diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 464ee235b..9ee218503 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -118,14 +118,10 @@ jobs: - name: Rust Cache uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - # Non-test code only. `--all-targets` would additionally compile the - # test modules, ~10 of which still use ungated `std::os::unix` - # (alias_repository, xtream_repository, bplustree/v3/publish, - # iptv/stalker/client, api_utils, api/model/download, setup_api). - # Widening this step means gating those first; the recording module - # tree is already `--all-targets`-clean on Windows. + # `--all-targets` also compiles the test modules, so Unix-only test + # helpers and imports must stay behind `#[cfg(unix)]`. - name: Check Windows build - run: cargo check --package tuliprox + run: cargo check --workspace --all-targets # Job 3: Build and Test (takes longer) test: diff --git a/CHANGELOG.md b/CHANGELOG.md index 72274a1fb..1b20b25cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -160,6 +160,13 @@ ## 🌟 New Features +- **Optional fast start for shared live HLS.** `reverse_proxy.hls_cache.startup.mode` selects `conservative` + (default, unchanged behavior), `first_ready` (admits playback after one processed start segment) or `progressive` + (streams the raw TS start segment while the shared origin fill is still running). Progressive replay memory, + reader lifetime and deferred repairs are bounded by the new `startup` limits; revision files share the existing + disk cache limits. Archive, encrypted and fMP4/MAP sessions keep the existing path. The Web UI exposes the mode + and its limits. See [Optional startup modes](docs/src/configuration/shared-hls-configuration.md#optional-startup-modes). + - **Personal Web UI table layouts.** Show or hide optional columns, reorder them with mouse, touch, or keyboard, and save layouts per account. Restore defaults, cancel drafts, and resolve conflicting saves without losing edits. Settings are stored under `/user_settings/`; installations without Web UI authentication share one layout. @@ -1075,6 +1082,14 @@ ## 🐛 Fixes +- **Shared HLS retries an origin account reserved for another session.** A bound account that is briefly reserved + by a competing session or an in-flight binding handoff is now retried within the existing acquire budget instead + of failing the segment fetch immediately. Ownership checks and rebinding rules are unchanged; a permanently + foreign reservation fails once that budget is spent. + +- **Shared HLS recovers from missing segments at the head of a new live window.** HTTP 404/410 responses for the + first segments of an unpublished live window advance the startup head instead of blocking the initial playlist. + - **Table pagination and settings errors.** Pagination clamps navigation and page ranges to valid bounds and rejects unsupported page sizes. Internal settings task failures return HTTP 500; oversized request bodies return HTTP 413, while other body read failures return HTTP 400. Account and token checks remain active after waiting for settings locks. diff --git a/Cargo.lock b/Cargo.lock index 40e3d4189..2a3fd0fc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4615,7 +4615,6 @@ version = "3.3.143" dependencies = [ "arc-swap", "blake3", - "bytes", "chrono", "chrono-tz", "fastrand", diff --git a/Makefile b/Makefile index a2afb109a..6935047c6 100644 --- a/Makefile +++ b/Makefile @@ -258,7 +258,7 @@ doctor-check: ## Verify the DVR doctor still matches the routes it reports on .PHONY: lint lint: ## Run clippy linter (Nightly) @echo "==> Running clippy (nightly)" - $(CARGO_NIGHTLY) clippy --workspace -- -D warnings + $(CARGO_NIGHTLY) clippy --workspace --all-targets -- -D warnings .PHONY: lint-fix lint-fix: ## Automatically fix clippy suggestions (Nightly) diff --git a/backend/app/src/api/api_utils/activation.rs b/backend/app/src/api/api_utils/activation.rs new file mode 100644 index 000000000..eedad6ec8 --- /dev/null +++ b/backend/app/src/api/api_utils/activation.rs @@ -0,0 +1,280 @@ +use super::{ + classify_playback_request, resolve_admission_with_strategies, AdmissionRequest, EvictionReentryGuard, + PlaybackRequestClass, PlaybackRequestFacts, +}; +use crate::{ + api::model::{AppState, PendingProviderReason}, + auth::Fingerprint, + model::{ConfigInput, ProxyUserCredentials}, +}; +use shared::{ + model::{PlaylistItemType, UserConnectionPermission, VirtualId}, + utils::current_time_secs, +}; +use std::{net::SocketAddr, sync::Arc}; + +pub(super) struct SessionActivationRequest<'a> { + pub(super) fingerprint: &'a Fingerprint, + pub(super) input: &'a ConfigInput, + pub(super) user: &'a ProxyUserCredentials, + pub(super) session_token: &'a str, + pub(super) request_class: Option, + pub(super) virtual_id: VirtualId, + pub(super) item_type: PlaylistItemType, + pub(super) stream_url: &'a str, + pub(super) connection_permission: UserConnectionPermission, + pub(super) connection_kind: crate::api::model::ConnectionKind, + pub(super) granted_grace_mode: Option, + pub(super) socket_bound: bool, +} + +pub(super) struct PlaybackActivationResult { + pub(super) admission: crate::api::model::ConnectionAdmission, + pub(super) grace_mode: Option, + pub(super) grace_context: Option, + pub(super) placeholder_transition_version: Option, +} + +/// # Panics +#[allow(clippy::too_many_lines)] +pub(super) async fn activate_session_before_stream_open( + app_state: &Arc, + request: SessionActivationRequest<'_>, +) -> PlaybackActivationResult { + let SessionActivationRequest { + fingerprint, + input, + user, + session_token, + request_class, + virtual_id, + item_type, + stream_url, + connection_permission, + connection_kind, + granted_grace_mode, + socket_bound, + } = request; + // Classify based on current session state, not the pre-computed value. + // If caller passes FollowUp, verify the session is still counted under the guard. + // A stale FollowUp would bypass admission — reclassify to catch this. + let (effective_request_class, loaded_session) = if let Some(request_class) = request_class { + if matches!(request_class, PlaybackRequestClass::FollowUp | PlaybackRequestClass::Activate) { + // Re-read session under the guard to ensure the counted lease is still held or acquired. + // If it is no longer counted, classify it from the current lifecycle so + // stale FollowUp requests cannot bypass admission. + // If it became counted, classify it so stale Activate requests don't double count. + let current_session = + app_state.active_users.get_and_update_user_session(&user.username, session_token).await; + let classified = classify_playback_request(PlaybackRequestFacts { + existing_session: current_session.as_ref(), + prepare_only: false, + terminate: false, + }); + (classified, Some(current_session)) + } else { + (request_class, None) + } + } else { + let existing_session = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; + let classified = classify_playback_request(PlaybackRequestFacts { + existing_session: existing_session.as_ref(), + prepare_only: false, + terminate: false, + }); + (classified, Some(existing_session)) + }; + let limits_enabled = app_state.app_config.config.load().user_access_control + && (user.max_connections > 0 || user.soft_connections > 0); + // Prepare: session setup without admission cost. The caller handles the actual activation. + // FollowUp: already counted, no re-admission needed. + // GracePeriod: grace already granted, no re-evaluation needed. + // No limits: skip admission entirely. + // GracePeriod permission is already resolved — skip admission strategies (re-run + // would evict the same session again). But we must still materialize the grace + // lifecycle (PendingProvider / GraceActive) so the session state is consistent. + if connection_permission == UserConnectionPermission::GracePeriod { + if loaded_session.as_ref().is_none_or(Option::is_none) { + app_state + .active_users + .ensure_user_session_placeholder(crate::api::model::CreateUserSessionParams { + user, + session_token, + virtual_id: virtual_id.get(), + provider: input.name.as_ref(), + stream_url, + addr: &fingerprint.addr, + connection_permission, + connection_kind: Some(connection_kind), + socket_bound, + }) + .await; + } + // Materialize grace lifecycle under the guard so the session state is consistent. + let current_session = match loaded_session { + Some(Some(session)) => Some(session), + _ => app_state.active_users.get_and_update_user_session(&user.username, session_token).await, + }; + let (_, resolved_grace) = match current_session.as_ref().map(|s| &s.lifecycle) { + Some(crate::api::model::PlaybackLifecycle::PendingProvider { .. }) => { + // Session already in PendingProvider — refresh deadline. + let deadline = current_time_secs().saturating_add(app_state.get_grace_options().timeout_secs); + let _ = app_state + .active_users + .mark_pending_provider(&user.username, session_token, PendingProviderReason::GraceHold, deadline) + .await; + ( + crate::api::model::PlaybackLifecycle::PendingProvider { + data: crate::api::model::PendingProviderState { + reason_code: PendingProviderReason::GraceHold, + created_at: current_time_secs(), + deadline, + version: current_session.as_ref().map_or(0, |s| { + if let crate::api::model::PlaybackLifecycle::PendingProvider { data } = &s.lifecycle { + data.version + } else { + 0 + } + }), + wake_source: None, + }, + }, + Some(crate::api::model::GraceMode::Hold), + ) + } + Some(crate::api::model::PlaybackLifecycle::GraceActive) => { + (crate::api::model::PlaybackLifecycle::GraceActive, Some(crate::api::model::GraceMode::Instant)) + } + _ => { + let hold_stream = granted_grace_mode.map_or_else( + || item_type.is_live() || item_type.is_live_adaptive(), + |mode| matches!(mode, crate::api::model::GraceMode::Hold), + ); + if hold_stream { + let deadline = current_time_secs().saturating_add(app_state.get_grace_options().timeout_secs); + let _ = app_state + .active_users + .mark_pending_provider( + &user.username, + session_token, + PendingProviderReason::GraceHold, + deadline, + ) + .await; + ( + crate::api::model::PlaybackLifecycle::PendingProvider { + data: crate::api::model::PendingProviderState { + reason_code: PendingProviderReason::GraceHold, + created_at: current_time_secs(), + deadline, + version: 1, + wake_source: None, + }, + }, + Some(crate::api::model::GraceMode::Hold), + ) + } else { + app_state.active_users.mark_grace_active(&user.username, session_token).await; + (crate::api::model::PlaybackLifecycle::GraceActive, Some(crate::api::model::GraceMode::Instant)) + } + } + }; + return PlaybackActivationResult { + admission: crate::api::model::ConnectionAdmission::from_permission( + connection_permission, + Some(connection_kind), + ), + grace_mode: resolved_grace, + grace_context: None, + placeholder_transition_version: None, + }; + } + // No limits: skip admission entirely. FollowUp / Prepare: no re-admission needed. + if !limits_enabled + || effective_request_class == PlaybackRequestClass::FollowUp + || effective_request_class == PlaybackRequestClass::Prepare + { + return PlaybackActivationResult { + admission: crate::api::model::ConnectionAdmission::from_permission( + connection_permission, + Some(connection_kind), + ), + grace_mode: None, + grace_context: None, + placeholder_transition_version: None, + }; + } + + let placeholder_transition_version = Some( + app_state + .active_users + .ensure_user_session_placeholder(crate::api::model::CreateUserSessionParams { + user, + session_token, + virtual_id: virtual_id.get(), + provider: input.name.as_ref(), + stream_url, + addr: &fingerprint.addr, + connection_permission, + connection_kind: Some(connection_kind), + socket_bound, + }) + .await, + ); + + let result = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + client_ip: &fingerprint.client_ip, + request_addr: &fingerprint.addr, + use_session_admission: true, + session_token: Some(session_token), + activate_unbound_session: true, + eviction_reentry_guard: if socket_bound { + EvictionReentryGuard::SocketPlayback { virtual_id } + } else { + EvictionReentryGuard::Session(session_token) + }, + }, + ) + .await; + let admission = result.admission; + let grace_mode = result.grace_mode; + let grace_context = result.grace_context; + + if admission.permission() == UserConnectionPermission::GracePeriod { + if matches!(grace_mode, Some(crate::api::model::GraceMode::Hold)) { + // Hold: session waits for provider slot. Does not count until provider is acquired. + let deadline = current_time_secs().saturating_add(app_state.get_grace_options().timeout_secs); + let _ = app_state + .active_users + .mark_pending_provider(&user.username, session_token, PendingProviderReason::GraceHold, deadline) + .await; + } else if matches!(grace_mode, Some(crate::api::model::GraceMode::Instant)) { + // Instant: session is provisionally active immediately. Counts against admission limits + // until the grace window resolves (success -> Active, failure -> Expired). + app_state.active_users.mark_grace_active(&user.username, session_token).await; + } + } + + PlaybackActivationResult { admission, grace_mode, grace_context, placeholder_transition_version } +} + +/// Session and account binding a forced resource request must still belong to after acquiring capacity. +#[derive(Clone, Copy)] +pub(super) struct CurrentSessionGuard<'a> { + pub(super) username: &'a str, + pub(super) token: &'a str, + pub(super) identity: tuliprox_session::SessionIdentity, +} + +pub(super) async fn cleanup_forced_reopen_addrs( + app_state: &Arc, + session_owner: &str, + cleanup_addrs: &[SocketAddr], +) { + app_state.active_provider.release_playback_connections_await(session_owner, cleanup_addrs).await; +} diff --git a/backend/app/src/api/api_utils/auth.rs b/backend/app/src/api/api_utils/auth.rs new file mode 100644 index 000000000..91baa29cc --- /dev/null +++ b/backend/app/src/api/api_utils/auth.rs @@ -0,0 +1,144 @@ +use crate::{ + api::model::{AppState, UserApiRequest}, + model::{AppConfig, ConfigInput, ConfigTarget, ProxyUserCredentials}, +}; +use shared::model::ProxyType; +use std::sync::Arc; + +pub fn get_user_target_by_username( + username: &str, + app_state: &Arc, +) -> Option<(Arc, Arc)> { + if !username.is_empty() { + return app_state.app_config.get_target_for_username(username); + } + None +} + +pub fn get_user_target_by_credentials<'a>( + username: &str, + password: &str, + api_req: &'a UserApiRequest, + app_state: &'a AppState, +) -> Option<(Arc, Arc)> { + if !username.is_empty() && !password.is_empty() { + app_state.app_config.get_target_for_user(username, password) + } else { + let token = api_req.token.as_str().trim(); + if token.is_empty() { + None + } else { + app_state.app_config.get_target_for_user_by_token(token) + } + } +} + +pub fn get_user_target<'a>( + api_req: &'a UserApiRequest, + app_state: &'a AppState, +) -> Option<(Arc, Arc)> { + let username = api_req.username.as_str().trim(); + let password = api_req.password.as_str().trim(); + get_user_target_by_credentials(username, password, api_req, app_state) +} + +pub fn get_username_from_auth_header(token: &str, app_state: &Arc) -> Option { + let config = app_state.app_config.config.load(); + let web_auth_config = config.web_ui.as_ref()?.auth.as_ref()?; + // This hand-rolled its own `decode` with a bare `Validation::new`, which + // checks `exp` and nothing else - no issuer. + crate::auth::verify_token(token, web_auth_config.secret.as_bytes(), &web_auth_config.issuer) + .map(|token_data| token_data.claims.username) +} + +pub fn create_api_proxy_user(app_state: &Arc) -> ProxyUserCredentials { + let config = app_state.app_config.config.load(); + + let server = config + .web_ui + .as_ref() + .and_then(|web_ui| web_ui.player_server.as_ref()) + .map_or("default", |server_name| server_name.as_str()); + + ProxyUserCredentials { + username: "api_user".to_string(), + password: "api_user".to_string(), + token: None, + proxy: ProxyType::Reverse(None), + server: Some(server.to_string()), + epg_timeshift: None, + epg_request_timeshift: None, + created_at: None, + exp_date: None, + max_connections: 0, + status: None, + output_clusters: shared::model::ClusterFlags::all(), + ui_enabled: false, + comment: None, + priority: 0, + soft_connections: 0, + soft_priority: 0, + t_is_api_user: true, + network_access: None, + plan: None, + filter: None, + raw_output_clusters: None, + raw_max_connections: 0, + raw_soft_connections: 0, + raw_proxy: Some(ProxyType::Reverse(None)), + t_filter: None, + t_has_unresolved_plan: false, + t_has_invalid_filter: false, + } +} + +/// The user internal recording playback runs as. Its server resolves to the +/// local API listener (`AppConfig::get_user_server_info`), and its HLS URLs +/// carry this identity to every follow-up request. +pub fn create_recording_proxy_user(app_state: &Arc) -> ProxyUserCredentials { + use base64::Engine; + use sha2::Digest; + let mut user = create_api_proxy_user(app_state); + user.username = crate::model::RECORDING_PROXY_USERNAME.to_string(); + let digest = sha2::Sha256::new() + .chain_update(b"tuliprox/internal-recording-user") + .chain_update(app_state.app_config.access_token_secret) + .finalize(); + user.password = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(&digest[..]); + user.server = None; + user +} + +/// Recording headers configure the upstream capture. They are layered under +/// the input's own headers, which keep precedence, and above the worker's +/// request headers. The worker's `User-Agent` stays on the local request so +/// the stream remains identifiable as a recording. +pub(crate) fn with_recording_headers(app_config: &AppConfig, input: Arc) -> Arc { + let config = app_config.config.load(); + let Some(recording) = config.recording() else { return input }; + let recording_headers = recording.origin_headers(&config); + if recording_headers.keys().all(|name| input.headers.keys().any(|key| key.eq_ignore_ascii_case(name.as_str()))) { + return input; + } + let mut merged = input.as_ref().clone(); + for (name, value) in recording_headers { + let Ok(value) = value.to_str() else { continue }; + if !merged.headers.keys().any(|key| key.eq_ignore_ascii_case(name.as_str())) { + merged.headers.insert(name.as_str().to_string(), value.to_string()); + } + } + Arc::new(merged) +} + +/// The input as seen by `user`: recording playback adds the recording headers. +pub(crate) fn input_for_user( + app_config: &AppConfig, + user: &ProxyUserCredentials, + input: Arc, +) -> Arc { + if user.is_recording_proxy_user() { + with_recording_headers(app_config, input) + } else { + input + } +} diff --git a/backend/app/src/api/api_utils/catchup.rs b/backend/app/src/api/api_utils/catchup.rs new file mode 100644 index 000000000..842fffaeb --- /dev/null +++ b/backend/app/src/api/api_utils/catchup.rs @@ -0,0 +1,282 @@ +use super::{get_stream_config_u64, mark_response_as_uncompressed, try_unwrap_body}; +use crate::{ + api::{ + endpoints::hls_api::{hls_media_playlist_wrap_enabled, wrap_media_playlist, HlsMediaPlaylistWrap}, + model::{AppState, BoxedProviderStream, StreamDetails, StreamError}, + }, + auth::Fingerprint, + model::{ConfigInput, ConfigTarget, PlaybackKind, ProxyUserCredentials}, + processing::parser::hls::{classify_hls_playlist, rewrite_hls, HlsPlaylistKind, RewriteHlsProps}, +}; +use axum::{ + body::Body, + http::{header, StatusCode}, + response::IntoResponse, +}; +use bytes::{Bytes, BytesMut}; +use futures::{stream, StreamExt}; +use shared::{ + concat_string, + defaults::default_catchup_session_ttl_secs, + model::{UserConnectionPermission, VirtualId}, +}; +use std::{sync::Arc, time::Duration}; +use tuliprox_hls::api::MAX_HLS_MANIFEST_BYTES; +use url::Url; + +pub(super) enum CatchupPayload { + Direct(BoxedProviderStream), + HlsManifest(Bytes), +} + +pub(super) struct DetectedCatchupHlsResponseParams<'a> { + pub(super) app_state: &'a Arc, + pub(super) stream_details: StreamDetails, + pub(super) manifest: Bytes, + pub(super) user: &'a ProxyUserCredentials, + pub(super) target: &'a ConfigTarget, + pub(super) input: &'a ConfigInput, + pub(super) fingerprint: &'a Fingerprint, + pub(super) session_token: &'a str, + pub(super) virtual_id: VirtualId, + pub(super) connection_permission: UserConnectionPermission, + pub(super) connection_kind: crate::api::model::ConnectionKind, + pub(super) fallback_stream_url: &'a str, +} + +#[allow(clippy::too_many_lines)] +pub(super) async fn detected_catchup_hls_response( + params: DetectedCatchupHlsResponseParams<'_>, +) -> axum::response::Response { + let DetectedCatchupHlsResponseParams { + app_state, + mut stream_details, + manifest, + user, + target, + input, + fingerprint, + session_token, + virtual_id, + connection_permission, + connection_kind, + fallback_stream_url, + } = params; + + let Some(provider) = stream_details.provider_name.clone() else { + cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token).await; + return StatusCode::BAD_GATEWAY.into_response(); + }; + let Some(server_info) = app_state.app_config.get_user_server_info(user) else { + cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token).await; + return StatusCode::BAD_GATEWAY.into_response(); + }; + let Ok(content) = std::str::from_utf8(&manifest) else { + cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token).await; + return StatusCode::BAD_GATEWAY.into_response(); + }; + let playlist_kind = classify_hls_playlist(content); + if playlist_kind == HlsPlaylistKind::Invalid { + cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token).await; + return StatusCode::BAD_GATEWAY.into_response(); + } + + let response_url = stream_details + .stream_info + .as_ref() + .and_then(|(_, _, response_url, _)| response_url.as_ref()) + .map_or_else(|| fallback_stream_url.to_string(), ToString::to_string); + let base_url = server_info.get_base_url(); + let encrypt_secret = app_state.get_encrypt_secret(); + let rewritten = rewrite_hls( + user, + &RewriteHlsProps { + secret: &encrypt_secret, + base_url: &base_url, + content, + hls_url: response_url, + target_id: target.id, + virtual_id: virtual_id.get(), + input_id: input.id, + user_token: Some(session_token), + origin_provider: Some(&provider), + playlist_kind: Some(playlist_kind), + }, + ); + + let request_url = stream_details.request_url.as_deref().unwrap_or(fallback_stream_url); + let created_session_token = app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user, + session_token, + virtual_id: virtual_id.get(), + provider: &provider, + stream_url: request_url, + addr: &fingerprint.addr, + connection_permission, + connection_kind: Some(connection_kind), + socket_bound: false, + }) + .await; + if let Some(stream_index) = stream_details.user_agent_stream_index { + app_state + .active_users + .set_user_agent_stream_index_if_absent(&user.username, &created_session_token, stream_index) + .await; + } + if !stream_details.provider_session_headers.is_empty() { + app_state + .active_users + .update_session_provider_response_headers_from( + &user.username, + &created_session_token, + &stream_details.provider_session_headers, + stream_details + .stream_info + .as_ref() + .and_then(|(_, _, url, _)| url.as_ref()) + .map_or(request_url, Url::as_str), + ) + .await; + } + app_state.active_provider.refresh_adaptive_playback_lease( + &provider, + &created_session_token, + PlaybackKind::Catchup, + get_catchup_session_ttl_secs(app_state), + ); + let binding_tag = stream_details.provider_handle.as_ref().and_then(|handle| handle.handle()?.binding_tag); + app_state.connection_manager.release_managed_provider_handle(stream_details.provider_handle.take()); + app_state + .active_users + .release_unbound_session_reservation(&user.username, &created_session_token, None, false) + .await; + app_state.active_users.clear_unbound_session_addr(&user.username, &created_session_token, &fingerprint.addr).await; + + if playlist_kind == HlsPlaylistKind::Media && hls_media_playlist_wrap_enabled(app_state, target, user) { + let master = wrap_media_playlist( + HlsMediaPlaylistWrap { + app_state, + user, + base_url: &base_url, + target_id: target.id, + input, + virtual_id: virtual_id.get(), + session_token: &created_session_token, + // The canonical catch-up URL, not the provider-resolved one, so a refresh can + // resolve it to whichever account is selected then. + sealed_url: fallback_stream_url, + provider: &provider, + binding_tag, + known_bitrate_bps: None, + stream_ref: None, + }, + rewritten, + ) + .await; + return catchup_hls_manifest_response(master); + } + catchup_hls_manifest_response(rewritten) +} + +pub(super) fn catchup_hls_manifest_response(content: String) -> axum::response::Response { + let mut response = try_unwrap_body!(axum::response::Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, crate::api::static_headers::CT_M3U.clone()) + .header(header::CACHE_CONTROL, crate::api::static_headers::CC_NO_STORE.clone()) + .body(Body::from(content))); + mark_response_as_uncompressed(&mut response); + response +} + +pub(super) async fn cleanup_failed_detected_catchup_hls( + app_state: &Arc, + stream_details: &mut StreamDetails, + username: &str, + session_token: &str, +) { + app_state.connection_manager.release_managed_provider_handle(stream_details.provider_handle.take()); + app_state.active_users.terminate_session(username, session_token).await; + app_state.active_provider.clear_provider_reservation(session_token); +} + +pub(super) async fn probe_catchup_payload( + stream: BoxedProviderStream, + deadline: Duration, +) -> Result { + tokio::time::timeout(deadline, probe_catchup_payload_inner(stream)) + .await + .map_err(|_| StreamError::Stream("catch-up payload probe timed out".to_string()))? +} + +pub(super) async fn probe_catchup_payload_inner( + mut stream: BoxedProviderStream, +) -> Result { + const HLS_SIGNATURE: &[u8] = b"#EXTM3U"; + + let mut prefix = BytesMut::new(); + while prefix.len() < HLS_SIGNATURE.len() { + let Some(chunk) = stream.next().await else { + return Ok(CatchupPayload::Direct(stream::once(async move { Ok(prefix.freeze()) }).chain(stream).boxed())); + }; + prefix.extend_from_slice(&chunk?); + } + + if !prefix.starts_with(HLS_SIGNATURE) { + return Ok(CatchupPayload::Direct(stream::once(async move { Ok(prefix.freeze()) }).chain(stream).boxed())); + } + if prefix.len() > MAX_HLS_MANIFEST_BYTES { + return Err(StreamError::Stream("catch-up HLS manifest exceeds size limit".to_string())); + } + + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + if prefix.len().saturating_add(chunk.len()) > MAX_HLS_MANIFEST_BYTES { + return Err(StreamError::Stream("catch-up HLS manifest exceeds size limit".to_string())); + } + prefix.extend_from_slice(&chunk); + } + + Ok(CatchupPayload::HlsManifest(prefix.freeze())) +} + +pub(crate) fn get_catchup_session_ttl_secs(app_state: &Arc) -> u64 { + get_stream_config_u64(app_state, |stream| stream.catchup_session_ttl_secs, default_catchup_session_ttl_secs()) +} + +pub fn create_catchup_session_key( + fingerprint: &Fingerprint, + username: &str, + virtual_id: u32, + archive_discriminator: &str, +) -> String { + concat_string!( + "catchup|", + &fingerprint.key, + "|", + username, + "|", + &virtual_id.to_string(), + "|", + archive_discriminator.trim_matches('/') + ) +} + +pub fn create_m3u_catchup_session_key( + fingerprint: &Fingerprint, + username: &str, + virtual_id: u32, + archive_discriminator: &str, +) -> String { + concat_string!( + "m3u-catchup|", + &fingerprint.key, + "|", + username, + "|", + &virtual_id.to_string(), + "|", + archive_discriminator + ) +} diff --git a/backend/app/src/api/api_utils/clock.rs b/backend/app/src/api/api_utils/clock.rs new file mode 100644 index 000000000..49c46ae12 --- /dev/null +++ b/backend/app/src/api/api_utils/clock.rs @@ -0,0 +1,20 @@ +use super::PROCESS_START; +use crate::BUILD_TIMESTAMP; +use chrono::{DateTime, Utc}; + +pub fn get_server_time() -> String { + chrono::offset::Local::now().with_timezone(&chrono::Local).format("%Y-%m-%d %H:%M:%S %Z").to_string() +} + +/// Anchors the uptime clock; call once at process startup. +pub fn init_uptime_clock() { let _ = *PROCESS_START; } + +pub fn get_uptime_secs() -> u64 { PROCESS_START.elapsed().as_secs() } + +pub fn get_build_time() -> Option { + BUILD_TIMESTAMP + .to_string() + .parse::>() + .ok() + .map(|datetime| datetime.format("%Y-%m-%d %H:%M:%S %Z").to_string()) +} diff --git a/backend/app/src/api/api_utils/local_files.rs b/backend/app/src/api/api_utils/local_files.rs new file mode 100644 index 000000000..e0b9222cf --- /dev/null +++ b/backend/app/src/api/api_utils/local_files.rs @@ -0,0 +1,412 @@ +use super::{ + activate_session_before_stream_open, get_stream_throttle, is_socket_bound_playback_session, is_throttled_stream, + mark_response_as_uncompressed, prepare_stream_metering, record_connect_failed_attempt, reentry_suppressed_response, + stream_admission_rejected_response, try_unwrap_body, ConnectFailedAttempt, PlaybackRequestClass, + SessionActivationRequest, +}; +use crate::{ + api::{ + model::{ + create_active_client_stream, create_custom_video_stream_response, AppState, CustomVideoStreamType, + StreamDetails, StreamError, ThrottledStream, + }, + static_headers::CT_OCTET, + }, + auth::Fingerprint, + model::{ConfigInput, ConfigTarget, ProxyUserCredentials}, + utils::{ + async_file_reader, get_file_extension, + request::{content_type_from_ext, parse_range}, + }, +}; +use axum::{ + body::Body, + http::{header, HeaderMap, HeaderValue, Response, StatusCode}, + response::IntoResponse, +}; +use chrono::{DateTime, Utc}; +use futures::{StreamExt, TryStreamExt}; +use log::{error, info, log_enabled, trace}; +use shared::{ + model::{ConnectFailureReason, FailureStage, StreamChannel, UserConnectionPermission, VirtualId}, + utils::{extract_extension_from_url, human_readable_kbps, sanitize_sensitive_info}, +}; +use std::{ + io::SeekFrom, + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use tokio_util::io::ReaderStream; + +#[allow(clippy::missing_panics_doc)] +pub async fn serve_file(file_path: &Path, mime_type: String, cache_control: Option<&str>) -> impl IntoResponse + Send { + match tokio::fs::try_exists(file_path).await { + Ok(exists) => { + if !exists { + return StatusCode::NOT_FOUND.into_response(); + } + } + Err(err) => { + error!("Failed to open file {}, {err:?}", file_path.display()); + return StatusCode::NOT_FOUND.into_response(); + } + } + + match tokio::fs::File::open(file_path).await { + Ok(file) => { + let last_modified = file.metadata().await.ok().and_then(|m| m.modified().ok()).map(|m| { + let dt: DateTime = m.into(); + dt.format("%a, %d %b %Y %H:%M:%S GMT").to_string() + }); + + let reader = async_file_reader(file); + let stream = ReaderStream::new(reader); + let body = Body::from_stream(stream); + + let mut builder = axum::response::Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, mime_type) + .header(header::CACHE_CONTROL, cache_control.unwrap_or("no-cache")); + + if let Some(lm) = last_modified { + builder = builder.header(header::LAST_MODIFIED, lm); + } + + try_unwrap_body!(builder.body(body)) + } + Err(_) => internal_server_error!(), + } +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +pub(crate) async fn local_stream_response( + fingerprint: &Fingerprint, + app_state: &Arc, + pli: StreamChannel, + req_headers: &HeaderMap, + input: &ConfigInput, + _target: &ConfigTarget, + user: &ProxyUserCredentials, + connection_permission: UserConnectionPermission, + connection_kind: crate::api::model::ConnectionKind, + playback_session_token: Option<&str>, + request_class: Option, + check_path: bool, +) -> impl IntoResponse + Send { + let _transition_guard = if let Some(session_token) = playback_session_token { + Some(app_state.active_users.acquire_playback_transition(&user.username, session_token).await) + } else { + None + }; + if log_enabled!(log::Level::Trace) { + trace!("Try to open stream {}", sanitize_sensitive_info(&pli.url)); + } + + let mut connection_permission = connection_permission; + let mut grace_mode = None; + if connection_permission == UserConnectionPermission::Exhausted { + let allow_session_reopen = if let Some(session_token) = playback_session_token { + user.max_connections > 0 + && app_state + .active_users + .connection_permission_for_session( + &user.username, + user.max_connections, + user.soft_connections, + session_token, + ) + .await + != UserConnectionPermission::Exhausted + } else { + false + }; + if !allow_session_reopen { + record_connect_failed_attempt(ConnectFailedAttempt { + app_state, + fingerprint, + user, + stream_channel: pli.clone(), + provider_name: input.name.clone(), + req_headers, + reason: ConnectFailureReason::UserConnectionsExhausted, + failure_stage: FailureStage::Admission, + }); + return create_custom_video_stream_response( + &app_state.provider_stream_ctx(), + &fingerprint.addr, + CustomVideoStreamType::UserConnectionsExhausted, + ) + .into_response(); + } + connection_permission = UserConnectionPermission::Allowed; + } + + let path = PathBuf::from(pli.url.strip_prefix("file://").unwrap_or(&pli.url)); + + let Ok(mut file) = tokio::fs::File::open(&path).await else { return StatusCode::NOT_FOUND.into_response() }; + let Ok(opened_metadata) = file.metadata().await else { return internal_server_error!() }; + + // Canonicalize and validate the path + let canonical = match tokio::fs::canonicalize(&path).await { + Ok(canonical) => canonical, + Err(err) => { + error!("Local file path is corrupt {}: {err}", path.display()); + return StatusCode::NOT_FOUND.into_response(); + } + }; + + if check_path { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let Ok(canonical_metadata) = tokio::fs::metadata(&canonical).await else { return internal_server_error!() }; + if opened_metadata.dev() != canonical_metadata.dev() || opened_metadata.ino() != canonical_metadata.ino() { + error!("TOCTOU race detected: file swapped during local_stream_response"); + return StatusCode::FORBIDDEN.into_response(); + } + } + #[cfg(windows)] + match same_windows_file_identity(&file, &canonical).await { + Ok(true) => {} + Ok(false) => { + error!("TOCTOU race detected: file swapped during local_stream_response"); + return StatusCode::FORBIDDEN.into_response(); + } + Err(err) => { + error!("Could not verify local file identity {}: {err}", canonical.display()); + return internal_server_error!(); + } + } + #[cfg(not(any(unix, windows)))] + { + error!("Secure local file identity validation is unsupported on this platform"); + return StatusCode::FORBIDDEN.into_response(); + } + + let Some(library_paths) = app_state + .app_config + .config + .load() + .library + .as_ref() + .map(|lib| lib.scan_directories.iter().map(|dir| dir.path.clone()).collect::>()) + else { + return StatusCode::NOT_FOUND.into_response(); + }; + + // Verify path is within allowed media directories + // (requires configuration of allowed base paths) + if !is_path_within_allowed_directories(&canonical, &library_paths) { + return StatusCode::FORBIDDEN.into_response(); + } + } + + let file_size = opened_metadata.len(); + + let range = req_headers.get("range").and_then(|v| v.to_str().ok()).and_then(parse_range); + + let (start, end) = if let Some((req_start, req_end)) = range { + if file_size == 0 || req_start >= file_size { + return StatusCode::RANGE_NOT_SATISFIABLE.into_response(); + } + let end = req_end.unwrap_or(file_size - 1).min(file_size - 1); + if end < req_start { + return StatusCode::RANGE_NOT_SATISFIABLE.into_response(); + } + (req_start, end) + } else { + if file_size == 0 { + // Serve empty file + let body = axum::body::Body::empty(); + let mut response = Response::new(body); + *response.status_mut() = StatusCode::OK; + let headers = response.headers_mut(); + if let Some(ext) = get_file_extension(&pli.url) { + let ct = content_type_from_ext(&ext); + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(ct)); + } else { + headers.insert(header::CONTENT_TYPE, CT_OCTET.clone()); //HeaderValue::from_static("application/octet-stream")); + } + headers.insert("Accept-Ranges", HeaderValue::from_static("bytes")); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from_static("0")); + return response.into_response(); + } + (0, file_size - 1) + }; + + let content_length = end - start + 1; + + if start > 0 { + if let Err(_err) = file.seek(SeekFrom::Start(start)).await { + return internal_server_error!(); + } + } + + let stream = + ReaderStream::new(file.take(content_length)).map_err(|err| StreamError::Stream(err.to_string())).boxed(); + let throttle_kbps = usize::try_from(get_stream_throttle(app_state)).unwrap_or_default(); + let stream = if is_throttled_stream(pli.item_type, throttle_kbps) { + info!("Stream throttling active: {}", human_readable_kbps(u64::try_from(throttle_kbps).unwrap_or_default())); + ThrottledStream::new(stream, throttle_kbps).boxed() + } else { + stream + }; + let socket_bound = is_socket_bound_playback_session(pli.item_type, extract_extension_from_url(&pli.url)); + let mut connection_kind = connection_kind; + if let Some(session_token) = playback_session_token { + let activation = activate_session_before_stream_open( + app_state, + SessionActivationRequest { + fingerprint, + input, + user, + session_token, + request_class, + virtual_id: VirtualId::new(pli.virtual_id), + item_type: pli.item_type, + stream_url: &pli.url, + connection_permission, + connection_kind, + granted_grace_mode: None, + socket_bound, + }, + ) + .await; + grace_mode = activation.grace_mode; + connection_permission = activation.admission.permission(); + connection_kind = activation.admission.kind().unwrap_or(connection_kind); + + if connection_permission == UserConnectionPermission::Exhausted { + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + activation.placeholder_transition_version, + activation.placeholder_transition_version.is_some(), + ) + .await; + if activation.admission.is_reentry_suppressed() { + return reentry_suppressed_response(); + } + return create_custom_video_stream_response( + &app_state.provider_stream_ctx(), + &fingerprint.addr, + CustomVideoStreamType::UserConnectionsExhausted, + ) + .into_response(); + } + } + let mut grace_period_options = app_state.get_grace_options(); + if connection_permission != UserConnectionPermission::GracePeriod { + grace_period_options.period_millis = 0; + } + if let Some(resolved_mode) = grace_mode { + grace_period_options.hold_stream = matches!(resolved_mode, crate::api::model::GraceMode::Hold); + } + let resolved_connection_kind = if let Some(session_token) = playback_session_token { + app_state + .active_users + .get_and_update_user_session(&user.username, session_token) + .await + .and_then(|session| session.connection_kind) + .unwrap_or(connection_kind) + } else { + connection_kind + }; + if let Some(session_token) = playback_session_token { + let _ = app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user, + session_token, + virtual_id: pli.virtual_id, + provider: input.name.as_ref(), + stream_url: &pli.url, + addr: &fingerprint.addr, + connection_permission, + connection_kind: Some(resolved_connection_kind), + socket_bound, + }) + .await; + } + let metering = prepare_stream_metering(app_state, &pli.url, false, true, false); + let stream = match create_active_client_stream(crate::api::model::ActiveClientStreamParams { + stream_details: StreamDetails::from_stream(stream, grace_period_options), + app_state, + user, + connection_permission, + connection_kind: resolved_connection_kind, + fingerprint, + stream_channel: pli.clone(), + socket_bound, + session_token: playback_session_token, + req_headers, + meter_uid: metering.meter_uid, + meter_stream: metering.meter_stream, + }) + .await + { + Ok(stream) => stream, + Err(error) => return stream_admission_rejected_response(error, &user.username), + }; + + let mut response = Response::new(axum::body::Body::from_stream(stream)); + + *response.status_mut() = if range.is_some() { StatusCode::PARTIAL_CONTENT } else { StatusCode::OK }; + + let headers = response.headers_mut(); + if let Some(ext) = get_file_extension(&pli.url) { + let ct = content_type_from_ext(&ext); + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(ct)); + } else { + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("application/octet-stream")); + } + headers.insert("Accept-Ranges", HeaderValue::from_static("bytes")); + if let Ok(header_value) = HeaderValue::from_str(&content_length.to_string()) { + headers.insert(header::CONTENT_LENGTH, header_value); + } + + if range.is_some() { + if let Ok(header_value) = HeaderValue::from_str(&format!("bytes {start}-{end}/{file_size}")) { + headers.insert(header::CONTENT_RANGE, header_value); + } + } + + mark_response_as_uncompressed(&mut response); + response +} + +pub(super) fn is_path_within_allowed_directories(sub_path: &Path, root_paths: &[String]) -> bool { + for root_path in root_paths { + if sub_path.starts_with(PathBuf::from(root_path)) { + return true; + } + } + false +} + +#[cfg(windows)] +pub(super) async fn same_windows_file_identity( + opened_file: &tokio::fs::File, + canonical_path: &Path, +) -> std::io::Result { + let canonical_file = tokio::fs::File::open(canonical_path).await?; + Ok(windows_file_identity(opened_file)? == windows_file_identity(&canonical_file)?) +} + +#[cfg(windows)] +pub(super) fn windows_file_identity(file: &tokio::fs::File) -> std::io::Result<(u32, u32, u32)> { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION}; + + let mut info = unsafe { std::mem::zeroed::() }; + // SAFETY: `file.as_raw_handle()` is a live file handle for the duration of + // the call, and `info` is a writable output buffer for the WinAPI function. + let ok = unsafe { GetFileInformationByHandle(file.as_raw_handle().cast(), &raw mut info) }; + if ok == 0 { + return Err(std::io::Error::last_os_error()); + } + Ok((info.dwVolumeSerialNumber, info.nFileIndexHigh, info.nFileIndexLow)) +} diff --git a/backend/app/src/api/api_utils/media_server.rs b/backend/app/src/api/api_utils/media_server.rs new file mode 100644 index 000000000..13374bdb7 --- /dev/null +++ b/backend/app/src/api/api_utils/media_server.rs @@ -0,0 +1,163 @@ +use super::is_hop_by_hop_response_header; +use crate::{ + api::model::{AppState, BoxedProviderStream, ProviderStreamInfo, StreamError}, + media_server::{ + playback::{ + media_server_image_response as open_media_server_proxy_image_response, + media_server_stream_response as open_media_server_proxy_stream_response, parse_media_server_image_ref, + parse_media_server_stream_ref, + }, + MediaServerError, MediaServerErrorKind, MediaServerHttpClient, MediaServerImageRef, + }, + model::ConfigInput, +}; +use axum::{ + body::Body, + http::{header, HeaderMap, Response, StatusCode}, +}; +use futures::{StreamExt, TryStreamExt}; +use shared::model::InputType; +use std::sync::Arc; +use url::Url; + +pub(super) fn is_media_server_playback_url(input: &ConfigInput, stream_url: &str) -> bool { + input.input_type == InputType::Plex || is_media_server_stream_ref_url(stream_url) +} + +pub(super) fn is_media_server_stream_ref_url(stream_url: &str) -> bool { + Url::parse(stream_url).is_ok_and(|url| url.scheme() == "media-server") +} + +pub(super) async fn open_media_server_stream_for_input( + app_state: &Arc, + input: &ConfigInput, + stream_url: &str, + req_headers: &HeaderMap, +) -> Result<(BoxedProviderStream, ProviderStreamInfo), MediaServerError> { + let stream_ref = parse_media_server_stream_ref(&input.name, stream_url)?; + let range = req_headers.get(header::RANGE).and_then(|value| value.to_str().ok()); + let http_client = MediaServerHttpClient::new(app_state.http_clients.default.load().as_ref().clone()); + + let response = match input.input_type { + InputType::Plex => { + let client = input.plex_catalog_client(http_client)?; + open_media_server_proxy_stream_response(&client, &stream_ref, range).await? + } + InputType::Emby | InputType::Jellyfin => { + return Err(MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) + .provider("media-server") + .detail("media-server playback proxy is not implemented for this input type")); + } + InputType::M3u + | InputType::Xtream + | InputType::M3uBatch + | InputType::XtreamBatch + | InputType::Stalker + | InputType::StalkerBatch + | InputType::Library + | InputType::Staged => { + return Err(MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) + .provider("media-server") + .detail("playlist item is not backed by a media-server input")); + } + }; + + let headers = response + .headers + .iter() + .filter(|(key, _)| !is_hop_by_hop_response_header(key)) + .filter_map(|(key, value)| value.to_str().ok().map(|value| (key.to_string(), value.to_string()))) + .collect::>(); + let status = response.status; + let stream = response.body.map_err(|err| StreamError::Stream(err.to_string())).boxed(); + Ok((stream, Some((headers, status, None, None)))) +} + +pub(super) async fn open_media_server_image_resource( + app_state: &Arc, + resource_url: &str, +) -> Result, MediaServerError> { + let image_ref = parse_media_server_image_ref(resource_url)?; + let input_name = media_server_image_input_name(&image_ref); + let input = app_state.app_config.get_input_by_name(input_name).ok_or_else(|| { + MediaServerError::new(MediaServerErrorKind::MediaServerItemNotFound) + .provider("media-server") + .detail("media-server image input was not found") + })?; + let http_client = MediaServerHttpClient::new(app_state.http_clients.default.load().as_ref().clone()); + + let response = match input.input_type { + InputType::Plex => { + let client = input.plex_catalog_client(http_client)?; + open_media_server_proxy_image_response(&client, &image_ref).await? + } + InputType::Emby | InputType::Jellyfin => { + return Err(MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) + .provider("media-server") + .detail("media-server image proxy is not implemented for this input type")); + } + InputType::M3u + | InputType::Xtream + | InputType::M3uBatch + | InputType::XtreamBatch + | InputType::Stalker + | InputType::StalkerBatch + | InputType::Library + | InputType::Staged => { + return Err(MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) + .provider("media-server") + .detail("media-server image input is not backed by a media-server input")); + } + }; + + let mut builder = Response::builder().status(response.status); + for (key, value) in &response.headers { + if !is_hop_by_hop_response_header(key) { + builder = builder.header(key, value); + } + } + let body = response.body.map_err(|err| StreamError::Stream(err.to_string())); + builder.body(Body::from_stream(body)).map_err(|err| { + MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) + .provider("media-server") + .detail(format!("media-server image response build failed: {err}")) + }) +} + +pub(super) fn media_server_image_error_status(err: &MediaServerError) -> StatusCode { + match err.kind { + MediaServerErrorKind::MediaServerItemNotFound | MediaServerErrorKind::NoDirectPlayableMediaServerSource => { + StatusCode::NOT_FOUND + } + MediaServerErrorKind::MediaServerStreamOpenFailed if is_media_server_image_validation_error(err) => { + StatusCode::BAD_REQUEST + } + MediaServerErrorKind::MediaServerStreamOpenFailed + | MediaServerErrorKind::MediaServerAuthDenied + | MediaServerErrorKind::MediaServerUnavailable + | MediaServerErrorKind::MediaServerLibraryUnavailable + | MediaServerErrorKind::MediaServerLibraryTypeUnsupported + | MediaServerErrorKind::MediaServerCatalogDecodeFailed + | MediaServerErrorKind::MediaServerCatalogPageStalled + | MediaServerErrorKind::MediaServerCatalogIncomplete + | MediaServerErrorKind::MediaServerRateLimited + | MediaServerErrorKind::MediaServerDiscoveryFailed => StatusCode::BAD_GATEWAY, + } +} + +pub(super) fn is_media_server_image_validation_error(err: &MediaServerError) -> bool { + err.detail_text().is_some_and(|detail| { + detail.contains("resource URL is not a media server image URL") + || detail.contains("media server image URL is missing required path parts") + || detail.contains("unsupported media server image URL scheme") + || detail.contains("media-server image input is not backed by a media-server input") + }) +} + +pub(super) fn media_server_image_input_name(image_ref: &MediaServerImageRef) -> &Arc { + match image_ref { + MediaServerImageRef::Emby { input_name, .. } + | MediaServerImageRef::Jellyfin { input_name, .. } + | MediaServerImageRef::Plex { input_name, .. } => input_name, + } +} diff --git a/backend/app/src/api/api_utils/metering.rs b/backend/app/src/api/api_utils/metering.rs new file mode 100644 index 000000000..488578176 --- /dev/null +++ b/backend/app/src/api/api_utils/metering.rs @@ -0,0 +1,101 @@ +use crate::api::model::AppState; +use shared::model::PlaylistItemType; +use std::sync::Arc; + +#[derive(Default)] +pub(super) struct StreamMeteringConfig { + pub(super) meter_uid: u32, + pub(super) meter_stream: bool, + pub(super) pending_shared_registration: Option, +} + +impl StreamMeteringConfig { + pub(super) fn commit_shared_registration(&mut self) { + if let Some(registration) = self.pending_shared_registration.take() { + registration.commit(); + } + } +} + +pub(super) fn is_throttled_stream(item_type: PlaylistItemType, throttle_kbps: usize) -> bool { + throttle_kbps > 0 + && matches!( + item_type, + PlaylistItemType::Video + | PlaylistItemType::Series + | PlaylistItemType::SeriesInfo + | PlaylistItemType::Catchup + | PlaylistItemType::LocalVideo + | PlaylistItemType::LocalSeries + | PlaylistItemType::LocalSeriesInfo + ) +} + +pub(super) fn get_stream_throttle(app_state: &Arc) -> u64 { + app_state + .app_config + .config + .load() + .reverse_proxy + .as_ref() + .and_then(|reverse_proxy| reverse_proxy.stream.as_ref()) + .map(|stream| stream.throttle_kbps) + .unwrap_or_default() +} + +pub(super) fn is_stream_metrics_enabled(app_state: &Arc) -> bool { + app_state + .app_config + .config + .load() + .reverse_proxy + .as_ref() + .and_then(|reverse_proxy| reverse_proxy.stream.as_ref()) + .is_some_and(|stream| stream.metrics_enabled) +} + +pub(super) fn prepare_stream_metering( + app_state: &Arc, + stream_url: &str, + share_stream: bool, + has_stream: bool, + has_deferred_provider_open: bool, +) -> StreamMeteringConfig { + if !is_stream_metrics_enabled(app_state) { + return StreamMeteringConfig::default(); + } + + if share_stream { + let (meter_uid, pending_registration) = app_state + .shared_stream_manager + .reserve_meter_uid(stream_url, || app_state.connection_manager.next_stream_uid()); + return StreamMeteringConfig { + meter_uid, + meter_stream: has_stream || has_deferred_provider_open, + pending_shared_registration: pending_registration, + }; + } else if has_stream || has_deferred_provider_open { + let meter_uid = app_state.connection_manager.next_stream_uid(); + return StreamMeteringConfig { meter_uid, meter_stream: true, pending_shared_registration: None }; + } + + StreamMeteringConfig::default() +} + +pub(super) fn resolve_stream_config_u64( + stream_config: Option<&crate::model::StreamConfig>, + selector: impl FnOnce(&crate::model::StreamConfig) -> u64, + default_value: u64, +) -> u64 { + stream_config.map_or(default_value, selector) +} + +pub(super) fn get_stream_config_u64( + app_state: &Arc, + selector: impl FnOnce(&crate::model::StreamConfig) -> u64, + default_value: u64, +) -> u64 { + let config = app_state.app_config.config.load(); + let stream_config = config.reverse_proxy.as_ref().and_then(|reverse_proxy| reverse_proxy.stream.as_ref()); + resolve_stream_config_u64(stream_config, selector, default_value) +} diff --git a/backend/app/src/api/api_utils/mod.rs b/backend/app/src/api/api_utils/mod.rs index ccbc83ba1..94fab5a28 100644 --- a/backend/app/src/api/api_utils/mod.rs +++ b/backend/app/src/api/api_utils/mod.rs @@ -4,117 +4,19 @@ pub use crate::repository::{ }; use crate::{ api::{ - endpoints::{ - hls_api::{hls_media_playlist_wrap_enabled, wrap_media_playlist, HlsMediaPlaylistWrap}, - xtream_api::{get_xtream_player_api_stream_url, ApiStreamContext}, - }, - model::{ - create_active_client_stream, create_channel_unavailable_stream, create_custom_video_stream_response, - create_provider_connections_exhausted_stream, get_custom_stream_response_error_status, - get_stream_response_with_headers, is_custom_video_stream_enabled, open_provider_stream_with_lifecycle, - tee_stream, AppState, BoxedProviderStream, CustomVideoStreamType, PendingProviderReason, PlaybackLeaseRef, - ProviderAllocation, ProviderConfig, ProviderStreamCustomReason, ProviderStreamFactoryOptions, - ProviderStreamInfo, ProviderStreamOpenLifecycle, ProviderStreamState, SharedStreamCtx, SharedStreamManager, - StreamAdmissionError, StreamDetails, StreamError, StreamingStrategy, ThrottledStream, UserApiRequest, - UserSession, - }, + endpoints::xtream_api::ApiStreamContext, + model::{AppState, PlaybackLeaseRef}, }, auth::Fingerprint, - media_server::{ - playback::{ - media_server_image_response as open_media_server_proxy_image_response, - media_server_stream_response as open_media_server_proxy_stream_response, parse_media_server_image_ref, - parse_media_server_stream_ref, - }, - MediaServerError, MediaServerErrorKind, MediaServerHttpClient, MediaServerImageRef, - }, - model::{ - AppConfig, ConfigInput, ConfigInputFlags, ConfigTarget, InputUserInfo, PlaybackKind, ProxyUserCredentials, - }, - processing::{ - parser::hls::{classify_hls_playlist, rewrite_hls, HlsPlaylistKind, RewriteHlsProps}, - processor::re_resolve_stalker_url, - }, - repository::{load_first_input_m3u_stream_url_by_keys, load_input_m3u_stream_url}, - utils::{ - async_file_reader, async_file_writer, classify_output_resource_url, classify_resource_hop, - create_new_file_for_write, debug_if_enabled, get_file_extension, request, - request::{ - classify_resource_destination, content_type_from_ext, parse_range, send_with_retry_and_provider, - ResourceDestination, - }, - trace_if_enabled, ResourceHop, - }, - BUILD_TIMESTAMP, + model::{ConfigInput, ConfigTarget, ProxyUserCredentials}, }; -use arc_swap::ArcSwapOption; -use axum::{ - body::Body, - http::{header, HeaderMap, HeaderName, HeaderValue, Response, StatusCode}, - response::IntoResponse, -}; -use bytes::{Bytes, BytesMut}; -use chrono::{DateTime, Utc}; -use futures::{stream, Stream, StreamExt, TryStreamExt}; -use log::{debug, error, info, log_enabled, trace, warn}; -use serde::Serialize; -use shared::{ - concat_string, - defaults::{DASH_EXT, HLS_EXT}, - model::{ - ConfigTargetOptions, InputFetchMethod, InputType, PlaylistEntry, PlaylistItemType, ProxyType, - ResourceOutputPolicy, StalkerStreamKind, StreamChannel, StreamInfo, TargetType, UserConnectionPermission, - VirtualId, XtreamCluster, - }, - utils::{ - bin_serialize, current_time_secs, extract_extension_from_url, get_credentials_from_url, human_readable_kbps, - is_account_query_key, is_sanitize_sensitive_info_enabled, m3u_flussonic_live_file_lookup_key, - m3u_stream_url_identity, replace_url_extension, sanitize_sensitive_info, trim_slash, Internable, - CONTENT_TYPE_CBOR, CONTENT_TYPE_JSON, - }, -}; -use smallvec::SmallVec; +use axum::http::HeaderMap; +use shared::model::{PlaylistEntry, StreamChannel, TargetType, XtreamCluster}; use std::{ - borrow::Cow, - collections::HashMap, - convert::Infallible, - io::SeekFrom, net::SocketAddr, - path::{Path, PathBuf}, sync::{Arc, LazyLock}, time::Duration, }; -use tokio::{ - io::{AsyncReadExt, AsyncSeekExt}, - sync::RwLock, -}; -use tokio_util::io::ReaderStream; -use tuliprox_hls::api::MAX_HLS_MANIFEST_BYTES; -use url::Url; - -pub(crate) fn resolve_request_url_for_logging<'a>(input: &ConfigInput, stream_url: &'a str) -> Cow<'a, str> { - if !is_sanitize_sensitive_info_enabled() { - return Cow::Borrowed(stream_url); - } - if is_media_server_playback_url(input, stream_url) { - return Cow::Borrowed("media-server://"); - } - - let provider = input.get_resolve_provider(stream_url); - if let Ok(url) = Url::parse(stream_url) { - return Cow::Owned(request::preview_request_target_for_logging(&url, provider.as_ref())); - } - - input - .resolve_url(stream_url) - .ok() - .and_then(|resolved| { - Url::parse(resolved.as_ref()) - .ok() - .map(|url| Cow::Owned(request::preview_request_target_for_logging(&url, provider.as_ref()))) - }) - .unwrap_or(Cow::Borrowed(stream_url)) -} pub(crate) struct ConnectFailedAttempt<'a> { pub app_state: &'a Arc, @@ -127,88 +29,6 @@ pub(crate) struct ConnectFailedAttempt<'a> { pub failure_stage: FailureStage, } -pub(crate) fn record_connect_failed_attempt(attempt: ConnectFailedAttempt<'_>) { - let user_agent = attempt - .req_headers - .get(header::USER_AGENT) - .and_then(|value| value.to_str().ok()) - .unwrap_or_default() - .to_string(); - let info = StreamInfo::new(shared::model::StreamInfoParams { - uid: 0, - meter_uid: 0, - username: &attempt.user.username, - addr: &attempt.fingerprint.addr, - client_ip: &attempt.fingerprint.client_ip, - provider: attempt.provider_name, - stream_channel: attempt.stream_channel, - user_agent, - country_code: None, - session_token: None, - }); - // Resolve target_name from target_id using the stable target config name. - let target_name = - attempt.app_state.app_config.get_target_by_id(info.channel.target_id).as_deref().map(|t| (&t.name).intern()); - attempt.app_state.connection_manager.record_connect_failed_with_provider_failure( - &info, - attempt.reason, - attempt.failure_stage, - None, - None, - target_name, - ); -} - -fn admission_failure_video_type(reason: ConnectFailureReason) -> Option { - match reason { - ConnectFailureReason::UserAccountExpired => Some(CustomVideoStreamType::UserAccountExpired), - ConnectFailureReason::UserConnectionsExhausted => Some(CustomVideoStreamType::UserConnectionsExhausted), - ConnectFailureReason::ProviderConnectionsExhausted => Some(CustomVideoStreamType::ProviderConnectionsExhausted), - _ => None, - } -} - -pub(crate) fn admission_failure_response( - app_state: &Arc, - fingerprint: &Fingerprint, - user: &ProxyUserCredentials, - stream_channel: StreamChannel, - provider_name: Arc, - req_headers: &HeaderMap, - reason: ConnectFailureReason, -) -> axum::response::Response { - record_connect_failed_attempt(ConnectFailedAttempt { - app_state, - fingerprint, - user, - stream_channel, - provider_name, - req_headers, - reason, - failure_stage: FailureStage::Admission, - }); - let Some(video_type) = admission_failure_video_type(reason) else { - error!("Unsupported admission failure reason: {reason:?}"); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - }; - create_custom_video_stream_response(&app_state.provider_stream_ctx(), &fingerprint.addr, video_type).into_response() -} - -/// Produces a defined non-success response for a rejected playback request. Unlike the -/// upstream provider response, this carries no Content-Length/Content-Range so downstream -/// players receive an unambiguous error instead of an empty success body. -fn stream_admission_rejected_response(error: StreamAdmissionError, username: &str) -> axum::response::Response { - error!("Stream admission rejected for user {username}: {error:?}"); - axum::response::Response::builder() - .status(StatusCode::SERVICE_UNAVAILABLE) - .header("x-tuliprox-rejection", "admission_rejected") - .body(axum::body::Body::empty()) - .unwrap_or_else(|_| StatusCode::SERVICE_UNAVAILABLE.into_response()) -} - -/// Produces a quiet termination response when a recently evicted stream retries. -pub fn reentry_suppressed_response() -> axum::response::Response { StatusCode::NO_CONTENT.into_response() } - #[macro_export] macro_rules! try_option_bad_request { ($option:expr, $msg_is_error:expr, $msg:expr) => { @@ -318,19 +138,8 @@ macro_rules! try_result_not_found { }; } -use crate::{ - api::{ - panel_api::{can_provision_on_exhausted, create_panel_api_provisioning_stream_details}, - static_headers::CT_OCTET, - }, - utils::LRUResourceCache, -}; pub use internal_server_error; -use shared::{ - defaults::{default_catchup_session_ttl_secs, default_hls_session_ttl_secs}, - error::TuliproxError, - model::{ConnectFailureReason, FailureStage}, -}; +use shared::model::{ConnectFailureReason, FailureStage}; pub use try_option_bad_request; pub use try_option_forbidden; pub use try_result_bad_request; @@ -349,137 +158,15 @@ pub(crate) use tuliprox_session::{ }, stream_options::{get_stream_options, StreamOptions, StreamResponseMode}, }; -use tuliprox_session::{ProviderSessionHeaders, SessionProviderHeaders}; - -pub fn get_server_time() -> String { - chrono::offset::Local::now().with_timezone(&chrono::Local).format("%Y-%m-%d %H:%M:%S %Z").to_string() -} static PROCESS_START: LazyLock = LazyLock::new(std::time::Instant::now); -/// Anchors the uptime clock; call once at process startup. -pub fn init_uptime_clock() { let _ = *PROCESS_START; } - -pub fn get_uptime_secs() -> u64 { PROCESS_START.elapsed().as_secs() } - -pub fn get_build_time() -> Option { - BUILD_TIMESTAMP - .to_string() - .parse::>() - .ok() - .map(|datetime| datetime.format("%Y-%m-%d %H:%M:%S %Z").to_string()) -} - // Response-compression opt-out moved to `tuliprox_core::utils`; re-exported so // api call sites keep their names. pub(crate) use tuliprox_core::utils::response_compression::{ mark_response_as_uncompressed, should_compress_response_extensions, }; -#[derive(Default)] -struct StreamMeteringConfig { - meter_uid: u32, - meter_stream: bool, - pending_shared_registration: Option, -} - -impl StreamMeteringConfig { - fn commit_shared_registration(&mut self) { - if let Some(registration) = self.pending_shared_registration.take() { - registration.commit(); - } - } -} - -#[allow(clippy::missing_panics_doc)] -pub async fn serve_file(file_path: &Path, mime_type: String, cache_control: Option<&str>) -> impl IntoResponse + Send { - match tokio::fs::try_exists(file_path).await { - Ok(exists) => { - if !exists { - return StatusCode::NOT_FOUND.into_response(); - } - } - Err(err) => { - error!("Failed to open file {}, {err:?}", file_path.display()); - return StatusCode::NOT_FOUND.into_response(); - } - } - - match tokio::fs::File::open(file_path).await { - Ok(file) => { - let last_modified = file.metadata().await.ok().and_then(|m| m.modified().ok()).map(|m| { - let dt: DateTime = m.into(); - dt.format("%a, %d %b %Y %H:%M:%S GMT").to_string() - }); - - let reader = async_file_reader(file); - let stream = ReaderStream::new(reader); - let body = Body::from_stream(stream); - - let mut builder = axum::response::Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, mime_type) - .header(header::CACHE_CONTROL, cache_control.unwrap_or("no-cache")); - - if let Some(lm) = last_modified { - builder = builder.header(header::LAST_MODIFIED, lm); - } - - try_unwrap_body!(builder.body(body)) - } - Err(_) => internal_server_error!(), - } -} - -pub fn get_user_target_by_username( - username: &str, - app_state: &Arc, -) -> Option<(Arc, Arc)> { - if !username.is_empty() { - return app_state.app_config.get_target_for_username(username); - } - None -} - -pub fn get_user_target_by_credentials<'a>( - username: &str, - password: &str, - api_req: &'a UserApiRequest, - app_state: &'a AppState, -) -> Option<(Arc, Arc)> { - if !username.is_empty() && !password.is_empty() { - app_state.app_config.get_target_for_user(username, password) - } else { - let token = api_req.token.as_str().trim(); - if token.is_empty() { - None - } else { - app_state.app_config.get_target_for_user_by_token(token) - } - } -} - -pub fn get_user_target<'a>( - api_req: &'a UserApiRequest, - app_state: &'a AppState, -) -> Option<(Arc, Arc)> { - let username = api_req.username.as_str().trim(); - let password = api_req.password.as_str().trim(); - get_user_target_by_credentials(username, password, api_req, app_state) -} - -struct StreamingAcquireOptions<'a> { - force_provider: Option<&'a Arc>, - allow_forced_provider_fallback: bool, - allow_provider_grace: bool, - user_priority: i8, - connection_kind: crate::api::model::ConnectionKind, - session_owner: Option<&'a str>, - playback_kind: PlaybackKind, - accept_requested_stream_url: bool, - capacity_wait_timeout: Option, -} - pub struct ForceStreamRequestContext<'a> { pub req_headers: &'a HeaderMap, pub input: &'a Arc, @@ -488,860 +175,14 @@ pub struct ForceStreamRequestContext<'a> { pub(crate) content_representation: crate::api::model::ProviderContentRepresentationMode, } -struct SessionActivationRequest<'a> { - fingerprint: &'a Fingerprint, - input: &'a ConfigInput, - user: &'a ProxyUserCredentials, - session_token: &'a str, - request_class: Option, - virtual_id: VirtualId, - item_type: PlaylistItemType, - stream_url: &'a str, - connection_permission: UserConnectionPermission, - connection_kind: crate::api::model::ConnectionKind, - granted_grace_mode: Option, - socket_bound: bool, -} - -struct PlaybackActivationResult { - admission: crate::api::model::ConnectionAdmission, - grace_mode: Option, - grace_context: Option, - placeholder_transition_version: Option, -} - -/// # Panics -#[allow(clippy::too_many_lines)] -async fn activate_session_before_stream_open( - app_state: &Arc, - request: SessionActivationRequest<'_>, -) -> PlaybackActivationResult { - let SessionActivationRequest { - fingerprint, - input, - user, - session_token, - request_class, - virtual_id, - item_type, - stream_url, - connection_permission, - connection_kind, - granted_grace_mode, - socket_bound, - } = request; - // Classify based on current session state, not the pre-computed value. - // If caller passes FollowUp, verify the session is still counted under the guard. - // A stale FollowUp would bypass admission — reclassify to catch this. - let (effective_request_class, loaded_session) = if let Some(request_class) = request_class { - if matches!(request_class, PlaybackRequestClass::FollowUp | PlaybackRequestClass::Activate) { - // Re-read session under the guard to ensure the counted lease is still held or acquired. - // If it is no longer counted, classify it from the current lifecycle so - // stale FollowUp requests cannot bypass admission. - // If it became counted, classify it so stale Activate requests don't double count. - let current_session = - app_state.active_users.get_and_update_user_session(&user.username, session_token).await; - let classified = classify_playback_request(PlaybackRequestFacts { - existing_session: current_session.as_ref(), - prepare_only: false, - terminate: false, - }); - (classified, Some(current_session)) - } else { - (request_class, None) - } - } else { - let existing_session = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; - let classified = classify_playback_request(PlaybackRequestFacts { - existing_session: existing_session.as_ref(), - prepare_only: false, - terminate: false, - }); - (classified, Some(existing_session)) - }; - let limits_enabled = app_state.app_config.config.load().user_access_control - && (user.max_connections > 0 || user.soft_connections > 0); - // Prepare: session setup without admission cost. The caller handles the actual activation. - // FollowUp: already counted, no re-admission needed. - // GracePeriod: grace already granted, no re-evaluation needed. - // No limits: skip admission entirely. - // GracePeriod permission is already resolved — skip admission strategies (re-run - // would evict the same session again). But we must still materialize the grace - // lifecycle (PendingProvider / GraceActive) so the session state is consistent. - if connection_permission == UserConnectionPermission::GracePeriod { - if loaded_session.as_ref().is_none_or(Option::is_none) { - app_state - .active_users - .ensure_user_session_placeholder(crate::api::model::CreateUserSessionParams { - user, - session_token, - virtual_id: virtual_id.get(), - provider: input.name.as_ref(), - stream_url, - addr: &fingerprint.addr, - connection_permission, - connection_kind: Some(connection_kind), - socket_bound, - }) - .await; - } - // Materialize grace lifecycle under the guard so the session state is consistent. - let current_session = match loaded_session { - Some(Some(session)) => Some(session), - _ => app_state.active_users.get_and_update_user_session(&user.username, session_token).await, - }; - let (_, resolved_grace) = match current_session.as_ref().map(|s| &s.lifecycle) { - Some(crate::api::model::PlaybackLifecycle::PendingProvider { .. }) => { - // Session already in PendingProvider — refresh deadline. - let deadline = current_time_secs().saturating_add(app_state.get_grace_options().timeout_secs); - let _ = app_state - .active_users - .mark_pending_provider(&user.username, session_token, PendingProviderReason::GraceHold, deadline) - .await; - ( - crate::api::model::PlaybackLifecycle::PendingProvider { - data: crate::api::model::PendingProviderState { - reason_code: PendingProviderReason::GraceHold, - created_at: current_time_secs(), - deadline, - version: current_session.as_ref().map_or(0, |s| { - if let crate::api::model::PlaybackLifecycle::PendingProvider { data } = &s.lifecycle { - data.version - } else { - 0 - } - }), - wake_source: None, - }, - }, - Some(crate::api::model::GraceMode::Hold), - ) - } - Some(crate::api::model::PlaybackLifecycle::GraceActive) => { - (crate::api::model::PlaybackLifecycle::GraceActive, Some(crate::api::model::GraceMode::Instant)) - } - _ => { - let hold_stream = granted_grace_mode.map_or_else( - || item_type.is_live() || item_type.is_live_adaptive(), - |mode| matches!(mode, crate::api::model::GraceMode::Hold), - ); - if hold_stream { - let deadline = current_time_secs().saturating_add(app_state.get_grace_options().timeout_secs); - let _ = app_state - .active_users - .mark_pending_provider( - &user.username, - session_token, - PendingProviderReason::GraceHold, - deadline, - ) - .await; - ( - crate::api::model::PlaybackLifecycle::PendingProvider { - data: crate::api::model::PendingProviderState { - reason_code: PendingProviderReason::GraceHold, - created_at: current_time_secs(), - deadline, - version: 1, - wake_source: None, - }, - }, - Some(crate::api::model::GraceMode::Hold), - ) - } else { - app_state.active_users.mark_grace_active(&user.username, session_token).await; - (crate::api::model::PlaybackLifecycle::GraceActive, Some(crate::api::model::GraceMode::Instant)) - } - } - }; - return PlaybackActivationResult { - admission: crate::api::model::ConnectionAdmission::from_permission( - connection_permission, - Some(connection_kind), - ), - grace_mode: resolved_grace, - grace_context: None, - placeholder_transition_version: None, - }; - } - // No limits: skip admission entirely. FollowUp / Prepare: no re-admission needed. - if !limits_enabled - || effective_request_class == PlaybackRequestClass::FollowUp - || effective_request_class == PlaybackRequestClass::Prepare - { - return PlaybackActivationResult { - admission: crate::api::model::ConnectionAdmission::from_permission( - connection_permission, - Some(connection_kind), - ), - grace_mode: None, - grace_context: None, - placeholder_transition_version: None, - }; - } - - let placeholder_transition_version = Some( - app_state - .active_users - .ensure_user_session_placeholder(crate::api::model::CreateUserSessionParams { - user, - session_token, - virtual_id: virtual_id.get(), - provider: input.name.as_ref(), - stream_url, - addr: &fingerprint.addr, - connection_permission, - connection_kind: Some(connection_kind), - socket_bound, - }) - .await, - ); - - let result = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - client_ip: &fingerprint.client_ip, - request_addr: &fingerprint.addr, - use_session_admission: true, - session_token: Some(session_token), - activate_unbound_session: true, - eviction_reentry_guard: if socket_bound { - EvictionReentryGuard::SocketPlayback { virtual_id } - } else { - EvictionReentryGuard::Session(session_token) - }, - }, - ) - .await; - let admission = result.admission; - let grace_mode = result.grace_mode; - let grace_context = result.grace_context; - - if admission.permission() == UserConnectionPermission::GracePeriod { - if matches!(grace_mode, Some(crate::api::model::GraceMode::Hold)) { - // Hold: session waits for provider slot. Does not count until provider is acquired. - let deadline = current_time_secs().saturating_add(app_state.get_grace_options().timeout_secs); - let _ = app_state - .active_users - .mark_pending_provider(&user.username, session_token, PendingProviderReason::GraceHold, deadline) - .await; - } else if matches!(grace_mode, Some(crate::api::model::GraceMode::Instant)) { - // Instant: session is provisionally active immediately. Counts against admission limits - // until the grace window resolves (success -> Active, failure -> Expired). - app_state.active_users.mark_grace_active(&user.username, session_token).await; - } - } - - PlaybackActivationResult { admission, grace_mode, grace_context, placeholder_transition_version } -} - -pub fn get_stream_alternative_url( - stream_url: &str, - input: &ConfigInput, - alias_input: &Arc, -) -> Option { - if input.input_type.is_m3u() && input.get_matched_config_by_url(stream_url).is_none() { - return get_stream_alternative_url_m3u(stream_url, input, alias_input); - } - - let (source_base_url, source_username, source_password, matched_via_external_signature) = - if let Some(matched) = input.get_matched_config_by_url(stream_url) { - (matched.0.to_string(), matched.1.cloned(), matched.2.cloned(), false) - } else { - let (base_url, username, password) = find_input_account_by_signature(stream_url, input)?; - (base_url, username, password, true) - }; - if matched_via_external_signature && !input.input_type.is_m3u() { - return None; - } - let alt_input_user_info = alias_input.get_user_info()?; - - let modified = stream_url.replacen(&source_base_url, &alt_input_user_info.base_url, 1); - let mut url = Url::parse(&modified).ok()?; - - if let (Some(old_username), Some(old_password)) = (source_username, source_password) { - let auth_updated = rewrite_url_auth_fields( - &mut url, - &old_username, - &old_password, - &alt_input_user_info.username, - &alt_input_user_info.password, - ); - if !auth_updated { - return None; - } - } - - Some(url.to_string()) -} - -fn get_stream_alternative_url_m3u( - stream_url: &str, - input: &ConfigInput, - alias_input: &Arc, -) -> Option { - if let Some(source_config_url) = find_input_account_by_query_signature(stream_url, input) { - return rewrite_account_query_fields(stream_url, source_config_url, &alias_input.url); - } - - if let Some((source_base_url, source_username, source_password)) = - find_input_account_by_signature(stream_url, input) - { - let Some(alt_input_user_info) = alias_input.get_user_info() else { - return Some(stream_url.to_string()); - }; - let modified = stream_url.replacen(&source_base_url, &alt_input_user_info.base_url, 1); - let mut url = Url::parse(&modified).ok()?; - - if let (Some(old_username), Some(old_password)) = (source_username, source_password) { - let auth_updated = rewrite_url_auth_fields( - &mut url, - &old_username, - &old_password, - &alt_input_user_info.username, - &alt_input_user_info.password, - ); - if !auth_updated { - return None; - } - } - - return Some(url.to_string()); - } - let Some(alt_input_user_info) = alias_input.get_user_info() else { - let Ok(url) = Url::parse(stream_url) else { - return None; - }; - if providerless_m3u_url_has_explicit_credentials(&url) { - return None; - } - return Some(stream_url.to_string()); - }; - if stream_url_has_account_signature(stream_url, &alt_input_user_info) { - return None; - } - Some(stream_url.to_string()) -} - -fn find_input_account_by_query_signature<'a>(stream_url: &str, input: &'a ConfigInput) -> Option<&'a str> { - if stream_url_account_query_matches(stream_url, &input.url) { - return Some(&input.url); - } - - input.aliases.as_ref().and_then(|aliases| { - aliases - .iter() - .find(|alias| stream_url_account_query_matches(stream_url, &alias.url)) - .map(|alias| alias.url.as_str()) - }) -} - -fn stream_url_account_query_matches(stream_url: &str, config_url: &str) -> bool { - let (Ok(stream_url), Ok(config_url)) = (Url::parse(stream_url), Url::parse(config_url)) else { - return false; - }; - - config_url.query_pairs().any(|(config_key, config_value)| { - is_account_query_key(&config_key) - && stream_url.query_pairs().any(|(stream_key, stream_value)| { - stream_key.eq_ignore_ascii_case(&config_key) && stream_value == config_value - }) - }) -} - -fn rewrite_account_query_fields(stream_url: &str, source_config_url: &str, target_config_url: &str) -> Option { - let mut stream_url = Url::parse(stream_url).ok()?; - let source_config_url = Url::parse(source_config_url).ok()?; - let target_config_url = Url::parse(target_config_url).ok()?; - let source_pairs: Vec<_> = source_config_url - .query_pairs() - .filter(|(key, _)| is_account_query_key(key)) - .map(|(key, value)| (key.into_owned(), value.into_owned())) - .collect(); - let target_pairs: Vec<_> = target_config_url - .query_pairs() - .filter(|(key, _)| is_account_query_key(key)) - .map(|(key, value)| (key.into_owned(), value.into_owned())) - .collect(); - let unambiguous_cross_key_target = - if source_pairs.len() == 1 && target_pairs.len() == 1 { target_pairs.first() } else { None }; - let present_source_pairs: Vec<_> = source_pairs - .iter() - .filter(|(source_key, source_value)| { - stream_url.query_pairs().any(|(stream_key, stream_value)| { - stream_key.eq_ignore_ascii_case(source_key) && stream_value == source_value.as_str() - }) - }) - .collect(); - let replacements: Vec<_> = present_source_pairs - .iter() - .filter_map(|(source_key, source_value)| { - target_pairs - .iter() - .find(|(target_key, _)| target_key.eq_ignore_ascii_case(source_key)) - .or(unambiguous_cross_key_target) - .map(|(target_key, target_value)| { - (source_key.clone(), source_value.clone(), target_key.clone(), target_value.clone()) - }) - }) - .collect(); - - if replacements.len() != present_source_pairs.len() || replacements.is_empty() { - return None; - } - - let mut replaced = false; - let stream_pairs: Vec<(String, String)> = stream_url - .query_pairs() - .map(|(key, value)| { - if let Some((_, _, target_key, target_value)) = - replacements.iter().find(|(source_key, source_value, _, _)| { - key.eq_ignore_ascii_case(source_key) && value == source_value.as_str() - }) - { - replaced = true; - (target_key.clone(), target_value.clone()) - } else { - (key.into_owned(), value.into_owned()) - } - }) - .collect(); - - if !replaced { - return None; - } - stream_url - .query_pairs_mut() - .clear() - .extend_pairs(stream_pairs.iter().map(|(key, value)| (key.as_str(), value.as_str()))); - Some(stream_url.to_string()) -} - -fn providerless_m3u_url_has_explicit_credentials(url: &Url) -> bool { - !url.username().is_empty() - || url.password().is_some() - || url - .query_pairs() - .any(|(key, _)| key.eq_ignore_ascii_case("username") || key.eq_ignore_ascii_case("password")) -} - -/// Look for an account signature in the stream URL that matches the input -/// itself or one of its configured aliases. Returns the matching entry's -/// `(base_url, username, password)` so the caller can rewrite only the -/// account-specific parts of the URL while preserving the original host/path. -/// -/// This helper is used for safe credential rewrites when Tuliprox switches -/// from one account to another. It is not the general trust gate for M3U -/// foreign hosts: plain external URLs from a stored M3U playlist item may be -/// accepted without a matching signature, while unrelated credential-bearing -/// URLs still fail closed unless they provably match the input or one of its -/// aliases. -fn find_input_account_by_signature( - stream_url: &str, - input: &ConfigInput, -) -> Option<(String, Option, Option)> { - // Try the input's main account first. - if let Some(user_info) = input.get_user_info() { - if stream_url_account_matches(stream_url, &user_info) { - return Some((input.url.clone(), Some(user_info.username), Some(user_info.password))); - } - } - // Then try each alias, if any. The input_type is inherited from the - // parent input for all aliases — see ConfigInputAlias definition. - if let Some(aliases) = input.aliases.as_ref() { - for alias in aliases { - if let Some(user_info) = - InputUserInfo::new(input.input_type, alias.username.as_deref(), alias.password.as_deref(), &alias.url) - { - if stream_url_account_matches(stream_url, &user_info) { - return Some((alias.url.clone(), Some(user_info.username), Some(user_info.password))); - } - } - } - } - None -} - -fn rewrite_url_auth_fields( - url: &mut Url, - old_username: &str, - old_password: &str, - new_username: &str, - new_password: &str, -) -> bool { - if rewrite_query_auth_fields(url, new_username, new_password) { - return true; - } - - if url.username() == old_username && url.password() == Some(old_password) { - return url.set_username(new_username).is_ok() && url.set_password(Some(new_password)).is_ok(); - } - - rewrite_path_auth_fields(url, old_username, old_password, new_username, new_password) -} - -fn rewrite_query_auth_fields(url: &mut Url, new_username: &str, new_password: &str) -> bool { - let mut has_username = false; - let mut has_password = false; - let pairs: Vec<(String, String)> = url - .query_pairs() - .map(|(key, value)| { - if key.eq_ignore_ascii_case("username") { - has_username = true; - (key.into_owned(), new_username.to_string()) - } else if key.eq_ignore_ascii_case("password") { - has_password = true; - (key.into_owned(), new_password.to_string()) - } else { - (key.into_owned(), value.into_owned()) - } - }) - .collect(); - - if !(has_username && has_password) { - return false; - } - - url.query_pairs_mut().clear().extend_pairs(pairs.iter().map(|(key, value)| (key.as_str(), value.as_str()))); - true -} - -fn collect_path_segments(url: &Url) -> Option> { - url.path_segments().map(|segments| segments.map(ToOwned::to_owned).collect::>()) -} - -fn find_path_auth_segment_index(segments: &[String], username: &str, password: &str) -> Option { - segments.windows(2).position(|pair| { - pair.first().is_some_and(|segment| segment == username) - && pair.get(1).is_some_and(|segment| segment == password) - }) -} - -fn rewrite_path_auth_fields( - url: &mut Url, - old_username: &str, - old_password: &str, - new_username: &str, - new_password: &str, -) -> bool { - let Some(mut segments) = collect_path_segments(url) else { - return false; - }; - - let credential_index = find_path_auth_segment_index(&segments, old_username, old_password); - let Some(credential_index) = credential_index else { - return false; - }; - - segments[credential_index] = new_username.to_string(); - segments[credential_index + 1] = new_password.to_string(); - - let Ok(mut path_segments) = url.path_segments_mut() else { - return false; - }; - path_segments.clear().extend(segments.iter().map(String::as_str)); - true -} - -fn stream_url_matches_provider(stream_url: &str, provider_cfg: &ProviderConfig) -> bool { - let account_query_matches = - provider_cfg.input_type.is_m3u() && stream_url_account_query_matches(stream_url, &provider_cfg.url); - let Some(user_info) = provider_cfg.get_user_info() else { - return account_query_matches; - }; - if stream_url_base_matches(stream_url, &user_info.base_url) { - // Same-host fast path: both base URL and account identity must match. - return stream_url_account_matches(stream_url, &user_info); - } - if !provider_cfg.input_type.is_m3u() { - return false; - } - // For M3U inputs, the stored playlist entry itself is the trust anchor. - // Open external URLs are therefore allowed, but external URLs that carry - // explicit account markers must still match the selected provider account. - if stream_url_has_account_signature(stream_url, &user_info) { - return stream_url_account_matches(stream_url, &user_info); - } - true -} - -fn stream_url_base_matches(stream_url: &str, base_url: &str) -> bool { - stream_url - .strip_prefix(base_url) - .is_some_and(|remaining| remaining.is_empty() || remaining.starts_with(['/', '?', '#'])) -} - -fn stream_url_account_matches(stream_url: &str, user_info: &crate::model::InputUserInfo) -> bool { - let Ok(url) = Url::parse(stream_url) else { - return false; - }; - - let (url_username, url_password) = get_credentials_from_url(&url); - if let (Some(url_username), Some(url_password)) = (url_username.as_deref(), url_password.as_deref()) { - return url_username == user_info.username && url_password == user_info.password; - } - - let mut has_query_username = false; - let mut has_query_password = false; - for (key, value) in url.query_pairs() { - if key.eq_ignore_ascii_case("username") { - has_query_username = value == user_info.username; - } else if key.eq_ignore_ascii_case("password") { - has_query_password = value == user_info.password; - } - } - if has_query_username || has_query_password { - return has_query_username && has_query_password; - } - - let Some(segments) = collect_path_segments(&url) else { - return false; - }; - - find_path_auth_segment_index(&segments, &user_info.username, &user_info.password).is_some() -} - -fn stream_url_has_account_signature(stream_url: &str, user_info: &crate::model::InputUserInfo) -> bool { - let Ok(url) = Url::parse(stream_url) else { - return false; - }; - - let (url_username, url_password) = get_credentials_from_url(&url); - if url_username.is_some() && url_password.is_some() { - return true; - } - - let mut has_query_username = false; - let mut has_query_password = false; - for (key, _) in url.query_pairs() { - if key.eq_ignore_ascii_case("username") { - has_query_username = true; - } else if key.eq_ignore_ascii_case("password") { - has_query_password = true; - } - } - if has_query_username || has_query_password { - return has_query_username && has_query_password; - } - - // Path-based credentials: some Xtream endpoints embed the account in the URL - // path (e.g. /live///...). Only flag a signature when the - // consecutive segments actually match the configured user/pass — arbitrary - // open paths must not be treated as account signatures. - if let Some(segments) = collect_path_segments(&url) { - if find_path_auth_segment_index(&segments, &user_info.username, &user_info.password).is_some() { - return true; - } - } - - false -} - -/// Maps a Flussonic archive URL of another account onto the alias account. -/// -/// The URL index only holds live URLs, so an archive URL derived from the main account's -/// live URL never matches. The archive lives next to the channel's live file, so the alias -/// URL of that sibling supplies the alias credentials and the archive file name is kept. -async fn resolve_m3u_alias_flussonic_archive_url( - app_config: &Arc, - provider_name: &Arc, - stream_url: &str, -) -> Option { - let requested = Url::parse(stream_url).ok()?; - let archive_file = requested.path_segments()?.next_back()?; - crate::iptv::m3u::parse_flussonic_archive_file(archive_file)?; - // The live file the archive was derived from is probed first: same stem, then same transport. - let archive_lower = archive_file.to_ascii_lowercase(); - let archive_is_hls = archive_lower.ends_with(HLS_EXT); - let archive_stem = archive_lower.split_once('-').map(|(stem, _)| stem); - let mut live_files = crate::iptv::m3u::FLUSSONIC_LIVE_FILES; - live_files.sort_by_key(|live_file| { - let same_stem = - archive_stem.is_some_and(|stem| live_file.split_once('.').is_some_and(|(name, _)| name == stem)); - let same_transport = live_file.ends_with(HLS_EXT) == archive_is_hls; - (!same_stem, !same_transport) - }); - // Each sibling is probed by its exact identity, then by its case-folded Flussonic key. - let keys: Vec = live_files - .iter() - .filter_map(|live_file| { - let mut sibling = requested.clone(); - sibling.path_segments_mut().ok()?.pop().push(live_file); - Some([m3u_stream_url_identity(sibling.as_str()), m3u_flussonic_live_file_lookup_key(sibling.as_str())]) - }) - .flatten() - .flatten() - .collect(); - let alias_live_url = match load_first_input_m3u_stream_url_by_keys(app_config, provider_name, keys).await { - Ok(url) => url?, - Err(err) => { - debug_if_enabled!( - "Failed to resolve M3U archive sibling URL for provider {}: {}", - sanitize_sensitive_info(provider_name), - sanitize_sensitive_info(&err.to_string()) - ); - return None; - } - }; - let mut alias_url = Url::parse(&alias_live_url).ok()?; - alias_url.path_segments_mut().ok()?.pop().push(archive_file); - Some(alias_url.into()) -} - -pub(crate) async fn select_provider_stream_url( - stream_url: &str, - input: &ConfigInput, - provider_cfg: &Arc, - accept_requested_stream_url: bool, - app_config: &Arc, -) -> Option<(Arc, String)> { - if accept_requested_stream_url { - return Some((provider_cfg.name.clone(), stream_url.to_string())); - } - - if provider_cfg.input_type.is_m3u() { - match load_input_m3u_stream_url(app_config, &provider_cfg.name, stream_url).await { - Ok(Some(provider_stream_url)) => { - debug_if_enabled!( - "M3U alias URL lookup: provider={} requested_url={} index_match=true resolved_url={}", - sanitize_sensitive_info(&provider_cfg.name), - sanitize_sensitive_info(resolve_request_url_for_logging(input, stream_url).as_ref()), - sanitize_sensitive_info(resolve_request_url_for_logging(input, &provider_stream_url).as_ref()) - ); - return Some((provider_cfg.name.clone(), provider_stream_url.to_string())); - } - Ok(None) => { - if let Some(provider_stream_url) = - resolve_m3u_alias_flussonic_archive_url(app_config, &provider_cfg.name, stream_url).await - { - debug_if_enabled!( - "M3U alias URL lookup: provider={} requested_url={} index_match=archive_sibling resolved_url={}", - sanitize_sensitive_info(&provider_cfg.name), - sanitize_sensitive_info(resolve_request_url_for_logging(input, stream_url).as_ref()), - sanitize_sensitive_info(resolve_request_url_for_logging(input, &provider_stream_url).as_ref()) - ); - return Some((provider_cfg.name.clone(), provider_stream_url)); - } - debug_if_enabled!( - "M3U alias URL lookup: provider={} requested_url={} index_match=false", - sanitize_sensitive_info(&provider_cfg.name), - sanitize_sensitive_info(resolve_request_url_for_logging(input, stream_url).as_ref()) - ); - } - Err(err) => { - debug_if_enabled!( - "Failed to resolve M3U stream URL for provider {}: {}", - sanitize_sensitive_info(&provider_cfg.name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - } - - if stream_url_matches_provider(stream_url, provider_cfg) { - Some((provider_cfg.name.clone(), stream_url.to_string())) - } else { - get_stream_alternative_url(stream_url, input, provider_cfg).map(|url| (provider_cfg.name.clone(), url)) - } -} - -/// Stored Xtream VOD entries belong to their named input even when their URL uses a CDN host. -fn resolve_xtream_vod_provider_url( - stream_url: &str, - input: &ConfigInput, - provider_cfg: &ProviderConfig, - channel: &StreamChannel, -) -> Option { - if input.input_type != InputType::Xtream - || provider_cfg.input_type != InputType::Xtream - || channel.item_type != PlaylistItemType::Video - || channel.input_name != input.name - || channel.url.as_ref() != stream_url - || channel.provider_id == 0 - { - return None; - } - let url = Url::parse(stream_url).ok()?; - if !matches!(url.scheme(), "http" | "https") { - return None; - } - let is_main = provider_cfg.name == input.name; - if !is_main - && !input.aliases.as_ref().is_some_and(|aliases| aliases.iter().any(|alias| alias.name == provider_cfg.name)) - { - return None; - } - let selected = provider_cfg.get_user_info()?; - let origin = input.get_user_info()?; - if is_main - && selected.base_url == origin.base_url - && selected.username == origin.username - && selected.password == origin.password - { - return Some(stream_url.to_string()); - } - - // A direct source may carry account-bound tokens. A different account obtains its own redirect. - let extension = channel - .technical - .as_ref() - .map(|technical| technical.container.as_str()) - .filter(|container| !container.is_empty()) - .map(|container| match container { - "mpegts" => "ts", - "hls" => "m3u8", - "dash" => "mpd", - other => other, - }) - .or_else(|| extract_extension_from_url(&channel.url).and_then(|extension| extension.strip_prefix('.'))); - let base_url = selected.base_url.trim_end_matches('/'); - let mut url = format!("{base_url}/movie/{}/{}/{}", selected.username, selected.password, channel.provider_id); - if let Some(extension) = extension { - url.push('.'); - url.push_str(extension); - } - Some(url) -} - -fn create_unmapped_provider_stream(app_config: &AppConfig) -> ProviderStreamState { - ProviderStreamState::Custom { - response: create_channel_unavailable_stream(app_config, &[], StatusCode::OK), - reason: ProviderStreamCustomReason::UnmappedProviderUrl, - } -} - -/// Splits a provider open into stream, response info and session headers. An upstream error -/// of a finite HLS resource carries no stream, only its status and forwarded headers. -fn split_provider_stream_open( - open: crate::api::model::ProviderStreamOpen, -) -> (Option, ProviderStreamInfo, ProviderSessionHeaders) { - match open { - crate::api::model::ProviderStreamOpen::Stream(response) => { - (Some(response.stream), response.info, response.provider_session_headers) - } - crate::api::model::ProviderStreamOpen::UpstreamStatus { status, headers } => { - (None, Some((headers, status, None, None)), ProviderSessionHeaders::default()) - } - } -} - -/// Session and account binding a forced resource request must still belong to after acquiring capacity. -#[derive(Clone, Copy)] -struct CurrentSessionGuard<'a> { - username: &'a str, - token: &'a str, - identity: tuliprox_session::SessionIdentity, -} - /// Upper bound a manifest refresh waits for its pinned account to free a slot. Parallel /// playbacks on one account free their slots within seconds; a 503 would stall the player. pub(crate) const HLS_MANIFEST_CAPACITY_WAIT: Duration = Duration::from_secs(5); + /// Upper bound a media object (segment, init, key) waits. It stays well below a typical /// segment duration, so the player keeps buffer to retry after a 503. const HLS_MEDIA_CAPACITY_WAIT: Duration = Duration::from_secs(2); + /// Lower bound between two acquisition attempts that only wait for a lease to expire. const LEASE_EXPIRY_RECHECK_FLOOR: Duration = Duration::from_millis(10); @@ -1355,818 +196,6 @@ pub(crate) struct ExactProviderAcquire<'a> { pub lease: Option>, } -/// Acquires a slot on exactly `request.provider`, waiting at most `capacity_wait` for one to -/// free up. Only slot and lease releases of this provider wake the waiter; reserved capacity -/// held by idle leases is retried when the next lease expires. -pub(crate) async fn acquire_exact_provider_handle( - app_state: &Arc, - request: &ExactProviderAcquire<'_>, - capacity_wait: Option, -) -> Option { - let deadline = capacity_wait.map(|wait| tokio::time::Instant::now() + wait); - let acquire = || { - app_state.active_provider.acquire_exact_connection_with_lease_for_session_until( - request.provider, - request.addr, - request.allow_grace, - request.priority, - request.kind, - request.lease, - deadline, - ) - }; - let Some(deadline) = deadline else { - return acquire().await; - }; - let capacity_notify = app_state.active_provider.provider_capacity_notify(request.provider); - loop { - let notified = capacity_notify.notified(); - tokio::pin!(notified); - notified.as_mut().enable(); - if app_state.connection_manager.is_shutting_down() || tokio::time::Instant::now() >= deadline { - return None; - } - if let Some(handle) = acquire().await { - return Some(handle); - } - let earliest_retry = tokio::time::Instant::now() + LEASE_EXPIRY_RECHECK_FLOOR; - let wake_at = app_state - .active_provider - .next_lease_expiry() - .map_or(deadline, |expiry| expiry.max(earliest_retry).min(deadline)); - tokio::select! { - () = tokio::time::sleep_until(wake_at) => {}, - () = &mut notified => {}, - } - } -} - -/// Waits for finite resources to release capacity without superseding an active body. -async fn acquire_exact_stream_provider_handle( - app_state: &Arc, - provider: &Arc, - fingerprint: &Fingerprint, - options: &StreamingAcquireOptions<'_>, -) -> Option { - let request = ExactProviderAcquire { - provider, - addr: &fingerprint.addr, - allow_grace: options.allow_provider_grace, - priority: options.user_priority, - kind: options.connection_kind, - lease: options.session_owner.map(|owner| PlaybackLeaseRef::new(owner, options.playback_kind)), - }; - acquire_exact_provider_handle(app_state, &request, options.capacity_wait_timeout).await -} - -async fn acquire_stream_provider_handle( - app_state: &Arc, - input: &ConfigInput, - fingerprint: &Fingerprint, - options: &StreamingAcquireOptions<'_>, -) -> Option { - let lease = options.session_owner.map(|owner| PlaybackLeaseRef::new(owner, options.playback_kind)); - let managed = |handle| tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), handle); - match options.force_provider { - Some(provider) => { - // First try to stay on the exact pinned provider account without over-allocating. - if let Some(handle) = acquire_exact_stream_provider_handle(app_state, provider, fingerprint, options).await - { - Some(managed(handle)) - } else if options.allow_forced_provider_fallback { - debug_if_enabled!( - "Pinned provider {} unavailable for {}; falling back to lineup allocation", - sanitize_sensitive_info(provider), - sanitize_sensitive_info(&fingerprint.addr.to_string()) - ); - app_state - .active_provider - .acquire_connection_with_lease_for_session_await( - &input.name, - &fingerprint.addr, - options.allow_provider_grace, - options.user_priority, - options.connection_kind, - lease, - ) - .await - .map(managed) - } else { - debug_if_enabled!( - "Pinned provider {} unavailable for {}; strict provider affinity prevents fallback", - sanitize_sensitive_info(provider), - sanitize_sensitive_info(&fingerprint.addr.to_string()) - ); - None - } - } - None => app_state - .active_provider - .acquire_connection_with_lease_for_session_await( - &input.name, - &fingerprint.addr, - options.allow_provider_grace, - options.user_priority, - options.connection_kind, - lease, - ) - .await - .map(managed), - } -} - -fn allows_provider_pool_failover(item_type: PlaylistItemType) -> bool { - matches!( - item_type, - PlaylistItemType::Video - | PlaylistItemType::LocalVideo - | PlaylistItemType::Series - | PlaylistItemType::SeriesInfo - | PlaylistItemType::LocalSeries - | PlaylistItemType::LocalSeriesInfo - ) -} - -pub(crate) fn resolve_redirect_location<'a>( - input: Option<&ConfigInput>, - stream_url: &'a str, -) -> Result, TuliproxError> { - input.map_or(Ok(Cow::Borrowed(stream_url)), |input| input.resolve_url(stream_url)) -} - -fn get_redirect_alternative_url(app_state: &Arc, redirect_url: &Arc, input: &ConfigInput) -> Arc { - if let Some((base_url, username, password)) = input.get_matched_config_by_url(redirect_url) { - if let Some(provider_cfg) = app_state.active_provider.get_next_provider(&input.name) { - let mut new_url = redirect_url.replacen(base_url, provider_cfg.url.as_str(), 1); - if let (Some(old_username), Some(old_password)) = (username, password) { - if let (Some(new_username), Some(new_password)) = - (provider_cfg.username.as_ref(), provider_cfg.password.as_ref()) - { - new_url = new_url.replacen(old_username, new_username, 1); - new_url = new_url.replacen(old_password, new_password, 1); - return new_url.into(); - } - // one has credentials the other not, something not right - return redirect_url.clone(); - } - return new_url.into(); - } - } - redirect_url.clone() -} - -/// Determines the appropriate streaming strategy for the given input and stream URL. -/// -/// This function attempts to acquire a connection to a streaming provider, either using a forced provider -/// (if specified), or based on the input name. It then selects a corresponding `StreamingOption`: -/// -/// - If no connections are available (`Exhausted`), it returns a custom stream indicating exhaustion. -/// - If a connection is available or in a grace period, it constructs a streaming URL accordingly: -/// - If the URL already targets the selected provider account, the original URL is reused. -/// - Otherwise, an alternative URL is generated based on the provider and input. -/// -/// The function returns: -/// - an optional `ProviderConnectionGuard` to manage the connection's lifecycle, -/// - a `ProviderStreamState` describing how the stream state is, -/// - and optional HTTP headers to include in the request. -/// -/// This logic helps abstract the decision-making behind provider selection and stream URL resolution. -#[cfg(test)] -async fn resolve_streaming_strategy( - app_state: &Arc, - stream_url: &str, - fingerprint: &Fingerprint, - input: &ConfigInput, - options: StreamingAcquireOptions<'_>, - stream_channel: Option<&StreamChannel>, -) -> StreamingStrategy { - resolve_streaming_strategy_with_provider_handle( - app_state, - stream_url, - fingerprint, - input, - options, - stream_channel, - None, - ) - .await -} - -async fn resolve_streaming_strategy_with_provider_handle( - app_state: &Arc, - stream_url: &str, - fingerprint: &Fingerprint, - input: &ConfigInput, - options: StreamingAcquireOptions<'_>, - stream_channel: Option<&StreamChannel>, - preacquired_provider_handle: Option, -) -> StreamingStrategy { - // Recording requests transfer the slot acquired by the worker into this - // provider-body owner. Normal playback allocates here as before. - let mut provider_connection_handle = match preacquired_provider_handle { - Some(handle) => Some(handle), - None => acquire_stream_provider_handle(app_state, input, fingerprint, &options).await, - }; - - // panel_api provisioning/loading is handled later in the stream creation flow - - let mut release_failed_mapping = false; - let stream_response_params = if let Some(allocation) = - provider_connection_handle.as_ref().and_then(|managed| managed.handle()).map(|ph| &ph.allocation) - { - match allocation { - ProviderAllocation::Exhausted => { - debug!("Provider {} is exhausted. No connections allowed.", input.name); - let stream = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); - ProviderStreamState::Custom { response: stream, reason: ProviderStreamCustomReason::ProviderExhausted } - } - ProviderAllocation::Available(ref provider_cfg) | ProviderAllocation::GracePeriod(ref provider_cfg) => { - // If a forced/pinned provider was requested but allocation fell back to another account, - // the session's stream_url still points to the old provider and cannot be accepted as-is; - // it must be resolved or rewritten for the newly allocated provider account. - let accept_requested_stream_url = (options.accept_requested_stream_url - || input.input_type.is_stalker()) - && options.force_provider.is_none_or(|forced| forced.as_ref() == provider_cfg.name.as_ref()); - // Keep the URL only when it already targets the selected provider account. Hot reload can leave old - // alias URLs in persisted playlists until the next processing run. - let selected_stream = select_provider_stream_url( - stream_url, - input, - provider_cfg, - accept_requested_stream_url, - &app_state.app_config, - ) - .await - .or_else(|| { - let url = resolve_xtream_vod_provider_url(stream_url, input, provider_cfg, stream_channel?)?; - Some((Arc::clone(&provider_cfg.name), url)) - }); - if let Some((selected_provider_name, url)) = selected_stream { - debug_if_enabled!( - "provider session: input={} provider_cfg={} user={} allocation={} stream_url={}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&provider_cfg.name), - sanitize_sensitive_info( - provider_cfg.get_user_info().as_ref().map_or_else(|| "?", |u| u.username.as_str()) - ), - allocation.short_key(), - sanitize_sensitive_info(resolve_request_url_for_logging(input, &url).as_ref()) - ); - - if matches!(allocation, ProviderAllocation::Available(_)) { - ProviderStreamState::Available(Some(selected_provider_name.intern()), url.intern()) - } else { - ProviderStreamState::GracePeriod(Some(selected_provider_name.intern()), url.intern()) - } - } else { - debug_if_enabled!( - "provider session rejected: input={} provider_cfg={} allocation={} stream_url={} reason=unmapped_provider_url", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&provider_cfg.name), - allocation.short_key(), - sanitize_sensitive_info(resolve_request_url_for_logging(input, stream_url).as_ref()) - ); - release_failed_mapping = true; - create_unmapped_provider_stream(&app_state.app_config) - } - } - } - } else { - debug!("Provider {} is exhausted. No connections allowed.", input.name); - let stream = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); - ProviderStreamState::Custom { response: stream, reason: ProviderStreamCustomReason::ProviderExhausted } - }; - - if release_failed_mapping { - // The managed owner releases the provider slot synchronously on drop; no - // task-per-drop and no lossy cleanup message. - drop(provider_connection_handle.take()); - } - - StreamingStrategy { - provider_handle: provider_connection_handle, - provider_stream_state: stream_response_params, - input_headers: Some(input.headers.clone()), - } -} - -fn get_grace_period_millis( - connection_permission: UserConnectionPermission, - stream_response_params: &ProviderStreamState, - config_grace_period_millis: u64, -) -> u64 { - if config_grace_period_millis > 0 - && ( - matches!(stream_response_params, ProviderStreamState::GracePeriod(_, _)) // provider grace period - || connection_permission == UserConnectionPermission::GracePeriod - // user grace period - ) - { - config_grace_period_millis - } else { - 0 - } -} - -fn should_defer_provider_open_for_grace_hold( - provider_grace_active: bool, - hold_stream: bool, - item_type: PlaylistItemType, - is_reopen: bool, -) -> bool { - if !(provider_grace_active && hold_stream) { - return false; - } - - // Catch-up must open immediately so its payload can be classified before response headers are committed. - if item_type == PlaylistItemType::Catchup { - return false; - } - - // v3.3.0 opened provider-affine VOD/Series reopens immediately, even when - // provider grace was temporarily in effect. Parking these requests in GracePending - // was introduced later and breaks players like libmpv during seek/reopen retries. - // Keep hold-stream behavior for live/admission paths, but restore direct-open behavior - // for provider-affine on-demand session reopens. - !(!item_type.is_live() && item_type.requires_provider_affinity() && is_reopen) -} - -fn should_refresh_stalker_playback(input_type: InputType, request_url_valid: bool, status: Option) -> bool { - input_type.is_stalker() && (!request_url_valid || status.is_some_and(|status| status.is_client_error())) -} - -fn needs_initial_stalker_resolution(input_type: InputType, stream_url: &str) -> bool { - input_type.is_stalker() && stream_url.is_empty() -} - -fn stalker_stream_kind(cluster: XtreamCluster, item_type: PlaylistItemType) -> StalkerStreamKind { - if item_type == PlaylistItemType::Catchup { - StalkerStreamKind::Archive - } else { - match cluster { - XtreamCluster::Live => StalkerStreamKind::Live, - XtreamCluster::Video => StalkerStreamKind::Movie, - XtreamCluster::Series => StalkerStreamKind::Episode, - } - } -} - -async fn re_resolve_stalker_url_singleflight( - app_state: &Arc, - input: &ConfigInput, - provider_id: u32, - kind: StalkerStreamKind, - force_refresh: bool, -) -> Result>, TuliproxError> { - let entry_lock = app_state.stalker_resolve_coordinator.guard_for(input.id, provider_id).await; - let _flight = entry_lock.lock().await; - let client = app_state.http_clients.default.load().as_ref().clone(); - re_resolve_stalker_url(&app_state.app_config, &client, input, provider_id, kind, force_refresh).await -} - -pub(crate) async fn resolve_initial_stalker_playback_url( - app_state: &Arc, - input: &ConfigInput, - provider_id: u32, - cluster: XtreamCluster, - item_type: PlaylistItemType, - stream_url: &Arc, -) -> Result, TuliproxError> { - if !needs_initial_stalker_resolution(input.input_type, stream_url) { - return Ok(Arc::clone(stream_url)); - } - re_resolve_stalker_url_singleflight(app_state, input, provider_id, stalker_stream_kind(cluster, item_type), false) - .await? - .ok_or_else(|| { - TuliproxError::RepositoryStalker(format!( - "Stalker playback URL could not be resolved for input '{}' and provider id {provider_id}", - input.name - )) - }) -} - -pub(crate) async fn resolve_stream_user_agent_index( - app_state: &AppState, - input: &ConfigInput, - has_provider_handle: bool, - username: &str, - session_owner: Option<&str>, -) -> Option { - if input.has_flag(ConfigInputFlags::UserAgentStreamIndex) && has_provider_handle { - if let Some(session_token) = session_owner { - app_state - .active_users - .get_or_assign_user_agent_stream_index(username, session_token) - .await - .or_else(|| Some(app_state.active_users.next_user_agent_stream_index())) - } else { - Some(app_state.active_users.next_user_agent_stream_index()) - } - } else { - None - } -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines, clippy::fn_params_excessive_bools)] -async fn create_stream_response_details( - app_state: &Arc, - stream_options: &StreamOptions, - stream_url: &str, - username: &str, - fingerprint: &Fingerprint, - req_headers: &HeaderMap, - input: &Arc, - stream_channel: &StreamChannel, - item_type: PlaylistItemType, - content_representation: crate::api::model::ProviderContentRepresentationMode, - share_stream: bool, - connection_permission: UserConnectionPermission, - force_provider: Option<&Arc>, - allow_forced_provider_fallback: bool, - allow_provider_grace: bool, - virtual_id: VirtualId, - user_priority: i8, - connection_kind: crate::api::model::ConnectionKind, - is_reopen: bool, - session_owner: Option<&str>, - session_headers: Option<&HashMap>, - accept_requested_stream_url: bool, - grace_hold_override: Option, - grace_resolution_context: Option, - current_session: Option>, - preacquired_provider_handle: Option, -) -> Result { - let mut streaming_strategy = resolve_streaming_strategy_with_provider_handle( - app_state, - stream_url, - fingerprint, - input, - StreamingAcquireOptions { - force_provider, - allow_forced_provider_fallback, - allow_provider_grace, - user_priority, - connection_kind, - session_owner, - playback_kind: PlaybackKind::classify(item_type, extract_extension_from_url(stream_url)), - accept_requested_stream_url, - capacity_wait_timeout: (stream_options.response_mode == StreamResponseMode::HlsResource) - .then_some(HLS_MEDIA_CAPACITY_WAIT), - }, - Some(stream_channel), - preacquired_provider_handle, - ) - .await; - // A capacity wait can outlive the session snapshot: revalidate the account binding and - // read the cookies valid now, before any upstream request is sent. - let current_session_headers; - let session_headers = match current_session { - Some(guard) => { - match app_state - .active_users - .current_session_provider_headers(guard.username, guard.token, guard.identity, stream_url) - .await - { - SessionProviderHeaders::NoSession => { - app_state - .connection_manager - .release_managed_provider_handle(streaming_strategy.provider_handle.take()); - return Err(TuliproxError::Errors("playback session ended or switched provider account")); - } - SessionProviderHeaders::NoHeaders => current_session_headers = None, - SessionProviderHeaders::Headers(headers) => current_session_headers = Some(headers), - } - current_session_headers.as_ref() - } - None => session_headers, - }; - let user_agent_stream_index = resolve_stream_user_agent_index( - app_state, - input, - streaming_strategy.provider_handle.is_some(), - username, - session_owner, - ) - .await; - let mut grace_period_options = app_state.get_grace_options(); - grace_period_options.period_millis = get_grace_period_millis( - connection_permission, - &streaming_strategy.provider_stream_state, - grace_period_options.period_millis, - ); - if let Some(hold) = grace_hold_override { - grace_period_options.hold_stream = hold; - } - let provider_grace_active = - matches!(streaming_strategy.provider_stream_state, ProviderStreamState::GracePeriod(_, _)); - - let guard_provider_name = streaming_strategy - .provider_handle - .as_ref() - .and_then(|managed| managed.handle()) - .and_then(|handle| handle.allocation.get_provider_name()); - - if matches!( - streaming_strategy.provider_stream_state, - ProviderStreamState::Custom { reason: ProviderStreamCustomReason::ProviderExhausted, .. } - ) && can_provision_on_exhausted(app_state, input) - { - if let Some(managed) = streaming_strategy.provider_handle.take() { - drop(managed); - } - debug_if_enabled!( - "panel_api: provider connections exhausted; sending provisioning stream for input {}", - sanitize_sensitive_info(&input.name) - ); - let mut details = create_panel_api_provisioning_stream_details( - app_state, - input, - guard_provider_name.clone().or_else(|| Some(input.name.clone())), - &grace_period_options, - fingerprint.addr, - virtual_id, - ); - details.content_representation = content_representation; - return Ok(details); - } - - match streaming_strategy.provider_stream_state { - // custom stream means we display our own stream like connection exhausted, channel-unavailable... - ProviderStreamState::Custom { response: provider_stream, reason } => { - let (stream, stream_info) = provider_stream; - // When allocation is exhausted or no connection was acquired, guard_provider_name is None. - // Use input.name as fallback so the provider field is never empty. - let provider_name = guard_provider_name.clone().unwrap_or_else(|| input.name.clone()); - Ok(StreamDetails { - shared_subscriber_id: None, - stream, - stream_info, - provider_name: Some(provider_name), - request_url: None, - session_headers: session_headers.cloned(), - provider_session_headers: ProviderSessionHeaders::default(), - user_agent_stream_index, - grace_period: grace_period_options, - provider_grace_active: false, - disable_provider_grace: false, - reconnect_flag: None, - provider_handle: streaming_strategy.provider_handle.take(), - content_representation, - grace_resolution_context, - custom_reason: Some(reason), - response_mode: stream_options.response_mode, - session_registration: None, - }) - } - ProviderStreamState::Available(_provider_name, request_url) - | ProviderStreamState::GracePeriod(_provider_name, request_url) => { - let mut request_url = request_url; - debug_if_enabled!( - "Provider stream selection: allocated_provider={} actual_request_url={}", - sanitize_sensitive_info(guard_provider_name.as_deref().unwrap_or("?")), - sanitize_sensitive_info(resolve_request_url_for_logging(input, request_url.as_ref()).as_ref()) - ); - let defer_provider_stream_until_grace_check = if should_defer_provider_open_for_grace_hold( - provider_grace_active, - grace_period_options.hold_stream, - item_type, - is_reopen, - ) { - if let Some(provider_name) = guard_provider_name.as_ref() { - app_state.active_provider.is_over_limit(provider_name) - } else { - false - } - } else { - false - }; - let is_fallback_provider = force_provider - .is_some_and(|forced| guard_provider_name.as_ref().is_some_and(|allocated| allocated != forced)); - let session_headers = if is_fallback_provider { None } else { session_headers }; - let (stream, stream_info, provider_session_headers, reconnect_flag) = - if defer_provider_stream_until_grace_check { - debug_if_enabled!( - "Deferring provider stream open until grace check completes for {}", - sanitize_sensitive_info(resolve_request_url_for_logging(input, request_url.as_ref()).as_ref()) - ); - (None, None, ProviderSessionHeaders::default(), None) - } else if is_media_server_stream_ref_url(request_url.as_ref()) { - match open_media_server_stream_for_input(app_state, input, request_url.as_ref(), req_headers).await - { - Ok((stream, stream_info)) => { - (Some(stream), stream_info, ProviderSessionHeaders::default(), None) - } - Err(err) => { - error!("Can't open media-server stream: {err}"); - (None, None, ProviderSessionHeaders::default(), None) - } - } - } else { - let parsed_url = Url::parse(&request_url); - let request_url_valid = parsed_url.is_ok(); - let ((mut stream, mut stream_info, mut provider_session_headers), mut reconnect_flag) = - if let Ok(url) = parsed_url { - let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); - let disabled_headers = app_state.get_disabled_headers(); - let mut provider_stream_factory_options = - ProviderStreamFactoryOptions::new(&crate::api::model::ProviderStreamFactoryParams { - addr: fingerprint.addr, - item_type, - share_stream, - stream_options, - stream_url: &url, - req_headers, - input_headers: streaming_strategy.input_headers.as_ref(), - session_headers, - disabled_headers: disabled_headers.as_ref(), - default_user_agent: default_user_agent.as_deref(), - username: Some(username), - client_ip: Some(&fingerprint.client_ip), - stream_channel: Some(stream_channel), - connect_failure_stage: Some(FailureStage::ProviderOpen), - content_representation, - }); - - if let Some(stream_index) = user_agent_stream_index { - provider_stream_factory_options.apply_user_agent_stream_index(stream_index); - } - - let provider_config = input.get_resolve_provider(url.as_ref()); - provider_stream_factory_options.set_provider(provider_config); - provider_stream_factory_options.apply_input_options(input); - if input.input_type.is_stalker() { - provider_stream_factory_options.require_public_destination(); - } - - let reconnect_flag = provider_stream_factory_options.get_reconnect_flag_clone(); - let lifecycle = streaming_strategy - .provider_handle - .as_ref() - .and_then(ProviderStreamOpenLifecycle::from_managed); - let provider_stream = match open_provider_stream_with_lifecycle( - &app_state.provider_stream_ctx(), - &app_state.http_clients.default.load(), - provider_stream_factory_options, - lifecycle, - ) - .await - { - None => (None, None, ProviderSessionHeaders::default()), - Some(open) => split_provider_stream_open(open), - }; - (provider_stream, Some(reconnect_flag)) - } else { - ((None, None, ProviderSessionHeaders::default()), None) - }; - let should_refresh_stalker = should_refresh_stalker_playback( - input.input_type, - request_url_valid, - stream_info.as_ref().map(|(_, status, _, _)| *status), - ); - if should_refresh_stalker { - let force_stalker_refresh = - stream_info.as_ref().is_some_and(|(_, status, _, _)| status.is_client_error()); - let kind = stalker_stream_kind(stream_channel.cluster, item_type); - let resolve_result = re_resolve_stalker_url_singleflight( - app_state, - input, - stream_channel.provider_id, - kind, - force_stalker_refresh, - ) - .await; - match resolve_result { - Ok(Some(refreshed_url)) => { - if let Ok(url) = Url::parse(&refreshed_url) { - let default_user_agent = - app_state.app_config.config.load().default_user_agent.clone(); - let disabled_headers = app_state.get_disabled_headers(); - let mut options = ProviderStreamFactoryOptions::new( - &crate::api::model::ProviderStreamFactoryParams { - addr: fingerprint.addr, - item_type, - share_stream, - stream_options, - stream_url: &url, - req_headers, - input_headers: streaming_strategy.input_headers.as_ref(), - session_headers, - disabled_headers: disabled_headers.as_ref(), - default_user_agent: default_user_agent.as_deref(), - username: Some(username), - client_ip: Some(&fingerprint.client_ip), - stream_channel: Some(stream_channel), - connect_failure_stage: Some(FailureStage::ProviderOpen), - content_representation, - }, - ); - if let Some(stream_index) = user_agent_stream_index { - options.apply_user_agent_stream_index(stream_index); - } - options.set_provider(input.get_resolve_provider(url.as_ref())); - options.apply_input_options(input); - options.require_public_destination(); - if let Some(m) = streaming_strategy.provider_handle.as_mut() { - let _ = m.renew_opening_tokens(); - } - let retry_reconnect_flag = options.get_reconnect_flag_clone(); - let retry_lifecycle = streaming_strategy - .provider_handle - .as_ref() - .and_then(ProviderStreamOpenLifecycle::from_managed); - let retried = open_provider_stream_with_lifecycle( - &app_state.provider_stream_ctx(), - &app_state.http_clients.default.load(), - options, - retry_lifecycle, - ) - .await; - if let Some(open) = retried { - (stream, stream_info, provider_session_headers) = - split_provider_stream_open(open); - reconnect_flag = Some(retry_reconnect_flag); - request_url = refreshed_url; - } else { - // Keep the original stream/stream_info: the upstream response - // might still be serveable, and its status is needed for reporting. - debug!("Stalker re-resolve retry could not open a stream, keeping original provider response"); - } - } - } - Ok(None) => {} - Err(err) => { - warn!( - "Failed to refresh Stalker playback URL: {}", - sanitize_sensitive_info(&err.to_string()) - ); - } - } - } - (stream, stream_info, provider_session_headers, reconnect_flag) - }; - - if is_fallback_provider && stream.is_some() { - if let Some(token) = session_owner { - let _ = - app_state.active_users.update_session_provider_headers(username, token, &HashMap::new()).await; - } - } - - if log_enabled!(log::Level::Debug) { - if let Some((headers, status_code, response_url, _custom_video_type)) = stream_info.as_ref() { - debug!( - "Responding stream request {} with status {}, headers {:?}", - sanitize_sensitive_info(response_url.as_ref().map_or(stream_url, |s| s.as_str())), - status_code, - headers - ); - } - } - - // An intentional deferred open must retain its grace allocation until body polling - // resumes the provider request. Other failed opens release their allocation here. - let failed_open = stream.is_none() - || matches!(stream_info.as_ref(), Some((_, _, _, Some(custom))) if *custom != CustomVideoStreamType::Provisioning); - let provider_handle = if failed_open && !defer_provider_stream_until_grace_check { - // The managed owner releases the provider slot synchronously on drop; - // a failed open must not rely on a lossy cleanup message. - drop(streaming_strategy.provider_handle.take()); - match stream_info.as_ref() { - // Finite HLS resources hand routine upstream errors (e.g. live-edge 404) to the client. - Some((_, status, _, None)) if stream.is_none() && !status.is_success() => { - debug!("Provider answered {status} for {}", sanitize_sensitive_info(&request_url)); - } - _ => error!("Can't open stream {}", sanitize_sensitive_info(&request_url)), - } - None - } else { - streaming_strategy.provider_handle.take() - }; - - Ok(StreamDetails { - shared_subscriber_id: None, - stream, - stream_info, - provider_name: guard_provider_name.clone(), - request_url: Some(request_url.clone()), - session_headers: session_headers.cloned(), - provider_session_headers, - user_agent_stream_index, - grace_period: grace_period_options, - provider_grace_active, - disable_provider_grace: false, - reconnect_flag, - provider_handle, - content_representation, - grace_resolution_context, - custom_reason: None, - response_mode: stream_options.response_mode, - session_registration: None, - }) - } - } -} - pub struct RedirectParams<'a, P> where P: PlaylistEntry, @@ -2183,2322 +212,10 @@ where pub action_path: &'a str, } -impl

RedirectParams<'_, P> -where - P: PlaylistEntry, -{ - pub fn get_query_path(&self, provider_id: u32, url: &str) -> String { - let extension = self.stream_ext.map_or_else( - || extract_extension_from_url(url).map_or_else(String::new, ToString::to_string), - ToString::to_string, - ); - - // if there is an action_path (like for timeshift duration/start), it will be added in front of the stream_id - if self.action_path.is_empty() { - concat_string!(&provider_id.to_string(), &extension) - } else { - concat_string!(&trim_slash(self.action_path), "/", &provider_id.to_string(), &extension) - } - } -} - -pub async fn redirect_response<'a, P>( - app_state: &Arc, - params: &'a RedirectParams<'a, P>, -) -> Option -where - P: PlaylistEntry, -{ - let item_type = params.item.get_item_type(); - let provider_url = params.item.get_provider_url(); - if is_media_server_playback_url(params.input, provider_url.as_ref()) { - return None; - } - - let redirect_request = params.user.proxy.is_redirect(item_type) || params.target.is_force_redirect(item_type); - let is_hls_request = item_type == PlaylistItemType::LiveHls || params.stream_ext == Some(HLS_EXT); - let is_dash_request = - (!is_hls_request && item_type == PlaylistItemType::LiveDash) || params.stream_ext == Some(DASH_EXT); - - // Recording playback keeps provider headers and proxy handling on this listener. - // DASH redirects because relative MPD resource URLs are not rewritten. - if params.user.is_recording_proxy_user() && !is_dash_request { - return None; - } - - if params.target_type == TargetType::M3u { - if redirect_request || is_dash_request { - let redirect_url: Arc = if is_hls_request { - replace_url_extension(&provider_url, HLS_EXT).into() - } else { - provider_url.clone() - }; - let redirect_url = - if is_dash_request { replace_url_extension(&redirect_url, DASH_EXT).into() } else { redirect_url }; - let redirect_url = get_redirect_alternative_url(app_state, &redirect_url, params.input); - let redirect_url = match resolve_redirect_location(Some(params.input), &redirect_url) { - Ok(url) => url, - Err(err) => { - error!("Failed to resolve redirect url: {}", sanitize_sensitive_info(&err.to_string())); - return Some(StatusCode::BAD_REQUEST.into_response()); - } - }; - debug_if_enabled!("Redirecting stream request to {}", sanitize_sensitive_info(redirect_url.as_ref())); - return Some(redirect(redirect_url.as_ref()).into_response()); - } - } else if params.target_type == TargetType::Xtream { - let Some(provider_id) = params.provider_id else { - return Some(StatusCode::BAD_REQUEST.into_response()); - }; - - if redirect_request || is_dash_request { - let target_name = params.target.name.as_str(); - let virtual_id = params.item.get_virtual_id(); - let stream_url = match get_xtream_player_api_stream_url( - params.input, - params.req_context, - ¶ms.get_query_path(provider_id, &provider_url), - &provider_url, - ) { - None => { - error!( - "Can't find stream url for target {target_name}, context {}, stream_id {virtual_id}", - params.req_context - ); - return Some(StatusCode::BAD_REQUEST.into_response()); - } - Some(url) => match app_state.active_provider.get_next_provider(¶ms.input.name) { - Some(provider_cfg) => match get_stream_alternative_url(&url, params.input, &provider_cfg) { - Some(stream_url) => stream_url, - None => return Some(StatusCode::BAD_REQUEST.into_response()), - }, - None => url.to_string(), - }, - }; - let stream_url = match resolve_redirect_location(Some(params.input), &stream_url) { - Ok(url) => url, - Err(err) => { - error!("Failed to resolve redirect url: {}", sanitize_sensitive_info(&err.to_string())); - return Some(StatusCode::BAD_REQUEST.into_response()); - } - }; - - // hls or dash redirect - if is_dash_request { - let redirect_url = if is_hls_request { - &replace_url_extension(&stream_url, HLS_EXT) - } else { - &replace_url_extension(&stream_url, DASH_EXT) - }; - debug_if_enabled!( - "Redirecting stream request to {}", - sanitize_sensitive_info(resolve_request_url_for_logging(params.input, redirect_url).as_ref()) - ); - return Some(redirect(redirect_url).into_response()); - } - - debug_if_enabled!( - "Redirecting stream request to {}", - sanitize_sensitive_info(resolve_request_url_for_logging(params.input, stream_url.as_ref()).as_ref()) - ); - return Some(redirect(stream_url.as_ref()).into_response()); - } - } - - None -} - -fn is_media_server_playback_url(input: &ConfigInput, stream_url: &str) -> bool { - input.input_type == InputType::Plex || is_media_server_stream_ref_url(stream_url) -} - -fn is_media_server_stream_ref_url(stream_url: &str) -> bool { - Url::parse(stream_url).is_ok_and(|url| url.scheme() == "media-server") -} - -fn is_throttled_stream(item_type: PlaylistItemType, throttle_kbps: usize) -> bool { - throttle_kbps > 0 - && matches!( - item_type, - PlaylistItemType::Video - | PlaylistItemType::Series - | PlaylistItemType::SeriesInfo - | PlaylistItemType::Catchup - | PlaylistItemType::LocalVideo - | PlaylistItemType::LocalSeries - | PlaylistItemType::LocalSeriesInfo - ) -} - -fn prepare_body_stream(app_state: &Arc, item_type: PlaylistItemType, stream: S) -> axum::body::Body -where - S: futures::Stream> + Send + 'static, -{ - let throttle_kbps = usize::try_from(get_stream_throttle(app_state)).unwrap_or_default(); - let body_stream = if is_throttled_stream(item_type, throttle_kbps) { - info!("Stream throttling active: {}", human_readable_kbps(u64::try_from(throttle_kbps).unwrap_or_default())); - axum::body::Body::from_stream(ThrottledStream::new(stream.boxed(), throttle_kbps)) - } else { - axum::body::Body::from_stream(stream) - }; - body_stream -} - -async fn open_media_server_stream_for_input( - app_state: &Arc, - input: &ConfigInput, - stream_url: &str, - req_headers: &HeaderMap, -) -> Result<(BoxedProviderStream, ProviderStreamInfo), MediaServerError> { - let stream_ref = parse_media_server_stream_ref(&input.name, stream_url)?; - let range = req_headers.get(header::RANGE).and_then(|value| value.to_str().ok()); - let http_client = MediaServerHttpClient::new(app_state.http_clients.default.load().as_ref().clone()); - - let response = match input.input_type { - InputType::Plex => { - let client = input.plex_catalog_client(http_client)?; - open_media_server_proxy_stream_response(&client, &stream_ref, range).await? - } - InputType::Emby | InputType::Jellyfin => { - return Err(MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) - .provider("media-server") - .detail("media-server playback proxy is not implemented for this input type")); - } - InputType::M3u - | InputType::Xtream - | InputType::M3uBatch - | InputType::XtreamBatch - | InputType::Stalker - | InputType::StalkerBatch - | InputType::Library - | InputType::Staged => { - return Err(MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) - .provider("media-server") - .detail("playlist item is not backed by a media-server input")); - } - }; - - let headers = response - .headers - .iter() - .filter(|(key, _)| !is_hop_by_hop_response_header(key)) - .filter_map(|(key, value)| value.to_str().ok().map(|value| (key.to_string(), value.to_string()))) - .collect::>(); - let status = response.status; - let stream = response.body.map_err(|err| StreamError::Stream(err.to_string())).boxed(); - Ok((stream, Some((headers, status, None, None)))) -} - -fn is_hop_by_hop_response_header(name: &HeaderName) -> bool { - matches!( - name.as_str(), - "connection" - | "keep-alive" - | "proxy-authenticate" - | "proxy-authorization" - | "te" - | "trailer" - | "trailers" - | "transfer-encoding" - | "upgrade" - ) -} - -fn no_custom_video_fallback_status(app_config: &AppConfig) -> StatusCode { - // Two reasons we have no custom-video response: - // 1. Operator disabled `custom_stream_response_enabled` → return the - // configured fallback status (e.g. 502) so reverse proxies handle the - // socket consistently. - // 2. Operator enabled custom-video but the concrete resource is missing - // → return `400` so downstream `proxy_intercept_errors on;` (Nginx) - // can sever the socket instead of looping on `200 OK`. - // Collapsing both into the configured status code broke the Nginx-intercept - // contract that the operator relied on by enabling custom-video in the - // first place. - if is_custom_video_stream_enabled(app_config) { - StatusCode::BAD_REQUEST - } else { - get_custom_stream_response_error_status(app_config) - } -} - -pub async fn force_provider_stream_response( - fingerprint: &Fingerprint, - app_state: &Arc, - user_session: &UserSession, - stream_channel: StreamChannel, - ctx: ForceStreamRequestContext<'_>, - grace_mode: Option, -) -> axum::response::Response { - force_stream_response( - fingerprint, - app_state, - user_session, - stream_channel, - ctx, - grace_mode, - StreamResponseMode::Stream, - ) - .await -} - -pub(crate) async fn force_hls_resource_response( - fingerprint: &Fingerprint, - app_state: &Arc, - user_session: &UserSession, - stream_channel: StreamChannel, - mut ctx: ForceStreamRequestContext<'_>, - grace_mode: Option, -) -> axum::response::Response { - // `resolve_stream_channel` classified the item with `hls_playback_kind`, like the manifest paths. - let kind = if stream_channel.item_type == PlaylistItemType::Catchup { - PlaybackKind::Catchup - } else { - PlaybackKind::LiveHls - }; - ctx.session_reservation_ttl_secs = get_hls_playback_ttl_secs(app_state, kind); - force_stream_response( - fingerprint, - app_state, - user_session, - stream_channel, - ctx, - grace_mode, - StreamResponseMode::HlsResource, - ) - .await -} - -/// Status a finite HLS resource answers with instead of a fallback body, or `None` on success. -fn hls_resource_failure_status(stream_details: &StreamDetails) -> Option { - match stream_details.stream_info.as_ref() { - Some((_, status, _, _)) if !status.is_success() => Some(*status), - Some((_, _, _, Some(_))) => Some(StatusCode::SERVICE_UNAVAILABLE), - _ if stream_details.custom_reason.is_some() => Some(StatusCode::SERVICE_UNAVAILABLE), - _ if !stream_details.has_stream() => Some(StatusCode::BAD_GATEWAY), - _ => None, - } -} - -/// Answers a failed finite HLS resource with its status and the forwarded upstream headers. -/// Its provider slot is freed and an entry reservation no body ever used is released; a session -/// that streams elsewhere keeps its reservation (see `release_unbound_session_reservation`). -async fn reject_hls_resource( - app_state: &Arc, - username: &str, - session_token: &str, - stream_details: StreamDetails, - status: StatusCode, -) -> axum::response::Response { - let StreamDetails { provider_handle, stream_info, .. } = stream_details; - app_state.connection_manager.release_managed_provider_handle(provider_handle); - app_state.active_users.release_unbound_session_reservation(username, session_token, None, false).await; - let mut response = status.into_response(); - if let Some((headers, _, _, None)) = stream_info { - for (name, value) in headers { - if let (Ok(name), Ok(value)) = (HeaderName::try_from(name), HeaderValue::try_from(value)) { - response.headers_mut().insert(name, value); - } - } - } - response -} - -#[allow(clippy::too_many_lines)] -async fn force_stream_response( - fingerprint: &Fingerprint, - app_state: &Arc, - user_session: &UserSession, - mut stream_channel: StreamChannel, - ctx: ForceStreamRequestContext<'_>, - grace_mode: Option, - response_mode: StreamResponseMode, -) -> axum::response::Response { - let hls_resource = response_mode == StreamResponseMode::HlsResource; - // Finite HLS resources stay pinned to the session account and never rebind or clean up - // other sockets, so they skip the session transition gate. Holding it across the - // capacity wait would serialize parallel audio and video rendition requests. - let _transition_guard = if hls_resource { - None - } else { - Some(app_state.active_users.acquire_playback_transition(&ctx.user.username, &user_session.token).await) - }; - let stream_options = get_stream_options(&app_state.app_config, response_mode); - let share_stream = false; - let connection_permission = UserConnectionPermission::Allowed; - let item_type = stream_channel.item_type; - - // Forced reopens must clear stale provider slots before reacquiring. For adaptive HLS/DASH - // and Catchup sessions we only target old active stream sockets of the same session, never - // manifest-only session addresses, otherwise the controlling playlist request gets torn down. - let cleanup_addrs = if hls_resource { - Vec::new() - } else if item_type.is_live_adaptive() || item_type == PlaylistItemType::Catchup { - app_state - .active_users - .adaptive_session_stream_cleanup_addrs(&ctx.user.username, &user_session.token, &fingerprint.addr) - .await - } else { - session_reacquire_cleanup_addrs(user_session, &fingerprint.addr) - }; - - if cleanup_addrs.is_empty() { - debug_if_enabled!( - "Forced reopen cleanup had no stale targets for item_type={item_type:?} session={} current_addr={}", - sanitize_sensitive_info(&user_session.token), - sanitize_sensitive_info(&fingerprint.addr.to_string()) - ); - } else { - debug_if_enabled!( - "Forced reopen cleanup releasing {} stale target(s) for item_type={item_type:?} session={} current_addr={}", - cleanup_addrs.len(), - sanitize_sensitive_info(&user_session.token), - sanitize_sensitive_info(&fingerprint.addr.to_string()) - ); - cleanup_forced_reopen_addrs(app_state, &user_session.token, &cleanup_addrs).await; - } - - // HLS child URLs remain bound to the manifest account before the first media byte. - // Other streams prefer the account after media starts or while its allocation is active, - // and can fall back to the lineup when that account is exhausted. - let preferred_provider = (hls_resource - || item_type.is_live() - || user_session.media_started.load(std::sync::atomic::Ordering::Acquire) - || app_state.active_provider.should_reuse_playback_provider(&user_session.token, &user_session.provider)) - .then_some(&user_session.provider); - // Child URLs and their signed tokens belong to the account that fetched the playlist. - let allow_forced_provider_fallback = - !hls_resource && (!item_type.requires_provider_affinity() || allows_provider_pool_failover(item_type)); - // Never allow provider-side grace for forced seek/session reacquire. - // Over-allocation here would break provider-side one-connection limits. - let allow_provider_grace = false; - let connection_kind = user_session.connection_kind.unwrap_or(crate::api::model::ConnectionKind::Normal); - - let session_identity = user_session.identity(); - let current_session = hls_resource.then_some(CurrentSessionGuard { - username: &ctx.user.username, - token: &user_session.token, - identity: session_identity, - }); - // Finite resources read their cookies after the capacity wait instead of from this snapshot. - let provider_session_headers = preferred_provider - .filter(|_| !hls_resource) - .and_then(|_| user_session.provider_session_headers_for(&user_session.stream_url)); - let create_details = create_stream_response_details( - app_state, - &stream_options, - &user_session.stream_url, - &ctx.user.username, - fingerprint, - ctx.req_headers, - ctx.input, - &stream_channel, - item_type, - ctx.content_representation, - share_stream, - connection_permission, - preferred_provider, - allow_forced_provider_fallback, - allow_provider_grace, - VirtualId::new(stream_channel.virtual_id), - connection_priority_for_kind(ctx.user, connection_kind), - connection_kind, - true, - Some(user_session.token.as_str()), - provider_session_headers.as_deref(), - hls_resource || preferred_provider.is_some(), - grace_mode.map(|mode| matches!(mode, crate::api::model::GraceMode::Hold)), - None, - current_session, - None, - ); - // The FORBIDDEN exits below leave the session reservation alone: they only fire once this - // session identity is gone, and after an account switch the reservation belongs to the new - // binding. Dropping the cancelled future frees any provider slot it held. - let details_result = if hls_resource { - tokio::select! { - result = create_details => result, - () = app_state.active_users.wait_for_playback_session_end( - &ctx.user.username, &user_session.token, session_identity, - ) => return StatusCode::FORBIDDEN.into_response(), - } - } else { - create_details.await - }; - let mut stream_details = match details_result { - Ok(stream_details) => stream_details, - Err(err) => { - if hls_resource - && !app_state - .active_users - .playback_session_is_current(&ctx.user.username, &user_session.token, session_identity) - .await - { - return StatusCode::FORBIDDEN.into_response(); - } - app_state - .active_users - .release_unbound_session_reservation(&ctx.user.username, &user_session.token, None, false) - .await; - error!("Failed to stream: {err}"); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - }; - - if hls_resource { - if !app_state - .active_users - .playback_session_is_current(&ctx.user.username, &user_session.token, session_identity) - .await - { - return StatusCode::FORBIDDEN.into_response(); - } - stream_details.session_registration = Some(tuliprox_session::PlaybackSessionRegistration { - identity: session_identity, - enforce_limits: app_state.app_config.config.load().user_access_control, - grace_admitted: user_session.permission == UserConnectionPermission::GracePeriod, - }); - if let Some(status) = hls_resource_failure_status(&stream_details) { - return reject_hls_resource(app_state, &ctx.user.username, &user_session.token, stream_details, status) - .await; - } - } - - let deferred_grace_hold_stream = stream_details.has_deferred_provider_open(); - - if stream_details.has_stream() || deferred_grace_hold_stream { - let selected_provider = stream_details.provider_name.clone(); - let selected_request_url = stream_details.request_url.clone(); - let selected_provider_headers = stream_details.provider_session_headers.clone(); - let metering = prepare_stream_metering( - app_state, - user_session.stream_url.as_ref(), - false, - stream_details.stream.is_some(), - stream_details.has_deferred_provider_open(), - ); - let provider_response = - stream_details.stream_info.as_ref().map(|(h, sc, url, cvt)| (h.clone(), *sc, url.clone(), *cvt)); - if ctx.session_reservation_ttl_secs > 0 { - if let Some(provider_name) = stream_details.provider_name.as_ref() { - app_state.active_provider.refresh_adaptive_playback_lease( - provider_name, - &user_session.token, - PlaybackKind::classify(item_type, extract_extension_from_url(user_session.stream_url.as_ref())), - ctx.session_reservation_ttl_secs, - ); - } - } - if let Some(allocated_provider) = stream_details.provider_name.as_ref() { - if allocated_provider.as_ref() != user_session.provider.as_ref() { - let new_stream_url = - stream_details.request_url.as_deref().map_or_else(|| user_session.stream_url.clone(), Arc::from); - app_state - .active_users - .update_session_provider_binding( - &ctx.user.username, - &user_session.token, - Arc::clone(allocated_provider), - new_stream_url, - ) - .await; - } - if allocated_provider.as_ref() != user_session.provider.as_ref() - || !stream_details.provider_session_headers.is_empty() - { - let cookie_origin = provider_response - .as_ref() - .and_then(|(_, _, url, _)| url.as_ref()) - .map_or(user_session.stream_url.as_ref(), Url::as_str); - let active_users = &app_state.active_users; - if hls_resource { - // Cookies from the previous account never enter a jar after an account switch. - active_users - .update_current_session_provider_response_headers_from( - &ctx.user.username, - &user_session.token, - session_identity, - &stream_details.provider_session_headers, - cookie_origin, - ) - .await; - } else { - active_users - .update_session_provider_response_headers_from( - &ctx.user.username, - &user_session.token, - &stream_details.provider_session_headers, - cookie_origin, - ) - .await; - } - } - } - app_state.active_users.update_session_addr(&ctx.user.username, &user_session.token, &fingerprint.addr).await; - stream_channel.shared = share_stream; - let socket_bound = user_session.socket_bound; - let stream = match create_active_client_stream(crate::api::model::ActiveClientStreamParams { - stream_details, - app_state, - user: ctx.user, - connection_permission, - connection_kind: user_session.connection_kind.unwrap_or(crate::api::model::ConnectionKind::Normal), - fingerprint, - stream_channel, - socket_bound, - session_token: Some(&user_session.token), - req_headers: ctx.req_headers, - meter_uid: metering.meter_uid, - meter_stream: metering.meter_stream, - }) - .await - { - Ok(stream) => stream, - Err(error) => { - app_state - .active_users - .release_unbound_session_reservation(&ctx.user.username, &user_session.token, None, false) - .await; - return stream_admission_rejected_response(error, &ctx.user.username); - } - }; - - if let Some(provider) = selected_provider.as_deref() { - let session_url = selected_request_url.as_deref().unwrap_or(user_session.stream_url.as_ref()); - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: ctx.user, - session_token: &user_session.token, - virtual_id: user_session.virtual_id, - provider, - stream_url: session_url, - addr: &fingerprint.addr, - connection_permission, - connection_kind: user_session.connection_kind, - socket_bound: user_session.socket_bound, - }) - .await; - if !selected_provider_headers.is_empty() { - let cookie_origin = - provider_response.as_ref().and_then(|(_, _, url, _)| url.as_ref()).map_or(session_url, Url::as_str); - app_state - .active_users - .update_session_provider_response_headers_from( - &ctx.user.username, - &user_session.token, - &selected_provider_headers, - cookie_origin, - ) - .await; - } - } - - let (status_code, header_map) = get_stream_response_with_headers(provider_response.map(|(h, s, _, _)| (h, s))); - let mut response = axum::response::Response::builder().status(status_code); - for (key, value) in &header_map { - response = response.header(key, value); - } - - let body_stream = prepare_body_stream(app_state, item_type, stream); - debug_if_enabled!( - "Streaming provider forced stream request from {}", - sanitize_sensitive_info( - resolve_request_url_for_logging(ctx.input, user_session.stream_url.as_ref()).as_ref() - ) - ); - let mut response = try_unwrap_body!(response.body(body_stream)); - mark_response_as_uncompressed(&mut response); - return response; - } - - app_state.connection_manager.release_managed_provider_handle(stream_details.provider_handle); - app_state - .active_users - .release_unbound_session_reservation(&ctx.user.username, &user_session.token, None, false) - .await; - if let (Some(stream), _stream_info) = - create_channel_unavailable_stream(&app_state.app_config, &[], StatusCode::SERVICE_UNAVAILABLE) - { - app_state - .connection_manager - .update_stream_detail(&fingerprint.addr, CustomVideoStreamType::ChannelUnavailable) - .await; - debug!("Streaming custom stream"); - let mut response = try_unwrap_body!(axum::response::Response::builder() - .status(StatusCode::OK) - .body(axum::body::Body::from_stream(stream))); - mark_response_as_uncompressed(&mut response); - response - } else { - no_custom_video_fallback_status(&app_state.app_config).into_response() - } -} - -#[allow(clippy::too_many_arguments)] -pub(crate) async fn stream_response( - fingerprint: &Fingerprint, - app_state: &Arc, - session_token: &str, - request_class: Option, - stream_channel: StreamChannel, - stream_url: &str, - pinned_provider: Option<&Arc>, - req_headers: &HeaderMap, - input: &Arc, - target: &Arc, - user: &ProxyUserCredentials, - connection_permission: UserConnectionPermission, - connection_kind: crate::api::model::ConnectionKind, - allow_exhausted_shared_reconnect: bool, - grace_mode: Option, -) -> axum::response::Response { - stream_response_with_provider_handle( - fingerprint, - app_state, - session_token, - request_class, - stream_channel, - stream_url, - pinned_provider, - req_headers, - input, - target, - user, - connection_permission, - connection_kind, - allow_exhausted_shared_reconnect, - grace_mode, - None, - ) - .await - .into_response() -} - -/// # Panics -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -pub(crate) async fn stream_response_with_provider_handle( - fingerprint: &Fingerprint, - app_state: &Arc, - session_token: &str, - request_class: Option, - mut stream_channel: StreamChannel, - stream_url: &str, - pinned_provider: Option<&Arc>, - req_headers: &HeaderMap, - input: &Arc, - target: &Arc, - user: &ProxyUserCredentials, - connection_permission: UserConnectionPermission, - connection_kind: crate::api::model::ConnectionKind, - allow_exhausted_shared_reconnect: bool, - grace_mode: Option, - preacquired_provider_handle: Option, -) -> impl IntoResponse + Send { - let _transition_guard = app_state.active_users.acquire_playback_transition(&user.username, session_token).await; - let request_log_stream_url = resolve_request_url_for_logging(input, stream_url); - if log_enabled!(log::Level::Trace) { - trace!("Try to open stream {}", sanitize_sensitive_info(request_log_stream_url.as_ref())); - } - - let virtual_id = stream_channel.virtual_id; - let item_type = stream_channel.item_type; - let playback_extension = extract_extension_from_url(stream_url); - let socket_bound = is_socket_bound_playback_session(item_type, playback_extension); - let mut connection_permission = connection_permission; - let mut connection_kind = connection_kind; - let activation = activate_session_before_stream_open( - app_state, - SessionActivationRequest { - fingerprint, - input, - user, - session_token, - request_class, - virtual_id: VirtualId::new(virtual_id), - item_type, - stream_url, - connection_permission, - connection_kind, - granted_grace_mode: grace_mode, - socket_bound, - }, - ) - .await; - let grace_mode = activation.grace_mode.or(grace_mode); - connection_permission = activation.admission.permission(); - connection_kind = activation.admission.kind().unwrap_or(connection_kind); - - let allow_shared_reuse = - connection_permission != UserConnectionPermission::Exhausted || allow_exhausted_shared_reconnect; - - let share_stream = is_stream_share_enabled(item_type, target); - let _shared_lock = if share_stream { - let write_lock = app_state.app_config.file_locks.write_lock_str(stream_url).await; - - if allow_shared_reuse { - if let Some(value) = try_shared_stream_response_if_any( - app_state, - stream_url, - fingerprint, - user, - connection_permission, - connection_kind, - stream_channel.clone(), - session_token, - req_headers, - activation.placeholder_transition_version, - ) - .await - { - return value.into_response(); - } - } - Some(write_lock) - } else { - // Opportunistic cross-target sharing: if another target already runs a shared stream - // for the same provider URL, subscribe to it instead of opening a separate connection. - if item_type == PlaylistItemType::Live && allow_shared_reuse { - if let Some(value) = try_shared_stream_response_if_any( - app_state, - stream_url, - fingerprint, - user, - connection_permission, - connection_kind, - stream_channel.clone(), - session_token, - req_headers, - activation.placeholder_transition_version, - ) - .await - { - debug_if_enabled!("Opportunistic shared stream reuse for {}", sanitize_sensitive_info(stream_url)); - return value.into_response(); - } - } - None - }; - - if connection_permission == UserConnectionPermission::Exhausted { - app_state - .active_users - .release_unbound_session_reservation( - &user.username, - session_token, - activation.placeholder_transition_version, - activation.placeholder_transition_version.is_some(), - ) - .await; - if activation.admission.is_reentry_suppressed() { - return reentry_suppressed_response(); - } - record_connect_failed_attempt(ConnectFailedAttempt { - app_state, - fingerprint, - user, - stream_channel: stream_channel.clone(), - provider_name: input.name.clone(), - req_headers, - reason: ConnectFailureReason::UserConnectionsExhausted, - failure_stage: FailureStage::Admission, - }); - return create_custom_video_stream_response( - &app_state.provider_stream_ctx(), - &fingerprint.addr, - CustomVideoStreamType::UserConnectionsExhausted, - ) - .into_response(); - } - - let stream_options = get_stream_options(&app_state.app_config, StreamResponseMode::Stream); - let session_state = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; - let pinned_provider = pinned_provider.filter(|provider| { - item_type.is_live() - || session_state - .as_ref() - .is_some_and(|session| session.media_started.load(std::sync::atomic::Ordering::Acquire)) - || app_state.active_provider.should_reuse_playback_provider(session_token, provider) - }); - let provider_session_headers = - session_state.as_ref().and_then(|session| session.provider_session_headers_for(stream_url)); - let mut stream_details = match create_stream_response_details( - app_state, - &stream_options, - stream_url, - &user.username, - fingerprint, - req_headers, - input, - &stream_channel, - item_type, - if item_type == PlaylistItemType::Catchup { - crate::api::model::ProviderContentRepresentationMode::Identity - } else { - crate::api::model::ProviderContentRepresentationMode::PreserveOrigin - }, - share_stream, - connection_permission, - pinned_provider, - pinned_provider.is_none() || allows_provider_pool_failover(item_type), - true, - VirtualId::new(stream_channel.virtual_id), - connection_priority_for_kind(user, connection_kind), - connection_kind, - false, - Some(session_token), - provider_session_headers.as_deref(), - pinned_provider.is_some(), - grace_mode.map(|m| matches!(m, crate::api::model::GraceMode::Hold)), - activation.grace_context.clone(), - None, - preacquired_provider_handle, - ) - .await - { - Ok(stream_details) => stream_details, - Err(err) => { - app_state - .active_users - .release_unbound_session_reservation( - &user.username, - session_token, - activation.placeholder_transition_version, - activation.placeholder_transition_version.is_some(), - ) - .await; - error!("Failed to stream: {err}"); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - }; - - if item_type == PlaylistItemType::Catchup { - if let Some(provider_stream) = stream_details.stream.take() { - let probe_deadline = Duration::from_millis(app_state.hls.proxy.origin_manifest_timeout_ms().max(1)); - match probe_catchup_payload(provider_stream, probe_deadline).await { - Ok(CatchupPayload::Direct(provider_stream)) => stream_details.stream = Some(provider_stream), - Ok(CatchupPayload::HlsManifest(manifest)) => { - return detected_catchup_hls_response(DetectedCatchupHlsResponseParams { - app_state, - stream_details, - manifest, - user, - target, - input, - fingerprint, - session_token, - virtual_id: VirtualId::new(virtual_id), - connection_permission, - connection_kind, - fallback_stream_url: stream_url, - }) - .await; - } - Err(err) => { - error!("Failed to inspect catch-up payload: {err}"); - cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token) - .await; - return StatusCode::BAD_GATEWAY.into_response(); - } - } - } - } - - // When no provider stream is available, still create an ActiveClientStream if a grace period - // needs to resolve (provider-grace with hold_stream, or user-grace). The grace task will - // determine the correct mode (UserExhausted / ProviderExhausted / Inner) and serve the - // appropriate custom video or terminate cleanly. - let deferred_grace_hold_stream = - stream_details.has_deferred_provider_open() || connection_permission == UserConnectionPermission::GracePeriod; - - if stream_details.has_stream() || deferred_grace_hold_stream { - // let content_length = get_stream_content_length(provider_response.as_ref()); - let provider_response = stream_details - .stream_info - .as_ref() - .map(|(h, sc, response_url, cvt)| (h.clone(), *sc, response_url.clone(), *cvt)); - let provider_name = stream_details.provider_name.clone(); - let actual_request_url = stream_details.request_url.clone().unwrap_or_else(|| Arc::::from(stream_url)); - let log_actual_request_url = resolve_request_url_for_logging(input, actual_request_url.as_ref()); - - debug_if_enabled!( - "Provider request mapping: allocated_provider={} actual_request_url={}", - sanitize_sensitive_info(provider_name.as_deref().unwrap_or("?")), - sanitize_sensitive_info(log_actual_request_url.as_ref()) - ); - - if let Some((headers, status, _response_url, Some(CustomVideoStreamType::Provisioning))) = - stream_details.stream_info.as_ref() - { - debug_if_enabled!("panel_api provisioning response to client: status={} headers={:?}", status, headers); - } - - // Captured before `stream_details` is moved into `create_active_client_stream`. - // The pinning rule is centralized in `should_pin_provider_for_session` so it stays - // testable in isolation and in sync with the call site below. - let should_pin_provider = should_pin_provider_for_session(&stream_details, app_state, item_type); - let user_agent_stream_index = stream_details.user_agent_stream_index; - // Persist this before response construction because some item types skip the later - // create_user_session path, including placeholder sessions created by ensure_user_session_placeholder. - if let Some(stream_index) = user_agent_stream_index { - app_state - .active_users - .set_user_agent_stream_index_if_absent(&user.username, session_token, stream_index) - .await; - } - - let mut is_stream_shared = share_stream && !stream_details.has_deferred_provider_open(); - if let Some((_header, _status_code, _url, Some(_custom_video))) = stream_details.stream_info.as_ref() { - if stream_details.stream.is_some() { - is_stream_shared = false; - } - } - let shared_subscriber_id = - tuliprox_core::model::SharedSubscriberId::from_stream_uid(app_state.connection_manager.next_stream_uid()); - let mut pending_shared_cleanup = if is_stream_shared { - match SharedStreamManager::reserve_subscriber_cleanup( - &app_state.connection_manager, - shared_subscriber_id, - fingerprint.addr, - ) - .await - { - Ok(cleanup) => Some(cleanup), - Err(reason) => { - app_state - .active_users - .release_unbound_session_reservation( - &user.username, - session_token, - activation.placeholder_transition_version, - activation.placeholder_transition_version.is_some(), - ) - .await; - return stream_admission_rejected_response(reason.into(), &user.username); - } - } - } else { - None - }; - let mut metering = prepare_stream_metering( - app_state, - stream_url, - is_stream_shared, - stream_details.stream.is_some(), - stream_details.has_deferred_provider_open(), - ); - if let (Some(cleanup), Some(request_id)) = ( - pending_shared_cleanup.as_mut(), - stream_details - .provider_handle - .as_ref() - .and_then(|managed| managed.handle()) - .and_then(|handle| handle.playback_request_id), - ) { - cleanup.set_provider_request_identity(session_token, request_id); - } - let provider_handle = if is_stream_shared && !stream_details.has_deferred_provider_open() { - // Transfer the ManagedProviderHandle to the shared-stream manager. - // Ownership stays with the managed type across all awaits; the shared-stream - // manager disarms it only once it holds the write lock and commits to a new - // shared origin. If the future is cancelled before that point the Drop guard - // releases the slot. - stream_details.provider_handle.take() - } else { - None - }; - - stream_channel.shared = is_stream_shared; - if is_stream_shared { - stream_channel.shared_joined_existing = Some(false); - stream_channel.shared_stream_id = Some(u64::from(metering.meter_uid)); - } else { - stream_channel.shared_joined_existing = None; - stream_channel.shared_stream_id = None; - } - if is_stream_shared { - stream_details.shared_subscriber_id = - pending_shared_cleanup.as_ref().map(tuliprox_session::PendingSharedSubscriberCleanup::capability); - } - // In the no-limits path there may be no placeholder yet. The body needs the - // session's media flag before its first byte; create that session now. - let created_media_session = if !is_stream_shared - && !item_type.is_live() - && item_type.requires_provider_affinity() - && app_state.active_users.media_started_flag(&user.username, session_token).await.is_none() - { - if let Some(provider) = provider_name.as_deref() { - app_state - .active_users - .ensure_user_session_placeholder(crate::api::model::CreateUserSessionParams { - user, - session_token, - virtual_id, - provider, - stream_url: actual_request_url.as_ref(), - addr: &fingerprint.addr, - connection_permission, - connection_kind: Some(connection_kind), - socket_bound, - }) - .await; - true - } else { - false - } - } else { - false - }; - let stream = match create_active_client_stream(crate::api::model::ActiveClientStreamParams { - stream_details, - app_state, - user, - connection_permission, - connection_kind, - fingerprint, - stream_channel, - socket_bound, - session_token: Some(session_token), - req_headers, - meter_uid: metering.meter_uid, - meter_stream: metering.meter_stream, - }) - .await - { - Ok(stream) => stream, - Err(error) => { - if created_media_session { - app_state.active_users.terminate_session(&user.username, session_token).await; - } else { - app_state - .active_users - .release_unbound_session_reservation( - &user.username, - session_token, - activation.placeholder_transition_version, - activation.placeholder_transition_version.is_some(), - ) - .await; - } - return stream_admission_rejected_response(error, &user.username); - } - }; - let stream_resp = if is_stream_shared { - debug_if_enabled!( - "Streaming shared stream request from {}", - sanitize_sensitive_info(log_actual_request_url.as_ref()) - ); - let Some(pending_shared_cleanup) = pending_shared_cleanup else { - return stream_admission_rejected_response(StreamAdmissionError::RegistrationRejected, &user.username); - }; - // Shared Stream response - let shared_headers = provider_response.as_ref().map_or_else(Vec::new, |(h, _, _, _)| h.clone()); - if let Some((broadcast_stream, _shared_provider, _cleanup_capability)) = - SharedStreamManager::register_shared_stream( - SharedStreamCtx { - app_config: &app_state.app_config, - shared_stream_manager: &app_state.shared_stream_manager, - active_provider: &app_state.active_provider, - connection_manager: &app_state.connection_manager, - }, - stream_url, - stream, - &fingerprint.addr, - shared_subscriber_id, - shared_headers, - stream_options.buffer_size, - provider_handle, - pending_shared_cleanup, - connection_priority_for_kind(user, connection_kind), - connection_kind, - ) - .await - { - metering.commit_shared_registration(); - let (status_code, header_map) = - get_stream_response_with_headers(provider_response.map(|(h, s, _, _)| (h, s))); - let mut response = axum::response::Response::builder().status(status_code); - for (key, value) in &header_map { - response = response.header(key, value); - } - let mut response = try_unwrap_body!(response.body(axum::body::Body::from_stream(broadcast_stream))); - mark_response_as_uncompressed(&mut response); - response - } else { - StatusCode::BAD_REQUEST.into_response() - } - } else { - // Previously, we always persisted the provider's final request URL into the session. - // For VOD-like playback that can be the wrong thing to reuse later: a seek or reopen - // should start from the canonical playback entrypoint, not from a provider-specific - // redirected target that happened to be used for an earlier request. - // For Movies/Series/Catchup we therefore keep the canonical request URL in the session. - // That avoids "session poisoning" where later seeks/resumes inherit a non-canonical URL. - // For live playback we still keep the redirected URL when available, because staying on - // the chosen upstream edge/server is often desirable there. - let session_url: Cow<'_, str> = if matches!( - item_type, - PlaylistItemType::Catchup - | PlaylistItemType::Video - | PlaylistItemType::LocalVideo - | PlaylistItemType::Series - | PlaylistItemType::LocalSeries - | PlaylistItemType::SeriesInfo - | PlaylistItemType::LocalSeriesInfo - ) { - Cow::Owned(actual_request_url.to_string()) - } else { - provider_response - .as_ref() - .and_then(|(_, _, u, _)| u.as_ref()) - .map_or_else(|| Cow::Owned(actual_request_url.to_string()), |url| Cow::Owned(url.to_string())) - }; - let log_session_url = resolve_request_url_for_logging(input, session_url.as_ref()); - if log_enabled!(log::Level::Debug) { - if log_session_url.eq(log_actual_request_url.as_ref()) { - debug!( - "Streaming stream request from {}", - sanitize_sensitive_info(log_actual_request_url.as_ref()) - ); - } else { - debug!( - "Streaming stream request for {} from {}", - sanitize_sensitive_info(log_actual_request_url.as_ref()), - sanitize_sensitive_info(log_session_url.as_ref()) - ); - } - } - let (status_code, header_map) = - get_stream_response_with_headers(provider_response.map(|(h, s, _, _)| (h, s))); - let mut response = axum::response::Response::builder().status(status_code); - for (key, value) in &header_map { - response = response.header(key, value); - } - - if let Some(provider) = provider_name { - if matches!( - item_type, - PlaylistItemType::LiveHls - | PlaylistItemType::LiveDash - | PlaylistItemType::Video - | PlaylistItemType::Series - | PlaylistItemType::SeriesInfo - | PlaylistItemType::LocalSeries - | PlaylistItemType::LocalSeriesInfo - | PlaylistItemType::Catchup - ) { - let _ = app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user, - session_token, - virtual_id, - provider: &provider, - stream_url: &session_url, - addr: &fingerprint.addr, - connection_permission, - connection_kind: Some(connection_kind), - socket_bound, - }) - .await; - if let Some(stream_index) = user_agent_stream_index { - app_state - .active_users - .set_user_agent_stream_index_if_absent(&user.username, session_token, stream_index) - .await; - } - if should_pin_provider { - let reservation_ttl_secs = get_session_reservation_ttl_secs(app_state, item_type); - if reservation_ttl_secs > 0 { - app_state.active_provider.refresh_adaptive_playback_lease( - &provider, - session_token, - PlaybackKind::classify(item_type, playback_extension), - reservation_ttl_secs, - ); - } - } - } - } - - let body_stream = prepare_body_stream(app_state, item_type, stream); - let mut response = try_unwrap_body!(response.body(body_stream)); - mark_response_as_uncompressed(&mut response); - response - }; - - return stream_resp.into_response(); - } - app_state.connection_manager.release_managed_provider_handle(stream_details.provider_handle); - app_state - .active_users - .release_unbound_session_reservation( - &user.username, - session_token, - activation.placeholder_transition_version, - activation.placeholder_transition_version.is_some(), - ) - .await; - if stream_details.custom_reason == Some(ProviderStreamCustomReason::ProviderExhausted) { - record_connect_failed_attempt(ConnectFailedAttempt { - app_state, - fingerprint, - user, - stream_channel, - provider_name: stream_details.provider_name.unwrap_or_else(|| input.name.clone()), - req_headers, - reason: ConnectFailureReason::ProviderConnectionsExhausted, - failure_stage: FailureStage::Admission, - }); - return create_custom_video_stream_response( - &app_state.provider_stream_ctx(), - &fingerprint.addr, - CustomVideoStreamType::ProviderConnectionsExhausted, - ) - .into_response(); - } - no_custom_video_fallback_status(&app_state.app_config).into_response() -} - -enum CatchupPayload { - Direct(BoxedProviderStream), - HlsManifest(Bytes), -} - -struct DetectedCatchupHlsResponseParams<'a> { - app_state: &'a Arc, - stream_details: StreamDetails, - manifest: Bytes, - user: &'a ProxyUserCredentials, - target: &'a ConfigTarget, - input: &'a ConfigInput, - fingerprint: &'a Fingerprint, - session_token: &'a str, - virtual_id: VirtualId, - connection_permission: UserConnectionPermission, - connection_kind: crate::api::model::ConnectionKind, - fallback_stream_url: &'a str, -} - -#[allow(clippy::too_many_lines)] -async fn detected_catchup_hls_response(params: DetectedCatchupHlsResponseParams<'_>) -> axum::response::Response { - let DetectedCatchupHlsResponseParams { - app_state, - mut stream_details, - manifest, - user, - target, - input, - fingerprint, - session_token, - virtual_id, - connection_permission, - connection_kind, - fallback_stream_url, - } = params; - - let Some(provider) = stream_details.provider_name.clone() else { - cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token).await; - return StatusCode::BAD_GATEWAY.into_response(); - }; - let Some(server_info) = app_state.app_config.get_user_server_info(user) else { - cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token).await; - return StatusCode::BAD_GATEWAY.into_response(); - }; - let Ok(content) = std::str::from_utf8(&manifest) else { - cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token).await; - return StatusCode::BAD_GATEWAY.into_response(); - }; - let playlist_kind = classify_hls_playlist(content); - if playlist_kind == HlsPlaylistKind::Invalid { - cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token).await; - return StatusCode::BAD_GATEWAY.into_response(); - } - - let response_url = stream_details - .stream_info - .as_ref() - .and_then(|(_, _, response_url, _)| response_url.as_ref()) - .map_or_else(|| fallback_stream_url.to_string(), ToString::to_string); - let base_url = server_info.get_base_url(); - let encrypt_secret = app_state.get_encrypt_secret(); - let rewritten = rewrite_hls( - user, - &RewriteHlsProps { - secret: &encrypt_secret, - base_url: &base_url, - content, - hls_url: response_url, - target_id: target.id, - virtual_id: virtual_id.get(), - input_id: input.id, - user_token: Some(session_token), - origin_provider: Some(&provider), - playlist_kind: Some(playlist_kind), - }, - ); - - let request_url = stream_details.request_url.as_deref().unwrap_or(fallback_stream_url); - let created_session_token = app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user, - session_token, - virtual_id: virtual_id.get(), - provider: &provider, - stream_url: request_url, - addr: &fingerprint.addr, - connection_permission, - connection_kind: Some(connection_kind), - socket_bound: false, - }) - .await; - if let Some(stream_index) = stream_details.user_agent_stream_index { - app_state - .active_users - .set_user_agent_stream_index_if_absent(&user.username, &created_session_token, stream_index) - .await; - } - if !stream_details.provider_session_headers.is_empty() { - app_state - .active_users - .update_session_provider_response_headers_from( - &user.username, - &created_session_token, - &stream_details.provider_session_headers, - stream_details - .stream_info - .as_ref() - .and_then(|(_, _, url, _)| url.as_ref()) - .map_or(request_url, Url::as_str), - ) - .await; - } - app_state.active_provider.refresh_adaptive_playback_lease( - &provider, - &created_session_token, - PlaybackKind::Catchup, - get_catchup_session_ttl_secs(app_state), - ); - let binding_tag = stream_details.provider_handle.as_ref().and_then(|handle| handle.handle()?.binding_tag); - app_state.connection_manager.release_managed_provider_handle(stream_details.provider_handle.take()); - app_state - .active_users - .release_unbound_session_reservation(&user.username, &created_session_token, None, false) - .await; - app_state.active_users.clear_unbound_session_addr(&user.username, &created_session_token, &fingerprint.addr).await; - - if playlist_kind == HlsPlaylistKind::Media && hls_media_playlist_wrap_enabled(app_state, target, user) { - let master = wrap_media_playlist( - HlsMediaPlaylistWrap { - app_state, - user, - base_url: &base_url, - target_id: target.id, - input, - virtual_id: virtual_id.get(), - session_token: &created_session_token, - // The canonical catch-up URL, not the provider-resolved one, so a refresh can - // resolve it to whichever account is selected then. - sealed_url: fallback_stream_url, - provider: &provider, - binding_tag, - known_bitrate_bps: None, - stream_ref: None, - }, - rewritten, - ) - .await; - return catchup_hls_manifest_response(master); - } - catchup_hls_manifest_response(rewritten) -} - -fn catchup_hls_manifest_response(content: String) -> axum::response::Response { - let mut response = try_unwrap_body!(axum::response::Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, crate::api::static_headers::CT_M3U.clone()) - .header(header::CACHE_CONTROL, crate::api::static_headers::CC_NO_STORE.clone()) - .body(Body::from(content))); - mark_response_as_uncompressed(&mut response); - response -} - -async fn cleanup_failed_detected_catchup_hls( - app_state: &Arc, - stream_details: &mut StreamDetails, - username: &str, - session_token: &str, -) { - app_state.connection_manager.release_managed_provider_handle(stream_details.provider_handle.take()); - app_state.active_users.terminate_session(username, session_token).await; - app_state.active_provider.clear_provider_reservation(session_token); -} - -async fn probe_catchup_payload(stream: BoxedProviderStream, deadline: Duration) -> Result { - tokio::time::timeout(deadline, probe_catchup_payload_inner(stream)) - .await - .map_err(|_| StreamError::Stream("catch-up payload probe timed out".to_string()))? -} - -async fn probe_catchup_payload_inner(mut stream: BoxedProviderStream) -> Result { - const HLS_SIGNATURE: &[u8] = b"#EXTM3U"; - - let mut prefix = BytesMut::new(); - while prefix.len() < HLS_SIGNATURE.len() { - let Some(chunk) = stream.next().await else { - return Ok(CatchupPayload::Direct(stream::once(async move { Ok(prefix.freeze()) }).chain(stream).boxed())); - }; - prefix.extend_from_slice(&chunk?); - } - - if !prefix.starts_with(HLS_SIGNATURE) { - return Ok(CatchupPayload::Direct(stream::once(async move { Ok(prefix.freeze()) }).chain(stream).boxed())); - } - if prefix.len() > MAX_HLS_MANIFEST_BYTES { - return Err(StreamError::Stream("catch-up HLS manifest exceeds size limit".to_string())); - } - - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - if prefix.len().saturating_add(chunk.len()) > MAX_HLS_MANIFEST_BYTES { - return Err(StreamError::Stream("catch-up HLS manifest exceeds size limit".to_string())); - } - prefix.extend_from_slice(&chunk); - } - - Ok(CatchupPayload::HlsManifest(prefix.freeze())) -} - -fn get_stream_throttle(app_state: &Arc) -> u64 { - app_state - .app_config - .config - .load() - .reverse_proxy - .as_ref() - .and_then(|reverse_proxy| reverse_proxy.stream.as_ref()) - .map(|stream| stream.throttle_kbps) - .unwrap_or_default() -} - -fn is_stream_metrics_enabled(app_state: &Arc) -> bool { - app_state - .app_config - .config - .load() - .reverse_proxy - .as_ref() - .and_then(|reverse_proxy| reverse_proxy.stream.as_ref()) - .is_some_and(|stream| stream.metrics_enabled) -} - -fn prepare_stream_metering( - app_state: &Arc, - stream_url: &str, - share_stream: bool, - has_stream: bool, - has_deferred_provider_open: bool, -) -> StreamMeteringConfig { - if !is_stream_metrics_enabled(app_state) { - return StreamMeteringConfig::default(); - } - - if share_stream { - let (meter_uid, pending_registration) = app_state - .shared_stream_manager - .reserve_meter_uid(stream_url, || app_state.connection_manager.next_stream_uid()); - return StreamMeteringConfig { - meter_uid, - meter_stream: has_stream || has_deferred_provider_open, - pending_shared_registration: pending_registration, - }; - } else if has_stream || has_deferred_provider_open { - let meter_uid = app_state.connection_manager.next_stream_uid(); - return StreamMeteringConfig { meter_uid, meter_stream: true, pending_shared_registration: None }; - } - - StreamMeteringConfig::default() -} - -fn resolve_stream_config_u64( - stream_config: Option<&crate::model::StreamConfig>, - selector: impl FnOnce(&crate::model::StreamConfig) -> u64, - default_value: u64, -) -> u64 { - stream_config.map_or(default_value, selector) -} - -fn get_stream_config_u64( - app_state: &Arc, - selector: impl FnOnce(&crate::model::StreamConfig) -> u64, - default_value: u64, -) -> u64 { - let config = app_state.app_config.config.load(); - let stream_config = config.reverse_proxy.as_ref().and_then(|reverse_proxy| reverse_proxy.stream.as_ref()); - resolve_stream_config_u64(stream_config, selector, default_value) -} - -pub(crate) fn get_hls_session_ttl_secs(app_state: &Arc) -> u64 { - get_stream_config_u64(app_state, |stream| stream.hls_session_ttl_secs, default_hls_session_ttl_secs()) -} - -async fn cleanup_forced_reopen_addrs(app_state: &Arc, session_owner: &str, cleanup_addrs: &[SocketAddr]) { - app_state.active_provider.release_playback_connections_await(session_owner, cleanup_addrs).await; -} - -pub(crate) fn get_catchup_session_ttl_secs(app_state: &Arc) -> u64 { - get_stream_config_u64(app_state, |stream| stream.catchup_session_ttl_secs, default_catchup_session_ttl_secs()) -} - -/// Reconnect window of an HLS playback lease: archive playback keeps the catchup window. -pub(crate) fn get_hls_playback_ttl_secs(app_state: &Arc, kind: PlaybackKind) -> u64 { - if kind == PlaybackKind::Catchup { - get_catchup_session_ttl_secs(app_state) - } else { - get_hls_session_ttl_secs(app_state) - } -} - -pub(crate) fn get_session_reservation_ttl_secs(app_state: &Arc, item_type: PlaylistItemType) -> u64 { - match item_type { - PlaylistItemType::LiveHls | PlaylistItemType::LiveDash => get_hls_session_ttl_secs(app_state), - PlaylistItemType::Catchup => get_catchup_session_ttl_secs(app_state), - _ => 0, - } -} - -/// Whether the session should pin the provider account via `refresh_provider_reservation`. -/// -/// A non-Provisioning custom video (`ChannelUnavailable`, `ProviderConnectionsExhausted`, …) means -/// the upstream open already failed. The provider connection slot was released by -/// `create_provider_stream`, and the custom video is a local fallback served to the client. -/// Pinning the provider via `refresh_provider_reservation` would hold the provider account for -/// the configured session TTL (e.g. `catchup_session_ttl_secs`), blocking other sessions of -/// the same family from using it even though the slot is already free. -/// -/// Only `Provisioning` custom videos represent a real provider handoff that benefits from -/// keeping the same provider pinned, and real provider streams (`stream_info` carries no -/// `CustomVideoStreamType`) obviously qualify. -pub(crate) fn should_pin_provider_for_session( - stream_details: &StreamDetails, - _app_state: &Arc, - _item_type: PlaylistItemType, -) -> bool { - !matches!( - stream_details.stream_info.as_ref(), - Some((_, _, _, Some(cv))) if *cv != CustomVideoStreamType::Provisioning - ) -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -async fn try_shared_stream_response_if_any( - app_state: &Arc, - stream_url: &str, - fingerprint: &Fingerprint, - user: &ProxyUserCredentials, - connect_permission: UserConnectionPermission, - connection_kind: crate::api::model::ConnectionKind, - mut stream_channel: StreamChannel, - session_token: &str, - req_headers: &HeaderMap, - placeholder_transition_version: Option, -) -> Option { - let subscriber_id = - tuliprox_core::model::SharedSubscriberId::from_stream_uid(app_state.connection_manager.next_stream_uid()); - let shared_subscription = SharedStreamManager::subscribe_shared_stream( - SharedStreamCtx { - app_config: &app_state.app_config, - shared_stream_manager: &app_state.shared_stream_manager, - active_provider: &app_state.active_provider, - connection_manager: &app_state.connection_manager, - }, - stream_url, - &fingerprint.addr, - subscriber_id, - connection_priority_for_kind(user, connection_kind), - connection_kind, - ) - .await; - let (stream, provider, cleanup_capability) = match shared_subscription { - Ok(Some(subscription)) => subscription, - Ok(None) => return None, - Err(reason) => { - app_state - .active_users - .release_unbound_session_reservation( - &user.username, - session_token, - placeholder_transition_version, - placeholder_transition_version.is_some(), - ) - .await; - return Some(stream_admission_rejected_response(reason.into(), &user.username)); - } - }; - debug_if_enabled!("Using shared stream {}", sanitize_sensitive_info(stream_url)); - if let Some(headers) = app_state.shared_stream_manager.get_shared_state_headers(stream_url).await { - let (status_code, header_map) = get_stream_response_with_headers(Some((headers.clone(), StatusCode::OK))); - let mut grace_period_options = app_state.get_grace_options(); - if connect_permission != UserConnectionPermission::GracePeriod { - grace_period_options.period_millis = 0; - } - let mut stream_details = StreamDetails::from_stream(stream, grace_period_options); - stream_details.shared_subscriber_id = Some(cleanup_capability); - - stream_details.provider_name = provider; - let socket_bound = - is_socket_bound_playback_session(stream_channel.item_type, extract_extension_from_url(stream_url)); - // A shared origin may have no provider allocation (e.g. a local/unknown owner), in - // which case `provider` is `None`. The subscriber must still be tracked as a user - // session, so fall back to a stable placeholder name rather than skipping session - // creation; `is_over_limit` on an unknown provider is a no-op. - let provider_name = stream_details.provider_name.as_deref().unwrap_or("shared"); - let _ = app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user, - session_token, - virtual_id: stream_channel.virtual_id, - provider: provider_name, - stream_url, - addr: &fingerprint.addr, - connection_permission: connect_permission, - connection_kind: Some(connection_kind), - socket_bound, - }) - .await; - stream_channel.shared = true; - stream_channel.shared_joined_existing = Some(true); - // Joining an existing origin reuses its meter identity. When metrics are disabled - // no meter exists to reserve; reserving one here would leave an ownerless entry that - // teardown cannot match and remove. - let (meter_uid, pending_registration) = if is_stream_metrics_enabled(app_state) { - app_state - .shared_stream_manager - .reserve_meter_uid(stream_url, || app_state.connection_manager.next_stream_uid()) - } else { - (0, None) - }; - stream_channel.shared_stream_id = Some(u64::from(meter_uid)); - // The pending registration is carried through admission so its drop guard rolls - // the reservation back if the client stream is never created. - let mut metering = - StreamMeteringConfig { meter_uid, meter_stream: false, pending_shared_registration: pending_registration }; - let stream = match create_active_client_stream(crate::api::model::ActiveClientStreamParams { - stream_details, - app_state, - user, - connection_permission: connect_permission, - connection_kind, - fingerprint, - stream_channel, - socket_bound, - session_token: Some(session_token), - req_headers, - meter_uid: metering.meter_uid, - meter_stream: metering.meter_stream, - }) - .await - { - Ok(stream) => { - metering.commit_shared_registration(); - stream.boxed() - } - Err(error) => { - app_state - .active_users - .release_unbound_session_reservation( - &user.username, - session_token, - placeholder_transition_version, - placeholder_transition_version.is_some(), - ) - .await; - return Some(stream_admission_rejected_response(error, &user.username)); - } - }; - let mut response = axum::response::Response::builder().status(status_code); - for (key, value) in &header_map { - response = response.header(key, value); - } - let mut response = response.body(axum::body::Body::from_stream(stream)).ok()?; - mark_response_as_uncompressed(&mut response); - return Some(response); - } - None -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -pub(crate) async fn local_stream_response( - fingerprint: &Fingerprint, - app_state: &Arc, - pli: StreamChannel, - req_headers: &HeaderMap, - input: &ConfigInput, - _target: &ConfigTarget, - user: &ProxyUserCredentials, - connection_permission: UserConnectionPermission, - connection_kind: crate::api::model::ConnectionKind, - playback_session_token: Option<&str>, - request_class: Option, - check_path: bool, -) -> impl IntoResponse + Send { - let _transition_guard = if let Some(session_token) = playback_session_token { - Some(app_state.active_users.acquire_playback_transition(&user.username, session_token).await) - } else { - None - }; - if log_enabled!(log::Level::Trace) { - trace!("Try to open stream {}", sanitize_sensitive_info(&pli.url)); - } - - let mut connection_permission = connection_permission; - let mut grace_mode = None; - if connection_permission == UserConnectionPermission::Exhausted { - let allow_session_reopen = if let Some(session_token) = playback_session_token { - user.max_connections > 0 - && app_state - .active_users - .connection_permission_for_session( - &user.username, - user.max_connections, - user.soft_connections, - session_token, - ) - .await - != UserConnectionPermission::Exhausted - } else { - false - }; - if !allow_session_reopen { - record_connect_failed_attempt(ConnectFailedAttempt { - app_state, - fingerprint, - user, - stream_channel: pli.clone(), - provider_name: input.name.clone(), - req_headers, - reason: ConnectFailureReason::UserConnectionsExhausted, - failure_stage: FailureStage::Admission, - }); - return create_custom_video_stream_response( - &app_state.provider_stream_ctx(), - &fingerprint.addr, - CustomVideoStreamType::UserConnectionsExhausted, - ) - .into_response(); - } - connection_permission = UserConnectionPermission::Allowed; - } - - let path = PathBuf::from(pli.url.strip_prefix("file://").unwrap_or(&pli.url)); - - let Ok(mut file) = tokio::fs::File::open(&path).await else { return StatusCode::NOT_FOUND.into_response() }; - let Ok(opened_metadata) = file.metadata().await else { return internal_server_error!() }; - - // Canonicalize and validate the path - let canonical = match tokio::fs::canonicalize(&path).await { - Ok(canonical) => canonical, - Err(err) => { - error!("Local file path is corrupt {}: {err}", path.display()); - return StatusCode::NOT_FOUND.into_response(); - } - }; - - if check_path { - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - let Ok(canonical_metadata) = tokio::fs::metadata(&canonical).await else { return internal_server_error!() }; - if opened_metadata.dev() != canonical_metadata.dev() || opened_metadata.ino() != canonical_metadata.ino() { - error!("TOCTOU race detected: file swapped during local_stream_response"); - return StatusCode::FORBIDDEN.into_response(); - } - } - #[cfg(windows)] - match same_windows_file_identity(&file, &canonical).await { - Ok(true) => {} - Ok(false) => { - error!("TOCTOU race detected: file swapped during local_stream_response"); - return StatusCode::FORBIDDEN.into_response(); - } - Err(err) => { - error!("Could not verify local file identity {}: {err}", canonical.display()); - return internal_server_error!(); - } - } - #[cfg(not(any(unix, windows)))] - { - error!("Secure local file identity validation is unsupported on this platform"); - return StatusCode::FORBIDDEN.into_response(); - } - - let Some(library_paths) = app_state - .app_config - .config - .load() - .library - .as_ref() - .map(|lib| lib.scan_directories.iter().map(|dir| dir.path.clone()).collect::>()) - else { - return StatusCode::NOT_FOUND.into_response(); - }; - - // Verify path is within allowed media directories - // (requires configuration of allowed base paths) - if !is_path_within_allowed_directories(&canonical, &library_paths) { - return StatusCode::FORBIDDEN.into_response(); - } - } - - let file_size = opened_metadata.len(); - - let range = req_headers.get("range").and_then(|v| v.to_str().ok()).and_then(parse_range); - - let (start, end) = if let Some((req_start, req_end)) = range { - if file_size == 0 || req_start >= file_size { - return StatusCode::RANGE_NOT_SATISFIABLE.into_response(); - } - let end = req_end.unwrap_or(file_size - 1).min(file_size - 1); - if end < req_start { - return StatusCode::RANGE_NOT_SATISFIABLE.into_response(); - } - (req_start, end) - } else { - if file_size == 0 { - // Serve empty file - let body = axum::body::Body::empty(); - let mut response = Response::new(body); - *response.status_mut() = StatusCode::OK; - let headers = response.headers_mut(); - if let Some(ext) = get_file_extension(&pli.url) { - let ct = content_type_from_ext(&ext); - headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(ct)); - } else { - headers.insert(header::CONTENT_TYPE, CT_OCTET.clone()); //HeaderValue::from_static("application/octet-stream")); - } - headers.insert("Accept-Ranges", HeaderValue::from_static("bytes")); - headers.insert(header::CONTENT_LENGTH, HeaderValue::from_static("0")); - return response.into_response(); - } - (0, file_size - 1) - }; - - let content_length = end - start + 1; - - if start > 0 { - if let Err(_err) = file.seek(SeekFrom::Start(start)).await { - return internal_server_error!(); - } - } - - let stream = - ReaderStream::new(file.take(content_length)).map_err(|err| StreamError::Stream(err.to_string())).boxed(); - let throttle_kbps = usize::try_from(get_stream_throttle(app_state)).unwrap_or_default(); - let stream = if is_throttled_stream(pli.item_type, throttle_kbps) { - info!("Stream throttling active: {}", human_readable_kbps(u64::try_from(throttle_kbps).unwrap_or_default())); - ThrottledStream::new(stream, throttle_kbps).boxed() - } else { - stream - }; - let socket_bound = is_socket_bound_playback_session(pli.item_type, extract_extension_from_url(&pli.url)); - let mut connection_kind = connection_kind; - if let Some(session_token) = playback_session_token { - let activation = activate_session_before_stream_open( - app_state, - SessionActivationRequest { - fingerprint, - input, - user, - session_token, - request_class, - virtual_id: VirtualId::new(pli.virtual_id), - item_type: pli.item_type, - stream_url: &pli.url, - connection_permission, - connection_kind, - granted_grace_mode: None, - socket_bound, - }, - ) - .await; - grace_mode = activation.grace_mode; - connection_permission = activation.admission.permission(); - connection_kind = activation.admission.kind().unwrap_or(connection_kind); - - if connection_permission == UserConnectionPermission::Exhausted { - app_state - .active_users - .release_unbound_session_reservation( - &user.username, - session_token, - activation.placeholder_transition_version, - activation.placeholder_transition_version.is_some(), - ) - .await; - if activation.admission.is_reentry_suppressed() { - return reentry_suppressed_response(); - } - return create_custom_video_stream_response( - &app_state.provider_stream_ctx(), - &fingerprint.addr, - CustomVideoStreamType::UserConnectionsExhausted, - ) - .into_response(); - } - } - let mut grace_period_options = app_state.get_grace_options(); - if connection_permission != UserConnectionPermission::GracePeriod { - grace_period_options.period_millis = 0; - } - if let Some(resolved_mode) = grace_mode { - grace_period_options.hold_stream = matches!(resolved_mode, crate::api::model::GraceMode::Hold); - } - let resolved_connection_kind = if let Some(session_token) = playback_session_token { - app_state - .active_users - .get_and_update_user_session(&user.username, session_token) - .await - .and_then(|session| session.connection_kind) - .unwrap_or(connection_kind) - } else { - connection_kind - }; - if let Some(session_token) = playback_session_token { - let _ = app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user, - session_token, - virtual_id: pli.virtual_id, - provider: input.name.as_ref(), - stream_url: &pli.url, - addr: &fingerprint.addr, - connection_permission, - connection_kind: Some(resolved_connection_kind), - socket_bound, - }) - .await; - } - let metering = prepare_stream_metering(app_state, &pli.url, false, true, false); - let stream = match create_active_client_stream(crate::api::model::ActiveClientStreamParams { - stream_details: StreamDetails::from_stream(stream, grace_period_options), - app_state, - user, - connection_permission, - connection_kind: resolved_connection_kind, - fingerprint, - stream_channel: pli.clone(), - socket_bound, - session_token: playback_session_token, - req_headers, - meter_uid: metering.meter_uid, - meter_stream: metering.meter_stream, - }) - .await - { - Ok(stream) => stream, - Err(error) => return stream_admission_rejected_response(error, &user.username), - }; - - let mut response = Response::new(axum::body::Body::from_stream(stream)); - - *response.status_mut() = if range.is_some() { StatusCode::PARTIAL_CONTENT } else { StatusCode::OK }; - - let headers = response.headers_mut(); - if let Some(ext) = get_file_extension(&pli.url) { - let ct = content_type_from_ext(&ext); - headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(ct)); - } else { - headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("application/octet-stream")); - } - headers.insert("Accept-Ranges", HeaderValue::from_static("bytes")); - if let Ok(header_value) = HeaderValue::from_str(&content_length.to_string()) { - headers.insert(header::CONTENT_LENGTH, header_value); - } - - if range.is_some() { - if let Ok(header_value) = HeaderValue::from_str(&format!("bytes {start}-{end}/{file_size}")) { - headers.insert(header::CONTENT_RANGE, header_value); - } - } - - mark_response_as_uncompressed(&mut response); - response -} - -fn is_path_within_allowed_directories(sub_path: &Path, root_paths: &[String]) -> bool { - for root_path in root_paths { - if sub_path.starts_with(PathBuf::from(root_path)) { - return true; - } - } - false -} - -pub fn is_stream_share_enabled(item_type: PlaylistItemType, target: &ConfigTarget) -> bool { - (item_type == PlaylistItemType::Live/* || item_type == PlaylistItemType::LiveHls */) - && target.options.as_ref().is_some_and(ConfigTargetOptions::share_live_mpeg_ts_enabled) -} - -pub fn is_hls_stream_share_enabled(target: &ConfigTarget) -> bool { - target.options.as_ref().is_some_and(ConfigTargetOptions::share_live_hls_enabled) -} - -fn get_add_cache_content( - res_url: &str, - mime_type: Option, - cache: &Arc>>, -) -> Arc { - let resource_url = String::from(res_url); - let cache = Arc::clone(cache); - let add_cache_content: Arc = Arc::new(move |size| { - let res_url = resource_url.clone(); - let mime_type = mime_type.clone(); - // todo spawn, replace with unboundchannel - let cache = Arc::clone(&cache); - tokio::spawn(async move { - if let Some(cache) = cache.load().as_ref() { - let _ = cache.write().await.add_content(&res_url, mime_type, size); - } - }); - }); - add_cache_content -} - -fn get_mime_type(headers: &HeaderMap, resource_url: &str) -> Option { - headers - .get(header::CONTENT_TYPE) - .and_then(|v| v.to_str().ok()) // Option<&str> - .map(ToString::to_string) // Option - .or_else(|| { - // fallback to guess - mime_guess::from_path(resource_url).first_raw().map(ToString::to_string) - }) -} - -#[cfg(windows)] -async fn same_windows_file_identity(opened_file: &tokio::fs::File, canonical_path: &Path) -> std::io::Result { - let canonical_file = tokio::fs::File::open(canonical_path).await?; - Ok(windows_file_identity(opened_file)? == windows_file_identity(&canonical_file)?) -} - -#[cfg(windows)] -fn windows_file_identity(file: &tokio::fs::File) -> std::io::Result<(u32, u32, u32)> { - use std::os::windows::io::AsRawHandle; - use windows_sys::Win32::Storage::FileSystem::{GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION}; - - let mut info = unsafe { std::mem::zeroed::() }; - // SAFETY: `file.as_raw_handle()` is a live file handle for the duration of - // the call, and `info` is a writable output buffer for the WinAPI function. - let ok = unsafe { GetFileInformationByHandle(file.as_raw_handle().cast(), &raw mut info) }; - if ok == 0 { - return Err(std::io::Error::last_os_error()); - } - Ok((info.dwVolumeSerialNumber, info.nFileIndexHigh, info.nFileIndexLow)) -} - -async fn build_resource_stream_response( - app_state: &Arc, - cache_key: Option<&str>, - fetch_policy: ResourceFetchPolicy, - resource_url: &str, - response: reqwest::Response, -) -> axum::response::Response { - let sanitized_resource_url = sanitize_sensitive_info(resource_url); - let status = response.status(); - let mut response_builder = axum::response::Response::builder().status(status); - let mime_type = get_mime_type(response.headers(), resource_url); - let has_content_range = response.headers().contains_key(header::CONTENT_RANGE); - for (key, value) in response.headers() { - if !is_hop_by_hop_response_header(key) - && (fetch_policy == ResourceFetchPolicy::Public - || matches!( - *key, - header::CONTENT_TYPE - | header::CONTENT_LENGTH - | header::CONTENT_ENCODING - | header::CONTENT_RANGE - | header::ACCEPT_RANGES - | header::ETAG - | header::LAST_MODIFIED - | header::CACHE_CONTROL - )) - { - response_builder = response_builder.header(key, value); - } - } - - if !response_builder.headers_ref().is_some_and(|h| h.contains_key(header::CACHE_CONTROL)) { - response_builder = response_builder.header(header::CACHE_CONTROL, "public, max-age=14400"); - } - - let byte_stream = response.bytes_stream().map_err(|err| StreamError::reqwest(&err)); - // Cache only complete responses (200 OK without Content-Range) - let can_cache = status == StatusCode::OK && !has_content_range; - if can_cache { - if let Some(cache_key) = cache_key { - debug!("Caching eligible resource stream {sanitized_resource_url}"); - let cache_resource_path = if let Some(cache) = app_state.cache.load().as_ref() { - Some(cache.write().await.store_path(cache_key, mime_type.as_deref())) - } else { - None - }; - if let Some(resource_path) = cache_resource_path { - match create_new_file_for_write(&resource_path).await { - Ok(file) => { - debug!("Persisting resource stream {sanitized_resource_url} to {}", resource_path.display()); - let writer = async_file_writer(file); - let add_cache_content = get_add_cache_content(cache_key, mime_type, &app_state.cache); - let tee = tee_stream(byte_stream, writer, &resource_path, add_cache_content); - return try_unwrap_body!(response_builder.body(axum::body::Body::from_stream(tee))); - } - Err(err) => { - warn!( - "Failed to create cache file {} for {sanitized_resource_url}: {err}", - resource_path.display() - ); - } - } - } else { - debug!( - "Resource cache unavailable; streaming response for {sanitized_resource_url} without persistence" - ); - } - } - } - - try_unwrap_body!(response_builder.body(axum::body::Body::from_stream(byte_stream))) -} - /// Upper bound of redirect hops a resource route follows. Every hop is classified, so the number of /// requests a provider or EPG entry can trigger through one resource link stays bounded. const RESOURCE_REDIRECT_LIMIT: u8 = 5; -/// Whether outbound requests of this configuration may leave through a proxy, including one provided by -/// the environment, because the public resource client honours both. -fn proxy_in_use(app_state: &Arc) -> bool { - app_state.app_config.config.load().proxy.is_some() || crate::model::proxy_env_present() -} - -async fn fetch_resource_with_retry( - app_state: &Arc, - url: &Url, - fetch_policy: ResourceFetchPolicy, - resource_url: &str, - req_headers: &HashMap>, - input: Option<&ConfigInput>, -) -> Option { - let cache_key = fetch_policy.cache_key(resource_url); - let config = app_state.app_config.config.load(); - let default_user_agent = config.default_user_agent.clone(); - drop(config); - - let disabled_headers = app_state.get_disabled_headers(); - let mut current_url = url.clone(); - let mut current_headers = req_headers.clone(); - let mut current_input = input; - let mut method = input.map_or(InputFetchMethod::GET, |i| i.method); - - for redirects in 0..=RESOURCE_REDIRECT_LIMIT { - // Every hop is routed on its own, so a redirect cannot move a request to an egress the hop - // itself would not have used. - let hop = classify_resource_hop(current_url.as_str(), proxy_in_use(app_state)).await; - if hop == ResourceHop::Blocked { - debug!("Refused resource destination local to this host: {}", sanitize_sensitive_info(resource_url)); - return None; - } - let use_proxy_aware_client = hop == ResourceHop::ViaProxy; - let provider_config = current_input.and_then(|i| i.get_resolve_provider(current_url.as_str())); - let response = match send_with_retry_and_provider( - &app_state.app_config, - ¤t_url, - provider_config.as_ref(), - // Hand redirects back instead of retrying them: the loop below follows them itself, so that - // every hop is classified and no client-side redirect policy decides where the request ends. - true, - |resolved_url| { - let http_client = if use_proxy_aware_client { - app_state.http_clients.resource_public_no_redirect.load() - } else { - app_state.http_clients.resource_no_redirect.load() - }; - request::get_client_request( - &http_client, - method, - current_input.map(|i| &i.headers), - resolved_url, - Some(¤t_headers), - disabled_headers.as_ref(), - default_user_agent.as_deref(), - ) - }, - ) - .await - { - Ok(response) => response, - Err(err) => { - debug!( - "Resource fetch failed for {}: {}", - sanitize_sensitive_info(resource_url), - sanitize_sensitive_info(&err.to_string()) - ); - return None; - } - }; - - let status = response.status(); - if status.is_redirection() { - if redirects == RESOURCE_REDIRECT_LIMIT { - debug!("Resource redirect limit reached for {}", sanitize_sensitive_info(resource_url)); - return None; - } - let next_url = response - .headers() - .get(header::LOCATION) - .and_then(|location| location.to_str().ok()) - .and_then(|location| response.url().join(location).ok()); - let Some(next_url) = next_url else { - debug!("Resource redirect has no usable location for {}", sanitize_sensitive_info(resource_url)); - return None; - }; - let same_origin = response.url().scheme() == next_url.scheme() - && response.url().host_str() == next_url.host_str() - && response.url().port_or_known_default() == next_url.port_or_known_default(); - if !same_origin { - current_headers.retain(|key, _| request::is_safe_cross_origin_redirect_header(key)); - current_input = None; - } - if !matches!(status, StatusCode::TEMPORARY_REDIRECT | StatusCode::PERMANENT_REDIRECT) { - method = InputFetchMethod::GET; - } - current_url = next_url; - continue; - } - - if status.is_success() { - return Some( - build_resource_stream_response(app_state, cache_key, fetch_policy, resource_url, response).await, - ); - } - - debug_if_enabled!("Failed to open resource got status {status} for {}", sanitize_sensitive_info(resource_url)); - if !fetch_policy.relays_upstream_response() { - return Some(status.into_response()); - } - let mut response_builder = axum::response::Response::builder().status(status); - for (key, value) in response.headers() { - if !is_hop_by_hop_response_header(key) { - response_builder = response_builder.header(key, value); - } - } - let stream = response.bytes_stream().map_err(|err| StreamError::reqwest(&err)); - return Some(try_unwrap_body!(response_builder.body(axum::body::Body::from_stream(stream)))); - } - None -} - /// How the client-visible answer of a resource route is shaped. /// /// The variant mirrors the classification of the destination the route serves: a [`Public`] destination @@ -4513,800 +230,113 @@ pub enum ResourceFetchPolicy { NonPublic, } -pub fn resource_input_for_url<'a>(input: Option<&'a ConfigInput>, resource_url: &str) -> Option<&'a ConfigInput> { - let input = input?; - let source = Url::parse(&input.url).ok()?; - let resource = Url::parse(resource_url).ok()?; - (source.scheme() == resource.scheme() - && source.host_str() == resource.host_str() - && source.port_or_known_default() == resource.port_or_known_default()) - .then_some(input) -} - -pub async fn resource_redirect_or_proxy( - app_state: &Arc, - resource_url: &str, - req_headers: &HeaderMap, - input: Option<&ConfigInput>, -) -> axum::response::Response { - let Some(resource_url) = shared::model::persisted_resource_url(resource_url) else { - return StatusCode::BAD_REQUEST.into_response(); - }; - let resource_url = resource_url.as_ref(); - if resource_url.starts_with("/api/v1/library/thumbnail/") { - return redirect(resource_url).into_response(); - } - if resource_url.starts_with("media-server://image/") { - return resource_response(app_state, ResourceFetchPolicy::NonPublic, resource_url, req_headers, None) - .await - .into_response(); - } - let Ok(url) = Url::parse(resource_url) else { - return StatusCode::BAD_REQUEST.into_response(); - }; - if !matches!(url.scheme(), "http" | "https") { - return StatusCode::BAD_REQUEST.into_response(); - } - let Some(host) = url.host_str() else { - return StatusCode::BAD_REQUEST.into_response(); - }; - if classify_resource_destination(host).await == ResourceDestination::Public { - redirect(resource_url).into_response() - } else { - resource_proxy_response(app_state, resource_url, req_headers, resource_input_for_url(input, resource_url)).await - } -} - -pub async fn resource_proxy_response( - app_state: &Arc, - resource_url: &str, - req_headers: &HeaderMap, - input: Option<&ConfigInput>, -) -> axum::response::Response { - let Some(resource_url) = shared::model::persisted_resource_url(resource_url) else { - return StatusCode::BAD_REQUEST.into_response(); - }; - let resource_url = resource_url.as_ref(); - let input = resource_input_for_url(input, resource_url); - match classify_output_resource_url(resource_url).await { - ResourceOutputPolicy::Direct if resource_url.starts_with("/api/v1/library/thumbnail/") => { - redirect(resource_url).into_response() - } - ResourceOutputPolicy::Direct => { - resource_response(app_state, ResourceFetchPolicy::Public, resource_url, req_headers, input) - .await - .into_response() - } - ResourceOutputPolicy::Proxy => { - resource_response(app_state, ResourceFetchPolicy::NonPublic, resource_url, req_headers, input) - .await - .into_response() - } - ResourceOutputPolicy::Blocked => StatusCode::FORBIDDEN.into_response(), - } -} - -impl ResourceFetchPolicy { - const fn cache_key(self, resource_url: &str) -> Option<&str> { - match self { - Self::Public => Some(resource_url), - Self::NonPublic => None, - } - } - - /// Whether an upstream response may be relayed to the client. - /// - /// A public destination was chosen by the user's own configuration, so what it answered is what - /// the client asked for. A resource proxy whose destination is deliberately hidden from the client - /// must not become a reader for that destination: status, headers and body of an upstream error can - /// disclose more about the internal service than the link itself. - const fn relays_upstream_response(self) -> bool { matches!(self, Self::Public) } -} - -/// Callers must pass an already-decoded resource URL (not a `resource://v1/` locator). -/// [`resource_redirect_or_proxy`] and [`resource_proxy_response`] handle the decode before -/// dispatching here; XMLTV and Web-UI endpoints pass URLs from authenticated tokens. -pub async fn resource_response( - app_state: &Arc, - fetch_policy: ResourceFetchPolicy, - resource_url: &str, - req_headers: &HeaderMap, - input: Option<&ConfigInput>, -) -> impl IntoResponse + Send { - if resource_url.is_empty() { - return StatusCode::NO_CONTENT.into_response(); - } - - if resource_url.starts_with("media-server://image/") { - return match open_media_server_image_resource(app_state, resource_url).await { - Ok(response) => response, - Err(err) => { - let status = media_server_image_error_status(&err); - match status { - StatusCode::BAD_REQUEST => warn!("Invalid media-server image resource URL: {err}"), - StatusCode::NOT_FOUND => debug!("Media-server image resource was not found: {err}"), - _ => error!("Can't open media-server image from upstream: {err}"), - } - status.into_response() - } - }; - } - let filter: HeaderFilter = Some(Box::new(request::is_safe_cross_origin_redirect_header)); - let req_headers = get_headers_from_request(req_headers, &filter); - let cache_key = fetch_policy.cache_key(resource_url); - if let (Some(cache_key), Some(cache)) = (cache_key, app_state.cache.load().as_ref()) { - let cache_hit = { - let mut guard = cache.write().await; - guard.get_content(cache_key) - }; - - if let Some((resource_path, mime_type)) = cache_hit { - trace_if_enabled!("Responding resource from cache {}", sanitize_sensitive_info(resource_url)); - return serve_file( - &resource_path, - mime_type.unwrap_or_else(|| mime::APPLICATION_OCTET_STREAM.to_string()), - Some("public, max-age=14400"), - ) - .await - .into_response(); - } - } - trace_if_enabled!("Try to fetch resource {}", sanitize_sensitive_info(resource_url)); - if let Ok(url) = Url::parse(resource_url) { - // A resource URL is chosen by playlist or EPG content, so the destination is never fetched - // blindly: an address local to this host (loopback, link-local, cloud metadata) would turn - // the proxy into a reader for the proxy host itself. Private network destinations stay - // allowed, because self-hosted services are the reason this route exists. - let Some(host) = url.host_str() else { - return StatusCode::BAD_REQUEST.into_response(); - }; - if classify_resource_destination(host).await == ResourceDestination::Blocked { - debug!("Refused resource destination local to this host: {}", sanitize_sensitive_info(resource_url)); - return StatusCode::FORBIDDEN.into_response(); - } - if let Some(resp) = - fetch_resource_with_retry(app_state, &url, fetch_policy, resource_url, &req_headers, input).await - { - return resp; - } - // Upstream failure after retries - return StatusCode::BAD_GATEWAY.into_response(); - } - error!("Url is malformed {}", sanitize_sensitive_info(resource_url)); - StatusCode::BAD_REQUEST.into_response() -} - -async fn open_media_server_image_resource( - app_state: &Arc, - resource_url: &str, -) -> Result, MediaServerError> { - let image_ref = parse_media_server_image_ref(resource_url)?; - let input_name = media_server_image_input_name(&image_ref); - let input = app_state.app_config.get_input_by_name(input_name).ok_or_else(|| { - MediaServerError::new(MediaServerErrorKind::MediaServerItemNotFound) - .provider("media-server") - .detail("media-server image input was not found") - })?; - let http_client = MediaServerHttpClient::new(app_state.http_clients.default.load().as_ref().clone()); - - let response = match input.input_type { - InputType::Plex => { - let client = input.plex_catalog_client(http_client)?; - open_media_server_proxy_image_response(&client, &image_ref).await? - } - InputType::Emby | InputType::Jellyfin => { - return Err(MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) - .provider("media-server") - .detail("media-server image proxy is not implemented for this input type")); - } - InputType::M3u - | InputType::Xtream - | InputType::M3uBatch - | InputType::XtreamBatch - | InputType::Stalker - | InputType::StalkerBatch - | InputType::Library - | InputType::Staged => { - return Err(MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) - .provider("media-server") - .detail("media-server image input is not backed by a media-server input")); - } - }; - - let mut builder = Response::builder().status(response.status); - for (key, value) in &response.headers { - if !is_hop_by_hop_response_header(key) { - builder = builder.header(key, value); - } - } - let body = response.body.map_err(|err| StreamError::Stream(err.to_string())); - builder.body(Body::from_stream(body)).map_err(|err| { - MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) - .provider("media-server") - .detail(format!("media-server image response build failed: {err}")) - }) -} - -fn media_server_image_error_status(err: &MediaServerError) -> StatusCode { - match err.kind { - MediaServerErrorKind::MediaServerItemNotFound | MediaServerErrorKind::NoDirectPlayableMediaServerSource => { - StatusCode::NOT_FOUND - } - MediaServerErrorKind::MediaServerStreamOpenFailed if is_media_server_image_validation_error(err) => { - StatusCode::BAD_REQUEST - } - MediaServerErrorKind::MediaServerStreamOpenFailed - | MediaServerErrorKind::MediaServerAuthDenied - | MediaServerErrorKind::MediaServerUnavailable - | MediaServerErrorKind::MediaServerLibraryUnavailable - | MediaServerErrorKind::MediaServerLibraryTypeUnsupported - | MediaServerErrorKind::MediaServerCatalogDecodeFailed - | MediaServerErrorKind::MediaServerCatalogPageStalled - | MediaServerErrorKind::MediaServerCatalogIncomplete - | MediaServerErrorKind::MediaServerRateLimited - | MediaServerErrorKind::MediaServerDiscoveryFailed => StatusCode::BAD_GATEWAY, - } -} - -fn is_media_server_image_validation_error(err: &MediaServerError) -> bool { - err.detail_text().is_some_and(|detail| { - detail.contains("resource URL is not a media server image URL") - || detail.contains("media server image URL is missing required path parts") - || detail.contains("unsupported media server image URL scheme") - || detail.contains("media-server image input is not backed by a media-server input") - }) -} - -fn media_server_image_input_name(image_ref: &MediaServerImageRef) -> &Arc { - match image_ref { - MediaServerImageRef::Emby { input_name, .. } - | MediaServerImageRef::Jellyfin { input_name, .. } - | MediaServerImageRef::Plex { input_name, .. } => input_name, - } -} - -pub fn separate_number_and_remainder(input: &str) -> (&str, Option<&str>) { - input.rfind('.').map_or_else( - || (input, None), - |dot_index| { - let number_part = &input[..dot_index]; - let rest = &input[dot_index..]; - (number_part, if rest.len() < 2 { None } else { Some(rest) }) - }, - ) -} - -/// # Panics -pub fn empty_json_list_response() -> axum::response::Response { - try_unwrap_body!(axum::response::Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, crate::api::static_headers::CT_JSON.clone()) - .body("[]".to_owned())) -} - -pub fn get_username_from_auth_header(token: &str, app_state: &Arc) -> Option { - let config = app_state.app_config.config.load(); - let web_auth_config = config.web_ui.as_ref()?.auth.as_ref()?; - // This hand-rolled its own `decode` with a bare `Validation::new`, which - // checks `exp` and nothing else - no issuer. - crate::auth::verify_token(token, web_auth_config.secret.as_bytes(), &web_auth_config.issuer) - .map(|token_data| token_data.claims.username) -} - -pub fn redirect(url: &str) -> impl IntoResponse { - try_unwrap_body!(axum::response::Response::builder() - .status(StatusCode::FOUND) - .header(header::LOCATION, url) - .body(Body::empty())) -} - -pub fn is_seek_request(cluster: XtreamCluster, req_headers: &HeaderMap) -> bool { - // seek only for non-live streams - if cluster == XtreamCluster::Live { - return false; - } - - // seek requests contains range header - let range = req_headers.get("range").and_then(|h| h.to_str().ok()).map(ToString::to_string); - - if let Some(range) = range { - if range.starts_with("bytes=") { - return true; - } - } - false -} - -pub fn is_seekable_media_request(cluster: XtreamCluster, req_headers: &HeaderMap, extension: Option<&str>) -> bool { - !extension.is_some_and(|ext| ext.eq_ignore_ascii_case(HLS_EXT)) && is_seek_request(cluster, req_headers) -} - -pub fn bin_response(data: &T) -> impl IntoResponse + Send { - match bin_serialize(data) { - Ok(body) => ([(header::CONTENT_TYPE, CONTENT_TYPE_CBOR)], body).into_response(), - Err(_) => internal_server_error!(), - } -} - -pub fn json_response(data: &T) -> impl IntoResponse + Send { - (StatusCode::OK, axum::Json(data)).into_response() -} - -pub fn json_or_bin_response(accept: Option<&str>, data: &T) -> impl IntoResponse + Send { - if accept.is_some_and(|a| a.contains(CONTENT_TYPE_CBOR)) { - return bin_response(data).into_response(); - } - json_response(data).into_response() -} - -pub fn stream_json_or_bin_response

( - accept: Option<&str>, - data: Box + Send>, -) -> axum::response::Response -where - P: serde::Serialize + Send + 'static, -{ - if accept.is_some_and(|a| a.contains(CONTENT_TYPE_CBOR)) { - return stream_bin_array(data); - } - stream_json_array(data) -} - -pub fn stream_json_or_bin_response_stream(accept: Option<&str>, data: S) -> axum::response::Response -where - P: serde::Serialize + Send + 'static, - S: Stream + Send + Unpin + 'static, -{ - if accept.is_some_and(|a| a.contains(CONTENT_TYPE_CBOR)) { - return stream_bin_array_stream(data); - } - stream_json_array_stream(data) -} - -pub fn stream_json_or_bin_response_try_stream(accept: Option<&str>, data: S) -> axum::response::Response -where - P: serde::Serialize + Send + 'static, - S: Stream> + Send + Unpin + 'static, - E: std::fmt::Display + Send + 'static, -{ - if accept.is_some_and(|value| value.contains(CONTENT_TYPE_CBOR)) { - return stream_bin_array_try_stream(data); - } - stream_json_array_try_stream(data) -} - -pub fn create_session_fingerprint( - fingerprint: &Fingerprint, - username: &str, - virtual_id: u32, - socket_bound: bool, -) -> String { - if socket_bound { - concat_string!(&fingerprint.addr.to_string(), "|", username, "|", &virtual_id.to_string()) - } else { - concat_string!(&fingerprint.key, "|", username, "|", &virtual_id.to_string()) - } -} - -pub(crate) fn create_playback_session_fingerprint( - fingerprint: &Fingerprint, - username: &str, - virtual_id: u32, - item_type: PlaylistItemType, - extension: Option<&str>, -) -> String { - // This scopes the session identity, not the session address-tracking policy. - // Adaptive and seekable playback use the stable client fingerprint so follow-up - // requests on new sockets reuse the same logical session. Plain live playback - // remains socket-bound so a separate connection creates a separate session. - let session_bound = is_session_based_playback(item_type, extension); - let socket_bound = !session_bound && is_socket_bound_playback_session(item_type, extension); - create_session_fingerprint(fingerprint, username, virtual_id, socket_bound) -} - -pub fn create_catchup_session_key( - fingerprint: &Fingerprint, - username: &str, - virtual_id: u32, - archive_discriminator: &str, -) -> String { - concat_string!( - "catchup|", - &fingerprint.key, - "|", - username, - "|", - &virtual_id.to_string(), - "|", - archive_discriminator.trim_matches('/') - ) -} - -pub fn create_m3u_catchup_session_key( - fingerprint: &Fingerprint, - username: &str, - virtual_id: u32, - archive_discriminator: &str, -) -> String { - concat_string!( - "m3u-catchup|", - &fingerprint.key, - "|", - username, - "|", - &virtual_id.to_string(), - "|", - archive_discriminator - ) -} - -pub(crate) fn is_session_based_playback(item_type: PlaylistItemType, extension: Option<&str>) -> bool { - item_type.is_live_adaptive() || matches!(extension, Some(ext) if ext == HLS_EXT || ext == DASH_EXT) -} - -pub(crate) fn is_socket_bound_playback_session(item_type: PlaylistItemType, extension: Option<&str>) -> bool { - item_type.uses_socket_bound_session() && !is_session_based_playback(item_type, extension) -} - -fn session_reacquire_cleanup_addrs(user_session: &UserSession, current_addr: &SocketAddr) -> Vec { - let mut addrs: SmallVec<[SocketAddr; 4]> = SmallVec::new(); - if user_session.addr != *current_addr { - addrs.push(user_session.addr); - } - for addr in &user_session.active_addrs { - if *addr != *current_addr && !addrs.contains(addr) { - addrs.push(*addr); - } - } - addrs.into_vec() -} - -pub(crate) fn should_allow_exhausted_shared_reconnect( - share_stream: bool, - user_session: Option<&UserSession>, - requested_virtual_id: u32, - requested_stream_url: &str, -) -> bool { - share_stream - && user_session.is_some_and(|session| { - session.permission != UserConnectionPermission::Exhausted - && session.virtual_id == requested_virtual_id - && session.stream_url.as_ref() == requested_stream_url - }) -} - -pub fn stream_json_array

(iter: Box + Send>) -> axum::response::Response -where - P: serde::Serialize + Send + 'static, -{ - let stream = stream::unfold((iter, true), |(mut iter, first)| async move { - match iter.next() { - Some(item) => { - let mut json = String::new(); - if !first { - json.push(','); - } - let element = serde_json::to_string(&item).ok()?; - json.push_str(&element); - Some((Ok::(Bytes::from(json)), (iter, false))) - } - None => None, - } - }); - - let body = Body::from_stream(coalesce_byte_stream( - stream::once(async { Ok::<_, Infallible>(Bytes::from_static(b"[")) }) - .chain(stream) - .chain(stream::once(async { Ok::<_, Infallible>(Bytes::from_static(b"]")) })), - )); - - try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_JSON).body(body)) -} - -pub fn stream_bin_array

(iter: Box + Send>) -> axum::response::Response -where - P: serde::Serialize + Send + 'static, -{ - let stream = stream::unfold(iter, |mut iter| async move { - match iter.next() { - Some(item) => { - match bin_serialize(&item) { - Ok(buf) => Some((Ok::(Bytes::from(buf)), iter)), - Err(err) => { - warn!("CBOR serialization error in stream: {err}"); - Some((Ok::(Bytes::new()), iter)) // skip errors, continue - } - } - } - None => None, - } - }); - - let body = Body::from_stream(coalesce_byte_stream( - stream::once(async { - // CBOR: start indefinite-length array - Ok::<_, Infallible>(Bytes::from_static(&[0x9f])) - }) - .chain(stream) - .chain(stream::once(async { - // CBOR: end indefinite-length array - Ok::<_, Infallible>(Bytes::from_static(&[0xff])) - })), - )); - - try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_CBOR).body(body)) -} - -pub fn stream_json_array_stream(stream: S) -> axum::response::Response -where - P: serde::Serialize + Send + 'static, - S: Stream + Send + Unpin + 'static, -{ - let stream = stream::unfold((stream, true), |(mut stream, first)| async move { - match stream.next().await { - Some(item) => { - let mut json = String::new(); - if !first { - json.push(','); - } - let element = serde_json::to_string(&item).ok()?; - json.push_str(&element); - Some((Ok::(Bytes::from(json)), (stream, false))) - } - None => None, - } - }); - - let body = Body::from_stream(coalesce_byte_stream( - stream::once(async { Ok::<_, Infallible>(Bytes::from_static(b"[")) }) - .chain(stream) - .chain(stream::once(async { Ok::<_, Infallible>(Bytes::from_static(b"]")) })), - )); - - try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_JSON).body(body)) -} - -fn stream_json_array_try_stream(stream: S) -> axum::response::Response -where - P: serde::Serialize + Send + 'static, - S: Stream> + Send + Unpin + 'static, - E: std::fmt::Display + Send + 'static, -{ - let stream = stream::unfold((stream, true, false), |(mut stream, first, failed)| async move { - if failed { - return None; - } - match stream.next().await { - Some(Ok(item)) => { - let serialized = serde_json::to_vec(&item).map_err(|error| error.to_string()); - let bytes = serialized.map(|serialized| { - if first { - Bytes::from(serialized) - } else { - let mut framed = Vec::with_capacity(serialized.len() + 1); - framed.push(b','); - framed.extend_from_slice(&serialized); - Bytes::from(framed) - } - }); - let failed = bytes.is_err(); - Some((bytes, (stream, false, failed))) - } - Some(Err(error)) => Some((Err(error.to_string()), (stream, first, true))), - None => None, - } - }); - - let body = Body::from_stream(coalesce_byte_stream( - stream::once(async { Ok::<_, String>(Bytes::from_static(b"[")) }) - .chain(stream) - .chain(stream::once(async { Ok::<_, String>(Bytes::from_static(b"]")) })), - )); - try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_JSON).body(body)) -} - -pub fn stream_bin_array_stream(stream: S) -> axum::response::Response -where - P: serde::Serialize + Send + 'static, - S: Stream + Send + Unpin + 'static, -{ - let stream = stream::unfold(stream, |mut stream| async move { - match stream.next().await { - Some(item) => match bin_serialize(&item) { - Ok(buf) => Some((Ok::(Bytes::from(buf)), stream)), - Err(err) => { - warn!("CBOR serialization error in stream: {err}"); - Some((Ok::(Bytes::new()), stream)) - } - }, - None => None, - } - }); - - let body = Body::from_stream(coalesce_byte_stream( - stream::once(async { Ok::<_, Infallible>(Bytes::from_static(&[0x9f])) }) - .chain(stream) - .chain(stream::once(async { Ok::<_, Infallible>(Bytes::from_static(&[0xff])) })), - )); - - try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_CBOR).body(body)) -} - -fn stream_bin_array_try_stream(stream: S) -> axum::response::Response -where - P: serde::Serialize + Send + 'static, - S: Stream> + Send + Unpin + 'static, - E: std::fmt::Display + Send + 'static, -{ - let stream = stream::unfold((stream, false), |(mut stream, failed)| async move { - if failed { - return None; - } - match stream.next().await { - Some(Ok(item)) => { - let bytes = bin_serialize(&item).map(Bytes::from).map_err(|error| error.to_string()); - let failed = bytes.is_err(); - Some((bytes, (stream, failed))) - } - Some(Err(error)) => Some((Err(error.to_string()), (stream, true))), - None => None, - } - }); - - let body = Body::from_stream(coalesce_byte_stream( - stream::once(async { Ok::<_, String>(Bytes::from_static(&[0x9f])) }) - .chain(stream) - .chain(stream::once(async { Ok::<_, String>(Bytes::from_static(&[0xff])) })), - )); - try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_CBOR).body(body)) -} - const API_STREAM_CHUNK_SIZE: usize = 64 * 1024; -pub(crate) fn coalesce_byte_stream(stream: S) -> impl Stream> -where - S: Stream> + Send + 'static, - E: Send + 'static, -{ - stream::unfold((Box::pin(stream), None, false), |(mut stream, pending_error, finished)| async move { - if let Some(error) = pending_error { - return Some((Err(error), (stream, None, true))); - } - if finished { - return None; - } - - let mut chunk = BytesMut::with_capacity(API_STREAM_CHUNK_SIZE); - loop { - match stream.next().await { - Some(Ok(bytes)) if chunk.is_empty() && bytes.len() >= API_STREAM_CHUNK_SIZE => { - return Some((Ok(bytes), (stream, None, false))); - } - Some(Ok(bytes)) => { - chunk.extend_from_slice(&bytes); - if chunk.len() >= API_STREAM_CHUNK_SIZE { - return Some((Ok(chunk.freeze()), (stream, None, false))); - } - } - Some(Err(error)) if chunk.is_empty() => { - return Some((Err(error), (stream, None, true))); - } - Some(Err(error)) => { - return Some((Ok(chunk.freeze()), (stream, Some(error), false))); - } - None if chunk.is_empty() => return None, - None => return Some((Ok(chunk.freeze()), (stream, None, true))), - } - } - }) - .fuse() -} - -pub fn create_api_proxy_user(app_state: &Arc) -> ProxyUserCredentials { - let config = app_state.app_config.config.load(); - - let server = config - .web_ui - .as_ref() - .and_then(|web_ui| web_ui.player_server.as_ref()) - .map_or("default", |server_name| server_name.as_str()); - - ProxyUserCredentials { - username: "api_user".to_string(), - password: "api_user".to_string(), - token: None, - proxy: ProxyType::Reverse(None), - server: Some(server.to_string()), - epg_timeshift: None, - epg_request_timeshift: None, - created_at: None, - exp_date: None, - max_connections: 0, - status: None, - output_clusters: shared::model::ClusterFlags::all(), - ui_enabled: false, - comment: None, - priority: 0, - soft_connections: 0, - soft_priority: 0, - t_is_api_user: true, - network_access: None, - plan: None, - filter: None, - raw_output_clusters: None, - raw_max_connections: 0, - raw_soft_connections: 0, - raw_proxy: Some(ProxyType::Reverse(None)), - t_filter: None, - t_has_unresolved_plan: false, - t_has_invalid_filter: false, - } -} - -/// The user internal recording playback runs as. Its server resolves to the -/// local API listener (`AppConfig::get_user_server_info`), and its HLS URLs -/// carry this identity to every follow-up request. -pub fn create_recording_proxy_user(app_state: &Arc) -> ProxyUserCredentials { - use base64::Engine; - use sha2::Digest; - let mut user = create_api_proxy_user(app_state); - user.username = crate::model::RECORDING_PROXY_USERNAME.to_string(); - let digest = sha2::Sha256::new() - .chain_update(b"tuliprox/internal-recording-user") - .chain_update(app_state.app_config.access_token_secret) - .finalize(); - user.password = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(&digest[..]); - user.server = None; - user -} - -/// Recording headers configure the upstream capture. They are layered under -/// the input's own headers, which keep precedence, and above the worker's -/// request headers. The worker's `User-Agent` stays on the local request so -/// the stream remains identifiable as a recording. -pub(crate) fn with_recording_headers(app_config: &AppConfig, input: Arc) -> Arc { - let config = app_config.config.load(); - let Some(recording) = config.recording() else { return input }; - let recording_headers = recording.origin_headers(&config); - if recording_headers.keys().all(|name| input.headers.keys().any(|key| key.eq_ignore_ascii_case(name.as_str()))) { - return input; - } - let mut merged = input.as_ref().clone(); - for (name, value) in recording_headers { - let Ok(value) = value.to_str() else { continue }; - if !merged.headers.keys().any(|key| key.eq_ignore_ascii_case(name.as_str())) { - merged.headers.insert(name.as_str().to_string(), value.to_string()); - } - } - Arc::new(merged) -} - -/// The input as seen by `user`: recording playback adds the recording headers. -pub(crate) fn input_for_user( - app_config: &AppConfig, - user: &ProxyUserCredentials, - input: Arc, -) -> Arc { - if user.is_recording_proxy_user() { - with_recording_headers(app_config, input) - } else { - input - } -} - -pub fn empty_json_response_as_object() -> axum::http::Result { - axum::response::Response::builder() - .status(axum::http::StatusCode::OK) - .header(axum::http::header::CONTENT_TYPE, crate::api::static_headers::CT_JSON.clone()) - .body(axum::body::Body::from("{}".as_bytes())) -} - -pub fn empty_json_response_as_array() -> axum::http::Result { - axum::response::Response::builder() - .status(axum::http::StatusCode::OK) - .header(axum::http::header::CONTENT_TYPE, crate::api::static_headers::CT_JSON.clone()) - .body(axum::body::Body::from("[]".as_bytes())) -} - #[cfg(test)] mod tests; + +mod activation; +mod auth; +mod catchup; +mod clock; +mod local_files; +mod media_server; +mod metering; +mod provider_acquire; +mod provider_urls; +mod resources; +mod response; +mod serialization; +mod session_identity; +mod stalker; +mod strategy; +mod streaming; + +#[cfg(test)] +use self::metering::resolve_stream_config_u64; +#[cfg(test)] +use self::provider_urls::find_input_account_by_signature; +#[cfg(test)] +use self::provider_urls::stream_url_matches_provider; +#[cfg(test)] +use self::response::admission_failure_video_type; +#[cfg(test)] +use self::stalker::needs_initial_stalker_resolution; +#[cfg(test)] +use self::strategy::resolve_streaming_strategy; +use self::{ + activation::{ + activate_session_before_stream_open, cleanup_forced_reopen_addrs, CurrentSessionGuard, SessionActivationRequest, + }, + catchup::{ + cleanup_failed_detected_catchup_hls, detected_catchup_hls_response, probe_catchup_payload, CatchupPayload, + DetectedCatchupHlsResponseParams, + }, + media_server::{ + is_media_server_playback_url, is_media_server_stream_ref_url, media_server_image_error_status, + open_media_server_image_resource, open_media_server_stream_for_input, + }, + metering::{ + get_stream_config_u64, get_stream_throttle, is_stream_metrics_enabled, is_throttled_stream, + prepare_stream_metering, StreamMeteringConfig, + }, + provider_acquire::{ + allows_provider_pool_failover, resolve_streaming_strategy_with_provider_handle, split_provider_stream_open, + }, + provider_urls::{get_redirect_alternative_url, resolve_xtream_vod_provider_url}, + resources::hls_resource_failure_status, + response::stream_admission_rejected_response, + session_identity::session_reacquire_cleanup_addrs, + stalker::{re_resolve_stalker_url_singleflight, should_refresh_stalker_playback, stalker_stream_kind}, + strategy::{get_grace_period_millis, should_defer_provider_open_for_grace_hold}, + streaming::{ + create_stream_response_details, force_stream_response, is_hop_by_hop_response_header, + no_custom_video_fallback_status, prepare_body_stream, try_shared_stream_response_if_any, + StreamingAcquireOptions, + }, +}; +pub use self::{ + auth::{ + create_api_proxy_user, create_recording_proxy_user, get_user_target, get_user_target_by_credentials, + get_user_target_by_username, get_username_from_auth_header, + }, + catchup::{create_catchup_session_key, create_m3u_catchup_session_key}, + clock::{get_build_time, get_server_time, get_uptime_secs, init_uptime_clock}, + local_files::serve_file, + provider_urls::get_stream_alternative_url, + resources::{resource_input_for_url, resource_proxy_response, resource_redirect_or_proxy, resource_response}, + response::{ + bin_response, empty_json_list_response, empty_json_response_as_array, empty_json_response_as_object, + json_or_bin_response, json_response, redirect, reentry_suppressed_response, separate_number_and_remainder, + }, + serialization::{ + stream_bin_array, stream_bin_array_stream, stream_json_array, stream_json_array_stream, + stream_json_or_bin_response, stream_json_or_bin_response_stream, stream_json_or_bin_response_try_stream, + }, + session_identity::create_session_fingerprint, + strategy::redirect_response, + streaming::{ + force_provider_stream_response, is_hls_stream_share_enabled, is_seek_request, is_seekable_media_request, + is_stream_share_enabled, + }, +}; +pub(crate) use self::{ + auth::{input_for_user, with_recording_headers}, + catchup::get_catchup_session_ttl_secs, + local_files::local_stream_response, + provider_acquire::{acquire_exact_provider_handle, stream_response_with_provider_handle}, + provider_urls::select_provider_stream_url, + resources::force_hls_resource_response, + response::admission_failure_response, + serialization::coalesce_byte_stream, + session_identity::{ + create_playback_session_fingerprint, get_hls_session_ttl_secs, get_session_reservation_ttl_secs, + is_session_based_playback, is_socket_bound_playback_session, should_pin_provider_for_session, + }, + stalker::resolve_initial_stalker_playback_url, + strategy::resolve_redirect_location, + streaming::{ + get_hls_playback_ttl_secs, record_connect_failed_attempt, resolve_request_url_for_logging, + resolve_stream_user_agent_index, should_allow_exhausted_shared_reconnect, stream_response, + }, +}; diff --git a/backend/app/src/api/api_utils/provider_acquire.rs b/backend/app/src/api/api_utils/provider_acquire.rs new file mode 100644 index 000000000..06e3e4772 --- /dev/null +++ b/backend/app/src/api/api_utils/provider_acquire.rs @@ -0,0 +1,870 @@ +use super::{ + activate_session_before_stream_open, cleanup_failed_detected_catchup_hls, connection_priority_for_kind, + create_stream_response_details, detected_catchup_hls_response, get_session_reservation_ttl_secs, + get_stream_options, is_socket_bound_playback_session, is_stream_share_enabled, mark_response_as_uncompressed, + no_custom_video_fallback_status, prepare_body_stream, prepare_stream_metering, probe_catchup_payload, + record_connect_failed_attempt, reentry_suppressed_response, resolve_request_url_for_logging, + resolve_xtream_vod_provider_url, select_provider_stream_url, should_pin_provider_for_session, + stream_admission_rejected_response, try_shared_stream_response_if_any, try_unwrap_body, CatchupPayload, + ConnectFailedAttempt, DetectedCatchupHlsResponseParams, ExactProviderAcquire, PlaybackRequestClass, + SessionActivationRequest, StreamResponseMode, StreamingAcquireOptions, LEASE_EXPIRY_RECHECK_FLOOR, +}; +use crate::{ + api::model::{ + create_active_client_stream, create_channel_unavailable_stream, create_custom_video_stream_response, + create_provider_connections_exhausted_stream, get_stream_response_with_headers, AppState, BoxedProviderStream, + CustomVideoStreamType, PlaybackLeaseRef, ProviderAllocation, ProviderStreamCustomReason, ProviderStreamInfo, + ProviderStreamState, SharedStreamCtx, SharedStreamManager, StreamAdmissionError, StreamingStrategy, + }, + auth::Fingerprint, + model::{AppConfig, ConfigInput, ConfigTarget, PlaybackKind, ProxyUserCredentials}, + utils::debug_if_enabled, +}; +use axum::{ + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; +use log::{debug, error, log_enabled, trace}; +use shared::{ + model::{ConnectFailureReason, FailureStage, PlaylistItemType, StreamChannel, UserConnectionPermission, VirtualId}, + utils::{extract_extension_from_url, sanitize_sensitive_info, Internable}, +}; +use std::{borrow::Cow, sync::Arc, time::Duration}; +use tuliprox_session::ProviderSessionHeaders; + +pub(super) fn create_unmapped_provider_stream(app_config: &AppConfig) -> ProviderStreamState { + ProviderStreamState::Custom { + response: create_channel_unavailable_stream(app_config, &[], StatusCode::OK), + reason: ProviderStreamCustomReason::UnmappedProviderUrl, + } +} + +/// Splits a provider open into stream, response info and session headers. An upstream error +/// of a finite HLS resource carries no stream, only its status and forwarded headers. +pub(super) fn split_provider_stream_open( + open: crate::api::model::ProviderStreamOpen, +) -> (Option, ProviderStreamInfo, ProviderSessionHeaders) { + match open { + crate::api::model::ProviderStreamOpen::Stream(response) => { + (Some(response.stream), response.info, response.provider_session_headers) + } + crate::api::model::ProviderStreamOpen::UpstreamStatus { status, headers } => { + (None, Some((headers, status, None, None)), ProviderSessionHeaders::default()) + } + } +} + +/// Acquires a slot on exactly `request.provider`, waiting at most `capacity_wait` for one to +/// free up. Only slot and lease releases of this provider wake the waiter; reserved capacity +/// held by idle leases is retried when the next lease expires. +pub(crate) async fn acquire_exact_provider_handle( + app_state: &Arc, + request: &ExactProviderAcquire<'_>, + capacity_wait: Option, +) -> Option { + let deadline = capacity_wait.map(|wait| tokio::time::Instant::now() + wait); + let acquire = || { + app_state.active_provider.acquire_exact_connection_with_lease_for_session_until( + request.provider, + request.addr, + request.allow_grace, + request.priority, + request.kind, + request.lease, + deadline, + ) + }; + let Some(deadline) = deadline else { + return acquire().await; + }; + let capacity_notify = app_state.active_provider.provider_capacity_notify(request.provider); + loop { + let notified = capacity_notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if app_state.connection_manager.is_shutting_down() || tokio::time::Instant::now() >= deadline { + return None; + } + if let Some(handle) = acquire().await { + return Some(handle); + } + let earliest_retry = tokio::time::Instant::now() + LEASE_EXPIRY_RECHECK_FLOOR; + let wake_at = app_state + .active_provider + .next_lease_expiry() + .map_or(deadline, |expiry| expiry.max(earliest_retry).min(deadline)); + tokio::select! { + () = tokio::time::sleep_until(wake_at) => {}, + () = &mut notified => {}, + } + } +} + +/// Waits for finite resources to release capacity without superseding an active body. +pub(super) async fn acquire_exact_stream_provider_handle( + app_state: &Arc, + provider: &Arc, + fingerprint: &Fingerprint, + options: &StreamingAcquireOptions<'_>, +) -> Option { + let request = ExactProviderAcquire { + provider, + addr: &fingerprint.addr, + allow_grace: options.allow_provider_grace, + priority: options.user_priority, + kind: options.connection_kind, + lease: options.session_owner.map(|owner| PlaybackLeaseRef::new(owner, options.playback_kind)), + }; + acquire_exact_provider_handle(app_state, &request, options.capacity_wait_timeout).await +} + +pub(super) async fn acquire_stream_provider_handle( + app_state: &Arc, + input: &ConfigInput, + fingerprint: &Fingerprint, + options: &StreamingAcquireOptions<'_>, +) -> Option { + let lease = options.session_owner.map(|owner| PlaybackLeaseRef::new(owner, options.playback_kind)); + let managed = |handle| tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), handle); + match options.force_provider { + Some(provider) => { + // First try to stay on the exact pinned provider account without over-allocating. + if let Some(handle) = acquire_exact_stream_provider_handle(app_state, provider, fingerprint, options).await + { + Some(managed(handle)) + } else if options.allow_forced_provider_fallback { + debug_if_enabled!( + "Pinned provider {} unavailable for {}; falling back to lineup allocation", + sanitize_sensitive_info(provider), + sanitize_sensitive_info(&fingerprint.addr.to_string()) + ); + app_state + .active_provider + .acquire_connection_with_lease_for_session_await( + &input.name, + &fingerprint.addr, + options.allow_provider_grace, + options.user_priority, + options.connection_kind, + lease, + ) + .await + .map(managed) + } else { + debug_if_enabled!( + "Pinned provider {} unavailable for {}; strict provider affinity prevents fallback", + sanitize_sensitive_info(provider), + sanitize_sensitive_info(&fingerprint.addr.to_string()) + ); + None + } + } + None => app_state + .active_provider + .acquire_connection_with_lease_for_session_await( + &input.name, + &fingerprint.addr, + options.allow_provider_grace, + options.user_priority, + options.connection_kind, + lease, + ) + .await + .map(managed), + } +} + +pub(super) fn allows_provider_pool_failover(item_type: PlaylistItemType) -> bool { + matches!( + item_type, + PlaylistItemType::Video + | PlaylistItemType::LocalVideo + | PlaylistItemType::Series + | PlaylistItemType::SeriesInfo + | PlaylistItemType::LocalSeries + | PlaylistItemType::LocalSeriesInfo + ) +} + +pub(super) async fn resolve_streaming_strategy_with_provider_handle( + app_state: &Arc, + stream_url: &str, + fingerprint: &Fingerprint, + input: &ConfigInput, + options: StreamingAcquireOptions<'_>, + stream_channel: Option<&StreamChannel>, + preacquired_provider_handle: Option, +) -> StreamingStrategy { + // Recording requests transfer the slot acquired by the worker into this + // provider-body owner. Normal playback allocates here as before. + let mut provider_connection_handle = match preacquired_provider_handle { + Some(handle) => Some(handle), + None => acquire_stream_provider_handle(app_state, input, fingerprint, &options).await, + }; + + // panel_api provisioning/loading is handled later in the stream creation flow + + let mut release_failed_mapping = false; + let stream_response_params = if let Some(allocation) = + provider_connection_handle.as_ref().and_then(|managed| managed.handle()).map(|ph| &ph.allocation) + { + match allocation { + ProviderAllocation::Exhausted => { + debug!("Provider {} is exhausted. No connections allowed.", input.name); + let stream = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); + ProviderStreamState::Custom { response: stream, reason: ProviderStreamCustomReason::ProviderExhausted } + } + ProviderAllocation::Available(ref provider_cfg) | ProviderAllocation::GracePeriod(ref provider_cfg) => { + // If a forced/pinned provider was requested but allocation fell back to another account, + // the session's stream_url still points to the old provider and cannot be accepted as-is; + // it must be resolved or rewritten for the newly allocated provider account. + let accept_requested_stream_url = (options.accept_requested_stream_url + || input.input_type.is_stalker()) + && options.force_provider.is_none_or(|forced| forced.as_ref() == provider_cfg.name.as_ref()); + // Keep the URL only when it already targets the selected provider account. Hot reload can leave old + // alias URLs in persisted playlists until the next processing run. + let selected_stream = select_provider_stream_url( + stream_url, + input, + provider_cfg, + accept_requested_stream_url, + &app_state.app_config, + ) + .await + .or_else(|| { + let url = resolve_xtream_vod_provider_url(stream_url, input, provider_cfg, stream_channel?)?; + Some((Arc::clone(&provider_cfg.name), url)) + }); + if let Some((selected_provider_name, url)) = selected_stream { + debug_if_enabled!( + "provider session: input={} provider_cfg={} user={} allocation={} stream_url={}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&provider_cfg.name), + sanitize_sensitive_info( + provider_cfg.get_user_info().as_ref().map_or_else(|| "?", |u| u.username.as_str()) + ), + allocation.short_key(), + sanitize_sensitive_info(resolve_request_url_for_logging(input, &url).as_ref()) + ); + + if matches!(allocation, ProviderAllocation::Available(_)) { + ProviderStreamState::Available(Some(selected_provider_name.intern()), url.intern()) + } else { + ProviderStreamState::GracePeriod(Some(selected_provider_name.intern()), url.intern()) + } + } else { + debug_if_enabled!( + "provider session rejected: input={} provider_cfg={} allocation={} stream_url={} reason=unmapped_provider_url", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&provider_cfg.name), + allocation.short_key(), + sanitize_sensitive_info(resolve_request_url_for_logging(input, stream_url).as_ref()) + ); + release_failed_mapping = true; + create_unmapped_provider_stream(&app_state.app_config) + } + } + } + } else { + debug!("Provider {} is exhausted. No connections allowed.", input.name); + let stream = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); + ProviderStreamState::Custom { response: stream, reason: ProviderStreamCustomReason::ProviderExhausted } + }; + + if release_failed_mapping { + // The managed owner releases the provider slot synchronously on drop; no + // task-per-drop and no lossy cleanup message. + drop(provider_connection_handle.take()); + } + + StreamingStrategy { + provider_handle: provider_connection_handle, + provider_stream_state: stream_response_params, + input_headers: Some(input.headers.clone()), + } +} + +/// # Panics +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +pub(crate) async fn stream_response_with_provider_handle( + fingerprint: &Fingerprint, + app_state: &Arc, + session_token: &str, + request_class: Option, + mut stream_channel: StreamChannel, + stream_url: &str, + pinned_provider: Option<&Arc>, + req_headers: &HeaderMap, + input: &Arc, + target: &Arc, + user: &ProxyUserCredentials, + connection_permission: UserConnectionPermission, + connection_kind: crate::api::model::ConnectionKind, + allow_exhausted_shared_reconnect: bool, + grace_mode: Option, + preacquired_provider_handle: Option, +) -> impl IntoResponse + Send { + let _transition_guard = app_state.active_users.acquire_playback_transition(&user.username, session_token).await; + let request_log_stream_url = resolve_request_url_for_logging(input, stream_url); + if log_enabled!(log::Level::Trace) { + trace!("Try to open stream {}", sanitize_sensitive_info(request_log_stream_url.as_ref())); + } + + let virtual_id = stream_channel.virtual_id; + let item_type = stream_channel.item_type; + let playback_extension = extract_extension_from_url(stream_url); + let socket_bound = is_socket_bound_playback_session(item_type, playback_extension); + let mut connection_permission = connection_permission; + let mut connection_kind = connection_kind; + let activation = activate_session_before_stream_open( + app_state, + SessionActivationRequest { + fingerprint, + input, + user, + session_token, + request_class, + virtual_id: VirtualId::new(virtual_id), + item_type, + stream_url, + connection_permission, + connection_kind, + granted_grace_mode: grace_mode, + socket_bound, + }, + ) + .await; + let grace_mode = activation.grace_mode.or(grace_mode); + connection_permission = activation.admission.permission(); + connection_kind = activation.admission.kind().unwrap_or(connection_kind); + + let allow_shared_reuse = + connection_permission != UserConnectionPermission::Exhausted || allow_exhausted_shared_reconnect; + + let share_stream = is_stream_share_enabled(item_type, target); + let _shared_lock = if share_stream { + let write_lock = app_state.app_config.file_locks.write_lock_str(stream_url).await; + + if allow_shared_reuse { + if let Some(value) = try_shared_stream_response_if_any( + app_state, + stream_url, + fingerprint, + user, + connection_permission, + connection_kind, + stream_channel.clone(), + session_token, + req_headers, + activation.placeholder_transition_version, + ) + .await + { + return value.into_response(); + } + } + Some(write_lock) + } else { + // Opportunistic cross-target sharing: if another target already runs a shared stream + // for the same provider URL, subscribe to it instead of opening a separate connection. + if item_type == PlaylistItemType::Live && allow_shared_reuse { + if let Some(value) = try_shared_stream_response_if_any( + app_state, + stream_url, + fingerprint, + user, + connection_permission, + connection_kind, + stream_channel.clone(), + session_token, + req_headers, + activation.placeholder_transition_version, + ) + .await + { + debug_if_enabled!("Opportunistic shared stream reuse for {}", sanitize_sensitive_info(stream_url)); + return value.into_response(); + } + } + None + }; + + if connection_permission == UserConnectionPermission::Exhausted { + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + activation.placeholder_transition_version, + activation.placeholder_transition_version.is_some(), + ) + .await; + if activation.admission.is_reentry_suppressed() { + return reentry_suppressed_response(); + } + record_connect_failed_attempt(ConnectFailedAttempt { + app_state, + fingerprint, + user, + stream_channel: stream_channel.clone(), + provider_name: input.name.clone(), + req_headers, + reason: ConnectFailureReason::UserConnectionsExhausted, + failure_stage: FailureStage::Admission, + }); + return create_custom_video_stream_response( + &app_state.provider_stream_ctx(), + &fingerprint.addr, + CustomVideoStreamType::UserConnectionsExhausted, + ) + .into_response(); + } + + let stream_options = get_stream_options(&app_state.app_config, StreamResponseMode::Stream); + let session_state = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; + let pinned_provider = pinned_provider.filter(|provider| { + item_type.is_live() + || session_state + .as_ref() + .is_some_and(|session| session.media_started.load(std::sync::atomic::Ordering::Acquire)) + || app_state.active_provider.should_reuse_playback_provider(session_token, provider) + }); + let provider_session_headers = + session_state.as_ref().and_then(|session| session.provider_session_headers_for(stream_url)); + let mut stream_details = match create_stream_response_details( + app_state, + &stream_options, + stream_url, + &user.username, + fingerprint, + req_headers, + input, + &stream_channel, + item_type, + if item_type == PlaylistItemType::Catchup { + crate::api::model::ProviderContentRepresentationMode::Identity + } else { + crate::api::model::ProviderContentRepresentationMode::PreserveOrigin + }, + share_stream, + connection_permission, + pinned_provider, + pinned_provider.is_none() || allows_provider_pool_failover(item_type), + true, + VirtualId::new(stream_channel.virtual_id), + connection_priority_for_kind(user, connection_kind), + connection_kind, + false, + Some(session_token), + provider_session_headers.as_deref(), + pinned_provider.is_some(), + grace_mode.map(|m| matches!(m, crate::api::model::GraceMode::Hold)), + activation.grace_context.clone(), + None, + preacquired_provider_handle, + ) + .await + { + Ok(stream_details) => stream_details, + Err(err) => { + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + activation.placeholder_transition_version, + activation.placeholder_transition_version.is_some(), + ) + .await; + error!("Failed to stream: {err}"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + + if item_type == PlaylistItemType::Catchup { + if let Some(provider_stream) = stream_details.stream.take() { + let probe_deadline = Duration::from_millis(app_state.hls.proxy.origin_manifest_timeout_ms().max(1)); + match probe_catchup_payload(provider_stream, probe_deadline).await { + Ok(CatchupPayload::Direct(provider_stream)) => stream_details.stream = Some(provider_stream), + Ok(CatchupPayload::HlsManifest(manifest)) => { + return detected_catchup_hls_response(DetectedCatchupHlsResponseParams { + app_state, + stream_details, + manifest, + user, + target, + input, + fingerprint, + session_token, + virtual_id: VirtualId::new(virtual_id), + connection_permission, + connection_kind, + fallback_stream_url: stream_url, + }) + .await; + } + Err(err) => { + error!("Failed to inspect catch-up payload: {err}"); + cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token) + .await; + return StatusCode::BAD_GATEWAY.into_response(); + } + } + } + } + + // When no provider stream is available, still create an ActiveClientStream if a grace period + // needs to resolve (provider-grace with hold_stream, or user-grace). The grace task will + // determine the correct mode (UserExhausted / ProviderExhausted / Inner) and serve the + // appropriate custom video or terminate cleanly. + let deferred_grace_hold_stream = + stream_details.has_deferred_provider_open() || connection_permission == UserConnectionPermission::GracePeriod; + + if stream_details.has_stream() || deferred_grace_hold_stream { + // let content_length = get_stream_content_length(provider_response.as_ref()); + let provider_response = stream_details + .stream_info + .as_ref() + .map(|(h, sc, response_url, cvt)| (h.clone(), *sc, response_url.clone(), *cvt)); + let provider_name = stream_details.provider_name.clone(); + let actual_request_url = stream_details.request_url.clone().unwrap_or_else(|| Arc::::from(stream_url)); + let log_actual_request_url = resolve_request_url_for_logging(input, actual_request_url.as_ref()); + + debug_if_enabled!( + "Provider request mapping: allocated_provider={} actual_request_url={}", + sanitize_sensitive_info(provider_name.as_deref().unwrap_or("?")), + sanitize_sensitive_info(log_actual_request_url.as_ref()) + ); + + if let Some((headers, status, _response_url, Some(CustomVideoStreamType::Provisioning))) = + stream_details.stream_info.as_ref() + { + debug_if_enabled!("panel_api provisioning response to client: status={} headers={:?}", status, headers); + } + + // Captured before `stream_details` is moved into `create_active_client_stream`. + // The pinning rule is centralized in `should_pin_provider_for_session` so it stays + // testable in isolation and in sync with the call site below. + let should_pin_provider = should_pin_provider_for_session(&stream_details, app_state, item_type); + let user_agent_stream_index = stream_details.user_agent_stream_index; + // Persist this before response construction because some item types skip the later + // create_user_session path, including placeholder sessions created by ensure_user_session_placeholder. + if let Some(stream_index) = user_agent_stream_index { + app_state + .active_users + .set_user_agent_stream_index_if_absent(&user.username, session_token, stream_index) + .await; + } + + let mut is_stream_shared = share_stream && !stream_details.has_deferred_provider_open(); + if let Some((_header, _status_code, _url, Some(_custom_video))) = stream_details.stream_info.as_ref() { + if stream_details.stream.is_some() { + is_stream_shared = false; + } + } + let shared_subscriber_id = + tuliprox_core::model::SharedSubscriberId::from_stream_uid(app_state.connection_manager.next_stream_uid()); + let mut pending_shared_cleanup = if is_stream_shared { + match SharedStreamManager::reserve_subscriber_cleanup( + &app_state.connection_manager, + shared_subscriber_id, + fingerprint.addr, + ) + .await + { + Ok(cleanup) => Some(cleanup), + Err(reason) => { + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + activation.placeholder_transition_version, + activation.placeholder_transition_version.is_some(), + ) + .await; + return stream_admission_rejected_response(reason.into(), &user.username); + } + } + } else { + None + }; + let mut metering = prepare_stream_metering( + app_state, + stream_url, + is_stream_shared, + stream_details.stream.is_some(), + stream_details.has_deferred_provider_open(), + ); + if let (Some(cleanup), Some(request_id)) = ( + pending_shared_cleanup.as_mut(), + stream_details + .provider_handle + .as_ref() + .and_then(|managed| managed.handle()) + .and_then(|handle| handle.playback_request_id), + ) { + cleanup.set_provider_request_identity(session_token, request_id); + } + let provider_handle = if is_stream_shared && !stream_details.has_deferred_provider_open() { + // Transfer the ManagedProviderHandle to the shared-stream manager. + // Ownership stays with the managed type across all awaits; the shared-stream + // manager disarms it only once it holds the write lock and commits to a new + // shared origin. If the future is cancelled before that point the Drop guard + // releases the slot. + stream_details.provider_handle.take() + } else { + None + }; + + stream_channel.shared = is_stream_shared; + if is_stream_shared { + stream_channel.shared_joined_existing = Some(false); + stream_channel.shared_stream_id = Some(u64::from(metering.meter_uid)); + } else { + stream_channel.shared_joined_existing = None; + stream_channel.shared_stream_id = None; + } + if is_stream_shared { + stream_details.shared_subscriber_id = + pending_shared_cleanup.as_ref().map(tuliprox_session::PendingSharedSubscriberCleanup::capability); + } + // In the no-limits path there may be no placeholder yet. The body needs the + // session's media flag before its first byte; create that session now. + let created_media_session = if !is_stream_shared + && !item_type.is_live() + && item_type.requires_provider_affinity() + && app_state.active_users.media_started_flag(&user.username, session_token).await.is_none() + { + if let Some(provider) = provider_name.as_deref() { + app_state + .active_users + .ensure_user_session_placeholder(crate::api::model::CreateUserSessionParams { + user, + session_token, + virtual_id, + provider, + stream_url: actual_request_url.as_ref(), + addr: &fingerprint.addr, + connection_permission, + connection_kind: Some(connection_kind), + socket_bound, + }) + .await; + true + } else { + false + } + } else { + false + }; + let stream = match create_active_client_stream(crate::api::model::ActiveClientStreamParams { + stream_details, + app_state, + user, + connection_permission, + connection_kind, + fingerprint, + stream_channel, + socket_bound, + session_token: Some(session_token), + req_headers, + meter_uid: metering.meter_uid, + meter_stream: metering.meter_stream, + }) + .await + { + Ok(stream) => stream, + Err(error) => { + if created_media_session { + app_state.active_users.terminate_session(&user.username, session_token).await; + } else { + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + activation.placeholder_transition_version, + activation.placeholder_transition_version.is_some(), + ) + .await; + } + return stream_admission_rejected_response(error, &user.username); + } + }; + let stream_resp = if is_stream_shared { + debug_if_enabled!( + "Streaming shared stream request from {}", + sanitize_sensitive_info(log_actual_request_url.as_ref()) + ); + let Some(pending_shared_cleanup) = pending_shared_cleanup else { + return stream_admission_rejected_response(StreamAdmissionError::RegistrationRejected, &user.username); + }; + // Shared Stream response + let shared_headers = provider_response.as_ref().map_or_else(Vec::new, |(h, _, _, _)| h.clone()); + if let Some((broadcast_stream, _shared_provider, _cleanup_capability)) = + SharedStreamManager::register_shared_stream( + SharedStreamCtx { + app_config: &app_state.app_config, + shared_stream_manager: &app_state.shared_stream_manager, + active_provider: &app_state.active_provider, + connection_manager: &app_state.connection_manager, + }, + stream_url, + stream, + &fingerprint.addr, + shared_subscriber_id, + shared_headers, + stream_options.buffer_size, + provider_handle, + pending_shared_cleanup, + connection_priority_for_kind(user, connection_kind), + connection_kind, + ) + .await + { + metering.commit_shared_registration(); + let (status_code, header_map) = + get_stream_response_with_headers(provider_response.map(|(h, s, _, _)| (h, s))); + let mut response = axum::response::Response::builder().status(status_code); + for (key, value) in &header_map { + response = response.header(key, value); + } + let mut response = try_unwrap_body!(response.body(axum::body::Body::from_stream(broadcast_stream))); + mark_response_as_uncompressed(&mut response); + response + } else { + StatusCode::BAD_REQUEST.into_response() + } + } else { + // Previously, we always persisted the provider's final request URL into the session. + // For VOD-like playback that can be the wrong thing to reuse later: a seek or reopen + // should start from the canonical playback entrypoint, not from a provider-specific + // redirected target that happened to be used for an earlier request. + // For Movies/Series/Catchup we therefore keep the canonical request URL in the session. + // That avoids "session poisoning" where later seeks/resumes inherit a non-canonical URL. + // For live playback we still keep the redirected URL when available, because staying on + // the chosen upstream edge/server is often desirable there. + let session_url: Cow<'_, str> = if matches!( + item_type, + PlaylistItemType::Catchup + | PlaylistItemType::Video + | PlaylistItemType::LocalVideo + | PlaylistItemType::Series + | PlaylistItemType::LocalSeries + | PlaylistItemType::SeriesInfo + | PlaylistItemType::LocalSeriesInfo + ) { + Cow::Owned(actual_request_url.to_string()) + } else { + provider_response + .as_ref() + .and_then(|(_, _, u, _)| u.as_ref()) + .map_or_else(|| Cow::Owned(actual_request_url.to_string()), |url| Cow::Owned(url.to_string())) + }; + let log_session_url = resolve_request_url_for_logging(input, session_url.as_ref()); + if log_enabled!(log::Level::Debug) { + if log_session_url.eq(log_actual_request_url.as_ref()) { + debug!( + "Streaming stream request from {}", + sanitize_sensitive_info(log_actual_request_url.as_ref()) + ); + } else { + debug!( + "Streaming stream request for {} from {}", + sanitize_sensitive_info(log_actual_request_url.as_ref()), + sanitize_sensitive_info(log_session_url.as_ref()) + ); + } + } + let (status_code, header_map) = + get_stream_response_with_headers(provider_response.map(|(h, s, _, _)| (h, s))); + let mut response = axum::response::Response::builder().status(status_code); + for (key, value) in &header_map { + response = response.header(key, value); + } + + if let Some(provider) = provider_name { + if matches!( + item_type, + PlaylistItemType::LiveHls + | PlaylistItemType::LiveDash + | PlaylistItemType::Video + | PlaylistItemType::Series + | PlaylistItemType::SeriesInfo + | PlaylistItemType::LocalSeries + | PlaylistItemType::LocalSeriesInfo + | PlaylistItemType::Catchup + ) { + let _ = app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user, + session_token, + virtual_id, + provider: &provider, + stream_url: &session_url, + addr: &fingerprint.addr, + connection_permission, + connection_kind: Some(connection_kind), + socket_bound, + }) + .await; + if let Some(stream_index) = user_agent_stream_index { + app_state + .active_users + .set_user_agent_stream_index_if_absent(&user.username, session_token, stream_index) + .await; + } + if should_pin_provider { + let reservation_ttl_secs = get_session_reservation_ttl_secs(app_state, item_type); + if reservation_ttl_secs > 0 { + app_state.active_provider.refresh_adaptive_playback_lease( + &provider, + session_token, + PlaybackKind::classify(item_type, playback_extension), + reservation_ttl_secs, + ); + } + } + } + } + + let body_stream = prepare_body_stream(app_state, item_type, stream); + let mut response = try_unwrap_body!(response.body(body_stream)); + mark_response_as_uncompressed(&mut response); + response + }; + + return stream_resp.into_response(); + } + app_state.connection_manager.release_managed_provider_handle(stream_details.provider_handle); + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + activation.placeholder_transition_version, + activation.placeholder_transition_version.is_some(), + ) + .await; + if stream_details.custom_reason == Some(ProviderStreamCustomReason::ProviderExhausted) { + record_connect_failed_attempt(ConnectFailedAttempt { + app_state, + fingerprint, + user, + stream_channel, + provider_name: stream_details.provider_name.unwrap_or_else(|| input.name.clone()), + req_headers, + reason: ConnectFailureReason::ProviderConnectionsExhausted, + failure_stage: FailureStage::Admission, + }); + return create_custom_video_stream_response( + &app_state.provider_stream_ctx(), + &fingerprint.addr, + CustomVideoStreamType::ProviderConnectionsExhausted, + ) + .into_response(); + } + no_custom_video_fallback_status(&app_state.app_config).into_response() +} diff --git a/backend/app/src/api/api_utils/provider_urls.rs b/backend/app/src/api/api_utils/provider_urls.rs new file mode 100644 index 000000000..779fea732 --- /dev/null +++ b/backend/app/src/api/api_utils/provider_urls.rs @@ -0,0 +1,614 @@ +use super::resolve_request_url_for_logging; +use crate::{ + api::model::{AppState, ProviderConfig}, + model::{AppConfig, ConfigInput, InputUserInfo}, + repository::{load_first_input_m3u_stream_url_by_keys, load_input_m3u_stream_url}, + utils::debug_if_enabled, +}; +use shared::{ + defaults::HLS_EXT, + model::{InputType, PlaylistItemType, StreamChannel}, + utils::{ + extract_extension_from_url, get_credentials_from_url, is_account_query_key, m3u_flussonic_live_file_lookup_key, + m3u_stream_url_identity, sanitize_sensitive_info, + }, +}; +use std::sync::Arc; +use url::Url; + +pub fn get_stream_alternative_url( + stream_url: &str, + input: &ConfigInput, + alias_input: &Arc, +) -> Option { + if input.input_type.is_m3u() && input.get_matched_config_by_url(stream_url).is_none() { + return get_stream_alternative_url_m3u(stream_url, input, alias_input); + } + + let (source_base_url, source_username, source_password, matched_via_external_signature) = + if let Some(matched) = input.get_matched_config_by_url(stream_url) { + (matched.0.to_string(), matched.1.cloned(), matched.2.cloned(), false) + } else { + let (base_url, username, password) = find_input_account_by_signature(stream_url, input)?; + (base_url, username, password, true) + }; + if matched_via_external_signature && !input.input_type.is_m3u() { + return None; + } + let alt_input_user_info = alias_input.get_user_info()?; + + let modified = stream_url.replacen(&source_base_url, &alt_input_user_info.base_url, 1); + let mut url = Url::parse(&modified).ok()?; + + if let (Some(old_username), Some(old_password)) = (source_username, source_password) { + let auth_updated = rewrite_url_auth_fields( + &mut url, + &old_username, + &old_password, + &alt_input_user_info.username, + &alt_input_user_info.password, + ); + if !auth_updated { + return None; + } + } + + Some(url.to_string()) +} + +pub(super) fn get_stream_alternative_url_m3u( + stream_url: &str, + input: &ConfigInput, + alias_input: &Arc, +) -> Option { + if let Some(source_config_url) = find_input_account_by_query_signature(stream_url, input) { + return rewrite_account_query_fields(stream_url, source_config_url, &alias_input.url); + } + + if let Some((source_base_url, source_username, source_password)) = + find_input_account_by_signature(stream_url, input) + { + let Some(alt_input_user_info) = alias_input.get_user_info() else { + return Some(stream_url.to_string()); + }; + let modified = stream_url.replacen(&source_base_url, &alt_input_user_info.base_url, 1); + let mut url = Url::parse(&modified).ok()?; + + if let (Some(old_username), Some(old_password)) = (source_username, source_password) { + let auth_updated = rewrite_url_auth_fields( + &mut url, + &old_username, + &old_password, + &alt_input_user_info.username, + &alt_input_user_info.password, + ); + if !auth_updated { + return None; + } + } + + return Some(url.to_string()); + } + let Some(alt_input_user_info) = alias_input.get_user_info() else { + let Ok(url) = Url::parse(stream_url) else { + return None; + }; + if providerless_m3u_url_has_explicit_credentials(&url) { + return None; + } + return Some(stream_url.to_string()); + }; + if stream_url_has_account_signature(stream_url, &alt_input_user_info) { + return None; + } + Some(stream_url.to_string()) +} + +pub(super) fn find_input_account_by_query_signature<'a>(stream_url: &str, input: &'a ConfigInput) -> Option<&'a str> { + if stream_url_account_query_matches(stream_url, &input.url) { + return Some(&input.url); + } + + input.aliases.as_ref().and_then(|aliases| { + aliases + .iter() + .find(|alias| stream_url_account_query_matches(stream_url, &alias.url)) + .map(|alias| alias.url.as_str()) + }) +} + +pub(super) fn stream_url_account_query_matches(stream_url: &str, config_url: &str) -> bool { + let (Ok(stream_url), Ok(config_url)) = (Url::parse(stream_url), Url::parse(config_url)) else { + return false; + }; + + config_url.query_pairs().any(|(config_key, config_value)| { + is_account_query_key(&config_key) + && stream_url.query_pairs().any(|(stream_key, stream_value)| { + stream_key.eq_ignore_ascii_case(&config_key) && stream_value == config_value + }) + }) +} + +pub(super) fn rewrite_account_query_fields( + stream_url: &str, + source_config_url: &str, + target_config_url: &str, +) -> Option { + let mut stream_url = Url::parse(stream_url).ok()?; + let source_config_url = Url::parse(source_config_url).ok()?; + let target_config_url = Url::parse(target_config_url).ok()?; + let source_pairs: Vec<_> = source_config_url + .query_pairs() + .filter(|(key, _)| is_account_query_key(key)) + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); + let target_pairs: Vec<_> = target_config_url + .query_pairs() + .filter(|(key, _)| is_account_query_key(key)) + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); + let unambiguous_cross_key_target = + if source_pairs.len() == 1 && target_pairs.len() == 1 { target_pairs.first() } else { None }; + let present_source_pairs: Vec<_> = source_pairs + .iter() + .filter(|(source_key, source_value)| { + stream_url.query_pairs().any(|(stream_key, stream_value)| { + stream_key.eq_ignore_ascii_case(source_key) && stream_value == source_value.as_str() + }) + }) + .collect(); + let replacements: Vec<_> = present_source_pairs + .iter() + .filter_map(|(source_key, source_value)| { + target_pairs + .iter() + .find(|(target_key, _)| target_key.eq_ignore_ascii_case(source_key)) + .or(unambiguous_cross_key_target) + .map(|(target_key, target_value)| { + (source_key.clone(), source_value.clone(), target_key.clone(), target_value.clone()) + }) + }) + .collect(); + + if replacements.len() != present_source_pairs.len() || replacements.is_empty() { + return None; + } + + let mut replaced = false; + let stream_pairs: Vec<(String, String)> = stream_url + .query_pairs() + .map(|(key, value)| { + if let Some((_, _, target_key, target_value)) = + replacements.iter().find(|(source_key, source_value, _, _)| { + key.eq_ignore_ascii_case(source_key) && value == source_value.as_str() + }) + { + replaced = true; + (target_key.clone(), target_value.clone()) + } else { + (key.into_owned(), value.into_owned()) + } + }) + .collect(); + + if !replaced { + return None; + } + stream_url + .query_pairs_mut() + .clear() + .extend_pairs(stream_pairs.iter().map(|(key, value)| (key.as_str(), value.as_str()))); + Some(stream_url.to_string()) +} + +pub(super) fn providerless_m3u_url_has_explicit_credentials(url: &Url) -> bool { + !url.username().is_empty() + || url.password().is_some() + || url + .query_pairs() + .any(|(key, _)| key.eq_ignore_ascii_case("username") || key.eq_ignore_ascii_case("password")) +} + +/// Look for an account signature in the stream URL that matches the input +/// itself or one of its configured aliases. Returns the matching entry's +/// `(base_url, username, password)` so the caller can rewrite only the +/// account-specific parts of the URL while preserving the original host/path. +/// +/// This helper is used for safe credential rewrites when Tuliprox switches +/// from one account to another. It is not the general trust gate for M3U +/// foreign hosts: plain external URLs from a stored M3U playlist item may be +/// accepted without a matching signature, while unrelated credential-bearing +/// URLs still fail closed unless they provably match the input or one of its +/// aliases. +pub(super) fn find_input_account_by_signature( + stream_url: &str, + input: &ConfigInput, +) -> Option<(String, Option, Option)> { + // Try the input's main account first. + if let Some(user_info) = input.get_user_info() { + if stream_url_account_matches(stream_url, &user_info) { + return Some((input.url.clone(), Some(user_info.username), Some(user_info.password))); + } + } + // Then try each alias, if any. The input_type is inherited from the + // parent input for all aliases — see ConfigInputAlias definition. + if let Some(aliases) = input.aliases.as_ref() { + for alias in aliases { + if let Some(user_info) = + InputUserInfo::new(input.input_type, alias.username.as_deref(), alias.password.as_deref(), &alias.url) + { + if stream_url_account_matches(stream_url, &user_info) { + return Some((alias.url.clone(), Some(user_info.username), Some(user_info.password))); + } + } + } + } + None +} + +pub(super) fn rewrite_url_auth_fields( + url: &mut Url, + old_username: &str, + old_password: &str, + new_username: &str, + new_password: &str, +) -> bool { + if rewrite_query_auth_fields(url, new_username, new_password) { + return true; + } + + if url.username() == old_username && url.password() == Some(old_password) { + return url.set_username(new_username).is_ok() && url.set_password(Some(new_password)).is_ok(); + } + + rewrite_path_auth_fields(url, old_username, old_password, new_username, new_password) +} + +pub(super) fn rewrite_query_auth_fields(url: &mut Url, new_username: &str, new_password: &str) -> bool { + let mut has_username = false; + let mut has_password = false; + let pairs: Vec<(String, String)> = url + .query_pairs() + .map(|(key, value)| { + if key.eq_ignore_ascii_case("username") { + has_username = true; + (key.into_owned(), new_username.to_string()) + } else if key.eq_ignore_ascii_case("password") { + has_password = true; + (key.into_owned(), new_password.to_string()) + } else { + (key.into_owned(), value.into_owned()) + } + }) + .collect(); + + if !(has_username && has_password) { + return false; + } + + url.query_pairs_mut().clear().extend_pairs(pairs.iter().map(|(key, value)| (key.as_str(), value.as_str()))); + true +} + +pub(super) fn collect_path_segments(url: &Url) -> Option> { + url.path_segments().map(|segments| segments.map(ToOwned::to_owned).collect::>()) +} + +pub(super) fn find_path_auth_segment_index(segments: &[String], username: &str, password: &str) -> Option { + segments.windows(2).position(|pair| { + pair.first().is_some_and(|segment| segment == username) + && pair.get(1).is_some_and(|segment| segment == password) + }) +} + +pub(super) fn rewrite_path_auth_fields( + url: &mut Url, + old_username: &str, + old_password: &str, + new_username: &str, + new_password: &str, +) -> bool { + let Some(mut segments) = collect_path_segments(url) else { + return false; + }; + + let credential_index = find_path_auth_segment_index(&segments, old_username, old_password); + let Some(credential_index) = credential_index else { + return false; + }; + + segments[credential_index] = new_username.to_string(); + segments[credential_index + 1] = new_password.to_string(); + + let Ok(mut path_segments) = url.path_segments_mut() else { + return false; + }; + path_segments.clear().extend(segments.iter().map(String::as_str)); + true +} + +pub(super) fn stream_url_matches_provider(stream_url: &str, provider_cfg: &ProviderConfig) -> bool { + let account_query_matches = + provider_cfg.input_type.is_m3u() && stream_url_account_query_matches(stream_url, &provider_cfg.url); + let Some(user_info) = provider_cfg.get_user_info() else { + return account_query_matches; + }; + if stream_url_base_matches(stream_url, &user_info.base_url) { + // Same-host fast path: both base URL and account identity must match. + return stream_url_account_matches(stream_url, &user_info); + } + if !provider_cfg.input_type.is_m3u() { + return false; + } + // For M3U inputs, the stored playlist entry itself is the trust anchor. + // Open external URLs are therefore allowed, but external URLs that carry + // explicit account markers must still match the selected provider account. + if stream_url_has_account_signature(stream_url, &user_info) { + return stream_url_account_matches(stream_url, &user_info); + } + true +} + +pub(super) fn stream_url_base_matches(stream_url: &str, base_url: &str) -> bool { + stream_url + .strip_prefix(base_url) + .is_some_and(|remaining| remaining.is_empty() || remaining.starts_with(['/', '?', '#'])) +} + +pub(super) fn stream_url_account_matches(stream_url: &str, user_info: &crate::model::InputUserInfo) -> bool { + let Ok(url) = Url::parse(stream_url) else { + return false; + }; + + let (url_username, url_password) = get_credentials_from_url(&url); + if let (Some(url_username), Some(url_password)) = (url_username.as_deref(), url_password.as_deref()) { + return url_username == user_info.username && url_password == user_info.password; + } + + let mut has_query_username = false; + let mut has_query_password = false; + for (key, value) in url.query_pairs() { + if key.eq_ignore_ascii_case("username") { + has_query_username = value == user_info.username; + } else if key.eq_ignore_ascii_case("password") { + has_query_password = value == user_info.password; + } + } + if has_query_username || has_query_password { + return has_query_username && has_query_password; + } + + let Some(segments) = collect_path_segments(&url) else { + return false; + }; + + find_path_auth_segment_index(&segments, &user_info.username, &user_info.password).is_some() +} + +pub(super) fn stream_url_has_account_signature(stream_url: &str, user_info: &crate::model::InputUserInfo) -> bool { + let Ok(url) = Url::parse(stream_url) else { + return false; + }; + + let (url_username, url_password) = get_credentials_from_url(&url); + if url_username.is_some() && url_password.is_some() { + return true; + } + + let mut has_query_username = false; + let mut has_query_password = false; + for (key, _) in url.query_pairs() { + if key.eq_ignore_ascii_case("username") { + has_query_username = true; + } else if key.eq_ignore_ascii_case("password") { + has_query_password = true; + } + } + if has_query_username || has_query_password { + return has_query_username && has_query_password; + } + + // Path-based credentials: some Xtream endpoints embed the account in the URL + // path (e.g. /live///...). Only flag a signature when the + // consecutive segments actually match the configured user/pass — arbitrary + // open paths must not be treated as account signatures. + if let Some(segments) = collect_path_segments(&url) { + if find_path_auth_segment_index(&segments, &user_info.username, &user_info.password).is_some() { + return true; + } + } + + false +} + +/// Maps a Flussonic archive URL of another account onto the alias account. +/// +/// The URL index only holds live URLs, so an archive URL derived from the main account's +/// live URL never matches. The archive lives next to the channel's live file, so the alias +/// URL of that sibling supplies the alias credentials and the archive file name is kept. +pub(super) async fn resolve_m3u_alias_flussonic_archive_url( + app_config: &Arc, + provider_name: &Arc, + stream_url: &str, +) -> Option { + let requested = Url::parse(stream_url).ok()?; + let archive_file = requested.path_segments()?.next_back()?; + crate::iptv::m3u::parse_flussonic_archive_file(archive_file)?; + // The live file the archive was derived from is probed first: same stem, then same transport. + let archive_lower = archive_file.to_ascii_lowercase(); + let archive_is_hls = archive_lower.ends_with(HLS_EXT); + let archive_stem = archive_lower.split_once('-').map(|(stem, _)| stem); + let mut live_files = crate::iptv::m3u::FLUSSONIC_LIVE_FILES; + live_files.sort_by_key(|live_file| { + let same_stem = + archive_stem.is_some_and(|stem| live_file.split_once('.').is_some_and(|(name, _)| name == stem)); + let same_transport = live_file.ends_with(HLS_EXT) == archive_is_hls; + (!same_stem, !same_transport) + }); + // Each sibling is probed by its exact identity, then by its case-folded Flussonic key. + let keys: Vec = live_files + .iter() + .filter_map(|live_file| { + let mut sibling = requested.clone(); + sibling.path_segments_mut().ok()?.pop().push(live_file); + Some([m3u_stream_url_identity(sibling.as_str()), m3u_flussonic_live_file_lookup_key(sibling.as_str())]) + }) + .flatten() + .flatten() + .collect(); + let alias_live_url = match load_first_input_m3u_stream_url_by_keys(app_config, provider_name, keys).await { + Ok(url) => url?, + Err(err) => { + debug_if_enabled!( + "Failed to resolve M3U archive sibling URL for provider {}: {}", + sanitize_sensitive_info(provider_name), + sanitize_sensitive_info(&err.to_string()) + ); + return None; + } + }; + let mut alias_url = Url::parse(&alias_live_url).ok()?; + alias_url.path_segments_mut().ok()?.pop().push(archive_file); + Some(alias_url.into()) +} + +pub(crate) async fn select_provider_stream_url( + stream_url: &str, + input: &ConfigInput, + provider_cfg: &Arc, + accept_requested_stream_url: bool, + app_config: &Arc, +) -> Option<(Arc, String)> { + if accept_requested_stream_url { + return Some((provider_cfg.name.clone(), stream_url.to_string())); + } + + if provider_cfg.input_type.is_m3u() { + match load_input_m3u_stream_url(app_config, &provider_cfg.name, stream_url).await { + Ok(Some(provider_stream_url)) => { + debug_if_enabled!( + "M3U alias URL lookup: provider={} requested_url={} index_match=true resolved_url={}", + sanitize_sensitive_info(&provider_cfg.name), + sanitize_sensitive_info(resolve_request_url_for_logging(input, stream_url).as_ref()), + sanitize_sensitive_info(resolve_request_url_for_logging(input, &provider_stream_url).as_ref()) + ); + return Some((provider_cfg.name.clone(), provider_stream_url.to_string())); + } + Ok(None) => { + if let Some(provider_stream_url) = + resolve_m3u_alias_flussonic_archive_url(app_config, &provider_cfg.name, stream_url).await + { + debug_if_enabled!( + "M3U alias URL lookup: provider={} requested_url={} index_match=archive_sibling resolved_url={}", + sanitize_sensitive_info(&provider_cfg.name), + sanitize_sensitive_info(resolve_request_url_for_logging(input, stream_url).as_ref()), + sanitize_sensitive_info(resolve_request_url_for_logging(input, &provider_stream_url).as_ref()) + ); + return Some((provider_cfg.name.clone(), provider_stream_url)); + } + debug_if_enabled!( + "M3U alias URL lookup: provider={} requested_url={} index_match=false", + sanitize_sensitive_info(&provider_cfg.name), + sanitize_sensitive_info(resolve_request_url_for_logging(input, stream_url).as_ref()) + ); + } + Err(err) => { + debug_if_enabled!( + "Failed to resolve M3U stream URL for provider {}: {}", + sanitize_sensitive_info(&provider_cfg.name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + } + + if stream_url_matches_provider(stream_url, provider_cfg) { + Some((provider_cfg.name.clone(), stream_url.to_string())) + } else { + get_stream_alternative_url(stream_url, input, provider_cfg).map(|url| (provider_cfg.name.clone(), url)) + } +} + +/// Stored Xtream VOD entries belong to their named input even when their URL uses a CDN host. +pub(super) fn resolve_xtream_vod_provider_url( + stream_url: &str, + input: &ConfigInput, + provider_cfg: &ProviderConfig, + channel: &StreamChannel, +) -> Option { + if input.input_type != InputType::Xtream + || provider_cfg.input_type != InputType::Xtream + || channel.item_type != PlaylistItemType::Video + || channel.input_name != input.name + || channel.url.as_ref() != stream_url + || channel.provider_id == 0 + { + return None; + } + let url = Url::parse(stream_url).ok()?; + if !matches!(url.scheme(), "http" | "https") { + return None; + } + let is_main = provider_cfg.name == input.name; + if !is_main + && !input.aliases.as_ref().is_some_and(|aliases| aliases.iter().any(|alias| alias.name == provider_cfg.name)) + { + return None; + } + let selected = provider_cfg.get_user_info()?; + let origin = input.get_user_info()?; + if is_main + && selected.base_url == origin.base_url + && selected.username == origin.username + && selected.password == origin.password + { + return Some(stream_url.to_string()); + } + + // A direct source may carry account-bound tokens. A different account obtains its own redirect. + let extension = channel + .technical + .as_ref() + .map(|technical| technical.container.as_str()) + .filter(|container| !container.is_empty()) + .map(|container| match container { + "mpegts" => "ts", + "hls" => "m3u8", + "dash" => "mpd", + other => other, + }) + .or_else(|| extract_extension_from_url(&channel.url).and_then(|extension| extension.strip_prefix('.'))); + let base_url = selected.base_url.trim_end_matches('/'); + let mut url = format!("{base_url}/movie/{}/{}/{}", selected.username, selected.password, channel.provider_id); + if let Some(extension) = extension { + url.push('.'); + url.push_str(extension); + } + Some(url) +} + +pub(super) fn get_redirect_alternative_url( + app_state: &Arc, + redirect_url: &Arc, + input: &ConfigInput, +) -> Arc { + if let Some((base_url, username, password)) = input.get_matched_config_by_url(redirect_url) { + if let Some(provider_cfg) = app_state.active_provider.get_next_provider(&input.name) { + let mut new_url = redirect_url.replacen(base_url, provider_cfg.url.as_str(), 1); + if let (Some(old_username), Some(old_password)) = (username, password) { + if let (Some(new_username), Some(new_password)) = + (provider_cfg.username.as_ref(), provider_cfg.password.as_ref()) + { + new_url = new_url.replacen(old_username, new_username, 1); + new_url = new_url.replacen(old_password, new_password, 1); + return new_url.into(); + } + // one has credentials the other not, something not right + return redirect_url.clone(); + } + return new_url.into(); + } + } + redirect_url.clone() +} diff --git a/backend/app/src/api/api_utils/resources.rs b/backend/app/src/api/api_utils/resources.rs new file mode 100644 index 000000000..f2aedab49 --- /dev/null +++ b/backend/app/src/api/api_utils/resources.rs @@ -0,0 +1,442 @@ +use super::{ + force_stream_response, get_headers_from_request, get_hls_playback_ttl_secs, is_hop_by_hop_response_header, + media_server_image_error_status, open_media_server_image_resource, redirect, serve_file, try_unwrap_body, + ForceStreamRequestContext, HeaderFilter, ResourceFetchPolicy, StreamResponseMode, RESOURCE_REDIRECT_LIMIT, +}; +use crate::{ + api::model::{tee_stream, AppState, StreamDetails, StreamError, UserSession}, + auth::Fingerprint, + model::{ConfigInput, PlaybackKind}, + utils::{ + async_file_writer, classify_output_resource_url, classify_resource_hop, create_new_file_for_write, + debug_if_enabled, request, + request::{classify_resource_destination, send_with_retry_and_provider, ResourceDestination}, + trace_if_enabled, LRUResourceCache, ResourceHop, + }, +}; +use arc_swap::ArcSwapOption; +use axum::{ + http::{header, HeaderMap, StatusCode}, + response::IntoResponse, +}; +use futures::TryStreamExt; +use log::{debug, error, warn}; +use shared::{ + model::{InputFetchMethod, PlaylistItemType, ResourceOutputPolicy, StreamChannel}, + utils::sanitize_sensitive_info, +}; +use std::{collections::HashMap, sync::Arc}; +use tokio::sync::RwLock; +use url::Url; + +pub(crate) async fn force_hls_resource_response( + fingerprint: &Fingerprint, + app_state: &Arc, + user_session: &UserSession, + stream_channel: StreamChannel, + mut ctx: ForceStreamRequestContext<'_>, + grace_mode: Option, +) -> axum::response::Response { + // `resolve_stream_channel` classified the item with `hls_playback_kind`, like the manifest paths. + let kind = if stream_channel.item_type == PlaylistItemType::Catchup { + PlaybackKind::Catchup + } else { + PlaybackKind::LiveHls + }; + ctx.session_reservation_ttl_secs = get_hls_playback_ttl_secs(app_state, kind); + force_stream_response( + fingerprint, + app_state, + user_session, + stream_channel, + ctx, + grace_mode, + StreamResponseMode::HlsResource, + ) + .await +} + +/// Status a finite HLS resource answers with instead of a fallback body, or `None` on success. +pub(super) fn hls_resource_failure_status(stream_details: &StreamDetails) -> Option { + match stream_details.stream_info.as_ref() { + Some((_, status, _, _)) if !status.is_success() => Some(*status), + Some((_, _, _, Some(_))) => Some(StatusCode::SERVICE_UNAVAILABLE), + _ if stream_details.custom_reason.is_some() => Some(StatusCode::SERVICE_UNAVAILABLE), + _ if !stream_details.has_stream() => Some(StatusCode::BAD_GATEWAY), + _ => None, + } +} + +pub(super) fn get_add_cache_content( + res_url: &str, + mime_type: Option, + cache: &Arc>>, +) -> Arc { + let resource_url = String::from(res_url); + let cache = Arc::clone(cache); + let add_cache_content: Arc = Arc::new(move |size| { + let res_url = resource_url.clone(); + let mime_type = mime_type.clone(); + // todo spawn, replace with unboundchannel + let cache = Arc::clone(&cache); + tokio::spawn(async move { + if let Some(cache) = cache.load().as_ref() { + let _ = cache.write().await.add_content(&res_url, mime_type, size); + } + }); + }); + add_cache_content +} + +pub(super) fn get_mime_type(headers: &HeaderMap, resource_url: &str) -> Option { + headers + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) // Option<&str> + .map(ToString::to_string) // Option + .or_else(|| { + // fallback to guess + mime_guess::from_path(resource_url).first_raw().map(ToString::to_string) + }) +} + +pub(super) async fn build_resource_stream_response( + app_state: &Arc, + cache_key: Option<&str>, + fetch_policy: ResourceFetchPolicy, + resource_url: &str, + response: reqwest::Response, +) -> axum::response::Response { + let sanitized_resource_url = sanitize_sensitive_info(resource_url); + let status = response.status(); + let mut response_builder = axum::response::Response::builder().status(status); + let mime_type = get_mime_type(response.headers(), resource_url); + let has_content_range = response.headers().contains_key(header::CONTENT_RANGE); + for (key, value) in response.headers() { + if !is_hop_by_hop_response_header(key) + && (fetch_policy == ResourceFetchPolicy::Public + || matches!( + *key, + header::CONTENT_TYPE + | header::CONTENT_LENGTH + | header::CONTENT_ENCODING + | header::CONTENT_RANGE + | header::ACCEPT_RANGES + | header::ETAG + | header::LAST_MODIFIED + | header::CACHE_CONTROL + )) + { + response_builder = response_builder.header(key, value); + } + } + + if !response_builder.headers_ref().is_some_and(|h| h.contains_key(header::CACHE_CONTROL)) { + response_builder = response_builder.header(header::CACHE_CONTROL, "public, max-age=14400"); + } + + let byte_stream = response.bytes_stream().map_err(|err| StreamError::reqwest(&err)); + // Cache only complete responses (200 OK without Content-Range) + let can_cache = status == StatusCode::OK && !has_content_range; + if can_cache { + if let Some(cache_key) = cache_key { + debug!("Caching eligible resource stream {sanitized_resource_url}"); + let cache_resource_path = if let Some(cache) = app_state.cache.load().as_ref() { + Some(cache.write().await.store_path(cache_key, mime_type.as_deref())) + } else { + None + }; + if let Some(resource_path) = cache_resource_path { + match create_new_file_for_write(&resource_path).await { + Ok(file) => { + debug!("Persisting resource stream {sanitized_resource_url} to {}", resource_path.display()); + let writer = async_file_writer(file); + let add_cache_content = get_add_cache_content(cache_key, mime_type, &app_state.cache); + let tee = tee_stream(byte_stream, writer, &resource_path, add_cache_content); + return try_unwrap_body!(response_builder.body(axum::body::Body::from_stream(tee))); + } + Err(err) => { + warn!( + "Failed to create cache file {} for {sanitized_resource_url}: {err}", + resource_path.display() + ); + } + } + } else { + debug!( + "Resource cache unavailable; streaming response for {sanitized_resource_url} without persistence" + ); + } + } + } + + try_unwrap_body!(response_builder.body(axum::body::Body::from_stream(byte_stream))) +} + +/// Whether outbound requests of this configuration may leave through a proxy, including one provided by +/// the environment, because the public resource client honours both. +pub(super) fn proxy_in_use(app_state: &Arc) -> bool { + app_state.app_config.config.load().proxy.is_some() || crate::model::proxy_env_present() +} + +pub(super) async fn fetch_resource_with_retry( + app_state: &Arc, + url: &Url, + fetch_policy: ResourceFetchPolicy, + resource_url: &str, + req_headers: &HashMap>, + input: Option<&ConfigInput>, +) -> Option { + let cache_key = fetch_policy.cache_key(resource_url); + let config = app_state.app_config.config.load(); + let default_user_agent = config.default_user_agent.clone(); + drop(config); + + let disabled_headers = app_state.get_disabled_headers(); + let mut current_url = url.clone(); + let mut current_headers = req_headers.clone(); + let mut current_input = input; + let mut method = input.map_or(InputFetchMethod::GET, |i| i.method); + + for redirects in 0..=RESOURCE_REDIRECT_LIMIT { + // Every hop is routed on its own, so a redirect cannot move a request to an egress the hop + // itself would not have used. + let hop = classify_resource_hop(current_url.as_str(), proxy_in_use(app_state)).await; + if hop == ResourceHop::Blocked { + debug!("Refused resource destination local to this host: {}", sanitize_sensitive_info(resource_url)); + return None; + } + let use_proxy_aware_client = hop == ResourceHop::ViaProxy; + let provider_config = current_input.and_then(|i| i.get_resolve_provider(current_url.as_str())); + let response = match send_with_retry_and_provider( + &app_state.app_config, + ¤t_url, + provider_config.as_ref(), + // Hand redirects back instead of retrying them: the loop below follows them itself, so that + // every hop is classified and no client-side redirect policy decides where the request ends. + true, + |resolved_url| { + let http_client = if use_proxy_aware_client { + app_state.http_clients.resource_public_no_redirect.load() + } else { + app_state.http_clients.resource_no_redirect.load() + }; + request::get_client_request( + &http_client, + method, + current_input.map(|i| &i.headers), + resolved_url, + Some(¤t_headers), + disabled_headers.as_ref(), + default_user_agent.as_deref(), + ) + }, + ) + .await + { + Ok(response) => response, + Err(err) => { + debug!( + "Resource fetch failed for {}: {}", + sanitize_sensitive_info(resource_url), + sanitize_sensitive_info(&err.to_string()) + ); + return None; + } + }; + + let status = response.status(); + if status.is_redirection() { + if redirects == RESOURCE_REDIRECT_LIMIT { + debug!("Resource redirect limit reached for {}", sanitize_sensitive_info(resource_url)); + return None; + } + let next_url = response + .headers() + .get(header::LOCATION) + .and_then(|location| location.to_str().ok()) + .and_then(|location| response.url().join(location).ok()); + let Some(next_url) = next_url else { + debug!("Resource redirect has no usable location for {}", sanitize_sensitive_info(resource_url)); + return None; + }; + let same_origin = response.url().scheme() == next_url.scheme() + && response.url().host_str() == next_url.host_str() + && response.url().port_or_known_default() == next_url.port_or_known_default(); + if !same_origin { + current_headers.retain(|key, _| request::is_safe_cross_origin_redirect_header(key)); + current_input = None; + } + if !matches!(status, StatusCode::TEMPORARY_REDIRECT | StatusCode::PERMANENT_REDIRECT) { + method = InputFetchMethod::GET; + } + current_url = next_url; + continue; + } + + if status.is_success() { + return Some( + build_resource_stream_response(app_state, cache_key, fetch_policy, resource_url, response).await, + ); + } + + debug_if_enabled!("Failed to open resource got status {status} for {}", sanitize_sensitive_info(resource_url)); + if !fetch_policy.relays_upstream_response() { + return Some(status.into_response()); + } + let mut response_builder = axum::response::Response::builder().status(status); + for (key, value) in response.headers() { + if !is_hop_by_hop_response_header(key) { + response_builder = response_builder.header(key, value); + } + } + let stream = response.bytes_stream().map_err(|err| StreamError::reqwest(&err)); + return Some(try_unwrap_body!(response_builder.body(axum::body::Body::from_stream(stream)))); + } + None +} + +pub fn resource_input_for_url<'a>(input: Option<&'a ConfigInput>, resource_url: &str) -> Option<&'a ConfigInput> { + let input = input?; + let source = Url::parse(&input.url).ok()?; + let resource = Url::parse(resource_url).ok()?; + (source.scheme() == resource.scheme() + && source.host_str() == resource.host_str() + && source.port_or_known_default() == resource.port_or_known_default()) + .then_some(input) +} + +pub async fn resource_redirect_or_proxy( + app_state: &Arc, + resource_url: &str, + req_headers: &HeaderMap, + input: Option<&ConfigInput>, +) -> axum::response::Response { + let Some(resource_url) = shared::model::persisted_resource_url(resource_url) else { + return StatusCode::BAD_REQUEST.into_response(); + }; + let resource_url = resource_url.as_ref(); + if resource_url.starts_with("/api/v1/library/thumbnail/") { + return redirect(resource_url).into_response(); + } + if resource_url.starts_with("media-server://image/") { + return resource_response(app_state, ResourceFetchPolicy::NonPublic, resource_url, req_headers, None) + .await + .into_response(); + } + let Ok(url) = Url::parse(resource_url) else { + return StatusCode::BAD_REQUEST.into_response(); + }; + if !matches!(url.scheme(), "http" | "https") { + return StatusCode::BAD_REQUEST.into_response(); + } + let Some(host) = url.host_str() else { + return StatusCode::BAD_REQUEST.into_response(); + }; + if classify_resource_destination(host).await == ResourceDestination::Public { + redirect(resource_url).into_response() + } else { + resource_proxy_response(app_state, resource_url, req_headers, resource_input_for_url(input, resource_url)).await + } +} + +pub async fn resource_proxy_response( + app_state: &Arc, + resource_url: &str, + req_headers: &HeaderMap, + input: Option<&ConfigInput>, +) -> axum::response::Response { + let Some(resource_url) = shared::model::persisted_resource_url(resource_url) else { + return StatusCode::BAD_REQUEST.into_response(); + }; + let resource_url = resource_url.as_ref(); + let input = resource_input_for_url(input, resource_url); + match classify_output_resource_url(resource_url).await { + ResourceOutputPolicy::Direct if resource_url.starts_with("/api/v1/library/thumbnail/") => { + redirect(resource_url).into_response() + } + ResourceOutputPolicy::Direct => { + resource_response(app_state, ResourceFetchPolicy::Public, resource_url, req_headers, input) + .await + .into_response() + } + ResourceOutputPolicy::Proxy => { + resource_response(app_state, ResourceFetchPolicy::NonPublic, resource_url, req_headers, input) + .await + .into_response() + } + ResourceOutputPolicy::Blocked => StatusCode::FORBIDDEN.into_response(), + } +} + +/// Callers must pass an already-decoded resource URL (not a `resource://v1/` locator). +/// [`resource_redirect_or_proxy`] and [`resource_proxy_response`] handle the decode before +/// dispatching here; XMLTV and Web-UI endpoints pass URLs from authenticated tokens. +pub async fn resource_response( + app_state: &Arc, + fetch_policy: ResourceFetchPolicy, + resource_url: &str, + req_headers: &HeaderMap, + input: Option<&ConfigInput>, +) -> impl IntoResponse + Send { + if resource_url.is_empty() { + return StatusCode::NO_CONTENT.into_response(); + } + + if resource_url.starts_with("media-server://image/") { + return match open_media_server_image_resource(app_state, resource_url).await { + Ok(response) => response, + Err(err) => { + let status = media_server_image_error_status(&err); + match status { + StatusCode::BAD_REQUEST => warn!("Invalid media-server image resource URL: {err}"), + StatusCode::NOT_FOUND => debug!("Media-server image resource was not found: {err}"), + _ => error!("Can't open media-server image from upstream: {err}"), + } + status.into_response() + } + }; + } + let filter: HeaderFilter = Some(Box::new(request::is_safe_cross_origin_redirect_header)); + let req_headers = get_headers_from_request(req_headers, &filter); + let cache_key = fetch_policy.cache_key(resource_url); + if let (Some(cache_key), Some(cache)) = (cache_key, app_state.cache.load().as_ref()) { + let cache_hit = { + let mut guard = cache.write().await; + guard.get_content(cache_key) + }; + + if let Some((resource_path, mime_type)) = cache_hit { + trace_if_enabled!("Responding resource from cache {}", sanitize_sensitive_info(resource_url)); + return serve_file( + &resource_path, + mime_type.unwrap_or_else(|| mime::APPLICATION_OCTET_STREAM.to_string()), + Some("public, max-age=14400"), + ) + .await + .into_response(); + } + } + trace_if_enabled!("Try to fetch resource {}", sanitize_sensitive_info(resource_url)); + if let Ok(url) = Url::parse(resource_url) { + // A resource URL is chosen by playlist or EPG content, so the destination is never fetched + // blindly: an address local to this host (loopback, link-local, cloud metadata) would turn + // the proxy into a reader for the proxy host itself. Private network destinations stay + // allowed, because self-hosted services are the reason this route exists. + let Some(host) = url.host_str() else { + return StatusCode::BAD_REQUEST.into_response(); + }; + if classify_resource_destination(host).await == ResourceDestination::Blocked { + debug!("Refused resource destination local to this host: {}", sanitize_sensitive_info(resource_url)); + return StatusCode::FORBIDDEN.into_response(); + } + if let Some(resp) = + fetch_resource_with_retry(app_state, &url, fetch_policy, resource_url, &req_headers, input).await + { + return resp; + } + // Upstream failure after retries + return StatusCode::BAD_GATEWAY.into_response(); + } + error!("Url is malformed {}", sanitize_sensitive_info(resource_url)); + StatusCode::BAD_REQUEST.into_response() +} diff --git a/backend/app/src/api/api_utils/response.rs b/backend/app/src/api/api_utils/response.rs new file mode 100644 index 000000000..f538f8a11 --- /dev/null +++ b/backend/app/src/api/api_utils/response.rs @@ -0,0 +1,129 @@ +use super::{record_connect_failed_attempt, try_unwrap_body, ConnectFailedAttempt}; +use crate::{ + api::model::{create_custom_video_stream_response, AppState, CustomVideoStreamType, StreamAdmissionError}, + auth::Fingerprint, + model::ProxyUserCredentials, +}; +use axum::{ + body::Body, + http::{header, HeaderMap, StatusCode}, + response::IntoResponse, +}; +use log::error; +use serde::Serialize; +use shared::{ + model::{ConnectFailureReason, FailureStage, StreamChannel}, + utils::{bin_serialize, CONTENT_TYPE_CBOR}, +}; +use std::sync::Arc; + +pub(super) fn admission_failure_video_type(reason: ConnectFailureReason) -> Option { + match reason { + ConnectFailureReason::UserAccountExpired => Some(CustomVideoStreamType::UserAccountExpired), + ConnectFailureReason::UserConnectionsExhausted => Some(CustomVideoStreamType::UserConnectionsExhausted), + ConnectFailureReason::ProviderConnectionsExhausted => Some(CustomVideoStreamType::ProviderConnectionsExhausted), + _ => None, + } +} + +pub(crate) fn admission_failure_response( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + stream_channel: StreamChannel, + provider_name: Arc, + req_headers: &HeaderMap, + reason: ConnectFailureReason, +) -> axum::response::Response { + record_connect_failed_attempt(ConnectFailedAttempt { + app_state, + fingerprint, + user, + stream_channel, + provider_name, + req_headers, + reason, + failure_stage: FailureStage::Admission, + }); + let Some(video_type) = admission_failure_video_type(reason) else { + error!("Unsupported admission failure reason: {reason:?}"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + create_custom_video_stream_response(&app_state.provider_stream_ctx(), &fingerprint.addr, video_type).into_response() +} + +/// Produces a defined non-success response for a rejected playback request. Unlike the +/// upstream provider response, this carries no Content-Length/Content-Range so downstream +/// players receive an unambiguous error instead of an empty success body. +pub(super) fn stream_admission_rejected_response( + error: StreamAdmissionError, + username: &str, +) -> axum::response::Response { + error!("Stream admission rejected for user {username}: {error:?}"); + axum::response::Response::builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .header("x-tuliprox-rejection", "admission_rejected") + .body(axum::body::Body::empty()) + .unwrap_or_else(|_| StatusCode::SERVICE_UNAVAILABLE.into_response()) +} + +/// Produces a quiet termination response when a recently evicted stream retries. +pub fn reentry_suppressed_response() -> axum::response::Response { StatusCode::NO_CONTENT.into_response() } + +pub fn separate_number_and_remainder(input: &str) -> (&str, Option<&str>) { + input.rfind('.').map_or_else( + || (input, None), + |dot_index| { + let number_part = &input[..dot_index]; + let rest = &input[dot_index..]; + (number_part, if rest.len() < 2 { None } else { Some(rest) }) + }, + ) +} + +/// # Panics +pub fn empty_json_list_response() -> axum::response::Response { + try_unwrap_body!(axum::response::Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, crate::api::static_headers::CT_JSON.clone()) + .body("[]".to_owned())) +} + +pub fn redirect(url: &str) -> impl IntoResponse { + try_unwrap_body!(axum::response::Response::builder() + .status(StatusCode::FOUND) + .header(header::LOCATION, url) + .body(Body::empty())) +} + +pub fn bin_response(data: &T) -> impl IntoResponse + Send { + match bin_serialize(data) { + Ok(body) => ([(header::CONTENT_TYPE, CONTENT_TYPE_CBOR)], body).into_response(), + Err(_) => internal_server_error!(), + } +} + +pub fn json_response(data: &T) -> impl IntoResponse + Send { + (StatusCode::OK, axum::Json(data)).into_response() +} + +pub fn json_or_bin_response(accept: Option<&str>, data: &T) -> impl IntoResponse + Send { + if accept.is_some_and(|a| a.contains(CONTENT_TYPE_CBOR)) { + return bin_response(data).into_response(); + } + json_response(data).into_response() +} + +pub fn empty_json_response_as_object() -> axum::http::Result { + axum::response::Response::builder() + .status(axum::http::StatusCode::OK) + .header(axum::http::header::CONTENT_TYPE, crate::api::static_headers::CT_JSON.clone()) + .body(axum::body::Body::from("{}".as_bytes())) +} + +pub fn empty_json_response_as_array() -> axum::http::Result { + axum::response::Response::builder() + .status(axum::http::StatusCode::OK) + .header(axum::http::header::CONTENT_TYPE, crate::api::static_headers::CT_JSON.clone()) + .body(axum::body::Body::from("[]".as_bytes())) +} diff --git a/backend/app/src/api/api_utils/serialization.rs b/backend/app/src/api/api_utils/serialization.rs new file mode 100644 index 000000000..3e82cb80a --- /dev/null +++ b/backend/app/src/api/api_utils/serialization.rs @@ -0,0 +1,272 @@ +use super::{try_unwrap_body, API_STREAM_CHUNK_SIZE}; +use axum::{ + body::Body, + http::{header, Response}, + response::IntoResponse, +}; +use bytes::{Bytes, BytesMut}; +use futures::{stream, Stream, StreamExt}; +use log::warn; +use shared::utils::{bin_serialize, CONTENT_TYPE_CBOR, CONTENT_TYPE_JSON}; +use std::convert::Infallible; + +pub fn stream_json_or_bin_response

( + accept: Option<&str>, + data: Box + Send>, +) -> axum::response::Response +where + P: serde::Serialize + Send + 'static, +{ + if accept.is_some_and(|a| a.contains(CONTENT_TYPE_CBOR)) { + return stream_bin_array(data); + } + stream_json_array(data) +} + +pub fn stream_json_or_bin_response_stream(accept: Option<&str>, data: S) -> axum::response::Response +where + P: serde::Serialize + Send + 'static, + S: Stream + Send + Unpin + 'static, +{ + if accept.is_some_and(|a| a.contains(CONTENT_TYPE_CBOR)) { + return stream_bin_array_stream(data); + } + stream_json_array_stream(data) +} + +pub fn stream_json_or_bin_response_try_stream(accept: Option<&str>, data: S) -> axum::response::Response +where + P: serde::Serialize + Send + 'static, + S: Stream> + Send + Unpin + 'static, + E: std::fmt::Display + Send + 'static, +{ + if accept.is_some_and(|value| value.contains(CONTENT_TYPE_CBOR)) { + return stream_bin_array_try_stream(data); + } + stream_json_array_try_stream(data) +} + +pub fn stream_json_array

(iter: Box + Send>) -> axum::response::Response +where + P: serde::Serialize + Send + 'static, +{ + let stream = stream::unfold((iter, true), |(mut iter, first)| async move { + match iter.next() { + Some(item) => { + let mut json = String::new(); + if !first { + json.push(','); + } + let element = serde_json::to_string(&item).ok()?; + json.push_str(&element); + Some((Ok::(Bytes::from(json)), (iter, false))) + } + None => None, + } + }); + + let body = Body::from_stream(coalesce_byte_stream( + stream::once(async { Ok::<_, Infallible>(Bytes::from_static(b"[")) }) + .chain(stream) + .chain(stream::once(async { Ok::<_, Infallible>(Bytes::from_static(b"]")) })), + )); + + try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_JSON).body(body)) +} + +pub fn stream_bin_array

(iter: Box + Send>) -> axum::response::Response +where + P: serde::Serialize + Send + 'static, +{ + let stream = stream::unfold(iter, |mut iter| async move { + match iter.next() { + Some(item) => { + match bin_serialize(&item) { + Ok(buf) => Some((Ok::(Bytes::from(buf)), iter)), + Err(err) => { + warn!("CBOR serialization error in stream: {err}"); + Some((Ok::(Bytes::new()), iter)) // skip errors, continue + } + } + } + None => None, + } + }); + + let body = Body::from_stream(coalesce_byte_stream( + stream::once(async { + // CBOR: start indefinite-length array + Ok::<_, Infallible>(Bytes::from_static(&[0x9f])) + }) + .chain(stream) + .chain(stream::once(async { + // CBOR: end indefinite-length array + Ok::<_, Infallible>(Bytes::from_static(&[0xff])) + })), + )); + + try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_CBOR).body(body)) +} + +pub fn stream_json_array_stream(stream: S) -> axum::response::Response +where + P: serde::Serialize + Send + 'static, + S: Stream + Send + Unpin + 'static, +{ + let stream = stream::unfold((stream, true), |(mut stream, first)| async move { + match stream.next().await { + Some(item) => { + let mut json = String::new(); + if !first { + json.push(','); + } + let element = serde_json::to_string(&item).ok()?; + json.push_str(&element); + Some((Ok::(Bytes::from(json)), (stream, false))) + } + None => None, + } + }); + + let body = Body::from_stream(coalesce_byte_stream( + stream::once(async { Ok::<_, Infallible>(Bytes::from_static(b"[")) }) + .chain(stream) + .chain(stream::once(async { Ok::<_, Infallible>(Bytes::from_static(b"]")) })), + )); + + try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_JSON).body(body)) +} + +pub(super) fn stream_json_array_try_stream(stream: S) -> axum::response::Response +where + P: serde::Serialize + Send + 'static, + S: Stream> + Send + Unpin + 'static, + E: std::fmt::Display + Send + 'static, +{ + let stream = stream::unfold((stream, true, false), |(mut stream, first, failed)| async move { + if failed { + return None; + } + match stream.next().await { + Some(Ok(item)) => { + let serialized = serde_json::to_vec(&item).map_err(|error| error.to_string()); + let bytes = serialized.map(|serialized| { + if first { + Bytes::from(serialized) + } else { + let mut framed = Vec::with_capacity(serialized.len() + 1); + framed.push(b','); + framed.extend_from_slice(&serialized); + Bytes::from(framed) + } + }); + let failed = bytes.is_err(); + Some((bytes, (stream, false, failed))) + } + Some(Err(error)) => Some((Err(error.to_string()), (stream, first, true))), + None => None, + } + }); + + let body = Body::from_stream(coalesce_byte_stream( + stream::once(async { Ok::<_, String>(Bytes::from_static(b"[")) }) + .chain(stream) + .chain(stream::once(async { Ok::<_, String>(Bytes::from_static(b"]")) })), + )); + try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_JSON).body(body)) +} + +pub fn stream_bin_array_stream(stream: S) -> axum::response::Response +where + P: serde::Serialize + Send + 'static, + S: Stream + Send + Unpin + 'static, +{ + let stream = stream::unfold(stream, |mut stream| async move { + match stream.next().await { + Some(item) => match bin_serialize(&item) { + Ok(buf) => Some((Ok::(Bytes::from(buf)), stream)), + Err(err) => { + warn!("CBOR serialization error in stream: {err}"); + Some((Ok::(Bytes::new()), stream)) + } + }, + None => None, + } + }); + + let body = Body::from_stream(coalesce_byte_stream( + stream::once(async { Ok::<_, Infallible>(Bytes::from_static(&[0x9f])) }) + .chain(stream) + .chain(stream::once(async { Ok::<_, Infallible>(Bytes::from_static(&[0xff])) })), + )); + + try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_CBOR).body(body)) +} + +pub(super) fn stream_bin_array_try_stream(stream: S) -> axum::response::Response +where + P: serde::Serialize + Send + 'static, + S: Stream> + Send + Unpin + 'static, + E: std::fmt::Display + Send + 'static, +{ + let stream = stream::unfold((stream, false), |(mut stream, failed)| async move { + if failed { + return None; + } + match stream.next().await { + Some(Ok(item)) => { + let bytes = bin_serialize(&item).map(Bytes::from).map_err(|error| error.to_string()); + let failed = bytes.is_err(); + Some((bytes, (stream, failed))) + } + Some(Err(error)) => Some((Err(error.to_string()), (stream, true))), + None => None, + } + }); + + let body = Body::from_stream(coalesce_byte_stream( + stream::once(async { Ok::<_, String>(Bytes::from_static(&[0x9f])) }) + .chain(stream) + .chain(stream::once(async { Ok::<_, String>(Bytes::from_static(&[0xff])) })), + )); + try_unwrap_body!(Response::builder().header(header::CONTENT_TYPE, CONTENT_TYPE_CBOR).body(body)) +} + +pub(crate) fn coalesce_byte_stream(stream: S) -> impl Stream> +where + S: Stream> + Send + 'static, + E: Send + 'static, +{ + stream::unfold((Box::pin(stream), None, false), |(mut stream, pending_error, finished)| async move { + if let Some(error) = pending_error { + return Some((Err(error), (stream, None, true))); + } + if finished { + return None; + } + + let mut chunk = BytesMut::with_capacity(API_STREAM_CHUNK_SIZE); + loop { + match stream.next().await { + Some(Ok(bytes)) if chunk.is_empty() && bytes.len() >= API_STREAM_CHUNK_SIZE => { + return Some((Ok(bytes), (stream, None, false))); + } + Some(Ok(bytes)) => { + chunk.extend_from_slice(&bytes); + if chunk.len() >= API_STREAM_CHUNK_SIZE { + return Some((Ok(chunk.freeze()), (stream, None, false))); + } + } + Some(Err(error)) if chunk.is_empty() => { + return Some((Err(error), (stream, None, true))); + } + Some(Err(error)) => { + return Some((Ok(chunk.freeze()), (stream, Some(error), false))); + } + None if chunk.is_empty() => return None, + None => return Some((Ok(chunk.freeze()), (stream, None, true))), + } + } + }) + .fuse() +} diff --git a/backend/app/src/api/api_utils/session_identity.rs b/backend/app/src/api/api_utils/session_identity.rs new file mode 100644 index 000000000..23e72d1cc --- /dev/null +++ b/backend/app/src/api/api_utils/session_identity.rs @@ -0,0 +1,100 @@ +use super::{get_catchup_session_ttl_secs, get_stream_config_u64}; +use crate::{ + api::model::{AppState, CustomVideoStreamType, StreamDetails, UserSession}, + auth::Fingerprint, +}; +use shared::{ + concat_string, + defaults::{default_hls_session_ttl_secs, DASH_EXT, HLS_EXT}, + model::PlaylistItemType, +}; +use smallvec::SmallVec; +use std::{net::SocketAddr, sync::Arc}; + +pub(crate) fn get_hls_session_ttl_secs(app_state: &Arc) -> u64 { + get_stream_config_u64(app_state, |stream| stream.hls_session_ttl_secs, default_hls_session_ttl_secs()) +} + +pub(crate) fn get_session_reservation_ttl_secs(app_state: &Arc, item_type: PlaylistItemType) -> u64 { + match item_type { + PlaylistItemType::LiveHls | PlaylistItemType::LiveDash => get_hls_session_ttl_secs(app_state), + PlaylistItemType::Catchup => get_catchup_session_ttl_secs(app_state), + _ => 0, + } +} + +/// Whether the session should pin the provider account via `refresh_provider_reservation`. +/// +/// A non-Provisioning custom video (`ChannelUnavailable`, `ProviderConnectionsExhausted`, …) means +/// the upstream open already failed. The provider connection slot was released by +/// `create_provider_stream`, and the custom video is a local fallback served to the client. +/// Pinning the provider via `refresh_provider_reservation` would hold the provider account for +/// the configured session TTL (e.g. `catchup_session_ttl_secs`), blocking other sessions of +/// the same family from using it even though the slot is already free. +/// +/// Only `Provisioning` custom videos represent a real provider handoff that benefits from +/// keeping the same provider pinned, and real provider streams (`stream_info` carries no +/// `CustomVideoStreamType`) obviously qualify. +pub(crate) fn should_pin_provider_for_session( + stream_details: &StreamDetails, + _app_state: &Arc, + _item_type: PlaylistItemType, +) -> bool { + !matches!( + stream_details.stream_info.as_ref(), + Some((_, _, _, Some(cv))) if *cv != CustomVideoStreamType::Provisioning + ) +} + +pub fn create_session_fingerprint( + fingerprint: &Fingerprint, + username: &str, + virtual_id: u32, + socket_bound: bool, +) -> String { + if socket_bound { + concat_string!(&fingerprint.addr.to_string(), "|", username, "|", &virtual_id.to_string()) + } else { + concat_string!(&fingerprint.key, "|", username, "|", &virtual_id.to_string()) + } +} + +pub(crate) fn create_playback_session_fingerprint( + fingerprint: &Fingerprint, + username: &str, + virtual_id: u32, + item_type: PlaylistItemType, + extension: Option<&str>, +) -> String { + // This scopes the session identity, not the session address-tracking policy. + // Adaptive and seekable playback use the stable client fingerprint so follow-up + // requests on new sockets reuse the same logical session. Plain live playback + // remains socket-bound so a separate connection creates a separate session. + let session_bound = is_session_based_playback(item_type, extension); + let socket_bound = !session_bound && is_socket_bound_playback_session(item_type, extension); + create_session_fingerprint(fingerprint, username, virtual_id, socket_bound) +} + +pub(crate) fn is_session_based_playback(item_type: PlaylistItemType, extension: Option<&str>) -> bool { + item_type.is_live_adaptive() || matches!(extension, Some(ext) if ext == HLS_EXT || ext == DASH_EXT) +} + +pub(crate) fn is_socket_bound_playback_session(item_type: PlaylistItemType, extension: Option<&str>) -> bool { + item_type.uses_socket_bound_session() && !is_session_based_playback(item_type, extension) +} + +pub(super) fn session_reacquire_cleanup_addrs( + user_session: &UserSession, + current_addr: &SocketAddr, +) -> Vec { + let mut addrs: SmallVec<[SocketAddr; 4]> = SmallVec::new(); + if user_session.addr != *current_addr { + addrs.push(user_session.addr); + } + for addr in &user_session.active_addrs { + if *addr != *current_addr && !addrs.contains(addr) { + addrs.push(*addr); + } + } + addrs.into_vec() +} diff --git a/backend/app/src/api/api_utils/stalker.rs b/backend/app/src/api/api_utils/stalker.rs new file mode 100644 index 000000000..21d548519 --- /dev/null +++ b/backend/app/src/api/api_utils/stalker.rs @@ -0,0 +1,65 @@ +use crate::{api::model::AppState, model::ConfigInput, processing::processor::re_resolve_stalker_url}; +use axum::http::StatusCode; +use shared::{ + error::TuliproxError, + model::{InputType, PlaylistItemType, StalkerStreamKind, XtreamCluster}, +}; +use std::sync::Arc; + +pub(super) fn should_refresh_stalker_playback( + input_type: InputType, + request_url_valid: bool, + status: Option, +) -> bool { + input_type.is_stalker() && (!request_url_valid || status.is_some_and(|status| status.is_client_error())) +} + +pub(super) fn needs_initial_stalker_resolution(input_type: InputType, stream_url: &str) -> bool { + input_type.is_stalker() && stream_url.is_empty() +} + +pub(super) fn stalker_stream_kind(cluster: XtreamCluster, item_type: PlaylistItemType) -> StalkerStreamKind { + if item_type == PlaylistItemType::Catchup { + StalkerStreamKind::Archive + } else { + match cluster { + XtreamCluster::Live => StalkerStreamKind::Live, + XtreamCluster::Video => StalkerStreamKind::Movie, + XtreamCluster::Series => StalkerStreamKind::Episode, + } + } +} + +pub(super) async fn re_resolve_stalker_url_singleflight( + app_state: &Arc, + input: &ConfigInput, + provider_id: u32, + kind: StalkerStreamKind, + force_refresh: bool, +) -> Result>, TuliproxError> { + let entry_lock = app_state.stalker_resolve_coordinator.guard_for(input.id, provider_id).await; + let _flight = entry_lock.lock().await; + let client = app_state.http_clients.default.load().as_ref().clone(); + re_resolve_stalker_url(&app_state.app_config, &client, input, provider_id, kind, force_refresh).await +} + +pub(crate) async fn resolve_initial_stalker_playback_url( + app_state: &Arc, + input: &ConfigInput, + provider_id: u32, + cluster: XtreamCluster, + item_type: PlaylistItemType, + stream_url: &Arc, +) -> Result, TuliproxError> { + if !needs_initial_stalker_resolution(input.input_type, stream_url) { + return Ok(Arc::clone(stream_url)); + } + re_resolve_stalker_url_singleflight(app_state, input, provider_id, stalker_stream_kind(cluster, item_type), false) + .await? + .ok_or_else(|| { + TuliproxError::RepositoryStalker(format!( + "Stalker playback URL could not be resolved for input '{}' and provider id {provider_id}", + input.name + )) + }) +} diff --git a/backend/app/src/api/api_utils/strategy.rs b/backend/app/src/api/api_utils/strategy.rs new file mode 100644 index 000000000..7f5514004 --- /dev/null +++ b/backend/app/src/api/api_utils/strategy.rs @@ -0,0 +1,220 @@ +use super::{ + get_redirect_alternative_url, get_stream_alternative_url, is_media_server_playback_url, redirect, + resolve_request_url_for_logging, RedirectParams, +}; +#[cfg(test)] +use super::{resolve_streaming_strategy_with_provider_handle, StreamingAcquireOptions}; +#[cfg(test)] +use crate::api::model::StreamingStrategy; +#[cfg(test)] +use crate::auth::Fingerprint; +use crate::{ + api::{ + endpoints::xtream_api::get_xtream_player_api_stream_url, + model::{AppState, ProviderStreamState}, + }, + model::ConfigInput, + utils::debug_if_enabled, +}; +use axum::{http::StatusCode, response::IntoResponse}; +use log::error; +#[cfg(test)] +use shared::model::StreamChannel; +use shared::{ + defaults::{DASH_EXT, HLS_EXT}, + error::TuliproxError, + model::{PlaylistEntry, PlaylistItemType, TargetType, UserConnectionPermission}, + utils::{replace_url_extension, sanitize_sensitive_info}, +}; +use std::{borrow::Cow, sync::Arc}; + +pub(crate) fn resolve_redirect_location<'a>( + input: Option<&ConfigInput>, + stream_url: &'a str, +) -> Result, TuliproxError> { + input.map_or(Ok(Cow::Borrowed(stream_url)), |input| input.resolve_url(stream_url)) +} + +/// Determines the appropriate streaming strategy for the given input and stream URL. +/// +/// This function attempts to acquire a connection to a streaming provider, either using a forced provider +/// (if specified), or based on the input name. It then selects a corresponding `StreamingOption`: +/// +/// - If no connections are available (`Exhausted`), it returns a custom stream indicating exhaustion. +/// - If a connection is available or in a grace period, it constructs a streaming URL accordingly: +/// - If the URL already targets the selected provider account, the original URL is reused. +/// - Otherwise, an alternative URL is generated based on the provider and input. +/// +/// The function returns: +/// - an optional `ProviderConnectionGuard` to manage the connection's lifecycle, +/// - a `ProviderStreamState` describing how the stream state is, +/// - and optional HTTP headers to include in the request. +/// +/// This logic helps abstract the decision-making behind provider selection and stream URL resolution. +#[cfg(test)] +pub(super) async fn resolve_streaming_strategy( + app_state: &Arc, + stream_url: &str, + fingerprint: &Fingerprint, + input: &ConfigInput, + options: StreamingAcquireOptions<'_>, + stream_channel: Option<&StreamChannel>, +) -> StreamingStrategy { + resolve_streaming_strategy_with_provider_handle( + app_state, + stream_url, + fingerprint, + input, + options, + stream_channel, + None, + ) + .await +} + +pub(super) fn get_grace_period_millis( + connection_permission: UserConnectionPermission, + stream_response_params: &ProviderStreamState, + config_grace_period_millis: u64, +) -> u64 { + if config_grace_period_millis > 0 + && ( + matches!(stream_response_params, ProviderStreamState::GracePeriod(_, _)) // provider grace period + || connection_permission == UserConnectionPermission::GracePeriod + // user grace period + ) + { + config_grace_period_millis + } else { + 0 + } +} + +pub(super) fn should_defer_provider_open_for_grace_hold( + provider_grace_active: bool, + hold_stream: bool, + item_type: PlaylistItemType, + is_reopen: bool, +) -> bool { + if !(provider_grace_active && hold_stream) { + return false; + } + + // Catch-up must open immediately so its payload can be classified before response headers are committed. + if item_type == PlaylistItemType::Catchup { + return false; + } + + // v3.3.0 opened provider-affine VOD/Series reopens immediately, even when + // provider grace was temporarily in effect. Parking these requests in GracePending + // was introduced later and breaks players like libmpv during seek/reopen retries. + // Keep hold-stream behavior for live/admission paths, but restore direct-open behavior + // for provider-affine on-demand session reopens. + !(!item_type.is_live() && item_type.requires_provider_affinity() && is_reopen) +} + +pub async fn redirect_response<'a, P>( + app_state: &Arc, + params: &'a RedirectParams<'a, P>, +) -> Option +where + P: PlaylistEntry, +{ + let item_type = params.item.get_item_type(); + let provider_url = params.item.get_provider_url(); + if is_media_server_playback_url(params.input, provider_url.as_ref()) { + return None; + } + + let redirect_request = params.user.proxy.is_redirect(item_type) || params.target.is_force_redirect(item_type); + let is_hls_request = item_type == PlaylistItemType::LiveHls || params.stream_ext == Some(HLS_EXT); + let is_dash_request = + (!is_hls_request && item_type == PlaylistItemType::LiveDash) || params.stream_ext == Some(DASH_EXT); + + // Recording playback keeps provider headers and proxy handling on this listener. + // DASH redirects because relative MPD resource URLs are not rewritten. + if params.user.is_recording_proxy_user() && !is_dash_request { + return None; + } + + if params.target_type == TargetType::M3u { + if redirect_request || is_dash_request { + let redirect_url: Arc = if is_hls_request { + replace_url_extension(&provider_url, HLS_EXT).into() + } else { + provider_url.clone() + }; + let redirect_url = + if is_dash_request { replace_url_extension(&redirect_url, DASH_EXT).into() } else { redirect_url }; + let redirect_url = get_redirect_alternative_url(app_state, &redirect_url, params.input); + let redirect_url = match resolve_redirect_location(Some(params.input), &redirect_url) { + Ok(url) => url, + Err(err) => { + error!("Failed to resolve redirect url: {}", sanitize_sensitive_info(&err.to_string())); + return Some(StatusCode::BAD_REQUEST.into_response()); + } + }; + debug_if_enabled!("Redirecting stream request to {}", sanitize_sensitive_info(redirect_url.as_ref())); + return Some(redirect(redirect_url.as_ref()).into_response()); + } + } else if params.target_type == TargetType::Xtream { + let Some(provider_id) = params.provider_id else { + return Some(StatusCode::BAD_REQUEST.into_response()); + }; + + if redirect_request || is_dash_request { + let target_name = params.target.name.as_str(); + let virtual_id = params.item.get_virtual_id(); + let stream_url = match get_xtream_player_api_stream_url( + params.input, + params.req_context, + ¶ms.get_query_path(provider_id, &provider_url), + &provider_url, + ) { + None => { + error!( + "Can't find stream url for target {target_name}, context {}, stream_id {virtual_id}", + params.req_context + ); + return Some(StatusCode::BAD_REQUEST.into_response()); + } + Some(url) => match app_state.active_provider.get_next_provider(¶ms.input.name) { + Some(provider_cfg) => match get_stream_alternative_url(&url, params.input, &provider_cfg) { + Some(stream_url) => stream_url, + None => return Some(StatusCode::BAD_REQUEST.into_response()), + }, + None => url.to_string(), + }, + }; + let stream_url = match resolve_redirect_location(Some(params.input), &stream_url) { + Ok(url) => url, + Err(err) => { + error!("Failed to resolve redirect url: {}", sanitize_sensitive_info(&err.to_string())); + return Some(StatusCode::BAD_REQUEST.into_response()); + } + }; + + // hls or dash redirect + if is_dash_request { + let redirect_url = if is_hls_request { + &replace_url_extension(&stream_url, HLS_EXT) + } else { + &replace_url_extension(&stream_url, DASH_EXT) + }; + debug_if_enabled!( + "Redirecting stream request to {}", + sanitize_sensitive_info(resolve_request_url_for_logging(params.input, redirect_url).as_ref()) + ); + return Some(redirect(redirect_url).into_response()); + } + + debug_if_enabled!( + "Redirecting stream request to {}", + sanitize_sensitive_info(resolve_request_url_for_logging(params.input, stream_url.as_ref()).as_ref()) + ); + return Some(redirect(stream_url.as_ref()).into_response()); + } + } + + None +} diff --git a/backend/app/src/api/api_utils/streaming.rs b/backend/app/src/api/api_utils/streaming.rs new file mode 100644 index 000000000..cea64b0d5 --- /dev/null +++ b/backend/app/src/api/api_utils/streaming.rs @@ -0,0 +1,1232 @@ +use super::{ + allows_provider_pool_failover, cleanup_forced_reopen_addrs, connection_priority_for_kind, + get_catchup_session_ttl_secs, get_grace_period_millis, get_hls_session_ttl_secs, get_stream_options, + get_stream_throttle, hls_resource_failure_status, is_media_server_playback_url, is_media_server_stream_ref_url, + is_socket_bound_playback_session, is_stream_metrics_enabled, is_throttled_stream, mark_response_as_uncompressed, + open_media_server_stream_for_input, prepare_stream_metering, re_resolve_stalker_url_singleflight, + resolve_streaming_strategy_with_provider_handle, session_reacquire_cleanup_addrs, + should_defer_provider_open_for_grace_hold, should_refresh_stalker_playback, split_provider_stream_open, + stalker_stream_kind, stream_admission_rejected_response, stream_response_with_provider_handle, try_unwrap_body, + ConnectFailedAttempt, CurrentSessionGuard, ForceStreamRequestContext, PlaybackRequestClass, RedirectParams, + ResourceFetchPolicy, StreamMeteringConfig, StreamOptions, StreamResponseMode, HLS_MEDIA_CAPACITY_WAIT, +}; +use crate::{ + api::{ + model::{ + create_active_client_stream, create_channel_unavailable_stream, get_custom_stream_response_error_status, + get_stream_response_with_headers, is_custom_video_stream_enabled, open_provider_stream_with_lifecycle, + AppState, CustomVideoStreamType, ProviderStreamCustomReason, ProviderStreamFactoryOptions, + ProviderStreamOpenLifecycle, ProviderStreamState, SharedStreamCtx, SharedStreamManager, StreamDetails, + StreamError, ThrottledStream, UserSession, + }, + panel_api::{can_provision_on_exhausted, create_panel_api_provisioning_stream_details}, + }, + auth::Fingerprint, + model::{AppConfig, ConfigInput, ConfigInputFlags, ConfigTarget, PlaybackKind, ProxyUserCredentials}, + utils::{debug_if_enabled, request}, +}; +use axum::{ + http::{header, HeaderMap, HeaderName, HeaderValue, StatusCode}, + response::IntoResponse, +}; +use futures::StreamExt; +use log::{debug, error, info, log_enabled, warn}; +use shared::{ + concat_string, + defaults::HLS_EXT, + error::TuliproxError, + model::{ + ConfigTargetOptions, FailureStage, PlaylistEntry, PlaylistItemType, StreamChannel, StreamInfo, + UserConnectionPermission, VirtualId, XtreamCluster, + }, + utils::{ + extract_extension_from_url, human_readable_kbps, is_sanitize_sensitive_info_enabled, sanitize_sensitive_info, + trim_slash, Internable, + }, +}; +use std::{borrow::Cow, collections::HashMap, sync::Arc, time::Duration}; +use tuliprox_session::{ProviderSessionHeaders, SessionProviderHeaders}; +use url::Url; + +pub(crate) fn resolve_request_url_for_logging<'a>(input: &ConfigInput, stream_url: &'a str) -> Cow<'a, str> { + if !is_sanitize_sensitive_info_enabled() { + return Cow::Borrowed(stream_url); + } + if is_media_server_playback_url(input, stream_url) { + return Cow::Borrowed("media-server://"); + } + + let provider = input.get_resolve_provider(stream_url); + if let Ok(url) = Url::parse(stream_url) { + return Cow::Owned(request::preview_request_target_for_logging(&url, provider.as_ref())); + } + + input + .resolve_url(stream_url) + .ok() + .and_then(|resolved| { + Url::parse(resolved.as_ref()) + .ok() + .map(|url| Cow::Owned(request::preview_request_target_for_logging(&url, provider.as_ref()))) + }) + .unwrap_or(Cow::Borrowed(stream_url)) +} + +pub(crate) fn record_connect_failed_attempt(attempt: ConnectFailedAttempt<'_>) { + let user_agent = attempt + .req_headers + .get(header::USER_AGENT) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + let info = StreamInfo::new(shared::model::StreamInfoParams { + uid: 0, + meter_uid: 0, + username: &attempt.user.username, + addr: &attempt.fingerprint.addr, + client_ip: &attempt.fingerprint.client_ip, + provider: attempt.provider_name, + stream_channel: attempt.stream_channel, + user_agent, + country_code: None, + session_token: None, + }); + // Resolve target_name from target_id using the stable target config name. + let target_name = + attempt.app_state.app_config.get_target_by_id(info.channel.target_id).as_deref().map(|t| (&t.name).intern()); + attempt.app_state.connection_manager.record_connect_failed_with_provider_failure( + &info, + attempt.reason, + attempt.failure_stage, + None, + None, + target_name, + ); +} + +pub(super) struct StreamingAcquireOptions<'a> { + pub(super) force_provider: Option<&'a Arc>, + pub(super) allow_forced_provider_fallback: bool, + pub(super) allow_provider_grace: bool, + pub(super) user_priority: i8, + pub(super) connection_kind: crate::api::model::ConnectionKind, + pub(super) session_owner: Option<&'a str>, + pub(super) playback_kind: PlaybackKind, + pub(super) accept_requested_stream_url: bool, + pub(super) capacity_wait_timeout: Option, +} + +pub(crate) async fn resolve_stream_user_agent_index( + app_state: &AppState, + input: &ConfigInput, + has_provider_handle: bool, + username: &str, + session_owner: Option<&str>, +) -> Option { + if input.has_flag(ConfigInputFlags::UserAgentStreamIndex) && has_provider_handle { + if let Some(session_token) = session_owner { + app_state + .active_users + .get_or_assign_user_agent_stream_index(username, session_token) + .await + .or_else(|| Some(app_state.active_users.next_user_agent_stream_index())) + } else { + Some(app_state.active_users.next_user_agent_stream_index()) + } + } else { + None + } +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines, clippy::fn_params_excessive_bools)] +pub(super) async fn create_stream_response_details( + app_state: &Arc, + stream_options: &StreamOptions, + stream_url: &str, + username: &str, + fingerprint: &Fingerprint, + req_headers: &HeaderMap, + input: &Arc, + stream_channel: &StreamChannel, + item_type: PlaylistItemType, + content_representation: crate::api::model::ProviderContentRepresentationMode, + share_stream: bool, + connection_permission: UserConnectionPermission, + force_provider: Option<&Arc>, + allow_forced_provider_fallback: bool, + allow_provider_grace: bool, + virtual_id: VirtualId, + user_priority: i8, + connection_kind: crate::api::model::ConnectionKind, + is_reopen: bool, + session_owner: Option<&str>, + session_headers: Option<&HashMap>, + accept_requested_stream_url: bool, + grace_hold_override: Option, + grace_resolution_context: Option, + current_session: Option>, + preacquired_provider_handle: Option, +) -> Result { + let mut streaming_strategy = resolve_streaming_strategy_with_provider_handle( + app_state, + stream_url, + fingerprint, + input, + StreamingAcquireOptions { + force_provider, + allow_forced_provider_fallback, + allow_provider_grace, + user_priority, + connection_kind, + session_owner, + playback_kind: PlaybackKind::classify(item_type, extract_extension_from_url(stream_url)), + accept_requested_stream_url, + capacity_wait_timeout: (stream_options.response_mode == StreamResponseMode::HlsResource) + .then_some(HLS_MEDIA_CAPACITY_WAIT), + }, + Some(stream_channel), + preacquired_provider_handle, + ) + .await; + // A capacity wait can outlive the session snapshot: revalidate the account binding and + // read the cookies valid now, before any upstream request is sent. + let current_session_headers; + let session_headers = match current_session { + Some(guard) => { + match app_state + .active_users + .current_session_provider_headers(guard.username, guard.token, guard.identity, stream_url) + .await + { + SessionProviderHeaders::NoSession => { + app_state + .connection_manager + .release_managed_provider_handle(streaming_strategy.provider_handle.take()); + return Err(TuliproxError::Errors("playback session ended or switched provider account")); + } + SessionProviderHeaders::NoHeaders => current_session_headers = None, + SessionProviderHeaders::Headers(headers) => current_session_headers = Some(headers), + } + current_session_headers.as_ref() + } + None => session_headers, + }; + let user_agent_stream_index = resolve_stream_user_agent_index( + app_state, + input, + streaming_strategy.provider_handle.is_some(), + username, + session_owner, + ) + .await; + let mut grace_period_options = app_state.get_grace_options(); + grace_period_options.period_millis = get_grace_period_millis( + connection_permission, + &streaming_strategy.provider_stream_state, + grace_period_options.period_millis, + ); + if let Some(hold) = grace_hold_override { + grace_period_options.hold_stream = hold; + } + let provider_grace_active = + matches!(streaming_strategy.provider_stream_state, ProviderStreamState::GracePeriod(_, _)); + + let guard_provider_name = streaming_strategy + .provider_handle + .as_ref() + .and_then(|managed| managed.handle()) + .and_then(|handle| handle.allocation.get_provider_name()); + + if matches!( + streaming_strategy.provider_stream_state, + ProviderStreamState::Custom { reason: ProviderStreamCustomReason::ProviderExhausted, .. } + ) && can_provision_on_exhausted(app_state, input) + { + if let Some(managed) = streaming_strategy.provider_handle.take() { + drop(managed); + } + debug_if_enabled!( + "panel_api: provider connections exhausted; sending provisioning stream for input {}", + sanitize_sensitive_info(&input.name) + ); + let mut details = create_panel_api_provisioning_stream_details( + app_state, + input, + guard_provider_name.clone().or_else(|| Some(input.name.clone())), + &grace_period_options, + fingerprint.addr, + virtual_id, + ); + details.content_representation = content_representation; + return Ok(details); + } + + match streaming_strategy.provider_stream_state { + // custom stream means we display our own stream like connection exhausted, channel-unavailable... + ProviderStreamState::Custom { response: provider_stream, reason } => { + let (stream, stream_info) = provider_stream; + // When allocation is exhausted or no connection was acquired, guard_provider_name is None. + // Use input.name as fallback so the provider field is never empty. + let provider_name = guard_provider_name.clone().unwrap_or_else(|| input.name.clone()); + Ok(StreamDetails { + shared_subscriber_id: None, + stream, + stream_info, + provider_name: Some(provider_name), + request_url: None, + session_headers: session_headers.cloned(), + provider_session_headers: ProviderSessionHeaders::default(), + user_agent_stream_index, + grace_period: grace_period_options, + provider_grace_active: false, + disable_provider_grace: false, + reconnect_flag: None, + provider_handle: streaming_strategy.provider_handle.take(), + content_representation, + grace_resolution_context, + custom_reason: Some(reason), + response_mode: stream_options.response_mode, + session_registration: None, + }) + } + ProviderStreamState::Available(_provider_name, request_url) + | ProviderStreamState::GracePeriod(_provider_name, request_url) => { + let mut request_url = request_url; + debug_if_enabled!( + "Provider stream selection: allocated_provider={} actual_request_url={}", + sanitize_sensitive_info(guard_provider_name.as_deref().unwrap_or("?")), + sanitize_sensitive_info(resolve_request_url_for_logging(input, request_url.as_ref()).as_ref()) + ); + let defer_provider_stream_until_grace_check = if should_defer_provider_open_for_grace_hold( + provider_grace_active, + grace_period_options.hold_stream, + item_type, + is_reopen, + ) { + if let Some(provider_name) = guard_provider_name.as_ref() { + app_state.active_provider.is_over_limit(provider_name) + } else { + false + } + } else { + false + }; + let is_fallback_provider = force_provider + .is_some_and(|forced| guard_provider_name.as_ref().is_some_and(|allocated| allocated != forced)); + let session_headers = if is_fallback_provider { None } else { session_headers }; + let (stream, stream_info, provider_session_headers, reconnect_flag) = + if defer_provider_stream_until_grace_check { + debug_if_enabled!( + "Deferring provider stream open until grace check completes for {}", + sanitize_sensitive_info(resolve_request_url_for_logging(input, request_url.as_ref()).as_ref()) + ); + (None, None, ProviderSessionHeaders::default(), None) + } else if is_media_server_stream_ref_url(request_url.as_ref()) { + match open_media_server_stream_for_input(app_state, input, request_url.as_ref(), req_headers).await + { + Ok((stream, stream_info)) => { + (Some(stream), stream_info, ProviderSessionHeaders::default(), None) + } + Err(err) => { + error!("Can't open media-server stream: {err}"); + (None, None, ProviderSessionHeaders::default(), None) + } + } + } else { + let parsed_url = Url::parse(&request_url); + let request_url_valid = parsed_url.is_ok(); + let ((mut stream, mut stream_info, mut provider_session_headers), mut reconnect_flag) = + if let Ok(url) = parsed_url { + let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); + let disabled_headers = app_state.get_disabled_headers(); + let mut provider_stream_factory_options = + ProviderStreamFactoryOptions::new(&crate::api::model::ProviderStreamFactoryParams { + addr: fingerprint.addr, + item_type, + share_stream, + stream_options, + stream_url: &url, + req_headers, + input_headers: streaming_strategy.input_headers.as_ref(), + session_headers, + disabled_headers: disabled_headers.as_ref(), + default_user_agent: default_user_agent.as_deref(), + username: Some(username), + client_ip: Some(&fingerprint.client_ip), + stream_channel: Some(stream_channel), + connect_failure_stage: Some(FailureStage::ProviderOpen), + content_representation, + }); + + if let Some(stream_index) = user_agent_stream_index { + provider_stream_factory_options.apply_user_agent_stream_index(stream_index); + } + + let provider_config = input.get_resolve_provider(url.as_ref()); + provider_stream_factory_options.set_provider(provider_config); + provider_stream_factory_options.apply_input_options(input); + if input.input_type.is_stalker() { + provider_stream_factory_options.require_public_destination(); + } + + let reconnect_flag = provider_stream_factory_options.get_reconnect_flag_clone(); + let lifecycle = streaming_strategy + .provider_handle + .as_ref() + .and_then(ProviderStreamOpenLifecycle::from_managed); + let provider_stream = match open_provider_stream_with_lifecycle( + &app_state.provider_stream_ctx(), + &app_state.http_clients.default.load(), + provider_stream_factory_options, + lifecycle, + ) + .await + { + None => (None, None, ProviderSessionHeaders::default()), + Some(open) => split_provider_stream_open(open), + }; + (provider_stream, Some(reconnect_flag)) + } else { + ((None, None, ProviderSessionHeaders::default()), None) + }; + let should_refresh_stalker = should_refresh_stalker_playback( + input.input_type, + request_url_valid, + stream_info.as_ref().map(|(_, status, _, _)| *status), + ); + if should_refresh_stalker { + let force_stalker_refresh = + stream_info.as_ref().is_some_and(|(_, status, _, _)| status.is_client_error()); + let kind = stalker_stream_kind(stream_channel.cluster, item_type); + let resolve_result = re_resolve_stalker_url_singleflight( + app_state, + input, + stream_channel.provider_id, + kind, + force_stalker_refresh, + ) + .await; + match resolve_result { + Ok(Some(refreshed_url)) => { + if let Ok(url) = Url::parse(&refreshed_url) { + let default_user_agent = + app_state.app_config.config.load().default_user_agent.clone(); + let disabled_headers = app_state.get_disabled_headers(); + let mut options = ProviderStreamFactoryOptions::new( + &crate::api::model::ProviderStreamFactoryParams { + addr: fingerprint.addr, + item_type, + share_stream, + stream_options, + stream_url: &url, + req_headers, + input_headers: streaming_strategy.input_headers.as_ref(), + session_headers, + disabled_headers: disabled_headers.as_ref(), + default_user_agent: default_user_agent.as_deref(), + username: Some(username), + client_ip: Some(&fingerprint.client_ip), + stream_channel: Some(stream_channel), + connect_failure_stage: Some(FailureStage::ProviderOpen), + content_representation, + }, + ); + if let Some(stream_index) = user_agent_stream_index { + options.apply_user_agent_stream_index(stream_index); + } + options.set_provider(input.get_resolve_provider(url.as_ref())); + options.apply_input_options(input); + options.require_public_destination(); + if let Some(m) = streaming_strategy.provider_handle.as_mut() { + let _ = m.renew_opening_tokens(); + } + let retry_reconnect_flag = options.get_reconnect_flag_clone(); + let retry_lifecycle = streaming_strategy + .provider_handle + .as_ref() + .and_then(ProviderStreamOpenLifecycle::from_managed); + let retried = open_provider_stream_with_lifecycle( + &app_state.provider_stream_ctx(), + &app_state.http_clients.default.load(), + options, + retry_lifecycle, + ) + .await; + if let Some(open) = retried { + (stream, stream_info, provider_session_headers) = + split_provider_stream_open(open); + reconnect_flag = Some(retry_reconnect_flag); + request_url = refreshed_url; + } else { + // Keep the original stream/stream_info: the upstream response + // might still be serveable, and its status is needed for reporting. + debug!("Stalker re-resolve retry could not open a stream, keeping original provider response"); + } + } + } + Ok(None) => {} + Err(err) => { + warn!( + "Failed to refresh Stalker playback URL: {}", + sanitize_sensitive_info(&err.to_string()) + ); + } + } + } + (stream, stream_info, provider_session_headers, reconnect_flag) + }; + + if is_fallback_provider && stream.is_some() { + if let Some(token) = session_owner { + let _ = + app_state.active_users.update_session_provider_headers(username, token, &HashMap::new()).await; + } + } + + if log_enabled!(log::Level::Debug) { + if let Some((headers, status_code, response_url, _custom_video_type)) = stream_info.as_ref() { + debug!( + "Responding stream request {} with status {}, headers {:?}", + sanitize_sensitive_info(response_url.as_ref().map_or(stream_url, |s| s.as_str())), + status_code, + headers + ); + } + } + + // An intentional deferred open must retain its grace allocation until body polling + // resumes the provider request. Other failed opens release their allocation here. + let failed_open = stream.is_none() + || matches!(stream_info.as_ref(), Some((_, _, _, Some(custom))) if *custom != CustomVideoStreamType::Provisioning); + let provider_handle = if failed_open && !defer_provider_stream_until_grace_check { + // The managed owner releases the provider slot synchronously on drop; + // a failed open must not rely on a lossy cleanup message. + drop(streaming_strategy.provider_handle.take()); + match stream_info.as_ref() { + // Finite HLS resources hand routine upstream errors (e.g. live-edge 404) to the client. + Some((_, status, _, None)) if stream.is_none() && !status.is_success() => { + debug!("Provider answered {status} for {}", sanitize_sensitive_info(&request_url)); + } + _ => error!("Can't open stream {}", sanitize_sensitive_info(&request_url)), + } + None + } else { + streaming_strategy.provider_handle.take() + }; + + Ok(StreamDetails { + shared_subscriber_id: None, + stream, + stream_info, + provider_name: guard_provider_name.clone(), + request_url: Some(request_url.clone()), + session_headers: session_headers.cloned(), + provider_session_headers, + user_agent_stream_index, + grace_period: grace_period_options, + provider_grace_active, + disable_provider_grace: false, + reconnect_flag, + provider_handle, + content_representation, + grace_resolution_context, + custom_reason: None, + response_mode: stream_options.response_mode, + session_registration: None, + }) + } + } +} + +impl

RedirectParams<'_, P> +where + P: PlaylistEntry, +{ + pub fn get_query_path(&self, provider_id: u32, url: &str) -> String { + let extension = self.stream_ext.map_or_else( + || extract_extension_from_url(url).map_or_else(String::new, ToString::to_string), + ToString::to_string, + ); + + // if there is an action_path (like for timeshift duration/start), it will be added in front of the stream_id + if self.action_path.is_empty() { + concat_string!(&provider_id.to_string(), &extension) + } else { + concat_string!(&trim_slash(self.action_path), "/", &provider_id.to_string(), &extension) + } + } +} + +pub(super) fn prepare_body_stream( + app_state: &Arc, + item_type: PlaylistItemType, + stream: S, +) -> axum::body::Body +where + S: futures::Stream> + Send + 'static, +{ + let throttle_kbps = usize::try_from(get_stream_throttle(app_state)).unwrap_or_default(); + let body_stream = if is_throttled_stream(item_type, throttle_kbps) { + info!("Stream throttling active: {}", human_readable_kbps(u64::try_from(throttle_kbps).unwrap_or_default())); + axum::body::Body::from_stream(ThrottledStream::new(stream.boxed(), throttle_kbps)) + } else { + axum::body::Body::from_stream(stream) + }; + body_stream +} + +pub(super) fn is_hop_by_hop_response_header(name: &HeaderName) -> bool { + matches!( + name.as_str(), + "connection" + | "keep-alive" + | "proxy-authenticate" + | "proxy-authorization" + | "te" + | "trailer" + | "trailers" + | "transfer-encoding" + | "upgrade" + ) +} + +pub(super) fn no_custom_video_fallback_status(app_config: &AppConfig) -> StatusCode { + // Two reasons we have no custom-video response: + // 1. Operator disabled `custom_stream_response_enabled` → return the + // configured fallback status (e.g. 502) so reverse proxies handle the + // socket consistently. + // 2. Operator enabled custom-video but the concrete resource is missing + // → return `400` so downstream `proxy_intercept_errors on;` (Nginx) + // can sever the socket instead of looping on `200 OK`. + // Collapsing both into the configured status code broke the Nginx-intercept + // contract that the operator relied on by enabling custom-video in the + // first place. + if is_custom_video_stream_enabled(app_config) { + StatusCode::BAD_REQUEST + } else { + get_custom_stream_response_error_status(app_config) + } +} + +pub async fn force_provider_stream_response( + fingerprint: &Fingerprint, + app_state: &Arc, + user_session: &UserSession, + stream_channel: StreamChannel, + ctx: ForceStreamRequestContext<'_>, + grace_mode: Option, +) -> axum::response::Response { + force_stream_response( + fingerprint, + app_state, + user_session, + stream_channel, + ctx, + grace_mode, + StreamResponseMode::Stream, + ) + .await +} + +/// Answers a failed finite HLS resource with its status and the forwarded upstream headers. +/// Its provider slot is freed and an entry reservation no body ever used is released; a session +/// that streams elsewhere keeps its reservation (see `release_unbound_session_reservation`). +pub(super) async fn reject_hls_resource( + app_state: &Arc, + username: &str, + session_token: &str, + stream_details: StreamDetails, + status: StatusCode, +) -> axum::response::Response { + let StreamDetails { provider_handle, stream_info, .. } = stream_details; + app_state.connection_manager.release_managed_provider_handle(provider_handle); + app_state.active_users.release_unbound_session_reservation(username, session_token, None, false).await; + let mut response = status.into_response(); + if let Some((headers, _, _, None)) = stream_info { + for (name, value) in headers { + if let (Ok(name), Ok(value)) = (HeaderName::try_from(name), HeaderValue::try_from(value)) { + response.headers_mut().insert(name, value); + } + } + } + response +} + +#[allow(clippy::too_many_lines)] +pub(super) async fn force_stream_response( + fingerprint: &Fingerprint, + app_state: &Arc, + user_session: &UserSession, + mut stream_channel: StreamChannel, + ctx: ForceStreamRequestContext<'_>, + grace_mode: Option, + response_mode: StreamResponseMode, +) -> axum::response::Response { + let hls_resource = response_mode == StreamResponseMode::HlsResource; + // Finite HLS resources stay pinned to the session account and never rebind or clean up + // other sockets, so they skip the session transition gate. Holding it across the + // capacity wait would serialize parallel audio and video rendition requests. + let _transition_guard = if hls_resource { + None + } else { + Some(app_state.active_users.acquire_playback_transition(&ctx.user.username, &user_session.token).await) + }; + let stream_options = get_stream_options(&app_state.app_config, response_mode); + let share_stream = false; + let connection_permission = UserConnectionPermission::Allowed; + let item_type = stream_channel.item_type; + + // Forced reopens must clear stale provider slots before reacquiring. For adaptive HLS/DASH + // and Catchup sessions we only target old active stream sockets of the same session, never + // manifest-only session addresses, otherwise the controlling playlist request gets torn down. + let cleanup_addrs = if hls_resource { + Vec::new() + } else if item_type.is_live_adaptive() || item_type == PlaylistItemType::Catchup { + app_state + .active_users + .adaptive_session_stream_cleanup_addrs(&ctx.user.username, &user_session.token, &fingerprint.addr) + .await + } else { + session_reacquire_cleanup_addrs(user_session, &fingerprint.addr) + }; + + if cleanup_addrs.is_empty() { + debug_if_enabled!( + "Forced reopen cleanup had no stale targets for item_type={item_type:?} session={} current_addr={}", + sanitize_sensitive_info(&user_session.token), + sanitize_sensitive_info(&fingerprint.addr.to_string()) + ); + } else { + debug_if_enabled!( + "Forced reopen cleanup releasing {} stale target(s) for item_type={item_type:?} session={} current_addr={}", + cleanup_addrs.len(), + sanitize_sensitive_info(&user_session.token), + sanitize_sensitive_info(&fingerprint.addr.to_string()) + ); + cleanup_forced_reopen_addrs(app_state, &user_session.token, &cleanup_addrs).await; + } + + // HLS child URLs remain bound to the manifest account before the first media byte. + // Other streams prefer the account after media starts or while its allocation is active, + // and can fall back to the lineup when that account is exhausted. + let preferred_provider = (hls_resource + || item_type.is_live() + || user_session.media_started.load(std::sync::atomic::Ordering::Acquire) + || app_state.active_provider.should_reuse_playback_provider(&user_session.token, &user_session.provider)) + .then_some(&user_session.provider); + // Child URLs and their signed tokens belong to the account that fetched the playlist. + let allow_forced_provider_fallback = + !hls_resource && (!item_type.requires_provider_affinity() || allows_provider_pool_failover(item_type)); + // Never allow provider-side grace for forced seek/session reacquire. + // Over-allocation here would break provider-side one-connection limits. + let allow_provider_grace = false; + let connection_kind = user_session.connection_kind.unwrap_or(crate::api::model::ConnectionKind::Normal); + + let session_identity = user_session.identity(); + let current_session = hls_resource.then_some(CurrentSessionGuard { + username: &ctx.user.username, + token: &user_session.token, + identity: session_identity, + }); + // Finite resources read their cookies after the capacity wait instead of from this snapshot. + let provider_session_headers = preferred_provider + .filter(|_| !hls_resource) + .and_then(|_| user_session.provider_session_headers_for(&user_session.stream_url)); + let create_details = create_stream_response_details( + app_state, + &stream_options, + &user_session.stream_url, + &ctx.user.username, + fingerprint, + ctx.req_headers, + ctx.input, + &stream_channel, + item_type, + ctx.content_representation, + share_stream, + connection_permission, + preferred_provider, + allow_forced_provider_fallback, + allow_provider_grace, + VirtualId::new(stream_channel.virtual_id), + connection_priority_for_kind(ctx.user, connection_kind), + connection_kind, + true, + Some(user_session.token.as_str()), + provider_session_headers.as_deref(), + hls_resource || preferred_provider.is_some(), + grace_mode.map(|mode| matches!(mode, crate::api::model::GraceMode::Hold)), + None, + current_session, + None, + ); + // The FORBIDDEN exits below leave the session reservation alone: they only fire once this + // session identity is gone, and after an account switch the reservation belongs to the new + // binding. Dropping the cancelled future frees any provider slot it held. + let details_result = if hls_resource { + tokio::select! { + result = create_details => result, + () = app_state.active_users.wait_for_playback_session_end( + &ctx.user.username, &user_session.token, session_identity, + ) => return StatusCode::FORBIDDEN.into_response(), + } + } else { + create_details.await + }; + let mut stream_details = match details_result { + Ok(stream_details) => stream_details, + Err(err) => { + if hls_resource + && !app_state + .active_users + .playback_session_is_current(&ctx.user.username, &user_session.token, session_identity) + .await + { + return StatusCode::FORBIDDEN.into_response(); + } + app_state + .active_users + .release_unbound_session_reservation(&ctx.user.username, &user_session.token, None, false) + .await; + error!("Failed to stream: {err}"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + + if hls_resource { + if !app_state + .active_users + .playback_session_is_current(&ctx.user.username, &user_session.token, session_identity) + .await + { + return StatusCode::FORBIDDEN.into_response(); + } + stream_details.session_registration = Some(tuliprox_session::PlaybackSessionRegistration { + identity: session_identity, + enforce_limits: app_state.app_config.config.load().user_access_control, + grace_admitted: user_session.permission == UserConnectionPermission::GracePeriod, + }); + if let Some(status) = hls_resource_failure_status(&stream_details) { + return reject_hls_resource(app_state, &ctx.user.username, &user_session.token, stream_details, status) + .await; + } + } + + let deferred_grace_hold_stream = stream_details.has_deferred_provider_open(); + + if stream_details.has_stream() || deferred_grace_hold_stream { + let selected_provider = stream_details.provider_name.clone(); + let selected_request_url = stream_details.request_url.clone(); + let selected_provider_headers = stream_details.provider_session_headers.clone(); + let metering = prepare_stream_metering( + app_state, + user_session.stream_url.as_ref(), + false, + stream_details.stream.is_some(), + stream_details.has_deferred_provider_open(), + ); + let provider_response = + stream_details.stream_info.as_ref().map(|(h, sc, url, cvt)| (h.clone(), *sc, url.clone(), *cvt)); + if ctx.session_reservation_ttl_secs > 0 { + if let Some(provider_name) = stream_details.provider_name.as_ref() { + app_state.active_provider.refresh_adaptive_playback_lease( + provider_name, + &user_session.token, + PlaybackKind::classify(item_type, extract_extension_from_url(user_session.stream_url.as_ref())), + ctx.session_reservation_ttl_secs, + ); + } + } + if let Some(allocated_provider) = stream_details.provider_name.as_ref() { + if allocated_provider.as_ref() != user_session.provider.as_ref() { + let new_stream_url = + stream_details.request_url.as_deref().map_or_else(|| user_session.stream_url.clone(), Arc::from); + app_state + .active_users + .update_session_provider_binding( + &ctx.user.username, + &user_session.token, + Arc::clone(allocated_provider), + new_stream_url, + ) + .await; + } + if allocated_provider.as_ref() != user_session.provider.as_ref() + || !stream_details.provider_session_headers.is_empty() + { + let cookie_origin = provider_response + .as_ref() + .and_then(|(_, _, url, _)| url.as_ref()) + .map_or(user_session.stream_url.as_ref(), Url::as_str); + let active_users = &app_state.active_users; + if hls_resource { + // Cookies from the previous account never enter a jar after an account switch. + active_users + .update_current_session_provider_response_headers_from( + &ctx.user.username, + &user_session.token, + session_identity, + &stream_details.provider_session_headers, + cookie_origin, + ) + .await; + } else { + active_users + .update_session_provider_response_headers_from( + &ctx.user.username, + &user_session.token, + &stream_details.provider_session_headers, + cookie_origin, + ) + .await; + } + } + } + app_state.active_users.update_session_addr(&ctx.user.username, &user_session.token, &fingerprint.addr).await; + stream_channel.shared = share_stream; + let socket_bound = user_session.socket_bound; + let stream = match create_active_client_stream(crate::api::model::ActiveClientStreamParams { + stream_details, + app_state, + user: ctx.user, + connection_permission, + connection_kind: user_session.connection_kind.unwrap_or(crate::api::model::ConnectionKind::Normal), + fingerprint, + stream_channel, + socket_bound, + session_token: Some(&user_session.token), + req_headers: ctx.req_headers, + meter_uid: metering.meter_uid, + meter_stream: metering.meter_stream, + }) + .await + { + Ok(stream) => stream, + Err(error) => { + app_state + .active_users + .release_unbound_session_reservation(&ctx.user.username, &user_session.token, None, false) + .await; + return stream_admission_rejected_response(error, &ctx.user.username); + } + }; + + if let Some(provider) = selected_provider.as_deref() { + let session_url = selected_request_url.as_deref().unwrap_or(user_session.stream_url.as_ref()); + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: ctx.user, + session_token: &user_session.token, + virtual_id: user_session.virtual_id, + provider, + stream_url: session_url, + addr: &fingerprint.addr, + connection_permission, + connection_kind: user_session.connection_kind, + socket_bound: user_session.socket_bound, + }) + .await; + if !selected_provider_headers.is_empty() { + let cookie_origin = + provider_response.as_ref().and_then(|(_, _, url, _)| url.as_ref()).map_or(session_url, Url::as_str); + app_state + .active_users + .update_session_provider_response_headers_from( + &ctx.user.username, + &user_session.token, + &selected_provider_headers, + cookie_origin, + ) + .await; + } + } + + let (status_code, header_map) = get_stream_response_with_headers(provider_response.map(|(h, s, _, _)| (h, s))); + let mut response = axum::response::Response::builder().status(status_code); + for (key, value) in &header_map { + response = response.header(key, value); + } + + let body_stream = prepare_body_stream(app_state, item_type, stream); + debug_if_enabled!( + "Streaming provider forced stream request from {}", + sanitize_sensitive_info( + resolve_request_url_for_logging(ctx.input, user_session.stream_url.as_ref()).as_ref() + ) + ); + let mut response = try_unwrap_body!(response.body(body_stream)); + mark_response_as_uncompressed(&mut response); + return response; + } + + app_state.connection_manager.release_managed_provider_handle(stream_details.provider_handle); + app_state + .active_users + .release_unbound_session_reservation(&ctx.user.username, &user_session.token, None, false) + .await; + if let (Some(stream), _stream_info) = + create_channel_unavailable_stream(&app_state.app_config, &[], StatusCode::SERVICE_UNAVAILABLE) + { + app_state + .connection_manager + .update_stream_detail(&fingerprint.addr, CustomVideoStreamType::ChannelUnavailable) + .await; + debug!("Streaming custom stream"); + let mut response = try_unwrap_body!(axum::response::Response::builder() + .status(StatusCode::OK) + .body(axum::body::Body::from_stream(stream))); + mark_response_as_uncompressed(&mut response); + response + } else { + no_custom_video_fallback_status(&app_state.app_config).into_response() + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn stream_response( + fingerprint: &Fingerprint, + app_state: &Arc, + session_token: &str, + request_class: Option, + stream_channel: StreamChannel, + stream_url: &str, + pinned_provider: Option<&Arc>, + req_headers: &HeaderMap, + input: &Arc, + target: &Arc, + user: &ProxyUserCredentials, + connection_permission: UserConnectionPermission, + connection_kind: crate::api::model::ConnectionKind, + allow_exhausted_shared_reconnect: bool, + grace_mode: Option, +) -> axum::response::Response { + stream_response_with_provider_handle( + fingerprint, + app_state, + session_token, + request_class, + stream_channel, + stream_url, + pinned_provider, + req_headers, + input, + target, + user, + connection_permission, + connection_kind, + allow_exhausted_shared_reconnect, + grace_mode, + None, + ) + .await + .into_response() +} + +/// Reconnect window of an HLS playback lease: archive playback keeps the catchup window. +pub(crate) fn get_hls_playback_ttl_secs(app_state: &Arc, kind: PlaybackKind) -> u64 { + if kind == PlaybackKind::Catchup { + get_catchup_session_ttl_secs(app_state) + } else { + get_hls_session_ttl_secs(app_state) + } +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +pub(super) async fn try_shared_stream_response_if_any( + app_state: &Arc, + stream_url: &str, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + connect_permission: UserConnectionPermission, + connection_kind: crate::api::model::ConnectionKind, + mut stream_channel: StreamChannel, + session_token: &str, + req_headers: &HeaderMap, + placeholder_transition_version: Option, +) -> Option { + let subscriber_id = + tuliprox_core::model::SharedSubscriberId::from_stream_uid(app_state.connection_manager.next_stream_uid()); + let shared_subscription = SharedStreamManager::subscribe_shared_stream( + SharedStreamCtx { + app_config: &app_state.app_config, + shared_stream_manager: &app_state.shared_stream_manager, + active_provider: &app_state.active_provider, + connection_manager: &app_state.connection_manager, + }, + stream_url, + &fingerprint.addr, + subscriber_id, + connection_priority_for_kind(user, connection_kind), + connection_kind, + ) + .await; + let (stream, provider, cleanup_capability) = match shared_subscription { + Ok(Some(subscription)) => subscription, + Ok(None) => return None, + Err(reason) => { + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + placeholder_transition_version, + placeholder_transition_version.is_some(), + ) + .await; + return Some(stream_admission_rejected_response(reason.into(), &user.username)); + } + }; + debug_if_enabled!("Using shared stream {}", sanitize_sensitive_info(stream_url)); + if let Some(headers) = app_state.shared_stream_manager.get_shared_state_headers(stream_url).await { + let (status_code, header_map) = get_stream_response_with_headers(Some((headers.clone(), StatusCode::OK))); + let mut grace_period_options = app_state.get_grace_options(); + if connect_permission != UserConnectionPermission::GracePeriod { + grace_period_options.period_millis = 0; + } + let mut stream_details = StreamDetails::from_stream(stream, grace_period_options); + stream_details.shared_subscriber_id = Some(cleanup_capability); + + stream_details.provider_name = provider; + let socket_bound = + is_socket_bound_playback_session(stream_channel.item_type, extract_extension_from_url(stream_url)); + // A shared origin may have no provider allocation (e.g. a local/unknown owner), in + // which case `provider` is `None`. The subscriber must still be tracked as a user + // session, so fall back to a stable placeholder name rather than skipping session + // creation; `is_over_limit` on an unknown provider is a no-op. + let provider_name = stream_details.provider_name.as_deref().unwrap_or("shared"); + let _ = app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user, + session_token, + virtual_id: stream_channel.virtual_id, + provider: provider_name, + stream_url, + addr: &fingerprint.addr, + connection_permission: connect_permission, + connection_kind: Some(connection_kind), + socket_bound, + }) + .await; + stream_channel.shared = true; + stream_channel.shared_joined_existing = Some(true); + // Joining an existing origin reuses its meter identity. When metrics are disabled + // no meter exists to reserve; reserving one here would leave an ownerless entry that + // teardown cannot match and remove. + let (meter_uid, pending_registration) = if is_stream_metrics_enabled(app_state) { + app_state + .shared_stream_manager + .reserve_meter_uid(stream_url, || app_state.connection_manager.next_stream_uid()) + } else { + (0, None) + }; + stream_channel.shared_stream_id = Some(u64::from(meter_uid)); + // The pending registration is carried through admission so its drop guard rolls + // the reservation back if the client stream is never created. + let mut metering = + StreamMeteringConfig { meter_uid, meter_stream: false, pending_shared_registration: pending_registration }; + let stream = match create_active_client_stream(crate::api::model::ActiveClientStreamParams { + stream_details, + app_state, + user, + connection_permission: connect_permission, + connection_kind, + fingerprint, + stream_channel, + socket_bound, + session_token: Some(session_token), + req_headers, + meter_uid: metering.meter_uid, + meter_stream: metering.meter_stream, + }) + .await + { + Ok(stream) => { + metering.commit_shared_registration(); + stream.boxed() + } + Err(error) => { + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + session_token, + placeholder_transition_version, + placeholder_transition_version.is_some(), + ) + .await; + return Some(stream_admission_rejected_response(error, &user.username)); + } + }; + let mut response = axum::response::Response::builder().status(status_code); + for (key, value) in &header_map { + response = response.header(key, value); + } + let mut response = response.body(axum::body::Body::from_stream(stream)).ok()?; + mark_response_as_uncompressed(&mut response); + return Some(response); + } + None +} + +pub fn is_stream_share_enabled(item_type: PlaylistItemType, target: &ConfigTarget) -> bool { + (item_type == PlaylistItemType::Live/* || item_type == PlaylistItemType::LiveHls */) + && target.options.as_ref().is_some_and(ConfigTargetOptions::share_live_mpeg_ts_enabled) +} + +pub fn is_hls_stream_share_enabled(target: &ConfigTarget) -> bool { + target.options.as_ref().is_some_and(ConfigTargetOptions::share_live_hls_enabled) +} + +impl ResourceFetchPolicy { + pub(super) const fn cache_key(self, resource_url: &str) -> Option<&str> { + match self { + Self::Public => Some(resource_url), + Self::NonPublic => None, + } + } + + /// Whether an upstream response may be relayed to the client. + /// + /// A public destination was chosen by the user's own configuration, so what it answered is what + /// the client asked for. A resource proxy whose destination is deliberately hidden from the client + /// must not become a reader for that destination: status, headers and body of an upstream error can + /// disclose more about the internal service than the link itself. + pub(super) const fn relays_upstream_response(self) -> bool { matches!(self, Self::Public) } +} + +pub fn is_seek_request(cluster: XtreamCluster, req_headers: &HeaderMap) -> bool { + // seek only for non-live streams + if cluster == XtreamCluster::Live { + return false; + } + + // seek requests contains range header + let range = req_headers.get("range").and_then(|h| h.to_str().ok()).map(ToString::to_string); + + if let Some(range) = range { + if range.starts_with("bytes=") { + return true; + } + } + false +} + +pub fn is_seekable_media_request(cluster: XtreamCluster, req_headers: &HeaderMap, extension: Option<&str>) -> bool { + !extension.is_some_and(|ext| ext.eq_ignore_ascii_case(HLS_EXT)) && is_seek_request(cluster, req_headers) +} + +pub(crate) fn should_allow_exhausted_shared_reconnect( + share_stream: bool, + user_session: Option<&UserSession>, + requested_virtual_id: u32, + requested_stream_url: &str, +) -> bool { + share_stream + && user_session.is_some_and(|session| { + session.permission != UserConnectionPermission::Exhausted + && session.virtual_id == requested_virtual_id + && session.stream_url.as_ref() == requested_stream_url + }) +} diff --git a/backend/app/src/api/api_utils/tests.rs b/backend/app/src/api/api_utils/tests.rs index 671dbc1e0..7394fd0be 100644 --- a/backend/app/src/api/api_utils/tests.rs +++ b/backend/app/src/api/api_utils/tests.rs @@ -1,11015 +1,38 @@ use super::*; -use crate::{ - api::model::{ - ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, ConnectionManager, EventManager, - MetadataUpdateManager, PlaylistStorageState, ProviderConfig as RuntimeProviderConfig, ProviderConfigConnection, - SharedStreamManager, - }, - auth::Fingerprint, - model::{ - AppConfig, Config, ConfigInput, ConfigInputAlias, ConfigProvider, ConfigTarget, GracePeriodOptions, - MediaToolCapabilities, NetworkAccess, ProxyUserCredentials, SourcesConfig, StreamHistoryConfig, - }, - repository::GeoIp, - utils::FileLockManager, -}; -use arc_swap::{ArcSwap, ArcSwapOption}; -use axum::{ - extract::{ConnectInfo, State}, - http::{HeaderMap, Response, StatusCode}, - routing::get, - Router, -}; -use bytes::Bytes; -use futures::stream; -use http_body_util::BodyExt; -use shared::{ - defaults::{default_catchup_session_ttl_secs, default_hls_session_ttl_secs}, - foundation::Filter, - model::{ - AdmissionStrategy, ClusterFlags, ConfigPaths, ConfigProviderDto, ConfigTargetOptions, GeoIpUnavailablePolicy, - InputFetchMethod, InputType, PlaylistItem, PlaylistItemHeader, PlaylistItemType, ProcessingOrder, - ProviderUrlSelectionPolicy, ProxyType, StreamChannel, TargetType, XtreamCluster, - }, - utils::Internable, -}; -use std::{borrow::Cow, collections::HashMap, net::SocketAddr, sync::Arc}; -use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpListener, - sync::RwLock, -}; -use tuliprox_core::utils::response_compression::should_compress_response; -use tuliprox_session::{ - admission::{evaluate_remaining_strategies_after_grace, get_effective_admission_strategies}, - AdmissionRejectionReason, GraceResolutionContext, + +mod admission; +mod catchup; +mod cleanup; +mod http_load; +mod http_smoke; +mod http_soak; +mod local_files; +mod network_access; +mod provider_leases; +mod provider_urls; +mod resource_proxy; +mod serialization; +mod session_identity; +mod stalker; +mod stream_responses; +mod support; + +use self::support::{ + create_test_app_config, create_test_app_state, create_test_app_state_for_config, + create_test_app_state_with_stream_config, create_test_dual_provider_app_config, + create_test_dual_provider_app_state, create_test_fingerprint, create_test_live_channel, create_test_local_channel, + create_test_provider_app_config, create_test_provider_app_state, create_test_shared_target, load_test_channel, + load_test_rss_kib, load_test_session, load_test_user, mock_geoip, spawn_controlled_fake_origin, + spawn_legacy_hls_test_origin, spawn_load_test_origin, spawn_range_aware_test_origin, user_with_network_access, + validate_load_response, FakeOriginMode, }; -#[test] -fn stalker_playback_refreshes_invalid_or_rejected_urls() { - assert!(should_refresh_stalker_playback(InputType::Stalker, false, None)); - assert!(should_refresh_stalker_playback(InputType::Stalker, true, Some(StatusCode::UNAUTHORIZED))); - assert!(!should_refresh_stalker_playback(InputType::Stalker, true, Some(StatusCode::OK))); - assert!(!should_refresh_stalker_playback(InputType::Xtream, false, None)); -} - -#[test] -fn initial_stalker_playback_resolves_only_empty_urls() { - assert!(needs_initial_stalker_resolution(InputType::Stalker, "")); - assert!(!needs_initial_stalker_resolution(InputType::Stalker, "https://stream.example/live.ts")); - assert!(!needs_initial_stalker_resolution(InputType::Xtream, "")); - assert_eq!(stalker_stream_kind(XtreamCluster::Live, PlaylistItemType::Catchup), StalkerStreamKind::Archive); -} - -#[test] -fn stream_admission_rejection_is_503_without_upstream_headers() { - use crate::api::model::StreamAdmissionError; - - for error in [ - StreamAdmissionError::CleanupReceiverClosed, - StreamAdmissionError::CleanupAdmissionTimeout, - StreamAdmissionError::RegistrationRejected, - ] { - let response = stream_admission_rejected_response(error, "test-user"); - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert!(response.headers().get("content-length").is_none(), "must not leak upstream content-length"); - assert!(response.headers().get("content-range").is_none(), "must not leak upstream content-range"); - } -} - -fn test_runtime_provider(url: &str, username: &str, password: &str) -> Arc { - test_runtime_provider_with_type(url, username, password, InputType::Xtream) -} - -#[tokio::test] -async fn streamed_json_array_coalesces_small_entries() { - let response = stream_json_array_stream(stream::iter(0..4_096u32)); - let mut body = response.into_body(); - let mut frames = 0usize; - let mut bytes = Vec::new(); - while let Some(frame) = body.frame().await { - let Ok(frame) = frame else { - return; - }; - if let Ok(data) = frame.into_data() { - frames += 1; - bytes.extend_from_slice(&data); - } - } - assert!(frames <= 2, "small JSON entries should be coalesced, got {frames} frames"); - let decoded = serde_json::from_slice::>(&bytes); - assert!(decoded.is_ok_and(|values| values.len() == 4_096)); -} - -#[tokio::test] -async fn catchup_payload_probe_detects_fragmented_hls() { - let source = - stream::iter([Ok::<_, StreamError>(Bytes::from_static(b"#EX")), Ok(Bytes::from_static(b"TM3U\nsegment.ts\n"))]) - .boxed(); - - let result = probe_catchup_payload(source, std::time::Duration::from_secs(1)).await; - - assert!(matches!(&result, Ok(CatchupPayload::HlsManifest(_)))); - if let Ok(CatchupPayload::HlsManifest(manifest)) = result { - assert_eq!(manifest, b"#EXTM3U\nsegment.ts\n".as_slice()); - } -} - -#[tokio::test] -async fn catchup_payload_probe_replays_ts_bytes() { - let expected = Bytes::from_static(b"\x47direct-ts-payload"); - let source = stream::iter([Ok::<_, StreamError>(expected.clone())]).boxed(); - - let result = probe_catchup_payload(source, std::time::Duration::from_secs(1)).await; - - assert!(matches!(&result, Ok(CatchupPayload::Direct(_)))); - if let Ok(CatchupPayload::Direct(mut stream)) = result { - let mut actual = Vec::new(); - while let Some(chunk) = stream.next().await { - if let Ok(chunk) = chunk { - actual.extend_from_slice(&chunk); - } - } - assert_eq!(actual, expected.as_ref()); - } -} - -#[tokio::test] -async fn catchup_payload_probe_replays_partial_signature_at_eof() { - let expected = Bytes::from_static(b"#EXT"); - let source = stream::iter([Ok::<_, StreamError>(expected.clone())]).boxed(); - - let result = probe_catchup_payload(source, std::time::Duration::from_secs(1)).await; - - assert!(matches!(&result, Ok(CatchupPayload::Direct(_)))); - if let Ok(CatchupPayload::Direct(mut stream)) = result { - let actual = stream.next().await.and_then(Result::ok); - assert_eq!(actual.as_ref(), Some(&expected)); - assert!(stream.next().await.is_none()); - } -} - -#[tokio::test] -async fn catchup_payload_probe_rejects_oversized_manifest() { - let oversized = vec![b'x'; MAX_HLS_MANIFEST_BYTES]; - let source = - stream::iter([Ok::<_, StreamError>(Bytes::from_static(b"#EXTM3U")), Ok(Bytes::from(oversized))]).boxed(); - - let result = probe_catchup_payload(source, std::time::Duration::from_secs(1)).await; - - assert!(result.is_err()); -} - -#[tokio::test] -async fn coalesced_stream_remains_finished_when_polled_again() { - let stream = coalesce_byte_stream(stream::empty::>()); - futures::pin_mut!(stream); - - assert!(stream.next().await.is_none()); - assert!(stream.next().await.is_none()); -} - -fn test_runtime_provider_with_type( - url: &str, - username: &str, - password: &str, - input_type: InputType, -) -> Arc { - let url = if input_type == InputType::M3u { - format!("{url}/playlist.m3u8?username={username}&password={password}") - } else { - url.to_string() - }; - let input = ConfigInput { - name: "provider".intern(), - url, - username: Some(username.to_string()), - password: Some(password.to_string()), - input_type, - ..ConfigInput::default() - }; - Arc::new(RuntimeProviderConfig::new( - &input, - Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), - Arc::new(|_, _| {}), - )) -} - -fn test_runtime_provider_without_credentials(url: &str, input_type: InputType) -> Arc { - let input = ConfigInput { name: "provider".intern(), url: url.to_string(), input_type, ..ConfigInput::default() }; - Arc::new(RuntimeProviderConfig::new( - &input, - Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), - Arc::new(|_, _| {}), - )) -} - -#[test] -fn test_is_seek_request() { - let mut headers = HeaderMap::new(); - - // No range header - assert!(!is_seek_request(XtreamCluster::Video, &headers)); - - // Range: bytes=0- (Should be true now to allow session takeover on restart) - headers.insert("range", "bytes=0-".parse().unwrap()); - assert!(is_seek_request(XtreamCluster::Video, &headers)); - - // Range: bytes=100- (Should be true) - headers.insert("range", "bytes=100-".parse().unwrap()); - assert!(is_seek_request(XtreamCluster::Video, &headers)); - - // Range: bytes=100-200 (Should be true) - headers.insert("range", "bytes=100-200".parse().unwrap()); - assert!(is_seek_request(XtreamCluster::Video, &headers)); - - // Live cluster should always return false - headers.insert("range", "bytes=100-".parse().unwrap()); - assert!(!is_seek_request(XtreamCluster::Live, &headers)); -} - -#[test] -fn hls_manifests_are_not_forced_as_seek_responses() { - let mut headers = HeaderMap::new(); - headers.insert("range", HeaderValue::from_static("bytes=0-")); - - assert!(!is_seekable_media_request(XtreamCluster::Video, &headers, Some(HLS_EXT))); - assert!(is_seekable_media_request(XtreamCluster::Video, &headers, Some(".ts"))); -} - -#[test] -fn resolve_redirect_location_resolves_provider_scheme_urls() { - let provider = ConfigProvider::from(&ConfigProviderDto { - name: "develop".intern(), - urls: vec!["https://provider.example".intern()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, - dns: None, - }); - let input = ConfigInput { - name: "provider".intern(), - provider_configs: Some(vec![Arc::new(provider)]), - ..ConfigInput::default() - }; - - let resolved = - resolve_redirect_location(Some(&input), "provider://develop/live/provider-user/provider-pass/33486.m3u8") - .expect("provider url should resolve"); - - assert_eq!(resolved, "https://provider.example/live/provider-user/provider-pass/33486.m3u8"); -} - -#[test] -fn stream_alternative_url_keeps_unmatched_urls_unchanged() { - let input = ConfigInput { - name: "source".intern(), - url: "http://source.example".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - input_type: InputType::Xtream, - ..ConfigInput::default() - }; - let alias = test_runtime_provider("http://alias.example", "alias-user", "alias-pass"); - let stream_url = "http://other.example/live/source-user/source-pass/123.ts"; - - let rewritten = get_stream_alternative_url(stream_url, &input, &alias); - - assert_eq!(rewritten, None); -} - -#[test] -fn stream_alternative_url_rewrites_only_query_auth_fields() { - let input = ConfigInput { - name: "source".intern(), - url: "http://source.example".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - input_type: InputType::Xtream, - ..ConfigInput::default() - }; - let alias = test_runtime_provider("http://alias.example", "alias-user", "alias-pass"); - let stream_url = "http://source.example/player?token=source-user&username=source-user&password=source-pass"; - - let rewritten = get_stream_alternative_url(stream_url, &input, &alias); - - assert_eq!( - rewritten, - Some("http://alias.example/player?token=source-user&username=alias-user&password=alias-pass".to_string()) - ); -} - -#[test] -fn stream_url_matches_provider_requires_base_url_and_account_identity() { - let provider = test_runtime_provider("http://same.example", "selected-user", "selected-pass"); - - assert!(stream_url_matches_provider("http://same.example/live/selected-user/selected-pass/123.ts", &provider)); - assert!(stream_url_matches_provider( - "http://same.example/timeshift/selected-user/selected-pass/30/2026-06-15:20-00/123.ts", - &provider - )); - assert!(stream_url_matches_provider( - "http://same.example/future-route/selected-user/selected-pass/opaque/123.ts", - &provider - )); - assert!(!stream_url_matches_provider("http://same.example/live/other-user/other-pass/123.ts", &provider)); - assert!(!stream_url_matches_provider( - "http://same.example/timeshift/other-user/other-pass/30/2026-06-15:20-00/123.ts", - &provider - )); - assert!(!stream_url_matches_provider( - "http://same.example/future-route/other-user/other-pass/opaque/123.ts", - &provider - )); -} - -#[test] -fn stream_url_matches_provider_accepts_external_playlist_url_for_m3u_without_account_signature() { - let provider = - test_runtime_provider_with_type("http://provider.example", "selected-user", "selected-pass", InputType::M3u); - - assert!(stream_url_matches_provider( - "https://hlspackager.akamaized.net/live/DB/ALYAUM_TV/HLS/ALYAUM_TV.m3u8", - &provider - )); - assert!(stream_url_matches_provider( - "https://shd-gcp-live.edgenextcdn.net/live/bitmovin-mbc-1/15cf99af5de54063fdabfefe66adc075/index.m3u8", - &provider - )); -} - -#[test] -fn stream_url_matches_provider_rejects_external_cdn_url_with_wrong_account_signature() { - let provider = test_runtime_provider("http://provider.example", "selected-user", "selected-pass"); - - assert!(!stream_url_matches_provider("http://cdn.example/live/other-user/other-pass/123.ts", &provider)); - assert!(!stream_url_matches_provider( - "http://cdn.example/segment.ts?username=other-user&password=other-pass", - &provider - )); -} - -#[test] -fn stream_url_matches_provider_rejects_external_cdn_url_with_wrong_account_signature_for_m3u() { - let provider = - test_runtime_provider_with_type("http://provider.example", "selected-user", "selected-pass", InputType::M3u); - - assert!(!stream_url_matches_provider( - "http://cdn.example/segment.ts?username=other-user&password=other-pass", - &provider - )); -} - -#[test] -fn stream_url_matches_provider_detects_m3u_path_credentials_against_alias_account() { - // Regression: a cross-host M3U URL whose path embeds the alias's - // account credentials must be detected as an account signature and - // validated, not silently allowed as an open URL. - let provider = - test_runtime_provider_with_type("http://provider.example", "selected-user", "selected-pass", InputType::M3u); - - // Matching path credentials -> allowed (account matches). - assert!(stream_url_matches_provider("http://cdn.example/live/selected-user/selected-pass/123.ts", &provider)); -} - -#[test] -fn stream_url_matches_provider_rejects_open_external_cdn_url_without_account_signature_for_xtream() { - let provider = test_runtime_provider("http://provider.example", "selected-user", "selected-pass"); - - assert!(!stream_url_matches_provider("http://cdn.example/open/playlist.m3u8", &provider)); - assert!(!stream_url_matches_provider("http://cdn.example/open/segment.ts?key=signedopaque", &provider)); -} - -#[test] -fn stream_url_matches_provider_rejects_external_cdn_url_for_xtream_even_with_valid_account_signature() { - let provider = test_runtime_provider("http://provider.example", "selected-user", "selected-pass"); - - assert!(!stream_url_matches_provider("http://cdn.example/live/selected-user/selected-pass/123.ts", &provider)); - assert!(!stream_url_matches_provider( - "http://cdn.example/segment.ts?username=selected-user&password=selected-pass", - &provider - )); -} - -#[test] -fn find_input_account_by_signature_matches_main_input_and_alias_accounts() { - let input = ConfigInput { - name: "source".intern(), - url: "http://provider.example".to_string(), - username: Some("main-user".to_string()), - password: Some("main-pass".to_string()), - input_type: InputType::Xtream, - aliases: Some(vec![ConfigInputAlias { - id: 2, - name: "alias".intern(), - url: "http://alias.example".to_string(), - username: Some("alias-user".to_string()), - password: Some("alias-pass".to_string()), - max_connections: 1, - priority: 0, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }; - - let main = find_input_account_by_signature("http://cdn.example/live/main-user/main-pass/1.ts", &input); - assert_eq!( - main, - Some(("http://provider.example".to_string(), Some("main-user".to_string()), Some("main-pass".to_string()),)) - ); - - let alias = find_input_account_by_signature("http://cdn.example/live/alias-user/alias-pass/1.ts", &input); - assert_eq!( - alias, - Some(("http://alias.example".to_string(), Some("alias-user".to_string()), Some("alias-pass".to_string()),)) - ); - - assert_eq!(find_input_account_by_signature("http://cdn.example/live/other/other/1.ts", &input), None); - assert_eq!(find_input_account_by_signature("http://cdn.example/open/playlist.m3u8", &input), None); -} - -#[test] -fn get_stream_alternative_url_rewrites_external_cdn_url_with_valid_account_signature_for_alias_account() { - let input = ConfigInput { - name: "source".intern(), - url: "http://provider.example/playlist.m3u8?username=source-user&password=source-pass".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let alias = test_runtime_provider_with_type("http://alias.example", "alias-user", "alias-pass", InputType::M3u); - let stream_url = "http://cdn.example/live/source-user/source-pass/123.ts"; - - let rewritten = get_stream_alternative_url(stream_url, &input, &alias); - assert_eq!(rewritten, Some("http://cdn.example/live/alias-user/alias-pass/123.ts".to_string())); -} - -#[test] -fn get_stream_alternative_url_rewrites_opaque_m3u_token_for_alias_account() { - let input = ConfigInput { - name: "source".intern(), - url: "http://playlist.example/a.m3u?token=provider-a-token".to_string(), - input_type: InputType::M3u, - aliases: Some(vec![ConfigInputAlias { - id: 2, - name: "alias".intern(), - url: "http://playlist.example/b.m3u?token=provider-b-token".to_string(), - username: None, - password: None, - max_connections: 0, - priority: 0, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }; - let alias = test_runtime_provider_without_credentials( - "http://playlist.example/b.m3u?token=provider-b-token", - InputType::M3u, - ); - - let rewritten = - get_stream_alternative_url("http://stream.example/channel/segment.ts?token=provider-a-token", &input, &alias); - - assert_eq!(rewritten, Some("http://stream.example/channel/segment.ts?token=provider-b-token".to_string())); -} - -#[test] -fn get_stream_alternative_url_rewrites_single_opaque_m3u_credential_with_different_key() { - let input = ConfigInput { - name: "source".intern(), - url: "http://playlist.example/a.m3u?token=provider-a-token".to_string(), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let alias = test_runtime_provider_without_credentials( - "http://playlist.example/b.m3u?api_key=provider-b-key", - InputType::M3u, - ); - - let rewritten = get_stream_alternative_url( - "http://stream.example/segment.ts?token=provider-a-token&quality=hd", - &input, - &alias, - ); - - assert_eq!(rewritten, Some("http://stream.example/segment.ts?api_key=provider-b-key&quality=hd".to_string())); -} - -#[test] -fn get_stream_alternative_url_rejects_ambiguous_cross_key_credential_mapping() { - let input = ConfigInput { - name: "source".intern(), - url: "http://playlist.example/a.m3u?token=provider-a-token".to_string(), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let alias = test_runtime_provider_without_credentials( - "http://playlist.example/b.m3u?api_key=provider-b-key&auth=provider-b-auth", - InputType::M3u, - ); - - assert_eq!( - get_stream_alternative_url("http://stream.example/segment.ts?token=provider-a-token", &input, &alias), - None - ); -} - -#[test] -fn get_stream_alternative_url_rejects_partial_source_credential_mapping() { - let input = ConfigInput { - name: "source".intern(), - url: "http://playlist.example/a.m3u?token=provider-a-token&api_key=provider-a-key".to_string(), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let alias = test_runtime_provider_without_credentials( - "http://playlist.example/b.m3u?token=provider-b-token", - InputType::M3u, - ); - - assert_eq!( - get_stream_alternative_url( - "http://stream.example/segment.ts?token=provider-a-token&api_key=provider-a-key&quality=hd", - &input, - &alias, - ), - None - ); -} - -#[tokio::test] -async fn select_provider_stream_url_rewrites_opaque_m3u_token_for_allocated_alias() { - let input = ConfigInput { - name: "provider-a".intern(), - url: "http://playlist.example/a.m3u?token=provider-a-token".to_string(), - input_type: InputType::M3u, - aliases: Some(vec![ConfigInputAlias { - id: 2, - name: "provider-b".intern(), - url: "http://playlist.example/b.m3u?token=provider-b-token".to_string(), - username: None, - password: None, - max_connections: 0, - priority: 0, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }; - let alias = test_runtime_provider_without_credentials( - "http://playlist.example/b.m3u?token=provider-b-token", - InputType::M3u, - ); - - let temp = tempfile::tempdir().expect("temp dir should be created"); - let app_config = create_test_dual_provider_app_config(); - let mut config = (*app_config.config.load_full()).clone(); - config.storage_dir = temp.path().to_string_lossy().into_owned(); - app_config.config.store(Arc::new(config)); - let app_config = Arc::new(app_config); - let selected = select_provider_stream_url( - "http://stream.example/channel/segment.ts?token=provider-a-token", - &input, - &alias, - false, - &app_config, - ) - .await; - - assert_eq!( - selected, - Some(("provider".intern(), "http://stream.example/channel/segment.ts?token=provider-b-token".to_string(),)) - ); -} - -async fn independent_stream_token_alias_fixture( -) -> (tempfile::TempDir, Arc, ConfigInput, Arc) { - independent_stream_token_alias_fixture_with(&["http://stream.example:4000/323/mono.m3u8?token=backup-stream-token"]) - .await -} - -/// Primary and backup M3U accounts whose stream tokens are independent of their playlist -/// keys; only the backup alias playlist with `alias_urls` is persisted. -async fn independent_stream_token_alias_fixture_with( - alias_urls: &[&str], -) -> (tempfile::TempDir, Arc, ConfigInput, Arc) { - use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemHeader}; - use tuliprox_repository::{get_input_m3u_playlist_file_path, get_input_storage_path, persist_input_m3u_playlist}; - - let temp = tempfile::tempdir().expect("temp dir should be created"); - let app_config = create_test_dual_provider_app_config(); - let mut config = (*app_config.config.load_full()).clone(); - config.storage_dir = temp.path().to_string_lossy().into_owned(); - app_config.config.store(Arc::new(config)); - let app_config = Arc::new(app_config); - - let input = ConfigInput { - name: "primary-account".intern(), - url: "http://playlist.example/list.m3u?access_key=primary-playlist-key".to_string(), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let alias_input = ConfigInput { - name: "backup-account".intern(), - url: "http://playlist.example/list.m3u?access_key=backup-playlist-key".to_string(), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let alias = Arc::new(RuntimeProviderConfig::new( - &alias_input, - Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), - Arc::new(|_, _| {}), - )); - - let storage_path = get_input_storage_path(&alias.name, &app_config.config.load().storage_dir) - .await - .expect("alias storage should be created"); - let playlist_path = get_input_m3u_playlist_file_path(&storage_path, &alias.name); - let playlist = vec![PlaylistGroup { - id: 1, - title: "Live".intern(), - channels: alias_urls - .iter() - .enumerate() - .map(|(index, url)| PlaylistItem { - header: PlaylistItemHeader { - id: format!("channel-323-{index}").intern(), - input_stream_id: format!("channel-323-{index}").intern(), - url: (*url).intern(), - item_type: PlaylistItemType::Live, - xtream_cluster: XtreamCluster::Live, - ..PlaylistItemHeader::default() - }, - }) - .collect(), - xtream_cluster: XtreamCluster::Live, - }]; - persist_input_m3u_playlist(&app_config, &playlist_path, &playlist).await.expect("alias playlist should persist"); - (temp, app_config, input, alias) -} - -#[tokio::test] -async fn select_provider_stream_url_uses_persisted_alias_url_for_independent_stream_token() { - let (_temp, app_config, input, alias) = independent_stream_token_alias_fixture().await; - - let selected = select_provider_stream_url( - "http://stream.example:4000/323/mono.m3u8?token=primary-stream-token", - &input, - &alias, - false, - &app_config, - ) - .await; - - assert_eq!( - selected, - Some(( - "backup-account".intern(), - "http://stream.example:4000/323/mono.m3u8?token=backup-stream-token".to_string(), - )) - ); -} - -#[tokio::test] -async fn select_provider_stream_url_maps_flussonic_archive_to_alias_stream_token() { - let (_temp, app_config, input, alias) = independent_stream_token_alias_fixture().await; - - for archive_file in ["archive-1791225557-14400.m3u8", "mono-1791225557-14400.m3u8", "timeshift_abs-1791225557.ts"] { - let selected = select_provider_stream_url( - &format!("http://stream.example:4000/323/{archive_file}?token=primary-stream-token"), - &input, - &alias, - false, - &app_config, - ) - .await; - - assert_eq!( - selected, - Some(( - "backup-account".intern(), - format!("http://stream.example:4000/323/{archive_file}?token=backup-stream-token"), - )), - "{archive_file}" - ); - } -} - -#[tokio::test] -async fn select_provider_stream_url_prefers_archive_source_live_file() { - let (_temp, app_config, input, alias) = independent_stream_token_alias_fixture_with(&[ - "http://stream.example:4000/323/index.m3u8?token=index-token", - "http://stream.example:4000/323/mono.ts?token=mono-ts-token", - "http://stream.example:4000/323/mono.m3u8?token=mono-hls-token", - ]) - .await; - - for (archive_file, token) in [ - ("mono-1791225557-14400.m3u8", "mono-hls-token"), - ("mono-1791225557-14400.ts", "mono-ts-token"), - ("index-1791225557-14400.m3u8", "index-token"), - ] { - let selected = select_provider_stream_url( - &format!("http://stream.example:4000/323/{archive_file}?token=primary-stream-token"), - &input, - &alias, - false, - &app_config, - ) - .await; - - assert_eq!( - selected.map(|(_, url)| url), - Some(format!("http://stream.example:4000/323/{archive_file}?token={token}")), - "{archive_file}" - ); - } -} - -#[tokio::test] -async fn select_provider_stream_url_matches_uppercase_alias_live_file_for_archives_only() { - let (_temp, app_config, input, alias) = independent_stream_token_alias_fixture_with(&[ - "http://stream.example:4000/323/MONO.M3U8?token=backup-stream-token", - "http://stream.example:4000/324/INDEX.m3u8?token=upper-token", - "http://stream.example:4000/324/index.m3u8?token=lower-token", - ]) - .await; - - for (requested, expected) in [ - // A Flussonic archive finds its live file in any case. - ( - "http://stream.example:4000/323/mono-1791225557-14400.m3u8?token=primary-stream-token", - "http://stream.example:4000/323/mono-1791225557-14400.m3u8?token=backup-stream-token", - ), - // Exact lookups keep case-sensitive paths of other providers distinct. - ( - "http://stream.example:4000/324/INDEX.m3u8?token=primary-stream-token", - "http://stream.example:4000/324/INDEX.m3u8?token=upper-token", - ), - ( - "http://stream.example:4000/324/index.m3u8?token=primary-stream-token", - "http://stream.example:4000/324/index.m3u8?token=lower-token", - ), - ] { - let selected = select_provider_stream_url(requested, &input, &alias, false, &app_config).await; - assert_eq!(selected.map(|(_, url)| url).as_deref(), Some(expected), "{requested}"); - } -} - -#[tokio::test] -async fn select_provider_stream_url_keeps_unindexed_non_archive_url() { - let (_temp, app_config, input, alias) = independent_stream_token_alias_fixture().await; - let segment = "http://stream.example:4000/323/tracks-v1a1/dvr-2026/10/05/18/26/seg.ts?token=backup-stream-token"; - - let selected = select_provider_stream_url(segment, &input, &alias, false, &app_config).await; - - assert_eq!(selected, Some(("backup-account".intern(), segment.to_string()))); -} - -#[test] -fn get_stream_alternative_url_rewrites_timeshift_path_credentials_for_alias_account() { - let input = ConfigInput { - name: "source".intern(), - url: "http://provider.example".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - input_type: InputType::Xtream, - ..ConfigInput::default() - }; - let alias = test_runtime_provider("http://alias.example", "alias-user", "alias-pass"); - let stream_url = "http://provider.example/timeshift/source-user/source-pass/30/2026-06-15:20-00/123.ts"; - - let rewritten = get_stream_alternative_url(stream_url, &input, &alias); - assert_eq!( - rewritten, - Some("http://alias.example/timeshift/alias-user/alias-pass/30/2026-06-15:20-00/123.ts".to_string()) - ); -} - -#[test] -fn get_stream_alternative_url_rewrites_future_route_path_credentials_for_alias_account() { - let input = ConfigInput { - name: "source".intern(), - url: "http://provider.example".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - input_type: InputType::Xtream, - ..ConfigInput::default() - }; - let alias = test_runtime_provider("http://alias.example", "alias-user", "alias-pass"); - let stream_url = "http://provider.example/future-route/source-user/source-pass/opaque/123.ts"; - - let rewritten = get_stream_alternative_url(stream_url, &input, &alias); - assert_eq!(rewritten, Some("http://alias.example/future-route/alias-user/alias-pass/opaque/123.ts".to_string())); -} - -#[test] -fn get_stream_alternative_url_keeps_open_external_playlist_url_for_m3u() { - let input = ConfigInput { - name: "source".intern(), - url: "http://provider.example/playlist.m3u8?username=source-user&password=source-pass".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let alias = test_runtime_provider_with_type("http://alias.example", "alias-user", "alias-pass", InputType::M3u); - let stream_url = "https://cnbc-live.akamaized.net/cnbc/master.m3u8"; - - assert_eq!(get_stream_alternative_url(stream_url, &input, &alias), Some(stream_url.to_string())); -} - -#[test] -fn get_stream_alternative_url_keeps_open_external_multisegment_playlist_url_for_m3u() { - let input = ConfigInput { - name: "source".intern(), - url: "http://provider.example/playlist.m3u8?username=source-user&password=source-pass".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let alias = test_runtime_provider_with_type("http://alias.example", "alias-user", "alias-pass", InputType::M3u); - let stream_url = "https://hnpsechtsc.turknet.ercdn.net/xpnvudnlsv/cnbc-e/cnbc-e.m3u8"; - - assert_eq!(get_stream_alternative_url(stream_url, &input, &alias), Some(stream_url.to_string())); -} - -#[test] -fn get_stream_alternative_url_keeps_open_external_m3u_url_for_provider_without_credentials() { - let input = ConfigInput { - name: "source".intern(), - url: "http://provider.example/playlist.m3u8".to_string(), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let provider = test_runtime_provider_without_credentials("http://provider.example/playlist.m3u8", InputType::M3u); - let stream_url = "http://s.only4.tv/17113/video.m3u8?token=abc"; - - assert_eq!(get_stream_alternative_url(stream_url, &input, &provider), Some(stream_url.to_string())); -} - -#[test] -fn get_stream_alternative_url_rejects_query_credentials_for_provider_without_credentials() { - let input = ConfigInput { - name: "source".intern(), - url: "http://provider.example/playlist.m3u8".to_string(), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let provider = test_runtime_provider_without_credentials("http://provider.example/playlist.m3u8", InputType::M3u); - - assert_eq!( - get_stream_alternative_url( - "http://cdn.example/segment.ts?username=other-user&password=other-pass", - &input, - &provider - ), - None - ); -} - -#[test] -fn get_stream_alternative_url_rejects_basic_auth_credentials_for_provider_without_credentials() { - let input = ConfigInput { - name: "source".intern(), - url: "http://provider.example/playlist.m3u8".to_string(), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let provider = test_runtime_provider_without_credentials("http://provider.example/playlist.m3u8", InputType::M3u); - - assert_eq!(get_stream_alternative_url("http://user:pass@cdn.example/segment.ts", &input, &provider), None); -} - -#[test] -fn get_stream_alternative_url_rejects_external_m3u_url_with_unmatched_account_signature() { - let input = ConfigInput { - name: "source".intern(), - url: "http://provider.example/playlist.m3u8?username=source-user&password=source-pass".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let alias = test_runtime_provider_with_type("http://alias.example", "alias-user", "alias-pass", InputType::M3u); - - assert_eq!( - get_stream_alternative_url( - "http://cdn.example/segment.ts?username=other-user&password=other-pass", - &input, - &alias, - ), - None - ); -} - -#[test] -fn get_stream_alternative_url_does_not_passthrough_arbitrary_open_external_url_for_xtream() { - let input = ConfigInput { - name: "source".intern(), - url: "http://provider.example".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - input_type: InputType::Xtream, - ..ConfigInput::default() - }; - let alias = test_runtime_provider("http://alias.example", "alias-user", "alias-pass"); - let stream_url = "http://cdn.example/open/playlist.m3u8"; - - assert_eq!(get_stream_alternative_url(stream_url, &input, &alias), None); -} - -#[test] -fn xtream_vod_provider_url_requires_stored_input_identity() -> Result<(), Box> { - let config = create_test_dual_provider_app_config(); - let input = config.sources.load().inputs.first().cloned().ok_or("missing input")?; - let provider = RuntimeProviderConfig::new( - &input, - Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), - Arc::new(|_, _| {}), - ); - let stream_url = "http://cdn.example/r2/movie.mp4"; - let mut channel = create_test_live_channel(stream_url); - channel.item_type = PlaylistItemType::Video; - channel.cluster = XtreamCluster::Video; - channel.provider_id = 100; - assert_eq!(resolve_xtream_vod_provider_url(stream_url, &input, &provider, &channel), Some(stream_url.to_string())); - assert_eq!( - resolve_xtream_vod_provider_url("http://unrelated.example/movie.mp4", &input, &provider, &channel), - None - ); - channel.input_name = "other-input".intern(); - assert_eq!(resolve_xtream_vod_provider_url(stream_url, &input, &provider, &channel), None); - channel.input_name = Arc::clone(&input.name); - channel.item_type = PlaylistItemType::Live; - assert_eq!(resolve_xtream_vod_provider_url(stream_url, &input, &provider, &channel), None); - channel.item_type = PlaylistItemType::Video; - channel.provider_id = 0; - assert_eq!(resolve_xtream_vod_provider_url(stream_url, &input, &provider, &channel), None); - channel.provider_id = 100; - channel.url = "file:///internal/movie.mp4".intern(); - assert_eq!(resolve_xtream_vod_provider_url(&channel.url, &input, &provider, &channel), None); - let unrelated = test_runtime_provider("http://unrelated.example", "other-user", "other-pass"); - channel.url = stream_url.intern(); - assert_eq!(resolve_xtream_vod_provider_url(stream_url, &input, &unrelated, &channel), None); - Ok(()) -} - -#[tokio::test] -async fn xtream_vod_m3u_reverse_proxies_external_sources_and_alias_redirects_without_leaking_urls( -) -> Result<(), Box> { - for use_alias in [false, true] { - for source_path in ["/r2/movie.mp4", "/stream/account-bound-token"] { - let (cdn_addr, cdn_task) = spawn_legacy_hls_test_origin( - "HTTP/1.1 206 Partial Content\r\nContent-Type: video/mp4\r\nContent-Length: 4\r\nContent-Range: bytes 4-7/12\r\nAccept-Ranges: bytes\r\nReferer: http://private-provider.example/user/pass\r\nContent-Location: http://private-cdn.example/movie.mp4\r\nLocation: http://private-cdn.example/movie.mp4\r\nLink: \r\nConnection: close\r\n\r\n".to_string(), - b"DATA".to_vec(), - ).await; - let cdn_url = format!("http://{cdn_addr}/internal/media.mp4?token=private-cdn-token"); - let (redirect_addr, redirect_task) = spawn_legacy_hls_test_origin( - format!("HTTP/1.1 302 Found\r\nLocation: {cdn_url}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"), - Vec::new(), - ) - .await; - let config = create_test_dual_provider_app_config(); - let mut input = (**config.sources.load().inputs.first().ok_or("missing input")?).clone(); - let alias = input.aliases.as_mut().and_then(|aliases| aliases.first_mut()).ok_or("missing alias")?; - alias.url = format!("http://{redirect_addr}"); - let input = Arc::new(input); - config - .sources - .store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(config)); - let busy_addr = SocketAddr::from(([127, 0, 0, 1], 55401)); - let busy = if use_alias { - app_state.active_provider.acquire_exact_connection_with_grace( - &input.name, - &busy_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ) - } else { - None - }; - assert!(!use_alias || busy.is_some()); - let source_url = format!("http://{redirect_addr}{source_path}"); - let item = shared::model::M3uPlaylistItem::from(&PlaylistItem { - header: PlaylistItemHeader { - id: "900".intern(), - input_stream_id: "100".intern(), - virtual_id: VirtualId::new(100), - input_name: Arc::clone(&input.name), - url: source_url.intern(), - item_type: PlaylistItemType::Video, - xtream_cluster: XtreamCluster::Video, - additional_properties: Some(shared::model::StreamProperties::Video(Box::new( - shared::model::VideoStreamProperties { - container_extension: "mp4".intern(), - ..Default::default() - }, - ))), - ..Default::default() - }, - }); - let mut target = create_test_shared_target(); - target.options = None; - target.output = vec![tuliprox_core::model::TargetOutput::M3u(tuliprox_core::model::M3uTargetOutput::from( - &shared::model::M3uTargetOutputDto::default(), - ))]; - let mut user = load_test_user("vod-reverse-viewer"); - user.proxy = ProxyType::Reverse(None); - let mut headers = HeaderMap::new(); - headers.insert(header::RANGE, HeaderValue::from_static("bytes=4-7")); - let response = crate::api::endpoints::m3u_api::m3u_api_stream_loaded( - Arc::new(user), - Arc::new(target), - &create_test_fingerprint(SocketAddr::from(([127, 0, 0, 1], 55402))), - &headers, - &app_state, - item, - Arc::clone(&input), - Some("mp4"), - None, - None, - ) - .await - .into_response(); - assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response.headers().get(header::CONTENT_RANGE), Some(&HeaderValue::from_static("bytes 4-7/12"))); - for name in ["location", "content-location", "referer", "link"] { - assert!(!response.headers().contains_key(name)); - } - for value in response.headers().values() { - let value = value.to_str()?; - assert!(!value.contains(&redirect_addr.to_string())); - assert!(!value.contains(&cdn_addr.to_string())); - assert!(!value.contains("private-cdn-token")); - assert!(!value.contains("user2/pass2")); - } - let body = response.into_body().collect().await?.to_bytes(); - assert_eq!(body.as_ref(), b"DATA"); - let redirect_request = tokio::time::timeout(Duration::from_secs(5), redirect_task).await??; - let cdn_request = tokio::time::timeout(Duration::from_secs(5), cdn_task).await??; - let expected_path = if use_alias { "/movie/user2/pass2/100.mp4" } else { source_path }; - assert!(redirect_request.starts_with(&format!("GET {expected_path} "))); - assert!(cdn_request.starts_with("GET /internal/media.mp4?token=private-cdn-token ")); - assert!(redirect_request.to_ascii_lowercase().contains("range: bytes=4-7")); - assert!(cdn_request.to_ascii_lowercase().contains("range: bytes=4-7")); - app_state.active_provider.release_connection(&busy_addr); - } - } - Ok(()) -} - -#[test] -fn media_server_proxy_response_header_filter_drops_hop_by_hop_headers() { - for name in [ - "connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailer", - "trailers", - "transfer-encoding", - "upgrade", - ] { - assert!(is_hop_by_hop_response_header(&HeaderName::from_static(name))); - } - assert!(!is_hop_by_hop_response_header(&header::CONTENT_TYPE)); -} - -#[test] -fn media_server_image_error_status_classifies_client_and_upstream_failures() { - let parse_error = MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) - .detail("media server image URL is missing required path parts"); - assert_eq!(media_server_image_error_status(&parse_error), StatusCode::BAD_REQUEST); - - let not_found = MediaServerError::new(MediaServerErrorKind::MediaServerItemNotFound) - .detail("plex media-server image URL is missing image_path"); - assert_eq!(media_server_image_error_status(¬_found), StatusCode::NOT_FOUND); - - let upstream = MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) - .detail("media-server image request failed"); - assert_eq!(media_server_image_error_status(&upstream), StatusCode::BAD_GATEWAY); -} - -#[test] -fn media_server_playback_urls_are_proxy_only_redirect_guard_candidates() { - let plex_input = ConfigInput { input_type: InputType::Plex, ..ConfigInput::default() }; - let emby_input = ConfigInput { input_type: InputType::Emby, ..ConfigInput::default() }; - let m3u_input = ConfigInput { input_type: InputType::M3u, ..ConfigInput::default() }; - - assert!(is_media_server_playback_url( - &plex_input, - "media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fredacted" - )); - assert!(is_media_server_playback_url( - &m3u_input, - "media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fredacted" - )); - assert!(is_media_server_playback_url(&plex_input, "https://plex.example/stream.mkv")); - assert!(!is_media_server_playback_url(&emby_input, "https://emby.example/stream.mkv")); - assert!(!is_media_server_playback_url(&m3u_input, "https://provider.example/stream.mkv")); - assert!(!is_media_server_stream_ref_url("https://provider.example/stream.mkv")); - assert!(is_media_server_stream_ref_url("media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fredacted")); - assert_eq!( - resolve_request_url_for_logging( - &plex_input, - "media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fredacted" - ) - .as_ref(), - "media-server://" - ); -} - -#[test] -fn test_resolve_request_url_for_logging_respects_sanitization_setting() { - static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); - let _guard = LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - struct SanitizeSettingGuard(bool); - impl Drop for SanitizeSettingGuard { - fn drop(&mut self) { shared::utils::set_sanitize_sensitive_info(self.0); } - } - - let prev = shared::utils::is_sanitize_sensitive_info_enabled(); - let _restore = SanitizeSettingGuard(prev); - - let m3u_input = ConfigInput { input_type: InputType::M3u, ..ConfigInput::default() }; - let media_ref = "media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fsecret_key"; - let standard_url = "http://provider.example/live/stream.m3u8"; - - // 1. When sanitization is enabled: - shared::utils::set_sanitize_sensitive_info(true); - assert_eq!(resolve_request_url_for_logging(&m3u_input, media_ref).as_ref(), "media-server://"); - assert_eq!(resolve_request_url_for_logging(&m3u_input, standard_url).as_ref(), standard_url); - - // 2. When sanitization is disabled: - shared::utils::set_sanitize_sensitive_info(false); - assert_eq!(resolve_request_url_for_logging(&m3u_input, media_ref).as_ref(), media_ref); - assert_eq!(resolve_request_url_for_logging(&m3u_input, standard_url).as_ref(), standard_url); -} - -#[test] -fn test_streaming_response_extension_disables_compression() { - let mut response = Response::new(()); - mark_response_as_uncompressed(&mut response); - - assert!(!should_compress_response(&response)); -} - -async fn spawn_legacy_hls_test_origin( - response_head: String, - response_body: Vec, -) -> (SocketAddr, tokio::task::JoinHandle) { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let origin_addr = listener.local_addr().expect("test origin address"); - let origin_task = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.expect("test origin accepts request"); - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 1024]; - let read = socket.read(&mut chunk).await.expect("test origin reads request"); - if read == 0 { - break; - } - request.extend_from_slice(&chunk[..read]); - } - socket.write_all(response_head.as_bytes()).await.expect("test origin writes response headers"); - socket.write_all(&response_body).await.expect("test origin writes response body"); - String::from_utf8_lossy(&request).into_owned() - }); - (origin_addr, origin_task) -} - -async fn spawn_range_aware_test_origin( - body: &'static [u8], - connections: usize, -) -> (SocketAddr, tokio::task::JoinHandle>) { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let origin_addr = listener.local_addr().expect("test origin address"); - let origin_task = tokio::spawn(async move { - let mut requests = Vec::new(); - for _ in 0..connections { - let Ok((mut socket, _)) = listener.accept().await else { break }; - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 1024]; - let read = socket.read(&mut chunk).await.expect("test origin reads request"); - if read == 0 { - break; - } - request.extend_from_slice(&chunk[..read]); - } - let request_lower = String::from_utf8_lossy(&request).to_ascii_lowercase(); - requests.push(request_lower.clone()); - - let range = request_lower - .lines() - .find_map(|line| line.strip_prefix("range: bytes=")) - .and_then(|spec| spec.split_once('-')) - .and_then(|(start, end)| { - let start: usize = start.trim().parse().ok()?; - let end: usize = - if end.trim().is_empty() { body.len().saturating_sub(1) } else { end.trim().parse().ok()? }; - Some((start, end.min(body.len().saturating_sub(1)))) - }) - .unwrap_or((0, body.len().saturating_sub(1))); - - let slice = &body[range.0..=range.1]; - let head = format!( - "HTTP/1.1 206 Partial Content\r\nContent-Type: video/mp2t\r\nContent-Range: bytes {}-{}/{}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - range.0, - range.1, - body.len(), - slice.len() - ); - socket.write_all(head.as_bytes()).await.expect("test origin writes response headers"); - socket.write_all(slice).await.expect("test origin writes response body"); - } - requests - }); - (origin_addr, origin_task) -} - -#[allow(dead_code)] -#[derive(Clone)] -enum FakeOriginMode { - EmptyBody, - BlockFirstChunk(Arc), - ErrorBeforeFirstChunk, - ErrorAfterFirstChunk(Vec), - FixedBytes(Vec), -} - -async fn spawn_controlled_fake_origin( - mode: FakeOriginMode, - max_requests: usize, -) -> (SocketAddr, tokio::task::JoinHandle>) { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let origin_addr = listener.local_addr().expect("test origin address"); - let origin_task = tokio::spawn(async move { - let mut requests = Vec::new(); - for _ in 0..max_requests { - let Ok((mut socket, _)) = listener.accept().await else { break }; - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 1024]; - let Ok(read) = socket.read(&mut chunk).await else { break }; - if read == 0 { - break; - } - request.extend_from_slice(&chunk[..read]); - } - if request.is_empty() { - continue; - } - let request_lower = String::from_utf8_lossy(&request).to_ascii_lowercase(); - requests.push(request_lower); - - match &mode { - FakeOriginMode::EmptyBody => { - let head = - "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; - let _ = socket.write_all(head.as_bytes()).await; - let _ = socket.shutdown().await; - } - FakeOriginMode::BlockFirstChunk(notify) => { - let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"; - let _ = socket.write_all(head.as_bytes()).await; - let mut probe = [0_u8; 1]; - tokio::select! { - _ = notify.notified() => {}, - _ = socket.read(&mut probe) => {}, - } - let _ = socket.shutdown().await; - } - FakeOriginMode::ErrorBeforeFirstChunk => { - let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"; - let _ = socket.write_all(head.as_bytes()).await; - let _ = socket.shutdown().await; - } - FakeOriginMode::ErrorAfterFirstChunk(chunk) => { - let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"; - let _ = socket.write_all(head.as_bytes()).await; - let chunk_hdr = format!("{:x}\r\n", chunk.len()); - let _ = socket.write_all(chunk_hdr.as_bytes()).await; - let _ = socket.write_all(chunk).await; - let _ = socket.write_all(b"\r\n").await; - let _ = socket.flush().await; - let _ = socket.shutdown().await; - } - FakeOriginMode::FixedBytes(bytes) => { - let head = format!( - "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - bytes.len() - ); - let _ = socket.write_all(head.as_bytes()).await; - let _ = socket.write_all(bytes).await; - let _ = socket.shutdown().await; - } - } - } - requests - }); - (origin_addr, origin_task) -} - -async fn forced_legacy_hls_test_response( - origin_addr: SocketAddr, - request_headers: &HeaderMap, - client_port: u16, -) -> axum::response::Response { - let origin_url = format!("http://{origin_addr}/segment.ts"); - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::from([("Accept-Encoding".to_string(), "gzip".to_string())]), - url: format!("http://{origin_addr}"), - enabled: true, - priority: 0, - max_connections: 1, - method: InputFetchMethod::default(), - ..ConfigInput::default() - }); - let app_config = create_test_provider_app_config(); - app_config.sources.store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(app_config)); - let client_addr = SocketAddr::from(([127, 0, 0, 1], client_port)); - let fingerprint = create_test_fingerprint(client_addr); - let mut user = ProxyUserCredentials::default(); - user.username = "viewer".to_string(); - let session = UserSession { - token: format!("legacy-hls-marker-{client_port}"), - transition_version: 1, - virtual_id: 41, - provider: Arc::clone(&input.name), - stream_url: origin_url.as_str().intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), - user_agent_stream_index: None, - addr: client_addr, - socket_bound: false, - active_addrs: vec![client_addr], - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - }; - let mut stream_channel = create_test_local_channel(&origin_url); - stream_channel.provider_id = u32::from(input.id); - stream_channel.input_name = Arc::clone(&input.name); - stream_channel.item_type = PlaylistItemType::Catchup; - stream_channel.cluster = XtreamCluster::Live; - stream_channel.url = origin_url.as_str().intern(); - - force_provider_stream_response( - &fingerprint, - &app_state, - &session, - stream_channel, - ForceStreamRequestContext { - req_headers: request_headers, - input: &input, - user: &user, - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response() -} - -#[tokio::test] -async fn forced_hls_provider_response_disables_compression_and_streams_identity_bytes() { - const IDENTITY_BODY: &[u8] = b"legacy hls identity segment"; - - let mut encoder = async_compression::tokio::write::GzipEncoder::new(Vec::new()); - encoder.write_all(IDENTITY_BODY).await.expect("gzip test body encodes"); - encoder.shutdown().await.expect("gzip test encoder finishes"); - let encoded_body = encoder.into_inner(); - - let response_head = format!( - "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - encoded_body.len() - ); - let (origin_addr, origin_task) = spawn_legacy_hls_test_origin(response_head, encoded_body).await; - let mut request_headers = HeaderMap::new(); - request_headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); - let response = forced_legacy_hls_test_response(origin_addr, &request_headers, 55_310).await; - - assert_eq!(response.status(), StatusCode::OK); - assert!(!should_compress_response(&response)); - assert!(!response.headers().contains_key(header::CONTENT_ENCODING)); - assert!(!response.headers().contains_key(header::CONTENT_LENGTH)); - let body = response.into_body().collect().await.expect("legacy HLS response body").to_bytes(); - assert_eq!(body.as_ref(), IDENTITY_BODY); - - let request = origin_task.await.expect("test origin task completes").to_ascii_lowercase(); - assert!(request.contains("\r\naccept-encoding: identity\r\n")); -} - -#[tokio::test] -async fn forced_hls_unencoded_partial_response_preserves_range_and_disables_compression() { - const PARTIAL_BODY: &[u8] = b"cdef"; - let response_head = format!( - "HTTP/1.1 206 Partial Content\r\nContent-Type: video/mp2t\r\nContent-Range: bytes 2-5/10\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - PARTIAL_BODY.len() - ); - let (origin_addr, origin_task) = spawn_legacy_hls_test_origin(response_head, PARTIAL_BODY.to_vec()).await; - let mut request_headers = HeaderMap::new(); - request_headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); - request_headers.insert(header::RANGE, HeaderValue::from_static("bytes=2-")); - - let response = forced_legacy_hls_test_response(origin_addr, &request_headers, 55_311).await; - - assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - assert!(!should_compress_response(&response)); - assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); - assert_eq!(response.headers()[header::CONTENT_LENGTH], "4"); - assert!(!response.headers().contains_key(header::CONTENT_ENCODING)); - let body = response.into_body().collect().await.expect("legacy HLS partial body").to_bytes(); - assert_eq!(body.as_ref(), PARTIAL_BODY); - - let request = origin_task.await.expect("test origin task completes").to_ascii_lowercase(); - assert!(request.contains("\r\naccept-encoding: identity\r\n")); - assert!(request.contains("\r\nrange: bytes=2-\r\n")); -} - -#[tokio::test] -async fn forced_reopen_stays_on_pinned_provider_account() { - const ACCOUNT_A_BODY: &[u8] = b"account-a-marker"; - const ACCOUNT_B_BODY: &[u8] = b"account-b-marker"; - - let head_a = format!( - "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - ACCOUNT_A_BODY.len() - ); - let head_b = format!( - "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - ACCOUNT_B_BODY.len() - ); - let (origin_a, task_a) = spawn_legacy_hls_test_origin(head_a, ACCOUNT_A_BODY.to_vec()).await; - let (origin_b, task_b) = spawn_legacy_hls_test_origin(head_b, ACCOUNT_B_BODY.to_vec()).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_a}"), - username: Some("user-a".to_string()), - password: Some("pass-a".to_string()), - enabled: true, - priority: 0, - max_connections: 1, - method: InputFetchMethod::default(), - aliases: Some(vec![ConfigInputAlias { - id: 2, - name: "provider_2".intern(), - url: format!("http://{origin_b}"), - username: Some("user-b".to_string()), - password: Some("pass-b".to_string()), - priority: 1, - max_connections: 1, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }); - let app_config = Arc::new(create_test_provider_app_config()); - app_config.sources.store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(app_config); - - let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_400)); - let fingerprint = create_test_fingerprint(client_addr); - let mut user = ProxyUserCredentials::default(); - user.username = "viewer".to_string(); - let session = UserSession { - token: "stickiness-token".to_string(), - transition_version: 1, - virtual_id: 41, - provider: Arc::clone(&input.name), - stream_url: format!("http://{origin_a}/live/1.ts").intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), - user_agent_stream_index: None, - addr: client_addr, - socket_bound: false, - active_addrs: vec![client_addr], - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - }; - let mut stream_channel = create_test_live_channel(&format!("http://{origin_a}/live/1.ts")); - stream_channel.provider_id = 1; - stream_channel.input_name = Arc::clone(&input.name); - stream_channel.item_type = PlaylistItemType::Catchup; - stream_channel.cluster = XtreamCluster::Live; - stream_channel.url = session.stream_url.clone(); - - let response = force_provider_stream_response( - &fingerprint, - &app_state, - &session, - stream_channel, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input, - user: &user, - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::OK); - let body = response.into_body().collect().await.expect("stream body").to_bytes(); - assert_eq!(body.as_ref(), ACCOUNT_A_BODY, "forced reopen must stay on the pinned provider account A"); - - let request_a = task_a.await.expect("origin A task completes").to_ascii_lowercase(); - assert!(!request_a.is_empty(), "origin A must have received the seek request"); - - let request_b = tokio::time::timeout(std::time::Duration::from_millis(250), task_b).await; - assert!(request_b.is_err(), "origin B must not receive a seek from an A-affine session"); -} - -#[tokio::test] -async fn forced_series_reopen_uses_free_pool_account_and_updates_session_pin() { - const ACCOUNT_B_BODY: &[u8] = b"account-b-marker"; - - let head_a = "HTTP/1.1 200 OK\r\nContent-Length: 1\r\nConnection: close\r\n\r\na".to_string(); - let head_b = format!( - "HTTP/1.1 200 OK\r\nContent-Type: video/x-matroska\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - ACCOUNT_B_BODY.len() - ); - let (origin_a, mut task_a) = spawn_legacy_hls_test_origin(head_a, vec![b'a']).await; - let (origin_b, task_b) = spawn_legacy_hls_test_origin(head_b, ACCOUNT_B_BODY.to_vec()).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_a}"), - username: Some("user-a".to_string()), - password: Some("pass-a".to_string()), - enabled: true, - priority: 0, - max_connections: 1, - method: InputFetchMethod::default(), - aliases: Some(vec![ConfigInputAlias { - id: 2, - name: "provider_2".intern(), - url: format!("http://{origin_b}"), - username: Some("user-b".to_string()), - password: Some("pass-b".to_string()), - priority: 1, - max_connections: 1, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }); - let app_config = Arc::new(create_test_provider_app_config()); - app_config.sources.store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(app_config); - let busy_addr: SocketAddr = "127.0.0.1:55410".parse().unwrap_or_else(|_| unreachable!()); - let busy = app_state.active_provider.acquire_exact_connection_with_grace( - &input.name, - &busy_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ); - assert!(busy.is_some(), "setup must occupy the pinned provider account"); - - let client_addr: SocketAddr = "127.0.0.1:55411".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(client_addr); - let mut user = ProxyUserCredentials::default(); - user.username = "viewer-pool-failover".to_string(); - let stream_url = format!("http://{origin_a}/series/user-a/pass-a/1.mkv"); - let session = UserSession { - token: "series-pool-token".to_string(), - media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), - virtual_id: 42, - provider: Arc::clone(&input.name), - stream_url: stream_url.clone().intern(), - addr: client_addr, - active_addrs: vec![client_addr], - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - }; - let mut stream_channel = create_test_live_channel(&stream_url); - stream_channel.virtual_id = session.virtual_id; - stream_channel.provider_id = 1; - stream_channel.input_name = Arc::clone(&input.name); - stream_channel.item_type = PlaylistItemType::Series; - stream_channel.cluster = XtreamCluster::Series; - stream_channel.url = session.stream_url.clone(); - - let response = force_provider_stream_response( - &fingerprint, - &app_state, - &session, - stream_channel, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input, - user: &user, - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::OK); - let body = response.into_body().collect().await.expect("stream body").to_bytes(); - assert_eq!(body.as_ref(), ACCOUNT_B_BODY, "reopen must use the free provider account B"); - task_b.await.expect("origin B task completes"); - - let updated = app_state - .active_users - .get_and_update_user_session(&user.username, &session.token) - .await - .expect("reopened session is persisted"); - assert_eq!(updated.provider.as_ref(), "provider_2"); - assert!(updated.stream_url.contains("/series/user-b/pass-b/1.mkv")); - - assert!(tokio::time::timeout(std::time::Duration::from_millis(100), &mut task_a).await.is_err()); - task_a.abort(); - app_state.active_provider.release_connection(&busy_addr); -} - -#[tokio::test] -async fn forced_reopen_sends_provider_cookies_only_to_the_origin_that_set_them() { - for (cookie_from_stream_origin, expect_cookie) in [(true, true), (false, false)] { - let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: 4\r\nConnection: close\r\n\r\n"; - let (origin, task) = spawn_legacy_hls_test_origin(head.to_string(), b"live".to_vec()).await; - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - url: format!("http://{origin}"), - username: Some("user".to_string()), - password: Some("pass".to_string()), - enabled: true, - max_connections: 1, - ..ConfigInput::default() - }); - let app_config = Arc::new(create_test_provider_app_config()); - app_config - .sources - .store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(app_config); - - let stream_url = format!("http://{origin}/live/1.ts"); - let cookie_source = - if cookie_from_stream_origin { stream_url.clone() } else { "http://cdn.example/live/1.ts".to_string() }; - let mut cookies = tuliprox_session::ProviderSessionCookieStore::default(); - cookies.update( - &Url::parse(&cookie_source).expect("cookie source url"), - &tuliprox_session::ProviderSessionHeaders { - headers: HashMap::new(), - cookies: vec!["sid=scoped; Path=/".to_string()], - }, - ); - let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_410)); - let mut user = ProxyUserCredentials::default(); - user.username = "viewer".to_string(); - let session = UserSession { - token: "cookie-scope-token".to_string(), - virtual_id: 41, - provider: Arc::clone(&input.name), - stream_url: stream_url.as_str().intern(), - provider_session_cookies: Arc::new(cookies), - media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), - addr: client_addr, - active_addrs: vec![client_addr], - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - }; - let mut stream_channel = create_test_live_channel(&stream_url); - stream_channel.provider_id = 1; - stream_channel.input_name = Arc::clone(&input.name); - - let response = force_provider_stream_response( - &create_test_fingerprint(client_addr), - &app_state, - &session, - stream_channel, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input, - user: &user, - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - assert_eq!(response.status(), StatusCode::OK); - drop(response.into_body().collect().await.expect("stream body")); - - let request = task.await.expect("origin task completes").to_ascii_lowercase(); - assert_eq!( - request.contains("cookie: sid=scoped"), - expect_cookie, - "cookie_from_stream_origin={cookie_from_stream_origin}: {request}" - ); - } -} - -#[tokio::test] -async fn overlapping_vod_range_requests_return_correct_account_bytes() { - const VOD_BODY: &[u8] = b"0123456789abcdefghij"; - - let (origin_addr, origin_task) = spawn_range_aware_test_origin(VOD_BODY, 2).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - enabled: true, - priority: 0, - max_connections: 2, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(config)); - - let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_500)); - let fingerprint = create_test_fingerprint(client_addr); - let mut user = ProxyUserCredentials::default(); - user.username = "viewer".to_string(); - - let make_session = |token: &str| UserSession { - token: token.to_string(), - transition_version: 1, - virtual_id: 42, - provider: Arc::clone(&input.name), - stream_url: format!("http://{origin_addr}/movie/1.ts").intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), - user_agent_stream_index: None, - addr: client_addr, - socket_bound: false, - active_addrs: vec![client_addr], - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - }; - - let make_channel = || { - let mut channel = create_test_local_channel(&format!("http://{origin_addr}/movie/1.ts")); - channel.provider_id = 1; - channel.input_name = Arc::clone(&input.name); - channel.item_type = PlaylistItemType::Video; - channel.cluster = XtreamCluster::Video; - channel.url = format!("http://{origin_addr}/movie/1.ts").intern(); - channel - }; - - let build_request = |range: &'static str| { - let mut headers = HeaderMap::new(); - headers.insert(header::RANGE, HeaderValue::from_static(range)); - let session = make_session(match range { - "bytes=0-9" => "vod-range-1", - _ => "vod-range-2", - }); - (headers, session) - }; - - let (first_headers, first_session) = build_request("bytes=0-9"); - let first = force_provider_stream_response( - &fingerprint, - &app_state, - &first_session, - make_channel(), - ForceStreamRequestContext { - req_headers: &first_headers, - input: &input, - user: &user, - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(first.headers().get(header::CONTENT_RANGE).and_then(|value| value.to_str().ok()), Some("bytes 0-9/20")); - let first_body = first.into_body().collect().await.expect("first range body").to_bytes(); - assert_eq!(first_body.as_ref(), &VOD_BODY[0..10]); - - let (second_headers, second_session) = build_request("bytes=5-14"); - let second = force_provider_stream_response( - &fingerprint, - &app_state, - &second_session, - make_channel(), - ForceStreamRequestContext { - req_headers: &second_headers, - input: &input, - user: &user, - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!( - second.headers().get(header::CONTENT_RANGE).and_then(|value| value.to_str().ok()), - Some("bytes 5-14/20") - ); - let second_body = second.into_body().collect().await.expect("second range body").to_bytes(); - assert_eq!(second_body.as_ref(), &VOD_BODY[5..15]); - - let requests = origin_task.await.expect("origin task completes"); - assert_eq!(requests.len(), 2, "both range requests must reach the same provider account"); - assert!(requests[0].contains("range: bytes=0-9")); - assert!(requests[1].contains("range: bytes=5-14")); -} - -#[tokio::test] -async fn parallel_series_range_requests_keep_both_claims_active() { - const SERIES_BODY: &[u8] = b"0123456789abcdefghij"; - - let (origin_addr, origin_task) = spawn_range_aware_test_origin(SERIES_BODY, 2).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - enabled: true, - priority: 0, - max_connections: 2, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(config)); - - let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_501)); - let fingerprint = create_test_fingerprint(client_addr); - let mut user = ProxyUserCredentials::default(); - user.username = "viewer".to_string(); - - let make_session = |token: &str| UserSession { - token: token.to_string(), - transition_version: 1, - virtual_id: 43, - provider: Arc::clone(&input.name), - stream_url: format!("http://{origin_addr}/series/1.ts").intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), - user_agent_stream_index: None, - addr: client_addr, - socket_bound: false, - active_addrs: vec![client_addr], - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - }; - - let make_channel = || { - let mut channel = create_test_local_channel(&format!("http://{origin_addr}/series/1.ts")); - channel.provider_id = 1; - channel.input_name = Arc::clone(&input.name); - channel.item_type = PlaylistItemType::Series; - channel.cluster = XtreamCluster::Series; - channel.url = format!("http://{origin_addr}/series/1.ts").intern(); - channel - }; - - let first_headers = { - let mut headers = HeaderMap::new(); - headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-9")); - headers - }; - let second_headers = { - let mut headers = HeaderMap::new(); - headers.insert(header::RANGE, HeaderValue::from_static("bytes=5-14")); - headers - }; - - // Open both range requests before consuming either body, so both claims are - // active at the same time. This exercises concurrent claims of one playback - // rather than a sequential request/consume cycle. - let first = force_provider_stream_response( - &fingerprint, - &app_state, - &make_session("series-range-1"), - make_channel(), - ForceStreamRequestContext { - req_headers: &first_headers, - input: &input, - user: &user, - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - let second = force_provider_stream_response( - &fingerprint, - &app_state, - &make_session("series-range-2"), - make_channel(), - ForceStreamRequestContext { - req_headers: &second_headers, - input: &input, - user: &user, - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(app_state.active_users.active_streams().await.len(), 2, "both range claims must be active concurrently"); - - let first_body = first.into_body().collect().await.expect("first series range body").to_bytes(); - assert_eq!(first_body.as_ref(), &SERIES_BODY[0..10]); - - let second_body = second.into_body().collect().await.expect("second series range body").to_bytes(); - assert_eq!(second_body.as_ref(), &SERIES_BODY[5..15]); - - let requests = origin_task.await.expect("origin task completes"); - assert_eq!(requests.len(), 2, "both series range requests must reach the same provider account"); - assert!(requests[0].contains("range: bytes=0-9")); - assert!(requests[1].contains("range: bytes=5-14")); -} - -async fn spawn_load_test_origin(body: &'static [u8]) -> SocketAddr { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("load origin binds"); - let addr = listener.local_addr().expect("load origin address"); - let app = axum::Router::new().fallback(move || async move { - ([(axum::http::header::CONTENT_TYPE, "video/mp2t")], Bytes::from_static(body)) - }); - tokio::spawn(async move { - let _ = axum::serve(listener, app).await; - }); - addr -} - -fn load_test_latency_percentile(sorted: &[u64], p: f64) -> u64 { - if sorted.is_empty() { - return 0; - } - let index = ((sorted.len() as f64 - 1.0) * p).round() as usize; - sorted[index.min(sorted.len() - 1)] -} - -fn validate_load_response(status: StatusCode, payload: &[u8], expected: &[u8]) -> Result<(), String> { - if status != StatusCode::OK { - return Err(format!("unexpected HTTP status {status}")); - } - if payload != expected { - return Err(format!("payload mismatch: expected {} bytes, received {}", expected.len(), payload.len())); - } - Ok(()) -} - -#[derive(Debug, Clone)] -struct SoakResourceSample { - elapsed_secs: f64, - rss_kib: Option, - provider_connections: usize, - lease_starting: usize, - lease_active: usize, - lease_idle: usize, - user_requests: usize, - user_sessions: usize, - shared_origins: usize, - shared_subscribers: usize, - shared_meters: usize, - running_tasks: usize, - completed: u64, - planned_aborted: u64, - unexpected_failed: u64, -} - -fn median_u64(values: &mut [u64]) -> Option { - if values.is_empty() { - return None; - } - values.sort_unstable(); - Some(values[values.len() / 2]) -} - -#[cfg(target_os = "linux")] -fn load_test_rss_kib() -> Option { - let Ok(content) = std::fs::read_to_string("/proc/self/status") else { - return None; - }; - for line in content.lines() { - if let Some(rest) = line.strip_prefix("VmRSS:") { - return rest.split_whitespace().next().and_then(|v| v.parse::().ok()); - } - } - None -} - -#[cfg(not(target_os = "linux"))] -fn load_test_rss_kib() -> Option { None } - -#[tokio::test(flavor = "multi_thread", worker_threads = 8)] -#[ignore = "HTTP load test: cargo +stable test -p tuliprox --release --bin tuliprox -- --ignored --nocapture http_provider_lease_load_test"] -async fn http_provider_lease_load_test() { - const TS_PACKET: [u8; 188] = [0x47; 188]; - let origin_addr = spawn_load_test_origin(&TS_PACKET).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - enabled: true, - priority: 0, - max_connections: 0, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app = create_test_app_state_for_config(Arc::new(config)); - - // Warm-up so allocator and connection pools are exercised before measurement. - for index in 0..32u16 { - let addr = SocketAddr::from(([127, 0, 0, 1], 45_000 + index)); - let fingerprint = create_test_fingerprint(addr); - let session = load_test_session(&input, origin_addr, &format!("warmup-{index}"), addr); - let channel = load_test_channel(&input, origin_addr); - let response = force_provider_stream_response( - &fingerprint, - &app, - &session, - channel, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input, - user: &load_test_user(&format!("warmup-user-{index}")), - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - let status = response.status(); - let body = response.into_body().collect().await.expect("warm-up body collects").to_bytes(); - validate_load_response(status, &body, &TS_PACKET).expect("warm-up response must match the fixture"); - } - - let baseline_rss_kib = load_test_rss_kib(); - eprintln!("HTTP provider lease load test (unlimited provider, real HTTP body, 5 repetitions per concurrency)"); - for concurrency in [1usize, 5, 50, 200] { - let rounds_per_task = 20; - let repetitions = 5; - let mut all_samples = Vec::<(u64, u64)>::new(); - let mut total_ops = 0u64; - let mut total_elapsed_secs = 0f64; - for rep in 0..repetitions { - let samples = Arc::new(std::sync::Mutex::new(Vec::<(u64, u64)>::new())); - let started = std::time::Instant::now(); - let mut tasks = tokio::task::JoinSet::new(); - for task_index in 0..concurrency { - let app = Arc::clone(&app); - let input = Arc::clone(&input); - let samples = Arc::clone(&samples); - let username = format!("load-user-{rep}-{task_index}"); - tasks.spawn(async move { - for round in 0..rounds_per_task { - let addr = SocketAddr::from(([127, 0, 0, 1], 46_000 + (task_index as u16))); - let fingerprint = create_test_fingerprint(addr); - let token = format!("load-{rep}-{task_index}-{round}"); - let session = load_test_session(&input, origin_addr, &token, addr); - let channel = load_test_channel(&input, origin_addr); - let user = load_test_user(&username); - - let acquire_at = std::time::Instant::now(); - let response = force_provider_stream_response( - &fingerprint, - &app, - &session, - channel, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input, - user: &user, - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - let acquire_us = acquire_at.elapsed().as_micros() as u64; - - let status = response.status(); - let body = response.into_body(); - let body_at = std::time::Instant::now(); - let collected = body.collect().await.expect("load body collects").to_bytes(); - validate_load_response(status, &collected, &TS_PACKET) - .expect("load response must match the TS packet"); - let body_us = body_at.elapsed().as_micros() as u64; - - samples.lock().unwrap().push((acquire_us, body_us)); - } - }); - } - while let Some(result) = tasks.join_next().await { - result.expect("load task must not fail"); - } - - total_elapsed_secs += started.elapsed().as_secs_f64(); - let rep_samples = Arc::try_unwrap(samples).expect("samples unique").into_inner().unwrap(); - total_ops += rep_samples.len() as u64; - all_samples.extend(rep_samples); - } - - let mut acquire = Vec::with_capacity(all_samples.len()); - let mut body = Vec::with_capacity(all_samples.len()); - for (a, b) in all_samples { - acquire.push(a); - body.push(b); - } - acquire.sort_unstable(); - body.sort_unstable(); - let throughput = total_ops as f64 / total_elapsed_secs.max(f64::EPSILON); - let byte_throughput = total_ops as f64 * TS_PACKET.len() as f64 / total_elapsed_secs.max(f64::EPSILON); - let rss_delta_kib = load_test_rss_kib() - .zip(baseline_rss_kib) - .map(|(rss, baseline)| rss.saturating_sub(baseline).to_string()) - .unwrap_or_else(|| "unsupported".to_string()); - eprintln!( - "concurrency={concurrency:>3} ops={total_ops:>5} throughput={throughput:>9.1} ops/s {byte_throughput:>11.1} bytes/s | acquire p50/p95/p99={}/{}/{}us | body p50/p95/p99={}/{}/{}us | rss_delta_kib={rss_delta_kib}", - load_test_latency_percentile(&acquire, 0.50), - load_test_latency_percentile(&acquire, 0.95), - load_test_latency_percentile(&acquire, 0.99), - load_test_latency_percentile(&body, 0.50), - load_test_latency_percentile(&body, 0.95), - load_test_latency_percentile(&body, 0.99), - ); - let (user_requests, user_sessions) = app.active_users.playback_resource_counts().await; - let (shared_origins, shared_subscribers) = app.shared_stream_manager.resource_counts().await; - let shared_meters = app.shared_stream_manager.meter_count(); - let leases = app.active_provider.provider_lease_usage(&input.name); - eprintln!( - "[load resource] concurrency={concurrency} conns={} starting={} active={} idle={} user_requests={user_requests} user_sessions={user_sessions} shared_origins={shared_origins} shared_subscribers={shared_subscribers} shared_meters={shared_meters}", - app.active_provider.get_provider_connections_count(), - leases.starting, - leases.active, - leases.idle, - ); - } - - tokio::time::timeout(std::time::Duration::from_secs(5), async { - loop { - let leases = app.active_provider.provider_lease_usage(&input.name); - let (user_requests, user_sessions) = app.active_users.playback_resource_counts().await; - let (shared_origins, shared_subscribers) = app.shared_stream_manager.resource_counts().await; - let shared_meters = app.shared_stream_manager.meter_count(); - if app.active_provider.get_provider_connections_count() == 0 - && leases.total() == 0 - && user_requests == 0 - && user_sessions == 0 - && shared_origins == 0 - && shared_subscribers == 0 - && shared_meters == 0 - { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("load resources must drain to baseline"); - assert_eq!(app.active_provider.get_provider_connections_count(), 0, "provider slots must return to baseline"); - let usage = app.active_provider.provider_lease_usage(&input.name); - assert_eq!(usage.total(), 0, "lease table must return to baseline after HTTP churn"); -} - -fn load_test_user(username: &str) -> ProxyUserCredentials { - let mut user = ProxyUserCredentials::default(); - user.username = username.to_string(); - user -} - -fn load_test_session(input: &ConfigInput, origin_addr: SocketAddr, token: &str, addr: SocketAddr) -> UserSession { - UserSession { - token: token.to_string(), - transition_version: 1, - virtual_id: 42, - provider: Arc::clone(&input.name), - stream_url: format!("http://{origin_addr}/live/42.ts").intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), - user_agent_stream_index: None, - addr, - socket_bound: false, - active_addrs: vec![addr], - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - } -} - -fn load_test_channel(input: &ConfigInput, origin_addr: SocketAddr) -> StreamChannel { - let mut channel = create_test_live_channel(&format!("http://{origin_addr}/live/42.ts")); - channel.provider_id = u32::from(input.id); - channel.input_name = Arc::clone(&input.name); - channel.url = format!("http://{origin_addr}/live/42.ts").intern(); - channel -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 8)] -#[ignore = "30-minute soak: TULIPROX_SOAK_SECS=60 cargo +stable test -p tuliprox --release --bin tuliprox -- --ignored --nocapture http_provider_lease_soak_test"] -async fn http_provider_lease_soak_test() { - let duration_secs: u64 = std::env::var("TULIPROX_SOAK_SECS").ok().and_then(|v| v.parse().ok()).unwrap_or(1800); - const SOAK_PACKET: [u8; 188] = [0x47; 188]; - let origin_addr = spawn_load_test_origin(&SOAK_PACKET).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - enabled: true, - priority: 0, - max_connections: 0, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app = create_test_app_state_for_config(Arc::new(config)); - - let baseline_rss_kib = load_test_rss_kib(); - let completed = Arc::new(std::sync::atomic::AtomicU64::new(0)); - let aborted = Arc::new(std::sync::atomic::AtomicU64::new(0)); - let unexpected_failed = Arc::new(std::sync::atomic::AtomicU64::new(0)); - let peak_connections = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let running_tasks = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let resource_samples = Arc::new(std::sync::Mutex::new(Vec::::new())); - - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(duration_secs); - let sampler_stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let sampler_stop_task = Arc::clone(&sampler_stop); - let app_sampler = Arc::clone(&app); - let sampler_input_name = Arc::clone(&input.name); - let sampler_completed = Arc::clone(&completed); - let sampler_aborted = Arc::clone(&aborted); - let sampler_failed = Arc::clone(&unexpected_failed); - let sampler_running = Arc::clone(&running_tasks); - let sampler_samples = Arc::clone(&resource_samples); - let sampler_task = tokio::spawn(async move { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(5)); - let start_time = std::time::Instant::now(); - while !sampler_stop_task.load(std::sync::atomic::Ordering::Relaxed) { - interval.tick().await; - let rss = load_test_rss_kib().map_or_else(|| "unsupported".to_string(), |rss| rss.to_string()); - let conns = app_sampler.active_provider.get_provider_connections_count(); - let leases = app_sampler.active_provider.provider_lease_usage(&sampler_input_name); - let (user_requests, user_sessions) = app_sampler.active_users.playback_resource_counts().await; - let (shared_origins, shared_subscribers) = app_sampler.shared_stream_manager.resource_counts().await; - let shared_meters = app_sampler.shared_stream_manager.meter_count(); - let sample = SoakResourceSample { - elapsed_secs: start_time.elapsed().as_secs_f64(), - rss_kib: load_test_rss_kib(), - provider_connections: conns, - lease_starting: leases.starting, - lease_active: leases.active, - lease_idle: leases.idle, - user_requests, - user_sessions, - shared_origins, - shared_subscribers, - shared_meters, - running_tasks: sampler_running.load(std::sync::atomic::Ordering::Relaxed), - completed: sampler_completed.load(std::sync::atomic::Ordering::Relaxed), - planned_aborted: sampler_aborted.load(std::sync::atomic::Ordering::Relaxed), - unexpected_failed: sampler_failed.load(std::sync::atomic::Ordering::Relaxed), - }; - eprintln!( - "[soak time_series] t={:4.1}s rss_kib={} conns={} starting={} active={} idle={} user_requests={} user_sessions={} shared_origins={} shared_subscribers={} shared_meters={} running_tasks={} completed={} planned_aborted={} unexpected_failed={}", - sample.elapsed_secs, - rss, - sample.provider_connections, - sample.lease_starting, - sample.lease_active, - sample.lease_idle, - sample.user_requests, - sample.user_sessions, - sample.shared_origins, - sample.shared_subscribers, - sample.shared_meters, - sample.running_tasks, - sample.completed, - sample.planned_aborted, - sample.unexpected_failed, - ); - sampler_samples.lock().unwrap().push(sample); - } - }); - - let mut tasks = tokio::task::JoinSet::new(); - for task_index in 0..16u16 { - let app = Arc::clone(&app); - let input = Arc::clone(&input); - let completed = Arc::clone(&completed); - let aborted = Arc::clone(&aborted); - let unexpected_failed = Arc::clone(&unexpected_failed); - let peak_connections = Arc::clone(&peak_connections); - let running_tasks = Arc::clone(&running_tasks); - tasks.spawn(async move { - running_tasks.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let result = async { - let addr = SocketAddr::from(([127, 0, 0, 1], 47_000 + task_index)); - let fingerprint = create_test_fingerprint(addr); - let username = format!("soak-user-{task_index}"); - let user = load_test_user(&username); - let token = format!("soak-reconnect-family-{task_index}"); - let mut round = 0u64; - while std::time::Instant::now() < deadline { - round += 1; - let session = load_test_session(&input, origin_addr, &token, addr); - let channel = load_test_channel(&input, origin_addr); - let response = force_provider_stream_response( - &fingerprint, - &app, - &session, - channel, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input, - user: &user, - session_reservation_ttl_secs: 2, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - let status = response.status(); - if status != StatusCode::OK { - unexpected_failed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - return Err(format!("soak response status was {status}")); - } - // Alternate full playback (collect) and abort (drop without consuming). - if round % 2 == 0 { - let collected = match response.into_body().collect().await { - Ok(body) => body.to_bytes(), - Err(err) => { - unexpected_failed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - return Err(format!("soak body collection failed: {err}")); - } - }; - if let Err(err) = validate_load_response(status, &collected, &SOAK_PACKET) { - unexpected_failed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - return Err(err); - } - completed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - } else { - drop(response); - aborted.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - } - - let current = app.active_provider.get_provider_connections_count(); - peak_connections.fetch_max(current, std::sync::atomic::Ordering::Relaxed); - } - Ok::<(), String>(()) - } - .await; - running_tasks.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); - result - }); - } - while let Some(result) = tasks.join_next().await { - result.expect("soak task must join").unwrap_or_else(|err| panic!("soak task failed: {err}")); - } - - sampler_stop.store(true, std::sync::atomic::Ordering::Relaxed); - let _ = sampler_task.await; - - let completed = completed.load(std::sync::atomic::Ordering::Relaxed); - let aborted = aborted.load(std::sync::atomic::Ordering::Relaxed); - let peak = peak_connections.load(std::sync::atomic::Ordering::Relaxed); - let unexpected_failed = unexpected_failed.load(std::sync::atomic::Ordering::Relaxed); - let rss_delta_kib = load_test_rss_kib() - .zip(baseline_rss_kib) - .map(|(rss, baseline)| rss.saturating_sub(baseline).to_string()) - .unwrap_or_else(|| "unsupported".to_string()); - eprintln!( - "soak duration={duration_secs}s completed={completed} planned_aborted={aborted} unexpected_failed={unexpected_failed} peak_connections={peak} rss_delta_kib={rss_delta_kib}" - ); - - let samples = Arc::try_unwrap(resource_samples).expect("resource samples unique").into_inner().unwrap(); - assert!(!samples.is_empty(), "soak must record at least one resource sample"); - let third = (samples.len() / 3).max(1); - let middle_start = third.min(samples.len()); - let middle_end = (third.saturating_mul(2)).min(samples.len()); - let last_start = middle_end.min(samples.len()); - let mut middle_rss = - samples[middle_start..middle_end].iter().filter_map(|sample| sample.rss_kib).collect::>(); - let mut last_rss = samples[last_start..].iter().filter_map(|sample| sample.rss_kib).collect::>(); - match (median_u64(&mut middle_rss), median_u64(&mut last_rss)) { - (Some(middle), Some(last)) => { - let delta = i128::from(last) - i128::from(middle); - eprintln!("soak rss rolling-median middle_third={middle}KiB last_third={last}KiB delta={delta}KiB"); - } - _ => eprintln!("soak rss rolling-median unsupported on this platform"), - } - - tokio::time::timeout(std::time::Duration::from_secs(5), async { - loop { - let leases = app.active_provider.provider_lease_usage(&input.name); - let (user_requests, user_sessions) = app.active_users.playback_resource_counts().await; - let (shared_origins, shared_subscribers) = app.shared_stream_manager.resource_counts().await; - let shared_meters = app.shared_stream_manager.meter_count(); - if app.active_provider.get_provider_connections_count() == 0 - && leases.total() == 0 - && user_requests == 0 - && user_sessions == 0 - && shared_origins == 0 - && shared_subscribers == 0 - && shared_meters == 0 - { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("soak resources must drain to baseline"); - - assert_eq!( - app.active_provider.get_provider_connections_count(), - 0, - "provider slots must return to baseline after soak" - ); - let usage = app.active_provider.provider_lease_usage(&input.name); - assert_eq!(usage.total(), 0, "lease table must return to baseline after soak"); - assert_eq!(unexpected_failed, 0, "soak must not hide failed operations"); - assert!(peak <= 16, "active connections must stay bounded by the churn workers, observed {peak}"); -} - -#[tokio::test] -async fn direct_ts_eof_before_first_byte_releases_slot_without_idle_lease() { - let (origin_addr, origin_task) = spawn_controlled_fake_origin(FakeOriginMode::EmptyBody, 1).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - enabled: true, - priority: 0, - max_connections: 1, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(config)); - - let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_601)); - let fingerprint = create_test_fingerprint(client_addr); - let mut user = ProxyUserCredentials::default(); - user.username = "viewer_ts_eof".to_string(); - - let session = UserSession { - token: "session-eof".to_string(), - transition_version: 1, - virtual_id: 42, - provider: Arc::clone(&input.name), - stream_url: format!("http://{origin_addr}/live/42.ts").intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), - user_agent_stream_index: None, - addr: client_addr, - socket_bound: false, - active_addrs: vec![client_addr], - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - }; - - let channel = load_test_channel(&input, origin_addr); - - let response = force_provider_stream_response( - &fingerprint, - &app_state, - &session, - channel, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input, - user: &user, - session_reservation_ttl_secs: 15, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::OK); - let body_bytes = response.into_body().collect().await.expect("empty body collects").to_bytes(); - assert!(body_bytes.is_empty(), "expected 0 payload bytes on immediate EOF"); - - let requests = origin_task.await.expect("origin finishes"); - assert_eq!(requests.len(), 1); - - for _ in 0..50 { - if app_state.active_provider.get_provider_connections_count() == 0 - && app_state.active_provider.provider_lease_usage(&input.name).total() == 0 - { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - - assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "physical slot must be released"); - let lease_usage = app_state.active_provider.provider_lease_usage(&input.name); - assert_eq!(lease_usage.starting, 0, "no starting leases remaining"); - assert_eq!(lease_usage.active, 0, "no active leases remaining"); - assert_eq!(lease_usage.idle, 0, "unconfirmed stream must NOT transition to an idle lease"); - assert_eq!(lease_usage.total(), 0, "total leases must be 0"); - assert_eq!(app_state.active_users.active_streams().await.len(), 0, "active user stream must be cleaned up"); -} - -#[tokio::test] -async fn direct_ts_abort_while_waiting_for_first_byte_releases_exact_request() { - let notify = Arc::new(tokio::sync::Notify::new()); - let (origin_addr, origin_task) = - spawn_controlled_fake_origin(FakeOriginMode::BlockFirstChunk(Arc::clone(¬ify)), 1).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - enabled: true, - priority: 0, - max_connections: 1, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(config)); - - let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_602)); - let fingerprint = create_test_fingerprint(client_addr); - let mut user = ProxyUserCredentials::default(); - user.username = "viewer_ts_abort".to_string(); - - let session = UserSession { - token: "session-abort-before-first-byte".to_string(), - transition_version: 1, - virtual_id: 42, - provider: Arc::clone(&input.name), - stream_url: format!("http://{origin_addr}/live/42.ts").intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), - user_agent_stream_index: None, - addr: client_addr, - socket_bound: false, - active_addrs: vec![client_addr], - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - }; - - let channel = load_test_channel(&input, origin_addr); - - let response = force_provider_stream_response( - &fingerprint, - &app_state, - &session, - channel, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input, - user: &user, - session_reservation_ttl_secs: 15, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(app_state.active_provider.get_provider_connections_count(), 1); - - drop(response); - notify.notify_waiters(); - - let _ = origin_task.await; - - for _ in 0..50 { - if app_state.active_provider.get_provider_connections_count() == 0 - && app_state.active_provider.provider_lease_usage(&input.name).total() == 0 - { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - - assert_eq!( - app_state.active_provider.get_provider_connections_count(), - 0, - "physical slot must be released on abort" - ); - let lease_usage = app_state.active_provider.provider_lease_usage(&input.name); - assert_eq!(lease_usage.total(), 0, "no residual lease claim after unconfirmed abort"); - assert_eq!(app_state.active_users.active_streams().await.len(), 0, "active user stream must be cleaned up"); -} - -#[tokio::test] -async fn provider_error_after_first_byte_finishes_only_matching_request() { - const TS_CHUNK_188: &[u8] = &[0x47; 188]; - const SIBLING_CHUNK: &[u8] = &[0x47; 376]; - - let (origin_fail_addr, origin_fail_task) = - spawn_controlled_fake_origin(FakeOriginMode::ErrorAfterFirstChunk(TS_CHUNK_188.to_vec()), 1).await; - let (origin_ok_addr, origin_ok_task) = - spawn_controlled_fake_origin(FakeOriginMode::FixedBytes(SIBLING_CHUNK.to_vec()), 1).await; - - let input_fail = Arc::new(ConfigInput { - id: 1, - name: "provider_fail".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_fail_addr}"), - enabled: true, - priority: 0, - max_connections: 2, - ..ConfigInput::default() - }); - let input_ok = Arc::new(ConfigInput { - id: 2, - name: "provider_ok".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_ok_addr}"), - enabled: true, - priority: 1, - max_connections: 2, - ..ConfigInput::default() - }); - - let mut config = create_test_provider_app_config(); - config.sources = Arc::new(ArcSwap::from_pointee(SourcesConfig { - inputs: vec![Arc::clone(&input_fail), Arc::clone(&input_ok)], - ..SourcesConfig::default() - })); - let app_state = create_test_app_state_for_config(Arc::new(config)); - - let addr_fail = SocketAddr::from(([127, 0, 0, 1], 55_603)); - let addr_ok = SocketAddr::from(([127, 0, 0, 1], 55_604)); - - let user = load_test_user("viewer_multistream"); - let session_fail = load_test_session(&input_fail, origin_fail_addr, "session-fail", addr_fail); - let session_ok = load_test_session(&input_ok, origin_ok_addr, "session-ok", addr_ok); - - let channel_fail = load_test_channel(&input_fail, origin_fail_addr); - let channel_ok = load_test_channel(&input_ok, origin_ok_addr); - - let resp_fail = force_provider_stream_response( - &create_test_fingerprint(addr_fail), - &app_state, - &session_fail, - channel_fail, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input_fail, - user: &user, - session_reservation_ttl_secs: 15, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - let resp_ok = force_provider_stream_response( - &create_test_fingerprint(addr_ok), - &app_state, - &session_ok, - channel_ok, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input_ok, - user: &user, - session_reservation_ttl_secs: 15, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - assert_eq!(resp_fail.status(), StatusCode::OK); - assert_eq!(resp_ok.status(), StatusCode::OK); - assert_eq!( - app_state.active_provider.get_provider_connections_count(), - 2, - "both providers have an active connection" - ); - - let mut fail_body = resp_fail.into_body(); - let first_frame = fail_body.frame().await.expect("first frame exists").expect("first frame is valid"); - let first_chunk = first_frame.into_data().expect("data chunk exists"); - assert_eq!(first_chunk.as_ref(), TS_CHUNK_188, "first byte was confirmed with exact chunk bytes"); - - let second_frame_res = fail_body.frame().await; - assert!( - second_frame_res.is_none() || second_frame_res.as_ref().is_some_and(Result::is_err), - "stream must terminate or error after first chunk" - ); - drop(fail_body); - - for _ in 0..50 { - if app_state.active_provider.provider_lease_usage(&input_fail.name).total() == 0 { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - - assert_eq!( - app_state.active_provider.provider_lease_usage(&input_fail.name).total(), - 0, - "failed provider leases cleaned up" - ); - assert_eq!( - app_state.active_provider.provider_lease_usage(&input_ok.name).total(), - 1, - "sibling provider lease remains intact" - ); - assert_eq!(app_state.active_provider.get_provider_connections_count(), 1, "sibling connection still active"); - - let ok_bytes = resp_ok.into_body().collect().await.expect("sibling body collects").to_bytes(); - assert_eq!(ok_bytes.as_ref(), SIBLING_CHUNK); - - for _ in 0..50 { - if app_state.active_provider.get_provider_connections_count() == 0 { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - - assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "all slots returned to baseline"); - let _ = origin_fail_task.await; - let _ = origin_ok_task.await; -} - -#[tokio::test] -async fn no_first_byte_failure_allows_next_request_to_prefer_provider_a() { - const TS_CHUNK: &[u8] = &[0x47; 188]; - let listener_a = TcpListener::bind("127.0.0.1:0").await.expect("origin A binds"); - let origin_a_addr = listener_a.local_addr().expect("origin A address"); - let task_a = tokio::spawn(async move { - let mut requests = Vec::new(); - for round in 0..2 { - let Ok((mut socket, _)) = listener_a.accept().await else { break }; - let mut req = Vec::new(); - while !req.windows(4).any(|w| w == b"\r\n\r\n") { - let mut chunk = [0u8; 512]; - let Ok(read) = socket.read(&mut chunk).await else { break }; - if read == 0 { - break; - } - req.extend_from_slice(&chunk[..read]); - } - requests.push(String::from_utf8_lossy(&req).to_ascii_lowercase()); - if round == 0 { - let head = - "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; - let _ = socket.write_all(head.as_bytes()).await; - let _ = socket.shutdown().await; - } else { - let head = format!( - "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - TS_CHUNK.len() - ); - let _ = socket.write_all(head.as_bytes()).await; - let _ = socket.write_all(TS_CHUNK).await; - let _ = socket.shutdown().await; - } - } - requests - }); - - let (origin_b_addr, _task_b) = spawn_controlled_fake_origin(FakeOriginMode::FixedBytes(TS_CHUNK.to_vec()), 1).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_A".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_a_addr}"), - enabled: true, - priority: 0, - max_connections: 1, - aliases: Some(vec![ConfigInputAlias { - id: 2, - name: "provider_B".intern(), - url: format!("http://{origin_b_addr}"), - username: None, - password: None, - priority: 1, - max_connections: 1, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }); - - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(config)); - - let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_605)); - let fingerprint = create_test_fingerprint(client_addr); - let user = load_test_user("viewer_prefer_a"); - - let session_1 = load_test_session(&input, origin_a_addr, "session-1", client_addr); - let channel_1 = load_test_channel(&input, origin_a_addr); - - let resp_1 = force_provider_stream_response( - &fingerprint, - &app_state, - &session_1, - channel_1, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input, - user: &user, - session_reservation_ttl_secs: 15, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - assert_eq!(resp_1.status(), StatusCode::OK); - let b1 = resp_1.into_body().collect().await.expect("body 1 collects").to_bytes(); - assert!(b1.is_empty(), "request 1 had early EOF"); - - for _ in 0..50 { - if app_state.active_provider.get_provider_connections_count() == 0 - && app_state.active_provider.provider_lease_usage(&input.name).total() == 0 - { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - - assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "slot on A must be free"); - assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0, "no phantom lease on A"); - - let session_2 = load_test_session(&input, origin_a_addr, "session-2", client_addr); - let channel_2 = load_test_channel(&input, origin_a_addr); - - let resp_2 = force_provider_stream_response( - &fingerprint, - &app_state, - &session_2, - channel_2, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input, - user: &user, - session_reservation_ttl_secs: 15, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - assert_eq!(resp_2.status(), StatusCode::OK); - let b2 = resp_2.into_body().collect().await.expect("body 2 collects").to_bytes(); - assert_eq!(b2.as_ref(), TS_CHUNK, "request 2 successfully used Provider A and read payload"); - - let reqs_a = task_a.await.expect("task A completed"); - assert_eq!(reqs_a.len(), 2, "both requests 1 and 2 were routed to Provider A!"); -} - -#[tokio::test] -async fn parallel_vod_abort_preserves_sibling_claim_and_provider_stickiness() { - const VOD_BODY: &[u8] = b"0123456789abcdefghij"; - - let (origin_addr, origin_task) = spawn_range_aware_test_origin(VOD_BODY, 3).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - enabled: true, - priority: 0, - max_connections: 2, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(config)); - - let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_701)); - let fingerprint = create_test_fingerprint(client_addr); - let mut user = ProxyUserCredentials::default(); - user.username = "viewer_vod_parallel".to_string(); - - let make_session = |token: &str| UserSession { - token: token.to_string(), - transition_version: 1, - virtual_id: 42, - provider: Arc::clone(&input.name), - stream_url: format!("http://{origin_addr}/movie/1.ts").intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), - user_agent_stream_index: None, - addr: client_addr, - socket_bound: false, - active_addrs: vec![client_addr], - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - }; - - let make_channel = || { - let mut channel = create_test_local_channel(&format!("http://{origin_addr}/movie/1.ts")); - channel.provider_id = 1; - channel.input_name = Arc::clone(&input.name); - channel.item_type = PlaylistItemType::Video; - channel.cluster = XtreamCluster::Video; - channel.url = format!("http://{origin_addr}/movie/1.ts").intern(); - channel - }; - - let build_request = |range: &'static str, token: &'static str| { - let mut headers = HeaderMap::new(); - headers.insert(header::RANGE, HeaderValue::from_static(range)); - let session = make_session(token); - (headers, session) - }; - - let (first_headers, first_session) = build_request("bytes=0-9", "vod-playback"); - let first = force_provider_stream_response( - &fingerprint, - &app_state, - &first_session, - make_channel(), - ForceStreamRequestContext { - req_headers: &first_headers, - input: &input, - user: &user, - session_reservation_ttl_secs: 15, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); - - let (second_headers, second_session) = build_request("bytes=5-14", "vod-playback"); - let second = force_provider_stream_response( - &fingerprint, - &app_state, - &second_session, - make_channel(), - ForceStreamRequestContext { - req_headers: &second_headers, - input: &input, - user: &user, - session_reservation_ttl_secs: 15, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(app_state.active_users.active_streams().await.len(), 1, "affine claims share one playback row"); - assert_eq!(app_state.active_users.playback_resource_counts().await.0, 2, "both request claims active"); - - drop(first); - - tokio::time::timeout(std::time::Duration::from_secs(2), async { - while app_state.active_users.playback_resource_counts().await.0 != 1 { - tokio::task::yield_now().await; - } - }) - .await - .expect("aborted VOD body must release only its request claim"); - assert_eq!(app_state.active_users.active_streams().await.len(), 1, "sibling playback row remains active"); - - let second_body = second.into_body().collect().await.expect("second body collects").to_bytes(); - assert_eq!(second_body.as_ref(), &VOD_BODY[5..15]); - - let (third_headers, third_session) = build_request("bytes=10-19", "vod-playback"); - let third = force_provider_stream_response( - &fingerprint, - &app_state, - &third_session, - make_channel(), - ForceStreamRequestContext { - req_headers: &third_headers, - input: &input, - user: &user, - session_reservation_ttl_secs: 15, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - assert_eq!(third.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!( - third.headers().get(header::CONTENT_RANGE).and_then(|value| value.to_str().ok()), - Some("bytes 10-19/20") - ); - let third_body = third.into_body().collect().await.expect("third body collects").to_bytes(); - assert_eq!(third_body.as_ref(), &VOD_BODY[10..20]); - - let requests = origin_task.await.expect("origin task completes"); - assert_eq!(requests.len(), 3, "all 3 range requests must reach the same provider account"); - assert!(requests[0].contains("range: bytes=0-9")); - assert!(requests[1].contains("range: bytes=5-14")); - assert!(requests[2].contains("range: bytes=10-19")); - - tokio::time::timeout(std::time::Duration::from_secs(2), async { - while app_state.active_provider.get_provider_connections_count() != 0 { - tokio::task::yield_now().await; - } - }) - .await - .expect("VOD provider connections must drain"); - assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); - app_state.active_provider.clear_provider_reservation("vod-playback"); - assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0); -} - -#[tokio::test] -async fn parallel_series_abort_then_seek_reuses_account_without_stale_claim() { - const SERIES_BODY: &[u8] = b"0123456789abcdefghij"; - - let (origin_addr, origin_task) = spawn_range_aware_test_origin(SERIES_BODY, 3).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - enabled: true, - priority: 0, - max_connections: 2, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(config)); - - let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_702)); - let fingerprint = create_test_fingerprint(client_addr); - let mut user = ProxyUserCredentials::default(); - user.username = "viewer_series_parallel".to_string(); - - let make_session = |token: &str| UserSession { - token: token.to_string(), - transition_version: 1, - virtual_id: 43, - provider: Arc::clone(&input.name), - stream_url: format!("http://{origin_addr}/series/1.ts").intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), - user_agent_stream_index: None, - addr: client_addr, - socket_bound: false, - active_addrs: vec![client_addr], - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - }; - - let make_channel = || { - let mut channel = create_test_local_channel(&format!("http://{origin_addr}/series/1.ts")); - channel.provider_id = 1; - channel.input_name = Arc::clone(&input.name); - channel.item_type = PlaylistItemType::Series; - channel.cluster = XtreamCluster::Series; - channel.url = format!("http://{origin_addr}/series/1.ts").intern(); - channel - }; - - let build_request = |range: &'static str, token: &'static str| { - let mut headers = HeaderMap::new(); - headers.insert(header::RANGE, HeaderValue::from_static(range)); - let session = make_session(token); - (headers, session) - }; - - let (first_headers, first_session) = build_request("bytes=0-9", "series-playback"); - let first = force_provider_stream_response( - &fingerprint, - &app_state, - &first_session, - make_channel(), - ForceStreamRequestContext { - req_headers: &first_headers, - input: &input, - user: &user, - session_reservation_ttl_secs: 15, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); - - let (second_headers, second_session) = build_request("bytes=5-14", "series-playback"); - let second = force_provider_stream_response( - &fingerprint, - &app_state, - &second_session, - make_channel(), - ForceStreamRequestContext { - req_headers: &second_headers, - input: &input, - user: &user, - session_reservation_ttl_secs: 15, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(app_state.active_users.active_streams().await.len(), 1, "affine claims share one playback row"); - assert_eq!(app_state.active_users.playback_resource_counts().await.0, 2, "both request claims active"); - - drop(first); - - tokio::time::timeout(std::time::Duration::from_secs(2), async { - while app_state.active_users.playback_resource_counts().await.0 != 1 { - tokio::task::yield_now().await; - } - }) - .await - .expect("aborted Series body must release only its request claim"); - assert_eq!(app_state.active_users.active_streams().await.len(), 1, "sibling playback row remains active"); - - let second_body = second.into_body().collect().await.expect("second body collects").to_bytes(); - assert_eq!(second_body.as_ref(), &SERIES_BODY[5..15]); - - let (third_headers, third_session) = build_request("bytes=12-19", "series-playback"); - let third = force_provider_stream_response( - &fingerprint, - &app_state, - &third_session, - make_channel(), - ForceStreamRequestContext { - req_headers: &third_headers, - input: &input, - user: &user, - session_reservation_ttl_secs: 15, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - assert_eq!(third.status(), StatusCode::PARTIAL_CONTENT); - let third_body = third.into_body().collect().await.expect("third body collects").to_bytes(); - assert_eq!(third_body.as_ref(), &SERIES_BODY[12..20]); - - let requests = origin_task.await.expect("origin task completes"); - assert_eq!(requests.len(), 3, "all series requests reach same provider"); - - tokio::time::timeout(std::time::Duration::from_secs(2), async { - while app_state.active_provider.get_provider_connections_count() != 0 { - tokio::task::yield_now().await; - } - }) - .await - .expect("series provider connections must drain"); - assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); - app_state.active_provider.clear_provider_reservation("series-playback"); - assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0); -} - -#[tokio::test] -async fn stale_direct_cleanup_after_rebind_preserves_successor_request() { - let app_state = create_test_provider_app_state(); - let provider_name: Arc = Arc::from("provider_1"); - let owner = "test-rebind-owner"; - let addr_x = SocketAddr::from(([127, 0, 0, 1], 55_631)); - let addr_y = SocketAddr::from(([127, 0, 0, 1], 55_632)); - - let req_x = tuliprox_core::model::PlaybackRequestId::from_raw(101); - let lease_ref_x = - tuliprox_session::PlaybackLeaseRef { owner, kind: tuliprox_core::model::PlaybackKind::Vod, request_id: req_x }; - let handle_x = app_state - .active_provider - .acquire_exact_connection_with_lease_for_session( - &provider_name, - &addr_x, - false, - 0, - crate::api::model::ConnectionKind::Normal, - Some(lease_ref_x), - ) - .expect("request X acquires provider"); - let stale_tag = handle_x.binding_tag.expect("request X binding tag"); - app_state.active_provider.refresh_playback_lease(&provider_name, &lease_ref_x, 30); - app_state.active_provider.confirm_identified_playback_activity(owner, req_x); - app_state.active_provider.release_handle(&handle_x); - app_state.active_provider.finish_identified_playback_request( - owner, - req_x, - tuliprox_core::model::PlaybackRequestOutcome::Completed, - ); - assert_eq!(app_state.active_provider.provider_lease_usage(&provider_name).idle, 1); - - let req_y = tuliprox_core::model::PlaybackRequestId::from_raw(102); - let lease_ref_y = - tuliprox_session::PlaybackLeaseRef { owner, kind: tuliprox_core::model::PlaybackKind::Vod, request_id: req_y }; - let handle_y = app_state - .active_provider - .acquire_exact_connection_with_lease_for_session( - &provider_name, - &addr_y, - false, - 0, - crate::api::model::ConnectionKind::Normal, - Some(lease_ref_y), - ) - .expect("request Y reacquires provider"); - app_state.active_provider.refresh_playback_lease(&provider_name, &lease_ref_y, 30); - app_state.active_provider.confirm_identified_playback_activity(owner, req_y); - - app_state.active_provider.finish_identified_playback_request( - owner, - req_x, - tuliprox_core::model::PlaybackRequestOutcome::Completed, - ); - - let usage_after_stale_finish = app_state.active_provider.provider_lease_usage(&provider_name); - assert_eq!(usage_after_stale_finish.active, 1, "successor request Y must keep the active lease alive"); - - app_state.active_provider.clear_identified_provider_reservation(owner, &provider_name, Some(stale_tag)); - - let usage_after_stale_tag = app_state.active_provider.provider_lease_usage(&provider_name); - assert_eq!(usage_after_stale_tag.active, 1, "stale clear tag must not erase active successor lease"); - - app_state.active_provider.release_handle(&handle_y); - app_state.active_provider.finish_identified_playback_request( - owner, - req_y, - tuliprox_core::model::PlaybackRequestOutcome::ServerShutdown, - ); - - let final_usage = app_state.active_provider.provider_lease_usage(&provider_name); - assert_eq!(final_usage.total(), 0, "request Y has finished"); -} - -#[tokio::test] -async fn stale_hls_cleanup_after_rebind_preserves_current_origin_and_segments() { - let app_state = create_test_provider_app_state(); - let provider_name: Arc = Arc::from("provider_1"); - let owner = "test-hls-rebind-owner"; - let addr_x = SocketAddr::from(([127, 0, 0, 1], 55_633)); - let addr_y = SocketAddr::from(([127, 0, 0, 1], 55_634)); - - let req_x = tuliprox_core::model::PlaybackRequestId::from_raw(201); - let lease_ref_x = tuliprox_session::PlaybackLeaseRef { - owner, - kind: tuliprox_core::model::PlaybackKind::LiveHls, - request_id: req_x, - }; - let handle_x = app_state - .active_provider - .acquire_exact_connection_with_lease_for_session( - &provider_name, - &addr_x, - false, - 0, - crate::api::model::ConnectionKind::Normal, - Some(lease_ref_x), - ) - .expect("request X acquires HLS provider"); - let stale_tag = handle_x.binding_tag.expect("request X binding tag"); - app_state.active_provider.refresh_playback_lease(&provider_name, &lease_ref_x, 30); - app_state.active_provider.confirm_identified_playback_activity(owner, req_x); - app_state.active_provider.release_handle(&handle_x); - app_state.active_provider.finish_identified_playback_request( - owner, - req_x, - tuliprox_core::model::PlaybackRequestOutcome::Completed, - ); - - let req_y = tuliprox_core::model::PlaybackRequestId::from_raw(202); - let lease_ref_y = tuliprox_session::PlaybackLeaseRef { - owner, - kind: tuliprox_core::model::PlaybackKind::LiveHls, - request_id: req_y, - }; - let handle_y = app_state - .active_provider - .acquire_exact_connection_with_lease_for_session( - &provider_name, - &addr_y, - false, - 0, - crate::api::model::ConnectionKind::Normal, - Some(lease_ref_y), - ) - .expect("request Y reacquires HLS provider"); - app_state.active_provider.refresh_playback_lease(&provider_name, &lease_ref_y, 30); - app_state.active_provider.confirm_identified_playback_activity(owner, req_y); - - app_state.active_provider.finish_identified_playback_request( - owner, - req_x, - tuliprox_core::model::PlaybackRequestOutcome::Completed, - ); - - let usage = app_state.active_provider.provider_lease_usage(&provider_name); - assert!(usage.active >= 1 || usage.idle >= 1, "active/reconnect lease remains for successor"); - - app_state.active_provider.clear_identified_provider_reservation(owner, &provider_name, Some(stale_tag)); - - let usage2 = app_state.active_provider.provider_lease_usage(&provider_name); - assert_eq!(usage2.total(), usage.total(), "stale clear tag must not drop the HLS lease"); - - app_state.active_provider.release_handle(&handle_y); - app_state.active_provider.finish_identified_playback_request( - owner, - req_y, - tuliprox_core::model::PlaybackRequestOutcome::ServerShutdown, - ); - assert_eq!(app_state.active_provider.provider_lease_usage(&provider_name).total(), 0); -} - -#[tokio::test] -async fn shared_proxy_clients_keep_independent_bodies_and_cleanup() { - let mut target = create_test_shared_target(); - target.options.as_mut().unwrap().share_live_streams.mpeg_ts = true; - let target = Arc::new(target); - - let listener = TcpListener::bind("127.0.0.1:0").await.expect("shared origin binds"); - let origin_addr = listener.local_addr().expect("shared origin addr"); - let app = axum::Router::new().fallback(|| async { - let (tx, rx) = tokio::sync::mpsc::channel::>(16); - tokio::spawn(async move { - for i in 0..10u8 { - let _ = tx.send(Ok(Bytes::from(vec![i; 188]))).await; - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - } - }); - let body_stream = tokio_stream::wrappers::ReceiverStream::new(rx); - axum::response::Response::builder() - .header(axum::http::header::CONTENT_TYPE, "video/mp2t") - .body(axum::body::Body::from_stream(body_stream)) - .unwrap() - }); - let origin_task = tokio::spawn(async move { - let _ = axum::serve(listener, app).await; - }); - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_shared_p".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - username: Some("user".to_string()), - password: Some("pass".to_string()), - enabled: true, - priority: 0, - max_connections: 5, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(config)); - - let proxy_addr = SocketAddr::from(([127, 0, 0, 1], 55_190)); - let fp1 = Fingerprint::new("client1|ua1".to_string(), "127.0.0.1".to_string(), proxy_addr); - let fp2 = Fingerprint::new("client2|ua2".to_string(), "127.0.0.1".to_string(), proxy_addr); - - let user1 = load_test_user("user_shared_1"); - let user2 = load_test_user("user_shared_2"); - let stream_url = format!("http://{origin_addr}/live/user/pass/shared.ts"); - let mut channel1 = load_test_channel(&input, origin_addr); - channel1.url = stream_url.as_str().intern(); - let mut channel2 = load_test_channel(&input, origin_addr); - channel2.url = stream_url.as_str().intern(); - - let resp1 = stream_response( - &fp1, - &app_state, - "shared-session-1", - None, - channel1, - &stream_url, - None, - &HeaderMap::default(), - &input, - &target, - &user1, - UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Normal, - false, - None, - ) - .await - .into_response(); - assert_eq!(resp1.status(), StatusCode::OK); - - let resp2 = stream_response( - &fp2, - &app_state, - "shared-session-2", - None, - channel2, - &stream_url, - None, - &HeaderMap::default(), - &input, - &target, - &user2, - UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Normal, - false, - None, - ) - .await - .into_response(); - assert_eq!(resp2.status(), StatusCode::OK); - - assert_eq!(app_state.active_provider.get_provider_connections_count(), 1, "only 1 shared origin connection"); - - drop(resp1); - - let body2 = resp2.into_body().collect().await.expect("client 2 body collects").to_bytes(); - assert!(!body2.is_empty(), "client 2 receives stream bytes after client 1 dropped"); - drop(body2); - - for _ in 0..50 { - if app_state.active_provider.get_provider_connections_count() == 0 { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); - origin_task.abort(); -} - -#[tokio::test] -async fn shared_admission_rejection_does_not_leak_meter_registration() { - let mut target = create_test_shared_target(); - target.options.as_mut().unwrap().share_live_streams.mpeg_ts = true; - let target = Arc::new(target); - - let origin_addr = spawn_load_test_origin(b"adm-test-ts-chunk").await; - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_shared_adm".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - username: Some("user".to_string()), - password: Some("pass".to_string()), - enabled: true, - priority: 0, - max_connections: 5, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - config.config = Arc::new(ArcSwap::from_pointee(Config { - reverse_proxy: Some(crate::model::ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: crate::model::ResourceRetryConfig::default(), - disabled_header: None, - stream: Some(crate::model::StreamConfig { metrics_enabled: true, ..crate::model::StreamConfig::default() }), - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }), - ..Config::default() - })); - let app_state = create_test_app_state_for_config(Arc::new(config)); - - let cleanup_tx = app_state.connection_manager.cleanup_tx(); - let mut permits = Vec::new(); - while let Ok(permit) = cleanup_tx.clone().try_reserve_owned() { - permits.push(permit); - } - - let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_195)); - let fp = create_test_fingerprint(client_addr); - let user = load_test_user("user_adm_fail"); - let stream_url = format!("http://{origin_addr}/live/user/pass/adm.ts"); - let mut channel = load_test_channel(&input, origin_addr); - channel.url = stream_url.as_str().intern(); - - let resp = stream_response( - &fp, - &app_state, - "shared-session-adm", - None, - channel, - &stream_url, - None, - &HeaderMap::default(), - &input, - &target, - &user, - UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Normal, - false, - None, - ) - .await - .into_response(); - - assert_eq!( - resp.status(), - StatusCode::SERVICE_UNAVAILABLE, - "shared admission failure must yield 503 Service Unavailable" - ); - assert_eq!( - app_state.active_provider.get_provider_connections_count(), - 0, - "admission failure must NOT fall through to opening a provider connection" - ); - assert_eq!(app_state.shared_stream_manager.meter_count(), 0, "rejected admission must not retain a meter ID"); - - drop(permits); -} - -#[tokio::test] -async fn shared_start_abort_does_not_leak_meter_registration() { - let mut target = create_test_shared_target(); - target.options.as_mut().unwrap().share_live_streams.mpeg_ts = true; - let target = Arc::new(target); - - let notify = Arc::new(tokio::sync::Notify::new()); - let (origin_addr, origin_task) = - spawn_controlled_fake_origin(FakeOriginMode::BlockFirstChunk(Arc::clone(¬ify)), 1).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_shared_abort".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - username: Some("user".to_string()), - password: Some("pass".to_string()), - enabled: true, - priority: 0, - max_connections: 2, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - config.config = Arc::new(ArcSwap::from_pointee(Config { - reverse_proxy: Some(crate::model::ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: crate::model::ResourceRetryConfig::default(), - disabled_header: None, - stream: Some(crate::model::StreamConfig { metrics_enabled: true, ..crate::model::StreamConfig::default() }), - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }), - ..Config::default() - })); - let app_state = create_test_app_state_for_config(Arc::new(config)); - - let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_196)); - let fp = create_test_fingerprint(client_addr); - let user = load_test_user("user_shared_abort"); - let stream_url = format!("http://{origin_addr}/live/user/pass/shared_abort.ts"); - let mut channel = load_test_channel(&input, origin_addr); - channel.url = stream_url.as_str().intern(); - - let resp = stream_response( - &fp, - &app_state, - "shared-session-abort", - None, - channel, - &stream_url, - None, - &HeaderMap::default(), - &input, - &target, - &user, - UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Normal, - false, - None, - ) - .await - .into_response(); - - assert_eq!(resp.status(), StatusCode::OK); - drop(resp); - notify.notify_waiters(); - let _ = origin_task.await; - - tokio::time::timeout(std::time::Duration::from_secs(2), async { - while app_state.active_provider.get_provider_connections_count() != 0 - || app_state.active_provider.provider_lease_usage(&input.name).total() != 0 - || !app_state.active_users.active_streams().await.is_empty() - || app_state.shared_stream_manager.meter_count() != 0 - { - tokio::task::yield_now().await; - } - }) - .await - .expect("shared abort resources must return to baseline"); - - assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "provider slot released"); - assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0, "provider request released"); - assert_eq!(app_state.active_users.active_streams().await.len(), 0, "user claim released"); - assert_eq!(app_state.shared_stream_manager.meter_count(), 0, "aborted origin meter registration released"); -} - -#[tokio::test] -async fn catchup_abort_seek_and_window_change_preserve_correct_affinity() { - const CATCHUP_BODY_A: &[u8] = b"catchup-origin-a-archive-payload"; - const CATCHUP_BODY_B: &[u8] = b"catchup-origin-b-archive-payload"; - - let (origin_a, task_a) = spawn_range_aware_test_origin(CATCHUP_BODY_A, 2).await; - let (origin_b, task_b) = spawn_range_aware_test_origin(CATCHUP_BODY_B, 1).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_a}"), - enabled: true, - priority: 0, - max_connections: 2, - aliases: Some(vec![ConfigInputAlias { - id: 2, - name: "provider_2".intern(), - url: format!("http://{origin_b}"), - username: None, - password: None, - priority: 1, - max_connections: 2, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(config)); - - let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_620)); - let fingerprint = create_test_fingerprint(client_addr); - let user = load_test_user("viewer_catchup"); - - let make_catchup_session = |token: &str, window: &str| UserSession { - token: token.to_string(), - transition_version: 1, - virtual_id: 41, - provider: Arc::clone(&input.name), - stream_url: format!("http://{origin_a}/timeshift/1.ts?window={window}").intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), - user_agent_stream_index: None, - addr: client_addr, - socket_bound: false, - active_addrs: vec![client_addr], - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - }; - - let make_catchup_channel = |window: &str| { - let mut channel = create_test_local_channel(&format!("http://{origin_a}/timeshift/1.ts?window={window}")); - channel.provider_id = 1; - channel.input_name = Arc::clone(&input.name); - channel.item_type = PlaylistItemType::Catchup; - channel.cluster = XtreamCluster::Live; - channel.url = format!("http://{origin_a}/timeshift/1.ts?window={window}").intern(); - channel - }; - - let session_w1_1 = make_catchup_session("catchup-w1", "w1"); - let session_w1_2 = make_catchup_session("catchup-w1", "w1"); - - let mut h1 = HeaderMap::new(); - h1.insert(header::RANGE, HeaderValue::from_static("bytes=0-9")); - let mut h2 = HeaderMap::new(); - h2.insert(header::RANGE, HeaderValue::from_static("bytes=5-14")); - - let resp1 = force_provider_stream_response( - &fingerprint, - &app_state, - &session_w1_1, - make_catchup_channel("w1"), - ForceStreamRequestContext { - req_headers: &h1, - input: &input, - user: &user, - session_reservation_ttl_secs: 10, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - let resp2 = force_provider_stream_response( - &fingerprint, - &app_state, - &session_w1_2, - make_catchup_channel("w1"), - ForceStreamRequestContext { - req_headers: &h2, - input: &input, - user: &user, - session_reservation_ttl_secs: 10, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - assert_eq!(resp1.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(resp2.status(), StatusCode::PARTIAL_CONTENT); - - let mut session_w2 = make_catchup_session("catchup-w2", "w2"); - session_w2.provider = "provider_2".intern(); - session_w2.stream_url = format!("http://{origin_b}/timeshift/1.ts?window=w2").intern(); - let mut channel_w2 = make_catchup_channel("w2"); - channel_w2.url = Arc::clone(&session_w2.stream_url); - let mut h_w2 = HeaderMap::new(); - h_w2.insert(header::RANGE, HeaderValue::from_static("bytes=0-9")); - let resp_w2 = force_provider_stream_response( - &fingerprint, - &app_state, - &session_w2, - channel_w2, - ForceStreamRequestContext { - req_headers: &h_w2, - input: &input, - user: &user, - session_reservation_ttl_secs: 10, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - assert_eq!(resp_w2.status(), StatusCode::PARTIAL_CONTENT); - let bytes_w2 = resp_w2.into_body().collect().await.expect("window w2 body").to_bytes(); - assert_eq!(bytes_w2.as_ref(), &CATCHUP_BODY_B[..10], "independent window uses available provider B"); - - drop(resp1); - let bytes2 = resp2.into_body().collect().await.expect("resp2 body").to_bytes(); - assert_eq!(bytes2.as_ref(), &CATCHUP_BODY_A[5..15]); - tokio::time::timeout(std::time::Duration::from_secs(2), async { - while app_state.active_provider.get_provider_connections_count() != 0 { - tokio::task::yield_now().await; - } - }) - .await - .expect("parallel catchup requests must release their physical connections"); - - let reqs_a = task_a.await.expect("task A completes"); - assert_eq!(reqs_a.len(), 2, "both requests in window w1 stayed on provider A"); - let reqs_b = task_b.await.expect("task B completes"); - assert_eq!(reqs_b.len(), 1, "the independent window used provider B while A was full"); - - tokio::time::timeout(std::time::Duration::from_secs(2), async { - while app_state.active_provider.get_provider_connections_count() != 0 { - tokio::task::yield_now().await; - } - }) - .await - .expect("catchup provider connections must drain"); - assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); - app_state.active_provider.clear_provider_reservation("catchup-w1"); - app_state.active_provider.clear_provider_reservation("catchup-w2"); - assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0); - assert_eq!(app_state.active_provider.provider_lease_usage(&"provider_2".intern()).total(), 0); -} - -#[tokio::test] -async fn dash_stream_request_remains_redirect_and_preserves_provider_affinity() { - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_dash".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: "http://provider-dash.example".to_string(), - username: Some("dash-user".to_string()), - password: Some("dash-pass".to_string()), - enabled: true, - priority: 0, - max_connections: 2, - ..ConfigInput::default() - }); - let target = Arc::new(ConfigTarget { - curation: None, - id: 1, - name: "target_dash".to_string(), - enabled: true, - options: None, - sort: None, - filter: Filter::default().into(), - output: Vec::new(), - rename: None, - mapping_ids: None, - mapping: Arc::new(ArcSwapOption::default()), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - }); - - let mut app_cfg = create_test_provider_app_config(); - app_cfg.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - app_cfg.config = Arc::new(ArcSwap::from_pointee(Config { - reverse_proxy: Some(crate::model::ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: crate::model::ResourceRetryConfig::default(), - disabled_header: None, - stream: None, - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }), - ..Config::default() - })); - let app_state = create_test_app_state_for_config(Arc::new(app_cfg)); - - let pli = PlaylistItem { - header: PlaylistItemHeader { - id: "1".intern(), - name: "DASH Live".intern(), - group: "Live".intern(), - title: "DASH Live".intern(), - url: "http://provider-dash.example/live/dash-user/dash-pass/1.mpd".intern(), - item_type: PlaylistItemType::LiveDash, - xtream_cluster: XtreamCluster::Live, - virtual_id: shared::model::VirtualId(1), - ..PlaylistItemHeader::default() - }, - }; - - let mut user = load_test_user("dash_viewer"); - user.proxy = ProxyType::Reverse(None); - for target_type in [TargetType::M3u, TargetType::Xtream] { - let redirect_params = RedirectParams { - item: &pli, - provider_id: pli.get_provider_id(), - cluster: XtreamCluster::Live, - target_type, - target: &target, - input: &input, - user: &user, - stream_ext: Some(shared::defaults::DASH_EXT), - req_context: crate::api::endpoints::xtream_api::ApiStreamContext::Live, - action_path: "", - }; - - let resp = redirect_response(&app_state, &redirect_params) - .await - .expect("DASH must return a redirect response") - .into_response(); - assert_eq!(resp.status(), StatusCode::FOUND, "DASH request must be HTTP 302 Found redirect"); - let location = resp.headers().get(axum::http::header::LOCATION).expect("location header present"); - let location_str = location.to_str().expect("valid location string"); - assert!( - location_str.contains("provider-dash.example"), - "redirect location must point to provider URL, got {location_str}" - ); - } - - let dash_ttl = get_session_reservation_ttl_secs(&app_state, PlaylistItemType::LiveDash); - assert_eq!(dash_ttl, default_hls_session_ttl_secs(), "DASH preserves provider affinity TTL"); -} - -#[tokio::test] -async fn http_provider_lease_load_test_negative_control_detects_payload_corruption() { - const TS_PACKET_VALID: [u8; 188] = [0x47; 188]; - const TS_PACKET_CORRUPT: [u8; 188] = [0; 188]; - - let (origin_addr, origin_task) = - spawn_controlled_fake_origin(FakeOriginMode::FixedBytes(TS_PACKET_CORRUPT.to_vec()), 1).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_neg_ctrl".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - enabled: true, - priority: 0, - max_connections: 1, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app = create_test_app_state_for_config(Arc::new(config)); - - let addr = SocketAddr::from(([127, 0, 0, 1], 55_650)); - let fingerprint = create_test_fingerprint(addr); - let session = load_test_session(&input, origin_addr, "neg-session", addr); - let channel = load_test_channel(&input, origin_addr); - let user = load_test_user("neg_ctrl_user"); - - let response = force_provider_stream_response( - &fingerprint, - &app, - &session, - channel, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input, - user: &user, - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - let status = response.status(); - let body = response.into_body().collect().await.expect("body collects").to_bytes(); - assert!( - validate_load_response(status, &body, &TS_PACKET_VALID).is_err(), - "negative control must be rejected by the same validator as the load harness" - ); - - let _ = origin_task.await; -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn http_provider_lease_smoke_test() { - const TS_PACKET: [u8; 188] = [0x47; 188]; - let origin_addr = spawn_load_test_origin(&TS_PACKET).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_smoke".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - enabled: true, - priority: 0, - max_connections: 0, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app = create_test_app_state_for_config(Arc::new(config)); - - for concurrency in [1usize, 2] { - let rounds = 5; - let mut tasks = tokio::task::JoinSet::new(); - for task_idx in 0..concurrency { - let app = Arc::clone(&app); - let input = Arc::clone(&input); - let user = load_test_user(&format!("smoke-user-{task_idx}")); - tasks.spawn(async move { - for r in 0..rounds { - let addr = SocketAddr::from(([127, 0, 0, 1], 48_000 + (task_idx as u16))); - let fingerprint = create_test_fingerprint(addr); - let token = format!("smoke-{task_idx}-{r}"); - let session = load_test_session(&input, origin_addr, &token, addr); - let channel = load_test_channel(&input, origin_addr); - - let response = force_provider_stream_response( - &fingerprint, - &app, - &session, - channel, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input, - user: &user, - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - let status = response.status(); - let body = response.into_body().collect().await.expect("smoke body collects").to_bytes(); - validate_load_response(status, &body, &TS_PACKET).expect("smoke response must match the TS packet"); - } - }); - } - while let Some(res) = tasks.join_next().await { - res.expect("smoke task must succeed"); - } - } - - for _ in 0..50 { - if app.active_provider.get_provider_connections_count() == 0 - && app.active_provider.provider_lease_usage(&input.name).total() == 0 - { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - - assert_eq!(app.active_provider.get_provider_connections_count(), 0, "smoke slots baseline"); - assert_eq!(app.active_provider.provider_lease_usage(&input.name).total(), 0, "smoke lease baseline"); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn http_provider_lease_downstream_smoke_test() { - const TS_PACKET: [u8; 188] = [0x47; 188]; - let origin_addr = spawn_load_test_origin(&TS_PACKET).await; - - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_downstream".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: format!("http://{origin_addr}"), - enabled: true, - priority: 0, - max_connections: 0, - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app = create_test_app_state_for_config(Arc::new(config)); - - #[derive(Clone)] - struct DownstreamState { - app: Arc, - input: Arc, - origin_addr: SocketAddr, - } - - async fn stream_handler( - State(state): State, - ConnectInfo(addr): ConnectInfo, - ) -> axum::response::Response { - let fingerprint = create_test_fingerprint(addr); - let user = load_test_user("downstream-user"); - let session = load_test_session(&state.input, state.origin_addr, "downstream-token", addr); - let channel = load_test_channel(&state.input, state.origin_addr); - force_provider_stream_response( - &fingerprint, - &state.app, - &session, - channel, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &state.input, - user: &user, - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response() - } - - let router = Router::new().route("/stream", get(stream_handler)).with_state(DownstreamState { - app: Arc::clone(&app), - input: Arc::clone(&input), - origin_addr, - }); - let listener = TcpListener::bind("127.0.0.1:0").await.expect("downstream listener binds"); - let downstream_addr = listener.local_addr().expect("downstream address"); - let server = tokio::spawn(async move { - let _ = axum::serve(listener, router.into_make_service_with_connect_info::()).await; - }); - - let client = reqwest::Client::new(); - for concurrency in [1usize, 2] { - let mut tasks = tokio::task::JoinSet::new(); - for task_idx in 0..concurrency { - let client = client.clone(); - let url = format!("http://{downstream_addr}/stream"); - tasks.spawn(async move { - for round in 0..3 { - let response = client.get(&url).send().await.expect("downstream response"); - let status = response.status(); - let body = response.bytes().await.expect("downstream body"); - validate_load_response(status, &body, &TS_PACKET) - .unwrap_or_else(|err| panic!("downstream task {task_idx} round {round}: {err}")); - } - }); - } - while let Some(res) = tasks.join_next().await { - res.expect("downstream task must succeed"); - } - } - - for _ in 0..50 { - if app.active_provider.get_provider_connections_count() == 0 - && app.active_provider.provider_lease_usage(&input.name).total() == 0 - { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - assert_eq!(app.active_provider.get_provider_connections_count(), 0, "downstream slots baseline"); - assert_eq!(app.active_provider.provider_lease_usage(&input.name).total(), 0, "downstream lease baseline"); - - server.abort(); -} - -#[test] -fn test_regular_response_keeps_compression_enabled() { - let response = Response::new(()); - - assert!(should_compress_response(&response)); -} - -#[test] -fn test_get_stream_config_u64_uses_default_when_stream_config_missing() { - assert_eq!( - resolve_stream_config_u64(None, |stream| stream.hls_session_ttl_secs, default_hls_session_ttl_secs()), - default_hls_session_ttl_secs() - ); - assert_eq!( - resolve_stream_config_u64(None, |stream| stream.catchup_session_ttl_secs, default_catchup_session_ttl_secs()), - default_catchup_session_ttl_secs() - ); -} - -#[tokio::test] -async fn test_get_session_reservation_ttl_secs_uses_hls_ttl_for_live_dash() { - let app_state = create_test_app_state(); - assert_eq!( - get_session_reservation_ttl_secs(&app_state, PlaylistItemType::LiveDash), - default_hls_session_ttl_secs() - ); -} - -#[test] -fn provider_affinity_policy_matches_stream_types() { - assert!(!PlaylistItemType::Live.requires_provider_affinity()); - assert!(!PlaylistItemType::LiveUnknown.requires_provider_affinity()); - assert!(PlaylistItemType::LiveHls.requires_provider_affinity()); - assert!(PlaylistItemType::LiveDash.requires_provider_affinity()); - assert!(PlaylistItemType::Video.requires_provider_affinity()); - assert!(PlaylistItemType::Series.requires_provider_affinity()); - assert!(PlaylistItemType::Catchup.requires_provider_affinity()); - - assert!(!allows_provider_pool_failover(PlaylistItemType::LiveHls)); - assert!(!allows_provider_pool_failover(PlaylistItemType::LiveDash)); - assert!(allows_provider_pool_failover(PlaylistItemType::Video)); - assert!(allows_provider_pool_failover(PlaylistItemType::Series)); - assert!(!allows_provider_pool_failover(PlaylistItemType::Catchup)); -} - -#[tokio::test] -async fn resolve_streaming_strategy_honors_forced_provider_fallback_policy() { - let app_state = create_test_dual_provider_app_state(); - let input_name = "provider_1".intern(); - let input = - app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); - let pinned_provider = "provider_1".intern(); - let busy_addr: SocketAddr = "127.0.0.1:55301".parse().unwrap_or_else(|_| unreachable!()); - let strict_addr: SocketAddr = "127.0.0.1:55302".parse().unwrap_or_else(|_| unreachable!()); - let fallback_addr: SocketAddr = "127.0.0.1:55303".parse().unwrap_or_else(|_| unreachable!()); - let stream_url = "http://provider-1.example/movie/user1/pass1/1.mkv"; - - let busy = app_state.active_provider.acquire_exact_connection_with_grace( - &pinned_provider, - &busy_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ); - assert!(busy.is_some(), "setup should occupy the pinned provider"); - - let strict = resolve_streaming_strategy( - &app_state, - stream_url, - &create_test_fingerprint(strict_addr), - &input, - StreamingAcquireOptions { - force_provider: Some(&pinned_provider), - allow_forced_provider_fallback: false, - allow_provider_grace: false, - user_priority: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - session_owner: Some("vod-session"), - playback_kind: crate::model::PlaybackKind::Vod, - accept_requested_stream_url: true, - capacity_wait_timeout: None, - }, - None, - ) - .await; - assert!(strict.provider_handle.is_none(), "strict provider affinity should not allocate a different provider"); - assert!( - matches!( - strict.provider_stream_state, - ProviderStreamState::Custom { reason: ProviderStreamCustomReason::ProviderExhausted, .. } - ), - "strict provider affinity should fail closed when the pinned provider is unavailable" - ); - - let fallback = resolve_streaming_strategy( - &app_state, - stream_url, - &create_test_fingerprint(fallback_addr), - &input, - StreamingAcquireOptions { - force_provider: Some(&pinned_provider), - allow_forced_provider_fallback: true, - allow_provider_grace: false, - user_priority: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - session_owner: Some("live-session"), - playback_kind: crate::model::PlaybackKind::LiveTs, - accept_requested_stream_url: true, - capacity_wait_timeout: None, - }, - None, - ) - .await; - let (ProviderStreamState::Available(Some(fallback_provider), fallback_url) - | ProviderStreamState::GracePeriod(Some(fallback_provider), fallback_url)) = fallback.provider_stream_state - else { - panic!("fallback-enabled request should allocate a provider") - }; - assert_eq!(fallback_provider.as_ref(), "provider_2"); - assert_eq!(fallback_url.as_ref(), "http://provider-2.example/movie/user2/pass2/1.mkv"); - - app_state.active_provider.release_connection(&busy_addr); - app_state.active_provider.release_connection(&strict_addr); - app_state.active_provider.release_connection(&fallback_addr); -} - -#[tokio::test] -async fn resolve_streaming_strategy_rewrites_url_on_fallback_even_when_accept_requested_stream_url_is_true() { - let app_state = create_test_dual_provider_app_state(); - let input_name = "provider_1".intern(); - let input = - app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); - let pinned_provider = "provider_1".intern(); - let busy_addr: SocketAddr = "127.0.0.1:55304".parse().unwrap_or_else(|_| unreachable!()); - let fallback_addr: SocketAddr = "127.0.0.1:55305".parse().unwrap_or_else(|_| unreachable!()); - let stream_url = "http://provider-1.example/movie/user1/pass1/1.mkv"; - - let busy = app_state.active_provider.acquire_exact_connection_with_grace( - &pinned_provider, - &busy_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ); - assert!(busy.is_some(), "setup should occupy the pinned provider"); - - let fallback = resolve_streaming_strategy( - &app_state, - stream_url, - &create_test_fingerprint(fallback_addr), - &input, - StreamingAcquireOptions { - force_provider: Some(&pinned_provider), - allow_forced_provider_fallback: true, - allow_provider_grace: false, - user_priority: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - session_owner: Some("vod-session"), - playback_kind: crate::model::PlaybackKind::Vod, - accept_requested_stream_url: true, - capacity_wait_timeout: None, - }, - None, - ) - .await; - - let (ProviderStreamState::Available(Some(fallback_provider), url) - | ProviderStreamState::GracePeriod(Some(fallback_provider), url)) = fallback.provider_stream_state - else { - panic!("fallback-enabled request should allocate fallback provider") - }; - assert_eq!(fallback_provider.as_ref(), "provider_2"); - assert_eq!(url.as_ref(), "http://provider-2.example/movie/user2/pass2/1.mkv"); - - app_state.active_provider.release_connection(&busy_addr); - app_state.active_provider.release_connection(&fallback_addr); -} - -#[tokio::test] -async fn create_stream_response_details_preserves_stored_headers_when_fallback_open_fails() { - let app_state = create_test_dual_provider_app_state(); - let input_name = "provider_1".intern(); - let input = - app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); - let pinned_provider = "provider_1".intern(); - let busy_addr: SocketAddr = "127.0.0.1:55306".parse().unwrap_or_else(|_| unreachable!()); - let reacquire_addr: SocketAddr = "127.0.0.1:55307".parse().unwrap_or_else(|_| unreachable!()); - let stream_url = "http://provider-1.example/movie/user1/pass1/1.mkv"; - - let user = load_test_user("test-fallback-headers-user"); - let session_token = "sess-fallback-headers-1"; - let initial_headers = HashMap::from([("cookie".to_string(), "old_prov_sess=1".to_string())]); - let created = app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token, - virtual_id: 1, - provider: &pinned_provider, - stream_url, - addr: &reacquire_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - assert!(!created.is_empty()); - app_state.active_users.update_session_provider_headers(&user.username, session_token, &initial_headers).await; - - let busy = app_state.active_provider.acquire_exact_connection_with_grace( - &pinned_provider, - &busy_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ); - assert!(busy.is_some(), "setup should occupy the pinned provider"); - - let channel = create_test_live_channel(stream_url); - let details = create_stream_response_details( - &app_state, - &get_stream_options(&app_state.app_config, StreamResponseMode::Stream), - stream_url, - &user.username, - &create_test_fingerprint(reacquire_addr), - &HeaderMap::new(), - &input, - &channel, - PlaylistItemType::Video, - crate::api::model::ProviderContentRepresentationMode::Identity, - false, - UserConnectionPermission::Allowed, - Some(&pinned_provider), - true, - false, - VirtualId::new(channel.virtual_id), - 0, - crate::api::model::ConnectionKind::Normal, - true, - Some(session_token), - Some(&initial_headers), - true, - None, - None, - None, - None, - ) - .await - .unwrap_or_else(|err| panic!("create_stream_response_details should succeed: {err}")); - - assert_eq!(details.provider_name.as_deref(), Some("provider_2")); - assert!(details.session_headers.is_none(), "fallback request should not pass pinned provider session headers"); - assert!(details.stream.is_none(), "test setup should fail to open the fallback provider stream"); - - let session = app_state - .active_users - .get_and_update_user_session(&user.username, session_token) - .await - .expect("session should exist"); - assert_eq!( - session.provider_session_headers, initial_headers, - "stored session headers should be retained when the fallback provider cannot be opened" - ); - - app_state.active_provider.release_connection(&busy_addr); - app_state.active_provider.release_connection(&reacquire_addr); -} - -#[tokio::test] -async fn force_provider_stream_response_clears_stored_headers_after_fallback_open_succeeds() { - const FALLBACK_BODY: &[u8] = b"fallback-provider"; - let response_head = format!( - "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - FALLBACK_BODY.len() - ); - let (origin_addr, origin_task) = spawn_legacy_hls_test_origin(response_head, FALLBACK_BODY.to_vec()).await; - let app_config = create_test_dual_provider_app_config(); - let Some(configured_input) = app_config.sources.load().inputs.first().cloned() else { unreachable!() }; - let mut fallback_input = (*configured_input).clone(); - let Some(aliases) = fallback_input.aliases.as_mut() else { unreachable!() }; - let Some(fallback_alias) = aliases.first_mut() else { unreachable!() }; - fallback_alias.url = format!("http://{origin_addr}"); - app_config - .sources - .store(Arc::new(SourcesConfig { inputs: vec![Arc::new(fallback_input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(app_config)); - let input_name = "provider_1".intern(); - let input = - app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); - let pinned_provider = "provider_1".intern(); - let busy_addr: SocketAddr = "127.0.0.1:55308".parse().unwrap_or_else(|_| unreachable!()); - let reacquire_addr: SocketAddr = "127.0.0.1:55309".parse().unwrap_or_else(|_| unreachable!()); - let stream_url = "http://provider-1.example/movie/user1/pass1/1.mkv"; - - let user = load_test_user("test-fallback-force-user"); - let session_token = "sess-fallback-force-1"; - let initial_headers = HashMap::from([("cookie".to_string(), "old_pinned_token=abc".to_string())]); - let created = app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token, - virtual_id: 1, - provider: &pinned_provider, - stream_url, - addr: &reacquire_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - assert!(!created.is_empty()); - app_state.active_users.update_session_provider_headers(&user.username, session_token, &initial_headers).await; - - let busy = app_state.active_provider.acquire_exact_connection_with_grace( - &pinned_provider, - &busy_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ); - assert!(busy.is_some(), "setup should occupy the pinned provider"); - - let session = app_state - .active_users - .get_and_update_user_session(&user.username, session_token) - .await - .expect("session should exist"); - - let channel = create_test_live_channel(stream_url); - let response = force_provider_stream_response( - &create_test_fingerprint(reacquire_addr), - &app_state, - &session, - channel, - ForceStreamRequestContext { - req_headers: &HeaderMap::new(), - input: &input, - user: &user, - session_reservation_ttl_secs: 0, - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - None, - ) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::OK); - let body = response.into_body().collect().await.expect("fallback stream body").to_bytes(); - assert_eq!(body.as_ref(), FALLBACK_BODY); - let request = tokio::time::timeout(std::time::Duration::from_secs(5), origin_task) - .await - .expect("fallback request was not sent within 5 seconds") - .expect("fallback origin task completes"); - assert!(!request.is_empty(), "fallback provider must receive the stream request"); - - let updated_session = app_state - .active_users - .get_and_update_user_session(&user.username, session_token) - .await - .expect("session should exist"); - assert!( - updated_session.provider_session_headers.is_empty(), - "stored session headers must be cleared after the fallback provider stream opens" - ); - - app_state.active_provider.release_connection(&busy_addr); -} - -#[tokio::test] -async fn resolve_streaming_strategy_rewrites_stale_alias_url_to_selected_main_provider() { - let app_state = create_test_dual_provider_app_state(); - let input_name = "provider_1".intern(); - let input = - app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); - let addr: SocketAddr = "127.0.0.1:55304".parse().unwrap_or_else(|_| unreachable!()); - - let strategy = resolve_streaming_strategy( - &app_state, - "http://provider-2.example/live/user2/pass2/100.ts", - &create_test_fingerprint(addr), - &input, - StreamingAcquireOptions { - force_provider: None, - allow_forced_provider_fallback: false, - allow_provider_grace: false, - user_priority: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - session_owner: Some("live-session"), - playback_kind: crate::model::PlaybackKind::LiveTs, - accept_requested_stream_url: false, - capacity_wait_timeout: None, - }, - None, - ) - .await; - - let ProviderStreamState::Available(Some(provider), url) = strategy.provider_stream_state else { - panic!("request should allocate the main provider") - }; - assert_eq!(provider.as_ref(), "provider_1"); - assert_eq!(url.as_ref(), "http://provider-1.example/live/user1/pass1/100.ts"); - - app_state.active_provider.release_connection(&addr); -} - -#[tokio::test] -async fn resolve_streaming_strategy_rewrites_opaque_m3u_token_after_alias_allocation() { - let app_config = create_test_dual_provider_app_config(); - let input = Arc::new(ConfigInput { - id: 1, - name: "provider-a".intern(), - input_type: InputType::M3u, - url: "http://playlist.example/a.m3u?token=provider-a-token".to_string(), - enabled: true, - max_connections: 1, - aliases: Some(vec![ConfigInputAlias { - id: 2, - name: "provider-b".intern(), - url: "http://playlist.example/b.m3u?token=provider-b-token".to_string(), - username: None, - password: None, - max_connections: 0, - priority: 1, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }); - app_config.sources.store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(app_config)); - let provider_a = "provider-a".intern(); - let busy_addr: SocketAddr = "127.0.0.1:55306".parse().unwrap_or_else(|_| unreachable!()); - let alias_addr: SocketAddr = "127.0.0.1:55307".parse().unwrap_or_else(|_| unreachable!()); - - let busy = app_state.active_provider.acquire_exact_connection_with_grace( - &provider_a, - &busy_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ); - assert!(busy.is_some(), "setup should occupy provider A"); - - let strategy = resolve_streaming_strategy( - &app_state, - "http://stream.example/channel/segment.ts?token=provider-a-token", - &create_test_fingerprint(alias_addr), - &input, - StreamingAcquireOptions { - force_provider: None, - allow_forced_provider_fallback: false, - allow_provider_grace: false, - user_priority: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - session_owner: Some("live-session"), - playback_kind: crate::model::PlaybackKind::LiveTs, - accept_requested_stream_url: false, - capacity_wait_timeout: None, - }, - None, - ) - .await; - - let ProviderStreamState::Available(Some(provider), url) = strategy.provider_stream_state else { - panic!("request should allocate provider B") - }; - assert_eq!(provider.as_ref(), "provider-b"); - assert_eq!(url.as_ref(), "http://stream.example/channel/segment.ts?token=provider-b-token"); - - app_state.active_provider.release_connection(&busy_addr); - app_state.active_provider.release_connection(&alias_addr); -} - -#[tokio::test] -async fn resolve_streaming_strategy_rejects_unmapped_provider_url() { - let app_state = create_test_dual_provider_app_state(); - let input_name = "provider_1".intern(); - let input = - app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); - let addr: SocketAddr = "127.0.0.1:55305".parse().unwrap_or_else(|_| unreachable!()); - - let strategy = resolve_streaming_strategy( - &app_state, - "http://unmapped.example/live/user1/pass1/100.ts", - &create_test_fingerprint(addr), - &input, - StreamingAcquireOptions { - force_provider: None, - allow_forced_provider_fallback: false, - allow_provider_grace: false, - user_priority: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - session_owner: Some("live-session"), - playback_kind: crate::model::PlaybackKind::LiveTs, - accept_requested_stream_url: false, - capacity_wait_timeout: None, - }, - None, - ) - .await; - - assert!(strategy.provider_handle.is_none()); - assert!(matches!( - strategy.provider_stream_state, - ProviderStreamState::Custom { reason: ProviderStreamCustomReason::UnmappedProviderUrl, .. } - )); - - app_state.active_provider.release_connection(&addr); -} - -#[tokio::test] -async fn resolve_streaming_strategy_accepts_stalker_portal_url() { - let app_config = create_test_provider_app_config(); - let Some(configured_input) = app_config.sources.load().inputs.first().cloned() else { unreachable!() }; - let mut stalker_input = (*configured_input).clone(); - stalker_input.input_type = InputType::Stalker; - stalker_input.username = None; - stalker_input.password = None; - app_config - .sources - .store(Arc::new(SourcesConfig { inputs: vec![Arc::new(stalker_input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(app_config)); - let input_name = "provider_1".intern(); - let input = - app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); - let addr: SocketAddr = "127.0.0.1:55307".parse().unwrap_or_else(|_| unreachable!()); - let stream_url = "http://line.example/play/live.php?mac=00:11:22:33:44:55&stream=347&extension=ts&play_token=abc"; - - let strategy = resolve_streaming_strategy( - &app_state, - stream_url, - &create_test_fingerprint(addr), - &input, - StreamingAcquireOptions { - force_provider: None, - allow_forced_provider_fallback: false, - allow_provider_grace: false, - user_priority: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - session_owner: Some("live-session"), - playback_kind: crate::model::PlaybackKind::LiveTs, - accept_requested_stream_url: false, - capacity_wait_timeout: None, - }, - None, - ) - .await; - - let ProviderStreamState::Available(Some(provider), url) = strategy.provider_stream_state else { unreachable!() }; - assert_eq!(provider.as_ref(), "provider_1"); - assert_eq!(url.as_ref(), stream_url); - - app_state.active_provider.release_connection(&addr); -} - -#[tokio::test] -async fn resolve_streaming_strategy_rejects_stalker_url_after_forced_provider_fallback() { - let app_config = create_test_dual_provider_app_config(); - let Some(configured_input) = app_config.sources.load().inputs.first().cloned() else { unreachable!() }; - let mut stalker_input = (*configured_input).clone(); - stalker_input.input_type = InputType::Stalker; - app_config - .sources - .store(Arc::new(SourcesConfig { inputs: vec![Arc::new(stalker_input)], ..SourcesConfig::default() })); - let app_state = create_test_app_state_for_config(Arc::new(app_config)); - let input_name = "provider_1".intern(); - let input = - app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); - let pinned_provider = "provider_1".intern(); - let busy_addr: SocketAddr = "127.0.0.1:55308".parse().unwrap_or_else(|_| unreachable!()); - let fallback_addr: SocketAddr = "127.0.0.1:55309".parse().unwrap_or_else(|_| unreachable!()); - let stream_url = "http://line.example/play/live.php?mac=00:11:22:33:44:55&stream=347&extension=ts&play_token=abc"; - - let busy = app_state.active_provider.acquire_exact_connection_with_grace( - &pinned_provider, - &busy_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ); - assert!(busy.is_some(), "setup should occupy the pinned provider"); - - let strategy = resolve_streaming_strategy( - &app_state, - stream_url, - &create_test_fingerprint(fallback_addr), - &input, - StreamingAcquireOptions { - force_provider: Some(&pinned_provider), - allow_forced_provider_fallback: true, - allow_provider_grace: false, - user_priority: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - session_owner: Some("live-session"), - playback_kind: crate::model::PlaybackKind::LiveTs, - accept_requested_stream_url: false, - capacity_wait_timeout: None, - }, - None, - ) - .await; - - assert!(matches!( - strategy.provider_stream_state, - ProviderStreamState::Custom { reason: ProviderStreamCustomReason::UnmappedProviderUrl, .. } - )); - - app_state.active_provider.release_connection(&busy_addr); - app_state.active_provider.release_connection(&fallback_addr); -} - -#[tokio::test] -async fn resolve_streaming_strategy_accepts_session_requested_stream_url() { - let app_state = create_test_dual_provider_app_state(); - let input_name = "provider_1".intern(); - let input = - app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); - let addr: SocketAddr = "127.0.0.1:55306".parse().unwrap_or_else(|_| unreachable!()); - let trusted_url = "http://unmapped.example/live/user1/pass1/100.ts"; - let strategy = resolve_streaming_strategy( - &app_state, - trusted_url, - &create_test_fingerprint(addr), - &input, - StreamingAcquireOptions { - force_provider: None, - allow_forced_provider_fallback: false, - allow_provider_grace: false, - user_priority: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - session_owner: Some("live-session"), - playback_kind: crate::model::PlaybackKind::LiveTs, - accept_requested_stream_url: true, - capacity_wait_timeout: None, - }, - None, - ) - .await; - - let ProviderStreamState::Available(Some(provider), url) = strategy.provider_stream_state else { - panic!("session-requested URL should be accepted for the pinned provider") - }; - assert_eq!(provider.as_ref(), "provider_1"); - assert_eq!(url.as_ref(), trusted_url); - - app_state.active_provider.release_connection(&addr); -} - -#[test] -fn test_should_allow_exhausted_shared_reconnect_only_for_matching_shared_session() { - let session = UserSession { - transition_version: 1, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - token: "tok".to_string(), - virtual_id: 282, - provider: Arc::::from("provider"), - stream_url: Arc::::from("http://provider/live/449924.ts"), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), - user_agent_stream_index: None, - addr: "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()), - socket_bound: false, - active_addrs: vec!["127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!())], - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - }; - - assert!(should_allow_exhausted_shared_reconnect(true, Some(&session), 282, "http://provider/live/449924.ts")); - assert!(!should_allow_exhausted_shared_reconnect(false, Some(&session), 282, "http://provider/live/449924.ts")); - assert!(!should_allow_exhausted_shared_reconnect(true, Some(&session), 999, "http://provider/live/449924.ts")); - assert!(!should_allow_exhausted_shared_reconnect(true, Some(&session), 282, "http://provider/live/other.ts")); -} - -fn create_test_app_config() -> AppConfig { - let input = Arc::new(ConfigInput { - id: 1, - name: "local_media".intern(), - input_type: InputType::Library, - headers: HashMap::default(), - url: "file:///tmp".to_string(), - enabled: true, - priority: 0, - max_connections: 1, - method: InputFetchMethod::default(), - aliases: None, - ..ConfigInput::default() - }); - let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; - - AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config::default())), - sources: Arc::new(ArcSwap::from_pointee(sources)), - hdhomerun: Arc::new(ArcSwapOption::default()), - api_proxy: Arc::new(ArcSwapOption::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - } -} - -fn create_test_provider_app_config() -> AppConfig { - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: "http://provider-1.example".to_string(), - username: Some("user1".to_string()), - password: Some("pass1".to_string()), - enabled: true, - priority: 0, - max_connections: 1, - method: InputFetchMethod::default(), - aliases: None, - ..ConfigInput::default() - }); - let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; - - AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config::default())), - sources: Arc::new(ArcSwap::from_pointee(sources)), - hdhomerun: Arc::new(ArcSwapOption::default()), - api_proxy: Arc::new(ArcSwapOption::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - } -} - -fn create_test_dual_provider_app_config() -> AppConfig { - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: "http://provider-1.example".to_string(), - username: Some("user1".to_string()), - password: Some("pass1".to_string()), - enabled: true, - priority: 0, - max_connections: 1, - method: InputFetchMethod::default(), - aliases: Some(vec![ConfigInputAlias { - id: 2, - name: "provider_2".intern(), - url: "http://provider-2.example".to_string(), - username: Some("user2".to_string()), - password: Some("pass2".to_string()), - priority: 1, - max_connections: 1, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }); - let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; - - AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config::default())), - sources: Arc::new(ArcSwap::from_pointee(sources)), - hdhomerun: Arc::new(ArcSwapOption::default()), - api_proxy: Arc::new(ArcSwapOption::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - } -} - -fn create_test_app_state() -> Arc { create_test_app_state_for_config(Arc::new(create_test_app_config())) } - -#[tokio::test] -async fn create_api_proxy_user_defaults_output_clusters_to_all() { - let app_state = create_test_app_state(); - let user = create_api_proxy_user(&app_state); - assert_eq!(user.output_clusters, ClusterFlags::all()); -} - -fn create_test_provider_app_state() -> Arc { - create_test_app_state_for_config(Arc::new(create_test_provider_app_config())) -} - -fn create_test_dual_provider_app_state() -> Arc { - create_test_app_state_for_config(Arc::new(create_test_dual_provider_app_config())) -} - -fn create_test_app_state_for_config(app_cfg: Arc) -> Arc { - let event_manager = Arc::new(EventManager::new()); - let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); - let history_config = Some(StreamHistoryConfig::default()); - active_provider.set_shared_stream_manager(&shared_stream_manager); - - let geoip = Arc::new(ArcSwapOption::::default()); - let config = app_cfg.config.load(); - let active_users = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); - let connection_manager = Arc::new(ConnectionManager::new( - &active_users, - &active_provider, - &shared_stream_manager, - &event_manager, - history_config.as_ref(), - )); - - let tokens = CancelTokens::default(); - let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); - - Arc::new(AppState { - recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( - Arc::clone(&active_provider), - Arc::clone(&connection_manager), - ), - app_config: app_cfg, - http_clients: Arc::default(), - recordings: Arc::new(crate::api::model::RecordingQueue::new()), - cache: Arc::new(ArcSwapOption::default()), - shared_stream_manager, - hls: crate::api::model::HlsState::new(Arc::new(crate::api::model::HlsProxyManager::new())), - stalker_resolve_coordinator: crate::api::model::StalkerResolveCoordinator::default(), - active_users, - active_provider, - connection_manager, - event_manager, - cancel_tokens: ArcSwap::from_pointee(tokens), - playlists: Arc::new(PlaylistStorageState::new()), - geoip, - metadata_manager, - auth: crate::api::model::AuthState::for_tests(), - playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), - }) -} - -fn create_test_fingerprint(addr: std::net::SocketAddr) -> Fingerprint { - Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr) -} - -#[tokio::test] -async fn resource_cache_is_used_only_by_matching_public_fetch_policy() { - const CACHED_BODY: &[u8] = b"cached image"; - const UPSTREAM_BODY: &[u8] = b"upstream image"; - - let app_state = create_test_app_state(); - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache_dir = temp_dir.path().to_string_lossy(); - let mut cache = LRUResourceCache::new(1024, cache_dir.as_ref()); - let resource_url = "http://1.1.1.1/icon.png"; - let cached_path = cache.store_path(resource_url, Some("image/png")); - tokio::fs::write(&cached_path, CACHED_BODY).await.expect("write cached image"); - cache.add_content(resource_url, Some("image/png".to_string()), CACHED_BODY.len()).expect("register cached image"); - app_state.cache.store(Some(Arc::new(RwLock::new(cache)))); - - let response_head = format!( - "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - UPSTREAM_BODY.len() - ); - let (upstream_addr, upstream_task) = spawn_legacy_hls_test_origin(response_head, UPSTREAM_BODY.to_vec()).await; - let proxy = reqwest::Proxy::http(format!("http://{upstream_addr}")).expect("mock proxy URL"); - let mock_client = Arc::new(reqwest::Client::builder().proxy(proxy).build().expect("mock upstream client")); - // The destination is public, so the hop is fetched through the public resource client; both resource - // clients are mocked so the assertion below is about the cache, not about which client performed the - // request. - app_state.http_clients.resource_public_no_redirect.store(Arc::clone(&mock_client)); - app_state.http_clients.resource_no_redirect.store(mock_client); - - let public_response = - resource_response(&app_state, ResourceFetchPolicy::Public, resource_url, &HeaderMap::new(), None) - .await - .into_response(); - assert_eq!(public_response.status(), StatusCode::OK); - let public_body = public_response.into_body().collect().await.expect("read cached image").to_bytes(); - assert_eq!(public_body, Bytes::from_static(CACHED_BODY)); - - let response = resource_response(&app_state, ResourceFetchPolicy::NonPublic, resource_url, &HeaderMap::new(), None) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::OK); - let response_body = response.into_body().collect().await.expect("read upstream image").to_bytes(); - assert_eq!(response_body, Bytes::from_static(UPSTREAM_BODY)); - assert_ne!(response_body, Bytes::from_static(CACHED_BODY)); - - let upstream_request = upstream_task.await.expect("mock upstream task completes"); - assert!(upstream_request.starts_with("GET http://1.1.1.1/icon.png HTTP/1.1\r\n")); -} - -#[tokio::test] -async fn no_redirect_resource_ignores_configured_proxy() { - let app_state = create_test_app_state(); - let response_head = - "HTTP/1.1 302 Found\r\nLocation: http://10.0.0.2/other.png\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; - let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head.to_string(), Vec::new()).await; - app_state.app_config.config.store(Arc::new(Config { - proxy: Some(crate::model::ProxyConfig { url: format!("http://{proxy_addr}"), username: None, password: None }), - ..Config::default() - })); - let client = crate::api::model::create_resource_http_client_no_redirect(&app_state.app_config) - .expect("resource HTTP client"); - // 192.0.2.1 (TEST-NET-1) is not routable, so a direct attempt fails by timing out or refusing the - // connection. A response would mean the request was answered by the proxy after all. - let result = client.get("http://192.0.2.1/icon.png").timeout(std::time::Duration::from_millis(100)).send().await; - assert!( - result.as_ref().is_err_and(|err| err.is_timeout() || err.is_connect()), - "the direct path must be attempted: {result:?}" - ); - assert!( - tokio::time::timeout(std::time::Duration::from_millis(100), proxy_task).await.is_err(), - "resource client must not send requests through the configured proxy" - ); -} - -#[tokio::test] -async fn public_resource_client_honours_the_configured_proxy() { - // Counterpart of `no_redirect_resource_ignores_configured_proxy`: a resource hop that can leave the - // local network must go through the configured proxy, or the fetch discloses this host's address. - let app_state = create_test_app_state(); - let response_head = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 3\r\nConnection: close\r\n\r\n"; - let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head.to_string(), b"png".to_vec()).await; - app_state.app_config.config.store(Arc::new(Config { - proxy: Some(crate::model::ProxyConfig { url: format!("http://{proxy_addr}"), username: None, password: None }), - ..Config::default() - })); - let client = crate::api::model::create_resource_public_http_client_no_redirect(&app_state.app_config) - .expect("resource HTTP client"); - - let response = client.get("http://8.8.8.8/icon.png").send().await.expect("request reaches the proxy"); - - assert_eq!(response.status(), StatusCode::OK); - let request = tokio::time::timeout(std::time::Duration::from_secs(2), proxy_task) - .await - .expect("request must reach proxy") - .expect("proxy task"); - assert!(request.starts_with("GET http://8.8.8.8/icon.png HTTP/1.1\r\n"), "{request}"); -} - -#[tokio::test] -async fn an_unresolved_resource_name_is_fetched_through_the_configured_proxy() { - // A resource name that does not resolve locally is still resolvable through a configured proxy (for - // example with remote DNS), so the fetch must not be pinned to the direct client. - let app_state = create_test_app_state(); - let response_head = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 3\r\nConnection: close\r\n\r\n"; - let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head.to_string(), b"png".to_vec()).await; - app_state.app_config.config.store(Arc::new(Config { - proxy: Some(crate::model::ProxyConfig { url: format!("http://{proxy_addr}"), username: None, password: None }), - ..Config::default() - })); - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(format!("http://{proxy_addr}")).expect("test proxy")) - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("mock proxy client"); - app_state.http_clients.resource_public_no_redirect.store(Arc::new(client)); - - let response = - resource_proxy_response(&app_state, "http://unresolved.invalid/logo.png", &HeaderMap::new(), None).await; - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response.into_body().collect().await.expect("image body").to_bytes(), Bytes::from_static(b"png")); - let request = tokio::time::timeout(std::time::Duration::from_secs(2), proxy_task) - .await - .expect("request must reach proxy") - .expect("proxy task"); - assert!(request.starts_with("GET http://unresolved.invalid/logo.png HTTP/1.1\r\n"), "{request}"); -} - -#[tokio::test] -async fn public_resource_client_reaches_a_proxy_on_loopback() { - // A proxy is commonly configured on the loopback interface (`http://localhost:8118`), so the - // connect-time guard of the public resource client must not reject the proxy host itself. - let app_state = create_test_app_state(); - let response_head = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 3\r\nConnection: close\r\n\r\n"; - let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head.to_string(), b"png".to_vec()).await; - app_state.app_config.config.store(Arc::new(Config { - proxy: Some(crate::model::ProxyConfig { - url: format!("http://localhost:{}", proxy_addr.port()), - username: None, - password: None, - }), - ..Config::default() - })); - let client = crate::api::model::create_resource_public_http_client_no_redirect(&app_state.app_config) - .expect("resource HTTP client"); - - let response = client.get("http://8.8.8.8/icon.png").send().await.expect("request reaches the proxy"); - - assert_eq!(response.status(), StatusCode::OK); - let request = tokio::time::timeout(std::time::Duration::from_secs(2), proxy_task) - .await - .expect("request must reach proxy") - .expect("proxy task"); - assert!(request.starts_with("GET http://8.8.8.8/icon.png HTTP/1.1\r\n"), "{request}"); -} - -#[tokio::test] -async fn no_redirect_resource_refuses_destinations_local_to_this_host() { - let app_state = create_test_app_state(); - let (proxy_addr, proxy_task) = - spawn_legacy_hls_test_origin("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n".to_string(), Vec::new()).await; - app_state.app_config.config.store(Arc::new(Config { - proxy: Some(crate::model::ProxyConfig { url: format!("http://{proxy_addr}"), username: None, password: None }), - ..Config::default() - })); - let client = crate::api::model::create_resource_http_client_no_redirect(&app_state.app_config) - .expect("resource HTTP client"); - app_state.http_clients.resource_no_redirect.store(Arc::new(client)); - - for local_only in ["http://127.0.0.1/icon.png", "http://169.254.169.254/latest/meta-data/", "http://[::1]/icon.png"] - { - let response = - resource_response(&app_state, ResourceFetchPolicy::NonPublic, local_only, &HeaderMap::new(), None) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::FORBIDDEN, "{local_only}"); - } - - // The guard rejects before a request is built, so nothing may reach the configured proxy. - assert!( - tokio::time::timeout(std::time::Duration::from_millis(100), proxy_task).await.is_err(), - "no request may reach the proxy for local-only destinations" - ); -} - -#[tokio::test] -async fn resource_redirect_hides_private_destination_and_upstream_location() { - let app_state = create_test_app_state(); - let response_head = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nLocation: http://10.0.0.2/private\r\nContent-Length: 3\r\nConnection: close\r\n\r\n"; - let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head.to_string(), b"png".to_vec()).await; - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(format!("http://{proxy_addr}")).expect("test proxy")) - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("mock upstream client"); - app_state.http_clients.resource_no_redirect.store(Arc::new(client)); - - let private = resource_redirect_or_proxy(&app_state, "http://10.0.0.1/icon.png", &HeaderMap::new(), None).await; - assert_eq!(private.status(), StatusCode::OK); - assert!(private.headers().get("location").is_none()); - assert_eq!(private.into_body().collect().await.expect("private icon body").to_bytes(), Bytes::from_static(b"png")); - let request = proxy_task.await.expect("mock request"); - assert!(request.starts_with("GET http://10.0.0.1/icon.png HTTP/1.1\r\n")); - - let public = resource_redirect_or_proxy(&app_state, "http://8.8.8.8/icon.png", &HeaderMap::new(), None).await; - assert_eq!(public.status(), StatusCode::FOUND); - assert_eq!(public.headers().get("location").and_then(|value| value.to_str().ok()), Some("http://8.8.8.8/icon.png")); - - let local_thumbnail = - resource_redirect_or_proxy(&app_state, "/api/v1/library/thumbnail/item", &HeaderMap::new(), None).await; - assert_eq!(local_thumbnail.status(), StatusCode::FOUND); - assert_eq!( - local_thumbnail.headers().get("location").and_then(|value| value.to_str().ok()), - Some("/api/v1/library/thumbnail/item") - ); - - let blocked = resource_redirect_or_proxy(&app_state, "http://127.0.0.1/icon.png", &HeaderMap::new(), None).await; - assert_eq!(blocked.status(), StatusCode::FORBIDDEN); - assert!(blocked.headers().get("location").is_none()); -} - -#[tokio::test] -async fn proxied_m3u_logo_follows_private_redirect_without_forwarding_credentials() { - let app_state = create_test_app_state(); - let listener = TcpListener::bind("127.0.0.1:0").await.expect("mock proxy binds"); - let proxy_addr = listener.local_addr().expect("mock proxy address"); - let proxy_task = tokio::spawn(async move { - let mut requests = Vec::new(); - for response in [ - "HTTP/1.1 302 Found\r\nLocation: http://10.0.0.2/icon.png\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", - "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 3\r\nConnection: close\r\n\r\npng", - ] { - let (mut socket, _) = listener.accept().await.expect("mock proxy accepts request"); - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 1024]; - let read = socket.read(&mut chunk).await.expect("mock proxy reads request"); - if read == 0 { - break; - } - request.extend_from_slice(&chunk[..read]); - } - socket.write_all(response.as_bytes()).await.expect("mock proxy writes response"); - requests.push(String::from_utf8_lossy(&request).into_owned()); - } - requests - }); - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(format!("http://{proxy_addr}")).expect("mock proxy URL")) - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("mock resource client"); - app_state.http_clients.resource_no_redirect.store(Arc::new(client)); - let input = ConfigInput { - url: "http://10.0.0.1/playlist.m3u".to_string(), - headers: HashMap::from([("Authorization".to_string(), "Bearer secret".to_string())]), - ..ConfigInput::default() - }; - let mut headers = HeaderMap::new(); - headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer player-secret")); - - let response = resource_proxy_response(&app_state, "http://10.0.0.1/logo.png", &headers, Some(&input)).await; - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response.into_body().collect().await.expect("image body").to_bytes(), Bytes::from_static(b"png")); - - let requests = proxy_task.await.expect("mock proxy task"); - assert!(requests[0].starts_with("GET http://10.0.0.1/logo.png HTTP/1.1\r\n")); - assert!(requests[0].to_ascii_lowercase().contains("authorization: bearer secret")); - assert!(!requests[0].contains("player-secret")); - assert!(requests[1].starts_with("GET http://10.0.0.2/icon.png HTTP/1.1\r\n")); - assert!(!requests[1].to_ascii_lowercase().contains("authorization:")); -} - -#[test] -fn configured_resource_headers_are_limited_to_the_input_origin() { - let input = ConfigInput { url: "http://10.0.0.1:8080/playlist.m3u".to_string(), ..ConfigInput::default() }; - assert!(resource_input_for_url(Some(&input), "http://10.0.0.1:8080/logo.png").is_some()); - assert!(resource_input_for_url(Some(&input), "http://10.0.0.2:8080/logo.png").is_none()); - assert!(resource_input_for_url(Some(&input), "http://10.0.0.1:8081/logo.png").is_none()); - assert!(resource_input_for_url(Some(&input), "https://10.0.0.1:8080/logo.png").is_none()); -} - -#[tokio::test] -async fn public_resource_is_fetched_through_the_configured_proxy() { - // Regression lock: a resource fetch that leaves the local network must use the configured proxy, - // otherwise it discloses the operator's address to the resource host. The Web UI resource route - // reaches this path, because it wraps every icon of a public destination into a proxy link. - let app_state = create_test_app_state(); - let response_head = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 3\r\nConnection: close\r\n\r\n"; - let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head.to_string(), b"png".to_vec()).await; - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(format!("http://{proxy_addr}")).expect("test proxy")) - .build() - .expect("mock proxy client"); - app_state.http_clients.resource_public_no_redirect.store(Arc::new(client)); - - let response = resource_proxy_response(&app_state, "http://8.8.8.8/logo.png", &HeaderMap::new(), None).await; - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response.into_body().collect().await.expect("image body").to_bytes(), Bytes::from_static(b"png")); - let request = tokio::time::timeout(std::time::Duration::from_secs(2), proxy_task) - .await - .expect("request must reach proxy") - .expect("proxy task"); - assert!(request.starts_with("GET http://8.8.8.8/logo.png HTTP/1.1\r\n"), "{request}"); -} - -#[tokio::test] -async fn proxied_resource_follows_a_redirect_into_the_public_network_through_the_configured_proxy() { - // Regression lock: the hop decides which client is used. A network-internal destination that - // redirects to a public host must not make the follow-up request directly, or the redirect would - // disclose the operator's address to that host despite a configured proxy. - let app_state = create_test_app_state(); - let redirect_head = - "HTTP/1.1 302 Found\r\nLocation: http://8.8.8.8/logo.png\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; - let (private_proxy_addr, private_proxy_task) = - spawn_legacy_hls_test_origin(redirect_head.to_string(), Vec::new()).await; - let private_client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(format!("http://{private_proxy_addr}")).expect("test proxy")) - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("mock private client"); - app_state.http_clients.resource_no_redirect.store(Arc::new(private_client)); - - let response_head = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 3\r\nConnection: close\r\n\r\n"; - let (public_proxy_addr, public_proxy_task) = - spawn_legacy_hls_test_origin(response_head.to_string(), b"png".to_vec()).await; - let public_client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(format!("http://{public_proxy_addr}")).expect("test proxy")) - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("mock public client"); - app_state.http_clients.resource_public_no_redirect.store(Arc::new(public_client)); - - let response = resource_proxy_response(&app_state, "http://10.0.0.1/logo.png", &HeaderMap::new(), None).await; - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response.into_body().collect().await.expect("image body").to_bytes(), Bytes::from_static(b"png")); - let first_hop = private_proxy_task.await.expect("first hop request"); - assert!(first_hop.starts_with("GET http://10.0.0.1/logo.png HTTP/1.1\r\n"), "{first_hop}"); - let second_hop = tokio::time::timeout(std::time::Duration::from_secs(2), public_proxy_task) - .await - .expect("public hop must be fetched") - .expect("public hop request"); - assert!(second_hop.starts_with("GET http://8.8.8.8/logo.png HTTP/1.1\r\n"), "{second_hop}"); -} - -#[tokio::test] -async fn public_resource_redirect_to_a_destination_local_to_this_host_is_refused() { - // Regression lock: a public host that redirects to loopback, link-local, or a cloud metadata - // endpoint must not turn the resource route into a reader for this host. The redirect target is - // never requested, and its location is never relayed to the client. - let app_state = create_test_app_state(); - let listener = TcpListener::bind("127.0.0.1:0").await.expect("mock proxy binds"); - let proxy_addr = listener.local_addr().expect("mock proxy address"); - let proxy_task = tokio::spawn(async move { - let mut requests = Vec::new(); - loop { - let Ok(accepted) = tokio::time::timeout(std::time::Duration::from_millis(500), listener.accept()).await - else { - break; - }; - let Ok((mut socket, _)) = accepted else { break }; - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 1024]; - match socket.read(&mut chunk).await { - Ok(0) | Err(_) => break, - Ok(read) => request.extend_from_slice(&chunk[..read]), - } - } - requests.push(String::from_utf8_lossy(&request).into_owned()); - let head = "HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1/secret\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; - let _ = socket.write_all(head.as_bytes()).await; - } - requests - }); - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(format!("http://{proxy_addr}")).expect("test proxy")) - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("mock client"); - app_state.http_clients.resource_public_no_redirect.store(Arc::new(client)); - - let response = resource_proxy_response(&app_state, "http://8.8.8.8/logo.png", &HeaderMap::new(), None).await; - - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); - assert!(response.headers().get("location").is_none(), "the local destination must not be relayed"); - let requests = proxy_task.await.expect("mock proxy task"); - assert_eq!(requests.len(), 1, "the local destination must not be requested: {requests:?}"); -} - -#[tokio::test] -async fn no_redirect_resource_does_not_relay_upstream_error_details() { - let app_state = create_test_app_state(); - let body = b"internal service error".to_vec(); - let response_head = - format!("HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\nContent-Length: {}\r\n\r\n", body.len()); - let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head, body.clone()).await; - app_state.app_config.config.store(Arc::new(Config { - proxy: Some(crate::model::ProxyConfig { url: format!("http://{proxy_addr}"), username: None, password: None }), - ..Config::default() - })); - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(format!("http://{proxy_addr}")).expect("test proxy")) - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("mock upstream client"); - app_state.http_clients.resource_no_redirect.store(Arc::new(client)); - - let response = resource_response( - &app_state, - ResourceFetchPolicy::NonPublic, - "http://10.0.0.1/icon.png", - &HeaderMap::new(), - None, - ) - .await - .into_response(); - - // The client learns that the fetch failed, not what the destination answered. - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); - let relayed = response.into_body().collect().await.expect("response body").to_bytes(); - assert!(relayed.is_empty(), "upstream error body must not be relayed: {relayed:?}"); - - let request = tokio::time::timeout(std::time::Duration::from_secs(2), proxy_task) - .await - .expect("request must reach proxy") - .expect("proxy task"); - assert!(request.starts_with("GET http://10.0.0.1/icon.png HTTP/1.1\r\n"), "{request}"); -} - -#[tokio::test] -async fn resource_client_refuses_names_that_resolve_to_local_addresses() { - let app_state = create_test_app_state(); - let (origin_addr, _origin_task) = - spawn_legacy_hls_test_origin("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n".to_string(), Vec::new()).await; - let guarded = crate::api::model::create_resource_http_client_no_redirect(&app_state.app_config) - .expect("resource HTTP client"); - - // `localhost` resolves to loopback on the client's DNS layer, so the request must fail while the - // name is resolved and never reach the origin that is listening on that loopback address. - let error = guarded - .get(format!("http://localhost:{}/icon.png", origin_addr.port())) - .send() - .await - .expect_err("a name resolving to a local address must be refused"); - - // reqwest only reports the outer failure, so the cause chain has to be walked to see the refusal. - let mut causes = error.to_string(); - let mut source = std::error::Error::source(&error); - while let Some(cause) = source { - causes.push_str(&format!(" | {cause}")); - source = cause.source(); - } - assert!(causes.contains("local to this host"), "{causes}"); -} - -fn create_test_fingerprint_with_user_agent(addr: std::net::SocketAddr, user_agent: &str) -> Fingerprint { - Fingerprint::new(format!("{}|{user_agent}", addr.ip()), addr.ip().to_string(), addr) -} - -fn create_test_app_state_with_stream_config(stream: crate::model::StreamConfig) -> Arc { - let config = Config { - reverse_proxy: Some(crate::model::ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: crate::model::ResourceRetryConfig::default(), - disabled_header: None, - stream: Some(stream), - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }), - user_access_control: true, - ..Config::default() - }; - - let mut app_cfg = create_test_app_config(); - app_cfg.config = Arc::new(ArcSwap::from_pointee(config)); - create_test_app_state_for_config(Arc::new(app_cfg)) -} - -fn create_test_local_channel(url: &str) -> StreamChannel { - StreamChannel { - target_id: 1, - virtual_id: 41, - provider_id: 0, - input_name: "library".intern(), - item_type: PlaylistItemType::LocalVideo, - cluster: XtreamCluster::Video, - group: "Local Movies".intern(), - title: "Local Test".intern(), - url: url.into(), - shared: false, - shared_joined_existing: None, - shared_stream_id: None, - technical: None, - epg_channel_id: None, - epg_reference_ts: None, - upstream_user_agent: None, - } -} - -fn create_test_live_channel(url: &str) -> StreamChannel { - StreamChannel { - target_id: 1, - virtual_id: 42, - provider_id: 1, - input_name: "provider_1".intern(), - item_type: PlaylistItemType::Live, - cluster: XtreamCluster::Live, - group: "Live".intern(), - title: "Shared Live".intern(), - url: url.into(), - shared: false, - shared_joined_existing: None, - shared_stream_id: None, - technical: None, - epg_channel_id: None, - epg_reference_ts: None, - upstream_user_agent: None, - } -} - -fn create_test_session( - token: &str, - item_type: PlaylistItemType, - lifecycle: crate::api::model::PlaybackLifecycle, -) -> UserSession { - UserSession { - token: token.to_string(), - transition_version: 1, - virtual_id: 42, - provider: Arc::::from("provider-a"), - stream_url: Arc::::from(match item_type { - PlaylistItemType::LiveHls => "http://provider-1.example/live/42.m3u8", - _ => "http://provider-1.example/live/42.ts", - }), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), - user_agent_stream_index: None, - addr: "127.0.0.1:55555".parse().unwrap_or_else(|_| unreachable!()), - socket_bound: item_type.uses_socket_bound_session(), - active_addrs: Vec::new(), - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle, - ..Default::default() - } -} - -#[test] -fn classify_playback_request_marks_adaptive_playlist_request_as_prepare() { - let request_class = classify_playback_request(PlaybackRequestFacts { - existing_session: None, - prepare_only: true, - terminate: false, - }); - - assert_eq!(request_class, PlaybackRequestClass::Prepare); -} - -#[test] -fn classify_playback_request_marks_preserved_session_as_activate() { - let session = create_test_session( - "tok-preserved", - PlaylistItemType::LiveHls, - crate::api::model::PlaybackLifecycle::Preserved, - ); - - let request_class = classify_playback_request(PlaybackRequestFacts { - existing_session: Some(&session), - prepare_only: false, - terminate: false, - }); - - assert_eq!(request_class, PlaybackRequestClass::Activate); -} - -#[test] -fn classify_playback_request_marks_counted_session_as_follow_up() { - let session = - create_test_session("tok-active", PlaylistItemType::LiveHls, crate::api::model::PlaybackLifecycle::Active); - - let request_class = classify_playback_request(PlaybackRequestFacts { - existing_session: Some(&session), - prepare_only: false, - terminate: false, - }); - - assert_eq!(request_class, PlaybackRequestClass::FollowUp); -} - -/// `PendingProvider` must NOT be classified as `FollowUp`. -/// `PendingProvider` has no counted lease yet — the session is still waiting -/// for a provider slot. A new request on a `PendingProvider` session should -/// be `Activate` so that full admission evaluation happens, not a cheap -/// `FollowUp` skip. -#[test] -fn classify_playback_request_marks_pending_provider_as_activate_not_follow_up() { - let session = create_test_session( - "tok-pending", - PlaylistItemType::LiveHls, - crate::api::model::PlaybackLifecycle::PendingProvider { - data: crate::api::model::PendingProviderState { - reason_code: crate::api::model::PendingProviderReason::GraceHold, - created_at: 1, - deadline: 30, - version: 1, - wake_source: None, - }, - }, - ); - - let request_class = classify_playback_request(PlaybackRequestFacts { - existing_session: Some(&session), - prepare_only: false, - terminate: false, - }); - - assert_eq!( - request_class, - PlaybackRequestClass::Activate, - "PendingProvider should not be FollowUp - it has no counted lease yet" - ); -} - -/// `Active` without a counted lease must NOT be classified as `FollowUp`. -/// `FollowUp` should only be returned when the session actually owns a -/// counted admission lease. A session with `Active` lifecycle but no counted -/// lease should go through `Activate` so that the counted lease is reacquired. -#[test] -fn classify_playback_request_marks_active_without_counted_as_activate_not_follow_up() { - let mut session = create_test_session( - "tok-active-uncounted", - PlaylistItemType::LiveHls, - crate::api::model::PlaybackLifecycle::Active, // counted=false via is_counted() - ); - // Manually force counted=false by setting to Prepared lifecycle, then restoring - // Note: is_counted() returns false for Prepared, true for Active - // For this test we need a session that is Active lifecycle but not counted - // The new model derives counted from lifecycle, so we must use a different lifecycle - // to represent "not counted". Use Prepared instead. - session.lifecycle = crate::api::model::PlaybackLifecycle::Prepared; - - let request_class = classify_playback_request(PlaybackRequestFacts { - existing_session: Some(&session), - prepare_only: false, - terminate: false, - }); - - assert_eq!( - request_class, - PlaybackRequestClass::Activate, - "Active session with counted=false should not be FollowUp" - ); -} - -/// Prepared sessions must be classified as Activate. -#[test] -fn classify_playback_request_marks_prepared_session_as_activate() { - let session = - create_test_session("tok-prepared", PlaylistItemType::LiveHls, crate::api::model::PlaybackLifecycle::Prepared); - - let request_class = classify_playback_request(PlaybackRequestFacts { - existing_session: Some(&session), - prepare_only: false, - terminate: false, - }); - - assert_eq!(request_class, PlaybackRequestClass::Activate); -} - -/// `GraceActive` without counted lease must NOT be classified as `FollowUp`. -#[test] -fn classify_playback_request_marks_grace_active_without_counted_as_activate() { - let mut session = create_test_session( - "tok-grace-uncounted", - PlaylistItemType::LiveHls, - crate::api::model::PlaybackLifecycle::Active, // is_counted() = true for GraceActive - ); - // Test scenario: session has GraceActive lifecycle but we need it NOT counted - // This represents the edge case before grace task resolves. Use Prepared lifecycle - // to model "not counted" since is_counted() returns false for Prepared. - session.lifecycle = crate::api::model::PlaybackLifecycle::Prepared; - - let request_class = classify_playback_request(PlaybackRequestFacts { - existing_session: Some(&session), - prepare_only: false, - terminate: false, - }); - - assert_eq!( - request_class, - PlaybackRequestClass::Activate, - "GraceActive session with counted=false should not be FollowUp" - ); -} - -#[tokio::test] -async fn activate_session_before_stream_open_skips_placeholder_for_follow_up_session() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest]), - }); - let addr: SocketAddr = "127.0.0.1:55220".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - let input = app_state.app_config.sources.load().inputs[0].clone(); - let mut user = ProxyUserCredentials::default(); - user.username = "follow-up-user".to_string(); - user.max_connections = 1; - let mut channel = create_test_live_channel("http://provider-1.example/live/55220.m3u8"); - channel.item_type = PlaylistItemType::LiveHls; - channel.virtual_id = 55220; - - app_state.connection_manager.add_connection(&addr).await; - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: "tok-follow-up", - virtual_id: channel.virtual_id, - provider: input.name.as_ref(), - stream_url: channel.url.as_ref(), - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: true, - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 1, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 10, - fingerprint: &fingerprint, - provider: input.name.clone(), - stream_channel: &channel, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-follow-up"), - }) - .await; - - let activation = activate_session_before_stream_open( - &app_state, - SessionActivationRequest { - fingerprint: &fingerprint, - input: input.as_ref(), - user: &user, - session_token: "tok-follow-up", - request_class: None, - virtual_id: VirtualId::new(channel.virtual_id), - item_type: PlaylistItemType::LiveHls, - stream_url: channel.url.as_ref(), - connection_permission: UserConnectionPermission::Allowed, - connection_kind: crate::api::model::ConnectionKind::Normal, - granted_grace_mode: None, - socket_bound: true, - }, - ) - .await; - - assert_eq!(activation.admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(activation.admission.kind(), Some(crate::api::model::ConnectionKind::Normal)); - assert_eq!(activation.grace_mode, None); - assert!( - activation.placeholder_transition_version.is_none(), - "follow-up activation must not create a placeholder session" - ); -} - -#[tokio::test] -async fn activate_session_before_stream_open_revalidates_precomputed_follow_up_request_class() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest]), - }); - let addr: SocketAddr = "127.0.0.1:55221".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - let input = app_state.app_config.sources.load().inputs[0].clone(); - let mut user = ProxyUserCredentials::default(); - user.username = "precomputed-follow-up-user".to_string(); - user.max_connections = 1; - let mut channel = create_test_live_channel("http://provider-1.example/live/55221.m3u8"); - channel.item_type = PlaylistItemType::LiveHls; - channel.virtual_id = 55221; - - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: "tok-precomputed-follow-up", - virtual_id: channel.virtual_id, - provider: input.name.as_ref(), - stream_url: channel.url.as_ref(), - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: true, - }) - .await; - - let activation = activate_session_before_stream_open( - &app_state, - SessionActivationRequest { - fingerprint: &fingerprint, - input: input.as_ref(), - user: &user, - session_token: "tok-precomputed-follow-up", - request_class: Some(PlaybackRequestClass::FollowUp), - virtual_id: VirtualId::new(channel.virtual_id), - item_type: PlaylistItemType::LiveHls, - stream_url: channel.url.as_ref(), - connection_permission: UserConnectionPermission::Allowed, - connection_kind: crate::api::model::ConnectionKind::Normal, - granted_grace_mode: None, - socket_bound: true, - }, - ) - .await; - - assert_eq!(activation.admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(activation.admission.kind(), Some(crate::api::model::ConnectionKind::Normal)); - assert_eq!(activation.grace_mode, None); - assert!( - activation.placeholder_transition_version.is_some(), - "precomputed FollowUp must be revalidated against the current uncounted lifecycle" - ); -} - -// stale FollowUp revalidation -#[tokio::test] -async fn activate_session_before_stream_open_stale_follow_up_reclassified_on_counted_lease_release() { - // Scenario: pre-computed FollowUp, but session's counted lease was released before - // the guard was acquired. Must reclassify to Activate so admission runs. - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest]), - }); - let addr: SocketAddr = "127.0.0.1:55230".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - let input = app_state.app_config.sources.load().inputs[0].clone(); - let mut user = ProxyUserCredentials::default(); - user.username = "stale-followup-user".to_string(); - user.max_connections = 1; - let mut channel = create_test_live_channel("http://provider-1.example/live/55230.m3u8"); - channel.item_type = PlaylistItemType::LiveHls; - channel.virtual_id = 55230; - - // Session created in Active (counted) state. - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: "tok-stale-followup", - virtual_id: channel.virtual_id, - provider: input.name.as_ref(), - stream_url: channel.url.as_ref(), - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: true, - }) - .await; - - // Simulate the counted lease being released before activation: - // expire the session so it no longer has a counted lease. - app_state.active_users.terminate_session(&user.username, "tok-stale-followup").await; - - // Call activate with stale FollowUp. Must NOT skip admission — reclassification - // to Activate must run so the placeholder is created. - let activation = activate_session_before_stream_open( - &app_state, - SessionActivationRequest { - fingerprint: &fingerprint, - input: input.as_ref(), - user: &user, - session_token: "tok-stale-followup", - request_class: Some(PlaybackRequestClass::FollowUp), - virtual_id: VirtualId::new(channel.virtual_id), - item_type: PlaylistItemType::LiveHls, - stream_url: channel.url.as_ref(), - connection_permission: UserConnectionPermission::Allowed, - connection_kind: crate::api::model::ConnectionKind::Normal, - granted_grace_mode: None, - socket_bound: true, - }, - ) - .await; - - // Must NOT skip — placeholder must be created since session is expired. - assert!( - activation.placeholder_transition_version.is_some(), - "stale FollowUp with expired session must run admission and create placeholder" - ); -} - -// pre-resolved Grace materialization -#[tokio::test] -async fn activate_session_before_stream_open_pre_resolved_grace_period_materializes_pending_provider() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), - }); - let addr: SocketAddr = "127.0.0.1:55231".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - let input = app_state.app_config.sources.load().inputs[0].clone(); - let mut user = ProxyUserCredentials::default(); - user.username = "pre-resolved-grace-user".to_string(); - user.max_connections = 1; - let mut channel = create_test_live_channel("http://provider-1.example/live/55231.m3u8"); - channel.item_type = PlaylistItemType::LiveHls; - channel.virtual_id = 55231; - - // A pre-resolved grace grant reaches activation before a session exists. - let activation = activate_session_before_stream_open( - &app_state, - SessionActivationRequest { - fingerprint: &fingerprint, - input: input.as_ref(), - user: &user, - session_token: "tok-pre-resolved-grace", - request_class: None, - virtual_id: VirtualId::new(channel.virtual_id), - item_type: PlaylistItemType::LiveHls, - stream_url: channel.url.as_ref(), - connection_permission: UserConnectionPermission::GracePeriod, - connection_kind: crate::api::model::ConnectionKind::Normal, - granted_grace_mode: Some(crate::api::model::GraceMode::Hold), - socket_bound: true, - }, - ) - .await; - - assert_eq!(activation.admission.permission(), UserConnectionPermission::GracePeriod); - assert_eq!(activation.grace_mode, Some(crate::api::model::GraceMode::Hold)); - - let session = app_state.active_users.get_and_update_user_session(&user.username, "tok-pre-resolved-grace").await; - assert!( - session.is_some_and(|s| matches!(s.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. })), - "pre-resolved GracePeriod must materialize as PendingProvider lifecycle" - ); -} - -#[tokio::test] -async fn pre_resolved_instant_grace_creates_counted_live_session() { - let app_state = create_test_app_state(); - let addr: SocketAddr = "127.0.0.1:55233".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - let input = app_state.app_config.sources.load().inputs[0].clone(); - let mut user = ProxyUserCredentials::default(); - user.username = "instant-grace-live-user".to_string(); - user.max_connections = 1; - let channel = create_test_live_channel("http://provider-1.example/live/55233.ts"); - - let activation = activate_session_before_stream_open( - &app_state, - SessionActivationRequest { - fingerprint: &fingerprint, - input: input.as_ref(), - user: &user, - session_token: "tok-instant-grace", - request_class: Some(PlaybackRequestClass::Activate), - virtual_id: VirtualId::new(channel.virtual_id), - item_type: PlaylistItemType::Live, - stream_url: channel.url.as_ref(), - connection_permission: UserConnectionPermission::GracePeriod, - connection_kind: crate::api::model::ConnectionKind::Normal, - granted_grace_mode: Some(crate::api::model::GraceMode::Instant), - socket_bound: true, - }, - ) - .await; - - assert_eq!(activation.grace_mode, Some(crate::api::model::GraceMode::Instant)); - assert_eq!(app_state.active_users.user_connections(&user.username).await, 1); - let session = app_state.active_users.get_and_update_user_session(&user.username, "tok-instant-grace").await; - assert!(session.is_some_and(|session| session.lifecycle == crate::api::model::PlaybackLifecycle::GraceActive)); -} - -/// `activate_session_before_stream_open` skips placeholder for Prepare class. -#[tokio::test] -async fn activate_session_before_stream_open_skips_placeholder_for_prepare() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest]), - }); - let addr: SocketAddr = "127.0.0.1:55222".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - let input = app_state.app_config.sources.load().inputs[0].clone(); - let mut user = ProxyUserCredentials::default(); - user.username = "prepare-user".to_string(); - user.max_connections = 1; - - let activation = activate_session_before_stream_open( - &app_state, - SessionActivationRequest { - fingerprint: &fingerprint, - input: input.as_ref(), - user: &user, - session_token: "tok-prepare", - // Explicitly pass Prepare class — placeholder and admission should be skipped. - request_class: Some(PlaybackRequestClass::Prepare), - virtual_id: VirtualId::new(55222), - item_type: PlaylistItemType::LiveHls, - stream_url: "http://provider.example/live/test.ts", - connection_permission: UserConnectionPermission::Allowed, - connection_kind: crate::api::model::ConnectionKind::Normal, - granted_grace_mode: None, - socket_bound: true, - }, - ) - .await; - - // Prepare returns Allowed without running admission strategies. - assert_eq!(activation.admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(activation.grace_mode, None); - assert!( - activation.placeholder_transition_version.is_none(), - "Prepare activation must not create a placeholder session" - ); -} - -/// `resolve_playback_request_admission` with `prepare_only = true` returns `Prepare` class. -#[tokio::test] -async fn resolve_playback_request_admission_prepare_only_returns_prepare_class() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), - }); - let addr: SocketAddr = "127.0.0.1:55223".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - let mut user = ProxyUserCredentials::default(); - user.username = "prepare-only-user".to_string(); - user.max_connections = 1; - - let (admission, grace_mode, request_class) = resolve_playback_request_admission( - &app_state.admission_ctx(), - &user, - &fingerprint, - None, - "tok-prepare-only", - false, - EvictionReentryGuard::Session("tok-prepare-only"), - true, // prepare_only - false, // terminate - ) - .await; - - assert_eq!(request_class, PlaybackRequestClass::Prepare); - // Prepare returns Allowed without running strategies. - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(grace_mode, None); -} - -/// `resolve_playback_request_admission` with `terminate = true` returns `Terminate` class -/// and calls `terminate_session` on the existing session. -#[tokio::test] -async fn resolve_playback_request_admission_terminate_returns_terminate_class() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), - }); - let addr: SocketAddr = "127.0.0.1:55224".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - let mut user = ProxyUserCredentials::default(); - user.username = "terminate-user".to_string(); - user.max_connections = 2; - - // First create a session. - let session_token = "tok-terminate"; - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token, - virtual_id: 55224, - provider: "test-provider", - stream_url: "http://provider.example/test.ts", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - // Verify session exists. - let before = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; - assert!(before.is_some(), "session should exist before terminate"); - - let (admission, grace_mode, request_class) = resolve_playback_request_admission( - &app_state.admission_ctx(), - &user, - &fingerprint, - before.as_ref(), - session_token, - false, - EvictionReentryGuard::Session(session_token), - false, // prepare_only - true, // terminate - ) - .await; - - assert_eq!(request_class, PlaybackRequestClass::Terminate); - assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); - assert_eq!(grace_mode, None); - - // Session should be expired after terminate. - let after = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; - assert!(after.is_none(), "session should be removed after terminate"); -} - -/// `classify_playback_request` returns `Terminate` when `terminate = true`. -#[test] -fn classify_playback_request_returns_terminate_when_flag_set() { - let request_class = classify_playback_request(PlaybackRequestFacts { - existing_session: None, - prepare_only: false, - terminate: true, - }); - assert_eq!(request_class, PlaybackRequestClass::Terminate); -} - -#[tokio::test] -async fn activate_session_before_stream_open_marks_pending_provider_for_grace_hold() { - let stream_cfg = crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), - }; - let mut app_cfg = create_test_app_config(); - app_cfg.config = Arc::new(ArcSwap::from_pointee(Config { - user_access_control: true, - reverse_proxy: Some(crate::model::ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: crate::model::ResourceRetryConfig::default(), - disabled_header: None, - stream: Some(stream_cfg), - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }), - ..Config::default() - })); - let app_state = create_test_app_state_for_config(Arc::new(app_cfg)); - let first_addr: SocketAddr = "127.0.0.1:55230".parse().unwrap_or_else(|_| unreachable!()); - let second_addr: SocketAddr = "127.0.0.1:55231".parse().unwrap_or_else(|_| unreachable!()); - let first_fingerprint = create_test_fingerprint(first_addr); - let second_fingerprint = create_test_fingerprint(second_addr); - let input = app_state.app_config.sources.load().inputs[0].clone(); - let mut user = ProxyUserCredentials::default(); - user.username = "grace-hold-user".to_string(); - user.max_connections = 1; - let first_channel = create_test_live_channel("http://provider-1.example/live/1.ts"); - let mut second_channel = create_test_live_channel("http://provider-1.example/live/2.m3u8"); - second_channel.item_type = PlaylistItemType::LiveHls; - second_channel.virtual_id = 55231; - - app_state.connection_manager.add_connection(&first_addr).await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 1, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 10, - fingerprint: &first_fingerprint, - provider: input.name.clone(), - stream_channel: &first_channel, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-first"), - }) - .await; - - let activation = activate_session_before_stream_open( - &app_state, - SessionActivationRequest { - fingerprint: &second_fingerprint, - input: input.as_ref(), - user: &user, - session_token: "tok-grace-hold", - request_class: None, - virtual_id: VirtualId::new(second_channel.virtual_id), - item_type: PlaylistItemType::LiveHls, - stream_url: second_channel.url.as_ref(), - connection_permission: UserConnectionPermission::Allowed, - connection_kind: crate::api::model::ConnectionKind::Normal, - granted_grace_mode: None, - socket_bound: false, - }, - ) - .await; - - assert_eq!(activation.admission.permission(), UserConnectionPermission::GracePeriod); - assert_eq!(activation.grace_mode, Some(crate::api::model::GraceMode::Hold)); - - let session = app_state - .active_users - .get_and_update_user_session(&user.username, "tok-grace-hold") - .await - .expect("placeholder session should exist"); - let crate::api::model::PlaybackLifecycle::PendingProvider { data: pending } = &session.lifecycle else { - panic!("grace hold should mark pending provider state") - }; - assert!(matches!(pending.reason_code, crate::api::model::PendingProviderReason::GraceHold)); - assert!(pending.deadline >= pending.created_at); - assert_eq!(app_state.active_users.user_connections(&user.username).await, 1); - assert!( - !session.lifecycle.is_counted(), - "pending provider placeholder must not consume an active user lease before commit" - ); -} - -#[tokio::test] -async fn activate_session_before_stream_open_does_not_commit_user_lease_before_provider_success() { - let stream_cfg = crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 0, - grace_period_timeout_secs: 8, - grace_period_hold_stream: false, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: None, - }; - let mut app_cfg = create_test_app_config(); - app_cfg.config = Arc::new(ArcSwap::from_pointee(Config { - user_access_control: true, - reverse_proxy: Some(crate::model::ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: crate::model::ResourceRetryConfig::default(), - disabled_header: None, - stream: Some(stream_cfg), - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }), - ..Config::default() - })); - let app_state = create_test_app_state_for_config(Arc::new(app_cfg)); - let addr: SocketAddr = "127.0.0.1:55232".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - let input = app_state.app_config.sources.load().inputs[0].clone(); - let mut user = ProxyUserCredentials::default(); - user.username = "atomic-commit-user".to_string(); - user.max_connections = 1; - let channel = create_test_live_channel("http://provider-1.example/live/3.ts"); - - let activation = activate_session_before_stream_open( - &app_state, - SessionActivationRequest { - fingerprint: &fingerprint, - input: input.as_ref(), - user: &user, - session_token: "tok-atomic-commit", - request_class: None, - virtual_id: VirtualId::new(channel.virtual_id), - item_type: channel.item_type, - stream_url: channel.url.as_ref(), - connection_permission: UserConnectionPermission::Allowed, - connection_kind: crate::api::model::ConnectionKind::Normal, - granted_grace_mode: None, - socket_bound: false, - }, - ) - .await; - - assert_eq!(activation.admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(activation.grace_mode, None); - - let session = app_state - .active_users - .get_and_update_user_session(&user.username, "tok-atomic-commit") - .await - .expect("placeholder session should exist"); - assert_eq!( - app_state.active_users.user_connections(&user.username).await, - 0, - "allowed activation should stay provisional until provider acquisition and stream commit succeed" - ); - assert!( - !session.lifecycle.is_counted(), - "placeholder session must stay uncounted until the provider side has been committed" - ); - assert!(!matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. })); -} - -fn create_test_shared_target() -> ConfigTarget { - ConfigTarget { - curation: None, - id: 1, - enabled: true, - name: "shared".to_string(), - options: Some(ConfigTargetOptions { - share_live_streams: shared::model::ConfigTargetShareLiveStreams { mpeg_ts: true, ..Default::default() }, - ..ConfigTargetOptions::default() - }), - sort: None, - filter: Filter::default().into(), - output: Vec::new(), - rename: None, - mapping_ids: None, - mapping: Arc::new(ArcSwapOption::default()), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - } -} - -#[test] -fn admission_failure_reason_maps_to_custom_video_type() { - assert!(matches!( - admission_failure_video_type(ConnectFailureReason::UserAccountExpired), - Some(CustomVideoStreamType::UserAccountExpired) - )); - assert!(matches!( - admission_failure_video_type(ConnectFailureReason::UserConnectionsExhausted), - Some(CustomVideoStreamType::UserConnectionsExhausted) - )); - assert!(matches!( - admission_failure_video_type(ConnectFailureReason::ProviderConnectionsExhausted), - Some(CustomVideoStreamType::ProviderConnectionsExhausted) - )); - assert!(admission_failure_video_type(ConnectFailureReason::ProviderError).is_none()); -} - -#[tokio::test] -async fn effective_admission_strategies_use_legacy_grace_when_field_missing() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: None, - }); - - assert_eq!( - get_effective_admission_strategies(&app_state.admission_ctx()).as_ref(), - &[shared::model::AdmissionStrategy::GraceHoldStream][..] - ); -} - -#[tokio::test] -async fn effective_admission_strategies_respect_explicit_empty_list() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![]), - }); - - assert!(get_effective_admission_strategies(&app_state.admission_ctx()).is_empty()); -} - -#[tokio::test] -async fn grace_context_is_populated_when_grace_strategy_is_actually_granted() { - // Use a DIFFERENT session token than the pre-existing counted session. - // Otherwise session-admission may treat it as a valid reopen and skip the exhausted path. - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![ - AdmissionStrategy::EvictUserSameIpOldest, - AdmissionStrategy::GraceHoldStream, - AdmissionStrategy::EvictUserOldest, - ]), - }); - - let addr1: SocketAddr = "127.0.0.1:55401".parse().unwrap_or_else(|_| unreachable!()); - let addr2: SocketAddr = "10.0.0.5:55402".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint1 = create_test_fingerprint(addr1); - let fingerprint2 = create_test_fingerprint(addr2); - // addr1 and addr2 have DIFFERENT IPs. - // EvictUserSameIpOldest will NOT match (different IP), so GraceHoldStream is evaluated. - let mut user = ProxyUserCredentials::default(); - user.username = "user-grace-ctx".to_string(); - user.max_connections = 1; - - // Register the connection first so update_connection succeeds - app_state.connection_manager.add_connection(&addr1).await; - - // Create the session — lifecycle starts as Prepared (uncounted) - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: "tok-existing-counted", - virtual_id: 55401, - provider: "provider_1", - stream_url: "http://provider-1.example/live/55401.m3u8", - addr: &addr1, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: true, - }) - .await; - - // update_connection promotes the session to Active (counted) and creates a stream. - // This exhausts the user's single slot (max_connections = 1). - app_state - .active_users - .update_connection(crate::api::model::ActiveUserConnectionParams { - uid: 55401, - meter_uid: 55401, - username: "user-grace-ctx", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint1, - provider: "provider_1".intern(), - stream_channel: &create_test_live_channel("http://provider-1.example/live/55401.m3u8"), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-existing-counted"), - }) - .await - .expect("stream should be created"); - - // Now the new request finds the slot exhausted and the grace strategy kicks in. - let result = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - client_ip: &fingerprint2.client_ip, - request_addr: &fingerprint2.addr, - use_session_admission: true, - session_token: Some("tok-new-request"), - activate_unbound_session: true, - eviction_reentry_guard: EvictionReentryGuard::Session("tok-new-request"), - }, - ) - .await; - - assert_eq!(result.admission.permission(), UserConnectionPermission::GracePeriod, "grace should be granted"); - assert!(matches!(result.grace_mode, Some(crate::api::model::GraceMode::Hold))); - let ctx = result.grace_context.expect("grace_context must be present when grace is granted"); - assert_eq!(ctx.strategy_index, 1, "GraceHoldStream is at index 1"); - assert_eq!(ctx.strategies.len(), 3); - assert!(matches!(ctx.strategies[ctx.strategy_index], AdmissionStrategy::GraceHoldStream)); -} - -#[tokio::test] -async fn evaluate_remaining_strategies_evicts_after_used_grace() { - // Strategies: [GraceHoldStream, EvictUserOldest] - // Grace was used at index 0, so only EvictUserOldest (index 1) is evaluated. - // The new request stays retryable while the evicted stream still owns its provider slot. - let strategies = vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]; - let grace_context = GraceResolutionContext { strategy_index: 0, strategies: strategies.into(), kind: None }; - - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]), - }); - let provider_config = create_test_provider_app_config(); - app_state.app_config.sources.store(provider_config.sources.load_full()); - app_state.active_provider.update_config(&app_state.app_config); - - let addr1: SocketAddr = "127.0.0.1:55701".parse().unwrap_or_else(|_| unreachable!()); - let addr2: SocketAddr = "10.0.0.5:55702".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint1 = create_test_fingerprint(addr1); - let fingerprint2 = create_test_fingerprint(addr2); - - app_state.connection_manager.add_connection(&addr1).await; - app_state.connection_manager.add_connection(&addr2).await; - - let mut user = ProxyUserCredentials::default(); - user.username = "remaining-evict".to_string(); - user.max_connections = 1; - - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: "tok-counted", - virtual_id: 55701, - provider: "provider-evict", - stream_url: "http://provider.example/live/1.ts", - addr: &addr1, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - app_state - .active_users - .update_connection(crate::api::model::ActiveUserConnectionParams { - uid: 55701, - meter_uid: 55701, - username: "remaining-evict", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint1, - provider: "provider-evict".intern(), - stream_channel: &create_test_live_channel("http://provider.example/live/1.ts"), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-counted"), - }) - .await - .expect("stream should be created"); - - let provider_handle = app_state - .active_provider - .acquire_connection_with_grace_for_session( - &"provider_1".intern(), - &addr1, - false, - 0, - crate::api::model::ConnectionKind::Normal, - Some("tok-counted"), - ) - .expect("old stream should occupy the only provider slot"); - assert!(app_state.active_provider.register_body_owner(provider_handle.allocation_id)); - let close_rx = app_state.connection_manager.register_close_socket(addr1); - let manager = Arc::clone(&app_state.connection_manager); - let provider = Arc::clone(&app_state.active_provider); - let release_body = Arc::new(tokio::sync::Notify::new()); - let release_body_after_timeout = Arc::clone(&release_body); - let close_task = tokio::spawn(async move { - assert_eq!(close_rx.await.expect("kick close signal"), shared::model::DisconnectReason::ClientKicked); - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - manager.release_provider_deferred(&addr1).await; - assert_eq!(provider.get_provider_connections_count(), 1, "body owner still holds provider capacity"); - release_body_after_timeout.notified().await; - provider.release_handle(&provider_handle); - provider_handle.completion_token.as_ref().expect("body completion token").cancel(); - manager.unregister_close_socket(&addr1); - }); - - let request = || AdmissionRequest { - username: "remaining-evict", - max_connections: 1, - soft_connections: 0, - client_ip: &fingerprint2.client_ip, - request_addr: &fingerprint2.addr, - use_session_admission: true, - session_token: Some("tok-new"), - activate_unbound_session: true, - eviction_reentry_guard: EvictionReentryGuard::Session("tok-new"), - }; - let result = tokio::time::timeout( - std::time::Duration::from_secs(5), - evaluate_remaining_strategies_after_grace( - &app_state.admission_ctx(), - request(), - &grace_context, - Some(crate::api::model::ConnectionKind::Normal), - ), - ) - .await - .expect("admission must remain retryable while the provider slot is held"); - - assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted); - assert_eq!(app_state.active_provider.get_provider_connections_count(), 1); - - let retry = tokio::time::timeout( - std::time::Duration::from_secs(5), - resolve_admission_with_strategies(&app_state.admission_ctx(), request()), - ) - .await - .expect("retry must remain bounded while the provider slot is held"); - assert_eq!(retry.admission.permission(), UserConnectionPermission::Exhausted); - - release_body.notify_one(); - close_task.await.expect("old transport cleanup"); - let result = resolve_admission_with_strategies(&app_state.admission_ctx(), request()).await; - - assert_eq!( - result.admission.permission(), - UserConnectionPermission::Allowed, - "EvictUserOldest should free the slot" - ); - assert!(result.grace_context.is_none(), "no grace context on eviction success"); - assert_eq!( - app_state.active_provider.get_provider_connections_count(), - 0, - "admission must not return while the evicted stream still owns the provider slot" - ); - let replacement = app_state.active_provider.acquire_connection_with_grace_for_session( - &"provider_1".intern(), - &addr2, - false, - 0, - crate::api::model::ConnectionKind::Normal, - Some("tok-new"), - ); - assert!(replacement.is_some(), "newly admitted stream must acquire the freed provider slot"); - if let Some(replacement) = replacement { - app_state.active_provider.release_handle(&replacement); - } -} - -#[tokio::test] -async fn evaluate_remaining_strategies_skips_no_match_and_uses_later_eviction() { - // Strategies: [GraceHoldStream, EvictUserSameIpOldest, EvictUserOldest] - // Grace was at index 0, remaining are EvictUserSameIpOldest (index 1) and EvictUserOldest (index 2). - // The existing counted session is at a DIFFERENT IP, so EvictUserSameIpOldest -> NoMatch. - // EvictUserOldest succeeds -> Allowed. - let strategies = vec![ - AdmissionStrategy::GraceHoldStream, - AdmissionStrategy::EvictUserSameIpOldest, - AdmissionStrategy::EvictUserOldest, - ]; - let grace_context = GraceResolutionContext { strategy_index: 0, strategies: strategies.into(), kind: None }; - - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![ - AdmissionStrategy::GraceHoldStream, - AdmissionStrategy::EvictUserSameIpOldest, - AdmissionStrategy::EvictUserOldest, - ]), - }); - - let addr1: SocketAddr = "127.0.0.1:55801".parse().unwrap_or_else(|_| unreachable!()); - let addr2: SocketAddr = "10.0.0.5:55802".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint1 = create_test_fingerprint(addr1); - let fingerprint2 = create_test_fingerprint(addr2); - - app_state.connection_manager.add_connection(&addr1).await; - app_state.connection_manager.add_connection(&addr2).await; - - let mut user = ProxyUserCredentials::default(); - user.username = "remaining-skip-no-match".to_string(); - user.max_connections = 1; - - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: "tok-counted", - virtual_id: 55801, - provider: "provider-skip", - stream_url: "http://provider.example/live/1.ts", - addr: &addr1, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - app_state - .active_users - .update_connection(crate::api::model::ActiveUserConnectionParams { - uid: 55801, - meter_uid: 55801, - username: "remaining-skip-no-match", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint1, - provider: "provider-skip".intern(), - stream_channel: &create_test_live_channel("http://provider.example/live/1.ts"), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-counted"), - }) - .await - .expect("stream should be created"); - - let result = evaluate_remaining_strategies_after_grace( - &app_state.admission_ctx(), - AdmissionRequest { - username: "remaining-skip-no-match", - max_connections: 1, - soft_connections: 0, - client_ip: &fingerprint2.client_ip, - request_addr: &fingerprint2.addr, - use_session_admission: true, - session_token: Some("tok-new"), - activate_unbound_session: true, - eviction_reentry_guard: EvictionReentryGuard::Session("tok-new"), - }, - &grace_context, - Some(crate::api::model::ConnectionKind::Normal), - ) - .await; - - assert_eq!( - result.admission.permission(), - UserConnectionPermission::Allowed, - "EvictUserSameIpOldest should NoMatch, EvictUserOldest should succeed" - ); -} - -#[tokio::test] -async fn evaluate_remaining_strategies_empty_slice_denies() { - // Strategies: [GraceHoldStream] - // Grace was at index 0, remaining slice is empty -> exhausted. - let strategies = vec![AdmissionStrategy::GraceHoldStream]; - let grace_context = GraceResolutionContext { strategy_index: 0, strategies: strategies.into(), kind: None }; - - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), - }); - - let addr: SocketAddr = "10.0.0.5:55901".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - - let result = evaluate_remaining_strategies_after_grace( - &app_state.admission_ctx(), - AdmissionRequest { - username: "no-remaining-strategies", - max_connections: 1, - soft_connections: 0, - client_ip: &fingerprint.client_ip, - request_addr: &fingerprint.addr, - use_session_admission: true, - session_token: Some("tok-new"), - activate_unbound_session: true, - eviction_reentry_guard: EvictionReentryGuard::Session("tok-new"), - }, - &grace_context, - None, - ) - .await; - - assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted, "empty remaining slice should deny"); -} - -#[tokio::test] -async fn evaluate_remaining_strategies_preserves_soft_kind_on_exhausted() { - // Strategies: [GraceHoldStream] - // Grace was at index 0, remaining slice is empty -> exhausted. - // grace_context.kind is Soft — must be preserved in the exhausted result. - let strategies = vec![AdmissionStrategy::GraceHoldStream]; - let grace_context = GraceResolutionContext { - strategy_index: 0, - strategies: strategies.into(), - kind: Some(crate::api::model::ConnectionKind::Soft), - }; - - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), - }); - - let addr: SocketAddr = "10.0.0.6:55902".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - - let result = evaluate_remaining_strategies_after_grace( - &app_state.admission_ctx(), - AdmissionRequest { - username: "soft-kind-user", - max_connections: 1, - soft_connections: 0, - client_ip: &fingerprint.client_ip, - request_addr: &fingerprint.addr, - use_session_admission: true, - session_token: Some("tok-soft"), - activate_unbound_session: true, - eviction_reentry_guard: EvictionReentryGuard::Session("tok-soft"), - }, - &grace_context, - Some(crate::api::model::ConnectionKind::Soft), - ) - .await; - - assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted, "empty remaining slice should deny"); - assert_eq!( - result.admission.kind(), - Some(crate::api::model::ConnectionKind::Soft), - "exhausted result must preserve the original Soft connection kind" - ); -} - -#[tokio::test] -async fn evaluate_remaining_strategies_does_not_retry_used_prefix() { - // Strategies: [GraceHoldStream, GraceInstantStream, EvictUserOldest] - // Grace was at index 1 (GraceInstantStream). - // Remaining slice: [EvictUserOldest] (index 2). - // GraceHoldStream (index 0) must NOT be re-evaluated. - let strategies = vec![ - AdmissionStrategy::GraceHoldStream, - AdmissionStrategy::GraceInstantStream, - AdmissionStrategy::EvictUserOldest, - ]; - let strategies_for_config = strategies.clone(); - let grace_context = GraceResolutionContext { strategy_index: 1, strategies: strategies.into(), kind: None }; - - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(strategies_for_config), - }); - - let addr1: SocketAddr = "127.0.0.1:56001".parse().unwrap_or_else(|_| unreachable!()); - let addr2: SocketAddr = "10.0.0.5:56002".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint1 = create_test_fingerprint(addr1); - let fingerprint2 = create_test_fingerprint(addr2); - - app_state.connection_manager.add_connection(&addr1).await; - app_state.connection_manager.add_connection(&addr2).await; - - let mut user = ProxyUserCredentials::default(); - user.username = "remaining-no-retry".to_string(); - user.max_connections = 1; - - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: "tok-counted", - virtual_id: 56001, - provider: "provider-no-retry", - stream_url: "http://provider.example/live/1.ts", - addr: &addr1, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - app_state - .active_users - .update_connection(crate::api::model::ActiveUserConnectionParams { - uid: 56001, - meter_uid: 56001, - username: "remaining-no-retry", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint1, - provider: "provider-no-retry".intern(), - stream_channel: &create_test_live_channel("http://provider.example/live/1.ts"), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-counted"), - }) - .await - .expect("stream should be created"); - - let result = evaluate_remaining_strategies_after_grace( - &app_state.admission_ctx(), - AdmissionRequest { - username: "remaining-no-retry", - max_connections: 1, - soft_connections: 0, - client_ip: &fingerprint2.client_ip, - request_addr: &fingerprint2.addr, - use_session_admission: true, - session_token: Some("tok-new"), - activate_unbound_session: true, - eviction_reentry_guard: EvictionReentryGuard::Session("tok-new"), - }, - &grace_context, - Some(crate::api::model::ConnectionKind::Normal), - ) - .await; - - assert_eq!( - result.admission.permission(), - UserConnectionPermission::Allowed, - "only EvictUserOldest should be evaluated, not GraceHoldStream" - ); -} - -#[tokio::test] -async fn evaluate_remaining_strategies_empty_slice_uses_original_kind_not_context_kind() { - // grace_context.kind = Normal, original_kind = Soft - // remaining slice is empty -> exhausted result must use original_kind. - // This proves the empty-slice branch uses original_kind, not grace_context.kind. - let strategies = vec![AdmissionStrategy::GraceHoldStream]; - let grace_context = GraceResolutionContext { - strategy_index: 0, - strategies: strategies.into(), - kind: Some(crate::api::model::ConnectionKind::Normal), - }; - let original_kind = Some(crate::api::model::ConnectionKind::Soft); - - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), - }); - - let addr: SocketAddr = "10.0.0.7:55903".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - - let result = evaluate_remaining_strategies_after_grace( - &app_state.admission_ctx(), - AdmissionRequest { - username: "kind-mismatch-empty", - max_connections: 1, - soft_connections: 0, - client_ip: &fingerprint.client_ip, - request_addr: &fingerprint.addr, - use_session_admission: true, - session_token: Some("tok-empty"), - activate_unbound_session: true, - eviction_reentry_guard: EvictionReentryGuard::Session("tok-empty"), - }, - &grace_context, - original_kind, - ) - .await; - - assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted); - assert_eq!( - result.admission.kind(), - original_kind, - "exhausted result must use original_kind (Soft), not grace_context.kind (Normal)" - ); -} - -#[tokio::test] -async fn evaluate_remaining_strategies_later_grace_uses_original_kind_not_context_kind() { - // grace_context.kind = Normal, original_kind = Soft - // Strategies: [GraceHoldStream, GraceInstantStream] - // Grace was used at index 0 (GraceHoldStream). - // Remaining slice contains GraceInstantStream (index 1). - // When the helper returns Grace for the remaining strategy, the new - // GraceResolutionContext.kind must be original_kind (Soft), not grace_context.kind (Normal). - // This proves build_grace_ctx uses original_kind as source of truth. - let strategies = vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::GraceInstantStream]; - let grace_context = GraceResolutionContext { - strategy_index: 0, - strategies: strategies.into(), - kind: Some(crate::api::model::ConnectionKind::Normal), - }; - let original_kind = Some(crate::api::model::ConnectionKind::Soft); - - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::GraceInstantStream]), - }); - - let addr1: SocketAddr = "127.0.0.1:55710".parse().unwrap_or_else(|_| unreachable!()); - let addr2: SocketAddr = "10.0.0.8:55711".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint1 = create_test_fingerprint(addr1); - let fingerprint2 = create_test_fingerprint(addr2); - - app_state.connection_manager.add_connection(&addr1).await; - app_state.connection_manager.add_connection(&addr2).await; - - let mut user = ProxyUserCredentials::default(); - user.username = "kind-mismatch-grace".to_string(); - user.max_connections = 1; - - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: "tok-counted-grace", - virtual_id: 55710, - provider: "provider-grace-kind", - stream_url: "http://provider.example/live/1.ts", - addr: &addr1, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - app_state - .active_users - .update_connection(crate::api::model::ActiveUserConnectionParams { - uid: 55710, - meter_uid: 55710, - username: "kind-mismatch-grace", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint1, - provider: "provider-grace-kind".intern(), - stream_channel: &create_test_live_channel("http://provider.example/live/1.ts"), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-counted-grace"), - }) - .await - .expect("stream should be created"); - - let result = evaluate_remaining_strategies_after_grace( - &app_state.admission_ctx(), - AdmissionRequest { - username: "kind-mismatch-grace", - max_connections: 1, - soft_connections: 0, - client_ip: &fingerprint2.client_ip, - request_addr: &fingerprint2.addr, - use_session_admission: true, - session_token: Some("tok-new-grace"), - activate_unbound_session: true, - eviction_reentry_guard: EvictionReentryGuard::Session("tok-new-grace"), - }, - &grace_context, - original_kind, - ) - .await; - - assert_eq!( - result.admission.permission(), - UserConnectionPermission::GracePeriod, - "remaining GraceInstantStream should grant GracePeriod" - ); - assert!(result.grace_context.is_some(), "grace_context must be present when grace is granted"); - assert_eq!( - result.grace_context.as_ref().unwrap().kind, - original_kind, - "GraceResolutionContext.kind in the result must be original_kind (Soft), not grace_context.kind (Normal)" - ); -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn resolve_admission_with_strategies_falls_through_after_failed_grace_grant() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]), - }); - - let first_addr: std::net::SocketAddr = "127.0.0.1:55151".parse().unwrap_or_else(|_| unreachable!()); - let second_addr: std::net::SocketAddr = "127.0.0.1:55152".parse().unwrap_or_else(|_| unreachable!()); - let first_fingerprint = create_test_fingerprint(first_addr); - let second_fingerprint = create_test_fingerprint(second_addr); - - app_state.connection_manager.add_connection(&first_addr).await; - app_state.connection_manager.add_connection(&second_addr).await; - - let mut session_user = ProxyUserCredentials::default(); - session_user.username = "fallthrough".to_string(); - session_user.max_connections = 1; - session_user.soft_connections = 1; - - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &session_user, - session_token: "tok-first", - virtual_id: 1, - provider: "provider-a", - stream_url: "http://provider-1.example/live/1.ts", - addr: &first_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 1, - username: "fallthrough", - max_connections: 1, - soft_connections: 1, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 10, - fingerprint: &first_fingerprint, - provider: "provider-a".intern(), - stream_channel: &create_test_live_channel("http://provider-1.example/live/1.ts"), - user_agent: std::borrow::Cow::Borrowed("ua"), - session_token: Some("tok-first"), - }) - .await; - - assert!(app_state.active_users.grant_grace("fallthrough").await); - - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &session_user, - session_token: "tok-second", - virtual_id: 2, - provider: "provider-a", - stream_url: "http://provider-1.example/live/2.ts", - addr: &second_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Soft), - socket_bound: false, - }) - .await; - - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 2, - username: "fallthrough", - max_connections: 1, - soft_connections: 1, - connection_kind: crate::api::model::ConnectionKind::Soft, - priority: 0, - soft_priority: 10, - fingerprint: &second_fingerprint, - provider: "provider-a".intern(), - stream_channel: &create_test_live_channel("http://provider-1.example/live/2.ts"), - user_agent: std::borrow::Cow::Borrowed("ua"), - session_token: Some("tok-second"), - }) - .await; - - let result = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: "fallthrough", - max_connections: 1, - soft_connections: 1, - client_ip: "127.0.0.1", - request_addr: &"127.0.0.1:55153".parse().unwrap_or_else(|_| unreachable!()), - use_session_admission: true, - session_token: Some("tok-third"), - activate_unbound_session: false, - eviction_reentry_guard: EvictionReentryGuard::Session("tok-third"), - }, - ) - .await; - let admission = result.admission; - let grace_mode = result.grace_mode; - - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(admission.kind(), Some(crate::api::model::ConnectionKind::Normal)); - assert_eq!(grace_mode, None); -} - -#[tokio::test] -async fn resolve_admission_with_strategies_allows_existing_session_even_when_user_is_at_limit() { - let app_state = create_test_app_state(); - let addr = "127.0.0.1:55154".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - let mut user = ProxyUserCredentials::default(); - user.username = "session-admission".to_string(); - user.max_connections = 1; - - app_state.connection_manager.add_connection(&addr).await; - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: "vod-session", - virtual_id: 1, - provider: "provider-a", - stream_url: "http://provider-1.example/movie/1.mkv", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 1, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 10, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &create_test_live_channel("http://provider-1.example/movie/1.mkv"), - user_agent: std::borrow::Cow::Borrowed("ua"), - session_token: Some("vod-session"), - }) - .await; - - let session_based = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - client_ip: &fingerprint.client_ip, - request_addr: &fingerprint.addr, - use_session_admission: true, - session_token: Some("vod-session"), - activate_unbound_session: false, - eviction_reentry_guard: EvictionReentryGuard::Session("vod-session"), - }, - ) - .await; - assert_eq!(session_based.admission.permission(), UserConnectionPermission::Allowed); - - let connection_based = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - client_ip: &fingerprint.client_ip, - request_addr: &fingerprint.addr, - use_session_admission: false, - session_token: Some("vod-session"), - activate_unbound_session: false, - eviction_reentry_guard: EvictionReentryGuard::Session("vod-session"), - }, - ) - .await; - assert_eq!(connection_based.admission.permission(), UserConnectionPermission::Exhausted); -} - -#[tokio::test] -async fn resolve_admission_with_strategies_prevents_recently_evicted_playback_ping_pong() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 0, - grace_period_timeout_secs: 8, - grace_period_hold_stream: false, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::EvictUserOldest]), - }); - - let victim_addr: std::net::SocketAddr = "127.0.0.1:55181".parse().unwrap_or_else(|_| unreachable!()); - let reconnect_addr: std::net::SocketAddr = "127.0.0.1:55182".parse().unwrap_or_else(|_| unreachable!()); - let winner_addr: std::net::SocketAddr = "127.0.0.1:55183".parse().unwrap_or_else(|_| unreachable!()); - let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); - let reconnect_fingerprint = create_test_fingerprint_with_user_agent(reconnect_addr, "player/1.0"); - let winner_fingerprint = create_test_fingerprint_with_user_agent(winner_addr, "winner/1.0"); - let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9001.ts"); - victim_channel.virtual_id = 9001; - let mut winner_channel = create_test_live_channel("http://provider-1.example/live/9002.ts"); - winner_channel.virtual_id = 9002; - - let mut session_user = ProxyUserCredentials::default(); - session_user.username = "loop-user".to_string(); - - app_state.connection_manager.add_connection(&victim_addr).await; - app_state.connection_manager.add_connection(&winner_addr).await; - - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &session_user, - session_token: "session-victim", - virtual_id: 9001, - provider: "provider-a", - stream_url: "http://provider-1.example/live/9001.ts", - addr: &victim_addr, - connection_permission: shared::model::UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &session_user, - session_token: "session-winner", - virtual_id: 9002, - provider: "provider-a", - stream_url: "http://provider-1.example/live/9002.ts", - addr: &winner_addr, - connection_permission: shared::model::UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 1, - username: "loop-user", - max_connections: 2, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &victim_fingerprint, - provider: "provider-a".intern(), - stream_channel: &victim_channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some("session-victim"), - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 2, - username: "loop-user", - max_connections: 2, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &winner_fingerprint, - provider: "provider-a".intern(), - stream_channel: &winner_channel, - user_agent: std::borrow::Cow::Borrowed("winner/1.0"), - session_token: Some("session-winner"), - }) - .await; - - app_state - .active_users - .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) - .await; - app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; - - let result = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: "loop-user", - max_connections: 1, - soft_connections: 0, - client_ip: &reconnect_fingerprint.client_ip, - request_addr: &reconnect_fingerprint.addr, - use_session_admission: true, - session_token: Some("socket-reconnect"), - activate_unbound_session: false, - eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9001) }, - }, - ) - .await; - let admission = result.admission; - let grace_mode = result.grace_mode; - - assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); - assert_eq!(admission.rejection_reason(), Some(AdmissionRejectionReason::RecentEvictionReentry)); - assert!(admission.is_reentry_suppressed()); - assert_eq!(grace_mode, None); - let active_streams = app_state.active_users.active_streams().await; - assert_eq!(active_streams.len(), 1); - assert_eq!(active_streams[0].channel.virtual_id, 9002); - - // A player may retry the winning plain-TS stream on another socket before the - // previous HTTP body has drained. That is a replacement of the same playback, - // not the evicted channel trying to reclaim its slot. - app_state - .active_users - .mark_recent_eviction_guard_for_addr(&winner_addr, reconnect_addr, std::time::Duration::from_secs(3)) - .await; - app_state.connection_manager.release_connection_as_kicked(&winner_addr).await; - app_state.connection_manager.add_connection(&reconnect_addr).await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 3, - username: "loop-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &reconnect_fingerprint, - provider: "provider-a".intern(), - stream_channel: &winner_channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: None, - }) - .await; - - let retry_addr: std::net::SocketAddr = "127.0.0.1:55184".parse().unwrap_or_else(|_| unreachable!()); - let retry = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: "loop-user", - max_connections: 1, - soft_connections: 0, - client_ip: &reconnect_fingerprint.client_ip, - request_addr: &retry_addr, - use_session_admission: true, - session_token: Some("socket-retry"), - activate_unbound_session: false, - eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9002) }, - }, - ) - .await; - - assert_eq!(retry.admission.permission(), UserConnectionPermission::Allowed); - assert!(app_state.active_users.active_streams().await.is_empty()); -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn resolve_admission_with_strategies_allows_other_channel_after_recent_eviction() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 0, - grace_period_timeout_secs: 8, - grace_period_hold_stream: false, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::EvictUserOldest]), - }); - - let victim_addr: std::net::SocketAddr = "127.0.0.1:55184".parse().unwrap_or_else(|_| unreachable!()); - let winner_addr: std::net::SocketAddr = "127.0.0.1:55185".parse().unwrap_or_else(|_| unreachable!()); - let new_addr: std::net::SocketAddr = "127.0.0.1:55186".parse().unwrap_or_else(|_| unreachable!()); - let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); - let winner_fingerprint = create_test_fingerprint_with_user_agent(winner_addr, "winner/1.0"); - let new_fingerprint = create_test_fingerprint_with_user_agent(new_addr, "player/1.0"); - let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9101.ts"); - victim_channel.virtual_id = 9101; - let mut winner_channel = create_test_live_channel("http://provider-1.example/live/9102.ts"); - winner_channel.virtual_id = 9102; - let mut session_user = ProxyUserCredentials::default(); - session_user.username = "loop-user-2".to_string(); - - app_state.connection_manager.add_connection(&victim_addr).await; - app_state.connection_manager.add_connection(&winner_addr).await; - - // Create sessions before update_connection so streams are linked to counted sessions - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &session_user, - session_token: "session-victim", - virtual_id: 9101, - provider: "provider-a", - stream_url: "http://provider-1.example/live/9101.ts", - addr: &victim_addr, - connection_permission: shared::model::UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &session_user, - session_token: "session-winner", - virtual_id: 9102, - provider: "provider-a", - stream_url: "http://provider-1.example/live/9102.ts", - addr: &winner_addr, - connection_permission: shared::model::UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 1, - username: "loop-user-2", - max_connections: 2, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &victim_fingerprint, - provider: "provider-a".intern(), - stream_channel: &victim_channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some("session-victim"), - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 2, - username: "loop-user-2", - max_connections: 2, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &winner_fingerprint, - provider: "provider-a".intern(), - stream_channel: &winner_channel, - user_agent: std::borrow::Cow::Borrowed("winner/1.0"), - session_token: Some("session-winner"), - }) - .await; - - app_state - .active_users - .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) - .await; - app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; - - let result = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: "loop-user-2", - max_connections: 1, - soft_connections: 0, - client_ip: &new_fingerprint.client_ip, - request_addr: &new_fingerprint.addr, - use_session_admission: true, - session_token: Some("session-new"), - activate_unbound_session: false, - eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9103) }, - }, - ) - .await; - let admission = result.admission; - - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - assert!(app_state.active_users.active_streams().await.is_empty()); -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn resolve_admission_with_strategies_does_not_suppress_different_session_on_same_channel() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 0, - grace_period_timeout_secs: 8, - grace_period_hold_stream: false, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::EvictUserOldest]), - }); - - let victim_addr: std::net::SocketAddr = "127.0.0.1:55190".parse().unwrap_or_else(|_| unreachable!()); - let winner_addr: std::net::SocketAddr = "127.0.0.1:55191".parse().unwrap_or_else(|_| unreachable!()); - let new_addr: std::net::SocketAddr = "127.0.0.1:55192".parse().unwrap_or_else(|_| unreachable!()); - let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); - let winner_fingerprint = create_test_fingerprint_with_user_agent(winner_addr, "player/1.0"); - let new_fingerprint = create_test_fingerprint_with_user_agent(new_addr, "player/1.0"); - let mut channel = create_test_live_channel("http://provider-1.example/live/9301.m3u8"); - channel.virtual_id = 9301; - channel.item_type = PlaylistItemType::LiveHls; - let mut session_user = ProxyUserCredentials::default(); - session_user.username = "loop-user-4".to_string(); - - app_state.connection_manager.add_connection(&victim_addr).await; - app_state.connection_manager.add_connection(&winner_addr).await; - - // Create sessions before update_connection so streams are linked to counted sessions - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &session_user, - session_token: "session-victim", - virtual_id: 9301, - provider: "provider-a", - stream_url: "http://provider-1.example/live/9301.m3u8", - addr: &victim_addr, - connection_permission: shared::model::UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &session_user, - session_token: "session-winner", - virtual_id: 9301, - provider: "provider-a", - stream_url: "http://provider-1.example/live/9301.m3u8", - addr: &winner_addr, - connection_permission: shared::model::UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 1, - username: "loop-user-4", - max_connections: 2, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &victim_fingerprint, - provider: "provider-a".intern(), - stream_channel: &channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some("session-victim"), - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 2, - username: "loop-user-4", - max_connections: 2, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &winner_fingerprint, - provider: "provider-a".intern(), - stream_channel: &channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some("session-winner"), - }) - .await; - - app_state - .active_users - .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) - .await; - app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; - - let result = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: "loop-user-4", - max_connections: 1, - soft_connections: 0, - client_ip: &new_fingerprint.client_ip, - request_addr: &new_fingerprint.addr, - use_session_admission: true, - session_token: Some("session-other"), - activate_unbound_session: false, - eviction_reentry_guard: EvictionReentryGuard::Session("session-other"), - }, - ) - .await; - let admission = result.admission; - let grace_mode = result.grace_mode; - - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(grace_mode, None); - assert!(app_state.active_users.active_streams().await.is_empty()); -} - -#[tokio::test] -async fn resolve_admission_with_strategies_allows_recently_evicted_playback_when_soft_slot_is_free() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 0, - grace_period_timeout_secs: 8, - grace_period_hold_stream: false, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::EvictUserOldest]), - }); - - let victim_addr: std::net::SocketAddr = "127.0.0.1:55187".parse().unwrap_or_else(|_| unreachable!()); - let reconnect_addr: std::net::SocketAddr = "127.0.0.1:55188".parse().unwrap_or_else(|_| unreachable!()); - let winner_addr: std::net::SocketAddr = "127.0.0.1:55189".parse().unwrap_or_else(|_| unreachable!()); - let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); - let reconnect_fingerprint = create_test_fingerprint_with_user_agent(reconnect_addr, "player/1.0"); - let winner_fingerprint = create_test_fingerprint_with_user_agent(winner_addr, "winner/1.0"); - let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9201.ts"); - victim_channel.virtual_id = 9201; - let mut winner_channel = create_test_live_channel("http://provider-1.example/live/9202.ts"); - winner_channel.virtual_id = 9202; - - app_state.connection_manager.add_connection(&victim_addr).await; - app_state.connection_manager.add_connection(&winner_addr).await; - - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 1, - username: "loop-user-3", - max_connections: 2, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &victim_fingerprint, - provider: "provider-a".intern(), - stream_channel: &victim_channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some("session-victim"), - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 2, - username: "loop-user-3", - max_connections: 2, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &winner_fingerprint, - provider: "provider-a".intern(), - stream_channel: &winner_channel, - user_agent: std::borrow::Cow::Borrowed("winner/1.0"), - session_token: Some("session-winner"), - }) - .await; - - app_state - .active_users - .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) - .await; - app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; - - let result = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: "loop-user-3", - max_connections: 1, - soft_connections: 1, - client_ip: &reconnect_fingerprint.client_ip, - request_addr: &reconnect_fingerprint.addr, - use_session_admission: true, - session_token: Some("socket-reconnect"), - activate_unbound_session: false, - eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9201) }, - }, - ) - .await; - let admission = result.admission; - let grace_mode = result.grace_mode; - - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); - assert_eq!(grace_mode, None); - - let active_streams = app_state.active_users.active_streams().await; - assert_eq!(active_streams.len(), 1); - assert_eq!(active_streams[0].channel.virtual_id, 9202); -} - -#[tokio::test] -async fn local_stream_response_registers_active_local_stream() { - let app_state = create_test_app_state(); - let temp_dir = tempfile::tempdir().expect("tempdir"); - let file_path = temp_dir.path().join("local-test.mkv"); - tokio::fs::write(&file_path, Bytes::from_static(b"local-stream")).await.expect("write local file"); - - let addr = "127.0.0.1:55123".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - let channel = create_test_local_channel(&format!("file://{}", file_path.display())); - let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; - let user = ProxyUserCredentials::default(); - let target = ConfigTarget { - curation: None, - id: 1, - enabled: true, - name: "test".to_string(), - options: None, - sort: None, - filter: Filter::default().into(), - output: Vec::new(), - rename: None, - mapping_ids: None, - mapping: Arc::new(ArcSwapOption::default()), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - }; - - let _response = local_stream_response( - &fingerprint, - &app_state, - channel, - &HeaderMap::default(), - &input, - &target, - &user, - UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Normal, - None, - None, - false, - ) - .await - .into_response(); - - let active_streams = app_state.active_users.active_streams().await; - assert_eq!(active_streams.len(), 1, "local file streaming should register an active stream"); - assert_eq!(active_streams[0].channel.item_type, PlaylistItemType::LocalVideo); -} - -#[tokio::test] -async fn local_stream_response_rechecks_limits_before_registering_socket_bound_streams() { - let mut app_cfg = create_test_app_config(); - let config = Config { user_access_control: true, ..Config::default() }; - app_cfg.config = Arc::new(ArcSwap::from_pointee(config)); - let app_state = create_test_app_state_for_config(Arc::new(app_cfg)); - let temp_dir = tempfile::tempdir().expect("tempdir"); - let file_path = temp_dir.path().join("local-race-test.mkv"); - tokio::fs::write(&file_path, Bytes::from_static(b"local-stream")).await.expect("write local file"); - - let first_addr = "127.0.0.1:55131".parse().unwrap_or_else(|_| unreachable!()); - let second_addr = "127.0.0.1:55132".parse().unwrap_or_else(|_| unreachable!()); - let first_fingerprint = create_test_fingerprint(first_addr); - let second_fingerprint = create_test_fingerprint(second_addr); - let channel = create_test_local_channel(&format!("file://{}", file_path.display())); - let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; - let mut user = ProxyUserCredentials::default(); - user.username = "local-limit-user".to_string(); - user.max_connections = 1; - let target = ConfigTarget { - curation: None, - id: 1, - enabled: true, - name: "test".to_string(), - options: None, - sort: None, - filter: Filter::default().into(), - output: Vec::new(), - rename: None, - mapping_ids: None, - mapping: Arc::new(ArcSwapOption::default()), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - }; - let first_token = create_session_fingerprint(&first_fingerprint, &user.username, channel.virtual_id, true); - let second_token = create_session_fingerprint(&second_fingerprint, &user.username, channel.virtual_id, true); - - let _first_response = local_stream_response( - &first_fingerprint, - &app_state, - channel.clone(), - &HeaderMap::default(), - &input, - &target, - &user, - UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Normal, - Some(&first_token), - None, - false, - ) - .await - .into_response(); - - let _second_response = local_stream_response( - &second_fingerprint, - &app_state, - channel, - &HeaderMap::default(), - &input, - &target, - &user, - UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Normal, - Some(&second_token), - None, - false, - ) - .await - .into_response(); - - assert_eq!(app_state.active_users.user_connections(&user.username).await, 1); - assert_eq!(app_state.active_users.active_streams().await.len(), 1); - assert_eq!( - app_state - .active_users - .connection_admission_for_session( - &user.username, - user.max_connections, - user.soft_connections, - &second_token - ) - .await - .permission(), - UserConnectionPermission::Exhausted, - "failed second open must not leave a placeholder session that bypasses admission" - ); -} - -#[tokio::test] -async fn stream_response_preserves_soft_kind_for_shared_reuse() { - let app_state = create_test_provider_app_state(); - let stream_url = "http://provider-1.example/live/shared.ts"; - let input_name = "provider_1".intern(); - let input = app_state.app_config.get_input_by_name(&input_name).expect("provider input should exist"); - let target = Arc::new(create_test_shared_target()); - - let owner_addr = "127.0.0.1:55140".parse().unwrap_or_else(|_| unreachable!()); - let owner_handle = app_state - .active_provider - .acquire_connection(&input.name, &owner_addr, 0, crate::api::model::ConnectionKind::Normal) - .expect("owner allocation should exist"); - let subscriber_id = - tuliprox_core::model::SharedSubscriberId::from_stream_uid(app_state.connection_manager.next_stream_uid()); - let pending_cleanup = - SharedStreamManager::reserve_subscriber_cleanup(&app_state.connection_manager, subscriber_id, owner_addr) - .await - .expect("shared cleanup admission should succeed"); - let shared_stream = stream::pending::>(); - let registered = SharedStreamManager::register_shared_stream( - SharedStreamCtx { - app_config: &app_state.app_config, - shared_stream_manager: &app_state.shared_stream_manager, - active_provider: &app_state.active_provider, - connection_manager: &app_state.connection_manager, - }, - stream_url, - shared_stream, - &owner_addr, - subscriber_id, - Vec::new(), - 1, - Some(tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), owner_handle)), - pending_cleanup, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .await; - assert!(registered.is_some(), "shared stream should register"); - - let mut user = ProxyUserCredentials::default(); - user.username = "soft-user".to_string(); - user.max_connections = 1; - user.soft_connections = 1; - user.priority = 0; - user.soft_priority = 9; - - let normal_addr = "127.0.0.1:55141".parse().unwrap_or_else(|_| unreachable!()); - let normal_fingerprint = create_test_fingerprint(normal_addr); - let normal_channel = create_test_live_channel("http://provider-1.example/live/normal.ts"); - app_state.active_users.add_connection(&normal_addr).await; - app_state - .active_users - .update_connection(crate::api::model::ActiveUserConnectionParams { - uid: 1001, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: user.priority, - soft_priority: user.soft_priority, - fingerprint: &normal_fingerprint, - provider: input.name.clone(), - stream_channel: &normal_channel, - user_agent: Cow::Borrowed("ua"), - session_token: Some("normal-session"), - }) - .await - .expect("normal stream should register"); - - let admission = - app_state.active_users.connection_admission(&user.username, user.max_connections, user.soft_connections).await; - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); - - let soft_addr = "127.0.0.1:55142".parse().unwrap_or_else(|_| unreachable!()); - let soft_fingerprint = create_test_fingerprint(soft_addr); - let response = stream_response( - &soft_fingerprint, - &app_state, - "soft-session", - None, - create_test_live_channel(stream_url), - stream_url, - None, - &HeaderMap::default(), - &input, - &target, - &user, - admission.permission(), - admission.kind().unwrap_or(crate::api::model::ConnectionKind::Normal), - false, - None, - ) - .await - .into_response(); - assert_eq!(response.status(), StatusCode::OK); - - let session_admission = app_state - .active_users - .connection_admission_for_session(&user.username, user.max_connections, user.soft_connections, "soft-session") - .await; - assert_eq!(session_admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); -} - -#[tokio::test] -async fn failed_provider_open_preserves_other_allocation_on_same_socket() -> Result<(), Box> { - let listener = TcpListener::bind("127.0.0.1:0").await?; - let upstream = listener.local_addr()?; - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await?; - socket.write_all(b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await - }); - let mut config = create_test_provider_app_config(); - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - enabled: true, - input_type: InputType::Xtream, - url: format!("http://{upstream}"), - username: Some("user1".to_string()), - password: Some("pass1".to_string()), - max_connections: 2, - ..ConfigInput::default() - }); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app = create_test_app_state_for_config(Arc::new(config)); - let addr = "127.0.0.1:55144".parse()?; - let live = app - .active_provider - .acquire_connection(&input.name, &addr, 0, crate::api::model::ConnectionKind::Normal) - .ok_or("live allocation missing")?; - let url = format!("http://{upstream}/live/user1/pass1/100.ts"); - let channel = create_test_live_channel(&url); - let details = create_stream_response_details( - &app, - &get_stream_options(&app.app_config, StreamResponseMode::Stream), - &url, - "failing-user", - &create_test_fingerprint(addr), - &HeaderMap::new(), - &input, - &channel, - PlaylistItemType::Live, - crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, - false, - UserConnectionPermission::Allowed, - None, - true, - false, - VirtualId::new(channel.virtual_id), - 0, - crate::api::model::ConnectionKind::Normal, - false, - Some("failed-session"), - None, - false, - None, - None, - None, - None, - ) - .await?; - assert!(details.provider_handle.is_none()); - assert_eq!(app.active_provider.get_provider_connections_count(), 1); - assert!(!live.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); - app.active_provider.release_handle(&live); - server.await??; - Ok(()) -} - -#[tokio::test] -async fn recording_provider_reservation_opens_without_second_slot() -> Result<(), Box> { - let listener = TcpListener::bind("127.0.0.1:0").await?; - let upstream = listener.local_addr()?; - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await?; - socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\ndata").await - }); - let mut config = create_test_provider_app_config(); - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - enabled: true, - input_type: InputType::Xtream, - url: format!("http://{upstream}"), - username: Some("user1".to_string()), - password: Some("pass1".to_string()), - max_connections: 1, - ..ConfigInput::default() - }); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app = create_test_app_state_for_config(Arc::new(config)); - let reserved = - app.active_provider.acquire_connection_for_download(&input.name, 0).ok_or("recording reservation missing")?; - let allocation_id = reserved.allocation_id; - let claimed = app - .active_provider - .claim_download_connection(allocation_id, &input.name) - .ok_or("recording reservation claim missing")?; - let managed = tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app.active_provider), claimed); - let addr = "127.0.0.1:55145".parse()?; - let url = format!("http://{upstream}/live/user1/pass1/100.ts"); - let channel = create_test_live_channel(&url); - let details = create_stream_response_details( - &app, - &get_stream_options(&app.app_config, StreamResponseMode::Stream), - &url, - "recording-user", - &create_test_fingerprint(addr), - &HeaderMap::new(), - &input, - &channel, - PlaylistItemType::Live, - crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, - false, - UserConnectionPermission::Allowed, - None, - true, - false, - VirtualId::new(channel.virtual_id), - 0, - crate::api::model::ConnectionKind::Normal, - false, - Some("recording-session"), - None, - false, - None, - None, - None, - Some(managed), - ) - .await?; - - assert!(details.stream.is_some(), "the claimed reservation must open the provider body"); - assert!(details.custom_reason.is_none(), "the request must not fall back to ProviderExhausted"); - assert_eq!(app.active_provider.get_provider_connections_count(), 1, "recording consumes exactly one slot"); - - drop(details); - app.active_provider.complete_release(allocation_id); - server.await??; - Ok(()) -} - -#[tokio::test] -async fn ts_direct_capacity_spreads_across_priority_groups_then_rejects_sixth() { - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_a".intern(), - enabled: true, - input_type: InputType::Xtream, - url: "http://127.0.0.1:1".to_string(), - username: Some("user-a".to_string()), - password: Some("pass-a".to_string()), - priority: 0, - max_connections: 2, - aliases: Some(vec![ConfigInputAlias { - id: 2, - name: "provider_b".intern(), - url: "http://127.0.0.1:2".to_string(), - username: Some("user-b".to_string()), - password: Some("pass-b".to_string()), - priority: 1, - max_connections: 3, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }); - let mut config = create_test_provider_app_config(); - config.sources = - Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); - let app = create_test_app_state_for_config(Arc::new(config)); - - let addr: SocketAddr = "127.0.0.1:55120".parse().unwrap_or_else(|_| unreachable!()); - let mut handles = Vec::new(); - for _ in 0..5 { - let handle = app.active_provider.acquire_connection_with_grace( - &input.name, - &addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ); - assert!(handle.is_some(), "five allocations must succeed against A(2)+B(3)"); - handles.push(handle.expect("allocation present")); - } - - let sixth = app.active_provider.acquire_connection_with_grace( - &input.name, - &addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ); - assert!(sixth.is_none(), "sixth start must be rejected once A(2)+B(3) are exhausted"); - - let active = app.active_provider.active_connections().expect("active connection map present"); - assert_eq!(active.get("provider_a").copied(), Some(5), "multi lineup must hold five allocations in total"); - - let capacities = app.active_provider.provider_capacities_for_input(&input.name); - let current_of = |name: &str| capacities.iter().find(|(n, _, _)| n.as_ref() == name).map(|(_, cur, _)| *cur); - assert_eq!(current_of("provider_a"), Some(2), "priority group A must hold two allocations"); - assert_eq!(current_of("provider_b"), Some(3), "priority group B must hold three allocations"); - - for handle in &handles { - app.active_provider.release_handle(handle); - } - assert_eq!(app.active_provider.get_provider_connections_count(), 0); -} - -#[tokio::test] -async fn stream_response_rolls_back_provisional_user_activation_when_provider_open_fails() { - let mut app_cfg = create_test_provider_app_config(); - app_cfg.config = Arc::new(ArcSwap::from_pointee(Config { - user_access_control: true, - custom_stream_response_enabled: true, - ..Config::default() - })); - let app_state = create_test_app_state_for_config(Arc::new(app_cfg)); - let addr = "127.0.0.1:55143".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - let input_name = "provider_1".intern(); - let input = app_state.app_config.get_input_by_name(&input_name).expect("provider input should exist"); - let target = Arc::new(ConfigTarget { - curation: None, - id: 1, - enabled: true, - name: "test".to_string(), - options: None, - sort: None, - filter: Filter::default().into(), - output: Vec::new(), - rename: None, - mapping_ids: None, - mapping: Arc::new(ArcSwapOption::default()), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - }); - let mut user = ProxyUserCredentials::default(); - user.username = "rollback-user".to_string(); - user.max_connections = 1; - let stream_url = "provider://bad-url"; - let channel = create_test_live_channel(stream_url); - - let response = stream_response( - &fingerprint, - &app_state, - "rollback-session", - None, - channel, - stream_url, - None, - &HeaderMap::default(), - &input, - &target, - &user, - UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Normal, - false, - None, - ) - .await - .into_response(); - - // Custom-video stream is enabled (`custom_stream_response_enabled: true` - // in this fixture), so a missing resource must return 400 — the - // Nginx `proxy_intercept_errors on;` contract requires 4xx so the - // socket is severed instead of looping on a 200 OK fallback body. - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - assert_eq!( - app_state.active_users.user_connections(&user.username).await, - 0, - "failed provider open must rollback provisional user activation" - ); - assert!( - app_state.active_users.get_and_update_user_session(&user.username, "rollback-session").await.is_none(), - "failed provider open must remove the provisional placeholder session" - ); -} - -/// Regression test for: when a catchup request fails upstream (e.g. provider returns -/// 4xx/5xx) the connection-slot is released, but the provider account was being -/// pinned via `refresh_provider_reservation` for `catchup_session_ttl_secs`. This -/// blocked other sessions of the same family from acquiring the same provider even -/// though the slot was already free. The fix delegates the pinning decision to -/// `should_pin_provider_for_session` and skips the reservation when the response -/// is a non-Provisioning custom video (failure fallback). Provisioning custom videos -/// must keep their reservation since they represent a successful provider handoff. -#[tokio::test] -async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_video() { - let app_state = create_test_app_state(); - let no_video_details = StreamDetails { - shared_subscriber_id: None, - stream: None, - stream_info: Some((Vec::new(), StatusCode::OK, None, None)), - provider_name: Some("provider_1".intern()), - request_url: None, - session_headers: None, - provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), - user_agent_stream_index: None, - grace_period: GracePeriodOptions::default(), - provider_grace_active: false, - disable_provider_grace: false, - reconnect_flag: None, - provider_handle: None, - content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, - grace_resolution_context: None, - custom_reason: None, - response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), - session_registration: None, - }; - assert!( - should_pin_provider_for_session(&no_video_details, &app_state, PlaylistItemType::Catchup), - "a real provider stream (no CustomVideoStreamType) must pin the provider" - ); - - let provisioning_details = StreamDetails { - shared_subscriber_id: None, - stream: None, - stream_info: Some((Vec::new(), StatusCode::OK, None, Some(CustomVideoStreamType::Provisioning))), - provider_name: Some("provider_1".intern()), - request_url: None, - session_headers: None, - provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), - user_agent_stream_index: None, - grace_period: GracePeriodOptions::default(), - provider_grace_active: false, - disable_provider_grace: false, - reconnect_flag: None, - provider_handle: None, - content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, - grace_resolution_context: None, - custom_reason: None, - response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), - session_registration: None, - }; - assert!( - should_pin_provider_for_session(&provisioning_details, &app_state, PlaylistItemType::Catchup), - "a Provisioning custom video represents a successful provider handoff and must pin" - ); - - for failure_type in [ - CustomVideoStreamType::ChannelUnavailable, - CustomVideoStreamType::ProviderConnectionsExhausted, - CustomVideoStreamType::UserConnectionsExhausted, - CustomVideoStreamType::UserAccountExpired, - CustomVideoStreamType::LowPriorityPreempted, - ] { - let failure_details = StreamDetails { - shared_subscriber_id: None, - stream: None, - stream_info: Some((Vec::new(), StatusCode::BAD_REQUEST, None, Some(failure_type))), - provider_name: Some("provider_1".intern()), - request_url: None, - session_headers: None, - provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), - user_agent_stream_index: None, - grace_period: GracePeriodOptions::default(), - provider_grace_active: false, - disable_provider_grace: false, - reconnect_flag: None, - provider_handle: None, - content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, - grace_resolution_context: None, - custom_reason: None, - response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), - session_registration: None, - }; - assert!( - !should_pin_provider_for_session(&failure_details, &app_state, PlaylistItemType::Catchup), - "{failure_type:?} is a failure fallback — must NOT pin the provider" - ); - } -} - -#[tokio::test] -async fn local_stream_response_disables_response_compression() { - let app_state = create_test_app_state(); - let temp_dir = tempfile::tempdir().expect("tempdir"); - let file_path = temp_dir.path().join("local-test.mkv"); - tokio::fs::write(&file_path, Bytes::from_static(b"local-stream")).await.expect("write local file"); - - let addr = "127.0.0.1:55124".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint(addr); - let channel = create_test_local_channel(&format!("file://{}", file_path.display())); - let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; - let user = ProxyUserCredentials::default(); - let target = ConfigTarget { - curation: None, - id: 1, - enabled: true, - name: "test".to_string(), - options: None, - sort: None, - filter: Filter::default().into(), - output: Vec::new(), - rename: None, - mapping_ids: None, - mapping: Arc::new(ArcSwapOption::default()), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - }; - - let response = local_stream_response( - &fingerprint, - &app_state, - channel, - &HeaderMap::default(), - &input, - &target, - &user, - UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Normal, - None, - None, - false, - ) - .await - .into_response(); - - assert!(!should_compress_response(&response)); -} - -#[tokio::test] -async fn local_stream_response_reuses_stable_playback_session_token_across_reopens() { - let app_state = create_test_app_state(); - let temp_dir = tempfile::tempdir().expect("tempdir"); - let file_path = temp_dir.path().join("local-test.mkv"); - tokio::fs::write(&file_path, Bytes::from_static(b"local-stream")).await.expect("write local file"); - - let channel = create_test_local_channel(&format!("file://{}", file_path.display())); - let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; - let user = ProxyUserCredentials::default(); - let target = ConfigTarget { - curation: None, - id: 1, - enabled: true, - name: "test".to_string(), - options: None, - sort: None, - filter: Filter::default().into(), - output: Vec::new(), - rename: None, - mapping_ids: None, - mapping: Arc::new(ArcSwapOption::default()), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - }; - let playback_session_token = "local-playback-token"; - - let first_fingerprint = create_test_fingerprint("127.0.0.1:55125".parse().unwrap_or_else(|_| unreachable!())); - let second_fingerprint = create_test_fingerprint("127.0.0.1:55126".parse().unwrap_or_else(|_| unreachable!())); - - let _first_response = local_stream_response( - &first_fingerprint, - &app_state, - channel.clone(), - &HeaderMap::default(), - &input, - &target, - &user, - UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Normal, - Some(playback_session_token), - None, - false, - ) - .await - .into_response(); - - let _second_response = local_stream_response( - &second_fingerprint, - &app_state, - channel, - &HeaderMap::default(), - &input, - &target, - &user, - UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Normal, - Some(playback_session_token), - None, - false, - ) - .await - .into_response(); - - let active_streams = app_state.active_users.active_streams().await; - assert_eq!(active_streams.len(), 1, "stable playback token should reuse the tracked local connection"); - assert_eq!(active_streams[0].session_token.as_deref(), Some(playback_session_token)); - assert_eq!(active_streams[0].addr, second_fingerprint.addr); -} - -#[tokio::test] -async fn local_stream_response_allows_exhausted_reopen_for_same_playback_session_token() { - let app_state = create_test_app_state(); - let temp_dir = tempfile::tempdir().expect("tempdir"); - let file_path = temp_dir.path().join("local-test.mkv"); - tokio::fs::write(&file_path, Bytes::from_static(b"local-stream")).await.expect("write local file"); - - let channel = create_test_local_channel(&format!("file://{}", file_path.display())); - let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; - let mut user = ProxyUserCredentials::default(); - user.username = "user1".to_string(); - user.max_connections = 1; - let target = ConfigTarget { - curation: None, - id: 1, - enabled: true, - name: "test".to_string(), - options: None, - sort: None, - filter: Filter::default().into(), - output: Vec::new(), - rename: None, - mapping_ids: None, - mapping: Arc::new(ArcSwapOption::default()), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - }; - let playback_session_token = "local-playback-token"; - - let first_fingerprint = create_test_fingerprint("127.0.0.1:55127".parse().unwrap_or_else(|_| unreachable!())); - let second_fingerprint = create_test_fingerprint("127.0.0.1:55128".parse().unwrap_or_else(|_| unreachable!())); - - let _first_response = local_stream_response( - &first_fingerprint, - &app_state, - channel.clone(), - &HeaderMap::default(), - &input, - &target, - &user, - UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Normal, - Some(playback_session_token), - None, - false, - ) - .await - .into_response(); - - let second_response = local_stream_response( - &second_fingerprint, - &app_state, - channel, - &HeaderMap::default(), - &input, - &target, - &user, - UserConnectionPermission::Exhausted, - crate::api::model::ConnectionKind::Normal, - Some(playback_session_token), - None, - false, - ) - .await - .into_response(); - - assert_eq!(second_response.status(), StatusCode::OK); - - let active_streams = app_state.active_users.active_streams().await; - assert_eq!(active_streams.len(), 1); - assert_eq!(active_streams[0].session_token.as_deref(), Some(playback_session_token)); - assert_eq!(active_streams[0].addr, second_fingerprint.addr); -} - -#[tokio::test] -async fn local_stream_response_preserves_soft_kind_across_reopens() { - let app_state = create_test_app_state(); - let temp_dir = tempfile::tempdir().expect("tempdir"); - let file_path = temp_dir.path().join("local-soft-test.mkv"); - tokio::fs::write(&file_path, Bytes::from_static(b"local-stream")).await.expect("write local file"); - - let channel = create_test_local_channel(&format!("file://{}", file_path.display())); - let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; - let mut user = ProxyUserCredentials::default(); - user.username = "soft-local-user".to_string(); - user.max_connections = 1; - user.soft_connections = 1; - let target = ConfigTarget { - curation: None, - id: 1, - enabled: true, - name: "test".to_string(), - options: None, - sort: None, - filter: Filter::default().into(), - output: Vec::new(), - rename: None, - mapping_ids: None, - mapping: Arc::new(ArcSwapOption::default()), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - }; - let playback_session_token = "local-soft-playback-token"; - - let first_fingerprint = create_test_fingerprint("127.0.0.1:55129".parse().unwrap_or_else(|_| unreachable!())); - let second_fingerprint = create_test_fingerprint("127.0.0.1:55130".parse().unwrap_or_else(|_| unreachable!())); - - let _first_response = local_stream_response( - &first_fingerprint, - &app_state, - channel.clone(), - &HeaderMap::default(), - &input, - &target, - &user, - UserConnectionPermission::Allowed, - crate::api::model::ConnectionKind::Soft, - Some(playback_session_token), - None, - false, - ) - .await - .into_response(); - - let second_response = local_stream_response( - &second_fingerprint, - &app_state, - channel, - &HeaderMap::default(), - &input, - &target, - &user, - UserConnectionPermission::Exhausted, - crate::api::model::ConnectionKind::Normal, - Some(playback_session_token), - None, - false, - ) - .await - .into_response(); - - assert_eq!(second_response.status(), StatusCode::OK); - - let session_admission = app_state - .active_users - .connection_admission_for_session( - &user.username, - user.max_connections, - user.soft_connections, - playback_session_token, - ) - .await; - assert_eq!(session_admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); -} - -#[tokio::test] -async fn activated_session_admission_keeps_hls_placeholders_uncounted_via_api_utils() { - let app_state = create_test_app_state(); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - user.max_connections = 1; - - let first_addr: std::net::SocketAddr = "127.0.0.1:55177".parse().unwrap_or_else(|_| unreachable!()); - let second_addr: std::net::SocketAddr = "127.0.0.1:55178".parse().unwrap_or_else(|_| unreachable!()); - let first_fingerprint = create_test_fingerprint(first_addr); - let second_fingerprint = create_test_fingerprint(second_addr); - - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: "tok-hls-first", - virtual_id: 7101, - provider: "provider-a", - stream_url: "http://provider-1.example/live/7101.m3u8", - addr: &first_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: "tok-hls-second", - virtual_id: 7102, - provider: "provider-a", - stream_url: "http://provider-1.example/live/7102.m3u8", - addr: &second_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let first_admission = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - client_ip: &first_fingerprint.client_ip, - request_addr: &first_fingerprint.addr, - use_session_admission: true, - session_token: Some("tok-hls-first"), - activate_unbound_session: true, - eviction_reentry_guard: EvictionReentryGuard::Session("tok-hls-first"), - }, - ) - .await; - let second_admission = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - client_ip: &second_fingerprint.client_ip, - request_addr: &second_fingerprint.addr, - use_session_admission: true, - session_token: Some("tok-hls-second"), - activate_unbound_session: true, - eviction_reentry_guard: EvictionReentryGuard::Session("tok-hls-second"), - }, - ) - .await; - - assert_eq!(first_admission.admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(second_admission.admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(app_state.active_users.user_connections(&user.username).await, 0); -} - -#[tokio::test] -async fn socket_bound_playback_sessions_enforce_hard_limits_per_socket() { - let app_state = create_test_app_state(); - let mut user = ProxyUserCredentials::default(); - user.username = "user1".to_string(); - user.max_connections = 1; - - let first_addr: std::net::SocketAddr = "127.0.0.1:55171".parse().unwrap_or_else(|_| unreachable!()); - let second_addr: std::net::SocketAddr = "127.0.0.1:55172".parse().unwrap_or_else(|_| unreachable!()); - let first_fingerprint = create_test_fingerprint(first_addr); - let first_token = create_session_fingerprint(&first_fingerprint, &user.username, 5001, true); - let second_fingerprint = create_test_fingerprint(second_addr); - let second_token = create_session_fingerprint(&second_fingerprint, &user.username, 5001, true); - - app_state.connection_manager.add_connection(&first_addr).await; - app_state.connection_manager.add_connection(&second_addr).await; - - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: &first_token, - virtual_id: 5001, - provider: "provider-a", - stream_url: "http://provider-1.example/vod/5001.ts", - addr: &first_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: true, - }) - .await; - - app_state - .active_users - .update_connection(crate::api::model::ActiveUserConnectionParams { - uid: 5001, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &first_fingerprint, - provider: "provider-a".intern(), - stream_channel: &create_test_live_channel("http://provider-1.example/vod/5001.ts"), - user_agent: std::borrow::Cow::Borrowed("ua"), - session_token: Some(&first_token), - }) - .await; - - let admission = app_state - .active_users - .connection_admission_for_session(&user.username, user.max_connections, user.soft_connections, &second_token) - .await; - assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn resolve_admission_with_strategies_evicts_preserved_hls_session_for_same_user_ts_request() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![ - AdmissionStrategy::EvictUserSameIpOldest, - AdmissionStrategy::EvictUserSameIpLatest, - AdmissionStrategy::GraceHoldStream, - AdmissionStrategy::EvictUserOldest, - AdmissionStrategy::EvictUserLatest, - ]), - }); - - let hls_addr: std::net::SocketAddr = "127.0.0.1:55176".parse().unwrap_or_else(|_| unreachable!()); - let ts_addr: std::net::SocketAddr = "127.0.0.1:55177".parse().unwrap_or_else(|_| unreachable!()); - let hls_fingerprint = create_test_fingerprint_with_user_agent(hls_addr, "player/1.0"); - let ts_fingerprint = create_test_fingerprint_with_user_agent(ts_addr, "player/1.0"); - let mut user = ProxyUserCredentials::default(); - user.username = "same-user".to_string(); - user.max_connections = 1; - - app_state.connection_manager.add_connection(&hls_addr).await; - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: "tok-hls-preserved", - virtual_id: 5001, - provider: "provider-a", - stream_url: "http://provider-1.example/live/5001.m3u8", - addr: &hls_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 1, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &hls_fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { - item_type: PlaylistItemType::LiveHls, - virtual_id: 5001, - ..create_test_live_channel("http://provider-1.example/live/5001.m3u8") - }, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some("tok-hls-preserved"), - }) - .await; - - app_state.connection_manager.release_connection(&hls_addr).await; - assert_eq!(app_state.active_users.user_connections(&user.username).await, 0); - assert!(app_state.active_users.active_streams().await.is_empty()); - - let mut close_rx = app_state.connection_manager.get_close_connection_channel(); - let result = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - client_ip: &ts_fingerprint.client_ip, - request_addr: &ts_fingerprint.addr, - use_session_admission: false, - session_token: None, - activate_unbound_session: false, - eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(5001) }, - }, - ) - .await; - let admission = result.admission; - let grace_mode = result.grace_mode; - - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(grace_mode, None); - assert!(app_state.active_users.active_streams().await.is_empty()); - assert_eq!( - tokio::time::timeout(std::time::Duration::from_millis(100), close_rx.recv()).await.ok().and_then(Result::ok), - Some(crate::api::model::CloseConnectionSignal::WithReason( - hls_addr, - shared::model::DisconnectReason::ClientKicked, - )) - ); - assert!( - app_state.active_users.get_and_update_user_session(&user.username, "tok-hls-preserved").await.is_none(), - "preserved session should be removed once the TS request evicts it" - ); -} - -#[tokio::test] -async fn socket_bound_playback_sessions_still_allow_soft_slots() { - let app_state = create_test_app_state(); - let mut user = ProxyUserCredentials::default(); - user.username = "soft-user".to_string(); - user.max_connections = 1; - user.soft_connections = 1; - user.priority = 0; - user.soft_priority = 9; - - let first_addr: std::net::SocketAddr = "127.0.0.1:55173".parse().unwrap_or_else(|_| unreachable!()); - let second_addr: std::net::SocketAddr = "127.0.0.1:55174".parse().unwrap_or_else(|_| unreachable!()); - let third_addr: std::net::SocketAddr = "127.0.0.1:55175".parse().unwrap_or_else(|_| unreachable!()); - let first_fingerprint = create_test_fingerprint(first_addr); - let second_fingerprint = create_test_fingerprint(second_addr); - let first_token = create_session_fingerprint(&first_fingerprint, &user.username, 6001, true); - let second_token = create_session_fingerprint(&second_fingerprint, &user.username, 6001, true); - let third_fingerprint = create_test_fingerprint(third_addr); - let third_token = create_session_fingerprint(&third_fingerprint, &user.username, 6001, true); - - app_state.connection_manager.add_connection(&first_addr).await; - app_state.connection_manager.add_connection(&second_addr).await; - - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: &first_token, - virtual_id: 6001, - provider: "provider-a", - stream_url: "http://provider-1.example/vod/6001.ts", - addr: &first_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: true, - }) - .await; - app_state - .active_users - .update_connection(crate::api::model::ActiveUserConnectionParams { - uid: 6001, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: user.priority, - soft_priority: user.soft_priority, - fingerprint: &first_fingerprint, - provider: "provider-a".intern(), - stream_channel: &create_test_live_channel("http://provider-1.example/vod/6001.ts"), - user_agent: std::borrow::Cow::Borrowed("ua"), - session_token: Some(&first_token), - }) - .await; - - let second_admission = app_state - .active_users - .connection_admission_for_session(&user.username, user.max_connections, user.soft_connections, &second_token) - .await; - assert_eq!(second_admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(second_admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); - - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: &second_token, - virtual_id: 6001, - provider: "provider-a", - stream_url: "http://provider-1.example/vod/6001.ts", - addr: &second_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Soft), - socket_bound: true, - }) - .await; - app_state - .active_users - .update_connection(crate::api::model::ActiveUserConnectionParams { - uid: 6002, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: crate::api::model::ConnectionKind::Soft, - priority: user.priority, - soft_priority: user.soft_priority, - fingerprint: &second_fingerprint, - provider: "provider-a".intern(), - stream_channel: &create_test_live_channel("http://provider-1.example/vod/6002.ts"), - user_agent: std::borrow::Cow::Borrowed("ua"), - session_token: Some(&second_token), - }) - .await; - - let third_admission = app_state - .active_users - .connection_admission_for_session(&user.username, user.max_connections, user.soft_connections, &third_token) - .await; - assert_eq!(third_admission.permission(), UserConnectionPermission::Exhausted); -} - -#[test] -fn session_based_playback_matches_adaptive_types_and_extensions() { - assert!(is_session_based_playback(PlaylistItemType::LiveHls, None)); - assert!(is_session_based_playback(PlaylistItemType::LiveDash, None)); - assert!(is_session_based_playback(PlaylistItemType::Live, Some(HLS_EXT))); - assert!(is_session_based_playback(PlaylistItemType::Live, Some(DASH_EXT))); - assert!(!is_session_based_playback(PlaylistItemType::Video, None)); -} - -#[test] -fn create_session_fingerprint_switches_between_logical_and_socket_bound_keys() { - let fingerprint = create_test_fingerprint("127.0.0.1:55176".parse().unwrap_or_else(|_| unreachable!())); - let logical = create_session_fingerprint(&fingerprint, "user1", 7001, false); - let socket_bound = create_session_fingerprint(&fingerprint, "user1", 7001, true); - - assert_ne!(logical, socket_bound); - assert!(logical.contains(&fingerprint.key)); - assert!(socket_bound.contains(&fingerprint.addr.to_string())); -} - -#[test] -fn adaptive_playback_session_fingerprint_is_logical_across_initial_sockets() { - let Some(first_addr) = "127.0.0.1:55177".parse().ok() else { - return; - }; - let Some(second_addr) = "127.0.0.1:55178".parse().ok() else { - return; - }; - let first = Fingerprint::new("10.0.0.6|player".to_string(), "10.0.0.6".to_string(), first_addr); - let second = Fingerprint::new(first.key.clone(), first.client_ip.clone(), second_addr); - - let first_token = create_playback_session_fingerprint(&first, "user1", 7002, PlaylistItemType::Live, Some(HLS_EXT)); - let second_token = - create_playback_session_fingerprint(&second, "user1", 7002, PlaylistItemType::Live, Some(HLS_EXT)); - - assert_eq!(first_token, second_token); - assert!(first_token.contains(&first.key)); - assert!(!first_token.contains(&first.addr.to_string())); - assert!(!second_token.contains(&second.addr.to_string())); -} - -#[test] -fn playback_session_fingerprint_keeps_ts_socket_bound_but_vod_logical() { - let first_addr: SocketAddr = "127.0.0.1:55179".parse().unwrap_or_else(|_| unreachable!()); - let second_addr: SocketAddr = "127.0.0.1:55180".parse().unwrap_or_else(|_| unreachable!()); - let first = Fingerprint::new("10.0.0.7|player".to_string(), "10.0.0.7".to_string(), first_addr); - let second = Fingerprint::new(first.key.clone(), first.client_ip.clone(), second_addr); - - let first_ts = create_playback_session_fingerprint(&first, "user1", 7003, PlaylistItemType::Live, None); - let second_ts = create_playback_session_fingerprint(&second, "user1", 7003, PlaylistItemType::Live, None); - let first_vod = create_playback_session_fingerprint(&first, "user1", 7003, PlaylistItemType::Video, None); - let second_vod = create_playback_session_fingerprint(&second, "user1", 7003, PlaylistItemType::Video, None); - - assert_ne!(first_ts, second_ts, "plain TS live remains socket-bound"); - assert_eq!(first_vod, second_vod, "VOD remains logical across reopen/seek sockets"); -} - -#[test] -fn catchup_session_key_is_sticky_only_within_the_same_archive_window() { - let addr: SocketAddr = "127.0.0.1:55181".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = Fingerprint::new("10.0.0.8|player".to_string(), "10.0.0.8".to_string(), addr); - - let first = create_catchup_session_key(&fingerprint, "user1", 7004, "/timeshift/3600/1700000000/"); - let same = create_catchup_session_key(&fingerprint, "user1", 7004, "timeshift/3600/1700000000"); - let seeked = create_catchup_session_key(&fingerprint, "user1", 7004, "timeshift/3600/1700000300"); - let resized = create_catchup_session_key(&fingerprint, "user1", 7004, "timeshift/1800/1700000000"); - - assert_eq!(first, same, "cosmetic path separators must not break provider stickiness"); - assert_ne!(first, seeked, "seeking must identify the new archive window"); - assert_ne!(first, resized, "changing duration must identify the new archive window"); -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn xtream_hls_then_ts_uses_distinct_tokens_and_evicts_old_hls_session() { - let mut app_cfg = create_test_app_config(); - let config = Config { - user_access_control: true, - reverse_proxy: Some(crate::model::ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: crate::model::ResourceRetryConfig::default(), - disabled_header: None, - stream: Some(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 2_000, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![ - AdmissionStrategy::EvictUserSameIpOldest, - AdmissionStrategy::EvictUserSameIpLatest, - AdmissionStrategy::GraceHoldStream, - AdmissionStrategy::EvictUserOldest, - AdmissionStrategy::EvictUserLatest, - ]), - }), - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }), - ..Config::default() - }; - app_cfg.config = Arc::new(ArcSwap::from_pointee(config)); - let app_state = create_test_app_state_for_config(Arc::new(app_cfg)); - let hls_addr: SocketAddr = "127.0.0.1:55186".parse().unwrap_or_else(|_| unreachable!()); - let ts_addr: SocketAddr = "127.0.0.1:55187".parse().unwrap_or_else(|_| unreachable!()); - let hls_fingerprint = create_test_fingerprint_with_user_agent(hls_addr, "libmpv"); - let ts_fingerprint = create_test_fingerprint_with_user_agent(ts_addr, "libmpv"); - let mut user = ProxyUserCredentials::default(); - user.username = "xtream-hls-ts".to_string(); - user.max_connections = 1; - - let virtual_id = 7811; - let hls_token = create_session_fingerprint(&hls_fingerprint, &user.username, virtual_id, false); - let ts_token = create_session_fingerprint(&ts_fingerprint, &user.username, virtual_id, true); - assert_ne!(hls_token, ts_token, "Xtream .m3u8 and .ts must not share the same playback token"); - - let mut hls_channel = create_test_live_channel("http://provider-1.example/live/7811.m3u8"); - hls_channel.virtual_id = virtual_id; - hls_channel.item_type = PlaylistItemType::LiveHls; - let mut ts_channel = create_test_live_channel("http://provider-1.example/live/7811.ts"); - ts_channel.virtual_id = virtual_id; - - app_state.connection_manager.add_connection(&hls_addr).await; - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: &hls_token, - virtual_id, - provider: "provider_1", - stream_url: hls_channel.url.as_ref(), - addr: &hls_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 1, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &hls_fingerprint, - provider: "provider_1".intern(), - stream_channel: &hls_channel, - user_agent: Cow::Borrowed("libmpv"), - session_token: Some(&hls_token), - }) - .await; - - app_state.connection_manager.release_connection(&hls_addr).await; - assert_eq!( - app_state.active_users.user_connections(&user.username).await, - 0, - "the preserved HLS playback must reserve capacity only virtually" - ); - assert_eq!(app_state.active_users.active_users_and_connections().await, (0, 0)); - assert!( - app_state.active_users.get_and_update_user_session(&user.username, &hls_token).await.is_some(), - "preserved HLS session should still exist before the competing TS request" - ); - assert_eq!( - app_state - .active_users - .connection_admission(&user.username, user.max_connections, user.soft_connections) - .await - .permission(), - UserConnectionPermission::Exhausted, - "the preserved HLS playback must still reserve the user's only slot before the TS request is evaluated" - ); - assert_eq!( - app_state.active_users.get_eviction_candidates(&user.username, &ts_fingerprint.client_ip).await.len(), - 1, - "the preserved HLS playback should be the single eviction candidate for the competing TS request" - ); - - let (ts_admission, ts_grace_mode, request_class) = resolve_playback_request_admission( - &app_state.admission_ctx(), - &user, - &ts_fingerprint, - None, - &ts_token, - false, - EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(virtual_id) }, - false, - false, - ) - .await; - assert_eq!(request_class, PlaybackRequestClass::Activate); - assert_eq!(ts_admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(ts_grace_mode, None); - assert!( - app_state - .active_users - .get_and_update_user_session(&user.username, &hls_token) - .await - .is_none(), - "the competing TS activation must remove the old preserved HLS session even though there is no live socket left to kick" - ); - assert_eq!( - app_state.active_users.user_connections(&user.username).await, - 0, - "eviction must not leave a real slot before the TS stream commits" - ); - assert_eq!(app_state.active_users.active_users_and_connections().await, (0, 0)); - - app_state.connection_manager.add_connection(&ts_addr).await; - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: &ts_token, - virtual_id, - provider: "provider_1", - stream_url: ts_channel.url.as_ref(), - addr: &ts_addr, - connection_permission: ts_admission.permission(), - connection_kind: ts_admission.kind(), - socket_bound: true, - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 2, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &ts_fingerprint, - provider: "provider_1".intern(), - stream_channel: &ts_channel, - user_agent: Cow::Borrowed("libmpv"), - session_token: Some(&ts_token), - }) - .await; - - assert_eq!(app_state.active_users.user_connections(&user.username).await, 1); - assert_eq!(app_state.active_users.active_users_and_connections().await, (1, 1)); - let active_streams = app_state.active_users.active_streams().await; - assert_eq!(active_streams.len(), 1); - assert_eq!(active_streams.first().and_then(|stream| stream.session_token.as_deref()), Some(ts_token.as_str())); - assert!( - app_state - .active_users - .get_and_update_user_session(&user.username, &hls_token) - .await - .is_none(), - "after the competing TS request, the old Xtream HLS session must be gone so later /hls segment fetches cannot revive it" - ); - assert!( - app_state.active_users.get_and_update_user_session(&user.username, &ts_token).await.is_some(), - "the winning TS playback should remain tracked under its socket-bound Xtream token" - ); -} - -#[test] -fn socket_bound_playback_session_matches_only_plain_live_playback() { - assert!(is_socket_bound_playback_session(PlaylistItemType::Live, None)); - assert!(!is_socket_bound_playback_session(PlaylistItemType::Live, Some(HLS_EXT))); - assert!(!is_socket_bound_playback_session(PlaylistItemType::Live, Some(DASH_EXT))); - assert!(!is_socket_bound_playback_session(PlaylistItemType::LiveHls, None)); - assert!(!is_socket_bound_playback_session(PlaylistItemType::Video, None)); - assert!(!is_socket_bound_playback_session(PlaylistItemType::Series, None)); - assert!(!is_socket_bound_playback_session(PlaylistItemType::Catchup, None)); -} - -#[test] -fn session_reacquire_cleanup_addrs_excludes_current_and_deduplicates() { - let primary: SocketAddr = "127.0.0.1:55191".parse().unwrap_or_else(|_| unreachable!()); - let overlap: SocketAddr = "127.0.0.1:55192".parse().unwrap_or_else(|_| unreachable!()); - let seek: SocketAddr = "127.0.0.1:55193".parse().unwrap_or_else(|_| unreachable!()); - let session = UserSession { - token: "tok-vod".to_string(), - transition_version: 1, - virtual_id: 9001, - provider: "provider-a".intern(), - stream_url: "http://localhost/movie.mkv".intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), - user_agent_stream_index: None, - addr: seek, - socket_bound: false, - active_addrs: vec![primary, overlap, seek, overlap], - ts: 1, - started_at: 1, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - lifecycle: crate::api::model::PlaybackLifecycle::Active, - ..Default::default() - }; - - assert_eq!(session_reacquire_cleanup_addrs(&session, &seek), vec![primary, overlap]); -} - -#[tokio::test] -async fn intentional_deferred_open_retains_provider_grace_handle() { - let app_state = create_test_provider_app_state(); - let provider_name = "provider_1".intern(); - let holder_addr: SocketAddr = "127.0.0.1:55230".parse().unwrap_or_else(|_| unreachable!()); - let deferred_addr: SocketAddr = "127.0.0.1:55231".parse().unwrap_or_else(|_| unreachable!()); - let holder_handle = app_state - .active_provider - .acquire_exact_connection_with_grace( - &provider_name, - &holder_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .expect("holder occupies the provider slot"); - let input = app_state.app_config.get_input_by_name(&provider_name).expect("provider input"); - let stream_url = "http://provider-1.example/live/user1/pass1/100.m3u8"; - let mut channel = create_test_live_channel(stream_url); - channel.item_type = PlaylistItemType::LiveHls; - let fingerprint = create_test_fingerprint(deferred_addr); - - let mut details = create_stream_response_details( - &app_state, - &get_stream_options(&app_state.app_config, StreamResponseMode::Stream), - stream_url, - "deferred-user", - &fingerprint, - &HeaderMap::new(), - &input, - &channel, - PlaylistItemType::LiveHls, - crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, - false, - UserConnectionPermission::Allowed, - None, - true, - true, - VirtualId::new(channel.virtual_id), - 0, - crate::api::model::ConnectionKind::Normal, - false, - Some("deferred-session"), - None, - false, - Some(true), - None, - None, - None, - ) - .await - .expect("provider grace creates deferred stream details"); - - assert!(details.stream.is_none()); - assert!(details.has_deferred_provider_open()); - assert!(details.provider_handle.is_some(), "deferred open must retain its provider allocation"); - - app_state.connection_manager.release_managed_provider_handle(details.provider_handle.take()); - app_state.connection_manager.release_provider_handle(Some(holder_handle)); -} - -#[test] -fn grace_hold_defers_live_and_fresh_video_but_not_catchup_or_affine_reopens() { - assert!(should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::LiveHls, false)); - assert!(should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::Video, false)); - assert!(!should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::Catchup, false)); - assert!(!should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::Catchup, true)); - assert!(!should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::Video, true)); - assert!(!should_defer_provider_open_for_grace_hold(true, false, PlaylistItemType::Video, true)); -} - -#[tokio::test] -async fn forced_reopen_cleanup_for_adaptive_streams_does_not_close_client_socket() { - let app_state = create_test_app_state(); - let addr: SocketAddr = "127.0.0.1:55220".parse().unwrap_or_else(|_| unreachable!()); - let mut close_rx = app_state.connection_manager.get_close_connection_channel(); - - cleanup_forced_reopen_addrs(&app_state, "adaptive-owner", &[addr]).await; - - let signal = - tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()).await.ok().and_then(Result::ok); - assert!(signal.is_none(), "adaptive cleanup should not hard-close the previous client socket"); -} - -#[tokio::test] -async fn forced_reopen_cleanup_for_non_adaptive_streams_preserves_client_socket() { - let app_state = create_test_app_state(); - let addr: SocketAddr = "127.0.0.1:55221".parse().unwrap_or_else(|_| unreachable!()); - let mut close_rx = app_state.connection_manager.get_close_connection_channel(); - - cleanup_forced_reopen_addrs(&app_state, "live-owner", &[addr]).await; - - let signal = - tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()).await.ok().and_then(Result::ok); - assert!(signal.is_none(), "a reopen must not close unrelated requests on the same proxy socket"); -} - -#[tokio::test] -async fn get_query_path_strips_extension_for_live_with_flag() { - use crate::model::ConfigInputFlags; - use shared::model::{InputType, PlaylistItemType, XtreamCluster, XtreamPlaylistItem}; - - let mut input = ConfigInput { - id: 1, - name: "provider_with_flag".intern(), - input_type: InputType::Xtream, - ..ConfigInput::default() - }; - let mut options = crate::model::ConfigInputOptions::defaults().clone(); - options.flags.set(ConfigInputFlags::XtreamLiveStreamWithoutExtension); - input.options = Some(options); - - let sources = SourcesConfig { inputs: vec![Arc::new(input)], ..SourcesConfig::default() }; - let mut app_cfg_raw = create_test_app_config(); - app_cfg_raw.sources = Arc::new(ArcSwap::from_pointee(sources)); - let app_state = create_test_app_state_for_config(Arc::new(app_cfg_raw)); - - let pli = XtreamPlaylistItem { - virtual_id: VirtualId::new(100), - provider_id: 1, - name: "test".intern(), - logo: "".intern(), - logo_small: "".intern(), - group: "".intern(), - title: "".intern(), - parent_code: "".intern(), - rec: "".intern(), - url: "http://example.com/123".intern(), - epg_channel_id: None, - xtream_cluster: XtreamCluster::Live, - additional_properties: None, - item_type: PlaylistItemType::Live, - category_id: 0, - input_name: "provider_with_flag".intern(), - channel_no: 0, - source_ordinal: 0, - input_stream_id: "1".intern(), - upstream_user_agent: None, - }; - - let hls_ext = shared::defaults::HLS_EXT.to_string(); - let (query_path, extension) = - crate::api::endpoints::xtream_api::get_query_path("", Some(&hls_ext), &pli, &app_state); - - assert_eq!(extension, ""); - assert_eq!(query_path, "1"); - - let dash_ext = shared::defaults::DASH_EXT.to_string(); - let (query_path, extension) = - crate::api::endpoints::xtream_api::get_query_path("", Some(&dash_ext), &pli, &app_state); - - assert_eq!(extension, ""); - assert_eq!(query_path, "1"); -} - -// ========================================================================================= -// evaluate_network_access tests -// ========================================================================================= - -/// Run `evaluate_network_access` for a synthetic user built from -/// `network_access` and assert the decision matches `expected`. Centralizes -/// the boilerplate (`user_with_network_access` + geoip setup + call + -/// assert) shared by every `evaluate_network_access` test below. -fn assert_network_decision( - network_access: Option, - geoip: &Arc>, - ip: &str, - expected: NetworkAccessDecision, -) { - let user = user_with_network_access(network_access); - assert_eq!(evaluate_network_access(&user, ip, geoip, GeoIpUnavailablePolicy::Deny), expected); -} - -/// `Arc>` with no `GeoIP` database loaded. -fn empty_geoip() -> Arc> { Arc::new(ArcSwapOption::::default()) } - -/// `Arc>` with a mock `GeoIP` that always reports the -/// given country for any lookup. -fn mock_geoip(country: &str) -> Arc> { - Arc::new(ArcSwapOption::from(Some(Arc::new(GeoIp::test_new(country))))) -} - -/// Helper to create a test user with specific network access -fn user_with_network_access(network_access: Option) -> ProxyUserCredentials { - ProxyUserCredentials { - username: "test".to_string(), - password: "test".to_string(), - token: None, - proxy: ProxyType::default(), - server: None, - epg_timeshift: None, - epg_request_timeshift: None, - created_at: None, - exp_date: None, - max_connections: 0, - status: None, - output_clusters: ClusterFlags::all(), - ui_enabled: true, - comment: None, - priority: 0, - soft_connections: 0, - soft_priority: 0, - t_is_api_user: false, - network_access, - plan: None, - filter: None, - raw_output_clusters: None, - raw_max_connections: 0, - raw_soft_connections: 0, - raw_proxy: Some(ProxyType::default()), - t_filter: None, - t_has_unresolved_plan: false, - t_has_invalid_filter: false, - } -} - -#[test] -fn no_config_allows_all() { - assert_network_decision(None, &empty_geoip(), "192.168.1.1", NetworkAccessDecision::Allowed); -} - -#[test] -fn empty_config_allows_all() { - assert_network_decision( - Some(NetworkAccess { allowed_countries: vec![], allowed_networks: vec![] }), - &empty_geoip(), - "192.168.1.1", - NetworkAccessDecision::Allowed, - ); -} - -#[test] -fn cidr_match_allows() { - assert_network_decision( - Some(NetworkAccess { allowed_countries: vec![], allowed_networks: vec!["192.168.1.0/24".parse().unwrap()] }), - &empty_geoip(), - "192.168.1.42", - NetworkAccessDecision::Allowed, - ); -} - -#[test] -fn cidr_miss_denies() { - assert_network_decision( - Some(NetworkAccess { allowed_countries: vec![], allowed_networks: vec!["192.168.1.0/24".parse().unwrap()] }), - &empty_geoip(), - "10.0.0.1", - NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCidrMatch), - ); -} - -#[test] -fn country_match_allows() { - assert_network_decision( - Some(NetworkAccess { allowed_countries: vec!["DE".to_string()], allowed_networks: vec![] }), - &mock_geoip("DE"), - "8.8.8.8", - NetworkAccessDecision::Allowed, - ); -} - -#[test] -fn country_miss_denies() { - assert_network_decision( - Some(NetworkAccess { allowed_countries: vec!["DE".to_string()], allowed_networks: vec![] }), - &mock_geoip("US"), - "8.8.8.8", - NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCountryMatch), - ); -} - -#[test] -fn no_geoip_denies_on_country_restriction() { - assert_network_decision( - Some(NetworkAccess { allowed_countries: vec!["DE".to_string()], allowed_networks: vec![] }), - &empty_geoip(), - "8.8.8.8", - NetworkAccessDecision::Denied(NetworkAccessDenyReason::GeoIpUnavailable), - ); -} - -#[test] -fn ipv4_vs_ipv6_denies_gracefully() { - assert_network_decision( - Some(NetworkAccess { allowed_countries: vec![], allowed_networks: vec!["2001:db8::/32".parse().unwrap()] }), - &empty_geoip(), - "192.168.1.1", - NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCidrMatch), - ); -} - -#[test] -fn ipv6_vs_ipv4_denies_gracefully() { - assert_network_decision( - Some(NetworkAccess { allowed_countries: vec![], allowed_networks: vec!["192.168.1.0/24".parse().unwrap()] }), - &empty_geoip(), - "2001:db8::1", - NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCidrMatch), - ); -} - -#[test] -fn either_cidr_or_country_match_allows() { - assert_network_decision( - Some(NetworkAccess { - allowed_countries: vec!["US".to_string()], - allowed_networks: vec!["192.168.1.0/24".parse().unwrap()], - }), - &mock_geoip("DE"), - "192.168.1.42", - NetworkAccessDecision::Allowed, - ); -} - -#[test] -fn single_ip_cidr() { - let user = user_with_network_access(Some(NetworkAccess { - allowed_countries: vec![], - allowed_networks: vec!["192.168.1.1/32".parse().unwrap()], - })); - let geoip = empty_geoip(); - assert_eq!( - evaluate_network_access(&user, "192.168.1.1", &geoip, GeoIpUnavailablePolicy::Deny), - NetworkAccessDecision::Allowed - ); - assert_eq!( - evaluate_network_access(&user, "192.168.1.2", &geoip, GeoIpUnavailablePolicy::Deny), - NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCidrMatch) - ); -} - -// ========================================================================================= -// network denied reason tests -// ========================================================================================= - -// The three `network_denied_reason_*` cases remain because they cover the -// `NetworkAccessDenyReason`-focused API surface directly, while -// `cidr_miss_denies`, `country_miss_denies`, and -// `no_geoip_denies_on_country_restriction` above assert the same deny -// reasons through broader `evaluate_network_access(...)` behavior. The -// overlap is intentional so both the general decision path and the -// reason-reporting-focused path stay pinned by tests. - -#[test] -fn network_denied_reason_country_unknown_when_geoip_loaded_but_unknown_ip() { - // GeoIP is loaded (not None), but lookup returns None for this IP (private/unknown). - // We need a GeoIP that only covers a private range, so public IPs get None. - // Use a CIDR-only restriction (no country rules) so we can verify - // that when countries ARE checked, lookup None gives "country_unknown". - let user = user_with_network_access(Some(NetworkAccess { - allowed_countries: vec!["DE".to_string()], - allowed_networks: vec!["192.168.1.0/24".parse().unwrap()], // miss CIDR first - })); - // Use the real GeoIp::new() which only seeds private ranges. - // For 8.8.8.8 (public), lookup returns None. - let geoip = Arc::new(ArcSwapOption::from(Some(Arc::new(GeoIp::new())))); - // CIDR miss -> country check -> geoip loaded but lookup returns None - assert_eq!( - evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Deny), - NetworkAccessDecision::Denied(NetworkAccessDenyReason::CountryUnknown) - ); -} - -#[test] -fn network_denied_reason_none_when_allowed() { - let user = user_with_network_access(Some(NetworkAccess { - allowed_countries: vec!["DE".to_string()], - allowed_networks: vec![], - })); - let mock_geoip = Arc::new(ArcSwapOption::from(Some(Arc::new(GeoIp::test_new("DE"))))); - assert_eq!( - evaluate_network_access(&user, "8.8.8.8", &mock_geoip, GeoIpUnavailablePolicy::Deny), - NetworkAccessDecision::Allowed - ); -} - -#[test] -fn network_denied_reason_none_when_no_config() { - let user = user_with_network_access(None); - let geoip = Arc::new(ArcSwapOption::::default()); - assert_eq!( - evaluate_network_access(&user, "192.168.1.1", &geoip, GeoIpUnavailablePolicy::Deny), - NetworkAccessDecision::Allowed - ); -} - -// ========================================================================================= -// GeoIP unavailable policy tests -// ========================================================================================= - -#[test] -fn geoip_unavailable_default_deny_denies() { - // Country rule exists but GeoIP is unavailable — default policy is Deny - let user = user_with_network_access(Some(NetworkAccess { - allowed_countries: vec!["DE".to_string()], - allowed_networks: vec![], - })); - let geoip = Arc::new(ArcSwapOption::::default()); - let decision = evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Deny); - assert_eq!(decision, NetworkAccessDecision::Denied(NetworkAccessDenyReason::GeoIpUnavailable)); -} - -#[test] -fn geoip_unavailable_explicit_allow_allows() { - // Country rule exists, GeoIP unavailable, but policy is Allow — allows - let user = user_with_network_access(Some(NetworkAccess { - allowed_countries: vec!["DE".to_string()], - allowed_networks: vec![], - })); - let geoip = Arc::new(ArcSwapOption::::default()); - let decision = evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Allow); - assert_eq!(decision, NetworkAccessDecision::AllowedGeoIpUnavailable); -} - -#[test] -fn geoip_unavailable_cidr_only_still_denies() { - // CIDR only rules, no match — should deny even with Allow policy - let user = user_with_network_access(Some(NetworkAccess { - allowed_countries: vec![], - allowed_networks: vec!["192.168.1.0/24".parse().unwrap()], - })); - let geoip = Arc::new(ArcSwapOption::::default()); - let decision = evaluate_network_access(&user, "10.0.0.1", &geoip, GeoIpUnavailablePolicy::Allow); - assert_eq!(decision, NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCidrMatch)); -} - -#[test] -fn geoip_unavailable_cidr_match_allows() { - // CIDR match always allows, regardless of policy - let user = user_with_network_access(Some(NetworkAccess { - allowed_countries: vec!["DE".to_string()], - allowed_networks: vec!["10.0.0.0/8".parse().unwrap()], - })); - let geoip = Arc::new(ArcSwapOption::::default()); - let decision = evaluate_network_access(&user, "10.0.0.1", &geoip, GeoIpUnavailablePolicy::Deny); - assert_eq!(decision, NetworkAccessDecision::Allowed); -} - -#[test] -fn geoip_loaded_country_mismatch_still_denies() { - // Loaded GeoIP but country doesn't match — should deny under Allow policy - let user = user_with_network_access(Some(NetworkAccess { - allowed_countries: vec!["DE".to_string()], - allowed_networks: vec![], - })); - let mock_geoip = Arc::new(ArcSwapOption::from(Some(Arc::new(GeoIp::test_new("US"))))); - let decision = evaluate_network_access(&user, "8.8.8.8", &mock_geoip, GeoIpUnavailablePolicy::Allow); - assert_eq!(decision, NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCountryMatch)); -} - -#[test] -fn geoip_loaded_unknown_country_still_denies() { - // Loaded GeoIP but lookup returns None — should deny under Allow policy - let user = user_with_network_access(Some(NetworkAccess { - allowed_countries: vec!["DE".to_string()], - allowed_networks: vec!["192.168.1.0/24".parse().unwrap()], - })); - let geoip = Arc::new(ArcSwapOption::from(Some(Arc::new(GeoIp::new())))); - let decision = evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Allow); - assert_eq!(decision, NetworkAccessDecision::Denied(NetworkAccessDenyReason::CountryUnknown)); -} - -#[test] -fn malformed_ip_denies_even_when_geoip_unavailable_policy_is_allow() { - let user = user_with_network_access(Some(NetworkAccess::from(&shared::model::NetworkAccessDto { - allowed_countries: Some(vec!["DE".to_string()]), - allowed_networks: None, - }))); - let geoip = Arc::new(ArcSwapOption::::default()); - - let decision = evaluate_network_access(&user, "not-an-ip", &geoip, GeoIpUnavailablePolicy::Allow); - - assert_eq!(decision, NetworkAccessDecision::Denied(NetworkAccessDenyReason::MalformedClientIp)); -} - -#[test] -fn evaluate_network_access_respects_allow_policy() { - // verify evaluate_network_access returns AllowedGeoIpUnavailable with Allow policy - let user = user_with_network_access(Some(NetworkAccess { - allowed_countries: vec!["DE".to_string()], - allowed_networks: vec![], - })); - let geoip = Arc::new(ArcSwapOption::::default()); - // Default deny policy should return Denied - assert_eq!( - evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Deny), - NetworkAccessDecision::Denied(NetworkAccessDenyReason::GeoIpUnavailable) - ); - // Allow policy should return AllowedGeoIpUnavailable - assert_eq!( - evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Allow), - NetworkAccessDecision::AllowedGeoIpUnavailable - ); -} - -#[tokio::test] -async fn test_channel_unavailable_fallback_does_not_register_body_owner_or_leak_slot() { - let app_state = create_test_provider_app_state(); - let provider_name = "provider_1".intern(); - let mut cfg = (**app_state.app_config.config.load()).clone(); - cfg.custom_stream_response_enabled = true; - app_state.app_config.config.store(std::sync::Arc::new(cfg)); - let addr: std::net::SocketAddr = "127.0.0.1:49080".parse().unwrap(); - - let custom_video = crate::model::CustomStreamResponse { - channel_unavailable: Some(crate::api::model::TransportStreamBuffer::new(b"channel-unavailable-bytes".to_vec())), - user_connections_exhausted: None, - provider_connections_exhausted: None, - low_priority_preempted: None, - user_account_expired: None, - panel_api_provisioning: None, - hls_session_or_lease_expired: None, - panel_api_provisioning_hls_segments: Vec::new(), - }; - app_state.app_config.custom_stream_response.store(Some(std::sync::Arc::new(custom_video))); - - let handle = app_state - .active_provider - .acquire_connection(&provider_name, &addr, 0, tuliprox_session::ConnectionKind::Normal) - .expect("connection should be acquired"); - - assert_eq!(app_state.active_provider.get_provider_connections_count(), 1); - - let (channel_unavail_stream, info) = - create_channel_unavailable_stream(&app_state.app_config, &[], axum::http::StatusCode::OK); - assert!(channel_unavail_stream.is_some()); - - let factory_response = tuliprox_session::ProviderStreamFactoryResponse { - stream: channel_unavail_stream.unwrap(), - info, - provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), - has_upstream_owner: false, - }; - - if factory_response.has_upstream_owner { - app_state.active_provider.register_body_owner(handle.allocation_id); - } - - app_state.active_provider.release_handle(&handle); - - assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); - assert_eq!(app_state.active_provider.active_connections().map(|m| m.values().sum::()).unwrap_or(0), 0); -} - -#[tokio::test] -async fn test_provider_open_cancelled_during_header_wait_releases_slot_without_leak() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let local_addr = listener.local_addr().unwrap(); - - let accepted = std::sync::Arc::new(tokio::sync::Notify::new()); - let accepted_clone = std::sync::Arc::clone(&accepted); - - let server_task = tokio::spawn(async move { - if let Ok((mut socket, _)) = listener.accept().await { - accepted_clone.notify_waiters(); - let mut buf = [0u8; 1024]; - loop { - use tokio::io::AsyncReadExt; - match socket.read(&mut buf).await { - Ok(0) | Err(_) => break, - Ok(_) => {} - } - } - } - }); - - let app_state = create_test_provider_app_state(); - let provider_name = "provider_1".intern(); - let client_addr: std::net::SocketAddr = "127.0.0.1:49081".parse().unwrap(); - - let handle = app_state - .active_provider - .acquire_connection(&provider_name, &client_addr, 0, tuliprox_session::ConnectionKind::Normal) - .expect("connection should be acquired"); - - assert_eq!(app_state.active_provider.get_provider_connections_count(), 1); - - let url = url::Url::parse(&format!("http://{local_addr}/stream.ts")).unwrap(); - let req_headers = axum::http::HeaderMap::new(); - let stream_options = get_stream_options(&app_state.app_config, StreamResponseMode::Stream); - let mut options = - crate::api::model::ProviderStreamFactoryOptions::new(&crate::api::model::ProviderStreamFactoryParams { - addr: client_addr, - item_type: PlaylistItemType::Live, - share_stream: false, - stream_options: &stream_options, - stream_url: &url, - req_headers: &req_headers, - input_headers: None, - session_headers: None, - disabled_headers: None, - default_user_agent: None, - username: None, - client_ip: None, - stream_channel: None, - connect_failure_stage: None, - content_representation: tuliprox_session::ProviderContentRepresentationMode::PreserveOrigin, - }); - options.set_provider_handle_tokens( - handle.cancel_token.clone(), - handle.completion_token.clone(), - Some(handle.close_reason.clone()), - ); - app_state.active_provider.mark_opening(handle.allocation_id); - - let ctx = app_state.provider_stream_ctx(); - let client = app_state.http_clients.default.load().as_ref().clone(); - - let open_task = - tokio::spawn(async move { crate::api::model::create_provider_stream(&ctx, &client, options).await }); - - // Wait until TCP connection is accepted and waiting for response headers - accepted.notified().await; - - // Now cancel the handle while it is waiting for response headers - handle.cancel_token.as_ref().unwrap().cancel(); - - // The open task should finish quickly because it is cancelled - let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), open_task) - .await - .expect("open task should abort upon cancellation") - .expect("task join"); - assert!(outcome.is_none()); - - // Release the handle - app_state.active_provider.release_handle(&handle); - - // Yield to allow background completion tasks to process - tokio::task::yield_now().await; - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); - assert_eq!(app_state.active_provider.active_connections().map(|m| m.values().sum::()).unwrap_or(0), 0); - - let _ = server_task.await; -} - -#[tokio::test] -async fn test_reentry_suppression_response_behavior() { - let response = crate::api::api_utils::reentry_suppressed_response(); - assert_eq!(response.status(), StatusCode::NO_CONTENT); - assert!(!response.headers().contains_key("x-tuliprox-rejection")); - assert!(!response.headers().contains_key("content-type")); -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn test_eviction_strategy_falls_back_when_candidate_protected_by_reentry() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 0, - grace_period_timeout_secs: 8, - grace_period_hold_stream: false, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest, AdmissionStrategy::EvictUserOldest]), - }); - - let victim_addr: SocketAddr = "127.0.0.1:55301".parse().unwrap_or_else(|_| unreachable!()); - let winner_addr: SocketAddr = "127.0.0.1:55302".parse().unwrap_or_else(|_| unreachable!()); - let other_ip_addr: SocketAddr = "10.0.0.1:55303".parse().unwrap_or_else(|_| unreachable!()); - let retry_addr: SocketAddr = "127.0.0.1:55304".parse().unwrap_or_else(|_| unreachable!()); - - let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); - let winner_fingerprint = create_test_fingerprint_with_user_agent(winner_addr, "winner/1.0"); - let other_ip_fingerprint = create_test_fingerprint_with_user_agent(other_ip_addr, "other/1.0"); - let retry_fingerprint = create_test_fingerprint_with_user_agent(retry_addr, "player/1.0"); - - let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9501.ts"); - victim_channel.virtual_id = 9501; - let winner_channel = create_test_live_channel("http://provider-1.example/live/9502.ts"); - let other_channel = create_test_live_channel("http://provider-1.example/live/9503.ts"); - - let mut session_user = ProxyUserCredentials::default(); - session_user.username = "fallback-user".to_string(); - - app_state.connection_manager.add_connection(&victim_addr).await; - app_state.connection_manager.add_connection(&winner_addr).await; - app_state.connection_manager.add_connection(&other_ip_addr).await; - - // Create user sessions so streams are linked to counted sessions - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &session_user, - session_token: "session-victim", - virtual_id: 9501, - provider: "provider-a", - stream_url: "http://provider-1.example/live/9501.ts", - addr: &victim_addr, - connection_permission: shared::model::UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &session_user, - session_token: "session-winner", - virtual_id: 9502, - provider: "provider-a", - stream_url: "http://provider-1.example/live/9502.ts", - addr: &winner_addr, - connection_permission: shared::model::UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &session_user, - session_token: "session-other", - virtual_id: 9503, - provider: "provider-a", - stream_url: "http://provider-1.example/live/9503.ts", - addr: &other_ip_addr, - connection_permission: shared::model::UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - // Stream 0: victim on same IP (127.0.0.1), channel 9501 - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 99, - username: "fallback-user", - max_connections: 3, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &victim_fingerprint, - provider: "provider-a".intern(), - stream_channel: &victim_channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some("session-victim"), - }) - .await; - - // Stream 1: winner on same IP (127.0.0.1), channel 9502 - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 1, - username: "fallback-user", - max_connections: 2, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &winner_fingerprint, - provider: "provider-a".intern(), - stream_channel: &winner_channel, - user_agent: std::borrow::Cow::Borrowed("winner/1.0"), - session_token: Some("session-winner"), - }) - .await; - - // Stream 2: stream on different IP (10.0.0.1), channel 9503 - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 2, - username: "fallback-user", - max_connections: 2, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &other_ip_fingerprint, - provider: "provider-a".intern(), - stream_channel: &other_channel, - user_agent: std::borrow::Cow::Borrowed("other/1.0"), - session_token: Some("session-other"), - }) - .await; - - // Mark winner_addr as protected against reentry of victim_addr (channel 9501) - app_state - .active_users - .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) - .await; - app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; - - // Retry for channel 9501 arrives from same IP (127.0.0.1). - // EvictUserSameIpOldest will target winner_addr (same IP), but winner_addr is protected! - // It should be suppressed, filtered out, and Fall B will continue to EvictUserOldest. - // EvictUserOldest will then target other_ip_addr (which is NOT protected). - let result = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: "fallback-user", - max_connections: 2, - soft_connections: 0, - client_ip: &retry_fingerprint.client_ip, - request_addr: &retry_fingerprint.addr, - use_session_admission: true, - session_token: Some("session-retry"), - activate_unbound_session: false, - eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9501) }, - }, - ) - .await; - - // Request must be Admitted because the fallback strategy evicted the unprotected stream! - assert_eq!(result.admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(result.admission.rejection_reason(), None); - - // Now test rejection when NO other candidate exists (only the protected stream). - // EvictUserSameIpOldest suppresses winner_addr; EvictUserOldest finds no candidates left. - let result_exhausted = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: "fallback-user", - max_connections: 1, - soft_connections: 0, - client_ip: &retry_fingerprint.client_ip, - request_addr: &retry_fingerprint.addr, - use_session_admission: true, - session_token: Some("session-retry-2"), - activate_unbound_session: false, - eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9501) }, - }, - ) - .await; - - assert_eq!(result_exhausted.admission.permission(), UserConnectionPermission::Exhausted); - assert_eq!(result_exhausted.admission.rejection_reason(), Some(AdmissionRejectionReason::RecentEvictionReentry)); - assert!(result_exhausted.admission.is_reentry_suppressed()); -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn test_mixed_suppressed_and_legitimate_eviction_reports_exhaustion() { - // A protected candidate is skipped, then a legitimate (but uncounted) candidate is - // evicted and admission still fails. That residual failure is real exhaustion and - // must not be mislabelled as `RecentEvictionReentry`. - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 0, - grace_period_timeout_secs: 8, - grace_period_hold_stream: false, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest, AdmissionStrategy::EvictUserOldest]), - }); - - let victim_addr: SocketAddr = "127.0.0.1:55321".parse().unwrap_or_else(|_| unreachable!()); - let protected_addr: SocketAddr = "127.0.0.1:55322".parse().unwrap_or_else(|_| unreachable!()); - let orphan_addr: SocketAddr = "10.0.0.1:55323".parse().unwrap_or_else(|_| unreachable!()); - let request_addr: SocketAddr = "127.0.0.1:55324".parse().unwrap_or_else(|_| unreachable!()); - - let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); - let protected_fingerprint = create_test_fingerprint_with_user_agent(protected_addr, "protected/1.0"); - let orphan_fingerprint = create_test_fingerprint_with_user_agent(orphan_addr, "orphan/1.0"); - let request_fingerprint = create_test_fingerprint_with_user_agent(request_addr, "player/1.0"); - - let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9701.ts"); - victim_channel.virtual_id = 9701; - let mut protected_channel = create_test_live_channel("http://provider-1.example/live/9702.ts"); - protected_channel.virtual_id = 9702; - let mut orphan_channel = create_test_live_channel("http://provider-1.example/live/9703.ts"); - orphan_channel.virtual_id = 9703; - - let mut session_user = ProxyUserCredentials::default(); - session_user.username = "mixed-user".to_string(); - - app_state.connection_manager.add_connection(&victim_addr).await; - app_state.connection_manager.add_connection(&protected_addr).await; - app_state.connection_manager.add_connection(&orphan_addr).await; - - // The guard source: a counted stream on the requested channel (9701). It supplies the - // socket reentry key and is released before the retry is resolved. - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &session_user, - session_token: "session-victim", - virtual_id: 9701, - provider: "provider-a", - stream_url: "http://provider-1.example/live/9701.ts", - addr: &victim_addr, - connection_permission: shared::model::UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 91, - username: "mixed-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &victim_fingerprint, - provider: "provider-a".intern(), - stream_channel: &victim_channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some("session-victim"), - }) - .await; - - // The protected candidate: counted, same IP as the retry, different channel. - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &session_user, - session_token: "session-protected", - virtual_id: 9702, - provider: "provider-a", - stream_url: "http://provider-1.example/live/9702.ts", - addr: &protected_addr, - connection_permission: shared::model::UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 92, - username: "mixed-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &protected_fingerprint, - provider: "provider-a".intern(), - stream_channel: &protected_channel, - user_agent: std::borrow::Cow::Borrowed("protected/1.0"), - session_token: Some("session-protected"), - }) - .await; - - // The legitimate fallback candidate: an orphan stream (no session), so evicting it - // frees no counted connection and admission stays exhausted. - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 93, - username: "mixed-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &orphan_fingerprint, - provider: "provider-a".intern(), - stream_channel: &orphan_channel, - user_agent: std::borrow::Cow::Borrowed("orphan/1.0"), - session_token: None, - }) - .await; - - app_state - .active_users - .mark_recent_eviction_guard_for_addr(&victim_addr, protected_addr, std::time::Duration::from_secs(3)) - .await; - app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; - - let result = resolve_admission_with_strategies( - &app_state.admission_ctx(), - AdmissionRequest { - username: "mixed-user", - max_connections: 1, - soft_connections: 0, - client_ip: &request_fingerprint.client_ip, - request_addr: &request_fingerprint.addr, - use_session_admission: true, - session_token: Some("session-retry"), - activate_unbound_session: false, - eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9701) }, - }, - ) - .await; - - assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted); - assert_eq!(result.admission.rejection_reason(), Some(AdmissionRejectionReason::UserConnectionsExhausted)); - assert!(!result.admission.is_reentry_suppressed()); -} - -fn connection_denied_count(app_state: &Arc) -> u64 { - app_state - .active_users - .events() - .stats() - .emitted() - .into_iter() - .find_map(|(kind, count)| (kind == shared::model::EventKind::ConnectionDenied).then_some(count)) - .unwrap_or(0) -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn test_reentry_suppression_does_not_emit_connection_denied() { - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 0, - grace_period_timeout_secs: 8, - grace_period_hold_stream: false, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest, AdmissionStrategy::EvictUserOldest]), - }); - - let victim_addr: SocketAddr = "127.0.0.1:55331".parse().unwrap_or_else(|_| unreachable!()); - let protected_addr: SocketAddr = "127.0.0.1:55332".parse().unwrap_or_else(|_| unreachable!()); - let request_addr: SocketAddr = "127.0.0.1:55333".parse().unwrap_or_else(|_| unreachable!()); - let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); - let protected_fingerprint = create_test_fingerprint_with_user_agent(protected_addr, "protected/1.0"); - let request_fingerprint = create_test_fingerprint_with_user_agent(request_addr, "player/1.0"); - - let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9711.ts"); - victim_channel.virtual_id = 9711; - let mut protected_channel = create_test_live_channel("http://provider-1.example/live/9712.ts"); - protected_channel.virtual_id = 9712; - - let mut user = ProxyUserCredentials::default(); - user.username = "event-reentry-user".to_string(); - user.max_connections = 1; - user.soft_connections = 0; - - app_state.connection_manager.add_connection(&victim_addr).await; - app_state.connection_manager.add_connection(&protected_addr).await; - - for (token, virtual_id, url, addr, fingerprint, channel, meter_uid) in [ - ( - "session-victim", - 9711, - "http://provider-1.example/live/9711.ts", - &victim_addr, - &victim_fingerprint, - &victim_channel, - 201_u32, - ), - ( - "session-protected", - 9712, - "http://provider-1.example/live/9712.ts", - &protected_addr, - &protected_fingerprint, - &protected_channel, - 202_u32, - ), - ] { - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: token, - virtual_id, - provider: "provider-a", - stream_url: url, - addr, - connection_permission: shared::model::UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid, - username: "event-reentry-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint, - provider: "provider-a".intern(), - stream_channel: channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some(token), - }) - .await; - } - - app_state - .active_users - .mark_recent_eviction_guard_for_addr(&victim_addr, protected_addr, std::time::Duration::from_secs(3)) - .await; - app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; - - let (admission, _, _) = crate::api::api_utils::resolve_playback_request_admission( - &app_state.admission_ctx(), - &user, - &request_fingerprint, - None, - "session-event", - false, - EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9711) }, - false, - false, - ) - .await; - - assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); - assert!(admission.is_reentry_suppressed()); - assert_eq!(connection_denied_count(&app_state), 0, "a suppressed reentry must not emit ConnectionDenied"); - assert_eq!( - app_state.active_users.reentry_suppressed_total(), - 1, - "a suppressed reentry must be counted once as a diagnostic" - ); -} - -#[tokio::test] -async fn test_real_exhaustion_emits_connection_denied() { - // Negative control: a genuine connection-limit denial must still emit the event. - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 0, - grace_period_timeout_secs: 8, - grace_period_hold_stream: false, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![]), - }); - - let addr: SocketAddr = "127.0.0.1:55341".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = create_test_fingerprint_with_user_agent(addr, "player/1.0"); - let mut user = ProxyUserCredentials::default(); - user.username = "event-limit-user".to_string(); - user.max_connections = 1; - user.soft_connections = 0; - - app_state.connection_manager.add_connection(&addr).await; - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user: &user, - session_token: "session-limit", - virtual_id: 9721, - provider: "provider-a", - stream_url: "http://provider-1.example/live/9721.ts", - addr: &addr, - connection_permission: shared::model::UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - // A stream row is what makes the session count towards `max_connections`. - let channel = create_test_live_channel("http://provider-1.example/live/9721.ts"); - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 211, - username: "event-limit-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some("session-limit"), - }) - .await; - - let (admission, _, _) = crate::api::api_utils::resolve_playback_request_admission( - &app_state.admission_ctx(), - &user, - &fingerprint, - None, - "session-new", - false, - EvictionReentryGuard::Session("session-new"), - false, - false, - ) - .await; - - assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); - assert_eq!(admission.rejection_reason(), Some(AdmissionRejectionReason::UserConnectionsExhausted)); - assert_eq!(connection_denied_count(&app_state), 1, "a real limit denial must emit ConnectionDenied"); - assert_eq!( - app_state.active_users.reentry_suppressed_total(), - 0, - "a real limit denial must not be counted as reentry suppression" - ); -} +mod admission_eviction_policy; +mod admission_request_classification; +mod admission_session_admission; +mod admission_strategy_chain; + +mod stream_responses_provider_capacity; +mod stream_responses_provider_fallback; +mod stream_responses_response_headers; +mod stream_responses_session_activation; diff --git a/backend/app/src/api/api_utils/tests/admission.rs b/backend/app/src/api/api_utils/tests/admission.rs new file mode 100644 index 000000000..4f23f6a9d --- /dev/null +++ b/backend/app/src/api/api_utils/tests/admission.rs @@ -0,0 +1,54 @@ +use crate::{ + api::model::{AppState, UserSession}, + auth::Fingerprint, +}; +use shared::model::{PlaylistItemType, UserConnectionPermission}; +use std::{collections::HashMap, sync::Arc}; + +pub(in crate::api::api_utils::tests) fn create_test_fingerprint_with_user_agent( + addr: std::net::SocketAddr, + user_agent: &str, +) -> Fingerprint { + Fingerprint::new(format!("{}|{user_agent}", addr.ip()), addr.ip().to_string(), addr) +} + +pub(in crate::api::api_utils::tests) fn create_test_session( + token: &str, + item_type: PlaylistItemType, + lifecycle: crate::api::model::PlaybackLifecycle, +) -> UserSession { + UserSession { + token: token.to_string(), + transition_version: 1, + virtual_id: 42, + provider: Arc::::from("provider-a"), + stream_url: Arc::::from(match item_type { + PlaylistItemType::LiveHls => "http://provider-1.example/live/42.m3u8", + _ => "http://provider-1.example/live/42.ts", + }), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), + user_agent_stream_index: None, + addr: "127.0.0.1:55555".parse().unwrap_or_else(|_| unreachable!()), + socket_bound: item_type.uses_socket_bound_session(), + active_addrs: Vec::new(), + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle, + ..Default::default() + } +} + +pub(in crate::api::api_utils::tests) fn connection_denied_count(app_state: &Arc) -> u64 { + app_state + .active_users + .events() + .stats() + .emitted() + .into_iter() + .find_map(|(kind, count)| (kind == shared::model::EventKind::ConnectionDenied).then_some(count)) + .unwrap_or(0) +} diff --git a/backend/app/src/api/api_utils/tests/admission_eviction_policy.rs b/backend/app/src/api/api_utils/tests/admission_eviction_policy.rs new file mode 100644 index 000000000..65c189438 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/admission_eviction_policy.rs @@ -0,0 +1,1471 @@ +use super::{ + admission::{connection_denied_count, create_test_fingerprint_with_user_agent}, + create_session_fingerprint, create_test_app_config, create_test_app_state_for_config, + create_test_app_state_with_stream_config, create_test_live_channel, resolve_admission_with_strategies, + resolve_playback_request_admission, AdmissionRequest, EvictionReentryGuard, PlaybackRequestClass, +}; +use crate::model::{Config, ProxyUserCredentials}; +use arc_swap::ArcSwap; +use axum::http::StatusCode; +use shared::{ + model::{AdmissionStrategy, PlaylistItemType, StreamChannel, UserConnectionPermission, VirtualId}, + utils::Internable, +}; +use std::{borrow::Cow, net::SocketAddr, sync::Arc}; +use tuliprox_session::AdmissionRejectionReason; + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn resolve_admission_with_strategies_prevents_recently_evicted_playback_ping_pong() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 0, + grace_period_timeout_secs: 8, + grace_period_hold_stream: false, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserOldest]), + }); + + let victim_addr: std::net::SocketAddr = "127.0.0.1:55181".parse().unwrap_or_else(|_| unreachable!()); + let reconnect_addr: std::net::SocketAddr = "127.0.0.1:55182".parse().unwrap_or_else(|_| unreachable!()); + let winner_addr: std::net::SocketAddr = "127.0.0.1:55183".parse().unwrap_or_else(|_| unreachable!()); + let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); + let reconnect_fingerprint = create_test_fingerprint_with_user_agent(reconnect_addr, "player/1.0"); + let winner_fingerprint = create_test_fingerprint_with_user_agent(winner_addr, "winner/1.0"); + let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9001.ts"); + victim_channel.virtual_id = 9001; + let mut winner_channel = create_test_live_channel("http://provider-1.example/live/9002.ts"); + winner_channel.virtual_id = 9002; + + let mut session_user = ProxyUserCredentials::default(); + session_user.username = "loop-user".to_string(); + + app_state.connection_manager.add_connection(&victim_addr).await; + app_state.connection_manager.add_connection(&winner_addr).await; + + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-victim", + virtual_id: 9001, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9001.ts", + addr: &victim_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-winner", + virtual_id: 9002, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9002.ts", + addr: &winner_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: "loop-user", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &victim_fingerprint, + provider: "provider-a".intern(), + stream_channel: &victim_channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("session-victim"), + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 2, + username: "loop-user", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &winner_fingerprint, + provider: "provider-a".intern(), + stream_channel: &winner_channel, + user_agent: std::borrow::Cow::Borrowed("winner/1.0"), + session_token: Some("session-winner"), + }) + .await; + + app_state + .active_users + .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) + .await; + app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; + + let result = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: "loop-user", + max_connections: 1, + soft_connections: 0, + client_ip: &reconnect_fingerprint.client_ip, + request_addr: &reconnect_fingerprint.addr, + use_session_admission: true, + session_token: Some("socket-reconnect"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9001) }, + }, + ) + .await; + let admission = result.admission; + let grace_mode = result.grace_mode; + + assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); + assert_eq!(admission.rejection_reason(), Some(AdmissionRejectionReason::RecentEvictionReentry)); + assert!(admission.is_reentry_suppressed()); + assert_eq!(grace_mode, None); + let active_streams = app_state.active_users.active_streams().await; + assert_eq!(active_streams.len(), 1); + assert_eq!(active_streams[0].channel.virtual_id, 9002); + + // A player may retry the winning plain-TS stream on another socket before the + // previous HTTP body has drained. That is a replacement of the same playback, + // not the evicted channel trying to reclaim its slot. + app_state + .active_users + .mark_recent_eviction_guard_for_addr(&winner_addr, reconnect_addr, std::time::Duration::from_secs(3)) + .await; + app_state.connection_manager.release_connection_as_kicked(&winner_addr).await; + app_state.connection_manager.add_connection(&reconnect_addr).await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 3, + username: "loop-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &reconnect_fingerprint, + provider: "provider-a".intern(), + stream_channel: &winner_channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: None, + }) + .await; + + let retry_addr: std::net::SocketAddr = "127.0.0.1:55184".parse().unwrap_or_else(|_| unreachable!()); + let retry = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: "loop-user", + max_connections: 1, + soft_connections: 0, + client_ip: &reconnect_fingerprint.client_ip, + request_addr: &retry_addr, + use_session_admission: true, + session_token: Some("socket-retry"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9002) }, + }, + ) + .await; + + assert_eq!(retry.admission.permission(), UserConnectionPermission::Allowed); + assert!(app_state.active_users.active_streams().await.is_empty()); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn resolve_admission_with_strategies_allows_other_channel_after_recent_eviction() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 0, + grace_period_timeout_secs: 8, + grace_period_hold_stream: false, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserOldest]), + }); + + let victim_addr: std::net::SocketAddr = "127.0.0.1:55184".parse().unwrap_or_else(|_| unreachable!()); + let winner_addr: std::net::SocketAddr = "127.0.0.1:55185".parse().unwrap_or_else(|_| unreachable!()); + let new_addr: std::net::SocketAddr = "127.0.0.1:55186".parse().unwrap_or_else(|_| unreachable!()); + let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); + let winner_fingerprint = create_test_fingerprint_with_user_agent(winner_addr, "winner/1.0"); + let new_fingerprint = create_test_fingerprint_with_user_agent(new_addr, "player/1.0"); + let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9101.ts"); + victim_channel.virtual_id = 9101; + let mut winner_channel = create_test_live_channel("http://provider-1.example/live/9102.ts"); + winner_channel.virtual_id = 9102; + let mut session_user = ProxyUserCredentials::default(); + session_user.username = "loop-user-2".to_string(); + + app_state.connection_manager.add_connection(&victim_addr).await; + app_state.connection_manager.add_connection(&winner_addr).await; + + // Create sessions before update_connection so streams are linked to counted sessions + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-victim", + virtual_id: 9101, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9101.ts", + addr: &victim_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-winner", + virtual_id: 9102, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9102.ts", + addr: &winner_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: "loop-user-2", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &victim_fingerprint, + provider: "provider-a".intern(), + stream_channel: &victim_channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("session-victim"), + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 2, + username: "loop-user-2", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &winner_fingerprint, + provider: "provider-a".intern(), + stream_channel: &winner_channel, + user_agent: std::borrow::Cow::Borrowed("winner/1.0"), + session_token: Some("session-winner"), + }) + .await; + + app_state + .active_users + .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) + .await; + app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; + + let result = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: "loop-user-2", + max_connections: 1, + soft_connections: 0, + client_ip: &new_fingerprint.client_ip, + request_addr: &new_fingerprint.addr, + use_session_admission: true, + session_token: Some("session-new"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9103) }, + }, + ) + .await; + let admission = result.admission; + + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert!(app_state.active_users.active_streams().await.is_empty()); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn resolve_admission_with_strategies_does_not_suppress_different_session_on_same_channel() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 0, + grace_period_timeout_secs: 8, + grace_period_hold_stream: false, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserOldest]), + }); + + let victim_addr: std::net::SocketAddr = "127.0.0.1:55190".parse().unwrap_or_else(|_| unreachable!()); + let winner_addr: std::net::SocketAddr = "127.0.0.1:55191".parse().unwrap_or_else(|_| unreachable!()); + let new_addr: std::net::SocketAddr = "127.0.0.1:55192".parse().unwrap_or_else(|_| unreachable!()); + let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); + let winner_fingerprint = create_test_fingerprint_with_user_agent(winner_addr, "player/1.0"); + let new_fingerprint = create_test_fingerprint_with_user_agent(new_addr, "player/1.0"); + let mut channel = create_test_live_channel("http://provider-1.example/live/9301.m3u8"); + channel.virtual_id = 9301; + channel.item_type = PlaylistItemType::LiveHls; + let mut session_user = ProxyUserCredentials::default(); + session_user.username = "loop-user-4".to_string(); + + app_state.connection_manager.add_connection(&victim_addr).await; + app_state.connection_manager.add_connection(&winner_addr).await; + + // Create sessions before update_connection so streams are linked to counted sessions + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-victim", + virtual_id: 9301, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9301.m3u8", + addr: &victim_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-winner", + virtual_id: 9301, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9301.m3u8", + addr: &winner_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: "loop-user-4", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &victim_fingerprint, + provider: "provider-a".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("session-victim"), + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 2, + username: "loop-user-4", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &winner_fingerprint, + provider: "provider-a".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("session-winner"), + }) + .await; + + app_state + .active_users + .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) + .await; + app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; + + let result = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: "loop-user-4", + max_connections: 1, + soft_connections: 0, + client_ip: &new_fingerprint.client_ip, + request_addr: &new_fingerprint.addr, + use_session_admission: true, + session_token: Some("session-other"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::Session("session-other"), + }, + ) + .await; + let admission = result.admission; + let grace_mode = result.grace_mode; + + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(grace_mode, None); + assert!(app_state.active_users.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn resolve_admission_with_strategies_allows_recently_evicted_playback_when_soft_slot_is_free() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 0, + grace_period_timeout_secs: 8, + grace_period_hold_stream: false, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserOldest]), + }); + + let victim_addr: std::net::SocketAddr = "127.0.0.1:55187".parse().unwrap_or_else(|_| unreachable!()); + let reconnect_addr: std::net::SocketAddr = "127.0.0.1:55188".parse().unwrap_or_else(|_| unreachable!()); + let winner_addr: std::net::SocketAddr = "127.0.0.1:55189".parse().unwrap_or_else(|_| unreachable!()); + let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); + let reconnect_fingerprint = create_test_fingerprint_with_user_agent(reconnect_addr, "player/1.0"); + let winner_fingerprint = create_test_fingerprint_with_user_agent(winner_addr, "winner/1.0"); + let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9201.ts"); + victim_channel.virtual_id = 9201; + let mut winner_channel = create_test_live_channel("http://provider-1.example/live/9202.ts"); + winner_channel.virtual_id = 9202; + + app_state.connection_manager.add_connection(&victim_addr).await; + app_state.connection_manager.add_connection(&winner_addr).await; + + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: "loop-user-3", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &victim_fingerprint, + provider: "provider-a".intern(), + stream_channel: &victim_channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("session-victim"), + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 2, + username: "loop-user-3", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &winner_fingerprint, + provider: "provider-a".intern(), + stream_channel: &winner_channel, + user_agent: std::borrow::Cow::Borrowed("winner/1.0"), + session_token: Some("session-winner"), + }) + .await; + + app_state + .active_users + .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) + .await; + app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; + + let result = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: "loop-user-3", + max_connections: 1, + soft_connections: 1, + client_ip: &reconnect_fingerprint.client_ip, + request_addr: &reconnect_fingerprint.addr, + use_session_admission: true, + session_token: Some("socket-reconnect"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9201) }, + }, + ) + .await; + let admission = result.admission; + let grace_mode = result.grace_mode; + + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); + assert_eq!(grace_mode, None); + + let active_streams = app_state.active_users.active_streams().await; + assert_eq!(active_streams.len(), 1); + assert_eq!(active_streams[0].channel.virtual_id, 9202); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn resolve_admission_with_strategies_evicts_preserved_hls_session_for_same_user_ts_request() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![ + AdmissionStrategy::EvictUserSameIpOldest, + AdmissionStrategy::EvictUserSameIpLatest, + AdmissionStrategy::GraceHoldStream, + AdmissionStrategy::EvictUserOldest, + AdmissionStrategy::EvictUserLatest, + ]), + }); + + let hls_addr: std::net::SocketAddr = "127.0.0.1:55176".parse().unwrap_or_else(|_| unreachable!()); + let ts_addr: std::net::SocketAddr = "127.0.0.1:55177".parse().unwrap_or_else(|_| unreachable!()); + let hls_fingerprint = create_test_fingerprint_with_user_agent(hls_addr, "player/1.0"); + let ts_fingerprint = create_test_fingerprint_with_user_agent(ts_addr, "player/1.0"); + let mut user = ProxyUserCredentials::default(); + user.username = "same-user".to_string(); + user.max_connections = 1; + + app_state.connection_manager.add_connection(&hls_addr).await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "tok-hls-preserved", + virtual_id: 5001, + provider: "provider-a", + stream_url: "http://provider-1.example/live/5001.m3u8", + addr: &hls_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &hls_fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { + item_type: PlaylistItemType::LiveHls, + virtual_id: 5001, + ..create_test_live_channel("http://provider-1.example/live/5001.m3u8") + }, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("tok-hls-preserved"), + }) + .await; + + app_state.connection_manager.release_connection(&hls_addr).await; + assert_eq!(app_state.active_users.user_connections(&user.username).await, 0); + assert!(app_state.active_users.active_streams().await.is_empty()); + + let mut close_rx = app_state.connection_manager.get_close_connection_channel(); + let result = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + client_ip: &ts_fingerprint.client_ip, + request_addr: &ts_fingerprint.addr, + use_session_admission: false, + session_token: None, + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(5001) }, + }, + ) + .await; + let admission = result.admission; + let grace_mode = result.grace_mode; + + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(grace_mode, None); + assert!(app_state.active_users.active_streams().await.is_empty()); + assert_eq!( + tokio::time::timeout(std::time::Duration::from_millis(100), close_rx.recv()).await.ok().and_then(Result::ok), + Some(crate::api::model::CloseConnectionSignal::WithReason( + hls_addr, + shared::model::DisconnectReason::ClientKicked, + )) + ); + assert!( + app_state.active_users.get_and_update_user_session(&user.username, "tok-hls-preserved").await.is_none(), + "preserved session should be removed once the TS request evicts it" + ); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn xtream_hls_then_ts_uses_distinct_tokens_and_evicts_old_hls_session() { + let mut app_cfg = create_test_app_config(); + let config = Config { + user_access_control: true, + reverse_proxy: Some(crate::model::ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: crate::model::ResourceRetryConfig::default(), + disabled_header: None, + stream: Some(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![ + AdmissionStrategy::EvictUserSameIpOldest, + AdmissionStrategy::EvictUserSameIpLatest, + AdmissionStrategy::GraceHoldStream, + AdmissionStrategy::EvictUserOldest, + AdmissionStrategy::EvictUserLatest, + ]), + }), + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }), + ..Config::default() + }; + app_cfg.config = Arc::new(ArcSwap::from_pointee(config)); + let app_state = create_test_app_state_for_config(Arc::new(app_cfg)); + let hls_addr: SocketAddr = "127.0.0.1:55186".parse().unwrap_or_else(|_| unreachable!()); + let ts_addr: SocketAddr = "127.0.0.1:55187".parse().unwrap_or_else(|_| unreachable!()); + let hls_fingerprint = create_test_fingerprint_with_user_agent(hls_addr, "libmpv"); + let ts_fingerprint = create_test_fingerprint_with_user_agent(ts_addr, "libmpv"); + let mut user = ProxyUserCredentials::default(); + user.username = "xtream-hls-ts".to_string(); + user.max_connections = 1; + + let virtual_id = 7811; + let hls_token = create_session_fingerprint(&hls_fingerprint, &user.username, virtual_id, false); + let ts_token = create_session_fingerprint(&ts_fingerprint, &user.username, virtual_id, true); + assert_ne!(hls_token, ts_token, "Xtream .m3u8 and .ts must not share the same playback token"); + + let mut hls_channel = create_test_live_channel("http://provider-1.example/live/7811.m3u8"); + hls_channel.virtual_id = virtual_id; + hls_channel.item_type = PlaylistItemType::LiveHls; + let mut ts_channel = create_test_live_channel("http://provider-1.example/live/7811.ts"); + ts_channel.virtual_id = virtual_id; + + app_state.connection_manager.add_connection(&hls_addr).await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: &hls_token, + virtual_id, + provider: "provider_1", + stream_url: hls_channel.url.as_ref(), + addr: &hls_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &hls_fingerprint, + provider: "provider_1".intern(), + stream_channel: &hls_channel, + user_agent: Cow::Borrowed("libmpv"), + session_token: Some(&hls_token), + }) + .await; + + app_state.connection_manager.release_connection(&hls_addr).await; + assert_eq!( + app_state.active_users.user_connections(&user.username).await, + 0, + "the preserved HLS playback must reserve capacity only virtually" + ); + assert_eq!(app_state.active_users.active_users_and_connections().await, (0, 0)); + assert!( + app_state.active_users.get_and_update_user_session(&user.username, &hls_token).await.is_some(), + "preserved HLS session should still exist before the competing TS request" + ); + assert_eq!( + app_state + .active_users + .connection_admission(&user.username, user.max_connections, user.soft_connections) + .await + .permission(), + UserConnectionPermission::Exhausted, + "the preserved HLS playback must still reserve the user's only slot before the TS request is evaluated" + ); + assert_eq!( + app_state.active_users.get_eviction_candidates(&user.username, &ts_fingerprint.client_ip).await.len(), + 1, + "the preserved HLS playback should be the single eviction candidate for the competing TS request" + ); + + let (ts_admission, ts_grace_mode, request_class) = resolve_playback_request_admission( + &app_state.admission_ctx(), + &user, + &ts_fingerprint, + None, + &ts_token, + false, + EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(virtual_id) }, + false, + false, + ) + .await; + assert_eq!(request_class, PlaybackRequestClass::Activate); + assert_eq!(ts_admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(ts_grace_mode, None); + assert!( + app_state + .active_users + .get_and_update_user_session(&user.username, &hls_token) + .await + .is_none(), + "the competing TS activation must remove the old preserved HLS session even though there is no live socket left to kick" + ); + assert_eq!( + app_state.active_users.user_connections(&user.username).await, + 0, + "eviction must not leave a real slot before the TS stream commits" + ); + assert_eq!(app_state.active_users.active_users_and_connections().await, (0, 0)); + + app_state.connection_manager.add_connection(&ts_addr).await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: &ts_token, + virtual_id, + provider: "provider_1", + stream_url: ts_channel.url.as_ref(), + addr: &ts_addr, + connection_permission: ts_admission.permission(), + connection_kind: ts_admission.kind(), + socket_bound: true, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 2, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &ts_fingerprint, + provider: "provider_1".intern(), + stream_channel: &ts_channel, + user_agent: Cow::Borrowed("libmpv"), + session_token: Some(&ts_token), + }) + .await; + + assert_eq!(app_state.active_users.user_connections(&user.username).await, 1); + assert_eq!(app_state.active_users.active_users_and_connections().await, (1, 1)); + let active_streams = app_state.active_users.active_streams().await; + assert_eq!(active_streams.len(), 1); + assert_eq!(active_streams.first().and_then(|stream| stream.session_token.as_deref()), Some(ts_token.as_str())); + assert!( + app_state + .active_users + .get_and_update_user_session(&user.username, &hls_token) + .await + .is_none(), + "after the competing TS request, the old Xtream HLS session must be gone so later /hls segment fetches cannot revive it" + ); + assert!( + app_state.active_users.get_and_update_user_session(&user.username, &ts_token).await.is_some(), + "the winning TS playback should remain tracked under its socket-bound Xtream token" + ); +} + +#[tokio::test] +async fn test_reentry_suppression_response_behavior() { + let response = crate::api::api_utils::reentry_suppressed_response(); + assert_eq!(response.status(), StatusCode::NO_CONTENT); + assert!(!response.headers().contains_key("x-tuliprox-rejection")); + assert!(!response.headers().contains_key("content-type")); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn test_eviction_strategy_falls_back_when_candidate_protected_by_reentry() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 0, + grace_period_timeout_secs: 8, + grace_period_hold_stream: false, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest, AdmissionStrategy::EvictUserOldest]), + }); + + let victim_addr: SocketAddr = "127.0.0.1:55301".parse().unwrap_or_else(|_| unreachable!()); + let winner_addr: SocketAddr = "127.0.0.1:55302".parse().unwrap_or_else(|_| unreachable!()); + let other_ip_addr: SocketAddr = "10.0.0.1:55303".parse().unwrap_or_else(|_| unreachable!()); + let retry_addr: SocketAddr = "127.0.0.1:55304".parse().unwrap_or_else(|_| unreachable!()); + + let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); + let winner_fingerprint = create_test_fingerprint_with_user_agent(winner_addr, "winner/1.0"); + let other_ip_fingerprint = create_test_fingerprint_with_user_agent(other_ip_addr, "other/1.0"); + let retry_fingerprint = create_test_fingerprint_with_user_agent(retry_addr, "player/1.0"); + + let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9501.ts"); + victim_channel.virtual_id = 9501; + let winner_channel = create_test_live_channel("http://provider-1.example/live/9502.ts"); + let other_channel = create_test_live_channel("http://provider-1.example/live/9503.ts"); + + let mut session_user = ProxyUserCredentials::default(); + session_user.username = "fallback-user".to_string(); + + app_state.connection_manager.add_connection(&victim_addr).await; + app_state.connection_manager.add_connection(&winner_addr).await; + app_state.connection_manager.add_connection(&other_ip_addr).await; + + // Create user sessions so streams are linked to counted sessions + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-victim", + virtual_id: 9501, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9501.ts", + addr: &victim_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-winner", + virtual_id: 9502, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9502.ts", + addr: &winner_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-other", + virtual_id: 9503, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9503.ts", + addr: &other_ip_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + // Stream 0: victim on same IP (127.0.0.1), channel 9501 + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 99, + username: "fallback-user", + max_connections: 3, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &victim_fingerprint, + provider: "provider-a".intern(), + stream_channel: &victim_channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("session-victim"), + }) + .await; + + // Stream 1: winner on same IP (127.0.0.1), channel 9502 + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: "fallback-user", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &winner_fingerprint, + provider: "provider-a".intern(), + stream_channel: &winner_channel, + user_agent: std::borrow::Cow::Borrowed("winner/1.0"), + session_token: Some("session-winner"), + }) + .await; + + // Stream 2: stream on different IP (10.0.0.1), channel 9503 + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 2, + username: "fallback-user", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &other_ip_fingerprint, + provider: "provider-a".intern(), + stream_channel: &other_channel, + user_agent: std::borrow::Cow::Borrowed("other/1.0"), + session_token: Some("session-other"), + }) + .await; + + // Mark winner_addr as protected against reentry of victim_addr (channel 9501) + app_state + .active_users + .mark_recent_eviction_guard_for_addr(&victim_addr, winner_addr, std::time::Duration::from_secs(3)) + .await; + app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; + + // Retry for channel 9501 arrives from same IP (127.0.0.1). + // EvictUserSameIpOldest will target winner_addr (same IP), but winner_addr is protected! + // It should be suppressed, filtered out, and Fall B will continue to EvictUserOldest. + // EvictUserOldest will then target other_ip_addr (which is NOT protected). + let result = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: "fallback-user", + max_connections: 2, + soft_connections: 0, + client_ip: &retry_fingerprint.client_ip, + request_addr: &retry_fingerprint.addr, + use_session_admission: true, + session_token: Some("session-retry"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9501) }, + }, + ) + .await; + + // Request must be Admitted because the fallback strategy evicted the unprotected stream! + assert_eq!(result.admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(result.admission.rejection_reason(), None); + + // Now test rejection when NO other candidate exists (only the protected stream). + // EvictUserSameIpOldest suppresses winner_addr; EvictUserOldest finds no candidates left. + let result_exhausted = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: "fallback-user", + max_connections: 1, + soft_connections: 0, + client_ip: &retry_fingerprint.client_ip, + request_addr: &retry_fingerprint.addr, + use_session_admission: true, + session_token: Some("session-retry-2"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9501) }, + }, + ) + .await; + + assert_eq!(result_exhausted.admission.permission(), UserConnectionPermission::Exhausted); + assert_eq!(result_exhausted.admission.rejection_reason(), Some(AdmissionRejectionReason::RecentEvictionReentry)); + assert!(result_exhausted.admission.is_reentry_suppressed()); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn test_mixed_suppressed_and_legitimate_eviction_reports_exhaustion() { + // A protected candidate is skipped, then a legitimate (but uncounted) candidate is + // evicted and admission still fails. That residual failure is real exhaustion and + // must not be mislabelled as `RecentEvictionReentry`. + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 0, + grace_period_timeout_secs: 8, + grace_period_hold_stream: false, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest, AdmissionStrategy::EvictUserOldest]), + }); + + let victim_addr: SocketAddr = "127.0.0.1:55321".parse().unwrap_or_else(|_| unreachable!()); + let protected_addr: SocketAddr = "127.0.0.1:55322".parse().unwrap_or_else(|_| unreachable!()); + let orphan_addr: SocketAddr = "10.0.0.1:55323".parse().unwrap_or_else(|_| unreachable!()); + let request_addr: SocketAddr = "127.0.0.1:55324".parse().unwrap_or_else(|_| unreachable!()); + + let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); + let protected_fingerprint = create_test_fingerprint_with_user_agent(protected_addr, "protected/1.0"); + let orphan_fingerprint = create_test_fingerprint_with_user_agent(orphan_addr, "orphan/1.0"); + let request_fingerprint = create_test_fingerprint_with_user_agent(request_addr, "player/1.0"); + + let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9701.ts"); + victim_channel.virtual_id = 9701; + let mut protected_channel = create_test_live_channel("http://provider-1.example/live/9702.ts"); + protected_channel.virtual_id = 9702; + let mut orphan_channel = create_test_live_channel("http://provider-1.example/live/9703.ts"); + orphan_channel.virtual_id = 9703; + + let mut session_user = ProxyUserCredentials::default(); + session_user.username = "mixed-user".to_string(); + + app_state.connection_manager.add_connection(&victim_addr).await; + app_state.connection_manager.add_connection(&protected_addr).await; + app_state.connection_manager.add_connection(&orphan_addr).await; + + // The guard source: a counted stream on the requested channel (9701). It supplies the + // socket reentry key and is released before the retry is resolved. + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-victim", + virtual_id: 9701, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9701.ts", + addr: &victim_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 91, + username: "mixed-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &victim_fingerprint, + provider: "provider-a".intern(), + stream_channel: &victim_channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("session-victim"), + }) + .await; + + // The protected candidate: counted, same IP as the retry, different channel. + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "session-protected", + virtual_id: 9702, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9702.ts", + addr: &protected_addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 92, + username: "mixed-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &protected_fingerprint, + provider: "provider-a".intern(), + stream_channel: &protected_channel, + user_agent: std::borrow::Cow::Borrowed("protected/1.0"), + session_token: Some("session-protected"), + }) + .await; + + // The legitimate fallback candidate: an orphan stream (no session), so evicting it + // frees no counted connection and admission stays exhausted. + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 93, + username: "mixed-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &orphan_fingerprint, + provider: "provider-a".intern(), + stream_channel: &orphan_channel, + user_agent: std::borrow::Cow::Borrowed("orphan/1.0"), + session_token: None, + }) + .await; + + app_state + .active_users + .mark_recent_eviction_guard_for_addr(&victim_addr, protected_addr, std::time::Duration::from_secs(3)) + .await; + app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; + + let result = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: "mixed-user", + max_connections: 1, + soft_connections: 0, + client_ip: &request_fingerprint.client_ip, + request_addr: &request_fingerprint.addr, + use_session_admission: true, + session_token: Some("session-retry"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9701) }, + }, + ) + .await; + + assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted); + assert_eq!(result.admission.rejection_reason(), Some(AdmissionRejectionReason::UserConnectionsExhausted)); + assert!(!result.admission.is_reentry_suppressed()); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn test_reentry_suppression_does_not_emit_connection_denied() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 0, + grace_period_timeout_secs: 8, + grace_period_hold_stream: false, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest, AdmissionStrategy::EvictUserOldest]), + }); + + let victim_addr: SocketAddr = "127.0.0.1:55331".parse().unwrap_or_else(|_| unreachable!()); + let protected_addr: SocketAddr = "127.0.0.1:55332".parse().unwrap_or_else(|_| unreachable!()); + let request_addr: SocketAddr = "127.0.0.1:55333".parse().unwrap_or_else(|_| unreachable!()); + let victim_fingerprint = create_test_fingerprint_with_user_agent(victim_addr, "player/1.0"); + let protected_fingerprint = create_test_fingerprint_with_user_agent(protected_addr, "protected/1.0"); + let request_fingerprint = create_test_fingerprint_with_user_agent(request_addr, "player/1.0"); + + let mut victim_channel = create_test_live_channel("http://provider-1.example/live/9711.ts"); + victim_channel.virtual_id = 9711; + let mut protected_channel = create_test_live_channel("http://provider-1.example/live/9712.ts"); + protected_channel.virtual_id = 9712; + + let mut user = ProxyUserCredentials::default(); + user.username = "event-reentry-user".to_string(); + user.max_connections = 1; + user.soft_connections = 0; + + app_state.connection_manager.add_connection(&victim_addr).await; + app_state.connection_manager.add_connection(&protected_addr).await; + + for (token, virtual_id, url, addr, fingerprint, channel, meter_uid) in [ + ( + "session-victim", + 9711, + "http://provider-1.example/live/9711.ts", + &victim_addr, + &victim_fingerprint, + &victim_channel, + 201_u32, + ), + ( + "session-protected", + 9712, + "http://provider-1.example/live/9712.ts", + &protected_addr, + &protected_fingerprint, + &protected_channel, + 202_u32, + ), + ] { + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: token, + virtual_id, + provider: "provider-a", + stream_url: url, + addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid, + username: "event-reentry-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint, + provider: "provider-a".intern(), + stream_channel: channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some(token), + }) + .await; + } + + app_state + .active_users + .mark_recent_eviction_guard_for_addr(&victim_addr, protected_addr, std::time::Duration::from_secs(3)) + .await; + app_state.connection_manager.release_connection_as_kicked(&victim_addr).await; + + let (admission, _, _) = crate::api::api_utils::resolve_playback_request_admission( + &app_state.admission_ctx(), + &user, + &request_fingerprint, + None, + "session-event", + false, + EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(9711) }, + false, + false, + ) + .await; + + assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); + assert!(admission.is_reentry_suppressed()); + assert_eq!(connection_denied_count(&app_state), 0, "a suppressed reentry must not emit ConnectionDenied"); + assert_eq!( + app_state.active_users.reentry_suppressed_total(), + 1, + "a suppressed reentry must be counted once as a diagnostic" + ); +} + +#[tokio::test] +async fn test_real_exhaustion_emits_connection_denied() { + // Negative control: a genuine connection-limit denial must still emit the event. + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 0, + grace_period_timeout_secs: 8, + grace_period_hold_stream: false, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![]), + }); + + let addr: SocketAddr = "127.0.0.1:55341".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint_with_user_agent(addr, "player/1.0"); + let mut user = ProxyUserCredentials::default(); + user.username = "event-limit-user".to_string(); + user.max_connections = 1; + user.soft_connections = 0; + + app_state.connection_manager.add_connection(&addr).await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "session-limit", + virtual_id: 9721, + provider: "provider-a", + stream_url: "http://provider-1.example/live/9721.ts", + addr: &addr, + connection_permission: shared::model::UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + // A stream row is what makes the session count towards `max_connections`. + let channel = create_test_live_channel("http://provider-1.example/live/9721.ts"); + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 211, + username: "event-limit-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("session-limit"), + }) + .await; + + let (admission, _, _) = crate::api::api_utils::resolve_playback_request_admission( + &app_state.admission_ctx(), + &user, + &fingerprint, + None, + "session-new", + false, + EvictionReentryGuard::Session("session-new"), + false, + false, + ) + .await; + + assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); + assert_eq!(admission.rejection_reason(), Some(AdmissionRejectionReason::UserConnectionsExhausted)); + assert_eq!(connection_denied_count(&app_state), 1, "a real limit denial must emit ConnectionDenied"); + assert_eq!( + app_state.active_users.reentry_suppressed_total(), + 0, + "a real limit denial must not be counted as reentry suppression" + ); +} diff --git a/backend/app/src/api/api_utils/tests/admission_request_classification.rs b/backend/app/src/api/api_utils/tests/admission_request_classification.rs new file mode 100644 index 000000000..77b942e74 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/admission_request_classification.rs @@ -0,0 +1,286 @@ +use super::{ + admission::create_test_session, classify_playback_request, create_test_app_state_with_stream_config, + create_test_fingerprint, resolve_playback_request_admission, EvictionReentryGuard, PlaybackRequestClass, + PlaybackRequestFacts, +}; +use crate::model::ProxyUserCredentials; +use shared::model::{AdmissionStrategy, PlaylistItemType, UserConnectionPermission}; +use std::net::SocketAddr; + +#[test] +fn classify_playback_request_marks_adaptive_playlist_request_as_prepare() { + let request_class = classify_playback_request(PlaybackRequestFacts { + existing_session: None, + prepare_only: true, + terminate: false, + }); + + assert_eq!(request_class, PlaybackRequestClass::Prepare); +} + +#[test] +fn classify_playback_request_marks_preserved_session_as_activate() { + let session = create_test_session( + "tok-preserved", + PlaylistItemType::LiveHls, + crate::api::model::PlaybackLifecycle::Preserved, + ); + + let request_class = classify_playback_request(PlaybackRequestFacts { + existing_session: Some(&session), + prepare_only: false, + terminate: false, + }); + + assert_eq!(request_class, PlaybackRequestClass::Activate); +} + +#[test] +fn classify_playback_request_marks_counted_session_as_follow_up() { + let session = + create_test_session("tok-active", PlaylistItemType::LiveHls, crate::api::model::PlaybackLifecycle::Active); + + let request_class = classify_playback_request(PlaybackRequestFacts { + existing_session: Some(&session), + prepare_only: false, + terminate: false, + }); + + assert_eq!(request_class, PlaybackRequestClass::FollowUp); +} + +/// `PendingProvider` must NOT be classified as `FollowUp`. +/// `PendingProvider` has no counted lease yet — the session is still waiting +/// for a provider slot. A new request on a `PendingProvider` session should +/// be `Activate` so that full admission evaluation happens, not a cheap +/// `FollowUp` skip. +#[test] +fn classify_playback_request_marks_pending_provider_as_activate_not_follow_up() { + let session = create_test_session( + "tok-pending", + PlaylistItemType::LiveHls, + crate::api::model::PlaybackLifecycle::PendingProvider { + data: crate::api::model::PendingProviderState { + reason_code: crate::api::model::PendingProviderReason::GraceHold, + created_at: 1, + deadline: 30, + version: 1, + wake_source: None, + }, + }, + ); + + let request_class = classify_playback_request(PlaybackRequestFacts { + existing_session: Some(&session), + prepare_only: false, + terminate: false, + }); + + assert_eq!( + request_class, + PlaybackRequestClass::Activate, + "PendingProvider should not be FollowUp - it has no counted lease yet" + ); +} + +/// `Active` without a counted lease must NOT be classified as `FollowUp`. +/// `FollowUp` should only be returned when the session actually owns a +/// counted admission lease. A session with `Active` lifecycle but no counted +/// lease should go through `Activate` so that the counted lease is reacquired. +#[test] +fn classify_playback_request_marks_active_without_counted_as_activate_not_follow_up() { + let mut session = create_test_session( + "tok-active-uncounted", + PlaylistItemType::LiveHls, + crate::api::model::PlaybackLifecycle::Active, // counted=false via is_counted() + ); + // Manually force counted=false by setting to Prepared lifecycle, then restoring + // Note: is_counted() returns false for Prepared, true for Active + // For this test we need a session that is Active lifecycle but not counted + // The new model derives counted from lifecycle, so we must use a different lifecycle + // to represent "not counted". Use Prepared instead. + session.lifecycle = crate::api::model::PlaybackLifecycle::Prepared; + + let request_class = classify_playback_request(PlaybackRequestFacts { + existing_session: Some(&session), + prepare_only: false, + terminate: false, + }); + + assert_eq!( + request_class, + PlaybackRequestClass::Activate, + "Active session with counted=false should not be FollowUp" + ); +} + +/// Prepared sessions must be classified as Activate. +#[test] +fn classify_playback_request_marks_prepared_session_as_activate() { + let session = + create_test_session("tok-prepared", PlaylistItemType::LiveHls, crate::api::model::PlaybackLifecycle::Prepared); + + let request_class = classify_playback_request(PlaybackRequestFacts { + existing_session: Some(&session), + prepare_only: false, + terminate: false, + }); + + assert_eq!(request_class, PlaybackRequestClass::Activate); +} + +/// `GraceActive` without counted lease must NOT be classified as `FollowUp`. +#[test] +fn classify_playback_request_marks_grace_active_without_counted_as_activate() { + let mut session = create_test_session( + "tok-grace-uncounted", + PlaylistItemType::LiveHls, + crate::api::model::PlaybackLifecycle::Active, // is_counted() = true for GraceActive + ); + // Test scenario: session has GraceActive lifecycle but we need it NOT counted + // This represents the edge case before grace task resolves. Use Prepared lifecycle + // to model "not counted" since is_counted() returns false for Prepared. + session.lifecycle = crate::api::model::PlaybackLifecycle::Prepared; + + let request_class = classify_playback_request(PlaybackRequestFacts { + existing_session: Some(&session), + prepare_only: false, + terminate: false, + }); + + assert_eq!( + request_class, + PlaybackRequestClass::Activate, + "GraceActive session with counted=false should not be FollowUp" + ); +} + +/// `resolve_playback_request_admission` with `prepare_only = true` returns `Prepare` class. +#[tokio::test] +async fn resolve_playback_request_admission_prepare_only_returns_prepare_class() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), + }); + let addr: SocketAddr = "127.0.0.1:55223".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + let mut user = ProxyUserCredentials::default(); + user.username = "prepare-only-user".to_string(); + user.max_connections = 1; + + let (admission, grace_mode, request_class) = resolve_playback_request_admission( + &app_state.admission_ctx(), + &user, + &fingerprint, + None, + "tok-prepare-only", + false, + EvictionReentryGuard::Session("tok-prepare-only"), + true, // prepare_only + false, // terminate + ) + .await; + + assert_eq!(request_class, PlaybackRequestClass::Prepare); + // Prepare returns Allowed without running strategies. + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(grace_mode, None); +} + +/// `resolve_playback_request_admission` with `terminate = true` returns `Terminate` class +/// and calls `terminate_session` on the existing session. +#[tokio::test] +async fn resolve_playback_request_admission_terminate_returns_terminate_class() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), + }); + let addr: SocketAddr = "127.0.0.1:55224".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + let mut user = ProxyUserCredentials::default(); + user.username = "terminate-user".to_string(); + user.max_connections = 2; + + // First create a session. + let session_token = "tok-terminate"; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token, + virtual_id: 55224, + provider: "test-provider", + stream_url: "http://provider.example/test.ts", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + // Verify session exists. + let before = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; + assert!(before.is_some(), "session should exist before terminate"); + + let (admission, grace_mode, request_class) = resolve_playback_request_admission( + &app_state.admission_ctx(), + &user, + &fingerprint, + before.as_ref(), + session_token, + false, + EvictionReentryGuard::Session(session_token), + false, // prepare_only + true, // terminate + ) + .await; + + assert_eq!(request_class, PlaybackRequestClass::Terminate); + assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); + assert_eq!(grace_mode, None); + + // Session should be expired after terminate. + let after = app_state.active_users.get_and_update_user_session(&user.username, session_token).await; + assert!(after.is_none(), "session should be removed after terminate"); +} + +/// `classify_playback_request` returns `Terminate` when `terminate = true`. +#[test] +fn classify_playback_request_returns_terminate_when_flag_set() { + let request_class = classify_playback_request(PlaybackRequestFacts { + existing_session: None, + prepare_only: false, + terminate: true, + }); + assert_eq!(request_class, PlaybackRequestClass::Terminate); +} diff --git a/backend/app/src/api/api_utils/tests/admission_session_admission.rs b/backend/app/src/api/api_utils/tests/admission_session_admission.rs new file mode 100644 index 000000000..cd02fedfe --- /dev/null +++ b/backend/app/src/api/api_utils/tests/admission_session_admission.rs @@ -0,0 +1,451 @@ +use super::{ + activate_session_before_stream_open, admission_failure_video_type, create_test_app_config, create_test_app_state, + create_test_app_state_for_config, create_test_fingerprint, create_test_live_channel, + create_test_provider_app_config, create_test_shared_target, load_test_channel, load_test_user, + resolve_admission_with_strategies, should_allow_exhausted_shared_reconnect, + should_defer_provider_open_for_grace_hold, spawn_load_test_origin, stream_admission_rejected_response, + stream_response, AdmissionRequest, EvictionReentryGuard, SessionActivationRequest, +}; +use crate::{ + api::model::{CustomVideoStreamType, UserSession}, + model::{Config, ConfigInput, ProxyUserCredentials, SourcesConfig}, +}; +use arc_swap::ArcSwap; +use axum::{ + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; +use shared::{ + model::{ + AdmissionStrategy, ConnectFailureReason, InputType, PlaylistItemType, UserConnectionPermission, VirtualId, + }, + utils::Internable, +}; +use std::{borrow::Cow, collections::HashMap, net::SocketAddr, sync::Arc}; + +#[test] +fn stream_admission_rejection_is_503_without_upstream_headers() { + use crate::api::model::StreamAdmissionError; + + for error in [ + StreamAdmissionError::CleanupReceiverClosed, + StreamAdmissionError::CleanupAdmissionTimeout, + StreamAdmissionError::RegistrationRejected, + ] { + let response = stream_admission_rejected_response(error, "test-user"); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response.headers().get("content-length").is_none(), "must not leak upstream content-length"); + assert!(response.headers().get("content-range").is_none(), "must not leak upstream content-range"); + } +} + +#[tokio::test] +async fn shared_admission_rejection_does_not_leak_meter_registration() { + let mut target = create_test_shared_target(); + target.options.as_mut().unwrap().share_live_streams.mpeg_ts = true; + let target = Arc::new(target); + + let origin_addr = spawn_load_test_origin(b"adm-test-ts-chunk").await; + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_shared_adm".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + username: Some("user".to_string()), + password: Some("pass".to_string()), + enabled: true, + priority: 0, + max_connections: 5, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + config.config = Arc::new(ArcSwap::from_pointee(Config { + reverse_proxy: Some(crate::model::ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: crate::model::ResourceRetryConfig::default(), + disabled_header: None, + stream: Some(crate::model::StreamConfig { metrics_enabled: true, ..crate::model::StreamConfig::default() }), + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }), + ..Config::default() + })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let cleanup_tx = app_state.connection_manager.cleanup_tx(); + let mut permits = Vec::new(); + while let Ok(permit) = cleanup_tx.clone().try_reserve_owned() { + permits.push(permit); + } + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_195)); + let fp = create_test_fingerprint(client_addr); + let user = load_test_user("user_adm_fail"); + let stream_url = format!("http://{origin_addr}/live/user/pass/adm.ts"); + let mut channel = load_test_channel(&input, origin_addr); + channel.url = stream_url.as_str().intern(); + + let resp = stream_response( + &fp, + &app_state, + "shared-session-adm", + None, + channel, + &stream_url, + None, + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + false, + None, + ) + .await + .into_response(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "shared admission failure must yield 503 Service Unavailable" + ); + assert_eq!( + app_state.active_provider.get_provider_connections_count(), + 0, + "admission failure must NOT fall through to opening a provider connection" + ); + assert_eq!(app_state.shared_stream_manager.meter_count(), 0, "rejected admission must not retain a meter ID"); + + drop(permits); +} + +#[test] +fn test_should_allow_exhausted_shared_reconnect_only_for_matching_shared_session() { + let session = UserSession { + transition_version: 1, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + token: "tok".to_string(), + virtual_id: 282, + provider: Arc::::from("provider"), + stream_url: Arc::::from("http://provider/live/449924.ts"), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), + user_agent_stream_index: None, + addr: "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()), + socket_bound: false, + active_addrs: vec!["127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!())], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + + assert!(should_allow_exhausted_shared_reconnect(true, Some(&session), 282, "http://provider/live/449924.ts")); + assert!(!should_allow_exhausted_shared_reconnect(false, Some(&session), 282, "http://provider/live/449924.ts")); + assert!(!should_allow_exhausted_shared_reconnect(true, Some(&session), 999, "http://provider/live/449924.ts")); + assert!(!should_allow_exhausted_shared_reconnect(true, Some(&session), 282, "http://provider/live/other.ts")); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn activate_session_before_stream_open_marks_pending_provider_for_grace_hold() { + let stream_cfg = crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), + }; + let mut app_cfg = create_test_app_config(); + app_cfg.config = Arc::new(ArcSwap::from_pointee(Config { + user_access_control: true, + reverse_proxy: Some(crate::model::ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: crate::model::ResourceRetryConfig::default(), + disabled_header: None, + stream: Some(stream_cfg), + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }), + ..Config::default() + })); + let app_state = create_test_app_state_for_config(Arc::new(app_cfg)); + let first_addr: SocketAddr = "127.0.0.1:55230".parse().unwrap_or_else(|_| unreachable!()); + let second_addr: SocketAddr = "127.0.0.1:55231".parse().unwrap_or_else(|_| unreachable!()); + let first_fingerprint = create_test_fingerprint(first_addr); + let second_fingerprint = create_test_fingerprint(second_addr); + let input = app_state.app_config.sources.load().inputs[0].clone(); + let mut user = ProxyUserCredentials::default(); + user.username = "grace-hold-user".to_string(); + user.max_connections = 1; + let first_channel = create_test_live_channel("http://provider-1.example/live/1.ts"); + let mut second_channel = create_test_live_channel("http://provider-1.example/live/2.m3u8"); + second_channel.item_type = PlaylistItemType::LiveHls; + second_channel.virtual_id = 55231; + + app_state.connection_manager.add_connection(&first_addr).await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 10, + fingerprint: &first_fingerprint, + provider: input.name.clone(), + stream_channel: &first_channel, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-first"), + }) + .await; + + let activation = activate_session_before_stream_open( + &app_state, + SessionActivationRequest { + fingerprint: &second_fingerprint, + input: input.as_ref(), + user: &user, + session_token: "tok-grace-hold", + request_class: None, + virtual_id: VirtualId::new(second_channel.virtual_id), + item_type: PlaylistItemType::LiveHls, + stream_url: second_channel.url.as_ref(), + connection_permission: UserConnectionPermission::Allowed, + connection_kind: crate::api::model::ConnectionKind::Normal, + granted_grace_mode: None, + socket_bound: false, + }, + ) + .await; + + assert_eq!(activation.admission.permission(), UserConnectionPermission::GracePeriod); + assert_eq!(activation.grace_mode, Some(crate::api::model::GraceMode::Hold)); + + let session = app_state + .active_users + .get_and_update_user_session(&user.username, "tok-grace-hold") + .await + .expect("placeholder session should exist"); + let crate::api::model::PlaybackLifecycle::PendingProvider { data: pending } = &session.lifecycle else { + panic!("grace hold should mark pending provider state") + }; + assert!(matches!(pending.reason_code, crate::api::model::PendingProviderReason::GraceHold)); + assert!(pending.deadline >= pending.created_at); + assert_eq!(app_state.active_users.user_connections(&user.username).await, 1); + assert!( + !session.lifecycle.is_counted(), + "pending provider placeholder must not consume an active user lease before commit" + ); +} + +#[test] +fn admission_failure_reason_maps_to_custom_video_type() { + assert!(matches!( + admission_failure_video_type(ConnectFailureReason::UserAccountExpired), + Some(CustomVideoStreamType::UserAccountExpired) + )); + assert!(matches!( + admission_failure_video_type(ConnectFailureReason::UserConnectionsExhausted), + Some(CustomVideoStreamType::UserConnectionsExhausted) + )); + assert!(matches!( + admission_failure_video_type(ConnectFailureReason::ProviderConnectionsExhausted), + Some(CustomVideoStreamType::ProviderConnectionsExhausted) + )); + assert!(admission_failure_video_type(ConnectFailureReason::ProviderError).is_none()); +} + +#[tokio::test] +async fn resolve_admission_with_strategies_allows_existing_session_even_when_user_is_at_limit() { + let app_state = create_test_app_state(); + let addr = "127.0.0.1:55154".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + let mut user = ProxyUserCredentials::default(); + user.username = "session-admission".to_string(); + user.max_connections = 1; + + app_state.connection_manager.add_connection(&addr).await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "vod-session", + virtual_id: 1, + provider: "provider-a", + stream_url: "http://provider-1.example/movie/1.mkv", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 10, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &create_test_live_channel("http://provider-1.example/movie/1.mkv"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some("vod-session"), + }) + .await; + + let session_based = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + client_ip: &fingerprint.client_ip, + request_addr: &fingerprint.addr, + use_session_admission: true, + session_token: Some("vod-session"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::Session("vod-session"), + }, + ) + .await; + assert_eq!(session_based.admission.permission(), UserConnectionPermission::Allowed); + + let connection_based = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + client_ip: &fingerprint.client_ip, + request_addr: &fingerprint.addr, + use_session_admission: false, + session_token: Some("vod-session"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::Session("vod-session"), + }, + ) + .await; + assert_eq!(connection_based.admission.permission(), UserConnectionPermission::Exhausted); +} + +#[tokio::test] +async fn activated_session_admission_keeps_hls_placeholders_uncounted_via_api_utils() { + let app_state = create_test_app_state(); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.max_connections = 1; + + let first_addr: std::net::SocketAddr = "127.0.0.1:55177".parse().unwrap_or_else(|_| unreachable!()); + let second_addr: std::net::SocketAddr = "127.0.0.1:55178".parse().unwrap_or_else(|_| unreachable!()); + let first_fingerprint = create_test_fingerprint(first_addr); + let second_fingerprint = create_test_fingerprint(second_addr); + + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "tok-hls-first", + virtual_id: 7101, + provider: "provider-a", + stream_url: "http://provider-1.example/live/7101.m3u8", + addr: &first_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "tok-hls-second", + virtual_id: 7102, + provider: "provider-a", + stream_url: "http://provider-1.example/live/7102.m3u8", + addr: &second_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let first_admission = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + client_ip: &first_fingerprint.client_ip, + request_addr: &first_fingerprint.addr, + use_session_admission: true, + session_token: Some("tok-hls-first"), + activate_unbound_session: true, + eviction_reentry_guard: EvictionReentryGuard::Session("tok-hls-first"), + }, + ) + .await; + let second_admission = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + client_ip: &second_fingerprint.client_ip, + request_addr: &second_fingerprint.addr, + use_session_admission: true, + session_token: Some("tok-hls-second"), + activate_unbound_session: true, + eviction_reentry_guard: EvictionReentryGuard::Session("tok-hls-second"), + }, + ) + .await; + + assert_eq!(first_admission.admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(second_admission.admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(app_state.active_users.user_connections(&user.username).await, 0); +} + +#[test] +fn grace_hold_defers_live_and_fresh_video_but_not_catchup_or_affine_reopens() { + assert!(should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::LiveHls, false)); + assert!(should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::Video, false)); + assert!(!should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::Catchup, false)); + assert!(!should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::Catchup, true)); + assert!(!should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::Video, true)); + assert!(!should_defer_provider_open_for_grace_hold(true, false, PlaylistItemType::Video, true)); +} diff --git a/backend/app/src/api/api_utils/tests/admission_strategy_chain.rs b/backend/app/src/api/api_utils/tests/admission_strategy_chain.rs new file mode 100644 index 000000000..97e947b06 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/admission_strategy_chain.rs @@ -0,0 +1,868 @@ +use super::{ + create_test_app_state_with_stream_config, create_test_fingerprint, create_test_live_channel, + create_test_provider_app_config, resolve_admission_with_strategies, AdmissionRequest, EvictionReentryGuard, +}; +use crate::model::ProxyUserCredentials; +use shared::{ + model::{AdmissionStrategy, UserConnectionPermission}, + utils::Internable, +}; +use std::{borrow::Cow, net::SocketAddr, sync::Arc}; +use tuliprox_session::{ + admission::{evaluate_remaining_strategies_after_grace, get_effective_admission_strategies}, + GraceResolutionContext, +}; + +#[tokio::test] +async fn effective_admission_strategies_use_legacy_grace_when_field_missing() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: None, + }); + + assert_eq!( + get_effective_admission_strategies(&app_state.admission_ctx()).as_ref(), + &[shared::model::AdmissionStrategy::GraceHoldStream][..] + ); +} + +#[tokio::test] +async fn effective_admission_strategies_respect_explicit_empty_list() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![]), + }); + + assert!(get_effective_admission_strategies(&app_state.admission_ctx()).is_empty()); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn evaluate_remaining_strategies_evicts_after_used_grace() { + // Strategies: [GraceHoldStream, EvictUserOldest] + // Grace was used at index 0, so only EvictUserOldest (index 1) is evaluated. + // The new request stays retryable while the evicted stream still owns its provider slot. + let strategies = vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]; + let grace_context = GraceResolutionContext { strategy_index: 0, strategies: strategies.into(), kind: None }; + + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]), + }); + let provider_config = create_test_provider_app_config(); + app_state.app_config.sources.store(provider_config.sources.load_full()); + app_state.active_provider.update_config(&app_state.app_config); + + let addr1: SocketAddr = "127.0.0.1:55701".parse().unwrap_or_else(|_| unreachable!()); + let addr2: SocketAddr = "10.0.0.5:55702".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint1 = create_test_fingerprint(addr1); + let fingerprint2 = create_test_fingerprint(addr2); + + app_state.connection_manager.add_connection(&addr1).await; + app_state.connection_manager.add_connection(&addr2).await; + + let mut user = ProxyUserCredentials::default(); + user.username = "remaining-evict".to_string(); + user.max_connections = 1; + + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "tok-counted", + virtual_id: 55701, + provider: "provider-evict", + stream_url: "http://provider.example/live/1.ts", + addr: &addr1, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + app_state + .active_users + .update_connection(crate::api::model::ActiveUserConnectionParams { + uid: 55701, + meter_uid: 55701, + username: "remaining-evict", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint1, + provider: "provider-evict".intern(), + stream_channel: &create_test_live_channel("http://provider.example/live/1.ts"), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-counted"), + }) + .await + .expect("stream should be created"); + + let provider_handle = app_state + .active_provider + .acquire_connection_with_grace_for_session( + &"provider_1".intern(), + &addr1, + false, + 0, + crate::api::model::ConnectionKind::Normal, + Some("tok-counted"), + ) + .expect("old stream should occupy the only provider slot"); + assert!(app_state.active_provider.register_body_owner(provider_handle.allocation_id)); + let close_rx = app_state.connection_manager.register_close_socket(addr1); + let manager = Arc::clone(&app_state.connection_manager); + let provider = Arc::clone(&app_state.active_provider); + let release_body = Arc::new(tokio::sync::Notify::new()); + let release_body_after_timeout = Arc::clone(&release_body); + let close_task = tokio::spawn(async move { + assert_eq!(close_rx.await.expect("kick close signal"), shared::model::DisconnectReason::ClientKicked); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + manager.release_provider_deferred(&addr1).await; + assert_eq!(provider.get_provider_connections_count(), 1, "body owner still holds provider capacity"); + release_body_after_timeout.notified().await; + provider.release_handle(&provider_handle); + provider_handle.completion_token.as_ref().expect("body completion token").cancel(); + manager.unregister_close_socket(&addr1); + }); + + let request = || AdmissionRequest { + username: "remaining-evict", + max_connections: 1, + soft_connections: 0, + client_ip: &fingerprint2.client_ip, + request_addr: &fingerprint2.addr, + use_session_admission: true, + session_token: Some("tok-new"), + activate_unbound_session: true, + eviction_reentry_guard: EvictionReentryGuard::Session("tok-new"), + }; + let result = tokio::time::timeout( + std::time::Duration::from_secs(5), + evaluate_remaining_strategies_after_grace( + &app_state.admission_ctx(), + request(), + &grace_context, + Some(crate::api::model::ConnectionKind::Normal), + ), + ) + .await + .expect("admission must remain retryable while the provider slot is held"); + + assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted); + assert_eq!(app_state.active_provider.get_provider_connections_count(), 1); + + let retry = tokio::time::timeout( + std::time::Duration::from_secs(5), + resolve_admission_with_strategies(&app_state.admission_ctx(), request()), + ) + .await + .expect("retry must remain bounded while the provider slot is held"); + assert_eq!(retry.admission.permission(), UserConnectionPermission::Exhausted); + + release_body.notify_one(); + close_task.await.expect("old transport cleanup"); + let result = resolve_admission_with_strategies(&app_state.admission_ctx(), request()).await; + + assert_eq!( + result.admission.permission(), + UserConnectionPermission::Allowed, + "EvictUserOldest should free the slot" + ); + assert!(result.grace_context.is_none(), "no grace context on eviction success"); + assert_eq!( + app_state.active_provider.get_provider_connections_count(), + 0, + "admission must not return while the evicted stream still owns the provider slot" + ); + let replacement = app_state.active_provider.acquire_connection_with_grace_for_session( + &"provider_1".intern(), + &addr2, + false, + 0, + crate::api::model::ConnectionKind::Normal, + Some("tok-new"), + ); + assert!(replacement.is_some(), "newly admitted stream must acquire the freed provider slot"); + if let Some(replacement) = replacement { + app_state.active_provider.release_handle(&replacement); + } +} + +#[tokio::test] +async fn evaluate_remaining_strategies_skips_no_match_and_uses_later_eviction() { + // Strategies: [GraceHoldStream, EvictUserSameIpOldest, EvictUserOldest] + // Grace was at index 0, remaining are EvictUserSameIpOldest (index 1) and EvictUserOldest (index 2). + // The existing counted session is at a DIFFERENT IP, so EvictUserSameIpOldest -> NoMatch. + // EvictUserOldest succeeds -> Allowed. + let strategies = vec![ + AdmissionStrategy::GraceHoldStream, + AdmissionStrategy::EvictUserSameIpOldest, + AdmissionStrategy::EvictUserOldest, + ]; + let grace_context = GraceResolutionContext { strategy_index: 0, strategies: strategies.into(), kind: None }; + + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![ + AdmissionStrategy::GraceHoldStream, + AdmissionStrategy::EvictUserSameIpOldest, + AdmissionStrategy::EvictUserOldest, + ]), + }); + + let addr1: SocketAddr = "127.0.0.1:55801".parse().unwrap_or_else(|_| unreachable!()); + let addr2: SocketAddr = "10.0.0.5:55802".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint1 = create_test_fingerprint(addr1); + let fingerprint2 = create_test_fingerprint(addr2); + + app_state.connection_manager.add_connection(&addr1).await; + app_state.connection_manager.add_connection(&addr2).await; + + let mut user = ProxyUserCredentials::default(); + user.username = "remaining-skip-no-match".to_string(); + user.max_connections = 1; + + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "tok-counted", + virtual_id: 55801, + provider: "provider-skip", + stream_url: "http://provider.example/live/1.ts", + addr: &addr1, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + app_state + .active_users + .update_connection(crate::api::model::ActiveUserConnectionParams { + uid: 55801, + meter_uid: 55801, + username: "remaining-skip-no-match", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint1, + provider: "provider-skip".intern(), + stream_channel: &create_test_live_channel("http://provider.example/live/1.ts"), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-counted"), + }) + .await + .expect("stream should be created"); + + let result = evaluate_remaining_strategies_after_grace( + &app_state.admission_ctx(), + AdmissionRequest { + username: "remaining-skip-no-match", + max_connections: 1, + soft_connections: 0, + client_ip: &fingerprint2.client_ip, + request_addr: &fingerprint2.addr, + use_session_admission: true, + session_token: Some("tok-new"), + activate_unbound_session: true, + eviction_reentry_guard: EvictionReentryGuard::Session("tok-new"), + }, + &grace_context, + Some(crate::api::model::ConnectionKind::Normal), + ) + .await; + + assert_eq!( + result.admission.permission(), + UserConnectionPermission::Allowed, + "EvictUserSameIpOldest should NoMatch, EvictUserOldest should succeed" + ); +} + +#[tokio::test] +async fn evaluate_remaining_strategies_empty_slice_denies() { + // Strategies: [GraceHoldStream] + // Grace was at index 0, remaining slice is empty -> exhausted. + let strategies = vec![AdmissionStrategy::GraceHoldStream]; + let grace_context = GraceResolutionContext { strategy_index: 0, strategies: strategies.into(), kind: None }; + + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), + }); + + let addr: SocketAddr = "10.0.0.5:55901".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + + let result = evaluate_remaining_strategies_after_grace( + &app_state.admission_ctx(), + AdmissionRequest { + username: "no-remaining-strategies", + max_connections: 1, + soft_connections: 0, + client_ip: &fingerprint.client_ip, + request_addr: &fingerprint.addr, + use_session_admission: true, + session_token: Some("tok-new"), + activate_unbound_session: true, + eviction_reentry_guard: EvictionReentryGuard::Session("tok-new"), + }, + &grace_context, + None, + ) + .await; + + assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted, "empty remaining slice should deny"); +} + +#[tokio::test] +async fn evaluate_remaining_strategies_preserves_soft_kind_on_exhausted() { + // Strategies: [GraceHoldStream] + // Grace was at index 0, remaining slice is empty -> exhausted. + // grace_context.kind is Soft — must be preserved in the exhausted result. + let strategies = vec![AdmissionStrategy::GraceHoldStream]; + let grace_context = GraceResolutionContext { + strategy_index: 0, + strategies: strategies.into(), + kind: Some(crate::api::model::ConnectionKind::Soft), + }; + + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), + }); + + let addr: SocketAddr = "10.0.0.6:55902".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + + let result = evaluate_remaining_strategies_after_grace( + &app_state.admission_ctx(), + AdmissionRequest { + username: "soft-kind-user", + max_connections: 1, + soft_connections: 0, + client_ip: &fingerprint.client_ip, + request_addr: &fingerprint.addr, + use_session_admission: true, + session_token: Some("tok-soft"), + activate_unbound_session: true, + eviction_reentry_guard: EvictionReentryGuard::Session("tok-soft"), + }, + &grace_context, + Some(crate::api::model::ConnectionKind::Soft), + ) + .await; + + assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted, "empty remaining slice should deny"); + assert_eq!( + result.admission.kind(), + Some(crate::api::model::ConnectionKind::Soft), + "exhausted result must preserve the original Soft connection kind" + ); +} + +#[tokio::test] +async fn evaluate_remaining_strategies_does_not_retry_used_prefix() { + // Strategies: [GraceHoldStream, GraceInstantStream, EvictUserOldest] + // Grace was at index 1 (GraceInstantStream). + // Remaining slice: [EvictUserOldest] (index 2). + // GraceHoldStream (index 0) must NOT be re-evaluated. + let strategies = vec![ + AdmissionStrategy::GraceHoldStream, + AdmissionStrategy::GraceInstantStream, + AdmissionStrategy::EvictUserOldest, + ]; + let strategies_for_config = strategies.clone(); + let grace_context = GraceResolutionContext { strategy_index: 1, strategies: strategies.into(), kind: None }; + + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(strategies_for_config), + }); + + let addr1: SocketAddr = "127.0.0.1:56001".parse().unwrap_or_else(|_| unreachable!()); + let addr2: SocketAddr = "10.0.0.5:56002".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint1 = create_test_fingerprint(addr1); + let fingerprint2 = create_test_fingerprint(addr2); + + app_state.connection_manager.add_connection(&addr1).await; + app_state.connection_manager.add_connection(&addr2).await; + + let mut user = ProxyUserCredentials::default(); + user.username = "remaining-no-retry".to_string(); + user.max_connections = 1; + + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "tok-counted", + virtual_id: 56001, + provider: "provider-no-retry", + stream_url: "http://provider.example/live/1.ts", + addr: &addr1, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + app_state + .active_users + .update_connection(crate::api::model::ActiveUserConnectionParams { + uid: 56001, + meter_uid: 56001, + username: "remaining-no-retry", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint1, + provider: "provider-no-retry".intern(), + stream_channel: &create_test_live_channel("http://provider.example/live/1.ts"), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-counted"), + }) + .await + .expect("stream should be created"); + + let result = evaluate_remaining_strategies_after_grace( + &app_state.admission_ctx(), + AdmissionRequest { + username: "remaining-no-retry", + max_connections: 1, + soft_connections: 0, + client_ip: &fingerprint2.client_ip, + request_addr: &fingerprint2.addr, + use_session_admission: true, + session_token: Some("tok-new"), + activate_unbound_session: true, + eviction_reentry_guard: EvictionReentryGuard::Session("tok-new"), + }, + &grace_context, + Some(crate::api::model::ConnectionKind::Normal), + ) + .await; + + assert_eq!( + result.admission.permission(), + UserConnectionPermission::Allowed, + "only EvictUserOldest should be evaluated, not GraceHoldStream" + ); +} + +#[tokio::test] +async fn evaluate_remaining_strategies_empty_slice_uses_original_kind_not_context_kind() { + // grace_context.kind = Normal, original_kind = Soft + // remaining slice is empty -> exhausted result must use original_kind. + // This proves the empty-slice branch uses original_kind, not grace_context.kind. + let strategies = vec![AdmissionStrategy::GraceHoldStream]; + let grace_context = GraceResolutionContext { + strategy_index: 0, + strategies: strategies.into(), + kind: Some(crate::api::model::ConnectionKind::Normal), + }; + let original_kind = Some(crate::api::model::ConnectionKind::Soft); + + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), + }); + + let addr: SocketAddr = "10.0.0.7:55903".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + + let result = evaluate_remaining_strategies_after_grace( + &app_state.admission_ctx(), + AdmissionRequest { + username: "kind-mismatch-empty", + max_connections: 1, + soft_connections: 0, + client_ip: &fingerprint.client_ip, + request_addr: &fingerprint.addr, + use_session_admission: true, + session_token: Some("tok-empty"), + activate_unbound_session: true, + eviction_reentry_guard: EvictionReentryGuard::Session("tok-empty"), + }, + &grace_context, + original_kind, + ) + .await; + + assert_eq!(result.admission.permission(), UserConnectionPermission::Exhausted); + assert_eq!( + result.admission.kind(), + original_kind, + "exhausted result must use original_kind (Soft), not grace_context.kind (Normal)" + ); +} + +#[tokio::test] +async fn evaluate_remaining_strategies_later_grace_uses_original_kind_not_context_kind() { + // grace_context.kind = Normal, original_kind = Soft + // Strategies: [GraceHoldStream, GraceInstantStream] + // Grace was used at index 0 (GraceHoldStream). + // Remaining slice contains GraceInstantStream (index 1). + // When the helper returns Grace for the remaining strategy, the new + // GraceResolutionContext.kind must be original_kind (Soft), not grace_context.kind (Normal). + // This proves build_grace_ctx uses original_kind as source of truth. + let strategies = vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::GraceInstantStream]; + let grace_context = GraceResolutionContext { + strategy_index: 0, + strategies: strategies.into(), + kind: Some(crate::api::model::ConnectionKind::Normal), + }; + let original_kind = Some(crate::api::model::ConnectionKind::Soft); + + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::GraceInstantStream]), + }); + + let addr1: SocketAddr = "127.0.0.1:55710".parse().unwrap_or_else(|_| unreachable!()); + let addr2: SocketAddr = "10.0.0.8:55711".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint1 = create_test_fingerprint(addr1); + let fingerprint2 = create_test_fingerprint(addr2); + + app_state.connection_manager.add_connection(&addr1).await; + app_state.connection_manager.add_connection(&addr2).await; + + let mut user = ProxyUserCredentials::default(); + user.username = "kind-mismatch-grace".to_string(); + user.max_connections = 1; + + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "tok-counted-grace", + virtual_id: 55710, + provider: "provider-grace-kind", + stream_url: "http://provider.example/live/1.ts", + addr: &addr1, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + app_state + .active_users + .update_connection(crate::api::model::ActiveUserConnectionParams { + uid: 55710, + meter_uid: 55710, + username: "kind-mismatch-grace", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint1, + provider: "provider-grace-kind".intern(), + stream_channel: &create_test_live_channel("http://provider.example/live/1.ts"), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-counted-grace"), + }) + .await + .expect("stream should be created"); + + let result = evaluate_remaining_strategies_after_grace( + &app_state.admission_ctx(), + AdmissionRequest { + username: "kind-mismatch-grace", + max_connections: 1, + soft_connections: 0, + client_ip: &fingerprint2.client_ip, + request_addr: &fingerprint2.addr, + use_session_admission: true, + session_token: Some("tok-new-grace"), + activate_unbound_session: true, + eviction_reentry_guard: EvictionReentryGuard::Session("tok-new-grace"), + }, + &grace_context, + original_kind, + ) + .await; + + assert_eq!( + result.admission.permission(), + UserConnectionPermission::GracePeriod, + "remaining GraceInstantStream should grant GracePeriod" + ); + assert!(result.grace_context.is_some(), "grace_context must be present when grace is granted"); + assert_eq!( + result.grace_context.as_ref().unwrap().kind, + original_kind, + "GraceResolutionContext.kind in the result must be original_kind (Soft), not grace_context.kind (Normal)" + ); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn resolve_admission_with_strategies_falls_through_after_failed_grace_grant() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]), + }); + + let first_addr: std::net::SocketAddr = "127.0.0.1:55151".parse().unwrap_or_else(|_| unreachable!()); + let second_addr: std::net::SocketAddr = "127.0.0.1:55152".parse().unwrap_or_else(|_| unreachable!()); + let first_fingerprint = create_test_fingerprint(first_addr); + let second_fingerprint = create_test_fingerprint(second_addr); + + app_state.connection_manager.add_connection(&first_addr).await; + app_state.connection_manager.add_connection(&second_addr).await; + + let mut session_user = ProxyUserCredentials::default(); + session_user.username = "fallthrough".to_string(); + session_user.max_connections = 1; + session_user.soft_connections = 1; + + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "tok-first", + virtual_id: 1, + provider: "provider-a", + stream_url: "http://provider-1.example/live/1.ts", + addr: &first_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: "fallthrough", + max_connections: 1, + soft_connections: 1, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 10, + fingerprint: &first_fingerprint, + provider: "provider-a".intern(), + stream_channel: &create_test_live_channel("http://provider-1.example/live/1.ts"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some("tok-first"), + }) + .await; + + assert!(app_state.active_users.grant_grace("fallthrough").await); + + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &session_user, + session_token: "tok-second", + virtual_id: 2, + provider: "provider-a", + stream_url: "http://provider-1.example/live/2.ts", + addr: &second_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Soft), + socket_bound: false, + }) + .await; + + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 2, + username: "fallthrough", + max_connections: 1, + soft_connections: 1, + connection_kind: crate::api::model::ConnectionKind::Soft, + priority: 0, + soft_priority: 10, + fingerprint: &second_fingerprint, + provider: "provider-a".intern(), + stream_channel: &create_test_live_channel("http://provider-1.example/live/2.ts"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some("tok-second"), + }) + .await; + + let result = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: "fallthrough", + max_connections: 1, + soft_connections: 1, + client_ip: "127.0.0.1", + request_addr: &"127.0.0.1:55153".parse().unwrap_or_else(|_| unreachable!()), + use_session_admission: true, + session_token: Some("tok-third"), + activate_unbound_session: false, + eviction_reentry_guard: EvictionReentryGuard::Session("tok-third"), + }, + ) + .await; + let admission = result.admission; + let grace_mode = result.grace_mode; + + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(admission.kind(), Some(crate::api::model::ConnectionKind::Normal)); + assert_eq!(grace_mode, None); +} diff --git a/backend/app/src/api/api_utils/tests/catchup.rs b/backend/app/src/api/api_utils/tests/catchup.rs new file mode 100644 index 000000000..50b7eceb6 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/catchup.rs @@ -0,0 +1,272 @@ +use super::{ + create_catchup_session_key, create_test_app_state_for_config, create_test_fingerprint, create_test_local_channel, + create_test_provider_app_config, force_provider_stream_response, load_test_user, probe_catchup_payload, + spawn_range_aware_test_origin, CatchupPayload, ForceStreamRequestContext, +}; +use crate::{ + api::model::{StreamError, UserSession}, + auth::Fingerprint, + model::{ConfigInput, ConfigInputAlias, SourcesConfig}, +}; +use arc_swap::ArcSwap; +use axum::{ + http::{header, HeaderMap, HeaderValue, StatusCode}, + response::IntoResponse, +}; +use bytes::Bytes; +use futures::{stream, StreamExt}; +use http_body_util::BodyExt; +use shared::{ + model::{InputType, PlaylistItemType, UserConnectionPermission, XtreamCluster}, + utils::Internable, +}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; +use tuliprox_hls::api::MAX_HLS_MANIFEST_BYTES; + +#[tokio::test] +async fn catchup_payload_probe_detects_fragmented_hls() { + let source = + stream::iter([Ok::<_, StreamError>(Bytes::from_static(b"#EX")), Ok(Bytes::from_static(b"TM3U\nsegment.ts\n"))]) + .boxed(); + + let result = probe_catchup_payload(source, std::time::Duration::from_secs(1)).await; + + assert!(matches!(&result, Ok(CatchupPayload::HlsManifest(_)))); + if let Ok(CatchupPayload::HlsManifest(manifest)) = result { + assert_eq!(manifest, b"#EXTM3U\nsegment.ts\n".as_slice()); + } +} + +#[tokio::test] +async fn catchup_payload_probe_replays_ts_bytes() { + let expected = Bytes::from_static(b"\x47direct-ts-payload"); + let source = stream::iter([Ok::<_, StreamError>(expected.clone())]).boxed(); + + let result = probe_catchup_payload(source, std::time::Duration::from_secs(1)).await; + + assert!(matches!(&result, Ok(CatchupPayload::Direct(_)))); + if let Ok(CatchupPayload::Direct(mut stream)) = result { + let mut actual = Vec::new(); + while let Some(chunk) = stream.next().await { + if let Ok(chunk) = chunk { + actual.extend_from_slice(&chunk); + } + } + assert_eq!(actual, expected.as_ref()); + } +} + +#[tokio::test] +async fn catchup_payload_probe_replays_partial_signature_at_eof() { + let expected = Bytes::from_static(b"#EXT"); + let source = stream::iter([Ok::<_, StreamError>(expected.clone())]).boxed(); + + let result = probe_catchup_payload(source, std::time::Duration::from_secs(1)).await; + + assert!(matches!(&result, Ok(CatchupPayload::Direct(_)))); + if let Ok(CatchupPayload::Direct(mut stream)) = result { + let actual = stream.next().await.and_then(Result::ok); + assert_eq!(actual.as_ref(), Some(&expected)); + assert!(stream.next().await.is_none()); + } +} + +#[tokio::test] +async fn catchup_payload_probe_rejects_oversized_manifest() { + let oversized = vec![b'x'; MAX_HLS_MANIFEST_BYTES]; + let source = + stream::iter([Ok::<_, StreamError>(Bytes::from_static(b"#EXTM3U")), Ok(Bytes::from(oversized))]).boxed(); + + let result = probe_catchup_payload(source, std::time::Duration::from_secs(1)).await; + + assert!(result.is_err()); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn catchup_abort_seek_and_window_change_preserve_correct_affinity() { + const CATCHUP_BODY_A: &[u8] = b"catchup-origin-a-archive-payload"; + const CATCHUP_BODY_B: &[u8] = b"catchup-origin-b-archive-payload"; + + let (origin_a, task_a) = spawn_range_aware_test_origin(CATCHUP_BODY_A, 2).await; + let (origin_b, task_b) = spawn_range_aware_test_origin(CATCHUP_BODY_B, 1).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_a}"), + enabled: true, + priority: 0, + max_connections: 2, + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "provider_2".intern(), + url: format!("http://{origin_b}"), + username: None, + password: None, + priority: 1, + max_connections: 2, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_620)); + let fingerprint = create_test_fingerprint(client_addr); + let user = load_test_user("viewer_catchup"); + + let make_catchup_session = |token: &str, window: &str| UserSession { + token: token.to_string(), + transition_version: 1, + virtual_id: 41, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_a}/timeshift/1.ts?window={window}").intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + + let make_catchup_channel = |window: &str| { + let mut channel = create_test_local_channel(&format!("http://{origin_a}/timeshift/1.ts?window={window}")); + channel.provider_id = 1; + channel.input_name = Arc::clone(&input.name); + channel.item_type = PlaylistItemType::Catchup; + channel.cluster = XtreamCluster::Live; + channel.url = format!("http://{origin_a}/timeshift/1.ts?window={window}").intern(); + channel + }; + + let session_w1_1 = make_catchup_session("catchup-w1", "w1"); + let session_w1_2 = make_catchup_session("catchup-w1", "w1"); + + let mut h1 = HeaderMap::new(); + h1.insert(header::RANGE, HeaderValue::from_static("bytes=0-9")); + let mut h2 = HeaderMap::new(); + h2.insert(header::RANGE, HeaderValue::from_static("bytes=5-14")); + + let resp1 = force_provider_stream_response( + &fingerprint, + &app_state, + &session_w1_1, + make_catchup_channel("w1"), + ForceStreamRequestContext { + req_headers: &h1, + input: &input, + user: &user, + session_reservation_ttl_secs: 10, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + let resp2 = force_provider_stream_response( + &fingerprint, + &app_state, + &session_w1_2, + make_catchup_channel("w1"), + ForceStreamRequestContext { + req_headers: &h2, + input: &input, + user: &user, + session_reservation_ttl_secs: 10, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(resp1.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(resp2.status(), StatusCode::PARTIAL_CONTENT); + + let mut session_w2 = make_catchup_session("catchup-w2", "w2"); + session_w2.provider = "provider_2".intern(); + session_w2.stream_url = format!("http://{origin_b}/timeshift/1.ts?window=w2").intern(); + let mut channel_w2 = make_catchup_channel("w2"); + channel_w2.url = Arc::clone(&session_w2.stream_url); + let mut h_w2 = HeaderMap::new(); + h_w2.insert(header::RANGE, HeaderValue::from_static("bytes=0-9")); + let resp_w2 = force_provider_stream_response( + &fingerprint, + &app_state, + &session_w2, + channel_w2, + ForceStreamRequestContext { + req_headers: &h_w2, + input: &input, + user: &user, + session_reservation_ttl_secs: 10, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(resp_w2.status(), StatusCode::PARTIAL_CONTENT); + let bytes_w2 = resp_w2.into_body().collect().await.expect("window w2 body").to_bytes(); + assert_eq!(bytes_w2.as_ref(), &CATCHUP_BODY_B[..10], "independent window uses available provider B"); + + drop(resp1); + let bytes2 = resp2.into_body().collect().await.expect("resp2 body").to_bytes(); + assert_eq!(bytes2.as_ref(), &CATCHUP_BODY_A[5..15]); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_provider.get_provider_connections_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("parallel catchup requests must release their physical connections"); + + let reqs_a = task_a.await.expect("task A completes"); + assert_eq!(reqs_a.len(), 2, "both requests in window w1 stayed on provider A"); + let reqs_b = task_b.await.expect("task B completes"); + assert_eq!(reqs_b.len(), 1, "the independent window used provider B while A was full"); + + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_provider.get_provider_connections_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("catchup provider connections must drain"); + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + app_state.active_provider.clear_provider_reservation("catchup-w1"); + app_state.active_provider.clear_provider_reservation("catchup-w2"); + assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0); + assert_eq!(app_state.active_provider.provider_lease_usage(&"provider_2".intern()).total(), 0); +} + +#[test] +fn catchup_session_key_is_sticky_only_within_the_same_archive_window() { + let addr: SocketAddr = "127.0.0.1:55181".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = Fingerprint::new("10.0.0.8|player".to_string(), "10.0.0.8".to_string(), addr); + + let first = create_catchup_session_key(&fingerprint, "user1", 7004, "/timeshift/3600/1700000000/"); + let same = create_catchup_session_key(&fingerprint, "user1", 7004, "timeshift/3600/1700000000"); + let seeked = create_catchup_session_key(&fingerprint, "user1", 7004, "timeshift/3600/1700000300"); + let resized = create_catchup_session_key(&fingerprint, "user1", 7004, "timeshift/1800/1700000000"); + + assert_eq!(first, same, "cosmetic path separators must not break provider stickiness"); + assert_ne!(first, seeked, "seeking must identify the new archive window"); + assert_ne!(first, resized, "changing duration must identify the new archive window"); +} diff --git a/backend/app/src/api/api_utils/tests/cleanup.rs b/backend/app/src/api/api_utils/tests/cleanup.rs new file mode 100644 index 000000000..daf204f95 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/cleanup.rs @@ -0,0 +1,1453 @@ +use super::{ + activate_session_before_stream_open, cleanup_forced_reopen_addrs, create_test_app_state, + create_test_app_state_for_config, create_test_app_state_with_stream_config, create_test_fingerprint, + create_test_live_channel, create_test_local_channel, create_test_provider_app_config, + create_test_provider_app_state, create_test_shared_target, force_provider_stream_response, get_stream_options, + load_test_channel, load_test_session, load_test_user, session_reacquire_cleanup_addrs, + spawn_controlled_fake_origin, spawn_range_aware_test_origin, stream_response, FakeOriginMode, + ForceStreamRequestContext, PlaybackRequestClass, SessionActivationRequest, StreamResponseMode, +}; +use crate::{ + api::model::{create_channel_unavailable_stream, UserSession}, + auth::Fingerprint, + model::{Config, ConfigInput, ConfigInputAlias, ProxyUserCredentials, SourcesConfig}, +}; +use arc_swap::ArcSwap; +use axum::{ + http::{header, HeaderMap, HeaderValue, StatusCode}, + response::IntoResponse, +}; +use bytes::Bytes; +use http_body_util::BodyExt; +use shared::{ + model::{AdmissionStrategy, InputType, PlaylistItemType, UserConnectionPermission, VirtualId, XtreamCluster}, + utils::Internable, +}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, +}; + +#[tokio::test] +async fn direct_ts_eof_before_first_byte_releases_slot_without_idle_lease() { + let (origin_addr, origin_task) = spawn_controlled_fake_origin(FakeOriginMode::EmptyBody, 1).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 1, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_601)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer_ts_eof".to_string(); + + let session = UserSession { + token: "session-eof".to_string(), + transition_version: 1, + virtual_id: 42, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/live/42.ts").intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + + let channel = load_test_channel(&input, origin_addr); + + let response = force_provider_stream_response( + &fingerprint, + &app_state, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::OK); + let body_bytes = response.into_body().collect().await.expect("empty body collects").to_bytes(); + assert!(body_bytes.is_empty(), "expected 0 payload bytes on immediate EOF"); + + let requests = origin_task.await.expect("origin finishes"); + assert_eq!(requests.len(), 1); + + for _ in 0..50 { + if app_state.active_provider.get_provider_connections_count() == 0 + && app_state.active_provider.provider_lease_usage(&input.name).total() == 0 + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "physical slot must be released"); + let lease_usage = app_state.active_provider.provider_lease_usage(&input.name); + assert_eq!(lease_usage.starting, 0, "no starting leases remaining"); + assert_eq!(lease_usage.active, 0, "no active leases remaining"); + assert_eq!(lease_usage.idle, 0, "unconfirmed stream must NOT transition to an idle lease"); + assert_eq!(lease_usage.total(), 0, "total leases must be 0"); + assert_eq!(app_state.active_users.active_streams().await.len(), 0, "active user stream must be cleaned up"); +} + +#[tokio::test] +async fn direct_ts_abort_while_waiting_for_first_byte_releases_exact_request() { + let notify = Arc::new(tokio::sync::Notify::new()); + let (origin_addr, origin_task) = + spawn_controlled_fake_origin(FakeOriginMode::BlockFirstChunk(Arc::clone(¬ify)), 1).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 1, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_602)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer_ts_abort".to_string(); + + let session = UserSession { + token: "session-abort-before-first-byte".to_string(), + transition_version: 1, + virtual_id: 42, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/live/42.ts").intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + + let channel = load_test_channel(&input, origin_addr); + + let response = force_provider_stream_response( + &fingerprint, + &app_state, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(app_state.active_provider.get_provider_connections_count(), 1); + + drop(response); + notify.notify_waiters(); + + let _ = origin_task.await; + + for _ in 0..50 { + if app_state.active_provider.get_provider_connections_count() == 0 + && app_state.active_provider.provider_lease_usage(&input.name).total() == 0 + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + assert_eq!( + app_state.active_provider.get_provider_connections_count(), + 0, + "physical slot must be released on abort" + ); + let lease_usage = app_state.active_provider.provider_lease_usage(&input.name); + assert_eq!(lease_usage.total(), 0, "no residual lease claim after unconfirmed abort"); + assert_eq!(app_state.active_users.active_streams().await.len(), 0, "active user stream must be cleaned up"); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn provider_error_after_first_byte_finishes_only_matching_request() { + const TS_CHUNK_188: &[u8] = &[0x47; 188]; + const SIBLING_CHUNK: &[u8] = &[0x47; 376]; + + let (origin_fail_addr, origin_fail_task) = + spawn_controlled_fake_origin(FakeOriginMode::ErrorAfterFirstChunk(TS_CHUNK_188.to_vec()), 1).await; + let (origin_ok_addr, origin_ok_task) = + spawn_controlled_fake_origin(FakeOriginMode::FixedBytes(SIBLING_CHUNK.to_vec()), 1).await; + + let input_fail = Arc::new(ConfigInput { + id: 1, + name: "provider_fail".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_fail_addr}"), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let input_ok = Arc::new(ConfigInput { + id: 2, + name: "provider_ok".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_ok_addr}"), + enabled: true, + priority: 1, + max_connections: 2, + ..ConfigInput::default() + }); + + let mut config = create_test_provider_app_config(); + config.sources = Arc::new(ArcSwap::from_pointee(SourcesConfig { + inputs: vec![Arc::clone(&input_fail), Arc::clone(&input_ok)], + ..SourcesConfig::default() + })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let addr_fail = SocketAddr::from(([127, 0, 0, 1], 55_603)); + let addr_ok = SocketAddr::from(([127, 0, 0, 1], 55_604)); + + let user = load_test_user("viewer_multistream"); + let session_fail = load_test_session(&input_fail, origin_fail_addr, "session-fail", addr_fail); + let session_ok = load_test_session(&input_ok, origin_ok_addr, "session-ok", addr_ok); + + let channel_fail = load_test_channel(&input_fail, origin_fail_addr); + let channel_ok = load_test_channel(&input_ok, origin_ok_addr); + + let resp_fail = force_provider_stream_response( + &create_test_fingerprint(addr_fail), + &app_state, + &session_fail, + channel_fail, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input_fail, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + let resp_ok = force_provider_stream_response( + &create_test_fingerprint(addr_ok), + &app_state, + &session_ok, + channel_ok, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input_ok, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(resp_fail.status(), StatusCode::OK); + assert_eq!(resp_ok.status(), StatusCode::OK); + assert_eq!( + app_state.active_provider.get_provider_connections_count(), + 2, + "both providers have an active connection" + ); + + let mut fail_body = resp_fail.into_body(); + let first_frame = fail_body.frame().await.expect("first frame exists").expect("first frame is valid"); + let first_chunk = first_frame.into_data().expect("data chunk exists"); + assert_eq!(first_chunk.as_ref(), TS_CHUNK_188, "first byte was confirmed with exact chunk bytes"); + + let second_frame_res = fail_body.frame().await; + assert!( + second_frame_res.is_none() || second_frame_res.as_ref().is_some_and(Result::is_err), + "stream must terminate or error after first chunk" + ); + drop(fail_body); + + for _ in 0..50 { + if app_state.active_provider.provider_lease_usage(&input_fail.name).total() == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + assert_eq!( + app_state.active_provider.provider_lease_usage(&input_fail.name).total(), + 0, + "failed provider leases cleaned up" + ); + assert_eq!( + app_state.active_provider.provider_lease_usage(&input_ok.name).total(), + 1, + "sibling provider lease remains intact" + ); + assert_eq!(app_state.active_provider.get_provider_connections_count(), 1, "sibling connection still active"); + + let ok_bytes = resp_ok.into_body().collect().await.expect("sibling body collects").to_bytes(); + assert_eq!(ok_bytes.as_ref(), SIBLING_CHUNK); + + for _ in 0..50 { + if app_state.active_provider.get_provider_connections_count() == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "all slots returned to baseline"); + let _ = origin_fail_task.await; + let _ = origin_ok_task.await; +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn no_first_byte_failure_allows_next_request_to_prefer_provider_a() { + const TS_CHUNK: &[u8] = &[0x47; 188]; + let listener_a = TcpListener::bind("127.0.0.1:0").await.expect("origin A binds"); + let primary_origin_addr = listener_a.local_addr().expect("origin A address"); + let task_a = tokio::spawn(async move { + let mut requests = Vec::new(); + for round in 0..2 { + let Ok((mut socket, _)) = listener_a.accept().await else { break }; + let mut req = Vec::new(); + while !req.windows(4).any(|w| w == b"\r\n\r\n") { + let mut chunk = [0u8; 512]; + let Ok(read) = socket.read(&mut chunk).await else { break }; + if read == 0 { + break; + } + req.extend_from_slice(&chunk[..read]); + } + requests.push(String::from_utf8_lossy(&req).to_ascii_lowercase()); + if round == 0 { + let head = + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.shutdown().await; + } else { + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + TS_CHUNK.len() + ); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(TS_CHUNK).await; + let _ = socket.shutdown().await; + } + } + requests + }); + + let (secondary_origin_addr, _task_b) = + spawn_controlled_fake_origin(FakeOriginMode::FixedBytes(TS_CHUNK.to_vec()), 1).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_A".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{primary_origin_addr}"), + enabled: true, + priority: 0, + max_connections: 1, + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "provider_B".intern(), + url: format!("http://{secondary_origin_addr}"), + username: None, + password: None, + priority: 1, + max_connections: 1, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }); + + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_605)); + let fingerprint = create_test_fingerprint(client_addr); + let user = load_test_user("viewer_prefer_a"); + + let session_1 = load_test_session(&input, primary_origin_addr, "session-1", client_addr); + let channel_1 = load_test_channel(&input, primary_origin_addr); + + let resp_1 = force_provider_stream_response( + &fingerprint, + &app_state, + &session_1, + channel_1, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(resp_1.status(), StatusCode::OK); + let b1 = resp_1.into_body().collect().await.expect("body 1 collects").to_bytes(); + assert!(b1.is_empty(), "request 1 had early EOF"); + + for _ in 0..50 { + if app_state.active_provider.get_provider_connections_count() == 0 + && app_state.active_provider.provider_lease_usage(&input.name).total() == 0 + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "slot on A must be free"); + assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0, "no phantom lease on A"); + + let session_2 = load_test_session(&input, primary_origin_addr, "session-2", client_addr); + let channel_2 = load_test_channel(&input, primary_origin_addr); + + let resp_2 = force_provider_stream_response( + &fingerprint, + &app_state, + &session_2, + channel_2, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(resp_2.status(), StatusCode::OK); + let b2 = resp_2.into_body().collect().await.expect("body 2 collects").to_bytes(); + assert_eq!(b2.as_ref(), TS_CHUNK, "request 2 successfully used Provider A and read payload"); + + let reqs_a = task_a.await.expect("task A completed"); + assert_eq!(reqs_a.len(), 2, "both requests 1 and 2 were routed to Provider A!"); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn parallel_vod_abort_preserves_sibling_claim_and_provider_stickiness() { + const VOD_BODY: &[u8] = b"0123456789abcdefghij"; + + let (origin_addr, origin_task) = spawn_range_aware_test_origin(VOD_BODY, 3).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_701)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer_vod_parallel".to_string(); + + let make_session = |token: &str| UserSession { + token: token.to_string(), + transition_version: 1, + virtual_id: 42, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/movie/1.ts").intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + + let make_channel = || { + let mut channel = create_test_local_channel(&format!("http://{origin_addr}/movie/1.ts")); + channel.provider_id = 1; + channel.input_name = Arc::clone(&input.name); + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + channel.url = format!("http://{origin_addr}/movie/1.ts").intern(); + channel + }; + + let build_request = |range: &'static str, token: &'static str| { + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static(range)); + let session = make_session(token); + (headers, session) + }; + + let (first_headers, first_session) = build_request("bytes=0-9", "vod-playback"); + let first = force_provider_stream_response( + &fingerprint, + &app_state, + &first_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &first_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + + let (second_headers, second_session) = build_request("bytes=5-14", "vod-playback"); + let second = force_provider_stream_response( + &fingerprint, + &app_state, + &second_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &second_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(app_state.active_users.active_streams().await.len(), 1, "affine claims share one playback row"); + assert_eq!(app_state.active_users.playback_resource_counts().await.0, 2, "both request claims active"); + + drop(first); + + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_users.playback_resource_counts().await.0 != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("aborted VOD body must release only its request claim"); + assert_eq!(app_state.active_users.active_streams().await.len(), 1, "sibling playback row remains active"); + + let second_body = second.into_body().collect().await.expect("second body collects").to_bytes(); + assert_eq!(second_body.as_ref(), &VOD_BODY[5..15]); + + let (third_headers, third_session) = build_request("bytes=10-19", "vod-playback"); + let third = force_provider_stream_response( + &fingerprint, + &app_state, + &third_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &third_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(third.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!( + third.headers().get(header::CONTENT_RANGE).and_then(|value| value.to_str().ok()), + Some("bytes 10-19/20") + ); + let third_body = third.into_body().collect().await.expect("third body collects").to_bytes(); + assert_eq!(third_body.as_ref(), &VOD_BODY[10..20]); + + let requests = origin_task.await.expect("origin task completes"); + assert_eq!(requests.len(), 3, "all 3 range requests must reach the same provider account"); + assert!(requests[0].contains("range: bytes=0-9")); + assert!(requests[1].contains("range: bytes=5-14")); + assert!(requests[2].contains("range: bytes=10-19")); + + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_provider.get_provider_connections_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("VOD provider connections must drain"); + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + app_state.active_provider.clear_provider_reservation("vod-playback"); + assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn parallel_series_abort_then_seek_reuses_account_without_stale_claim() { + const SERIES_BODY: &[u8] = b"0123456789abcdefghij"; + + let (origin_addr, origin_task) = spawn_range_aware_test_origin(SERIES_BODY, 3).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_702)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer_series_parallel".to_string(); + + let make_session = |token: &str| UserSession { + token: token.to_string(), + transition_version: 1, + virtual_id: 43, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/series/1.ts").intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + + let make_channel = || { + let mut channel = create_test_local_channel(&format!("http://{origin_addr}/series/1.ts")); + channel.provider_id = 1; + channel.input_name = Arc::clone(&input.name); + channel.item_type = PlaylistItemType::Series; + channel.cluster = XtreamCluster::Series; + channel.url = format!("http://{origin_addr}/series/1.ts").intern(); + channel + }; + + let build_request = |range: &'static str, token: &'static str| { + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static(range)); + let session = make_session(token); + (headers, session) + }; + + let (first_headers, first_session) = build_request("bytes=0-9", "series-playback"); + let first = force_provider_stream_response( + &fingerprint, + &app_state, + &first_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &first_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + + let (second_headers, second_session) = build_request("bytes=5-14", "series-playback"); + let second = force_provider_stream_response( + &fingerprint, + &app_state, + &second_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &second_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(app_state.active_users.active_streams().await.len(), 1, "affine claims share one playback row"); + assert_eq!(app_state.active_users.playback_resource_counts().await.0, 2, "both request claims active"); + + drop(first); + + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_users.playback_resource_counts().await.0 != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("aborted Series body must release only its request claim"); + assert_eq!(app_state.active_users.active_streams().await.len(), 1, "sibling playback row remains active"); + + let second_body = second.into_body().collect().await.expect("second body collects").to_bytes(); + assert_eq!(second_body.as_ref(), &SERIES_BODY[5..15]); + + let (third_headers, third_session) = build_request("bytes=12-19", "series-playback"); + let third = force_provider_stream_response( + &fingerprint, + &app_state, + &third_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &third_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 15, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(third.status(), StatusCode::PARTIAL_CONTENT); + let third_body = third.into_body().collect().await.expect("third body collects").to_bytes(); + assert_eq!(third_body.as_ref(), &SERIES_BODY[12..20]); + + let requests = origin_task.await.expect("origin task completes"); + assert_eq!(requests.len(), 3, "all series requests reach same provider"); + + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_provider.get_provider_connections_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("series provider connections must drain"); + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + app_state.active_provider.clear_provider_reservation("series-playback"); + assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0); +} + +#[tokio::test] +async fn stale_direct_cleanup_after_rebind_preserves_successor_request() { + let app_state = create_test_provider_app_state(); + let provider_name: Arc = Arc::from("provider_1"); + let owner = "test-rebind-owner"; + let addr_x = SocketAddr::from(([127, 0, 0, 1], 55_631)); + let addr_y = SocketAddr::from(([127, 0, 0, 1], 55_632)); + + let req_x = tuliprox_core::model::PlaybackRequestId::from_raw(101); + let lease_ref_x = + tuliprox_session::PlaybackLeaseRef { owner, kind: tuliprox_core::model::PlaybackKind::Vod, request_id: req_x }; + let handle_x = app_state + .active_provider + .acquire_exact_connection_with_lease_for_session( + &provider_name, + &addr_x, + false, + 0, + crate::api::model::ConnectionKind::Normal, + Some(lease_ref_x), + ) + .expect("request X acquires provider"); + let stale_tag = handle_x.binding_tag.expect("request X binding tag"); + app_state.active_provider.refresh_playback_lease(&provider_name, &lease_ref_x, 30); + app_state.active_provider.confirm_identified_playback_activity(owner, req_x); + app_state.active_provider.release_handle(&handle_x); + app_state.active_provider.finish_identified_playback_request( + owner, + req_x, + tuliprox_core::model::PlaybackRequestOutcome::Completed, + ); + assert_eq!(app_state.active_provider.provider_lease_usage(&provider_name).idle, 1); + + let req_y = tuliprox_core::model::PlaybackRequestId::from_raw(102); + let lease_ref_y = + tuliprox_session::PlaybackLeaseRef { owner, kind: tuliprox_core::model::PlaybackKind::Vod, request_id: req_y }; + let handle_y = app_state + .active_provider + .acquire_exact_connection_with_lease_for_session( + &provider_name, + &addr_y, + false, + 0, + crate::api::model::ConnectionKind::Normal, + Some(lease_ref_y), + ) + .expect("request Y reacquires provider"); + app_state.active_provider.refresh_playback_lease(&provider_name, &lease_ref_y, 30); + app_state.active_provider.confirm_identified_playback_activity(owner, req_y); + + app_state.active_provider.finish_identified_playback_request( + owner, + req_x, + tuliprox_core::model::PlaybackRequestOutcome::Completed, + ); + + let usage_after_stale_finish = app_state.active_provider.provider_lease_usage(&provider_name); + assert_eq!(usage_after_stale_finish.active, 1, "successor request Y must keep the active lease alive"); + + app_state.active_provider.clear_identified_provider_reservation(owner, &provider_name, Some(stale_tag)); + + let usage_after_stale_tag = app_state.active_provider.provider_lease_usage(&provider_name); + assert_eq!(usage_after_stale_tag.active, 1, "stale clear tag must not erase active successor lease"); + + app_state.active_provider.release_handle(&handle_y); + app_state.active_provider.finish_identified_playback_request( + owner, + req_y, + tuliprox_core::model::PlaybackRequestOutcome::ServerShutdown, + ); + + let final_usage = app_state.active_provider.provider_lease_usage(&provider_name); + assert_eq!(final_usage.total(), 0, "request Y has finished"); +} + +#[tokio::test] +async fn stale_hls_cleanup_after_rebind_preserves_current_origin_and_segments() { + let app_state = create_test_provider_app_state(); + let provider_name: Arc = Arc::from("provider_1"); + let owner = "test-hls-rebind-owner"; + let addr_x = SocketAddr::from(([127, 0, 0, 1], 55_633)); + let addr_y = SocketAddr::from(([127, 0, 0, 1], 55_634)); + + let req_x = tuliprox_core::model::PlaybackRequestId::from_raw(201); + let lease_ref_x = tuliprox_session::PlaybackLeaseRef { + owner, + kind: tuliprox_core::model::PlaybackKind::LiveHls, + request_id: req_x, + }; + let handle_x = app_state + .active_provider + .acquire_exact_connection_with_lease_for_session( + &provider_name, + &addr_x, + false, + 0, + crate::api::model::ConnectionKind::Normal, + Some(lease_ref_x), + ) + .expect("request X acquires HLS provider"); + let stale_tag = handle_x.binding_tag.expect("request X binding tag"); + app_state.active_provider.refresh_playback_lease(&provider_name, &lease_ref_x, 30); + app_state.active_provider.confirm_identified_playback_activity(owner, req_x); + app_state.active_provider.release_handle(&handle_x); + app_state.active_provider.finish_identified_playback_request( + owner, + req_x, + tuliprox_core::model::PlaybackRequestOutcome::Completed, + ); + + let req_y = tuliprox_core::model::PlaybackRequestId::from_raw(202); + let lease_ref_y = tuliprox_session::PlaybackLeaseRef { + owner, + kind: tuliprox_core::model::PlaybackKind::LiveHls, + request_id: req_y, + }; + let handle_y = app_state + .active_provider + .acquire_exact_connection_with_lease_for_session( + &provider_name, + &addr_y, + false, + 0, + crate::api::model::ConnectionKind::Normal, + Some(lease_ref_y), + ) + .expect("request Y reacquires HLS provider"); + app_state.active_provider.refresh_playback_lease(&provider_name, &lease_ref_y, 30); + app_state.active_provider.confirm_identified_playback_activity(owner, req_y); + + app_state.active_provider.finish_identified_playback_request( + owner, + req_x, + tuliprox_core::model::PlaybackRequestOutcome::Completed, + ); + + let usage = app_state.active_provider.provider_lease_usage(&provider_name); + assert!(usage.active >= 1 || usage.idle >= 1, "active/reconnect lease remains for successor"); + + app_state.active_provider.clear_identified_provider_reservation(owner, &provider_name, Some(stale_tag)); + + let usage2 = app_state.active_provider.provider_lease_usage(&provider_name); + assert_eq!(usage2.total(), usage.total(), "stale clear tag must not drop the HLS lease"); + + app_state.active_provider.release_handle(&handle_y); + app_state.active_provider.finish_identified_playback_request( + owner, + req_y, + tuliprox_core::model::PlaybackRequestOutcome::ServerShutdown, + ); + assert_eq!(app_state.active_provider.provider_lease_usage(&provider_name).total(), 0); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn shared_proxy_clients_keep_independent_bodies_and_cleanup() { + let mut target = create_test_shared_target(); + target.options.as_mut().unwrap().share_live_streams.mpeg_ts = true; + let target = Arc::new(target); + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("shared origin binds"); + let origin_addr = listener.local_addr().expect("shared origin addr"); + let app = axum::Router::new().fallback(|| async { + let (tx, rx) = tokio::sync::mpsc::channel::>(16); + tokio::spawn(async move { + for i in 0..10u8 { + let _ = tx.send(Ok(Bytes::from(vec![i; 188]))).await; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + }); + let body_stream = tokio_stream::wrappers::ReceiverStream::new(rx); + axum::response::Response::builder() + .header(axum::http::header::CONTENT_TYPE, "video/mp2t") + .body(axum::body::Body::from_stream(body_stream)) + .unwrap() + }); + let origin_task = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_shared_p".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + username: Some("user".to_string()), + password: Some("pass".to_string()), + enabled: true, + priority: 0, + max_connections: 5, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let proxy_addr = SocketAddr::from(([127, 0, 0, 1], 55_190)); + let fp1 = Fingerprint::new("client1|ua1".to_string(), "127.0.0.1".to_string(), proxy_addr); + let fp2 = Fingerprint::new("client2|ua2".to_string(), "127.0.0.1".to_string(), proxy_addr); + + let user1 = load_test_user("user_shared_1"); + let user2 = load_test_user("user_shared_2"); + let stream_url = format!("http://{origin_addr}/live/user/pass/shared.ts"); + let mut channel1 = load_test_channel(&input, origin_addr); + channel1.url = stream_url.as_str().intern(); + let mut channel2 = load_test_channel(&input, origin_addr); + channel2.url = stream_url.as_str().intern(); + + let resp1 = stream_response( + &fp1, + &app_state, + "shared-session-1", + None, + channel1, + &stream_url, + None, + &HeaderMap::default(), + &input, + &target, + &user1, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + false, + None, + ) + .await + .into_response(); + assert_eq!(resp1.status(), StatusCode::OK); + + let resp2 = stream_response( + &fp2, + &app_state, + "shared-session-2", + None, + channel2, + &stream_url, + None, + &HeaderMap::default(), + &input, + &target, + &user2, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + false, + None, + ) + .await + .into_response(); + assert_eq!(resp2.status(), StatusCode::OK); + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 1, "only 1 shared origin connection"); + + drop(resp1); + + let body2 = resp2.into_body().collect().await.expect("client 2 body collects").to_bytes(); + assert!(!body2.is_empty(), "client 2 receives stream bytes after client 1 dropped"); + drop(body2); + + for _ in 0..50 { + if app_state.active_provider.get_provider_connections_count() == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + origin_task.abort(); +} + +#[tokio::test] +async fn shared_start_abort_does_not_leak_meter_registration() { + let mut target = create_test_shared_target(); + target.options.as_mut().unwrap().share_live_streams.mpeg_ts = true; + let target = Arc::new(target); + + let notify = Arc::new(tokio::sync::Notify::new()); + let (origin_addr, origin_task) = + spawn_controlled_fake_origin(FakeOriginMode::BlockFirstChunk(Arc::clone(¬ify)), 1).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_shared_abort".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + username: Some("user".to_string()), + password: Some("pass".to_string()), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + config.config = Arc::new(ArcSwap::from_pointee(Config { + reverse_proxy: Some(crate::model::ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: crate::model::ResourceRetryConfig::default(), + disabled_header: None, + stream: Some(crate::model::StreamConfig { metrics_enabled: true, ..crate::model::StreamConfig::default() }), + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }), + ..Config::default() + })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_196)); + let fp = create_test_fingerprint(client_addr); + let user = load_test_user("user_shared_abort"); + let stream_url = format!("http://{origin_addr}/live/user/pass/shared_abort.ts"); + let mut channel = load_test_channel(&input, origin_addr); + channel.url = stream_url.as_str().intern(); + + let resp = stream_response( + &fp, + &app_state, + "shared-session-abort", + None, + channel, + &stream_url, + None, + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + false, + None, + ) + .await + .into_response(); + + assert_eq!(resp.status(), StatusCode::OK); + drop(resp); + notify.notify_waiters(); + let _ = origin_task.await; + + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while app_state.active_provider.get_provider_connections_count() != 0 + || app_state.active_provider.provider_lease_usage(&input.name).total() != 0 + || !app_state.active_users.active_streams().await.is_empty() + || app_state.shared_stream_manager.meter_count() != 0 + { + tokio::task::yield_now().await; + } + }) + .await + .expect("shared abort resources must return to baseline"); + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0, "provider slot released"); + assert_eq!(app_state.active_provider.provider_lease_usage(&input.name).total(), 0, "provider request released"); + assert_eq!(app_state.active_users.active_streams().await.len(), 0, "user claim released"); + assert_eq!(app_state.shared_stream_manager.meter_count(), 0, "aborted origin meter registration released"); +} + +// stale FollowUp revalidation +#[tokio::test] +async fn activate_session_before_stream_open_stale_follow_up_reclassified_on_counted_lease_release() { + // Scenario: pre-computed FollowUp, but session's counted lease was released before + // the guard was acquired. Must reclassify to Activate so admission runs. + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest]), + }); + let addr: SocketAddr = "127.0.0.1:55230".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + let input = app_state.app_config.sources.load().inputs[0].clone(); + let mut user = ProxyUserCredentials::default(); + user.username = "stale-followup-user".to_string(); + user.max_connections = 1; + let mut channel = create_test_live_channel("http://provider-1.example/live/55230.m3u8"); + channel.item_type = PlaylistItemType::LiveHls; + channel.virtual_id = 55230; + + // Session created in Active (counted) state. + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "tok-stale-followup", + virtual_id: channel.virtual_id, + provider: input.name.as_ref(), + stream_url: channel.url.as_ref(), + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: true, + }) + .await; + + // Simulate the counted lease being released before activation: + // expire the session so it no longer has a counted lease. + app_state.active_users.terminate_session(&user.username, "tok-stale-followup").await; + + // Call activate with stale FollowUp. Must NOT skip admission — reclassification + // to Activate must run so the placeholder is created. + let activation = activate_session_before_stream_open( + &app_state, + SessionActivationRequest { + fingerprint: &fingerprint, + input: input.as_ref(), + user: &user, + session_token: "tok-stale-followup", + request_class: Some(PlaybackRequestClass::FollowUp), + virtual_id: VirtualId::new(channel.virtual_id), + item_type: PlaylistItemType::LiveHls, + stream_url: channel.url.as_ref(), + connection_permission: UserConnectionPermission::Allowed, + connection_kind: crate::api::model::ConnectionKind::Normal, + granted_grace_mode: None, + socket_bound: true, + }, + ) + .await; + + // Must NOT skip — placeholder must be created since session is expired. + assert!( + activation.placeholder_transition_version.is_some(), + "stale FollowUp with expired session must run admission and create placeholder" + ); +} + +#[test] +fn session_reacquire_cleanup_addrs_excludes_current_and_deduplicates() { + let primary: SocketAddr = "127.0.0.1:55191".parse().unwrap_or_else(|_| unreachable!()); + let overlap: SocketAddr = "127.0.0.1:55192".parse().unwrap_or_else(|_| unreachable!()); + let seek: SocketAddr = "127.0.0.1:55193".parse().unwrap_or_else(|_| unreachable!()); + let session = UserSession { + token: "tok-vod".to_string(), + transition_version: 1, + virtual_id: 9001, + provider: "provider-a".intern(), + stream_url: "http://localhost/movie.mkv".intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), + user_agent_stream_index: None, + addr: seek, + socket_bound: false, + active_addrs: vec![primary, overlap, seek, overlap], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + + assert_eq!(session_reacquire_cleanup_addrs(&session, &seek), vec![primary, overlap]); +} + +#[tokio::test] +async fn forced_reopen_cleanup_for_adaptive_streams_does_not_close_client_socket() { + let app_state = create_test_app_state(); + let addr: SocketAddr = "127.0.0.1:55220".parse().unwrap_or_else(|_| unreachable!()); + let mut close_rx = app_state.connection_manager.get_close_connection_channel(); + + cleanup_forced_reopen_addrs(&app_state, "adaptive-owner", &[addr]).await; + + let signal = + tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()).await.ok().and_then(Result::ok); + assert!(signal.is_none(), "adaptive cleanup should not hard-close the previous client socket"); +} + +#[tokio::test] +async fn forced_reopen_cleanup_for_non_adaptive_streams_preserves_client_socket() { + let app_state = create_test_app_state(); + let addr: SocketAddr = "127.0.0.1:55221".parse().unwrap_or_else(|_| unreachable!()); + let mut close_rx = app_state.connection_manager.get_close_connection_channel(); + + cleanup_forced_reopen_addrs(&app_state, "live-owner", &[addr]).await; + + let signal = + tokio::time::timeout(std::time::Duration::from_millis(50), close_rx.recv()).await.ok().and_then(Result::ok); + assert!(signal.is_none(), "a reopen must not close unrelated requests on the same proxy socket"); +} + +#[tokio::test] +async fn test_channel_unavailable_fallback_does_not_register_body_owner_or_leak_slot() { + let app_state = create_test_provider_app_state(); + let provider_name = "provider_1".intern(); + let mut cfg = (**app_state.app_config.config.load()).clone(); + cfg.custom_stream_response_enabled = true; + app_state.app_config.config.store(std::sync::Arc::new(cfg)); + let addr: std::net::SocketAddr = "127.0.0.1:49080".parse().unwrap(); + + let custom_video = crate::model::CustomStreamResponse { + channel_unavailable: Some(crate::api::model::TransportStreamBuffer::new(b"channel-unavailable-bytes".to_vec())), + user_connections_exhausted: None, + provider_connections_exhausted: None, + low_priority_preempted: None, + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: Vec::new(), + }; + app_state.app_config.custom_stream_response.store(Some(std::sync::Arc::new(custom_video))); + + let handle = app_state + .active_provider + .acquire_connection(&provider_name, &addr, 0, tuliprox_session::ConnectionKind::Normal) + .expect("connection should be acquired"); + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 1); + + let (channel_unavail_stream, info) = + create_channel_unavailable_stream(&app_state.app_config, &[], axum::http::StatusCode::OK); + assert!(channel_unavail_stream.is_some()); + + let factory_response = tuliprox_session::ProviderStreamFactoryResponse { + stream: channel_unavail_stream.unwrap(), + info, + provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), + has_upstream_owner: false, + }; + + if factory_response.has_upstream_owner { + app_state.active_provider.register_body_owner(handle.allocation_id); + } + + app_state.active_provider.release_handle(&handle); + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + assert_eq!(app_state.active_provider.active_connections().map_or(0, |m| m.values().sum::()), 0); +} + +#[tokio::test] +async fn test_provider_open_cancelled_during_header_wait_releases_slot_without_leak() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + + let accepted = std::sync::Arc::new(tokio::sync::Notify::new()); + let accepted_clone = std::sync::Arc::clone(&accepted); + + let server_task = tokio::spawn(async move { + if let Ok((mut socket, _)) = listener.accept().await { + accepted_clone.notify_waiters(); + let mut buf = [0u8; 1024]; + loop { + use tokio::io::AsyncReadExt; + match socket.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + } + } + }); + + let app_state = create_test_provider_app_state(); + let provider_name = "provider_1".intern(); + let client_addr: std::net::SocketAddr = "127.0.0.1:49081".parse().unwrap(); + + let handle = app_state + .active_provider + .acquire_connection(&provider_name, &client_addr, 0, tuliprox_session::ConnectionKind::Normal) + .expect("connection should be acquired"); + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 1); + + let url = url::Url::parse(&format!("http://{local_addr}/stream.ts")).unwrap(); + let req_headers = axum::http::HeaderMap::new(); + let stream_options = get_stream_options(&app_state.app_config, StreamResponseMode::Stream); + let mut options = + crate::api::model::ProviderStreamFactoryOptions::new(&crate::api::model::ProviderStreamFactoryParams { + addr: client_addr, + item_type: PlaylistItemType::Live, + share_stream: false, + stream_options: &stream_options, + stream_url: &url, + req_headers: &req_headers, + input_headers: None, + session_headers: None, + disabled_headers: None, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: tuliprox_session::ProviderContentRepresentationMode::PreserveOrigin, + }); + options.set_provider_handle_tokens( + handle.cancel_token.clone(), + handle.completion_token.clone(), + Some(handle.close_reason.clone()), + ); + app_state.active_provider.mark_opening(handle.allocation_id); + + let ctx = app_state.provider_stream_ctx(); + let client = app_state.http_clients.default.load().as_ref().clone(); + + let open_task = + tokio::spawn(async move { crate::api::model::create_provider_stream(&ctx, &client, options).await }); + + // Wait until TCP connection is accepted and waiting for response headers + accepted.notified().await; + + // Now cancel the handle while it is waiting for response headers + handle.cancel_token.as_ref().unwrap().cancel(); + + // The open task should finish quickly because it is cancelled + let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), open_task) + .await + .expect("open task should abort upon cancellation") + .expect("task join"); + assert!(outcome.is_none()); + + // Release the handle + app_state.active_provider.release_handle(&handle); + + // Yield to allow background completion tasks to process + tokio::task::yield_now().await; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + assert_eq!(app_state.active_provider.active_connections().map_or(0, |m| m.values().sum::()), 0); + + let _ = server_task.await; +} diff --git a/backend/app/src/api/api_utils/tests/http_load.rs b/backend/app/src/api/api_utils/tests/http_load.rs new file mode 100644 index 000000000..d39e87bbe --- /dev/null +++ b/backend/app/src/api/api_utils/tests/http_load.rs @@ -0,0 +1,260 @@ +use super::{ + create_test_app_state_for_config, create_test_fingerprint, create_test_provider_app_config, + force_provider_stream_response, load_test_channel, load_test_rss_kib, load_test_session, load_test_user, + spawn_controlled_fake_origin, spawn_load_test_origin, validate_load_response, FakeOriginMode, + ForceStreamRequestContext, +}; +use crate::model::{ConfigInput, SourcesConfig}; +use arc_swap::ArcSwap; +use axum::{http::HeaderMap, response::IntoResponse}; +use http_body_util::BodyExt; +use shared::{model::InputType, utils::Internable}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; + +// Percentiles of a few thousand latency samples tolerate float rounding. +#[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation, clippy::cast_sign_loss)] +pub(in crate::api::api_utils::tests) fn load_test_latency_percentile(sorted: &[u64], p: f64) -> u64 { + if sorted.is_empty() { + return 0; + } + let index = ((sorted.len() as f64 - 1.0) * p).round() as usize; + sorted[index.min(sorted.len() - 1)] +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +#[ignore = "HTTP load test: cargo +stable test -p tuliprox --release --bin tuliprox -- --ignored --nocapture http_provider_lease_load_test"] +#[allow(clippy::too_many_lines, clippy::cast_precision_loss)] +async fn http_provider_lease_load_test() { + const TS_PACKET: [u8; 188] = [0x47; 188]; + let origin_addr = spawn_load_test_origin(&TS_PACKET).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 0, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + + // Warm-up so allocator and connection pools are exercised before measurement. + for index in 0..32u16 { + let addr = SocketAddr::from(([127, 0, 0, 1], 45_000 + index)); + let fingerprint = create_test_fingerprint(addr); + let session = load_test_session(&input, origin_addr, &format!("warmup-{index}"), addr); + let channel = load_test_channel(&input, origin_addr); + let response = force_provider_stream_response( + &fingerprint, + &app, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &load_test_user(&format!("warmup-user-{index}")), + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + let status = response.status(); + let body = response.into_body().collect().await.expect("warm-up body collects").to_bytes(); + validate_load_response(status, &body, &TS_PACKET).expect("warm-up response must match the fixture"); + } + + let baseline_rss_kib = load_test_rss_kib(); + eprintln!("HTTP provider lease load test (unlimited provider, real HTTP body, 5 repetitions per concurrency)"); + for concurrency in [1usize, 5, 50, 200] { + let rounds_per_task = 20; + let repetitions = 5; + let mut all_samples = Vec::<(u64, u64)>::new(); + let mut total_ops = 0u64; + let mut total_elapsed_secs = 0f64; + for rep in 0..repetitions { + let samples = Arc::new(std::sync::Mutex::new(Vec::<(u64, u64)>::new())); + let started = std::time::Instant::now(); + let mut tasks = tokio::task::JoinSet::new(); + for task_index in 0..concurrency { + let app = Arc::clone(&app); + let input = Arc::clone(&input); + let samples = Arc::clone(&samples); + let username = format!("load-user-{rep}-{task_index}"); + tasks.spawn(async move { + for round in 0..rounds_per_task { + let addr = SocketAddr::from(( + [127, 0, 0, 1], + 46_000 + u16::try_from(task_index).expect("load task index fits a port offset"), + )); + let fingerprint = create_test_fingerprint(addr); + let token = format!("load-{rep}-{task_index}-{round}"); + let session = load_test_session(&input, origin_addr, &token, addr); + let channel = load_test_channel(&input, origin_addr); + let user = load_test_user(&username); + + let acquire_at = std::time::Instant::now(); + let response = force_provider_stream_response( + &fingerprint, + &app, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + let acquire_us = u64::try_from(acquire_at.elapsed().as_micros()).unwrap_or(u64::MAX); + + let status = response.status(); + let body = response.into_body(); + let body_at = std::time::Instant::now(); + let collected = body.collect().await.expect("load body collects").to_bytes(); + validate_load_response(status, &collected, &TS_PACKET) + .expect("load response must match the TS packet"); + let body_us = u64::try_from(body_at.elapsed().as_micros()).unwrap_or(u64::MAX); + + samples.lock().unwrap().push((acquire_us, body_us)); + } + }); + } + while let Some(result) = tasks.join_next().await { + result.expect("load task must not fail"); + } + + total_elapsed_secs += started.elapsed().as_secs_f64(); + let rep_samples = Arc::try_unwrap(samples).expect("samples unique").into_inner().unwrap(); + total_ops += rep_samples.len() as u64; + all_samples.extend(rep_samples); + } + + let mut acquire = Vec::with_capacity(all_samples.len()); + let mut body = Vec::with_capacity(all_samples.len()); + for (a, b) in all_samples { + acquire.push(a); + body.push(b); + } + acquire.sort_unstable(); + body.sort_unstable(); + let throughput = total_ops as f64 / total_elapsed_secs.max(f64::EPSILON); + let byte_throughput = total_ops as f64 * TS_PACKET.len() as f64 / total_elapsed_secs.max(f64::EPSILON); + let rss_delta_kib = load_test_rss_kib() + .zip(baseline_rss_kib) + .map_or_else(|| "unsupported".to_string(), |(rss, baseline)| rss.saturating_sub(baseline).to_string()); + eprintln!( + "concurrency={concurrency:>3} ops={total_ops:>5} throughput={throughput:>9.1} ops/s {byte_throughput:>11.1} bytes/s | acquire p50/p95/p99={}/{}/{}us | body p50/p95/p99={}/{}/{}us | rss_delta_kib={rss_delta_kib}", + load_test_latency_percentile(&acquire, 0.50), + load_test_latency_percentile(&acquire, 0.95), + load_test_latency_percentile(&acquire, 0.99), + load_test_latency_percentile(&body, 0.50), + load_test_latency_percentile(&body, 0.95), + load_test_latency_percentile(&body, 0.99), + ); + let (user_requests, user_sessions) = app.active_users.playback_resource_counts().await; + let (shared_origins, shared_subscribers) = app.shared_stream_manager.resource_counts().await; + let shared_meters = app.shared_stream_manager.meter_count(); + let leases = app.active_provider.provider_lease_usage(&input.name); + eprintln!( + "[load resource] concurrency={concurrency} conns={} starting={} active={} idle={} user_requests={user_requests} user_sessions={user_sessions} shared_origins={shared_origins} shared_subscribers={shared_subscribers} shared_meters={shared_meters}", + app.active_provider.get_provider_connections_count(), + leases.starting, + leases.active, + leases.idle, + ); + } + + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let leases = app.active_provider.provider_lease_usage(&input.name); + let (user_requests, user_sessions) = app.active_users.playback_resource_counts().await; + let (shared_origins, shared_subscribers) = app.shared_stream_manager.resource_counts().await; + let shared_meters = app.shared_stream_manager.meter_count(); + if app.active_provider.get_provider_connections_count() == 0 + && leases.total() == 0 + && user_requests == 0 + && user_sessions == 0 + && shared_origins == 0 + && shared_subscribers == 0 + && shared_meters == 0 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("load resources must drain to baseline"); + assert_eq!(app.active_provider.get_provider_connections_count(), 0, "provider slots must return to baseline"); + let usage = app.active_provider.provider_lease_usage(&input.name); + assert_eq!(usage.total(), 0, "lease table must return to baseline after HTTP churn"); +} + +#[tokio::test] +async fn http_provider_lease_load_test_negative_control_detects_payload_corruption() { + const TS_PACKET_VALID: [u8; 188] = [0x47; 188]; + const TS_PACKET_CORRUPT: [u8; 188] = [0; 188]; + + let (origin_addr, origin_task) = + spawn_controlled_fake_origin(FakeOriginMode::FixedBytes(TS_PACKET_CORRUPT.to_vec()), 1).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_neg_ctrl".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 1, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + + let addr = SocketAddr::from(([127, 0, 0, 1], 55_650)); + let fingerprint = create_test_fingerprint(addr); + let session = load_test_session(&input, origin_addr, "neg-session", addr); + let channel = load_test_channel(&input, origin_addr); + let user = load_test_user("neg_ctrl_user"); + + let response = force_provider_stream_response( + &fingerprint, + &app, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + let status = response.status(); + let body = response.into_body().collect().await.expect("body collects").to_bytes(); + assert!( + validate_load_response(status, &body, &TS_PACKET_VALID).is_err(), + "negative control must be rejected by the same validator as the load harness" + ); + + let _ = origin_task.await; +} diff --git a/backend/app/src/api/api_utils/tests/http_smoke.rs b/backend/app/src/api/api_utils/tests/http_smoke.rs new file mode 100644 index 000000000..0dd175b0c --- /dev/null +++ b/backend/app/src/api/api_utils/tests/http_smoke.rs @@ -0,0 +1,201 @@ +use super::{ + create_test_app_state_for_config, create_test_fingerprint, create_test_provider_app_config, + force_provider_stream_response, load_test_channel, load_test_session, load_test_user, spawn_load_test_origin, + validate_load_response, ForceStreamRequestContext, +}; +use crate::{ + api::model::AppState, + model::{ConfigInput, SourcesConfig}, +}; +use arc_swap::ArcSwap; +use axum::{ + extract::{ConnectInfo, State}, + http::HeaderMap, + response::IntoResponse, + routing::get, + Router, +}; +use http_body_util::BodyExt; +use shared::{model::InputType, utils::Internable}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; +use tokio::net::TcpListener; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn http_provider_lease_smoke_test() { + const TS_PACKET: [u8; 188] = [0x47; 188]; + let origin_addr = spawn_load_test_origin(&TS_PACKET).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_smoke".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 0, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + + for concurrency in [1usize, 2] { + let rounds = 5; + let mut tasks = tokio::task::JoinSet::new(); + for task_idx in 0..concurrency { + let app = Arc::clone(&app); + let input = Arc::clone(&input); + let user = load_test_user(&format!("smoke-user-{task_idx}")); + tasks.spawn(async move { + for r in 0..rounds { + let addr = SocketAddr::from(( + [127, 0, 0, 1], + 48_000 + u16::try_from(task_idx).expect("smoke task index fits a port offset"), + )); + let fingerprint = create_test_fingerprint(addr); + let token = format!("smoke-{task_idx}-{r}"); + let session = load_test_session(&input, origin_addr, &token, addr); + let channel = load_test_channel(&input, origin_addr); + + let response = force_provider_stream_response( + &fingerprint, + &app, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + let status = response.status(); + let body = response.into_body().collect().await.expect("smoke body collects").to_bytes(); + validate_load_response(status, &body, &TS_PACKET).expect("smoke response must match the TS packet"); + } + }); + } + while let Some(res) = tasks.join_next().await { + res.expect("smoke task must succeed"); + } + } + + for _ in 0..50 { + if app.active_provider.get_provider_connections_count() == 0 + && app.active_provider.provider_lease_usage(&input.name).total() == 0 + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + assert_eq!(app.active_provider.get_provider_connections_count(), 0, "smoke slots baseline"); + assert_eq!(app.active_provider.provider_lease_usage(&input.name).total(), 0, "smoke lease baseline"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn http_provider_lease_downstream_smoke_test() { + #[derive(Clone)] + struct DownstreamState { + app: Arc, + input: Arc, + origin_addr: SocketAddr, + } + + async fn stream_handler( + State(state): State, + ConnectInfo(addr): ConnectInfo, + ) -> axum::response::Response { + let fingerprint = create_test_fingerprint(addr); + let user = load_test_user("downstream-user"); + let session = load_test_session(&state.input, state.origin_addr, "downstream-token", addr); + let channel = load_test_channel(&state.input, state.origin_addr); + force_provider_stream_response( + &fingerprint, + &state.app, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &state.input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response() + } + + const TS_PACKET: [u8; 188] = [0x47; 188]; + let origin_addr = spawn_load_test_origin(&TS_PACKET).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_downstream".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 0, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + + let router = Router::new().route("/stream", get(stream_handler)).with_state(DownstreamState { + app: Arc::clone(&app), + input: Arc::clone(&input), + origin_addr, + }); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("downstream listener binds"); + let downstream_addr = listener.local_addr().expect("downstream address"); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, router.into_make_service_with_connect_info::()).await; + }); + + let client = reqwest::Client::new(); + for concurrency in [1usize, 2] { + let mut tasks = tokio::task::JoinSet::new(); + for task_idx in 0..concurrency { + let client = client.clone(); + let url = format!("http://{downstream_addr}/stream"); + tasks.spawn(async move { + for round in 0..3 { + let response = client.get(&url).send().await.expect("downstream response"); + let status = response.status(); + let body = response.bytes().await.expect("downstream body"); + validate_load_response(status, &body, &TS_PACKET) + .unwrap_or_else(|err| panic!("downstream task {task_idx} round {round}: {err}")); + } + }); + } + while let Some(res) = tasks.join_next().await { + res.expect("downstream task must succeed"); + } + } + + for _ in 0..50 { + if app.active_provider.get_provider_connections_count() == 0 + && app.active_provider.provider_lease_usage(&input.name).total() == 0 + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert_eq!(app.active_provider.get_provider_connections_count(), 0, "downstream slots baseline"); + assert_eq!(app.active_provider.provider_lease_usage(&input.name).total(), 0, "downstream lease baseline"); + + server.abort(); +} diff --git a/backend/app/src/api/api_utils/tests/http_soak.rs b/backend/app/src/api/api_utils/tests/http_soak.rs new file mode 100644 index 000000000..becb3ca93 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/http_soak.rs @@ -0,0 +1,274 @@ +use super::{ + create_test_app_state_for_config, create_test_fingerprint, create_test_provider_app_config, + force_provider_stream_response, load_test_channel, load_test_rss_kib, load_test_session, load_test_user, + spawn_load_test_origin, validate_load_response, ForceStreamRequestContext, +}; +use crate::model::{ConfigInput, SourcesConfig}; +use arc_swap::ArcSwap; +use axum::{ + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; +use http_body_util::BodyExt; +use shared::{model::InputType, utils::Internable}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; + +#[derive(Debug, Clone)] +pub(in crate::api::api_utils::tests) struct SoakResourceSample { + pub(in crate::api::api_utils::tests) elapsed_secs: f64, + pub(in crate::api::api_utils::tests) rss_kib: Option, + pub(in crate::api::api_utils::tests) provider_connections: usize, + pub(in crate::api::api_utils::tests) lease_starting: usize, + pub(in crate::api::api_utils::tests) lease_active: usize, + pub(in crate::api::api_utils::tests) lease_idle: usize, + pub(in crate::api::api_utils::tests) user_requests: usize, + pub(in crate::api::api_utils::tests) user_sessions: usize, + pub(in crate::api::api_utils::tests) shared_origins: usize, + pub(in crate::api::api_utils::tests) shared_subscribers: usize, + pub(in crate::api::api_utils::tests) shared_meters: usize, + pub(in crate::api::api_utils::tests) running_tasks: usize, + pub(in crate::api::api_utils::tests) completed: u64, + pub(in crate::api::api_utils::tests) planned_aborted: u64, + pub(in crate::api::api_utils::tests) unexpected_failed: u64, +} + +pub(in crate::api::api_utils::tests) fn median_u64(values: &mut [u64]) -> Option { + if values.is_empty() { + return None; + } + values.sort_unstable(); + Some(values[values.len() / 2]) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +#[ignore = "30-minute soak: TULIPROX_SOAK_SECS=60 cargo +stable test -p tuliprox --release --bin tuliprox -- --ignored --nocapture http_provider_lease_soak_test"] +#[allow(clippy::too_many_lines)] +async fn http_provider_lease_soak_test() { + const SOAK_PACKET: [u8; 188] = [0x47; 188]; + let duration_secs: u64 = std::env::var("TULIPROX_SOAK_SECS").ok().and_then(|v| v.parse().ok()).unwrap_or(1800); + let origin_addr = spawn_load_test_origin(&SOAK_PACKET).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 0, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + + let baseline_rss_kib = load_test_rss_kib(); + let completed = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let aborted = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let unexpected_failed = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let peak_connections = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let running_tasks = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let resource_samples = Arc::new(std::sync::Mutex::new(Vec::::new())); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(duration_secs); + let sampler_stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let sampler_stop_task = Arc::clone(&sampler_stop); + let app_sampler = Arc::clone(&app); + let sampler_input_name = Arc::clone(&input.name); + let sampler_completed = Arc::clone(&completed); + let sampler_aborted = Arc::clone(&aborted); + let sampler_failed = Arc::clone(&unexpected_failed); + let sampler_running = Arc::clone(&running_tasks); + let sampler_samples = Arc::clone(&resource_samples); + let sampler_task = tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(5)); + let start_time = std::time::Instant::now(); + while !sampler_stop_task.load(std::sync::atomic::Ordering::Relaxed) { + interval.tick().await; + let rss = load_test_rss_kib().map_or_else(|| "unsupported".to_string(), |rss| rss.to_string()); + let conns = app_sampler.active_provider.get_provider_connections_count(); + let leases = app_sampler.active_provider.provider_lease_usage(&sampler_input_name); + let (user_requests, user_sessions) = app_sampler.active_users.playback_resource_counts().await; + let (shared_origins, shared_subscribers) = app_sampler.shared_stream_manager.resource_counts().await; + let shared_meters = app_sampler.shared_stream_manager.meter_count(); + let sample = SoakResourceSample { + elapsed_secs: start_time.elapsed().as_secs_f64(), + rss_kib: load_test_rss_kib(), + provider_connections: conns, + lease_starting: leases.starting, + lease_active: leases.active, + lease_idle: leases.idle, + user_requests, + user_sessions, + shared_origins, + shared_subscribers, + shared_meters, + running_tasks: sampler_running.load(std::sync::atomic::Ordering::Relaxed), + completed: sampler_completed.load(std::sync::atomic::Ordering::Relaxed), + planned_aborted: sampler_aborted.load(std::sync::atomic::Ordering::Relaxed), + unexpected_failed: sampler_failed.load(std::sync::atomic::Ordering::Relaxed), + }; + eprintln!( + "[soak time_series] t={:4.1}s rss_kib={} conns={} starting={} active={} idle={} user_requests={} user_sessions={} shared_origins={} shared_subscribers={} shared_meters={} running_tasks={} completed={} planned_aborted={} unexpected_failed={}", + sample.elapsed_secs, + rss, + sample.provider_connections, + sample.lease_starting, + sample.lease_active, + sample.lease_idle, + sample.user_requests, + sample.user_sessions, + sample.shared_origins, + sample.shared_subscribers, + sample.shared_meters, + sample.running_tasks, + sample.completed, + sample.planned_aborted, + sample.unexpected_failed, + ); + sampler_samples.lock().unwrap().push(sample); + } + }); + + let mut tasks = tokio::task::JoinSet::new(); + for task_index in 0..16u16 { + let app = Arc::clone(&app); + let input = Arc::clone(&input); + let completed = Arc::clone(&completed); + let aborted = Arc::clone(&aborted); + let unexpected_failed = Arc::clone(&unexpected_failed); + let peak_connections = Arc::clone(&peak_connections); + let running_tasks = Arc::clone(&running_tasks); + tasks.spawn(async move { + running_tasks.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let result = async { + let addr = SocketAddr::from(([127, 0, 0, 1], 47_000 + task_index)); + let fingerprint = create_test_fingerprint(addr); + let username = format!("soak-user-{task_index}"); + let user = load_test_user(&username); + let token = format!("soak-reconnect-family-{task_index}"); + let mut round = 0u64; + while std::time::Instant::now() < deadline { + round += 1; + let session = load_test_session(&input, origin_addr, &token, addr); + let channel = load_test_channel(&input, origin_addr); + let response = force_provider_stream_response( + &fingerprint, + &app, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 2, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + let status = response.status(); + if status != StatusCode::OK { + unexpected_failed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return Err(format!("soak response status was {status}")); + } + // Alternate full playback (collect) and abort (drop without consuming). + if round.is_multiple_of(2) { + let collected = match response.into_body().collect().await { + Ok(body) => body.to_bytes(), + Err(err) => { + unexpected_failed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return Err(format!("soak body collection failed: {err}")); + } + }; + if let Err(err) = validate_load_response(status, &collected, &SOAK_PACKET) { + unexpected_failed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return Err(err); + } + completed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } else { + drop(response); + aborted.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + + let current = app.active_provider.get_provider_connections_count(); + peak_connections.fetch_max(current, std::sync::atomic::Ordering::Relaxed); + } + Ok::<(), String>(()) + } + .await; + running_tasks.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + result + }); + } + while let Some(result) = tasks.join_next().await { + result.expect("soak task must join").unwrap_or_else(|err| panic!("soak task failed: {err}")); + } + + sampler_stop.store(true, std::sync::atomic::Ordering::Relaxed); + let _ = sampler_task.await; + + let completed = completed.load(std::sync::atomic::Ordering::Relaxed); + let aborted = aborted.load(std::sync::atomic::Ordering::Relaxed); + let peak = peak_connections.load(std::sync::atomic::Ordering::Relaxed); + let unexpected_failed = unexpected_failed.load(std::sync::atomic::Ordering::Relaxed); + let rss_delta_kib = load_test_rss_kib() + .zip(baseline_rss_kib) + .map_or_else(|| "unsupported".to_string(), |(rss, baseline)| rss.saturating_sub(baseline).to_string()); + eprintln!( + "soak duration={duration_secs}s completed={completed} planned_aborted={aborted} unexpected_failed={unexpected_failed} peak_connections={peak} rss_delta_kib={rss_delta_kib}" + ); + + let samples = Arc::try_unwrap(resource_samples).expect("resource samples unique").into_inner().unwrap(); + assert!(!samples.is_empty(), "soak must record at least one resource sample"); + let third = (samples.len() / 3).max(1); + let middle_start = third.min(samples.len()); + let middle_end = (third.saturating_mul(2)).min(samples.len()); + let last_start = middle_end.min(samples.len()); + let mut middle_rss = + samples[middle_start..middle_end].iter().filter_map(|sample| sample.rss_kib).collect::>(); + let mut last_rss = samples[last_start..].iter().filter_map(|sample| sample.rss_kib).collect::>(); + match (median_u64(&mut middle_rss), median_u64(&mut last_rss)) { + (Some(middle), Some(last)) => { + let delta = i128::from(last) - i128::from(middle); + eprintln!("soak rss rolling-median middle_third={middle}KiB last_third={last}KiB delta={delta}KiB"); + } + _ => eprintln!("soak rss rolling-median unsupported on this platform"), + } + + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let leases = app.active_provider.provider_lease_usage(&input.name); + let (user_requests, user_sessions) = app.active_users.playback_resource_counts().await; + let (shared_origins, shared_subscribers) = app.shared_stream_manager.resource_counts().await; + let shared_meters = app.shared_stream_manager.meter_count(); + if app.active_provider.get_provider_connections_count() == 0 + && leases.total() == 0 + && user_requests == 0 + && user_sessions == 0 + && shared_origins == 0 + && shared_subscribers == 0 + && shared_meters == 0 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("soak resources must drain to baseline"); + + assert_eq!( + app.active_provider.get_provider_connections_count(), + 0, + "provider slots must return to baseline after soak" + ); + let usage = app.active_provider.provider_lease_usage(&input.name); + assert_eq!(usage.total(), 0, "lease table must return to baseline after soak"); + assert_eq!(unexpected_failed, 0, "soak must not hide failed operations"); + assert!(peak <= 16, "active connections must stay bounded by the churn workers, observed {peak}"); +} diff --git a/backend/app/src/api/api_utils/tests/local_files.rs b/backend/app/src/api/api_utils/tests/local_files.rs new file mode 100644 index 000000000..929201e30 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/local_files.rs @@ -0,0 +1,447 @@ +use super::{ + create_session_fingerprint, create_test_app_config, create_test_app_state, create_test_app_state_for_config, + create_test_fingerprint, create_test_local_channel, local_stream_response, +}; +use crate::model::{Config, ConfigInput, ConfigTarget, ProxyUserCredentials}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use axum::{ + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; +use bytes::Bytes; +use shared::{ + foundation::Filter, + model::{InputType, PlaylistItemType, ProcessingOrder, UserConnectionPermission}, +}; +use std::sync::Arc; +use tuliprox_core::utils::response_compression::should_compress_response; + +#[tokio::test] +async fn local_stream_response_registers_active_local_stream() { + let app_state = create_test_app_state(); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let file_path = temp_dir.path().join("local-test.mkv"); + tokio::fs::write(&file_path, Bytes::from_static(b"local-stream")).await.expect("write local file"); + + let addr = "127.0.0.1:55123".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + let channel = create_test_local_channel(&format!("file://{}", file_path.display())); + let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; + let user = ProxyUserCredentials::default(); + let target = ConfigTarget { + curation: None, + id: 1, + enabled: true, + name: "test".to_string(), + options: None, + sort: None, + filter: Filter::default().into(), + output: Vec::new(), + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::default()), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }; + + let _response = local_stream_response( + &fingerprint, + &app_state, + channel, + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + None, + None, + false, + ) + .await + .into_response(); + + let active_streams = app_state.active_users.active_streams().await; + assert_eq!(active_streams.len(), 1, "local file streaming should register an active stream"); + assert_eq!(active_streams[0].channel.item_type, PlaylistItemType::LocalVideo); +} + +#[tokio::test] +async fn local_stream_response_rechecks_limits_before_registering_socket_bound_streams() { + let mut app_cfg = create_test_app_config(); + let config = Config { user_access_control: true, ..Config::default() }; + app_cfg.config = Arc::new(ArcSwap::from_pointee(config)); + let app_state = create_test_app_state_for_config(Arc::new(app_cfg)); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let file_path = temp_dir.path().join("local-race-test.mkv"); + tokio::fs::write(&file_path, Bytes::from_static(b"local-stream")).await.expect("write local file"); + + let first_addr = "127.0.0.1:55131".parse().unwrap_or_else(|_| unreachable!()); + let second_addr = "127.0.0.1:55132".parse().unwrap_or_else(|_| unreachable!()); + let first_fingerprint = create_test_fingerprint(first_addr); + let second_fingerprint = create_test_fingerprint(second_addr); + let channel = create_test_local_channel(&format!("file://{}", file_path.display())); + let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; + let mut user = ProxyUserCredentials::default(); + user.username = "local-limit-user".to_string(); + user.max_connections = 1; + let target = ConfigTarget { + curation: None, + id: 1, + enabled: true, + name: "test".to_string(), + options: None, + sort: None, + filter: Filter::default().into(), + output: Vec::new(), + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::default()), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }; + let first_token = create_session_fingerprint(&first_fingerprint, &user.username, channel.virtual_id, true); + let second_token = create_session_fingerprint(&second_fingerprint, &user.username, channel.virtual_id, true); + + let _first_response = local_stream_response( + &first_fingerprint, + &app_state, + channel.clone(), + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + Some(&first_token), + None, + false, + ) + .await + .into_response(); + + let _second_response = local_stream_response( + &second_fingerprint, + &app_state, + channel, + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + Some(&second_token), + None, + false, + ) + .await + .into_response(); + + assert_eq!(app_state.active_users.user_connections(&user.username).await, 1); + assert_eq!(app_state.active_users.active_streams().await.len(), 1); + assert_eq!( + app_state + .active_users + .connection_admission_for_session( + &user.username, + user.max_connections, + user.soft_connections, + &second_token + ) + .await + .permission(), + UserConnectionPermission::Exhausted, + "failed second open must not leave a placeholder session that bypasses admission" + ); +} + +#[tokio::test] +async fn local_stream_response_disables_response_compression() { + let app_state = create_test_app_state(); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let file_path = temp_dir.path().join("local-test.mkv"); + tokio::fs::write(&file_path, Bytes::from_static(b"local-stream")).await.expect("write local file"); + + let addr = "127.0.0.1:55124".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + let channel = create_test_local_channel(&format!("file://{}", file_path.display())); + let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; + let user = ProxyUserCredentials::default(); + let target = ConfigTarget { + curation: None, + id: 1, + enabled: true, + name: "test".to_string(), + options: None, + sort: None, + filter: Filter::default().into(), + output: Vec::new(), + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::default()), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }; + + let response = local_stream_response( + &fingerprint, + &app_state, + channel, + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + None, + None, + false, + ) + .await + .into_response(); + + assert!(!should_compress_response(&response)); +} + +#[tokio::test] +async fn local_stream_response_reuses_stable_playback_session_token_across_reopens() { + let app_state = create_test_app_state(); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let file_path = temp_dir.path().join("local-test.mkv"); + tokio::fs::write(&file_path, Bytes::from_static(b"local-stream")).await.expect("write local file"); + + let channel = create_test_local_channel(&format!("file://{}", file_path.display())); + let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; + let user = ProxyUserCredentials::default(); + let target = ConfigTarget { + curation: None, + id: 1, + enabled: true, + name: "test".to_string(), + options: None, + sort: None, + filter: Filter::default().into(), + output: Vec::new(), + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::default()), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }; + let playback_session_token = "local-playback-token"; + + let first_fingerprint = create_test_fingerprint("127.0.0.1:55125".parse().unwrap_or_else(|_| unreachable!())); + let second_fingerprint = create_test_fingerprint("127.0.0.1:55126".parse().unwrap_or_else(|_| unreachable!())); + + let _first_response = local_stream_response( + &first_fingerprint, + &app_state, + channel.clone(), + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + Some(playback_session_token), + None, + false, + ) + .await + .into_response(); + + let _second_response = local_stream_response( + &second_fingerprint, + &app_state, + channel, + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + Some(playback_session_token), + None, + false, + ) + .await + .into_response(); + + let active_streams = app_state.active_users.active_streams().await; + assert_eq!(active_streams.len(), 1, "stable playback token should reuse the tracked local connection"); + assert_eq!(active_streams[0].session_token.as_deref(), Some(playback_session_token)); + assert_eq!(active_streams[0].addr, second_fingerprint.addr); +} + +#[tokio::test] +async fn local_stream_response_allows_exhausted_reopen_for_same_playback_session_token() { + let app_state = create_test_app_state(); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let file_path = temp_dir.path().join("local-test.mkv"); + tokio::fs::write(&file_path, Bytes::from_static(b"local-stream")).await.expect("write local file"); + + let channel = create_test_local_channel(&format!("file://{}", file_path.display())); + let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; + let mut user = ProxyUserCredentials::default(); + user.username = "user1".to_string(); + user.max_connections = 1; + let target = ConfigTarget { + curation: None, + id: 1, + enabled: true, + name: "test".to_string(), + options: None, + sort: None, + filter: Filter::default().into(), + output: Vec::new(), + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::default()), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }; + let playback_session_token = "local-playback-token"; + + let first_fingerprint = create_test_fingerprint("127.0.0.1:55127".parse().unwrap_or_else(|_| unreachable!())); + let second_fingerprint = create_test_fingerprint("127.0.0.1:55128".parse().unwrap_or_else(|_| unreachable!())); + + let _first_response = local_stream_response( + &first_fingerprint, + &app_state, + channel.clone(), + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + Some(playback_session_token), + None, + false, + ) + .await + .into_response(); + + let second_response = local_stream_response( + &second_fingerprint, + &app_state, + channel, + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Exhausted, + crate::api::model::ConnectionKind::Normal, + Some(playback_session_token), + None, + false, + ) + .await + .into_response(); + + assert_eq!(second_response.status(), StatusCode::OK); + + let active_streams = app_state.active_users.active_streams().await; + assert_eq!(active_streams.len(), 1); + assert_eq!(active_streams[0].session_token.as_deref(), Some(playback_session_token)); + assert_eq!(active_streams[0].addr, second_fingerprint.addr); +} + +#[tokio::test] +async fn local_stream_response_preserves_soft_kind_across_reopens() { + let app_state = create_test_app_state(); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let file_path = temp_dir.path().join("local-soft-test.mkv"); + tokio::fs::write(&file_path, Bytes::from_static(b"local-stream")).await.expect("write local file"); + + let channel = create_test_local_channel(&format!("file://{}", file_path.display())); + let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; + let mut user = ProxyUserCredentials::default(); + user.username = "soft-local-user".to_string(); + user.max_connections = 1; + user.soft_connections = 1; + let target = ConfigTarget { + curation: None, + id: 1, + enabled: true, + name: "test".to_string(), + options: None, + sort: None, + filter: Filter::default().into(), + output: Vec::new(), + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::default()), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }; + let playback_session_token = "local-soft-playback-token"; + + let first_fingerprint = create_test_fingerprint("127.0.0.1:55129".parse().unwrap_or_else(|_| unreachable!())); + let second_fingerprint = create_test_fingerprint("127.0.0.1:55130".parse().unwrap_or_else(|_| unreachable!())); + + let _first_response = local_stream_response( + &first_fingerprint, + &app_state, + channel.clone(), + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Soft, + Some(playback_session_token), + None, + false, + ) + .await + .into_response(); + + let second_response = local_stream_response( + &second_fingerprint, + &app_state, + channel, + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Exhausted, + crate::api::model::ConnectionKind::Normal, + Some(playback_session_token), + None, + false, + ) + .await + .into_response(); + + assert_eq!(second_response.status(), StatusCode::OK); + + let session_admission = app_state + .active_users + .connection_admission_for_session( + &user.username, + user.max_connections, + user.soft_connections, + playback_session_token, + ) + .await; + assert_eq!(session_admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); +} diff --git a/backend/app/src/api/api_utils/tests/network_access.rs b/backend/app/src/api/api_utils/tests/network_access.rs new file mode 100644 index 000000000..988aab5b3 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/network_access.rs @@ -0,0 +1,296 @@ +use super::{ + evaluate_network_access, mock_geoip, user_with_network_access, NetworkAccessDecision, NetworkAccessDenyReason, +}; +use crate::{model::NetworkAccess, repository::GeoIp}; +use arc_swap::ArcSwapOption; +use shared::model::GeoIpUnavailablePolicy; +use std::sync::Arc; + +// ========================================================================================= +// evaluate_network_access tests +// ========================================================================================= + +/// Run `evaluate_network_access` for a synthetic user built from +/// `network_access` and assert the decision matches `expected`. Centralizes +/// the boilerplate (`user_with_network_access` + geoip setup + call + +/// assert) shared by every `evaluate_network_access` test below. +pub(in crate::api::api_utils::tests) fn assert_network_decision( + network_access: Option, + geoip: &Arc>, + ip: &str, + expected: NetworkAccessDecision, +) { + let user = user_with_network_access(network_access); + assert_eq!(evaluate_network_access(&user, ip, geoip, GeoIpUnavailablePolicy::Deny), expected); +} + +/// `Arc>` with no `GeoIP` database loaded. +pub(in crate::api::api_utils::tests) fn empty_geoip() -> Arc> { + Arc::new(ArcSwapOption::::default()) +} + +#[test] +fn no_config_allows_all() { + assert_network_decision(None, &empty_geoip(), "192.168.1.1", NetworkAccessDecision::Allowed); +} + +#[test] +fn empty_config_allows_all() { + assert_network_decision( + Some(NetworkAccess { allowed_countries: vec![], allowed_networks: vec![] }), + &empty_geoip(), + "192.168.1.1", + NetworkAccessDecision::Allowed, + ); +} + +#[test] +fn cidr_match_allows() { + assert_network_decision( + Some(NetworkAccess { allowed_countries: vec![], allowed_networks: vec!["192.168.1.0/24".parse().unwrap()] }), + &empty_geoip(), + "192.168.1.42", + NetworkAccessDecision::Allowed, + ); +} + +#[test] +fn cidr_miss_denies() { + assert_network_decision( + Some(NetworkAccess { allowed_countries: vec![], allowed_networks: vec!["192.168.1.0/24".parse().unwrap()] }), + &empty_geoip(), + "10.0.0.1", + NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCidrMatch), + ); +} + +#[test] +fn country_match_allows() { + assert_network_decision( + Some(NetworkAccess { allowed_countries: vec!["DE".to_string()], allowed_networks: vec![] }), + &mock_geoip("DE"), + "8.8.8.8", + NetworkAccessDecision::Allowed, + ); +} + +#[test] +fn country_miss_denies() { + assert_network_decision( + Some(NetworkAccess { allowed_countries: vec!["DE".to_string()], allowed_networks: vec![] }), + &mock_geoip("US"), + "8.8.8.8", + NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCountryMatch), + ); +} + +#[test] +fn no_geoip_denies_on_country_restriction() { + assert_network_decision( + Some(NetworkAccess { allowed_countries: vec!["DE".to_string()], allowed_networks: vec![] }), + &empty_geoip(), + "8.8.8.8", + NetworkAccessDecision::Denied(NetworkAccessDenyReason::GeoIpUnavailable), + ); +} + +#[test] +fn ipv4_vs_ipv6_denies_gracefully() { + assert_network_decision( + Some(NetworkAccess { allowed_countries: vec![], allowed_networks: vec!["2001:db8::/32".parse().unwrap()] }), + &empty_geoip(), + "192.168.1.1", + NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCidrMatch), + ); +} + +#[test] +fn ipv6_vs_ipv4_denies_gracefully() { + assert_network_decision( + Some(NetworkAccess { allowed_countries: vec![], allowed_networks: vec!["192.168.1.0/24".parse().unwrap()] }), + &empty_geoip(), + "2001:db8::1", + NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCidrMatch), + ); +} + +#[test] +fn either_cidr_or_country_match_allows() { + assert_network_decision( + Some(NetworkAccess { + allowed_countries: vec!["US".to_string()], + allowed_networks: vec!["192.168.1.0/24".parse().unwrap()], + }), + &mock_geoip("DE"), + "192.168.1.42", + NetworkAccessDecision::Allowed, + ); +} + +#[test] +fn single_ip_cidr() { + let user = user_with_network_access(Some(NetworkAccess { + allowed_countries: vec![], + allowed_networks: vec!["192.168.1.1/32".parse().unwrap()], + })); + let geoip = empty_geoip(); + assert_eq!( + evaluate_network_access(&user, "192.168.1.1", &geoip, GeoIpUnavailablePolicy::Deny), + NetworkAccessDecision::Allowed + ); + assert_eq!( + evaluate_network_access(&user, "192.168.1.2", &geoip, GeoIpUnavailablePolicy::Deny), + NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCidrMatch) + ); +} + +// ========================================================================================= +// network denied reason tests +// ========================================================================================= + +// The three `network_denied_reason_*` cases remain because they cover the +// `NetworkAccessDenyReason`-focused API surface directly, while +// `cidr_miss_denies`, `country_miss_denies`, and +// `no_geoip_denies_on_country_restriction` above assert the same deny +// reasons through broader `evaluate_network_access(...)` behavior. The +// overlap is intentional so both the general decision path and the +// reason-reporting-focused path stay pinned by tests. + +#[test] +fn network_denied_reason_country_unknown_when_geoip_loaded_but_unknown_ip() { + // GeoIP is loaded (not None), but lookup returns None for this IP (private/unknown). + // We need a GeoIP that only covers a private range, so public IPs get None. + // Use a CIDR-only restriction (no country rules) so we can verify + // that when countries ARE checked, lookup None gives "country_unknown". + let user = user_with_network_access(Some(NetworkAccess { + allowed_countries: vec!["DE".to_string()], + allowed_networks: vec!["192.168.1.0/24".parse().unwrap()], // miss CIDR first + })); + // Use the real GeoIp::new() which only seeds private ranges. + // For 8.8.8.8 (public), lookup returns None. + let geoip = Arc::new(ArcSwapOption::from(Some(Arc::new(GeoIp::new())))); + // CIDR miss -> country check -> geoip loaded but lookup returns None + assert_eq!( + evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Deny), + NetworkAccessDecision::Denied(NetworkAccessDenyReason::CountryUnknown) + ); +} + +#[test] +fn network_denied_reason_none_when_no_config() { + let user = user_with_network_access(None); + let geoip = Arc::new(ArcSwapOption::::default()); + assert_eq!( + evaluate_network_access(&user, "192.168.1.1", &geoip, GeoIpUnavailablePolicy::Deny), + NetworkAccessDecision::Allowed + ); +} + +// ========================================================================================= +// GeoIP unavailable policy tests +// ========================================================================================= + +#[test] +fn geoip_unavailable_default_deny_denies() { + // Country rule exists but GeoIP is unavailable — default policy is Deny + let user = user_with_network_access(Some(NetworkAccess { + allowed_countries: vec!["DE".to_string()], + allowed_networks: vec![], + })); + let geoip = Arc::new(ArcSwapOption::::default()); + let decision = evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Deny); + assert_eq!(decision, NetworkAccessDecision::Denied(NetworkAccessDenyReason::GeoIpUnavailable)); +} + +#[test] +fn geoip_unavailable_explicit_allow_allows() { + // Country rule exists, GeoIP unavailable, but policy is Allow — allows + let user = user_with_network_access(Some(NetworkAccess { + allowed_countries: vec!["DE".to_string()], + allowed_networks: vec![], + })); + let geoip = Arc::new(ArcSwapOption::::default()); + let decision = evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Allow); + assert_eq!(decision, NetworkAccessDecision::AllowedGeoIpUnavailable); +} + +#[test] +fn geoip_unavailable_cidr_only_still_denies() { + // CIDR only rules, no match — should deny even with Allow policy + let user = user_with_network_access(Some(NetworkAccess { + allowed_countries: vec![], + allowed_networks: vec!["192.168.1.0/24".parse().unwrap()], + })); + let geoip = Arc::new(ArcSwapOption::::default()); + let decision = evaluate_network_access(&user, "10.0.0.1", &geoip, GeoIpUnavailablePolicy::Allow); + assert_eq!(decision, NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCidrMatch)); +} + +#[test] +fn geoip_unavailable_cidr_match_allows() { + // CIDR match always allows, regardless of policy + let user = user_with_network_access(Some(NetworkAccess { + allowed_countries: vec!["DE".to_string()], + allowed_networks: vec!["10.0.0.0/8".parse().unwrap()], + })); + let geoip = Arc::new(ArcSwapOption::::default()); + let decision = evaluate_network_access(&user, "10.0.0.1", &geoip, GeoIpUnavailablePolicy::Deny); + assert_eq!(decision, NetworkAccessDecision::Allowed); +} + +#[test] +fn geoip_loaded_country_mismatch_still_denies() { + // Loaded GeoIP but country doesn't match — should deny under Allow policy + let user = user_with_network_access(Some(NetworkAccess { + allowed_countries: vec!["DE".to_string()], + allowed_networks: vec![], + })); + let mock_geoip = Arc::new(ArcSwapOption::from(Some(Arc::new(GeoIp::test_new("US"))))); + let decision = evaluate_network_access(&user, "8.8.8.8", &mock_geoip, GeoIpUnavailablePolicy::Allow); + assert_eq!(decision, NetworkAccessDecision::Denied(NetworkAccessDenyReason::NoCountryMatch)); +} + +#[test] +fn geoip_loaded_unknown_country_still_denies() { + // Loaded GeoIP but lookup returns None — should deny under Allow policy + let user = user_with_network_access(Some(NetworkAccess { + allowed_countries: vec!["DE".to_string()], + allowed_networks: vec!["192.168.1.0/24".parse().unwrap()], + })); + let geoip = Arc::new(ArcSwapOption::from(Some(Arc::new(GeoIp::new())))); + let decision = evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Allow); + assert_eq!(decision, NetworkAccessDecision::Denied(NetworkAccessDenyReason::CountryUnknown)); +} + +#[test] +fn malformed_ip_denies_even_when_geoip_unavailable_policy_is_allow() { + let user = user_with_network_access(Some(NetworkAccess::from(&shared::model::NetworkAccessDto { + allowed_countries: Some(vec!["DE".to_string()]), + allowed_networks: None, + }))); + let geoip = Arc::new(ArcSwapOption::::default()); + + let decision = evaluate_network_access(&user, "not-an-ip", &geoip, GeoIpUnavailablePolicy::Allow); + + assert_eq!(decision, NetworkAccessDecision::Denied(NetworkAccessDenyReason::MalformedClientIp)); +} + +#[test] +fn evaluate_network_access_respects_allow_policy() { + // verify evaluate_network_access returns AllowedGeoIpUnavailable with Allow policy + let user = user_with_network_access(Some(NetworkAccess { + allowed_countries: vec!["DE".to_string()], + allowed_networks: vec![], + })); + let geoip = Arc::new(ArcSwapOption::::default()); + // Default deny policy should return Denied + assert_eq!( + evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Deny), + NetworkAccessDecision::Denied(NetworkAccessDenyReason::GeoIpUnavailable) + ); + // Allow policy should return AllowedGeoIpUnavailable + assert_eq!( + evaluate_network_access(&user, "8.8.8.8", &geoip, GeoIpUnavailablePolicy::Allow), + NetworkAccessDecision::AllowedGeoIpUnavailable + ); +} diff --git a/backend/app/src/api/api_utils/tests/provider_leases.rs b/backend/app/src/api/api_utils/tests/provider_leases.rs new file mode 100644 index 000000000..124de44a2 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/provider_leases.rs @@ -0,0 +1,553 @@ +use super::{ + create_test_app_state_for_config, create_test_fingerprint, create_test_live_channel, create_test_local_channel, + create_test_provider_app_config, force_provider_stream_response, get_session_reservation_ttl_secs, load_test_user, + redirect_response, spawn_legacy_hls_test_origin, spawn_range_aware_test_origin, ForceStreamRequestContext, + RedirectParams, +}; +use crate::{ + api::model::UserSession, + model::{Config, ConfigInput, ConfigInputAlias, ConfigTarget, ProxyUserCredentials, SourcesConfig}, +}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use axum::{ + http::{header, HeaderMap, HeaderValue, StatusCode}, + response::IntoResponse, +}; +use http_body_util::BodyExt; +use shared::{ + defaults::default_hls_session_ttl_secs, + foundation::Filter, + model::{ + InputFetchMethod, InputType, PlaylistEntry, PlaylistItem, PlaylistItemHeader, PlaylistItemType, + ProcessingOrder, ProxyType, TargetType, UserConnectionPermission, XtreamCluster, + }, + utils::Internable, +}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; +use url::Url; + +#[tokio::test] +async fn forced_reopen_stays_on_pinned_provider_account() { + const ACCOUNT_A_BODY: &[u8] = b"account-a-marker"; + const ACCOUNT_B_BODY: &[u8] = b"account-b-marker"; + + let head_a = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + ACCOUNT_A_BODY.len() + ); + let head_b = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + ACCOUNT_B_BODY.len() + ); + let (origin_a, task_a) = spawn_legacy_hls_test_origin(head_a, ACCOUNT_A_BODY.to_vec()).await; + let (origin_b, task_b) = spawn_legacy_hls_test_origin(head_b, ACCOUNT_B_BODY.to_vec()).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_a}"), + username: Some("user-a".to_string()), + password: Some("pass-a".to_string()), + enabled: true, + priority: 0, + max_connections: 1, + method: InputFetchMethod::default(), + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "provider_2".intern(), + url: format!("http://{origin_b}"), + username: Some("user-b".to_string()), + password: Some("pass-b".to_string()), + priority: 1, + max_connections: 1, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }); + let app_config = Arc::new(create_test_provider_app_config()); + app_config.sources.store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(app_config); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_400)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer".to_string(); + let session = UserSession { + token: "stickiness-token".to_string(), + transition_version: 1, + virtual_id: 41, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_a}/live/1.ts").intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + let mut stream_channel = create_test_live_channel(&format!("http://{origin_a}/live/1.ts")); + stream_channel.provider_id = 1; + stream_channel.input_name = Arc::clone(&input.name); + stream_channel.item_type = PlaylistItemType::Catchup; + stream_channel.cluster = XtreamCluster::Live; + stream_channel.url = session.stream_url.clone(); + + let response = force_provider_stream_response( + &fingerprint, + &app_state, + &session, + stream_channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::OK); + let body = response.into_body().collect().await.expect("stream body").to_bytes(); + assert_eq!(body.as_ref(), ACCOUNT_A_BODY, "forced reopen must stay on the pinned provider account A"); + + let request_a = task_a.await.expect("origin A task completes").to_ascii_lowercase(); + assert!(!request_a.is_empty(), "origin A must have received the seek request"); + + let request_b = tokio::time::timeout(std::time::Duration::from_millis(250), task_b).await; + assert!(request_b.is_err(), "origin B must not receive a seek from an A-affine session"); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn forced_series_reopen_uses_free_pool_account_and_updates_session_pin() { + const ACCOUNT_B_BODY: &[u8] = b"account-b-marker"; + + let head_a = "HTTP/1.1 200 OK\r\nContent-Length: 1\r\nConnection: close\r\n\r\na".to_string(); + let head_b = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/x-matroska\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + ACCOUNT_B_BODY.len() + ); + let (origin_a, mut task_a) = spawn_legacy_hls_test_origin(head_a, vec![b'a']).await; + let (origin_b, task_b) = spawn_legacy_hls_test_origin(head_b, ACCOUNT_B_BODY.to_vec()).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_a}"), + username: Some("user-a".to_string()), + password: Some("pass-a".to_string()), + enabled: true, + priority: 0, + max_connections: 1, + method: InputFetchMethod::default(), + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "provider_2".intern(), + url: format!("http://{origin_b}"), + username: Some("user-b".to_string()), + password: Some("pass-b".to_string()), + priority: 1, + max_connections: 1, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }); + let app_config = Arc::new(create_test_provider_app_config()); + app_config.sources.store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(app_config); + let busy_addr: SocketAddr = "127.0.0.1:55410".parse().unwrap_or_else(|_| unreachable!()); + let busy = app_state.active_provider.acquire_exact_connection_with_grace( + &input.name, + &busy_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ); + assert!(busy.is_some(), "setup must occupy the pinned provider account"); + + let client_addr: SocketAddr = "127.0.0.1:55411".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer-pool-failover".to_string(); + let stream_url = format!("http://{origin_a}/series/user-a/pass-a/1.mkv"); + let session = UserSession { + token: "series-pool-token".to_string(), + media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), + virtual_id: 42, + provider: Arc::clone(&input.name), + stream_url: stream_url.clone().intern(), + addr: client_addr, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + let mut stream_channel = create_test_live_channel(&stream_url); + stream_channel.virtual_id = session.virtual_id; + stream_channel.provider_id = 1; + stream_channel.input_name = Arc::clone(&input.name); + stream_channel.item_type = PlaylistItemType::Series; + stream_channel.cluster = XtreamCluster::Series; + stream_channel.url = session.stream_url.clone(); + + let response = force_provider_stream_response( + &fingerprint, + &app_state, + &session, + stream_channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::OK); + let body = response.into_body().collect().await.expect("stream body").to_bytes(); + assert_eq!(body.as_ref(), ACCOUNT_B_BODY, "reopen must use the free provider account B"); + task_b.await.expect("origin B task completes"); + + let updated = app_state + .active_users + .get_and_update_user_session(&user.username, &session.token) + .await + .expect("reopened session is persisted"); + assert_eq!(updated.provider.as_ref(), "provider_2"); + assert!(updated.stream_url.contains("/series/user-b/pass-b/1.mkv")); + + assert!(tokio::time::timeout(std::time::Duration::from_millis(100), &mut task_a).await.is_err()); + task_a.abort(); + app_state.active_provider.release_connection(&busy_addr); +} + +#[tokio::test] +async fn forced_reopen_sends_provider_cookies_only_to_the_origin_that_set_them() { + for (cookie_from_stream_origin, expect_cookie) in [(true, true), (false, false)] { + let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: 4\r\nConnection: close\r\n\r\n"; + let (origin, task) = spawn_legacy_hls_test_origin(head.to_string(), b"live".to_vec()).await; + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + url: format!("http://{origin}"), + username: Some("user".to_string()), + password: Some("pass".to_string()), + enabled: true, + max_connections: 1, + ..ConfigInput::default() + }); + let app_config = Arc::new(create_test_provider_app_config()); + app_config + .sources + .store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(app_config); + + let stream_url = format!("http://{origin}/live/1.ts"); + let cookie_source = + if cookie_from_stream_origin { stream_url.clone() } else { "http://cdn.example/live/1.ts".to_string() }; + let mut cookies = tuliprox_session::ProviderSessionCookieStore::default(); + cookies.update( + &Url::parse(&cookie_source).expect("cookie source url"), + &tuliprox_session::ProviderSessionHeaders { + headers: HashMap::new(), + cookies: vec!["sid=scoped; Path=/".to_string()], + }, + ); + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_410)); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer".to_string(); + let session = UserSession { + token: "cookie-scope-token".to_string(), + virtual_id: 41, + provider: Arc::clone(&input.name), + stream_url: stream_url.as_str().intern(), + provider_session_cookies: Arc::new(cookies), + media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), + addr: client_addr, + active_addrs: vec![client_addr], + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + let mut stream_channel = create_test_live_channel(&stream_url); + stream_channel.provider_id = 1; + stream_channel.input_name = Arc::clone(&input.name); + + let response = force_provider_stream_response( + &create_test_fingerprint(client_addr), + &app_state, + &session, + stream_channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(response.status(), StatusCode::OK); + drop(response.into_body().collect().await.expect("stream body")); + + let request = task.await.expect("origin task completes").to_ascii_lowercase(); + assert_eq!( + request.contains("cookie: sid=scoped"), + expect_cookie, + "cookie_from_stream_origin={cookie_from_stream_origin}: {request}" + ); + } +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn parallel_series_range_requests_keep_both_claims_active() { + const SERIES_BODY: &[u8] = b"0123456789abcdefghij"; + + let (origin_addr, origin_task) = spawn_range_aware_test_origin(SERIES_BODY, 2).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_501)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer".to_string(); + + let make_session = |token: &str| UserSession { + token: token.to_string(), + transition_version: 1, + virtual_id: 43, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/series/1.ts").intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + + let make_channel = || { + let mut channel = create_test_local_channel(&format!("http://{origin_addr}/series/1.ts")); + channel.provider_id = 1; + channel.input_name = Arc::clone(&input.name); + channel.item_type = PlaylistItemType::Series; + channel.cluster = XtreamCluster::Series; + channel.url = format!("http://{origin_addr}/series/1.ts").intern(); + channel + }; + + let first_headers = { + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-9")); + headers + }; + let second_headers = { + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static("bytes=5-14")); + headers + }; + + // Open both range requests before consuming either body, so both claims are + // active at the same time. This exercises concurrent claims of one playback + // rather than a sequential request/consume cycle. + let first = force_provider_stream_response( + &fingerprint, + &app_state, + &make_session("series-range-1"), + make_channel(), + ForceStreamRequestContext { + req_headers: &first_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + let second = force_provider_stream_response( + &fingerprint, + &app_state, + &make_session("series-range-2"), + make_channel(), + ForceStreamRequestContext { + req_headers: &second_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(app_state.active_users.active_streams().await.len(), 2, "both range claims must be active concurrently"); + + let first_body = first.into_body().collect().await.expect("first series range body").to_bytes(); + assert_eq!(first_body.as_ref(), &SERIES_BODY[0..10]); + + let second_body = second.into_body().collect().await.expect("second series range body").to_bytes(); + assert_eq!(second_body.as_ref(), &SERIES_BODY[5..15]); + + let requests = origin_task.await.expect("origin task completes"); + assert_eq!(requests.len(), 2, "both series range requests must reach the same provider account"); + assert!(requests[0].contains("range: bytes=0-9")); + assert!(requests[1].contains("range: bytes=5-14")); +} + +#[tokio::test] +async fn dash_stream_request_remains_redirect_and_preserves_provider_affinity() { + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_dash".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: "http://provider-dash.example".to_string(), + username: Some("dash-user".to_string()), + password: Some("dash-pass".to_string()), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let target = Arc::new(ConfigTarget { + curation: None, + id: 1, + name: "target_dash".to_string(), + enabled: true, + options: None, + sort: None, + filter: Filter::default().into(), + output: Vec::new(), + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::default()), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }); + + let mut app_cfg = create_test_provider_app_config(); + app_cfg.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + app_cfg.config = Arc::new(ArcSwap::from_pointee(Config { + reverse_proxy: Some(crate::model::ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: crate::model::ResourceRetryConfig::default(), + disabled_header: None, + stream: None, + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }), + ..Config::default() + })); + let app_state = create_test_app_state_for_config(Arc::new(app_cfg)); + + let pli = PlaylistItem { + header: PlaylistItemHeader { + id: "1".intern(), + name: "DASH Live".intern(), + group: "Live".intern(), + title: "DASH Live".intern(), + url: "http://provider-dash.example/live/dash-user/dash-pass/1.mpd".intern(), + item_type: PlaylistItemType::LiveDash, + xtream_cluster: XtreamCluster::Live, + virtual_id: shared::model::VirtualId(1), + ..PlaylistItemHeader::default() + }, + }; + + let mut user = load_test_user("dash_viewer"); + user.proxy = ProxyType::Reverse(None); + for target_type in [TargetType::M3u, TargetType::Xtream] { + let redirect_params = RedirectParams { + item: &pli, + provider_id: pli.get_provider_id(), + cluster: XtreamCluster::Live, + target_type, + target: &target, + input: &input, + user: &user, + stream_ext: Some(shared::defaults::DASH_EXT), + req_context: crate::api::endpoints::xtream_api::ApiStreamContext::Live, + action_path: "", + }; + + let resp = redirect_response(&app_state, &redirect_params) + .await + .expect("DASH must return a redirect response") + .into_response(); + assert_eq!(resp.status(), StatusCode::FOUND, "DASH request must be HTTP 302 Found redirect"); + let location = resp.headers().get(axum::http::header::LOCATION).expect("location header present"); + let location_str = location.to_str().expect("valid location string"); + assert!( + location_str.contains("provider-dash.example"), + "redirect location must point to provider URL, got {location_str}" + ); + } + + let dash_ttl = get_session_reservation_ttl_secs(&app_state, PlaylistItemType::LiveDash); + assert_eq!(dash_ttl, default_hls_session_ttl_secs(), "DASH preserves provider affinity TTL"); +} diff --git a/backend/app/src/api/api_utils/tests/provider_urls.rs b/backend/app/src/api/api_utils/tests/provider_urls.rs new file mode 100644 index 000000000..4cb92f4e3 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/provider_urls.rs @@ -0,0 +1,1135 @@ +use super::{ + create_test_app_config, create_test_app_state_for_config, create_test_dual_provider_app_config, + create_test_dual_provider_app_state, create_test_fingerprint, create_test_live_channel, create_test_local_channel, + create_test_provider_app_config, create_test_shared_target, find_input_account_by_signature, + force_provider_stream_response, get_stream_alternative_url, load_test_user, resolve_redirect_location, + resolve_streaming_strategy, resolve_xtream_vod_provider_url, select_provider_stream_url, + spawn_legacy_hls_test_origin, spawn_range_aware_test_origin, stream_url_matches_provider, + ForceStreamRequestContext, StreamingAcquireOptions, +}; +use crate::{ + api::model::{ProviderConfig as RuntimeProviderConfig, ProviderConfigConnection, ProviderStreamState, UserSession}, + model::{AppConfig, ConfigInput, ConfigInputAlias, ConfigProvider, ProxyUserCredentials, SourcesConfig}, +}; +use arc_swap::ArcSwap; +use axum::{ + http::{header, HeaderMap, HeaderValue, StatusCode}, + response::IntoResponse, +}; +use http_body_util::BodyExt; +use shared::{ + model::{ + ConfigProviderDto, InputType, PlaylistItem, PlaylistItemHeader, PlaylistItemType, ProviderUrlSelectionPolicy, + ProxyType, UserConnectionPermission, VirtualId, XtreamCluster, + }, + utils::Internable, +}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration}; + +pub(in crate::api::api_utils::tests) fn test_runtime_provider( + url: &str, + username: &str, + password: &str, +) -> Arc { + test_runtime_provider_with_type(url, username, password, InputType::Xtream) +} + +pub(in crate::api::api_utils::tests) fn test_runtime_provider_with_type( + url: &str, + username: &str, + password: &str, + input_type: InputType, +) -> Arc { + let url = if input_type == InputType::M3u { + format!("{url}/playlist.m3u8?username={username}&password={password}") + } else { + url.to_string() + }; + let input = ConfigInput { + name: "provider".intern(), + url, + username: Some(username.to_string()), + password: Some(password.to_string()), + input_type, + ..ConfigInput::default() + }; + Arc::new(RuntimeProviderConfig::new( + &input, + Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), + Arc::new(|_, _| {}), + )) +} + +pub(in crate::api::api_utils::tests) fn test_runtime_provider_without_credentials( + url: &str, + input_type: InputType, +) -> Arc { + let input = ConfigInput { name: "provider".intern(), url: url.to_string(), input_type, ..ConfigInput::default() }; + Arc::new(RuntimeProviderConfig::new( + &input, + Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), + Arc::new(|_, _| {}), + )) +} + +#[test] +fn resolve_redirect_location_resolves_provider_scheme_urls() { + let provider = ConfigProvider::from(&ConfigProviderDto { + name: "develop".intern(), + urls: vec!["https://provider.example".intern()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + }); + let input = ConfigInput { + name: "provider".intern(), + provider_configs: Some(vec![Arc::new(provider)]), + ..ConfigInput::default() + }; + + let resolved = + resolve_redirect_location(Some(&input), "provider://develop/live/provider-user/provider-pass/33486.m3u8") + .expect("provider url should resolve"); + + assert_eq!(resolved, "https://provider.example/live/provider-user/provider-pass/33486.m3u8"); +} + +#[test] +fn stream_alternative_url_keeps_unmatched_urls_unchanged() { + let input = ConfigInput { + name: "source".intern(), + url: "http://source.example".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + input_type: InputType::Xtream, + ..ConfigInput::default() + }; + let alias = test_runtime_provider("http://alias.example", "alias-user", "alias-pass"); + let stream_url = "http://other.example/live/source-user/source-pass/123.ts"; + + let rewritten = get_stream_alternative_url(stream_url, &input, &alias); + + assert_eq!(rewritten, None); +} + +#[test] +fn stream_alternative_url_rewrites_only_query_auth_fields() { + let input = ConfigInput { + name: "source".intern(), + url: "http://source.example".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + input_type: InputType::Xtream, + ..ConfigInput::default() + }; + let alias = test_runtime_provider("http://alias.example", "alias-user", "alias-pass"); + let stream_url = "http://source.example/player?token=source-user&username=source-user&password=source-pass"; + + let rewritten = get_stream_alternative_url(stream_url, &input, &alias); + + assert_eq!( + rewritten, + Some("http://alias.example/player?token=source-user&username=alias-user&password=alias-pass".to_string()) + ); +} + +#[test] +fn stream_url_matches_provider_requires_base_url_and_account_identity() { + let provider = test_runtime_provider("http://same.example", "selected-user", "selected-pass"); + + assert!(stream_url_matches_provider("http://same.example/live/selected-user/selected-pass/123.ts", &provider)); + assert!(stream_url_matches_provider( + "http://same.example/timeshift/selected-user/selected-pass/30/2026-06-15:20-00/123.ts", + &provider + )); + assert!(stream_url_matches_provider( + "http://same.example/future-route/selected-user/selected-pass/opaque/123.ts", + &provider + )); + assert!(!stream_url_matches_provider("http://same.example/live/other-user/other-pass/123.ts", &provider)); + assert!(!stream_url_matches_provider( + "http://same.example/timeshift/other-user/other-pass/30/2026-06-15:20-00/123.ts", + &provider + )); + assert!(!stream_url_matches_provider( + "http://same.example/future-route/other-user/other-pass/opaque/123.ts", + &provider + )); +} + +#[test] +fn stream_url_matches_provider_accepts_external_playlist_url_for_m3u_without_account_signature() { + let provider = + test_runtime_provider_with_type("http://provider.example", "selected-user", "selected-pass", InputType::M3u); + + assert!(stream_url_matches_provider( + "https://hlspackager.akamaized.net/live/DB/ALYAUM_TV/HLS/ALYAUM_TV.m3u8", + &provider + )); + assert!(stream_url_matches_provider( + "https://shd-gcp-live.edgenextcdn.net/live/bitmovin-mbc-1/15cf99af5de54063fdabfefe66adc075/index.m3u8", + &provider + )); +} + +#[test] +fn stream_url_matches_provider_rejects_external_cdn_url_with_wrong_account_signature() { + let provider = test_runtime_provider("http://provider.example", "selected-user", "selected-pass"); + + assert!(!stream_url_matches_provider("http://cdn.example/live/other-user/other-pass/123.ts", &provider)); + assert!(!stream_url_matches_provider( + "http://cdn.example/segment.ts?username=other-user&password=other-pass", + &provider + )); +} + +#[test] +fn stream_url_matches_provider_rejects_external_cdn_url_with_wrong_account_signature_for_m3u() { + let provider = + test_runtime_provider_with_type("http://provider.example", "selected-user", "selected-pass", InputType::M3u); + + assert!(!stream_url_matches_provider( + "http://cdn.example/segment.ts?username=other-user&password=other-pass", + &provider + )); +} + +#[test] +fn stream_url_matches_provider_detects_m3u_path_credentials_against_alias_account() { + // Regression: a cross-host M3U URL whose path embeds the alias's + // account credentials must be detected as an account signature and + // validated, not silently allowed as an open URL. + let provider = + test_runtime_provider_with_type("http://provider.example", "selected-user", "selected-pass", InputType::M3u); + + // Matching path credentials -> allowed (account matches). + assert!(stream_url_matches_provider("http://cdn.example/live/selected-user/selected-pass/123.ts", &provider)); +} + +#[test] +fn stream_url_matches_provider_rejects_open_external_cdn_url_without_account_signature_for_xtream() { + let provider = test_runtime_provider("http://provider.example", "selected-user", "selected-pass"); + + assert!(!stream_url_matches_provider("http://cdn.example/open/playlist.m3u8", &provider)); + assert!(!stream_url_matches_provider("http://cdn.example/open/segment.ts?key=signedopaque", &provider)); +} + +#[test] +fn stream_url_matches_provider_rejects_external_cdn_url_for_xtream_even_with_valid_account_signature() { + let provider = test_runtime_provider("http://provider.example", "selected-user", "selected-pass"); + + assert!(!stream_url_matches_provider("http://cdn.example/live/selected-user/selected-pass/123.ts", &provider)); + assert!(!stream_url_matches_provider( + "http://cdn.example/segment.ts?username=selected-user&password=selected-pass", + &provider + )); +} + +#[test] +fn find_input_account_by_signature_matches_main_input_and_alias_accounts() { + let input = ConfigInput { + name: "source".intern(), + url: "http://provider.example".to_string(), + username: Some("main-user".to_string()), + password: Some("main-pass".to_string()), + input_type: InputType::Xtream, + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "alias".intern(), + url: "http://alias.example".to_string(), + username: Some("alias-user".to_string()), + password: Some("alias-pass".to_string()), + max_connections: 1, + priority: 0, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }; + + let main = find_input_account_by_signature("http://cdn.example/live/main-user/main-pass/1.ts", &input); + assert_eq!( + main, + Some(("http://provider.example".to_string(), Some("main-user".to_string()), Some("main-pass".to_string()),)) + ); + + let alias = find_input_account_by_signature("http://cdn.example/live/alias-user/alias-pass/1.ts", &input); + assert_eq!( + alias, + Some(("http://alias.example".to_string(), Some("alias-user".to_string()), Some("alias-pass".to_string()),)) + ); + + assert_eq!(find_input_account_by_signature("http://cdn.example/live/other/other/1.ts", &input), None); + assert_eq!(find_input_account_by_signature("http://cdn.example/open/playlist.m3u8", &input), None); +} + +#[test] +fn get_stream_alternative_url_rewrites_external_cdn_url_with_valid_account_signature_for_alias_account() { + let input = ConfigInput { + name: "source".intern(), + url: "http://provider.example/playlist.m3u8?username=source-user&password=source-pass".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + input_type: InputType::M3u, + ..ConfigInput::default() + }; + let alias = test_runtime_provider_with_type("http://alias.example", "alias-user", "alias-pass", InputType::M3u); + let stream_url = "http://cdn.example/live/source-user/source-pass/123.ts"; + + let rewritten = get_stream_alternative_url(stream_url, &input, &alias); + assert_eq!(rewritten, Some("http://cdn.example/live/alias-user/alias-pass/123.ts".to_string())); +} + +#[test] +fn get_stream_alternative_url_rewrites_opaque_m3u_token_for_alias_account() { + let input = ConfigInput { + name: "source".intern(), + url: "http://playlist.example/a.m3u?token=provider-a-token".to_string(), + input_type: InputType::M3u, + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "alias".intern(), + url: "http://playlist.example/b.m3u?token=provider-b-token".to_string(), + username: None, + password: None, + max_connections: 0, + priority: 0, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }; + let alias = test_runtime_provider_without_credentials( + "http://playlist.example/b.m3u?token=provider-b-token", + InputType::M3u, + ); + + let rewritten = + get_stream_alternative_url("http://stream.example/channel/segment.ts?token=provider-a-token", &input, &alias); + + assert_eq!(rewritten, Some("http://stream.example/channel/segment.ts?token=provider-b-token".to_string())); +} + +#[test] +fn get_stream_alternative_url_rewrites_single_opaque_m3u_credential_with_different_key() { + let input = ConfigInput { + name: "source".intern(), + url: "http://playlist.example/a.m3u?token=provider-a-token".to_string(), + input_type: InputType::M3u, + ..ConfigInput::default() + }; + let alias = test_runtime_provider_without_credentials( + "http://playlist.example/b.m3u?api_key=provider-b-key", + InputType::M3u, + ); + + let rewritten = get_stream_alternative_url( + "http://stream.example/segment.ts?token=provider-a-token&quality=hd", + &input, + &alias, + ); + + assert_eq!(rewritten, Some("http://stream.example/segment.ts?api_key=provider-b-key&quality=hd".to_string())); +} + +#[test] +fn get_stream_alternative_url_rejects_ambiguous_cross_key_credential_mapping() { + let input = ConfigInput { + name: "source".intern(), + url: "http://playlist.example/a.m3u?token=provider-a-token".to_string(), + input_type: InputType::M3u, + ..ConfigInput::default() + }; + let alias = test_runtime_provider_without_credentials( + "http://playlist.example/b.m3u?api_key=provider-b-key&auth=provider-b-auth", + InputType::M3u, + ); + + assert_eq!( + get_stream_alternative_url("http://stream.example/segment.ts?token=provider-a-token", &input, &alias), + None + ); +} + +#[test] +fn get_stream_alternative_url_rejects_partial_source_credential_mapping() { + let input = ConfigInput { + name: "source".intern(), + url: "http://playlist.example/a.m3u?token=provider-a-token&api_key=provider-a-key".to_string(), + input_type: InputType::M3u, + ..ConfigInput::default() + }; + let alias = test_runtime_provider_without_credentials( + "http://playlist.example/b.m3u?token=provider-b-token", + InputType::M3u, + ); + + assert_eq!( + get_stream_alternative_url( + "http://stream.example/segment.ts?token=provider-a-token&api_key=provider-a-key&quality=hd", + &input, + &alias, + ), + None + ); +} + +#[tokio::test] +async fn select_provider_stream_url_rewrites_opaque_m3u_token_for_allocated_alias() { + let input = ConfigInput { + name: "provider-a".intern(), + url: "http://playlist.example/a.m3u?token=provider-a-token".to_string(), + input_type: InputType::M3u, + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "provider-b".intern(), + url: "http://playlist.example/b.m3u?token=provider-b-token".to_string(), + username: None, + password: None, + max_connections: 0, + priority: 0, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }; + let alias = test_runtime_provider_without_credentials( + "http://playlist.example/b.m3u?token=provider-b-token", + InputType::M3u, + ); + + let temp = tempfile::tempdir().expect("temp dir should be created"); + let app_config = create_test_dual_provider_app_config(); + let mut config = (*app_config.config.load_full()).clone(); + config.storage_dir = temp.path().to_string_lossy().into_owned(); + app_config.config.store(Arc::new(config)); + let app_config = Arc::new(app_config); + let selected = select_provider_stream_url( + "http://stream.example/channel/segment.ts?token=provider-a-token", + &input, + &alias, + false, + &app_config, + ) + .await; + + assert_eq!( + selected, + Some(("provider".intern(), "http://stream.example/channel/segment.ts?token=provider-b-token".to_string(),)) + ); +} + +pub(in crate::api::api_utils::tests) async fn independent_stream_token_alias_fixture( +) -> (tempfile::TempDir, Arc, ConfigInput, Arc) { + independent_stream_token_alias_fixture_with(&["http://stream.example:4000/323/mono.m3u8?token=backup-stream-token"]) + .await +} + +/// Primary and backup M3U accounts whose stream tokens are independent of their playlist +/// keys; only the backup alias playlist with `alias_urls` is persisted. +pub(in crate::api::api_utils::tests) async fn independent_stream_token_alias_fixture_with( + alias_urls: &[&str], +) -> (tempfile::TempDir, Arc, ConfigInput, Arc) { + use shared::model::{PlaylistGroup, PlaylistItem, PlaylistItemHeader}; + use tuliprox_repository::{get_input_m3u_playlist_file_path, get_input_storage_path, persist_input_m3u_playlist}; + + let temp = tempfile::tempdir().expect("temp dir should be created"); + let app_config = create_test_dual_provider_app_config(); + let mut config = (*app_config.config.load_full()).clone(); + config.storage_dir = temp.path().to_string_lossy().into_owned(); + app_config.config.store(Arc::new(config)); + let app_config = Arc::new(app_config); + + let input = ConfigInput { + name: "primary-account".intern(), + url: "http://playlist.example/list.m3u?access_key=primary-playlist-key".to_string(), + input_type: InputType::M3u, + ..ConfigInput::default() + }; + let alias_input = ConfigInput { + name: "backup-account".intern(), + url: "http://playlist.example/list.m3u?access_key=backup-playlist-key".to_string(), + input_type: InputType::M3u, + ..ConfigInput::default() + }; + let alias = Arc::new(RuntimeProviderConfig::new( + &alias_input, + Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), + Arc::new(|_, _| {}), + )); + + let storage_path = get_input_storage_path(&alias.name, &app_config.config.load().storage_dir) + .await + .expect("alias storage should be created"); + let playlist_path = get_input_m3u_playlist_file_path(&storage_path, &alias.name); + let playlist = vec![PlaylistGroup { + id: 1, + title: "Live".intern(), + channels: alias_urls + .iter() + .enumerate() + .map(|(index, url)| PlaylistItem { + header: PlaylistItemHeader { + id: format!("channel-323-{index}").intern(), + input_stream_id: format!("channel-323-{index}").intern(), + url: (*url).intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }) + .collect(), + xtream_cluster: XtreamCluster::Live, + }]; + persist_input_m3u_playlist(&app_config, &playlist_path, &playlist).await.expect("alias playlist should persist"); + (temp, app_config, input, alias) +} + +#[tokio::test] +async fn select_provider_stream_url_uses_persisted_alias_url_for_independent_stream_token() { + let (_temp, app_config, input, alias) = independent_stream_token_alias_fixture().await; + + let selected = select_provider_stream_url( + "http://stream.example:4000/323/mono.m3u8?token=primary-stream-token", + &input, + &alias, + false, + &app_config, + ) + .await; + + assert_eq!( + selected, + Some(( + "backup-account".intern(), + "http://stream.example:4000/323/mono.m3u8?token=backup-stream-token".to_string(), + )) + ); +} + +#[tokio::test] +async fn select_provider_stream_url_maps_flussonic_archive_to_alias_stream_token() { + let (_temp, app_config, input, alias) = independent_stream_token_alias_fixture().await; + + for archive_file in ["archive-1791225557-14400.m3u8", "mono-1791225557-14400.m3u8", "timeshift_abs-1791225557.ts"] { + let selected = select_provider_stream_url( + &format!("http://stream.example:4000/323/{archive_file}?token=primary-stream-token"), + &input, + &alias, + false, + &app_config, + ) + .await; + + assert_eq!( + selected, + Some(( + "backup-account".intern(), + format!("http://stream.example:4000/323/{archive_file}?token=backup-stream-token"), + )), + "{archive_file}" + ); + } +} + +#[tokio::test] +async fn select_provider_stream_url_prefers_archive_source_live_file() { + let (_temp, app_config, input, alias) = independent_stream_token_alias_fixture_with(&[ + "http://stream.example:4000/323/index.m3u8?token=index-token", + "http://stream.example:4000/323/mono.ts?token=mono-ts-token", + "http://stream.example:4000/323/mono.m3u8?token=mono-hls-token", + ]) + .await; + + for (archive_file, token) in [ + ("mono-1791225557-14400.m3u8", "mono-hls-token"), + ("mono-1791225557-14400.ts", "mono-ts-token"), + ("index-1791225557-14400.m3u8", "index-token"), + ] { + let selected = select_provider_stream_url( + &format!("http://stream.example:4000/323/{archive_file}?token=primary-stream-token"), + &input, + &alias, + false, + &app_config, + ) + .await; + + assert_eq!( + selected.map(|(_, url)| url), + Some(format!("http://stream.example:4000/323/{archive_file}?token={token}")), + "{archive_file}" + ); + } +} + +#[tokio::test] +async fn select_provider_stream_url_matches_uppercase_alias_live_file_for_archives_only() { + let (_temp, app_config, input, alias) = independent_stream_token_alias_fixture_with(&[ + "http://stream.example:4000/323/MONO.M3U8?token=backup-stream-token", + "http://stream.example:4000/324/INDEX.m3u8?token=upper-token", + "http://stream.example:4000/324/index.m3u8?token=lower-token", + ]) + .await; + + for (requested, expected) in [ + // A Flussonic archive finds its live file in any case. + ( + "http://stream.example:4000/323/mono-1791225557-14400.m3u8?token=primary-stream-token", + "http://stream.example:4000/323/mono-1791225557-14400.m3u8?token=backup-stream-token", + ), + // Exact lookups keep case-sensitive paths of other providers distinct. + ( + "http://stream.example:4000/324/INDEX.m3u8?token=primary-stream-token", + "http://stream.example:4000/324/INDEX.m3u8?token=upper-token", + ), + ( + "http://stream.example:4000/324/index.m3u8?token=primary-stream-token", + "http://stream.example:4000/324/index.m3u8?token=lower-token", + ), + ] { + let selected = select_provider_stream_url(requested, &input, &alias, false, &app_config).await; + assert_eq!(selected.map(|(_, url)| url).as_deref(), Some(expected), "{requested}"); + } +} + +#[tokio::test] +async fn select_provider_stream_url_keeps_unindexed_non_archive_url() { + let (_temp, app_config, input, alias) = independent_stream_token_alias_fixture().await; + let segment = "http://stream.example:4000/323/tracks-v1a1/dvr-2026/10/05/18/26/seg.ts?token=backup-stream-token"; + + let selected = select_provider_stream_url(segment, &input, &alias, false, &app_config).await; + + assert_eq!(selected, Some(("backup-account".intern(), segment.to_string()))); +} + +#[test] +fn get_stream_alternative_url_rewrites_timeshift_path_credentials_for_alias_account() { + let input = ConfigInput { + name: "source".intern(), + url: "http://provider.example".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + input_type: InputType::Xtream, + ..ConfigInput::default() + }; + let alias = test_runtime_provider("http://alias.example", "alias-user", "alias-pass"); + let stream_url = "http://provider.example/timeshift/source-user/source-pass/30/2026-06-15:20-00/123.ts"; + + let rewritten = get_stream_alternative_url(stream_url, &input, &alias); + assert_eq!( + rewritten, + Some("http://alias.example/timeshift/alias-user/alias-pass/30/2026-06-15:20-00/123.ts".to_string()) + ); +} + +#[test] +fn get_stream_alternative_url_rewrites_future_route_path_credentials_for_alias_account() { + let input = ConfigInput { + name: "source".intern(), + url: "http://provider.example".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + input_type: InputType::Xtream, + ..ConfigInput::default() + }; + let alias = test_runtime_provider("http://alias.example", "alias-user", "alias-pass"); + let stream_url = "http://provider.example/future-route/source-user/source-pass/opaque/123.ts"; + + let rewritten = get_stream_alternative_url(stream_url, &input, &alias); + assert_eq!(rewritten, Some("http://alias.example/future-route/alias-user/alias-pass/opaque/123.ts".to_string())); +} + +#[test] +fn get_stream_alternative_url_keeps_open_external_playlist_url_for_m3u() { + let input = ConfigInput { + name: "source".intern(), + url: "http://provider.example/playlist.m3u8?username=source-user&password=source-pass".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + input_type: InputType::M3u, + ..ConfigInput::default() + }; + let alias = test_runtime_provider_with_type("http://alias.example", "alias-user", "alias-pass", InputType::M3u); + let stream_url = "https://cnbc-live.akamaized.net/cnbc/master.m3u8"; + + assert_eq!(get_stream_alternative_url(stream_url, &input, &alias), Some(stream_url.to_string())); +} + +#[test] +fn get_stream_alternative_url_keeps_open_external_multisegment_playlist_url_for_m3u() { + let input = ConfigInput { + name: "source".intern(), + url: "http://provider.example/playlist.m3u8?username=source-user&password=source-pass".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + input_type: InputType::M3u, + ..ConfigInput::default() + }; + let alias = test_runtime_provider_with_type("http://alias.example", "alias-user", "alias-pass", InputType::M3u); + let stream_url = "https://hnpsechtsc.turknet.ercdn.net/xpnvudnlsv/cnbc-e/cnbc-e.m3u8"; + + assert_eq!(get_stream_alternative_url(stream_url, &input, &alias), Some(stream_url.to_string())); +} + +#[test] +fn get_stream_alternative_url_keeps_open_external_m3u_url_for_provider_without_credentials() { + let input = ConfigInput { + name: "source".intern(), + url: "http://provider.example/playlist.m3u8".to_string(), + input_type: InputType::M3u, + ..ConfigInput::default() + }; + let provider = test_runtime_provider_without_credentials("http://provider.example/playlist.m3u8", InputType::M3u); + let stream_url = "http://s.only4.tv/17113/video.m3u8?token=abc"; + + assert_eq!(get_stream_alternative_url(stream_url, &input, &provider), Some(stream_url.to_string())); +} + +#[test] +fn get_stream_alternative_url_rejects_query_credentials_for_provider_without_credentials() { + let input = ConfigInput { + name: "source".intern(), + url: "http://provider.example/playlist.m3u8".to_string(), + input_type: InputType::M3u, + ..ConfigInput::default() + }; + let provider = test_runtime_provider_without_credentials("http://provider.example/playlist.m3u8", InputType::M3u); + + assert_eq!( + get_stream_alternative_url( + "http://cdn.example/segment.ts?username=other-user&password=other-pass", + &input, + &provider + ), + None + ); +} + +#[test] +fn get_stream_alternative_url_rejects_basic_auth_credentials_for_provider_without_credentials() { + let input = ConfigInput { + name: "source".intern(), + url: "http://provider.example/playlist.m3u8".to_string(), + input_type: InputType::M3u, + ..ConfigInput::default() + }; + let provider = test_runtime_provider_without_credentials("http://provider.example/playlist.m3u8", InputType::M3u); + + assert_eq!(get_stream_alternative_url("http://user:pass@cdn.example/segment.ts", &input, &provider), None); +} + +#[test] +fn get_stream_alternative_url_rejects_external_m3u_url_with_unmatched_account_signature() { + let input = ConfigInput { + name: "source".intern(), + url: "http://provider.example/playlist.m3u8?username=source-user&password=source-pass".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + input_type: InputType::M3u, + ..ConfigInput::default() + }; + let alias = test_runtime_provider_with_type("http://alias.example", "alias-user", "alias-pass", InputType::M3u); + + assert_eq!( + get_stream_alternative_url( + "http://cdn.example/segment.ts?username=other-user&password=other-pass", + &input, + &alias, + ), + None + ); +} + +#[test] +fn get_stream_alternative_url_does_not_passthrough_arbitrary_open_external_url_for_xtream() { + let input = ConfigInput { + name: "source".intern(), + url: "http://provider.example".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + input_type: InputType::Xtream, + ..ConfigInput::default() + }; + let alias = test_runtime_provider("http://alias.example", "alias-user", "alias-pass"); + let stream_url = "http://cdn.example/open/playlist.m3u8"; + + assert_eq!(get_stream_alternative_url(stream_url, &input, &alias), None); +} + +#[test] +fn xtream_vod_provider_url_requires_stored_input_identity() -> Result<(), Box> { + let config = create_test_dual_provider_app_config(); + let input = config.sources.load().inputs.first().cloned().ok_or("missing input")?; + let provider = RuntimeProviderConfig::new( + &input, + Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), + Arc::new(|_, _| {}), + ); + let stream_url = "http://cdn.example/r2/movie.mp4"; + let mut channel = create_test_live_channel(stream_url); + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + channel.provider_id = 100; + assert_eq!(resolve_xtream_vod_provider_url(stream_url, &input, &provider, &channel), Some(stream_url.to_string())); + assert_eq!( + resolve_xtream_vod_provider_url("http://unrelated.example/movie.mp4", &input, &provider, &channel), + None + ); + channel.input_name = "other-input".intern(); + assert_eq!(resolve_xtream_vod_provider_url(stream_url, &input, &provider, &channel), None); + channel.input_name = Arc::clone(&input.name); + channel.item_type = PlaylistItemType::Live; + assert_eq!(resolve_xtream_vod_provider_url(stream_url, &input, &provider, &channel), None); + channel.item_type = PlaylistItemType::Video; + channel.provider_id = 0; + assert_eq!(resolve_xtream_vod_provider_url(stream_url, &input, &provider, &channel), None); + channel.provider_id = 100; + channel.url = "file:///internal/movie.mp4".intern(); + assert_eq!(resolve_xtream_vod_provider_url(&channel.url, &input, &provider, &channel), None); + let unrelated = test_runtime_provider("http://unrelated.example", "other-user", "other-pass"); + channel.url = stream_url.intern(); + assert_eq!(resolve_xtream_vod_provider_url(stream_url, &input, &unrelated, &channel), None); + Ok(()) +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn xtream_vod_m3u_reverse_proxies_external_sources_and_alias_redirects_without_leaking_urls( +) -> Result<(), Box> { + for use_alias in [false, true] { + for source_path in ["/r2/movie.mp4", "/stream/account-bound-token"] { + let (cdn_addr, cdn_task) = spawn_legacy_hls_test_origin( + "HTTP/1.1 206 Partial Content\r\nContent-Type: video/mp4\r\nContent-Length: 4\r\nContent-Range: bytes 4-7/12\r\nAccept-Ranges: bytes\r\nReferer: http://private-provider.example/user/pass\r\nContent-Location: http://private-cdn.example/movie.mp4\r\nLocation: http://private-cdn.example/movie.mp4\r\nLink: \r\nConnection: close\r\n\r\n".to_string(), + b"DATA".to_vec(), + ).await; + let cdn_url = format!("http://{cdn_addr}/internal/media.mp4?token=private-cdn-token"); + let (redirect_addr, redirect_task) = spawn_legacy_hls_test_origin( + format!("HTTP/1.1 302 Found\r\nLocation: {cdn_url}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"), + Vec::new(), + ) + .await; + let config = create_test_dual_provider_app_config(); + let mut input = (**config.sources.load().inputs.first().ok_or("missing input")?).clone(); + let alias = input.aliases.as_mut().and_then(|aliases| aliases.first_mut()).ok_or("missing alias")?; + alias.url = format!("http://{redirect_addr}"); + let input = Arc::new(input); + config + .sources + .store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + let busy_addr = SocketAddr::from(([127, 0, 0, 1], 55401)); + let busy = if use_alias { + app_state.active_provider.acquire_exact_connection_with_grace( + &input.name, + &busy_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ) + } else { + None + }; + assert!(!use_alias || busy.is_some()); + let source_url = format!("http://{redirect_addr}{source_path}"); + let item = shared::model::M3uPlaylistItem::from(&PlaylistItem { + header: PlaylistItemHeader { + id: "900".intern(), + input_stream_id: "100".intern(), + virtual_id: VirtualId::new(100), + input_name: Arc::clone(&input.name), + url: source_url.intern(), + item_type: PlaylistItemType::Video, + xtream_cluster: XtreamCluster::Video, + additional_properties: Some(shared::model::StreamProperties::Video(Box::new( + shared::model::VideoStreamProperties { + container_extension: "mp4".intern(), + ..Default::default() + }, + ))), + ..Default::default() + }, + }); + let mut target = create_test_shared_target(); + target.options = None; + target.output = vec![tuliprox_core::model::TargetOutput::M3u(tuliprox_core::model::M3uTargetOutput::from( + &shared::model::M3uTargetOutputDto::default(), + ))]; + let mut user = load_test_user("vod-reverse-viewer"); + user.proxy = ProxyType::Reverse(None); + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static("bytes=4-7")); + let response = crate::api::endpoints::m3u_api::m3u_api_stream_loaded( + Arc::new(user), + Arc::new(target), + &create_test_fingerprint(SocketAddr::from(([127, 0, 0, 1], 55402))), + &headers, + &app_state, + item, + Arc::clone(&input), + Some("mp4"), + None, + None, + ) + .await + .into_response(); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers().get(header::CONTENT_RANGE), Some(&HeaderValue::from_static("bytes 4-7/12"))); + for name in ["location", "content-location", "referer", "link"] { + assert!(!response.headers().contains_key(name)); + } + for value in response.headers().values() { + let value = value.to_str()?; + assert!(!value.contains(&redirect_addr.to_string())); + assert!(!value.contains(&cdn_addr.to_string())); + assert!(!value.contains("private-cdn-token")); + assert!(!value.contains("user2/pass2")); + } + let body = response.into_body().collect().await?.to_bytes(); + assert_eq!(body.as_ref(), b"DATA"); + let redirect_request = tokio::time::timeout(Duration::from_secs(5), redirect_task).await??; + let cdn_request = tokio::time::timeout(Duration::from_secs(5), cdn_task).await??; + let expected_path = if use_alias { "/movie/user2/pass2/100.mp4" } else { source_path }; + assert!(redirect_request.starts_with(&format!("GET {expected_path} "))); + assert!(cdn_request.starts_with("GET /internal/media.mp4?token=private-cdn-token ")); + assert!(redirect_request.to_ascii_lowercase().contains("range: bytes=4-7")); + assert!(cdn_request.to_ascii_lowercase().contains("range: bytes=4-7")); + app_state.active_provider.release_connection(&busy_addr); + } + } + Ok(()) +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn overlapping_vod_range_requests_return_correct_account_bytes() { + const VOD_BODY: &[u8] = b"0123456789abcdefghij"; + + let (origin_addr, origin_task) = spawn_range_aware_test_origin(VOD_BODY, 2).await; + + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 2, + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(config)); + + let client_addr = SocketAddr::from(([127, 0, 0, 1], 55_500)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer".to_string(); + + let make_session = |token: &str| UserSession { + token: token.to_string(), + transition_version: 1, + virtual_id: 42, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/movie/1.ts").intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + + let make_channel = || { + let mut channel = create_test_local_channel(&format!("http://{origin_addr}/movie/1.ts")); + channel.provider_id = 1; + channel.input_name = Arc::clone(&input.name); + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + channel.url = format!("http://{origin_addr}/movie/1.ts").intern(); + channel + }; + + let build_request = |range: &'static str| { + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static(range)); + let session = make_session(match range { + "bytes=0-9" => "vod-range-1", + _ => "vod-range-2", + }); + (headers, session) + }; + + let (first_headers, first_session) = build_request("bytes=0-9"); + let first = force_provider_stream_response( + &fingerprint, + &app_state, + &first_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &first_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(first.headers().get(header::CONTENT_RANGE).and_then(|value| value.to_str().ok()), Some("bytes 0-9/20")); + let first_body = first.into_body().collect().await.expect("first range body").to_bytes(); + assert_eq!(first_body.as_ref(), &VOD_BODY[0..10]); + + let (second_headers, second_session) = build_request("bytes=5-14"); + let second = force_provider_stream_response( + &fingerprint, + &app_state, + &second_session, + make_channel(), + ForceStreamRequestContext { + req_headers: &second_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!( + second.headers().get(header::CONTENT_RANGE).and_then(|value| value.to_str().ok()), + Some("bytes 5-14/20") + ); + let second_body = second.into_body().collect().await.expect("second range body").to_bytes(); + assert_eq!(second_body.as_ref(), &VOD_BODY[5..15]); + + let requests = origin_task.await.expect("origin task completes"); + assert_eq!(requests.len(), 2, "both range requests must reach the same provider account"); + assert!(requests[0].contains("range: bytes=0-9")); + assert!(requests[1].contains("range: bytes=5-14")); +} + +#[tokio::test] +async fn resolve_streaming_strategy_rewrites_url_on_fallback_even_when_accept_requested_stream_url_is_true() { + let app_state = create_test_dual_provider_app_state(); + let input_name = "provider_1".intern(); + let input = + app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); + let pinned_provider = "provider_1".intern(); + let busy_addr: SocketAddr = "127.0.0.1:55304".parse().unwrap_or_else(|_| unreachable!()); + let fallback_addr: SocketAddr = "127.0.0.1:55305".parse().unwrap_or_else(|_| unreachable!()); + let stream_url = "http://provider-1.example/movie/user1/pass1/1.mkv"; + + let busy = app_state.active_provider.acquire_exact_connection_with_grace( + &pinned_provider, + &busy_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ); + assert!(busy.is_some(), "setup should occupy the pinned provider"); + + let fallback = resolve_streaming_strategy( + &app_state, + stream_url, + &create_test_fingerprint(fallback_addr), + &input, + StreamingAcquireOptions { + force_provider: Some(&pinned_provider), + allow_forced_provider_fallback: true, + allow_provider_grace: false, + user_priority: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + session_owner: Some("vod-session"), + playback_kind: crate::model::PlaybackKind::Vod, + accept_requested_stream_url: true, + capacity_wait_timeout: None, + }, + None, + ) + .await; + + let (ProviderStreamState::Available(Some(fallback_provider), url) + | ProviderStreamState::GracePeriod(Some(fallback_provider), url)) = fallback.provider_stream_state + else { + panic!("fallback-enabled request should allocate fallback provider") + }; + assert_eq!(fallback_provider.as_ref(), "provider_2"); + assert_eq!(url.as_ref(), "http://provider-2.example/movie/user2/pass2/1.mkv"); + + app_state.active_provider.release_connection(&busy_addr); + app_state.active_provider.release_connection(&fallback_addr); +} + +#[tokio::test] +async fn get_query_path_strips_extension_for_live_with_flag() { + use crate::model::ConfigInputFlags; + use shared::model::{InputType, PlaylistItemType, XtreamCluster, XtreamPlaylistItem}; + + let mut input = ConfigInput { + id: 1, + name: "provider_with_flag".intern(), + input_type: InputType::Xtream, + ..ConfigInput::default() + }; + let mut options = crate::model::ConfigInputOptions::defaults().clone(); + options.flags.set(ConfigInputFlags::XtreamLiveStreamWithoutExtension); + input.options = Some(options); + + let sources = SourcesConfig { inputs: vec![Arc::new(input)], ..SourcesConfig::default() }; + let mut app_cfg_raw = create_test_app_config(); + app_cfg_raw.sources = Arc::new(ArcSwap::from_pointee(sources)); + let app_state = create_test_app_state_for_config(Arc::new(app_cfg_raw)); + + let pli = XtreamPlaylistItem { + virtual_id: VirtualId::new(100), + provider_id: 1, + name: "test".intern(), + logo: "".intern(), + logo_small: "".intern(), + group: "".intern(), + title: "".intern(), + parent_code: "".intern(), + rec: "".intern(), + url: "http://example.com/123".intern(), + epg_channel_id: None, + xtream_cluster: XtreamCluster::Live, + additional_properties: None, + item_type: PlaylistItemType::Live, + category_id: 0, + input_name: "provider_with_flag".intern(), + channel_no: 0, + source_ordinal: 0, + input_stream_id: "1".intern(), + upstream_user_agent: None, + }; + + let hls_ext = shared::defaults::HLS_EXT.to_string(); + let (query_path, extension) = + crate::api::endpoints::xtream_api::get_query_path("", Some(&hls_ext), &pli, &app_state); + + assert_eq!(extension, ""); + assert_eq!(query_path, "1"); + + let dash_ext = shared::defaults::DASH_EXT.to_string(); + let (query_path, extension) = + crate::api::endpoints::xtream_api::get_query_path("", Some(&dash_ext), &pli, &app_state); + + assert_eq!(extension, ""); + assert_eq!(query_path, "1"); +} diff --git a/backend/app/src/api/api_utils/tests/resource_proxy.rs b/backend/app/src/api/api_utils/tests/resource_proxy.rs new file mode 100644 index 000000000..9a3414d60 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/resource_proxy.rs @@ -0,0 +1,443 @@ +use super::{ + create_test_app_state, media_server_image_error_status, resource_input_for_url, resource_proxy_response, + resource_redirect_or_proxy, resource_response, spawn_legacy_hls_test_origin, ResourceFetchPolicy, +}; +use crate::{ + media_server::{MediaServerError, MediaServerErrorKind}, + model::{Config, ConfigInput}, + utils::LRUResourceCache, +}; +use axum::{ + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; +use bytes::Bytes; +use http_body_util::BodyExt; +use std::{fmt::Write as _, sync::Arc}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::RwLock, +}; + +#[test] +fn media_server_image_error_status_classifies_client_and_upstream_failures() { + let parse_error = MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) + .detail("media server image URL is missing required path parts"); + assert_eq!(media_server_image_error_status(&parse_error), StatusCode::BAD_REQUEST); + + let not_found = MediaServerError::new(MediaServerErrorKind::MediaServerItemNotFound) + .detail("plex media-server image URL is missing image_path"); + assert_eq!(media_server_image_error_status(¬_found), StatusCode::NOT_FOUND); + + let upstream = MediaServerError::new(MediaServerErrorKind::MediaServerStreamOpenFailed) + .detail("media-server image request failed"); + assert_eq!(media_server_image_error_status(&upstream), StatusCode::BAD_GATEWAY); +} + +#[tokio::test] +async fn resource_cache_is_used_only_by_matching_public_fetch_policy() { + const CACHED_BODY: &[u8] = b"cached image"; + const UPSTREAM_BODY: &[u8] = b"upstream image"; + + let app_state = create_test_app_state(); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache_dir = temp_dir.path().to_string_lossy(); + let mut cache = LRUResourceCache::new(1024, cache_dir.as_ref()); + let resource_url = "http://1.1.1.1/icon.png"; + let cached_path = cache.store_path(resource_url, Some("image/png")); + tokio::fs::write(&cached_path, CACHED_BODY).await.expect("write cached image"); + cache.add_content(resource_url, Some("image/png".to_string()), CACHED_BODY.len()).expect("register cached image"); + app_state.cache.store(Some(Arc::new(RwLock::new(cache)))); + + let response_head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + UPSTREAM_BODY.len() + ); + let (upstream_addr, upstream_task) = spawn_legacy_hls_test_origin(response_head, UPSTREAM_BODY.to_vec()).await; + let proxy = reqwest::Proxy::http(format!("http://{upstream_addr}")).expect("mock proxy URL"); + let mock_client = Arc::new(reqwest::Client::builder().proxy(proxy).build().expect("mock upstream client")); + // The destination is public, so the hop is fetched through the public resource client; both resource + // clients are mocked so the assertion below is about the cache, not about which client performed the + // request. + app_state.http_clients.resource_public_no_redirect.store(Arc::clone(&mock_client)); + app_state.http_clients.resource_no_redirect.store(mock_client); + + let public_response = + resource_response(&app_state, ResourceFetchPolicy::Public, resource_url, &HeaderMap::new(), None) + .await + .into_response(); + assert_eq!(public_response.status(), StatusCode::OK); + let public_body = public_response.into_body().collect().await.expect("read cached image").to_bytes(); + assert_eq!(public_body, Bytes::from_static(CACHED_BODY)); + + let response = resource_response(&app_state, ResourceFetchPolicy::NonPublic, resource_url, &HeaderMap::new(), None) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::OK); + let response_body = response.into_body().collect().await.expect("read upstream image").to_bytes(); + assert_eq!(response_body, Bytes::from_static(UPSTREAM_BODY)); + assert_ne!(response_body, Bytes::from_static(CACHED_BODY)); + + let upstream_request = upstream_task.await.expect("mock upstream task completes"); + assert!(upstream_request.starts_with("GET http://1.1.1.1/icon.png HTTP/1.1\r\n")); +} + +#[tokio::test] +async fn no_redirect_resource_ignores_configured_proxy() { + let app_state = create_test_app_state(); + let response_head = + "HTTP/1.1 302 Found\r\nLocation: http://10.0.0.2/other.png\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head.to_string(), Vec::new()).await; + app_state.app_config.config.store(Arc::new(Config { + proxy: Some(crate::model::ProxyConfig { url: format!("http://{proxy_addr}"), username: None, password: None }), + ..Config::default() + })); + let client = crate::api::model::create_resource_http_client_no_redirect(&app_state.app_config) + .expect("resource HTTP client"); + // 192.0.2.1 (TEST-NET-1) is not routable, so a direct attempt fails by timing out or refusing the + // connection. A response would mean the request was answered by the proxy after all. + let result = client.get("http://192.0.2.1/icon.png").timeout(std::time::Duration::from_millis(100)).send().await; + assert!( + result.as_ref().is_err_and(|err| err.is_timeout() || err.is_connect()), + "the direct path must be attempted: {result:?}" + ); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), proxy_task).await.is_err(), + "resource client must not send requests through the configured proxy" + ); +} + +#[tokio::test] +async fn public_resource_client_honours_the_configured_proxy() { + // Counterpart of `no_redirect_resource_ignores_configured_proxy`: a resource hop that can leave the + // local network must go through the configured proxy, or the fetch discloses this host's address. + let app_state = create_test_app_state(); + let response_head = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 3\r\nConnection: close\r\n\r\n"; + let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head.to_string(), b"png".to_vec()).await; + app_state.app_config.config.store(Arc::new(Config { + proxy: Some(crate::model::ProxyConfig { url: format!("http://{proxy_addr}"), username: None, password: None }), + ..Config::default() + })); + let client = crate::api::model::create_resource_public_http_client_no_redirect(&app_state.app_config) + .expect("resource HTTP client"); + + let response = client.get("http://8.8.8.8/icon.png").send().await.expect("request reaches the proxy"); + + assert_eq!(response.status(), StatusCode::OK); + let request = tokio::time::timeout(std::time::Duration::from_secs(2), proxy_task) + .await + .expect("request must reach proxy") + .expect("proxy task"); + assert!(request.starts_with("GET http://8.8.8.8/icon.png HTTP/1.1\r\n"), "{request}"); +} + +#[tokio::test] +async fn an_unresolved_resource_name_is_fetched_through_the_configured_proxy() { + // A resource name that does not resolve locally is still resolvable through a configured proxy (for + // example with remote DNS), so the fetch must not be pinned to the direct client. + let app_state = create_test_app_state(); + let response_head = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 3\r\nConnection: close\r\n\r\n"; + let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head.to_string(), b"png".to_vec()).await; + app_state.app_config.config.store(Arc::new(Config { + proxy: Some(crate::model::ProxyConfig { url: format!("http://{proxy_addr}"), username: None, password: None }), + ..Config::default() + })); + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://{proxy_addr}")).expect("test proxy")) + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("mock proxy client"); + app_state.http_clients.resource_public_no_redirect.store(Arc::new(client)); + + let response = + resource_proxy_response(&app_state, "http://unresolved.invalid/logo.png", &HeaderMap::new(), None).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.into_body().collect().await.expect("image body").to_bytes(), Bytes::from_static(b"png")); + let request = tokio::time::timeout(std::time::Duration::from_secs(2), proxy_task) + .await + .expect("request must reach proxy") + .expect("proxy task"); + assert!(request.starts_with("GET http://unresolved.invalid/logo.png HTTP/1.1\r\n"), "{request}"); +} + +#[tokio::test] +async fn public_resource_client_reaches_a_proxy_on_loopback() { + // A proxy is commonly configured on the loopback interface (`http://localhost:8118`), so the + // connect-time guard of the public resource client must not reject the proxy host itself. + let app_state = create_test_app_state(); + let response_head = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 3\r\nConnection: close\r\n\r\n"; + let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head.to_string(), b"png".to_vec()).await; + app_state.app_config.config.store(Arc::new(Config { + proxy: Some(crate::model::ProxyConfig { + url: format!("http://localhost:{}", proxy_addr.port()), + username: None, + password: None, + }), + ..Config::default() + })); + let client = crate::api::model::create_resource_public_http_client_no_redirect(&app_state.app_config) + .expect("resource HTTP client"); + + let response = client.get("http://8.8.8.8/icon.png").send().await.expect("request reaches the proxy"); + + assert_eq!(response.status(), StatusCode::OK); + let request = tokio::time::timeout(std::time::Duration::from_secs(2), proxy_task) + .await + .expect("request must reach proxy") + .expect("proxy task"); + assert!(request.starts_with("GET http://8.8.8.8/icon.png HTTP/1.1\r\n"), "{request}"); +} + +#[tokio::test] +async fn no_redirect_resource_refuses_destinations_local_to_this_host() { + let app_state = create_test_app_state(); + let (proxy_addr, proxy_task) = + spawn_legacy_hls_test_origin("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n".to_string(), Vec::new()).await; + app_state.app_config.config.store(Arc::new(Config { + proxy: Some(crate::model::ProxyConfig { url: format!("http://{proxy_addr}"), username: None, password: None }), + ..Config::default() + })); + let client = crate::api::model::create_resource_http_client_no_redirect(&app_state.app_config) + .expect("resource HTTP client"); + app_state.http_clients.resource_no_redirect.store(Arc::new(client)); + + for local_only in ["http://127.0.0.1/icon.png", "http://169.254.169.254/latest/meta-data/", "http://[::1]/icon.png"] + { + let response = + resource_response(&app_state, ResourceFetchPolicy::NonPublic, local_only, &HeaderMap::new(), None) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::FORBIDDEN, "{local_only}"); + } + + // The guard rejects before a request is built, so nothing may reach the configured proxy. + assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), proxy_task).await.is_err(), + "no request may reach the proxy for local-only destinations" + ); +} + +#[tokio::test] +async fn resource_redirect_hides_private_destination_and_upstream_location() { + let app_state = create_test_app_state(); + let response_head = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nLocation: http://10.0.0.2/private\r\nContent-Length: 3\r\nConnection: close\r\n\r\n"; + let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head.to_string(), b"png".to_vec()).await; + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://{proxy_addr}")).expect("test proxy")) + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("mock upstream client"); + app_state.http_clients.resource_no_redirect.store(Arc::new(client)); + + let private = resource_redirect_or_proxy(&app_state, "http://10.0.0.1/icon.png", &HeaderMap::new(), None).await; + assert_eq!(private.status(), StatusCode::OK); + assert!(private.headers().get("location").is_none()); + assert_eq!(private.into_body().collect().await.expect("private icon body").to_bytes(), Bytes::from_static(b"png")); + let request = proxy_task.await.expect("mock request"); + assert!(request.starts_with("GET http://10.0.0.1/icon.png HTTP/1.1\r\n")); + + let public = resource_redirect_or_proxy(&app_state, "http://8.8.8.8/icon.png", &HeaderMap::new(), None).await; + assert_eq!(public.status(), StatusCode::FOUND); + assert_eq!(public.headers().get("location").and_then(|value| value.to_str().ok()), Some("http://8.8.8.8/icon.png")); + + let local_thumbnail = + resource_redirect_or_proxy(&app_state, "/api/v1/library/thumbnail/item", &HeaderMap::new(), None).await; + assert_eq!(local_thumbnail.status(), StatusCode::FOUND); + assert_eq!( + local_thumbnail.headers().get("location").and_then(|value| value.to_str().ok()), + Some("/api/v1/library/thumbnail/item") + ); + + let blocked = resource_redirect_or_proxy(&app_state, "http://127.0.0.1/icon.png", &HeaderMap::new(), None).await; + assert_eq!(blocked.status(), StatusCode::FORBIDDEN); + assert!(blocked.headers().get("location").is_none()); +} + +#[test] +fn configured_resource_headers_are_limited_to_the_input_origin() { + let input = ConfigInput { url: "http://10.0.0.1:8080/playlist.m3u".to_string(), ..ConfigInput::default() }; + assert!(resource_input_for_url(Some(&input), "http://10.0.0.1:8080/logo.png").is_some()); + assert!(resource_input_for_url(Some(&input), "http://10.0.0.2:8080/logo.png").is_none()); + assert!(resource_input_for_url(Some(&input), "http://10.0.0.1:8081/logo.png").is_none()); + assert!(resource_input_for_url(Some(&input), "https://10.0.0.1:8080/logo.png").is_none()); +} + +#[tokio::test] +async fn public_resource_is_fetched_through_the_configured_proxy() { + // Regression lock: a resource fetch that leaves the local network must use the configured proxy, + // otherwise it discloses the operator's address to the resource host. The Web UI resource route + // reaches this path, because it wraps every icon of a public destination into a proxy link. + let app_state = create_test_app_state(); + let response_head = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 3\r\nConnection: close\r\n\r\n"; + let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head.to_string(), b"png".to_vec()).await; + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://{proxy_addr}")).expect("test proxy")) + .build() + .expect("mock proxy client"); + app_state.http_clients.resource_public_no_redirect.store(Arc::new(client)); + + let response = resource_proxy_response(&app_state, "http://8.8.8.8/logo.png", &HeaderMap::new(), None).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.into_body().collect().await.expect("image body").to_bytes(), Bytes::from_static(b"png")); + let request = tokio::time::timeout(std::time::Duration::from_secs(2), proxy_task) + .await + .expect("request must reach proxy") + .expect("proxy task"); + assert!(request.starts_with("GET http://8.8.8.8/logo.png HTTP/1.1\r\n"), "{request}"); +} + +#[tokio::test] +async fn proxied_resource_follows_a_redirect_into_the_public_network_through_the_configured_proxy() { + // Regression lock: the hop decides which client is used. A network-internal destination that + // redirects to a public host must not make the follow-up request directly, or the redirect would + // disclose the operator's address to that host despite a configured proxy. + let app_state = create_test_app_state(); + let redirect_head = + "HTTP/1.1 302 Found\r\nLocation: http://8.8.8.8/logo.png\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let (private_proxy_addr, private_proxy_task) = + spawn_legacy_hls_test_origin(redirect_head.to_string(), Vec::new()).await; + let private_client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://{private_proxy_addr}")).expect("test proxy")) + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("mock private client"); + app_state.http_clients.resource_no_redirect.store(Arc::new(private_client)); + + let response_head = "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 3\r\nConnection: close\r\n\r\n"; + let (public_proxy_addr, public_proxy_task) = + spawn_legacy_hls_test_origin(response_head.to_string(), b"png".to_vec()).await; + let public_client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://{public_proxy_addr}")).expect("test proxy")) + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("mock public client"); + app_state.http_clients.resource_public_no_redirect.store(Arc::new(public_client)); + + let response = resource_proxy_response(&app_state, "http://10.0.0.1/logo.png", &HeaderMap::new(), None).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.into_body().collect().await.expect("image body").to_bytes(), Bytes::from_static(b"png")); + let first_hop = private_proxy_task.await.expect("first hop request"); + assert!(first_hop.starts_with("GET http://10.0.0.1/logo.png HTTP/1.1\r\n"), "{first_hop}"); + let second_hop = tokio::time::timeout(std::time::Duration::from_secs(2), public_proxy_task) + .await + .expect("public hop must be fetched") + .expect("public hop request"); + assert!(second_hop.starts_with("GET http://8.8.8.8/logo.png HTTP/1.1\r\n"), "{second_hop}"); +} + +#[tokio::test] +async fn public_resource_redirect_to_a_destination_local_to_this_host_is_refused() { + // Regression lock: a public host that redirects to loopback, link-local, or a cloud metadata + // endpoint must not turn the resource route into a reader for this host. The redirect target is + // never requested, and its location is never relayed to the client. + let app_state = create_test_app_state(); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("mock proxy binds"); + let proxy_addr = listener.local_addr().expect("mock proxy address"); + let proxy_task = tokio::spawn(async move { + let mut requests = Vec::new(); + loop { + let Ok(accepted) = tokio::time::timeout(std::time::Duration::from_millis(500), listener.accept()).await + else { + break; + }; + let Ok((mut socket, _)) = accepted else { break }; + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + match socket.read(&mut chunk).await { + Ok(0) | Err(_) => break, + Ok(read) => request.extend_from_slice(&chunk[..read]), + } + } + requests.push(String::from_utf8_lossy(&request).into_owned()); + let head = "HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1/secret\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + } + requests + }); + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://{proxy_addr}")).expect("test proxy")) + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("mock client"); + app_state.http_clients.resource_public_no_redirect.store(Arc::new(client)); + + let response = resource_proxy_response(&app_state, "http://8.8.8.8/logo.png", &HeaderMap::new(), None).await; + + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + assert!(response.headers().get("location").is_none(), "the local destination must not be relayed"); + let requests = proxy_task.await.expect("mock proxy task"); + assert_eq!(requests.len(), 1, "the local destination must not be requested: {requests:?}"); +} + +#[tokio::test] +async fn no_redirect_resource_does_not_relay_upstream_error_details() { + let app_state = create_test_app_state(); + let body = b"internal service error".to_vec(); + let response_head = + format!("HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\nContent-Length: {}\r\n\r\n", body.len()); + let (proxy_addr, proxy_task) = spawn_legacy_hls_test_origin(response_head, body.clone()).await; + app_state.app_config.config.store(Arc::new(Config { + proxy: Some(crate::model::ProxyConfig { url: format!("http://{proxy_addr}"), username: None, password: None }), + ..Config::default() + })); + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://{proxy_addr}")).expect("test proxy")) + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("mock upstream client"); + app_state.http_clients.resource_no_redirect.store(Arc::new(client)); + + let response = resource_response( + &app_state, + ResourceFetchPolicy::NonPublic, + "http://10.0.0.1/icon.png", + &HeaderMap::new(), + None, + ) + .await + .into_response(); + + // The client learns that the fetch failed, not what the destination answered. + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + let relayed = response.into_body().collect().await.expect("response body").to_bytes(); + assert!(relayed.is_empty(), "upstream error body must not be relayed: {relayed:?}"); + + let request = tokio::time::timeout(std::time::Duration::from_secs(2), proxy_task) + .await + .expect("request must reach proxy") + .expect("proxy task"); + assert!(request.starts_with("GET http://10.0.0.1/icon.png HTTP/1.1\r\n"), "{request}"); +} + +#[tokio::test] +async fn resource_client_refuses_names_that_resolve_to_local_addresses() { + let app_state = create_test_app_state(); + let (origin_addr, _origin_task) = + spawn_legacy_hls_test_origin("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n".to_string(), Vec::new()).await; + let guarded = crate::api::model::create_resource_http_client_no_redirect(&app_state.app_config) + .expect("resource HTTP client"); + + // `localhost` resolves to loopback on the client's DNS layer, so the request must fail while the + // name is resolved and never reach the origin that is listening on that loopback address. + let error = guarded + .get(format!("http://localhost:{}/icon.png", origin_addr.port())) + .send() + .await + .expect_err("a name resolving to a local address must be refused"); + + // reqwest only reports the outer failure, so the cause chain has to be walked to see the refusal. + let mut causes = error.to_string(); + let mut source = std::error::Error::source(&error); + while let Some(cause) = source { + let _ = write!(causes, " | {cause}"); + source = cause.source(); + } + assert!(causes.contains("local to this host"), "{causes}"); +} diff --git a/backend/app/src/api/api_utils/tests/serialization.rs b/backend/app/src/api/api_utils/tests/serialization.rs new file mode 100644 index 000000000..121db2a70 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/serialization.rs @@ -0,0 +1,33 @@ +use super::{coalesce_byte_stream, stream_json_array_stream}; +use bytes::Bytes; +use futures::{stream, StreamExt}; +use http_body_util::BodyExt; + +#[tokio::test] +async fn streamed_json_array_coalesces_small_entries() { + let response = stream_json_array_stream(stream::iter(0..4_096u32)); + let mut body = response.into_body(); + let mut frames = 0usize; + let mut bytes = Vec::new(); + while let Some(frame) = body.frame().await { + let Ok(frame) = frame else { + return; + }; + if let Ok(data) = frame.into_data() { + frames += 1; + bytes.extend_from_slice(&data); + } + } + assert!(frames <= 2, "small JSON entries should be coalesced, got {frames} frames"); + let decoded = serde_json::from_slice::>(&bytes); + assert!(decoded.is_ok_and(|values| values.len() == 4_096)); +} + +#[tokio::test] +async fn coalesced_stream_remains_finished_when_polled_again() { + let stream = coalesce_byte_stream(stream::empty::>()); + futures::pin_mut!(stream); + + assert!(stream.next().await.is_none()); + assert!(stream.next().await.is_none()); +} diff --git a/backend/app/src/api/api_utils/tests/session_identity.rs b/backend/app/src/api/api_utils/tests/session_identity.rs new file mode 100644 index 000000000..cefcfa1f4 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/session_identity.rs @@ -0,0 +1,267 @@ +use super::{ + allows_provider_pool_failover, create_playback_session_fingerprint, create_session_fingerprint, + create_test_app_state, create_test_fingerprint, create_test_live_channel, get_session_reservation_ttl_secs, + is_session_based_playback, is_socket_bound_playback_session, +}; +use crate::{auth::Fingerprint, model::ProxyUserCredentials}; +use shared::{ + defaults::{default_hls_session_ttl_secs, DASH_EXT, HLS_EXT}, + model::{PlaylistItemType, UserConnectionPermission}, + utils::Internable, +}; +use std::net::SocketAddr; + +#[tokio::test] +async fn test_get_session_reservation_ttl_secs_uses_hls_ttl_for_live_dash() { + let app_state = create_test_app_state(); + assert_eq!( + get_session_reservation_ttl_secs(&app_state, PlaylistItemType::LiveDash), + default_hls_session_ttl_secs() + ); +} + +#[test] +fn provider_affinity_policy_matches_stream_types() { + assert!(!PlaylistItemType::Live.requires_provider_affinity()); + assert!(!PlaylistItemType::LiveUnknown.requires_provider_affinity()); + assert!(PlaylistItemType::LiveHls.requires_provider_affinity()); + assert!(PlaylistItemType::LiveDash.requires_provider_affinity()); + assert!(PlaylistItemType::Video.requires_provider_affinity()); + assert!(PlaylistItemType::Series.requires_provider_affinity()); + assert!(PlaylistItemType::Catchup.requires_provider_affinity()); + + assert!(!allows_provider_pool_failover(PlaylistItemType::LiveHls)); + assert!(!allows_provider_pool_failover(PlaylistItemType::LiveDash)); + assert!(allows_provider_pool_failover(PlaylistItemType::Video)); + assert!(allows_provider_pool_failover(PlaylistItemType::Series)); + assert!(!allows_provider_pool_failover(PlaylistItemType::Catchup)); +} + +#[tokio::test] +async fn socket_bound_playback_sessions_enforce_hard_limits_per_socket() { + let app_state = create_test_app_state(); + let mut user = ProxyUserCredentials::default(); + user.username = "user1".to_string(); + user.max_connections = 1; + + let first_addr: std::net::SocketAddr = "127.0.0.1:55171".parse().unwrap_or_else(|_| unreachable!()); + let second_addr: std::net::SocketAddr = "127.0.0.1:55172".parse().unwrap_or_else(|_| unreachable!()); + let first_fingerprint = create_test_fingerprint(first_addr); + let first_token = create_session_fingerprint(&first_fingerprint, &user.username, 5001, true); + let second_fingerprint = create_test_fingerprint(second_addr); + let second_token = create_session_fingerprint(&second_fingerprint, &user.username, 5001, true); + + app_state.connection_manager.add_connection(&first_addr).await; + app_state.connection_manager.add_connection(&second_addr).await; + + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: &first_token, + virtual_id: 5001, + provider: "provider-a", + stream_url: "http://provider-1.example/vod/5001.ts", + addr: &first_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: true, + }) + .await; + + app_state + .active_users + .update_connection(crate::api::model::ActiveUserConnectionParams { + uid: 5001, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &first_fingerprint, + provider: "provider-a".intern(), + stream_channel: &create_test_live_channel("http://provider-1.example/vod/5001.ts"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some(&first_token), + }) + .await; + + let admission = app_state + .active_users + .connection_admission_for_session(&user.username, user.max_connections, user.soft_connections, &second_token) + .await; + assert_eq!(admission.permission(), UserConnectionPermission::Exhausted); +} + +#[tokio::test] +async fn socket_bound_playback_sessions_still_allow_soft_slots() { + let app_state = create_test_app_state(); + let mut user = ProxyUserCredentials::default(); + user.username = "soft-user".to_string(); + user.max_connections = 1; + user.soft_connections = 1; + user.priority = 0; + user.soft_priority = 9; + + let first_addr: std::net::SocketAddr = "127.0.0.1:55173".parse().unwrap_or_else(|_| unreachable!()); + let second_addr: std::net::SocketAddr = "127.0.0.1:55174".parse().unwrap_or_else(|_| unreachable!()); + let third_addr: std::net::SocketAddr = "127.0.0.1:55175".parse().unwrap_or_else(|_| unreachable!()); + let first_fingerprint = create_test_fingerprint(first_addr); + let second_fingerprint = create_test_fingerprint(second_addr); + let first_token = create_session_fingerprint(&first_fingerprint, &user.username, 6001, true); + let second_token = create_session_fingerprint(&second_fingerprint, &user.username, 6001, true); + let third_fingerprint = create_test_fingerprint(third_addr); + let third_token = create_session_fingerprint(&third_fingerprint, &user.username, 6001, true); + + app_state.connection_manager.add_connection(&first_addr).await; + app_state.connection_manager.add_connection(&second_addr).await; + + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: &first_token, + virtual_id: 6001, + provider: "provider-a", + stream_url: "http://provider-1.example/vod/6001.ts", + addr: &first_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: true, + }) + .await; + app_state + .active_users + .update_connection(crate::api::model::ActiveUserConnectionParams { + uid: 6001, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: user.priority, + soft_priority: user.soft_priority, + fingerprint: &first_fingerprint, + provider: "provider-a".intern(), + stream_channel: &create_test_live_channel("http://provider-1.example/vod/6001.ts"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some(&first_token), + }) + .await; + + let second_admission = app_state + .active_users + .connection_admission_for_session(&user.username, user.max_connections, user.soft_connections, &second_token) + .await; + assert_eq!(second_admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(second_admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); + + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: &second_token, + virtual_id: 6001, + provider: "provider-a", + stream_url: "http://provider-1.example/vod/6001.ts", + addr: &second_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Soft), + socket_bound: true, + }) + .await; + app_state + .active_users + .update_connection(crate::api::model::ActiveUserConnectionParams { + uid: 6002, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: crate::api::model::ConnectionKind::Soft, + priority: user.priority, + soft_priority: user.soft_priority, + fingerprint: &second_fingerprint, + provider: "provider-a".intern(), + stream_channel: &create_test_live_channel("http://provider-1.example/vod/6002.ts"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some(&second_token), + }) + .await; + + let third_admission = app_state + .active_users + .connection_admission_for_session(&user.username, user.max_connections, user.soft_connections, &third_token) + .await; + assert_eq!(third_admission.permission(), UserConnectionPermission::Exhausted); +} + +#[test] +fn session_based_playback_matches_adaptive_types_and_extensions() { + assert!(is_session_based_playback(PlaylistItemType::LiveHls, None)); + assert!(is_session_based_playback(PlaylistItemType::LiveDash, None)); + assert!(is_session_based_playback(PlaylistItemType::Live, Some(HLS_EXT))); + assert!(is_session_based_playback(PlaylistItemType::Live, Some(DASH_EXT))); + assert!(!is_session_based_playback(PlaylistItemType::Video, None)); +} + +#[test] +fn create_session_fingerprint_switches_between_logical_and_socket_bound_keys() { + let fingerprint = create_test_fingerprint("127.0.0.1:55176".parse().unwrap_or_else(|_| unreachable!())); + let logical = create_session_fingerprint(&fingerprint, "user1", 7001, false); + let socket_bound = create_session_fingerprint(&fingerprint, "user1", 7001, true); + + assert_ne!(logical, socket_bound); + assert!(logical.contains(&fingerprint.key)); + assert!(socket_bound.contains(&fingerprint.addr.to_string())); +} + +#[test] +fn adaptive_playback_session_fingerprint_is_logical_across_initial_sockets() { + let Some(first_addr) = "127.0.0.1:55177".parse().ok() else { + return; + }; + let Some(second_addr) = "127.0.0.1:55178".parse().ok() else { + return; + }; + let first = Fingerprint::new("10.0.0.6|player".to_string(), "10.0.0.6".to_string(), first_addr); + let second = Fingerprint::new(first.key.clone(), first.client_ip.clone(), second_addr); + + let first_token = create_playback_session_fingerprint(&first, "user1", 7002, PlaylistItemType::Live, Some(HLS_EXT)); + let second_token = + create_playback_session_fingerprint(&second, "user1", 7002, PlaylistItemType::Live, Some(HLS_EXT)); + + assert_eq!(first_token, second_token); + assert!(first_token.contains(&first.key)); + assert!(!first_token.contains(&first.addr.to_string())); + assert!(!second_token.contains(&second.addr.to_string())); +} + +#[test] +fn playback_session_fingerprint_keeps_ts_socket_bound_but_vod_logical() { + let first_addr: SocketAddr = "127.0.0.1:55179".parse().unwrap_or_else(|_| unreachable!()); + let second_addr: SocketAddr = "127.0.0.1:55180".parse().unwrap_or_else(|_| unreachable!()); + let first = Fingerprint::new("10.0.0.7|player".to_string(), "10.0.0.7".to_string(), first_addr); + let second = Fingerprint::new(first.key.clone(), first.client_ip.clone(), second_addr); + + let first_ts = create_playback_session_fingerprint(&first, "user1", 7003, PlaylistItemType::Live, None); + let second_ts = create_playback_session_fingerprint(&second, "user1", 7003, PlaylistItemType::Live, None); + let first_vod = create_playback_session_fingerprint(&first, "user1", 7003, PlaylistItemType::Video, None); + let second_vod = create_playback_session_fingerprint(&second, "user1", 7003, PlaylistItemType::Video, None); + + assert_ne!(first_ts, second_ts, "plain TS live remains socket-bound"); + assert_eq!(first_vod, second_vod, "VOD remains logical across reopen/seek sockets"); +} + +#[test] +fn socket_bound_playback_session_matches_only_plain_live_playback() { + assert!(is_socket_bound_playback_session(PlaylistItemType::Live, None)); + assert!(!is_socket_bound_playback_session(PlaylistItemType::Live, Some(HLS_EXT))); + assert!(!is_socket_bound_playback_session(PlaylistItemType::Live, Some(DASH_EXT))); + assert!(!is_socket_bound_playback_session(PlaylistItemType::LiveHls, None)); + assert!(!is_socket_bound_playback_session(PlaylistItemType::Video, None)); + assert!(!is_socket_bound_playback_session(PlaylistItemType::Series, None)); + assert!(!is_socket_bound_playback_session(PlaylistItemType::Catchup, None)); +} diff --git a/backend/app/src/api/api_utils/tests/stalker.rs b/backend/app/src/api/api_utils/tests/stalker.rs new file mode 100644 index 000000000..dde107e67 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/stalker.rs @@ -0,0 +1,132 @@ +use super::{ + create_test_app_state_for_config, create_test_dual_provider_app_config, create_test_fingerprint, + create_test_provider_app_config, needs_initial_stalker_resolution, resolve_streaming_strategy, + should_refresh_stalker_playback, stalker_stream_kind, StreamingAcquireOptions, +}; +use crate::{ + api::model::{ProviderStreamCustomReason, ProviderStreamState}, + model::SourcesConfig, +}; +use axum::http::StatusCode; +use shared::{ + model::{InputType, PlaylistItemType, StalkerStreamKind, XtreamCluster}, + utils::Internable, +}; +use std::{net::SocketAddr, sync::Arc}; + +#[test] +fn stalker_playback_refreshes_invalid_or_rejected_urls() { + assert!(should_refresh_stalker_playback(InputType::Stalker, false, None)); + assert!(should_refresh_stalker_playback(InputType::Stalker, true, Some(StatusCode::UNAUTHORIZED))); + assert!(!should_refresh_stalker_playback(InputType::Stalker, true, Some(StatusCode::OK))); + assert!(!should_refresh_stalker_playback(InputType::Xtream, false, None)); +} + +#[test] +fn initial_stalker_playback_resolves_only_empty_urls() { + assert!(needs_initial_stalker_resolution(InputType::Stalker, "")); + assert!(!needs_initial_stalker_resolution(InputType::Stalker, "https://stream.example/live.ts")); + assert!(!needs_initial_stalker_resolution(InputType::Xtream, "")); + assert_eq!(stalker_stream_kind(XtreamCluster::Live, PlaylistItemType::Catchup), StalkerStreamKind::Archive); +} + +#[tokio::test] +async fn resolve_streaming_strategy_accepts_stalker_portal_url() { + let app_config = create_test_provider_app_config(); + let Some(configured_input) = app_config.sources.load().inputs.first().cloned() else { unreachable!() }; + let mut stalker_input = (*configured_input).clone(); + stalker_input.input_type = InputType::Stalker; + stalker_input.username = None; + stalker_input.password = None; + app_config + .sources + .store(Arc::new(SourcesConfig { inputs: vec![Arc::new(stalker_input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(app_config)); + let input_name = "provider_1".intern(); + let input = + app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); + let addr: SocketAddr = "127.0.0.1:55307".parse().unwrap_or_else(|_| unreachable!()); + let stream_url = "http://line.example/play/live.php?mac=00:11:22:33:44:55&stream=347&extension=ts&play_token=abc"; + + let strategy = resolve_streaming_strategy( + &app_state, + stream_url, + &create_test_fingerprint(addr), + &input, + StreamingAcquireOptions { + force_provider: None, + allow_forced_provider_fallback: false, + allow_provider_grace: false, + user_priority: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + session_owner: Some("live-session"), + playback_kind: crate::model::PlaybackKind::LiveTs, + accept_requested_stream_url: false, + capacity_wait_timeout: None, + }, + None, + ) + .await; + + let ProviderStreamState::Available(Some(provider), url) = strategy.provider_stream_state else { unreachable!() }; + assert_eq!(provider.as_ref(), "provider_1"); + assert_eq!(url.as_ref(), stream_url); + + app_state.active_provider.release_connection(&addr); +} + +#[tokio::test] +async fn resolve_streaming_strategy_rejects_stalker_url_after_forced_provider_fallback() { + let app_config = create_test_dual_provider_app_config(); + let Some(configured_input) = app_config.sources.load().inputs.first().cloned() else { unreachable!() }; + let mut stalker_input = (*configured_input).clone(); + stalker_input.input_type = InputType::Stalker; + app_config + .sources + .store(Arc::new(SourcesConfig { inputs: vec![Arc::new(stalker_input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(app_config)); + let input_name = "provider_1".intern(); + let input = + app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); + let pinned_provider = "provider_1".intern(); + let busy_addr: SocketAddr = "127.0.0.1:55308".parse().unwrap_or_else(|_| unreachable!()); + let fallback_addr: SocketAddr = "127.0.0.1:55309".parse().unwrap_or_else(|_| unreachable!()); + let stream_url = "http://line.example/play/live.php?mac=00:11:22:33:44:55&stream=347&extension=ts&play_token=abc"; + + let busy = app_state.active_provider.acquire_exact_connection_with_grace( + &pinned_provider, + &busy_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ); + assert!(busy.is_some(), "setup should occupy the pinned provider"); + + let strategy = resolve_streaming_strategy( + &app_state, + stream_url, + &create_test_fingerprint(fallback_addr), + &input, + StreamingAcquireOptions { + force_provider: Some(&pinned_provider), + allow_forced_provider_fallback: true, + allow_provider_grace: false, + user_priority: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + session_owner: Some("live-session"), + playback_kind: crate::model::PlaybackKind::LiveTs, + accept_requested_stream_url: false, + capacity_wait_timeout: None, + }, + None, + ) + .await; + + assert!(matches!( + strategy.provider_stream_state, + ProviderStreamState::Custom { reason: ProviderStreamCustomReason::UnmappedProviderUrl, .. } + )); + + app_state.active_provider.release_connection(&busy_addr); + app_state.active_provider.release_connection(&fallback_addr); +} diff --git a/backend/app/src/api/api_utils/tests/stream_responses.rs b/backend/app/src/api/api_utils/tests/stream_responses.rs new file mode 100644 index 000000000..b87920d8b --- /dev/null +++ b/backend/app/src/api/api_utils/tests/stream_responses.rs @@ -0,0 +1,84 @@ +use super::{ + create_test_app_state_for_config, create_test_fingerprint, create_test_local_channel, + create_test_provider_app_config, force_provider_stream_response, ForceStreamRequestContext, +}; +use crate::{ + api::model::UserSession, + model::{ConfigInput, ProxyUserCredentials, SourcesConfig}, +}; +use axum::{http::HeaderMap, response::IntoResponse}; +use shared::{ + model::{InputFetchMethod, InputType, PlaylistItemType, UserConnectionPermission, XtreamCluster}, + utils::Internable, +}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; + +pub(in crate::api::api_utils::tests) async fn forced_legacy_hls_test_response( + origin_addr: SocketAddr, + request_headers: &HeaderMap, + client_port: u16, +) -> axum::response::Response { + let origin_url = format!("http://{origin_addr}/segment.ts"); + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::from([("Accept-Encoding".to_string(), "gzip".to_string())]), + url: format!("http://{origin_addr}"), + enabled: true, + priority: 0, + max_connections: 1, + method: InputFetchMethod::default(), + ..ConfigInput::default() + }); + let app_config = create_test_provider_app_config(); + app_config.sources.store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(app_config)); + let client_addr = SocketAddr::from(([127, 0, 0, 1], client_port)); + let fingerprint = create_test_fingerprint(client_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "viewer".to_string(); + let session = UserSession { + token: format!("legacy-hls-marker-{client_port}"), + transition_version: 1, + virtual_id: 41, + provider: Arc::clone(&input.name), + stream_url: origin_url.as_str().intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(true)), + user_agent_stream_index: None, + addr: client_addr, + socket_bound: false, + active_addrs: vec![client_addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + }; + let mut stream_channel = create_test_local_channel(&origin_url); + stream_channel.provider_id = u32::from(input.id); + stream_channel.input_name = Arc::clone(&input.name); + stream_channel.item_type = PlaylistItemType::Catchup; + stream_channel.cluster = XtreamCluster::Live; + stream_channel.url = origin_url.as_str().intern(); + + force_provider_stream_response( + &fingerprint, + &app_state, + &session, + stream_channel, + ForceStreamRequestContext { + req_headers: request_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response() +} diff --git a/backend/app/src/api/api_utils/tests/stream_responses_provider_capacity.rs b/backend/app/src/api/api_utils/tests/stream_responses_provider_capacity.rs new file mode 100644 index 000000000..cd1c2df03 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/stream_responses_provider_capacity.rs @@ -0,0 +1,382 @@ +use super::{ + create_stream_response_details, create_test_app_state, create_test_app_state_for_config, create_test_fingerprint, + create_test_live_channel, create_test_provider_app_config, create_test_provider_app_state, get_stream_options, + should_pin_provider_for_session, StreamResponseMode, +}; +use crate::{ + api::model::{CustomVideoStreamType, StreamDetails}, + model::{ConfigInput, ConfigInputAlias, GracePeriodOptions, SourcesConfig}, +}; +use arc_swap::ArcSwap; +use axum::http::{HeaderMap, StatusCode}; +use shared::{ + model::{InputType, PlaylistItemType, UserConnectionPermission, VirtualId}, + utils::Internable, +}; +use std::{net::SocketAddr, sync::Arc}; +use tokio::{io::AsyncWriteExt, net::TcpListener}; + +#[tokio::test] +async fn failed_provider_open_preserves_other_allocation_on_same_socket() -> Result<(), Box> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let upstream = listener.local_addr()?; + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + socket.write_all(b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await + }); + let mut config = create_test_provider_app_config(); + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + enabled: true, + input_type: InputType::Xtream, + url: format!("http://{upstream}"), + username: Some("user1".to_string()), + password: Some("pass1".to_string()), + max_connections: 2, + ..ConfigInput::default() + }); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + let addr = "127.0.0.1:55144".parse()?; + let live = app + .active_provider + .acquire_connection(&input.name, &addr, 0, crate::api::model::ConnectionKind::Normal) + .ok_or("live allocation missing")?; + let url = format!("http://{upstream}/live/user1/pass1/100.ts"); + let channel = create_test_live_channel(&url); + let details = create_stream_response_details( + &app, + &get_stream_options(&app.app_config, StreamResponseMode::Stream), + &url, + "failing-user", + &create_test_fingerprint(addr), + &HeaderMap::new(), + &input, + &channel, + PlaylistItemType::Live, + crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, + false, + UserConnectionPermission::Allowed, + None, + true, + false, + VirtualId::new(channel.virtual_id), + 0, + crate::api::model::ConnectionKind::Normal, + false, + Some("failed-session"), + None, + false, + None, + None, + None, + None, + ) + .await?; + assert!(details.provider_handle.is_none()); + assert_eq!(app.active_provider.get_provider_connections_count(), 1); + assert!(!live.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); + app.active_provider.release_handle(&live); + server.await??; + Ok(()) +} + +#[tokio::test] +async fn recording_provider_reservation_opens_without_second_slot() -> Result<(), Box> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let upstream = listener.local_addr()?; + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\ndata").await + }); + let mut config = create_test_provider_app_config(); + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + enabled: true, + input_type: InputType::Xtream, + url: format!("http://{upstream}"), + username: Some("user1".to_string()), + password: Some("pass1".to_string()), + max_connections: 1, + ..ConfigInput::default() + }); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + let reserved = + app.active_provider.acquire_connection_for_download(&input.name, 0).ok_or("recording reservation missing")?; + let allocation_id = reserved.allocation_id; + let claimed = app + .active_provider + .claim_download_connection(allocation_id, &input.name) + .ok_or("recording reservation claim missing")?; + let managed = tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app.active_provider), claimed); + let addr = "127.0.0.1:55145".parse()?; + let url = format!("http://{upstream}/live/user1/pass1/100.ts"); + let channel = create_test_live_channel(&url); + let details = create_stream_response_details( + &app, + &get_stream_options(&app.app_config, StreamResponseMode::Stream), + &url, + "recording-user", + &create_test_fingerprint(addr), + &HeaderMap::new(), + &input, + &channel, + PlaylistItemType::Live, + crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, + false, + UserConnectionPermission::Allowed, + None, + true, + false, + VirtualId::new(channel.virtual_id), + 0, + crate::api::model::ConnectionKind::Normal, + false, + Some("recording-session"), + None, + false, + None, + None, + None, + Some(managed), + ) + .await?; + + assert!(details.stream.is_some(), "the claimed reservation must open the provider body"); + assert!(details.custom_reason.is_none(), "the request must not fall back to ProviderExhausted"); + assert_eq!(app.active_provider.get_provider_connections_count(), 1, "recording consumes exactly one slot"); + + drop(details); + app.active_provider.complete_release(allocation_id); + server.await??; + Ok(()) +} + +#[tokio::test] +async fn ts_direct_capacity_spreads_across_priority_groups_then_rejects_sixth() { + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_a".intern(), + enabled: true, + input_type: InputType::Xtream, + url: "http://127.0.0.1:1".to_string(), + username: Some("user-a".to_string()), + password: Some("pass-a".to_string()), + priority: 0, + max_connections: 2, + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "provider_b".intern(), + url: "http://127.0.0.1:2".to_string(), + username: Some("user-b".to_string()), + password: Some("pass-b".to_string()), + priority: 1, + max_connections: 3, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }); + let mut config = create_test_provider_app_config(); + config.sources = + Arc::new(ArcSwap::from_pointee(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app = create_test_app_state_for_config(Arc::new(config)); + + let addr: SocketAddr = "127.0.0.1:55120".parse().unwrap_or_else(|_| unreachable!()); + let mut handles = Vec::new(); + for _ in 0..5 { + let handle = app.active_provider.acquire_connection_with_grace( + &input.name, + &addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ); + assert!(handle.is_some(), "five allocations must succeed against A(2)+B(3)"); + handles.push(handle.expect("allocation present")); + } + + let sixth = app.active_provider.acquire_connection_with_grace( + &input.name, + &addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ); + assert!(sixth.is_none(), "sixth start must be rejected once A(2)+B(3) are exhausted"); + + let active = app.active_provider.active_connections().expect("active connection map present"); + assert_eq!(active.get("provider_a").copied(), Some(5), "multi lineup must hold five allocations in total"); + + let capacities = app.active_provider.provider_capacities_for_input(&input.name); + let current_of = |name: &str| capacities.iter().find(|(n, _, _)| n.as_ref() == name).map(|(_, cur, _)| *cur); + assert_eq!(current_of("provider_a"), Some(2), "priority group A must hold two allocations"); + assert_eq!(current_of("provider_b"), Some(3), "priority group B must hold three allocations"); + + for handle in &handles { + app.active_provider.release_handle(handle); + } + assert_eq!(app.active_provider.get_provider_connections_count(), 0); +} + +/// Regression test for: when a catchup request fails upstream (e.g. provider returns +/// 4xx/5xx) the connection-slot is released, but the provider account was being +/// pinned via `refresh_provider_reservation` for `catchup_session_ttl_secs`. This +/// blocked other sessions of the same family from acquiring the same provider even +/// though the slot was already free. The fix delegates the pinning decision to +/// `should_pin_provider_for_session` and skips the reservation when the response +/// is a non-Provisioning custom video (failure fallback). Provisioning custom videos +/// must keep their reservation since they represent a successful provider handoff. +#[tokio::test] +async fn should_pin_provider_for_session_skips_reservation_on_failure_custom_video() { + let app_state = create_test_app_state(); + let no_video_details = StreamDetails { + shared_subscriber_id: None, + stream: None, + stream_info: Some((Vec::new(), StatusCode::OK, None, None)), + provider_name: Some("provider_1".intern()), + request_url: None, + session_headers: None, + provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), + user_agent_stream_index: None, + grace_period: GracePeriodOptions::default(), + provider_grace_active: false, + disable_provider_grace: false, + reconnect_flag: None, + provider_handle: None, + content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, + grace_resolution_context: None, + custom_reason: None, + response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), + session_registration: None, + }; + assert!( + should_pin_provider_for_session(&no_video_details, &app_state, PlaylistItemType::Catchup), + "a real provider stream (no CustomVideoStreamType) must pin the provider" + ); + + let provisioning_details = StreamDetails { + shared_subscriber_id: None, + stream: None, + stream_info: Some((Vec::new(), StatusCode::OK, None, Some(CustomVideoStreamType::Provisioning))), + provider_name: Some("provider_1".intern()), + request_url: None, + session_headers: None, + provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), + user_agent_stream_index: None, + grace_period: GracePeriodOptions::default(), + provider_grace_active: false, + disable_provider_grace: false, + reconnect_flag: None, + provider_handle: None, + content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, + grace_resolution_context: None, + custom_reason: None, + response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), + session_registration: None, + }; + assert!( + should_pin_provider_for_session(&provisioning_details, &app_state, PlaylistItemType::Catchup), + "a Provisioning custom video represents a successful provider handoff and must pin" + ); + + for failure_type in [ + CustomVideoStreamType::ChannelUnavailable, + CustomVideoStreamType::ProviderConnectionsExhausted, + CustomVideoStreamType::UserConnectionsExhausted, + CustomVideoStreamType::UserAccountExpired, + CustomVideoStreamType::LowPriorityPreempted, + ] { + let failure_details = StreamDetails { + shared_subscriber_id: None, + stream: None, + stream_info: Some((Vec::new(), StatusCode::BAD_REQUEST, None, Some(failure_type))), + provider_name: Some("provider_1".intern()), + request_url: None, + session_headers: None, + provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), + user_agent_stream_index: None, + grace_period: GracePeriodOptions::default(), + provider_grace_active: false, + disable_provider_grace: false, + reconnect_flag: None, + provider_handle: None, + content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, + grace_resolution_context: None, + custom_reason: None, + response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), + session_registration: None, + }; + assert!( + !should_pin_provider_for_session(&failure_details, &app_state, PlaylistItemType::Catchup), + "{failure_type:?} is a failure fallback — must NOT pin the provider" + ); + } +} + +#[tokio::test] +async fn intentional_deferred_open_retains_provider_grace_handle() { + let app_state = create_test_provider_app_state(); + let provider_name = "provider_1".intern(); + let holder_addr: SocketAddr = "127.0.0.1:55230".parse().unwrap_or_else(|_| unreachable!()); + let deferred_addr: SocketAddr = "127.0.0.1:55231".parse().unwrap_or_else(|_| unreachable!()); + let holder_handle = app_state + .active_provider + .acquire_exact_connection_with_grace( + &provider_name, + &holder_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .expect("holder occupies the provider slot"); + let input = app_state.app_config.get_input_by_name(&provider_name).expect("provider input"); + let stream_url = "http://provider-1.example/live/user1/pass1/100.m3u8"; + let mut channel = create_test_live_channel(stream_url); + channel.item_type = PlaylistItemType::LiveHls; + let fingerprint = create_test_fingerprint(deferred_addr); + + let mut details = create_stream_response_details( + &app_state, + &get_stream_options(&app_state.app_config, StreamResponseMode::Stream), + stream_url, + "deferred-user", + &fingerprint, + &HeaderMap::new(), + &input, + &channel, + PlaylistItemType::LiveHls, + crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, + false, + UserConnectionPermission::Allowed, + None, + true, + true, + VirtualId::new(channel.virtual_id), + 0, + crate::api::model::ConnectionKind::Normal, + false, + Some("deferred-session"), + None, + false, + Some(true), + None, + None, + None, + ) + .await + .expect("provider grace creates deferred stream details"); + + assert!(details.stream.is_none()); + assert!(details.has_deferred_provider_open()); + assert!(details.provider_handle.is_some(), "deferred open must retain its provider allocation"); + + app_state.connection_manager.release_managed_provider_handle(details.provider_handle.take()); + app_state.connection_manager.release_provider_handle(Some(holder_handle)); +} diff --git a/backend/app/src/api/api_utils/tests/stream_responses_provider_fallback.rs b/backend/app/src/api/api_utils/tests/stream_responses_provider_fallback.rs new file mode 100644 index 000000000..67f602c30 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/stream_responses_provider_fallback.rs @@ -0,0 +1,472 @@ +use super::{ + create_stream_response_details, create_test_app_state_for_config, create_test_dual_provider_app_config, + create_test_dual_provider_app_state, create_test_fingerprint, create_test_live_channel, + force_provider_stream_response, get_stream_options, load_test_user, resolve_streaming_strategy, + spawn_legacy_hls_test_origin, ForceStreamRequestContext, StreamResponseMode, StreamingAcquireOptions, +}; +use crate::{ + api::model::{ProviderStreamCustomReason, ProviderStreamState}, + model::{ConfigInput, ConfigInputAlias, SourcesConfig}, +}; +use axum::{ + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; +use http_body_util::BodyExt; +use shared::{ + model::{InputType, PlaylistItemType, UserConnectionPermission, VirtualId}, + utils::Internable, +}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; + +#[tokio::test] +async fn resolve_streaming_strategy_honors_forced_provider_fallback_policy() { + let app_state = create_test_dual_provider_app_state(); + let input_name = "provider_1".intern(); + let input = + app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); + let pinned_provider = "provider_1".intern(); + let busy_addr: SocketAddr = "127.0.0.1:55301".parse().unwrap_or_else(|_| unreachable!()); + let strict_addr: SocketAddr = "127.0.0.1:55302".parse().unwrap_or_else(|_| unreachable!()); + let fallback_addr: SocketAddr = "127.0.0.1:55303".parse().unwrap_or_else(|_| unreachable!()); + let stream_url = "http://provider-1.example/movie/user1/pass1/1.mkv"; + + let busy = app_state.active_provider.acquire_exact_connection_with_grace( + &pinned_provider, + &busy_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ); + assert!(busy.is_some(), "setup should occupy the pinned provider"); + + let strict = resolve_streaming_strategy( + &app_state, + stream_url, + &create_test_fingerprint(strict_addr), + &input, + StreamingAcquireOptions { + force_provider: Some(&pinned_provider), + allow_forced_provider_fallback: false, + allow_provider_grace: false, + user_priority: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + session_owner: Some("vod-session"), + playback_kind: crate::model::PlaybackKind::Vod, + accept_requested_stream_url: true, + capacity_wait_timeout: None, + }, + None, + ) + .await; + assert!(strict.provider_handle.is_none(), "strict provider affinity should not allocate a different provider"); + assert!( + matches!( + strict.provider_stream_state, + ProviderStreamState::Custom { reason: ProviderStreamCustomReason::ProviderExhausted, .. } + ), + "strict provider affinity should fail closed when the pinned provider is unavailable" + ); + + let fallback = resolve_streaming_strategy( + &app_state, + stream_url, + &create_test_fingerprint(fallback_addr), + &input, + StreamingAcquireOptions { + force_provider: Some(&pinned_provider), + allow_forced_provider_fallback: true, + allow_provider_grace: false, + user_priority: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + session_owner: Some("live-session"), + playback_kind: crate::model::PlaybackKind::LiveTs, + accept_requested_stream_url: true, + capacity_wait_timeout: None, + }, + None, + ) + .await; + let (ProviderStreamState::Available(Some(fallback_provider), fallback_url) + | ProviderStreamState::GracePeriod(Some(fallback_provider), fallback_url)) = fallback.provider_stream_state + else { + panic!("fallback-enabled request should allocate a provider") + }; + assert_eq!(fallback_provider.as_ref(), "provider_2"); + assert_eq!(fallback_url.as_ref(), "http://provider-2.example/movie/user2/pass2/1.mkv"); + + app_state.active_provider.release_connection(&busy_addr); + app_state.active_provider.release_connection(&strict_addr); + app_state.active_provider.release_connection(&fallback_addr); +} + +#[tokio::test] +async fn create_stream_response_details_preserves_stored_headers_when_fallback_open_fails() { + let app_state = create_test_dual_provider_app_state(); + let input_name = "provider_1".intern(); + let input = + app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); + let pinned_provider = "provider_1".intern(); + let busy_addr: SocketAddr = "127.0.0.1:55306".parse().unwrap_or_else(|_| unreachable!()); + let reacquire_addr: SocketAddr = "127.0.0.1:55307".parse().unwrap_or_else(|_| unreachable!()); + let stream_url = "http://provider-1.example/movie/user1/pass1/1.mkv"; + + let user = load_test_user("test-fallback-headers-user"); + let session_token = "sess-fallback-headers-1"; + let initial_headers = HashMap::from([("cookie".to_string(), "old_prov_sess=1".to_string())]); + let created = app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token, + virtual_id: 1, + provider: &pinned_provider, + stream_url, + addr: &reacquire_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + assert!(!created.is_empty()); + app_state.active_users.update_session_provider_headers(&user.username, session_token, &initial_headers).await; + + let busy = app_state.active_provider.acquire_exact_connection_with_grace( + &pinned_provider, + &busy_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ); + assert!(busy.is_some(), "setup should occupy the pinned provider"); + + let channel = create_test_live_channel(stream_url); + let details = create_stream_response_details( + &app_state, + &get_stream_options(&app_state.app_config, StreamResponseMode::Stream), + stream_url, + &user.username, + &create_test_fingerprint(reacquire_addr), + &HeaderMap::new(), + &input, + &channel, + PlaylistItemType::Video, + crate::api::model::ProviderContentRepresentationMode::Identity, + false, + UserConnectionPermission::Allowed, + Some(&pinned_provider), + true, + false, + VirtualId::new(channel.virtual_id), + 0, + crate::api::model::ConnectionKind::Normal, + true, + Some(session_token), + Some(&initial_headers), + true, + None, + None, + None, + None, + ) + .await + .unwrap_or_else(|err| panic!("create_stream_response_details should succeed: {err}")); + + assert_eq!(details.provider_name.as_deref(), Some("provider_2")); + assert!(details.session_headers.is_none(), "fallback request should not pass pinned provider session headers"); + assert!(details.stream.is_none(), "test setup should fail to open the fallback provider stream"); + + let session = app_state + .active_users + .get_and_update_user_session(&user.username, session_token) + .await + .expect("session should exist"); + assert_eq!( + session.provider_session_headers, initial_headers, + "stored session headers should be retained when the fallback provider cannot be opened" + ); + + app_state.active_provider.release_connection(&busy_addr); + app_state.active_provider.release_connection(&reacquire_addr); +} + +#[tokio::test] +async fn force_provider_stream_response_clears_stored_headers_after_fallback_open_succeeds() { + const FALLBACK_BODY: &[u8] = b"fallback-provider"; + let response_head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + FALLBACK_BODY.len() + ); + let (origin_addr, origin_task) = spawn_legacy_hls_test_origin(response_head, FALLBACK_BODY.to_vec()).await; + let app_config = create_test_dual_provider_app_config(); + let Some(configured_input) = app_config.sources.load().inputs.first().cloned() else { unreachable!() }; + let mut fallback_input = (*configured_input).clone(); + let Some(aliases) = fallback_input.aliases.as_mut() else { unreachable!() }; + let Some(fallback_alias) = aliases.first_mut() else { unreachable!() }; + fallback_alias.url = format!("http://{origin_addr}"); + app_config + .sources + .store(Arc::new(SourcesConfig { inputs: vec![Arc::new(fallback_input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(app_config)); + let input_name = "provider_1".intern(); + let input = + app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); + let pinned_provider = "provider_1".intern(); + let busy_addr: SocketAddr = "127.0.0.1:55308".parse().unwrap_or_else(|_| unreachable!()); + let reacquire_addr: SocketAddr = "127.0.0.1:55309".parse().unwrap_or_else(|_| unreachable!()); + let stream_url = "http://provider-1.example/movie/user1/pass1/1.mkv"; + + let user = load_test_user("test-fallback-force-user"); + let session_token = "sess-fallback-force-1"; + let initial_headers = HashMap::from([("cookie".to_string(), "old_pinned_token=abc".to_string())]); + let created = app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token, + virtual_id: 1, + provider: &pinned_provider, + stream_url, + addr: &reacquire_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + assert!(!created.is_empty()); + app_state.active_users.update_session_provider_headers(&user.username, session_token, &initial_headers).await; + + let busy = app_state.active_provider.acquire_exact_connection_with_grace( + &pinned_provider, + &busy_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ); + assert!(busy.is_some(), "setup should occupy the pinned provider"); + + let session = app_state + .active_users + .get_and_update_user_session(&user.username, session_token) + .await + .expect("session should exist"); + + let channel = create_test_live_channel(stream_url); + let response = force_provider_stream_response( + &create_test_fingerprint(reacquire_addr), + &app_state, + &session, + channel, + ForceStreamRequestContext { + req_headers: &HeaderMap::new(), + input: &input, + user: &user, + session_reservation_ttl_secs: 0, + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + None, + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::OK); + let body = response.into_body().collect().await.expect("fallback stream body").to_bytes(); + assert_eq!(body.as_ref(), FALLBACK_BODY); + let request = tokio::time::timeout(std::time::Duration::from_secs(5), origin_task) + .await + .expect("fallback request was not sent within 5 seconds") + .expect("fallback origin task completes"); + assert!(!request.is_empty(), "fallback provider must receive the stream request"); + + let updated_session = app_state + .active_users + .get_and_update_user_session(&user.username, session_token) + .await + .expect("session should exist"); + assert!( + updated_session.provider_session_headers.is_empty(), + "stored session headers must be cleared after the fallback provider stream opens" + ); + + app_state.active_provider.release_connection(&busy_addr); +} + +#[tokio::test] +async fn resolve_streaming_strategy_rewrites_stale_alias_url_to_selected_main_provider() { + let app_state = create_test_dual_provider_app_state(); + let input_name = "provider_1".intern(); + let input = + app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); + let addr: SocketAddr = "127.0.0.1:55304".parse().unwrap_or_else(|_| unreachable!()); + + let strategy = resolve_streaming_strategy( + &app_state, + "http://provider-2.example/live/user2/pass2/100.ts", + &create_test_fingerprint(addr), + &input, + StreamingAcquireOptions { + force_provider: None, + allow_forced_provider_fallback: false, + allow_provider_grace: false, + user_priority: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + session_owner: Some("live-session"), + playback_kind: crate::model::PlaybackKind::LiveTs, + accept_requested_stream_url: false, + capacity_wait_timeout: None, + }, + None, + ) + .await; + + let ProviderStreamState::Available(Some(provider), url) = strategy.provider_stream_state else { + panic!("request should allocate the main provider") + }; + assert_eq!(provider.as_ref(), "provider_1"); + assert_eq!(url.as_ref(), "http://provider-1.example/live/user1/pass1/100.ts"); + + app_state.active_provider.release_connection(&addr); +} + +#[tokio::test] +async fn resolve_streaming_strategy_rewrites_opaque_m3u_token_after_alias_allocation() { + let app_config = create_test_dual_provider_app_config(); + let input = Arc::new(ConfigInput { + id: 1, + name: "provider-a".intern(), + input_type: InputType::M3u, + url: "http://playlist.example/a.m3u?token=provider-a-token".to_string(), + enabled: true, + max_connections: 1, + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "provider-b".intern(), + url: "http://playlist.example/b.m3u?token=provider-b-token".to_string(), + username: None, + password: None, + max_connections: 0, + priority: 1, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }); + app_config.sources.store(Arc::new(SourcesConfig { inputs: vec![Arc::clone(&input)], ..SourcesConfig::default() })); + let app_state = create_test_app_state_for_config(Arc::new(app_config)); + let provider_a = "provider-a".intern(); + let busy_addr: SocketAddr = "127.0.0.1:55306".parse().unwrap_or_else(|_| unreachable!()); + let alias_addr: SocketAddr = "127.0.0.1:55307".parse().unwrap_or_else(|_| unreachable!()); + + let busy = app_state.active_provider.acquire_exact_connection_with_grace( + &provider_a, + &busy_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ); + assert!(busy.is_some(), "setup should occupy provider A"); + + let strategy = resolve_streaming_strategy( + &app_state, + "http://stream.example/channel/segment.ts?token=provider-a-token", + &create_test_fingerprint(alias_addr), + &input, + StreamingAcquireOptions { + force_provider: None, + allow_forced_provider_fallback: false, + allow_provider_grace: false, + user_priority: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + session_owner: Some("live-session"), + playback_kind: crate::model::PlaybackKind::LiveTs, + accept_requested_stream_url: false, + capacity_wait_timeout: None, + }, + None, + ) + .await; + + let ProviderStreamState::Available(Some(provider), url) = strategy.provider_stream_state else { + panic!("request should allocate provider B") + }; + assert_eq!(provider.as_ref(), "provider-b"); + assert_eq!(url.as_ref(), "http://stream.example/channel/segment.ts?token=provider-b-token"); + + app_state.active_provider.release_connection(&busy_addr); + app_state.active_provider.release_connection(&alias_addr); +} + +#[tokio::test] +async fn resolve_streaming_strategy_rejects_unmapped_provider_url() { + let app_state = create_test_dual_provider_app_state(); + let input_name = "provider_1".intern(); + let input = + app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); + let addr: SocketAddr = "127.0.0.1:55305".parse().unwrap_or_else(|_| unreachable!()); + + let strategy = resolve_streaming_strategy( + &app_state, + "http://unmapped.example/live/user1/pass1/100.ts", + &create_test_fingerprint(addr), + &input, + StreamingAcquireOptions { + force_provider: None, + allow_forced_provider_fallback: false, + allow_provider_grace: false, + user_priority: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + session_owner: Some("live-session"), + playback_kind: crate::model::PlaybackKind::LiveTs, + accept_requested_stream_url: false, + capacity_wait_timeout: None, + }, + None, + ) + .await; + + assert!(strategy.provider_handle.is_none()); + assert!(matches!( + strategy.provider_stream_state, + ProviderStreamState::Custom { reason: ProviderStreamCustomReason::UnmappedProviderUrl, .. } + )); + + app_state.active_provider.release_connection(&addr); +} + +#[tokio::test] +async fn resolve_streaming_strategy_accepts_session_requested_stream_url() { + let app_state = create_test_dual_provider_app_state(); + let input_name = "provider_1".intern(); + let input = + app_state.app_config.sources.load().get_input_by_name(&input_name).cloned().unwrap_or_else(|| unreachable!()); + let addr: SocketAddr = "127.0.0.1:55306".parse().unwrap_or_else(|_| unreachable!()); + let trusted_url = "http://unmapped.example/live/user1/pass1/100.ts"; + let strategy = resolve_streaming_strategy( + &app_state, + trusted_url, + &create_test_fingerprint(addr), + &input, + StreamingAcquireOptions { + force_provider: None, + allow_forced_provider_fallback: false, + allow_provider_grace: false, + user_priority: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + session_owner: Some("live-session"), + playback_kind: crate::model::PlaybackKind::LiveTs, + accept_requested_stream_url: true, + capacity_wait_timeout: None, + }, + None, + ) + .await; + + let ProviderStreamState::Available(Some(provider), url) = strategy.provider_stream_state else { + panic!("session-requested URL should be accepted for the pinned provider") + }; + assert_eq!(provider.as_ref(), "provider_1"); + assert_eq!(url.as_ref(), trusted_url); + + app_state.active_provider.release_connection(&addr); +} diff --git a/backend/app/src/api/api_utils/tests/stream_responses_response_headers.rs b/backend/app/src/api/api_utils/tests/stream_responses_response_headers.rs new file mode 100644 index 000000000..26ed21ec4 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/stream_responses_response_headers.rs @@ -0,0 +1,401 @@ +use super::{ + create_api_proxy_user, create_test_app_state, create_test_fingerprint, create_test_live_channel, + create_test_provider_app_state, create_test_shared_target, evaluate_network_access, is_hop_by_hop_response_header, + is_media_server_playback_url, is_media_server_stream_ref_url, is_seek_request, is_seekable_media_request, + mark_response_as_uncompressed, resolve_request_url_for_logging, resolve_stream_config_u64, resource_proxy_response, + spawn_legacy_hls_test_origin, stream_response, stream_responses::forced_legacy_hls_test_response, + user_with_network_access, NetworkAccessDecision, +}; +use crate::{ + api::model::{SharedStreamCtx, SharedStreamManager}, + model::{ConfigInput, NetworkAccess, ProxyUserCredentials}, + repository::GeoIp, +}; +use arc_swap::ArcSwapOption; +use axum::{ + http::{header, HeaderMap, HeaderName, HeaderValue, Response, StatusCode}, + response::IntoResponse, +}; +use bytes::Bytes; +use futures::stream; +use http_body_util::BodyExt; +use shared::{ + defaults::{default_catchup_session_ttl_secs, default_hls_session_ttl_secs, HLS_EXT}, + model::{ClusterFlags, GeoIpUnavailablePolicy, InputType, UserConnectionPermission, XtreamCluster}, + utils::Internable, +}; +use std::{borrow::Cow, collections::HashMap, sync::Arc}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, +}; +use tuliprox_core::utils::response_compression::should_compress_response; + +#[test] +fn test_is_seek_request() { + let mut headers = HeaderMap::new(); + + // No range header + assert!(!is_seek_request(XtreamCluster::Video, &headers)); + + // Range: bytes=0- (Should be true now to allow session takeover on restart) + headers.insert("range", "bytes=0-".parse().unwrap()); + assert!(is_seek_request(XtreamCluster::Video, &headers)); + + // Range: bytes=100- (Should be true) + headers.insert("range", "bytes=100-".parse().unwrap()); + assert!(is_seek_request(XtreamCluster::Video, &headers)); + + // Range: bytes=100-200 (Should be true) + headers.insert("range", "bytes=100-200".parse().unwrap()); + assert!(is_seek_request(XtreamCluster::Video, &headers)); + + // Live cluster should always return false + headers.insert("range", "bytes=100-".parse().unwrap()); + assert!(!is_seek_request(XtreamCluster::Live, &headers)); +} + +#[test] +fn hls_manifests_are_not_forced_as_seek_responses() { + let mut headers = HeaderMap::new(); + headers.insert("range", HeaderValue::from_static("bytes=0-")); + + assert!(!is_seekable_media_request(XtreamCluster::Video, &headers, Some(HLS_EXT))); + assert!(is_seekable_media_request(XtreamCluster::Video, &headers, Some(".ts"))); +} + +#[test] +fn media_server_proxy_response_header_filter_drops_hop_by_hop_headers() { + for name in [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "trailers", + "transfer-encoding", + "upgrade", + ] { + assert!(is_hop_by_hop_response_header(&HeaderName::from_static(name))); + } + assert!(!is_hop_by_hop_response_header(&header::CONTENT_TYPE)); +} + +#[test] +fn media_server_playback_urls_are_proxy_only_redirect_guard_candidates() { + let plex_input = ConfigInput { input_type: InputType::Plex, ..ConfigInput::default() }; + let emby_input = ConfigInput { input_type: InputType::Emby, ..ConfigInput::default() }; + let m3u_input = ConfigInput { input_type: InputType::M3u, ..ConfigInput::default() }; + + assert!(is_media_server_playback_url( + &plex_input, + "media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fredacted" + )); + assert!(is_media_server_playback_url( + &m3u_input, + "media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fredacted" + )); + assert!(is_media_server_playback_url(&plex_input, "https://plex.example/stream.mkv")); + assert!(!is_media_server_playback_url(&emby_input, "https://emby.example/stream.mkv")); + assert!(!is_media_server_playback_url(&m3u_input, "https://provider.example/stream.mkv")); + assert!(!is_media_server_stream_ref_url("https://provider.example/stream.mkv")); + assert!(is_media_server_stream_ref_url("media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fredacted")); + assert_eq!( + resolve_request_url_for_logging( + &plex_input, + "media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fredacted" + ) + .as_ref(), + "media-server://" + ); +} + +#[test] +fn test_resolve_request_url_for_logging_respects_sanitization_setting() { + static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + struct SanitizeSettingGuard(bool); + impl Drop for SanitizeSettingGuard { + fn drop(&mut self) { shared::utils::set_sanitize_sensitive_info(self.0); } + } + let _guard = LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + + let prev = shared::utils::is_sanitize_sensitive_info_enabled(); + let _restore = SanitizeSettingGuard(prev); + + let m3u_input = ConfigInput { input_type: InputType::M3u, ..ConfigInput::default() }; + let media_ref = "media-server://plex/server/rating?part_key=%2Flibrary%2Fparts%2Fsecret_key"; + let standard_url = "http://provider.example/live/stream.m3u8"; + + // 1. When sanitization is enabled: + shared::utils::set_sanitize_sensitive_info(true); + assert_eq!(resolve_request_url_for_logging(&m3u_input, media_ref).as_ref(), "media-server://"); + assert_eq!(resolve_request_url_for_logging(&m3u_input, standard_url).as_ref(), standard_url); + + // 2. When sanitization is disabled: + shared::utils::set_sanitize_sensitive_info(false); + assert_eq!(resolve_request_url_for_logging(&m3u_input, media_ref).as_ref(), media_ref); + assert_eq!(resolve_request_url_for_logging(&m3u_input, standard_url).as_ref(), standard_url); +} + +#[test] +fn test_streaming_response_extension_disables_compression() { + let mut response = Response::new(()); + mark_response_as_uncompressed(&mut response); + + assert!(!should_compress_response(&response)); +} + +#[tokio::test] +async fn forced_hls_provider_response_disables_compression_and_streams_identity_bytes() { + const IDENTITY_BODY: &[u8] = b"legacy hls identity segment"; + + let mut encoder = async_compression::tokio::write::GzipEncoder::new(Vec::new()); + encoder.write_all(IDENTITY_BODY).await.expect("gzip test body encodes"); + encoder.shutdown().await.expect("gzip test encoder finishes"); + let encoded_body = encoder.into_inner(); + + let response_head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + encoded_body.len() + ); + let (origin_addr, origin_task) = spawn_legacy_hls_test_origin(response_head, encoded_body).await; + let mut request_headers = HeaderMap::new(); + request_headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); + let response = forced_legacy_hls_test_response(origin_addr, &request_headers, 55_310).await; + + assert_eq!(response.status(), StatusCode::OK); + assert!(!should_compress_response(&response)); + assert!(!response.headers().contains_key(header::CONTENT_ENCODING)); + assert!(!response.headers().contains_key(header::CONTENT_LENGTH)); + let body = response.into_body().collect().await.expect("legacy HLS response body").to_bytes(); + assert_eq!(body.as_ref(), IDENTITY_BODY); + + let request = origin_task.await.expect("test origin task completes").to_ascii_lowercase(); + assert!(request.contains("\r\naccept-encoding: identity\r\n")); +} + +#[tokio::test] +async fn forced_hls_unencoded_partial_response_preserves_range_and_disables_compression() { + const PARTIAL_BODY: &[u8] = b"cdef"; + let response_head = format!( + "HTTP/1.1 206 Partial Content\r\nContent-Type: video/mp2t\r\nContent-Range: bytes 2-5/10\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + PARTIAL_BODY.len() + ); + let (origin_addr, origin_task) = spawn_legacy_hls_test_origin(response_head, PARTIAL_BODY.to_vec()).await; + let mut request_headers = HeaderMap::new(); + request_headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); + request_headers.insert(header::RANGE, HeaderValue::from_static("bytes=2-")); + + let response = forced_legacy_hls_test_response(origin_addr, &request_headers, 55_311).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert!(!should_compress_response(&response)); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "4"); + assert!(!response.headers().contains_key(header::CONTENT_ENCODING)); + let body = response.into_body().collect().await.expect("legacy HLS partial body").to_bytes(); + assert_eq!(body.as_ref(), PARTIAL_BODY); + + let request = origin_task.await.expect("test origin task completes").to_ascii_lowercase(); + assert!(request.contains("\r\naccept-encoding: identity\r\n")); + assert!(request.contains("\r\nrange: bytes=2-\r\n")); +} + +#[test] +fn test_regular_response_keeps_compression_enabled() { + let response = Response::new(()); + + assert!(should_compress_response(&response)); +} + +#[test] +fn test_get_stream_config_u64_uses_default_when_stream_config_missing() { + assert_eq!( + resolve_stream_config_u64(None, |stream| stream.hls_session_ttl_secs, default_hls_session_ttl_secs()), + default_hls_session_ttl_secs() + ); + assert_eq!( + resolve_stream_config_u64(None, |stream| stream.catchup_session_ttl_secs, default_catchup_session_ttl_secs()), + default_catchup_session_ttl_secs() + ); +} + +#[tokio::test] +async fn create_api_proxy_user_defaults_output_clusters_to_all() { + let app_state = create_test_app_state(); + let user = create_api_proxy_user(&app_state); + assert_eq!(user.output_clusters, ClusterFlags::all()); +} + +#[tokio::test] +async fn proxied_m3u_logo_follows_private_redirect_without_forwarding_credentials() { + let app_state = create_test_app_state(); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("mock proxy binds"); + let proxy_addr = listener.local_addr().expect("mock proxy address"); + let proxy_task = tokio::spawn(async move { + let mut requests = Vec::new(); + for response in [ + "HTTP/1.1 302 Found\r\nLocation: http://10.0.0.2/icon.png\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + "HTTP/1.1 200 OK\r\nContent-Type: image/png\r\nContent-Length: 3\r\nConnection: close\r\n\r\npng", + ] { + let (mut socket, _) = listener.accept().await.expect("mock proxy accepts request"); + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let read = socket.read(&mut chunk).await.expect("mock proxy reads request"); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + socket.write_all(response.as_bytes()).await.expect("mock proxy writes response"); + requests.push(String::from_utf8_lossy(&request).into_owned()); + } + requests + }); + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(format!("http://{proxy_addr}")).expect("mock proxy URL")) + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("mock resource client"); + app_state.http_clients.resource_no_redirect.store(Arc::new(client)); + let input = ConfigInput { + url: "http://10.0.0.1/playlist.m3u".to_string(), + headers: HashMap::from([("Authorization".to_string(), "Bearer secret".to_string())]), + ..ConfigInput::default() + }; + let mut headers = HeaderMap::new(); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer player-secret")); + + let response = resource_proxy_response(&app_state, "http://10.0.0.1/logo.png", &headers, Some(&input)).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.into_body().collect().await.expect("image body").to_bytes(), Bytes::from_static(b"png")); + + let requests = proxy_task.await.expect("mock proxy task"); + assert!(requests[0].starts_with("GET http://10.0.0.1/logo.png HTTP/1.1\r\n")); + assert!(requests[0].to_ascii_lowercase().contains("authorization: bearer secret")); + assert!(!requests[0].contains("player-secret")); + assert!(requests[1].starts_with("GET http://10.0.0.2/icon.png HTTP/1.1\r\n")); + assert!(!requests[1].to_ascii_lowercase().contains("authorization:")); +} + +#[tokio::test] +async fn stream_response_preserves_soft_kind_for_shared_reuse() { + let app_state = create_test_provider_app_state(); + let stream_url = "http://provider-1.example/live/shared.ts"; + let input_name = "provider_1".intern(); + let input = app_state.app_config.get_input_by_name(&input_name).expect("provider input should exist"); + let target = Arc::new(create_test_shared_target()); + + let owner_addr = "127.0.0.1:55140".parse().unwrap_or_else(|_| unreachable!()); + let owner_handle = app_state + .active_provider + .acquire_connection(&input.name, &owner_addr, 0, crate::api::model::ConnectionKind::Normal) + .expect("owner allocation should exist"); + let subscriber_id = + tuliprox_core::model::SharedSubscriberId::from_stream_uid(app_state.connection_manager.next_stream_uid()); + let pending_cleanup = + SharedStreamManager::reserve_subscriber_cleanup(&app_state.connection_manager, subscriber_id, owner_addr) + .await + .expect("shared cleanup admission should succeed"); + let shared_stream = stream::pending::>(); + let registered = SharedStreamManager::register_shared_stream( + SharedStreamCtx { + app_config: &app_state.app_config, + shared_stream_manager: &app_state.shared_stream_manager, + active_provider: &app_state.active_provider, + connection_manager: &app_state.connection_manager, + }, + stream_url, + shared_stream, + &owner_addr, + subscriber_id, + Vec::new(), + 1, + Some(tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.active_provider), owner_handle)), + pending_cleanup, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .await; + assert!(registered.is_some(), "shared stream should register"); + + let mut user = ProxyUserCredentials::default(); + user.username = "soft-user".to_string(); + user.max_connections = 1; + user.soft_connections = 1; + user.priority = 0; + user.soft_priority = 9; + + let normal_addr = "127.0.0.1:55141".parse().unwrap_or_else(|_| unreachable!()); + let normal_fingerprint = create_test_fingerprint(normal_addr); + let normal_channel = create_test_live_channel("http://provider-1.example/live/normal.ts"); + app_state.active_users.add_connection(&normal_addr).await; + app_state + .active_users + .update_connection(crate::api::model::ActiveUserConnectionParams { + uid: 1001, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: user.priority, + soft_priority: user.soft_priority, + fingerprint: &normal_fingerprint, + provider: input.name.clone(), + stream_channel: &normal_channel, + user_agent: Cow::Borrowed("ua"), + session_token: Some("normal-session"), + }) + .await + .expect("normal stream should register"); + + let admission = + app_state.active_users.connection_admission(&user.username, user.max_connections, user.soft_connections).await; + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); + + let soft_addr = "127.0.0.1:55142".parse().unwrap_or_else(|_| unreachable!()); + let soft_fingerprint = create_test_fingerprint(soft_addr); + let response = stream_response( + &soft_fingerprint, + &app_state, + "soft-session", + None, + create_test_live_channel(stream_url), + stream_url, + None, + &HeaderMap::default(), + &input, + &target, + &user, + admission.permission(), + admission.kind().unwrap_or(crate::api::model::ConnectionKind::Normal), + false, + None, + ) + .await + .into_response(); + assert_eq!(response.status(), StatusCode::OK); + + let session_admission = app_state + .active_users + .connection_admission_for_session(&user.username, user.max_connections, user.soft_connections, "soft-session") + .await; + assert_eq!(session_admission.kind(), Some(crate::api::model::ConnectionKind::Soft)); +} + +#[test] +fn network_denied_reason_none_when_allowed() { + let user = user_with_network_access(Some(NetworkAccess { + allowed_countries: vec!["DE".to_string()], + allowed_networks: vec![], + })); + let mock_geoip = Arc::new(ArcSwapOption::from(Some(Arc::new(GeoIp::test_new("DE"))))); + assert_eq!( + evaluate_network_access(&user, "8.8.8.8", &mock_geoip, GeoIpUnavailablePolicy::Deny), + NetworkAccessDecision::Allowed + ); +} diff --git a/backend/app/src/api/api_utils/tests/stream_responses_session_activation.rs b/backend/app/src/api/api_utils/tests/stream_responses_session_activation.rs new file mode 100644 index 000000000..a191a6223 --- /dev/null +++ b/backend/app/src/api/api_utils/tests/stream_responses_session_activation.rs @@ -0,0 +1,604 @@ +use super::{ + activate_session_before_stream_open, create_test_app_config, create_test_app_state, + create_test_app_state_for_config, create_test_app_state_with_stream_config, create_test_fingerprint, + create_test_live_channel, create_test_provider_app_config, resolve_admission_with_strategies, stream_response, + AdmissionRequest, EvictionReentryGuard, PlaybackRequestClass, SessionActivationRequest, +}; +use crate::model::{Config, ConfigTarget, ProxyUserCredentials}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use axum::{ + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; +use shared::{ + foundation::Filter, + model::{AdmissionStrategy, PlaylistItemType, ProcessingOrder, UserConnectionPermission, VirtualId}, + utils::Internable, +}; +use std::{borrow::Cow, net::SocketAddr, sync::Arc}; + +#[tokio::test] +async fn activate_session_before_stream_open_skips_placeholder_for_follow_up_session() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest]), + }); + let addr: SocketAddr = "127.0.0.1:55220".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + let input = app_state.app_config.sources.load().inputs[0].clone(); + let mut user = ProxyUserCredentials::default(); + user.username = "follow-up-user".to_string(); + user.max_connections = 1; + let mut channel = create_test_live_channel("http://provider-1.example/live/55220.m3u8"); + channel.item_type = PlaylistItemType::LiveHls; + channel.virtual_id = 55220; + + app_state.connection_manager.add_connection(&addr).await; + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "tok-follow-up", + virtual_id: channel.virtual_id, + provider: input.name.as_ref(), + stream_url: channel.url.as_ref(), + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: true, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 10, + fingerprint: &fingerprint, + provider: input.name.clone(), + stream_channel: &channel, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-follow-up"), + }) + .await; + + let activation = activate_session_before_stream_open( + &app_state, + SessionActivationRequest { + fingerprint: &fingerprint, + input: input.as_ref(), + user: &user, + session_token: "tok-follow-up", + request_class: None, + virtual_id: VirtualId::new(channel.virtual_id), + item_type: PlaylistItemType::LiveHls, + stream_url: channel.url.as_ref(), + connection_permission: UserConnectionPermission::Allowed, + connection_kind: crate::api::model::ConnectionKind::Normal, + granted_grace_mode: None, + socket_bound: true, + }, + ) + .await; + + assert_eq!(activation.admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(activation.admission.kind(), Some(crate::api::model::ConnectionKind::Normal)); + assert_eq!(activation.grace_mode, None); + assert!( + activation.placeholder_transition_version.is_none(), + "follow-up activation must not create a placeholder session" + ); +} + +#[tokio::test] +async fn activate_session_before_stream_open_revalidates_precomputed_follow_up_request_class() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest]), + }); + let addr: SocketAddr = "127.0.0.1:55221".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + let input = app_state.app_config.sources.load().inputs[0].clone(); + let mut user = ProxyUserCredentials::default(); + user.username = "precomputed-follow-up-user".to_string(); + user.max_connections = 1; + let mut channel = create_test_live_channel("http://provider-1.example/live/55221.m3u8"); + channel.item_type = PlaylistItemType::LiveHls; + channel.virtual_id = 55221; + + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "tok-precomputed-follow-up", + virtual_id: channel.virtual_id, + provider: input.name.as_ref(), + stream_url: channel.url.as_ref(), + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: true, + }) + .await; + + let activation = activate_session_before_stream_open( + &app_state, + SessionActivationRequest { + fingerprint: &fingerprint, + input: input.as_ref(), + user: &user, + session_token: "tok-precomputed-follow-up", + request_class: Some(PlaybackRequestClass::FollowUp), + virtual_id: VirtualId::new(channel.virtual_id), + item_type: PlaylistItemType::LiveHls, + stream_url: channel.url.as_ref(), + connection_permission: UserConnectionPermission::Allowed, + connection_kind: crate::api::model::ConnectionKind::Normal, + granted_grace_mode: None, + socket_bound: true, + }, + ) + .await; + + assert_eq!(activation.admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(activation.admission.kind(), Some(crate::api::model::ConnectionKind::Normal)); + assert_eq!(activation.grace_mode, None); + assert!( + activation.placeholder_transition_version.is_some(), + "precomputed FollowUp must be revalidated against the current uncounted lifecycle" + ); +} + +// pre-resolved Grace materialization +#[tokio::test] +async fn activate_session_before_stream_open_pre_resolved_grace_period_materializes_pending_provider() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), + }); + let addr: SocketAddr = "127.0.0.1:55231".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + let input = app_state.app_config.sources.load().inputs[0].clone(); + let mut user = ProxyUserCredentials::default(); + user.username = "pre-resolved-grace-user".to_string(); + user.max_connections = 1; + let mut channel = create_test_live_channel("http://provider-1.example/live/55231.m3u8"); + channel.item_type = PlaylistItemType::LiveHls; + channel.virtual_id = 55231; + + // A pre-resolved grace grant reaches activation before a session exists. + let activation = activate_session_before_stream_open( + &app_state, + SessionActivationRequest { + fingerprint: &fingerprint, + input: input.as_ref(), + user: &user, + session_token: "tok-pre-resolved-grace", + request_class: None, + virtual_id: VirtualId::new(channel.virtual_id), + item_type: PlaylistItemType::LiveHls, + stream_url: channel.url.as_ref(), + connection_permission: UserConnectionPermission::GracePeriod, + connection_kind: crate::api::model::ConnectionKind::Normal, + granted_grace_mode: Some(crate::api::model::GraceMode::Hold), + socket_bound: true, + }, + ) + .await; + + assert_eq!(activation.admission.permission(), UserConnectionPermission::GracePeriod); + assert_eq!(activation.grace_mode, Some(crate::api::model::GraceMode::Hold)); + + let session = app_state.active_users.get_and_update_user_session(&user.username, "tok-pre-resolved-grace").await; + assert!( + session.is_some_and(|s| matches!(s.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. })), + "pre-resolved GracePeriod must materialize as PendingProvider lifecycle" + ); +} + +#[tokio::test] +async fn pre_resolved_instant_grace_creates_counted_live_session() { + let app_state = create_test_app_state(); + let addr: SocketAddr = "127.0.0.1:55233".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + let input = app_state.app_config.sources.load().inputs[0].clone(); + let mut user = ProxyUserCredentials::default(); + user.username = "instant-grace-live-user".to_string(); + user.max_connections = 1; + let channel = create_test_live_channel("http://provider-1.example/live/55233.ts"); + + let activation = activate_session_before_stream_open( + &app_state, + SessionActivationRequest { + fingerprint: &fingerprint, + input: input.as_ref(), + user: &user, + session_token: "tok-instant-grace", + request_class: Some(PlaybackRequestClass::Activate), + virtual_id: VirtualId::new(channel.virtual_id), + item_type: PlaylistItemType::Live, + stream_url: channel.url.as_ref(), + connection_permission: UserConnectionPermission::GracePeriod, + connection_kind: crate::api::model::ConnectionKind::Normal, + granted_grace_mode: Some(crate::api::model::GraceMode::Instant), + socket_bound: true, + }, + ) + .await; + + assert_eq!(activation.grace_mode, Some(crate::api::model::GraceMode::Instant)); + assert_eq!(app_state.active_users.user_connections(&user.username).await, 1); + let session = app_state.active_users.get_and_update_user_session(&user.username, "tok-instant-grace").await; + assert!(session.is_some_and(|session| session.lifecycle == crate::api::model::PlaybackLifecycle::GraceActive)); +} + +/// `activate_session_before_stream_open` skips placeholder for Prepare class. +#[tokio::test] +async fn activate_session_before_stream_open_skips_placeholder_for_prepare() { + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::EvictUserSameIpOldest]), + }); + let addr: SocketAddr = "127.0.0.1:55222".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + let input = app_state.app_config.sources.load().inputs[0].clone(); + let mut user = ProxyUserCredentials::default(); + user.username = "prepare-user".to_string(); + user.max_connections = 1; + + let activation = activate_session_before_stream_open( + &app_state, + SessionActivationRequest { + fingerprint: &fingerprint, + input: input.as_ref(), + user: &user, + session_token: "tok-prepare", + // Explicitly pass Prepare class — placeholder and admission should be skipped. + request_class: Some(PlaybackRequestClass::Prepare), + virtual_id: VirtualId::new(55222), + item_type: PlaylistItemType::LiveHls, + stream_url: "http://provider.example/live/test.ts", + connection_permission: UserConnectionPermission::Allowed, + connection_kind: crate::api::model::ConnectionKind::Normal, + granted_grace_mode: None, + socket_bound: true, + }, + ) + .await; + + // Prepare returns Allowed without running admission strategies. + assert_eq!(activation.admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(activation.grace_mode, None); + assert!( + activation.placeholder_transition_version.is_none(), + "Prepare activation must not create a placeholder session" + ); +} + +#[tokio::test] +async fn activate_session_before_stream_open_does_not_commit_user_lease_before_provider_success() { + let stream_cfg = crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 0, + grace_period_timeout_secs: 8, + grace_period_hold_stream: false, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: None, + }; + let mut app_cfg = create_test_app_config(); + app_cfg.config = Arc::new(ArcSwap::from_pointee(Config { + user_access_control: true, + reverse_proxy: Some(crate::model::ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: crate::model::ResourceRetryConfig::default(), + disabled_header: None, + stream: Some(stream_cfg), + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }), + ..Config::default() + })); + let app_state = create_test_app_state_for_config(Arc::new(app_cfg)); + let addr: SocketAddr = "127.0.0.1:55232".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + let input = app_state.app_config.sources.load().inputs[0].clone(); + let mut user = ProxyUserCredentials::default(); + user.username = "atomic-commit-user".to_string(); + user.max_connections = 1; + let channel = create_test_live_channel("http://provider-1.example/live/3.ts"); + + let activation = activate_session_before_stream_open( + &app_state, + SessionActivationRequest { + fingerprint: &fingerprint, + input: input.as_ref(), + user: &user, + session_token: "tok-atomic-commit", + request_class: None, + virtual_id: VirtualId::new(channel.virtual_id), + item_type: channel.item_type, + stream_url: channel.url.as_ref(), + connection_permission: UserConnectionPermission::Allowed, + connection_kind: crate::api::model::ConnectionKind::Normal, + granted_grace_mode: None, + socket_bound: false, + }, + ) + .await; + + assert_eq!(activation.admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(activation.grace_mode, None); + + let session = app_state + .active_users + .get_and_update_user_session(&user.username, "tok-atomic-commit") + .await + .expect("placeholder session should exist"); + assert_eq!( + app_state.active_users.user_connections(&user.username).await, + 0, + "allowed activation should stay provisional until provider acquisition and stream commit succeed" + ); + assert!( + !session.lifecycle.is_counted(), + "placeholder session must stay uncounted until the provider side has been committed" + ); + assert!(!matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. })); +} + +#[tokio::test] +async fn grace_context_is_populated_when_grace_strategy_is_actually_granted() { + // Use a DIFFERENT session token than the pre-existing counted session. + // Otherwise session-admission may treat it as a valid reopen and skip the exhausted path. + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 2_000, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![ + AdmissionStrategy::EvictUserSameIpOldest, + AdmissionStrategy::GraceHoldStream, + AdmissionStrategy::EvictUserOldest, + ]), + }); + + let addr1: SocketAddr = "127.0.0.1:55401".parse().unwrap_or_else(|_| unreachable!()); + let addr2: SocketAddr = "10.0.0.5:55402".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint1 = create_test_fingerprint(addr1); + let fingerprint2 = create_test_fingerprint(addr2); + // addr1 and addr2 have DIFFERENT IPs. + // EvictUserSameIpOldest will NOT match (different IP), so GraceHoldStream is evaluated. + let mut user = ProxyUserCredentials::default(); + user.username = "user-grace-ctx".to_string(); + user.max_connections = 1; + + // Register the connection first so update_connection succeeds + app_state.connection_manager.add_connection(&addr1).await; + + // Create the session — lifecycle starts as Prepared (uncounted) + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user: &user, + session_token: "tok-existing-counted", + virtual_id: 55401, + provider: "provider_1", + stream_url: "http://provider-1.example/live/55401.m3u8", + addr: &addr1, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: true, + }) + .await; + + // update_connection promotes the session to Active (counted) and creates a stream. + // This exhausts the user's single slot (max_connections = 1). + app_state + .active_users + .update_connection(crate::api::model::ActiveUserConnectionParams { + uid: 55401, + meter_uid: 55401, + username: "user-grace-ctx", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint1, + provider: "provider_1".intern(), + stream_channel: &create_test_live_channel("http://provider-1.example/live/55401.m3u8"), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-existing-counted"), + }) + .await + .expect("stream should be created"); + + // Now the new request finds the slot exhausted and the grace strategy kicks in. + let result = resolve_admission_with_strategies( + &app_state.admission_ctx(), + AdmissionRequest { + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + client_ip: &fingerprint2.client_ip, + request_addr: &fingerprint2.addr, + use_session_admission: true, + session_token: Some("tok-new-request"), + activate_unbound_session: true, + eviction_reentry_guard: EvictionReentryGuard::Session("tok-new-request"), + }, + ) + .await; + + assert_eq!(result.admission.permission(), UserConnectionPermission::GracePeriod, "grace should be granted"); + assert!(matches!(result.grace_mode, Some(crate::api::model::GraceMode::Hold))); + let ctx = result.grace_context.expect("grace_context must be present when grace is granted"); + assert_eq!(ctx.strategy_index, 1, "GraceHoldStream is at index 1"); + assert_eq!(ctx.strategies.len(), 3); + assert!(matches!(ctx.strategies[ctx.strategy_index], AdmissionStrategy::GraceHoldStream)); +} + +#[tokio::test] +async fn stream_response_rolls_back_provisional_user_activation_when_provider_open_fails() { + let mut app_cfg = create_test_provider_app_config(); + app_cfg.config = Arc::new(ArcSwap::from_pointee(Config { + user_access_control: true, + custom_stream_response_enabled: true, + ..Config::default() + })); + let app_state = create_test_app_state_for_config(Arc::new(app_cfg)); + let addr = "127.0.0.1:55143".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = create_test_fingerprint(addr); + let input_name = "provider_1".intern(); + let input = app_state.app_config.get_input_by_name(&input_name).expect("provider input should exist"); + let target = Arc::new(ConfigTarget { + curation: None, + id: 1, + enabled: true, + name: "test".to_string(), + options: None, + sort: None, + filter: Filter::default().into(), + output: Vec::new(), + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::default()), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }); + let mut user = ProxyUserCredentials::default(); + user.username = "rollback-user".to_string(); + user.max_connections = 1; + let stream_url = "provider://bad-url"; + let channel = create_test_live_channel(stream_url); + + let response = stream_response( + &fingerprint, + &app_state, + "rollback-session", + None, + channel, + stream_url, + None, + &HeaderMap::default(), + &input, + &target, + &user, + UserConnectionPermission::Allowed, + crate::api::model::ConnectionKind::Normal, + false, + None, + ) + .await + .into_response(); + + // Custom-video stream is enabled (`custom_stream_response_enabled: true` + // in this fixture), so a missing resource must return 400 — the + // Nginx `proxy_intercept_errors on;` contract requires 4xx so the + // socket is severed instead of looping on a 200 OK fallback body. + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!( + app_state.active_users.user_connections(&user.username).await, + 0, + "failed provider open must rollback provisional user activation" + ); + assert!( + app_state.active_users.get_and_update_user_session(&user.username, "rollback-session").await.is_none(), + "failed provider open must remove the provisional placeholder session" + ); +} diff --git a/backend/app/src/api/api_utils/tests/support.rs b/backend/app/src/api/api_utils/tests/support.rs new file mode 100644 index 000000000..5a262921b --- /dev/null +++ b/backend/app/src/api/api_utils/tests/support.rs @@ -0,0 +1,610 @@ +use crate::{ + api::model::{ + ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, ConnectionManager, EventManager, + MetadataUpdateManager, PlaylistStorageState, SharedStreamManager, UserSession, + }, + auth::Fingerprint, + model::{ + AppConfig, Config, ConfigInput, ConfigInputAlias, ConfigTarget, MediaToolCapabilities, NetworkAccess, + ProxyUserCredentials, SourcesConfig, StreamHistoryConfig, + }, + repository::GeoIp, + utils::FileLockManager, +}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use axum::http::StatusCode; +use bytes::Bytes; +use shared::{ + foundation::Filter, + model::{ + ClusterFlags, ConfigPaths, ConfigTargetOptions, InputFetchMethod, InputType, PlaylistItemType, ProcessingOrder, + ProxyType, StreamChannel, UserConnectionPermission, XtreamCluster, + }, + utils::Internable, +}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, +}; + +pub(in crate::api::api_utils::tests) async fn spawn_legacy_hls_test_origin( + response_head: String, + response_body: Vec, +) -> (SocketAddr, tokio::task::JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let origin_addr = listener.local_addr().expect("test origin address"); + let origin_task = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("test origin accepts request"); + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let read = socket.read(&mut chunk).await.expect("test origin reads request"); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + socket.write_all(response_head.as_bytes()).await.expect("test origin writes response headers"); + socket.write_all(&response_body).await.expect("test origin writes response body"); + String::from_utf8_lossy(&request).into_owned() + }); + (origin_addr, origin_task) +} + +pub(in crate::api::api_utils::tests) async fn spawn_range_aware_test_origin( + body: &'static [u8], + connections: usize, +) -> (SocketAddr, tokio::task::JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let origin_addr = listener.local_addr().expect("test origin address"); + let origin_task = tokio::spawn(async move { + let mut requests = Vec::new(); + for _ in 0..connections { + let Ok((mut socket, _)) = listener.accept().await else { break }; + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let read = socket.read(&mut chunk).await.expect("test origin reads request"); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + let request_lower = String::from_utf8_lossy(&request).to_ascii_lowercase(); + requests.push(request_lower.clone()); + + let range = request_lower + .lines() + .find_map(|line| line.strip_prefix("range: bytes=")) + .and_then(|spec| spec.split_once('-')) + .and_then(|(start, end)| { + let start: usize = start.trim().parse().ok()?; + let end: usize = + if end.trim().is_empty() { body.len().saturating_sub(1) } else { end.trim().parse().ok()? }; + Some((start, end.min(body.len().saturating_sub(1)))) + }) + .unwrap_or((0, body.len().saturating_sub(1))); + + let slice = &body[range.0..=range.1]; + let head = format!( + "HTTP/1.1 206 Partial Content\r\nContent-Type: video/mp2t\r\nContent-Range: bytes {}-{}/{}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + range.0, + range.1, + body.len(), + slice.len() + ); + socket.write_all(head.as_bytes()).await.expect("test origin writes response headers"); + socket.write_all(slice).await.expect("test origin writes response body"); + } + requests + }); + (origin_addr, origin_task) +} + +#[allow(dead_code)] +#[derive(Clone)] +pub(in crate::api::api_utils::tests) enum FakeOriginMode { + EmptyBody, + BlockFirstChunk(Arc), + ErrorBeforeFirstChunk, + ErrorAfterFirstChunk(Vec), + FixedBytes(Vec), +} + +pub(in crate::api::api_utils::tests) async fn spawn_controlled_fake_origin( + mode: FakeOriginMode, + max_requests: usize, +) -> (SocketAddr, tokio::task::JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let origin_addr = listener.local_addr().expect("test origin address"); + let origin_task = tokio::spawn(async move { + let mut requests = Vec::new(); + for _ in 0..max_requests { + let Ok((mut socket, _)) = listener.accept().await else { break }; + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let Ok(read) = socket.read(&mut chunk).await else { break }; + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + if request.is_empty() { + continue; + } + let request_lower = String::from_utf8_lossy(&request).to_ascii_lowercase(); + requests.push(request_lower); + + match &mode { + FakeOriginMode::EmptyBody => { + let head = + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.shutdown().await; + } + FakeOriginMode::BlockFirstChunk(notify) => { + let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let mut probe = [0_u8; 1]; + tokio::select! { + () = notify.notified() => {}, + _ = socket.read(&mut probe) => {}, + } + let _ = socket.shutdown().await; + } + FakeOriginMode::ErrorBeforeFirstChunk => { + let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.shutdown().await; + } + FakeOriginMode::ErrorAfterFirstChunk(chunk) => { + let head = "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(head.as_bytes()).await; + let chunk_hdr = format!("{:x}\r\n", chunk.len()); + let _ = socket.write_all(chunk_hdr.as_bytes()).await; + let _ = socket.write_all(chunk).await; + let _ = socket.write_all(b"\r\n").await; + let _ = socket.flush().await; + let _ = socket.shutdown().await; + } + FakeOriginMode::FixedBytes(bytes) => { + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + bytes.len() + ); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(bytes).await; + let _ = socket.shutdown().await; + } + } + } + requests + }); + (origin_addr, origin_task) +} + +pub(in crate::api::api_utils::tests) async fn spawn_load_test_origin(body: &'static [u8]) -> SocketAddr { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("load origin binds"); + let addr = listener.local_addr().expect("load origin address"); + let app = axum::Router::new().fallback(move || async move { + ([(axum::http::header::CONTENT_TYPE, "video/mp2t")], Bytes::from_static(body)) + }); + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + addr +} + +pub(in crate::api::api_utils::tests) fn validate_load_response( + status: StatusCode, + payload: &[u8], + expected: &[u8], +) -> Result<(), String> { + if status != StatusCode::OK { + return Err(format!("unexpected HTTP status {status}")); + } + if payload != expected { + return Err(format!("payload mismatch: expected {} bytes, received {}", expected.len(), payload.len())); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +pub(in crate::api::api_utils::tests) fn load_test_rss_kib() -> Option { + let Ok(content) = std::fs::read_to_string("/proc/self/status") else { + return None; + }; + for line in content.lines() { + if let Some(rest) = line.strip_prefix("VmRSS:") { + return rest.split_whitespace().next().and_then(|v| v.parse::().ok()); + } + } + None +} + +#[cfg(not(target_os = "linux"))] +pub(in crate::api::api_utils::tests) fn load_test_rss_kib() -> Option { None } + +pub(in crate::api::api_utils::tests) fn load_test_user(username: &str) -> ProxyUserCredentials { + let mut user = ProxyUserCredentials::default(); + user.username = username.to_string(); + user +} + +pub(in crate::api::api_utils::tests) fn load_test_session( + input: &ConfigInput, + origin_addr: SocketAddr, + token: &str, + addr: SocketAddr, +) -> UserSession { + UserSession { + token: token.to_string(), + transition_version: 1, + virtual_id: 42, + provider: Arc::clone(&input.name), + stream_url: format!("http://{origin_addr}/live/42.ts").intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), + user_agent_stream_index: None, + addr, + socket_bound: false, + active_addrs: vec![addr], + ts: 1, + started_at: 1, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + lifecycle: crate::api::model::PlaybackLifecycle::Active, + ..Default::default() + } +} + +pub(in crate::api::api_utils::tests) fn load_test_channel( + input: &ConfigInput, + origin_addr: SocketAddr, +) -> StreamChannel { + let mut channel = create_test_live_channel(&format!("http://{origin_addr}/live/42.ts")); + channel.provider_id = u32::from(input.id); + channel.input_name = Arc::clone(&input.name); + channel.url = format!("http://{origin_addr}/live/42.ts").intern(); + channel +} + +pub(in crate::api::api_utils::tests) fn create_test_app_config() -> AppConfig { + let input = Arc::new(ConfigInput { + id: 1, + name: "local_media".intern(), + input_type: InputType::Library, + headers: HashMap::default(), + url: "file:///tmp".to_string(), + enabled: true, + priority: 0, + max_connections: 1, + method: InputFetchMethod::default(), + aliases: None, + ..ConfigInput::default() + }); + let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; + + AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(sources)), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + } +} + +pub(in crate::api::api_utils::tests) fn create_test_provider_app_config() -> AppConfig { + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: "http://provider-1.example".to_string(), + username: Some("user1".to_string()), + password: Some("pass1".to_string()), + enabled: true, + priority: 0, + max_connections: 1, + method: InputFetchMethod::default(), + aliases: None, + ..ConfigInput::default() + }); + let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; + + AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(sources)), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + } +} + +pub(in crate::api::api_utils::tests) fn create_test_dual_provider_app_config() -> AppConfig { + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: "http://provider-1.example".to_string(), + username: Some("user1".to_string()), + password: Some("pass1".to_string()), + enabled: true, + priority: 0, + max_connections: 1, + method: InputFetchMethod::default(), + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "provider_2".intern(), + url: "http://provider-2.example".to_string(), + username: Some("user2".to_string()), + password: Some("pass2".to_string()), + priority: 1, + max_connections: 1, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }); + let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; + + AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(sources)), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + } +} + +pub(in crate::api::api_utils::tests) fn create_test_app_state() -> Arc { + create_test_app_state_for_config(Arc::new(create_test_app_config())) +} + +pub(in crate::api::api_utils::tests) fn create_test_provider_app_state() -> Arc { + create_test_app_state_for_config(Arc::new(create_test_provider_app_config())) +} + +pub(in crate::api::api_utils::tests) fn create_test_dual_provider_app_state() -> Arc { + create_test_app_state_for_config(Arc::new(create_test_dual_provider_app_config())) +} + +pub(in crate::api::api_utils::tests) fn create_test_app_state_for_config(app_cfg: Arc) -> Arc { + let event_manager = Arc::new(EventManager::new()); + let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); + let history_config = Some(StreamHistoryConfig::default()); + active_provider.set_shared_stream_manager(&shared_stream_manager); + + let geoip = Arc::new(ArcSwapOption::::default()); + let config = app_cfg.config.load(); + let active_users = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); + let connection_manager = Arc::new(ConnectionManager::new( + &active_users, + &active_provider, + &shared_stream_manager, + &event_manager, + history_config.as_ref(), + )); + + let tokens = CancelTokens::default(); + let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); + + Arc::new(AppState { + recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( + Arc::clone(&active_provider), + Arc::clone(&connection_manager), + ), + app_config: app_cfg, + http_clients: Arc::default(), + recordings: Arc::new(crate::api::model::RecordingQueue::new()), + cache: Arc::new(ArcSwapOption::default()), + shared_stream_manager, + hls: crate::api::model::HlsState::new(Arc::new(crate::api::model::HlsProxyManager::new())), + stalker_resolve_coordinator: crate::api::model::StalkerResolveCoordinator::default(), + active_users, + active_provider, + connection_manager, + event_manager, + cancel_tokens: ArcSwap::from_pointee(tokens), + playlists: Arc::new(PlaylistStorageState::new()), + geoip, + metadata_manager, + auth: crate::api::model::AuthState::for_tests(), + playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), + }) +} + +pub(in crate::api::api_utils::tests) fn create_test_fingerprint(addr: std::net::SocketAddr) -> Fingerprint { + Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr) +} + +pub(in crate::api::api_utils::tests) fn create_test_app_state_with_stream_config( + stream: crate::model::StreamConfig, +) -> Arc { + let config = Config { + reverse_proxy: Some(crate::model::ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: crate::model::ResourceRetryConfig::default(), + disabled_header: None, + stream: Some(stream), + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }), + user_access_control: true, + ..Config::default() + }; + + let mut app_cfg = create_test_app_config(); + app_cfg.config = Arc::new(ArcSwap::from_pointee(config)); + create_test_app_state_for_config(Arc::new(app_cfg)) +} + +pub(in crate::api::api_utils::tests) fn create_test_local_channel(url: &str) -> StreamChannel { + StreamChannel { + target_id: 1, + virtual_id: 41, + provider_id: 0, + input_name: "library".intern(), + item_type: PlaylistItemType::LocalVideo, + cluster: XtreamCluster::Video, + group: "Local Movies".intern(), + title: "Local Test".intern(), + url: url.into(), + shared: false, + shared_joined_existing: None, + shared_stream_id: None, + technical: None, + epg_channel_id: None, + epg_reference_ts: None, + upstream_user_agent: None, + } +} + +pub(in crate::api::api_utils::tests) fn create_test_live_channel(url: &str) -> StreamChannel { + StreamChannel { + target_id: 1, + virtual_id: 42, + provider_id: 1, + input_name: "provider_1".intern(), + item_type: PlaylistItemType::Live, + cluster: XtreamCluster::Live, + group: "Live".intern(), + title: "Shared Live".intern(), + url: url.into(), + shared: false, + shared_joined_existing: None, + shared_stream_id: None, + technical: None, + epg_channel_id: None, + epg_reference_ts: None, + upstream_user_agent: None, + } +} + +pub(in crate::api::api_utils::tests) fn create_test_shared_target() -> ConfigTarget { + ConfigTarget { + curation: None, + id: 1, + enabled: true, + name: "shared".to_string(), + options: Some(ConfigTargetOptions { + share_live_streams: shared::model::ConfigTargetShareLiveStreams { mpeg_ts: true, ..Default::default() }, + ..ConfigTargetOptions::default() + }), + sort: None, + filter: Filter::default().into(), + output: Vec::new(), + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::default()), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + } +} + +/// `Arc>` with a mock `GeoIP` that always reports the +/// given country for any lookup. +pub(in crate::api::api_utils::tests) fn mock_geoip(country: &str) -> Arc> { + Arc::new(ArcSwapOption::from(Some(Arc::new(GeoIp::test_new(country))))) +} + +/// Helper to create a test user with specific network access +pub(in crate::api::api_utils::tests) fn user_with_network_access( + network_access: Option, +) -> ProxyUserCredentials { + ProxyUserCredentials { + username: "test".to_string(), + password: "test".to_string(), + token: None, + proxy: ProxyType::default(), + server: None, + epg_timeshift: None, + epg_request_timeshift: None, + created_at: None, + exp_date: None, + max_connections: 0, + status: None, + output_clusters: ClusterFlags::all(), + ui_enabled: true, + comment: None, + priority: 0, + soft_connections: 0, + soft_priority: 0, + t_is_api_user: false, + network_access, + plan: None, + filter: None, + raw_output_clusters: None, + raw_max_connections: 0, + raw_soft_connections: 0, + raw_proxy: Some(ProxyType::default()), + t_filter: None, + t_has_unresolved_plan: false, + t_has_invalid_filter: false, + } +} diff --git a/backend/app/src/api/endpoints/hls_api/manifest.rs b/backend/app/src/api/endpoints/hls_api/manifest.rs index 837c716d2..602d58731 100644 --- a/backend/app/src/api/endpoints/hls_api/manifest.rs +++ b/backend/app/src/api/endpoints/hls_api/manifest.rs @@ -1,1993 +1,7 @@ #![allow(clippy::wildcard_imports)] + use super::*; -#[derive(Debug, Deserialize)] -pub(super) struct HlsProxyTerminalSegmentPathParams { - pub(super) proxy_session_id: String, - pub(super) hls_access_lease_id: String, - pub(super) generation: String, - pub(super) terminal_file: String, -} - -pub(super) fn hls_custom_video_type_for_failure_reason(reason: ConnectFailureReason) -> CustomVideoStreamType { - match reason { - ConnectFailureReason::UserAccountExpired => CustomVideoStreamType::UserAccountExpired, - ConnectFailureReason::UserConnectionsExhausted => CustomVideoStreamType::UserConnectionsExhausted, - ConnectFailureReason::ProviderConnectionsExhausted => CustomVideoStreamType::ProviderConnectionsExhausted, - ConnectFailureReason::Preempted => CustomVideoStreamType::LowPriorityPreempted, - ConnectFailureReason::Provisioning => CustomVideoStreamType::Provisioning, - ConnectFailureReason::SessionExpired => CustomVideoStreamType::HlsSessionOrLeaseExpired, - ConnectFailureReason::ProviderError - | ConnectFailureReason::ProviderClosed - | ConnectFailureReason::ChannelUnavailable => CustomVideoStreamType::ChannelUnavailable, - } -} - -pub(crate) async fn hls_custom_video_manifest_response( - app_state: &Arc, - user: &ProxyUserCredentials, - video_type: CustomVideoStreamType, - fallback_status: StatusCode, -) -> axum::response::Response { - hls_custom_video_manifest_response_with_virtual_id(app_state, user, video_type, fallback_status, None).await -} - -pub(crate) async fn hls_admission_failure_manifest_response( - app_state: &Arc, - fingerprint: &Fingerprint, - user: &ProxyUserCredentials, - stream_channel: StreamChannel, - provider_name: Arc, - req_headers: &HeaderMap, - reason: ConnectFailureReason, -) -> axum::response::Response { - record_connect_failed_attempt(ConnectFailedAttempt { - app_state, - fingerprint, - user, - stream_channel, - provider_name, - req_headers, - reason, - failure_stage: FailureStage::Admission, - }); - hls_custom_video_manifest_response( - app_state, - user, - hls_custom_video_type_for_failure_reason(reason), - StatusCode::FORBIDDEN, - ) - .await -} - -pub(super) fn apply_hls_proxy_public_path_prefix(hls_content: String, server_path: Option<&str>) -> String { - let Some(path_prefix) = normalize_hls_proxy_public_path_prefix(server_path) else { - return hls_content; - }; - - let uri_attr_prefix = format!("URI=\"{path_prefix}/hls/shared/live/"); - let hls_content = hls_content.replace("URI=\"/hls/shared/live/", &uri_attr_prefix); - if hls_content.is_empty() { - return hls_content; - } - let mut prefixed = String::with_capacity(hls_content.len().saturating_add(path_prefix.len().saturating_mul(4))); - - for part in hls_content.split_inclusive('\n') { - let (line, line_ending) = split_hls_line_ending(part); - if line.starts_with("/hls/shared/live/") { - prefixed.push_str(&path_prefix); - } - prefixed.push_str(line); - prefixed.push_str(line_ending); - } - - prefixed -} - -pub(super) fn normalize_hls_proxy_public_path_prefix(server_path: Option<&str>) -> Option { - let path = server_path?.trim().trim_matches('/'); - if path.is_empty() { - return None; - } - Some(format!("/{path}")) -} - -pub(super) fn split_hls_line_ending(part: &str) -> (&str, &str) { - if let Some(line) = part.strip_suffix("\r\n") { - (line, "\r\n") - } else if let Some(line) = part.strip_suffix('\n') { - (line, "\n") - } else { - (part, "") - } -} - -pub(super) fn materialize_hls_access_manifest( - hls_content: &str, - lease_id: &HlsAccessLeaseId, - server_path: Option<&str>, -) -> String { - let hls_content = hls_content.replace(HLS_ACCESS_LEASE_ID_PLACEHOLDER, &lease_id.0); - apply_hls_proxy_public_path_prefix(hls_content, server_path) -} - -pub(super) fn hls_access_manifest_uses_startup_view(lease_state: HlsAccessLeaseState) -> bool { - matches!(lease_state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Idle) -} - -pub(super) fn materialize_shared_hls_access_manifest( - hls_content: &str, - lease_id: &HlsAccessLeaseId, - lease_state: HlsAccessLeaseState, - strip: &crate::model::StripConfig, - window_policy: HlsManifestWindowPolicy, - mode: &'static str, - server_path: Option<&str>, -) -> HlsMaterializedSharedManifest { - let (response_body, initial_strip_outcome) = if hls_access_manifest_uses_startup_view(lease_state) { - let view = materialize_initial_hls_strip_view(hls_content, strip, window_policy); - (view.body, Some(view.outcome)) - } else { - (Cow::Borrowed(hls_content), None) - }; - HlsMaterializedSharedManifest { - body: materialize_hls_access_manifest(&response_body, lease_id, server_path), - mode, - initial_strip_outcome, - } -} - -pub(super) struct HlsMaterializedSharedManifest { - pub(super) body: String, - pub(super) mode: &'static str, - pub(super) initial_strip_outcome: Option, -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub(super) enum HlsInitialStripLeaseSkipReason { - LeaseActivated, - LeaseNotStartupView, -} - -impl HlsInitialStripLeaseSkipReason { - pub(super) const fn as_log_reason(self) -> &'static str { - match self { - Self::LeaseActivated => "lease-activated", - Self::LeaseNotStartupView => "lease-not-startup-view", - } - } -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub(super) enum HlsInitialStripPublicationDiagnostic { - Applied { mode: &'static str, strip_mode: &'static str, configured: u64, effective: usize, visible_segments: usize }, - Skipped { mode: &'static str, reason: HlsInitialStripSkipReason, visible_segments: usize }, - SkippedForLeaseState { mode: &'static str, reason: HlsInitialStripLeaseSkipReason }, -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub(super) enum HlsInitialStripPublicationStatus { - NotCommitted, - Committed, -} - -pub(super) fn hls_initial_strip_publication_diagnostic( - publication_status: HlsInitialStripPublicationStatus, - lease_state: HlsAccessLeaseState, - materialized: &HlsMaterializedSharedManifest, -) -> Option { - match publication_status { - HlsInitialStripPublicationStatus::NotCommitted => return None, - HlsInitialStripPublicationStatus::Committed => {} - } - Some(match &materialized.initial_strip_outcome { - Some(HlsInitialStripOutcome::Applied { mode: strip_mode, configured, effective, visible_segments }) => { - HlsInitialStripPublicationDiagnostic::Applied { - mode: materialized.mode, - strip_mode, - configured: *configured, - effective: *effective, - visible_segments: *visible_segments, - } - } - Some(HlsInitialStripOutcome::Skipped { reason, visible_segments }) => { - HlsInitialStripPublicationDiagnostic::Skipped { - mode: materialized.mode, - reason: *reason, - visible_segments: *visible_segments, - } - } - None => HlsInitialStripPublicationDiagnostic::SkippedForLeaseState { - mode: materialized.mode, - reason: if lease_state == HlsAccessLeaseState::Activated { - HlsInitialStripLeaseSkipReason::LeaseActivated - } else { - HlsInitialStripLeaseSkipReason::LeaseNotStartupView - }, - }, - }) -} - -pub(super) fn hls_entry_master_playlist_response( - proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, - bandwidth: HlsMasterBandwidth, - server_path: Option<&str>, -) -> HlsEntryMasterPlaylistResponse { - let path_prefix = normalize_hls_proxy_public_path_prefix(server_path).unwrap_or_default(); - let media_playlist_uri = format!("{path_prefix}{}", hls_canonical_manifest_path(proxy_session_id, access_lease_id)); - let body = HlsSingleVariantMasterPlaylist::new(bandwidth, media_playlist_uri).render().into_bytes(); - let content_length = body.len(); - let response = try_unwrap_body!(axum::response::Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, "application/vnd.apple.mpegurl") - .header(header::CACHE_CONTROL, "private, no-store, no-cache, must-revalidate") - .header(header::CONTENT_LENGTH, content_length) - .body(Body::from(body))); - HlsEntryMasterPlaylistResponse { response, content_length } -} - -pub(super) fn hls_canonical_manifest_path( - proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, -) -> String { - format!("/hls/shared/live/{}/{}/manifest.m3u8", proxy_session_id.0, access_lease_id.0) -} - -pub(super) fn hls_canonical_retry_after_response() -> axum::response::Response { - try_unwrap_body!(axum::response::Response::builder() - .status(axum::http::StatusCode::SERVICE_UNAVAILABLE) - .header(axum::http::header::RETRY_AFTER, cold_start_retry_after_seconds().to_string()) - .body(axum::body::Body::empty())) -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum HlsCanonicalOwnerRegistration { - Join(HlsCanonicalOwnerRegistrationKind), - FailClosed(HlsCanonicalOwnerRegistrationFailure), -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum HlsCanonicalOwnerRegistrationKind { - Scheduled, - AlreadyOwned, -} - -impl HlsCanonicalOwnerRegistrationKind { - pub(super) const fn as_label(self) -> &'static str { - match self { - Self::Scheduled => "scheduled", - Self::AlreadyOwned => "already_owned", - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum HlsCanonicalOwnerRegistrationFailure { - CapacityExceeded, - RuntimeUnavailable, -} - -impl HlsCanonicalOwnerRegistrationFailure { - pub(super) const fn as_label(self) -> &'static str { - match self { - Self::CapacityExceeded => "capacity_exceeded", - Self::RuntimeUnavailable => "runtime_unavailable", - } - } -} - -pub(super) const fn hls_canonical_owner_registration( - registration: HlsAvailabilityReevaluationRegistration, -) -> HlsCanonicalOwnerRegistration { - match registration { - HlsAvailabilityReevaluationRegistration::Scheduled => { - HlsCanonicalOwnerRegistration::Join(HlsCanonicalOwnerRegistrationKind::Scheduled) - } - HlsAvailabilityReevaluationRegistration::AlreadyOwned | HlsAvailabilityReevaluationRegistration::Superseded => { - HlsCanonicalOwnerRegistration::Join(HlsCanonicalOwnerRegistrationKind::AlreadyOwned) - } - HlsAvailabilityReevaluationRegistration::CapacityExceeded => { - HlsCanonicalOwnerRegistration::FailClosed(HlsCanonicalOwnerRegistrationFailure::CapacityExceeded) - } - HlsAvailabilityReevaluationRegistration::RuntimeUnavailable => { - HlsCanonicalOwnerRegistration::FailClosed(HlsCanonicalOwnerRegistrationFailure::RuntimeUnavailable) - } - } -} - -pub(super) fn hls_availability_reevaluation_registration_failure_response( - failure: HlsCanonicalOwnerRegistrationFailure, -) -> axum::response::Response { - warn!("HLS availability reevaluation not registered: reason={}", failure.as_label()); - hls_canonical_retry_after_response() -} - -pub(super) enum HlsCanonicalOwnerResolution { - Live(axum::response::Response), - Terminal(axum::response::Response), - Standalone(axum::response::Response), - FailedClosed { reason: HlsCanonicalOwnerFailureReason, response: axum::response::Response }, -} - -impl HlsCanonicalOwnerResolution { - pub(super) const fn outcome_label(&self) -> &'static str { - match self { - Self::Live(_) => "live", - Self::Terminal(_) => "terminal", - Self::Standalone(_) => "standalone", - Self::FailedClosed { reason, .. } => reason.as_label(), - } - } - - pub(super) fn into_response(self) -> axum::response::Response { - match self { - Self::Live(response) - | Self::Terminal(response) - | Self::Standalone(response) - | Self::FailedClosed { response, .. } => response, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum HlsCanonicalOwnerFailureReason { - Superseded, - DeadlineElapsed, - LeaseUnavailable, -} - -impl HlsCanonicalOwnerFailureReason { - pub(super) const fn as_label(self) -> &'static str { - match self { - Self::Superseded => "superseded", - Self::DeadlineElapsed => "deadline_elapsed", - Self::LeaseUnavailable => "lease_unavailable", - } - } - - pub(super) fn response(self) -> axum::response::Response { - let reason = match self { - Self::DeadlineElapsed => HlsTerminalFailedClosedReason::SafeCommitDeadlineElapsed, - Self::Superseded | Self::LeaseUnavailable => HlsTerminalFailedClosedReason::LeaseStateUnavailable, - }; - hls_terminal_failed_closed_response(reason) - } -} - -pub(super) struct HlsCanonicalOwnerPending { - pub(super) deadline_ms: u64, - pub(super) current_session_available: bool, -} - -pub(super) enum HlsCanonicalOwnerEvaluation { - Resolved(HlsCanonicalOwnerResolution), - Pending(HlsCanonicalOwnerPending), -} - -pub(super) fn hls_canonical_owner_failed(reason: HlsCanonicalOwnerFailureReason) -> HlsCanonicalOwnerEvaluation { - HlsCanonicalOwnerEvaluation::Resolved(HlsCanonicalOwnerResolution::FailedClosed { - reason, - response: reason.response(), - }) -} - -pub(super) struct HlsCanonicalOwnerHandoffContext<'a> { - pub(super) app_state: &'a Arc, - pub(super) proxy_session_id: &'a ProxySessionId, - pub(super) access_lease_id: &'a HlsAccessLeaseId, - pub(super) expected_lease_issued_at_ms: Option, - pub(super) strip: &'a crate::model::StripConfig, - pub(super) server_path: Option<&'a str>, - pub(super) manifest_commit_requirement: HlsManifestCommitRequirement, - pub(super) manifest_boundary_rendered_at_ms: u64, - pub(super) bandwidth_learning: HlsRuntimeBandwidthLearningContext<'a>, - pub(super) request_deadline_ms: u64, - pub(super) safe_session: String, -} - -pub(super) fn hls_canonical_owner_lease_deadline_ms(lease: &HlsAccessLease) -> u64 { - if lease.state == HlsAccessLeaseState::Pending { - lease.pending_deadline_ms().unwrap_or(lease.valid_until_ms) - } else { - lease.valid_until_ms - } -} - -pub(super) fn hls_canonical_owner_request_deadline_ms( - lease: &HlsAccessLease, - wait_timeout: Duration, - now_ms: u64, -) -> u64 { - let lease_deadline_ms = hls_canonical_owner_lease_deadline_ms(lease); - if wait_timeout.is_zero() && lease.state == HlsAccessLeaseState::Pending { - lease_deadline_ms - } else { - lease_deadline_ms.min(now_ms.saturating_add(duration_to_millis_saturating(wait_timeout))) - } -} - -pub(super) async fn evaluate_hls_canonical_owner_handoff( - context: &HlsCanonicalOwnerHandoffContext<'_>, -) -> HlsCanonicalOwnerEvaluation { - let now_ms = current_time_millis(); - let Some(lease) = context - .app_state - .hls - .proxy - .access_lease_response_snapshot(context.access_lease_id, context.proxy_session_id, now_ms) - .await - else { - return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable); - }; - if context.expected_lease_issued_at_ms != Some(lease.issued_at_ms) { - return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable); - } - match &lease.playback_mode { - HlsLeasePlaybackMode::TerminalTail(_) => { - let Some(response) = - hls_terminal_playback_response(&lease, context.proxy_session_id, context.access_lease_id) - else { - return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable); - }; - return HlsCanonicalOwnerEvaluation::Resolved(HlsCanonicalOwnerResolution::Terminal(response)); - } - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => { - return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable); - } - HlsLeasePlaybackMode::Live => {} - } - if matches!( - lease.state, - HlsAccessLeaseState::PolicyRevoking | HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied - ) { - return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable); - } - - let current_session = - context.app_state.hls.proxy.sessions().get_by_proxy_session_id(context.proxy_session_id).await; - if let Some(current_session) = current_session.as_ref() { - let options = hls_cached_manifest_options_for_requirement( - Duration::ZERO, - context.manifest_commit_requirement, - context.manifest_boundary_rendered_at_ms, - ); - if let Some(response) = try_hls_cached_manifest_response( - context.app_state, - current_session, - context.access_lease_id, - lease.state, - context.strip, - context.server_path, - options, - context.bandwidth_learning, - ) - .await - .filter(|response| response.status() == StatusCode::OK) - { - return HlsCanonicalOwnerEvaluation::Resolved(HlsCanonicalOwnerResolution::Live(response)); - } - } - HlsCanonicalOwnerEvaluation::Pending(HlsCanonicalOwnerPending { - deadline_ms: hls_canonical_owner_lease_deadline_ms(&lease).min(context.request_deadline_ms), - current_session_available: current_session.is_some(), - }) -} - -pub(super) async fn finalize_hls_canonical_owner_handoff( - context: &HlsCanonicalOwnerHandoffContext<'_>, - pending: HlsCanonicalOwnerPending, - deadline_elapsed: bool, -) -> HlsCanonicalOwnerResolution { - match evaluate_hls_canonical_owner_handoff(context).await { - HlsCanonicalOwnerEvaluation::Resolved(resolution) => resolution, - HlsCanonicalOwnerEvaluation::Pending(current) => { - let response = hls_unpublished_lease_channel_unavailable_response( - context.app_state, - context.proxy_session_id, - context.access_lease_id, - ) - .await; - if response.status() == StatusCode::OK { - return HlsCanonicalOwnerResolution::Standalone(response); - } - let reason = if deadline_elapsed { - HlsCanonicalOwnerFailureReason::DeadlineElapsed - } else if !current.current_session_available && !pending.current_session_available { - HlsCanonicalOwnerFailureReason::Superseded - } else { - HlsCanonicalOwnerFailureReason::LeaseUnavailable - }; - HlsCanonicalOwnerResolution::FailedClosed { reason, response: reason.response() } - } - } -} - -pub(super) async fn join_hls_canonical_manifest_owner( - context: HlsCanonicalOwnerHandoffContext<'_>, - registration: HlsCanonicalOwnerRegistrationKind, -) -> axum::response::Response { - let started_at = tokio::time::Instant::now(); - let coordinator = context.app_state.hls.proxy.availability_reevaluations(); - let resolution = loop { - let mut observer = coordinator.observe_owner(context.proxy_session_id); - let pending = match evaluate_hls_canonical_owner_handoff(&context).await { - HlsCanonicalOwnerEvaluation::Resolved(resolution) => break resolution, - HlsCanonicalOwnerEvaluation::Pending(pending) => pending, - }; - let now_ms = current_time_millis(); - if now_ms >= pending.deadline_ms { - break finalize_hls_canonical_owner_handoff(&context, pending, true).await; - } - let Some(observer) = observer.as_mut() else { - break finalize_hls_canonical_owner_handoff(&context, pending, false).await; - }; - let remaining_ms = pending.deadline_ms.saturating_sub(now_ms); - match tokio::time::timeout(Duration::from_millis(remaining_ms), observer.changed()).await { - Ok( - HlsAvailabilityReevaluationObservation::EvidenceChanged - | HlsAvailabilityReevaluationObservation::OwnerFinished, - ) => {} - Err(_) => break finalize_hls_canonical_owner_handoff(&context, pending, true).await, - } - }; - debug!( - "HLS canonical manifest owner handoff completed: session={} proxy_session={} lease={} registration={} outcome={} wait_ms={}", - context.safe_session, - safe_proxy_session_id(context.proxy_session_id), - safe_hls_access_lease_id(context.access_lease_id), - registration.as_label(), - resolution.outcome_label(), - duration_to_millis_saturating(started_at.elapsed()) - ); - resolution.into_response() -} - -pub(super) async fn hls_direct_refresh_follow_up( - app_state: &Arc, - session: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - refresh_request: OriginRefreshRequest, - outcome: HlsOriginRefreshTriggerOutcome, -) -> Option { - match outcome { - HlsOriginRefreshTriggerOutcome::Started - | HlsOriginRefreshTriggerOutcome::SessionUnavailable - | HlsOriginRefreshTriggerOutcome::InFlight - | HlsOriginRefreshTriggerOutcome::DebouncedUntil { .. } => return None, - HlsOriginRefreshTriggerOutcome::RecoveryPressureSuperseded => { - warn!("HLS direct origin refresh evidence superseded; scheduling current availability reevaluation"); - } - HlsOriginRefreshTriggerOutcome::RecoveryPressureStateContention => { - warn!("HLS direct origin refresh state contended; scheduling current availability reevaluation"); - } - } - let Some(owner_key) = app_state.hls.proxy.availability_reevaluation_owner_key(session, proxy_session_id).await - else { - warn!("HLS direct origin refresh follow-up unavailable: reason=session_superseded"); - return Some(hls_canonical_retry_after_response()); - }; - match register_hls_availability_reevaluation(app_state.hls_ctx(), Arc::clone(session), owner_key, refresh_request) { - HlsAvailabilityReevaluationRegistration::Scheduled - | HlsAvailabilityReevaluationRegistration::AlreadyOwned - | HlsAvailabilityReevaluationRegistration::Superseded => None, - HlsAvailabilityReevaluationRegistration::CapacityExceeded => { - warn!("HLS direct origin refresh follow-up unavailable: reason=capacity_exceeded"); - Some(hls_canonical_retry_after_response()) - } - HlsAvailabilityReevaluationRegistration::RuntimeUnavailable => { - warn!("HLS direct origin refresh follow-up unavailable: reason=runtime_unavailable"); - Some(hls_canonical_retry_after_response()) - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum HlsManifestRefreshOrdering { - Background, - AwaitBeforeTerminalEvaluation, -} - -pub(super) async fn trigger_hls_canonical_manifest_refresh( - app_state: &Arc, - session: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, - refresh_request: OriginRefreshRequest, - ordering: HlsManifestRefreshOrdering, -) -> Option { - match ordering { - HlsManifestRefreshOrdering::Background => { - let outcome = maybe_trigger_origin_refresh_with_outcome(refresh_request.clone()).await; - hls_direct_refresh_follow_up(app_state, session, proxy_session_id, refresh_request, outcome).await - } - HlsManifestRefreshOrdering::AwaitBeforeTerminalEvaluation => { - // An already-owned refresh cannot be joined through this call, but - // it must not bypass the lease-specific terminal decision. The - // in-flight owner will still publish its eventual progress/failure. - let _refresh_started = trigger_origin_refresh_sync(refresh_request).await; - let now_ms = current_time_millis(); - let Some(lease) = - app_state.hls.proxy.access_lease_response_snapshot(access_lease_id, proxy_session_id, now_ms).await - else { - return Some(hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::LeaseStateUnavailable)); - }; - resolve_hls_terminal_manifest_state(app_state, session, proxy_session_id, access_lease_id, lease, now_ms) - .await - .err() - .map(|response| *response) - } - } -} - -pub(super) fn hls_canonical_status_response(status: StatusCode) -> axum::response::Response { - if status == StatusCode::SERVICE_UNAVAILABLE { - hls_canonical_retry_after_response() - } else { - status.into_response() - } -} - -pub(super) struct HlsEntryOriginAccountReservation { - pub(super) request_url: String, - pub(super) session_token: String, - pub(super) provider_handle: Option, - pub(super) selected_provider_config: Option>, -} - -#[allow(clippy::too_many_arguments)] -pub(super) async fn try_reserve_hls_entry_origin_account_for_redirect( - app_state: &Arc, - fingerprint: &Fingerprint, - user: &ProxyUserCredentials, - input: &ConfigInput, - virtual_id: u32, - request_url: &str, - user_session_token: &str, - session_owner: &str, - playback_kind: PlaybackKind, - reservation_ttl_secs: u64, - connection_permission: UserConnectionPermission, - connection_kind: crate::api::model::ConnectionKind, - create_user_session: bool, -) -> Option { - let provider_handle = app_state.active_provider.acquire_connection_with_lease_for_session( - &input.name, - &fingerprint.addr, - false, - connection_priority_for_kind(user, connection_kind), - connection_kind, - Some(PlaybackLeaseRef::new(session_owner, playback_kind)), - )?; - - let Some(provider_config) = provider_handle.allocation.get_provider_config() else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); - return None; - }; - let Some((_provider_name, stream_url)) = - select_provider_stream_url(request_url, input, &provider_config, false, &app_state.app_config).await - else { - app_state.connection_manager.release_provider_handle(Some(provider_handle)); - return None; - }; - - let session_token = if create_user_session { - app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user, - session_token: user_session_token, - virtual_id, - provider: &provider_config.name, - stream_url: &stream_url, - addr: &fingerprint.addr, - connection_permission, - connection_kind: Some(connection_kind), - socket_bound: PlaylistItemType::LiveHls.uses_socket_bound_session(), - }) - .await - } else { - user_session_token.to_string() - }; - - app_state.active_provider.refresh_adaptive_playback_lease( - &provider_config.name, - session_owner, - playback_kind, - reservation_ttl_secs, - ); - - Some(HlsEntryOriginAccountReservation { - request_url: stream_url, - session_token, - provider_handle: Some(provider_handle), - selected_provider_config: Some(provider_config), - }) -} - -#[allow(clippy::too_many_arguments)] -pub(super) async fn try_reserve_hls_virtual_entry_origin_account_for_redirect( - app_state: &Arc, - fingerprint: &Fingerprint, - user: &ProxyUserCredentials, - target: &Arc, - input: &ConfigInput, - stream_identity: &HlsEntryStreamIdentity, -) -> bool { - let virtual_id = stream_identity.virtual_id(); - let session_token = - create_playback_session_fingerprint(fingerprint, &user.username, virtual_id, PlaylistItemType::LiveHls, None); - let (connection_admission, _, _) = resolve_playback_request_admission( - &app_state.admission_ctx(), - user, - fingerprint, - None, - &session_token, - false, - EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(virtual_id) }, - false, - false, - ) - .await; - if connection_admission.permission() == UserConnectionPermission::Exhausted { - return false; - } - - let Some(channel) = get_stream_channel(app_state, target, virtual_id).await else { - return false; - }; - let Ok(origin_playlist_url) = - resolve_hls_origin_playlist_url(app_state, target, input, virtual_id, channel.url.as_ref()).await - else { - return false; - }; - let Some(hls_cache_origin) = build_hls_origin_resolution(input, &origin_playlist_url) else { - return false; - }; - let Some(connection_kind) = connection_admission.kind() else { - return false; - }; - let (shared_hls_session_owner, reservation_ttl_secs) = if hls_cache_enabled_for_user(app_state, target, user) { - let origin_source = build_hls_origin_source(input, stream_identity.stream_ref()); - let proxy_session_id = build_proxy_session_id(&origin_source.session_key(), &app_state.get_encrypt_secret()); - let reservation_ttl_secs = match app_state.hls.proxy.sessions().get_by_key(&origin_source.session_key()).await { - Some(session) => hls_origin_account_reservation_ttl_secs_for_session(&session).await, - None => hls_origin_account_reservation_ttl_secs_fallback(), - }; - (Some(build_hls_origin_session_owner(&proxy_session_id)), reservation_ttl_secs) - } else { - (None, get_hls_session_ttl_secs(app_state)) - }; - let session_owner = shared_hls_session_owner.as_deref().unwrap_or(session_token.as_str()); - - let Some(reservation) = try_reserve_hls_entry_origin_account_for_redirect( - app_state, - fingerprint, - user, - input, - virtual_id, - hls_cache_origin.session_entry_url.as_str(), - &session_token, - session_owner, - PlaybackKind::LiveHls, - reservation_ttl_secs, - connection_admission.permission(), - connection_kind, - false, - ) - .await - else { - return false; - }; - - app_state.connection_manager.release_provider_handle(reservation.provider_handle); - true -} - -pub(super) async fn mark_hls_provisioning_handoff_discontinuity( - app_state: &Arc, - input: &ConfigInput, - stream_identity: &HlsEntryStreamIdentity, - access_lease_id: Option<&HlsAccessLeaseId>, - now_ms: u64, -) -> bool { - if !hls_cache_configured(app_state) { - return false; - } - let origin_source = build_hls_origin_source(input, stream_identity.stream_ref()); - let Some(session) = app_state.hls.proxy.sessions().get_by_key(&origin_source.session_key()).await else { - return false; - }; - mark_hls_provisioning_handoff_discontinuity_once_for_session( - app_state, - &session, - input, - stream_identity.virtual_id(), - access_lease_id, - now_ms, - ) - .await -} - -pub(super) async fn mark_hls_provisioning_handoff_discontinuity_once_for_session( - app_state: &Arc, - session: &HlsSessionHandle, - input: &ConfigInput, - virtual_id: u32, - access_lease_id: Option<&HlsAccessLeaseId>, - now_ms: u64, -) -> bool { - let proxy_session_id = session.read().await.proxy_session_id.clone(); - if !app_state.hls.provisioning.mark_handoff_once( - &input.name, - virtual_id, - Some(&proxy_session_id), - access_lease_id, - now_ms, - ) { - debug!( - "HLS provisioning handoff discontinuity already marked: proxy_session={}", - safe_proxy_session_id(&proxy_session_id) - ); - return false; - } - mark_hls_provisioning_handoff_discontinuity_for_session(session, now_ms).await; - ensure_shared_hls_provisioning_handoff_gap(app_state, session, now_ms).await; - true -} - -pub(super) async fn mark_hls_provisioning_handoff_discontinuity_for_session(session: &HlsSessionHandle, now_ms: u64) { - let discontinuity_sequence = hls_provisioning_discontinuity_sequence(now_ms); - let proxy_session_id = { - let mut session = session.write().await; - session.mark_pending_handoff_discontinuity(discontinuity_sequence); - session.proxy_session_id.clone() - }; - debug!( - "HLS provisioning handoff discontinuity marked: proxy_session={} discontinuity_sequence={}", - safe_proxy_session_id(&proxy_session_id), - discontinuity_sequence - ); -} - -pub(super) fn clear_hls_provisioning_handoff_consumer( - app_state: &Arc, - input: &ConfigInput, - virtual_id: u32, - now_ms: u64, -) { - if !app_state.hls.provisioning.take_ready_slot_for_consumer(&input.name, virtual_id, now_ms) { - app_state.hls.provisioning.clear_consumer(&input.name, virtual_id); - } -} - -pub(super) async fn maybe_mark_hls_provisioning_handoff_for_canonical_manifest( - app_state: &Arc, - session: &HlsSessionHandle, - input: &ConfigInput, - virtual_id: u32, - access_lease_id: &HlsAccessLeaseId, - now_ms: u64, -) -> Option { - if !app_state.hls.provisioning.has_consumer(&input.name, virtual_id, now_ms) { - return None; - } - let previous_manifest_rendered_at_ms = latest_shared_hls_manifest_rendered_at_ms(session).await; - mark_hls_provisioning_handoff_discontinuity_once_for_session( - app_state, - session, - input, - virtual_id, - Some(access_lease_id), - now_ms, - ) - .await - .then_some(previous_manifest_rendered_at_ms) -} - -pub(super) async fn latest_shared_hls_manifest_rendered_at_ms(session: &HlsSessionHandle) -> u64 { - let session = session.read().await; - session - .last_rendered_manifest - .as_ref() - .map_or(0, |rendered| rendered.rendered_at_ms) - .max(session.transient.last_manifest_rendered_at_ms.unwrap_or(0)) -} - -#[allow(clippy::too_many_arguments)] -pub(in crate::api) async fn hls_panel_provisioning_poll_manifest_response( - app_state: &Arc, - fingerprint: &Fingerprint, - user: &ProxyUserCredentials, - target: &Arc, - input: &ConfigInput, - stream_identity: &HlsEntryStreamIdentity, - original_hls_entry_path: &str, - server_path: Option<&str>, -) -> axum::response::Response { - hls_panel_provisioning_poll_response( - app_state, - fingerprint, - user, - target, - input, - stream_identity, - original_hls_entry_path, - server_path, - HlsProvisioningPollResponseKind::Legacy, - ) - .await -} - -pub(super) enum HlsProvisioningPollResponseKind { - Legacy, -} - -impl HlsProvisioningPollResponseKind { - pub(super) fn access_lease_id(&self) -> Option<&HlsAccessLeaseId> { - match self { - Self::Legacy => None, - } - } -} - -#[allow(clippy::too_many_arguments)] -pub(super) async fn hls_panel_provisioning_poll_response( - app_state: &Arc, - fingerprint: &Fingerprint, - user: &ProxyUserCredentials, - target: &Arc, - input: &ConfigInput, - stream_identity: &HlsEntryStreamIdentity, - ready_redirect_path: &str, - server_path: Option<&str>, - response_kind: HlsProvisioningPollResponseKind, -) -> axum::response::Response { - let virtual_id = stream_identity.virtual_id(); - let now_ms = current_time_millis(); - app_state.hls.provisioning.touch_consumer(Arc::clone(&input.name), virtual_id, now_ms); - - let existing_status = app_state.hls.provisioning.consumer_status(&input.name, virtual_id, now_ms); - - if try_reserve_hls_virtual_entry_origin_account_for_redirect( - app_state, - fingerprint, - user, - target, - input, - stream_identity, - ) - .await - { - mark_hls_provisioning_handoff_discontinuity( - app_state, - input, - stream_identity, - response_kind.access_lease_id(), - now_ms, - ) - .await; - clear_hls_provisioning_handoff_consumer(app_state, input, virtual_id, current_time_millis()); - return hls_virtual_entry_redirect_response(ready_redirect_path, server_path); - } - - let provisioning_enabled = can_provision_on_exhausted(app_state.as_ref(), input); - if provisioning_enabled { - start_hls_panel_provisioning_once(app_state, input); - } - - let status = existing_status.unwrap_or(if provisioning_enabled { - HlsProvisioningStatus::InProgress - } else { - HlsProvisioningStatus::ProviderExhausted - }); - - match status { - HlsProvisioningStatus::Ready | HlsProvisioningStatus::InProgress => { - hls_custom_video_manifest_response_with_virtual_id( - app_state, - user, - CustomVideoStreamType::Provisioning, - StatusCode::SERVICE_UNAVAILABLE, - Some(virtual_id), - ) - .await - } - HlsProvisioningStatus::ProviderExhausted => { - hls_custom_video_manifest_response_with_virtual_id( - app_state, - user, - CustomVideoStreamType::ProviderConnectionsExhausted, - StatusCode::SERVICE_UNAVAILABLE, - Some(virtual_id), - ) - .await - } - } -} - -pub(super) async fn hls_panel_provisioning_or_status_response( - app_state: &Arc, - user: &ProxyUserCredentials, - input: &ConfigInput, - virtual_id: u32, - _original_hls_entry_path: &str, - server_path: Option<&str>, - fallback_status: StatusCode, -) -> axum::response::Response { - try_hls_panel_provisioning_manifest_response( - app_state, - user, - input, - virtual_id, - HlsPanelProvisioningRedirectPaths { waiting_manifest_path: None }, - server_path, - fallback_status, - ) - .await - .unwrap_or_else(|| fallback_status.into_response()) -} - -#[derive(Debug, Clone)] -pub(super) struct SharedHlsProvisioningSegmentPlan { - pub(super) proxy_seq: u64, - pub(super) physical_index: usize, - pub(super) cache_key: SegmentCacheKey, - pub(super) segment_kind: SharedHlsProvisioningLocalSegmentKind, -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub(super) enum SharedHlsProvisioningLocalSegmentKind { - Provisioning, - Gap, -} - -pub(super) fn shared_hls_provisioning_segment_plans( - session: &HlsSession, - physical_segment_count: usize, -) -> Vec { - let existing_provisioning_segments = - session.segments.values().filter(|entry| is_hls_provisioning_segment(entry)).count(); - let append_count = if existing_provisioning_segments == 0 { 3 } else { 1 }; - let start_proxy_seq = session.proxy_next_seq.unwrap_or(0); - (0..append_count) - .filter_map(|offset| { - let proxy_seq = start_proxy_seq.checked_add(u64::try_from(offset).ok()?)?; - if session.segments.contains_key(&proxy_seq) { - return None; - } - Some(SharedHlsProvisioningSegmentPlan { - proxy_seq, - physical_index: (existing_provisioning_segments + offset) % physical_segment_count, - cache_key: SegmentCacheKey::new(session.proxy_session_id.clone(), proxy_seq, "ts"), - segment_kind: SharedHlsProvisioningLocalSegmentKind::Provisioning, - }) - }) - .collect() -} - -pub(super) fn shared_hls_provisioning_segment_entry( - plan: SharedHlsProvisioningSegmentPlan, - content_length: u64, - duration_ms: u64, - now_ms: u64, -) -> SegmentEntry { - let origin_epoch = match plan.segment_kind { - SharedHlsProvisioningLocalSegmentKind::Provisioning => HLS_PROVISIONING_ORIGIN_EPOCH, - SharedHlsProvisioningLocalSegmentKind::Gap => HLS_PROVISIONING_GAP_ORIGIN_EPOCH, - }; - SegmentEntry { - origin_key: OriginSegmentKey { - origin_epoch, - effective_host_id: 0, - host_local_sequence: plan.proxy_seq, - host_local_index: u32::try_from(plan.proxy_seq).unwrap_or(u32::MAX), - }, - proxy_seq: plan.proxy_seq, - duration_ms, - proxy_file_ext: "ts".to_string(), - content_type: "video/mp2t".to_string(), - cache_key: plan.cache_key, - discontinuity_before: false, - program_date_time: None, - daterange_tags_before: Vec::new(), - origin_byte_range: None, - map_ref: None, - encryption: None, - origin_fetch_ref: None, - status: SegmentCacheStatus::Ready { content_length, ready_at_ms: now_ms }, - last_rendered_at_ms: None, - access: Arc::new(CacheAccessState::new()), - } -} - -pub(super) async fn commit_shared_hls_provisioning_segments( - app_state: &Arc, - session: &HlsSessionHandle, - plans: &[SharedHlsProvisioningSegmentPlan], - provisioning_segments: &[TransportStreamBuffer], -) -> Option> { - let mut committed = Vec::with_capacity(plans.len()); - for plan in plans { - let video = provisioning_segments.get(plan.physical_index)?; - let duration_ms = video.duration_ms().unwrap_or(HLS_PROVISIONING_SEGMENT_DURATION_MS); - let metadata = match app_state - .hls - .proxy - .segment_cache() - .write_bytes_and_commit(&plan.cache_key, video.as_bytes()) - .await - { - Ok(metadata) => metadata, - Err(err) => { - let safe_proxy_session = { - let session_guard = session.read().await; - safe_proxy_session_id(&session_guard.proxy_session_id) - }; - warn!( - "HLS provisioning segment cache commit failed for shared manifest: proxy_session={} seq={} error={err}", - safe_proxy_session, plan.proxy_seq - ); - return None; - } - }; - committed.push((plan.clone(), metadata.size, duration_ms)); - } - Some(committed) -} - -pub(super) async fn ensure_shared_hls_provisioning_handoff_gap( - app_state: &Arc, - session: &HlsSessionHandle, - now_ms: u64, -) -> bool { - let custom_stream_response = app_state.app_config.custom_stream_response.load(); - let Some(provisioning_segments) = custom_stream_response - .as_ref() - .map(|response| response.panel_api_provisioning_hls_segments.clone()) - .filter(|segments| !segments.is_empty()) - else { - return false; - }; - let plan = { - let session_guard = session.read().await; - if !session_guard.segments.values().any(is_hls_provisioning_segment) - || session_guard.segments.values().any(is_hls_provisioning_gap_segment) - { - return false; - } - let proxy_seq = session_guard.proxy_next_seq.unwrap_or(0); - if session_guard.segments.contains_key(&proxy_seq) { - return false; - } - let existing_provisioning_segments = - session_guard.segments.values().filter(|entry| is_hls_provisioning_segment(entry)).count(); - SharedHlsProvisioningSegmentPlan { - proxy_seq, - physical_index: existing_provisioning_segments % provisioning_segments.len(), - cache_key: SegmentCacheKey::new(session_guard.proxy_session_id.clone(), proxy_seq, "ts"), - segment_kind: SharedHlsProvisioningLocalSegmentKind::Gap, - } - }; - let Some(committed) = commit_shared_hls_provisioning_segments( - app_state, - session, - std::slice::from_ref(&plan), - &provisioning_segments, - ) - .await - else { - return false; - }; - let mut session_guard = session.write().await; - let mut inserted = false; - for (plan, content_length, duration_ms) in committed { - if session_guard.segments.contains_key(&plan.proxy_seq) { - continue; - } - if session_guard.publishable_origin_head_proxy_seq.is_none() { - session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq); - } - session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq); - session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1)); - session_guard - .segments - .insert(plan.proxy_seq, shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms)); - inserted = true; - } - if inserted { - session_guard.target_duration = Some(HLS_PROVISIONING_TARGET_DURATION_SECS); - session_guard.independent_segments = true; - } - inserted -} - -pub(super) async fn hls_shared_provisioning_timeline_manifest_response( - app_state: &Arc, - session: &HlsSessionHandle, - access_lease_id: &HlsAccessLeaseId, - access_lease_state: HlsAccessLeaseState, - strip: &crate::model::StripConfig, - server_path: Option<&str>, -) -> Option { - let custom_stream_response = app_state.app_config.custom_stream_response.load(); - let provisioning_segments = custom_stream_response - .as_ref() - .map(|response| response.panel_api_provisioning_hls_segments.clone()) - .filter(|segments| !segments.is_empty())?; - let now_ms = current_time_millis(); - let plans = { - let session_guard = session.read().await; - shared_hls_provisioning_segment_plans(&session_guard, provisioning_segments.len()) - }; - if plans.is_empty() { - let mut session_guard = session.write().await; - session_guard.render_and_store_manifest(now_ms).ok()?; - } else { - let committed = - commit_shared_hls_provisioning_segments(app_state, session, &plans, &provisioning_segments).await?; - let mut session_guard = session.write().await; - for (plan, content_length, duration_ms) in committed { - if session_guard.segments.contains_key(&plan.proxy_seq) { - continue; - } - if session_guard.publishable_origin_head_proxy_seq.is_none() { - session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq); - } - session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq); - session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1)); - session_guard.segments.insert( - plan.proxy_seq, - shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms), - ); - } - session_guard.target_duration = Some(HLS_PROVISIONING_TARGET_DURATION_SECS); - session_guard.independent_segments = true; - session_guard.render_and_store_manifest(now_ms).ok()?; - } - - try_hls_cached_manifest_response( - app_state, - session, - access_lease_id, - access_lease_state, - strip, - server_path, - HlsCachedManifestOptions::initial(Duration::ZERO), - HlsRuntimeBandwidthLearningContext::Disabled, - ) - .await -} - -#[allow(clippy::too_many_arguments)] -pub(super) async fn hls_shared_provisioning_or_provider_exhausted_response( - app_state: &Arc, - session: &HlsSessionHandle, - username: &str, - input: &ConfigInput, - virtual_id: u32, - access_lease_id: &HlsAccessLeaseId, - access_lease_state: HlsAccessLeaseState, - strip: &crate::model::StripConfig, - server_path: Option<&str>, -) -> axum::response::Response { - let Some((_user, _target)) = app_state.app_config.get_target_for_username(username) else { - return hls_canonical_retry_after_response(); - }; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let now_ms = current_time_millis(); - let provisioning_enabled = can_provision_on_exhausted(app_state.as_ref(), input); - if provisioning_enabled { - app_state.hls.provisioning.touch_consumer(Arc::clone(&input.name), virtual_id, now_ms); - start_hls_panel_provisioning_once(app_state, input); - if let Some(HlsProvisioningStatus::ProviderExhausted) = - app_state.hls.provisioning.consumer_status(&input.name, virtual_id, now_ms) - { - return hls_runtime_or_standalone_custom_tail_response( - app_state, - session, - &proxy_session_id, - access_lease_id, - HlsRuntimeCustomTailReason::ProviderConnectionsExhausted, - StatusCode::SERVICE_UNAVAILABLE, - ) - .await; - } - if let Some(response) = hls_shared_provisioning_timeline_manifest_response( - app_state, - session, - access_lease_id, - access_lease_state, - strip, - server_path, - ) - .await - { - return response; - } - } - - let provider_exhausted_custom_response_available = is_custom_video_stream_enabled(&app_state.app_config) - && app_state - .app_config - .custom_stream_response - .load() - .as_ref() - .and_then(|response| response.provider_connections_exhausted.as_ref()) - .is_some(); - if provider_exhausted_custom_response_available { - return hls_runtime_or_standalone_custom_tail_response( - app_state, - session, - &proxy_session_id, - access_lease_id, - HlsRuntimeCustomTailReason::ProviderConnectionsExhausted, - StatusCode::SERVICE_UNAVAILABLE, - ) - .await; - } - hls_canonical_retry_after_response() -} - -pub(super) enum HlsProviderExhaustedResolution { - RetryAcquire, - Response(axum::response::Response), -} - -#[allow(clippy::too_many_arguments)] -pub(super) async fn hls_provider_connections_exhausted_manifest_resolution( - app_state: &Arc, - session: &HlsSessionHandle, - username: &str, - input: &ConfigInput, - virtual_id: u32, - access_lease_id: &HlsAccessLeaseId, - access_lease_state: HlsAccessLeaseState, - strip: &crate::model::StripConfig, - server_path: Option<&str>, - allow_grace_hold: bool, -) -> HlsProviderExhaustedResolution { - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let grace_options = app_state.get_grace_options(); - if allow_grace_hold && grace_options.hold_stream && grace_options.period_millis > 0 { - debug!( - "HLS provider connections exhausted; holding canonical manifest for grace: proxy_session={} lease={} hold_ms={}", - safe_proxy_session_id(&proxy_session_id), - safe_hls_access_lease_id(access_lease_id), - grace_options.period_millis - ); - let capacity_notify = app_state.connection_manager.capacity_notified(); - let deadline = tokio::time::Instant::now() + Duration::from_millis(grace_options.period_millis); - let wake_reason = tokio::select! { - () = capacity_notify.notified() => "capacity-notified", - () = tokio::time::sleep_until(deadline) => "timeout", - }; - debug!( - "HLS provider connections exhausted grace hold completed: proxy_session={} lease={} reason={wake_reason}", - safe_proxy_session_id(&proxy_session_id), - safe_hls_access_lease_id(access_lease_id) - ); - return HlsProviderExhaustedResolution::RetryAcquire; - } - - HlsProviderExhaustedResolution::Response( - hls_shared_provisioning_or_provider_exhausted_response( - app_state, - session, - username, - input, - virtual_id, - access_lease_id, - access_lease_state, - strip, - server_path, - ) - .await, - ) -} - -#[allow(clippy::too_many_arguments)] -pub(super) async fn prepare_hls_canonical_manifest_origin_runtime( - app_state: &Arc, - session: &HlsSessionHandle, - context: &HlsAccessContext, - origin: &HlsCacheManifestOrigin<'_>, - path_proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, - access_lease_state: HlsAccessLeaseState, - fingerprint: &Fingerprint, - server_path: Option<&str>, - now_ms: u64, -) -> Result> { - let mut allow_grace_hold = true; - loop { - let origin_policy = hls_effective_origin_acquire_policy(session).await; - match prepare_hls_origin_runtime( - app_state, - session, - origin.input, - origin.raw_request_url, - origin.session_entry_url.as_str(), - path_proxy_session_id, - fingerprint, - origin_policy.connection_kind, - origin_policy.priority, - HlsOriginWorkKind::Manifest, - HlsOriginWorkClass::ManifestInteractive, - now_ms, - ) - .await - { - Ok(prepared) => return Ok(prepared), - Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { - reason: HlsOriginRuntimeNoAccountReason::OriginBindingPreempted, - }) => { - return Err(Box::new( - hls_runtime_or_standalone_custom_tail_response( - app_state, - session, - path_proxy_session_id, - access_lease_id, - HlsRuntimeCustomTailReason::LowPriorityPreempted, - StatusCode::SERVICE_UNAVAILABLE, - ) - .await, - )); - } - Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { - reason: HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted, - }) => { - let strip = app_state.hls.proxy.strip(); - match hls_provider_connections_exhausted_manifest_resolution( - app_state, - session, - &context.username, - origin.input, - context.virtual_id, - access_lease_id, - access_lease_state, - &strip, - server_path, - allow_grace_hold, - ) - .await - { - HlsProviderExhaustedResolution::RetryAcquire => { - allow_grace_hold = false; - } - HlsProviderExhaustedResolution::Response(response) => return Err(Box::new(response)), - } - } - Err(HlsOriginRuntimeAcquireError::Fatal(status)) => { - return Err(Box::new(hls_canonical_status_response(status))) - } - } - } -} - -pub(super) async fn apply_hls_user_agent_stream_index( - session: &HlsSessionHandle, - headers: &mut HeaderMap, - enabled: bool, - active_users: &crate::api::model::ActiveUserManager, -) { - if !enabled { - return; - } - let stream_index = { - let mut session = session.write().await; - if session.user_agent_stream_index.is_none() { - session.user_agent_stream_index = Some(active_users.next_user_agent_stream_index()); - } - session.user_agent_stream_index - }; - if let Some(stream_index) = stream_index { - request::append_user_agent_stream_index(headers, stream_index); - } -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -pub(super) async fn try_hls_cache_canonical_manifest_response( - app_state: &Arc, - fingerprint: &Fingerprint, - context: &HlsAccessContext, - path_proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, - access_lease_state: HlsAccessLeaseState, - origin: HlsCacheManifestOrigin<'_>, - mut headers: HeaderMap, - server_path: Option<&str>, - _original_hls_entry_path: &str, - refresh_ordering: HlsManifestRefreshOrdering, -) -> Option { - if !hls_cache_configured(app_state) { - return None; - } - if origin.origin_source.input_id != context.input_id || origin.origin_source.stream_ref != context.stream_ref { - return Some(StatusCode::NOT_FOUND.into_response()); - } - - let session_key = origin.origin_source.session_key(); - let expected_proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - if &expected_proxy_session_id != path_proxy_session_id { - return Some(StatusCode::NOT_FOUND.into_response()); - } - let now_ms = current_time_millis(); - let rewrite_secret = app_state.get_encrypt_secret(); - let (session, session_outcome) = app_state - .hls - .proxy - .get_or_create_session_with_source_and_outcome( - session_key, - origin.origin_source.clone(), - &rewrite_secret, - now_ms, - ) - .await; - if access_lease_state == HlsAccessLeaseState::Activated { - let timing = hls_access_lease_timing_for_session(app_state, &session).await; - match app_state - .hls - .proxy - .touch_manifest_access_lease( - access_lease_id, - path_proxy_session_id, - now_ms, - Some(timing), - None, - hls_access_lease_ttl_ms(app_state), - ) - .await - { - HlsAccessLeaseTouch::Touched { .. } => {} - HlsAccessLeaseTouch::Denied => { - return Some( - hls_runtime_or_standalone_custom_tail_response( - app_state, - &session, - path_proxy_session_id, - access_lease_id, - HlsRuntimeCustomTailReason::UserConnectionsExhausted, - StatusCode::FORBIDDEN, - ) - .await, - ); - } - HlsAccessLeaseTouch::Expired | HlsAccessLeaseTouch::UnknownLease | HlsAccessLeaseTouch::SessionMismatch => { - return Some(StatusCode::NOT_FOUND.into_response()); - } - } - } - app_state - .hls - .proxy - .sync_session_access_lease_count_and_detach_if_needed( - &app_state.active_users, - &app_state.active_provider, - &session, - path_proxy_session_id, - now_ms, - ) - .await; - let prepared_origin = match prepare_hls_canonical_manifest_origin_runtime( - app_state, - &session, - context, - &origin, - path_proxy_session_id, - access_lease_id, - access_lease_state, - fingerprint, - server_path, - now_ms, - ) - .await - { - Ok(prepared) => prepared, - Err(response) => return Some(*response), - }; - apply_hls_user_agent_stream_index( - &session, - &mut headers, - origin.input.has_flag(ConfigInputFlags::UserAgentStreamIndex), - &app_state.active_users, - ) - .await; - let url_failover_provider = effective_hls_url_failover_provider_for_fetch_url( - &prepared_origin.fetch_url, - prepared_origin.url_failover_provider.clone(), - origin.session_entry_url.url_failover_provider(), - ); - let origin_entry = LiveHlsOriginEntry::parse_with_provider_configs( - &prepared_origin.fetch_url, - url_failover_provider, - prepared_origin.runtime_provider_config.clone(), - )?; - { - let mut session_guard = session.write().await; - if session_guard.is_gc_marked_for_removal() { - return Some(hls_canonical_retry_after_response()); - } - if prepared_origin.origin_account_binding_to_store.is_some() { - session_guard.replace_origin_account_binding(prepared_origin.origin_account_binding_to_store); - } - } - mark_hls_authorized_manifest_access(app_state, &session, now_ms).await; - let selected_account = session.read().await.origin_account_binding.as_ref().map_or_else( - || "".to_string(), - |binding| sanitize_sensitive_info(binding.account_name.as_ref()).to_string(), - ); - debug!( - "HLS origin account selected: proxy_session={} account={}", - safe_proxy_session_id(path_proxy_session_id), - selected_account - ); - let reservation_ttl_secs = hls_origin_account_reservation_ttl_secs_for_session(&session).await; - let previous_manifest_rendered_at_ms = latest_shared_hls_manifest_rendered_at_ms(&session).await; - let handoff_previous_rendered_at_ms = maybe_mark_hls_provisioning_handoff_for_canonical_manifest( - app_state, - &session, - origin.input, - context.virtual_id, - access_lease_id, - now_ms, - ) - .await; - let manifest_commit_requirement = - hls_manifest_commit_requirement(&session, session_outcome, handoff_previous_rendered_at_ms, now_ms).await; - let hls_ctx = app_state.hls_ctx(); - let acceptance_evaluation = - hls_manifest_acceptance_directive_for_session(&hls_ctx, &session, path_proxy_session_id).await; - let (acceptance_directive, availability_reevaluation_owner_key) = match acceptance_evaluation { - HlsManifestAcceptanceEvaluationOutcome::Evaluated(directive) => (directive, None), - HlsManifestAcceptanceEvaluationOutcome::StateContention { owner_key } => { - (HlsManifestAcceptanceDirective::none(), Some(owner_key)) - } - HlsManifestAcceptanceEvaluationOutcome::SessionSuperseded => { - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); - return Some(hls_canonical_retry_after_response()); - } - }; - let manifest_boundary_rendered_at_ms = handoff_previous_rendered_at_ms.unwrap_or(previous_manifest_rendered_at_ms); - let wait_timeout = - hls_manifest_wait_timeout_for_requirement(app_state, &session, manifest_commit_requirement).await; - let cached_manifest_options = hls_cached_manifest_options_for_requirement( - wait_timeout, - manifest_commit_requirement, - manifest_boundary_rendered_at_ms, - ); - let bandwidth_learning = match context.known_bitrate_bps { - Some(_) => HlsRuntimeBandwidthLearningContext::Disabled, - None => HlsRuntimeBandwidthLearningContext::Eligible(origin.input), - }; - - let origin_policy = hls_effective_origin_acquire_policy(&session).await; - let origin_provider_session_headers = session.read().await.origin_provider_session_headers.clone(); - let mut preacquired_provider_handle = prepared_origin.preacquired_origin_account_handle; - let mut origin_io = HlsOriginIoContext { - ctx: hls_ctx.clone(), - client_addr: fingerprint.addr, - allow_grace: HlsOriginWorkClass::ManifestInteractive.allows_grace(), - priority: origin_policy.priority, - connection_kind: origin_policy.connection_kind, - reservation_ttl_secs, - preacquired_provider_handle: None, - started_generation: None, - }; - if availability_reevaluation_owner_key.is_none() { - if let Some(provider_handle) = preacquired_provider_handle.take() { - origin_io = origin_io.with_preacquired_provider_handle(provider_handle); - } - } - - let refresh_request = OriginRefreshRequest { - app_config: Arc::clone(&app_state.app_config), - session: Arc::clone(&session), - origin_entry, - headers, - origin_provider_session_headers, - client: app_state.http_clients.default.load().as_ref().clone(), - no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(), - use_manual_redirects: app_state.should_use_manual_redirects(), - segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()), - hls_proxy: Arc::clone(&app_state.hls.proxy), - segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()), - segment_worker_pool: Arc::clone(app_state.hls.proxy.segment_worker_pool()), - map_worker_pool: Arc::clone(app_state.hls.proxy.map_worker_pool()), - origin_manifest_timeout_ms: app_state.hls.proxy.origin_manifest_timeout_ms(), - manifest_recovery_burst: app_state.hls.proxy.manifest_recovery_burst(), - strip: app_state.hls.proxy.strip().clone(), - retry_policy: RetryPolicy::default(), - reverse_proxy_rewrite_secret: rewrite_secret.to_vec(), - transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(), - manifest_commit_requirement, - fresh_manifest_requirement_generation: None, - acceptance_directive, - access_lease_id: Some(access_lease_id.clone()), - disabled_headers: app_state.get_disabled_headers(), - now_ms, - origin_io: Some(origin_io), - post_refresh_runtime: Some(HlsPostRefreshRuntime { ctx: hls_ctx.downgrade() }), - }; - let refresh_ordering = if session_outcome == HlsSessionStoreOutcome::Reused { - refresh_ordering - } else { - HlsManifestRefreshOrdering::Background - }; - if let Some(owner_key) = availability_reevaluation_owner_key { - app_state.connection_manager.release_provider_handle(preacquired_provider_handle); - touch_initial_manifest_access_lease_window( - app_state, - access_lease_id, - path_proxy_session_id, - access_lease_state, - wait_timeout, - now_ms, - ) - .await; - let owner_wait_lease = app_state - .hls - .proxy - .access_lease_response_snapshot(access_lease_id, path_proxy_session_id, current_time_millis()) - .await; - let expected_lease_issued_at_ms = owner_wait_lease.as_ref().map(|lease| lease.issued_at_ms); - let request_deadline_ms = owner_wait_lease - .as_ref() - .map_or(now_ms, |lease| hls_canonical_owner_request_deadline_ms(lease, wait_timeout, now_ms)); - let safe_session = { - let session = session.read().await; - safe_session_key(&session.key) - }; - let registration = - register_hls_availability_reevaluation(hls_ctx, Arc::clone(&session), owner_key, refresh_request); - return Some(match hls_canonical_owner_registration(registration) { - HlsCanonicalOwnerRegistration::Join(registration) => { - let strip = app_state.hls.proxy.strip(); - join_hls_canonical_manifest_owner( - HlsCanonicalOwnerHandoffContext { - app_state, - proxy_session_id: path_proxy_session_id, - access_lease_id, - expected_lease_issued_at_ms, - strip: &strip, - server_path, - manifest_commit_requirement, - manifest_boundary_rendered_at_ms, - bandwidth_learning, - request_deadline_ms, - safe_session, - }, - registration, - ) - .await - } - HlsCanonicalOwnerRegistration::FailClosed(failure) => { - hls_availability_reevaluation_registration_failure_response(failure) - } - }); - } - if handoff_previous_rendered_at_ms.is_some() { - touch_initial_manifest_access_lease_window( - app_state, - access_lease_id, - path_proxy_session_id, - access_lease_state, - wait_timeout, - now_ms, - ) - .await; - if let Some(response) = trigger_hls_canonical_manifest_refresh( - app_state, - &session, - path_proxy_session_id, - access_lease_id, - refresh_request, - refresh_ordering, - ) - .await - { - return Some(response); - } - let strip = app_state.hls.proxy.strip(); - if let Some(response) = try_hls_cached_manifest_response( - app_state, - &session, - access_lease_id, - access_lease_state, - &strip, - server_path, - cached_manifest_options, - bandwidth_learning, - ) - .await - { - clear_hls_provisioning_handoff_consumer(app_state, origin.input, context.virtual_id, current_time_millis()); - return Some(response); - } - return Some(StatusCode::SERVICE_UNAVAILABLE.into_response()); - } - match session_outcome { - HlsSessionStoreOutcome::Created => { - touch_initial_manifest_access_lease_window( - app_state, - access_lease_id, - path_proxy_session_id, - access_lease_state, - wait_timeout, - now_ms, - ) - .await; - if let Some(response) = trigger_hls_canonical_manifest_refresh( - app_state, - &session, - path_proxy_session_id, - access_lease_id, - refresh_request, - refresh_ordering, - ) - .await - { - return Some(response); - } - let strip = app_state.hls.proxy.strip(); - if let Some(response) = try_hls_cached_manifest_response( - app_state, - &session, - access_lease_id, - access_lease_state, - &strip, - server_path, - cached_manifest_options, - bandwidth_learning, - ) - .await - { - return Some(response); - } - } - HlsSessionStoreOutcome::Reused => { - if let Some(response) = trigger_hls_canonical_manifest_refresh( - app_state, - &session, - path_proxy_session_id, - access_lease_id, - refresh_request, - refresh_ordering, - ) - .await - { - if refresh_ordering == HlsManifestRefreshOrdering::AwaitBeforeTerminalEvaluation - && response.status() == StatusCode::SERVICE_UNAVAILABLE - { - let strip = app_state.hls.proxy.strip(); - if let Some(live_response) = try_hls_cached_manifest_response( - app_state, - &session, - access_lease_id, - access_lease_state, - &strip, - server_path, - HlsCachedManifestOptions::initial(Duration::ZERO), - bandwidth_learning, - ) - .await - .filter(|candidate| candidate.status() == StatusCode::OK) - { - return Some(live_response); - } - } - return Some(response); - } - touch_initial_manifest_access_lease_window( - app_state, - access_lease_id, - path_proxy_session_id, - access_lease_state, - wait_timeout, - now_ms, - ) - .await; - let strip = app_state.hls.proxy.strip(); - if let Some(response) = try_hls_cached_manifest_response( - app_state, - &session, - access_lease_id, - access_lease_state, - &strip, - server_path, - cached_manifest_options, - bandwidth_learning, - ) - .await - { - return Some(response); - } - } - } - - Some(hls_unpublished_lease_channel_unavailable_response(app_state, path_proxy_session_id, access_lease_id).await) -} - -pub(super) fn hls_initial_manifest_decision_wait_timeout(app_state: &Arc) -> Duration { - Duration::from_secs(app_state.hls.proxy.initial_manifest_wait_timeout_secs()) -} - -pub(super) async fn hls_manifest_wait_timeout_for_requirement( - app_state: &Arc, - session: &HlsSessionHandle, - requirement: HlsManifestCommitRequirement, -) -> Duration { - match requirement { - HlsManifestCommitRequirement::FreshCommitRequired { .. } => { - hls_initial_manifest_decision_wait_timeout(app_state) - } - HlsManifestCommitRequirement::CommittedManifestAllowed => { - hls_initial_manifest_wait_timeout(app_state, session).await - } - } -} - -pub(super) async fn touch_initial_manifest_access_lease_window( - app_state: &Arc, - access_lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - access_lease_state: HlsAccessLeaseState, - wait_timeout: Duration, - now_ms: u64, -) { - if wait_timeout.is_zero() || access_lease_state != HlsAccessLeaseState::Pending { - return; - } - let wait_timeout_ms = duration_to_millis_saturating(wait_timeout); - let deadline_ms = now_ms.saturating_add(wait_timeout_ms.max(hls_pending_bootstrap_window_ms(app_state))); - let touch = app_state - .hls - .proxy - .touch_manifest_access_lease( - access_lease_id, - proxy_session_id, - now_ms, - None, - Some(HlsAccessLeasePendingDeadline::Bootstrap { deadline_ms }), - hls_access_lease_ttl_ms(app_state), - ) - .await; - let failure = match touch { - HlsAccessLeaseTouch::Touched { .. } => return, - HlsAccessLeaseTouch::Expired => "expired", - HlsAccessLeaseTouch::Denied => "denied", - HlsAccessLeaseTouch::UnknownLease => "unknown-lease", - HlsAccessLeaseTouch::SessionMismatch => "session-mismatch", - }; - debug!( - "HLS initial manifest lease window not extended: lease={} proxy_session={} outcome={failure}", - safe_hls_access_lease_id(access_lease_id), - safe_proxy_session_id(proxy_session_id) - ); -} - -pub(super) async fn hls_initial_manifest_wait_timeout( - app_state: &Arc, - session: &HlsSessionHandle, -) -> Duration { - let session = session.read().await; - if matches!( - session.account_binding_protection(current_time_millis()), - HlsAccountBindingProtection::NoMediaYet | HlsAccountBindingProtection::Expired - ) { - hls_initial_manifest_decision_wait_timeout(app_state) - } else { - Duration::ZERO - } -} - -pub(super) struct HlsCachedManifestRead { - pub(super) transient_body: Option, - pub(super) rendered_body: Option, - pub(super) should_wait: bool, - pub(super) wait_for_initial_commit: bool, -} - pub(super) struct HlsCachedTransientManifestRead { pub(super) body: Arc, pub(super) template: Arc, @@ -1997,641 +11,66 @@ pub(super) struct HlsCachedTransientManifestRead { published_resource_ids: HlsPublishedTransientResourceIds, } -pub(super) async fn read_hls_cached_manifest( - session: &HlsSessionHandle, - options: HlsCachedManifestOptions, - started_at_ms: u64, -) -> HlsCachedManifestRead { - let session = session.read().await; - let now_ms = current_time_millis(); - let should_wait = session.initial_manifest_commit_work_pending(); - let committed_body = hls_committed_manifest_body_for_request(&session, options, started_at_ms, now_ms); - let (transient_body, rendered_body) = match committed_body { - Some(HlsCommittedManifestBody::Transient(body)) => ( - session.transient.last_manifest_template().zip(session.transient.last_manifest_commit_identity()).map( - |(template, source_commit_identity)| HlsCachedTransientManifestRead { - body, - template, - source_commit_identity, - window_policy: session.transient.last_manifest_window_policy(), - finalized_manifest_generation: session.transient.current_finalized_manifest_generation(), - published_resource_ids: session.transient.last_manifest_published_resource_ids(), - }, - ), - None, - ), - Some(HlsCommittedManifestBody::Normal(body)) => (None, Some(body)), - None => (None, None), - }; - let wait_for_initial_commit = hls_should_wait_for_initial_manifest_commit( - &session, - transient_body.is_some() || rendered_body.is_some(), - should_wait, - options, - now_ms, - ); - HlsCachedManifestRead { transient_body, rendered_body, should_wait, wait_for_initial_commit } -} - -pub(super) struct HlsCachedManifestViewContext<'a> { - pub(super) proxy_session_id: &'a ProxySessionId, - pub(super) access_lease_id: &'a HlsAccessLeaseId, - pub(super) access_lease_state: HlsAccessLeaseState, - pub(super) strip: &'a crate::model::StripConfig, - pub(super) server_path: Option<&'a str>, - pub(super) bandwidth_learning: HlsRuntimeBandwidthLearningContext<'a>, -} - -#[derive(Clone, Copy)] -pub(super) enum HlsRuntimeBandwidthLearningContext<'a> { - Disabled, - Eligible(&'a ConfigInput), -} - -impl HlsCachedManifestViewContext<'_> { - pub(super) fn new<'a>( - proxy_session_id: &'a ProxySessionId, - access_lease_id: &'a HlsAccessLeaseId, - access_lease_state: HlsAccessLeaseState, - strip: &'a crate::model::StripConfig, - server_path: Option<&'a str>, - bandwidth_learning: HlsRuntimeBandwidthLearningContext<'a>, - ) -> HlsCachedManifestViewContext<'a> { - HlsCachedManifestViewContext { - proxy_session_id, - access_lease_id, - access_lease_state, - strip, - server_path, - bandwidth_learning, - } - } - - pub(super) fn materialize( - &self, - body: &str, - mode: &'static str, - window_policy: HlsManifestWindowPolicy, - ) -> HlsMaterializedSharedManifest { - materialize_shared_hls_access_manifest( - body, - self.access_lease_id, - self.access_lease_state, - self.strip, - window_policy, - mode, - self.server_path, - ) - } - - pub(super) async fn finish( - &self, - app_state: &Arc, - session: &HlsSessionHandle, - materialized: HlsMaterializedSharedManifest, - strip_diagnostic: HlsInitialStripPublicationDiagnostic, - ) -> axum::response::Response { - touch_pending_manifest_follow_up_window(app_state, session, self.access_lease_id, self.access_lease_state) - .await; - drop(spawn_hls_runtime_bandwidth_persistence(app_state, session, self.bandwidth_learning)); - mark_successful_canonical_manifest_activity(app_state, session, current_time_millis()).await; - log_hls_initial_strip_publication(self.proxy_session_id, self.access_lease_id, strip_diagnostic); - hls_response(materialized.body).into_response() - } -} - -pub(super) fn spawn_hls_runtime_bandwidth_persistence( - app_state: &Arc, - session: &HlsSessionHandle, - context: HlsRuntimeBandwidthLearningContext<'_>, -) -> Option> { - let input = match context { - HlsRuntimeBandwidthLearningContext::Disabled => return None, - HlsRuntimeBandwidthLearningContext::Eligible(input) => input.clone(), - }; - let (bitrate_bps, proxy_session_id, stream_ref) = { - let Ok(mut session_guard) = session.try_write() else { - return None; - }; - let bitrate_bps = session_guard.begin_bandwidth_persistence(current_time_millis())?; - (bitrate_bps, session_guard.proxy_session_id.clone(), session_guard.origin_source.stream_ref.clone()) - }; - let app_config = Arc::clone(&app_state.app_config); - let hls_proxy = Arc::clone(&app_state.hls.proxy); - let session = Arc::clone(session); - - Some(tokio::spawn(async move { - let outcome = match persist_input_live_bitrate_bps(&app_config, &input, &stream_ref, bitrate_bps).await { - Ok(repository_outcome) => hls_bandwidth_persistence_outcome(repository_outcome, &proxy_session_id), - Err(err) => { - error!( - "HLS runtime bandwidth persistence failed: proxy_session={} error={}", - safe_proxy_session_id(&proxy_session_id), - sanitize_sensitive_info(&err.to_string()) - ); - HlsBandwidthPersistenceOutcome::RetryAfter - } - }; - let Some(current_session) = hls_proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await else { - return; - }; - if !Arc::ptr_eq(¤t_session, &session) { - return; - } - current_session.write().await.finish_bandwidth_persistence(bitrate_bps, outcome, current_time_millis()); - })) -} - -pub(super) fn hls_bandwidth_persistence_outcome( - repository_outcome: LiveBitratePersistenceOutcome, - proxy_session_id: &ProxySessionId, -) -> HlsBandwidthPersistenceOutcome { - match repository_outcome { - LiveBitratePersistenceOutcome::Updated | LiveBitratePersistenceOutcome::AlreadyEqualOrHigher => { - HlsBandwidthPersistenceOutcome::Persisted - } - LiveBitratePersistenceOutcome::MissingDatabase => { - debug!( - "HLS runtime bandwidth persistence deferred: proxy_session={} reason=missing_database", - safe_proxy_session_id(proxy_session_id) - ); - HlsBandwidthPersistenceOutcome::RetryAfter - } - LiveBitratePersistenceOutcome::MissingStreamItem => { - debug!( - "HLS runtime bandwidth persistence deferred: proxy_session={} reason=missing_stream_item", - safe_proxy_session_id(proxy_session_id) - ); - HlsBandwidthPersistenceOutcome::RetryAfter - } - LiveBitratePersistenceOutcome::PermanentlyInapplicable(reason) => { - debug!( - "HLS runtime bandwidth persistence skipped: proxy_session={} reason={}", - safe_proxy_session_id(proxy_session_id), - reason.log_label() - ); - HlsBandwidthPersistenceOutcome::PermanentlyInapplicable - } - } -} - -pub(super) fn hls_cached_manifest_temporarily_unavailable() -> axum::response::Response { - hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS) -} - -pub(super) fn observe_hls_lease_manifest_snapshot_derivation( - app_state: &AppState, - proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, - derivation: Result, HlsManifestLimitViolation>, -) -> Result, ()> { - match derivation { - Ok(snapshot) => { - if let Some(snapshot) = snapshot.as_ref() { - app_state.hls.proxy.metrics().record_lease_snapshot_segments(snapshot.visible_segments.len()); - } - Ok(snapshot) - } - Err(violation) => { - app_state.hls.proxy.metrics().record_manifest_limit_rejection(); - warn!( - "HLS lease manifest snapshot rejected: proxy_session={} lease={} reason=manifest-representation-limit kind={} actual={} limit={}", - safe_proxy_session_id(proxy_session_id), - safe_hls_access_lease_id(access_lease_id), - violation.kind.as_log_value(), - violation.actual, - violation.limit - ); - Err(()) - } - } -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -pub(super) async fn try_hls_cached_manifest_response( - app_state: &Arc, - session: &HlsSessionHandle, - access_lease_id: &HlsAccessLeaseId, - access_lease_state: HlsAccessLeaseState, - strip: &crate::model::StripConfig, - server_path: Option<&str>, - options: HlsCachedManifestOptions, - bandwidth_learning: HlsRuntimeBandwidthLearningContext<'_>, -) -> Option { - let started_at = tokio::time::Instant::now(); - let started_at_ms = current_time_millis(); - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let Some(publication_guard) = app_state - .hls - .proxy - .prepare_access_lease_manifest_publication(access_lease_id, &proxy_session_id, started_at_ms) - .await - else { - return Some(hls_cached_manifest_temporarily_unavailable()); - }; - let view = HlsCachedManifestViewContext::new( - &proxy_session_id, - access_lease_id, - access_lease_state, - strip, - server_path, - bandwidth_learning, - ); - loop { - let cached = read_hls_cached_manifest(session, options, started_at_ms).await; - if !cached.wait_for_initial_commit { - let prepared = if let Some(transient) = cached.transient_body { - let materialized = view.materialize(&transient.body, "transient", transient.window_policy); - let published_resource_ids = if transient.window_policy.preserves_full_manifest() { - transient.published_resource_ids.clone() - } else { - HlsPublishedTransientResourceIds::from_manifest_body(&materialized.body) - }; - let delivered_at_ms = current_time_millis(); - let snapshot_input = if transient.window_policy.preserves_full_manifest() { - HlsLeaseManifestSnapshotInput::TransientPassthroughTemplate { - template: &transient.template, - source_commit_identity: transient.source_commit_identity, - uri_materialization: HlsLeaseManifestUriMaterialization::new( - access_lease_id, - normalize_hls_proxy_public_path_prefix(server_path).map(Arc::from), - ), - finalized_manifest_generation: transient.finalized_manifest_generation, - } - } else { - HlsLeaseManifestSnapshotInput::TransientPassthrough { - materialized_body: &materialized.body, - source_commit_identity: transient.source_commit_identity, - finalized_manifest_generation: transient.finalized_manifest_generation, - } - }; - let snapshot = observe_hls_lease_manifest_snapshot_derivation( - app_state, - &proxy_session_id, - access_lease_id, - derive_hls_lease_manifest_snapshot(&snapshot_input, delivered_at_ms), - ); - let Ok(snapshot) = snapshot else { - return Some(hls_cached_manifest_temporarily_unavailable()); - }; - let Some(snapshot) = snapshot else { - return Some(hls_cached_manifest_temporarily_unavailable()); - }; - Some((materialized, snapshot, published_resource_ids, delivered_at_ms)) - } else if let Some(body) = cached.rendered_body { - let materialized = view.materialize(&body, "normal", HlsManifestWindowPolicy::ApplyLiveWindow); - let published_resource_ids = HlsPublishedTransientResourceIds::from_manifest_body(&materialized.body); - let delivered_at_ms = current_time_millis(); - let snapshot = { - let session = session.read().await; - observe_hls_lease_manifest_snapshot_derivation( - app_state, - &proxy_session_id, - access_lease_id, - derive_hls_lease_manifest_snapshot( - &HlsLeaseManifestSnapshotInput::NormalCacheTimeline { - session: &session, - committed_body: &body, - materialized_body: &materialized.body, - stripped_tail_segments: stripped_tail_segments(&materialized), - }, - delivered_at_ms, - ), - ) - }; - let Ok(snapshot) = snapshot else { - return Some(hls_cached_manifest_temporarily_unavailable()); - }; - let Some(snapshot) = snapshot else { - if access_lease_state != HlsAccessLeaseState::Pending { - return None; - } - if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await { - continue; - } - return Some(hls_cached_manifest_temporarily_unavailable()); - }; - Some((materialized, snapshot, published_resource_ids, delivered_at_ms)) - } else { - None - }; - if let Some((materialized, snapshot, published_resource_ids, delivered_at_ms)) = prepared { - if access_lease_state == HlsAccessLeaseState::Pending - && !hls_startup_admission_allows_snapshot(&app_state.hls_ctx(), session, &snapshot, delivered_at_ms) - .await - { - if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await { - continue; - } - return Some(hls_cached_manifest_temporarily_unavailable()); - } - let startup_snapshot = snapshot.clone(); - let admission_at_ms = current_time_millis(); - let outcome = app_state - .hls - .proxy - .commit_access_lease_manifest_publication_with_resources( - access_lease_id, - &proxy_session_id, - publication_guard, - snapshot, - published_resource_ids, - admission_at_ms, - ) - .await; - if let Some(snapshot_generation) = outcome.snapshot_generation() { - let published_at_ms = current_time_millis(); - let first_startup_publication = - app_state.hls.proxy.startup_observability().record_manifest_publication( - access_lease_id, - snapshot_generation, - admission_at_ms, - published_at_ms, - Arc::from(startup_snapshot.visible_proxy_seqs().collect::>()), - ); - if first_startup_publication && hls_access_manifest_uses_startup_view(access_lease_state) { - app_state.hls.proxy.spawn_access_lease_repair_prewarm( - Arc::clone(session), - access_lease_id.clone(), - startup_snapshot, - snapshot_generation, - ); - } - } - let publication_status = if outcome.is_committed() { - HlsInitialStripPublicationStatus::Committed - } else { - HlsInitialStripPublicationStatus::NotCommitted - }; - let Some(strip_diagnostic) = - hls_initial_strip_publication_diagnostic(publication_status, access_lease_state, &materialized) - else { - return Some(hls_cached_manifest_temporarily_unavailable()); - }; - return Some(view.finish(app_state, session, materialized, strip_diagnostic).await); - } - } - if options.wait_timeout.is_zero() || !cached.should_wait || started_at.elapsed() >= options.wait_timeout { - return None; - } - let remaining = options.wait_timeout.saturating_sub(started_at.elapsed()); - tokio::time::sleep(remaining.min(HLS_MANIFEST_WAIT_POLL_INTERVAL)).await; - } -} - -pub(super) async fn wait_for_hls_startup_evidence(started_at: tokio::time::Instant, wait_timeout: Duration) -> bool { - let elapsed = started_at.elapsed(); - if wait_timeout.is_zero() || elapsed >= wait_timeout { - return false; - } - tokio::time::sleep(wait_timeout.saturating_sub(elapsed).min(Duration::from_millis(25))).await; - true -} - -pub(super) async fn mark_successful_canonical_manifest_activity( - app_state: &Arc, - session: &HlsSessionHandle, - now_ms: u64, -) { - mark_hls_authorized_media_access(app_state, session, now_ms).await; -} - -pub(super) async fn mark_hls_authorized_manifest_access( - app_state: &Arc, - session: &HlsSessionHandle, - now_ms: u64, -) { - session.write().await.mark_authorized_manifest_access(now_ms); - app_state.hls.proxy.schedule_session_idle_for_handle(session).await; -} - -pub(super) async fn mark_hls_authorized_media_access( - app_state: &Arc, - session: &HlsSessionHandle, - now_ms: u64, -) { - session.write().await.mark_authorized_media_access(now_ms); - app_state.hls.proxy.schedule_session_idle_for_handle(session).await; -} - -pub(super) fn hls_cache_configured(app_state: &Arc) -> bool { - let config = app_state.app_config.config.load(); - config.reverse_proxy.as_ref().is_some_and(|reverse_proxy| reverse_proxy.hls_cache.is_some()) -} - -pub(super) fn hls_cache_enabled_for_target(app_state: &Arc, target: &ConfigTarget) -> bool { - hls_cache_configured(app_state) && is_hls_stream_share_enabled(target) -} - -pub(super) fn hls_cache_enabled_for_user( - app_state: &Arc, - target: &ConfigTarget, - user: &ProxyUserCredentials, -) -> bool { - // Recording headers can select different origin content from ordinary playback. - !user.is_recording_proxy_user() && hls_cache_enabled_for_target(app_state, target) -} - -pub(super) struct HlsAccessManifestRequestContext { - pub(super) input: Arc, - pub(super) hls_url: String, - pub(super) session_entry_url: HlsOriginEntryUrl, - pub(super) original_hls_entry_path: String, - pub(super) origin_source: HlsOriginSource, - pub(super) headers: HeaderMap, - pub(super) server_path: Option, -} - -pub(super) async fn resolve_hls_playback_manifest_request_context( - app_state: &Arc, - access_context: &HlsAccessContext, - req_headers: &HeaderMap, -) -> Result { - let Some((user, target)) = app_state.app_config.get_target_for_username(&access_context.username) else { - return Err(StatusCode::NOT_FOUND); - }; - if !hls_cache_enabled_for_target(app_state, &target) { - return Err(StatusCode::NOT_FOUND); - } - let Some(input) = app_state.app_config.get_input_by_id(access_context.input_id) else { - return Err(StatusCode::NOT_FOUND); - }; - if app_state - .active_users - .is_user_blocked_for_stream(&user.username, VirtualId::new(access_context.virtual_id)) - .await - { - return Err(StatusCode::FORBIDDEN); - } - let Some(channel) = get_stream_channel(app_state, &target, access_context.virtual_id).await else { - return Err(StatusCode::NOT_FOUND); - }; - let origin_playlist_url = if let Some(archive_url) = access_context.archive_origin_url.as_ref() { - archive_url.clone() - } else { - resolve_hls_origin_playlist_url(app_state, &target, &input, access_context.virtual_id, channel.url.as_ref()) - .await? - }; - let Some(hls_cache_origin) = build_hls_origin_resolution(&input, &origin_playlist_url) else { - return Err(StatusCode::SERVICE_UNAVAILABLE); - }; - let origin_source = build_hls_origin_source_for_playback( - &input, - access_context.stream_ref.clone(), - access_context.epg_reference_ts, - Some(&origin_playlist_url), - ); - let Some(server_info) = app_state.app_config.get_user_server_info(&user) else { - return Err(StatusCode::SERVICE_UNAVAILABLE); - }; - let disabled_headers = app_state.get_disabled_headers(); - let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); - let headers = build_hls_manifest_request_headers( - &input.headers, - req_headers, - disabled_headers.as_ref(), - default_user_agent.as_deref(), - channel.upstream_user_agent.as_deref(), - ); - - let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, access_context.virtual_id); - - Ok(HlsAccessManifestRequestContext { - input, - hls_url: hls_cache_origin.hls_url, - session_entry_url: hls_cache_origin.session_entry_url, - original_hls_entry_path, - origin_source, - headers, - server_path: server_info.path.clone(), - }) -} - -pub(super) async fn hls_manifest_preflight_refresh_ordering( - app_state: &Arc, - session: &HlsSessionHandle, - lease: &HlsAccessLease, - proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, - now_ms: u64, -) -> Result> { - match hls_manifest_terminal_preflight(session, lease, now_ms).await { - HlsManifestTerminalPreflight::ServeCommittedPlayback => { - Err(Box::new(hls_terminal_playback_response(lease, proxy_session_id, access_lease_id).unwrap_or_else( - || hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::RuntimeUnavailable), - ))) - } - HlsManifestTerminalPreflight::BootstrapPendingLease => Ok(HlsManifestRefreshOrdering::Background), - HlsManifestTerminalPreflight::RefreshBeforeTerminalEvaluation => { - Ok(HlsManifestRefreshOrdering::AwaitBeforeTerminalEvaluation) - } - HlsManifestTerminalPreflight::EvaluateTerminal => resolve_hls_terminal_manifest_state( - app_state, - session, - proxy_session_id, - access_lease_id, - lease.clone(), - now_ms, - ) - .await - .map(|_| HlsManifestRefreshOrdering::Background), - HlsManifestTerminalPreflight::FailClosed { reason } => { - Err(Box::new(hls_terminal_failed_closed_response(reason))) - } - } -} - -pub(super) async fn hls_proxy_manifest( - fingerprint: Fingerprint, - axum::extract::Path(params): axum::extract::Path, - axum::extract::State(app_state): axum::extract::State>, - headers: HeaderMap, -) -> axum::response::Response { - let proxy_session_id = ProxySessionId(params.proxy_session_id); - let access_lease_id = HlsAccessLeaseId(params.hls_access_lease_id); - let now_ms = current_time_millis(); - let access_lease_snapshot = - app_state.hls.proxy.access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms).await; - if let Some(lease) = access_lease_snapshot.as_ref() { - let standalone_policy_response_required = lease.playback_mode == HlsLeasePlaybackMode::Ended - && lease - .runtime_policy_denial_reason() - .is_some_and(HlsRuntimeCustomTailReason::permits_unpublished_lease_standalone_tail); - if let (false, Some(response)) = ( - standalone_policy_response_required, - hls_terminal_playback_response(lease, &proxy_session_id, &access_lease_id), - ) { - return response; - } - } - let session = app_state.hls.proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await; - if let Some(session) = session.as_ref() { - app_state - .hls - .proxy - .sync_session_access_lease_count_and_detach_if_needed( - &app_state.active_users, - &app_state.active_provider, - session, - &proxy_session_id, - now_ms, - ) - .await; - } - let (access_context, access_lease_state) = match hls_manifest_access_context_and_state( - &app_state, - &fingerprint, - &proxy_session_id, - &access_lease_id, - access_lease_snapshot.as_ref(), - now_ms, - ) - .await - { - Ok(context_and_state) => context_and_state, - Err(response) => return *response, - }; - let refresh_ordering = if let (Some(session), Some(lease)) = (session.as_ref(), access_lease_snapshot.as_ref()) { - match hls_manifest_preflight_refresh_ordering( - &app_state, - session, - lease, - &proxy_session_id, - &access_lease_id, - now_ms, - ) - .await - { - Ok(ordering) => ordering, - Err(response) => return *response, - } - } else { - HlsManifestRefreshOrdering::Background - }; - let request_context = - match resolve_hls_playback_manifest_request_context(&app_state, &access_context, &headers).await { - Ok(context) => context, - Err(status) => return hls_canonical_status_response(status), - }; - - try_hls_cache_canonical_manifest_response( - &app_state, - &fingerprint, - &access_context, - &proxy_session_id, - &access_lease_id, - access_lease_state, - HlsCacheManifestOrigin { - raw_request_url: &request_context.hls_url, - session_entry_url: request_context.session_entry_url.clone(), - input: &request_context.input, - origin_source: request_context.origin_source, - }, - request_context.headers, - request_context.server_path.as_deref(), - &request_context.original_hls_entry_path, - refresh_ordering, - ) - .await - .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()) -} +mod access; +mod admission; +mod cached; +mod canonical_owner; +mod provisioning; +mod publication; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use access::{ + hls_cache_configured, hls_cache_enabled_for_target, hls_cache_enabled_for_user, + hls_manifest_preflight_refresh_ordering, hls_proxy_manifest, mark_hls_authorized_manifest_access, + mark_hls_authorized_media_access, mark_successful_canonical_manifest_activity, + resolve_hls_playback_manifest_request_context, HlsAccessManifestRequestContext, +}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use admission::{ + apply_hls_user_agent_stream_index, hls_initial_manifest_decision_wait_timeout, hls_initial_manifest_wait_timeout, + hls_manifest_wait_timeout_for_requirement, prepare_hls_canonical_manifest_origin_runtime, + touch_initial_manifest_access_lease_window, try_hls_cache_canonical_manifest_response, +}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use cached::{ + hls_bandwidth_persistence_outcome, hls_cached_manifest_temporarily_unavailable, + observe_hls_lease_manifest_snapshot_derivation, read_hls_cached_manifest, spawn_hls_runtime_bandwidth_persistence, + try_hls_cached_manifest_response, wait_for_hls_startup_evidence, HlsCachedManifestRead, + HlsCachedManifestViewContext, HlsRuntimeBandwidthLearningContext, +}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use canonical_owner::{ + evaluate_hls_canonical_owner_handoff, finalize_hls_canonical_owner_handoff, + hls_availability_reevaluation_registration_failure_response, hls_canonical_manifest_path, + hls_canonical_owner_failed, hls_canonical_owner_lease_deadline_ms, hls_canonical_owner_registration, + hls_canonical_owner_request_deadline_ms, hls_canonical_retry_after_response, hls_canonical_status_response, + hls_direct_refresh_follow_up, join_hls_canonical_manifest_owner, trigger_hls_canonical_manifest_refresh, + HlsCanonicalOwnerEvaluation, HlsCanonicalOwnerFailureReason, HlsCanonicalOwnerHandoffContext, + HlsCanonicalOwnerPending, HlsCanonicalOwnerRegistration, HlsCanonicalOwnerRegistrationFailure, + HlsCanonicalOwnerRegistrationKind, HlsCanonicalOwnerResolution, HlsManifestRefreshOrdering, +}; +pub(in crate::api) use provisioning::hls_panel_provisioning_poll_manifest_response; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use provisioning::{ + clear_hls_provisioning_handoff_consumer, commit_shared_hls_provisioning_segments, + ensure_shared_hls_provisioning_handoff_gap, hls_panel_provisioning_or_status_response, + hls_panel_provisioning_poll_response, hls_provider_connections_exhausted_manifest_resolution, + hls_shared_provisioning_or_provider_exhausted_response, hls_shared_provisioning_timeline_manifest_response, + latest_shared_hls_manifest_rendered_at_ms, mark_hls_provisioning_handoff_discontinuity, + mark_hls_provisioning_handoff_discontinuity_for_session, + mark_hls_provisioning_handoff_discontinuity_once_for_session, + maybe_mark_hls_provisioning_handoff_for_canonical_manifest, shared_hls_provisioning_segment_entry, + shared_hls_provisioning_segment_plans, try_reserve_hls_entry_origin_account_for_redirect, + try_reserve_hls_virtual_entry_origin_account_for_redirect, HlsEntryOriginAccountReservation, + HlsProviderExhaustedResolution, HlsProvisioningPollResponseKind, SharedHlsProvisioningLocalSegmentKind, + SharedHlsProvisioningSegmentPlan, +}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use publication::{ + apply_hls_proxy_public_path_prefix, hls_access_manifest_uses_startup_view, + hls_custom_video_type_for_failure_reason, hls_entry_master_playlist_response, + hls_initial_strip_publication_diagnostic, materialize_hls_access_manifest, materialize_shared_hls_access_manifest, + normalize_hls_proxy_public_path_prefix, split_hls_line_ending, HlsInitialStripLeaseSkipReason, + HlsInitialStripPublicationDiagnostic, HlsInitialStripPublicationStatus, HlsMaterializedSharedManifest, + HlsProxyTerminalSegmentPathParams, +}; +pub(crate) use publication::{hls_admission_failure_manifest_response, hls_custom_video_manifest_response}; diff --git a/backend/app/src/api/endpoints/hls_api/manifest/access.rs b/backend/app/src/api/endpoints/hls_api/manifest/access.rs new file mode 100644 index 000000000..c82372d2a --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/manifest/access.rs @@ -0,0 +1,275 @@ +use super::{ + build_hls_manifest_request_headers, build_hls_origin_resolution, build_hls_origin_source_for_playback, + build_virtual_hls_entry_path, get_stream_channel, hls_canonical_status_response, + hls_manifest_access_context_and_state, hls_manifest_terminal_preflight, hls_terminal_failed_closed_response, + hls_terminal_playback_response, resolve_hls_origin_playlist_url, resolve_hls_terminal_manifest_state, + try_hls_cache_canonical_manifest_response, HlsCacheManifestOrigin, HlsManifestRefreshOrdering, + HlsManifestTerminalPreflight, HlsOriginEntryUrl, HlsProxyManifestPathParams, +}; +use crate::{ + api::{api_utils::is_hls_stream_share_enabled, model::AppState}, + auth::Fingerprint, + model::{ConfigInput, ConfigTarget, ProxyUserCredentials}, +}; +use axum::{ + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; +use shared::model::VirtualId; +use std::sync::Arc; +use tuliprox_core::utils::current_time_millis; +use tuliprox_hls::api::{ + HlsAccessContext, HlsAccessLease, HlsAccessLeaseId, HlsLeasePlaybackMode, HlsOriginSource, + HlsRuntimeCustomTailReason, HlsSessionHandle, HlsTerminalFailedClosedReason, ProxySessionId, +}; + +pub(in crate::api::endpoints::hls_api) async fn mark_successful_canonical_manifest_activity( + app_state: &Arc, + session: &HlsSessionHandle, + now_ms: u64, +) { + mark_hls_authorized_media_access(app_state, session, now_ms).await; +} + +pub(in crate::api::endpoints::hls_api) async fn mark_hls_authorized_manifest_access( + app_state: &Arc, + session: &HlsSessionHandle, + now_ms: u64, +) { + session.write().await.mark_authorized_manifest_access(now_ms); + app_state.hls.proxy.schedule_session_idle_for_handle(session).await; +} + +pub(in crate::api::endpoints::hls_api) async fn mark_hls_authorized_media_access( + app_state: &Arc, + session: &HlsSessionHandle, + now_ms: u64, +) { + session.write().await.mark_authorized_media_access(now_ms); + app_state.hls.proxy.schedule_session_idle_for_handle(session).await; +} + +pub(in crate::api::endpoints::hls_api) fn hls_cache_configured(app_state: &Arc) -> bool { + let config = app_state.app_config.config.load(); + config.reverse_proxy.as_ref().is_some_and(|reverse_proxy| reverse_proxy.hls_cache.is_some()) +} + +pub(in crate::api::endpoints::hls_api) fn hls_cache_enabled_for_target( + app_state: &Arc, + target: &ConfigTarget, +) -> bool { + hls_cache_configured(app_state) && is_hls_stream_share_enabled(target) +} + +pub(in crate::api::endpoints::hls_api) fn hls_cache_enabled_for_user( + app_state: &Arc, + target: &ConfigTarget, + user: &ProxyUserCredentials, +) -> bool { + // Recording headers can select different origin content from ordinary playback. + !user.is_recording_proxy_user() && hls_cache_enabled_for_target(app_state, target) +} + +pub(in crate::api::endpoints::hls_api) struct HlsAccessManifestRequestContext { + pub(in crate::api::endpoints::hls_api) input: Arc, + pub(in crate::api::endpoints::hls_api) hls_url: String, + pub(in crate::api::endpoints::hls_api) session_entry_url: HlsOriginEntryUrl, + pub(in crate::api::endpoints::hls_api) original_hls_entry_path: String, + pub(in crate::api::endpoints::hls_api) origin_source: HlsOriginSource, + pub(in crate::api::endpoints::hls_api) headers: HeaderMap, + pub(in crate::api::endpoints::hls_api) server_path: Option, +} + +pub(in crate::api::endpoints::hls_api) async fn resolve_hls_playback_manifest_request_context( + app_state: &Arc, + access_context: &HlsAccessContext, + req_headers: &HeaderMap, +) -> Result { + let Some((user, target)) = app_state.app_config.get_target_for_username(&access_context.username) else { + return Err(StatusCode::NOT_FOUND); + }; + if !hls_cache_enabled_for_target(app_state, &target) { + return Err(StatusCode::NOT_FOUND); + } + let Some(input) = app_state.app_config.get_input_by_id(access_context.input_id) else { + return Err(StatusCode::NOT_FOUND); + }; + if app_state + .active_users + .is_user_blocked_for_stream(&user.username, VirtualId::new(access_context.virtual_id)) + .await + { + return Err(StatusCode::FORBIDDEN); + } + let Some(channel) = get_stream_channel(app_state, &target, access_context.virtual_id).await else { + return Err(StatusCode::NOT_FOUND); + }; + let origin_playlist_url = if let Some(archive_url) = access_context.archive_origin_url.as_ref() { + archive_url.clone() + } else { + resolve_hls_origin_playlist_url(app_state, &target, &input, access_context.virtual_id, channel.url.as_ref()) + .await? + }; + let Some(hls_cache_origin) = build_hls_origin_resolution(&input, &origin_playlist_url) else { + return Err(StatusCode::SERVICE_UNAVAILABLE); + }; + let origin_source = build_hls_origin_source_for_playback( + &input, + access_context.stream_ref.clone(), + access_context.epg_reference_ts, + Some(&origin_playlist_url), + ); + let Some(server_info) = app_state.app_config.get_user_server_info(&user) else { + return Err(StatusCode::SERVICE_UNAVAILABLE); + }; + let disabled_headers = app_state.get_disabled_headers(); + let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); + let headers = build_hls_manifest_request_headers( + &input.headers, + req_headers, + disabled_headers.as_ref(), + default_user_agent.as_deref(), + channel.upstream_user_agent.as_deref(), + ); + + let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, access_context.virtual_id); + + Ok(HlsAccessManifestRequestContext { + input, + hls_url: hls_cache_origin.hls_url, + session_entry_url: hls_cache_origin.session_entry_url, + original_hls_entry_path, + origin_source, + headers, + server_path: server_info.path.clone(), + }) +} + +pub(in crate::api::endpoints::hls_api) async fn hls_manifest_preflight_refresh_ordering( + app_state: &Arc, + session: &HlsSessionHandle, + lease: &HlsAccessLease, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + now_ms: u64, +) -> Result> { + match hls_manifest_terminal_preflight(session, lease, now_ms).await { + HlsManifestTerminalPreflight::ServeCommittedPlayback => { + Err(Box::new(hls_terminal_playback_response(lease, proxy_session_id, access_lease_id).unwrap_or_else( + || hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::RuntimeUnavailable), + ))) + } + HlsManifestTerminalPreflight::BootstrapPendingLease => Ok(HlsManifestRefreshOrdering::Background), + HlsManifestTerminalPreflight::RefreshBeforeTerminalEvaluation => { + Ok(HlsManifestRefreshOrdering::AwaitBeforeTerminalEvaluation) + } + HlsManifestTerminalPreflight::EvaluateTerminal => resolve_hls_terminal_manifest_state( + app_state, + session, + proxy_session_id, + access_lease_id, + lease.clone(), + now_ms, + ) + .await + .map(|_| HlsManifestRefreshOrdering::Background), + HlsManifestTerminalPreflight::FailClosed { reason } => { + Err(Box::new(hls_terminal_failed_closed_response(reason))) + } + } +} + +pub(in crate::api::endpoints::hls_api) async fn hls_proxy_manifest( + fingerprint: Fingerprint, + axum::extract::Path(params): axum::extract::Path, + axum::extract::State(app_state): axum::extract::State>, + headers: HeaderMap, +) -> axum::response::Response { + let proxy_session_id = ProxySessionId(params.proxy_session_id); + let access_lease_id = HlsAccessLeaseId(params.hls_access_lease_id); + let now_ms = current_time_millis(); + let access_lease_snapshot = + app_state.hls.proxy.access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms).await; + if let Some(lease) = access_lease_snapshot.as_ref() { + let standalone_policy_response_required = lease.playback_mode == HlsLeasePlaybackMode::Ended + && lease + .runtime_policy_denial_reason() + .is_some_and(HlsRuntimeCustomTailReason::permits_unpublished_lease_standalone_tail); + if let (false, Some(response)) = ( + standalone_policy_response_required, + hls_terminal_playback_response(lease, &proxy_session_id, &access_lease_id), + ) { + return response; + } + } + let session = app_state.hls.proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await; + if let Some(session) = session.as_ref() { + app_state + .hls + .proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + session, + &proxy_session_id, + now_ms, + ) + .await; + } + let (access_context, access_lease_state) = match hls_manifest_access_context_and_state( + &app_state, + &fingerprint, + &proxy_session_id, + &access_lease_id, + access_lease_snapshot.as_ref(), + now_ms, + ) + .await + { + Ok(context_and_state) => context_and_state, + Err(response) => return *response, + }; + let refresh_ordering = if let (Some(session), Some(lease)) = (session.as_ref(), access_lease_snapshot.as_ref()) { + match hls_manifest_preflight_refresh_ordering( + &app_state, + session, + lease, + &proxy_session_id, + &access_lease_id, + now_ms, + ) + .await + { + Ok(ordering) => ordering, + Err(response) => return *response, + } + } else { + HlsManifestRefreshOrdering::Background + }; + let request_context = + match resolve_hls_playback_manifest_request_context(&app_state, &access_context, &headers).await { + Ok(context) => context, + Err(status) => return hls_canonical_status_response(status), + }; + + try_hls_cache_canonical_manifest_response( + &app_state, + &fingerprint, + &access_context, + &proxy_session_id, + &access_lease_id, + access_lease_state, + HlsCacheManifestOrigin { + raw_request_url: &request_context.hls_url, + session_entry_url: request_context.session_entry_url.clone(), + input: &request_context.input, + origin_source: request_context.origin_source, + }, + request_context.headers, + request_context.server_path.as_deref(), + &request_context.original_hls_entry_path, + refresh_ordering, + ) + .await + .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()) +} diff --git a/backend/app/src/api/endpoints/hls_api/manifest/admission.rs b/backend/app/src/api/endpoints/hls_api/manifest/admission.rs new file mode 100644 index 000000000..d9c6554ef --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/manifest/admission.rs @@ -0,0 +1,631 @@ +use super::{ + clear_hls_provisioning_handoff_consumer, duration_to_millis_saturating, + effective_hls_url_failover_provider_for_fetch_url, hls_access_lease_timing_for_session, hls_access_lease_ttl_ms, + hls_availability_reevaluation_registration_failure_response, hls_cache_configured, + hls_canonical_owner_registration, hls_canonical_owner_request_deadline_ms, hls_canonical_retry_after_response, + hls_canonical_status_response, hls_effective_origin_acquire_policy, + hls_origin_account_reservation_ttl_secs_for_session, hls_pending_bootstrap_window_ms, + hls_provider_connections_exhausted_manifest_resolution, hls_runtime_or_standalone_custom_tail_response, + hls_unpublished_lease_channel_unavailable_response, join_hls_canonical_manifest_owner, + latest_shared_hls_manifest_rendered_at_ms, mark_hls_authorized_manifest_access, + maybe_mark_hls_provisioning_handoff_for_canonical_manifest, prepare_hls_origin_runtime, + trigger_hls_canonical_manifest_refresh, try_hls_cached_manifest_response, HlsCacheManifestOrigin, + HlsCanonicalOwnerHandoffContext, HlsCanonicalOwnerRegistration, HlsManifestRefreshOrdering, + HlsOriginRuntimeAcquireError, HlsOriginRuntimeNoAccountReason, HlsOriginWorkKind, HlsProviderExhaustedResolution, + HlsRuntimeBandwidthLearningContext, PreparedHlsOriginRuntime, +}; +use crate::{api::model::AppState, auth::Fingerprint, model::ConfigInputFlags, utils::request}; +use axum::{ + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; +use log::debug; +use shared::utils::sanitize_sensitive_info; +use std::{sync::Arc, time::Duration}; +use tuliprox_core::utils::current_time_millis; +use tuliprox_hls::api::{ + build_proxy_session_id, hls_cached_manifest_options_for_requirement, hls_manifest_acceptance_directive_for_session, + hls_manifest_commit_requirement, register_hls_availability_reevaluation, safe_hls_access_lease_id, + safe_proxy_session_id, safe_session_key, HlsAccessContext, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, + HlsAccessLeaseState, HlsAccessLeaseTouch, HlsAccountBindingProtection, HlsCachedManifestOptions, + HlsManifestAcceptanceDirective, HlsManifestAcceptanceEvaluationOutcome, HlsManifestCommitRequirement, + HlsOriginIoContext, HlsOriginWorkClass, HlsPostRefreshRuntime, HlsRuntimeCustomTailReason, HlsSessionHandle, + HlsSessionStoreOutcome, LiveHlsOriginEntry, OriginRefreshRequest, ProxySessionId, RetryPolicy, +}; + +#[allow(clippy::too_many_arguments)] +pub(in crate::api::endpoints::hls_api) async fn prepare_hls_canonical_manifest_origin_runtime( + app_state: &Arc, + session: &HlsSessionHandle, + context: &HlsAccessContext, + origin: &HlsCacheManifestOrigin<'_>, + path_proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + fingerprint: &Fingerprint, + server_path: Option<&str>, + now_ms: u64, +) -> Result> { + let mut allow_grace_hold = true; + loop { + let origin_policy = hls_effective_origin_acquire_policy(session).await; + match prepare_hls_origin_runtime( + app_state, + session, + origin.input, + origin.raw_request_url, + origin.session_entry_url.as_str(), + path_proxy_session_id, + fingerprint, + origin_policy.connection_kind, + origin_policy.priority, + HlsOriginWorkKind::Manifest, + HlsOriginWorkClass::ManifestInteractive, + now_ms, + ) + .await + { + Ok(prepared) => return Ok(prepared), + Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: HlsOriginRuntimeNoAccountReason::OriginBindingPreempted, + }) => { + return Err(Box::new( + hls_runtime_or_standalone_custom_tail_response( + app_state, + session, + path_proxy_session_id, + access_lease_id, + HlsRuntimeCustomTailReason::LowPriorityPreempted, + StatusCode::SERVICE_UNAVAILABLE, + ) + .await, + )); + } + Err(HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted, + }) => { + let strip = app_state.hls.proxy.strip(); + match hls_provider_connections_exhausted_manifest_resolution( + app_state, + session, + &context.username, + origin.input, + context.virtual_id, + access_lease_id, + access_lease_state, + &strip, + server_path, + allow_grace_hold, + ) + .await + { + HlsProviderExhaustedResolution::RetryAcquire => { + allow_grace_hold = false; + } + HlsProviderExhaustedResolution::Response(response) => return Err(Box::new(response)), + } + } + Err(HlsOriginRuntimeAcquireError::Fatal(status)) => { + return Err(Box::new(hls_canonical_status_response(status))) + } + } + } +} + +pub(in crate::api::endpoints::hls_api) async fn apply_hls_user_agent_stream_index( + session: &HlsSessionHandle, + headers: &mut HeaderMap, + enabled: bool, + active_users: &crate::api::model::ActiveUserManager, +) { + if !enabled { + return; + } + let stream_index = { + let mut session = session.write().await; + if session.user_agent_stream_index.is_none() { + session.user_agent_stream_index = Some(active_users.next_user_agent_stream_index()); + } + session.user_agent_stream_index + }; + if let Some(stream_index) = stream_index { + request::append_user_agent_stream_index(headers, stream_index); + } +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +pub(in crate::api::endpoints::hls_api) async fn try_hls_cache_canonical_manifest_response( + app_state: &Arc, + fingerprint: &Fingerprint, + context: &HlsAccessContext, + path_proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + origin: HlsCacheManifestOrigin<'_>, + mut headers: HeaderMap, + server_path: Option<&str>, + _original_hls_entry_path: &str, + refresh_ordering: HlsManifestRefreshOrdering, +) -> Option { + if !hls_cache_configured(app_state) { + return None; + } + if origin.origin_source.input_id != context.input_id || origin.origin_source.stream_ref != context.stream_ref { + return Some(StatusCode::NOT_FOUND.into_response()); + } + + let session_key = origin.origin_source.session_key(); + let expected_proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + if &expected_proxy_session_id != path_proxy_session_id { + return Some(StatusCode::NOT_FOUND.into_response()); + } + let now_ms = current_time_millis(); + let rewrite_secret = app_state.get_encrypt_secret(); + let (session, session_outcome) = app_state + .hls + .proxy + .get_or_create_session_with_source_and_outcome( + session_key, + origin.origin_source.clone(), + &rewrite_secret, + now_ms, + ) + .await; + if access_lease_state == HlsAccessLeaseState::Activated { + let timing = hls_access_lease_timing_for_session(app_state, &session).await; + match app_state + .hls + .proxy + .touch_manifest_access_lease( + access_lease_id, + path_proxy_session_id, + now_ms, + Some(timing), + None, + hls_access_lease_ttl_ms(app_state), + ) + .await + { + HlsAccessLeaseTouch::Touched { .. } => {} + HlsAccessLeaseTouch::Denied => { + return Some( + hls_runtime_or_standalone_custom_tail_response( + app_state, + &session, + path_proxy_session_id, + access_lease_id, + HlsRuntimeCustomTailReason::UserConnectionsExhausted, + StatusCode::FORBIDDEN, + ) + .await, + ); + } + HlsAccessLeaseTouch::Expired | HlsAccessLeaseTouch::UnknownLease | HlsAccessLeaseTouch::SessionMismatch => { + return Some(StatusCode::NOT_FOUND.into_response()); + } + } + } + app_state + .hls + .proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + path_proxy_session_id, + now_ms, + ) + .await; + let prepared_origin = match prepare_hls_canonical_manifest_origin_runtime( + app_state, + &session, + context, + &origin, + path_proxy_session_id, + access_lease_id, + access_lease_state, + fingerprint, + server_path, + now_ms, + ) + .await + { + Ok(prepared) => prepared, + Err(response) => return Some(*response), + }; + apply_hls_user_agent_stream_index( + &session, + &mut headers, + origin.input.has_flag(ConfigInputFlags::UserAgentStreamIndex), + &app_state.active_users, + ) + .await; + let url_failover_provider = effective_hls_url_failover_provider_for_fetch_url( + &prepared_origin.fetch_url, + prepared_origin.url_failover_provider.clone(), + origin.session_entry_url.url_failover_provider(), + ); + let origin_entry = LiveHlsOriginEntry::parse_with_provider_configs( + &prepared_origin.fetch_url, + url_failover_provider, + prepared_origin.runtime_provider_config.clone(), + )?; + { + let mut session_guard = session.write().await; + if session_guard.is_gc_marked_for_removal() { + return Some(hls_canonical_retry_after_response()); + } + if prepared_origin.origin_account_binding_to_store.is_some() { + session_guard.replace_origin_account_binding(prepared_origin.origin_account_binding_to_store); + } + } + mark_hls_authorized_manifest_access(app_state, &session, now_ms).await; + let selected_account = session.read().await.origin_account_binding.as_ref().map_or_else( + || "".to_string(), + |binding| sanitize_sensitive_info(binding.account_name.as_ref()).to_string(), + ); + debug!( + "HLS origin account selected: proxy_session={} account={}", + safe_proxy_session_id(path_proxy_session_id), + selected_account + ); + let reservation_ttl_secs = hls_origin_account_reservation_ttl_secs_for_session(&session).await; + let previous_manifest_rendered_at_ms = latest_shared_hls_manifest_rendered_at_ms(&session).await; + let handoff_previous_rendered_at_ms = maybe_mark_hls_provisioning_handoff_for_canonical_manifest( + app_state, + &session, + origin.input, + context.virtual_id, + access_lease_id, + now_ms, + ) + .await; + let manifest_commit_requirement = + hls_manifest_commit_requirement(&session, session_outcome, handoff_previous_rendered_at_ms, now_ms).await; + let hls_ctx = app_state.hls_ctx(); + let acceptance_evaluation = + hls_manifest_acceptance_directive_for_session(&hls_ctx, &session, path_proxy_session_id).await; + let (acceptance_directive, availability_reevaluation_owner_key) = match acceptance_evaluation { + HlsManifestAcceptanceEvaluationOutcome::Evaluated(directive) => (directive, None), + HlsManifestAcceptanceEvaluationOutcome::StateContention { owner_key } => { + (HlsManifestAcceptanceDirective::none(), Some(owner_key)) + } + HlsManifestAcceptanceEvaluationOutcome::SessionSuperseded => { + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); + return Some(hls_canonical_retry_after_response()); + } + }; + let manifest_boundary_rendered_at_ms = handoff_previous_rendered_at_ms.unwrap_or(previous_manifest_rendered_at_ms); + let wait_timeout = + hls_manifest_wait_timeout_for_requirement(app_state, &session, manifest_commit_requirement).await; + let cached_manifest_options = hls_cached_manifest_options_for_requirement( + wait_timeout, + manifest_commit_requirement, + manifest_boundary_rendered_at_ms, + ); + let bandwidth_learning = match context.known_bitrate_bps { + Some(_) => HlsRuntimeBandwidthLearningContext::Disabled, + None => HlsRuntimeBandwidthLearningContext::Eligible(origin.input), + }; + + let origin_policy = hls_effective_origin_acquire_policy(&session).await; + let origin_provider_session_headers = session.read().await.origin_provider_session_headers.clone(); + let mut preacquired_provider_handle = prepared_origin.preacquired_origin_account_handle; + let mut origin_io = HlsOriginIoContext { + ctx: hls_ctx.clone(), + client_addr: fingerprint.addr, + allow_grace: HlsOriginWorkClass::ManifestInteractive.allows_grace(), + priority: origin_policy.priority, + connection_kind: origin_policy.connection_kind, + reservation_ttl_secs, + preacquired_provider_handle: None, + started_generation: None, + }; + if availability_reevaluation_owner_key.is_none() { + if let Some(provider_handle) = preacquired_provider_handle.take() { + origin_io = origin_io.with_preacquired_provider_handle(provider_handle); + } + } + + let refresh_request = OriginRefreshRequest { + app_config: Arc::clone(&app_state.app_config), + session: Arc::clone(&session), + origin_entry, + headers, + origin_provider_session_headers, + client: app_state.http_clients.default.load().as_ref().clone(), + no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(), + use_manual_redirects: app_state.should_use_manual_redirects(), + segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()), + hls_proxy: Arc::clone(&app_state.hls.proxy), + segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()), + segment_worker_pool: Arc::clone(app_state.hls.proxy.segment_worker_pool()), + map_worker_pool: Arc::clone(app_state.hls.proxy.map_worker_pool()), + origin_manifest_timeout_ms: app_state.hls.proxy.origin_manifest_timeout_ms(), + manifest_recovery_burst: app_state.hls.proxy.manifest_recovery_burst(), + strip: app_state.hls.proxy.strip().clone(), + retry_policy: RetryPolicy::default(), + reverse_proxy_rewrite_secret: rewrite_secret.to_vec(), + transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(), + manifest_commit_requirement, + fresh_manifest_requirement_generation: None, + acceptance_directive, + access_lease_id: Some(access_lease_id.clone()), + disabled_headers: app_state.get_disabled_headers(), + now_ms, + origin_io: Some(origin_io), + post_refresh_runtime: Some(HlsPostRefreshRuntime { ctx: hls_ctx.downgrade() }), + }; + let refresh_ordering = if session_outcome == HlsSessionStoreOutcome::Reused { + refresh_ordering + } else { + HlsManifestRefreshOrdering::Background + }; + if let Some(owner_key) = availability_reevaluation_owner_key { + app_state.connection_manager.release_provider_handle(preacquired_provider_handle); + touch_initial_manifest_access_lease_window( + app_state, + access_lease_id, + path_proxy_session_id, + access_lease_state, + wait_timeout, + now_ms, + ) + .await; + let owner_wait_lease = app_state + .hls + .proxy + .access_lease_response_snapshot(access_lease_id, path_proxy_session_id, current_time_millis()) + .await; + let expected_lease_issued_at_ms = owner_wait_lease.as_ref().map(|lease| lease.issued_at_ms); + let request_deadline_ms = owner_wait_lease + .as_ref() + .map_or(now_ms, |lease| hls_canonical_owner_request_deadline_ms(lease, wait_timeout, now_ms)); + let safe_session = { + let session = session.read().await; + safe_session_key(&session.key) + }; + let registration = + register_hls_availability_reevaluation(hls_ctx, Arc::clone(&session), owner_key, refresh_request); + return Some(match hls_canonical_owner_registration(registration) { + HlsCanonicalOwnerRegistration::Join(registration) => { + let strip = app_state.hls.proxy.strip(); + join_hls_canonical_manifest_owner( + HlsCanonicalOwnerHandoffContext { + app_state, + proxy_session_id: path_proxy_session_id, + access_lease_id, + expected_lease_issued_at_ms, + strip: &strip, + server_path, + manifest_commit_requirement, + manifest_boundary_rendered_at_ms, + bandwidth_learning, + request_deadline_ms, + safe_session, + }, + registration, + ) + .await + } + HlsCanonicalOwnerRegistration::FailClosed(failure) => { + hls_availability_reevaluation_registration_failure_response(failure) + } + }); + } + if handoff_previous_rendered_at_ms.is_some() { + touch_initial_manifest_access_lease_window( + app_state, + access_lease_id, + path_proxy_session_id, + access_lease_state, + wait_timeout, + now_ms, + ) + .await; + if let Some(response) = trigger_hls_canonical_manifest_refresh( + app_state, + &session, + path_proxy_session_id, + access_lease_id, + refresh_request, + refresh_ordering, + ) + .await + { + return Some(response); + } + let strip = app_state.hls.proxy.strip(); + if let Some(response) = try_hls_cached_manifest_response( + app_state, + &session, + access_lease_id, + access_lease_state, + &strip, + server_path, + cached_manifest_options, + bandwidth_learning, + ) + .await + { + clear_hls_provisioning_handoff_consumer(app_state, origin.input, context.virtual_id, current_time_millis()); + return Some(response); + } + return Some(StatusCode::SERVICE_UNAVAILABLE.into_response()); + } + match session_outcome { + HlsSessionStoreOutcome::Created => { + touch_initial_manifest_access_lease_window( + app_state, + access_lease_id, + path_proxy_session_id, + access_lease_state, + wait_timeout, + now_ms, + ) + .await; + if let Some(response) = trigger_hls_canonical_manifest_refresh( + app_state, + &session, + path_proxy_session_id, + access_lease_id, + refresh_request, + refresh_ordering, + ) + .await + { + return Some(response); + } + let strip = app_state.hls.proxy.strip(); + if let Some(response) = try_hls_cached_manifest_response( + app_state, + &session, + access_lease_id, + access_lease_state, + &strip, + server_path, + cached_manifest_options, + bandwidth_learning, + ) + .await + { + return Some(response); + } + } + HlsSessionStoreOutcome::Reused => { + if let Some(response) = trigger_hls_canonical_manifest_refresh( + app_state, + &session, + path_proxy_session_id, + access_lease_id, + refresh_request, + refresh_ordering, + ) + .await + { + if refresh_ordering == HlsManifestRefreshOrdering::AwaitBeforeTerminalEvaluation + && response.status() == StatusCode::SERVICE_UNAVAILABLE + { + let strip = app_state.hls.proxy.strip(); + if let Some(live_response) = try_hls_cached_manifest_response( + app_state, + &session, + access_lease_id, + access_lease_state, + &strip, + server_path, + HlsCachedManifestOptions::initial(Duration::ZERO), + bandwidth_learning, + ) + .await + .filter(|candidate| candidate.status() == StatusCode::OK) + { + return Some(live_response); + } + } + return Some(response); + } + touch_initial_manifest_access_lease_window( + app_state, + access_lease_id, + path_proxy_session_id, + access_lease_state, + wait_timeout, + now_ms, + ) + .await; + let strip = app_state.hls.proxy.strip(); + if let Some(response) = try_hls_cached_manifest_response( + app_state, + &session, + access_lease_id, + access_lease_state, + &strip, + server_path, + cached_manifest_options, + bandwidth_learning, + ) + .await + { + return Some(response); + } + } + } + + Some(hls_unpublished_lease_channel_unavailable_response(app_state, path_proxy_session_id, access_lease_id).await) +} + +pub(in crate::api::endpoints::hls_api) fn hls_initial_manifest_decision_wait_timeout( + app_state: &Arc, +) -> Duration { + Duration::from_secs(app_state.hls.proxy.initial_manifest_wait_timeout_secs()) +} + +pub(in crate::api::endpoints::hls_api) async fn hls_manifest_wait_timeout_for_requirement( + app_state: &Arc, + session: &HlsSessionHandle, + requirement: HlsManifestCommitRequirement, +) -> Duration { + match requirement { + HlsManifestCommitRequirement::FreshCommitRequired { .. } => { + hls_initial_manifest_decision_wait_timeout(app_state) + } + HlsManifestCommitRequirement::CommittedManifestAllowed => { + hls_initial_manifest_wait_timeout(app_state, session).await + } + } +} + +pub(in crate::api::endpoints::hls_api) async fn touch_initial_manifest_access_lease_window( + app_state: &Arc, + access_lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + access_lease_state: HlsAccessLeaseState, + wait_timeout: Duration, + now_ms: u64, +) { + if wait_timeout.is_zero() || access_lease_state != HlsAccessLeaseState::Pending { + return; + } + let wait_timeout_ms = duration_to_millis_saturating(wait_timeout); + let deadline_ms = now_ms.saturating_add(wait_timeout_ms.max(hls_pending_bootstrap_window_ms(app_state))); + let touch = app_state + .hls + .proxy + .touch_manifest_access_lease( + access_lease_id, + proxy_session_id, + now_ms, + None, + Some(HlsAccessLeasePendingDeadline::Bootstrap { deadline_ms }), + hls_access_lease_ttl_ms(app_state), + ) + .await; + let failure = match touch { + HlsAccessLeaseTouch::Touched { .. } => return, + HlsAccessLeaseTouch::Expired => "expired", + HlsAccessLeaseTouch::Denied => "denied", + HlsAccessLeaseTouch::UnknownLease => "unknown-lease", + HlsAccessLeaseTouch::SessionMismatch => "session-mismatch", + }; + debug!( + "HLS initial manifest lease window not extended: lease={} proxy_session={} outcome={failure}", + safe_hls_access_lease_id(access_lease_id), + safe_proxy_session_id(proxy_session_id) + ); +} + +pub(in crate::api::endpoints::hls_api) async fn hls_initial_manifest_wait_timeout( + app_state: &Arc, + session: &HlsSessionHandle, +) -> Duration { + let session = session.read().await; + if matches!( + session.account_binding_protection(current_time_millis()), + HlsAccountBindingProtection::NoMediaYet | HlsAccountBindingProtection::Expired + ) { + hls_initial_manifest_decision_wait_timeout(app_state) + } else { + Duration::ZERO + } +} diff --git a/backend/app/src/api/endpoints/hls_api/manifest/cached.rs b/backend/app/src/api/endpoints/hls_api/manifest/cached.rs new file mode 100644 index 000000000..0b7e2a3dc --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/manifest/cached.rs @@ -0,0 +1,428 @@ +use super::{ + hls_access_manifest_uses_startup_view, hls_initial_strip_publication_diagnostic, hls_response, + hls_temporary_resource_unavailable_response, log_hls_initial_strip_publication, + mark_successful_canonical_manifest_activity, materialize_shared_hls_access_manifest, + normalize_hls_proxy_public_path_prefix, stripped_tail_segments, touch_pending_manifest_follow_up_window, + HlsCachedTransientManifestRead, HlsInitialStripPublicationDiagnostic, HlsInitialStripPublicationStatus, + HlsMaterializedSharedManifest, HLS_MANIFEST_WAIT_POLL_INTERVAL, HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, +}; +use crate::{ + api::model::AppState, + model::ConfigInput, + processing::parser::hls::origin_manifest::HlsManifestWindowPolicy, + repository::{persist_input_live_bitrate_bps, LiveBitratePersistenceOutcome}, +}; +use axum::response::IntoResponse; +use log::{debug, error, warn}; +use shared::utils::sanitize_sensitive_info; +use std::{sync::Arc, time::Duration}; +use tuliprox_core::utils::current_time_millis; +use tuliprox_hls::api::{ + derive_hls_lease_manifest_snapshot, hls_committed_manifest_body_for_request, + hls_should_wait_for_initial_manifest_commit, hls_startup_admission_allows_snapshot, safe_hls_access_lease_id, + safe_proxy_session_id, HlsAccessLeaseId, HlsAccessLeaseState, HlsBandwidthPersistenceOutcome, + HlsCachedManifestOptions, HlsCommittedManifestBody, HlsLeaseManifestSnapshot, HlsLeaseManifestSnapshotInput, + HlsLeaseManifestUriMaterialization, HlsManifestLimitViolation, HlsPublishedTransientResourceIds, HlsSessionHandle, + ProxySessionId, +}; + +pub(in crate::api::endpoints::hls_api) struct HlsCachedManifestRead { + pub(in crate::api::endpoints::hls_api) transient_body: Option, + pub(in crate::api::endpoints::hls_api) rendered_body: Option, + pub(in crate::api::endpoints::hls_api) should_wait: bool, + pub(in crate::api::endpoints::hls_api) wait_for_initial_commit: bool, +} + +pub(in crate::api::endpoints::hls_api) async fn read_hls_cached_manifest( + session: &HlsSessionHandle, + options: HlsCachedManifestOptions, + started_at_ms: u64, +) -> HlsCachedManifestRead { + let session = session.read().await; + let now_ms = current_time_millis(); + let should_wait = session.initial_manifest_commit_work_pending(); + let committed_body = hls_committed_manifest_body_for_request(&session, options, started_at_ms, now_ms); + let (transient_body, rendered_body) = match committed_body { + Some(HlsCommittedManifestBody::Transient(body)) => ( + session.transient.last_manifest_template().zip(session.transient.last_manifest_commit_identity()).map( + |(template, source_commit_identity)| HlsCachedTransientManifestRead { + body, + template, + source_commit_identity, + window_policy: session.transient.last_manifest_window_policy(), + finalized_manifest_generation: session.transient.current_finalized_manifest_generation(), + published_resource_ids: session.transient.last_manifest_published_resource_ids(), + }, + ), + None, + ), + Some(HlsCommittedManifestBody::Normal(body)) => (None, Some(body)), + None => (None, None), + }; + let wait_for_initial_commit = hls_should_wait_for_initial_manifest_commit( + &session, + transient_body.is_some() || rendered_body.is_some(), + should_wait, + options, + now_ms, + ); + HlsCachedManifestRead { transient_body, rendered_body, should_wait, wait_for_initial_commit } +} + +pub(in crate::api::endpoints::hls_api) struct HlsCachedManifestViewContext<'a> { + pub(in crate::api::endpoints::hls_api) proxy_session_id: &'a ProxySessionId, + pub(in crate::api::endpoints::hls_api) access_lease_id: &'a HlsAccessLeaseId, + pub(in crate::api::endpoints::hls_api) access_lease_state: HlsAccessLeaseState, + pub(in crate::api::endpoints::hls_api) strip: &'a crate::model::StripConfig, + pub(in crate::api::endpoints::hls_api) server_path: Option<&'a str>, + pub(in crate::api::endpoints::hls_api) bandwidth_learning: HlsRuntimeBandwidthLearningContext<'a>, +} + +#[derive(Clone, Copy)] +pub(in crate::api::endpoints::hls_api) enum HlsRuntimeBandwidthLearningContext<'a> { + Disabled, + Eligible(&'a ConfigInput), +} + +impl HlsCachedManifestViewContext<'_> { + pub(in crate::api::endpoints::hls_api) fn new<'a>( + proxy_session_id: &'a ProxySessionId, + access_lease_id: &'a HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + strip: &'a crate::model::StripConfig, + server_path: Option<&'a str>, + bandwidth_learning: HlsRuntimeBandwidthLearningContext<'a>, + ) -> HlsCachedManifestViewContext<'a> { + HlsCachedManifestViewContext { + proxy_session_id, + access_lease_id, + access_lease_state, + strip, + server_path, + bandwidth_learning, + } + } + + pub(in crate::api::endpoints::hls_api) fn materialize( + &self, + body: &str, + mode: &'static str, + window_policy: HlsManifestWindowPolicy, + ) -> HlsMaterializedSharedManifest { + materialize_shared_hls_access_manifest( + body, + self.access_lease_id, + self.access_lease_state, + self.strip, + window_policy, + mode, + self.server_path, + ) + } + + pub(in crate::api::endpoints::hls_api) async fn finish( + &self, + app_state: &Arc, + session: &HlsSessionHandle, + materialized: HlsMaterializedSharedManifest, + strip_diagnostic: HlsInitialStripPublicationDiagnostic, + ) -> axum::response::Response { + touch_pending_manifest_follow_up_window(app_state, session, self.access_lease_id, self.access_lease_state) + .await; + drop(spawn_hls_runtime_bandwidth_persistence(app_state, session, self.bandwidth_learning)); + mark_successful_canonical_manifest_activity(app_state, session, current_time_millis()).await; + log_hls_initial_strip_publication(self.proxy_session_id, self.access_lease_id, strip_diagnostic); + hls_response(materialized.body).into_response() + } +} + +pub(in crate::api::endpoints::hls_api) fn spawn_hls_runtime_bandwidth_persistence( + app_state: &Arc, + session: &HlsSessionHandle, + context: HlsRuntimeBandwidthLearningContext<'_>, +) -> Option> { + let input = match context { + HlsRuntimeBandwidthLearningContext::Disabled => return None, + HlsRuntimeBandwidthLearningContext::Eligible(input) => input.clone(), + }; + let (bitrate_bps, proxy_session_id, stream_ref) = { + let Ok(mut session_guard) = session.try_write() else { + return None; + }; + let bitrate_bps = session_guard.begin_bandwidth_persistence(current_time_millis())?; + (bitrate_bps, session_guard.proxy_session_id.clone(), session_guard.origin_source.stream_ref.clone()) + }; + let app_config = Arc::clone(&app_state.app_config); + let hls_proxy = Arc::clone(&app_state.hls.proxy); + let session = Arc::clone(session); + + Some(tokio::spawn(async move { + let outcome = match persist_input_live_bitrate_bps(&app_config, &input, &stream_ref, bitrate_bps).await { + Ok(repository_outcome) => hls_bandwidth_persistence_outcome(repository_outcome, &proxy_session_id), + Err(err) => { + error!( + "HLS runtime bandwidth persistence failed: proxy_session={} error={}", + safe_proxy_session_id(&proxy_session_id), + sanitize_sensitive_info(&err.to_string()) + ); + HlsBandwidthPersistenceOutcome::RetryAfter + } + }; + let Some(current_session) = hls_proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await else { + return; + }; + if !Arc::ptr_eq(¤t_session, &session) { + return; + } + current_session.write().await.finish_bandwidth_persistence(bitrate_bps, outcome, current_time_millis()); + })) +} + +pub(in crate::api::endpoints::hls_api) fn hls_bandwidth_persistence_outcome( + repository_outcome: LiveBitratePersistenceOutcome, + proxy_session_id: &ProxySessionId, +) -> HlsBandwidthPersistenceOutcome { + match repository_outcome { + LiveBitratePersistenceOutcome::Updated | LiveBitratePersistenceOutcome::AlreadyEqualOrHigher => { + HlsBandwidthPersistenceOutcome::Persisted + } + LiveBitratePersistenceOutcome::MissingDatabase => { + debug!( + "HLS runtime bandwidth persistence deferred: proxy_session={} reason=missing_database", + safe_proxy_session_id(proxy_session_id) + ); + HlsBandwidthPersistenceOutcome::RetryAfter + } + LiveBitratePersistenceOutcome::MissingStreamItem => { + debug!( + "HLS runtime bandwidth persistence deferred: proxy_session={} reason=missing_stream_item", + safe_proxy_session_id(proxy_session_id) + ); + HlsBandwidthPersistenceOutcome::RetryAfter + } + LiveBitratePersistenceOutcome::PermanentlyInapplicable(reason) => { + debug!( + "HLS runtime bandwidth persistence skipped: proxy_session={} reason={}", + safe_proxy_session_id(proxy_session_id), + reason.log_label() + ); + HlsBandwidthPersistenceOutcome::PermanentlyInapplicable + } + } +} + +pub(in crate::api::endpoints::hls_api) fn hls_cached_manifest_temporarily_unavailable() -> axum::response::Response { + hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS) +} + +pub(in crate::api::endpoints::hls_api) fn observe_hls_lease_manifest_snapshot_derivation( + app_state: &AppState, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + derivation: Result, HlsManifestLimitViolation>, +) -> Result, ()> { + match derivation { + Ok(snapshot) => { + if let Some(snapshot) = snapshot.as_ref() { + app_state.hls.proxy.metrics().record_lease_snapshot_segments(snapshot.visible_segments.len()); + } + Ok(snapshot) + } + Err(violation) => { + app_state.hls.proxy.metrics().record_manifest_limit_rejection(); + warn!( + "HLS lease manifest snapshot rejected: proxy_session={} lease={} reason=manifest-representation-limit kind={} actual={} limit={}", + safe_proxy_session_id(proxy_session_id), + safe_hls_access_lease_id(access_lease_id), + violation.kind.as_log_value(), + violation.actual, + violation.limit + ); + Err(()) + } + } +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +pub(in crate::api::endpoints::hls_api) async fn try_hls_cached_manifest_response( + app_state: &Arc, + session: &HlsSessionHandle, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + strip: &crate::model::StripConfig, + server_path: Option<&str>, + options: HlsCachedManifestOptions, + bandwidth_learning: HlsRuntimeBandwidthLearningContext<'_>, +) -> Option { + let started_at = tokio::time::Instant::now(); + let started_at_ms = current_time_millis(); + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let Some(publication_guard) = app_state + .hls + .proxy + .prepare_access_lease_manifest_publication(access_lease_id, &proxy_session_id, started_at_ms) + .await + else { + return Some(hls_cached_manifest_temporarily_unavailable()); + }; + let view = HlsCachedManifestViewContext::new( + &proxy_session_id, + access_lease_id, + access_lease_state, + strip, + server_path, + bandwidth_learning, + ); + loop { + let cached = read_hls_cached_manifest(session, options, started_at_ms).await; + if !cached.wait_for_initial_commit { + let prepared = if let Some(transient) = cached.transient_body { + let materialized = view.materialize(&transient.body, "transient", transient.window_policy); + let published_resource_ids = if transient.window_policy.preserves_full_manifest() { + transient.published_resource_ids.clone() + } else { + HlsPublishedTransientResourceIds::from_manifest_body(&materialized.body) + }; + let delivered_at_ms = current_time_millis(); + let snapshot_input = if transient.window_policy.preserves_full_manifest() { + HlsLeaseManifestSnapshotInput::TransientPassthroughTemplate { + template: &transient.template, + source_commit_identity: transient.source_commit_identity, + uri_materialization: HlsLeaseManifestUriMaterialization::new( + access_lease_id, + normalize_hls_proxy_public_path_prefix(server_path).map(Arc::from), + ), + finalized_manifest_generation: transient.finalized_manifest_generation, + } + } else { + HlsLeaseManifestSnapshotInput::TransientPassthrough { + materialized_body: &materialized.body, + source_commit_identity: transient.source_commit_identity, + finalized_manifest_generation: transient.finalized_manifest_generation, + } + }; + let snapshot = observe_hls_lease_manifest_snapshot_derivation( + app_state, + &proxy_session_id, + access_lease_id, + derive_hls_lease_manifest_snapshot(&snapshot_input, delivered_at_ms), + ); + let Ok(snapshot) = snapshot else { + return Some(hls_cached_manifest_temporarily_unavailable()); + }; + let Some(snapshot) = snapshot else { + return Some(hls_cached_manifest_temporarily_unavailable()); + }; + Some((materialized, snapshot, published_resource_ids, delivered_at_ms)) + } else if let Some(body) = cached.rendered_body { + let materialized = view.materialize(&body, "normal", HlsManifestWindowPolicy::ApplyLiveWindow); + let published_resource_ids = HlsPublishedTransientResourceIds::from_manifest_body(&materialized.body); + let delivered_at_ms = current_time_millis(); + let snapshot = { + let session = session.read().await; + observe_hls_lease_manifest_snapshot_derivation( + app_state, + &proxy_session_id, + access_lease_id, + derive_hls_lease_manifest_snapshot( + &HlsLeaseManifestSnapshotInput::NormalCacheTimeline { + session: &session, + committed_body: &body, + materialized_body: &materialized.body, + stripped_tail_segments: stripped_tail_segments(&materialized), + }, + delivered_at_ms, + ), + ) + }; + let Ok(snapshot) = snapshot else { + return Some(hls_cached_manifest_temporarily_unavailable()); + }; + let Some(snapshot) = snapshot else { + if access_lease_state != HlsAccessLeaseState::Pending { + return None; + } + if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await { + continue; + } + return Some(hls_cached_manifest_temporarily_unavailable()); + }; + Some((materialized, snapshot, published_resource_ids, delivered_at_ms)) + } else { + None + }; + if let Some((materialized, snapshot, published_resource_ids, delivered_at_ms)) = prepared { + if access_lease_state == HlsAccessLeaseState::Pending + && !hls_startup_admission_allows_snapshot(&app_state.hls_ctx(), session, &snapshot, delivered_at_ms) + .await + { + if wait_for_hls_startup_evidence(started_at, options.wait_timeout).await { + continue; + } + return Some(hls_cached_manifest_temporarily_unavailable()); + } + let startup_snapshot = snapshot.clone(); + let admission_at_ms = current_time_millis(); + let outcome = app_state + .hls + .proxy + .commit_access_lease_manifest_publication_with_resources( + access_lease_id, + &proxy_session_id, + publication_guard, + snapshot, + published_resource_ids, + admission_at_ms, + ) + .await; + if let Some(snapshot_generation) = outcome.snapshot_generation() { + let published_at_ms = current_time_millis(); + let first_startup_publication = + app_state.hls.proxy.startup_observability().record_manifest_publication( + access_lease_id, + snapshot_generation, + admission_at_ms, + published_at_ms, + Arc::from(startup_snapshot.visible_proxy_seqs().collect::>()), + ); + if first_startup_publication && hls_access_manifest_uses_startup_view(access_lease_state) { + app_state.hls.proxy.spawn_access_lease_repair_prewarm( + Arc::clone(session), + access_lease_id.clone(), + startup_snapshot, + snapshot_generation, + ); + } + } + let publication_status = if outcome.is_committed() { + HlsInitialStripPublicationStatus::Committed + } else { + HlsInitialStripPublicationStatus::NotCommitted + }; + let Some(strip_diagnostic) = + hls_initial_strip_publication_diagnostic(publication_status, access_lease_state, &materialized) + else { + return Some(hls_cached_manifest_temporarily_unavailable()); + }; + return Some(view.finish(app_state, session, materialized, strip_diagnostic).await); + } + } + if options.wait_timeout.is_zero() || !cached.should_wait || started_at.elapsed() >= options.wait_timeout { + return None; + } + let remaining = options.wait_timeout.saturating_sub(started_at.elapsed()); + tokio::time::sleep(remaining.min(HLS_MANIFEST_WAIT_POLL_INTERVAL)).await; + } +} + +pub(in crate::api::endpoints::hls_api) async fn wait_for_hls_startup_evidence( + started_at: tokio::time::Instant, + wait_timeout: Duration, +) -> bool { + let elapsed = started_at.elapsed(); + if wait_timeout.is_zero() || elapsed >= wait_timeout { + return false; + } + tokio::time::sleep(wait_timeout.saturating_sub(elapsed).min(Duration::from_millis(25))).await; + true +} diff --git a/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs b/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs new file mode 100644 index 000000000..f303f27db --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/manifest/canonical_owner.rs @@ -0,0 +1,423 @@ +use super::{ + duration_to_millis_saturating, hls_terminal_failed_closed_response, hls_terminal_playback_response, + hls_unpublished_lease_channel_unavailable_response, resolve_hls_terminal_manifest_state, + try_hls_cached_manifest_response, HlsRuntimeBandwidthLearningContext, +}; +use crate::api::{api_utils::try_unwrap_body, model::AppState}; +use axum::{http::StatusCode, response::IntoResponse}; +use log::{debug, warn}; +use std::{sync::Arc, time::Duration}; +use tuliprox_core::utils::current_time_millis; +use tuliprox_hls::api::{ + cold_start_retry_after_seconds, hls_cached_manifest_options_for_requirement, + maybe_trigger_origin_refresh_with_outcome, register_hls_availability_reevaluation, safe_hls_access_lease_id, + safe_proxy_session_id, trigger_origin_refresh_sync, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, + HlsAvailabilityReevaluationObservation, HlsAvailabilityReevaluationRegistration, HlsLeasePlaybackMode, + HlsManifestCommitRequirement, HlsOriginRefreshTriggerOutcome, HlsSessionHandle, HlsTerminalFailedClosedReason, + OriginRefreshRequest, ProxySessionId, +}; + +pub(in crate::api::endpoints::hls_api) fn hls_canonical_manifest_path( + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, +) -> String { + format!("/hls/shared/live/{}/{}/manifest.m3u8", proxy_session_id.0, access_lease_id.0) +} + +pub(in crate::api::endpoints::hls_api) fn hls_canonical_retry_after_response() -> axum::response::Response { + try_unwrap_body!(axum::response::Response::builder() + .status(axum::http::StatusCode::SERVICE_UNAVAILABLE) + .header(axum::http::header::RETRY_AFTER, cold_start_retry_after_seconds().to_string()) + .body(axum::body::Body::empty())) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(in crate::api::endpoints::hls_api) enum HlsCanonicalOwnerRegistration { + Join(HlsCanonicalOwnerRegistrationKind), + FailClosed(HlsCanonicalOwnerRegistrationFailure), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(in crate::api::endpoints::hls_api) enum HlsCanonicalOwnerRegistrationKind { + Scheduled, + AlreadyOwned, +} + +impl HlsCanonicalOwnerRegistrationKind { + pub(in crate::api::endpoints::hls_api) const fn as_label(self) -> &'static str { + match self { + Self::Scheduled => "scheduled", + Self::AlreadyOwned => "already_owned", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(in crate::api::endpoints::hls_api) enum HlsCanonicalOwnerRegistrationFailure { + CapacityExceeded, + RuntimeUnavailable, +} + +impl HlsCanonicalOwnerRegistrationFailure { + pub(in crate::api::endpoints::hls_api) const fn as_label(self) -> &'static str { + match self { + Self::CapacityExceeded => "capacity_exceeded", + Self::RuntimeUnavailable => "runtime_unavailable", + } + } +} + +pub(in crate::api::endpoints::hls_api) const fn hls_canonical_owner_registration( + registration: HlsAvailabilityReevaluationRegistration, +) -> HlsCanonicalOwnerRegistration { + match registration { + HlsAvailabilityReevaluationRegistration::Scheduled => { + HlsCanonicalOwnerRegistration::Join(HlsCanonicalOwnerRegistrationKind::Scheduled) + } + HlsAvailabilityReevaluationRegistration::AlreadyOwned | HlsAvailabilityReevaluationRegistration::Superseded => { + HlsCanonicalOwnerRegistration::Join(HlsCanonicalOwnerRegistrationKind::AlreadyOwned) + } + HlsAvailabilityReevaluationRegistration::CapacityExceeded => { + HlsCanonicalOwnerRegistration::FailClosed(HlsCanonicalOwnerRegistrationFailure::CapacityExceeded) + } + HlsAvailabilityReevaluationRegistration::RuntimeUnavailable => { + HlsCanonicalOwnerRegistration::FailClosed(HlsCanonicalOwnerRegistrationFailure::RuntimeUnavailable) + } + } +} + +pub(in crate::api::endpoints::hls_api) fn hls_availability_reevaluation_registration_failure_response( + failure: HlsCanonicalOwnerRegistrationFailure, +) -> axum::response::Response { + warn!("HLS availability reevaluation not registered: reason={}", failure.as_label()); + hls_canonical_retry_after_response() +} + +pub(in crate::api::endpoints::hls_api) enum HlsCanonicalOwnerResolution { + Live(axum::response::Response), + Terminal(axum::response::Response), + Standalone(axum::response::Response), + FailedClosed { reason: HlsCanonicalOwnerFailureReason, response: axum::response::Response }, +} + +impl HlsCanonicalOwnerResolution { + pub(in crate::api::endpoints::hls_api) const fn outcome_label(&self) -> &'static str { + match self { + Self::Live(_) => "live", + Self::Terminal(_) => "terminal", + Self::Standalone(_) => "standalone", + Self::FailedClosed { reason, .. } => reason.as_label(), + } + } + + pub(in crate::api::endpoints::hls_api) fn into_response(self) -> axum::response::Response { + match self { + Self::Live(response) + | Self::Terminal(response) + | Self::Standalone(response) + | Self::FailedClosed { response, .. } => response, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(in crate::api::endpoints::hls_api) enum HlsCanonicalOwnerFailureReason { + Superseded, + DeadlineElapsed, + LeaseUnavailable, +} + +impl HlsCanonicalOwnerFailureReason { + pub(in crate::api::endpoints::hls_api) const fn as_label(self) -> &'static str { + match self { + Self::Superseded => "superseded", + Self::DeadlineElapsed => "deadline_elapsed", + Self::LeaseUnavailable => "lease_unavailable", + } + } + + pub(in crate::api::endpoints::hls_api) fn response(self) -> axum::response::Response { + let reason = match self { + Self::DeadlineElapsed => HlsTerminalFailedClosedReason::SafeCommitDeadlineElapsed, + Self::Superseded | Self::LeaseUnavailable => HlsTerminalFailedClosedReason::LeaseStateUnavailable, + }; + hls_terminal_failed_closed_response(reason) + } +} + +pub(in crate::api::endpoints::hls_api) struct HlsCanonicalOwnerPending { + pub(in crate::api::endpoints::hls_api) deadline_ms: u64, + pub(in crate::api::endpoints::hls_api) current_session_available: bool, +} + +pub(in crate::api::endpoints::hls_api) enum HlsCanonicalOwnerEvaluation { + Resolved(HlsCanonicalOwnerResolution), + Pending(HlsCanonicalOwnerPending), +} + +pub(in crate::api::endpoints::hls_api) fn hls_canonical_owner_failed( + reason: HlsCanonicalOwnerFailureReason, +) -> HlsCanonicalOwnerEvaluation { + HlsCanonicalOwnerEvaluation::Resolved(HlsCanonicalOwnerResolution::FailedClosed { + reason, + response: reason.response(), + }) +} + +pub(in crate::api::endpoints::hls_api) struct HlsCanonicalOwnerHandoffContext<'a> { + pub(in crate::api::endpoints::hls_api) app_state: &'a Arc, + pub(in crate::api::endpoints::hls_api) proxy_session_id: &'a ProxySessionId, + pub(in crate::api::endpoints::hls_api) access_lease_id: &'a HlsAccessLeaseId, + pub(in crate::api::endpoints::hls_api) expected_lease_issued_at_ms: Option, + pub(in crate::api::endpoints::hls_api) strip: &'a crate::model::StripConfig, + pub(in crate::api::endpoints::hls_api) server_path: Option<&'a str>, + pub(in crate::api::endpoints::hls_api) manifest_commit_requirement: HlsManifestCommitRequirement, + pub(in crate::api::endpoints::hls_api) manifest_boundary_rendered_at_ms: u64, + pub(in crate::api::endpoints::hls_api) bandwidth_learning: HlsRuntimeBandwidthLearningContext<'a>, + pub(in crate::api::endpoints::hls_api) request_deadline_ms: u64, + pub(in crate::api::endpoints::hls_api) safe_session: String, +} + +pub(in crate::api::endpoints::hls_api) fn hls_canonical_owner_lease_deadline_ms(lease: &HlsAccessLease) -> u64 { + if lease.state == HlsAccessLeaseState::Pending { + lease.pending_deadline_ms().unwrap_or(lease.valid_until_ms) + } else { + lease.valid_until_ms + } +} + +pub(in crate::api::endpoints::hls_api) fn hls_canonical_owner_request_deadline_ms( + lease: &HlsAccessLease, + wait_timeout: Duration, + now_ms: u64, +) -> u64 { + let lease_deadline_ms = hls_canonical_owner_lease_deadline_ms(lease); + if wait_timeout.is_zero() && lease.state == HlsAccessLeaseState::Pending { + lease_deadline_ms + } else { + lease_deadline_ms.min(now_ms.saturating_add(duration_to_millis_saturating(wait_timeout))) + } +} + +pub(in crate::api::endpoints::hls_api) async fn evaluate_hls_canonical_owner_handoff( + context: &HlsCanonicalOwnerHandoffContext<'_>, +) -> HlsCanonicalOwnerEvaluation { + let now_ms = current_time_millis(); + let Some(lease) = context + .app_state + .hls + .proxy + .access_lease_response_snapshot(context.access_lease_id, context.proxy_session_id, now_ms) + .await + else { + return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable); + }; + if context.expected_lease_issued_at_ms != Some(lease.issued_at_ms) { + return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable); + } + match &lease.playback_mode { + HlsLeasePlaybackMode::TerminalTail(_) => { + let Some(response) = + hls_terminal_playback_response(&lease, context.proxy_session_id, context.access_lease_id) + else { + return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable); + }; + return HlsCanonicalOwnerEvaluation::Resolved(HlsCanonicalOwnerResolution::Terminal(response)); + } + HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => { + return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable); + } + HlsLeasePlaybackMode::Live => {} + } + if matches!( + lease.state, + HlsAccessLeaseState::PolicyRevoking | HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied + ) { + return hls_canonical_owner_failed(HlsCanonicalOwnerFailureReason::LeaseUnavailable); + } + + let current_session = + context.app_state.hls.proxy.sessions().get_by_proxy_session_id(context.proxy_session_id).await; + if let Some(current_session) = current_session.as_ref() { + let options = hls_cached_manifest_options_for_requirement( + Duration::ZERO, + context.manifest_commit_requirement, + context.manifest_boundary_rendered_at_ms, + ); + if let Some(response) = try_hls_cached_manifest_response( + context.app_state, + current_session, + context.access_lease_id, + lease.state, + context.strip, + context.server_path, + options, + context.bandwidth_learning, + ) + .await + .filter(|response| response.status() == StatusCode::OK) + { + return HlsCanonicalOwnerEvaluation::Resolved(HlsCanonicalOwnerResolution::Live(response)); + } + } + HlsCanonicalOwnerEvaluation::Pending(HlsCanonicalOwnerPending { + deadline_ms: hls_canonical_owner_lease_deadline_ms(&lease).min(context.request_deadline_ms), + current_session_available: current_session.is_some(), + }) +} + +pub(in crate::api::endpoints::hls_api) async fn finalize_hls_canonical_owner_handoff( + context: &HlsCanonicalOwnerHandoffContext<'_>, + pending: HlsCanonicalOwnerPending, + deadline_elapsed: bool, +) -> HlsCanonicalOwnerResolution { + match evaluate_hls_canonical_owner_handoff(context).await { + HlsCanonicalOwnerEvaluation::Resolved(resolution) => resolution, + HlsCanonicalOwnerEvaluation::Pending(current) => { + let response = hls_unpublished_lease_channel_unavailable_response( + context.app_state, + context.proxy_session_id, + context.access_lease_id, + ) + .await; + if response.status() == StatusCode::OK { + return HlsCanonicalOwnerResolution::Standalone(response); + } + let reason = if deadline_elapsed { + HlsCanonicalOwnerFailureReason::DeadlineElapsed + } else if !current.current_session_available && !pending.current_session_available { + HlsCanonicalOwnerFailureReason::Superseded + } else { + HlsCanonicalOwnerFailureReason::LeaseUnavailable + }; + HlsCanonicalOwnerResolution::FailedClosed { reason, response: reason.response() } + } + } +} + +pub(in crate::api::endpoints::hls_api) async fn join_hls_canonical_manifest_owner( + context: HlsCanonicalOwnerHandoffContext<'_>, + registration: HlsCanonicalOwnerRegistrationKind, +) -> axum::response::Response { + let started_at = tokio::time::Instant::now(); + let coordinator = context.app_state.hls.proxy.availability_reevaluations(); + let resolution = loop { + let mut observer = coordinator.observe_owner(context.proxy_session_id); + let pending = match evaluate_hls_canonical_owner_handoff(&context).await { + HlsCanonicalOwnerEvaluation::Resolved(resolution) => break resolution, + HlsCanonicalOwnerEvaluation::Pending(pending) => pending, + }; + let now_ms = current_time_millis(); + if now_ms >= pending.deadline_ms { + break finalize_hls_canonical_owner_handoff(&context, pending, true).await; + } + let Some(observer) = observer.as_mut() else { + break finalize_hls_canonical_owner_handoff(&context, pending, false).await; + }; + let remaining_ms = pending.deadline_ms.saturating_sub(now_ms); + match tokio::time::timeout(Duration::from_millis(remaining_ms), observer.changed()).await { + Ok( + HlsAvailabilityReevaluationObservation::EvidenceChanged + | HlsAvailabilityReevaluationObservation::OwnerFinished, + ) => {} + Err(_) => break finalize_hls_canonical_owner_handoff(&context, pending, true).await, + } + }; + debug!( + "HLS canonical manifest owner handoff completed: session={} proxy_session={} lease={} registration={} outcome={} wait_ms={}", + context.safe_session, + safe_proxy_session_id(context.proxy_session_id), + safe_hls_access_lease_id(context.access_lease_id), + registration.as_label(), + resolution.outcome_label(), + duration_to_millis_saturating(started_at.elapsed()) + ); + resolution.into_response() +} + +pub(in crate::api::endpoints::hls_api) async fn hls_direct_refresh_follow_up( + app_state: &Arc, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + refresh_request: OriginRefreshRequest, + outcome: HlsOriginRefreshTriggerOutcome, +) -> Option { + match outcome { + HlsOriginRefreshTriggerOutcome::Started + | HlsOriginRefreshTriggerOutcome::SessionUnavailable + | HlsOriginRefreshTriggerOutcome::InFlight + | HlsOriginRefreshTriggerOutcome::DebouncedUntil { .. } => return None, + HlsOriginRefreshTriggerOutcome::RecoveryPressureSuperseded => { + warn!("HLS direct origin refresh evidence superseded; scheduling current availability reevaluation"); + } + HlsOriginRefreshTriggerOutcome::RecoveryPressureStateContention => { + warn!("HLS direct origin refresh state contended; scheduling current availability reevaluation"); + } + } + let Some(owner_key) = app_state.hls.proxy.availability_reevaluation_owner_key(session, proxy_session_id).await + else { + warn!("HLS direct origin refresh follow-up unavailable: reason=session_superseded"); + return Some(hls_canonical_retry_after_response()); + }; + match register_hls_availability_reevaluation(app_state.hls_ctx(), Arc::clone(session), owner_key, refresh_request) { + HlsAvailabilityReevaluationRegistration::Scheduled + | HlsAvailabilityReevaluationRegistration::AlreadyOwned + | HlsAvailabilityReevaluationRegistration::Superseded => None, + HlsAvailabilityReevaluationRegistration::CapacityExceeded => { + warn!("HLS direct origin refresh follow-up unavailable: reason=capacity_exceeded"); + Some(hls_canonical_retry_after_response()) + } + HlsAvailabilityReevaluationRegistration::RuntimeUnavailable => { + warn!("HLS direct origin refresh follow-up unavailable: reason=runtime_unavailable"); + Some(hls_canonical_retry_after_response()) + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::api::endpoints::hls_api) enum HlsManifestRefreshOrdering { + Background, + AwaitBeforeTerminalEvaluation, +} + +pub(in crate::api::endpoints::hls_api) async fn trigger_hls_canonical_manifest_refresh( + app_state: &Arc, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + refresh_request: OriginRefreshRequest, + ordering: HlsManifestRefreshOrdering, +) -> Option { + match ordering { + HlsManifestRefreshOrdering::Background => { + let outcome = maybe_trigger_origin_refresh_with_outcome(refresh_request.clone()).await; + hls_direct_refresh_follow_up(app_state, session, proxy_session_id, refresh_request, outcome).await + } + HlsManifestRefreshOrdering::AwaitBeforeTerminalEvaluation => { + // An already-owned refresh cannot be joined through this call, but + // it must not bypass the lease-specific terminal decision. The + // in-flight owner will still publish its eventual progress/failure. + let _refresh_started = trigger_origin_refresh_sync(refresh_request).await; + let now_ms = current_time_millis(); + let Some(lease) = + app_state.hls.proxy.access_lease_response_snapshot(access_lease_id, proxy_session_id, now_ms).await + else { + return Some(hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::LeaseStateUnavailable)); + }; + resolve_hls_terminal_manifest_state(app_state, session, proxy_session_id, access_lease_id, lease, now_ms) + .await + .err() + .map(|response| *response) + } + } +} + +pub(in crate::api::endpoints::hls_api) fn hls_canonical_status_response( + status: StatusCode, +) -> axum::response::Response { + if status == StatusCode::SERVICE_UNAVAILABLE { + hls_canonical_retry_after_response() + } else { + status.into_response() + } +} diff --git a/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs b/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs new file mode 100644 index 000000000..3820bc4d7 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/manifest/provisioning.rs @@ -0,0 +1,800 @@ +use super::{ + build_hls_origin_resolution, build_hls_origin_source, get_stream_channel, hls_cache_configured, + hls_cache_enabled_for_user, hls_canonical_retry_after_response, hls_origin_account_reservation_ttl_secs_fallback, + hls_origin_account_reservation_ttl_secs_for_session, hls_runtime_or_standalone_custom_tail_response, + resolve_hls_origin_playlist_url, try_hls_cached_manifest_response, HlsEntryStreamIdentity, + HlsRuntimeBandwidthLearningContext, +}; +use crate::{ + api::{ + api_utils::{ + connection_priority_for_kind, create_playback_session_fingerprint, get_hls_session_ttl_secs, + resolve_playback_request_admission, select_provider_stream_url, EvictionReentryGuard, + }, + model::{ + hls_custom_video_manifest_response_with_virtual_id, hls_provisioning_discontinuity_sequence, + hls_virtual_entry_redirect_response, is_custom_video_stream_enabled, start_hls_panel_provisioning_once, + try_hls_panel_provisioning_manifest_response, AppState, CustomVideoStreamType, + HlsPanelProvisioningRedirectPaths, HlsProvisioningStatus, PlaybackLeaseRef, + ProviderConfig as RuntimeProviderConfig, ProviderHandle, TransportStreamBuffer, + }, + panel_api::can_provision_on_exhausted, + }, + auth::Fingerprint, + model::{ConfigInput, ConfigTarget, PlaybackKind, ProxyUserCredentials}, +}; +use axum::{http::StatusCode, response::IntoResponse}; +use log::{debug, warn}; +use shared::model::{PlaylistItemType, UserConnectionPermission, VirtualId}; +use std::{sync::Arc, time::Duration}; +use tuliprox_core::utils::current_time_millis; +use tuliprox_hls::api::{ + build_hls_origin_session_owner, build_proxy_session_id, is_hls_provisioning_gap_segment, + is_hls_provisioning_segment, safe_hls_access_lease_id, safe_proxy_session_id, CacheAccessState, HlsAccessLeaseId, + HlsAccessLeaseState, HlsCachedManifestOptions, HlsRuntimeCustomTailReason, HlsSession, HlsSessionHandle, + OriginSegmentKey, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, HLS_PROVISIONING_GAP_ORIGIN_EPOCH, + HLS_PROVISIONING_ORIGIN_EPOCH, HLS_PROVISIONING_SEGMENT_DURATION_MS, HLS_PROVISIONING_TARGET_DURATION_SECS, +}; + +pub(in crate::api::endpoints::hls_api) struct HlsEntryOriginAccountReservation { + pub(in crate::api::endpoints::hls_api) request_url: String, + pub(in crate::api::endpoints::hls_api) session_token: String, + pub(in crate::api::endpoints::hls_api) provider_handle: Option, + pub(in crate::api::endpoints::hls_api) selected_provider_config: Option>, +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::api::endpoints::hls_api) async fn try_reserve_hls_entry_origin_account_for_redirect( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + input: &ConfigInput, + virtual_id: u32, + request_url: &str, + user_session_token: &str, + session_owner: &str, + playback_kind: PlaybackKind, + reservation_ttl_secs: u64, + connection_permission: UserConnectionPermission, + connection_kind: crate::api::model::ConnectionKind, + create_user_session: bool, +) -> Option { + let provider_handle = app_state.active_provider.acquire_connection_with_lease_for_session( + &input.name, + &fingerprint.addr, + false, + connection_priority_for_kind(user, connection_kind), + connection_kind, + Some(PlaybackLeaseRef::new(session_owner, playback_kind)), + )?; + + let Some(provider_config) = provider_handle.allocation.get_provider_config() else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)); + return None; + }; + let Some((_provider_name, stream_url)) = + select_provider_stream_url(request_url, input, &provider_config, false, &app_state.app_config).await + else { + app_state.connection_manager.release_provider_handle(Some(provider_handle)); + return None; + }; + + let session_token = if create_user_session { + app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user, + session_token: user_session_token, + virtual_id, + provider: &provider_config.name, + stream_url: &stream_url, + addr: &fingerprint.addr, + connection_permission, + connection_kind: Some(connection_kind), + socket_bound: PlaylistItemType::LiveHls.uses_socket_bound_session(), + }) + .await + } else { + user_session_token.to_string() + }; + + app_state.active_provider.refresh_adaptive_playback_lease( + &provider_config.name, + session_owner, + playback_kind, + reservation_ttl_secs, + ); + + Some(HlsEntryOriginAccountReservation { + request_url: stream_url, + session_token, + provider_handle: Some(provider_handle), + selected_provider_config: Some(provider_config), + }) +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::api::endpoints::hls_api) async fn try_reserve_hls_virtual_entry_origin_account_for_redirect( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + target: &Arc, + input: &ConfigInput, + stream_identity: &HlsEntryStreamIdentity, +) -> bool { + let virtual_id = stream_identity.virtual_id(); + let session_token = + create_playback_session_fingerprint(fingerprint, &user.username, virtual_id, PlaylistItemType::LiveHls, None); + let (connection_admission, _, _) = resolve_playback_request_admission( + &app_state.admission_ctx(), + user, + fingerprint, + None, + &session_token, + false, + EvictionReentryGuard::SocketPlayback { virtual_id: VirtualId::new(virtual_id) }, + false, + false, + ) + .await; + if connection_admission.permission() == UserConnectionPermission::Exhausted { + return false; + } + + let Some(channel) = get_stream_channel(app_state, target, virtual_id).await else { + return false; + }; + let Ok(origin_playlist_url) = + resolve_hls_origin_playlist_url(app_state, target, input, virtual_id, channel.url.as_ref()).await + else { + return false; + }; + let Some(hls_cache_origin) = build_hls_origin_resolution(input, &origin_playlist_url) else { + return false; + }; + let Some(connection_kind) = connection_admission.kind() else { + return false; + }; + let (shared_hls_session_owner, reservation_ttl_secs) = if hls_cache_enabled_for_user(app_state, target, user) { + let origin_source = build_hls_origin_source(input, stream_identity.stream_ref()); + let proxy_session_id = build_proxy_session_id(&origin_source.session_key(), &app_state.get_encrypt_secret()); + let reservation_ttl_secs = match app_state.hls.proxy.sessions().get_by_key(&origin_source.session_key()).await { + Some(session) => hls_origin_account_reservation_ttl_secs_for_session(&session).await, + None => hls_origin_account_reservation_ttl_secs_fallback(), + }; + (Some(build_hls_origin_session_owner(&proxy_session_id)), reservation_ttl_secs) + } else { + (None, get_hls_session_ttl_secs(app_state)) + }; + let session_owner = shared_hls_session_owner.as_deref().unwrap_or(session_token.as_str()); + + let Some(reservation) = try_reserve_hls_entry_origin_account_for_redirect( + app_state, + fingerprint, + user, + input, + virtual_id, + hls_cache_origin.session_entry_url.as_str(), + &session_token, + session_owner, + PlaybackKind::LiveHls, + reservation_ttl_secs, + connection_admission.permission(), + connection_kind, + false, + ) + .await + else { + return false; + }; + + app_state.connection_manager.release_provider_handle(reservation.provider_handle); + true +} + +pub(in crate::api::endpoints::hls_api) async fn mark_hls_provisioning_handoff_discontinuity( + app_state: &Arc, + input: &ConfigInput, + stream_identity: &HlsEntryStreamIdentity, + access_lease_id: Option<&HlsAccessLeaseId>, + now_ms: u64, +) -> bool { + if !hls_cache_configured(app_state) { + return false; + } + let origin_source = build_hls_origin_source(input, stream_identity.stream_ref()); + let Some(session) = app_state.hls.proxy.sessions().get_by_key(&origin_source.session_key()).await else { + return false; + }; + mark_hls_provisioning_handoff_discontinuity_once_for_session( + app_state, + &session, + input, + stream_identity.virtual_id(), + access_lease_id, + now_ms, + ) + .await +} + +pub(in crate::api::endpoints::hls_api) async fn mark_hls_provisioning_handoff_discontinuity_once_for_session( + app_state: &Arc, + session: &HlsSessionHandle, + input: &ConfigInput, + virtual_id: u32, + access_lease_id: Option<&HlsAccessLeaseId>, + now_ms: u64, +) -> bool { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + if !app_state.hls.provisioning.mark_handoff_once( + &input.name, + virtual_id, + Some(&proxy_session_id), + access_lease_id, + now_ms, + ) { + debug!( + "HLS provisioning handoff discontinuity already marked: proxy_session={}", + safe_proxy_session_id(&proxy_session_id) + ); + return false; + } + mark_hls_provisioning_handoff_discontinuity_for_session(session, now_ms).await; + ensure_shared_hls_provisioning_handoff_gap(app_state, session, now_ms).await; + true +} + +pub(in crate::api::endpoints::hls_api) async fn mark_hls_provisioning_handoff_discontinuity_for_session( + session: &HlsSessionHandle, + now_ms: u64, +) { + let discontinuity_sequence = hls_provisioning_discontinuity_sequence(now_ms); + let proxy_session_id = { + let mut session = session.write().await; + session.mark_pending_handoff_discontinuity(discontinuity_sequence); + session.proxy_session_id.clone() + }; + debug!( + "HLS provisioning handoff discontinuity marked: proxy_session={} discontinuity_sequence={}", + safe_proxy_session_id(&proxy_session_id), + discontinuity_sequence + ); +} + +pub(in crate::api::endpoints::hls_api) fn clear_hls_provisioning_handoff_consumer( + app_state: &Arc, + input: &ConfigInput, + virtual_id: u32, + now_ms: u64, +) { + if !app_state.hls.provisioning.take_ready_slot_for_consumer(&input.name, virtual_id, now_ms) { + app_state.hls.provisioning.clear_consumer(&input.name, virtual_id); + } +} + +pub(in crate::api::endpoints::hls_api) async fn maybe_mark_hls_provisioning_handoff_for_canonical_manifest( + app_state: &Arc, + session: &HlsSessionHandle, + input: &ConfigInput, + virtual_id: u32, + access_lease_id: &HlsAccessLeaseId, + now_ms: u64, +) -> Option { + if !app_state.hls.provisioning.has_consumer(&input.name, virtual_id, now_ms) { + return None; + } + let previous_manifest_rendered_at_ms = latest_shared_hls_manifest_rendered_at_ms(session).await; + mark_hls_provisioning_handoff_discontinuity_once_for_session( + app_state, + session, + input, + virtual_id, + Some(access_lease_id), + now_ms, + ) + .await + .then_some(previous_manifest_rendered_at_ms) +} + +pub(in crate::api::endpoints::hls_api) async fn latest_shared_hls_manifest_rendered_at_ms( + session: &HlsSessionHandle, +) -> u64 { + let session = session.read().await; + session + .last_rendered_manifest + .as_ref() + .map_or(0, |rendered| rendered.rendered_at_ms) + .max(session.transient.last_manifest_rendered_at_ms.unwrap_or(0)) +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::api) async fn hls_panel_provisioning_poll_manifest_response( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + target: &Arc, + input: &ConfigInput, + stream_identity: &HlsEntryStreamIdentity, + original_hls_entry_path: &str, + server_path: Option<&str>, +) -> axum::response::Response { + hls_panel_provisioning_poll_response( + app_state, + fingerprint, + user, + target, + input, + stream_identity, + original_hls_entry_path, + server_path, + HlsProvisioningPollResponseKind::Legacy, + ) + .await +} + +pub(in crate::api::endpoints::hls_api) enum HlsProvisioningPollResponseKind { + Legacy, +} + +impl HlsProvisioningPollResponseKind { + pub(in crate::api::endpoints::hls_api) fn access_lease_id(&self) -> Option<&HlsAccessLeaseId> { + match self { + Self::Legacy => None, + } + } +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::api::endpoints::hls_api) async fn hls_panel_provisioning_poll_response( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + target: &Arc, + input: &ConfigInput, + stream_identity: &HlsEntryStreamIdentity, + ready_redirect_path: &str, + server_path: Option<&str>, + response_kind: HlsProvisioningPollResponseKind, +) -> axum::response::Response { + let virtual_id = stream_identity.virtual_id(); + let now_ms = current_time_millis(); + app_state.hls.provisioning.touch_consumer(Arc::clone(&input.name), virtual_id, now_ms); + + let existing_status = app_state.hls.provisioning.consumer_status(&input.name, virtual_id, now_ms); + + if try_reserve_hls_virtual_entry_origin_account_for_redirect( + app_state, + fingerprint, + user, + target, + input, + stream_identity, + ) + .await + { + mark_hls_provisioning_handoff_discontinuity( + app_state, + input, + stream_identity, + response_kind.access_lease_id(), + now_ms, + ) + .await; + clear_hls_provisioning_handoff_consumer(app_state, input, virtual_id, current_time_millis()); + return hls_virtual_entry_redirect_response(ready_redirect_path, server_path); + } + + let provisioning_enabled = can_provision_on_exhausted(app_state.as_ref(), input); + if provisioning_enabled { + start_hls_panel_provisioning_once(app_state, input); + } + + let status = existing_status.unwrap_or(if provisioning_enabled { + HlsProvisioningStatus::InProgress + } else { + HlsProvisioningStatus::ProviderExhausted + }); + + match status { + HlsProvisioningStatus::Ready | HlsProvisioningStatus::InProgress => { + hls_custom_video_manifest_response_with_virtual_id( + app_state, + user, + CustomVideoStreamType::Provisioning, + StatusCode::SERVICE_UNAVAILABLE, + Some(virtual_id), + ) + .await + } + HlsProvisioningStatus::ProviderExhausted => { + hls_custom_video_manifest_response_with_virtual_id( + app_state, + user, + CustomVideoStreamType::ProviderConnectionsExhausted, + StatusCode::SERVICE_UNAVAILABLE, + Some(virtual_id), + ) + .await + } + } +} + +pub(in crate::api::endpoints::hls_api) async fn hls_panel_provisioning_or_status_response( + app_state: &Arc, + user: &ProxyUserCredentials, + input: &ConfigInput, + virtual_id: u32, + _original_hls_entry_path: &str, + server_path: Option<&str>, + fallback_status: StatusCode, +) -> axum::response::Response { + try_hls_panel_provisioning_manifest_response( + app_state, + user, + input, + virtual_id, + HlsPanelProvisioningRedirectPaths { waiting_manifest_path: None }, + server_path, + fallback_status, + ) + .await + .unwrap_or_else(|| fallback_status.into_response()) +} + +#[derive(Debug, Clone)] +pub(in crate::api::endpoints::hls_api) struct SharedHlsProvisioningSegmentPlan { + pub(in crate::api::endpoints::hls_api) proxy_seq: u64, + pub(in crate::api::endpoints::hls_api) physical_index: usize, + pub(in crate::api::endpoints::hls_api) cache_key: SegmentCacheKey, + pub(in crate::api::endpoints::hls_api) segment_kind: SharedHlsProvisioningLocalSegmentKind, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(in crate::api::endpoints::hls_api) enum SharedHlsProvisioningLocalSegmentKind { + Provisioning, + Gap, +} + +pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_plans( + session: &HlsSession, + physical_segment_count: usize, +) -> Vec { + let existing_provisioning_segments = + session.segments.values().filter(|entry| is_hls_provisioning_segment(entry)).count(); + let append_count = if existing_provisioning_segments == 0 { 3 } else { 1 }; + let start_proxy_seq = session.proxy_next_seq.unwrap_or(0); + (0..append_count) + .filter_map(|offset| { + let proxy_seq = start_proxy_seq.checked_add(u64::try_from(offset).ok()?)?; + if session.segments.contains_key(&proxy_seq) { + return None; + } + Some(SharedHlsProvisioningSegmentPlan { + proxy_seq, + physical_index: (existing_provisioning_segments + offset) % physical_segment_count, + cache_key: SegmentCacheKey::new(session.proxy_session_id.clone(), proxy_seq, "ts"), + segment_kind: SharedHlsProvisioningLocalSegmentKind::Provisioning, + }) + }) + .collect() +} + +pub(in crate::api::endpoints::hls_api) fn shared_hls_provisioning_segment_entry( + plan: SharedHlsProvisioningSegmentPlan, + content_length: u64, + duration_ms: u64, + now_ms: u64, +) -> SegmentEntry { + let origin_epoch = match plan.segment_kind { + SharedHlsProvisioningLocalSegmentKind::Provisioning => HLS_PROVISIONING_ORIGIN_EPOCH, + SharedHlsProvisioningLocalSegmentKind::Gap => HLS_PROVISIONING_GAP_ORIGIN_EPOCH, + }; + SegmentEntry { + origin_key: OriginSegmentKey { + origin_epoch, + effective_host_id: 0, + host_local_sequence: plan.proxy_seq, + host_local_index: u32::try_from(plan.proxy_seq).unwrap_or(u32::MAX), + }, + proxy_seq: plan.proxy_seq, + duration_ms, + proxy_file_ext: "ts".to_string(), + content_type: "video/mp2t".to_string(), + cache_key: plan.cache_key, + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + encryption: None, + origin_fetch_ref: None, + status: SegmentCacheStatus::Ready { content_length, ready_at_ms: now_ms }, + last_rendered_at_ms: None, + access: Arc::new(CacheAccessState::new()), + } +} + +pub(in crate::api::endpoints::hls_api) async fn commit_shared_hls_provisioning_segments( + app_state: &Arc, + session: &HlsSessionHandle, + plans: &[SharedHlsProvisioningSegmentPlan], + provisioning_segments: &[TransportStreamBuffer], +) -> Option> { + let mut committed = Vec::with_capacity(plans.len()); + for plan in plans { + let video = provisioning_segments.get(plan.physical_index)?; + let duration_ms = video.duration_ms().unwrap_or(HLS_PROVISIONING_SEGMENT_DURATION_MS); + let metadata = match app_state + .hls + .proxy + .segment_cache() + .write_bytes_and_commit(&plan.cache_key, video.as_bytes()) + .await + { + Ok(metadata) => metadata, + Err(err) => { + let safe_proxy_session = { + let session_guard = session.read().await; + safe_proxy_session_id(&session_guard.proxy_session_id) + }; + warn!( + "HLS provisioning segment cache commit failed for shared manifest: proxy_session={} seq={} error={err}", + safe_proxy_session, plan.proxy_seq + ); + return None; + } + }; + committed.push((plan.clone(), metadata.size, duration_ms)); + } + Some(committed) +} + +pub(in crate::api::endpoints::hls_api) async fn ensure_shared_hls_provisioning_handoff_gap( + app_state: &Arc, + session: &HlsSessionHandle, + now_ms: u64, +) -> bool { + let custom_stream_response = app_state.app_config.custom_stream_response.load(); + let Some(provisioning_segments) = custom_stream_response + .as_ref() + .map(|response| response.panel_api_provisioning_hls_segments.clone()) + .filter(|segments| !segments.is_empty()) + else { + return false; + }; + let plan = { + let session_guard = session.read().await; + if !session_guard.segments.values().any(is_hls_provisioning_segment) + || session_guard.segments.values().any(is_hls_provisioning_gap_segment) + { + return false; + } + let proxy_seq = session_guard.proxy_next_seq.unwrap_or(0); + if session_guard.segments.contains_key(&proxy_seq) { + return false; + } + let existing_provisioning_segments = + session_guard.segments.values().filter(|entry| is_hls_provisioning_segment(entry)).count(); + SharedHlsProvisioningSegmentPlan { + proxy_seq, + physical_index: existing_provisioning_segments % provisioning_segments.len(), + cache_key: SegmentCacheKey::new(session_guard.proxy_session_id.clone(), proxy_seq, "ts"), + segment_kind: SharedHlsProvisioningLocalSegmentKind::Gap, + } + }; + let Some(committed) = commit_shared_hls_provisioning_segments( + app_state, + session, + std::slice::from_ref(&plan), + &provisioning_segments, + ) + .await + else { + return false; + }; + let mut session_guard = session.write().await; + let mut inserted = false; + for (plan, content_length, duration_ms) in committed { + if session_guard.segments.contains_key(&plan.proxy_seq) { + continue; + } + if session_guard.publishable_origin_head_proxy_seq.is_none() { + session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq); + } + session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq); + session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1)); + session_guard + .segments + .insert(plan.proxy_seq, shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms)); + inserted = true; + } + if inserted { + session_guard.target_duration = Some(HLS_PROVISIONING_TARGET_DURATION_SECS); + session_guard.independent_segments = true; + } + inserted +} + +pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_timeline_manifest_response( + app_state: &Arc, + session: &HlsSessionHandle, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + strip: &crate::model::StripConfig, + server_path: Option<&str>, +) -> Option { + let custom_stream_response = app_state.app_config.custom_stream_response.load(); + let provisioning_segments = custom_stream_response + .as_ref() + .map(|response| response.panel_api_provisioning_hls_segments.clone()) + .filter(|segments| !segments.is_empty())?; + let now_ms = current_time_millis(); + let plans = { + let session_guard = session.read().await; + shared_hls_provisioning_segment_plans(&session_guard, provisioning_segments.len()) + }; + if plans.is_empty() { + let mut session_guard = session.write().await; + session_guard.render_and_store_manifest(now_ms).ok()?; + } else { + let committed = + commit_shared_hls_provisioning_segments(app_state, session, &plans, &provisioning_segments).await?; + let mut session_guard = session.write().await; + for (plan, content_length, duration_ms) in committed { + if session_guard.segments.contains_key(&plan.proxy_seq) { + continue; + } + if session_guard.publishable_origin_head_proxy_seq.is_none() { + session_guard.publishable_origin_head_proxy_seq = Some(plan.proxy_seq); + } + session_guard.publishable_origin_tail_proxy_seq = Some(plan.proxy_seq); + session_guard.proxy_next_seq = Some(plan.proxy_seq.saturating_add(1)); + session_guard.segments.insert( + plan.proxy_seq, + shared_hls_provisioning_segment_entry(plan, content_length, duration_ms, now_ms), + ); + } + session_guard.target_duration = Some(HLS_PROVISIONING_TARGET_DURATION_SECS); + session_guard.independent_segments = true; + session_guard.render_and_store_manifest(now_ms).ok()?; + } + + try_hls_cached_manifest_response( + app_state, + session, + access_lease_id, + access_lease_state, + strip, + server_path, + HlsCachedManifestOptions::initial(Duration::ZERO), + HlsRuntimeBandwidthLearningContext::Disabled, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::api::endpoints::hls_api) async fn hls_shared_provisioning_or_provider_exhausted_response( + app_state: &Arc, + session: &HlsSessionHandle, + username: &str, + input: &ConfigInput, + virtual_id: u32, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + strip: &crate::model::StripConfig, + server_path: Option<&str>, +) -> axum::response::Response { + let Some((_user, _target)) = app_state.app_config.get_target_for_username(username) else { + return hls_canonical_retry_after_response(); + }; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let now_ms = current_time_millis(); + let provisioning_enabled = can_provision_on_exhausted(app_state.as_ref(), input); + if provisioning_enabled { + app_state.hls.provisioning.touch_consumer(Arc::clone(&input.name), virtual_id, now_ms); + start_hls_panel_provisioning_once(app_state, input); + if let Some(HlsProvisioningStatus::ProviderExhausted) = + app_state.hls.provisioning.consumer_status(&input.name, virtual_id, now_ms) + { + return hls_runtime_or_standalone_custom_tail_response( + app_state, + session, + &proxy_session_id, + access_lease_id, + HlsRuntimeCustomTailReason::ProviderConnectionsExhausted, + StatusCode::SERVICE_UNAVAILABLE, + ) + .await; + } + if let Some(response) = hls_shared_provisioning_timeline_manifest_response( + app_state, + session, + access_lease_id, + access_lease_state, + strip, + server_path, + ) + .await + { + return response; + } + } + + let provider_exhausted_custom_response_available = is_custom_video_stream_enabled(&app_state.app_config) + && app_state + .app_config + .custom_stream_response + .load() + .as_ref() + .and_then(|response| response.provider_connections_exhausted.as_ref()) + .is_some(); + if provider_exhausted_custom_response_available { + return hls_runtime_or_standalone_custom_tail_response( + app_state, + session, + &proxy_session_id, + access_lease_id, + HlsRuntimeCustomTailReason::ProviderConnectionsExhausted, + StatusCode::SERVICE_UNAVAILABLE, + ) + .await; + } + hls_canonical_retry_after_response() +} + +pub(in crate::api::endpoints::hls_api) enum HlsProviderExhaustedResolution { + RetryAcquire, + Response(axum::response::Response), +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::api::endpoints::hls_api) async fn hls_provider_connections_exhausted_manifest_resolution( + app_state: &Arc, + session: &HlsSessionHandle, + username: &str, + input: &ConfigInput, + virtual_id: u32, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + strip: &crate::model::StripConfig, + server_path: Option<&str>, + allow_grace_hold: bool, +) -> HlsProviderExhaustedResolution { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let grace_options = app_state.get_grace_options(); + if allow_grace_hold && grace_options.hold_stream && grace_options.period_millis > 0 { + debug!( + "HLS provider connections exhausted; holding canonical manifest for grace: proxy_session={} lease={} hold_ms={}", + safe_proxy_session_id(&proxy_session_id), + safe_hls_access_lease_id(access_lease_id), + grace_options.period_millis + ); + let capacity_notify = app_state.connection_manager.capacity_notified(); + let deadline = tokio::time::Instant::now() + Duration::from_millis(grace_options.period_millis); + let wake_reason = tokio::select! { + () = capacity_notify.notified() => "capacity-notified", + () = tokio::time::sleep_until(deadline) => "timeout", + }; + debug!( + "HLS provider connections exhausted grace hold completed: proxy_session={} lease={} reason={wake_reason}", + safe_proxy_session_id(&proxy_session_id), + safe_hls_access_lease_id(access_lease_id) + ); + return HlsProviderExhaustedResolution::RetryAcquire; + } + + HlsProviderExhaustedResolution::Response( + hls_shared_provisioning_or_provider_exhausted_response( + app_state, + session, + username, + input, + virtual_id, + access_lease_id, + access_lease_state, + strip, + server_path, + ) + .await, + ) +} diff --git a/backend/app/src/api/endpoints/hls_api/manifest/publication.rs b/backend/app/src/api/endpoints/hls_api/manifest/publication.rs new file mode 100644 index 000000000..092fef3c3 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/manifest/publication.rs @@ -0,0 +1,262 @@ +use super::{hls_canonical_manifest_path, HlsEntryMasterPlaylistResponse}; +use crate::{ + api::{ + api_utils::{record_connect_failed_attempt, try_unwrap_body, ConnectFailedAttempt}, + model::{ + hls_cache::initial_strip::{ + materialize_initial_hls_strip_view, HlsInitialStripOutcome, HlsInitialStripSkipReason, + }, + hls_custom_video_manifest_response_with_virtual_id, AppState, CustomVideoStreamType, + }, + }, + auth::Fingerprint, + model::ProxyUserCredentials, + processing::parser::hls::origin_manifest::HlsManifestWindowPolicy, +}; +use axum::{ + body::Body, + http::{header, HeaderMap, StatusCode}, + response::IntoResponse, +}; +use serde::Deserialize; +use shared::model::{ConnectFailureReason, FailureStage, StreamChannel}; +use std::{borrow::Cow, sync::Arc}; +use tuliprox_hls::api::{ + HlsAccessLeaseId, HlsAccessLeaseState, HlsMasterBandwidth, HlsSingleVariantMasterPlaylist, ProxySessionId, + HLS_ACCESS_LEASE_ID_PLACEHOLDER, +}; + +#[derive(Debug, Deserialize)] +pub(in crate::api::endpoints::hls_api) struct HlsProxyTerminalSegmentPathParams { + pub(in crate::api::endpoints::hls_api) proxy_session_id: String, + pub(in crate::api::endpoints::hls_api) hls_access_lease_id: String, + pub(in crate::api::endpoints::hls_api) generation: String, + pub(in crate::api::endpoints::hls_api) terminal_file: String, +} + +pub(in crate::api::endpoints::hls_api) fn hls_custom_video_type_for_failure_reason( + reason: ConnectFailureReason, +) -> CustomVideoStreamType { + match reason { + ConnectFailureReason::UserAccountExpired => CustomVideoStreamType::UserAccountExpired, + ConnectFailureReason::UserConnectionsExhausted => CustomVideoStreamType::UserConnectionsExhausted, + ConnectFailureReason::ProviderConnectionsExhausted => CustomVideoStreamType::ProviderConnectionsExhausted, + ConnectFailureReason::Preempted => CustomVideoStreamType::LowPriorityPreempted, + ConnectFailureReason::Provisioning => CustomVideoStreamType::Provisioning, + ConnectFailureReason::SessionExpired => CustomVideoStreamType::HlsSessionOrLeaseExpired, + ConnectFailureReason::ProviderError + | ConnectFailureReason::ProviderClosed + | ConnectFailureReason::ChannelUnavailable => CustomVideoStreamType::ChannelUnavailable, + } +} + +pub(crate) async fn hls_custom_video_manifest_response( + app_state: &Arc, + user: &ProxyUserCredentials, + video_type: CustomVideoStreamType, + fallback_status: StatusCode, +) -> axum::response::Response { + hls_custom_video_manifest_response_with_virtual_id(app_state, user, video_type, fallback_status, None).await +} + +pub(crate) async fn hls_admission_failure_manifest_response( + app_state: &Arc, + fingerprint: &Fingerprint, + user: &ProxyUserCredentials, + stream_channel: StreamChannel, + provider_name: Arc, + req_headers: &HeaderMap, + reason: ConnectFailureReason, +) -> axum::response::Response { + record_connect_failed_attempt(ConnectFailedAttempt { + app_state, + fingerprint, + user, + stream_channel, + provider_name, + req_headers, + reason, + failure_stage: FailureStage::Admission, + }); + hls_custom_video_manifest_response( + app_state, + user, + hls_custom_video_type_for_failure_reason(reason), + StatusCode::FORBIDDEN, + ) + .await +} + +pub(in crate::api::endpoints::hls_api) fn apply_hls_proxy_public_path_prefix( + hls_content: String, + server_path: Option<&str>, +) -> String { + let Some(path_prefix) = normalize_hls_proxy_public_path_prefix(server_path) else { + return hls_content; + }; + + let uri_attr_prefix = format!("URI=\"{path_prefix}/hls/shared/live/"); + let hls_content = hls_content.replace("URI=\"/hls/shared/live/", &uri_attr_prefix); + if hls_content.is_empty() { + return hls_content; + } + let mut prefixed = String::with_capacity(hls_content.len().saturating_add(path_prefix.len().saturating_mul(4))); + + for part in hls_content.split_inclusive('\n') { + let (line, line_ending) = split_hls_line_ending(part); + if line.starts_with("/hls/shared/live/") { + prefixed.push_str(&path_prefix); + } + prefixed.push_str(line); + prefixed.push_str(line_ending); + } + + prefixed +} + +pub(in crate::api::endpoints::hls_api) fn normalize_hls_proxy_public_path_prefix( + server_path: Option<&str>, +) -> Option { + let path = server_path?.trim().trim_matches('/'); + if path.is_empty() { + return None; + } + Some(format!("/{path}")) +} + +pub(in crate::api::endpoints::hls_api) fn split_hls_line_ending(part: &str) -> (&str, &str) { + if let Some(line) = part.strip_suffix("\r\n") { + (line, "\r\n") + } else if let Some(line) = part.strip_suffix('\n') { + (line, "\n") + } else { + (part, "") + } +} + +pub(in crate::api::endpoints::hls_api) fn materialize_hls_access_manifest( + hls_content: &str, + lease_id: &HlsAccessLeaseId, + server_path: Option<&str>, +) -> String { + let hls_content = hls_content.replace(HLS_ACCESS_LEASE_ID_PLACEHOLDER, &lease_id.0); + apply_hls_proxy_public_path_prefix(hls_content, server_path) +} + +pub(in crate::api::endpoints::hls_api) fn hls_access_manifest_uses_startup_view( + lease_state: HlsAccessLeaseState, +) -> bool { + matches!(lease_state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Idle) +} + +pub(in crate::api::endpoints::hls_api) fn materialize_shared_hls_access_manifest( + hls_content: &str, + lease_id: &HlsAccessLeaseId, + lease_state: HlsAccessLeaseState, + strip: &crate::model::StripConfig, + window_policy: HlsManifestWindowPolicy, + mode: &'static str, + server_path: Option<&str>, +) -> HlsMaterializedSharedManifest { + let (response_body, initial_strip_outcome) = if hls_access_manifest_uses_startup_view(lease_state) { + let view = materialize_initial_hls_strip_view(hls_content, strip, window_policy); + (view.body, Some(view.outcome)) + } else { + (Cow::Borrowed(hls_content), None) + }; + HlsMaterializedSharedManifest { + body: materialize_hls_access_manifest(&response_body, lease_id, server_path), + mode, + initial_strip_outcome, + } +} + +pub(in crate::api::endpoints::hls_api) struct HlsMaterializedSharedManifest { + pub(in crate::api::endpoints::hls_api) body: String, + pub(in crate::api::endpoints::hls_api) mode: &'static str, + pub(in crate::api::endpoints::hls_api) initial_strip_outcome: Option, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(in crate::api::endpoints::hls_api) enum HlsInitialStripLeaseSkipReason { + LeaseActivated, + LeaseNotStartupView, +} + +impl HlsInitialStripLeaseSkipReason { + pub(in crate::api::endpoints::hls_api) const fn as_log_reason(self) -> &'static str { + match self { + Self::LeaseActivated => "lease-activated", + Self::LeaseNotStartupView => "lease-not-startup-view", + } + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(in crate::api::endpoints::hls_api) enum HlsInitialStripPublicationDiagnostic { + Applied { mode: &'static str, strip_mode: &'static str, configured: u64, effective: usize, visible_segments: usize }, + Skipped { mode: &'static str, reason: HlsInitialStripSkipReason, visible_segments: usize }, + SkippedForLeaseState { mode: &'static str, reason: HlsInitialStripLeaseSkipReason }, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(in crate::api::endpoints::hls_api) enum HlsInitialStripPublicationStatus { + NotCommitted, + Committed, +} + +pub(in crate::api::endpoints::hls_api) fn hls_initial_strip_publication_diagnostic( + publication_status: HlsInitialStripPublicationStatus, + lease_state: HlsAccessLeaseState, + materialized: &HlsMaterializedSharedManifest, +) -> Option { + match publication_status { + HlsInitialStripPublicationStatus::NotCommitted => return None, + HlsInitialStripPublicationStatus::Committed => {} + } + Some(match &materialized.initial_strip_outcome { + Some(HlsInitialStripOutcome::Applied { mode: strip_mode, configured, effective, visible_segments }) => { + HlsInitialStripPublicationDiagnostic::Applied { + mode: materialized.mode, + strip_mode, + configured: *configured, + effective: *effective, + visible_segments: *visible_segments, + } + } + Some(HlsInitialStripOutcome::Skipped { reason, visible_segments }) => { + HlsInitialStripPublicationDiagnostic::Skipped { + mode: materialized.mode, + reason: *reason, + visible_segments: *visible_segments, + } + } + None => HlsInitialStripPublicationDiagnostic::SkippedForLeaseState { + mode: materialized.mode, + reason: if lease_state == HlsAccessLeaseState::Activated { + HlsInitialStripLeaseSkipReason::LeaseActivated + } else { + HlsInitialStripLeaseSkipReason::LeaseNotStartupView + }, + }, + }) +} + +pub(in crate::api::endpoints::hls_api) fn hls_entry_master_playlist_response( + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + bandwidth: HlsMasterBandwidth, + server_path: Option<&str>, +) -> HlsEntryMasterPlaylistResponse { + let path_prefix = normalize_hls_proxy_public_path_prefix(server_path).unwrap_or_default(); + let media_playlist_uri = format!("{path_prefix}{}", hls_canonical_manifest_path(proxy_session_id, access_lease_id)); + let body = HlsSingleVariantMasterPlaylist::new(bandwidth, media_playlist_uri).render().into_bytes(); + let content_length = body.len(); + let response = try_unwrap_body!(axum::response::Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "application/vnd.apple.mpegurl") + .header(header::CACHE_CONTROL, "private, no-store, no-cache, must-revalidate") + .header(header::CONTENT_LENGTH, content_length) + .body(Body::from(body))); + HlsEntryMasterPlaylistResponse { response, content_length } +} diff --git a/backend/app/src/api/endpoints/hls_api/mod.rs b/backend/app/src/api/endpoints/hls_api/mod.rs index c705b96ba..10c21bf12 100644 --- a/backend/app/src/api/endpoints/hls_api/mod.rs +++ b/backend/app/src/api/endpoints/hls_api/mod.rs @@ -14,112 +14,51 @@ use super::{ use crate::{ api::{ api_utils::{ - connection_priority_for_kind, create_api_proxy_user, create_m3u_catchup_session_key, - create_playback_session_fingerprint, create_recording_proxy_user, create_session_fingerprint, - force_hls_resource_response, get_headers_from_request, get_hls_session_ttl_secs, - get_stream_alternative_url, input_for_user, is_hls_stream_share_enabled, local_stream_response, - record_connect_failed_attempt, resolve_playback_request_admission, select_provider_stream_url, - try_option_bad_request, try_unwrap_body, ConnectFailedAttempt, EvictionReentryGuard, HeaderFilter, + connection_priority_for_kind, create_playback_session_fingerprint, force_hls_resource_response, + get_hls_session_ttl_secs, get_stream_alternative_url, resolve_playback_request_admission, + select_provider_stream_url, EvictionReentryGuard, }, model::{ - hls_cache::initial_strip::{ - materialize_initial_hls_strip_view, HlsInitialStripOutcome, HlsInitialStripSkipReason, - }, - hls_custom_video_manifest_response_for_access_lease, hls_custom_video_manifest_response_with_virtual_id, - hls_provisioning_discontinuity_sequence, hls_virtual_entry_redirect_response, - is_custom_video_stream_enabled, start_hls_panel_provisioning_once, - try_hls_panel_provisioning_manifest_response, AppState, ConnectionHistoryMode, CustomVideoStreamType, - GraceMode, HlsPanelProvisioningRedirectPaths, HlsProvisioningStatus, PlaybackLeaseRef, ProviderAllocation, - ProviderConfig as RuntimeProviderConfig, ProviderHandle, StreamMeterHandle, TransportStreamBuffer, - UserSession, + AppState, CustomVideoStreamType, GraceMode, PlaybackLeaseRef, ProviderAllocation, + ProviderConfig as RuntimeProviderConfig, ProviderHandle, UserSession, }, - panel_api::can_provision_on_exhausted, }, auth::{check_network_access_only, Fingerprint}, - model::{ - ConfigInput, ConfigInputFlags, ConfigProvider, ConfigTarget, InputSource, PlaybackKind, ProxyUserCredentials, - ReverseProxyDisabledHeaderConfig, - }, - processing::parser::hls::{ - get_hls_session_token_and_url_from_token, origin_manifest::HlsManifestWindowPolicy, rewrite_hls, - RewriteHlsProps, - }, - repository::{ - load_input_live_bitrate_bps, m3u_get_item_for_stream_id, persist_input_live_bitrate_bps, storage_const, - xtream_get_item_for_stream_id, LiveBitratePersistenceOutcome, - }, - utils::{content_coding::OutboundContentCodingPolicy, debug_if_enabled, request, request::is_file_url}, + model::{ConfigInput, ConfigProvider, ConfigTarget, PlaybackKind, ProxyUserCredentials}, + processing::parser::hls::origin_manifest::HlsManifestWindowPolicy, + repository::{load_input_live_bitrate_bps, m3u_get_item_for_stream_id, xtream_get_item_for_stream_id}, + utils::debug_if_enabled, }; use axum::{ - body::Body, - http::{header, HeaderMap, HeaderValue, Method, StatusCode}, + http::{header, HeaderMap, HeaderValue, StatusCode}, response::IntoResponse, }; -use futures::FutureExt; -use log::{debug, error, warn}; +use log::{debug, warn}; use serde::Deserialize; -use sha2::{Digest, Sha256}; use shared::{ - defaults::HLS_EXT, model::{ - ConnectFailureReason, FailureStage, InputType, PlaylistEntry, PlaylistItemType, StreamChannel, StreamInfo, - StreamProperties, TargetType, UserConnectionPermission, VirtualId, XtreamCluster, - }, - utils::{ - generate_random_string, is_m3u_catchup_session_token, replace_url_extension, sanitize_sensitive_info, - Internable, PROVIDER_SCHEME_PREFIX, + ConnectFailureReason, InputType, PlaylistEntry, PlaylistItemType, StreamChannel, StreamProperties, TargetType, + UserConnectionPermission, XtreamCluster, }, + utils::{is_m3u_catchup_session_token, sanitize_sensitive_info, Internable, PROVIDER_SCHEME_PREFIX}, }; -use std::{borrow::Cow, collections::HashMap, sync::Arc, time::Duration}; +use std::{collections::HashMap, sync::Arc, time::Duration}; use tuliprox_core::utils::current_time_millis; use tuliprox_hls::{ api::{ begin_hls_origin_account_io_bounded, build_hls_origin_session_owner, build_proxy_session_id, - cold_start_retry_after_seconds, commit_hls_runtime_custom_tail, derive_hls_lease_manifest_snapshot, - extract_hls_provider_session_headers, fetch_and_commit_hls_transient_origin_response_with_attempt_prepare, - fetch_hls_transient_origin_response_with_attempt_prepare, finite_hls_terminal_key_response, - force_identity_without_range, hls_cached_manifest_options_for_requirement, - hls_committed_manifest_body_for_request, hls_manifest_acceptance_directive_for_session, - hls_manifest_commit_requirement, hls_object_body_deadline, hls_origin_account_status, - hls_should_wait_for_initial_manifest_commit, hls_startup_admission_allows_snapshot, - hls_transient_object_fetch_failure, hls_transient_origin_response, is_hls_provisioning_gap_segment, - is_hls_provisioning_segment, maybe_trigger_origin_refresh_with_outcome, new_hls_access_lease_id, - origin_account_binding_from_allocation, record_successful_transient_segment_fetch, - record_temporary_transient_segment_fetch_failure, register_hls_availability_reevaluation, - resolve_hls_transient_object_cache_action, safe_hls_access_lease_id, safe_proxy_session_id, safe_session_key, - safe_user_session_token, scrub_hls_origin_headers, serve_hls_map_cache_outcome, - serve_hls_segment_cache_outcome, serve_hls_transient_object_cache_outcome, - serve_hls_transient_object_cache_response, should_remove_hls_origin_header, trigger_origin_refresh_sync, - validate_hls_access_lease, CacheAccessState, HlsAccessAdmissionMode, HlsAccessContext, HlsAccessLease, - HlsAccessLeaseActivation, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseState, - HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsAccessLeaseValidationError, HlsAccountBindingProtection, - HlsAccountOverlapTiming, HlsAvailabilityReevaluationObservation, HlsAvailabilityReevaluationRegistration, - HlsBandwidthPersistenceOutcome, HlsBoundAccountAcquireErrorKind, HlsCacheResponseContext, - HlsCachedManifestOptions, HlsCommittedManifestBody, HlsEffectiveOriginAcquirePolicy, HlsLeaseManifestSnapshot, - HlsLeaseManifestSnapshotInput, HlsLeaseManifestUriMaterialization, HlsLeasePlaybackMode, - HlsLeaseStartupAdmissionState, HlsLogIdentity, HlsManifestAcceptanceDirective, - HlsManifestAcceptanceEvaluationOutcome, HlsManifestCommitIdentity, HlsManifestCommitRequirement, - HlsManifestLimitViolation, HlsMapFile, HlsMasterBandwidth, HlsMasterBandwidthSelection, - HlsMediaActivityCommitOutcome, HlsMediaActivityMarker, HlsMediaLeaseIdentity, HlsOriginAccountBinding, - HlsOriginAccountBindingMode, HlsOriginAccountDetachedReason, HlsOriginAccountStatus, HlsOriginIoContext, - HlsOriginRefreshTriggerOutcome, HlsOriginResourceClients, HlsOriginResourceFetchError, HlsOriginSource, - HlsOriginSourceKind, HlsOriginWorkClass, HlsPlaybackFamilyKey, HlsPostRefreshRuntime, - HlsPublishedTransientResourceIds, HlsQosMeterInit, HlsQosRuntimeConfig, HlsResourceFetchAttempt, - HlsResourceServeFailure, HlsResourceServeOutcome, HlsRuntimeCustomTailOutcome, HlsRuntimeCustomTailReason, - HlsRuntimeCustomTailRequest, HlsSegmentFile, HlsSession, HlsSessionHandle, HlsSessionKey, HlsSessionMode, - HlsSessionStoreOutcome, HlsSingleVariantMasterPlaylist, HlsTerminalFailedClosedReason, HlsTerminalSegmentPath, - HlsTransientCacheCommitContext, HlsTransientDecodedOriginResponse, HlsTransientDirectResponseContext, - HlsTransientManifestTemplate, HlsTransientObjectCacheAction, HlsTransientObjectFetchFailure, - HlsTransientObjectFetchFinalizer, HlsTransientOriginCacheFetchRequest, HlsTransientOriginFetchRequest, - HlsTransientOriginIoGuard, HlsTransientResourceLeaseContext, LiveHlsOriginEntry, OriginRefreshRequest, - OriginSegmentKey, ProxySessionId, RetryPolicy, SegmentCacheKey, SegmentCacheStatus, SegmentDemandFetchOutcome, - SegmentEntry, SegmentFetchContext, SegmentFetchPolicy, TransientManifestGeneration, TransientObjectFetchToken, - TransientObjectUnavailableState, TransientPassthroughState, TransientResourceFile, TransientResourceId, - TransientResourceKind, TransientResourceRef, HLS_ACCESS_LEASE_ID_PLACEHOLDER, - HLS_PROVISIONING_GAP_ORIGIN_EPOCH, HLS_PROVISIONING_ORIGIN_EPOCH, HLS_PROVISIONING_SEGMENT_DURATION_MS, - HLS_PROVISIONING_TARGET_DURATION_SECS, MAX_HLS_MANIFEST_BYTES, + cold_start_retry_after_seconds, hls_object_body_deadline, hls_origin_account_status, new_hls_access_lease_id, + origin_account_binding_from_allocation, safe_hls_access_lease_id, safe_proxy_session_id, safe_session_key, + safe_user_session_token, HlsAccessContext, HlsAccessLease, HlsAccessLeaseState, HlsAccountBindingProtection, + HlsEffectiveOriginAcquirePolicy, HlsLogIdentity, HlsManifestCommitIdentity, HlsMasterBandwidth, + HlsMasterBandwidthSelection, HlsMediaLeaseIdentity, HlsOriginAccountBinding, HlsOriginAccountBindingMode, + HlsOriginAccountDetachedReason, HlsOriginAccountStatus, HlsOriginIoContext, HlsOriginSource, + HlsOriginSourceKind, HlsOriginWorkClass, HlsPlaybackFamilyKey, HlsPublishedTransientResourceIds, + HlsSegmentFile, HlsSessionHandle, HlsSessionMode, HlsSingleVariantMasterPlaylist, HlsTransientManifestTemplate, + HlsTransientOriginIoGuard, ProxySessionId, SegmentCacheStatus, TransientManifestGeneration, + TransientResourceFile, }, - HlsCtx, MAX_MANUAL_REDIRECTS, + HlsCtx, }; use url::Url; diff --git a/backend/app/src/api/endpoints/hls_api/segment.rs b/backend/app/src/api/endpoints/hls_api/segment.rs index 0879b612d..cfe09388f 100644 --- a/backend/app/src/api/endpoints/hls_api/segment.rs +++ b/backend/app/src/api/endpoints/hls_api/segment.rs @@ -1,751 +1,6 @@ #![allow(clippy::wildcard_imports)] + use super::*; -use crate::{ - api::api_utils::{ - acquire_exact_provider_handle, get_hls_playback_ttl_secs, ExactProviderAcquire, HLS_MANIFEST_CAPACITY_WAIT, - }, - processing::parser::hls::{classify_hls_playlist, HlsManifestSource, HlsPlaylistKind, HlsResourceKind}, -}; -use tuliprox_core::model::{PlaybackRequestId, PlaybackRequestOutcome, ProviderBindingTag}; - -pub(super) fn log_hls_initial_strip_publication( - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, - diagnostic: HlsInitialStripPublicationDiagnostic, -) { - match diagnostic { - HlsInitialStripPublicationDiagnostic::Applied { mode, strip_mode, configured, effective, visible_segments } => { - debug!( - "HLS initial strip applied: mode={} lease={} proxy_session={} strip_mode={} configured={} effective={} visible_segments={}", - mode, - safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(proxy_session_id), - strip_mode, - configured, - effective, - visible_segments - ); - } - HlsInitialStripPublicationDiagnostic::Skipped { mode, reason, visible_segments } => { - debug!( - "HLS initial strip skipped: mode={} lease={} proxy_session={} reason={} visible_segments={}", - mode, - safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(proxy_session_id), - reason.as_log_reason(), - visible_segments - ); - } - HlsInitialStripPublicationDiagnostic::SkippedForLeaseState { mode, reason } => { - debug!( - "HLS initial strip skipped: mode={} lease={} proxy_session={} reason={}", - mode, - safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(proxy_session_id), - reason.as_log_reason() - ); - } - } -} - -pub(super) fn stripped_tail_segments(materialized: &HlsMaterializedSharedManifest) -> usize { - materialized.initial_strip_outcome.as_ref().map_or(0, |outcome| match outcome { - HlsInitialStripOutcome::Applied { effective, .. } => *effective, - HlsInitialStripOutcome::Skipped { .. } => 0, - }) -} - -pub(super) fn hls_access_lease_ttl_ms(app_state: &Arc) -> u64 { - app_state.hls.proxy.session_idle_timeout_ms() -} - -pub(super) fn duration_to_millis_saturating(duration: Duration) -> u64 { - u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) -} - -pub(super) fn hls_pending_bootstrap_window_ms(app_state: &Arc) -> u64 { - duration_to_millis_saturating(hls_initial_manifest_decision_wait_timeout(app_state)) -} - -pub(super) async fn hls_access_lease_timing_for_session( - app_state: &Arc, - session: &HlsSessionHandle, -) -> HlsAccessLeaseTiming { - let timing = session.read().await.account_overlap_timing(); - let active_window_ms = timing.hard_active_window_ms.saturating_mul(2); - HlsAccessLeaseTiming { active_window_ms, valid_window_ms: hls_access_lease_ttl_ms(app_state) } -} - -pub(super) async fn touch_pending_manifest_follow_up_window( - app_state: &Arc, - session: &HlsSessionHandle, - access_lease_id: &HlsAccessLeaseId, - access_lease_state: HlsAccessLeaseState, -) { - if access_lease_state != HlsAccessLeaseState::Pending { - return; - } - let (proxy_session_id, target_duration) = { - let session = session.read().await; - (session.proxy_session_id.clone(), session.target_duration) - }; - let now_ms = current_time_millis(); - if !app_state - .hls - .proxy - .mark_pending_manifest_follow_up_for_lease(access_lease_id, &proxy_session_id, now_ms, target_duration) - .await - { - debug!( - "HLS pending manifest follow-up skipped: lease={} proxy_session={} reason=expired-or-generation-race", - safe_hls_access_lease_id(access_lease_id), - safe_proxy_session_id(&proxy_session_id) - ); - } -} - -pub(super) struct HlsResourceAccess { - pub(super) session: HlsSessionHandle, - pub(super) access_context: HlsAccessContext, - pub(super) lease: HlsAccessLease, -} - -pub(super) async fn prepare_hls_resource_access( - app_state: &Arc, - fingerprint: &Fingerprint, - proxy_session_id: &ProxySessionId, - hls_access_lease_id: &str, - now_ms: u64, - request_kind: &'static str, -) -> Result> { - let Some(session) = app_state.hls.proxy.sessions().get_by_proxy_session_id(proxy_session_id).await else { - return Err(Box::new(StatusCode::NOT_FOUND.into_response())); - }; - app_state - .hls - .proxy - .sync_session_access_lease_count_and_detach_if_needed( - &app_state.active_users, - &app_state.active_provider, - &session, - proxy_session_id, - now_ms, - ) - .await; - let access_context = match validate_hls_proxy_access_request( - app_state, - fingerprint, - proxy_session_id, - hls_access_lease_id, - now_ms, - hls_access_lease_timing_for_session(app_state, &session).await, - request_kind, - ) - .await - { - Ok(context) => context, - Err(err) => { - return Err(Box::new(hls_resource_access_lease_validation_response(&err))); - } - }; - reclaim_hls_account_overlap_if_needed(app_state, &session, now_ms).await; - let Some(lease) = - app_state.hls.proxy.access_lease_response_snapshot(&access_context.lease_id, proxy_session_id, now_ms).await - else { - return Err(Box::new(StatusCode::NOT_FOUND.into_response())); - }; - Ok(HlsResourceAccess { session, access_context, lease }) -} - -pub(super) fn hls_lease_allows_live_origin_work(lease: &HlsAccessLease) -> bool { - matches!(lease.state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle) - && lease.playback_mode == HlsLeasePlaybackMode::Live -} - -pub(super) fn hls_lease_allows_cached_segment(lease: &HlsAccessLease, proxy_seq: u64) -> bool { - match &lease.playback_mode { - HlsLeasePlaybackMode::Live => true, - HlsLeasePlaybackMode::TerminalTail(plan) => plan.protected_base_proxy_seqs.contains(&proxy_seq), - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => false, - } -} - -pub(super) async fn current_hls_resource_lease( - app_state: &Arc, - access_context: &HlsAccessContext, -) -> Option { - app_state - .hls - .proxy - .access_lease_response_snapshot( - &access_context.lease_id, - &access_context.proxy_session_id, - current_time_millis(), - ) - .await -} - -pub(super) async fn hls_live_lease_identity_is_current( - app_state: &Arc, - access_context: &HlsAccessContext, - expected_identity: HlsMediaLeaseIdentity, -) -> bool { - current_hls_resource_lease(app_state, access_context).await.is_some_and(|lease| { - lease.playback_mode == HlsLeasePlaybackMode::Live && lease.media_identity() == Some(expected_identity) - }) -} - -pub(super) fn create_hls_cache_user_session_token( - fingerprint: &Fingerprint, - username: &str, - virtual_id: u32, - existing_session_token: Option<&str>, - archive_reference: Option, -) -> String { - let base = - hls_entry_user_session_token(fingerprint, username, virtual_id, existing_session_token, archive_reference); - format!("{base}|hls-cache|{}", generate_random_string(16)) -} - -pub(super) fn is_hls_media_activity_status(status: StatusCode) -> bool { - matches!(status, StatusCode::OK | StatusCode::PARTIAL_CONTENT) -} - -pub(super) async fn hls_cache_response_context( - app_state: &Arc, - session: &HlsSessionHandle, - access_context: &HlsAccessContext, - lease_identity: HlsMediaLeaseIdentity, - now_ms: u64, -) -> HlsCacheResponseContext { - let qos_meter = app_state.hls.proxy.qos().meter_for_access_lease(&access_context.lease_id).await; - let (log_identity, session_owner, playback_request_id) = { - let session = session.read().await; - let binding = session.origin_account_binding.as_ref().filter(|binding| binding.is_active()); - ( - HlsLogIdentity::from_session(&session), - binding.map(|binding| binding.session_owner.clone()), - binding.and_then(|binding| binding.playback_request_id), - ) - }; - HlsCacheResponseContext::new( - access_context.lease_id.clone(), - log_identity, - app_state.hls.proxy.cache_duration_seconds(), - Arc::clone(app_state.hls.proxy.metrics()), - Arc::clone(app_state.hls.proxy.segment_repair()), - qos_meter, - Some( - HlsMediaActivityMarker::new( - Arc::clone(&app_state.hls.proxy), - Arc::clone(session), - access_context.proxy_session_id.clone(), - access_context.lease_id.clone(), - lease_identity, - ) - .with_active_provider( - Arc::clone(&app_state.active_provider), - session_owner, - playback_request_id, - ), - ), - now_ms, - ) -} - -pub(super) fn hls_qos_meter_init( - app_state: &Arc, - qos_config: HlsQosRuntimeConfig, -) -> Option { - if !qos_config.live_metering_enabled { - return None; - } - let meter_uid = app_state.connection_manager.next_stream_uid(); - let meter = Arc::new(StreamMeterHandle::new(meter_uid, Arc::downgrade(&app_state.event_manager))); - Some(HlsQosMeterInit { meter_uid, meter }) -} - -pub(super) async fn register_hls_cache_stream_for_successful_media_response( - app_state: &Arc, - fingerprint: &Fingerprint, - headers: &HeaderMap, - access_context: &HlsAccessContext, - session: &HlsSessionHandle, - response_context: &HlsCacheResponseContext, -) { - if ensure_hls_cache_stream_registered(app_state, fingerprint, headers, access_context, session).await.is_none() { - debug!( - "HLS media registration skipped: lease={} reason=session-or-connection-unavailable", - safe_hls_access_lease_id(&access_context.lease_id) - ); - } - response_context.set_qos_meter(app_state.hls.proxy.qos().meter_for_access_lease(&access_context.lease_id).await); -} - -pub(super) async fn hls_proxy_segment( - fingerprint: Fingerprint, - axum::extract::Path(params): axum::extract::Path, - axum::extract::State(app_state): axum::extract::State>, - headers: HeaderMap, -) -> axum::response::Response { - let proxy_session_id = ProxySessionId(params.proxy_session_id); - let now_ms = current_time_millis(); - let HlsResourceAccess { session, access_context, lease } = match prepare_hls_resource_access( - &app_state, - &fingerprint, - &proxy_session_id, - ¶ms.hls_access_lease_id, - now_ms, - "segment", - ) - .await - { - Ok(access) => access, - Err(response) => return *response, - }; - let Some(segment_file) = HlsSegmentFile::parse(¶ms.segment_file) else { - return hls_resource_channel_unavailable_response(&app_state, &access_context); - }; - if !hls_lease_allows_cached_segment(&lease, segment_file.proxy_seq) { - return StatusCode::NOT_FOUND.into_response(); - } - let allows_origin_work = hls_lease_allows_live_origin_work(&lease); - if let Err(response) = - validate_hls_segment_entry(&app_state, &session, &access_context, &segment_file, allows_origin_work).await - { - return *response; - } - let demand_result = if allows_origin_work { - demand_fetch_hls_live_segment( - &app_state, - &session, - &segment_file, - &access_context, - &fingerprint, - &headers, - now_ms, - ) - .await - } else { - Ok(()) - }; - if let Err(response) = demand_result { - return *response; - } - - serve_hls_segment_for_current_lease( - &app_state, - &session, - &access_context, - &fingerprint, - &headers, - segment_file, - now_ms, - ) - .await -} - -pub(super) async fn validate_hls_segment_entry( - app_state: &Arc, - session: &HlsSessionHandle, - access_context: &HlsAccessContext, - segment_file: &HlsSegmentFile, - allows_origin_work: bool, -) -> Result<(), Box> { - let session = session.read().await; - if session.is_gc_marked_for_removal() { - return Err(Box::new(StatusCode::NOT_FOUND.into_response())); - } - let Some(entry) = session.segments.get(&segment_file.proxy_seq) else { - return Err(Box::new(hls_resource_channel_unavailable_response(app_state, access_context))); - }; - if entry.proxy_file_ext != segment_file.extension { - return Err(Box::new(hls_resource_channel_unavailable_response(app_state, access_context))); - } - if !allows_origin_work && !matches!(&entry.status, SegmentCacheStatus::Ready { .. }) { - return Err(Box::new(StatusCode::NOT_FOUND.into_response())); - } - Ok(()) -} - -pub(super) async fn demand_fetch_hls_live_segment( - app_state: &Arc, - session: &HlsSessionHandle, - segment_file: &HlsSegmentFile, - access_context: &HlsAccessContext, - fingerprint: &Fingerprint, - headers: &HeaderMap, - now_ms: u64, -) -> Result<(), Box> { - let preacquired_provider_handle = if hls_segment_request_requires_origin_work(session, segment_file).await { - match prepare_hls_origin_binding_for_authorized_resource_work( - app_state, - session, - access_context, - fingerprint, - headers, - HlsOriginWorkKind::Segment, - now_ms, - ) - .await - { - Ok(handle) => handle, - Err(err) => { - return Err(Box::new(hls_origin_runtime_resource_failure_response(app_state, access_context, err))) - } - } - } else { - None - }; - match demand_fetch_hls_segment_if_needed( - app_state, - session, - segment_file, - access_context, - fingerprint, - preacquired_provider_handle, - now_ms, - ) - .await - { - SegmentDemandFetchOutcome::NotFound => { - Err(Box::new(hls_resource_channel_unavailable_response(app_state, access_context))) - } - SegmentDemandFetchOutcome::Ready - | SegmentDemandFetchOutcome::QueuedOrFetching - | SegmentDemandFetchOutcome::Unavailable - | SegmentDemandFetchOutcome::TimedOut => Ok(()), - } -} - -pub(super) async fn serve_hls_segment_for_current_lease( - app_state: &Arc, - session: &HlsSessionHandle, - access_context: &HlsAccessContext, - fingerprint: &Fingerprint, - headers: &HeaderMap, - segment_file: HlsSegmentFile, - now_ms: u64, -) -> axum::response::Response { - let Some(current_lease) = current_hls_resource_lease(app_state, access_context).await else { - return StatusCode::NOT_FOUND.into_response(); - }; - if !hls_lease_allows_cached_segment(¤t_lease, segment_file.proxy_seq) { - return StatusCode::NOT_FOUND.into_response(); - } - let current_allows_origin_work = hls_lease_allows_live_origin_work(¤t_lease); - let Some(lease_identity) = current_lease.media_identity() else { - return StatusCode::NOT_FOUND.into_response(); - }; - let response_context = hls_cache_response_context(app_state, session, access_context, lease_identity, now_ms).await; - let response = hls_resource_serve_outcome_response( - app_state, - access_context, - serve_hls_segment_cache_outcome( - Arc::clone(app_state.hls.proxy.segment_cache()), - Arc::clone(session), - segment_file, - headers.get(header::RANGE).cloned(), - &response_context, - ) - .await, - ); - if current_allows_origin_work && is_hls_media_activity_status(response.status()) { - register_hls_cache_stream_for_successful_media_response( - app_state, - fingerprint, - headers, - access_context, - session, - &response_context, - ) - .await; - } - response -} - -pub(super) async fn hls_proxy_terminal_segment( - fingerprint: Fingerprint, - axum::extract::Path(params): axum::extract::Path, - axum::extract::State(app_state): axum::extract::State>, - method: Method, - headers: HeaderMap, -) -> axum::response::Response { - let proxy_session_id = ProxySessionId(params.proxy_session_id); - let Some(path) = HlsTerminalSegmentPath::parse(¶ms.generation, ¶ms.terminal_file) else { - return StatusCode::NOT_FOUND.into_response(); - }; - let now_ms = current_time_millis(); - let access_lease_id = HlsAccessLeaseId(params.hls_access_lease_id.clone()); - let immutable_replay_plan = - app_state.hls.proxy.access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms).await.and_then( - |lease| match lease.playback_mode { - HlsLeasePlaybackMode::TerminalTail(plan) if plan.matches_route(&proxy_session_id, &access_lease_id) => { - Some(plan) - } - HlsLeasePlaybackMode::Live - | HlsLeasePlaybackMode::TerminalTail(_) - | HlsLeasePlaybackMode::TerminalUnavailable { .. } - | HlsLeasePlaybackMode::Ended => None, - }, - ); - let access = prepare_hls_resource_access( - &app_state, - &fingerprint, - &proxy_session_id, - ¶ms.hls_access_lease_id, - now_ms, - "terminal-segment", - ) - .await; - let Ok(HlsResourceAccess { session, access_context, lease }) = access else { - return immutable_replay_plan - .and_then(|plan| { - terminal_segment_immutable_replay_response( - &plan, - path, - headers.get(header::RANGE), - method == Method::HEAD, - ) - }) - .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); - }; - let Some(current_lease) = current_hls_resource_lease(&app_state, &access_context).await else { - return StatusCode::NOT_FOUND.into_response(); - }; - let Some(plan) = - terminal_tail_plan_for_current_route(&lease, ¤t_lease, &proxy_session_id, &access_context.lease_id) - else { - return StatusCode::NOT_FOUND.into_response(); - }; - if method == Method::HEAD { - return terminal_segment_head_response(&plan, path, headers.get(header::RANGE)) - .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); - } - let Some(lease_identity) = current_lease.media_identity() else { - return StatusCode::NOT_FOUND.into_response(); - }; - let response_context = - hls_cache_response_context(&app_state, &session, &access_context, lease_identity, now_ms).await; - let Some(response) = - terminal_segment_get_response(&plan, path, headers.get(header::RANGE), &response_context, &proxy_session_id) - else { - return StatusCode::NOT_FOUND.into_response(); - }; - if is_hls_media_activity_status(response.status()) { - response_context.mark_media_activity().await; - register_hls_cache_stream_for_successful_media_response( - &app_state, - &fingerprint, - &headers, - &access_context, - &session, - &response_context, - ) - .await; - } - response -} - -pub(super) async fn demand_fetch_hls_segment_if_needed( - app_state: &Arc, - session: &HlsSessionHandle, - segment_file: &HlsSegmentFile, - access_context: &HlsAccessContext, - fingerprint: &Fingerprint, - preacquired_provider_handle: Option, - now_ms: u64, -) -> SegmentDemandFetchOutcome { - let context = build_hls_segment_fetch_context( - app_state, - session, - Some(access_context.lease_id.clone()), - fingerprint, - preacquired_provider_handle, - ) - .await; - app_state.hls.proxy.segment_worker_pool().demand_fetch_and_wait(context, segment_file, now_ms).await -} - -pub(super) async fn build_hls_segment_fetch_context( - app_state: &Arc, - session: &HlsSessionHandle, - repair_access_lease_id: Option, - fingerprint: &Fingerprint, - preacquired_provider_handle: Option, -) -> SegmentFetchContext { - let (headers, origin_provider_session_headers, origin_policy, reservation_ttl_secs) = { - let session = session.read().await; - ( - session.origin_request_headers.clone(), - session.origin_provider_session_headers.clone(), - session.effective_origin_acquire_policy_or_default(), - session.account_overlap_timing().reservation_ttl_secs(), - ) - }; - let mut origin_io = HlsOriginIoContext { - ctx: app_state.hls_ctx(), - client_addr: fingerprint.addr, - allow_grace: HlsOriginWorkClass::Demand.allows_grace(), - priority: origin_policy.priority, - connection_kind: origin_policy.connection_kind, - reservation_ttl_secs, - preacquired_provider_handle: None, - started_generation: None, - }; - if let Some(provider_handle) = preacquired_provider_handle { - origin_io = origin_io.with_preacquired_provider_handle(provider_handle); - } - SegmentFetchContext { - session: Arc::clone(session), - segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()), - segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()), - repair_access_lease_id, - headers, - origin_provider_session_headers, - client: app_state.http_clients.default.load().as_ref().clone(), - no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(), - use_manual_redirects: app_state.should_use_manual_redirects(), - origin_io: Some(origin_io), - } -} - -pub(super) async fn hls_effective_origin_acquire_policy(session: &HlsSessionHandle) -> HlsEffectiveOriginAcquirePolicy { - session.read().await.effective_origin_acquire_policy_or_default() -} - -pub(super) async fn hls_origin_account_reservation_ttl_secs_for_session(session: &HlsSessionHandle) -> u64 { - session.read().await.account_overlap_timing().reservation_ttl_secs() -} - -pub(super) fn hls_origin_account_reservation_ttl_secs_fallback() -> u64 { - HlsAccountOverlapTiming::from_target_duration_secs(None).reservation_ttl_secs() -} - -pub(super) async fn hls_proxy_map( - fingerprint: Fingerprint, - axum::extract::Path(params): axum::extract::Path, - axum::extract::State(app_state): axum::extract::State>, - headers: HeaderMap, -) -> axum::response::Response { - let proxy_session_id = ProxySessionId(params.proxy_session_id); - let now_ms = current_time_millis(); - let HlsResourceAccess { session, access_context, lease } = match prepare_hls_resource_access( - &app_state, - &fingerprint, - &proxy_session_id, - ¶ms.hls_access_lease_id, - now_ms, - "map", - ) - .await - { - Ok(access) => access, - Err(response) => return *response, - }; - if !hls_lease_allows_live_origin_work(&lease) { - return StatusCode::NOT_FOUND.into_response(); - } - let Some(map_file) = HlsMapFile::parse(¶ms.map_file) else { - return hls_resource_channel_unavailable_response(&app_state, &access_context); - }; - { - let session_guard = session.read().await; - if session_guard.is_gc_marked_for_removal() { - return StatusCode::NOT_FOUND.into_response(); - } - let Some(entry) = session_guard.maps.get(&map_file.proxy_map_id.into()) else { - return hls_resource_channel_unavailable_response(&app_state, &access_context); - }; - if entry.proxy_file_ext != map_file.extension { - return hls_resource_channel_unavailable_response(&app_state, &access_context); - } - } - - let Some(current_lease) = current_hls_resource_lease(&app_state, &access_context).await else { - return StatusCode::NOT_FOUND.into_response(); - }; - if current_lease.playback_mode != HlsLeasePlaybackMode::Live { - return StatusCode::NOT_FOUND.into_response(); - } - let Some(lease_identity) = current_lease.media_identity() else { - return StatusCode::NOT_FOUND.into_response(); - }; - let response_context = - hls_cache_response_context(&app_state, &session, &access_context, lease_identity, now_ms).await; - let response = hls_resource_serve_outcome_response( - &app_state, - &access_context, - serve_hls_map_cache_outcome( - Arc::clone(app_state.hls.proxy.segment_cache()), - Arc::clone(&session), - map_file, - headers.get(header::RANGE).cloned(), - &response_context, - ) - .await, - ); - if is_hls_media_activity_status(response.status()) { - register_hls_cache_stream_for_successful_media_response( - &app_state, - &fingerprint, - &headers, - &access_context, - &session, - &response_context, - ) - .await; - } - response -} - -pub(super) async fn hls_proxy_resource( - fingerprint: Fingerprint, - axum::extract::Path(params): axum::extract::Path, - axum::extract::State(app_state): axum::extract::State>, - headers: HeaderMap, -) -> axum::response::Response { - let proxy_session_id = ProxySessionId(params.proxy_session_id); - let now_ms = current_time_millis(); - let HlsResourceAccess { session, access_context, lease } = match prepare_hls_resource_access( - &app_state, - &fingerprint, - &proxy_session_id, - ¶ms.hls_access_lease_id, - now_ms, - "resource", - ) - .await - { - Ok(access) => access, - Err(response) => return *response, - }; - let Some(resource_file) = TransientResourceFile::parse(¶ms.resource_file) else { - return hls_resource_channel_unavailable_response(&app_state, &access_context); - }; - let Some(lease_identity) = lease.media_identity() else { - return StatusCode::NOT_FOUND.into_response(); - }; - let context = HlsResourceEndpointContext { - app_state: &app_state, - session: &session, - fingerprint: &fingerprint, - headers: &headers, - access_context: &access_context, - lease_identity, - published_resource_ids: lease.published_transient_resource_ids().clone(), - resource_file, - range_header: headers.get(header::RANGE).cloned(), - now_ms, - }; - match &lease.playback_mode { - HlsLeasePlaybackMode::Live => serve_hls_live_transient_resource(context).await, - HlsLeasePlaybackMode::TerminalTail(_) => serve_hls_terminal_key_resource(context, &lease.playback_mode).await, - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => { - StatusCode::NOT_FOUND.into_response() - } - } -} pub(super) struct HlsResourceEndpointContext<'a> { pub(super) app_state: &'a Arc, @@ -760,2570 +15,71 @@ pub(super) struct HlsResourceEndpointContext<'a> { pub(super) now_ms: u64, } -pub(super) async fn serve_hls_terminal_key_resource( - context: HlsResourceEndpointContext<'_>, - playback_mode: &HlsLeasePlaybackMode, -) -> axum::response::Response { - let HlsLeasePlaybackMode::TerminalTail(plan) = playback_mode else { - return StatusCode::NOT_FOUND.into_response(); - }; - let proxy_session_id = &context.access_context.proxy_session_id; - let Some(binding) = - plan.terminal_key_binding(proxy_session_id, &context.access_context.lease_id, &context.resource_file) - else { - return StatusCode::NOT_FOUND.into_response(); - }; - if !context.session.read().await.terminal_key_binding_is_current( - &context.access_context.lease_id, - plan.generation, - &binding, - ) { - return StatusCode::NOT_FOUND.into_response(); - } - let response_context = hls_cache_response_context( - context.app_state, - context.session, - context.access_context, - context.lease_identity, - context.now_ms, - ) - .await; - let response = finite_hls_terminal_key_response( - binding.bytes(), - context.range_header.as_ref(), - binding.content_type(), - "private, max-age=300, immutable", - &response_context, - proxy_session_id, - context.resource_file.resource_id.0, - ); - if is_hls_media_activity_status(response.status()) { - response_context.mark_media_activity().await; - register_hls_cache_stream_for_successful_media_response( - context.app_state, - context.fingerprint, - context.headers, - context.access_context, - context.session, - &response_context, - ) - .await; - } - response -} - -pub(super) async fn serve_hls_live_transient_resource( - context: HlsResourceEndpointContext<'_>, -) -> axum::response::Response { - let cache_duration_ms = context.app_state.hls.proxy.cache_duration_seconds().saturating_mul(1_000); - let Ok(cache_resolution) = resolve_hls_transient_object_cache_action( - context.session, - &context.access_context.proxy_session_id, - HlsTransientResourceLeaseContext { - access_lease_id: &context.access_context.lease_id, - lease_issued_at_ms: context.lease_identity.lease_issued_at_ms(), - published_resource_ids: &context.published_resource_ids, - }, - &context.resource_file, - context.range_header.as_ref(), - context.now_ms, - cache_duration_ms, - ) - .await - else { - return hls_resource_channel_unavailable_response(context.app_state, context.access_context); - }; - let resource = cache_resolution.resource; - let origin_headers = cache_resolution.origin_headers; - let origin_provider_session_headers = cache_resolution.origin_provider_session_headers; - let cache_action = cache_resolution.action; - - match cache_action { - HlsTransientObjectCacheAction::ServeReady => { - return serve_transient_object_cache_response_and_mark_or_unavailable(TransientObjectCacheServeContext { - app_state: context.app_state, - session: context.session, - fingerprint: context.fingerprint, - headers: context.headers, - access_context: context.access_context, - lease_identity: context.lease_identity, - resource_file: context.resource_file, - range_header: context.range_header, - now_ms: context.now_ms, - }) - .await; - } - HlsTransientObjectCacheAction::WaitForFetch(notifier) => { - return wait_for_transient_object_cache_fetch(TransientObjectWaitContext { - app_state: context.app_state, - session: context.session, - fingerprint: context.fingerprint, - headers: context.headers, - access_context: context.access_context, - lease_identity: context.lease_identity, - resource_file: context.resource_file, - range_header: context.range_header, - notifier, - }) - .await; - } - HlsTransientObjectCacheAction::FetchAndCache(_) | HlsTransientObjectCacheAction::PassthroughNoCache => {} - } - - fetch_or_passthrough_transient_resource(HlsTransientPassthroughContext { - endpoint: context, - resource, - cache_action, - origin_headers, - origin_provider_session_headers, - cache_duration_ms, - }) - .await -} - -pub(super) struct HlsTransientPassthroughContext<'a> { - pub(super) endpoint: HlsResourceEndpointContext<'a>, - pub(super) resource: TransientResourceRef, - pub(super) cache_action: HlsTransientObjectCacheAction, - pub(super) origin_headers: HeaderMap, - pub(super) origin_provider_session_headers: HeaderMap, - pub(super) cache_duration_ms: u64, -} - -pub(super) async fn fetch_or_passthrough_transient_resource( - context: HlsTransientPassthroughContext<'_>, -) -> axum::response::Response { - let HlsTransientPassthroughContext { - endpoint, - resource, - cache_action, - origin_headers, - origin_provider_session_headers, - cache_duration_ms, - } = context; - if let HlsTransientObjectCacheAction::FetchAndCache(fetch_token) = cache_action { - return fetch_and_cache_transient_origin_response(HlsTransientEndpointCacheFetchContext { - app_state: endpoint.app_state, - session: endpoint.session, - fingerprint: endpoint.fingerprint, - headers: endpoint.headers, - access_context: endpoint.access_context, - lease_identity: endpoint.lease_identity, - resource: &resource, - resource_file: endpoint.resource_file, - fetch_token: *fetch_token, - origin_headers, - origin_provider_session_headers, - range_header: endpoint.range_header, - cache_duration_ms, - }) - .await; - } - - let policy = endpoint.app_state.hls.proxy.segment_fetch_policy(); - let fetch_result = fetch_transient_origin_response_with_provider_io(HlsTransientEndpointOriginFetchRequest { - app_state: endpoint.app_state, - session: endpoint.session, - access_context: endpoint.access_context, - fingerprint: endpoint.fingerprint, - headers: endpoint.headers, - resource: &resource, - resource_file: &endpoint.resource_file, - origin_headers, - origin_provider_session_headers, - range_header: endpoint.range_header.clone(), - policy: policy.clone(), - }) - .await; - serve_hls_transient_passthrough_result(endpoint, resource, policy, fetch_result).await -} - -#[allow(clippy::too_many_lines)] -pub(super) async fn serve_hls_transient_passthrough_result( - endpoint: HlsResourceEndpointContext<'_>, - resource: TransientResourceRef, - policy: SegmentFetchPolicy, - fetch_result: HlsTransientOriginFetchResult, -) -> axum::response::Response { - match fetch_result.result { - Ok(response) => { - if response.decoded.status.is_success() { - let activity_outcome = endpoint - .app_state - .hls - .proxy - .mark_authorized_media_access_for_lease_if_identity_matches( - endpoint.session, - &endpoint.access_context.lease_id, - &endpoint.access_context.proxy_session_id, - endpoint.lease_identity, - endpoint.now_ms, - ) - .await; - match activity_outcome { - HlsMediaActivityCommitOutcome::Committed => {} - HlsMediaActivityCommitOutcome::StaleLeaseIdentity => { - debug!( - "HLS transient media response discarded: lease={} proxy_session={} reason=playback-generation-race", - safe_hls_access_lease_id(&endpoint.access_context.lease_id), - safe_proxy_session_id(&endpoint.access_context.proxy_session_id) - ); - return StatusCode::NOT_FOUND.into_response(); - } - HlsMediaActivityCommitOutcome::DeferredLockContention => { - debug!( - "HLS transient media response deferred: lease={} proxy_session={} reason=lock-contention", - safe_hls_access_lease_id(&endpoint.access_context.lease_id), - safe_proxy_session_id(&endpoint.access_context.proxy_session_id) - ); - return StatusCode::SERVICE_UNAVAILABLE.into_response(); - } - } - if ensure_hls_cache_stream_registered( - endpoint.app_state, - endpoint.fingerprint, - endpoint.headers, - endpoint.access_context, - endpoint.session, - ) - .await - .is_none() - { - debug!( - "HLS transient media registration skipped: lease={} reason=session-or-connection-unavailable", - safe_hls_access_lease_id(&endpoint.access_context.lease_id) - ); - } - } - let media_marker = if matches!(resource.kind, TransientResourceKind::Segment | TransientResourceKind::Part) - && response.decoded.status.is_success() - { - hls_cache_response_context( - endpoint.app_state, - endpoint.session, - endpoint.access_context, - endpoint.lease_identity, - endpoint.now_ms, - ) - .await - .media_activity_marker - } else { - None - }; - let response = hls_transient_origin_response( - response, - HlsTransientDirectResponseContext { - session: Arc::clone(endpoint.session), - resource, - policy: policy.clone(), - now_ms: endpoint.now_ms, - log_identity: { - let session = endpoint.session.read().await; - HlsLogIdentity::from_session(&session) - }, - }, - ); - if let Some(marker) = media_marker { - marker.confirm_media_response(response) - } else { - response - } - } - Err(err) => { - if matches!(err, HlsOriginResourceFetchError::ProviderUnavailable(_)) { - if let Some(runtime_err) = fetch_result.runtime_prepare_error { - return hls_origin_runtime_resource_failure_response( - endpoint.app_state, - endpoint.access_context, - runtime_err, - ); - } - } - match hls_transient_object_fetch_failure(&err) { - HlsTransientObjectFetchFailure::Retryable => { - let failed_at_ms = current_time_millis(); - if record_temporary_transient_segment_fetch_failure( - endpoint.session, - &resource, - &policy, - failed_at_ms, - ) - .await - { - hls_resource_channel_unavailable_response(endpoint.app_state, endpoint.access_context) - } else { - hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS) - } - } - HlsTransientObjectFetchFailure::Permanent { status: _ } => { - hls_resource_channel_unavailable_response(endpoint.app_state, endpoint.access_context) - } - } - } - } -} - -pub(super) struct HlsTransientEndpointOriginFetchRequest<'a> { - pub(super) app_state: &'a Arc, - pub(super) session: &'a HlsSessionHandle, - pub(super) access_context: &'a HlsAccessContext, - pub(super) fingerprint: &'a Fingerprint, - pub(super) headers: &'a HeaderMap, - pub(super) resource: &'a TransientResourceRef, - pub(super) resource_file: &'a TransientResourceFile, - pub(super) origin_headers: HeaderMap, - pub(super) origin_provider_session_headers: HeaderMap, - pub(super) range_header: Option, - pub(super) policy: SegmentFetchPolicy, -} - -pub(super) struct HlsTransientOriginFetchResult { - pub(super) result: - Result>, HlsOriginResourceFetchError>, - pub(super) runtime_prepare_error: Option, -} - -pub(super) async fn fetch_transient_origin_response_with_provider_io( - request: HlsTransientEndpointOriginFetchRequest<'_>, -) -> HlsTransientOriginFetchResult { - let clients = HlsOriginResourceClients { - client: request.app_state.http_clients.default.load().as_ref().clone(), - no_redirect_client: request.app_state.http_clients.no_redirect.load().as_ref().clone(), - use_manual_redirects: request.app_state.should_use_manual_redirects(), - }; - let log_identity = { - let session = request.session.read().await; - HlsLogIdentity::from_session(&session) - }; - let fetch_request = HlsTransientOriginFetchRequest { - resolved_origin_uri: request.resource.resolved_origin_uri.clone(), - origin_headers: request.origin_headers, - origin_provider_session_headers: request.origin_provider_session_headers, - range_header: request.range_header, - resource_file: request.resource_file.clone(), - resource_kind: request.resource.kind, - clients, - policy: request.policy, - log_identity, - }; - let runtime_prepare_error = Arc::new(tokio::sync::Mutex::new(None)); - let prepare_attempt = hls_transient_origin_prepare_closure( - request.app_state, - request.session, - request.access_context, - request.fingerprint, - request.headers, - &runtime_prepare_error, - ); - let result = fetch_hls_transient_origin_response_with_attempt_prepare(fetch_request, prepare_attempt).await; - let runtime_prepare_error = *runtime_prepare_error.lock().await; - HlsTransientOriginFetchResult { result, runtime_prepare_error } -} - -/// Builds the shared per-attempt prepare closure for transient origin fetches. -/// Runtime acquire failures are captured in `runtime_prepare_error` and mapped -/// to a provider-unavailable fetch error so the retry loop can proceed uniformly. -pub(super) fn hls_transient_origin_prepare_closure( - app_state: &Arc, - session: &HlsSessionHandle, - access_context: &HlsAccessContext, - fingerprint: &Fingerprint, - headers: &HeaderMap, - runtime_prepare_error: &Arc>>, -) -> impl FnMut( - HlsResourceFetchAttempt, -) -> futures::future::BoxFuture< - 'static, - Result, HlsOriginResourceFetchError>, -> { - let app_state = Arc::clone(app_state); - let session = Arc::clone(session); - let access_context = access_context.clone(); - let fingerprint = fingerprint.clone(); - let headers = headers.clone(); - let runtime_prepare_error = Arc::clone(runtime_prepare_error); - move |_attempt| { - let app_state = Arc::clone(&app_state); - let session = Arc::clone(&session); - let access_context = access_context.clone(); - let fingerprint = fingerprint.clone(); - let headers = headers.clone(); - let runtime_prepare_error = Arc::clone(&runtime_prepare_error); - async move { - match prepare_hls_transient_origin_io_for_authorized_resource_work( - &app_state, - &session, - &access_context, - &fingerprint, - &headers, - current_time_millis(), - ) - .await - { - Ok(guard) => Ok(guard), - Err(err) => { - *runtime_prepare_error.lock().await = Some(err); - Err(HlsOriginResourceFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Unavailable)) - } - } - } - .boxed() - } -} - -#[allow(clippy::too_many_arguments)] -pub(super) struct HlsTransientEndpointCacheFetchContext<'a> { - pub(super) app_state: &'a Arc, - pub(super) session: &'a HlsSessionHandle, - pub(super) fingerprint: &'a Fingerprint, - pub(super) headers: &'a HeaderMap, - pub(super) access_context: &'a HlsAccessContext, - pub(super) lease_identity: HlsMediaLeaseIdentity, - pub(super) resource: &'a TransientResourceRef, - pub(super) resource_file: TransientResourceFile, - pub(super) fetch_token: TransientObjectFetchToken, - pub(super) origin_headers: HeaderMap, - pub(super) origin_provider_session_headers: HeaderMap, - pub(super) range_header: Option, - pub(super) cache_duration_ms: u64, -} - -pub(super) struct TransientObjectWaitContext<'a> { - pub(super) app_state: &'a Arc, - pub(super) session: &'a HlsSessionHandle, - pub(super) fingerprint: &'a Fingerprint, - pub(super) headers: &'a HeaderMap, - pub(super) access_context: &'a HlsAccessContext, - pub(super) lease_identity: HlsMediaLeaseIdentity, - pub(super) resource_file: TransientResourceFile, - pub(super) range_header: Option, - pub(super) notifier: Arc, -} - -pub(super) struct TransientObjectCacheServeContext<'a> { - pub(super) app_state: &'a Arc, - pub(super) session: &'a HlsSessionHandle, - pub(super) fingerprint: &'a Fingerprint, - pub(super) headers: &'a HeaderMap, - pub(super) access_context: &'a HlsAccessContext, - pub(super) lease_identity: HlsMediaLeaseIdentity, - pub(super) resource_file: TransientResourceFile, - pub(super) range_header: Option, - pub(super) now_ms: u64, -} - -pub(super) async fn serve_transient_object_cache_response_and_mark( - context: TransientObjectCacheServeContext<'_>, -) -> axum::response::Response { - if !hls_live_lease_identity_is_current(context.app_state, context.access_context, context.lease_identity).await { - return StatusCode::NOT_FOUND.into_response(); - } - let response_context = hls_cache_response_context( - context.app_state, - context.session, - context.access_context, - context.lease_identity, - context.now_ms, - ) - .await; - let response = hls_resource_serve_outcome_response( - context.app_state, - context.access_context, - serve_hls_transient_object_cache_outcome( - Arc::clone(context.app_state.hls.proxy.segment_cache()), - Arc::clone(context.session), - context.resource_file, - context.range_header, - &response_context, - ) - .await, - ); - if is_hls_media_activity_status(response.status()) { - register_hls_cache_stream_for_successful_media_response( - context.app_state, - context.fingerprint, - context.headers, - context.access_context, - context.session, - &response_context, - ) - .await; - } - response -} - -pub(super) async fn serve_transient_object_cache_response_and_mark_or_unavailable( - context: TransientObjectCacheServeContext<'_>, -) -> axum::response::Response { - serve_transient_object_cache_response_and_mark(context).await -} - -pub(super) async fn wait_for_transient_object_cache_fetch( - context: TransientObjectWaitContext<'_>, -) -> axum::response::Response { - let wait_timeout = context.app_state.hls.proxy.segment_fetch_policy().origin_object_wait_timeout(); - let safe_resource_id = safe_transient_resource_id(&context.resource_file.resource_id); - debug!( - "HLS transient object wait started: resource_id={} lease={} state=inflight", - safe_resource_id, - safe_hls_access_lease_id(&context.access_context.lease_id) - ); - let wait_result = tokio::time::timeout(wait_timeout, context.notifier.notified()).await; - if wait_result.is_err() { - debug!( - "HLS transient object wait timed out: resource_id={} lease={} state=inflight", - safe_resource_id, - safe_hls_access_lease_id(&context.access_context.lease_id) - ); - return hls_transient_object_unavailable_response( - context.app_state, - context.session, - &context.resource_file, - current_time_millis(), - context.access_context, - ) - .await; - } - let response = serve_transient_object_cache_response_and_mark_or_unavailable(TransientObjectCacheServeContext { - app_state: context.app_state, - session: context.session, - fingerprint: context.fingerprint, - headers: context.headers, - access_context: context.access_context, - lease_identity: context.lease_identity, - resource_file: context.resource_file, - range_header: context.range_header, - now_ms: current_time_millis(), - }) - .await; - debug!( - "HLS transient object wait completed: resource_id={} lease={} status={}", - safe_resource_id, - safe_hls_access_lease_id(&context.access_context.lease_id), - response.status() - ); - response -} - -pub(super) fn safe_transient_resource_id(resource_id: &TransientResourceId) -> String { - // Truncate at the first char boundary at or before byte 8 to avoid allocating - // a temporary `String` of 8 chars (and a second UTF-8 walk via `len()`). - let full = resource_id.0.as_str(); - let truncate_at = full.char_indices().nth(8).map_or(full.len(), |(byte_idx, _)| byte_idx); - if truncate_at == full.len() { - return full.to_owned(); - } - let mut out = String::with_capacity(truncate_at + 3); - out.push_str(&full[..truncate_at]); - out.push_str("..."); - out -} - -pub(super) async fn validate_hls_proxy_access_request( - app_state: &Arc, - fingerprint: &Fingerprint, - proxy_session_id: &ProxySessionId, - hls_access_lease_id: &str, - now_ms: u64, - timing: HlsAccessLeaseTiming, - request_kind: &'static str, -) -> Result { - let context = validate_hls_proxy_access_context( - app_state, - fingerprint, - proxy_session_id, - hls_access_lease_id, - now_ms, - HlsAccessAdmissionMode::ResourceAccess, - ) - .await?; - let startup_admission_pending = app_state - .hls - .proxy - .access_lease_response_snapshot(&context.lease_id, proxy_session_id, now_ms) - .await - .is_some_and(|lease| { - lease.state == HlsAccessLeaseState::Pending - && lease.startup_admission == HlsLeaseStartupAdmissionState::Pending - }); - if startup_admission_pending { - return Err(HlsAccessLeaseValidationError::AvailabilityPending); - } - match app_state.hls.proxy.activate_access_lease(&context.lease_id, proxy_session_id, now_ms, timing).await { - HlsAccessLeaseActivation::Activated { .. } => { - debug!( - "HLS access lease accepted: lease={} proxy_session={} user_session={} request={request_kind}", - safe_hls_access_lease_id(&context.lease_id), - safe_proxy_session_id(proxy_session_id), - safe_user_session_token(&context.user_session_token) - ); - Ok(context) - } - HlsAccessLeaseActivation::Denied => { - warn!( - "HLS access lease rejected: lease={} proxy_session={} user_session={} request={request_kind} reason=denied", - safe_hls_access_lease_id(&context.lease_id), - safe_proxy_session_id(proxy_session_id), - safe_user_session_token(&context.user_session_token) - ); - let (runtime_tail, reason) = app_state - .hls - .proxy - .access_lease_response_snapshot(&context.lease_id, proxy_session_id, now_ms) - .await - .map_or((None, None), |lease| { - (lease.runtime_policy_revocation_outcome(), lease.runtime_policy_denial_reason()) - }); - Err(HlsAccessLeaseValidationError::AdmissionDenied { runtime_tail, reason }) - } - HlsAccessLeaseActivation::Expired - | HlsAccessLeaseActivation::UnknownLease - | HlsAccessLeaseActivation::SessionMismatch => { - warn!( - "HLS access lease rejected: lease={} proxy_session={} user_session={} request={request_kind} reason=expired", - safe_hls_access_lease_id(&context.lease_id), - safe_proxy_session_id(proxy_session_id), - safe_user_session_token(&context.user_session_token) - ); - Err(HlsAccessLeaseValidationError::Expired) - } - } -} - -pub(super) async fn ensure_hls_cache_stream_registered( - app_state: &Arc, - fingerprint: &Fingerprint, - req_headers: &HeaderMap, - access: &HlsAccessContext, - session: &HlsSessionHandle, -) -> Option { - let (proxy_session_id, origin_source, origin_account_binding) = { - let session = session.read().await; - if session.is_gc_marked_for_removal() { - return None; - } - (session.proxy_session_id.clone(), session.origin_source.clone(), session.origin_account_binding.clone()) - }; - let user = app_state.app_config.get_user_credentials(&access.username)?; - let user_session = - app_state.active_users.get_and_update_user_session(&access.username, &access.user_session_token).await?; - let connection_kind = user_session.connection_kind?; - let priority = connection_priority_for_kind(&user, connection_kind); - let mut stream_channel = build_hls_cache_stream_channel(app_state, access, &origin_source, &proxy_session_id).await; - let provider = hls_cache_stats_provider(&origin_source, origin_account_binding.as_ref(), &user_session); - let user_agent = req_headers - .get(header::USER_AGENT) - .map_or_else(|| Cow::Borrowed(""), |value| String::from_utf8_lossy(value.as_bytes())); - - stream_channel.url = Arc::from(hls_cache_stream_stats_url(&proxy_session_id)); - // Panel Streams/History read this item_type. Shared HLS transport is still HLS, but - // archive/catchup leases must never be published as Live/LiveHls. - let panel_archive_reference = origin_source - .archive_reference - .or(access.epg_reference_ts) - .or_else(|| access.archive_origin_url.as_deref().and_then(m3u_archive_epg_reference_ts)) - .or_else(|| m3u_catchup_epg_reference_from_session_token(&access.user_session_token)) - .or(stream_channel.epg_reference_ts); - let is_archive_playback = panel_archive_reference.is_some() - || access.archive_origin_url.is_some() - || origin_source.archive_reference.is_some() - || is_m3u_catchup_session_token(&access.user_session_token) - || stream_channel.item_type == PlaylistItemType::Catchup; - if is_archive_playback { - stream_channel.item_type = PlaylistItemType::Catchup; - stream_channel.cluster = XtreamCluster::Video; - stream_channel.epg_reference_ts = panel_archive_reference; - } else { - stream_channel.item_type = PlaylistItemType::LiveHls; - stream_channel.cluster = PlaylistItemType::LiveHls.cluster(); - } - let shared_stream_id = hls_cache_shared_stream_id(&proxy_session_id); - stream_channel.shared = true; - stream_channel.shared_stream_id = Some(shared_stream_id); - stream_channel.shared_joined_existing = Some( - hls_cache_shared_joined_existing(app_state, shared_stream_id, &access.username, &access.user_session_token) - .await, - ); - let qos_config = HlsQosRuntimeConfig::from_app_config(&app_state.app_config); - let qos_registration = app_state - .hls - .proxy - .qos() - .ensure_access_lease( - &access.lease_id, - &proxy_session_id, - current_time_millis(), - hls_qos_meter_init(app_state, qos_config), - ) - .await; - if let Some(meter) = qos_registration.register_meter.as_ref() { - app_state.event_manager.register_meter(Arc::clone(meter)).await; - } - let history_mode = if qos_registration.emit_connect_record { - ConnectionHistoryMode::EmitConnect - } else { - ConnectionHistoryMode::RefreshOnly - }; - - app_state - .connection_manager - .update_connection_with_history_mode( - crate::api::model::ConnectionParams { - meter_uid: qos_registration.meter_uid, - username: &access.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind, - priority, - soft_priority: user.soft_priority, - fingerprint, - provider, - stream_channel: &stream_channel, - user_agent, - session_token: Some(&access.user_session_token), - }, - history_mode, - ) - .await -} - -pub(super) fn hls_cache_stats_provider( - origin_source: &HlsOriginSource, - origin_account_binding: Option<&HlsOriginAccountBinding>, - user_session: &UserSession, -) -> Arc { - origin_account_binding.filter(|binding| binding.is_active()).map_or_else( - || { - if user_session.provider.is_empty() { - Arc::clone(&origin_source.input_name) - } else { - Arc::clone(&user_session.provider) - } - }, - |binding| Arc::clone(&binding.account_name), - ) -} - -pub(super) async fn build_hls_cache_stream_channel( - app_state: &Arc, - access: &HlsAccessContext, - origin_source: &HlsOriginSource, - proxy_session_id: &ProxySessionId, -) -> StreamChannel { - let mut channel = if let Some((_, target)) = app_state.app_config.get_target_for_username(&access.username) { - if let Some(mut channel) = get_stream_channel(app_state, &target, access.virtual_id).await { - channel.url = Arc::from(hls_cache_stream_stats_url(proxy_session_id)); - channel - } else { - fallback_hls_cache_stream_channel(target.id, access.virtual_id, origin_source, proxy_session_id) - } - } else { - fallback_hls_cache_stream_channel(0, access.virtual_id, origin_source, proxy_session_id) - }; - - let archive_reference = access - .epg_reference_ts - .or_else(|| access.archive_origin_url.as_deref().and_then(m3u_archive_epg_reference_ts)) - .or_else(|| m3u_catchup_epg_reference_from_session_token(&access.user_session_token)); - - if archive_reference.is_some() - || access.archive_origin_url.is_some() - || is_m3u_catchup_session_token(&access.user_session_token) - { - channel.item_type = PlaylistItemType::Catchup; - channel.cluster = XtreamCluster::Video; - channel.epg_reference_ts = archive_reference; - } else { - channel.item_type = PlaylistItemType::LiveHls; - channel.cluster = PlaylistItemType::LiveHls.cluster(); - channel.epg_reference_ts = None; - } - channel -} - -pub(super) fn fallback_hls_cache_stream_channel( - target_id: u16, - virtual_id: u32, - origin_source: &HlsOriginSource, - proxy_session_id: &ProxySessionId, -) -> StreamChannel { - let unknown = "Unknown".intern(); - StreamChannel { - target_id, - virtual_id, - provider_id: 0, - input_name: Arc::clone(&origin_source.input_name), - item_type: PlaylistItemType::LiveHls, - cluster: XtreamCluster::Live, - group: unknown.clone(), - title: unknown, - url: Arc::from(hls_cache_stream_stats_url(proxy_session_id)), - shared: false, - shared_joined_existing: None, - shared_stream_id: None, - technical: None, - epg_channel_id: None, - epg_reference_ts: None, - upstream_user_agent: None, - } -} - -pub(super) fn hls_cache_stream_stats_url(proxy_session_id: &ProxySessionId) -> String { - format!("/hls/shared/live/{}/manifest.m3u8", proxy_session_id.0) -} - -pub(super) fn hls_cache_shared_stream_id(proxy_session_id: &ProxySessionId) -> u64 { - let digest = Sha256::digest(proxy_session_id.0.as_bytes()); - digest.iter().take(8).fold(0_u64, |value, byte| (value << 8) | u64::from(*byte)) -} - -pub(super) async fn hls_cache_shared_joined_existing( - app_state: &Arc, - shared_stream_id: u64, - username: &str, - session_token: &str, -) -> bool { - let streams = app_state.active_users.active_streams().await; - if let Some(existing) = streams.iter().find(|stream| { - stream.username == username - && stream.session_token.as_deref() == Some(session_token) - && stream.channel.shared - && stream.channel.shared_stream_id == Some(shared_stream_id) - }) { - return existing.channel.shared_joined_existing.unwrap_or(false); - } - - streams.iter().any(|stream| { - stream.channel.shared - && stream.channel.shared_stream_id == Some(shared_stream_id) - && (stream.username != username || stream.session_token.as_deref() != Some(session_token)) - }) -} - -pub(super) async fn validate_hls_proxy_access_context( - app_state: &Arc, - fingerprint: &Fingerprint, - proxy_session_id: &ProxySessionId, - hls_access_lease_id: &str, - now_ms: u64, - admission_mode: HlsAccessAdmissionMode, -) -> Result { - validate_hls_access_lease( - &app_state.hls_ctx(), - fingerprint, - proxy_session_id, - &HlsAccessLeaseId(hls_access_lease_id.to_string()), - now_ms, - admission_mode, - ) - .await -} - -pub(super) async fn hls_custom_video_manifest_response_for_username( - app_state: &Arc, - username: &str, - video_type: CustomVideoStreamType, - fallback_status: StatusCode, -) -> axum::response::Response { - if let Some(user) = app_state.app_config.get_user_credentials(username) { - return hls_custom_video_manifest_response(app_state, &user, video_type, fallback_status).await; - } - fallback_status.into_response() -} - -pub(super) async fn hls_custom_video_manifest_response_for_lease( - app_state: &Arc, - lease: &HlsAccessLease, - video_type: CustomVideoStreamType, - fallback_status: StatusCode, -) -> axum::response::Response { - let Some(user) = app_state.app_config.get_user_credentials(&lease.username) else { - return fallback_status.into_response(); - }; - hls_custom_video_manifest_response_for_access_lease(app_state, &user, video_type, fallback_status, lease).await -} - -pub(super) async fn hls_runtime_custom_tail_response( - app_state: &Arc, - session: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, - reason: HlsRuntimeCustomTailReason, - fallback_status: StatusCode, -) -> axum::response::Response { - let outcome = commit_hls_runtime_custom_tail( - app_state.hls_ctx(), - HlsRuntimeCustomTailRequest { - session: Arc::clone(session), - proxy_session_id: proxy_session_id.clone(), - lease_id: access_lease_id.clone(), - reason, - now_ms: current_time_millis(), - }, - ) - .await; - if matches!(outcome, HlsRuntimeCustomTailOutcome::Committed | HlsRuntimeCustomTailOutcome::AlreadyCommitted) { - let now_ms = current_time_millis(); - if let Some(lease) = - app_state.hls.proxy.access_lease_response_snapshot(access_lease_id, proxy_session_id, now_ms).await - { - if let Some(response) = hls_terminal_playback_response(&lease, proxy_session_id, access_lease_id) { - return response; - } - } - } - if outcome == HlsRuntimeCustomTailOutcome::PendingOwnerRegistered { - return hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS); - } - fallback_status.into_response() -} - -pub(super) async fn hls_runtime_or_standalone_custom_tail_response( - app_state: &Arc, - session: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, - reason: HlsRuntimeCustomTailReason, - fallback_status: StatusCode, -) -> axum::response::Response { - let now_ms = current_time_millis(); - let Some(lease) = - app_state.hls.proxy.access_lease_response_snapshot(access_lease_id, proxy_session_id, now_ms).await - else { - return fallback_status.into_response(); - }; - match &lease.playback_mode { - HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } => { - if let Some(response) = hls_terminal_playback_response(&lease, proxy_session_id, access_lease_id) { - return response; - } - } - HlsLeasePlaybackMode::Live - if lease.last_manifest_snapshot.is_some() - && matches!(lease.state, HlsAccessLeaseState::Activated | HlsAccessLeaseState::PolicyRevoking) => - { - return hls_runtime_custom_tail_response( - app_state, - session, - proxy_session_id, - access_lease_id, - reason, - fallback_status, - ) - .await; - } - HlsLeasePlaybackMode::Ended if !reason.permits_unpublished_lease_standalone_tail() => { - return fallback_status.into_response(); - } - HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::Ended => {} - } - hls_custom_video_manifest_response_for_lease(app_state, &lease, reason.video_type(), fallback_status).await -} - -pub(super) async fn hls_manifest_channel_unavailable_response_for_username( - app_state: &Arc, - username: &str, -) -> axum::response::Response { - hls_custom_video_manifest_response_for_username( - app_state, - username, - CustomVideoStreamType::ChannelUnavailable, - StatusCode::NOT_FOUND, - ) - .await -} - -/// Resolves the final canonical-manifest fallback after refresh and cached-live -/// publication have both produced no response. -pub(super) async fn hls_unpublished_lease_channel_unavailable_response( - app_state: &Arc, - proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, -) -> axum::response::Response { - let reason = HlsRuntimeCustomTailReason::ChannelUnavailable; - let lease = app_state - .hls - .proxy - .access_lease_response_snapshot(access_lease_id, proxy_session_id, current_time_millis()) - .await; - if let Some(lease) = lease.filter(|lease| lease.permits_unpublished_standalone_tail(reason)) { - return hls_custom_video_manifest_response_for_lease( - app_state, - &lease, - CustomVideoStreamType::ChannelUnavailable, - StatusCode::NOT_FOUND, - ) - .await; - } - StatusCode::SERVICE_UNAVAILABLE.into_response() -} - -pub(super) async fn hls_manifest_access_denial_runtime_response( - app_state: &Arc, - proxy_session_id: &ProxySessionId, - lease_snapshot: Option<&HlsAccessLease>, - reason: HlsRuntimeCustomTailReason, - fallback_status: StatusCode, -) -> axum::response::Response { - let Some(lease) = lease_snapshot else { - return fallback_status.into_response(); - }; - let Some(session) = app_state.hls.proxy.sessions().get_by_proxy_session_id(proxy_session_id).await else { - return fallback_status.into_response(); - }; - hls_runtime_or_standalone_custom_tail_response( - app_state, - &session, - proxy_session_id, - &lease.lease_id, - reason, - fallback_status, - ) - .await -} - -pub(super) fn hls_resource_channel_unavailable_response( - _app_state: &Arc, - _access_context: &HlsAccessContext, -) -> axum::response::Response { - StatusCode::NOT_FOUND.into_response() -} - -pub(super) fn hls_origin_runtime_resource_failure_response( - _app_state: &Arc, - _access_context: &HlsAccessContext, - err: HlsOriginRuntimeAcquireError, -) -> axum::response::Response { - match err { - HlsOriginRuntimeAcquireError::NoAccountAvailable { .. } => StatusCode::SERVICE_UNAVAILABLE.into_response(), - HlsOriginRuntimeAcquireError::Fatal(status) => hls_canonical_status_response(status), - } -} - -pub(super) fn hls_resource_serve_outcome_response( - app_state: &Arc, - access_context: &HlsAccessContext, - outcome: HlsResourceServeOutcome, -) -> axum::response::Response { - match outcome { - HlsResourceServeOutcome::Ready(response) => response, - HlsResourceServeOutcome::Failure(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms }) => { - hls_temporary_resource_unavailable_response(retry_after_ms) - } - HlsResourceServeOutcome::Failure( - HlsResourceServeFailure::Missing - | HlsResourceServeFailure::Expired - | HlsResourceServeFailure::PermanentFailed { .. }, - ) => hls_resource_channel_unavailable_response(app_state, access_context), - } -} - -pub(super) async fn hls_manifest_access_lease_validation_response( - app_state: &Arc, - proxy_session_id: &ProxySessionId, - lease_snapshot: Option<&HlsAccessLease>, - err: HlsAccessLeaseValidationError, -) -> axum::response::Response { - match err { - HlsAccessLeaseValidationError::AdmissionDenied { reason, .. } => { - hls_manifest_access_denial_runtime_response( - app_state, - proxy_session_id, - lease_snapshot, - reason.unwrap_or(HlsRuntimeCustomTailReason::UserConnectionsExhausted), - StatusCode::FORBIDDEN, - ) - .await - } - HlsAccessLeaseValidationError::AvailabilityPending => { - hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS) - } - HlsAccessLeaseValidationError::UserSessionMissing { .. } => { - hls_manifest_access_denial_runtime_response( - app_state, - proxy_session_id, - lease_snapshot, - HlsRuntimeCustomTailReason::SessionOrLeaseExpired, - StatusCode::NOT_FOUND, - ) - .await - } - HlsAccessLeaseValidationError::UserAccountExpired { .. } => { - hls_manifest_access_denial_runtime_response( - app_state, - proxy_session_id, - lease_snapshot, - HlsRuntimeCustomTailReason::UserAccountExpired, - StatusCode::FORBIDDEN, - ) - .await - } - HlsAccessLeaseValidationError::Expired => StatusCode::NOT_FOUND.into_response(), - } -} - -pub(super) fn hls_resource_access_lease_validation_response( - err: &HlsAccessLeaseValidationError, -) -> axum::response::Response { - match err { - HlsAccessLeaseValidationError::AdmissionDenied { .. } - | HlsAccessLeaseValidationError::UserAccountExpired { .. } => StatusCode::FORBIDDEN.into_response(), - HlsAccessLeaseValidationError::AvailabilityPending => { - hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS) - } - HlsAccessLeaseValidationError::UserSessionMissing { .. } | HlsAccessLeaseValidationError::Expired => { - StatusCode::NOT_FOUND.into_response() - } - } -} - -pub(super) async fn hls_manifest_access_context_and_state( - app_state: &Arc, - fingerprint: &Fingerprint, - proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, - access_lease_snapshot: Option<&HlsAccessLease>, - now_ms: u64, -) -> Result<(HlsAccessContext, HlsAccessLeaseState), Box> { - app_state.hls.proxy.startup_observability().record_media_manifest_request(access_lease_id, now_ms); - let access_context = match validate_hls_proxy_access_context( - app_state, - fingerprint, - proxy_session_id, - &access_lease_id.0, - now_ms, - HlsAccessAdmissionMode::ManifestPrepare, - ) - .await - { - Ok(context) => context, - Err(err) => { - warn!( - "HLS access lease rejected: lease={} proxy_session={} user_session=none reason={err:?}", - safe_hls_access_lease_id(access_lease_id), - safe_proxy_session_id(proxy_session_id) - ); - return Err(Box::new( - hls_manifest_access_lease_validation_response(app_state, proxy_session_id, access_lease_snapshot, err) - .await, - )); - } - }; - if access_lease_snapshot.is_none() - && app_state.hls.proxy.sessions().get_by_proxy_session_id(proxy_session_id).await.is_none() - && app_state.hls.proxy.expired_session_marker(proxy_session_id, now_ms).await.is_some() - { - return Err(Box::new(StatusCode::NOT_FOUND.into_response())); - } - debug!( - "HLS access lease accepted: lease={} proxy_session={} user_session={} request=manifest", - safe_hls_access_lease_id(&access_context.lease_id), - safe_proxy_session_id(proxy_session_id), - safe_user_session_token(&access_context.user_session_token) - ); - - let access_lease_state = match app_state - .hls - .proxy - .touch_manifest_access_lease( - &access_context.lease_id, - proxy_session_id, - now_ms, - None, - Some(HlsAccessLeasePendingDeadline::Bootstrap { - deadline_ms: now_ms.saturating_add(hls_pending_bootstrap_window_ms(app_state)), - }), - hls_access_lease_ttl_ms(app_state), - ) - .await - { - HlsAccessLeaseTouch::Touched { lease } => lease.state, - HlsAccessLeaseTouch::Denied => { - return Err(Box::new( - hls_manifest_access_denial_runtime_response( - app_state, - proxy_session_id, - access_lease_snapshot, - HlsRuntimeCustomTailReason::UserConnectionsExhausted, - StatusCode::FORBIDDEN, - ) - .await, - )); - } - HlsAccessLeaseTouch::Expired | HlsAccessLeaseTouch::UnknownLease | HlsAccessLeaseTouch::SessionMismatch => { - return Err(Box::new(StatusCode::NOT_FOUND.into_response())); - } - }; - - Ok((access_context, access_lease_state)) -} - -pub(super) async fn hls_transient_object_unavailable_response( - app_state: &Arc, - session: &HlsSessionHandle, - resource_file: &TransientResourceFile, - now_ms: u64, - access_context: &HlsAccessContext, -) -> axum::response::Response { - let state = { - let session = session.read().await; - let key = TransientPassthroughState::transient_object_key( - &session.proxy_session_id, - &resource_file.resource_id, - resource_file.extension.clone(), - ); - session.transient.object_unavailable_state(&key, now_ms) - }; - match state { - TransientObjectUnavailableState::Fetching => { - hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS) - } - TransientObjectUnavailableState::FailedRetryable { retry_after_ms } => { - hls_temporary_resource_unavailable_response(retry_after_ms) - } - TransientObjectUnavailableState::FailedPermanent | TransientObjectUnavailableState::Missing => { - hls_resource_channel_unavailable_response(app_state, access_context) - } - } -} - -#[allow(clippy::too_many_lines)] -pub(super) async fn fetch_and_cache_transient_origin_response( - context: HlsTransientEndpointCacheFetchContext<'_>, -) -> axum::response::Response { - let policy = context.app_state.hls.proxy.segment_fetch_policy(); - let mut fetch_finalizer = HlsTransientObjectFetchFinalizer::new( - Arc::clone(context.session), - Arc::clone(context.app_state.hls.proxy.segment_cache()), - context.fetch_token.clone(), - HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, - ); - let clients = HlsOriginResourceClients { - client: context.app_state.http_clients.default.load().as_ref().clone(), - no_redirect_client: context.app_state.http_clients.no_redirect.load().as_ref().clone(), - use_manual_redirects: context.app_state.should_use_manual_redirects(), - }; - let log_identity = { - let session = context.session.read().await; - HlsLogIdentity::from_session(&session) - }; - let fetch_request = HlsTransientOriginFetchRequest { - resolved_origin_uri: context.resource.resolved_origin_uri.clone(), - origin_headers: context.origin_headers.clone(), - origin_provider_session_headers: context.origin_provider_session_headers.clone(), - range_header: None, - resource_file: context.resource_file.clone(), - resource_kind: context.resource.kind, - clients, - policy: policy.clone(), - log_identity: log_identity.clone(), - }; - let cache_fetch_request = HlsTransientOriginCacheFetchRequest { - fetch: fetch_request, - commit: HlsTransientCacheCommitContext { - segment_cache: Arc::clone(context.app_state.hls.proxy.segment_cache()), - segment_repair: Arc::clone(context.app_state.hls.proxy.segment_repair()), - session: Arc::clone(context.session), - proxy_session_id: context.access_context.proxy_session_id.clone(), - log_identity: log_identity.clone(), - access_lease_id: context.access_context.lease_id.clone(), - resource: context.resource.clone(), - resource_file: context.resource_file.clone(), - fetch_token: context.fetch_token.clone(), - cache_duration_ms: context.cache_duration_ms, - }, - }; - let runtime_prepare_error = Arc::new(tokio::sync::Mutex::new(None)); - let prepare_attempt = hls_transient_origin_prepare_closure( - context.app_state, - context.session, - context.access_context, - context.fingerprint, - context.headers, - &runtime_prepare_error, - ); - let final_failure = - match fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(cache_fetch_request, prepare_attempt) - .await - { - Ok(()) => { - let ready_at_ms = current_time_millis(); - let response_context = hls_cache_response_context( - context.app_state, - context.session, - context.access_context, - context.lease_identity, - ready_at_ms, - ) - .await; - let response = serve_hls_transient_object_cache_response( - Arc::clone(context.app_state.hls.proxy.segment_cache()), - Arc::clone(context.session), - context.resource_file.clone(), - context.range_header.clone(), - &response_context, - ) - .await; - if is_hls_media_activity_status(response.status()) { - register_hls_cache_stream_for_successful_media_response( - context.app_state, - context.fingerprint, - context.headers, - context.access_context, - context.session, - &response_context, - ) - .await; - } - record_successful_transient_segment_fetch(context.session, context.resource).await; - fetch_finalizer.complete(); - return response; - } - Err(err) => { - if matches!(err, HlsOriginResourceFetchError::ProviderUnavailable(_)) { - let runtime_prepare_error = *runtime_prepare_error.lock().await; - if let Some(runtime_err) = runtime_prepare_error { - context.session.write().await.fail_transient_object_retryable_if_current( - &context.fetch_token, - current_time_millis(), - HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, - ); - return hls_origin_runtime_resource_failure_response( - context.app_state, - context.access_context, - runtime_err, - ); - } - } - hls_transient_object_fetch_failure(&err) - } - }; - - let failed_at_ms = current_time_millis(); - match final_failure { - HlsTransientObjectFetchFailure::Retryable => { - if record_temporary_transient_segment_fetch_failure( - context.session, - context.resource, - &policy, - failed_at_ms, - ) - .await - { - context.session.write().await.fail_transient_object_permanent_if_current( - &context.fetch_token, - failed_at_ms, - None, - ); - } else { - context.session.write().await.fail_transient_object_retryable_if_current( - &context.fetch_token, - failed_at_ms, - HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, - ); - } - } - HlsTransientObjectFetchFailure::Permanent { status } => { - context.session.write().await.fail_transient_object_permanent_if_current( - &context.fetch_token, - failed_at_ms, - status, - ); - } - } - hls_transient_object_unavailable_response( - context.app_state, - context.session, - &context.resource_file, - failed_at_ms, - context.access_context, - ) - .await -} - -use tuliprox_core::utils::current_time_millis; - -pub(super) async fn release_prepared_hls_manifest_session( - app_state: &Arc, - username: &str, - session_token: &str, - addr: &std::net::SocketAddr, -) { - let _transition_guard = app_state.active_users.acquire_playback_transition(username, session_token).await; - app_state.active_users.release_unbound_session_reservation(username, session_token, None, false).await; - app_state.active_users.clear_unbound_session_addr(username, session_token, addr).await; -} - -pub(super) async fn terminate_failed_hls_manifest_session( - app_state: &Arc, - username: &str, - session_token: &str, - provider_name: Option<&Arc>, - binding_tag: Option, - request_id: Option, -) { - let _transition_guard = app_state.active_users.acquire_playback_transition(username, session_token).await; - app_state.active_users.terminate_session(username, session_token).await; - let mut cleared = false; - if let Some(request_id) = request_id { - app_state.active_provider.finish_identified_playback_request( - session_token, - request_id, - PlaybackRequestOutcome::ProviderFailed, - ); - cleared = true; - } else if let (Some(provider_name), Some(binding_tag)) = (provider_name, binding_tag) { - // A failed manifest lets the next entry fall back to another provider, but only - // when the preference still belongs to this binding. - app_state.active_provider.forget_identified_provider_affinity(session_token, provider_name, binding_tag); - app_state.active_provider.clear_identified_provider_reservation( - session_token, - provider_name, - Some(binding_tag), - ); - cleared = true; - } - if !cleared { - app_state.active_provider.clear_provider_reservation(session_token); - } -} - -pub(super) fn normalize_xtream_live_hls_url(hls_url: &str, input: &ConfigInput) -> String { - if !input.input_type.is_xtream() || !input.has_flag(ConfigInputFlags::XtreamLiveStreamUsePrefix) { - return hls_url.to_string(); - } - - let (Some(username), Some(password)) = (input.username.as_deref(), input.password.as_deref()) else { - return hls_url.to_string(); - }; - - let Ok(mut parsed) = Url::parse(hls_url) else { - return hls_url.to_string(); - }; - let Some(segments) = parsed.path_segments() else { - return hls_url.to_string(); - }; - - let parts: Vec<&str> = segments.collect(); - if parts.len() >= 3 && parts[0] == username && parts[1] == password { - parsed.set_path(&format!("/live/{}", parts.join("/"))); - return parsed.to_string(); - } - - hls_url.to_string() -} - -pub(super) fn ensure_hls_manifest_extension(url: &str) -> String { - let with_extension = replace_url_extension(url, HLS_EXT); - let (base_url, suffix) = match with_extension.find(['?', '#'].as_ref()) { - Some(pos) => (&with_extension[..pos], &with_extension[pos..]), - None => (with_extension.as_str(), ""), - }; - let Some(path_without_ext) = base_url.strip_suffix(HLS_EXT) else { - return with_extension; - }; - format!("{}{}{}", path_without_ext.trim_end_matches('.'), HLS_EXT, suffix) -} - -pub(super) fn build_hls_manifest_request_headers( - input_headers: &HashMap, - req_headers: &HeaderMap, - disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, - default_user_agent: Option<&str>, - upstream_user_agent: Option<&str>, -) -> HeaderMap { - let input_headers = input_headers - .iter() - .filter(|(key, _)| !should_remove_hls_origin_header(key, disabled_headers)) - .map(|(key, value)| (key.clone(), value.clone())) - .collect::>(); - let disabled_headers_for_filter = disabled_headers.cloned(); - let filter_header: HeaderFilter = Some(Box::new(move |name: &str| { - !name.eq_ignore_ascii_case("range") - && !should_remove_hls_origin_header(name, disabled_headers_for_filter.as_ref()) - })); - let forwarded = get_headers_from_request(req_headers, &filter_header); - let mut headers = - request::get_request_headers(Some(&input_headers), Some(&forwarded), disabled_headers, default_user_agent); - request::overlay_upstream_user_agent(&mut headers, upstream_user_agent, disabled_headers); - scrub_hls_origin_headers(&mut headers, disabled_headers); - force_identity_without_range(&mut headers); - headers -} - -pub(super) async fn download_legacy_hls_manifest( - app_state: &Arc, - input: &InputSource, - headers: &HeaderMap, -) -> Result<(String, String, HeaderMap), std::io::Error> { - let deadline = Duration::from_millis(app_state.hls.proxy.origin_manifest_timeout_ms().max(1)); - let fetch_options = request::RequestFetchOptions::with_attempt_idle_timeout(deadline) - .with_content_coding(OutboundContentCodingPolicy::Identity); - let body_options = request::TextContentBodyOptions::hls_manifest(MAX_HLS_MANIFEST_BYTES, deadline); - let options = request::TextContentFetchOptions::new(fetch_options, body_options); - - if app_state.should_use_manual_redirects() { - request::download_text_content_with_manual_redirects_and_headers_and_options( - &app_state.app_config, - &app_state.http_clients.no_redirect.load(), - input, - Some(headers), - false, - MAX_MANUAL_REDIRECTS, - options, - ) - .await - } else { - request::download_text_content_with_headers_and_options( - &app_state.app_config, - &app_state.http_clients.default.load(), - input, - Some(headers), - false, - options, - ) - .await - } -} - -pub(super) struct HlsCacheManifestOrigin<'a> { - pub(super) raw_request_url: &'a str, - pub(super) session_entry_url: HlsOriginEntryUrl, - pub(super) input: &'a ConfigInput, - pub(super) origin_source: HlsOriginSource, -} - -pub(super) struct HlsCacheOriginResolution { - pub(super) hls_url: String, - pub(super) session_entry_url: HlsOriginEntryUrl, -} - -pub(in crate::api) fn build_virtual_hls_entry_path( - target: &ConfigTarget, - input: &ConfigInput, - user: &ProxyUserCredentials, - virtual_id: u32, -) -> String { - if input.input_type.is_m3u() && !target.has_output(TargetType::Xtream) { - format!("/{}/live/{}/{}/{}{HLS_EXT}", storage_const::M3U_STREAM_PATH, user.username, user.password, virtual_id) - } else { - format!("/live/{}/{}/{}{HLS_EXT}", user.username, user.password, virtual_id) - } -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -pub(in crate::api) async fn handle_hls_stream_request( - fingerprint: &Fingerprint, - app_state: &Arc, - user: &ProxyUserCredentials, - target: &ConfigTarget, - user_session: Option<&UserSession>, - session_token_hint: Option<&str>, - hls_url: &str, - archive_reference: Option, - stream_context: HlsEntryStreamContext, - input: &ConfigInput, - req_headers: &HeaderMap, - connection_permission: UserConnectionPermission, - connection_kind: Option, - original_hls_entry_path: &str, - stage: HlsRequestStage<'_>, -) -> impl IntoResponse + Send { - let virtual_id = stream_context.virtual_id(); - if app_state.active_users.is_user_blocked_for_stream(&user.username, VirtualId::new(virtual_id)).await { - return axum::http::StatusCode::BAD_REQUEST.into_response(); - } - - let stream_ref = stream_context.stream_ref().to_string(); - // Variant URLs were normalized before sealing or come from an upstream master; use them as is. - let url = if stage.normalizes_url() { - let normalized_hls_url = normalize_xtream_live_hls_url(hls_url, input); - if normalized_hls_url != hls_url { - debug_if_enabled!( - "Normalized xtream hls url from {} to {}", - sanitize_sensitive_info(hls_url), - sanitize_sensitive_info(&normalized_hls_url) - ); - } - ensure_hls_manifest_extension(&normalized_hls_url) - } else { - hls_url.to_string() - }; - // The wrapper seals the normalized entry URL, not the provider-resolved one, so every - // refresh can resolve it to whichever account is selected then. - let entry_url = url.clone(); - // Recover archive context when callers (esp. Xtream timeshift) pass None but the - // resolved provider URL / catchup session still carries Flussonic archive markers. - let archive_reference = archive_reference.or_else(|| m3u_archive_epg_reference_ts(&url)).or_else(|| { - user_session - .map(|session| session.token.as_str()) - .or(session_token_hint) - .and_then(m3u_catchup_epg_reference_from_session_token) - }); - let hls_cache_origin = build_hls_origin_resolution(input, &url); - let hls_origin_source = hls_cache_origin.as_ref().map(|_| { - build_hls_origin_source_for_playback(input, stream_ref.clone(), archive_reference, Some(url.as_str())) - }); - let server_info = app_state.app_config.get_user_server_info(user); - let Some(server_info) = server_info else { - return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); - }; - - let disabled_headers = app_state.get_disabled_headers(); - let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); - let mut headers = build_hls_manifest_request_headers( - &input.headers, - req_headers, - disabled_headers.as_ref(), - default_user_agent.as_deref(), - stream_context.identity().upstream_user_agent(), - ); - - if hls_cache_enabled_for_user(app_state, target, user) { - let Some(origin_source) = hls_origin_source.clone() else { - return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); - }; - return create_hls_cache_entry_master_playlist_response( - app_state, - fingerprint, - user, - origin_source, - virtual_id, - user_session, - stream_context.known_bitrate_bps(), - session_token_hint, - if archive_reference.is_some() { - url.as_str() - } else { - hls_cache_origin.as_ref().map_or(url.as_str(), |origin| origin.session_entry_url.as_str()) - }, - input, - connection_permission, - connection_kind, - server_info.path.as_deref(), - ) - .await; - } - - let fallback_connection_kind = connection_kind.unwrap_or(crate::api::model::ConnectionKind::Normal); - let (request_url, session_token, provider_handle, selected_provider_config) = if let Some(session) = user_session { - let pinned_provider = if session.provider.is_empty() { &input.name } else { &session.provider }; - let pinned_kind = hls_playback_kind(archive_reference, &url, Some(session.token.as_str())); - let session_kind = - session.connection_kind.or(connection_kind).unwrap_or(crate::api::model::ConnectionKind::Normal); - // Manifest refreshes of parallel playbacks on one account free their slots within seconds; - // waiting briefly beats an immediate 503 that stalls the player. - let acquired = acquire_exact_provider_handle( - app_state, - &ExactProviderAcquire { - provider: pinned_provider, - addr: &fingerprint.addr, - allow_grace: false, - priority: connection_priority_for_kind(user, session_kind), - kind: session_kind, - lease: Some(PlaybackLeaseRef::new(session.token.as_str(), pinned_kind)), - }, - Some(HLS_MANIFEST_CAPACITY_WAIT), - ) - .await; - let provider_handle = if let Some(handle) = acquired { - Some(handle) - } else { - debug_if_enabled!( - "HLS pinned provider {} unavailable for {}; aborting allocation to prevent mid-session migration", - sanitize_sensitive_info(pinned_provider), - sanitize_sensitive_info(&fingerprint.addr.to_string()) - ); - None - }; - - if provider_handle.is_none() { - // The capacity wait can outlive the session snapshot: an ended or rebound session must - // not trigger panel provisioning for the stale pinned account. - if app_state.active_users.session_identity(&user.username, &session.token).await != Some(session.identity()) - { - return StatusCode::SERVICE_UNAVAILABLE.into_response(); - } - return hls_panel_provisioning_or_status_response( - app_state, - user, - input, - virtual_id, - original_hls_entry_path, - server_info.path.as_deref(), - StatusCode::SERVICE_UNAVAILABLE, - ) - .await; - } - match provider_handle.as_ref().map(|handle| &handle.allocation) { - Some(ProviderAllocation::Exhausted) => (url, None, provider_handle, None), - Some(ProviderAllocation::Available(cfg) | ProviderAllocation::GracePeriod(cfg)) => { - let selected_provider_config = Arc::clone(cfg); - // URL acceptance follows the provider that produced the sealed URL, not the session: - // entry URLs are re-resolved to the selected account on every fetch, child URLs - // (often CDN URLs) are fetched as sealed and only from their origin provider. - let accept_requested_stream_url = match stage { - HlsRequestStage::Variant { source: HlsManifestSource::Child, origin_provider } => { - if origin_provider != Some(selected_provider_config.name.as_ref()) { - // A stale child URI from an earlier account: reject only this request, - // the playback itself stays valid after the player refetches its master. - app_state.connection_manager.release_provider_handle(provider_handle); - debug_if_enabled!( - "HLS child manifest from provider {} rejected for selected provider {}", - sanitize_sensitive_info(origin_provider.unwrap_or("")), - sanitize_sensitive_info(&selected_provider_config.name) - ); - return StatusCode::NOT_FOUND.into_response(); - } - true - } - HlsRequestStage::Entry | HlsRequestStage::LegacyVariant | HlsRequestStage::Variant { .. } => false, - }; - let Some((_provider_name, stream_url)) = select_provider_stream_url( - &url, - input, - &selected_provider_config, - accept_requested_stream_url, - &app_state.app_config, - ) - .await - else { - app_state.connection_manager.release_provider_handle(provider_handle); - return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); - }; - // The capacity wait can outlive the session snapshot: a session that ended or switched - // accounts meanwhile must not be rebound, and cookies rotated or expired during the - // wait are read now. - let session_headers = app_state - .active_users - .provider_headers_for_session_identity( - &user.username, - &session.token, - session.identity(), - &stream_url, - ) - .await; - if session_headers == tuliprox_session::SessionProviderHeaders::NoSession { - app_state.connection_manager.release_provider_handle(provider_handle); - debug_if_enabled!( - "HLS session {} changed while waiting for pinned provider {}", - sanitize_sensitive_info(&session.token), - sanitize_sensitive_info(pinned_provider) - ); - return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); - } - if session.provider.as_ref() == selected_provider_config.name.as_ref() { - if let tuliprox_session::SessionProviderHeaders::Headers(session_headers) = session_headers { - append_user_session_provider_headers(&mut headers, &session_headers); - } - } - let session_token = app_state - .active_users - .create_user_session(crate::api::model::CreateUserSessionParams { - user, - session_token: &session.token, - virtual_id, - provider: &selected_provider_config.name, - stream_url: &stream_url, - addr: &fingerprint.addr, - connection_permission, - connection_kind: session.connection_kind.or(connection_kind), - socket_bound: PlaylistItemType::LiveHls.uses_socket_bound_session(), - }) - .await; - let session_ttl_secs = get_hls_playback_ttl_secs(app_state, pinned_kind); - app_state.active_provider.refresh_adaptive_playback_lease( - &selected_provider_config.name, - &session_token, - pinned_kind, - session_ttl_secs, - ); - (stream_url, Some(session_token), provider_handle, Some(selected_provider_config)) - } - None => (url, None, None, None), - } - } else { - // Append/shift catchup must keep an m3u-catchup session token even when shared HLS - // cache is off; otherwise rewritten segments register as LiveHls in the panel. - let user_session_token = hls_entry_user_session_token( - fingerprint, - &user.username, - virtual_id, - session_token_hint, - archive_reference, - ); - let hls_session_owner = if hls_cache_enabled_for_user(app_state, target, user) { - let session_key = HlsSessionKey::new(input.id, stream_context.stream_ref()); - let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - Some(build_hls_origin_session_owner(&proxy_session_id)) - } else { - None - }; - let session_owner = hls_session_owner.as_deref().unwrap_or(user_session_token.as_str()); - // Archive playback keeps its own reconnect window semantics, so the lease must - // be classified as catchup rather than plain live HLS. - let playback_kind = hls_playback_kind(archive_reference, &url, Some(user_session_token.as_str())); - let hls_session_ttl_secs = get_hls_playback_ttl_secs(app_state, playback_kind); - let Some(reservation) = try_reserve_hls_entry_origin_account_for_redirect( - app_state, - fingerprint, - user, - input, - virtual_id, - &url, - &user_session_token, - session_owner, - playback_kind, - hls_session_ttl_secs, - connection_permission, - fallback_connection_kind, - true, - ) - .await - else { - return hls_panel_provisioning_or_status_response( - app_state, - user, - input, - virtual_id, - original_hls_entry_path, - server_info.path.as_deref(), - StatusCode::SERVICE_UNAVAILABLE, - ) - .await; - }; - debug_if_enabled!( - "API endpoint [HLS] create_session_fingerprint user={} virtual_id={virtual_id} provider={} stream_url={}", - sanitize_sensitive_info(&user.username), - reservation.selected_provider_config.as_ref().map_or("", |provider| provider.name.as_ref()), - sanitize_sensitive_info(&reservation.request_url) - ); - ( - reservation.request_url, - Some(reservation.session_token), - reservation.provider_handle, - reservation.selected_provider_config, - ) - }; - - // The session as created or reserved above; manifest response cookies may only enter this - // binding, not one an account switch installed while the playlist downloaded. - let session_identity = match session_token.as_deref() { - Some(token) => app_state.active_users.session_identity(&user.username, token).await, - None => None, - }; - let provider_binding_tag = provider_handle.as_ref().and_then(|handle| handle.binding_tag); - let provider_request_id = provider_handle.as_ref().and_then(|handle| handle.playback_request_id); - let selected_provider_name = selected_provider_config.as_ref().map(|cfg| Arc::clone(&cfg.name)); - - // The entry request already downloaded this playlist; serve it once to the immediate variant - // request when it is still bound to the same provider binding. - if matches!(stage, HlsRequestStage::Variant { source: HlsManifestSource::Entry, .. }) { - if let (Some(session_token), Some(provider)) = (session_token.as_deref(), selected_provider_name.as_deref()) { - if let Some(entry) = app_state - .hls - .playlist_handoff - .take(session_token, hls_url) - .filter(|entry| entry.matches(provider, provider_binding_tag)) - { - app_state.connection_manager.release_provider_handle(provider_handle); - release_prepared_hls_manifest_session(app_state, &user.username, session_token, &fingerprint.addr) - .await; - return hls_response(entry.content).into_response(); - } - } - } - - let user_agent_stream_index = crate::api::api_utils::resolve_stream_user_agent_index( - app_state, - input, - provider_handle.is_some(), - &user.username, - session_token.as_deref(), - ) - .await; - if let Some(stream_index) = user_agent_stream_index { - request::append_user_agent_stream_index(&mut headers, stream_index); - if let Some(session_token) = session_token.as_deref() { - app_state - .active_users - .set_user_agent_stream_index_if_absent(&user.username, session_token, stream_index) - .await; - } - } - - // Playlist requests only need the chosen provider account to derive the URL and pin the session. - // Holding the provider slot until the first segment request causes stale active connections and - // breaks forced same-account reuse on the next HLS/Catchup stream request. - app_state.connection_manager.release_provider_handle(provider_handle); - - let input_source = InputSource::from(input).with_url(request_url); - let download_result = match download_legacy_hls_manifest(app_state, &input_source, &headers).await { - Ok((content, response_url, response_headers)) => match classify_hls_playlist(&content) { - // A refresh keeps its session on a malformed body; the player retries the playlist. - HlsPlaylistKind::Invalid if stage != HlsRequestStage::Entry => { - warn!("Upstream HLS refresh returned no playlist for virtual_id={virtual_id}; keeping session"); - if let Some(session_token) = session_token.as_deref() { - release_prepared_hls_manifest_session(app_state, &user.username, session_token, &fingerprint.addr) - .await; - } - return StatusCode::BAD_GATEWAY.into_response(); - } - HlsPlaylistKind::Invalid => { - Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "upstream response is not an HLS playlist")) - } - playlist_kind => Ok((content, response_url, response_headers, playlist_kind)), - }, - Err(err) => Err(err), - }; - match download_result { - Ok((content, response_url, response_headers, playlist_kind)) => { - let encrypt_secret = app_state.get_encrypt_secret(); - let base_url = server_info.get_base_url(); - let rewrite_hls_props = RewriteHlsProps { - secret: &encrypt_secret, - base_url: &base_url, - content: &content, - hls_url: response_url, - target_id: target.id, - virtual_id, - input_id: input.id, - user_token: session_token.as_deref(), - origin_provider: selected_provider_name.as_deref(), - playlist_kind: Some(playlist_kind), - }; - let hls_content = rewrite_hls(user, &rewrite_hls_props); - if let Some(session_token) = session_token.as_deref() { - let session_headers = extract_hls_provider_session_headers(&response_headers); - if let (false, Some(identity)) = (session_headers.is_empty(), session_identity) { - app_state - .active_users - .update_current_session_provider_response_headers_from( - &user.username, - session_token, - identity, - &session_headers, - &rewrite_hls_props.hls_url, - ) - .await; - } - release_prepared_hls_manifest_session(app_state, &user.username, session_token, &fingerprint.addr) - .await; - } - if let (HlsRequestStage::Entry, HlsPlaylistKind::Media, Some(session_token), Some(provider)) = - (stage, playlist_kind, session_token.as_deref(), selected_provider_name.as_ref()) - { - if hls_media_playlist_wrap_enabled(app_state, target, user) { - let master = wrap_media_playlist( - HlsMediaPlaylistWrap { - app_state, - user, - base_url: &base_url, - target_id: target.id, - input, - virtual_id, - session_token, - sealed_url: &entry_url, - provider, - binding_tag: provider_binding_tag, - known_bitrate_bps: stream_context.known_bitrate_bps(), - stream_ref: Some(&stream_ref), - }, - hls_content, - ) - .await; - return hls_response(master).into_response(); - } - } - hls_response(hls_content).into_response() - } - Err(err) => { - error!("Failed to download m3u8: {}", request::text_response_error_log_label(&err)); - if let Some(session_token) = session_token.as_deref() { - terminate_failed_hls_manifest_session( - app_state, - &user.username, - session_token, - selected_provider_name.as_ref(), - provider_binding_tag, - provider_request_id, - ) - .await; - } - - hls_custom_video_manifest_response( - app_state, - user, - CustomVideoStreamType::ChannelUnavailable, - StatusCode::NOT_FOUND, - ) - .await - } - } -} - -pub(super) async fn get_stream_channel( - app_state: &Arc, - target: &Arc, - virtual_id: u32, -) -> Option { - if target.has_output(TargetType::Xtream) { - if let Ok(pli) = - xtream_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target, None).await - { - return Some(pli.to_stream_channel(target.id)); - } - } - let target_id = target.id; - m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target) - .await - .ok() - .map(|pli| pli.to_stream_channel(target_id)) -} - -pub(super) fn hls_stream_context_or_unavailable( - item: &impl PlaylistEntry, - virtual_id: u32, -) -> Result { - HlsEntryStreamContext::from_playlist_item(item).ok_or_else(|| { - warn!("HLS input stream identity missing for virtual_id={virtual_id}; refresh target playlist"); - StatusCode::SERVICE_UNAVAILABLE - }) -} - -pub(in crate::api) async fn resolve_hls_virtual_source_for_target( - app_state: &Arc, - target: &Arc, - virtual_id: u32, -) -> Result { - let (input_name, stream_context) = if target.has_output(TargetType::Xtream) { - if let Ok(item) = - xtream_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target, None).await - { - let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; - (Arc::clone(&item.input_name), stream_context) - } else { - let item = m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target) - .await - .map_err(|_| StatusCode::NOT_FOUND)?; - let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; - (Arc::clone(&item.input_name), stream_context) - } - } else { - let item = m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target) - .await - .map_err(|_| StatusCode::NOT_FOUND)?; - let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; - (Arc::clone(&item.input_name), stream_context) - }; - let input = app_state.app_config.get_input_by_name(&input_name).ok_or(StatusCode::NOT_FOUND)?; - Ok(HlsResolvedVirtualSource { input, stream_context }) -} - -pub(super) async fn resolve_hls_origin_playlist_url( - app_state: &Arc, - target: &Arc, - input: &ConfigInput, - virtual_id: u32, - fallback_url: &str, -) -> Result { - if input.input_type.is_xtream() && target.has_output(TargetType::Xtream) { - let pli = xtream_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target, None) - .await - .map_err(|_| StatusCode::NOT_FOUND)?; - let hls_extension = format!(".{HLS_EXT}"); - let (query_path, _) = get_query_path("", Some(&hls_extension), &pli, app_state); - return get_xtream_player_api_stream_url(input, ApiStreamContext::Live, &query_path, &pli.url) - .map(|url| url.to_string()) - .ok_or(StatusCode::SERVICE_UNAVAILABLE); - } - - Ok(fallback_url.to_string()) -} - -/// Archive (Catchup) or live HLS playback, decided the same way for every HLS path so leases, -/// reservations and TTLs agree. Append/shift catchup often loses utc/utcstart on rewritten -/// segment URLs; the archive path or the session token still identifies archive playback. -pub(super) fn hls_playback_kind( - archive_reference: Option, - url: &str, - session_token: Option<&str>, -) -> PlaybackKind { - if archive_reference.is_some() - || looks_like_archive_media_path(url) - || session_token.is_some_and(is_m3u_catchup_session_token) - { - PlaybackKind::Catchup - } else { - PlaybackKind::LiveHls - } -} - -pub(super) async fn resolve_stream_channel( - app_state: &Arc, - target: &Arc, - input: &Arc, - virtual_id: u32, - hls_url: &str, - archive_reference: Option, - session_token: Option<&str>, -) -> StreamChannel { - let unknown = "Unknown".intern(); - let mut channel = match get_stream_channel(app_state, target, virtual_id).await { - Some(mut channel) => { - channel.url = Arc::from(hls_url); - channel - } - None => StreamChannel { - target_id: target.id, - virtual_id, - provider_id: 0, - input_name: Arc::clone(&input.name), - item_type: PlaylistItemType::LiveHls, - cluster: XtreamCluster::Live, - group: unknown.clone(), - title: unknown, - url: Arc::from(hls_url), - shared: false, - shared_joined_existing: None, - shared_stream_id: None, - technical: None, - epg_channel_id: None, - epg_reference_ts: None, - upstream_user_agent: None, - }, - }; - - let archive_reference = archive_reference.or_else(|| epg_reference_ts_from_date_tree_path(hls_url)); - if hls_playback_kind(archive_reference, hls_url, session_token) == PlaybackKind::Catchup { - channel.item_type = PlaylistItemType::Catchup; - channel.cluster = XtreamCluster::Video; - channel.epg_reference_ts = archive_reference; - } else { - channel.item_type = PlaylistItemType::LiveHls; - channel.epg_reference_ts = None; - } - channel -} - -pub(super) fn hls_entry_user_session_token( - fingerprint: &Fingerprint, - username: &str, - virtual_id: u32, - session_token_hint: Option<&str>, - archive_reference: Option, -) -> String { - // Live hints only arrive from sealed tokens validated by `hls_session_hint_matches_requester`; - // entry routes pass the plain fingerprint key, which never carries the `|hls|` suffix. - if let Some(hint) = - session_token_hint.filter(|token| is_m3u_catchup_session_token(token) || is_live_hls_session_token(token)) - { - return hint.to_string(); - } - if let Some(timestamp) = archive_reference { - return create_m3u_catchup_session_key(fingerprint, username, virtual_id, &format!("archive|{timestamp}|0")); - } - let base = create_playback_session_fingerprint(fingerprint, username, virtual_id, PlaylistItemType::LiveHls, None); - format!("{base}|hls|{}", generate_random_string(16)) -} - -#[allow(clippy::too_many_lines)] -pub(super) async fn hls_api_stream( - fingerprint: Fingerprint, - req_headers: HeaderMap, - axum::extract::RawQuery(raw_query): axum::extract::RawQuery, - axum::extract::Path(params): axum::extract::Path, - axum::extract::State(app_state): axum::extract::State>, -) -> impl IntoResponse + Send { - let internal_user = match params.username.as_str() { - crate::model::RECORDING_PROXY_USERNAME => Some(create_recording_proxy_user(&app_state)), - "api_user" => Some(create_api_proxy_user(&app_state)), - _ => None, - } - .filter(|internal| crate::auth::constant_time_eq(params.password.as_bytes(), internal.password.as_bytes())); - let (user, target) = if let Some(internal_user) = internal_user { - let Some(target) = app_state.app_config.get_target_by_id(params.target_id) else { - return axum::http::StatusCode::BAD_REQUEST.into_response(); - }; - (Arc::new(internal_user), target) - } else { - let Some((user, target)) = app_state.app_config.get_target_for_user(¶ms.username, ¶ms.password) else { - // Credential failure is an auth error, not a malformed request - return app_state.app_config.get_auth_error_status().into_response(); - }; - if target.id != params.target_id { - return axum::http::StatusCode::BAD_REQUEST.into_response(); - } - (user, target) - }; - - // Nested path = relative origin segment that leaked past rewrite_hls (e.g. dvr-YYYY/...). - if params.token.contains('/') { - let Some((token, relative_path)) = params.token.split_once('/') else { - return StatusCode::BAD_REQUEST.into_response(); - }; - let encrypt_secret = app_state.get_encrypt_secret(); - let Some(decoded_hls_token) = get_hls_session_token_and_url_from_token(&encrypt_secret, token) else { - return StatusCode::BAD_REQUEST.into_response(); - }; - let lookup_session_token = decoded_hls_token - .session_token - .clone() - .unwrap_or_else(|| create_session_fingerprint(&fingerprint, &user.username, params.stream_id, false)); - let Some(input) = app_state.app_config.get_input_by_id(params.input_id) else { - return StatusCode::BAD_REQUEST.into_response(); - }; - let input = input_for_user(&app_state.app_config, &user, input); - let Some(session) = app_state - .active_users - .find_latest_session_for_target_stream( - &user.username, - target.id, - input.name.as_ref(), - params.stream_id, - lookup_session_token.as_str(), - ) - .await - else { - return StatusCode::NOT_FOUND.into_response(); - }; - if !legacy_hls_route_allowed_with_cache( - hls_cache_enabled_for_user(&app_state, &target, &user), - decoded_hls_token.session_token.as_deref(), - Some(session.token.as_str()), - ) { - return hls_custom_video_manifest_response_for_username( - &app_state, - &user.username, - CustomVideoStreamType::ChannelUnavailable, - StatusCode::NOT_FOUND, - ) - .await; - } - return hls_api_stream_leaked_relative( - fingerprint, - req_headers, - app_state, - user, - target, - input, - params.stream_id, - session, - decoded_hls_token.url, - relative_path.to_string(), - raw_query.as_deref(), - ) - .await; - } - - hls_api_stream_resolved( - fingerprint, - req_headers, - app_state, - user, - target, - params.input_id, - params.stream_id, - params.token, - ) - .await -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -pub(super) async fn hls_api_stream_resolved( - fingerprint: Fingerprint, - req_headers: HeaderMap, - app_state: Arc, - user: Arc, - target: Arc, - input_id: u16, - stream_id: u32, - token: String, -) -> axum::response::Response { - // Network access check only - permission check is done later with full stream info - if let Err(e) = check_network_access_only(&user, &fingerprint, &app_state.app_config, &app_state.geoip) { - return e.into_player_response(app_state.app_config.get_auth_error_status()); - } - let target_name = &target.name; - let virtual_id = stream_id; - let input = try_option_bad_request!( - app_state.app_config.get_input_by_id(input_id), - true, - format!("Can't find input {} for target {target_name}, stream_id {virtual_id}, hls", input_id) - ); - let input = input_for_user(&app_state.app_config, &user, input); - - if user.permission_denied(&app_state.app_config) { - let stream_channel = resolve_stream_channel(&app_state, &target, &input, virtual_id, "", None, None).await; - return hls_admission_failure_manifest_response( - &app_state, - &fingerprint, - &user, - stream_channel, - input.name.clone(), - &req_headers, - ConnectFailureReason::UserAccountExpired, - ) - .await; - } - - debug_if_enabled!("ID chain for hls endpoint: request_stream_id={stream_id} -> virtual_id={virtual_id}"); - let encrypt_secret = app_state.get_encrypt_secret(); - let Some(decoded_hls_token) = get_hls_session_token_and_url_from_token(&encrypt_secret, &token) else { - return axum::http::StatusCode::BAD_REQUEST.into_response(); - }; - // Manifest vs. media is decided once from the token kind; legacy tokens use the URL heuristic. - let is_manifest = decoded_hls_token.is_manifest(); - let lookup_session_token = decoded_hls_token - .session_token - .clone() - .unwrap_or_else(|| create_session_fingerprint(&fingerprint, &user.username, virtual_id, false)); - let mut user_session = - app_state.active_users.get_and_update_user_session(&user.username, &lookup_session_token).await; - if !legacy_hls_route_allowed_with_cache( - hls_cache_enabled_for_user(&app_state, &target, &user), - decoded_hls_token.session_token.as_deref(), - user_session.as_ref().map(|session| session.token.as_str()), - ) { - return hls_manifest_channel_unavailable_response_for_username(&app_state, &user.username).await; - } - - if let Some(session) = &mut user_session { - let decoded_archive_reference = - resolve_m3u_archive_reference(&decoded_hls_token.url, Some(lookup_session_token.as_str())); - if session.permission == UserConnectionPermission::Exhausted { - let stream_channel = resolve_stream_channel( - &app_state, - &target, - &input, - virtual_id, - &decoded_hls_token.url, - decoded_archive_reference, - Some(session.token.as_str()), - ) - .await; - return hls_admission_failure_manifest_response( - &app_state, - &fingerprint, - &user, - stream_channel, - session.provider.clone(), - &req_headers, - ConnectFailureReason::UserConnectionsExhausted, - ) - .await; - } - - if app_state.active_provider.is_over_limit(&session.provider) { - let stream_channel = resolve_stream_channel( - &app_state, - &target, - &input, - virtual_id, - &decoded_hls_token.url, - decoded_archive_reference, - Some(session.token.as_str()), - ) - .await; - return hls_admission_failure_manifest_response( - &app_state, - &fingerprint, - &user, - stream_channel, - session.provider.clone(), - &req_headers, - ConnectFailureReason::ProviderConnectionsExhausted, - ) - .await; - } - - let hls_url = match decoded_hls_token.session_token.as_deref() { - Some(session_token) if session.token.eq(session_token) => decoded_hls_token.url.as_str(), - None => decoded_hls_token.url.as_str(), - Some(_) => return axum::http::StatusCode::BAD_REQUEST.into_response(), - }; - // A media URI resolved by another provider account would be fetched with that account's - // credentials; the player refetches its playlist from the session's provider instead. - if decoded_hls_token.kind == Some(HlsResourceKind::Media) - && !session.provider.is_empty() - && decoded_hls_token.origin_provider.as_deref().is_some_and(|origin| origin != session.provider.as_ref()) - { - debug_if_enabled!( - "HLS media URI from provider {} rejected for session provider {}", - sanitize_sensitive_info(decoded_hls_token.origin_provider.as_deref().unwrap_or_default()), - sanitize_sensitive_info(&session.provider) - ); - return StatusCode::NOT_FOUND.into_response(); - } - let hls_url = hls_url.intern(); - // Recover utc/utcstart from the prior playlist URL before overwriting with a segment URL - // that usually drops append/shift query params. - let archive_reference = resolve_m3u_archive_reference(&hls_url, Some(session.token.as_str())) - .or_else(|| m3u_archive_epg_reference_ts(session.stream_url.as_ref())); - session.stream_url = hls_url.clone(); - if session.virtual_id == virtual_id { - app_state.connection_manager.touch_http_activity(&user.username, &session.token, &fingerprint.addr).await; - } else { - return axum::http::StatusCode::BAD_REQUEST.into_response(); - } - - let (connection_admission, grace_mode, request_class) = - crate::api::api_utils::resolve_playback_request_admission( - &app_state.admission_ctx(), - &user, - &fingerprint, - Some(session), - &session.token, - true, - crate::api::api_utils::EvictionReentryGuard::Session(&session.token), - // HLS playlist requests are explicit Prepare: they set up session metadata - // but do not consume an admission slot. Segment and other media requests use Activate. - is_manifest, - false, - ) - .await; - let connection_permission = connection_admission.permission(); - let connection_kind = connection_admission.kind().or(session.connection_kind); - session.permission = connection_permission; - if let Some(connection_kind) = connection_kind { - session.connection_kind = Some(connection_kind); - } - if connection_permission == UserConnectionPermission::Exhausted - || (connection_permission == UserConnectionPermission::GracePeriod && connection_kind.is_none()) - { - if connection_admission.is_reentry_suppressed() { - return crate::api::api_utils::reentry_suppressed_response(); - } - let provider = if session.provider.is_empty() { input.name.clone() } else { session.provider.clone() }; - let stream_channel = resolve_stream_channel( - &app_state, - &target, - &input, - virtual_id, - &session.stream_url, - archive_reference, - Some(session.token.as_str()), - ) - .await; - return hls_admission_failure_manifest_response( - &app_state, - &fingerprint, - &user, - stream_channel, - provider, - &req_headers, - ConnectFailureReason::UserConnectionsExhausted, - ) - .await; - } - let fallback_connection_kind = connection_kind.unwrap_or(crate::api::model::ConnectionKind::Normal); - - if is_manifest { - let manifest_stage = match decoded_hls_token.kind { - Some(HlsResourceKind::Manifest(source)) => { - HlsRequestStage::Variant { source, origin_provider: decoded_hls_token.origin_provider.as_deref() } - } - Some(HlsResourceKind::Media) | None => HlsRequestStage::LegacyVariant, - }; - let source = match resolve_hls_virtual_source_for_target(&app_state, &target, virtual_id).await { - Ok(source) if source.input.id == input.id => source, - Ok(source) => { - warn!( - "HLS input context mismatch for virtual_id={virtual_id}: expected_input_id={}, resolved_input_id={}", - input.id, source.input.id - ); - return StatusCode::SERVICE_UNAVAILABLE.into_response(); - } - Err(status) => return status.into_response(), - }; - let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, virtual_id); - return handle_hls_stream_request( - &fingerprint, - &app_state, - &user, - &target, - Some(session), - None, - &session.stream_url, - archive_reference, - source.stream_context, - &input, - &req_headers, - connection_permission, - connection_kind, - &original_hls_entry_path, - manifest_stage, - ) - .await - .into_response(); - } - - if is_file_url(&session.stream_url) { - let stream_channel = resolve_stream_channel( - &app_state, - &target, - &input, - virtual_id, - &hls_url, - archive_reference, - Some(session.token.as_str()), - ) - .await; - return local_stream_response( - &fingerprint, - &app_state, - stream_channel, - &req_headers, - &input, - &target, - &user, - connection_permission, - fallback_connection_kind, - Some(&session.token), - Some(request_class), - false, - ) - .await - .into_response(); - } - - let stream_channel = resolve_stream_channel( - &app_state, - &target, - &input, - virtual_id, - &hls_url, - archive_reference, - Some(session.token.as_str()), - ) - .await; - force_hls_resource_response( - &fingerprint, - &app_state, - session, - stream_channel, - crate::api::api_utils::ForceStreamRequestContext { - req_headers: &req_headers, - input: &input, - user: &user, - session_reservation_ttl_secs: get_hls_session_ttl_secs(&app_state), - content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, - }, - grace_mode, - ) - .await - .into_response() - } else { - recreate_hls_session_from_token( - &fingerprint, - &req_headers, - &app_state, - &user, - &target, - &input, - virtual_id, - &decoded_hls_token, - ) - .await - } -} +mod access; +mod entry; +mod fallback; +mod legacy; +mod registration; +mod resources; +mod routing; +mod segments; +mod transient_cache; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use access::{ + create_hls_cache_user_session_token, current_hls_resource_lease, duration_to_millis_saturating, + hls_access_lease_timing_for_session, hls_access_lease_ttl_ms, hls_cache_response_context, + hls_lease_allows_cached_segment, hls_lease_allows_live_origin_work, hls_live_lease_identity_is_current, + hls_pending_bootstrap_window_ms, hls_qos_meter_init, is_hls_media_activity_status, + log_hls_initial_strip_publication, prepare_hls_resource_access, + register_hls_cache_stream_for_successful_media_response, stripped_tail_segments, + touch_pending_manifest_follow_up_window, validate_hls_proxy_access_context, validate_hls_proxy_access_request, + HlsResourceAccess, +}; +pub(in crate::api) use entry::{build_virtual_hls_entry_path, handle_hls_stream_request}; +pub(super) use entry::{get_stream_channel, HlsCacheManifestOrigin, HlsCacheOriginResolution}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use fallback::{ + hls_custom_video_manifest_response_for_lease, hls_custom_video_manifest_response_for_username, + hls_manifest_access_context_and_state, hls_manifest_access_denial_runtime_response, + hls_manifest_access_lease_validation_response, hls_manifest_channel_unavailable_response_for_username, + hls_origin_runtime_resource_failure_response, hls_resource_access_lease_validation_response, + hls_resource_channel_unavailable_response, hls_resource_serve_outcome_response, hls_runtime_custom_tail_response, + hls_runtime_or_standalone_custom_tail_response, hls_transient_object_unavailable_response, + hls_unpublished_lease_channel_unavailable_response, +}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use legacy::{ + build_hls_manifest_request_headers, download_legacy_hls_manifest, ensure_hls_manifest_extension, + normalize_xtream_live_hls_url, release_prepared_hls_manifest_session, terminate_failed_hls_manifest_session, +}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use registration::{ + build_hls_cache_stream_channel, ensure_hls_cache_stream_registered, fallback_hls_cache_stream_channel, + hls_cache_shared_joined_existing, hls_cache_shared_stream_id, hls_cache_stats_provider, hls_cache_stream_stats_url, +}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use resources::{ + fetch_or_passthrough_transient_resource, hls_proxy_map, hls_proxy_resource, serve_hls_live_transient_resource, + serve_hls_terminal_key_resource, serve_hls_transient_passthrough_result, HlsTransientPassthroughContext, +}; +pub(in crate::api) use routing::resolve_hls_virtual_source_for_target; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use routing::{ + hls_api_stream, hls_api_stream_resolved, hls_entry_user_session_token, hls_playback_kind, + hls_stream_context_or_unavailable, resolve_hls_origin_playlist_url, resolve_stream_channel, +}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use segments::{ + build_hls_segment_fetch_context, demand_fetch_hls_live_segment, demand_fetch_hls_segment_if_needed, + hls_effective_origin_acquire_policy, hls_origin_account_reservation_ttl_secs_fallback, + hls_origin_account_reservation_ttl_secs_for_session, hls_proxy_segment, hls_proxy_terminal_segment, + serve_hls_segment_for_current_lease, validate_hls_segment_entry, +}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(super) use transient_cache::{ + fetch_and_cache_transient_origin_response, fetch_transient_origin_response_with_provider_io, + hls_transient_origin_prepare_closure, safe_transient_resource_id, serve_transient_object_cache_response_and_mark, + serve_transient_object_cache_response_and_mark_or_unavailable, wait_for_transient_object_cache_fetch, + HlsTransientEndpointCacheFetchContext, HlsTransientEndpointOriginFetchRequest, HlsTransientOriginFetchResult, + TransientObjectCacheServeContext, TransientObjectWaitContext, +}; diff --git a/backend/app/src/api/endpoints/hls_api/segment/access.rs b/backend/app/src/api/endpoints/hls_api/segment/access.rs new file mode 100644 index 000000000..89f18c52b --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/segment/access.rs @@ -0,0 +1,393 @@ +use super::{ + ensure_hls_cache_stream_registered, hls_entry_user_session_token, hls_initial_manifest_decision_wait_timeout, + hls_resource_access_lease_validation_response, reclaim_hls_account_overlap_if_needed, + HlsInitialStripPublicationDiagnostic, HlsMaterializedSharedManifest, +}; +use crate::{ + api::model::{hls_cache::initial_strip::HlsInitialStripOutcome, AppState, StreamMeterHandle}, + auth::Fingerprint, +}; +use axum::{ + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; +use log::{debug, warn}; +use shared::utils::generate_random_string; +use std::{sync::Arc, time::Duration}; +use tuliprox_core::utils::current_time_millis; +use tuliprox_hls::api::{ + safe_hls_access_lease_id, safe_proxy_session_id, safe_user_session_token, validate_hls_access_lease, + HlsAccessAdmissionMode, HlsAccessContext, HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseId, + HlsAccessLeaseState, HlsAccessLeaseTiming, HlsAccessLeaseValidationError, HlsCacheResponseContext, + HlsLeasePlaybackMode, HlsLeaseStartupAdmissionState, HlsLogIdentity, HlsMediaActivityMarker, HlsMediaLeaseIdentity, + HlsQosMeterInit, HlsQosRuntimeConfig, HlsSessionHandle, ProxySessionId, +}; + +pub(in crate::api::endpoints::hls_api) fn log_hls_initial_strip_publication( + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + diagnostic: HlsInitialStripPublicationDiagnostic, +) { + match diagnostic { + HlsInitialStripPublicationDiagnostic::Applied { mode, strip_mode, configured, effective, visible_segments } => { + debug!( + "HLS initial strip applied: mode={} lease={} proxy_session={} strip_mode={} configured={} effective={} visible_segments={}", + mode, + safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(proxy_session_id), + strip_mode, + configured, + effective, + visible_segments + ); + } + HlsInitialStripPublicationDiagnostic::Skipped { mode, reason, visible_segments } => { + debug!( + "HLS initial strip skipped: mode={} lease={} proxy_session={} reason={} visible_segments={}", + mode, + safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(proxy_session_id), + reason.as_log_reason(), + visible_segments + ); + } + HlsInitialStripPublicationDiagnostic::SkippedForLeaseState { mode, reason } => { + debug!( + "HLS initial strip skipped: mode={} lease={} proxy_session={} reason={}", + mode, + safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(proxy_session_id), + reason.as_log_reason() + ); + } + } +} + +pub(in crate::api::endpoints::hls_api) fn stripped_tail_segments( + materialized: &HlsMaterializedSharedManifest, +) -> usize { + materialized.initial_strip_outcome.as_ref().map_or(0, |outcome| match outcome { + HlsInitialStripOutcome::Applied { effective, .. } => *effective, + HlsInitialStripOutcome::Skipped { .. } => 0, + }) +} + +pub(in crate::api::endpoints::hls_api) fn hls_access_lease_ttl_ms(app_state: &Arc) -> u64 { + app_state.hls.proxy.session_idle_timeout_ms() +} + +pub(in crate::api::endpoints::hls_api) fn duration_to_millis_saturating(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + +pub(in crate::api::endpoints::hls_api) fn hls_pending_bootstrap_window_ms(app_state: &Arc) -> u64 { + duration_to_millis_saturating(hls_initial_manifest_decision_wait_timeout(app_state)) +} + +pub(in crate::api::endpoints::hls_api) async fn hls_access_lease_timing_for_session( + app_state: &Arc, + session: &HlsSessionHandle, +) -> HlsAccessLeaseTiming { + let timing = session.read().await.account_overlap_timing(); + let active_window_ms = timing.hard_active_window_ms.saturating_mul(2); + HlsAccessLeaseTiming { active_window_ms, valid_window_ms: hls_access_lease_ttl_ms(app_state) } +} + +pub(in crate::api::endpoints::hls_api) async fn touch_pending_manifest_follow_up_window( + app_state: &Arc, + session: &HlsSessionHandle, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, +) { + if access_lease_state != HlsAccessLeaseState::Pending { + return; + } + let (proxy_session_id, target_duration) = { + let session = session.read().await; + (session.proxy_session_id.clone(), session.target_duration) + }; + let now_ms = current_time_millis(); + if !app_state + .hls + .proxy + .mark_pending_manifest_follow_up_for_lease(access_lease_id, &proxy_session_id, now_ms, target_duration) + .await + { + debug!( + "HLS pending manifest follow-up skipped: lease={} proxy_session={} reason=expired-or-generation-race", + safe_hls_access_lease_id(access_lease_id), + safe_proxy_session_id(&proxy_session_id) + ); + } +} + +pub(in crate::api::endpoints::hls_api) struct HlsResourceAccess { + pub(in crate::api::endpoints::hls_api) session: HlsSessionHandle, + pub(in crate::api::endpoints::hls_api) access_context: HlsAccessContext, + pub(in crate::api::endpoints::hls_api) lease: HlsAccessLease, +} + +pub(in crate::api::endpoints::hls_api) async fn prepare_hls_resource_access( + app_state: &Arc, + fingerprint: &Fingerprint, + proxy_session_id: &ProxySessionId, + hls_access_lease_id: &str, + now_ms: u64, + request_kind: &'static str, +) -> Result> { + let Some(session) = app_state.hls.proxy.sessions().get_by_proxy_session_id(proxy_session_id).await else { + return Err(Box::new(StatusCode::NOT_FOUND.into_response())); + }; + app_state + .hls + .proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + proxy_session_id, + now_ms, + ) + .await; + let access_context = match validate_hls_proxy_access_request( + app_state, + fingerprint, + proxy_session_id, + hls_access_lease_id, + now_ms, + hls_access_lease_timing_for_session(app_state, &session).await, + request_kind, + ) + .await + { + Ok(context) => context, + Err(err) => { + return Err(Box::new(hls_resource_access_lease_validation_response(&err))); + } + }; + reclaim_hls_account_overlap_if_needed(app_state, &session, now_ms).await; + let Some(lease) = + app_state.hls.proxy.access_lease_response_snapshot(&access_context.lease_id, proxy_session_id, now_ms).await + else { + return Err(Box::new(StatusCode::NOT_FOUND.into_response())); + }; + Ok(HlsResourceAccess { session, access_context, lease }) +} + +pub(in crate::api::endpoints::hls_api) fn hls_lease_allows_live_origin_work(lease: &HlsAccessLease) -> bool { + matches!(lease.state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle) + && lease.playback_mode == HlsLeasePlaybackMode::Live +} + +pub(in crate::api::endpoints::hls_api) fn hls_lease_allows_cached_segment( + lease: &HlsAccessLease, + proxy_seq: u64, +) -> bool { + match &lease.playback_mode { + HlsLeasePlaybackMode::Live => true, + HlsLeasePlaybackMode::TerminalTail(plan) => plan.protected_base_proxy_seqs.contains(&proxy_seq), + HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => false, + } +} + +pub(in crate::api::endpoints::hls_api) async fn current_hls_resource_lease( + app_state: &Arc, + access_context: &HlsAccessContext, +) -> Option { + app_state + .hls + .proxy + .access_lease_response_snapshot( + &access_context.lease_id, + &access_context.proxy_session_id, + current_time_millis(), + ) + .await +} + +pub(in crate::api::endpoints::hls_api) async fn hls_live_lease_identity_is_current( + app_state: &Arc, + access_context: &HlsAccessContext, + expected_identity: HlsMediaLeaseIdentity, +) -> bool { + current_hls_resource_lease(app_state, access_context).await.is_some_and(|lease| { + lease.playback_mode == HlsLeasePlaybackMode::Live && lease.media_identity() == Some(expected_identity) + }) +} + +pub(in crate::api::endpoints::hls_api) fn create_hls_cache_user_session_token( + fingerprint: &Fingerprint, + username: &str, + virtual_id: u32, + existing_session_token: Option<&str>, + archive_reference: Option, +) -> String { + let base = + hls_entry_user_session_token(fingerprint, username, virtual_id, existing_session_token, archive_reference); + format!("{base}|hls-cache|{}", generate_random_string(16)) +} + +pub(in crate::api::endpoints::hls_api) fn is_hls_media_activity_status(status: StatusCode) -> bool { + matches!(status, StatusCode::OK | StatusCode::PARTIAL_CONTENT) +} + +pub(in crate::api::endpoints::hls_api) async fn hls_cache_response_context( + app_state: &Arc, + session: &HlsSessionHandle, + access_context: &HlsAccessContext, + lease_identity: HlsMediaLeaseIdentity, + now_ms: u64, +) -> HlsCacheResponseContext { + let qos_meter = app_state.hls.proxy.qos().meter_for_access_lease(&access_context.lease_id).await; + let (log_identity, session_owner, playback_request_id) = { + let session = session.read().await; + let binding = session.origin_account_binding.as_ref().filter(|binding| binding.is_active()); + ( + HlsLogIdentity::from_session(&session), + binding.map(|binding| binding.session_owner.clone()), + binding.and_then(|binding| binding.playback_request_id), + ) + }; + HlsCacheResponseContext::new( + access_context.lease_id.clone(), + log_identity, + app_state.hls.proxy.cache_duration_seconds(), + Arc::clone(app_state.hls.proxy.metrics()), + Arc::clone(app_state.hls.proxy.segment_repair()), + qos_meter, + Some( + HlsMediaActivityMarker::new( + Arc::clone(&app_state.hls.proxy), + Arc::clone(session), + access_context.proxy_session_id.clone(), + access_context.lease_id.clone(), + lease_identity, + ) + .with_active_provider( + Arc::clone(&app_state.active_provider), + session_owner, + playback_request_id, + ), + ), + now_ms, + ) +} + +pub(in crate::api::endpoints::hls_api) fn hls_qos_meter_init( + app_state: &Arc, + qos_config: HlsQosRuntimeConfig, +) -> Option { + if !qos_config.live_metering_enabled { + return None; + } + let meter_uid = app_state.connection_manager.next_stream_uid(); + let meter = Arc::new(StreamMeterHandle::new(meter_uid, Arc::downgrade(&app_state.event_manager))); + Some(HlsQosMeterInit { meter_uid, meter }) +} + +pub(in crate::api::endpoints::hls_api) async fn register_hls_cache_stream_for_successful_media_response( + app_state: &Arc, + fingerprint: &Fingerprint, + headers: &HeaderMap, + access_context: &HlsAccessContext, + session: &HlsSessionHandle, + response_context: &HlsCacheResponseContext, +) { + if ensure_hls_cache_stream_registered(app_state, fingerprint, headers, access_context, session).await.is_none() { + debug!( + "HLS media registration skipped: lease={} reason=session-or-connection-unavailable", + safe_hls_access_lease_id(&access_context.lease_id) + ); + } + response_context.set_qos_meter(app_state.hls.proxy.qos().meter_for_access_lease(&access_context.lease_id).await); +} + +pub(in crate::api::endpoints::hls_api) async fn validate_hls_proxy_access_request( + app_state: &Arc, + fingerprint: &Fingerprint, + proxy_session_id: &ProxySessionId, + hls_access_lease_id: &str, + now_ms: u64, + timing: HlsAccessLeaseTiming, + request_kind: &'static str, +) -> Result { + let context = validate_hls_proxy_access_context( + app_state, + fingerprint, + proxy_session_id, + hls_access_lease_id, + now_ms, + HlsAccessAdmissionMode::ResourceAccess, + ) + .await?; + let startup_admission_pending = app_state + .hls + .proxy + .access_lease_response_snapshot(&context.lease_id, proxy_session_id, now_ms) + .await + .is_some_and(|lease| { + lease.state == HlsAccessLeaseState::Pending + && lease.startup_admission == HlsLeaseStartupAdmissionState::Pending + }); + if startup_admission_pending { + return Err(HlsAccessLeaseValidationError::AvailabilityPending); + } + match app_state.hls.proxy.activate_access_lease(&context.lease_id, proxy_session_id, now_ms, timing).await { + HlsAccessLeaseActivation::Activated { .. } => { + debug!( + "HLS access lease accepted: lease={} proxy_session={} user_session={} request={request_kind}", + safe_hls_access_lease_id(&context.lease_id), + safe_proxy_session_id(proxy_session_id), + safe_user_session_token(&context.user_session_token) + ); + Ok(context) + } + HlsAccessLeaseActivation::Denied => { + warn!( + "HLS access lease rejected: lease={} proxy_session={} user_session={} request={request_kind} reason=denied", + safe_hls_access_lease_id(&context.lease_id), + safe_proxy_session_id(proxy_session_id), + safe_user_session_token(&context.user_session_token) + ); + let (runtime_tail, reason) = app_state + .hls + .proxy + .access_lease_response_snapshot(&context.lease_id, proxy_session_id, now_ms) + .await + .map_or((None, None), |lease| { + (lease.runtime_policy_revocation_outcome(), lease.runtime_policy_denial_reason()) + }); + Err(HlsAccessLeaseValidationError::AdmissionDenied { runtime_tail, reason }) + } + HlsAccessLeaseActivation::Expired + | HlsAccessLeaseActivation::UnknownLease + | HlsAccessLeaseActivation::SessionMismatch => { + warn!( + "HLS access lease rejected: lease={} proxy_session={} user_session={} request={request_kind} reason=expired", + safe_hls_access_lease_id(&context.lease_id), + safe_proxy_session_id(proxy_session_id), + safe_user_session_token(&context.user_session_token) + ); + Err(HlsAccessLeaseValidationError::Expired) + } + } +} + +pub(in crate::api::endpoints::hls_api) async fn validate_hls_proxy_access_context( + app_state: &Arc, + fingerprint: &Fingerprint, + proxy_session_id: &ProxySessionId, + hls_access_lease_id: &str, + now_ms: u64, + admission_mode: HlsAccessAdmissionMode, +) -> Result { + validate_hls_access_lease( + &app_state.hls_ctx(), + fingerprint, + proxy_session_id, + &HlsAccessLeaseId(hls_access_lease_id.to_string()), + now_ms, + admission_mode, + ) + .await +} diff --git a/backend/app/src/api/endpoints/hls_api/segment/entry.rs b/backend/app/src/api/endpoints/hls_api/segment/entry.rs new file mode 100644 index 000000000..60541c190 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/segment/entry.rs @@ -0,0 +1,536 @@ +use super::{ + append_user_session_provider_headers, build_hls_manifest_request_headers, build_hls_origin_resolution, + build_hls_origin_source_for_playback, create_hls_cache_entry_master_playlist_response, + download_legacy_hls_manifest, ensure_hls_manifest_extension, hls_cache_enabled_for_user, + hls_custom_video_manifest_response, hls_entry_user_session_token, hls_media_playlist_wrap_enabled, + hls_panel_provisioning_or_status_response, hls_playback_kind, hls_response, m3u_archive_epg_reference_ts, + m3u_catchup_epg_reference_from_session_token, normalize_xtream_live_hls_url, release_prepared_hls_manifest_session, + terminate_failed_hls_manifest_session, try_reserve_hls_entry_origin_account_for_redirect, wrap_media_playlist, + HlsEntryStreamContext, HlsMediaPlaylistWrap, HlsOriginEntryUrl, HlsRequestStage, +}; +use crate::{ + api::{ + api_utils::{ + acquire_exact_provider_handle, connection_priority_for_kind, get_hls_playback_ttl_secs, + select_provider_stream_url, ExactProviderAcquire, HLS_MANIFEST_CAPACITY_WAIT, + }, + model::{AppState, CustomVideoStreamType, PlaybackLeaseRef, ProviderAllocation, UserSession}, + }, + auth::Fingerprint, + model::{ConfigInput, ConfigTarget, InputSource, ProxyUserCredentials}, + processing::parser::hls::{ + classify_hls_playlist, rewrite_hls, HlsManifestSource, HlsPlaylistKind, RewriteHlsProps, + }, + repository::{m3u_get_item_for_stream_id, storage_const, xtream_get_item_for_stream_id}, + utils::{debug_if_enabled, request}, +}; +use axum::{ + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; +use log::{error, warn}; +use shared::{ + defaults::HLS_EXT, + model::{PlaylistItemType, StreamChannel, TargetType, UserConnectionPermission, VirtualId}, + utils::sanitize_sensitive_info, +}; +use std::sync::Arc; +use tuliprox_hls::api::{ + build_hls_origin_session_owner, build_proxy_session_id, extract_hls_provider_session_headers, HlsOriginSource, + HlsSessionKey, +}; + +pub(in crate::api::endpoints::hls_api) struct HlsCacheManifestOrigin<'a> { + pub(in crate::api::endpoints::hls_api) raw_request_url: &'a str, + pub(in crate::api::endpoints::hls_api) session_entry_url: HlsOriginEntryUrl, + pub(in crate::api::endpoints::hls_api) input: &'a ConfigInput, + pub(in crate::api::endpoints::hls_api) origin_source: HlsOriginSource, +} + +pub(in crate::api::endpoints::hls_api) struct HlsCacheOriginResolution { + pub(in crate::api::endpoints::hls_api) hls_url: String, + pub(in crate::api::endpoints::hls_api) session_entry_url: HlsOriginEntryUrl, +} + +pub(in crate::api) fn build_virtual_hls_entry_path( + target: &ConfigTarget, + input: &ConfigInput, + user: &ProxyUserCredentials, + virtual_id: u32, +) -> String { + if input.input_type.is_m3u() && !target.has_output(TargetType::Xtream) { + format!("/{}/live/{}/{}/{}{HLS_EXT}", storage_const::M3U_STREAM_PATH, user.username, user.password, virtual_id) + } else { + format!("/live/{}/{}/{}{HLS_EXT}", user.username, user.password, virtual_id) + } +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +pub(in crate::api) async fn handle_hls_stream_request( + fingerprint: &Fingerprint, + app_state: &Arc, + user: &ProxyUserCredentials, + target: &ConfigTarget, + user_session: Option<&UserSession>, + session_token_hint: Option<&str>, + hls_url: &str, + archive_reference: Option, + stream_context: HlsEntryStreamContext, + input: &ConfigInput, + req_headers: &HeaderMap, + connection_permission: UserConnectionPermission, + connection_kind: Option, + original_hls_entry_path: &str, + stage: HlsRequestStage<'_>, +) -> impl IntoResponse + Send { + let virtual_id = stream_context.virtual_id(); + if app_state.active_users.is_user_blocked_for_stream(&user.username, VirtualId::new(virtual_id)).await { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + } + + let stream_ref = stream_context.stream_ref().to_string(); + // Variant URLs were normalized before sealing or come from an upstream master; use them as is. + let url = if stage.normalizes_url() { + let normalized_hls_url = normalize_xtream_live_hls_url(hls_url, input); + if normalized_hls_url != hls_url { + debug_if_enabled!( + "Normalized xtream hls url from {} to {}", + sanitize_sensitive_info(hls_url), + sanitize_sensitive_info(&normalized_hls_url) + ); + } + ensure_hls_manifest_extension(&normalized_hls_url) + } else { + hls_url.to_string() + }; + // The wrapper seals the normalized entry URL, not the provider-resolved one, so every + // refresh can resolve it to whichever account is selected then. + let entry_url = url.clone(); + // Recover archive context when callers (esp. Xtream timeshift) pass None but the + // resolved provider URL / catchup session still carries Flussonic archive markers. + let archive_reference = archive_reference.or_else(|| m3u_archive_epg_reference_ts(&url)).or_else(|| { + user_session + .map(|session| session.token.as_str()) + .or(session_token_hint) + .and_then(m3u_catchup_epg_reference_from_session_token) + }); + let hls_cache_origin = build_hls_origin_resolution(input, &url); + let hls_origin_source = hls_cache_origin.as_ref().map(|_| { + build_hls_origin_source_for_playback(input, stream_ref.clone(), archive_reference, Some(url.as_str())) + }); + let server_info = app_state.app_config.get_user_server_info(user); + let Some(server_info) = server_info else { + return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; + + let disabled_headers = app_state.get_disabled_headers(); + let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); + let mut headers = build_hls_manifest_request_headers( + &input.headers, + req_headers, + disabled_headers.as_ref(), + default_user_agent.as_deref(), + stream_context.identity().upstream_user_agent(), + ); + + if hls_cache_enabled_for_user(app_state, target, user) { + let Some(origin_source) = hls_origin_source.clone() else { + return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; + return create_hls_cache_entry_master_playlist_response( + app_state, + fingerprint, + user, + origin_source, + virtual_id, + user_session, + stream_context.known_bitrate_bps(), + session_token_hint, + if archive_reference.is_some() { + url.as_str() + } else { + hls_cache_origin.as_ref().map_or(url.as_str(), |origin| origin.session_entry_url.as_str()) + }, + input, + connection_permission, + connection_kind, + server_info.path.as_deref(), + ) + .await; + } + + let fallback_connection_kind = connection_kind.unwrap_or(crate::api::model::ConnectionKind::Normal); + let (request_url, session_token, provider_handle, selected_provider_config) = if let Some(session) = user_session { + let pinned_provider = if session.provider.is_empty() { &input.name } else { &session.provider }; + let pinned_kind = hls_playback_kind(archive_reference, &url, Some(session.token.as_str())); + let session_kind = + session.connection_kind.or(connection_kind).unwrap_or(crate::api::model::ConnectionKind::Normal); + // Manifest refreshes of parallel playbacks on one account free their slots within seconds; + // waiting briefly beats an immediate 503 that stalls the player. + let acquired = acquire_exact_provider_handle( + app_state, + &ExactProviderAcquire { + provider: pinned_provider, + addr: &fingerprint.addr, + allow_grace: false, + priority: connection_priority_for_kind(user, session_kind), + kind: session_kind, + lease: Some(PlaybackLeaseRef::new(session.token.as_str(), pinned_kind)), + }, + Some(HLS_MANIFEST_CAPACITY_WAIT), + ) + .await; + let provider_handle = if let Some(handle) = acquired { + Some(handle) + } else { + debug_if_enabled!( + "HLS pinned provider {} unavailable for {}; aborting allocation to prevent mid-session migration", + sanitize_sensitive_info(pinned_provider), + sanitize_sensitive_info(&fingerprint.addr.to_string()) + ); + None + }; + + if provider_handle.is_none() { + // The capacity wait can outlive the session snapshot: an ended or rebound session must + // not trigger panel provisioning for the stale pinned account. + if app_state.active_users.session_identity(&user.username, &session.token).await != Some(session.identity()) + { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + return hls_panel_provisioning_or_status_response( + app_state, + user, + input, + virtual_id, + original_hls_entry_path, + server_info.path.as_deref(), + StatusCode::SERVICE_UNAVAILABLE, + ) + .await; + } + match provider_handle.as_ref().map(|handle| &handle.allocation) { + Some(ProviderAllocation::Exhausted) => (url, None, provider_handle, None), + Some(ProviderAllocation::Available(cfg) | ProviderAllocation::GracePeriod(cfg)) => { + let selected_provider_config = Arc::clone(cfg); + // URL acceptance follows the provider that produced the sealed URL, not the session: + // entry URLs are re-resolved to the selected account on every fetch, child URLs + // (often CDN URLs) are fetched as sealed and only from their origin provider. + let accept_requested_stream_url = match stage { + HlsRequestStage::Variant { source: HlsManifestSource::Child, origin_provider } => { + if origin_provider != Some(selected_provider_config.name.as_ref()) { + // A stale child URI from an earlier account: reject only this request, + // the playback itself stays valid after the player refetches its master. + app_state.connection_manager.release_provider_handle(provider_handle); + debug_if_enabled!( + "HLS child manifest from provider {} rejected for selected provider {}", + sanitize_sensitive_info(origin_provider.unwrap_or("")), + sanitize_sensitive_info(&selected_provider_config.name) + ); + return StatusCode::NOT_FOUND.into_response(); + } + true + } + HlsRequestStage::Entry | HlsRequestStage::LegacyVariant | HlsRequestStage::Variant { .. } => false, + }; + let Some((_provider_name, stream_url)) = select_provider_stream_url( + &url, + input, + &selected_provider_config, + accept_requested_stream_url, + &app_state.app_config, + ) + .await + else { + app_state.connection_manager.release_provider_handle(provider_handle); + return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; + // The capacity wait can outlive the session snapshot: a session that ended or switched + // accounts meanwhile must not be rebound, and cookies rotated or expired during the + // wait are read now. + let session_headers = app_state + .active_users + .provider_headers_for_session_identity( + &user.username, + &session.token, + session.identity(), + &stream_url, + ) + .await; + if session_headers == tuliprox_session::SessionProviderHeaders::NoSession { + app_state.connection_manager.release_provider_handle(provider_handle); + debug_if_enabled!( + "HLS session {} changed while waiting for pinned provider {}", + sanitize_sensitive_info(&session.token), + sanitize_sensitive_info(pinned_provider) + ); + return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + if session.provider.as_ref() == selected_provider_config.name.as_ref() { + if let tuliprox_session::SessionProviderHeaders::Headers(session_headers) = session_headers { + append_user_session_provider_headers(&mut headers, &session_headers); + } + } + let session_token = app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user, + session_token: &session.token, + virtual_id, + provider: &selected_provider_config.name, + stream_url: &stream_url, + addr: &fingerprint.addr, + connection_permission, + connection_kind: session.connection_kind.or(connection_kind), + socket_bound: PlaylistItemType::LiveHls.uses_socket_bound_session(), + }) + .await; + let session_ttl_secs = get_hls_playback_ttl_secs(app_state, pinned_kind); + app_state.active_provider.refresh_adaptive_playback_lease( + &selected_provider_config.name, + &session_token, + pinned_kind, + session_ttl_secs, + ); + (stream_url, Some(session_token), provider_handle, Some(selected_provider_config)) + } + None => (url, None, None, None), + } + } else { + // Append/shift catchup must keep an m3u-catchup session token even when shared HLS + // cache is off; otherwise rewritten segments register as LiveHls in the panel. + let user_session_token = hls_entry_user_session_token( + fingerprint, + &user.username, + virtual_id, + session_token_hint, + archive_reference, + ); + let hls_session_owner = if hls_cache_enabled_for_user(app_state, target, user) { + let session_key = HlsSessionKey::new(input.id, stream_context.stream_ref()); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + Some(build_hls_origin_session_owner(&proxy_session_id)) + } else { + None + }; + let session_owner = hls_session_owner.as_deref().unwrap_or(user_session_token.as_str()); + // Archive playback keeps its own reconnect window semantics, so the lease must + // be classified as catchup rather than plain live HLS. + let playback_kind = hls_playback_kind(archive_reference, &url, Some(user_session_token.as_str())); + let hls_session_ttl_secs = get_hls_playback_ttl_secs(app_state, playback_kind); + let Some(reservation) = try_reserve_hls_entry_origin_account_for_redirect( + app_state, + fingerprint, + user, + input, + virtual_id, + &url, + &user_session_token, + session_owner, + playback_kind, + hls_session_ttl_secs, + connection_permission, + fallback_connection_kind, + true, + ) + .await + else { + return hls_panel_provisioning_or_status_response( + app_state, + user, + input, + virtual_id, + original_hls_entry_path, + server_info.path.as_deref(), + StatusCode::SERVICE_UNAVAILABLE, + ) + .await; + }; + debug_if_enabled!( + "API endpoint [HLS] create_session_fingerprint user={} virtual_id={virtual_id} provider={} stream_url={}", + sanitize_sensitive_info(&user.username), + reservation.selected_provider_config.as_ref().map_or("", |provider| provider.name.as_ref()), + sanitize_sensitive_info(&reservation.request_url) + ); + ( + reservation.request_url, + Some(reservation.session_token), + reservation.provider_handle, + reservation.selected_provider_config, + ) + }; + + // The session as created or reserved above; manifest response cookies may only enter this + // binding, not one an account switch installed while the playlist downloaded. + let session_identity = match session_token.as_deref() { + Some(token) => app_state.active_users.session_identity(&user.username, token).await, + None => None, + }; + let provider_binding_tag = provider_handle.as_ref().and_then(|handle| handle.binding_tag); + let provider_request_id = provider_handle.as_ref().and_then(|handle| handle.playback_request_id); + let selected_provider_name = selected_provider_config.as_ref().map(|cfg| Arc::clone(&cfg.name)); + + // The entry request already downloaded this playlist; serve it once to the immediate variant + // request when it is still bound to the same provider binding. + if matches!(stage, HlsRequestStage::Variant { source: HlsManifestSource::Entry, .. }) { + if let (Some(session_token), Some(provider)) = (session_token.as_deref(), selected_provider_name.as_deref()) { + if let Some(entry) = app_state + .hls + .playlist_handoff + .take(session_token, hls_url) + .filter(|entry| entry.matches(provider, provider_binding_tag)) + { + app_state.connection_manager.release_provider_handle(provider_handle); + release_prepared_hls_manifest_session(app_state, &user.username, session_token, &fingerprint.addr) + .await; + return hls_response(entry.content).into_response(); + } + } + } + + let user_agent_stream_index = crate::api::api_utils::resolve_stream_user_agent_index( + app_state, + input, + provider_handle.is_some(), + &user.username, + session_token.as_deref(), + ) + .await; + if let Some(stream_index) = user_agent_stream_index { + request::append_user_agent_stream_index(&mut headers, stream_index); + if let Some(session_token) = session_token.as_deref() { + app_state + .active_users + .set_user_agent_stream_index_if_absent(&user.username, session_token, stream_index) + .await; + } + } + + // Playlist requests only need the chosen provider account to derive the URL and pin the session. + // Holding the provider slot until the first segment request causes stale active connections and + // breaks forced same-account reuse on the next HLS/Catchup stream request. + app_state.connection_manager.release_provider_handle(provider_handle); + + let input_source = InputSource::from(input).with_url(request_url); + let download_result = match download_legacy_hls_manifest(app_state, &input_source, &headers).await { + Ok((content, response_url, response_headers)) => match classify_hls_playlist(&content) { + // A refresh keeps its session on a malformed body; the player retries the playlist. + HlsPlaylistKind::Invalid if stage != HlsRequestStage::Entry => { + warn!("Upstream HLS refresh returned no playlist for virtual_id={virtual_id}; keeping session"); + if let Some(session_token) = session_token.as_deref() { + release_prepared_hls_manifest_session(app_state, &user.username, session_token, &fingerprint.addr) + .await; + } + return StatusCode::BAD_GATEWAY.into_response(); + } + HlsPlaylistKind::Invalid => { + Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "upstream response is not an HLS playlist")) + } + playlist_kind => Ok((content, response_url, response_headers, playlist_kind)), + }, + Err(err) => Err(err), + }; + match download_result { + Ok((content, response_url, response_headers, playlist_kind)) => { + let encrypt_secret = app_state.get_encrypt_secret(); + let base_url = server_info.get_base_url(); + let rewrite_hls_props = RewriteHlsProps { + secret: &encrypt_secret, + base_url: &base_url, + content: &content, + hls_url: response_url, + target_id: target.id, + virtual_id, + input_id: input.id, + user_token: session_token.as_deref(), + origin_provider: selected_provider_name.as_deref(), + playlist_kind: Some(playlist_kind), + }; + let hls_content = rewrite_hls(user, &rewrite_hls_props); + if let Some(session_token) = session_token.as_deref() { + let session_headers = extract_hls_provider_session_headers(&response_headers); + if let (false, Some(identity)) = (session_headers.is_empty(), session_identity) { + app_state + .active_users + .update_current_session_provider_response_headers_from( + &user.username, + session_token, + identity, + &session_headers, + &rewrite_hls_props.hls_url, + ) + .await; + } + release_prepared_hls_manifest_session(app_state, &user.username, session_token, &fingerprint.addr) + .await; + } + if let (HlsRequestStage::Entry, HlsPlaylistKind::Media, Some(session_token), Some(provider)) = + (stage, playlist_kind, session_token.as_deref(), selected_provider_name.as_ref()) + { + if hls_media_playlist_wrap_enabled(app_state, target, user) { + let master = wrap_media_playlist( + HlsMediaPlaylistWrap { + app_state, + user, + base_url: &base_url, + target_id: target.id, + input, + virtual_id, + session_token, + sealed_url: &entry_url, + provider, + binding_tag: provider_binding_tag, + known_bitrate_bps: stream_context.known_bitrate_bps(), + stream_ref: Some(&stream_ref), + }, + hls_content, + ) + .await; + return hls_response(master).into_response(); + } + } + hls_response(hls_content).into_response() + } + Err(err) => { + error!("Failed to download m3u8: {}", request::text_response_error_log_label(&err)); + if let Some(session_token) = session_token.as_deref() { + terminate_failed_hls_manifest_session( + app_state, + &user.username, + session_token, + selected_provider_name.as_ref(), + provider_binding_tag, + provider_request_id, + ) + .await; + } + + hls_custom_video_manifest_response( + app_state, + user, + CustomVideoStreamType::ChannelUnavailable, + StatusCode::NOT_FOUND, + ) + .await + } + } +} + +pub(in crate::api::endpoints::hls_api) async fn get_stream_channel( + app_state: &Arc, + target: &Arc, + virtual_id: u32, +) -> Option { + if target.has_output(TargetType::Xtream) { + if let Ok(pli) = + xtream_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target, None).await + { + return Some(pli.to_stream_channel(target.id)); + } + } + let target_id = target.id; + m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target) + .await + .ok() + .map(|pli| pli.to_stream_channel(target_id)) +} diff --git a/backend/app/src/api/endpoints/hls_api/segment/fallback.rs b/backend/app/src/api/endpoints/hls_api/segment/fallback.rs new file mode 100644 index 000000000..fca2112b1 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/segment/fallback.rs @@ -0,0 +1,389 @@ +use super::{ + hls_access_lease_ttl_ms, hls_canonical_status_response, hls_custom_video_manifest_response, + hls_pending_bootstrap_window_ms, hls_temporary_resource_unavailable_response, hls_terminal_playback_response, + validate_hls_proxy_access_context, HlsOriginRuntimeAcquireError, HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, +}; +use crate::{ + api::model::{hls_custom_video_manifest_response_for_access_lease, AppState, CustomVideoStreamType}, + auth::Fingerprint, +}; +use axum::{http::StatusCode, response::IntoResponse}; +use log::{debug, warn}; +use std::sync::Arc; +use tuliprox_core::utils::current_time_millis; +use tuliprox_hls::api::{ + commit_hls_runtime_custom_tail, safe_hls_access_lease_id, safe_proxy_session_id, safe_user_session_token, + HlsAccessAdmissionMode, HlsAccessContext, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, + HlsAccessLeaseState, HlsAccessLeaseTouch, HlsAccessLeaseValidationError, HlsLeasePlaybackMode, + HlsResourceServeFailure, HlsResourceServeOutcome, HlsRuntimeCustomTailOutcome, HlsRuntimeCustomTailReason, + HlsRuntimeCustomTailRequest, HlsSessionHandle, ProxySessionId, TransientObjectUnavailableState, + TransientPassthroughState, TransientResourceFile, +}; + +pub(in crate::api::endpoints::hls_api) async fn hls_custom_video_manifest_response_for_username( + app_state: &Arc, + username: &str, + video_type: CustomVideoStreamType, + fallback_status: StatusCode, +) -> axum::response::Response { + if let Some(user) = app_state.app_config.get_user_credentials(username) { + return hls_custom_video_manifest_response(app_state, &user, video_type, fallback_status).await; + } + fallback_status.into_response() +} + +pub(in crate::api::endpoints::hls_api) async fn hls_custom_video_manifest_response_for_lease( + app_state: &Arc, + lease: &HlsAccessLease, + video_type: CustomVideoStreamType, + fallback_status: StatusCode, +) -> axum::response::Response { + let Some(user) = app_state.app_config.get_user_credentials(&lease.username) else { + return fallback_status.into_response(); + }; + hls_custom_video_manifest_response_for_access_lease(app_state, &user, video_type, fallback_status, lease).await +} + +pub(in crate::api::endpoints::hls_api) async fn hls_runtime_custom_tail_response( + app_state: &Arc, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + reason: HlsRuntimeCustomTailReason, + fallback_status: StatusCode, +) -> axum::response::Response { + let outcome = commit_hls_runtime_custom_tail( + app_state.hls_ctx(), + HlsRuntimeCustomTailRequest { + session: Arc::clone(session), + proxy_session_id: proxy_session_id.clone(), + lease_id: access_lease_id.clone(), + reason, + now_ms: current_time_millis(), + }, + ) + .await; + if matches!(outcome, HlsRuntimeCustomTailOutcome::Committed | HlsRuntimeCustomTailOutcome::AlreadyCommitted) { + let now_ms = current_time_millis(); + if let Some(lease) = + app_state.hls.proxy.access_lease_response_snapshot(access_lease_id, proxy_session_id, now_ms).await + { + if let Some(response) = hls_terminal_playback_response(&lease, proxy_session_id, access_lease_id) { + return response; + } + } + } + if outcome == HlsRuntimeCustomTailOutcome::PendingOwnerRegistered { + return hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS); + } + fallback_status.into_response() +} + +pub(in crate::api::endpoints::hls_api) async fn hls_runtime_or_standalone_custom_tail_response( + app_state: &Arc, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + reason: HlsRuntimeCustomTailReason, + fallback_status: StatusCode, +) -> axum::response::Response { + let now_ms = current_time_millis(); + let Some(lease) = + app_state.hls.proxy.access_lease_response_snapshot(access_lease_id, proxy_session_id, now_ms).await + else { + return fallback_status.into_response(); + }; + match &lease.playback_mode { + HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } => { + if let Some(response) = hls_terminal_playback_response(&lease, proxy_session_id, access_lease_id) { + return response; + } + } + HlsLeasePlaybackMode::Live + if lease.last_manifest_snapshot.is_some() + && matches!(lease.state, HlsAccessLeaseState::Activated | HlsAccessLeaseState::PolicyRevoking) => + { + return hls_runtime_custom_tail_response( + app_state, + session, + proxy_session_id, + access_lease_id, + reason, + fallback_status, + ) + .await; + } + HlsLeasePlaybackMode::Ended if !reason.permits_unpublished_lease_standalone_tail() => { + return fallback_status.into_response(); + } + HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::Ended => {} + } + hls_custom_video_manifest_response_for_lease(app_state, &lease, reason.video_type(), fallback_status).await +} + +pub(in crate::api::endpoints::hls_api) async fn hls_manifest_channel_unavailable_response_for_username( + app_state: &Arc, + username: &str, +) -> axum::response::Response { + hls_custom_video_manifest_response_for_username( + app_state, + username, + CustomVideoStreamType::ChannelUnavailable, + StatusCode::NOT_FOUND, + ) + .await +} + +/// Resolves the final canonical-manifest fallback after refresh and cached-live +/// publication have both produced no response. +pub(in crate::api::endpoints::hls_api) async fn hls_unpublished_lease_channel_unavailable_response( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, +) -> axum::response::Response { + let reason = HlsRuntimeCustomTailReason::ChannelUnavailable; + let lease = app_state + .hls + .proxy + .access_lease_response_snapshot(access_lease_id, proxy_session_id, current_time_millis()) + .await; + if let Some(lease) = lease.filter(|lease| lease.permits_unpublished_standalone_tail(reason)) { + return hls_custom_video_manifest_response_for_lease( + app_state, + &lease, + CustomVideoStreamType::ChannelUnavailable, + StatusCode::NOT_FOUND, + ) + .await; + } + StatusCode::SERVICE_UNAVAILABLE.into_response() +} + +pub(in crate::api::endpoints::hls_api) async fn hls_manifest_access_denial_runtime_response( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + lease_snapshot: Option<&HlsAccessLease>, + reason: HlsRuntimeCustomTailReason, + fallback_status: StatusCode, +) -> axum::response::Response { + let Some(lease) = lease_snapshot else { + return fallback_status.into_response(); + }; + let Some(session) = app_state.hls.proxy.sessions().get_by_proxy_session_id(proxy_session_id).await else { + return fallback_status.into_response(); + }; + hls_runtime_or_standalone_custom_tail_response( + app_state, + &session, + proxy_session_id, + &lease.lease_id, + reason, + fallback_status, + ) + .await +} + +pub(in crate::api::endpoints::hls_api) fn hls_resource_channel_unavailable_response( + _app_state: &Arc, + _access_context: &HlsAccessContext, +) -> axum::response::Response { + StatusCode::NOT_FOUND.into_response() +} + +pub(in crate::api::endpoints::hls_api) fn hls_origin_runtime_resource_failure_response( + _app_state: &Arc, + _access_context: &HlsAccessContext, + err: HlsOriginRuntimeAcquireError, +) -> axum::response::Response { + match err { + HlsOriginRuntimeAcquireError::NoAccountAvailable { .. } => StatusCode::SERVICE_UNAVAILABLE.into_response(), + HlsOriginRuntimeAcquireError::Fatal(status) => hls_canonical_status_response(status), + } +} + +pub(in crate::api::endpoints::hls_api) fn hls_resource_serve_outcome_response( + app_state: &Arc, + access_context: &HlsAccessContext, + outcome: HlsResourceServeOutcome, +) -> axum::response::Response { + match outcome { + HlsResourceServeOutcome::Ready(response) => response, + HlsResourceServeOutcome::Failure(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms }) => { + hls_temporary_resource_unavailable_response(retry_after_ms) + } + HlsResourceServeOutcome::Failure( + HlsResourceServeFailure::Missing + | HlsResourceServeFailure::Expired + | HlsResourceServeFailure::PermanentFailed { .. }, + ) => hls_resource_channel_unavailable_response(app_state, access_context), + } +} + +pub(in crate::api::endpoints::hls_api) async fn hls_manifest_access_lease_validation_response( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + lease_snapshot: Option<&HlsAccessLease>, + err: HlsAccessLeaseValidationError, +) -> axum::response::Response { + match err { + HlsAccessLeaseValidationError::AdmissionDenied { reason, .. } => { + hls_manifest_access_denial_runtime_response( + app_state, + proxy_session_id, + lease_snapshot, + reason.unwrap_or(HlsRuntimeCustomTailReason::UserConnectionsExhausted), + StatusCode::FORBIDDEN, + ) + .await + } + HlsAccessLeaseValidationError::AvailabilityPending => { + hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS) + } + HlsAccessLeaseValidationError::UserSessionMissing { .. } => { + hls_manifest_access_denial_runtime_response( + app_state, + proxy_session_id, + lease_snapshot, + HlsRuntimeCustomTailReason::SessionOrLeaseExpired, + StatusCode::NOT_FOUND, + ) + .await + } + HlsAccessLeaseValidationError::UserAccountExpired { .. } => { + hls_manifest_access_denial_runtime_response( + app_state, + proxy_session_id, + lease_snapshot, + HlsRuntimeCustomTailReason::UserAccountExpired, + StatusCode::FORBIDDEN, + ) + .await + } + HlsAccessLeaseValidationError::Expired => StatusCode::NOT_FOUND.into_response(), + } +} + +pub(in crate::api::endpoints::hls_api) fn hls_resource_access_lease_validation_response( + err: &HlsAccessLeaseValidationError, +) -> axum::response::Response { + match err { + HlsAccessLeaseValidationError::AdmissionDenied { .. } + | HlsAccessLeaseValidationError::UserAccountExpired { .. } => StatusCode::FORBIDDEN.into_response(), + HlsAccessLeaseValidationError::AvailabilityPending => { + hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS) + } + HlsAccessLeaseValidationError::UserSessionMissing { .. } | HlsAccessLeaseValidationError::Expired => { + StatusCode::NOT_FOUND.into_response() + } + } +} + +pub(in crate::api::endpoints::hls_api) async fn hls_manifest_access_context_and_state( + app_state: &Arc, + fingerprint: &Fingerprint, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + access_lease_snapshot: Option<&HlsAccessLease>, + now_ms: u64, +) -> Result<(HlsAccessContext, HlsAccessLeaseState), Box> { + app_state.hls.proxy.startup_observability().record_media_manifest_request(access_lease_id, now_ms); + let access_context = match validate_hls_proxy_access_context( + app_state, + fingerprint, + proxy_session_id, + &access_lease_id.0, + now_ms, + HlsAccessAdmissionMode::ManifestPrepare, + ) + .await + { + Ok(context) => context, + Err(err) => { + warn!( + "HLS access lease rejected: lease={} proxy_session={} user_session=none reason={err:?}", + safe_hls_access_lease_id(access_lease_id), + safe_proxy_session_id(proxy_session_id) + ); + return Err(Box::new( + hls_manifest_access_lease_validation_response(app_state, proxy_session_id, access_lease_snapshot, err) + .await, + )); + } + }; + if access_lease_snapshot.is_none() + && app_state.hls.proxy.sessions().get_by_proxy_session_id(proxy_session_id).await.is_none() + && app_state.hls.proxy.expired_session_marker(proxy_session_id, now_ms).await.is_some() + { + return Err(Box::new(StatusCode::NOT_FOUND.into_response())); + } + debug!( + "HLS access lease accepted: lease={} proxy_session={} user_session={} request=manifest", + safe_hls_access_lease_id(&access_context.lease_id), + safe_proxy_session_id(proxy_session_id), + safe_user_session_token(&access_context.user_session_token) + ); + + let access_lease_state = match app_state + .hls + .proxy + .touch_manifest_access_lease( + &access_context.lease_id, + proxy_session_id, + now_ms, + None, + Some(HlsAccessLeasePendingDeadline::Bootstrap { + deadline_ms: now_ms.saturating_add(hls_pending_bootstrap_window_ms(app_state)), + }), + hls_access_lease_ttl_ms(app_state), + ) + .await + { + HlsAccessLeaseTouch::Touched { lease } => lease.state, + HlsAccessLeaseTouch::Denied => { + return Err(Box::new( + hls_manifest_access_denial_runtime_response( + app_state, + proxy_session_id, + access_lease_snapshot, + HlsRuntimeCustomTailReason::UserConnectionsExhausted, + StatusCode::FORBIDDEN, + ) + .await, + )); + } + HlsAccessLeaseTouch::Expired | HlsAccessLeaseTouch::UnknownLease | HlsAccessLeaseTouch::SessionMismatch => { + return Err(Box::new(StatusCode::NOT_FOUND.into_response())); + } + }; + + Ok((access_context, access_lease_state)) +} + +pub(in crate::api::endpoints::hls_api) async fn hls_transient_object_unavailable_response( + app_state: &Arc, + session: &HlsSessionHandle, + resource_file: &TransientResourceFile, + now_ms: u64, + access_context: &HlsAccessContext, +) -> axum::response::Response { + let state = { + let session = session.read().await; + let key = TransientPassthroughState::transient_object_key( + &session.proxy_session_id, + &resource_file.resource_id, + resource_file.extension.clone(), + ); + session.transient.object_unavailable_state(&key, now_ms) + }; + match state { + TransientObjectUnavailableState::Fetching => { + hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS) + } + TransientObjectUnavailableState::FailedRetryable { retry_after_ms } => { + hls_temporary_resource_unavailable_response(retry_after_ms) + } + TransientObjectUnavailableState::FailedPermanent | TransientObjectUnavailableState::Missing => { + hls_resource_channel_unavailable_response(app_state, access_context) + } + } +} diff --git a/backend/app/src/api/endpoints/hls_api/segment/legacy.rs b/backend/app/src/api/endpoints/hls_api/segment/legacy.rs new file mode 100644 index 000000000..048b3761d --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/segment/legacy.rs @@ -0,0 +1,162 @@ +use crate::{ + api::{ + api_utils::{get_headers_from_request, HeaderFilter}, + model::AppState, + }, + model::{ConfigInput, ConfigInputFlags, InputSource, ReverseProxyDisabledHeaderConfig}, + utils::{content_coding::OutboundContentCodingPolicy, request}, +}; +use axum::http::HeaderMap; +use shared::{defaults::HLS_EXT, utils::replace_url_extension}; +use std::{collections::HashMap, sync::Arc, time::Duration}; +use tuliprox_core::model::{PlaybackRequestId, PlaybackRequestOutcome, ProviderBindingTag}; +use tuliprox_hls::{ + api::{ + force_identity_without_range, scrub_hls_origin_headers, should_remove_hls_origin_header, MAX_HLS_MANIFEST_BYTES, + }, + MAX_MANUAL_REDIRECTS, +}; +use url::Url; + +pub(in crate::api::endpoints::hls_api) async fn release_prepared_hls_manifest_session( + app_state: &Arc, + username: &str, + session_token: &str, + addr: &std::net::SocketAddr, +) { + let _transition_guard = app_state.active_users.acquire_playback_transition(username, session_token).await; + app_state.active_users.release_unbound_session_reservation(username, session_token, None, false).await; + app_state.active_users.clear_unbound_session_addr(username, session_token, addr).await; +} + +pub(in crate::api::endpoints::hls_api) async fn terminate_failed_hls_manifest_session( + app_state: &Arc, + username: &str, + session_token: &str, + provider_name: Option<&Arc>, + binding_tag: Option, + request_id: Option, +) { + let _transition_guard = app_state.active_users.acquire_playback_transition(username, session_token).await; + app_state.active_users.terminate_session(username, session_token).await; + let mut cleared = false; + if let Some(request_id) = request_id { + app_state.active_provider.finish_identified_playback_request( + session_token, + request_id, + PlaybackRequestOutcome::ProviderFailed, + ); + cleared = true; + } else if let (Some(provider_name), Some(binding_tag)) = (provider_name, binding_tag) { + // A failed manifest lets the next entry fall back to another provider, but only + // when the preference still belongs to this binding. + app_state.active_provider.forget_identified_provider_affinity(session_token, provider_name, binding_tag); + app_state.active_provider.clear_identified_provider_reservation( + session_token, + provider_name, + Some(binding_tag), + ); + cleared = true; + } + if !cleared { + app_state.active_provider.clear_provider_reservation(session_token); + } +} + +pub(in crate::api::endpoints::hls_api) fn normalize_xtream_live_hls_url(hls_url: &str, input: &ConfigInput) -> String { + if !input.input_type.is_xtream() || !input.has_flag(ConfigInputFlags::XtreamLiveStreamUsePrefix) { + return hls_url.to_string(); + } + + let (Some(username), Some(password)) = (input.username.as_deref(), input.password.as_deref()) else { + return hls_url.to_string(); + }; + + let Ok(mut parsed) = Url::parse(hls_url) else { + return hls_url.to_string(); + }; + let Some(segments) = parsed.path_segments() else { + return hls_url.to_string(); + }; + + let parts: Vec<&str> = segments.collect(); + if parts.len() >= 3 && parts[0] == username && parts[1] == password { + parsed.set_path(&format!("/live/{}", parts.join("/"))); + return parsed.to_string(); + } + + hls_url.to_string() +} + +pub(in crate::api::endpoints::hls_api) fn ensure_hls_manifest_extension(url: &str) -> String { + let with_extension = replace_url_extension(url, HLS_EXT); + let (base_url, suffix) = match with_extension.find(['?', '#'].as_ref()) { + Some(pos) => (&with_extension[..pos], &with_extension[pos..]), + None => (with_extension.as_str(), ""), + }; + let Some(path_without_ext) = base_url.strip_suffix(HLS_EXT) else { + return with_extension; + }; + format!("{}{}{}", path_without_ext.trim_end_matches('.'), HLS_EXT, suffix) +} + +pub(in crate::api::endpoints::hls_api) fn build_hls_manifest_request_headers( + input_headers: &HashMap, + req_headers: &HeaderMap, + disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, + default_user_agent: Option<&str>, + upstream_user_agent: Option<&str>, +) -> HeaderMap { + let input_headers = input_headers + .iter() + .filter(|(key, _)| !should_remove_hls_origin_header(key, disabled_headers)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect::>(); + let disabled_headers_for_filter = disabled_headers.cloned(); + let filter_header: HeaderFilter = Some(Box::new(move |name: &str| { + !name.eq_ignore_ascii_case("range") + && !should_remove_hls_origin_header(name, disabled_headers_for_filter.as_ref()) + })); + let forwarded = get_headers_from_request(req_headers, &filter_header); + let mut headers = + request::get_request_headers(Some(&input_headers), Some(&forwarded), disabled_headers, default_user_agent); + request::overlay_upstream_user_agent(&mut headers, upstream_user_agent, disabled_headers); + scrub_hls_origin_headers(&mut headers, disabled_headers); + force_identity_without_range(&mut headers); + headers +} + +pub(in crate::api::endpoints::hls_api) async fn download_legacy_hls_manifest( + app_state: &Arc, + input: &InputSource, + headers: &HeaderMap, +) -> Result<(String, String, HeaderMap), std::io::Error> { + let deadline = Duration::from_millis(app_state.hls.proxy.origin_manifest_timeout_ms().max(1)); + let fetch_options = request::RequestFetchOptions::with_attempt_idle_timeout(deadline) + .with_content_coding(OutboundContentCodingPolicy::Identity); + let body_options = request::TextContentBodyOptions::hls_manifest(MAX_HLS_MANIFEST_BYTES, deadline); + let options = request::TextContentFetchOptions::new(fetch_options, body_options); + + if app_state.should_use_manual_redirects() { + request::download_text_content_with_manual_redirects_and_headers_and_options( + &app_state.app_config, + &app_state.http_clients.no_redirect.load(), + input, + Some(headers), + false, + MAX_MANUAL_REDIRECTS, + options, + ) + .await + } else { + request::download_text_content_with_headers_and_options( + &app_state.app_config, + &app_state.http_clients.default.load(), + input, + Some(headers), + false, + options, + ) + .await + } +} diff --git a/backend/app/src/api/endpoints/hls_api/segment/registration.rs b/backend/app/src/api/endpoints/hls_api/segment/registration.rs new file mode 100644 index 000000000..935159283 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/segment/registration.rs @@ -0,0 +1,231 @@ +use super::{ + get_stream_channel, hls_qos_meter_init, m3u_archive_epg_reference_ts, m3u_catchup_epg_reference_from_session_token, +}; +use crate::{ + api::{ + api_utils::connection_priority_for_kind, + model::{AppState, ConnectionHistoryMode, UserSession}, + }, + auth::Fingerprint, +}; +use axum::http::{header, HeaderMap}; +use sha2::{Digest, Sha256}; +use shared::{ + model::{PlaylistItemType, StreamChannel, StreamInfo, XtreamCluster}, + utils::{is_m3u_catchup_session_token, Internable}, +}; +use std::{borrow::Cow, sync::Arc}; +use tuliprox_core::utils::current_time_millis; +use tuliprox_hls::api::{ + HlsAccessContext, HlsOriginAccountBinding, HlsOriginSource, HlsQosRuntimeConfig, HlsSessionHandle, ProxySessionId, +}; + +pub(in crate::api::endpoints::hls_api) async fn ensure_hls_cache_stream_registered( + app_state: &Arc, + fingerprint: &Fingerprint, + req_headers: &HeaderMap, + access: &HlsAccessContext, + session: &HlsSessionHandle, +) -> Option { + let (proxy_session_id, origin_source, origin_account_binding) = { + let session = session.read().await; + if session.is_gc_marked_for_removal() { + return None; + } + (session.proxy_session_id.clone(), session.origin_source.clone(), session.origin_account_binding.clone()) + }; + let user = app_state.app_config.get_user_credentials(&access.username)?; + let user_session = + app_state.active_users.get_and_update_user_session(&access.username, &access.user_session_token).await?; + let connection_kind = user_session.connection_kind?; + let priority = connection_priority_for_kind(&user, connection_kind); + let mut stream_channel = build_hls_cache_stream_channel(app_state, access, &origin_source, &proxy_session_id).await; + let provider = hls_cache_stats_provider(&origin_source, origin_account_binding.as_ref(), &user_session); + let user_agent = req_headers + .get(header::USER_AGENT) + .map_or_else(|| Cow::Borrowed(""), |value| String::from_utf8_lossy(value.as_bytes())); + + stream_channel.url = Arc::from(hls_cache_stream_stats_url(&proxy_session_id)); + // Panel Streams/History read this item_type. Shared HLS transport is still HLS, but + // archive/catchup leases must never be published as Live/LiveHls. + let panel_archive_reference = origin_source + .archive_reference + .or(access.epg_reference_ts) + .or_else(|| access.archive_origin_url.as_deref().and_then(m3u_archive_epg_reference_ts)) + .or_else(|| m3u_catchup_epg_reference_from_session_token(&access.user_session_token)) + .or(stream_channel.epg_reference_ts); + let is_archive_playback = panel_archive_reference.is_some() + || access.archive_origin_url.is_some() + || origin_source.archive_reference.is_some() + || is_m3u_catchup_session_token(&access.user_session_token) + || stream_channel.item_type == PlaylistItemType::Catchup; + if is_archive_playback { + stream_channel.item_type = PlaylistItemType::Catchup; + stream_channel.cluster = XtreamCluster::Video; + stream_channel.epg_reference_ts = panel_archive_reference; + } else { + stream_channel.item_type = PlaylistItemType::LiveHls; + stream_channel.cluster = PlaylistItemType::LiveHls.cluster(); + } + let shared_stream_id = hls_cache_shared_stream_id(&proxy_session_id); + stream_channel.shared = true; + stream_channel.shared_stream_id = Some(shared_stream_id); + stream_channel.shared_joined_existing = Some( + hls_cache_shared_joined_existing(app_state, shared_stream_id, &access.username, &access.user_session_token) + .await, + ); + let qos_config = HlsQosRuntimeConfig::from_app_config(&app_state.app_config); + let qos_registration = app_state + .hls + .proxy + .qos() + .ensure_access_lease( + &access.lease_id, + &proxy_session_id, + current_time_millis(), + hls_qos_meter_init(app_state, qos_config), + ) + .await; + if let Some(meter) = qos_registration.register_meter.as_ref() { + app_state.event_manager.register_meter(Arc::clone(meter)).await; + } + let history_mode = if qos_registration.emit_connect_record { + ConnectionHistoryMode::EmitConnect + } else { + ConnectionHistoryMode::RefreshOnly + }; + + app_state + .connection_manager + .update_connection_with_history_mode( + crate::api::model::ConnectionParams { + meter_uid: qos_registration.meter_uid, + username: &access.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind, + priority, + soft_priority: user.soft_priority, + fingerprint, + provider, + stream_channel: &stream_channel, + user_agent, + session_token: Some(&access.user_session_token), + }, + history_mode, + ) + .await +} + +pub(in crate::api::endpoints::hls_api) fn hls_cache_stats_provider( + origin_source: &HlsOriginSource, + origin_account_binding: Option<&HlsOriginAccountBinding>, + user_session: &UserSession, +) -> Arc { + origin_account_binding.filter(|binding| binding.is_active()).map_or_else( + || { + if user_session.provider.is_empty() { + Arc::clone(&origin_source.input_name) + } else { + Arc::clone(&user_session.provider) + } + }, + |binding| Arc::clone(&binding.account_name), + ) +} + +pub(in crate::api::endpoints::hls_api) async fn build_hls_cache_stream_channel( + app_state: &Arc, + access: &HlsAccessContext, + origin_source: &HlsOriginSource, + proxy_session_id: &ProxySessionId, +) -> StreamChannel { + let mut channel = if let Some((_, target)) = app_state.app_config.get_target_for_username(&access.username) { + if let Some(mut channel) = get_stream_channel(app_state, &target, access.virtual_id).await { + channel.url = Arc::from(hls_cache_stream_stats_url(proxy_session_id)); + channel + } else { + fallback_hls_cache_stream_channel(target.id, access.virtual_id, origin_source, proxy_session_id) + } + } else { + fallback_hls_cache_stream_channel(0, access.virtual_id, origin_source, proxy_session_id) + }; + + let archive_reference = access + .epg_reference_ts + .or_else(|| access.archive_origin_url.as_deref().and_then(m3u_archive_epg_reference_ts)) + .or_else(|| m3u_catchup_epg_reference_from_session_token(&access.user_session_token)); + + if archive_reference.is_some() + || access.archive_origin_url.is_some() + || is_m3u_catchup_session_token(&access.user_session_token) + { + channel.item_type = PlaylistItemType::Catchup; + channel.cluster = XtreamCluster::Video; + channel.epg_reference_ts = archive_reference; + } else { + channel.item_type = PlaylistItemType::LiveHls; + channel.cluster = PlaylistItemType::LiveHls.cluster(); + channel.epg_reference_ts = None; + } + channel +} + +pub(in crate::api::endpoints::hls_api) fn fallback_hls_cache_stream_channel( + target_id: u16, + virtual_id: u32, + origin_source: &HlsOriginSource, + proxy_session_id: &ProxySessionId, +) -> StreamChannel { + let unknown = "Unknown".intern(); + StreamChannel { + target_id, + virtual_id, + provider_id: 0, + input_name: Arc::clone(&origin_source.input_name), + item_type: PlaylistItemType::LiveHls, + cluster: XtreamCluster::Live, + group: unknown.clone(), + title: unknown, + url: Arc::from(hls_cache_stream_stats_url(proxy_session_id)), + shared: false, + shared_joined_existing: None, + shared_stream_id: None, + technical: None, + epg_channel_id: None, + epg_reference_ts: None, + upstream_user_agent: None, + } +} + +pub(in crate::api::endpoints::hls_api) fn hls_cache_stream_stats_url(proxy_session_id: &ProxySessionId) -> String { + format!("/hls/shared/live/{}/manifest.m3u8", proxy_session_id.0) +} + +pub(in crate::api::endpoints::hls_api) fn hls_cache_shared_stream_id(proxy_session_id: &ProxySessionId) -> u64 { + let digest = Sha256::digest(proxy_session_id.0.as_bytes()); + digest.iter().take(8).fold(0_u64, |value, byte| (value << 8) | u64::from(*byte)) +} + +pub(in crate::api::endpoints::hls_api) async fn hls_cache_shared_joined_existing( + app_state: &Arc, + shared_stream_id: u64, + username: &str, + session_token: &str, +) -> bool { + let streams = app_state.active_users.active_streams().await; + if let Some(existing) = streams.iter().find(|stream| { + stream.username == username + && stream.session_token.as_deref() == Some(session_token) + && stream.channel.shared + && stream.channel.shared_stream_id == Some(shared_stream_id) + }) { + return existing.channel.shared_joined_existing.unwrap_or(false); + } + + streams.iter().any(|stream| { + stream.channel.shared + && stream.channel.shared_stream_id == Some(shared_stream_id) + && (stream.username != username || stream.session_token.as_deref() != Some(session_token)) + }) +} diff --git a/backend/app/src/api/endpoints/hls_api/segment/resources.rs b/backend/app/src/api/endpoints/hls_api/segment/resources.rs new file mode 100644 index 000000000..f9c68d7ea --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/segment/resources.rs @@ -0,0 +1,455 @@ +use super::{ + current_hls_resource_lease, ensure_hls_cache_stream_registered, fetch_and_cache_transient_origin_response, + fetch_transient_origin_response_with_provider_io, hls_cache_response_context, hls_lease_allows_live_origin_work, + hls_origin_runtime_resource_failure_response, hls_resource_channel_unavailable_response, + hls_resource_serve_outcome_response, hls_temporary_resource_unavailable_response, is_hls_media_activity_status, + prepare_hls_resource_access, register_hls_cache_stream_for_successful_media_response, + serve_transient_object_cache_response_and_mark_or_unavailable, wait_for_transient_object_cache_fetch, + HlsProxyMapPathParams, HlsProxyResourcePathParams, HlsResourceAccess, HlsResourceEndpointContext, + HlsTransientEndpointCacheFetchContext, HlsTransientEndpointOriginFetchRequest, HlsTransientOriginFetchResult, + TransientObjectCacheServeContext, TransientObjectWaitContext, HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, +}; +use crate::{api::model::AppState, auth::Fingerprint}; +use axum::{ + http::{header, HeaderMap, StatusCode}, + response::IntoResponse, +}; +use log::debug; +use std::sync::Arc; +use tuliprox_core::utils::current_time_millis; +use tuliprox_hls::api::{ + finite_hls_terminal_key_response, hls_transient_object_fetch_failure, hls_transient_origin_response, + record_temporary_transient_segment_fetch_failure, resolve_hls_transient_object_cache_action, + safe_hls_access_lease_id, safe_proxy_session_id, serve_hls_map_cache_outcome, HlsLeasePlaybackMode, HlsLogIdentity, + HlsMapFile, HlsMediaActivityCommitOutcome, HlsOriginResourceFetchError, HlsTransientDirectResponseContext, + HlsTransientObjectCacheAction, HlsTransientObjectFetchFailure, HlsTransientResourceLeaseContext, ProxySessionId, + SegmentFetchPolicy, TransientResourceFile, TransientResourceKind, TransientResourceRef, +}; + +pub(in crate::api::endpoints::hls_api) async fn hls_proxy_map( + fingerprint: Fingerprint, + axum::extract::Path(params): axum::extract::Path, + axum::extract::State(app_state): axum::extract::State>, + headers: HeaderMap, +) -> axum::response::Response { + let proxy_session_id = ProxySessionId(params.proxy_session_id); + let now_ms = current_time_millis(); + let HlsResourceAccess { session, access_context, lease } = match prepare_hls_resource_access( + &app_state, + &fingerprint, + &proxy_session_id, + ¶ms.hls_access_lease_id, + now_ms, + "map", + ) + .await + { + Ok(access) => access, + Err(response) => return *response, + }; + if !hls_lease_allows_live_origin_work(&lease) { + return StatusCode::NOT_FOUND.into_response(); + } + let Some(map_file) = HlsMapFile::parse(¶ms.map_file) else { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + }; + { + let session_guard = session.read().await; + if session_guard.is_gc_marked_for_removal() { + return StatusCode::NOT_FOUND.into_response(); + } + let Some(entry) = session_guard.maps.get(&map_file.proxy_map_id.into()) else { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + }; + if entry.proxy_file_ext != map_file.extension { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + } + } + + let Some(current_lease) = current_hls_resource_lease(&app_state, &access_context).await else { + return StatusCode::NOT_FOUND.into_response(); + }; + if current_lease.playback_mode != HlsLeasePlaybackMode::Live { + return StatusCode::NOT_FOUND.into_response(); + } + let Some(lease_identity) = current_lease.media_identity() else { + return StatusCode::NOT_FOUND.into_response(); + }; + let response_context = + hls_cache_response_context(&app_state, &session, &access_context, lease_identity, now_ms).await; + let response = hls_resource_serve_outcome_response( + &app_state, + &access_context, + serve_hls_map_cache_outcome( + Arc::clone(app_state.hls.proxy.segment_cache()), + Arc::clone(&session), + map_file, + headers.get(header::RANGE).cloned(), + &response_context, + ) + .await, + ); + if is_hls_media_activity_status(response.status()) { + register_hls_cache_stream_for_successful_media_response( + &app_state, + &fingerprint, + &headers, + &access_context, + &session, + &response_context, + ) + .await; + } + response +} + +pub(in crate::api::endpoints::hls_api) async fn hls_proxy_resource( + fingerprint: Fingerprint, + axum::extract::Path(params): axum::extract::Path, + axum::extract::State(app_state): axum::extract::State>, + headers: HeaderMap, +) -> axum::response::Response { + let proxy_session_id = ProxySessionId(params.proxy_session_id); + let now_ms = current_time_millis(); + let HlsResourceAccess { session, access_context, lease } = match prepare_hls_resource_access( + &app_state, + &fingerprint, + &proxy_session_id, + ¶ms.hls_access_lease_id, + now_ms, + "resource", + ) + .await + { + Ok(access) => access, + Err(response) => return *response, + }; + let Some(resource_file) = TransientResourceFile::parse(¶ms.resource_file) else { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + }; + let Some(lease_identity) = lease.media_identity() else { + return StatusCode::NOT_FOUND.into_response(); + }; + let context = HlsResourceEndpointContext { + app_state: &app_state, + session: &session, + fingerprint: &fingerprint, + headers: &headers, + access_context: &access_context, + lease_identity, + published_resource_ids: lease.published_transient_resource_ids().clone(), + resource_file, + range_header: headers.get(header::RANGE).cloned(), + now_ms, + }; + match &lease.playback_mode { + HlsLeasePlaybackMode::Live => serve_hls_live_transient_resource(context).await, + HlsLeasePlaybackMode::TerminalTail(_) => serve_hls_terminal_key_resource(context, &lease.playback_mode).await, + HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => { + StatusCode::NOT_FOUND.into_response() + } + } +} + +pub(in crate::api::endpoints::hls_api) async fn serve_hls_terminal_key_resource( + context: HlsResourceEndpointContext<'_>, + playback_mode: &HlsLeasePlaybackMode, +) -> axum::response::Response { + let HlsLeasePlaybackMode::TerminalTail(plan) = playback_mode else { + return StatusCode::NOT_FOUND.into_response(); + }; + let proxy_session_id = &context.access_context.proxy_session_id; + let Some(binding) = + plan.terminal_key_binding(proxy_session_id, &context.access_context.lease_id, &context.resource_file) + else { + return StatusCode::NOT_FOUND.into_response(); + }; + if !context.session.read().await.terminal_key_binding_is_current( + &context.access_context.lease_id, + plan.generation, + &binding, + ) { + return StatusCode::NOT_FOUND.into_response(); + } + let response_context = hls_cache_response_context( + context.app_state, + context.session, + context.access_context, + context.lease_identity, + context.now_ms, + ) + .await; + let response = finite_hls_terminal_key_response( + binding.bytes(), + context.range_header.as_ref(), + binding.content_type(), + "private, max-age=300, immutable", + &response_context, + proxy_session_id, + context.resource_file.resource_id.0, + ); + if is_hls_media_activity_status(response.status()) { + response_context.mark_media_activity().await; + register_hls_cache_stream_for_successful_media_response( + context.app_state, + context.fingerprint, + context.headers, + context.access_context, + context.session, + &response_context, + ) + .await; + } + response +} + +pub(in crate::api::endpoints::hls_api) async fn serve_hls_live_transient_resource( + context: HlsResourceEndpointContext<'_>, +) -> axum::response::Response { + let cache_duration_ms = context.app_state.hls.proxy.cache_duration_seconds().saturating_mul(1_000); + let Ok(cache_resolution) = resolve_hls_transient_object_cache_action( + context.session, + &context.access_context.proxy_session_id, + HlsTransientResourceLeaseContext { + access_lease_id: &context.access_context.lease_id, + lease_issued_at_ms: context.lease_identity.lease_issued_at_ms(), + published_resource_ids: &context.published_resource_ids, + }, + &context.resource_file, + context.range_header.as_ref(), + context.now_ms, + cache_duration_ms, + ) + .await + else { + return hls_resource_channel_unavailable_response(context.app_state, context.access_context); + }; + let resource = cache_resolution.resource; + let origin_headers = cache_resolution.origin_headers; + let origin_provider_session_headers = cache_resolution.origin_provider_session_headers; + let cache_action = cache_resolution.action; + + match cache_action { + HlsTransientObjectCacheAction::ServeReady => { + return serve_transient_object_cache_response_and_mark_or_unavailable(TransientObjectCacheServeContext { + app_state: context.app_state, + session: context.session, + fingerprint: context.fingerprint, + headers: context.headers, + access_context: context.access_context, + lease_identity: context.lease_identity, + resource_file: context.resource_file, + range_header: context.range_header, + now_ms: context.now_ms, + }) + .await; + } + HlsTransientObjectCacheAction::WaitForFetch(notifier) => { + return wait_for_transient_object_cache_fetch(TransientObjectWaitContext { + app_state: context.app_state, + session: context.session, + fingerprint: context.fingerprint, + headers: context.headers, + access_context: context.access_context, + lease_identity: context.lease_identity, + resource_file: context.resource_file, + range_header: context.range_header, + notifier, + }) + .await; + } + HlsTransientObjectCacheAction::FetchAndCache(_) | HlsTransientObjectCacheAction::PassthroughNoCache => {} + } + + fetch_or_passthrough_transient_resource(HlsTransientPassthroughContext { + endpoint: context, + resource, + cache_action, + origin_headers, + origin_provider_session_headers, + cache_duration_ms, + }) + .await +} + +pub(in crate::api::endpoints::hls_api) struct HlsTransientPassthroughContext<'a> { + pub(in crate::api::endpoints::hls_api) endpoint: HlsResourceEndpointContext<'a>, + pub(in crate::api::endpoints::hls_api) resource: TransientResourceRef, + pub(in crate::api::endpoints::hls_api) cache_action: HlsTransientObjectCacheAction, + pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap, + pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap, + pub(in crate::api::endpoints::hls_api) cache_duration_ms: u64, +} + +pub(in crate::api::endpoints::hls_api) async fn fetch_or_passthrough_transient_resource( + context: HlsTransientPassthroughContext<'_>, +) -> axum::response::Response { + let HlsTransientPassthroughContext { + endpoint, + resource, + cache_action, + origin_headers, + origin_provider_session_headers, + cache_duration_ms, + } = context; + if let HlsTransientObjectCacheAction::FetchAndCache(fetch_token) = cache_action { + return fetch_and_cache_transient_origin_response(HlsTransientEndpointCacheFetchContext { + app_state: endpoint.app_state, + session: endpoint.session, + fingerprint: endpoint.fingerprint, + headers: endpoint.headers, + access_context: endpoint.access_context, + lease_identity: endpoint.lease_identity, + resource: &resource, + resource_file: endpoint.resource_file, + fetch_token: *fetch_token, + origin_headers, + origin_provider_session_headers, + range_header: endpoint.range_header, + cache_duration_ms, + }) + .await; + } + + let policy = endpoint.app_state.hls.proxy.segment_fetch_policy(); + let fetch_result = fetch_transient_origin_response_with_provider_io(HlsTransientEndpointOriginFetchRequest { + app_state: endpoint.app_state, + session: endpoint.session, + access_context: endpoint.access_context, + fingerprint: endpoint.fingerprint, + headers: endpoint.headers, + resource: &resource, + resource_file: &endpoint.resource_file, + origin_headers, + origin_provider_session_headers, + range_header: endpoint.range_header.clone(), + policy: policy.clone(), + }) + .await; + serve_hls_transient_passthrough_result(endpoint, resource, policy, fetch_result).await +} + +#[allow(clippy::too_many_lines)] +pub(in crate::api::endpoints::hls_api) async fn serve_hls_transient_passthrough_result( + endpoint: HlsResourceEndpointContext<'_>, + resource: TransientResourceRef, + policy: SegmentFetchPolicy, + fetch_result: HlsTransientOriginFetchResult, +) -> axum::response::Response { + match fetch_result.result { + Ok(response) => { + if response.decoded.status.is_success() { + let activity_outcome = endpoint + .app_state + .hls + .proxy + .mark_authorized_media_access_for_lease_if_identity_matches( + endpoint.session, + &endpoint.access_context.lease_id, + &endpoint.access_context.proxy_session_id, + endpoint.lease_identity, + endpoint.now_ms, + ) + .await; + match activity_outcome { + HlsMediaActivityCommitOutcome::Committed => {} + HlsMediaActivityCommitOutcome::StaleLeaseIdentity => { + debug!( + "HLS transient media response discarded: lease={} proxy_session={} reason=playback-generation-race", + safe_hls_access_lease_id(&endpoint.access_context.lease_id), + safe_proxy_session_id(&endpoint.access_context.proxy_session_id) + ); + return StatusCode::NOT_FOUND.into_response(); + } + HlsMediaActivityCommitOutcome::DeferredLockContention => { + debug!( + "HLS transient media response deferred: lease={} proxy_session={} reason=lock-contention", + safe_hls_access_lease_id(&endpoint.access_context.lease_id), + safe_proxy_session_id(&endpoint.access_context.proxy_session_id) + ); + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + } + if ensure_hls_cache_stream_registered( + endpoint.app_state, + endpoint.fingerprint, + endpoint.headers, + endpoint.access_context, + endpoint.session, + ) + .await + .is_none() + { + debug!( + "HLS transient media registration skipped: lease={} reason=session-or-connection-unavailable", + safe_hls_access_lease_id(&endpoint.access_context.lease_id) + ); + } + } + let media_marker = if matches!(resource.kind, TransientResourceKind::Segment | TransientResourceKind::Part) + && response.decoded.status.is_success() + { + hls_cache_response_context( + endpoint.app_state, + endpoint.session, + endpoint.access_context, + endpoint.lease_identity, + endpoint.now_ms, + ) + .await + .media_activity_marker + } else { + None + }; + let response = hls_transient_origin_response( + response, + HlsTransientDirectResponseContext { + session: Arc::clone(endpoint.session), + resource, + policy: policy.clone(), + now_ms: endpoint.now_ms, + log_identity: { + let session = endpoint.session.read().await; + HlsLogIdentity::from_session(&session) + }, + }, + ); + if let Some(marker) = media_marker { + marker.confirm_media_response(response) + } else { + response + } + } + Err(err) => { + if matches!(err, HlsOriginResourceFetchError::ProviderUnavailable(_)) { + if let Some(runtime_err) = fetch_result.runtime_prepare_error { + return hls_origin_runtime_resource_failure_response( + endpoint.app_state, + endpoint.access_context, + runtime_err, + ); + } + } + match hls_transient_object_fetch_failure(&err) { + HlsTransientObjectFetchFailure::Retryable => { + let failed_at_ms = current_time_millis(); + if record_temporary_transient_segment_fetch_failure( + endpoint.session, + &resource, + &policy, + failed_at_ms, + ) + .await + { + hls_resource_channel_unavailable_response(endpoint.app_state, endpoint.access_context) + } else { + hls_temporary_resource_unavailable_response(HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS) + } + } + HlsTransientObjectFetchFailure::Permanent { status: _ } => { + hls_resource_channel_unavailable_response(endpoint.app_state, endpoint.access_context) + } + } + } + } +} diff --git a/backend/app/src/api/endpoints/hls_api/segment/routing.rs b/backend/app/src/api/endpoints/hls_api/segment/routing.rs new file mode 100644 index 000000000..4ccf9b7b0 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/segment/routing.rs @@ -0,0 +1,585 @@ +use super::{ + build_virtual_hls_entry_path, epg_reference_ts_from_date_tree_path, get_query_path, get_stream_channel, + get_xtream_player_api_stream_url, handle_hls_stream_request, hls_admission_failure_manifest_response, + hls_api_stream_leaked_relative, hls_cache_enabled_for_user, hls_custom_video_manifest_response_for_username, + hls_manifest_channel_unavailable_response_for_username, is_live_hls_session_token, + legacy_hls_route_allowed_with_cache, looks_like_archive_media_path, m3u_archive_epg_reference_ts, + recreate_hls_session_from_token, resolve_m3u_archive_reference, ApiStreamContext, HlsApiPathParams, + HlsEntryStreamContext, HlsRequestStage, HlsResolvedVirtualSource, +}; +use crate::{ + api::{ + api_utils::{ + create_api_proxy_user, create_m3u_catchup_session_key, create_playback_session_fingerprint, + create_recording_proxy_user, create_session_fingerprint, force_hls_resource_response, + get_hls_session_ttl_secs, input_for_user, local_stream_response, try_option_bad_request, + }, + model::{AppState, CustomVideoStreamType}, + }, + auth::{check_network_access_only, Fingerprint}, + model::{ConfigInput, ConfigTarget, PlaybackKind, ProxyUserCredentials}, + processing::parser::hls::{get_hls_session_token_and_url_from_token, HlsResourceKind}, + repository::{m3u_get_item_for_stream_id, xtream_get_item_for_stream_id}, + utils::{debug_if_enabled, request::is_file_url}, +}; +use axum::{ + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; +use log::{debug, error, warn}; +use shared::{ + defaults::HLS_EXT, + model::{ + ConnectFailureReason, PlaylistEntry, PlaylistItemType, StreamChannel, TargetType, UserConnectionPermission, + XtreamCluster, + }, + utils::{generate_random_string, is_m3u_catchup_session_token, sanitize_sensitive_info, Internable}, +}; +use std::sync::Arc; + +pub(in crate::api::endpoints::hls_api) fn hls_stream_context_or_unavailable( + item: &impl PlaylistEntry, + virtual_id: u32, +) -> Result { + HlsEntryStreamContext::from_playlist_item(item).ok_or_else(|| { + warn!("HLS input stream identity missing for virtual_id={virtual_id}; refresh target playlist"); + StatusCode::SERVICE_UNAVAILABLE + }) +} + +pub(in crate::api) async fn resolve_hls_virtual_source_for_target( + app_state: &Arc, + target: &Arc, + virtual_id: u32, +) -> Result { + let (input_name, stream_context) = if target.has_output(TargetType::Xtream) { + if let Ok(item) = + xtream_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target, None).await + { + let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; + (Arc::clone(&item.input_name), stream_context) + } else { + let item = m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target) + .await + .map_err(|_| StatusCode::NOT_FOUND)?; + let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; + (Arc::clone(&item.input_name), stream_context) + } + } else { + let item = m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target) + .await + .map_err(|_| StatusCode::NOT_FOUND)?; + let stream_context = hls_stream_context_or_unavailable(&item, virtual_id)?; + (Arc::clone(&item.input_name), stream_context) + }; + let input = app_state.app_config.get_input_by_name(&input_name).ok_or(StatusCode::NOT_FOUND)?; + Ok(HlsResolvedVirtualSource { input, stream_context }) +} + +pub(in crate::api::endpoints::hls_api) async fn resolve_hls_origin_playlist_url( + app_state: &Arc, + target: &Arc, + input: &ConfigInput, + virtual_id: u32, + fallback_url: &str, +) -> Result { + if input.input_type.is_xtream() && target.has_output(TargetType::Xtream) { + let pli = xtream_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, target, None) + .await + .map_err(|_| StatusCode::NOT_FOUND)?; + let hls_extension = format!(".{HLS_EXT}"); + let (query_path, _) = get_query_path("", Some(&hls_extension), &pli, app_state); + return get_xtream_player_api_stream_url(input, ApiStreamContext::Live, &query_path, &pli.url) + .map(|url| url.to_string()) + .ok_or(StatusCode::SERVICE_UNAVAILABLE); + } + + Ok(fallback_url.to_string()) +} + +/// Archive (Catchup) or live HLS playback, decided the same way for every HLS path so leases, +/// reservations and TTLs agree. Append/shift catchup often loses utc/utcstart on rewritten +/// segment URLs; the archive path or the session token still identifies archive playback. +pub(in crate::api::endpoints::hls_api) fn hls_playback_kind( + archive_reference: Option, + url: &str, + session_token: Option<&str>, +) -> PlaybackKind { + if archive_reference.is_some() + || looks_like_archive_media_path(url) + || session_token.is_some_and(is_m3u_catchup_session_token) + { + PlaybackKind::Catchup + } else { + PlaybackKind::LiveHls + } +} + +pub(in crate::api::endpoints::hls_api) async fn resolve_stream_channel( + app_state: &Arc, + target: &Arc, + input: &Arc, + virtual_id: u32, + hls_url: &str, + archive_reference: Option, + session_token: Option<&str>, +) -> StreamChannel { + let unknown = "Unknown".intern(); + let mut channel = match get_stream_channel(app_state, target, virtual_id).await { + Some(mut channel) => { + channel.url = Arc::from(hls_url); + channel + } + None => StreamChannel { + target_id: target.id, + virtual_id, + provider_id: 0, + input_name: Arc::clone(&input.name), + item_type: PlaylistItemType::LiveHls, + cluster: XtreamCluster::Live, + group: unknown.clone(), + title: unknown, + url: Arc::from(hls_url), + shared: false, + shared_joined_existing: None, + shared_stream_id: None, + technical: None, + epg_channel_id: None, + epg_reference_ts: None, + upstream_user_agent: None, + }, + }; + + let archive_reference = archive_reference.or_else(|| epg_reference_ts_from_date_tree_path(hls_url)); + if hls_playback_kind(archive_reference, hls_url, session_token) == PlaybackKind::Catchup { + channel.item_type = PlaylistItemType::Catchup; + channel.cluster = XtreamCluster::Video; + channel.epg_reference_ts = archive_reference; + } else { + channel.item_type = PlaylistItemType::LiveHls; + channel.epg_reference_ts = None; + } + channel +} + +pub(in crate::api::endpoints::hls_api) fn hls_entry_user_session_token( + fingerprint: &Fingerprint, + username: &str, + virtual_id: u32, + session_token_hint: Option<&str>, + archive_reference: Option, +) -> String { + // Live hints only arrive from sealed tokens validated by `hls_session_hint_matches_requester`; + // entry routes pass the plain fingerprint key, which never carries the `|hls|` suffix. + if let Some(hint) = + session_token_hint.filter(|token| is_m3u_catchup_session_token(token) || is_live_hls_session_token(token)) + { + return hint.to_string(); + } + if let Some(timestamp) = archive_reference { + return create_m3u_catchup_session_key(fingerprint, username, virtual_id, &format!("archive|{timestamp}|0")); + } + let base = create_playback_session_fingerprint(fingerprint, username, virtual_id, PlaylistItemType::LiveHls, None); + format!("{base}|hls|{}", generate_random_string(16)) +} + +#[allow(clippy::too_many_lines)] +pub(in crate::api::endpoints::hls_api) async fn hls_api_stream( + fingerprint: Fingerprint, + req_headers: HeaderMap, + axum::extract::RawQuery(raw_query): axum::extract::RawQuery, + axum::extract::Path(params): axum::extract::Path, + axum::extract::State(app_state): axum::extract::State>, +) -> impl IntoResponse + Send { + let internal_user = match params.username.as_str() { + crate::model::RECORDING_PROXY_USERNAME => Some(create_recording_proxy_user(&app_state)), + "api_user" => Some(create_api_proxy_user(&app_state)), + _ => None, + } + .filter(|internal| crate::auth::constant_time_eq(params.password.as_bytes(), internal.password.as_bytes())); + let (user, target) = if let Some(internal_user) = internal_user { + let Some(target) = app_state.app_config.get_target_by_id(params.target_id) else { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + }; + (Arc::new(internal_user), target) + } else { + let Some((user, target)) = app_state.app_config.get_target_for_user(¶ms.username, ¶ms.password) else { + // Credential failure is an auth error, not a malformed request + return app_state.app_config.get_auth_error_status().into_response(); + }; + if target.id != params.target_id { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + } + (user, target) + }; + + // Nested path = relative origin segment that leaked past rewrite_hls (e.g. dvr-YYYY/...). + if params.token.contains('/') { + let Some((token, relative_path)) = params.token.split_once('/') else { + return StatusCode::BAD_REQUEST.into_response(); + }; + let encrypt_secret = app_state.get_encrypt_secret(); + let Some(decoded_hls_token) = get_hls_session_token_and_url_from_token(&encrypt_secret, token) else { + return StatusCode::BAD_REQUEST.into_response(); + }; + let lookup_session_token = decoded_hls_token + .session_token + .clone() + .unwrap_or_else(|| create_session_fingerprint(&fingerprint, &user.username, params.stream_id, false)); + let Some(input) = app_state.app_config.get_input_by_id(params.input_id) else { + return StatusCode::BAD_REQUEST.into_response(); + }; + let input = input_for_user(&app_state.app_config, &user, input); + let Some(session) = app_state + .active_users + .find_latest_session_for_target_stream( + &user.username, + target.id, + input.name.as_ref(), + params.stream_id, + lookup_session_token.as_str(), + ) + .await + else { + return StatusCode::NOT_FOUND.into_response(); + }; + if !legacy_hls_route_allowed_with_cache( + hls_cache_enabled_for_user(&app_state, &target, &user), + decoded_hls_token.session_token.as_deref(), + Some(session.token.as_str()), + ) { + return hls_custom_video_manifest_response_for_username( + &app_state, + &user.username, + CustomVideoStreamType::ChannelUnavailable, + StatusCode::NOT_FOUND, + ) + .await; + } + return hls_api_stream_leaked_relative( + fingerprint, + req_headers, + app_state, + user, + target, + input, + params.stream_id, + session, + decoded_hls_token.url, + relative_path.to_string(), + raw_query.as_deref(), + ) + .await; + } + + hls_api_stream_resolved( + fingerprint, + req_headers, + app_state, + user, + target, + params.input_id, + params.stream_id, + params.token, + ) + .await +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +pub(in crate::api::endpoints::hls_api) async fn hls_api_stream_resolved( + fingerprint: Fingerprint, + req_headers: HeaderMap, + app_state: Arc, + user: Arc, + target: Arc, + input_id: u16, + stream_id: u32, + token: String, +) -> axum::response::Response { + // Network access check only - permission check is done later with full stream info + if let Err(e) = check_network_access_only(&user, &fingerprint, &app_state.app_config, &app_state.geoip) { + return e.into_player_response(app_state.app_config.get_auth_error_status()); + } + let target_name = &target.name; + let virtual_id = stream_id; + let input = try_option_bad_request!( + app_state.app_config.get_input_by_id(input_id), + true, + format!("Can't find input {} for target {target_name}, stream_id {virtual_id}, hls", input_id) + ); + let input = input_for_user(&app_state.app_config, &user, input); + + if user.permission_denied(&app_state.app_config) { + let stream_channel = resolve_stream_channel(&app_state, &target, &input, virtual_id, "", None, None).await; + return hls_admission_failure_manifest_response( + &app_state, + &fingerprint, + &user, + stream_channel, + input.name.clone(), + &req_headers, + ConnectFailureReason::UserAccountExpired, + ) + .await; + } + + debug_if_enabled!("ID chain for hls endpoint: request_stream_id={stream_id} -> virtual_id={virtual_id}"); + let encrypt_secret = app_state.get_encrypt_secret(); + let Some(decoded_hls_token) = get_hls_session_token_and_url_from_token(&encrypt_secret, &token) else { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + }; + // Manifest vs. media is decided once from the token kind; legacy tokens use the URL heuristic. + let is_manifest = decoded_hls_token.is_manifest(); + let lookup_session_token = decoded_hls_token + .session_token + .clone() + .unwrap_or_else(|| create_session_fingerprint(&fingerprint, &user.username, virtual_id, false)); + let mut user_session = + app_state.active_users.get_and_update_user_session(&user.username, &lookup_session_token).await; + if !legacy_hls_route_allowed_with_cache( + hls_cache_enabled_for_user(&app_state, &target, &user), + decoded_hls_token.session_token.as_deref(), + user_session.as_ref().map(|session| session.token.as_str()), + ) { + return hls_manifest_channel_unavailable_response_for_username(&app_state, &user.username).await; + } + + if let Some(session) = &mut user_session { + let decoded_archive_reference = + resolve_m3u_archive_reference(&decoded_hls_token.url, Some(lookup_session_token.as_str())); + if session.permission == UserConnectionPermission::Exhausted { + let stream_channel = resolve_stream_channel( + &app_state, + &target, + &input, + virtual_id, + &decoded_hls_token.url, + decoded_archive_reference, + Some(session.token.as_str()), + ) + .await; + return hls_admission_failure_manifest_response( + &app_state, + &fingerprint, + &user, + stream_channel, + session.provider.clone(), + &req_headers, + ConnectFailureReason::UserConnectionsExhausted, + ) + .await; + } + + if app_state.active_provider.is_over_limit(&session.provider) { + let stream_channel = resolve_stream_channel( + &app_state, + &target, + &input, + virtual_id, + &decoded_hls_token.url, + decoded_archive_reference, + Some(session.token.as_str()), + ) + .await; + return hls_admission_failure_manifest_response( + &app_state, + &fingerprint, + &user, + stream_channel, + session.provider.clone(), + &req_headers, + ConnectFailureReason::ProviderConnectionsExhausted, + ) + .await; + } + + let hls_url = match decoded_hls_token.session_token.as_deref() { + Some(session_token) if session.token.eq(session_token) => decoded_hls_token.url.as_str(), + None => decoded_hls_token.url.as_str(), + Some(_) => return axum::http::StatusCode::BAD_REQUEST.into_response(), + }; + // A media URI resolved by another provider account would be fetched with that account's + // credentials; the player refetches its playlist from the session's provider instead. + if decoded_hls_token.kind == Some(HlsResourceKind::Media) + && !session.provider.is_empty() + && decoded_hls_token.origin_provider.as_deref().is_some_and(|origin| origin != session.provider.as_ref()) + { + debug_if_enabled!( + "HLS media URI from provider {} rejected for session provider {}", + sanitize_sensitive_info(decoded_hls_token.origin_provider.as_deref().unwrap_or_default()), + sanitize_sensitive_info(&session.provider) + ); + return StatusCode::NOT_FOUND.into_response(); + } + let hls_url = hls_url.intern(); + // Recover utc/utcstart from the prior playlist URL before overwriting with a segment URL + // that usually drops append/shift query params. + let archive_reference = resolve_m3u_archive_reference(&hls_url, Some(session.token.as_str())) + .or_else(|| m3u_archive_epg_reference_ts(session.stream_url.as_ref())); + session.stream_url = hls_url.clone(); + if session.virtual_id == virtual_id { + app_state.connection_manager.touch_http_activity(&user.username, &session.token, &fingerprint.addr).await; + } else { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + } + + let (connection_admission, grace_mode, request_class) = + crate::api::api_utils::resolve_playback_request_admission( + &app_state.admission_ctx(), + &user, + &fingerprint, + Some(session), + &session.token, + true, + crate::api::api_utils::EvictionReentryGuard::Session(&session.token), + // HLS playlist requests are explicit Prepare: they set up session metadata + // but do not consume an admission slot. Segment and other media requests use Activate. + is_manifest, + false, + ) + .await; + let connection_permission = connection_admission.permission(); + let connection_kind = connection_admission.kind().or(session.connection_kind); + session.permission = connection_permission; + if let Some(connection_kind) = connection_kind { + session.connection_kind = Some(connection_kind); + } + if connection_permission == UserConnectionPermission::Exhausted + || (connection_permission == UserConnectionPermission::GracePeriod && connection_kind.is_none()) + { + if connection_admission.is_reentry_suppressed() { + return crate::api::api_utils::reentry_suppressed_response(); + } + let provider = if session.provider.is_empty() { input.name.clone() } else { session.provider.clone() }; + let stream_channel = resolve_stream_channel( + &app_state, + &target, + &input, + virtual_id, + &session.stream_url, + archive_reference, + Some(session.token.as_str()), + ) + .await; + return hls_admission_failure_manifest_response( + &app_state, + &fingerprint, + &user, + stream_channel, + provider, + &req_headers, + ConnectFailureReason::UserConnectionsExhausted, + ) + .await; + } + let fallback_connection_kind = connection_kind.unwrap_or(crate::api::model::ConnectionKind::Normal); + + if is_manifest { + let manifest_stage = match decoded_hls_token.kind { + Some(HlsResourceKind::Manifest(source)) => { + HlsRequestStage::Variant { source, origin_provider: decoded_hls_token.origin_provider.as_deref() } + } + Some(HlsResourceKind::Media) | None => HlsRequestStage::LegacyVariant, + }; + let source = match resolve_hls_virtual_source_for_target(&app_state, &target, virtual_id).await { + Ok(source) if source.input.id == input.id => source, + Ok(source) => { + warn!( + "HLS input context mismatch for virtual_id={virtual_id}: expected_input_id={}, resolved_input_id={}", + input.id, source.input.id + ); + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + Err(status) => return status.into_response(), + }; + let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, virtual_id); + return handle_hls_stream_request( + &fingerprint, + &app_state, + &user, + &target, + Some(session), + None, + &session.stream_url, + archive_reference, + source.stream_context, + &input, + &req_headers, + connection_permission, + connection_kind, + &original_hls_entry_path, + manifest_stage, + ) + .await + .into_response(); + } + + if is_file_url(&session.stream_url) { + let stream_channel = resolve_stream_channel( + &app_state, + &target, + &input, + virtual_id, + &hls_url, + archive_reference, + Some(session.token.as_str()), + ) + .await; + return local_stream_response( + &fingerprint, + &app_state, + stream_channel, + &req_headers, + &input, + &target, + &user, + connection_permission, + fallback_connection_kind, + Some(&session.token), + Some(request_class), + false, + ) + .await + .into_response(); + } + + let stream_channel = resolve_stream_channel( + &app_state, + &target, + &input, + virtual_id, + &hls_url, + archive_reference, + Some(session.token.as_str()), + ) + .await; + force_hls_resource_response( + &fingerprint, + &app_state, + session, + stream_channel, + crate::api::api_utils::ForceStreamRequestContext { + req_headers: &req_headers, + input: &input, + user: &user, + session_reservation_ttl_secs: get_hls_session_ttl_secs(&app_state), + content_representation: crate::api::model::ProviderContentRepresentationMode::Identity, + }, + grace_mode, + ) + .await + .into_response() + } else { + recreate_hls_session_from_token( + &fingerprint, + &req_headers, + &app_state, + &user, + &target, + &input, + virtual_id, + &decoded_hls_token, + ) + .await + } +} diff --git a/backend/app/src/api/endpoints/hls_api/segment/segments.rs b/backend/app/src/api/endpoints/hls_api/segment/segments.rs new file mode 100644 index 000000000..59393842f --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/segment/segments.rs @@ -0,0 +1,371 @@ +use super::{ + current_hls_resource_lease, hls_cache_response_context, hls_lease_allows_cached_segment, + hls_lease_allows_live_origin_work, hls_origin_runtime_resource_failure_response, + hls_resource_channel_unavailable_response, hls_resource_serve_outcome_response, + hls_segment_request_requires_origin_work, is_hls_media_activity_status, + prepare_hls_origin_binding_for_authorized_resource_work, prepare_hls_resource_access, + register_hls_cache_stream_for_successful_media_response, terminal_segment_get_response, + terminal_segment_head_response, terminal_segment_immutable_replay_response, terminal_tail_plan_for_current_route, + HlsOriginWorkKind, HlsProxySegmentPathParams, HlsProxyTerminalSegmentPathParams, HlsResourceAccess, +}; +use crate::{ + api::model::{AppState, ProviderHandle}, + auth::Fingerprint, +}; +use axum::{ + http::{header, HeaderMap, Method, StatusCode}, + response::IntoResponse, +}; +use std::sync::Arc; +use tuliprox_core::utils::current_time_millis; +use tuliprox_hls::api::{ + serve_hls_segment_cache_outcome, HlsAccessContext, HlsAccessLeaseId, HlsAccountOverlapTiming, + HlsEffectiveOriginAcquirePolicy, HlsLeasePlaybackMode, HlsOriginIoContext, HlsOriginWorkClass, HlsSegmentFile, + HlsSessionHandle, HlsTerminalSegmentPath, ProxySessionId, SegmentCacheStatus, SegmentDemandFetchOutcome, + SegmentFetchContext, +}; + +pub(in crate::api::endpoints::hls_api) async fn hls_proxy_segment( + fingerprint: Fingerprint, + axum::extract::Path(params): axum::extract::Path, + axum::extract::State(app_state): axum::extract::State>, + headers: HeaderMap, +) -> axum::response::Response { + let proxy_session_id = ProxySessionId(params.proxy_session_id); + let now_ms = current_time_millis(); + let HlsResourceAccess { session, access_context, lease } = match prepare_hls_resource_access( + &app_state, + &fingerprint, + &proxy_session_id, + ¶ms.hls_access_lease_id, + now_ms, + "segment", + ) + .await + { + Ok(access) => access, + Err(response) => return *response, + }; + let Some(segment_file) = HlsSegmentFile::parse(¶ms.segment_file) else { + return hls_resource_channel_unavailable_response(&app_state, &access_context); + }; + if !hls_lease_allows_cached_segment(&lease, segment_file.proxy_seq) { + return StatusCode::NOT_FOUND.into_response(); + } + let allows_origin_work = hls_lease_allows_live_origin_work(&lease); + if let Err(response) = + validate_hls_segment_entry(&app_state, &session, &access_context, &segment_file, allows_origin_work).await + { + return *response; + } + let revision_delivery = session.read().await.startup.is_some(); + let demand_result = if allows_origin_work && !revision_delivery { + demand_fetch_hls_live_segment( + &app_state, + &session, + &segment_file, + &access_context, + &fingerprint, + &headers, + now_ms, + ) + .await + } else { + Ok(()) + }; + if let Err(response) = demand_result { + return *response; + } + + serve_hls_segment_for_current_lease( + &app_state, + &session, + &access_context, + &fingerprint, + &headers, + segment_file, + now_ms, + ) + .await +} + +pub(in crate::api::endpoints::hls_api) async fn validate_hls_segment_entry( + app_state: &Arc, + session: &HlsSessionHandle, + access_context: &HlsAccessContext, + segment_file: &HlsSegmentFile, + allows_origin_work: bool, +) -> Result<(), Box> { + let session = session.read().await; + if session.is_gc_marked_for_removal() { + return Err(Box::new(StatusCode::NOT_FOUND.into_response())); + } + let Some(entry) = session.segments.get(&segment_file.proxy_seq) else { + return Err(Box::new(hls_resource_channel_unavailable_response(app_state, access_context))); + }; + if entry.proxy_file_ext != segment_file.extension { + return Err(Box::new(hls_resource_channel_unavailable_response(app_state, access_context))); + } + if !allows_origin_work && !matches!(&entry.status, SegmentCacheStatus::Ready { .. }) { + return Err(Box::new(StatusCode::NOT_FOUND.into_response())); + } + Ok(()) +} + +pub(in crate::api::endpoints::hls_api) async fn demand_fetch_hls_live_segment( + app_state: &Arc, + session: &HlsSessionHandle, + segment_file: &HlsSegmentFile, + access_context: &HlsAccessContext, + fingerprint: &Fingerprint, + headers: &HeaderMap, + now_ms: u64, +) -> Result<(), Box> { + let preacquired_provider_handle = if hls_segment_request_requires_origin_work(session, segment_file).await { + match prepare_hls_origin_binding_for_authorized_resource_work( + app_state, + session, + access_context, + fingerprint, + headers, + HlsOriginWorkKind::Segment, + now_ms, + ) + .await + { + Ok(handle) => handle, + Err(err) => { + return Err(Box::new(hls_origin_runtime_resource_failure_response(app_state, access_context, err))) + } + } + } else { + None + }; + match demand_fetch_hls_segment_if_needed( + app_state, + session, + segment_file, + access_context, + fingerprint, + preacquired_provider_handle, + now_ms, + ) + .await + { + SegmentDemandFetchOutcome::NotFound => { + Err(Box::new(hls_resource_channel_unavailable_response(app_state, access_context))) + } + SegmentDemandFetchOutcome::Ready + | SegmentDemandFetchOutcome::QueuedOrFetching + | SegmentDemandFetchOutcome::Unavailable + | SegmentDemandFetchOutcome::TimedOut => Ok(()), + } +} + +pub(in crate::api::endpoints::hls_api) async fn serve_hls_segment_for_current_lease( + app_state: &Arc, + session: &HlsSessionHandle, + access_context: &HlsAccessContext, + fingerprint: &Fingerprint, + headers: &HeaderMap, + segment_file: HlsSegmentFile, + now_ms: u64, +) -> axum::response::Response { + let Some(current_lease) = current_hls_resource_lease(app_state, access_context).await else { + return StatusCode::NOT_FOUND.into_response(); + }; + if !hls_lease_allows_cached_segment(¤t_lease, segment_file.proxy_seq) { + return StatusCode::NOT_FOUND.into_response(); + } + let current_allows_origin_work = hls_lease_allows_live_origin_work(¤t_lease); + let Some(lease_identity) = current_lease.media_identity() else { + return StatusCode::NOT_FOUND.into_response(); + }; + let response_context = hls_cache_response_context(app_state, session, access_context, lease_identity, now_ms).await; + let response = hls_resource_serve_outcome_response( + app_state, + access_context, + serve_hls_segment_cache_outcome( + Arc::clone(app_state.hls.proxy.segment_cache()), + Arc::clone(session), + segment_file, + headers.get(header::RANGE).cloned(), + &response_context, + ) + .await, + ); + if current_allows_origin_work && is_hls_media_activity_status(response.status()) { + register_hls_cache_stream_for_successful_media_response( + app_state, + fingerprint, + headers, + access_context, + session, + &response_context, + ) + .await; + } + response +} + +pub(in crate::api::endpoints::hls_api) async fn hls_proxy_terminal_segment( + fingerprint: Fingerprint, + axum::extract::Path(params): axum::extract::Path, + axum::extract::State(app_state): axum::extract::State>, + method: Method, + headers: HeaderMap, +) -> axum::response::Response { + let proxy_session_id = ProxySessionId(params.proxy_session_id); + let Some(path) = HlsTerminalSegmentPath::parse(¶ms.generation, ¶ms.terminal_file) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let now_ms = current_time_millis(); + let access_lease_id = HlsAccessLeaseId(params.hls_access_lease_id.clone()); + let immutable_replay_plan = + app_state.hls.proxy.access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms).await.and_then( + |lease| match lease.playback_mode { + HlsLeasePlaybackMode::TerminalTail(plan) if plan.matches_route(&proxy_session_id, &access_lease_id) => { + Some(plan) + } + HlsLeasePlaybackMode::Live + | HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } + | HlsLeasePlaybackMode::Ended => None, + }, + ); + let access = prepare_hls_resource_access( + &app_state, + &fingerprint, + &proxy_session_id, + ¶ms.hls_access_lease_id, + now_ms, + "terminal-segment", + ) + .await; + let Ok(HlsResourceAccess { session, access_context, lease }) = access else { + return immutable_replay_plan + .and_then(|plan| { + terminal_segment_immutable_replay_response( + &plan, + path, + headers.get(header::RANGE), + method == Method::HEAD, + ) + }) + .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); + }; + let Some(current_lease) = current_hls_resource_lease(&app_state, &access_context).await else { + return StatusCode::NOT_FOUND.into_response(); + }; + let Some(plan) = + terminal_tail_plan_for_current_route(&lease, ¤t_lease, &proxy_session_id, &access_context.lease_id) + else { + return StatusCode::NOT_FOUND.into_response(); + }; + if method == Method::HEAD { + return terminal_segment_head_response(&plan, path, headers.get(header::RANGE)) + .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); + } + let Some(lease_identity) = current_lease.media_identity() else { + return StatusCode::NOT_FOUND.into_response(); + }; + let response_context = + hls_cache_response_context(&app_state, &session, &access_context, lease_identity, now_ms).await; + let Some(response) = + terminal_segment_get_response(&plan, path, headers.get(header::RANGE), &response_context, &proxy_session_id) + else { + return StatusCode::NOT_FOUND.into_response(); + }; + if is_hls_media_activity_status(response.status()) { + response_context.mark_media_activity().await; + register_hls_cache_stream_for_successful_media_response( + &app_state, + &fingerprint, + &headers, + &access_context, + &session, + &response_context, + ) + .await; + } + response +} + +pub(in crate::api::endpoints::hls_api) async fn demand_fetch_hls_segment_if_needed( + app_state: &Arc, + session: &HlsSessionHandle, + segment_file: &HlsSegmentFile, + access_context: &HlsAccessContext, + fingerprint: &Fingerprint, + preacquired_provider_handle: Option, + now_ms: u64, +) -> SegmentDemandFetchOutcome { + let context = build_hls_segment_fetch_context( + app_state, + session, + Some(access_context.lease_id.clone()), + fingerprint, + preacquired_provider_handle, + ) + .await; + app_state.hls.proxy.segment_worker_pool().demand_fetch_and_wait(context, segment_file, now_ms).await +} + +pub(in crate::api::endpoints::hls_api) async fn build_hls_segment_fetch_context( + app_state: &Arc, + session: &HlsSessionHandle, + repair_access_lease_id: Option, + fingerprint: &Fingerprint, + preacquired_provider_handle: Option, +) -> SegmentFetchContext { + let (headers, origin_provider_session_headers, origin_policy, reservation_ttl_secs) = { + let session = session.read().await; + ( + session.origin_request_headers.clone(), + session.origin_provider_session_headers.clone(), + session.effective_origin_acquire_policy_or_default(), + session.account_overlap_timing().reservation_ttl_secs(), + ) + }; + let mut origin_io = HlsOriginIoContext { + ctx: app_state.hls_ctx(), + client_addr: fingerprint.addr, + allow_grace: HlsOriginWorkClass::Demand.allows_grace(), + priority: origin_policy.priority, + connection_kind: origin_policy.connection_kind, + reservation_ttl_secs, + preacquired_provider_handle: None, + started_generation: None, + }; + if let Some(provider_handle) = preacquired_provider_handle { + origin_io = origin_io.with_preacquired_provider_handle(provider_handle); + } + SegmentFetchContext { + session: Arc::clone(session), + segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()), + segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()), + repair_access_lease_id, + headers, + origin_provider_session_headers, + client: app_state.http_clients.default.load().as_ref().clone(), + no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(), + use_manual_redirects: app_state.should_use_manual_redirects(), + origin_io: Some(origin_io), + } +} + +pub(in crate::api::endpoints::hls_api) async fn hls_effective_origin_acquire_policy( + session: &HlsSessionHandle, +) -> HlsEffectiveOriginAcquirePolicy { + session.read().await.effective_origin_acquire_policy_or_default() +} + +pub(in crate::api::endpoints::hls_api) async fn hls_origin_account_reservation_ttl_secs_for_session( + session: &HlsSessionHandle, +) -> u64 { + session.read().await.account_overlap_timing().reservation_ttl_secs() +} + +pub(in crate::api::endpoints::hls_api) fn hls_origin_account_reservation_ttl_secs_fallback() -> u64 { + HlsAccountOverlapTiming::from_target_duration_secs(None).reservation_ttl_secs() +} diff --git a/backend/app/src/api/endpoints/hls_api/segment/transient_cache.rs b/backend/app/src/api/endpoints/hls_api/segment/transient_cache.rs new file mode 100644 index 000000000..82ffb062e --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/segment/transient_cache.rs @@ -0,0 +1,437 @@ +use super::{ + hls_cache_response_context, hls_live_lease_identity_is_current, hls_origin_runtime_resource_failure_response, + hls_resource_serve_outcome_response, hls_transient_object_unavailable_response, is_hls_media_activity_status, + prepare_hls_transient_origin_io_for_authorized_resource_work, + register_hls_cache_stream_for_successful_media_response, HlsOriginRuntimeAcquireError, + HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, +}; +use crate::{api::model::AppState, auth::Fingerprint}; +use axum::{ + http::{HeaderMap, HeaderValue, StatusCode}, + response::IntoResponse, +}; +use futures::FutureExt; +use log::debug; +use std::sync::Arc; +use tuliprox_core::utils::current_time_millis; +use tuliprox_hls::api::{ + fetch_and_commit_hls_transient_origin_response_with_attempt_prepare, + fetch_hls_transient_origin_response_with_attempt_prepare, hls_transient_object_fetch_failure, + record_successful_transient_segment_fetch, record_temporary_transient_segment_fetch_failure, + safe_hls_access_lease_id, serve_hls_transient_object_cache_outcome, serve_hls_transient_object_cache_response, + HlsAccessContext, HlsBoundAccountAcquireErrorKind, HlsLogIdentity, HlsMediaLeaseIdentity, HlsOriginResourceClients, + HlsOriginResourceFetchError, HlsResourceFetchAttempt, HlsSessionHandle, HlsTransientCacheCommitContext, + HlsTransientDecodedOriginResponse, HlsTransientObjectFetchFailure, HlsTransientObjectFetchFinalizer, + HlsTransientOriginCacheFetchRequest, HlsTransientOriginFetchRequest, HlsTransientOriginIoGuard, SegmentFetchPolicy, + TransientObjectFetchToken, TransientResourceFile, TransientResourceId, TransientResourceRef, +}; + +pub(in crate::api::endpoints::hls_api) struct HlsTransientEndpointOriginFetchRequest<'a> { + pub(in crate::api::endpoints::hls_api) app_state: &'a Arc, + pub(in crate::api::endpoints::hls_api) session: &'a HlsSessionHandle, + pub(in crate::api::endpoints::hls_api) access_context: &'a HlsAccessContext, + pub(in crate::api::endpoints::hls_api) fingerprint: &'a Fingerprint, + pub(in crate::api::endpoints::hls_api) headers: &'a HeaderMap, + pub(in crate::api::endpoints::hls_api) resource: &'a TransientResourceRef, + pub(in crate::api::endpoints::hls_api) resource_file: &'a TransientResourceFile, + pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap, + pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap, + pub(in crate::api::endpoints::hls_api) range_header: Option, + pub(in crate::api::endpoints::hls_api) policy: SegmentFetchPolicy, +} + +pub(in crate::api::endpoints::hls_api) struct HlsTransientOriginFetchResult { + pub(in crate::api::endpoints::hls_api) result: + Result>, HlsOriginResourceFetchError>, + pub(in crate::api::endpoints::hls_api) runtime_prepare_error: Option, +} + +pub(in crate::api::endpoints::hls_api) async fn fetch_transient_origin_response_with_provider_io( + request: HlsTransientEndpointOriginFetchRequest<'_>, +) -> HlsTransientOriginFetchResult { + let clients = HlsOriginResourceClients { + client: request.app_state.http_clients.default.load().as_ref().clone(), + no_redirect_client: request.app_state.http_clients.no_redirect.load().as_ref().clone(), + use_manual_redirects: request.app_state.should_use_manual_redirects(), + }; + let log_identity = { + let session = request.session.read().await; + HlsLogIdentity::from_session(&session) + }; + let fetch_request = HlsTransientOriginFetchRequest { + resolved_origin_uri: request.resource.resolved_origin_uri.clone(), + origin_headers: request.origin_headers, + origin_provider_session_headers: request.origin_provider_session_headers, + range_header: request.range_header, + resource_file: request.resource_file.clone(), + resource_kind: request.resource.kind, + clients, + policy: request.policy, + log_identity, + }; + let runtime_prepare_error = Arc::new(tokio::sync::Mutex::new(None)); + let prepare_attempt = hls_transient_origin_prepare_closure( + request.app_state, + request.session, + request.access_context, + request.fingerprint, + request.headers, + &runtime_prepare_error, + ); + let result = fetch_hls_transient_origin_response_with_attempt_prepare(fetch_request, prepare_attempt).await; + let runtime_prepare_error = *runtime_prepare_error.lock().await; + HlsTransientOriginFetchResult { result, runtime_prepare_error } +} + +/// Builds the shared per-attempt prepare closure for transient origin fetches. +/// Runtime acquire failures are captured in `runtime_prepare_error` and mapped +/// to a provider-unavailable fetch error so the retry loop can proceed uniformly. +pub(in crate::api::endpoints::hls_api) fn hls_transient_origin_prepare_closure( + app_state: &Arc, + session: &HlsSessionHandle, + access_context: &HlsAccessContext, + fingerprint: &Fingerprint, + headers: &HeaderMap, + runtime_prepare_error: &Arc>>, +) -> impl FnMut( + HlsResourceFetchAttempt, +) -> futures::future::BoxFuture< + 'static, + Result, HlsOriginResourceFetchError>, +> { + let app_state = Arc::clone(app_state); + let session = Arc::clone(session); + let access_context = access_context.clone(); + let fingerprint = fingerprint.clone(); + let headers = headers.clone(); + let runtime_prepare_error = Arc::clone(runtime_prepare_error); + move |_attempt| { + let app_state = Arc::clone(&app_state); + let session = Arc::clone(&session); + let access_context = access_context.clone(); + let fingerprint = fingerprint.clone(); + let headers = headers.clone(); + let runtime_prepare_error = Arc::clone(&runtime_prepare_error); + async move { + match prepare_hls_transient_origin_io_for_authorized_resource_work( + &app_state, + &session, + &access_context, + &fingerprint, + &headers, + current_time_millis(), + ) + .await + { + Ok(guard) => Ok(guard), + Err(err) => { + *runtime_prepare_error.lock().await = Some(err); + Err(HlsOriginResourceFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Unavailable)) + } + } + } + .boxed() + } +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::api::endpoints::hls_api) struct HlsTransientEndpointCacheFetchContext<'a> { + pub(in crate::api::endpoints::hls_api) app_state: &'a Arc, + pub(in crate::api::endpoints::hls_api) session: &'a HlsSessionHandle, + pub(in crate::api::endpoints::hls_api) fingerprint: &'a Fingerprint, + pub(in crate::api::endpoints::hls_api) headers: &'a HeaderMap, + pub(in crate::api::endpoints::hls_api) access_context: &'a HlsAccessContext, + pub(in crate::api::endpoints::hls_api) lease_identity: HlsMediaLeaseIdentity, + pub(in crate::api::endpoints::hls_api) resource: &'a TransientResourceRef, + pub(in crate::api::endpoints::hls_api) resource_file: TransientResourceFile, + pub(in crate::api::endpoints::hls_api) fetch_token: TransientObjectFetchToken, + pub(in crate::api::endpoints::hls_api) origin_headers: HeaderMap, + pub(in crate::api::endpoints::hls_api) origin_provider_session_headers: HeaderMap, + pub(in crate::api::endpoints::hls_api) range_header: Option, + pub(in crate::api::endpoints::hls_api) cache_duration_ms: u64, +} + +pub(in crate::api::endpoints::hls_api) struct TransientObjectWaitContext<'a> { + pub(in crate::api::endpoints::hls_api) app_state: &'a Arc, + pub(in crate::api::endpoints::hls_api) session: &'a HlsSessionHandle, + pub(in crate::api::endpoints::hls_api) fingerprint: &'a Fingerprint, + pub(in crate::api::endpoints::hls_api) headers: &'a HeaderMap, + pub(in crate::api::endpoints::hls_api) access_context: &'a HlsAccessContext, + pub(in crate::api::endpoints::hls_api) lease_identity: HlsMediaLeaseIdentity, + pub(in crate::api::endpoints::hls_api) resource_file: TransientResourceFile, + pub(in crate::api::endpoints::hls_api) range_header: Option, + pub(in crate::api::endpoints::hls_api) notifier: Arc, +} + +pub(in crate::api::endpoints::hls_api) struct TransientObjectCacheServeContext<'a> { + pub(in crate::api::endpoints::hls_api) app_state: &'a Arc, + pub(in crate::api::endpoints::hls_api) session: &'a HlsSessionHandle, + pub(in crate::api::endpoints::hls_api) fingerprint: &'a Fingerprint, + pub(in crate::api::endpoints::hls_api) headers: &'a HeaderMap, + pub(in crate::api::endpoints::hls_api) access_context: &'a HlsAccessContext, + pub(in crate::api::endpoints::hls_api) lease_identity: HlsMediaLeaseIdentity, + pub(in crate::api::endpoints::hls_api) resource_file: TransientResourceFile, + pub(in crate::api::endpoints::hls_api) range_header: Option, + pub(in crate::api::endpoints::hls_api) now_ms: u64, +} + +pub(in crate::api::endpoints::hls_api) async fn serve_transient_object_cache_response_and_mark( + context: TransientObjectCacheServeContext<'_>, +) -> axum::response::Response { + if !hls_live_lease_identity_is_current(context.app_state, context.access_context, context.lease_identity).await { + return StatusCode::NOT_FOUND.into_response(); + } + let response_context = hls_cache_response_context( + context.app_state, + context.session, + context.access_context, + context.lease_identity, + context.now_ms, + ) + .await; + let response = hls_resource_serve_outcome_response( + context.app_state, + context.access_context, + serve_hls_transient_object_cache_outcome( + Arc::clone(context.app_state.hls.proxy.segment_cache()), + Arc::clone(context.session), + context.resource_file, + context.range_header, + &response_context, + ) + .await, + ); + if is_hls_media_activity_status(response.status()) { + register_hls_cache_stream_for_successful_media_response( + context.app_state, + context.fingerprint, + context.headers, + context.access_context, + context.session, + &response_context, + ) + .await; + } + response +} + +pub(in crate::api::endpoints::hls_api) async fn serve_transient_object_cache_response_and_mark_or_unavailable( + context: TransientObjectCacheServeContext<'_>, +) -> axum::response::Response { + serve_transient_object_cache_response_and_mark(context).await +} + +pub(in crate::api::endpoints::hls_api) async fn wait_for_transient_object_cache_fetch( + context: TransientObjectWaitContext<'_>, +) -> axum::response::Response { + let wait_timeout = context.app_state.hls.proxy.segment_fetch_policy().origin_object_wait_timeout(); + let safe_resource_id = safe_transient_resource_id(&context.resource_file.resource_id); + debug!( + "HLS transient object wait started: resource_id={} lease={} state=inflight", + safe_resource_id, + safe_hls_access_lease_id(&context.access_context.lease_id) + ); + let wait_result = tokio::time::timeout(wait_timeout, context.notifier.notified()).await; + if wait_result.is_err() { + debug!( + "HLS transient object wait timed out: resource_id={} lease={} state=inflight", + safe_resource_id, + safe_hls_access_lease_id(&context.access_context.lease_id) + ); + return hls_transient_object_unavailable_response( + context.app_state, + context.session, + &context.resource_file, + current_time_millis(), + context.access_context, + ) + .await; + } + let response = serve_transient_object_cache_response_and_mark_or_unavailable(TransientObjectCacheServeContext { + app_state: context.app_state, + session: context.session, + fingerprint: context.fingerprint, + headers: context.headers, + access_context: context.access_context, + lease_identity: context.lease_identity, + resource_file: context.resource_file, + range_header: context.range_header, + now_ms: current_time_millis(), + }) + .await; + debug!( + "HLS transient object wait completed: resource_id={} lease={} status={}", + safe_resource_id, + safe_hls_access_lease_id(&context.access_context.lease_id), + response.status() + ); + response +} + +pub(in crate::api::endpoints::hls_api) fn safe_transient_resource_id(resource_id: &TransientResourceId) -> String { + // Truncate at the first char boundary at or before byte 8 to avoid allocating + // a temporary `String` of 8 chars (and a second UTF-8 walk via `len()`). + let full = resource_id.0.as_str(); + let truncate_at = full.char_indices().nth(8).map_or(full.len(), |(byte_idx, _)| byte_idx); + if truncate_at == full.len() { + return full.to_owned(); + } + let mut out = String::with_capacity(truncate_at + 3); + out.push_str(&full[..truncate_at]); + out.push_str("..."); + out +} + +#[allow(clippy::too_many_lines)] +pub(in crate::api::endpoints::hls_api) async fn fetch_and_cache_transient_origin_response( + context: HlsTransientEndpointCacheFetchContext<'_>, +) -> axum::response::Response { + let policy = context.app_state.hls.proxy.segment_fetch_policy(); + let mut fetch_finalizer = HlsTransientObjectFetchFinalizer::new( + Arc::clone(context.session), + Arc::clone(context.app_state.hls.proxy.segment_cache()), + context.fetch_token.clone(), + HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, + ); + let clients = HlsOriginResourceClients { + client: context.app_state.http_clients.default.load().as_ref().clone(), + no_redirect_client: context.app_state.http_clients.no_redirect.load().as_ref().clone(), + use_manual_redirects: context.app_state.should_use_manual_redirects(), + }; + let log_identity = { + let session = context.session.read().await; + HlsLogIdentity::from_session(&session) + }; + let fetch_request = HlsTransientOriginFetchRequest { + resolved_origin_uri: context.resource.resolved_origin_uri.clone(), + origin_headers: context.origin_headers.clone(), + origin_provider_session_headers: context.origin_provider_session_headers.clone(), + range_header: None, + resource_file: context.resource_file.clone(), + resource_kind: context.resource.kind, + clients, + policy: policy.clone(), + log_identity: log_identity.clone(), + }; + let cache_fetch_request = HlsTransientOriginCacheFetchRequest { + fetch: fetch_request, + commit: HlsTransientCacheCommitContext { + segment_cache: Arc::clone(context.app_state.hls.proxy.segment_cache()), + segment_repair: Arc::clone(context.app_state.hls.proxy.segment_repair()), + session: Arc::clone(context.session), + proxy_session_id: context.access_context.proxy_session_id.clone(), + log_identity: log_identity.clone(), + access_lease_id: context.access_context.lease_id.clone(), + resource: context.resource.clone(), + resource_file: context.resource_file.clone(), + fetch_token: context.fetch_token.clone(), + cache_duration_ms: context.cache_duration_ms, + }, + }; + let runtime_prepare_error = Arc::new(tokio::sync::Mutex::new(None)); + let prepare_attempt = hls_transient_origin_prepare_closure( + context.app_state, + context.session, + context.access_context, + context.fingerprint, + context.headers, + &runtime_prepare_error, + ); + let final_failure = + match fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(cache_fetch_request, prepare_attempt) + .await + { + Ok(()) => { + let ready_at_ms = current_time_millis(); + let response_context = hls_cache_response_context( + context.app_state, + context.session, + context.access_context, + context.lease_identity, + ready_at_ms, + ) + .await; + let response = serve_hls_transient_object_cache_response( + Arc::clone(context.app_state.hls.proxy.segment_cache()), + Arc::clone(context.session), + context.resource_file.clone(), + context.range_header.clone(), + &response_context, + ) + .await; + if is_hls_media_activity_status(response.status()) { + register_hls_cache_stream_for_successful_media_response( + context.app_state, + context.fingerprint, + context.headers, + context.access_context, + context.session, + &response_context, + ) + .await; + } + record_successful_transient_segment_fetch(context.session, context.resource).await; + fetch_finalizer.complete(); + return response; + } + Err(err) => { + if matches!(err, HlsOriginResourceFetchError::ProviderUnavailable(_)) { + let runtime_prepare_error = *runtime_prepare_error.lock().await; + if let Some(runtime_err) = runtime_prepare_error { + context.session.write().await.fail_transient_object_retryable_if_current( + &context.fetch_token, + current_time_millis(), + HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, + ); + return hls_origin_runtime_resource_failure_response( + context.app_state, + context.access_context, + runtime_err, + ); + } + } + hls_transient_object_fetch_failure(&err) + } + }; + + let failed_at_ms = current_time_millis(); + match final_failure { + HlsTransientObjectFetchFailure::Retryable => { + if record_temporary_transient_segment_fetch_failure( + context.session, + context.resource, + &policy, + failed_at_ms, + ) + .await + { + context.session.write().await.fail_transient_object_permanent_if_current( + &context.fetch_token, + failed_at_ms, + None, + ); + } else { + context.session.write().await.fail_transient_object_retryable_if_current( + &context.fetch_token, + failed_at_ms, + HLS_TEMPORARY_RESOURCE_RETRY_AFTER_MS, + ); + } + } + HlsTransientObjectFetchFailure::Permanent { status } => { + context.session.write().await.fail_transient_object_permanent_if_current( + &context.fetch_token, + failed_at_ms, + status, + ); + } + } + hls_transient_object_unavailable_response( + context.app_state, + context.session, + &context.resource_file, + failed_at_ms, + context.access_context, + ) + .await +} diff --git a/backend/app/src/api/endpoints/hls_api/tests.rs b/backend/app/src/api/endpoints/hls_api/tests.rs index cd1b45847..2fa27cc69 100644 --- a/backend/app/src/api/endpoints/hls_api/tests.rs +++ b/backend/app/src/api/endpoints/hls_api/tests.rs @@ -3,76 +3,56 @@ use super::{ hls_manifest_terminal_preflight, hls_terminal_endpoint_action, hls_terminal_failed_closed_response, HlsManifestTerminalPreflight, HlsTerminalEndpointAction, }, - apply_hls_user_agent_stream_index, build_hls_manifest_request_headers, extract_hls_provider_session_headers, - hls_api_register, hls_availability_reevaluation_registration_failure_response, hls_canonical_owner_registration, + apply_hls_user_agent_stream_index, build_hls_manifest_request_headers, hls_api_register, + hls_availability_reevaluation_registration_failure_response, hls_canonical_owner_registration, hls_temporary_resource_unavailable_response, m3u_archive_epg_reference_ts, m3u_catchup_epg_reference_from_session_token, resolve_leaked_hls_relative_origin, HlsCanonicalOwnerRegistration, - HlsCanonicalOwnerRegistrationFailure, HlsCanonicalOwnerRegistrationKind, MAX_HLS_MANIFEST_BYTES, + HlsCanonicalOwnerRegistrationFailure, HlsCanonicalOwnerRegistrationKind, }; use crate::{ api::model::{ - begin_hls_origin_account_io, build_hls_custom_video_manifest_body, build_proxy_session_id, - build_terminal_tail_plan, build_transient_resource_id, commit_terminal_tail_if_lease_reserve_requires_cutover, - finish_hls_origin_account_io, prepare_terminal_base_evidence, prepared_terminal_bundle_key, - snapshot_terminal_media_asset, trigger_origin_refresh_sync, ActiveProviderManager, ActiveUserManager, AppState, - CacheAccessState, CancelTokens, ConnectionKind, ConnectionManager, CreateUserSessionParams, - CustomVideoStreamType, EventManager, HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, - HlsAccessAdmissionMode, HlsAccessContext, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, - HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsAccessLeaseValidationError, - HlsAvailabilityReevaluationFinishReason, HlsAvailabilityReevaluationMode, - HlsAvailabilityReevaluationRegistration, HlsBandwidthPersistenceState, HlsEffectiveOriginAcquirePolicy, - HlsFreshManifestRequiredReason, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, - HlsLifecycleEvent, HlsLifecycleEventKey, HlsManifestAcceptanceDirective, - HlsManifestAcceptanceEvaluationOutcome, HlsManifestAcceptanceExhaustionReason, HlsManifestAcceptanceTrigger, - HlsManifestCommitIdentity, HlsManifestCommitRequirement, HlsManifestDeliveryMode, HlsMapSignature, - HlsMediaContainer, HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsOriginAccountBinding, - HlsOriginAccountBindingMode, HlsOriginAccountDetachedReason, HlsOriginIoContext, HlsOriginPathCondition, - HlsOriginSource, HlsOriginSourceKind, HlsPlaybackFamilyKey, HlsPreparedTerminalBundleState, HlsProxyManager, - HlsPublishedTransientResourceIds, HlsRecoveryEtaMs, HlsRecoveryTimingPolicy, HlsRecoveryWorkload, - HlsRuntimeCustomTailAssetIdentity, HlsRuntimeCustomTailReason, HlsSegmentFile, HlsSession, HlsSessionHandle, - HlsSessionKey, HlsSessionMode, HlsSessionStoreOutcome, HlsTerminalAssetIdentity, HlsTerminalBaseMediaState, - HlsTerminalBaseProtection, HlsTerminalBaseSegmentAvailability, HlsTerminalFailedClosedReason, - HlsTerminalMediaAsset, HlsTerminalMediaPreparationState, HlsTerminalResolution, HlsTerminalSegmentPath, - HlsTerminalTailBuildInput, HlsTerminalTailCompatibility, HlsTerminalTailGeneration, HlsTerminalTailPlan, - HlsTerminalTailProtection, HlsTransitionMarginMs, LiveHlsOriginEntry, MapCacheStatus, MapEntry, - MetadataUpdateManager, OriginMapKey, OriginRefreshRequest, OriginSegmentFetchRef, OriginSegmentKey, - PlaybackLifecycle, PlaylistStorageState, ProviderConfig as RuntimeProviderConfig, ProviderConfigConnection, - ProxyMapId, ProxySessionId, RenderedManifest, RetryPolicy, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, - SegmentFetchPriority, SharedStreamManager, TransientObjectCacheKey, TransientObjectCacheStatus, - TransientResourceId, TransientResourceKind, TransientResourceRef, TransportStreamBuffer, UserSession, + build_terminal_tail_plan, build_transient_resource_id, + hls_cache::initial_strip::{HlsInitialStripOutcome, HlsInitialStripSkipReason}, + prepare_terminal_base_evidence, prepared_terminal_bundle_key, snapshot_terminal_media_asset, + ActiveProviderManager, ActiveUserManager, AppState, CacheAccessState, CancelTokens, ConnectionKind, + ConnectionManager, CreateUserSessionParams, EventManager, HlsAccessContext, HlsAccessLease, HlsAccessLeaseId, + HlsAccessLeaseState, HlsAccessLeaseTiming, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, + HlsLeasePlaybackMode, HlsManifestCommitIdentity, HlsManifestCommitRequirement, HlsManifestDeliveryMode, + HlsMediaContainer, HlsOriginAccountBinding, HlsPlaybackFamilyKey, HlsPreparedTerminalBundleState, + HlsProxyManager, HlsPublishedTransientResourceIds, HlsRuntimeCustomTailAssetIdentity, HlsSession, + HlsSessionHandle, HlsSessionKey, HlsSessionMode, HlsTerminalAssetIdentity, HlsTerminalBaseMediaState, + HlsTerminalBaseProtection, HlsTerminalBaseSegmentAvailability, HlsTerminalMediaAsset, + HlsTerminalTailBuildInput, HlsTerminalTailGeneration, HlsTerminalTailPlan, HlsTerminalTailProtection, + MapCacheStatus, MetadataUpdateManager, OriginSegmentFetchRef, OriginSegmentKey, PlaylistStorageState, + ProxyMapId, ProxySessionId, RenderedManifest, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, + SharedStreamManager, TransientResourceKind, TransientResourceRef, TransportStreamBuffer, HLS_TERMINAL_TAIL_SEGMENT_COUNT, }, auth::Fingerprint, model::{ - ApiProxyConfig, ApiProxyServerInfo, AppConfig, Config, ConfigInput, ConfigProvider, ConfigSource, ConfigTarget, - CustomStreamResponse, HlsCacheConfig, ProxyUserCredentials, ReverseProxyConfig, - ReverseProxyDisabledHeaderConfig, SourcesConfig, StripConfig, TargetUser, + ApiProxyConfig, ApiProxyServerInfo, AppConfig, Config, ConfigInput, ConfigSource, ConfigTarget, + CustomStreamResponse, ProxyUserCredentials, ReverseProxyConfig, SourcesConfig, StripConfig, TargetUser, }, processing::parser::hls::{ + get_hls_session_token_and_url_from_token, origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}, - rewrite_hls, RewriteHlsProps, }, repository::GeoIp, }; -use aes::{ - cipher::{Block, BlockEncrypt, KeyInit}, - Aes128, -}; +use aes::{cipher::Block, Aes128}; use arc_swap::{ArcSwap, ArcSwapOption}; use axum::{ body::Body, extract::ConnectInfo, - http::{header, HeaderMap, HeaderName, HeaderValue, Method, Request, Response, StatusCode}, + http::{header, HeaderMap, HeaderValue, Method, Request, Response, StatusCode}, response::IntoResponse, }; use http_body_util::BodyExt; use shared::{ model::{ - provider_saturation::build_group_lookup, ConfigPaths, ConfigProviderDto, ConfigTargetDto, ConfigTargetOptions, - ConfigTargetShareLiveStreams, HlsCacheConfigDto, HlsManifestRecoveryBurstConfigDto, - HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripConfigDto, HlsStripMode, InputType, - M3uPlaylistItem, M3uTargetOutputDto, PlaylistItem, PlaylistItemHeader, PlaylistItemType, - ProviderUrlSelectionPolicy, ReverseProxyConfigDto, StreamConfigDto, StreamProperties, TargetOutputDto, + provider_saturation::build_group_lookup, ConfigPaths, ConfigTargetDto, ConfigTargetOptions, + ConfigTargetShareLiveStreams, HlsCacheConfigDto, HlsStripMode, InputType, M3uPlaylistItem, M3uTargetOutputDto, + PlaylistItem, PlaylistItemHeader, PlaylistItemType, ReverseProxyConfigDto, StreamConfigDto, TargetOutputDto, UserConnectionPermission, VirtualId, XtreamCluster, XtreamTargetOutputDto, }, utils::Internable, @@ -93,13306 +73,67 @@ use tokio::{ sync::RwLock, }; use tower::ServiceExt; - -fn path_has_extension(path: &str, extension: &str) -> bool { - std::path::Path::new(path).extension().is_some_and(|actual| actual.eq_ignore_ascii_case(extension)) -} - -#[test] -fn archive_epg_reference_supports_query_and_path_formats() { - assert_eq!( - m3u_archive_epg_reference_ts("http://provider/live/42.m3u8?utc=1700000000&lutc=1700003600"), - Some(1_700_000_000) - ); - assert_eq!( - m3u_archive_epg_reference_ts("http://provider/live/archive-1700003600-1700007200.m3u8"), - Some(1_700_003_600) - ); - assert_eq!(m3u_archive_epg_reference_ts("http://provider/live/timeshift_abs-1700007200.ts"), Some(1_700_007_200)); - assert_eq!( - m3u_archive_epg_reference_ts("http://provider/live/42.m3u8?start=1700000000&end=1700003600"), - Some(1_700_000_000) - ); -} - -#[test] -fn archive_epg_reference_rejects_plain_start_queries() { - assert_eq!(m3u_archive_epg_reference_ts("http://provider/live/42.m3u8?start=1700000000"), None); -} - -#[test] -fn date_tree_path_recovers_bittv_archive_epg_reference() { - assert_eq!(super::epg_reference_ts_from_date_tree_path("2026/07/24/14/13/38-06800.ts"), Some(1_784_902_418)); - assert_eq!( - super::epg_reference_ts_from_date_tree_path("dvr-2026/07/24/14/13/38-06800.ts"), - super::epg_reference_ts_from_date_tree_path("2026/07/24/14/13/38-06800.ts") - ); - assert!(super::looks_like_archive_media_path("2026/07/24/14/13/38-06800.ts")); -} - -#[test] -fn archive_media_path_does_not_accept_plain_202_prefixed_segments() { - assert!(!super::looks_like_archive_media_path("2026.ts")); - assert!(!super::looks_like_archive_media_path("202_media.ts")); -} - -#[test] -fn session_token_recovers_archive_epg_reference_when_media_url_lost_markers() { - assert_eq!( - m3u_catchup_epg_reference_from_session_token("m3u-catchup|user|42|archive|1717200000|3600"), - Some(1_717_200_000) - ); - assert_eq!(m3u_catchup_epg_reference_from_session_token("m3u-catchup|user|42|live"), None); -} - -#[test] -fn append_catchup_session_hint_keeps_m3u_catchup_token_without_shared_hls_cache() { - let fingerprint = test_fingerprint(); - let hint = "m3u-catchup|fp|alice|42|deadbeef"; - let token = super::hls_entry_user_session_token(&fingerprint, "alice", 42, Some(hint), Some(1_717_200_000)); - assert_eq!(token, hint); - assert!(super::is_m3u_catchup_session_token(&token)); - - let from_archive = super::hls_entry_user_session_token(&fingerprint, "alice", 42, None, Some(1_717_200_000)); - assert!(from_archive.contains("|archive|1717200000|0")); - assert!(super::is_m3u_catchup_session_token(&from_archive)); - assert!(super::is_m3u_catchup_session_token("m3u-catchup|fp|alice|42|timeshift_abs|1717200000|0")); -} - -#[test] -fn live_hls_entry_tokens_are_unique_but_share_a_stable_provider_owner() { - let fingerprint = test_fingerprint(); - let first = super::hls_entry_user_session_token(&fingerprint, "alice", 42, None, None); - let second = super::hls_entry_user_session_token(&fingerprint, "alice", 42, None, None); - - assert_ne!(first, second); - assert!(first.contains("|hls|")); - assert!(second.contains("|hls|")); - let first_lease = tuliprox_session::PlaybackLeaseRef::new(&first, tuliprox_core::model::PlaybackKind::LiveHls); - let second_lease = tuliprox_session::PlaybackLeaseRef::new(&second, tuliprox_core::model::PlaybackKind::LiveHls); - assert_ne!(first_lease.owner, second_lease.owner); - assert_eq!(first_lease.provider_owner(), second_lease.provider_owner()); - assert_ne!(first_lease.request_id, second_lease.request_id); - let other_channel = super::hls_entry_user_session_token(&fingerprint, "alice", 43, None, None); - let other_user = super::hls_entry_user_session_token(&fingerprint, "bob", 42, None, None); - assert_ne!( - first_lease.provider_owner(), - tuliprox_session::PlaybackLeaseRef::new(&other_channel, first_lease.kind).provider_owner() - ); - assert_ne!( - first_lease.provider_owner(), - tuliprox_session::PlaybackLeaseRef::new(&other_user, first_lease.kind).provider_owner() - ); -} - -#[test] -fn leaked_dvr_relative_joins_against_media_playlist_and_dvr_session_root() { - assert_eq!( - resolve_leaked_hls_relative_origin( - "http://cdn.example/big/aa_1/media.m3u8", - "dvr-2026/07/26/15/30/59-06000.ts", - Some("token=abc"), - ), - Some("http://cdn.example/big/aa_1/dvr-2026/07/26/15/30/59-06000.ts?token=abc".to_string()) - ); - assert_eq!( - resolve_leaked_hls_relative_origin( - "http://cdn.example/big/aa_1/dvr-2026/07/26/15/30/59-06000.ts?token=old", - "dvr-2026/07/26/15/31/05-06000.ts", - Some("token=new"), - ), - Some("http://cdn.example/big/aa_1/dvr-2026/07/26/15/31/05-06000.ts?token=new".to_string()) - ); - assert_eq!( - resolve_leaked_hls_relative_origin("http://cdn.example/big/aa_1/media.m3u8", "segment001.ts", None,), - None - ); - assert_eq!( - resolve_leaked_hls_relative_origin( - "http://cdn.example/big/aa_1/media.m3u8", - "dvr-2026/07/26/15/30/../59-06000.ts", - None, - ), - None - ); - assert_eq!(resolve_leaked_hls_relative_origin("http://cdn.example/live.m3u8", "./dvr-2026/a.ts", None), None); -} - -#[test] -fn archive_epg_reference_supports_contextual_start_aliases() { - assert_eq!( - m3u_archive_epg_reference_ts("http://provider/live/42.m3u8?offset=-3600&utcstart=1717200000"), - Some(1_717_200_000) - ); - assert_eq!( - m3u_archive_epg_reference_ts("http://provider/live/42.m3u8?timestamp=1717200000&offset=120"), - Some(1_717_200_000) - ); -} - -#[test] -fn canonical_manifest_joins_authoritative_owner_and_fails_closed_without_one() { - assert_eq!( - hls_canonical_owner_registration(HlsAvailabilityReevaluationRegistration::Scheduled), - HlsCanonicalOwnerRegistration::Join(HlsCanonicalOwnerRegistrationKind::Scheduled) - ); - assert_eq!( - hls_canonical_owner_registration(HlsAvailabilityReevaluationRegistration::AlreadyOwned), - HlsCanonicalOwnerRegistration::Join(HlsCanonicalOwnerRegistrationKind::AlreadyOwned) - ); - assert_eq!( - hls_canonical_owner_registration(HlsAvailabilityReevaluationRegistration::Superseded), - HlsCanonicalOwnerRegistration::Join(HlsCanonicalOwnerRegistrationKind::AlreadyOwned) - ); - for (registration, failure) in [ - ( - HlsAvailabilityReevaluationRegistration::CapacityExceeded, - HlsCanonicalOwnerRegistrationFailure::CapacityExceeded, - ), - ( - HlsAvailabilityReevaluationRegistration::RuntimeUnavailable, - HlsCanonicalOwnerRegistrationFailure::RuntimeUnavailable, - ), - ] { - assert_eq!(hls_canonical_owner_registration(registration), HlsCanonicalOwnerRegistration::FailClosed(failure)); - let response = hls_availability_reevaluation_registration_failure_response(failure); - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert!(response.headers().contains_key(header::RETRY_AFTER)); - } -} - -struct CanonicalOwnerHandoffFixture { - app_state: Arc, - session: HlsSessionHandle, - proxy_session_id: ProxySessionId, - leases: Vec<(HlsAccessLeaseId, u64)>, - strip: StripConfig, -} - -impl CanonicalOwnerHandoffFixture { - async fn new(lease_ids: &[&str]) -> Self { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "owner-handoff"), &app_state.get_encrypt_secret(), 100) - .await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let issued_at_ms = super::current_time_millis(); - let mut leases = Vec::with_capacity(lease_ids.len()); - for lease_id in lease_ids { - let lease_id = HlsAccessLeaseId((*lease_id).to_string()); - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), - proxy_session_id.clone(), - "hls-user".to_string(), - format!("{}-session", lease_id.0), - 1, - "owner-handoff".to_string(), - 12345, - issued_at_ms, - 60_000, - )) - .await; - leases.push((lease_id, issued_at_ms)); - } - Self { - app_state, - session, - proxy_session_id, - leases, - strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, - } - } - - async fn safe_session(&self) -> String { - let session = self.session.read().await; - super::safe_session_key(&session.key) - } - - fn handoff_context( - &self, - lease_index: usize, - safe_session: String, - request_deadline_ms: u64, - ) -> super::HlsCanonicalOwnerHandoffContext<'_> { - let (lease_id, issued_at_ms) = &self.leases[lease_index]; - super::HlsCanonicalOwnerHandoffContext { - app_state: &self.app_state, - proxy_session_id: &self.proxy_session_id, - access_lease_id: lease_id, - expected_lease_issued_at_ms: Some(*issued_at_ms), - strip: &self.strip, - server_path: None, - manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, - manifest_boundary_rendered_at_ms: 0, - bandwidth_learning: super::HlsRuntimeBandwidthLearningContext::Disabled, - request_deadline_ms, - safe_session, - } - } -} - -async fn publish_owner_handoff_test_manifest(session: &HlsSessionHandle) { - let now_ms = super::current_time_millis(); - let manifest = normal_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ - #EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n", - ); - let mut session = session.write().await; - session.apply_origin_manifest(&manifest).expect("owner handoff manifest maps"); - for segment in session.segments.values_mut() { - segment.status = SegmentCacheStatus::Ready { content_length: 1_000, ready_at_ms: now_ms }; - } - session.advance_media_readiness_generation(); - session.render_and_store_manifest(now_ms).expect("owner handoff manifest renders"); - session.mark_authorized_media_access(now_ms); -} - -#[tokio::test] -async fn scheduled_owner_does_not_return_transient_503() { - let fixture = CanonicalOwnerHandoffFixture::new(&["scheduled-lease"]).await; - let owner_key = fixture - .app_state - .hls - .proxy - .availability_reevaluation_owner_key(&fixture.session, &fixture.proxy_session_id) - .await - .expect("owner key"); - let coordinator = fixture.app_state.hls.proxy.availability_reevaluations(); - let started = Arc::new(tokio::sync::Notify::new()); - let release = Arc::new(tokio::sync::Notify::new()); - let completed = Arc::new(tokio::sync::Notify::new()); - let task_started = Arc::clone(&started); - let task_release = Arc::clone(&release); - let task_completed = Arc::clone(&completed); - let task_session = Arc::clone(&fixture.session); - let task_app_state = Arc::clone(&fixture.app_state); - let task_proxy_session_id = fixture.proxy_session_id.clone(); - let task_owner_key = owner_key.clone(); - assert_eq!( - coordinator.register( - owner_key, - HlsAvailabilityReevaluationMode::RecoveryPressure, - move |ownership| async move { - task_started.notify_one(); - task_release.notified().await; - publish_owner_handoff_test_manifest(&task_session).await; - task_app_state.hls.proxy.notify_session_evidence_changed(&task_proxy_session_id); - let _ = ownership.finish_cycle(&task_owner_key, HlsAvailabilityReevaluationFinishReason::Evaluated); - task_completed.notify_one(); - }, - ), - HlsAvailabilityReevaluationRegistration::Scheduled - ); - started.notified().await; - let deadline_ms = super::current_time_millis().saturating_add(60_000); - let safe_session = fixture.safe_session().await; - let mut response = Box::pin(super::join_hls_canonical_manifest_owner( - fixture.handoff_context(0, safe_session, deadline_ms), - HlsCanonicalOwnerRegistrationKind::Scheduled, - )); - - assert!(matches!(futures::poll!(response.as_mut()), std::task::Poll::Pending)); - release.notify_one(); - - let response = response.await; - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::RETRY_AFTER)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("live manifest utf8"); - assert!(body.contains("/scheduled-lease/")); - completed.notified().await; -} - -#[tokio::test] -async fn already_owned_work_is_joined_without_duplicate_refresh() { - let fixture = CanonicalOwnerHandoffFixture::new(&["first-lease", "second-lease"]).await; - let owner_key = fixture - .app_state - .hls - .proxy - .availability_reevaluation_owner_key(&fixture.session, &fixture.proxy_session_id) - .await - .expect("owner key"); - let coordinator = fixture.app_state.hls.proxy.availability_reevaluations(); - let release = Arc::new(tokio::sync::Notify::new()); - let completed = Arc::new(tokio::sync::Notify::new()); - let task_release = Arc::clone(&release); - let task_completed = Arc::clone(&completed); - let task_session = Arc::clone(&fixture.session); - let task_app_state = Arc::clone(&fixture.app_state); - let task_proxy_session_id = fixture.proxy_session_id.clone(); - let task_owner_key = owner_key.clone(); - assert_eq!( - coordinator.register( - owner_key.clone(), - HlsAvailabilityReevaluationMode::RecoveryPressure, - move |ownership| async move { - task_release.notified().await; - publish_owner_handoff_test_manifest(&task_session).await; - task_app_state.hls.proxy.notify_session_evidence_changed(&task_proxy_session_id); - let _ = ownership.finish_cycle(&task_owner_key, HlsAvailabilityReevaluationFinishReason::Evaluated); - task_completed.notify_one(); - }, - ), - HlsAvailabilityReevaluationRegistration::Scheduled - ); - let duplicate_owner_runs = Arc::new(AtomicUsize::new(0)); - let task_duplicate_owner_runs = Arc::clone(&duplicate_owner_runs); - assert_eq!( - coordinator.register(owner_key, HlsAvailabilityReevaluationMode::RecoveryPressure, move |_| async move { - task_duplicate_owner_runs.fetch_add(1, Ordering::SeqCst); - },), - HlsAvailabilityReevaluationRegistration::AlreadyOwned - ); - let deadline_ms = super::current_time_millis().saturating_add(60_000); - let mut first = Box::pin(super::join_hls_canonical_manifest_owner( - fixture.handoff_context(0, fixture.safe_session().await, deadline_ms), - HlsCanonicalOwnerRegistrationKind::AlreadyOwned, - )); - let mut second = Box::pin(super::join_hls_canonical_manifest_owner( - fixture.handoff_context(1, fixture.safe_session().await, deadline_ms), - HlsCanonicalOwnerRegistrationKind::AlreadyOwned, - )); - assert!(matches!(futures::poll!(first.as_mut()), std::task::Poll::Pending)); - assert!(matches!(futures::poll!(second.as_mut()), std::task::Poll::Pending)); - - release.notify_one(); - let (first, second) = tokio::join!(first, second); - - assert_eq!(first.status(), StatusCode::OK); - assert_eq!(second.status(), StatusCode::OK); - let first_body = String::from_utf8(response_body(first).await.to_vec()).expect("first manifest utf8"); - let second_body = String::from_utf8(response_body(second).await.to_vec()).expect("second manifest utf8"); - assert!(first_body.contains("/first-lease/")); - assert!(!first_body.contains("/second-lease/")); - assert!(second_body.contains("/second-lease/")); - assert!(!second_body.contains("/first-lease/")); - assert_eq!(duplicate_owner_runs.load(Ordering::SeqCst), 0); - completed.notified().await; - tokio::task::yield_now().await; - assert_eq!(coordinator.owner_count(), 0); -} - -#[tokio::test] -async fn canonical_owner_join_preserves_bounded_deadline_failure() { - let fixture = CanonicalOwnerHandoffFixture::new(&["deadline-lease"]).await; - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.leases[0].0, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("pending deadline lease"); - assert_eq!( - super::hls_canonical_owner_request_deadline_ms(&lease, Duration::ZERO, super::current_time_millis(),), - lease.pending_deadline_ms().expect("pending lease deadline") - ); - let owner_key = fixture - .app_state - .hls - .proxy - .availability_reevaluation_owner_key(&fixture.session, &fixture.proxy_session_id) - .await - .expect("owner key"); - let coordinator = fixture.app_state.hls.proxy.availability_reevaluations(); - assert_eq!( - coordinator.register(owner_key, HlsAvailabilityReevaluationMode::RecoveryPressure, |ownership| async move { - ownership.cancelled().await; - },), - HlsAvailabilityReevaluationRegistration::Scheduled - ); - let response = super::join_hls_canonical_manifest_owner( - fixture.handoff_context(0, fixture.safe_session().await, super::current_time_millis().saturating_sub(1)), - HlsCanonicalOwnerRegistrationKind::Scheduled, - ) - .await; - - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert!(!response.headers().contains_key(header::RETRY_AFTER)); - coordinator.cancel_session(&fixture.proxy_session_id); -} - -async fn terminal_generation_for_lease( - app_state: &Arc, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, -) -> HlsTerminalTailGeneration { - let lease = app_state - .hls - .proxy - .access_lease_response_snapshot(lease_id, proxy_session_id, super::current_time_millis()) - .await - .expect("terminal lease remains stored"); - let HlsLeasePlaybackMode::TerminalTail(plan) = lease.playback_mode else { - panic!("lease remains terminal"); - }; - plan.generation -} - -#[tokio::test] -async fn completed_owner_resolves_new_lease_standalone_without_reusing_old_terminal() { - let temp_dir = tempfile::tempdir().expect("owner handoff cache tempdir"); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); - enable_channel_unavailable_custom_response(&app_state); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"old-terminal-base").await; - let old_lease_id = HlsAccessLeaseId(format!("test-access-lease-{proxy_session_id}")); - terminalize_existing_test_lease(&app_state, &proxy_session_id, &old_lease_id.0, 123).await; - let proxy_session_id = ProxySessionId(proxy_session_id); - let now_ms = super::current_time_millis(); - let old_generation = terminal_generation_for_lease(&app_state, &proxy_session_id, &old_lease_id).await; - let new_lease_id = HlsAccessLeaseId("new-owner-handoff-lease".to_string()); - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - new_lease_id.clone(), - HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), - proxy_session_id.clone(), - "hls-user".to_string(), - "new-owner-handoff-session".to_string(), - 1, - "12345".to_string(), - 12345, - now_ms, - 60_000, - )) - .await; - let session = - app_state.hls.proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.expect("shared session"); - let owner_key = - app_state.hls.proxy.availability_reevaluation_owner_key(&session, &proxy_session_id).await.expect("owner key"); - let release = Arc::new(tokio::sync::Notify::new()); - let task_release = Arc::clone(&release); - let task_owner_key = owner_key.clone(); - let coordinator = app_state.hls.proxy.availability_reevaluations(); - assert_eq!( - coordinator.register( - owner_key, - HlsAvailabilityReevaluationMode::RecoveryPressure, - move |ownership| async move { - task_release.notified().await; - let _ = ownership.finish_cycle(&task_owner_key, HlsAvailabilityReevaluationFinishReason::Evaluated); - }, - ), - HlsAvailabilityReevaluationRegistration::Scheduled - ); - let safe_session = { - let session = session.read().await; - super::safe_session_key(&session.key) - }; - let strip = StripConfig { mode: HlsStripMode::Segments, value: 0 }; - let mut response = Box::pin(super::join_hls_canonical_manifest_owner( - super::HlsCanonicalOwnerHandoffContext { - app_state: &app_state, - proxy_session_id: &proxy_session_id, - access_lease_id: &new_lease_id, - expected_lease_issued_at_ms: Some(now_ms), - strip: &strip, - server_path: None, - manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, - manifest_boundary_rendered_at_ms: 0, - bandwidth_learning: super::HlsRuntimeBandwidthLearningContext::Disabled, - request_deadline_ms: now_ms.saturating_add(60_000), - safe_session, - }, - HlsCanonicalOwnerRegistrationKind::Scheduled, - )); - assert!(matches!(futures::poll!(response.as_mut()), std::task::Poll::Pending)); - - release.notify_one(); - let response = response.await; - - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - assert!(!response.headers().contains_key(header::RETRY_AFTER)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("standalone manifest utf8"); - assert!(body.contains("#EXT-X-ENDLIST")); - assert!(!body.contains("/hls/shared/live/")); - assert_eq!( - terminal_generation_for_lease(&app_state, &proxy_session_id, &old_lease_id).await, - old_generation, - "new lease fallback cannot reactivate old terminal lease" - ); -} - -#[test] -fn hls_cache_session_tokens_separate_live_and_archive_playback() { - let fingerprint = test_fingerprint(); - let live = super::create_hls_cache_user_session_token(&fingerprint, "user", 31, None, None); - let archive = super::create_hls_cache_user_session_token(&fingerprint, "user", 31, None, Some(1_784_898_000)); - - assert!(!super::is_m3u_catchup_session_token(&live)); - assert!(super::is_m3u_catchup_session_token(&archive)); - assert_ne!(live, archive); -} - -#[test] -fn hls_cache_session_token_preserves_existing_m3u_catchup_identity() { - let fingerprint = test_fingerprint(); - let existing = "m3u-catchup|fp|user|31|archive|1784898000|3600"; - let token = - super::create_hls_cache_user_session_token(&fingerprint, "user", 31, Some(existing), Some(1_784_898_000)); - - assert!(token.starts_with(existing)); - assert!(token.contains("|hls-cache|")); -} - -#[test] -fn hls_terminal_commit_endpoint_resolution_mapping_is_exhaustive() { - assert_eq!(hls_terminal_endpoint_action(HlsTerminalResolution::LiveAllowed), HlsTerminalEndpointAction::ServeLive); - assert_eq!( - hls_terminal_endpoint_action(HlsTerminalResolution::Committed), - HlsTerminalEndpointAction::ReloadTerminal - ); - assert_eq!(hls_terminal_endpoint_action(HlsTerminalResolution::Reevaluate), HlsTerminalEndpointAction::Reevaluate); - assert_eq!( - hls_terminal_endpoint_action(HlsTerminalResolution::Pending { retry_after_ms: 250 }), - HlsTerminalEndpointAction::RetryAfter { retry_after_ms: 250 } - ); - - for reason in [ - HlsTerminalFailedClosedReason::LeaseStateUnavailable, - HlsTerminalFailedClosedReason::BundleNotReadyWithoutOwner, - HlsTerminalFailedClosedReason::BundleIncompatible, - HlsTerminalFailedClosedReason::SafeCommitDeadlineElapsed, - HlsTerminalFailedClosedReason::RetryCapacityExceeded, - HlsTerminalFailedClosedReason::RetryAttemptsExhausted, - HlsTerminalFailedClosedReason::RuntimeUnavailable, - ] { - assert_eq!( - hls_terminal_endpoint_action(HlsTerminalResolution::FailedClosed { reason }), - HlsTerminalEndpointAction::FailClosed { reason } - ); - } -} - -#[tokio::test] -async fn hls_manifest_terminal_preflight_distinguishes_bootstrap_refresh_and_invalid_missing_snapshot() { - let app_state = test_app_state(); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "preflight-stream"), &app_state.get_encrypt_secret(), 1_000) - .await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let mut lease = HlsAccessLease::pending( - HlsAccessLeaseId("preflight-lease".to_string()), - HlsPlaybackFamilyKey::new("hls-user", "preflight-client"), - proxy_session_id, - "hls-user".to_string(), - "preflight-user-session".to_string(), - 1, - "preflight-stream".to_string(), - 12345, - 1_000, - 120_000, - ); - - assert_eq!( - hls_manifest_terminal_preflight(&session, &lease, 2_000).await, - HlsManifestTerminalPreflight::BootstrapPendingLease - ); - - lease.state = HlsAccessLeaseState::Activated; - assert_eq!( - hls_manifest_terminal_preflight(&session, &lease, 2_000).await, - HlsManifestTerminalPreflight::FailClosed { reason: HlsTerminalFailedClosedReason::LeaseStateUnavailable } - ); - - lease.last_manifest_snapshot = Some(HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(1_500), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 1, - delivered_at_ms: 1_500, - first_proxy_seq: 0, - last_proxy_seq: 0, - visible_segments: Arc::from([]), - discontinuity_sequence: 0, - target_duration_ms: 4_000, - playlist_duration_ms: 0, - last_visible_media_end_ms: 0, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - }); - { - let mut session = session.write().await; - session.origin_control.target_duration_snapshot_ms = Some(4_000); - session.origin_control.last_media_progress_at_ms = Some(2_000); - } - assert_eq!( - hls_manifest_terminal_preflight(&session, &lease, 8_000).await, - HlsManifestTerminalPreflight::RefreshBeforeTerminalEvaluation - ); - session.write().await.origin_control.last_media_progress_at_ms = Some(7_999); - assert_eq!( - hls_manifest_terminal_preflight(&session, &lease, 8_000).await, - HlsManifestTerminalPreflight::EvaluateTerminal - ); - - lease.playback_mode = HlsLeasePlaybackMode::Ended; - assert_eq!( - hls_manifest_terminal_preflight(&session, &lease, 8_000).await, - HlsManifestTerminalPreflight::ServeCommittedPlayback - ); -} - -#[tokio::test] -async fn hls_manifest_terminal_preflight_keeps_capacity_recovery_out_of_sync_terminal_wait() { - let app_state = test_app_state(); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "capacity-preflight"), &app_state.get_encrypt_secret(), 1_000) - .await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - { - let mut session = session.write().await; - let OriginManifestParseOutcome::Normal(manifest) = parse_origin_media_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:8\n#EXT-X-MEDIA-SEQUENCE:0\n\ - #EXTINF:4.0,\n0.ts\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n", - "http://origin.example/live/index.m3u8", - ) else { - panic!("capacity preflight manifest parses"); - }; - session.apply_origin_manifest(&manifest).expect("capacity preflight timeline applies"); - for segment in session.segments.values_mut() { - segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1_000 }; - } - session.segments.get_mut(&1).expect("deferred segment").status = - SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Prefetch, deferred_at_ms: 2_000 }; - session.origin_control.target_duration_snapshot_ms = Some(8_000); - session.origin_control.last_media_progress_at_ms = Some(1_000); - } - let mut lease = HlsAccessLease::pending( - HlsAccessLeaseId("capacity-preflight-lease".to_string()), - HlsPlaybackFamilyKey::new("capacity-user", "capacity-client"), - proxy_session_id, - "capacity-user".to_string(), - "capacity-user-session".to_string(), - 1, - "capacity-preflight".to_string(), - 1, - 1_000, - 60_000, - ); - lease.state = HlsAccessLeaseState::Activated; - lease.last_manifest_snapshot = Some(HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(1), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 1, - delivered_at_ms: 1_000, - first_proxy_seq: 0, - last_proxy_seq: 0, - visible_segments: Arc::from([HlsLeaseManifestSegment { - proxy_seq: 0, - duration_ms: 4_000, - uri: "000000.ts".to_string().into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }]), - discontinuity_sequence: 0, - target_duration_ms: 8_000, - playlist_duration_ms: 4_000, - last_visible_media_end_ms: 4_000, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - }); - - assert_eq!( - hls_manifest_terminal_preflight(&session, &lease, 20_000).await, - HlsManifestTerminalPreflight::EvaluateTerminal, - ); - - session.write().await.segments.get_mut(&1).expect("recovered segment").status = - SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 20_001 }; - assert_eq!( - hls_manifest_terminal_preflight(&session, &lease, 20_001).await, - HlsManifestTerminalPreflight::RefreshBeforeTerminalEvaluation, - ); -} - -#[test] -fn hls_terminal_commit_endpoint_pending_and_failed_closed_headers_are_distinct() { - let pending = hls_temporary_resource_unavailable_response(250); - assert_eq!(pending.status(), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!(pending.headers().get(header::RETRY_AFTER).and_then(|value| value.to_str().ok()), Some("1")); - assert!(pending.headers().get(header::LOCATION).is_none()); - - let failed_closed = hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::RetryAttemptsExhausted); - assert_eq!(failed_closed.status(), StatusCode::SERVICE_UNAVAILABLE); - assert!(failed_closed.headers().get(header::RETRY_AFTER).is_none()); - assert!(failed_closed.headers().get(header::LOCATION).is_none()); -} - -#[test] -fn extract_hls_provider_session_headers_converts_set_cookie_to_cookie_header() { - let mut headers = HeaderMap::new(); - headers.append("set-cookie", "sid=abc; Path=/; HttpOnly".parse().expect("valid cookie")); - headers.append("set-cookie", "pref=1; Secure".parse().expect("valid cookie")); - - let session_headers = extract_hls_provider_session_headers(&headers); - - assert_eq!(session_headers.headers.get("cookie").map(String::as_str), Some("sid=abc; pref=1")); -} - -fn test_app_config() -> Arc { - let mut hls_user = ProxyUserCredentials::default(); - hls_user.username = "hls-user".to_string(); - hls_user.password = "hls-pass".to_string(); - hls_user.max_connections = 1; - let api_proxy = ApiProxyConfig { - server: vec![ApiProxyServerInfo { - name: "default".to_string(), - protocol: "https".to_string(), - host: "example.test".to_string(), - port: None, - timezone: "UTC".to_string(), - message: String::new(), - path: Some("iptv".to_string()), - }], - user: vec![TargetUser { target: "default".to_string(), credentials: vec![Arc::new(hls_user)] }], - ..Default::default() - }; - Arc::new(AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config { custom_stream_response_enabled: true, ..Default::default() })), - sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), - hdhomerun: Arc::new(ArcSwapOption::empty()), - api_proxy: Arc::new(ArcSwapOption::from(Some(Arc::new(api_proxy)))), - file_locks: Arc::new(crate::utils::FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::empty()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(crate::model::MediaToolCapabilities::default()), - }) -} - -fn hls_custom_video_test_user() -> ProxyUserCredentials { - let mut user = ProxyUserCredentials::default(); - user.username = "viewer".to_string(); - user.password = "secret".to_string(); - user -} - -fn test_hls_share_target(hls_enabled: bool) -> ConfigTarget { - ConfigTarget::from(&ConfigTargetDto { - id: 1, - name: "default".to_string(), - options: Some(ConfigTargetOptions { - share_live_streams: ConfigTargetShareLiveStreams { hls: hls_enabled, mpeg_ts: false }, - ..Default::default() - }), - output: vec![TargetOutputDto::Xtream(XtreamTargetOutputDto::default())], - ..Default::default() - }) -} - -fn test_m3u_hls_share_target() -> ConfigTarget { - ConfigTarget::from(&ConfigTargetDto { - id: 2, - name: "m3u-target".to_string(), - options: Some(ConfigTargetOptions { - share_live_streams: ConfigTargetShareLiveStreams { hls: true, mpeg_ts: false }, - ..Default::default() - }), - output: vec![TargetOutputDto::M3u(M3uTargetOutputDto::default())], - use_memory_cache: true, - ..Default::default() - }) -} - -fn test_m3u_hls_item(input: &ConfigInput, virtual_id: u32, input_stream_id: &str, url: &str) -> M3uPlaylistItem { - M3uPlaylistItem::from(&PlaylistItem { - header: PlaylistItemHeader { - id: input_stream_id.intern(), - virtual_id: VirtualId::new(virtual_id), - input_name: Arc::clone(&input.name), - url: url.intern(), - item_type: PlaylistItemType::LiveHls, - xtream_cluster: XtreamCluster::Live, - input_stream_id: input_stream_id.intern(), - ..PlaylistItemHeader::default() - }, - }) -} - -fn test_hls_entry_stream_context( - virtual_id: u32, - input_stream_id: &str, - known_bitrate_bps: Option, -) -> super::HlsEntryStreamContext { - super::HlsEntryStreamContext { - identity: super::HlsEntryStreamIdentity::new(virtual_id, input_stream_id).expect("valid input stream identity"), - known_bitrate_bps, - } -} - -async fn cache_test_m3u_hls_item(app_state: &Arc, target: &ConfigTarget, item: M3uPlaylistItem) { - let mut playlist = crate::repository::BPlusTree::new(); - playlist.insert(item.virtual_id.get(), item); - app_state - .playlists - .cache_playlist(&target.name, crate::api::model::PlaylistStorage::M3uPlaylist(Box::new(playlist))) - .await; -} - -fn test_hls_input() -> ConfigInput { - ConfigInput { - id: 1, - name: Arc::from("test-input"), - input_type: InputType::Xtream, - url: "http://origin.example.com".to_string(), - username: Some("user".to_string()), - password: Some("pass".to_string()), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - } -} - -#[derive(Clone, Copy)] -enum TestLiveBitrateRepositoryState { - MissingDatabase, - MissingStreamItem, - ExistingHigher, - Update, - PermanentlyInapplicable, - RepositoryIoError, -} - -impl TestLiveBitrateRepositoryState { - const fn input_name(self) -> &'static str { - match self { - Self::MissingDatabase => "bandwidth-missing-database", - Self::MissingStreamItem => "bandwidth-missing-item", - Self::ExistingHigher => "bandwidth-existing-higher", - Self::Update => "bandwidth-update", - Self::PermanentlyInapplicable => "bandwidth-inapplicable", - Self::RepositoryIoError => "bandwidth-io-error", - } - } -} - -fn prepare_test_live_bitrate_repository( - input: &ConfigInput, - storage_root: &std::path::Path, - repository_state: TestLiveBitrateRepositoryState, -) { - let storage_path = - crate::repository::build_input_storage_path(&input.name, storage_root.to_string_lossy().as_ref()); - let database_path = crate::repository::get_input_m3u_playlist_file_path(&storage_path, &input.name); - match repository_state { - TestLiveBitrateRepositoryState::MissingDatabase | TestLiveBitrateRepositoryState::PermanentlyInapplicable => {} - TestLiveBitrateRepositoryState::RepositoryIoError => { - std::fs::create_dir_all(&storage_path).expect("input storage"); - std::fs::write(&database_path, b"invalid btree data").expect("corrupt input tree"); - } - TestLiveBitrateRepositoryState::MissingStreamItem - | TestLiveBitrateRepositoryState::ExistingHigher - | TestLiveBitrateRepositoryState::Update => { - std::fs::create_dir_all(&storage_path).expect("input storage"); - let (stream_ref, stored_bitrate) = match repository_state { - TestLiveBitrateRepositoryState::MissingStreamItem => ("different-channel", 0), - TestLiveBitrateRepositoryState::ExistingHigher => ("channel-a", 3_000_000), - TestLiveBitrateRepositoryState::Update => ("channel-a", 0), - TestLiveBitrateRepositoryState::MissingDatabase - | TestLiveBitrateRepositoryState::PermanentlyInapplicable - | TestLiveBitrateRepositoryState::RepositoryIoError => unreachable!(), - }; - let mut item = test_m3u_hls_item(input, 12345, stream_ref, "http://origin.test/live.m3u8"); - item.additional_properties = Some(StreamProperties::Live(Box::new(shared::model::LiveStreamProperties { - bitrate: stored_bitrate, - ..Default::default() - }))); - let mut tree = crate::repository::BPlusTree::new(); - tree.insert(Arc::clone(&item.provider_id), item); - tree.store(&database_path).expect("input tree"); - } - } -} - -async fn prepare_runtime_bandwidth_session( - app_state: &Arc, - input: &ConfigInput, - manifest_rendered_at_ms: u64, -) -> HlsSessionHandle { - let origin_source = super::build_hls_origin_source(input, "channel-a"); - let (session, _) = app_state - .hls - .proxy - .get_or_create_session_with_source_and_outcome( - origin_source.session_key(), - origin_source, - &app_state.get_encrypt_secret(), - manifest_rendered_at_ms, - ) - .await; - let manifest = normal_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ - #EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n", - ); - { - let mut session_guard = session.write().await; - session_guard.apply_origin_manifest(&manifest).expect("runtime learning timeline"); - for entry in session_guard.segments.values_mut() { - entry.status = - SegmentCacheStatus::Ready { content_length: 1_000_000, ready_at_ms: manifest_rendered_at_ms }; - } - session_guard.advance_media_readiness_generation(); - session_guard.render_and_store_manifest(manifest_rendered_at_ms).expect("runtime learning manifest"); - session_guard.mark_authorized_media_access(manifest_rendered_at_ms); - } - session -} - -async fn hls_runtime_bandwidth_manifest_case( - repository_state: TestLiveBitrateRepositoryState, -) -> (HlsBandwidthPersistenceState, Option) { - let temp = tempfile::tempdir().expect("temp dir"); - let app_state = test_app_state(); - let current_config = app_state.app_config.config.load(); - app_state.app_config.config.store(Arc::new(Config { - storage_dir: temp.path().to_string_lossy().into_owned(), - ..current_config.as_ref().clone() - })); - let input = ConfigInput { - id: 7, - name: Arc::from(repository_state.input_name()), - input_type: if matches!(repository_state, TestLiveBitrateRepositoryState::PermanentlyInapplicable) { - InputType::Library - } else { - InputType::M3u - }, - ..ConfigInput::default() - }; - prepare_test_live_bitrate_repository(&input, temp.path(), repository_state); - let manifest_rendered_at_ms = super::current_time_millis(); - let session = prepare_runtime_bandwidth_session(&app_state, &input, manifest_rendered_at_ms).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let access_lease_id = HlsAccessLeaseId(format!("{}-lease", repository_state.input_name())); - let now_ms = super::current_time_millis(); - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - access_lease_id.clone(), - HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), - proxy_session_id, - "hls-user".to_string(), - "hls-session-token".to_string(), - input.id, - "channel-a".to_string(), - 12345, - now_ms, - 60_000, - )) - .await; - - let response = super::try_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Pending, - &StripConfig { mode: HlsStripMode::Segments, value: 0 }, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - super::HlsRuntimeBandwidthLearningContext::Eligible(&input), - ) - .await - .expect("cached media manifest response"); - assert_eq!(response.status(), StatusCode::OK); - assert!(!response_body(response).await.is_empty()); - - let bandwidth_persistence = tokio::time::timeout(Duration::from_secs(10), async { - loop { - let state = session.read().await.bandwidth_persistence; - if !matches!(state, HlsBandwidthPersistenceState::Idle | HlsBandwidthPersistenceState::InFlight { .. }) { - break state; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("bandwidth persistence completion"); - let stored_bitrate = if matches!( - repository_state, - TestLiveBitrateRepositoryState::ExistingHigher | TestLiveBitrateRepositoryState::Update - ) { - crate::repository::load_input_live_bitrate_bps(&app_state.app_config, &input, "channel-a") - .await - .expect("stored bitrate read") - } else { - None - }; - (bandwidth_persistence, stored_bitrate) -} - -#[tokio::test] -async fn hls_runtime_bandwidth_missing_database_retries_without_failing_media_manifest() { - let (state, _) = hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::MissingDatabase).await; - - assert!(matches!(state, HlsBandwidthPersistenceState::RetryAfter { .. })); -} - -#[tokio::test] -async fn hls_runtime_bandwidth_missing_item_retries_without_failing_media_manifest() { - let (state, _) = hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::MissingStreamItem).await; - - assert!(matches!(state, HlsBandwidthPersistenceState::RetryAfter { .. })); -} - -#[tokio::test] -async fn hls_runtime_bandwidth_existing_higher_completes_without_failing_media_manifest() { - let (state, stored_bitrate) = - hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::ExistingHigher).await; - - assert!(matches!(state, HlsBandwidthPersistenceState::Persisted { bitrate_bps: 2_000_000 })); - assert_eq!(stored_bitrate, Some(3_000_000)); -} - -#[tokio::test] -async fn hls_runtime_bandwidth_update_completes_without_failing_media_manifest() { - let (state, stored_bitrate) = hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::Update).await; - - assert!(matches!(state, HlsBandwidthPersistenceState::Persisted { bitrate_bps: 2_000_000 })); - assert_eq!(stored_bitrate, Some(2_000_000)); -} - -#[tokio::test] -async fn hls_runtime_bandwidth_inapplicable_and_io_error_do_not_fail_media_manifest() { - let (inapplicable, _) = - hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::PermanentlyInapplicable).await; - let (io_error, _) = hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::RepositoryIoError).await; - - assert!(matches!(inapplicable, HlsBandwidthPersistenceState::PermanentlyInapplicable { bitrate_bps: 2_000_000 })); - assert!(matches!(io_error, HlsBandwidthPersistenceState::RetryAfter { .. })); -} - -#[tokio::test] -async fn hls_runtime_bandwidth_persistence_is_entry_gated_and_deduplicated() { - let temp = tempfile::tempdir().expect("temp dir"); - let app_state = test_app_state(); - let current_config = app_state.app_config.config.load(); - app_state.app_config.config.store(Arc::new(Config { - storage_dir: temp.path().to_string_lossy().into_owned(), - ..current_config.as_ref().clone() - })); - let input = ConfigInput { - id: 7, - name: Arc::from("runtime-bandwidth-input"), - input_type: InputType::M3u, - ..ConfigInput::default() - }; - let storage_path = crate::repository::build_input_storage_path(&input.name, temp.path().to_string_lossy().as_ref()); - std::fs::create_dir_all(&storage_path).expect("input storage"); - let item = test_m3u_hls_item(&input, 12345, "channel-a", "http://origin.test/live.m3u8"); - let mut tree = crate::repository::BPlusTree::new(); - tree.insert(Arc::clone(&item.provider_id), item); - tree.store(&crate::repository::get_input_m3u_playlist_file_path(&storage_path, &input.name)).expect("input tree"); - - let origin_source = super::build_hls_origin_source(&input, "channel-a"); - let (session, _) = app_state - .hls - .proxy - .get_or_create_session_with_source_and_outcome( - origin_source.session_key(), - origin_source, - &app_state.get_encrypt_secret(), - 1, - ) - .await; - { - let manifest = normal_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ - #EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n", - ); - let mut session_guard = session.write().await; - session_guard.apply_origin_manifest(&manifest).expect("runtime learning timeline"); - for entry in session_guard.segments.values_mut() { - entry.status = SegmentCacheStatus::Ready { content_length: 1_000_000, ready_at_ms: 2 }; - } - } - - assert!(super::spawn_hls_runtime_bandwidth_persistence( - &app_state, - &session, - super::HlsRuntimeBandwidthLearningContext::Disabled, - ) - .is_none()); - assert_eq!( - crate::repository::load_input_live_bitrate_bps(&app_state.app_config, &input, "channel-a") - .await - .expect("unknown bitrate read"), - None - ); - - let task = super::spawn_hls_runtime_bandwidth_persistence( - &app_state, - &session, - super::HlsRuntimeBandwidthLearningContext::Eligible(&input), - ) - .expect("runtime persistence task"); - task.await.expect("runtime persistence completion"); - - assert_eq!( - crate::repository::load_input_live_bitrate_bps(&app_state.app_config, &input, "channel-a") - .await - .expect("persisted bitrate read"), - Some(2_000_000) - ); - assert!(super::spawn_hls_runtime_bandwidth_persistence( - &app_state, - &session, - super::HlsRuntimeBandwidthLearningContext::Eligible(&input), - ) - .is_none()); -} - -fn store_test_sources_with_target(app_state: &Arc, input: ConfigInput, target: ConfigTarget) { - let input = Arc::new(input); - let inputs = vec![Arc::clone(&input)]; - app_state.app_config.sources.store(Arc::new(SourcesConfig { - batch_files: vec![], - provider: vec![], - group_lookup: build_group_lookup(&inputs), - inputs, - sources: vec![ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![Arc::new(target)] }], - templates: None, - })); -} - -fn configure_default_test_server(app_state: &Arc) { - let users = - app_state.app_config.api_proxy.load_full().as_ref().map_or_else(Vec::new, |api_proxy| api_proxy.user.clone()); - app_state.app_config.api_proxy.store(Some(Arc::new(ApiProxyConfig { - server: vec![ApiProxyServerInfo { - name: "default".to_string(), - protocol: "http".to_string(), - host: "127.0.0.1".to_string(), - port: Some("8901".to_string()), - timezone: "UTC".to_string(), - message: String::new(), - path: None, - }], - user: users, - ..Default::default() - }))); -} - -#[test] -fn hls_custom_video_manifest_body_is_none_for_non_provisioning() { - let user = hls_custom_video_test_user(); - let manifest = build_hls_custom_video_manifest_body( - "https://example.test/iptv/", - &user, - CustomVideoStreamType::UserConnectionsExhausted, - ); - - assert!(manifest.is_none(), "non-provisioning custom video types have no static manifest body"); -} - -#[tokio::test] -async fn hls_initial_manifest_decision_wait_timeout_defaults_to_ninety_seconds() { - let app_state = test_app_state(); - assert_eq!(super::hls_initial_manifest_decision_wait_timeout(&app_state), Duration::from_secs(90)); -} - -#[tokio::test] -async fn hls_manifest_channel_unavailable_renders_inline_without_redirect() { - let app_state = test_app_state(); - enable_channel_unavailable_custom_response(&app_state); - - let response = super::hls_manifest_channel_unavailable_response_for_username(&app_state, "hls-user").await; - - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - assert!(body.contains("#EXT-X-ENDLIST")); -} - -#[tokio::test] -async fn hls_manifest_channel_unavailable_falls_back_to_not_found_when_custom_response_is_disabled() { - let app_state = test_app_state(); - disable_custom_stream_response(&app_state); - - let response = super::hls_manifest_channel_unavailable_response_for_username(&app_state, "hls-user").await; - - assert_eq!(response.status(), StatusCode::NOT_FOUND); -} - -const RECOVERY_LOGICAL_LARGE_SEGMENT_BYTES: u64 = 20 * 1024 * 1024; - -struct RecoveryBeforeCutoverFixture { - _temp_dir: tempfile::TempDir, - origin: TestSegmentOrigin, - origin_phase: Arc, - app_state: Arc, - session: HlsSessionHandle, - proxy_session_id: ProxySessionId, - lease_id: HlsAccessLeaseId, - refresh: OriginRefreshRequest, -} - -async fn assert_initial_recovery_window(fixture: &RecoveryBeforeCutoverFixture) { - let response = try_test_hls_cached_manifest_response( - &fixture.app_state, - &fixture.session, - &fixture.lease_id, - HlsAccessLeaseState::Pending, - &StripConfig { mode: HlsStripMode::Segments, value: 3 }, - None, - super::HlsCachedManifestOptions::initial(Duration::from_secs(10)), - ) - .await - .expect("READY initial manifest"); - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - assert_eq!(media_uri_count(&body), 3); - assert!(body.contains("/000000.ts")); - assert!(body.contains("/000002.ts")); - assert!(!body.contains("/000003.ts")); - let now_ms = super::current_time_millis(); - assert!(fixture - .app_state - .hls - .proxy - .activate_access_lease( - &fixture.lease_id, - &fixture.proxy_session_id, - now_ms, - HlsAccessLeaseTiming { active_window_ms: 120_000, valid_window_ms: 180_000 }, - ) - .await - .is_activated()); - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) - .await - .expect("active stripped lease"); - let snapshot = lease.last_manifest_snapshot.as_ref().expect("lease manifest snapshot"); - assert_eq!(snapshot.visible_segments.len(), 3); - assert_eq!(snapshot.last_proxy_seq, 2); - let evidence = - prepare_terminal_base_evidence(&fixture.session, fixture.app_state.hls.proxy.segment_cache(), snapshot, now_ms) - .await; - assert_eq!(evidence.track_signature(), Some(terminal_test_asset().track_signature().clone())); - evidence.release(); - extend_ready_segment_as_sparse_file(&fixture.app_state, &fixture.session, 2, RECOVERY_LOGICAL_LARGE_SEGMENT_BYTES) - .await; - let uri = format!("/hls/shared/live/{}/{}/000002.ts", fixture.proxy_session_id.0, fixture.lease_id.0); - let segment = get_response(Arc::clone(&fixture.app_state), &uri, None).await; - assert_eq!(segment.status(), StatusCode::OK); - assert_eq!(segment.headers()[header::CONTENT_LENGTH], RECOVERY_LOGICAL_LARGE_SEGMENT_BYTES.to_string()); - assert_eq!( - u64::try_from(response_body(segment).await.len()).unwrap_or(u64::MAX), - RECOVERY_LOGICAL_LARGE_SEGMENT_BYTES - ); -} - -async fn recovery_before_cutover_fixture() -> RecoveryBeforeCutoverFixture { - let temp_dir = tempfile::tempdir().expect("recovery cache tempdir"); - let unchanged_manifest = Arc::<[u8]>::from(regression_origin_manifest(100, 6)); - let progressed_manifest = Arc::<[u8]>::from(regression_origin_manifest(101, 6)); - let segment = Arc::<[u8]>::from( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) - .as_slice(), - ); - let origin_phase = Arc::new(AtomicUsize::new(0)); - let phase = Arc::clone(&origin_phase); - let origin = spawn_test_binary_origin(Arc::new(move |path| { - if path_has_extension(path, "m3u8") { - return match phase.load(Ordering::SeqCst) { - 0 => TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&unchanged_manifest)), - 1 => TestBinaryOriginResponse::new( - StatusCode::PROXY_AUTHENTICATION_REQUIRED, - Arc::<[u8]>::from(&b"retry"[..]), - ), - _ => TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&progressed_manifest)), - }; - } - TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&segment)) - })) - .await; - let input = ConfigInput { - id: 1, - name: Arc::from("recovery-regression-input"), - input_type: InputType::M3u, - url: origin.base_url.clone(), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let app_state = - test_app_state_with_hls_proxy_and_inputs(test_beast_hls_proxy(temp_dir.path()), vec![Arc::new(input)]); - enable_hls_cache(&app_state); - create_active_hls_user_session(&app_state).await; - let manifest_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 1_000) - .await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("recovery-before-cutover".to_string()); - prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &lease_id).await; - let refresh = - regression_origin_refresh_request(&app_state, Arc::clone(&session), &manifest_url, Some(lease_id.clone())); - let fixture = RecoveryBeforeCutoverFixture { - _temp_dir: temp_dir, - origin, - origin_phase, - app_state, - session, - proxy_session_id, - lease_id, - refresh, - }; - assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); - wait_for_ready_timeline(&fixture.session, 6).await; - assert_initial_recovery_window(&fixture).await; - fixture -} - -async fn run_recovery_outage(fixture: &mut RecoveryBeforeCutoverFixture) -> (u64, Option) { - let progress_generation = fixture.session.read().await.origin_control.progress_generation; - for _ in 0..2 { - fixture.refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; - assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); - } - assert_eq!(fixture.session.read().await.origin_control.progress_generation, progress_generation); - fixture.origin_phase.store(1, Ordering::SeqCst); - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - assert_eq!(fixture.app_state.hls.proxy.manifest_recovery_burst().level.plan(), plan); - fixture.refresh.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; - let requests_before = fixture.origin.manifest_request_count(); - let mut last_episode_generation = None; - for _ in 0..3 { - fixture.refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; - assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("407 exhaustion cannot remove the lease"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - let session = fixture.session.read().await; - let episode = session.origin_control.acceptance_episode.as_ref().expect("bounded acceptance evidence"); - assert!(episode.full_burst_completed); - assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); - last_episode_generation = Some(episode.generation.0); - } - assert!( - fixture.origin.manifest_request_count().saturating_sub(requests_before) - >= plan.total_candidates().saturating_mul(3) - ); - let uri = format!("/hls/shared/live/{}/{}/000003.ts", fixture.proxy_session_id.0, fixture.lease_id.0); - let cached = get_response(Arc::clone(&fixture.app_state), &uri, None).await; - assert_eq!(cached.status(), StatusCode::OK); - assert!(!response_body(cached).await.is_empty()); - (progress_generation, last_episode_generation) -} - -async fn assert_recovery_after_outage( - fixture: &mut RecoveryBeforeCutoverFixture, - progress_generation: u64, - last_episode_generation: Option, -) { - fixture.origin_phase.store(2, Ordering::SeqCst); - fixture.refresh.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; - fixture.refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; - assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); - wait_for_ready_timeline(&fixture.session, 7).await; - { - let session = fixture.session.read().await; - assert_eq!(session.origin_seq_highwater, Some(106)); - assert_eq!(session.proxy_next_seq, Some(7)); - assert!(session.origin_control.progress_generation > progress_generation); - assert!(session.origin_control.acceptance_episode.is_none()); - assert!(last_episode_generation - .is_some_and(|generation| session.origin_control.acceptance_generation.0 > generation)); - } - let response = try_test_hls_cached_manifest_response( - &fixture.app_state, - &fixture.session, - &fixture.lease_id, - HlsAccessLeaseState::Activated, - &StripConfig { mode: HlsStripMode::Segments, value: 3 }, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - ) - .await - .expect("recovered normal manifest"); - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("recovered manifest utf8"); - assert!(body.contains("/000006.ts")); - assert!(!body.contains("/terminal/")); - assert!(!body.contains("#EXT-X-ENDLIST")); - let uri = format!("/hls/shared/live/{}/{}/000006.ts", fixture.proxy_session_id.0, fixture.lease_id.0); - let segment = get_response(Arc::clone(&fixture.app_state), &uri, None).await; - assert_eq!(segment.status(), StatusCode::OK); - assert!(!response_body(segment).await.is_empty()); - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("recovered lease remains stored"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); -} - -#[tokio::test] -async fn recovers_before_lease_cutover_without_terminal_tail() { - let mut fixture = recovery_before_cutover_fixture().await; - let (progress_generation, last_episode_generation) = run_recovery_outage(&mut fixture).await; - assert_recovery_after_outage(&mut fixture, progress_generation, last_episode_generation).await; -} - -struct StaleOriginServers { - pinned: TestSegmentOrigin, - alternative: TestSegmentOrigin, - pinned_phase: Arc, - alternative_phase: Arc, - burst_candidates: Arc, - pinned_candidates: Arc, -} - -async fn spawn_stale_origin_servers() -> StaleOriginServers { - let pinned_manifest = Arc::<[u8]>::from(regression_origin_manifest(100, 6)); - let alternative_manifest = Arc::<[u8]>::from(regression_origin_manifest(200, 6)); - let continued_manifest = Arc::<[u8]>::from(regression_origin_manifest(201, 6)); - let segment = Arc::<[u8]>::from( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) - .as_slice(), - ); - let alternative_phase = Arc::new(AtomicUsize::new(0)); - let alternative_phase_for_handler = Arc::clone(&alternative_phase); - let alternative_segment = Arc::clone(&segment); - let alternative = spawn_test_binary_origin(Arc::new(move |path| { - if path_has_extension(path, "m3u8") { - let body = if alternative_phase_for_handler.load(Ordering::SeqCst) == 0 { - Arc::clone(&alternative_manifest) - } else { - Arc::clone(&continued_manifest) - }; - return TestBinaryOriginResponse::new(StatusCode::OK, body); - } - TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&alternative_segment)) - })) - .await; - let alternative_url = - format!("{}/live/user/pass/12345.m3u8", alternative.base_url.replacen("127.0.0.1", "localhost", 1)); - let pinned_phase = Arc::new(AtomicUsize::new(0)); - let burst_candidates = Arc::new(AtomicUsize::new(0)); - let pinned_candidates = Arc::new(AtomicUsize::new(0)); - let handler_phase = Arc::clone(&pinned_phase); - let handler_burst_candidates = Arc::clone(&burst_candidates); - let handler_pinned_candidates = Arc::clone(&pinned_candidates); - let handler_pinned_manifest = Arc::clone(&pinned_manifest); - let pinned_segment = Arc::clone(&segment); - let pinned = spawn_test_binary_origin(Arc::new(move |path| { - if !path_has_extension(path, "m3u8") { - return TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&pinned_segment)); - } - match handler_phase.load(Ordering::SeqCst) { - 0 => TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&handler_pinned_manifest)), - 1 => { - let index = handler_burst_candidates.fetch_add(1, Ordering::SeqCst); - if index.is_multiple_of(2) { - handler_pinned_candidates.fetch_add(1, Ordering::SeqCst); - TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&handler_pinned_manifest)) - } else { - TestBinaryOriginResponse::redirect(alternative_url.clone()) - } - } - _ => TestBinaryOriginResponse::redirect(alternative_url.clone()), - } - })) - .await; - StaleOriginServers { pinned, alternative, pinned_phase, alternative_phase, burst_candidates, pinned_candidates } -} - -struct StaleOriginFixture { - _temp_dir: tempfile::TempDir, - servers: StaleOriginServers, - app_state: Arc, - session: HlsSessionHandle, - proxy_session_id: ProxySessionId, - lease_id: HlsAccessLeaseId, - refresh: OriginRefreshRequest, - initial_visible_tail: u64, - initial_progress_generation: u64, - initial_origin_epoch: u64, - initial_progress_at_ms: Option, -} - -async fn stale_origin_fixture() -> StaleOriginFixture { - let temp_dir = tempfile::tempdir().expect("stale-origin cache tempdir"); - let servers = spawn_stale_origin_servers().await; - let input = ConfigInput { - id: 1, - name: Arc::from("reachable-stale-origin-input"), - input_type: InputType::M3u, - url: servers.pinned.base_url.clone(), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let app_state = - test_app_state_with_hls_proxy_and_inputs(test_beast_hls_proxy(temp_dir.path()), vec![Arc::new(input)]); - enable_hls_cache(&app_state); - create_active_hls_user_session(&app_state).await; - let manifest_url = format!("{}/live/user/pass/12345.m3u8", servers.pinned.base_url); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 1_000) - .await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("reachable-stale-origin".to_string()); - prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &lease_id).await; - let refresh = - regression_origin_refresh_request(&app_state, Arc::clone(&session), &manifest_url, Some(lease_id.clone())); - assert!(trigger_origin_refresh_sync(refresh.clone()).await); - wait_for_ready_timeline(&session, 6).await; - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &lease_id, - HlsAccessLeaseState::Pending, - &StripConfig { mode: HlsStripMode::Segments, value: 3 }, - None, - super::HlsCachedManifestOptions::initial(Duration::from_secs(10)), - ) - .await - .expect("READY pinned-origin manifest"); - assert_eq!(response.status(), StatusCode::OK); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0")); - assert_eq!(media_uri_count(&body), 3); - let now_ms = super::current_time_millis(); - assert!(app_state - .hls - .proxy - .activate_access_lease( - &lease_id, - &proxy_session_id, - now_ms, - HlsAccessLeaseTiming { active_window_ms: 120_000, valid_window_ms: 180_000 }, - ) - .await - .is_activated()); - let lease = app_state - .hls - .proxy - .access_lease_response_snapshot(&lease_id, &proxy_session_id, now_ms) - .await - .expect("activated pinned-origin lease"); - let initial_visible_tail = - lease.last_manifest_snapshot.as_ref().expect("initial lease manifest snapshot").last_proxy_seq; - let (initial_progress_generation, initial_origin_epoch, initial_progress_at_ms) = { - let session = session.read().await; - assert_eq!(session.origin_seq_highwater, Some(105)); - assert_eq!(session.last_effective_manifest_host.as_deref(), Some("127.0.0.1")); - ( - session.origin_control.progress_generation, - session.origin_epoch, - session.origin_control.last_media_progress_at_ms, - ) - }; - StaleOriginFixture { - _temp_dir: temp_dir, - servers, - app_state, - session, - proxy_session_id, - lease_id, - refresh, - initial_visible_tail, - initial_progress_generation, - initial_origin_epoch, - initial_progress_at_ms, - } -} - -async fn observe_stale_origin(fixture: &mut StaleOriginFixture) -> HlsManifestAcceptanceDirective { - let requests_before = fixture.servers.pinned.manifest_request_count(); - for _ in 0..2 { - fixture.refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; - assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); - } - { - let session = fixture.session.read().await; - assert_eq!(session.origin_seq_highwater, Some(105)); - assert_eq!(session.origin_control.progress_generation, fixture.initial_progress_generation); - assert_eq!(session.origin_control.last_media_progress_at_ms, fixture.initial_progress_at_ms); - assert_eq!(session.origin_refresh.consecutive_failures, 0); - } - assert_eq!(fixture.servers.pinned.manifest_request_count().saturating_sub(requests_before), 2); - assert_eq!(fixture.servers.alternative.manifest_request_count(), 0); - let response = try_test_hls_cached_manifest_response( - &fixture.app_state, - &fixture.session, - &fixture.lease_id, - HlsAccessLeaseState::Activated, - &StripConfig { mode: HlsStripMode::Segments, value: 3 }, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - ) - .await - .expect("reachable stale origin keeps the committed live manifest"); - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("stale live manifest utf8"); - assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0")); - assert!(!body.contains("/terminal/")); - assert!(!body.contains("#EXT-X-ENDLIST")); - { - let mut session = fixture.session.write().await; - for segment in session.segments.values_mut().filter(|segment| segment.proxy_seq > fixture.initial_visible_tail) - { - segment.duration_ms = 10_000; - } - session.origin_control.last_media_progress_at_ms = Some(0); - session.advance_media_readiness_generation(); - } - let directive = match crate::api::model::hls_manifest_acceptance_directive_for_session( - &fixture.app_state.hls_ctx(), - &fixture.session, - &fixture.proxy_session_id, - ) - .await - { - HlsManifestAcceptanceEvaluationOutcome::Evaluated(directive) => directive, - other => panic!("stale progress evidence must evaluate: {other:?}"), - }; - assert!(directive.trigger.recovery_required()); - assert_eq!(fixture.session.read().await.origin_control.path_condition, HlsOriginPathCondition::PublicationLate); - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("stale progress evidence keeps the lease stored"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - directive -} - -async fn assert_stale_origin_handoff( - fixture: &mut StaleOriginFixture, - directive: HlsManifestAcceptanceDirective, -) -> u64 { - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - assert_eq!(fixture.app_state.hls.proxy.manifest_recovery_burst().level.plan(), plan); - fixture.servers.pinned_phase.store(1, Ordering::SeqCst); - let requests_before = fixture.servers.pinned.manifest_request_count(); - fixture.refresh.acceptance_directive = directive; - fixture.refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; - assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); - wait_for_ready_timeline(&fixture.session, 12).await; - assert_eq!( - fixture.servers.pinned.manifest_request_count().saturating_sub(requests_before), - plan.total_candidates() - ); - assert_eq!(fixture.servers.burst_candidates.load(Ordering::SeqCst), plan.total_candidates()); - assert!(fixture.servers.pinned_candidates.load(Ordering::SeqCst) > 0); - assert!(fixture.servers.alternative.manifest_request_count() >= 2); - let progress_generation = { - let session = fixture.session.read().await; - assert_eq!(session.origin_seq_highwater, Some(205)); - assert_eq!(session.proxy_next_seq, Some(12)); - assert_eq!(session.last_effective_manifest_host.as_deref(), Some("localhost")); - assert_eq!(session.origin_epoch, fixture.initial_origin_epoch.saturating_add(1)); - assert!(session.origin_control.progress_generation > fixture.initial_progress_generation); - session.origin_control.progress_generation - }; - let response = try_test_hls_cached_manifest_response( - &fixture.app_state, - &fixture.session, - &fixture.lease_id, - HlsAccessLeaseState::Activated, - &StripConfig { mode: HlsStripMode::Segments, value: 3 }, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - ) - .await - .expect("cross-host recovery manifest"); - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("recovered manifest utf8"); - assert!(body.contains("/000006.ts")); - assert!(body.contains("#EXT-X-DISCONTINUITY")); - assert!(!body.contains("/terminal/")); - assert!(!body.contains("#EXT-X-ENDLIST")); - let uri = format!("/hls/shared/live/{}/{}/000006.ts", fixture.proxy_session_id.0, fixture.lease_id.0); - let segment = get_response(Arc::clone(&fixture.app_state), &uri, None).await; - assert_eq!(segment.status(), StatusCode::OK); - assert!(!response_body(segment).await.is_empty()); - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("recovered lease remains stored"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - progress_generation -} - -async fn assert_stale_origin_continuation(fixture: &mut StaleOriginFixture, progress_generation: u64) { - fixture.servers.alternative_phase.store(1, Ordering::SeqCst); - fixture.servers.pinned_phase.store(2, Ordering::SeqCst); - fixture.refresh.acceptance_directive = HlsManifestAcceptanceDirective::none(); - fixture.refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; - assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); - wait_for_ready_timeline(&fixture.session, 13).await; - { - let session = fixture.session.read().await; - assert_eq!(session.origin_seq_highwater, Some(206)); - assert_eq!(session.proxy_next_seq, Some(13)); - assert_eq!(session.origin_epoch, fixture.initial_origin_epoch.saturating_add(1)); - assert!(session.origin_control.progress_generation > progress_generation); - } - let response = try_test_hls_cached_manifest_response( - &fixture.app_state, - &fixture.session, - &fixture.lease_id, - HlsAccessLeaseState::Activated, - &StripConfig { mode: HlsStripMode::Segments, value: 3 }, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - ) - .await - .expect("continued alternative-origin timeline"); - assert_eq!(response.status(), StatusCode::OK); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("continued manifest utf8"); - assert!(body.contains("/000012.ts")); - assert!(!body.contains("/terminal/")); - assert!(!body.contains("#EXT-X-ENDLIST")); - let uri = format!("/hls/shared/live/{}/{}/000012.ts", fixture.proxy_session_id.0, fixture.lease_id.0); - let segment = get_response(Arc::clone(&fixture.app_state), &uri, None).await; - assert_eq!(segment.status(), StatusCode::OK); - assert!(!response_body(segment).await.is_empty()); -} - -#[tokio::test] -async fn reachable_stale_origin_hands_off_to_progressed_origin_without_terminal_tail() { - let mut fixture = stale_origin_fixture().await; - let directive = observe_stale_origin(&mut fixture).await; - let progress_generation = assert_stale_origin_handoff(&mut fixture, directive).await; - assert_stale_origin_continuation(&mut fixture, progress_generation).await; -} - -#[tokio::test] -async fn terminal_lease_manifest_is_inline_immutable_endlist_on_canonical_path() { - const LIVE_TAIL_BYTES: &[u8] = b"original-live-tail"; - let temp_dir = tempfile::tempdir().expect("tempdir"); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", LIVE_TAIL_BYTES).await; - let lease_id = format!("test-access-lease-{proxy_session_id}"); - terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; - let cursor_before = app_state - .hls - .proxy - .access_lease_response_snapshot( - &HlsAccessLeaseId(lease_id.clone()), - &ProxySessionId(proxy_session_id.clone()), - super::current_time_millis(), - ) - .await - .expect("terminal lease before media read") - .playback_cursor; - let (generation, segment_count) = terminal_test_plan_shape(&app_state, &proxy_session_id, &lease_id).await; - let manifest_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/manifest.m3u8"); - let mut reloaded_api_proxy = - app_state.app_config.api_proxy.load_full().as_deref().cloned().expect("test API proxy config"); - reloaded_api_proxy.server[0].path = Some("reloaded".to_string()); - app_state.app_config.api_proxy.store(Some(Arc::new(reloaded_api_proxy))); - - let response = get_response(Arc::clone(&app_state), &manifest_uri, None).await; - - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("terminal manifest is utf8"); - let repeated = get_response(Arc::clone(&app_state), &manifest_uri, None).await; - assert_eq!(repeated.status(), StatusCode::OK); - assert!(!repeated.headers().contains_key(header::LOCATION)); - assert_eq!(response_body(repeated).await, body.as_bytes()); - let live_tail = format!("/{proxy_session_id}/{lease_id}/000123.ts"); - let terminal_prefix = format!("/{proxy_session_id}/{lease_id}/terminal/{generation}/"); - assert!(body.contains(&live_tail)); - assert!(body.contains("/iptv/hls/shared/live/")); - assert!(!body.contains("/reloaded/hls/shared/live/")); - assert_eq!(body.matches(&terminal_prefix).count(), usize::from(segment_count)); - assert_eq!(body.matches("#EXT-X-DISCONTINUITY\n").count(), 1); - assert!(body.ends_with("#EXT-X-ENDLIST\n")); - assert!(body.find(&live_tail) < body.find("#EXT-X-DISCONTINUITY\n")); - assert!(body.find("#EXT-X-DISCONTINUITY\n") < body.find(&terminal_prefix)); - - let live_tail_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/000123.ts"); - let live_tail_response = get_response(Arc::clone(&app_state), &live_tail_uri, Some("bytes=0-")).await; - assert_eq!(live_tail_response.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(live_tail_response.headers()[header::CONTENT_LENGTH], LIVE_TAIL_BYTES.len().to_string()); - assert_eq!( - live_tail_response.headers()[header::CONTENT_RANGE], - format!("bytes 0-{}/{}", LIVE_TAIL_BYTES.len() - 1, LIVE_TAIL_BYTES.len()) - ); - assert_eq!(response_body(live_tail_response).await, bytes::Bytes::from_static(LIVE_TAIL_BYTES)); - let cursor_after = app_state - .hls - .proxy - .access_lease_response_snapshot( - &HlsAccessLeaseId(lease_id), - &ProxySessionId(proxy_session_id), - super::current_time_millis(), - ) - .await - .expect("terminal lease after media read") - .playback_cursor; - assert_eq!(cursor_after, cursor_before); -} - -struct PreparedTerminalCutoverFixture { - _temp_dir: tempfile::TempDir, - origin: TestSegmentOrigin, - app_state: Arc, - session: HlsSessionHandle, - proxy_session_id: ProxySessionId, - lease_id: HlsAccessLeaseId, - request_url: String, - base_manifest: HlsLeaseManifestSnapshot, - asset_buffer: TransportStreamBuffer, - asset: Arc, -} - -async fn prepare_terminal_cutover_bundle( - app_state: &Arc, - base_manifest: &HlsLeaseManifestSnapshot, -) -> (TransportStreamBuffer, Arc) { - let asset_buffer = app_state - .app_config - .custom_stream_response - .load_full() - .as_ref() - .and_then(|responses| responses.channel_unavailable.as_ref()) - .cloned() - .expect("configured terminal renderer"); - let asset = snapshot_terminal_media_asset(&asset_buffer).expect("terminal asset snapshot"); - let key = prepared_terminal_bundle_key(&asset, base_manifest.target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); - let state = app_state.hls.proxy.start_prepared_terminal_bundle( - Arc::clone(&asset), - base_manifest.target_duration_ms, - HLS_TERMINAL_TAIL_SEGMENT_COUNT, - ); - let state = match state { - HlsPreparedTerminalBundleState::Preparing { .. } => app_state - .hls - .proxy - .wait_for_prepared_terminal_bundle(key) - .await - .expect("prepared terminal bundle completion"), - state => state, - }; - assert!(matches!( - state, - HlsPreparedTerminalBundleState::Ready { ref bundle } - if bundle.key == key - && bundle.segments.len() == usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT) - )); - assert_eq!(asset_buffer.finite_hls_render_count(), usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); - (asset_buffer, asset) -} - -async fn prepared_terminal_cutover_fixture() -> PreparedTerminalCutoverFixture { - let temp_dir = tempfile::tempdir().expect("terminal cutover tempdir"); - let origin = spawn_test_encrypted_hls_origin( - AES_TEST_MANIFEST, - Arc::from(AES_TEST_KEY_BYTES), - Arc::from(AES_TEST_PLAINTEXT_SEGMENT), - ) - .await; - let input = ConfigInput { - id: 1, - name: Arc::from("terminal-regression-input"), - input_type: InputType::M3u, - url: origin.base_url.clone(), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let app_state = - test_app_state_with_hls_proxy_and_inputs(test_beast_hls_proxy(temp_dir.path()), vec![Arc::new(input.clone())]); - enable_hls_cache(&app_state); - enable_channel_unavailable_custom_response(&app_state); - create_active_hls_user_session(&app_state).await; - let request_url = format!("{}/channel/index.m3u8", origin.base_url); - let origin_source = super::build_hls_origin_source(&input, "12345"); - let session_key = origin_source.session_key(); - let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - let lease_id = HlsAccessLeaseId("prepared-terminal-cutover".to_string()); - let access_context = test_hls_access_context(proxy_session_id.clone(), lease_id.clone()); - prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &lease_id).await; - let response = super::try_hls_cache_canonical_manifest_response( - &app_state, - &test_fingerprint(), - &access_context, - &proxy_session_id, - &lease_id, - HlsAccessLeaseState::Pending, - super::HlsCacheManifestOrigin { - raw_request_url: &request_url, - session_entry_url: super::HlsOriginEntryUrl::direct_http(&request_url), - input: &input, - origin_source, - }, - HeaderMap::new(), - None, - "/m3u-stream/live/hls-user/hls-pass/12345.m3u8", - super::HlsManifestRefreshOrdering::Background, - ) - .await - .expect("AES shared session cold start"); - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("AES manifest utf8"); - assert!(body.contains("#EXT-X-KEY:METHOD=AES-128")); - let session = app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("AES shared session"); - wait_for_ready_timeline(&session, 6).await; - let now_ms = super::current_time_millis(); - assert!(app_state - .hls - .proxy - .activate_access_lease( - &lease_id, - &proxy_session_id, - now_ms, - HlsAccessLeaseTiming { active_window_ms: 120_000, valid_window_ms: 180_000 }, - ) - .await - .is_activated()); - let lease = app_state - .hls - .proxy - .access_lease_response_snapshot(&lease_id, &proxy_session_id, now_ms) - .await - .expect("live AES lease"); - let base_manifest = lease.last_manifest_snapshot.as_ref().expect("frozen AES lease manifest").clone(); - assert_eq!(base_manifest.target_duration_ms, 12_000); - assert_eq!(base_manifest.visible_segments.len(), 3); - assert!(base_manifest.active_encryption.is_some()); - let (asset_buffer, asset) = prepare_terminal_cutover_bundle(&app_state, &base_manifest).await; - for segment in base_manifest.visible_segments.iter() { - let response = get_response(Arc::clone(&app_state), &segment.uri, None).await; - assert_eq!(response.status(), StatusCode::OK); - assert!(!response_body(response).await.is_empty()); - } - PreparedTerminalCutoverFixture { - _temp_dir: temp_dir, - origin, - app_state, - session, - proxy_session_id, - lease_id, - request_url, - base_manifest, - asset_buffer, - asset, - } -} - -async fn apply_terminal_cutover_pressure(fixture: &PreparedTerminalCutoverFixture) { - let mut session = fixture.session.write().await; - session.origin_control.path_condition = HlsOriginPathCondition::HardFetchFailure; - let transition_buffer_seq = fixture.base_manifest.last_proxy_seq.saturating_add(1); - let commit_guard_seq = transition_buffer_seq.saturating_add(1); - for segment in - session.segments.values_mut().filter(|segment| segment.proxy_seq > fixture.base_manifest.last_proxy_seq) - { - if segment.proxy_seq == transition_buffer_seq { - segment.duration_ms = 1_000; - } else if segment.proxy_seq == commit_guard_seq { - segment.duration_ms = 2_800; - } else { - segment.status = SegmentCacheStatus::Expired; - } - } - session.advance_media_readiness_generation(); -} - -async fn terminal_cutover_acceptance_directive( - fixture: &PreparedTerminalCutoverFixture, -) -> HlsManifestAcceptanceDirective { - apply_terminal_cutover_pressure(fixture).await; - let directive = match crate::api::model::hls_manifest_acceptance_directive_for_session( - &fixture.app_state.hls_ctx(), - &fixture.session, - &fixture.proxy_session_id, - ) - .await - { - HlsManifestAcceptanceEvaluationOutcome::Evaluated(directive) => directive, - other => panic!("deterministic recovery-pressure snapshot must evaluate: {other:?}"), - }; - assert_eq!(directive.trigger, HlsManifestAcceptanceTrigger::RecoveryRequired); - let bundle_key = prepared_terminal_bundle_key( - &fixture.asset, - fixture.base_manifest.target_duration_ms, - HLS_TERMINAL_TAIL_SEGMENT_COUNT, - ); - let timing = directive.timing_seed.expect("prepared acceptance timing seed"); - assert_eq!(timing.required_terminal_media_key, Some(bundle_key)); - assert_eq!(timing.terminal_media_preparation, HlsTerminalMediaPreparationState::Ready { key: bundle_key }); - directive -} - -async fn exhaust_terminal_cutover_recovery( - fixture: &PreparedTerminalCutoverFixture, - directive: HlsManifestAcceptanceDirective, -) -> HlsAccessLease { - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let requests_before = fixture.origin.manifest_request_count(); - let mut refresh = regression_origin_refresh_request( - &fixture.app_state, - Arc::clone(&fixture.session), - &fixture.request_url, - Some(fixture.lease_id.clone()), - ); - refresh.acceptance_directive = directive; - refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; - assert!(trigger_origin_refresh_sync(refresh).await); - assert!(fixture.origin.manifest_request_count().saturating_sub(requests_before) >= plan.total_candidates()); - apply_terminal_cutover_pressure(fixture).await; - let bundle_key = prepared_terminal_bundle_key( - &fixture.asset, - fixture.base_manifest.target_duration_ms, - HLS_TERMINAL_TAIL_SEGMENT_COUNT, - ); - { - let session = fixture.session.read().await; - let episode = session.origin_control.acceptance_episode.as_ref().expect("exhausted acceptance episode"); - assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); - assert!(episode.full_burst_completed); - assert_eq!(episode.timing().required_terminal_media_key, Some(bundle_key)); - assert_eq!( - episode.timing().terminal_media_preparation, - HlsTerminalMediaPreparationState::Ready { key: bundle_key } - ); - assert!(episode.exhaustion_reason().is_some()); - } - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("pressured live lease"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - assert_eq!( - lease.playback_cursor.highest_contiguous_completed_proxy_seq, - Some(fixture.base_manifest.last_proxy_seq) - ); - { - let session = fixture.session.read().await; - let ready_after_tail = session - .segments - .values() - .filter(|segment| segment.proxy_seq > fixture.base_manifest.last_proxy_seq) - .filter_map(|segment| { - matches!(segment.status, SegmentCacheStatus::Ready { .. }) - .then_some((segment.proxy_seq, segment.duration_ms)) - }) - .collect::>(); - assert_eq!( - ready_after_tail, - vec![ - (fixture.base_manifest.last_proxy_seq.saturating_add(1), 1_000), - (fixture.base_manifest.last_proxy_seq.saturating_add(2), 2_800), - ] - ); - assert!(session.origin_control.path_condition.is_degraded()); - } - lease -} - -async fn commit_prepared_terminal_cutover( - fixture: &PreparedTerminalCutoverFixture, - pressured_lease: &HlsAccessLease, -) -> (u64, u64) { - let cutover_now_ms = pressured_lease - .playback_cursor - .first_segment_completed_at_ms - .expect("measured lease playback start") - .saturating_add(fixture.base_manifest.playlist_duration_ms); - let first = commit_terminal_tail_if_lease_reserve_requires_cutover( - &fixture.app_state.hls_ctx(), - &fixture.session, - &fixture.proxy_session_id, - pressured_lease, - cutover_now_ms, - ) - .await; - assert_eq!(first, HlsTerminalResolution::Committed); - let terminal_lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("terminal lease remains stored"); - let second = commit_terminal_tail_if_lease_reserve_requires_cutover( - &fixture.app_state.hls_ctx(), - &fixture.session, - &fixture.proxy_session_id, - &terminal_lease, - cutover_now_ms.saturating_add(1), - ) - .await; - assert_eq!(second, HlsTerminalResolution::Committed); - let HlsLeasePlaybackMode::TerminalTail(plan) = terminal_lease.playback_mode else { - panic!("prepared terminal tail must commit"); - }; - assert_eq!(plan.segment_count, HLS_TERMINAL_TAIL_SEGMENT_COUNT); - (plan.generation.0, cutover_now_ms) -} - -async fn assert_prepared_terminal_cutover_manifest(fixture: &PreparedTerminalCutoverFixture, generation: u64) { - let manifest_uri = format!("/hls/shared/live/{}/{}/manifest.m3u8", fixture.proxy_session_id.0, fixture.lease_id.0); - let response = get_response(Arc::clone(&fixture.app_state), &manifest_uri, None).await; - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("terminal manifest utf8"); - let live_tail_path = format!( - "/{}/{}/{:06}.ts", - fixture.proxy_session_id.0, fixture.lease_id.0, fixture.base_manifest.last_proxy_seq - ); - let terminal_prefix = format!("/{}/{}/terminal/{generation}/", fixture.proxy_session_id.0, fixture.lease_id.0); - assert!(body.contains(&live_tail_path)); - assert_eq!(body.matches("#EXT-X-DISCONTINUITY\n").count(), 1); - assert!(body.find(&live_tail_path) < body.find("#EXT-X-DISCONTINUITY\n")); - let key_reset = body.find("#EXT-X-KEY:METHOD=NONE\n").expect("AES-to-clear key reset"); - let discontinuity = body.find("#EXT-X-DISCONTINUITY\n").expect("terminal discontinuity"); - assert!(key_reset < discontinuity); - assert_eq!(body.matches(&terminal_prefix).count(), usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); - for index in 0..HLS_TERMINAL_TAIL_SEGMENT_COUNT { - assert!(body.contains(&format!("{terminal_prefix}{index}.ts"))); - } - let duration_ms = fixture.asset.duration_ms(); - let extinf = format!("#EXTINF:{}.{:03},", duration_ms / 1_000, duration_ms % 1_000); - assert_eq!(body.matches(&extinf).count(), usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); - assert!(body.ends_with("#EXT-X-ENDLIST\n")); - let renders_before = fixture.asset_buffer.finite_hls_render_count(); - let zero_uri = - format!("/hls/shared/live/{}/{}/terminal/{generation}/0.ts", fixture.proxy_session_id.0, fixture.lease_id.0); - let one_uri = - format!("/hls/shared/live/{}/{}/terminal/{generation}/1.ts", fixture.proxy_session_id.0, fixture.lease_id.0); - let zero = response_body(get_response(Arc::clone(&fixture.app_state), &zero_uri, None).await).await; - let one = response_body(get_response(Arc::clone(&fixture.app_state), &one_uri, None).await).await; - assert_ne!(zero, one); - assert_eq!(fixture.asset_buffer.finite_hls_render_count(), renders_before); -} - -async fn assert_terminal_cutover_sticky_after_recovery( - fixture: &PreparedTerminalCutoverFixture, - generation: u64, - cutover_now_ms: u64, -) { - { - let mut session = fixture.session.write().await; - let recovered_manifest = - normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:83\n#EXT-X-TARGETDURATION:12\n#EXTINF:12.0,\n83.ts\n"); - session.apply_origin_manifest(&recovered_manifest).expect("later shared-session recovery"); - session.origin_control.record_media_progress(cutover_now_ms.saturating_add(1), 12_000); - } - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("sticky terminal lease"); - assert!(matches!( - lease.playback_mode, - HlsLeasePlaybackMode::TerminalTail(ref plan) if plan.generation.0 == generation - )); -} - -#[tokio::test] -async fn commits_prepared_terminal_tail_once_when_recovery_misses_deadline() { - let fixture = prepared_terminal_cutover_fixture().await; - let directive = terminal_cutover_acceptance_directive(&fixture).await; - let pressured_lease = exhaust_terminal_cutover_recovery(&fixture, directive).await; - let (generation, cutover_now_ms) = commit_prepared_terminal_cutover(&fixture, &pressured_lease).await; - assert_prepared_terminal_cutover_manifest(&fixture, generation).await; - assert_terminal_cutover_sticky_after_recovery(&fixture, generation, cutover_now_ms).await; -} - -#[tokio::test] -async fn warm_fmp4_map_cutover_without_ready_reserve_fails_closed_without_a_ts_splice() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); - enable_channel_unavailable_custom_response(&app_state); - let proxy_session_id = ProxySessionId(map_hls_map(&app_state, b"fmp4-init", true).await); - let lease_id = HlsAccessLeaseId(format!("test-access-lease-{}", proxy_session_id.0)); - let now_ms = super::current_time_millis(); - let (base_proxy_seq, duration_ms) = { - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&proxy_session_id) - .await - .expect("fMP4 shared session"); - let session = session.read().await; - let entry = session.segments.values().next().expect("fMP4 media entry"); - (entry.proxy_seq, entry.duration_ms) - }; - let snapshot = HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(now_ms), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 0, - delivered_at_ms: now_ms, - first_proxy_seq: base_proxy_seq, - last_proxy_seq: base_proxy_seq, - visible_segments: Arc::from([HlsLeaseManifestSegment { - proxy_seq: base_proxy_seq, - duration_ms, - uri: format!("/iptv/hls/shared/live/{}/{}/{base_proxy_seq:06}.m4s", proxy_session_id.0, lease_id.0).into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }]), - discontinuity_sequence: 0, - target_duration_ms: terminal_test_asset().duration_ms().saturating_add(1_000), - playlist_duration_ms: duration_ms, - last_visible_media_end_ms: duration_ms, - active_map: Some(HlsMapSignature { fingerprint: [7; 32], container: HlsMediaContainer::FragmentedMp4 }), - active_encryption: None, - container: HlsMediaContainer::FragmentedMp4, - }; - publish_test_manifest_and_exhaust_configured_acceptance(&app_state, &proxy_session_id, &lease_id, snapshot, now_ms) - .await; - let manifest_uri = format!("/hls/shared/live/{}/{}/manifest.m3u8", proxy_session_id.0, lease_id.0); - - let response = get_response(Arc::clone(&app_state), &manifest_uri, None).await; - - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert!(!response.headers().contains_key(header::LOCATION)); - assert!(response_body(response).await.is_empty()); - let lease = app_state - .hls - .proxy - .access_lease_response_snapshot(&lease_id, &proxy_session_id, super::current_time_millis()) - .await - .expect("failed-closed lease snapshot"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - let terminal_uri = format!("/hls/shared/live/{}/{}/terminal/1/0.ts", proxy_session_id.0, lease_id.0); - let terminal_response = get_response(app_state, &terminal_uri, None).await; - assert_eq!(terminal_response.status(), StatusCode::NOT_FOUND); - assert!(!terminal_response.headers().contains_key(header::LOCATION)); - assert!(response_body(terminal_response).await.is_empty()); -} - -async fn terminal_head_content_length(app_state: &Arc, proxy_session_id: &str, segment_uri: &str) -> usize { - let head_response = request_response(Arc::clone(app_state), Method::HEAD, segment_uri, None).await; - assert_eq!(head_response.status(), StatusCode::OK); - assert_eq!(head_response.headers()[header::CONTENT_TYPE], "video/mp2t"); - assert_eq!(head_response.headers()[header::ACCEPT_RANGES], "bytes"); - assert!(head_response.headers()[header::CACHE_CONTROL].to_str().is_ok_and(|value| value.contains("immutable"))); - let content_length = head_response.headers()[header::CONTENT_LENGTH] - .to_str() - .expect("HEAD content length") - .parse::() - .expect("HEAD content length value"); - assert!(content_length > 0); - assert!(response_body(head_response).await.is_empty()); - assert_eq!(hls_session_last_media_at_ms(app_state, proxy_session_id).await, None); - assert_no_hls_cache_stream_registered(app_state).await; - - let head_range = request_response(Arc::clone(app_state), Method::HEAD, segment_uri, Some("bytes=0-187")).await; - assert_eq!(head_range.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(head_range.headers()[header::CONTENT_LENGTH], "188"); - let expected_content_range = format!("bytes 0-187/{content_length}"); - assert_eq!(head_range.headers()[header::CONTENT_RANGE].to_str().ok(), Some(expected_content_range.as_str())); - assert!(response_body(head_range).await.is_empty()); - assert_eq!(hls_session_last_media_at_ms(app_state, proxy_session_id).await, None); - assert_no_hls_cache_stream_registered(app_state).await; - content_length -} - -#[tokio::test] -async fn hls_terminal_response_serves_prepared_finite_full_and_range_bytes_per_index() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"original-live-tail").await; - let lease_id = format!("test-access-lease-{proxy_session_id}"); - let terminal_renderer = terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; - let renders_before_requests = terminal_renderer.finite_hls_render_count(); - let finalizations_before_requests = terminal_renderer.finite_hls_finalize_count(); - assert_eq!(renders_before_requests, usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); - assert_eq!(finalizations_before_requests, usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); - let (generation, _) = terminal_test_plan_shape(&app_state, &proxy_session_id, &lease_id).await; - let segment_zero_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/0.ts"); - let segment_one_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/1.ts"); - let repair_before = app_state.hls.proxy.segment_repair().stats().await; - let provider_connections_before = app_state.active_provider.get_provider_connections_count(); - - let head_content_length = terminal_head_content_length(&app_state, &proxy_session_id, &segment_zero_uri).await; - - let segment_zero_response = get_response(Arc::clone(&app_state), &segment_zero_uri, None).await; - assert_eq!(segment_zero_response.status(), StatusCode::OK); - assert!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await.is_some()); - assert!(!tuliprox_core::utils::response_compression::should_compress_response(&segment_zero_response)); - assert_eq!(segment_zero_response.headers()[header::CONTENT_TYPE], "video/mp2t"); - assert_eq!(segment_zero_response.headers()[header::ACCEPT_RANGES], "bytes"); - assert!(segment_zero_response.headers()[header::CACHE_CONTROL] - .to_str() - .is_ok_and(|value| value.contains("immutable"))); - let declared_length = segment_zero_response.headers()[header::CONTENT_LENGTH] - .to_str() - .expect("finite content length header") - .parse::() - .expect("finite content length value"); - assert_eq!(declared_length, head_content_length); - let segment_zero = response_body(segment_zero_response).await; - assert_eq!(segment_zero.len(), declared_length); - assert_hls_cache_stream_registered(&app_state, &proxy_session_id).await; - assert!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await.is_some()); - assert_eq!(app_state.hls.proxy.segment_repair().stats().await, repair_before); - assert_eq!(app_state.active_provider.get_provider_connections_count(), provider_connections_before); - - let segment_zero_again = response_body(get_response(Arc::clone(&app_state), &segment_zero_uri, None).await).await; - let segment_one = response_body(get_response(Arc::clone(&app_state), &segment_one_uri, None).await).await; - assert_eq!(segment_zero, segment_zero_again, "same terminal index is immutable"); - assert_ne!(segment_zero, segment_one, "successive terminal indices advance timestamps and continuity"); - - let range_response = get_response(Arc::clone(&app_state), &segment_zero_uri, Some("bytes=0-187")).await; - assert_eq!(range_response.status(), StatusCode::PARTIAL_CONTENT); - assert!(!tuliprox_core::utils::response_compression::should_compress_response(&range_response)); - assert_eq!(range_response.headers()[header::CONTENT_LENGTH], "188"); - let expected_content_range = format!("bytes 0-187/{declared_length}"); - assert_eq!(range_response.headers()[header::CONTENT_RANGE].to_str().ok(), Some(expected_content_range.as_str())); - assert_eq!(range_response.headers()[header::ACCEPT_RANGES], "bytes"); - assert!(range_response.headers()[header::CACHE_CONTROL].to_str().is_ok_and(|value| value.contains("immutable"))); - assert_eq!(response_body(range_response).await, segment_zero.slice(..188)); - - let unsatisfiable_range = format!("bytes={declared_length}-"); - let unsatisfiable_response = get_response(app_state, &segment_zero_uri, Some(&unsatisfiable_range)).await; - assert_eq!(unsatisfiable_response.status(), StatusCode::RANGE_NOT_SATISFIABLE); - assert!(!tuliprox_core::utils::response_compression::should_compress_response(&unsatisfiable_response)); - let expected_unsatisfied_content_range = format!("bytes */{declared_length}"); - assert_eq!( - unsatisfiable_response.headers()[header::CONTENT_RANGE].to_str().ok(), - Some(expected_unsatisfied_content_range.as_str()) - ); - assert!(response_body(unsatisfiable_response).await.is_empty()); - assert_eq!( - terminal_renderer.finite_hls_render_count(), - renders_before_requests, - "terminal HTTP serving must not invoke the TS writer again" - ); - assert_eq!( - terminal_renderer.finite_hls_finalize_count(), - finalizations_before_requests, - "terminal HTTP serving must not invoke lease-specific TS finalization again" - ); -} - -async fn assert_runtime_policy_terminal_segments( - fixture: &RuntimePolicyEndpointFixture, - terminal_prefix: &str, - segment_count: u16, -) { - for index in [0, 1, segment_count.saturating_sub(1)] { - let uri = format!("{terminal_prefix}{index}.ts"); - let response = get_response(Arc::clone(&fixture.app_state), &uri, None).await; - assert_eq!(response.status(), StatusCode::OK, "terminal segment {index}"); - assert!(!response_body(response).await.is_empty()); - } -} - -#[tokio::test] -async fn resource_access_denial_commits_and_preserves_user_exhausted_tail() { - let fixture = runtime_policy_endpoint_fixture(true).await; - let session = fixture - .app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&fixture.proxy_session_id) - .await - .expect("runtime policy session"); - let origin_refresh_before = session.read().await.origin_refresh.clone(); - mark_hls_user_session_exhausted(&fixture.app_state).await; - - let denied_live = get_response(Arc::clone(&fixture.app_state), &fixture.live_segment_uri, None).await; - assert_eq!(denied_live.status(), StatusCode::FORBIDDEN); - let revoking = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("revoking lease snapshot"); - assert_eq!(revoking.state, HlsAccessLeaseState::PolicyRevoking); - assert_eq!(revoking.playback_mode, HlsLeasePlaybackMode::Live); - assert_eq!( - revoking.runtime_policy_revocation.as_ref().map(|revocation| revocation.reason), - Some(HlsRuntimeCustomTailReason::UserConnectionsExhausted) - ); - - let pending_manifest = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; - assert_eq!(pending_manifest.status(), StatusCode::SERVICE_UNAVAILABLE); - let committed_plan = wait_for_runtime_policy_terminal_plan(&fixture).await; - let manifest_response = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; - assert_eq!(manifest_response.status(), StatusCode::OK); - assert!(!manifest_response.headers().contains_key(header::LOCATION)); - let manifest = String::from_utf8(response_body(manifest_response).await.to_vec()) - .expect("runtime policy terminal manifest utf8"); - assert!(manifest.ends_with("#EXT-X-ENDLIST\n")); - assert!(!manifest.contains("/cvs/hls/")); - - let committed = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("committed runtime policy lease"); - assert_eq!(committed.state, HlsAccessLeaseState::Denied); - let HlsLeasePlaybackMode::TerminalTail(plan) = committed.playback_mode else { - panic!("resource denial must commit a finite terminal plan"); - }; - assert_eq!(plan.generation, committed_plan.generation); - assert_eq!(plan.reason, HlsRuntimeCustomTailReason::UserConnectionsExhausted); - assert!(plan.segment_count >= 2); - let terminal_prefix = format!( - "/hls/shared/live/{}/{}/terminal/{}/", - fixture.proxy_session_id.0, fixture.lease_id.0, plan.generation.0 - ); - assert_eq!(manifest.matches(&terminal_prefix).count(), usize::from(plan.segment_count)); - - assert_runtime_policy_terminal_segments(&fixture, &terminal_prefix, plan.segment_count).await; - let replay = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; - assert_eq!(replay.status(), StatusCode::OK); - assert_eq!( - String::from_utf8(response_body(replay).await.to_vec()).expect("replayed runtime policy manifest utf8"), - manifest - ); - assert_eq!(get_status(Arc::clone(&fixture.app_state), &fixture.live_segment_uri).await, StatusCode::FORBIDDEN); - let retained = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("retained runtime policy plan"); - assert!(matches!( - retained.playback_mode, - HlsLeasePlaybackMode::TerminalTail(ref current) - if current.generation == plan.generation - && current.reason == HlsRuntimeCustomTailReason::UserConnectionsExhausted - )); - assert_eq!(session.read().await.origin_refresh, origin_refresh_before); -} - -#[tokio::test] -async fn manifest_touch_denied_replays_policy_tail_instead_of_standalone_clock() { - let fixture = runtime_policy_endpoint_fixture(true).await; - let _ = fixture - .app_state - .hls - .proxy - .begin_runtime_policy_revocation( - &fixture.lease_id, - &fixture.proxy_session_id, - HlsRuntimeCustomTailReason::UserConnectionsExhausted, - super::current_time_millis(), - ) - .await; - assert_eq!( - fixture - .app_state - .hls - .proxy - .touch_manifest_access_lease( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - None, - None, - 60_000, - ) - .await, - HlsAccessLeaseTouch::Denied - ); - - let pending = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; - assert_eq!(pending.status(), StatusCode::SERVICE_UNAVAILABLE); - let committed_plan = wait_for_runtime_policy_terminal_plan(&fixture).await; - let response = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; - assert_eq!(response.status(), StatusCode::OK); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("policy touch manifest utf8"); - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("touch-denied lease"); - let HlsLeasePlaybackMode::TerminalTail(plan) = lease.playback_mode else { - panic!("touch denial must retain a lease-bound terminal plan"); - }; - assert_eq!(plan.generation, committed_plan.generation); - assert!(body.contains(&format!( - "/hls/shared/live/{}/{}/terminal/{}/0.ts", - fixture.proxy_session_id.0, fixture.lease_id.0, plan.generation.0 - ))); - assert!(!body.contains("/cvs/hls/")); - assert!(body.ends_with("#EXT-X-ENDLIST\n")); -} - -#[tokio::test] -async fn cold_user_denial_uses_standalone_finite_response() { - let fixture = runtime_policy_endpoint_fixture(false).await; - mark_hls_user_session_exhausted(&fixture.app_state).await; - - let denial = get_response(Arc::clone(&fixture.app_state), &fixture.live_segment_uri, None).await; - assert_eq!(denial.status(), StatusCode::FORBIDDEN); - let denied = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("cold denied lease"); - assert_eq!(denied.state, HlsAccessLeaseState::Denied); - assert_eq!(denied.playback_mode, HlsLeasePlaybackMode::Ended); - assert!(denied.runtime_policy_revocation.is_none()); - assert_eq!(denied.runtime_policy_denial_reason(), Some(HlsRuntimeCustomTailReason::UserConnectionsExhausted)); - - let response = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; - assert_eq!(response.status(), StatusCode::OK); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("standalone policy manifest utf8"); - assert!(body.contains(&format!("/cvs/hls/{}/", fixture.lease_id.0))); - assert!(!body.contains("/hls-user/")); - assert!(!body.contains("/hls-pass/")); - assert!(!body.contains("/user_connections_exhausted/")); - assert!( - !body.contains(&format!("/hls/shared/live/{}/{}/terminal/", fixture.proxy_session_id.0, fixture.lease_id.0)) - ); - assert!(body.ends_with("#EXT-X-ENDLIST\n")); -} - -#[tokio::test] -async fn hls_terminal_response_body_after_lease_denial_does_not_extend_shared_session_activity() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"original-live-tail").await; - let lease_id = format!("test-access-lease-{proxy_session_id}"); - terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; - let (generation, _) = terminal_test_plan_shape(&app_state, &proxy_session_id, &lease_id).await; - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) - .await - .expect("terminal session"); - let segment_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/0.ts"); - let response = get_response(Arc::clone(&app_state), &segment_uri, None).await; - assert_eq!(response.status(), StatusCode::OK); - let activity_after_authorized_response = - session.read().await.activity.last_authorized_media_at_ms.expect("terminal GET marks media activity"); - - let _ = app_state - .hls - .proxy - .deny_access_lease( - &HlsAccessLeaseId(lease_id), - tuliprox_hls::HlsAccessLeaseDenialMode::PreserveCommittedFiniteTail, - ) - .await; - assert!(!response_body(response).await.is_empty()); - - assert_eq!( - session.read().await.activity.last_authorized_media_at_ms, - Some(activity_after_authorized_response), - "body completion after denial must not extend activity" - ); -} - -#[tokio::test] -async fn hls_terminal_response_rejects_stale_malformed_and_out_of_bounds_paths() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"original-live-tail").await; - let lease_id = format!("test-access-lease-{proxy_session_id}"); - terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; - let (generation, segment_count) = terminal_test_plan_shape(&app_state, &proxy_session_id, &lease_id).await; - let stale_generation = generation.saturating_add(1); - let stale_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{stale_generation}/0.ts"); - let out_of_bounds_uri = - format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/{segment_count}.ts"); - - let malformed_generation = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/01/0.ts"); - let non_numeric_generation = - format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/not-a-generation/0.ts"); - let overflowing_generation = - format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/18446744073709551616/0.ts"); - let malformed_file = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/00.ts"); - let wrong_extension = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/0.m4s"); - - for uri in [ - stale_uri, - out_of_bounds_uri, - malformed_generation, - non_numeric_generation, - overflowing_generation, - malformed_file, - wrong_extension, - ] { - let response = get_response(Arc::clone(&app_state), &uri, None).await; - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert!(!response.headers().contains_key(header::LOCATION)); - } -} - -#[tokio::test] -async fn expired_route_replays_already_committed_custom_tail() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"original-live-tail").await; - let lease_id = format!("test-access-lease-{proxy_session_id}"); - terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; - let (generation, _) = terminal_test_plan_shape(&app_state, &proxy_session_id, &lease_id).await; - let proxy_session_key = ProxySessionId(proxy_session_id.clone()); - let lease_key = HlsAccessLeaseId(lease_id.clone()); - let expired_at_ms = super::current_time_millis().saturating_sub(1); - { - let mut leases = app_state.hls.proxy.access_leases().write().await; - let mut lease = leases.remove_access_lease(&lease_key).expect("terminal lease exists"); - lease.valid_until_ms = expired_at_ms; - leases.prepare_access_lease(lease); - } - let manifest_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/manifest.m3u8"); - let segment_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/0.ts"); - - let manifest_response = get_response(Arc::clone(&app_state), &manifest_uri, None).await; - let segment_response = get_response(Arc::clone(&app_state), &segment_uri, None).await; - - assert_eq!(manifest_response.status(), StatusCode::OK); - assert!(!manifest_response.headers().contains_key(header::LOCATION)); - assert!(String::from_utf8(response_body(manifest_response).await.to_vec()) - .expect("expired committed manifest utf8") - .ends_with("#EXT-X-ENDLIST\n")); - assert_eq!(segment_response.status(), StatusCode::OK); - assert!(!response_body(segment_response).await.is_empty()); - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&proxy_session_key) - .await - .expect("terminal session exists"); - assert!(session.read().await.terminal_tail_protection(&lease_key).is_some()); - - let cleanup_at_ms = super::current_time_millis(); - app_state - .hls - .proxy - .handle_lifecycle_event( - &app_state.active_users, - &app_state.active_provider, - HlsLifecycleEvent { - key: HlsLifecycleEventKey::AccessLeaseValidity { - lease_id: lease_key.clone(), - proxy_session_id: proxy_session_key, - }, - due_at_ms: cleanup_at_ms, - }, - cleanup_at_ms, - ) - .await; - - assert!(!session.read().await.has_terminal_tail_protections()); - assert!(app_state - .hls - .proxy - .access_lease_response_snapshot(&lease_key, &ProxySessionId(proxy_session_id), cleanup_at_ms) - .await - .is_none()); - assert_eq!(get_response(Arc::clone(&app_state), &manifest_uri, None).await.status(), StatusCode::NOT_FOUND); - assert_eq!(get_response(app_state, &segment_uri, None).await.status(), StatusCode::NOT_FOUND); -} - -#[tokio::test] -async fn expired_route_without_session_evidence_returns_not_found_instead_of_unanchored_manifest() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"live-without-lease").await; - enable_channel_unavailable_custom_response(&app_state); - let missing_lease = "expired-without-base-evidence"; - let manifest_uri = format!("/hls/shared/live/{proxy_session_id}/{missing_lease}/manifest.m3u8"); - - let response = get_response(app_state, &manifest_uri, None).await; - - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert!(!response.headers().contains_key(header::LOCATION)); - assert!(response_body(response).await.is_empty()); -} - -#[tokio::test] -async fn hls_terminal_response_normal_segment_map_and_resource_routes_never_serve_terminal_fallbacks() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"original-live-tail").await; - let lease_id = format!("test-access-lease-{proxy_session_id}"); - terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; - let normal_segment_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/000123.ts"); - - let normal_segment_response = get_response(Arc::clone(&app_state), &normal_segment_uri, None).await; - - assert_eq!(normal_segment_response.status(), StatusCode::OK); - assert!(!normal_segment_response.headers().contains_key(header::LOCATION)); - assert_eq!(response_body(normal_segment_response).await, bytes::Bytes::from_static(b"original-live-tail")); - - let map_temp_dir = tempfile::tempdir().expect("map tempdir"); - let map_app_state = - test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(map_temp_dir.path(), 300))); - let map_proxy_session_id = map_hls_map(&map_app_state, b"original-map", true).await; - let map_lease_id = format!("test-access-lease-{map_proxy_session_id}"); - let map_session = map_app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&ProxySessionId(map_proxy_session_id.clone())) - .await - .expect("map session exists"); - let map_base_proxy_seq = *map_session.read().await.segments.keys().next().expect("map manifest has media"); - terminalize_existing_test_lease(&map_app_state, &map_proxy_session_id, &map_lease_id, map_base_proxy_seq).await; - let map_uri = format!("/hls/shared/live/{map_proxy_session_id}/{map_lease_id}/map/000000.mp4"); - let map_response = get_response(map_app_state, &map_uri, None).await; - assert_eq!(map_response.status(), StatusCode::NOT_FOUND); - assert!(!map_response.headers().contains_key(header::LOCATION)); - assert!(response_body(map_response).await.is_empty()); - - let resource_temp_dir = tempfile::tempdir().expect("resource tempdir"); - let resource_app_state = - test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(resource_temp_dir.path(), 300))); - let (resource_proxy_session_id, resource_id) = - map_transient_resource(&resource_app_state, "http://origin.example.com/old.ts", "ts", true).await; - let resource_lease_id = format!("test-access-lease-{resource_proxy_session_id}"); - terminalize_existing_test_lease(&resource_app_state, &resource_proxy_session_id, &resource_lease_id, 0).await; - let resource_uri = format!("/hls/shared/live/{resource_proxy_session_id}/{resource_lease_id}/r/{resource_id}.ts"); - let resource_response = get_response(resource_app_state, &resource_uri, None).await; - assert_eq!(resource_response.status(), StatusCode::NOT_FOUND); - assert!(!resource_response.headers().contains_key(header::LOCATION)); - assert!(response_body(resource_response).await.is_empty()); -} - -#[tokio::test] -async fn delayed_live_resource_completion_revalidates_after_terminal_cutover_without_a_sleep() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"ready-before-cutover").await; - let lease_id = HlsAccessLeaseId(format!("test-access-lease-{proxy_session_id}")); - let proxy_session_key = ProxySessionId(proxy_session_id.clone()); - let access_context = test_hls_access_context(proxy_session_key.clone(), lease_id.clone()); - let live_identity = app_state - .hls - .proxy - .access_lease_response_snapshot(&lease_id, &proxy_session_key, super::current_time_millis()) - .await - .and_then(|lease| lease.media_identity()) - .expect("live lease identity"); - let (release_sender, release_receiver) = tokio::sync::oneshot::channel(); - let app_state_for_completion = Arc::clone(&app_state); - let access_context_for_completion = access_context.clone(); - let completion = tokio::spawn(async move { - let _ = release_receiver.await; - super::hls_live_lease_identity_is_current( - &app_state_for_completion, - &access_context_for_completion, - live_identity, - ) - .await - }); - - terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id.0, 123).await; - release_sender.send(()).expect("release delayed completion after cutover"); - - assert!(!completion.await.expect("controlled completion task")); -} - -async fn prepare_other_live_lease( - app_state: &Arc, - proxy_session_id: &ProxySessionId, - now_ms: u64, -) -> HlsAccessLeaseId { - let lease_id = HlsAccessLeaseId("other-live-lease".to_string()); - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), - proxy_session_id.clone(), - "hls-user".to_string(), - "hls-session-token".to_string(), - 1, - "12345".to_string(), - 12345, - now_ms, - 60_000, - )) - .await; - lease_id -} - -fn assert_terminal_plan_unchanged( - playback_mode: &HlsLeasePlaybackMode, - expected_generation: HlsTerminalTailGeneration, - terminal_path: HlsTerminalSegmentPath, - expected_bytes: &bytes::Bytes, -) { - let HlsLeasePlaybackMode::TerminalTail(plan) = playback_mode else { - panic!("shared lease operation cannot reactivate the terminal lease"); - }; - assert_eq!(plan.generation, expected_generation); - assert_eq!(plan.segment_bytes(terminal_path).as_ref(), Some(expected_bytes)); -} - -async fn assert_conflicted_standalone_fallback( - app_state: &Arc, - session: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, -) { - let response = - super::hls_unpublished_lease_channel_unavailable_response(app_state, proxy_session_id, lease_id).await; - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - assert!(!response.headers().contains_key(header::RETRY_AFTER)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("standalone manifest utf8"); - assert!(body.contains("#EXT-X-ENDLIST")); - assert!( - !body.contains("/hls/shared/live/"), - "standalone response must not expose a normal segment without a readiness path" - ); - assert_eq!( - session.read().await.origin_control.path_condition, - HlsOriginPathCondition::AcceptanceConflict, - "lease-local fallback must not relax deterministic conflict evidence" - ); -} - -#[tokio::test] -async fn reused_conflicted_session_standalone_fallback_preserves_terminal_lease_during_recovery() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); - enable_channel_unavailable_custom_response(&app_state); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"original-live-tail").await; - let terminal_lease_id = format!("test-access-lease-{proxy_session_id}"); - terminalize_existing_test_lease(&app_state, &proxy_session_id, &terminal_lease_id, 123).await; - let proxy_session_key = ProxySessionId(proxy_session_id.clone()); - let now_ms = super::current_time_millis(); - let terminal_before = app_state - .hls - .proxy - .access_lease_response_snapshot(&HlsAccessLeaseId(terminal_lease_id.clone()), &proxy_session_key, now_ms) - .await - .expect("terminal lease exists before the other lease"); - let HlsLeasePlaybackMode::TerminalTail(terminal_plan_before) = terminal_before.playback_mode else { - panic!("original lease is terminal"); - }; - let terminal_path = HlsTerminalSegmentPath { generation: terminal_plan_before.generation, index: 0 }; - let terminal_bytes_before = - terminal_plan_before.segment_bytes(terminal_path).expect("terminal segment zero is immutable"); - let other_lease_id = prepare_other_live_lease(&app_state, &proxy_session_key, now_ms).await; - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&proxy_session_key) - .await - .expect("shared session exists"); - session.write().await.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict; - - assert_conflicted_standalone_fallback(&app_state, &session, &proxy_session_key, &other_lease_id).await; - - let terminal_after_fallback = app_state - .hls - .proxy - .access_lease_response_snapshot( - &HlsAccessLeaseId(terminal_lease_id.clone()), - &proxy_session_key, - now_ms.saturating_add(1), - ) - .await - .expect("terminal lease remains stored after standalone fallback"); - assert_terminal_plan_unchanged( - &terminal_after_fallback.playback_mode, - terminal_plan_before.generation, - terminal_path, - &terminal_bytes_before, - ); - - assert!(app_state - .hls - .proxy - .activate_access_lease( - &other_lease_id, - &proxy_session_key, - now_ms, - HlsAccessLeaseTiming { active_window_ms: 5_000, valid_window_ms: 60_000 }, - ) - .await - .is_activated()); - - let recovered_manifest = - normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:124\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n124.ts\n"); - { - let mut session = session.write().await; - session.apply_origin_manifest(&recovered_manifest).expect("recovery manifest commits"); - session.origin_control.record_media_progress(now_ms.saturating_add(1), 4_000); - } - - let terminal = app_state - .hls - .proxy - .access_lease_response_snapshot( - &HlsAccessLeaseId(terminal_lease_id), - &proxy_session_key, - now_ms.saturating_add(2), - ) - .await - .expect("terminal lease remains stored"); - let other = app_state - .hls - .proxy - .access_lease_response_snapshot(&other_lease_id, &proxy_session_key, now_ms.saturating_add(2)) - .await - .expect("other live lease remains stored"); - assert_terminal_plan_unchanged( - &terminal.playback_mode, - terminal_plan_before.generation, - terminal_path, - &terminal_bytes_before, - ); - assert_eq!(other.playback_mode, HlsLeasePlaybackMode::Live); -} - -#[test] -fn hls_custom_video_manifest_uses_live_six_segment_window_for_provisioning() { - let user = hls_custom_video_test_user(); - let manifest = - build_hls_custom_video_manifest_body("https://example.test/iptv", &user, CustomVideoStreamType::Provisioning) - .expect("provisioning manifest does not depend on one looping asset duration"); - - assert!(manifest.contains("#EXT-X-TARGETDURATION:2")); - assert!(manifest.contains("#EXT-X-MEDIA-SEQUENCE:0")); - assert!(manifest.contains("#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-INDEPENDENT-SEGMENTS\n")); - assert!(!manifest.contains("#EXT-X-DISCONTINUITY-SEQUENCE")); - assert!(!manifest.contains("#EXT-X-SESSION-DATA")); - assert!(!manifest.contains("#EXT-X-ENDLIST")); - assert!(!manifest.contains("#EXT-X-DISCONTINUITY\n")); - assert_eq!(manifest.matches("#EXTINF:2.000000,").count(), 6); - for index in 0..6 { - assert!( - manifest.contains(&format!("https://example.test/iptv/cvs/hls/viewer/secret/provisioning_{index:03}.ts")) - ); - } - assert!(!manifest.contains("https://example.test/iptv/cvs/hls/viewer/secret/provisioning.ts")); - assert_eq!( - crate::api::model::hls_panel_provisioning_manifest_path(&user, 80510), - "/cvs/hls/viewer/secret/provisioning.m3u8?id=80510" - ); - assert!(!manifest.contains("provisioning.ts?")); - assert!(!manifest.contains("virtual_id")); - assert!(!manifest.contains("&seq=")); -} - -#[test] -fn hls_response_uses_rfc8216_content_type_and_remains_tower_compressible() { - let response = super::hls_response("#EXTM3U\n".to_string()).into_response(); - - assert_eq!(response.headers().get(header::CONTENT_TYPE).unwrap(), "application/vnd.apple.mpegurl"); - assert!(tuliprox_core::utils::response_compression::should_compress_response(&response)); -} - -#[test] -fn virtual_hls_entry_path_uses_single_manifest_extension() { - let user = hls_custom_video_test_user(); - let xtream_target = ConfigTarget::from(&shared::model::ConfigTargetDto { - output: vec![shared::model::TargetOutputDto::Xtream(shared::model::XtreamTargetOutputDto::default())], - ..Default::default() - }); - let xtream_input = ConfigInput { input_type: InputType::Xtream, ..ConfigInput::default() }; - let m3u_target = ConfigTarget::from(&shared::model::ConfigTargetDto::default()); - let m3u_input = ConfigInput { input_type: InputType::M3u, ..ConfigInput::default() }; - - let xtream_path = super::build_virtual_hls_entry_path(&xtream_target, &xtream_input, &user, 59); - let m3u_path = super::build_virtual_hls_entry_path(&m3u_target, &m3u_input, &user, 59); - - assert_eq!(xtream_path, "/live/viewer/secret/59.m3u8"); - assert_eq!(m3u_path, "/m3u-stream/live/viewer/secret/59.m3u8"); - assert!(!xtream_path.contains("..m3u8")); - assert!(!m3u_path.contains("..m3u8")); -} - -#[test] -fn hls_manifest_headers_apply_disabled_headers_and_default_user_agent_policy() { - let mut input_headers = HashMap::new(); - input_headers.insert("User-Agent".to_string(), "Input-UA".to_string()); - input_headers.insert("Accept-Language".to_string(), "de".to_string()); - input_headers.insert("Accept-Encoding".to_string(), "gzip".to_string()); - input_headers.insert("Authorization".to_string(), "Bearer input-secret".to_string()); - input_headers.insert("X-Origin-Secret".to_string(), "input-secret".to_string()); - - let mut request_headers = HeaderMap::new(); - request_headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-")); - request_headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); - request_headers.insert(header::USER_AGENT, HeaderValue::from_static("Client-UA")); - request_headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer client-secret")); - request_headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); - request_headers.insert(HeaderName::from_static("proxy-authorization"), HeaderValue::from_static("Basic secret")); - request_headers.insert(header::HOST, HeaderValue::from_static("proxy.example.com")); - request_headers.insert(HeaderName::from_static("x-blocked"), HeaderValue::from_static("client")); - request_headers.insert(HeaderName::from_static("cf-ray"), HeaderValue::from_static("cf")); - - let disabled = ReverseProxyDisabledHeaderConfig { - referer_header: false, - x_header: true, - cloudflare_header: true, - custom_header: vec!["X-Origin-Secret".to_string()], - }; - let headers = build_hls_manifest_request_headers( - &input_headers, - &request_headers, - Some(&disabled), - Some("Default-UA"), - Some("Channel-UA"), - ); - - assert_eq!(headers.get(header::USER_AGENT).expect("user agent"), "Channel-UA"); - assert_eq!(headers.get("accept-language").expect("accept language"), "de"); - assert_eq!(headers.get(header::ACCEPT_ENCODING).expect("accept encoding"), "identity"); - assert!(!headers.contains_key(header::RANGE)); - assert!(!headers.contains_key(header::AUTHORIZATION)); - assert!(!headers.contains_key(header::COOKIE)); - assert!(!headers.contains_key("proxy-authorization")); - assert!(!headers.contains_key(header::HOST)); - assert!(!headers.contains_key("x-origin-secret")); - assert!(!headers.contains_key("x-blocked")); - assert!(!headers.contains_key("cf-ray")); -} - -#[tokio::test] -async fn hls_user_agent_stream_index_is_stable_for_the_session() { - let app_state = test_app_state(); - let session = Arc::new(tokio::sync::RwLock::new(HlsSession::new(HlsSessionKey::new(1, "stream-1"), b"secret", 0))); - let mut manifest_headers = HeaderMap::new(); - manifest_headers.insert(header::USER_AGENT, HeaderValue::from_static("VLC/3.0")); - - apply_hls_user_agent_stream_index(&session, &mut manifest_headers, true, &app_state.active_users).await; - let stream_index = session.read().await.user_agent_stream_index.unwrap_or_default(); - assert_ne!(stream_index, 0); - assert_eq!( - manifest_headers.get(header::USER_AGENT).and_then(|value| value.to_str().ok()), - Some(format!("VLC/3.0 {stream_index}").as_str()) - ); - - let mut segment_headers = HeaderMap::new(); - segment_headers.insert(header::USER_AGENT, HeaderValue::from_static("VLC/3.0")); - apply_hls_user_agent_stream_index(&session, &mut segment_headers, true, &app_state.active_users).await; - - assert_eq!( - segment_headers.get(header::USER_AGENT).and_then(|value| value.to_str().ok()), - Some(format!("VLC/3.0 {stream_index}").as_str()) - ); - assert_eq!(session.read().await.user_agent_stream_index, Some(stream_index)); -} - -#[tokio::test] -async fn legacy_hls_manifest_decodes_supported_origin_codings_and_enforces_identity() { - const MANIFEST: &str = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nsegment.ts\n"; - - for coding in ["gzip", "deflate", "br", "zstd"] { - let encoded = encode_test_manifest(coding, MANIFEST.as_bytes()).await; - let origin = spawn_test_encoded_manifest_origin(Some(coding), encoded, Duration::ZERO).await; - let input = legacy_manifest_test_input(&origin); - let client_headers = legacy_manifest_test_client_headers(); - - let (manifest, final_url, _) = super::download_legacy_hls_manifest(&test_app_state(), &input, &client_headers) - .await - .unwrap_or_else(|error| panic!("{coding} manifest should decode: {error}")); - - assert_eq!(manifest, MANIFEST, "coding={coding}"); - assert_eq!(final_url, input.url, "coding={coding}"); - let requests = origin.requests.lock().await; - assert_eq!(requests.len(), 1, "coding={coding}"); - assert!( - requests[0].to_ascii_lowercase().contains("\r\naccept-encoding: identity\r\n"), - "coding={coding}, request={}", - requests[0] - ); - } -} - -#[tokio::test] -async fn legacy_hls_manifest_handles_identity_and_headerless_gzip_magic() { - const MANIFEST: &[u8] = b"#EXTM3U\n#EXT-X-TARGETDURATION:4\n"; - let cases = [("identity", MANIFEST.to_vec()), ("gzip-magic", encode_test_manifest("gzip", MANIFEST).await)]; - - for (case, body) in cases { - let origin = spawn_test_encoded_manifest_origin(None, body, Duration::ZERO).await; - let input = legacy_manifest_test_input(&origin); - - let (manifest, _, _) = - super::download_legacy_hls_manifest(&test_app_state(), &input, &legacy_manifest_test_client_headers()) - .await - .unwrap_or_else(|error| panic!("{case} manifest should decode: {error}")); - - assert_eq!(manifest.as_bytes(), MANIFEST, "case={case}"); - } -} - -#[tokio::test] -async fn legacy_hls_manifest_limit_applies_after_decompression() { - let decoded = vec![b'x'; MAX_HLS_MANIFEST_BYTES + 1]; - let origin = - spawn_test_encoded_manifest_origin(Some("gzip"), encode_test_manifest("gzip", &decoded).await, Duration::ZERO) - .await; - let input = legacy_manifest_test_input(&origin); - - let error = super::download_legacy_hls_manifest(&test_app_state(), &input, &legacy_manifest_test_client_headers()) - .await - .expect_err("decoded manifest above limit must fail"); - - assert!(matches!( - error.get_ref().and_then(|source| source.downcast_ref()), - Some(crate::utils::content_coding::ContentBodyReadError::LimitExceeded { limit }) - if *limit == MAX_HLS_MANIFEST_BYTES - )); -} - -#[tokio::test] -async fn legacy_hls_manifest_deadline_includes_full_body_read() { - let origin = spawn_test_encoded_manifest_origin(None, b"#EXTM3U\n".to_vec(), Duration::from_millis(100)).await; - let input = legacy_manifest_test_input(&origin); - let hls_config = HlsCacheConfig::from(&HlsCacheConfigDto { - origin_manifest_timeout_ms: shared::model::Millis::new(10), - ..Default::default() - }); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); - - let error = super::download_legacy_hls_manifest(&app_state, &input, &legacy_manifest_test_client_headers()) - .await - .expect_err("complete manifest body read must honor the deadline"); - - assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); -} - -#[tokio::test] -async fn legacy_hls_manifest_distinguishes_invalid_utf8_from_decoder_failure() { - let invalid_utf8_origin = spawn_test_encoded_manifest_origin(None, vec![0xff], Duration::ZERO).await; - let invalid_utf8_input = legacy_manifest_test_input(&invalid_utf8_origin); - let invalid_utf8 = super::download_legacy_hls_manifest( - &test_app_state(), - &invalid_utf8_input, - &legacy_manifest_test_client_headers(), - ) - .await - .expect_err("invalid UTF-8 must fail"); - - let corrupt_origin = - spawn_test_encoded_manifest_origin(Some("gzip"), vec![0x1f, 0x8b, 0x08, 0x00], Duration::ZERO).await; - let corrupt_input = legacy_manifest_test_input(&corrupt_origin); - let decoder_failure = - super::download_legacy_hls_manifest(&test_app_state(), &corrupt_input, &legacy_manifest_test_client_headers()) - .await - .expect_err("corrupt gzip must fail"); - - assert_eq!(invalid_utf8.kind(), std::io::ErrorKind::InvalidData); - assert!(crate::utils::content_coding::content_decoding_error_from_io(&decoder_failure).is_some()); -} - -#[test] -fn hls_proxy_public_path_prefix_rewrites_only_proxy_hls_uri_surfaces() { - let body = concat!( - "#EXTM3U\n", - "#EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/proxy-id/r/key.key\",IV=0x1\n", - "#EXT-X-MAP:URI=\"/hls/shared/live/proxy-id/map/000000.mp4\",BYTERANGE=\"10@0\"\n", - "#EXT-X-PART:DURATION=1.0,URI=\"/hls/shared/live/proxy-id/r/part.m4s\"\n", - "#EXT-X-MEDIA-SEQUENCE:7\n", - "#EXTINF:4.0,\n", - "/hls/shared/live/proxy-id/000007.ts\n", - "https://origin.example.com/not-proxy.ts\n", - ); - - let prefixed = super::apply_hls_proxy_public_path_prefix(body.to_string(), Some("/iptv/")); - - assert!(prefixed.contains("URI=\"/iptv/hls/shared/live/proxy-id/r/key.key\"")); - assert!(prefixed.contains("URI=\"/iptv/hls/shared/live/proxy-id/map/000000.mp4\"")); - assert!(prefixed.contains("URI=\"/iptv/hls/shared/live/proxy-id/r/part.m4s\"")); - assert!(prefixed.contains("\n/iptv/hls/shared/live/proxy-id/000007.ts\n")); - assert!(prefixed.contains("#EXT-X-MEDIA-SEQUENCE:7")); - assert!(prefixed.contains("https://origin.example.com/not-proxy.ts")); -} - -#[test] -fn hls_proxy_public_path_prefix_keeps_body_unchanged_without_server_path() { - let body = "#EXTM3U\n#EXTINF:4.0,\n/hls/shared/live/proxy-id/000007.ts\n".to_string(); - - assert_eq!(super::apply_hls_proxy_public_path_prefix(body.clone(), None), body); - assert_eq!(super::apply_hls_proxy_public_path_prefix(body.clone(), Some("/")), body); -} - -#[test] -fn hls_manifest_materialization_uses_proxy_paths_without_provider_or_legacy_route() { - let body = format!( - "#EXTM3U\n#EXTINF:4.0,\n/hls/shared/live/proxy-id/{}/000123.ts\n", - crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER - ); - let lease_id = HlsAccessLeaseId("access-lease".to_string()); - - let materialized = super::materialize_hls_access_manifest(&body, &lease_id, Some("/iptv")); - - assert!(materialized.contains("/iptv/hls/shared/live/proxy-id/access-lease/000123.ts")); - assert!(!materialized.contains("provider://")); - assert!(!materialized.contains("/hls/hls-user/")); - assert!(!materialized.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); -} - -#[test] -fn hls_cache_origin_entry_url_preserves_provider_scheme_as_failover() { - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "demo".into(), - urls: vec!["http://origin.example.com".into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, - dns: None, - })); - let input = ConfigInput { provider_configs: Some(vec![Arc::clone(&provider)]), ..ConfigInput::default() }; - - let origin = - super::resolve_hls_cache_origin_entry_url(&input, "provider://demo/live/account-a/token-a/1025130.m3u8") - .expect("provider entry url should resolve"); - - assert_eq!(origin.session_entry_url.as_str(), "provider://demo/live/account-a/token-a/1025130.m3u8"); - assert_eq!( - origin.session_entry_url.url_failover_provider().expect("provider failover config").name.as_ref(), - "demo" - ); - let provider_key = super::build_hls_origin_source(&input, "1025130").session_key(); - let direct_key = super::build_hls_origin_source(&input, "1025130").session_key(); - assert_eq!(provider_key, direct_key); - assert_eq!(provider_key.stable_value(), "input:0|hls|1025130"); - assert!(!provider_key.stable_value().contains("provider://")); - assert!(!provider_key.stable_value().contains("origin.example.com")); -} - -#[test] -fn hls_cache_origin_entry_url_does_not_attach_url_failover_provider_to_http_url() { - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "demo".into(), - urls: vec!["http://origin.example.com".into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, - dns: None, - })); - let input = ConfigInput { provider_configs: Some(vec![provider]), ..ConfigInput::default() }; - - let origin = super::resolve_hls_cache_origin_entry_url( - &input, - "http://origin.example.com/live/account-a/token-a/1025130.m3u8", - ) - .expect("http entry url should resolve"); - - assert_eq!(origin.session_entry_url.as_str(), "http://origin.example.com/live/account-a/token-a/1025130.m3u8"); - assert!(origin.session_entry_url.url_failover_provider().is_none()); -} - -#[test] -fn hls_origin_source_kind_covers_xtream_m3u_and_direct_media_playlist() { - assert_eq!(super::hls_origin_source_kind(InputType::Xtream), HlsOriginSourceKind::XtreamLive); - assert_eq!(super::hls_origin_source_kind(InputType::M3u), HlsOriginSourceKind::M3uMediaPlaylist); - assert_eq!(super::hls_origin_source_kind(InputType::Library), HlsOriginSourceKind::DirectMediaPlaylist); -} - -#[test] -fn hls_manifest_extension_helper_does_not_create_double_dot_urls() { - assert_eq!( - super::ensure_hls_manifest_extension("http://origin.example.com/live/user/pass/1025123.m3u8"), - "http://origin.example.com/live/user/pass/1025123.m3u8" - ); - assert_eq!( - super::ensure_hls_manifest_extension("http://origin.example.com/live/user/pass/1025123..m3u8"), - "http://origin.example.com/live/user/pass/1025123.m3u8" - ); - assert_eq!( - super::ensure_hls_manifest_extension("http://origin.example.com/live/user/pass/1025123..?token=1"), - "http://origin.example.com/live/user/pass/1025123.m3u8?token=1" - ); - assert_eq!( - super::ensure_hls_manifest_extension("provider://mirror/live/user/pass/1025123..m3u8"), - "provider://mirror/live/user/pass/1025123.m3u8" - ); -} - -#[test] -fn hls_origin_resolution_preserves_legacy_built_xtream_origin_url() { - let input = ConfigInput { - id: 7, - name: Arc::from("xtream"), - input_type: InputType::Xtream, - url: "http://origin.example.com/base".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - ..ConfigInput::default() - }; - - let origin = super::build_hls_origin_resolution(&input, "http://other.example.com/live/other/creds/1025126.m3u8") - .expect("xtream origin should resolve"); - - assert_eq!(origin.session_entry_url.as_str(), "http://other.example.com/live/other/creds/1025126.m3u8"); - assert_eq!(origin.hls_url, origin.session_entry_url.as_str()); - assert!(origin.session_entry_url.url_failover_provider().is_none()); -} - -#[test] -fn hls_origin_resolution_keeps_provider_failover_out_of_identity() { - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "mirror-group".into(), - urls: vec!["http://mirror-a.example.com".into(), "http://mirror-b.example.com".into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, - dns: None, - })); - let input = ConfigInput { - id: 7, - name: Arc::from("xtream"), - input_type: InputType::Xtream, - url: "provider://mirror-group".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - provider_configs: Some(vec![Arc::clone(&provider)]), - ..ConfigInput::default() - }; - - let origin = - super::build_hls_origin_resolution(&input, "provider://mirror-group/live/source-user/source-pass/1025126.m3u8") - .expect("provider failover origin should resolve"); - let failover_key = super::build_hls_origin_source(&input, "80510").session_key(); - let direct_key = super::build_hls_origin_source(&input, "80510").session_key(); - - assert_eq!(origin.session_entry_url.as_str(), "provider://mirror-group/live/source-user/source-pass/1025126.m3u8"); - assert!(origin.session_entry_url.url_failover_provider().is_some()); - assert_eq!(failover_key, direct_key); - assert!(!failover_key.stable_value().contains("provider://")); - assert!(!failover_key.stable_value().contains("mirror-a.example.com")); - assert!(!failover_key.stable_value().contains("mirror-b.example.com")); -} - -#[test] -fn hls_origin_resolution_uses_m3u_playlist_item_url() { - let input = ConfigInput { - id: 9, - name: Arc::from("m3u"), - input_type: InputType::M3u, - url: "http://playlist.example.com/list.m3u".to_string(), - ..ConfigInput::default() - }; - - let origin = super::build_hls_origin_resolution(&input, "http://media.example.com/live/channel/index.m3u8") - .expect("m3u hls origin should resolve"); - let source = super::build_hls_origin_source(&input, "stable-item"); - - assert_eq!(origin.session_entry_url.as_str(), "http://media.example.com/live/channel/index.m3u8"); - assert_eq!(source.source_kind, HlsOriginSourceKind::M3uMediaPlaylist); - assert_eq!(source.session_key().stable_value(), "input:9|hls|stable-item"); -} - -#[test] -fn provider_failover_mirror_change_keeps_same_hls_session_identity() { - let provider_a = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "mirror-group".into(), - urls: vec!["http://mirror-a.example.com".into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, - dns: None, - })); - let provider_b = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "mirror-group".into(), - urls: vec!["http://mirror-b.example.com".into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, - dns: None, - })); - let input_a = ConfigInput { - id: 7, - name: Arc::from("xtream"), - input_type: InputType::Xtream, - url: "provider://mirror-group".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - provider_configs: Some(vec![provider_a]), - ..ConfigInput::default() - }; - let input_b = ConfigInput { provider_configs: Some(vec![provider_b]), ..input_a.clone() }; - - let origin_a = super::build_hls_origin_resolution(&input_a, "provider://mirror-group/a.m3u8") - .expect("provider failover origin a should resolve"); - let origin_b = super::build_hls_origin_resolution(&input_b, "provider://mirror-group/b.m3u8") - .expect("provider failover origin b should resolve"); - let key_a = super::build_hls_origin_source(&input_a, "80510").session_key(); - let key_b = super::build_hls_origin_source(&input_b, "80510").session_key(); - let secret = b"rewrite-secret"; - - assert!(origin_a.session_entry_url.url_failover_provider().is_some()); - assert!(origin_b.session_entry_url.url_failover_provider().is_some()); - assert_eq!(key_a, key_b); - assert_eq!(key_a.stable_value(), "input:7|hls|80510"); - assert_eq!(build_proxy_session_id(&key_a, secret), build_proxy_session_id(&key_b, secret)); - assert!(!key_a.stable_value().contains("provider://")); - assert!(!key_a.stable_value().contains("mirror-a.example.com")); - assert!(!key_a.stable_value().contains("mirror-b.example.com")); -} - -#[test] -fn hls_runtime_origin_fetch_url_uses_selected_provider_account() { - let input = ConfigInput { - name: Arc::from("source"), - url: "http://source.example.com".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - input_type: InputType::Xtream, - ..ConfigInput::default() - }; - let provider_input = ConfigInput { - id: 7, - name: Arc::from("selected-provider"), - url: "http://provider.example.com".to_string(), - username: Some("provider-user".to_string()), - password: Some("provider-pass".to_string()), - input_type: InputType::Xtream, - max_connections: 1, - ..ConfigInput::default() - }; - let provider = Arc::new(RuntimeProviderConfig::new( - &provider_input, - Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), - Arc::new(|_, _| {}), - )); - - let fetch_url = super::build_hls_origin_fetch_url( - &input, - "http://source.example.com/live/source-user/source-pass/12345.m3u8", - "http://source.example.com/live/source-user/source-pass/12345.m3u8", - Some(&provider), - ) - .expect("fetch url should be rewritten"); - - assert_eq!(fetch_url, "http://provider.example.com/live/provider-user/provider-pass/12345.m3u8"); - - let failover_provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "demo".into(), - urls: vec!["http://mirror-a.example.com".into(), "http://mirror-b.example.com".into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, - dns: None, - })); - let provider_scheme_input = ConfigInput { - name: Arc::from("source"), - url: "provider://demo".to_string(), - username: Some("source-user".to_string()), - password: Some("source-pass".to_string()), - input_type: InputType::Xtream, - provider_configs: Some(vec![Arc::clone(&failover_provider)]), - ..ConfigInput::default() - }; - let provider_scheme_without_account_rewrite = super::build_hls_origin_fetch_url( - &provider_scheme_input, - "provider://demo/live/source-user/source-pass/12345.m3u8", - "provider://demo/live/source-user/source-pass/12345.m3u8", - None, - ) - .expect("provider scheme fetch url should remain failover capable"); - assert_eq!(provider_scheme_without_account_rewrite, "provider://demo/live/source-user/source-pass/12345.m3u8"); - - let provider_scheme_fetch_url = super::build_hls_origin_fetch_url( - &provider_scheme_input, - "provider://demo/live/source-user/source-pass/12345.m3u8", - "provider://demo/live/source-user/source-pass/12345.m3u8", - Some(&provider), - ) - .expect("provider scheme fetch url should use selected runtime account without losing failover"); - - assert_eq!(provider_scheme_fetch_url, "provider://demo/live/provider-user/provider-pass/12345.m3u8"); - let origin_entry = super::LiveHlsOriginEntry::parse_with_provider_configs( - &provider_scheme_without_account_rewrite, - Some(Arc::clone(&failover_provider)), - Some(Arc::clone(&provider)), - ) - .expect("provider origin entry"); - let input_source = origin_entry.to_input_source(); - assert_eq!(input_source.url, provider_scheme_without_account_rewrite); - assert_eq!(input_source.username.as_deref(), Some("provider-user")); - assert_eq!(input_source.password.as_deref(), Some("provider-pass")); - - let failover_context = super::hls_url_failover_provider_for_origin_context( - &provider_scheme_input, - "provider://demo/live/source-user/source-pass/12345.m3u8", - "provider://demo/live/source-user/source-pass/12345.m3u8", - &provider_scheme_fetch_url, - ) - .expect("provider failover context"); - assert_eq!(failover_context.name.as_ref(), "demo"); -} - -#[test] -fn hls_origin_entry_attaches_url_failover_provider_only_to_provider_scheme_fetch_url() { - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "demo".into(), - urls: vec!["http://mirror.example.com".into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, - dns: None, - })); - - let http_provider = super::effective_hls_url_failover_provider_for_fetch_url( - "http://provider.example.com/live/user/pass/12345.m3u8", - None, - Some(Arc::clone(&provider)), - ); - let http_entry = super::LiveHlsOriginEntry::parse_with_url_failover_provider( - "http://provider.example.com/live/user/pass/12345.m3u8", - http_provider, - ) - .expect("http origin entry"); - assert!(http_entry.url_failover_provider().is_none()); - - let provider_scheme_provider = super::effective_hls_url_failover_provider_for_fetch_url( - "provider://demo/live/user/pass/12345.m3u8", - None, - Some(Arc::clone(&provider)), - ); - let provider_entry = super::LiveHlsOriginEntry::parse_with_url_failover_provider( - "provider://demo/live/user/pass/12345.m3u8", - provider_scheme_provider, - ) - .expect("provider origin entry"); - assert_eq!(provider_entry.url_failover_provider().expect("provider").name.as_ref(), "demo"); -} - -#[tokio::test] -async fn hls_access_lease_validity_uses_session_idle_timeout_not_cache_duration() { - let hls_dto = HlsCacheConfigDto { - cache_duration: shared::model::Secs::new(900), - session_idle_timeout: shared::model::Secs::new(42), - ..Default::default() - }; - let hls_config = HlsCacheConfig::from(&hls_dto); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); - - assert_eq!(super::hls_access_lease_ttl_ms(&app_state), 42_000); -} - -#[tokio::test] -async fn hls_access_lease_active_window_uses_two_target_durations() { - let app_state = test_app_state(); - let key = HlsSessionKey::new(1, "access-window-stream"); - let (session, _) = - app_state.hls.proxy.get_or_create_session_with_outcome(key, b"secret", super::current_time_millis()).await; - session.write().await.target_duration = Some(11); - - let timing = super::hls_access_lease_timing_for_session(&app_state, &session).await; - - assert_eq!(timing.active_window_ms, 22_000); - assert_eq!(timing.valid_window_ms, super::hls_access_lease_ttl_ms(&app_state)); -} - -#[tokio::test] -async fn hls_lifecycle_active_timer_moves_access_lease_to_idle() { - let app_state = test_app_state(); - let now_ms = super::current_time_millis(); - let lease_id = HlsAccessLeaseId("lifecycle-lease".to_string()); - let key = HlsSessionKey::new(1, "lifecycle-stream"); - let (session, _) = app_state.hls.proxy.get_or_create_session_with_outcome(key, b"secret", now_ms).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("hls-user", "client"), - proxy_session_id.clone(), - "hls-user".to_string(), - "hls-session-token".to_string(), - 1, - "lifecycle-stream".to_string(), - 123, - now_ms, - 60_000, - )) - .await; - assert!(app_state - .hls - .proxy - .activate_access_lease( - &lease_id, - &proxy_session_id, - now_ms, - HlsAccessLeaseTiming { active_window_ms: 1, valid_window_ms: 60_000 }, - ) - .await - .is_activated()); - session.write().await.activity.active_access_lease_count = 1; - - app_state - .hls - .proxy - .handle_lifecycle_event( - &app_state.active_users, - &app_state.active_provider, - HlsLifecycleEvent { - key: HlsLifecycleEventKey::AccessLeaseActive { - lease_id: lease_id.clone(), - proxy_session_id: proxy_session_id.clone(), - }, - due_at_ms: now_ms.saturating_add(1), - }, - now_ms.saturating_add(2), - ) - .await; - - assert_eq!( - app_state.hls.proxy.access_leases().write().await.lease_state(&lease_id, now_ms.saturating_add(2)), - Some(HlsAccessLeaseState::Idle) - ); - assert_eq!(session.read().await.activity.active_access_lease_count, 0); -} - -#[tokio::test] -async fn hls_lifecycle_validity_timer_removes_expired_access_lease() { - let mut hls_cache = HlsCacheConfigDto::default(); - hls_cache.segment_repair.max_level = HlsSegmentRepairMode::Low; - let hls_config = HlsCacheConfig::from(&hls_cache); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); - let now_ms = super::current_time_millis(); - let proxy_session_id = ProxySessionId("lifecycle-validity-proxy".to_string()); - let lease_id = HlsAccessLeaseId("lifecycle-validity-lease".to_string()); - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("hls-user", "client"), - proxy_session_id.clone(), - "hls-user".to_string(), - "hls-session-token".to_string(), - 1, - "lifecycle-stream".to_string(), - 123, - now_ms, - 1, - )) - .await; - assert!(app_state - .hls - .proxy - .activate_access_lease( - &lease_id, - &proxy_session_id, - now_ms, - HlsAccessLeaseTiming { active_window_ms: 1, valid_window_ms: 1 }, - ) - .await - .is_activated()); - let repair_before = app_state.hls.proxy.segment_repair().stats().await; - assert_eq!(repair_before.windows, 1); - assert_eq!(repair_before.generations, 1); - - app_state - .hls - .proxy - .handle_lifecycle_event( - &app_state.active_users, - &app_state.active_provider, - HlsLifecycleEvent { - key: HlsLifecycleEventKey::AccessLeaseValidity { - lease_id: lease_id.clone(), - proxy_session_id: proxy_session_id.clone(), - }, - due_at_ms: now_ms.saturating_add(1), - }, - now_ms.saturating_add(2), - ) - .await; - - assert_eq!( - app_state.hls.proxy.access_leases().write().await.lease_state(&lease_id, now_ms.saturating_add(2)), - None - ); - let repair_after = app_state.hls.proxy.segment_repair().stats().await; - assert_eq!(repair_after.windows, 0); - assert_eq!(repair_after.generations, 0); -} - -#[tokio::test] -async fn hls_lifecycle_validity_timer_removes_expired_pending_access_lease() { - let app_state = test_app_state(); - let now_ms = super::current_time_millis(); - let key = HlsSessionKey::new(1, "pending-expiry-stream"); - let (session, _) = app_state.hls.proxy.get_or_create_session_with_outcome(key, b"secret", now_ms).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("pending-expiry-lease".to_string()); - let active_lease_id = HlsAccessLeaseId("active-soft-lease".to_string()); - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("hls-user", "client"), - proxy_session_id.clone(), - "hls-user".to_string(), - "hls-session-token".to_string(), - 1, - "pending-expiry-stream".to_string(), - 123, - now_ms, - 1, - )) - .await; - app_state - .hls - .proxy - .prepare_access_lease( - HlsAccessLease::pending( - active_lease_id.clone(), - HlsPlaybackFamilyKey::new("soft-user", "client"), - proxy_session_id.clone(), - "soft-user".to_string(), - "soft-session-token".to_string(), - 1, - "pending-expiry-stream".to_string(), - 123, - now_ms, - 60_000, - ) - .with_origin_acquire_policy(ConnectionKind::Soft, 20), - ) - .await; - assert!(app_state - .hls - .proxy - .activate_access_lease( - &active_lease_id, - &proxy_session_id, - now_ms, - HlsAccessLeaseTiming { active_window_ms: 60_000, valid_window_ms: 60_000 }, - ) - .await - .is_activated()); - - app_state - .hls - .proxy - .handle_lifecycle_event( - &app_state.active_users, - &app_state.active_provider, - HlsLifecycleEvent { - key: HlsLifecycleEventKey::AccessLeaseValidity { - lease_id: lease_id.clone(), - proxy_session_id: proxy_session_id.clone(), - }, - due_at_ms: now_ms.saturating_add(1), - }, - now_ms.saturating_add(2), - ) - .await; - - assert_eq!( - app_state.hls.proxy.access_leases().write().await.lease_state(&lease_id, now_ms.saturating_add(2)), - None - ); - let session = session.read().await; - assert_eq!(session.activity.active_access_lease_count, 1); - let effective_policy = session.effective_origin_acquire_policy_or_default(); - assert_eq!(effective_policy.connection_kind, ConnectionKind::Soft); - assert_eq!(effective_policy.priority, 20); -} - -#[tokio::test] -async fn hls_lifecycle_session_idle_timer_removes_idle_session() { - let mut hls_dto = HlsCacheConfigDto { session_idle_timeout: shared::model::Secs::new(1), ..Default::default() }; - hls_dto.segment_repair.max_level = HlsSegmentRepairMode::Low; - let hls_config = HlsCacheConfig::from(&hls_dto); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); - let now_ms = super::current_time_millis(); - let key = HlsSessionKey::new(1, "expired-session"); - let (session, _) = - app_state.hls.proxy.get_or_create_session_with_outcome(key, b"secret", now_ms.saturating_sub(2_000)).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("session-idle-cleanup-lease".to_string()); - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("hls-user", "client"), - proxy_session_id.clone(), - "hls-user".to_string(), - "hls-session-token".to_string(), - 1, - "expired-session".to_string(), - 123, - now_ms, - 60_000, - )) - .await; - assert!(app_state - .hls - .proxy - .activate_access_lease( - &lease_id, - &proxy_session_id, - now_ms, - HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 }, - ) - .await - .is_activated()); - assert_eq!(app_state.hls.proxy.access_leases().read().await.len(), 1); - assert_eq!(app_state.hls.proxy.segment_repair().stats().await.windows, 1); - - app_state - .hls - .proxy - .handle_lifecycle_event( - &app_state.active_users, - &app_state.active_provider, - HlsLifecycleEvent { - key: HlsLifecycleEventKey::SessionIdle { proxy_session_id: proxy_session_id.clone() }, - due_at_ms: now_ms.saturating_sub(1_000), - }, - now_ms, - ) - .await; - - assert!(app_state.hls.proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.is_none()); - assert_eq!(app_state.hls.proxy.access_leases().read().await.len(), 0); - let repair_after = app_state.hls.proxy.segment_repair().stats().await; - assert_eq!(repair_after.windows, 0); - assert_eq!(repair_after.generations, 0); -} - -#[tokio::test] -async fn hls_gc_session_removal_cleans_access_leases_and_repair_state() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let mut hls_dto = HlsCacheConfigDto { - cache_path: Some(temp_dir.path().to_string_lossy().into_owned()), - session_idle_timeout: shared::model::Secs::new(1), - ..Default::default() - }; - hls_dto.segment_repair.max_level = HlsSegmentRepairMode::Low; - let hls_config = HlsCacheConfig::from(&hls_dto); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); - let now_ms = super::current_time_millis(); - let key = HlsSessionKey::new(1, "gc-cleanup-session"); - let (session, _) = - app_state.hls.proxy.get_or_create_session_with_outcome(key, b"secret", now_ms.saturating_sub(2_000)).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("gc-cleanup-lease".to_string()); - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("hls-user", "client"), - proxy_session_id.clone(), - "hls-user".to_string(), - "hls-session-token".to_string(), - 1, - "gc-cleanup-session".to_string(), - 123, - now_ms, - 60_000, - )) - .await; - assert!(app_state - .hls - .proxy - .activate_access_lease( - &lease_id, - &proxy_session_id, - now_ms, - HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 }, - ) - .await - .is_activated()); - assert_eq!(app_state.hls.proxy.access_leases().read().await.len(), 1); - assert_eq!(app_state.hls.proxy.segment_repair().stats().await.windows, 1); - - let report = app_state.hls.proxy.run_garbage_collection_once(now_ms).await.expect("gc should run"); - - assert_eq!(report.sessions_deleted, 1); - assert!(app_state.hls.proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.is_none()); - assert_eq!(app_state.hls.proxy.access_leases().read().await.len(), 0); - let repair_after = app_state.hls.proxy.segment_repair().stats().await; - assert_eq!(repair_after.windows, 0); - assert_eq!(repair_after.generations, 0); -} - -fn test_app_state() -> Arc { test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::new())) } - -#[tokio::test] -async fn lease_snapshot_limit_rejection_is_controlled_and_observable() { - let app_state = test_app_state(); - let proxy_session_id = ProxySessionId("snapshot-limit-session".to_string()); - let access_lease_id = HlsAccessLeaseId("snapshot-limit-lease".to_string()); - let oversized_uri = "x".repeat(MAX_HLS_MANIFEST_BYTES + 1); - let body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{oversized_uri}\n"); - let derivation = super::derive_hls_lease_manifest_snapshot( - &super::HlsLeaseManifestSnapshotInput::TransientPassthrough { - materialized_body: &body, - source_commit_identity: HlsManifestCommitIdentity::committed(1, 1), - finalized_manifest_generation: None, - }, - 2, - ); - - assert!(super::observe_hls_lease_manifest_snapshot_derivation( - &app_state, - &proxy_session_id, - &access_lease_id, - derivation, - ) - .is_err()); - assert_eq!(app_state.hls.proxy.metrics().snapshot().manifest_limit_rejections, 1); -} - -fn test_beast_hls_proxy(cache_path: &std::path::Path) -> Arc { - let config = HlsCacheConfig::from(&HlsCacheConfigDto { - cache_path: Some(cache_path.to_string_lossy().into_owned()), - strip: HlsStripConfigDto { mode: HlsStripMode::Segments, value: 3 }, - max_segments_prefetch: 6, - manifest_recovery_burst: HlsManifestRecoveryBurstConfigDto { level: HlsManifestRecoveryBurstLevel::Beast }, - ..HlsCacheConfigDto::default() - }); - Arc::new(HlsProxyManager::from_hls_cache_config(Some(&config))) -} - -fn disable_custom_stream_response(app_state: &Arc) { - app_state.app_config.config.store(Arc::new(Config { custom_stream_response_enabled: false, ..Default::default() })); -} - -fn enable_provider_exhausted_custom_response(app_state: &Arc) { - app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { - channel_unavailable: None, - user_connections_exhausted: None, - provider_connections_exhausted: Some(test_custom_video_buffer()), - low_priority_preempted: None, - user_account_expired: None, - panel_api_provisioning: None, - hls_session_or_lease_expired: None, - panel_api_provisioning_hls_segments: Vec::new(), - }))); -} - -fn enable_runtime_policy_custom_responses(app_state: &Arc) { - app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { - channel_unavailable: None, - user_connections_exhausted: Some(TransportStreamBuffer::new( - include_bytes!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../test/fixtures/hls/user_connections_exhausted.ts" - )) - .to_vec(), - )), - provider_connections_exhausted: Some(TransportStreamBuffer::new( - include_bytes!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../test/fixtures/hls/provider_connections_exhausted.ts" - )) - .to_vec(), - )), - low_priority_preempted: Some(TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/low_priority_preempted.ts")) - .to_vec(), - )), - user_account_expired: None, - panel_api_provisioning: None, - hls_session_or_lease_expired: None, - panel_api_provisioning_hls_segments: Vec::new(), - }))); -} - -fn enable_channel_unavailable_custom_response(app_state: &Arc) { - let config = app_state.app_config.config.load(); - app_state - .app_config - .config - .store(Arc::new(Config { custom_stream_response_enabled: true, ..config.as_ref().clone() })); - app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { - channel_unavailable: Some(test_custom_video_buffer()), - user_connections_exhausted: None, - provider_connections_exhausted: None, - low_priority_preempted: None, - user_account_expired: None, - panel_api_provisioning: None, - hls_session_or_lease_expired: None, - panel_api_provisioning_hls_segments: Vec::new(), - }))); -} - -fn test_custom_video_buffer() -> TransportStreamBuffer { - TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")).to_vec(), - ) -} - -fn enable_hls_provisioning_custom_response(app_state: &Arc) { - let mut ts_packet = vec![0_u8; 188]; - ts_packet[0] = 0x47; - let provisioning_segments = (0..6) - .map(|index| { - let mut packet = ts_packet.clone(); - packet[1] = u8::try_from(index).expect("test index fits"); - TransportStreamBuffer::new(packet) - }) - .collect(); - app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { - channel_unavailable: None, - user_connections_exhausted: None, - provider_connections_exhausted: None, - low_priority_preempted: None, - user_account_expired: None, - panel_api_provisioning: None, - hls_session_or_lease_expired: None, - panel_api_provisioning_hls_segments: provisioning_segments, - }))); -} - -fn test_app_state_with_inputs(inputs: Vec>) -> Arc { - test_app_state_with_hls_proxy_and_inputs(Arc::new(HlsProxyManager::new()), inputs) -} - -fn enable_hls_cache(app_state: &Arc) { - let config = Config { - reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { - hls_cache: Some(HlsCacheConfigDto::default()), - ..Default::default() - })), - ..Default::default() - }; - app_state.app_config.config.store(Arc::new(config)); -} - -fn test_app_state_with_hls_proxy(hls_proxy: Arc) -> Arc { - test_app_state_with_hls_proxy_and_inputs(hls_proxy, Vec::new()) -} - -fn test_app_state_with_hls_proxy_and_inputs( - hls_proxy: Arc, - inputs: Vec>, -) -> Arc { - let app_config = test_app_config(); - if !inputs.is_empty() { - app_config.sources.store(Arc::new(SourcesConfig { - batch_files: vec![], - provider: vec![], - group_lookup: build_group_lookup(&inputs), - inputs, - sources: vec![], - templates: None, - })); - } - let event_manager = Arc::new(EventManager::new()); - let active_provider = Arc::new(ActiveProviderManager::new(&app_config, &event_manager)); - let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); - active_provider.set_shared_stream_manager(&shared_stream_manager); - - let geoip = Arc::new(ArcSwapOption::::default()); - let config = app_config.config.load(); - let active_users = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); - let connection_manager = - Arc::new(ConnectionManager::new(&active_users, &active_provider, &shared_stream_manager, &event_manager, None)); - let cancel_tokens = CancelTokens::default(); - let metadata_manager = Arc::new(MetadataUpdateManager::new(cancel_tokens.metadata.clone())); - - Arc::new(AppState { - recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( - Arc::clone(&active_provider), - Arc::clone(&connection_manager), - ), - app_config, - http_clients: Arc::new(tuliprox_core::model::HttpClients::new( - reqwest::Client::new(), - reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("no-redirect client builds"), - reqwest::Client::new(), - reqwest::Client::new(), - reqwest::Client::new(), - )), - recordings: Arc::new(crate::api::model::RecordingQueue::new()), - cache: Arc::new(ArcSwapOption::default()), - shared_stream_manager, - hls: crate::api::model::HlsState::new(hls_proxy), - stalker_resolve_coordinator: crate::api::model::StalkerResolveCoordinator::default(), - active_users, - active_provider, - connection_manager, - event_manager, - cancel_tokens: ArcSwap::from_pointee(cancel_tokens), - playlists: Arc::new(PlaylistStorageState::new()), - geoip, - metadata_manager, - auth: crate::api::model::AuthState::for_tests(), - playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), - }) -} - -async fn create_bound_hls_test_session( - app_state: &Arc, - input: &ConfigInput, - stream_ref: &str, - account_name: &str, - now_ms: u64, -) -> HlsSessionHandle { - let origin_source = super::build_hls_origin_source(input, stream_ref); - let session_key = origin_source.session_key(); - let (session, _) = app_state - .hls - .proxy - .get_or_create_session_with_source_and_outcome( - session_key, - origin_source, - &app_state.get_encrypt_secret(), - now_ms, - ) - .await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - { - let mut session_guard = session.write().await; - session_guard.origin_account_binding = Some(HlsOriginAccountBinding::new( - Arc::clone(&input.name), - Arc::from(account_name), - &proxy_session_id, - now_ms, - )); - } - session -} - -async fn create_unbound_hls_test_session( - app_state: &Arc, - input: &ConfigInput, - stream_ref: &str, - now_ms: u64, -) -> HlsSessionHandle { - let origin_source = super::build_hls_origin_source(input, stream_ref); - let session_key = origin_source.session_key(); - app_state - .hls - .proxy - .get_or_create_session_with_source_and_outcome( - session_key, - origin_source, - &app_state.get_encrypt_secret(), - now_ms, - ) - .await - .0 -} - -fn test_hls_origin_io_context(app_state: &Arc) -> HlsOriginIoContext { - HlsOriginIoContext { - ctx: app_state.hls_ctx(), - client_addr: test_fingerprint().addr, - allow_grace: false, - priority: 0, - connection_kind: ConnectionKind::Normal, - reservation_ttl_secs: 60, - preacquired_provider_handle: None, - started_generation: None, - } -} - -fn single_hls_provider_input(name: &str) -> ConfigInput { - ConfigInput { - id: 1, - name: Arc::from(name), - input_type: InputType::Xtream, - url: "http://account.example.com".to_string(), - username: Some("account-user".to_string()), - password: Some("account-pass".to_string()), - enabled: true, - priority: 0, - max_connections: 1, - ..ConfigInput::default() - } -} - -fn overlap_provider_input() -> ConfigInput { - ConfigInput { - id: 1, - name: Arc::from("overlap-input"), - input_type: InputType::Xtream, - url: "http://root.example.com".to_string(), - username: Some("root-user".to_string()), - password: Some("root-pass".to_string()), - enabled: true, - priority: 10, - max_connections: 1, - aliases: Some(vec![crate::model::ConfigInputAlias { - id: 2, - name: Arc::from("account-a"), - url: "http://account.example.com".to_string(), - username: Some("account-user".to_string()), - password: Some("account-pass".to_string()), - priority: 0, - max_connections: 1, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - } -} - -#[tokio::test] -async fn hls_origin_account_io_lease_allows_parallel_same_session_origin_work() { - let input = overlap_provider_input(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; - let binding = session.read().await.origin_account_binding.clone().expect("binding exists"); - let origin_io = test_hls_origin_io_context(&app_state); - - let first = begin_hls_origin_account_io(&origin_io, &session, &binding) - .await - .expect("first same-session origin io acquires provider account"); - wait_for_provider_connection_count(&app_state, 1).await; - let second = begin_hls_origin_account_io(&origin_io, &session, &binding) - .await - .expect("second same-session origin io joins session lease"); - wait_for_provider_connection_count(&app_state, 1).await; - assert_eq!( - session - .read() - .await - .origin_account_io_lease - .as_ref() - .expect("session provider lease exists") - .active_io_count - .load(std::sync::atomic::Ordering::Relaxed), - 2 - ); - - finish_hls_origin_account_io(&origin_io, &session, first, true).await; - wait_for_provider_connection_count(&app_state, 1).await; - assert_eq!( - session - .read() - .await - .origin_account_io_lease - .as_ref() - .expect("session provider lease remains while second io is active") - .active_io_count - .load(std::sync::atomic::Ordering::Relaxed), - 1 - ); - - finish_hls_origin_account_io(&origin_io, &session, second, true).await; - wait_for_provider_connection_count(&app_state, 0).await; - assert!(!app_state - .active_provider - .is_provider_reserved_for_other_session(&binding.account_name, Some("other-hls-session"))); -} - -#[tokio::test] -async fn hls_origin_account_io_lease_blocks_other_hls_sessions_for_same_account() { - let input = overlap_provider_input(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let first_session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; - let second_session = create_bound_hls_test_session(&app_state, &input, "67890", "account-a", 1_000).await; - let first_binding = first_session.read().await.origin_account_binding.clone().expect("binding exists"); - let second_binding = second_session.read().await.origin_account_binding.clone().expect("binding exists"); - let origin_io = test_hls_origin_io_context(&app_state); - - let first = begin_hls_origin_account_io(&origin_io, &first_session, &first_binding) - .await - .expect("first session acquires account"); - wait_for_provider_connection_count(&app_state, 1).await; - - assert!(begin_hls_origin_account_io(&origin_io, &second_session, &second_binding).await.is_err()); - wait_for_provider_connection_count(&app_state, 1).await; - - finish_hls_origin_account_io(&origin_io, &first_session, first, true).await; - wait_for_provider_connection_count(&app_state, 0).await; -} - -#[tokio::test] -async fn transient_origin_binding_requires_runtime_prepare_for_missing_or_detached_account() { - let input = single_hls_provider_input("known-account"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let proxy_session_id = ProxySessionId("shared-hls-session".to_string()); - let known_binding = - HlsOriginAccountBinding::new(Arc::clone(&input.name), Arc::clone(&input.name), &proxy_session_id, 1_000); - let missing_binding = - HlsOriginAccountBinding::new(Arc::clone(&input.name), Arc::from("removed-account"), &proxy_session_id, 1_000); - let mut detached_binding = known_binding.clone(); - detached_binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, 2_000); - - let hls_ctx = app_state.hls_ctx(); - assert!(!super::hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, &known_binding)); - assert!(super::hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, &missing_binding)); - assert!(super::hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, &detached_binding)); -} - -#[tokio::test] -async fn hls_account_overlap_selects_soft_candidate_but_not_hard_active() { - let app_state = test_app_state(); - let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; - let session = create_bound_hls_test_session(&app_state, &input, "old", "account-a", 1_000).await; - { - let mut session = session.write().await; - session.target_duration = Some(10); - session.mark_authorized_media_access(1_000); - } - let new_proxy_session_id = ProxySessionId("new-session".to_string()); - - let hard_candidate = - super::find_hls_account_overlap_candidate(&app_state, &input, &new_proxy_session_id, 5_000).await; - assert!(hard_candidate.is_none(), "hard-active sessions must not be overbooked"); - - let delayed_candidate = - super::find_hls_account_overlap_candidate(&app_state, &input, &new_proxy_session_id, 12_000).await; - assert!(delayed_candidate.is_none(), "soft-active candidate must respect the dynamic overlap delay"); - - let soft_candidate = super::find_hls_account_overlap_candidate(&app_state, &input, &new_proxy_session_id, 21_000) - .await - .expect("soft-active session can be overbooked"); - assert_eq!(soft_candidate.account_name.as_ref(), "account-a"); - assert_eq!(soft_candidate.last_media_at_ms, 1_000); - assert_eq!(soft_candidate.soft_overlap_eligible_at_ms, 21_000); - assert_eq!(soft_candidate.soft_overlap_delay_ms, 20_000); - assert_eq!(soft_candidate.reclaim_until_ms, 31_000); -} - -#[test] -fn hls_soft_overlap_delay_scales_with_tuliprox_target_pressure() { - assert_eq!(super::hls_soft_overlap_delay_ms(10_000, 1, 1), 20_000); - assert_eq!(super::hls_soft_overlap_delay_ms(10_000, 3, 2), 15_000); - assert_eq!(super::hls_soft_overlap_delay_ms(10_000, 4, 2), 10_000); -} - -#[tokio::test] -async fn hls_account_overlap_reclaim_preempts_speculative_session() { - let app_state = test_app_state(); - let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; - let winner = create_bound_hls_test_session(&app_state, &input, "winner", "account-a", 1_000).await; - let loser = create_bound_hls_test_session(&app_state, &input, "loser", "account-a", 1_000).await; - let winner_proxy_session_id = winner.read().await.proxy_session_id.clone(); - let loser_proxy_session_id = loser.read().await.proxy_session_id.clone(); - { - let mut loser = loser.write().await; - loser.origin_account_binding = Some(HlsOriginAccountBinding::speculative_from( - Arc::clone(&input.name), - Arc::from("account-a"), - &loser_proxy_session_id, - winner_proxy_session_id.clone(), - 20_000, - 2_000, - )); - } - let loser_generation = loser.read().await.activity.origin_work_generation; - - super::reclaim_hls_account_overlap_if_needed(&app_state, &winner, 10_000).await; - - assert!(app_state.hls.proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 10_000).await); - assert!(!app_state.hls.proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 25_000).await); - let loser_binding_mode = loser.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); - assert!(matches!( - loser_binding_mode, - HlsOriginAccountBindingMode::Detached { reason: HlsOriginAccountDetachedReason::ReclaimedByOriginalOwner, .. } - )); - assert_eq!(loser.read().await.activity.origin_work_generation, loser_generation + 1); - let winner_binding_mode = winner.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); - assert!(matches!(winner_binding_mode, HlsOriginAccountBindingMode::Active)); -} - -#[tokio::test] -async fn hls_account_overlap_promotes_speculative_session_after_soft_window() { - let app_state = test_app_state(); - let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; - let displaced = create_bound_hls_test_session(&app_state, &input, "displaced", "account-a", 1_000).await; - let promoted = create_bound_hls_test_session(&app_state, &input, "promoted", "account-a", 1_000).await; - let displaced_proxy_session_id = displaced.read().await.proxy_session_id.clone(); - let promoted_proxy_session_id = promoted.read().await.proxy_session_id.clone(); - { - let mut promoted = promoted.write().await; - promoted.origin_account_binding = Some(HlsOriginAccountBinding::speculative_from( - Arc::clone(&input.name), - Arc::from("account-a"), - &promoted_proxy_session_id, - displaced_proxy_session_id, - 20_000, - 2_000, - )); - } - let displaced_generation = displaced.read().await.activity.origin_work_generation; - - super::promote_elapsed_hls_account_overlaps(&app_state, 20_001).await; - - let displaced_binding_mode = displaced.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); - assert!(matches!( - displaced_binding_mode, - HlsOriginAccountBindingMode::Detached { reason: HlsOriginAccountDetachedReason::SoftWindowElapsed, .. } - )); - assert_eq!(displaced.read().await.activity.origin_work_generation, displaced_generation + 1); - let promoted_binding_mode = promoted.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); - assert!(matches!(promoted_binding_mode, HlsOriginAccountBindingMode::Active)); -} - -#[tokio::test] -async fn hls_account_binding_soft_expiry_retains_session_and_reacquires_on_authorized_manifest_work() { - let input = overlap_provider_input(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let session = create_bound_hls_test_session(&app_state, &input, "12345", input.name.as_ref(), 1_000).await; - { - let mut session = session.write().await; - session.target_duration = Some(1); - session.mark_authorized_media_access(1_000); - session.activity.last_delivered_media_at_ms = Some(1_000); - } - let old_generation = session.read().await.activity.origin_work_generation; - - super::detach_unprotected_hls_origin_account_bindings(&app_state, 4_001).await; - - { - let session = session.read().await; - let binding = session.origin_account_binding.as_ref().expect("detached binding is retained"); - assert!(matches!( - binding.binding_mode, - HlsOriginAccountBindingMode::Detached { reason: HlsOriginAccountDetachedReason::SoftWindowElapsed, .. } - )); - assert_eq!(session.activity.origin_work_generation, old_generation + 1); - } - - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let prepared_origin = super::prepare_hls_origin_runtime( - &app_state, - &session, - &input, - "http://root.example.com/live/root-user/root-pass/12345.m3u8", - "http://root.example.com/live/root-user/root-pass/12345.m3u8", - &proxy_session_id, - &test_fingerprint(), - ConnectionKind::Normal, - 0, - super::HlsOriginWorkKind::Manifest, - super::HlsOriginWorkClass::ManifestInteractive, - 4_100, - ) - .await - .expect("authorized origin work can reacquire a detached binding"); - - let binding = - prepared_origin.origin_account_binding_to_store.as_ref().expect("new binding should be stored by caller"); - assert_eq!(binding.account_name.as_ref(), "account-a"); - assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active)); - assert_eq!(prepared_origin.fetch_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8"); - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); -} - -#[tokio::test] -async fn hls_origin_runtime_uses_soft_overlap_before_grace_for_interactive_work() { - let input = single_hls_provider_input("soft-overlap-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let old_session = create_bound_hls_test_session(&app_state, &input, "old", input.name.as_ref(), 1_000).await; - let old_proxy_session_id = old_session.read().await.proxy_session_id.clone(); - { - let mut session = old_session.write().await; - session.target_duration = Some(10); - session.mark_authorized_media_access(1_000); - session.activity.last_delivered_media_at_ms = Some(1_000); - } - let old_binding = old_session.read().await.origin_account_binding.clone().expect("binding exists"); - app_state.active_provider.refresh_provider_reservation(&old_binding.account_name, &old_binding.session_owner, 60); - app_state.active_provider.confirm_playback_activity(&old_binding.session_owner); - // The binding acquired a reservation without a handle; capture its tag so the - // preempt path can prove it still targets this exact incarnation. - let binding_tag = app_state.active_provider.binding_tag_for_owner(&old_binding.session_owner); - old_session.write().await.origin_account_binding.as_mut().expect("binding exists").provider_binding_tag = - binding_tag; - - let new_session = create_unbound_hls_test_session(&app_state, &input, "new", 12_000).await; - let new_proxy_session_id = new_session.read().await.proxy_session_id.clone(); - let prepared_origin = super::prepare_hls_origin_runtime( - &app_state, - &new_session, - &input, - "http://account.example.com/live/account-user/account-pass/new.m3u8", - "http://account.example.com/live/account-user/account-pass/new.m3u8", - &new_proxy_session_id, - &test_fingerprint_with_addr(test_addr_with_port(55201)), - ConnectionKind::Normal, - 0, - super::HlsOriginWorkKind::Manifest, - super::HlsOriginWorkClass::ManifestInteractive, - 21_000, - ) - .await - .expect("interactive work should use soft-active overlap before grace"); - - let binding = - prepared_origin.origin_account_binding_to_store.as_ref().expect("speculative binding should be prepared"); - assert_eq!(binding.account_name, input.name); - assert!(matches!( - &binding.binding_mode, - HlsOriginAccountBindingMode::Speculative { - displaced_proxy_session_id, - .. - } if displaced_proxy_session_id == &old_proxy_session_id - )); - assert!(matches!( - prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), - Some(super::ProviderAllocation::Available(_)) - )); - - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); -} - -#[tokio::test] -async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() { - let input = single_hls_provider_input("policy-preempt-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let soft_session = create_bound_hls_test_session(&app_state, &input, "soft", input.name.as_ref(), 1_000).await; - let soft_generation = { - let mut session = soft_session.write().await; - session.target_duration = Some(10); - session.mark_authorized_media_access(10_000); - session.activity.last_delivered_media_at_ms = Some(10_000); - session.reconcile_effective_origin_acquire_policy( - Some(HlsEffectiveOriginAcquirePolicy::new(ConnectionKind::Soft, 0, 10_000)), - 10_000, - ); - session.activity.origin_work_generation - }; - let soft_binding = soft_session.read().await.origin_account_binding.clone().expect("soft binding exists"); - app_state.active_provider.refresh_provider_reservation(&soft_binding.account_name, &soft_binding.session_owner, 60); - app_state.active_provider.confirm_playback_activity(&soft_binding.session_owner); - // The binding acquired a reservation without a handle; capture its tag so the - // preempt path can prove it still targets this exact incarnation. - let binding_tag = app_state.active_provider.binding_tag_for_owner(&soft_binding.session_owner); - soft_session.write().await.origin_account_binding.as_mut().expect("binding exists").provider_binding_tag = - binding_tag; - - let normal_session = create_unbound_hls_test_session(&app_state, &input, "normal", 10_500).await; - let normal_proxy_session_id = normal_session.read().await.proxy_session_id.clone(); - let prepared_origin = super::prepare_hls_origin_runtime( - &app_state, - &normal_session, - &input, - "http://account.example.com/live/account-user/account-pass/normal.m3u8", - "http://account.example.com/live/account-user/account-pass/normal.m3u8", - &normal_proxy_session_id, - &test_fingerprint_with_addr(test_addr_with_port(55231)), - ConnectionKind::Normal, - 0, - super::HlsOriginWorkKind::Manifest, - super::HlsOriginWorkClass::ManifestInteractive, - 10_500, - ) - .await - .expect("normal HLS policy should preempt active soft HLS binding"); - - let new_binding = prepared_origin - .origin_account_binding_to_store - .as_ref() - .expect("preempting session should receive active binding"); - assert_eq!(new_binding.account_name, input.name); - assert!(matches!(new_binding.binding_mode, HlsOriginAccountBindingMode::Active)); - assert!(matches!( - prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), - Some(super::ProviderAllocation::Available(_)) - )); - let soft_session = soft_session.read().await; - assert_eq!(soft_session.activity.origin_work_generation, soft_generation + 1); - assert!(matches!( - soft_session.origin_account_binding.as_ref().map(|binding| &binding.binding_mode), - Some(HlsOriginAccountBindingMode::Detached { - reason: HlsOriginAccountDetachedReason::PreemptedByHigherPriority, - .. - }) - )); - drop(soft_session); - - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); -} - -#[tokio::test] -async fn preempted_origin_runtime_commits_low_priority_tail_without_redirect_or_origin_fetch() { - let fixture = runtime_policy_endpoint_fixture(true).await; - let input = single_hls_provider_input("runtime-preempted-input"); - let session = fixture - .app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&fixture.proxy_session_id) - .await - .expect("runtime policy session"); - let now_ms = super::current_time_millis(); - let mut binding = HlsOriginAccountBinding::new( - Arc::clone(&input.name), - Arc::from("preempted-account"), - &fixture.proxy_session_id, - now_ms, - ); - binding.detach(HlsOriginAccountDetachedReason::PreemptedByHigherPriority, now_ms); - session.write().await.replace_origin_account_binding(Some(binding)); - let origin_refresh_before = session.read().await.origin_refresh.clone(); - let origin = super::HlsCacheManifestOrigin { - raw_request_url: "http://account.example.com/live/account-user/account-pass/12345.m3u8", - session_entry_url: super::HlsOriginEntryUrl::direct_http( - "http://account.example.com/live/account-user/account-pass/12345.m3u8", - ), - input: &input, - origin_source: super::build_hls_origin_source(&input, "12345"), - }; - let context = test_hls_access_context(fixture.proxy_session_id.clone(), fixture.lease_id.clone()); - - let result = super::prepare_hls_canonical_manifest_origin_runtime( - &fixture.app_state, - &session, - &context, - &origin, - &fixture.proxy_session_id, - &fixture.lease_id, - HlsAccessLeaseState::Activated, - &test_fingerprint(), - None, - now_ms, - ) - .await; - let Err(response) = result else { - panic!("detached origin binding must resolve to the lease-bound policy tail"); - }; - - assert!(!response.headers().contains_key(header::LOCATION)); - let plan = wait_for_runtime_policy_terminal_plan(&fixture).await; - assert_eq!(plan.reason, HlsRuntimeCustomTailReason::LowPriorityPreempted); - assert_eq!(plan.segment_duration_ms, 10_027); - assert_eq!(session.read().await.origin_refresh, origin_refresh_before); - let replay = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; - assert_eq!(replay.status(), StatusCode::OK); - assert!(!replay.headers().contains_key(header::LOCATION)); - let body = String::from_utf8(response_body(replay).await.to_vec()).expect("preemption manifest utf8"); - assert!(body.contains("/terminal/")); - assert!(!body.contains("/cvs/hls/")); - assert!(body.ends_with("#EXT-X-ENDLIST\n")); -} - -#[tokio::test] -async fn hls_origin_policy_preemption_rejects_soft_request_against_active_normal_binding() { - let input = single_hls_provider_input("policy-no-preempt-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let normal_session = create_bound_hls_test_session(&app_state, &input, "normal", input.name.as_ref(), 1_000).await; - { - let mut session = normal_session.write().await; - session.target_duration = Some(10); - session.mark_authorized_media_access(10_000); - session.reconcile_effective_origin_acquire_policy( - Some(HlsEffectiveOriginAcquirePolicy::new(ConnectionKind::Normal, 0, 10_000)), - 10_000, - ); - } - let normal_binding = normal_session.read().await.origin_account_binding.clone().expect("normal binding exists"); - app_state.active_provider.refresh_provider_reservation( - &normal_binding.account_name, - &normal_binding.session_owner, - 60, - ); - app_state.active_provider.confirm_playback_activity(&normal_binding.session_owner); - let soft_session = create_unbound_hls_test_session(&app_state, &input, "soft", 10_500).await; - let soft_proxy_session_id = soft_session.read().await.proxy_session_id.clone(); - - let result = super::prepare_hls_origin_policy_preempt_runtime( - &app_state, - &soft_session, - &input, - "http://account.example.com/live/account-user/account-pass/soft.m3u8", - "http://account.example.com/live/account-user/account-pass/soft.m3u8", - &soft_proxy_session_id, - &test_fingerprint_with_addr(test_addr_with_port(55232)), - ConnectionKind::Soft, - -100, - crate::model::PlaybackKind::LiveHls, - 10_500, - ) - .await; - - assert!(result.is_err()); - assert!(matches!( - normal_session.read().await.origin_account_binding.as_ref().map(|binding| &binding.binding_mode), - Some(HlsOriginAccountBindingMode::Active) - )); - assert!(app_state - .active_provider - .is_provider_reserved_for_other_session(&normal_binding.account_name, Some("unrelated-session"))); -} - -#[tokio::test] -async fn hls_origin_runtime_uses_grace_as_interactive_fallback_after_overlap_fails() { - let input = single_hls_provider_input("grace-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let occupied = app_state - .active_provider - .acquire_connection_with_grace_for_session( - &input.name, - &test_addr_with_port(55211), - false, - 0, - ConnectionKind::Normal, - Some("external-owner"), - ) - .expect("test should occupy the only provider account"); - let session = create_unbound_hls_test_session(&app_state, &input, "12345", 2_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - - let prepared_origin = super::prepare_hls_origin_runtime( - &app_state, - &session, - &input, - "http://account.example.com/live/account-user/account-pass/12345.m3u8", - "http://account.example.com/live/account-user/account-pass/12345.m3u8", - &proxy_session_id, - &test_fingerprint_with_addr(test_addr_with_port(55212)), - ConnectionKind::Normal, - 0, - super::HlsOriginWorkKind::Manifest, - super::HlsOriginWorkClass::ManifestInteractive, - 2_000, - ) - .await - .expect("interactive work can use grace when normal acquire and overlap fail"); - - assert!(matches!( - prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), - Some(super::ProviderAllocation::GracePeriod(_)) - )); - assert_eq!( - prepared_origin - .origin_account_binding_to_store - .as_ref() - .expect("grace binding should still bind the selected account") - .account_name - .as_ref(), - input.name.as_ref() - ); - - app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); - app_state.connection_manager.release_provider_handle(Some(occupied)); -} - -#[tokio::test] -async fn hls_provider_exhausted_without_provisioning_returns_custom_manifest() { - let input = single_hls_provider_input("provider-exhausted-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - store_test_sources_with_target( - &app_state, - input.clone(), - ConfigTarget::from(&ConfigTargetDto { id: 1, name: "default".to_string(), ..Default::default() }), - ); - enable_provider_exhausted_custom_response(&app_state); - let session = create_unbound_hls_test_session(&app_state, &input, "provider-exhausted-session", 1_000).await; - let access_lease_id = HlsAccessLeaseId("provider-exhausted-lease".to_string()); - let proxy_session_id = session.read().await.proxy_session_id.clone(); - prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; - let strip = app_state.hls.proxy.strip(); - - let response = super::hls_shared_provisioning_or_provider_exhausted_response( - &app_state, - &session, - "hls-user", - &input, - 59, - &access_lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - ) - .await; - - assert_eq!(response.status(), StatusCode::OK); - let body = response_body(response).await; - let manifest = String::from_utf8(body.to_vec()).expect("manifest is utf8"); - assert!(manifest.contains("#EXTM3U")); - assert!(manifest.contains(&format!("/cvs/hls/{}/", access_lease_id.0))); - assert!(!manifest.contains("/hls-user/")); - assert!(!manifest.contains("/hls-pass/")); - assert!(!manifest.contains("/provider_connections_exhausted/")); -} - -#[tokio::test] -async fn hls_provider_exhausted_grace_hold_waits_for_grace_period_before_retry() { - let input = single_hls_provider_input("provider-grace-hold-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - app_state.app_config.config.store(Arc::new(Config { - reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { - stream: Some(StreamConfigDto { - grace_period_millis: 20, - grace_period_timeout_secs: 10, - grace_period_hold_stream: true, - ..Default::default() - }), - ..Default::default() - })), - ..Default::default() - })); - let session = create_unbound_hls_test_session(&app_state, &input, "provider-grace-session", 1_000).await; - let access_lease_id = HlsAccessLeaseId("provider-grace-lease".to_string()); - let strip = app_state.hls.proxy.strip(); - let resolution = super::hls_provider_connections_exhausted_manifest_resolution( - &app_state, - &session, - "hls-user", - &input, - 59, - &access_lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - true, - ); - - let resolution = - tokio::time::timeout(Duration::from_millis(500), resolution).await.expect("grace hold deadline should wake"); - - assert!(matches!(resolution, super::HlsProviderExhaustedResolution::RetryAcquire)); -} - -#[tokio::test] -async fn provider_grace_expiry_commits_lease_bound_provider_exhausted_tail() { - let fixture = runtime_policy_endpoint_fixture(true).await; - let input = single_hls_provider_input("runtime-provider-grace-input"); - store_test_sources_with_target( - &fixture.app_state, - input.clone(), - ConfigTarget::from(&ConfigTargetDto { id: 1, name: "default".to_string(), ..Default::default() }), - ); - let current = fixture.app_state.app_config.config.load(); - fixture.app_state.app_config.config.store(Arc::new(Config { - reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { - stream: Some(StreamConfigDto { - grace_period_millis: 20, - grace_period_timeout_secs: 10, - grace_period_hold_stream: true, - ..Default::default() - }), - ..Default::default() - })), - ..current.as_ref().clone() - })); - let strip = fixture.app_state.hls.proxy.strip(); - let session = fixture - .app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&fixture.proxy_session_id) - .await - .expect("runtime provider session"); - - let grace = super::hls_provider_connections_exhausted_manifest_resolution( - &fixture.app_state, - &session, - "hls-user", - &input, - 12345, - &fixture.lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - true, - ) - .await; - assert!(matches!(grace, super::HlsProviderExhaustedResolution::RetryAcquire)); - - let response = super::hls_provider_connections_exhausted_manifest_resolution( - &fixture.app_state, - &session, - "hls-user", - &input, - 12345, - &fixture.lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - false, - ) - .await; - let super::HlsProviderExhaustedResolution::Response(response) = response else { - panic!("expired grace must resolve to a finite response"); - }; - assert!(!response.headers().contains_key(header::LOCATION)); - - let plan = wait_for_runtime_policy_terminal_plan(&fixture).await; - assert_eq!(plan.reason, HlsRuntimeCustomTailReason::ProviderConnectionsExhausted); - assert_eq!(plan.segment_duration_ms, 10_027); - let replay = super::hls_provider_connections_exhausted_manifest_resolution( - &fixture.app_state, - &session, - "hls-user", - &input, - 12345, - &fixture.lease_id, - HlsAccessLeaseState::Denied, - &strip, - None, - false, - ) - .await; - let super::HlsProviderExhaustedResolution::Response(replay) = replay else { - panic!("committed provider tail must replay"); - }; - assert_eq!(replay.status(), StatusCode::OK); - assert!(!replay.headers().contains_key(header::LOCATION)); - let body = String::from_utf8(response_body(replay).await.to_vec()).expect("provider terminal manifest utf8"); - assert!(body.contains("/terminal/")); - assert!(!body.contains("/cvs/hls/")); - assert!(body.ends_with("#EXT-X-ENDLIST\n")); -} - -#[tokio::test] -async fn shared_provisioning_timeline_manifest_uses_canonical_hls_session_segments() { - let input = single_hls_provider_input("shared-provisioning-timeline-input"); - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy_and_inputs(hls_proxy, vec![Arc::new(input.clone())]); - enable_hls_provisioning_custom_response(&app_state); - let session = create_unbound_hls_test_session(&app_state, &input, "shared-prov-timeline-12345", 1_000).await; - let access_lease_id = HlsAccessLeaseId("timeline-lease".to_string()); - let proxy_session_id = session.read().await.proxy_session_id.clone(); - activate_test_hls_access_lease( - &app_state, - &proxy_session_id, - &access_lease_id.0, - super::current_time_millis(), - 60_000, - ) - .await; - let strip = app_state.hls.proxy.strip(); - - let response = super::hls_shared_provisioning_timeline_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - ) - .await - .expect("provisioning manifest should render"); - - assert_eq!(response.status(), StatusCode::OK); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest is utf8"); - assert!(body.contains("#EXT-X-VERSION:7\n")); - assert!(body.contains("#EXT-X-INDEPENDENT-SEGMENTS\n")); - assert!(body.contains("#EXT-X-TARGETDURATION:2\n")); - assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0\n")); - assert!(body.contains("#EXTINF:2.000,\n")); - assert!(!body.contains("#EXTINF:12.000,\n")); - assert!(body.contains("/hls/shared/live/")); - assert!(body.contains("/000000.ts?pseq=0")); - assert!(body.contains("/000001.ts?pseq=1")); - assert!(body.contains("/000002.ts?pseq=2")); - assert!(body.matches("#EXTINF:").count() <= 6); - assert!(!body.contains("/cvs/hls/")); - { - let session = session.read().await; - assert_eq!(session.proxy_next_seq, Some(3)); - assert_eq!(session.publishable_origin_head_proxy_seq, Some(0)); - assert_eq!(session.publishable_origin_tail_proxy_seq, Some(2)); - assert_eq!(session.segments.len(), 3); - } - - let response = super::hls_shared_provisioning_timeline_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - ) - .await - .expect("subsequent manifest should append one segment"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest is utf8"); - assert!(body.contains("/000003.ts?pseq=3")); - assert!(body.matches("#EXTINF:").count() <= 6); - assert_eq!(session.read().await.proxy_next_seq, Some(4)); -} - -#[tokio::test] -async fn stale_provisioning_segments_do_not_trigger_canonical_handoff() { - let input = single_hls_provider_input("stale-provisioning-handoff-input"); - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy_and_inputs(hls_proxy, vec![Arc::new(input.clone())]); - enable_hls_provisioning_custom_response(&app_state); - let session = create_unbound_hls_test_session(&app_state, &input, "54321", 1_000).await; - let initial_lease_id = HlsAccessLeaseId("initial-provisioning-lease".to_string()); - let strip = app_state.hls.proxy.strip(); - - super::hls_shared_provisioning_timeline_manifest_response( - &app_state, - &session, - &initial_lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - ) - .await - .expect("provisioning manifest should render local segments"); - { - let session_guard = session.read().await; - assert!(session_guard.segments.values().any(crate::api::model::is_hls_provisioning_segment)); - assert_eq!(session_guard.segments.len(), 3); - assert_eq!(session_guard.pending_handoff_discontinuity_sequence, None); - } - - let new_lease_id = HlsAccessLeaseId("new-playback-lease".to_string()); - let previous_rendered_at = super::maybe_mark_hls_provisioning_handoff_for_canonical_manifest( - &app_state, - &session, - &input, - 54321, - &new_lease_id, - 2_000, - ) - .await; - - assert_eq!(previous_rendered_at, None); - let session_guard = session.read().await; - assert_eq!(session_guard.segments.len(), 3); - assert_eq!(session_guard.pending_handoff_discontinuity_sequence, None); -} - -#[tokio::test] -async fn provisioning_handoff_finds_shared_session_by_input_stream_id_not_virtual_id() { - let input = single_hls_provider_input("origin-id-handoff-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - enable_hls_cache(&app_state); - let origin_session = create_unbound_hls_test_session(&app_state, &input, "80510", 1_000).await; - let virtual_id_session = create_unbound_hls_test_session(&app_state, &input, "1001", 1_000).await; - let stream_identity = super::HlsEntryStreamIdentity::new(1001, "80510").expect("input stream identity"); - - assert!( - super::mark_hls_provisioning_handoff_discontinuity(&app_state, &input, &stream_identity, None, 2_000,).await - ); - - assert!(origin_session.read().await.pending_handoff_discontinuity_sequence.is_some()); - assert_eq!(virtual_id_session.read().await.pending_handoff_discontinuity_sequence, None); -} - -#[tokio::test] -async fn shared_provisioning_handoff_continues_proxy_sequence_for_origin_segments() { - let input = single_hls_provider_input("shared-provisioning-handoff-input"); - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy_and_inputs(hls_proxy, vec![Arc::new(input.clone())]); - enable_hls_provisioning_custom_response(&app_state); - let session = create_unbound_hls_test_session(&app_state, &input, "shared-handoff-12345", 1_000).await; - let access_lease_id = HlsAccessLeaseId("handoff-lease".to_string()); - let strip = app_state.hls.proxy.strip(); - super::hls_shared_provisioning_timeline_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - ) - .await - .expect("provisioning manifest should render"); - - let manifest = match parse_origin_media_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:4025\n#EXTINF:4.0,\n4025.ts\n#EXTINF:4.0,\n4026.ts\n#EXTINF:4.0,\n4027.ts\n", - "http://origin.example/live/stream.m3u8", - ) { - OriginManifestParseOutcome::Normal(manifest) => manifest, - OriginManifestParseOutcome::TransientPassthrough { reason } => { - panic!("expected normal manifest: {reason:?}") - } - }; - let rendered = { - let mut session_guard = session.write().await; - session_guard.mark_pending_handoff_discontinuity(0); - drop(session_guard); - assert!( - super::ensure_shared_hls_provisioning_handoff_gap(&app_state, &session, 2_000).await, - "handoff should append one gap segment" - ); - let mut session_guard = session.write().await; - session_guard.apply_origin_manifest(&manifest).expect("origin manifest should map"); - for proxy_seq in 4..=6 { - session_guard.segments.get_mut(&proxy_seq).expect("origin segment").status = - SegmentCacheStatus::Ready { content_length: 1024, ready_at_ms: 2_000 }; - } - session_guard.render_and_store_manifest(2_000).expect("handoff manifest should render") - }; - - assert!(rendered.body.contains("#EXT-X-MEDIA-SEQUENCE:1\n")); - assert!(rendered.body.contains("#EXT-X-TARGETDURATION:4\n")); - assert!(rendered.body.contains("/000002.ts?pseq=2")); - assert!(rendered.body.contains("/000004.ts")); - assert!(rendered.body.contains("/000005.ts")); - assert!(rendered.body.contains("/000006.ts")); - assert!(!rendered.body.contains("/004025.ts")); - let provisioning_tail = rendered.body.find("/000002.ts?pseq=2").expect("provisioning tail is rendered"); - let gap_tag = rendered.body.find("#EXT-X-GAP\n").expect("handoff gap tag is rendered"); - let gap_uri = rendered.body.find("/000003.ts?pseq=3").expect("handoff gap uri is rendered"); - let discontinuity = rendered - .body - .find("#EXT-X-DISCONTINUITY\n#EXTINF:4.000,\n/hls/shared/live/") - .expect("origin handoff discontinuity is rendered"); - let first_origin = rendered.body.find("/000004.ts").expect("first origin segment is rendered"); - assert!(provisioning_tail < gap_tag); - assert!(gap_tag < gap_uri); - assert!(gap_uri < discontinuity); - assert!(discontinuity < first_origin); -} - -#[tokio::test] -async fn hls_origin_runtime_background_skips_soft_overlap_and_grace() { - let input = single_hls_provider_input("background-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let old_session = create_bound_hls_test_session(&app_state, &input, "old", input.name.as_ref(), 1_000).await; - { - let mut session = old_session.write().await; - session.target_duration = Some(10); - session.mark_authorized_media_access(1_000); - session.activity.last_delivered_media_at_ms = Some(1_000); - } - let old_binding = old_session.read().await.origin_account_binding.clone().expect("binding exists"); - app_state.active_provider.refresh_provider_reservation(&old_binding.account_name, &old_binding.session_owner, 60); - app_state.active_provider.confirm_playback_activity(&old_binding.session_owner); - let new_session = create_unbound_hls_test_session(&app_state, &input, "new", 12_000).await; - let new_proxy_session_id = new_session.read().await.proxy_session_id.clone(); - - let result = super::prepare_hls_origin_runtime( - &app_state, - &new_session, - &input, - "http://account.example.com/live/account-user/account-pass/new.m3u8", - "http://account.example.com/live/account-user/account-pass/new.m3u8", - &new_proxy_session_id, - &test_fingerprint_with_addr(test_addr_with_port(55221)), - ConnectionKind::Normal, - 0, - super::HlsOriginWorkKind::Segment, - super::HlsOriginWorkClass::Background, - 12_000, - ) - .await; - - assert_eq!( - result.err(), - Some(super::HlsOriginRuntimeAcquireError::NoAccountAvailable { - reason: super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted - }) - ); - let old_session = old_session.read().await; - assert!(matches!( - old_session.origin_account_binding.as_ref().expect("old binding remains").binding_mode, - HlsOriginAccountBindingMode::Active - )); - assert!(new_session.read().await.origin_account_binding.is_none()); -} - -#[test] -fn hls_detached_origin_binding_reclaimed_by_owner_maps_to_preempted_no_account_reason() { - let proxy_session_id = ProxySessionId("preempted-session".to_string()); - let mut binding = - HlsOriginAccountBinding::new(Arc::from("input-a"), Arc::from("account-a"), &proxy_session_id, 1_000); - binding.detach(HlsOriginAccountDetachedReason::ReclaimedByOriginalOwner, 2_000); - - assert_eq!( - super::hls_no_account_reason_for_binding(Some(&binding)), - super::HlsOriginRuntimeNoAccountReason::OriginBindingPreempted - ); -} - -#[test] -fn hls_detached_origin_binding_soft_window_elapsed_maps_to_exhausted_no_account_reason() { - let proxy_session_id = ProxySessionId("soft-window-session".to_string()); - let mut binding = - HlsOriginAccountBinding::new(Arc::from("input-a"), Arc::from("account-a"), &proxy_session_id, 1_000); - binding.detach(HlsOriginAccountDetachedReason::SoftWindowElapsed, 2_000); - - assert_eq!( - super::hls_no_account_reason_for_binding(Some(&binding)), - super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted - ); -} - -#[tokio::test] -async fn hls_account_binding_without_media_activity_expires_after_startup_window() { - let input = overlap_provider_input(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; - { - let mut session = session.write().await; - session.target_duration = Some(12); - session.mark_authorized_manifest_access(1_000); - } - let old_generation = session.read().await.activity.origin_work_generation; - - super::detach_unprotected_hls_origin_account_bindings(&app_state, 5_000).await; - assert!(session.read().await.origin_account_binding.as_ref().is_some_and(|binding| binding.is_active())); - super::detach_unprotected_hls_origin_account_bindings(&app_state, 60_000).await; - - let session = session.read().await; - let binding = session.origin_account_binding.as_ref().expect("binding remains"); - assert!(matches!( - binding.binding_mode, - HlsOriginAccountBindingMode::Detached { reason: HlsOriginAccountDetachedReason::SoftWindowElapsed, .. } - )); - assert_eq!(session.activity.origin_work_generation, old_generation + 1); - assert_eq!(session.activity.last_authorized_media_at_ms, None); - assert_eq!(session.account_overlap_timing().target_duration_ms, 12_000); -} - -#[tokio::test] -async fn hls_hard_manifest_failure_forces_next_fresh_commit() { - let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); - { - let mut session = session.write().await; - session.last_rendered_manifest = Some(RenderedManifest { - body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), - first_proxy_seq: 1, - last_proxy_seq: 1, - playlist_duration_ms: 4_000, - valid_until_ms: 5_000, - render_gap_segments: 0, - rendered_at_ms: 1_000, - discontinuity_sequence: 0, - target_duration_ms: 4_000, - segment_proxy_seqs: vec![1], - }); - session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::PreviousHardManifestFailure); - } - - assert_eq!( - super::hls_manifest_commit_requirement(&session, HlsSessionStoreOutcome::Reused, None, 2_000).await, - HlsManifestCommitRequirement::FreshCommitRequired { - reason: HlsFreshManifestRequiredReason::PreviousHardManifestFailure - } - ); -} - -#[tokio::test] -async fn hls_normal_expired_session_allows_committed_manifest_while_manifest_valid() { - let now_ms = 100_000; - let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); - { - let mut session = session.write().await; - let proxy_session_id = session.proxy_session_id.clone(); - session.target_duration = Some(10); - session.mark_authorized_media_access(1_000); - session.origin_account_binding = Some(HlsOriginAccountBinding::new( - Arc::from("test-input"), - Arc::from("test-account"), - &proxy_session_id, - now_ms, - )); - session.last_rendered_manifest = Some(RenderedManifest { - body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), - first_proxy_seq: 1, - last_proxy_seq: 1, - playlist_duration_ms: 4_000, - valid_until_ms: now_ms.saturating_add(10_000), - render_gap_segments: 0, - rendered_at_ms: now_ms.saturating_sub(1_000), - discontinuity_sequence: 0, - target_duration_ms: 4_000, - segment_proxy_seqs: vec![1], - }); - } - - assert_eq!( - super::hls_manifest_commit_requirement(&session, HlsSessionStoreOutcome::Reused, None, now_ms).await, - HlsManifestCommitRequirement::CommittedManifestAllowed - ); -} - -#[tokio::test] -async fn hls_normal_expired_session_requires_fresh_commit_after_manifest_validity() { - let now_ms = 100_000; - let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); - { - let mut session = session.write().await; - let proxy_session_id = session.proxy_session_id.clone(); - session.target_duration = Some(10); - session.mark_authorized_media_access(1_000); - session.origin_account_binding = Some(HlsOriginAccountBinding::new( - Arc::from("test-input"), - Arc::from("test-account"), - &proxy_session_id, - now_ms, - )); - session.last_rendered_manifest = Some(RenderedManifest { - body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), - first_proxy_seq: 1, - last_proxy_seq: 1, - playlist_duration_ms: 4_000, - valid_until_ms: now_ms.saturating_sub(1), - render_gap_segments: 0, - rendered_at_ms: now_ms.saturating_sub(10_000), - discontinuity_sequence: 0, - target_duration_ms: 4_000, - segment_proxy_seqs: vec![1], - }); - } - - assert_eq!( - super::hls_manifest_commit_requirement(&session, HlsSessionStoreOutcome::Reused, None, now_ms).await, - HlsManifestCommitRequirement::FreshCommitRequired { - reason: HlsFreshManifestRequiredReason::ExpiredRevalidation - } - ); -} - -#[tokio::test] -async fn hls_ready_cache_hit_does_not_require_origin_reacquire_when_binding_is_detached() { - let app_state = test_app_state(); - let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; - let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - { - let mut session = session.write().await; - session - .origin_account_binding - .as_mut() - .expect("binding exists") - .detach(HlsOriginAccountDetachedReason::Cleanup, 2_000); - session.segments.insert( - 123, - SegmentEntry { - origin_key: OriginSegmentKey { - origin_epoch: 0, - effective_host_id: 0, - host_local_sequence: 123, - host_local_index: 123, - }, - proxy_seq: 123, - duration_ms: 4_000, - proxy_file_ext: "ts".to_string(), - content_type: "video/mp2t".to_string(), - cache_key: SegmentCacheKey::new(proxy_session_id, 123, "ts"), - discontinuity_before: false, - program_date_time: None, - daterange_tags_before: Vec::new(), - origin_byte_range: None, - map_ref: None, - encryption: None, - origin_fetch_ref: Some(OriginSegmentFetchRef { - resolved_origin_url: "http://origin.example.com/123.ts".to_string(), - byte_range: None, - valid_until_ms: None, - }), - status: SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 1_000 }, - last_rendered_at_ms: None, - access: Arc::new(CacheAccessState::new()), - }, - ); - } - - let segment_file = HlsSegmentFile { proxy_seq: 123, extension: "ts".to_string() }; - assert!(!super::hls_segment_request_requires_origin_work(&session, &segment_file).await); - assert!(super::hls_origin_binding_needs_reacquire(&session).await); -} - -async fn activate_test_hls_access_lease( - app_state: &Arc, - proxy_session_id: &ProxySessionId, - lease_id: &str, - now_ms: u64, - ttl_ms: u64, -) { - let lease_id = HlsAccessLeaseId(lease_id.to_string()); - let valid_window_ms = ttl_ms.saturating_mul(10).max(ttl_ms); - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), - proxy_session_id.clone(), - "hls-user".to_string(), - "hls-session-token".to_string(), - 1, - "12345".to_string(), - 12345, - now_ms, - valid_window_ms, - )) - .await; - assert!(app_state - .hls - .proxy - .activate_access_lease( - &lease_id, - proxy_session_id, - now_ms, - HlsAccessLeaseTiming { active_window_ms: ttl_ms, valid_window_ms }, - ) - .await - .is_activated()); -} - -async fn register_test_hls_stream_for_lease_release( - app_state: &Arc, - session: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - provider: &str, -) { - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - user.max_connections = 1; - app_state - .active_users - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "hls-session-token", - virtual_id: 12345, - provider, - stream_url: "http://origin.example.com/live/user/pass/12345.m3u8", - addr: &test_addr(), - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - let mut stream_channel = - super::fallback_hls_cache_stream_channel(0, 12345, &session.read().await.origin_source, proxy_session_id); - stream_channel.shared = true; - stream_channel.shared_stream_id = Some(super::hls_cache_shared_stream_id(proxy_session_id)); - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: ConnectionKind::Normal, - priority: user.priority, - soft_priority: user.soft_priority, - fingerprint: &test_fingerprint(), - provider: Arc::from(provider), - stream_channel: &stream_channel, - user_agent: std::borrow::Cow::Borrowed("test"), - session_token: Some("hls-session-token"), - }) - .await; -} - -fn test_segment_entry(proxy_session_id: &ProxySessionId, proxy_seq: u64, status: SegmentCacheStatus) -> SegmentEntry { - SegmentEntry { - origin_key: OriginSegmentKey { - origin_epoch: 0, - effective_host_id: 0, - host_local_sequence: proxy_seq, - host_local_index: u32::try_from(proxy_seq).unwrap_or(u32::MAX), - }, - proxy_seq, - duration_ms: 4_000, - proxy_file_ext: "ts".to_string(), - content_type: "video/mp2t".to_string(), - cache_key: SegmentCacheKey::new(proxy_session_id.clone(), proxy_seq, "ts"), - discontinuity_before: false, - program_date_time: None, - daterange_tags_before: Vec::new(), - origin_byte_range: None, - map_ref: None, - encryption: None, - origin_fetch_ref: Some(OriginSegmentFetchRef { - resolved_origin_url: format!("http://origin.example.com/{proxy_seq}.ts"), - byte_range: None, - valid_until_ms: None, - }), - status, - last_rendered_at_ms: None, - access: Arc::new(CacheAccessState::new()), - } -} - -#[tokio::test] -async fn hls_access_lease_idle_releases_user_but_keeps_origin_binding_and_queues() { - let input = overlap_provider_input(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let binding = session.read().await.origin_account_binding.clone().expect("binding exists"); - let account_name = Arc::from("account-a"); - app_state.active_provider.refresh_provider_reservation(&account_name, &binding.session_owner, 60); - app_state.active_provider.confirm_playback_activity(&binding.session_owner); - assert!(app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner"))); - activate_test_hls_access_lease(&app_state, &proxy_session_id, "detach-lease", 1_000, 1_000).await; - register_test_hls_stream_for_lease_release(&app_state, &session, &proxy_session_id, input.name.as_ref()).await; - assert_eq!(app_state.active_users.active_streams().await.len(), 1); - app_state - .hls - .proxy - .sync_session_access_lease_count_and_detach_if_needed( - &app_state.active_users, - &app_state.active_provider, - &session, - &proxy_session_id, - 1_000, - ) - .await; - { - let mut session = session.write().await; - assert_eq!(session.activity.active_access_lease_count, 1); - session.segments.insert(1, test_segment_entry(&proxy_session_id, 1, SegmentCacheStatus::Discovered)); - session.queue_segment_fetch_candidate(1, SegmentFetchPriority::Prefetch, 1_100); - session.segments.insert( - 2, - test_segment_entry( - &proxy_session_id, - 2, - SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 1_000 }, - ), - ); - let map_id = ProxyMapId(1); - let mut map = MapEntry::new( - &proxy_session_id, - map_id, - OriginMapKey { - origin_epoch: 0, - resolved_origin_uri: "http://origin.example.com/init.mp4".to_string(), - byte_range: None, - }, - "mp4".to_string(), - ); - map.status = MapCacheStatus::Queued { queued_at_ms: 1_100 }; - session.maps.insert(map_id, map); - } - let old_generation = session.read().await.activity.origin_work_generation; - - app_state - .hls - .proxy - .sync_session_access_lease_count_and_detach_if_needed( - &app_state.active_users, - &app_state.active_provider, - &session, - &proxy_session_id, - 3_000, - ) - .await; - - { - let session = session.read().await; - assert_eq!(session.activity.active_access_lease_count, 0); - let binding = session.origin_account_binding.as_ref().expect("binding is retained"); - assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active)); - assert_eq!(session.activity.origin_work_generation, old_generation); - assert!(!session.segment_prefetch_queue.is_empty()); - assert!(matches!( - session.segments.get(&1).expect("queued segment remains").status, - SegmentCacheStatus::Queued { .. } - )); - assert!(matches!( - session.segments.get(&2).expect("ready segment remains").status, - SegmentCacheStatus::Ready { .. } - )); - assert!(matches!(session.maps.get(&ProxyMapId(1)).expect("map remains").status, MapCacheStatus::Queued { .. })); - } - assert!(app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner"))); - assert!(app_state.active_users.active_streams().await.is_empty()); -} - -#[tokio::test] -async fn hls_access_lease_sync_keeps_binding_and_queue_when_active_count_remains_positive() { - let input = overlap_provider_input(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - activate_test_hls_access_lease(&app_state, &proxy_session_id, "still-active-lease", 1_000, 10_000).await; - { - let mut session = session.write().await; - session.activity.active_access_lease_count = 1; - session.segments.insert(1, test_segment_entry(&proxy_session_id, 1, SegmentCacheStatus::Discovered)); - session.queue_segment_fetch_candidate(1, SegmentFetchPriority::Prefetch, 1_100); - } - let old_generation = session.read().await.activity.origin_work_generation; - - app_state - .hls - .proxy - .sync_session_access_lease_count_and_detach_if_needed( - &app_state.active_users, - &app_state.active_provider, - &session, - &proxy_session_id, - 1_500, - ) - .await; - - let session = session.read().await; - assert_eq!(session.activity.active_access_lease_count, 1); - assert!(matches!( - session.origin_account_binding.as_ref().expect("binding exists").binding_mode, - HlsOriginAccountBindingMode::Active - )); - assert_eq!(session.activity.origin_work_generation, old_generation); - assert!(!session.segment_prefetch_queue.is_empty()); - assert!(matches!( - session.segments.get(&1).expect("segment remains queued").status, - SegmentCacheStatus::Queued { .. } - )); -} - -#[tokio::test] -async fn hls_access_lease_sync_zero_to_zero_is_idempotent() { - let input = overlap_provider_input(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let old_generation = session.read().await.activity.origin_work_generation; - - app_state - .hls - .proxy - .sync_session_access_lease_count_and_detach_if_needed( - &app_state.active_users, - &app_state.active_provider, - &session, - &proxy_session_id, - 3_000, - ) - .await; - - let session = session.read().await; - assert_eq!(session.activity.active_access_lease_count, 0); - assert!(matches!( - session.origin_account_binding.as_ref().expect("binding exists").binding_mode, - HlsOriginAccountBindingMode::Active - )); - assert_eq!(session.activity.origin_work_generation, old_generation); -} - -#[tokio::test] -async fn hls_access_lease_gc_prepass_releases_user_without_detaching_origin_binding() { - let input = overlap_provider_input(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - activate_test_hls_access_lease(&app_state, &proxy_session_id, "gc-expired-lease", 1_000, 1_000).await; - app_state - .hls - .proxy - .sync_session_access_lease_count_and_detach_if_needed( - &app_state.active_users, - &app_state.active_provider, - &session, - &proxy_session_id, - 1_000, - ) - .await; - let old_generation = session.read().await.activity.origin_work_generation; - - app_state - .hls - .proxy - .sync_all_session_access_leases_and_detach_if_needed(&app_state.active_users, &app_state.active_provider, 3_000) - .await; - let _ = app_state.hls.proxy.run_garbage_collection_once(3_000).await.expect("gc should run"); - - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&proxy_session_id) - .await - .expect("detach keeps shared hls session"); - let session = session.read().await; - assert_eq!(session.activity.active_access_lease_count, 0); - assert!(matches!( - session.origin_account_binding.as_ref().expect("binding exists").binding_mode, - HlsOriginAccountBindingMode::Active - )); - assert_eq!(session.activity.origin_work_generation, old_generation); -} - -fn test_addr() -> SocketAddr { "127.0.0.1:55123".parse().unwrap_or_else(|_| unreachable!()) } - -fn test_addr_with_port(port: u16) -> SocketAddr { SocketAddr::from(([127, 0, 0, 1], port)) } - -fn test_fingerprint() -> Fingerprint { Fingerprint::new("test".to_string(), "127.0.0.1".to_string(), test_addr()) } - -fn test_fingerprint_with_addr(addr: SocketAddr) -> Fingerprint { - Fingerprint::new(format!("test-{}", addr.port()), "127.0.0.1".to_string(), addr) -} - -async fn create_active_hls_user_session(app_state: &Arc) { - create_active_hls_user_session_with( - app_state, - "hls-session-token", - "origin-provider", - "http://origin.example.com/live/12345.m3u8", - test_addr(), - ) - .await; -} - -async fn create_active_hls_user_session_with( - app_state: &Arc, - session_token: &str, - provider: &str, - stream_url: &str, - addr: SocketAddr, -) { - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - user.max_connections = 1; - app_state - .active_users - .create_user_session(CreateUserSessionParams { - user: &user, - session_token, - virtual_id: 12345, - provider, - stream_url, - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; -} - -async fn mark_hls_user_session_exhausted(app_state: &Arc) { - let user = app_state.app_config.get_user_credentials("hls-user").expect("configured HLS test user"); - let addr = test_addr(); - app_state - .active_users - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "hls-session-token", - virtual_id: 12345, - provider: "origin-provider", - stream_url: "http://origin.example.com/live/12345.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Exhausted, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; -} - -fn test_hls_access_context(proxy_session_id: ProxySessionId, access_lease_id: HlsAccessLeaseId) -> HlsAccessContext { - test_hls_access_context_with(proxy_session_id, access_lease_id, "hls-session-token", test_fingerprint().key) -} - -fn test_hls_access_context_with( - proxy_session_id: ProxySessionId, - access_lease_id: HlsAccessLeaseId, - user_session_token: &str, - client_fingerprint: String, -) -> HlsAccessContext { - HlsAccessContext { - username: "hls-user".to_string(), - user_session_token: user_session_token.to_string(), - proxy_session_id, - input_id: 1, - stream_ref: "12345".to_string(), - virtual_id: 12345, - known_bitrate_bps: None, - lease_id: access_lease_id, - family_key: HlsPlaybackFamilyKey::new("hls-user", client_fingerprint), - epg_reference_ts: None, - archive_origin_url: None, - } -} - -#[tokio::test] -async fn hls_cache_stream_channel_uses_archive_epg_context() { - let app_state = test_app_state(); - let mut access = test_hls_access_context( - ProxySessionId("proxy-archive".to_string()), - HlsAccessLeaseId("lease-archive".to_string()), - ); - access.epg_reference_ts = Some(1_784_898_000); - access.archive_origin_url = Some("http://provider/channel/timeshift_abs-1784898000.m3u8".to_string()); - let origin_source = - HlsOriginSource::new(1, Arc::from("test-input"), "80510", HlsOriginSourceKind::M3uMediaPlaylist) - .with_archive_reference(1_784_898_000); - - let channel = super::build_hls_cache_stream_channel( - &app_state, - &access, - &origin_source, - &ProxySessionId("proxy-archive".to_string()), - ) - .await; - - assert_eq!(channel.item_type, PlaylistItemType::Catchup); - assert_eq!(channel.cluster, XtreamCluster::Video); - assert_eq!(channel.epg_reference_ts, Some(1_784_898_000)); -} - -#[tokio::test] -async fn hls_cache_manifest_context_restores_leased_archive_origin() -> Result<(), StatusCode> { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let input = ConfigInput { - id: 1, - name: Arc::from("test-input"), - input_type: InputType::M3u, - enabled: true, - ..ConfigInput::default() - }; - let mut target = test_m3u_hls_share_target(); - target.name = "default".to_string(); - store_test_sources_with_target(&app_state, input.clone(), target.clone()); - cache_test_m3u_hls_item( - &app_state, - &target, - test_m3u_hls_item(&input, 12345, "80510", "http://provider/channel/mono.m3u8"), - ) - .await; - - let archive_url = "http://provider/channel/timeshift_abs-1784898000.m3u8"; - let mut access = test_hls_access_context( - ProxySessionId("proxy-archive".to_string()), - HlsAccessLeaseId("lease-archive".to_string()), - ); - access.stream_ref = "80510".to_string(); - access.epg_reference_ts = Some(1_784_898_000); - access.archive_origin_url = Some(archive_url.to_string()); - - let context = super::resolve_hls_playback_manifest_request_context(&app_state, &access, &HeaderMap::new()).await?; - - assert_eq!(context.hls_url, archive_url); - assert_eq!(context.origin_source.stream_ref, "80510"); - assert_eq!(context.origin_source.archive_reference, Some(1_784_898_000)); - Ok(()) -} - -async fn prepare_pending_test_hls_access_lease( - app_state: &Arc, - proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, -) { - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - access_lease_id.clone(), - HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), - proxy_session_id.clone(), - "hls-user".to_string(), - "hls-session-token".to_string(), - 1, - "12345".to_string(), - 12345, - super::current_time_millis(), - super::hls_access_lease_ttl_ms(app_state), - )) - .await; -} - -#[tokio::test] -async fn hls_cache_manifest_cold_start_synchronously_returns_initial_manifest() { - let origin = spawn_test_segment_origin( - b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:123\n#EXTINF:4.0,\n000123.ts\n#EXTINF:4.0,\n000124.ts\n#EXTINF:4.0,\n000125.ts\n", - ) - .await; - let input_name = Arc::::from("test-input"); - let input = ConfigInput { - id: 1, - name: Arc::clone(&input_name), - input_type: InputType::Xtream, - url: origin.base_url.clone(), - username: Some("user".to_string()), - password: Some("pass".to_string()), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - enable_hls_cache(&app_state); - create_active_hls_user_session(&app_state).await; - let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); - let origin_source = super::build_hls_origin_source(&input, "12345"); - let session_key = origin_source.session_key(); - let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); - prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; - - let response = super::try_hls_cache_canonical_manifest_response( - &app_state, - &test_fingerprint(), - &access_context, - &proxy_session_id, - &access_lease_id, - HlsAccessLeaseState::Pending, - super::HlsCacheManifestOrigin { - raw_request_url: &request_url, - session_entry_url: super::HlsOriginEntryUrl::direct_http(&request_url), - input: &input, - origin_source, - }, - HeaderMap::new(), - None, - "/live/hls-user/hls-pass/12345.m3u8", - super::HlsManifestRefreshOrdering::Background, - ) - .await - .expect("hls cache should handle valid live hls entrypoint"); - - assert_eq!(response.status(), StatusCode::OK); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); - assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0")); - assert!(body.contains(&format!("/hls/shared/live/{}/{}/000000.ts", proxy_session_id.0, access_lease_id.0))); - assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); - let session = app_state - .hls - .proxy - .sessions() - .get_by_key(&session_key) - .await - .expect("cold start should create shared hls session"); - { - let session = session.read().await; - assert!(session.last_rendered_manifest.is_some()); - let binding = session.origin_account_binding.as_ref().expect("plain http input still has account binding"); - assert_eq!(binding.input_name.as_ref(), "test-input"); - assert_eq!(binding.account_name.as_ref(), "test-input"); - } - assert!(app_state.active_users.active_streams().await.is_empty()); -} - -#[tokio::test] -async fn canonical_recovery_from_provisioning_marks_normal_handoff_boundary() { - let origin = spawn_test_segment_origin( - b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:123\n#EXTINF:4.0,\n000123.ts\n#EXTINF:4.0,\n000124.ts\n#EXTINF:4.0,\n000125.ts\n", - ) - .await; - let input = ConfigInput { - id: 1, - name: Arc::from("test-input"), - input_type: InputType::Xtream, - url: origin.base_url.clone(), - username: Some("user".to_string()), - password: Some("pass".to_string()), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - enable_hls_cache(&app_state); - create_active_hls_user_session(&app_state).await; - let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); - let origin_source = super::build_hls_origin_source(&input, "12345"); - let session_key = origin_source.session_key(); - let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); - prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; - app_state.hls.provisioning.touch_consumer(Arc::clone(&input.name), 12345, super::current_time_millis()); - - let response = super::try_hls_cache_canonical_manifest_response( - &app_state, - &test_fingerprint(), - &access_context, - &proxy_session_id, - &access_lease_id, - HlsAccessLeaseState::Pending, - super::HlsCacheManifestOrigin { - raw_request_url: &request_url, - session_entry_url: super::HlsOriginEntryUrl::direct_http(&request_url), - input: &input, - origin_source, - }, - HeaderMap::new(), - None, - "/live/hls-user/hls-pass/12345.m3u8", - super::HlsManifestRefreshOrdering::Background, - ) - .await - .expect("canonical hls cache should recover from provisioning"); - - assert_eq!(response.status(), StatusCode::OK); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); - assert!(body.contains("#EXT-X-DISCONTINUITY\n#EXTINF:4.000,")); - assert!(!app_state.hls.provisioning.has_consumer(&input.name, 12345, super::current_time_millis())); -} - -#[tokio::test] -async fn canonical_recovery_from_provisioning_marks_transient_handoff_boundary() { - let origin = spawn_test_segment_origin( - b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:123\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.key\"\n#EXTINF:4.0,\n000123.ts\n#EXTINF:4.0,\n000124.ts\n#EXTINF:4.0,\n000125.ts\n", - ) - .await; - let input = ConfigInput { - id: 1, - name: Arc::from("test-input"), - input_type: InputType::Xtream, - url: origin.base_url.clone(), - username: Some("user".to_string()), - password: Some("pass".to_string()), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - enable_hls_cache(&app_state); - create_active_hls_user_session(&app_state).await; - let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); - let origin_source = super::build_hls_origin_source(&input, "12345"); - let session_key = origin_source.session_key(); - let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); - prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; - app_state.hls.provisioning.touch_consumer(Arc::clone(&input.name), 12345, super::current_time_millis()); - - let response = super::try_hls_cache_canonical_manifest_response( - &app_state, - &test_fingerprint(), - &access_context, - &proxy_session_id, - &access_lease_id, - HlsAccessLeaseState::Activated, - super::HlsCacheManifestOrigin { - raw_request_url: &request_url, - session_entry_url: super::HlsOriginEntryUrl::direct_http(&request_url), - input: &input, - origin_source, - }, - HeaderMap::new(), - None, - "/live/hls-user/hls-pass/12345.m3u8", - super::HlsManifestRefreshOrdering::Background, - ) - .await - .expect("canonical hls cache should recover from provisioning"); - - assert_eq!(response.status(), StatusCode::OK); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); - assert!(body.contains("#EXT-X-DISCONTINUITY\n#EXTINF:4.0,")); - assert!(!app_state.hls.provisioning.has_consumer(&input.name, 12345, super::current_time_millis())); -} - -#[tokio::test] -async fn hls_cache_manifest_cold_start_supports_m3u_hls_origin_source() { - let origin = spawn_test_segment_origin( - b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:77\n#EXTINF:4.0,\nseg-77.ts\n#EXTINF:4.0,\nseg-78.ts\n#EXTINF:4.0,\nseg-79.ts\n", - ) - .await; - let input = ConfigInput { - id: 1, - name: Arc::from("m3u-input"), - input_type: InputType::M3u, - url: origin.base_url.clone(), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - enable_hls_cache(&app_state); - create_active_hls_user_session(&app_state).await; - let request_url = format!("{}/channel/index.m3u8", origin.base_url); - let origin_source = super::build_hls_origin_source(&input, "12345"); - let session_key = origin_source.session_key(); - let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); - prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; - - let response = super::try_hls_cache_canonical_manifest_response( - &app_state, - &test_fingerprint(), - &access_context, - &proxy_session_id, - &access_lease_id, - HlsAccessLeaseState::Pending, - super::HlsCacheManifestOrigin { - raw_request_url: &request_url, - session_entry_url: super::HlsOriginEntryUrl::direct_http(&request_url), - input: &input, - origin_source, - }, - HeaderMap::new(), - None, - "/m3u-stream/live/hls-user/hls-pass/12345.m3u8", - super::HlsManifestRefreshOrdering::Background, - ) - .await - .expect("hls cache should handle m3u hls media playlists"); - - assert_eq!(response.status(), StatusCode::OK); - let session = - app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("m3u hls should create shared session"); - let session = session.read().await; - assert_eq!(session.origin_source.source_kind, HlsOriginSourceKind::M3uMediaPlaylist); - let binding = session.origin_account_binding.as_ref().expect("m3u hls input still has account binding"); - assert_eq!(binding.input_name.as_ref(), "m3u-input"); - assert_eq!(binding.account_name.as_ref(), "m3u-input"); -} - -const AES_TEST_MANIFEST: &[u8] = b"#EXTM3U\n#EXT-X-VERSION:5\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:77\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\",KEYFORMAT=\"identity\",KEYFORMATVERSIONS=\"1\"\n#EXTINF:12,\n77.ts\n#EXTINF:12,\n78.ts\n#EXTINF:12,\n79.ts\n#EXTINF:12,\n80.ts\n#EXTINF:12,\n81.ts\n#EXTINF:12,\n82.ts\n"; -const AES_TEST_KEY_BYTES: &[u8] = b"0123456789abcdef"; -const AES_TEST_ROTATED_KEY_BYTES: &[u8] = b"fedcba9876543210"; -const AES_TEST_PLAINTEXT_SEGMENT: &[u8] = - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); - -struct AesEndpointFixture { - _temp_dir: tempfile::TempDir, - origin: TestSegmentOrigin, - input: ConfigInput, - app_state: Arc, - request_url: String, - session: HlsSessionHandle, - proxy_session_id: ProxySessionId, - access_lease_id: HlsAccessLeaseId, - key_uri: String, - base_manifest: HlsLeaseManifestSnapshot, - asset: Arc, -} - -async fn assert_aes_live_endpoint( - app_state: &Arc, - origin: &TestSegmentOrigin, - access_lease_id: &HlsAccessLeaseId, - live_body: &str, -) -> String { - let key_uri = live_body - .lines() - .find(|line| line.starts_with("#EXT-X-KEY:METHOD=AES-128")) - .and_then(|line| line.split_once("URI=\"").map(|(_, tail)| tail)) - .and_then(|tail| tail.split_once('"').map(|(uri, _)| uri.to_string())) - .expect("opaque key URI"); - assert!(key_uri.contains(&format!("/{}/r/", access_lease_id.0))); - assert!(!live_body.contains("key.bin")); - assert!(live_body.contains("#EXT-X-VERSION:5\n")); - let proxy_media_sequence = live_body - .lines() - .find_map(|line| line.strip_prefix("#EXT-X-MEDIA-SEQUENCE:")) - .and_then(|value| value.parse::().ok()) - .expect("proxy media sequence"); - assert_ne!(proxy_media_sequence, 77); - assert!(live_body.contains("IV=0x0000000000000000000000000000004d")); - assert_eq!(origin.key_request_count(), 1); - let segment_uris = live_body - .lines() - .filter(|line| line.starts_with("/hls/shared/live/") && !line.contains("/r/")) - .collect::>(); - assert_eq!(segment_uris.len(), 6); - for (index, segment_uri) in segment_uris.into_iter().enumerate() { - let response = get_response(Arc::clone(app_state), segment_uri, None).await; - assert_eq!(response.status(), StatusCode::OK); - let served = response_body(response).await; - let origin_sequence = 77_u64.saturating_add(u64::try_from(index).expect("test segment index")); - let expected = encrypt_test_aes128_cbc_pkcs7( - AES_TEST_PLAINTEXT_SEGMENT, - AES_TEST_KEY_BYTES, - test_hls_sequence_iv(origin_sequence), - ); - assert_eq!(served.as_ref(), expected.as_slice()); - assert_ne!(served.as_ref(), AES_TEST_PLAINTEXT_SEGMENT); - assert_eq!(origin.key_request_count(), 1); - } - key_uri -} - -async fn aes_endpoint_fixture() -> AesEndpointFixture { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let origin = spawn_test_encrypted_hls_origin( - AES_TEST_MANIFEST, - Arc::from(AES_TEST_KEY_BYTES), - Arc::from(AES_TEST_PLAINTEXT_SEGMENT), - ) - .await; - let input = ConfigInput { - id: 1, - name: Arc::from("encrypted-m3u-input"), - input_type: InputType::M3u, - url: origin.base_url.clone(), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let app_state = test_app_state_with_hls_proxy_and_inputs( - Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)), - vec![Arc::new(input.clone())], - ); - enable_hls_cache(&app_state); - create_active_hls_user_session(&app_state).await; - let request_url = format!("{}/channel/index.m3u8", origin.base_url); - let origin_source = super::build_hls_origin_source(&input, "12345"); - let session_key = origin_source.session_key(); - let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - let access_lease_id = HlsAccessLeaseId("encrypted-normal-lease".to_string()); - let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); - prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; - let response = super::try_hls_cache_canonical_manifest_response( - &app_state, - &test_fingerprint(), - &access_context, - &proxy_session_id, - &access_lease_id, - HlsAccessLeaseState::Pending, - super::HlsCacheManifestOrigin { - raw_request_url: &request_url, - session_entry_url: super::HlsOriginEntryUrl::direct_http(&request_url), - input: &input, - origin_source, - }, - HeaderMap::new(), - None, - "/m3u-stream/live/hls-user/hls-pass/12345.m3u8", - super::HlsManifestRefreshOrdering::Background, - ) - .await - .expect("compatible AES origin should enter normal HLS cache"); - assert_eq!(response.status(), StatusCode::OK); - let live_body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - let key_uri = assert_aes_live_endpoint(&app_state, &origin, &access_lease_id, &live_body).await; - assert_eq!(origin.segment_request_count(), 6); - let session = app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("normal encrypted session"); - assert_eq!(session.read().await.mode, HlsSessionMode::NormalCacheTimeline); - let lease = app_state - .hls - .proxy - .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::current_time_millis()) - .await - .expect("live encrypted lease snapshot"); - let base_manifest = lease.last_manifest_snapshot.expect("authoritative lease manifest"); - assert_eq!(base_manifest.delivery_mode, HlsManifestDeliveryMode::NormalCacheTimeline); - assert_eq!( - base_manifest.active_encryption.as_ref().and_then(|encryption| encryption.iv.as_deref()), - Some("0x00000000000000000000000000000052") - ); - let asset = terminal_test_asset(); - let evidence = prepare_terminal_base_evidence( - &session, - app_state.hls.proxy.segment_cache(), - &base_manifest, - super::current_time_millis(), - ) - .await; - assert_eq!(evidence.track_signature(), Some(asset.track_signature().clone())); - assert_eq!(evidence.key_bindings().len(), 1); - assert_eq!(origin.key_request_count(), 1); - evidence.release(); - AesEndpointFixture { - _temp_dir: temp_dir, - origin, - input, - app_state, - request_url, - session, - proxy_session_id, - access_lease_id, - key_uri, - base_manifest, - asset, - } -} - -async fn install_aes_terminal_plan(fixture: &AesEndpointFixture) -> TransientObjectCacheKey { - let key_response = get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, None).await; - assert_eq!(key_response.status(), StatusCode::OK); - assert_eq!(response_body(key_response).await.as_ref(), AES_TEST_KEY_BYTES); - assert_eq!(fixture.origin.key_request_count(), 1); - let evidence_key_id = { - let mut session = fixture.session.write().await; - let key_id = session - .transient - .resources - .values() - .find(|resource| resource.kind == TransientResourceKind::Key) - .map(|resource| resource.id.clone()) - .expect("normal key resource"); - session.last_rendered_manifest = None; - session.transient.last_manifest_body = None; - session.transient.resources.get_mut(&key_id).expect("key resource").expires_at_ms = 0; - for (key, object) in &mut session.transient.object_cache { - if key.transient_resource_id() == &key_id { - object.expires_at_ms = 0; - } - } - key_id - }; - let evidence = prepare_terminal_base_evidence( - &fixture.session, - fixture.app_state.hls.proxy.segment_cache(), - &fixture.base_manifest, - super::current_time_millis(), - ) - .await; - assert_eq!(evidence.track_signature(), Some(fixture.asset.track_signature().clone())); - assert_eq!(fixture.origin.key_request_count(), 1); - fixture - .app_state - .hls - .proxy - .run_garbage_collection_once(super::current_time_millis()) - .await - .expect("evidence-pinned GC"); - { - let session = fixture.session.read().await; - assert!(session.transient.resources.contains_key(&evidence_key_id)); - assert!(session.transient.object_cache.keys().any(|key| key.transient_resource_id() == &evidence_key_id)); - } - let plan = build_terminal_tail_plan(HlsTerminalTailBuildInput { - generation: HlsTerminalTailGeneration(23), - created_at_ms: super::current_time_millis(), - base_availability: evidence.availability(), - base_track_signature: evidence.track_signature(), - base_splice_evidence: evidence.splice_evidence().cloned(), - terminal_splice_evidence: Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&fixture.asset)), - base_timing: evidence.timing().cloned(), - base_key_bindings: evidence.key_bindings(), - expected_asset: HlsRuntimeCustomTailAssetIdentity::channel_unavailable(HlsTerminalAssetIdentity::from_asset( - &fixture.asset, - )), - base_manifest: fixture.base_manifest.clone(), - anchored_bundle: HlsTerminalTailBuildInput::anchored_bundle_for_test( - &fixture.asset, - fixture.base_manifest.target_duration_ms, - ), - asset: Arc::clone(&fixture.asset), - }) - .expect("READY AES key permits safe terminal reset"); - let protection = HlsTerminalTailProtection { - generation: plan.generation, - base_proxy_seqs: Arc::clone(&plan.protected_base_proxy_seqs), - key_bindings: plan.key_bindings(), - }; - assert_eq!(protection.key_bindings.len(), 1); - assert_eq!(protection.key_bindings[0].resource_id(), &evidence_key_id); - let frozen_source_cache_key = protection.key_bindings[0].source_cache_key().clone(); - { - let mut leases = fixture.app_state.hls.proxy.access_leases().write().await; - let mut lease = leases.remove_access_lease(&fixture.access_lease_id).expect("live lease"); - lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(Arc::new(plan)); - leases.prepare_access_lease(lease); - } - fixture.session.write().await.install_terminal_tail_protection(fixture.access_lease_id.clone(), protection); - evidence.release(); - frozen_source_cache_key -} - -async fn assert_aes_terminal_endpoints(fixture: &AesEndpointFixture) { - let manifest_uri = - format!("/hls/shared/live/{}/{}/manifest.m3u8", fixture.proxy_session_id.0, fixture.access_lease_id.0); - let response = get_response(Arc::clone(&fixture.app_state), &manifest_uri, None).await; - assert_eq!(response.status(), StatusCode::OK); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("terminal utf8"); - let key = body.find("#EXT-X-KEY:METHOD=AES-128").expect("base AES key"); - let reset = body.find("#EXT-X-KEY:METHOD=NONE").expect("clear key reset"); - let discontinuity = body[reset..].find("#EXT-X-DISCONTINUITY").expect("terminal discontinuity") + reset; - assert!(key < reset && reset < discontinuity); - assert!(body.contains("IV=0x00000000000000000000000000000052")); - assert!(body.contains("#EXT-X-VERSION:5\n")); - assert!(body.ends_with("#EXT-X-ENDLIST\n")); - let key_response = get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, None).await; - assert_eq!(key_response.status(), StatusCode::OK); - assert_eq!(response_body(key_response).await.as_ref(), AES_TEST_KEY_BYTES); - assert_eq!(fixture.origin.key_request_count(), 1); - let range = get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, Some("bytes=4-7")).await; - assert_eq!(range.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response_body(range).await.as_ref(), &AES_TEST_KEY_BYTES[4..=7]); - let unsatisfiable = get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, Some("bytes=16-")).await; - assert_eq!(unsatisfiable.status(), StatusCode::RANGE_NOT_SATISFIABLE); - assert_eq!(unsatisfiable.headers()[header::CONTENT_RANGE], "bytes */16"); -} - -async fn assert_aes_rotated_live_key( - fixture: &AesEndpointFixture, - frozen_source_cache_key: &TransientObjectCacheKey, -) -> String { - let manifest_requests_before = fixture.origin.manifest_request_count(); - let segment_requests_before = fixture.origin.segment_request_count(); - fixture.origin.set_key_bytes(Arc::from(AES_TEST_ROTATED_KEY_BYTES)).await; - let live_lease_id = HlsAccessLeaseId("encrypted-rotated-live-lease".to_string()); - let access_context = test_hls_access_context(fixture.proxy_session_id.clone(), live_lease_id.clone()); - prepare_pending_test_hls_access_lease(&fixture.app_state, &fixture.proxy_session_id, &live_lease_id).await; - let response = super::try_hls_cache_canonical_manifest_response( - &fixture.app_state, - &test_fingerprint(), - &access_context, - &fixture.proxy_session_id, - &live_lease_id, - HlsAccessLeaseState::Pending, - super::HlsCacheManifestOrigin { - raw_request_url: &fixture.request_url, - session_entry_url: super::HlsOriginEntryUrl::direct_http(&fixture.request_url), - input: &fixture.input, - origin_source: super::build_hls_origin_source(&fixture.input, "12345"), - }, - HeaderMap::new(), - None, - "/m3u-stream/live/hls-user/hls-pass/12345.m3u8", - super::HlsManifestRefreshOrdering::Background, - ) - .await - .expect("a new live lease should reuse the recovered shared session"); - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(fixture.origin.manifest_request_count(), manifest_requests_before.saturating_add(1)); - assert_eq!(fixture.origin.segment_request_count(), segment_requests_before); - let live_lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&live_lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await - .expect("rotated live lease snapshot"); - assert_eq!(live_lease.playback_mode, HlsLeasePlaybackMode::Live); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("rotated manifest utf8"); - let rotated_key_uri = body - .lines() - .find(|line| line.starts_with("#EXT-X-KEY:METHOD=AES-128")) - .and_then(|line| line.split_once("URI=\"").map(|(_, tail)| tail)) - .and_then(|tail| tail.split_once('"').map(|(uri, _)| uri.to_string())) - .expect("rotated live key URI"); - let key_response = get_response(Arc::clone(&fixture.app_state), &rotated_key_uri, None).await; - assert_eq!(key_response.status(), StatusCode::OK); - assert_eq!(response_body(key_response).await.as_ref(), AES_TEST_ROTATED_KEY_BYTES); - assert_eq!(fixture.origin.key_request_count(), 2); - assert!(fixture - .app_state - .hls - .proxy - .segment_cache() - .metadata(frozen_source_cache_key) - .await - .expect("A metadata lookup") - .is_none()); - let terminal_key = get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, None).await; - assert_eq!(terminal_key.status(), StatusCode::OK); - assert_eq!(response_body(terminal_key).await.as_ref(), AES_TEST_KEY_BYTES); - assert_eq!(fixture.origin.key_request_count(), 2); - rotated_key_uri -} - -async fn expire_aes_terminal_lease(fixture: &AesEndpointFixture, rotated_key_uri: &str) { - let expired_at_ms = super::current_time_millis().saturating_sub(1); - { - let mut leases = fixture.app_state.hls.proxy.access_leases().write().await; - let mut lease = leases.remove_access_lease(&fixture.access_lease_id).expect("terminal lease"); - lease.valid_until_ms = expired_at_ms; - leases.prepare_access_lease(lease); - } - fixture - .app_state - .hls - .proxy - .handle_lifecycle_event( - &fixture.app_state.active_users, - &fixture.app_state.active_provider, - HlsLifecycleEvent { - key: HlsLifecycleEventKey::AccessLeaseValidity { - lease_id: fixture.access_lease_id.clone(), - proxy_session_id: fixture.proxy_session_id.clone(), - }, - due_at_ms: expired_at_ms, - }, - super::current_time_millis(), - ) - .await; - fixture.app_state.hls.proxy.run_garbage_collection_once(super::current_time_millis()).await.expect("released GC"); - assert!(fixture.session.read().await.terminal_tail_protection(&fixture.access_lease_id).is_none()); - assert_eq!( - get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, None).await.status(), - StatusCode::NOT_FOUND - ); - let live_key = get_response(Arc::clone(&fixture.app_state), rotated_key_uri, None).await; - assert_eq!(live_key.status(), StatusCode::OK); - assert_eq!(response_body(live_key).await.as_ref(), AES_TEST_ROTATED_KEY_BYTES); -} - -#[tokio::test] -async fn aes_128_normal_origin_key_and_terminal_lifecycle_are_endpoint_safe() { - let fixture = aes_endpoint_fixture().await; - let frozen_source_cache_key = install_aes_terminal_plan(&fixture).await; - assert_aes_terminal_endpoints(&fixture).await; - let rotated_key_uri = assert_aes_rotated_live_key(&fixture, &frozen_source_cache_key).await; - expire_aes_terminal_lease(&fixture, &rotated_key_uri).await; -} - -#[tokio::test] -async fn hls_cache_manifest_unpublished_lease_uses_same_finite_fallback_for_created_and_reused_session() { - let mut rendered_bodies = Vec::new(); - for expected_outcome in [HlsSessionStoreOutcome::Created, HlsSessionStoreOutcome::Reused] { - let input_name = Arc::::from("test-input"); - let origin = spawn_test_status_origin(StatusCode::NOT_FOUND, b"missing").await; - let input = ConfigInput { - id: 1, - name: Arc::clone(&input_name), - input_type: InputType::Xtream, - url: origin.base_url.clone(), - username: Some("user".to_string()), - password: Some("pass".to_string()), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - enable_hls_cache(&app_state); - enable_channel_unavailable_custom_response(&app_state); - create_active_hls_user_session(&app_state).await; - let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); - let origin_source = super::build_hls_origin_source(&input, "12345"); - let session_key = origin_source.session_key(); - let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - if expected_outcome == HlsSessionStoreOutcome::Reused { - let (session, outcome) = app_state - .hls - .proxy - .get_or_create_session_with_source_and_outcome( - session_key, - origin_source.clone(), - &app_state.get_encrypt_secret(), - super::current_time_millis(), - ) - .await; - assert_eq!(outcome, HlsSessionStoreOutcome::Created); - session.write().await.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict; - } - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); - prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; - - let response = super::try_hls_cache_canonical_manifest_response( - &app_state, - &test_fingerprint(), - &access_context, - &proxy_session_id, - &access_lease_id, - HlsAccessLeaseState::Pending, - super::HlsCacheManifestOrigin { - raw_request_url: request_url.as_str(), - session_entry_url: super::HlsOriginEntryUrl::direct_http(request_url.as_str()), - input: &input, - origin_source, - }, - HeaderMap::new(), - None, - "/live/hls-user/hls-pass/12345.m3u8", - super::HlsManifestRefreshOrdering::Background, - ) - .await - .expect("hls cache should handle valid live hls entrypoint"); - - assert_eq!(response.status(), StatusCode::OK, "session outcome: {expected_outcome:?}"); - assert!(response.headers().get(header::RETRY_AFTER).is_none(), "custom response must not expose retry-after"); - assert!(!response.headers().contains_key(header::LOCATION)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - assert!(body.contains("#EXT-X-ENDLIST")); - assert!(!body.contains("/hls/shared/live/"), "standalone response must not expose an unready normal URI"); - rendered_bodies.push(body); - - let snapshot = app_state - .hls - .proxy - .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::current_time_millis()) - .await - .expect("lease remains available for strict cold-start handling"); - assert_eq!(snapshot.playback_mode, HlsLeasePlaybackMode::Live); - assert!(snapshot.last_manifest_snapshot.is_none()); - } - assert_eq!(rendered_bodies[0], rendered_bodies[1]); -} - -#[tokio::test] -async fn hls_cache_manifest_cold_start_client_abort_does_not_leave_refresh_in_flight() { - let origin = spawn_test_transient_origin_with_delayed_response( - "200 OK", - &[("Content-Type", "application/vnd.apple.mpegurl")], - "#EXTM3U\n#EXT-X-VERSION:3\n", - Duration::from_millis(100), - ) - .await; - let input = ConfigInput { - id: 1, - name: Arc::from("test-input"), - input_type: InputType::Xtream, - url: origin.base_url.clone(), - username: Some("user".to_string()), - password: Some("pass".to_string()), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let hls_dto = - HlsCacheConfigDto { origin_manifest_timeout_ms: shared::model::Millis::new(1_000), ..Default::default() }; - let hls_config = HlsCacheConfig::from(&hls_dto); - let app_state = test_app_state_with_hls_proxy_and_inputs( - Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config)), - vec![Arc::new(input.clone())], - ); - enable_hls_cache(&app_state); - create_active_hls_user_session(&app_state).await; - let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); - let origin_source = super::build_hls_origin_source(&input, "12345"); - let session_key = origin_source.session_key(); - let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); - prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; - - let app_state_for_request = Arc::clone(&app_state); - let request_handle = tokio::spawn(async move { - super::try_hls_cache_canonical_manifest_response( - &app_state_for_request, - &test_fingerprint(), - &access_context, - &proxy_session_id, - &access_lease_id, - HlsAccessLeaseState::Pending, - super::HlsCacheManifestOrigin { - raw_request_url: &request_url, - session_entry_url: super::HlsOriginEntryUrl::direct_http(&request_url), - input: &input, - origin_source, - }, - HeaderMap::new(), - None, - "/live/hls-user/hls-pass/12345.m3u8", - super::HlsManifestRefreshOrdering::Background, - ) - .await - }); - - let session = wait_for_hls_test_session(&app_state, &session_key).await; - wait_for_hls_refresh_in_flight(&session).await; - request_handle.abort(); - let _ = request_handle.await; - - for _ in 0..200 { - if !session.read().await.origin_refresh.in_flight { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - let session = session.read().await; - assert!(!session.origin_refresh.in_flight); - assert!(session.origin_refresh.last_fetch_finished_at_ms.is_some()); -} - -#[tokio::test] -async fn hls_cache_canonical_prepare_service_unavailable_sets_retry_after() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - create_active_hls_user_session(&app_state).await; - let input = ConfigInput { - id: 1, - name: Arc::from("alias-input"), - aliases: Some(vec![crate::model::ConfigInputAlias { - id: 2, - name: Arc::from("alias-account"), - url: "http://alias.example.com".to_string(), - username: Some("alias-user".to_string()), - password: Some("alias-pass".to_string()), - priority: 0, - max_connections: 1, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }; - let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; - let origin_source = super::build_hls_origin_source(&input, "12345"); - let session_key = origin_source.session_key(); - let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); - - let response = super::try_hls_cache_canonical_manifest_response( - &app_state, - &test_fingerprint(), - &access_context, - &proxy_session_id, - &access_lease_id, - HlsAccessLeaseState::Pending, - super::HlsCacheManifestOrigin { - raw_request_url: request_url, - session_entry_url: super::HlsOriginEntryUrl::direct_http(request_url), - input: &input, - origin_source, - }, - HeaderMap::new(), - None, - "/live/hls-user/hls-pass/12345.m3u8", - super::HlsManifestRefreshOrdering::Background, - ) - .await - .expect("canonical hls cache response"); - - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!(response.headers().get(header::RETRY_AFTER).expect("retry after"), "2"); -} - -#[tokio::test] -async fn hls_origin_account_rebind_failure_sets_backoff_without_changing_session_identity() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - create_active_hls_user_session(&app_state).await; - let input = ConfigInput { id: 1, name: Arc::from("stale-input"), ..ConfigInput::default() }; - let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; - let origin_source = super::build_hls_origin_source(&input, "12345"); - let session_key = origin_source.session_key(); - let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); - let (session, _) = app_state - .hls - .proxy - .get_or_create_session_with_source_and_outcome( - session_key.clone(), - origin_source.clone(), - &app_state.get_encrypt_secret(), - 1_000, - ) - .await; - { - let mut session_guard = session.write().await; - session_guard.origin_account_binding = Some(HlsOriginAccountBinding::new( - Arc::clone(&input.name), - Arc::from("removed-account"), - &proxy_session_id, - 1_000, - )); - } - - let response = super::try_hls_cache_canonical_manifest_response( - &app_state, - &test_fingerprint(), - &access_context, - &proxy_session_id, - &access_lease_id, - HlsAccessLeaseState::Pending, - super::HlsCacheManifestOrigin { - raw_request_url: request_url, - session_entry_url: super::HlsOriginEntryUrl::direct_http(request_url), - input: &input, - origin_source, - }, - HeaderMap::new(), - None, - "/live/hls-user/hls-pass/12345.m3u8", - super::HlsManifestRefreshOrdering::Background, - ) - .await - .expect("canonical hls cache response"); - - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!(response.headers().get(header::RETRY_AFTER).expect("retry after"), "2"); - let session_guard = session.read().await; - assert_eq!(session_guard.key, session_key); - assert_eq!(session_guard.proxy_session_id, proxy_session_id); - let binding = session_guard.origin_account_binding.as_ref().expect("stale binding remains"); - assert_eq!(binding.account_name.as_ref(), "removed-account"); - assert_eq!(binding.generation, 0); - assert_eq!(session_guard.origin_account_rebind.consecutive_rebind_failures, 1); - assert!(session_guard.origin_account_rebind.next_rebind_allowed_at_ms.is_some()); -} - -#[tokio::test] -async fn hls_cache_entry_returns_master_playlist_without_origin_refresh_or_session_creation() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; - let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; - let origin_source = super::build_hls_origin_source(&input, "12345"); - let session_key = origin_source.session_key(); - - let response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - origin_source, - 12345, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - Some("/iptv"), - ) - .await; - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - response.headers().get(header::CONTENT_TYPE).and_then(|value| value.to_str().ok()), - Some("application/vnd.apple.mpegurl") - ); - assert_eq!( - response.headers().get(header::CACHE_CONTROL).and_then(|value| value.to_str().ok()), - Some("private, no-store, no-cache, must-revalidate") - ); - assert!(response.headers().get(header::LOCATION).is_none()); - assert!(response.headers().get(header::CONTENT_ENCODING).is_none()); - assert!(response.headers().get(header::VARY).is_none()); - let content_length = response - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.parse::().ok()) - .expect("content length"); - let body = response_body(response).await; - assert_eq!(content_length, body.len()); - let body = std::str::from_utf8(&body).expect("master playlist should be UTF-8"); - assert_eq!(body.matches("/iptv").count(), 1); - let variant_uri = body.lines().nth(2).expect("single variant URI"); - assert!(variant_uri.starts_with("/iptv/hls/shared/live/")); - assert!(variant_uri.ends_with("/manifest.m3u8")); - let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(variant_uri).to_string()); - assert_eq!(access_lease_id.0.len(), 22); - let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(variant_uri).to_string()); - let lease = app_state - .hls - .proxy - .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::current_time_millis()) - .await - .expect("entry access lease"); - assert!(app_state - .active_users - .get_and_update_user_session(&lease.username, &lease.user_session_token) - .await - .is_some()); - assert!(app_state.hls.proxy.sessions().get_by_key(&session_key).await.is_none()); - assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0); - assert!(app_state.active_users.active_streams().await.is_empty()); -} - -#[test] -fn hls_master_playlist_response_diagnostic_reports_complete_response_contract_for_every_bandwidth_source() { - for selection in [ - super::HlsMasterBandwidthSelection::resolve(Some(2_500_000), Some(2_000_000)), - super::HlsMasterBandwidthSelection::resolve(None, Some(2_000_000)), - super::HlsMasterBandwidthSelection::resolve(None, None), - ] { - let rendered = super::hls_entry_master_playlist_response( - &ProxySessionId("diagnostic-proxy-session".to_string()), - &HlsAccessLeaseId("diagnostic-lease".to_string()), - selection.bandwidth(), - Some("/iptv"), - ); - let fields = super::HlsMasterPlaylistResponseDiagnostic { - lease: "lease-safe".to_string(), - session: "session-safe".to_string(), - proxy_session: "proxy-safe".to_string(), - user_session: "user-safe".to_string(), - virtual_id: 12345, - bandwidth_bps: selection.bandwidth().advertised_bps(), - bandwidth_source: selection.source().as_log_value(), - content_length: rendered.content_length, - } - .to_string(); - - assert!(fields.contains("lease=lease-safe session=session-safe proxy_session=proxy-safe")); - assert!(fields.contains("user_session=user-safe virtual_id=12345")); - assert!(fields.contains(&format!( - "bandwidth_bps={} bandwidth_source={}", - selection.bandwidth().advertised_bps(), - selection.source().as_log_value() - ))); - assert!(fields.contains("status=200")); - assert!(fields.ends_with(&format!("content_length={}", rendered.content_length))); - assert_eq!( - rendered - .response - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.parse::().ok()), - Some(rendered.content_length) - ); - } -} - -struct SharedRequestFlowFixture { - _origin: TestSegmentOrigin, - app_state: Arc, - input: ConfigInput, - target: ConfigTarget, - user: ProxyUserCredentials, - origin_manifest_url: String, - entry_path: String, -} - -async fn shared_request_flow_fixture() -> SharedRequestFlowFixture { - const MANIFEST: &[u8] = b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n\ - #EXT-X-MEDIA-SEQUENCE:123\n#EXTINF:4.0,\n000123.ts\n#EXTINF:4.0,\n000124.ts\n\ - #EXTINF:4.0,\n000125.ts\n#EXTINF:4.0,\n000126.ts\n#EXTINF:4.0,\n000127.ts\n\ - #EXTINF:4.0,\n000128.ts\n"; - let origin = spawn_test_segment_origin(MANIFEST).await; - let input = ConfigInput { - id: 1, - name: Arc::from("request-flow-input"), - input_type: InputType::M3u, - url: origin.base_url.clone(), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let mut target = test_m3u_hls_share_target(); - target.name = "default".to_string(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - enable_hls_cache(&app_state); - configure_default_test_server(&app_state); - store_test_sources_with_target(&app_state, input.clone(), target.clone()); - let origin_manifest_url = format!("{}/channel/index.m3u8", origin.base_url); - cache_test_m3u_hls_item(&app_state, &target, test_m3u_hls_item(&input, 12345, "channel-a", &origin_manifest_url)) - .await; - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - user.password = "hls-pass".to_string(); - let entry_path = super::build_virtual_hls_entry_path(&target, &input, &user, 12345); - SharedRequestFlowFixture { _origin: origin, app_state, input, target, user, origin_manifest_url, entry_path } -} - -async fn shared_request_flow_entry(fixture: &SharedRequestFlowFixture, fingerprint: &Fingerprint) -> Response { - super::handle_hls_stream_request( - fingerprint, - &fixture.app_state, - &fixture.user, - &fixture.target, - None, - None, - &fixture.origin_manifest_url, - None, - test_hls_entry_stream_context(12345, "channel-a", Some(2_500_000)), - &fixture.input, - &HeaderMap::new(), - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - &fixture.entry_path, - super::HlsRequestStage::Entry, - ) - .await - .into_response() -} - -#[tokio::test] -async fn hls_cache_archive_entry_uses_distinct_identity_and_preserves_origin() -> Result<(), &'static str> { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - configure_default_test_server(&app_state); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - user.password = "hls-pass".to_string(); - let input = test_hls_input(); - let target = test_hls_share_target(true); - let archive_url = "http://origin.example.com/live/user/pass/timeshift_abs-1784898000.m3u8"; - - let response = super::handle_hls_stream_request( - &test_fingerprint(), - &app_state, - &user, - &target, - None, - None, - archive_url, - Some(1_784_898_000), - test_hls_entry_stream_context(12345, "80510", None), - &input, - &HeaderMap::new(), - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - &super::build_virtual_hls_entry_path(&target, &input, &user, 12345), - super::HlsRequestStage::Entry, - ) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::OK); - let (_, media_playlist_uri) = single_variant_master_playlist(response).await; - let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&media_playlist_uri).to_string()); - let live_source = super::build_hls_origin_source(&input, "80510"); - let archive_source = - super::build_hls_origin_source_for_playback(&input, "80510", Some(1_784_898_000), Some(archive_url)); - assert_ne!(proxy_session_id, build_proxy_session_id(&live_source.session_key(), &app_state.get_encrypt_secret())); - assert_eq!( - proxy_session_id, - build_proxy_session_id(&archive_source.session_key(), &app_state.get_encrypt_secret()) - ); - - let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&media_playlist_uri).to_string()); - let lease = app_state - .hls - .proxy - .access_leases() - .write() - .await - .response_snapshot(&access_lease_id, &proxy_session_id, super::current_time_millis()) - .ok_or("archive access lease")?; - assert_eq!(lease.stream_ref, "80510"); - assert_eq!(lease.epg_reference_ts, Some(1_784_898_000)); - assert_eq!(lease.archive_origin_url.as_deref(), Some(archive_url)); - assert!(super::is_m3u_catchup_session_token(&lease.user_session_token)); - Ok(()) -} - -struct BoundedFlussonicFixture { - _temp: tempfile::TempDir, - origin: TestSegmentOrigin, - app_state: Arc, - requests: Arc>>, - catchup_template: String, - live_uri: String, -} - -async fn bounded_flussonic_request( - app_state: Arc, - url: &str, -) -> Result, Box> { - let parsed = url::Url::parse(url)?; - let mut uri = parsed.path().to_owned(); - if let Some(query) = parsed.query() { - uri.push('?'); - uri.push_str(query); - } - let router = crate::api::endpoints::m3u_api::m3u_api_register().merge(hls_api_register()).with_state(app_state); - let mut request = Request::builder().uri(uri).body(Body::empty())?; - request.extensions_mut().insert(ConnectInfo(test_addr())); - Ok(router.oneshot(request).await?) -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum BoundedArchiveOrigin { - Media, - Missing, - SingleVariantMaster, -} - -async fn bounded_flussonic_origin( - archive_origin: BoundedArchiveOrigin, - requests: &Arc>>, -) -> Result> { - let recorded = Arc::clone(requests); - let base = Arc::new(std::sync::OnceLock::::new()); - let origin_base = Arc::clone(&base); - let segment = Arc::<[u8]>::from( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) - .as_slice(), - ); - let origin = spawn_test_binary_origin(Arc::new(move |path| { - if let Ok(mut paths) = recorded.lock() { - paths.push(path.to_owned()); - } - let resource = path.split('?').next().unwrap_or(path); - if resource == "/input.m3u" { - let Some(host) = origin_base.get() else { - return TestBinaryOriginResponse::new(StatusCode::INTERNAL_SERVER_ERROR, Arc::from(&b"origin"[..])); - }; - let text = format!( - "#EXTM3U\n#EXTINF:-1 catchup=\"fs\" catchup-days=\"7\",Channel\n{host}/channel/mono.m3u8?token=a%2Fb\n" - ); - return TestBinaryOriginResponse::new(StatusCode::OK, Arc::from(text.into_bytes())); - } - if resource.ends_with(".ts") - && ["/channel/archive-", "/channel/live/", "/channel/tracks-v1/archive-"] - .iter() - .any(|prefix| resource.starts_with(prefix)) - { - return TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&segment)); - } - let archive_file = resource - .strip_prefix("/channel/") - .and_then(|file| file.strip_suffix(".m3u8")) - .filter(|file| file.starts_with("archive-")); - if let (BoundedArchiveOrigin::SingleVariantMaster, Some(archive)) = (archive_origin, archive_file) { - let master = format!( - "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=9150000,RESOLUTION=1920x1080\ntracks-v1/{archive}.ts.m3u8\n" - ); - return TestBinaryOriginResponse::new(StatusCode::OK, Arc::from(master.into_bytes())); - } - let media_directory = if resource == "/channel/mono.m3u8" { - Some("live") - } else if archive_origin == BoundedArchiveOrigin::SingleVariantMaster { - resource - .strip_prefix("/channel/tracks-v1/") - .and_then(|file| file.strip_suffix(".ts.m3u8")) - .filter(|file| file.starts_with("archive-")) - } else { - archive_file - }; - if let Some(directory) = media_directory { - if archive_origin == BoundedArchiveOrigin::Missing && directory != "live" { - return TestBinaryOriginResponse::new(StatusCode::NOT_FOUND, Arc::from(&b"unavailable"[..])); - } - let mut manifest = String::from_utf8_lossy(®ression_origin_manifest(123, 6)).into_owned(); - for sequence in 123..129 { - manifest = - manifest.replace(&format!("{sequence}.ts"), &format!("{directory}/{sequence}.ts?token=a%2Fb")); - } - if directory != "live" { - manifest.push_str("#EXT-X-ENDLIST\n"); - } - return TestBinaryOriginResponse::new(StatusCode::OK, Arc::from(manifest.into_bytes())); - } - TestBinaryOriginResponse::new(StatusCode::NOT_FOUND, Arc::from(&b"unexpected origin path"[..])) - })) - .await; - base.set(origin.base_url.clone()).map_err(|_| "origin already configured")?; - Ok(origin) -} - -async fn bounded_flussonic_hls_fixture( - archive_origin: BoundedArchiveOrigin, -) -> Result> { - use shared::model::{ConfigInputOptionsDto, FlussonicHlsCatchup, ProxyType}; - use tuliprox_core::model::ConfigInputOptions; - use tuliprox_repository::{ensure_target_storage_path, m3u_write_playlist}; - - let temp = tempfile::tempdir()?; - let requests = Arc::new(std::sync::Mutex::new(Vec::new())); - let origin = bounded_flussonic_origin(archive_origin, &requests).await?; - let options: ConfigInputOptionsDto = serde_json::from_str(r#"{"flussonic_hls_catchup":"bounded_archive"}"#)?; - assert_eq!(options.flussonic_hls_catchup, FlussonicHlsCatchup::BoundedArchive); - let input = ConfigInput { - id: 1, - name: Arc::from("bounded-flussonic"), - input_type: InputType::M3u, - url: format!("{}/input.m3u", origin.base_url), - enabled: true, - options: Some(ConfigInputOptions::from(&options)), - ..ConfigInput::default() - }; - let mut target = test_m3u_hls_share_target(); - target.name = "default".to_owned(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - enable_hls_cache(&app_state); - let mut config = (*app_state.app_config.config.load_full()).clone(); - config.storage_dir = temp.path().to_string_lossy().into_owned(); - config.custom_stream_response_enabled = false; - app_state.app_config.config.store(Arc::new(config)); - configure_default_test_server(&app_state); - store_test_sources_with_target(&app_state, input.clone(), target.clone()); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_owned(); - user.password = "hls-pass".to_owned(); - user.proxy = ProxyType::Reverse(None); - let mut proxy = app_state.app_config.api_proxy.load_full().ok_or("api proxy")?.as_ref().clone(); - proxy.user = vec![TargetUser { target: target.name.clone(), credentials: vec![Arc::new(user)] }]; - app_state.app_config.api_proxy.store(Some(Arc::new(proxy))); - let (mut groups, errors) = crate::iptv::m3u::download_m3u_playlist( - &app_state.app_config, - &reqwest::Client::new(), - &app_state.app_config.config.load_full(), - &input, - ) - .await; - assert_eq!(errors.len(), 0, "import errors: {errors:?}"); - let item = groups.first_mut().and_then(|group| group.channels.first_mut()).ok_or("imported item")?; - item.header.virtual_id = VirtualId::new(12345); - assert_eq!(item.header.url.as_ref(), format!("{}/channel/mono.m3u8?token=a%2Fb", origin.base_url)); - let target_path = ensure_target_storage_path(&app_state.app_config.config.load(), &target.name).await?; - m3u_write_playlist( - &app_state.app_config, - &target, - target.get_m3u_output().ok_or("M3U output")?, - &target_path, - &groups, - false, - ) - .await?; - let response = bounded_flussonic_request( - Arc::clone(&app_state), - "http://proxy/get.php?username=hls-user&password=hls-pass&type=m3u_plus", - ) - .await?; - assert_eq!(response.status(), StatusCode::OK); - let export = String::from_utf8(response_body(response).await.to_vec())?; - let catchup_template = export - .split("catchup-source=\"") - .nth(1) - .and_then(|tail| tail.split('"').next()) - .ok_or("exported catchup source")? - .to_owned(); - assert!(export.contains("catchup=\"default\"")); - assert!(!export.contains("a%2Fb")); - let live_uri = - export.lines().find(|line| !line.starts_with('#') && !line.is_empty()).ok_or("exported live URL")?.to_owned(); - Ok(BoundedFlussonicFixture { _temp: temp, origin, app_state, requests, catchup_template, live_uri }) -} - -async fn bounded_flussonic_hls_entry( - fixture: &BoundedFlussonicFixture, - start: i64, - duration: i64, -) -> Result, Box> { - let url = - fixture.catchup_template.replace("{utc}", &start.to_string()).replace("{duration}", &duration.to_string()); - bounded_flussonic_request(Arc::clone(&fixture.app_state), &url).await -} - -#[tokio::test] -async fn bounded_flussonic_hls_preserves_master_child_segments_and_duration_identity( -) -> Result<(), Box> { - let fixture = bounded_flussonic_hls_fixture(BoundedArchiveOrigin::Media).await?; - let mut identities = std::collections::HashSet::new(); - for (start, duration) in [(1_784_898_000, 3600), (1_784_898_000, 7200), (1_784_898_600, 3600)] { - let entry = bounded_flussonic_hls_entry(&fixture, start, duration).await?; - assert_eq!(entry.status(), StatusCode::OK); - let (_, uri) = single_variant_master_playlist(entry).await; - let proxy_id = ProxySessionId(proxy_session_id_from_variant_uri(&uri).to_owned()); - assert!(identities.insert(proxy_id.clone())); - let lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&uri).to_owned()); - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&lease_id, &proxy_id, super::current_time_millis()) - .await - .ok_or("lease")?; - let archive = format!("archive-{start}-{duration}"); - let expected_origin = format!("{}/channel/{archive}.m3u8?token=a%2Fb", fixture.origin.base_url); - assert_eq!(lease.archive_origin_url.as_deref(), Some(expected_origin.as_str())); - assert!(lease.user_session_token.contains(&format!("|archive|{start}|{duration}|hls-cache|"))); - let media = get_response(Arc::clone(&fixture.app_state), &uri, None).await; - assert_eq!(media.status(), StatusCode::OK); - let body = String::from_utf8(response_body(media).await.to_vec())?; - let segment_uri = - body.lines().find(|line| line.starts_with("/hls/") && path_has_extension(line, "ts")).ok_or("segment")?; - let segment = get_response(Arc::clone(&fixture.app_state), segment_uri, None).await; - assert_eq!(segment.status(), StatusCode::OK); - assert_ne!(response_body(segment).await, bytes::Bytes::new()); - let reload = get_response(Arc::clone(&fixture.app_state), &uri, None).await; - assert_eq!(reload.status(), StatusCode::OK); - let reload_body = String::from_utf8(response_body(reload).await.to_vec())?; - assert_eq!(manifest_media_sequence(&body), manifest_media_sequence(&reload_body)); - let paths = fixture.requests.lock().map_err(|_| "request log")?; - assert!(paths.iter().any(|p| p == &format!("/channel/{archive}.m3u8?token=a%2Fb"))); - assert!(paths.iter().any(|p| p.starts_with(&format!("/channel/{archive}/")) && p.ends_with(".ts?token=a%2Fb"))); - assert!(!paths.iter().any(|p| p.contains("timeshift_abs") || p.contains("mono.m3u8"))); - } - Ok(()) -} - -#[tokio::test] -async fn bounded_flussonic_hls_resolves_single_variant_archive_master() -> Result<(), Box> { - let fixture = bounded_flussonic_hls_fixture(BoundedArchiveOrigin::SingleVariantMaster).await?; - let entry = bounded_flussonic_hls_entry(&fixture, 1_784_898_000, 3600).await?; - assert_eq!(entry.status(), StatusCode::OK); - let (_, uri) = single_variant_master_playlist(entry).await; - let media = get_response(Arc::clone(&fixture.app_state), &uri, None).await; - assert_eq!(media.status(), StatusCode::OK); - let body = String::from_utf8(response_body(media).await.to_vec())?; - let segment_uri = - body.lines().find(|line| line.starts_with("/hls/") && path_has_extension(line, "ts")).ok_or("segment")?; - let segment = get_response(Arc::clone(&fixture.app_state), segment_uri, None).await; - assert_eq!(segment.status(), StatusCode::OK); - assert_ne!(response_body(segment).await, bytes::Bytes::new()); - let reload = get_response(Arc::clone(&fixture.app_state), &uri, None).await; - assert_eq!(reload.status(), StatusCode::OK); - let reload_body = String::from_utf8(response_body(reload).await.to_vec())?; - assert_eq!(manifest_media_sequence(&body), manifest_media_sequence(&reload_body)); - let paths = fixture.requests.lock().map_err(|_| "request log")?; - assert!(paths.iter().any(|p| p == "/channel/archive-1784898000-3600.m3u8?token=a%2Fb")); - assert!(paths.iter().any(|p| p == "/channel/tracks-v1/archive-1784898000-3600.ts.m3u8?token=a%2Fb")); - assert!(paths - .iter() - .any(|p| p.starts_with("/channel/tracks-v1/archive-1784898000-3600/") && p.ends_with(".ts?token=a%2Fb"))); - assert!(!paths.iter().any(|p| p.contains("timeshift_abs") || p.contains("mono.m3u8"))); - Ok(()) -} - -#[tokio::test] -async fn bounded_flussonic_hls_404_is_unavailable_without_live_or_timeshift_fallback( -) -> Result<(), Box> { - let fixture = bounded_flussonic_hls_fixture(BoundedArchiveOrigin::Missing).await?; - let entry = bounded_flussonic_hls_entry(&fixture, 1_784_898_000, 3600).await?; - assert_eq!(entry.status(), StatusCode::OK); - let (_, uri) = single_variant_master_playlist(entry).await; - let response = get_response(Arc::clone(&fixture.app_state), &uri, None).await; - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert!(!response.headers().contains_key(header::LOCATION)); - { - let paths = fixture.requests.lock().map_err(|_| "request log")?; - assert!(paths.iter().any(|p| p == "/channel/archive-1784898000-3600.m3u8?token=a%2Fb")); - assert!(!paths.iter().any(|p| p.contains("timeshift_abs") || p.contains("mono.m3u8"))); - } - let live = bounded_flussonic_request(Arc::clone(&fixture.app_state), &fixture.live_uri).await?; - assert_eq!(live.status(), StatusCode::OK); - let (_, live_manifest) = single_variant_master_playlist(live).await; - let media = get_response(Arc::clone(&fixture.app_state), &live_manifest, None).await; - assert_eq!(media.status(), StatusCode::OK); - let body = String::from_utf8(response_body(media).await.to_vec())?; - let segment_uri = - body.lines().find(|line| line.starts_with("/hls/") && path_has_extension(line, "ts")).ok_or("live segment")?; - let segment = get_response(Arc::clone(&fixture.app_state), segment_uri, None).await; - assert_eq!(segment.status(), StatusCode::OK); - assert_ne!(response_body(segment).await, bytes::Bytes::new()); - let paths = fixture.requests.lock().map_err(|_| "request log")?; - assert!(paths.iter().any(|p| p == "/channel/mono.m3u8?token=a%2Fb")); - assert!(paths.iter().any(|p| p.starts_with("/channel/live/") && p.ends_with(".ts?token=a%2Fb"))); - assert!(!paths.iter().any(|p| p.contains("timeshift_abs"))); - Ok(()) -} - -#[tokio::test] -async fn shared_hls_request_flow_keeps_media_playlist_lease_bound_across_reloads() { - let fixture = shared_request_flow_fixture().await; - let app_state = &fixture.app_state; - let entry_response = shared_request_flow_entry(&fixture, &test_fingerprint()).await; - - assert_eq!(entry_response.status(), StatusCode::OK); - assert!(!entry_response.headers().contains_key(header::LOCATION)); - let (_, media_playlist_uri) = single_variant_master_playlist(entry_response).await; - let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&media_playlist_uri).to_string()); - let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&media_playlist_uri).to_string()); - - let first_media_response = get_response(Arc::clone(app_state), &media_playlist_uri, None).await; - assert_eq!(first_media_response.status(), StatusCode::OK); - assert!(!first_media_response.headers().contains_key(header::LOCATION)); - let first_media_body = - String::from_utf8(response_body(first_media_response).await.to_vec()).expect("media playlist utf8"); - let segment_uri = first_media_body - .lines() - .find(|line| line.starts_with("/hls/shared/live/") && path_has_extension(line, "ts")) - .expect("lease-bound segment URI") - .to_string(); - let lease_path = format!("/{}/{}/", proxy_session_id.0, access_lease_id.0); - assert!(segment_uri.contains(&lease_path)); - - let reloaded_media_response = get_response(Arc::clone(app_state), &media_playlist_uri, None).await; - assert_eq!(reloaded_media_response.status(), StatusCode::OK); - assert!(!reloaded_media_response.headers().contains_key(header::LOCATION)); - let reloaded_media_body = - String::from_utf8(response_body(reloaded_media_response).await.to_vec()).expect("reloaded media playlist utf8"); - assert_eq!(manifest_media_sequence(&reloaded_media_body), manifest_media_sequence(&first_media_body)); - assert!(reloaded_media_body.lines().any(|line| line == segment_uri)); - assert_eq!(app_state.hls.proxy.access_leases().read().await.len(), 1); - - let segment_response = get_response(Arc::clone(app_state), &segment_uri, None).await; - assert_eq!(segment_response.status(), StatusCode::OK); - assert!(!response_body(segment_response).await.is_empty()); - let lease = app_state - .hls - .proxy - .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::current_time_millis()) - .await - .expect("request-flow access lease"); - assert_eq!(lease.state, HlsAccessLeaseState::Activated); - assert!(lease.last_manifest_snapshot.is_some()); - - let second_entry_response = - shared_request_flow_entry(&fixture, &test_fingerprint_with_addr(test_addr_with_port(55131))).await; - assert_eq!(second_entry_response.status(), StatusCode::OK); - let (_, second_media_playlist_uri) = single_variant_master_playlist(second_entry_response).await; - let second_proxy_session_id = - ProxySessionId(proxy_session_id_from_variant_uri(&second_media_playlist_uri).to_string()); - let second_access_lease_id = - HlsAccessLeaseId(access_lease_id_from_variant_uri(&second_media_playlist_uri).to_string()); - assert_eq!(second_proxy_session_id, proxy_session_id); - assert_ne!(second_access_lease_id, access_lease_id); - let second_pending_lease = app_state - .hls - .proxy - .access_lease_response_snapshot(&second_access_lease_id, &second_proxy_session_id, super::current_time_millis()) - .await - .expect("second pending request-flow lease"); - assert_eq!(second_pending_lease.state, HlsAccessLeaseState::Pending); - assert!(second_pending_lease.last_manifest_snapshot.is_none()); - assert_ne!(second_pending_lease.user_session_token, lease.user_session_token); - - let second_media_response = get_response(Arc::clone(app_state), &second_media_playlist_uri, None).await; - assert_eq!(second_media_response.status(), StatusCode::OK); - assert!(!second_media_response.headers().contains_key(header::LOCATION)); - let second_media_body = - String::from_utf8(response_body(second_media_response).await.to_vec()).expect("second media playlist utf8"); - assert_eq!(manifest_media_sequence(&second_media_body), manifest_media_sequence(&first_media_body)); - let second_published_lease = app_state - .hls - .proxy - .access_lease_response_snapshot(&second_access_lease_id, &second_proxy_session_id, super::current_time_millis()) - .await - .expect("second published request-flow lease"); - assert!(second_published_lease.last_manifest_snapshot.is_some()); - assert_eq!(second_published_lease.playback_mode, HlsLeasePlaybackMode::Live); - assert_eq!(app_state.hls.proxy.sessions().len().await, 1); -} - -struct PublicationLateFixture { - origin: TestSegmentOrigin, - origin_phase: Arc, - app_state: Arc, - session: HlsSessionHandle, - proxy_session_id: ProxySessionId, - access_lease_id: HlsAccessLeaseId, - media_playlist_uri: String, -} - -async fn publication_late_fixture() -> PublicationLateFixture { - let initial_manifest = Arc::<[u8]>::from(regression_origin_manifest(123, 6)); - let progressed_manifest = Arc::<[u8]>::from(regression_origin_manifest(124, 6)); - let segment = Arc::<[u8]>::from( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) - .as_slice(), - ); - let origin_phase = Arc::new(AtomicUsize::new(0)); - let origin_phase_for_handler = Arc::clone(&origin_phase); - let origin = spawn_test_binary_origin(Arc::new(move |path| { - if path_has_extension(path, "m3u8") { - return match origin_phase_for_handler.load(Ordering::SeqCst) { - 0 => TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&initial_manifest)), - 1 => TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&progressed_manifest)), - _ => TestBinaryOriginResponse::new( - StatusCode::PROXY_AUTHENTICATION_REQUIRED, - Arc::<[u8]>::from(&b"retry"[..]), - ), - }; - } - TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&segment)) - })) - .await; - let input = ConfigInput { - id: 1, - name: Arc::from("publication-late-request-input"), - input_type: InputType::M3u, - url: origin.base_url.clone(), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let mut target = test_m3u_hls_share_target(); - target.name = "default".to_string(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - enable_hls_cache(&app_state); - enable_channel_unavailable_custom_response(&app_state); - configure_default_test_server(&app_state); - store_test_sources_with_target(&app_state, input.clone(), target.clone()); - let origin_manifest_url = format!("{}/channel/index.m3u8", origin.base_url); - cache_test_m3u_hls_item(&app_state, &target, test_m3u_hls_item(&input, 12345, "channel-a", &origin_manifest_url)) - .await; - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - user.password = "hls-pass".to_string(); - let entry_path = super::build_virtual_hls_entry_path(&target, &input, &user, 12345); - let entry_response = super::handle_hls_stream_request( - &test_fingerprint(), - &app_state, - &user, - &target, - None, - None, - &origin_manifest_url, - None, - test_hls_entry_stream_context(12345, "channel-a", None), - &input, - &HeaderMap::new(), - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - &entry_path, - super::HlsRequestStage::Entry, - ) - .await - .into_response(); - let (_, media_playlist_uri) = single_variant_master_playlist(entry_response).await; - let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&media_playlist_uri).to_string()); - let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&media_playlist_uri).to_string()); - let initial_media_response = get_response(Arc::clone(&app_state), &media_playlist_uri, None).await; - assert_eq!(initial_media_response.status(), StatusCode::OK); - let initial_media_body = - String::from_utf8(response_body(initial_media_response).await.to_vec()).expect("initial media utf8"); - let last_segment_uri = initial_media_body - .lines() - .rfind(|line| line.starts_with("/hls/shared/live/") && path_has_extension(line, "ts")) - .expect("initial manifest segment"); - assert_eq!(get_status(Arc::clone(&app_state), last_segment_uri).await, StatusCode::OK); - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&proxy_session_id) - .await - .expect("shared publication-late session"); - PublicationLateFixture { - origin, - origin_phase, - app_state, - session, - proxy_session_id, - access_lease_id, - media_playlist_uri, - } -} - -async fn refresh_publication_late_fixture(fixture: &PublicationLateFixture) -> HlsAccessLease { - let progress_generation_before = fixture.session.read().await.origin_control.progress_generation; - { - let mut session = fixture.session.write().await; - session.origin_control.last_media_progress_at_ms = Some(0); - session.origin_refresh.next_fetch_allowed_at_ms = 0; - } - let requests_before = fixture.origin.manifest_request_count(); - fixture.origin_phase.store(1, Ordering::SeqCst); - let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("refreshed media utf8"); - assert!(!body.contains("#EXT-X-ENDLIST")); - assert!(!body.contains("/terminal/")); - assert!(fixture.origin.manifest_request_count() > requests_before); - { - let session = fixture.session.read().await; - assert!(session.origin_control.progress_generation > progress_generation_before); - assert_eq!(session.origin_seq_highwater, Some(129)); - } - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot( - &fixture.access_lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) - .await - .expect("publication-late lease remains stored"); - assert_eq!(lease.state, HlsAccessLeaseState::Activated); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - assert_eq!(fixture.app_state.hls.proxy.terminal_pending().owner_count(), 0); - lease -} - -async fn prepare_publication_late_terminal_pressure( - fixture: &PublicationLateFixture, - base_manifest: &HlsLeaseManifestSnapshot, -) { - let terminal_response = fixture.app_state.app_config.custom_stream_response.load_full(); - let terminal_asset = snapshot_terminal_media_asset( - terminal_response - .as_ref() - .and_then(|response| response.channel_unavailable.as_ref()) - .expect("configured terminal asset"), - ) - .expect("compatible terminal asset"); - let bundle_key = prepared_terminal_bundle_key( - &terminal_asset, - base_manifest.target_duration_ms, - HLS_TERMINAL_TAIL_SEGMENT_COUNT, - ); - let state = fixture.app_state.hls.proxy.start_prepared_terminal_bundle( - terminal_asset, - base_manifest.target_duration_ms, - HLS_TERMINAL_TAIL_SEGMENT_COUNT, - ); - let state = match state { - HlsPreparedTerminalBundleState::Preparing { .. } => fixture - .app_state - .hls - .proxy - .wait_for_prepared_terminal_bundle(bundle_key) - .await - .expect("terminal bundle completion"), - state => state, - }; - assert!(matches!(state, HlsPreparedTerminalBundleState::Ready { .. })); - let mut session = fixture.session.write().await; - for segment in session.segments.values_mut().filter(|segment| segment.proxy_seq > base_manifest.last_proxy_seq) { - segment.duration_ms = 1; - } - session.origin_control.last_media_progress_at_ms = Some(0); - session.origin_refresh.next_fetch_allowed_at_ms = 0; -} - -async fn assert_publication_late_terminal_result(fixture: &PublicationLateFixture) { - let requests_before = fixture.origin.manifest_request_count(); - fixture.origin_phase.store(2, Ordering::SeqCst); - let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot( - &fixture.access_lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) - .await - .expect("terminal lease remains stored"); - let recovery_plan = fixture.app_state.hls.proxy.manifest_recovery_burst().level.plan(); - assert!( - fixture.origin.manifest_request_count().saturating_sub(requests_before) >= recovery_plan.total_candidates() - ); - match lease.playback_mode { - HlsLeasePlaybackMode::TerminalTail(_) => { - assert_eq!(response.status(), StatusCode::OK); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("terminal manifest utf8"); - assert!(body.contains("#EXT-X-ENDLIST")); - assert!(body.contains("/terminal/")); - } - HlsLeasePlaybackMode::TerminalUnavailable { .. } => { - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - } - HlsLeasePlaybackMode::Live => { - assert_eq!(response.status(), StatusCode::OK); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("live manifest utf8"); - assert!(body.starts_with("#EXTM3U")); - assert!(!body.contains("#EXT-X-ENDLIST")); - } - HlsLeasePlaybackMode::Ended => { - panic!("active hard-failed lease must not become ended without terminal publication") - } - } -} - -#[tokio::test] -async fn publication_late_live_manifest_request_refreshes_before_terminal_evaluation() { - let fixture = publication_late_fixture().await; - let lease = refresh_publication_late_fixture(&fixture).await; - let base_manifest = lease.last_manifest_snapshot.as_ref().expect("live manifest snapshot"); - prepare_publication_late_terminal_pressure(&fixture, base_manifest).await; - assert_publication_late_terminal_result(&fixture).await; -} - -#[test] -fn hls_entry_stream_context_keeps_only_positive_live_bitrate() { - let input = ConfigInput { name: Arc::from("m3u-input"), ..ConfigInput::default() }; - let mut item = test_m3u_hls_item(&input, 1001, "channel-a", "http://origin.example.com/live.m3u8"); - item.additional_properties = Some(StreamProperties::Live(Box::new(shared::model::LiveStreamProperties { - bitrate: 2_500_000, - ..shared::model::LiveStreamProperties::default() - }))); - - let context = super::HlsEntryStreamContext::from_playlist_item(&item).expect("entry stream context"); - assert_eq!(context.virtual_id(), 1001); - assert_eq!(context.stream_ref(), "channel-a"); - assert_eq!(context.known_bitrate_bps(), Some(2_500_000)); - - if let Some(StreamProperties::Live(properties)) = item.additional_properties.as_mut() { - properties.bitrate = 0; - } - assert_eq!( - super::HlsEntryStreamContext::from_playlist_item(&item) - .expect("entry stream context without bitrate") - .known_bitrate_bps(), - None - ); -} - -#[tokio::test] -async fn hls_cache_entry_prefers_item_bitrate_then_loads_db_without_target_rebuild() { - let temp = tempfile::tempdir().expect("temp dir"); - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let mut config = app_state.app_config.config.load().as_ref().clone(); - config.storage_dir = temp.path().to_string_lossy().into_owned(); - app_state.app_config.config.store(Arc::new(config)); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - let input = - ConfigInput { id: 9, name: Arc::from("m3u-input"), input_type: InputType::M3u, ..ConfigInput::default() }; - let mut stored_item = test_m3u_hls_item(&input, 70001, "channel-a", "http://origin.example.com/live.m3u8"); - stored_item.additional_properties = Some(StreamProperties::Live(Box::new(shared::model::LiveStreamProperties { - bitrate: 2_500_000, - ..shared::model::LiveStreamProperties::default() - }))); - let input_storage = crate::repository::build_input_storage_path(&input.name, &temp.path().to_string_lossy()); - std::fs::create_dir_all(&input_storage).expect("input storage"); - let db_path = crate::repository::get_input_m3u_playlist_file_path(&input_storage, &input.name); - let mut tree = crate::repository::BPlusTree::new(); - tree.insert(Arc::::from("channel-a"), stored_item); - tree.store(&db_path).expect("input M3U metadata"); - let request_url = "http://origin.example.com/live.m3u8"; - - let db_response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - super::build_hls_origin_source(&input, "channel-a"), - 70001, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - let (db_bandwidth, db_variant_uri) = single_variant_master_playlist(db_response).await; - assert_eq!(db_bandwidth, 3_000_000); - let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&db_variant_uri).to_string()); - let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&db_variant_uri).to_string()); - let lease = app_state - .hls - .proxy - .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::current_time_millis()) - .await - .expect("DB-backed access lease"); - assert_eq!(lease.known_bitrate_bps, Some(2_500_000)); - - std::fs::write(&db_path, b"invalid metadata tree").expect("corrupt test metadata"); - let item_response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint_with_addr(test_addr_with_port(55127)), - &user, - super::build_hls_origin_source(&input, "channel-a"), - 70001, - None, - Some(3_000_000), - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - let (item_bandwidth, _) = single_variant_master_playlist(item_response).await; - assert_eq!(item_bandwidth, 3_600_000); - - let fallback_response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint_with_addr(test_addr_with_port(55128)), - &user, - super::build_hls_origin_source(&input, "channel-a"), - 70001, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - let (fallback_bandwidth, _) = single_variant_master_playlist(fallback_response).await; - assert_eq!(fallback_bandwidth, 1_000_000); -} - -#[tokio::test] -async fn hls_cache_entry_denies_access_lease_for_grace_without_slot_and_exhausted() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let user = app_state.app_config.get_user_credentials("hls-user").expect("test user should exist"); - let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; - let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; - let origin_source = super::build_hls_origin_source(&input, "12345"); - - for (connection_permission, connection_kind) in [ - (UserConnectionPermission::GracePeriod, None), - (UserConnectionPermission::Exhausted, Some(ConnectionKind::Normal)), - ] { - let response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - origin_source.clone(), - 12345, - None, - None, - None, - request_url, - &input, - connection_permission, - connection_kind, - Some("/iptv"), - ) - .await; - - assert_eq!(response.status(), StatusCode::OK); - let variant_uri = single_variant_uri(response).await; - let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&variant_uri).to_string()); - let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&variant_uri).to_string()); - let now_ms = super::current_time_millis(); - let snapshot = app_state - .hls - .proxy - .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms) - .await - .expect("denied lease should stay available for response rendering"); - assert_eq!(snapshot.state, HlsAccessLeaseState::Denied); - assert!(app_state - .hls - .proxy - .access_lease_session_snapshot(&proxy_session_id, now_ms) - .await - .effective_origin_policy - .is_none()); - - let err = super::validate_hls_proxy_access_context( - &app_state, - &test_fingerprint(), - &proxy_session_id, - &access_lease_id.0, - now_ms, - HlsAccessAdmissionMode::ManifestPrepare, - ) - .await - .expect_err("denied lease must surface as admission denied"); - assert!(matches!(err, HlsAccessLeaseValidationError::AdmissionDenied { runtime_tail: None, .. })); - } -} - -#[tokio::test] -async fn hls_cache_entry_master_playlist_uses_cache_when_target_hls_share_enabled() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - configure_default_test_server(&app_state); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - user.password = "hls-pass".to_string(); - let input = test_hls_input(); - let target = test_hls_share_target(true); - let original_hls_entry_path = super::build_virtual_hls_entry_path(&target, &input, &user, 1001); - - let response = super::handle_hls_stream_request( - &test_fingerprint(), - &app_state, - &user, - &target, - None, - None, - "http://origin.example.com/live/user/pass/1001.m3u8", - None, - test_hls_entry_stream_context(1001, "80510", Some(2_500_000)), - &input, - &HeaderMap::new(), - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - &original_hls_entry_path, - super::HlsRequestStage::Entry, - ) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::OK); - let (bandwidth, variant_uri) = single_variant_master_playlist(response).await; - assert_eq!(bandwidth, 3_000_000); - assert!(variant_uri.starts_with("/hls/shared/live/")); - assert!(variant_uri.ends_with("/manifest.m3u8")); - assert_eq!(app_state.hls.proxy.access_leases().read().await.len(), 1); - - let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&variant_uri).to_string()); - let origin_key = HlsSessionKey::new(input.id, "80510"); - let virtual_id_key = HlsSessionKey::new(input.id, "1001"); - assert_eq!(origin_key.stable_value(), "input:1|hls|80510"); - assert_eq!(proxy_session_id, build_proxy_session_id(&origin_key, &app_state.get_encrypt_secret())); - assert_ne!(proxy_session_id, build_proxy_session_id(&virtual_id_key, &app_state.get_encrypt_secret())); - - let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&variant_uri).to_string()); - let lease = app_state - .hls - .proxy - .access_leases() - .write() - .await - .response_snapshot(&access_lease_id, &proxy_session_id, super::current_time_millis()) - .expect("access lease"); - assert_eq!(lease.stream_ref, "80510"); - assert_eq!(lease.virtual_id, 1001); - assert_eq!(lease.known_bitrate_bps, Some(2_500_000)); -} - -#[tokio::test] -async fn hls_cache_entry_shares_content_session_across_targets_but_keeps_distinct_virtual_leases() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - configure_default_test_server(&app_state); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - user.password = "hls-pass".to_string(); - let input = test_hls_input(); - let first_target = test_hls_share_target(true); - let mut second_target = test_hls_share_target(true); - second_target.id = 2; - - let first_response = super::handle_hls_stream_request( - &test_fingerprint(), - &app_state, - &user, - &first_target, - None, - None, - "http://origin.example.com/live/user/pass/1001.m3u8", - None, - test_hls_entry_stream_context(1001, "80510", None), - &input, - &HeaderMap::new(), - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - &super::build_virtual_hls_entry_path(&first_target, &input, &user, 1001), - super::HlsRequestStage::Entry, - ) - .await - .into_response(); - let second_response = super::handle_hls_stream_request( - &test_fingerprint_with_addr(test_addr_with_port(55124)), - &app_state, - &user, - &second_target, - None, - None, - "http://origin.example.com/live/user/pass/9007.m3u8", - None, - test_hls_entry_stream_context(9007, "80510", None), - &input, - &HeaderMap::new(), - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - &super::build_virtual_hls_entry_path(&second_target, &input, &user, 9007), - super::HlsRequestStage::Entry, - ) - .await - .into_response(); - - assert_eq!(first_response.status(), StatusCode::OK); - assert_eq!(second_response.status(), StatusCode::OK); - let first_variant_uri = single_variant_uri(first_response).await; - let second_variant_uri = single_variant_uri(second_response).await; - assert_eq!( - proxy_session_id_from_variant_uri(&first_variant_uri), - proxy_session_id_from_variant_uri(&second_variant_uri) - ); - assert_ne!( - access_lease_id_from_variant_uri(&first_variant_uri), - access_lease_id_from_variant_uri(&second_variant_uri) - ); - - let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&first_variant_uri).to_string()); - let first_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&first_variant_uri).to_string()); - let second_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&second_variant_uri).to_string()); - let now_ms = super::current_time_millis(); - let mut leases = app_state.hls.proxy.access_leases().write().await; - let first_lease = leases.response_snapshot(&first_lease_id, &proxy_session_id, now_ms).expect("first access lease"); - let second_lease = - leases.response_snapshot(&second_lease_id, &proxy_session_id, now_ms).expect("second access lease"); - assert_eq!(first_lease.virtual_id, 1001); - assert_eq!(second_lease.virtual_id, 9007); - assert_eq!(first_lease.stream_ref, "80510"); - assert_eq!(second_lease.stream_ref, "80510"); -} - -#[tokio::test] -async fn hls_virtual_source_resolver_rejects_missing_input_stream_id_with_service_unavailable() { - let input = single_hls_provider_input("missing-origin-id-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let target = Arc::new(test_m3u_hls_share_target()); - let item = - test_m3u_hls_item(&input, 1001, "", "http://account.example.com/live/account-user/account-pass/channel.m3u8"); - cache_test_m3u_hls_item(&app_state, &target, item).await; - - let status = super::resolve_hls_virtual_source_for_target(&app_state, &target, 1001) - .await - .expect_err("missing input stream identity must fail safely"); - - assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); -} - -#[tokio::test] -async fn hls_cache_entry_uses_legacy_path_when_target_hls_share_disabled() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - configure_default_test_server(&app_state); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - user.password = "hls-pass".to_string(); - let input = test_hls_input(); - let target = test_hls_share_target(false); - let original_hls_entry_path = super::build_virtual_hls_entry_path(&target, &input, &user, 12345); - - let response = super::handle_hls_stream_request( - &test_fingerprint(), - &app_state, - &user, - &target, - None, - None, - "http://origin.example.com/live/user/pass/12345.m3u8", - None, - test_hls_entry_stream_context(12345, "80510", None), - &input, - &HeaderMap::new(), - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - &original_hls_entry_path, - super::HlsRequestStage::Entry, - ) - .await - .into_response(); - - let location = response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).unwrap_or(""); - assert!(!location.contains("/hls/shared/live/")); - assert!(app_state.hls.proxy.access_leases().read().await.is_empty()); - assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0); -} - -/// Origin that only answers requests carrying the configured recording header. -async fn spawn_recording_header_origin() -> (String, Arc>>, tokio::task::JoinHandle<()>) { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let base_url = format!("http://{}", listener.local_addr().expect("local addr")); - let requests = Arc::new(std::sync::Mutex::new(Vec::new())); - let seen = Arc::clone(&requests); - let task = tokio::spawn(async move { - while let Ok((mut socket, _)) = listener.accept().await { - let seen = Arc::clone(&seen); - tokio::spawn(async move { - let mut request = Vec::new(); - let mut chunk = [0_u8; 1024]; - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - match socket.read(&mut chunk).await { - Ok(0) | Err(_) => return, - Ok(read) => request.extend_from_slice(&chunk[..read]), - } - } - let request = String::from_utf8_lossy(&request).to_ascii_lowercase(); - let path = request.split_whitespace().nth(1).unwrap_or("/").to_string(); - seen.lock().expect("request log").push(request.clone()); - let (status, body): (&str, &[u8]) = if !request.contains("\r\nx-recording-test: custom\r\n") { - ("403 Forbidden", b"") - } else if path_has_extension(&path, "m3u8") { - ( - "200 OK", - b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\nseg1.ts\n", - ) - } else { - ("200 OK", b"segment-bytes") - }; - let head = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); - let _ = socket.write_all(head.as_bytes()).await; - let _ = socket.write_all(body).await; - }); - } - }); - (base_url, requests, task) -} - -fn recording_test_router(app_state: &Arc) -> axum::Router { - axum::Router::new() - .nest( - "/tuliprox/api/v1", - crate::api::endpoints::v1_api_playlist::v1_api_playlist_register_public(axum::Router::new()), - ) - .merge(hls_api_register()) - .merge(crate::api::endpoints::m3u_api::m3u_api_register()) - .merge(crate::api::endpoints::xtream_api::xtream_api_register()) - .with_state(Arc::clone(app_state)) -} - -async fn recording_test_response( - router: &axum::Router, - uri: &str, -) -> Result, Box> { - let mut request = Request::builder() - .uri(uri) - .header(header::USER_AGENT, shared::model::RECORDING_STREAM_USER_AGENT) - .body(Body::empty())?; - request.extensions_mut().insert(ConnectInfo(test_addr())); - Ok(router.clone().oneshot(request).await?) -} - -fn configure_recording_test_listener(app_state: &Arc, storage_root: &std::path::Path) { - let mut config = (*app_state.app_config.config.load_full()).clone(); - config.api.host = "0.0.0.0".to_string(); - config.api.port = 8901; - config.storage_dir = storage_root.to_string_lossy().into_owned(); - config.web_ui = Some(crate::model::WebUiConfig::from(&shared::model::WebUiConfigDto { - path: Some("/tuliprox".to_string()), - ..Default::default() - })); - let mut recording = crate::model::RecordingConfig::from(&shared::model::RecordingConfigDto::default()); - recording.headers = HashMap::from([("X-Recording-Test".to_string(), "custom".to_string())]); - config.video = Some(crate::model::VideoConfig { extensions: vec![], web_search: None, recording: Some(recording) }); - app_state.app_config.config.store(Arc::new(config)); -} - -fn recording_test_url( - app_state: &Arc, - target: &ConfigTarget, - input: &ConfigInput, - cluster: XtreamCluster, -) -> Result> { - tuliprox_dvr::recording::recording_url::build_stable_recording_url( - &app_state.app_config, - &target.name, - &input.name, - 12345, - cluster, - None, - ) - .ok_or_else(|| "recording URL missing".into()) -} - -fn local_recording_hls_url(manifest: &str) -> Result> { - let resource = - manifest.lines().find(|line| !line.is_empty() && !line.starts_with('#')).ok_or("HLS resource URI missing")?; - let url = url::Url::parse(resource)?; - assert_eq!(url.scheme(), "http"); - assert_eq!(url.host_str(), Some("127.0.0.1")); - assert_eq!(url.port(), Some(8901)); - assert!(url.path().starts_with("/hls/recording_user/"), "{url}"); - Ok(url) -} - -async fn cache_recording_test_item( - app_state: &Arc, - target: &ConfigTarget, - item: M3uPlaylistItem, -) -> Result<(), Box> { - use crate::{ - api::model::{PlaylistStorage, PlaylistXtreamStorage}, - repository::{BPlusTree, VirtualIdRecord}, - }; - if target.has_output(shared::model::TargetType::Xtream) { - let item = shared::model::XtreamPlaylistItem::from(&PlaylistItem::from(&item)); - let mut mapping = BPlusTree::new(); - mapping.insert( - item.virtual_id, - VirtualIdRecord::new( - item.provider_id, - item.virtual_id, - item.item_type, - VirtualId::new(0), - shared::model::UUIDType::default(), - ), - ); - app_state.playlists.cache_id_mapping(&target.name, mapping).await; - let mut live = BPlusTree::new(); - let mut vod = BPlusTree::new(); - let mut series = BPlusTree::new(); - match item.item_type.cluster() { - XtreamCluster::Live => live.insert(item.virtual_id.get(), item), - XtreamCluster::Video => vod.insert(item.virtual_id.get(), item), - XtreamCluster::Series => series.insert(item.virtual_id.get(), item), - } - app_state - .playlists - .cache_playlist( - &target.name, - PlaylistStorage::XtreamPlaylist(Box::new(PlaylistXtreamStorage { live, vod, series })), - ) - .await; - } else { - let storage_dir = app_state.app_config.config.load().storage_dir.clone(); - let input_storage = crate::repository::get_input_storage_path(&item.input_name, &storage_dir).await?; - let path = crate::repository::get_input_m3u_playlist_file_path(&input_storage, &item.input_name); - let group = shared::model::PlaylistGroup { - id: 1, - title: "recording".intern(), - xtream_cluster: item.item_type.cluster(), - channels: vec![PlaylistItem::from(&item)], - }; - crate::repository::persist_input_m3u_playlist(&app_state.app_config, &path, &[group]).await?; - cache_test_m3u_hls_item(app_state, target, item).await; - } - Ok(()) -} - -#[tokio::test] -async fn recording_hls_routes_keep_local_urls_and_headers_after_session_loss() -> Result<(), Box> -{ - for (output, share) in [ - (shared::model::TargetType::Xtream, false), - (shared::model::TargetType::M3u, false), - (shared::model::TargetType::Xtream, true), - (shared::model::TargetType::M3u, true), - ] { - let (origin_url, requests, origin_task) = spawn_recording_header_origin().await; - let input = ConfigInput { - id: 1, - name: Arc::from("recording-input"), - input_type: if output == shared::model::TargetType::Xtream { InputType::Xtream } else { InputType::M3u }, - url: origin_url.clone(), - username: Some("user".to_string()), - password: Some("pass".to_string()), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let mut target = test_hls_share_target(share); - target.use_memory_cache = true; - if output == shared::model::TargetType::M3u { - target.output = vec![crate::model::TargetOutput::M3u(crate::model::M3uTargetOutput::from( - &M3uTargetOutputDto::default(), - ))]; - } - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - store_test_sources_with_target(&app_state, input.clone(), target.clone()); - app_state.app_config.api_proxy.store(None); - if share { - enable_hls_cache(&app_state); - } - let storage = tempfile::tempdir()?; - configure_recording_test_listener(&app_state, storage.path()); - let manifest_url = format!("{origin_url}/live/user/pass/12345.m3u8"); - cache_recording_test_item(&app_state, &target, test_m3u_hls_item(&input, 12345, "12345", &manifest_url)) - .await?; - let router = recording_test_router(&app_state); - let entry = - recording_test_response(&router, &recording_test_url(&app_state, &target, &input, XtreamCluster::Live)?) - .await?; - assert_eq!(entry.status(), StatusCode::OK, "{output:?}"); - let master = String::from_utf8(entry.into_body().collect().await?.to_bytes().to_vec())?; - let variant_url = local_recording_hls_url(&master)?; - let mut public_credentials = variant_url.clone(); - public_credentials.set_path(&variant_url.path().replacen( - &crate::api::api_utils::create_recording_proxy_user(&app_state).password, - crate::model::RECORDING_PROXY_USERNAME, - 1, - )); - let denied = recording_test_response(&router, public_credentials.as_str()).await?; - assert!(!denied.status().is_success(), "the public recording_user password must not authorize HLS"); - let token = variant_url.path_segments().and_then(Iterator::last).ok_or("HLS token missing")?; - let decoded = super::get_hls_session_token_and_url_from_token(&app_state.get_encrypt_secret(), token) - .ok_or("HLS token cannot be decoded")?; - let session_token = decoded.session_token.ok_or("HLS session token missing")?; - let variant = recording_test_response(&router, variant_url.as_str()).await?; - assert_eq!(variant.status(), StatusCode::OK, "{output:?}"); - let media = String::from_utf8(variant.into_body().collect().await?.to_bytes().to_vec())?; - let segment_url = local_recording_hls_url(&media)?; - let segment = recording_test_response(&router, segment_url.as_str()).await?; - assert_eq!(segment.status(), StatusCode::OK, "{output:?}"); - assert_eq!(segment.into_body().collect().await?.to_bytes().as_ref(), b"segment-bytes"); - // The hand-off served the first variant; a refresh must fetch the origin again. - let refresh = recording_test_response(&router, variant_url.as_str()).await?; - assert_eq!(refresh.status(), StatusCode::OK, "{output:?}"); - let refreshed = String::from_utf8(refresh.into_body().collect().await?.to_bytes().to_vec())?; - local_recording_hls_url(&refreshed)?; - assert!(app_state.active_users.terminate_session(crate::model::RECORDING_PROXY_USERNAME, &session_token).await); - let recreated = recording_test_response(&router, variant_url.as_str()).await?; - assert_eq!(recreated.status(), StatusCode::OK, "{output:?}"); - let recreated = String::from_utf8(recreated.into_body().collect().await?.to_bytes().to_vec())?; - local_recording_hls_url(&recreated)?; - assert!(app_state - .active_users - .get_and_update_user_session(crate::model::RECORDING_PROXY_USERNAME, &session_token) - .await - .is_some()); - let requests = requests.lock().map_err(|error| error.to_string())?.clone(); - assert!(requests.iter().any(|request| request.contains("seg1.ts")), "{requests:?}"); - assert_eq!(requests.iter().filter(|request| request.contains("12345.m3u8")).count(), 3); - assert!(requests.iter().all(|request| request.contains("x-recording-test: custom")), "{requests:?}"); - origin_task.abort(); - } - Ok(()) -} - -#[tokio::test] -async fn recording_force_redirect_keeps_vod_and_series_on_the_provider_proxy() -> Result<(), Box> -{ - for (output, cluster, item_type) in [ - (shared::model::TargetType::M3u, XtreamCluster::Video, PlaylistItemType::Video), - (shared::model::TargetType::M3u, XtreamCluster::Series, PlaylistItemType::Series), - (shared::model::TargetType::Xtream, XtreamCluster::Video, PlaylistItemType::Video), - (shared::model::TargetType::Xtream, XtreamCluster::Series, PlaylistItemType::Series), - ] { - // The proxy answers for an origin that cannot be resolved directly. - let (proxy_url, requests, proxy_task) = spawn_recording_header_origin().await; - let provider_url = format!( - "http://recording-origin.invalid/{}/{}12345.mp4", - cluster.as_stream_type(), - if output == shared::model::TargetType::Xtream { "user/pass/" } else { "" } - ); - let input = ConfigInput { - id: 1, - name: Arc::from("recording-input"), - input_type: if output == shared::model::TargetType::Xtream { InputType::Xtream } else { InputType::M3u }, - username: Some("user".to_string()), - password: Some("pass".to_string()), - url: if output == shared::model::TargetType::Xtream { - "http://recording-origin.invalid".to_string() - } else { - provider_url.clone() - }, - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let mut target = if output == shared::model::TargetType::Xtream { - test_hls_share_target(false) - } else { - test_m3u_hls_share_target() - }; - target.use_memory_cache = true; - target.name = "default".to_string(); - if let Some(options) = target.options.as_mut() { - options.force_redirect = Some(shared::model::ClusterFlags::all()); - } - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - store_test_sources_with_target(&app_state, input.clone(), target.clone()); - let storage = tempfile::tempdir()?; - configure_recording_test_listener(&app_state, storage.path()); - let mut config = (*app_state.app_config.config.load_full()).clone(); - config.proxy = Some(crate::model::ProxyConfig { url: proxy_url, username: None, password: None }); - app_state.app_config.config.store(Arc::new(config)); - let client = crate::utils::request::create_client(&app_state.app_config).build()?; - app_state.http_clients.default.store(Arc::new(client.clone())); - app_state.http_clients.no_redirect.store(Arc::new(client)); - let mut item = test_m3u_hls_item(&input, 12345, "12345", &provider_url); - item.item_type = item_type; - cache_recording_test_item(&app_state, &target, item).await?; - let router = recording_test_router(&app_state); - let recording = - recording_test_response(&router, &recording_test_url(&app_state, &target, &input, cluster)?).await?; - assert_eq!( - recording.status(), - StatusCode::OK, - "{cluster:?}: the recording endpoint must relay the provider body" - ); - assert_eq!(recording.into_body().collect().await?.to_bytes().as_ref(), b"segment-bytes"); - let ordinary_url = if output == shared::model::TargetType::M3u { - format!( - "/{}/{}/hls-user/hls-pass/12345.mp4", - crate::repository::storage_const::M3U_STREAM_PATH, - cluster.as_stream_type() - ) - } else { - format!("/{}/hls-user/hls-pass/12345.mp4", cluster.as_stream_type()) - }; - let ordinary = recording_test_response(&router, &ordinary_url).await?; - assert!( - ordinary.status().is_redirection(), - "{output:?}: ordinary playback keeps force_redirect: {}", - ordinary.status() - ); - assert_eq!( - ordinary.headers().get(header::LOCATION).ok_or("redirect location missing")?.to_str()?, - provider_url - ); - let requests = requests.lock().map_err(|error| error.to_string())?.clone(); - assert_eq!(requests.len(), 1, "{requests:?}"); - assert!(requests[0].starts_with(&format!("get {provider_url} ")), "{requests:?}"); - proxy_task.abort(); - } - Ok(()) -} - -#[tokio::test] -async fn recording_m3u_dash_keeps_the_redirect_for_relative_media_urls() -> Result<(), Box> { - let (origin_url, requests, origin_task) = spawn_recording_header_origin().await; - let input = ConfigInput { - id: 1, - name: Arc::from("recording-input"), - input_type: InputType::M3u, - url: origin_url.clone(), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let mut target = test_m3u_hls_share_target(); - target.name = "default".to_string(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - store_test_sources_with_target(&app_state, input.clone(), target.clone()); - let storage = tempfile::tempdir()?; - configure_recording_test_listener(&app_state, storage.path()); - let router = recording_test_router(&app_state); - let manifest_url = format!("{origin_url}/live/manifest.mpd"); - for item_type in [PlaylistItemType::LiveDash, PlaylistItemType::Live] { - let mut item = test_m3u_hls_item(&input, 12345, "12345", &manifest_url); - item.item_type = item_type; - cache_recording_test_item(&app_state, &target, item).await?; - let response = - recording_test_response(&router, &recording_test_url(&app_state, &target, &input, XtreamCluster::Live)?) - .await?; - assert!(response.status().is_redirection(), "{item_type:?}: {}", response.status()); - assert_eq!( - response.headers().get(header::LOCATION).ok_or("redirect location missing")?.to_str()?, - manifest_url - ); - } - assert!(requests.lock().map_err(|error| error.to_string())?.is_empty()); - origin_task.abort(); - Ok(()) -} - -#[tokio::test] -async fn legacy_hls_token_route_renders_channel_unavailable_inline_when_target_hls_share_enabled() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - app_state.app_config.config.store(Arc::new(Config { - custom_stream_response_enabled: true, - reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { - hls_cache: Some(HlsCacheConfigDto::default()), - ..Default::default() - })), - ..Default::default() - })); - configure_default_test_server(&app_state); - enable_channel_unavailable_custom_response(&app_state); - let user = app_state.app_config.get_user_credentials("hls-user").expect("test user should exist"); - let input = test_hls_input(); - let target = test_hls_share_target(true); - store_test_sources_with_target(&app_state, input.clone(), target.clone()); - let encrypt_secret = app_state.get_encrypt_secret(); - let legacy_manifest = rewrite_hls( - &user, - &RewriteHlsProps { - secret: &encrypt_secret, - base_url: "", - content: "#EXTM3U\n#EXTINF:4.0,\nseg.ts\n", - hls_url: "http://origin.example.com/live/user/pass/12345.m3u8".to_string(), - target_id: target.id, - virtual_id: 12345, - input_id: input.id, - user_token: Some("legacy-session-token"), - origin_provider: None, - playlist_kind: None, - }, - ); - let token = legacy_manifest - .lines() - .find_map(|line| line.rsplit_once('/').map(|(_, token)| token.trim().to_string())) - .expect("legacy hls segment token should be rendered"); - - let response = super::hls_api_stream_resolved( - test_fingerprint(), - HeaderMap::new(), - Arc::clone(&app_state), - Arc::clone(&user), - Arc::new(target), - input.id, - 12345, - token, - ) - .await; - - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::LOCATION)); - assert_eq!(response.headers()[header::CONTENT_TYPE], "application/vnd.apple.mpegurl"); - assert!(app_state.hls.proxy.access_leases().read().await.is_empty()); - assert!(app_state.active_users.active_streams().await.is_empty()); -} - -#[tokio::test] -async fn legacy_hls_token_route_with_invalid_token_returns_bad_request_when_target_hls_share_enabled() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let user = app_state.app_config.get_user_credentials("hls-user").expect("test user should exist"); - let input = test_hls_input(); - let target = test_hls_share_target(true); - store_test_sources_with_target(&app_state, input.clone(), target.clone()); - - let response = super::hls_api_stream_resolved( - test_fingerprint(), - HeaderMap::new(), - Arc::clone(&app_state), - user, - Arc::new(target), - input.id, - 12345, - "not-a-valid-legacy-token.ts".to_string(), - ) - .await; - - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - assert!(app_state.hls.proxy.access_leases().read().await.is_empty()); - assert!(app_state.active_users.active_streams().await.is_empty()); -} - -#[test] -fn cache_enabled_legacy_hls_route_only_allows_existing_m3u_catchup_session() { - assert!(super::legacy_hls_route_allowed_with_cache(true, Some("m3u-catchup|session"), Some("m3u-catchup|session"))); - assert!(super::legacy_hls_route_allowed_with_cache(true, Some("catchup|session"), Some("catchup|session"))); - assert!(!super::legacy_hls_route_allowed_with_cache(true, Some("m3u-catchup|session"), Some("m3u-catchup|other"))); - assert!(!super::legacy_hls_route_allowed_with_cache(true, Some("legacy-session"), Some("legacy-session"))); - assert!(!super::legacy_hls_route_allowed_with_cache(true, None, None)); - assert!(super::legacy_hls_route_allowed_with_cache(false, None, None)); -} - -#[tokio::test] -async fn hls_cache_canonical_manifest_rejects_when_target_hls_share_disabled() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let input = test_hls_input(); - let target = test_hls_share_target(false); - store_test_sources_with_target(&app_state, input, target); - let access_context = HlsAccessContext { - username: "hls-user".to_string(), - user_session_token: "hls-session-token".to_string(), - proxy_session_id: ProxySessionId("proxy-session".to_string()), - input_id: 1, - stream_ref: "12345".to_string(), - virtual_id: 12345, - known_bitrate_bps: None, - lease_id: HlsAccessLeaseId("access-lease".to_string()), - family_key: HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), - epg_reference_ts: None, - archive_origin_url: None, - }; - - let Err(err) = - super::resolve_hls_playback_manifest_request_context(&app_state, &access_context, &HeaderMap::new()).await - else { - panic!("disabled target hls sharing should reject canonical cache path"); - }; - - assert_eq!(err, StatusCode::NOT_FOUND); -} - -#[tokio::test] -async fn hls_cache_entry_leases_update_effective_origin_acquire_policy_for_shared_session() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let mut soft_user = ProxyUserCredentials::default(); - soft_user.username = "soft-user".to_string(); - soft_user.soft_priority = 20; - let mut normal_user = ProxyUserCredentials::default(); - normal_user.username = "normal-user".to_string(); - normal_user.priority = -5; - let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; - let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; - let origin_source = super::build_hls_origin_source(&input, "12345"); - let session_key = origin_source.session_key(); - let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); - - let soft_response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &soft_user, - origin_source.clone(), - 12345, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Soft), - None, - ) - .await; - assert_eq!(soft_response.status(), StatusCode::OK); - let soft_snapshot = - app_state.hls.proxy.access_lease_session_snapshot(&proxy_session_id, super::current_time_millis()).await; - let soft_policy = soft_snapshot.effective_origin_policy.expect("soft policy"); - assert_eq!(soft_policy.connection_kind, ConnectionKind::Soft); - assert_eq!(soft_policy.priority, soft_user.soft_priority); - - // Different user/family, same shared HLS session. Normal media admission must upgrade - // the future origin-account acquire policy without changing the shared session identity. - let normal_response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &normal_user, - origin_source, - 12345, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - assert_eq!(normal_response.status(), StatusCode::OK); - let normal_snapshot = - app_state.hls.proxy.access_lease_session_snapshot(&proxy_session_id, super::current_time_millis()).await; - let normal_policy = normal_snapshot.effective_origin_policy.expect("normal policy"); - assert_eq!(normal_policy.connection_kind, ConnectionKind::Normal); - assert_eq!(normal_policy.priority, normal_user.priority); - - let (session, _) = app_state - .hls - .proxy - .get_or_create_session_with_source_and_outcome( - session_key, - super::build_hls_origin_source(&input, "12345"), - &app_state.get_encrypt_secret(), - super::current_time_millis(), - ) - .await; - app_state - .hls - .proxy - .sync_session_access_lease_count_and_detach_if_needed( - &app_state.active_users, - &app_state.active_provider, - &session, - &proxy_session_id, - super::current_time_millis(), - ) - .await; - let session_policy = session.read().await.effective_origin_acquire_policy_or_default(); - assert_eq!(session_policy.connection_kind, ConnectionKind::Normal); - assert_eq!(session_policy.priority, normal_user.priority); -} - -#[tokio::test] -async fn hls_entry_origin_reservation_requires_real_provider_handle() { - let app_state = test_app_state(); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - let input = single_hls_provider_input("missing-provider"); - - let reservation = super::try_reserve_hls_entry_origin_account_for_redirect( - &app_state, - &test_fingerprint(), - &user, - &input, - 12345, - "http://origin.example.com/live/source-user/source-pass/12345.m3u8", - "hls-session-token", - "hls-cache:test-session", - crate::model::PlaybackKind::LiveHls, - super::hls_origin_account_reservation_ttl_secs_fallback(), - UserConnectionPermission::Allowed, - ConnectionKind::Normal, - false, - ) - .await; - - assert!(reservation.is_none(), "provisioning redirect must not use an exhausted/counter-only check"); -} - -#[tokio::test] -async fn hls_entry_origin_reservation_blocks_foreign_sessions_only_after_media_confirmation() { - let input = single_hls_provider_input("available-provider"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - let session_owner = "hls-cache:test-session"; - - let reservation = super::try_reserve_hls_entry_origin_account_for_redirect( - &app_state, - &test_fingerprint(), - &user, - &input, - 12345, - "http://account.example.com/live/account-user/account-pass/12345.m3u8", - "hls-session-token", - session_owner, - crate::model::PlaybackKind::LiveHls, - super::hls_origin_account_reservation_ttl_secs_fallback(), - UserConnectionPermission::Allowed, - ConnectionKind::Normal, - false, - ) - .await - .expect("provider reservation should be acquired before provisioning redirect"); - - assert_eq!(reservation.request_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8"); - assert!(reservation.selected_provider_config.is_some()); - assert!(app_state.active_users.active_streams().await.is_empty()); - app_state.connection_manager.release_provider_handle(reservation.provider_handle); - - // A manifest-only entry reservation is still unconfirmed, so it must not block a - // different HLS session: an abandoned start or a manifest retry has to leave the - // provider capacity free for unrelated clients behind the same reverse proxy. - let other_handle = app_state.active_provider.acquire_connection_with_grace_for_session( - &input.name, - &test_addr_with_port(55251), - false, - 0, - ConnectionKind::Normal, - Some("other-owner"), - ); - assert!(other_handle.is_some(), "an unconfirmed entry reservation must not block other HLS sessions"); - app_state.connection_manager.release_provider_handle(other_handle); - - let same_owner_handle = app_state.active_provider.acquire_connection_with_grace_for_session( - &input.name, - &test_addr_with_port(55252), - false, - 0, - ConnectionKind::Normal, - Some(session_owner), - ); - assert!(same_owner_handle.is_some(), "reserved provider must be reusable by the same HLS session owner"); - app_state.connection_manager.release_provider_handle(same_owner_handle); - - // Once real media activity confirms the lease, it reserves the provider slot and - // other sessions are routed away from it. - app_state.active_provider.confirm_playback_activity(session_owner); - assert!( - app_state - .active_provider - .acquire_connection_with_grace_for_session( - &input.name, - &test_addr_with_port(55253), - false, - 0, - ConnectionKind::Normal, - Some("other-owner"), - ) - .is_none(), - "a confirmed reservation must stay blocked for other HLS sessions" - ); -} - -#[tokio::test] -async fn hls_entry_origin_reservation_uses_persisted_alias_manifest_url() { - use shared::model::PlaylistGroup; - use tuliprox_repository::{get_input_m3u_playlist_file_path, get_input_storage_path, persist_input_m3u_playlist}; - - let temp = tempfile::tempdir().expect("temp dir should be created"); - let input = ConfigInput { - id: 1, - name: Arc::from("primary-account"), - input_type: InputType::M3u, - url: "http://playlist.example/list.m3u?access_key=primary-playlist-key".to_string(), - enabled: true, - max_connections: 1, - aliases: Some(vec![crate::model::ConfigInputAlias { - id: 2, - name: Arc::from("alias-account"), - url: "http://playlist.example/list.m3u?access_key=alias-playlist-key".to_string(), - username: None, - password: None, - max_connections: 0, - priority: 0, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }; - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let mut config = (*app_state.app_config.config.load_full()).clone(); - config.storage_dir = temp.path().to_string_lossy().into_owned(); - app_state.app_config.config.store(Arc::new(config)); - - let alias_name = "alias-account".intern(); - let storage_path = get_input_storage_path(&alias_name, &app_state.app_config.config.load().storage_dir) - .await - .expect("alias storage should be created"); - let playlist_path = get_input_m3u_playlist_file_path(&storage_path, &alias_name); - let alias_playlist = vec![PlaylistGroup { - id: 1, - title: "Live".intern(), - channels: vec![PlaylistItem { - header: PlaylistItemHeader { - id: "news24hd".intern(), - input_stream_id: "news24hd".intern(), - url: "http://stream.example:4000/news24hd/mono.m3u8?token=alias-stream-token".intern(), - item_type: PlaylistItemType::Live, - xtream_cluster: XtreamCluster::Live, - ..PlaylistItemHeader::default() - }, - }], - xtream_cluster: XtreamCluster::Live, - }]; - persist_input_m3u_playlist(&app_state.app_config, &playlist_path, &alias_playlist) - .await - .expect("alias playlist should persist"); - - let primary_handle = app_state - .active_provider - .acquire_exact_connection_with_grace_for_session( - &input.name, - &test_fingerprint().addr, - false, - 0, - ConnectionKind::Normal, - Some("primary-session"), - ) - .expect("primary account should be allocated"); - - let reservation = super::try_reserve_hls_entry_origin_account_for_redirect( - &app_state, - &test_fingerprint(), - &{ - let mut creds = ProxyUserCredentials::default(); - creds.username = "hls-user".to_string(); - creds - }, - &input, - 12345, - "http://stream.example:4000/news24hd/mono.m3u8?token=primary-stream-token", - "alias-session-token", - "alias-session-owner", - crate::model::PlaybackKind::LiveHls, - super::hls_origin_account_reservation_ttl_secs_fallback(), - UserConnectionPermission::Allowed, - ConnectionKind::Normal, - false, - ) - .await - .expect("alias provider reservation should succeed"); - - assert_eq!( - reservation.selected_provider_config.as_ref().map(|provider| provider.name.as_ref()), - Some("alias-account") - ); - assert_eq!(reservation.request_url, "http://stream.example:4000/news24hd/mono.m3u8?token=alias-stream-token"); - - app_state.connection_manager.release_provider_handle(reservation.provider_handle); - app_state.connection_manager.release_provider_handle(Some(primary_handle)); -} - -#[tokio::test] -async fn hls_virtual_entry_reservation_uses_input_stream_id_for_shared_session_owner() { - let input = single_hls_provider_input("origin-id-reservation-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - enable_hls_cache(&app_state); - let target = Arc::new(test_m3u_hls_share_target()); - let item = test_m3u_hls_item( - &input, - 1001, - "80510", - "http://account.example.com/live/account-user/account-pass/80510.m3u8", - ); - cache_test_m3u_hls_item(&app_state, &target, item).await; - let stream_identity = super::HlsEntryStreamIdentity::new(1001, "80510").expect("input stream identity"); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - - assert!( - super::try_reserve_hls_virtual_entry_origin_account_for_redirect( - &app_state, - &test_fingerprint(), - &user, - &target, - &input, - &stream_identity, - ) - .await - ); - - let expected_key = HlsSessionKey::new(input.id, "80510"); - let expected_proxy_session_id = build_proxy_session_id(&expected_key, &app_state.get_encrypt_secret()); - let expected_owner = crate::api::model::build_hls_origin_session_owner(&expected_proxy_session_id); - let same_owner_handle = app_state.active_provider.acquire_connection_with_grace_for_session( - &input.name, - &test_addr_with_port(55253), - false, - 0, - ConnectionKind::Normal, - Some(&expected_owner), - ); - assert!(same_owner_handle.is_some(), "reservation must be owned by input:1|hls|80510, not virtual_id=1001"); - app_state.connection_manager.release_provider_handle(same_owner_handle); -} - -#[tokio::test] -async fn hls_cache_entry_creates_new_lease_for_same_user_session_and_proxy_session() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; - let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; - let origin_source = super::build_hls_origin_source(&input, "12345"); - - let first_response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - origin_source.clone(), - 12345, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - let first_variant_uri = single_variant_uri(first_response).await; - let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&first_variant_uri).to_string()); - let first_access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&first_variant_uri).to_string()); - let first_session_token = access_lease_session_token(&app_state, &proxy_session_id, &first_access_lease_id).await; - - let second_response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - origin_source, - 12345, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - let second_variant_uri = single_variant_uri(second_response).await; - assert_ne!( - access_lease_id_from_variant_uri(&first_variant_uri), - access_lease_id_from_variant_uri(&second_variant_uri) - ); - let second_access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&second_variant_uri).to_string()); - let second_session_token = access_lease_session_token(&app_state, &proxy_session_id, &second_access_lease_id).await; - assert_ne!(first_session_token, second_session_token); -} - -#[tokio::test] -async fn hls_cache_entry_creates_new_lease_after_manifest_touch() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; - let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; - let origin_source = super::build_hls_origin_source(&input, "12345"); - - let first_response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - origin_source.clone(), - 12345, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - let first_variant_uri = single_variant_uri(first_response).await; - let first_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&first_variant_uri).to_string()); - let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&first_variant_uri).to_string()); - let first_session_token = access_lease_session_token(&app_state, &proxy_session_id, &first_lease_id).await; - let now_ms = super::current_time_millis(); - - assert!(matches!( - app_state - .hls - .proxy - .touch_manifest_access_lease( - &first_lease_id, - &proxy_session_id, - now_ms, - None, - Some(super::HlsAccessLeasePendingDeadline::Bootstrap { - deadline_ms: now_ms.saturating_add(super::hls_pending_bootstrap_window_ms(&app_state)), - }), - super::hls_access_lease_ttl_ms(&app_state), - ) - .await, - HlsAccessLeaseTouch::Touched { .. } - )); - - let second_response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - origin_source, - 12345, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - let second_variant_uri = single_variant_uri(second_response).await; - - assert_ne!( - access_lease_id_from_variant_uri(&first_variant_uri), - access_lease_id_from_variant_uri(&second_variant_uri) - ); - let second_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&second_variant_uri).to_string()); - let second_session_token = access_lease_session_token(&app_state, &proxy_session_id, &second_lease_id).await; - assert_ne!(first_session_token, second_session_token); -} - -#[tokio::test] -async fn hls_cache_entry_does_not_reuse_activated_access_lease() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; - let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; - let origin_source = super::build_hls_origin_source(&input, "12345"); - - let first_response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - origin_source.clone(), - 12345, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - let first_variant_uri = single_variant_uri(first_response).await; - let first_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&first_variant_uri).to_string()); - let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&first_variant_uri).to_string()); - let now_ms = super::current_time_millis(); - assert!(app_state - .hls - .proxy - .activate_access_lease( - &first_lease_id, - &proxy_session_id, - now_ms, - HlsAccessLeaseTiming { - active_window_ms: 5_000, - valid_window_ms: super::hls_access_lease_ttl_ms(&app_state), - }, - ) - .await - .is_activated()); - - let second_response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - origin_source, - 12345, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - let second_variant_uri = single_variant_uri(second_response).await; - let second_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&second_variant_uri).to_string()); - - assert_ne!(first_lease_id, second_lease_id); - let first_session_token = access_lease_session_token(&app_state, &proxy_session_id, &first_lease_id).await; - let second_session_token = access_lease_session_token(&app_state, &proxy_session_id, &second_lease_id).await; - assert_ne!(first_session_token, second_session_token); - assert!( - app_state - .hls - .proxy - .touch_access_lease( - &first_lease_id, - super::current_time_millis(), - HlsAccessLeaseTiming { - active_window_ms: 5_000, - valid_window_ms: super::hls_access_lease_ttl_ms(&app_state), - }, - ) - .await - ); -} - -#[tokio::test] -async fn hls_cache_entry_ignores_existing_pending_lease_for_new_playback() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; - let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; - let origin_source = super::build_hls_origin_source(&input, "12345"); - let proxy_session_id = build_proxy_session_id(&origin_source.session_key(), &app_state.get_encrypt_secret()); - let old_lease_id = HlsAccessLeaseId("old-pending-lease".to_string()); - let old_session_token = "old-hls-session-token"; - let old_issued_at_ms = super::current_time_millis().saturating_sub(6_000); - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - old_lease_id.clone(), - HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), - proxy_session_id.clone(), - "hls-user".to_string(), - old_session_token.to_string(), - 1, - "12345".to_string(), - 12345, - old_issued_at_ms, - super::hls_access_lease_ttl_ms(&app_state), - )) - .await; - - let response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - origin_source, - 12345, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - let variant_uri = single_variant_uri(response).await; - let new_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&variant_uri).to_string()); - let new_session_token = access_lease_session_token(&app_state, &proxy_session_id, &new_lease_id).await; - - assert_ne!(old_lease_id, new_lease_id); - assert_ne!(old_session_token, new_session_token); -} - -#[tokio::test] -async fn hls_cache_parallel_real_playbacks_same_virtual_id_register_distinct_streams() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; - let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; - let origin_source = super::build_hls_origin_source(&input, "12345"); - let proxy_session_id = map_ready_segment_without_lease(&app_state, 123, "ts", b"0123456789").await; - - let first_response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - origin_source.clone(), - 12345, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - let first_variant_uri = single_variant_uri(first_response).await; - let first_lease_id = access_lease_id_from_variant_uri(&first_variant_uri); - assert_eq!(proxy_session_id_from_variant_uri(&first_variant_uri), proxy_session_id); - let proxy_session = ProxySessionId(proxy_session_id.clone()); - let first_session_token = - access_lease_session_token(&app_state, &proxy_session, &HlsAccessLeaseId(first_lease_id.to_string())).await; - publish_ready_test_manifest_for_lease( - &app_state, - &proxy_session, - &HlsAccessLeaseId(first_lease_id.to_string()), - 4_000, - ) - .await; - let first_segment_uri = format!("/hls/shared/live/{proxy_session_id}/{first_lease_id}/000123.ts"); - assert_eq!(get_status(Arc::clone(&app_state), &first_segment_uri).await, StatusCode::OK); - - let second_response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - origin_source, - 12345, - None, - None, - None, - request_url, - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - let second_variant_uri = single_variant_uri(second_response).await; - let second_lease_id = access_lease_id_from_variant_uri(&second_variant_uri); - assert_ne!(first_lease_id, second_lease_id); - assert_eq!(proxy_session_id_from_variant_uri(&second_variant_uri), proxy_session_id); - let second_session_token = - access_lease_session_token(&app_state, &proxy_session, &HlsAccessLeaseId(second_lease_id.to_string())).await; - assert_ne!(first_session_token, second_session_token); - publish_ready_test_manifest_for_lease( - &app_state, - &proxy_session, - &HlsAccessLeaseId(second_lease_id.to_string()), - 4_000, - ) - .await; - let second_segment_uri = format!("/hls/shared/live/{proxy_session_id}/{second_lease_id}/000123.ts"); - assert_eq!(get_status(Arc::clone(&app_state), &second_segment_uri).await, StatusCode::OK); - - let streams = app_state.active_users.active_streams().await; - assert_eq!(streams.len(), 2); - let first_stream = streams - .iter() - .find(|stream| stream.session_token.as_deref() == Some(first_session_token.as_str())) - .expect("first stream should be registered"); - let second_stream = streams - .iter() - .find(|stream| stream.session_token.as_deref() == Some(second_session_token.as_str())) - .expect("second stream should be registered"); - let shared_stream_id = super::hls_cache_shared_stream_id(&proxy_session); - assert_ne!(first_stream.session_token, second_stream.session_token); - assert_eq!(first_stream.channel.shared_stream_id, Some(shared_stream_id)); - assert_eq!(second_stream.channel.shared_stream_id, Some(shared_stream_id)); - assert_eq!(first_stream.channel.shared_joined_existing, Some(false)); - assert_eq!(second_stream.channel.shared_joined_existing, Some(true)); -} - -#[tokio::test] -async fn hls_cache_entry_master_playlist_for_xtream_uses_stream_ref_session_identity() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - let input = - ConfigInput { id: 7, name: Arc::from("xtream-input"), input_type: InputType::Xtream, ..ConfigInput::default() }; - let origin_source = super::build_hls_origin_source(&input, "80510"); - let expected_proxy_session_id = - build_proxy_session_id(&origin_source.session_key(), &app_state.get_encrypt_secret()); - - let response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - origin_source, - 80510, - None, - None, - None, - "http://origin.example.com/live/user/pass/80510.m3u8", - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - None, - ) - .await; - - assert_eq!(response.status(), StatusCode::OK); - let variant_uri = single_variant_uri(response).await; - assert!(variant_uri.starts_with(&format!("/hls/shared/live/{}/", expected_proxy_session_id.0))); - assert!(variant_uri.ends_with("/manifest.m3u8")); - assert!(app_state.hls.proxy.sessions().get_by_key(&HlsSessionKey::new(7, "80510")).await.is_none()); - assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0); -} - -#[tokio::test] -async fn hls_cache_entry_master_playlist_for_m3u_uses_stream_ref_session_identity() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let mut user = ProxyUserCredentials::default(); - user.username = "hls-user".to_string(); - let input = - ConfigInput { id: 9, name: Arc::from("m3u-input"), input_type: InputType::M3u, ..ConfigInput::default() }; - let origin_source = super::build_hls_origin_source(&input, "70001"); - let expected_proxy_session_id = - build_proxy_session_id(&origin_source.session_key(), &app_state.get_encrypt_secret()); - - let response = super::create_hls_cache_entry_master_playlist_response( - &app_state, - &test_fingerprint(), - &user, - origin_source, - 70001, - None, - None, - None, - "http://media.example.com/channel/playlist.m3u8", - &input, - UserConnectionPermission::Allowed, - Some(ConnectionKind::Normal), - Some("/iptv"), - ) - .await; - - assert_eq!(response.status(), StatusCode::OK); - let variant_uri = single_variant_uri(response).await; - assert!(variant_uri.starts_with(&format!("/iptv/hls/shared/live/{}/", expected_proxy_session_id.0))); - assert!(variant_uri.ends_with("/manifest.m3u8")); - assert!(app_state.hls.proxy.sessions().get_by_key(&HlsSessionKey::new(9, "70001")).await.is_none()); - assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0); -} - -#[tokio::test] -async fn hls_proxy_manifest_invalid_token_starts_no_origin_work() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - - let response = get_response( - Arc::clone(&app_state), - "/hls/shared/live/a8f31c9eQ7sLk92pV0mTaw/not-a-valid-token/manifest.m3u8", - None, - ) - .await; - - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0); - assert!(app_state.hls.proxy.sessions().is_empty().await); -} - -async fn prepare_server_path_manifest_session(app_state: &Arc) -> (HlsSessionHandle, ProxySessionId) { - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let proxy_session_id = { - let mut session = session.write().await; - session.origin_refresh.next_fetch_allowed_at_ms = u64::MAX; - let proxy_session_id = session.proxy_session_id.0.clone(); - let rendered_at_ms = super::current_time_millis(); - let map_id = ProxyMapId(0); - let mut map = MapEntry::new( - &session.proxy_session_id, - map_id, - OriginMapKey { - origin_epoch: 0, - resolved_origin_uri: "http://origin.example.com/init.mp4".to_string(), - byte_range: None, - }, - "mp4".to_string(), - ); - map.status = MapCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }; - session.maps.insert(map_id, map); - let mut segment = test_segment_entry( - &session.proxy_session_id, - 123, - SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }, - ); - segment.map_ref = Some(map_id); - session.segments.insert(123, segment); - session.advance_media_readiness_generation(); - session.last_rendered_manifest = Some(RenderedManifest { - body: format!( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:123\n#EXT-X-MAP:URI=\"/hls/shared/live/{proxy_session_id}/{}/map/000000.mp4\"\n#EXTINF:4.0,\n/hls/shared/live/{proxy_session_id}/{}/000123.ts\n", - crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER, - crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER - ), - first_proxy_seq: 123, - last_proxy_seq: 123, - playlist_duration_ms: 4_000, - valid_until_ms: rendered_at_ms.saturating_add(4_000), - render_gap_segments: 0, - rendered_at_ms, - discontinuity_sequence: 0, - target_duration_ms: 4_000, - segment_proxy_seqs: vec![123], - }); - record_test_normal_manifest_commit(&mut session, rendered_at_ms); - ProxySessionId(proxy_session_id) - }; - (session, proxy_session_id) -} - -#[tokio::test] -async fn hls_cache_manifest_response_applies_current_users_server_path_without_mutating_session_body() { - let input_name = Arc::::from("test-input"); - let input = ConfigInput { - id: 1, - name: Arc::clone(&input_name), - input_type: InputType::Xtream, - url: "http://origin.example.com".to_string(), - username: Some("user".to_string()), - password: Some("pass".to_string()), - max_connections: 1, - enabled: true, - ..ConfigInput::default() - }; - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - enable_hls_cache(&app_state); - create_active_hls_user_session(&app_state).await; - let (session, proxy_session_id) = prepare_server_path_manifest_session(&app_state).await; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); - prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; - - let response = super::try_hls_cache_canonical_manifest_response( - &app_state, - &test_fingerprint(), - &access_context, - &proxy_session_id, - &access_lease_id, - HlsAccessLeaseState::Pending, - super::HlsCacheManifestOrigin { - raw_request_url: "http://origin.example.com/live/user/pass/12345.m3u8", - session_entry_url: super::HlsOriginEntryUrl::direct_http( - "http://origin.example.com/live/user/pass/12345.m3u8", - ), - input: &input, - origin_source: super::build_hls_origin_source(&input, "12345"), - }, - HeaderMap::new(), - Some("/iptv"), - "/live/hls-user/hls-pass/12345.m3u8", - super::HlsManifestRefreshOrdering::Background, - ) - .await - .expect("hls cache should handle valid live hls entrypoint"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); - - assert!(body.contains(&format!("/iptv/hls/shared/live/{}/", proxy_session_id.0))); - assert!(body.contains("/map/000000.mp4")); - assert!(body.contains("/000123.ts")); - assert!(body.contains(&access_lease_id.0)); - assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); - let stored_body = session.read().await.last_rendered_manifest.as_ref().expect("stored manifest").body.clone(); - assert!(stored_body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); - assert!(stored_body.contains(&format!("/hls/shared/live/{}/", proxy_session_id.0))); - assert!(!stored_body.contains("/iptv/hls/shared/live/")); - assert!(!stored_body.contains(&access_lease_id.0)); - assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); -} - -fn transient_manifest_body(proxy_session_id: &str) -> String { - transient_manifest_body_from_sequence(proxy_session_id, 100, 6) -} - -fn transient_manifest_body_from_sequence(proxy_session_id: &str, first_sequence: u64, count: usize) -> String { - let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:10\n#EXT-X-MEDIA-SEQUENCE:{first_sequence}\n"); - for index in 0..count { - let sequence = first_sequence.saturating_add(u64::try_from(index).expect("test sequence index fits u64")); - body.push_str("#EXTINF:10.0,\n"); - let _ = writeln!( - body, - "/hls/shared/live/{proxy_session_id}/{}/r/seg{sequence}.ts", - crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER - ); - } - body -} - -fn media_uri_count(body: &str) -> usize { - body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).count() -} - -fn normal_manifest_body(proxy_session_id: &str) -> String { normal_manifest_body_from_sequence(proxy_session_id, 0, 6) } - -fn normal_manifest_body_from_sequence(proxy_session_id: &str, first_sequence: u64, count: usize) -> String { - let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:10\n#EXT-X-MEDIA-SEQUENCE:{first_sequence}\n"); - for index in 0..count { - let sequence = first_sequence.saturating_add(u64::try_from(index).expect("test sequence index fits u64")); - body.push_str("#EXTINF:10.0,\n"); - let _ = writeln!( - body, - "/hls/shared/live/{proxy_session_id}/{}/{sequence:06}.ts", - crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER - ); - } - body -} - -fn store_normal_manifest_body(session: &mut HlsSession, body: String, rendered_at_ms: u64) { - store_normal_manifest_body_range(session, body, 0, 6, rendered_at_ms); -} - -fn record_test_normal_manifest_commit(session: &mut HlsSession, rendered_at_ms: u64) { - let identity = session - .next_manifest_commit_identity(rendered_at_ms) - .expect("test manifest commit generation remains available"); - session.record_normal_manifest_commit_identity(identity); -} - -fn store_normal_manifest_body_range( - session: &mut HlsSession, - body: String, - first_proxy_seq: u64, - count: usize, - rendered_at_ms: u64, -) { - let last_proxy_seq = - first_proxy_seq.saturating_add(u64::try_from(count.saturating_sub(1)).expect("test count fits u64")); - let proxy_session_id = session.proxy_session_id.clone(); - for proxy_seq in first_proxy_seq..=last_proxy_seq { - let mut entry = test_segment_entry( - &proxy_session_id, - proxy_seq, - SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }, - ); - entry.duration_ms = 10_000; - session.segments.insert(proxy_seq, entry); - } - session.advance_media_readiness_generation(); - session.last_rendered_manifest = Some(RenderedManifest { - body, - first_proxy_seq, - last_proxy_seq, - playlist_duration_ms: 60_000, - valid_until_ms: rendered_at_ms.saturating_add(60_000), - render_gap_segments: 0, - rendered_at_ms, - discontinuity_sequence: 0, - target_duration_ms: 10_000, - segment_proxy_seqs: (first_proxy_seq..=last_proxy_seq).collect(), - }); - record_test_normal_manifest_commit(session, rendered_at_ms); -} - -async fn try_test_hls_cached_manifest_response( - app_state: &Arc, - session: &HlsSessionHandle, - access_lease_id: &HlsAccessLeaseId, - access_lease_state: HlsAccessLeaseState, - strip: &StripConfig, - server_path: Option<&str>, - options: super::HlsCachedManifestOptions, -) -> Option { - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let now_ms = super::current_time_millis(); - if app_state.hls.proxy.access_lease_response_snapshot(access_lease_id, &proxy_session_id, now_ms).await.is_none() { - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - access_lease_id.clone(), - HlsPlaybackFamilyKey::new("test-user", "manifest-test-client"), - proxy_session_id, - "test-user".to_string(), - "manifest-test-session".to_string(), - 1, - "12345".to_string(), - 12345, - now_ms, - 60_000, - )) - .await; - } - super::try_hls_cached_manifest_response( - app_state, - session, - access_lease_id, - access_lease_state, - strip, - server_path, - options, - super::HlsRuntimeBandwidthLearningContext::Disabled, - ) - .await -} - -#[test] -fn repeated_speculative_strip_candidates_yield_one_committed_applied_diagnostic() { - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let body = normal_manifest_body("proxy-session"); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - let candidates = (0..8) - .map(|_| { - super::materialize_shared_hls_access_manifest( - &body, - &access_lease_id, - HlsAccessLeaseState::Pending, - &strip, - super::HlsManifestWindowPolicy::ApplyLiveWindow, - "normal", - None, - ) - }) - .collect::>(); - - assert_eq!(candidates.len(), 8); - let candidate_count = candidates.len(); - let diagnostics = candidates - .iter() - .enumerate() - .filter_map(|(index, candidate)| { - super::hls_initial_strip_publication_diagnostic( - if index.saturating_add(1) == candidate_count { - super::HlsInitialStripPublicationStatus::Committed - } else { - super::HlsInitialStripPublicationStatus::NotCommitted - }, - HlsAccessLeaseState::Pending, - candidate, - ) - }) - .collect::>(); - - assert_eq!( - diagnostics, - vec![super::HlsInitialStripPublicationDiagnostic::Applied { - mode: "normal", - strip_mode: "segments", - configured: 3, - effective: 3, - visible_segments: 3, - }] - ); -} - -#[test] -fn committed_pending_strip_disabled_yields_one_skipped_diagnostic() { - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let materialized = super::materialize_shared_hls_access_manifest( - &normal_manifest_body("proxy-session"), - &access_lease_id, - HlsAccessLeaseState::Pending, - &StripConfig { mode: HlsStripMode::Segments, value: 0 }, - super::HlsManifestWindowPolicy::ApplyLiveWindow, - "normal", - None, - ); - - let diagnostics = [super::hls_initial_strip_publication_diagnostic( - super::HlsInitialStripPublicationStatus::Committed, - HlsAccessLeaseState::Pending, - &materialized, - ) - .expect("committed strip diagnostic")]; - - assert_eq!( - diagnostics, - [super::HlsInitialStripPublicationDiagnostic::Skipped { - mode: "normal", - reason: crate::api::model::hls_cache::initial_strip::HlsInitialStripSkipReason::StripDisabled, - visible_segments: 6, - }] - ); - assert_eq!(media_uri_count(&materialized.body), 6); -} - -#[test] -fn pending_playlist_type_manifest_ignores_three_to_six_window_and_strip() { - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - for segment_count in [2, 8] { - let body = transient_manifest_body_from_sequence("proxy-session", 100, segment_count).replacen( - "#EXTM3U\n", - "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n", - 1, - ); - let materialized = super::materialize_shared_hls_access_manifest( - &body, - &access_lease_id, - HlsAccessLeaseState::Pending, - &StripConfig { mode: HlsStripMode::Segments, value: 5 }, - super::HlsManifestWindowPolicy::PreserveFullManifest, - "transient", - None, - ); - - assert_eq!(media_uri_count(&materialized.body), segment_count); - assert!(materialized.body.contains("#EXT-X-PLAYLIST-TYPE:EVENT")); - assert!(!materialized.body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); - assert_eq!( - materialized.initial_strip_outcome, - Some(super::HlsInitialStripOutcome::Skipped { - reason: super::HlsInitialStripSkipReason::ManifestSemanticsPreserveFullManifest, - visible_segments: segment_count, - }) - ); - } -} - -#[test] -fn pending_endlist_only_manifest_keeps_complete_body_despite_strip() { - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let mut body = transient_manifest_body_from_sequence("proxy-session", 100, 8); - body.push_str("#EXT-X-ENDLIST\n"); - let window_policy = - crate::processing::parser::hls::origin_manifest::parse_manifest_semantics(&body).window_policy(); - assert_eq!(window_policy, super::HlsManifestWindowPolicy::PreserveFullManifest); - - let materialized = super::materialize_shared_hls_access_manifest( - &body, - &access_lease_id, - HlsAccessLeaseState::Pending, - &StripConfig { mode: HlsStripMode::Segments, value: 5 }, - window_policy, - "transient", - None, - ); - - assert_eq!(media_uri_count(&materialized.body), 8); - assert!(materialized.body.contains("#EXT-X-ENDLIST")); - assert_eq!( - materialized.initial_strip_outcome, - Some(super::HlsInitialStripOutcome::Skipped { - reason: super::HlsInitialStripSkipReason::ManifestSemanticsPreserveFullManifest, - visible_segments: 8, - }) - ); -} - -#[test] -fn committed_activated_manifest_yields_one_lease_state_skip_diagnostic() { - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let materialized = super::materialize_shared_hls_access_manifest( - &normal_manifest_body("proxy-session"), - &access_lease_id, - HlsAccessLeaseState::Activated, - &StripConfig { mode: HlsStripMode::Segments, value: 3 }, - super::HlsManifestWindowPolicy::ApplyLiveWindow, - "normal", - None, - ); - - let diagnostics = [super::hls_initial_strip_publication_diagnostic( - super::HlsInitialStripPublicationStatus::Committed, - HlsAccessLeaseState::Activated, - &materialized, - ) - .expect("committed strip diagnostic")]; - - assert_eq!( - diagnostics, - [super::HlsInitialStripPublicationDiagnostic::SkippedForLeaseState { - mode: "normal", - reason: super::HlsInitialStripLeaseSkipReason::LeaseActivated, - }] - ); - assert_eq!(media_uri_count(&materialized.body), 6); -} - -#[tokio::test(start_paused = true)] -async fn pending_strip_admission_timeout_does_not_commit_speculative_candidate() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let proxy_session_id = { - let mut session = session.write().await; - let proxy_session_id = session.proxy_session_id.clone(); - let rendered_at_ms = super::current_time_millis(); - store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id.0), rendered_at_ms); - session.segments.get_mut(&1).expect("visible test segment").status = SegmentCacheStatus::Discovered; - session.advance_media_readiness_generation(); - session.mark_authorized_media_access(rendered_at_ms); - proxy_session_id - }; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Pending, - &StripConfig { mode: HlsStripMode::Segments, value: 3 }, - None, - super::HlsCachedManifestOptions::committed_only(Duration::from_millis(75)), - ) - .await - .expect("timeout response"); - - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - let lease = app_state - .hls - .proxy - .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::current_time_millis()) - .await - .expect("pending lease remains available"); - assert!(lease.last_manifest_snapshot.is_none()); -} - -async fn publish_ready_test_manifest_for_lease( - app_state: &Arc, - proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, - target_duration_ms: u64, -) { - let session = - app_state.hls.proxy.sessions().get_by_proxy_session_id(proxy_session_id).await.expect("test session exists"); - let (proxy_seq, duration_ms) = { - let session = session.read().await; - session - .segments - .iter() - .find_map(|(proxy_seq, entry)| { - matches!(entry.status, SegmentCacheStatus::Ready { .. }).then_some((*proxy_seq, entry.duration_ms)) - }) - .expect("READY test segment") - }; - let now_ms = super::current_time_millis(); - let publication_guard = app_state - .hls - .proxy - .prepare_access_lease_manifest_publication(access_lease_id, proxy_session_id, now_ms) - .await - .expect("test lease accepts publication"); - let snapshot = HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(now_ms), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 0, - delivered_at_ms: now_ms, - first_proxy_seq: proxy_seq, - last_proxy_seq: proxy_seq, - visible_segments: Arc::from([HlsLeaseManifestSegment { - proxy_seq, - duration_ms, - uri: format!("/hls/shared/live/{}/{}/{proxy_seq:06}.ts", proxy_session_id.0, access_lease_id.0).into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }]), - discontinuity_sequence: 0, - target_duration_ms: target_duration_ms.max(duration_ms), - playlist_duration_ms: duration_ms, - last_visible_media_end_ms: duration_ms, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - }; - assert!(app_state - .hls - .proxy - .commit_access_lease_manifest_publication( - access_lease_id, - proxy_session_id, - publication_guard, - snapshot, - now_ms, - ) - .await - .is_committed()); -} - -async fn prepare_user_exhausted_terminal_bundle(app_state: &Arc, target_duration_ms: u64) { - let response = app_state.app_config.custom_stream_response.load_full().expect("runtime custom responses"); - let asset = response - .user_connections_exhausted - .as_ref() - .and_then(|buffer| snapshot_terminal_media_asset(buffer).ok()) - .expect("valid user-exhausted terminal asset"); - let key = prepared_terminal_bundle_key(&asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); - let state = - app_state.hls.proxy.start_prepared_terminal_bundle(asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); - let state = match state { - HlsPreparedTerminalBundleState::Preparing { .. } => app_state - .hls - .proxy - .wait_for_prepared_terminal_bundle(key) - .await - .expect("user-exhausted terminal bundle completion"), - state => state, - }; - assert!(matches!( - state, - HlsPreparedTerminalBundleState::Ready { ref bundle } if bundle.key == key - )); -} - -struct RuntimePolicyEndpointFixture { - _temp_dir: tempfile::TempDir, - app_state: Arc, - proxy_session_id: ProxySessionId, - lease_id: HlsAccessLeaseId, - manifest_uri: String, - live_segment_uri: String, -} - -async fn assert_runtime_policy_base_timing( - app_state: &Arc, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, - phase: &str, -) { - let session = - app_state.hls.proxy.sessions().get_by_proxy_session_id(proxy_session_id).await.expect("runtime policy session"); - let manifest = app_state - .hls - .proxy - .access_lease_response_snapshot(lease_id, proxy_session_id, super::current_time_millis()) - .await - .and_then(|lease| lease.last_manifest_snapshot) - .expect("runtime policy manifest"); - let evidence = prepare_terminal_base_evidence( - &session, - app_state.hls.proxy.segment_cache(), - &manifest, - super::current_time_millis(), - ) - .await; - assert!( - evidence.timing().is_some(), - "runtime policy base timing {phase}: {}", - evidence.track_evidence_reason_code() - ); - evidence.release(); -} - -async fn serve_and_wait_runtime_policy_base_segment( - app_state: &Arc, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, - live_segment_uri: &str, -) { - let initial_segment = get_response(Arc::clone(app_state), live_segment_uri, None).await; - assert_eq!(initial_segment.status(), StatusCode::OK); - assert!(!response_body(initial_segment).await.is_empty()); - tokio::time::timeout(Duration::from_secs(2), async { - loop { - let completed = app_state - .hls - .proxy - .access_lease_response_snapshot(lease_id, proxy_session_id, super::current_time_millis()) - .await - .and_then(|lease| lease.playback_cursor.highest_contiguous_completed_proxy_seq); - if completed == Some(123) { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("initial live segment completion"); -} - -async fn runtime_policy_endpoint_fixture(publish_manifest: bool) -> RuntimePolicyEndpointFixture { - const TARGET_DURATION_MS: u64 = 12_000; - - let temp_dir = tempfile::tempdir().expect("runtime policy cache tempdir"); - let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); - enable_runtime_policy_custom_responses(&app_state); - let live_bytes = - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); - let proxy_session_id = ProxySessionId(map_ready_segment_without_lease(&app_state, 123, "ts", live_bytes).await); - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&proxy_session_id) - .await - .expect("runtime policy session"); - session.write().await.segments.get_mut(&123).expect("runtime policy base segment").duration_ms = TARGET_DURATION_MS; - let lease_id = HlsAccessLeaseId(grant_hls_proxy_lease(&app_state, &proxy_session_id.0).await); - if publish_manifest { - publish_ready_test_manifest_for_lease(&app_state, &proxy_session_id, &lease_id, TARGET_DURATION_MS).await; - let target_duration_ms = app_state - .hls - .proxy - .access_lease_response_snapshot(&lease_id, &proxy_session_id, super::current_time_millis()) - .await - .and_then(|lease| lease.last_manifest_snapshot) - .map(|manifest| manifest.target_duration_ms) - .expect("published runtime policy target duration"); - prepare_user_exhausted_terminal_bundle(&app_state, target_duration_ms).await; - } - let manifest_uri = format!("/hls/shared/live/{}/{}/manifest.m3u8", proxy_session_id.0, lease_id.0); - let live_segment_uri = format!("/hls/shared/live/{}/{}/000123.ts", proxy_session_id.0, lease_id.0); - if publish_manifest { - assert_runtime_policy_base_timing(&app_state, &proxy_session_id, &lease_id, "before serve").await; - } - serve_and_wait_runtime_policy_base_segment(&app_state, &proxy_session_id, &lease_id, &live_segment_uri).await; - if publish_manifest { - assert_runtime_policy_base_timing(&app_state, &proxy_session_id, &lease_id, "after serve").await; - } - - RuntimePolicyEndpointFixture { - _temp_dir: temp_dir, - app_state, - proxy_session_id, - lease_id, - manifest_uri, - live_segment_uri, - } -} - -async fn wait_for_runtime_policy_terminal_plan(fixture: &RuntimePolicyEndpointFixture) -> Arc { - let plan = tokio::time::timeout(Duration::from_secs(10), async { - loop { - if let Some(HlsLeasePlaybackMode::TerminalTail(plan)) = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot( - &fixture.lease_id, - &fixture.proxy_session_id, - super::current_time_millis(), - ) - .await - .map(|lease| lease.playback_mode) - { - return plan; - } - tokio::task::yield_now().await; - } - }) - .await; - if let Ok(plan) = plan { - return plan; - } - let lease = fixture - .app_state - .hls - .proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, super::current_time_millis()) - .await; - let state = lease.as_ref().map_or("missing", |lease| lease.state.as_log_value()); - let playback = lease.as_ref().map_or("missing", |lease| match lease.playback_mode { - HlsLeasePlaybackMode::Live => "live", - HlsLeasePlaybackMode::TerminalTail(_) => "terminal-tail", - HlsLeasePlaybackMode::TerminalUnavailable { .. } => "terminal-unavailable", - HlsLeasePlaybackMode::Ended => "ended", - }); - panic!( - "runtime policy terminal owner deadline: state={state} playback={playback} owners={}", - fixture.app_state.hls.proxy.terminal_pending().owner_count() - ); -} - -#[tokio::test] -async fn hls_cache_pending_normal_manifest_applies_initial_strip_without_mutating_shared_body() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let _proxy_session_id = { - let mut session = session.write().await; - let proxy_session_id = session.proxy_session_id.0.clone(); - let rendered_at_ms = super::current_time_millis(); - store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id), rendered_at_ms); - session.mark_authorized_media_access(rendered_at_ms); - proxy_session_id - }; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - let stored_before = session.read().await.last_rendered_manifest.as_ref().expect("normal manifest").body.clone(); - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Pending, - &strip, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - ) - .await - .expect("normal manifest response"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - - assert_eq!(media_uri_count(&body), 3); - assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0\n")); - assert!(body.contains("/000000.ts")); - assert!(body.contains("/000001.ts")); - assert!(body.contains("/000002.ts")); - assert!(!body.contains("/000003.ts")); - assert!(body.contains(&access_lease_id.0)); - assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); - assert_eq!(session.read().await.last_rendered_manifest.as_ref().expect("stored manifest").body, stored_before); - assert_eq!(media_uri_count(&stored_before), 6); -} - -#[tokio::test] -async fn hls_cache_idle_normal_manifest_applies_initial_strip_without_mutating_shared_body() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let _proxy_session_id = { - let mut session = session.write().await; - let proxy_session_id = session.proxy_session_id.0.clone(); - let rendered_at_ms = super::current_time_millis(); - store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id), rendered_at_ms); - session.mark_authorized_media_access(rendered_at_ms); - proxy_session_id - }; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - let stored_before = session.read().await.last_rendered_manifest.as_ref().expect("normal manifest").body.clone(); - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Idle, - &strip, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - ) - .await - .expect("idle normal manifest response"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - - assert_eq!(media_uri_count(&body), 3); - assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0\n")); - assert!(body.contains("/000000.ts")); - assert!(body.contains("/000001.ts")); - assert!(body.contains("/000002.ts")); - assert!(!body.contains("/000003.ts")); - assert!(body.contains(&access_lease_id.0)); - assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); - assert_eq!(session.read().await.last_rendered_manifest.as_ref().expect("stored manifest").body, stored_before); - assert_eq!(media_uri_count(&stored_before), 6); -} - -#[tokio::test] -async fn hls_cache_activated_normal_manifest_skips_initial_strip() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let _proxy_session_id = { - let mut session = session.write().await; - let proxy_session_id = session.proxy_session_id.0.clone(); - let rendered_at_ms = super::current_time_millis(); - store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id), rendered_at_ms); - session.mark_authorized_media_access(rendered_at_ms); - proxy_session_id - }; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - ) - .await - .expect("normal manifest response"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - - assert_eq!(media_uri_count(&body), 6); - assert!(body.contains(&access_lease_id.0)); - assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); -} - -#[tokio::test] -async fn hls_cache_fresh_required_normal_manifest_does_not_serve_stale_committed_body() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let rendered_at_ms = { - let mut session = session.write().await; - let proxy_session_id = session.proxy_session_id.0.clone(); - let rendered_at_ms = super::current_time_millis(); - store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id), rendered_at_ms); - session.mark_authorized_media_access(rendered_at_ms); - rendered_at_ms - }; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - super::HlsCachedManifestOptions::initial(Duration::ZERO).requiring_newer_manifest(rendered_at_ms), - ) - .await; - - assert!(response.is_none()); -} - -#[tokio::test] -async fn hls_cache_fresh_required_normal_manifest_waits_for_newer_commit() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let (proxy_session_id, old_rendered_at_ms) = { - let mut session = session.write().await; - let proxy_session_id = session.proxy_session_id.clone(); - let old_rendered_at_ms = super::current_time_millis(); - store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id.0), old_rendered_at_ms); - session.origin_refresh.in_flight = true; - session.mark_authorized_media_access(old_rendered_at_ms); - (proxy_session_id, old_rendered_at_ms) - }; - let session_for_commit = Arc::clone(&session); - let proxy_session_for_body = proxy_session_id.0.clone(); - tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(20)).await; - let mut session = session_for_commit.write().await; - let rendered_at_ms = super::current_time_millis(); - let mut entry = test_segment_entry( - &session.proxy_session_id, - 100, - SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }, - ); - entry.duration_ms = 4_000; - session.segments.insert(100, entry); - session.advance_media_readiness_generation(); - session.last_rendered_manifest = Some(RenderedManifest { - body: format!( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n/hls/shared/live/{proxy_session_for_body}/{}/000100.ts\n", - crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER - ), - first_proxy_seq: 100, - last_proxy_seq: 100, - playlist_duration_ms: 4_000, - valid_until_ms: rendered_at_ms.saturating_add(4_000), - render_gap_segments: 0, - rendered_at_ms, - discontinuity_sequence: 0, - target_duration_ms: 4_000, - segment_proxy_seqs: vec![100], - }); - record_test_normal_manifest_commit(&mut session, rendered_at_ms); - session.origin_refresh.in_flight = false; - }); - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 0 }; - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - super::HlsCachedManifestOptions::initial(Duration::from_millis(200)) - .requiring_newer_manifest(old_rendered_at_ms), - ) - .await - .expect("fresh manifest response"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - - assert!(body.contains("000100.ts")); - assert!(!body.contains("000000.ts")); - assert!(body.contains(&access_lease_id.0)); -} - -#[tokio::test] -async fn hls_cache_pending_transient_manifest_applies_initial_strip_without_mutating_shared_body() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let _proxy_session_id = { - let mut session = session.write().await; - session.mode = - HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; - let proxy_session_id = session.proxy_session_id.0.clone(); - session.transient.replace_manifest_with_semantics(transient_manifest_body(&proxy_session_id), 100, None); - session.mark_authorized_media_access(super::current_time_millis()); - proxy_session_id - }; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - let stored_before = session.read().await.transient.last_manifest_body.clone().expect("transient manifest body"); - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Pending, - &strip, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - ) - .await - .expect("transient manifest response"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - - assert_eq!(media_uri_count(&body), 3); - assert!(body.contains(&access_lease_id.0)); - assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); - assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); - assert_eq!(session.read().await.transient.last_manifest_body.as_ref().expect("stored manifest"), &stored_before); - assert_eq!(media_uri_count(&stored_before), 6); -} - -fn assert_encrypted_transient_terminal_incompatibility(snapshot: HlsLeaseManifestSnapshot, now_ms: u64) { - let asset = terminal_test_asset(); - let bundle_target_duration_ms = snapshot.target_duration_ms.max(asset.duration_ms()); - let base_timing = Some(HlsTerminalTailBuildInput::base_timing_for_test(&asset, &snapshot)); - let base_splice_evidence = Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); - let terminal_splice_evidence = base_splice_evidence.clone(); - assert_eq!( - build_terminal_tail_plan(HlsTerminalTailBuildInput { - generation: HlsTerminalTailGeneration(1), - created_at_ms: now_ms, - base_availability: Arc::from([]), - base_track_signature: Some(asset.track_signature().clone()), - base_splice_evidence, - terminal_splice_evidence, - base_timing, - base_key_bindings: Arc::from([]), - expected_asset: HlsRuntimeCustomTailAssetIdentity::channel_unavailable( - HlsTerminalAssetIdentity::from_asset(&asset), - ), - anchored_bundle: HlsTerminalTailBuildInput::anchored_bundle_for_test(&asset, bundle_target_duration_ms,), - base_manifest: snapshot, - asset, - }), - Err(HlsTerminalTailCompatibility::TransientPassthroughUnsupported) - ); -} - -#[tokio::test] -async fn encrypted_transient_endpoint_stores_client_visible_key_and_typed_terminal_incompatibility() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let now_ms = super::current_time_millis(); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), now_ms) - .await; - let (proxy_session_id, access_lease_id) = { - let mut session = session.write().await; - session.mode = - HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; - let proxy_session_id = session.proxy_session_id.clone(); - let access_lease_id = HlsAccessLeaseId("encrypted-access-lease".to_string()); - let body = format!( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:20\n\ - #EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/{}/{}/r/key.bin\",IV=0x1,KEYFORMAT=\"identity\",KEYFORMATVERSIONS=\"1\"\n\ - #EXTINF:4.0,\n/hls/shared/live/{}/{}/r/20.ts\n\ - #EXTINF:4.0,\n/hls/shared/live/{}/{}/r/21.ts\n\ - #EXTINF:4.0,\n/hls/shared/live/{}/{}/r/22.ts\n", - proxy_session_id.0, - crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER, - proxy_session_id.0, - crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER, - proxy_session_id.0, - crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER, - proxy_session_id.0, - crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER, - ); - session.transient.replace_manifest_with_semantics(body, now_ms, Some(12_000)); - session.mark_authorized_media_access(now_ms); - (proxy_session_id, access_lease_id) - }; - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - access_lease_id.clone(), - HlsPlaybackFamilyKey::new("hls-user", "encrypted-client"), - proxy_session_id.clone(), - "hls-user".to_string(), - "encrypted-session".to_string(), - 1, - "12345".to_string(), - 12345, - now_ms, - 60_000, - )) - .await; - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Pending, - &StripConfig { mode: HlsStripMode::Segments, value: 0 }, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - ) - .await - .expect("encrypted transient response"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - let lease = app_state - .hls - .proxy - .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms) - .await - .expect("lease snapshot"); - let snapshot = lease.last_manifest_snapshot.expect("manifest snapshot"); - let encryption = snapshot.active_encryption.as_ref().expect("active encryption"); - - assert!(body.contains(&format!("URI=\"/hls/shared/live/{}/{}/r/key.bin\"", proxy_session_id.0, access_lease_id.0))); - assert_eq!(snapshot.delivery_mode, HlsManifestDeliveryMode::TransientPassthrough); - assert_eq!(encryption.method, "AES-128"); - assert_eq!(encryption.iv.as_deref(), Some("0x1")); - assert_eq!(encryption.key_format.as_deref(), Some("identity")); - assert_eq!(encryption.key_format_versions.as_deref(), Some("1")); - assert!(encryption.can_reset_to_clear); - - assert_encrypted_transient_terminal_incompatibility(snapshot, now_ms); -} - -#[tokio::test] -async fn hls_cache_idle_transient_manifest_applies_initial_strip_without_mutating_shared_body() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let _proxy_session_id = { - let mut session = session.write().await; - session.mode = - HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; - let proxy_session_id = session.proxy_session_id.0.clone(); - session.transient.replace_manifest_with_semantics(transient_manifest_body(&proxy_session_id), 100, None); - session.mark_authorized_media_access(super::current_time_millis()); - proxy_session_id - }; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - let stored_before = session.read().await.transient.last_manifest_body.clone().expect("transient manifest body"); - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Idle, - &strip, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - ) - .await - .expect("idle transient manifest response"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - - assert_eq!(media_uri_count(&body), 3); - assert!(body.contains(&access_lease_id.0)); - assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); - assert_eq!(session.read().await.transient.last_manifest_body.as_ref().expect("stored manifest"), &stored_before); - assert_eq!(media_uri_count(&stored_before), 6); -} - -#[tokio::test] -async fn hls_cache_activated_transient_manifest_skips_initial_strip() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let _proxy_session_id = { - let mut session = session.write().await; - session.mode = - HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; - let proxy_session_id = session.proxy_session_id.0.clone(); - session.transient.replace_manifest_with_semantics(transient_manifest_body(&proxy_session_id), 100, None); - session.mark_authorized_media_access(super::current_time_millis().saturating_sub(16_000)); - proxy_session_id - }; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - ) - .await - .expect("transient manifest response"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - - assert_eq!(media_uri_count(&body), 6); - assert!(body.contains(&access_lease_id.0)); - assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); -} - -#[tokio::test] -async fn hls_cache_transient_manifest_without_media_activity_is_not_served_from_committed_body() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let _proxy_session_id = { - let mut session = session.write().await; - session.mode = - HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; - let proxy_session_id = session.proxy_session_id.0.clone(); - let rendered_at_ms = super::current_time_millis(); - session.transient.replace_manifest_with_semantics( - transient_manifest_body(&proxy_session_id), - rendered_at_ms, - Some(60_000), - ); - proxy_session_id - }; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - ) - .await; - - assert!(response.is_none()); -} - -#[tokio::test] -async fn hls_cache_no_media_yet_transient_manifest_is_served_for_initial_canonical_response() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let _proxy_session_id = { - let mut session = session.write().await; - session.mode = - HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; - let proxy_session_id = session.proxy_session_id.0.clone(); - let rendered_at_ms = super::current_time_millis(); - session.transient.replace_manifest_with_semantics( - transient_manifest_body(&proxy_session_id), - rendered_at_ms, - Some(60_000), - ); - proxy_session_id - }; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Pending, - &strip, - None, - super::HlsCachedManifestOptions::initial(Duration::ZERO), - ) - .await - .expect("initial transient manifest response"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - - assert_eq!(media_uri_count(&body), 3); - assert!(body.contains(&access_lease_id.0)); - assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); -} - -#[tokio::test] -async fn hls_cache_transient_manifest_outside_soft_window_is_not_served_from_committed_body() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let _proxy_session_id = { - let mut session = session.write().await; - session.mode = - HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; - let proxy_session_id = session.proxy_session_id.0.clone(); - session.transient.replace_manifest_with_semantics(transient_manifest_body(&proxy_session_id), 100, None); - session.mark_authorized_media_access(super::current_time_millis().saturating_sub(60_000)); - proxy_session_id - }; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Activated, - &strip, - None, - super::HlsCachedManifestOptions::committed_only(Duration::ZERO), - ) - .await; - - assert!(response.is_none()); -} - -#[tokio::test] -async fn hls_cache_expired_transient_manifest_with_active_binding_is_served_while_manifest_valid() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let now_ms = super::current_time_millis(); - let _proxy_session_id = { - let mut session = session.write().await; - session.mode = - HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; - let proxy_session_id = session.proxy_session_id.clone(); - session.transient.replace_manifest_with_semantics( - transient_manifest_body(&proxy_session_id.0), - now_ms.saturating_sub(1_000), - Some(60_000), - ); - session.mark_authorized_media_access(now_ms.saturating_sub(60_000)); - session.origin_account_binding = Some(HlsOriginAccountBinding::new( - Arc::from("test-input"), - Arc::from("test-account"), - &proxy_session_id, - now_ms, - )); - session.origin_refresh.in_flight = true; - proxy_session_id - }; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Pending, - &strip, - None, - super::HlsCachedManifestOptions::initial(Duration::from_millis(200)), - ) - .await - .expect("valid committed transient manifest response"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - - assert_eq!(media_uri_count(&body), 3); - assert!(body.contains(&access_lease_id.0)); -} - -#[tokio::test] -async fn hls_cache_expired_transient_manifest_with_active_binding_is_not_served_after_manifest_validity() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let now_ms = super::current_time_millis(); - let _proxy_session_id = { - let mut session = session.write().await; - session.mode = - HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; - let proxy_session_id = session.proxy_session_id.clone(); - session.transient.replace_manifest_with_semantics( - transient_manifest_body(&proxy_session_id.0), - now_ms.saturating_sub(60_000), - Some(1_000), - ); - session.mark_authorized_media_access(now_ms.saturating_sub(60_000)); - session.origin_account_binding = Some(HlsOriginAccountBinding::new( - Arc::from("test-input"), - Arc::from("test-account"), - &proxy_session_id, - now_ms, - )); - proxy_session_id - }; - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Pending, - &strip, - None, - super::HlsCachedManifestOptions::initial(Duration::ZERO), - ) - .await; - - assert!(response.is_none()); -} - -#[tokio::test] -async fn hls_cache_no_media_yet_waits_for_first_normal_manifest_commit() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let proxy_session_id = { - let mut session = session.write().await; - session.origin_refresh.in_flight = true; - session.proxy_session_id.clone() - }; - let session_for_commit = Arc::clone(&session); - let proxy_session_for_body = proxy_session_id.0.clone(); - tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(20)).await; - let mut session = session_for_commit.write().await; - let rendered_at_ms = super::current_time_millis(); - let mut entry = test_segment_entry( - &session.proxy_session_id, - 100, - SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }, - ); - entry.duration_ms = 4_000; - session.segments.insert(100, entry); - session.advance_media_readiness_generation(); - session.last_rendered_manifest = Some(RenderedManifest { - body: format!( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n/hls/shared/live/{proxy_session_for_body}/{}/000100.ts\n", - crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER - ), - first_proxy_seq: 100, - last_proxy_seq: 100, - playlist_duration_ms: 4_000, - valid_until_ms: rendered_at_ms.saturating_add(4_000), - render_gap_segments: 0, - rendered_at_ms, - discontinuity_sequence: 0, - target_duration_ms: 4_000, - segment_proxy_seqs: vec![100], - }); - record_test_normal_manifest_commit(&mut session, rendered_at_ms); - session.origin_refresh.in_flight = false; - }); - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 0 }; - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Pending, - &strip, - None, - super::HlsCachedManifestOptions::initial(Duration::from_millis(200)), - ) - .await - .expect("initial normal manifest response"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - - assert!(body.contains(&access_lease_id.0)); - assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); -} - -#[tokio::test] -async fn hls_cache_no_media_yet_waits_for_first_transient_manifest_commit() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let proxy_session_id = { - let mut session = session.write().await; - session.mode = - HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; - session.origin_refresh.in_flight = true; - session.proxy_session_id.clone() - }; - let session_for_commit = Arc::clone(&session); - let proxy_session_for_body = proxy_session_id.0.clone(); - tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(20)).await; - let mut session = session_for_commit.write().await; - let rendered_at_ms = super::current_time_millis(); - session.transient.replace_manifest_with_semantics( - transient_manifest_body(&proxy_session_for_body), - rendered_at_ms, - Some(60_000), - ); - session.origin_refresh.in_flight = false; - }); - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Pending, - &strip, - None, - super::HlsCachedManifestOptions::initial(Duration::from_millis(200)), - ) - .await - .expect("initial transient manifest response"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - - assert_eq!(media_uri_count(&body), 3); - assert!(body.contains(&access_lease_id.0)); - assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); -} - -#[tokio::test] -async fn hls_cache_expired_transient_manifest_waits_for_revalidation_commit() { - let app_state = test_app_state(); - enable_hls_cache(&app_state); - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let proxy_session_id = { - let mut session = session.write().await; - session.mode = - HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; - session.mark_authorized_media_access(super::current_time_millis().saturating_sub(60_000)); - session.origin_refresh.in_flight = true; - session.proxy_session_id.clone() - }; - let session_for_commit = Arc::clone(&session); - let proxy_session_for_body = proxy_session_id.0.clone(); - tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(20)).await; - let mut session = session_for_commit.write().await; - let rendered_at_ms = super::current_time_millis(); - session.transient.replace_manifest_with_semantics( - transient_manifest_body(&proxy_session_for_body), - rendered_at_ms, - Some(60_000), - ); - session.origin_refresh.in_flight = false; - }); - let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); - let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; - - let response = try_test_hls_cached_manifest_response( - &app_state, - &session, - &access_lease_id, - HlsAccessLeaseState::Pending, - &strip, - None, - super::HlsCachedManifestOptions::initial(Duration::from_millis(200)), - ) - .await - .expect("revalidated transient manifest response"); - let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); - - assert_eq!(media_uri_count(&body), 3); - assert!(body.contains(&access_lease_id.0)); - assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); -} - -async fn grant_hls_proxy_lease(app_state: &Arc, proxy_session_id: &str) -> String { - create_active_hls_user_session(app_state).await; - let now_ms = super::current_time_millis(); - let lease_id = HlsAccessLeaseId(format!("test-access-lease-{proxy_session_id}")); - let family_key = HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key); - app_state - .hls - .proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - family_key, - ProxySessionId(proxy_session_id.to_string()), - "hls-user".to_string(), - "hls-session-token".to_string(), - 1, - "12345".to_string(), - 12345, - now_ms, - super::hls_access_lease_ttl_ms(app_state), - )) - .await; - assert!(app_state - .hls - .proxy - .activate_access_lease( - &lease_id, - &ProxySessionId(proxy_session_id.to_string()), - now_ms, - HlsAccessLeaseTiming { - active_window_ms: 5_000, - valid_window_ms: super::hls_access_lease_ttl_ms(app_state), - }, - ) - .await - .is_activated()); - lease_id.0 -} - -async fn hls_proxy_uri(app_state: &Arc, proxy_session_id: &str, suffix: &str) -> String { - let access_lease_id = grant_hls_proxy_lease(app_state, proxy_session_id).await; - let uri = format!("/hls/shared/live/{proxy_session_id}/{access_lease_id}/{suffix}"); - if suffix.starts_with("r/") { - publish_test_transient_resource_membership(app_state, proxy_session_id, &access_lease_id, &uri).await; - } - uri -} - -async fn publish_test_transient_resource_membership( - app_state: &Arc, - proxy_session_id: &str, - access_lease_id: &str, - resource_uri: &str, -) { - let proxy_session_id = ProxySessionId(proxy_session_id.to_string()); - let access_lease_id = HlsAccessLeaseId(access_lease_id.to_string()); - let now_ms = super::current_time_millis(); - let publication = app_state - .hls - .proxy - .prepare_access_lease_manifest_publication(&access_lease_id, &proxy_session_id, now_ms) - .await - .expect("test resource lease accepts manifest publication"); - let snapshot = HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::TransientPassthrough, - source_commit_identity: HlsManifestCommitIdentity::new(now_ms), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 0, - delivered_at_ms: now_ms, - first_proxy_seq: 0, - last_proxy_seq: 0, - visible_segments: Arc::from([]), - discontinuity_sequence: 0, - target_duration_ms: 4_000, - playlist_duration_ms: 0, - last_visible_media_end_ms: 0, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - }; - let published_resource_ids = HlsPublishedTransientResourceIds::from_manifest_body(resource_uri); - assert!(app_state - .hls - .proxy - .commit_access_lease_manifest_publication_with_resources( - &access_lease_id, - &proxy_session_id, - publication, - snapshot, - published_resource_ids, - now_ms, - ) - .await - .is_committed()); -} - -fn regression_origin_manifest(first_sequence: u64, segment_count: usize) -> Vec { - let mut manifest = - format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:{first_sequence}\n"); - for offset in 0..segment_count { - let sequence = first_sequence.saturating_add(u64::try_from(offset).unwrap_or(u64::MAX)); - let _ = writeln!(&mut manifest, "#EXTINF:12.0,\n{sequence}.ts"); - } - manifest.into_bytes() -} - -fn regression_origin_refresh_request( - app_state: &Arc, - session: HlsSessionHandle, - manifest_url: &str, - access_lease_id: Option, -) -> OriginRefreshRequest { - OriginRefreshRequest { - app_config: Arc::clone(&app_state.app_config), - session, - origin_entry: LiveHlsOriginEntry::parse(manifest_url).expect("regression origin entry"), - headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - client: app_state.http_clients.default.load().as_ref().clone(), - no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(), - use_manual_redirects: false, - segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()), - hls_proxy: Arc::clone(&app_state.hls.proxy), - segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()), - segment_worker_pool: Arc::clone(app_state.hls.proxy.segment_worker_pool()), - map_worker_pool: Arc::clone(app_state.hls.proxy.map_worker_pool()), - origin_manifest_timeout_ms: app_state.hls.proxy.origin_manifest_timeout_ms(), - manifest_recovery_burst: app_state.hls.proxy.manifest_recovery_burst(), - strip: app_state.hls.proxy.strip(), - retry_policy: RetryPolicy { delays_ms: [0; 5], jitter_max_ms: 0 }, - reverse_proxy_rewrite_secret: app_state.get_encrypt_secret().to_vec(), - transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(), - manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, - fresh_manifest_requirement_generation: None, - acceptance_directive: HlsManifestAcceptanceDirective::none(), - access_lease_id, - disabled_headers: None, - now_ms: super::current_time_millis(), - origin_io: None, - post_refresh_runtime: None, - } -} - -async fn wait_for_ready_timeline(session: &HlsSessionHandle, expected_ready: usize) { - let wait = tokio::time::timeout(Duration::from_secs(10), async { - loop { - let ready = session - .read() - .await - .segments - .values() - .filter(|segment| matches!(segment.status, SegmentCacheStatus::Ready { .. })) - .count(); - if ready >= expected_ready { - break; - } - tokio::task::yield_now().await; - } - }) - .await; - if wait.is_err() { - let session = session.read().await; - let statuses = - session.segments.values().map(|segment| (segment.proxy_seq, segment.status.clone())).collect::>(); - panic!("READY timeline deadline: expected={expected_ready} statuses={statuses:?}"); - } -} - -async fn extend_ready_segment_as_sparse_file( - app_state: &Arc, - session: &HlsSessionHandle, - proxy_seq: u64, - logical_size: u64, -) { - let cache_key = session.read().await.segments.get(&proxy_seq).expect("mapped sparse segment").cache_key.clone(); - let metadata = app_state - .hls - .proxy - .segment_cache() - .metadata(&cache_key) - .await - .expect("sparse cache metadata read") - .expect("READY sparse cache object"); - let file = tokio::fs::OpenOptions::new().write(true).open(&metadata.path).await.expect("sparse cache object opens"); - file.set_len(logical_size).await.expect("sparse cache object extends"); - let mut session = session.write().await; - let segment = session.segments.get_mut(&proxy_seq).expect("sparse segment remains mapped"); - segment.status = - SegmentCacheStatus::Ready { content_length: logical_size, ready_at_ms: super::current_time_millis() }; - session.advance_media_readiness_generation(); - session.render_and_store_manifest(super::current_time_millis()).expect("sparse timeline renders"); -} - -fn terminal_test_asset() -> Arc { - let bytes = bytes::Bytes::from_static(include_bytes!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../test/fixtures/hls/channel_unavailable.ts" - ))); - let buffer = TransportStreamBuffer::new(bytes.to_vec()); - snapshot_terminal_media_asset(&buffer).expect("terminal test asset is valid") -} - -async fn publish_test_manifest_and_exhaust_configured_acceptance( - app_state: &Arc, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, - snapshot: HlsLeaseManifestSnapshot, - now_ms: u64, -) { - let target_duration_ms = snapshot.target_duration_ms; - let publication_guard = app_state - .hls - .proxy - .prepare_access_lease_manifest_publication(lease_id, proxy_session_id, now_ms) - .await - .expect("live lease accepts publication preparation"); - assert!(app_state - .hls - .proxy - .commit_access_lease_manifest_publication(lease_id, proxy_session_id, publication_guard, snapshot, now_ms,) - .await - .is_committed()); - - let terminal_response = app_state.app_config.custom_stream_response.load_full(); - let terminal_asset = terminal_response - .as_ref() - .and_then(|responses| responses.channel_unavailable.as_ref()) - .and_then(|buffer| snapshot_terminal_media_asset(buffer).ok()) - .expect("configured terminal test asset"); - let terminal_key = - prepared_terminal_bundle_key(&terminal_asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); - let state = app_state.hls.proxy.start_prepared_terminal_bundle( - terminal_asset, - target_duration_ms, - HLS_TERMINAL_TAIL_SEGMENT_COUNT, - ); - let state = match state { - HlsPreparedTerminalBundleState::Preparing { .. } => app_state - .hls - .proxy - .wait_for_prepared_terminal_bundle(terminal_key) - .await - .expect("terminal bundle completion"), - state => state, - }; - assert!(matches!( - state, - HlsPreparedTerminalBundleState::Ready { ref bundle } if bundle.key == terminal_key - )); - - let session = - app_state.hls.proxy.sessions().get_by_proxy_session_id(proxy_session_id).await.expect("warm shared session"); - let mut session = session.write().await; - session.origin_control.record_media_progress(now_ms, target_duration_ms); - let burst_plan = app_state.hls.proxy.manifest_recovery_burst().level.plan(); - let operation_timeout = HlsOperationTimeoutMs::from_millis(app_state.hls.proxy.origin_manifest_timeout_ms()); - let expected_eta = HlsRecoveryEtaMs::from_millis(app_state.hls.proxy.origin_manifest_timeout_ms()); - let timing = HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput { - started_at_ms: now_ms, - burst_plan, - target_duration_ms, - transition_margin: HlsTransitionMarginMs::from_millis(target_duration_ms), - workload: HlsRecoveryWorkload::clear_fetch(), - observed_latency: HlsObservedRecoveryLatency::default(), - required_terminal_media_key: Some(terminal_key), - terminal_media_preparation: HlsTerminalMediaPreparationState::Ready { key: terminal_key }, - policy: HlsRecoveryTimingPolicy::new(operation_timeout, operation_timeout, expected_eta, expected_eta), - }); - session.origin_control.begin_acceptance_episode( - now_ms, - burst_plan, - HlsManifestAcceptanceTrigger::RecoveryRequired, - &timing, - ); - session.origin_control.path_condition = HlsOriginPathCondition::HardFetchFailure; - let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); - episode.record_full_burst(); - episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::AllFailed); - episode.hold_after_uncommitted_burst(None, None); -} - -async fn terminalize_existing_test_lease( - app_state: &Arc, - proxy_session_id: &str, - lease_id: &str, - base_proxy_seq: u64, -) -> TransportStreamBuffer { - let proxy_session_id = ProxySessionId(proxy_session_id.to_string()); - let lease_id = HlsAccessLeaseId(lease_id.to_string()); - let buffer = TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")).to_vec(), - ); - let asset = snapshot_terminal_media_asset(&buffer).expect("terminal test asset is valid"); - let base_manifest = HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(7), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 7, - delivered_at_ms: super::current_time_millis(), - first_proxy_seq: base_proxy_seq, - last_proxy_seq: base_proxy_seq, - visible_segments: Arc::from([HlsLeaseManifestSegment { - proxy_seq: base_proxy_seq, - duration_ms: 4_000, - uri: format!("/iptv/hls/shared/live/{}/{}/{base_proxy_seq:06}.ts", proxy_session_id.0, lease_id.0).into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }]), - discontinuity_sequence: 3, - target_duration_ms: asset.duration_ms().saturating_add(1_000), - playlist_duration_ms: 4_000, - last_visible_media_end_ms: 4_000, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - }; - let base_timing = Some(HlsTerminalTailBuildInput::base_timing_for_test(&asset, &base_manifest)); - let base_splice_evidence = Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); - let terminal_splice_evidence = base_splice_evidence.clone(); - let plan = build_terminal_tail_plan(HlsTerminalTailBuildInput { - generation: HlsTerminalTailGeneration(17), - created_at_ms: super::current_time_millis(), - base_availability: Arc::from([HlsTerminalBaseSegmentAvailability { - proxy_seq: base_proxy_seq, - media_state: HlsTerminalBaseMediaState::Ready, - required_map_ready: true, - required_key_ready: true, - protection: HlsTerminalBaseProtection::Protectable, - }]), - base_track_signature: Some(asset.track_signature().clone()), - base_splice_evidence, - terminal_splice_evidence, - base_timing, - base_key_bindings: Arc::from([]), - expected_asset: HlsRuntimeCustomTailAssetIdentity::channel_unavailable(HlsTerminalAssetIdentity::from_asset( - &asset, - )), - base_manifest: base_manifest.clone(), - anchored_bundle: HlsTerminalTailBuildInput::anchored_bundle_for_test(&asset, base_manifest.target_duration_ms), - asset, - }) - .expect("terminal test plan is compatible"); - let protection = HlsTerminalTailProtection { - generation: plan.generation, - base_proxy_seqs: Arc::clone(&plan.protected_base_proxy_seqs), - key_bindings: plan.key_bindings(), - }; - { - let mut leases = app_state.hls.proxy.access_leases().write().await; - let mut lease = leases.remove_access_lease(&lease_id).expect("test lease exists before terminal cutover"); - lease.last_manifest_snapshot = Some(base_manifest); - lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(Arc::new(plan)); - leases.prepare_access_lease(lease); - } - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&proxy_session_id) - .await - .expect("test session exists before terminal cutover"); - session.write().await.install_terminal_tail_protection(lease_id, protection); - buffer -} - -async fn terminal_test_plan_shape(app_state: &Arc, proxy_session_id: &str, lease_id: &str) -> (u64, u16) { - let snapshot = app_state - .hls - .proxy - .access_lease_response_snapshot( - &HlsAccessLeaseId(lease_id.to_string()), - &ProxySessionId(proxy_session_id.to_string()), - super::current_time_millis(), - ) - .await - .expect("terminal test lease snapshot exists"); - let HlsLeasePlaybackMode::TerminalTail(plan) = snapshot.playback_mode else { - panic!("terminal test lease keeps terminal playback mode"); - }; - (plan.generation.0, plan.segment_count) -} - -async fn wait_for_provider_connection_count(app_state: &Arc, expected: usize) { - for _ in 0..50 { - let actual = app_state.active_provider.get_provider_connections_count(); - if actual == expected { - return; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert_eq!(app_state.active_provider.get_provider_connections_count(), expected); -} - -fn normal_manifest(body: &str) -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest { - match parse_origin_media_manifest(body, "http://origin.example.com/live/final/index.m3u8") { - OriginManifestParseOutcome::Normal(manifest) => manifest, - OriginManifestParseOutcome::TransientPassthrough { reason } => { - panic!("expected normal manifest: {reason:?}") - } - } -} - -async fn map_segment(app_state: &Arc, proxy_seq: u64, extension: &str) -> String { - map_segment_with_origin_url(app_state, proxy_seq, extension, &format!("{proxy_seq}.{extension}")).await -} - -async fn map_segment_with_origin_url( - app_state: &Arc, - proxy_seq: u64, - _extension: &str, - origin_url: &str, -) -> String { - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let manifest = - normal_manifest(&format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:{proxy_seq}\n#EXTINF:4.0,\n{origin_url}\n")); - let mut session = session.write().await; - session.proxy_next_seq = Some(proxy_seq); - session.apply_origin_manifest(&manifest).expect("manifest should map"); - session.proxy_session_id.0.clone() -} - -async fn map_ready_segment(app_state: &Arc, proxy_seq: u64, extension: &str, body: &[u8]) -> String { - let proxy_session_id = map_ready_segment_without_lease(app_state, proxy_seq, extension, body).await; - grant_hls_proxy_lease(app_state, &proxy_session_id).await; - proxy_session_id -} - -async fn map_ready_segment_without_lease( - app_state: &Arc, - proxy_seq: u64, - extension: &str, - body: &[u8], -) -> String { - let proxy_session_id = map_segment(app_state, proxy_seq, extension).await; - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) - .await - .expect("session should exist"); - let cache_key = { - let session = session.read().await; - session.segments.get(&proxy_seq).expect("segment should be mapped").cache_key.clone() - }; - let metadata = app_state - .hls - .proxy - .segment_cache() - .write_bytes_and_commit(&cache_key, body) - .await - .expect("cache commit should succeed"); - { - let mut session = session.write().await; - session.segments.get_mut(&proxy_seq).expect("segment should be mapped").status = - SegmentCacheStatus::Ready { content_length: metadata.size, ready_at_ms: 200 }; - } - proxy_session_id -} - -async fn map_hls_map(app_state: &Arc, body: &[u8], grant_lease: bool) -> String { - let session = app_state - .hls - .proxy - .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) - .await; - let manifest = normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n000123.m4s\n"); - let proxy_session_id = { - let mut session = session.write().await; - session.apply_origin_manifest(&manifest).expect("manifest should map"); - session.proxy_session_id.0.clone() - }; - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) - .await - .expect("session should exist"); - let cache_key = { - let session = session.read().await; - session.maps.get(&ProxyMapId(0)).expect("map should be mapped").cache_key.clone() - }; - let metadata = app_state - .hls - .proxy - .segment_cache() - .write_bytes_and_commit(&cache_key, body) - .await - .expect("map cache commit should succeed"); - { - let mut session = session.write().await; - session.maps.get_mut(&ProxyMapId(0)).expect("map should be mapped").status = - MapCacheStatus::Ready { content_length: metadata.size, ready_at_ms: 200 }; - } - if grant_lease { - grant_hls_proxy_lease(app_state, &proxy_session_id).await; - } - proxy_session_id -} - -async fn map_transient_resource( - app_state: &Arc, - origin_url: &str, - extension: &str, - grant_lease: bool, -) -> (String, String) { - map_transient_resource_with_kind(app_state, origin_url, extension, grant_lease, TransientResourceKind::Segment) - .await -} - -async fn map_transient_resource_with_kind( - app_state: &Arc, - origin_url: &str, - extension: &str, - grant_lease: bool, - kind: TransientResourceKind, -) -> (String, String) { - static TRANSIENT_STREAM_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); - let secret = b"rewrite-secret"; - let now_ms = super::current_time_millis(); - let stream_ref = - format!("transient-{}", TRANSIENT_STREAM_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed)); - let session = app_state.hls.proxy.get_or_create_session(HlsSessionKey::new(1, &stream_ref), secret, now_ms).await; - let resource_id = build_transient_resource_id(origin_url, secret); - let proxy_session_id = { - let mut session = session.write().await; - session.mode = - HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; - session.transient.upsert_resources([TransientResourceRef::new( - kind, - origin_url, - secret, - now_ms, - 300_000, - Some(extension.to_string()), - )]); - session.proxy_session_id.0.clone() - }; - if grant_lease { - grant_hls_proxy_lease(app_state, &proxy_session_id).await; - } - (proxy_session_id, resource_id.0) -} - -async fn get_response(app_state: Arc, uri: &str, range: Option<&str>) -> Response { - request_response(app_state, Method::GET, uri, range).await -} - -async fn request_response(app_state: Arc, method: Method, uri: &str, range: Option<&str>) -> Response { - let router = hls_api_register().with_state(app_state); - let mut request = Request::builder().method(method).uri(uri); - if let Some(range) = range { - request = request.header(header::RANGE, range); - } - let mut request = request.body(Body::empty()).expect("request should build"); - request.extensions_mut().insert(ConnectInfo(test_addr())); - router.oneshot(request).await.expect("response") -} - -async fn get_status(app_state: Arc, uri: &str) -> StatusCode { - get_response(app_state, uri, None).await.status() -} - -async fn hls_session_last_media_at_ms(app_state: &Arc, proxy_session_id: &str) -> Option { - app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.to_string())) - .await - .expect("session should exist") - .read() - .await - .activity - .last_authorized_media_at_ms -} - -async fn assert_no_hls_cache_stream_registered(app_state: &Arc) { - assert!(app_state.active_users.active_streams().await.is_empty()); -} - -async fn response_body(response: Response) -> bytes::Bytes { - response.into_body().collect().await.expect("body should collect").to_bytes() -} - -async fn single_variant_master_playlist(response: Response) -> (u32, String) { - let body = response_body(response).await; - let body = std::str::from_utf8(&body).expect("master playlist should be UTF-8"); - let mut lines = body.lines(); - assert_eq!(lines.next(), Some("#EXTM3U")); - let bandwidth = lines - .next() - .and_then(|line| line.strip_prefix("#EXT-X-STREAM-INF:BANDWIDTH=")) - .and_then(|value| value.parse::().ok()) - .expect("positive master playlist bandwidth"); - let uri = lines.next().expect("single variant URI").to_string(); - assert!(lines.next().is_none(), "master playlist must contain exactly one variant"); - (bandwidth, uri) -} - -async fn single_variant_uri(response: Response) -> String { single_variant_master_playlist(response).await.1 } - -fn access_lease_id_from_variant_uri(uri: &str) -> &str { - uri.trim_end_matches("/manifest.m3u8").rsplit('/').next().expect("access lease id in variant URI") -} - -fn proxy_session_id_from_variant_uri(uri: &str) -> &str { - let mut parts = uri.trim_end_matches("/manifest.m3u8").rsplit('/'); - let _access_lease_id = parts.next().expect("access lease id in variant URI"); - parts.next().expect("proxy session id in variant URI") -} - -fn manifest_media_sequence(body: &str) -> u64 { - body.lines() - .find_map(|line| line.strip_prefix("#EXT-X-MEDIA-SEQUENCE:")) - .and_then(|value| value.parse().ok()) - .expect("media sequence") -} - -async fn access_lease_session_token( - app_state: &Arc, - proxy_session_id: &ProxySessionId, - access_lease_id: &HlsAccessLeaseId, -) -> String { - app_state - .hls - .proxy - .access_lease(access_lease_id, proxy_session_id, super::current_time_millis()) - .await - .expect("access lease should exist") - .user_session_token -} - -#[test] -fn transient_full_object_cacheable_request_accepts_open_zero_range() { - use crate::api::model::is_hls_transient_full_object_cacheable_request; - - assert!(is_hls_transient_full_object_cacheable_request(None)); - assert!(is_hls_transient_full_object_cacheable_request(Some(&HeaderValue::from_static("bytes=0-")))); - assert!(!is_hls_transient_full_object_cacheable_request(Some(&HeaderValue::from_static("bytes=4-")))); - assert!(!is_hls_transient_full_object_cacheable_request(Some(&HeaderValue::from_static("bytes=-4")))); - assert!(!is_hls_transient_full_object_cacheable_request(Some(&HeaderValue::from_static("bytes=0-1,4-5")))); -} - -async fn assert_hls_cache_stream_registered(app_state: &Arc, proxy_session_id: &str) { - let streams = app_state.active_users.active_streams().await; - assert_eq!(streams.len(), 1); - let stream = &streams[0]; - assert_eq!(stream.username, "hls-user"); - assert_eq!(stream.session_token.as_deref(), Some("hls-session-token")); - assert_eq!(stream.provider.as_ref(), "origin-provider"); - assert_eq!(stream.channel.item_type, PlaylistItemType::LiveHls); - assert!(stream.channel.shared); - assert_eq!( - stream.channel.shared_stream_id, - Some(super::hls_cache_shared_stream_id(&ProxySessionId(proxy_session_id.to_string()))) - ); - assert_eq!(stream.channel.shared_joined_existing, Some(false)); - assert_eq!(stream.channel.url.as_ref(), format!("/hls/shared/live/{proxy_session_id}/manifest.m3u8")); - assert!(!stream.channel.url.contains("test-access-lease")); - assert!(!stream.channel.url.contains("hls-session-token")); - assert!(!stream.channel.url.contains("origin.example.com")); - assert!(!stream.channel.url.contains("/hls/hls-user/")); -} - -fn stats_provider_test_user_session(provider: &str) -> UserSession { - UserSession { - token: "stats-session-token".to_string(), - transition_version: 0, - virtual_id: 12345, - provider: Arc::from(provider), - stream_url: Arc::from("http://origin.example.com/live/12345.m3u8"), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), - user_agent_stream_index: None, - addr: test_addr(), - socket_bound: false, - active_addrs: Vec::new(), - ts: 100, - started_at: 100, - permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - lifecycle: PlaybackLifecycle::Active, - ..Default::default() - } -} - -#[test] -fn hls_cache_stats_provider_prefers_active_origin_account_binding() { - let origin_source = HlsOriginSource::new(1, Arc::from("cdn-dev"), "12345", HlsOriginSourceKind::XtreamLive); - let proxy_session_id = ProxySessionId("stats-session".to_string()); - let binding = HlsOriginAccountBinding::new( - Arc::clone(&origin_source.input_name), - Arc::from("cdn-dev-alias"), - &proxy_session_id, - 100, - ); - let user_session = stats_provider_test_user_session("cdn-dev"); - - let provider = super::hls_cache_stats_provider(&origin_source, Some(&binding), &user_session); - - assert_eq!(provider.as_ref(), "cdn-dev-alias"); -} - -#[test] -fn hls_cache_stats_provider_falls_back_when_origin_account_binding_is_not_active() { - let origin_source = HlsOriginSource::new(1, Arc::from("cdn-dev"), "12345", HlsOriginSourceKind::XtreamLive); - let proxy_session_id = ProxySessionId("stats-session".to_string()); - let mut binding = HlsOriginAccountBinding::new( - Arc::clone(&origin_source.input_name), - Arc::from("cdn-dev-alias"), - &proxy_session_id, - 100, - ); - binding.detach(HlsOriginAccountDetachedReason::Cleanup, 200); - let user_session = stats_provider_test_user_session("session-provider"); - - let provider = super::hls_cache_stats_provider(&origin_source, Some(&binding), &user_session); - - assert_eq!(provider.as_ref(), "session-provider"); -} - -#[test] -fn hls_cache_stats_provider_falls_back_to_input_name_without_session_provider() { - let origin_source = HlsOriginSource::new(1, Arc::from("cdn-dev"), "12345", HlsOriginSourceKind::XtreamLive); - let user_session = stats_provider_test_user_session(""); - - let provider = super::hls_cache_stats_provider(&origin_source, None, &user_session); - - assert_eq!(provider.as_ref(), "cdn-dev"); -} - -async fn register_hls_cache_stream_for_stats_test( - app_state: &Arc, - session: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - session_token: &str, - fingerprint: &Fingerprint, - lease_id: &str, -) { - create_active_hls_user_session_with( - app_state, - session_token, - "origin-provider", - "http://origin.example.com/live/12345.m3u8", - fingerprint.addr, - ) - .await; - let context = test_hls_access_context_with( - proxy_session_id.clone(), - HlsAccessLeaseId(lease_id.to_string()), - session_token, - fingerprint.key.clone(), - ); - super::ensure_hls_cache_stream_registered(app_state, fingerprint, &HeaderMap::new(), &context, session) - .await - .expect("HLS stream registers"); -} - -fn find_stream_by_session_token( - streams: &[shared::model::StreamInfo], - session_token: &str, -) -> shared::model::StreamInfo { - streams - .iter() - .find(|stream| stream.session_token.as_deref() == Some(session_token)) - .unwrap_or_else(|| panic!("{session_token} stream should exist")) - .clone() -} - -#[tokio::test] -async fn hls_cache_stream_stats_mark_additional_viewers_as_joined_existing() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_ready_segment_without_lease(&app_state, 123, "ts", b"0123456789").await; - let proxy_session_id = ProxySessionId(proxy_session_id); - let shared_stream_id = super::hls_cache_shared_stream_id(&proxy_session_id); - let session = - app_state.hls.proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.expect("session should exist"); - let first_fingerprint = test_fingerprint(); - let second_fingerprint = test_fingerprint_with_addr(test_addr_with_port(55124)); - register_hls_cache_stream_for_stats_test( - &app_state, - &session, - &proxy_session_id, - "hls-session-token", - &first_fingerprint, - "first-access-lease", - ) - .await; - - let streams = app_state.active_users.active_streams().await; - let first_stream = find_stream_by_session_token(&streams, "hls-session-token"); - let first_meter_uid = first_stream.meter_uid; - assert!(first_stream.channel.shared); - assert_eq!(first_stream.channel.shared_stream_id, Some(shared_stream_id)); - assert_eq!(first_stream.channel.shared_joined_existing, Some(false)); - - register_hls_cache_stream_for_stats_test( - &app_state, - &session, - &proxy_session_id, - "hls-second-session-token", - &second_fingerprint, - "second-access-lease", - ) - .await; - - let streams = app_state.active_users.active_streams().await; - let first_stream = find_stream_by_session_token(&streams, "hls-session-token"); - let second_stream = find_stream_by_session_token(&streams, "hls-second-session-token"); - assert_eq!(first_stream.channel.shared_stream_id, Some(shared_stream_id)); - assert_eq!(first_stream.channel.shared_joined_existing, Some(false)); - assert_eq!(second_stream.channel.shared_stream_id, Some(shared_stream_id)); - assert_eq!(second_stream.channel.shared_joined_existing, Some(true)); - - register_hls_cache_stream_for_stats_test( - &app_state, - &session, - &proxy_session_id, - "hls-session-token", - &first_fingerprint, - "first-access-lease", - ) - .await; - - let streams = app_state.active_users.active_streams().await; - let first_stream = find_stream_by_session_token(&streams, "hls-session-token"); - assert_eq!(first_stream.channel.shared_stream_id, Some(shared_stream_id)); - assert_eq!(first_stream.channel.shared_joined_existing, Some(false)); - assert_eq!(first_stream.meter_uid, first_meter_uid); -} - -struct TestSegmentOrigin { - base_url: String, - key_requests: Arc, - manifest_requests: Arc, - segment_requests: Arc, - key_bytes: Option>>>, - task: tokio::task::JoinHandle<()>, -} - -impl Drop for TestSegmentOrigin { - fn drop(&mut self) { self.task.abort(); } -} - -impl TestSegmentOrigin { - fn key_request_count(&self) -> usize { self.key_requests.load(Ordering::SeqCst) } - - fn manifest_request_count(&self) -> usize { self.manifest_requests.load(Ordering::SeqCst) } - - fn segment_request_count(&self) -> usize { self.segment_requests.load(Ordering::SeqCst) } - - async fn set_key_bytes(&self, bytes: Arc<[u8]>) { - if let Some(key_bytes) = &self.key_bytes { - *key_bytes.write().await = bytes; - } - } -} - -async fn spawn_test_segment_origin(body: &'static [u8]) -> TestSegmentOrigin { - spawn_test_status_origin(StatusCode::OK, body).await -} - -async fn spawn_test_encrypted_hls_origin( - manifest: &'static [u8], - key_bytes: Arc<[u8]>, - plaintext_segment: Arc<[u8]>, -) -> TestSegmentOrigin { - let manifest = Arc::<[u8]>::from(manifest); - let key_bytes = Arc::new(RwLock::new(key_bytes)); - let key_bytes_for_task = Arc::clone(&key_bytes); - let key_requests = Arc::new(AtomicUsize::new(0)); - let key_requests_for_task = Arc::clone(&key_requests); - let manifest_requests = Arc::new(AtomicUsize::new(0)); - let manifest_requests_for_task = Arc::clone(&manifest_requests); - let segment_requests = Arc::new(AtomicUsize::new(0)); - let segment_requests_for_task = Arc::clone(&segment_requests); - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let addr = listener.local_addr().expect("local addr"); - let task = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let manifest = Arc::clone(&manifest); - let key_bytes = Arc::clone(&key_bytes_for_task); - let key_requests = Arc::clone(&key_requests_for_task); - let manifest_requests = Arc::clone(&manifest_requests_for_task); - let segment_requests = Arc::clone(&segment_requests_for_task); - let plaintext_segment = Arc::clone(&plaintext_segment); - tokio::spawn(async move { - let mut request = vec![0_u8; 2048]; - let Ok(read) = socket.read(&mut request).await else { - return; - }; - if read == 0 { - return; - } - let path = String::from_utf8_lossy(&request[..read]) - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .map_or_else(|| "/".to_string(), str::to_owned); - let current_key_bytes = Arc::clone(&*key_bytes.read().await); - let body = if path_has_extension(&path, "m3u8") { - manifest_requests.fetch_add(1, Ordering::SeqCst); - manifest - } else if path.ends_with("key.bin") { - key_requests.fetch_add(1, Ordering::SeqCst); - current_key_bytes - } else if let Some(origin_sequence) = path - .rsplit('/') - .next() - .and_then(|file| file.strip_suffix(".ts")) - .and_then(|value| value.parse::().ok()) - { - segment_requests.fetch_add(1, Ordering::SeqCst); - Arc::from(encrypt_test_aes128_cbc_pkcs7( - &plaintext_segment, - ¤t_key_bytes, - test_hls_sequence_iv(origin_sequence), - )) - } else { - Arc::<[u8]>::from([]) - }; - let response = - format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); - if socket.write_all(response.as_bytes()).await.is_ok() { - let _ = socket.write_all(&body).await; - } - }); - } - }); - TestSegmentOrigin { - base_url: format!("http://{addr}"), - key_requests, - manifest_requests, - segment_requests, - key_bytes: Some(key_bytes), - task, - } -} - -fn encrypt_test_aes128_cbc_pkcs7(plaintext: &[u8], key: &[u8], iv: [u8; 16]) -> Vec { - let padding_len = 16 - (plaintext.len() % 16); - let mut ciphertext = plaintext.to_vec(); - ciphertext.resize( - plaintext.len().saturating_add(padding_len), - u8::try_from(padding_len).expect("PKCS#7 AES-128 padding fits in u8"), - ); - let cipher = Aes128::new_from_slice(key).expect("test key has AES-128 length"); - let mut previous = iv; - for block in ciphertext.as_chunks_mut::<16>().0 { - for (byte, previous) in block.iter_mut().zip(previous) { - *byte ^= previous; - } - let mut encrypted = Block::::default(); - encrypted.copy_from_slice(block); - cipher.encrypt_block(&mut encrypted); - block.copy_from_slice(&encrypted); - previous.copy_from_slice(block); - } - ciphertext -} - -fn test_hls_sequence_iv(sequence: u64) -> [u8; 16] { - let mut iv = [0_u8; 16]; - iv[8..].copy_from_slice(&sequence.to_be_bytes()); - iv -} - -struct TestBinaryOriginResponse { - status: StatusCode, - location: Option, - body: Arc<[u8]>, -} - -impl TestBinaryOriginResponse { - fn new(status: StatusCode, body: Arc<[u8]>) -> Self { Self { status, location: None, body } } - - fn redirect(location: String) -> Self { - Self { status: StatusCode::FOUND, location: Some(location), body: Arc::from(&b""[..]) } - } -} - -type TestBinaryOriginHandler = Arc TestBinaryOriginResponse + Send + Sync>; - -async fn spawn_test_binary_origin(handler: TestBinaryOriginHandler) -> TestSegmentOrigin { - let key_requests = Arc::new(AtomicUsize::new(0)); - let manifest_requests = Arc::new(AtomicUsize::new(0)); - let segment_requests = Arc::new(AtomicUsize::new(0)); - let manifest_requests_for_task = Arc::clone(&manifest_requests); - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let addr = listener.local_addr().expect("local addr"); - let task = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let handler = Arc::clone(&handler); - let manifest_requests = Arc::clone(&manifest_requests_for_task); - tokio::spawn(async move { - let mut buf = vec![0_u8; 2048]; - let Ok(read) = socket.read(&mut buf).await else { - return; - }; - if read == 0 { - return; - } - let request = String::from_utf8_lossy(&buf[..read]); - let path = request.lines().next().and_then(|line| line.split_whitespace().nth(1)).unwrap_or("/"); - if path_has_extension(path, "m3u8") { - manifest_requests.fetch_add(1, Ordering::SeqCst); - } - let TestBinaryOriginResponse { status, location, body } = handler(path); - let reason = status.canonical_reason().unwrap_or("Status"); - let location_header = location.map_or_else(String::new, |location| format!("Location: {location}\r\n")); - let response = format!( - "HTTP/1.1 {} {reason}\r\n{location_header}Content-Length: {}\r\nConnection: close\r\n\r\n", - status.as_u16(), - body.len() - ); - let _ = socket.write_all(response.as_bytes()).await; - let _ = socket.write_all(&body).await; - }); - } - }); - TestSegmentOrigin { - base_url: format!("http://{addr}"), - key_requests, - manifest_requests, - segment_requests, - key_bytes: None, - task, - } -} - -async fn spawn_test_status_origin(status: StatusCode, body: &'static [u8]) -> TestSegmentOrigin { - let body = Arc::<[u8]>::from(body); - spawn_test_binary_origin(Arc::new(move |_path| TestBinaryOriginResponse::new(status, Arc::clone(&body)))).await -} - -struct TestEncodedManifestOrigin { - base_url: String, - requests: Arc>>, - task: tokio::task::JoinHandle<()>, -} - -impl Drop for TestEncodedManifestOrigin { - fn drop(&mut self) { self.task.abort(); } -} - -async fn spawn_test_encoded_manifest_origin( - content_encoding: Option<&'static str>, - body: Vec, - body_delay: Duration, -) -> TestEncodedManifestOrigin { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let addr = listener.local_addr().expect("local addr"); - let requests = Arc::new(tokio::sync::Mutex::new(Vec::new())); - let task_requests = Arc::clone(&requests); - let body = Arc::new(body); - let task = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let requests = Arc::clone(&task_requests); - let body = Arc::clone(&body); - tokio::spawn(async move { - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 2048]; - let Ok(read) = socket.read(&mut chunk).await else { - return; - }; - if read == 0 { - return; - } - request.extend_from_slice(&chunk[..read]); - } - requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); - let mut response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n", body.len()); - if let Some(content_encoding) = content_encoding { - let _ = writeln!(&mut response, "Content-Encoding: {content_encoding}\r"); - } - response.push_str("Connection: close\r\n\r\n"); - let _ = socket.write_all(response.as_bytes()).await; - if body_delay.is_zero() { - let _ = socket.write_all(body.as_slice()).await; - } else { - let split_at = body.len().min(4); - let _ = socket.write_all(&body[..split_at]).await; - tokio::time::sleep(body_delay).await; - let _ = socket.write_all(&body[split_at..]).await; - } - }); - } - }); - TestEncodedManifestOrigin { base_url: format!("http://{addr}"), requests, task } -} - -fn legacy_manifest_test_input(origin: &TestEncodedManifestOrigin) -> crate::model::InputSource { - crate::model::InputSource { - name: Arc::from("legacy-content-coding-test"), - url: format!("{}/manifest.m3u8", origin.base_url), - provider: None, - username: None, - password: None, - method: shared::model::InputFetchMethod::GET, - headers: HashMap::from([("Accept-Encoding".to_string(), "gzip".to_string())]), - } -} - -fn legacy_manifest_test_client_headers() -> HeaderMap { - let mut headers = HeaderMap::new(); - headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); - headers -} - -async fn encode_test_manifest(content_encoding: &str, body: &[u8]) -> Vec { - match content_encoding { - "gzip" => { - let mut encoder = async_compression::tokio::write::GzipEncoder::new(Vec::new()); - encoder.write_all(body).await.expect("gzip test body encodes"); - encoder.shutdown().await.expect("gzip test encoder finishes"); - encoder.into_inner() - } - "deflate" => { - let mut encoder = async_compression::tokio::write::DeflateEncoder::new(Vec::new()); - encoder.write_all(body).await.expect("deflate test body encodes"); - encoder.shutdown().await.expect("deflate test encoder finishes"); - encoder.into_inner() - } - "br" => { - let mut encoder = async_compression::tokio::write::BrotliEncoder::new(Vec::new()); - encoder.write_all(body).await.expect("brotli test body encodes"); - encoder.shutdown().await.expect("brotli test encoder finishes"); - encoder.into_inner() - } - "zstd" => { - let mut encoder = async_compression::tokio::write::ZstdEncoder::new(Vec::new()); - encoder.write_all(body).await.expect("zstd test body encodes"); - encoder.shutdown().await.expect("zstd test encoder finishes"); - encoder.into_inner() - } - _ => panic!("unsupported test Content-Encoding: {content_encoding}"), - } -} - -async fn wait_for_hls_test_session(app_state: &Arc, session_key: &HlsSessionKey) -> HlsSessionHandle { - for _ in 0..50 { - if let Some(session) = app_state.hls.proxy.sessions().get_by_key(session_key).await { - return session; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - panic!("shared hls test session should be created"); -} - -async fn wait_for_hls_refresh_in_flight(session: &HlsSessionHandle) { - for _ in 0..50 { - if session.read().await.origin_refresh.in_flight { - return; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - panic!("shared hls refresh should be in flight"); -} - -struct TestTransientOrigin { - base_url: String, - requests: Arc>>, - task: tokio::task::JoinHandle<()>, -} - -impl Drop for TestTransientOrigin { - fn drop(&mut self) { self.task.abort(); } -} - -async fn spawn_test_transient_origin() -> TestTransientOrigin { - spawn_test_transient_origin_with_response( - "206 Partial Content", - &[ - ("Content-Type", "video/mp2t"), - ("Content-Range", "bytes 2-15/16"), - ("Accept-Ranges", "bytes"), - ("Cache-Control", "no-store"), - ("ETag", "\"abc\""), - ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"), - ], - "transient-body", - ) - .await -} - -async fn spawn_test_transient_origin_with_response( - status_line: &'static str, - response_headers: &'static [(&'static str, &'static str)], - body: &'static str, -) -> TestTransientOrigin { - spawn_test_transient_origin_with_delayed_binary_response( - status_line, - response_headers, - body.as_bytes().to_vec(), - Duration::ZERO, - ) - .await -} - -async fn spawn_test_transient_origin_with_delayed_response( - status_line: &'static str, - response_headers: &'static [(&'static str, &'static str)], - body: &'static str, - response_delay: Duration, -) -> TestTransientOrigin { - spawn_test_transient_origin_with_delayed_binary_response( - status_line, - response_headers, - body.as_bytes().to_vec(), - response_delay, - ) - .await -} - -async fn spawn_test_transient_origin_with_delayed_binary_response( - status_line: &'static str, - response_headers: &'static [(&'static str, &'static str)], - body: Vec, - response_delay: Duration, -) -> TestTransientOrigin { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let addr = listener.local_addr().expect("local addr"); - let requests = Arc::new(tokio::sync::Mutex::new(Vec::new())); - let task_requests = Arc::clone(&requests); - let body = Arc::new(body); - let task = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let requests = Arc::clone(&task_requests); - let body = Arc::clone(&body); - tokio::spawn(async move { - let mut buf = vec![0_u8; 4096]; - let Ok(read) = socket.read(&mut buf).await else { - return; - }; - if read == 0 { - return; - } - let request = String::from_utf8_lossy(&buf[..read]).to_string(); - requests.lock().await.push(request); - if !response_delay.is_zero() { - tokio::time::sleep(response_delay).await; - } - let mut response_head = format!("HTTP/1.1 {status_line}\r\nContent-Length: {}\r\n", body.len()); - for (name, value) in response_headers { - let _ = writeln!(&mut response_head, "{name}: {value}\r"); - } - response_head.push_str("Connection: close\r\n\r\n"); - let _ = socket.write_all(response_head.as_bytes()).await; - let _ = socket.write_all(body.as_slice()).await; - }); - } - }); - TestTransientOrigin { base_url: format!("http://{addr}"), requests, task } -} - -#[tokio::test] -async fn valid_hls_proxy_segment_without_session_returns_not_found() { - let status = get_status(test_app_state(), "/hls/shared/live/a8f31c9eQ7sLk92pV0mTaw/000123.ts").await; - - assert_eq!(status, StatusCode::NOT_FOUND); -} - -#[tokio::test] -async fn valid_hls_proxy_segment_with_not_ready_session_returns_not_found() { - let app_state = test_app_state(); - let proxy_session_id = map_segment(&app_state, 123, "ts").await; - - let status = get_status(app_state, &format!("/hls/shared/live/{proxy_session_id}/000123.ts")).await; - - assert_eq!(status, StatusCode::NOT_FOUND); -} - -#[tokio::test] -async fn valid_hls_proxy_segment_with_not_ready_and_valid_lease_returns_service_unavailable() { - let app_state = test_app_state(); - let proxy_session_id = map_segment(&app_state, 123, "ts").await; - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) - .await - .expect("session should exist"); - session.write().await.segments.get_mut(&123).expect("segment should exist").origin_fetch_ref = None; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; - - let response = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!(response.headers()[header::RETRY_AFTER], "1"); - assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); - assert_no_hls_cache_stream_registered(&app_state).await; -} - -#[tokio::test] -async fn not_ready_hls_proxy_segment_with_fetch_ref_demand_fetches_and_returns_ok() { - let origin = spawn_test_segment_origin(b"0123456789").await; - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = - map_segment_with_origin_url(&app_state, 123, "ts", &format!("{}/seg.ts", origin.base_url)).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; - - let response = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); -} - -#[tokio::test] -async fn not_ready_hls_proxy_segment_with_range_waits_for_demand_fetch_then_returns_partial() { - let origin = spawn_test_segment_origin(b"0123456789").await; - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = - map_segment_with_origin_url(&app_state, 123, "ts", &format!("{}/seg.ts", origin.base_url)).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; - - let response = get_response(app_state, &uri, Some("bytes=2-5")).await; - - assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); - assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"2345")); -} - -#[tokio::test] -async fn not_ready_hls_proxy_segment_without_fetch_ref_returns_service_unavailable() { - let app_state = test_app_state(); - let proxy_session_id = map_segment(&app_state, 123, "ts").await; - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) - .await - .expect("session should exist"); - session.write().await.segments.get_mut(&123).expect("segment should exist").origin_fetch_ref = None; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; - - let response = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!(response.headers()[header::RETRY_AFTER], "1"); - assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); - assert_no_hls_cache_stream_registered(&app_state).await; -} - -#[tokio::test] -async fn ready_hls_proxy_segment_without_lease_returns_not_found() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_ready_segment_without_lease(&app_state, 123, "ts", b"0123456789").await; - - let status = get_status(app_state, &format!("/hls/shared/live/{proxy_session_id}/000123.ts")).await; - - assert_eq!(status, StatusCode::NOT_FOUND); -} - -#[tokio::test] -async fn invalid_normal_segment_uri_never_redirects_or_serves_terminal_media() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "broken.ts").await; - - let response = get_response(app_state, &uri, None).await; - - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert!(!response.headers().contains_key(header::LOCATION)); -} - -#[tokio::test] -async fn ready_hls_proxy_segment_marked_for_gc_returns_not_found_without_redirect() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) - .await - .expect("session should exist"); - session.write().await.mark_for_gc_removal(); - - let response = get_response(app_state, &uri, None).await; - - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert!(!response.headers().contains_key(header::LOCATION)); -} - -#[tokio::test] -async fn ready_hls_proxy_segment_without_range_returns_ok() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; - - let response = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp2t"); - assert_eq!(response.headers()[header::CONTENT_LENGTH], "10"); - assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); - assert_eq!(response.headers()[header::CACHE_CONTROL], "public, max-age=300, immutable"); - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) - .await - .expect("session should exist"); - { - let session = session.read().await; - let segment = session.segments.get(&123).expect("segment should exist"); - assert_eq!(segment.access.active_readers(), 1); - assert!(matches!(segment.status, SegmentCacheStatus::Ready { content_length: 10, .. })); - } - - assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); - - { - let session = session.read().await; - let segment = session.segments.get(&123).expect("segment should exist"); - assert_eq!(segment.access.active_readers(), 0); - assert!(segment.access.last_accessed_at_ms() > 0); - assert!(session.activity.last_authorized_media_at_ms.is_some()); - } - assert_hls_cache_stream_registered(&app_state, &proxy_session_id).await; -} - -#[tokio::test] -async fn ready_hls_proxy_segment_range_zero_open_returns_partial_content() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_ready_segment(&app_state, 123, "m4s", b"0123456789").await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.m4s").await; - - let response = get_response(app_state, &uri, Some("bytes=0-")).await; - - assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp4"); - assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 0-9/10"); - assert_eq!(response.headers()[header::CONTENT_LENGTH], "10"); - assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); -} - -#[tokio::test] -async fn ready_hls_proxy_segment_range_start_open_returns_partial_content_from_offset() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_ready_segment(&app_state, 123, "m4v", b"0123456789").await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.m4v").await; - - let response = get_response(app_state, &uri, Some("bytes=4-")).await; - - assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp4"); - assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 4-9/10"); - assert_eq!(response.headers()[header::CONTENT_LENGTH], "6"); - assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"456789")); -} - -#[tokio::test] -async fn ready_hls_proxy_segment_range_start_end_returns_partial_content() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; - - let response = get_response(app_state, &uri, Some("bytes=2-5")).await; - - assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); - assert_eq!(response.headers()[header::CONTENT_LENGTH], "4"); - assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"2345")); -} - -#[tokio::test] -async fn ready_hls_proxy_segment_suffix_range_returns_partial_content() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; - - let response = get_response(app_state, &uri, Some("bytes=-3")).await; - - assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 7-9/10"); - assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"789")); -} - -#[tokio::test] -async fn ready_hls_proxy_segment_unsatisfiable_range_returns_416() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; - - let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=99-")).await; - - assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); - assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes */10"); - assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); - assert_eq!(response.headers()[header::CONTENT_LENGTH], "0"); - assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); - assert_no_hls_cache_stream_registered(&app_state).await; -} - -#[tokio::test] -async fn ready_hls_proxy_segment_multi_range_returns_416() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; - - let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=0-1,4-5")).await; - - assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); - assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes */10"); - assert_eq!(response.headers()[header::CONTENT_LENGTH], "0"); - assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); - assert_no_hls_cache_stream_registered(&app_state).await; -} - -#[tokio::test] -async fn ready_hls_proxy_map_without_lease_returns_not_found() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_hls_map(&app_state, b"map-body", false).await; - - let status = get_status(app_state, &format!("/hls/shared/live/{proxy_session_id}/map/000000.mp4")).await; - - assert_eq!(status, StatusCode::NOT_FOUND); -} - -#[tokio::test] -async fn ready_hls_proxy_map_without_range_returns_ok() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_hls_map(&app_state, b"0123456789", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "map/000000.mp4").await; - - let response = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp4"); - assert_eq!(response.headers()[header::CONTENT_LENGTH], "10"); - assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); - assert!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await.is_some()); - assert_hls_cache_stream_registered(&app_state, &proxy_session_id).await; -} - -#[tokio::test] -async fn ready_hls_proxy_map_range_returns_partial_content() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_hls_map(&app_state, b"0123456789", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "map/000000.mp4").await; - - let response = get_response(app_state, &uri, Some("bytes=2-5")).await; - - assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); - assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"2345")); -} - -#[tokio::test] -async fn ready_hls_proxy_map_multi_range_returns_416_with_zero_content_length() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_hls_map(&app_state, b"0123456789", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "map/000000.mp4").await; - - let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=0-1,4-5")).await; - - assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); - assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes */10"); - assert_eq!(response.headers()[header::CONTENT_LENGTH], "0"); - assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); - assert_no_hls_cache_stream_registered(&app_state).await; -} - -#[tokio::test] -async fn hls_proxy_map_not_ready_with_valid_lease_returns_service_unavailable() { - let app_state = test_app_state(); - let session = - app_state.hls.proxy.get_or_create_session(HlsSessionKey::new(1, "12345"), b"rewrite-secret", 100).await; - let manifest = normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n000123.m4s\n"); - let proxy_session_id = { - let mut session = session.write().await; - session.apply_origin_manifest(&manifest).expect("manifest should map"); - session.proxy_session_id.0.clone() - }; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "map/000000.mp4").await; - - let response = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!(response.headers()[header::RETRY_AFTER], "1"); - assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); - assert_no_hls_cache_stream_registered(&app_state).await; -} - -#[tokio::test] -async fn transient_resource_without_lease_returns_not_found() { - let app_state = test_app_state(); - let origin = spawn_test_transient_origin().await; - let (proxy_session_id, resource_id) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", false).await; - - let status = get_status(app_state, &format!("/hls/shared/live/{proxy_session_id}/r/{resource_id}.ts")).await; - - assert_eq!(status, StatusCode::NOT_FOUND); -} - -#[tokio::test] -async fn transient_resource_with_valid_lease_streams_origin_response_and_headers() { - let app_state = test_app_state(); - let origin = spawn_test_transient_origin().await; - let (proxy_session_id, resource_id) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) - .await - .expect("session should exist"); - { - let mut session = session.write().await; - session.origin_request_headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); - session.origin_request_headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); - session - .origin_request_headers - .insert(HeaderName::from_static("proxy-authorization"), HeaderValue::from_static("Basic secret")); - session.origin_request_headers.insert(header::HOST, HeaderValue::from_static("proxy.example.com")); - session - .origin_request_headers - .insert(HeaderName::from_static("x-tuliprox-main-revision"), HeaderValue::from_static("secret")); - session.origin_request_headers.insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("de")); - } - - let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-15")).await; - - assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp2t"); - assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-15/16"); - assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); - assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); - assert_eq!(response.headers()[header::ETAG], "\"abc\""); - assert_eq!(response.headers()[header::LAST_MODIFIED], "Wed, 21 Oct 2015 07:28:00 GMT"); - assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"transient-body")); - assert!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await.is_some()); - let origin_requests = origin.requests.lock().await; - let origin_request = origin_requests.first().expect("origin request").to_ascii_lowercase(); - assert!(origin_request.contains("range: bytes=2-15")); - assert!(origin_request.contains("accept-language: de")); - assert!(!origin_request.contains("authorization: bearer secret")); - assert!(!origin_request.contains("cookie: sid=secret")); - assert!(!origin_request.contains("proxy-authorization: basic secret")); - assert!(!origin_request.contains("host: proxy.example.com")); - assert!(!origin_request.contains("x-tuliprox-main-revision")); - assert_hls_cache_stream_registered(&app_state, &proxy_session_id).await; -} - -#[tokio::test] -async fn transient_resource_without_range_is_cached_after_first_fetch() { - let app_state = test_app_state(); - let origin = - spawn_test_transient_origin_with_response("200 OK", &[("Content-Type", "video/mp2t")], "0123456789").await; - let (proxy_session_id, resource_id) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; - - let first = get_response(Arc::clone(&app_state), &uri, None).await; - let second = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(first.status(), StatusCode::OK); - assert_eq!(second.status(), StatusCode::OK); - assert_eq!(response_body(first).await, bytes::Bytes::from_static(b"0123456789")); - assert_eq!(response_body(second).await, bytes::Bytes::from_static(b"0123456789")); - assert_eq!(origin.requests.lock().await.len(), 1); -} - -#[tokio::test] -async fn transient_resource_range_from_zero_is_cached_as_full_object() { - let app_state = test_app_state(); - let origin = - spawn_test_transient_origin_with_response("200 OK", &[("Content-Type", "video/mp2t")], "0123456789").await; - let (proxy_session_id, resource_id) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; - - let first = get_response(Arc::clone(&app_state), &uri, Some("bytes=0-")).await; - let second = get_response(Arc::clone(&app_state), &uri, Some("bytes=4-")).await; - - assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response_body(first).await, bytes::Bytes::from_static(b"0123456789")); - assert_eq!(response_body(second).await, bytes::Bytes::from_static(b"456789")); - let requests = origin.requests.lock().await; - assert_eq!(requests.len(), 1); - let request = requests[0].to_ascii_lowercase(); - assert!(request.contains("accept-encoding: identity")); - assert!(!request.contains("\r\nrange:")); -} - -#[tokio::test] -async fn transient_cache_fill_rejects_identity_partial_without_ready_object_or_temp_file() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(Arc::clone(&hls_proxy)); - disable_custom_stream_response(&app_state); - let origin = spawn_test_transient_origin_with_response( - "206 Partial Content", - &[("Content-Type", "video/mp2t"), ("Content-Range", "bytes 0-3/10")], - "part", - ) - .await; - let (proxy_session_id, resource_id) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; - - let response = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(response.status(), StatusCode::NOT_FOUND); - let proxy_session_id = ProxySessionId(proxy_session_id); - let cache_key = TransientObjectCacheKey::new(proxy_session_id.clone(), TransientResourceId(resource_id), "ts"); - let session = hls_proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.expect("session should exist"); - let status = - session.read().await.transient.object_cache.get(&cache_key).expect("transient cache entry").status.clone(); - assert!(matches!(status, TransientObjectCacheStatus::FailedPermanent { status: None, .. })); - assert!(hls_proxy.segment_cache().metadata(&cache_key).await.expect("cache metadata reads").is_none()); - assert!(!hls_proxy.segment_cache().has_active_temp_files()); - assert_eq!(std::fs::read_dir(temp_dir.path()).expect("cache root reads").count(), 0); - - let requests = origin.requests.lock().await; - assert_eq!(requests.len(), 1); - let request = requests[0].to_ascii_lowercase(); - assert!(request.contains("accept-encoding: identity")); - assert!(!request.contains("\r\nrange:")); -} - -#[tokio::test] -async fn transient_resource_range_from_zero_waits_for_inflight_object_cache_fetch() { - let app_state = test_app_state(); - let origin = spawn_test_transient_origin_with_delayed_response( - "200 OK", - &[("Content-Type", "video/mp2t")], - "0123456789", - Duration::from_millis(150), - ) - .await; - let (proxy_session_id, resource_id) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; - - let first_app_state = Arc::clone(&app_state); - let first_uri = uri.clone(); - let first = tokio::spawn(async move { get_response(first_app_state, &first_uri, Some("bytes=0-")).await }); - for _ in 0..50 { - if origin.requests.lock().await.len() == 1 { - break; - } - tokio::time::sleep(Duration::from_millis(5)).await; - } - assert_eq!(origin.requests.lock().await.len(), 1); - - let second = get_response(Arc::clone(&app_state), &uri, Some("bytes=0-")).await; - let first = first.await.expect("first request joins"); - - assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response_body(first).await, bytes::Bytes::from_static(b"0123456789")); - assert_eq!(response_body(second).await, bytes::Bytes::from_static(b"0123456789")); - assert_eq!(origin.requests.lock().await.len(), 1); -} - -#[tokio::test] -async fn transient_resource_range_from_offset_without_ready_object_is_not_cached() { - let app_state = test_app_state(); - let origin = spawn_test_transient_origin().await; - let (proxy_session_id, resource_id) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; - - let first = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-15")).await; - let second = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-15")).await; - - assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(origin.requests.lock().await.len(), 2); -} - -#[tokio::test] -async fn transient_key_resource_is_not_cached() { - let app_state = test_app_state(); - let origin = spawn_test_transient_origin_with_response( - "200 OK", - &[("Content-Type", "application/octet-stream")], - "key-bytes", - ) - .await; - let (proxy_session_id, resource_id) = map_transient_resource_with_kind( - &app_state, - &format!("{}/key.bin", origin.base_url), - "key", - true, - TransientResourceKind::Key, - ) - .await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.key")).await; - - let first = get_response(Arc::clone(&app_state), &uri, None).await; - let second = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(first.status(), StatusCode::OK); - assert_eq!(second.status(), StatusCode::OK); - assert_eq!(response_body(first).await, bytes::Bytes::from_static(b"key-bytes")); - assert_eq!(response_body(second).await, bytes::Bytes::from_static(b"key-bytes")); - assert_eq!(origin.requests.lock().await.len(), 2); -} - -#[tokio::test] -async fn transient_resource_origin_error_does_not_mark_media_activity() { - let app_state = test_app_state(); - let origin = spawn_test_transient_origin_with_response( - "500 Internal Server Error", - &[("Content-Type", "text/plain")], - "origin-error", - ) - .await; - let (proxy_session_id, resource_id) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; - - let response = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!(response.headers().get(header::RETRY_AFTER).and_then(|value| value.to_str().ok()), Some("1")); - assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); - assert_no_hls_cache_stream_registered(&app_state).await; -} - -#[tokio::test] -async fn transient_resource_permanent_origin_error_never_redirects_to_manifest() { - let app_state = test_app_state(); - let origin = - spawn_test_transient_origin_with_response("404 Not Found", &[("Content-Type", "text/plain")], "missing").await; - let (proxy_session_id, resource_id) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; - - let response = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert!(!response.headers().contains_key(header::LOCATION)); - assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); - assert_no_hls_cache_stream_registered(&app_state).await; -} - -#[tokio::test] -async fn transient_resource_permanent_origin_error_returns_not_found_when_custom_response_disabled() { - let app_state = test_app_state(); - disable_custom_stream_response(&app_state); - let origin = - spawn_test_transient_origin_with_response("404 Not Found", &[("Content-Type", "text/plain")], "missing").await; - let (proxy_session_id, resource_id) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; - - let response = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); - assert_no_hls_cache_stream_registered(&app_state).await; -} - -#[tokio::test] -async fn transient_resource_holds_provider_handle_until_origin_body_is_finished() { - let input = overlap_provider_input(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let origin = spawn_test_transient_origin().await; - let (proxy_session_id, resource_id) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; - let proxy_session_id_value = ProxySessionId(proxy_session_id.clone()); - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&proxy_session_id_value) - .await - .expect("session should exist"); - { - let mut session = session.write().await; - session.origin_account_binding = Some(HlsOriginAccountBinding::new( - Arc::clone(&input.name), - Arc::from("account-a"), - &proxy_session_id_value, - super::current_time_millis(), - )); - } - let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; - - let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-15")).await; - - assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - wait_for_provider_connection_count(&app_state, 1).await; - assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"transient-body")); - wait_for_provider_connection_count(&app_state, 0).await; -} - -#[tokio::test] -async fn transient_decoder_failure_releases_origin_and_access_guards_once() { - let input = overlap_provider_input(); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let mut truncated = encode_test_manifest("gzip", b"transient decoder failure").await; - truncated.truncate(truncated.len().saturating_sub(8)); - let origin = spawn_test_transient_origin_with_delayed_binary_response( - "200 OK", - &[("Content-Type", "video/mp2t"), ("Content-Encoding", "gzip")], - truncated, - Duration::ZERO, - ) - .await; - let (proxy_session_id, resource_id) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; - let proxy_session_id_value = ProxySessionId(proxy_session_id.clone()); - let session = app_state - .hls - .proxy - .sessions() - .get_by_proxy_session_id(&proxy_session_id_value) - .await - .expect("session should exist"); - { - let mut session = session.write().await; - session.origin_account_binding = Some(HlsOriginAccountBinding::new( - Arc::clone(&input.name), - Arc::from("account-a"), - &proxy_session_id_value, - super::current_time_millis(), - )); - } - let resource = session - .read() - .await - .transient - .resources - .get(&TransientResourceId(resource_id.clone())) - .cloned() - .expect("transient resource exists"); - let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; - - let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-")).await; - - assert_eq!(response.status(), StatusCode::OK); - wait_for_provider_connection_count(&app_state, 1).await; - assert_eq!(resource.active_readers(), 1); - assert!(response.into_body().collect().await.is_err()); - wait_for_provider_connection_count(&app_state, 0).await; - assert_eq!(resource.active_readers(), 0); - tokio::task::yield_now().await; - assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); - assert_eq!(origin.requests.lock().await.len(), 1, "body failure must not start another origin request"); -} - -#[tokio::test] -async fn transient_unknown_resource_never_redirects_to_manifest() { - let app_state = test_app_state(); - let origin = spawn_test_transient_origin().await; - let (proxy_session_id, _) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "r/unknown.ts").await; - - let response = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert!(!response.headers().contains_key(header::LOCATION)); - assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); - assert_no_hls_cache_stream_registered(&app_state).await; -} - -#[tokio::test] -async fn transient_unknown_resource_returns_not_found_when_custom_response_disabled() { - let app_state = test_app_state(); - disable_custom_stream_response(&app_state); - let origin = spawn_test_transient_origin().await; - let (proxy_session_id, _) = - map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "r/unknown.ts").await; - - let response = get_response(Arc::clone(&app_state), &uri, None).await; - - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); - assert_no_hls_cache_stream_registered(&app_state).await; -} - -#[test] -fn transient_cross_origin_redirect_strips_sensitive_headers() { - let mut headers = HeaderMap::new(); - headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); - headers.insert(header::COOKIE, HeaderValue::from_static("session=secret")); - headers.insert(HeaderName::from_static("proxy-authorization"), HeaderValue::from_static("Basic secret")); - headers.insert(header::HOST, HeaderValue::from_static("origin.example.com")); - headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-1")); - - crate::api::model::scrub_hls_origin_headers(&mut headers, None); - - assert!(!headers.contains_key(header::AUTHORIZATION)); - assert!(!headers.contains_key(header::COOKIE)); - assert!(!headers.contains_key("proxy-authorization")); - assert!(!headers.contains_key(header::HOST)); - assert_eq!(headers[header::RANGE], "bytes=0-1"); -} - -#[tokio::test] -async fn ready_hls_proxy_segment_unknown_range_unit_is_ignored() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let app_state = test_app_state_with_hls_proxy(hls_proxy); - let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; - let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; - - let response = get_response(app_state, &uri, Some("items=0-1")).await; - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); -} - -#[tokio::test] -async fn invalid_hls_proxy_file_names_return_not_found() { - let app_state = test_app_state(); - - assert_eq!( - get_status(Arc::clone(&app_state), "/hls/shared/live/a8f31c9eQ7sLk92pV0mTaw/123.ts").await, - StatusCode::NOT_FOUND - ); - assert_eq!( - get_status(app_state, "/hls/shared/live/a8f31c9eQ7sLk92pV0mTaw/map/000123.exe").await, - StatusCode::NOT_FOUND - ); -} - -#[tokio::test] -async fn legacy_hls_route_remains_registered() { - let status = get_status(test_app_state(), "/hls/user/pass/1/2/3/not-a-token").await; - - assert_ne!(status, StatusCode::NOT_FOUND); -} - -#[tokio::test] -async fn terminate_failed_hls_manifest_session_releases_identified_provider_reservation() { - let input = single_hls_provider_input("failed-hls-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let username = "testuser"; - let session_token = "testuser|stream-1|hls|0123456789abcdef"; - let provider_name = Arc::clone(&input.name); - let addr = test_addr_with_port(55301); - - let handle = app_state - .active_provider - .acquire_connection_with_lease_for_session( - &provider_name, - &addr, - false, - 0, - ConnectionKind::Normal, - Some(tuliprox_session::PlaybackLeaseRef::new(session_token, crate::model::PlaybackKind::LiveHls)), - ) - .expect("handle should be acquired"); - - let binding_tag = handle.binding_tag; - let request_id = handle.playback_request_id; - assert!(binding_tag.is_some(), "handle should carry a binding tag"); - - app_state.connection_manager.release_provider_handle(Some(handle)); - - assert!( - app_state.active_provider.binding_tag_for_owner(session_token).is_some(), - "lease should still be active after releasing connection handle" - ); - - app_state.active_provider.clear_provider_reservation(session_token); - assert!( - app_state.active_provider.binding_tag_for_owner(session_token).is_some(), - "clear_provider_reservation must return early for public HLS token" - ); - - super::segment::terminate_failed_hls_manifest_session( - &app_state, - username, - session_token, - Some(&provider_name), - binding_tag, - request_id, - ) - .await; - - assert!( - app_state.active_provider.binding_tag_for_owner(session_token).is_none(), - "lease should be cleared after terminate_failed_hls_manifest_session with binding identity" - ); -} - -#[tokio::test] -async fn delayed_manifest_failure_of_older_binding_keeps_provider_affinity() { - let input = single_hls_provider_input("affinity-hls-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let provider_name = Arc::clone(&input.name); - let old_token = "testuser|stream-1|hls|0123456789abcdef"; - let new_token = "testuser|stream-1|hls|fedcba9876543210"; - let acquire = |token: &'static str, port: u16| { - app_state - .active_provider - .acquire_connection_with_lease_for_session( - &provider_name, - &test_addr_with_port(port), - false, - 0, - ConnectionKind::Normal, - Some(tuliprox_session::PlaybackLeaseRef::new(token, crate::model::PlaybackKind::LiveHls)), - ) - .expect("handle should be acquired") - }; - - // The older binding never produced media and ended. - let old = acquire(old_token, 55311); - let (old_tag, old_request) = (old.binding_tag, old.playback_request_id); - let old_request_id = old_request.expect("old request id"); - app_state.connection_manager.release_provider_handle(Some(old)); - app_state.active_provider.finish_identified_playback_request( - old_token, - old_request_id, - tuliprox_core::model::PlaybackRequestOutcome::FailedBeforeMedia, - ); - - // The successor binding confirms media and owns the provider preference. - let current = acquire(new_token, 55312); - let current_request = current.playback_request_id.expect("current request id"); - app_state.active_provider.refresh_adaptive_playback_lease( - &provider_name, - new_token, - crate::model::PlaybackKind::LiveHls, - 15, - ); - app_state.active_provider.confirm_identified_playback_activity(new_token, current_request); - app_state.connection_manager.release_provider_handle(Some(current)); - assert_eq!(app_state.active_provider.provider_affinity_for_owner(new_token), Some(Arc::clone(&provider_name))); - - for (tag, request) in [(old_tag, old_request), (old_tag, None)] { - super::segment::terminate_failed_hls_manifest_session( - &app_state, - "testuser", - old_token, - Some(&provider_name), - tag, - request, - ) - .await; - } - - assert_eq!( - app_state.active_provider.provider_affinity_for_owner(new_token), - Some(Arc::clone(&provider_name)), - "a delayed failure of the older binding must not end the successor's provider preference" - ); -} - -#[tokio::test] -async fn terminating_unknown_old_hls_session_keeps_newer_provider_lease() { - use axum::response::IntoResponse; - let input = single_hls_provider_input("terminate-hls-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let provider_name = Arc::clone(&input.name); - let old_token = "testuser|stream-1|hls|0123456789abcdef"; - let new_token = "testuser|stream-1|hls|fedcba9876543210"; - let current = app_state - .active_provider - .acquire_connection_with_lease_for_session( - &provider_name, - &test_addr_with_port(55321), - false, - 0, - ConnectionKind::Normal, - Some(tuliprox_session::PlaybackLeaseRef::new(new_token, crate::model::PlaybackKind::LiveHls)), - ) - .expect("handle should be acquired"); - let request_id = current.playback_request_id.expect("request id"); - app_state.active_provider.refresh_adaptive_playback_lease( - &provider_name, - new_token, - crate::model::PlaybackKind::LiveHls, - 15, - ); - app_state.active_provider.confirm_identified_playback_activity(new_token, request_id); - app_state.connection_manager.release_provider_handle(Some(current)); - app_state.active_provider.finish_identified_playback_request( - new_token, - request_id, - tuliprox_core::model::PlaybackRequestOutcome::Completed, - ); - let binding_tag = app_state.active_provider.binding_tag_for_owner(new_token); - assert!(binding_tag.is_some(), "the newer playback keeps an idle lease"); - - let response = crate::api::endpoints::v1_api_user::terminate_user_session( - axum::extract::State(Arc::clone(&app_state)), - axum::extract::Path(("testuser".to_string(), old_token.to_string())), - ) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::NO_CONTENT); - assert_eq!(app_state.active_provider.binding_tag_for_owner(new_token), binding_tag); - assert_eq!(app_state.active_provider.provider_affinity_for_owner(new_token), Some(provider_name)); -} - -#[tokio::test] -async fn terminate_failed_hls_manifest_session_preserves_shared_lease_when_request_id_available() { - let mut input = single_hls_provider_input("shared-hls-input"); - input.max_connections = 2; - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let username = "testuser"; - let session_token = "testuser|stream-1|hls|0123456789abcdef"; - let provider_name = Arc::clone(&input.name); - let addr = test_addr_with_port(55302); - - let handle1 = app_state - .active_provider - .acquire_connection_with_lease_for_session( - &provider_name, - &addr, - false, - 0, - ConnectionKind::Normal, - Some(tuliprox_session::PlaybackLeaseRef::new(session_token, crate::model::PlaybackKind::LiveHls)), - ) - .expect("handle1 should be acquired"); - - let handle2 = app_state - .active_provider - .acquire_connection_with_lease_for_session( - &provider_name, - &addr, - false, - 0, - ConnectionKind::Normal, - Some(tuliprox_session::PlaybackLeaseRef::new(session_token, crate::model::PlaybackKind::LiveHls)), - ) - .expect("handle2 should be acquired"); - - let binding_tag = handle1.binding_tag; - let request_id1 = handle1.playback_request_id; - let request_id2 = handle2.playback_request_id; - assert!(request_id1.is_some() && request_id2.is_some() && request_id1 != request_id2); - - app_state.connection_manager.release_provider_handle(Some(handle1)); - - super::segment::terminate_failed_hls_manifest_session( - &app_state, - username, - session_token, - Some(&provider_name), - binding_tag, - request_id1, - ) - .await; - - assert!( - app_state.active_provider.binding_tag_for_owner(session_token).is_some(), - "shared lease must remain active for surviving request" - ); - - app_state.connection_manager.release_provider_handle(Some(handle2)); - super::segment::terminate_failed_hls_manifest_session( - &app_state, - username, - session_token, - Some(&provider_name), - binding_tag, - request_id2, - ) - .await; - - assert!( - app_state.active_provider.binding_tag_for_owner(session_token).is_none(), - "lease should be cleared once all requests are terminated" - ); -} - -#[tokio::test] -async fn terminate_failed_hls_manifest_session_clears_identified_reservation_without_request_id() { - let input = single_hls_provider_input("no-request-id-hls-input"); - let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); - let username = "testuser"; - let session_token = "testuser|stream-1|hls|0123456789abcdef"; - let provider_name = Arc::clone(&input.name); - let addr = test_addr_with_port(55303); - - let handle = app_state - .active_provider - .acquire_connection_with_lease_for_session( - &provider_name, - &addr, - false, - 0, - ConnectionKind::Normal, - Some(tuliprox_session::PlaybackLeaseRef::new(session_token, crate::model::PlaybackKind::LiveHls)), - ) - .expect("handle should be acquired"); - - let binding_tag = handle.binding_tag; - assert!(binding_tag.is_some()); - - app_state.connection_manager.release_provider_handle(Some(handle)); - - assert!(app_state.active_provider.binding_tag_for_owner(session_token).is_some(), "lease should still be active"); - - super::segment::terminate_failed_hls_manifest_session( - &app_state, - username, - session_token, - Some(&provider_name), - binding_tag, - None, - ) - .await; - - assert!( - app_state.active_provider.binding_tag_for_owner(session_token).is_none(), - "lease should be cleared via binding tag when no request ID is available" - ); -} +use tuliprox_hls::api::{ + derive_hls_lease_manifest_snapshot, hls_manifest_commit_requirement, HlsAccessLeasePendingDeadline, + HlsCachedManifestOptions, HlsLeaseManifestSnapshotInput, LiveHlsOriginEntry, +}; mod owner_token; + +mod archive; +mod bandwidth; +mod encrypted_media; +mod entry_identity; +mod headers; +mod lease_lifecycle; +mod manifest_startup; +mod observability; +mod owner_handoff; +mod provider_admission; +mod runtime_policy; +mod segment_responses; +mod support; +mod terminal_resources; +mod terminal_tail; +mod transient_resources; + +use self::support::{ + access_lease_id_from_variant_uri, access_lease_session_token, activate_test_hls_access_lease, + assert_hls_cache_stream_registered, assert_no_hls_cache_stream_registered, cache_recording_test_item, + cache_test_m3u_hls_item, configure_default_test_server, configure_recording_test_listener, + create_active_hls_user_session, create_active_hls_user_session_with, create_bound_hls_test_session, + create_unbound_hls_test_session, disable_custom_stream_response, enable_channel_unavailable_custom_response, + enable_hls_cache, encode_test_manifest, encrypt_test_aes128_cbc_pkcs7, get_response, get_status, + hls_custom_video_test_user, hls_proxy_uri, hls_session_last_media_at_ms, legacy_manifest_test_client_headers, + legacy_manifest_test_input, manifest_media_sequence, map_hls_map, map_ready_segment, + map_ready_segment_without_lease, map_segment, map_segment_with_origin_url, map_transient_resource, + map_transient_resource_with_kind, mark_hls_user_session_exhausted, media_uri_count, normal_manifest, + normal_manifest_body, overlap_provider_input, path_has_extension, prepare_pending_test_hls_access_lease, + proxy_session_id_from_variant_uri, publish_owner_handoff_test_manifest, publish_ready_test_manifest_for_lease, + record_test_normal_manifest_commit, recording_test_response, recording_test_router, recording_test_url, + regression_origin_manifest, request_response, response_body, runtime_policy_endpoint_fixture, + single_hls_provider_input, single_variant_master_playlist, single_variant_uri, spawn_recording_header_origin, + spawn_test_binary_origin, spawn_test_encoded_manifest_origin, spawn_test_encrypted_hls_origin, + spawn_test_segment_origin, spawn_test_status_origin, spawn_test_transient_origin, + spawn_test_transient_origin_with_delayed_binary_response, spawn_test_transient_origin_with_delayed_response, + spawn_test_transient_origin_with_response, store_normal_manifest_body, store_test_sources_with_target, + terminal_test_asset, terminal_test_plan_shape, terminalize_existing_test_lease, test_addr, test_addr_with_port, + test_app_state, test_app_state_with_hls_proxy, test_app_state_with_hls_proxy_and_inputs, + test_app_state_with_inputs, test_custom_video_buffer, test_fingerprint, test_fingerprint_with_addr, + test_hls_access_context, test_hls_access_context_with, test_hls_entry_stream_context, test_hls_input, + test_hls_sequence_iv, test_hls_share_target, test_m3u_hls_item, test_m3u_hls_share_target, test_segment_entry, + transient_manifest_body, transient_manifest_body_from_sequence, try_test_hls_cached_manifest_response, + wait_for_provider_connection_count, wait_for_runtime_policy_terminal_plan, CanonicalOwnerHandoffFixture, + RuntimePolicyEndpointFixture, TestBinaryOriginResponse, TestSegmentOrigin, AES_TEST_KEY_BYTES, AES_TEST_MANIFEST, + AES_TEST_PLAINTEXT_SEGMENT, +}; + +mod provider_admission_account_overlap; +mod provider_admission_provider_binding; +mod provider_admission_provider_identity; +mod provider_admission_provider_lease_release; + +mod terminal_tail_recovery_before_cutover; +mod terminal_tail_terminal_cutover; +mod terminal_tail_terminal_fallback; +mod terminal_tail_terminal_preflight; diff --git a/backend/app/src/api/endpoints/hls_api/tests/archive.rs b/backend/app/src/api/endpoints/hls_api/tests/archive.rs new file mode 100644 index 000000000..ada0dd319 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/archive.rs @@ -0,0 +1,600 @@ +use super::{ + access_lease_id_from_variant_uri, cache_test_m3u_hls_item, configure_default_test_server, enable_hls_cache, + get_response, hls_api_register, m3u_archive_epg_reference_ts, m3u_catchup_epg_reference_from_session_token, + manifest_media_sequence, path_has_extension, proxy_session_id_from_variant_uri, regression_origin_manifest, + resolve_leaked_hls_relative_origin, response_body, single_variant_master_playlist, spawn_test_binary_origin, + store_test_sources_with_target, test_addr, test_app_state, test_app_state_with_inputs, test_fingerprint, + test_hls_access_context, test_hls_entry_stream_context, test_hls_input, test_hls_share_target, test_m3u_hls_item, + test_m3u_hls_share_target, TestBinaryOriginResponse, TestSegmentOrigin, +}; +use crate::{ + api::model::{ + build_proxy_session_id, AppState, ConnectionKind, HlsAccessLeaseId, HlsOriginSource, HlsOriginSourceKind, + ProxySessionId, + }, + model::{ConfigInput, ProxyUserCredentials, TargetUser}, +}; +use axum::{ + body::Body, + extract::ConnectInfo, + http::{header, HeaderMap, Request, Response, StatusCode}, + response::IntoResponse, +}; +use shared::model::{InputType, PlaylistItemType, UserConnectionPermission, VirtualId, XtreamCluster}; +use std::sync::Arc; +use tower::ServiceExt; + +#[test] +fn archive_epg_reference_supports_query_and_path_formats() { + assert_eq!( + m3u_archive_epg_reference_ts("http://provider/live/42.m3u8?utc=1700000000&lutc=1700003600"), + Some(1_700_000_000) + ); + assert_eq!( + m3u_archive_epg_reference_ts("http://provider/live/archive-1700003600-1700007200.m3u8"), + Some(1_700_003_600) + ); + assert_eq!(m3u_archive_epg_reference_ts("http://provider/live/timeshift_abs-1700007200.ts"), Some(1_700_007_200)); + assert_eq!( + m3u_archive_epg_reference_ts("http://provider/live/42.m3u8?start=1700000000&end=1700003600"), + Some(1_700_000_000) + ); +} + +#[test] +fn archive_epg_reference_rejects_plain_start_queries() { + assert_eq!(m3u_archive_epg_reference_ts("http://provider/live/42.m3u8?start=1700000000"), None); +} + +#[test] +fn date_tree_path_recovers_bittv_archive_epg_reference() { + assert_eq!(super::super::epg_reference_ts_from_date_tree_path("2026/07/24/14/13/38-06800.ts"), Some(1_784_902_418)); + assert_eq!( + super::super::epg_reference_ts_from_date_tree_path("dvr-2026/07/24/14/13/38-06800.ts"), + super::super::epg_reference_ts_from_date_tree_path("2026/07/24/14/13/38-06800.ts") + ); + assert!(super::super::looks_like_archive_media_path("2026/07/24/14/13/38-06800.ts")); +} + +#[test] +fn archive_media_path_does_not_accept_plain_202_prefixed_segments() { + assert!(!super::super::looks_like_archive_media_path("2026.ts")); + assert!(!super::super::looks_like_archive_media_path("202_media.ts")); +} + +#[test] +fn session_token_recovers_archive_epg_reference_when_media_url_lost_markers() { + assert_eq!( + m3u_catchup_epg_reference_from_session_token("m3u-catchup|user|42|archive|1717200000|3600"), + Some(1_717_200_000) + ); + assert_eq!(m3u_catchup_epg_reference_from_session_token("m3u-catchup|user|42|live"), None); +} + +#[test] +fn append_catchup_session_hint_keeps_m3u_catchup_token_without_shared_hls_cache() { + let fingerprint = test_fingerprint(); + let hint = "m3u-catchup|fp|alice|42|deadbeef"; + let token = super::super::hls_entry_user_session_token(&fingerprint, "alice", 42, Some(hint), Some(1_717_200_000)); + assert_eq!(token, hint); + assert!(super::super::is_m3u_catchup_session_token(&token)); + + let from_archive = super::super::hls_entry_user_session_token(&fingerprint, "alice", 42, None, Some(1_717_200_000)); + assert!(from_archive.contains("|archive|1717200000|0")); + assert!(super::super::is_m3u_catchup_session_token(&from_archive)); + assert!(super::super::is_m3u_catchup_session_token("m3u-catchup|fp|alice|42|timeshift_abs|1717200000|0")); +} + +#[test] +fn leaked_dvr_relative_joins_against_media_playlist_and_dvr_session_root() { + assert_eq!( + resolve_leaked_hls_relative_origin( + "http://cdn.example/big/aa_1/media.m3u8", + "dvr-2026/07/26/15/30/59-06000.ts", + Some("token=abc"), + ), + Some("http://cdn.example/big/aa_1/dvr-2026/07/26/15/30/59-06000.ts?token=abc".to_string()) + ); + assert_eq!( + resolve_leaked_hls_relative_origin( + "http://cdn.example/big/aa_1/dvr-2026/07/26/15/30/59-06000.ts?token=old", + "dvr-2026/07/26/15/31/05-06000.ts", + Some("token=new"), + ), + Some("http://cdn.example/big/aa_1/dvr-2026/07/26/15/31/05-06000.ts?token=new".to_string()) + ); + assert_eq!( + resolve_leaked_hls_relative_origin("http://cdn.example/big/aa_1/media.m3u8", "segment001.ts", None,), + None + ); + assert_eq!( + resolve_leaked_hls_relative_origin( + "http://cdn.example/big/aa_1/media.m3u8", + "dvr-2026/07/26/15/30/../59-06000.ts", + None, + ), + None + ); + assert_eq!(resolve_leaked_hls_relative_origin("http://cdn.example/live.m3u8", "./dvr-2026/a.ts", None), None); +} + +#[test] +fn archive_epg_reference_supports_contextual_start_aliases() { + assert_eq!( + m3u_archive_epg_reference_ts("http://provider/live/42.m3u8?offset=-3600&utcstart=1717200000"), + Some(1_717_200_000) + ); + assert_eq!( + m3u_archive_epg_reference_ts("http://provider/live/42.m3u8?timestamp=1717200000&offset=120"), + Some(1_717_200_000) + ); +} + +#[test] +fn hls_cache_session_tokens_separate_live_and_archive_playback() { + let fingerprint = test_fingerprint(); + let live = super::super::create_hls_cache_user_session_token(&fingerprint, "user", 31, None, None); + let archive = + super::super::create_hls_cache_user_session_token(&fingerprint, "user", 31, None, Some(1_784_898_000)); + + assert!(!super::super::is_m3u_catchup_session_token(&live)); + assert!(super::super::is_m3u_catchup_session_token(&archive)); + assert_ne!(live, archive); +} + +#[test] +fn hls_cache_session_token_preserves_existing_m3u_catchup_identity() { + let fingerprint = test_fingerprint(); + let existing = "m3u-catchup|fp|user|31|archive|1784898000|3600"; + let token = super::super::create_hls_cache_user_session_token( + &fingerprint, + "user", + 31, + Some(existing), + Some(1_784_898_000), + ); + + assert!(token.starts_with(existing)); + assert!(token.contains("|hls-cache|")); +} + +#[tokio::test] +async fn hls_cache_stream_channel_uses_archive_epg_context() { + let app_state = test_app_state(); + let mut access = test_hls_access_context( + ProxySessionId("proxy-archive".to_string()), + HlsAccessLeaseId("lease-archive".to_string()), + ); + access.epg_reference_ts = Some(1_784_898_000); + access.archive_origin_url = Some("http://provider/channel/timeshift_abs-1784898000.m3u8".to_string()); + let origin_source = + HlsOriginSource::new(1, Arc::from("test-input"), "80510", HlsOriginSourceKind::M3uMediaPlaylist) + .with_archive_reference(1_784_898_000); + + let channel = super::super::build_hls_cache_stream_channel( + &app_state, + &access, + &origin_source, + &ProxySessionId("proxy-archive".to_string()), + ) + .await; + + assert_eq!(channel.item_type, PlaylistItemType::Catchup); + assert_eq!(channel.cluster, XtreamCluster::Video); + assert_eq!(channel.epg_reference_ts, Some(1_784_898_000)); +} + +#[tokio::test] +async fn hls_cache_manifest_context_restores_leased_archive_origin() -> Result<(), StatusCode> { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let input = ConfigInput { + id: 1, + name: Arc::from("test-input"), + input_type: InputType::M3u, + enabled: true, + ..ConfigInput::default() + }; + let mut target = test_m3u_hls_share_target(); + target.name = "default".to_string(); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + cache_test_m3u_hls_item( + &app_state, + &target, + test_m3u_hls_item(&input, 12345, "80510", "http://provider/channel/mono.m3u8"), + ) + .await; + + let archive_url = "http://provider/channel/timeshift_abs-1784898000.m3u8"; + let mut access = test_hls_access_context( + ProxySessionId("proxy-archive".to_string()), + HlsAccessLeaseId("lease-archive".to_string()), + ); + access.stream_ref = "80510".to_string(); + access.epg_reference_ts = Some(1_784_898_000); + access.archive_origin_url = Some(archive_url.to_string()); + + let context = + super::super::resolve_hls_playback_manifest_request_context(&app_state, &access, &HeaderMap::new()).await?; + + assert_eq!(context.hls_url, archive_url); + assert_eq!(context.origin_source.stream_ref, "80510"); + assert_eq!(context.origin_source.archive_reference, Some(1_784_898_000)); + Ok(()) +} + +#[tokio::test] +async fn hls_cache_archive_entry_uses_distinct_identity_and_preserves_origin() -> Result<(), &'static str> { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + configure_default_test_server(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.password = "hls-pass".to_string(); + let input = test_hls_input(); + let target = test_hls_share_target(true); + let archive_url = "http://origin.example.com/live/user/pass/timeshift_abs-1784898000.m3u8"; + + let response = super::super::handle_hls_stream_request( + &test_fingerprint(), + &app_state, + &user, + &target, + None, + None, + archive_url, + Some(1_784_898_000), + test_hls_entry_stream_context(12345, "80510", None), + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &super::super::build_virtual_hls_entry_path(&target, &input, &user, 12345), + super::super::HlsRequestStage::Entry, + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::OK); + let (_, media_playlist_uri) = single_variant_master_playlist(response).await; + let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&media_playlist_uri).to_string()); + let live_source = super::super::build_hls_origin_source(&input, "80510"); + let archive_source = + super::super::build_hls_origin_source_for_playback(&input, "80510", Some(1_784_898_000), Some(archive_url)); + assert_ne!(proxy_session_id, build_proxy_session_id(&live_source.session_key(), &app_state.get_encrypt_secret())); + assert_eq!( + proxy_session_id, + build_proxy_session_id(&archive_source.session_key(), &app_state.get_encrypt_secret()) + ); + + let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&media_playlist_uri).to_string()); + let lease = app_state + .hls + .proxy + .access_leases() + .write() + .await + .response_snapshot(&access_lease_id, &proxy_session_id, super::super::current_time_millis()) + .ok_or("archive access lease")?; + assert_eq!(lease.stream_ref, "80510"); + assert_eq!(lease.epg_reference_ts, Some(1_784_898_000)); + assert_eq!(lease.archive_origin_url.as_deref(), Some(archive_url)); + assert!(super::super::is_m3u_catchup_session_token(&lease.user_session_token)); + Ok(()) +} + +pub(in crate::api::endpoints::hls_api::tests) struct BoundedFlussonicFixture { + pub(in crate::api::endpoints::hls_api::tests) _temp: tempfile::TempDir, + pub(in crate::api::endpoints::hls_api::tests) origin: TestSegmentOrigin, + pub(in crate::api::endpoints::hls_api::tests) app_state: Arc, + pub(in crate::api::endpoints::hls_api::tests) requests: Arc>>, + pub(in crate::api::endpoints::hls_api::tests) catchup_template: String, + pub(in crate::api::endpoints::hls_api::tests) live_uri: String, +} + +pub(in crate::api::endpoints::hls_api::tests) async fn bounded_flussonic_request( + app_state: Arc, + url: &str, +) -> Result, Box> { + let parsed = url::Url::parse(url)?; + let mut uri = parsed.path().to_owned(); + if let Some(query) = parsed.query() { + uri.push('?'); + uri.push_str(query); + } + let router = crate::api::endpoints::m3u_api::m3u_api_register().merge(hls_api_register()).with_state(app_state); + let mut request = Request::builder().uri(uri).body(Body::empty())?; + request.extensions_mut().insert(ConnectInfo(test_addr())); + Ok(router.oneshot(request).await?) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(in crate::api::endpoints::hls_api::tests) enum BoundedArchiveOrigin { + Media, + Missing, + SingleVariantMaster, +} + +pub(in crate::api::endpoints::hls_api::tests) async fn bounded_flussonic_origin( + archive_origin: BoundedArchiveOrigin, + requests: &Arc>>, +) -> Result> { + let recorded = Arc::clone(requests); + let base = Arc::new(std::sync::OnceLock::::new()); + let origin_base = Arc::clone(&base); + let segment = Arc::<[u8]>::from( + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + .as_slice(), + ); + let origin = spawn_test_binary_origin(Arc::new(move |path| { + if let Ok(mut paths) = recorded.lock() { + paths.push(path.to_owned()); + } + let resource = path.split('?').next().unwrap_or(path); + if resource == "/input.m3u" { + let Some(host) = origin_base.get() else { + return TestBinaryOriginResponse::new(StatusCode::INTERNAL_SERVER_ERROR, Arc::from(&b"origin"[..])); + }; + let text = format!( + "#EXTM3U\n#EXTINF:-1 catchup=\"fs\" catchup-days=\"7\",Channel\n{host}/channel/mono.m3u8?token=a%2Fb\n" + ); + return TestBinaryOriginResponse::new(StatusCode::OK, Arc::from(text.into_bytes())); + } + if std::path::Path::new(resource).extension().is_some_and(|extension| extension.eq_ignore_ascii_case("ts")) + && ["/channel/archive-", "/channel/live/", "/channel/tracks-v1/archive-"] + .iter() + .any(|prefix| resource.starts_with(prefix)) + { + return TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&segment)); + } + let archive_file = resource + .strip_prefix("/channel/") + .and_then(|file| file.strip_suffix(".m3u8")) + .filter(|file| file.starts_with("archive-")); + if let (BoundedArchiveOrigin::SingleVariantMaster, Some(archive)) = (archive_origin, archive_file) { + let master = format!( + "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=9150000,RESOLUTION=1920x1080\ntracks-v1/{archive}.ts.m3u8\n" + ); + return TestBinaryOriginResponse::new(StatusCode::OK, Arc::from(master.into_bytes())); + } + let media_directory = if resource == "/channel/mono.m3u8" { + Some("live") + } else if archive_origin == BoundedArchiveOrigin::SingleVariantMaster { + resource + .strip_prefix("/channel/tracks-v1/") + .and_then(|file| file.strip_suffix(".ts.m3u8")) + .filter(|file| file.starts_with("archive-")) + } else { + archive_file + }; + if let Some(directory) = media_directory { + if archive_origin == BoundedArchiveOrigin::Missing && directory != "live" { + return TestBinaryOriginResponse::new(StatusCode::NOT_FOUND, Arc::from(&b"unavailable"[..])); + } + let mut manifest = String::from_utf8_lossy(®ression_origin_manifest(123, 6)).into_owned(); + for sequence in 123..129 { + manifest = + manifest.replace(&format!("{sequence}.ts"), &format!("{directory}/{sequence}.ts?token=a%2Fb")); + } + if directory != "live" { + manifest.push_str("#EXT-X-ENDLIST\n"); + } + return TestBinaryOriginResponse::new(StatusCode::OK, Arc::from(manifest.into_bytes())); + } + TestBinaryOriginResponse::new(StatusCode::NOT_FOUND, Arc::from(&b"unexpected origin path"[..])) + })) + .await; + base.set(origin.base_url.clone()).map_err(|_| "origin already configured")?; + Ok(origin) +} + +pub(in crate::api::endpoints::hls_api::tests) async fn bounded_flussonic_hls_fixture( + archive_origin: BoundedArchiveOrigin, +) -> Result> { + use shared::model::{ConfigInputOptionsDto, FlussonicHlsCatchup, ProxyType}; + use tuliprox_core::model::ConfigInputOptions; + use tuliprox_repository::{ensure_target_storage_path, m3u_write_playlist}; + + let temp = tempfile::tempdir()?; + let requests = Arc::new(std::sync::Mutex::new(Vec::new())); + let origin = bounded_flussonic_origin(archive_origin, &requests).await?; + let options: ConfigInputOptionsDto = serde_json::from_str(r#"{"flussonic_hls_catchup":"bounded_archive"}"#)?; + assert_eq!(options.flussonic_hls_catchup, FlussonicHlsCatchup::BoundedArchive); + let input = ConfigInput { + id: 1, + name: Arc::from("bounded-flussonic"), + input_type: InputType::M3u, + url: format!("{}/input.m3u", origin.base_url), + enabled: true, + options: Some(ConfigInputOptions::from(&options)), + ..ConfigInput::default() + }; + let mut target = test_m3u_hls_share_target(); + target.name = "default".to_owned(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + let mut config = (*app_state.app_config.config.load_full()).clone(); + config.storage_dir = temp.path().to_string_lossy().into_owned(); + config.custom_stream_response_enabled = false; + app_state.app_config.config.store(Arc::new(config)); + configure_default_test_server(&app_state); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_owned(); + user.password = "hls-pass".to_owned(); + user.proxy = ProxyType::Reverse(None); + let mut proxy = app_state.app_config.api_proxy.load_full().ok_or("api proxy")?.as_ref().clone(); + proxy.user = vec![TargetUser { target: target.name.clone(), credentials: vec![Arc::new(user)] }]; + app_state.app_config.api_proxy.store(Some(Arc::new(proxy))); + let (mut groups, errors) = crate::iptv::m3u::download_m3u_playlist( + &app_state.app_config, + &reqwest::Client::new(), + &app_state.app_config.config.load_full(), + &input, + ) + .await; + assert_eq!(errors.len(), 0, "import errors: {errors:?}"); + let item = groups.first_mut().and_then(|group| group.channels.first_mut()).ok_or("imported item")?; + item.header.virtual_id = VirtualId::new(12345); + assert_eq!(item.header.url.as_ref(), format!("{}/channel/mono.m3u8?token=a%2Fb", origin.base_url)); + let target_path = ensure_target_storage_path(&app_state.app_config.config.load(), &target.name).await?; + m3u_write_playlist( + &app_state.app_config, + &target, + target.get_m3u_output().ok_or("M3U output")?, + &target_path, + &groups, + false, + ) + .await?; + let response = bounded_flussonic_request( + Arc::clone(&app_state), + "http://proxy/get.php?username=hls-user&password=hls-pass&type=m3u_plus", + ) + .await?; + assert_eq!(response.status(), StatusCode::OK); + let export = String::from_utf8(response_body(response).await.to_vec())?; + let catchup_template = export + .split("catchup-source=\"") + .nth(1) + .and_then(|tail| tail.split('"').next()) + .ok_or("exported catchup source")? + .to_owned(); + assert!(export.contains("catchup=\"default\"")); + assert!(!export.contains("a%2Fb")); + let live_uri = + export.lines().find(|line| !line.starts_with('#') && !line.is_empty()).ok_or("exported live URL")?.to_owned(); + Ok(BoundedFlussonicFixture { _temp: temp, origin, app_state, requests, catchup_template, live_uri }) +} + +pub(in crate::api::endpoints::hls_api::tests) async fn bounded_flussonic_hls_entry( + fixture: &BoundedFlussonicFixture, + start: i64, + duration: i64, +) -> Result, Box> { + let url = + fixture.catchup_template.replace("{utc}", &start.to_string()).replace("{duration}", &duration.to_string()); + bounded_flussonic_request(Arc::clone(&fixture.app_state), &url).await +} + +#[tokio::test] +async fn bounded_flussonic_hls_preserves_master_child_segments_and_duration_identity( +) -> Result<(), Box> { + let fixture = bounded_flussonic_hls_fixture(BoundedArchiveOrigin::Media).await?; + let mut identities = std::collections::HashSet::new(); + for (start, duration) in [(1_784_898_000, 3600), (1_784_898_000, 7200), (1_784_898_600, 3600)] { + let entry = bounded_flussonic_hls_entry(&fixture, start, duration).await?; + assert_eq!(entry.status(), StatusCode::OK); + let (_, uri) = single_variant_master_playlist(entry).await; + let proxy_id = ProxySessionId(proxy_session_id_from_variant_uri(&uri).to_owned()); + assert!(identities.insert(proxy_id.clone())); + let lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&uri).to_owned()); + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot(&lease_id, &proxy_id, super::super::current_time_millis()) + .await + .ok_or("lease")?; + let archive = format!("archive-{start}-{duration}"); + let expected_origin = format!("{}/channel/{archive}.m3u8?token=a%2Fb", fixture.origin.base_url); + assert_eq!(lease.archive_origin_url.as_deref(), Some(expected_origin.as_str())); + assert!(lease.user_session_token.contains(&format!("|archive|{start}|{duration}|hls-cache|"))); + let media = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(media.status(), StatusCode::OK); + let body = String::from_utf8(response_body(media).await.to_vec())?; + let segment_uri = + body.lines().find(|line| line.starts_with("/hls/") && path_has_extension(line, "ts")).ok_or("segment")?; + let segment = get_response(Arc::clone(&fixture.app_state), segment_uri, None).await; + assert_eq!(segment.status(), StatusCode::OK); + assert_ne!(response_body(segment).await, bytes::Bytes::new()); + let reload = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(reload.status(), StatusCode::OK); + let reload_body = String::from_utf8(response_body(reload).await.to_vec())?; + assert_eq!(manifest_media_sequence(&body), manifest_media_sequence(&reload_body)); + let paths = fixture.requests.lock().map_err(|_| "request log")?; + assert!(paths.iter().any(|p| p == &format!("/channel/{archive}.m3u8?token=a%2Fb"))); + assert!(paths.iter().any(|p| p.starts_with(&format!("/channel/{archive}/")) && p.ends_with(".ts?token=a%2Fb"))); + assert!(!paths.iter().any(|p| p.contains("timeshift_abs") || p.contains("mono.m3u8"))); + } + Ok(()) +} + +#[tokio::test] +async fn bounded_flussonic_hls_resolves_single_variant_archive_master() -> Result<(), Box> { + let fixture = bounded_flussonic_hls_fixture(BoundedArchiveOrigin::SingleVariantMaster).await?; + let entry = bounded_flussonic_hls_entry(&fixture, 1_784_898_000, 3600).await?; + assert_eq!(entry.status(), StatusCode::OK); + let (_, uri) = single_variant_master_playlist(entry).await; + let media = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(media.status(), StatusCode::OK); + let body = String::from_utf8(response_body(media).await.to_vec())?; + let segment_uri = + body.lines().find(|line| line.starts_with("/hls/") && path_has_extension(line, "ts")).ok_or("segment")?; + let segment = get_response(Arc::clone(&fixture.app_state), segment_uri, None).await; + assert_eq!(segment.status(), StatusCode::OK); + assert_ne!(response_body(segment).await, bytes::Bytes::new()); + let reload = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(reload.status(), StatusCode::OK); + let reload_body = String::from_utf8(response_body(reload).await.to_vec())?; + assert_eq!(manifest_media_sequence(&body), manifest_media_sequence(&reload_body)); + let paths = fixture.requests.lock().map_err(|_| "request log")?; + assert!(paths.iter().any(|p| p == "/channel/archive-1784898000-3600.m3u8?token=a%2Fb")); + assert!(paths.iter().any(|p| p == "/channel/tracks-v1/archive-1784898000-3600.ts.m3u8?token=a%2Fb")); + assert!(paths + .iter() + .any(|p| p.starts_with("/channel/tracks-v1/archive-1784898000-3600/") && p.ends_with(".ts?token=a%2Fb"))); + assert!(!paths.iter().any(|p| p.contains("timeshift_abs") || p.contains("mono.m3u8"))); + Ok(()) +} + +#[tokio::test] +async fn bounded_flussonic_hls_404_is_unavailable_without_live_or_timeshift_fallback( +) -> Result<(), Box> { + let fixture = bounded_flussonic_hls_fixture(BoundedArchiveOrigin::Missing).await?; + let entry = bounded_flussonic_hls_entry(&fixture, 1_784_898_000, 3600).await?; + assert_eq!(entry.status(), StatusCode::OK); + let (_, uri) = single_variant_master_playlist(entry).await; + let response = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!(!response.headers().contains_key(header::LOCATION)); + { + let paths = fixture.requests.lock().map_err(|_| "request log")?; + assert!(paths.iter().any(|p| p == "/channel/archive-1784898000-3600.m3u8?token=a%2Fb")); + assert!(!paths.iter().any(|p| p.contains("timeshift_abs") || p.contains("mono.m3u8"))); + } + let live = bounded_flussonic_request(Arc::clone(&fixture.app_state), &fixture.live_uri).await?; + assert_eq!(live.status(), StatusCode::OK); + let (_, live_manifest) = single_variant_master_playlist(live).await; + let media = get_response(Arc::clone(&fixture.app_state), &live_manifest, None).await; + assert_eq!(media.status(), StatusCode::OK); + let body = String::from_utf8(response_body(media).await.to_vec())?; + let segment_uri = + body.lines().find(|line| line.starts_with("/hls/") && path_has_extension(line, "ts")).ok_or("live segment")?; + let segment = get_response(Arc::clone(&fixture.app_state), segment_uri, None).await; + assert_eq!(segment.status(), StatusCode::OK); + assert_ne!(response_body(segment).await, bytes::Bytes::new()); + let paths = fixture.requests.lock().map_err(|_| "request log")?; + assert!(paths.iter().any(|p| p == "/channel/mono.m3u8?token=a%2Fb")); + assert!(paths.iter().any(|p| p.starts_with("/channel/live/") && p.ends_with(".ts?token=a%2Fb"))); + assert!(!paths.iter().any(|p| p.contains("timeshift_abs"))); + Ok(()) +} + +#[test] +fn cache_enabled_legacy_hls_route_only_allows_existing_m3u_catchup_session() { + assert!(super::super::legacy_hls_route_allowed_with_cache( + true, + Some("m3u-catchup|session"), + Some("m3u-catchup|session") + )); + assert!(super::super::legacy_hls_route_allowed_with_cache(true, Some("catchup|session"), Some("catchup|session"))); + assert!(!super::super::legacy_hls_route_allowed_with_cache( + true, + Some("m3u-catchup|session"), + Some("m3u-catchup|other") + )); + assert!(!super::super::legacy_hls_route_allowed_with_cache(true, Some("legacy-session"), Some("legacy-session"))); + assert!(!super::super::legacy_hls_route_allowed_with_cache(true, None, None)); + assert!(super::super::legacy_hls_route_allowed_with_cache(false, None, None)); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/bandwidth.rs b/backend/app/src/api/endpoints/hls_api/tests/bandwidth.rs new file mode 100644 index 000000000..5883cfa25 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/bandwidth.rs @@ -0,0 +1,477 @@ +use super::{ + access_lease_id_from_variant_uri, enable_hls_cache, normal_manifest, proxy_session_id_from_variant_uri, + response_body, single_variant_master_playlist, test_addr_with_port, test_app_state, test_fingerprint, + test_fingerprint_with_addr, test_m3u_hls_item, +}; +use crate::{ + api::model::{ + AppState, ConnectionKind, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, HlsBandwidthPersistenceState, + HlsPlaybackFamilyKey, HlsSessionHandle, ProxySessionId, SegmentCacheStatus, + }, + model::{Config, ConfigInput, ProxyUserCredentials, StripConfig}, +}; +use axum::http::{header, StatusCode}; +use shared::model::{HlsStripMode, InputType, StreamProperties, UserConnectionPermission}; +use std::{sync::Arc, time::Duration}; + +#[derive(Clone, Copy)] +pub(in crate::api::endpoints::hls_api::tests) enum TestLiveBitrateRepositoryState { + MissingDatabase, + MissingStreamItem, + ExistingHigher, + Update, + PermanentlyInapplicable, + RepositoryIoError, +} + +impl TestLiveBitrateRepositoryState { + pub(in crate::api::endpoints::hls_api::tests) const fn input_name(self) -> &'static str { + match self { + Self::MissingDatabase => "bandwidth-missing-database", + Self::MissingStreamItem => "bandwidth-missing-item", + Self::ExistingHigher => "bandwidth-existing-higher", + Self::Update => "bandwidth-update", + Self::PermanentlyInapplicable => "bandwidth-inapplicable", + Self::RepositoryIoError => "bandwidth-io-error", + } + } +} + +pub(in crate::api::endpoints::hls_api::tests) fn prepare_test_live_bitrate_repository( + input: &ConfigInput, + storage_root: &std::path::Path, + repository_state: TestLiveBitrateRepositoryState, +) { + let storage_path = + crate::repository::build_input_storage_path(&input.name, storage_root.to_string_lossy().as_ref()); + let database_path = crate::repository::get_input_m3u_playlist_file_path(&storage_path, &input.name); + match repository_state { + TestLiveBitrateRepositoryState::MissingDatabase | TestLiveBitrateRepositoryState::PermanentlyInapplicable => {} + TestLiveBitrateRepositoryState::RepositoryIoError => { + std::fs::create_dir_all(&storage_path).expect("input storage"); + std::fs::write(&database_path, b"invalid btree data").expect("corrupt input tree"); + } + TestLiveBitrateRepositoryState::MissingStreamItem + | TestLiveBitrateRepositoryState::ExistingHigher + | TestLiveBitrateRepositoryState::Update => { + std::fs::create_dir_all(&storage_path).expect("input storage"); + let (stream_ref, stored_bitrate) = match repository_state { + TestLiveBitrateRepositoryState::MissingStreamItem => ("different-channel", 0), + TestLiveBitrateRepositoryState::ExistingHigher => ("channel-a", 3_000_000), + TestLiveBitrateRepositoryState::Update => ("channel-a", 0), + TestLiveBitrateRepositoryState::MissingDatabase + | TestLiveBitrateRepositoryState::PermanentlyInapplicable + | TestLiveBitrateRepositoryState::RepositoryIoError => unreachable!(), + }; + let mut item = test_m3u_hls_item(input, 12345, stream_ref, "http://origin.test/live.m3u8"); + item.additional_properties = Some(StreamProperties::Live(Box::new(shared::model::LiveStreamProperties { + bitrate: stored_bitrate, + ..Default::default() + }))); + let mut tree = crate::repository::BPlusTree::new(); + tree.insert(Arc::clone(&item.provider_id), item); + tree.store(&database_path).expect("input tree"); + } + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn prepare_runtime_bandwidth_session( + app_state: &Arc, + input: &ConfigInput, + manifest_rendered_at_ms: u64, +) -> HlsSessionHandle { + let origin_source = super::super::build_hls_origin_source(input, "channel-a"); + let (session, _) = app_state + .hls + .proxy + .get_or_create_session_with_source_and_outcome( + origin_source.session_key(), + origin_source, + &app_state.get_encrypt_secret(), + manifest_rendered_at_ms, + ) + .await; + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ + #EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n", + ); + { + let mut session_guard = session.write().await; + session_guard.apply_origin_manifest(&manifest).expect("runtime learning timeline"); + for entry in session_guard.segments.values_mut() { + entry.status = + SegmentCacheStatus::Ready { content_length: 1_000_000, ready_at_ms: manifest_rendered_at_ms }; + } + session_guard.advance_media_readiness_generation(); + session_guard.render_and_store_manifest(manifest_rendered_at_ms).expect("runtime learning manifest"); + session_guard.mark_authorized_media_access(manifest_rendered_at_ms); + } + session +} + +pub(in crate::api::endpoints::hls_api::tests) async fn hls_runtime_bandwidth_manifest_case( + repository_state: TestLiveBitrateRepositoryState, +) -> (HlsBandwidthPersistenceState, Option) { + let temp = tempfile::tempdir().expect("temp dir"); + let app_state = test_app_state(); + let current_config = app_state.app_config.config.load(); + app_state.app_config.config.store(Arc::new(Config { + storage_dir: temp.path().to_string_lossy().into_owned(), + ..current_config.as_ref().clone() + })); + let input = ConfigInput { + id: 7, + name: Arc::from(repository_state.input_name()), + input_type: if matches!(repository_state, TestLiveBitrateRepositoryState::PermanentlyInapplicable) { + InputType::Library + } else { + InputType::M3u + }, + ..ConfigInput::default() + }; + prepare_test_live_bitrate_repository(&input, temp.path(), repository_state); + let manifest_rendered_at_ms = super::super::current_time_millis(); + let session = prepare_runtime_bandwidth_session(&app_state, &input, manifest_rendered_at_ms).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let access_lease_id = HlsAccessLeaseId(format!("{}-lease", repository_state.input_name())); + let now_ms = super::super::current_time_millis(); + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + access_lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), + proxy_session_id, + "hls-user".to_string(), + "hls-session-token".to_string(), + input.id, + "channel-a".to_string(), + 12345, + now_ms, + 60_000, + )) + .await; + + let response = super::super::try_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &StripConfig { mode: HlsStripMode::Segments, value: 0 }, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + super::super::HlsRuntimeBandwidthLearningContext::Eligible(&input), + ) + .await + .expect("cached media manifest response"); + assert_eq!(response.status(), StatusCode::OK); + assert!(!response_body(response).await.is_empty()); + + let bandwidth_persistence = tokio::time::timeout(Duration::from_secs(10), async { + loop { + let state = session.read().await.bandwidth_persistence; + if !matches!(state, HlsBandwidthPersistenceState::Idle | HlsBandwidthPersistenceState::InFlight { .. }) { + break state; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("bandwidth persistence completion"); + let stored_bitrate = if matches!( + repository_state, + TestLiveBitrateRepositoryState::ExistingHigher | TestLiveBitrateRepositoryState::Update + ) { + crate::repository::load_input_live_bitrate_bps(&app_state.app_config, &input, "channel-a") + .await + .expect("stored bitrate read") + } else { + None + }; + (bandwidth_persistence, stored_bitrate) +} + +#[tokio::test] +async fn hls_runtime_bandwidth_missing_database_retries_without_failing_media_manifest() { + let (state, _) = hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::MissingDatabase).await; + + assert!(matches!(state, HlsBandwidthPersistenceState::RetryAfter { .. })); +} + +#[tokio::test] +async fn hls_runtime_bandwidth_missing_item_retries_without_failing_media_manifest() { + let (state, _) = hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::MissingStreamItem).await; + + assert!(matches!(state, HlsBandwidthPersistenceState::RetryAfter { .. })); +} + +#[tokio::test] +async fn hls_runtime_bandwidth_existing_higher_completes_without_failing_media_manifest() { + let (state, stored_bitrate) = + hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::ExistingHigher).await; + + assert!(matches!(state, HlsBandwidthPersistenceState::Persisted { bitrate_bps: 2_000_000 })); + assert_eq!(stored_bitrate, Some(3_000_000)); +} + +#[tokio::test] +async fn hls_runtime_bandwidth_update_completes_without_failing_media_manifest() { + let (state, stored_bitrate) = hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::Update).await; + + assert!(matches!(state, HlsBandwidthPersistenceState::Persisted { bitrate_bps: 2_000_000 })); + assert_eq!(stored_bitrate, Some(2_000_000)); +} + +#[tokio::test] +async fn hls_runtime_bandwidth_inapplicable_and_io_error_do_not_fail_media_manifest() { + let (inapplicable, _) = + hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::PermanentlyInapplicable).await; + let (io_error, _) = hls_runtime_bandwidth_manifest_case(TestLiveBitrateRepositoryState::RepositoryIoError).await; + + assert!(matches!(inapplicable, HlsBandwidthPersistenceState::PermanentlyInapplicable { bitrate_bps: 2_000_000 })); + assert!(matches!(io_error, HlsBandwidthPersistenceState::RetryAfter { .. })); +} + +#[tokio::test] +async fn hls_runtime_bandwidth_persistence_is_entry_gated_and_deduplicated() { + let temp = tempfile::tempdir().expect("temp dir"); + let app_state = test_app_state(); + let current_config = app_state.app_config.config.load(); + app_state.app_config.config.store(Arc::new(Config { + storage_dir: temp.path().to_string_lossy().into_owned(), + ..current_config.as_ref().clone() + })); + let input = ConfigInput { + id: 7, + name: Arc::from("runtime-bandwidth-input"), + input_type: InputType::M3u, + ..ConfigInput::default() + }; + let storage_path = crate::repository::build_input_storage_path(&input.name, temp.path().to_string_lossy().as_ref()); + std::fs::create_dir_all(&storage_path).expect("input storage"); + let item = test_m3u_hls_item(&input, 12345, "channel-a", "http://origin.test/live.m3u8"); + let mut tree = crate::repository::BPlusTree::new(); + tree.insert(Arc::clone(&item.provider_id), item); + tree.store(&crate::repository::get_input_m3u_playlist_file_path(&storage_path, &input.name)).expect("input tree"); + + let origin_source = super::super::build_hls_origin_source(&input, "channel-a"); + let (session, _) = app_state + .hls + .proxy + .get_or_create_session_with_source_and_outcome( + origin_source.session_key(), + origin_source, + &app_state.get_encrypt_secret(), + 1, + ) + .await; + { + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ + #EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n", + ); + let mut session_guard = session.write().await; + session_guard.apply_origin_manifest(&manifest).expect("runtime learning timeline"); + for entry in session_guard.segments.values_mut() { + entry.status = SegmentCacheStatus::Ready { content_length: 1_000_000, ready_at_ms: 2 }; + } + } + + assert!(super::super::spawn_hls_runtime_bandwidth_persistence( + &app_state, + &session, + super::super::HlsRuntimeBandwidthLearningContext::Disabled, + ) + .is_none()); + assert_eq!( + crate::repository::load_input_live_bitrate_bps(&app_state.app_config, &input, "channel-a") + .await + .expect("unknown bitrate read"), + None + ); + + let task = super::super::spawn_hls_runtime_bandwidth_persistence( + &app_state, + &session, + super::super::HlsRuntimeBandwidthLearningContext::Eligible(&input), + ) + .expect("runtime persistence task"); + task.await.expect("runtime persistence completion"); + + assert_eq!( + crate::repository::load_input_live_bitrate_bps(&app_state.app_config, &input, "channel-a") + .await + .expect("persisted bitrate read"), + Some(2_000_000) + ); + assert!(super::super::spawn_hls_runtime_bandwidth_persistence( + &app_state, + &session, + super::super::HlsRuntimeBandwidthLearningContext::Eligible(&input), + ) + .is_none()); +} + +#[test] +fn hls_master_playlist_response_diagnostic_reports_complete_response_contract_for_every_bandwidth_source() { + for selection in [ + super::super::HlsMasterBandwidthSelection::resolve(Some(2_500_000), Some(2_000_000)), + super::super::HlsMasterBandwidthSelection::resolve(None, Some(2_000_000)), + super::super::HlsMasterBandwidthSelection::resolve(None, None), + ] { + let rendered = super::super::hls_entry_master_playlist_response( + &ProxySessionId("diagnostic-proxy-session".to_string()), + &HlsAccessLeaseId("diagnostic-lease".to_string()), + selection.bandwidth(), + Some("/iptv"), + ); + let fields = super::super::HlsMasterPlaylistResponseDiagnostic { + lease: "lease-safe".to_string(), + session: "session-safe".to_string(), + proxy_session: "proxy-safe".to_string(), + user_session: "user-safe".to_string(), + virtual_id: 12345, + bandwidth_bps: selection.bandwidth().advertised_bps(), + bandwidth_source: selection.source().as_log_value(), + content_length: rendered.content_length, + } + .to_string(); + + assert!(fields.contains("lease=lease-safe session=session-safe proxy_session=proxy-safe")); + assert!(fields.contains("user_session=user-safe virtual_id=12345")); + assert!(fields.contains(&format!( + "bandwidth_bps={} bandwidth_source={}", + selection.bandwidth().advertised_bps(), + selection.source().as_log_value() + ))); + assert!(fields.contains("status=200")); + assert!(fields.ends_with(&format!("content_length={}", rendered.content_length))); + assert_eq!( + rendered + .response + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()), + Some(rendered.content_length) + ); + } +} + +#[test] +fn hls_entry_stream_context_keeps_only_positive_live_bitrate() { + let input = ConfigInput { name: Arc::from("m3u-input"), ..ConfigInput::default() }; + let mut item = test_m3u_hls_item(&input, 1001, "channel-a", "http://origin.example.com/live.m3u8"); + item.additional_properties = Some(StreamProperties::Live(Box::new(shared::model::LiveStreamProperties { + bitrate: 2_500_000, + ..shared::model::LiveStreamProperties::default() + }))); + + let context = super::super::HlsEntryStreamContext::from_playlist_item(&item).expect("entry stream context"); + assert_eq!(context.virtual_id(), 1001); + assert_eq!(context.stream_ref(), "channel-a"); + assert_eq!(context.known_bitrate_bps(), Some(2_500_000)); + + if let Some(StreamProperties::Live(properties)) = item.additional_properties.as_mut() { + properties.bitrate = 0; + } + assert_eq!( + super::super::HlsEntryStreamContext::from_playlist_item(&item) + .expect("entry stream context without bitrate") + .known_bitrate_bps(), + None + ); +} + +#[tokio::test] +async fn hls_cache_entry_prefers_item_bitrate_then_loads_db_without_target_rebuild() { + let temp = tempfile::tempdir().expect("temp dir"); + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut config = app_state.app_config.config.load().as_ref().clone(); + config.storage_dir = temp.path().to_string_lossy().into_owned(); + app_state.app_config.config.store(Arc::new(config)); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = + ConfigInput { id: 9, name: Arc::from("m3u-input"), input_type: InputType::M3u, ..ConfigInput::default() }; + let mut stored_item = test_m3u_hls_item(&input, 70001, "channel-a", "http://origin.example.com/live.m3u8"); + stored_item.additional_properties = Some(StreamProperties::Live(Box::new(shared::model::LiveStreamProperties { + bitrate: 2_500_000, + ..shared::model::LiveStreamProperties::default() + }))); + let input_storage = crate::repository::build_input_storage_path(&input.name, &temp.path().to_string_lossy()); + std::fs::create_dir_all(&input_storage).expect("input storage"); + let db_path = crate::repository::get_input_m3u_playlist_file_path(&input_storage, &input.name); + let mut tree = crate::repository::BPlusTree::new(); + tree.insert(Arc::::from("channel-a"), stored_item); + tree.store(&db_path).expect("input M3U metadata"); + let request_url = "http://origin.example.com/live.m3u8"; + + let db_response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + super::super::build_hls_origin_source(&input, "channel-a"), + 70001, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + let (db_bandwidth, db_variant_uri) = single_variant_master_playlist(db_response).await; + assert_eq!(db_bandwidth, 3_000_000); + let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&db_variant_uri).to_string()); + let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&db_variant_uri).to_string()); + let lease = app_state + .hls + .proxy + .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::super::current_time_millis()) + .await + .expect("DB-backed access lease"); + assert_eq!(lease.known_bitrate_bps, Some(2_500_000)); + + std::fs::write(&db_path, b"invalid metadata tree").expect("corrupt test metadata"); + let item_response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint_with_addr(test_addr_with_port(55127)), + &user, + super::super::build_hls_origin_source(&input, "channel-a"), + 70001, + None, + Some(3_000_000), + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + let (item_bandwidth, _) = single_variant_master_playlist(item_response).await; + assert_eq!(item_bandwidth, 3_600_000); + + let fallback_response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint_with_addr(test_addr_with_port(55128)), + &user, + super::super::build_hls_origin_source(&input, "channel-a"), + 70001, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + let (fallback_bandwidth, _) = single_variant_master_playlist(fallback_response).await; + assert_eq!(fallback_bandwidth, 1_000_000); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/encrypted_media.rs b/backend/app/src/api/endpoints/hls_api/tests/encrypted_media.rs new file mode 100644 index 000000000..6875afd3e --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/encrypted_media.rs @@ -0,0 +1,554 @@ +use super::{ + create_active_hls_user_session, enable_hls_cache, encrypt_test_aes128_cbc_pkcs7, get_response, hls_proxy_uri, + map_transient_resource_with_kind, prepare_pending_test_hls_access_lease, response_body, + spawn_test_encrypted_hls_origin, spawn_test_transient_origin_with_response, terminal_test_asset, test_app_state, + test_app_state_with_hls_proxy_and_inputs, test_fingerprint, test_hls_access_context, test_hls_sequence_iv, + try_test_hls_cached_manifest_response, TestSegmentOrigin, AES_TEST_KEY_BYTES, AES_TEST_MANIFEST, + AES_TEST_PLAINTEXT_SEGMENT, +}; +use crate::{ + api::model::{ + build_proxy_session_id, build_terminal_tail_plan, prepare_terminal_base_evidence, AppState, HlsAccessLease, + HlsAccessLeaseId, HlsAccessLeaseState, HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, HlsLifecycleEvent, + HlsLifecycleEventKey, HlsManifestDeliveryMode, HlsPlaybackFamilyKey, HlsProxyManager, + HlsRuntimeCustomTailAssetIdentity, HlsSessionHandle, HlsSessionKey, HlsSessionMode, HlsTerminalAssetIdentity, + HlsTerminalMediaAsset, HlsTerminalTailBuildInput, HlsTerminalTailCompatibility, HlsTerminalTailGeneration, + HlsTerminalTailProtection, ProxySessionId, TransientObjectCacheKey, TransientResourceKind, + }, + model::{ConfigInput, StripConfig}, +}; +use axum::http::{header, HeaderMap, StatusCode}; +use shared::model::{HlsStripMode, InputType}; +use std::{sync::Arc, time::Duration}; + +pub(in crate::api::endpoints::hls_api::tests) const AES_TEST_ROTATED_KEY_BYTES: &[u8] = b"fedcba9876543210"; + +pub(in crate::api::endpoints::hls_api::tests) struct AesEndpointFixture { + pub(in crate::api::endpoints::hls_api::tests) _temp_dir: tempfile::TempDir, + pub(in crate::api::endpoints::hls_api::tests) origin: TestSegmentOrigin, + pub(in crate::api::endpoints::hls_api::tests) input: ConfigInput, + pub(in crate::api::endpoints::hls_api::tests) app_state: Arc, + pub(in crate::api::endpoints::hls_api::tests) request_url: String, + pub(in crate::api::endpoints::hls_api::tests) session: HlsSessionHandle, + pub(in crate::api::endpoints::hls_api::tests) proxy_session_id: ProxySessionId, + pub(in crate::api::endpoints::hls_api::tests) access_lease_id: HlsAccessLeaseId, + pub(in crate::api::endpoints::hls_api::tests) key_uri: String, + pub(in crate::api::endpoints::hls_api::tests) base_manifest: HlsLeaseManifestSnapshot, + pub(in crate::api::endpoints::hls_api::tests) asset: Arc, +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_aes_live_endpoint( + app_state: &Arc, + origin: &TestSegmentOrigin, + access_lease_id: &HlsAccessLeaseId, + live_body: &str, +) -> String { + let key_uri = live_body + .lines() + .find(|line| line.starts_with("#EXT-X-KEY:METHOD=AES-128")) + .and_then(|line| line.split_once("URI=\"").map(|(_, tail)| tail)) + .and_then(|tail| tail.split_once('"').map(|(uri, _)| uri.to_string())) + .expect("opaque key URI"); + assert!(key_uri.contains(&format!("/{}/r/", access_lease_id.0))); + assert!(!live_body.contains("key.bin")); + assert!(live_body.contains("#EXT-X-VERSION:5\n")); + let proxy_media_sequence = live_body + .lines() + .find_map(|line| line.strip_prefix("#EXT-X-MEDIA-SEQUENCE:")) + .and_then(|value| value.parse::().ok()) + .expect("proxy media sequence"); + assert_ne!(proxy_media_sequence, 77); + assert!(live_body.contains("IV=0x0000000000000000000000000000004d")); + assert_eq!(origin.key_request_count(), 1); + let segment_uris = live_body + .lines() + .filter(|line| line.starts_with("/hls/shared/live/") && !line.contains("/r/")) + .collect::>(); + assert_eq!(segment_uris.len(), 6); + for (index, segment_uri) in segment_uris.into_iter().enumerate() { + let response = get_response(Arc::clone(app_state), segment_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + let served = response_body(response).await; + let origin_sequence = 77_u64.saturating_add(u64::try_from(index).expect("test segment index")); + let expected = encrypt_test_aes128_cbc_pkcs7( + AES_TEST_PLAINTEXT_SEGMENT, + AES_TEST_KEY_BYTES, + test_hls_sequence_iv(origin_sequence), + ); + assert_eq!(served.as_ref(), expected.as_slice()); + assert_ne!(served.as_ref(), AES_TEST_PLAINTEXT_SEGMENT); + assert_eq!(origin.key_request_count(), 1); + } + key_uri +} + +pub(in crate::api::endpoints::hls_api::tests) async fn aes_endpoint_fixture() -> AesEndpointFixture { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let origin = spawn_test_encrypted_hls_origin( + AES_TEST_MANIFEST, + Arc::from(AES_TEST_KEY_BYTES), + Arc::from(AES_TEST_PLAINTEXT_SEGMENT), + ) + .await; + let input = ConfigInput { + id: 1, + name: Arc::from("encrypted-m3u-input"), + input_type: InputType::M3u, + url: origin.base_url.clone(), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_hls_proxy_and_inputs( + Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)), + vec![Arc::new(input.clone())], + ); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/channel/index.m3u8", origin.base_url); + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("encrypted-normal-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + let response = super::super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::super::HlsCacheManifestOrigin { + raw_request_url: &request_url, + session_entry_url: super::super::HlsOriginEntryUrl::direct_http(&request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/m3u-stream/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + .expect("compatible AES origin should enter normal HLS cache"); + assert_eq!(response.status(), StatusCode::OK); + let live_body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + let key_uri = assert_aes_live_endpoint(&app_state, &origin, &access_lease_id, &live_body).await; + assert_eq!(origin.segment_request_count(), 6); + let session = app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("normal encrypted session"); + assert_eq!(session.read().await.mode, HlsSessionMode::NormalCacheTimeline); + let lease = app_state + .hls + .proxy + .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::super::current_time_millis()) + .await + .expect("live encrypted lease snapshot"); + let base_manifest = lease.last_manifest_snapshot.expect("authoritative lease manifest"); + assert_eq!(base_manifest.delivery_mode, HlsManifestDeliveryMode::NormalCacheTimeline); + assert_eq!( + base_manifest.active_encryption.as_ref().and_then(|encryption| encryption.iv.as_deref()), + Some("0x00000000000000000000000000000052") + ); + let asset = terminal_test_asset(); + let evidence = prepare_terminal_base_evidence( + &session, + app_state.hls.proxy.segment_cache(), + &base_manifest, + super::super::current_time_millis(), + ) + .await; + assert_eq!(evidence.track_signature(), Some(asset.track_signature().clone())); + assert_eq!(evidence.key_bindings().len(), 1); + assert_eq!(origin.key_request_count(), 1); + evidence.release(); + AesEndpointFixture { + _temp_dir: temp_dir, + origin, + input, + app_state, + request_url, + session, + proxy_session_id, + access_lease_id, + key_uri, + base_manifest, + asset, + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn install_aes_terminal_plan( + fixture: &AesEndpointFixture, +) -> TransientObjectCacheKey { + let key_response = get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, None).await; + assert_eq!(key_response.status(), StatusCode::OK); + assert_eq!(response_body(key_response).await.as_ref(), AES_TEST_KEY_BYTES); + assert_eq!(fixture.origin.key_request_count(), 1); + let evidence_key_id = { + let mut session = fixture.session.write().await; + let key_id = session + .transient + .resources + .values() + .find(|resource| resource.kind == TransientResourceKind::Key) + .map(|resource| resource.id.clone()) + .expect("normal key resource"); + session.last_rendered_manifest = None; + session.transient.last_manifest_body = None; + session.transient.resources.get_mut(&key_id).expect("key resource").expires_at_ms = 0; + for (key, object) in &mut session.transient.object_cache { + if key.transient_resource_id() == &key_id { + object.expires_at_ms = 0; + } + } + key_id + }; + let evidence = prepare_terminal_base_evidence( + &fixture.session, + fixture.app_state.hls.proxy.segment_cache(), + &fixture.base_manifest, + super::super::current_time_millis(), + ) + .await; + assert_eq!(evidence.track_signature(), Some(fixture.asset.track_signature().clone())); + assert_eq!(fixture.origin.key_request_count(), 1); + fixture + .app_state + .hls + .proxy + .run_garbage_collection_once(super::super::current_time_millis()) + .await + .expect("evidence-pinned GC"); + { + let session = fixture.session.read().await; + assert!(session.transient.resources.contains_key(&evidence_key_id)); + assert!(session.transient.object_cache.keys().any(|key| key.transient_resource_id() == &evidence_key_id)); + } + let plan = build_terminal_tail_plan(HlsTerminalTailBuildInput { + generation: HlsTerminalTailGeneration(23), + created_at_ms: super::super::current_time_millis(), + base_availability: evidence.availability(), + base_track_signature: evidence.track_signature(), + base_splice_evidence: evidence.splice_evidence().cloned(), + terminal_splice_evidence: Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&fixture.asset)), + base_timing: evidence.timing().cloned(), + base_key_bindings: evidence.key_bindings(), + expected_asset: HlsRuntimeCustomTailAssetIdentity::channel_unavailable(HlsTerminalAssetIdentity::from_asset( + &fixture.asset, + )), + base_manifest: fixture.base_manifest.clone(), + anchored_bundle: HlsTerminalTailBuildInput::anchored_bundle_for_test( + &fixture.asset, + fixture.base_manifest.target_duration_ms, + ), + asset: Arc::clone(&fixture.asset), + }) + .expect("READY AES key permits safe terminal reset"); + let protection = HlsTerminalTailProtection { + generation: plan.generation, + base_proxy_seqs: Arc::clone(&plan.protected_base_proxy_seqs), + key_bindings: plan.key_bindings(), + }; + assert_eq!(protection.key_bindings.len(), 1); + assert_eq!(protection.key_bindings[0].resource_id(), &evidence_key_id); + let frozen_source_cache_key = protection.key_bindings[0].source_cache_key().clone(); + { + let mut leases = fixture.app_state.hls.proxy.access_leases().write().await; + let mut lease = leases.remove_access_lease(&fixture.access_lease_id).expect("live lease"); + lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(Arc::new(plan)); + leases.prepare_access_lease(lease); + } + fixture.session.write().await.install_terminal_tail_protection(fixture.access_lease_id.clone(), protection); + evidence.release(); + frozen_source_cache_key +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_aes_terminal_endpoints(fixture: &AesEndpointFixture) { + let manifest_uri = + format!("/hls/shared/live/{}/{}/manifest.m3u8", fixture.proxy_session_id.0, fixture.access_lease_id.0); + let response = get_response(Arc::clone(&fixture.app_state), &manifest_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("terminal utf8"); + let key = body.find("#EXT-X-KEY:METHOD=AES-128").expect("base AES key"); + let reset = body.find("#EXT-X-KEY:METHOD=NONE").expect("clear key reset"); + let discontinuity = body[reset..].find("#EXT-X-DISCONTINUITY").expect("terminal discontinuity") + reset; + assert!(key < reset && reset < discontinuity); + assert!(body.contains("IV=0x00000000000000000000000000000052")); + assert!(body.contains("#EXT-X-VERSION:5\n")); + assert!(body.ends_with("#EXT-X-ENDLIST\n")); + let key_response = get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, None).await; + assert_eq!(key_response.status(), StatusCode::OK); + assert_eq!(response_body(key_response).await.as_ref(), AES_TEST_KEY_BYTES); + assert_eq!(fixture.origin.key_request_count(), 1); + let range = get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, Some("bytes=4-7")).await; + assert_eq!(range.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response_body(range).await.as_ref(), &AES_TEST_KEY_BYTES[4..=7]); + let unsatisfiable = get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, Some("bytes=16-")).await; + assert_eq!(unsatisfiable.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert_eq!(unsatisfiable.headers()[header::CONTENT_RANGE], "bytes */16"); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_aes_rotated_live_key( + fixture: &AesEndpointFixture, + frozen_source_cache_key: &TransientObjectCacheKey, +) -> String { + let manifest_requests_before = fixture.origin.manifest_request_count(); + let segment_requests_before = fixture.origin.segment_request_count(); + fixture.origin.set_key_bytes(Arc::from(AES_TEST_ROTATED_KEY_BYTES)).await; + let live_lease_id = HlsAccessLeaseId("encrypted-rotated-live-lease".to_string()); + let access_context = test_hls_access_context(fixture.proxy_session_id.clone(), live_lease_id.clone()); + prepare_pending_test_hls_access_lease(&fixture.app_state, &fixture.proxy_session_id, &live_lease_id).await; + let response = super::super::try_hls_cache_canonical_manifest_response( + &fixture.app_state, + &test_fingerprint(), + &access_context, + &fixture.proxy_session_id, + &live_lease_id, + HlsAccessLeaseState::Pending, + super::super::HlsCacheManifestOrigin { + raw_request_url: &fixture.request_url, + session_entry_url: super::super::HlsOriginEntryUrl::direct_http(&fixture.request_url), + input: &fixture.input, + origin_source: super::super::build_hls_origin_source(&fixture.input, "12345"), + }, + HeaderMap::new(), + None, + "/m3u-stream/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + .expect("a new live lease should reuse the recovered shared session"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(fixture.origin.manifest_request_count(), manifest_requests_before.saturating_add(1)); + assert_eq!(fixture.origin.segment_request_count(), segment_requests_before); + let live_lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot(&live_lease_id, &fixture.proxy_session_id, super::super::current_time_millis()) + .await + .expect("rotated live lease snapshot"); + assert_eq!(live_lease.playback_mode, HlsLeasePlaybackMode::Live); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("rotated manifest utf8"); + let rotated_key_uri = body + .lines() + .find(|line| line.starts_with("#EXT-X-KEY:METHOD=AES-128")) + .and_then(|line| line.split_once("URI=\"").map(|(_, tail)| tail)) + .and_then(|tail| tail.split_once('"').map(|(uri, _)| uri.to_string())) + .expect("rotated live key URI"); + let key_response = get_response(Arc::clone(&fixture.app_state), &rotated_key_uri, None).await; + assert_eq!(key_response.status(), StatusCode::OK); + assert_eq!(response_body(key_response).await.as_ref(), AES_TEST_ROTATED_KEY_BYTES); + assert_eq!(fixture.origin.key_request_count(), 2); + assert!(fixture + .app_state + .hls + .proxy + .segment_cache() + .metadata(frozen_source_cache_key) + .await + .expect("A metadata lookup") + .is_none()); + let terminal_key = get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, None).await; + assert_eq!(terminal_key.status(), StatusCode::OK); + assert_eq!(response_body(terminal_key).await.as_ref(), AES_TEST_KEY_BYTES); + assert_eq!(fixture.origin.key_request_count(), 2); + rotated_key_uri +} + +pub(in crate::api::endpoints::hls_api::tests) async fn expire_aes_terminal_lease( + fixture: &AesEndpointFixture, + rotated_key_uri: &str, +) { + let expired_at_ms = super::super::current_time_millis().saturating_sub(1); + { + let mut leases = fixture.app_state.hls.proxy.access_leases().write().await; + let mut lease = leases.remove_access_lease(&fixture.access_lease_id).expect("terminal lease"); + lease.valid_until_ms = expired_at_ms; + leases.prepare_access_lease(lease); + } + fixture + .app_state + .hls + .proxy + .handle_lifecycle_event( + &fixture.app_state.active_users, + &fixture.app_state.active_provider, + HlsLifecycleEvent { + key: HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: fixture.access_lease_id.clone(), + proxy_session_id: fixture.proxy_session_id.clone(), + }, + due_at_ms: expired_at_ms, + }, + super::super::current_time_millis(), + ) + .await; + fixture + .app_state + .hls + .proxy + .run_garbage_collection_once(super::super::current_time_millis()) + .await + .expect("released GC"); + assert!(fixture.session.read().await.terminal_tail_protection(&fixture.access_lease_id).is_none()); + assert_eq!( + get_response(Arc::clone(&fixture.app_state), &fixture.key_uri, None).await.status(), + StatusCode::NOT_FOUND + ); + let live_key = get_response(Arc::clone(&fixture.app_state), rotated_key_uri, None).await; + assert_eq!(live_key.status(), StatusCode::OK); + assert_eq!(response_body(live_key).await.as_ref(), AES_TEST_ROTATED_KEY_BYTES); +} + +#[tokio::test] +async fn aes_128_normal_origin_key_and_terminal_lifecycle_are_endpoint_safe() { + let fixture = aes_endpoint_fixture().await; + let frozen_source_cache_key = install_aes_terminal_plan(&fixture).await; + assert_aes_terminal_endpoints(&fixture).await; + let rotated_key_uri = assert_aes_rotated_live_key(&fixture, &frozen_source_cache_key).await; + expire_aes_terminal_lease(&fixture, &rotated_key_uri).await; +} + +pub(in crate::api::endpoints::hls_api::tests) fn assert_encrypted_transient_terminal_incompatibility( + snapshot: HlsLeaseManifestSnapshot, + now_ms: u64, +) { + let asset = terminal_test_asset(); + let bundle_target_duration_ms = snapshot.target_duration_ms.max(asset.duration_ms()); + let base_timing = Some(HlsTerminalTailBuildInput::base_timing_for_test(&asset, &snapshot)); + let base_splice_evidence = Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); + let terminal_splice_evidence = base_splice_evidence.clone(); + assert_eq!( + build_terminal_tail_plan(HlsTerminalTailBuildInput { + generation: HlsTerminalTailGeneration(1), + created_at_ms: now_ms, + base_availability: Arc::from([]), + base_track_signature: Some(asset.track_signature().clone()), + base_splice_evidence, + terminal_splice_evidence, + base_timing, + base_key_bindings: Arc::from([]), + expected_asset: HlsRuntimeCustomTailAssetIdentity::channel_unavailable( + HlsTerminalAssetIdentity::from_asset(&asset), + ), + anchored_bundle: HlsTerminalTailBuildInput::anchored_bundle_for_test(&asset, bundle_target_duration_ms,), + base_manifest: snapshot, + asset, + }), + Err(HlsTerminalTailCompatibility::TransientPassthroughUnsupported) + ); +} + +#[tokio::test] +async fn encrypted_transient_endpoint_stores_client_visible_key_and_typed_terminal_incompatibility() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let now_ms = super::super::current_time_millis(); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), now_ms) + .await; + let (proxy_session_id, access_lease_id) = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.clone(); + let access_lease_id = HlsAccessLeaseId("encrypted-access-lease".to_string()); + let body = format!( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:20\n\ + #EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/{}/{}/r/key.bin\",IV=0x1,KEYFORMAT=\"identity\",KEYFORMATVERSIONS=\"1\"\n\ + #EXTINF:4.0,\n/hls/shared/live/{}/{}/r/20.ts\n\ + #EXTINF:4.0,\n/hls/shared/live/{}/{}/r/21.ts\n\ + #EXTINF:4.0,\n/hls/shared/live/{}/{}/r/22.ts\n", + proxy_session_id.0, + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER, + proxy_session_id.0, + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER, + proxy_session_id.0, + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER, + proxy_session_id.0, + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER, + ); + session.transient.replace_manifest_with_semantics(body, now_ms, Some(12_000)); + session.mark_authorized_media_access(now_ms); + (proxy_session_id, access_lease_id) + }; + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + access_lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", "encrypted-client"), + proxy_session_id.clone(), + "hls-user".to_string(), + "encrypted-session".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + 60_000, + )) + .await; + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &StripConfig { mode: HlsStripMode::Segments, value: 0 }, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("encrypted transient response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + let lease = app_state + .hls + .proxy + .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms) + .await + .expect("lease snapshot"); + let snapshot = lease.last_manifest_snapshot.expect("manifest snapshot"); + let encryption = snapshot.active_encryption.as_ref().expect("active encryption"); + + assert!(body.contains(&format!("URI=\"/hls/shared/live/{}/{}/r/key.bin\"", proxy_session_id.0, access_lease_id.0))); + assert_eq!(snapshot.delivery_mode, HlsManifestDeliveryMode::TransientPassthrough); + assert_eq!(encryption.method, "AES-128"); + assert_eq!(encryption.iv.as_deref(), Some("0x1")); + assert_eq!(encryption.key_format.as_deref(), Some("identity")); + assert_eq!(encryption.key_format_versions.as_deref(), Some("1")); + assert!(encryption.can_reset_to_clear); + + assert_encrypted_transient_terminal_incompatibility(snapshot, now_ms); +} + +#[tokio::test] +async fn transient_key_resource_is_not_cached() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin_with_response( + "200 OK", + &[("Content-Type", "application/octet-stream")], + "key-bytes", + ) + .await; + let (proxy_session_id, resource_id) = map_transient_resource_with_kind( + &app_state, + &format!("{}/key.bin", origin.base_url), + "key", + true, + TransientResourceKind::Key, + ) + .await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.key")).await; + + let first = get_response(Arc::clone(&app_state), &uri, None).await; + let second = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(first.status(), StatusCode::OK); + assert_eq!(second.status(), StatusCode::OK); + assert_eq!(response_body(first).await, bytes::Bytes::from_static(b"key-bytes")); + assert_eq!(response_body(second).await, bytes::Bytes::from_static(b"key-bytes")); + assert_eq!(origin.requests.lock().await.len(), 2); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs b/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs new file mode 100644 index 000000000..8ececa9af --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/entry_identity.rs @@ -0,0 +1,1016 @@ +use super::{ + access_lease_id_from_variant_uri, access_lease_session_token, cache_test_m3u_hls_item, + configure_default_test_server, enable_channel_unavailable_custom_response, enable_hls_cache, get_response, + hls_custom_video_test_user, proxy_session_id_from_variant_uri, response_body, single_hls_provider_input, + single_variant_master_playlist, single_variant_uri, store_test_sources_with_target, test_addr_with_port, + test_app_state, test_app_state_with_inputs, test_fingerprint, test_fingerprint_with_addr, + test_hls_entry_stream_context, test_hls_input, test_hls_share_target, test_m3u_hls_item, test_m3u_hls_share_target, +}; +use crate::{ + api::model::{ + build_proxy_session_id, ConnectionKind, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseTouch, + HlsOriginSourceKind, HlsPlaybackFamilyKey, HlsSessionKey, ProxySessionId, + }, + model::{Config, ConfigInput, ConfigTarget, ProxyUserCredentials, ReverseProxyConfig}, + processing::parser::hls::{rewrite_hls, RewriteHlsProps}, +}; +use axum::{ + http::{header, HeaderMap, StatusCode}, + response::IntoResponse, +}; +use shared::{ + model::{ + HlsCacheConfigDto, InputType, PlaylistItem, PlaylistItemHeader, PlaylistItemType, ReverseProxyConfigDto, + UserConnectionPermission, XtreamCluster, + }, + utils::Internable, +}; +use std::sync::Arc; + +#[test] +fn virtual_hls_entry_path_uses_single_manifest_extension() { + let user = hls_custom_video_test_user(); + let xtream_target = ConfigTarget::from(&shared::model::ConfigTargetDto { + output: vec![shared::model::TargetOutputDto::Xtream(shared::model::XtreamTargetOutputDto::default())], + ..Default::default() + }); + let xtream_input = ConfigInput { input_type: InputType::Xtream, ..ConfigInput::default() }; + let m3u_target = ConfigTarget::from(&shared::model::ConfigTargetDto::default()); + let m3u_input = ConfigInput { input_type: InputType::M3u, ..ConfigInput::default() }; + + let xtream_path = super::super::build_virtual_hls_entry_path(&xtream_target, &xtream_input, &user, 59); + let m3u_path = super::super::build_virtual_hls_entry_path(&m3u_target, &m3u_input, &user, 59); + + assert_eq!(xtream_path, "/live/viewer/secret/59.m3u8"); + assert_eq!(m3u_path, "/m3u-stream/live/viewer/secret/59.m3u8"); + assert!(!xtream_path.contains("..m3u8")); + assert!(!m3u_path.contains("..m3u8")); +} + +#[test] +fn hls_origin_source_kind_covers_xtream_m3u_and_direct_media_playlist() { + assert_eq!(super::super::hls_origin_source_kind(InputType::Xtream), HlsOriginSourceKind::XtreamLive); + assert_eq!(super::super::hls_origin_source_kind(InputType::M3u), HlsOriginSourceKind::M3uMediaPlaylist); + assert_eq!(super::super::hls_origin_source_kind(InputType::Library), HlsOriginSourceKind::DirectMediaPlaylist); +} + +#[test] +fn hls_origin_resolution_preserves_legacy_built_xtream_origin_url() { + let input = ConfigInput { + id: 7, + name: Arc::from("xtream"), + input_type: InputType::Xtream, + url: "http://origin.example.com/base".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + ..ConfigInput::default() + }; + + let origin = + super::super::build_hls_origin_resolution(&input, "http://other.example.com/live/other/creds/1025126.m3u8") + .expect("xtream origin should resolve"); + + assert_eq!(origin.session_entry_url.as_str(), "http://other.example.com/live/other/creds/1025126.m3u8"); + assert_eq!(origin.hls_url, origin.session_entry_url.as_str()); + assert!(origin.session_entry_url.url_failover_provider().is_none()); +} + +#[test] +fn hls_origin_resolution_uses_m3u_playlist_item_url() { + let input = ConfigInput { + id: 9, + name: Arc::from("m3u"), + input_type: InputType::M3u, + url: "http://playlist.example.com/list.m3u".to_string(), + ..ConfigInput::default() + }; + + let origin = super::super::build_hls_origin_resolution(&input, "http://media.example.com/live/channel/index.m3u8") + .expect("m3u hls origin should resolve"); + let source = super::super::build_hls_origin_source(&input, "stable-item"); + + assert_eq!(origin.session_entry_url.as_str(), "http://media.example.com/live/channel/index.m3u8"); + assert_eq!(source.source_kind, HlsOriginSourceKind::M3uMediaPlaylist); + assert_eq!(source.session_key().stable_value(), "input:9|hls|stable-item"); +} + +#[tokio::test] +async fn hls_cache_entry_returns_master_playlist_without_origin_refresh_or_session_creation() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + + let response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 12345, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + Some("/iptv"), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers().get(header::CONTENT_TYPE).and_then(|value| value.to_str().ok()), + Some("application/vnd.apple.mpegurl") + ); + assert_eq!( + response.headers().get(header::CACHE_CONTROL).and_then(|value| value.to_str().ok()), + Some("private, no-store, no-cache, must-revalidate") + ); + assert!(response.headers().get(header::LOCATION).is_none()); + assert!(response.headers().get(header::CONTENT_ENCODING).is_none()); + assert!(response.headers().get(header::VARY).is_none()); + let content_length = response + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .expect("content length"); + let body = response_body(response).await; + assert_eq!(content_length, body.len()); + let body = std::str::from_utf8(&body).expect("master playlist should be UTF-8"); + assert_eq!(body.matches("/iptv").count(), 1); + let variant_uri = body.lines().nth(2).expect("single variant URI"); + assert!(variant_uri.starts_with("/iptv/hls/shared/live/")); + assert!(variant_uri.ends_with("/manifest.m3u8")); + let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(variant_uri).to_string()); + assert_eq!(access_lease_id.0.len(), 22); + let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(variant_uri).to_string()); + let lease = app_state + .hls + .proxy + .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::super::current_time_millis()) + .await + .expect("entry access lease"); + assert!(app_state + .active_users + .get_and_update_user_session(&lease.username, &lease.user_session_token) + .await + .is_some()); + assert!(app_state.hls.proxy.sessions().get_by_key(&session_key).await.is_none()); + assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0); + assert!(app_state.active_users.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn hls_cache_entry_master_playlist_uses_cache_when_target_hls_share_enabled() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + configure_default_test_server(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.password = "hls-pass".to_string(); + let input = test_hls_input(); + let target = test_hls_share_target(true); + let original_hls_entry_path = super::super::build_virtual_hls_entry_path(&target, &input, &user, 1001); + + let response = super::super::handle_hls_stream_request( + &test_fingerprint(), + &app_state, + &user, + &target, + None, + None, + "http://origin.example.com/live/user/pass/1001.m3u8", + None, + test_hls_entry_stream_context(1001, "80510", Some(2_500_000)), + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &original_hls_entry_path, + super::super::HlsRequestStage::Entry, + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::OK); + let (bandwidth, variant_uri) = single_variant_master_playlist(response).await; + assert_eq!(bandwidth, 3_000_000); + assert!(variant_uri.starts_with("/hls/shared/live/")); + assert!(variant_uri.ends_with("/manifest.m3u8")); + assert_eq!(app_state.hls.proxy.access_leases().read().await.len(), 1); + + let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&variant_uri).to_string()); + let origin_key = HlsSessionKey::new(input.id, "80510"); + let virtual_id_key = HlsSessionKey::new(input.id, "1001"); + assert_eq!(origin_key.stable_value(), "input:1|hls|80510"); + assert_eq!(proxy_session_id, build_proxy_session_id(&origin_key, &app_state.get_encrypt_secret())); + assert_ne!(proxy_session_id, build_proxy_session_id(&virtual_id_key, &app_state.get_encrypt_secret())); + + let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&variant_uri).to_string()); + let lease = app_state + .hls + .proxy + .access_leases() + .write() + .await + .response_snapshot(&access_lease_id, &proxy_session_id, super::super::current_time_millis()) + .expect("access lease"); + assert_eq!(lease.stream_ref, "80510"); + assert_eq!(lease.virtual_id, 1001); + assert_eq!(lease.known_bitrate_bps, Some(2_500_000)); +} + +#[tokio::test] +async fn hls_cache_entry_shares_content_session_across_targets_but_keeps_distinct_virtual_leases() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + configure_default_test_server(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.password = "hls-pass".to_string(); + let input = test_hls_input(); + let first_target = test_hls_share_target(true); + let mut second_target = test_hls_share_target(true); + second_target.id = 2; + + let first_response = super::super::handle_hls_stream_request( + &test_fingerprint(), + &app_state, + &user, + &first_target, + None, + None, + "http://origin.example.com/live/user/pass/1001.m3u8", + None, + test_hls_entry_stream_context(1001, "80510", None), + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &super::super::build_virtual_hls_entry_path(&first_target, &input, &user, 1001), + super::super::HlsRequestStage::Entry, + ) + .await + .into_response(); + let second_response = super::super::handle_hls_stream_request( + &test_fingerprint_with_addr(test_addr_with_port(55124)), + &app_state, + &user, + &second_target, + None, + None, + "http://origin.example.com/live/user/pass/9007.m3u8", + None, + test_hls_entry_stream_context(9007, "80510", None), + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &super::super::build_virtual_hls_entry_path(&second_target, &input, &user, 9007), + super::super::HlsRequestStage::Entry, + ) + .await + .into_response(); + + assert_eq!(first_response.status(), StatusCode::OK); + assert_eq!(second_response.status(), StatusCode::OK); + let first_variant_uri = single_variant_uri(first_response).await; + let second_variant_uri = single_variant_uri(second_response).await; + assert_eq!( + proxy_session_id_from_variant_uri(&first_variant_uri), + proxy_session_id_from_variant_uri(&second_variant_uri) + ); + assert_ne!( + access_lease_id_from_variant_uri(&first_variant_uri), + access_lease_id_from_variant_uri(&second_variant_uri) + ); + + let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&first_variant_uri).to_string()); + let first_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&first_variant_uri).to_string()); + let second_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&second_variant_uri).to_string()); + let now_ms = super::super::current_time_millis(); + let mut leases = app_state.hls.proxy.access_leases().write().await; + let first_lease = leases.response_snapshot(&first_lease_id, &proxy_session_id, now_ms).expect("first access lease"); + let second_lease = + leases.response_snapshot(&second_lease_id, &proxy_session_id, now_ms).expect("second access lease"); + assert_eq!(first_lease.virtual_id, 1001); + assert_eq!(second_lease.virtual_id, 9007); + assert_eq!(first_lease.stream_ref, "80510"); + assert_eq!(second_lease.stream_ref, "80510"); +} + +#[tokio::test] +async fn hls_cache_entry_uses_legacy_path_when_target_hls_share_disabled() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + configure_default_test_server(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.password = "hls-pass".to_string(); + let input = test_hls_input(); + let target = test_hls_share_target(false); + let original_hls_entry_path = super::super::build_virtual_hls_entry_path(&target, &input, &user, 12345); + + let response = super::super::handle_hls_stream_request( + &test_fingerprint(), + &app_state, + &user, + &target, + None, + None, + "http://origin.example.com/live/user/pass/12345.m3u8", + None, + test_hls_entry_stream_context(12345, "80510", None), + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &original_hls_entry_path, + super::super::HlsRequestStage::Entry, + ) + .await + .into_response(); + + let location = response.headers().get(header::LOCATION).and_then(|value| value.to_str().ok()).unwrap_or(""); + assert!(!location.contains("/hls/shared/live/")); + assert!(app_state.hls.proxy.access_leases().read().await.is_empty()); + assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0); +} + +#[tokio::test] +async fn legacy_hls_token_route_renders_channel_unavailable_inline_when_target_hls_share_enabled() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + app_state.app_config.config.store(Arc::new(Config { + custom_stream_response_enabled: true, + reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { + hls_cache: Some(HlsCacheConfigDto::default()), + ..Default::default() + })), + ..Default::default() + })); + configure_default_test_server(&app_state); + enable_channel_unavailable_custom_response(&app_state); + let user = app_state.app_config.get_user_credentials("hls-user").expect("test user should exist"); + let input = test_hls_input(); + let target = test_hls_share_target(true); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + let encrypt_secret = app_state.get_encrypt_secret(); + let legacy_manifest = rewrite_hls( + &user, + &RewriteHlsProps { + secret: &encrypt_secret, + base_url: "", + content: "#EXTM3U\n#EXTINF:4.0,\nseg.ts\n", + hls_url: "http://origin.example.com/live/user/pass/12345.m3u8".to_string(), + target_id: target.id, + virtual_id: 12345, + input_id: input.id, + user_token: Some("legacy-session-token"), + origin_provider: None, + playlist_kind: None, + }, + ); + let token = legacy_manifest + .lines() + .find_map(|line| line.rsplit_once('/').map(|(_, token)| token.trim().to_string())) + .expect("legacy hls segment token should be rendered"); + + let response = super::super::hls_api_stream_resolved( + test_fingerprint(), + HeaderMap::new(), + Arc::clone(&app_state), + Arc::clone(&user), + Arc::new(target), + input.id, + 12345, + token, + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + assert_eq!(response.headers()[header::CONTENT_TYPE], "application/vnd.apple.mpegurl"); + assert!(app_state.hls.proxy.access_leases().read().await.is_empty()); + assert!(app_state.active_users.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn legacy_hls_token_route_with_invalid_token_returns_bad_request_when_target_hls_share_enabled() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let user = app_state.app_config.get_user_credentials("hls-user").expect("test user should exist"); + let input = test_hls_input(); + let target = test_hls_share_target(true); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + + let response = super::super::hls_api_stream_resolved( + test_fingerprint(), + HeaderMap::new(), + Arc::clone(&app_state), + user, + Arc::new(target), + input.id, + 12345, + "not-a-valid-legacy-token.ts".to_string(), + ) + .await; + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(app_state.hls.proxy.access_leases().read().await.is_empty()); + assert!(app_state.active_users.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn hls_cache_entry_leases_update_effective_origin_acquire_policy_for_shared_session() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut soft_user = ProxyUserCredentials::default(); + soft_user.username = "soft-user".to_string(); + soft_user.soft_priority = 20; + let mut normal_user = ProxyUserCredentials::default(); + normal_user.username = "normal-user".to_string(); + normal_user.priority = -5; + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + + let soft_response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &soft_user, + origin_source.clone(), + 12345, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Soft), + None, + ) + .await; + assert_eq!(soft_response.status(), StatusCode::OK); + let soft_snapshot = + app_state.hls.proxy.access_lease_session_snapshot(&proxy_session_id, super::super::current_time_millis()).await; + let soft_policy = soft_snapshot.effective_origin_policy.expect("soft policy"); + assert_eq!(soft_policy.connection_kind, ConnectionKind::Soft); + assert_eq!(soft_policy.priority, soft_user.soft_priority); + + // Different user/family, same shared HLS session. Normal media admission must upgrade + // the future origin-account acquire policy without changing the shared session identity. + let normal_response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &normal_user, + origin_source, + 12345, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + assert_eq!(normal_response.status(), StatusCode::OK); + let normal_snapshot = + app_state.hls.proxy.access_lease_session_snapshot(&proxy_session_id, super::super::current_time_millis()).await; + let normal_policy = normal_snapshot.effective_origin_policy.expect("normal policy"); + assert_eq!(normal_policy.connection_kind, ConnectionKind::Normal); + assert_eq!(normal_policy.priority, normal_user.priority); + + let (session, _) = app_state + .hls + .proxy + .get_or_create_session_with_source_and_outcome( + session_key, + super::super::build_hls_origin_source(&input, "12345"), + &app_state.get_encrypt_secret(), + super::super::current_time_millis(), + ) + .await; + app_state + .hls + .proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + &proxy_session_id, + super::super::current_time_millis(), + ) + .await; + let session_policy = session.read().await.effective_origin_acquire_policy_or_default(); + assert_eq!(session_policy.connection_kind, ConnectionKind::Normal); + assert_eq!(session_policy.priority, normal_user.priority); +} + +#[tokio::test] +async fn hls_entry_origin_reservation_blocks_foreign_sessions_only_after_media_confirmation() { + let input = single_hls_provider_input("available-provider"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let session_owner = "hls-cache:test-session"; + + let reservation = super::super::try_reserve_hls_entry_origin_account_for_redirect( + &app_state, + &test_fingerprint(), + &user, + &input, + 12345, + "http://account.example.com/live/account-user/account-pass/12345.m3u8", + "hls-session-token", + session_owner, + crate::model::PlaybackKind::LiveHls, + super::super::hls_origin_account_reservation_ttl_secs_fallback(), + UserConnectionPermission::Allowed, + ConnectionKind::Normal, + false, + ) + .await + .expect("provider reservation should be acquired before provisioning redirect"); + + assert_eq!(reservation.request_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8"); + assert!(reservation.selected_provider_config.is_some()); + assert!(app_state.active_users.active_streams().await.is_empty()); + app_state.connection_manager.release_provider_handle(reservation.provider_handle); + + // A manifest-only entry reservation is still unconfirmed, so it must not block a + // different HLS session: an abandoned start or a manifest retry has to leave the + // provider capacity free for unrelated clients behind the same reverse proxy. + let other_handle = app_state.active_provider.acquire_connection_with_grace_for_session( + &input.name, + &test_addr_with_port(55251), + false, + 0, + ConnectionKind::Normal, + Some("other-owner"), + ); + assert!(other_handle.is_some(), "an unconfirmed entry reservation must not block other HLS sessions"); + app_state.connection_manager.release_provider_handle(other_handle); + + let same_owner_handle = app_state.active_provider.acquire_connection_with_grace_for_session( + &input.name, + &test_addr_with_port(55252), + false, + 0, + ConnectionKind::Normal, + Some(session_owner), + ); + assert!(same_owner_handle.is_some(), "reserved provider must be reusable by the same HLS session owner"); + app_state.connection_manager.release_provider_handle(same_owner_handle); + + // Once real media activity confirms the lease, it reserves the provider slot and + // other sessions are routed away from it. + app_state.active_provider.confirm_playback_activity(session_owner); + assert!( + app_state + .active_provider + .acquire_connection_with_grace_for_session( + &input.name, + &test_addr_with_port(55253), + false, + 0, + ConnectionKind::Normal, + Some("other-owner"), + ) + .is_none(), + "a confirmed reservation must stay blocked for other HLS sessions" + ); +} + +#[tokio::test] +async fn hls_entry_origin_reservation_uses_persisted_alias_manifest_url() { + use shared::model::PlaylistGroup; + use tuliprox_repository::{get_input_m3u_playlist_file_path, get_input_storage_path, persist_input_m3u_playlist}; + + let temp = tempfile::tempdir().expect("temp dir should be created"); + let input = ConfigInput { + id: 1, + name: Arc::from("primary-account"), + input_type: InputType::M3u, + url: "http://playlist.example/list.m3u?access_key=primary-playlist-key".to_string(), + enabled: true, + max_connections: 1, + aliases: Some(vec![crate::model::ConfigInputAlias { + id: 2, + name: Arc::from("alias-account"), + url: "http://playlist.example/list.m3u?access_key=alias-playlist-key".to_string(), + username: None, + password: None, + max_connections: 0, + priority: 0, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let mut config = (*app_state.app_config.config.load_full()).clone(); + config.storage_dir = temp.path().to_string_lossy().into_owned(); + app_state.app_config.config.store(Arc::new(config)); + + let alias_name = "alias-account".intern(); + let storage_path = get_input_storage_path(&alias_name, &app_state.app_config.config.load().storage_dir) + .await + .expect("alias storage should be created"); + let playlist_path = get_input_m3u_playlist_file_path(&storage_path, &alias_name); + let alias_playlist = vec![PlaylistGroup { + id: 1, + title: "Live".intern(), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { + id: "news24hd".intern(), + input_stream_id: "news24hd".intern(), + url: "http://stream.example:4000/news24hd/mono.m3u8?token=alias-stream-token".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }], + xtream_cluster: XtreamCluster::Live, + }]; + persist_input_m3u_playlist(&app_state.app_config, &playlist_path, &alias_playlist) + .await + .expect("alias playlist should persist"); + + let primary_handle = app_state + .active_provider + .acquire_exact_connection_with_grace_for_session( + &input.name, + &test_fingerprint().addr, + false, + 0, + ConnectionKind::Normal, + Some("primary-session"), + ) + .expect("primary account should be allocated"); + + let reservation = super::super::try_reserve_hls_entry_origin_account_for_redirect( + &app_state, + &test_fingerprint(), + &{ + let mut creds = ProxyUserCredentials::default(); + creds.username = "hls-user".to_string(); + creds + }, + &input, + 12345, + "http://stream.example:4000/news24hd/mono.m3u8?token=primary-stream-token", + "alias-session-token", + "alias-session-owner", + crate::model::PlaybackKind::LiveHls, + super::super::hls_origin_account_reservation_ttl_secs_fallback(), + UserConnectionPermission::Allowed, + ConnectionKind::Normal, + false, + ) + .await + .expect("alias provider reservation should succeed"); + + assert_eq!( + reservation.selected_provider_config.as_ref().map(|provider| provider.name.as_ref()), + Some("alias-account") + ); + assert_eq!(reservation.request_url, "http://stream.example:4000/news24hd/mono.m3u8?token=alias-stream-token"); + + app_state.connection_manager.release_provider_handle(reservation.provider_handle); + app_state.connection_manager.release_provider_handle(Some(primary_handle)); +} + +#[tokio::test] +async fn hls_virtual_entry_reservation_uses_input_stream_id_for_shared_session_owner() { + let input = single_hls_provider_input("origin-id-reservation-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + let target = Arc::new(test_m3u_hls_share_target()); + let item = test_m3u_hls_item( + &input, + 1001, + "80510", + "http://account.example.com/live/account-user/account-pass/80510.m3u8", + ); + cache_test_m3u_hls_item(&app_state, &target, item).await; + let stream_identity = super::super::HlsEntryStreamIdentity::new(1001, "80510").expect("input stream identity"); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + + assert!( + super::super::try_reserve_hls_virtual_entry_origin_account_for_redirect( + &app_state, + &test_fingerprint(), + &user, + &target, + &input, + &stream_identity, + ) + .await + ); + + let expected_key = HlsSessionKey::new(input.id, "80510"); + let expected_proxy_session_id = build_proxy_session_id(&expected_key, &app_state.get_encrypt_secret()); + let expected_owner = crate::api::model::build_hls_origin_session_owner(&expected_proxy_session_id); + let same_owner_handle = app_state.active_provider.acquire_connection_with_grace_for_session( + &input.name, + &test_addr_with_port(55253), + false, + 0, + ConnectionKind::Normal, + Some(&expected_owner), + ); + assert!(same_owner_handle.is_some(), "reservation must be owned by input:1|hls|80510, not virtual_id=1001"); + app_state.connection_manager.release_provider_handle(same_owner_handle); +} + +#[tokio::test] +async fn hls_cache_entry_creates_new_lease_for_same_user_session_and_proxy_session() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + + let first_response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + origin_source.clone(), + 12345, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + let first_variant_uri = single_variant_uri(first_response).await; + let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&first_variant_uri).to_string()); + let first_access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&first_variant_uri).to_string()); + let first_session_token = access_lease_session_token(&app_state, &proxy_session_id, &first_access_lease_id).await; + + let second_response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 12345, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + let second_variant_uri = single_variant_uri(second_response).await; + assert_ne!( + access_lease_id_from_variant_uri(&first_variant_uri), + access_lease_id_from_variant_uri(&second_variant_uri) + ); + let second_access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&second_variant_uri).to_string()); + let second_session_token = access_lease_session_token(&app_state, &proxy_session_id, &second_access_lease_id).await; + assert_ne!(first_session_token, second_session_token); +} + +#[tokio::test] +async fn hls_cache_entry_creates_new_lease_after_manifest_touch() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + + let first_response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + origin_source.clone(), + 12345, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + let first_variant_uri = single_variant_uri(first_response).await; + let first_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&first_variant_uri).to_string()); + let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&first_variant_uri).to_string()); + let first_session_token = access_lease_session_token(&app_state, &proxy_session_id, &first_lease_id).await; + let now_ms = super::super::current_time_millis(); + + assert!(matches!( + app_state + .hls + .proxy + .touch_manifest_access_lease( + &first_lease_id, + &proxy_session_id, + now_ms, + None, + Some(super::HlsAccessLeasePendingDeadline::Bootstrap { + deadline_ms: now_ms.saturating_add(super::super::hls_pending_bootstrap_window_ms(&app_state)), + }), + super::super::hls_access_lease_ttl_ms(&app_state), + ) + .await, + HlsAccessLeaseTouch::Touched { .. } + )); + + let second_response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 12345, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + let second_variant_uri = single_variant_uri(second_response).await; + + assert_ne!( + access_lease_id_from_variant_uri(&first_variant_uri), + access_lease_id_from_variant_uri(&second_variant_uri) + ); + let second_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&second_variant_uri).to_string()); + let second_session_token = access_lease_session_token(&app_state, &proxy_session_id, &second_lease_id).await; + assert_ne!(first_session_token, second_session_token); +} + +#[tokio::test] +async fn hls_cache_entry_ignores_existing_pending_lease_for_new_playback() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let proxy_session_id = build_proxy_session_id(&origin_source.session_key(), &app_state.get_encrypt_secret()); + let old_lease_id = HlsAccessLeaseId("old-pending-lease".to_string()); + let old_session_token = "old-hls-session-token"; + let old_issued_at_ms = super::super::current_time_millis().saturating_sub(6_000); + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + old_lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), + proxy_session_id.clone(), + "hls-user".to_string(), + old_session_token.to_string(), + 1, + "12345".to_string(), + 12345, + old_issued_at_ms, + super::super::hls_access_lease_ttl_ms(&app_state), + )) + .await; + + let response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 12345, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + let variant_uri = single_variant_uri(response).await; + let new_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&variant_uri).to_string()); + let new_session_token = access_lease_session_token(&app_state, &proxy_session_id, &new_lease_id).await; + + assert_ne!(old_lease_id, new_lease_id); + assert_ne!(old_session_token, new_session_token); +} + +#[tokio::test] +async fn hls_cache_entry_master_playlist_for_xtream_uses_stream_ref_session_identity() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = + ConfigInput { id: 7, name: Arc::from("xtream-input"), input_type: InputType::Xtream, ..ConfigInput::default() }; + let origin_source = super::super::build_hls_origin_source(&input, "80510"); + let expected_proxy_session_id = + build_proxy_session_id(&origin_source.session_key(), &app_state.get_encrypt_secret()); + + let response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 80510, + None, + None, + None, + "http://origin.example.com/live/user/pass/80510.m3u8", + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let variant_uri = single_variant_uri(response).await; + assert!(variant_uri.starts_with(&format!("/hls/shared/live/{}/", expected_proxy_session_id.0))); + assert!(variant_uri.ends_with("/manifest.m3u8")); + assert!(app_state.hls.proxy.sessions().get_by_key(&HlsSessionKey::new(7, "80510")).await.is_none()); + assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0); +} + +#[tokio::test] +async fn hls_cache_entry_master_playlist_for_m3u_uses_stream_ref_session_identity() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = + ConfigInput { id: 9, name: Arc::from("m3u-input"), input_type: InputType::M3u, ..ConfigInput::default() }; + let origin_source = super::super::build_hls_origin_source(&input, "70001"); + let expected_proxy_session_id = + build_proxy_session_id(&origin_source.session_key(), &app_state.get_encrypt_secret()); + + let response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 70001, + None, + None, + None, + "http://media.example.com/channel/playlist.m3u8", + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + Some("/iptv"), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let variant_uri = single_variant_uri(response).await; + assert!(variant_uri.starts_with(&format!("/iptv/hls/shared/live/{}/", expected_proxy_session_id.0))); + assert!(variant_uri.ends_with("/manifest.m3u8")); + assert!(app_state.hls.proxy.sessions().get_by_key(&HlsSessionKey::new(9, "70001")).await.is_none()); + assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0); +} + +#[tokio::test] +async fn hls_proxy_manifest_invalid_token_starts_no_origin_work() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + + let response = get_response( + Arc::clone(&app_state), + "/hls/shared/live/a8f31c9eQ7sLk92pV0mTaw/not-a-valid-token/manifest.m3u8", + None, + ) + .await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(app_state.hls.proxy.metrics().snapshot().refresh_started, 0); + assert!(app_state.hls.proxy.sessions().is_empty().await); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/headers.rs b/backend/app/src/api/endpoints/hls_api/tests/headers.rs new file mode 100644 index 000000000..2d3205159 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/headers.rs @@ -0,0 +1,383 @@ +use super::{ + apply_hls_user_agent_stream_index, assert_hls_cache_stream_registered, build_hls_manifest_request_headers, + cache_recording_test_item, configure_recording_test_listener, enable_hls_cache, encode_test_manifest, get_response, + hls_proxy_uri, hls_session_last_media_at_ms, legacy_manifest_test_client_headers, legacy_manifest_test_input, + map_transient_resource, recording_test_response, recording_test_router, recording_test_url, response_body, + spawn_recording_header_origin, spawn_test_encoded_manifest_origin, spawn_test_transient_origin, + store_test_sources_with_target, test_app_state, test_app_state_with_inputs, test_hls_share_target, + test_m3u_hls_item, +}; +use crate::{ + api::model::{HlsSession, HlsSessionKey, ProxySessionId}, + model::{ConfigInput, ReverseProxyDisabledHeaderConfig}, +}; +use axum::{ + http::{header, HeaderMap, HeaderName, HeaderValue, StatusCode}, + response::IntoResponse, +}; +use http_body_util::BodyExt; +use shared::model::{InputType, M3uTargetOutputDto, XtreamCluster}; +use std::{collections::HashMap, sync::Arc, time::Duration}; +use tuliprox_hls::api::{extract_hls_provider_session_headers, MAX_HLS_MANIFEST_BYTES}; + +#[test] +fn extract_hls_provider_session_headers_converts_set_cookie_to_cookie_header() { + let mut headers = HeaderMap::new(); + headers.append("set-cookie", "sid=abc; Path=/; HttpOnly".parse().expect("valid cookie")); + headers.append("set-cookie", "pref=1; Secure".parse().expect("valid cookie")); + + let session_headers = extract_hls_provider_session_headers(&headers); + + assert_eq!(session_headers.headers.get("cookie").map(String::as_str), Some("sid=abc; pref=1")); +} + +#[test] +fn hls_response_uses_rfc8216_content_type_and_remains_tower_compressible() { + let response = super::super::hls_response("#EXTM3U\n".to_string()).into_response(); + + assert_eq!(response.headers().get(header::CONTENT_TYPE).unwrap(), "application/vnd.apple.mpegurl"); + assert!(tuliprox_core::utils::response_compression::should_compress_response(&response)); +} + +#[test] +fn hls_manifest_headers_apply_disabled_headers_and_default_user_agent_policy() { + let mut input_headers = HashMap::new(); + input_headers.insert("User-Agent".to_string(), "Input-UA".to_string()); + input_headers.insert("Accept-Language".to_string(), "de".to_string()); + input_headers.insert("Accept-Encoding".to_string(), "gzip".to_string()); + input_headers.insert("Authorization".to_string(), "Bearer input-secret".to_string()); + input_headers.insert("X-Origin-Secret".to_string(), "input-secret".to_string()); + + let mut request_headers = HeaderMap::new(); + request_headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-")); + request_headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); + request_headers.insert(header::USER_AGENT, HeaderValue::from_static("Client-UA")); + request_headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer client-secret")); + request_headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); + request_headers.insert(HeaderName::from_static("proxy-authorization"), HeaderValue::from_static("Basic secret")); + request_headers.insert(header::HOST, HeaderValue::from_static("proxy.example.com")); + request_headers.insert(HeaderName::from_static("x-blocked"), HeaderValue::from_static("client")); + request_headers.insert(HeaderName::from_static("cf-ray"), HeaderValue::from_static("cf")); + + let disabled = ReverseProxyDisabledHeaderConfig { + referer_header: false, + x_header: true, + cloudflare_header: true, + custom_header: vec!["X-Origin-Secret".to_string()], + }; + let headers = build_hls_manifest_request_headers( + &input_headers, + &request_headers, + Some(&disabled), + Some("Default-UA"), + Some("Channel-UA"), + ); + + assert_eq!(headers.get(header::USER_AGENT).expect("user agent"), "Channel-UA"); + assert_eq!(headers.get("accept-language").expect("accept language"), "de"); + assert_eq!(headers.get(header::ACCEPT_ENCODING).expect("accept encoding"), "identity"); + assert!(!headers.contains_key(header::RANGE)); + assert!(!headers.contains_key(header::AUTHORIZATION)); + assert!(!headers.contains_key(header::COOKIE)); + assert!(!headers.contains_key("proxy-authorization")); + assert!(!headers.contains_key(header::HOST)); + assert!(!headers.contains_key("x-origin-secret")); + assert!(!headers.contains_key("x-blocked")); + assert!(!headers.contains_key("cf-ray")); +} + +#[tokio::test] +async fn hls_user_agent_stream_index_is_stable_for_the_session() { + let app_state = test_app_state(); + let session = Arc::new(tokio::sync::RwLock::new(HlsSession::new(HlsSessionKey::new(1, "stream-1"), b"secret", 0))); + let mut manifest_headers = HeaderMap::new(); + manifest_headers.insert(header::USER_AGENT, HeaderValue::from_static("VLC/3.0")); + + apply_hls_user_agent_stream_index(&session, &mut manifest_headers, true, &app_state.active_users).await; + let stream_index = session.read().await.user_agent_stream_index.unwrap_or_default(); + assert_ne!(stream_index, 0); + assert_eq!( + manifest_headers.get(header::USER_AGENT).and_then(|value| value.to_str().ok()), + Some(format!("VLC/3.0 {stream_index}").as_str()) + ); + + let mut segment_headers = HeaderMap::new(); + segment_headers.insert(header::USER_AGENT, HeaderValue::from_static("VLC/3.0")); + apply_hls_user_agent_stream_index(&session, &mut segment_headers, true, &app_state.active_users).await; + + assert_eq!( + segment_headers.get(header::USER_AGENT).and_then(|value| value.to_str().ok()), + Some(format!("VLC/3.0 {stream_index}").as_str()) + ); + assert_eq!(session.read().await.user_agent_stream_index, Some(stream_index)); +} + +#[tokio::test] +async fn legacy_hls_manifest_decodes_supported_origin_codings_and_enforces_identity() { + const MANIFEST: &str = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nsegment.ts\n"; + + for coding in ["gzip", "deflate", "br", "zstd"] { + let encoded = encode_test_manifest(coding, MANIFEST.as_bytes()).await; + let origin = spawn_test_encoded_manifest_origin(Some(coding), encoded, Duration::ZERO).await; + let input = legacy_manifest_test_input(&origin); + let client_headers = legacy_manifest_test_client_headers(); + + let (manifest, final_url, _) = + super::super::download_legacy_hls_manifest(&test_app_state(), &input, &client_headers) + .await + .unwrap_or_else(|error| panic!("{coding} manifest should decode: {error}")); + + assert_eq!(manifest, MANIFEST, "coding={coding}"); + assert_eq!(final_url, input.url, "coding={coding}"); + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1, "coding={coding}"); + assert!( + requests[0].to_ascii_lowercase().contains("\r\naccept-encoding: identity\r\n"), + "coding={coding}, request={}", + requests[0] + ); + } +} + +#[tokio::test] +async fn legacy_hls_manifest_handles_identity_and_headerless_gzip_magic() { + const MANIFEST: &[u8] = b"#EXTM3U\n#EXT-X-TARGETDURATION:4\n"; + let cases = [("identity", MANIFEST.to_vec()), ("gzip-magic", encode_test_manifest("gzip", MANIFEST).await)]; + + for (case, body) in cases { + let origin = spawn_test_encoded_manifest_origin(None, body, Duration::ZERO).await; + let input = legacy_manifest_test_input(&origin); + + let (manifest, _, _) = super::super::download_legacy_hls_manifest( + &test_app_state(), + &input, + &legacy_manifest_test_client_headers(), + ) + .await + .unwrap_or_else(|error| panic!("{case} manifest should decode: {error}")); + + assert_eq!(manifest.as_bytes(), MANIFEST, "case={case}"); + } +} + +#[tokio::test] +async fn legacy_hls_manifest_limit_applies_after_decompression() { + let decoded = vec![b'x'; MAX_HLS_MANIFEST_BYTES + 1]; + let origin = + spawn_test_encoded_manifest_origin(Some("gzip"), encode_test_manifest("gzip", &decoded).await, Duration::ZERO) + .await; + let input = legacy_manifest_test_input(&origin); + + let error = + super::super::download_legacy_hls_manifest(&test_app_state(), &input, &legacy_manifest_test_client_headers()) + .await + .expect_err("decoded manifest above limit must fail"); + + assert!(matches!( + error.get_ref().and_then(|source| source.downcast_ref()), + Some(crate::utils::content_coding::ContentBodyReadError::LimitExceeded { limit }) + if *limit == MAX_HLS_MANIFEST_BYTES + )); +} + +#[test] +fn hls_proxy_public_path_prefix_rewrites_only_proxy_hls_uri_surfaces() { + let body = concat!( + "#EXTM3U\n", + "#EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/proxy-id/r/key.key\",IV=0x1\n", + "#EXT-X-MAP:URI=\"/hls/shared/live/proxy-id/map/000000.mp4\",BYTERANGE=\"10@0\"\n", + "#EXT-X-PART:DURATION=1.0,URI=\"/hls/shared/live/proxy-id/r/part.m4s\"\n", + "#EXT-X-MEDIA-SEQUENCE:7\n", + "#EXTINF:4.0,\n", + "/hls/shared/live/proxy-id/000007.ts\n", + "https://origin.example.com/not-proxy.ts\n", + ); + + let prefixed = super::super::apply_hls_proxy_public_path_prefix(body.to_string(), Some("/iptv/")); + + assert!(prefixed.contains("URI=\"/iptv/hls/shared/live/proxy-id/r/key.key\"")); + assert!(prefixed.contains("URI=\"/iptv/hls/shared/live/proxy-id/map/000000.mp4\"")); + assert!(prefixed.contains("URI=\"/iptv/hls/shared/live/proxy-id/r/part.m4s\"")); + assert!(prefixed.contains("\n/iptv/hls/shared/live/proxy-id/000007.ts\n")); + assert!(prefixed.contains("#EXT-X-MEDIA-SEQUENCE:7")); + assert!(prefixed.contains("https://origin.example.com/not-proxy.ts")); +} + +#[test] +fn hls_proxy_public_path_prefix_keeps_body_unchanged_without_server_path() { + let body = "#EXTM3U\n#EXTINF:4.0,\n/hls/shared/live/proxy-id/000007.ts\n".to_string(); + + assert_eq!(super::super::apply_hls_proxy_public_path_prefix(body.clone(), None), body); + assert_eq!(super::super::apply_hls_proxy_public_path_prefix(body.clone(), Some("/")), body); +} + +pub(in crate::api::endpoints::hls_api::tests) fn local_recording_hls_url( + manifest: &str, +) -> Result> { + let resource = + manifest.lines().find(|line| !line.is_empty() && !line.starts_with('#')).ok_or("HLS resource URI missing")?; + let url = url::Url::parse(resource)?; + assert_eq!(url.scheme(), "http"); + assert_eq!(url.host_str(), Some("127.0.0.1")); + assert_eq!(url.port(), Some(8901)); + assert!(url.path().starts_with("/hls/recording_user/"), "{url}"); + Ok(url) +} + +#[tokio::test] +async fn recording_hls_routes_keep_local_urls_and_headers_after_session_loss() -> Result<(), Box> +{ + for (output, share) in [ + (shared::model::TargetType::Xtream, false), + (shared::model::TargetType::M3u, false), + (shared::model::TargetType::Xtream, true), + (shared::model::TargetType::M3u, true), + ] { + let (origin_url, requests, origin_task) = spawn_recording_header_origin().await; + let input = ConfigInput { + id: 1, + name: Arc::from("recording-input"), + input_type: if output == shared::model::TargetType::Xtream { InputType::Xtream } else { InputType::M3u }, + url: origin_url.clone(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let mut target = test_hls_share_target(share); + target.use_memory_cache = true; + if output == shared::model::TargetType::M3u { + target.output = vec![crate::model::TargetOutput::M3u(crate::model::M3uTargetOutput::from( + &M3uTargetOutputDto::default(), + ))]; + } + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + app_state.app_config.api_proxy.store(None); + if share { + enable_hls_cache(&app_state); + } + let storage = tempfile::tempdir()?; + configure_recording_test_listener(&app_state, storage.path()); + let manifest_url = format!("{origin_url}/live/user/pass/12345.m3u8"); + cache_recording_test_item(&app_state, &target, test_m3u_hls_item(&input, 12345, "12345", &manifest_url)) + .await?; + let router = recording_test_router(&app_state); + let entry = + recording_test_response(&router, &recording_test_url(&app_state, &target, &input, XtreamCluster::Live)?) + .await?; + assert_eq!(entry.status(), StatusCode::OK, "{output:?}"); + let master = String::from_utf8(entry.into_body().collect().await?.to_bytes().to_vec())?; + let variant_url = local_recording_hls_url(&master)?; + let mut public_credentials = variant_url.clone(); + public_credentials.set_path(&variant_url.path().replacen( + &crate::api::api_utils::create_recording_proxy_user(&app_state).password, + crate::model::RECORDING_PROXY_USERNAME, + 1, + )); + let denied = recording_test_response(&router, public_credentials.as_str()).await?; + assert!(!denied.status().is_success(), "the public recording_user password must not authorize HLS"); + let token = variant_url.path_segments().and_then(Iterator::last).ok_or("HLS token missing")?; + let decoded = super::get_hls_session_token_and_url_from_token(&app_state.get_encrypt_secret(), token) + .ok_or("HLS token cannot be decoded")?; + let session_token = decoded.session_token.ok_or("HLS session token missing")?; + let variant = recording_test_response(&router, variant_url.as_str()).await?; + assert_eq!(variant.status(), StatusCode::OK, "{output:?}"); + let media = String::from_utf8(variant.into_body().collect().await?.to_bytes().to_vec())?; + let segment_url = local_recording_hls_url(&media)?; + let segment = recording_test_response(&router, segment_url.as_str()).await?; + assert_eq!(segment.status(), StatusCode::OK, "{output:?}"); + assert_eq!(segment.into_body().collect().await?.to_bytes().as_ref(), b"segment-bytes"); + // The hand-off served the first variant; a refresh must fetch the origin again. + let refresh = recording_test_response(&router, variant_url.as_str()).await?; + assert_eq!(refresh.status(), StatusCode::OK, "{output:?}"); + let refreshed = String::from_utf8(refresh.into_body().collect().await?.to_bytes().to_vec())?; + local_recording_hls_url(&refreshed)?; + assert!(app_state.active_users.terminate_session(crate::model::RECORDING_PROXY_USERNAME, &session_token).await); + let recreated = recording_test_response(&router, variant_url.as_str()).await?; + assert_eq!(recreated.status(), StatusCode::OK, "{output:?}"); + let recreated = String::from_utf8(recreated.into_body().collect().await?.to_bytes().to_vec())?; + local_recording_hls_url(&recreated)?; + assert!(app_state + .active_users + .get_and_update_user_session(crate::model::RECORDING_PROXY_USERNAME, &session_token) + .await + .is_some()); + let requests = requests.lock().map_err(|error| error.to_string())?.clone(); + assert!(requests.iter().any(|request| request.contains("seg1.ts")), "{requests:?}"); + assert_eq!(requests.iter().filter(|request| request.contains("12345.m3u8")).count(), 3); + assert!(requests.iter().all(|request| request.contains("x-recording-test: custom")), "{requests:?}"); + origin_task.abort(); + } + Ok(()) +} + +#[tokio::test] +async fn transient_resource_with_valid_lease_streams_origin_response_and_headers() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin().await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + { + let mut session = session.write().await; + session.origin_request_headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); + session.origin_request_headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); + session + .origin_request_headers + .insert(HeaderName::from_static("proxy-authorization"), HeaderValue::from_static("Basic secret")); + session.origin_request_headers.insert(header::HOST, HeaderValue::from_static("proxy.example.com")); + session + .origin_request_headers + .insert(HeaderName::from_static("x-tuliprox-main-revision"), HeaderValue::from_static("secret")); + session.origin_request_headers.insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("de")); + } + + let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-15")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp2t"); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-15/16"); + assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + assert_eq!(response.headers()[header::ETAG], "\"abc\""); + assert_eq!(response.headers()[header::LAST_MODIFIED], "Wed, 21 Oct 2015 07:28:00 GMT"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"transient-body")); + assert!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await.is_some()); + let origin_requests = origin.requests.lock().await; + let origin_request = origin_requests.first().expect("origin request").to_ascii_lowercase(); + assert!(origin_request.contains("range: bytes=2-15")); + assert!(origin_request.contains("accept-language: de")); + assert!(!origin_request.contains("authorization: bearer secret")); + assert!(!origin_request.contains("cookie: sid=secret")); + assert!(!origin_request.contains("proxy-authorization: basic secret")); + assert!(!origin_request.contains("host: proxy.example.com")); + assert!(!origin_request.contains("x-tuliprox-main-revision")); + assert_hls_cache_stream_registered(&app_state, &proxy_session_id).await; +} + +#[test] +fn transient_cross_origin_redirect_strips_sensitive_headers() { + let mut headers = HeaderMap::new(); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); + headers.insert(header::COOKIE, HeaderValue::from_static("session=secret")); + headers.insert(HeaderName::from_static("proxy-authorization"), HeaderValue::from_static("Basic secret")); + headers.insert(header::HOST, HeaderValue::from_static("origin.example.com")); + headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-1")); + + crate::api::model::scrub_hls_origin_headers(&mut headers, None); + + assert!(!headers.contains_key(header::AUTHORIZATION)); + assert!(!headers.contains_key(header::COOKIE)); + assert!(!headers.contains_key("proxy-authorization")); + assert!(!headers.contains_key(header::HOST)); + assert_eq!(headers[header::RANGE], "bytes=0-1"); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/lease_lifecycle.rs b/backend/app/src/api/endpoints/hls_api/tests/lease_lifecycle.rs new file mode 100644 index 000000000..1d2ff6e26 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/lease_lifecycle.rs @@ -0,0 +1,668 @@ +use super::{ + access_lease_id_from_variant_uri, access_lease_session_token, create_bound_hls_test_session, enable_hls_cache, + get_response, hls_proxy_uri, map_ready_segment, media_uri_count, normal_manifest_body, overlap_provider_input, + proxy_session_id_from_variant_uri, response_body, single_variant_uri, store_normal_manifest_body, test_app_state, + test_app_state_with_hls_proxy, test_app_state_with_inputs, test_fingerprint, transient_manifest_body, + try_test_hls_cached_manifest_response, +}; +use crate::{ + api::model::{ + ConnectionKind, HlsAccessAdmissionMode, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, + HlsAccessLeaseTiming, HlsAccessLeaseValidationError, HlsLifecycleEvent, HlsLifecycleEventKey, + HlsOriginAccountBindingMode, HlsPlaybackFamilyKey, HlsProxyManager, HlsSessionKey, HlsSessionMode, + ProxySessionId, + }, + model::{ConfigInput, HlsCacheConfig, ProxyUserCredentials, StripConfig}, +}; +use axum::http::{header, StatusCode}; +use shared::model::{HlsCacheConfigDto, HlsSegmentRepairMode, HlsStripMode, UserConnectionPermission}; +use std::{sync::Arc, time::Duration}; + +#[tokio::test] +async fn hls_access_lease_validity_uses_session_idle_timeout_not_cache_duration() { + let hls_dto = HlsCacheConfigDto { + cache_duration: shared::model::Secs::new(900), + session_idle_timeout: shared::model::Secs::new(42), + ..Default::default() + }; + let hls_config = HlsCacheConfig::from(&hls_dto); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); + + assert_eq!(super::super::hls_access_lease_ttl_ms(&app_state), 42_000); +} + +#[tokio::test] +async fn hls_access_lease_active_window_uses_two_target_durations() { + let app_state = test_app_state(); + let key = HlsSessionKey::new(1, "access-window-stream"); + let (session, _) = app_state + .hls + .proxy + .get_or_create_session_with_outcome(key, b"secret", super::super::current_time_millis()) + .await; + session.write().await.target_duration = Some(11); + + let timing = super::super::hls_access_lease_timing_for_session(&app_state, &session).await; + + assert_eq!(timing.active_window_ms, 22_000); + assert_eq!(timing.valid_window_ms, super::super::hls_access_lease_ttl_ms(&app_state)); +} + +#[tokio::test] +async fn hls_lifecycle_active_timer_moves_access_lease_to_idle() { + let app_state = test_app_state(); + let now_ms = super::super::current_time_millis(); + let lease_id = HlsAccessLeaseId("lifecycle-lease".to_string()); + let key = HlsSessionKey::new(1, "lifecycle-stream"); + let (session, _) = app_state.hls.proxy.get_or_create_session_with_outcome(key, b"secret", now_ms).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", "client"), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "lifecycle-stream".to_string(), + 123, + now_ms, + 60_000, + )) + .await; + assert!(app_state + .hls + .proxy + .activate_access_lease( + &lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 1, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + session.write().await.activity.active_access_lease_count = 1; + + app_state + .hls + .proxy + .handle_lifecycle_event( + &app_state.active_users, + &app_state.active_provider, + HlsLifecycleEvent { + key: HlsLifecycleEventKey::AccessLeaseActive { + lease_id: lease_id.clone(), + proxy_session_id: proxy_session_id.clone(), + }, + due_at_ms: now_ms.saturating_add(1), + }, + now_ms.saturating_add(2), + ) + .await; + + assert_eq!( + app_state.hls.proxy.access_leases().write().await.lease_state(&lease_id, now_ms.saturating_add(2)), + Some(HlsAccessLeaseState::Idle) + ); + assert_eq!(session.read().await.activity.active_access_lease_count, 0); +} + +#[tokio::test] +async fn hls_lifecycle_validity_timer_removes_expired_access_lease() { + let mut hls_cache = HlsCacheConfigDto::default(); + hls_cache.segment_repair.max_level = HlsSegmentRepairMode::Low; + let hls_config = HlsCacheConfig::from(&hls_cache); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); + let now_ms = super::super::current_time_millis(); + let proxy_session_id = ProxySessionId("lifecycle-validity-proxy".to_string()); + let lease_id = HlsAccessLeaseId("lifecycle-validity-lease".to_string()); + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", "client"), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "lifecycle-stream".to_string(), + 123, + now_ms, + 1, + )) + .await; + assert!(app_state + .hls + .proxy + .activate_access_lease( + &lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 1, valid_window_ms: 1 }, + ) + .await + .is_activated()); + let repair_before = app_state.hls.proxy.segment_repair().stats().await; + assert_eq!(repair_before.windows, 1); + assert_eq!(repair_before.generations, 1); + + app_state + .hls + .proxy + .handle_lifecycle_event( + &app_state.active_users, + &app_state.active_provider, + HlsLifecycleEvent { + key: HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: lease_id.clone(), + proxy_session_id: proxy_session_id.clone(), + }, + due_at_ms: now_ms.saturating_add(1), + }, + now_ms.saturating_add(2), + ) + .await; + + assert_eq!( + app_state.hls.proxy.access_leases().write().await.lease_state(&lease_id, now_ms.saturating_add(2)), + None + ); + let repair_after = app_state.hls.proxy.segment_repair().stats().await; + assert_eq!(repair_after.windows, 0); + assert_eq!(repair_after.generations, 0); +} + +#[tokio::test] +async fn hls_lifecycle_validity_timer_removes_expired_pending_access_lease() { + let app_state = test_app_state(); + let now_ms = super::super::current_time_millis(); + let key = HlsSessionKey::new(1, "pending-expiry-stream"); + let (session, _) = app_state.hls.proxy.get_or_create_session_with_outcome(key, b"secret", now_ms).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("pending-expiry-lease".to_string()); + let active_lease_id = HlsAccessLeaseId("active-soft-lease".to_string()); + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", "client"), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "pending-expiry-stream".to_string(), + 123, + now_ms, + 1, + )) + .await; + app_state + .hls + .proxy + .prepare_access_lease( + HlsAccessLease::pending( + active_lease_id.clone(), + HlsPlaybackFamilyKey::new("soft-user", "client"), + proxy_session_id.clone(), + "soft-user".to_string(), + "soft-session-token".to_string(), + 1, + "pending-expiry-stream".to_string(), + 123, + now_ms, + 60_000, + ) + .with_origin_acquire_policy(ConnectionKind::Soft, 20), + ) + .await; + assert!(app_state + .hls + .proxy + .activate_access_lease( + &active_lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 60_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + + app_state + .hls + .proxy + .handle_lifecycle_event( + &app_state.active_users, + &app_state.active_provider, + HlsLifecycleEvent { + key: HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: lease_id.clone(), + proxy_session_id: proxy_session_id.clone(), + }, + due_at_ms: now_ms.saturating_add(1), + }, + now_ms.saturating_add(2), + ) + .await; + + assert_eq!( + app_state.hls.proxy.access_leases().write().await.lease_state(&lease_id, now_ms.saturating_add(2)), + None + ); + let session = session.read().await; + assert_eq!(session.activity.active_access_lease_count, 1); + let effective_policy = session.effective_origin_acquire_policy_or_default(); + assert_eq!(effective_policy.connection_kind, ConnectionKind::Soft); + assert_eq!(effective_policy.priority, 20); +} + +#[tokio::test] +async fn hls_lifecycle_session_idle_timer_removes_idle_session() { + let mut hls_dto = HlsCacheConfigDto { session_idle_timeout: shared::model::Secs::new(1), ..Default::default() }; + hls_dto.segment_repair.max_level = HlsSegmentRepairMode::Low; + let hls_config = HlsCacheConfig::from(&hls_dto); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); + let now_ms = super::super::current_time_millis(); + let key = HlsSessionKey::new(1, "expired-session"); + let (session, _) = + app_state.hls.proxy.get_or_create_session_with_outcome(key, b"secret", now_ms.saturating_sub(2_000)).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("session-idle-cleanup-lease".to_string()); + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", "client"), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "expired-session".to_string(), + 123, + now_ms, + 60_000, + )) + .await; + assert!(app_state + .hls + .proxy + .activate_access_lease( + &lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + assert_eq!(app_state.hls.proxy.access_leases().read().await.len(), 1); + assert_eq!(app_state.hls.proxy.segment_repair().stats().await.windows, 1); + + app_state + .hls + .proxy + .handle_lifecycle_event( + &app_state.active_users, + &app_state.active_provider, + HlsLifecycleEvent { + key: HlsLifecycleEventKey::SessionIdle { proxy_session_id: proxy_session_id.clone() }, + due_at_ms: now_ms.saturating_sub(1_000), + }, + now_ms, + ) + .await; + + assert!(app_state.hls.proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.is_none()); + assert_eq!(app_state.hls.proxy.access_leases().read().await.len(), 0); + let repair_after = app_state.hls.proxy.segment_repair().stats().await; + assert_eq!(repair_after.windows, 0); + assert_eq!(repair_after.generations, 0); +} + +#[tokio::test] +async fn hls_gc_session_removal_cleans_access_leases_and_repair_state() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let mut hls_dto = HlsCacheConfigDto { + cache_path: Some(temp_dir.path().to_string_lossy().into_owned()), + session_idle_timeout: shared::model::Secs::new(1), + ..Default::default() + }; + hls_dto.segment_repair.max_level = HlsSegmentRepairMode::Low; + let hls_config = HlsCacheConfig::from(&hls_dto); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); + let now_ms = super::super::current_time_millis(); + let key = HlsSessionKey::new(1, "gc-cleanup-session"); + let (session, _) = + app_state.hls.proxy.get_or_create_session_with_outcome(key, b"secret", now_ms.saturating_sub(2_000)).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("gc-cleanup-lease".to_string()); + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", "client"), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "gc-cleanup-session".to_string(), + 123, + now_ms, + 60_000, + )) + .await; + assert!(app_state + .hls + .proxy + .activate_access_lease( + &lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + assert_eq!(app_state.hls.proxy.access_leases().read().await.len(), 1); + assert_eq!(app_state.hls.proxy.segment_repair().stats().await.windows, 1); + + let report = app_state.hls.proxy.run_garbage_collection_once(now_ms).await.expect("gc should run"); + + assert_eq!(report.sessions_deleted, 1); + assert!(app_state.hls.proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.is_none()); + assert_eq!(app_state.hls.proxy.access_leases().read().await.len(), 0); + let repair_after = app_state.hls.proxy.segment_repair().stats().await; + assert_eq!(repair_after.windows, 0); + assert_eq!(repair_after.generations, 0); +} + +#[tokio::test] +async fn hls_access_lease_sync_zero_to_zero_is_idempotent() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let old_generation = session.read().await.activity.origin_work_generation; + + app_state + .hls + .proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + &proxy_session_id, + 3_000, + ) + .await; + + let session = session.read().await; + assert_eq!(session.activity.active_access_lease_count, 0); + assert!(matches!( + session.origin_account_binding.as_ref().expect("binding exists").binding_mode, + HlsOriginAccountBindingMode::Active + )); + assert_eq!(session.activity.origin_work_generation, old_generation); +} + +#[tokio::test] +async fn hls_cache_entry_denies_access_lease_for_grace_without_slot_and_exhausted() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let user = app_state.app_config.get_user_credentials("hls-user").expect("test user should exist"); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + + for (connection_permission, connection_kind) in [ + (UserConnectionPermission::GracePeriod, None), + (UserConnectionPermission::Exhausted, Some(ConnectionKind::Normal)), + ] { + let response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + origin_source.clone(), + 12345, + None, + None, + None, + request_url, + &input, + connection_permission, + connection_kind, + Some("/iptv"), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let variant_uri = single_variant_uri(response).await; + let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&variant_uri).to_string()); + let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&variant_uri).to_string()); + let now_ms = super::super::current_time_millis(); + let snapshot = app_state + .hls + .proxy + .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, now_ms) + .await + .expect("denied lease should stay available for response rendering"); + assert_eq!(snapshot.state, HlsAccessLeaseState::Denied); + assert!(app_state + .hls + .proxy + .access_lease_session_snapshot(&proxy_session_id, now_ms) + .await + .effective_origin_policy + .is_none()); + + let err = super::super::validate_hls_proxy_access_context( + &app_state, + &test_fingerprint(), + &proxy_session_id, + &access_lease_id.0, + now_ms, + HlsAccessAdmissionMode::ManifestPrepare, + ) + .await + .expect_err("denied lease must surface as admission denied"); + assert!(matches!(err, HlsAccessLeaseValidationError::AdmissionDenied { runtime_tail: None, .. })); + } +} + +#[tokio::test] +async fn hls_cache_entry_does_not_reuse_activated_access_lease() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + + let first_response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + origin_source.clone(), + 12345, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + let first_variant_uri = single_variant_uri(first_response).await; + let first_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&first_variant_uri).to_string()); + let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&first_variant_uri).to_string()); + let now_ms = super::super::current_time_millis(); + assert!(app_state + .hls + .proxy + .activate_access_lease( + &first_lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { + active_window_ms: 5_000, + valid_window_ms: super::super::hls_access_lease_ttl_ms(&app_state), + }, + ) + .await + .is_activated()); + + let second_response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 12345, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + let second_variant_uri = single_variant_uri(second_response).await; + let second_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&second_variant_uri).to_string()); + + assert_ne!(first_lease_id, second_lease_id); + let first_session_token = access_lease_session_token(&app_state, &proxy_session_id, &first_lease_id).await; + let second_session_token = access_lease_session_token(&app_state, &proxy_session_id, &second_lease_id).await; + assert_ne!(first_session_token, second_session_token); + assert!( + app_state + .hls + .proxy + .touch_access_lease( + &first_lease_id, + super::super::current_time_millis(), + HlsAccessLeaseTiming { + active_window_ms: 5_000, + valid_window_ms: super::super::hls_access_lease_ttl_ms(&app_state), + }, + ) + .await + ); +} + +#[tokio::test] +async fn hls_cache_idle_normal_manifest_applies_initial_strip_without_mutating_shared_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::super::current_time_millis(); + store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id), rendered_at_ms); + session.mark_authorized_media_access(rendered_at_ms); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + let stored_before = session.read().await.last_rendered_manifest.as_ref().expect("normal manifest").body.clone(); + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Idle, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("idle normal manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0\n")); + assert!(body.contains("/000000.ts")); + assert!(body.contains("/000001.ts")); + assert!(body.contains("/000002.ts")); + assert!(!body.contains("/000003.ts")); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + assert_eq!(session.read().await.last_rendered_manifest.as_ref().expect("stored manifest").body, stored_before); + assert_eq!(media_uri_count(&stored_before), 6); +} + +#[tokio::test] +async fn hls_cache_idle_transient_manifest_applies_initial_strip_without_mutating_shared_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.0.clone(); + session.transient.replace_manifest_with_semantics(transient_manifest_body(&proxy_session_id), 100, None); + session.mark_authorized_media_access(super::super::current_time_millis()); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + let stored_before = session.read().await.transient.last_manifest_body.clone().expect("transient manifest body"); + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Idle, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("idle transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + assert_eq!(session.read().await.transient.last_manifest_body.as_ref().expect("stored manifest"), &stored_before); + assert_eq!(media_uri_count(&stored_before), 6); +} + +#[tokio::test] +async fn ready_hls_proxy_segment_marked_for_gc_returns_not_found_without_redirect() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + session.write().await.mark_for_gc_removal(); + + let response = get_response(app_state, &uri, None).await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!(!response.headers().contains_key(header::LOCATION)); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/manifest_startup.rs b/backend/app/src/api/endpoints/hls_api/tests/manifest_startup.rs new file mode 100644 index 000000000..fb3ae44cc --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/manifest_startup.rs @@ -0,0 +1,1557 @@ +use super::{ + access_lease_id_from_variant_uri, access_lease_session_token, assert_no_hls_cache_stream_registered, + cache_recording_test_item, cache_test_m3u_hls_item, configure_default_test_server, + configure_recording_test_listener, create_active_hls_user_session, disable_custom_stream_response, + enable_channel_unavailable_custom_response, enable_hls_cache, get_response, get_status, + hls_availability_reevaluation_registration_failure_response, hls_canonical_owner_registration, hls_proxy_uri, + hls_session_last_media_at_ms, legacy_manifest_test_client_headers, legacy_manifest_test_input, + manifest_media_sequence, map_hls_map, map_ready_segment_without_lease, media_uri_count, normal_manifest, + normal_manifest_body, path_has_extension, prepare_pending_test_hls_access_lease, proxy_session_id_from_variant_uri, + publish_ready_test_manifest_for_lease, record_test_normal_manifest_commit, recording_test_response, + recording_test_router, recording_test_url, response_body, single_hls_provider_input, + single_variant_master_playlist, single_variant_uri, spawn_recording_header_origin, + spawn_test_encoded_manifest_origin, spawn_test_segment_origin, spawn_test_transient_origin_with_delayed_response, + store_normal_manifest_body, store_test_sources_with_target, test_addr_with_port, test_app_state, + test_app_state_with_hls_proxy, test_app_state_with_hls_proxy_and_inputs, test_app_state_with_inputs, + test_fingerprint, test_fingerprint_with_addr, test_hls_access_context, test_hls_entry_stream_context, + test_hls_input, test_hls_share_target, test_m3u_hls_item, test_m3u_hls_share_target, test_segment_entry, + transient_manifest_body_from_sequence, try_test_hls_cached_manifest_response, HlsCanonicalOwnerRegistration, + HlsCanonicalOwnerRegistrationFailure, HlsCanonicalOwnerRegistrationKind, TestSegmentOrigin, +}; +use crate::{ + api::model::{ + build_proxy_session_id, AppState, ConnectionKind, HlsAccessContext, HlsAccessLeaseId, HlsAccessLeaseState, + HlsAvailabilityReevaluationRegistration, HlsFreshManifestRequiredReason, HlsLeasePlaybackMode, + HlsManifestCommitIdentity, HlsManifestCommitRequirement, HlsOriginAccountBinding, HlsOriginSourceKind, + HlsPlaybackFamilyKey, HlsProxyManager, HlsSession, HlsSessionHandle, HlsSessionKey, HlsSessionStoreOutcome, + MapCacheStatus, MapEntry, OriginMapKey, ProxyMapId, ProxySessionId, RenderedManifest, SegmentCacheStatus, + }, + auth::Fingerprint, + model::{ConfigInput, ConfigTarget, HlsCacheConfig, ProxyUserCredentials, StripConfig}, +}; +use axum::{ + body::Body, + http::{header, HeaderMap, Response, StatusCode}, + response::IntoResponse, +}; +use shared::model::{ + HlsCacheConfigDto, HlsStripMode, InputType, PlaylistItemType, UserConnectionPermission, XtreamCluster, +}; +use std::{sync::Arc, time::Duration}; +use tokio::sync::RwLock; +use tuliprox_hls::api::MAX_HLS_MANIFEST_BYTES; + +#[test] +fn canonical_manifest_joins_authoritative_owner_and_fails_closed_without_one() { + assert_eq!( + hls_canonical_owner_registration(HlsAvailabilityReevaluationRegistration::Scheduled), + HlsCanonicalOwnerRegistration::Join(HlsCanonicalOwnerRegistrationKind::Scheduled) + ); + assert_eq!( + hls_canonical_owner_registration(HlsAvailabilityReevaluationRegistration::AlreadyOwned), + HlsCanonicalOwnerRegistration::Join(HlsCanonicalOwnerRegistrationKind::AlreadyOwned) + ); + assert_eq!( + hls_canonical_owner_registration(HlsAvailabilityReevaluationRegistration::Superseded), + HlsCanonicalOwnerRegistration::Join(HlsCanonicalOwnerRegistrationKind::AlreadyOwned) + ); + for (registration, failure) in [ + ( + HlsAvailabilityReevaluationRegistration::CapacityExceeded, + HlsCanonicalOwnerRegistrationFailure::CapacityExceeded, + ), + ( + HlsAvailabilityReevaluationRegistration::RuntimeUnavailable, + HlsCanonicalOwnerRegistrationFailure::RuntimeUnavailable, + ), + ] { + assert_eq!(hls_canonical_owner_registration(registration), HlsCanonicalOwnerRegistration::FailClosed(failure)); + let response = hls_availability_reevaluation_registration_failure_response(failure); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response.headers().contains_key(header::RETRY_AFTER)); + } +} + +#[tokio::test] +async fn hls_initial_manifest_decision_wait_timeout_defaults_to_ninety_seconds() { + let app_state = test_app_state(); + assert_eq!(super::super::hls_initial_manifest_decision_wait_timeout(&app_state), Duration::from_secs(90)); +} + +#[tokio::test] +async fn hls_manifest_channel_unavailable_renders_inline_without_redirect() { + let app_state = test_app_state(); + enable_channel_unavailable_custom_response(&app_state); + + let response = super::super::hls_manifest_channel_unavailable_response_for_username(&app_state, "hls-user").await; + + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + assert!(body.contains("#EXT-X-ENDLIST")); +} + +#[tokio::test] +async fn hls_manifest_channel_unavailable_falls_back_to_not_found_when_custom_response_is_disabled() { + let app_state = test_app_state(); + disable_custom_stream_response(&app_state); + + let response = super::super::hls_manifest_channel_unavailable_response_for_username(&app_state, "hls-user").await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn legacy_hls_manifest_deadline_includes_full_body_read() { + let origin = spawn_test_encoded_manifest_origin(None, b"#EXTM3U\n".to_vec(), Duration::from_millis(100)).await; + let input = legacy_manifest_test_input(&origin); + let hls_config = HlsCacheConfig::from(&HlsCacheConfigDto { + origin_manifest_timeout_ms: shared::model::Millis::new(10), + ..Default::default() + }); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config))); + + let error = super::super::download_legacy_hls_manifest(&app_state, &input, &legacy_manifest_test_client_headers()) + .await + .expect_err("complete manifest body read must honor the deadline"); + + assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); +} + +#[tokio::test] +async fn legacy_hls_manifest_distinguishes_invalid_utf8_from_decoder_failure() { + let invalid_utf8_origin = spawn_test_encoded_manifest_origin(None, vec![0xff], Duration::ZERO).await; + let invalid_utf8_input = legacy_manifest_test_input(&invalid_utf8_origin); + let invalid_utf8 = super::super::download_legacy_hls_manifest( + &test_app_state(), + &invalid_utf8_input, + &legacy_manifest_test_client_headers(), + ) + .await + .expect_err("invalid UTF-8 must fail"); + + let corrupt_origin = + spawn_test_encoded_manifest_origin(Some("gzip"), vec![0x1f, 0x8b, 0x08, 0x00], Duration::ZERO).await; + let corrupt_input = legacy_manifest_test_input(&corrupt_origin); + let decoder_failure = super::super::download_legacy_hls_manifest( + &test_app_state(), + &corrupt_input, + &legacy_manifest_test_client_headers(), + ) + .await + .expect_err("corrupt gzip must fail"); + + assert_eq!(invalid_utf8.kind(), std::io::ErrorKind::InvalidData); + assert!(crate::utils::content_coding::content_decoding_error_from_io(&decoder_failure).is_some()); +} + +#[test] +fn hls_manifest_extension_helper_does_not_create_double_dot_urls() { + assert_eq!( + super::super::ensure_hls_manifest_extension("http://origin.example.com/live/user/pass/1025123.m3u8"), + "http://origin.example.com/live/user/pass/1025123.m3u8" + ); + assert_eq!( + super::super::ensure_hls_manifest_extension("http://origin.example.com/live/user/pass/1025123..m3u8"), + "http://origin.example.com/live/user/pass/1025123.m3u8" + ); + assert_eq!( + super::super::ensure_hls_manifest_extension("http://origin.example.com/live/user/pass/1025123..?token=1"), + "http://origin.example.com/live/user/pass/1025123.m3u8?token=1" + ); + assert_eq!( + super::super::ensure_hls_manifest_extension("provider://mirror/live/user/pass/1025123..m3u8"), + "provider://mirror/live/user/pass/1025123.m3u8" + ); +} + +#[tokio::test] +async fn lease_snapshot_limit_rejection_is_controlled_and_observable() { + let app_state = test_app_state(); + let proxy_session_id = ProxySessionId("snapshot-limit-session".to_string()); + let access_lease_id = HlsAccessLeaseId("snapshot-limit-lease".to_string()); + let oversized_uri = "x".repeat(MAX_HLS_MANIFEST_BYTES + 1); + let body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{oversized_uri}\n"); + let derivation = super::derive_hls_lease_manifest_snapshot( + &super::HlsLeaseManifestSnapshotInput::TransientPassthrough { + materialized_body: &body, + source_commit_identity: HlsManifestCommitIdentity::committed(1, 1), + finalized_manifest_generation: None, + }, + 2, + ); + + assert!(super::super::observe_hls_lease_manifest_snapshot_derivation( + &app_state, + &proxy_session_id, + &access_lease_id, + derivation, + ) + .is_err()); + assert_eq!(app_state.hls.proxy.metrics().snapshot().manifest_limit_rejections, 1); +} + +#[tokio::test] +async fn hls_hard_manifest_failure_forces_next_fresh_commit() { + let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); + { + let mut session = session.write().await; + session.last_rendered_manifest = Some(RenderedManifest { + body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), + first_proxy_seq: 1, + last_proxy_seq: 1, + playlist_duration_ms: 4_000, + valid_until_ms: 5_000, + render_gap_segments: 0, + rendered_at_ms: 1_000, + discontinuity_sequence: 0, + target_duration_ms: 4_000, + segment_proxy_seqs: vec![1], + }); + session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::PreviousHardManifestFailure); + } + + assert_eq!( + super::hls_manifest_commit_requirement(&session, HlsSessionStoreOutcome::Reused, None, 2_000).await, + HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::PreviousHardManifestFailure + } + ); +} + +#[tokio::test] +async fn hls_normal_expired_session_allows_committed_manifest_while_manifest_valid() { + let now_ms = 100_000; + let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); + { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.clone(); + session.target_duration = Some(10); + session.mark_authorized_media_access(1_000); + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::from("test-input"), + Arc::from("test-account"), + &proxy_session_id, + now_ms, + )); + session.last_rendered_manifest = Some(RenderedManifest { + body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), + first_proxy_seq: 1, + last_proxy_seq: 1, + playlist_duration_ms: 4_000, + valid_until_ms: now_ms.saturating_add(10_000), + render_gap_segments: 0, + rendered_at_ms: now_ms.saturating_sub(1_000), + discontinuity_sequence: 0, + target_duration_ms: 4_000, + segment_proxy_seqs: vec![1], + }); + } + + assert_eq!( + super::hls_manifest_commit_requirement(&session, HlsSessionStoreOutcome::Reused, None, now_ms).await, + HlsManifestCommitRequirement::CommittedManifestAllowed + ); +} + +#[tokio::test] +async fn hls_normal_expired_session_requires_fresh_commit_after_manifest_validity() { + let now_ms = 100_000; + let session = Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))); + { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.clone(); + session.target_duration = Some(10); + session.mark_authorized_media_access(1_000); + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::from("test-input"), + Arc::from("test-account"), + &proxy_session_id, + now_ms, + )); + session.last_rendered_manifest = Some(RenderedManifest { + body: "#EXTM3U\n#EXTINF:4.0,\n000001.ts\n".to_string(), + first_proxy_seq: 1, + last_proxy_seq: 1, + playlist_duration_ms: 4_000, + valid_until_ms: now_ms.saturating_sub(1), + render_gap_segments: 0, + rendered_at_ms: now_ms.saturating_sub(10_000), + discontinuity_sequence: 0, + target_duration_ms: 4_000, + segment_proxy_seqs: vec![1], + }); + } + + assert_eq!( + super::hls_manifest_commit_requirement(&session, HlsSessionStoreOutcome::Reused, None, now_ms).await, + HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation + } + ); +} + +#[tokio::test] +async fn hls_cache_manifest_cold_start_synchronously_returns_initial_manifest() { + let origin = spawn_test_segment_origin( + b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:123\n#EXTINF:4.0,\n000123.ts\n#EXTINF:4.0,\n000124.ts\n#EXTINF:4.0,\n000125.ts\n", + ) + .await; + let input_name = Arc::::from("test-input"); + let input = ConfigInput { + id: 1, + name: Arc::clone(&input_name), + input_type: InputType::Xtream, + url: origin.base_url.clone(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + + let response = super::super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::super::HlsCacheManifestOrigin { + raw_request_url: &request_url, + session_entry_url: super::super::HlsOriginEntryUrl::direct_http(&request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + .expect("hls cache should handle valid live hls entrypoint"); + + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); + assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0")); + assert!(body.contains(&format!("/hls/shared/live/{}/{}/000000.ts", proxy_session_id.0, access_lease_id.0))); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + let session = app_state + .hls + .proxy + .sessions() + .get_by_key(&session_key) + .await + .expect("cold start should create shared hls session"); + { + let session = session.read().await; + assert!(session.last_rendered_manifest.is_some()); + let binding = session.origin_account_binding.as_ref().expect("plain http input still has account binding"); + assert_eq!(binding.input_name.as_ref(), "test-input"); + assert_eq!(binding.account_name.as_ref(), "test-input"); + } + assert!(app_state.active_users.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn hls_cache_manifest_cold_start_supports_m3u_hls_origin_source() { + let origin = spawn_test_segment_origin( + b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:77\n#EXTINF:4.0,\nseg-77.ts\n#EXTINF:4.0,\nseg-78.ts\n#EXTINF:4.0,\nseg-79.ts\n", + ) + .await; + let input = ConfigInput { + id: 1, + name: Arc::from("m3u-input"), + input_type: InputType::M3u, + url: origin.base_url.clone(), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/channel/index.m3u8", origin.base_url); + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + + let response = super::super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::super::HlsCacheManifestOrigin { + raw_request_url: &request_url, + session_entry_url: super::super::HlsOriginEntryUrl::direct_http(&request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/m3u-stream/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + .expect("hls cache should handle m3u hls media playlists"); + + assert_eq!(response.status(), StatusCode::OK); + let session = + app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("m3u hls should create shared session"); + let session = session.read().await; + assert_eq!(session.origin_source.source_kind, HlsOriginSourceKind::M3uMediaPlaylist); + let binding = session.origin_account_binding.as_ref().expect("m3u hls input still has account binding"); + assert_eq!(binding.input_name.as_ref(), "m3u-input"); + assert_eq!(binding.account_name.as_ref(), "m3u-input"); +} + +#[tokio::test] +async fn hls_cache_manifest_cold_start_client_abort_does_not_leave_refresh_in_flight() { + let origin = spawn_test_transient_origin_with_delayed_response( + "200 OK", + &[("Content-Type", "application/vnd.apple.mpegurl")], + "#EXTM3U\n#EXT-X-VERSION:3\n", + Duration::from_millis(100), + ) + .await; + let input = ConfigInput { + id: 1, + name: Arc::from("test-input"), + input_type: InputType::Xtream, + url: origin.base_url.clone(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let hls_dto = + HlsCacheConfigDto { origin_manifest_timeout_ms: shared::model::Millis::new(1_000), ..Default::default() }; + let hls_config = HlsCacheConfig::from(&hls_dto); + let app_state = test_app_state_with_hls_proxy_and_inputs( + Arc::new(HlsProxyManager::with_hls_cache_config(&hls_config)), + vec![Arc::new(input.clone())], + ); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + + let app_state_for_request = Arc::clone(&app_state); + let request_handle = tokio::spawn(async move { + super::super::try_hls_cache_canonical_manifest_response( + &app_state_for_request, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::super::HlsCacheManifestOrigin { + raw_request_url: &request_url, + session_entry_url: super::super::HlsOriginEntryUrl::direct_http(&request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + }); + + let session = wait_for_hls_test_session(&app_state, &session_key).await; + wait_for_hls_refresh_in_flight(&session).await; + request_handle.abort(); + let _ = request_handle.await; + + for _ in 0..200 { + if !session.read().await.origin_refresh.in_flight { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + let session = session.read().await; + assert!(!session.origin_refresh.in_flight); + assert!(session.origin_refresh.last_fetch_finished_at_ms.is_some()); +} + +#[tokio::test] +async fn hls_cache_canonical_prepare_service_unavailable_sets_retry_after() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let input = ConfigInput { + id: 1, + name: Arc::from("alias-input"), + aliases: Some(vec![crate::model::ConfigInputAlias { + id: 2, + name: Arc::from("alias-account"), + url: "http://alias.example.com".to_string(), + username: Some("alias-user".to_string()), + password: Some("alias-pass".to_string()), + priority: 0, + max_connections: 1, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + + let response = super::super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::super::HlsCacheManifestOrigin { + raw_request_url: request_url, + session_entry_url: super::super::HlsOriginEntryUrl::direct_http(request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + .expect("canonical hls cache response"); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers().get(header::RETRY_AFTER).expect("retry after"), "2"); +} + +pub(in crate::api::endpoints::hls_api::tests) struct SharedRequestFlowFixture { + pub(in crate::api::endpoints::hls_api::tests) _origin: TestSegmentOrigin, + pub(in crate::api::endpoints::hls_api::tests) app_state: Arc, + pub(in crate::api::endpoints::hls_api::tests) input: ConfigInput, + pub(in crate::api::endpoints::hls_api::tests) target: ConfigTarget, + pub(in crate::api::endpoints::hls_api::tests) user: ProxyUserCredentials, + pub(in crate::api::endpoints::hls_api::tests) origin_manifest_url: String, + pub(in crate::api::endpoints::hls_api::tests) entry_path: String, +} + +pub(in crate::api::endpoints::hls_api::tests) async fn shared_request_flow_fixture() -> SharedRequestFlowFixture { + const MANIFEST: &[u8] = b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n\ + #EXT-X-MEDIA-SEQUENCE:123\n#EXTINF:4.0,\n000123.ts\n#EXTINF:4.0,\n000124.ts\n\ + #EXTINF:4.0,\n000125.ts\n#EXTINF:4.0,\n000126.ts\n#EXTINF:4.0,\n000127.ts\n\ + #EXTINF:4.0,\n000128.ts\n"; + let origin = spawn_test_segment_origin(MANIFEST).await; + let input = ConfigInput { + id: 1, + name: Arc::from("request-flow-input"), + input_type: InputType::M3u, + url: origin.base_url.clone(), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let mut target = test_m3u_hls_share_target(); + target.name = "default".to_string(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + configure_default_test_server(&app_state); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + let origin_manifest_url = format!("{}/channel/index.m3u8", origin.base_url); + cache_test_m3u_hls_item(&app_state, &target, test_m3u_hls_item(&input, 12345, "channel-a", &origin_manifest_url)) + .await; + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.password = "hls-pass".to_string(); + let entry_path = super::super::build_virtual_hls_entry_path(&target, &input, &user, 12345); + SharedRequestFlowFixture { _origin: origin, app_state, input, target, user, origin_manifest_url, entry_path } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn shared_request_flow_entry( + fixture: &SharedRequestFlowFixture, + fingerprint: &Fingerprint, +) -> Response { + super::super::handle_hls_stream_request( + fingerprint, + &fixture.app_state, + &fixture.user, + &fixture.target, + None, + None, + &fixture.origin_manifest_url, + None, + test_hls_entry_stream_context(12345, "channel-a", Some(2_500_000)), + &fixture.input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &fixture.entry_path, + super::super::HlsRequestStage::Entry, + ) + .await + .into_response() +} + +#[tokio::test] +async fn shared_hls_request_flow_keeps_media_playlist_lease_bound_across_reloads() { + let fixture = shared_request_flow_fixture().await; + let app_state = &fixture.app_state; + let entry_response = shared_request_flow_entry(&fixture, &test_fingerprint()).await; + + assert_eq!(entry_response.status(), StatusCode::OK); + assert!(!entry_response.headers().contains_key(header::LOCATION)); + let (_, media_playlist_uri) = single_variant_master_playlist(entry_response).await; + let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&media_playlist_uri).to_string()); + let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&media_playlist_uri).to_string()); + + let first_media_response = get_response(Arc::clone(app_state), &media_playlist_uri, None).await; + assert_eq!(first_media_response.status(), StatusCode::OK); + assert!(!first_media_response.headers().contains_key(header::LOCATION)); + let first_media_body = + String::from_utf8(response_body(first_media_response).await.to_vec()).expect("media playlist utf8"); + let segment_uri = first_media_body + .lines() + .find(|line| line.starts_with("/hls/shared/live/") && path_has_extension(line, "ts")) + .expect("lease-bound segment URI") + .to_string(); + let lease_path = format!("/{}/{}/", proxy_session_id.0, access_lease_id.0); + assert!(segment_uri.contains(&lease_path)); + + let reloaded_media_response = get_response(Arc::clone(app_state), &media_playlist_uri, None).await; + assert_eq!(reloaded_media_response.status(), StatusCode::OK); + assert!(!reloaded_media_response.headers().contains_key(header::LOCATION)); + let reloaded_media_body = + String::from_utf8(response_body(reloaded_media_response).await.to_vec()).expect("reloaded media playlist utf8"); + assert_eq!(manifest_media_sequence(&reloaded_media_body), manifest_media_sequence(&first_media_body)); + assert!(reloaded_media_body.lines().any(|line| line == segment_uri)); + assert_eq!(app_state.hls.proxy.access_leases().read().await.len(), 1); + + let segment_response = get_response(Arc::clone(app_state), &segment_uri, None).await; + assert_eq!(segment_response.status(), StatusCode::OK); + assert!(!response_body(segment_response).await.is_empty()); + let lease = app_state + .hls + .proxy + .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::super::current_time_millis()) + .await + .expect("request-flow access lease"); + assert_eq!(lease.state, HlsAccessLeaseState::Activated); + assert!(lease.last_manifest_snapshot.is_some()); + + let second_entry_response = + shared_request_flow_entry(&fixture, &test_fingerprint_with_addr(test_addr_with_port(55131))).await; + assert_eq!(second_entry_response.status(), StatusCode::OK); + let (_, second_media_playlist_uri) = single_variant_master_playlist(second_entry_response).await; + let second_proxy_session_id = + ProxySessionId(proxy_session_id_from_variant_uri(&second_media_playlist_uri).to_string()); + let second_access_lease_id = + HlsAccessLeaseId(access_lease_id_from_variant_uri(&second_media_playlist_uri).to_string()); + assert_eq!(second_proxy_session_id, proxy_session_id); + assert_ne!(second_access_lease_id, access_lease_id); + let second_pending_lease = app_state + .hls + .proxy + .access_lease_response_snapshot( + &second_access_lease_id, + &second_proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("second pending request-flow lease"); + assert_eq!(second_pending_lease.state, HlsAccessLeaseState::Pending); + assert!(second_pending_lease.last_manifest_snapshot.is_none()); + assert_ne!(second_pending_lease.user_session_token, lease.user_session_token); + + let second_media_response = get_response(Arc::clone(app_state), &second_media_playlist_uri, None).await; + assert_eq!(second_media_response.status(), StatusCode::OK); + assert!(!second_media_response.headers().contains_key(header::LOCATION)); + let second_media_body = + String::from_utf8(response_body(second_media_response).await.to_vec()).expect("second media playlist utf8"); + assert_eq!(manifest_media_sequence(&second_media_body), manifest_media_sequence(&first_media_body)); + let second_published_lease = app_state + .hls + .proxy + .access_lease_response_snapshot( + &second_access_lease_id, + &second_proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("second published request-flow lease"); + assert!(second_published_lease.last_manifest_snapshot.is_some()); + assert_eq!(second_published_lease.playback_mode, HlsLeasePlaybackMode::Live); + assert_eq!(app_state.hls.proxy.sessions().len().await, 1); +} + +#[tokio::test] +async fn hls_virtual_source_resolver_rejects_missing_input_stream_id_with_service_unavailable() { + let input = single_hls_provider_input("missing-origin-id-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let target = Arc::new(test_m3u_hls_share_target()); + let item = + test_m3u_hls_item(&input, 1001, "", "http://account.example.com/live/account-user/account-pass/channel.m3u8"); + cache_test_m3u_hls_item(&app_state, &target, item).await; + + let status = super::super::resolve_hls_virtual_source_for_target(&app_state, &target, 1001) + .await + .expect_err("missing input stream identity must fail safely"); + + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); +} + +#[tokio::test] +async fn recording_m3u_dash_keeps_the_redirect_for_relative_media_urls() -> Result<(), Box> { + let (origin_url, requests, origin_task) = spawn_recording_header_origin().await; + let input = ConfigInput { + id: 1, + name: Arc::from("recording-input"), + input_type: InputType::M3u, + url: origin_url.clone(), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let mut target = test_m3u_hls_share_target(); + target.name = "default".to_string(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + let storage = tempfile::tempdir()?; + configure_recording_test_listener(&app_state, storage.path()); + let router = recording_test_router(&app_state); + let manifest_url = format!("{origin_url}/live/manifest.mpd"); + for item_type in [PlaylistItemType::LiveDash, PlaylistItemType::Live] { + let mut item = test_m3u_hls_item(&input, 12345, "12345", &manifest_url); + item.item_type = item_type; + cache_recording_test_item(&app_state, &target, item).await?; + let response = + recording_test_response(&router, &recording_test_url(&app_state, &target, &input, XtreamCluster::Live)?) + .await?; + assert!(response.status().is_redirection(), "{item_type:?}: {}", response.status()); + assert_eq!( + response.headers().get(header::LOCATION).ok_or("redirect location missing")?.to_str()?, + manifest_url + ); + } + assert!(requests.lock().map_err(|error| error.to_string())?.is_empty()); + origin_task.abort(); + Ok(()) +} + +#[tokio::test] +async fn hls_cache_canonical_manifest_rejects_when_target_hls_share_disabled() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let input = test_hls_input(); + let target = test_hls_share_target(false); + store_test_sources_with_target(&app_state, input, target); + let access_context = HlsAccessContext { + username: "hls-user".to_string(), + user_session_token: "hls-session-token".to_string(), + proxy_session_id: ProxySessionId("proxy-session".to_string()), + input_id: 1, + stream_ref: "12345".to_string(), + virtual_id: 12345, + known_bitrate_bps: None, + lease_id: HlsAccessLeaseId("access-lease".to_string()), + family_key: HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), + epg_reference_ts: None, + archive_origin_url: None, + }; + + let Err(err) = + super::super::resolve_hls_playback_manifest_request_context(&app_state, &access_context, &HeaderMap::new()) + .await + else { + panic!("disabled target hls sharing should reject canonical cache path"); + }; + + assert_eq!(err, StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn hls_cache_parallel_real_playbacks_same_virtual_id_register_distinct_streams() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = ConfigInput { id: 1, name: Arc::from("test-input"), ..Default::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let proxy_session_id = map_ready_segment_without_lease(&app_state, 123, "ts", b"0123456789").await; + + let first_response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + origin_source.clone(), + 12345, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + let first_variant_uri = single_variant_uri(first_response).await; + let first_lease_id = access_lease_id_from_variant_uri(&first_variant_uri); + assert_eq!(proxy_session_id_from_variant_uri(&first_variant_uri), proxy_session_id); + let proxy_session = ProxySessionId(proxy_session_id.clone()); + let first_session_token = + access_lease_session_token(&app_state, &proxy_session, &HlsAccessLeaseId(first_lease_id.to_string())).await; + publish_ready_test_manifest_for_lease( + &app_state, + &proxy_session, + &HlsAccessLeaseId(first_lease_id.to_string()), + 4_000, + ) + .await; + let first_segment_uri = format!("/hls/shared/live/{proxy_session_id}/{first_lease_id}/000123.ts"); + assert_eq!(get_status(Arc::clone(&app_state), &first_segment_uri).await, StatusCode::OK); + + let second_response = super::super::create_hls_cache_entry_master_playlist_response( + &app_state, + &test_fingerprint(), + &user, + origin_source, + 12345, + None, + None, + None, + request_url, + &input, + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + None, + ) + .await; + let second_variant_uri = single_variant_uri(second_response).await; + let second_lease_id = access_lease_id_from_variant_uri(&second_variant_uri); + assert_ne!(first_lease_id, second_lease_id); + assert_eq!(proxy_session_id_from_variant_uri(&second_variant_uri), proxy_session_id); + let second_session_token = + access_lease_session_token(&app_state, &proxy_session, &HlsAccessLeaseId(second_lease_id.to_string())).await; + assert_ne!(first_session_token, second_session_token); + publish_ready_test_manifest_for_lease( + &app_state, + &proxy_session, + &HlsAccessLeaseId(second_lease_id.to_string()), + 4_000, + ) + .await; + let second_segment_uri = format!("/hls/shared/live/{proxy_session_id}/{second_lease_id}/000123.ts"); + assert_eq!(get_status(Arc::clone(&app_state), &second_segment_uri).await, StatusCode::OK); + + let streams = app_state.active_users.active_streams().await; + assert_eq!(streams.len(), 2); + let first_stream = streams + .iter() + .find(|stream| stream.session_token.as_deref() == Some(first_session_token.as_str())) + .expect("first stream should be registered"); + let second_stream = streams + .iter() + .find(|stream| stream.session_token.as_deref() == Some(second_session_token.as_str())) + .expect("second stream should be registered"); + let shared_stream_id = super::super::hls_cache_shared_stream_id(&proxy_session); + assert_ne!(first_stream.session_token, second_stream.session_token); + assert_eq!(first_stream.channel.shared_stream_id, Some(shared_stream_id)); + assert_eq!(second_stream.channel.shared_stream_id, Some(shared_stream_id)); + assert_eq!(first_stream.channel.shared_joined_existing, Some(false)); + assert_eq!(second_stream.channel.shared_joined_existing, Some(true)); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn prepare_server_path_manifest_session( + app_state: &Arc, +) -> (HlsSessionHandle, ProxySessionId) { + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let proxy_session_id = { + let mut session = session.write().await; + session.origin_refresh.next_fetch_allowed_at_ms = u64::MAX; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::super::current_time_millis(); + let map_id = ProxyMapId(0); + let mut map = MapEntry::new( + &session.proxy_session_id, + map_id, + OriginMapKey { + origin_epoch: 0, + resolved_origin_uri: "http://origin.example.com/init.mp4".to_string(), + byte_range: None, + }, + "mp4".to_string(), + ); + map.status = MapCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }; + session.maps.insert(map_id, map); + let mut segment = test_segment_entry( + &session.proxy_session_id, + 123, + SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }, + ); + segment.map_ref = Some(map_id); + session.segments.insert(123, segment); + session.advance_media_readiness_generation(); + session.last_rendered_manifest = Some(RenderedManifest { + body: format!( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:123\n#EXT-X-MAP:URI=\"/hls/shared/live/{proxy_session_id}/{}/map/000000.mp4\"\n#EXTINF:4.0,\n/hls/shared/live/{proxy_session_id}/{}/000123.ts\n", + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER, + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER + ), + first_proxy_seq: 123, + last_proxy_seq: 123, + playlist_duration_ms: 4_000, + valid_until_ms: rendered_at_ms.saturating_add(4_000), + render_gap_segments: 0, + rendered_at_ms, + discontinuity_sequence: 0, + target_duration_ms: 4_000, + segment_proxy_seqs: vec![123], + }); + record_test_normal_manifest_commit(&mut session, rendered_at_ms); + ProxySessionId(proxy_session_id) + }; + (session, proxy_session_id) +} + +#[tokio::test] +async fn hls_cache_manifest_response_applies_current_users_server_path_without_mutating_session_body() { + let input_name = Arc::::from("test-input"); + let input = ConfigInput { + id: 1, + name: Arc::clone(&input_name), + input_type: InputType::Xtream, + url: "http://origin.example.com".to_string(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let (session, proxy_session_id) = prepare_server_path_manifest_session(&app_state).await; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + + let response = super::super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::super::HlsCacheManifestOrigin { + raw_request_url: "http://origin.example.com/live/user/pass/12345.m3u8", + session_entry_url: super::super::HlsOriginEntryUrl::direct_http( + "http://origin.example.com/live/user/pass/12345.m3u8", + ), + input: &input, + origin_source: super::super::build_hls_origin_source(&input, "12345"), + }, + HeaderMap::new(), + Some("/iptv"), + "/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + .expect("hls cache should handle valid live hls entrypoint"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); + + assert!(body.contains(&format!("/iptv/hls/shared/live/{}/", proxy_session_id.0))); + assert!(body.contains("/map/000000.mp4")); + assert!(body.contains("/000123.ts")); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + let stored_body = session.read().await.last_rendered_manifest.as_ref().expect("stored manifest").body.clone(); + assert!(stored_body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + assert!(stored_body.contains(&format!("/hls/shared/live/{}/", proxy_session_id.0))); + assert!(!stored_body.contains("/iptv/hls/shared/live/")); + assert!(!stored_body.contains(&access_lease_id.0)); + assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); +} + +#[test] +fn pending_playlist_type_manifest_ignores_three_to_six_window_and_strip() { + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + for segment_count in [2, 8] { + let body = transient_manifest_body_from_sequence("proxy-session", 100, segment_count).replacen( + "#EXTM3U\n", + "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n", + 1, + ); + let materialized = super::super::materialize_shared_hls_access_manifest( + &body, + &access_lease_id, + HlsAccessLeaseState::Pending, + &StripConfig { mode: HlsStripMode::Segments, value: 5 }, + super::super::HlsManifestWindowPolicy::PreserveFullManifest, + "transient", + None, + ); + + assert_eq!(media_uri_count(&materialized.body), segment_count); + assert!(materialized.body.contains("#EXT-X-PLAYLIST-TYPE:EVENT")); + assert!(!materialized.body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + assert_eq!( + materialized.initial_strip_outcome, + Some(super::HlsInitialStripOutcome::Skipped { + reason: super::HlsInitialStripSkipReason::ManifestSemanticsPreserveFullManifest, + visible_segments: segment_count, + }) + ); + } +} + +#[test] +fn pending_endlist_only_manifest_keeps_complete_body_despite_strip() { + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let mut body = transient_manifest_body_from_sequence("proxy-session", 100, 8); + body.push_str("#EXT-X-ENDLIST\n"); + let window_policy = + crate::processing::parser::hls::origin_manifest::parse_manifest_semantics(&body).window_policy(); + assert_eq!(window_policy, super::super::HlsManifestWindowPolicy::PreserveFullManifest); + + let materialized = super::super::materialize_shared_hls_access_manifest( + &body, + &access_lease_id, + HlsAccessLeaseState::Pending, + &StripConfig { mode: HlsStripMode::Segments, value: 5 }, + window_policy, + "transient", + None, + ); + + assert_eq!(media_uri_count(&materialized.body), 8); + assert!(materialized.body.contains("#EXT-X-ENDLIST")); + assert_eq!( + materialized.initial_strip_outcome, + Some(super::HlsInitialStripOutcome::Skipped { + reason: super::HlsInitialStripSkipReason::ManifestSemanticsPreserveFullManifest, + visible_segments: 8, + }) + ); +} + +#[tokio::test(start_paused = true)] +async fn pending_strip_admission_timeout_does_not_commit_speculative_candidate() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let proxy_session_id = { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.clone(); + let rendered_at_ms = super::super::current_time_millis(); + store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id.0), rendered_at_ms); + session.segments.get_mut(&1).expect("visible test segment").status = SegmentCacheStatus::Discovered; + session.advance_media_readiness_generation(); + session.mark_authorized_media_access(rendered_at_ms); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &StripConfig { mode: HlsStripMode::Segments, value: 3 }, + None, + super::HlsCachedManifestOptions::committed_only(Duration::from_millis(75)), + ) + .await + .expect("timeout response"); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + let lease = app_state + .hls + .proxy + .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::super::current_time_millis()) + .await + .expect("pending lease remains available"); + assert!(lease.last_manifest_snapshot.is_none()); +} + +#[tokio::test] +async fn hls_cache_pending_normal_manifest_applies_initial_strip_without_mutating_shared_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::super::current_time_millis(); + store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id), rendered_at_ms); + session.mark_authorized_media_access(rendered_at_ms); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + let stored_before = session.read().await.last_rendered_manifest.as_ref().expect("normal manifest").body.clone(); + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("normal manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0\n")); + assert!(body.contains("/000000.ts")); + assert!(body.contains("/000001.ts")); + assert!(body.contains("/000002.ts")); + assert!(!body.contains("/000003.ts")); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + assert_eq!(session.read().await.last_rendered_manifest.as_ref().expect("stored manifest").body, stored_before); + assert_eq!(media_uri_count(&stored_before), 6); +} + +#[tokio::test] +async fn hls_cache_activated_normal_manifest_skips_initial_strip() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::super::current_time_millis(); + store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id), rendered_at_ms); + session.mark_authorized_media_access(rendered_at_ms); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("normal manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 6); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); +} + +#[tokio::test] +async fn hls_cache_fresh_required_normal_manifest_does_not_serve_stale_committed_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let rendered_at_ms = { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::super::current_time_millis(); + store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id), rendered_at_ms); + session.mark_authorized_media_access(rendered_at_ms); + rendered_at_ms + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::ZERO).requiring_newer_manifest(rendered_at_ms), + ) + .await; + + assert!(response.is_none()); +} + +#[tokio::test] +async fn hls_cache_fresh_required_normal_manifest_waits_for_newer_commit() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let (proxy_session_id, old_rendered_at_ms) = { + let mut session = session.write().await; + let proxy_session_id = session.proxy_session_id.clone(); + let old_rendered_at_ms = super::super::current_time_millis(); + store_normal_manifest_body(&mut session, normal_manifest_body(&proxy_session_id.0), old_rendered_at_ms); + session.origin_refresh.in_flight = true; + session.mark_authorized_media_access(old_rendered_at_ms); + (proxy_session_id, old_rendered_at_ms) + }; + let session_for_commit = Arc::clone(&session); + let proxy_session_for_body = proxy_session_id.0.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + let mut session = session_for_commit.write().await; + let rendered_at_ms = super::super::current_time_millis(); + let mut entry = test_segment_entry( + &session.proxy_session_id, + 100, + SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }, + ); + entry.duration_ms = 4_000; + session.segments.insert(100, entry); + session.advance_media_readiness_generation(); + session.last_rendered_manifest = Some(RenderedManifest { + body: format!( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n/hls/shared/live/{proxy_session_for_body}/{}/000100.ts\n", + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER + ), + first_proxy_seq: 100, + last_proxy_seq: 100, + playlist_duration_ms: 4_000, + valid_until_ms: rendered_at_ms.saturating_add(4_000), + render_gap_segments: 0, + rendered_at_ms, + discontinuity_sequence: 0, + target_duration_ms: 4_000, + segment_proxy_seqs: vec![100], + }); + record_test_normal_manifest_commit(&mut session, rendered_at_ms); + session.origin_refresh.in_flight = false; + }); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 0 }; + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::from_millis(200)) + .requiring_newer_manifest(old_rendered_at_ms), + ) + .await + .expect("fresh manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert!(body.contains("000100.ts")); + assert!(!body.contains("000000.ts")); + assert!(body.contains(&access_lease_id.0)); +} + +#[tokio::test] +async fn hls_cache_no_media_yet_waits_for_first_normal_manifest_commit() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let proxy_session_id = { + let mut session = session.write().await; + session.origin_refresh.in_flight = true; + session.proxy_session_id.clone() + }; + let session_for_commit = Arc::clone(&session); + let proxy_session_for_body = proxy_session_id.0.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + let mut session = session_for_commit.write().await; + let rendered_at_ms = super::super::current_time_millis(); + let mut entry = test_segment_entry( + &session.proxy_session_id, + 100, + SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }, + ); + entry.duration_ms = 4_000; + session.segments.insert(100, entry); + session.advance_media_readiness_generation(); + session.last_rendered_manifest = Some(RenderedManifest { + body: format!( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:100\n#EXTINF:4.0,\n/hls/shared/live/{proxy_session_for_body}/{}/000100.ts\n", + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER + ), + first_proxy_seq: 100, + last_proxy_seq: 100, + playlist_duration_ms: 4_000, + valid_until_ms: rendered_at_ms.saturating_add(4_000), + render_gap_segments: 0, + rendered_at_ms, + discontinuity_sequence: 0, + target_duration_ms: 4_000, + segment_proxy_seqs: vec![100], + }); + record_test_normal_manifest_commit(&mut session, rendered_at_ms); + session.origin_refresh.in_flight = false; + }); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 0 }; + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::from_millis(200)), + ) + .await + .expect("initial normal manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert!(body.contains(&access_lease_id.0)); + assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn wait_for_hls_test_session( + app_state: &Arc, + session_key: &HlsSessionKey, +) -> HlsSessionHandle { + for _ in 0..50 { + if let Some(session) = app_state.hls.proxy.sessions().get_by_key(session_key).await { + return session; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("shared hls test session should be created"); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn wait_for_hls_refresh_in_flight(session: &HlsSessionHandle) { + for _ in 0..50 { + if session.read().await.origin_refresh.in_flight { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("shared hls refresh should be in flight"); +} + +#[tokio::test] +async fn ready_hls_proxy_map_without_lease_returns_not_found() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_hls_map(&app_state, b"map-body", false).await; + + let status = get_status(app_state, &format!("/hls/shared/live/{proxy_session_id}/map/000000.mp4")).await; + + assert_eq!(status, StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn hls_proxy_map_not_ready_with_valid_lease_returns_service_unavailable() { + let app_state = test_app_state(); + let session = + app_state.hls.proxy.get_or_create_session(HlsSessionKey::new(1, "12345"), b"rewrite-secret", 100).await; + let manifest = normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n000123.m4s\n"); + let proxy_session_id = { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest should map"); + session.proxy_session_id.0.clone() + }; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "map/000000.mp4").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers()[header::RETRY_AFTER], "1"); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; +} + +#[tokio::test] +async fn invalid_hls_proxy_file_names_return_not_found() { + let app_state = test_app_state(); + + assert_eq!( + get_status(Arc::clone(&app_state), "/hls/shared/live/a8f31c9eQ7sLk92pV0mTaw/123.ts").await, + StatusCode::NOT_FOUND + ); + assert_eq!( + get_status(app_state, "/hls/shared/live/a8f31c9eQ7sLk92pV0mTaw/map/000123.exe").await, + StatusCode::NOT_FOUND + ); +} + +#[tokio::test] +async fn legacy_hls_route_remains_registered() { + let status = get_status(test_app_state(), "/hls/user/pass/1/2/3/not-a-token").await; + + assert_ne!(status, StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn terminate_failed_hls_manifest_session_preserves_shared_lease_when_request_id_available() { + let mut input = single_hls_provider_input("shared-hls-input"); + input.max_connections = 2; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let username = "testuser"; + let session_token = "testuser|stream-1|hls|0123456789abcdef"; + let provider_name = Arc::clone(&input.name); + let addr = test_addr_with_port(55302); + + let handle1 = app_state + .active_provider + .acquire_connection_with_lease_for_session( + &provider_name, + &addr, + false, + 0, + ConnectionKind::Normal, + Some(tuliprox_session::PlaybackLeaseRef::new(session_token, crate::model::PlaybackKind::LiveHls)), + ) + .expect("handle1 should be acquired"); + + let handle2 = app_state + .active_provider + .acquire_connection_with_lease_for_session( + &provider_name, + &addr, + false, + 0, + ConnectionKind::Normal, + Some(tuliprox_session::PlaybackLeaseRef::new(session_token, crate::model::PlaybackKind::LiveHls)), + ) + .expect("handle2 should be acquired"); + + let binding_tag = handle1.binding_tag; + let request_id1 = handle1.playback_request_id; + let request_id2 = handle2.playback_request_id; + assert!(request_id1.is_some() && request_id2.is_some() && request_id1 != request_id2); + + app_state.connection_manager.release_provider_handle(Some(handle1)); + + super::super::segment::terminate_failed_hls_manifest_session( + &app_state, + username, + session_token, + Some(&provider_name), + binding_tag, + request_id1, + ) + .await; + + assert!( + app_state.active_provider.binding_tag_for_owner(session_token).is_some(), + "shared lease must remain active for surviving request" + ); + + app_state.connection_manager.release_provider_handle(Some(handle2)); + super::super::segment::terminate_failed_hls_manifest_session( + &app_state, + username, + session_token, + Some(&provider_name), + binding_tag, + request_id2, + ) + .await; + + assert!( + app_state.active_provider.binding_tag_for_owner(session_token).is_none(), + "lease should be cleared once all requests are terminated" + ); +} + +#[tokio::test] +async fn terminate_failed_hls_manifest_session_clears_identified_reservation_without_request_id() { + let input = single_hls_provider_input("no-request-id-hls-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let username = "testuser"; + let session_token = "testuser|stream-1|hls|0123456789abcdef"; + let provider_name = Arc::clone(&input.name); + let addr = test_addr_with_port(55303); + + let handle = app_state + .active_provider + .acquire_connection_with_lease_for_session( + &provider_name, + &addr, + false, + 0, + ConnectionKind::Normal, + Some(tuliprox_session::PlaybackLeaseRef::new(session_token, crate::model::PlaybackKind::LiveHls)), + ) + .expect("handle should be acquired"); + + let binding_tag = handle.binding_tag; + assert!(binding_tag.is_some()); + + app_state.connection_manager.release_provider_handle(Some(handle)); + + assert!(app_state.active_provider.binding_tag_for_owner(session_token).is_some(), "lease should still be active"); + + super::super::segment::terminate_failed_hls_manifest_session( + &app_state, + username, + session_token, + Some(&provider_name), + binding_tag, + None, + ) + .await; + + assert!( + app_state.active_provider.binding_tag_for_owner(session_token).is_none(), + "lease should be cleared via binding tag when no request ID is available" + ); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/observability.rs b/backend/app/src/api/endpoints/hls_api/tests/observability.rs new file mode 100644 index 000000000..937837fae --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/observability.rs @@ -0,0 +1,225 @@ +use super::{ + create_active_hls_user_session_with, map_ready_segment_without_lease, media_uri_count, normal_manifest_body, + test_addr_with_port, test_app_state_with_hls_proxy, test_fingerprint, test_fingerprint_with_addr, + test_hls_access_context_with, +}; +use crate::{ + api::model::{AppState, HlsAccessLeaseId, HlsAccessLeaseState, HlsProxyManager, HlsSessionHandle, ProxySessionId}, + auth::Fingerprint, + model::StripConfig, +}; +use axum::http::HeaderMap; +use shared::model::HlsStripMode; +use std::sync::Arc; + +#[test] +fn repeated_speculative_strip_candidates_yield_one_committed_applied_diagnostic() { + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let body = normal_manifest_body("proxy-session"); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + let candidates = (0..8) + .map(|_| { + super::super::materialize_shared_hls_access_manifest( + &body, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + super::super::HlsManifestWindowPolicy::ApplyLiveWindow, + "normal", + None, + ) + }) + .collect::>(); + + assert_eq!(candidates.len(), 8); + let candidate_count = candidates.len(); + let diagnostics = candidates + .iter() + .enumerate() + .filter_map(|(index, candidate)| { + super::super::hls_initial_strip_publication_diagnostic( + if index.saturating_add(1) == candidate_count { + super::super::HlsInitialStripPublicationStatus::Committed + } else { + super::super::HlsInitialStripPublicationStatus::NotCommitted + }, + HlsAccessLeaseState::Pending, + candidate, + ) + }) + .collect::>(); + + assert_eq!( + diagnostics, + vec![super::super::HlsInitialStripPublicationDiagnostic::Applied { + mode: "normal", + strip_mode: "segments", + configured: 3, + effective: 3, + visible_segments: 3, + }] + ); +} + +#[test] +fn committed_pending_strip_disabled_yields_one_skipped_diagnostic() { + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let materialized = super::super::materialize_shared_hls_access_manifest( + &normal_manifest_body("proxy-session"), + &access_lease_id, + HlsAccessLeaseState::Pending, + &StripConfig { mode: HlsStripMode::Segments, value: 0 }, + super::super::HlsManifestWindowPolicy::ApplyLiveWindow, + "normal", + None, + ); + + let diagnostics = [super::super::hls_initial_strip_publication_diagnostic( + super::super::HlsInitialStripPublicationStatus::Committed, + HlsAccessLeaseState::Pending, + &materialized, + ) + .expect("committed strip diagnostic")]; + + assert_eq!( + diagnostics, + [super::super::HlsInitialStripPublicationDiagnostic::Skipped { + mode: "normal", + reason: crate::api::model::hls_cache::initial_strip::HlsInitialStripSkipReason::StripDisabled, + visible_segments: 6, + }] + ); + assert_eq!(media_uri_count(&materialized.body), 6); +} + +#[test] +fn committed_activated_manifest_yields_one_lease_state_skip_diagnostic() { + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let materialized = super::super::materialize_shared_hls_access_manifest( + &normal_manifest_body("proxy-session"), + &access_lease_id, + HlsAccessLeaseState::Activated, + &StripConfig { mode: HlsStripMode::Segments, value: 3 }, + super::super::HlsManifestWindowPolicy::ApplyLiveWindow, + "normal", + None, + ); + + let diagnostics = [super::super::hls_initial_strip_publication_diagnostic( + super::super::HlsInitialStripPublicationStatus::Committed, + HlsAccessLeaseState::Activated, + &materialized, + ) + .expect("committed strip diagnostic")]; + + assert_eq!( + diagnostics, + [super::super::HlsInitialStripPublicationDiagnostic::SkippedForLeaseState { + mode: "normal", + reason: super::super::HlsInitialStripLeaseSkipReason::LeaseActivated, + }] + ); + assert_eq!(media_uri_count(&materialized.body), 6); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn register_hls_cache_stream_for_stats_test( + app_state: &Arc, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + session_token: &str, + fingerprint: &Fingerprint, + lease_id: &str, +) { + create_active_hls_user_session_with( + app_state, + session_token, + "origin-provider", + "http://origin.example.com/live/12345.m3u8", + fingerprint.addr, + ) + .await; + let context = test_hls_access_context_with( + proxy_session_id.clone(), + HlsAccessLeaseId(lease_id.to_string()), + session_token, + fingerprint.key.clone(), + ); + super::super::ensure_hls_cache_stream_registered(app_state, fingerprint, &HeaderMap::new(), &context, session) + .await + .expect("HLS stream registers"); +} + +pub(in crate::api::endpoints::hls_api::tests) fn find_stream_by_session_token( + streams: &[shared::model::StreamInfo], + session_token: &str, +) -> shared::model::StreamInfo { + streams + .iter() + .find(|stream| stream.session_token.as_deref() == Some(session_token)) + .unwrap_or_else(|| panic!("{session_token} stream should exist")) + .clone() +} + +#[tokio::test] +async fn hls_cache_stream_stats_mark_additional_viewers_as_joined_existing() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment_without_lease(&app_state, 123, "ts", b"0123456789").await; + let proxy_session_id = ProxySessionId(proxy_session_id); + let shared_stream_id = super::super::hls_cache_shared_stream_id(&proxy_session_id); + let session = + app_state.hls.proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.expect("session should exist"); + let first_fingerprint = test_fingerprint(); + let second_fingerprint = test_fingerprint_with_addr(test_addr_with_port(55124)); + register_hls_cache_stream_for_stats_test( + &app_state, + &session, + &proxy_session_id, + "hls-session-token", + &first_fingerprint, + "first-access-lease", + ) + .await; + + let streams = app_state.active_users.active_streams().await; + let first_stream = find_stream_by_session_token(&streams, "hls-session-token"); + let first_meter_uid = first_stream.meter_uid; + assert!(first_stream.channel.shared); + assert_eq!(first_stream.channel.shared_stream_id, Some(shared_stream_id)); + assert_eq!(first_stream.channel.shared_joined_existing, Some(false)); + + register_hls_cache_stream_for_stats_test( + &app_state, + &session, + &proxy_session_id, + "hls-second-session-token", + &second_fingerprint, + "second-access-lease", + ) + .await; + + let streams = app_state.active_users.active_streams().await; + let first_stream = find_stream_by_session_token(&streams, "hls-session-token"); + let second_stream = find_stream_by_session_token(&streams, "hls-second-session-token"); + assert_eq!(first_stream.channel.shared_stream_id, Some(shared_stream_id)); + assert_eq!(first_stream.channel.shared_joined_existing, Some(false)); + assert_eq!(second_stream.channel.shared_stream_id, Some(shared_stream_id)); + assert_eq!(second_stream.channel.shared_joined_existing, Some(true)); + + register_hls_cache_stream_for_stats_test( + &app_state, + &session, + &proxy_session_id, + "hls-session-token", + &first_fingerprint, + "first-access-lease", + ) + .await; + + let streams = app_state.active_users.active_streams().await; + let first_stream = find_stream_by_session_token(&streams, "hls-session-token"); + assert_eq!(first_stream.channel.shared_stream_id, Some(shared_stream_id)); + assert_eq!(first_stream.channel.shared_joined_existing, Some(false)); + assert_eq!(first_stream.meter_uid, first_meter_uid); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/owner_handoff.rs b/backend/app/src/api/endpoints/hls_api/tests/owner_handoff.rs new file mode 100644 index 000000000..46b7b94ce --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/owner_handoff.rs @@ -0,0 +1,571 @@ +use super::{ + activate_test_hls_access_lease, create_active_hls_user_session, create_unbound_hls_test_session, enable_hls_cache, + hls_custom_video_test_user, prepare_pending_test_hls_access_lease, publish_owner_handoff_test_manifest, + response_body, single_hls_provider_input, spawn_test_segment_origin, store_test_sources_with_target, + test_app_state_with_hls_proxy_and_inputs, test_app_state_with_inputs, test_custom_video_buffer, test_fingerprint, + test_hls_access_context, CanonicalOwnerHandoffFixture, HlsCanonicalOwnerRegistrationKind, +}; +use crate::{ + api::model::{ + build_hls_custom_video_manifest_body, build_proxy_session_id, AppState, CustomVideoStreamType, + HlsAccessLeaseId, HlsAccessLeaseState, HlsAvailabilityReevaluationFinishReason, + HlsAvailabilityReevaluationMode, HlsAvailabilityReevaluationRegistration, HlsProxyManager, SegmentCacheStatus, + TransportStreamBuffer, + }, + model::{ConfigInput, ConfigTarget, CustomStreamResponse}, + processing::parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}, +}; +use axum::http::{header, HeaderMap, StatusCode}; +use shared::model::{ConfigTargetDto, InputType}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +#[tokio::test] +async fn already_owned_work_is_joined_without_duplicate_refresh() { + let fixture = CanonicalOwnerHandoffFixture::new(&["first-lease", "second-lease"]).await; + let owner_key = fixture + .app_state + .hls + .proxy + .availability_reevaluation_owner_key(&fixture.session, &fixture.proxy_session_id) + .await + .expect("owner key"); + let coordinator = fixture.app_state.hls.proxy.availability_reevaluations(); + let release = Arc::new(tokio::sync::Notify::new()); + let completed = Arc::new(tokio::sync::Notify::new()); + let task_release = Arc::clone(&release); + let task_completed = Arc::clone(&completed); + let task_session = Arc::clone(&fixture.session); + let task_app_state = Arc::clone(&fixture.app_state); + let task_proxy_session_id = fixture.proxy_session_id.clone(); + let task_owner_key = owner_key.clone(); + assert_eq!( + coordinator.register( + owner_key.clone(), + HlsAvailabilityReevaluationMode::RecoveryPressure, + move |ownership| async move { + task_release.notified().await; + publish_owner_handoff_test_manifest(&task_session).await; + task_app_state.hls.proxy.notify_session_evidence_changed(&task_proxy_session_id); + let _ = ownership.finish_cycle(&task_owner_key, HlsAvailabilityReevaluationFinishReason::Evaluated); + task_completed.notify_one(); + }, + ), + HlsAvailabilityReevaluationRegistration::Scheduled + ); + let duplicate_owner_runs = Arc::new(AtomicUsize::new(0)); + let task_duplicate_owner_runs = Arc::clone(&duplicate_owner_runs); + assert_eq!( + coordinator.register(owner_key, HlsAvailabilityReevaluationMode::RecoveryPressure, move |_| async move { + task_duplicate_owner_runs.fetch_add(1, Ordering::SeqCst); + },), + HlsAvailabilityReevaluationRegistration::AlreadyOwned + ); + let deadline_ms = super::super::current_time_millis().saturating_add(60_000); + let mut first = Box::pin(super::super::join_hls_canonical_manifest_owner( + fixture.handoff_context(0, fixture.safe_session().await, deadline_ms), + HlsCanonicalOwnerRegistrationKind::AlreadyOwned, + )); + let mut second = Box::pin(super::super::join_hls_canonical_manifest_owner( + fixture.handoff_context(1, fixture.safe_session().await, deadline_ms), + HlsCanonicalOwnerRegistrationKind::AlreadyOwned, + )); + assert!(matches!(futures::poll!(first.as_mut()), std::task::Poll::Pending)); + assert!(matches!(futures::poll!(second.as_mut()), std::task::Poll::Pending)); + + release.notify_one(); + let (first, second) = tokio::join!(first, second); + + assert_eq!(first.status(), StatusCode::OK); + assert_eq!(second.status(), StatusCode::OK); + let first_body = String::from_utf8(response_body(first).await.to_vec()).expect("first manifest utf8"); + let second_body = String::from_utf8(response_body(second).await.to_vec()).expect("second manifest utf8"); + assert!(first_body.contains("/first-lease/")); + assert!(!first_body.contains("/second-lease/")); + assert!(second_body.contains("/second-lease/")); + assert!(!second_body.contains("/first-lease/")); + assert_eq!(duplicate_owner_runs.load(Ordering::SeqCst), 0); + completed.notified().await; + tokio::task::yield_now().await; + assert_eq!(coordinator.owner_count(), 0); +} + +#[tokio::test] +async fn canonical_owner_join_preserves_bounded_deadline_failure() { + let fixture = CanonicalOwnerHandoffFixture::new(&["deadline-lease"]).await; + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.leases[0].0, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("pending deadline lease"); + assert_eq!( + super::super::hls_canonical_owner_request_deadline_ms( + &lease, + Duration::ZERO, + super::super::current_time_millis(), + ), + lease.pending_deadline_ms().expect("pending lease deadline") + ); + let owner_key = fixture + .app_state + .hls + .proxy + .availability_reevaluation_owner_key(&fixture.session, &fixture.proxy_session_id) + .await + .expect("owner key"); + let coordinator = fixture.app_state.hls.proxy.availability_reevaluations(); + assert_eq!( + coordinator.register(owner_key, HlsAvailabilityReevaluationMode::RecoveryPressure, |ownership| async move { + ownership.cancelled().await; + },), + HlsAvailabilityReevaluationRegistration::Scheduled + ); + let response = super::super::join_hls_canonical_manifest_owner( + fixture.handoff_context(0, fixture.safe_session().await, super::super::current_time_millis().saturating_sub(1)), + HlsCanonicalOwnerRegistrationKind::Scheduled, + ) + .await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(!response.headers().contains_key(header::RETRY_AFTER)); + coordinator.cancel_session(&fixture.proxy_session_id); +} + +#[test] +fn hls_custom_video_manifest_body_is_none_for_non_provisioning() { + let user = hls_custom_video_test_user(); + let manifest = build_hls_custom_video_manifest_body( + "https://example.test/iptv/", + &user, + CustomVideoStreamType::UserConnectionsExhausted, + ); + + assert!(manifest.is_none(), "non-provisioning custom video types have no static manifest body"); +} + +#[test] +fn hls_custom_video_manifest_uses_live_six_segment_window_for_provisioning() { + let user = hls_custom_video_test_user(); + let manifest = + build_hls_custom_video_manifest_body("https://example.test/iptv", &user, CustomVideoStreamType::Provisioning) + .expect("provisioning manifest does not depend on one looping asset duration"); + + assert!(manifest.contains("#EXT-X-TARGETDURATION:2")); + assert!(manifest.contains("#EXT-X-MEDIA-SEQUENCE:0")); + assert!(manifest.contains("#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-INDEPENDENT-SEGMENTS\n")); + assert!(!manifest.contains("#EXT-X-DISCONTINUITY-SEQUENCE")); + assert!(!manifest.contains("#EXT-X-SESSION-DATA")); + assert!(!manifest.contains("#EXT-X-ENDLIST")); + assert!(!manifest.contains("#EXT-X-DISCONTINUITY\n")); + assert_eq!(manifest.matches("#EXTINF:2.000000,").count(), 6); + for index in 0..6 { + assert!( + manifest.contains(&format!("https://example.test/iptv/cvs/hls/viewer/secret/provisioning_{index:03}.ts")) + ); + } + assert!(!manifest.contains("https://example.test/iptv/cvs/hls/viewer/secret/provisioning.ts")); + assert_eq!( + crate::api::model::hls_panel_provisioning_manifest_path(&user, 80510), + "/cvs/hls/viewer/secret/provisioning.m3u8?id=80510" + ); + assert!(!manifest.contains("provisioning.ts?")); + assert!(!manifest.contains("virtual_id")); + assert!(!manifest.contains("&seq=")); +} + +pub(in crate::api::endpoints::hls_api::tests) fn enable_provider_exhausted_custom_response(app_state: &Arc) { + app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { + channel_unavailable: None, + user_connections_exhausted: None, + provider_connections_exhausted: Some(test_custom_video_buffer()), + low_priority_preempted: None, + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: Vec::new(), + }))); +} + +pub(in crate::api::endpoints::hls_api::tests) fn enable_hls_provisioning_custom_response(app_state: &Arc) { + let mut ts_packet = vec![0_u8; 188]; + ts_packet[0] = 0x47; + let provisioning_segments = (0..6) + .map(|index| { + let mut packet = ts_packet.clone(); + packet[1] = u8::try_from(index).expect("test index fits"); + TransportStreamBuffer::new(packet) + }) + .collect(); + app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { + channel_unavailable: None, + user_connections_exhausted: None, + provider_connections_exhausted: None, + low_priority_preempted: None, + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: provisioning_segments, + }))); +} + +#[tokio::test] +async fn hls_provider_exhausted_without_provisioning_returns_custom_manifest() { + let input = single_hls_provider_input("provider-exhausted-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + store_test_sources_with_target( + &app_state, + input.clone(), + ConfigTarget::from(&ConfigTargetDto { id: 1, name: "default".to_string(), ..Default::default() }), + ); + enable_provider_exhausted_custom_response(&app_state); + let session = create_unbound_hls_test_session(&app_state, &input, "provider-exhausted-session", 1_000).await; + let access_lease_id = HlsAccessLeaseId("provider-exhausted-lease".to_string()); + let proxy_session_id = session.read().await.proxy_session_id.clone(); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + let strip = app_state.hls.proxy.strip(); + + let response = super::super::hls_shared_provisioning_or_provider_exhausted_response( + &app_state, + &session, + "hls-user", + &input, + 59, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let body = response_body(response).await; + let manifest = String::from_utf8(body.to_vec()).expect("manifest is utf8"); + assert!(manifest.contains("#EXTM3U")); + assert!(manifest.contains(&format!("/cvs/hls/{}/", access_lease_id.0))); + assert!(!manifest.contains("/hls-user/")); + assert!(!manifest.contains("/hls-pass/")); + assert!(!manifest.contains("/provider_connections_exhausted/")); +} + +#[tokio::test] +async fn shared_provisioning_timeline_manifest_uses_canonical_hls_session_segments() { + let input = single_hls_provider_input("shared-provisioning-timeline-input"); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy_and_inputs(hls_proxy, vec![Arc::new(input.clone())]); + enable_hls_provisioning_custom_response(&app_state); + let session = create_unbound_hls_test_session(&app_state, &input, "shared-prov-timeline-12345", 1_000).await; + let access_lease_id = HlsAccessLeaseId("timeline-lease".to_string()); + let proxy_session_id = session.read().await.proxy_session_id.clone(); + activate_test_hls_access_lease( + &app_state, + &proxy_session_id, + &access_lease_id.0, + super::super::current_time_millis(), + 60_000, + ) + .await; + let strip = app_state.hls.proxy.strip(); + + let response = super::super::hls_shared_provisioning_timeline_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + ) + .await + .expect("provisioning manifest should render"); + + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest is utf8"); + assert!(body.contains("#EXT-X-VERSION:7\n")); + assert!(body.contains("#EXT-X-INDEPENDENT-SEGMENTS\n")); + assert!(body.contains("#EXT-X-TARGETDURATION:2\n")); + assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0\n")); + assert!(body.contains("#EXTINF:2.000,\n")); + assert!(!body.contains("#EXTINF:12.000,\n")); + assert!(body.contains("/hls/shared/live/")); + assert!(body.contains("/000000.ts?pseq=0")); + assert!(body.contains("/000001.ts?pseq=1")); + assert!(body.contains("/000002.ts?pseq=2")); + assert!(body.matches("#EXTINF:").count() <= 6); + assert!(!body.contains("/cvs/hls/")); + { + let session = session.read().await; + assert_eq!(session.proxy_next_seq, Some(3)); + assert_eq!(session.publishable_origin_head_proxy_seq, Some(0)); + assert_eq!(session.publishable_origin_tail_proxy_seq, Some(2)); + assert_eq!(session.segments.len(), 3); + } + + let response = super::super::hls_shared_provisioning_timeline_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + ) + .await + .expect("subsequent manifest should append one segment"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest is utf8"); + assert!(body.contains("/000003.ts?pseq=3")); + assert!(body.matches("#EXTINF:").count() <= 6); + assert_eq!(session.read().await.proxy_next_seq, Some(4)); +} + +#[tokio::test] +async fn stale_provisioning_segments_do_not_trigger_canonical_handoff() { + let input = single_hls_provider_input("stale-provisioning-handoff-input"); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy_and_inputs(hls_proxy, vec![Arc::new(input.clone())]); + enable_hls_provisioning_custom_response(&app_state); + let session = create_unbound_hls_test_session(&app_state, &input, "54321", 1_000).await; + let initial_lease_id = HlsAccessLeaseId("initial-provisioning-lease".to_string()); + let strip = app_state.hls.proxy.strip(); + + super::super::hls_shared_provisioning_timeline_manifest_response( + &app_state, + &session, + &initial_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + ) + .await + .expect("provisioning manifest should render local segments"); + { + let session_guard = session.read().await; + assert!(session_guard.segments.values().any(crate::api::model::is_hls_provisioning_segment)); + assert_eq!(session_guard.segments.len(), 3); + assert_eq!(session_guard.pending_handoff_discontinuity_sequence, None); + } + + let new_lease_id = HlsAccessLeaseId("new-playback-lease".to_string()); + let previous_rendered_at = super::super::maybe_mark_hls_provisioning_handoff_for_canonical_manifest( + &app_state, + &session, + &input, + 54321, + &new_lease_id, + 2_000, + ) + .await; + + assert_eq!(previous_rendered_at, None); + let session_guard = session.read().await; + assert_eq!(session_guard.segments.len(), 3); + assert_eq!(session_guard.pending_handoff_discontinuity_sequence, None); +} + +#[tokio::test] +async fn provisioning_handoff_finds_shared_session_by_input_stream_id_not_virtual_id() { + let input = single_hls_provider_input("origin-id-handoff-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + let origin_session = create_unbound_hls_test_session(&app_state, &input, "80510", 1_000).await; + let virtual_id_session = create_unbound_hls_test_session(&app_state, &input, "1001", 1_000).await; + let stream_identity = super::super::HlsEntryStreamIdentity::new(1001, "80510").expect("input stream identity"); + + assert!( + super::super::mark_hls_provisioning_handoff_discontinuity(&app_state, &input, &stream_identity, None, 2_000,) + .await + ); + + assert!(origin_session.read().await.pending_handoff_discontinuity_sequence.is_some()); + assert_eq!(virtual_id_session.read().await.pending_handoff_discontinuity_sequence, None); +} + +#[tokio::test] +async fn shared_provisioning_handoff_continues_proxy_sequence_for_origin_segments() { + let input = single_hls_provider_input("shared-provisioning-handoff-input"); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy_and_inputs(hls_proxy, vec![Arc::new(input.clone())]); + enable_hls_provisioning_custom_response(&app_state); + let session = create_unbound_hls_test_session(&app_state, &input, "shared-handoff-12345", 1_000).await; + let access_lease_id = HlsAccessLeaseId("handoff-lease".to_string()); + let strip = app_state.hls.proxy.strip(); + super::super::hls_shared_provisioning_timeline_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + ) + .await + .expect("provisioning manifest should render"); + + let manifest = match parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:4025\n#EXTINF:4.0,\n4025.ts\n#EXTINF:4.0,\n4026.ts\n#EXTINF:4.0,\n4027.ts\n", + "http://origin.example/live/stream.m3u8", + ) { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + }; + let rendered = { + let mut session_guard = session.write().await; + session_guard.mark_pending_handoff_discontinuity(0); + drop(session_guard); + assert!( + super::super::ensure_shared_hls_provisioning_handoff_gap(&app_state, &session, 2_000).await, + "handoff should append one gap segment" + ); + let mut session_guard = session.write().await; + session_guard.apply_origin_manifest(&manifest).expect("origin manifest should map"); + for proxy_seq in 4..=6 { + session_guard.segments.get_mut(&proxy_seq).expect("origin segment").status = + SegmentCacheStatus::Ready { content_length: 1024, ready_at_ms: 2_000 }; + } + session_guard.render_and_store_manifest(2_000).expect("handoff manifest should render") + }; + + assert!(rendered.body.contains("#EXT-X-MEDIA-SEQUENCE:1\n")); + assert!(rendered.body.contains("#EXT-X-TARGETDURATION:4\n")); + assert!(rendered.body.contains("/000002.ts?pseq=2")); + assert!(rendered.body.contains("/000004.ts")); + assert!(rendered.body.contains("/000005.ts")); + assert!(rendered.body.contains("/000006.ts")); + assert!(!rendered.body.contains("/004025.ts")); + let provisioning_tail = rendered.body.find("/000002.ts?pseq=2").expect("provisioning tail is rendered"); + let gap_tag = rendered.body.find("#EXT-X-GAP\n").expect("handoff gap tag is rendered"); + let gap_uri = rendered.body.find("/000003.ts?pseq=3").expect("handoff gap uri is rendered"); + let discontinuity = rendered + .body + .find("#EXT-X-DISCONTINUITY\n#EXTINF:4.000,\n/hls/shared/live/") + .expect("origin handoff discontinuity is rendered"); + let first_origin = rendered.body.find("/000004.ts").expect("first origin segment is rendered"); + assert!(provisioning_tail < gap_tag); + assert!(gap_tag < gap_uri); + assert!(gap_uri < discontinuity); + assert!(discontinuity < first_origin); +} + +#[tokio::test] +async fn canonical_recovery_from_provisioning_marks_normal_handoff_boundary() { + let origin = spawn_test_segment_origin( + b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:123\n#EXTINF:4.0,\n000123.ts\n#EXTINF:4.0,\n000124.ts\n#EXTINF:4.0,\n000125.ts\n", + ) + .await; + let input = ConfigInput { + id: 1, + name: Arc::from("test-input"), + input_type: InputType::Xtream, + url: origin.base_url.clone(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + app_state.hls.provisioning.touch_consumer(Arc::clone(&input.name), 12345, super::super::current_time_millis()); + + let response = super::super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::super::HlsCacheManifestOrigin { + raw_request_url: &request_url, + session_entry_url: super::super::HlsOriginEntryUrl::direct_http(&request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + .expect("canonical hls cache should recover from provisioning"); + + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); + assert!(body.contains("#EXT-X-DISCONTINUITY\n#EXTINF:4.000,")); + assert!(!app_state.hls.provisioning.has_consumer(&input.name, 12345, super::super::current_time_millis())); +} + +#[tokio::test] +async fn canonical_recovery_from_provisioning_marks_transient_handoff_boundary() { + let origin = spawn_test_segment_origin( + b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:123\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.key\"\n#EXTINF:4.0,\n000123.ts\n#EXTINF:4.0,\n000124.ts\n#EXTINF:4.0,\n000125.ts\n", + ) + .await; + let input = ConfigInput { + id: 1, + name: Arc::from("test-input"), + input_type: InputType::Xtream, + url: origin.base_url.clone(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + app_state.hls.provisioning.touch_consumer(Arc::clone(&input.name), 12345, super::super::current_time_millis()); + + let response = super::super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Activated, + super::super::HlsCacheManifestOrigin { + raw_request_url: &request_url, + session_entry_url: super::super::HlsOriginEntryUrl::direct_http(&request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + .expect("canonical hls cache should recover from provisioning"); + + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest should be utf8"); + assert!(body.contains("#EXT-X-DISCONTINUITY\n#EXTINF:4.0,")); + assert!(!app_state.hls.provisioning.has_consumer(&input.name, 12345, super::super::current_time_millis())); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission.rs new file mode 100644 index 000000000..2650ff9bd --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission.rs @@ -0,0 +1,98 @@ +use super::{test_addr, test_fingerprint}; +use crate::{ + api::model::{ + AppState, ConnectionKind, CreateUserSessionParams, HlsOriginIoContext, HlsSessionHandle, PlaybackLifecycle, + ProxySessionId, UserSession, + }, + model::ProxyUserCredentials, +}; +use shared::model::UserConnectionPermission; +use std::{collections::HashMap, sync::Arc}; + +pub(in crate::api::endpoints::hls_api::tests) fn test_hls_origin_io_context( + app_state: &Arc, +) -> HlsOriginIoContext { + HlsOriginIoContext { + ctx: app_state.hls_ctx(), + client_addr: test_fingerprint().addr, + allow_grace: false, + priority: 0, + connection_kind: ConnectionKind::Normal, + reservation_ttl_secs: 60, + preacquired_provider_handle: None, + started_generation: None, + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn register_test_hls_stream_for_lease_release( + app_state: &Arc, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + provider: &str, +) { + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.max_connections = 1; + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "hls-session-token", + virtual_id: 12345, + provider, + stream_url: "http://origin.example.com/live/user/pass/12345.m3u8", + addr: &test_addr(), + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + let mut stream_channel = super::super::fallback_hls_cache_stream_channel( + 0, + 12345, + &session.read().await.origin_source, + proxy_session_id, + ); + stream_channel.shared = true; + stream_channel.shared_stream_id = Some(super::super::hls_cache_shared_stream_id(proxy_session_id)); + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Normal, + priority: user.priority, + soft_priority: user.soft_priority, + fingerprint: &test_fingerprint(), + provider: Arc::from(provider), + stream_channel: &stream_channel, + user_agent: std::borrow::Cow::Borrowed("test"), + session_token: Some("hls-session-token"), + }) + .await; +} + +pub(in crate::api::endpoints::hls_api::tests) fn stats_provider_test_user_session(provider: &str) -> UserSession { + UserSession { + token: "stats-session-token".to_string(), + transition_version: 0, + virtual_id: 12345, + provider: Arc::from(provider), + stream_url: Arc::from("http://origin.example.com/live/12345.m3u8"), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), + user_agent_stream_index: None, + addr: test_addr(), + socket_bound: false, + active_addrs: Vec::new(), + ts: 100, + started_at: 100, + permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + lifecycle: PlaybackLifecycle::Active, + ..Default::default() + } +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs new file mode 100644 index 000000000..44df8368f --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_account_overlap.rs @@ -0,0 +1,689 @@ +use super::{ + create_bound_hls_test_session, create_unbound_hls_test_session, get_response, overlap_provider_input, + provider_admission::test_hls_origin_io_context, response_body, runtime_policy_endpoint_fixture, + single_hls_provider_input, store_test_sources_with_target, test_addr_with_port, test_app_state, + test_app_state_with_inputs, test_fingerprint, test_fingerprint_with_addr, test_hls_access_context, + wait_for_provider_connection_count, wait_for_runtime_policy_terminal_plan, +}; +use crate::{ + api::model::{ + begin_hls_origin_account_io, finish_hls_origin_account_io, ConnectionKind, HlsAccessLeaseId, + HlsAccessLeaseState, HlsEffectiveOriginAcquirePolicy, HlsOriginAccountBinding, HlsOriginAccountBindingMode, + HlsOriginAccountDetachedReason, HlsRuntimeCustomTailReason, ProxySessionId, + }, + model::{Config, ConfigInput, ConfigTarget, ReverseProxyConfig}, +}; +use axum::http::{header, StatusCode}; +use shared::model::{ConfigTargetDto, ReverseProxyConfigDto, StreamConfigDto}; +use std::{sync::Arc, time::Duration}; + +#[tokio::test] +async fn hls_origin_account_io_lease_allows_parallel_same_session_origin_work() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let binding = session.read().await.origin_account_binding.clone().expect("binding exists"); + let origin_io = test_hls_origin_io_context(&app_state); + + let first = begin_hls_origin_account_io(&origin_io, &session, &binding) + .await + .expect("first same-session origin io acquires provider account"); + wait_for_provider_connection_count(&app_state, 1).await; + let second = begin_hls_origin_account_io(&origin_io, &session, &binding) + .await + .expect("second same-session origin io joins session lease"); + wait_for_provider_connection_count(&app_state, 1).await; + assert_eq!( + session + .read() + .await + .origin_account_io_lease + .as_ref() + .expect("session provider lease exists") + .active_io_count + .load(std::sync::atomic::Ordering::Relaxed), + 2 + ); + + finish_hls_origin_account_io(&origin_io, &session, first, true).await; + wait_for_provider_connection_count(&app_state, 1).await; + assert_eq!( + session + .read() + .await + .origin_account_io_lease + .as_ref() + .expect("session provider lease remains while second io is active") + .active_io_count + .load(std::sync::atomic::Ordering::Relaxed), + 1 + ); + + finish_hls_origin_account_io(&origin_io, &session, second, true).await; + wait_for_provider_connection_count(&app_state, 0).await; + assert!(!app_state + .active_provider + .is_provider_reserved_for_other_session(&binding.account_name, Some("other-hls-session"))); +} + +#[tokio::test] +async fn hls_origin_account_io_lease_blocks_other_hls_sessions_for_same_account() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let first_session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let second_session = create_bound_hls_test_session(&app_state, &input, "67890", "account-a", 1_000).await; + let first_binding = first_session.read().await.origin_account_binding.clone().expect("binding exists"); + let second_binding = second_session.read().await.origin_account_binding.clone().expect("binding exists"); + let origin_io = test_hls_origin_io_context(&app_state); + + let first = begin_hls_origin_account_io(&origin_io, &first_session, &first_binding) + .await + .expect("first session acquires account"); + wait_for_provider_connection_count(&app_state, 1).await; + + assert!(begin_hls_origin_account_io(&origin_io, &second_session, &second_binding).await.is_err()); + wait_for_provider_connection_count(&app_state, 1).await; + + finish_hls_origin_account_io(&origin_io, &first_session, first, true).await; + wait_for_provider_connection_count(&app_state, 0).await; +} + +#[tokio::test] +async fn hls_account_overlap_selects_soft_candidate_but_not_hard_active() { + let app_state = test_app_state(); + let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; + let session = create_bound_hls_test_session(&app_state, &input, "old", "account-a", 1_000).await; + { + let mut session = session.write().await; + session.target_duration = Some(10); + session.mark_authorized_media_access(1_000); + } + let new_proxy_session_id = ProxySessionId("new-session".to_string()); + + let hard_candidate = + super::super::find_hls_account_overlap_candidate(&app_state, &input, &new_proxy_session_id, 5_000).await; + assert!(hard_candidate.is_none(), "hard-active sessions must not be overbooked"); + + let delayed_candidate = + super::super::find_hls_account_overlap_candidate(&app_state, &input, &new_proxy_session_id, 12_000).await; + assert!(delayed_candidate.is_none(), "soft-active candidate must respect the dynamic overlap delay"); + + let soft_candidate = + super::super::find_hls_account_overlap_candidate(&app_state, &input, &new_proxy_session_id, 21_000) + .await + .expect("soft-active session can be overbooked"); + assert_eq!(soft_candidate.account_name.as_ref(), "account-a"); + assert_eq!(soft_candidate.last_media_at_ms, 1_000); + assert_eq!(soft_candidate.soft_overlap_eligible_at_ms, 21_000); + assert_eq!(soft_candidate.soft_overlap_delay_ms, 20_000); + assert_eq!(soft_candidate.reclaim_until_ms, 31_000); +} + +#[test] +fn hls_soft_overlap_delay_scales_with_tuliprox_target_pressure() { + assert_eq!(super::super::hls_soft_overlap_delay_ms(10_000, 1, 1), 20_000); + assert_eq!(super::super::hls_soft_overlap_delay_ms(10_000, 3, 2), 15_000); + assert_eq!(super::super::hls_soft_overlap_delay_ms(10_000, 4, 2), 10_000); +} + +#[tokio::test] +async fn hls_account_overlap_reclaim_preempts_speculative_session() { + let app_state = test_app_state(); + let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; + let winner = create_bound_hls_test_session(&app_state, &input, "winner", "account-a", 1_000).await; + let loser = create_bound_hls_test_session(&app_state, &input, "loser", "account-a", 1_000).await; + let winner_proxy_session_id = winner.read().await.proxy_session_id.clone(); + let loser_proxy_session_id = loser.read().await.proxy_session_id.clone(); + { + let mut loser = loser.write().await; + loser.origin_account_binding = Some(HlsOriginAccountBinding::speculative_from( + Arc::clone(&input.name), + Arc::from("account-a"), + &loser_proxy_session_id, + winner_proxy_session_id.clone(), + 20_000, + 2_000, + )); + } + let loser_generation = loser.read().await.activity.origin_work_generation; + + super::super::reclaim_hls_account_overlap_if_needed(&app_state, &winner, 10_000).await; + + assert!(app_state.hls.proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 10_000).await); + assert!(!app_state.hls.proxy.is_account_overlap_cooling_down(&input.name, &Arc::from("account-a"), 25_000).await); + let loser_binding_mode = loser.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); + assert!(matches!( + loser_binding_mode, + HlsOriginAccountBindingMode::Detached { reason: HlsOriginAccountDetachedReason::ReclaimedByOriginalOwner, .. } + )); + assert_eq!(loser.read().await.activity.origin_work_generation, loser_generation + 1); + let winner_binding_mode = winner.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); + assert!(matches!(winner_binding_mode, HlsOriginAccountBindingMode::Active)); +} + +#[tokio::test] +async fn hls_account_overlap_promotes_speculative_session_after_soft_window() { + let app_state = test_app_state(); + let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; + let displaced = create_bound_hls_test_session(&app_state, &input, "displaced", "account-a", 1_000).await; + let promoted = create_bound_hls_test_session(&app_state, &input, "promoted", "account-a", 1_000).await; + let displaced_proxy_session_id = displaced.read().await.proxy_session_id.clone(); + let promoted_proxy_session_id = promoted.read().await.proxy_session_id.clone(); + { + let mut promoted = promoted.write().await; + promoted.origin_account_binding = Some(HlsOriginAccountBinding::speculative_from( + Arc::clone(&input.name), + Arc::from("account-a"), + &promoted_proxy_session_id, + displaced_proxy_session_id, + 20_000, + 2_000, + )); + } + let displaced_generation = displaced.read().await.activity.origin_work_generation; + + super::super::promote_elapsed_hls_account_overlaps(&app_state, 20_001).await; + + let displaced_binding_mode = displaced.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); + assert!(matches!( + displaced_binding_mode, + HlsOriginAccountBindingMode::Detached { reason: HlsOriginAccountDetachedReason::SoftWindowElapsed, .. } + )); + assert_eq!(displaced.read().await.activity.origin_work_generation, displaced_generation + 1); + let promoted_binding_mode = promoted.read().await.origin_account_binding.as_ref().unwrap().binding_mode.clone(); + assert!(matches!(promoted_binding_mode, HlsOriginAccountBindingMode::Active)); +} + +#[tokio::test] +async fn hls_origin_runtime_uses_soft_overlap_before_grace_for_interactive_work() { + let input = single_hls_provider_input("soft-overlap-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let old_session = create_bound_hls_test_session(&app_state, &input, "old", input.name.as_ref(), 1_000).await; + let old_proxy_session_id = old_session.read().await.proxy_session_id.clone(); + { + let mut session = old_session.write().await; + session.target_duration = Some(10); + session.mark_authorized_media_access(1_000); + session.activity.last_delivered_media_at_ms = Some(1_000); + } + let old_binding = old_session.read().await.origin_account_binding.clone().expect("binding exists"); + app_state.active_provider.refresh_provider_reservation(&old_binding.account_name, &old_binding.session_owner, 60); + app_state.active_provider.confirm_playback_activity(&old_binding.session_owner); + // The binding acquired a reservation without a handle; capture its tag so the + // preempt path can prove it still targets this exact incarnation. + let binding_tag = app_state.active_provider.binding_tag_for_owner(&old_binding.session_owner); + old_session.write().await.origin_account_binding.as_mut().expect("binding exists").provider_binding_tag = + binding_tag; + + let new_session = create_unbound_hls_test_session(&app_state, &input, "new", 12_000).await; + let new_proxy_session_id = new_session.read().await.proxy_session_id.clone(); + let prepared_origin = super::super::prepare_hls_origin_runtime( + &app_state, + &new_session, + &input, + "http://account.example.com/live/account-user/account-pass/new.m3u8", + "http://account.example.com/live/account-user/account-pass/new.m3u8", + &new_proxy_session_id, + &test_fingerprint_with_addr(test_addr_with_port(55201)), + ConnectionKind::Normal, + 0, + super::super::HlsOriginWorkKind::Manifest, + super::super::HlsOriginWorkClass::ManifestInteractive, + 21_000, + ) + .await + .expect("interactive work should use soft-active overlap before grace"); + + let binding = + prepared_origin.origin_account_binding_to_store.as_ref().expect("speculative binding should be prepared"); + assert_eq!(binding.account_name, input.name); + assert!(matches!( + &binding.binding_mode, + HlsOriginAccountBindingMode::Speculative { + displaced_proxy_session_id, + .. + } if displaced_proxy_session_id == &old_proxy_session_id + )); + assert!(matches!( + prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), + Some(super::super::ProviderAllocation::Available(_)) + )); + + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); +} + +#[tokio::test] +async fn hls_origin_runtime_normal_policy_preempts_active_soft_hls_binding() { + let input = single_hls_provider_input("policy-preempt-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let soft_session = create_bound_hls_test_session(&app_state, &input, "soft", input.name.as_ref(), 1_000).await; + let soft_generation = { + let mut session = soft_session.write().await; + session.target_duration = Some(10); + session.mark_authorized_media_access(10_000); + session.activity.last_delivered_media_at_ms = Some(10_000); + session.reconcile_effective_origin_acquire_policy( + Some(HlsEffectiveOriginAcquirePolicy::new(ConnectionKind::Soft, 0, 10_000)), + 10_000, + ); + session.activity.origin_work_generation + }; + let soft_binding = soft_session.read().await.origin_account_binding.clone().expect("soft binding exists"); + app_state.active_provider.refresh_provider_reservation(&soft_binding.account_name, &soft_binding.session_owner, 60); + app_state.active_provider.confirm_playback_activity(&soft_binding.session_owner); + // The binding acquired a reservation without a handle; capture its tag so the + // preempt path can prove it still targets this exact incarnation. + let binding_tag = app_state.active_provider.binding_tag_for_owner(&soft_binding.session_owner); + soft_session.write().await.origin_account_binding.as_mut().expect("binding exists").provider_binding_tag = + binding_tag; + + let normal_session = create_unbound_hls_test_session(&app_state, &input, "normal", 10_500).await; + let normal_proxy_session_id = normal_session.read().await.proxy_session_id.clone(); + let prepared_origin = super::super::prepare_hls_origin_runtime( + &app_state, + &normal_session, + &input, + "http://account.example.com/live/account-user/account-pass/normal.m3u8", + "http://account.example.com/live/account-user/account-pass/normal.m3u8", + &normal_proxy_session_id, + &test_fingerprint_with_addr(test_addr_with_port(55231)), + ConnectionKind::Normal, + 0, + super::super::HlsOriginWorkKind::Manifest, + super::super::HlsOriginWorkClass::ManifestInteractive, + 10_500, + ) + .await + .expect("normal HLS policy should preempt active soft HLS binding"); + + let new_binding = prepared_origin + .origin_account_binding_to_store + .as_ref() + .expect("preempting session should receive active binding"); + assert_eq!(new_binding.account_name, input.name); + assert!(matches!(new_binding.binding_mode, HlsOriginAccountBindingMode::Active)); + assert!(matches!( + prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), + Some(super::super::ProviderAllocation::Available(_)) + )); + let soft_session = soft_session.read().await; + assert_eq!(soft_session.activity.origin_work_generation, soft_generation + 1); + assert!(matches!( + soft_session.origin_account_binding.as_ref().map(|binding| &binding.binding_mode), + Some(HlsOriginAccountBindingMode::Detached { + reason: HlsOriginAccountDetachedReason::PreemptedByHigherPriority, + .. + }) + )); + drop(soft_session); + + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); +} + +#[tokio::test] +async fn preempted_origin_runtime_commits_low_priority_tail_without_redirect_or_origin_fetch() { + let fixture = runtime_policy_endpoint_fixture(true).await; + let input = single_hls_provider_input("runtime-preempted-input"); + let session = fixture + .app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&fixture.proxy_session_id) + .await + .expect("runtime policy session"); + let now_ms = super::super::current_time_millis(); + let mut binding = HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from("preempted-account"), + &fixture.proxy_session_id, + now_ms, + ); + binding.detach(HlsOriginAccountDetachedReason::PreemptedByHigherPriority, now_ms); + session.write().await.replace_origin_account_binding(Some(binding)); + let origin_refresh_before = session.read().await.origin_refresh.clone(); + let origin = super::super::HlsCacheManifestOrigin { + raw_request_url: "http://account.example.com/live/account-user/account-pass/12345.m3u8", + session_entry_url: super::super::HlsOriginEntryUrl::direct_http( + "http://account.example.com/live/account-user/account-pass/12345.m3u8", + ), + input: &input, + origin_source: super::super::build_hls_origin_source(&input, "12345"), + }; + let context = test_hls_access_context(fixture.proxy_session_id.clone(), fixture.lease_id.clone()); + + let result = super::super::prepare_hls_canonical_manifest_origin_runtime( + &fixture.app_state, + &session, + &context, + &origin, + &fixture.proxy_session_id, + &fixture.lease_id, + HlsAccessLeaseState::Activated, + &test_fingerprint(), + None, + now_ms, + ) + .await; + let Err(response) = result else { + panic!("detached origin binding must resolve to the lease-bound policy tail"); + }; + + assert!(!response.headers().contains_key(header::LOCATION)); + let plan = wait_for_runtime_policy_terminal_plan(&fixture).await; + assert_eq!(plan.reason, HlsRuntimeCustomTailReason::LowPriorityPreempted); + assert_eq!(plan.segment_duration_ms, 10_027); + assert_eq!(session.read().await.origin_refresh, origin_refresh_before); + let replay = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; + assert_eq!(replay.status(), StatusCode::OK); + assert!(!replay.headers().contains_key(header::LOCATION)); + let body = String::from_utf8(response_body(replay).await.to_vec()).expect("preemption manifest utf8"); + assert!(body.contains("/terminal/")); + assert!(!body.contains("/cvs/hls/")); + assert!(body.ends_with("#EXT-X-ENDLIST\n")); +} + +#[tokio::test] +async fn hls_origin_policy_preemption_rejects_soft_request_against_active_normal_binding() { + let input = single_hls_provider_input("policy-no-preempt-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let normal_session = create_bound_hls_test_session(&app_state, &input, "normal", input.name.as_ref(), 1_000).await; + { + let mut session = normal_session.write().await; + session.target_duration = Some(10); + session.mark_authorized_media_access(10_000); + session.reconcile_effective_origin_acquire_policy( + Some(HlsEffectiveOriginAcquirePolicy::new(ConnectionKind::Normal, 0, 10_000)), + 10_000, + ); + } + let normal_binding = normal_session.read().await.origin_account_binding.clone().expect("normal binding exists"); + app_state.active_provider.refresh_provider_reservation( + &normal_binding.account_name, + &normal_binding.session_owner, + 60, + ); + app_state.active_provider.confirm_playback_activity(&normal_binding.session_owner); + let soft_session = create_unbound_hls_test_session(&app_state, &input, "soft", 10_500).await; + let soft_proxy_session_id = soft_session.read().await.proxy_session_id.clone(); + + let result = super::super::prepare_hls_origin_policy_preempt_runtime( + &app_state, + &soft_session, + &input, + "http://account.example.com/live/account-user/account-pass/soft.m3u8", + "http://account.example.com/live/account-user/account-pass/soft.m3u8", + &soft_proxy_session_id, + &test_fingerprint_with_addr(test_addr_with_port(55232)), + ConnectionKind::Soft, + -100, + crate::model::PlaybackKind::LiveHls, + 10_500, + ) + .await; + + assert!(result.is_err()); + assert!(matches!( + normal_session.read().await.origin_account_binding.as_ref().map(|binding| &binding.binding_mode), + Some(HlsOriginAccountBindingMode::Active) + )); + assert!(app_state + .active_provider + .is_provider_reserved_for_other_session(&normal_binding.account_name, Some("unrelated-session"))); +} + +#[tokio::test] +async fn hls_origin_runtime_uses_grace_as_interactive_fallback_after_overlap_fails() { + let input = single_hls_provider_input("grace-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let occupied = app_state + .active_provider + .acquire_connection_with_grace_for_session( + &input.name, + &test_addr_with_port(55211), + false, + 0, + ConnectionKind::Normal, + Some("external-owner"), + ) + .expect("test should occupy the only provider account"); + let session = create_unbound_hls_test_session(&app_state, &input, "12345", 2_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + + let prepared_origin = super::super::prepare_hls_origin_runtime( + &app_state, + &session, + &input, + "http://account.example.com/live/account-user/account-pass/12345.m3u8", + "http://account.example.com/live/account-user/account-pass/12345.m3u8", + &proxy_session_id, + &test_fingerprint_with_addr(test_addr_with_port(55212)), + ConnectionKind::Normal, + 0, + super::super::HlsOriginWorkKind::Manifest, + super::super::HlsOriginWorkClass::ManifestInteractive, + 2_000, + ) + .await + .expect("interactive work can use grace when normal acquire and overlap fail"); + + assert!(matches!( + prepared_origin.preacquired_origin_account_handle.as_ref().map(|handle| &handle.allocation), + Some(super::super::ProviderAllocation::GracePeriod(_)) + )); + assert_eq!( + prepared_origin + .origin_account_binding_to_store + .as_ref() + .expect("grace binding should still bind the selected account") + .account_name + .as_ref(), + input.name.as_ref() + ); + + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); + app_state.connection_manager.release_provider_handle(Some(occupied)); +} + +#[tokio::test] +async fn hls_provider_exhausted_grace_hold_waits_for_grace_period_before_retry() { + let input = single_hls_provider_input("provider-grace-hold-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + app_state.app_config.config.store(Arc::new(Config { + reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { + stream: Some(StreamConfigDto { + grace_period_millis: 20, + grace_period_timeout_secs: 10, + grace_period_hold_stream: true, + ..Default::default() + }), + ..Default::default() + })), + ..Default::default() + })); + let session = create_unbound_hls_test_session(&app_state, &input, "provider-grace-session", 1_000).await; + let access_lease_id = HlsAccessLeaseId("provider-grace-lease".to_string()); + let strip = app_state.hls.proxy.strip(); + let resolution = super::super::hls_provider_connections_exhausted_manifest_resolution( + &app_state, + &session, + "hls-user", + &input, + 59, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + true, + ); + + let resolution = + tokio::time::timeout(Duration::from_millis(500), resolution).await.expect("grace hold deadline should wake"); + + assert!(matches!(resolution, super::super::HlsProviderExhaustedResolution::RetryAcquire)); +} + +#[tokio::test] +async fn provider_grace_expiry_commits_lease_bound_provider_exhausted_tail() { + let fixture = runtime_policy_endpoint_fixture(true).await; + let input = single_hls_provider_input("runtime-provider-grace-input"); + store_test_sources_with_target( + &fixture.app_state, + input.clone(), + ConfigTarget::from(&ConfigTargetDto { id: 1, name: "default".to_string(), ..Default::default() }), + ); + let current = fixture.app_state.app_config.config.load(); + fixture.app_state.app_config.config.store(Arc::new(Config { + reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { + stream: Some(StreamConfigDto { + grace_period_millis: 20, + grace_period_timeout_secs: 10, + grace_period_hold_stream: true, + ..Default::default() + }), + ..Default::default() + })), + ..current.as_ref().clone() + })); + let strip = fixture.app_state.hls.proxy.strip(); + let session = fixture + .app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&fixture.proxy_session_id) + .await + .expect("runtime provider session"); + + let grace = super::super::hls_provider_connections_exhausted_manifest_resolution( + &fixture.app_state, + &session, + "hls-user", + &input, + 12345, + &fixture.lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + true, + ) + .await; + assert!(matches!(grace, super::super::HlsProviderExhaustedResolution::RetryAcquire)); + + let response = super::super::hls_provider_connections_exhausted_manifest_resolution( + &fixture.app_state, + &session, + "hls-user", + &input, + 12345, + &fixture.lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + false, + ) + .await; + let super::super::HlsProviderExhaustedResolution::Response(response) = response else { + panic!("expired grace must resolve to a finite response"); + }; + assert!(!response.headers().contains_key(header::LOCATION)); + + let plan = wait_for_runtime_policy_terminal_plan(&fixture).await; + assert_eq!(plan.reason, HlsRuntimeCustomTailReason::ProviderConnectionsExhausted); + assert_eq!(plan.segment_duration_ms, 10_027); + let replay = super::super::hls_provider_connections_exhausted_manifest_resolution( + &fixture.app_state, + &session, + "hls-user", + &input, + 12345, + &fixture.lease_id, + HlsAccessLeaseState::Denied, + &strip, + None, + false, + ) + .await; + let super::super::HlsProviderExhaustedResolution::Response(replay) = replay else { + panic!("committed provider tail must replay"); + }; + assert_eq!(replay.status(), StatusCode::OK); + assert!(!replay.headers().contains_key(header::LOCATION)); + let body = String::from_utf8(response_body(replay).await.to_vec()).expect("provider terminal manifest utf8"); + assert!(body.contains("/terminal/")); + assert!(!body.contains("/cvs/hls/")); + assert!(body.ends_with("#EXT-X-ENDLIST\n")); +} + +#[tokio::test] +async fn hls_origin_runtime_background_skips_soft_overlap_and_grace() { + let input = single_hls_provider_input("background-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let old_session = create_bound_hls_test_session(&app_state, &input, "old", input.name.as_ref(), 1_000).await; + { + let mut session = old_session.write().await; + session.target_duration = Some(10); + session.mark_authorized_media_access(1_000); + session.activity.last_delivered_media_at_ms = Some(1_000); + } + let old_binding = old_session.read().await.origin_account_binding.clone().expect("binding exists"); + app_state.active_provider.refresh_provider_reservation(&old_binding.account_name, &old_binding.session_owner, 60); + app_state.active_provider.confirm_playback_activity(&old_binding.session_owner); + let new_session = create_unbound_hls_test_session(&app_state, &input, "new", 12_000).await; + let new_proxy_session_id = new_session.read().await.proxy_session_id.clone(); + + let result = super::super::prepare_hls_origin_runtime( + &app_state, + &new_session, + &input, + "http://account.example.com/live/account-user/account-pass/new.m3u8", + "http://account.example.com/live/account-user/account-pass/new.m3u8", + &new_proxy_session_id, + &test_fingerprint_with_addr(test_addr_with_port(55221)), + ConnectionKind::Normal, + 0, + super::super::HlsOriginWorkKind::Segment, + super::super::HlsOriginWorkClass::Background, + 12_000, + ) + .await; + + assert_eq!( + result.err(), + Some(super::super::HlsOriginRuntimeAcquireError::NoAccountAvailable { + reason: super::super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted + }) + ); + let old_session = old_session.read().await; + assert!(matches!( + old_session.origin_account_binding.as_ref().expect("old binding remains").binding_mode, + HlsOriginAccountBindingMode::Active + )); + assert!(new_session.read().await.origin_account_binding.is_none()); +} + +#[test] +fn hls_detached_origin_binding_reclaimed_by_owner_maps_to_preempted_no_account_reason() { + let proxy_session_id = ProxySessionId("preempted-session".to_string()); + let mut binding = + HlsOriginAccountBinding::new(Arc::from("input-a"), Arc::from("account-a"), &proxy_session_id, 1_000); + binding.detach(HlsOriginAccountDetachedReason::ReclaimedByOriginalOwner, 2_000); + + assert_eq!( + super::super::hls_no_account_reason_for_binding(Some(&binding)), + super::super::HlsOriginRuntimeNoAccountReason::OriginBindingPreempted + ); +} + +#[test] +fn hls_detached_origin_binding_soft_window_elapsed_maps_to_exhausted_no_account_reason() { + let proxy_session_id = ProxySessionId("soft-window-session".to_string()); + let mut binding = + HlsOriginAccountBinding::new(Arc::from("input-a"), Arc::from("account-a"), &proxy_session_id, 1_000); + binding.detach(HlsOriginAccountDetachedReason::SoftWindowElapsed, 2_000); + + assert_eq!( + super::super::hls_no_account_reason_for_binding(Some(&binding)), + super::super::HlsOriginRuntimeNoAccountReason::ProviderConnectionsExhausted + ); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_binding.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_binding.rs new file mode 100644 index 000000000..e65886aa3 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_binding.rs @@ -0,0 +1,496 @@ +use super::{ + cache_recording_test_item, configure_recording_test_listener, create_active_hls_user_session, + create_bound_hls_test_session, enable_hls_cache, media_uri_count, overlap_provider_input, + provider_admission::stats_provider_test_user_session, recording_test_response, recording_test_router, + recording_test_url, response_body, single_hls_provider_input, spawn_recording_header_origin, + store_test_sources_with_target, test_app_state, test_app_state_with_inputs, test_fingerprint, + test_hls_access_context, test_hls_share_target, test_m3u_hls_item, test_m3u_hls_share_target, + transient_manifest_body, try_test_hls_cached_manifest_response, +}; +use crate::{ + api::model::{ + build_proxy_session_id, CacheAccessState, ConnectionKind, HlsAccessLeaseId, HlsAccessLeaseState, + HlsOriginAccountBinding, HlsOriginAccountBindingMode, HlsOriginAccountDetachedReason, HlsOriginSource, + HlsOriginSourceKind, HlsSegmentFile, HlsSessionKey, HlsSessionMode, OriginSegmentFetchRef, OriginSegmentKey, + ProxySessionId, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, + }, + model::{ConfigInput, ProxyUserCredentials, StripConfig}, +}; +use axum::http::{header, HeaderMap, StatusCode}; +use http_body_util::BodyExt; +use shared::model::{HlsStripMode, InputType, PlaylistItemType, UserConnectionPermission, XtreamCluster}; +use std::{sync::Arc, time::Duration}; + +#[tokio::test] +async fn transient_origin_binding_requires_runtime_prepare_for_missing_or_detached_account() { + let input = single_hls_provider_input("known-account"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let proxy_session_id = ProxySessionId("shared-hls-session".to_string()); + let known_binding = + HlsOriginAccountBinding::new(Arc::clone(&input.name), Arc::clone(&input.name), &proxy_session_id, 1_000); + let missing_binding = + HlsOriginAccountBinding::new(Arc::clone(&input.name), Arc::from("removed-account"), &proxy_session_id, 1_000); + let mut detached_binding = known_binding.clone(); + detached_binding.detach(HlsOriginAccountDetachedReason::AccountMissingOrExpired, 2_000); + + let hls_ctx = app_state.hls_ctx(); + assert!(!super::super::hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, &known_binding)); + assert!(super::super::hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, &missing_binding)); + assert!(super::super::hls_transient_origin_binding_requires_runtime_prepare(&hls_ctx, &detached_binding)); +} + +#[tokio::test] +async fn hls_account_binding_soft_expiry_retains_session_and_reacquires_on_authorized_manifest_work() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", input.name.as_ref(), 1_000).await; + { + let mut session = session.write().await; + session.target_duration = Some(1); + session.mark_authorized_media_access(1_000); + session.activity.last_delivered_media_at_ms = Some(1_000); + } + let old_generation = session.read().await.activity.origin_work_generation; + + super::super::detach_unprotected_hls_origin_account_bindings(&app_state, 4_001).await; + + { + let session = session.read().await; + let binding = session.origin_account_binding.as_ref().expect("detached binding is retained"); + assert!(matches!( + binding.binding_mode, + HlsOriginAccountBindingMode::Detached { reason: HlsOriginAccountDetachedReason::SoftWindowElapsed, .. } + )); + assert_eq!(session.activity.origin_work_generation, old_generation + 1); + } + + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let prepared_origin = super::super::prepare_hls_origin_runtime( + &app_state, + &session, + &input, + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + "http://root.example.com/live/root-user/root-pass/12345.m3u8", + &proxy_session_id, + &test_fingerprint(), + ConnectionKind::Normal, + 0, + super::super::HlsOriginWorkKind::Manifest, + super::super::HlsOriginWorkClass::ManifestInteractive, + 4_100, + ) + .await + .expect("authorized origin work can reacquire a detached binding"); + + let binding = + prepared_origin.origin_account_binding_to_store.as_ref().expect("new binding should be stored by caller"); + assert_eq!(binding.account_name.as_ref(), "account-a"); + assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active)); + assert_eq!(prepared_origin.fetch_url, "http://account.example.com/live/account-user/account-pass/12345.m3u8"); + app_state.connection_manager.release_provider_handle(prepared_origin.preacquired_origin_account_handle); +} + +#[tokio::test] +async fn hls_account_binding_without_media_activity_expires_after_startup_window() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + { + let mut session = session.write().await; + session.target_duration = Some(12); + session.mark_authorized_manifest_access(1_000); + } + let old_generation = session.read().await.activity.origin_work_generation; + + super::super::detach_unprotected_hls_origin_account_bindings(&app_state, 5_000).await; + assert!(session + .read() + .await + .origin_account_binding + .as_ref() + .is_some_and(tuliprox_hls::api::HlsOriginAccountBinding::is_active)); + super::super::detach_unprotected_hls_origin_account_bindings(&app_state, 60_000).await; + + let session = session.read().await; + let binding = session.origin_account_binding.as_ref().expect("binding remains"); + assert!(matches!( + binding.binding_mode, + HlsOriginAccountBindingMode::Detached { reason: HlsOriginAccountDetachedReason::SoftWindowElapsed, .. } + )); + assert_eq!(session.activity.origin_work_generation, old_generation + 1); + assert_eq!(session.activity.last_authorized_media_at_ms, None); + assert_eq!(session.account_overlap_timing().target_duration_ms, 12_000); +} + +#[tokio::test] +async fn hls_ready_cache_hit_does_not_require_origin_reacquire_when_binding_is_detached() { + let app_state = test_app_state(); + let input = ConfigInput { id: 1, name: Arc::from("overlap-input"), ..ConfigInput::default() }; + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + { + let mut session = session.write().await; + session + .origin_account_binding + .as_mut() + .expect("binding exists") + .detach(HlsOriginAccountDetachedReason::Cleanup, 2_000); + session.segments.insert( + 123, + SegmentEntry { + origin_key: OriginSegmentKey { + origin_epoch: 0, + effective_host_id: 0, + host_local_sequence: 123, + host_local_index: 123, + }, + proxy_seq: 123, + duration_ms: 4_000, + proxy_file_ext: "ts".to_string(), + content_type: "video/mp2t".to_string(), + cache_key: SegmentCacheKey::new(proxy_session_id, 123, "ts"), + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + encryption: None, + origin_fetch_ref: Some(OriginSegmentFetchRef { + resolved_origin_url: "http://origin.example.com/123.ts".to_string(), + byte_range: None, + valid_until_ms: None, + }), + status: SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 1_000 }, + last_rendered_at_ms: None, + access: Arc::new(CacheAccessState::new()), + }, + ); + } + + let segment_file = HlsSegmentFile { proxy_seq: 123, extension: "ts".to_string() }; + assert!(!super::super::hls_segment_request_requires_origin_work(&session, &segment_file).await); + assert!(super::super::hls_origin_binding_needs_reacquire(&session).await); +} + +#[tokio::test] +async fn hls_origin_account_rebind_failure_sets_backoff_without_changing_session_identity() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let input = ConfigInput { id: 1, name: Arc::from("stale-input"), ..ConfigInput::default() }; + let request_url = "http://origin.example.com/live/user/pass/12345.m3u8"; + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + let (session, _) = app_state + .hls + .proxy + .get_or_create_session_with_source_and_outcome( + session_key.clone(), + origin_source.clone(), + &app_state.get_encrypt_secret(), + 1_000, + ) + .await; + { + let mut session_guard = session.write().await; + session_guard.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from("removed-account"), + &proxy_session_id, + 1_000, + )); + } + + let response = super::super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::super::HlsCacheManifestOrigin { + raw_request_url: request_url, + session_entry_url: super::super::HlsOriginEntryUrl::direct_http(request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + .expect("canonical hls cache response"); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers().get(header::RETRY_AFTER).expect("retry after"), "2"); + let session_guard = session.read().await; + assert_eq!(session_guard.key, session_key); + assert_eq!(session_guard.proxy_session_id, proxy_session_id); + let binding = session_guard.origin_account_binding.as_ref().expect("stale binding remains"); + assert_eq!(binding.account_name.as_ref(), "removed-account"); + assert_eq!(binding.generation, 0); + assert_eq!(session_guard.origin_account_rebind.consecutive_rebind_failures, 1); + assert!(session_guard.origin_account_rebind.next_rebind_allowed_at_ms.is_some()); +} + +#[tokio::test] +async fn recording_force_redirect_keeps_vod_and_series_on_the_provider_proxy() -> Result<(), Box> +{ + for (output, cluster, item_type) in [ + (shared::model::TargetType::M3u, XtreamCluster::Video, PlaylistItemType::Video), + (shared::model::TargetType::M3u, XtreamCluster::Series, PlaylistItemType::Series), + (shared::model::TargetType::Xtream, XtreamCluster::Video, PlaylistItemType::Video), + (shared::model::TargetType::Xtream, XtreamCluster::Series, PlaylistItemType::Series), + ] { + // The proxy answers for an origin that cannot be resolved directly. + let (proxy_url, requests, proxy_task) = spawn_recording_header_origin().await; + let provider_url = format!( + "http://recording-origin.invalid/{}/{}12345.mp4", + cluster.as_stream_type(), + if output == shared::model::TargetType::Xtream { "user/pass/" } else { "" } + ); + let input = ConfigInput { + id: 1, + name: Arc::from("recording-input"), + input_type: if output == shared::model::TargetType::Xtream { InputType::Xtream } else { InputType::M3u }, + username: Some("user".to_string()), + password: Some("pass".to_string()), + url: if output == shared::model::TargetType::Xtream { + "http://recording-origin.invalid".to_string() + } else { + provider_url.clone() + }, + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let mut target = if output == shared::model::TargetType::Xtream { + test_hls_share_target(false) + } else { + test_m3u_hls_share_target() + }; + target.use_memory_cache = true; + target.name = "default".to_string(); + if let Some(options) = target.options.as_mut() { + options.force_redirect = Some(shared::model::ClusterFlags::all()); + } + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + let storage = tempfile::tempdir()?; + configure_recording_test_listener(&app_state, storage.path()); + let mut config = (*app_state.app_config.config.load_full()).clone(); + config.proxy = Some(crate::model::ProxyConfig { url: proxy_url, username: None, password: None }); + app_state.app_config.config.store(Arc::new(config)); + let client = crate::utils::request::create_client(&app_state.app_config).build()?; + app_state.http_clients.default.store(Arc::new(client.clone())); + app_state.http_clients.no_redirect.store(Arc::new(client)); + let mut item = test_m3u_hls_item(&input, 12345, "12345", &provider_url); + item.item_type = item_type; + cache_recording_test_item(&app_state, &target, item).await?; + let router = recording_test_router(&app_state); + let recording = + recording_test_response(&router, &recording_test_url(&app_state, &target, &input, cluster)?).await?; + assert_eq!( + recording.status(), + StatusCode::OK, + "{cluster:?}: the recording endpoint must relay the provider body" + ); + assert_eq!(recording.into_body().collect().await?.to_bytes().as_ref(), b"segment-bytes"); + let ordinary_url = if output == shared::model::TargetType::M3u { + format!( + "/{}/{}/hls-user/hls-pass/12345.mp4", + crate::repository::storage_const::M3U_STREAM_PATH, + cluster.as_stream_type() + ) + } else { + format!("/{}/hls-user/hls-pass/12345.mp4", cluster.as_stream_type()) + }; + let ordinary = recording_test_response(&router, &ordinary_url).await?; + assert!( + ordinary.status().is_redirection(), + "{output:?}: ordinary playback keeps force_redirect: {}", + ordinary.status() + ); + assert_eq!( + ordinary.headers().get(header::LOCATION).ok_or("redirect location missing")?.to_str()?, + provider_url + ); + let requests = requests.lock().map_err(|error| error.to_string())?.clone(); + assert_eq!(requests.len(), 1, "{requests:?}"); + assert!(requests[0].starts_with(&format!("get {provider_url} ")), "{requests:?}"); + proxy_task.abort(); + } + Ok(()) +} + +#[tokio::test] +async fn hls_entry_origin_reservation_requires_real_provider_handle() { + let app_state = test_app_state(); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + let input = single_hls_provider_input("missing-provider"); + + let reservation = super::super::try_reserve_hls_entry_origin_account_for_redirect( + &app_state, + &test_fingerprint(), + &user, + &input, + 12345, + "http://origin.example.com/live/source-user/source-pass/12345.m3u8", + "hls-session-token", + "hls-cache:test-session", + crate::model::PlaybackKind::LiveHls, + super::super::hls_origin_account_reservation_ttl_secs_fallback(), + UserConnectionPermission::Allowed, + ConnectionKind::Normal, + false, + ) + .await; + + assert!(reservation.is_none(), "provisioning redirect must not use an exhausted/counter-only check"); +} + +#[tokio::test] +async fn hls_cache_expired_transient_manifest_with_active_binding_is_served_while_manifest_valid() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let now_ms = super::super::current_time_millis(); + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.clone(); + session.transient.replace_manifest_with_semantics( + transient_manifest_body(&proxy_session_id.0), + now_ms.saturating_sub(1_000), + Some(60_000), + ); + session.mark_authorized_media_access(now_ms.saturating_sub(60_000)); + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::from("test-input"), + Arc::from("test-account"), + &proxy_session_id, + now_ms, + )); + session.origin_refresh.in_flight = true; + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::from_millis(200)), + ) + .await + .expect("valid committed transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains(&access_lease_id.0)); +} + +#[tokio::test] +async fn hls_cache_expired_transient_manifest_with_active_binding_is_not_served_after_manifest_validity() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let now_ms = super::super::current_time_millis(); + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.clone(); + session.transient.replace_manifest_with_semantics( + transient_manifest_body(&proxy_session_id.0), + now_ms.saturating_sub(60_000), + Some(1_000), + ); + session.mark_authorized_media_access(now_ms.saturating_sub(60_000)); + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::from("test-input"), + Arc::from("test-account"), + &proxy_session_id, + now_ms, + )); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await; + + assert!(response.is_none()); +} + +#[test] +fn hls_cache_stats_provider_prefers_active_origin_account_binding() { + let origin_source = HlsOriginSource::new(1, Arc::from("cdn-dev"), "12345", HlsOriginSourceKind::XtreamLive); + let proxy_session_id = ProxySessionId("stats-session".to_string()); + let binding = HlsOriginAccountBinding::new( + Arc::clone(&origin_source.input_name), + Arc::from("cdn-dev-alias"), + &proxy_session_id, + 100, + ); + let user_session = stats_provider_test_user_session("cdn-dev"); + + let provider = super::super::hls_cache_stats_provider(&origin_source, Some(&binding), &user_session); + + assert_eq!(provider.as_ref(), "cdn-dev-alias"); +} + +#[test] +fn hls_cache_stats_provider_falls_back_when_origin_account_binding_is_not_active() { + let origin_source = HlsOriginSource::new(1, Arc::from("cdn-dev"), "12345", HlsOriginSourceKind::XtreamLive); + let proxy_session_id = ProxySessionId("stats-session".to_string()); + let mut binding = HlsOriginAccountBinding::new( + Arc::clone(&origin_source.input_name), + Arc::from("cdn-dev-alias"), + &proxy_session_id, + 100, + ); + binding.detach(HlsOriginAccountDetachedReason::Cleanup, 200); + let user_session = stats_provider_test_user_session("session-provider"); + + let provider = super::super::hls_cache_stats_provider(&origin_source, Some(&binding), &user_session); + + assert_eq!(provider.as_ref(), "session-provider"); +} + +#[test] +fn hls_cache_stats_provider_falls_back_to_input_name_without_session_provider() { + let origin_source = HlsOriginSource::new(1, Arc::from("cdn-dev"), "12345", HlsOriginSourceKind::XtreamLive); + let user_session = stats_provider_test_user_session(""); + + let provider = super::super::hls_cache_stats_provider(&origin_source, None, &user_session); + + assert_eq!(provider.as_ref(), "cdn-dev"); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_identity.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_identity.rs new file mode 100644 index 000000000..2a7c9fa48 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_identity.rs @@ -0,0 +1,301 @@ +use super::test_fingerprint; +use crate::{ + api::model::{ + build_proxy_session_id, HlsAccessLeaseId, ProviderConfig as RuntimeProviderConfig, ProviderConfigConnection, + }, + model::{ConfigInput, ConfigProvider}, +}; +use shared::model::{ConfigProviderDto, InputType, ProviderUrlSelectionPolicy}; +use std::sync::Arc; + +#[test] +fn live_hls_entry_tokens_are_unique_but_share_a_stable_provider_owner() { + let fingerprint = test_fingerprint(); + let first = super::super::hls_entry_user_session_token(&fingerprint, "alice", 42, None, None); + let second = super::super::hls_entry_user_session_token(&fingerprint, "alice", 42, None, None); + + assert_ne!(first, second); + assert!(first.contains("|hls|")); + assert!(second.contains("|hls|")); + let first_lease = tuliprox_session::PlaybackLeaseRef::new(&first, tuliprox_core::model::PlaybackKind::LiveHls); + let second_lease = tuliprox_session::PlaybackLeaseRef::new(&second, tuliprox_core::model::PlaybackKind::LiveHls); + assert_ne!(first_lease.owner, second_lease.owner); + assert_eq!(first_lease.provider_owner(), second_lease.provider_owner()); + assert_ne!(first_lease.request_id, second_lease.request_id); + let other_channel = super::super::hls_entry_user_session_token(&fingerprint, "alice", 43, None, None); + let other_user = super::super::hls_entry_user_session_token(&fingerprint, "bob", 42, None, None); + assert_ne!( + first_lease.provider_owner(), + tuliprox_session::PlaybackLeaseRef::new(&other_channel, first_lease.kind).provider_owner() + ); + assert_ne!( + first_lease.provider_owner(), + tuliprox_session::PlaybackLeaseRef::new(&other_user, first_lease.kind).provider_owner() + ); +} + +#[test] +fn hls_manifest_materialization_uses_proxy_paths_without_provider_or_legacy_route() { + let body = format!( + "#EXTM3U\n#EXTINF:4.0,\n/hls/shared/live/proxy-id/{}/000123.ts\n", + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER + ); + let lease_id = HlsAccessLeaseId("access-lease".to_string()); + + let materialized = super::super::materialize_hls_access_manifest(&body, &lease_id, Some("/iptv")); + + assert!(materialized.contains("/iptv/hls/shared/live/proxy-id/access-lease/000123.ts")); + assert!(!materialized.contains("provider://")); + assert!(!materialized.contains("/hls/hls-user/")); + assert!(!materialized.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); +} + +#[test] +fn hls_cache_origin_entry_url_preserves_provider_scheme_as_failover() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec!["http://origin.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let input = ConfigInput { provider_configs: Some(vec![Arc::clone(&provider)]), ..ConfigInput::default() }; + + let origin = + super::super::resolve_hls_cache_origin_entry_url(&input, "provider://demo/live/account-a/token-a/1025130.m3u8") + .expect("provider entry url should resolve"); + + assert_eq!(origin.session_entry_url.as_str(), "provider://demo/live/account-a/token-a/1025130.m3u8"); + assert_eq!( + origin.session_entry_url.url_failover_provider().expect("provider failover config").name.as_ref(), + "demo" + ); + let provider_key = super::super::build_hls_origin_source(&input, "1025130").session_key(); + let direct_key = super::super::build_hls_origin_source(&input, "1025130").session_key(); + assert_eq!(provider_key, direct_key); + assert_eq!(provider_key.stable_value(), "input:0|hls|1025130"); + assert!(!provider_key.stable_value().contains("provider://")); + assert!(!provider_key.stable_value().contains("origin.example.com")); +} + +#[test] +fn hls_cache_origin_entry_url_does_not_attach_url_failover_provider_to_http_url() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec!["http://origin.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let input = ConfigInput { provider_configs: Some(vec![provider]), ..ConfigInput::default() }; + + let origin = super::super::resolve_hls_cache_origin_entry_url( + &input, + "http://origin.example.com/live/account-a/token-a/1025130.m3u8", + ) + .expect("http entry url should resolve"); + + assert_eq!(origin.session_entry_url.as_str(), "http://origin.example.com/live/account-a/token-a/1025130.m3u8"); + assert!(origin.session_entry_url.url_failover_provider().is_none()); +} + +#[test] +fn hls_origin_resolution_keeps_provider_failover_out_of_identity() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "mirror-group".into(), + urls: vec!["http://mirror-a.example.com".into(), "http://mirror-b.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let input = ConfigInput { + id: 7, + name: Arc::from("xtream"), + input_type: InputType::Xtream, + url: "provider://mirror-group".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + provider_configs: Some(vec![Arc::clone(&provider)]), + ..ConfigInput::default() + }; + + let origin = super::super::build_hls_origin_resolution( + &input, + "provider://mirror-group/live/source-user/source-pass/1025126.m3u8", + ) + .expect("provider failover origin should resolve"); + let failover_key = super::super::build_hls_origin_source(&input, "80510").session_key(); + let direct_key = super::super::build_hls_origin_source(&input, "80510").session_key(); + + assert_eq!(origin.session_entry_url.as_str(), "provider://mirror-group/live/source-user/source-pass/1025126.m3u8"); + assert!(origin.session_entry_url.url_failover_provider().is_some()); + assert_eq!(failover_key, direct_key); + assert!(!failover_key.stable_value().contains("provider://")); + assert!(!failover_key.stable_value().contains("mirror-a.example.com")); + assert!(!failover_key.stable_value().contains("mirror-b.example.com")); +} + +#[test] +fn provider_failover_mirror_change_keeps_same_hls_session_identity() { + let provider_a = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "mirror-group".into(), + urls: vec!["http://mirror-a.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let provider_b = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "mirror-group".into(), + urls: vec!["http://mirror-b.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let input_a = ConfigInput { + id: 7, + name: Arc::from("xtream"), + input_type: InputType::Xtream, + url: "provider://mirror-group".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + provider_configs: Some(vec![provider_a]), + ..ConfigInput::default() + }; + let input_b = ConfigInput { provider_configs: Some(vec![provider_b]), ..input_a.clone() }; + + let origin_a = super::super::build_hls_origin_resolution(&input_a, "provider://mirror-group/a.m3u8") + .expect("provider failover origin a should resolve"); + let origin_b = super::super::build_hls_origin_resolution(&input_b, "provider://mirror-group/b.m3u8") + .expect("provider failover origin b should resolve"); + let key_a = super::super::build_hls_origin_source(&input_a, "80510").session_key(); + let key_b = super::super::build_hls_origin_source(&input_b, "80510").session_key(); + let secret = b"rewrite-secret"; + + assert!(origin_a.session_entry_url.url_failover_provider().is_some()); + assert!(origin_b.session_entry_url.url_failover_provider().is_some()); + assert_eq!(key_a, key_b); + assert_eq!(key_a.stable_value(), "input:7|hls|80510"); + assert_eq!(build_proxy_session_id(&key_a, secret), build_proxy_session_id(&key_b, secret)); + assert!(!key_a.stable_value().contains("provider://")); + assert!(!key_a.stable_value().contains("mirror-a.example.com")); + assert!(!key_a.stable_value().contains("mirror-b.example.com")); +} + +#[test] +fn hls_runtime_origin_fetch_url_uses_selected_provider_account() { + let input = ConfigInput { + name: Arc::from("source"), + url: "http://source.example.com".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + input_type: InputType::Xtream, + ..ConfigInput::default() + }; + let provider_input = ConfigInput { + id: 7, + name: Arc::from("selected-provider"), + url: "http://provider.example.com".to_string(), + username: Some("provider-user".to_string()), + password: Some("provider-pass".to_string()), + input_type: InputType::Xtream, + max_connections: 1, + ..ConfigInput::default() + }; + let provider = Arc::new(RuntimeProviderConfig::new( + &provider_input, + Arc::new(std::sync::RwLock::new(ProviderConfigConnection::default())), + Arc::new(|_, _| {}), + )); + + let fetch_url = super::super::build_hls_origin_fetch_url( + &input, + "http://source.example.com/live/source-user/source-pass/12345.m3u8", + "http://source.example.com/live/source-user/source-pass/12345.m3u8", + Some(&provider), + ) + .expect("fetch url should be rewritten"); + + assert_eq!(fetch_url, "http://provider.example.com/live/provider-user/provider-pass/12345.m3u8"); + + let failover_provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec!["http://mirror-a.example.com".into(), "http://mirror-b.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let provider_scheme_input = ConfigInput { + name: Arc::from("source"), + url: "provider://demo".to_string(), + username: Some("source-user".to_string()), + password: Some("source-pass".to_string()), + input_type: InputType::Xtream, + provider_configs: Some(vec![Arc::clone(&failover_provider)]), + ..ConfigInput::default() + }; + let provider_scheme_without_account_rewrite = super::super::build_hls_origin_fetch_url( + &provider_scheme_input, + "provider://demo/live/source-user/source-pass/12345.m3u8", + "provider://demo/live/source-user/source-pass/12345.m3u8", + None, + ) + .expect("provider scheme fetch url should remain failover capable"); + assert_eq!(provider_scheme_without_account_rewrite, "provider://demo/live/source-user/source-pass/12345.m3u8"); + + let provider_scheme_fetch_url = super::super::build_hls_origin_fetch_url( + &provider_scheme_input, + "provider://demo/live/source-user/source-pass/12345.m3u8", + "provider://demo/live/source-user/source-pass/12345.m3u8", + Some(&provider), + ) + .expect("provider scheme fetch url should use selected runtime account without losing failover"); + + assert_eq!(provider_scheme_fetch_url, "provider://demo/live/provider-user/provider-pass/12345.m3u8"); + let origin_entry = super::LiveHlsOriginEntry::parse_with_provider_configs( + &provider_scheme_without_account_rewrite, + Some(Arc::clone(&failover_provider)), + Some(Arc::clone(&provider)), + ) + .expect("provider origin entry"); + let input_source = origin_entry.to_input_source(); + assert_eq!(input_source.url, provider_scheme_without_account_rewrite); + assert_eq!(input_source.username.as_deref(), Some("provider-user")); + assert_eq!(input_source.password.as_deref(), Some("provider-pass")); + + let failover_context = super::super::hls_url_failover_provider_for_origin_context( + &provider_scheme_input, + "provider://demo/live/source-user/source-pass/12345.m3u8", + "provider://demo/live/source-user/source-pass/12345.m3u8", + &provider_scheme_fetch_url, + ) + .expect("provider failover context"); + assert_eq!(failover_context.name.as_ref(), "demo"); +} + +#[test] +fn hls_origin_entry_attaches_url_failover_provider_only_to_provider_scheme_fetch_url() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec!["http://mirror.example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + + let http_provider = super::super::effective_hls_url_failover_provider_for_fetch_url( + "http://provider.example.com/live/user/pass/12345.m3u8", + None, + Some(Arc::clone(&provider)), + ); + let http_entry = super::LiveHlsOriginEntry::parse_with_url_failover_provider( + "http://provider.example.com/live/user/pass/12345.m3u8", + http_provider, + ) + .expect("http origin entry"); + assert!(http_entry.url_failover_provider().is_none()); + + let provider_scheme_provider = super::super::effective_hls_url_failover_provider_for_fetch_url( + "provider://demo/live/user/pass/12345.m3u8", + None, + Some(Arc::clone(&provider)), + ); + let provider_entry = super::LiveHlsOriginEntry::parse_with_url_failover_provider( + "provider://demo/live/user/pass/12345.m3u8", + provider_scheme_provider, + ) + .expect("provider origin entry"); + assert_eq!(provider_entry.url_failover_provider().expect("provider").name.as_ref(), "demo"); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_lease_release.rs b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_lease_release.rs new file mode 100644 index 000000000..eeeb814e6 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/provider_admission_provider_lease_release.rs @@ -0,0 +1,383 @@ +use super::{ + activate_test_hls_access_lease, create_bound_hls_test_session, get_response, hls_proxy_uri, map_transient_resource, + overlap_provider_input, provider_admission::register_test_hls_stream_for_lease_release, response_body, + single_hls_provider_input, spawn_test_transient_origin, test_addr_with_port, test_app_state_with_inputs, + test_segment_entry, wait_for_provider_connection_count, +}; +use crate::api::model::{ + ConnectionKind, HlsOriginAccountBinding, HlsOriginAccountBindingMode, MapCacheStatus, MapEntry, OriginMapKey, + ProxyMapId, ProxySessionId, SegmentCacheStatus, SegmentFetchPriority, +}; +use axum::http::StatusCode; +use std::sync::Arc; + +#[tokio::test] +async fn hls_access_lease_idle_releases_user_but_keeps_origin_binding_and_queues() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let binding = session.read().await.origin_account_binding.clone().expect("binding exists"); + let account_name = Arc::from("account-a"); + app_state.active_provider.refresh_provider_reservation(&account_name, &binding.session_owner, 60); + app_state.active_provider.confirm_playback_activity(&binding.session_owner); + assert!(app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner"))); + activate_test_hls_access_lease(&app_state, &proxy_session_id, "detach-lease", 1_000, 1_000).await; + register_test_hls_stream_for_lease_release(&app_state, &session, &proxy_session_id, input.name.as_ref()).await; + assert_eq!(app_state.active_users.active_streams().await.len(), 1); + app_state + .hls + .proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + &proxy_session_id, + 1_000, + ) + .await; + { + let mut session = session.write().await; + assert_eq!(session.activity.active_access_lease_count, 1); + session.segments.insert(1, test_segment_entry(&proxy_session_id, 1, SegmentCacheStatus::Discovered)); + session.queue_segment_fetch_candidate(1, SegmentFetchPriority::Prefetch, 1_100); + session.segments.insert( + 2, + test_segment_entry( + &proxy_session_id, + 2, + SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 1_000 }, + ), + ); + let map_id = ProxyMapId(1); + let mut map = MapEntry::new( + &proxy_session_id, + map_id, + OriginMapKey { + origin_epoch: 0, + resolved_origin_uri: "http://origin.example.com/init.mp4".to_string(), + byte_range: None, + }, + "mp4".to_string(), + ); + map.status = MapCacheStatus::Queued { queued_at_ms: 1_100 }; + session.maps.insert(map_id, map); + } + let old_generation = session.read().await.activity.origin_work_generation; + + app_state + .hls + .proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + &proxy_session_id, + 3_000, + ) + .await; + + { + let session = session.read().await; + assert_eq!(session.activity.active_access_lease_count, 0); + let binding = session.origin_account_binding.as_ref().expect("binding is retained"); + assert!(matches!(binding.binding_mode, HlsOriginAccountBindingMode::Active)); + assert_eq!(session.activity.origin_work_generation, old_generation); + assert!(!session.segment_prefetch_queue.is_empty()); + assert!(matches!( + session.segments.get(&1).expect("queued segment remains").status, + SegmentCacheStatus::Queued { .. } + )); + assert!(matches!( + session.segments.get(&2).expect("ready segment remains").status, + SegmentCacheStatus::Ready { .. } + )); + assert!(matches!(session.maps.get(&ProxyMapId(1)).expect("map remains").status, MapCacheStatus::Queued { .. })); + } + assert!(app_state.active_provider.is_provider_reserved_for_other_session(&account_name, Some("other-owner"))); + assert!(app_state.active_users.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn hls_access_lease_sync_keeps_binding_and_queue_when_active_count_remains_positive() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + activate_test_hls_access_lease(&app_state, &proxy_session_id, "still-active-lease", 1_000, 10_000).await; + { + let mut session = session.write().await; + session.activity.active_access_lease_count = 1; + session.segments.insert(1, test_segment_entry(&proxy_session_id, 1, SegmentCacheStatus::Discovered)); + session.queue_segment_fetch_candidate(1, SegmentFetchPriority::Prefetch, 1_100); + } + let old_generation = session.read().await.activity.origin_work_generation; + + app_state + .hls + .proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + &proxy_session_id, + 1_500, + ) + .await; + + let session = session.read().await; + assert_eq!(session.activity.active_access_lease_count, 1); + assert!(matches!( + session.origin_account_binding.as_ref().expect("binding exists").binding_mode, + HlsOriginAccountBindingMode::Active + )); + assert_eq!(session.activity.origin_work_generation, old_generation); + assert!(!session.segment_prefetch_queue.is_empty()); + assert!(matches!( + session.segments.get(&1).expect("segment remains queued").status, + SegmentCacheStatus::Queued { .. } + )); +} + +#[tokio::test] +async fn hls_access_lease_gc_prepass_releases_user_without_detaching_origin_binding() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let session = create_bound_hls_test_session(&app_state, &input, "12345", "account-a", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + activate_test_hls_access_lease(&app_state, &proxy_session_id, "gc-expired-lease", 1_000, 1_000).await; + app_state + .hls + .proxy + .sync_session_access_lease_count_and_detach_if_needed( + &app_state.active_users, + &app_state.active_provider, + &session, + &proxy_session_id, + 1_000, + ) + .await; + let old_generation = session.read().await.activity.origin_work_generation; + + app_state + .hls + .proxy + .sync_all_session_access_leases_and_detach_if_needed(&app_state.active_users, &app_state.active_provider, 3_000) + .await; + let _ = app_state.hls.proxy.run_garbage_collection_once(3_000).await.expect("gc should run"); + + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&proxy_session_id) + .await + .expect("detach keeps shared hls session"); + let session = session.read().await; + assert_eq!(session.activity.active_access_lease_count, 0); + assert!(matches!( + session.origin_account_binding.as_ref().expect("binding exists").binding_mode, + HlsOriginAccountBindingMode::Active + )); + assert_eq!(session.activity.origin_work_generation, old_generation); +} + +#[tokio::test] +async fn transient_resource_holds_provider_handle_until_origin_body_is_finished() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let origin = spawn_test_transient_origin().await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let proxy_session_id_value = ProxySessionId(proxy_session_id.clone()); + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&proxy_session_id_value) + .await + .expect("session should exist"); + { + let mut session = session.write().await; + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from("account-a"), + &proxy_session_id_value, + super::super::current_time_millis(), + )); + } + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-15")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + wait_for_provider_connection_count(&app_state, 1).await; + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"transient-body")); + wait_for_provider_connection_count(&app_state, 0).await; +} + +#[tokio::test] +async fn terminate_failed_hls_manifest_session_releases_identified_provider_reservation() { + let input = single_hls_provider_input("failed-hls-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let username = "testuser"; + let session_token = "testuser|stream-1|hls|0123456789abcdef"; + let provider_name = Arc::clone(&input.name); + let addr = test_addr_with_port(55301); + + let handle = app_state + .active_provider + .acquire_connection_with_lease_for_session( + &provider_name, + &addr, + false, + 0, + ConnectionKind::Normal, + Some(tuliprox_session::PlaybackLeaseRef::new(session_token, crate::model::PlaybackKind::LiveHls)), + ) + .expect("handle should be acquired"); + + let binding_tag = handle.binding_tag; + let request_id = handle.playback_request_id; + assert!(binding_tag.is_some(), "handle should carry a binding tag"); + + app_state.connection_manager.release_provider_handle(Some(handle)); + + assert!( + app_state.active_provider.binding_tag_for_owner(session_token).is_some(), + "lease should still be active after releasing connection handle" + ); + + app_state.active_provider.clear_provider_reservation(session_token); + assert!( + app_state.active_provider.binding_tag_for_owner(session_token).is_some(), + "clear_provider_reservation must return early for public HLS token" + ); + + super::super::segment::terminate_failed_hls_manifest_session( + &app_state, + username, + session_token, + Some(&provider_name), + binding_tag, + request_id, + ) + .await; + + assert!( + app_state.active_provider.binding_tag_for_owner(session_token).is_none(), + "lease should be cleared after terminate_failed_hls_manifest_session with binding identity" + ); +} + +#[tokio::test] +async fn delayed_manifest_failure_of_older_binding_keeps_provider_affinity() { + let input = single_hls_provider_input("affinity-hls-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let provider_name = Arc::clone(&input.name); + let old_token = "testuser|stream-1|hls|0123456789abcdef"; + let new_token = "testuser|stream-1|hls|fedcba9876543210"; + let acquire = |token: &'static str, port: u16| { + app_state + .active_provider + .acquire_connection_with_lease_for_session( + &provider_name, + &test_addr_with_port(port), + false, + 0, + ConnectionKind::Normal, + Some(tuliprox_session::PlaybackLeaseRef::new(token, crate::model::PlaybackKind::LiveHls)), + ) + .expect("handle should be acquired") + }; + + // The older binding never produced media and ended. + let old = acquire(old_token, 55311); + let (old_tag, old_request) = (old.binding_tag, old.playback_request_id); + let old_request_id = old_request.expect("old request id"); + app_state.connection_manager.release_provider_handle(Some(old)); + app_state.active_provider.finish_identified_playback_request( + old_token, + old_request_id, + tuliprox_core::model::PlaybackRequestOutcome::FailedBeforeMedia, + ); + + // The successor binding confirms media and owns the provider preference. + let current = acquire(new_token, 55312); + let current_request = current.playback_request_id.expect("current request id"); + app_state.active_provider.refresh_adaptive_playback_lease( + &provider_name, + new_token, + crate::model::PlaybackKind::LiveHls, + 15, + ); + app_state.active_provider.confirm_identified_playback_activity(new_token, current_request); + app_state.connection_manager.release_provider_handle(Some(current)); + assert_eq!(app_state.active_provider.provider_affinity_for_owner(new_token), Some(Arc::clone(&provider_name))); + + for (tag, request) in [(old_tag, old_request), (old_tag, None)] { + super::super::segment::terminate_failed_hls_manifest_session( + &app_state, + "testuser", + old_token, + Some(&provider_name), + tag, + request, + ) + .await; + } + + assert_eq!( + app_state.active_provider.provider_affinity_for_owner(new_token), + Some(Arc::clone(&provider_name)), + "a delayed failure of the older binding must not end the successor's provider preference" + ); +} + +#[tokio::test] +async fn terminating_unknown_old_hls_session_keeps_newer_provider_lease() { + use axum::response::IntoResponse; + let input = single_hls_provider_input("terminate-hls-input"); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let provider_name = Arc::clone(&input.name); + let old_token = "testuser|stream-1|hls|0123456789abcdef"; + let new_token = "testuser|stream-1|hls|fedcba9876543210"; + let current = app_state + .active_provider + .acquire_connection_with_lease_for_session( + &provider_name, + &test_addr_with_port(55321), + false, + 0, + ConnectionKind::Normal, + Some(tuliprox_session::PlaybackLeaseRef::new(new_token, crate::model::PlaybackKind::LiveHls)), + ) + .expect("handle should be acquired"); + let request_id = current.playback_request_id.expect("request id"); + app_state.active_provider.refresh_adaptive_playback_lease( + &provider_name, + new_token, + crate::model::PlaybackKind::LiveHls, + 15, + ); + app_state.active_provider.confirm_identified_playback_activity(new_token, request_id); + app_state.connection_manager.release_provider_handle(Some(current)); + app_state.active_provider.finish_identified_playback_request( + new_token, + request_id, + tuliprox_core::model::PlaybackRequestOutcome::Completed, + ); + let binding_tag = app_state.active_provider.binding_tag_for_owner(new_token); + assert!(binding_tag.is_some(), "the newer playback keeps an idle lease"); + + let response = crate::api::endpoints::v1_api_user::terminate_user_session( + axum::extract::State(Arc::clone(&app_state)), + axum::extract::Path(("testuser".to_string(), old_token.to_string())), + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::NO_CONTENT); + assert_eq!(app_state.active_provider.binding_tag_for_owner(new_token), binding_tag); + assert_eq!(app_state.active_provider.provider_affinity_for_owner(new_token), Some(provider_name)); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/runtime_policy.rs b/backend/app/src/api/endpoints/hls_api/tests/runtime_policy.rs new file mode 100644 index 000000000..b89b53bf9 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/runtime_policy.rs @@ -0,0 +1,177 @@ +use super::{ + get_response, get_status, mark_hls_user_session_exhausted, response_body, runtime_policy_endpoint_fixture, + wait_for_runtime_policy_terminal_plan, RuntimePolicyEndpointFixture, +}; +use crate::api::model::{HlsAccessLeaseState, HlsAccessLeaseTouch, HlsLeasePlaybackMode, HlsRuntimeCustomTailReason}; +use axum::http::{header, StatusCode}; +use std::sync::Arc; + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_runtime_policy_terminal_segments( + fixture: &RuntimePolicyEndpointFixture, + terminal_prefix: &str, + segment_count: u16, +) { + for index in [0, 1, segment_count.saturating_sub(1)] { + let uri = format!("{terminal_prefix}{index}.ts"); + let response = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(response.status(), StatusCode::OK, "terminal segment {index}"); + assert!(!response_body(response).await.is_empty()); + } +} + +#[tokio::test] +async fn resource_access_denial_commits_and_preserves_user_exhausted_tail() { + let fixture = runtime_policy_endpoint_fixture(true).await; + let session = fixture + .app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&fixture.proxy_session_id) + .await + .expect("runtime policy session"); + let origin_refresh_before = session.read().await.origin_refresh.clone(); + mark_hls_user_session_exhausted(&fixture.app_state).await; + + let denied_live = get_response(Arc::clone(&fixture.app_state), &fixture.live_segment_uri, None).await; + assert_eq!(denied_live.status(), StatusCode::FORBIDDEN); + let revoking = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("revoking lease snapshot"); + assert_eq!(revoking.state, HlsAccessLeaseState::PolicyRevoking); + assert_eq!(revoking.playback_mode, HlsLeasePlaybackMode::Live); + assert_eq!( + revoking.runtime_policy_revocation.as_ref().map(|revocation| revocation.reason), + Some(HlsRuntimeCustomTailReason::UserConnectionsExhausted) + ); + + let pending_manifest = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; + assert_eq!(pending_manifest.status(), StatusCode::SERVICE_UNAVAILABLE); + let committed_plan = wait_for_runtime_policy_terminal_plan(&fixture).await; + let manifest_response = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; + assert_eq!(manifest_response.status(), StatusCode::OK); + assert!(!manifest_response.headers().contains_key(header::LOCATION)); + let manifest = String::from_utf8(response_body(manifest_response).await.to_vec()) + .expect("runtime policy terminal manifest utf8"); + assert!(manifest.ends_with("#EXT-X-ENDLIST\n")); + assert!(!manifest.contains("/cvs/hls/")); + + let committed = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("committed runtime policy lease"); + assert_eq!(committed.state, HlsAccessLeaseState::Denied); + let HlsLeasePlaybackMode::TerminalTail(plan) = committed.playback_mode else { + panic!("resource denial must commit a finite terminal plan"); + }; + assert_eq!(plan.generation, committed_plan.generation); + assert_eq!(plan.reason, HlsRuntimeCustomTailReason::UserConnectionsExhausted); + assert!(plan.segment_count >= 2); + let terminal_prefix = format!( + "/hls/shared/live/{}/{}/terminal/{}/", + fixture.proxy_session_id.0, fixture.lease_id.0, plan.generation.0 + ); + assert_eq!(manifest.matches(&terminal_prefix).count(), usize::from(plan.segment_count)); + + assert_runtime_policy_terminal_segments(&fixture, &terminal_prefix, plan.segment_count).await; + let replay = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; + assert_eq!(replay.status(), StatusCode::OK); + assert_eq!( + String::from_utf8(response_body(replay).await.to_vec()).expect("replayed runtime policy manifest utf8"), + manifest + ); + assert_eq!(get_status(Arc::clone(&fixture.app_state), &fixture.live_segment_uri).await, StatusCode::FORBIDDEN); + let retained = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("retained runtime policy plan"); + assert!(matches!( + retained.playback_mode, + HlsLeasePlaybackMode::TerminalTail(ref current) + if current.generation == plan.generation + && current.reason == HlsRuntimeCustomTailReason::UserConnectionsExhausted + )); + assert_eq!(session.read().await.origin_refresh, origin_refresh_before); +} + +#[tokio::test] +async fn manifest_touch_denied_replays_policy_tail_instead_of_standalone_clock() { + let fixture = runtime_policy_endpoint_fixture(true).await; + let _ = fixture + .app_state + .hls + .proxy + .begin_runtime_policy_revocation( + &fixture.lease_id, + &fixture.proxy_session_id, + HlsRuntimeCustomTailReason::UserConnectionsExhausted, + super::super::current_time_millis(), + ) + .await; + assert_eq!( + fixture + .app_state + .hls + .proxy + .touch_manifest_access_lease( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + None, + None, + 60_000, + ) + .await, + HlsAccessLeaseTouch::Denied + ); + + let pending = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; + assert_eq!(pending.status(), StatusCode::SERVICE_UNAVAILABLE); + let committed_plan = wait_for_runtime_policy_terminal_plan(&fixture).await; + let response = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("policy touch manifest utf8"); + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("touch-denied lease"); + let HlsLeasePlaybackMode::TerminalTail(plan) = lease.playback_mode else { + panic!("touch denial must retain a lease-bound terminal plan"); + }; + assert_eq!(plan.generation, committed_plan.generation); + assert!(body.contains(&format!( + "/hls/shared/live/{}/{}/terminal/{}/0.ts", + fixture.proxy_session_id.0, fixture.lease_id.0, plan.generation.0 + ))); + assert!(!body.contains("/cvs/hls/")); + assert!(body.ends_with("#EXT-X-ENDLIST\n")); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/segment_responses.rs b/backend/app/src/api/endpoints/hls_api/tests/segment_responses.rs new file mode 100644 index 000000000..e618c582b --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/segment_responses.rs @@ -0,0 +1,320 @@ +use super::{ + assert_hls_cache_stream_registered, assert_no_hls_cache_stream_registered, get_response, get_status, hls_proxy_uri, + hls_session_last_media_at_ms, map_hls_map, map_ready_segment, map_ready_segment_without_lease, map_segment, + map_segment_with_origin_url, response_body, spawn_test_segment_origin, test_app_state, + test_app_state_with_hls_proxy, +}; +use crate::api::model::{HlsProxyManager, ProxySessionId, SegmentCacheStatus}; +use axum::http::{header, StatusCode}; +use std::sync::Arc; + +#[tokio::test] +async fn valid_hls_proxy_segment_without_session_returns_not_found() { + let status = get_status(test_app_state(), "/hls/shared/live/a8f31c9eQ7sLk92pV0mTaw/000123.ts").await; + + assert_eq!(status, StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn valid_hls_proxy_segment_with_not_ready_session_returns_not_found() { + let app_state = test_app_state(); + let proxy_session_id = map_segment(&app_state, 123, "ts").await; + + let status = get_status(app_state, &format!("/hls/shared/live/{proxy_session_id}/000123.ts")).await; + + assert_eq!(status, StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn valid_hls_proxy_segment_with_not_ready_and_valid_lease_returns_service_unavailable() { + let app_state = test_app_state(); + let proxy_session_id = map_segment(&app_state, 123, "ts").await; + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + session.write().await.segments.get_mut(&123).expect("segment should exist").origin_fetch_ref = None; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers()[header::RETRY_AFTER], "1"); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; +} + +#[tokio::test] +async fn not_ready_hls_proxy_segment_with_fetch_ref_demand_fetches_and_returns_ok() { + let origin = spawn_test_segment_origin(b"0123456789").await; + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = + map_segment_with_origin_url(&app_state, 123, "ts", &format!("{}/seg.ts", origin.base_url)).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); +} + +#[tokio::test] +async fn not_ready_hls_proxy_segment_with_range_waits_for_demand_fetch_then_returns_partial() { + let origin = spawn_test_segment_origin(b"0123456789").await; + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = + map_segment_with_origin_url(&app_state, 123, "ts", &format!("{}/seg.ts", origin.base_url)).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(app_state, &uri, Some("bytes=2-5")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"2345")); +} + +#[tokio::test] +async fn not_ready_hls_proxy_segment_without_fetch_ref_returns_service_unavailable() { + let app_state = test_app_state(); + let proxy_session_id = map_segment(&app_state, 123, "ts").await; + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + session.write().await.segments.get_mut(&123).expect("segment should exist").origin_fetch_ref = None; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers()[header::RETRY_AFTER], "1"); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; +} + +#[tokio::test] +async fn ready_hls_proxy_segment_without_lease_returns_not_found() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment_without_lease(&app_state, 123, "ts", b"0123456789").await; + + let status = get_status(app_state, &format!("/hls/shared/live/{proxy_session_id}/000123.ts")).await; + + assert_eq!(status, StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn ready_hls_proxy_segment_without_range_returns_ok() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp2t"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "10"); + assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); + assert_eq!(response.headers()[header::CACHE_CONTROL], "public, max-age=300, immutable"); + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + { + let session = session.read().await; + let segment = session.segments.get(&123).expect("segment should exist"); + assert_eq!(segment.access.active_readers(), 1); + assert!(matches!(segment.status, SegmentCacheStatus::Ready { content_length: 10, .. })); + } + + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); + + { + let session = session.read().await; + let segment = session.segments.get(&123).expect("segment should exist"); + assert_eq!(segment.access.active_readers(), 0); + assert!(segment.access.last_accessed_at_ms() > 0); + assert!(session.activity.last_authorized_media_at_ms.is_some()); + } + assert_hls_cache_stream_registered(&app_state, &proxy_session_id).await; +} + +#[tokio::test] +async fn ready_hls_proxy_segment_range_zero_open_returns_partial_content() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "m4s", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.m4s").await; + + let response = get_response(app_state, &uri, Some("bytes=0-")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp4"); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 0-9/10"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "10"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); +} + +#[tokio::test] +async fn ready_hls_proxy_segment_range_start_open_returns_partial_content_from_offset() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "m4v", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.m4v").await; + + let response = get_response(app_state, &uri, Some("bytes=4-")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp4"); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 4-9/10"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "6"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"456789")); +} + +#[tokio::test] +async fn ready_hls_proxy_segment_range_start_end_returns_partial_content() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(app_state, &uri, Some("bytes=2-5")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "4"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"2345")); +} + +#[tokio::test] +async fn ready_hls_proxy_segment_suffix_range_returns_partial_content() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(app_state, &uri, Some("bytes=-3")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 7-9/10"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"789")); +} + +#[tokio::test] +async fn ready_hls_proxy_segment_unsatisfiable_range_returns_416() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=99-")).await; + + assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes */10"); + assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "0"); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; +} + +#[tokio::test] +async fn ready_hls_proxy_segment_multi_range_returns_416() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=0-1,4-5")).await; + + assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes */10"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "0"); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; +} + +#[tokio::test] +async fn ready_hls_proxy_map_without_range_returns_ok() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_hls_map(&app_state, b"0123456789", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "map/000000.mp4").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()[header::CONTENT_TYPE], "video/mp4"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "10"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); + assert!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await.is_some()); + assert_hls_cache_stream_registered(&app_state, &proxy_session_id).await; +} + +#[tokio::test] +async fn ready_hls_proxy_map_range_returns_partial_content() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_hls_map(&app_state, b"0123456789", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "map/000000.mp4").await; + + let response = get_response(app_state, &uri, Some("bytes=2-5")).await; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"2345")); +} + +#[tokio::test] +async fn ready_hls_proxy_map_multi_range_returns_416_with_zero_content_length() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_hls_map(&app_state, b"0123456789", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "map/000000.mp4").await; + + let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=0-1,4-5")).await; + + assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes */10"); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "0"); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; +} + +#[tokio::test] +async fn ready_hls_proxy_segment_unknown_range_unit_is_ignored() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "000123.ts").await; + + let response = get_response(app_state, &uri, Some("items=0-1")).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body(response).await, bytes::Bytes::from_static(b"0123456789")); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/support.rs b/backend/app/src/api/endpoints/hls_api/tests/support.rs new file mode 100644 index 000000000..545a791e1 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/support.rs @@ -0,0 +1,62 @@ +mod app_state; +mod crypto; +mod http; +mod manifest; +mod origins; +mod owner_handoff; +mod recording; +mod runtime_policy; +mod sessions; +mod terminal; + +pub(in crate::api::endpoints::hls_api::tests) use self::{ + app_state::{ + cache_test_m3u_hls_item, configure_default_test_server, enable_hls_cache, hls_custom_video_test_user, + overlap_provider_input, path_has_extension, publish_test_transient_resource_membership, + single_hls_provider_input, single_variant_master_playlist, store_test_sources_with_target, test_app_state, + test_app_state_with_hls_proxy, test_app_state_with_hls_proxy_and_inputs, test_app_state_with_inputs, + test_custom_video_buffer, test_hls_access_context, test_hls_access_context_with, test_hls_entry_stream_context, + test_hls_input, test_hls_share_target, test_m3u_hls_item, test_m3u_hls_share_target, test_segment_entry, + }, + crypto::{ + encrypt_test_aes128_cbc_pkcs7, spawn_test_encrypted_hls_origin, test_hls_sequence_iv, AES_TEST_KEY_BYTES, + AES_TEST_MANIFEST, AES_TEST_PLAINTEXT_SEGMENT, + }, + http::{ + access_lease_id_from_variant_uri, disable_custom_stream_response, enable_channel_unavailable_custom_response, + get_response, get_status, hls_proxy_uri, proxy_session_id_from_variant_uri, request_response, response_body, + single_variant_uri, try_test_hls_cached_manifest_response, + }, + manifest::{ + legacy_manifest_test_client_headers, legacy_manifest_test_input, manifest_media_sequence, map_hls_map, + map_ready_segment, map_ready_segment_without_lease, map_segment, map_segment_with_origin_url, + map_transient_resource, map_transient_resource_with_kind, media_uri_count, normal_manifest, + normal_manifest_body, publish_ready_test_manifest_for_lease, record_test_normal_manifest_commit, + regression_origin_manifest, store_normal_manifest_body, transient_manifest_body, + transient_manifest_body_from_sequence, + }, + origins::{ + encode_test_manifest, spawn_test_binary_origin, spawn_test_encoded_manifest_origin, spawn_test_segment_origin, + spawn_test_status_origin, spawn_test_transient_origin, + spawn_test_transient_origin_with_delayed_binary_response, spawn_test_transient_origin_with_delayed_response, + spawn_test_transient_origin_with_response, TestBinaryOriginResponse, TestEncodedManifestOrigin, + TestSegmentOrigin, + }, + owner_handoff::{publish_owner_handoff_test_manifest, CanonicalOwnerHandoffFixture}, + recording::{ + cache_recording_test_item, configure_recording_test_listener, recording_test_response, recording_test_router, + recording_test_url, spawn_recording_header_origin, + }, + runtime_policy::{ + runtime_policy_endpoint_fixture, wait_for_runtime_policy_terminal_plan, RuntimePolicyEndpointFixture, + }, + sessions::{ + access_lease_session_token, activate_test_hls_access_lease, assert_hls_cache_stream_registered, + assert_no_hls_cache_stream_registered, create_active_hls_user_session, create_active_hls_user_session_with, + create_bound_hls_test_session, create_unbound_hls_test_session, grant_hls_proxy_lease, + hls_session_last_media_at_ms, mark_hls_user_session_exhausted, prepare_pending_test_hls_access_lease, + test_addr, test_addr_with_port, test_fingerprint, test_fingerprint_with_addr, + wait_for_provider_connection_count, + }, + terminal::{terminal_test_asset, terminal_test_plan_shape, terminalize_existing_test_lease}, +}; diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs b/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs new file mode 100644 index 000000000..ee59bdec1 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/support/app_state.rs @@ -0,0 +1,458 @@ +use super::{ + super::{ + build_group_lookup, ActiveProviderManager, ActiveUserManager, ApiProxyConfig, ApiProxyServerInfo, AppConfig, + AppState, Arc, ArcSwap, ArcSwapOption, Body, CacheAccessState, CancelTokens, Config, ConfigInput, ConfigPaths, + ConfigSource, ConfigTarget, ConfigTargetDto, ConfigTargetOptions, ConfigTargetShareLiveStreams, + ConnectionManager, EventManager, GeoIp, HlsAccessContext, HlsAccessLeaseId, HlsCacheConfigDto, + HlsLeaseManifestSnapshot, HlsManifestCommitIdentity, HlsManifestDeliveryMode, HlsMediaContainer, + HlsPlaybackFamilyKey, HlsProxyManager, HlsPublishedTransientResourceIds, InputType, Internable, + M3uPlaylistItem, M3uTargetOutputDto, MetadataUpdateManager, OriginSegmentFetchRef, OriginSegmentKey, + PlaylistItem, PlaylistItemHeader, PlaylistItemType, PlaylistStorageState, ProxySessionId, ProxyUserCredentials, + Response, ReverseProxyConfig, ReverseProxyConfigDto, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, + SharedStreamManager, SourcesConfig, TargetOutputDto, TargetUser, TransportStreamBuffer, VirtualId, + XtreamCluster, XtreamTargetOutputDto, + }, + response_body, test_fingerprint, +}; + +pub(in crate::api::endpoints::hls_api::tests) fn path_has_extension(path: &str, extension: &str) -> bool { + std::path::Path::new(path).extension().is_some_and(|actual| actual.eq_ignore_ascii_case(extension)) +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_app_config() -> Arc { + let mut hls_user = ProxyUserCredentials::default(); + hls_user.username = "hls-user".to_string(); + hls_user.password = "hls-pass".to_string(); + hls_user.max_connections = 1; + let api_proxy = ApiProxyConfig { + server: vec![ApiProxyServerInfo { + name: "default".to_string(), + protocol: "https".to_string(), + host: "example.test".to_string(), + port: None, + timezone: "UTC".to_string(), + message: String::new(), + path: Some("iptv".to_string()), + }], + user: vec![TargetUser { target: "default".to_string(), credentials: vec![Arc::new(hls_user)] }], + ..Default::default() + }; + Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config { custom_stream_response_enabled: true, ..Default::default() })), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::empty()), + api_proxy: Arc::new(ArcSwapOption::from(Some(Arc::new(api_proxy)))), + file_locks: Arc::new(crate::utils::FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::empty()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(crate::model::MediaToolCapabilities::default()), + }) +} + +pub(in crate::api::endpoints::hls_api::tests) fn hls_custom_video_test_user() -> ProxyUserCredentials { + let mut user = ProxyUserCredentials::default(); + user.username = "viewer".to_string(); + user.password = "secret".to_string(); + user +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_hls_share_target(hls_enabled: bool) -> ConfigTarget { + ConfigTarget::from(&ConfigTargetDto { + id: 1, + name: "default".to_string(), + options: Some(ConfigTargetOptions { + share_live_streams: ConfigTargetShareLiveStreams { hls: hls_enabled, mpeg_ts: false }, + ..Default::default() + }), + output: vec![TargetOutputDto::Xtream(XtreamTargetOutputDto::default())], + ..Default::default() + }) +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_m3u_hls_share_target() -> ConfigTarget { + ConfigTarget::from(&ConfigTargetDto { + id: 2, + name: "m3u-target".to_string(), + options: Some(ConfigTargetOptions { + share_live_streams: ConfigTargetShareLiveStreams { hls: true, mpeg_ts: false }, + ..Default::default() + }), + output: vec![TargetOutputDto::M3u(M3uTargetOutputDto::default())], + use_memory_cache: true, + ..Default::default() + }) +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_m3u_hls_item( + input: &ConfigInput, + virtual_id: u32, + input_stream_id: &str, + url: &str, +) -> M3uPlaylistItem { + M3uPlaylistItem::from(&PlaylistItem { + header: PlaylistItemHeader { + id: input_stream_id.intern(), + virtual_id: VirtualId::new(virtual_id), + input_name: Arc::clone(&input.name), + url: url.intern(), + item_type: PlaylistItemType::LiveHls, + xtream_cluster: XtreamCluster::Live, + input_stream_id: input_stream_id.intern(), + ..PlaylistItemHeader::default() + }, + }) +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_hls_entry_stream_context( + virtual_id: u32, + input_stream_id: &str, + known_bitrate_bps: Option, +) -> super::super::super::HlsEntryStreamContext { + super::super::super::HlsEntryStreamContext { + identity: super::super::super::HlsEntryStreamIdentity::new(virtual_id, input_stream_id) + .expect("valid input stream identity"), + known_bitrate_bps, + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn cache_test_m3u_hls_item( + app_state: &Arc, + target: &ConfigTarget, + item: M3uPlaylistItem, +) { + let mut playlist = crate::repository::BPlusTree::new(); + playlist.insert(item.virtual_id.get(), item); + app_state + .playlists + .cache_playlist(&target.name, crate::api::model::PlaylistStorage::M3uPlaylist(Box::new(playlist))) + .await; +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_hls_input() -> ConfigInput { + ConfigInput { + id: 1, + name: Arc::from("test-input"), + input_type: InputType::Xtream, + url: "http://origin.example.com".to_string(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + } +} + +pub(in crate::api::endpoints::hls_api::tests) fn store_test_sources_with_target( + app_state: &Arc, + input: ConfigInput, + target: ConfigTarget, +) { + let input = Arc::new(input); + let inputs = vec![Arc::clone(&input)]; + app_state.app_config.sources.store(Arc::new(SourcesConfig { + batch_files: vec![], + provider: vec![], + group_lookup: build_group_lookup(&inputs), + inputs, + sources: vec![ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![Arc::new(target)] }], + templates: None, + })); +} + +pub(in crate::api::endpoints::hls_api::tests) fn configure_default_test_server(app_state: &Arc) { + let users = + app_state.app_config.api_proxy.load_full().as_ref().map_or_else(Vec::new, |api_proxy| api_proxy.user.clone()); + app_state.app_config.api_proxy.store(Some(Arc::new(ApiProxyConfig { + server: vec![ApiProxyServerInfo { + name: "default".to_string(), + protocol: "http".to_string(), + host: "127.0.0.1".to_string(), + port: Some("8901".to_string()), + timezone: "UTC".to_string(), + message: String::new(), + path: None, + }], + user: users, + ..Default::default() + }))); +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_app_state() -> Arc { + test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::new())) +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_custom_video_buffer() -> TransportStreamBuffer { + TransportStreamBuffer::new( + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")).to_vec(), + ) +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_app_state_with_inputs( + inputs: Vec>, +) -> Arc { + test_app_state_with_hls_proxy_and_inputs(Arc::new(HlsProxyManager::new()), inputs) +} + +pub(in crate::api::endpoints::hls_api::tests) fn enable_hls_cache(app_state: &Arc) { + let config = Config { + reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { + hls_cache: Some(HlsCacheConfigDto::default()), + ..Default::default() + })), + ..Default::default() + }; + app_state.app_config.config.store(Arc::new(config)); +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_app_state_with_hls_proxy( + hls_proxy: Arc, +) -> Arc { + test_app_state_with_hls_proxy_and_inputs(hls_proxy, Vec::new()) +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_app_state_with_hls_proxy_and_inputs( + hls_proxy: Arc, + inputs: Vec>, +) -> Arc { + let app_config = test_app_config(); + if !inputs.is_empty() { + app_config.sources.store(Arc::new(SourcesConfig { + batch_files: vec![], + provider: vec![], + group_lookup: build_group_lookup(&inputs), + inputs, + sources: vec![], + templates: None, + })); + } + let event_manager = Arc::new(EventManager::new()); + let active_provider = Arc::new(ActiveProviderManager::new(&app_config, &event_manager)); + let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); + active_provider.set_shared_stream_manager(&shared_stream_manager); + + let geoip = Arc::new(ArcSwapOption::::default()); + let config = app_config.config.load(); + let active_users = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); + let connection_manager = + Arc::new(ConnectionManager::new(&active_users, &active_provider, &shared_stream_manager, &event_manager, None)); + let cancel_tokens = CancelTokens::default(); + let metadata_manager = Arc::new(MetadataUpdateManager::new(cancel_tokens.metadata.clone())); + + Arc::new(AppState { + recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( + Arc::clone(&active_provider), + Arc::clone(&connection_manager), + ), + app_config, + http_clients: Arc::new(tuliprox_core::model::HttpClients::new( + reqwest::Client::new(), + reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client builds"), + reqwest::Client::new(), + reqwest::Client::new(), + reqwest::Client::new(), + )), + recordings: Arc::new(crate::api::model::RecordingQueue::new()), + cache: Arc::new(ArcSwapOption::default()), + shared_stream_manager, + hls: crate::api::model::HlsState::new(hls_proxy), + stalker_resolve_coordinator: crate::api::model::StalkerResolveCoordinator::default(), + active_users, + active_provider, + connection_manager, + event_manager, + cancel_tokens: ArcSwap::from_pointee(cancel_tokens), + playlists: Arc::new(PlaylistStorageState::new()), + geoip, + metadata_manager, + auth: crate::api::model::AuthState::for_tests(), + playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), + }) +} + +pub(in crate::api::endpoints::hls_api::tests) fn single_hls_provider_input(name: &str) -> ConfigInput { + ConfigInput { + id: 1, + name: Arc::from(name), + input_type: InputType::Xtream, + url: "http://account.example.com".to_string(), + username: Some("account-user".to_string()), + password: Some("account-pass".to_string()), + enabled: true, + priority: 0, + max_connections: 1, + ..ConfigInput::default() + } +} + +pub(in crate::api::endpoints::hls_api::tests) fn overlap_provider_input() -> ConfigInput { + ConfigInput { + id: 1, + name: Arc::from("overlap-input"), + input_type: InputType::Xtream, + url: "http://root.example.com".to_string(), + username: Some("root-user".to_string()), + password: Some("root-pass".to_string()), + enabled: true, + priority: 10, + max_connections: 1, + aliases: Some(vec![crate::model::ConfigInputAlias { + id: 2, + name: Arc::from("account-a"), + url: "http://account.example.com".to_string(), + username: Some("account-user".to_string()), + password: Some("account-pass".to_string()), + priority: 0, + max_connections: 1, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + } +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_segment_entry( + proxy_session_id: &ProxySessionId, + proxy_seq: u64, + status: SegmentCacheStatus, +) -> SegmentEntry { + SegmentEntry { + origin_key: OriginSegmentKey { + origin_epoch: 0, + effective_host_id: 0, + host_local_sequence: proxy_seq, + host_local_index: u32::try_from(proxy_seq).unwrap_or(u32::MAX), + }, + proxy_seq, + duration_ms: 4_000, + proxy_file_ext: "ts".to_string(), + content_type: "video/mp2t".to_string(), + cache_key: SegmentCacheKey::new(proxy_session_id.clone(), proxy_seq, "ts"), + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + encryption: None, + origin_fetch_ref: Some(OriginSegmentFetchRef { + resolved_origin_url: format!("http://origin.example.com/{proxy_seq}.ts"), + byte_range: None, + valid_until_ms: None, + }), + status, + last_rendered_at_ms: None, + access: Arc::new(CacheAccessState::new()), + } +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_hls_access_context( + proxy_session_id: ProxySessionId, + access_lease_id: HlsAccessLeaseId, +) -> HlsAccessContext { + test_hls_access_context_with(proxy_session_id, access_lease_id, "hls-session-token", test_fingerprint().key) +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_hls_access_context_with( + proxy_session_id: ProxySessionId, + access_lease_id: HlsAccessLeaseId, + user_session_token: &str, + client_fingerprint: String, +) -> HlsAccessContext { + HlsAccessContext { + username: "hls-user".to_string(), + user_session_token: user_session_token.to_string(), + proxy_session_id, + input_id: 1, + stream_ref: "12345".to_string(), + virtual_id: 12345, + known_bitrate_bps: None, + lease_id: access_lease_id, + family_key: HlsPlaybackFamilyKey::new("hls-user", client_fingerprint), + epg_reference_ts: None, + archive_origin_url: None, + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn publish_test_transient_resource_membership( + app_state: &Arc, + proxy_session_id: &str, + access_lease_id: &str, + resource_uri: &str, +) { + let proxy_session_id = ProxySessionId(proxy_session_id.to_string()); + let access_lease_id = HlsAccessLeaseId(access_lease_id.to_string()); + let now_ms = super::super::super::current_time_millis(); + let publication = app_state + .hls + .proxy + .prepare_access_lease_manifest_publication(&access_lease_id, &proxy_session_id, now_ms) + .await + .expect("test resource lease accepts manifest publication"); + let snapshot = HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::TransientPassthrough, + source_commit_identity: HlsManifestCommitIdentity::new(now_ms), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 0, + delivered_at_ms: now_ms, + first_proxy_seq: 0, + last_proxy_seq: 0, + visible_segments: Arc::from([]), + discontinuity_sequence: 0, + target_duration_ms: 4_000, + playlist_duration_ms: 0, + last_visible_media_end_ms: 0, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + }; + let published_resource_ids = HlsPublishedTransientResourceIds::from_manifest_body(resource_uri); + assert!(app_state + .hls + .proxy + .commit_access_lease_manifest_publication_with_resources( + &access_lease_id, + &proxy_session_id, + publication, + snapshot, + published_resource_ids, + now_ms, + ) + .await + .is_committed()); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn single_variant_master_playlist( + response: Response, +) -> (u32, String) { + let body = response_body(response).await; + let body = std::str::from_utf8(&body).expect("master playlist should be UTF-8"); + let mut lines = body.lines(); + assert_eq!(lines.next(), Some("#EXTM3U")); + let bandwidth = lines + .next() + .and_then(|line| line.strip_prefix("#EXT-X-STREAM-INF:BANDWIDTH=")) + .and_then(|value| value.parse::().ok()) + .expect("positive master playlist bandwidth"); + let uri = lines.next().expect("single variant URI").to_string(); + assert!(lines.next().is_none(), "master playlist must contain exactly one variant"); + (bandwidth, uri) +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/crypto.rs b/backend/app/src/api/endpoints/hls_api/tests/support/crypto.rs new file mode 100644 index 000000000..bc3c708dd --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/support/crypto.rs @@ -0,0 +1,124 @@ +use super::{ + super::{Aes128, Arc, AsyncReadExt, AsyncWriteExt, AtomicUsize, Block, Ordering, RwLock, TcpListener}, + path_has_extension, TestSegmentOrigin, +}; +use aes::cipher::{BlockEncrypt, KeyInit as _}; + +pub(in crate::api::endpoints::hls_api::tests) const AES_TEST_MANIFEST: &[u8] = b"#EXTM3U\n#EXT-X-VERSION:5\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:77\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\",KEYFORMAT=\"identity\",KEYFORMATVERSIONS=\"1\"\n#EXTINF:12,\n77.ts\n#EXTINF:12,\n78.ts\n#EXTINF:12,\n79.ts\n#EXTINF:12,\n80.ts\n#EXTINF:12,\n81.ts\n#EXTINF:12,\n82.ts\n"; + +pub(in crate::api::endpoints::hls_api::tests) const AES_TEST_KEY_BYTES: &[u8] = b"0123456789abcdef"; + +pub(in crate::api::endpoints::hls_api::tests) const AES_TEST_PLAINTEXT_SEGMENT: &[u8] = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); + +pub(in crate::api::endpoints::hls_api::tests) async fn spawn_test_encrypted_hls_origin( + manifest: &'static [u8], + key_bytes: Arc<[u8]>, + plaintext_segment: Arc<[u8]>, +) -> TestSegmentOrigin { + let manifest = Arc::<[u8]>::from(manifest); + let key_bytes = Arc::new(RwLock::new(key_bytes)); + let key_bytes_for_task = Arc::clone(&key_bytes); + let key_requests = Arc::new(AtomicUsize::new(0)); + let key_requests_for_task = Arc::clone(&key_requests); + let manifest_requests = Arc::new(AtomicUsize::new(0)); + let manifest_requests_for_task = Arc::clone(&manifest_requests); + let segment_requests = Arc::new(AtomicUsize::new(0)); + let segment_requests_for_task = Arc::clone(&segment_requests); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let manifest = Arc::clone(&manifest); + let key_bytes = Arc::clone(&key_bytes_for_task); + let key_requests = Arc::clone(&key_requests_for_task); + let manifest_requests = Arc::clone(&manifest_requests_for_task); + let segment_requests = Arc::clone(&segment_requests_for_task); + let plaintext_segment = Arc::clone(&plaintext_segment); + tokio::spawn(async move { + let mut request = vec![0_u8; 2048]; + let Ok(read) = socket.read(&mut request).await else { + return; + }; + if read == 0 { + return; + } + let path = String::from_utf8_lossy(&request[..read]) + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .map_or_else(|| "/".to_string(), str::to_owned); + let current_key_bytes = Arc::clone(&*key_bytes.read().await); + let body = if path_has_extension(&path, "m3u8") { + manifest_requests.fetch_add(1, Ordering::SeqCst); + manifest + } else if path.ends_with("key.bin") { + key_requests.fetch_add(1, Ordering::SeqCst); + current_key_bytes + } else if let Some(origin_sequence) = path + .rsplit('/') + .next() + .and_then(|file| file.strip_suffix(".ts")) + .and_then(|value| value.parse::().ok()) + { + segment_requests.fetch_add(1, Ordering::SeqCst); + Arc::from(encrypt_test_aes128_cbc_pkcs7( + &plaintext_segment, + ¤t_key_bytes, + test_hls_sequence_iv(origin_sequence), + )) + } else { + Arc::<[u8]>::from([]) + }; + let response = + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + if socket.write_all(response.as_bytes()).await.is_ok() { + let _ = socket.write_all(&body).await; + } + }); + } + }); + TestSegmentOrigin { + base_url: format!("http://{addr}"), + key_requests, + manifest_requests, + segment_requests, + key_bytes: Some(key_bytes), + task, + } +} + +pub(in crate::api::endpoints::hls_api::tests) fn encrypt_test_aes128_cbc_pkcs7( + plaintext: &[u8], + key: &[u8], + iv: [u8; 16], +) -> Vec { + let padding_len = 16 - (plaintext.len() % 16); + let mut ciphertext = plaintext.to_vec(); + ciphertext.resize( + plaintext.len().saturating_add(padding_len), + u8::try_from(padding_len).expect("PKCS#7 AES-128 padding fits in u8"), + ); + let cipher = Aes128::new_from_slice(key).expect("test key has AES-128 length"); + let mut previous = iv; + for block in ciphertext.as_chunks_mut::<16>().0 { + for (byte, previous) in block.iter_mut().zip(previous) { + *byte ^= previous; + } + let mut encrypted = Block::::default(); + encrypted.copy_from_slice(block); + cipher.encrypt_block(&mut encrypted); + block.copy_from_slice(&encrypted); + previous.copy_from_slice(block); + } + ciphertext +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_hls_sequence_iv(sequence: u64) -> [u8; 16] { + let mut iv = [0_u8; 16]; + iv[8..].copy_from_slice(&sequence.to_be_bytes()); + iv +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/http.rs b/backend/app/src/api/endpoints/hls_api/tests/support/http.rs new file mode 100644 index 000000000..5fd38a880 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/support/http.rs @@ -0,0 +1,132 @@ +use super::{ + super::{ + header, hls_api_register, AppState, Arc, Body, BodyExt, Config, ConnectInfo, CustomStreamResponse, + HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, HlsPlaybackFamilyKey, HlsSessionHandle, Method, Request, + Response, ServiceExt, StatusCode, StripConfig, + }, + grant_hls_proxy_lease, publish_test_transient_resource_membership, single_variant_master_playlist, test_addr, + test_custom_video_buffer, +}; + +pub(in crate::api::endpoints::hls_api::tests) fn disable_custom_stream_response(app_state: &Arc) { + app_state.app_config.config.store(Arc::new(Config { custom_stream_response_enabled: false, ..Default::default() })); +} + +pub(in crate::api::endpoints::hls_api::tests) fn enable_channel_unavailable_custom_response(app_state: &Arc) { + let config = app_state.app_config.config.load(); + app_state + .app_config + .config + .store(Arc::new(Config { custom_stream_response_enabled: true, ..config.as_ref().clone() })); + app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { + channel_unavailable: Some(test_custom_video_buffer()), + user_connections_exhausted: None, + provider_connections_exhausted: None, + low_priority_preempted: None, + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: Vec::new(), + }))); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn try_test_hls_cached_manifest_response( + app_state: &Arc, + session: &HlsSessionHandle, + access_lease_id: &HlsAccessLeaseId, + access_lease_state: HlsAccessLeaseState, + strip: &StripConfig, + server_path: Option<&str>, + options: super::super::HlsCachedManifestOptions, +) -> Option { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let now_ms = super::super::super::current_time_millis(); + if app_state.hls.proxy.access_lease_response_snapshot(access_lease_id, &proxy_session_id, now_ms).await.is_none() { + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + access_lease_id.clone(), + HlsPlaybackFamilyKey::new("test-user", "manifest-test-client"), + proxy_session_id, + "test-user".to_string(), + "manifest-test-session".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + 60_000, + )) + .await; + } + super::super::super::try_hls_cached_manifest_response( + app_state, + session, + access_lease_id, + access_lease_state, + strip, + server_path, + options, + super::super::super::HlsRuntimeBandwidthLearningContext::Disabled, + ) + .await +} + +pub(in crate::api::endpoints::hls_api::tests) async fn hls_proxy_uri( + app_state: &Arc, + proxy_session_id: &str, + suffix: &str, +) -> String { + let access_lease_id = grant_hls_proxy_lease(app_state, proxy_session_id).await; + let uri = format!("/hls/shared/live/{proxy_session_id}/{access_lease_id}/{suffix}"); + if suffix.starts_with("r/") { + publish_test_transient_resource_membership(app_state, proxy_session_id, &access_lease_id, &uri).await; + } + uri +} + +pub(in crate::api::endpoints::hls_api::tests) async fn get_response( + app_state: Arc, + uri: &str, + range: Option<&str>, +) -> Response { + request_response(app_state, Method::GET, uri, range).await +} + +pub(in crate::api::endpoints::hls_api::tests) async fn request_response( + app_state: Arc, + method: Method, + uri: &str, + range: Option<&str>, +) -> Response { + let router = hls_api_register().with_state(app_state); + let mut request = Request::builder().method(method).uri(uri); + if let Some(range) = range { + request = request.header(header::RANGE, range); + } + let mut request = request.body(Body::empty()).expect("request should build"); + request.extensions_mut().insert(ConnectInfo(test_addr())); + router.oneshot(request).await.expect("response") +} + +pub(in crate::api::endpoints::hls_api::tests) async fn get_status(app_state: Arc, uri: &str) -> StatusCode { + get_response(app_state, uri, None).await.status() +} + +pub(in crate::api::endpoints::hls_api::tests) async fn response_body(response: Response) -> bytes::Bytes { + response.into_body().collect().await.expect("body should collect").to_bytes() +} + +pub(in crate::api::endpoints::hls_api::tests) async fn single_variant_uri(response: Response) -> String { + single_variant_master_playlist(response).await.1 +} + +pub(in crate::api::endpoints::hls_api::tests) fn access_lease_id_from_variant_uri(uri: &str) -> &str { + uri.trim_end_matches("/manifest.m3u8").rsplit('/').next().expect("access lease id in variant URI") +} + +pub(in crate::api::endpoints::hls_api::tests) fn proxy_session_id_from_variant_uri(uri: &str) -> &str { + let mut parts = uri.trim_end_matches("/manifest.m3u8").rsplit('/'); + let _access_lease_id = parts.next().expect("access lease id in variant URI"); + parts.next().expect("proxy session id in variant URI") +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/manifest.rs b/backend/app/src/api/endpoints/hls_api/tests/support/manifest.rs new file mode 100644 index 000000000..45fd3bc4c --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/support/manifest.rs @@ -0,0 +1,388 @@ +use super::{ + super::{ + build_transient_resource_id, header, parse_origin_media_manifest, AppState, Arc, HashMap, HeaderMap, + HeaderValue, HlsAccessLeaseId, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsManifestCommitIdentity, + HlsManifestDeliveryMode, HlsMediaContainer, HlsSession, HlsSessionKey, HlsSessionMode, MapCacheStatus, + OriginManifestParseOutcome, ProxyMapId, ProxySessionId, RenderedManifest, SegmentCacheStatus, + TransientResourceKind, TransientResourceRef, + }, + grant_hls_proxy_lease, test_segment_entry, TestEncodedManifestOrigin, +}; +use std::fmt::Write as _; + +pub(in crate::api::endpoints::hls_api::tests) fn transient_manifest_body(proxy_session_id: &str) -> String { + transient_manifest_body_from_sequence(proxy_session_id, 100, 6) +} + +pub(in crate::api::endpoints::hls_api::tests) fn transient_manifest_body_from_sequence( + proxy_session_id: &str, + first_sequence: u64, + count: usize, +) -> String { + let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:10\n#EXT-X-MEDIA-SEQUENCE:{first_sequence}\n"); + for index in 0..count { + let sequence = first_sequence.saturating_add(u64::try_from(index).expect("test sequence index fits u64")); + body.push_str("#EXTINF:10.0,\n"); + let _ = writeln!( + body, + "/hls/shared/live/{proxy_session_id}/{}/r/seg{sequence}.ts", + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER + ); + } + body +} + +pub(in crate::api::endpoints::hls_api::tests) fn media_uri_count(body: &str) -> usize { + body.lines().filter(|line| !line.is_empty() && !line.starts_with('#')).count() +} + +pub(in crate::api::endpoints::hls_api::tests) fn normal_manifest_body(proxy_session_id: &str) -> String { + normal_manifest_body_from_sequence(proxy_session_id, 0, 6) +} + +pub(in crate::api::endpoints::hls_api::tests) fn normal_manifest_body_from_sequence( + proxy_session_id: &str, + first_sequence: u64, + count: usize, +) -> String { + let mut body = format!("#EXTM3U\n#EXT-X-TARGETDURATION:10\n#EXT-X-MEDIA-SEQUENCE:{first_sequence}\n"); + for index in 0..count { + let sequence = first_sequence.saturating_add(u64::try_from(index).expect("test sequence index fits u64")); + body.push_str("#EXTINF:10.0,\n"); + let _ = writeln!( + body, + "/hls/shared/live/{proxy_session_id}/{}/{sequence:06}.ts", + crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER + ); + } + body +} + +pub(in crate::api::endpoints::hls_api::tests) fn store_normal_manifest_body( + session: &mut HlsSession, + body: String, + rendered_at_ms: u64, +) { + store_normal_manifest_body_range(session, body, 0, 6, rendered_at_ms); +} + +pub(in crate::api::endpoints::hls_api::tests) fn record_test_normal_manifest_commit( + session: &mut HlsSession, + rendered_at_ms: u64, +) { + let identity = session + .next_manifest_commit_identity(rendered_at_ms) + .expect("test manifest commit generation remains available"); + session.record_normal_manifest_commit_identity(identity); +} + +pub(in crate::api::endpoints::hls_api::tests) fn store_normal_manifest_body_range( + session: &mut HlsSession, + body: String, + first_proxy_seq: u64, + count: usize, + rendered_at_ms: u64, +) { + let last_proxy_seq = + first_proxy_seq.saturating_add(u64::try_from(count.saturating_sub(1)).expect("test count fits u64")); + let proxy_session_id = session.proxy_session_id.clone(); + for proxy_seq in first_proxy_seq..=last_proxy_seq { + let mut entry = test_segment_entry( + &proxy_session_id, + proxy_seq, + SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }, + ); + entry.duration_ms = 10_000; + session.segments.insert(proxy_seq, entry); + } + session.advance_media_readiness_generation(); + session.last_rendered_manifest = Some(RenderedManifest { + body, + first_proxy_seq, + last_proxy_seq, + playlist_duration_ms: 60_000, + valid_until_ms: rendered_at_ms.saturating_add(60_000), + render_gap_segments: 0, + rendered_at_ms, + discontinuity_sequence: 0, + target_duration_ms: 10_000, + segment_proxy_seqs: (first_proxy_seq..=last_proxy_seq).collect(), + }); + record_test_normal_manifest_commit(session, rendered_at_ms); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn publish_ready_test_manifest_for_lease( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, + target_duration_ms: u64, +) { + let session = + app_state.hls.proxy.sessions().get_by_proxy_session_id(proxy_session_id).await.expect("test session exists"); + let (proxy_seq, duration_ms) = { + let session = session.read().await; + session + .segments + .iter() + .find_map(|(proxy_seq, entry)| { + matches!(entry.status, SegmentCacheStatus::Ready { .. }).then_some((*proxy_seq, entry.duration_ms)) + }) + .expect("READY test segment") + }; + let now_ms = super::super::super::current_time_millis(); + let publication_guard = app_state + .hls + .proxy + .prepare_access_lease_manifest_publication(access_lease_id, proxy_session_id, now_ms) + .await + .expect("test lease accepts publication"); + let snapshot = HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(now_ms), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 0, + delivered_at_ms: now_ms, + first_proxy_seq: proxy_seq, + last_proxy_seq: proxy_seq, + visible_segments: Arc::from([HlsLeaseManifestSegment { + proxy_seq, + duration_ms, + uri: format!("/hls/shared/live/{}/{}/{proxy_seq:06}.ts", proxy_session_id.0, access_lease_id.0).into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }]), + discontinuity_sequence: 0, + target_duration_ms: target_duration_ms.max(duration_ms), + playlist_duration_ms: duration_ms, + last_visible_media_end_ms: duration_ms, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + }; + assert!(app_state + .hls + .proxy + .commit_access_lease_manifest_publication( + access_lease_id, + proxy_session_id, + publication_guard, + snapshot, + now_ms, + ) + .await + .is_committed()); +} + +pub(in crate::api::endpoints::hls_api::tests) fn regression_origin_manifest( + first_sequence: u64, + segment_count: usize, +) -> Vec { + let mut manifest = + format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:{first_sequence}\n"); + for offset in 0..segment_count { + let sequence = first_sequence.saturating_add(u64::try_from(offset).unwrap_or(u64::MAX)); + let _ = writeln!(&mut manifest, "#EXTINF:12.0,\n{sequence}.ts"); + } + manifest.into_bytes() +} + +pub(in crate::api::endpoints::hls_api::tests) fn normal_manifest( + body: &str, +) -> crate::processing::parser::hls::origin_manifest::ParsedOriginManifest { + match parse_origin_media_manifest(body, "http://origin.example.com/live/final/index.m3u8") { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn map_segment( + app_state: &Arc, + proxy_seq: u64, + extension: &str, +) -> String { + map_segment_with_origin_url(app_state, proxy_seq, extension, &format!("{proxy_seq}.{extension}")).await +} + +pub(in crate::api::endpoints::hls_api::tests) async fn map_segment_with_origin_url( + app_state: &Arc, + proxy_seq: u64, + _extension: &str, + origin_url: &str, +) -> String { + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let manifest = + normal_manifest(&format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:{proxy_seq}\n#EXTINF:4.0,\n{origin_url}\n")); + let mut session = session.write().await; + session.proxy_next_seq = Some(proxy_seq); + session.apply_origin_manifest(&manifest).expect("manifest should map"); + session.proxy_session_id.0.clone() +} + +pub(in crate::api::endpoints::hls_api::tests) async fn map_ready_segment( + app_state: &Arc, + proxy_seq: u64, + extension: &str, + body: &[u8], +) -> String { + let proxy_session_id = map_ready_segment_without_lease(app_state, proxy_seq, extension, body).await; + grant_hls_proxy_lease(app_state, &proxy_session_id).await; + proxy_session_id +} + +pub(in crate::api::endpoints::hls_api::tests) async fn map_ready_segment_without_lease( + app_state: &Arc, + proxy_seq: u64, + extension: &str, + body: &[u8], +) -> String { + let proxy_session_id = map_segment(app_state, proxy_seq, extension).await; + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + let cache_key = { + let session = session.read().await; + session.segments.get(&proxy_seq).expect("segment should be mapped").cache_key.clone() + }; + let metadata = app_state + .hls + .proxy + .segment_cache() + .write_bytes_and_commit(&cache_key, body) + .await + .expect("cache commit should succeed"); + { + let mut session = session.write().await; + session.segments.get_mut(&proxy_seq).expect("segment should be mapped").status = + SegmentCacheStatus::Ready { content_length: metadata.size, ready_at_ms: 200 }; + } + proxy_session_id +} + +pub(in crate::api::endpoints::hls_api::tests) async fn map_hls_map( + app_state: &Arc, + body: &[u8], + grant_lease: bool, +) -> String { + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let manifest = normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n000123.m4s\n"); + let proxy_session_id = { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest should map"); + session.proxy_session_id.0.clone() + }; + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) + .await + .expect("session should exist"); + let cache_key = { + let session = session.read().await; + session.maps.get(&ProxyMapId(0)).expect("map should be mapped").cache_key.clone() + }; + let metadata = app_state + .hls + .proxy + .segment_cache() + .write_bytes_and_commit(&cache_key, body) + .await + .expect("map cache commit should succeed"); + { + let mut session = session.write().await; + session.maps.get_mut(&ProxyMapId(0)).expect("map should be mapped").status = + MapCacheStatus::Ready { content_length: metadata.size, ready_at_ms: 200 }; + } + if grant_lease { + grant_hls_proxy_lease(app_state, &proxy_session_id).await; + } + proxy_session_id +} + +pub(in crate::api::endpoints::hls_api::tests) async fn map_transient_resource( + app_state: &Arc, + origin_url: &str, + extension: &str, + grant_lease: bool, +) -> (String, String) { + map_transient_resource_with_kind(app_state, origin_url, extension, grant_lease, TransientResourceKind::Segment) + .await +} + +pub(in crate::api::endpoints::hls_api::tests) async fn map_transient_resource_with_kind( + app_state: &Arc, + origin_url: &str, + extension: &str, + grant_lease: bool, + kind: TransientResourceKind, +) -> (String, String) { + static TRANSIENT_STREAM_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + let secret = b"rewrite-secret"; + let now_ms = super::super::super::current_time_millis(); + let stream_ref = + format!("transient-{}", TRANSIENT_STREAM_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed)); + let session = app_state.hls.proxy.get_or_create_session(HlsSessionKey::new(1, &stream_ref), secret, now_ms).await; + let resource_id = build_transient_resource_id(origin_url, secret); + let proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + session.transient.upsert_resources([TransientResourceRef::new( + kind, + origin_url, + secret, + now_ms, + 300_000, + Some(extension.to_string()), + )]); + session.proxy_session_id.0.clone() + }; + if grant_lease { + grant_hls_proxy_lease(app_state, &proxy_session_id).await; + } + (proxy_session_id, resource_id.0) +} + +pub(in crate::api::endpoints::hls_api::tests) fn manifest_media_sequence(body: &str) -> u64 { + body.lines() + .find_map(|line| line.strip_prefix("#EXT-X-MEDIA-SEQUENCE:")) + .and_then(|value| value.parse().ok()) + .expect("media sequence") +} + +pub(in crate::api::endpoints::hls_api::tests) fn legacy_manifest_test_input( + origin: &TestEncodedManifestOrigin, +) -> crate::model::InputSource { + crate::model::InputSource { + name: Arc::from("legacy-content-coding-test"), + url: format!("{}/manifest.m3u8", origin.base_url), + provider: None, + username: None, + password: None, + method: shared::model::InputFetchMethod::GET, + headers: HashMap::from([("Accept-Encoding".to_string(), "gzip".to_string())]), + } +} + +pub(in crate::api::endpoints::hls_api::tests) fn legacy_manifest_test_client_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); + headers +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/origins.rs b/backend/app/src/api/endpoints/hls_api/tests/support/origins.rs new file mode 100644 index 000000000..bed96a471 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/support/origins.rs @@ -0,0 +1,315 @@ +use super::{ + super::{Arc, AsyncReadExt, AsyncWriteExt, AtomicUsize, Duration, Ordering, RwLock, StatusCode, TcpListener}, + path_has_extension, +}; +use std::fmt::Write as _; + +pub(in crate::api::endpoints::hls_api::tests) struct TestSegmentOrigin { + pub(in crate::api::endpoints::hls_api::tests) base_url: String, + pub(in crate::api::endpoints::hls_api::tests) key_requests: Arc, + pub(in crate::api::endpoints::hls_api::tests) manifest_requests: Arc, + pub(in crate::api::endpoints::hls_api::tests) segment_requests: Arc, + pub(in crate::api::endpoints::hls_api::tests) key_bytes: Option>>>, + pub(in crate::api::endpoints::hls_api::tests) task: tokio::task::JoinHandle<()>, +} + +impl Drop for TestSegmentOrigin { + fn drop(&mut self) { self.task.abort(); } +} + +impl TestSegmentOrigin { + pub(in crate::api::endpoints::hls_api::tests) fn key_request_count(&self) -> usize { + self.key_requests.load(Ordering::SeqCst) + } + + pub(in crate::api::endpoints::hls_api::tests) fn manifest_request_count(&self) -> usize { + self.manifest_requests.load(Ordering::SeqCst) + } + + pub(in crate::api::endpoints::hls_api::tests) fn segment_request_count(&self) -> usize { + self.segment_requests.load(Ordering::SeqCst) + } + + pub(in crate::api::endpoints::hls_api::tests) async fn set_key_bytes(&self, bytes: Arc<[u8]>) { + if let Some(key_bytes) = &self.key_bytes { + *key_bytes.write().await = bytes; + } + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn spawn_test_segment_origin( + body: &'static [u8], +) -> TestSegmentOrigin { + spawn_test_status_origin(StatusCode::OK, body).await +} + +pub(in crate::api::endpoints::hls_api::tests) struct TestBinaryOriginResponse { + pub(in crate::api::endpoints::hls_api::tests) status: StatusCode, + pub(in crate::api::endpoints::hls_api::tests) location: Option, + pub(in crate::api::endpoints::hls_api::tests) body: Arc<[u8]>, +} + +impl TestBinaryOriginResponse { + pub(in crate::api::endpoints::hls_api::tests) fn new(status: StatusCode, body: Arc<[u8]>) -> Self { + Self { status, location: None, body } + } + + pub(in crate::api::endpoints::hls_api::tests) fn redirect(location: String) -> Self { + Self { status: StatusCode::FOUND, location: Some(location), body: Arc::from(&b""[..]) } + } +} + +pub(in crate::api::endpoints::hls_api::tests) type TestBinaryOriginHandler = + Arc TestBinaryOriginResponse + Send + Sync>; + +pub(in crate::api::endpoints::hls_api::tests) async fn spawn_test_binary_origin( + handler: TestBinaryOriginHandler, +) -> TestSegmentOrigin { + let key_requests = Arc::new(AtomicUsize::new(0)); + let manifest_requests = Arc::new(AtomicUsize::new(0)); + let segment_requests = Arc::new(AtomicUsize::new(0)); + let manifest_requests_for_task = Arc::clone(&manifest_requests); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let handler = Arc::clone(&handler); + let manifest_requests = Arc::clone(&manifest_requests_for_task); + tokio::spawn(async move { + let mut buf = vec![0_u8; 2048]; + let Ok(read) = socket.read(&mut buf).await else { + return; + }; + if read == 0 { + return; + } + let request = String::from_utf8_lossy(&buf[..read]); + let path = request.lines().next().and_then(|line| line.split_whitespace().nth(1)).unwrap_or("/"); + if path_has_extension(path, "m3u8") { + manifest_requests.fetch_add(1, Ordering::SeqCst); + } + let TestBinaryOriginResponse { status, location, body } = handler(path); + let reason = status.canonical_reason().unwrap_or("Status"); + let location_header = location.map_or_else(String::new, |location| format!("Location: {location}\r\n")); + let response = format!( + "HTTP/1.1 {} {reason}\r\n{location_header}Content-Length: {}\r\nConnection: close\r\n\r\n", + status.as_u16(), + body.len() + ); + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.write_all(&body).await; + }); + } + }); + TestSegmentOrigin { + base_url: format!("http://{addr}"), + key_requests, + manifest_requests, + segment_requests, + key_bytes: None, + task, + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn spawn_test_status_origin( + status: StatusCode, + body: &'static [u8], +) -> TestSegmentOrigin { + let body = Arc::<[u8]>::from(body); + spawn_test_binary_origin(Arc::new(move |_path| TestBinaryOriginResponse::new(status, Arc::clone(&body)))).await +} + +pub(in crate::api::endpoints::hls_api::tests) struct TestEncodedManifestOrigin { + pub(in crate::api::endpoints::hls_api::tests) base_url: String, + pub(in crate::api::endpoints::hls_api::tests) requests: Arc>>, + pub(in crate::api::endpoints::hls_api::tests) task: tokio::task::JoinHandle<()>, +} + +impl Drop for TestEncodedManifestOrigin { + fn drop(&mut self) { self.task.abort(); } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn spawn_test_encoded_manifest_origin( + content_encoding: Option<&'static str>, + body: Vec, + body_delay: Duration, +) -> TestEncodedManifestOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let requests = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let body = Arc::new(body); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + let body = Arc::clone(&body); + tokio::spawn(async move { + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 2048]; + let Ok(read) = socket.read(&mut chunk).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); + let mut response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n", body.len()); + if let Some(content_encoding) = content_encoding { + let _ = writeln!(&mut response, "Content-Encoding: {content_encoding}\r"); + } + response.push_str("Connection: close\r\n\r\n"); + let _ = socket.write_all(response.as_bytes()).await; + if body_delay.is_zero() { + let _ = socket.write_all(body.as_slice()).await; + } else { + let split_at = body.len().min(4); + let _ = socket.write_all(&body[..split_at]).await; + tokio::time::sleep(body_delay).await; + let _ = socket.write_all(&body[split_at..]).await; + } + }); + } + }); + TestEncodedManifestOrigin { base_url: format!("http://{addr}"), requests, task } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn encode_test_manifest( + content_encoding: &str, + body: &[u8], +) -> Vec { + match content_encoding { + "gzip" => { + let mut encoder = async_compression::tokio::write::GzipEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("gzip test body encodes"); + encoder.shutdown().await.expect("gzip test encoder finishes"); + encoder.into_inner() + } + "deflate" => { + let mut encoder = async_compression::tokio::write::DeflateEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("deflate test body encodes"); + encoder.shutdown().await.expect("deflate test encoder finishes"); + encoder.into_inner() + } + "br" => { + let mut encoder = async_compression::tokio::write::BrotliEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("brotli test body encodes"); + encoder.shutdown().await.expect("brotli test encoder finishes"); + encoder.into_inner() + } + "zstd" => { + let mut encoder = async_compression::tokio::write::ZstdEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("zstd test body encodes"); + encoder.shutdown().await.expect("zstd test encoder finishes"); + encoder.into_inner() + } + _ => panic!("unsupported test Content-Encoding: {content_encoding}"), + } +} + +pub(in crate::api::endpoints::hls_api::tests) struct TestTransientOrigin { + pub(in crate::api::endpoints::hls_api::tests) base_url: String, + pub(in crate::api::endpoints::hls_api::tests) requests: Arc>>, + pub(in crate::api::endpoints::hls_api::tests) task: tokio::task::JoinHandle<()>, +} + +impl Drop for TestTransientOrigin { + fn drop(&mut self) { self.task.abort(); } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn spawn_test_transient_origin() -> TestTransientOrigin { + spawn_test_transient_origin_with_response( + "206 Partial Content", + &[ + ("Content-Type", "video/mp2t"), + ("Content-Range", "bytes 2-15/16"), + ("Accept-Ranges", "bytes"), + ("Cache-Control", "no-store"), + ("ETag", "\"abc\""), + ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"), + ], + "transient-body", + ) + .await +} + +pub(in crate::api::endpoints::hls_api::tests) async fn spawn_test_transient_origin_with_response( + status_line: &'static str, + response_headers: &'static [(&'static str, &'static str)], + body: &'static str, +) -> TestTransientOrigin { + spawn_test_transient_origin_with_delayed_binary_response( + status_line, + response_headers, + body.as_bytes().to_vec(), + Duration::ZERO, + ) + .await +} + +pub(in crate::api::endpoints::hls_api::tests) async fn spawn_test_transient_origin_with_delayed_response( + status_line: &'static str, + response_headers: &'static [(&'static str, &'static str)], + body: &'static str, + response_delay: Duration, +) -> TestTransientOrigin { + spawn_test_transient_origin_with_delayed_binary_response( + status_line, + response_headers, + body.as_bytes().to_vec(), + response_delay, + ) + .await +} + +pub(in crate::api::endpoints::hls_api::tests) async fn spawn_test_transient_origin_with_delayed_binary_response( + status_line: &'static str, + response_headers: &'static [(&'static str, &'static str)], + body: Vec, + response_delay: Duration, +) -> TestTransientOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let requests = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let body = Arc::new(body); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + let body = Arc::clone(&body); + tokio::spawn(async move { + let mut buf = vec![0_u8; 4096]; + let Ok(read) = socket.read(&mut buf).await else { + return; + }; + if read == 0 { + return; + } + let request = String::from_utf8_lossy(&buf[..read]).to_string(); + requests.lock().await.push(request); + if !response_delay.is_zero() { + tokio::time::sleep(response_delay).await; + } + let mut response_head = format!("HTTP/1.1 {status_line}\r\nContent-Length: {}\r\n", body.len()); + for (name, value) in response_headers { + let _ = writeln!(&mut response_head, "{name}: {value}\r"); + } + response_head.push_str("Connection: close\r\n\r\n"); + let _ = socket.write_all(response_head.as_bytes()).await; + let _ = socket.write_all(body.as_slice()).await; + }); + } + }); + TestTransientOrigin { base_url: format!("http://{addr}"), requests, task } +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/owner_handoff.rs b/backend/app/src/api/endpoints/hls_api/tests/support/owner_handoff.rs new file mode 100644 index 000000000..96d57b6c6 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/support/owner_handoff.rs @@ -0,0 +1,100 @@ +use super::{ + super::{ + AppState, Arc, HlsAccessLease, HlsAccessLeaseId, HlsManifestCommitRequirement, HlsPlaybackFamilyKey, + HlsSessionHandle, HlsSessionKey, HlsStripMode, ProxySessionId, SegmentCacheStatus, StripConfig, + }, + enable_hls_cache, normal_manifest, test_app_state, test_fingerprint, +}; + +pub(in crate::api::endpoints::hls_api::tests) struct CanonicalOwnerHandoffFixture { + pub(in crate::api::endpoints::hls_api::tests) app_state: Arc, + pub(in crate::api::endpoints::hls_api::tests) session: HlsSessionHandle, + pub(in crate::api::endpoints::hls_api::tests) proxy_session_id: ProxySessionId, + pub(in crate::api::endpoints::hls_api::tests) leases: Vec<(HlsAccessLeaseId, u64)>, + pub(in crate::api::endpoints::hls_api::tests) strip: StripConfig, +} + +impl CanonicalOwnerHandoffFixture { + pub(in crate::api::endpoints::hls_api::tests) async fn new(lease_ids: &[&str]) -> Self { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "owner-handoff"), &app_state.get_encrypt_secret(), 100) + .await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let issued_at_ms = super::super::super::current_time_millis(); + let mut leases = Vec::with_capacity(lease_ids.len()); + for lease_id in lease_ids { + let lease_id = HlsAccessLeaseId((*lease_id).to_string()); + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), + proxy_session_id.clone(), + "hls-user".to_string(), + format!("{}-session", lease_id.0), + 1, + "owner-handoff".to_string(), + 12345, + issued_at_ms, + 60_000, + )) + .await; + leases.push((lease_id, issued_at_ms)); + } + Self { + app_state, + session, + proxy_session_id, + leases, + strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, + } + } + + pub(in crate::api::endpoints::hls_api::tests) async fn safe_session(&self) -> String { + let session = self.session.read().await; + super::super::super::safe_session_key(&session.key) + } + + pub(in crate::api::endpoints::hls_api::tests) fn handoff_context( + &self, + lease_index: usize, + safe_session: String, + request_deadline_ms: u64, + ) -> super::super::super::HlsCanonicalOwnerHandoffContext<'_> { + let (lease_id, issued_at_ms) = &self.leases[lease_index]; + super::super::super::HlsCanonicalOwnerHandoffContext { + app_state: &self.app_state, + proxy_session_id: &self.proxy_session_id, + access_lease_id: lease_id, + expected_lease_issued_at_ms: Some(*issued_at_ms), + strip: &self.strip, + server_path: None, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + manifest_boundary_rendered_at_ms: 0, + bandwidth_learning: super::super::super::HlsRuntimeBandwidthLearningContext::Disabled, + request_deadline_ms, + safe_session, + } + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn publish_owner_handoff_test_manifest(session: &HlsSessionHandle) { + let now_ms = super::super::super::current_time_millis(); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ + #EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n", + ); + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("owner handoff manifest maps"); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1_000, ready_at_ms: now_ms }; + } + session.advance_media_readiness_generation(); + session.render_and_store_manifest(now_ms).expect("owner handoff manifest renders"); + session.mark_authorized_media_access(now_ms); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/recording.rs b/backend/app/src/api/endpoints/hls_api/tests/support/recording.rs new file mode 100644 index 000000000..3d771d2e5 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/support/recording.rs @@ -0,0 +1,162 @@ +use super::{ + super::{ + header, hls_api_register, AppState, Arc, AsyncReadExt, AsyncWriteExt, Body, ConfigInput, ConfigTarget, + ConnectInfo, HashMap, Internable, M3uPlaylistItem, PlaylistItem, Request, Response, ServiceExt, TcpListener, + VirtualId, XtreamCluster, + }, + cache_test_m3u_hls_item, path_has_extension, test_addr, +}; + +/// Origin that only answers requests carrying the configured recording header. +pub(in crate::api::endpoints::hls_api::tests) async fn spawn_recording_header_origin( +) -> (String, Arc>>, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let base_url = format!("http://{}", listener.local_addr().expect("local addr")); + let requests = Arc::new(std::sync::Mutex::new(Vec::new())); + let seen = Arc::clone(&requests); + let task = tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let seen = Arc::clone(&seen); + tokio::spawn(async move { + let mut request = Vec::new(); + let mut chunk = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + match socket.read(&mut chunk).await { + Ok(0) | Err(_) => return, + Ok(read) => request.extend_from_slice(&chunk[..read]), + } + } + let request = String::from_utf8_lossy(&request).to_ascii_lowercase(); + let path = request.split_whitespace().nth(1).unwrap_or("/").to_string(); + seen.lock().expect("request log").push(request.clone()); + let (status, body): (&str, &[u8]) = if !request.contains("\r\nx-recording-test: custom\r\n") { + ("403 Forbidden", b"") + } else if path_has_extension(&path, "m3u8") { + ( + "200 OK", + b"#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\nseg1.ts\n", + ) + } else { + ("200 OK", b"segment-bytes") + }; + let head = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(body).await; + }); + } + }); + (base_url, requests, task) +} + +pub(in crate::api::endpoints::hls_api::tests) fn recording_test_router(app_state: &Arc) -> axum::Router { + axum::Router::new() + .nest( + "/tuliprox/api/v1", + crate::api::endpoints::v1_api_playlist::v1_api_playlist_register_public(axum::Router::new()), + ) + .merge(hls_api_register()) + .merge(crate::api::endpoints::m3u_api::m3u_api_register()) + .merge(crate::api::endpoints::xtream_api::xtream_api_register()) + .with_state(Arc::clone(app_state)) +} + +pub(in crate::api::endpoints::hls_api::tests) async fn recording_test_response( + router: &axum::Router, + uri: &str, +) -> Result, Box> { + let mut request = Request::builder() + .uri(uri) + .header(header::USER_AGENT, shared::model::RECORDING_STREAM_USER_AGENT) + .body(Body::empty())?; + request.extensions_mut().insert(ConnectInfo(test_addr())); + Ok(router.clone().oneshot(request).await?) +} + +pub(in crate::api::endpoints::hls_api::tests) fn configure_recording_test_listener( + app_state: &Arc, + storage_root: &std::path::Path, +) { + let mut config = (*app_state.app_config.config.load_full()).clone(); + config.api.host = "0.0.0.0".to_string(); + config.api.port = 8901; + config.storage_dir = storage_root.to_string_lossy().into_owned(); + config.web_ui = Some(crate::model::WebUiConfig::from(&shared::model::WebUiConfigDto { + path: Some("/tuliprox".to_string()), + ..Default::default() + })); + let mut recording = crate::model::RecordingConfig::from(&shared::model::RecordingConfigDto::default()); + recording.headers = HashMap::from([("X-Recording-Test".to_string(), "custom".to_string())]); + config.video = Some(crate::model::VideoConfig { extensions: vec![], web_search: None, recording: Some(recording) }); + app_state.app_config.config.store(Arc::new(config)); +} + +pub(in crate::api::endpoints::hls_api::tests) fn recording_test_url( + app_state: &Arc, + target: &ConfigTarget, + input: &ConfigInput, + cluster: XtreamCluster, +) -> Result> { + tuliprox_dvr::recording::recording_url::build_stable_recording_url( + &app_state.app_config, + &target.name, + &input.name, + 12345, + cluster, + None, + ) + .ok_or_else(|| "recording URL missing".into()) +} + +pub(in crate::api::endpoints::hls_api::tests) async fn cache_recording_test_item( + app_state: &Arc, + target: &ConfigTarget, + item: M3uPlaylistItem, +) -> Result<(), Box> { + use crate::{ + api::model::{PlaylistStorage, PlaylistXtreamStorage}, + repository::{BPlusTree, VirtualIdRecord}, + }; + if target.has_output(shared::model::TargetType::Xtream) { + let item = shared::model::XtreamPlaylistItem::from(&PlaylistItem::from(&item)); + let mut mapping = BPlusTree::new(); + mapping.insert( + item.virtual_id, + VirtualIdRecord::new( + item.provider_id, + item.virtual_id, + item.item_type, + VirtualId::new(0), + shared::model::UUIDType::default(), + ), + ); + app_state.playlists.cache_id_mapping(&target.name, mapping).await; + let mut live = BPlusTree::new(); + let mut vod = BPlusTree::new(); + let mut series = BPlusTree::new(); + match item.item_type.cluster() { + XtreamCluster::Live => live.insert(item.virtual_id.get(), item), + XtreamCluster::Video => vod.insert(item.virtual_id.get(), item), + XtreamCluster::Series => series.insert(item.virtual_id.get(), item), + } + app_state + .playlists + .cache_playlist( + &target.name, + PlaylistStorage::XtreamPlaylist(Box::new(PlaylistXtreamStorage { live, vod, series })), + ) + .await; + } else { + let storage_dir = app_state.app_config.config.load().storage_dir.clone(); + let input_storage = crate::repository::get_input_storage_path(&item.input_name, &storage_dir).await?; + let path = crate::repository::get_input_m3u_playlist_file_path(&input_storage, &item.input_name); + let group = shared::model::PlaylistGroup { + id: 1, + title: "recording".intern(), + xtream_cluster: item.item_type.cluster(), + channels: vec![PlaylistItem::from(&item)], + }; + crate::repository::persist_input_m3u_playlist(&app_state.app_config, &path, &[group]).await?; + cache_test_m3u_hls_item(app_state, target, item).await; + } + Ok(()) +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs b/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs new file mode 100644 index 000000000..6dbaca4d7 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/support/runtime_policy.rs @@ -0,0 +1,233 @@ +use super::{ + super::{ + prepare_terminal_base_evidence, prepared_terminal_bundle_key, snapshot_terminal_media_asset, AppState, Arc, + CustomStreamResponse, Duration, HlsAccessLeaseId, HlsLeasePlaybackMode, HlsPreparedTerminalBundleState, + HlsProxyManager, HlsTerminalTailPlan, ProxySessionId, StatusCode, TransportStreamBuffer, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, + }, + get_response, grant_hls_proxy_lease, map_ready_segment_without_lease, publish_ready_test_manifest_for_lease, + response_body, test_app_state_with_hls_proxy, +}; + +pub(in crate::api::endpoints::hls_api::tests) fn enable_runtime_policy_custom_responses(app_state: &Arc) { + app_state.app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { + channel_unavailable: None, + user_connections_exhausted: Some(TransportStreamBuffer::new( + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/user_connections_exhausted.ts" + )) + .to_vec(), + )), + provider_connections_exhausted: Some(TransportStreamBuffer::new( + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/provider_connections_exhausted.ts" + )) + .to_vec(), + )), + low_priority_preempted: Some(TransportStreamBuffer::new( + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/low_priority_preempted.ts")) + .to_vec(), + )), + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: Vec::new(), + }))); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn prepare_user_exhausted_terminal_bundle( + app_state: &Arc, + target_duration_ms: u64, +) { + let response = app_state.app_config.custom_stream_response.load_full().expect("runtime custom responses"); + let asset = response + .user_connections_exhausted + .as_ref() + .and_then(|buffer| snapshot_terminal_media_asset(buffer).ok()) + .expect("valid user-exhausted terminal asset"); + let key = prepared_terminal_bundle_key(&asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); + let state = + app_state.hls.proxy.start_prepared_terminal_bundle(asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); + let state = match state { + HlsPreparedTerminalBundleState::Preparing { .. } => app_state + .hls + .proxy + .wait_for_prepared_terminal_bundle(key) + .await + .expect("user-exhausted terminal bundle completion"), + state => state, + }; + assert!(matches!( + state, + HlsPreparedTerminalBundleState::Ready { ref bundle } if bundle.key == key + )); +} + +pub(in crate::api::endpoints::hls_api::tests) struct RuntimePolicyEndpointFixture { + pub(in crate::api::endpoints::hls_api::tests) _temp_dir: tempfile::TempDir, + pub(in crate::api::endpoints::hls_api::tests) app_state: Arc, + pub(in crate::api::endpoints::hls_api::tests) proxy_session_id: ProxySessionId, + pub(in crate::api::endpoints::hls_api::tests) lease_id: HlsAccessLeaseId, + pub(in crate::api::endpoints::hls_api::tests) manifest_uri: String, + pub(in crate::api::endpoints::hls_api::tests) live_segment_uri: String, +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_runtime_policy_base_timing( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + phase: &str, +) { + let session = + app_state.hls.proxy.sessions().get_by_proxy_session_id(proxy_session_id).await.expect("runtime policy session"); + let manifest = app_state + .hls + .proxy + .access_lease_response_snapshot(lease_id, proxy_session_id, super::super::super::current_time_millis()) + .await + .and_then(|lease| lease.last_manifest_snapshot) + .expect("runtime policy manifest"); + let evidence = prepare_terminal_base_evidence( + &session, + app_state.hls.proxy.segment_cache(), + &manifest, + super::super::super::current_time_millis(), + ) + .await; + assert!( + evidence.timing().is_some(), + "runtime policy base timing {phase}: {}", + evidence.track_evidence_reason_code() + ); + evidence.release(); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn serve_and_wait_runtime_policy_base_segment( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + live_segment_uri: &str, +) { + let initial_segment = get_response(Arc::clone(app_state), live_segment_uri, None).await; + assert_eq!(initial_segment.status(), StatusCode::OK); + assert!(!response_body(initial_segment).await.is_empty()); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let completed = app_state + .hls + .proxy + .access_lease_response_snapshot(lease_id, proxy_session_id, super::super::super::current_time_millis()) + .await + .and_then(|lease| lease.playback_cursor.highest_contiguous_completed_proxy_seq); + if completed == Some(123) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("initial live segment completion"); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn runtime_policy_endpoint_fixture( + publish_manifest: bool, +) -> RuntimePolicyEndpointFixture { + const TARGET_DURATION_MS: u64 = 12_000; + + let temp_dir = tempfile::tempdir().expect("runtime policy cache tempdir"); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); + enable_runtime_policy_custom_responses(&app_state); + let live_bytes = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); + let proxy_session_id = ProxySessionId(map_ready_segment_without_lease(&app_state, 123, "ts", live_bytes).await); + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&proxy_session_id) + .await + .expect("runtime policy session"); + session.write().await.segments.get_mut(&123).expect("runtime policy base segment").duration_ms = TARGET_DURATION_MS; + let lease_id = HlsAccessLeaseId(grant_hls_proxy_lease(&app_state, &proxy_session_id.0).await); + if publish_manifest { + publish_ready_test_manifest_for_lease(&app_state, &proxy_session_id, &lease_id, TARGET_DURATION_MS).await; + let target_duration_ms = app_state + .hls + .proxy + .access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::super::current_time_millis()) + .await + .and_then(|lease| lease.last_manifest_snapshot) + .map(|manifest| manifest.target_duration_ms) + .expect("published runtime policy target duration"); + prepare_user_exhausted_terminal_bundle(&app_state, target_duration_ms).await; + } + let manifest_uri = format!("/hls/shared/live/{}/{}/manifest.m3u8", proxy_session_id.0, lease_id.0); + let live_segment_uri = format!("/hls/shared/live/{}/{}/000123.ts", proxy_session_id.0, lease_id.0); + if publish_manifest { + assert_runtime_policy_base_timing(&app_state, &proxy_session_id, &lease_id, "before serve").await; + } + serve_and_wait_runtime_policy_base_segment(&app_state, &proxy_session_id, &lease_id, &live_segment_uri).await; + if publish_manifest { + assert_runtime_policy_base_timing(&app_state, &proxy_session_id, &lease_id, "after serve").await; + } + + RuntimePolicyEndpointFixture { + _temp_dir: temp_dir, + app_state, + proxy_session_id, + lease_id, + manifest_uri, + live_segment_uri, + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn wait_for_runtime_policy_terminal_plan( + fixture: &RuntimePolicyEndpointFixture, +) -> Arc { + let plan = tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Some(HlsLeasePlaybackMode::TerminalTail(plan)) = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::super::current_time_millis(), + ) + .await + .map(|lease| lease.playback_mode) + { + return plan; + } + tokio::task::yield_now().await; + } + }) + .await; + if let Ok(plan) = plan { + return plan; + } + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::super::current_time_millis(), + ) + .await; + let state = lease.as_ref().map_or("missing", |lease| lease.state.as_log_value()); + let playback = lease.as_ref().map_or("missing", |lease| match lease.playback_mode { + HlsLeasePlaybackMode::Live => "live", + HlsLeasePlaybackMode::TerminalTail(_) => "terminal-tail", + HlsLeasePlaybackMode::TerminalUnavailable { .. } => "terminal-unavailable", + HlsLeasePlaybackMode::Ended => "ended", + }); + panic!( + "runtime policy terminal owner deadline: state={state} playback={playback} owners={}", + fixture.app_state.hls.proxy.terminal_pending().owner_count() + ); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/sessions.rs b/backend/app/src/api/endpoints/hls_api/tests/support/sessions.rs new file mode 100644 index 000000000..e29da4016 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/support/sessions.rs @@ -0,0 +1,307 @@ +use super::super::{ + AppState, Arc, ConfigInput, ConnectionKind, CreateUserSessionParams, Duration, Fingerprint, HlsAccessLease, + HlsAccessLeaseId, HlsAccessLeaseTiming, HlsOriginAccountBinding, HlsPlaybackFamilyKey, HlsSessionHandle, + PlaylistItemType, ProxySessionId, ProxyUserCredentials, SocketAddr, UserConnectionPermission, +}; + +pub(in crate::api::endpoints::hls_api::tests) async fn create_bound_hls_test_session( + app_state: &Arc, + input: &ConfigInput, + stream_ref: &str, + account_name: &str, + now_ms: u64, +) -> HlsSessionHandle { + let origin_source = super::super::super::build_hls_origin_source(input, stream_ref); + let session_key = origin_source.session_key(); + let (session, _) = app_state + .hls + .proxy + .get_or_create_session_with_source_and_outcome( + session_key, + origin_source, + &app_state.get_encrypt_secret(), + now_ms, + ) + .await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + { + let mut session_guard = session.write().await; + session_guard.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from(account_name), + &proxy_session_id, + now_ms, + )); + } + session +} + +pub(in crate::api::endpoints::hls_api::tests) async fn create_unbound_hls_test_session( + app_state: &Arc, + input: &ConfigInput, + stream_ref: &str, + now_ms: u64, +) -> HlsSessionHandle { + let origin_source = super::super::super::build_hls_origin_source(input, stream_ref); + let session_key = origin_source.session_key(); + app_state + .hls + .proxy + .get_or_create_session_with_source_and_outcome( + session_key, + origin_source, + &app_state.get_encrypt_secret(), + now_ms, + ) + .await + .0 +} + +pub(in crate::api::endpoints::hls_api::tests) async fn activate_test_hls_access_lease( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + lease_id: &str, + now_ms: u64, + ttl_ms: u64, +) { + let lease_id = HlsAccessLeaseId(lease_id.to_string()); + let valid_window_ms = ttl_ms.saturating_mul(10).max(ttl_ms); + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + valid_window_ms, + )) + .await; + assert!(app_state + .hls + .proxy + .activate_access_lease( + &lease_id, + proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: ttl_ms, valid_window_ms }, + ) + .await + .is_activated()); +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_addr() -> SocketAddr { + "127.0.0.1:55123".parse().unwrap_or_else(|_| unreachable!()) +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_addr_with_port(port: u16) -> SocketAddr { + SocketAddr::from(([127, 0, 0, 1], port)) +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_fingerprint() -> Fingerprint { + Fingerprint::new("test".to_string(), "127.0.0.1".to_string(), test_addr()) +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_fingerprint_with_addr(addr: SocketAddr) -> Fingerprint { + Fingerprint::new(format!("test-{}", addr.port()), "127.0.0.1".to_string(), addr) +} + +pub(in crate::api::endpoints::hls_api::tests) async fn create_active_hls_user_session(app_state: &Arc) { + create_active_hls_user_session_with( + app_state, + "hls-session-token", + "origin-provider", + "http://origin.example.com/live/12345.m3u8", + test_addr(), + ) + .await; +} + +pub(in crate::api::endpoints::hls_api::tests) async fn create_active_hls_user_session_with( + app_state: &Arc, + session_token: &str, + provider: &str, + stream_url: &str, + addr: SocketAddr, +) { + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.max_connections = 1; + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token, + virtual_id: 12345, + provider, + stream_url, + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; +} + +pub(in crate::api::endpoints::hls_api::tests) async fn mark_hls_user_session_exhausted(app_state: &Arc) { + let user = app_state.app_config.get_user_credentials("hls-user").expect("configured HLS test user"); + let addr = test_addr(); + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "hls-session-token", + virtual_id: 12345, + provider: "origin-provider", + stream_url: "http://origin.example.com/live/12345.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Exhausted, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; +} + +pub(in crate::api::endpoints::hls_api::tests) async fn prepare_pending_test_hls_access_lease( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, +) { + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + access_lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "12345".to_string(), + 12345, + super::super::super::current_time_millis(), + super::super::super::hls_access_lease_ttl_ms(app_state), + )) + .await; +} + +pub(in crate::api::endpoints::hls_api::tests) async fn grant_hls_proxy_lease( + app_state: &Arc, + proxy_session_id: &str, +) -> String { + create_active_hls_user_session(app_state).await; + let now_ms = super::super::super::current_time_millis(); + let lease_id = HlsAccessLeaseId(format!("test-access-lease-{proxy_session_id}")); + let family_key = HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key); + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + family_key, + ProxySessionId(proxy_session_id.to_string()), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + super::super::super::hls_access_lease_ttl_ms(app_state), + )) + .await; + assert!(app_state + .hls + .proxy + .activate_access_lease( + &lease_id, + &ProxySessionId(proxy_session_id.to_string()), + now_ms, + HlsAccessLeaseTiming { + active_window_ms: 5_000, + valid_window_ms: super::super::super::hls_access_lease_ttl_ms(app_state), + }, + ) + .await + .is_activated()); + lease_id.0 +} + +pub(in crate::api::endpoints::hls_api::tests) async fn wait_for_provider_connection_count( + app_state: &Arc, + expected: usize, +) { + for _ in 0..50 { + let actual = app_state.active_provider.get_provider_connections_count(); + if actual == expected { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(app_state.active_provider.get_provider_connections_count(), expected); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn hls_session_last_media_at_ms( + app_state: &Arc, + proxy_session_id: &str, +) -> Option { + app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.to_string())) + .await + .expect("session should exist") + .read() + .await + .activity + .last_authorized_media_at_ms +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_no_hls_cache_stream_registered( + app_state: &Arc, +) { + assert!(app_state.active_users.active_streams().await.is_empty()); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn access_lease_session_token( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + access_lease_id: &HlsAccessLeaseId, +) -> String { + app_state + .hls + .proxy + .access_lease(access_lease_id, proxy_session_id, super::super::super::current_time_millis()) + .await + .expect("access lease should exist") + .user_session_token +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_hls_cache_stream_registered( + app_state: &Arc, + proxy_session_id: &str, +) { + let streams = app_state.active_users.active_streams().await; + assert_eq!(streams.len(), 1); + let stream = &streams[0]; + assert_eq!(stream.username, "hls-user"); + assert_eq!(stream.session_token.as_deref(), Some("hls-session-token")); + assert_eq!(stream.provider.as_ref(), "origin-provider"); + assert_eq!(stream.channel.item_type, PlaylistItemType::LiveHls); + assert!(stream.channel.shared); + assert_eq!( + stream.channel.shared_stream_id, + Some(super::super::super::hls_cache_shared_stream_id(&ProxySessionId(proxy_session_id.to_string()))) + ); + assert_eq!(stream.channel.shared_joined_existing, Some(false)); + assert_eq!(stream.channel.url.as_ref(), format!("/hls/shared/live/{proxy_session_id}/manifest.m3u8")); + assert!(!stream.channel.url.contains("test-access-lease")); + assert!(!stream.channel.url.contains("hls-session-token")); + assert!(!stream.channel.url.contains("origin.example.com")); + assert!(!stream.channel.url.contains("/hls/hls-user/")); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/support/terminal.rs b/backend/app/src/api/endpoints/hls_api/tests/support/terminal.rs new file mode 100644 index 000000000..893887834 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/support/terminal.rs @@ -0,0 +1,124 @@ +use super::super::{ + build_terminal_tail_plan, snapshot_terminal_media_asset, AppState, Arc, HlsAccessLeaseId, HlsLeaseManifestSegment, + HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, HlsManifestCommitIdentity, HlsManifestDeliveryMode, + HlsMediaContainer, HlsRuntimeCustomTailAssetIdentity, HlsTerminalAssetIdentity, HlsTerminalBaseMediaState, + HlsTerminalBaseProtection, HlsTerminalBaseSegmentAvailability, HlsTerminalMediaAsset, HlsTerminalTailBuildInput, + HlsTerminalTailGeneration, HlsTerminalTailProtection, ProxySessionId, TransportStreamBuffer, +}; + +pub(in crate::api::endpoints::hls_api::tests) fn terminal_test_asset() -> Arc { + let bytes = bytes::Bytes::from_static(include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/channel_unavailable.ts" + ))); + let buffer = TransportStreamBuffer::new(bytes.to_vec()); + snapshot_terminal_media_asset(&buffer).expect("terminal test asset is valid") +} + +pub(in crate::api::endpoints::hls_api::tests) async fn terminalize_existing_test_lease( + app_state: &Arc, + proxy_session_id: &str, + lease_id: &str, + base_proxy_seq: u64, +) -> TransportStreamBuffer { + let proxy_session_id = ProxySessionId(proxy_session_id.to_string()); + let lease_id = HlsAccessLeaseId(lease_id.to_string()); + let buffer = TransportStreamBuffer::new( + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")).to_vec(), + ); + let asset = snapshot_terminal_media_asset(&buffer).expect("terminal test asset is valid"); + let base_manifest = HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(7), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 7, + delivered_at_ms: super::super::super::current_time_millis(), + first_proxy_seq: base_proxy_seq, + last_proxy_seq: base_proxy_seq, + visible_segments: Arc::from([HlsLeaseManifestSegment { + proxy_seq: base_proxy_seq, + duration_ms: 4_000, + uri: format!("/iptv/hls/shared/live/{}/{}/{base_proxy_seq:06}.ts", proxy_session_id.0, lease_id.0).into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }]), + discontinuity_sequence: 3, + target_duration_ms: asset.duration_ms().saturating_add(1_000), + playlist_duration_ms: 4_000, + last_visible_media_end_ms: 4_000, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + }; + let base_timing = Some(HlsTerminalTailBuildInput::base_timing_for_test(&asset, &base_manifest)); + let base_splice_evidence = Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); + let terminal_splice_evidence = base_splice_evidence.clone(); + let plan = build_terminal_tail_plan(HlsTerminalTailBuildInput { + generation: HlsTerminalTailGeneration(17), + created_at_ms: super::super::super::current_time_millis(), + base_availability: Arc::from([HlsTerminalBaseSegmentAvailability { + proxy_seq: base_proxy_seq, + media_state: HlsTerminalBaseMediaState::Ready, + required_map_ready: true, + required_key_ready: true, + protection: HlsTerminalBaseProtection::Protectable, + }]), + base_track_signature: Some(asset.track_signature().clone()), + base_splice_evidence, + terminal_splice_evidence, + base_timing, + base_key_bindings: Arc::from([]), + expected_asset: HlsRuntimeCustomTailAssetIdentity::channel_unavailable(HlsTerminalAssetIdentity::from_asset( + &asset, + )), + base_manifest: base_manifest.clone(), + anchored_bundle: HlsTerminalTailBuildInput::anchored_bundle_for_test(&asset, base_manifest.target_duration_ms), + asset, + }) + .expect("terminal test plan is compatible"); + let protection = HlsTerminalTailProtection { + generation: plan.generation, + base_proxy_seqs: Arc::clone(&plan.protected_base_proxy_seqs), + key_bindings: plan.key_bindings(), + }; + { + let mut leases = app_state.hls.proxy.access_leases().write().await; + let mut lease = leases.remove_access_lease(&lease_id).expect("test lease exists before terminal cutover"); + lease.last_manifest_snapshot = Some(base_manifest); + lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(Arc::new(plan)); + leases.prepare_access_lease(lease); + } + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&proxy_session_id) + .await + .expect("test session exists before terminal cutover"); + session.write().await.install_terminal_tail_protection(lease_id, protection); + buffer +} + +pub(in crate::api::endpoints::hls_api::tests) async fn terminal_test_plan_shape( + app_state: &Arc, + proxy_session_id: &str, + lease_id: &str, +) -> (u64, u16) { + let snapshot = app_state + .hls + .proxy + .access_lease_response_snapshot( + &HlsAccessLeaseId(lease_id.to_string()), + &ProxySessionId(proxy_session_id.to_string()), + super::super::super::current_time_millis(), + ) + .await + .expect("terminal test lease snapshot exists"); + let HlsLeasePlaybackMode::TerminalTail(plan) = snapshot.playback_mode else { + panic!("terminal test lease keeps terminal playback mode"); + }; + (plan.generation.0, plan.segment_count) +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/terminal_resources.rs b/backend/app/src/api/endpoints/hls_api/tests/terminal_resources.rs new file mode 100644 index 000000000..eb5d658e1 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/terminal_resources.rs @@ -0,0 +1,328 @@ +use super::{ + assert_hls_cache_stream_registered, assert_no_hls_cache_stream_registered, + enable_channel_unavailable_custom_response, get_response, hls_session_last_media_at_ms, map_hls_map, + map_ready_segment, map_transient_resource, request_response, response_body, terminal_test_plan_shape, + terminalize_existing_test_lease, test_app_state_with_hls_proxy, +}; +use crate::api::model::{ + AppState, HlsAccessLeaseId, HlsLifecycleEvent, HlsLifecycleEventKey, HlsProxyManager, ProxySessionId, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, +}; +use axum::http::{header, Method, StatusCode}; +use std::sync::Arc; + +pub(in crate::api::endpoints::hls_api::tests) async fn terminal_head_content_length( + app_state: &Arc, + proxy_session_id: &str, + segment_uri: &str, +) -> usize { + let head_response = request_response(Arc::clone(app_state), Method::HEAD, segment_uri, None).await; + assert_eq!(head_response.status(), StatusCode::OK); + assert_eq!(head_response.headers()[header::CONTENT_TYPE], "video/mp2t"); + assert_eq!(head_response.headers()[header::ACCEPT_RANGES], "bytes"); + assert!(head_response.headers()[header::CACHE_CONTROL].to_str().is_ok_and(|value| value.contains("immutable"))); + let content_length = head_response.headers()[header::CONTENT_LENGTH] + .to_str() + .expect("HEAD content length") + .parse::() + .expect("HEAD content length value"); + assert!(content_length > 0); + assert!(response_body(head_response).await.is_empty()); + assert_eq!(hls_session_last_media_at_ms(app_state, proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(app_state).await; + + let head_range = request_response(Arc::clone(app_state), Method::HEAD, segment_uri, Some("bytes=0-187")).await; + assert_eq!(head_range.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(head_range.headers()[header::CONTENT_LENGTH], "188"); + let expected_content_range = format!("bytes 0-187/{content_length}"); + assert_eq!(head_range.headers()[header::CONTENT_RANGE].to_str().ok(), Some(expected_content_range.as_str())); + assert!(response_body(head_range).await.is_empty()); + assert_eq!(hls_session_last_media_at_ms(app_state, proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(app_state).await; + content_length +} + +#[tokio::test] +async fn hls_terminal_response_serves_prepared_finite_full_and_range_bytes_per_index() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"original-live-tail").await; + let lease_id = format!("test-access-lease-{proxy_session_id}"); + let terminal_renderer = terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; + let renders_before_requests = terminal_renderer.finite_hls_render_count(); + let finalizations_before_requests = terminal_renderer.finite_hls_finalize_count(); + assert_eq!(renders_before_requests, usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); + assert_eq!(finalizations_before_requests, usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); + let (generation, _) = terminal_test_plan_shape(&app_state, &proxy_session_id, &lease_id).await; + let segment_zero_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/0.ts"); + let segment_one_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/1.ts"); + let repair_before = app_state.hls.proxy.segment_repair().stats().await; + let provider_connections_before = app_state.active_provider.get_provider_connections_count(); + + let head_content_length = terminal_head_content_length(&app_state, &proxy_session_id, &segment_zero_uri).await; + + let segment_zero_response = get_response(Arc::clone(&app_state), &segment_zero_uri, None).await; + assert_eq!(segment_zero_response.status(), StatusCode::OK); + assert!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await.is_some()); + assert!(!tuliprox_core::utils::response_compression::should_compress_response(&segment_zero_response)); + assert_eq!(segment_zero_response.headers()[header::CONTENT_TYPE], "video/mp2t"); + assert_eq!(segment_zero_response.headers()[header::ACCEPT_RANGES], "bytes"); + assert!(segment_zero_response.headers()[header::CACHE_CONTROL] + .to_str() + .is_ok_and(|value| value.contains("immutable"))); + let declared_length = segment_zero_response.headers()[header::CONTENT_LENGTH] + .to_str() + .expect("finite content length header") + .parse::() + .expect("finite content length value"); + assert_eq!(declared_length, head_content_length); + let segment_zero = response_body(segment_zero_response).await; + assert_eq!(segment_zero.len(), declared_length); + assert_hls_cache_stream_registered(&app_state, &proxy_session_id).await; + assert!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await.is_some()); + assert_eq!(app_state.hls.proxy.segment_repair().stats().await, repair_before); + assert_eq!(app_state.active_provider.get_provider_connections_count(), provider_connections_before); + + let segment_zero_again = response_body(get_response(Arc::clone(&app_state), &segment_zero_uri, None).await).await; + let segment_one = response_body(get_response(Arc::clone(&app_state), &segment_one_uri, None).await).await; + assert_eq!(segment_zero, segment_zero_again, "same terminal index is immutable"); + assert_ne!(segment_zero, segment_one, "successive terminal indices advance timestamps and continuity"); + + let range_response = get_response(Arc::clone(&app_state), &segment_zero_uri, Some("bytes=0-187")).await; + assert_eq!(range_response.status(), StatusCode::PARTIAL_CONTENT); + assert!(!tuliprox_core::utils::response_compression::should_compress_response(&range_response)); + assert_eq!(range_response.headers()[header::CONTENT_LENGTH], "188"); + let expected_content_range = format!("bytes 0-187/{declared_length}"); + assert_eq!(range_response.headers()[header::CONTENT_RANGE].to_str().ok(), Some(expected_content_range.as_str())); + assert_eq!(range_response.headers()[header::ACCEPT_RANGES], "bytes"); + assert!(range_response.headers()[header::CACHE_CONTROL].to_str().is_ok_and(|value| value.contains("immutable"))); + assert_eq!(response_body(range_response).await, segment_zero.slice(..188)); + + let unsatisfiable_range = format!("bytes={declared_length}-"); + let unsatisfiable_response = get_response(app_state, &segment_zero_uri, Some(&unsatisfiable_range)).await; + assert_eq!(unsatisfiable_response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert!(!tuliprox_core::utils::response_compression::should_compress_response(&unsatisfiable_response)); + let expected_unsatisfied_content_range = format!("bytes */{declared_length}"); + assert_eq!( + unsatisfiable_response.headers()[header::CONTENT_RANGE].to_str().ok(), + Some(expected_unsatisfied_content_range.as_str()) + ); + assert!(response_body(unsatisfiable_response).await.is_empty()); + assert_eq!( + terminal_renderer.finite_hls_render_count(), + renders_before_requests, + "terminal HTTP serving must not invoke the TS writer again" + ); + assert_eq!( + terminal_renderer.finite_hls_finalize_count(), + finalizations_before_requests, + "terminal HTTP serving must not invoke lease-specific TS finalization again" + ); +} + +#[tokio::test] +async fn hls_terminal_response_body_after_lease_denial_does_not_extend_shared_session_activity() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"original-live-tail").await; + let lease_id = format!("test-access-lease-{proxy_session_id}"); + terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; + let (generation, _) = terminal_test_plan_shape(&app_state, &proxy_session_id, &lease_id).await; + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&ProxySessionId(proxy_session_id.clone())) + .await + .expect("terminal session"); + let segment_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/0.ts"); + let response = get_response(Arc::clone(&app_state), &segment_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + let activity_after_authorized_response = + session.read().await.activity.last_authorized_media_at_ms.expect("terminal GET marks media activity"); + + let _ = app_state + .hls + .proxy + .deny_access_lease( + &HlsAccessLeaseId(lease_id), + tuliprox_hls::HlsAccessLeaseDenialMode::PreserveCommittedFiniteTail, + ) + .await; + assert!(!response_body(response).await.is_empty()); + + assert_eq!( + session.read().await.activity.last_authorized_media_at_ms, + Some(activity_after_authorized_response), + "body completion after denial must not extend activity" + ); +} + +#[tokio::test] +async fn hls_terminal_response_rejects_stale_malformed_and_out_of_bounds_paths() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"original-live-tail").await; + let lease_id = format!("test-access-lease-{proxy_session_id}"); + terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; + let (generation, segment_count) = terminal_test_plan_shape(&app_state, &proxy_session_id, &lease_id).await; + let stale_generation = generation.saturating_add(1); + let stale_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{stale_generation}/0.ts"); + let out_of_bounds_uri = + format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/{segment_count}.ts"); + + let malformed_generation = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/01/0.ts"); + let non_numeric_generation = + format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/not-a-generation/0.ts"); + let overflowing_generation = + format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/18446744073709551616/0.ts"); + let malformed_file = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/00.ts"); + let wrong_extension = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/0.m4s"); + + for uri in [ + stale_uri, + out_of_bounds_uri, + malformed_generation, + non_numeric_generation, + overflowing_generation, + malformed_file, + wrong_extension, + ] { + let response = get_response(Arc::clone(&app_state), &uri, None).await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!(!response.headers().contains_key(header::LOCATION)); + } +} + +#[tokio::test] +async fn expired_route_replays_already_committed_custom_tail() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"original-live-tail").await; + let lease_id = format!("test-access-lease-{proxy_session_id}"); + terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; + let (generation, _) = terminal_test_plan_shape(&app_state, &proxy_session_id, &lease_id).await; + let proxy_session_key = ProxySessionId(proxy_session_id.clone()); + let lease_key = HlsAccessLeaseId(lease_id.clone()); + let expired_at_ms = super::super::current_time_millis().saturating_sub(1); + { + let mut leases = app_state.hls.proxy.access_leases().write().await; + let mut lease = leases.remove_access_lease(&lease_key).expect("terminal lease exists"); + lease.valid_until_ms = expired_at_ms; + leases.prepare_access_lease(lease); + } + let manifest_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/manifest.m3u8"); + let segment_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/terminal/{generation}/0.ts"); + + let manifest_response = get_response(Arc::clone(&app_state), &manifest_uri, None).await; + let segment_response = get_response(Arc::clone(&app_state), &segment_uri, None).await; + + assert_eq!(manifest_response.status(), StatusCode::OK); + assert!(!manifest_response.headers().contains_key(header::LOCATION)); + assert!(String::from_utf8(response_body(manifest_response).await.to_vec()) + .expect("expired committed manifest utf8") + .ends_with("#EXT-X-ENDLIST\n")); + assert_eq!(segment_response.status(), StatusCode::OK); + assert!(!response_body(segment_response).await.is_empty()); + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&proxy_session_key) + .await + .expect("terminal session exists"); + assert!(session.read().await.terminal_tail_protection(&lease_key).is_some()); + + let cleanup_at_ms = super::super::current_time_millis(); + app_state + .hls + .proxy + .handle_lifecycle_event( + &app_state.active_users, + &app_state.active_provider, + HlsLifecycleEvent { + key: HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: lease_key.clone(), + proxy_session_id: proxy_session_key, + }, + due_at_ms: cleanup_at_ms, + }, + cleanup_at_ms, + ) + .await; + + assert!(!session.read().await.has_terminal_tail_protections()); + assert!(app_state + .hls + .proxy + .access_lease_response_snapshot(&lease_key, &ProxySessionId(proxy_session_id), cleanup_at_ms) + .await + .is_none()); + assert_eq!(get_response(Arc::clone(&app_state), &manifest_uri, None).await.status(), StatusCode::NOT_FOUND); + assert_eq!(get_response(app_state, &segment_uri, None).await.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn expired_route_without_session_evidence_returns_not_found_instead_of_unanchored_manifest() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"live-without-lease").await; + enable_channel_unavailable_custom_response(&app_state); + let missing_lease = "expired-without-base-evidence"; + let manifest_uri = format!("/hls/shared/live/{proxy_session_id}/{missing_lease}/manifest.m3u8"); + + let response = get_response(app_state, &manifest_uri, None).await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!(!response.headers().contains_key(header::LOCATION)); + assert!(response_body(response).await.is_empty()); +} + +#[tokio::test] +async fn hls_terminal_response_normal_segment_map_and_resource_routes_never_serve_terminal_fallbacks() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"original-live-tail").await; + let lease_id = format!("test-access-lease-{proxy_session_id}"); + terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; + let normal_segment_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/000123.ts"); + + let normal_segment_response = get_response(Arc::clone(&app_state), &normal_segment_uri, None).await; + + assert_eq!(normal_segment_response.status(), StatusCode::OK); + assert!(!normal_segment_response.headers().contains_key(header::LOCATION)); + assert_eq!(response_body(normal_segment_response).await, bytes::Bytes::from_static(b"original-live-tail")); + + let map_temp_dir = tempfile::tempdir().expect("map tempdir"); + let map_app_state = + test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(map_temp_dir.path(), 300))); + let map_proxy_session_id = map_hls_map(&map_app_state, b"original-map", true).await; + let map_lease_id = format!("test-access-lease-{map_proxy_session_id}"); + let map_session = map_app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&ProxySessionId(map_proxy_session_id.clone())) + .await + .expect("map session exists"); + let map_base_proxy_seq = *map_session.read().await.segments.keys().next().expect("map manifest has media"); + terminalize_existing_test_lease(&map_app_state, &map_proxy_session_id, &map_lease_id, map_base_proxy_seq).await; + let map_uri = format!("/hls/shared/live/{map_proxy_session_id}/{map_lease_id}/map/000000.mp4"); + let map_response = get_response(map_app_state, &map_uri, None).await; + assert_eq!(map_response.status(), StatusCode::NOT_FOUND); + assert!(!map_response.headers().contains_key(header::LOCATION)); + assert!(response_body(map_response).await.is_empty()); + + let resource_temp_dir = tempfile::tempdir().expect("resource tempdir"); + let resource_app_state = + test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(resource_temp_dir.path(), 300))); + let (resource_proxy_session_id, resource_id) = + map_transient_resource(&resource_app_state, "http://origin.example.com/old.ts", "ts", true).await; + let resource_lease_id = format!("test-access-lease-{resource_proxy_session_id}"); + terminalize_existing_test_lease(&resource_app_state, &resource_proxy_session_id, &resource_lease_id, 0).await; + let resource_uri = format!("/hls/shared/live/{resource_proxy_session_id}/{resource_lease_id}/r/{resource_id}.ts"); + let resource_response = get_response(resource_app_state, &resource_uri, None).await; + assert_eq!(resource_response.status(), StatusCode::NOT_FOUND); + assert!(!resource_response.headers().contains_key(header::LOCATION)); + assert!(response_body(resource_response).await.is_empty()); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs new file mode 100644 index 000000000..5090b3914 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail.rs @@ -0,0 +1,1494 @@ +use super::{ + access_lease_id_from_variant_uri, cache_test_m3u_hls_item, configure_default_test_server, + create_active_hls_user_session, enable_channel_unavailable_custom_response, enable_hls_cache, get_response, + get_status, media_uri_count, normal_manifest, path_has_extension, prepare_pending_test_hls_access_lease, + proxy_session_id_from_variant_uri, regression_origin_manifest, response_body, single_variant_master_playlist, + spawn_test_binary_origin, spawn_test_encrypted_hls_origin, store_test_sources_with_target, terminal_test_asset, + test_app_state_with_hls_proxy_and_inputs, test_app_state_with_inputs, test_fingerprint, test_hls_access_context, + test_hls_entry_stream_context, test_m3u_hls_item, test_m3u_hls_share_target, try_test_hls_cached_manifest_response, + TestBinaryOriginResponse, TestSegmentOrigin, AES_TEST_KEY_BYTES, AES_TEST_MANIFEST, AES_TEST_PLAINTEXT_SEGMENT, +}; +use crate::{ + api::model::{ + build_proxy_session_id, commit_terminal_tail_if_lease_reserve_requires_cutover, prepare_terminal_base_evidence, + prepared_terminal_bundle_key, snapshot_terminal_media_asset, trigger_origin_refresh_sync, AppState, + ConnectionKind, HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, HlsAccessLease, HlsAccessLeaseId, + HlsAccessLeaseState, HlsAccessLeaseTiming, HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, + HlsManifestAcceptanceDirective, HlsManifestAcceptanceEvaluationOutcome, HlsManifestAcceptanceExhaustionReason, + HlsManifestAcceptanceTrigger, HlsManifestCommitRequirement, HlsObservedRecoveryLatency, HlsOperationTimeoutMs, + HlsOriginPathCondition, HlsPlaybackFamilyKey, HlsPreparedTerminalBundleState, HlsProxyManager, + HlsRecoveryEtaMs, HlsRecoveryTimingPolicy, HlsRecoveryWorkload, HlsSessionHandle, HlsSessionKey, + HlsTerminalMediaAsset, HlsTerminalMediaPreparationState, HlsTerminalResolution, HlsTerminalSegmentPath, + HlsTerminalTailGeneration, HlsTransitionMarginMs, LiveHlsOriginEntry, OriginRefreshRequest, ProxySessionId, + RetryPolicy, SegmentCacheStatus, TransportStreamBuffer, HLS_TERMINAL_TAIL_SEGMENT_COUNT, + }, + model::{ConfigInput, HlsCacheConfig, ProxyUserCredentials, StripConfig}, +}; +use axum::{ + http::{header, HeaderMap, StatusCode}, + response::IntoResponse, +}; +use shared::model::{ + HlsCacheConfigDto, HlsManifestRecoveryBurstConfigDto, HlsManifestRecoveryBurstLevel, HlsStripConfigDto, + HlsStripMode, InputType, UserConnectionPermission, +}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +pub(in crate::api::endpoints::hls_api::tests) async fn terminal_generation_for_lease( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, +) -> HlsTerminalTailGeneration { + let lease = app_state + .hls + .proxy + .access_lease_response_snapshot(lease_id, proxy_session_id, super::super::current_time_millis()) + .await + .expect("terminal lease remains stored"); + let HlsLeasePlaybackMode::TerminalTail(plan) = lease.playback_mode else { + panic!("lease remains terminal"); + }; + plan.generation +} + +pub(in crate::api::endpoints::hls_api::tests) const RECOVERY_LOGICAL_LARGE_SEGMENT_BYTES: u64 = 20 * 1024 * 1024; + +pub(in crate::api::endpoints::hls_api::tests) struct RecoveryBeforeCutoverFixture { + pub(in crate::api::endpoints::hls_api::tests) _temp_dir: tempfile::TempDir, + pub(in crate::api::endpoints::hls_api::tests) origin: TestSegmentOrigin, + pub(in crate::api::endpoints::hls_api::tests) origin_phase: Arc, + pub(in crate::api::endpoints::hls_api::tests) app_state: Arc, + pub(in crate::api::endpoints::hls_api::tests) session: HlsSessionHandle, + pub(in crate::api::endpoints::hls_api::tests) proxy_session_id: ProxySessionId, + pub(in crate::api::endpoints::hls_api::tests) lease_id: HlsAccessLeaseId, + pub(in crate::api::endpoints::hls_api::tests) refresh: OriginRefreshRequest, +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_initial_recovery_window( + fixture: &RecoveryBeforeCutoverFixture, +) { + let response = try_test_hls_cached_manifest_response( + &fixture.app_state, + &fixture.session, + &fixture.lease_id, + HlsAccessLeaseState::Pending, + &StripConfig { mode: HlsStripMode::Segments, value: 3 }, + None, + super::HlsCachedManifestOptions::initial(Duration::from_secs(10)), + ) + .await + .expect("READY initial manifest"); + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains("/000000.ts")); + assert!(body.contains("/000002.ts")); + assert!(!body.contains("/000003.ts")); + let now_ms = super::super::current_time_millis(); + assert!(fixture + .app_state + .hls + .proxy + .activate_access_lease( + &fixture.lease_id, + &fixture.proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 120_000, valid_window_ms: 180_000 }, + ) + .await + .is_activated()); + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, now_ms) + .await + .expect("active stripped lease"); + let snapshot = lease.last_manifest_snapshot.as_ref().expect("lease manifest snapshot"); + assert_eq!(snapshot.visible_segments.len(), 3); + assert_eq!(snapshot.last_proxy_seq, 2); + let evidence = + prepare_terminal_base_evidence(&fixture.session, fixture.app_state.hls.proxy.segment_cache(), snapshot, now_ms) + .await; + assert_eq!(evidence.track_signature(), Some(terminal_test_asset().track_signature().clone())); + evidence.release(); + extend_ready_segment_as_sparse_file(&fixture.app_state, &fixture.session, 2, RECOVERY_LOGICAL_LARGE_SEGMENT_BYTES) + .await; + let uri = format!("/hls/shared/live/{}/{}/000002.ts", fixture.proxy_session_id.0, fixture.lease_id.0); + let segment = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(segment.status(), StatusCode::OK); + assert_eq!(segment.headers()[header::CONTENT_LENGTH], RECOVERY_LOGICAL_LARGE_SEGMENT_BYTES.to_string()); + assert_eq!( + u64::try_from(response_body(segment).await.len()).unwrap_or(u64::MAX), + RECOVERY_LOGICAL_LARGE_SEGMENT_BYTES + ); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn recovery_before_cutover_fixture() -> RecoveryBeforeCutoverFixture +{ + let temp_dir = tempfile::tempdir().expect("recovery cache tempdir"); + let unchanged_manifest = Arc::<[u8]>::from(regression_origin_manifest(100, 6)); + let progressed_manifest = Arc::<[u8]>::from(regression_origin_manifest(101, 6)); + let segment = Arc::<[u8]>::from( + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + .as_slice(), + ); + let origin_phase = Arc::new(AtomicUsize::new(0)); + let phase = Arc::clone(&origin_phase); + let origin = spawn_test_binary_origin(Arc::new(move |path| { + if path_has_extension(path, "m3u8") { + return match phase.load(Ordering::SeqCst) { + 0 => TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&unchanged_manifest)), + 1 => TestBinaryOriginResponse::new( + StatusCode::PROXY_AUTHENTICATION_REQUIRED, + Arc::<[u8]>::from(&b"retry"[..]), + ), + _ => TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&progressed_manifest)), + }; + } + TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&segment)) + })) + .await; + let input = ConfigInput { + id: 1, + name: Arc::from("recovery-regression-input"), + input_type: InputType::M3u, + url: origin.base_url.clone(), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = + test_app_state_with_hls_proxy_and_inputs(test_beast_hls_proxy(temp_dir.path()), vec![Arc::new(input)]); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let manifest_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 1_000) + .await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("recovery-before-cutover".to_string()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &lease_id).await; + let refresh = + regression_origin_refresh_request(&app_state, Arc::clone(&session), &manifest_url, Some(lease_id.clone())); + let fixture = RecoveryBeforeCutoverFixture { + _temp_dir: temp_dir, + origin, + origin_phase, + app_state, + session, + proxy_session_id, + lease_id, + refresh, + }; + assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); + wait_for_ready_timeline(&fixture.session, 6).await; + assert_initial_recovery_window(&fixture).await; + fixture +} + +pub(in crate::api::endpoints::hls_api::tests) async fn run_recovery_outage( + fixture: &mut RecoveryBeforeCutoverFixture, +) -> (u64, Option) { + let progress_generation = fixture.session.read().await.origin_control.progress_generation; + for _ in 0..2 { + fixture.refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); + } + assert_eq!(fixture.session.read().await.origin_control.progress_generation, progress_generation); + fixture.origin_phase.store(1, Ordering::SeqCst); + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + assert_eq!(fixture.app_state.hls.proxy.manifest_recovery_burst().level.plan(), plan); + fixture.refresh.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; + let requests_before = fixture.origin.manifest_request_count(); + let mut last_episode_generation = None; + for _ in 0..3 { + fixture.refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("407 exhaustion cannot remove the lease"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + let session = fixture.session.read().await; + let episode = session.origin_control.acceptance_episode.as_ref().expect("bounded acceptance evidence"); + assert!(episode.full_burst_completed); + assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); + last_episode_generation = Some(episode.generation.0); + } + assert!( + fixture.origin.manifest_request_count().saturating_sub(requests_before) + >= plan.total_candidates().saturating_mul(3) + ); + let uri = format!("/hls/shared/live/{}/{}/000003.ts", fixture.proxy_session_id.0, fixture.lease_id.0); + let cached = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(cached.status(), StatusCode::OK); + assert!(!response_body(cached).await.is_empty()); + (progress_generation, last_episode_generation) +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_recovery_after_outage( + fixture: &mut RecoveryBeforeCutoverFixture, + progress_generation: u64, + last_episode_generation: Option, +) { + fixture.origin_phase.store(2, Ordering::SeqCst); + fixture.refresh.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; + fixture.refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); + wait_for_ready_timeline(&fixture.session, 7).await; + { + let session = fixture.session.read().await; + assert_eq!(session.origin_seq_highwater, Some(106)); + assert_eq!(session.proxy_next_seq, Some(7)); + assert!(session.origin_control.progress_generation > progress_generation); + assert!(session.origin_control.acceptance_episode.is_none()); + assert!(last_episode_generation + .is_some_and(|generation| session.origin_control.acceptance_generation.0 > generation)); + } + let response = try_test_hls_cached_manifest_response( + &fixture.app_state, + &fixture.session, + &fixture.lease_id, + HlsAccessLeaseState::Activated, + &StripConfig { mode: HlsStripMode::Segments, value: 3 }, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("recovered normal manifest"); + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("recovered manifest utf8"); + assert!(body.contains("/000006.ts")); + assert!(!body.contains("/terminal/")); + assert!(!body.contains("#EXT-X-ENDLIST")); + let uri = format!("/hls/shared/live/{}/{}/000006.ts", fixture.proxy_session_id.0, fixture.lease_id.0); + let segment = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(segment.status(), StatusCode::OK); + assert!(!response_body(segment).await.is_empty()); + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("recovered lease remains stored"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); +} + +pub(in crate::api::endpoints::hls_api::tests) struct StaleOriginServers { + pub(in crate::api::endpoints::hls_api::tests) pinned: TestSegmentOrigin, + pub(in crate::api::endpoints::hls_api::tests) alternative: TestSegmentOrigin, + pub(in crate::api::endpoints::hls_api::tests) pinned_phase: Arc, + pub(in crate::api::endpoints::hls_api::tests) alternative_phase: Arc, + pub(in crate::api::endpoints::hls_api::tests) burst_candidates: Arc, + pub(in crate::api::endpoints::hls_api::tests) pinned_candidates: Arc, +} + +pub(in crate::api::endpoints::hls_api::tests) async fn spawn_stale_origin_servers() -> StaleOriginServers { + let pinned_manifest = Arc::<[u8]>::from(regression_origin_manifest(100, 6)); + let alternative_manifest = Arc::<[u8]>::from(regression_origin_manifest(200, 6)); + let continued_manifest = Arc::<[u8]>::from(regression_origin_manifest(201, 6)); + let segment = Arc::<[u8]>::from( + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + .as_slice(), + ); + let alternative_phase = Arc::new(AtomicUsize::new(0)); + let alternative_phase_for_handler = Arc::clone(&alternative_phase); + let alternative_segment = Arc::clone(&segment); + let alternative = spawn_test_binary_origin(Arc::new(move |path| { + if path_has_extension(path, "m3u8") { + let body = if alternative_phase_for_handler.load(Ordering::SeqCst) == 0 { + Arc::clone(&alternative_manifest) + } else { + Arc::clone(&continued_manifest) + }; + return TestBinaryOriginResponse::new(StatusCode::OK, body); + } + TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&alternative_segment)) + })) + .await; + let alternative_url = + format!("{}/live/user/pass/12345.m3u8", alternative.base_url.replacen("127.0.0.1", "localhost", 1)); + let pinned_phase = Arc::new(AtomicUsize::new(0)); + let burst_candidates = Arc::new(AtomicUsize::new(0)); + let pinned_candidates = Arc::new(AtomicUsize::new(0)); + let handler_phase = Arc::clone(&pinned_phase); + let handler_burst_candidates = Arc::clone(&burst_candidates); + let handler_pinned_candidates = Arc::clone(&pinned_candidates); + let handler_pinned_manifest = Arc::clone(&pinned_manifest); + let pinned_segment = Arc::clone(&segment); + let pinned = spawn_test_binary_origin(Arc::new(move |path| { + if !path_has_extension(path, "m3u8") { + return TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&pinned_segment)); + } + match handler_phase.load(Ordering::SeqCst) { + 0 => TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&handler_pinned_manifest)), + 1 => { + let index = handler_burst_candidates.fetch_add(1, Ordering::SeqCst); + if index.is_multiple_of(2) { + handler_pinned_candidates.fetch_add(1, Ordering::SeqCst); + TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&handler_pinned_manifest)) + } else { + TestBinaryOriginResponse::redirect(alternative_url.clone()) + } + } + _ => TestBinaryOriginResponse::redirect(alternative_url.clone()), + } + })) + .await; + StaleOriginServers { pinned, alternative, pinned_phase, alternative_phase, burst_candidates, pinned_candidates } +} + +pub(in crate::api::endpoints::hls_api::tests) struct StaleOriginFixture { + pub(in crate::api::endpoints::hls_api::tests) _temp_dir: tempfile::TempDir, + pub(in crate::api::endpoints::hls_api::tests) servers: StaleOriginServers, + pub(in crate::api::endpoints::hls_api::tests) app_state: Arc, + pub(in crate::api::endpoints::hls_api::tests) session: HlsSessionHandle, + pub(in crate::api::endpoints::hls_api::tests) proxy_session_id: ProxySessionId, + pub(in crate::api::endpoints::hls_api::tests) lease_id: HlsAccessLeaseId, + pub(in crate::api::endpoints::hls_api::tests) refresh: OriginRefreshRequest, + pub(in crate::api::endpoints::hls_api::tests) initial_visible_tail: u64, + pub(in crate::api::endpoints::hls_api::tests) initial_progress_generation: u64, + pub(in crate::api::endpoints::hls_api::tests) initial_origin_epoch: u64, + pub(in crate::api::endpoints::hls_api::tests) initial_progress_at_ms: Option, +} + +pub(in crate::api::endpoints::hls_api::tests) async fn stale_origin_fixture() -> StaleOriginFixture { + let temp_dir = tempfile::tempdir().expect("stale-origin cache tempdir"); + let servers = spawn_stale_origin_servers().await; + let input = ConfigInput { + id: 1, + name: Arc::from("reachable-stale-origin-input"), + input_type: InputType::M3u, + url: servers.pinned.base_url.clone(), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = + test_app_state_with_hls_proxy_and_inputs(test_beast_hls_proxy(temp_dir.path()), vec![Arc::new(input)]); + enable_hls_cache(&app_state); + create_active_hls_user_session(&app_state).await; + let manifest_url = format!("{}/live/user/pass/12345.m3u8", servers.pinned.base_url); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 1_000) + .await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("reachable-stale-origin".to_string()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &lease_id).await; + let refresh = + regression_origin_refresh_request(&app_state, Arc::clone(&session), &manifest_url, Some(lease_id.clone())); + assert!(trigger_origin_refresh_sync(refresh.clone()).await); + wait_for_ready_timeline(&session, 6).await; + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &lease_id, + HlsAccessLeaseState::Pending, + &StripConfig { mode: HlsStripMode::Segments, value: 3 }, + None, + super::HlsCachedManifestOptions::initial(Duration::from_secs(10)), + ) + .await + .expect("READY pinned-origin manifest"); + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0")); + assert_eq!(media_uri_count(&body), 3); + let now_ms = super::super::current_time_millis(); + assert!(app_state + .hls + .proxy + .activate_access_lease( + &lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 120_000, valid_window_ms: 180_000 }, + ) + .await + .is_activated()); + let lease = app_state + .hls + .proxy + .access_lease_response_snapshot(&lease_id, &proxy_session_id, now_ms) + .await + .expect("activated pinned-origin lease"); + let initial_visible_tail = + lease.last_manifest_snapshot.as_ref().expect("initial lease manifest snapshot").last_proxy_seq; + let (initial_progress_generation, initial_origin_epoch, initial_progress_at_ms) = { + let session = session.read().await; + assert_eq!(session.origin_seq_highwater, Some(105)); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some("127.0.0.1")); + ( + session.origin_control.progress_generation, + session.origin_epoch, + session.origin_control.last_media_progress_at_ms, + ) + }; + StaleOriginFixture { + _temp_dir: temp_dir, + servers, + app_state, + session, + proxy_session_id, + lease_id, + refresh, + initial_visible_tail, + initial_progress_generation, + initial_origin_epoch, + initial_progress_at_ms, + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn observe_stale_origin( + fixture: &mut StaleOriginFixture, +) -> HlsManifestAcceptanceDirective { + let requests_before = fixture.servers.pinned.manifest_request_count(); + for _ in 0..2 { + fixture.refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); + } + { + let session = fixture.session.read().await; + assert_eq!(session.origin_seq_highwater, Some(105)); + assert_eq!(session.origin_control.progress_generation, fixture.initial_progress_generation); + assert_eq!(session.origin_control.last_media_progress_at_ms, fixture.initial_progress_at_ms); + assert_eq!(session.origin_refresh.consecutive_failures, 0); + } + assert_eq!(fixture.servers.pinned.manifest_request_count().saturating_sub(requests_before), 2); + assert_eq!(fixture.servers.alternative.manifest_request_count(), 0); + let response = try_test_hls_cached_manifest_response( + &fixture.app_state, + &fixture.session, + &fixture.lease_id, + HlsAccessLeaseState::Activated, + &StripConfig { mode: HlsStripMode::Segments, value: 3 }, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("reachable stale origin keeps the committed live manifest"); + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("stale live manifest utf8"); + assert!(body.contains("#EXT-X-MEDIA-SEQUENCE:0")); + assert!(!body.contains("/terminal/")); + assert!(!body.contains("#EXT-X-ENDLIST")); + { + let mut session = fixture.session.write().await; + for segment in session.segments.values_mut().filter(|segment| segment.proxy_seq > fixture.initial_visible_tail) + { + segment.duration_ms = 10_000; + } + session.origin_control.last_media_progress_at_ms = Some(0); + session.advance_media_readiness_generation(); + } + let directive = match crate::api::model::hls_manifest_acceptance_directive_for_session( + &fixture.app_state.hls_ctx(), + &fixture.session, + &fixture.proxy_session_id, + ) + .await + { + HlsManifestAcceptanceEvaluationOutcome::Evaluated(directive) => directive, + other => panic!("stale progress evidence must evaluate: {other:?}"), + }; + assert!(directive.trigger.recovery_required()); + assert_eq!(fixture.session.read().await.origin_control.path_condition, HlsOriginPathCondition::PublicationLate); + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("stale progress evidence keeps the lease stored"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + directive +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_stale_origin_handoff( + fixture: &mut StaleOriginFixture, + directive: HlsManifestAcceptanceDirective, +) -> u64 { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + assert_eq!(fixture.app_state.hls.proxy.manifest_recovery_burst().level.plan(), plan); + fixture.servers.pinned_phase.store(1, Ordering::SeqCst); + let requests_before = fixture.servers.pinned.manifest_request_count(); + fixture.refresh.acceptance_directive = directive; + fixture.refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); + wait_for_ready_timeline(&fixture.session, 12).await; + assert_eq!( + fixture.servers.pinned.manifest_request_count().saturating_sub(requests_before), + plan.total_candidates() + ); + assert_eq!(fixture.servers.burst_candidates.load(Ordering::SeqCst), plan.total_candidates()); + assert!(fixture.servers.pinned_candidates.load(Ordering::SeqCst) > 0); + assert!(fixture.servers.alternative.manifest_request_count() >= 2); + let progress_generation = { + let session = fixture.session.read().await; + assert_eq!(session.origin_seq_highwater, Some(205)); + assert_eq!(session.proxy_next_seq, Some(12)); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some("localhost")); + assert_eq!(session.origin_epoch, fixture.initial_origin_epoch.saturating_add(1)); + assert!(session.origin_control.progress_generation > fixture.initial_progress_generation); + session.origin_control.progress_generation + }; + let response = try_test_hls_cached_manifest_response( + &fixture.app_state, + &fixture.session, + &fixture.lease_id, + HlsAccessLeaseState::Activated, + &StripConfig { mode: HlsStripMode::Segments, value: 3 }, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("cross-host recovery manifest"); + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("recovered manifest utf8"); + assert!(body.contains("/000006.ts")); + assert!(body.contains("#EXT-X-DISCONTINUITY")); + assert!(!body.contains("/terminal/")); + assert!(!body.contains("#EXT-X-ENDLIST")); + let uri = format!("/hls/shared/live/{}/{}/000006.ts", fixture.proxy_session_id.0, fixture.lease_id.0); + let segment = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(segment.status(), StatusCode::OK); + assert!(!response_body(segment).await.is_empty()); + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("recovered lease remains stored"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + progress_generation +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_stale_origin_continuation( + fixture: &mut StaleOriginFixture, + progress_generation: u64, +) { + fixture.servers.alternative_phase.store(1, Ordering::SeqCst); + fixture.servers.pinned_phase.store(2, Ordering::SeqCst); + fixture.refresh.acceptance_directive = HlsManifestAcceptanceDirective::none(); + fixture.refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + assert!(trigger_origin_refresh_sync(fixture.refresh.clone()).await); + wait_for_ready_timeline(&fixture.session, 13).await; + { + let session = fixture.session.read().await; + assert_eq!(session.origin_seq_highwater, Some(206)); + assert_eq!(session.proxy_next_seq, Some(13)); + assert_eq!(session.origin_epoch, fixture.initial_origin_epoch.saturating_add(1)); + assert!(session.origin_control.progress_generation > progress_generation); + } + let response = try_test_hls_cached_manifest_response( + &fixture.app_state, + &fixture.session, + &fixture.lease_id, + HlsAccessLeaseState::Activated, + &StripConfig { mode: HlsStripMode::Segments, value: 3 }, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("continued alternative-origin timeline"); + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("continued manifest utf8"); + assert!(body.contains("/000012.ts")); + assert!(!body.contains("/terminal/")); + assert!(!body.contains("#EXT-X-ENDLIST")); + let uri = format!("/hls/shared/live/{}/{}/000012.ts", fixture.proxy_session_id.0, fixture.lease_id.0); + let segment = get_response(Arc::clone(&fixture.app_state), &uri, None).await; + assert_eq!(segment.status(), StatusCode::OK); + assert!(!response_body(segment).await.is_empty()); +} + +pub(in crate::api::endpoints::hls_api::tests) struct PreparedTerminalCutoverFixture { + pub(in crate::api::endpoints::hls_api::tests) _temp_dir: tempfile::TempDir, + pub(in crate::api::endpoints::hls_api::tests) origin: TestSegmentOrigin, + pub(in crate::api::endpoints::hls_api::tests) app_state: Arc, + pub(in crate::api::endpoints::hls_api::tests) session: HlsSessionHandle, + pub(in crate::api::endpoints::hls_api::tests) proxy_session_id: ProxySessionId, + pub(in crate::api::endpoints::hls_api::tests) lease_id: HlsAccessLeaseId, + pub(in crate::api::endpoints::hls_api::tests) request_url: String, + pub(in crate::api::endpoints::hls_api::tests) base_manifest: HlsLeaseManifestSnapshot, + pub(in crate::api::endpoints::hls_api::tests) asset_buffer: TransportStreamBuffer, + pub(in crate::api::endpoints::hls_api::tests) asset: Arc, +} + +pub(in crate::api::endpoints::hls_api::tests) async fn prepare_terminal_cutover_bundle( + app_state: &Arc, + base_manifest: &HlsLeaseManifestSnapshot, +) -> (TransportStreamBuffer, Arc) { + let asset_buffer = app_state + .app_config + .custom_stream_response + .load_full() + .as_ref() + .and_then(|responses| responses.channel_unavailable.as_ref()) + .cloned() + .expect("configured terminal renderer"); + let asset = snapshot_terminal_media_asset(&asset_buffer).expect("terminal asset snapshot"); + let key = prepared_terminal_bundle_key(&asset, base_manifest.target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); + let state = app_state.hls.proxy.start_prepared_terminal_bundle( + Arc::clone(&asset), + base_manifest.target_duration_ms, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, + ); + let state = match state { + HlsPreparedTerminalBundleState::Preparing { .. } => app_state + .hls + .proxy + .wait_for_prepared_terminal_bundle(key) + .await + .expect("prepared terminal bundle completion"), + state => state, + }; + assert!(matches!( + state, + HlsPreparedTerminalBundleState::Ready { ref bundle } + if bundle.key == key + && bundle.segments.len() == usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT) + )); + assert_eq!(asset_buffer.finite_hls_render_count(), usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); + (asset_buffer, asset) +} + +pub(in crate::api::endpoints::hls_api::tests) async fn prepared_terminal_cutover_fixture( +) -> PreparedTerminalCutoverFixture { + let temp_dir = tempfile::tempdir().expect("terminal cutover tempdir"); + let origin = spawn_test_encrypted_hls_origin( + AES_TEST_MANIFEST, + Arc::from(AES_TEST_KEY_BYTES), + Arc::from(AES_TEST_PLAINTEXT_SEGMENT), + ) + .await; + let input = ConfigInput { + id: 1, + name: Arc::from("terminal-regression-input"), + input_type: InputType::M3u, + url: origin.base_url.clone(), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = + test_app_state_with_hls_proxy_and_inputs(test_beast_hls_proxy(temp_dir.path()), vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + enable_channel_unavailable_custom_response(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/channel/index.m3u8", origin.base_url); + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + let lease_id = HlsAccessLeaseId("prepared-terminal-cutover".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &lease_id).await; + let response = super::super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &lease_id, + HlsAccessLeaseState::Pending, + super::super::HlsCacheManifestOrigin { + raw_request_url: &request_url, + session_entry_url: super::super::HlsOriginEntryUrl::direct_http(&request_url), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/m3u-stream/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + .expect("AES shared session cold start"); + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("AES manifest utf8"); + assert!(body.contains("#EXT-X-KEY:METHOD=AES-128")); + let session = app_state.hls.proxy.sessions().get_by_key(&session_key).await.expect("AES shared session"); + wait_for_ready_timeline(&session, 6).await; + let now_ms = super::super::current_time_millis(); + assert!(app_state + .hls + .proxy + .activate_access_lease( + &lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 120_000, valid_window_ms: 180_000 }, + ) + .await + .is_activated()); + let lease = app_state + .hls + .proxy + .access_lease_response_snapshot(&lease_id, &proxy_session_id, now_ms) + .await + .expect("live AES lease"); + let base_manifest = lease.last_manifest_snapshot.as_ref().expect("frozen AES lease manifest").clone(); + assert_eq!(base_manifest.target_duration_ms, 12_000); + assert_eq!(base_manifest.visible_segments.len(), 3); + assert!(base_manifest.active_encryption.is_some()); + let (asset_buffer, asset) = prepare_terminal_cutover_bundle(&app_state, &base_manifest).await; + for segment in base_manifest.visible_segments.iter() { + let response = get_response(Arc::clone(&app_state), &segment.uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert!(!response_body(response).await.is_empty()); + } + PreparedTerminalCutoverFixture { + _temp_dir: temp_dir, + origin, + app_state, + session, + proxy_session_id, + lease_id, + request_url, + base_manifest, + asset_buffer, + asset, + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn apply_terminal_cutover_pressure( + fixture: &PreparedTerminalCutoverFixture, +) { + let mut session = fixture.session.write().await; + session.origin_control.path_condition = HlsOriginPathCondition::HardFetchFailure; + let transition_buffer_seq = fixture.base_manifest.last_proxy_seq.saturating_add(1); + let commit_guard_seq = transition_buffer_seq.saturating_add(1); + for segment in + session.segments.values_mut().filter(|segment| segment.proxy_seq > fixture.base_manifest.last_proxy_seq) + { + if segment.proxy_seq == transition_buffer_seq { + segment.duration_ms = 1_000; + } else if segment.proxy_seq == commit_guard_seq { + segment.duration_ms = 2_800; + } else { + segment.status = SegmentCacheStatus::Expired; + } + } + session.advance_media_readiness_generation(); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn terminal_cutover_acceptance_directive( + fixture: &PreparedTerminalCutoverFixture, +) -> HlsManifestAcceptanceDirective { + apply_terminal_cutover_pressure(fixture).await; + let directive = match crate::api::model::hls_manifest_acceptance_directive_for_session( + &fixture.app_state.hls_ctx(), + &fixture.session, + &fixture.proxy_session_id, + ) + .await + { + HlsManifestAcceptanceEvaluationOutcome::Evaluated(directive) => directive, + other => panic!("deterministic recovery-pressure snapshot must evaluate: {other:?}"), + }; + assert_eq!(directive.trigger, HlsManifestAcceptanceTrigger::RecoveryRequired); + let bundle_key = prepared_terminal_bundle_key( + &fixture.asset, + fixture.base_manifest.target_duration_ms, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, + ); + let timing = directive.timing_seed.expect("prepared acceptance timing seed"); + assert_eq!(timing.required_terminal_media_key, Some(bundle_key)); + assert_eq!(timing.terminal_media_preparation, HlsTerminalMediaPreparationState::Ready { key: bundle_key }); + directive +} + +pub(in crate::api::endpoints::hls_api::tests) async fn exhaust_terminal_cutover_recovery( + fixture: &PreparedTerminalCutoverFixture, + directive: HlsManifestAcceptanceDirective, +) -> HlsAccessLease { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let requests_before = fixture.origin.manifest_request_count(); + let mut refresh = regression_origin_refresh_request( + &fixture.app_state, + Arc::clone(&fixture.session), + &fixture.request_url, + Some(fixture.lease_id.clone()), + ); + refresh.acceptance_directive = directive; + refresh.now_ms = fixture.session.read().await.origin_refresh.next_fetch_allowed_at_ms; + assert!(trigger_origin_refresh_sync(refresh).await); + assert!(fixture.origin.manifest_request_count().saturating_sub(requests_before) >= plan.total_candidates()); + apply_terminal_cutover_pressure(fixture).await; + let bundle_key = prepared_terminal_bundle_key( + &fixture.asset, + fixture.base_manifest.target_duration_ms, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, + ); + { + let session = fixture.session.read().await; + let episode = session.origin_control.acceptance_episode.as_ref().expect("exhausted acceptance episode"); + assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); + assert!(episode.full_burst_completed); + assert_eq!(episode.timing().required_terminal_media_key, Some(bundle_key)); + assert_eq!( + episode.timing().terminal_media_preparation, + HlsTerminalMediaPreparationState::Ready { key: bundle_key } + ); + assert!(episode.exhaustion_reason().is_some()); + } + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("pressured live lease"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + assert_eq!( + lease.playback_cursor.highest_contiguous_completed_proxy_seq, + Some(fixture.base_manifest.last_proxy_seq) + ); + { + let session = fixture.session.read().await; + let ready_after_tail = session + .segments + .values() + .filter(|segment| segment.proxy_seq > fixture.base_manifest.last_proxy_seq) + .filter_map(|segment| { + matches!(segment.status, SegmentCacheStatus::Ready { .. }) + .then_some((segment.proxy_seq, segment.duration_ms)) + }) + .collect::>(); + assert_eq!( + ready_after_tail, + vec![ + (fixture.base_manifest.last_proxy_seq.saturating_add(1), 1_000), + (fixture.base_manifest.last_proxy_seq.saturating_add(2), 2_800), + ] + ); + assert!(session.origin_control.path_condition.is_degraded()); + } + lease +} + +pub(in crate::api::endpoints::hls_api::tests) async fn commit_prepared_terminal_cutover( + fixture: &PreparedTerminalCutoverFixture, + pressured_lease: &HlsAccessLease, +) -> (u64, u64) { + let cutover_now_ms = pressured_lease + .playback_cursor + .first_segment_completed_at_ms + .expect("measured lease playback start") + .saturating_add(fixture.base_manifest.playlist_duration_ms); + let first = commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.app_state.hls_ctx(), + &fixture.session, + &fixture.proxy_session_id, + pressured_lease, + cutover_now_ms, + ) + .await; + assert_eq!(first, HlsTerminalResolution::Committed); + let terminal_lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("terminal lease remains stored"); + let second = commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.app_state.hls_ctx(), + &fixture.session, + &fixture.proxy_session_id, + &terminal_lease, + cutover_now_ms.saturating_add(1), + ) + .await; + assert_eq!(second, HlsTerminalResolution::Committed); + let HlsLeasePlaybackMode::TerminalTail(plan) = terminal_lease.playback_mode else { + panic!("prepared terminal tail must commit"); + }; + assert_eq!(plan.segment_count, HLS_TERMINAL_TAIL_SEGMENT_COUNT); + (plan.generation.0, cutover_now_ms) +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_prepared_terminal_cutover_manifest( + fixture: &PreparedTerminalCutoverFixture, + generation: u64, +) { + let manifest_uri = format!("/hls/shared/live/{}/{}/manifest.m3u8", fixture.proxy_session_id.0, fixture.lease_id.0); + let response = get_response(Arc::clone(&fixture.app_state), &manifest_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("terminal manifest utf8"); + let live_tail_path = format!( + "/{}/{}/{:06}.ts", + fixture.proxy_session_id.0, fixture.lease_id.0, fixture.base_manifest.last_proxy_seq + ); + let terminal_prefix = format!("/{}/{}/terminal/{generation}/", fixture.proxy_session_id.0, fixture.lease_id.0); + assert!(body.contains(&live_tail_path)); + assert_eq!(body.matches("#EXT-X-DISCONTINUITY\n").count(), 1); + assert!(body.find(&live_tail_path) < body.find("#EXT-X-DISCONTINUITY\n")); + let key_reset = body.find("#EXT-X-KEY:METHOD=NONE\n").expect("AES-to-clear key reset"); + let discontinuity = body.find("#EXT-X-DISCONTINUITY\n").expect("terminal discontinuity"); + assert!(key_reset < discontinuity); + assert_eq!(body.matches(&terminal_prefix).count(), usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); + for index in 0..HLS_TERMINAL_TAIL_SEGMENT_COUNT { + assert!(body.contains(&format!("{terminal_prefix}{index}.ts"))); + } + let duration_ms = fixture.asset.duration_ms(); + let extinf = format!("#EXTINF:{}.{:03},", duration_ms / 1_000, duration_ms % 1_000); + assert_eq!(body.matches(&extinf).count(), usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); + assert!(body.ends_with("#EXT-X-ENDLIST\n")); + let renders_before = fixture.asset_buffer.finite_hls_render_count(); + let zero_uri = + format!("/hls/shared/live/{}/{}/terminal/{generation}/0.ts", fixture.proxy_session_id.0, fixture.lease_id.0); + let one_uri = + format!("/hls/shared/live/{}/{}/terminal/{generation}/1.ts", fixture.proxy_session_id.0, fixture.lease_id.0); + let zero = response_body(get_response(Arc::clone(&fixture.app_state), &zero_uri, None).await).await; + let one = response_body(get_response(Arc::clone(&fixture.app_state), &one_uri, None).await).await; + assert_ne!(zero, one); + assert_eq!(fixture.asset_buffer.finite_hls_render_count(), renders_before); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_terminal_cutover_sticky_after_recovery( + fixture: &PreparedTerminalCutoverFixture, + generation: u64, + cutover_now_ms: u64, +) { + { + let mut session = fixture.session.write().await; + let recovered_manifest = + normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:83\n#EXT-X-TARGETDURATION:12\n#EXTINF:12.0,\n83.ts\n"); + session.apply_origin_manifest(&recovered_manifest).expect("later shared-session recovery"); + session.origin_control.record_media_progress(cutover_now_ms.saturating_add(1), 12_000); + } + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("sticky terminal lease"); + assert!(matches!( + lease.playback_mode, + HlsLeasePlaybackMode::TerminalTail(ref plan) if plan.generation.0 == generation + )); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn prepare_other_live_lease( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + now_ms: u64, +) -> HlsAccessLeaseId { + let lease_id = HlsAccessLeaseId("other-live-lease".to_string()); + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), + proxy_session_id.clone(), + "hls-user".to_string(), + "hls-session-token".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + 60_000, + )) + .await; + lease_id +} + +pub(in crate::api::endpoints::hls_api::tests) fn assert_terminal_plan_unchanged( + playback_mode: &HlsLeasePlaybackMode, + expected_generation: HlsTerminalTailGeneration, + terminal_path: HlsTerminalSegmentPath, + expected_bytes: &bytes::Bytes, +) { + let HlsLeasePlaybackMode::TerminalTail(plan) = playback_mode else { + panic!("shared lease operation cannot reactivate the terminal lease"); + }; + assert_eq!(plan.generation, expected_generation); + assert_eq!(plan.segment_bytes(terminal_path).as_ref(), Some(expected_bytes)); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_conflicted_standalone_fallback( + app_state: &Arc, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, +) { + let response = + super::super::hls_unpublished_lease_channel_unavailable_response(app_state, proxy_session_id, lease_id).await; + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + assert!(!response.headers().contains_key(header::RETRY_AFTER)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("standalone manifest utf8"); + assert!(body.contains("#EXT-X-ENDLIST")); + assert!( + !body.contains("/hls/shared/live/"), + "standalone response must not expose a normal segment without a readiness path" + ); + assert_eq!( + session.read().await.origin_control.path_condition, + HlsOriginPathCondition::AcceptanceConflict, + "lease-local fallback must not relax deterministic conflict evidence" + ); +} + +pub(in crate::api::endpoints::hls_api::tests) fn test_beast_hls_proxy( + cache_path: &std::path::Path, +) -> Arc { + let config = HlsCacheConfig::from(&HlsCacheConfigDto { + cache_path: Some(cache_path.to_string_lossy().into_owned()), + strip: HlsStripConfigDto { mode: HlsStripMode::Segments, value: 3 }, + max_segments_prefetch: 6, + manifest_recovery_burst: HlsManifestRecoveryBurstConfigDto { level: HlsManifestRecoveryBurstLevel::Beast }, + ..HlsCacheConfigDto::default() + }); + Arc::new(HlsProxyManager::from_hls_cache_config(Some(&config))) +} + +pub(in crate::api::endpoints::hls_api::tests) struct PublicationLateFixture { + pub(in crate::api::endpoints::hls_api::tests) origin: TestSegmentOrigin, + pub(in crate::api::endpoints::hls_api::tests) origin_phase: Arc, + pub(in crate::api::endpoints::hls_api::tests) app_state: Arc, + pub(in crate::api::endpoints::hls_api::tests) session: HlsSessionHandle, + pub(in crate::api::endpoints::hls_api::tests) proxy_session_id: ProxySessionId, + pub(in crate::api::endpoints::hls_api::tests) access_lease_id: HlsAccessLeaseId, + pub(in crate::api::endpoints::hls_api::tests) media_playlist_uri: String, +} + +pub(in crate::api::endpoints::hls_api::tests) async fn publication_late_fixture() -> PublicationLateFixture { + let initial_manifest = Arc::<[u8]>::from(regression_origin_manifest(123, 6)); + let progressed_manifest = Arc::<[u8]>::from(regression_origin_manifest(124, 6)); + let segment = Arc::<[u8]>::from( + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + .as_slice(), + ); + let origin_phase = Arc::new(AtomicUsize::new(0)); + let origin_phase_for_handler = Arc::clone(&origin_phase); + let origin = spawn_test_binary_origin(Arc::new(move |path| { + if path_has_extension(path, "m3u8") { + return match origin_phase_for_handler.load(Ordering::SeqCst) { + 0 => TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&initial_manifest)), + 1 => TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&progressed_manifest)), + _ => TestBinaryOriginResponse::new( + StatusCode::PROXY_AUTHENTICATION_REQUIRED, + Arc::<[u8]>::from(&b"retry"[..]), + ), + }; + } + TestBinaryOriginResponse::new(StatusCode::OK, Arc::clone(&segment)) + })) + .await; + let input = ConfigInput { + id: 1, + name: Arc::from("publication-late-request-input"), + input_type: InputType::M3u, + url: origin.base_url.clone(), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let mut target = test_m3u_hls_share_target(); + target.name = "default".to_string(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + enable_channel_unavailable_custom_response(&app_state); + configure_default_test_server(&app_state); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + let origin_manifest_url = format!("{}/channel/index.m3u8", origin.base_url); + cache_test_m3u_hls_item(&app_state, &target, test_m3u_hls_item(&input, 12345, "channel-a", &origin_manifest_url)) + .await; + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.password = "hls-pass".to_string(); + let entry_path = super::super::build_virtual_hls_entry_path(&target, &input, &user, 12345); + let entry_response = super::super::handle_hls_stream_request( + &test_fingerprint(), + &app_state, + &user, + &target, + None, + None, + &origin_manifest_url, + None, + test_hls_entry_stream_context(12345, "channel-a", None), + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &entry_path, + super::super::HlsRequestStage::Entry, + ) + .await + .into_response(); + let (_, media_playlist_uri) = single_variant_master_playlist(entry_response).await; + let proxy_session_id = ProxySessionId(proxy_session_id_from_variant_uri(&media_playlist_uri).to_string()); + let access_lease_id = HlsAccessLeaseId(access_lease_id_from_variant_uri(&media_playlist_uri).to_string()); + let initial_media_response = get_response(Arc::clone(&app_state), &media_playlist_uri, None).await; + assert_eq!(initial_media_response.status(), StatusCode::OK); + let initial_media_body = + String::from_utf8(response_body(initial_media_response).await.to_vec()).expect("initial media utf8"); + let last_segment_uri = initial_media_body + .lines() + .rfind(|line| line.starts_with("/hls/shared/live/") && path_has_extension(line, "ts")) + .expect("initial manifest segment"); + assert_eq!(get_status(Arc::clone(&app_state), last_segment_uri).await, StatusCode::OK); + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&proxy_session_id) + .await + .expect("shared publication-late session"); + PublicationLateFixture { + origin, + origin_phase, + app_state, + session, + proxy_session_id, + access_lease_id, + media_playlist_uri, + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn refresh_publication_late_fixture( + fixture: &PublicationLateFixture, +) -> HlsAccessLease { + let progress_generation_before = fixture.session.read().await.origin_control.progress_generation; + { + let mut session = fixture.session.write().await; + session.origin_control.last_media_progress_at_ms = Some(0); + session.origin_refresh.next_fetch_allowed_at_ms = 0; + } + let requests_before = fixture.origin.manifest_request_count(); + fixture.origin_phase.store(1, Ordering::SeqCst); + let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("refreshed media utf8"); + assert!(!body.contains("#EXT-X-ENDLIST")); + assert!(!body.contains("/terminal/")); + assert!(fixture.origin.manifest_request_count() > requests_before); + { + let session = fixture.session.read().await; + assert!(session.origin_control.progress_generation > progress_generation_before); + assert_eq!(session.origin_seq_highwater, Some(129)); + } + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.access_lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("publication-late lease remains stored"); + assert_eq!(lease.state, HlsAccessLeaseState::Activated); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + assert_eq!(fixture.app_state.hls.proxy.terminal_pending().owner_count(), 0); + lease +} + +pub(in crate::api::endpoints::hls_api::tests) async fn prepare_publication_late_terminal_pressure( + fixture: &PublicationLateFixture, + base_manifest: &HlsLeaseManifestSnapshot, +) { + let terminal_response = fixture.app_state.app_config.custom_stream_response.load_full(); + let terminal_asset = snapshot_terminal_media_asset( + terminal_response + .as_ref() + .and_then(|response| response.channel_unavailable.as_ref()) + .expect("configured terminal asset"), + ) + .expect("compatible terminal asset"); + let bundle_key = prepared_terminal_bundle_key( + &terminal_asset, + base_manifest.target_duration_ms, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, + ); + let state = fixture.app_state.hls.proxy.start_prepared_terminal_bundle( + terminal_asset, + base_manifest.target_duration_ms, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, + ); + let state = match state { + HlsPreparedTerminalBundleState::Preparing { .. } => fixture + .app_state + .hls + .proxy + .wait_for_prepared_terminal_bundle(bundle_key) + .await + .expect("terminal bundle completion"), + state => state, + }; + assert!(matches!(state, HlsPreparedTerminalBundleState::Ready { .. })); + let mut session = fixture.session.write().await; + for segment in session.segments.values_mut().filter(|segment| segment.proxy_seq > base_manifest.last_proxy_seq) { + segment.duration_ms = 1; + } + session.origin_control.last_media_progress_at_ms = Some(0); + session.origin_refresh.next_fetch_allowed_at_ms = 0; +} + +pub(in crate::api::endpoints::hls_api::tests) async fn assert_publication_late_terminal_result( + fixture: &PublicationLateFixture, +) { + let requests_before = fixture.origin.manifest_request_count(); + fixture.origin_phase.store(2, Ordering::SeqCst); + let response = get_response(Arc::clone(&fixture.app_state), &fixture.media_playlist_uri, None).await; + let lease = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.access_lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("terminal lease remains stored"); + let recovery_plan = fixture.app_state.hls.proxy.manifest_recovery_burst().level.plan(); + assert!( + fixture.origin.manifest_request_count().saturating_sub(requests_before) >= recovery_plan.total_candidates() + ); + match lease.playback_mode { + HlsLeasePlaybackMode::TerminalTail(_) => { + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("terminal manifest utf8"); + assert!(body.contains("#EXT-X-ENDLIST")); + assert!(body.contains("/terminal/")); + } + HlsLeasePlaybackMode::TerminalUnavailable { .. } => { + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + } + HlsLeasePlaybackMode::Live => { + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("live manifest utf8"); + assert!(body.starts_with("#EXTM3U")); + assert!(!body.contains("#EXT-X-ENDLIST")); + } + HlsLeasePlaybackMode::Ended => { + panic!("active hard-failed lease must not become ended without terminal publication") + } + } +} + +pub(in crate::api::endpoints::hls_api::tests) fn regression_origin_refresh_request( + app_state: &Arc, + session: HlsSessionHandle, + manifest_url: &str, + access_lease_id: Option, +) -> OriginRefreshRequest { + OriginRefreshRequest { + app_config: Arc::clone(&app_state.app_config), + session, + origin_entry: LiveHlsOriginEntry::parse(manifest_url).expect("regression origin entry"), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: app_state.http_clients.default.load().as_ref().clone(), + no_redirect_client: app_state.http_clients.no_redirect.load().as_ref().clone(), + use_manual_redirects: false, + segment_cache: Arc::clone(app_state.hls.proxy.segment_cache()), + hls_proxy: Arc::clone(&app_state.hls.proxy), + segment_repair: Arc::clone(app_state.hls.proxy.segment_repair()), + segment_worker_pool: Arc::clone(app_state.hls.proxy.segment_worker_pool()), + map_worker_pool: Arc::clone(app_state.hls.proxy.map_worker_pool()), + origin_manifest_timeout_ms: app_state.hls.proxy.origin_manifest_timeout_ms(), + manifest_recovery_burst: app_state.hls.proxy.manifest_recovery_burst(), + strip: app_state.hls.proxy.strip(), + retry_policy: RetryPolicy { delays_ms: [0; 5], jitter_max_ms: 0 }, + reverse_proxy_rewrite_secret: app_state.get_encrypt_secret().to_vec(), + transient_resource_ttl_ms: app_state.hls.proxy.transient_resource_ttl_ms(), + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + fresh_manifest_requirement_generation: None, + acceptance_directive: HlsManifestAcceptanceDirective::none(), + access_lease_id, + disabled_headers: None, + now_ms: super::super::current_time_millis(), + origin_io: None, + post_refresh_runtime: None, + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn wait_for_ready_timeline( + session: &HlsSessionHandle, + expected_ready: usize, +) { + let wait = tokio::time::timeout(Duration::from_secs(10), async { + loop { + let ready = session + .read() + .await + .segments + .values() + .filter(|segment| matches!(segment.status, SegmentCacheStatus::Ready { .. })) + .count(); + if ready >= expected_ready { + break; + } + tokio::task::yield_now().await; + } + }) + .await; + if wait.is_err() { + let session = session.read().await; + let statuses = + session.segments.values().map(|segment| (segment.proxy_seq, segment.status.clone())).collect::>(); + panic!("READY timeline deadline: expected={expected_ready} statuses={statuses:?}"); + } +} + +pub(in crate::api::endpoints::hls_api::tests) async fn extend_ready_segment_as_sparse_file( + app_state: &Arc, + session: &HlsSessionHandle, + proxy_seq: u64, + logical_size: u64, +) { + let cache_key = session.read().await.segments.get(&proxy_seq).expect("mapped sparse segment").cache_key.clone(); + let metadata = app_state + .hls + .proxy + .segment_cache() + .metadata(&cache_key) + .await + .expect("sparse cache metadata read") + .expect("READY sparse cache object"); + let file = tokio::fs::OpenOptions::new().write(true).open(&metadata.path).await.expect("sparse cache object opens"); + file.set_len(logical_size).await.expect("sparse cache object extends"); + let mut session = session.write().await; + let segment = session.segments.get_mut(&proxy_seq).expect("sparse segment remains mapped"); + segment.status = + SegmentCacheStatus::Ready { content_length: logical_size, ready_at_ms: super::super::current_time_millis() }; + session.advance_media_readiness_generation(); + session.render_and_store_manifest(super::super::current_time_millis()).expect("sparse timeline renders"); +} + +pub(in crate::api::endpoints::hls_api::tests) async fn publish_test_manifest_and_exhaust_configured_acceptance( + app_state: &Arc, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + snapshot: HlsLeaseManifestSnapshot, + now_ms: u64, +) { + let target_duration_ms = snapshot.target_duration_ms; + let publication_guard = app_state + .hls + .proxy + .prepare_access_lease_manifest_publication(lease_id, proxy_session_id, now_ms) + .await + .expect("live lease accepts publication preparation"); + assert!(app_state + .hls + .proxy + .commit_access_lease_manifest_publication(lease_id, proxy_session_id, publication_guard, snapshot, now_ms,) + .await + .is_committed()); + + let terminal_response = app_state.app_config.custom_stream_response.load_full(); + let terminal_asset = terminal_response + .as_ref() + .and_then(|responses| responses.channel_unavailable.as_ref()) + .and_then(|buffer| snapshot_terminal_media_asset(buffer).ok()) + .expect("configured terminal test asset"); + let terminal_key = + prepared_terminal_bundle_key(&terminal_asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); + let state = app_state.hls.proxy.start_prepared_terminal_bundle( + terminal_asset, + target_duration_ms, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, + ); + let state = match state { + HlsPreparedTerminalBundleState::Preparing { .. } => app_state + .hls + .proxy + .wait_for_prepared_terminal_bundle(terminal_key) + .await + .expect("terminal bundle completion"), + state => state, + }; + assert!(matches!( + state, + HlsPreparedTerminalBundleState::Ready { ref bundle } if bundle.key == terminal_key + )); + + let session = + app_state.hls.proxy.sessions().get_by_proxy_session_id(proxy_session_id).await.expect("warm shared session"); + let mut session = session.write().await; + session.origin_control.record_media_progress(now_ms, target_duration_ms); + let burst_plan = app_state.hls.proxy.manifest_recovery_burst().level.plan(); + let operation_timeout = HlsOperationTimeoutMs::from_millis(app_state.hls.proxy.origin_manifest_timeout_ms()); + let expected_eta = HlsRecoveryEtaMs::from_millis(app_state.hls.proxy.origin_manifest_timeout_ms()); + let timing = HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput { + started_at_ms: now_ms, + burst_plan, + target_duration_ms, + transition_margin: HlsTransitionMarginMs::from_millis(target_duration_ms), + workload: HlsRecoveryWorkload::clear_fetch(), + observed_latency: HlsObservedRecoveryLatency::default(), + required_terminal_media_key: Some(terminal_key), + terminal_media_preparation: HlsTerminalMediaPreparationState::Ready { key: terminal_key }, + policy: HlsRecoveryTimingPolicy::new(operation_timeout, operation_timeout, expected_eta, expected_eta), + }); + session.origin_control.begin_acceptance_episode( + now_ms, + burst_plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &timing, + ); + session.origin_control.path_condition = HlsOriginPathCondition::HardFetchFailure; + let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); + episode.record_full_burst(); + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::AllFailed); + episode.hold_after_uncommitted_burst(None, None); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_recovery_before_cutover.rs b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_recovery_before_cutover.rs new file mode 100644 index 000000000..bb8c07120 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_recovery_before_cutover.rs @@ -0,0 +1,19 @@ +use super::terminal_tail::{ + assert_recovery_after_outage, assert_stale_origin_continuation, assert_stale_origin_handoff, observe_stale_origin, + recovery_before_cutover_fixture, run_recovery_outage, stale_origin_fixture, +}; + +#[tokio::test] +async fn recovers_before_lease_cutover_without_terminal_tail() { + let mut fixture = recovery_before_cutover_fixture().await; + let (progress_generation, last_episode_generation) = run_recovery_outage(&mut fixture).await; + assert_recovery_after_outage(&mut fixture, progress_generation, last_episode_generation).await; +} + +#[tokio::test] +async fn reachable_stale_origin_hands_off_to_progressed_origin_without_terminal_tail() { + let mut fixture = stale_origin_fixture().await; + let directive = observe_stale_origin(&mut fixture).await; + let progress_generation = assert_stale_origin_handoff(&mut fixture, directive).await; + assert_stale_origin_continuation(&mut fixture, progress_generation).await; +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs new file mode 100644 index 000000000..2d7657594 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_cutover.rs @@ -0,0 +1,199 @@ +use super::{ + enable_channel_unavailable_custom_response, get_response, map_hls_map, map_ready_segment, response_body, + terminal_tail::{ + assert_prepared_terminal_cutover_manifest, assert_terminal_cutover_sticky_after_recovery, + commit_prepared_terminal_cutover, exhaust_terminal_cutover_recovery, prepared_terminal_cutover_fixture, + publish_test_manifest_and_exhaust_configured_acceptance, terminal_cutover_acceptance_directive, + }, + terminal_test_asset, terminal_test_plan_shape, terminalize_existing_test_lease, test_app_state_with_hls_proxy, + test_hls_access_context, +}; +use crate::api::model::{ + HlsAccessLeaseId, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, + HlsManifestCommitIdentity, HlsManifestDeliveryMode, HlsMapSignature, HlsMediaContainer, HlsProxyManager, + ProxySessionId, +}; +use axum::http::{header, StatusCode}; +use std::sync::Arc; + +#[tokio::test] +async fn terminal_lease_manifest_is_inline_immutable_endlist_on_canonical_path() { + const LIVE_TAIL_BYTES: &[u8] = b"original-live-tail"; + let temp_dir = tempfile::tempdir().expect("tempdir"); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", LIVE_TAIL_BYTES).await; + let lease_id = format!("test-access-lease-{proxy_session_id}"); + terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id, 123).await; + let cursor_before = app_state + .hls + .proxy + .access_lease_response_snapshot( + &HlsAccessLeaseId(lease_id.clone()), + &ProxySessionId(proxy_session_id.clone()), + super::super::current_time_millis(), + ) + .await + .expect("terminal lease before media read") + .playback_cursor; + let (generation, segment_count) = terminal_test_plan_shape(&app_state, &proxy_session_id, &lease_id).await; + let manifest_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/manifest.m3u8"); + let mut reloaded_api_proxy = + app_state.app_config.api_proxy.load_full().as_deref().cloned().expect("test API proxy config"); + reloaded_api_proxy.server[0].path = Some("reloaded".to_string()); + app_state.app_config.api_proxy.store(Some(Arc::new(reloaded_api_proxy))); + + let response = get_response(Arc::clone(&app_state), &manifest_uri, None).await; + + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("terminal manifest is utf8"); + let repeated = get_response(Arc::clone(&app_state), &manifest_uri, None).await; + assert_eq!(repeated.status(), StatusCode::OK); + assert!(!repeated.headers().contains_key(header::LOCATION)); + assert_eq!(response_body(repeated).await, body.as_bytes()); + let live_tail = format!("/{proxy_session_id}/{lease_id}/000123.ts"); + let terminal_prefix = format!("/{proxy_session_id}/{lease_id}/terminal/{generation}/"); + assert!(body.contains(&live_tail)); + assert!(body.contains("/iptv/hls/shared/live/")); + assert!(!body.contains("/reloaded/hls/shared/live/")); + assert_eq!(body.matches(&terminal_prefix).count(), usize::from(segment_count)); + assert_eq!(body.matches("#EXT-X-DISCONTINUITY\n").count(), 1); + assert!(body.ends_with("#EXT-X-ENDLIST\n")); + assert!(body.find(&live_tail) < body.find("#EXT-X-DISCONTINUITY\n")); + assert!(body.find("#EXT-X-DISCONTINUITY\n") < body.find(&terminal_prefix)); + + let live_tail_uri = format!("/hls/shared/live/{proxy_session_id}/{lease_id}/000123.ts"); + let live_tail_response = get_response(Arc::clone(&app_state), &live_tail_uri, Some("bytes=0-")).await; + assert_eq!(live_tail_response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(live_tail_response.headers()[header::CONTENT_LENGTH], LIVE_TAIL_BYTES.len().to_string()); + assert_eq!( + live_tail_response.headers()[header::CONTENT_RANGE], + format!("bytes 0-{}/{}", LIVE_TAIL_BYTES.len() - 1, LIVE_TAIL_BYTES.len()) + ); + assert_eq!(response_body(live_tail_response).await, bytes::Bytes::from_static(LIVE_TAIL_BYTES)); + let cursor_after = app_state + .hls + .proxy + .access_lease_response_snapshot( + &HlsAccessLeaseId(lease_id), + &ProxySessionId(proxy_session_id), + super::super::current_time_millis(), + ) + .await + .expect("terminal lease after media read") + .playback_cursor; + assert_eq!(cursor_after, cursor_before); +} + +#[tokio::test] +async fn commits_prepared_terminal_tail_once_when_recovery_misses_deadline() { + let fixture = prepared_terminal_cutover_fixture().await; + let directive = terminal_cutover_acceptance_directive(&fixture).await; + let pressured_lease = exhaust_terminal_cutover_recovery(&fixture, directive).await; + let (generation, cutover_now_ms) = commit_prepared_terminal_cutover(&fixture, &pressured_lease).await; + assert_prepared_terminal_cutover_manifest(&fixture, generation).await; + assert_terminal_cutover_sticky_after_recovery(&fixture, generation, cutover_now_ms).await; +} + +#[tokio::test] +async fn warm_fmp4_map_cutover_without_ready_reserve_fails_closed_without_a_ts_splice() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); + enable_channel_unavailable_custom_response(&app_state); + let proxy_session_id = ProxySessionId(map_hls_map(&app_state, b"fmp4-init", true).await); + let lease_id = HlsAccessLeaseId(format!("test-access-lease-{}", proxy_session_id.0)); + let now_ms = super::super::current_time_millis(); + let (base_proxy_seq, duration_ms) = { + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&proxy_session_id) + .await + .expect("fMP4 shared session"); + let session = session.read().await; + let entry = session.segments.values().next().expect("fMP4 media entry"); + (entry.proxy_seq, entry.duration_ms) + }; + let snapshot = HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(now_ms), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 0, + delivered_at_ms: now_ms, + first_proxy_seq: base_proxy_seq, + last_proxy_seq: base_proxy_seq, + visible_segments: Arc::from([HlsLeaseManifestSegment { + proxy_seq: base_proxy_seq, + duration_ms, + uri: format!("/iptv/hls/shared/live/{}/{}/{base_proxy_seq:06}.m4s", proxy_session_id.0, lease_id.0).into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }]), + discontinuity_sequence: 0, + target_duration_ms: terminal_test_asset().duration_ms().saturating_add(1_000), + playlist_duration_ms: duration_ms, + last_visible_media_end_ms: duration_ms, + active_map: Some(HlsMapSignature { fingerprint: [7; 32], container: HlsMediaContainer::FragmentedMp4 }), + active_encryption: None, + container: HlsMediaContainer::FragmentedMp4, + }; + publish_test_manifest_and_exhaust_configured_acceptance(&app_state, &proxy_session_id, &lease_id, snapshot, now_ms) + .await; + let manifest_uri = format!("/hls/shared/live/{}/{}/manifest.m3u8", proxy_session_id.0, lease_id.0); + + let response = get_response(Arc::clone(&app_state), &manifest_uri, None).await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(!response.headers().contains_key(header::LOCATION)); + assert!(response_body(response).await.is_empty()); + let lease = app_state + .hls + .proxy + .access_lease_response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis()) + .await + .expect("failed-closed lease snapshot"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + let terminal_uri = format!("/hls/shared/live/{}/{}/terminal/1/0.ts", proxy_session_id.0, lease_id.0); + let terminal_response = get_response(app_state, &terminal_uri, None).await; + assert_eq!(terminal_response.status(), StatusCode::NOT_FOUND); + assert!(!terminal_response.headers().contains_key(header::LOCATION)); + assert!(response_body(terminal_response).await.is_empty()); +} + +#[tokio::test] +async fn delayed_live_resource_completion_revalidates_after_terminal_cutover_without_a_sleep() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"ready-before-cutover").await; + let lease_id = HlsAccessLeaseId(format!("test-access-lease-{proxy_session_id}")); + let proxy_session_key = ProxySessionId(proxy_session_id.clone()); + let access_context = test_hls_access_context(proxy_session_key.clone(), lease_id.clone()); + let live_identity = app_state + .hls + .proxy + .access_lease_response_snapshot(&lease_id, &proxy_session_key, super::super::current_time_millis()) + .await + .and_then(|lease| lease.media_identity()) + .expect("live lease identity"); + let (release_sender, release_receiver) = tokio::sync::oneshot::channel(); + let app_state_for_completion = Arc::clone(&app_state); + let access_context_for_completion = access_context.clone(); + let completion = tokio::spawn(async move { + let _ = release_receiver.await; + super::super::hls_live_lease_identity_is_current( + &app_state_for_completion, + &access_context_for_completion, + live_identity, + ) + .await + }); + + terminalize_existing_test_lease(&app_state, &proxy_session_id, &lease_id.0, 123).await; + release_sender.send(()).expect("release delayed completion after cutover"); + + assert!(!completion.await.expect("controlled completion task")); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_fallback.rs b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_fallback.rs new file mode 100644 index 000000000..d90672698 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_fallback.rs @@ -0,0 +1,336 @@ +use super::{ + create_active_hls_user_session, enable_channel_unavailable_custom_response, enable_hls_cache, get_response, + map_ready_segment, mark_hls_user_session_exhausted, normal_manifest, prepare_pending_test_hls_access_lease, + response_body, runtime_policy_endpoint_fixture, spawn_test_status_origin, + terminal_tail::{ + assert_conflicted_standalone_fallback, assert_publication_late_terminal_result, assert_terminal_plan_unchanged, + prepare_other_live_lease, prepare_publication_late_terminal_pressure, publication_late_fixture, + refresh_publication_late_fixture, terminal_generation_for_lease, + }, + terminalize_existing_test_lease, test_app_state_with_hls_proxy, test_app_state_with_inputs, test_fingerprint, + test_hls_access_context, HlsCanonicalOwnerRegistrationKind, +}; +use crate::{ + api::model::{ + build_proxy_session_id, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, HlsAccessLeaseTiming, + HlsAvailabilityReevaluationFinishReason, HlsAvailabilityReevaluationMode, + HlsAvailabilityReevaluationRegistration, HlsLeasePlaybackMode, HlsManifestCommitRequirement, + HlsOriginPathCondition, HlsPlaybackFamilyKey, HlsProxyManager, HlsRuntimeCustomTailReason, + HlsSessionStoreOutcome, HlsTerminalSegmentPath, ProxySessionId, + }, + model::{ConfigInput, StripConfig}, +}; +use axum::http::{header, HeaderMap, StatusCode}; +use shared::model::{HlsStripMode, InputType}; +use std::sync::Arc; + +#[tokio::test] +async fn completed_owner_resolves_new_lease_standalone_without_reusing_old_terminal() { + let temp_dir = tempfile::tempdir().expect("owner handoff cache tempdir"); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); + enable_channel_unavailable_custom_response(&app_state); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"old-terminal-base").await; + let old_lease_id = HlsAccessLeaseId(format!("test-access-lease-{proxy_session_id}")); + terminalize_existing_test_lease(&app_state, &proxy_session_id, &old_lease_id.0, 123).await; + let proxy_session_id = ProxySessionId(proxy_session_id); + let now_ms = super::super::current_time_millis(); + let old_generation = terminal_generation_for_lease(&app_state, &proxy_session_id, &old_lease_id).await; + let new_lease_id = HlsAccessLeaseId("new-owner-handoff-lease".to_string()); + app_state + .hls + .proxy + .prepare_access_lease(HlsAccessLease::pending( + new_lease_id.clone(), + HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), + proxy_session_id.clone(), + "hls-user".to_string(), + "new-owner-handoff-session".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + 60_000, + )) + .await; + let session = + app_state.hls.proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.expect("shared session"); + let owner_key = + app_state.hls.proxy.availability_reevaluation_owner_key(&session, &proxy_session_id).await.expect("owner key"); + let release = Arc::new(tokio::sync::Notify::new()); + let task_release = Arc::clone(&release); + let task_owner_key = owner_key.clone(); + let coordinator = app_state.hls.proxy.availability_reevaluations(); + assert_eq!( + coordinator.register( + owner_key, + HlsAvailabilityReevaluationMode::RecoveryPressure, + move |ownership| async move { + task_release.notified().await; + let _ = ownership.finish_cycle(&task_owner_key, HlsAvailabilityReevaluationFinishReason::Evaluated); + }, + ), + HlsAvailabilityReevaluationRegistration::Scheduled + ); + let safe_session = { + let session = session.read().await; + super::super::safe_session_key(&session.key) + }; + let strip = StripConfig { mode: HlsStripMode::Segments, value: 0 }; + let mut response = Box::pin(super::super::join_hls_canonical_manifest_owner( + super::super::HlsCanonicalOwnerHandoffContext { + app_state: &app_state, + proxy_session_id: &proxy_session_id, + access_lease_id: &new_lease_id, + expected_lease_issued_at_ms: Some(now_ms), + strip: &strip, + server_path: None, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + manifest_boundary_rendered_at_ms: 0, + bandwidth_learning: super::super::HlsRuntimeBandwidthLearningContext::Disabled, + request_deadline_ms: now_ms.saturating_add(60_000), + safe_session, + }, + HlsCanonicalOwnerRegistrationKind::Scheduled, + )); + assert!(matches!(futures::poll!(response.as_mut()), std::task::Poll::Pending)); + + release.notify_one(); + let response = response.await; + + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::LOCATION)); + assert!(!response.headers().contains_key(header::RETRY_AFTER)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("standalone manifest utf8"); + assert!(body.contains("#EXT-X-ENDLIST")); + assert!(!body.contains("/hls/shared/live/")); + assert_eq!( + terminal_generation_for_lease(&app_state, &proxy_session_id, &old_lease_id).await, + old_generation, + "new lease fallback cannot reactivate old terminal lease" + ); +} + +#[tokio::test] +async fn cold_user_denial_uses_standalone_finite_response() { + let fixture = runtime_policy_endpoint_fixture(false).await; + mark_hls_user_session_exhausted(&fixture.app_state).await; + + let denial = get_response(Arc::clone(&fixture.app_state), &fixture.live_segment_uri, None).await; + assert_eq!(denial.status(), StatusCode::FORBIDDEN); + let denied = fixture + .app_state + .hls + .proxy + .access_lease_response_snapshot( + &fixture.lease_id, + &fixture.proxy_session_id, + super::super::current_time_millis(), + ) + .await + .expect("cold denied lease"); + assert_eq!(denied.state, HlsAccessLeaseState::Denied); + assert_eq!(denied.playback_mode, HlsLeasePlaybackMode::Ended); + assert!(denied.runtime_policy_revocation.is_none()); + assert_eq!(denied.runtime_policy_denial_reason(), Some(HlsRuntimeCustomTailReason::UserConnectionsExhausted)); + + let response = get_response(Arc::clone(&fixture.app_state), &fixture.manifest_uri, None).await; + assert_eq!(response.status(), StatusCode::OK); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("standalone policy manifest utf8"); + assert!(body.contains(&format!("/cvs/hls/{}/", fixture.lease_id.0))); + assert!(!body.contains("/hls-user/")); + assert!(!body.contains("/hls-pass/")); + assert!(!body.contains("/user_connections_exhausted/")); + assert!( + !body.contains(&format!("/hls/shared/live/{}/{}/terminal/", fixture.proxy_session_id.0, fixture.lease_id.0)) + ); + assert!(body.ends_with("#EXT-X-ENDLIST\n")); +} + +#[tokio::test] +async fn reused_conflicted_session_standalone_fallback_preserves_terminal_lease_during_recovery() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let app_state = test_app_state_with_hls_proxy(Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300))); + enable_channel_unavailable_custom_response(&app_state); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"original-live-tail").await; + let terminal_lease_id = format!("test-access-lease-{proxy_session_id}"); + terminalize_existing_test_lease(&app_state, &proxy_session_id, &terminal_lease_id, 123).await; + let proxy_session_key = ProxySessionId(proxy_session_id.clone()); + let now_ms = super::super::current_time_millis(); + let terminal_before = app_state + .hls + .proxy + .access_lease_response_snapshot(&HlsAccessLeaseId(terminal_lease_id.clone()), &proxy_session_key, now_ms) + .await + .expect("terminal lease exists before the other lease"); + let HlsLeasePlaybackMode::TerminalTail(terminal_plan_before) = terminal_before.playback_mode else { + panic!("original lease is terminal"); + }; + let terminal_path = HlsTerminalSegmentPath { generation: terminal_plan_before.generation, index: 0 }; + let terminal_bytes_before = + terminal_plan_before.segment_bytes(terminal_path).expect("terminal segment zero is immutable"); + let other_lease_id = prepare_other_live_lease(&app_state, &proxy_session_key, now_ms).await; + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&proxy_session_key) + .await + .expect("shared session exists"); + session.write().await.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict; + + assert_conflicted_standalone_fallback(&app_state, &session, &proxy_session_key, &other_lease_id).await; + + let terminal_after_fallback = app_state + .hls + .proxy + .access_lease_response_snapshot( + &HlsAccessLeaseId(terminal_lease_id.clone()), + &proxy_session_key, + now_ms.saturating_add(1), + ) + .await + .expect("terminal lease remains stored after standalone fallback"); + assert_terminal_plan_unchanged( + &terminal_after_fallback.playback_mode, + terminal_plan_before.generation, + terminal_path, + &terminal_bytes_before, + ); + + assert!(app_state + .hls + .proxy + .activate_access_lease( + &other_lease_id, + &proxy_session_key, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 5_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + + let recovered_manifest = + normal_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:124\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n124.ts\n"); + { + let mut session = session.write().await; + session.apply_origin_manifest(&recovered_manifest).expect("recovery manifest commits"); + session.origin_control.record_media_progress(now_ms.saturating_add(1), 4_000); + } + + let terminal = app_state + .hls + .proxy + .access_lease_response_snapshot( + &HlsAccessLeaseId(terminal_lease_id), + &proxy_session_key, + now_ms.saturating_add(2), + ) + .await + .expect("terminal lease remains stored"); + let other = app_state + .hls + .proxy + .access_lease_response_snapshot(&other_lease_id, &proxy_session_key, now_ms.saturating_add(2)) + .await + .expect("other live lease remains stored"); + assert_terminal_plan_unchanged( + &terminal.playback_mode, + terminal_plan_before.generation, + terminal_path, + &terminal_bytes_before, + ); + assert_eq!(other.playback_mode, HlsLeasePlaybackMode::Live); +} + +#[tokio::test] +async fn hls_cache_manifest_unpublished_lease_uses_same_finite_fallback_for_created_and_reused_session() { + let mut rendered_bodies = Vec::new(); + for expected_outcome in [HlsSessionStoreOutcome::Created, HlsSessionStoreOutcome::Reused] { + let input_name = Arc::::from("test-input"); + let origin = spawn_test_status_origin(StatusCode::NOT_FOUND, b"missing").await; + let input = ConfigInput { + id: 1, + name: Arc::clone(&input_name), + input_type: InputType::Xtream, + url: origin.base_url.clone(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + max_connections: 1, + enabled: true, + ..ConfigInput::default() + }; + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + enable_hls_cache(&app_state); + enable_channel_unavailable_custom_response(&app_state); + create_active_hls_user_session(&app_state).await; + let request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); + let origin_source = super::super::build_hls_origin_source(&input, "12345"); + let session_key = origin_source.session_key(); + let proxy_session_id = build_proxy_session_id(&session_key, &app_state.get_encrypt_secret()); + if expected_outcome == HlsSessionStoreOutcome::Reused { + let (session, outcome) = app_state + .hls + .proxy + .get_or_create_session_with_source_and_outcome( + session_key, + origin_source.clone(), + &app_state.get_encrypt_secret(), + super::super::current_time_millis(), + ) + .await; + assert_eq!(outcome, HlsSessionStoreOutcome::Created); + session.write().await.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict; + } + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let access_context = test_hls_access_context(proxy_session_id.clone(), access_lease_id.clone()); + prepare_pending_test_hls_access_lease(&app_state, &proxy_session_id, &access_lease_id).await; + + let response = super::super::try_hls_cache_canonical_manifest_response( + &app_state, + &test_fingerprint(), + &access_context, + &proxy_session_id, + &access_lease_id, + HlsAccessLeaseState::Pending, + super::super::HlsCacheManifestOrigin { + raw_request_url: request_url.as_str(), + session_entry_url: super::super::HlsOriginEntryUrl::direct_http(request_url.as_str()), + input: &input, + origin_source, + }, + HeaderMap::new(), + None, + "/live/hls-user/hls-pass/12345.m3u8", + super::super::HlsManifestRefreshOrdering::Background, + ) + .await + .expect("hls cache should handle valid live hls entrypoint"); + + assert_eq!(response.status(), StatusCode::OK, "session outcome: {expected_outcome:?}"); + assert!(response.headers().get(header::RETRY_AFTER).is_none(), "custom response must not expose retry-after"); + assert!(!response.headers().contains_key(header::LOCATION)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + assert!(body.contains("#EXT-X-ENDLIST")); + assert!(!body.contains("/hls/shared/live/"), "standalone response must not expose an unready normal URI"); + rendered_bodies.push(body); + + let snapshot = app_state + .hls + .proxy + .access_lease_response_snapshot(&access_lease_id, &proxy_session_id, super::super::current_time_millis()) + .await + .expect("lease remains available for strict cold-start handling"); + assert_eq!(snapshot.playback_mode, HlsLeasePlaybackMode::Live); + assert!(snapshot.last_manifest_snapshot.is_none()); + } + assert_eq!(rendered_bodies[0], rendered_bodies[1]); +} + +#[tokio::test] +async fn publication_late_live_manifest_request_refreshes_before_terminal_evaluation() { + let fixture = publication_late_fixture().await; + let lease = refresh_publication_late_fixture(&fixture).await; + let base_manifest = lease.last_manifest_snapshot.as_ref().expect("live manifest snapshot"); + prepare_publication_late_terminal_pressure(&fixture, base_manifest).await; + assert_publication_late_terminal_result(&fixture).await; +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_preflight.rs b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_preflight.rs new file mode 100644 index 000000000..b7305f90a --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/terminal_tail_terminal_preflight.rs @@ -0,0 +1,226 @@ +use super::{ + get_response, hls_manifest_terminal_preflight, hls_proxy_uri, hls_temporary_resource_unavailable_response, + hls_terminal_endpoint_action, hls_terminal_failed_closed_response, map_ready_segment, test_app_state, + test_app_state_with_hls_proxy, HlsManifestTerminalPreflight, HlsTerminalEndpointAction, +}; +use crate::{ + api::model::{ + HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, + HlsLeasePlaybackMode, HlsManifestCommitIdentity, HlsManifestDeliveryMode, HlsMediaContainer, + HlsPlaybackFamilyKey, HlsProxyManager, HlsSessionKey, HlsTerminalFailedClosedReason, HlsTerminalResolution, + SegmentCacheStatus, SegmentFetchPriority, + }, + processing::parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}, +}; +use axum::http::{header, StatusCode}; +use std::sync::Arc; + +#[test] +fn hls_terminal_commit_endpoint_resolution_mapping_is_exhaustive() { + assert_eq!(hls_terminal_endpoint_action(HlsTerminalResolution::LiveAllowed), HlsTerminalEndpointAction::ServeLive); + assert_eq!( + hls_terminal_endpoint_action(HlsTerminalResolution::Committed), + HlsTerminalEndpointAction::ReloadTerminal + ); + assert_eq!(hls_terminal_endpoint_action(HlsTerminalResolution::Reevaluate), HlsTerminalEndpointAction::Reevaluate); + assert_eq!( + hls_terminal_endpoint_action(HlsTerminalResolution::Pending { retry_after_ms: 250 }), + HlsTerminalEndpointAction::RetryAfter { retry_after_ms: 250 } + ); + + for reason in [ + HlsTerminalFailedClosedReason::LeaseStateUnavailable, + HlsTerminalFailedClosedReason::BundleNotReadyWithoutOwner, + HlsTerminalFailedClosedReason::BundleIncompatible, + HlsTerminalFailedClosedReason::SafeCommitDeadlineElapsed, + HlsTerminalFailedClosedReason::RetryCapacityExceeded, + HlsTerminalFailedClosedReason::RetryAttemptsExhausted, + HlsTerminalFailedClosedReason::RuntimeUnavailable, + ] { + assert_eq!( + hls_terminal_endpoint_action(HlsTerminalResolution::FailedClosed { reason }), + HlsTerminalEndpointAction::FailClosed { reason } + ); + } +} + +#[tokio::test] +async fn hls_manifest_terminal_preflight_distinguishes_bootstrap_refresh_and_invalid_missing_snapshot() { + let app_state = test_app_state(); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "preflight-stream"), &app_state.get_encrypt_secret(), 1_000) + .await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let mut lease = HlsAccessLease::pending( + HlsAccessLeaseId("preflight-lease".to_string()), + HlsPlaybackFamilyKey::new("hls-user", "preflight-client"), + proxy_session_id, + "hls-user".to_string(), + "preflight-user-session".to_string(), + 1, + "preflight-stream".to_string(), + 12345, + 1_000, + 120_000, + ); + + assert_eq!( + hls_manifest_terminal_preflight(&session, &lease, 2_000).await, + HlsManifestTerminalPreflight::BootstrapPendingLease + ); + + lease.state = HlsAccessLeaseState::Activated; + assert_eq!( + hls_manifest_terminal_preflight(&session, &lease, 2_000).await, + HlsManifestTerminalPreflight::FailClosed { reason: HlsTerminalFailedClosedReason::LeaseStateUnavailable } + ); + + lease.last_manifest_snapshot = Some(HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(1_500), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 1, + delivered_at_ms: 1_500, + first_proxy_seq: 0, + last_proxy_seq: 0, + visible_segments: Arc::from([]), + discontinuity_sequence: 0, + target_duration_ms: 4_000, + playlist_duration_ms: 0, + last_visible_media_end_ms: 0, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + }); + { + let mut session = session.write().await; + session.origin_control.target_duration_snapshot_ms = Some(4_000); + session.origin_control.last_media_progress_at_ms = Some(2_000); + } + assert_eq!( + hls_manifest_terminal_preflight(&session, &lease, 8_000).await, + HlsManifestTerminalPreflight::RefreshBeforeTerminalEvaluation + ); + session.write().await.origin_control.last_media_progress_at_ms = Some(7_999); + assert_eq!( + hls_manifest_terminal_preflight(&session, &lease, 8_000).await, + HlsManifestTerminalPreflight::EvaluateTerminal + ); + + lease.playback_mode = HlsLeasePlaybackMode::Ended; + assert_eq!( + hls_manifest_terminal_preflight(&session, &lease, 8_000).await, + HlsManifestTerminalPreflight::ServeCommittedPlayback + ); +} + +#[tokio::test] +async fn hls_manifest_terminal_preflight_keeps_capacity_recovery_out_of_sync_terminal_wait() { + let app_state = test_app_state(); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "capacity-preflight"), &app_state.get_encrypt_secret(), 1_000) + .await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + { + let mut session = session.write().await; + let OriginManifestParseOutcome::Normal(manifest) = parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:8\n#EXT-X-MEDIA-SEQUENCE:0\n\ + #EXTINF:4.0,\n0.ts\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n", + "http://origin.example/live/index.m3u8", + ) else { + panic!("capacity preflight manifest parses"); + }; + session.apply_origin_manifest(&manifest).expect("capacity preflight timeline applies"); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1_000 }; + } + session.segments.get_mut(&1).expect("deferred segment").status = + SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Prefetch, deferred_at_ms: 2_000 }; + session.origin_control.target_duration_snapshot_ms = Some(8_000); + session.origin_control.last_media_progress_at_ms = Some(1_000); + } + let mut lease = HlsAccessLease::pending( + HlsAccessLeaseId("capacity-preflight-lease".to_string()), + HlsPlaybackFamilyKey::new("capacity-user", "capacity-client"), + proxy_session_id, + "capacity-user".to_string(), + "capacity-user-session".to_string(), + 1, + "capacity-preflight".to_string(), + 1, + 1_000, + 60_000, + ); + lease.state = HlsAccessLeaseState::Activated; + lease.last_manifest_snapshot = Some(HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(1), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 1, + delivered_at_ms: 1_000, + first_proxy_seq: 0, + last_proxy_seq: 0, + visible_segments: Arc::from([HlsLeaseManifestSegment { + proxy_seq: 0, + duration_ms: 4_000, + uri: "000000.ts".to_string().into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }]), + discontinuity_sequence: 0, + target_duration_ms: 8_000, + playlist_duration_ms: 4_000, + last_visible_media_end_ms: 4_000, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + }); + + assert_eq!( + hls_manifest_terminal_preflight(&session, &lease, 20_000).await, + HlsManifestTerminalPreflight::EvaluateTerminal, + ); + + session.write().await.segments.get_mut(&1).expect("recovered segment").status = + SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 20_001 }; + assert_eq!( + hls_manifest_terminal_preflight(&session, &lease, 20_001).await, + HlsManifestTerminalPreflight::RefreshBeforeTerminalEvaluation, + ); +} + +#[test] +fn hls_terminal_commit_endpoint_pending_and_failed_closed_headers_are_distinct() { + let pending = hls_temporary_resource_unavailable_response(250); + assert_eq!(pending.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(pending.headers().get(header::RETRY_AFTER).and_then(|value| value.to_str().ok()), Some("1")); + assert!(pending.headers().get(header::LOCATION).is_none()); + + let failed_closed = hls_terminal_failed_closed_response(HlsTerminalFailedClosedReason::RetryAttemptsExhausted); + assert_eq!(failed_closed.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(failed_closed.headers().get(header::RETRY_AFTER).is_none()); + assert!(failed_closed.headers().get(header::LOCATION).is_none()); +} + +#[tokio::test] +async fn invalid_normal_segment_uri_never_redirects_or_serves_terminal_media() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(hls_proxy); + let proxy_session_id = map_ready_segment(&app_state, 123, "ts", b"0123456789").await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "broken.ts").await; + + let response = get_response(app_state, &uri, None).await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!(!response.headers().contains_key(header::LOCATION)); +} diff --git a/backend/app/src/api/endpoints/hls_api/tests/transient_resources.rs b/backend/app/src/api/endpoints/hls_api/tests/transient_resources.rs new file mode 100644 index 000000000..3d9f08171 --- /dev/null +++ b/backend/app/src/api/endpoints/hls_api/tests/transient_resources.rs @@ -0,0 +1,671 @@ +use super::{ + assert_no_hls_cache_stream_registered, disable_custom_stream_response, enable_hls_cache, encode_test_manifest, + get_response, get_status, hls_proxy_uri, hls_session_last_media_at_ms, map_transient_resource, media_uri_count, + overlap_provider_input, publish_owner_handoff_test_manifest, response_body, spawn_test_transient_origin, + spawn_test_transient_origin_with_delayed_binary_response, spawn_test_transient_origin_with_delayed_response, + spawn_test_transient_origin_with_response, test_app_state, test_app_state_with_hls_proxy, + test_app_state_with_inputs, transient_manifest_body, try_test_hls_cached_manifest_response, + wait_for_provider_connection_count, CanonicalOwnerHandoffFixture, HlsCanonicalOwnerRegistrationKind, +}; +use crate::{ + api::model::{ + HlsAccessLeaseId, HlsAccessLeaseState, HlsAvailabilityReevaluationFinishReason, + HlsAvailabilityReevaluationMode, HlsAvailabilityReevaluationRegistration, HlsOriginAccountBinding, + HlsProxyManager, HlsSessionKey, HlsSessionMode, ProxySessionId, TransientObjectCacheKey, + TransientObjectCacheStatus, TransientResourceId, + }, + model::StripConfig, +}; +use axum::http::{header, HeaderValue, StatusCode}; +use http_body_util::BodyExt; +use shared::model::HlsStripMode; +use std::{sync::Arc, time::Duration}; + +#[tokio::test] +async fn scheduled_owner_does_not_return_transient_503() { + let fixture = CanonicalOwnerHandoffFixture::new(&["scheduled-lease"]).await; + let owner_key = fixture + .app_state + .hls + .proxy + .availability_reevaluation_owner_key(&fixture.session, &fixture.proxy_session_id) + .await + .expect("owner key"); + let coordinator = fixture.app_state.hls.proxy.availability_reevaluations(); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let completed = Arc::new(tokio::sync::Notify::new()); + let task_started = Arc::clone(&started); + let task_release = Arc::clone(&release); + let task_completed = Arc::clone(&completed); + let task_session = Arc::clone(&fixture.session); + let task_app_state = Arc::clone(&fixture.app_state); + let task_proxy_session_id = fixture.proxy_session_id.clone(); + let task_owner_key = owner_key.clone(); + assert_eq!( + coordinator.register( + owner_key, + HlsAvailabilityReevaluationMode::RecoveryPressure, + move |ownership| async move { + task_started.notify_one(); + task_release.notified().await; + publish_owner_handoff_test_manifest(&task_session).await; + task_app_state.hls.proxy.notify_session_evidence_changed(&task_proxy_session_id); + let _ = ownership.finish_cycle(&task_owner_key, HlsAvailabilityReevaluationFinishReason::Evaluated); + task_completed.notify_one(); + }, + ), + HlsAvailabilityReevaluationRegistration::Scheduled + ); + started.notified().await; + let deadline_ms = super::super::current_time_millis().saturating_add(60_000); + let safe_session = fixture.safe_session().await; + let mut response = Box::pin(super::super::join_hls_canonical_manifest_owner( + fixture.handoff_context(0, safe_session, deadline_ms), + HlsCanonicalOwnerRegistrationKind::Scheduled, + )); + + assert!(matches!(futures::poll!(response.as_mut()), std::task::Poll::Pending)); + release.notify_one(); + + let response = response.await; + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::RETRY_AFTER)); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("live manifest utf8"); + assert!(body.contains("/scheduled-lease/")); + completed.notified().await; +} + +#[tokio::test] +async fn hls_cache_pending_transient_manifest_applies_initial_strip_without_mutating_shared_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.0.clone(); + session.transient.replace_manifest_with_semantics(transient_manifest_body(&proxy_session_id), 100, None); + session.mark_authorized_media_access(super::super::current_time_millis()); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + let stored_before = session.read().await.transient.last_manifest_body.clone().expect("transient manifest body"); + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); + assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); + assert_eq!(session.read().await.transient.last_manifest_body.as_ref().expect("stored manifest"), &stored_before); + assert_eq!(media_uri_count(&stored_before), 6); +} + +#[tokio::test] +async fn hls_cache_activated_transient_manifest_skips_initial_strip() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.0.clone(); + session.transient.replace_manifest_with_semantics(transient_manifest_body(&proxy_session_id), 100, None); + session.mark_authorized_media_access(super::super::current_time_millis().saturating_sub(16_000)); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await + .expect("transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 6); + assert!(body.contains(&access_lease_id.0)); + assert!(!body.contains(crate::api::model::HLS_ACCESS_LEASE_ID_PLACEHOLDER)); +} + +#[tokio::test] +async fn hls_cache_transient_manifest_without_media_activity_is_not_served_from_committed_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::super::current_time_millis(); + session.transient.replace_manifest_with_semantics( + transient_manifest_body(&proxy_session_id), + rendered_at_ms, + Some(60_000), + ); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await; + + assert!(response.is_none()); +} + +#[tokio::test] +async fn hls_cache_no_media_yet_transient_manifest_is_served_for_initial_canonical_response() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.0.clone(); + let rendered_at_ms = super::super::current_time_millis(); + session.transient.replace_manifest_with_semantics( + transient_manifest_body(&proxy_session_id), + rendered_at_ms, + Some(60_000), + ); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::ZERO), + ) + .await + .expect("initial transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains(&access_lease_id.0)); + assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); +} + +#[tokio::test] +async fn hls_cache_transient_manifest_outside_soft_window_is_not_served_from_committed_body() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let _proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + let proxy_session_id = session.proxy_session_id.0.clone(); + session.transient.replace_manifest_with_semantics(transient_manifest_body(&proxy_session_id), 100, None); + session.mark_authorized_media_access(super::super::current_time_millis().saturating_sub(60_000)); + proxy_session_id + }; + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Activated, + &strip, + None, + super::HlsCachedManifestOptions::committed_only(Duration::ZERO), + ) + .await; + + assert!(response.is_none()); +} + +#[tokio::test] +async fn hls_cache_no_media_yet_waits_for_first_transient_manifest_commit() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + session.origin_refresh.in_flight = true; + session.proxy_session_id.clone() + }; + let session_for_commit = Arc::clone(&session); + let proxy_session_for_body = proxy_session_id.0.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + let mut session = session_for_commit.write().await; + let rendered_at_ms = super::super::current_time_millis(); + session.transient.replace_manifest_with_semantics( + transient_manifest_body(&proxy_session_for_body), + rendered_at_ms, + Some(60_000), + ); + session.origin_refresh.in_flight = false; + }); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::from_millis(200)), + ) + .await + .expect("initial transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains(&access_lease_id.0)); + assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); +} + +#[tokio::test] +async fn hls_cache_expired_transient_manifest_waits_for_revalidation_commit() { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let session = app_state + .hls + .proxy + .get_or_create_session(HlsSessionKey::new(1, "12345"), &app_state.get_encrypt_secret(), 100) + .await; + let proxy_session_id = { + let mut session = session.write().await; + session.mode = + HlsSessionMode::TransientPassthrough { reason: crate::api::model::TransientPassthroughReason::ExtXKey }; + session.mark_authorized_media_access(super::super::current_time_millis().saturating_sub(60_000)); + session.origin_refresh.in_flight = true; + session.proxy_session_id.clone() + }; + let session_for_commit = Arc::clone(&session); + let proxy_session_for_body = proxy_session_id.0.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + let mut session = session_for_commit.write().await; + let rendered_at_ms = super::super::current_time_millis(); + session.transient.replace_manifest_with_semantics( + transient_manifest_body(&proxy_session_for_body), + rendered_at_ms, + Some(60_000), + ); + session.origin_refresh.in_flight = false; + }); + let access_lease_id = HlsAccessLeaseId("access-lease".to_string()); + let strip = StripConfig { mode: HlsStripMode::Segments, value: 3 }; + + let response = try_test_hls_cached_manifest_response( + &app_state, + &session, + &access_lease_id, + HlsAccessLeaseState::Pending, + &strip, + None, + super::HlsCachedManifestOptions::initial(Duration::from_millis(200)), + ) + .await + .expect("revalidated transient manifest response"); + let body = String::from_utf8(response_body(response).await.to_vec()).expect("manifest utf8"); + + assert_eq!(media_uri_count(&body), 3); + assert!(body.contains(&access_lease_id.0)); + assert!(session.read().await.activity.last_authorized_media_at_ms.is_some()); +} + +#[test] +fn transient_full_object_cacheable_request_accepts_open_zero_range() { + use crate::api::model::is_hls_transient_full_object_cacheable_request; + + assert!(is_hls_transient_full_object_cacheable_request(None)); + assert!(is_hls_transient_full_object_cacheable_request(Some(&HeaderValue::from_static("bytes=0-")))); + assert!(!is_hls_transient_full_object_cacheable_request(Some(&HeaderValue::from_static("bytes=4-")))); + assert!(!is_hls_transient_full_object_cacheable_request(Some(&HeaderValue::from_static("bytes=-4")))); + assert!(!is_hls_transient_full_object_cacheable_request(Some(&HeaderValue::from_static("bytes=0-1,4-5")))); +} + +#[tokio::test] +async fn transient_resource_without_lease_returns_not_found() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin().await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", false).await; + + let status = get_status(app_state, &format!("/hls/shared/live/{proxy_session_id}/r/{resource_id}.ts")).await; + + assert_eq!(status, StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn transient_resource_without_range_is_cached_after_first_fetch() { + let app_state = test_app_state(); + let origin = + spawn_test_transient_origin_with_response("200 OK", &[("Content-Type", "video/mp2t")], "0123456789").await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let first = get_response(Arc::clone(&app_state), &uri, None).await; + let second = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(first.status(), StatusCode::OK); + assert_eq!(second.status(), StatusCode::OK); + assert_eq!(response_body(first).await, bytes::Bytes::from_static(b"0123456789")); + assert_eq!(response_body(second).await, bytes::Bytes::from_static(b"0123456789")); + assert_eq!(origin.requests.lock().await.len(), 1); +} + +#[tokio::test] +async fn transient_resource_range_from_zero_is_cached_as_full_object() { + let app_state = test_app_state(); + let origin = + spawn_test_transient_origin_with_response("200 OK", &[("Content-Type", "video/mp2t")], "0123456789").await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let first = get_response(Arc::clone(&app_state), &uri, Some("bytes=0-")).await; + let second = get_response(Arc::clone(&app_state), &uri, Some("bytes=4-")).await; + + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response_body(first).await, bytes::Bytes::from_static(b"0123456789")); + assert_eq!(response_body(second).await, bytes::Bytes::from_static(b"456789")); + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1); + let request = requests[0].to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity")); + assert!(!request.contains("\r\nrange:")); +} + +#[tokio::test] +async fn transient_cache_fill_rejects_identity_partial_without_ready_object_or_temp_file() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let app_state = test_app_state_with_hls_proxy(Arc::clone(&hls_proxy)); + disable_custom_stream_response(&app_state); + let origin = spawn_test_transient_origin_with_response( + "206 Partial Content", + &[("Content-Type", "video/mp2t"), ("Content-Range", "bytes 0-3/10")], + "part", + ) + .await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + let proxy_session_id = ProxySessionId(proxy_session_id); + let cache_key = TransientObjectCacheKey::new(proxy_session_id.clone(), TransientResourceId(resource_id), "ts"); + let session = hls_proxy.sessions().get_by_proxy_session_id(&proxy_session_id).await.expect("session should exist"); + let status = + session.read().await.transient.object_cache.get(&cache_key).expect("transient cache entry").status.clone(); + assert!(matches!(status, TransientObjectCacheStatus::FailedPermanent { status: None, .. })); + assert!(hls_proxy.segment_cache().metadata(&cache_key).await.expect("cache metadata reads").is_none()); + assert!(!hls_proxy.segment_cache().has_active_temp_files()); + assert_eq!(std::fs::read_dir(temp_dir.path()).expect("cache root reads").count(), 0); + + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1); + let request = requests[0].to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity")); + assert!(!request.contains("\r\nrange:")); +} + +#[tokio::test] +async fn transient_resource_range_from_zero_waits_for_inflight_object_cache_fetch() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin_with_delayed_response( + "200 OK", + &[("Content-Type", "video/mp2t")], + "0123456789", + Duration::from_millis(150), + ) + .await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let first_app_state = Arc::clone(&app_state); + let first_uri = uri.clone(); + let first = tokio::spawn(async move { get_response(first_app_state, &first_uri, Some("bytes=0-")).await }); + for _ in 0..50 { + if origin.requests.lock().await.len() == 1 { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert_eq!(origin.requests.lock().await.len(), 1); + + let second = get_response(Arc::clone(&app_state), &uri, Some("bytes=0-")).await; + let first = first.await.expect("first request joins"); + + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response_body(first).await, bytes::Bytes::from_static(b"0123456789")); + assert_eq!(response_body(second).await, bytes::Bytes::from_static(b"0123456789")); + assert_eq!(origin.requests.lock().await.len(), 1); +} + +#[tokio::test] +async fn transient_resource_range_from_offset_without_ready_object_is_not_cached() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin().await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let first = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-15")).await; + let second = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-15")).await; + + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(origin.requests.lock().await.len(), 2); +} + +#[tokio::test] +async fn transient_resource_origin_error_does_not_mark_media_activity() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin_with_response( + "500 Internal Server Error", + &[("Content-Type", "text/plain")], + "origin-error", + ) + .await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers().get(header::RETRY_AFTER).and_then(|value| value.to_str().ok()), Some("1")); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; +} + +#[tokio::test] +async fn transient_resource_permanent_origin_error_never_redirects_to_manifest() { + let app_state = test_app_state(); + let origin = + spawn_test_transient_origin_with_response("404 Not Found", &[("Content-Type", "text/plain")], "missing").await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!(!response.headers().contains_key(header::LOCATION)); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; +} + +#[tokio::test] +async fn transient_resource_permanent_origin_error_returns_not_found_when_custom_response_disabled() { + let app_state = test_app_state(); + disable_custom_stream_response(&app_state); + let origin = + spawn_test_transient_origin_with_response("404 Not Found", &[("Content-Type", "text/plain")], "missing").await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; +} + +#[tokio::test] +async fn transient_decoder_failure_releases_origin_and_access_guards_once() { + let input = overlap_provider_input(); + let app_state = test_app_state_with_inputs(vec![Arc::new(input.clone())]); + let mut truncated = encode_test_manifest("gzip", b"transient decoder failure").await; + truncated.truncate(truncated.len().saturating_sub(8)); + let origin = spawn_test_transient_origin_with_delayed_binary_response( + "200 OK", + &[("Content-Type", "video/mp2t"), ("Content-Encoding", "gzip")], + truncated, + Duration::ZERO, + ) + .await; + let (proxy_session_id, resource_id) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let proxy_session_id_value = ProxySessionId(proxy_session_id.clone()); + let session = app_state + .hls + .proxy + .sessions() + .get_by_proxy_session_id(&proxy_session_id_value) + .await + .expect("session should exist"); + { + let mut session = session.write().await; + session.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::clone(&input.name), + Arc::from("account-a"), + &proxy_session_id_value, + super::super::current_time_millis(), + )); + } + let resource = session + .read() + .await + .transient + .resources + .get(&TransientResourceId(resource_id.clone())) + .cloned() + .expect("transient resource exists"); + let uri = hls_proxy_uri(&app_state, &proxy_session_id, &format!("r/{resource_id}.ts")).await; + + let response = get_response(Arc::clone(&app_state), &uri, Some("bytes=2-")).await; + + assert_eq!(response.status(), StatusCode::OK); + wait_for_provider_connection_count(&app_state, 1).await; + assert_eq!(resource.active_readers(), 1); + assert!(response.into_body().collect().await.is_err()); + wait_for_provider_connection_count(&app_state, 0).await; + assert_eq!(resource.active_readers(), 0); + tokio::task::yield_now().await; + assert_eq!(app_state.active_provider.get_provider_connections_count(), 0); + assert_eq!(origin.requests.lock().await.len(), 1, "body failure must not start another origin request"); +} + +#[tokio::test] +async fn transient_unknown_resource_never_redirects_to_manifest() { + let app_state = test_app_state(); + let origin = spawn_test_transient_origin().await; + let (proxy_session_id, _) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "r/unknown.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!(!response.headers().contains_key(header::LOCATION)); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; +} + +#[tokio::test] +async fn transient_unknown_resource_returns_not_found_when_custom_response_disabled() { + let app_state = test_app_state(); + disable_custom_stream_response(&app_state); + let origin = spawn_test_transient_origin().await; + let (proxy_session_id, _) = + map_transient_resource(&app_state, &format!("{}/seg.ts", origin.base_url), "ts", true).await; + let uri = hls_proxy_uri(&app_state, &proxy_session_id, "r/unknown.ts").await; + + let response = get_response(Arc::clone(&app_state), &uri, None).await; + + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(hls_session_last_media_at_ms(&app_state, &proxy_session_id).await, None); + assert_no_hls_cache_stream_registered(&app_state).await; +} diff --git a/backend/app/src/api/endpoints/library_api.rs b/backend/app/src/api/endpoints/library_api.rs index 4e6a666df..b33ddb51e 100644 --- a/backend/app/src/api/endpoints/library_api.rs +++ b/backend/app/src/api/endpoints/library_api.rs @@ -123,6 +123,55 @@ async fn get_thumbnail( serve_thumbnail_hash(&storage, &id, etag, &headers).await } +async fn serve_thumbnail_hash( + storage: &MetadataStorage, + hash: &str, + etag: String, + headers: &axum::http::HeaderMap, +) -> axum::response::Response { + if let Some(if_none_match) = headers.get(axum::http::header::IF_NONE_MATCH) { + if if_none_match.as_bytes() == etag.as_bytes() { + return axum::http::StatusCode::NOT_MODIFIED.into_response(); + } + } + + let thumb_path = storage.get_thumbnail_path(hash); + match tokio::fs::read(&thumb_path).await { + Ok(data) => { + let headers = [ + (axum::http::header::CONTENT_TYPE, "image/jpeg".to_string()), + (axum::http::header::CACHE_CONTROL, "max-age=86400, public".to_string()), + (axum::http::header::ETAG, etag), + ]; + (headers, data).into_response() + } + Err(_) => axum::http::StatusCode::NOT_FOUND.into_response(), + } +} + +/// Registers Library API routes. +pub fn library_api_register( + router: axum::Router>, + app_state: Option<&Arc>, +) -> axum::Router> { + match app_state { + Some(app_state) => router + .route( + "/library/status", + axum::routing::get(get_library_status).layer(permission_layer!(app_state, Permission::LibraryRead)), + ) + .route( + "/library/scan", + axum::routing::post(scan_library).layer(permission_layer!(app_state, Permission::LibraryWrite)), + ) + .route("/library/thumbnail/{uuid}", axum::routing::get(get_thumbnail)), + None => router + .route("/library/scan", axum::routing::post(scan_library)) + .route("/library/status", axum::routing::get(get_library_status)) + .route("/library/thumbnail/{uuid}", axum::routing::get(get_thumbnail)), + } +} + #[cfg(test)] mod tests { use super::*; @@ -173,52 +222,3 @@ mod tests { ); } } - -async fn serve_thumbnail_hash( - storage: &MetadataStorage, - hash: &str, - etag: String, - headers: &axum::http::HeaderMap, -) -> axum::response::Response { - if let Some(if_none_match) = headers.get(axum::http::header::IF_NONE_MATCH) { - if if_none_match.as_bytes() == etag.as_bytes() { - return axum::http::StatusCode::NOT_MODIFIED.into_response(); - } - } - - let thumb_path = storage.get_thumbnail_path(hash); - match tokio::fs::read(&thumb_path).await { - Ok(data) => { - let headers = [ - (axum::http::header::CONTENT_TYPE, "image/jpeg".to_string()), - (axum::http::header::CACHE_CONTROL, "max-age=86400, public".to_string()), - (axum::http::header::ETAG, etag), - ]; - (headers, data).into_response() - } - Err(_) => axum::http::StatusCode::NOT_FOUND.into_response(), - } -} - -/// Registers Library API routes. -pub fn library_api_register( - router: axum::Router>, - app_state: Option<&Arc>, -) -> axum::Router> { - match app_state { - Some(app_state) => router - .route( - "/library/status", - axum::routing::get(get_library_status).layer(permission_layer!(app_state, Permission::LibraryRead)), - ) - .route( - "/library/scan", - axum::routing::post(scan_library).layer(permission_layer!(app_state, Permission::LibraryWrite)), - ) - .route("/library/thumbnail/{uuid}", axum::routing::get(get_thumbnail)), - None => router - .route("/library/scan", axum::routing::post(scan_library)) - .route("/library/status", axum::routing::get(get_library_status)) - .route("/library/thumbnail/{uuid}", axum::routing::get(get_thumbnail)), - } -} diff --git a/backend/app/src/api/endpoints/target_bouquet_api.rs b/backend/app/src/api/endpoints/target_bouquet_api.rs index a6c5bfe3a..18069cb7e 100644 --- a/backend/app/src/api/endpoints/target_bouquet_api.rs +++ b/backend/app/src/api/endpoints/target_bouquet_api.rs @@ -333,8 +333,7 @@ mod tests { use tuliprox_core::model::{Config, SourcesConfig}; fn test_config_with_sources(temp_dir: &TempDir) -> (Config, SourcesConfig) { - let mut config = Config::default(); - config.storage_dir = temp_dir.path().to_string_lossy().to_string(); + let config = Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Config::default() }; let sources_dto = SourcesConfigDto { inputs: vec![ diff --git a/backend/app/src/api/endpoints/user_settings_api.rs b/backend/app/src/api/endpoints/user_settings_api.rs index 23d8bd3a9..5cbdb5c8c 100644 --- a/backend/app/src/api/endpoints/user_settings_api.rs +++ b/backend/app/src/api/endpoints/user_settings_api.rs @@ -503,6 +503,7 @@ mod tests { } #[tokio::test] + #[allow(clippy::too_many_lines)] async fn settings_are_isolated_between_authenticated_users() -> TestResult { let dir = tempfile::tempdir()?; let web_ui = WebUiConfigDto { @@ -621,6 +622,7 @@ mod tests { } #[tokio::test] + #[allow(clippy::too_many_lines)] async fn settings_auth_is_namespace_scoped_and_rechecks_deleted_users_after_waiting() -> TestResult { let dir = tempfile::tempdir()?; let web_ui = WebUiConfigDto { diff --git a/backend/app/src/api/endpoints/v1_api_config.rs b/backend/app/src/api/endpoints/v1_api_config.rs index cf7e719af..673cbae54 100644 --- a/backend/app/src/api/endpoints/v1_api_config.rs +++ b/backend/app/src/api/endpoints/v1_api_config.rs @@ -1103,7 +1103,10 @@ mod tests { require_matching_revision, }; use crate::model::ConfigProvider; - use axum::http::{HeaderMap, HeaderValue, StatusCode}; + use axum::{ + http::{HeaderMap, HeaderValue, StatusCode}, + response::IntoResponse, + }; use shared::{ model::{ ApiProxyConfigDto, ApiProxyServerInfoDto, AppConfigDto, ConfigDto, ConfigProviderDto, Permission, @@ -1466,7 +1469,6 @@ mod tests { .await .expect("save_config_sources must not deadlock"); - use axum::response::IntoResponse; assert_eq!(response.into_response().status(), StatusCode::OK); assert!(!tuliprox_repository::target_bouquet_exists(temp_dir.path(), "target_1").await); assert!(!tuliprox_repository::target_bouquet_exists(temp_dir.path(), "target_renamed").await); diff --git a/backend/app/src/api/endpoints/v1_api_playlist.rs b/backend/app/src/api/endpoints/v1_api_playlist.rs index 56c2af486..d2b213909 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist.rs @@ -1,212 +1,12 @@ // Recording-source resolution moved into the recording subsystem so it no // longer has to call back into `endpoints`. -use crate::{ - api::{ - api_utils::{ - create_api_proxy_user, json_or_bin_response, resource_proxy_response, try_option_bad_request, - try_result_bad_request, try_unwrap_body, - }, - auth_middleware::{check_permission, permission_layer, VerifiedClaims}, - endpoints::{ - api_playlist_utils::{ - get_playlist_for_custom_provider, get_playlist_for_input, get_playlist_for_target, - rewrite_resource_url, STALKER_RESOURCE_SCHEME, - }, - epg_grid_api, - extract_accept_header::ExtractAcceptHeader, - m3u_api::m3u_api_stream_loaded, - xmltv_api::{rewrite_epg_channel_resource_url, serve_epg_web_ui, stream_epg_api}, - xtream_api::{ - xtream_get_stream_info_response, xtream_player_api_stream_with_resolved_target, - xtream_player_api_stream_with_token, ApiStreamContext, ApiStreamRequest, - }, - }, - model::{ - recording::recording_source_resolution::{resolve_recording_config, resolve_recording_target}, - AppState, - }, - }, - auth::{create_access_token, verify_access_token}, - iptv::{stalker::client::validate_public_playable_url, xtream}, - model::{ - parse_xmltv_for_web_ui_from_file, parse_xmltv_for_web_ui_from_url, AppConfig, ConfigInput, ConfigInputFlags, - ConfigInputOptions, ConfigInputUpdateQuality, ConfigTarget, EpgSource, EpgSourceType, IcsDummyPolicy, - InputSource, ProcessTargets, SourcesConfig, - }, - processing::{ - epg::get_input_raw_epg_file_path, - parser::{ - ics::parse_ics_file_to_channel, - xmltv::{merge_epg_channels_by_priority_with_dummy_policies, EpgDummyPolicySource}, - }, - processor::re_resolve_stalker_url, - }, - repository::{ - iter_raw_m3u_target_playlist, iter_raw_xtream_target_playlist, m3u_get_item_for_stream_id, - xtream_get_item_for_stream_id, - }, - utils::{canonicalize_output_epg_id, canonicalize_untrusted_epg_id, file_exists_async, request, EpgIdOutputCase}, -}; -use axum::{response::IntoResponse, Router}; -use log::{debug, error}; -use serde::Deserialize; -use serde_json::json; -use shared::{ - error::TuliproxError, - foundation::{get_filter_detailed, Filter, ValueProvider}, - model::{ - permission::Permission, stalker::StalkerStreamKind, ConfigTargetOptions, EpgChannel, - InputPlaylistUpdateStatusDto, InputType, InputUpdateAction, InputUpdateCapabilities, InputUpdateRequest, - OperationRunAccepted, PersistedPlaylistUpdateClusterState, PersistedPlaylistUpdateClusterStatusDto, - PersistedPlaylistUpdateInputResult, PlaylistEpgRequest, PlaylistItem, PlaylistItemType, PlaylistRequest, - PlaylistUpdateRequestDto, PlaylistUpdateRequestPayload, PlaylistUpdateRunId, PlaylistUpdateState, - PlaylistUpdateStatusDto, PlaylistUrlResolveRequest, ProxyType, TargetType, UiPlaylistItem, VirtualId, - XtreamCluster, - }, - utils::{concat_path_leading_slash, open_web_ui_resource_url, sanitize_sensitive_info, Internable}, -}; -use std::{path::Path, str::FromStr, sync::Arc}; -use tokio_stream::StreamExt; -use url::Url; +use crate::api::{endpoints::extract_accept_header::ExtractAcceptHeader, model::AppState}; +use axum::response::IntoResponse; +use shared::model::{PlaylistRequest, XtreamCluster}; +use std::sync::Arc; const PLAYLIST_UPDATE_STATUS_READ_CONCURRENCY: usize = 8; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ManualUpdateEnqueueError { - Busy, - Unavailable, -} - -fn enqueue_manual_playlist_update( - sender: &tokio::sync::mpsc::Sender, - targets: Arc, - input_action: Option, -) -> Result { - let permit = sender.try_reserve().map_err(|error| match error { - tokio::sync::mpsc::error::TrySendError::Full(()) => ManualUpdateEnqueueError::Busy, - tokio::sync::mpsc::error::TrySendError::Closed(()) => ManualUpdateEnqueueError::Unavailable, - })?; - let run_id = PlaylistUpdateRunId::generate(); - permit.send(crate::api::model::ManualPlaylistUpdateRequest { run_id: run_id.clone(), targets, input_action }); - Ok(run_id) -} - -fn create_config_input_for_m3u(url: &str) -> ConfigInput { - ConfigInput { - id: 0, - name: "m3u_req".intern(), - input_type: InputType::M3u, - url: String::from(url), - enabled: true, - options: Some(ConfigInputOptions { - flags: ConfigInputFlags::XtreamLiveStreamUsePrefix | ConfigInputFlags::ResolveBackground, - update_quality: ConfigInputUpdateQuality::default(), - resolve_delay: shared::defaults::default_resolve_delay_secs(), - probe_delay: shared::defaults::default_probe_delay_secs(), - probe_live_interval_hours: 120, - resolve_filter: None, - probe_filter: None, - flussonic_hls_catchup: shared::model::FlussonicHlsCatchup::Native, - flussonic_hls_catchup_max_duration_secs: shared::model::default_flussonic_hls_catchup_max_duration_secs(), - }), - ..Default::default() - } -} - -fn create_config_input_for_xtream(username: &str, password: &str, host: &str) -> ConfigInput { - ConfigInput { - id: 0, - name: "xc_req".intern(), - input_type: InputType::Xtream, - url: String::from(host), - username: Some(String::from(username)), - password: Some(String::from(password)), - enabled: true, - options: Some(ConfigInputOptions { - flags: ConfigInputFlags::XtreamLiveStreamUsePrefix | ConfigInputFlags::ResolveBackground, - update_quality: ConfigInputUpdateQuality::default(), - resolve_delay: shared::defaults::default_resolve_delay_secs(), - probe_delay: shared::defaults::default_probe_delay_secs(), - probe_live_interval_hours: 120, - resolve_filter: None, - probe_filter: None, - flussonic_hls_catchup: shared::model::FlussonicHlsCatchup::Native, - flussonic_hls_catchup_max_duration_secs: shared::model::default_flussonic_hls_catchup_max_duration_secs(), - }), - ..Default::default() - } -} - -fn resolve_provider_url_with_input(input: &ConfigInput, url: &str) -> String { - match input.resolve_url(url) { - Ok(resolved) => resolved.into_owned(), - Err(err) => { - let sanitized_url = sanitize_sensitive_info(url); - let err_text = err.to_string(); - let sanitized_err = sanitize_sensitive_info(&err_text); - error!("resolve_provider_url_with_input failed for url '{sanitized_url}': {sanitized_err}"); - url.to_string() - } - } -} - -fn resolve_provider_url_for_request(app_config: &AppConfig, playlist_request: &PlaylistRequest, url: &str) -> String { - if !url.starts_with(shared::utils::PROVIDER_SCHEME_PREFIX) { - return url.to_string(); - } - - match playlist_request { - PlaylistRequest::Input(input_name) => app_config - .get_input_by_name(&input_name.intern()) - .map_or_else(|| url.to_string(), |input| resolve_provider_url_with_input(input.as_ref(), url)), - PlaylistRequest::Target(target_id) => app_config - .get_target_by_id(*target_id) - .and_then(|target| app_config.get_inputs_for_target(&target.name)) - .and_then(|inputs| { - let mut matches = inputs.into_iter().filter(|input| input.get_resolve_provider(url).is_some()); - let first = matches.next()?; - if matches.next().is_some() { - return None; - } - Some(resolve_provider_url_with_input(first.as_ref(), url)) - }) - .unwrap_or_else(|| url.to_string()), - PlaylistRequest::CustomXtream(_) | PlaylistRequest::CustomM3u(_) => url.to_string(), - } -} - -fn build_playlist_webplayer_url( - base_url: &str, - access_token: &str, - target_id: u16, - virtual_id: u32, - cluster: XtreamCluster, -) -> String { - format!( - "{base_url}/api/v1/playlist/webplayer/{access_token}/{}/{}/{}", - target_id, - cluster.as_stream_type(), - virtual_id - ) -} - -pub(in crate::api) fn build_webplayer_recording_url( - app_config: &crate::model::AppConfig, - target_id: u16, - virtual_id: u32, - cluster: XtreamCluster, -) -> Option { - let access_token = create_access_token(&app_config.access_token_secret, 30, crate::auth::scope::INTERNAL_PLAYER); - let config = app_config.config.load(); - let server_name = config - .web_ui - .as_ref() - .and_then(|web_ui| web_ui.player_server.as_ref()) - .map_or("default", |server_name| server_name.as_str()); - let server_info = app_config.get_server_info(server_name)?; - Some(build_playlist_webplayer_url(&server_info.get_base_url(), &access_token, target_id, virtual_id, cluster)) -} - #[derive(Debug, Clone, PartialEq, Eq)] pub(in crate::api) struct ResolvedRecordingSource { pub virtual_id: u32, @@ -221,646 +21,6 @@ pub(in crate::api) struct ResolvedRecordingSource { pub downloadable: bool, } -/// The playlist fields a recording needs, shared by Xtream and M3U items. -struct RecordingCandidateFields<'a> { - virtual_id: VirtualId, - input_name: &'a str, - name: &'a str, - title: &'a str, - group: &'a str, - url: &'a str, - item_type: PlaylistItemType, -} - -fn non_blank(value: &str) -> Option { - let value = value.trim(); - (!value.is_empty()).then(|| value.to_string()) -} - -fn recording_candidate(fields: &RecordingCandidateFields<'_>) -> ResolvedRecordingSource { - let RecordingCandidateFields { virtual_id, input_name, name, title, group, url, item_type } = *fields; - let is_episode = matches!(item_type, PlaylistItemType::Series | PlaylistItemType::LocalSeries); - ResolvedRecordingSource { - virtual_id: virtual_id.get(), - input_name: input_name.to_string(), - title: title.to_string(), - group: non_blank(group), - series_name: if is_episode { non_blank(name) } else { None }, - extension: shared::utils::extract_extension_from_url(url).map(str::to_string), - downloadable: item_type != PlaylistItemType::SeriesInfo, - } -} - -pub(in crate::api) async fn resolve_target_recording_source( - app_config: &crate::model::AppConfig, - target_name: &str, - input_name: &str, - virtual_id: u32, - cluster: XtreamCluster, -) -> Option { - // EPG grid rows carry the exact playlist id but do not know the input. - // Resolve it from the persisted item, then validate its configured scope. - let missing_input = input_name.trim().is_empty(); - let target = if missing_input { - find_target_by_name(app_config, target_name)? - } else { - resolve_recording_target(app_config, target_name, input_name)? - }; - let wanted = VirtualId::new(virtual_id); - let mut resolved = None; - if target.has_output(TargetType::Xtream) { - if let Some(mut items) = iter_raw_xtream_target_playlist(app_config, &target, cluster).await { - while let Some(entry) = items.next().await { - let Ok(item) = entry else { continue }; - if item.virtual_id == wanted { - resolved = Some(recording_candidate(&RecordingCandidateFields { - virtual_id: item.virtual_id, - input_name: &item.input_name, - name: &item.name, - title: &item.title, - group: &item.group, - url: &item.url, - item_type: item.item_type, - })); - break; - } - } - } - } - if resolved.is_none() && target.has_output(TargetType::M3u) { - if let Some(mut items) = iter_raw_m3u_target_playlist(app_config, &target, Some(cluster)).await { - while let Some(entry) = items.next().await { - let Ok(item) = entry else { continue }; - if item.virtual_id == wanted { - resolved = Some(recording_candidate(&RecordingCandidateFields { - virtual_id: item.virtual_id, - input_name: &item.input_name, - name: &item.name, - title: &item.title, - group: &item.group, - url: &item.url, - item_type: item.item_type, - })); - break; - } - } - } - } - let resolved = resolved?; - if missing_input { - return is_recording_input_in_target_scope(app_config, &target, &resolved.input_name).then_some(resolved); - } - (resolved.input_name == input_name).then_some(resolved) -} - -/// `SourcesConfigDto::prepare` rejects duplicate target names across sources. -fn find_target_by_name(app_config: &crate::model::AppConfig, target_name: &str) -> Option> { - app_config - .sources - .load() - .sources - .iter() - .flat_map(|source| &source.targets) - .find(|target| target.name == target_name) - .cloned() -} - -/// Whether `input_name` belongs to the source that configures `target`. -fn is_recording_input_in_target_scope( - app_config: &crate::model::AppConfig, - target: &ConfigTarget, - input_name: &str, -) -> bool { - resolve_recording_target(app_config, &target.name, input_name).is_some_and(|configured| configured.id == target.id) -} - -/// Compares a playlist item's EPG channel id against a requested one under the -/// target's configured output casing. -/// -/// The web UI reads channel ids from the target EPG database, where they are -/// written in the target's output case (`lowercase_ids`). The persisted -/// playlist keeps the source casing, so a raw comparison misses every channel -/// whose source id differs in case from the EPG database key. -fn epg_channel_id_matches( - item_epg_channel_id: Option<&Arc>, - requested: &Arc, - output_case: EpgIdOutputCase, -) -> bool { - item_epg_channel_id - .is_some_and(|item_id| canonicalize_output_epg_id(item_id, output_case).as_ref() == requested.as_ref()) -} - -/// Picks the channel to record from every live item that carries the requested -/// EPG channel id. -/// -/// Providers commonly publish one EPG id for several streams of the same -/// channel (HD, FHD, 4K, backup feeds). The web UI shows them as one EPG row -/// labelled with one of their titles, so that title is the best hint for -/// which stream the user meant. Without a matching title every candidate airs -/// the same programme, and the first one in playlist order is taken. -fn select_epg_channel_candidate( - candidates: Vec, - channel_name: Option<&str>, -) -> Option { - let hint = channel_name.map(str::trim).filter(|name| !name.is_empty()); - if let Some(hint) = hint { - if let Some(index) = candidates.iter().position(|candidate| candidate.title == hint) { - return candidates.into_iter().nth(index); - } - let hint = hint.to_lowercase(); - if let Some(index) = candidates.iter().position(|candidate| candidate.title.to_lowercase() == hint) { - return candidates.into_iter().nth(index); - } - } - candidates.into_iter().next() -} - -pub(in crate::api) async fn resolve_target_live_recording_source_by_epg_channel( - app_config: &crate::model::AppConfig, - target_name: &str, - input_name: &str, - epg_channel_id: &str, - channel_name: Option<&str>, -) -> Option { - let target = find_target_by_name(app_config, target_name)?; - let output_case = - EpgIdOutputCase::from_lowercase(target.options.as_ref().is_some_and(ConfigTargetOptions::lowercase_epg_ids)); - let requested = canonicalize_untrusted_epg_id(epg_channel_id, output_case); - let mut candidates = Vec::new(); - if target.has_output(TargetType::Xtream) { - if let Some(mut items) = iter_raw_xtream_target_playlist(app_config, &target, XtreamCluster::Live).await { - while let Some(entry) = items.next().await { - let Ok(item) = entry else { continue }; - if epg_channel_id_matches(item.epg_channel_id.as_ref(), &requested, output_case) { - candidates.push(recording_candidate(&RecordingCandidateFields { - virtual_id: item.virtual_id, - input_name: &item.input_name, - name: &item.name, - title: &item.title, - group: &item.group, - url: &item.url, - item_type: item.item_type, - })); - } - } - } - } else if target.has_output(TargetType::M3u) { - if let Some(mut items) = iter_raw_m3u_target_playlist(app_config, &target, Some(XtreamCluster::Live)).await { - while let Some(entry) = items.next().await { - let Ok(item) = entry else { continue }; - if epg_channel_id_matches(item.epg_channel_id.as_ref(), &requested, output_case) { - candidates.push(recording_candidate(&RecordingCandidateFields { - virtual_id: item.virtual_id, - input_name: &item.input_name, - name: &item.name, - title: &item.title, - group: &item.group, - url: &item.url, - item_type: item.item_type, - })); - } - } - } - } - // Apply input constraints before the title hint can select another feed. - candidates.retain(|candidate| { - (input_name.trim().is_empty() || candidate.input_name == input_name) - && is_recording_input_in_target_scope(app_config, &target, &candidate.input_name) - }); - select_epg_channel_candidate(candidates, channel_name) -} - -#[cfg(test)] -fn merge_epg_channels(mut channels_by_source: Vec<(i16, Vec)>) -> Vec { - channels_by_source.sort_by_key(|(priority, _)| *priority); - merge_epg_channels_by_priority_with_dummy_policies(channels_by_source, Vec::new()) -} - -async fn load_xmltv_epg_source_channels( - app_state: &Arc, - raw_epg_path: &Path, - resolved_url: &str, -) -> Result, TuliproxError> { - { - let _cache_lock = app_state.app_config.file_locks.read_lock(raw_epg_path).await; - if file_exists_async(raw_epg_path).await { - match parse_xmltv_for_web_ui_from_file(raw_epg_path).await { - Ok(channels) => return Ok(channels), - Err(file_err) => { - debug!( - "EPG file parse failed {}, trying upstream: {file_err}", - sanitize_sensitive_info(raw_epg_path.to_str().unwrap_or_default()) - ); - } - } - } - } - - parse_xmltv_for_web_ui_from_url(&app_state.app_config, &app_state.http_clients.default.load(), resolved_url).await -} - -async fn load_ics_epg_source_channels( - app_state: &Arc, - input: &ConfigInput, - epg_source: &EpgSource, - raw_epg_path: &Path, - storage_dir: &str, -) -> Result, TuliproxError> { - let ics_config = epg_source - .ics - .as_ref() - .ok_or_else(|| TuliproxError::ConfigEpg("ics configuration is required for ICS EPG sources".to_string()))?; - - let channel_id = epg_source - .channel_id - .clone() - .ok_or_else(|| TuliproxError::ConfigEpg("channel_id is required for ICS EPG sources".to_string()))?; - - { - let _cache_lock = app_state.app_config.file_locks.read_lock(raw_epg_path).await; - if file_exists_async(raw_epg_path).await { - match parse_ics_file_to_channel( - raw_epg_path, - channel_id.clone(), - epg_source.channel_title.clone(), - ics_config, - ) - .await - { - Ok(channel) => return Ok(vec![channel]), - Err(file_err) => { - debug!( - "ICS EPG file parse failed {}, redownloading source: {}", - sanitize_sensitive_info(raw_epg_path.to_str().unwrap_or_default()), - sanitize_sensitive_info(&file_err.to_string()) - ); - } - } - } - } - - let client = app_state.http_clients.default.load(); - request::get_input_epg_content_as_file( - &app_state.app_config, - &client, - input, - request::InputEpgFileRequest { - headers: None, - storage_dir, - url: &epg_source.url, - persist_path: raw_epg_path, - max_bytes: Some(ics_config.max_download_bytes), - }, - ) - .await?; - - let _cache_lock = app_state.app_config.file_locks.read_lock(raw_epg_path).await; - parse_ics_file_to_channel(raw_epg_path, channel_id, epg_source.channel_title.clone(), ics_config) - .await - .map(|channel| vec![channel]) -} - -async fn load_epg_channels_for_input( - app_state: &Arc, - input: &ConfigInput, -) -> Result>, TuliproxError> { - let Some(epg_config) = input.epg.as_ref() else { - return Ok(None); - }; - - let storage_dir = app_state.app_config.config.load().storage_dir.clone(); - let mut channels_by_source = Vec::new(); - let mut dummy_policies = Vec::new(); - let mut failed_sources = 0usize; - - for (source_order, epg_source) in epg_config.sources.iter().enumerate() { - let raw_epg_path = match get_input_raw_epg_file_path(epg_source, input, &storage_dir).await { - Ok(path) => path, - Err(err) => { - debug!( - "Skipping EPG source {}: {}", - sanitize_sensitive_info(epg_source.url.as_str()), - sanitize_sensitive_info(&err.to_string()) - ); - failed_sources += 1; - continue; - } - }; - - let source_result = match epg_source.source_type { - EpgSourceType::Xmltv => { - let resolved_url = resolve_provider_url_with_input(input, &epg_source.url); - load_xmltv_epg_source_channels(app_state, &raw_epg_path, &resolved_url).await - } - EpgSourceType::Ics => { - load_ics_epg_source_channels(app_state, input, epg_source, &raw_epg_path, &storage_dir).await - } - }; - - let source_channels = match source_result { - Ok(channels) => channels, - Err(err) => { - debug!( - "Skipping EPG source {}: {}", - sanitize_sensitive_info(epg_source.url.as_str()), - sanitize_sensitive_info(&err.to_string()) - ); - failed_sources += 1; - continue; - } - }; - if epg_source.source_type == EpgSourceType::Ics { - if let (Some(ics_config), Some(channel_id)) = (epg_source.ics.as_ref(), epg_source.channel_id.as_ref()) { - dummy_policies.push(EpgDummyPolicySource { - priority: epg_source.priority, - source_order, - channel_id: channel_id.clone(), - policy: IcsDummyPolicy { timezone: ics_config.timezone.clone(), config: ics_config.dummy.clone() }, - }); - } - } - channels_by_source.push((epg_source.priority, source_channels)); - } - - if channels_by_source.is_empty() { - if failed_sources > 0 { - Err(TuliproxError::Config(format!("All {failed_sources} EPG source(s) failed for input '{}'", input.name))) - } else { - Ok(None) - } - } else { - channels_by_source.sort_by_key(|(priority, _)| *priority); - Ok(Some(merge_epg_channels_by_priority_with_dummy_policies(channels_by_source, dummy_policies))) - } -} - -async fn playlist_update( - axum::extract::State(app_state): axum::extract::State>, - claims: Option>, - axum::extract::Json(payload): axum::extract::Json, -) -> impl axum::response::IntoResponse + Send { - let request = payload.into_request(); - let input_action = match request.manual_input_update() { - Ok(action) => action, - Err(error) => { - return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": error}))).into_response() - } - }; - if input_action.is_some_and(|request| request.action == InputUpdateAction::Rescan) { - let config = app_state.app_config.config.load(); - if !config.library.as_ref().is_some_and(|library| library.enabled) { - return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Library is not enabled"}))) - .into_response(); - } - if config.web_ui.as_ref().and_then(|web| web.auth.as_ref()).is_some() { - let Some(axum::Extension(VerifiedClaims(claims))) = claims else { - return axum::http::StatusCode::UNAUTHORIZED.into_response(); - }; - if check_permission::<{ Permission::LibraryWrite as u32 }>(&app_state, &claims, None).is_err() { - return axum::http::StatusCode::FORBIDDEN.into_response(); - } - } - } - let process_targets = resolve_manual_playlist_update_targets(&app_state.app_config.sources.load(), &request); - match process_targets { - Ok(valid_targets) => { - let valid_targets = Arc::new(valid_targets); - match enqueue_manual_playlist_update( - &app_state.playlist_updates.manual_update_sender, - valid_targets, - input_action, - ) { - Ok(run_id) => { - (axum::http::StatusCode::ACCEPTED, axum::Json(OperationRunAccepted::playlist_update(run_id))) - .into_response() - } - Err(ManualUpdateEnqueueError::Busy) => { - debug!("Manual playlist update rejected: another update is already queued"); - ( - axum::http::StatusCode::CONFLICT, - axum::Json(json!({"error": "Another playlist update is already queued; retry this request"})), - ) - .into_response() - } - Err(ManualUpdateEnqueueError::Unavailable) => { - debug!("Manual playlist update rejected: worker channel closed (server shutting down)"); - axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response() - } - } - } - Err(err) => { - error!("Failed playlist update {}", sanitize_sensitive_info(&err.to_string())); - (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": err.to_string()}))).into_response() - } - } -} - -fn resolve_manual_playlist_update_targets( - sources: &SourcesConfig, - request: &PlaylistUpdateRequestDto, -) -> Result { - let manual_action = - request.manual_input_update().map_err(|error| TuliproxError::ConfigSource(error.to_string()))?; - let manual_input = if let Some(manual) = manual_action { - let input = sources - .inputs - .iter() - .find(|input| input.id == manual.input_id) - .ok_or_else(|| TuliproxError::ConfigSource(format!("No input found for id {}", manual.input_id)))?; - if !InputUpdateCapabilities::for_input_type(input.input_type, input.enabled).supports_action(manual.action) { - return Err(TuliproxError::ConfigSource("Manual action is not supported for this input".to_string())); - } - // Only the additive action wire form requires IDs. Legacy input_refresh - // requests retain the existing name resolution, including empty = all. - if request.input_action.is_some() && request.target_ids.is_none() { - return Err(TuliproxError::ConfigSource("Manual input actions require explicit target IDs".to_string())); - } - Some(input) - } else { - None - }; - let Some(target_ids) = request.target_ids.as_deref() else { - let targets = (!request.targets.is_empty()).then_some(&request.targets); - return sources.validate_targets(targets); - }; - if !request.targets.is_empty() { - return Err(TuliproxError::ConfigSource( - "Manual playlist update cannot contain both target names and target IDs".to_string(), - )); - } - if target_ids.is_empty() { - return Err(TuliproxError::ConfigSource( - "Manual playlist update target ID selection must not be empty".to_string(), - )); - } - - let mut targets = Vec::with_capacity(target_ids.len()); - let mut target_names = Vec::with_capacity(target_ids.len()); - for target_id in target_ids { - let Some(target) = sources.get_target_by_id(*target_id) else { - return Err(TuliproxError::ConfigSource(format!("No target found for id {target_id}"))); - }; - if manual_input.is_some_and(|input| { - !sources.sources.iter().any(|source| { - source.inputs.contains(&input.name) && source.targets.iter().any(|target| target.id == *target_id) - }) - }) { - return Err(TuliproxError::ConfigSource("Target does not belong to the selected input".to_string())); - } - targets.push(target.id); - target_names.push(target.name.clone()); - } - - Ok(ProcessTargets { - enabled: true, - inputs: sources.inputs.iter().map(|input| input.id).collect(), - targets, - target_names, - }) -} - -async fn read_playlist_update_input_status( - read: impl FnOnce() -> crate::processing::input_cache::InputStatus + Send + 'static, -) -> Result { - tokio::task::spawn_blocking(read).await.map_err(|error| { - error!("Playlist update status reader failed: {error}"); - axum::http::StatusCode::INTERNAL_SERVER_ERROR - }) -} - -async fn playlist_update_status( - axum::extract::State(app_state): axum::extract::State>, -) -> Result, axum::http::StatusCode> { - let storage_dir = app_state.app_config.config.load().storage_dir.clone(); - let inputs = app_state.app_config.sources.load().inputs.clone(); - let status_reads = futures::stream::iter(inputs.into_iter().map(|input| { - let storage_dir = storage_dir.clone(); - let input_name = input.name.clone(); - async move { - let storage_path = - crate::processing::input_cache::resolve_input_storage_path(&storage_dir, &input_name).await; - let input_status = read_playlist_update_input_status(move || { - crate::processing::input_cache::load_input_status(&storage_path) - }) - .await?; - Ok::<_, axum::http::StatusCode>((input, input_status)) - } - })); - let status_reads = { - use futures::StreamExt as _; - status_reads.buffered(PLAYLIST_UPDATE_STATUS_READ_CONCURRENCY) - }; - let input_statuses: Vec<_> = futures::StreamExt::collect(status_reads).await; - let input_statuses = input_statuses.into_iter().collect::, _>>()?; - let mut statuses = Vec::with_capacity(input_statuses.len()); - for (input, input_status) in input_statuses { - let cluster_based = input.input_type.is_xtream() || input.input_type.is_stalker(); - let last_update = - input_status.clusters.values().map(|cluster| cluster.timestamp).filter(|timestamp| *timestamp > 0).max(); - let clusters = if cluster_based || input.input_type == InputType::M3u { - [XtreamCluster::Live, XtreamCluster::Video, XtreamCluster::Series] - .into_iter() - .filter_map(|cluster| { - input_status.clusters.get(cluster.as_ref()).map(|persisted| { - PersistedPlaylistUpdateClusterStatusDto { - cluster, - status: match &persisted.status { - crate::processing::input_cache::ClusterState::Ok => { - PersistedPlaylistUpdateClusterState::Ok - } - crate::processing::input_cache::ClusterState::Failed => { - PersistedPlaylistUpdateClusterState::Failed - } - }, - timestamp: persisted.timestamp, - last_update: persisted.last_update, - } - }) - }) - .collect() - } else { - Vec::new() - }; - let last_input_update = input_status.last_input_update.or_else(|| { - // Legacy cluster-free inputs already persisted a general default status. - // Do not infer an aggregate input/target or Quality result from clusters. - if cluster_based { - return None; - } - input_status.clusters.get("default").filter(|status| status.timestamp > 0).map(|status| { - PersistedPlaylistUpdateInputResult { - state: match status.status { - crate::processing::input_cache::ClusterState::Ok => PlaylistUpdateState::Success, - crate::processing::input_cache::ClusterState::Failed => PlaylistUpdateState::Failure, - }, - timestamp: status.timestamp, - } - }) - }); - statuses.push(InputPlaylistUpdateStatusDto { input_id: input.id, last_update, last_input_update, clusters }); - } - let active_updates = app_state - .event_manager - .playlist_update_snapshot() - .into_iter() - .filter(|event| event.input_id.is_some_and(|id| statuses.iter().any(|status| status.input_id == id))) - .collect(); - Ok(axum::Json(PlaylistUpdateStatusDto { inputs: statuses, active_updates })) -} - -async fn playlist_content( - accept: Option, - app_state: &Arc, - playlist_req: &PlaylistRequest, - cluster: XtreamCluster, -) -> impl IntoResponse + Send { - let client = app_state.http_clients.default.load(); - match playlist_req { - PlaylistRequest::Target(target_id) => get_playlist_for_target( - app_state.app_config.get_target_by_id(*target_id).as_deref(), - app_state, - cluster, - accept.as_deref(), - ) - .await - .into_response(), - PlaylistRequest::Input(input_name) => get_playlist_for_input( - app_state.app_config.get_input_by_name(&input_name.intern()).as_ref(), - app_state, - cluster, - accept.as_deref(), - ) - .await - .into_response(), - PlaylistRequest::CustomXtream(xtream) => match Url::parse(&xtream.url) { - Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => { - let input = Arc::new(create_config_input_for_xtream(&xtream.username, &xtream.password, &xtream.url)); - get_playlist_for_custom_provider(client.as_ref(), Some(&input), app_state, cluster, accept.as_deref()) - .await - .into_response() - } - _ => ( - axum::http::StatusCode::BAD_REQUEST, - axum::Json(json!({"error": "Invalid url scheme; only http/https are allowed"})), - ) - .into_response(), - }, - PlaylistRequest::CustomM3u(m3u) => match Url::parse(&m3u.url) { - Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => { - let input = Arc::new(create_config_input_for_m3u(&m3u.url)); - get_playlist_for_custom_provider(client.as_ref(), Some(&input), app_state, cluster, accept.as_deref()) - .await - .into_response() - } - _ => ( - axum::http::StatusCode::BAD_REQUEST, - axum::Json(json!({"error": "Invalid url scheme; only http/https are allowed"})), - ) - .into_response(), - }, - } -} - macro_rules! create_player_api_for_cluster { ($fn_name:ident, $cluster:expr) => { async fn $fn_name( @@ -874,3250 +34,65 @@ macro_rules! create_player_api_for_cluster { } create_player_api_for_cluster!(playlist_content_live, XtreamCluster::Live); + create_player_api_for_cluster!(playlist_content_vod, XtreamCluster::Video); + create_player_api_for_cluster!(playlist_content_series, XtreamCluster::Series); -async fn playlist_series_info( - axum::extract::Path((virtual_id, provider_id)): axum::extract::Path<(String, String)>, - axum::extract::State(app_state): axum::extract::State>, - axum::extract::Json(playlist_req): axum::extract::Json, -) -> impl IntoResponse + Send { - let provider_id = provider_id.trim().parse::().ok(); - - match playlist_req { - PlaylistRequest::Target(target_id) => { - if let Some(target) = app_state.app_config.get_target_by_id(target_id) { - if target.has_output(TargetType::Xtream) { - let mut user = create_api_proxy_user(&app_state); - user.proxy = ProxyType::Redirect; - return xtream_get_stream_info_response( - &app_state, - &user, - &target, - &virtual_id, - XtreamCluster::Series, - ) - .await - .into_response(); - } - } - } - - PlaylistRequest::Input(input_name) => { - if let Some(input) = app_state.app_config.get_input_by_name(&input_name.intern()) { - if input.input_type.is_xtream() { - // We cannot call `xtream_get_stream_info_response` directly here because that path - // depends on target-local virtual-id mapping (`xtream_get_item_for_stream_id`). - // Input/custom requests only provide provider_id, so we resolve series info from - // the upstream Xtream API using provider_id. - if let Some(provider_id) = provider_id { - if let Some(info_url) = - xtream::get_xtream_player_api_info_url(input.as_ref(), XtreamCluster::Series, provider_id) - { - let Ok(resolved_url) = input.resolve_url(&info_url) else { - return axum::http::StatusCode::NO_CONTENT.into_response(); - }; - let input_source = InputSource::from(input.as_ref()).with_url(resolved_url.to_string()); - if let Ok(content) = xtream::get_xtream_stream_info_content( - &app_state.app_config, - &app_state.http_clients.default.load(), - &input_source, - false, - ) - .await - { - return try_unwrap_body!(axum::response::Response::builder() - .status(axum::http::StatusCode::OK) - .header(axum::http::header::CONTENT_TYPE, mime::APPLICATION_JSON.to_string()) - .body(axum::body::Body::from(content))); - } - } - } - } - } - } - PlaylistRequest::CustomXtream(xtream_req) => { - if let Some(provider_id) = provider_id { - let input = create_config_input_for_xtream(&xtream_req.username, &xtream_req.password, &xtream_req.url); - if let Some(info_url) = - xtream::get_xtream_player_api_info_url(&input, XtreamCluster::Series, provider_id) - { - let input_source = InputSource::from(&input).with_url(info_url); - if let Ok(content) = xtream::get_xtream_stream_info_content( - &app_state.app_config, - &app_state.http_clients.default.load(), - &input_source, - false, - ) - .await - { - return try_unwrap_body!(axum::response::Response::builder() - .status(axum::http::StatusCode::OK) - .header(axum::http::header::CONTENT_TYPE, mime::APPLICATION_JSON.to_string()) - .body(axum::body::Body::from(content))); - } - } - } - } - PlaylistRequest::CustomM3u(_) => {} - } - axum::http::StatusCode::NO_CONTENT.into_response() -} - -fn playlist_webplayer( - axum::extract::State(app_state): axum::extract::State>, - target_id: u16, - virtual_id: u32, - cluster: XtreamCluster, -) -> impl axum::response::IntoResponse + Send { - let Some(url) = build_webplayer_recording_url(&app_state.app_config, target_id, virtual_id, cluster) else { - return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); - }; - url.into_response() -} - -async fn playlist_webplayer_stream( - fingerprint: crate::auth::Fingerprint, - axum::extract::Path((token, target_id, cluster, stream_id)): axum::extract::Path<(String, u16, String, String)>, - axum::extract::State(app_state): axum::extract::State>, - req_headers: axum::http::HeaderMap, -) -> impl IntoResponse + Send { - if !verify_access_token(&token, &app_state.app_config.access_token_secret, crate::auth::scope::INTERNAL_PLAYER) { - return axum::http::StatusCode::FORBIDDEN.into_response(); - } - - let ctxt = try_result_bad_request!(ApiStreamContext::from_str(cluster.as_str())); - let Some(target) = app_state.app_config.get_target_by_id(target_id) else { - return axum::http::StatusCode::BAD_REQUEST.into_response(); - }; - - if target.has_output(TargetType::Xtream) { - return xtream_player_api_stream_with_token( - &fingerprint, - &req_headers, - &app_state, - target_id, - ApiStreamRequest::from_access_token(ctxt, &token, &stream_id, ""), - ) - .await - .into_response(); - } - - if !target.has_output(TargetType::M3u) { - return axum::http::StatusCode::BAD_REQUEST.into_response(); - } - - let req_virtual_id: u32 = try_result_bad_request!(stream_id.trim().parse()); - let pli = try_result_bad_request!( - m3u_get_item_for_stream_id(req_virtual_id, &app_state.app_config, &app_state.playlists, &target).await, - true, - format!("Failed to read m3u item for stream id {req_virtual_id}") - ); - let input = try_option_bad_request!( - app_state.app_config.get_input_by_name(&pli.input_name), - true, - format!("Can't find input {} for target {}", pli.input_name, target.name) - ); - let user = Arc::new(create_api_proxy_user(&app_state)); - - m3u_api_stream_loaded(user, target, &fingerprint, &req_headers, &app_state, pli, input, None, None, None) - .await - .into_response() -} - -#[derive(Debug, Deserialize)] -struct RecordingStreamQuery { - target_name: String, - input_name: String, - provider_allocation_id: Option, -} - -async fn playlist_recording_stream( - fingerprint: crate::auth::Fingerprint, - axum::extract::Path((token, cluster, virtual_id)): axum::extract::Path<(String, String, u32)>, - axum::extract::Query(query): axum::extract::Query, - axum::extract::State(app_state): axum::extract::State>, - req_headers: axum::http::HeaderMap, -) -> impl IntoResponse + Send { - if !verify_access_token(&token, &app_state.app_config.access_token_secret, crate::auth::scope::INTERNAL_PLAYER) { - return axum::http::StatusCode::FORBIDDEN.into_response(); - } - let ctxt = try_result_bad_request!(ApiStreamContext::from_str(cluster.as_str())); - let resolved = { - let sources = app_state.app_config.sources.load(); - resolve_recording_config(sources.as_ref(), &query.target_name, &query.input_name) - }; - let Some(resolved) = resolved else { - return axum::http::StatusCode::BAD_REQUEST.into_response(); - }; - let input = crate::api::api_utils::with_recording_headers(&app_state.app_config, resolved.input); - if resolved.target.has_output(TargetType::Xtream) { - let stream_id = virtual_id.to_string(); - return xtream_player_api_stream_with_resolved_target( - &fingerprint, - &req_headers, - &app_state, - resolved.target, - super::xtream_api::InternalPlaybackSource::Recording(input), - ApiStreamRequest::from_access_token(ctxt, &token, &stream_id, ""), - query.provider_allocation_id, - ) - .await - .into_response(); - } - if !resolved.target.has_output(TargetType::M3u) { - return axum::http::StatusCode::BAD_REQUEST.into_response(); - } - - let pli = try_result_bad_request!( - m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, &resolved.target).await, - true, - format!("Failed to read m3u item for stream id {virtual_id}") - ); - if pli.input_name != input.name || pli.item_type.cluster() != ctxt.cluster() { - return axum::http::StatusCode::BAD_REQUEST.into_response(); - } - let user = Arc::new(crate::api::api_utils::create_recording_proxy_user(&app_state)); - m3u_api_stream_loaded( - user, - resolved.target, - &fingerprint, - &req_headers, - &app_state, - pli, - input, - None, - None, - query.provider_allocation_id, - ) - .await - .into_response() -} - -async fn playlist_epg( - ExtractAcceptHeader(accept): ExtractAcceptHeader, - axum::extract::State(app_state): axum::extract::State>, - axum::extract::Json(playlist_epg_req): axum::extract::Json, -) -> impl IntoResponse + Send { - match playlist_epg_req { - PlaylistEpgRequest::Target(target_id) => { - if let Some(target) = app_state.app_config.get_target_by_id(target_id) { - let config = &app_state.app_config.config.load(); - let epg_path = crate::api::endpoints::xmltv_api::get_epg_path_for_target_by_type( - config, - &target, - TargetType::Xtream, - ) - .or_else(|| { - crate::api::endpoints::xmltv_api::get_epg_path_for_target_by_type(config, &target, TargetType::M3u) - }); - if let Some(epg_path) = epg_path { - return serve_epg_web_ui(&app_state, accept.as_deref(), &epg_path, &target).await; - } - } - } - PlaylistEpgRequest::Input(input_name) => { - if let Some(input) = app_state.app_config.get_input_by_name(&input_name.intern()) { - match load_epg_channels_for_input(&app_state, input.as_ref()).await { - Ok(Some(epg)) => { - let config = app_state.app_config.config.load(); - let web_ui_path = - config.web_ui.as_ref().and_then(|w| w.path.as_ref()).map_or("", String::as_str); - let resource_url = concat_path_leading_slash(web_ui_path, "api/v1/playlist/resource"); - let encrypt_secret = app_state.get_encrypt_secret(); - let epg = epg - .into_iter() - .map(|channel| rewrite_epg_channel_resource_url(&encrypt_secret, &resource_url, channel)) - .collect::>(); - return json_or_bin_response(accept.as_deref(), &epg).into_response(); - } - Ok(None) => return axum::http::StatusCode::NO_CONTENT.into_response(), - Err(err) => { - error!( - "Failed to load input EPG for '{}': {}", - input.name, - sanitize_sensitive_info(&err.to_string()) - ); - return ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(serde_json::json!({"error": "Failed to load EPG"})), - ) - .into_response(); - } - } - } - } - PlaylistEpgRequest::Custom(url) => { - let valid_custom_url = Url::parse(&url).is_ok_and(|parsed| matches!(parsed.scheme(), "http" | "https")); - if !valid_custom_url { - return ( - axum::http::StatusCode::BAD_REQUEST, - axum::Json(serde_json::json!({"error": "Invalid EPG URL"})), - ) - .into_response(); - } - match parse_xmltv_for_web_ui_from_url(&app_state.app_config, &app_state.http_clients.default.load(), &url) - .await - { - Ok(epg) => { - let config = app_state.app_config.config.load(); - let web_ui_path = config.web_ui.as_ref().and_then(|w| w.path.as_ref()).map_or("", String::as_str); - let resource_url = concat_path_leading_slash(web_ui_path, "api/v1/playlist/resource"); - let encrypt_secret = app_state.get_encrypt_secret(); - let epg = epg - .into_iter() - .map(|channel| rewrite_epg_channel_resource_url(&encrypt_secret, &resource_url, channel)) - .collect::>(); - return json_or_bin_response(accept.as_deref(), &epg).into_response(); - } - Err(err) => { - error!("Failed to load custom EPG: {}", sanitize_sensitive_info(&err.to_string())); - return ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(serde_json::json!({"error": "Failed to load EPG"})), - ) - .into_response(); - } - } - } - } - axum::http::StatusCode::NO_CONTENT.into_response() -} - -async fn playlist_resource( - req_headers: axum::http::HeaderMap, - axum::extract::Path(resource): axum::extract::Path, - axum::extract::State(app_state): axum::extract::State>, -) -> impl IntoResponse + Send { - let encrypt_secret = app_state.get_encrypt_secret(); - if let Ok(resource_url) = open_web_ui_resource_url(&encrypt_secret, &resource) { - if let Some((input_id, cluster, provider_id)) = parse_stalker_resource(&resource_url) { - return stalker_resource_response(&app_state, input_id, cluster, provider_id).await; - } - // This route serves every icon the Web UI shows, so it is classified like the player routes: - // a public destination goes out through the proxy-aware fetch, which honours a configured - // proxy, while one that is not provably public is fetched directly so the request cannot leave - // through a proxy that has no route to it. - resource_proxy_response(&app_state, &resource_url, &req_headers, None).await.into_response() - } else { - axum::http::StatusCode::BAD_REQUEST.into_response() - } -} - -fn parse_stalker_resource(resource: &str) -> Option<(u16, XtreamCluster, u32)> { - let mut parts = resource.strip_prefix(STALKER_RESOURCE_SCHEME)?.split('/'); - let input_id = parts.next()?.parse().ok()?; - let cluster = parts.next()?.parse().ok()?; - let provider_id = parts.next()?.parse().ok()?; - parts.next().is_none().then_some((input_id, cluster, provider_id)) -} - -async fn stalker_resource_response( - app_state: &Arc, - input_id: u16, - cluster: XtreamCluster, - provider_id: u32, -) -> axum::response::Response { - let Some(input) = app_state.app_config.get_input_by_id(input_id).filter(|input| input.input_type.is_stalker()) - else { - return axum::http::StatusCode::NOT_FOUND.into_response(); - }; - let kind = match cluster { - XtreamCluster::Live => StalkerStreamKind::Live, - XtreamCluster::Video => StalkerStreamKind::Movie, - XtreamCluster::Series => StalkerStreamKind::Episode, - }; - let client = app_state.http_clients.default.load().as_ref().clone(); - match re_resolve_stalker_url(&app_state.app_config, &client, &input, provider_id, kind, false).await { - Ok(Some(resolved_url)) => { - let Ok(url) = Url::parse(&resolved_url) else { - return axum::http::StatusCode::BAD_GATEWAY.into_response(); - }; - if validate_public_playable_url(&url).await.is_err() { - return axum::http::StatusCode::BAD_GATEWAY.into_response(); - } - axum::response::Redirect::temporary(url.as_str()).into_response() - } - Ok(None) => axum::http::StatusCode::NOT_FOUND.into_response(), - Err(err) => { - error!("Failed to resolve Stalker preview stream: {}", sanitize_sensitive_info(&err.to_string())); - axum::http::StatusCode::BAD_GATEWAY.into_response() - } - } -} - -async fn playlist_resolve_url( - axum::extract::State(app_state): axum::extract::State>, - axum::extract::Json(request): axum::extract::Json, -) -> impl IntoResponse + Send { - match request { - PlaylistUrlResolveRequest::Webplayer { target_id, virtual_id, cluster } => { - playlist_webplayer(axum::extract::State(app_state), target_id, virtual_id, cluster).into_response() - } - PlaylistUrlResolveRequest::Provider { playlist_request, url } => { - resolve_provider_url_for_request(&app_state.app_config, &playlist_request, &url).into_response() - } - } -} - const FILTER_PREVIEW_DEFAULT_SAMPLES: u16 = 25; + const FILTER_PREVIEW_MAX_SAMPLES: u16 = 50; -#[derive(serde::Deserialize)] -struct FilterPreviewRequest { - target: u16, - filter: String, - #[serde(default)] - limit: Option, - #[serde(default)] - match_as_ascii: bool, -} - -#[derive(serde::Serialize)] -struct FilterPreviewItem { - name: String, - group: String, - item_type: String, -} - -impl From<&PlaylistItem> for FilterPreviewItem { - fn from(pli: &PlaylistItem) -> Self { - let header = &pli.header; - Self { - name: header.name.to_string(), - group: header.group.to_string(), - item_type: header.item_type.as_str().to_string(), - } - } -} - -#[derive(serde::Serialize, Default)] -struct FilterPreviewClusterStats { - total: usize, - matched: usize, -} - -#[derive(serde::Serialize, Default)] -struct FilterPreviewResponse { - total: usize, - matched: usize, - live: FilterPreviewClusterStats, - vod: FilterPreviewClusterStats, - series: FilterPreviewClusterStats, - sample_matched: Vec, - sample_excluded: Vec, -} - -impl FilterPreviewResponse { - fn observe(&mut self, pli: &PlaylistItem, filter: &Filter, match_as_ascii: bool, sample_limit: usize) { - let cluster_stats = match pli.header.xtream_cluster { - XtreamCluster::Live => &mut self.live, - XtreamCluster::Video => &mut self.vod, - XtreamCluster::Series => &mut self.series, - }; - self.total += 1; - cluster_stats.total += 1; - let provider = ValueProvider { pli, match_as_ascii }; - if filter.filter(&provider) { - self.matched += 1; - cluster_stats.matched += 1; - if self.sample_matched.len() < sample_limit { - self.sample_matched.push(FilterPreviewItem::from(pli)); - } - } else if self.sample_excluded.len() < sample_limit { - self.sample_excluded.push(FilterPreviewItem::from(pli)); - } - } -} - -/// Dry-run a filter DSL expression against a target's stored playlist -/// without touching processing or provider fetches. -async fn playlist_filter_preview( - axum::extract::State(app_state): axum::extract::State>, - axum::extract::Json(req): axum::extract::Json, -) -> impl IntoResponse + Send { - let filter = { - let sources = app_state.app_config.sources.load(); - match get_filter_detailed(&req.filter, sources.templates.as_deref()) { - Ok(filter) => filter, - Err((err, position)) => { - return ( - axum::http::StatusCode::UNPROCESSABLE_ENTITY, - axum::Json(json!({ - "error": err.to_string(), - "line": position.map(|p| p.line), - "column": position.map(|p| p.column), - })), - ) - .into_response() - } - } - }; - let Some(target) = app_state.app_config.get_target_by_id(req.target) else { - return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Unknown target"}))).into_response(); - }; - let sample_limit = usize::from(req.limit.unwrap_or(FILTER_PREVIEW_DEFAULT_SAMPLES).min(FILTER_PREVIEW_MAX_SAMPLES)); - - let mut response = FilterPreviewResponse::default(); - if target.has_output(TargetType::Xtream) { - let mut any_cluster_read = false; - for cluster in [XtreamCluster::Live, XtreamCluster::Video, XtreamCluster::Series] { - if let Some(mut iterator) = iter_raw_xtream_target_playlist(&app_state.app_config, &target, cluster).await { - any_cluster_read = true; - while let Some(entry) = iterator.next().await { - match entry { - Ok(item) => { - let pli = PlaylistItem::from(&item); - response.observe(&pli, &filter, req.match_as_ascii, sample_limit); - } - Err(err) => { - error!("Filter preview failed to read stored {cluster} playlist: {err}"); - return ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(json!({"error": "Failed to read stored playlist"})), - ) - .into_response(); - } - } - } - } - } - if !any_cluster_read { - return ( - axum::http::StatusCode::NOT_FOUND, - axum::Json(json!({"error": "Stored playlist is not available, update the playlist first"})), - ) - .into_response(); - } - } else if target.has_output(TargetType::M3u) { - let Some(mut iterator) = iter_raw_m3u_target_playlist(&app_state.app_config, &target, None).await else { - return ( - axum::http::StatusCode::NOT_FOUND, - axum::Json(json!({"error": "Stored playlist is not available, update the playlist first"})), - ) - .into_response(); - }; - while let Some(entry) = iterator.next().await { - match entry { - Ok(item) => { - let pli = PlaylistItem::from(&item); - response.observe(&pli, &filter, req.match_as_ascii, sample_limit); - } - Err(err) => { - error!("Filter preview failed to read stored m3u playlist: {err}"); - return ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(json!({"error": "Failed to read stored playlist"})), - ) - .into_response(); - } - } - } - } else { - return ( - axum::http::StatusCode::BAD_REQUEST, - axum::Json(json!({"error": "Target has no xtream or m3u output to preview"})), - ) - .into_response(); - } - - axum::Json(response).into_response() -} - -pub fn v1_api_playlist_register_protected(router: Router>) -> axum::Router> { - router - .route("/playlist/resolve_url", axum::routing::post(playlist_resolve_url)) - .route("/playlist/update", axum::routing::post(playlist_update)) - .route("/playlist/update/status", axum::routing::get(playlist_update_status)) - .route("/playlist/epg", axum::routing::post(playlist_epg)) - .route("/playlist/epg/stream", axum::routing::post(stream_epg_api)) - .route("/playlist/epg/groups", axum::routing::post(epg_grid_api::playlist_epg_groups)) - .route("/playlist/epg/grid", axum::routing::post(epg_grid_api::playlist_epg_grid)) - .route("/playlist/live", axum::routing::post(playlist_content_live)) - .route("/playlist/vod", axum::routing::post(playlist_content_vod)) - .route("/playlist/series", axum::routing::post(playlist_content_series)) - .route("/playlist/series_info/{virtual_id}/{provider_id}", axum::routing::post(playlist_series_info)) - .route("/playlist/series/episode/{virtual_id}", axum::routing::post(playlist_episode_item)) - .route("/playlist/filter/preview", axum::routing::post(playlist_filter_preview)) -} - -pub fn v1_api_playlist_register_public(router: Router>) -> axum::Router> { - router - .route("/playlist/resource/{resource}", axum::routing::get(playlist_resource)) - .route( - "/playlist/webplayer/{token}/{target_id}/{cluster}/{stream_id}", - axum::routing::get(playlist_webplayer_stream), - ) - .route("/playlist/recording/{token}/{cluster}/{virtual_id}", axum::routing::get(playlist_recording_stream)) -} - -pub fn v1_api_playlist_register_with_permissions( - router: Router>, - app_state: &Arc, -) -> axum::Router> { - let read_routes = Router::new() - .route("/update/status", axum::routing::get(playlist_update_status)) - .route("/live", axum::routing::post(playlist_content_live)) - .route("/vod", axum::routing::post(playlist_content_vod)) - .route("/series", axum::routing::post(playlist_content_series)) - .route("/resolve_url", axum::routing::post(playlist_resolve_url)) - .route("/series_info/{virtual_id}/{provider_id}", axum::routing::post(playlist_series_info)) - .route("/series/episode/{virtual_id}", axum::routing::post(playlist_episode_item)) - .route("/filter/preview", axum::routing::post(playlist_filter_preview)) - .layer(permission_layer!(app_state, Permission::PlaylistRead)); - - let write_routes = Router::new() - .route("/update", axum::routing::post(playlist_update)) - .layer(permission_layer!(app_state, Permission::PlaylistWrite)); - - let epg_routes = Router::new() - .route("/epg", axum::routing::post(playlist_epg)) - .route("/epg/stream", axum::routing::post(stream_epg_api)) - .route("/epg/groups", axum::routing::post(epg_grid_api::playlist_epg_groups)) - .route("/epg/grid", axum::routing::post(epg_grid_api::playlist_epg_grid)) - .layer(permission_layer!(app_state, Permission::EpgRead)); - - router.nest("/playlist", read_routes.merge(write_routes).merge(epg_routes)) -} - -async fn playlist_episode_item( - axum::extract::Path(virtual_id): axum::extract::Path, - axum::extract::State(app_state): axum::extract::State>, - axum::extract::Json(playlist_req): axum::extract::Json, -) -> impl IntoResponse + Send { - if let PlaylistRequest::Target(target_id) = playlist_req { - if let Some(target) = app_state.app_config.get_target_by_id(target_id) { - if target.has_output(TargetType::Xtream) { - if let Ok(vid) = virtual_id.parse::() { - if let Ok(pli) = xtream_get_item_for_stream_id( - vid, - &app_state.app_config, - &app_state.playlists, - &target, - Some(XtreamCluster::Series), - ) - .await - { - // The icon is wrapped like on every other Web UI path: the item carries the - // destination as it was stored, which may name a host internal to this instance. - let config = app_state.app_config.config.load(); - let web_ui_path = - config.web_ui.as_ref().and_then(|web_ui| web_ui.path.as_ref()).map_or("", String::as_str); - let resource_url = concat_path_leading_slash(web_ui_path, "api/v1/playlist/resource"); - drop(config); - let item = rewrite_resource_url( - &app_state.get_encrypt_secret(), - &resource_url, - UiPlaylistItem::from(pli), - ); - return axum::Json(json!(item)).into_response(); - } - } - } - } - } - axum::http::StatusCode::NO_CONTENT.into_response() -} +#[cfg(test)] +mod epg_channel_candidate_tests; #[cfg(test)] -mod epg_channel_candidate_tests { - use super::{select_epg_channel_candidate, ResolvedRecordingSource}; +mod tests; - fn candidate(virtual_id: u32, title: &str) -> ResolvedRecordingSource { - ResolvedRecordingSource { - virtual_id, - input_name: "input".to_string(), - title: title.to_string(), - group: None, - series_name: None, - extension: None, - downloadable: true, - } - } - - fn shared_epg_id_candidates() -> Vec { - vec![candidate(1, "┃NL┃ SBS 6 4K"), candidate(2, "┃CANAL+┃ SBS 6 HD"), candidate(3, "┃NLZIET┃ SBS 6 HD")] - } - - #[test] - fn shared_epg_id_prefers_exact_channel_name() { - let selected = select_epg_channel_candidate(shared_epg_id_candidates(), Some("┃NLZIET┃ SBS 6 HD")); - assert_eq!(selected.map(|c| c.virtual_id), Some(3)); - } - - #[test] - fn shared_epg_id_matches_lowercased_display_name() { - let selected = select_epg_channel_candidate(shared_epg_id_candidates(), Some(" ┃canal+┃ sbs 6 hd ")); - assert_eq!(selected.map(|c| c.virtual_id), Some(2)); - } - - #[test] - fn shared_epg_id_without_matching_name_takes_first() { - assert_eq!( - select_epg_channel_candidate(shared_epg_id_candidates(), Some("SBS 6")).map(|c| c.virtual_id), - Some(1) - ); - assert_eq!(select_epg_channel_candidate(shared_epg_id_candidates(), None).map(|c| c.virtual_id), Some(1)); - } - - #[test] - fn episode_candidate_carries_series_name_and_group() { - use super::{recording_candidate, RecordingCandidateFields}; - use shared::model::{PlaylistItemType, VirtualId}; - let fields = |item_type, group| RecordingCandidateFields { - virtual_id: VirtualId::new(7), - input_name: "input", - name: "The Show", - title: "The Show S01E02 Pilot", - group, - url: "http://example.test/series/7.mkv", - item_type, - }; - let episode = recording_candidate(&fields(PlaylistItemType::Series, "Crime")); - assert_eq!(episode.series_name.as_deref(), Some("The Show")); - assert_eq!(episode.group.as_deref(), Some("Crime")); - assert_eq!(episode.extension.as_deref().map(|ext| ext.trim_start_matches('.')), Some("mkv")); - - let film = recording_candidate(&fields(PlaylistItemType::Video, " ")); - assert_eq!(film.series_name, None, "only episodes carry a series name"); - assert_eq!(film.group, None, "a blank group is no group"); - } - - #[test] - fn no_candidate_resolves_nothing() { - assert!(select_epg_channel_candidate(Vec::new(), Some("SBS 6")).is_none()); - } -} +mod content; +mod epg; +mod error; +mod filter_preview; +mod recording; +mod resources; +mod routes; +mod update; +mod webplayer; #[cfg(test)] -mod tests { - use super::resolve_provider_url_for_request; - use crate::{ - api::model::{ - recording::recording_source_resolution::resolve_recording_config, ActiveProviderManager, ActiveUserManager, - AppState, ConnectionManager, EventManager, MetadataUpdateManager, PlaylistStorageState, RecordingQueue, - SharedStreamManager, - }, - model::{ - AppConfig, Config, ConfigInput, ConfigInputOptions, ConfigInputUpdateQuality, ConfigProvider, ConfigSource, - ConfigTarget, RecordingConfig, SourcesConfig, StreamHistoryConfig, - }, - processing::epg::{get_input_raw_epg_file_path, get_input_raw_xmltv_file_path}, - repository::GeoIp, - }; - use arc_swap::{ArcSwap, ArcSwapOption}; - use axum::{ - body::Body, - extract::{Path as AxumPath, Query, State}, - http::{Request, StatusCode}, - response::IntoResponse, - Json, Router, - }; - use chrono::Utc; - use serde_json::json; - use shared::{ - foundation::Filter, - model::{ - provider_saturation::build_group_lookup, ConfigPaths, ConfigProviderDto, EpgChannel, EpgConfigDto, - EpgProgramme, EpgSourceDto, EpgSourceTypeDto, IcsDummyConfigDto, IcsEpgSourceConfigDto, - InputRefreshOverride, InputRefreshPolicy, InputType, OperationRunAccepted, - PersistedPlaylistUpdateClusterSnapshot, PersistedPlaylistUpdateClusterState, - PersistedPlaylistUpdateQualityDecision, PersistedPlaylistUpdateQualitySnapshot, - PersistedPlaylistUpdateTechnicalState, PlaylistRequest, PlaylistUpdateDataSource, PlaylistUpdateRequestDto, - PlaylistUpdateRequestPayload, PlaylistUpdateRunId, PlaylistUpdateStatusDto, ProcessingOrder, XtreamCluster, - }, - utils::Internable, - }; - use std::{collections::HashMap, sync::Arc}; - use tempfile::tempdir; - use tokio::sync::mpsc; - use tokio_util::sync::CancellationToken; - use tower::ServiceExt; - use url::Url; - - fn manual_update_request( - input_id: u16, - policy: InputRefreshPolicy, - ) -> crate::api::model::ManualPlaylistUpdateRequest { - crate::api::model::ManualPlaylistUpdateRequest { - run_id: PlaylistUpdateRunId::generate(), - targets: Arc::new(crate::model::ProcessTargets { - enabled: true, - inputs: vec![input_id], - targets: Vec::new(), - target_names: Vec::new(), - }), - input_action: Some(shared::model::InputUpdateRequest { - input_id, - action: shared::model::InputUpdateAction::Provider(policy), - }), - } - } - - fn playlist_update_target(id: u16, name: &str) -> Arc { - Arc::new(ConfigTarget { - curation: None, - id, - enabled: true, - name: name.to_string(), - options: None, - sort: None, - filter: Filter::default().into(), - output: vec![], - rename: None, - mapping_ids: None, - mapping: Arc::default(), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - }) - } - - #[test] - fn manual_update_capabilities_reject_unsupported_actions_and_keep_all_required_inputs() { - use shared::model::{InputUpdateAction, InputUpdateRequest}; - for (input_type, allowed) in [ - (InputType::Xtream, 3), - (InputType::Stalker, 3), - (InputType::M3u, 3), - (InputType::Plex, 2), - (InputType::Library, 1), - (InputType::Jellyfin, 0), - (InputType::Emby, 0), - (InputType::M3uBatch, 0), - (InputType::XtreamBatch, 0), - (InputType::StalkerBatch, 0), - (InputType::Staged, 0), - ] { - for enabled in [false, true] { - let input = Arc::new(ConfigInput { - id: 2, - name: "selected".intern(), - input_type, - enabled, - ..ConfigInput::default() - }); - let companion = Arc::new(ConfigInput { - id: 3, - name: "companion".intern(), - input_type: InputType::M3u, - enabled: true, - ..ConfigInput::default() - }); - let sources = SourcesConfig { - inputs: vec![input.clone(), companion.clone()], - sources: vec![ConfigSource { - inputs: vec![input.name.clone(), companion.name.clone()], - targets: vec![playlist_update_target(20, "target")], - }], - ..SourcesConfig::default() - }; - for (index, action) in [ - InputUpdateAction::Provider(InputRefreshPolicy::NORMAL), - InputUpdateAction::Provider(InputRefreshPolicy::REFRESH), - InputUpdateAction::Provider(InputRefreshPolicy::FORCE), - InputUpdateAction::Rescan, - ] - .into_iter() - .enumerate() - { - let request = PlaylistUpdateRequestDto { - target_ids: Some(vec![20]), - input_action: Some(InputUpdateRequest { input_id: 2, action }), - ..PlaylistUpdateRequestDto::default() - }; - let result = super::resolve_manual_playlist_update_targets(&sources, &request); - let expected = - enabled && if input_type == InputType::Library { index == 3 } else { index < allowed }; - assert_eq!(result.is_ok(), expected, "{input_type} {action:?}"); - if let Ok(targets) = result { - assert_eq!(targets.targets, vec![20]); - assert_eq!(targets.inputs, vec![2, 3]); - } - } - } - } - } - - #[tokio::test] - async fn manual_update_library_rescan_uses_same_queue_and_requires_library_permission() { - use shared::model::{ - InputUpdateAction, InputUpdateRequest, LibraryConfigDto, WebAuthConfigDto, WebUiConfigDto, - }; - let input = Arc::new(ConfigInput { - id: 2, - name: "library".intern(), - input_type: InputType::Library, - enabled: true, - ..ConfigInput::default() - }); - let app_config = Arc::new(test_app_config( - input.clone(), - ConfigSource { inputs: vec![input.name.clone()], targets: vec![playlist_update_target(20, "target")] }, - )); - let mut config = (**app_config.config.load()).clone(); - config.library = - Some(crate::model::LibraryConfig::from(&LibraryConfigDto { enabled: true, ..LibraryConfigDto::default() })); - config.web_ui = Some(crate::model::WebUiConfig::from(&WebUiConfigDto { - auth: Some(WebAuthConfigDto::default()), - ..WebUiConfigDto::default() - })); - app_config.config.store(Arc::new(config)); - let mut sources = (**app_config.sources.load()).clone(); - sources.inputs.push(Arc::new(ConfigInput { - id: 3, - name: "other-input".intern(), - input_type: InputType::M3u, - enabled: true, - ..ConfigInput::default() - })); - sources.sources.push(ConfigSource { - inputs: vec!["other-input".intern()], - targets: vec![playlist_update_target(21, "unrelated-target")], - }); - app_config.sources.store(Arc::new(sources)); - let (sender, mut receiver) = mpsc::channel(1); - let app_state = test_app_state_with_manual_update_sender(app_config, sender); - let action = InputUpdateRequest { input_id: 2, action: InputUpdateAction::Rescan }; - let request = PlaylistUpdateRequestDto { - target_ids: Some(vec![20]), - input_action: Some(action), - ..PlaylistUpdateRequestDto::default() - }; - let mut claims: shared::model::Claims = - serde_json::from_value(serde_json::json!({"username":"test", "iss":"test", "iat":0, "exp":0})).unwrap(); - let rejected = super::playlist_update( - State(app_state.clone()), - Some(axum::Extension(super::VerifiedClaims(claims.clone()))), - Json(PlaylistUpdateRequestPayload::Current(request.clone())), - ) - .await - .into_response(); - assert_eq!(rejected.status(), StatusCode::FORBIDDEN); - assert!(receiver.try_recv().is_err()); - let unauthenticated = super::playlist_update( - State(app_state.clone()), - None, - Json(PlaylistUpdateRequestPayload::Current(request.clone())), - ) - .await - .into_response(); - assert_eq!(unauthenticated.status(), StatusCode::UNAUTHORIZED); - assert!(receiver.try_recv().is_err()); - claims.permissions.set(shared::model::permission::Permission::LibraryWrite); - // The legacy-name compatibility must never relax the new Rescan wire form. - let invalid_requests = [ - PlaylistUpdateRequestDto { target_ids: None, ..request.clone() }, - PlaylistUpdateRequestDto { targets: vec!["target".to_string()], target_ids: None, ..request.clone() }, - PlaylistUpdateRequestDto { target_ids: Some(Vec::new()), ..request.clone() }, - PlaylistUpdateRequestDto { target_ids: Some(vec![99]), ..request.clone() }, - PlaylistUpdateRequestDto { target_ids: Some(vec![21]), ..request.clone() }, - PlaylistUpdateRequestDto { targets: vec!["target".to_string()], ..request.clone() }, - PlaylistUpdateRequestDto { - input_refresh: Some(InputRefreshOverride { input_id: 2, policy: InputRefreshPolicy::NORMAL }), - ..request.clone() - }, - ]; - for invalid in invalid_requests { - let rejected = super::playlist_update( - State(app_state.clone()), - Some(axum::Extension(super::VerifiedClaims(claims.clone()))), - Json(PlaylistUpdateRequestPayload::Current(invalid)), - ) - .await - .into_response(); - assert_eq!(rejected.status(), StatusCode::BAD_REQUEST); - assert!(receiver.try_recv().is_err()); - } - let accepted = super::playlist_update( - State(app_state), - Some(axum::Extension(super::VerifiedClaims(claims))), - Json(PlaylistUpdateRequestPayload::Current(request)), - ) - .await - .into_response(); - assert_eq!(accepted.status(), StatusCode::ACCEPTED); - let queued = receiver.recv().await.unwrap(); - assert_eq!(queued.input_action, Some(action)); - assert_eq!(queued.targets.targets, vec![20]); - } - - #[tokio::test] - async fn manual_update_target_ids_keep_conflict_instead_of_accepting_a_dropped_force_request() { - for pending_policy in [InputRefreshPolicy::NORMAL, InputRefreshPolicy::REFRESH] { - let input = Arc::new(ConfigInput { - id: 17, - name: "force-input".intern(), - input_type: InputType::Xtream, - enabled: true, - ..ConfigInput::default() - }); - let app_config = Arc::new(test_app_config( - Arc::clone(&input), - ConfigSource { - inputs: vec![Arc::clone(&input.name)], - targets: vec![playlist_update_target(1, "queued-target")], - }, - )); - let (sender, mut receiver) = mpsc::channel(1); - sender.send(manual_update_request(1, InputRefreshPolicy::NORMAL)).await.unwrap(); - let running = receiver.recv().await.expect("running request"); - assert_eq!( - running.input_action.map(|request| request.action), - Some(shared::model::InputUpdateAction::Provider(InputRefreshPolicy::NORMAL)) - ); - sender.send(manual_update_request(2, pending_policy)).await.unwrap(); - let app_state = test_app_state_with_manual_update_sender(app_config, sender); - - let response = super::playlist_update( - State(app_state), - None, - Json(PlaylistUpdateRequestPayload::Current(PlaylistUpdateRequestDto { - targets: Vec::new(), - target_ids: Some(vec![1]), - input_refresh: Some(InputRefreshOverride { input_id: input.id, policy: InputRefreshPolicy::FORCE }), - input_action: None, - })), - ) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::CONFLICT); - let pending = receiver.recv().await.expect("pending request remains queued"); - assert_eq!( - pending.input_action, - Some(shared::model::InputUpdateRequest { - input_id: 2, - action: shared::model::InputUpdateAction::Provider(pending_policy) - }) - ); - assert!(receiver.try_recv().is_err()); - } - } - - #[tokio::test] - async fn manual_update_run_target_ids_and_refresh_override_keep_the_accepted_queue_identity() { - let input = Arc::new(ConfigInput { - id: 17, - name: "stable-input".intern(), - input_type: InputType::Xtream, - enabled: true, - ..ConfigInput::default() - }); - let app_config = Arc::new(test_app_config( - Arc::clone(&input), - ConfigSource { - inputs: vec![Arc::clone(&input.name)], - targets: vec![playlist_update_target(1, "shared-name"), playlist_update_target(4, "shared-name")], - }, - )); - let (sender, mut receiver) = mpsc::channel(1); - let app_state = test_app_state_with_manual_update_sender(app_config, sender); - let input_refresh = InputRefreshOverride { input_id: input.id, policy: InputRefreshPolicy::FORCE }; - - let response = super::playlist_update( - State(app_state), - None, - Json(PlaylistUpdateRequestPayload::Current(PlaylistUpdateRequestDto { - targets: Vec::new(), - target_ids: Some(vec![4, 1]), - input_refresh: Some(input_refresh), - input_action: None, - })), - ) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::ACCEPTED); - let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("accepted response body"); - let accepted = serde_json::from_slice::(&body).expect("accepted response payload"); - let queued = receiver.recv().await.expect("accepted update request"); - assert_eq!(accepted.run_id.as_ref(), Some(&queued.run_id)); - assert!(queued.targets.enabled); - assert_eq!(queued.targets.inputs, vec![input.id]); - assert_eq!(queued.targets.targets, vec![4, 1]); - assert_eq!(queued.targets.target_names, vec!["shared-name", "shared-name"]); - assert_eq!( - queued.input_action, - Some(shared::model::InputUpdateRequest { - input_id: input_refresh.input_id, - action: shared::model::InputUpdateAction::Provider(input_refresh.policy) - }) - ); - } - - #[tokio::test] - async fn manual_update_target_ids_reject_unknown_empty_and_ambiguous_selections() { - let input = Arc::new(ConfigInput { - id: 17, - name: "stable-input".intern(), - input_type: InputType::Xtream, - enabled: true, - ..ConfigInput::default() - }); - let app_config = Arc::new(test_app_config( - Arc::clone(&input), - ConfigSource { - inputs: vec![Arc::clone(&input.name)], - targets: vec![playlist_update_target(1, "known-target")], - }, - )); - let (sender, mut receiver) = mpsc::channel(1); - let app_state = test_app_state_with_manual_update_sender(app_config, sender); - let input_refresh = Some(InputRefreshOverride { input_id: input.id, policy: InputRefreshPolicy::REFRESH }); - let invalid_requests = [ - PlaylistUpdateRequestDto { - targets: vec!["known-target".to_string()], - input_action: Some(shared::model::InputUpdateRequest { - input_id: input.id, - action: shared::model::InputUpdateAction::Provider(InputRefreshPolicy::REFRESH), - }), - ..PlaylistUpdateRequestDto::default() - }, - PlaylistUpdateRequestDto { - targets: Vec::new(), - target_ids: Some(vec![99]), - input_refresh, - input_action: None, - }, - PlaylistUpdateRequestDto { - targets: Vec::new(), - target_ids: Some(Vec::new()), - input_refresh, - input_action: None, - }, - PlaylistUpdateRequestDto { - targets: vec!["known-target".to_string()], - target_ids: Some(vec![1]), - input_refresh, - input_action: None, - }, - ]; - - for request in invalid_requests { - let response = super::playlist_update( - State(Arc::clone(&app_state)), - None, - Json(PlaylistUpdateRequestPayload::Current(request)), - ) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - assert!(receiver.try_recv().is_err()); - } - } - - #[tokio::test] - async fn manual_update_target_ids_preserve_named_legacy_and_empty_bulk_requests() { - let input = Arc::new(ConfigInput { - id: 17, - name: "stable-input".intern(), - input_type: InputType::Xtream, - enabled: true, - ..ConfigInput::default() - }); - let app_config = Arc::new(test_app_config( - Arc::clone(&input), - ConfigSource { - inputs: vec![Arc::clone(&input.name)], - targets: vec![playlist_update_target(1, "shared-name"), playlist_update_target(4, "shared-name")], - }, - )); - let (sender, mut receiver) = mpsc::channel(1); - let app_state = test_app_state_with_manual_update_sender(app_config, sender); - let requests = [ - PlaylistUpdateRequestPayload::Current(PlaylistUpdateRequestDto { - targets: vec!["shared-name".to_string()], - target_ids: None, - input_refresh: None, - input_action: None, - }), - PlaylistUpdateRequestPayload::LegacyTargets(vec!["shared-name".to_string()]), - PlaylistUpdateRequestPayload::Current(PlaylistUpdateRequestDto { - targets: Vec::new(), - target_ids: None, - input_refresh: None, - input_action: None, - }), - ]; - - for (index, request) in requests.into_iter().enumerate() { - let response = - super::playlist_update(State(Arc::clone(&app_state)), None, Json(request)).await.into_response(); - - assert_eq!(response.status(), StatusCode::ACCEPTED); - let queued = receiver.recv().await.expect("accepted legacy request"); - assert_eq!(queued.input_action, None); - if index < 2 { - assert!(queued.targets.enabled); - assert_eq!(queued.targets.targets, vec![1, 4]); - assert_eq!(queued.targets.target_names, vec!["shared-name", "shared-name"]); - } else { - assert!(!queued.targets.enabled); - assert!(queued.targets.targets.is_empty()); - assert!(queued.targets.target_names.is_empty()); - } - } - } - - #[tokio::test] - async fn manual_update_m3u_force_is_queued_by_stable_ids_and_preserves_conflict() { - use shared::model::{InputUpdateAction, InputUpdateRequest}; - let input = Arc::new(ConfigInput { - id: 17, - name: "m3u-force".intern(), - input_type: InputType::M3u, - enabled: true, - ..ConfigInput::default() - }); - let config = Arc::new(test_app_config( - input.clone(), - ConfigSource { - inputs: vec![input.name.clone()], - targets: vec![playlist_update_target(1, "selected"), playlist_update_target(2, "other")], - }, - )); - let (sender, mut receiver) = mpsc::channel(1); - let app_state = test_app_state_with_manual_update_sender(config, sender); - let action = - InputUpdateRequest { input_id: input.id, action: InputUpdateAction::Provider(InputRefreshPolicy::FORCE) }; - let request = PlaylistUpdateRequestPayload::Current(PlaylistUpdateRequestDto { - target_ids: Some(vec![1]), - input_action: Some(action), - ..PlaylistUpdateRequestDto::default() - }); - let accepted = - super::playlist_update(State(app_state.clone()), None, Json(request.clone())).await.into_response(); - assert_eq!(accepted.status(), StatusCode::ACCEPTED); - let conflict = super::playlist_update(State(app_state), None, Json(request)).await.into_response(); - assert_eq!(conflict.status(), StatusCode::CONFLICT); - let queued = receiver.try_recv().unwrap(); - assert_eq!(queued.input_action, Some(action)); - assert_eq!(queued.targets.targets, [1]); - assert_eq!(queued.targets.inputs, [17]); - assert!(receiver.try_recv().is_err()); - } - - #[tokio::test] - async fn manual_update_legacy_input_refresh_preserves_named_and_all_scope_policy_and_queue_conflicts() { - let input = Arc::new(ConfigInput { - id: 17, - name: "provider".intern(), - input_type: InputType::Xtream, - enabled: true, - ..ConfigInput::default() - }); - let app_config = Arc::new(test_app_config( - input.clone(), - ConfigSource { - inputs: vec![input.name.clone()], - targets: vec![playlist_update_target(1, "selected"), playlist_update_target(4, "other")], - }, - )); - let (sender, mut receiver) = mpsc::channel(1); - let app_state = test_app_state_with_manual_update_sender(app_config, sender); - for policy in [InputRefreshPolicy::NORMAL, InputRefreshPolicy::REFRESH, InputRefreshPolicy::FORCE] { - for names in [vec!["selected".to_string()], Vec::new()] { - let expected = app_state - .app_config - .sources - .load() - .validate_targets((!names.is_empty()).then_some(&names)) - .unwrap(); - let request: PlaylistUpdateRequestPayload = serde_json::from_value(json!({ - "targets": names, - "input_refresh": {"input_id": input.id, "policy": policy} - })) - .unwrap(); - let response = - super::playlist_update(State(app_state.clone()), None, Json(request.clone())).await.into_response(); - assert_eq!(response.status(), StatusCode::ACCEPTED); - let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap(); - let accepted: OperationRunAccepted = serde_json::from_slice(&body).unwrap(); - let conflict = - super::playlist_update(State(app_state.clone()), None, Json(request)).await.into_response(); - assert_eq!(conflict.status(), StatusCode::CONFLICT); - let queued = receiver.try_recv().unwrap(); - assert_eq!(accepted.run_id.as_ref(), Some(&queued.run_id)); - assert_eq!(queued.targets.enabled, expected.enabled); - assert_eq!(queued.targets.inputs, expected.inputs); - assert_eq!(queued.targets.targets, expected.targets); - assert_eq!(queued.targets.target_names, expected.target_names); - assert_eq!( - queued.input_action, - Some(shared::model::InputUpdateRequest { - input_id: input.id, - action: shared::model::InputUpdateAction::Provider(policy), - }) - ); - assert!(receiver.try_recv().is_err()); - } - } - } - - #[tokio::test] - async fn m3u_update_quality_playlist_update_status_reads_newest_timestamp_from_canonically_resolved_input_path() { - let temp_dir = tempdir().expect("temp dir"); - let input = Arc::new(ConfigInput { - id: 7, - name: "timestamped/provider legacy".intern(), - input_type: InputType::Xtream, - enabled: true, - ..ConfigInput::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: Vec::new() }; - let app_config = test_app_config(Arc::clone(&input), source); - app_config.config.store(Arc::new(Config { - storage_dir: temp_dir.path().to_string_lossy().to_string(), - ..Config::default() - })); - let mut sources = app_config.sources.load().as_ref().clone(); - sources.inputs.push(Arc::new(ConfigInput { - id: 8, - name: "never-updated".intern(), - input_type: InputType::Stalker, - enabled: true, - ..ConfigInput::default() - })); - sources.inputs.push(Arc::new(ConfigInput { - id: 9, - name: "m3u-with-legacy-cluster".intern(), - input_type: InputType::M3u, - enabled: true, - ..ConfigInput::default() - })); - app_config.sources.store(Arc::new(sources)); - - let never_updated_path = temp_dir.path().join("input_never_updated"); - assert!(!never_updated_path.exists()); - let storage_path = crate::processing::input_cache::resolve_input_storage_path( - temp_dir.path().to_string_lossy().as_ref(), - &input.name, - ) - .await; - assert_eq!(storage_path.file_name().and_then(|name| name.to_str()), Some("input_timestamped_provider_legacy")); - let mut persisted = crate::processing::input_cache::InputStatus::default(); - persisted.clusters.insert( - XtreamCluster::Live.as_ref().to_string(), - crate::processing::input_cache::ClusterStatus { - status: crate::processing::input_cache::ClusterState::Ok, - timestamp: 101, - last_update: None, - }, - ); - persisted.clusters.insert( - XtreamCluster::Video.as_ref().to_string(), - crate::processing::input_cache::ClusterStatus { - status: crate::processing::input_cache::ClusterState::Failed, - timestamp: 303, - last_update: Some(PersistedPlaylistUpdateClusterSnapshot { - policy: Some(InputRefreshPolicy::REFRESH), - source: Some(PlaylistUpdateDataSource::Provider), - quality_guard_threshold: None, - quality: Some(PersistedPlaylistUpdateQualitySnapshot { - threshold: 95, - baseline_count: Some(4_812), - candidate_count: Some(3_104), - achieved_quality: Some(64), - decision: PersistedPlaylistUpdateQualityDecision::Rejected, - }), - active_count: Some(4_812), - technical_state: Some(PersistedPlaylistUpdateTechnicalState::Succeeded), - }), - }, - ); - persisted.clusters.insert( - XtreamCluster::Series.as_ref().to_string(), - crate::processing::input_cache::ClusterStatus { - status: crate::processing::input_cache::ClusterState::Ok, - timestamp: 202, - last_update: None, - }, - ); - crate::processing::input_cache::save_input_status(&storage_path, &persisted); - - let m3u_storage_path = crate::processing::input_cache::resolve_input_storage_path( - temp_dir.path().to_string_lossy().as_ref(), - "m3u-with-legacy-cluster", - ) - .await; - let mut m3u_persisted = crate::processing::input_cache::InputStatus::default(); - m3u_persisted.clusters.insert( - XtreamCluster::Live.as_ref().to_string(), - crate::processing::input_cache::ClusterStatus { - status: crate::processing::input_cache::ClusterState::Failed, - timestamp: 404, - last_update: None, - }, - ); - crate::processing::input_cache::save_input_status(&m3u_storage_path, &m3u_persisted); - - let app_state = test_app_state(Arc::new(app_config)); - let router = super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); - let response = router - .into_service::() - .oneshot(Request::builder().uri("/playlist/update/status").body(Body::empty()).expect("request")) - .await - .expect("response"); - - assert_eq!(response.status(), StatusCode::OK); - let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("body"); - let status = serde_json::from_slice::(&body).expect("status response"); - assert_eq!(status.inputs.len(), 3); - assert_eq!(status.inputs[0].input_id, 7); - assert_eq!(status.inputs[0].last_update, Some(303)); - assert_eq!(status.inputs[0].clusters.len(), 3); - assert_eq!(status.inputs[0].clusters[0].cluster, XtreamCluster::Live); - assert_eq!(status.inputs[0].clusters[0].status, PersistedPlaylistUpdateClusterState::Ok); - assert_eq!(status.inputs[0].clusters[0].timestamp, 101); - assert_eq!(status.inputs[0].clusters[1].cluster, XtreamCluster::Video); - assert_eq!(status.inputs[0].clusters[1].status, PersistedPlaylistUpdateClusterState::Failed); - assert_eq!(status.inputs[0].clusters[1].timestamp, 303); - assert_eq!( - status.inputs[0].clusters[1].last_update, - Some(PersistedPlaylistUpdateClusterSnapshot { - policy: Some(InputRefreshPolicy::REFRESH), - source: Some(PlaylistUpdateDataSource::Provider), - quality_guard_threshold: None, - quality: Some(PersistedPlaylistUpdateQualitySnapshot { - threshold: 95, - baseline_count: Some(4_812), - candidate_count: Some(3_104), - achieved_quality: Some(64), - decision: PersistedPlaylistUpdateQualityDecision::Rejected, - }), - active_count: Some(4_812), - technical_state: Some(PersistedPlaylistUpdateTechnicalState::Succeeded), - }) - ); - assert_eq!(status.inputs[0].clusters[2].cluster, XtreamCluster::Series); - assert_eq!(status.inputs[0].clusters[2].status, PersistedPlaylistUpdateClusterState::Ok); - assert_eq!(status.inputs[0].clusters[2].timestamp, 202); - assert_eq!(status.inputs[1].input_id, 8); - assert_eq!(status.inputs[1].last_update, None); - assert!(status.inputs[1].clusters.is_empty()); - assert_eq!(status.inputs[2].input_id, 9); - assert_eq!(status.inputs[2].last_update, Some(404)); - assert_eq!(status.inputs[2].clusters.len(), 1, "Only the actually persisted M3U cluster is exposed"); - assert_eq!(status.inputs[2].clusters[0].cluster, XtreamCluster::Live); - assert_eq!(status.inputs[2].clusters[0].status, PersistedPlaylistUpdateClusterState::Failed); - assert!(never_updated_path.is_dir()); - } - - #[tokio::test] - async fn playlist_update_status_reader_runs_off_the_async_worker() { - use crate::processing::input_cache::{ - load_input_status, save_input_status, ClusterState, ClusterStatus, InputStatus, - }; - - let temp = tempdir().unwrap(); - let expected = InputStatus { - clusters: HashMap::from([( - "live".to_owned(), - ClusterStatus { status: ClusterState::Ok, timestamp: 17, last_update: None }, - )]), - ..InputStatus::default() - }; - save_input_status(temp.path(), &expected); - let before = std::fs::read(temp.path().join("status.json")).unwrap(); - let worker = std::thread::current().id(); - let path = temp.path().to_path_buf(); - let actual = super::read_playlist_update_input_status(move || { - assert_ne!(std::thread::current().id(), worker); - load_input_status(&path) - }) - .await - .unwrap(); - - assert_eq!(actual.clusters, expected.clusters); - assert_eq!(std::fs::read(temp.path().join("status.json")).unwrap(), before); - } - - #[tokio::test] - async fn playlist_update_status_reader_join_failure_returns_server_error_not_empty_status() { - let result = super::read_playlist_update_input_status(|| panic!("simulated status reader failure")).await; - assert_eq!(result.unwrap_err(), StatusCode::INTERNAL_SERVER_ERROR); - } - - #[tokio::test] - async fn playlist_update_status_reload_returns_active_bus_facts_without_persisting_updating() { - use shared::model::{EventMessage, PlaylistUpdateProgressEvent, PlaylistUpdateState, PlaylistUpdateSummary}; - let temp = tempdir().unwrap(); - let input = Arc::new(ConfigInput { - id: 7, - name: "reload-input".intern(), - input_type: InputType::M3u, - enabled: true, - ..ConfigInput::default() - }); - let config = - test_app_config(input.clone(), ConfigSource { inputs: vec![input.name.clone()], targets: Vec::new() }); - config - .config - .store(Arc::new(Config { storage_dir: temp.path().to_string_lossy().into_owned(), ..Config::default() })); - let app = test_app_state(Arc::new(config)); - let path = - crate::processing::input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &input.name) - .await; - let persisted = crate::processing::input_cache::InputStatus { - last_input_update: Some(shared::model::PersistedPlaylistUpdateInputResult { - state: PlaylistUpdateState::Success, - timestamp: 1, - }), - ..crate::processing::input_cache::InputStatus::default() - }; - crate::processing::input_cache::save_input_status(&path, &persisted); - let before = std::fs::read(path.join("status.json")).unwrap(); - let mut progress = - PlaylistUpdateProgressEvent::for_run_input("running".into(), 17.into(), 7, "reload-input", "updating"); - app.event_manager.send_event(EventMessage::PlaylistUpdateProgress(progress.clone())); - let unknown = PlaylistUpdateProgressEvent::for_run_input("other".into(), 18.into(), 99, "removed", "updating"); - app.event_manager.send_event(EventMessage::PlaylistUpdateProgress(unknown)); - let Json(status) = super::playlist_update_status(State(app.clone())).await.unwrap(); - assert_eq!(status.active_updates, vec![progress.clone()]); - assert_eq!(status.inputs[0].last_input_update, persisted.last_input_update); - progress.state = Some(PlaylistUpdateState::Partial); - app.event_manager.send_event(EventMessage::PlaylistUpdateProgress(progress.clone())); - let Json(status) = super::playlist_update_status(State(app.clone())).await.unwrap(); - assert_eq!(status.active_updates, vec![progress], "completed input still belongs to a running target rebuild"); - app.event_manager.send_event(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::for_run( - "running".into(), - 17.into(), - PlaylistUpdateState::Failure, - ))); - let Json(status) = super::playlist_update_status(State(app)).await.unwrap(); - assert!(status.active_updates.is_empty()); - assert_eq!( - std::fs::read(path.join("status.json")).unwrap(), - before, - "read-only runtime projection never writes transient status" - ); - } - - #[tokio::test] - async fn playlist_update_status_reloads_staged_own_completion_without_default_or_parent_inference() { - use crate::processing::input_cache::{ - load_input_status, resolve_input_storage_path, save_input_status, ClusterState, ClusterStatus, InputStatus, - }; - use shared::model::{ - PersistedPlaylistUpdateClusterStatusDto, PersistedPlaylistUpdateInputResult, PlaylistUpdateState, - }; - for own_state in [Some(PlaylistUpdateState::Success), Some(PlaylistUpdateState::Failure), None] { - let temp = tempdir().unwrap(); - let parent = Arc::new(ConfigInput { - id: 7, - name: "parent".intern(), - input_type: InputType::Xtream, - ..ConfigInput::default() - }); - let app_config = test_app_config( - parent.clone(), - ConfigSource { inputs: vec![parent.name.clone()], targets: Vec::new() }, - ); - app_config.config.store(Arc::new(Config { - storage_dir: temp.path().to_string_lossy().into_owned(), - ..Config::default() - })); - let mut staged = ConfigInput { - id: 8, - name: "staged/provider own".intern(), - input_type: InputType::Staged, - staged_type: shared::model::StagedInputType::Xtream, - ..ConfigInput::default() - }; - staged.staged = Some(tuliprox_core::model::ConfigInputStaged { - for_input: Some(parent.name.clone()), - clusters: shared::model::ClusterFlags::Live, - }); - staged.resolve_staged_download_type(); - let mut sources = app_config.sources.load().as_ref().clone(); - sources.inputs.insert(0, Arc::new(staged.clone())); // Deliberately not ID or source order. - app_config.sources.store(Arc::new(sources)); - let path = resolve_input_storage_path(&temp.path().to_string_lossy(), &staged.name).await; - let snapshot = PersistedPlaylistUpdateClusterSnapshot { - policy: Some(InputRefreshPolicy::REFRESH), - source: Some(PlaylistUpdateDataSource::Cache), - quality_guard_threshold: Some(95), - ..PersistedPlaylistUpdateClusterSnapshot::default() - }; - let mut persisted = InputStatus::default(); - persisted.clusters.insert( - "live".into(), - ClusterStatus { status: ClusterState::Ok, timestamp: 123, last_update: Some(snapshot) }, - ); - persisted.last_input_update = - own_state.map(|state| PersistedPlaylistUpdateInputResult { state, timestamp: 456 }); - save_input_status(&path, &persisted); - let before = std::fs::read(path.join("status.json")).unwrap(); - let parent_path = resolve_input_storage_path(&temp.path().to_string_lossy(), &parent.name).await; - save_input_status( - &parent_path, - &InputStatus { - last_input_update: Some(PersistedPlaylistUpdateInputResult { - state: PlaylistUpdateState::Partial, - timestamp: 789, - }), - ..InputStatus::default() - }, - ); - let router = super::v1_api_playlist_register_protected(Router::new()) - .with_state(test_app_state(Arc::new(app_config))); - let response = router - .into_service::() - .oneshot(Request::builder().uri("/playlist/update/status").body(Body::empty()).unwrap()) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap(); - let dto: PlaylistUpdateStatusDto = serde_json::from_slice(&body).unwrap(); - let own = dto.inputs.iter().find(|input| input.input_id == 8).unwrap(); - assert_eq!(own.last_input_update, persisted.last_input_update); - assert_eq!(own.last_update, Some(123)); - assert_eq!( - own.clusters, - vec![PersistedPlaylistUpdateClusterStatusDto { - cluster: XtreamCluster::Live, - status: PersistedPlaylistUpdateClusterState::Ok, - timestamp: 123, - last_update: Some(snapshot) - }] - ); - assert_eq!( - dto.inputs.iter().find(|input| input.input_id == 7).unwrap().last_input_update.unwrap().state, - PlaylistUpdateState::Partial - ); - assert!(!load_input_status(&path).clusters.contains_key("default")); - assert_eq!( - std::fs::read(path.join("status.json")).unwrap(), - before, - "read endpoint must not rewrite canonical status" - ); - } - } - - #[tokio::test] - async fn playlist_update_status_exposes_last_input_results_and_legacy_default_without_inventing_clusters() { - use crate::processing::input_cache::{ - load_input_status, resolve_input_storage_path, save_input_status, ClusterState, ClusterStatus, InputStatus, - }; - use shared::model::{PersistedPlaylistUpdateInputResult, PlaylistUpdateState}; - for last_state in [ - None, - Some(PlaylistUpdateState::Success), - Some(PlaylistUpdateState::Partial), - Some(PlaylistUpdateState::Failure), - ] { - let temp = tempdir().unwrap(); - let app_config = test_app_config( - Arc::new(ConfigInput::default()), - ConfigSource { inputs: Vec::new(), targets: Vec::new() }, - ); - app_config.config.store(Arc::new(Config { - storage_dir: temp.path().to_string_lossy().into_owned(), - ..Config::default() - })); - let mut sources = (**app_config.sources.load()).clone(); - sources.inputs.clear(); - for (index, input_type) in [ - InputType::Xtream, - InputType::XtreamBatch, - InputType::Stalker, - InputType::StalkerBatch, - InputType::M3u, - InputType::M3uBatch, - InputType::Library, - InputType::Plex, - InputType::Emby, - InputType::Jellyfin, - InputType::Staged, - ] - .into_iter() - .enumerate() - { - let input = Arc::new(ConfigInput { - id: u16::try_from(index + 1).unwrap(), - name: format!("input-{index}").intern(), - input_type, - enabled: index % 2 == 0, - ..ConfigInput::default() - }); - let path = resolve_input_storage_path(&temp.path().to_string_lossy(), &input.name).await; - let mut persisted = InputStatus::default(); - persisted.clusters.insert( - "default".to_string(), - ClusterStatus { - status: if input.enabled { ClusterState::Ok } else { ClusterState::Failed }, - timestamp: 123, - last_update: None, - }, - ); - persisted.last_input_update = - last_state.map(|state| PersistedPlaylistUpdateInputResult { state, timestamp: 456 }); - save_input_status(&path, &persisted); - sources.inputs.push(input); - } - let inputs = sources.inputs.clone(); - app_config.sources.store(Arc::new(sources)); - let router = super::v1_api_playlist_register_protected(Router::new()) - .with_state(test_app_state(Arc::new(app_config))); - let response = router - .into_service::() - .oneshot(Request::builder().uri("/playlist/update/status").body(Body::empty()).unwrap()) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap(); - let dto: PlaylistUpdateStatusDto = serde_json::from_slice(&body).unwrap(); - assert_eq!(dto.inputs.len(), inputs.len()); - for (input, status) in inputs.iter().zip(dto.inputs) { - assert_eq!(status.input_id, input.id); - assert_eq!(status.last_update, Some(123), "Last update retains the cache timestamp contract"); - let expected = if let Some(state) = last_state { - Some(PersistedPlaylistUpdateInputResult { state, timestamp: 456 }) - } else if input.input_type.is_xtream() || input.input_type.is_stalker() { - None // Do not reconstruct an aggregate input result from incomplete legacy clusters. - } else { - Some(PersistedPlaylistUpdateInputResult { - state: if input.enabled { PlaylistUpdateState::Success } else { PlaylistUpdateState::Failure }, - timestamp: 123, - }) - }; - assert_eq!(status.last_input_update, expected); - assert!(status.clusters.is_empty(), "no synthetic Live/VOD/Series from default"); - let path = resolve_input_storage_path(&temp.path().to_string_lossy(), &input.name).await; - assert_eq!( - load_input_status(&path).last_input_update.map(|result| result.state), - last_state, - "read endpoint must not persist a legacy projection" - ); - } - } - } - - #[tokio::test] - async fn recording_headers_layer_under_input_headers_without_touching_the_worker_request() { - let mut recording = RecordingConfig::from(&shared::model::RecordingConfigDto::default()); - recording.headers = HashMap::from([ - ("User-Agent".to_string(), "VLC/3.0.16 LibVLC/3.0.16".to_string()), - ("Accept".to_string(), "video/*".to_string()), - ("X-Recording-Test".to_string(), "custom".to_string()), - ("invalid header".to_string(), "ignored".to_string()), - ]); - let state = crate::api::model::create_test_app_state(Config { - video: Some(tuliprox_core::model::VideoConfig { - extensions: vec![], - web_search: None, - recording: Some(recording), - }), - ..Default::default() - }); - let header = |input: &ConfigInput, name: &str| { - input.headers.iter().find(|(key, _)| key.eq_ignore_ascii_case(name)).map(|(_, value)| value.clone()) - }; - - let plain = Arc::new(ConfigInput { name: "input".into(), ..Default::default() }); - let merged = crate::api::api_utils::with_recording_headers(&state.app_config, Arc::clone(&plain)); - assert_eq!(header(&merged, "user-agent").as_deref(), Some("VLC/3.0.16 LibVLC/3.0.16")); - assert_eq!(header(&merged, "accept").as_deref(), Some("video/*")); - assert_eq!(header(&merged, "x-recording-test").as_deref(), Some("custom")); - assert_eq!(header(&merged, "invalid header"), None); - assert_eq!(merged.name, plain.name); - assert!(plain.headers.is_empty(), "the configured input must stay untouched"); - - let with_agent = Arc::new(ConfigInput { - name: "input".into(), - headers: HashMap::from([("user-agent".to_string(), "input-agent".to_string())]), - ..Default::default() - }); - let merged = crate::api::api_utils::with_recording_headers(&state.app_config, with_agent); - assert_eq!(header(&merged, "user-agent").as_deref(), Some("input-agent")); - assert_eq!(merged.headers.keys().filter(|key| key.eq_ignore_ascii_case("user-agent")).count(), 1); - assert_eq!(header(&merged, "accept").as_deref(), Some("video/*")); - } - - #[tokio::test] - async fn stable_recording_route_rejects_target_and_input_from_different_sources() { - let app_config = Arc::new(test_app_config( - Arc::new(ConfigInput { id: 7, name: "input-a".intern(), ..Default::default() }), - ConfigSource { - inputs: vec!["input-a".intern()], - targets: vec![Arc::new(ConfigTarget { - curation: None, - id: 11, - enabled: true, - name: "stable-target".to_string(), - options: None, - sort: None, - filter: Filter::default().into(), - output: vec![], - rename: None, - mapping_ids: None, - mapping: Arc::default(), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - })], - }, - )); - let app_state = test_app_state(Arc::clone(&app_config)); - let token = - crate::auth::create_access_token(&app_config.access_token_secret, 1, crate::auth::scope::INTERNAL_PLAYER); - let fingerprint = crate::auth::Fingerprint::new( - "test".to_string(), - "127.0.0.1".to_string(), - "127.0.0.1:1234".parse().expect("test socket address"), - ); - - let response = super::playlist_recording_stream( - fingerprint, - AxumPath((token, "live".to_string(), 42)), - Query(super::RecordingStreamQuery { - target_name: "stable-target".to_string(), - input_name: "input-b".to_string(), - provider_allocation_id: None, - }), - State(app_state), - axum::http::HeaderMap::new(), - ) - .await - .into_response(); - - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - } - - /// Generate an XMLTV datetime string in the format `YYYYMMDDHHmmss +0000` - /// offset by `hours_from_now` hours from the current time. - fn epg_dt(hours_from_now: i64) -> String { - let dt = Utc::now() + chrono::Duration::hours(hours_from_now); - dt.format("%Y%m%d%H%M%S %z").to_string() - } - - fn xmltv_source_dto(url: &str, priority: i16) -> EpgSourceDto { - EpgSourceDto { url: url.to_string(), priority, ..EpgSourceDto::default() } - } - - fn ics_source_dto(url: &str, channel_id: &str, priority: i16) -> EpgSourceDto { - EpgSourceDto { - source_type: EpgSourceTypeDto::Ics, - url: url.to_string(), - priority, - channel_id: Some(channel_id.to_string()), - channel_title: Some("Formula 1".to_string()), - ics: Some(IcsEpgSourceConfigDto::default()), - ..EpgSourceDto::default() - } - } - - fn test_app_config(input: Arc, source: ConfigSource) -> AppConfig { - let inputs = vec![input]; - let sources = SourcesConfig { - batch_files: vec![], - provider: vec![], - group_lookup: build_group_lookup(&inputs), - inputs, - sources: vec![source], - templates: None, - }; - - AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config::default())), - sources: Arc::new(ArcSwap::from_pointee(sources)), - hdhomerun: Arc::new(ArcSwapOption::empty()), - api_proxy: Arc::new(ArcSwapOption::empty()), - file_locks: Arc::new(crate::utils::FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::empty()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(crate::model::MediaToolCapabilities::default()), - } - } - - #[test] - fn recording_target_resolution_uses_stable_name_and_input_across_runtime_ids() { - let input_a = Arc::new(ConfigInput { id: 7, name: "input-a".intern(), ..Default::default() }); - let input_b = Arc::new(ConfigInput { id: 8, name: "input-b".intern(), ..Default::default() }); - let target = |id, name: &str| { - Arc::new(ConfigTarget { - curation: None, - id, - enabled: true, - name: name.to_string(), - options: None, - sort: None, - filter: Filter::default().into(), - output: vec![], - rename: None, - mapping_ids: None, - mapping: Arc::default(), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - }) - }; - let sources = |later_target_id| { - let inputs = vec![Arc::clone(&input_a), Arc::clone(&input_b)]; - SourcesConfig { - batch_files: vec![], - provider: vec![], - group_lookup: build_group_lookup(&inputs), - inputs, - sources: vec![ - ConfigSource { inputs: vec!["input-a".intern()], targets: vec![target(11, "target-a")] }, - ConfigSource { - inputs: vec!["input-b".intern()], - targets: vec![target(later_target_id, "stable-target")], - }, - ], - templates: None, - } - }; - let mut app_config = test_app_config( - Arc::new(ConfigInput { id: 0, name: "unused".intern(), ..Default::default() }), - ConfigSource { inputs: vec![], targets: vec![] }, - ); - app_config.sources = Arc::new(ArcSwap::from_pointee(sources(12))); - - let snapshot = app_config.sources.load(); - let resolved = resolve_recording_config(snapshot.as_ref(), "stable-target", "input-b") - .expect("stable source in first snapshot"); - assert_eq!(resolved.target.id, 12); - assert_eq!(resolved.input.name.as_ref(), "input-b"); - assert!(resolve_recording_config(snapshot.as_ref(), "stable-target", "input-a").is_none()); - drop(snapshot); - - app_config.sources.store(Arc::new(sources(37))); - let snapshot = app_config.sources.load(); - let resolved = resolve_recording_config(snapshot.as_ref(), "stable-target", "input-b") - .expect("stable source after reload"); - assert_eq!(resolved.target.id, 37); - } - - fn test_app_state(app_cfg: Arc) -> Arc { - let (manual_update_sender, _) = mpsc::channel::(1); - test_app_state_with_manual_update_sender(app_cfg, manual_update_sender) - } - - fn test_app_state_with_manual_update_sender( - app_cfg: Arc, - manual_update_sender: mpsc::Sender, - ) -> Arc { - let event_manager = Arc::new(EventManager::new()); - let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); - let history_config = Some(StreamHistoryConfig::default()); - active_provider.set_shared_stream_manager(&shared_stream_manager); - - let geoip = Arc::new(ArcSwapOption::::default()); - let config = app_cfg.config.load(); - let active_users = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); - let connection_manager = Arc::new(ConnectionManager::new( - &active_users, - &active_provider, - &shared_stream_manager, - &event_manager, - history_config.as_ref(), - )); - - let tokens = crate::api::model::CancelTokens { - scheduler: CancellationToken::new(), - hdhomerun: CancellationToken::new(), - file_watch: CancellationToken::new(), - provider_dns: CancellationToken::new(), - metadata: CancellationToken::new(), - qos_aggregation: CancellationToken::new(), - recordings: CancellationToken::new(), - hls_cache: CancellationToken::new(), - }; - let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); - - Arc::new(AppState { - recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( - Arc::clone(&active_provider), - Arc::clone(&connection_manager), - ), - app_config: app_cfg, - http_clients: Arc::default(), - recordings: Arc::new(crate::api::model::RecordingQueue::new()), - cache: Arc::new(ArcSwapOption::default()), - shared_stream_manager, - hls: crate::api::model::HlsState::new(Arc::new(crate::api::model::HlsProxyManager::new())), - stalker_resolve_coordinator: crate::api::model::StalkerResolveCoordinator::default(), - active_users, - active_provider, - connection_manager, - event_manager, - cancel_tokens: ArcSwap::from_pointee(tokens), - playlists: Arc::new(PlaylistStorageState::new()), - geoip, - metadata_manager, - auth: crate::api::model::AuthState::for_tests(), - playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests_with_sender(manual_update_sender), - }) - } - - async fn stalker_mock_handler(Query(params): Query>) -> impl IntoResponse { - let action = params.get("action").map_or("", String::as_str); - let portal_type = params.get("type").map_or("", String::as_str); - let page = params.get("p").map_or("1", String::as_str); - let response = match (portal_type, action, page) { - ("stb", "handshake", _) => json!({"js": {"token": "preview-token"}}), - ("stb", "get_profile", _) => json!({"js": {"status": 1, "max_connections": 1}}), - ("stb", "get_capabilities", _) => json!({"js": {}}), - ("itv", "get_genres", _) => json!({"js": [{"id": "10", "title": "News"}]}), - ("itv", "get_ordered_list", "1") => json!({ - "js": { - "data": { - "101": { - "id": "101", - "name": "Demo Channel", - "category_id": "10", - "cmd": "ffmpeg http://streams.example/live/101" - }, - "102": { - "id": "102", - "name": "Private Channel", - "category_id": "10", - "cmd": "ffmpeg http://streams.example/live/102" - } - } - } - }), - ("itv", "create_link", _) => { - let destination = if params.get("cmd").is_some_and(|cmd| cmd.ends_with("/102")) { - "http://127.0.0.1/live/102" - } else { - "http://8.8.8.8/live/101" - }; - json!({"js": {"cmd": format!("ffmpeg {destination}")}}) - } - _ => json!({"js": []}), - }; - Json(response) - } - - async fn spawn_stalker_mock_server() -> (String, tokio::task::JoinHandle<()>) { - let router = Router::new().route("/server/load.php", axum::routing::get(stalker_mock_handler)); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind mock stalker server"); - let base_url = format!("http://{}", listener.local_addr().expect("mock addr")); - let handle = tokio::spawn(async move { - axum::serve(listener, router).await.expect("serve mock stalker server"); - }); - (base_url, handle) - } - - #[test] - fn resolve_provider_url_for_input_request_rewrites_provider_scheme() { - let provider = ConfigProvider::from(&ConfigProviderDto { - name: "demo".intern(), - urls: vec!["http://provider.example".intern()], - provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), - dns: None, - }); - let input = Arc::new(ConfigInput { - id: 7, - name: "input".intern(), - provider_configs: Some(vec![Arc::new(provider)]), - ..Default::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_config = test_app_config(input, source); - let resolved = resolve_provider_url_for_request( - &app_config, - &PlaylistRequest::Input("input".to_string()), - "provider://demo/live/user/pass/1359.ts", - ); - - assert_eq!(resolved, "http://provider.example/live/user/pass/1359.ts"); - } - - #[test] - fn resolve_provider_url_for_target_request_rewrites_provider_scheme() { - let provider = ConfigProvider::from(&ConfigProviderDto { - name: "demo".intern(), - urls: vec!["http://provider.example".intern()], - provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), - dns: None, - }); - let input = Arc::new(ConfigInput { - id: 7, - name: "input".intern(), - provider_configs: Some(vec![Arc::new(provider)]), - ..Default::default() - }); - let target = Arc::new(ConfigTarget { - id: 11, - enabled: true, - name: "target".to_string(), - options: None, - sort: None, - filter: Filter::default().into(), - output: vec![], - rename: None, - mapping_ids: None, - mapping: Arc::default(), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - curation: None, - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![target] }; - let app_config = test_app_config(input, source); - let resolved = resolve_provider_url_for_request( - &app_config, - &PlaylistRequest::Target(11), - "provider://demo/live/user/pass/1359.ts", - ); - - assert_eq!(resolved, "http://provider.example/live/user/pass/1359.ts"); - } - - #[test] - fn resolve_provider_url_passthrough_for_unresolved_provider_input_request() { - let provider = ConfigProvider::from(&ConfigProviderDto { - name: "demo".intern(), - urls: vec!["http://provider.example".intern()], - provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), - dns: None, - }); - let input = Arc::new(ConfigInput { - id: 7, - name: "input".intern(), - provider_configs: Some(vec![Arc::new(provider)]), - ..Default::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_config = test_app_config(input, source); - let original = "provider://unknown/live/user/pass/1359.ts"; - let resolved = - resolve_provider_url_for_request(&app_config, &PlaylistRequest::Input("input".to_string()), original); - - assert_eq!(resolved, original); - } - - #[test] - fn resolve_provider_url_passthrough_for_unresolved_provider_target_request() { - let provider = ConfigProvider::from(&ConfigProviderDto { - name: "demo".intern(), - urls: vec!["http://provider.example".intern()], - provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), - dns: None, - }); - let input = Arc::new(ConfigInput { - id: 7, - name: "input".intern(), - provider_configs: Some(vec![Arc::new(provider)]), - ..Default::default() - }); - let target = Arc::new(ConfigTarget { - id: 11, - enabled: true, - name: "target".to_string(), - options: None, - sort: None, - filter: Filter::default().into(), - output: vec![], - rename: None, - mapping_ids: None, - mapping: Arc::default(), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - curation: None, - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![target] }; - let app_config = test_app_config(input, source); - let original = "provider://unknown/live/user/pass/1359.ts"; - let resolved = resolve_provider_url_for_request(&app_config, &PlaylistRequest::Target(11), original); - - assert_eq!(resolved, original); - } - - #[test] - fn resolve_provider_url_passthrough_for_ambiguous_target_request() { - let provider_a = ConfigProvider::from(&ConfigProviderDto { - name: "shared".intern(), - urls: vec!["http://provider-a.example".intern()], - provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), - dns: None, - }); - let provider_b = ConfigProvider::from(&ConfigProviderDto { - name: "shared".intern(), - urls: vec!["http://provider-b.example".intern()], - provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), - dns: None, - }); - let input_a = Arc::new(ConfigInput { - id: 7, - name: "input-a".intern(), - provider_configs: Some(vec![Arc::new(provider_a)]), - ..Default::default() - }); - let input_b = Arc::new(ConfigInput { - id: 8, - name: "input-b".intern(), - provider_configs: Some(vec![Arc::new(provider_b)]), - ..Default::default() - }); - let target = Arc::new(ConfigTarget { - curation: None, - id: 11, - enabled: true, - name: "target".to_string(), - options: None, - sort: None, - filter: Filter::default().into(), - output: vec![], - rename: None, - mapping_ids: None, - mapping: Arc::default(), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - }); - let source = - ConfigSource { inputs: vec![Arc::clone(&input_a.name), Arc::clone(&input_b.name)], targets: vec![target] }; - let inputs = vec![input_a, input_b]; - let sources = SourcesConfig { - batch_files: vec![], - provider: vec![], - group_lookup: build_group_lookup(&inputs), - inputs, - sources: vec![source], - templates: None, - }; - - let app_config = AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config::default())), - sources: Arc::new(ArcSwap::from_pointee(sources)), - hdhomerun: Arc::new(ArcSwapOption::empty()), - api_proxy: Arc::new(ArcSwapOption::empty()), - file_locks: Arc::new(crate::utils::FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::empty()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(crate::model::MediaToolCapabilities::default()), - }; - - let original = "provider://shared/live/user/pass/1359.ts"; - let resolved = resolve_provider_url_for_request(&app_config, &PlaylistRequest::Target(11), original); - - assert_eq!(resolved, original); - } - - #[test] - fn build_playlist_webplayer_url_uses_cluster_stream_type() { - let live = super::build_playlist_webplayer_url("http://player.example", "token123", 1, 42, XtreamCluster::Live); - let movie = - super::build_playlist_webplayer_url("http://player.example", "token123", 1, 42, XtreamCluster::Video); - let series = - super::build_playlist_webplayer_url("http://player.example", "token123", 1, 42, XtreamCluster::Series); - - assert_eq!(live, "http://player.example/api/v1/playlist/webplayer/token123/1/live/42"); - assert_eq!(movie, "http://player.example/api/v1/playlist/webplayer/token123/1/movie/42"); - assert_eq!(series, "http://player.example/api/v1/playlist/webplayer/token123/1/series/42"); - } - - #[test] - fn recording_source_descriptor_is_token_free_and_percent_encodes_names() { - let url = crate::api::model::recording::recording_source_resolution::build_recording_source_descriptor( - "News/HD &+", - "input/name ?+", - 42, - XtreamCluster::Video, - ) - .expect("valid recording source descriptor"); - let parsed = Url::parse(&url).expect("parse recording source descriptor"); - let query = parsed.query_pairs().collect::>(); - - assert_eq!(parsed.scheme(), "tuliprox-recording"); - assert_eq!(parsed.host_str(), Some("source")); - assert_eq!(query.get("target_name").map(std::convert::AsRef::as_ref), Some("News/HD &+")); - assert_eq!(query.get("input_name").map(std::convert::AsRef::as_ref), Some("input/name ?+")); - assert_eq!(query.get("virtual_id").map(std::convert::AsRef::as_ref), Some("42")); - assert_eq!(query.get("cluster").map(std::convert::AsRef::as_ref), Some("movie")); - assert!(!url.contains("token")); - } - - #[test] - fn future_scheduled_recording_descriptor_round_trips_without_token() { - let url = crate::api::model::recording::recording_source_resolution::build_recording_source_descriptor( - "stable-target", - "input-a", - 42, - XtreamCluster::Live, - ) - .expect("valid recording url"); - let download_cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { - directory: Some("/tmp".to_string()), - ..Default::default() - }); - let metadata = shared::model::RecordingMetadata::new_live( - shared::model::RecordingOwner::User(shared::model::UserId::from("web:alice")), - shared::model::RecordingVisibility::Private, - shared::model::recording::RecordingSource::new("stable-target", "42", "input-a"), - 1_700_000_000, - 1_700_003_600, - 0, - 0, - ); - let recording = crate::api::model::RecordingTask::new( - shared::model::RecordingKind::Live, - &url, - "recording.ts", - &download_cfg, - Some("input-a".intern()), - 0, - metadata, - ) - .expect("valid recording task"); - let persisted = RecordingQueue::to_persisted(&recording); - let restored = RecordingQueue::from_persisted(persisted.clone()).expect("restore recording task"); - - assert_eq!(persisted.url, url); - assert_eq!(restored.url.as_str(), url); - assert!(persisted.url.starts_with("tuliprox-recording://source?")); - assert!(!persisted.url.contains("token")); - assert!(!persisted.url.contains("/11/")); - assert!(persisted.url.contains("target_name=stable-target")); - } - - #[test] - fn epg_channel_match_applies_the_target_output_case() { - let item = "NPO3.nl".intern(); - let requested_lowercase = - crate::utils::canonicalize_untrusted_epg_id("npo3.nl", crate::utils::EpgIdOutputCase::LowercaseAscii); - - assert!(super::epg_channel_id_matches( - Some(&item), - &requested_lowercase, - crate::utils::EpgIdOutputCase::LowercaseAscii - )); - assert!(!super::epg_channel_id_matches( - Some(&item), - &requested_lowercase, - crate::utils::EpgIdOutputCase::Preserve - )); - } - - #[test] - fn epg_channel_match_rejects_absent_ids() { - let requested = "npo3.nl".intern(); - assert!(!super::epg_channel_id_matches(None, &requested, crate::utils::EpgIdOutputCase::Preserve)); - } - - #[test] - fn merge_epg_channels_prefers_higher_priority_metadata_and_fills_lower_priority_gaps() { - let low_priority = EpgChannel { - id: "demo.channel".intern(), - title: Some("Low".intern()), - icon: Some("http://low/icon.png".intern()), - programmes: vec![EpgProgramme::new_all( - 10, - 20, - "demo.channel".intern(), - Some("Low Show".intern()), - None, - None, - )], - }; - let high_priority = EpgChannel { - id: "demo.channel".intern(), - title: Some("High".intern()), - icon: Some("http://high/icon.png".intern()), - programmes: vec![EpgProgramme::new_all( - 30, - 40, - "demo.channel".intern(), - Some("High Show".intern()), - None, - None, - )], - }; - let same_priority = EpgChannel { - id: "demo.channel".intern(), - title: Some("Same".intern()), - icon: Some("http://same/icon.png".intern()), - programmes: vec![ - EpgProgramme::new_all(30, 40, "demo.channel".intern(), Some("Duplicate".intern()), None, None), - EpgProgramme::new_all(50, 60, "demo.channel".intern(), Some("Second Show".intern()), None, None), - ], - }; - - let channels = super::merge_epg_channels(vec![ - (10, vec![low_priority]), - (0, vec![high_priority]), - (0, vec![same_priority]), - ]); - - assert_eq!(channels.len(), 1); - assert_eq!(channels[0].title.as_deref(), Some("High")); - assert_eq!(channels[0].icon.as_deref(), Some("http://high/icon.png")); - assert_eq!(channels[0].programmes.len(), 3); - assert_eq!( - channels[0].programmes.iter().map(|programme| (programme.start, programme.stop)).collect::>(), - vec![(10, 20), (30, 40), (50, 60)], - ); - } - - #[tokio::test] - async fn load_epg_channels_for_input_uses_resolved_provider_epg_cache_file() { - let temp_dir = tempdir().expect("temp dir"); - let provider = ConfigProvider::from(&ConfigProviderDto { - name: "demo".intern(), - urls: vec!["http://provider.example".intern()], - provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), - dns: None, - }); - let input = Arc::new(ConfigInput { - id: 7, - name: "input".intern(), - epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { - sources: Some(vec![xmltv_source_dto("provider://demo/xmltv.php?username=user&password=pass", 0)]), - t_sources: vec![xmltv_source_dto("provider://demo/xmltv.php?username=user&password=pass", 0)], - smart_match: None, - })), - provider_configs: Some(vec![Arc::new(provider)]), - ..Default::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_config = test_app_config(input.clone(), source); - app_config.config.store(Arc::new(Config { - storage_dir: temp_dir.path().to_string_lossy().to_string(), - ..Default::default() - })); - let raw_epg_path = get_input_raw_xmltv_file_path( - "provider://demo/xmltv.php?username=user&password=pass", - input.as_ref(), - temp_dir.path().to_string_lossy().as_ref(), - ) - .await - .expect("epg path"); - if let Some(parent) = raw_epg_path.parent() { - tokio::fs::create_dir_all(parent).await.expect("epg dir"); - } - let prog_start = epg_dt(0); - let prog_stop = epg_dt(1); - tokio::fs::write( - &raw_epg_path, - format!( - r#" - - - Demo Channel - - - Morning Show - -"# - ), - ) - .await - .expect("write epg"); - - let app_state = test_app_state(Arc::new(app_config)); - let channels = - super::load_epg_channels_for_input(&app_state, input.as_ref()).await.expect("load epg").expect("channels"); - - assert_eq!(channels.len(), 1); - assert_eq!(channels[0].id.as_ref(), "demo.channel"); - assert_eq!(channels[0].programmes.len(), 1); - assert_eq!(channels[0].programmes[0].title.as_ref().map(std::convert::AsRef::as_ref), Some("Morning Show")); - } - - #[tokio::test] - async fn load_epg_channels_for_input_reads_cached_ics_source() { - let temp_dir = tempdir().expect("temp dir"); - let ics_dto = ics_source_dto("https://example.com/f1.ics", "f1.calendar", -10); - let input = Arc::new(ConfigInput { - id: 7, - name: "input".intern(), - epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { - sources: Some(vec![ics_dto.clone()]), - t_sources: vec![ics_dto], - smart_match: None, - })), - ..Default::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_config = test_app_config(input.clone(), source); - app_config.config.store(Arc::new(Config { - storage_dir: temp_dir.path().to_string_lossy().to_string(), - ..Default::default() - })); - let epg_source = &input.epg.as_ref().expect("epg").sources[0]; - let raw_epg_path = - get_input_raw_epg_file_path(epg_source, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) - .await - .expect("epg path"); - if let Some(parent) = raw_epg_path.parent() { - tokio::fs::create_dir_all(parent).await.expect("epg dir"); - } - tokio::fs::write( - &raw_epg_path, - "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:Practice 1\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT\nEND:VCALENDAR", - ) - .await - .expect("write ics"); - - let app_state = test_app_state(Arc::new(app_config)); - let channels = - super::load_epg_channels_for_input(&app_state, input.as_ref()).await.expect("load epg").expect("channels"); - - assert_eq!(channels.len(), 1); - assert_eq!(channels[0].id.as_ref(), "f1.calendar"); - assert_eq!(channels[0].title.as_deref(), Some("Formula 1")); - assert_eq!(channels[0].programmes[0].title.as_deref(), Some("Practice 1")); - } - - #[tokio::test] - async fn load_epg_channels_for_input_redownloads_invalid_cached_ics_source() { - let temp_dir = tempdir().expect("temp dir"); - tokio::fs::write( - temp_dir.path().join("valid.ics"), - "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:Downloaded\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT\nEND:VCALENDAR", - ) - .await - .expect("write valid ics source"); - let ics_dto = ics_source_dto("valid.ics", "f1.calendar", -10); - let input = Arc::new(ConfigInput { - id: 7, - name: "input".intern(), - epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { - sources: Some(vec![ics_dto.clone()]), - t_sources: vec![ics_dto], - smart_match: None, - })), - ..Default::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_config = test_app_config(input.clone(), source); - app_config.config.store(Arc::new(Config { - storage_dir: temp_dir.path().to_string_lossy().to_string(), - ..Default::default() - })); - let epg_source = &input.epg.as_ref().expect("epg").sources[0]; - let raw_epg_path = - get_input_raw_epg_file_path(epg_source, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) - .await - .expect("epg path"); - if let Some(parent) = raw_epg_path.parent() { - tokio::fs::create_dir_all(parent).await.expect("epg dir"); - } - tokio::fs::write(&raw_epg_path, "upstream returned an error page").await.expect("write invalid cache"); - - let app_state = test_app_state(Arc::new(app_config)); - let channels = - super::load_epg_channels_for_input(&app_state, input.as_ref()).await.expect("load epg").expect("channels"); - - assert_eq!(channels.len(), 1); - assert_eq!(channels[0].programmes[0].title.as_deref(), Some("Downloaded")); - assert_eq!( - tokio::fs::read_to_string(&raw_epg_path).await.expect("updated cache"), - "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:Downloaded\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT\nEND:VCALENDAR" - ); - } - - #[tokio::test] - async fn load_epg_channels_for_input_preserves_invalid_cache_when_refresh_fails() { - let temp_dir = tempdir().expect("temp dir"); - let invalid_cache = "upstream returned an error page"; - let ics_dto = ics_source_dto("missing.ics", "f1.calendar", -10); - let input = Arc::new(ConfigInput { - id: 7, - name: "input".intern(), - epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { - sources: Some(vec![ics_dto.clone()]), - t_sources: vec![ics_dto], - smart_match: None, - })), - ..Default::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_config = test_app_config(input.clone(), source); - app_config.config.store(Arc::new(Config { - storage_dir: temp_dir.path().to_string_lossy().to_string(), - ..Default::default() - })); - let epg_source = &input.epg.as_ref().expect("epg").sources[0]; - let raw_epg_path = - get_input_raw_epg_file_path(epg_source, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) - .await - .expect("epg path"); - if let Some(parent) = raw_epg_path.parent() { - tokio::fs::create_dir_all(parent).await.expect("epg dir"); - } - tokio::fs::write(&raw_epg_path, invalid_cache).await.expect("write invalid cache"); - - let app_state = test_app_state(Arc::new(app_config)); - assert!(super::load_epg_channels_for_input(&app_state, input.as_ref()).await.is_err()); - assert_eq!(tokio::fs::read_to_string(&raw_epg_path).await.expect("preserved cache"), invalid_cache); - } - - #[tokio::test] - async fn load_epg_channels_for_input_applies_ics_dummy_policy_in_preview() { - let temp_dir = tempdir().expect("temp dir"); - tokio::fs::write(temp_dir.path().join("empty.ics"), "BEGIN:VCALENDAR\nEND:VCALENDAR") - .await - .expect("write empty ics source"); - let mut ics_dto = ics_source_dto("empty.ics", "f1.calendar", -10); - ics_dto.ics = Some(IcsEpgSourceConfigDto { - dummy: IcsDummyConfigDto { - enabled: true, - title: "No F1".to_string(), - days_past: 0, - days_future: 0, - block_hours: 24, - ..IcsDummyConfigDto::default() - }, - ..IcsEpgSourceConfigDto::default() - }); - let input = Arc::new(ConfigInput { - id: 7, - name: "input".intern(), - epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { - sources: Some(vec![ics_dto.clone()]), - t_sources: vec![ics_dto], - smart_match: None, - })), - ..Default::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_config = test_app_config(input.clone(), source); - app_config.config.store(Arc::new(Config { - storage_dir: temp_dir.path().to_string_lossy().to_string(), - ..Default::default() - })); - - let app_state = test_app_state(Arc::new(app_config)); - let channels = - super::load_epg_channels_for_input(&app_state, input.as_ref()).await.expect("load epg").expect("channels"); - - assert_eq!(channels.len(), 1); - assert_eq!(channels[0].id.as_ref(), "f1.calendar"); - assert!(!channels[0].programmes.is_empty()); - assert!(channels[0].programmes.iter().all(|programme| programme.title.as_deref() == Some("No F1"))); - } - - #[tokio::test] - async fn load_epg_channels_for_input_prefers_high_priority_ics_dummy_policy() { - let temp_dir = tempdir().expect("temp dir"); - for filename in ["low.ics", "high.ics"] { - tokio::fs::write(temp_dir.path().join(filename), "BEGIN:VCALENDAR\nEND:VCALENDAR") - .await - .expect("write empty ics source"); - } - - let mut low_priority = ics_source_dto("low.ics", "f1.calendar", 10); - low_priority.ics = Some(IcsEpgSourceConfigDto { - dummy: IcsDummyConfigDto { - enabled: true, - title: "Low priority".to_string(), - days_past: 0, - days_future: 0, - block_hours: 24, - ..IcsDummyConfigDto::default() - }, - ..IcsEpgSourceConfigDto::default() - }); - let mut high_priority = ics_source_dto("high.ics", "f1.calendar", -10); - high_priority.ics = Some(IcsEpgSourceConfigDto { - dummy: IcsDummyConfigDto { - enabled: true, - title: "High priority".to_string(), - days_past: 0, - days_future: 0, - block_hours: 24, - ..IcsDummyConfigDto::default() - }, - ..IcsEpgSourceConfigDto::default() - }); - let input = Arc::new(ConfigInput { - id: 7, - name: "input".intern(), - epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { - sources: Some(vec![low_priority.clone(), high_priority.clone()]), - t_sources: vec![low_priority, high_priority], - smart_match: None, - })), - ..Default::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_config = test_app_config(input.clone(), source); - app_config.config.store(Arc::new(Config { - storage_dir: temp_dir.path().to_string_lossy().to_string(), - ..Default::default() - })); - - let app_state = test_app_state(Arc::new(app_config)); - let channels = - super::load_epg_channels_for_input(&app_state, input.as_ref()).await.expect("load epg").expect("channels"); - - assert_eq!(channels.len(), 1); - assert!(!channels[0].programmes.is_empty()); - assert!(channels[0].programmes.iter().all(|programme| programme.title.as_deref() == Some("High priority"))); - } - - #[tokio::test] - async fn playlist_epg_custom_route_rejects_invalid_url_scheme() { - let input = Arc::new(ConfigInput { id: 7, name: "input".intern(), ..Default::default() }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_state = test_app_state(Arc::new(test_app_config(input, source))); - let router = super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); - let request = Request::builder() - .method("POST") - .uri("/playlist/epg") - .header("content-type", "application/json") - .body(Body::from(r#"{"Custom":"ftp://example.com/epg.xml"}"#)) - .expect("request"); - - let response = router.into_service::().oneshot(request).await.expect("response"); - - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - } - - #[tokio::test] - async fn playlist_live_input_route_supports_stalker_inputs() { - let temp_dir = tempdir().expect("temp dir"); - let (base_url, server_handle) = spawn_stalker_mock_server().await; - let input = Arc::new(ConfigInput { - id: 7, - name: "stalker".intern(), - input_type: InputType::Stalker, - url: base_url, - enabled: true, - options: Some(ConfigInputOptions { - flags: crate::model::ConfigInputFlagsSet::new(), - update_quality: ConfigInputUpdateQuality::default(), - resolve_delay: shared::defaults::default_resolve_delay_secs(), - probe_delay: shared::defaults::default_probe_delay_secs(), - probe_live_interval_hours: 120, - resolve_filter: None, - probe_filter: None, - flussonic_hls_catchup: shared::model::FlussonicHlsCatchup::Native, - flussonic_hls_catchup_max_duration_secs: shared::model::default_flussonic_hls_catchup_max_duration_secs( - ), - }), - stalker: Some(crate::model::StalkerInputConfig { - device: None, - auth_mode: shared::model::StalkerAuthMode::Auto, - mag_preset: shared::model::StalkerMagPreset::GenericSafe, - endpoint_preference: shared::model::StalkerEndpointPreference::ServerLoad, - size_caps: None, - catalog_max_pages: None, - ..Default::default() - }), - ..Default::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_config = test_app_config(Arc::clone(&input), source); - let mut sources = app_config.sources.load().as_ref().clone(); - sources.inputs.push(Arc::new(ConfigInput { - id: 8, - name: "m3u".intern(), - input_type: InputType::M3u, - ..Default::default() - })); - app_config.sources.store(Arc::new(sources)); - app_config.config.store(Arc::new(Config { - storage_dir: temp_dir.path().to_string_lossy().to_string(), - ..Default::default() - })); - let app_state = test_app_state(Arc::new(app_config)); - let router = super::v1_api_playlist_register_protected(super::v1_api_playlist_register_public(Router::new())) - .with_state(Arc::clone(&app_state)); - let request = Request::builder() - .method("POST") - .uri("/playlist/live") - .header("content-type", "application/json") - .body(Body::from(r#"{"Input":"stalker"}"#)) - .expect("request"); - - let response = router.clone().into_service::().oneshot(request).await.expect("response"); - - let status = response.status(); - let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("body"); - let body_text = String::from_utf8(body.to_vec()).expect("utf8"); - assert_eq!(status, StatusCode::OK, "{body_text}"); - assert!(body_text.contains("Demo Channel"), "{body_text}"); - assert!(!body_text.contains("ffmpeg http://streams.example/live/101"), "{body_text}"); - let body_json: serde_json::Value = serde_json::from_str(&body_text).unwrap_or_default(); - let playback_url = body_json - .as_array() - .and_then(|items| items.first()) - .and_then(serde_json::Value::as_array) - .and_then(|item| item.get(6)) - .and_then(serde_json::Value::as_str) - .unwrap_or_default(); - assert!(!playback_url.is_empty(), "{body_text}"); - assert!(playback_url.starts_with("http") || playback_url.starts_with('/'), "{playback_url}"); - - let resource_path = playback_url - .find("/playlist/resource/") - .map(|index| &playback_url[index..]) - .expect("Stalker playback resource path"); - let response = router - .clone() - .into_service::() - .oneshot(Request::get(resource_path).body(Body::empty()).expect("resource request")) - .await - .expect("resource response"); - assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT, "{resource_path}"); - assert_eq!( - response.headers().get("location").and_then(|value| value.to_str().ok()), - Some("http://8.8.8.8/live/101") - ); - - let response = router - .clone() - .into_service::() - .oneshot(Request::get("/playlist/resource/*").body(Body::empty()).expect("malformed request")) - .await - .expect("malformed response"); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - - let legacy = shared::utils::obfuscate_text(&app_state.get_encrypt_secret(), "http://10.0.0.1/icon.png"); - let response = router - .clone() - .into_service::() - .oneshot( - Request::get(format!("/playlist/resource/{legacy}")).body(Body::empty()).expect("legacy token request"), - ) - .await - .expect("legacy token response"); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - - for locator in [ - format!("{}99/live/101", super::STALKER_RESOURCE_SCHEME), - format!("{}8/live/101", super::STALKER_RESOURCE_SCHEME), - ] { - let resource = shared::utils::seal_web_ui_resource_url(&app_state.get_encrypt_secret(), &locator); - let response = router - .clone() - .into_service::() - .oneshot( - Request::get(format!("/playlist/resource/{resource}")) - .body(Body::empty()) - .expect("not-found request"), - ) - .await - .expect("not-found response"); - assert_eq!(response.status(), StatusCode::NOT_FOUND, "{locator}"); - } - - let resource = shared::utils::seal_web_ui_resource_url( - &app_state.get_encrypt_secret(), - &format!("{}7/live/102", super::STALKER_RESOURCE_SCHEME), - ); - let response = router - .into_service::() - .oneshot( - Request::get(format!("/playlist/resource/{resource}")) - .body(Body::empty()) - .expect("private destination request"), - ) - .await - .expect("private destination response"); - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); - server_handle.abort(); - } - - /// The playback endpoints resolve a Stalker item from the catalog through the item's own - /// provider id. This is the path behind `Failed to resolve initial Stalker playback URL`, - /// so it needs the real id -> item -> `create_link` chain against a portal, not just the - /// Web UI preview route. - #[tokio::test] - async fn initial_stalker_playback_resolution_uses_the_requested_items_own_command() { - let temp_dir = tempdir().expect("temp dir"); - let (base_url, server_handle) = spawn_stalker_mock_server().await; - let input = Arc::new(ConfigInput { - id: 7, - name: "stalker".intern(), - input_type: InputType::Stalker, - url: base_url, - enabled: true, - options: Some(ConfigInputOptions { - flags: crate::model::ConfigInputFlagsSet::new(), - update_quality: ConfigInputUpdateQuality::default(), - resolve_delay: shared::defaults::default_resolve_delay_secs(), - probe_delay: shared::defaults::default_probe_delay_secs(), - probe_live_interval_hours: 120, - resolve_filter: None, - probe_filter: None, - flussonic_hls_catchup: shared::model::FlussonicHlsCatchup::Native, - flussonic_hls_catchup_max_duration_secs: shared::model::default_flussonic_hls_catchup_max_duration_secs( - ), - }), - stalker: Some(crate::model::StalkerInputConfig { - device: None, - auth_mode: shared::model::StalkerAuthMode::Auto, - mag_preset: shared::model::StalkerMagPreset::GenericSafe, - endpoint_preference: shared::model::StalkerEndpointPreference::ServerLoad, - size_caps: None, - catalog_max_pages: None, - ..Default::default() - }), - ..Default::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_config = test_app_config(Arc::clone(&input), source); - app_config.config.store(Arc::new(Config { - storage_dir: temp_dir.path().to_string_lossy().to_string(), - ..Default::default() - })); - let app_state = test_app_state(Arc::new(app_config)); - let router = super::v1_api_playlist_register_protected(super::v1_api_playlist_register_public(Router::new())) - .with_state(Arc::clone(&app_state)); - - // The catalog import publishes the generation the playback path reads. - let response = router - .clone() - .into_service::() - .oneshot( - Request::builder() - .method("POST") - .uri("/playlist/live") - .header("content-type", "application/json") - .body(Body::from(r#"{"Input":"stalker"}"#)) - .expect("request"), - ) - .await - .expect("catalog response"); - assert_eq!(response.status(), StatusCode::OK); - - let live_input = app_state.app_config.get_input_by_name(&"stalker".intern()).expect("stalker input"); - let unresolved: Arc = "".intern(); - let live = shared::model::PlaylistItemType::Live; - let resolve = |provider_id: u32| { - let app_state = Arc::clone(&app_state); - let live_input = Arc::clone(&live_input); - let unresolved = Arc::clone(&unresolved); - async move { - crate::api::api_utils::resolve_initial_stalker_playback_url( - &app_state, - &live_input, - provider_id, - XtreamCluster::Live, - live, - &unresolved, - ) - .await - } - }; - - // An id that is not in the catalog must not resolve, and must not invalidate the - // published generation the next request reads. - let err = resolve(999_999).await.expect_err("an unknown provider id cannot be resolved"); - assert!(err.to_string().contains("999999"), "{err}"); - - // Each item resolves through its own stored cmd: the mock portal answers a different - // destination per cmd, so a provider id that picked the wrong item would surface here. - assert_eq!( - resolve(101).await.expect("item 101 resolves").as_ref(), - "http://8.8.8.8/live/101", - "the resolved url must come from item 101's own cmd" - ); - assert_eq!( - resolve(102).await.expect("item 102 resolves").as_ref(), - "http://127.0.0.1/live/102", - "the resolved url must come from item 102's own cmd" - ); - - // An item that already carries a url is served as is. - let resolved_url: Arc = "http://stream.example/already-resolved.ts".intern(); - let passthrough = crate::api::api_utils::resolve_initial_stalker_playback_url( - &app_state, - &live_input, - 101, - XtreamCluster::Live, - live, - &resolved_url, - ) - .await - .expect("a resolved url is passed through"); - assert!(Arc::ptr_eq(&passthrough, &resolved_url)); - - server_handle.abort(); - } - - #[tokio::test] - async fn playlist_epg_input_route_returns_cached_provider_epg() { - let temp_dir = tempdir().expect("temp dir"); - let provider = ConfigProvider::from(&ConfigProviderDto { - name: "demo".intern(), - urls: vec!["http://provider.example".intern()], - provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), - dns: None, - }); - let input = Arc::new(ConfigInput { - id: 7, - name: "input".intern(), - epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { - sources: Some(vec![xmltv_source_dto("provider://demo/xmltv.php?username=user&password=pass", 0)]), - t_sources: vec![xmltv_source_dto("provider://demo/xmltv.php?username=user&password=pass", 0)], - smart_match: None, - })), - provider_configs: Some(vec![Arc::new(provider)]), - ..Default::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_config = test_app_config(input.clone(), source); - app_config.config.store(Arc::new(Config { - storage_dir: temp_dir.path().to_string_lossy().to_string(), - ..Default::default() - })); - let raw_epg_path = get_input_raw_xmltv_file_path( - "provider://demo/xmltv.php?username=user&password=pass", - input.as_ref(), - temp_dir.path().to_string_lossy().as_ref(), - ) - .await - .expect("epg path"); - if let Some(parent) = raw_epg_path.parent() { - tokio::fs::create_dir_all(parent).await.expect("epg dir"); - } - let prog_start = epg_dt(0); - let prog_stop = epg_dt(1); - tokio::fs::write( - &raw_epg_path, - format!( - r#" - - - Demo Channel - - - Morning Show - -"# - ), - ) - .await - .expect("write epg"); - - let app_state = test_app_state(Arc::new(app_config)); - let router = super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); - let request = Request::builder() - .method("POST") - .uri("/playlist/epg") - .header("content-type", "application/json") - .body(Body::from(r#"{"Input":"input"}"#)) - .expect("request"); - - let response = router.into_service::().oneshot(request).await.expect("response"); - - assert_eq!(response.status(), StatusCode::OK); - let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("body"); - let body_text = String::from_utf8(body.to_vec()).expect("utf8"); - assert!(body_text.contains("demo.channel"), "{body_text}"); - assert!(body_text.contains("Morning Show"), "{body_text}"); - } - - #[tokio::test] - async fn playlist_epg_input_route_returns_no_content_for_unknown_input_name() { - let provider = ConfigProvider::from(&ConfigProviderDto { - name: "demo".intern(), - urls: vec!["http://provider.example".intern()], - provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), - dns: None, - }); - let input = Arc::new(ConfigInput { - id: 7, - name: "input".intern(), - provider_configs: Some(vec![Arc::new(provider)]), - ..Default::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_state = test_app_state(Arc::new(test_app_config(input, source))); - let router = super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); - let request = Request::builder() - .method("POST") - .uri("/playlist/epg") - .header("content-type", "application/json") - .body(Body::from(r#"{"Input":"missing"}"#)) - .expect("request"); - - let response = router.into_service::().oneshot(request).await.expect("response"); - - assert_eq!(response.status(), StatusCode::NO_CONTENT); - } - - #[tokio::test] - #[allow(clippy::too_many_lines)] - async fn playlist_epg_input_route_merges_multiple_cached_sources_by_priority() { - let temp_dir = tempdir().expect("temp dir"); - let provider = ConfigProvider::from(&ConfigProviderDto { - name: "demo".intern(), - urls: vec!["http://provider.example".intern()], - provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), - dns: None, - }); - let primary_url = "provider://demo/xmltv-primary.php?username=user&password=pass"; - let secondary_url = "provider://demo/xmltv-secondary.php?username=user&password=pass"; - let input = Arc::new(ConfigInput { - id: 7, - name: "input".intern(), - epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { - sources: Some(vec![ - xmltv_source_dto(secondary_url, 10), - xmltv_source_dto(primary_url, 0), - xmltv_source_dto("provider://demo/xmltv-same-priority.php?username=user&password=pass", 0), - ]), - t_sources: vec![ - xmltv_source_dto(secondary_url, 10), - xmltv_source_dto(primary_url, 0), - xmltv_source_dto("provider://demo/xmltv-same-priority.php?username=user&password=pass", 0), - ], - smart_match: None, - })), - provider_configs: Some(vec![Arc::new(provider)]), - ..Default::default() - }); - let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; - let app_config = test_app_config(input.clone(), source); - app_config.config.store(Arc::new(Config { - storage_dir: temp_dir.path().to_string_lossy().to_string(), - ..Default::default() - })); - - let primary_path = - get_input_raw_xmltv_file_path(primary_url, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) - .await - .expect("primary epg path"); - let secondary_path = - get_input_raw_xmltv_file_path(secondary_url, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) - .await - .expect("secondary epg path"); - let same_priority_path = get_input_raw_xmltv_file_path( - "provider://demo/xmltv-same-priority.php?username=user&password=pass", - input.as_ref(), - temp_dir.path().to_string_lossy().as_ref(), - ) - .await - .expect("same priority epg path"); - - for path in [&primary_path, &secondary_path, &same_priority_path] { - if let Some(parent) = path.parent() { - tokio::fs::create_dir_all(parent).await.expect("epg dir"); - } - } - - let p0 = epg_dt(0); - let p1 = epg_dt(1); - let p2 = epg_dt(2); - let p3 = epg_dt(3); - - tokio::fs::write( - &secondary_path, - format!( - r#" - - - Secondary Channel - - - - Secondary Show - -"# - ), - ) - .await - .expect("write secondary epg"); - tokio::fs::write( - &primary_path, - format!( - r#" - - - Primary Channel - - - - Primary Show - -"# - ), - ) - .await - .expect("write primary epg"); - tokio::fs::write( - &same_priority_path, - format!( - r#" - - - Same Priority Channel - - - - Duplicate Show - - - Second Show - -"# - ), - ) - .await - .expect("write same priority epg"); - - let app_state = test_app_state(Arc::new(app_config)); - let router = super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); - let request = Request::builder() - .method("POST") - .uri("/playlist/epg") - .header("content-type", "application/json") - .body(Body::from(r#"{"Input":"input"}"#)) - .expect("request"); - - let response = router.into_service::().oneshot(request).await.expect("response"); - - assert_eq!(response.status(), StatusCode::OK); - let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("body"); - let body_text = String::from_utf8(body.to_vec()).expect("utf8"); - assert!(body_text.contains("Primary Channel"), "{body_text}"); - assert!(!body_text.contains("Secondary Channel"), "{body_text}"); - assert!(body_text.contains("Primary Show"), "{body_text}"); - assert!(body_text.contains("Second Show"), "{body_text}"); - assert!(!body_text.contains("Secondary Show"), "{body_text}"); - } -} +use self::epg::load_epg_channels_for_input; +#[cfg(test)] +use self::epg::merge_epg_channels; +#[cfg(test)] +use self::recording::epg_channel_id_matches; +#[cfg(test)] +use self::recording::select_epg_channel_candidate; +#[cfg(test)] +use self::recording::RecordingStreamQuery; +pub(in crate::api) use self::recording::{ + build_webplayer_recording_url, resolve_target_live_recording_source_by_epg_channel, resolve_target_recording_source, +}; +#[cfg(test)] +use self::recording::{recording_candidate, RecordingCandidateFields}; +#[cfg(test)] +use self::resources::resolve_provider_url_for_request; +pub use self::routes::{ + v1_api_playlist_register_protected, v1_api_playlist_register_public, v1_api_playlist_register_with_permissions, +}; +#[cfg(test)] +use self::update::read_playlist_update_input_status; +#[cfg(test)] +use self::update::resolve_manual_playlist_update_targets; +use self::{ + content::{playlist_content, playlist_episode_item, playlist_series_info}, + epg::playlist_epg, + error::ManualUpdateEnqueueError, + filter_preview::playlist_filter_preview, + recording::playlist_recording_stream, + resources::{ + create_config_input_for_m3u, create_config_input_for_xtream, playlist_resolve_url, playlist_resource, + resolve_provider_url_with_input, + }, + update::{playlist_update, playlist_update_status}, + webplayer::{build_playlist_webplayer_url, playlist_webplayer, playlist_webplayer_stream}, +}; diff --git a/backend/app/src/api/endpoints/v1_api_playlist/content.rs b/backend/app/src/api/endpoints/v1_api_playlist/content.rs new file mode 100644 index 000000000..fbda9b80a --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/content.rs @@ -0,0 +1,203 @@ +use super::{create_config_input_for_m3u, create_config_input_for_xtream}; +use crate::{ + api::{ + api_utils::{create_api_proxy_user, try_unwrap_body}, + endpoints::{ + api_playlist_utils::{ + get_playlist_for_custom_provider, get_playlist_for_input, get_playlist_for_target, rewrite_resource_url, + }, + xtream_api::xtream_get_stream_info_response, + }, + model::AppState, + }, + iptv::xtream, + model::InputSource, + repository::xtream_get_item_for_stream_id, +}; +use axum::response::IntoResponse; +use serde_json::json; +use shared::{ + model::{PlaylistRequest, ProxyType, TargetType, UiPlaylistItem, XtreamCluster}, + utils::{concat_path_leading_slash, Internable}, +}; +use std::sync::Arc; +use url::Url; + +pub(super) async fn playlist_content( + accept: Option, + app_state: &Arc, + playlist_req: &PlaylistRequest, + cluster: XtreamCluster, +) -> impl IntoResponse + Send { + let client = app_state.http_clients.default.load(); + match playlist_req { + PlaylistRequest::Target(target_id) => get_playlist_for_target( + app_state.app_config.get_target_by_id(*target_id).as_deref(), + app_state, + cluster, + accept.as_deref(), + ) + .await + .into_response(), + PlaylistRequest::Input(input_name) => get_playlist_for_input( + app_state.app_config.get_input_by_name(&input_name.intern()).as_ref(), + app_state, + cluster, + accept.as_deref(), + ) + .await + .into_response(), + PlaylistRequest::CustomXtream(xtream) => match Url::parse(&xtream.url) { + Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => { + let input = Arc::new(create_config_input_for_xtream(&xtream.username, &xtream.password, &xtream.url)); + get_playlist_for_custom_provider(client.as_ref(), Some(&input), app_state, cluster, accept.as_deref()) + .await + .into_response() + } + _ => ( + axum::http::StatusCode::BAD_REQUEST, + axum::Json(json!({"error": "Invalid url scheme; only http/https are allowed"})), + ) + .into_response(), + }, + PlaylistRequest::CustomM3u(m3u) => match Url::parse(&m3u.url) { + Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => { + let input = Arc::new(create_config_input_for_m3u(&m3u.url)); + get_playlist_for_custom_provider(client.as_ref(), Some(&input), app_state, cluster, accept.as_deref()) + .await + .into_response() + } + _ => ( + axum::http::StatusCode::BAD_REQUEST, + axum::Json(json!({"error": "Invalid url scheme; only http/https are allowed"})), + ) + .into_response(), + }, + } +} + +pub(super) async fn playlist_series_info( + axum::extract::Path((virtual_id, provider_id)): axum::extract::Path<(String, String)>, + axum::extract::State(app_state): axum::extract::State>, + axum::extract::Json(playlist_req): axum::extract::Json, +) -> impl IntoResponse + Send { + let provider_id = provider_id.trim().parse::().ok(); + + match playlist_req { + PlaylistRequest::Target(target_id) => { + if let Some(target) = app_state.app_config.get_target_by_id(target_id) { + if target.has_output(TargetType::Xtream) { + let mut user = create_api_proxy_user(&app_state); + user.proxy = ProxyType::Redirect; + return xtream_get_stream_info_response( + &app_state, + &user, + &target, + &virtual_id, + XtreamCluster::Series, + ) + .await + .into_response(); + } + } + } + + PlaylistRequest::Input(input_name) => { + if let Some(input) = app_state.app_config.get_input_by_name(&input_name.intern()) { + if input.input_type.is_xtream() { + // We cannot call `xtream_get_stream_info_response` directly here because that path + // depends on target-local virtual-id mapping (`xtream_get_item_for_stream_id`). + // Input/custom requests only provide provider_id, so we resolve series info from + // the upstream Xtream API using provider_id. + if let Some(provider_id) = provider_id { + if let Some(info_url) = + xtream::get_xtream_player_api_info_url(input.as_ref(), XtreamCluster::Series, provider_id) + { + let Ok(resolved_url) = input.resolve_url(&info_url) else { + return axum::http::StatusCode::NO_CONTENT.into_response(); + }; + let input_source = InputSource::from(input.as_ref()).with_url(resolved_url.to_string()); + if let Ok(content) = xtream::get_xtream_stream_info_content( + &app_state.app_config, + &app_state.http_clients.default.load(), + &input_source, + false, + ) + .await + { + return try_unwrap_body!(axum::response::Response::builder() + .status(axum::http::StatusCode::OK) + .header(axum::http::header::CONTENT_TYPE, mime::APPLICATION_JSON.to_string()) + .body(axum::body::Body::from(content))); + } + } + } + } + } + } + PlaylistRequest::CustomXtream(xtream_req) => { + if let Some(provider_id) = provider_id { + let input = create_config_input_for_xtream(&xtream_req.username, &xtream_req.password, &xtream_req.url); + if let Some(info_url) = + xtream::get_xtream_player_api_info_url(&input, XtreamCluster::Series, provider_id) + { + let input_source = InputSource::from(&input).with_url(info_url); + if let Ok(content) = xtream::get_xtream_stream_info_content( + &app_state.app_config, + &app_state.http_clients.default.load(), + &input_source, + false, + ) + .await + { + return try_unwrap_body!(axum::response::Response::builder() + .status(axum::http::StatusCode::OK) + .header(axum::http::header::CONTENT_TYPE, mime::APPLICATION_JSON.to_string()) + .body(axum::body::Body::from(content))); + } + } + } + } + PlaylistRequest::CustomM3u(_) => {} + } + axum::http::StatusCode::NO_CONTENT.into_response() +} + +pub(super) async fn playlist_episode_item( + axum::extract::Path(virtual_id): axum::extract::Path, + axum::extract::State(app_state): axum::extract::State>, + axum::extract::Json(playlist_req): axum::extract::Json, +) -> impl IntoResponse + Send { + if let PlaylistRequest::Target(target_id) = playlist_req { + if let Some(target) = app_state.app_config.get_target_by_id(target_id) { + if target.has_output(TargetType::Xtream) { + if let Ok(vid) = virtual_id.parse::() { + if let Ok(pli) = xtream_get_item_for_stream_id( + vid, + &app_state.app_config, + &app_state.playlists, + &target, + Some(XtreamCluster::Series), + ) + .await + { + // The icon is wrapped like on every other Web UI path: the item carries the + // destination as it was stored, which may name a host internal to this instance. + let config = app_state.app_config.config.load(); + let web_ui_path = + config.web_ui.as_ref().and_then(|web_ui| web_ui.path.as_ref()).map_or("", String::as_str); + let resource_url = concat_path_leading_slash(web_ui_path, "api/v1/playlist/resource"); + drop(config); + let item = rewrite_resource_url( + &app_state.get_encrypt_secret(), + &resource_url, + UiPlaylistItem::from(pli), + ); + return axum::Json(json!(item)).into_response(); + } + } + } + } + } + axum::http::StatusCode::NO_CONTENT.into_response() +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/epg.rs b/backend/app/src/api/endpoints/v1_api_playlist/epg.rs new file mode 100644 index 000000000..03c0acd9c --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/epg.rs @@ -0,0 +1,286 @@ +use super::resolve_provider_url_with_input; +use crate::{ + api::{ + api_utils::json_or_bin_response, + endpoints::{ + extract_accept_header::ExtractAcceptHeader, + xmltv_api::{rewrite_epg_channel_resource_url, serve_epg_web_ui}, + }, + model::AppState, + }, + model::{ + parse_xmltv_for_web_ui_from_file, parse_xmltv_for_web_ui_from_url, ConfigInput, EpgSource, EpgSourceType, + IcsDummyPolicy, + }, + processing::{ + epg::get_input_raw_epg_file_path, + parser::{ + ics::parse_ics_file_to_channel, + xmltv::{merge_epg_channels_by_priority_with_dummy_policies, EpgDummyPolicySource}, + }, + }, + utils::{file_exists_async, request}, +}; +use axum::response::IntoResponse; +use log::{debug, error}; +use shared::{ + error::TuliproxError, + model::{EpgChannel, PlaylistEpgRequest, TargetType}, + utils::{concat_path_leading_slash, sanitize_sensitive_info, Internable}, +}; +use std::{path::Path, sync::Arc}; +use url::Url; + +#[cfg(test)] +pub(super) fn merge_epg_channels(mut channels_by_source: Vec<(i16, Vec)>) -> Vec { + channels_by_source.sort_by_key(|(priority, _)| *priority); + merge_epg_channels_by_priority_with_dummy_policies(channels_by_source, Vec::new()) +} + +async fn load_xmltv_epg_source_channels( + app_state: &Arc, + raw_epg_path: &Path, + resolved_url: &str, +) -> Result, TuliproxError> { + { + let _cache_lock = app_state.app_config.file_locks.read_lock(raw_epg_path).await; + if file_exists_async(raw_epg_path).await { + match parse_xmltv_for_web_ui_from_file(raw_epg_path).await { + Ok(channels) => return Ok(channels), + Err(file_err) => { + debug!( + "EPG file parse failed {}, trying upstream: {file_err}", + sanitize_sensitive_info(raw_epg_path.to_str().unwrap_or_default()) + ); + } + } + } + } + + parse_xmltv_for_web_ui_from_url(&app_state.app_config, &app_state.http_clients.default.load(), resolved_url).await +} + +async fn load_ics_epg_source_channels( + app_state: &Arc, + input: &ConfigInput, + epg_source: &EpgSource, + raw_epg_path: &Path, + storage_dir: &str, +) -> Result, TuliproxError> { + let ics_config = epg_source + .ics + .as_ref() + .ok_or_else(|| TuliproxError::ConfigEpg("ics configuration is required for ICS EPG sources".to_string()))?; + + let channel_id = epg_source + .channel_id + .clone() + .ok_or_else(|| TuliproxError::ConfigEpg("channel_id is required for ICS EPG sources".to_string()))?; + + { + let _cache_lock = app_state.app_config.file_locks.read_lock(raw_epg_path).await; + if file_exists_async(raw_epg_path).await { + match parse_ics_file_to_channel( + raw_epg_path, + channel_id.clone(), + epg_source.channel_title.clone(), + ics_config, + ) + .await + { + Ok(channel) => return Ok(vec![channel]), + Err(file_err) => { + debug!( + "ICS EPG file parse failed {}, redownloading source: {}", + sanitize_sensitive_info(raw_epg_path.to_str().unwrap_or_default()), + sanitize_sensitive_info(&file_err.to_string()) + ); + } + } + } + } + + let client = app_state.http_clients.default.load(); + request::get_input_epg_content_as_file( + &app_state.app_config, + &client, + input, + request::InputEpgFileRequest { + headers: None, + storage_dir, + url: &epg_source.url, + persist_path: raw_epg_path, + max_bytes: Some(ics_config.max_download_bytes), + }, + ) + .await?; + + let _cache_lock = app_state.app_config.file_locks.read_lock(raw_epg_path).await; + parse_ics_file_to_channel(raw_epg_path, channel_id, epg_source.channel_title.clone(), ics_config) + .await + .map(|channel| vec![channel]) +} + +pub(super) async fn load_epg_channels_for_input( + app_state: &Arc, + input: &ConfigInput, +) -> Result>, TuliproxError> { + let Some(epg_config) = input.epg.as_ref() else { + return Ok(None); + }; + + let storage_dir = app_state.app_config.config.load().storage_dir.clone(); + let mut channels_by_source = Vec::new(); + let mut dummy_policies = Vec::new(); + let mut failed_sources = 0usize; + + for (source_order, epg_source) in epg_config.sources.iter().enumerate() { + let raw_epg_path = match get_input_raw_epg_file_path(epg_source, input, &storage_dir).await { + Ok(path) => path, + Err(err) => { + debug!( + "Skipping EPG source {}: {}", + sanitize_sensitive_info(epg_source.url.as_str()), + sanitize_sensitive_info(&err.to_string()) + ); + failed_sources += 1; + continue; + } + }; + + let source_result = match epg_source.source_type { + EpgSourceType::Xmltv => { + let resolved_url = resolve_provider_url_with_input(input, &epg_source.url); + load_xmltv_epg_source_channels(app_state, &raw_epg_path, &resolved_url).await + } + EpgSourceType::Ics => { + load_ics_epg_source_channels(app_state, input, epg_source, &raw_epg_path, &storage_dir).await + } + }; + + let source_channels = match source_result { + Ok(channels) => channels, + Err(err) => { + debug!( + "Skipping EPG source {}: {}", + sanitize_sensitive_info(epg_source.url.as_str()), + sanitize_sensitive_info(&err.to_string()) + ); + failed_sources += 1; + continue; + } + }; + if epg_source.source_type == EpgSourceType::Ics { + if let (Some(ics_config), Some(channel_id)) = (epg_source.ics.as_ref(), epg_source.channel_id.as_ref()) { + dummy_policies.push(EpgDummyPolicySource { + priority: epg_source.priority, + source_order, + channel_id: channel_id.clone(), + policy: IcsDummyPolicy { timezone: ics_config.timezone.clone(), config: ics_config.dummy.clone() }, + }); + } + } + channels_by_source.push((epg_source.priority, source_channels)); + } + + if channels_by_source.is_empty() { + if failed_sources > 0 { + Err(TuliproxError::Config(format!("All {failed_sources} EPG source(s) failed for input '{}'", input.name))) + } else { + Ok(None) + } + } else { + channels_by_source.sort_by_key(|(priority, _)| *priority); + Ok(Some(merge_epg_channels_by_priority_with_dummy_policies(channels_by_source, dummy_policies))) + } +} + +pub(super) async fn playlist_epg( + ExtractAcceptHeader(accept): ExtractAcceptHeader, + axum::extract::State(app_state): axum::extract::State>, + axum::extract::Json(playlist_epg_req): axum::extract::Json, +) -> impl IntoResponse + Send { + match playlist_epg_req { + PlaylistEpgRequest::Target(target_id) => { + if let Some(target) = app_state.app_config.get_target_by_id(target_id) { + let config = &app_state.app_config.config.load(); + let epg_path = crate::api::endpoints::xmltv_api::get_epg_path_for_target_by_type( + config, + &target, + TargetType::Xtream, + ) + .or_else(|| { + crate::api::endpoints::xmltv_api::get_epg_path_for_target_by_type(config, &target, TargetType::M3u) + }); + if let Some(epg_path) = epg_path { + return serve_epg_web_ui(&app_state, accept.as_deref(), &epg_path, &target).await; + } + } + } + PlaylistEpgRequest::Input(input_name) => { + if let Some(input) = app_state.app_config.get_input_by_name(&input_name.intern()) { + match load_epg_channels_for_input(&app_state, input.as_ref()).await { + Ok(Some(epg)) => { + let config = app_state.app_config.config.load(); + let web_ui_path = + config.web_ui.as_ref().and_then(|w| w.path.as_ref()).map_or("", String::as_str); + let resource_url = concat_path_leading_slash(web_ui_path, "api/v1/playlist/resource"); + let encrypt_secret = app_state.get_encrypt_secret(); + let epg = epg + .into_iter() + .map(|channel| rewrite_epg_channel_resource_url(&encrypt_secret, &resource_url, channel)) + .collect::>(); + return json_or_bin_response(accept.as_deref(), &epg).into_response(); + } + Ok(None) => return axum::http::StatusCode::NO_CONTENT.into_response(), + Err(err) => { + error!( + "Failed to load input EPG for '{}': {}", + input.name, + sanitize_sensitive_info(&err.to_string()) + ); + return ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(serde_json::json!({"error": "Failed to load EPG"})), + ) + .into_response(); + } + } + } + } + PlaylistEpgRequest::Custom(url) => { + let valid_custom_url = Url::parse(&url).is_ok_and(|parsed| matches!(parsed.scheme(), "http" | "https")); + if !valid_custom_url { + return ( + axum::http::StatusCode::BAD_REQUEST, + axum::Json(serde_json::json!({"error": "Invalid EPG URL"})), + ) + .into_response(); + } + match parse_xmltv_for_web_ui_from_url(&app_state.app_config, &app_state.http_clients.default.load(), &url) + .await + { + Ok(epg) => { + let config = app_state.app_config.config.load(); + let web_ui_path = config.web_ui.as_ref().and_then(|w| w.path.as_ref()).map_or("", String::as_str); + let resource_url = concat_path_leading_slash(web_ui_path, "api/v1/playlist/resource"); + let encrypt_secret = app_state.get_encrypt_secret(); + let epg = epg + .into_iter() + .map(|channel| rewrite_epg_channel_resource_url(&encrypt_secret, &resource_url, channel)) + .collect::>(); + return json_or_bin_response(accept.as_deref(), &epg).into_response(); + } + Err(err) => { + error!("Failed to load custom EPG: {}", sanitize_sensitive_info(&err.to_string())); + return ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(serde_json::json!({"error": "Failed to load EPG"})), + ) + .into_response(); + } + } + } + } + axum::http::StatusCode::NO_CONTENT.into_response() +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/epg_channel_candidate_tests.rs b/backend/app/src/api/endpoints/v1_api_playlist/epg_channel_candidate_tests.rs new file mode 100644 index 000000000..d434ecb98 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/epg_channel_candidate_tests.rs @@ -0,0 +1,3 @@ +use super::{select_epg_channel_candidate, ResolvedRecordingSource}; + +mod behavior; diff --git a/backend/app/src/api/endpoints/v1_api_playlist/epg_channel_candidate_tests/behavior.rs b/backend/app/src/api/endpoints/v1_api_playlist/epg_channel_candidate_tests/behavior.rs new file mode 100644 index 000000000..8ed046b51 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/epg_channel_candidate_tests/behavior.rs @@ -0,0 +1,67 @@ +use super::{select_epg_channel_candidate, ResolvedRecordingSource}; + +pub(in crate::api::endpoints::v1_api_playlist::epg_channel_candidate_tests) fn candidate( + virtual_id: u32, + title: &str, +) -> ResolvedRecordingSource { + ResolvedRecordingSource { + virtual_id, + input_name: "input".to_string(), + title: title.to_string(), + group: None, + series_name: None, + extension: None, + downloadable: true, + } +} + +pub(in crate::api::endpoints::v1_api_playlist::epg_channel_candidate_tests) fn shared_epg_id_candidates( +) -> Vec { + vec![candidate(1, "┃NL┃ SBS 6 4K"), candidate(2, "┃CANAL+┃ SBS 6 HD"), candidate(3, "┃NLZIET┃ SBS 6 HD")] +} + +#[test] +fn shared_epg_id_prefers_exact_channel_name() { + let selected = select_epg_channel_candidate(shared_epg_id_candidates(), Some("┃NLZIET┃ SBS 6 HD")); + assert_eq!(selected.map(|c| c.virtual_id), Some(3)); +} + +#[test] +fn shared_epg_id_matches_lowercased_display_name() { + let selected = select_epg_channel_candidate(shared_epg_id_candidates(), Some(" ┃canal+┃ sbs 6 hd ")); + assert_eq!(selected.map(|c| c.virtual_id), Some(2)); +} + +#[test] +fn shared_epg_id_without_matching_name_takes_first() { + assert_eq!(select_epg_channel_candidate(shared_epg_id_candidates(), Some("SBS 6")).map(|c| c.virtual_id), Some(1)); + assert_eq!(select_epg_channel_candidate(shared_epg_id_candidates(), None).map(|c| c.virtual_id), Some(1)); +} + +#[test] +fn episode_candidate_carries_series_name_and_group() { + use super::super::{recording_candidate, RecordingCandidateFields}; + use shared::model::{PlaylistItemType, VirtualId}; + let fields = |item_type, group| RecordingCandidateFields { + virtual_id: VirtualId::new(7), + input_name: "input", + name: "The Show", + title: "The Show S01E02 Pilot", + group, + url: "http://example.test/series/7.mkv", + item_type, + }; + let episode = recording_candidate(&fields(PlaylistItemType::Series, "Crime")); + assert_eq!(episode.series_name.as_deref(), Some("The Show")); + assert_eq!(episode.group.as_deref(), Some("Crime")); + assert_eq!(episode.extension.as_deref().map(|ext| ext.trim_start_matches('.')), Some("mkv")); + + let film = recording_candidate(&fields(PlaylistItemType::Video, " ")); + assert_eq!(film.series_name, None, "only episodes carry a series name"); + assert_eq!(film.group, None, "a blank group is no group"); +} + +#[test] +fn no_candidate_resolves_nothing() { + assert!(select_epg_channel_candidate(Vec::new(), Some("SBS 6")).is_none()); +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/error.rs b/backend/app/src/api/endpoints/v1_api_playlist/error.rs new file mode 100644 index 000000000..a1935ae8e --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/error.rs @@ -0,0 +1,5 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ManualUpdateEnqueueError { + Busy, + Unavailable, +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/filter_preview.rs b/backend/app/src/api/endpoints/v1_api_playlist/filter_preview.rs new file mode 100644 index 000000000..b18a9625c --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/filter_preview.rs @@ -0,0 +1,175 @@ +use super::{FILTER_PREVIEW_DEFAULT_SAMPLES, FILTER_PREVIEW_MAX_SAMPLES}; +use crate::{ + api::model::AppState, + repository::{iter_raw_m3u_target_playlist, iter_raw_xtream_target_playlist}, +}; +use axum::response::IntoResponse; +use log::error; +use serde_json::json; +use shared::{ + foundation::{get_filter_detailed, Filter, ValueProvider}, + model::{PlaylistItem, TargetType, XtreamCluster}, +}; +use std::sync::Arc; +use tokio_stream::StreamExt; + +#[derive(serde::Deserialize)] +pub(super) struct FilterPreviewRequest { + target: u16, + filter: String, + #[serde(default)] + limit: Option, + #[serde(default)] + match_as_ascii: bool, +} + +#[derive(serde::Serialize)] +struct FilterPreviewItem { + name: String, + group: String, + item_type: String, +} + +impl From<&PlaylistItem> for FilterPreviewItem { + fn from(pli: &PlaylistItem) -> Self { + let header = &pli.header; + Self { + name: header.name.to_string(), + group: header.group.to_string(), + item_type: header.item_type.as_str().to_string(), + } + } +} + +#[derive(serde::Serialize, Default)] +struct FilterPreviewClusterStats { + total: usize, + matched: usize, +} + +#[derive(serde::Serialize, Default)] +struct FilterPreviewResponse { + total: usize, + matched: usize, + live: FilterPreviewClusterStats, + vod: FilterPreviewClusterStats, + series: FilterPreviewClusterStats, + sample_matched: Vec, + sample_excluded: Vec, +} + +impl FilterPreviewResponse { + fn observe(&mut self, pli: &PlaylistItem, filter: &Filter, match_as_ascii: bool, sample_limit: usize) { + let cluster_stats = match pli.header.xtream_cluster { + XtreamCluster::Live => &mut self.live, + XtreamCluster::Video => &mut self.vod, + XtreamCluster::Series => &mut self.series, + }; + self.total += 1; + cluster_stats.total += 1; + let provider = ValueProvider { pli, match_as_ascii }; + if filter.filter(&provider) { + self.matched += 1; + cluster_stats.matched += 1; + if self.sample_matched.len() < sample_limit { + self.sample_matched.push(FilterPreviewItem::from(pli)); + } + } else if self.sample_excluded.len() < sample_limit { + self.sample_excluded.push(FilterPreviewItem::from(pli)); + } + } +} + +/// Dry-run a filter DSL expression against a target's stored playlist +/// without touching processing or provider fetches. +pub(super) async fn playlist_filter_preview( + axum::extract::State(app_state): axum::extract::State>, + axum::extract::Json(req): axum::extract::Json, +) -> impl IntoResponse + Send { + let filter = { + let sources = app_state.app_config.sources.load(); + match get_filter_detailed(&req.filter, sources.templates.as_deref()) { + Ok(filter) => filter, + Err((err, position)) => { + return ( + axum::http::StatusCode::UNPROCESSABLE_ENTITY, + axum::Json(json!({ + "error": err.to_string(), + "line": position.map(|p| p.line), + "column": position.map(|p| p.column), + })), + ) + .into_response() + } + } + }; + let Some(target) = app_state.app_config.get_target_by_id(req.target) else { + return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Unknown target"}))).into_response(); + }; + let sample_limit = usize::from(req.limit.unwrap_or(FILTER_PREVIEW_DEFAULT_SAMPLES).min(FILTER_PREVIEW_MAX_SAMPLES)); + + let mut response = FilterPreviewResponse::default(); + if target.has_output(TargetType::Xtream) { + let mut any_cluster_read = false; + for cluster in [XtreamCluster::Live, XtreamCluster::Video, XtreamCluster::Series] { + if let Some(mut iterator) = iter_raw_xtream_target_playlist(&app_state.app_config, &target, cluster).await { + any_cluster_read = true; + while let Some(entry) = iterator.next().await { + match entry { + Ok(item) => { + let pli = PlaylistItem::from(&item); + response.observe(&pli, &filter, req.match_as_ascii, sample_limit); + } + Err(err) => { + error!("Filter preview failed to read stored {cluster} playlist: {err}"); + return ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(json!({"error": "Failed to read stored playlist"})), + ) + .into_response(); + } + } + } + } + } + if !any_cluster_read { + return ( + axum::http::StatusCode::NOT_FOUND, + axum::Json(json!({"error": "Stored playlist is not available, update the playlist first"})), + ) + .into_response(); + } + } else if target.has_output(TargetType::M3u) { + let Some(mut iterator) = iter_raw_m3u_target_playlist(&app_state.app_config, &target, None).await else { + return ( + axum::http::StatusCode::NOT_FOUND, + axum::Json(json!({"error": "Stored playlist is not available, update the playlist first"})), + ) + .into_response(); + }; + while let Some(entry) = iterator.next().await { + match entry { + Ok(item) => { + let pli = PlaylistItem::from(&item); + response.observe(&pli, &filter, req.match_as_ascii, sample_limit); + } + Err(err) => { + error!("Filter preview failed to read stored m3u playlist: {err}"); + return ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(json!({"error": "Failed to read stored playlist"})), + ) + .into_response(); + } + } + } + } else { + return ( + axum::http::StatusCode::BAD_REQUEST, + axum::Json(json!({"error": "Target has no xtream or m3u output to preview"})), + ) + .into_response(); + } + + axum::Json(response).into_response() +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/recording.rs b/backend/app/src/api/endpoints/v1_api_playlist/recording.rs new file mode 100644 index 000000000..a5f8a6864 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/recording.rs @@ -0,0 +1,319 @@ +use super::{build_playlist_webplayer_url, ResolvedRecordingSource}; +use crate::{ + api::{ + api_utils::try_result_bad_request, + endpoints::{ + m3u_api::m3u_api_stream_loaded, + xtream_api::{xtream_player_api_stream_with_resolved_target, ApiStreamContext, ApiStreamRequest}, + }, + model::{ + recording::recording_source_resolution::{resolve_recording_config, resolve_recording_target}, + AppState, + }, + }, + auth::{create_access_token, verify_access_token}, + model::ConfigTarget, + repository::{iter_raw_m3u_target_playlist, iter_raw_xtream_target_playlist, m3u_get_item_for_stream_id}, + utils::{canonicalize_output_epg_id, canonicalize_untrusted_epg_id, EpgIdOutputCase}, +}; +use axum::response::IntoResponse; +use log::{debug, error}; +use serde::Deserialize; +use shared::model::{ConfigTargetOptions, PlaylistItemType, TargetType, VirtualId, XtreamCluster}; +use std::{str::FromStr, sync::Arc}; +use tokio_stream::StreamExt; + +pub(in crate::api) fn build_webplayer_recording_url( + app_config: &crate::model::AppConfig, + target_id: u16, + virtual_id: u32, + cluster: XtreamCluster, +) -> Option { + let access_token = create_access_token(&app_config.access_token_secret, 30, crate::auth::scope::INTERNAL_PLAYER); + let config = app_config.config.load(); + let server_name = config + .web_ui + .as_ref() + .and_then(|web_ui| web_ui.player_server.as_ref()) + .map_or("default", |server_name| server_name.as_str()); + let server_info = app_config.get_server_info(server_name)?; + Some(build_playlist_webplayer_url(&server_info.get_base_url(), &access_token, target_id, virtual_id, cluster)) +} + +/// The playlist fields a recording needs, shared by Xtream and M3U items. +pub(super) struct RecordingCandidateFields<'a> { + pub(super) virtual_id: VirtualId, + pub(super) input_name: &'a str, + pub(super) name: &'a str, + pub(super) title: &'a str, + pub(super) group: &'a str, + pub(super) url: &'a str, + pub(super) item_type: PlaylistItemType, +} + +fn non_blank(value: &str) -> Option { + let value = value.trim(); + (!value.is_empty()).then(|| value.to_string()) +} + +pub(super) fn recording_candidate(fields: &RecordingCandidateFields<'_>) -> ResolvedRecordingSource { + let RecordingCandidateFields { virtual_id, input_name, name, title, group, url, item_type } = *fields; + let is_episode = matches!(item_type, PlaylistItemType::Series | PlaylistItemType::LocalSeries); + ResolvedRecordingSource { + virtual_id: virtual_id.get(), + input_name: input_name.to_string(), + title: title.to_string(), + group: non_blank(group), + series_name: if is_episode { non_blank(name) } else { None }, + extension: shared::utils::extract_extension_from_url(url).map(str::to_string), + downloadable: item_type != PlaylistItemType::SeriesInfo, + } +} + +pub(in crate::api) async fn resolve_target_recording_source( + app_config: &crate::model::AppConfig, + target_name: &str, + input_name: &str, + virtual_id: u32, + cluster: XtreamCluster, +) -> Option { + // EPG grid rows carry the exact playlist id but do not know the input. + // Resolve it from the persisted item, then validate its configured scope. + let missing_input = input_name.trim().is_empty(); + let target = if missing_input { + find_target_by_name(app_config, target_name)? + } else { + resolve_recording_target(app_config, target_name, input_name)? + }; + let wanted = VirtualId::new(virtual_id); + let mut resolved = None; + if target.has_output(TargetType::Xtream) { + if let Some(mut items) = iter_raw_xtream_target_playlist(app_config, &target, cluster).await { + while let Some(entry) = items.next().await { + let Ok(item) = entry else { continue }; + if item.virtual_id == wanted { + resolved = Some(recording_candidate(&RecordingCandidateFields { + virtual_id: item.virtual_id, + input_name: &item.input_name, + name: &item.name, + title: &item.title, + group: &item.group, + url: &item.url, + item_type: item.item_type, + })); + break; + } + } + } + } + if resolved.is_none() && target.has_output(TargetType::M3u) { + if let Some(mut items) = iter_raw_m3u_target_playlist(app_config, &target, Some(cluster)).await { + while let Some(entry) = items.next().await { + let Ok(item) = entry else { continue }; + if item.virtual_id == wanted { + resolved = Some(recording_candidate(&RecordingCandidateFields { + virtual_id: item.virtual_id, + input_name: &item.input_name, + name: &item.name, + title: &item.title, + group: &item.group, + url: &item.url, + item_type: item.item_type, + })); + break; + } + } + } + } + let resolved = resolved?; + if missing_input { + return is_recording_input_in_target_scope(app_config, &target, &resolved.input_name).then_some(resolved); + } + (resolved.input_name == input_name).then_some(resolved) +} + +/// `SourcesConfigDto::prepare` rejects duplicate target names across sources. +fn find_target_by_name(app_config: &crate::model::AppConfig, target_name: &str) -> Option> { + app_config + .sources + .load() + .sources + .iter() + .flat_map(|source| &source.targets) + .find(|target| target.name == target_name) + .cloned() +} + +/// Whether `input_name` belongs to the source that configures `target`. +fn is_recording_input_in_target_scope( + app_config: &crate::model::AppConfig, + target: &ConfigTarget, + input_name: &str, +) -> bool { + resolve_recording_target(app_config, &target.name, input_name).is_some_and(|configured| configured.id == target.id) +} + +/// Compares a playlist item's EPG channel id against a requested one under the +/// target's configured output casing. +/// +/// The web UI reads channel ids from the target EPG database, where they are +/// written in the target's output case (`lowercase_ids`). The persisted +/// playlist keeps the source casing, so a raw comparison misses every channel +/// whose source id differs in case from the EPG database key. +pub(super) fn epg_channel_id_matches( + item_epg_channel_id: Option<&Arc>, + requested: &Arc, + output_case: EpgIdOutputCase, +) -> bool { + item_epg_channel_id + .is_some_and(|item_id| canonicalize_output_epg_id(item_id, output_case).as_ref() == requested.as_ref()) +} + +/// Picks the channel to record from every live item that carries the requested +/// EPG channel id. +/// +/// Providers commonly publish one EPG id for several streams of the same +/// channel (HD, FHD, 4K, backup feeds). The web UI shows them as one EPG row +/// labelled with one of their titles, so that title is the best hint for +/// which stream the user meant. Without a matching title every candidate airs +/// the same programme, and the first one in playlist order is taken. +pub(super) fn select_epg_channel_candidate( + candidates: Vec, + channel_name: Option<&str>, +) -> Option { + let hint = channel_name.map(str::trim).filter(|name| !name.is_empty()); + if let Some(hint) = hint { + if let Some(index) = candidates.iter().position(|candidate| candidate.title == hint) { + return candidates.into_iter().nth(index); + } + let hint = hint.to_lowercase(); + if let Some(index) = candidates.iter().position(|candidate| candidate.title.to_lowercase() == hint) { + return candidates.into_iter().nth(index); + } + } + candidates.into_iter().next() +} + +pub(in crate::api) async fn resolve_target_live_recording_source_by_epg_channel( + app_config: &crate::model::AppConfig, + target_name: &str, + input_name: &str, + epg_channel_id: &str, + channel_name: Option<&str>, +) -> Option { + let target = find_target_by_name(app_config, target_name)?; + let output_case = + EpgIdOutputCase::from_lowercase(target.options.as_ref().is_some_and(ConfigTargetOptions::lowercase_epg_ids)); + let requested = canonicalize_untrusted_epg_id(epg_channel_id, output_case); + let mut candidates = Vec::new(); + if target.has_output(TargetType::Xtream) { + if let Some(mut items) = iter_raw_xtream_target_playlist(app_config, &target, XtreamCluster::Live).await { + while let Some(entry) = items.next().await { + let Ok(item) = entry else { continue }; + if epg_channel_id_matches(item.epg_channel_id.as_ref(), &requested, output_case) { + candidates.push(recording_candidate(&RecordingCandidateFields { + virtual_id: item.virtual_id, + input_name: &item.input_name, + name: &item.name, + title: &item.title, + group: &item.group, + url: &item.url, + item_type: item.item_type, + })); + } + } + } + } else if target.has_output(TargetType::M3u) { + if let Some(mut items) = iter_raw_m3u_target_playlist(app_config, &target, Some(XtreamCluster::Live)).await { + while let Some(entry) = items.next().await { + let Ok(item) = entry else { continue }; + if epg_channel_id_matches(item.epg_channel_id.as_ref(), &requested, output_case) { + candidates.push(recording_candidate(&RecordingCandidateFields { + virtual_id: item.virtual_id, + input_name: &item.input_name, + name: &item.name, + title: &item.title, + group: &item.group, + url: &item.url, + item_type: item.item_type, + })); + } + } + } + } + // Apply input constraints before the title hint can select another feed. + candidates.retain(|candidate| { + (input_name.trim().is_empty() || candidate.input_name == input_name) + && is_recording_input_in_target_scope(app_config, &target, &candidate.input_name) + }); + select_epg_channel_candidate(candidates, channel_name) +} + +#[derive(Debug, Deserialize)] +pub(super) struct RecordingStreamQuery { + pub(super) target_name: String, + pub(super) input_name: String, + pub(super) provider_allocation_id: Option, +} + +pub(super) async fn playlist_recording_stream( + fingerprint: crate::auth::Fingerprint, + axum::extract::Path((token, cluster, virtual_id)): axum::extract::Path<(String, String, u32)>, + axum::extract::Query(query): axum::extract::Query, + axum::extract::State(app_state): axum::extract::State>, + req_headers: axum::http::HeaderMap, +) -> impl IntoResponse + Send { + if !verify_access_token(&token, &app_state.app_config.access_token_secret, crate::auth::scope::INTERNAL_PLAYER) { + return axum::http::StatusCode::FORBIDDEN.into_response(); + } + let ctxt = try_result_bad_request!(ApiStreamContext::from_str(cluster.as_str())); + let resolved = { + let sources = app_state.app_config.sources.load(); + resolve_recording_config(sources.as_ref(), &query.target_name, &query.input_name) + }; + let Some(resolved) = resolved else { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + }; + let input = crate::api::api_utils::with_recording_headers(&app_state.app_config, resolved.input); + if resolved.target.has_output(TargetType::Xtream) { + let stream_id = virtual_id.to_string(); + return xtream_player_api_stream_with_resolved_target( + &fingerprint, + &req_headers, + &app_state, + resolved.target, + super::super::xtream_api::InternalPlaybackSource::Recording(input), + ApiStreamRequest::from_access_token(ctxt, &token, &stream_id, ""), + query.provider_allocation_id, + ) + .await + .into_response(); + } + if !resolved.target.has_output(TargetType::M3u) { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + } + + let pli = try_result_bad_request!( + m3u_get_item_for_stream_id(virtual_id, &app_state.app_config, &app_state.playlists, &resolved.target).await, + true, + format!("Failed to read m3u item for stream id {virtual_id}") + ); + if pli.input_name != input.name || pli.item_type.cluster() != ctxt.cluster() { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + } + let user = Arc::new(crate::api::api_utils::create_recording_proxy_user(&app_state)); + m3u_api_stream_loaded( + user, + resolved.target, + &fingerprint, + &req_headers, + &app_state, + pli, + input, + None, + None, + query.provider_allocation_id, + ) + .await + .into_response() +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/resources.rs b/backend/app/src/api/endpoints/v1_api_playlist/resources.rs new file mode 100644 index 000000000..395005638 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/resources.rs @@ -0,0 +1,181 @@ +use super::playlist_webplayer; +use crate::{ + api::{ + api_utils::resource_proxy_response, endpoints::api_playlist_utils::STALKER_RESOURCE_SCHEME, model::AppState, + }, + iptv::stalker::client::validate_public_playable_url, + model::{AppConfig, ConfigInput, ConfigInputFlags, ConfigInputOptions, ConfigInputUpdateQuality}, + processing::processor::re_resolve_stalker_url, +}; +use axum::response::IntoResponse; +use log::error; +use shared::{ + model::{stalker::StalkerStreamKind, InputType, PlaylistRequest, PlaylistUrlResolveRequest, XtreamCluster}, + utils::{open_web_ui_resource_url, sanitize_sensitive_info, Internable}, +}; +use std::sync::Arc; +use url::Url; + +pub(super) fn create_config_input_for_m3u(url: &str) -> ConfigInput { + ConfigInput { + id: 0, + name: "m3u_req".intern(), + input_type: InputType::M3u, + url: String::from(url), + enabled: true, + options: Some(ConfigInputOptions { + flags: ConfigInputFlags::XtreamLiveStreamUsePrefix | ConfigInputFlags::ResolveBackground, + update_quality: ConfigInputUpdateQuality::default(), + resolve_delay: shared::defaults::default_resolve_delay_secs(), + probe_delay: shared::defaults::default_probe_delay_secs(), + probe_live_interval_hours: 120, + resolve_filter: None, + probe_filter: None, + flussonic_hls_catchup: shared::model::FlussonicHlsCatchup::Native, + flussonic_hls_catchup_max_duration_secs: shared::model::default_flussonic_hls_catchup_max_duration_secs(), + }), + ..Default::default() + } +} + +pub(super) fn create_config_input_for_xtream(username: &str, password: &str, host: &str) -> ConfigInput { + ConfigInput { + id: 0, + name: "xc_req".intern(), + input_type: InputType::Xtream, + url: String::from(host), + username: Some(String::from(username)), + password: Some(String::from(password)), + enabled: true, + options: Some(ConfigInputOptions { + flags: ConfigInputFlags::XtreamLiveStreamUsePrefix | ConfigInputFlags::ResolveBackground, + update_quality: ConfigInputUpdateQuality::default(), + resolve_delay: shared::defaults::default_resolve_delay_secs(), + probe_delay: shared::defaults::default_probe_delay_secs(), + probe_live_interval_hours: 120, + resolve_filter: None, + probe_filter: None, + flussonic_hls_catchup: shared::model::FlussonicHlsCatchup::Native, + flussonic_hls_catchup_max_duration_secs: shared::model::default_flussonic_hls_catchup_max_duration_secs(), + }), + ..Default::default() + } +} + +pub(super) fn resolve_provider_url_with_input(input: &ConfigInput, url: &str) -> String { + match input.resolve_url(url) { + Ok(resolved) => resolved.into_owned(), + Err(err) => { + let sanitized_url = sanitize_sensitive_info(url); + let err_text = err.to_string(); + let sanitized_err = sanitize_sensitive_info(&err_text); + error!("resolve_provider_url_with_input failed for url '{sanitized_url}': {sanitized_err}"); + url.to_string() + } + } +} + +pub(super) fn resolve_provider_url_for_request( + app_config: &AppConfig, + playlist_request: &PlaylistRequest, + url: &str, +) -> String { + if !url.starts_with(shared::utils::PROVIDER_SCHEME_PREFIX) { + return url.to_string(); + } + + match playlist_request { + PlaylistRequest::Input(input_name) => app_config + .get_input_by_name(&input_name.intern()) + .map_or_else(|| url.to_string(), |input| resolve_provider_url_with_input(input.as_ref(), url)), + PlaylistRequest::Target(target_id) => app_config + .get_target_by_id(*target_id) + .and_then(|target| app_config.get_inputs_for_target(&target.name)) + .and_then(|inputs| { + let mut matches = inputs.into_iter().filter(|input| input.get_resolve_provider(url).is_some()); + let first = matches.next()?; + if matches.next().is_some() { + return None; + } + Some(resolve_provider_url_with_input(first.as_ref(), url)) + }) + .unwrap_or_else(|| url.to_string()), + PlaylistRequest::CustomXtream(_) | PlaylistRequest::CustomM3u(_) => url.to_string(), + } +} + +pub(super) async fn playlist_resource( + req_headers: axum::http::HeaderMap, + axum::extract::Path(resource): axum::extract::Path, + axum::extract::State(app_state): axum::extract::State>, +) -> impl IntoResponse + Send { + let encrypt_secret = app_state.get_encrypt_secret(); + if let Ok(resource_url) = open_web_ui_resource_url(&encrypt_secret, &resource) { + if let Some((input_id, cluster, provider_id)) = parse_stalker_resource(&resource_url) { + return stalker_resource_response(&app_state, input_id, cluster, provider_id).await; + } + // This route serves every icon the Web UI shows, so it is classified like the player routes: + // a public destination goes out through the proxy-aware fetch, which honours a configured + // proxy, while one that is not provably public is fetched directly so the request cannot leave + // through a proxy that has no route to it. + resource_proxy_response(&app_state, &resource_url, &req_headers, None).await.into_response() + } else { + axum::http::StatusCode::BAD_REQUEST.into_response() + } +} + +fn parse_stalker_resource(resource: &str) -> Option<(u16, XtreamCluster, u32)> { + let mut parts = resource.strip_prefix(STALKER_RESOURCE_SCHEME)?.split('/'); + let input_id = parts.next()?.parse().ok()?; + let cluster = parts.next()?.parse().ok()?; + let provider_id = parts.next()?.parse().ok()?; + parts.next().is_none().then_some((input_id, cluster, provider_id)) +} + +async fn stalker_resource_response( + app_state: &Arc, + input_id: u16, + cluster: XtreamCluster, + provider_id: u32, +) -> axum::response::Response { + let Some(input) = app_state.app_config.get_input_by_id(input_id).filter(|input| input.input_type.is_stalker()) + else { + return axum::http::StatusCode::NOT_FOUND.into_response(); + }; + let kind = match cluster { + XtreamCluster::Live => StalkerStreamKind::Live, + XtreamCluster::Video => StalkerStreamKind::Movie, + XtreamCluster::Series => StalkerStreamKind::Episode, + }; + let client = app_state.http_clients.default.load().as_ref().clone(); + match re_resolve_stalker_url(&app_state.app_config, &client, &input, provider_id, kind, false).await { + Ok(Some(resolved_url)) => { + let Ok(url) = Url::parse(&resolved_url) else { + return axum::http::StatusCode::BAD_GATEWAY.into_response(); + }; + if validate_public_playable_url(&url).await.is_err() { + return axum::http::StatusCode::BAD_GATEWAY.into_response(); + } + axum::response::Redirect::temporary(url.as_str()).into_response() + } + Ok(None) => axum::http::StatusCode::NOT_FOUND.into_response(), + Err(err) => { + error!("Failed to resolve Stalker preview stream: {}", sanitize_sensitive_info(&err.to_string())); + axum::http::StatusCode::BAD_GATEWAY.into_response() + } + } +} + +pub(super) async fn playlist_resolve_url( + axum::extract::State(app_state): axum::extract::State>, + axum::extract::Json(request): axum::extract::Json, +) -> impl IntoResponse + Send { + match request { + PlaylistUrlResolveRequest::Webplayer { target_id, virtual_id, cluster } => { + playlist_webplayer(axum::extract::State(app_state), target_id, virtual_id, cluster).into_response() + } + PlaylistUrlResolveRequest::Provider { playlist_request, url } => { + resolve_provider_url_for_request(&app_state.app_config, &playlist_request, &url).into_response() + } + } +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/routes.rs b/backend/app/src/api/endpoints/v1_api_playlist/routes.rs new file mode 100644 index 000000000..078455242 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/routes.rs @@ -0,0 +1,69 @@ +use super::{ + playlist_content_live, playlist_content_series, playlist_content_vod, playlist_epg, playlist_episode_item, + playlist_filter_preview, playlist_recording_stream, playlist_resolve_url, playlist_resource, playlist_series_info, + playlist_update, playlist_update_status, playlist_webplayer_stream, +}; +use crate::api::{ + auth_middleware::permission_layer, + endpoints::{epg_grid_api, xmltv_api::stream_epg_api}, + model::AppState, +}; +use axum::Router; +use shared::model::permission::Permission; +use std::sync::Arc; + +pub fn v1_api_playlist_register_protected(router: Router>) -> axum::Router> { + router + .route("/playlist/resolve_url", axum::routing::post(playlist_resolve_url)) + .route("/playlist/update", axum::routing::post(playlist_update)) + .route("/playlist/update/status", axum::routing::get(playlist_update_status)) + .route("/playlist/epg", axum::routing::post(playlist_epg)) + .route("/playlist/epg/stream", axum::routing::post(stream_epg_api)) + .route("/playlist/epg/groups", axum::routing::post(epg_grid_api::playlist_epg_groups)) + .route("/playlist/epg/grid", axum::routing::post(epg_grid_api::playlist_epg_grid)) + .route("/playlist/live", axum::routing::post(playlist_content_live)) + .route("/playlist/vod", axum::routing::post(playlist_content_vod)) + .route("/playlist/series", axum::routing::post(playlist_content_series)) + .route("/playlist/series_info/{virtual_id}/{provider_id}", axum::routing::post(playlist_series_info)) + .route("/playlist/series/episode/{virtual_id}", axum::routing::post(playlist_episode_item)) + .route("/playlist/filter/preview", axum::routing::post(playlist_filter_preview)) +} + +pub fn v1_api_playlist_register_public(router: Router>) -> axum::Router> { + router + .route("/playlist/resource/{resource}", axum::routing::get(playlist_resource)) + .route( + "/playlist/webplayer/{token}/{target_id}/{cluster}/{stream_id}", + axum::routing::get(playlist_webplayer_stream), + ) + .route("/playlist/recording/{token}/{cluster}/{virtual_id}", axum::routing::get(playlist_recording_stream)) +} + +pub fn v1_api_playlist_register_with_permissions( + router: Router>, + app_state: &Arc, +) -> axum::Router> { + let read_routes = Router::new() + .route("/update/status", axum::routing::get(playlist_update_status)) + .route("/live", axum::routing::post(playlist_content_live)) + .route("/vod", axum::routing::post(playlist_content_vod)) + .route("/series", axum::routing::post(playlist_content_series)) + .route("/resolve_url", axum::routing::post(playlist_resolve_url)) + .route("/series_info/{virtual_id}/{provider_id}", axum::routing::post(playlist_series_info)) + .route("/series/episode/{virtual_id}", axum::routing::post(playlist_episode_item)) + .route("/filter/preview", axum::routing::post(playlist_filter_preview)) + .layer(permission_layer!(app_state, Permission::PlaylistRead)); + + let write_routes = Router::new() + .route("/update", axum::routing::post(playlist_update)) + .layer(permission_layer!(app_state, Permission::PlaylistWrite)); + + let epg_routes = Router::new() + .route("/epg", axum::routing::post(playlist_epg)) + .route("/epg/stream", axum::routing::post(stream_epg_api)) + .route("/epg/groups", axum::routing::post(epg_grid_api::playlist_epg_groups)) + .route("/epg/grid", axum::routing::post(epg_grid_api::playlist_epg_grid)) + .layer(permission_layer!(app_state, Permission::EpgRead)); + + router.nest("/playlist", read_routes.merge(write_routes).merge(epg_routes)) +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests.rs new file mode 100644 index 000000000..8897e75b6 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests.rs @@ -0,0 +1,19 @@ +use super::resolve_provider_url_for_request; + +mod admission; +mod behavior; +mod configuration; +mod lifecycle; +mod persistence; +mod protocol; +mod support; + +use self::support::{ + epg_dt, ics_source_dto, playlist_update_target, test_app_config, test_app_state, + test_app_state_with_manual_update_sender, xmltv_source_dto, +}; + +mod behavior_epg_preview; +mod behavior_manual_update; +mod behavior_provider_resolution; +mod behavior_recording_routes; diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/admission.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/admission.rs new file mode 100644 index 000000000..d2c9aecae --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/admission.rs @@ -0,0 +1,264 @@ +use super::{epg_dt, ics_source_dto, test_app_config, test_app_state, xmltv_source_dto}; +use crate::{ + model::{Config, ConfigInput, ConfigProvider, ConfigSource}, + processing::epg::get_input_raw_xmltv_file_path, +}; +use axum::{ + body::Body, + http::{Request, StatusCode}, + Router, +}; +use shared::{ + model::{ConfigProviderDto, EpgChannel, EpgConfigDto, EpgProgramme, IcsDummyConfigDto, IcsEpgSourceConfigDto}, + utils::Internable, +}; +use std::sync::Arc; +use tempfile::tempdir; +use tower::ServiceExt; + +#[test] +fn merge_epg_channels_prefers_higher_priority_metadata_and_fills_lower_priority_gaps() { + let low_priority = EpgChannel { + id: "demo.channel".intern(), + title: Some("Low".intern()), + icon: Some("http://low/icon.png".intern()), + programmes: vec![EpgProgramme::new_all(10, 20, "demo.channel".intern(), Some("Low Show".intern()), None, None)], + }; + let high_priority = EpgChannel { + id: "demo.channel".intern(), + title: Some("High".intern()), + icon: Some("http://high/icon.png".intern()), + programmes: vec![EpgProgramme::new_all( + 30, + 40, + "demo.channel".intern(), + Some("High Show".intern()), + None, + None, + )], + }; + let same_priority = EpgChannel { + id: "demo.channel".intern(), + title: Some("Same".intern()), + icon: Some("http://same/icon.png".intern()), + programmes: vec![ + EpgProgramme::new_all(30, 40, "demo.channel".intern(), Some("Duplicate".intern()), None, None), + EpgProgramme::new_all(50, 60, "demo.channel".intern(), Some("Second Show".intern()), None, None), + ], + }; + + let channels = super::super::merge_epg_channels(vec![ + (10, vec![low_priority]), + (0, vec![high_priority]), + (0, vec![same_priority]), + ]); + + assert_eq!(channels.len(), 1); + assert_eq!(channels[0].title.as_deref(), Some("High")); + assert_eq!(channels[0].icon.as_deref(), Some("http://high/icon.png")); + assert_eq!(channels[0].programmes.len(), 3); + assert_eq!( + channels[0].programmes.iter().map(|programme| (programme.start, programme.stop)).collect::>(), + vec![(10, 20), (30, 40), (50, 60)], + ); +} + +#[tokio::test] +async fn load_epg_channels_for_input_prefers_high_priority_ics_dummy_policy() { + let temp_dir = tempdir().expect("temp dir"); + for filename in ["low.ics", "high.ics"] { + tokio::fs::write(temp_dir.path().join(filename), "BEGIN:VCALENDAR\nEND:VCALENDAR") + .await + .expect("write empty ics source"); + } + + let mut low_priority = ics_source_dto("low.ics", "f1.calendar", 10); + low_priority.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { + enabled: true, + title: "Low priority".to_string(), + days_past: 0, + days_future: 0, + block_hours: 24, + ..IcsDummyConfigDto::default() + }, + ..IcsEpgSourceConfigDto::default() + }); + let mut high_priority = ics_source_dto("high.ics", "f1.calendar", -10); + high_priority.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { + enabled: true, + title: "High priority".to_string(), + days_past: 0, + days_future: 0, + block_hours: 24, + ..IcsDummyConfigDto::default() + }, + ..IcsEpgSourceConfigDto::default() + }); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { + sources: Some(vec![low_priority.clone(), high_priority.clone()]), + t_sources: vec![low_priority, high_priority], + smart_match: None, + })), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input.clone(), source); + app_config + .config + .store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Default::default() })); + + let app_state = test_app_state(Arc::new(app_config)); + let channels = super::super::load_epg_channels_for_input(&app_state, input.as_ref()) + .await + .expect("load epg") + .expect("channels"); + + assert_eq!(channels.len(), 1); + assert!(!channels[0].programmes.is_empty()); + assert!(channels[0].programmes.iter().all(|programme| programme.title.as_deref() == Some("High priority"))); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn playlist_epg_input_route_merges_multiple_cached_sources_by_priority() { + let temp_dir = tempdir().expect("temp dir"); + let provider = ConfigProvider::from(&ConfigProviderDto { + name: "demo".intern(), + urls: vec!["http://provider.example".intern()], + provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), + dns: None, + }); + let primary_url = "provider://demo/xmltv-primary.php?username=user&password=pass"; + let secondary_url = "provider://demo/xmltv-secondary.php?username=user&password=pass"; + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { + sources: Some(vec![ + xmltv_source_dto(secondary_url, 10), + xmltv_source_dto(primary_url, 0), + xmltv_source_dto("provider://demo/xmltv-same-priority.php?username=user&password=pass", 0), + ]), + t_sources: vec![ + xmltv_source_dto(secondary_url, 10), + xmltv_source_dto(primary_url, 0), + xmltv_source_dto("provider://demo/xmltv-same-priority.php?username=user&password=pass", 0), + ], + smart_match: None, + })), + provider_configs: Some(vec![Arc::new(provider)]), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input.clone(), source); + app_config + .config + .store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Default::default() })); + + let primary_path = + get_input_raw_xmltv_file_path(primary_url, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) + .await + .expect("primary epg path"); + let secondary_path = + get_input_raw_xmltv_file_path(secondary_url, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) + .await + .expect("secondary epg path"); + let same_priority_path = get_input_raw_xmltv_file_path( + "provider://demo/xmltv-same-priority.php?username=user&password=pass", + input.as_ref(), + temp_dir.path().to_string_lossy().as_ref(), + ) + .await + .expect("same priority epg path"); + + for path in [&primary_path, &secondary_path, &same_priority_path] { + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await.expect("epg dir"); + } + } + + let p0 = epg_dt(0); + let p1 = epg_dt(1); + let p2 = epg_dt(2); + let p3 = epg_dt(3); + + tokio::fs::write( + &secondary_path, + format!( + r#" + + + Secondary Channel + + + + Secondary Show + +"# + ), + ) + .await + .expect("write secondary epg"); + tokio::fs::write( + &primary_path, + format!( + r#" + + + Primary Channel + + + + Primary Show + +"# + ), + ) + .await + .expect("write primary epg"); + tokio::fs::write( + &same_priority_path, + format!( + r#" + + + Same Priority Channel + + + + Duplicate Show + + + Second Show + +"# + ), + ) + .await + .expect("write same priority epg"); + + let app_state = test_app_state(Arc::new(app_config)); + let router = super::super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); + let request = Request::builder() + .method("POST") + .uri("/playlist/epg") + .header("content-type", "application/json") + .body(Body::from(r#"{"Input":"input"}"#)) + .expect("request"); + + let response = router.into_service::().oneshot(request).await.expect("response"); + + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("body"); + let body_text = String::from_utf8(body.to_vec()).expect("utf8"); + assert!(body_text.contains("Primary Channel"), "{body_text}"); + assert!(!body_text.contains("Secondary Channel"), "{body_text}"); + assert!(body_text.contains("Primary Show"), "{body_text}"); + assert!(body_text.contains("Second Show"), "{body_text}"); + assert!(!body_text.contains("Secondary Show"), "{body_text}"); +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior.rs new file mode 100644 index 000000000..875fad719 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior.rs @@ -0,0 +1,56 @@ +use axum::{extract::Query, response::IntoResponse, Json, Router}; +use serde_json::json; +use std::collections::HashMap; + +pub(in crate::api::endpoints::v1_api_playlist::tests) async fn stalker_mock_handler( + Query(params): Query>, +) -> impl IntoResponse { + let action = params.get("action").map_or("", String::as_str); + let portal_type = params.get("type").map_or("", String::as_str); + let page = params.get("p").map_or("1", String::as_str); + let response = match (portal_type, action, page) { + ("stb", "handshake", _) => json!({"js": {"token": "preview-token"}}), + ("stb", "get_profile", _) => json!({"js": {"status": 1, "max_connections": 1}}), + ("stb", "get_capabilities", _) => json!({"js": {}}), + ("itv", "get_genres", _) => json!({"js": [{"id": "10", "title": "News"}]}), + ("itv", "get_ordered_list", "1") => json!({ + "js": { + "data": { + "101": { + "id": "101", + "name": "Demo Channel", + "category_id": "10", + "cmd": "ffmpeg http://streams.example/live/101" + }, + "102": { + "id": "102", + "name": "Private Channel", + "category_id": "10", + "cmd": "ffmpeg http://streams.example/live/102" + } + } + } + }), + ("itv", "create_link", _) => { + let destination = if params.get("cmd").is_some_and(|cmd| cmd.ends_with("/102")) { + "http://127.0.0.1/live/102" + } else { + "http://8.8.8.8/live/101" + }; + json!({"js": {"cmd": format!("ffmpeg {destination}")}}) + } + _ => json!({"js": []}), + }; + Json(response) +} + +pub(in crate::api::endpoints::v1_api_playlist::tests) async fn spawn_stalker_mock_server( +) -> (String, tokio::task::JoinHandle<()>) { + let router = Router::new().route("/server/load.php", axum::routing::get(stalker_mock_handler)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind mock stalker server"); + let base_url = format!("http://{}", listener.local_addr().expect("mock addr")); + let handle = tokio::spawn(async move { + axum::serve(listener, router).await.expect("serve mock stalker server"); + }); + (base_url, handle) +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_epg_preview.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_epg_preview.rs new file mode 100644 index 000000000..cbf145680 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_epg_preview.rs @@ -0,0 +1,378 @@ +use super::{epg_dt, ics_source_dto, test_app_config, test_app_state, xmltv_source_dto}; +use crate::{ + model::{Config, ConfigInput, ConfigProvider, ConfigSource}, + processing::epg::{get_input_raw_epg_file_path, get_input_raw_xmltv_file_path}, +}; +use axum::{ + body::Body, + http::{Request, StatusCode}, + Router, +}; +use shared::{ + model::{ConfigProviderDto, EpgConfigDto, IcsDummyConfigDto, IcsEpgSourceConfigDto}, + utils::Internable, +}; +use std::sync::Arc; +use tempfile::tempdir; +use tower::ServiceExt; + +#[tokio::test] +async fn load_epg_channels_for_input_uses_resolved_provider_epg_cache_file() { + let temp_dir = tempdir().expect("temp dir"); + let provider = ConfigProvider::from(&ConfigProviderDto { + name: "demo".intern(), + urls: vec!["http://provider.example".intern()], + provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), + dns: None, + }); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { + sources: Some(vec![xmltv_source_dto("provider://demo/xmltv.php?username=user&password=pass", 0)]), + t_sources: vec![xmltv_source_dto("provider://demo/xmltv.php?username=user&password=pass", 0)], + smart_match: None, + })), + provider_configs: Some(vec![Arc::new(provider)]), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input.clone(), source); + app_config + .config + .store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Default::default() })); + let raw_epg_path = get_input_raw_xmltv_file_path( + "provider://demo/xmltv.php?username=user&password=pass", + input.as_ref(), + temp_dir.path().to_string_lossy().as_ref(), + ) + .await + .expect("epg path"); + if let Some(parent) = raw_epg_path.parent() { + tokio::fs::create_dir_all(parent).await.expect("epg dir"); + } + let prog_start = epg_dt(0); + let prog_stop = epg_dt(1); + tokio::fs::write( + &raw_epg_path, + format!( + r#" + + + Demo Channel + + + Morning Show + +"# + ), + ) + .await + .expect("write epg"); + + let app_state = test_app_state(Arc::new(app_config)); + let channels = super::super::load_epg_channels_for_input(&app_state, input.as_ref()) + .await + .expect("load epg") + .expect("channels"); + + assert_eq!(channels.len(), 1); + assert_eq!(channels[0].id.as_ref(), "demo.channel"); + assert_eq!(channels[0].programmes.len(), 1); + assert_eq!(channels[0].programmes[0].title.as_ref().map(std::convert::AsRef::as_ref), Some("Morning Show")); +} + +#[tokio::test] +async fn load_epg_channels_for_input_reads_cached_ics_source() { + let temp_dir = tempdir().expect("temp dir"); + let ics_dto = ics_source_dto("https://example.com/f1.ics", "f1.calendar", -10); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { + sources: Some(vec![ics_dto.clone()]), + t_sources: vec![ics_dto], + smart_match: None, + })), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input.clone(), source); + app_config + .config + .store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Default::default() })); + let epg_source = &input.epg.as_ref().expect("epg").sources[0]; + let raw_epg_path = + get_input_raw_epg_file_path(epg_source, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) + .await + .expect("epg path"); + if let Some(parent) = raw_epg_path.parent() { + tokio::fs::create_dir_all(parent).await.expect("epg dir"); + } + tokio::fs::write( + &raw_epg_path, + "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:Practice 1\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT\nEND:VCALENDAR", + ) + .await + .expect("write ics"); + + let app_state = test_app_state(Arc::new(app_config)); + let channels = super::super::load_epg_channels_for_input(&app_state, input.as_ref()) + .await + .expect("load epg") + .expect("channels"); + + assert_eq!(channels.len(), 1); + assert_eq!(channels[0].id.as_ref(), "f1.calendar"); + assert_eq!(channels[0].title.as_deref(), Some("Formula 1")); + assert_eq!(channels[0].programmes[0].title.as_deref(), Some("Practice 1")); +} + +#[tokio::test] +async fn load_epg_channels_for_input_redownloads_invalid_cached_ics_source() { + let temp_dir = tempdir().expect("temp dir"); + tokio::fs::write( + temp_dir.path().join("valid.ics"), + "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:Downloaded\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT\nEND:VCALENDAR", + ) + .await + .expect("write valid ics source"); + let ics_dto = ics_source_dto("valid.ics", "f1.calendar", -10); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { + sources: Some(vec![ics_dto.clone()]), + t_sources: vec![ics_dto], + smart_match: None, + })), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input.clone(), source); + app_config + .config + .store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Default::default() })); + let epg_source = &input.epg.as_ref().expect("epg").sources[0]; + let raw_epg_path = + get_input_raw_epg_file_path(epg_source, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) + .await + .expect("epg path"); + if let Some(parent) = raw_epg_path.parent() { + tokio::fs::create_dir_all(parent).await.expect("epg dir"); + } + tokio::fs::write(&raw_epg_path, "upstream returned an error page").await.expect("write invalid cache"); + + let app_state = test_app_state(Arc::new(app_config)); + let channels = super::super::load_epg_channels_for_input(&app_state, input.as_ref()) + .await + .expect("load epg") + .expect("channels"); + + assert_eq!(channels.len(), 1); + assert_eq!(channels[0].programmes[0].title.as_deref(), Some("Downloaded")); + assert_eq!( + tokio::fs::read_to_string(&raw_epg_path).await.expect("updated cache"), + "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:Downloaded\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT\nEND:VCALENDAR" + ); +} + +#[tokio::test] +async fn load_epg_channels_for_input_preserves_invalid_cache_when_refresh_fails() { + let temp_dir = tempdir().expect("temp dir"); + let invalid_cache = "upstream returned an error page"; + let ics_dto = ics_source_dto("missing.ics", "f1.calendar", -10); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { + sources: Some(vec![ics_dto.clone()]), + t_sources: vec![ics_dto], + smart_match: None, + })), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input.clone(), source); + app_config + .config + .store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Default::default() })); + let epg_source = &input.epg.as_ref().expect("epg").sources[0]; + let raw_epg_path = + get_input_raw_epg_file_path(epg_source, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) + .await + .expect("epg path"); + if let Some(parent) = raw_epg_path.parent() { + tokio::fs::create_dir_all(parent).await.expect("epg dir"); + } + tokio::fs::write(&raw_epg_path, invalid_cache).await.expect("write invalid cache"); + + let app_state = test_app_state(Arc::new(app_config)); + assert!(super::super::load_epg_channels_for_input(&app_state, input.as_ref()).await.is_err()); + assert_eq!(tokio::fs::read_to_string(&raw_epg_path).await.expect("preserved cache"), invalid_cache); +} + +#[tokio::test] +async fn load_epg_channels_for_input_applies_ics_dummy_policy_in_preview() { + let temp_dir = tempdir().expect("temp dir"); + tokio::fs::write(temp_dir.path().join("empty.ics"), "BEGIN:VCALENDAR\nEND:VCALENDAR") + .await + .expect("write empty ics source"); + let mut ics_dto = ics_source_dto("empty.ics", "f1.calendar", -10); + ics_dto.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { + enabled: true, + title: "No F1".to_string(), + days_past: 0, + days_future: 0, + block_hours: 24, + ..IcsDummyConfigDto::default() + }, + ..IcsEpgSourceConfigDto::default() + }); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { + sources: Some(vec![ics_dto.clone()]), + t_sources: vec![ics_dto], + smart_match: None, + })), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input.clone(), source); + app_config + .config + .store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Default::default() })); + + let app_state = test_app_state(Arc::new(app_config)); + let channels = super::super::load_epg_channels_for_input(&app_state, input.as_ref()) + .await + .expect("load epg") + .expect("channels"); + + assert_eq!(channels.len(), 1); + assert_eq!(channels[0].id.as_ref(), "f1.calendar"); + assert!(!channels[0].programmes.is_empty()); + assert!(channels[0].programmes.iter().all(|programme| programme.title.as_deref() == Some("No F1"))); +} + +#[tokio::test] +async fn playlist_epg_custom_route_rejects_invalid_url_scheme() { + let input = Arc::new(ConfigInput { id: 7, name: "input".intern(), ..Default::default() }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_state = test_app_state(Arc::new(test_app_config(input, source))); + let router = super::super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); + let request = Request::builder() + .method("POST") + .uri("/playlist/epg") + .header("content-type", "application/json") + .body(Body::from(r#"{"Custom":"ftp://example.com/epg.xml"}"#)) + .expect("request"); + + let response = router.into_service::().oneshot(request).await.expect("response"); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn playlist_epg_input_route_returns_cached_provider_epg() { + let temp_dir = tempdir().expect("temp dir"); + let provider = ConfigProvider::from(&ConfigProviderDto { + name: "demo".intern(), + urls: vec!["http://provider.example".intern()], + provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), + dns: None, + }); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { + sources: Some(vec![xmltv_source_dto("provider://demo/xmltv.php?username=user&password=pass", 0)]), + t_sources: vec![xmltv_source_dto("provider://demo/xmltv.php?username=user&password=pass", 0)], + smart_match: None, + })), + provider_configs: Some(vec![Arc::new(provider)]), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input.clone(), source); + app_config + .config + .store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Default::default() })); + let raw_epg_path = get_input_raw_xmltv_file_path( + "provider://demo/xmltv.php?username=user&password=pass", + input.as_ref(), + temp_dir.path().to_string_lossy().as_ref(), + ) + .await + .expect("epg path"); + if let Some(parent) = raw_epg_path.parent() { + tokio::fs::create_dir_all(parent).await.expect("epg dir"); + } + let prog_start = epg_dt(0); + let prog_stop = epg_dt(1); + tokio::fs::write( + &raw_epg_path, + format!( + r#" + + + Demo Channel + + + Morning Show + +"# + ), + ) + .await + .expect("write epg"); + + let app_state = test_app_state(Arc::new(app_config)); + let router = super::super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); + let request = Request::builder() + .method("POST") + .uri("/playlist/epg") + .header("content-type", "application/json") + .body(Body::from(r#"{"Input":"input"}"#)) + .expect("request"); + + let response = router.into_service::().oneshot(request).await.expect("response"); + + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("body"); + let body_text = String::from_utf8(body.to_vec()).expect("utf8"); + assert!(body_text.contains("demo.channel"), "{body_text}"); + assert!(body_text.contains("Morning Show"), "{body_text}"); +} + +#[tokio::test] +async fn playlist_epg_input_route_returns_no_content_for_unknown_input_name() { + let provider = ConfigProvider::from(&ConfigProviderDto { + name: "demo".intern(), + urls: vec!["http://provider.example".intern()], + provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), + dns: None, + }); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + provider_configs: Some(vec![Arc::new(provider)]), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_state = test_app_state(Arc::new(test_app_config(input, source))); + let router = super::super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); + let request = Request::builder() + .method("POST") + .uri("/playlist/epg") + .header("content-type", "application/json") + .body(Body::from(r#"{"Input":"missing"}"#)) + .expect("request"); + + let response = router.into_service::().oneshot(request).await.expect("response"); + + assert_eq!(response.status(), StatusCode::NO_CONTENT); +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_manual_update.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_manual_update.rs new file mode 100644 index 000000000..0bdc1c88c --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_manual_update.rs @@ -0,0 +1,627 @@ +use super::{playlist_update_target, test_app_config, test_app_state, test_app_state_with_manual_update_sender}; +use crate::model::{Config, ConfigInput, ConfigSource, SourcesConfig}; +use axum::{ + body::Body, + extract::State, + http::{Request, StatusCode}, + response::IntoResponse, + Json, Router, +}; +use serde_json::json; +use shared::{ + model::{ + InputRefreshOverride, InputRefreshPolicy, InputType, OperationRunAccepted, + PersistedPlaylistUpdateClusterSnapshot, PersistedPlaylistUpdateClusterState, + PersistedPlaylistUpdateQualityDecision, PersistedPlaylistUpdateQualitySnapshot, + PersistedPlaylistUpdateTechnicalState, PlaylistUpdateDataSource, PlaylistUpdateRequestDto, + PlaylistUpdateRequestPayload, PlaylistUpdateStatusDto, XtreamCluster, + }, + utils::Internable, +}; +use std::{collections::HashMap, sync::Arc}; +use tempfile::tempdir; +use tokio::sync::mpsc; +use tower::ServiceExt; + +#[test] +fn manual_update_capabilities_reject_unsupported_actions_and_keep_all_required_inputs() { + use shared::model::{InputUpdateAction, InputUpdateRequest}; + for (input_type, allowed) in [ + (InputType::Xtream, 3), + (InputType::Stalker, 3), + (InputType::M3u, 3), + (InputType::Plex, 2), + (InputType::Library, 1), + (InputType::Jellyfin, 0), + (InputType::Emby, 0), + (InputType::M3uBatch, 0), + (InputType::XtreamBatch, 0), + (InputType::StalkerBatch, 0), + (InputType::Staged, 0), + ] { + for enabled in [false, true] { + let input = Arc::new(ConfigInput { + id: 2, + name: "selected".intern(), + input_type, + enabled, + ..ConfigInput::default() + }); + let companion = Arc::new(ConfigInput { + id: 3, + name: "companion".intern(), + input_type: InputType::M3u, + enabled: true, + ..ConfigInput::default() + }); + let sources = SourcesConfig { + inputs: vec![input.clone(), companion.clone()], + sources: vec![ConfigSource { + inputs: vec![input.name.clone(), companion.name.clone()], + targets: vec![playlist_update_target(20, "target")], + }], + ..SourcesConfig::default() + }; + for (index, action) in [ + InputUpdateAction::Provider(InputRefreshPolicy::NORMAL), + InputUpdateAction::Provider(InputRefreshPolicy::REFRESH), + InputUpdateAction::Provider(InputRefreshPolicy::FORCE), + InputUpdateAction::Rescan, + ] + .into_iter() + .enumerate() + { + let request = PlaylistUpdateRequestDto { + target_ids: Some(vec![20]), + input_action: Some(InputUpdateRequest { input_id: 2, action }), + ..PlaylistUpdateRequestDto::default() + }; + let result = super::super::resolve_manual_playlist_update_targets(&sources, &request); + let expected = enabled && if input_type == InputType::Library { index == 3 } else { index < allowed }; + assert_eq!(result.is_ok(), expected, "{input_type} {action:?}"); + if let Ok(targets) = result { + assert_eq!(targets.targets, vec![20]); + assert_eq!(targets.inputs, vec![2, 3]); + } + } + } + } +} + +#[tokio::test] +async fn manual_update_library_rescan_uses_same_queue_and_requires_library_permission() { + use shared::model::{InputUpdateAction, InputUpdateRequest, LibraryConfigDto, WebAuthConfigDto, WebUiConfigDto}; + let input = Arc::new(ConfigInput { + id: 2, + name: "library".intern(), + input_type: InputType::Library, + enabled: true, + ..ConfigInput::default() + }); + let app_config = Arc::new(test_app_config( + input.clone(), + ConfigSource { inputs: vec![input.name.clone()], targets: vec![playlist_update_target(20, "target")] }, + )); + let mut config = (**app_config.config.load()).clone(); + config.library = + Some(crate::model::LibraryConfig::from(&LibraryConfigDto { enabled: true, ..LibraryConfigDto::default() })); + config.web_ui = Some(crate::model::WebUiConfig::from(&WebUiConfigDto { + auth: Some(WebAuthConfigDto::default()), + ..WebUiConfigDto::default() + })); + app_config.config.store(Arc::new(config)); + let mut sources = (**app_config.sources.load()).clone(); + sources.inputs.push(Arc::new(ConfigInput { + id: 3, + name: "other-input".intern(), + input_type: InputType::M3u, + enabled: true, + ..ConfigInput::default() + })); + sources.sources.push(ConfigSource { + inputs: vec!["other-input".intern()], + targets: vec![playlist_update_target(21, "unrelated-target")], + }); + app_config.sources.store(Arc::new(sources)); + let (sender, mut receiver) = mpsc::channel(1); + let app_state = test_app_state_with_manual_update_sender(app_config, sender); + let action = InputUpdateRequest { input_id: 2, action: InputUpdateAction::Rescan }; + let request = PlaylistUpdateRequestDto { + target_ids: Some(vec![20]), + input_action: Some(action), + ..PlaylistUpdateRequestDto::default() + }; + let mut claims: shared::model::Claims = + serde_json::from_value(serde_json::json!({"username":"test", "iss":"test", "iat":0, "exp":0})).unwrap(); + let rejected = super::super::playlist_update( + State(app_state.clone()), + Some(axum::Extension(crate::api::auth_middleware::VerifiedClaims(claims.clone()))), + Json(PlaylistUpdateRequestPayload::Current(request.clone())), + ) + .await + .into_response(); + assert_eq!(rejected.status(), StatusCode::FORBIDDEN); + assert!(receiver.try_recv().is_err()); + let unauthenticated = super::super::playlist_update( + State(app_state.clone()), + None, + Json(PlaylistUpdateRequestPayload::Current(request.clone())), + ) + .await + .into_response(); + assert_eq!(unauthenticated.status(), StatusCode::UNAUTHORIZED); + assert!(receiver.try_recv().is_err()); + claims.permissions.set(shared::model::permission::Permission::LibraryWrite); + // The legacy-name compatibility must never relax the new Rescan wire form. + let invalid_requests = [ + PlaylistUpdateRequestDto { target_ids: None, ..request.clone() }, + PlaylistUpdateRequestDto { targets: vec!["target".to_string()], target_ids: None, ..request.clone() }, + PlaylistUpdateRequestDto { target_ids: Some(Vec::new()), ..request.clone() }, + PlaylistUpdateRequestDto { target_ids: Some(vec![99]), ..request.clone() }, + PlaylistUpdateRequestDto { target_ids: Some(vec![21]), ..request.clone() }, + PlaylistUpdateRequestDto { targets: vec!["target".to_string()], ..request.clone() }, + PlaylistUpdateRequestDto { + input_refresh: Some(InputRefreshOverride { input_id: 2, policy: InputRefreshPolicy::NORMAL }), + ..request.clone() + }, + ]; + for invalid in invalid_requests { + let rejected = super::super::playlist_update( + State(app_state.clone()), + Some(axum::Extension(crate::api::auth_middleware::VerifiedClaims(claims.clone()))), + Json(PlaylistUpdateRequestPayload::Current(invalid)), + ) + .await + .into_response(); + assert_eq!(rejected.status(), StatusCode::BAD_REQUEST); + assert!(receiver.try_recv().is_err()); + } + let accepted = super::super::playlist_update( + State(app_state), + Some(axum::Extension(crate::api::auth_middleware::VerifiedClaims(claims))), + Json(PlaylistUpdateRequestPayload::Current(request)), + ) + .await + .into_response(); + assert_eq!(accepted.status(), StatusCode::ACCEPTED); + let queued = receiver.recv().await.unwrap(); + assert_eq!(queued.input_action, Some(action)); + assert_eq!(queued.targets.targets, vec![20]); +} + +#[tokio::test] +async fn manual_update_run_target_ids_and_refresh_override_keep_the_accepted_queue_identity() { + let input = Arc::new(ConfigInput { + id: 17, + name: "stable-input".intern(), + input_type: InputType::Xtream, + enabled: true, + ..ConfigInput::default() + }); + let app_config = Arc::new(test_app_config( + Arc::clone(&input), + ConfigSource { + inputs: vec![Arc::clone(&input.name)], + targets: vec![playlist_update_target(1, "shared-name"), playlist_update_target(4, "shared-name")], + }, + )); + let (sender, mut receiver) = mpsc::channel(1); + let app_state = test_app_state_with_manual_update_sender(app_config, sender); + let input_refresh = InputRefreshOverride { input_id: input.id, policy: InputRefreshPolicy::FORCE }; + + let response = super::super::playlist_update( + State(app_state), + None, + Json(PlaylistUpdateRequestPayload::Current(PlaylistUpdateRequestDto { + targets: Vec::new(), + target_ids: Some(vec![4, 1]), + input_refresh: Some(input_refresh), + input_action: None, + })), + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::ACCEPTED); + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("accepted response body"); + let accepted = serde_json::from_slice::(&body).expect("accepted response payload"); + let queued = receiver.recv().await.expect("accepted update request"); + assert_eq!(accepted.run_id.as_ref(), Some(&queued.run_id)); + assert!(queued.targets.enabled); + assert_eq!(queued.targets.inputs, vec![input.id]); + assert_eq!(queued.targets.targets, vec![4, 1]); + assert_eq!(queued.targets.target_names, vec!["shared-name", "shared-name"]); + assert_eq!( + queued.input_action, + Some(shared::model::InputUpdateRequest { + input_id: input_refresh.input_id, + action: shared::model::InputUpdateAction::Provider(input_refresh.policy) + }) + ); +} + +#[tokio::test] +async fn manual_update_target_ids_reject_unknown_empty_and_ambiguous_selections() { + let input = Arc::new(ConfigInput { + id: 17, + name: "stable-input".intern(), + input_type: InputType::Xtream, + enabled: true, + ..ConfigInput::default() + }); + let app_config = Arc::new(test_app_config( + Arc::clone(&input), + ConfigSource { + inputs: vec![Arc::clone(&input.name)], + targets: vec![playlist_update_target(1, "known-target")], + }, + )); + let (sender, mut receiver) = mpsc::channel(1); + let app_state = test_app_state_with_manual_update_sender(app_config, sender); + let input_refresh = Some(InputRefreshOverride { input_id: input.id, policy: InputRefreshPolicy::REFRESH }); + let invalid_requests = [ + PlaylistUpdateRequestDto { + targets: vec!["known-target".to_string()], + input_action: Some(shared::model::InputUpdateRequest { + input_id: input.id, + action: shared::model::InputUpdateAction::Provider(InputRefreshPolicy::REFRESH), + }), + ..PlaylistUpdateRequestDto::default() + }, + PlaylistUpdateRequestDto { targets: Vec::new(), target_ids: Some(vec![99]), input_refresh, input_action: None }, + PlaylistUpdateRequestDto { + targets: Vec::new(), + target_ids: Some(Vec::new()), + input_refresh, + input_action: None, + }, + PlaylistUpdateRequestDto { + targets: vec!["known-target".to_string()], + target_ids: Some(vec![1]), + input_refresh, + input_action: None, + }, + ]; + + for request in invalid_requests { + let response = super::super::playlist_update( + State(Arc::clone(&app_state)), + None, + Json(PlaylistUpdateRequestPayload::Current(request)), + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(receiver.try_recv().is_err()); + } +} + +#[tokio::test] +async fn manual_update_target_ids_preserve_named_legacy_and_empty_bulk_requests() { + let input = Arc::new(ConfigInput { + id: 17, + name: "stable-input".intern(), + input_type: InputType::Xtream, + enabled: true, + ..ConfigInput::default() + }); + let app_config = Arc::new(test_app_config( + Arc::clone(&input), + ConfigSource { + inputs: vec![Arc::clone(&input.name)], + targets: vec![playlist_update_target(1, "shared-name"), playlist_update_target(4, "shared-name")], + }, + )); + let (sender, mut receiver) = mpsc::channel(1); + let app_state = test_app_state_with_manual_update_sender(app_config, sender); + let requests = [ + PlaylistUpdateRequestPayload::Current(PlaylistUpdateRequestDto { + targets: vec!["shared-name".to_string()], + target_ids: None, + input_refresh: None, + input_action: None, + }), + PlaylistUpdateRequestPayload::LegacyTargets(vec!["shared-name".to_string()]), + PlaylistUpdateRequestPayload::Current(PlaylistUpdateRequestDto { + targets: Vec::new(), + target_ids: None, + input_refresh: None, + input_action: None, + }), + ]; + + for (index, request) in requests.into_iter().enumerate() { + let response = + super::super::playlist_update(State(Arc::clone(&app_state)), None, Json(request)).await.into_response(); + + assert_eq!(response.status(), StatusCode::ACCEPTED); + let queued = receiver.recv().await.expect("accepted legacy request"); + assert_eq!(queued.input_action, None); + if index < 2 { + assert!(queued.targets.enabled); + assert_eq!(queued.targets.targets, vec![1, 4]); + assert_eq!(queued.targets.target_names, vec!["shared-name", "shared-name"]); + } else { + assert!(!queued.targets.enabled); + assert!(queued.targets.targets.is_empty()); + assert!(queued.targets.target_names.is_empty()); + } + } +} + +#[tokio::test] +async fn manual_update_m3u_force_is_queued_by_stable_ids_and_preserves_conflict() { + use shared::model::{InputUpdateAction, InputUpdateRequest}; + let input = Arc::new(ConfigInput { + id: 17, + name: "m3u-force".intern(), + input_type: InputType::M3u, + enabled: true, + ..ConfigInput::default() + }); + let config = Arc::new(test_app_config( + input.clone(), + ConfigSource { + inputs: vec![input.name.clone()], + targets: vec![playlist_update_target(1, "selected"), playlist_update_target(2, "other")], + }, + )); + let (sender, mut receiver) = mpsc::channel(1); + let app_state = test_app_state_with_manual_update_sender(config, sender); + let action = + InputUpdateRequest { input_id: input.id, action: InputUpdateAction::Provider(InputRefreshPolicy::FORCE) }; + let request = PlaylistUpdateRequestPayload::Current(PlaylistUpdateRequestDto { + target_ids: Some(vec![1]), + input_action: Some(action), + ..PlaylistUpdateRequestDto::default() + }); + let accepted = + super::super::playlist_update(State(app_state.clone()), None, Json(request.clone())).await.into_response(); + assert_eq!(accepted.status(), StatusCode::ACCEPTED); + let conflict = super::super::playlist_update(State(app_state), None, Json(request)).await.into_response(); + assert_eq!(conflict.status(), StatusCode::CONFLICT); + let queued = receiver.try_recv().unwrap(); + assert_eq!(queued.input_action, Some(action)); + assert_eq!(queued.targets.targets, [1]); + assert_eq!(queued.targets.inputs, [17]); + assert!(receiver.try_recv().is_err()); +} + +#[tokio::test] +async fn manual_update_legacy_input_refresh_preserves_named_and_all_scope_policy_and_queue_conflicts() { + let input = Arc::new(ConfigInput { + id: 17, + name: "provider".intern(), + input_type: InputType::Xtream, + enabled: true, + ..ConfigInput::default() + }); + let app_config = Arc::new(test_app_config( + input.clone(), + ConfigSource { + inputs: vec![input.name.clone()], + targets: vec![playlist_update_target(1, "selected"), playlist_update_target(4, "other")], + }, + )); + let (sender, mut receiver) = mpsc::channel(1); + let app_state = test_app_state_with_manual_update_sender(app_config, sender); + for policy in [InputRefreshPolicy::NORMAL, InputRefreshPolicy::REFRESH, InputRefreshPolicy::FORCE] { + for names in [vec!["selected".to_string()], Vec::new()] { + let expected = + app_state.app_config.sources.load().validate_targets((!names.is_empty()).then_some(&names)).unwrap(); + let request: PlaylistUpdateRequestPayload = serde_json::from_value(json!({ + "targets": names, + "input_refresh": {"input_id": input.id, "policy": policy} + })) + .unwrap(); + let response = super::super::playlist_update(State(app_state.clone()), None, Json(request.clone())) + .await + .into_response(); + assert_eq!(response.status(), StatusCode::ACCEPTED); + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let accepted: OperationRunAccepted = serde_json::from_slice(&body).unwrap(); + let conflict = + super::super::playlist_update(State(app_state.clone()), None, Json(request)).await.into_response(); + assert_eq!(conflict.status(), StatusCode::CONFLICT); + let queued = receiver.try_recv().unwrap(); + assert_eq!(accepted.run_id.as_ref(), Some(&queued.run_id)); + assert_eq!(queued.targets.enabled, expected.enabled); + assert_eq!(queued.targets.inputs, expected.inputs); + assert_eq!(queued.targets.targets, expected.targets); + assert_eq!(queued.targets.target_names, expected.target_names); + assert_eq!( + queued.input_action, + Some(shared::model::InputUpdateRequest { + input_id: input.id, + action: shared::model::InputUpdateAction::Provider(policy), + }) + ); + assert!(receiver.try_recv().is_err()); + } + } +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn m3u_update_quality_playlist_update_status_reads_newest_timestamp_from_canonically_resolved_input_path() { + let temp_dir = tempdir().expect("temp dir"); + let input = Arc::new(ConfigInput { + id: 7, + name: "timestamped/provider legacy".intern(), + input_type: InputType::Xtream, + enabled: true, + ..ConfigInput::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: Vec::new() }; + let app_config = test_app_config(Arc::clone(&input), source); + app_config + .config + .store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Config::default() })); + let mut sources = app_config.sources.load().as_ref().clone(); + sources.inputs.push(Arc::new(ConfigInput { + id: 8, + name: "never-updated".intern(), + input_type: InputType::Stalker, + enabled: true, + ..ConfigInput::default() + })); + sources.inputs.push(Arc::new(ConfigInput { + id: 9, + name: "m3u-with-legacy-cluster".intern(), + input_type: InputType::M3u, + enabled: true, + ..ConfigInput::default() + })); + app_config.sources.store(Arc::new(sources)); + + let never_updated_path = temp_dir.path().join("input_never_updated"); + assert!(!never_updated_path.exists()); + let storage_path = crate::processing::input_cache::resolve_input_storage_path( + temp_dir.path().to_string_lossy().as_ref(), + &input.name, + ) + .await; + assert_eq!(storage_path.file_name().and_then(|name| name.to_str()), Some("input_timestamped_provider_legacy")); + let mut persisted = crate::processing::input_cache::InputStatus::default(); + persisted.clusters.insert( + XtreamCluster::Live.as_ref().to_string(), + crate::processing::input_cache::ClusterStatus { + status: crate::processing::input_cache::ClusterState::Ok, + timestamp: 101, + last_update: None, + }, + ); + persisted.clusters.insert( + XtreamCluster::Video.as_ref().to_string(), + crate::processing::input_cache::ClusterStatus { + status: crate::processing::input_cache::ClusterState::Failed, + timestamp: 303, + last_update: Some(PersistedPlaylistUpdateClusterSnapshot { + policy: Some(InputRefreshPolicy::REFRESH), + source: Some(PlaylistUpdateDataSource::Provider), + quality_guard_threshold: None, + quality: Some(PersistedPlaylistUpdateQualitySnapshot { + threshold: 95, + baseline_count: Some(4_812), + candidate_count: Some(3_104), + achieved_quality: Some(64), + decision: PersistedPlaylistUpdateQualityDecision::Rejected, + }), + active_count: Some(4_812), + technical_state: Some(PersistedPlaylistUpdateTechnicalState::Succeeded), + }), + }, + ); + persisted.clusters.insert( + XtreamCluster::Series.as_ref().to_string(), + crate::processing::input_cache::ClusterStatus { + status: crate::processing::input_cache::ClusterState::Ok, + timestamp: 202, + last_update: None, + }, + ); + crate::processing::input_cache::save_input_status(&storage_path, &persisted); + + let m3u_storage_path = crate::processing::input_cache::resolve_input_storage_path( + temp_dir.path().to_string_lossy().as_ref(), + "m3u-with-legacy-cluster", + ) + .await; + let mut m3u_persisted = crate::processing::input_cache::InputStatus::default(); + m3u_persisted.clusters.insert( + XtreamCluster::Live.as_ref().to_string(), + crate::processing::input_cache::ClusterStatus { + status: crate::processing::input_cache::ClusterState::Failed, + timestamp: 404, + last_update: None, + }, + ); + crate::processing::input_cache::save_input_status(&m3u_storage_path, &m3u_persisted); + + let app_state = test_app_state(Arc::new(app_config)); + let router = super::super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); + let response = router + .into_service::() + .oneshot(Request::builder().uri("/playlist/update/status").body(Body::empty()).expect("request")) + .await + .expect("response"); + + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("body"); + let status = serde_json::from_slice::(&body).expect("status response"); + assert_eq!(status.inputs.len(), 3); + assert_eq!(status.inputs[0].input_id, 7); + assert_eq!(status.inputs[0].last_update, Some(303)); + assert_eq!(status.inputs[0].clusters.len(), 3); + assert_eq!(status.inputs[0].clusters[0].cluster, XtreamCluster::Live); + assert_eq!(status.inputs[0].clusters[0].status, PersistedPlaylistUpdateClusterState::Ok); + assert_eq!(status.inputs[0].clusters[0].timestamp, 101); + assert_eq!(status.inputs[0].clusters[1].cluster, XtreamCluster::Video); + assert_eq!(status.inputs[0].clusters[1].status, PersistedPlaylistUpdateClusterState::Failed); + assert_eq!(status.inputs[0].clusters[1].timestamp, 303); + assert_eq!( + status.inputs[0].clusters[1].last_update, + Some(PersistedPlaylistUpdateClusterSnapshot { + policy: Some(InputRefreshPolicy::REFRESH), + source: Some(PlaylistUpdateDataSource::Provider), + quality_guard_threshold: None, + quality: Some(PersistedPlaylistUpdateQualitySnapshot { + threshold: 95, + baseline_count: Some(4_812), + candidate_count: Some(3_104), + achieved_quality: Some(64), + decision: PersistedPlaylistUpdateQualityDecision::Rejected, + }), + active_count: Some(4_812), + technical_state: Some(PersistedPlaylistUpdateTechnicalState::Succeeded), + }) + ); + assert_eq!(status.inputs[0].clusters[2].cluster, XtreamCluster::Series); + assert_eq!(status.inputs[0].clusters[2].status, PersistedPlaylistUpdateClusterState::Ok); + assert_eq!(status.inputs[0].clusters[2].timestamp, 202); + assert_eq!(status.inputs[1].input_id, 8); + assert_eq!(status.inputs[1].last_update, None); + assert!(status.inputs[1].clusters.is_empty()); + assert_eq!(status.inputs[2].input_id, 9); + assert_eq!(status.inputs[2].last_update, Some(404)); + assert_eq!(status.inputs[2].clusters.len(), 1, "Only the actually persisted M3U cluster is exposed"); + assert_eq!(status.inputs[2].clusters[0].cluster, XtreamCluster::Live); + assert_eq!(status.inputs[2].clusters[0].status, PersistedPlaylistUpdateClusterState::Failed); + assert!(never_updated_path.is_dir()); +} + +#[tokio::test] +async fn playlist_update_status_reader_runs_off_the_async_worker() { + use crate::processing::input_cache::{ + load_input_status, save_input_status, ClusterState, ClusterStatus, InputStatus, + }; + + let temp = tempdir().unwrap(); + let expected = InputStatus { + clusters: HashMap::from([( + "live".to_owned(), + ClusterStatus { status: ClusterState::Ok, timestamp: 17, last_update: None }, + )]), + ..InputStatus::default() + }; + save_input_status(temp.path(), &expected); + let before = std::fs::read(temp.path().join("status.json")).unwrap(); + let worker = std::thread::current().id(); + let path = temp.path().to_path_buf(); + let actual = super::super::read_playlist_update_input_status(move || { + assert_ne!(std::thread::current().id(), worker); + load_input_status(&path) + }) + .await + .unwrap(); + + assert_eq!(actual.clusters, expected.clusters); + assert_eq!(std::fs::read(temp.path().join("status.json")).unwrap(), before); +} + +#[tokio::test] +async fn playlist_update_status_reader_join_failure_returns_server_error_not_empty_status() { + let result = super::super::read_playlist_update_input_status(|| panic!("simulated status reader failure")).await; + assert_eq!(result.unwrap_err(), StatusCode::INTERNAL_SERVER_ERROR); +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_provider_resolution.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_provider_resolution.rs new file mode 100644 index 000000000..dacd98980 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_provider_resolution.rs @@ -0,0 +1,514 @@ +use super::{behavior::spawn_stalker_mock_server, resolve_provider_url_for_request, test_app_config, test_app_state}; +use crate::model::{ + AppConfig, Config, ConfigInput, ConfigInputOptions, ConfigInputUpdateQuality, ConfigProvider, ConfigSource, + ConfigTarget, SourcesConfig, +}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use axum::{ + body::Body, + http::{Request, StatusCode}, + Router, +}; +use shared::{ + foundation::Filter, + model::{ + provider_saturation::build_group_lookup, ConfigPaths, ConfigProviderDto, InputType, PlaylistRequest, + ProcessingOrder, XtreamCluster, + }, + utils::Internable, +}; +use std::sync::Arc; +use tempfile::tempdir; +use tower::ServiceExt; + +#[test] +fn resolve_provider_url_for_input_request_rewrites_provider_scheme() { + let provider = ConfigProvider::from(&ConfigProviderDto { + name: "demo".intern(), + urls: vec!["http://provider.example".intern()], + provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), + dns: None, + }); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + provider_configs: Some(vec![Arc::new(provider)]), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input, source); + let resolved = resolve_provider_url_for_request( + &app_config, + &PlaylistRequest::Input("input".to_string()), + "provider://demo/live/user/pass/1359.ts", + ); + + assert_eq!(resolved, "http://provider.example/live/user/pass/1359.ts"); +} + +#[test] +fn resolve_provider_url_for_target_request_rewrites_provider_scheme() { + let provider = ConfigProvider::from(&ConfigProviderDto { + name: "demo".intern(), + urls: vec!["http://provider.example".intern()], + provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), + dns: None, + }); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + provider_configs: Some(vec![Arc::new(provider)]), + ..Default::default() + }); + let target = Arc::new(ConfigTarget { + id: 11, + enabled: true, + name: "target".to_string(), + options: None, + sort: None, + filter: Filter::default().into(), + output: vec![], + rename: None, + mapping_ids: None, + mapping: Arc::default(), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + curation: None, + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![target] }; + let app_config = test_app_config(input, source); + let resolved = resolve_provider_url_for_request( + &app_config, + &PlaylistRequest::Target(11), + "provider://demo/live/user/pass/1359.ts", + ); + + assert_eq!(resolved, "http://provider.example/live/user/pass/1359.ts"); +} + +#[test] +fn resolve_provider_url_passthrough_for_unresolved_provider_input_request() { + let provider = ConfigProvider::from(&ConfigProviderDto { + name: "demo".intern(), + urls: vec!["http://provider.example".intern()], + provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), + dns: None, + }); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + provider_configs: Some(vec![Arc::new(provider)]), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input, source); + let original = "provider://unknown/live/user/pass/1359.ts"; + let resolved = + resolve_provider_url_for_request(&app_config, &PlaylistRequest::Input("input".to_string()), original); + + assert_eq!(resolved, original); +} + +#[test] +fn resolve_provider_url_passthrough_for_unresolved_provider_target_request() { + let provider = ConfigProvider::from(&ConfigProviderDto { + name: "demo".intern(), + urls: vec!["http://provider.example".intern()], + provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), + dns: None, + }); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + provider_configs: Some(vec![Arc::new(provider)]), + ..Default::default() + }); + let target = Arc::new(ConfigTarget { + id: 11, + enabled: true, + name: "target".to_string(), + options: None, + sort: None, + filter: Filter::default().into(), + output: vec![], + rename: None, + mapping_ids: None, + mapping: Arc::default(), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + curation: None, + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![target] }; + let app_config = test_app_config(input, source); + let original = "provider://unknown/live/user/pass/1359.ts"; + let resolved = resolve_provider_url_for_request(&app_config, &PlaylistRequest::Target(11), original); + + assert_eq!(resolved, original); +} + +#[test] +fn resolve_provider_url_passthrough_for_ambiguous_target_request() { + let provider_a = ConfigProvider::from(&ConfigProviderDto { + name: "shared".intern(), + urls: vec!["http://provider-a.example".intern()], + provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), + dns: None, + }); + let provider_b = ConfigProvider::from(&ConfigProviderDto { + name: "shared".intern(), + urls: vec!["http://provider-b.example".intern()], + provider_url_selection_policy: shared::model::ProviderUrlSelectionPolicy::default(), + dns: None, + }); + let input_a = Arc::new(ConfigInput { + id: 7, + name: "input-a".intern(), + provider_configs: Some(vec![Arc::new(provider_a)]), + ..Default::default() + }); + let input_b = Arc::new(ConfigInput { + id: 8, + name: "input-b".intern(), + provider_configs: Some(vec![Arc::new(provider_b)]), + ..Default::default() + }); + let target = Arc::new(ConfigTarget { + curation: None, + id: 11, + enabled: true, + name: "target".to_string(), + options: None, + sort: None, + filter: Filter::default().into(), + output: vec![], + rename: None, + mapping_ids: None, + mapping: Arc::default(), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }); + let source = + ConfigSource { inputs: vec![Arc::clone(&input_a.name), Arc::clone(&input_b.name)], targets: vec![target] }; + let inputs = vec![input_a, input_b]; + let sources = SourcesConfig { + batch_files: vec![], + provider: vec![], + group_lookup: build_group_lookup(&inputs), + inputs, + sources: vec![source], + templates: None, + }; + + let app_config = AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(sources)), + hdhomerun: Arc::new(ArcSwapOption::empty()), + api_proxy: Arc::new(ArcSwapOption::empty()), + file_locks: Arc::new(crate::utils::FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::empty()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(crate::model::MediaToolCapabilities::default()), + }; + + let original = "provider://shared/live/user/pass/1359.ts"; + let resolved = resolve_provider_url_for_request(&app_config, &PlaylistRequest::Target(11), original); + + assert_eq!(resolved, original); +} + +#[test] +fn build_playlist_webplayer_url_uses_cluster_stream_type() { + let live = + super::super::build_playlist_webplayer_url("http://player.example", "token123", 1, 42, XtreamCluster::Live); + let movie = + super::super::build_playlist_webplayer_url("http://player.example", "token123", 1, 42, XtreamCluster::Video); + let series = + super::super::build_playlist_webplayer_url("http://player.example", "token123", 1, 42, XtreamCluster::Series); + + assert_eq!(live, "http://player.example/api/v1/playlist/webplayer/token123/1/live/42"); + assert_eq!(movie, "http://player.example/api/v1/playlist/webplayer/token123/1/movie/42"); + assert_eq!(series, "http://player.example/api/v1/playlist/webplayer/token123/1/series/42"); +} + +#[tokio::test] +async fn playlist_live_input_route_supports_stalker_inputs() { + let temp_dir = tempdir().expect("temp dir"); + let (base_url, server_handle) = spawn_stalker_mock_server().await; + let input = Arc::new(ConfigInput { + id: 7, + name: "stalker-preview".intern(), + input_type: InputType::Stalker, + url: base_url, + enabled: true, + options: Some(ConfigInputOptions { + flags: crate::model::ConfigInputFlagsSet::new(), + update_quality: ConfigInputUpdateQuality::default(), + resolve_delay: shared::defaults::default_resolve_delay_secs(), + probe_delay: shared::defaults::default_probe_delay_secs(), + probe_live_interval_hours: 120, + resolve_filter: None, + probe_filter: None, + flussonic_hls_catchup: shared::model::FlussonicHlsCatchup::Native, + flussonic_hls_catchup_max_duration_secs: shared::model::default_flussonic_hls_catchup_max_duration_secs(), + }), + stalker: Some(crate::model::StalkerInputConfig { + device: None, + auth_mode: shared::model::StalkerAuthMode::Auto, + mag_preset: shared::model::StalkerMagPreset::GenericSafe, + endpoint_preference: shared::model::StalkerEndpointPreference::ServerLoad, + size_caps: None, + catalog_max_pages: None, + ..Default::default() + }), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(Arc::clone(&input), source); + let mut sources = app_config.sources.load().as_ref().clone(); + sources.inputs.push(Arc::new(ConfigInput { + id: 8, + name: "m3u".intern(), + input_type: InputType::M3u, + ..Default::default() + })); + app_config.sources.store(Arc::new(sources)); + app_config + .config + .store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Default::default() })); + let app_state = test_app_state(Arc::new(app_config)); + let router = + super::super::v1_api_playlist_register_protected(super::super::v1_api_playlist_register_public(Router::new())) + .with_state(Arc::clone(&app_state)); + let request = Request::builder() + .method("POST") + .uri("/playlist/live") + .header("content-type", "application/json") + .body(Body::from(r#"{"Input":"stalker-preview"}"#)) + .expect("request"); + + let response = router.clone().into_service::().oneshot(request).await.expect("response"); + + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.expect("body"); + let body_text = String::from_utf8(body.to_vec()).expect("utf8"); + assert_eq!(status, StatusCode::OK, "{body_text}"); + assert!(body_text.contains("Demo Channel"), "{body_text}"); + assert!(!body_text.contains("ffmpeg http://streams.example/live/101"), "{body_text}"); + let body_json: serde_json::Value = serde_json::from_str(&body_text).unwrap_or_default(); + let playback_url = body_json + .as_array() + .and_then(|items| items.first()) + .and_then(serde_json::Value::as_array) + .and_then(|item| item.get(6)) + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + assert!(!playback_url.is_empty(), "{body_text}"); + assert!(playback_url.starts_with("http") || playback_url.starts_with('/'), "{playback_url}"); + + let resource_path = playback_url + .find("/playlist/resource/") + .map(|index| &playback_url[index..]) + .expect("Stalker playback resource path"); + let response = router + .clone() + .into_service::() + .oneshot(Request::get(resource_path).body(Body::empty()).expect("resource request")) + .await + .expect("resource response"); + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT, "{resource_path}"); + assert_eq!( + response.headers().get("location").and_then(|value| value.to_str().ok()), + Some("http://8.8.8.8/live/101") + ); + + let response = router + .clone() + .into_service::() + .oneshot(Request::get("/playlist/resource/*").body(Body::empty()).expect("malformed request")) + .await + .expect("malformed response"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + let legacy = shared::utils::obfuscate_text(&app_state.get_encrypt_secret(), "http://10.0.0.1/icon.png"); + let response = router + .clone() + .into_service::() + .oneshot( + Request::get(format!("/playlist/resource/{legacy}")).body(Body::empty()).expect("legacy token request"), + ) + .await + .expect("legacy token response"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + for locator in [ + format!("{}99/live/101", crate::api::endpoints::api_playlist_utils::STALKER_RESOURCE_SCHEME), + format!("{}8/live/101", crate::api::endpoints::api_playlist_utils::STALKER_RESOURCE_SCHEME), + ] { + let resource = shared::utils::seal_web_ui_resource_url(&app_state.get_encrypt_secret(), &locator); + let response = router + .clone() + .into_service::() + .oneshot( + Request::get(format!("/playlist/resource/{resource}")).body(Body::empty()).expect("not-found request"), + ) + .await + .expect("not-found response"); + assert_eq!(response.status(), StatusCode::NOT_FOUND, "{locator}"); + } + + let resource = shared::utils::seal_web_ui_resource_url( + &app_state.get_encrypt_secret(), + &format!("{}7/live/102", crate::api::endpoints::api_playlist_utils::STALKER_RESOURCE_SCHEME), + ); + let response = router + .into_service::() + .oneshot( + Request::get(format!("/playlist/resource/{resource}")) + .body(Body::empty()) + .expect("private destination request"), + ) + .await + .expect("private destination response"); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + server_handle.abort(); +} + +/// The playback endpoints resolve a Stalker item from the catalog through the item's own +/// provider id. This is the path behind `Failed to resolve initial Stalker playback URL`, +/// so it needs the real id -> item -> `create_link` chain against a portal, not just the +/// Web UI preview route. +#[tokio::test] +async fn initial_stalker_playback_resolution_uses_the_requested_items_own_command() { + let temp_dir = tempdir().expect("temp dir"); + let (base_url, server_handle) = spawn_stalker_mock_server().await; + let input = Arc::new(ConfigInput { + id: 7, + name: "stalker-playback".intern(), + input_type: InputType::Stalker, + url: base_url, + enabled: true, + options: Some(ConfigInputOptions { + flags: crate::model::ConfigInputFlagsSet::new(), + update_quality: ConfigInputUpdateQuality::default(), + resolve_delay: shared::defaults::default_resolve_delay_secs(), + probe_delay: shared::defaults::default_probe_delay_secs(), + probe_live_interval_hours: 120, + resolve_filter: None, + probe_filter: None, + flussonic_hls_catchup: shared::model::FlussonicHlsCatchup::Native, + flussonic_hls_catchup_max_duration_secs: shared::model::default_flussonic_hls_catchup_max_duration_secs(), + }), + stalker: Some(crate::model::StalkerInputConfig { + device: None, + auth_mode: shared::model::StalkerAuthMode::Auto, + mag_preset: shared::model::StalkerMagPreset::GenericSafe, + endpoint_preference: shared::model::StalkerEndpointPreference::ServerLoad, + size_caps: None, + catalog_max_pages: None, + ..Default::default() + }), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(Arc::clone(&input), source); + app_config + .config + .store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Default::default() })); + let app_state = test_app_state(Arc::new(app_config)); + let router = + super::super::v1_api_playlist_register_protected(super::super::v1_api_playlist_register_public(Router::new())) + .with_state(Arc::clone(&app_state)); + + // The catalog import publishes the generation the playback path reads. + let response = router + .clone() + .into_service::() + .oneshot( + Request::builder() + .method("POST") + .uri("/playlist/live") + .header("content-type", "application/json") + .body(Body::from(r#"{"Input":"stalker-playback"}"#)) + .expect("request"), + ) + .await + .expect("catalog response"); + assert_eq!(response.status(), StatusCode::OK); + + let live_input = app_state.app_config.get_input_by_name(&"stalker-playback".intern()).expect("stalker input"); + let unresolved: Arc = "".intern(); + let live = shared::model::PlaylistItemType::Live; + let resolve = |provider_id: u32| { + let app_state = Arc::clone(&app_state); + let live_input = Arc::clone(&live_input); + let unresolved = Arc::clone(&unresolved); + async move { + crate::api::api_utils::resolve_initial_stalker_playback_url( + &app_state, + &live_input, + provider_id, + XtreamCluster::Live, + live, + &unresolved, + ) + .await + } + }; + + // An id that is not in the catalog must not resolve, and must not invalidate the + // published generation the next request reads. + let err = resolve(999_999).await.expect_err("an unknown provider id cannot be resolved"); + assert!(err.to_string().contains("999999"), "{err}"); + + // Each item resolves through its own stored cmd: the mock portal answers a different + // destination per cmd, so a provider id that picked the wrong item would surface here. + assert_eq!( + resolve(101).await.expect("item 101 resolves").as_ref(), + "http://8.8.8.8/live/101", + "the resolved url must come from item 101's own cmd" + ); + assert_eq!( + resolve(102).await.expect("item 102 resolves").as_ref(), + "http://127.0.0.1/live/102", + "the resolved url must come from item 102's own cmd" + ); + + // An item that already carries a url is served as is. + let resolved_url: Arc = "http://stream.example/already-resolved.ts".intern(); + let passthrough = crate::api::api_utils::resolve_initial_stalker_playback_url( + &app_state, + &live_input, + 101, + XtreamCluster::Live, + live, + &resolved_url, + ) + .await + .expect("a resolved url is passed through"); + assert!(Arc::ptr_eq(&passthrough, &resolved_url)); + + server_handle.abort(); +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_recording_routes.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_recording_routes.rs new file mode 100644 index 000000000..61ce79965 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/behavior_recording_routes.rs @@ -0,0 +1,220 @@ +use super::{test_app_config, test_app_state}; +use crate::{ + api::model::{recording::recording_source_resolution::resolve_recording_config, RecordingQueue}, + model::{ConfigInput, ConfigSource, ConfigTarget, RecordingConfig, SourcesConfig}, +}; +use arc_swap::ArcSwap; +use axum::{ + extract::{Path as AxumPath, Query, State}, + http::StatusCode, + response::IntoResponse, +}; +use shared::{ + foundation::Filter, + model::{provider_saturation::build_group_lookup, ProcessingOrder, XtreamCluster}, + utils::Internable, +}; +use std::{collections::HashMap, sync::Arc}; +use url::Url; + +#[tokio::test] +async fn stable_recording_route_rejects_target_and_input_from_different_sources() { + let app_config = Arc::new(test_app_config( + Arc::new(ConfigInput { id: 7, name: "input-a".intern(), ..Default::default() }), + ConfigSource { + inputs: vec!["input-a".intern()], + targets: vec![Arc::new(ConfigTarget { + curation: None, + id: 11, + enabled: true, + name: "stable-target".to_string(), + options: None, + sort: None, + filter: Filter::default().into(), + output: vec![], + rename: None, + mapping_ids: None, + mapping: Arc::default(), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + })], + }, + )); + let app_state = test_app_state(Arc::clone(&app_config)); + let token = + crate::auth::create_access_token(&app_config.access_token_secret, 1, crate::auth::scope::INTERNAL_PLAYER); + let fingerprint = crate::auth::Fingerprint::new( + "test".to_string(), + "127.0.0.1".to_string(), + "127.0.0.1:1234".parse().expect("test socket address"), + ); + + let response = super::super::playlist_recording_stream( + fingerprint, + AxumPath((token, "live".to_string(), 42)), + Query(super::super::RecordingStreamQuery { + target_name: "stable-target".to_string(), + input_name: "input-b".to_string(), + provider_allocation_id: None, + }), + State(app_state), + axum::http::HeaderMap::new(), + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); +} + +#[test] +fn recording_target_resolution_uses_stable_name_and_input_across_runtime_ids() { + let input_a = Arc::new(ConfigInput { id: 7, name: "input-a".intern(), ..Default::default() }); + let input_b = Arc::new(ConfigInput { id: 8, name: "input-b".intern(), ..Default::default() }); + let target = |id, name: &str| { + Arc::new(ConfigTarget { + curation: None, + id, + enabled: true, + name: name.to_string(), + options: None, + sort: None, + filter: Filter::default().into(), + output: vec![], + rename: None, + mapping_ids: None, + mapping: Arc::default(), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }) + }; + let sources = |later_target_id| { + let inputs = vec![Arc::clone(&input_a), Arc::clone(&input_b)]; + SourcesConfig { + batch_files: vec![], + provider: vec![], + group_lookup: build_group_lookup(&inputs), + inputs, + sources: vec![ + ConfigSource { inputs: vec!["input-a".intern()], targets: vec![target(11, "target-a")] }, + ConfigSource { + inputs: vec!["input-b".intern()], + targets: vec![target(later_target_id, "stable-target")], + }, + ], + templates: None, + } + }; + let mut app_config = test_app_config( + Arc::new(ConfigInput { id: 0, name: "unused".intern(), ..Default::default() }), + ConfigSource { inputs: vec![], targets: vec![] }, + ); + app_config.sources = Arc::new(ArcSwap::from_pointee(sources(12))); + + let snapshot = app_config.sources.load(); + let resolved = resolve_recording_config(snapshot.as_ref(), "stable-target", "input-b") + .expect("stable source in first snapshot"); + assert_eq!(resolved.target.id, 12); + assert_eq!(resolved.input.name.as_ref(), "input-b"); + assert!(resolve_recording_config(snapshot.as_ref(), "stable-target", "input-a").is_none()); + drop(snapshot); + + app_config.sources.store(Arc::new(sources(37))); + let snapshot = app_config.sources.load(); + let resolved = + resolve_recording_config(snapshot.as_ref(), "stable-target", "input-b").expect("stable source after reload"); + assert_eq!(resolved.target.id, 37); +} + +#[test] +fn recording_source_descriptor_is_token_free_and_percent_encodes_names() { + let url = crate::api::model::recording::recording_source_resolution::build_recording_source_descriptor( + "News/HD &+", + "input/name ?+", + 42, + XtreamCluster::Video, + ) + .expect("valid recording source descriptor"); + let parsed = Url::parse(&url).expect("parse recording source descriptor"); + let query = parsed.query_pairs().collect::>(); + + assert_eq!(parsed.scheme(), "tuliprox-recording"); + assert_eq!(parsed.host_str(), Some("source")); + assert_eq!(query.get("target_name").map(std::convert::AsRef::as_ref), Some("News/HD &+")); + assert_eq!(query.get("input_name").map(std::convert::AsRef::as_ref), Some("input/name ?+")); + assert_eq!(query.get("virtual_id").map(std::convert::AsRef::as_ref), Some("42")); + assert_eq!(query.get("cluster").map(std::convert::AsRef::as_ref), Some("movie")); + assert!(!url.contains("token")); +} + +#[test] +fn future_scheduled_recording_descriptor_round_trips_without_token() { + let url = crate::api::model::recording::recording_source_resolution::build_recording_source_descriptor( + "stable-target", + "input-a", + 42, + XtreamCluster::Live, + ) + .expect("valid recording url"); + let download_cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { + directory: Some("/tmp".to_string()), + ..Default::default() + }); + let metadata = shared::model::RecordingMetadata::new_live( + shared::model::RecordingOwner::User(shared::model::UserId::from("web:alice")), + shared::model::RecordingVisibility::Private, + shared::model::recording::RecordingSource::new("stable-target", "42", "input-a"), + 1_700_000_000, + 1_700_003_600, + 0, + 0, + ); + let recording = crate::api::model::RecordingTask::new( + shared::model::RecordingKind::Live, + &url, + "recording.ts", + &download_cfg, + Some("input-a".intern()), + 0, + metadata, + ) + .expect("valid recording task"); + let persisted = RecordingQueue::to_persisted(&recording); + let restored = RecordingQueue::from_persisted(persisted.clone()).expect("restore recording task"); + + assert_eq!(persisted.url, url); + assert_eq!(restored.url.as_str(), url); + assert!(persisted.url.starts_with("tuliprox-recording://source?")); + assert!(!persisted.url.contains("token")); + assert!(!persisted.url.contains("/11/")); + assert!(persisted.url.contains("target_name=stable-target")); +} + +#[test] +fn epg_channel_match_applies_the_target_output_case() { + let item = "NPO3.nl".intern(); + let requested_lowercase = + crate::utils::canonicalize_untrusted_epg_id("npo3.nl", crate::utils::EpgIdOutputCase::LowercaseAscii); + + assert!(super::super::epg_channel_id_matches( + Some(&item), + &requested_lowercase, + crate::utils::EpgIdOutputCase::LowercaseAscii + )); + assert!(!super::super::epg_channel_id_matches( + Some(&item), + &requested_lowercase, + crate::utils::EpgIdOutputCase::Preserve + )); +} + +#[test] +fn epg_channel_match_rejects_absent_ids() { + let requested = "npo3.nl".intern(); + assert!(!super::super::epg_channel_id_matches(None, &requested, crate::utils::EpgIdOutputCase::Preserve)); +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/configuration.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/configuration.rs new file mode 100644 index 000000000..a5536c3ef --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/configuration.rs @@ -0,0 +1,211 @@ +use super::{test_app_config, test_app_state}; +use crate::model::{Config, ConfigInput, ConfigSource}; +use axum::{ + body::Body, + http::{Request, StatusCode}, + Router, +}; +use shared::{ + model::{ + InputRefreshPolicy, InputType, PersistedPlaylistUpdateClusterSnapshot, PersistedPlaylistUpdateClusterState, + PlaylistUpdateDataSource, PlaylistUpdateStatusDto, XtreamCluster, + }, + utils::Internable, +}; +use std::sync::Arc; +use tempfile::tempdir; +use tower::ServiceExt; + +#[tokio::test] +async fn playlist_update_status_reloads_staged_own_completion_without_default_or_parent_inference() { + use crate::processing::input_cache::{ + load_input_status, resolve_input_storage_path, save_input_status, ClusterState, ClusterStatus, InputStatus, + }; + use shared::model::{ + PersistedPlaylistUpdateClusterStatusDto, PersistedPlaylistUpdateInputResult, PlaylistUpdateState, + }; + for own_state in [Some(PlaylistUpdateState::Success), Some(PlaylistUpdateState::Failure), None] { + let temp = tempdir().unwrap(); + let parent = Arc::new(ConfigInput { + id: 7, + name: "parent".intern(), + input_type: InputType::Xtream, + ..ConfigInput::default() + }); + let app_config = + test_app_config(parent.clone(), ConfigSource { inputs: vec![parent.name.clone()], targets: Vec::new() }); + app_config + .config + .store(Arc::new(Config { storage_dir: temp.path().to_string_lossy().into_owned(), ..Config::default() })); + let mut staged = ConfigInput { + id: 8, + name: "staged/provider own".intern(), + input_type: InputType::Staged, + staged_type: shared::model::StagedInputType::Xtream, + ..ConfigInput::default() + }; + staged.staged = Some(tuliprox_core::model::ConfigInputStaged { + for_input: Some(parent.name.clone()), + clusters: shared::model::ClusterFlags::Live, + }); + staged.resolve_staged_download_type(); + let mut sources = app_config.sources.load().as_ref().clone(); + sources.inputs.insert(0, Arc::new(staged.clone())); // Deliberately not ID or source order. + app_config.sources.store(Arc::new(sources)); + let path = resolve_input_storage_path(&temp.path().to_string_lossy(), &staged.name).await; + let snapshot = PersistedPlaylistUpdateClusterSnapshot { + policy: Some(InputRefreshPolicy::REFRESH), + source: Some(PlaylistUpdateDataSource::Cache), + quality_guard_threshold: Some(95), + ..PersistedPlaylistUpdateClusterSnapshot::default() + }; + let mut persisted = InputStatus::default(); + persisted.clusters.insert( + "live".into(), + ClusterStatus { status: ClusterState::Ok, timestamp: 123, last_update: Some(snapshot) }, + ); + persisted.last_input_update = + own_state.map(|state| PersistedPlaylistUpdateInputResult { state, timestamp: 456 }); + save_input_status(&path, &persisted); + let before = std::fs::read(path.join("status.json")).unwrap(); + let parent_path = resolve_input_storage_path(&temp.path().to_string_lossy(), &parent.name).await; + save_input_status( + &parent_path, + &InputStatus { + last_input_update: Some(PersistedPlaylistUpdateInputResult { + state: PlaylistUpdateState::Partial, + timestamp: 789, + }), + ..InputStatus::default() + }, + ); + let router = super::super::v1_api_playlist_register_protected(Router::new()) + .with_state(test_app_state(Arc::new(app_config))); + let response = router + .into_service::() + .oneshot(Request::builder().uri("/playlist/update/status").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let dto: PlaylistUpdateStatusDto = serde_json::from_slice(&body).unwrap(); + let own = dto.inputs.iter().find(|input| input.input_id == 8).unwrap(); + assert_eq!(own.last_input_update, persisted.last_input_update); + assert_eq!(own.last_update, Some(123)); + assert_eq!( + own.clusters, + vec![PersistedPlaylistUpdateClusterStatusDto { + cluster: XtreamCluster::Live, + status: PersistedPlaylistUpdateClusterState::Ok, + timestamp: 123, + last_update: Some(snapshot) + }] + ); + assert_eq!( + dto.inputs.iter().find(|input| input.input_id == 7).unwrap().last_input_update.unwrap().state, + PlaylistUpdateState::Partial + ); + assert!(!load_input_status(&path).clusters.contains_key("default")); + assert_eq!( + std::fs::read(path.join("status.json")).unwrap(), + before, + "read endpoint must not rewrite canonical status" + ); + } +} + +#[tokio::test] +async fn playlist_update_status_exposes_last_input_results_and_legacy_default_without_inventing_clusters() { + use crate::processing::input_cache::{ + load_input_status, resolve_input_storage_path, save_input_status, ClusterState, ClusterStatus, InputStatus, + }; + use shared::model::{PersistedPlaylistUpdateInputResult, PlaylistUpdateState}; + for last_state in [ + None, + Some(PlaylistUpdateState::Success), + Some(PlaylistUpdateState::Partial), + Some(PlaylistUpdateState::Failure), + ] { + let temp = tempdir().unwrap(); + let app_config = + test_app_config(Arc::new(ConfigInput::default()), ConfigSource { inputs: Vec::new(), targets: Vec::new() }); + app_config + .config + .store(Arc::new(Config { storage_dir: temp.path().to_string_lossy().into_owned(), ..Config::default() })); + let mut sources = (**app_config.sources.load()).clone(); + sources.inputs.clear(); + for (index, input_type) in [ + InputType::Xtream, + InputType::XtreamBatch, + InputType::Stalker, + InputType::StalkerBatch, + InputType::M3u, + InputType::M3uBatch, + InputType::Library, + InputType::Plex, + InputType::Emby, + InputType::Jellyfin, + InputType::Staged, + ] + .into_iter() + .enumerate() + { + let input = Arc::new(ConfigInput { + id: u16::try_from(index + 1).unwrap(), + name: format!("input-{index}").intern(), + input_type, + enabled: index % 2 == 0, + ..ConfigInput::default() + }); + let path = resolve_input_storage_path(&temp.path().to_string_lossy(), &input.name).await; + let mut persisted = InputStatus::default(); + persisted.clusters.insert( + "default".to_string(), + ClusterStatus { + status: if input.enabled { ClusterState::Ok } else { ClusterState::Failed }, + timestamp: 123, + last_update: None, + }, + ); + persisted.last_input_update = + last_state.map(|state| PersistedPlaylistUpdateInputResult { state, timestamp: 456 }); + save_input_status(&path, &persisted); + sources.inputs.push(input); + } + let inputs = sources.inputs.clone(); + app_config.sources.store(Arc::new(sources)); + let router = super::super::v1_api_playlist_register_protected(Router::new()) + .with_state(test_app_state(Arc::new(app_config))); + let response = router + .into_service::() + .oneshot(Request::builder().uri("/playlist/update/status").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let dto: PlaylistUpdateStatusDto = serde_json::from_slice(&body).unwrap(); + assert_eq!(dto.inputs.len(), inputs.len()); + for (input, status) in inputs.iter().zip(dto.inputs) { + assert_eq!(status.input_id, input.id); + assert_eq!(status.last_update, Some(123), "Last update retains the cache timestamp contract"); + let expected = if let Some(state) = last_state { + Some(PersistedPlaylistUpdateInputResult { state, timestamp: 456 }) + } else if input.input_type.is_xtream() || input.input_type.is_stalker() { + None // Do not reconstruct an aggregate input result from incomplete legacy clusters. + } else { + Some(PersistedPlaylistUpdateInputResult { + state: if input.enabled { PlaylistUpdateState::Success } else { PlaylistUpdateState::Failure }, + timestamp: 123, + }) + }; + assert_eq!(status.last_input_update, expected); + assert!(status.clusters.is_empty(), "no synthetic Live/VOD/Series from default"); + let path = resolve_input_storage_path(&temp.path().to_string_lossy(), &input.name).await; + assert_eq!( + load_input_status(&path).last_input_update.map(|result| result.state), + last_state, + "read endpoint must not persist a legacy projection" + ); + } + } +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/lifecycle.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/lifecycle.rs new file mode 100644 index 000000000..1bebc77b0 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/lifecycle.rs @@ -0,0 +1,84 @@ +use super::{playlist_update_target, test_app_config, test_app_state_with_manual_update_sender}; +use crate::model::{ConfigInput, ConfigSource}; +use axum::{extract::State, http::StatusCode, response::IntoResponse, Json}; +use shared::{ + model::{ + InputRefreshOverride, InputRefreshPolicy, InputType, PlaylistUpdateRequestDto, PlaylistUpdateRequestPayload, + PlaylistUpdateRunId, + }, + utils::Internable, +}; +use std::sync::Arc; +use tokio::sync::mpsc; + +pub(in crate::api::endpoints::v1_api_playlist::tests) fn manual_update_request( + input_id: u16, + policy: InputRefreshPolicy, +) -> crate::api::model::ManualPlaylistUpdateRequest { + crate::api::model::ManualPlaylistUpdateRequest { + run_id: PlaylistUpdateRunId::generate(), + targets: Arc::new(crate::model::ProcessTargets { + enabled: true, + inputs: vec![input_id], + targets: Vec::new(), + target_names: Vec::new(), + }), + input_action: Some(shared::model::InputUpdateRequest { + input_id, + action: shared::model::InputUpdateAction::Provider(policy), + }), + } +} + +#[tokio::test] +async fn manual_update_target_ids_keep_conflict_instead_of_accepting_a_dropped_force_request() { + for pending_policy in [InputRefreshPolicy::NORMAL, InputRefreshPolicy::REFRESH] { + let input = Arc::new(ConfigInput { + id: 17, + name: "force-input".intern(), + input_type: InputType::Xtream, + enabled: true, + ..ConfigInput::default() + }); + let app_config = Arc::new(test_app_config( + Arc::clone(&input), + ConfigSource { + inputs: vec![Arc::clone(&input.name)], + targets: vec![playlist_update_target(1, "queued-target")], + }, + )); + let (sender, mut receiver) = mpsc::channel(1); + sender.send(manual_update_request(1, InputRefreshPolicy::NORMAL)).await.unwrap(); + let running = receiver.recv().await.expect("running request"); + assert_eq!( + running.input_action.map(|request| request.action), + Some(shared::model::InputUpdateAction::Provider(InputRefreshPolicy::NORMAL)) + ); + sender.send(manual_update_request(2, pending_policy)).await.unwrap(); + let app_state = test_app_state_with_manual_update_sender(app_config, sender); + + let response = super::super::playlist_update( + State(app_state), + None, + Json(PlaylistUpdateRequestPayload::Current(PlaylistUpdateRequestDto { + targets: Vec::new(), + target_ids: Some(vec![1]), + input_refresh: Some(InputRefreshOverride { input_id: input.id, policy: InputRefreshPolicy::FORCE }), + input_action: None, + })), + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::CONFLICT); + let pending = receiver.recv().await.expect("pending request remains queued"); + assert_eq!( + pending.input_action, + Some(shared::model::InputUpdateRequest { + input_id: 2, + action: shared::model::InputUpdateAction::Provider(pending_policy) + }) + ); + assert!(receiver.try_recv().is_err()); + } +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/persistence.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/persistence.rs new file mode 100644 index 000000000..c53329ee0 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/persistence.rs @@ -0,0 +1,59 @@ +use super::{test_app_config, test_app_state}; +use crate::model::{Config, ConfigInput, ConfigSource}; +use axum::{extract::State, Json}; +use shared::{model::InputType, utils::Internable}; +use std::sync::Arc; +use tempfile::tempdir; + +#[tokio::test] +async fn playlist_update_status_reload_returns_active_bus_facts_without_persisting_updating() { + use shared::model::{EventMessage, PlaylistUpdateProgressEvent, PlaylistUpdateState, PlaylistUpdateSummary}; + let temp = tempdir().unwrap(); + let input = Arc::new(ConfigInput { + id: 7, + name: "reload-input".intern(), + input_type: InputType::M3u, + enabled: true, + ..ConfigInput::default() + }); + let config = test_app_config(input.clone(), ConfigSource { inputs: vec![input.name.clone()], targets: Vec::new() }); + config + .config + .store(Arc::new(Config { storage_dir: temp.path().to_string_lossy().into_owned(), ..Config::default() })); + let app = test_app_state(Arc::new(config)); + let path = + crate::processing::input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &input.name).await; + let persisted = crate::processing::input_cache::InputStatus { + last_input_update: Some(shared::model::PersistedPlaylistUpdateInputResult { + state: PlaylistUpdateState::Success, + timestamp: 1, + }), + ..crate::processing::input_cache::InputStatus::default() + }; + crate::processing::input_cache::save_input_status(&path, &persisted); + let before = std::fs::read(path.join("status.json")).unwrap(); + let mut progress = + PlaylistUpdateProgressEvent::for_run_input("running".into(), 17.into(), 7, "reload-input", "updating"); + app.event_manager.send_event(EventMessage::PlaylistUpdateProgress(progress.clone())); + let unknown = PlaylistUpdateProgressEvent::for_run_input("other".into(), 18.into(), 99, "removed", "updating"); + app.event_manager.send_event(EventMessage::PlaylistUpdateProgress(unknown)); + let Json(status) = super::super::playlist_update_status(State(app.clone())).await.unwrap(); + assert_eq!(status.active_updates, vec![progress.clone()]); + assert_eq!(status.inputs[0].last_input_update, persisted.last_input_update); + progress.state = Some(PlaylistUpdateState::Partial); + app.event_manager.send_event(EventMessage::PlaylistUpdateProgress(progress.clone())); + let Json(status) = super::super::playlist_update_status(State(app.clone())).await.unwrap(); + assert_eq!(status.active_updates, vec![progress], "completed input still belongs to a running target rebuild"); + app.event_manager.send_event(EventMessage::PlaylistUpdate(PlaylistUpdateSummary::for_run( + "running".into(), + 17.into(), + PlaylistUpdateState::Failure, + ))); + let Json(status) = super::super::playlist_update_status(State(app)).await.unwrap(); + assert!(status.active_updates.is_empty()); + assert_eq!( + std::fs::read(path.join("status.json")).unwrap(), + before, + "read-only runtime projection never writes transient status" + ); +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/protocol.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/protocol.rs new file mode 100644 index 000000000..81dc6fe41 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/protocol.rs @@ -0,0 +1,43 @@ +use crate::model::{Config, ConfigInput, RecordingConfig}; +use std::{collections::HashMap, sync::Arc}; + +#[tokio::test] +async fn recording_headers_layer_under_input_headers_without_touching_the_worker_request() { + let mut recording = RecordingConfig::from(&shared::model::RecordingConfigDto::default()); + recording.headers = HashMap::from([ + ("User-Agent".to_string(), "VLC/3.0.16 LibVLC/3.0.16".to_string()), + ("Accept".to_string(), "video/*".to_string()), + ("X-Recording-Test".to_string(), "custom".to_string()), + ("invalid header".to_string(), "ignored".to_string()), + ]); + let state = crate::api::model::create_test_app_state(Config { + video: Some(tuliprox_core::model::VideoConfig { + extensions: vec![], + web_search: None, + recording: Some(recording), + }), + ..Default::default() + }); + let header = |input: &ConfigInput, name: &str| { + input.headers.iter().find(|(key, _)| key.eq_ignore_ascii_case(name)).map(|(_, value)| value.clone()) + }; + + let plain = Arc::new(ConfigInput { name: "input".into(), ..Default::default() }); + let merged = crate::api::api_utils::with_recording_headers(&state.app_config, Arc::clone(&plain)); + assert_eq!(header(&merged, "user-agent").as_deref(), Some("VLC/3.0.16 LibVLC/3.0.16")); + assert_eq!(header(&merged, "accept").as_deref(), Some("video/*")); + assert_eq!(header(&merged, "x-recording-test").as_deref(), Some("custom")); + assert_eq!(header(&merged, "invalid header"), None); + assert_eq!(merged.name, plain.name); + assert!(plain.headers.is_empty(), "the configured input must stay untouched"); + + let with_agent = Arc::new(ConfigInput { + name: "input".into(), + headers: HashMap::from([("user-agent".to_string(), "input-agent".to_string())]), + ..Default::default() + }); + let merged = crate::api::api_utils::with_recording_headers(&state.app_config, with_agent); + assert_eq!(header(&merged, "user-agent").as_deref(), Some("input-agent")); + assert_eq!(merged.headers.keys().filter(|key| key.eq_ignore_ascii_case("user-agent")).count(), 1); + assert_eq!(header(&merged, "accept").as_deref(), Some("video/*")); +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/tests/support.rs b/backend/app/src/api/endpoints/v1_api_playlist/tests/support.rs new file mode 100644 index 000000000..448dec1f9 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/tests/support.rs @@ -0,0 +1,174 @@ +use crate::{ + api::model::{ + ActiveProviderManager, ActiveUserManager, AppState, ConnectionManager, EventManager, MetadataUpdateManager, + PlaylistStorageState, SharedStreamManager, + }, + model::{AppConfig, Config, ConfigInput, ConfigSource, ConfigTarget, SourcesConfig, StreamHistoryConfig}, + repository::GeoIp, +}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use chrono::Utc; +use shared::{ + foundation::Filter, + model::{ + provider_saturation::build_group_lookup, ConfigPaths, EpgSourceDto, EpgSourceTypeDto, IcsEpgSourceConfigDto, + ProcessingOrder, + }, +}; +use std::sync::Arc; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +pub(in crate::api::endpoints::v1_api_playlist::tests) fn playlist_update_target( + id: u16, + name: &str, +) -> Arc { + Arc::new(ConfigTarget { + curation: None, + id, + enabled: true, + name: name.to_string(), + options: None, + sort: None, + filter: Filter::default().into(), + output: vec![], + rename: None, + mapping_ids: None, + mapping: Arc::default(), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }) +} + +/// Generate an XMLTV datetime string in the format `YYYYMMDDHHmmss +0000` +/// offset by `hours_from_now` hours from the current time. +pub(in crate::api::endpoints::v1_api_playlist::tests) fn epg_dt(hours_from_now: i64) -> String { + let dt = Utc::now() + chrono::Duration::hours(hours_from_now); + dt.format("%Y%m%d%H%M%S %z").to_string() +} + +pub(in crate::api::endpoints::v1_api_playlist::tests) fn xmltv_source_dto(url: &str, priority: i16) -> EpgSourceDto { + EpgSourceDto { url: url.to_string(), priority, ..EpgSourceDto::default() } +} + +pub(in crate::api::endpoints::v1_api_playlist::tests) fn ics_source_dto( + url: &str, + channel_id: &str, + priority: i16, +) -> EpgSourceDto { + EpgSourceDto { + source_type: EpgSourceTypeDto::Ics, + url: url.to_string(), + priority, + channel_id: Some(channel_id.to_string()), + channel_title: Some("Formula 1".to_string()), + ics: Some(IcsEpgSourceConfigDto::default()), + ..EpgSourceDto::default() + } +} + +pub(in crate::api::endpoints::v1_api_playlist::tests) fn test_app_config( + input: Arc, + source: ConfigSource, +) -> AppConfig { + let inputs = vec![input]; + let sources = SourcesConfig { + batch_files: vec![], + provider: vec![], + group_lookup: build_group_lookup(&inputs), + inputs, + sources: vec![source], + templates: None, + }; + + AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(sources)), + hdhomerun: Arc::new(ArcSwapOption::empty()), + api_proxy: Arc::new(ArcSwapOption::empty()), + file_locks: Arc::new(crate::utils::FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::empty()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(crate::model::MediaToolCapabilities::default()), + } +} + +pub(in crate::api::endpoints::v1_api_playlist::tests) fn test_app_state(app_cfg: Arc) -> Arc { + let (manual_update_sender, _) = mpsc::channel::(1); + test_app_state_with_manual_update_sender(app_cfg, manual_update_sender) +} + +pub(in crate::api::endpoints::v1_api_playlist::tests) fn test_app_state_with_manual_update_sender( + app_cfg: Arc, + manual_update_sender: mpsc::Sender, +) -> Arc { + let event_manager = Arc::new(EventManager::new()); + let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); + let history_config = Some(StreamHistoryConfig::default()); + active_provider.set_shared_stream_manager(&shared_stream_manager); + + let geoip = Arc::new(ArcSwapOption::::default()); + let config = app_cfg.config.load(); + let active_users = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); + let connection_manager = Arc::new(ConnectionManager::new( + &active_users, + &active_provider, + &shared_stream_manager, + &event_manager, + history_config.as_ref(), + )); + + let tokens = crate::api::model::CancelTokens { + scheduler: CancellationToken::new(), + hdhomerun: CancellationToken::new(), + file_watch: CancellationToken::new(), + provider_dns: CancellationToken::new(), + metadata: CancellationToken::new(), + qos_aggregation: CancellationToken::new(), + recordings: CancellationToken::new(), + hls_cache: CancellationToken::new(), + }; + let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); + + Arc::new(AppState { + recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( + Arc::clone(&active_provider), + Arc::clone(&connection_manager), + ), + app_config: app_cfg, + http_clients: Arc::default(), + recordings: Arc::new(crate::api::model::RecordingQueue::new()), + cache: Arc::new(ArcSwapOption::default()), + shared_stream_manager, + hls: crate::api::model::HlsState::new(Arc::new(crate::api::model::HlsProxyManager::new())), + stalker_resolve_coordinator: crate::api::model::StalkerResolveCoordinator::default(), + active_users, + active_provider, + connection_manager, + event_manager, + cancel_tokens: ArcSwap::from_pointee(tokens), + playlists: Arc::new(PlaylistStorageState::new()), + geoip, + metadata_manager, + auth: crate::api::model::AuthState::for_tests(), + playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests_with_sender(manual_update_sender), + }) +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/update.rs b/backend/app/src/api/endpoints/v1_api_playlist/update.rs new file mode 100644 index 000000000..5c2611cca --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/update.rs @@ -0,0 +1,250 @@ +use super::{ManualUpdateEnqueueError, PLAYLIST_UPDATE_STATUS_READ_CONCURRENCY}; +use crate::{ + api::{ + auth_middleware::{check_permission, VerifiedClaims}, + model::AppState, + }, + model::{ProcessTargets, SourcesConfig}, +}; +use axum::response::IntoResponse; +use log::{debug, error}; +use serde_json::json; +use shared::{ + error::TuliproxError, + model::{ + permission::Permission, InputPlaylistUpdateStatusDto, InputType, InputUpdateAction, InputUpdateCapabilities, + InputUpdateRequest, OperationRunAccepted, PersistedPlaylistUpdateClusterState, + PersistedPlaylistUpdateClusterStatusDto, PersistedPlaylistUpdateInputResult, PlaylistUpdateRequestDto, + PlaylistUpdateRequestPayload, PlaylistUpdateRunId, PlaylistUpdateState, PlaylistUpdateStatusDto, XtreamCluster, + }, + utils::sanitize_sensitive_info, +}; +use std::sync::Arc; + +fn enqueue_manual_playlist_update( + sender: &tokio::sync::mpsc::Sender, + targets: Arc, + input_action: Option, +) -> Result { + let permit = sender.try_reserve().map_err(|error| match error { + tokio::sync::mpsc::error::TrySendError::Full(()) => ManualUpdateEnqueueError::Busy, + tokio::sync::mpsc::error::TrySendError::Closed(()) => ManualUpdateEnqueueError::Unavailable, + })?; + let run_id = PlaylistUpdateRunId::generate(); + permit.send(crate::api::model::ManualPlaylistUpdateRequest { run_id: run_id.clone(), targets, input_action }); + Ok(run_id) +} + +pub(super) async fn playlist_update( + axum::extract::State(app_state): axum::extract::State>, + claims: Option>, + axum::extract::Json(payload): axum::extract::Json, +) -> impl axum::response::IntoResponse + Send { + let request = payload.into_request(); + let input_action = match request.manual_input_update() { + Ok(action) => action, + Err(error) => { + return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": error}))).into_response() + } + }; + if input_action.is_some_and(|request| request.action == InputUpdateAction::Rescan) { + let config = app_state.app_config.config.load(); + if !config.library.as_ref().is_some_and(|library| library.enabled) { + return (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": "Library is not enabled"}))) + .into_response(); + } + if config.web_ui.as_ref().and_then(|web| web.auth.as_ref()).is_some() { + let Some(axum::Extension(VerifiedClaims(claims))) = claims else { + return axum::http::StatusCode::UNAUTHORIZED.into_response(); + }; + if check_permission::<{ Permission::LibraryWrite as u32 }>(&app_state, &claims, None).is_err() { + return axum::http::StatusCode::FORBIDDEN.into_response(); + } + } + } + let process_targets = resolve_manual_playlist_update_targets(&app_state.app_config.sources.load(), &request); + match process_targets { + Ok(valid_targets) => { + let valid_targets = Arc::new(valid_targets); + match enqueue_manual_playlist_update( + &app_state.playlist_updates.manual_update_sender, + valid_targets, + input_action, + ) { + Ok(run_id) => { + (axum::http::StatusCode::ACCEPTED, axum::Json(OperationRunAccepted::playlist_update(run_id))) + .into_response() + } + Err(ManualUpdateEnqueueError::Busy) => { + debug!("Manual playlist update rejected: another update is already queued"); + ( + axum::http::StatusCode::CONFLICT, + axum::Json(json!({"error": "Another playlist update is already queued; retry this request"})), + ) + .into_response() + } + Err(ManualUpdateEnqueueError::Unavailable) => { + debug!("Manual playlist update rejected: worker channel closed (server shutting down)"); + axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response() + } + } + } + Err(err) => { + error!("Failed playlist update {}", sanitize_sensitive_info(&err.to_string())); + (axum::http::StatusCode::BAD_REQUEST, axum::Json(json!({"error": err.to_string()}))).into_response() + } + } +} + +pub(super) fn resolve_manual_playlist_update_targets( + sources: &SourcesConfig, + request: &PlaylistUpdateRequestDto, +) -> Result { + let manual_action = + request.manual_input_update().map_err(|error| TuliproxError::ConfigSource(error.to_string()))?; + let manual_input = if let Some(manual) = manual_action { + let input = sources + .inputs + .iter() + .find(|input| input.id == manual.input_id) + .ok_or_else(|| TuliproxError::ConfigSource(format!("No input found for id {}", manual.input_id)))?; + if !InputUpdateCapabilities::for_input_type(input.input_type, input.enabled).supports_action(manual.action) { + return Err(TuliproxError::ConfigSource("Manual action is not supported for this input".to_string())); + } + // Only the additive action wire form requires IDs. Legacy input_refresh + // requests retain the existing name resolution, including empty = all. + if request.input_action.is_some() && request.target_ids.is_none() { + return Err(TuliproxError::ConfigSource("Manual input actions require explicit target IDs".to_string())); + } + Some(input) + } else { + None + }; + let Some(target_ids) = request.target_ids.as_deref() else { + let targets = (!request.targets.is_empty()).then_some(&request.targets); + return sources.validate_targets(targets); + }; + if !request.targets.is_empty() { + return Err(TuliproxError::ConfigSource( + "Manual playlist update cannot contain both target names and target IDs".to_string(), + )); + } + if target_ids.is_empty() { + return Err(TuliproxError::ConfigSource( + "Manual playlist update target ID selection must not be empty".to_string(), + )); + } + + let mut targets = Vec::with_capacity(target_ids.len()); + let mut target_names = Vec::with_capacity(target_ids.len()); + for target_id in target_ids { + let Some(target) = sources.get_target_by_id(*target_id) else { + return Err(TuliproxError::ConfigSource(format!("No target found for id {target_id}"))); + }; + if manual_input.is_some_and(|input| { + !sources.sources.iter().any(|source| { + source.inputs.contains(&input.name) && source.targets.iter().any(|target| target.id == *target_id) + }) + }) { + return Err(TuliproxError::ConfigSource("Target does not belong to the selected input".to_string())); + } + targets.push(target.id); + target_names.push(target.name.clone()); + } + + Ok(ProcessTargets { + enabled: true, + inputs: sources.inputs.iter().map(|input| input.id).collect(), + targets, + target_names, + }) +} + +pub(super) async fn read_playlist_update_input_status( + read: impl FnOnce() -> crate::processing::input_cache::InputStatus + Send + 'static, +) -> Result { + tokio::task::spawn_blocking(read).await.map_err(|error| { + error!("Playlist update status reader failed: {error}"); + axum::http::StatusCode::INTERNAL_SERVER_ERROR + }) +} + +pub(super) async fn playlist_update_status( + axum::extract::State(app_state): axum::extract::State>, +) -> Result, axum::http::StatusCode> { + let storage_dir = app_state.app_config.config.load().storage_dir.clone(); + let inputs = app_state.app_config.sources.load().inputs.clone(); + let status_reads = futures::stream::iter(inputs.into_iter().map(|input| { + let storage_dir = storage_dir.clone(); + let input_name = input.name.clone(); + async move { + let storage_path = + crate::processing::input_cache::resolve_input_storage_path(&storage_dir, &input_name).await; + let input_status = read_playlist_update_input_status(move || { + crate::processing::input_cache::load_input_status(&storage_path) + }) + .await?; + Ok::<_, axum::http::StatusCode>((input, input_status)) + } + })); + let status_reads = { + use futures::StreamExt as _; + status_reads.buffered(PLAYLIST_UPDATE_STATUS_READ_CONCURRENCY) + }; + let input_statuses: Vec<_> = futures::StreamExt::collect(status_reads).await; + let input_statuses = input_statuses.into_iter().collect::, _>>()?; + let mut statuses = Vec::with_capacity(input_statuses.len()); + for (input, input_status) in input_statuses { + let cluster_based = input.input_type.is_xtream() || input.input_type.is_stalker(); + let last_update = + input_status.clusters.values().map(|cluster| cluster.timestamp).filter(|timestamp| *timestamp > 0).max(); + let clusters = if cluster_based || input.input_type == InputType::M3u { + [XtreamCluster::Live, XtreamCluster::Video, XtreamCluster::Series] + .into_iter() + .filter_map(|cluster| { + input_status.clusters.get(cluster.as_ref()).map(|persisted| { + PersistedPlaylistUpdateClusterStatusDto { + cluster, + status: match &persisted.status { + crate::processing::input_cache::ClusterState::Ok => { + PersistedPlaylistUpdateClusterState::Ok + } + crate::processing::input_cache::ClusterState::Failed => { + PersistedPlaylistUpdateClusterState::Failed + } + }, + timestamp: persisted.timestamp, + last_update: persisted.last_update, + } + }) + }) + .collect() + } else { + Vec::new() + }; + let last_input_update = input_status.last_input_update.or_else(|| { + // Legacy cluster-free inputs already persisted a general default status. + // Do not infer an aggregate input/target or Quality result from clusters. + if cluster_based { + return None; + } + input_status.clusters.get("default").filter(|status| status.timestamp > 0).map(|status| { + PersistedPlaylistUpdateInputResult { + state: match status.status { + crate::processing::input_cache::ClusterState::Ok => PlaylistUpdateState::Success, + crate::processing::input_cache::ClusterState::Failed => PlaylistUpdateState::Failure, + }, + timestamp: status.timestamp, + } + }) + }); + statuses.push(InputPlaylistUpdateStatusDto { input_id: input.id, last_update, last_input_update, clusters }); + } + let active_updates = app_state + .event_manager + .playlist_update_snapshot() + .into_iter() + .filter(|event| event.input_id.is_some_and(|id| statuses.iter().any(|status| status.input_id == id))) + .collect(); + Ok(axum::Json(PlaylistUpdateStatusDto { inputs: statuses, active_updates })) +} diff --git a/backend/app/src/api/endpoints/v1_api_playlist/webplayer.rs b/backend/app/src/api/endpoints/v1_api_playlist/webplayer.rs new file mode 100644 index 000000000..ecc0045e5 --- /dev/null +++ b/backend/app/src/api/endpoints/v1_api_playlist/webplayer.rs @@ -0,0 +1,93 @@ +use super::build_webplayer_recording_url; +use crate::{ + api::{ + api_utils::{create_api_proxy_user, try_option_bad_request, try_result_bad_request}, + endpoints::{ + m3u_api::m3u_api_stream_loaded, + xtream_api::{xtream_player_api_stream_with_token, ApiStreamContext, ApiStreamRequest}, + }, + model::AppState, + }, + auth::verify_access_token, + repository::m3u_get_item_for_stream_id, +}; +use axum::response::IntoResponse; +use log::{debug, error}; +use shared::model::{TargetType, XtreamCluster}; +use std::{str::FromStr, sync::Arc}; + +pub(super) fn build_playlist_webplayer_url( + base_url: &str, + access_token: &str, + target_id: u16, + virtual_id: u32, + cluster: XtreamCluster, +) -> String { + format!( + "{base_url}/api/v1/playlist/webplayer/{access_token}/{}/{}/{}", + target_id, + cluster.as_stream_type(), + virtual_id + ) +} + +pub(super) fn playlist_webplayer( + axum::extract::State(app_state): axum::extract::State>, + target_id: u16, + virtual_id: u32, + cluster: XtreamCluster, +) -> impl axum::response::IntoResponse + Send { + let Some(url) = build_webplayer_recording_url(&app_state.app_config, target_id, virtual_id, cluster) else { + return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; + url.into_response() +} + +pub(super) async fn playlist_webplayer_stream( + fingerprint: crate::auth::Fingerprint, + axum::extract::Path((token, target_id, cluster, stream_id)): axum::extract::Path<(String, u16, String, String)>, + axum::extract::State(app_state): axum::extract::State>, + req_headers: axum::http::HeaderMap, +) -> impl IntoResponse + Send { + if !verify_access_token(&token, &app_state.app_config.access_token_secret, crate::auth::scope::INTERNAL_PLAYER) { + return axum::http::StatusCode::FORBIDDEN.into_response(); + } + + let ctxt = try_result_bad_request!(ApiStreamContext::from_str(cluster.as_str())); + let Some(target) = app_state.app_config.get_target_by_id(target_id) else { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + }; + + if target.has_output(TargetType::Xtream) { + return xtream_player_api_stream_with_token( + &fingerprint, + &req_headers, + &app_state, + target_id, + ApiStreamRequest::from_access_token(ctxt, &token, &stream_id, ""), + ) + .await + .into_response(); + } + + if !target.has_output(TargetType::M3u) { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + } + + let req_virtual_id: u32 = try_result_bad_request!(stream_id.trim().parse()); + let pli = try_result_bad_request!( + m3u_get_item_for_stream_id(req_virtual_id, &app_state.app_config, &app_state.playlists, &target).await, + true, + format!("Failed to read m3u item for stream id {req_virtual_id}") + ); + let input = try_option_bad_request!( + app_state.app_config.get_input_by_name(&pli.input_name), + true, + format!("Can't find input {} for target {}", pli.input_name, target.name) + ); + let user = Arc::new(create_api_proxy_user(&app_state)); + + m3u_api_stream_loaded(user, target, &fingerprint, &req_headers, &app_state, pli, input, None, None, None) + .await + .into_response() +} diff --git a/backend/app/src/api/main_api.rs b/backend/app/src/api/main_api.rs index 663cc501d..0a4fe4ca6 100644 --- a/backend/app/src/api/main_api.rs +++ b/backend/app/src/api/main_api.rs @@ -1169,7 +1169,7 @@ mod tests { (status, body["status"].as_str().expect("status label").to_string()) } - async fn acquire(state: &Arc, input: &'static str, addr: &'static str) -> ProviderHandle { + fn acquire(state: &Arc, input: &'static str, addr: &'static str) -> ProviderHandle { let addr: SocketAddr = addr.parse().expect("socket addr"); state .active_provider @@ -1196,7 +1196,7 @@ mod tests { #[tokio::test] async fn ready_is_ready_while_capacity_remains() { let state = test_app_state(test_app_config(vec![test_input("prov", 2, true)])); - let allocation = acquire(&state, "prov", "127.0.0.1:45101").await; + let allocation = acquire(&state, "prov", "127.0.0.1:45101"); let (status, label) = call_ready(state).await; assert_eq!(status, axum::http::StatusCode::OK); assert_eq!(label, "ready"); @@ -1206,7 +1206,7 @@ mod tests { #[tokio::test] async fn ready_is_exhausted_when_all_groups_are_full() { let state = test_app_state(test_app_config(vec![test_input("prov", 1, true)])); - let allocation = acquire(&state, "prov", "127.0.0.1:45102").await; + let allocation = acquire(&state, "prov", "127.0.0.1:45102"); let (status, label) = call_ready(state).await; assert_eq!(status, axum::http::StatusCode::SERVICE_UNAVAILABLE); assert_eq!(label, "exhausted"); diff --git a/backend/app/src/api/model/streams/active_client_stream.rs b/backend/app/src/api/model/streams/active_client_stream.rs index 1da0865f8..b75023b18 100644 --- a/backend/app/src/api/model/streams/active_client_stream.rs +++ b/backend/app/src/api/model/streams/active_client_stream.rs @@ -1,131 +1,16 @@ use crate::{ - api::{ - model::{ - connection_manager::{ - PROVIDER_END_CLOSED, PROVIDER_END_ERROR, PROVIDER_END_NOT_SET, PROVIDER_END_PREEMPTED, - }, - open_provider_stream_with_lifecycle, uses_direct_body_idle_timeout, AppState, BoxedProviderStream, - CleanupEvent, ConnectionManager, CustomVideoStreamType, EventManager, MeteringStream, - PendingProviderWakeSource, ProviderStreamFactoryOptions, ProviderStreamOpen, ProviderStreamOpenLifecycle, - StreamDetails, StreamError, StreamMeterHandle, TimedClientStream, TransportStreamBuffer, - DIRECT_BODY_IDLE_TIMEOUT_SECS, - }, - panel_api::{can_provision_on_exhausted, find_input_by_provider_name, run_panel_api_provisioning_probe}, - }, + api::model::{AppState, StreamDetails}, auth::Fingerprint, - model::{ConfigInput, ProxyUserCredentials}, - utils::debug_if_enabled, -}; -use axum::http::{header::USER_AGENT, HeaderMap}; -use bytes::Bytes; -use futures::{task::AtomicWaker, Future, Stream, StreamExt}; -use log::{error, info, warn}; -use shared::{ - model::{FailureStage, StreamChannel, UserConnectionPermission, VirtualId}, - utils::{sanitize_sensitive_info, Internable}, -}; -use std::{ - net::SocketAddr, - pin::Pin, - sync::{ - atomic::{AtomicBool, AtomicU8, Ordering}, - Arc, - }, - task::{Context, Poll}, -}; -use tokio::sync::Notify; -use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; -use tuliprox_session::{ - stream_options::{get_stream_options, StreamResponseMode}, - ConnectionRejectionReason, + model::ProxyUserCredentials, }; +use axum::http::HeaderMap; +use shared::model::{StreamChannel, UserConnectionPermission}; +use std::sync::Arc; const BODY_IDLE_TIMEOUT_ERROR_CLASS: &str = "body_idle_timeout"; + const DIRECT_BODY_SOCKET_ACTIVITY_TOUCH_SECS: u64 = 1; -/// Discriminates which byte-stream the client is consuming at any moment. -/// Stored as `u8` in an `AtomicU8` for lock-free access inside `poll_next`. -/// Lower numeric values correspond to a live or custom stream; `GracePending` -/// (255) is a transient sentinel that parks the poll until the grace task resolves. -/// -/// Discriminants are implicit (declaration order) except for the `255` sentinel. -/// The byte mapping lives once, in [`StreamMode::try_from`], and is pinned by -/// `test_stream_mode_byte_values`. -#[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum StreamMode { - /// Forward bytes directly from the upstream provider. - Inner, - /// Show the "user connections exhausted" custom video. - UserExhausted, - /// Show the "provider connections exhausted" custom video. - ProviderExhausted, - /// Show the "channel unavailable" custom video. - ChannelUnavailable, - /// Show the provisioning/placeholder custom video while probing for capacity. - Provisioning, - /// Show the "low-priority preempted" custom video. - LowPriorityPreempted, - /// A recently evicted playback retried while this stream held a grace slot and - /// every remaining eviction candidate is reentry-protected. The body must end - /// without painting a user-visible "connections exhausted" error video. - ReentrySuppressed, - /// Transient: grace-period check is still in progress; `poll_next` must park. - GracePending = 255, -} - -impl TryFrom for StreamMode { - type Error = u8; - - /// Decodes the atomic mode flag. Returns `Err(value)` for a byte that was never a - /// valid [`StreamMode`]; callers must fail safe instead of inventing a live state - /// such as `GracePending`. - fn try_from(value: u8) -> Result { - match value { - 0 => Ok(Self::Inner), - 1 => Ok(Self::UserExhausted), - 2 => Ok(Self::ProviderExhausted), - 3 => Ok(Self::ChannelUnavailable), - 4 => Ok(Self::Provisioning), - 5 => Ok(Self::LowPriorityPreempted), - 6 => Ok(Self::ReentrySuppressed), - 255 => Ok(Self::GracePending), - other => Err(other), - } - } -} - -/// Publishes a new mode to the lock-free flag with release ordering. All writers go -/// through this so the `as u8` cast and the memory ordering live in one place. -fn store_stream_mode(flag: &AtomicU8, mode: StreamMode) { flag.store(mode as u8, Ordering::Release); } - -/// Holds the optional custom video buffers for each error/placeholder scenario. -/// Using named fields avoids the positional-indexing confusion of a 4-tuple. -struct CustomVideoBuffers { - user_exhausted: Option, - provider_exhausted: Option, - unavailable: Option, - provisioning: Option, - low_priority_preempted: Option, -} - -struct GraceProvisioningInfo { - input: Arc, - stop_signal: CancellationToken, -} - -#[derive(Clone)] -struct TimedStreamContext { - app_state: Arc, - duration_secs: u32, - virtual_id: VirtualId, -} - -struct DeferredProviderOpenContext { - app_state: Arc, - provider_stream_factory_options: ProviderStreamFactoryOptions, -} - pub(crate) struct ActiveClientStreamParams<'a> { pub stream_details: StreamDetails, pub app_state: &'a Arc, @@ -141,3787 +26,28 @@ pub(crate) struct ActiveClientStreamParams<'a> { pub meter_stream: bool, } -struct GracePeriodParams { - app_state: Arc, - stream_details: StreamDetails, - user_grace_period: bool, - user: ProxyUserCredentials, - fingerprint: Fingerprint, - virtual_id: VirtualId, - session_token: Option, - provisioning_info: Option, - waker: Option>, - hold_stream: bool, - capacity_notify: Arc, - pending_provider_version: Option, - // The `transition_version` of the session if it is in `GraceActive` lifecycle. - // Used by the grace task to confirm the session is still in `GraceActive` before resolving. - grace_active_version: Option, - // Set when the stream was admitted via a user-grace strategy. - // On user-grace failure, remaining strategies are evaluated before final deny. - grace_resolution_context: Option, - // The `ConnectionKind` from the original admission decision. - // Preserved in `GraceResolutionContext.kind` and also passed directly here - // so the grace task can use the runtime value when evaluating remaining strategies. - grace_kind: Option, - // Whether the session is `socket_bound`. Used to construct the correct - // `EvictionReentryGuard`. - socket_bound: bool, - shared_subscriber_id: Option, -} - -enum DeferredProviderOpenOutcome { - Stream(BoxedProviderStream), - Mode(StreamMode), - Failed, -} - -enum DeferredProviderOpenState { - Pending(Box), - Opening(Pin + Send>>), -} - -struct DirectBodyIdleTimeout { - enabled: bool, - deadline: Option, - sleep: Option>>, - last_socket_activity_touch: Option, -} - -impl DirectBodyIdleTimeout { - const fn disabled() -> Self { - Self { enabled: false, deadline: None, sleep: None, last_socket_activity_touch: None } - } - - const fn enabled() -> Self { Self { enabled: true, deadline: None, sleep: None, last_socket_activity_touch: None } } - - fn mark_progress(&mut self) -> bool { - if !self.enabled { - return false; - } - - let now = tokio::time::Instant::now(); - self.deadline = Some(now + tokio::time::Duration::from_secs(DIRECT_BODY_IDLE_TIMEOUT_SECS)); - self.sleep = None; - - let should_touch_socket = self.last_socket_activity_touch.is_none_or(|last_touch| { - now.duration_since(last_touch) >= tokio::time::Duration::from_secs(DIRECT_BODY_SOCKET_ACTIVITY_TOUCH_SECS) - }); - if should_touch_socket { - self.last_socket_activity_touch = Some(now); - } - should_touch_socket - } - - fn poll_expired(&mut self, cx: &mut Context<'_>) -> bool { - if !self.enabled { - return false; - } - - let deadline = *self.deadline.get_or_insert_with(|| { - tokio::time::Instant::now() + tokio::time::Duration::from_secs(DIRECT_BODY_IDLE_TIMEOUT_SECS) - }); - if tokio::time::Instant::now() >= deadline { - return true; - } - - self.sleep.get_or_insert_with(|| Box::pin(tokio::time::sleep_until(deadline))); - self.sleep.as_mut().is_some_and(|sleep| sleep.as_mut().poll(cx).is_ready()) - } -} - -#[allow(clippy::struct_excessive_bools)] -struct ActiveClientStreamState { - response_mode: StreamResponseMode, - inner: Option, - send_custom_stream_flag: Option>, - provider_handle: Option, - deferred_provider_open: Option, - timed_stream_context: Option, - preempt_cancelled: Option>>, - grace_task_handle: Option>, - /// Cancels a panel-api provisioning probe spawned by the grace task; aborting the - /// grace task alone would leave the probe running to its own timeout. - provisioning_stop_signal: Option, - provisionable: bool, - custom_video: CustomVideoBuffers, - meter: Option>, - event_manager: Arc, - waker: Option>, - connection_manager: Arc, - fingerprint: Arc, - stream_uid: Option, - provider_stopped: bool, - user_stream_released: bool, - /// Mirrors `user_stream_released` for the provider handle to guard against double-release - /// when preemption and Drop race. - provider_handle_released: bool, - custom_video_timeout_secs: u32, - custom_video_timeout_mode: Option, - custom_video_timeout_sleep: Option>>, - direct_body_idle_timeout: DirectBodyIdleTimeout, - /// Set once when the provider stream ends. Read once in Drop. Never queried during streaming. - /// Separate from `send_custom_stream_flag` (`StreamMode`). Uses `PROVIDER_END_*` constants. - provider_end_reason: AtomicU8, - provider_error_class: Option<&'static str>, - provider_http_status: Option, - /// Count of successful provider reconnections during this session (grace period / deferred open). - provider_reconnect_count: AtomicU8, - /// Playback lease owner (session token) whose provider slot is confirmed once real - /// media bytes reach the client. `None` when this stream has no provider lease. - lease_owner: Option>, - media_started: Option>, - /// Guards against emitting the confirmation more than once per stream. - lease_confirmed: bool, - lease_request_id: Option, - /// Guaranteed final cleanup for the registered request claim. The body owns this and - /// finishes it exactly once with the real provider outcome, so a full cleanup queue can - /// never leak the user request or provider lease. - request_cleanup: Option, -} - -impl ActiveClientStreamState { - fn mode_for_custom_video_type(video_type: CustomVideoStreamType) -> Option { - match video_type { - CustomVideoStreamType::ChannelUnavailable => Some(StreamMode::ChannelUnavailable), - CustomVideoStreamType::UserConnectionsExhausted => Some(StreamMode::UserExhausted), - CustomVideoStreamType::ProviderConnectionsExhausted => Some(StreamMode::ProviderExhausted), - CustomVideoStreamType::LowPriorityPreempted => Some(StreamMode::LowPriorityPreempted), - CustomVideoStreamType::Provisioning => Some(StreamMode::Provisioning), - CustomVideoStreamType::UserAccountExpired | CustomVideoStreamType::HlsSessionOrLeaseExpired => None, - } - } - - fn wrap_provider_stream(&self, stream: BoxedProviderStream) -> BoxedProviderStream { - let stream = if let Some(meter) = &self.meter { - MeteringStream::new(stream, Arc::clone(meter), Arc::clone(&self.event_manager)).boxed() - } else { - stream - }; - if let Some(ctx) = self.timed_stream_context.as_ref() { - TimedClientStream::new( - &ctx.app_state.app_config, - &ctx.app_state.connection_manager, - stream, - ctx.duration_secs, - self.fingerprint.addr, - ctx.virtual_id, - self.stream_uid, - ) - .boxed() - } else { - stream - } - } - - /// Maps a mode to the custom video it serves, if any. `ReentrySuppressed` and the - /// transparent forwarding modes deliberately have no custom video: a suppressed - /// reentry must never surface a user-visible error clip. - fn custom_video_type_for_mode(mode: StreamMode) -> Option { - match mode { - StreamMode::UserExhausted => Some(CustomVideoStreamType::UserConnectionsExhausted), - StreamMode::ProviderExhausted => Some(CustomVideoStreamType::ProviderConnectionsExhausted), - StreamMode::Provisioning => Some(CustomVideoStreamType::Provisioning), - StreamMode::LowPriorityPreempted => Some(CustomVideoStreamType::LowPriorityPreempted), - StreamMode::ChannelUnavailable => Some(CustomVideoStreamType::ChannelUnavailable), - StreamMode::Inner | StreamMode::GracePending | StreamMode::ReentrySuppressed => None, - } - } - - /// Drops the provider handle at most once (releasing the provider slot - /// synchronously) and detaches the provider body stream. The stream claim itself - /// is released separately through [`Self::release_user_stream`]. - fn release_provider_handle_and_detach_body(&mut self) { - self.inner = None; - if !self.provider_handle_released { - self.provider_handle_released = true; - drop(self.provider_handle.take()); - } - } - - fn release_user_stream(&mut self) { - if self.user_stream_released { - return; - } - self.user_stream_released = true; - if let Some(cleanup) = self.request_cleanup.as_mut() { - cleanup.finish( - self.lease_request_id, - self.provider_end_reason.load(Ordering::Relaxed), - self.provider_reconnect_count.load(Ordering::Relaxed), - self.provider_error_class, - self.provider_http_status, - ); - } - } - - /// Confirms the playback lease exactly once, when the first real provider media - /// bytes reach the client. Only a confirmed lease may reserve provider capacity - /// against other playbacks, so an abandoned or custom-video-only start does not - /// block unrelated clients behind the same reverse proxy. - fn confirm_lease_once(&mut self) { - if self.lease_confirmed { - return; - } - let Some(owner) = self.lease_owner.take() else { - return; - }; - self.lease_confirmed = true; - if let Some(media_started) = &self.media_started { - media_started.store(true, Ordering::Release); - } - let request_id = self.lease_request_id.or_else(|| { - self.provider_handle.as_ref().and_then(|managed| managed.handle()).and_then(|h| h.playback_request_id) - }); - // Confirm synchronously through the broker: a first-byte confirmation is a - // mandatory capacity transition and must not be dropped by queue pressure. - let provider_manager = &self.connection_manager.provider_manager; - if let Some(request_id) = request_id { - provider_manager.confirm_identified_playback_activity(&owner, request_id); - } else { - provider_manager.confirm_playback_activity(&owner); - } - } - - fn release_stream_and_provider_handle_once(&mut self) { - self.stop_grace_task(); - self.release_provider_handle_and_detach_body(); - self.release_user_stream(); - } - - fn stop_direct_body_idle_timeout(&mut self) { - self.provider_stopped = true; - self.preempt_cancelled = None; - self.inner = None; - self.provider_error_class = Some(BODY_IDLE_TIMEOUT_ERROR_CLASS); - self.provider_http_status = None; - let _ = self.provider_end_reason.compare_exchange( - PROVIDER_END_NOT_SET, - PROVIDER_END_ERROR, - Ordering::Relaxed, - Ordering::Relaxed, - ); - warn!( - "Direct body stream idle timeout after {DIRECT_BODY_IDLE_TIMEOUT_SECS}s for {}, terminating stream", - sanitize_sensitive_info(&self.fingerprint.addr.to_string()) - ); - self.release_stream_and_provider_handle_once(); - } - - fn mark_direct_body_progress(&mut self) { - if self.direct_body_idle_timeout.mark_progress() { - self.connection_manager.touch_direct_body_activity(&self.fingerprint.addr); - } - } - - fn stop_grace_task(&mut self) { - if let Some(task) = self.grace_task_handle.take() { - task.abort(); - } - if let Some(token) = self.provisioning_stop_signal.take() { - token.cancel(); - } - } - - fn clear_finished_grace_task(&mut self) { - if self.grace_task_handle.as_ref().is_some_and(tokio::task::JoinHandle::is_finished) { - self.grace_task_handle = None; - // If the task finished but the flag is still GRACE_PENDING (e.g. the task - // panicked or was cancelled before it could update the flag), reset the flag - // to INNER_STREAM so the client stream is not hung indefinitely. - if let Some(flag) = &self.send_custom_stream_flag { - let _ = flag.compare_exchange( - StreamMode::GracePending as u8, - StreamMode::Inner as u8, - Ordering::AcqRel, - Ordering::Relaxed, - ); - } - } - } - - fn stop_provider_stream_preempted(&mut self) -> bool { - if self.response_mode == StreamResponseMode::HlsResource { - self.provider_end_reason.store(PROVIDER_END_PREEMPTED, Ordering::Relaxed); - self.provider_error_class = Some("preempted"); - self.terminate_quietly(); - return false; - } - self.provider_stopped = true; - self.preempt_cancelled = None; - self.stop_grace_task(); - self.provider_end_reason.store(PROVIDER_END_PREEMPTED, Ordering::Relaxed); - self.provider_error_class = Some("preempted"); - - let is_superseded = self - .provider_handle - .as_ref() - .and_then(|m| m.handle()) - .is_some_and(|h| h.get_close_reason() == tuliprox_core::model::ProviderCloseReason::Superseded); - - let mut serve_preempted_custom = false; - if self.provider_handle.is_some() { - let managed = self.provider_handle.take(); - self.provider_handle_released = true; - // Synchronous provider-slot release; the custom-video detail update is a - // separate, best-effort UI effect. - drop(managed); - if !is_superseded && self.custom_video.low_priority_preempted.is_some() { - serve_preempted_custom = true; - if let Some(flag) = &self.send_custom_stream_flag { - store_stream_mode(flag, StreamMode::LowPriorityPreempted); - } else { - // Fallback: create_active_client_stream usually initializes this via stream_grace_period. - self.send_custom_stream_flag = - Some(Arc::new(AtomicU8::new(StreamMode::LowPriorityPreempted as u8))); - } - } else if let Some(flag) = &self.send_custom_stream_flag { - store_stream_mode(flag, StreamMode::Inner); - } - - if let Some(waker) = &self.waker { - waker.wake(); - } - - let addr = self.fingerprint.addr; - // Drop the provider stream immediately instead of replacing with an - // allocated empty stream — avoids a heap allocation on every preemption. - self.inner = None; - - debug_if_enabled!( - "Provider stream preempted for {}; stopping client stream", - sanitize_sensitive_info(&addr.to_string()) - ); - if serve_preempted_custom { - self.connection_manager.send_cleanup(CleanupEvent::UpdateDetailAndReleaseProvider { - addr, - stream_uid: self.stream_uid, - video_type: CustomVideoStreamType::LowPriorityPreempted, - handle: None, - }); - } else { - self.release_user_stream(); - } - } - serve_preempted_custom - } - - fn stop_provider_stream(&mut self, mode: StreamMode) { - self.provider_stopped = true; - self.preempt_cancelled = None; - self.stop_grace_task(); - - if self.provider_handle.is_some() { - // Synchronous provider-slot release; the custom-video detail update is a - // separate, best-effort UI effect. - self.release_provider_handle_and_detach_body(); - - if mode == StreamMode::ChannelUnavailable { - if let Some(flag) = &self.send_custom_stream_flag { - let _ = flag.compare_exchange( - StreamMode::Inner as u8, - StreamMode::ChannelUnavailable as u8, - Ordering::AcqRel, - Ordering::Relaxed, - ); - } - } - - if let Some(waker) = &self.waker { - waker.wake(); - } - - let addr = self.fingerprint.addr; - let reason = match mode { - StreamMode::ChannelUnavailable => "unavailable provider channel", - StreamMode::UserExhausted => "user grace period exhaustion", - StreamMode::ProviderExhausted => "provider grace period exhaustion", - StreamMode::Provisioning => "provider grace period provisioning", - StreamMode::LowPriorityPreempted => "low-priority preemption", - StreamMode::ReentrySuppressed => "reentry suppression", - StreamMode::Inner | StreamMode::GracePending => "stream mode transition", - }; - debug_if_enabled!( - "Provider stream stopped due to {reason} for {}", - sanitize_sensitive_info(&addr.to_string()) - ); - if let Some(video_type) = Self::custom_video_type_for_mode(mode) { - self.connection_manager.send_cleanup(CleanupEvent::UpdateDetailAndReleaseProvider { - addr, - stream_uid: self.stream_uid, - video_type, - handle: None, - }); - } else { - // No custom video for this mode: release the request without publishing - // a detail update so no error clip is shown. - self.release_user_stream(); - } - } - } - - /// Ends the response body without switching to a custom error video. Used for a - /// suppressed reentry retry, where a visible "connections exhausted" video would be - /// misleading: the request was declined by the reentry guard, not by a real limit. - fn terminate_quietly(&mut self) { - self.provider_stopped = true; - self.preempt_cancelled = None; - self.stop_grace_task(); - self.release_provider_handle_and_detach_body(); - self.release_user_stream(); - } - - fn reset_custom_video_timeout(&mut self) { - self.custom_video_timeout_mode = None; - self.custom_video_timeout_sleep = None; - } - - /// Returns whether the custom state can serve a body; finite HLS objects terminate on entry. - fn enter_custom_mode(&mut self, mode: StreamMode) -> bool { - if self.response_mode == StreamResponseMode::HlsResource { - self.terminate_quietly(); - return false; - } - if self.custom_video_timeout_mode != Some(mode) { - self.custom_video_timeout_mode = Some(mode); - self.custom_video_timeout_sleep = if self.custom_video_timeout_secs > 0 { - Some(Box::pin(tokio::time::sleep(tokio::time::Duration::from_secs(u64::from( - self.custom_video_timeout_secs, - ))))) - } else { - None - }; - } - - if !self.provider_stopped { - info!( - "Switching to {mode:?} custom video stream for {}", - sanitize_sensitive_info(&self.fingerprint.addr.to_string()) - ); - self.stop_provider_stream(mode); - } - true - } - - fn custom_video_timed_out(&mut self, cx: &mut Context<'_>, mode: StreamMode) -> bool { - if self.custom_video_timeout_secs == 0 { - return false; - } - - if self.custom_video_timeout_mode != Some(mode) { - return false; - } - - if let Some(timeout_sleep) = self.custom_video_timeout_sleep.as_mut() { - return timeout_sleep.as_mut().poll(cx).is_ready(); - } - - false - } -} - -fn wrap_timed_client_stream_if_needed( - app_state: &Arc, - stream: BoxedProviderStream, - addr: SocketAddr, - virtual_id: VirtualId, - stream_uid: Option, -) -> BoxedProviderStream { - let config = app_state.app_config.config.load(); - match config.sleep_timer_mins { - None => stream, - Some(mins) => { - let secs = u32::try_from((u64::from(mins) * 60).min(u64::from(u32::MAX))).unwrap_or(0); - if secs > 0 { - TimedClientStream::new( - &app_state.app_config, - &app_state.connection_manager, - stream, - secs, - addr, - virtual_id, - stream_uid, - ) - .boxed() - } else { - stream - } - } - } -} - -fn create_deferred_provider_open_future( - app_state: &Arc, - stream_details: &StreamDetails, - fingerprint: &Fingerprint, - stream_channel: &StreamChannel, - req_headers: &HeaderMap, -) -> Option { - if !stream_details.has_deferred_provider_open() { - return None; - } - - let provider_name = stream_details.provider_name.as_deref()?; - let request_url = stream_details.request_url.as_deref()?; - let input = find_input_by_provider_name(app_state.as_ref(), provider_name)?; - let stream_url = url::Url::parse(request_url).ok()?; - let stream_options = get_stream_options(&app_state.app_config, stream_details.response_mode); - let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); - let disabled_headers = app_state.get_disabled_headers(); - let mut provider_stream_factory_options = - ProviderStreamFactoryOptions::new(&crate::api::model::ProviderStreamFactoryParams { - addr: fingerprint.addr, - item_type: stream_channel.item_type, - share_stream: stream_channel.shared, - stream_options: &stream_options, - stream_url: &stream_url, - req_headers, - input_headers: Some(&input.headers), - session_headers: stream_details.session_headers.as_ref(), - disabled_headers: disabled_headers.as_ref(), - default_user_agent: default_user_agent.as_deref(), - username: None, - client_ip: Some(&fingerprint.client_ip), - stream_channel: Some(stream_channel), - connect_failure_stage: Some(FailureStage::ProviderOpen), - content_representation: stream_details.content_representation, - }) - .for_deferred_open(); - if stream_details.provider_handle.is_some() { - if let Some(stream_index) = stream_details.user_agent_stream_index { - provider_stream_factory_options.apply_user_agent_stream_index(stream_index); - } - } - provider_stream_factory_options.set_provider(input.get_resolve_provider(stream_url.as_ref())); - provider_stream_factory_options.apply_input_options(&input); - - Some(DeferredProviderOpenState::Pending(Box::new(DeferredProviderOpenContext { - app_state: Arc::clone(app_state), - provider_stream_factory_options, - }))) -} - -fn create_timed_stream_context(app_state: &Arc, virtual_id: VirtualId) -> Option { - let config = app_state.app_config.config.load(); - let mins = config.sleep_timer_mins?; - let duration_secs = u32::try_from((u64::from(mins) * 60).min(u64::from(u32::MAX))).unwrap_or(0); - (duration_secs > 0).then(|| TimedStreamContext { app_state: Arc::clone(app_state), duration_secs, virtual_id }) -} - -fn should_use_direct_body_idle_timeout(stream_channel: &StreamChannel) -> bool { - uses_direct_body_idle_timeout(stream_channel) -} - pub(in crate::api) struct ActiveClientStream { state: ActiveClientStreamState, } -impl Stream for ActiveClientStream { - type Item = Result; - - #[allow(clippy::too_many_lines)] - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - loop { - // 1. Preemption check (user priority feature) - if let Some(fut) = self.state.preempt_cancelled.as_mut() { - if fut.as_mut().poll(cx).is_ready() && !self.state.stop_provider_stream_preempted() { - return Poll::Ready(None); - } - } - - // 2. Grace task lifecycle management + waker registration - self.state.clear_finished_grace_task(); - if let Some(waker) = &self.state.waker { - waker.register(cx.waker()); - } - - // 3. Read atomic mode flag (set by grace task or stop_provider_stream) - let Some(mode) = (match &self.state.send_custom_stream_flag { - Some(flag) => StreamMode::try_from(flag.load(Ordering::Acquire)).ok(), - None => Some(StreamMode::Inner), - }) else { - // An unknown mode can only come from a corrupted flag. Never map it - // onto a live state such as `GracePending`; end the body safely. - debug_assert!(false, "unknown stream mode flag"); - error!( - "Unknown stream mode for {}, terminating stream", - sanitize_sensitive_info(&self.state.fingerprint.addr.to_string()) - ); - self.state.terminate_quietly(); - return Poll::Ready(None); - }; - - // Dispatch based on the current streaming phase. - match mode { - // Grace period: hold_stream=true, waiting for grace task to resolve - StreamMode::GracePending => { - self.state.reset_custom_video_timeout(); - return Poll::Pending; - } - - // Live streaming: forward bytes from upstream provider - StreamMode::Inner => { - self.state.reset_custom_video_timeout(); - - if self.state.inner.is_none() { - if let Some(deferred_provider_open) = self.state.deferred_provider_open.take() { - match deferred_provider_open { - DeferredProviderOpenState::Pending(context) => { - let app_state = Arc::clone(&context.app_state); - let client = { - let http_client = app_state.http_clients.default.load(); - http_client.as_ref().clone() - }; - let lifecycle = self - .state - .provider_handle - .as_ref() - .and_then(ProviderStreamOpenLifecycle::from_managed); - let future = Box::pin(async move { - match open_provider_stream_with_lifecycle( - &app_state.provider_stream_ctx(), - &client, - context.provider_stream_factory_options, - lifecycle, - ) - .await - { - Some(ProviderStreamOpen::Stream(response)) - if matches!(response.info, Some((_, _, _, Some(_)))) => - { - let Some((_headers, _status, _response_url, Some(custom_video_type))) = - response.info - else { - return DeferredProviderOpenOutcome::Failed; - }; - ActiveClientStreamState::mode_for_custom_video_type(custom_video_type) - .map_or( - DeferredProviderOpenOutcome::Failed, - DeferredProviderOpenOutcome::Mode, - ) - } - Some(ProviderStreamOpen::Stream(response)) => { - DeferredProviderOpenOutcome::Stream(response.stream) - } - Some(ProviderStreamOpen::UpstreamStatus { .. }) | None => { - DeferredProviderOpenOutcome::Failed - } - } - }); - self.state.deferred_provider_open = - Some(DeferredProviderOpenState::Opening(future)); - continue; - } - DeferredProviderOpenState::Opening(mut future) => match future.as_mut().poll(cx) { - Poll::Pending => { - self.state.deferred_provider_open = - Some(DeferredProviderOpenState::Opening(future)); - return Poll::Pending; - } - Poll::Ready(DeferredProviderOpenOutcome::Stream(stream)) => { - self.state.provider_reconnect_count.fetch_add(1, Ordering::Relaxed); - self.state.inner = Some(self.state.wrap_provider_stream(stream)); - self.state.mark_direct_body_progress(); - continue; - } - Poll::Ready(DeferredProviderOpenOutcome::Mode(mode)) => { - if !self.state.enter_custom_mode(mode) { - return Poll::Ready(None); - } - continue; - } - Poll::Ready(DeferredProviderOpenOutcome::Failed) => { - if !self.state.enter_custom_mode(StreamMode::ChannelUnavailable) { - return Poll::Ready(None); - } - continue; - } - }, - } - } - - if self.state.grace_task_handle.is_none() { - self.state.stop_provider_stream(StreamMode::ChannelUnavailable); - return Poll::Ready(None); - } - - return Poll::Pending; - } - - match self.state.inner.as_mut().map(|inner| Pin::new(inner).poll_next(cx)) { - Some(Poll::Ready(Some(Ok(bytes)))) => { - if !bytes.is_empty() { - self.state.confirm_lease_once(); - } - self.state.mark_direct_body_progress(); - return Poll::Ready(Some(Ok(bytes))); - } - Some(Poll::Ready(Some(Err(e)))) => { - error!("Inner stream error: {e:?}"); - self.state.provider_error_class = Some(e.provider_error_class()); - self.state.provider_http_status = e.provider_http_status(); - let _ = self.state.provider_end_reason.compare_exchange( - PROVIDER_END_NOT_SET, - PROVIDER_END_ERROR, - Ordering::Relaxed, - Ordering::Relaxed, - ); - if self.state.grace_task_handle.is_none() { - self.state.stop_provider_stream(StreamMode::ChannelUnavailable); - return Poll::Ready(None); - } - - return Poll::Pending; - } - Some(Poll::Ready(None)) | None => { - let _ = self.state.provider_end_reason.compare_exchange( - PROVIDER_END_NOT_SET, - PROVIDER_END_CLOSED, - Ordering::Relaxed, - Ordering::Relaxed, - ); - if self.state.grace_task_handle.is_none() { - self.state.stop_provider_stream(StreamMode::ChannelUnavailable); - return Poll::Ready(None); - } - - return Poll::Pending; - } - Some(Poll::Pending) => { - if self.state.direct_body_idle_timeout.poll_expired(cx) { - self.state.stop_direct_body_idle_timeout(); - return Poll::Ready(None); - } - return Poll::Pending; - } - } - } - - // Quiet termination: the reentry guard declined a retry of a recently - // evicted playback. End the body without a user-visible error video. - StreamMode::ReentrySuppressed => { - info!( - "Suppressing reentry retry for {}, terminating stream", - sanitize_sensitive_info(&self.state.fingerprint.addr.to_string()) - ); - self.state.terminate_quietly(); - return Poll::Ready(None); - } - - // Custom video modes: serve the appropriate buffer - video_mode => { - if self.state.custom_video_timeout_mode != Some(video_mode) - && !self.state.enter_custom_mode(video_mode) - { - return Poll::Ready(None); - } - - if self.state.custom_video_timed_out(cx, video_mode) { - info!( - "Custom video {video_mode:?} timed out for {}, terminating stream", - sanitize_sensitive_info(&self.state.fingerprint.addr.to_string()) - ); - return Poll::Ready(None); - } - - let is_provisioning = video_mode == StreamMode::Provisioning && self.state.provisionable; - - let buffer_opt = match video_mode { - StreamMode::UserExhausted => self.state.custom_video.user_exhausted.as_mut(), - StreamMode::ProviderExhausted => self.state.custom_video.provider_exhausted.as_mut(), - StreamMode::ChannelUnavailable => self.state.custom_video.unavailable.as_mut(), - StreamMode::Provisioning => self.state.custom_video.provisioning.as_mut(), - StreamMode::LowPriorityPreempted => self.state.custom_video.low_priority_preempted.as_mut(), - _ => None, - }; - - if let Some(buffer) = buffer_opt { - buffer.register_waker(cx.waker()); - if let Some(chunk) = buffer.next_chunk() { - return Poll::Ready(Some(Ok(chunk))); - } - - // Provisioning loops until preemption fires; all others terminate. - if is_provisioning { - return Poll::Pending; - } - - info!( - "Custom video {video_mode:?} buffer exhausted for {}, terminating stream", - sanitize_sensitive_info(&self.state.fingerprint.addr.to_string()) - ); - return Poll::Ready(None); - } - - // No custom video configured for this mode -> terminate immediately. - info!( - "No custom video configured for {video_mode:?} mode for {}, terminating stream", - sanitize_sensitive_info(&self.state.fingerprint.addr.to_string()) - ); - return Poll::Ready(None); - } - } - } - } -} - -impl Drop for ActiveClientStream { - fn drop(&mut self) { self.state.release_stream_and_provider_handle_once(); } -} - -/// Terminal failure while admitting a playback request. Carried to the response layer so a -/// rejected request surfaces as a non-success HTTP response instead of an empty provider body. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum StreamAdmissionError { - CleanupReceiverClosed, - CleanupAdmissionTimeout, - RegistrationRejected, -} - -impl From for StreamAdmissionError { - fn from(reason: ConnectionRejectionReason) -> Self { - match reason { - ConnectionRejectionReason::CleanupReceiverClosed => Self::CleanupReceiverClosed, - ConnectionRejectionReason::CleanupAdmissionTimeout => Self::CleanupAdmissionTimeout, - ConnectionRejectionReason::RegistrationFailed => Self::RegistrationRejected, - } - } -} - -#[allow(clippy::too_many_lines)] -pub(crate) async fn create_active_client_stream( - request: ActiveClientStreamParams<'_>, -) -> Result { - let ActiveClientStreamParams { - mut stream_details, - app_state, - user, - connection_permission, - connection_kind, - fingerprint, - stream_channel, - socket_bound, - session_token, - req_headers, - meter_uid, - meter_stream, - } = request; - if connection_permission == UserConnectionPermission::Exhausted { - error!("Something is wrong this should not happen"); - } - let response_mode = stream_details.response_mode; - let grant_user_grace_period = connection_permission == UserConnectionPermission::GracePeriod; - let username = user.username.as_str(); - let provider_name = stream_details.provider_name.clone().unwrap_or_else(|| "unknown".intern()); - - let user_agent = req_headers.get(USER_AGENT).map(|h| String::from_utf8_lossy(h.as_bytes())).unwrap_or_default(); - - let virtual_id = stream_channel.virtual_id; - let is_shared_source_stream = stream_channel.shared && stream_details.stream.is_some(); - let direct_body_idle_timeout = if should_use_direct_body_idle_timeout(&stream_channel) { - DirectBodyIdleTimeout::enabled() - } else { - DirectBodyIdleTimeout::disabled() - }; - // A shared response already owns a guaranteed ReleaseSharedSubscriber permit - // before this registration starts. That terminal event also removes this exact user request, so reserving - // another permit from the same bounded queue would create a circular admission - // dependency under saturation. - let shared_cleanup_capability = stream_details.shared_subscriber_id.take(); - let shared_subscriber_id = - shared_cleanup_capability.as_ref().map(tuliprox_session::SharedCleanupCapability::subscriber_id); - let request_uid = shared_subscriber_id.map_or_else( - || app_state.connection_manager.next_stream_uid(), - tuliprox_core::model::SharedSubscriberId::stream_uid, - ); - let provider_request_id = stream_details - .provider_handle - .as_ref() - .and_then(|managed| managed.handle()) - .and_then(|handle| handle.playback_request_id); - let connection = crate::api::model::ConnectionParams { - meter_uid, - username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind, - priority: user.priority, - soft_priority: user.soft_priority, - fingerprint, - provider: provider_name, - stream_channel: &stream_channel, - user_agent, - session_token, - }; - let mut registered_request = if let Some(capability) = shared_cleanup_capability.filter(|_| stream_channel.shared) { - app_state - .connection_manager - .update_connection_with_uid_using_shared_cleanup( - connection, - tuliprox_session::ConnectionHistoryMode::EmitConnect, - capability, - provider_request_id, - ) - .await - } else { - app_state - .connection_manager - .update_connection_with_uid_and_session_registration( - connection, - tuliprox_session::ConnectionHistoryMode::EmitConnect, - request_uid, - provider_request_id, - stream_details.session_registration.as_ref(), - ) - .await - }; - let request_uid = registered_request.request_uid; - let display_stream_uid = registered_request.display_uid(); - let stream_uid = Some(request_uid); - if registered_request.display_stream.is_none() { - // Admission was rejected (closed cleanup receiver, bounded timeout, or failed - // registration). No provider bytes may be delivered; the acquired provider - // handle and any open provider stream are released when `request` (and its - // `stream_details`) is dropped on return. - let reason = registered_request.rejection_reason().unwrap_or(ConnectionRejectionReason::RegistrationFailed); - error!("Stream admission rejected for user {username}: {reason:?}; dropping provider handle"); - return Err(reason.into()); - } - if let Some((_, _, _m_, Some(cvt))) = stream_details.stream_info.as_ref() { - app_state.connection_manager.update_stream_detail_by_uid(request_uid, *cvt).await; - } - - let meter = if meter_stream && meter_uid != 0 { - let meter = Arc::new(StreamMeterHandle::new(meter_uid, Arc::downgrade(&app_state.event_manager))); - app_state.event_manager.register_meter(Arc::clone(&meter)).await; - Some(meter) - } else { - None - }; - - // Shared broadcaster source (first subscriber path): feed provider bytes directly. - // Grace/custom handling is not needed here because this stream is only the fan-out source. - if is_shared_source_stream { - if let Some(stream) = stream_details.stream.take() { - let stream = if let Some(meter) = &meter { - MeteringStream::new(stream, Arc::clone(meter), Arc::clone(&app_state.event_manager)).boxed() - } else { - stream - }; - // The shared source owns the claim from here; disarm the registration rollback. - registered_request.commit(); - return Ok(stream); - } - } - - let provisioning_info = resolve_grace_period_provisioning(app_state, &stream_details); - let has_provisioning = provisioning_info.is_some(); - let provisioning_stop_signal = provisioning_info.as_ref().map(|info| info.stop_signal.clone()); - let hold_stream = stream_details.grace_period.hold_stream; - let capacity_notify = app_state.connection_manager.capacity_notified(); - let pending_provider_version = if hold_stream { - if let Some(token) = session_token { - app_state.active_users.pending_provider_version(&user.username, token).await - } else { - None - } - } else { - None - }; - // `GraceActive` version: populated when the session is in `GraceActive` lifecycle (GraceMode::Instant). - let grace_active_version = if let Some(token) = session_token { - app_state.active_users.grace_active_version(&user.username, token).await - } else { - None - }; - - let waker = Arc::new(AtomicWaker::new()); - let owned_session_token: Option = session_token.map(str::to_string); - // A provider-backed playback confirms its lease owner once media flows. Local - // media and streams without a provider handle carry no lease to confirm. - let lease_owner: Option> = session_token - .filter(|_| stream_details.provider_handle.is_some() || stream_details.has_deferred_provider_open()) - .map(Arc::::from); - let media_started = if let Some(owner) = lease_owner.as_deref() { - app_state.active_users.media_started_flag(&user.username, owner).await - } else { - None - }; - let owned_grace_ctx = stream_details.grace_resolution_context.clone(); - let lease_request_id = stream_details - .provider_handle - .as_ref() - .and_then(|managed| managed.handle()) - .and_then(|handle| handle.playback_request_id); - // Compute deferred-open metadata before taking the handle: the grace snapshot only - // needs the provider identity, while the body owns the actual release. - let has_deferred_open = stream_details.has_deferred_provider_open(); - let deferred_provider_open = - create_deferred_provider_open_future(app_state, &stream_details, fingerprint, &stream_channel, req_headers); - let timed_stream_context = deferred_provider_open - .as_ref() - .and_then(|_| create_timed_stream_context(app_state, VirtualId::new(virtual_id))); - // The body owns the provider slot from here on; the managed owner releases it - // synchronously on drop, never through a lossy cleanup message. - let mut provider_handle_preserved = stream_details.provider_handle.take(); - let stream_taken = stream_details.stream.take(); - let grace_waker = - if grant_user_grace_period || stream_details.provider_grace_active { Some(Arc::clone(&waker)) } else { None }; - let (grace_stop_flag, grace_task_handle) = stream_grace_period(GracePeriodParams { - app_state: Arc::clone(app_state), - stream_details, - user_grace_period: grant_user_grace_period, - user: user.clone(), - fingerprint: fingerprint.clone(), - virtual_id: VirtualId::new(virtual_id), - session_token: owned_session_token, - provisioning_info, - waker: grace_waker, - hold_stream, - capacity_notify, - pending_provider_version, - grace_active_version, - grace_resolution_context: owned_grace_ctx, - grace_kind: Some(connection_kind), - socket_bound, - shared_subscriber_id, - }); - - let cfg = &app_state.app_config; - let custom_response = cfg.custom_stream_response.load(); - let custom_video_timeout_secs = cfg.config.load().custom_stream_response_timeout_secs; - let custom_video = custom_response.as_ref().filter(|_| response_mode == StreamResponseMode::Stream).map_or( - CustomVideoBuffers { - user_exhausted: None, - provider_exhausted: None, - unavailable: None, - provisioning: None, - low_priority_preempted: None, - }, - |c| CustomVideoBuffers { - user_exhausted: c.user_connections_exhausted.clone(), - provider_exhausted: c.provider_connections_exhausted.clone(), - unavailable: c.channel_unavailable.clone(), - provisioning: c.panel_api_provisioning.clone(), - low_priority_preempted: c.low_priority_preempted.clone(), - }, - ); - - let stream: Option = match stream_taken { - None => { - if !has_deferred_open { - drop(provider_handle_preserved.take()); - } - None - } - Some(stream) => { - let stream = if let Some(meter) = &meter { - MeteringStream::new(stream, Arc::clone(meter), Arc::clone(&app_state.event_manager)).boxed() - } else { - stream - }; - Some(wrap_timed_client_stream_if_needed( - app_state, - stream, - fingerprint.addr, - VirtualId::new(virtual_id), - display_stream_uid, - )) - } - }; - - let preempt_cancelled = provider_handle_preserved - .as_ref() - .and_then(|managed| managed.handle()) - .and_then(|h| h.cancel_token.as_ref()) - .map(|token| Box::pin(token.clone().cancelled_owned())); - - let mut send_custom_stream_flag = grace_stop_flag; - if send_custom_stream_flag.is_none() && preempt_cancelled.is_some() && custom_video.low_priority_preempted.is_some() - { - send_custom_stream_flag = Some(Arc::new(AtomicU8::new(StreamMode::Inner as u8))); - } - - let state = ActiveClientStreamState { - response_mode, - inner: stream, - deferred_provider_open, - timed_stream_context, - preempt_cancelled, - grace_task_handle, - provisioning_stop_signal, - provider_handle: provider_handle_preserved, - send_custom_stream_flag, - provisionable: has_provisioning, - custom_video, - meter, - event_manager: Arc::clone(&app_state.event_manager), - waker: Some(waker), - connection_manager: Arc::clone(&app_state.connection_manager), - fingerprint: Arc::new(fingerprint.clone()), - stream_uid, - provider_stopped: false, - user_stream_released: false, - provider_handle_released: false, - custom_video_timeout_secs, - custom_video_timeout_mode: None, - custom_video_timeout_sleep: None, - direct_body_idle_timeout, - provider_end_reason: AtomicU8::new(PROVIDER_END_NOT_SET), - provider_error_class: None, - provider_http_status: None, - provider_reconnect_count: AtomicU8::new(0), - lease_owner, - media_started, - lease_confirmed: false, - lease_request_id, - request_cleanup: registered_request.into_body_cleanup(), - }; - - Ok(ActiveClientStream { state }.boxed()) -} - -fn resolve_grace_period_provisioning( - app_state: &Arc, - stream_details: &StreamDetails, -) -> Option { - if stream_details.disable_provider_grace || !stream_details.provider_grace_active { - return None; - } - let provider_name = stream_details.provider_name.as_deref(); - let input = provider_name.and_then(|name| find_input_by_provider_name(app_state.as_ref(), name))?; - if !can_provision_on_exhausted(app_state, &input) { - return None; - } - - let stop_signal = CancellationToken::new(); - Some(GraceProvisioningInfo { input, stop_signal }) -} - -#[allow(clippy::too_many_lines)] -fn stream_grace_period(request: GracePeriodParams) -> (Option>, Option>) { - let GracePeriodParams { - app_state, - stream_details, - user_grace_period, - user, - fingerprint, - virtual_id, - session_token, - provisioning_info, - waker, - hold_stream, - capacity_notify, - pending_provider_version, - grace_active_version, - grace_resolution_context, - grace_kind, - socket_bound, - shared_subscriber_id, - .. - } = request; - let grace_period = stream_details.grace_period; - let active_users = Arc::clone(&app_state.active_users); - let active_provider = Arc::clone(&app_state.active_provider); - let connection_manager = Arc::clone(&app_state.connection_manager); - - let provider_grace_check = if stream_details.provider_grace_active - && stream_details.provider_name.is_some() - && !stream_details.disable_provider_grace - { - stream_details.provider_name.clone() - } else { - None - }; - - let user_max_connections = user.max_connections; - let user_grace_check = if user_grace_period && user_max_connections > 0 { - let user_name = user.username.clone(); - Some((user_name, user_max_connections)) - } else { - None - }; - - if provider_grace_check.is_some() || user_grace_check.is_some() { - let stream_strategy_flag = - Arc::new(AtomicU8::new(if hold_stream { StreamMode::GracePending as u8 } else { StreamMode::Inner as u8 })); - let stream_strategy_flag_copy = Arc::clone(&stream_strategy_flag); - let grace_period_millis = grace_period.period_millis; - - let user_manager = Arc::clone(&active_users); - let provider_manager = Arc::clone(&active_provider); - let connection_manager = Arc::clone(&connection_manager); - let reconnect_flag = stream_details.reconnect_flag.clone(); - let fingerprint = fingerprint.clone(); - let app_state = Arc::clone(&app_state); - let pending_username = user.username.clone(); - // Clone owned copies of fields borrowed from `request` so the spawn sees owned values. - let grace_resolution_context = grace_resolution_context.clone(); - // Safety timeout: if async operations inside the grace task stall, force the flag - // out of GRACE_PENDING so the client stream is not hung indefinitely. - // Allow grace_period_millis for the intentional delay plus a 10-second buffer - // for the async connection checks that follow. - let grace_task_timeout = tokio::time::Duration::from_millis(grace_period_millis.saturating_add(10_000)); - // Clone handles for use in the timeout fallback, in case the inner async block is cancelled. - let flag_for_fallback = Arc::clone(&stream_strategy_flag_copy); - let waker_for_fallback = waker.clone(); - let session_token_timeout = session_token.clone(); - let active_users_timeout = Arc::clone(&active_users); - let pending_username_timeout = pending_username.clone(); - let grace_task_handle = tokio::spawn(async move { - let timed_out = tokio::time::timeout(grace_task_timeout, async move { - let deadline = tokio::time::Instant::now() + tokio::time::Duration::from_millis(grace_period_millis); - let mut pending_wake_source = PendingProviderWakeSource::Activated; - loop { - let capacity_wait = capacity_notify.notified(); - tokio::pin!(capacity_wait); - - let user_ok = match &user_grace_check { - Some((username, max_connections)) => { - user_manager.user_connections(username).await <= *max_connections - } - None => true, - }; - let provider_ok = match &provider_grace_check { - Some(provider_name) => !provider_manager.is_over_limit(provider_name), - None => true, - }; - if user_ok && provider_ok { - break; - } - - tokio::select! { - () = tokio::time::sleep_until(deadline) => { - pending_wake_source = PendingProviderWakeSource::Timeout; - break; - } - () = &mut capacity_wait => { - pending_wake_source = PendingProviderWakeSource::CapacityNotify; - } - } - } - - let mut updated = false; - if let Some((username, max_connections)) = user_grace_check { - let active_connections = user_manager.user_connections(&username).await; - if active_connections > max_connections { - // User-grace failed. Evaluate remaining strategies before final deny. - if let Some(ref ctx) = grace_resolution_context { - let eviction_guard = if socket_bound { - tuliprox_session::admission::EvictionReentryGuard::SocketPlayback { virtual_id } - } else { - tuliprox_session::admission::EvictionReentryGuard::Session( - // Defensive fallback: an empty token will not match any real session. - session_token.as_deref().unwrap_or_default(), - ) - }; - let remaining_result = - tuliprox_session::admission::evaluate_remaining_strategies_after_grace( - &app_state.admission_ctx(), - tuliprox_session::admission::AdmissionRequest { - username: &username, - max_connections, - soft_connections: user.soft_connections, - client_ip: &fingerprint.client_ip, - request_addr: &fingerprint.addr, - use_session_admission: true, - session_token: session_token.as_deref(), - activate_unbound_session: true, - eviction_reentry_guard: eviction_guard, - }, - ctx, - grace_kind, - ) - .await; - match remaining_result.admission.permission() { - shared::model::UserConnectionPermission::Allowed - | shared::model::UserConnectionPermission::GracePeriod => { - // Remaining strategy succeeded — proceed to Inner. - store_stream_mode(&stream_strategy_flag_copy, StreamMode::Inner); - // updated stays false - } - shared::model::UserConnectionPermission::Exhausted => { - let suppressed = remaining_result.admission.is_reentry_suppressed(); - if suppressed { - // Every remaining eviction candidate is reentry-protected, - // so this is a suppressed retry of a recently evicted - // playback. End the body quietly: a visible - // connections-exhausted video would be misleading because - // the request was declined by the reentry guard, not by a - // real connection limit. - store_stream_mode(&stream_strategy_flag_copy, StreamMode::ReentrySuppressed); - info!( - "Suppressing reentry retry for recently evicted playback of user {username}" - ); - } else { - // Remaining strategies exhausted — final UserExhausted. - store_stream_mode(&stream_strategy_flag_copy, StreamMode::UserExhausted); - connection_manager - .update_stream_detail( - &fingerprint.addr, - CustomVideoStreamType::UserConnectionsExhausted, - ) - .await; - info!("User connections exhausted for active clients: {username}"); - } - if let Some(id) = shared_subscriber_id { - connection_manager.shared_stream_manager.release_subscriber(id).await; - } - updated = true; - } - } - } else { - // No grace context — immediate UserExhausted. - store_stream_mode(&stream_strategy_flag_copy, StreamMode::UserExhausted); - connection_manager - .update_stream_detail( - &fingerprint.addr, - CustomVideoStreamType::UserConnectionsExhausted, - ) - .await; - if let Some(id) = shared_subscriber_id { - connection_manager.shared_stream_manager.release_subscriber(id).await; - } - info!("User connections exhausted for active clients: {username}"); - updated = true; - } - } - } - - if !updated { - if let Some(provider_name) = provider_grace_check { - if provider_manager.is_over_limit(&provider_name) { - if let Some(provisioning_info) = provisioning_info { - store_stream_mode(&stream_strategy_flag_copy, StreamMode::Provisioning); - connection_manager - .update_stream_detail(&fingerprint.addr, CustomVideoStreamType::Provisioning) - .await; - debug_if_enabled!( - "Provider grace period exhausted; provisioning for active clients: {provider_name}" - ); - let app_state = Arc::clone(&app_state); - let input_name = Arc::clone(&provisioning_info.input.name); - let stop_signal = provisioning_info.stop_signal; - let addr = fingerprint.addr; - tokio::spawn(async move { - if let Err(err) = run_panel_api_provisioning_probe( - app_state, - input_name, - stop_signal, - addr, - virtual_id, - ) - .await - { - error!("Error running Probe: {err:?}"); - } - }); - } else { - store_stream_mode(&stream_strategy_flag_copy, StreamMode::ProviderExhausted); - connection_manager - .update_stream_detail( - &fingerprint.addr, - CustomVideoStreamType::ProviderConnectionsExhausted, - ) - .await; - // Release the shared stream subscription to stop the subscriber loop - if let Some(id) = shared_subscriber_id { - connection_manager.shared_stream_manager.release_subscriber(id).await; - } - info!("Provider connections exhausted for active clients: {provider_name}"); - } - updated = true; - } - } - } - - if !updated { - store_stream_mode(&stream_strategy_flag_copy, StreamMode::Inner); - } - - // Resolve session lifecycle transitions. - // PendingProvider (Hold): activate on success, expire on failure. - if hold_stream { - if let (Some(token), Some(version)) = (session_token.as_deref(), pending_provider_version) { - let _transition_guard = - active_users.acquire_playback_transition(&pending_username, token).await; - if updated { - active_users - .expire_pending_provider(&pending_username, token, version, pending_wake_source) - .await; - } else { - active_users - .activate_pending_provider(&pending_username, token, version, pending_wake_source) - .await; - } - } - } - - // GraceActive (Instant): activate on success, expire on failure. - // grace_active_version is Some when the session is in `GraceActive` lifecycle. - if let (Some(token), Some(version)) = (session_token.as_deref(), grace_active_version) { - let _transition_guard = active_users.acquire_playback_transition(&pending_username, token).await; - if updated { - active_users.expire_grace_active(&pending_username, token, version).await; - } else { - active_users.activate_grace_active(&pending_username, token, version).await; - } - } - - if updated { - if let Some(flag) = reconnect_flag { - flag.cancel(); - } - } - - if let Some(w) = waker.as_ref() { - w.wake(); - } - }) - .await; - - if timed_out.is_err() { - // Grace task exceeded its budget without updating the flag — reset GRACE_PENDING - // to INNER_STREAM so the client stream is not hung indefinitely. - // Also resolve any session lifecycle to prevent inconsistent state: - // a PendingProvider / GraceActive session that was never resolved would cause - // the next admission attempt to incorrectly skip re-evaluation. - if let (Some(token), Some(version)) = (session_token_timeout.as_deref(), pending_provider_version) { - let _transition_guard = - active_users_timeout.acquire_playback_transition(&pending_username_timeout, token).await; - active_users_timeout - .expire_pending_provider( - &pending_username_timeout, - token, - version, - PendingProviderWakeSource::Timeout, - ) - .await; - } - if let (Some(token), Some(version)) = (session_token_timeout.as_deref(), grace_active_version) { - let _transition_guard = - active_users_timeout.acquire_playback_transition(&pending_username_timeout, token).await; - active_users_timeout.expire_grace_active(&pending_username_timeout, token, version).await; - } - error!("Grace period task timed out; resetting stream flag to prevent client hang"); - let _ = flag_for_fallback.compare_exchange( - StreamMode::GracePending as u8, - StreamMode::Inner as u8, - Ordering::AcqRel, - Ordering::Relaxed, - ); - if let Some(w) = waker_for_fallback.as_ref() { - w.wake(); - } - } - }); - return (Some(stream_strategy_flag), Some(grace_task_handle)); - } - (None, None) -} - #[cfg(test)] -mod tests { - use super::{ - super::buffered_stream::BufferedStream, create_active_client_stream, create_deferred_provider_open_future, - should_use_direct_body_idle_timeout, stream_grace_period, ActiveClientStream, ActiveClientStreamParams, - ActiveClientStreamState, CustomVideoBuffers, DeferredProviderOpenOutcome, DeferredProviderOpenState, - DirectBodyIdleTimeout, GracePeriodParams, StreamMode, TimedStreamContext, DIRECT_BODY_IDLE_TIMEOUT_SECS, - }; - use crate::{ - api::model::{ - connection_manager::PROVIDER_END_NOT_SET, ActiveProviderManager, ActiveUserManager, AppState, - BoxedProviderStream, CancelTokens, ConnectionManager, CreateUserSessionParams, CustomVideoStreamType, - EventManager, GraceResolutionContext, MetadataUpdateManager, PlaylistStorageState, - ProviderContentRepresentationMode, ProviderHandle, RecordingQueue, SharedStreamManager, StreamDetails, - StreamError, - }, - auth::Fingerprint, - model::{ - AppConfig, Config, ConfigInput, GracePeriodOptions, MediaToolCapabilities, ProxyUserCredentials, - SourcesConfig, StreamConfig, - }, - repository::GeoIp, - utils::FileLockManager, - }; - use arc_swap::{ArcSwap, ArcSwapOption}; - use axum::{body::Body, http::HeaderMap}; - use bytes::Bytes; - use futures::{pin_mut, StreamExt}; - use http_body_util::BodyExt; - use shared::{ - model::{ - AdmissionStrategy, ConfigPaths, InputFetchMethod, InputType, PlaylistItemType, StreamChannel, - UserConnectionPermission, VirtualId, XtreamCluster, - }, - utils::Internable, - }; - use std::{ - collections::HashMap, - net::SocketAddr, - pin::Pin, - sync::{ - atomic::{AtomicBool, AtomicU8, Ordering}, - Arc, - }, - task::{Context, Poll}, - time::Duration, - }; - use tokio::sync::{oneshot, Notify}; - use tokio_util::sync::CancellationToken; - - fn create_test_app_config() -> AppConfig { - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: "http://provider-1.example".to_string(), - username: Some("user1".to_string()), - password: Some("pass1".to_string()), - enabled: true, - priority: 0, - max_connections: 1, - method: InputFetchMethod::default(), - aliases: None, - ..ConfigInput::default() - }); - let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; - - AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config::default())), - sources: Arc::new(ArcSwap::from_pointee(sources)), - hdhomerun: Arc::new(ArcSwapOption::default()), - api_proxy: Arc::new(ArcSwapOption::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - } - } - - fn create_test_connection_manager() -> Arc { - let app_cfg = create_test_app_config(); - let event_manager = Arc::new(EventManager::new()); - let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let shared_manager = Arc::new(SharedStreamManager::new(Arc::clone(&provider_manager))); - provider_manager.set_shared_stream_manager(&shared_manager); - - let geo_ip = Arc::new(ArcSwapOption::::default()); - let config = app_cfg.config.load(); - let user_manager = Arc::new(ActiveUserManager::new(&config, &geo_ip, &event_manager)); - - Arc::new(ConnectionManager::new(&user_manager, &provider_manager, &shared_manager, &event_manager, None)) - } - - fn create_test_app_state() -> Arc { - let app_cfg = Arc::new(create_test_app_config()); - let event_manager = Arc::new(EventManager::new()); - let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); - active_provider.set_shared_stream_manager(&shared_stream_manager); - - let geoip = Arc::new(ArcSwapOption::::default()); - let config = app_cfg.config.load(); - let active_users = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); - let connection_manager = Arc::new(ConnectionManager::new( - &active_users, - &active_provider, - &shared_stream_manager, - &event_manager, - None, - )); - - let tokens = CancelTokens::default(); - let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); - - Arc::new(AppState { - recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( - Arc::clone(&active_provider), - Arc::clone(&connection_manager), - ), - app_config: app_cfg, - http_clients: Arc::default(), - recordings: Arc::new(RecordingQueue::new()), - cache: Arc::new(ArcSwapOption::default()), - shared_stream_manager, - hls: crate::api::model::HlsState::new(Arc::new(crate::api::model::HlsProxyManager::new())), - stalker_resolve_coordinator: crate::api::model::StalkerResolveCoordinator::default(), - active_users, - active_provider, - connection_manager, - event_manager, - cancel_tokens: ArcSwap::from_pointee(tokens), - playlists: Arc::new(PlaylistStorageState::new()), - geoip, - metadata_manager, - auth: crate::api::model::AuthState::for_tests(), - playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), - }) - } - - fn create_test_app_state_with_stream_config(stream: StreamConfig) -> Arc { - let config = Config { - reverse_proxy: Some(crate::model::ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: crate::model::ResourceRetryConfig::default(), - disabled_header: None, - stream: Some(stream), - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }), - user_access_control: true, - ..Config::default() - }; - - let mut app_cfg = create_test_app_config(); - app_cfg.config = Arc::new(ArcSwap::from_pointee(config)); - - let event_manager = Arc::new(EventManager::new()); - let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); - active_provider.set_shared_stream_manager(&shared_stream_manager); - - let geoip = Arc::new(ArcSwapOption::::default()); - let config_loaded = app_cfg.config.load(); - let active_users = Arc::new(ActiveUserManager::new(&config_loaded, &geoip, &event_manager)); - let connection_manager = Arc::new(ConnectionManager::new( - &active_users, - &active_provider, - &shared_stream_manager, - &event_manager, - None, - )); - - let tokens = CancelTokens::default(); - let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); - - Arc::new(AppState { - recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( - Arc::clone(&active_provider), - Arc::clone(&connection_manager), - ), - app_config: Arc::new(app_cfg), - http_clients: Arc::default(), - recordings: Arc::new(RecordingQueue::new()), - cache: Arc::new(ArcSwapOption::default()), - shared_stream_manager, - hls: crate::api::model::HlsState::new(Arc::new(crate::api::model::HlsProxyManager::new())), - stalker_resolve_coordinator: crate::api::model::StalkerResolveCoordinator::default(), - active_users, - active_provider, - connection_manager, - event_manager, - cancel_tokens: ArcSwap::from_pointee(tokens), - playlists: Arc::new(PlaylistStorageState::new()), - geoip, - metadata_manager, - auth: crate::api::model::AuthState::for_tests(), - playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), - }) - } - - fn create_test_user(username: &str) -> ProxyUserCredentials { - let mut user = ProxyUserCredentials::default(); - user.username = username.to_string(); - user.max_connections = 1; - user - } - - fn create_test_fingerprint(addr: std::net::SocketAddr) -> Fingerprint { - Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr) - } - - fn create_test_stream_channel(virtual_id: u32, url: &str) -> StreamChannel { - StreamChannel { - target_id: 1, - virtual_id, - provider_id: 1, - input_name: "input".intern(), - item_type: PlaylistItemType::Live, - cluster: XtreamCluster::Live, - group: "Live".intern(), - title: "Test Channel".intern(), - url: url.into(), - shared: false, - shared_joined_existing: None, - shared_stream_id: None, - technical: None, - epg_channel_id: None, - epg_reference_ts: None, - upstream_user_agent: None, - } - } - - fn create_test_shared_stream_channel(virtual_id: u32, url: &str) -> StreamChannel { - let mut channel = create_test_stream_channel(virtual_id, url); - channel.shared = true; - channel - } - - fn create_test_video_stream_channel(virtual_id: u32, url: &str) -> StreamChannel { - let mut channel = create_test_stream_channel(virtual_id, url); - channel.item_type = PlaylistItemType::Video; - channel.cluster = XtreamCluster::Video; - channel.group = "Movies".intern(); - channel.url = url.into(); - channel - } - - fn create_test_series_stream_channel(virtual_id: u32, url: &str) -> StreamChannel { - let mut channel = create_test_stream_channel(virtual_id, url); - channel.item_type = PlaylistItemType::Series; - channel.cluster = XtreamCluster::Series; - channel.group = "Series".intern(); - channel.url = url.into(); - channel - } - - #[derive(Clone, Default)] - struct DropTracker(Arc); - - impl DropTracker { - fn is_dropped(&self) -> bool { self.0.load(Ordering::Acquire) } - } - - struct DropTrackedProviderStream { - inner: BoxedProviderStream, - tracker: DropTracker, - } - - impl futures::Stream for DropTrackedProviderStream { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.inner.as_mut().poll_next(cx) - } - } - - impl Drop for DropTrackedProviderStream { - fn drop(&mut self) { self.tracker.0.store(true, Ordering::Release); } - } - - fn track_provider_stream(stream: BoxedProviderStream) -> (BoxedProviderStream, DropTracker) { - let tracker = DropTracker::default(); - let tracked_stream = DropTrackedProviderStream { inner: stream, tracker: tracker.clone() }.boxed(); - (tracked_stream, tracker) - } - - struct TestDirectStreamParams<'a> { - app_state: &'a Arc, - username: &'a str, - max_connections: u32, - addr: SocketAddr, - stream_channel: StreamChannel, - provider_stream: BoxedProviderStream, - provider_handle: Option, - } - - struct TestDirectStream { - stream: BoxedProviderStream, - uid: u32, - } - - async fn create_test_active_direct_stream(params: TestDirectStreamParams<'_>) -> TestDirectStream { - let TestDirectStreamParams { - app_state, - username, - max_connections, - addr, - stream_channel, - provider_stream, - provider_handle, - } = params; - let mut user = create_test_user(username); - user.max_connections = max_connections; - let fingerprint = create_test_fingerprint(addr); - let mut stream_details = StreamDetails::from_stream(provider_stream, GracePeriodOptions::default()); - if provider_handle.is_some() { - stream_details.provider_name = Some("provider_1".intern()); - } - stream_details.provider_handle = provider_handle.map(|handle| { - tuliprox_session::ManagedProviderHandle::new( - Arc::clone(&app_state.connection_manager.provider_manager), - handle, - ) - }); - let virtual_id = stream_channel.virtual_id; - - let stream = create_active_client_stream(ActiveClientStreamParams { - stream_details, - app_state, - user: &user, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: crate::api::model::ConnectionKind::Normal, - fingerprint: &fingerprint, - stream_channel, - socket_bound: false, - session_token: None, - req_headers: &HeaderMap::default(), - meter_uid: 0, - meter_stream: false, - }) - .await - .expect("direct test stream admission should succeed"); - let uid = app_state - .active_users - .active_streams() - .await - .into_iter() - .find(|active| { - active.username == username && active.addr == addr && active.channel.virtual_id == virtual_id - }) - .map(|active| active.uid) - .expect("direct test stream should be registered"); - - TestDirectStream { stream, uid } - } - - async fn acquire_test_provider_handle(app_state: &Arc, addr: SocketAddr) -> ProviderHandle { - app_state - .active_provider - .acquire_exact_connection_with_grace( - &"provider_1".intern(), - &addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .expect("direct test stream should acquire the provider slot") - } - - #[derive(Debug, Eq, PartialEq)] - struct TestLifecycleSnapshot { - active_counts: (usize, usize), - stream_uids: Vec, - provider_connections: usize, - } - - async fn lifecycle_snapshot(app_state: &Arc) -> TestLifecycleSnapshot { - let active_counts = app_state.active_users.active_users_and_connections().await; - let mut stream_uids = - app_state.active_users.active_streams().await.into_iter().map(|stream| stream.uid).collect::>(); - stream_uids.sort_unstable(); - TestLifecycleSnapshot { - active_counts, - stream_uids, - provider_connections: app_state.active_provider.get_provider_connections_count(), - } - } - - struct ExpectedLifecycle<'a> { - description: &'static str, - active_counts: (usize, usize), - stream_uids: &'a [u32], - provider_connections: usize, - dropped_streams: &'a [&'a DropTracker], - } - - async fn wait_for_lifecycle(app_state: &Arc, expected: ExpectedLifecycle<'_>) { - let mut expected_stream_uids = expected.stream_uids.to_vec(); - expected_stream_uids.sort_unstable(); - let expected_snapshot = TestLifecycleSnapshot { - active_counts: expected.active_counts, - stream_uids: expected_stream_uids, - provider_connections: expected.provider_connections, - }; - let completed = tokio::time::timeout(Duration::from_secs(1), async { - loop { - let snapshot = lifecycle_snapshot(app_state).await; - if snapshot == expected_snapshot && expected.dropped_streams.iter().all(|tracker| tracker.is_dropped()) - { - break; - } - tokio::task::yield_now().await; - } - }) - .await; - - if completed.is_err() { - let snapshot = lifecycle_snapshot(app_state).await; - let dropped = expected.dropped_streams.iter().map(|tracker| tracker.is_dropped()).collect::>(); - panic!( - "{} did not converge: expected={expected_snapshot:?}, actual={snapshot:?}, dropped={dropped:?}", - expected.description - ); - } - } - - fn create_deferred_provider_grace_details( - provider_name: &Arc, - provider_handle: tuliprox_session::ManagedProviderHandle, - ) -> StreamDetails { - StreamDetails { - shared_subscriber_id: None, - stream: None, - stream_info: None, - provider_name: Some(Arc::clone(provider_name)), - request_url: Some("http://provider-1.example/live/1".intern()), - session_headers: None, - provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), - user_agent_stream_index: None, - grace_period: GracePeriodOptions { period_millis: 100, timeout_secs: 0, hold_stream: true }, - provider_grace_active: true, - disable_provider_grace: false, - reconnect_flag: None, - provider_handle: Some(provider_handle), - content_representation: ProviderContentRepresentationMode::PreserveOrigin, - grace_resolution_context: None, - custom_reason: None, - response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), - session_registration: None, - } - } - - async fn start_deferred_provider_grace_resolution( - app_state: &Arc, - provider_name: &Arc, - deferred_addr: std::net::SocketAddr, - session_token: Option<&str>, - ) -> (Arc, tokio::task::JoinHandle<()>, crate::api::model::ProviderHandle) { - let deferred_handle = app_state - .active_provider - .acquire_exact_connection_with_grace( - provider_name, - &deferred_addr, - true, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .expect("deferred client should receive provider grace allocation"); - let mut stream_details = create_deferred_provider_grace_details( - provider_name, - tuliprox_session::ManagedProviderHandle::new( - Arc::clone(&app_state.connection_manager.provider_manager), - deferred_handle, - ), - ); - // The grace-resolution task only reads Copy fields of `stream_details` and never - // takes ownership of the handle, so disarm it back into the raw handle the caller - // releases explicitly after the grace task completes. - let deferred_provider_handle = stream_details - .provider_handle - .take() - .and_then(|mut managed| managed.disarm()) - .expect("deferred provider handle must be retained during grace"); - let test_user = create_test_user("grace-user"); - let test_fingerprint = create_test_fingerprint(deferred_addr); - let pending_provider_version = if let Some(token) = session_token { - app_state.active_users.pending_provider_version(&test_user.username, token).await - } else { - None - }; - let (flag, task) = stream_grace_period(GracePeriodParams { - app_state: Arc::clone(app_state), - stream_details, - user_grace_period: false, - user: test_user, - fingerprint: test_fingerprint, - virtual_id: VirtualId::new(1), - session_token: session_token.map(str::to_string), - provisioning_info: None, - waker: None, - hold_stream: true, - capacity_notify: app_state.connection_manager.capacity_notified(), - pending_provider_version, - grace_active_version: None, - grace_resolution_context: None, - grace_kind: None, - socket_bound: false, - shared_subscriber_id: None, - }); - ( - flag.expect("provider grace should install a mode flag"), - task.expect("provider grace should spawn a grace-resolution task"), - deferred_provider_handle, - ) - } - - fn custom_video_test_state(mode: StreamMode, provisionable: bool) -> ActiveClientStreamState { - let connection_manager = create_test_connection_manager(); - let addr = std::net::SocketAddr::from(([127, 0, 0, 1], 55001)); - - ActiveClientStreamState { - response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), - inner: None, - send_custom_stream_flag: Some(Arc::new(AtomicU8::new(mode as u8))), - provider_handle: None, - deferred_provider_open: None, - timed_stream_context: None, - preempt_cancelled: None, - grace_task_handle: None, - provisioning_stop_signal: None, - provisionable, - custom_video: CustomVideoBuffers { - user_exhausted: None, - provider_exhausted: None, - unavailable: None, - provisioning: None, - low_priority_preempted: None, - }, - meter: None, - event_manager: Arc::new(EventManager::new()), - waker: None, - connection_manager, - fingerprint: Arc::new(Fingerprint::new("fp-key".to_string(), "127.0.0.1".to_string(), addr)), - stream_uid: None, - provider_stopped: true, - user_stream_released: true, - provider_handle_released: true, - custom_video_timeout_secs: 5, - custom_video_timeout_mode: None, - custom_video_timeout_sleep: None, - direct_body_idle_timeout: DirectBodyIdleTimeout::disabled(), - provider_end_reason: AtomicU8::new(PROVIDER_END_NOT_SET), - provider_error_class: None, - provider_http_status: None, - provider_reconnect_count: AtomicU8::new(0), - lease_owner: None, - media_started: None, - lease_confirmed: false, - lease_request_id: None, - request_cleanup: None, - } - } - - async fn assert_missing_custom_video_terminates(mode: StreamMode, provisionable: bool) { - let stream = ActiveClientStream { state: custom_video_test_state(mode, provisionable) }; - pin_mut!(stream); - assert!(stream.next().await.is_none()); - } - - #[tokio::test] - async fn custom_response_modes_preserve_ts_fallback_and_terminate_hls_resources( - ) -> Result<(), Box> { - use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer; - use tuliprox_session::stream_options::StreamResponseMode; - - let mut packet = [0xff_u8; 188]; - packet[..4].copy_from_slice(&[0x47, 0x1f, 0xff, 0x10]); - let fallback = TransportStreamBuffer::new(packet.repeat(8)); - for mode in [ - StreamMode::UserExhausted, - StreamMode::ProviderExhausted, - StreamMode::ChannelUnavailable, - StreamMode::Provisioning, - StreamMode::LowPriorityPreempted, - ] { - for response_mode in [StreamResponseMode::Stream, StreamResponseMode::HlsResource] { - let mut state = custom_video_test_state(mode, true); - state.response_mode = response_mode; - state.custom_video = CustomVideoBuffers { - user_exhausted: Some(fallback.clone()), - provider_exhausted: Some(fallback.clone()), - unavailable: Some(fallback.clone()), - provisioning: Some(fallback.clone()), - low_priority_preempted: Some(fallback.clone()), - }; - let mut stream = ActiveClientStream { state }; - let chunk = tokio::time::timeout(Duration::from_secs(1), stream.next()).await?; - if response_mode == StreamResponseMode::Stream { - let bytes = chunk.ok_or("TS fallback missing")??; - assert_eq!(bytes.first(), Some(&0x47), "{mode:?}"); - } else { - assert!(chunk.is_none(), "HLS must terminate without TS bytes in {mode:?}"); - } - } - } - Ok(()) - } - - #[test] - fn test_custom_video_type_mapping_for_grace_modes() { - assert_eq!( - ActiveClientStreamState::custom_video_type_for_mode(StreamMode::UserExhausted), - Some(CustomVideoStreamType::UserConnectionsExhausted) - ); - assert_eq!( - ActiveClientStreamState::custom_video_type_for_mode(StreamMode::ProviderExhausted), - Some(CustomVideoStreamType::ProviderConnectionsExhausted) - ); - assert_eq!( - ActiveClientStreamState::custom_video_type_for_mode(StreamMode::Provisioning), - Some(CustomVideoStreamType::Provisioning) - ); - assert_eq!( - ActiveClientStreamState::custom_video_type_for_mode(StreamMode::LowPriorityPreempted), - Some(CustomVideoStreamType::LowPriorityPreempted) - ); - assert_eq!( - ActiveClientStreamState::custom_video_type_for_mode(StreamMode::ChannelUnavailable), - Some(CustomVideoStreamType::ChannelUnavailable) - ); - // A suppressed reentry must never map to a user-visible error clip. - assert_eq!(ActiveClientStreamState::custom_video_type_for_mode(StreamMode::ReentrySuppressed), None); - assert_eq!(ActiveClientStreamState::custom_video_type_for_mode(StreamMode::Inner), None); - assert_eq!(ActiveClientStreamState::custom_video_type_for_mode(StreamMode::GracePending), None); - } - - #[test] - fn test_stream_mode_byte_values_match_the_implicit_discriminants() { - // Discriminants are implicit (declaration order) except the 255 sentinel, so - // this pins the wire mapping against accidental reordering. - assert_eq!(StreamMode::Inner as u8, 0); - assert_eq!(StreamMode::UserExhausted as u8, 1); - assert_eq!(StreamMode::ProviderExhausted as u8, 2); - assert_eq!(StreamMode::ChannelUnavailable as u8, 3); - assert_eq!(StreamMode::Provisioning as u8, 4); - assert_eq!(StreamMode::LowPriorityPreempted as u8, 5); - assert_eq!(StreamMode::ReentrySuppressed as u8, 6); - assert_eq!(StreamMode::GracePending as u8, 255); - } - - #[test] - fn test_stream_mode_try_from_rejects_unknown_values() { - assert_eq!(StreamMode::try_from(0), Ok(StreamMode::Inner)); - assert_eq!(StreamMode::try_from(6), Ok(StreamMode::ReentrySuppressed)); - assert_eq!(StreamMode::try_from(255), Ok(StreamMode::GracePending)); - // Unknown values must not be silently mapped onto GracePending. - assert_eq!(StreamMode::try_from(7), Err(7)); - assert_eq!(StreamMode::try_from(200), Err(200)); - } - - #[test] - fn test_direct_body_idle_timeout_only_applies_to_direct_vod_series_streams() { - let video = create_test_video_stream_channel(1, "http://provider-1.example/movie/1.mkv"); - assert!(should_use_direct_body_idle_timeout(&video)); - - let mut series = create_test_video_stream_channel(2, "http://provider-1.example/series/2.mkv"); - series.item_type = PlaylistItemType::Series; - series.cluster = XtreamCluster::Series; - assert!(should_use_direct_body_idle_timeout(&series)); - - let live = create_test_stream_channel(3, "http://provider-1.example/live/3.ts"); - assert!(!should_use_direct_body_idle_timeout(&live)); - - let shared_video = { - let mut channel = video.clone(); - channel.shared = true; - channel - }; - assert!(!should_use_direct_body_idle_timeout(&shared_video)); - } - - #[tokio::test] - async fn test_provisioning_without_custom_video_terminates_immediately_with_timeout_configured() { - assert_missing_custom_video_terminates(StreamMode::Provisioning, true).await; - } - - #[tokio::test] - async fn test_user_exhausted_without_custom_video_terminates_immediately() { - assert_missing_custom_video_terminates(StreamMode::UserExhausted, false).await; - } - - #[tokio::test] - async fn test_provider_exhausted_without_custom_video_terminates_immediately() { - assert_missing_custom_video_terminates(StreamMode::ProviderExhausted, false).await; - } - - #[tokio::test] - async fn test_channel_unavailable_without_custom_video_terminates_immediately() { - assert_missing_custom_video_terminates(StreamMode::ChannelUnavailable, false).await; - } - - #[tokio::test] - async fn test_low_priority_preempted_without_custom_video_terminates_immediately() { - assert_missing_custom_video_terminates(StreamMode::LowPriorityPreempted, false).await; - } - - #[tokio::test(start_paused = true)] - async fn test_provider_grace_resolution_transitions_from_grace_pending_to_inner_when_capacity_notify_arrives() { - let app_state = create_test_app_state(); - let provider_name = "provider_1".intern(); - let holder_addr = "127.0.0.1:55010".parse().unwrap_or_else(|_| unreachable!()); - let deferred_addr = "127.0.0.1:55011".parse().unwrap_or_else(|_| unreachable!()); - - let holder_handle = app_state - .active_provider - .acquire_exact_connection_with_grace( - &provider_name, - &holder_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .expect("holder should consume the provider's live capacity"); - let (flag, grace_task, deferred_handle) = - start_deferred_provider_grace_resolution(&app_state, &provider_name, deferred_addr, None).await; - - assert_eq!( - StreamMode::try_from(flag.load(Ordering::Acquire)).unwrap(), - StreamMode::GracePending, - "provider grace resolution must begin in GracePending while provider capacity is exhausted" - ); - - app_state.connection_manager.release_provider_handle(Some(holder_handle)); - let join_result = tokio::time::timeout(Duration::from_millis(1), grace_task).await; - - assert!( - join_result.is_ok(), - "provider grace resolution stayed pending after capacity_notify should have fired" - ); - assert_eq!( - StreamMode::try_from(flag.load(Ordering::Acquire)).unwrap(), - StreamMode::Inner, - "capacity-notify should resolve provider grace from GracePending to Inner before the deadline" - ); - - app_state.connection_manager.release_provider_handle(Some(deferred_handle)); - } - - #[tokio::test(start_paused = true)] - async fn test_provider_grace_resolution_clears_pending_provider_on_capacity_notify() { - let app_state = create_test_app_state(); - let provider_name = "provider_1".intern(); - let holder_addr = "127.0.0.1:55024".parse().unwrap_or_else(|_| unreachable!()); - let deferred_addr = "127.0.0.1:55025".parse().unwrap_or_else(|_| unreachable!()); - let user = create_test_user("grace-user"); - - let _ = app_state - .active_users - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-grace", - virtual_id: 1, - provider: provider_name.as_ref(), - stream_url: "http://provider-1.example/live/1.ts", - addr: &deferred_addr, - connection_permission: UserConnectionPermission::GracePeriod, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - let _ = app_state - .active_users - .mark_pending_provider( - &user.username, - "tok-grace", - crate::api::model::PendingProviderReason::GraceHold, - 9_999, - ) - .await; - - let holder_handle = app_state - .active_provider - .acquire_exact_connection_with_grace( - &provider_name, - &holder_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .expect("holder should consume the provider's live capacity"); - let (_flag, grace_task, deferred_handle) = - start_deferred_provider_grace_resolution(&app_state, &provider_name, deferred_addr, Some("tok-grace")) - .await; - - app_state.connection_manager.release_provider_handle(Some(holder_handle)); - let join_result = tokio::time::timeout(Duration::from_millis(1), grace_task).await; - assert!(join_result.is_ok(), "grace task should finish after capacity notify"); - - let session = app_state - .active_users - .get_and_update_user_session(&user.username, "tok-grace") - .await - .expect("session should still exist"); - assert!( - !matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. }), - "capacity notify should clear pending provider state" - ); - assert_eq!(session.permission, UserConnectionPermission::Allowed); - - app_state.connection_manager.release_provider_handle(Some(deferred_handle)); - } - - #[tokio::test(start_paused = true)] - async fn test_provider_grace_resolution_ignores_stale_pending_version_on_capacity_notify() { - let app_state = create_test_app_state(); - let provider_name = "provider_1".intern(); - let holder_addr = "127.0.0.1:55026".parse().unwrap_or_else(|_| unreachable!()); - let deferred_addr = "127.0.0.1:55027".parse().unwrap_or_else(|_| unreachable!()); - let user = create_test_user("grace-user"); - - let _ = app_state - .active_users - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-grace-stale", - virtual_id: 1, - provider: provider_name.as_ref(), - stream_url: "http://provider-1.example/live/1.ts", - addr: &deferred_addr, - connection_permission: UserConnectionPermission::GracePeriod, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - let _ = app_state - .active_users - .mark_pending_provider( - &user.username, - "tok-grace-stale", - crate::api::model::PendingProviderReason::GraceHold, - 9_999, - ) - .await; - - let holder_handle = app_state - .active_provider - .acquire_exact_connection_with_grace( - &provider_name, - &holder_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .expect("holder should consume the provider's live capacity"); - let (_flag, grace_task, deferred_handle) = start_deferred_provider_grace_resolution( - &app_state, - &provider_name, - deferred_addr, - Some("tok-grace-stale"), - ) - .await; - - let replacement_version = app_state - .active_users - .mark_pending_provider( - &user.username, - "tok-grace-stale", - crate::api::model::PendingProviderReason::GraceHold, - 10_500, - ) - .await - .expect("replacement pending version should be created"); - - app_state.connection_manager.release_provider_handle(Some(holder_handle)); - let join_result = tokio::time::timeout(Duration::from_millis(1), grace_task).await; - assert!(join_result.is_ok(), "grace task should finish after capacity notify"); - - let session = app_state - .active_users - .get_and_update_user_session(&user.username, "tok-grace-stale") - .await - .expect("session should still exist"); - let crate::api::model::PlaybackLifecycle::PendingProvider { data: pending } = &session.lifecycle else { - panic!("stale grace task must not clear the replacement pending provider state") - }; - assert_eq!(pending.version, replacement_version); - assert!(pending.wake_source.is_none()); - assert_eq!(session.permission, UserConnectionPermission::GracePeriod); - - app_state.connection_manager.release_provider_handle(Some(deferred_handle)); - } - - #[tokio::test] - async fn deferred_provider_open_forces_identity_when_original_mode_preserves_origin() { - let app_state = create_test_app_state(); - let provider_name = "provider_1".intern(); - let addr = "127.0.0.1:55015".parse().unwrap(); - let provider_handle = app_state - .active_provider - .acquire_exact_connection_with_grace( - &provider_name, - &addr, - true, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .expect("deferred provider allocation"); - let stream_details = create_deferred_provider_grace_details( - &provider_name, - tuliprox_session::ManagedProviderHandle::new( - Arc::clone(&app_state.connection_manager.provider_manager), - provider_handle.clone(), - ), - ); - assert_eq!(stream_details.content_representation, ProviderContentRepresentationMode::PreserveOrigin); - let fingerprint = create_test_fingerprint(addr); - let mut stream_channel = create_test_stream_channel(1, "http://provider-1.example/live/1"); - stream_channel.item_type = PlaylistItemType::Catchup; - - let deferred = create_deferred_provider_open_future( - &app_state, - &stream_details, - &fingerprint, - &stream_channel, - &HeaderMap::new(), - ) - .expect("deferred provider open context"); - - let DeferredProviderOpenState::Pending(context) = deferred else { - panic!("new deferred open must start pending") - }; - assert_eq!( - context.provider_stream_factory_options.content_representation(), - ProviderContentRepresentationMode::Identity - ); - assert!(!context.provider_stream_factory_options.response_head_is_available()); - drop(context); - app_state.connection_manager.release_provider_handle(Some(provider_handle)); - } - - #[tokio::test(start_paused = true)] - async fn test_active_client_stream_deferred_provider_grace_retains_provider_handle_while_grace_pending() { - let app_state = create_test_app_state(); - let provider_name = "provider_1".intern(); - let holder_addr = "127.0.0.1:55012".parse().unwrap_or_else(|_| unreachable!()); - let deferred_addr = "127.0.0.1:55013".parse().unwrap_or_else(|_| unreachable!()); - let third_addr = "127.0.0.1:55014".parse().unwrap_or_else(|_| unreachable!()); - - let holder_handle = app_state - .active_provider - .acquire_exact_connection_with_grace( - &provider_name, - &holder_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .expect("holder should consume the provider's live capacity"); - let deferred_handle = app_state - .active_provider - .acquire_exact_connection_with_grace( - &provider_name, - &deferred_addr, - true, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .expect("deferred client should receive provider grace allocation"); - let stream_details = create_deferred_provider_grace_details( - &provider_name, - tuliprox_session::ManagedProviderHandle::new( - Arc::clone(&app_state.connection_manager.provider_manager), - deferred_handle.clone(), - ), - ); - let test_user = create_test_user("grace-user"); - let test_fingerprint = create_test_fingerprint(deferred_addr); - let stream = create_active_client_stream(ActiveClientStreamParams { - stream_details, - app_state: &app_state, - user: &test_user, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: crate::api::model::ConnectionKind::Normal, - fingerprint: &test_fingerprint, - stream_channel: create_test_stream_channel(1, "http://provider-1.example/live/1"), - socket_bound: true, - session_token: None, - req_headers: &HeaderMap::default(), - meter_uid: 0, - meter_stream: false, - }) - .await - .expect("deferred test stream admission should succeed"); - pin_mut!(stream); - - assert!( - matches!(futures::poll!(stream.next()), std::task::Poll::Pending), - "deferred active-client-stream should park in GracePending while waiting for provider grace resolution" - ); - - let third_handle = app_state.active_provider.acquire_exact_connection_with_grace( - &provider_name, - &third_addr, - true, - 0, - crate::api::model::ConnectionKind::Normal, - ); - - assert!( - third_handle.is_none(), - "deferred active-client-stream should retain the deferred provider grace reservation while GracePending" - ); - - app_state.connection_manager.release_provider_handle(Some(holder_handle)); - app_state.connection_manager.release_provider_handle(Some(deferred_handle)); - } - - #[tokio::test(start_paused = true)] - async fn test_active_client_stream_shared_deferred_provider_grace_retains_provider_handle_while_grace_pending() { - let app_state = create_test_app_state(); - let provider_name = "provider_1".intern(); - let holder_addr = "127.0.0.1:55017".parse().unwrap_or_else(|_| unreachable!()); - let deferred_addr = "127.0.0.1:55018".parse().unwrap_or_else(|_| unreachable!()); - let third_addr = "127.0.0.1:55019".parse().unwrap_or_else(|_| unreachable!()); - - let holder_handle = app_state - .active_provider - .acquire_exact_connection_with_grace( - &provider_name, - &holder_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .expect("holder should consume the provider's live capacity"); - let deferred_handle = app_state - .active_provider - .acquire_exact_connection_with_grace( - &provider_name, - &deferred_addr, - true, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .expect("deferred shared client should receive provider grace allocation"); - let stream_details = create_deferred_provider_grace_details( - &provider_name, - tuliprox_session::ManagedProviderHandle::new( - Arc::clone(&app_state.connection_manager.provider_manager), - deferred_handle.clone(), - ), - ); - let test_user = create_test_user("grace-user"); - let test_fingerprint = create_test_fingerprint(deferred_addr); - let stream = create_active_client_stream(ActiveClientStreamParams { - stream_details, - app_state: &app_state, - user: &test_user, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: crate::api::model::ConnectionKind::Normal, - fingerprint: &test_fingerprint, - stream_channel: create_test_shared_stream_channel(1, "http://provider-1.example/live/1"), - socket_bound: true, - session_token: None, - req_headers: &HeaderMap::default(), - meter_uid: 0, - meter_stream: false, - }) - .await - .expect("shared deferred test stream admission should succeed"); - pin_mut!(stream); - - assert!( - matches!(futures::poll!(stream.next()), std::task::Poll::Pending), - "shared deferred active-client-stream should stay pending instead of returning an empty stream" - ); - - let third_handle = app_state.active_provider.acquire_exact_connection_with_grace( - &provider_name, - &third_addr, - true, - 0, - crate::api::model::ConnectionKind::Normal, - ); - - assert!( - third_handle.is_none(), - "shared deferred active-client-stream should retain the deferred provider grace reservation while pending" - ); - - app_state.connection_manager.release_provider_handle(Some(holder_handle)); - app_state.connection_manager.release_provider_handle(Some(deferred_handle)); - } - - #[tokio::test(start_paused = true)] - async fn test_active_client_stream_deferred_provider_open_applies_sleep_timer_timeout() { - let app_state = create_test_app_state(); - let connection_manager = create_test_connection_manager(); - let addr = "127.0.0.1:55020".parse().unwrap_or_else(|_| unreachable!()); - let state = ActiveClientStreamState { - response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), - inner: None, - send_custom_stream_flag: Some(Arc::new(AtomicU8::new(StreamMode::Inner as u8))), - provider_handle: None, - deferred_provider_open: Some(DeferredProviderOpenState::Opening(Box::pin(async { - DeferredProviderOpenOutcome::Stream(futures::stream::pending::>().boxed()) - }))), - timed_stream_context: Some(TimedStreamContext { - app_state, - duration_secs: 1, - virtual_id: VirtualId::new(1), - }), - preempt_cancelled: None, - grace_task_handle: None, - provisioning_stop_signal: None, - provisionable: false, - custom_video: CustomVideoBuffers { - user_exhausted: None, - provider_exhausted: None, - unavailable: None, - provisioning: None, - low_priority_preempted: None, - }, - meter: None, - event_manager: Arc::new(EventManager::new()), - waker: None, - connection_manager, - fingerprint: Arc::new(Fingerprint::new("fp-timeout".to_string(), "127.0.0.1".to_string(), addr)), - stream_uid: None, - provider_stopped: false, - user_stream_released: true, - provider_handle_released: true, - custom_video_timeout_secs: 0, - custom_video_timeout_mode: None, - custom_video_timeout_sleep: None, - direct_body_idle_timeout: DirectBodyIdleTimeout::disabled(), - provider_end_reason: AtomicU8::new(PROVIDER_END_NOT_SET), - provider_error_class: None, - provider_http_status: None, - provider_reconnect_count: AtomicU8::new(0), - lease_owner: None, - media_started: None, - lease_confirmed: false, - lease_request_id: None, - request_cleanup: None, - }; - let stream = ActiveClientStream { state }; - pin_mut!(stream); - - assert!( - matches!(futures::poll!(stream.next()), std::task::Poll::Pending), - "deferred-open success should first install the wrapped upstream stream and park pending" - ); - - tokio::time::advance(Duration::from_secs(2)).await; - - let result = tokio::time::timeout(Duration::from_millis(1), stream.next()).await; - assert!(result.is_ok(), "deferred-open stream should stop once the configured sleep timer expires"); - match result { - Ok(joined) => assert!( - joined.is_none(), - "sleep timer should terminate the deferred-open stream without yielding bytes" - ), - Err(_) => unreachable!("timeout already checked"), - } - } - - #[tokio::test(start_paused = true)] - async fn test_provider_grace_resolution_transitions_from_grace_pending_to_provider_exhausted_at_deadline() { - let app_state = create_test_app_state(); - let provider_name = "provider_1".intern(); - let holder_addr = "127.0.0.1:55015".parse().unwrap_or_else(|_| unreachable!()); - let deferred_addr = "127.0.0.1:55016".parse().unwrap_or_else(|_| unreachable!()); - let user = create_test_user("grace-user"); - - let _ = app_state - .active_users - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-grace-timeout", - virtual_id: 1, - provider: provider_name.as_ref(), - stream_url: "http://provider-1.example/live/1.ts", - addr: &deferred_addr, - connection_permission: UserConnectionPermission::GracePeriod, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - let _ = app_state - .active_users - .mark_pending_provider( - &user.username, - "tok-grace-timeout", - crate::api::model::PendingProviderReason::GraceHold, - 9_999, - ) - .await; - - let holder_handle = app_state - .active_provider - .acquire_exact_connection_with_grace( - &provider_name, - &holder_addr, - false, - 0, - crate::api::model::ConnectionKind::Normal, - ) - .expect("holder should consume the provider's live capacity"); - let (flag, grace_task, deferred_handle) = start_deferred_provider_grace_resolution( - &app_state, - &provider_name, - deferred_addr, - Some("tok-grace-timeout"), - ) - .await; - - assert_eq!( - StreamMode::try_from(flag.load(Ordering::Acquire)).unwrap(), - StreamMode::GracePending, - "provider grace resolution must begin in GracePending while provider capacity is exhausted" - ); - - tokio::time::advance(Duration::from_millis(101)).await; - let task_result = grace_task.await; - - assert!( - task_result.is_ok(), - "grace-resolution task should complete once the deadline expires without capacity becoming available" - ); - assert_eq!( - StreamMode::try_from(flag.load(Ordering::Acquire)).unwrap(), - StreamMode::ProviderExhausted, - "provider grace resolution should transition from GracePending to ProviderExhausted when the deadline expires" - ); - - let session = app_state - .active_users - .get_and_update_user_session(&user.username, "tok-grace-timeout") - .await - .expect("session should still exist"); - assert!( - !matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. }), - "timeout expiry should clear pending provider state" - ); - assert_eq!(session.permission, UserConnectionPermission::Exhausted); - - app_state.connection_manager.release_provider_handle(Some(holder_handle)); - app_state.connection_manager.release_provider_handle(Some(deferred_handle)); - } - - /// Regression test: verifies that when user-grace fails and remaining strategies are - /// exhausted, the `grace_kind` (original `ConnectionKind = Soft`) flows through to - /// `evaluate_remaining_strategies_after_grace` and the session expires correctly. - /// - /// This test exercises the full `stream_grace_period` path with: - /// - `grace_kind = Some(Soft)` (passed directly from `create_active_client_stream`) - /// - `grace_resolution_context` pointing to `GraceHoldStream` (remaining slice is empty) - /// - user-grace failure (deadline expires, user still at connection limit) - /// - remaining strategies exhausted -> `expire_pending_provider` is called - #[tokio::test(start_paused = true)] - #[allow(clippy::too_many_lines)] - async fn test_user_grace_failure_preserves_soft_kind_on_exhausted() { - // Use GraceHoldStream only, remaining slice is empty after grace exhaustion. - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 100, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), - }); - - let provider_name = "provider_1".intern(); - // Two addresses: first holds the counted session, second triggers grace. - let first_addr: std::net::SocketAddr = "127.0.0.1:55201".parse().unwrap_or_else(|_| unreachable!()); - let second_addr: std::net::SocketAddr = "127.0.0.1:55202".parse().unwrap_or_else(|_| unreachable!()); - let first_fingerprint = create_test_fingerprint(first_addr); - - let mut user = create_test_user("grace-soft-user"); - user.max_connections = 1; // User has 1 hard slot; first session fills it, grace session exceeds it - user.soft_connections = 0; - - // First session: counted Normal, consumes the user's only hard slot. - app_state - .active_users - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-first", - virtual_id: 1, - provider: provider_name.as_ref(), - stream_url: "http://provider-1.example/live/1.ts", - addr: &first_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 1, - username: "grace-soft-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 10, - fingerprint: &first_fingerprint, - provider: provider_name.clone(), - stream_channel: &create_test_stream_channel(1, "http://provider-1.example/live/1.ts"), - user_agent: std::borrow::Cow::Borrowed("ua"), - session_token: Some("tok-first"), - }) - .await; - - // Second session: will be admitted via grace (user already at limit, grace is granted). - // The grace session itself is not yet counted, so user_connections returns 1 (just the - // first session). When the grace deadline expires, the first session is still present, - // user_connections (1) > max_connections (1) is NOT true. We need the first session to - // actually consume the hard slot so that the second (grace) session causes the over-limit - // check to fire when grace expires. Since max_connections=1, the first Normal session - // uses that slot, and the grace session would push to 2 > 1 — but PendingProvider sessions - // are not counted! So we need a different approach: create a second Normal session BEFORE - // the grace session so that user_connections = 2 when grace expires. - // - // Actually, the simpler path: make the first session Soft and max_connections=1. - // A Soft session IS counted (it increments connection_data.connections but uses the soft - // counter). When grace expires, user_connections = 1 (the Soft session), which is - // NOT > max_connections = 1. So user_ok = true, no failure. - // - // The correct setup: max_connections = 1, first session = Normal (uses the hard slot), - // second session = GracePeriod (PendingProvider, not counted). When grace deadline hits, - // user_connections = 1 (Normal only), max_connections = 1, so 1 > 1 is false — no failure. - // - // We need the grace session itself to trigger the over-limit check at deadline. - // But PendingProvider sessions are not counted in user_connections! - // - // So the only way for grace to fail at deadline is if there is already a DIFFERENT - // counted session taking up the slot, AND that session is still there at deadline. - // With max_connections=1 and first session Normal: when grace expires, user_conn=1, - // max=1, so 1 > 1 is false. - // - // The solution: we need TWO already-counted sessions at deadline, not one. - // But we can't create both before the grace session because the second would also be - // admitted via grace. - // - // Instead: make the first session consume the slot AND also expire it at deadline, - // so when grace expires, user_connections = 0, which is NOT > max_connections = 1. - // - // The grace session (tok-second) is in PendingProvider state and is NOT counted. - // To trigger user-grace failure, we need user_connections > max_connections at deadline. - // With max_connections=1: we need 2 counted sessions at grace deadline. - // Solution: create two Normal sessions BEFORE the grace session: - // tok-first -> Normal, counted (consumes hard slot) - // tok-preload -> Normal, counted (exceeds max, 2 > 1) - // tok-second -> GracePeriod, PendingProvider (NOT counted, grace session) - // - // At grace deadline: user_connections = 2 (tok-first + tok-preload), max = 1. - // 2 > 1 -> user_ok = false -> user-grace failure path entered. - let preload_addr: std::net::SocketAddr = "127.0.0.1:55203".parse().unwrap_or_else(|_| unreachable!()); - let preload_fingerprint = create_test_fingerprint(preload_addr); - - app_state - .active_users - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-preload", - virtual_id: 3, - provider: provider_name.as_ref(), - stream_url: "http://provider-1.example/live/3.ts", - addr: &preload_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 3, - username: "grace-soft-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 10, - fingerprint: &preload_fingerprint, - provider: provider_name.clone(), - stream_channel: &create_test_stream_channel(3, "http://provider-1.example/live/3.ts"), - user_agent: std::borrow::Cow::Borrowed("ua"), - session_token: Some("tok-preload"), - }) - .await; - - let second_fingerprint = create_test_fingerprint(second_addr); - - app_state - .active_users - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-second", - virtual_id: 2, - provider: provider_name.as_ref(), - stream_url: "http://provider-1.example/live/2.ts", - addr: &second_addr, - connection_permission: UserConnectionPermission::GracePeriod, - connection_kind: Some(crate::api::model::ConnectionKind::Soft), - socket_bound: false, - }) - .await; - - let _pending_version = app_state - .active_users - .mark_pending_provider( - "grace-soft-user", - "tok-second", - crate::api::model::PendingProviderReason::GraceHold, - 9_999, - ) - .await - .expect("pending version must be created for tok-second"); - - let grace_context = GraceResolutionContext { - strategy_index: 0, - strategies: [AdmissionStrategy::GraceHoldStream].into(), - kind: Some(crate::api::model::ConnectionKind::Soft), - }; - - let pending_version = app_state - .active_users - .pending_provider_version("grace-soft-user", "tok-second") - .await - .expect("pending version must be created for tok-second"); - - let stream_details = StreamDetails { - shared_subscriber_id: None, - stream: None, - stream_info: None, - provider_name: Some(provider_name), - request_url: Some("http://provider-1.example/live/2.ts".intern()), - session_headers: None, - provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), - user_agent_stream_index: None, - grace_period: GracePeriodOptions { period_millis: 100, timeout_secs: 0, hold_stream: true }, - provider_grace_active: false, - disable_provider_grace: false, - reconnect_flag: None, - provider_handle: None, - content_representation: ProviderContentRepresentationMode::PreserveOrigin, - grace_resolution_context: Some(grace_context.clone()), - custom_reason: None, - response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), - session_registration: None, - }; - - let (flag, grace_task) = stream_grace_period(GracePeriodParams { - app_state: Arc::clone(&app_state), - stream_details, - user_grace_period: true, - user: user.clone(), - fingerprint: second_fingerprint.clone(), - virtual_id: VirtualId::new(2), - session_token: Some("tok-second".to_string()), - provisioning_info: None, - waker: None, - hold_stream: true, - capacity_notify: app_state.connection_manager.capacity_notified(), - pending_provider_version: Some(pending_version), - grace_active_version: None, - grace_resolution_context: Some(grace_context), - grace_kind: Some(crate::api::model::ConnectionKind::Soft), - socket_bound: false, - shared_subscriber_id: None, - }); - - // Grace should be pending initially. - assert_eq!( - StreamMode::try_from(flag.as_ref().unwrap().load(Ordering::Acquire)).unwrap(), - StreamMode::GracePending, - "user grace should start in GracePending" - ); - - // Advance time past the grace deadline. - tokio::time::advance(Duration::from_millis(101)).await; - let _ = grace_task.expect("grace task should be spawned").await; - - // Remaining strategies are exhausted (only GraceHoldStream was configured, no eviction). - // The session should expire with UserExhausted. - assert_eq!( - StreamMode::try_from(flag.as_ref().unwrap().load(Ordering::Acquire)).unwrap(), - StreamMode::UserExhausted, - "exhausted remaining strategies should result in UserExhausted" - ); - - let session = app_state - .active_users - .get_and_update_user_session("grace-soft-user", "tok-second") - .await - .expect("session must exist after grace failure"); - - // Permission should be Exhausted (not GracePeriod). - assert_eq!( - session.permission, - UserConnectionPermission::Exhausted, - "session permission should be Exhausted after grace failure with exhausted strategies" - ); - - // Lifecycle should be Expired. - assert!( - matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::Expired), - "session lifecycle should be Expired after grace failure" - ); - - // The session's original connection_kind is Soft and must NOT be modified by the - // grace failure path. The grace_kind = Soft that was passed to - // evaluate_remaining_strategies_after_grace is verified by the fact that - // the session's kind remained Soft (it was set at creation time and is preserved - // through the grace failure flow since session.connection_kind is not changed). - assert_eq!( - session.connection_kind, - Some(crate::api::model::ConnectionKind::Soft), - "session connection_kind should remain Soft (unchanged from creation)" - ); - } - - #[tokio::test(start_paused = true)] - #[allow(clippy::too_many_lines)] - async fn test_user_grace_failure_reentry_suppression_terminates_quietly() { - // GraceHoldStream admits the request; the remaining EvictUserOldest candidate - // is reentry-protected, so the grace task must resolve to ReentrySuppressed - // instead of painting the user-visible connections-exhausted video. - let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { - retry: true, - metrics_enabled: true, - buffer: None, - grace_period_millis: 100, - grace_period_timeout_secs: 8, - grace_period_hold_stream: true, - hls_session_ttl_secs: 10, - catchup_session_ttl_secs: 10, - provider_affinity_ttl_secs: 120, - hls_wrap_media_playlist: true, - throttle_str: None, - throttle_kbps: 0, - shared_burst_buffer_mb: 1, - shared_subscriber_idle_timeout_secs: 300, - cleanup_queue_capacity: 4096, - recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), - admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]), - }); - - let provider_name = "provider_1".intern(); - let first_addr: std::net::SocketAddr = "127.0.0.1:55211".parse().unwrap_or_else(|_| unreachable!()); - let second_addr: std::net::SocketAddr = "127.0.0.1:55212".parse().unwrap_or_else(|_| unreachable!()); - let preload_addr: std::net::SocketAddr = "127.0.0.1:55213".parse().unwrap_or_else(|_| unreachable!()); - let first_fingerprint = create_test_fingerprint(first_addr); - let second_fingerprint = create_test_fingerprint(second_addr); - let preload_fingerprint = create_test_fingerprint(preload_addr); - - let mut user = create_test_user("grace-reentry-user"); - user.max_connections = 1; - user.soft_connections = 0; - - // Two counted Normal streams make the user over limit when grace expires. - app_state - .active_users - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-first", - virtual_id: 1, - provider: provider_name.as_ref(), - stream_url: "http://provider-1.example/live/1.ts", - addr: &first_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 1, - username: "grace-reentry-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 10, - fingerprint: &first_fingerprint, - provider: provider_name.clone(), - stream_channel: &create_test_stream_channel(1, "http://provider-1.example/live/1.ts"), - user_agent: std::borrow::Cow::Borrowed("ua"), - session_token: Some("tok-first"), - }) - .await; - app_state - .active_users - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-preload", - virtual_id: 3, - provider: provider_name.as_ref(), - stream_url: "http://provider-1.example/live/3.ts", - addr: &preload_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::api::model::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 3, - username: "grace-reentry-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 10, - fingerprint: &preload_fingerprint, - provider: provider_name.clone(), - stream_channel: &create_test_stream_channel(3, "http://provider-1.example/live/3.ts"), - user_agent: std::borrow::Cow::Borrowed("ua"), - session_token: Some("tok-preload"), - }) - .await; - - // Grace session on a distinct channel; its socket-bound stream row is what - // `mark_recent_eviction_guard_for_addr` keys the reentry guard on. - app_state - .active_users - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-second", - virtual_id: 2, - provider: provider_name.as_ref(), - stream_url: "http://provider-1.example/live/2.ts", - addr: &second_addr, - connection_permission: UserConnectionPermission::GracePeriod, - connection_kind: Some(crate::api::model::ConnectionKind::Soft), - socket_bound: false, - }) - .await; - app_state - .connection_manager - .update_connection(crate::api::model::ConnectionParams { - meter_uid: 2, - username: "grace-reentry-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::api::model::ConnectionKind::Normal, - priority: 0, - soft_priority: 10, - fingerprint: &second_fingerprint, - provider: provider_name.clone(), - stream_channel: &create_test_stream_channel(2, "http://provider-1.example/live/2.ts"), - user_agent: std::borrow::Cow::Borrowed("ua"), - session_token: Some("tok-second"), - }) - .await; - - // Protect the oldest candidate (channel 1) against a retry of this channel 2. - app_state - .active_users - .mark_recent_eviction_guard_for_addr(&second_addr, first_addr, Duration::from_secs(3)) - .await; - - let _pending_version = app_state - .active_users - .mark_pending_provider( - "grace-reentry-user", - "tok-second", - crate::api::model::PendingProviderReason::GraceHold, - 9_999, - ) - .await - .expect("pending version must be created for tok-second"); - - let grace_context = GraceResolutionContext { - strategy_index: 0, - strategies: [AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest].into(), - kind: Some(crate::api::model::ConnectionKind::Soft), - }; - let pending_version = app_state - .active_users - .pending_provider_version("grace-reentry-user", "tok-second") - .await - .expect("pending version must be created for tok-second"); - - let stream_details = StreamDetails { - shared_subscriber_id: None, - stream: None, - stream_info: None, - provider_name: Some(provider_name), - request_url: Some("http://provider-1.example/live/2.ts".intern()), - session_headers: None, - provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), - user_agent_stream_index: None, - grace_period: GracePeriodOptions { period_millis: 100, timeout_secs: 0, hold_stream: true }, - provider_grace_active: false, - disable_provider_grace: false, - reconnect_flag: None, - provider_handle: None, - content_representation: ProviderContentRepresentationMode::PreserveOrigin, - grace_resolution_context: Some(grace_context.clone()), - custom_reason: None, - response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), - session_registration: None, - }; - - let (flag, grace_task) = stream_grace_period(GracePeriodParams { - app_state: Arc::clone(&app_state), - stream_details, - user_grace_period: true, - user: user.clone(), - fingerprint: second_fingerprint.clone(), - virtual_id: VirtualId::new(2), - session_token: Some("tok-second".to_string()), - provisioning_info: None, - waker: None, - hold_stream: true, - capacity_notify: app_state.connection_manager.capacity_notified(), - pending_provider_version: Some(pending_version), - grace_active_version: None, - grace_resolution_context: Some(grace_context), - grace_kind: Some(crate::api::model::ConnectionKind::Soft), - socket_bound: true, - shared_subscriber_id: None, - }); - - assert_eq!( - StreamMode::try_from(flag.as_ref().unwrap().load(Ordering::Acquire)).unwrap(), - StreamMode::GracePending, - "user grace should start in GracePending" - ); - - tokio::time::advance(Duration::from_millis(101)).await; - let _ = grace_task.expect("grace task should be spawned").await; - - assert_eq!( - StreamMode::try_from(flag.as_ref().unwrap().load(Ordering::Acquire)).unwrap(), - StreamMode::ReentrySuppressed, - "reentry-protected remaining candidate must resolve to quiet termination" - ); - - let session = app_state - .active_users - .get_and_update_user_session("grace-reentry-user", "tok-second") - .await - .expect("session must exist after grace failure"); - assert!( - matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::Expired), - "session lifecycle should be Expired after reentry-suppressed grace failure" - ); - } - - #[tokio::test(start_paused = true)] - async fn test_active_client_stream_immediate_provider_stream_emits_meter_batches() { - let app_state = create_test_app_state(); - let mut meter_events = app_state.event_manager.get_meter_channel(); - app_state.event_manager.stream_meter_subscriber_connected(); - - let addr = "127.0.0.1:55030".parse().unwrap_or_else(|_| unreachable!()); - let test_user = create_test_user("meter-user"); - let test_fingerprint = create_test_fingerprint(addr); - let provider_stream = futures::stream::iter(vec![Ok::(Bytes::from_static(&[0_u8; 3072]))]) - .chain(futures::stream::pending()) - .boxed(); - let stream_details = StreamDetails::from_stream(provider_stream, GracePeriodOptions::default()); - - let stream = create_active_client_stream(ActiveClientStreamParams { - stream_details, - app_state: &app_state, - user: &test_user, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: crate::api::model::ConnectionKind::Normal, - fingerprint: &test_fingerprint, - stream_channel: create_test_stream_channel(1, "http://provider-1.example/live/1"), - socket_bound: true, - session_token: None, - req_headers: &HeaderMap::default(), - meter_uid: 55, - meter_stream: true, - }) - .await - .expect("metered test stream admission should succeed"); - pin_mut!(stream); - - let first_chunk = stream.next().await; - assert!( - matches!(first_chunk, Some(Ok(ref bytes)) if bytes.len() == 3072), - "immediate provider stream should yield the metered payload chunk" - ); - - tokio::time::advance(Duration::from_secs(3)).await; - tokio::task::yield_now().await; - - let entries = tokio::time::timeout(Duration::from_millis(1), meter_events.recv()) - .await - .expect("immediate provider stream should publish a meter batch after bytes are sent") - .expect("meter channel should stay open while the stream is active"); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].meter_uid, 55); - assert_eq!(entries[0].uids, vec![1]); - assert_eq!(entries[0].rate_kbps, 1); - assert_eq!(entries[0].total_kb, 3); - } - - #[tokio::test] - async fn direct_series_normal_eof_releases_full_lifecycle() { - let app_state = create_test_app_state(); - let addr = "127.0.0.1:55031".parse().unwrap_or_else(|_| unreachable!()); - let provider_handle = acquire_test_provider_handle(&app_state, addr).await; - let (provider_stream, tracker) = - track_provider_stream(futures::stream::once(async { Ok(Bytes::from_static(b"series-eof")) }).boxed()); - let direct = create_test_active_direct_stream(TestDirectStreamParams { - app_state: &app_state, - username: "series-eof-user", - max_connections: 1, - addr, - stream_channel: create_test_series_stream_channel(1, "http://provider-1.example/series/1.mkv"), - provider_stream, - provider_handle: Some(provider_handle), - }) - .await; - let active_uids = [direct.uid]; - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "registered Series EOF stream", - active_counts: (1, 1), - stream_uids: &active_uids, - provider_connections: 1, - dropped_streams: &[], - }, - ) - .await; - assert!(!tracker.is_dropped()); - - let body = Body::from_stream(direct.stream); - let collected = body.collect().await.expect("normal provider EOF should complete the response body"); - assert_eq!(collected.to_bytes(), Bytes::from_static(b"series-eof")); - - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "normal Series EOF cleanup", - active_counts: (0, 0), - stream_uids: &[], - provider_connections: 0, - dropped_streams: &[&tracker], - }, - ) - .await; - } - - #[tokio::test] - async fn direct_series_upstream_error_after_registration_releases_full_lifecycle() { - let app_state = create_test_app_state(); - let addr = "127.0.0.1:55032".parse().unwrap_or_else(|_| unreachable!()); - let provider_handle = acquire_test_provider_handle(&app_state, addr).await; - let provider_items = vec![ - Ok(Bytes::from_static(b"before-error")), - Err(StreamError::Stream("controlled upstream failure".to_string())), - ]; - let (provider_stream, tracker) = track_provider_stream(futures::stream::iter(provider_items).boxed()); - let direct = create_test_active_direct_stream(TestDirectStreamParams { - app_state: &app_state, - username: "series-error-user", - max_connections: 1, - addr, - stream_channel: create_test_series_stream_channel(2, "http://provider-1.example/series/2.mkv"), - provider_stream, - provider_handle: Some(provider_handle), - }) - .await; - let active_uids = [direct.uid]; - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "registered Series error stream", - active_counts: (1, 1), - stream_uids: &active_uids, - provider_connections: 1, - dropped_streams: &[], - }, - ) - .await; - - let body = Body::from_stream(direct.stream); - let collected = body - .collect() - .await - .expect("ActiveClientStream should convert the controlled upstream error into stream termination"); - assert_eq!(collected.to_bytes(), Bytes::from_static(b"before-error")); - - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "Series upstream error cleanup", - active_counts: (0, 0), - stream_uids: &[], - provider_connections: 0, - dropped_streams: &[&tracker], - }, - ) - .await; - } - - #[tokio::test] - async fn closed_buffered_consumer_releases_direct_series_full_lifecycle() { - let app_state = create_test_app_state(); - let addr = "127.0.0.1:55039".parse().unwrap_or_else(|_| unreachable!()); - let provider_handle = acquire_test_provider_handle(&app_state, addr).await; - let (gate_tx, gate_rx) = oneshot::channel(); - let gated_provider = futures::stream::once(async move { - gate_rx.await.expect("test should release the upstream chunk"); - Ok(Bytes::from_static(b"after-consumer-drop")) - }) - .chain(futures::stream::pending()) - .boxed(); - let (tracked_provider, tracker) = track_provider_stream(gated_provider); - let producer_cancel = CancellationToken::new(); - let buffered_provider = - BufferedStream::new(tracked_provider, 1, 0, producer_cancel.clone(), "controlled-test-stream").boxed(); - let direct = create_test_active_direct_stream(TestDirectStreamParams { - app_state: &app_state, - username: "series-closed-consumer-user", - max_connections: 1, - addr, - stream_channel: create_test_series_stream_channel(10, "http://provider-1.example/series/10.mkv"), - provider_stream: buffered_provider, - provider_handle: Some(provider_handle), - }) - .await; - let active_uids = [direct.uid]; - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "registered buffered Series stream", - active_counts: (1, 1), - stream_uids: &active_uids, - provider_connections: 1, - dropped_streams: &[], - }, - ) - .await; - - drop(Body::from_stream(direct.stream)); - gate_tx.send(()).expect("buffered producer should still own the gated upstream"); - - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "closed buffered Series consumer cleanup", - active_counts: (0, 0), - stream_uids: &[], - provider_connections: 0, - dropped_streams: &[&tracker], - }, - ) - .await; - assert!(producer_cancel.is_cancelled(), "closed consumer must cancel the buffered producer"); - } - - #[tokio::test] - async fn aborting_task_polling_direct_series_body_releases_full_lifecycle() { - let app_state = create_test_app_state(); - let addr = "127.0.0.1:55033".parse().unwrap_or_else(|_| unreachable!()); - let provider_handle = acquire_test_provider_handle(&app_state, addr).await; - let (provider_stream, tracker) = - track_provider_stream(futures::stream::pending::>().boxed()); - let direct = create_test_active_direct_stream(TestDirectStreamParams { - app_state: &app_state, - username: "series-abort-user", - max_connections: 1, - addr, - stream_channel: create_test_series_stream_channel(3, "http://provider-1.example/series/3.mkv"), - provider_stream, - provider_handle: Some(provider_handle), - }) - .await; - let active_uids = [direct.uid]; - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "registered Series task-abort stream", - active_counts: (1, 1), - stream_uids: &active_uids, - provider_connections: 1, - dropped_streams: &[], - }, - ) - .await; - - let started = Arc::new(Notify::new()); - let started_in_task = Arc::clone(&started); - let task = tokio::spawn(async move { - let mut body = Body::from_stream(direct.stream); - started_in_task.notify_one(); - let _ = body.frame().await; - }); - tokio::time::timeout(Duration::from_secs(1), started.notified()).await.expect("body polling task should start"); - task.abort(); - let join_error = tokio::time::timeout(Duration::from_secs(1), task) - .await - .expect("aborted body polling task should finish") - .expect_err("pending body polling task should be cancelled"); - assert!(join_error.is_cancelled()); - - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "aborted Series body task cleanup", - active_counts: (0, 0), - stream_uids: &[], - provider_connections: 0, - dropped_streams: &[&tracker], - }, - ) - .await; - } - - #[tokio::test] - async fn response_builder_error_after_registration_drops_direct_series_body() { - let app_state = create_test_app_state(); - let addr = "127.0.0.1:55034".parse().unwrap_or_else(|_| unreachable!()); - let provider_handle = acquire_test_provider_handle(&app_state, addr).await; - let (provider_stream, tracker) = - track_provider_stream(futures::stream::pending::>().boxed()); - let direct = create_test_active_direct_stream(TestDirectStreamParams { - app_state: &app_state, - username: "series-builder-error-user", - max_connections: 1, - addr, - stream_channel: create_test_series_stream_channel(4, "http://provider-1.example/series/4.mkv"), - provider_stream, - provider_handle: Some(provider_handle), - }) - .await; - let active_uids = [direct.uid]; - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "registered Series response-builder stream", - active_counts: (1, 1), - stream_uids: &active_uids, - provider_connections: 1, - dropped_streams: &[], - }, - ) - .await; - - let response = axum::response::Response::builder().status(10_000_u16).body(Body::from_stream(direct.stream)); - assert!(response.is_err(), "invalid status should fail response construction"); - - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "response-builder error cleanup", - active_counts: (0, 0), - stream_uids: &[], - provider_connections: 0, - dropped_streams: &[&tracker], - }, - ) - .await; - } - - #[tokio::test(start_paused = true)] - async fn test_direct_vod_body_idle_timeout_releases_active_stream() { - let app_state = create_test_app_state(); - let addr = "127.0.0.1:55035".parse().unwrap_or_else(|_| unreachable!()); - let provider_handle = acquire_test_provider_handle(&app_state, addr).await; - let pending_provider = - futures::stream::once(async { Ok(Bytes::from_static(b"vod")) }).chain(futures::stream::pending()).boxed(); - let (provider_stream, tracker) = track_provider_stream(pending_provider); - let direct = create_test_active_direct_stream(TestDirectStreamParams { - app_state: &app_state, - username: "vod-user", - max_connections: 1, - addr, - stream_channel: create_test_video_stream_channel(5, "http://provider-1.example/movie/5.mkv"), - provider_stream, - provider_handle: Some(provider_handle), - }) - .await; - let active_uids = [direct.uid]; - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "registered idle-timeout VOD stream", - active_counts: (1, 1), - stream_uids: &active_uids, - provider_connections: 1, - dropped_streams: &[], - }, - ) - .await; - let stream = direct.stream; - pin_mut!(stream); - - let first_chunk = stream.next().await; - assert!(matches!(first_chunk, Some(Ok(ref bytes)) if bytes.as_ref() == b"vod")); - assert!( - matches!(futures::poll!(stream.next()), std::task::Poll::Pending), - "pending VOD body should wait until the direct body idle timeout elapses" - ); - - tokio::time::advance(Duration::from_secs(DIRECT_BODY_IDLE_TIMEOUT_SECS)).await; - tokio::task::yield_now().await; - assert!(stream.next().await.is_none(), "VOD body idle timeout should terminate the stream"); - tokio::time::resume(); - - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "VOD body idle-timeout cleanup", - active_counts: (0, 0), - stream_uids: &[], - provider_connections: 0, - dropped_streams: &[&tracker], - }, - ) - .await; - } - - #[tokio::test] - async fn dropping_direct_series_body_releases_original_user_after_socket_owner_changes() { - let app_state = create_test_app_state(); - let addr = "127.0.0.1:55036".parse().unwrap_or_else(|_| unreachable!()); - let provider_handle = acquire_test_provider_handle(&app_state, addr).await; - let (first_provider_stream, first_tracker) = - track_provider_stream(futures::stream::pending::>().boxed()); - let first = create_test_active_direct_stream(TestDirectStreamParams { - app_state: &app_state, - username: "series-user-a", - max_connections: 1, - addr, - stream_channel: create_test_series_stream_channel(6, "http://provider-1.example/series/6.mkv"), - provider_stream: first_provider_stream, - provider_handle: Some(provider_handle), - }) - .await; - let (second_provider_stream, second_tracker) = - track_provider_stream(futures::stream::pending::>().boxed()); - let second = create_test_active_direct_stream(TestDirectStreamParams { - app_state: &app_state, - username: "series-user-b", - max_connections: 1, - addr, - stream_channel: create_test_series_stream_channel(7, "http://provider-1.example/series/7.mkv"), - provider_stream: second_provider_stream, - provider_handle: None, - }) - .await; - let both_uids = [first.uid, second.uid]; - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "two users sharing a reused socket address", - active_counts: (2, 2), - stream_uids: &both_uids, - provider_connections: 1, - dropped_streams: &[], - }, - ) - .await; - - let first_body = Body::from_stream(first.stream); - let second_body = Body::from_stream(second.stream); - drop(first_body); - let second_uid = [second.uid]; - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "original user Body drop after socket owner replacement", - active_counts: (1, 1), - stream_uids: &second_uid, - provider_connections: 0, - dropped_streams: &[&first_tracker], - }, - ) - .await; - assert!(!second_tracker.is_dropped()); - - drop(second_body); - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "replacement user Body drop cleanup", - active_counts: (0, 0), - stream_uids: &[], - provider_connections: 0, - dropped_streams: &[&first_tracker, &second_tracker], - }, - ) - .await; - } - - #[tokio::test] - async fn multiple_same_user_direct_series_connections_release_independently() { - let app_state = create_test_app_state(); - let first_addr = "127.0.0.1:55037".parse().unwrap_or_else(|_| unreachable!()); - let second_addr = "127.0.0.1:55038".parse().unwrap_or_else(|_| unreachable!()); - let (first_provider_stream, first_tracker) = - track_provider_stream(futures::stream::pending::>().boxed()); - let first = create_test_active_direct_stream(TestDirectStreamParams { - app_state: &app_state, - username: "multi-series-user", - max_connections: 2, - addr: first_addr, - stream_channel: create_test_series_stream_channel(8, "http://provider-1.example/series/8.mkv"), - provider_stream: first_provider_stream, - provider_handle: None, - }) - .await; - let (second_provider_stream, second_tracker) = - track_provider_stream(futures::stream::pending::>().boxed()); - let second = create_test_active_direct_stream(TestDirectStreamParams { - app_state: &app_state, - username: "multi-series-user", - max_connections: 2, - addr: second_addr, - stream_channel: create_test_series_stream_channel(9, "http://provider-1.example/series/9.mkv"), - provider_stream: second_provider_stream, - provider_handle: None, - }) - .await; - let both_uids = [first.uid, second.uid]; - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "two connections for one Series user", - active_counts: (1, 2), - stream_uids: &both_uids, - provider_connections: 0, - dropped_streams: &[], - }, - ) - .await; - - let first_body = Body::from_stream(first.stream); - let second_body = Body::from_stream(second.stream); - drop(first_body); - let second_uid = [second.uid]; - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "first same-user Series connection cleanup", - active_counts: (1, 1), - stream_uids: &second_uid, - provider_connections: 0, - dropped_streams: &[&first_tracker], - }, - ) - .await; - assert!(!second_tracker.is_dropped()); - - drop(second_body); - wait_for_lifecycle( - &app_state, - ExpectedLifecycle { - description: "last same-user Series connection cleanup", - active_counts: (0, 0), - stream_uids: &[], - provider_connections: 0, - dropped_streams: &[&first_tracker, &second_tracker], - }, - ) - .await; - } -} +mod tests; + +mod admission; +mod error; +mod grace; +mod provider_open; +mod state; +mod stream; + +pub(crate) use self::{admission::create_active_client_stream, error::StreamAdmissionError}; +use self::{ + admission::{ActiveClientStreamState, GracePeriodParams}, + grace::{resolve_grace_period_provisioning, stream_grace_period}, + provider_open::{create_deferred_provider_open_future, DeferredProviderOpenOutcome, DeferredProviderOpenState}, + state::{ + should_use_direct_body_idle_timeout, store_stream_mode, CustomVideoBuffers, DirectBodyIdleTimeout, + GraceProvisioningInfo, StreamMode, TimedStreamContext, + }, + stream::{create_timed_stream_context, wrap_timed_client_stream_if_needed}, +}; diff --git a/backend/app/src/api/model/streams/active_client_stream/admission.rs b/backend/app/src/api/model/streams/active_client_stream/admission.rs new file mode 100644 index 000000000..bfc78ac0f --- /dev/null +++ b/backend/app/src/api/model/streams/active_client_stream/admission.rs @@ -0,0 +1,403 @@ +use super::{ + create_deferred_provider_open_future, create_timed_stream_context, resolve_grace_period_provisioning, + should_use_direct_body_idle_timeout, stream_grace_period, wrap_timed_client_stream_if_needed, ActiveClientStream, + ActiveClientStreamParams, CustomVideoBuffers, DeferredProviderOpenState, DirectBodyIdleTimeout, + GraceProvisioningInfo, StreamAdmissionError, StreamMode, TimedStreamContext, +}; +use crate::{ + api::model::{ + connection_manager::PROVIDER_END_NOT_SET, AppState, BoxedProviderStream, ConnectionManager, EventManager, + MeteringStream, StreamDetails, StreamMeterHandle, + }, + auth::Fingerprint, + model::ProxyUserCredentials, +}; +use axum::http::header::USER_AGENT; +use futures::{task::AtomicWaker, StreamExt}; +use log::error; +use shared::{ + model::{UserConnectionPermission, VirtualId}, + utils::Internable, +}; +use std::{ + pin::Pin, + sync::{ + atomic::{AtomicBool, AtomicU8}, + Arc, + }, +}; +use tokio::sync::Notify; +use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; +use tuliprox_session::{stream_options::StreamResponseMode, ConnectionRejectionReason}; + +pub(super) struct GracePeriodParams { + pub(super) app_state: Arc, + pub(super) stream_details: StreamDetails, + pub(super) user_grace_period: bool, + pub(super) user: ProxyUserCredentials, + pub(super) fingerprint: Fingerprint, + pub(super) virtual_id: VirtualId, + pub(super) session_token: Option, + pub(super) provisioning_info: Option, + pub(super) waker: Option>, + pub(super) hold_stream: bool, + pub(super) capacity_notify: Arc, + pub(super) pending_provider_version: Option, + // The `transition_version` of the session if it is in `GraceActive` lifecycle. + // Used by the grace task to confirm the session is still in `GraceActive` before resolving. + pub(super) grace_active_version: Option, + // Set when the stream was admitted via a user-grace strategy. + // On user-grace failure, remaining strategies are evaluated before final deny. + pub(super) grace_resolution_context: Option, + // The `ConnectionKind` from the original admission decision. + // Preserved in `GraceResolutionContext.kind` and also passed directly here + // so the grace task can use the runtime value when evaluating remaining strategies. + pub(super) grace_kind: Option, + // Whether the session is `socket_bound`. Used to construct the correct + // `EvictionReentryGuard`. + pub(super) socket_bound: bool, + pub(super) shared_subscriber_id: Option, +} + +#[allow(clippy::struct_excessive_bools)] +pub(super) struct ActiveClientStreamState { + pub(super) response_mode: StreamResponseMode, + pub(super) inner: Option, + pub(super) send_custom_stream_flag: Option>, + pub(super) provider_handle: Option, + pub(super) deferred_provider_open: Option, + pub(super) timed_stream_context: Option, + pub(super) preempt_cancelled: Option>>, + pub(super) grace_task_handle: Option>, + /// Cancels a panel-api provisioning probe spawned by the grace task; aborting the + /// grace task alone would leave the probe running to its own timeout. + pub(super) provisioning_stop_signal: Option, + pub(super) provisionable: bool, + pub(super) custom_video: CustomVideoBuffers, + pub(super) meter: Option>, + pub(super) event_manager: Arc, + pub(super) waker: Option>, + pub(super) connection_manager: Arc, + pub(super) fingerprint: Arc, + pub(super) stream_uid: Option, + pub(super) provider_stopped: bool, + pub(super) user_stream_released: bool, + /// Mirrors `user_stream_released` for the provider handle to guard against double-release + /// when preemption and Drop race. + pub(super) provider_handle_released: bool, + pub(super) custom_video_timeout_secs: u32, + pub(super) custom_video_timeout_mode: Option, + pub(super) custom_video_timeout_sleep: Option>>, + pub(super) direct_body_idle_timeout: DirectBodyIdleTimeout, + /// Set once when the provider stream ends. Read once in Drop. Never queried during streaming. + /// Separate from `send_custom_stream_flag` (`StreamMode`). Uses `PROVIDER_END_*` constants. + pub(super) provider_end_reason: AtomicU8, + pub(super) provider_error_class: Option<&'static str>, + pub(super) provider_http_status: Option, + /// Count of successful provider reconnections during this session (grace period / deferred open). + pub(super) provider_reconnect_count: AtomicU8, + /// Playback lease owner (session token) whose provider slot is confirmed once real + /// media bytes reach the client. `None` when this stream has no provider lease. + pub(super) lease_owner: Option>, + pub(super) media_started: Option>, + /// Guards against emitting the confirmation more than once per stream. + pub(super) lease_confirmed: bool, + pub(super) lease_request_id: Option, + /// Guaranteed final cleanup for the registered request claim. The body owns this and + /// finishes it exactly once with the real provider outcome, so a full cleanup queue can + /// never leak the user request or provider lease. + pub(super) request_cleanup: Option, +} + +#[allow(clippy::too_many_lines)] +pub(crate) async fn create_active_client_stream( + request: ActiveClientStreamParams<'_>, +) -> Result { + let ActiveClientStreamParams { + mut stream_details, + app_state, + user, + connection_permission, + connection_kind, + fingerprint, + stream_channel, + socket_bound, + session_token, + req_headers, + meter_uid, + meter_stream, + } = request; + if connection_permission == UserConnectionPermission::Exhausted { + error!("Something is wrong this should not happen"); + } + let response_mode = stream_details.response_mode; + let grant_user_grace_period = connection_permission == UserConnectionPermission::GracePeriod; + let username = user.username.as_str(); + let provider_name = stream_details.provider_name.clone().unwrap_or_else(|| "unknown".intern()); + + let user_agent = req_headers.get(USER_AGENT).map(|h| String::from_utf8_lossy(h.as_bytes())).unwrap_or_default(); + + let virtual_id = stream_channel.virtual_id; + let is_shared_source_stream = stream_channel.shared && stream_details.stream.is_some(); + let direct_body_idle_timeout = if should_use_direct_body_idle_timeout(&stream_channel) { + DirectBodyIdleTimeout::enabled() + } else { + DirectBodyIdleTimeout::disabled() + }; + // A shared response already owns a guaranteed ReleaseSharedSubscriber permit + // before this registration starts. That terminal event also removes this exact user request, so reserving + // another permit from the same bounded queue would create a circular admission + // dependency under saturation. + let shared_cleanup_capability = stream_details.shared_subscriber_id.take(); + let shared_subscriber_id = + shared_cleanup_capability.as_ref().map(tuliprox_session::SharedCleanupCapability::subscriber_id); + let request_uid = shared_subscriber_id.map_or_else( + || app_state.connection_manager.next_stream_uid(), + tuliprox_core::model::SharedSubscriberId::stream_uid, + ); + let provider_request_id = stream_details + .provider_handle + .as_ref() + .and_then(|managed| managed.handle()) + .and_then(|handle| handle.playback_request_id); + let connection = crate::api::model::ConnectionParams { + meter_uid, + username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind, + priority: user.priority, + soft_priority: user.soft_priority, + fingerprint, + provider: provider_name, + stream_channel: &stream_channel, + user_agent, + session_token, + }; + let mut registered_request = if let Some(capability) = shared_cleanup_capability.filter(|_| stream_channel.shared) { + app_state + .connection_manager + .update_connection_with_uid_using_shared_cleanup( + connection, + tuliprox_session::ConnectionHistoryMode::EmitConnect, + capability, + provider_request_id, + ) + .await + } else { + app_state + .connection_manager + .update_connection_with_uid_and_session_registration( + connection, + tuliprox_session::ConnectionHistoryMode::EmitConnect, + request_uid, + provider_request_id, + stream_details.session_registration.as_ref(), + ) + .await + }; + let request_uid = registered_request.request_uid; + let display_stream_uid = registered_request.display_uid(); + let stream_uid = Some(request_uid); + if registered_request.display_stream.is_none() { + // Admission was rejected (closed cleanup receiver, bounded timeout, or failed + // registration). No provider bytes may be delivered; the acquired provider + // handle and any open provider stream are released when `request` (and its + // `stream_details`) is dropped on return. + let reason = registered_request.rejection_reason().unwrap_or(ConnectionRejectionReason::RegistrationFailed); + error!("Stream admission rejected for user {username}: {reason:?}; dropping provider handle"); + return Err(reason.into()); + } + if let Some((_, _, _m_, Some(cvt))) = stream_details.stream_info.as_ref() { + app_state.connection_manager.update_stream_detail_by_uid(request_uid, *cvt).await; + } + + let meter = if meter_stream && meter_uid != 0 { + let meter = Arc::new(StreamMeterHandle::new(meter_uid, Arc::downgrade(&app_state.event_manager))); + app_state.event_manager.register_meter(Arc::clone(&meter)).await; + Some(meter) + } else { + None + }; + + // Shared broadcaster source (first subscriber path): feed provider bytes directly. + // Grace/custom handling is not needed here because this stream is only the fan-out source. + if is_shared_source_stream { + if let Some(stream) = stream_details.stream.take() { + let stream = if let Some(meter) = &meter { + MeteringStream::new(stream, Arc::clone(meter), Arc::clone(&app_state.event_manager)).boxed() + } else { + stream + }; + // The shared source owns the claim from here; disarm the registration rollback. + registered_request.commit(); + return Ok(stream); + } + } + + let provisioning_info = resolve_grace_period_provisioning(app_state, &stream_details); + let has_provisioning = provisioning_info.is_some(); + let provisioning_stop_signal = provisioning_info.as_ref().map(|info| info.stop_signal.clone()); + let hold_stream = stream_details.grace_period.hold_stream; + let capacity_notify = app_state.connection_manager.capacity_notified(); + let pending_provider_version = if hold_stream { + if let Some(token) = session_token { + app_state.active_users.pending_provider_version(&user.username, token).await + } else { + None + } + } else { + None + }; + // `GraceActive` version: populated when the session is in `GraceActive` lifecycle (GraceMode::Instant). + let grace_active_version = if let Some(token) = session_token { + app_state.active_users.grace_active_version(&user.username, token).await + } else { + None + }; + + let waker = Arc::new(AtomicWaker::new()); + let owned_session_token: Option = session_token.map(str::to_string); + // A provider-backed playback confirms its lease owner once media flows. Local + // media and streams without a provider handle carry no lease to confirm. + let lease_owner: Option> = session_token + .filter(|_| stream_details.provider_handle.is_some() || stream_details.has_deferred_provider_open()) + .map(Arc::::from); + let media_started = if let Some(owner) = lease_owner.as_deref() { + app_state.active_users.media_started_flag(&user.username, owner).await + } else { + None + }; + let owned_grace_ctx = stream_details.grace_resolution_context.clone(); + let lease_request_id = stream_details + .provider_handle + .as_ref() + .and_then(|managed| managed.handle()) + .and_then(|handle| handle.playback_request_id); + // Compute deferred-open metadata before taking the handle: the grace snapshot only + // needs the provider identity, while the body owns the actual release. + let has_deferred_open = stream_details.has_deferred_provider_open(); + let deferred_provider_open = + create_deferred_provider_open_future(app_state, &stream_details, fingerprint, &stream_channel, req_headers); + let timed_stream_context = deferred_provider_open + .as_ref() + .and_then(|_| create_timed_stream_context(app_state, VirtualId::new(virtual_id))); + // The body owns the provider slot from here on; the managed owner releases it + // synchronously on drop, never through a lossy cleanup message. + let mut provider_handle_preserved = stream_details.provider_handle.take(); + let stream_taken = stream_details.stream.take(); + let grace_waker = + if grant_user_grace_period || stream_details.provider_grace_active { Some(Arc::clone(&waker)) } else { None }; + let (grace_stop_flag, grace_task_handle) = stream_grace_period(GracePeriodParams { + app_state: Arc::clone(app_state), + stream_details, + user_grace_period: grant_user_grace_period, + user: user.clone(), + fingerprint: fingerprint.clone(), + virtual_id: VirtualId::new(virtual_id), + session_token: owned_session_token, + provisioning_info, + waker: grace_waker, + hold_stream, + capacity_notify, + pending_provider_version, + grace_active_version, + grace_resolution_context: owned_grace_ctx, + grace_kind: Some(connection_kind), + socket_bound, + shared_subscriber_id, + }); + + let cfg = &app_state.app_config; + let custom_response = cfg.custom_stream_response.load(); + let custom_video_timeout_secs = cfg.config.load().custom_stream_response_timeout_secs; + let custom_video = custom_response.as_ref().filter(|_| response_mode == StreamResponseMode::Stream).map_or( + CustomVideoBuffers { + user_exhausted: None, + provider_exhausted: None, + unavailable: None, + provisioning: None, + low_priority_preempted: None, + }, + |c| CustomVideoBuffers { + user_exhausted: c.user_connections_exhausted.clone(), + provider_exhausted: c.provider_connections_exhausted.clone(), + unavailable: c.channel_unavailable.clone(), + provisioning: c.panel_api_provisioning.clone(), + low_priority_preempted: c.low_priority_preempted.clone(), + }, + ); + + let stream: Option = match stream_taken { + None => { + if !has_deferred_open { + drop(provider_handle_preserved.take()); + } + None + } + Some(stream) => { + let stream = if let Some(meter) = &meter { + MeteringStream::new(stream, Arc::clone(meter), Arc::clone(&app_state.event_manager)).boxed() + } else { + stream + }; + Some(wrap_timed_client_stream_if_needed( + app_state, + stream, + fingerprint.addr, + VirtualId::new(virtual_id), + display_stream_uid, + )) + } + }; + + let preempt_cancelled = provider_handle_preserved + .as_ref() + .and_then(|managed| managed.handle()) + .and_then(|h| h.cancel_token.as_ref()) + .map(|token| Box::pin(token.clone().cancelled_owned())); + + let mut send_custom_stream_flag = grace_stop_flag; + if send_custom_stream_flag.is_none() && preempt_cancelled.is_some() && custom_video.low_priority_preempted.is_some() + { + send_custom_stream_flag = Some(Arc::new(AtomicU8::new(StreamMode::Inner as u8))); + } + + let state = ActiveClientStreamState { + response_mode, + inner: stream, + deferred_provider_open, + timed_stream_context, + preempt_cancelled, + grace_task_handle, + provisioning_stop_signal, + provider_handle: provider_handle_preserved, + send_custom_stream_flag, + provisionable: has_provisioning, + custom_video, + meter, + event_manager: Arc::clone(&app_state.event_manager), + waker: Some(waker), + connection_manager: Arc::clone(&app_state.connection_manager), + fingerprint: Arc::new(fingerprint.clone()), + stream_uid, + provider_stopped: false, + user_stream_released: false, + provider_handle_released: false, + custom_video_timeout_secs, + custom_video_timeout_mode: None, + custom_video_timeout_sleep: None, + direct_body_idle_timeout, + provider_end_reason: AtomicU8::new(PROVIDER_END_NOT_SET), + provider_error_class: None, + provider_http_status: None, + provider_reconnect_count: AtomicU8::new(0), + lease_owner, + media_started, + lease_confirmed: false, + lease_request_id, + request_cleanup: registered_request.into_body_cleanup(), + }; + + Ok(ActiveClientStream { state }.boxed()) +} diff --git a/backend/app/src/api/model/streams/active_client_stream/error.rs b/backend/app/src/api/model/streams/active_client_stream/error.rs new file mode 100644 index 000000000..a54f46be8 --- /dev/null +++ b/backend/app/src/api/model/streams/active_client_stream/error.rs @@ -0,0 +1,20 @@ +/// Terminal failure while admitting a playback request. Carried to the response layer so a +/// rejected request surfaces as a non-success HTTP response instead of an empty provider body. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StreamAdmissionError { + CleanupReceiverClosed, + CleanupAdmissionTimeout, + RegistrationRejected, +} + +use tuliprox_session::ConnectionRejectionReason; + +impl From for StreamAdmissionError { + fn from(reason: ConnectionRejectionReason) -> Self { + match reason { + ConnectionRejectionReason::CleanupReceiverClosed => Self::CleanupReceiverClosed, + ConnectionRejectionReason::CleanupAdmissionTimeout => Self::CleanupAdmissionTimeout, + ConnectionRejectionReason::RegistrationFailed => Self::RegistrationRejected, + } + } +} diff --git a/backend/app/src/api/model/streams/active_client_stream/grace.rs b/backend/app/src/api/model/streams/active_client_stream/grace.rs new file mode 100644 index 000000000..30741a916 --- /dev/null +++ b/backend/app/src/api/model/streams/active_client_stream/grace.rs @@ -0,0 +1,383 @@ +use super::{store_stream_mode, ActiveClientStreamState, GracePeriodParams, GraceProvisioningInfo, StreamMode}; +use crate::{ + api::{ + model::{AppState, CustomVideoStreamType, PendingProviderWakeSource, StreamDetails}, + panel_api::{can_provision_on_exhausted, find_input_by_provider_name, run_panel_api_provisioning_probe}, + }, + utils::debug_if_enabled, +}; +use log::{error, info}; +use std::sync::{ + atomic::{AtomicU8, Ordering}, + Arc, +}; +use tokio_util::sync::CancellationToken; + +impl ActiveClientStreamState { + pub(super) fn stop_grace_task(&mut self) { + if let Some(task) = self.grace_task_handle.take() { + task.abort(); + } + if let Some(token) = self.provisioning_stop_signal.take() { + token.cancel(); + } + } + + pub(super) fn clear_finished_grace_task(&mut self) { + if self.grace_task_handle.as_ref().is_some_and(tokio::task::JoinHandle::is_finished) { + self.grace_task_handle = None; + // If the task finished but the flag is still GRACE_PENDING (e.g. the task + // panicked or was cancelled before it could update the flag), reset the flag + // to INNER_STREAM so the client stream is not hung indefinitely. + if let Some(flag) = &self.send_custom_stream_flag { + let _ = flag.compare_exchange( + StreamMode::GracePending as u8, + StreamMode::Inner as u8, + Ordering::AcqRel, + Ordering::Relaxed, + ); + } + } + } +} + +pub(super) fn resolve_grace_period_provisioning( + app_state: &Arc, + stream_details: &StreamDetails, +) -> Option { + if stream_details.disable_provider_grace || !stream_details.provider_grace_active { + return None; + } + let provider_name = stream_details.provider_name.as_deref(); + let input = provider_name.and_then(|name| find_input_by_provider_name(app_state.as_ref(), name))?; + if !can_provision_on_exhausted(app_state, &input) { + return None; + } + + let stop_signal = CancellationToken::new(); + Some(GraceProvisioningInfo { input, stop_signal }) +} + +#[allow(clippy::too_many_lines)] +pub(super) fn stream_grace_period( + request: GracePeriodParams, +) -> (Option>, Option>) { + let GracePeriodParams { + app_state, + stream_details, + user_grace_period, + user, + fingerprint, + virtual_id, + session_token, + provisioning_info, + waker, + hold_stream, + capacity_notify, + pending_provider_version, + grace_active_version, + grace_resolution_context, + grace_kind, + socket_bound, + shared_subscriber_id, + .. + } = request; + let grace_period = stream_details.grace_period; + let active_users = Arc::clone(&app_state.active_users); + let active_provider = Arc::clone(&app_state.active_provider); + let connection_manager = Arc::clone(&app_state.connection_manager); + + let provider_grace_check = if stream_details.provider_grace_active + && stream_details.provider_name.is_some() + && !stream_details.disable_provider_grace + { + stream_details.provider_name.clone() + } else { + None + }; + + let user_max_connections = user.max_connections; + let user_grace_check = if user_grace_period && user_max_connections > 0 { + let user_name = user.username.clone(); + Some((user_name, user_max_connections)) + } else { + None + }; + + if provider_grace_check.is_some() || user_grace_check.is_some() { + let stream_strategy_flag = + Arc::new(AtomicU8::new(if hold_stream { StreamMode::GracePending as u8 } else { StreamMode::Inner as u8 })); + let stream_strategy_flag_copy = Arc::clone(&stream_strategy_flag); + let grace_period_millis = grace_period.period_millis; + + let user_manager = Arc::clone(&active_users); + let provider_manager = Arc::clone(&active_provider); + let connection_manager = Arc::clone(&connection_manager); + let reconnect_flag = stream_details.reconnect_flag.clone(); + let fingerprint = fingerprint.clone(); + let app_state = Arc::clone(&app_state); + let pending_username = user.username.clone(); + // Clone owned copies of fields borrowed from `request` so the spawn sees owned values. + let grace_resolution_context = grace_resolution_context.clone(); + // Safety timeout: if async operations inside the grace task stall, force the flag + // out of GRACE_PENDING so the client stream is not hung indefinitely. + // Allow grace_period_millis for the intentional delay plus a 10-second buffer + // for the async connection checks that follow. + let grace_task_timeout = tokio::time::Duration::from_millis(grace_period_millis.saturating_add(10_000)); + // Clone handles for use in the timeout fallback, in case the inner async block is cancelled. + let flag_for_fallback = Arc::clone(&stream_strategy_flag_copy); + let waker_for_fallback = waker.clone(); + let session_token_timeout = session_token.clone(); + let active_users_timeout = Arc::clone(&active_users); + let pending_username_timeout = pending_username.clone(); + let grace_task_handle = tokio::spawn(async move { + let timed_out = tokio::time::timeout(grace_task_timeout, async move { + let deadline = tokio::time::Instant::now() + tokio::time::Duration::from_millis(grace_period_millis); + let mut pending_wake_source = PendingProviderWakeSource::Activated; + loop { + let capacity_wait = capacity_notify.notified(); + tokio::pin!(capacity_wait); + + let user_ok = match &user_grace_check { + Some((username, max_connections)) => { + user_manager.user_connections(username).await <= *max_connections + } + None => true, + }; + let provider_ok = match &provider_grace_check { + Some(provider_name) => !provider_manager.is_over_limit(provider_name), + None => true, + }; + if user_ok && provider_ok { + break; + } + + tokio::select! { + () = tokio::time::sleep_until(deadline) => { + pending_wake_source = PendingProviderWakeSource::Timeout; + break; + } + () = &mut capacity_wait => { + pending_wake_source = PendingProviderWakeSource::CapacityNotify; + } + } + } + + let mut updated = false; + if let Some((username, max_connections)) = user_grace_check { + let active_connections = user_manager.user_connections(&username).await; + if active_connections > max_connections { + // User-grace failed. Evaluate remaining strategies before final deny. + if let Some(ref ctx) = grace_resolution_context { + let eviction_guard = if socket_bound { + tuliprox_session::admission::EvictionReentryGuard::SocketPlayback { virtual_id } + } else { + tuliprox_session::admission::EvictionReentryGuard::Session( + // Defensive fallback: an empty token will not match any real session. + session_token.as_deref().unwrap_or_default(), + ) + }; + let remaining_result = + tuliprox_session::admission::evaluate_remaining_strategies_after_grace( + &app_state.admission_ctx(), + tuliprox_session::admission::AdmissionRequest { + username: &username, + max_connections, + soft_connections: user.soft_connections, + client_ip: &fingerprint.client_ip, + request_addr: &fingerprint.addr, + use_session_admission: true, + session_token: session_token.as_deref(), + activate_unbound_session: true, + eviction_reentry_guard: eviction_guard, + }, + ctx, + grace_kind, + ) + .await; + match remaining_result.admission.permission() { + shared::model::UserConnectionPermission::Allowed + | shared::model::UserConnectionPermission::GracePeriod => { + // Remaining strategy succeeded — proceed to Inner. + store_stream_mode(&stream_strategy_flag_copy, StreamMode::Inner); + // updated stays false + } + shared::model::UserConnectionPermission::Exhausted => { + let suppressed = remaining_result.admission.is_reentry_suppressed(); + if suppressed { + // Every remaining eviction candidate is reentry-protected, + // so this is a suppressed retry of a recently evicted + // playback. End the body quietly: a visible + // connections-exhausted video would be misleading because + // the request was declined by the reentry guard, not by a + // real connection limit. + store_stream_mode(&stream_strategy_flag_copy, StreamMode::ReentrySuppressed); + info!( + "Suppressing reentry retry for recently evicted playback of user {username}" + ); + } else { + // Remaining strategies exhausted — final UserExhausted. + store_stream_mode(&stream_strategy_flag_copy, StreamMode::UserExhausted); + connection_manager + .update_stream_detail( + &fingerprint.addr, + CustomVideoStreamType::UserConnectionsExhausted, + ) + .await; + info!("User connections exhausted for active clients: {username}"); + } + if let Some(id) = shared_subscriber_id { + connection_manager.shared_stream_manager.release_subscriber(id).await; + } + updated = true; + } + } + } else { + // No grace context — immediate UserExhausted. + store_stream_mode(&stream_strategy_flag_copy, StreamMode::UserExhausted); + connection_manager + .update_stream_detail( + &fingerprint.addr, + CustomVideoStreamType::UserConnectionsExhausted, + ) + .await; + if let Some(id) = shared_subscriber_id { + connection_manager.shared_stream_manager.release_subscriber(id).await; + } + info!("User connections exhausted for active clients: {username}"); + updated = true; + } + } + } + + if !updated { + if let Some(provider_name) = provider_grace_check { + if provider_manager.is_over_limit(&provider_name) { + if let Some(provisioning_info) = provisioning_info { + store_stream_mode(&stream_strategy_flag_copy, StreamMode::Provisioning); + connection_manager + .update_stream_detail(&fingerprint.addr, CustomVideoStreamType::Provisioning) + .await; + debug_if_enabled!( + "Provider grace period exhausted; provisioning for active clients: {provider_name}" + ); + let app_state = Arc::clone(&app_state); + let input_name = Arc::clone(&provisioning_info.input.name); + let stop_signal = provisioning_info.stop_signal; + let addr = fingerprint.addr; + tokio::spawn(async move { + if let Err(err) = run_panel_api_provisioning_probe( + app_state, + input_name, + stop_signal, + addr, + virtual_id, + ) + .await + { + error!("Error running Probe: {err:?}"); + } + }); + } else { + store_stream_mode(&stream_strategy_flag_copy, StreamMode::ProviderExhausted); + connection_manager + .update_stream_detail( + &fingerprint.addr, + CustomVideoStreamType::ProviderConnectionsExhausted, + ) + .await; + // Release the shared stream subscription to stop the subscriber loop + if let Some(id) = shared_subscriber_id { + connection_manager.shared_stream_manager.release_subscriber(id).await; + } + info!("Provider connections exhausted for active clients: {provider_name}"); + } + updated = true; + } + } + } + + if !updated { + store_stream_mode(&stream_strategy_flag_copy, StreamMode::Inner); + } + + // Resolve session lifecycle transitions. + // PendingProvider (Hold): activate on success, expire on failure. + if hold_stream { + if let (Some(token), Some(version)) = (session_token.as_deref(), pending_provider_version) { + let _transition_guard = + active_users.acquire_playback_transition(&pending_username, token).await; + if updated { + active_users + .expire_pending_provider(&pending_username, token, version, pending_wake_source) + .await; + } else { + active_users + .activate_pending_provider(&pending_username, token, version, pending_wake_source) + .await; + } + } + } + + // GraceActive (Instant): activate on success, expire on failure. + // grace_active_version is Some when the session is in `GraceActive` lifecycle. + if let (Some(token), Some(version)) = (session_token.as_deref(), grace_active_version) { + let _transition_guard = active_users.acquire_playback_transition(&pending_username, token).await; + if updated { + active_users.expire_grace_active(&pending_username, token, version).await; + } else { + active_users.activate_grace_active(&pending_username, token, version).await; + } + } + + if updated { + if let Some(flag) = reconnect_flag { + flag.cancel(); + } + } + + if let Some(w) = waker.as_ref() { + w.wake(); + } + }) + .await; + + if timed_out.is_err() { + // Grace task exceeded its budget without updating the flag — reset GRACE_PENDING + // to INNER_STREAM so the client stream is not hung indefinitely. + // Also resolve any session lifecycle to prevent inconsistent state: + // a PendingProvider / GraceActive session that was never resolved would cause + // the next admission attempt to incorrectly skip re-evaluation. + if let (Some(token), Some(version)) = (session_token_timeout.as_deref(), pending_provider_version) { + let _transition_guard = + active_users_timeout.acquire_playback_transition(&pending_username_timeout, token).await; + active_users_timeout + .expire_pending_provider( + &pending_username_timeout, + token, + version, + PendingProviderWakeSource::Timeout, + ) + .await; + } + if let (Some(token), Some(version)) = (session_token_timeout.as_deref(), grace_active_version) { + let _transition_guard = + active_users_timeout.acquire_playback_transition(&pending_username_timeout, token).await; + active_users_timeout.expire_grace_active(&pending_username_timeout, token, version).await; + } + error!("Grace period task timed out; resetting stream flag to prevent client hang"); + let _ = flag_for_fallback.compare_exchange( + StreamMode::GracePending as u8, + StreamMode::Inner as u8, + Ordering::AcqRel, + Ordering::Relaxed, + ); + if let Some(w) = waker_for_fallback.as_ref() { + w.wake(); + } + } + }); + return (Some(stream_strategy_flag), Some(grace_task_handle)); + } + (None, None) +} diff --git a/backend/app/src/api/model/streams/active_client_stream/provider_open.rs b/backend/app/src/api/model/streams/active_client_stream/provider_open.rs new file mode 100644 index 000000000..a37513e0e --- /dev/null +++ b/backend/app/src/api/model/streams/active_client_stream/provider_open.rs @@ -0,0 +1,237 @@ +use super::{store_stream_mode, ActiveClientStreamState, StreamMode}; +use crate::{ + api::{ + model::{ + connection_manager::PROVIDER_END_PREEMPTED, AppState, BoxedProviderStream, CleanupEvent, + CustomVideoStreamType, MeteringStream, ProviderStreamFactoryOptions, StreamDetails, TimedClientStream, + }, + panel_api::find_input_by_provider_name, + }, + auth::Fingerprint, + utils::debug_if_enabled, +}; +use axum::http::HeaderMap; +use futures::{Future, StreamExt}; +use shared::{ + model::{FailureStage, StreamChannel}, + utils::sanitize_sensitive_info, +}; +use std::{ + pin::Pin, + sync::{ + atomic::{AtomicU8, Ordering}, + Arc, + }, +}; +use tuliprox_session::stream_options::{get_stream_options, StreamResponseMode}; + +pub(super) struct DeferredProviderOpenContext { + pub(super) app_state: Arc, + pub(super) provider_stream_factory_options: ProviderStreamFactoryOptions, +} + +pub(super) enum DeferredProviderOpenOutcome { + Stream(BoxedProviderStream), + Mode(StreamMode), + Failed, +} + +pub(super) enum DeferredProviderOpenState { + Pending(Box), + Opening(Pin + Send>>), +} + +impl ActiveClientStreamState { + pub(super) fn wrap_provider_stream(&self, stream: BoxedProviderStream) -> BoxedProviderStream { + let stream = if let Some(meter) = &self.meter { + MeteringStream::new(stream, Arc::clone(meter), Arc::clone(&self.event_manager)).boxed() + } else { + stream + }; + if let Some(ctx) = self.timed_stream_context.as_ref() { + TimedClientStream::new( + &ctx.app_state.app_config, + &ctx.app_state.connection_manager, + stream, + ctx.duration_secs, + self.fingerprint.addr, + ctx.virtual_id, + self.stream_uid, + ) + .boxed() + } else { + stream + } + } + + pub(super) fn stop_provider_stream_preempted(&mut self) -> bool { + if self.response_mode == StreamResponseMode::HlsResource { + self.provider_end_reason.store(PROVIDER_END_PREEMPTED, Ordering::Relaxed); + self.provider_error_class = Some("preempted"); + self.terminate_quietly(); + return false; + } + self.provider_stopped = true; + self.preempt_cancelled = None; + self.stop_grace_task(); + self.provider_end_reason.store(PROVIDER_END_PREEMPTED, Ordering::Relaxed); + self.provider_error_class = Some("preempted"); + + let is_superseded = self + .provider_handle + .as_ref() + .and_then(|m| m.handle()) + .is_some_and(|h| h.get_close_reason() == tuliprox_core::model::ProviderCloseReason::Superseded); + + let mut serve_preempted_custom = false; + if self.provider_handle.is_some() { + let managed = self.provider_handle.take(); + self.provider_handle_released = true; + // Synchronous provider-slot release; the custom-video detail update is a + // separate, best-effort UI effect. + drop(managed); + if !is_superseded && self.custom_video.low_priority_preempted.is_some() { + serve_preempted_custom = true; + if let Some(flag) = &self.send_custom_stream_flag { + store_stream_mode(flag, StreamMode::LowPriorityPreempted); + } else { + // Fallback: create_active_client_stream usually initializes this via stream_grace_period. + self.send_custom_stream_flag = + Some(Arc::new(AtomicU8::new(StreamMode::LowPriorityPreempted as u8))); + } + } else if let Some(flag) = &self.send_custom_stream_flag { + store_stream_mode(flag, StreamMode::Inner); + } + + if let Some(waker) = &self.waker { + waker.wake(); + } + + let addr = self.fingerprint.addr; + // Drop the provider stream immediately instead of replacing with an + // allocated empty stream — avoids a heap allocation on every preemption. + self.inner = None; + + debug_if_enabled!( + "Provider stream preempted for {}; stopping client stream", + sanitize_sensitive_info(&addr.to_string()) + ); + if serve_preempted_custom { + self.connection_manager.send_cleanup(CleanupEvent::UpdateDetailAndReleaseProvider { + addr, + stream_uid: self.stream_uid, + video_type: CustomVideoStreamType::LowPriorityPreempted, + handle: None, + }); + } else { + self.release_user_stream(); + } + } + serve_preempted_custom + } + + pub(super) fn stop_provider_stream(&mut self, mode: StreamMode) { + self.provider_stopped = true; + self.preempt_cancelled = None; + self.stop_grace_task(); + + if self.provider_handle.is_some() { + // Synchronous provider-slot release; the custom-video detail update is a + // separate, best-effort UI effect. + self.release_provider_handle_and_detach_body(); + + if mode == StreamMode::ChannelUnavailable { + if let Some(flag) = &self.send_custom_stream_flag { + let _ = flag.compare_exchange( + StreamMode::Inner as u8, + StreamMode::ChannelUnavailable as u8, + Ordering::AcqRel, + Ordering::Relaxed, + ); + } + } + + if let Some(waker) = &self.waker { + waker.wake(); + } + + let addr = self.fingerprint.addr; + let reason = match mode { + StreamMode::ChannelUnavailable => "unavailable provider channel", + StreamMode::UserExhausted => "user grace period exhaustion", + StreamMode::ProviderExhausted => "provider grace period exhaustion", + StreamMode::Provisioning => "provider grace period provisioning", + StreamMode::LowPriorityPreempted => "low-priority preemption", + StreamMode::ReentrySuppressed => "reentry suppression", + StreamMode::Inner | StreamMode::GracePending => "stream mode transition", + }; + debug_if_enabled!( + "Provider stream stopped due to {reason} for {}", + sanitize_sensitive_info(&addr.to_string()) + ); + if let Some(video_type) = Self::custom_video_type_for_mode(mode) { + self.connection_manager.send_cleanup(CleanupEvent::UpdateDetailAndReleaseProvider { + addr, + stream_uid: self.stream_uid, + video_type, + handle: None, + }); + } else { + // No custom video for this mode: release the request without publishing + // a detail update so no error clip is shown. + self.release_user_stream(); + } + } + } +} + +pub(super) fn create_deferred_provider_open_future( + app_state: &Arc, + stream_details: &StreamDetails, + fingerprint: &Fingerprint, + stream_channel: &StreamChannel, + req_headers: &HeaderMap, +) -> Option { + if !stream_details.has_deferred_provider_open() { + return None; + } + + let provider_name = stream_details.provider_name.as_deref()?; + let request_url = stream_details.request_url.as_deref()?; + let input = find_input_by_provider_name(app_state.as_ref(), provider_name)?; + let stream_url = url::Url::parse(request_url).ok()?; + let stream_options = get_stream_options(&app_state.app_config, stream_details.response_mode); + let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); + let disabled_headers = app_state.get_disabled_headers(); + let mut provider_stream_factory_options = + ProviderStreamFactoryOptions::new(&crate::api::model::ProviderStreamFactoryParams { + addr: fingerprint.addr, + item_type: stream_channel.item_type, + share_stream: stream_channel.shared, + stream_options: &stream_options, + stream_url: &stream_url, + req_headers, + input_headers: Some(&input.headers), + session_headers: stream_details.session_headers.as_ref(), + disabled_headers: disabled_headers.as_ref(), + default_user_agent: default_user_agent.as_deref(), + username: None, + client_ip: Some(&fingerprint.client_ip), + stream_channel: Some(stream_channel), + connect_failure_stage: Some(FailureStage::ProviderOpen), + content_representation: stream_details.content_representation, + }) + .for_deferred_open(); + if stream_details.provider_handle.is_some() { + if let Some(stream_index) = stream_details.user_agent_stream_index { + provider_stream_factory_options.apply_user_agent_stream_index(stream_index); + } + } + provider_stream_factory_options.set_provider(input.get_resolve_provider(stream_url.as_ref())); + provider_stream_factory_options.apply_input_options(&input); + + Some(DeferredProviderOpenState::Pending(Box::new(DeferredProviderOpenContext { + app_state: Arc::clone(app_state), + provider_stream_factory_options, + }))) +} diff --git a/backend/app/src/api/model/streams/active_client_stream/state.rs b/backend/app/src/api/model/streams/active_client_stream/state.rs new file mode 100644 index 000000000..7f7f09216 --- /dev/null +++ b/backend/app/src/api/model/streams/active_client_stream/state.rs @@ -0,0 +1,252 @@ +use super::{ActiveClientStreamState, BODY_IDLE_TIMEOUT_ERROR_CLASS, DIRECT_BODY_SOCKET_ACTIVITY_TOUCH_SECS}; +use crate::{ + api::model::{ + connection_manager::{PROVIDER_END_ERROR, PROVIDER_END_NOT_SET}, + uses_direct_body_idle_timeout, AppState, CustomVideoStreamType, TransportStreamBuffer, + DIRECT_BODY_IDLE_TIMEOUT_SECS, + }, + model::ConfigInput, +}; +use futures::Future; +use log::{info, warn}; +use shared::{ + model::{StreamChannel, VirtualId}, + utils::sanitize_sensitive_info, +}; +use std::{ + pin::Pin, + sync::{ + atomic::{AtomicU8, Ordering}, + Arc, + }, + task::Context, +}; +use tokio_util::sync::CancellationToken; +use tuliprox_session::stream_options::StreamResponseMode; + +/// Discriminates which byte-stream the client is consuming at any moment. +/// Stored as `u8` in an `AtomicU8` for lock-free access inside `poll_next`. +/// Lower numeric values correspond to a live or custom stream; `GracePending` +/// (255) is a transient sentinel that parks the poll until the grace task resolves. +/// +/// Discriminants are implicit (declaration order) except for the `255` sentinel. +/// The byte mapping lives once, in [`StreamMode::try_from`], and is pinned by +/// `test_stream_mode_byte_values`. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum StreamMode { + /// Forward bytes directly from the upstream provider. + Inner, + /// Show the "user connections exhausted" custom video. + UserExhausted, + /// Show the "provider connections exhausted" custom video. + ProviderExhausted, + /// Show the "channel unavailable" custom video. + ChannelUnavailable, + /// Show the provisioning/placeholder custom video while probing for capacity. + Provisioning, + /// Show the "low-priority preempted" custom video. + LowPriorityPreempted, + /// A recently evicted playback retried while this stream held a grace slot and + /// every remaining eviction candidate is reentry-protected. The body must end + /// without painting a user-visible "connections exhausted" error video. + ReentrySuppressed, + /// Transient: grace-period check is still in progress; `poll_next` must park. + GracePending = 255, +} + +impl TryFrom for StreamMode { + type Error = u8; + + /// Decodes the atomic mode flag. Returns `Err(value)` for a byte that was never a + /// valid [`StreamMode`]; callers must fail safe instead of inventing a live state + /// such as `GracePending`. + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(Self::Inner), + 1 => Ok(Self::UserExhausted), + 2 => Ok(Self::ProviderExhausted), + 3 => Ok(Self::ChannelUnavailable), + 4 => Ok(Self::Provisioning), + 5 => Ok(Self::LowPriorityPreempted), + 6 => Ok(Self::ReentrySuppressed), + 255 => Ok(Self::GracePending), + other => Err(other), + } + } +} + +/// Publishes a new mode to the lock-free flag with release ordering. All writers go +/// through this so the `as u8` cast and the memory ordering live in one place. +pub(super) fn store_stream_mode(flag: &AtomicU8, mode: StreamMode) { flag.store(mode as u8, Ordering::Release); } + +/// Holds the optional custom video buffers for each error/placeholder scenario. +/// Using named fields avoids the positional-indexing confusion of a 4-tuple. +pub(super) struct CustomVideoBuffers { + pub(super) user_exhausted: Option, + pub(super) provider_exhausted: Option, + pub(super) unavailable: Option, + pub(super) provisioning: Option, + pub(super) low_priority_preempted: Option, +} + +pub(super) struct GraceProvisioningInfo { + pub(super) input: Arc, + pub(super) stop_signal: CancellationToken, +} + +#[derive(Clone)] +pub(super) struct TimedStreamContext { + pub(super) app_state: Arc, + pub(super) duration_secs: u32, + pub(super) virtual_id: VirtualId, +} + +pub(super) struct DirectBodyIdleTimeout { + pub(super) enabled: bool, + pub(super) deadline: Option, + pub(super) sleep: Option>>, + pub(super) last_socket_activity_touch: Option, +} + +impl DirectBodyIdleTimeout { + pub(super) const fn disabled() -> Self { + Self { enabled: false, deadline: None, sleep: None, last_socket_activity_touch: None } + } + + pub(super) const fn enabled() -> Self { + Self { enabled: true, deadline: None, sleep: None, last_socket_activity_touch: None } + } + + pub(super) fn mark_progress(&mut self) -> bool { + if !self.enabled { + return false; + } + + let now = tokio::time::Instant::now(); + self.deadline = Some(now + tokio::time::Duration::from_secs(DIRECT_BODY_IDLE_TIMEOUT_SECS)); + self.sleep = None; + + let should_touch_socket = self.last_socket_activity_touch.is_none_or(|last_touch| { + now.duration_since(last_touch) >= tokio::time::Duration::from_secs(DIRECT_BODY_SOCKET_ACTIVITY_TOUCH_SECS) + }); + if should_touch_socket { + self.last_socket_activity_touch = Some(now); + } + should_touch_socket + } + + pub(super) fn poll_expired(&mut self, cx: &mut Context<'_>) -> bool { + if !self.enabled { + return false; + } + + let deadline = *self.deadline.get_or_insert_with(|| { + tokio::time::Instant::now() + tokio::time::Duration::from_secs(DIRECT_BODY_IDLE_TIMEOUT_SECS) + }); + if tokio::time::Instant::now() >= deadline { + return true; + } + + self.sleep.get_or_insert_with(|| Box::pin(tokio::time::sleep_until(deadline))); + self.sleep.as_mut().is_some_and(|sleep| sleep.as_mut().poll(cx).is_ready()) + } +} + +impl ActiveClientStreamState { + pub(super) fn mode_for_custom_video_type(video_type: CustomVideoStreamType) -> Option { + match video_type { + CustomVideoStreamType::ChannelUnavailable => Some(StreamMode::ChannelUnavailable), + CustomVideoStreamType::UserConnectionsExhausted => Some(StreamMode::UserExhausted), + CustomVideoStreamType::ProviderConnectionsExhausted => Some(StreamMode::ProviderExhausted), + CustomVideoStreamType::LowPriorityPreempted => Some(StreamMode::LowPriorityPreempted), + CustomVideoStreamType::Provisioning => Some(StreamMode::Provisioning), + CustomVideoStreamType::UserAccountExpired | CustomVideoStreamType::HlsSessionOrLeaseExpired => None, + } + } + + /// Maps a mode to the custom video it serves, if any. `ReentrySuppressed` and the + /// transparent forwarding modes deliberately have no custom video: a suppressed + /// reentry must never surface a user-visible error clip. + pub(super) fn custom_video_type_for_mode(mode: StreamMode) -> Option { + match mode { + StreamMode::UserExhausted => Some(CustomVideoStreamType::UserConnectionsExhausted), + StreamMode::ProviderExhausted => Some(CustomVideoStreamType::ProviderConnectionsExhausted), + StreamMode::Provisioning => Some(CustomVideoStreamType::Provisioning), + StreamMode::LowPriorityPreempted => Some(CustomVideoStreamType::LowPriorityPreempted), + StreamMode::ChannelUnavailable => Some(CustomVideoStreamType::ChannelUnavailable), + StreamMode::Inner | StreamMode::GracePending | StreamMode::ReentrySuppressed => None, + } + } + + pub(super) fn stop_direct_body_idle_timeout(&mut self) { + self.provider_stopped = true; + self.preempt_cancelled = None; + self.inner = None; + self.provider_error_class = Some(BODY_IDLE_TIMEOUT_ERROR_CLASS); + self.provider_http_status = None; + let _ = self.provider_end_reason.compare_exchange( + PROVIDER_END_NOT_SET, + PROVIDER_END_ERROR, + Ordering::Relaxed, + Ordering::Relaxed, + ); + warn!( + "Direct body stream idle timeout after {DIRECT_BODY_IDLE_TIMEOUT_SECS}s for {}, terminating stream", + sanitize_sensitive_info(&self.fingerprint.addr.to_string()) + ); + self.release_stream_and_provider_handle_once(); + } + + pub(super) fn reset_custom_video_timeout(&mut self) { + self.custom_video_timeout_mode = None; + self.custom_video_timeout_sleep = None; + } + + /// Returns whether the custom state can serve a body; finite HLS objects terminate on entry. + pub(super) fn enter_custom_mode(&mut self, mode: StreamMode) -> bool { + if self.response_mode == StreamResponseMode::HlsResource { + self.terminate_quietly(); + return false; + } + if self.custom_video_timeout_mode != Some(mode) { + self.custom_video_timeout_mode = Some(mode); + self.custom_video_timeout_sleep = if self.custom_video_timeout_secs > 0 { + Some(Box::pin(tokio::time::sleep(tokio::time::Duration::from_secs(u64::from( + self.custom_video_timeout_secs, + ))))) + } else { + None + }; + } + + if !self.provider_stopped { + info!( + "Switching to {mode:?} custom video stream for {}", + sanitize_sensitive_info(&self.fingerprint.addr.to_string()) + ); + self.stop_provider_stream(mode); + } + true + } + + pub(super) fn custom_video_timed_out(&mut self, cx: &mut Context<'_>, mode: StreamMode) -> bool { + if self.custom_video_timeout_secs == 0 { + return false; + } + + if self.custom_video_timeout_mode != Some(mode) { + return false; + } + + if let Some(timeout_sleep) = self.custom_video_timeout_sleep.as_mut() { + return timeout_sleep.as_mut().poll(cx).is_ready(); + } + + false + } +} + +pub(super) fn should_use_direct_body_idle_timeout(stream_channel: &StreamChannel) -> bool { + uses_direct_body_idle_timeout(stream_channel) +} diff --git a/backend/app/src/api/model/streams/active_client_stream/stream.rs b/backend/app/src/api/model/streams/active_client_stream/stream.rs new file mode 100644 index 000000000..f04c21085 --- /dev/null +++ b/backend/app/src/api/model/streams/active_client_stream/stream.rs @@ -0,0 +1,392 @@ +use super::{ + ActiveClientStream, ActiveClientStreamState, DeferredProviderOpenOutcome, DeferredProviderOpenState, StreamMode, + TimedStreamContext, +}; +use crate::api::model::{ + connection_manager::{PROVIDER_END_CLOSED, PROVIDER_END_ERROR, PROVIDER_END_NOT_SET}, + open_provider_stream_with_lifecycle, AppState, BoxedProviderStream, ProviderStreamOpen, + ProviderStreamOpenLifecycle, StreamError, TimedClientStream, +}; +use bytes::Bytes; +use futures::{Future, Stream, StreamExt}; +use log::{error, info}; +use shared::{model::VirtualId, utils::sanitize_sensitive_info}; +use std::{ + net::SocketAddr, + pin::Pin, + sync::{atomic::Ordering, Arc}, + task::{Context, Poll}, +}; + +impl ActiveClientStreamState { + /// Drops the provider handle at most once (releasing the provider slot + /// synchronously) and detaches the provider body stream. The stream claim itself + /// is released separately through [`Self::release_user_stream`]. + pub(super) fn release_provider_handle_and_detach_body(&mut self) { + self.inner = None; + if !self.provider_handle_released { + self.provider_handle_released = true; + drop(self.provider_handle.take()); + } + } + + pub(super) fn release_user_stream(&mut self) { + if self.user_stream_released { + return; + } + self.user_stream_released = true; + if let Some(cleanup) = self.request_cleanup.as_mut() { + cleanup.finish( + self.lease_request_id, + self.provider_end_reason.load(Ordering::Relaxed), + self.provider_reconnect_count.load(Ordering::Relaxed), + self.provider_error_class, + self.provider_http_status, + ); + } + } + + /// Confirms the playback lease exactly once, when the first real provider media + /// bytes reach the client. Only a confirmed lease may reserve provider capacity + /// against other playbacks, so an abandoned or custom-video-only start does not + /// block unrelated clients behind the same reverse proxy. + fn confirm_lease_once(&mut self) { + if self.lease_confirmed { + return; + } + let Some(owner) = self.lease_owner.take() else { + return; + }; + self.lease_confirmed = true; + if let Some(media_started) = &self.media_started { + media_started.store(true, Ordering::Release); + } + let request_id = self.lease_request_id.or_else(|| { + self.provider_handle.as_ref().and_then(|managed| managed.handle()).and_then(|h| h.playback_request_id) + }); + // Confirm synchronously through the broker: a first-byte confirmation is a + // mandatory capacity transition and must not be dropped by queue pressure. + let provider_manager = &self.connection_manager.provider_manager; + if let Some(request_id) = request_id { + provider_manager.confirm_identified_playback_activity(&owner, request_id); + } else { + provider_manager.confirm_playback_activity(&owner); + } + } + + pub(super) fn release_stream_and_provider_handle_once(&mut self) { + self.stop_grace_task(); + self.release_provider_handle_and_detach_body(); + self.release_user_stream(); + } + + fn mark_direct_body_progress(&mut self) { + if self.direct_body_idle_timeout.mark_progress() { + self.connection_manager.touch_direct_body_activity(&self.fingerprint.addr); + } + } + + /// Ends the response body without switching to a custom error video. Used for a + /// suppressed reentry retry, where a visible "connections exhausted" video would be + /// misleading: the request was declined by the reentry guard, not by a real limit. + pub(super) fn terminate_quietly(&mut self) { + self.provider_stopped = true; + self.preempt_cancelled = None; + self.stop_grace_task(); + self.release_provider_handle_and_detach_body(); + self.release_user_stream(); + } +} + +pub(super) fn wrap_timed_client_stream_if_needed( + app_state: &Arc, + stream: BoxedProviderStream, + addr: SocketAddr, + virtual_id: VirtualId, + stream_uid: Option, +) -> BoxedProviderStream { + let config = app_state.app_config.config.load(); + match config.sleep_timer_mins { + None => stream, + Some(mins) => { + let secs = u32::try_from((u64::from(mins) * 60).min(u64::from(u32::MAX))).unwrap_or(0); + if secs > 0 { + TimedClientStream::new( + &app_state.app_config, + &app_state.connection_manager, + stream, + secs, + addr, + virtual_id, + stream_uid, + ) + .boxed() + } else { + stream + } + } + } +} + +pub(super) fn create_timed_stream_context( + app_state: &Arc, + virtual_id: VirtualId, +) -> Option { + let config = app_state.app_config.config.load(); + let mins = config.sleep_timer_mins?; + let duration_secs = u32::try_from((u64::from(mins) * 60).min(u64::from(u32::MAX))).unwrap_or(0); + (duration_secs > 0).then(|| TimedStreamContext { app_state: Arc::clone(app_state), duration_secs, virtual_id }) +} + +impl Stream for ActiveClientStream { + type Item = Result; + + #[allow(clippy::too_many_lines)] + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + loop { + // 1. Preemption check (user priority feature) + if let Some(fut) = self.state.preempt_cancelled.as_mut() { + if fut.as_mut().poll(cx).is_ready() && !self.state.stop_provider_stream_preempted() { + return Poll::Ready(None); + } + } + + // 2. Grace task lifecycle management + waker registration + self.state.clear_finished_grace_task(); + if let Some(waker) = &self.state.waker { + waker.register(cx.waker()); + } + + // 3. Read atomic mode flag (set by grace task or stop_provider_stream) + let Some(mode) = (match &self.state.send_custom_stream_flag { + Some(flag) => StreamMode::try_from(flag.load(Ordering::Acquire)).ok(), + None => Some(StreamMode::Inner), + }) else { + // An unknown mode can only come from a corrupted flag. Never map it + // onto a live state such as `GracePending`; end the body safely. + debug_assert!(false, "unknown stream mode flag"); + error!( + "Unknown stream mode for {}, terminating stream", + sanitize_sensitive_info(&self.state.fingerprint.addr.to_string()) + ); + self.state.terminate_quietly(); + return Poll::Ready(None); + }; + + // Dispatch based on the current streaming phase. + match mode { + // Grace period: hold_stream=true, waiting for grace task to resolve + StreamMode::GracePending => { + self.state.reset_custom_video_timeout(); + return Poll::Pending; + } + + // Live streaming: forward bytes from upstream provider + StreamMode::Inner => { + self.state.reset_custom_video_timeout(); + + if self.state.inner.is_none() { + if let Some(deferred_provider_open) = self.state.deferred_provider_open.take() { + match deferred_provider_open { + DeferredProviderOpenState::Pending(context) => { + let app_state = Arc::clone(&context.app_state); + let client = { + let http_client = app_state.http_clients.default.load(); + http_client.as_ref().clone() + }; + let lifecycle = self + .state + .provider_handle + .as_ref() + .and_then(ProviderStreamOpenLifecycle::from_managed); + let future = Box::pin(async move { + match open_provider_stream_with_lifecycle( + &app_state.provider_stream_ctx(), + &client, + context.provider_stream_factory_options, + lifecycle, + ) + .await + { + Some(ProviderStreamOpen::Stream(response)) + if matches!(response.info, Some((_, _, _, Some(_)))) => + { + let Some((_headers, _status, _response_url, Some(custom_video_type))) = + response.info + else { + return DeferredProviderOpenOutcome::Failed; + }; + ActiveClientStreamState::mode_for_custom_video_type(custom_video_type) + .map_or( + DeferredProviderOpenOutcome::Failed, + DeferredProviderOpenOutcome::Mode, + ) + } + Some(ProviderStreamOpen::Stream(response)) => { + DeferredProviderOpenOutcome::Stream(response.stream) + } + Some(ProviderStreamOpen::UpstreamStatus { .. }) | None => { + DeferredProviderOpenOutcome::Failed + } + } + }); + self.state.deferred_provider_open = + Some(DeferredProviderOpenState::Opening(future)); + continue; + } + DeferredProviderOpenState::Opening(mut future) => match future.as_mut().poll(cx) { + Poll::Pending => { + self.state.deferred_provider_open = + Some(DeferredProviderOpenState::Opening(future)); + return Poll::Pending; + } + Poll::Ready(DeferredProviderOpenOutcome::Stream(stream)) => { + self.state.provider_reconnect_count.fetch_add(1, Ordering::Relaxed); + self.state.inner = Some(self.state.wrap_provider_stream(stream)); + self.state.mark_direct_body_progress(); + continue; + } + Poll::Ready(DeferredProviderOpenOutcome::Mode(mode)) => { + if !self.state.enter_custom_mode(mode) { + return Poll::Ready(None); + } + continue; + } + Poll::Ready(DeferredProviderOpenOutcome::Failed) => { + if !self.state.enter_custom_mode(StreamMode::ChannelUnavailable) { + return Poll::Ready(None); + } + continue; + } + }, + } + } + + if self.state.grace_task_handle.is_none() { + self.state.stop_provider_stream(StreamMode::ChannelUnavailable); + return Poll::Ready(None); + } + + return Poll::Pending; + } + + match self.state.inner.as_mut().map(|inner| Pin::new(inner).poll_next(cx)) { + Some(Poll::Ready(Some(Ok(bytes)))) => { + if !bytes.is_empty() { + self.state.confirm_lease_once(); + } + self.state.mark_direct_body_progress(); + return Poll::Ready(Some(Ok(bytes))); + } + Some(Poll::Ready(Some(Err(e)))) => { + error!("Inner stream error: {e:?}"); + self.state.provider_error_class = Some(e.provider_error_class()); + self.state.provider_http_status = e.provider_http_status(); + let _ = self.state.provider_end_reason.compare_exchange( + PROVIDER_END_NOT_SET, + PROVIDER_END_ERROR, + Ordering::Relaxed, + Ordering::Relaxed, + ); + if self.state.grace_task_handle.is_none() { + self.state.stop_provider_stream(StreamMode::ChannelUnavailable); + return Poll::Ready(None); + } + + return Poll::Pending; + } + Some(Poll::Ready(None)) | None => { + let _ = self.state.provider_end_reason.compare_exchange( + PROVIDER_END_NOT_SET, + PROVIDER_END_CLOSED, + Ordering::Relaxed, + Ordering::Relaxed, + ); + if self.state.grace_task_handle.is_none() { + self.state.stop_provider_stream(StreamMode::ChannelUnavailable); + return Poll::Ready(None); + } + + return Poll::Pending; + } + Some(Poll::Pending) => { + if self.state.direct_body_idle_timeout.poll_expired(cx) { + self.state.stop_direct_body_idle_timeout(); + return Poll::Ready(None); + } + return Poll::Pending; + } + } + } + + // Quiet termination: the reentry guard declined a retry of a recently + // evicted playback. End the body without a user-visible error video. + StreamMode::ReentrySuppressed => { + info!( + "Suppressing reentry retry for {}, terminating stream", + sanitize_sensitive_info(&self.state.fingerprint.addr.to_string()) + ); + self.state.terminate_quietly(); + return Poll::Ready(None); + } + + // Custom video modes: serve the appropriate buffer + video_mode => { + if self.state.custom_video_timeout_mode != Some(video_mode) + && !self.state.enter_custom_mode(video_mode) + { + return Poll::Ready(None); + } + + if self.state.custom_video_timed_out(cx, video_mode) { + info!( + "Custom video {video_mode:?} timed out for {}, terminating stream", + sanitize_sensitive_info(&self.state.fingerprint.addr.to_string()) + ); + return Poll::Ready(None); + } + + let is_provisioning = video_mode == StreamMode::Provisioning && self.state.provisionable; + + let buffer_opt = match video_mode { + StreamMode::UserExhausted => self.state.custom_video.user_exhausted.as_mut(), + StreamMode::ProviderExhausted => self.state.custom_video.provider_exhausted.as_mut(), + StreamMode::ChannelUnavailable => self.state.custom_video.unavailable.as_mut(), + StreamMode::Provisioning => self.state.custom_video.provisioning.as_mut(), + StreamMode::LowPriorityPreempted => self.state.custom_video.low_priority_preempted.as_mut(), + _ => None, + }; + + if let Some(buffer) = buffer_opt { + buffer.register_waker(cx.waker()); + if let Some(chunk) = buffer.next_chunk() { + return Poll::Ready(Some(Ok(chunk))); + } + + // Provisioning loops until preemption fires; all others terminate. + if is_provisioning { + return Poll::Pending; + } + + info!( + "Custom video {video_mode:?} buffer exhausted for {}, terminating stream", + sanitize_sensitive_info(&self.state.fingerprint.addr.to_string()) + ); + return Poll::Ready(None); + } + + // No custom video configured for this mode -> terminate immediately. + info!( + "No custom video configured for {video_mode:?} mode for {}, terminating stream", + sanitize_sensitive_info(&self.state.fingerprint.addr.to_string()) + ); + return Poll::Ready(None); + } + } + } + } +} + +impl Drop for ActiveClientStream { + fn drop(&mut self) { self.state.release_stream_and_provider_handle_once(); } +} diff --git a/backend/app/src/api/model/streams/active_client_stream/tests.rs b/backend/app/src/api/model/streams/active_client_stream/tests.rs new file mode 100644 index 000000000..317b8827a --- /dev/null +++ b/backend/app/src/api/model/streams/active_client_stream/tests.rs @@ -0,0 +1,18 @@ +use super::{ + super::buffered_stream::BufferedStream, create_active_client_stream, create_deferred_provider_open_future, + should_use_direct_body_idle_timeout, stream_grace_period, ActiveClientStream, ActiveClientStreamParams, + ActiveClientStreamState, CustomVideoBuffers, DeferredProviderOpenOutcome, DeferredProviderOpenState, + DirectBodyIdleTimeout, GracePeriodParams, StreamMode, TimedStreamContext, +}; + +mod admission; +mod behavior; +mod configuration; +mod lifecycle; +mod support; + +use self::support::{ + assert_missing_custom_video_terminates, create_deferred_provider_grace_details, create_test_app_config, + create_test_app_state, create_test_connection_manager, create_test_fingerprint, create_test_stream_channel, + create_test_user, custom_video_test_state, +}; diff --git a/backend/app/src/api/model/streams/active_client_stream/tests/admission.rs b/backend/app/src/api/model/streams/active_client_stream/tests/admission.rs new file mode 100644 index 000000000..e1d9bc9c6 --- /dev/null +++ b/backend/app/src/api/model/streams/active_client_stream/tests/admission.rs @@ -0,0 +1,1119 @@ +use super::{ + assert_missing_custom_video_terminates, create_active_client_stream, create_deferred_provider_grace_details, + create_test_app_config, create_test_app_state, create_test_fingerprint, create_test_stream_channel, + create_test_user, stream_grace_period, ActiveClientStreamParams, ActiveClientStreamState, GracePeriodParams, + StreamMode, +}; +use crate::{ + api::model::{ + ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, ConnectionManager, CreateUserSessionParams, + CustomVideoStreamType, EventManager, GraceResolutionContext, MetadataUpdateManager, PlaylistStorageState, + ProviderContentRepresentationMode, RecordingQueue, SharedStreamManager, StreamDetails, + }, + model::{Config, GracePeriodOptions, StreamConfig}, + repository::GeoIp, +}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use axum::http::HeaderMap; +use futures::{pin_mut, StreamExt}; +use shared::{ + model::{AdmissionStrategy, StreamChannel, UserConnectionPermission, VirtualId}, + utils::Internable, +}; +use std::{ + sync::{ + atomic::{AtomicU8, Ordering}, + Arc, + }, + time::Duration, +}; + +pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_app_state_with_stream_config( + stream: StreamConfig, +) -> Arc { + let config = Config { + reverse_proxy: Some(crate::model::ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: crate::model::ResourceRetryConfig::default(), + disabled_header: None, + stream: Some(stream), + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }), + user_access_control: true, + ..Config::default() + }; + + let mut app_cfg = create_test_app_config(); + app_cfg.config = Arc::new(ArcSwap::from_pointee(config)); + + let event_manager = Arc::new(EventManager::new()); + let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); + active_provider.set_shared_stream_manager(&shared_stream_manager); + + let geoip = Arc::new(ArcSwapOption::::default()); + let config_loaded = app_cfg.config.load(); + let active_users = Arc::new(ActiveUserManager::new(&config_loaded, &geoip, &event_manager)); + let connection_manager = + Arc::new(ConnectionManager::new(&active_users, &active_provider, &shared_stream_manager, &event_manager, None)); + + let tokens = CancelTokens::default(); + let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); + + Arc::new(AppState { + recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( + Arc::clone(&active_provider), + Arc::clone(&connection_manager), + ), + app_config: Arc::new(app_cfg), + http_clients: Arc::default(), + recordings: Arc::new(RecordingQueue::new()), + cache: Arc::new(ArcSwapOption::default()), + shared_stream_manager, + hls: crate::api::model::HlsState::new(Arc::new(crate::api::model::HlsProxyManager::new())), + stalker_resolve_coordinator: crate::api::model::StalkerResolveCoordinator::default(), + active_users, + active_provider, + connection_manager, + event_manager, + cancel_tokens: ArcSwap::from_pointee(tokens), + playlists: Arc::new(PlaylistStorageState::new()), + geoip, + metadata_manager, + auth: crate::api::model::AuthState::for_tests(), + playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), + }) +} + +pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_shared_stream_channel( + virtual_id: u32, + url: &str, +) -> StreamChannel { + let mut channel = create_test_stream_channel(virtual_id, url); + channel.shared = true; + channel +} + +pub(in crate::api::model::streams::active_client_stream::tests) async fn start_deferred_provider_grace_resolution( + app_state: &Arc, + provider_name: &Arc, + deferred_addr: std::net::SocketAddr, + session_token: Option<&str>, +) -> (Arc, tokio::task::JoinHandle<()>, crate::api::model::ProviderHandle) { + let deferred_handle = app_state + .active_provider + .acquire_exact_connection_with_grace( + provider_name, + &deferred_addr, + true, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .expect("deferred client should receive provider grace allocation"); + let mut stream_details = create_deferred_provider_grace_details( + provider_name, + tuliprox_session::ManagedProviderHandle::new( + Arc::clone(&app_state.connection_manager.provider_manager), + deferred_handle, + ), + ); + // The grace-resolution task only reads Copy fields of `stream_details` and never + // takes ownership of the handle, so disarm it back into the raw handle the caller + // releases explicitly after the grace task completes. + let deferred_provider_handle = stream_details + .provider_handle + .take() + .and_then(|mut managed| managed.disarm()) + .expect("deferred provider handle must be retained during grace"); + let test_user = create_test_user("grace-user"); + let test_fingerprint = create_test_fingerprint(deferred_addr); + let pending_provider_version = if let Some(token) = session_token { + app_state.active_users.pending_provider_version(&test_user.username, token).await + } else { + None + }; + let (flag, task) = stream_grace_period(GracePeriodParams { + app_state: Arc::clone(app_state), + stream_details, + user_grace_period: false, + user: test_user, + fingerprint: test_fingerprint, + virtual_id: VirtualId::new(1), + session_token: session_token.map(str::to_string), + provisioning_info: None, + waker: None, + hold_stream: true, + capacity_notify: app_state.connection_manager.capacity_notified(), + pending_provider_version, + grace_active_version: None, + grace_resolution_context: None, + grace_kind: None, + socket_bound: false, + shared_subscriber_id: None, + }); + ( + flag.expect("provider grace should install a mode flag"), + task.expect("provider grace should spawn a grace-resolution task"), + deferred_provider_handle, + ) +} + +#[test] +fn test_custom_video_type_mapping_for_grace_modes() { + assert_eq!( + ActiveClientStreamState::custom_video_type_for_mode(StreamMode::UserExhausted), + Some(CustomVideoStreamType::UserConnectionsExhausted) + ); + assert_eq!( + ActiveClientStreamState::custom_video_type_for_mode(StreamMode::ProviderExhausted), + Some(CustomVideoStreamType::ProviderConnectionsExhausted) + ); + assert_eq!( + ActiveClientStreamState::custom_video_type_for_mode(StreamMode::Provisioning), + Some(CustomVideoStreamType::Provisioning) + ); + assert_eq!( + ActiveClientStreamState::custom_video_type_for_mode(StreamMode::LowPriorityPreempted), + Some(CustomVideoStreamType::LowPriorityPreempted) + ); + assert_eq!( + ActiveClientStreamState::custom_video_type_for_mode(StreamMode::ChannelUnavailable), + Some(CustomVideoStreamType::ChannelUnavailable) + ); + // A suppressed reentry must never map to a user-visible error clip. + assert_eq!(ActiveClientStreamState::custom_video_type_for_mode(StreamMode::ReentrySuppressed), None); + assert_eq!(ActiveClientStreamState::custom_video_type_for_mode(StreamMode::Inner), None); + assert_eq!(ActiveClientStreamState::custom_video_type_for_mode(StreamMode::GracePending), None); +} + +#[tokio::test] +async fn test_low_priority_preempted_without_custom_video_terminates_immediately() { + assert_missing_custom_video_terminates(StreamMode::LowPriorityPreempted, false).await; +} + +#[tokio::test(start_paused = true)] +async fn test_provider_grace_resolution_transitions_from_grace_pending_to_inner_when_capacity_notify_arrives() { + let app_state = create_test_app_state(); + let provider_name = "provider_1".intern(); + let holder_addr = "127.0.0.1:55010".parse().unwrap_or_else(|_| unreachable!()); + let deferred_addr = "127.0.0.1:55011".parse().unwrap_or_else(|_| unreachable!()); + + let holder_handle = app_state + .active_provider + .acquire_exact_connection_with_grace( + &provider_name, + &holder_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .expect("holder should consume the provider's live capacity"); + let (flag, grace_task, deferred_handle) = + start_deferred_provider_grace_resolution(&app_state, &provider_name, deferred_addr, None).await; + + assert_eq!( + StreamMode::try_from(flag.load(Ordering::Acquire)).unwrap(), + StreamMode::GracePending, + "provider grace resolution must begin in GracePending while provider capacity is exhausted" + ); + + app_state.connection_manager.release_provider_handle(Some(holder_handle)); + let join_result = tokio::time::timeout(Duration::from_millis(1), grace_task).await; + + assert!(join_result.is_ok(), "provider grace resolution stayed pending after capacity_notify should have fired"); + assert_eq!( + StreamMode::try_from(flag.load(Ordering::Acquire)).unwrap(), + StreamMode::Inner, + "capacity-notify should resolve provider grace from GracePending to Inner before the deadline" + ); + + app_state.connection_manager.release_provider_handle(Some(deferred_handle)); +} + +#[tokio::test(start_paused = true)] +async fn test_provider_grace_resolution_clears_pending_provider_on_capacity_notify() { + let app_state = create_test_app_state(); + let provider_name = "provider_1".intern(); + let holder_addr = "127.0.0.1:55024".parse().unwrap_or_else(|_| unreachable!()); + let deferred_addr = "127.0.0.1:55025".parse().unwrap_or_else(|_| unreachable!()); + let user = create_test_user("grace-user"); + + let _ = app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-grace", + virtual_id: 1, + provider: provider_name.as_ref(), + stream_url: "http://provider-1.example/live/1.ts", + addr: &deferred_addr, + connection_permission: UserConnectionPermission::GracePeriod, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + let _ = app_state + .active_users + .mark_pending_provider(&user.username, "tok-grace", crate::api::model::PendingProviderReason::GraceHold, 9_999) + .await; + + let holder_handle = app_state + .active_provider + .acquire_exact_connection_with_grace( + &provider_name, + &holder_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .expect("holder should consume the provider's live capacity"); + let (_flag, grace_task, deferred_handle) = + start_deferred_provider_grace_resolution(&app_state, &provider_name, deferred_addr, Some("tok-grace")).await; + + app_state.connection_manager.release_provider_handle(Some(holder_handle)); + let join_result = tokio::time::timeout(Duration::from_millis(1), grace_task).await; + assert!(join_result.is_ok(), "grace task should finish after capacity notify"); + + let session = app_state + .active_users + .get_and_update_user_session(&user.username, "tok-grace") + .await + .expect("session should still exist"); + assert!( + !matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. }), + "capacity notify should clear pending provider state" + ); + assert_eq!(session.permission, UserConnectionPermission::Allowed); + + app_state.connection_manager.release_provider_handle(Some(deferred_handle)); +} + +#[tokio::test(start_paused = true)] +async fn test_provider_grace_resolution_ignores_stale_pending_version_on_capacity_notify() { + let app_state = create_test_app_state(); + let provider_name = "provider_1".intern(); + let holder_addr = "127.0.0.1:55026".parse().unwrap_or_else(|_| unreachable!()); + let deferred_addr = "127.0.0.1:55027".parse().unwrap_or_else(|_| unreachable!()); + let user = create_test_user("grace-user"); + + let _ = app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-grace-stale", + virtual_id: 1, + provider: provider_name.as_ref(), + stream_url: "http://provider-1.example/live/1.ts", + addr: &deferred_addr, + connection_permission: UserConnectionPermission::GracePeriod, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + let _ = app_state + .active_users + .mark_pending_provider( + &user.username, + "tok-grace-stale", + crate::api::model::PendingProviderReason::GraceHold, + 9_999, + ) + .await; + + let holder_handle = app_state + .active_provider + .acquire_exact_connection_with_grace( + &provider_name, + &holder_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .expect("holder should consume the provider's live capacity"); + let (_flag, grace_task, deferred_handle) = + start_deferred_provider_grace_resolution(&app_state, &provider_name, deferred_addr, Some("tok-grace-stale")) + .await; + + let replacement_version = app_state + .active_users + .mark_pending_provider( + &user.username, + "tok-grace-stale", + crate::api::model::PendingProviderReason::GraceHold, + 10_500, + ) + .await + .expect("replacement pending version should be created"); + + app_state.connection_manager.release_provider_handle(Some(holder_handle)); + let join_result = tokio::time::timeout(Duration::from_millis(1), grace_task).await; + assert!(join_result.is_ok(), "grace task should finish after capacity notify"); + + let session = app_state + .active_users + .get_and_update_user_session(&user.username, "tok-grace-stale") + .await + .expect("session should still exist"); + let crate::api::model::PlaybackLifecycle::PendingProvider { data: pending } = &session.lifecycle else { + panic!("stale grace task must not clear the replacement pending provider state") + }; + assert_eq!(pending.version, replacement_version); + assert!(pending.wake_source.is_none()); + assert_eq!(session.permission, UserConnectionPermission::GracePeriod); + + app_state.connection_manager.release_provider_handle(Some(deferred_handle)); +} + +#[tokio::test(start_paused = true)] +async fn test_active_client_stream_deferred_provider_grace_retains_provider_handle_while_grace_pending() { + let app_state = create_test_app_state(); + let provider_name = "provider_1".intern(); + let holder_addr = "127.0.0.1:55012".parse().unwrap_or_else(|_| unreachable!()); + let deferred_addr = "127.0.0.1:55013".parse().unwrap_or_else(|_| unreachable!()); + let third_addr = "127.0.0.1:55014".parse().unwrap_or_else(|_| unreachable!()); + + let holder_handle = app_state + .active_provider + .acquire_exact_connection_with_grace( + &provider_name, + &holder_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .expect("holder should consume the provider's live capacity"); + let deferred_handle = app_state + .active_provider + .acquire_exact_connection_with_grace( + &provider_name, + &deferred_addr, + true, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .expect("deferred client should receive provider grace allocation"); + let stream_details = create_deferred_provider_grace_details( + &provider_name, + tuliprox_session::ManagedProviderHandle::new( + Arc::clone(&app_state.connection_manager.provider_manager), + deferred_handle.clone(), + ), + ); + let test_user = create_test_user("grace-user"); + let test_fingerprint = create_test_fingerprint(deferred_addr); + let stream = create_active_client_stream(ActiveClientStreamParams { + stream_details, + app_state: &app_state, + user: &test_user, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: crate::api::model::ConnectionKind::Normal, + fingerprint: &test_fingerprint, + stream_channel: create_test_stream_channel(1, "http://provider-1.example/live/1"), + socket_bound: true, + session_token: None, + req_headers: &HeaderMap::default(), + meter_uid: 0, + meter_stream: false, + }) + .await + .expect("deferred test stream admission should succeed"); + pin_mut!(stream); + + assert!( + matches!(futures::poll!(stream.next()), std::task::Poll::Pending), + "deferred active-client-stream should park in GracePending while waiting for provider grace resolution" + ); + + let third_handle = app_state.active_provider.acquire_exact_connection_with_grace( + &provider_name, + &third_addr, + true, + 0, + crate::api::model::ConnectionKind::Normal, + ); + + assert!( + third_handle.is_none(), + "deferred active-client-stream should retain the deferred provider grace reservation while GracePending" + ); + + app_state.connection_manager.release_provider_handle(Some(holder_handle)); + app_state.connection_manager.release_provider_handle(Some(deferred_handle)); +} + +#[tokio::test(start_paused = true)] +async fn test_active_client_stream_shared_deferred_provider_grace_retains_provider_handle_while_grace_pending() { + let app_state = create_test_app_state(); + let provider_name = "provider_1".intern(); + let holder_addr = "127.0.0.1:55017".parse().unwrap_or_else(|_| unreachable!()); + let deferred_addr = "127.0.0.1:55018".parse().unwrap_or_else(|_| unreachable!()); + let third_addr = "127.0.0.1:55019".parse().unwrap_or_else(|_| unreachable!()); + + let holder_handle = app_state + .active_provider + .acquire_exact_connection_with_grace( + &provider_name, + &holder_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .expect("holder should consume the provider's live capacity"); + let deferred_handle = app_state + .active_provider + .acquire_exact_connection_with_grace( + &provider_name, + &deferred_addr, + true, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .expect("deferred shared client should receive provider grace allocation"); + let stream_details = create_deferred_provider_grace_details( + &provider_name, + tuliprox_session::ManagedProviderHandle::new( + Arc::clone(&app_state.connection_manager.provider_manager), + deferred_handle.clone(), + ), + ); + let test_user = create_test_user("grace-user"); + let test_fingerprint = create_test_fingerprint(deferred_addr); + let stream = create_active_client_stream(ActiveClientStreamParams { + stream_details, + app_state: &app_state, + user: &test_user, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: crate::api::model::ConnectionKind::Normal, + fingerprint: &test_fingerprint, + stream_channel: create_test_shared_stream_channel(1, "http://provider-1.example/live/1"), + socket_bound: true, + session_token: None, + req_headers: &HeaderMap::default(), + meter_uid: 0, + meter_stream: false, + }) + .await + .expect("shared deferred test stream admission should succeed"); + pin_mut!(stream); + + assert!( + matches!(futures::poll!(stream.next()), std::task::Poll::Pending), + "shared deferred active-client-stream should stay pending instead of returning an empty stream" + ); + + let third_handle = app_state.active_provider.acquire_exact_connection_with_grace( + &provider_name, + &third_addr, + true, + 0, + crate::api::model::ConnectionKind::Normal, + ); + + assert!( + third_handle.is_none(), + "shared deferred active-client-stream should retain the deferred provider grace reservation while pending" + ); + + app_state.connection_manager.release_provider_handle(Some(holder_handle)); + app_state.connection_manager.release_provider_handle(Some(deferred_handle)); +} + +#[tokio::test(start_paused = true)] +async fn test_provider_grace_resolution_transitions_from_grace_pending_to_provider_exhausted_at_deadline() { + let app_state = create_test_app_state(); + let provider_name = "provider_1".intern(); + let holder_addr = "127.0.0.1:55015".parse().unwrap_or_else(|_| unreachable!()); + let deferred_addr = "127.0.0.1:55016".parse().unwrap_or_else(|_| unreachable!()); + let user = create_test_user("grace-user"); + + let _ = app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-grace-timeout", + virtual_id: 1, + provider: provider_name.as_ref(), + stream_url: "http://provider-1.example/live/1.ts", + addr: &deferred_addr, + connection_permission: UserConnectionPermission::GracePeriod, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + let _ = app_state + .active_users + .mark_pending_provider( + &user.username, + "tok-grace-timeout", + crate::api::model::PendingProviderReason::GraceHold, + 9_999, + ) + .await; + + let holder_handle = app_state + .active_provider + .acquire_exact_connection_with_grace( + &provider_name, + &holder_addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .expect("holder should consume the provider's live capacity"); + let (flag, grace_task, deferred_handle) = + start_deferred_provider_grace_resolution(&app_state, &provider_name, deferred_addr, Some("tok-grace-timeout")) + .await; + + assert_eq!( + StreamMode::try_from(flag.load(Ordering::Acquire)).unwrap(), + StreamMode::GracePending, + "provider grace resolution must begin in GracePending while provider capacity is exhausted" + ); + + tokio::time::advance(Duration::from_millis(101)).await; + let task_result = grace_task.await; + + assert!( + task_result.is_ok(), + "grace-resolution task should complete once the deadline expires without capacity becoming available" + ); + assert_eq!( + StreamMode::try_from(flag.load(Ordering::Acquire)).unwrap(), + StreamMode::ProviderExhausted, + "provider grace resolution should transition from GracePending to ProviderExhausted when the deadline expires" + ); + + let session = app_state + .active_users + .get_and_update_user_session(&user.username, "tok-grace-timeout") + .await + .expect("session should still exist"); + assert!( + !matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::PendingProvider { .. }), + "timeout expiry should clear pending provider state" + ); + assert_eq!(session.permission, UserConnectionPermission::Exhausted); + + app_state.connection_manager.release_provider_handle(Some(holder_handle)); + app_state.connection_manager.release_provider_handle(Some(deferred_handle)); +} + +/// Regression test: verifies that when user-grace fails and remaining strategies are +/// exhausted, the `grace_kind` (original `ConnectionKind = Soft`) flows through to +/// `evaluate_remaining_strategies_after_grace` and the session expires correctly. +/// +/// This test exercises the full `stream_grace_period` path with: +/// - `grace_kind = Some(Soft)` (passed directly from `create_active_client_stream`) +/// - `grace_resolution_context` pointing to `GraceHoldStream` (remaining slice is empty) +/// - user-grace failure (deadline expires, user still at connection limit) +/// - remaining strategies exhausted -> `expire_pending_provider` is called +#[tokio::test(start_paused = true)] +#[allow(clippy::too_many_lines)] +async fn test_user_grace_failure_preserves_soft_kind_on_exhausted() { + // Use GraceHoldStream only, remaining slice is empty after grace exhaustion. + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 100, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream]), + }); + + let provider_name = "provider_1".intern(); + // Two addresses: first holds the counted session, second triggers grace. + let first_addr: std::net::SocketAddr = "127.0.0.1:55201".parse().unwrap_or_else(|_| unreachable!()); + let second_addr: std::net::SocketAddr = "127.0.0.1:55202".parse().unwrap_or_else(|_| unreachable!()); + let first_fingerprint = create_test_fingerprint(first_addr); + + let mut user = create_test_user("grace-soft-user"); + user.max_connections = 1; // User has 1 hard slot; first session fills it, grace session exceeds it + user.soft_connections = 0; + + // First session: counted Normal, consumes the user's only hard slot. + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-first", + virtual_id: 1, + provider: provider_name.as_ref(), + stream_url: "http://provider-1.example/live/1.ts", + addr: &first_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: "grace-soft-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 10, + fingerprint: &first_fingerprint, + provider: provider_name.clone(), + stream_channel: &create_test_stream_channel(1, "http://provider-1.example/live/1.ts"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some("tok-first"), + }) + .await; + + // Second session: will be admitted via grace (user already at limit, grace is granted). + // The grace session itself is not yet counted, so user_connections returns 1 (just the + // first session). When the grace deadline expires, the first session is still present, + // user_connections (1) > max_connections (1) is NOT true. We need the first session to + // actually consume the hard slot so that the second (grace) session causes the over-limit + // check to fire when grace expires. Since max_connections=1, the first Normal session + // uses that slot, and the grace session would push to 2 > 1 — but PendingProvider sessions + // are not counted! So we need a different approach: create a second Normal session BEFORE + // the grace session so that user_connections = 2 when grace expires. + // + // Actually, the simpler path: make the first session Soft and max_connections=1. + // A Soft session IS counted (it increments connection_data.connections but uses the soft + // counter). When grace expires, user_connections = 1 (the Soft session), which is + // NOT > max_connections = 1. So user_ok = true, no failure. + // + // The correct setup: max_connections = 1, first session = Normal (uses the hard slot), + // second session = GracePeriod (PendingProvider, not counted). When grace deadline hits, + // user_connections = 1 (Normal only), max_connections = 1, so 1 > 1 is false — no failure. + // + // We need the grace session itself to trigger the over-limit check at deadline. + // But PendingProvider sessions are not counted in user_connections! + // + // So the only way for grace to fail at deadline is if there is already a DIFFERENT + // counted session taking up the slot, AND that session is still there at deadline. + // With max_connections=1 and first session Normal: when grace expires, user_conn=1, + // max=1, so 1 > 1 is false. + // + // The solution: we need TWO already-counted sessions at deadline, not one. + // But we can't create both before the grace session because the second would also be + // admitted via grace. + // + // Instead: make the first session consume the slot AND also expire it at deadline, + // so when grace expires, user_connections = 0, which is NOT > max_connections = 1. + // + // The grace session (tok-second) is in PendingProvider state and is NOT counted. + // To trigger user-grace failure, we need user_connections > max_connections at deadline. + // With max_connections=1: we need 2 counted sessions at grace deadline. + // Solution: create two Normal sessions BEFORE the grace session: + // tok-first -> Normal, counted (consumes hard slot) + // tok-preload -> Normal, counted (exceeds max, 2 > 1) + // tok-second -> GracePeriod, PendingProvider (NOT counted, grace session) + // + // At grace deadline: user_connections = 2 (tok-first + tok-preload), max = 1. + // 2 > 1 -> user_ok = false -> user-grace failure path entered. + let preload_addr: std::net::SocketAddr = "127.0.0.1:55203".parse().unwrap_or_else(|_| unreachable!()); + let preload_fingerprint = create_test_fingerprint(preload_addr); + + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-preload", + virtual_id: 3, + provider: provider_name.as_ref(), + stream_url: "http://provider-1.example/live/3.ts", + addr: &preload_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 3, + username: "grace-soft-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 10, + fingerprint: &preload_fingerprint, + provider: provider_name.clone(), + stream_channel: &create_test_stream_channel(3, "http://provider-1.example/live/3.ts"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some("tok-preload"), + }) + .await; + + let second_fingerprint = create_test_fingerprint(second_addr); + + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-second", + virtual_id: 2, + provider: provider_name.as_ref(), + stream_url: "http://provider-1.example/live/2.ts", + addr: &second_addr, + connection_permission: UserConnectionPermission::GracePeriod, + connection_kind: Some(crate::api::model::ConnectionKind::Soft), + socket_bound: false, + }) + .await; + + let _pending_version = app_state + .active_users + .mark_pending_provider( + "grace-soft-user", + "tok-second", + crate::api::model::PendingProviderReason::GraceHold, + 9_999, + ) + .await + .expect("pending version must be created for tok-second"); + + let grace_context = GraceResolutionContext { + strategy_index: 0, + strategies: [AdmissionStrategy::GraceHoldStream].into(), + kind: Some(crate::api::model::ConnectionKind::Soft), + }; + + let pending_version = app_state + .active_users + .pending_provider_version("grace-soft-user", "tok-second") + .await + .expect("pending version must be created for tok-second"); + + let stream_details = StreamDetails { + shared_subscriber_id: None, + stream: None, + stream_info: None, + provider_name: Some(provider_name), + request_url: Some("http://provider-1.example/live/2.ts".intern()), + session_headers: None, + provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), + user_agent_stream_index: None, + grace_period: GracePeriodOptions { period_millis: 100, timeout_secs: 0, hold_stream: true }, + provider_grace_active: false, + disable_provider_grace: false, + reconnect_flag: None, + provider_handle: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + grace_resolution_context: Some(grace_context.clone()), + custom_reason: None, + response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), + session_registration: None, + }; + + let (flag, grace_task) = stream_grace_period(GracePeriodParams { + app_state: Arc::clone(&app_state), + stream_details, + user_grace_period: true, + user: user.clone(), + fingerprint: second_fingerprint.clone(), + virtual_id: VirtualId::new(2), + session_token: Some("tok-second".to_string()), + provisioning_info: None, + waker: None, + hold_stream: true, + capacity_notify: app_state.connection_manager.capacity_notified(), + pending_provider_version: Some(pending_version), + grace_active_version: None, + grace_resolution_context: Some(grace_context), + grace_kind: Some(crate::api::model::ConnectionKind::Soft), + socket_bound: false, + shared_subscriber_id: None, + }); + + // Grace should be pending initially. + assert_eq!( + StreamMode::try_from(flag.as_ref().unwrap().load(Ordering::Acquire)).unwrap(), + StreamMode::GracePending, + "user grace should start in GracePending" + ); + + // Advance time past the grace deadline. + tokio::time::advance(Duration::from_millis(101)).await; + let _ = grace_task.expect("grace task should be spawned").await; + + // Remaining strategies are exhausted (only GraceHoldStream was configured, no eviction). + // The session should expire with UserExhausted. + assert_eq!( + StreamMode::try_from(flag.as_ref().unwrap().load(Ordering::Acquire)).unwrap(), + StreamMode::UserExhausted, + "exhausted remaining strategies should result in UserExhausted" + ); + + let session = app_state + .active_users + .get_and_update_user_session("grace-soft-user", "tok-second") + .await + .expect("session must exist after grace failure"); + + // Permission should be Exhausted (not GracePeriod). + assert_eq!( + session.permission, + UserConnectionPermission::Exhausted, + "session permission should be Exhausted after grace failure with exhausted strategies" + ); + + // Lifecycle should be Expired. + assert!( + matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::Expired), + "session lifecycle should be Expired after grace failure" + ); + + // The session's original connection_kind is Soft and must NOT be modified by the + // grace failure path. The grace_kind = Soft that was passed to + // evaluate_remaining_strategies_after_grace is verified by the fact that + // the session's kind remained Soft (it was set at creation time and is preserved + // through the grace failure flow since session.connection_kind is not changed). + assert_eq!( + session.connection_kind, + Some(crate::api::model::ConnectionKind::Soft), + "session connection_kind should remain Soft (unchanged from creation)" + ); +} + +#[tokio::test(start_paused = true)] +#[allow(clippy::too_many_lines)] +async fn test_user_grace_failure_reentry_suppression_terminates_quietly() { + // GraceHoldStream admits the request; the remaining EvictUserOldest candidate + // is reentry-protected, so the grace task must resolve to ReentrySuppressed + // instead of painting the user-visible connections-exhausted video. + let app_state = create_test_app_state_with_stream_config(crate::model::StreamConfig { + retry: true, + metrics_enabled: true, + buffer: None, + grace_period_millis: 100, + grace_period_timeout_secs: 8, + grace_period_hold_stream: true, + hls_session_ttl_secs: 10, + catchup_session_ttl_secs: 10, + provider_affinity_ttl_secs: 120, + hls_wrap_media_playlist: true, + throttle_str: None, + throttle_kbps: 0, + shared_burst_buffer_mb: 1, + shared_subscriber_idle_timeout_secs: 300, + cleanup_queue_capacity: 4096, + recent_eviction_reentry_ttl: std::time::Duration::from_millis(1500), + admission_strategies: Some(vec![AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest]), + }); + + let provider_name = "provider_1".intern(); + let first_addr: std::net::SocketAddr = "127.0.0.1:55211".parse().unwrap_or_else(|_| unreachable!()); + let second_addr: std::net::SocketAddr = "127.0.0.1:55212".parse().unwrap_or_else(|_| unreachable!()); + let preload_addr: std::net::SocketAddr = "127.0.0.1:55213".parse().unwrap_or_else(|_| unreachable!()); + let first_fingerprint = create_test_fingerprint(first_addr); + let second_fingerprint = create_test_fingerprint(second_addr); + let preload_fingerprint = create_test_fingerprint(preload_addr); + + let mut user = create_test_user("grace-reentry-user"); + user.max_connections = 1; + user.soft_connections = 0; + + // Two counted Normal streams make the user over limit when grace expires. + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-first", + virtual_id: 1, + provider: provider_name.as_ref(), + stream_url: "http://provider-1.example/live/1.ts", + addr: &first_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 1, + username: "grace-reentry-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 10, + fingerprint: &first_fingerprint, + provider: provider_name.clone(), + stream_channel: &create_test_stream_channel(1, "http://provider-1.example/live/1.ts"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some("tok-first"), + }) + .await; + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-preload", + virtual_id: 3, + provider: provider_name.as_ref(), + stream_url: "http://provider-1.example/live/3.ts", + addr: &preload_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::api::model::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 3, + username: "grace-reentry-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 10, + fingerprint: &preload_fingerprint, + provider: provider_name.clone(), + stream_channel: &create_test_stream_channel(3, "http://provider-1.example/live/3.ts"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some("tok-preload"), + }) + .await; + + // Grace session on a distinct channel; its socket-bound stream row is what + // `mark_recent_eviction_guard_for_addr` keys the reentry guard on. + app_state + .active_users + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-second", + virtual_id: 2, + provider: provider_name.as_ref(), + stream_url: "http://provider-1.example/live/2.ts", + addr: &second_addr, + connection_permission: UserConnectionPermission::GracePeriod, + connection_kind: Some(crate::api::model::ConnectionKind::Soft), + socket_bound: false, + }) + .await; + app_state + .connection_manager + .update_connection(crate::api::model::ConnectionParams { + meter_uid: 2, + username: "grace-reentry-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::api::model::ConnectionKind::Normal, + priority: 0, + soft_priority: 10, + fingerprint: &second_fingerprint, + provider: provider_name.clone(), + stream_channel: &create_test_stream_channel(2, "http://provider-1.example/live/2.ts"), + user_agent: std::borrow::Cow::Borrowed("ua"), + session_token: Some("tok-second"), + }) + .await; + + // Protect the oldest candidate (channel 1) against a retry of this channel 2. + app_state.active_users.mark_recent_eviction_guard_for_addr(&second_addr, first_addr, Duration::from_secs(3)).await; + + let _pending_version = app_state + .active_users + .mark_pending_provider( + "grace-reentry-user", + "tok-second", + crate::api::model::PendingProviderReason::GraceHold, + 9_999, + ) + .await + .expect("pending version must be created for tok-second"); + + let grace_context = GraceResolutionContext { + strategy_index: 0, + strategies: [AdmissionStrategy::GraceHoldStream, AdmissionStrategy::EvictUserOldest].into(), + kind: Some(crate::api::model::ConnectionKind::Soft), + }; + let pending_version = app_state + .active_users + .pending_provider_version("grace-reentry-user", "tok-second") + .await + .expect("pending version must be created for tok-second"); + + let stream_details = StreamDetails { + shared_subscriber_id: None, + stream: None, + stream_info: None, + provider_name: Some(provider_name), + request_url: Some("http://provider-1.example/live/2.ts".intern()), + session_headers: None, + provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), + user_agent_stream_index: None, + grace_period: GracePeriodOptions { period_millis: 100, timeout_secs: 0, hold_stream: true }, + provider_grace_active: false, + disable_provider_grace: false, + reconnect_flag: None, + provider_handle: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + grace_resolution_context: Some(grace_context.clone()), + custom_reason: None, + response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), + session_registration: None, + }; + + let (flag, grace_task) = stream_grace_period(GracePeriodParams { + app_state: Arc::clone(&app_state), + stream_details, + user_grace_period: true, + user: user.clone(), + fingerprint: second_fingerprint.clone(), + virtual_id: VirtualId::new(2), + session_token: Some("tok-second".to_string()), + provisioning_info: None, + waker: None, + hold_stream: true, + capacity_notify: app_state.connection_manager.capacity_notified(), + pending_provider_version: Some(pending_version), + grace_active_version: None, + grace_resolution_context: Some(grace_context), + grace_kind: Some(crate::api::model::ConnectionKind::Soft), + socket_bound: true, + shared_subscriber_id: None, + }); + + assert_eq!( + StreamMode::try_from(flag.as_ref().unwrap().load(Ordering::Acquire)).unwrap(), + StreamMode::GracePending, + "user grace should start in GracePending" + ); + + tokio::time::advance(Duration::from_millis(101)).await; + let _ = grace_task.expect("grace task should be spawned").await; + + assert_eq!( + StreamMode::try_from(flag.as_ref().unwrap().load(Ordering::Acquire)).unwrap(), + StreamMode::ReentrySuppressed, + "reentry-protected remaining candidate must resolve to quiet termination" + ); + + let session = app_state + .active_users + .get_and_update_user_session("grace-reentry-user", "tok-second") + .await + .expect("session must exist after grace failure"); + assert!( + matches!(session.lifecycle, crate::api::model::PlaybackLifecycle::Expired), + "session lifecycle should be Expired after reentry-suppressed grace failure" + ); +} diff --git a/backend/app/src/api/model/streams/active_client_stream/tests/behavior.rs b/backend/app/src/api/model/streams/active_client_stream/tests/behavior.rs new file mode 100644 index 000000000..d6b9d83a5 --- /dev/null +++ b/backend/app/src/api/model/streams/active_client_stream/tests/behavior.rs @@ -0,0 +1,264 @@ +use super::{ + assert_missing_custom_video_terminates, create_active_client_stream, create_deferred_provider_grace_details, + create_deferred_provider_open_future, create_test_app_state, create_test_connection_manager, + create_test_fingerprint, create_test_stream_channel, create_test_user, custom_video_test_state, ActiveClientStream, + ActiveClientStreamParams, ActiveClientStreamState, CustomVideoBuffers, DeferredProviderOpenOutcome, + DeferredProviderOpenState, DirectBodyIdleTimeout, StreamMode, TimedStreamContext, +}; +use crate::{ + api::model::{ + connection_manager::PROVIDER_END_NOT_SET, EventManager, ProviderContentRepresentationMode, StreamDetails, + StreamError, + }, + auth::Fingerprint, + model::GracePeriodOptions, +}; +use axum::http::HeaderMap; +use bytes::Bytes; +use futures::{pin_mut, StreamExt}; +use shared::{ + model::{PlaylistItemType, UserConnectionPermission, VirtualId}, + utils::Internable, +}; +use std::{ + sync::{atomic::AtomicU8, Arc}, + time::Duration, +}; + +#[tokio::test] +async fn custom_response_modes_preserve_ts_fallback_and_terminate_hls_resources( +) -> Result<(), Box> { + use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer; + use tuliprox_session::stream_options::StreamResponseMode; + + let mut packet = [0xff_u8; 188]; + packet[..4].copy_from_slice(&[0x47, 0x1f, 0xff, 0x10]); + let fallback = TransportStreamBuffer::new(packet.repeat(8)); + for mode in [ + StreamMode::UserExhausted, + StreamMode::ProviderExhausted, + StreamMode::ChannelUnavailable, + StreamMode::Provisioning, + StreamMode::LowPriorityPreempted, + ] { + for response_mode in [StreamResponseMode::Stream, StreamResponseMode::HlsResource] { + let mut state = custom_video_test_state(mode, true); + state.response_mode = response_mode; + state.custom_video = CustomVideoBuffers { + user_exhausted: Some(fallback.clone()), + provider_exhausted: Some(fallback.clone()), + unavailable: Some(fallback.clone()), + provisioning: Some(fallback.clone()), + low_priority_preempted: Some(fallback.clone()), + }; + let mut stream = ActiveClientStream { state }; + let chunk = tokio::time::timeout(Duration::from_secs(1), stream.next()).await?; + if response_mode == StreamResponseMode::Stream { + let bytes = chunk.ok_or("TS fallback missing")??; + assert_eq!(bytes.first(), Some(&0x47), "{mode:?}"); + } else { + assert!(chunk.is_none(), "HLS must terminate without TS bytes in {mode:?}"); + } + } + } + Ok(()) +} + +#[test] +fn test_stream_mode_byte_values_match_the_implicit_discriminants() { + // Discriminants are implicit (declaration order) except the 255 sentinel, so + // this pins the wire mapping against accidental reordering. + assert_eq!(StreamMode::Inner as u8, 0); + assert_eq!(StreamMode::UserExhausted as u8, 1); + assert_eq!(StreamMode::ProviderExhausted as u8, 2); + assert_eq!(StreamMode::ChannelUnavailable as u8, 3); + assert_eq!(StreamMode::Provisioning as u8, 4); + assert_eq!(StreamMode::LowPriorityPreempted as u8, 5); + assert_eq!(StreamMode::ReentrySuppressed as u8, 6); + assert_eq!(StreamMode::GracePending as u8, 255); +} + +#[test] +fn test_stream_mode_try_from_rejects_unknown_values() { + assert_eq!(StreamMode::try_from(0), Ok(StreamMode::Inner)); + assert_eq!(StreamMode::try_from(6), Ok(StreamMode::ReentrySuppressed)); + assert_eq!(StreamMode::try_from(255), Ok(StreamMode::GracePending)); + // Unknown values must not be silently mapped onto GracePending. + assert_eq!(StreamMode::try_from(7), Err(7)); + assert_eq!(StreamMode::try_from(200), Err(200)); +} + +#[tokio::test] +async fn test_user_exhausted_without_custom_video_terminates_immediately() { + assert_missing_custom_video_terminates(StreamMode::UserExhausted, false).await; +} + +#[tokio::test] +async fn test_provider_exhausted_without_custom_video_terminates_immediately() { + assert_missing_custom_video_terminates(StreamMode::ProviderExhausted, false).await; +} + +#[tokio::test] +async fn test_channel_unavailable_without_custom_video_terminates_immediately() { + assert_missing_custom_video_terminates(StreamMode::ChannelUnavailable, false).await; +} + +#[tokio::test] +async fn deferred_provider_open_forces_identity_when_original_mode_preserves_origin() { + let app_state = create_test_app_state(); + let provider_name = "provider_1".intern(); + let addr = "127.0.0.1:55015".parse().unwrap(); + let provider_handle = app_state + .active_provider + .acquire_exact_connection_with_grace(&provider_name, &addr, true, 0, crate::api::model::ConnectionKind::Normal) + .expect("deferred provider allocation"); + let stream_details = create_deferred_provider_grace_details( + &provider_name, + tuliprox_session::ManagedProviderHandle::new( + Arc::clone(&app_state.connection_manager.provider_manager), + provider_handle.clone(), + ), + ); + assert_eq!(stream_details.content_representation, ProviderContentRepresentationMode::PreserveOrigin); + let fingerprint = create_test_fingerprint(addr); + let mut stream_channel = create_test_stream_channel(1, "http://provider-1.example/live/1"); + stream_channel.item_type = PlaylistItemType::Catchup; + + let deferred = create_deferred_provider_open_future( + &app_state, + &stream_details, + &fingerprint, + &stream_channel, + &HeaderMap::new(), + ) + .expect("deferred provider open context"); + + let DeferredProviderOpenState::Pending(context) = deferred else { panic!("new deferred open must start pending") }; + assert_eq!( + context.provider_stream_factory_options.content_representation(), + ProviderContentRepresentationMode::Identity + ); + assert!(!context.provider_stream_factory_options.response_head_is_available()); + drop(context); + app_state.connection_manager.release_provider_handle(Some(provider_handle)); +} + +#[tokio::test(start_paused = true)] +async fn test_active_client_stream_deferred_provider_open_applies_sleep_timer_timeout() { + let app_state = create_test_app_state(); + let connection_manager = create_test_connection_manager(); + let addr = "127.0.0.1:55020".parse().unwrap_or_else(|_| unreachable!()); + let state = ActiveClientStreamState { + response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), + inner: None, + send_custom_stream_flag: Some(Arc::new(AtomicU8::new(StreamMode::Inner as u8))), + provider_handle: None, + deferred_provider_open: Some(DeferredProviderOpenState::Opening(Box::pin(async { + DeferredProviderOpenOutcome::Stream(futures::stream::pending::>().boxed()) + }))), + timed_stream_context: Some(TimedStreamContext { app_state, duration_secs: 1, virtual_id: VirtualId::new(1) }), + preempt_cancelled: None, + grace_task_handle: None, + provisioning_stop_signal: None, + provisionable: false, + custom_video: CustomVideoBuffers { + user_exhausted: None, + provider_exhausted: None, + unavailable: None, + provisioning: None, + low_priority_preempted: None, + }, + meter: None, + event_manager: Arc::new(EventManager::new()), + waker: None, + connection_manager, + fingerprint: Arc::new(Fingerprint::new("fp-timeout".to_string(), "127.0.0.1".to_string(), addr)), + stream_uid: None, + provider_stopped: false, + user_stream_released: true, + provider_handle_released: true, + custom_video_timeout_secs: 0, + custom_video_timeout_mode: None, + custom_video_timeout_sleep: None, + direct_body_idle_timeout: DirectBodyIdleTimeout::disabled(), + provider_end_reason: AtomicU8::new(PROVIDER_END_NOT_SET), + provider_error_class: None, + provider_http_status: None, + provider_reconnect_count: AtomicU8::new(0), + lease_owner: None, + media_started: None, + lease_confirmed: false, + lease_request_id: None, + request_cleanup: None, + }; + let stream = ActiveClientStream { state }; + pin_mut!(stream); + + assert!( + matches!(futures::poll!(stream.next()), std::task::Poll::Pending), + "deferred-open success should first install the wrapped upstream stream and park pending" + ); + + tokio::time::advance(Duration::from_secs(2)).await; + + let result = tokio::time::timeout(Duration::from_millis(1), stream.next()).await; + assert!(result.is_ok(), "deferred-open stream should stop once the configured sleep timer expires"); + match result { + Ok(joined) => { + assert!(joined.is_none(), "sleep timer should terminate the deferred-open stream without yielding bytes"); + } + Err(_) => unreachable!("timeout already checked"), + } +} + +#[tokio::test(start_paused = true)] +async fn test_active_client_stream_immediate_provider_stream_emits_meter_batches() { + let app_state = create_test_app_state(); + let mut meter_events = app_state.event_manager.get_meter_channel(); + app_state.event_manager.stream_meter_subscriber_connected(); + + let addr = "127.0.0.1:55030".parse().unwrap_or_else(|_| unreachable!()); + let test_user = create_test_user("meter-user"); + let test_fingerprint = create_test_fingerprint(addr); + let provider_stream = futures::stream::iter(vec![Ok::(Bytes::from_static(&[0_u8; 3072]))]) + .chain(futures::stream::pending()) + .boxed(); + let stream_details = StreamDetails::from_stream(provider_stream, GracePeriodOptions::default()); + + let stream = create_active_client_stream(ActiveClientStreamParams { + stream_details, + app_state: &app_state, + user: &test_user, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: crate::api::model::ConnectionKind::Normal, + fingerprint: &test_fingerprint, + stream_channel: create_test_stream_channel(1, "http://provider-1.example/live/1"), + socket_bound: true, + session_token: None, + req_headers: &HeaderMap::default(), + meter_uid: 55, + meter_stream: true, + }) + .await + .expect("metered test stream admission should succeed"); + pin_mut!(stream); + + let first_chunk = stream.next().await; + assert!( + matches!(first_chunk, Some(Ok(ref bytes)) if bytes.len() == 3072), + "immediate provider stream should yield the metered payload chunk" + ); + + tokio::time::advance(Duration::from_secs(3)).await; + tokio::task::yield_now().await; + + let entries = tokio::time::timeout(Duration::from_millis(1), meter_events.recv()) + .await + .expect("immediate provider stream should publish a meter batch after bytes are sent") + .expect("meter channel should stay open while the stream is active"); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].meter_uid, 55); + assert_eq!(entries[0].uids, vec![1]); + assert_eq!(entries[0].rate_kbps, 1); + assert_eq!(entries[0].total_kb, 3); +} diff --git a/backend/app/src/api/model/streams/active_client_stream/tests/configuration.rs b/backend/app/src/api/model/streams/active_client_stream/tests/configuration.rs new file mode 100644 index 000000000..4435346a0 --- /dev/null +++ b/backend/app/src/api/model/streams/active_client_stream/tests/configuration.rs @@ -0,0 +1,6 @@ +use super::{assert_missing_custom_video_terminates, StreamMode}; + +#[tokio::test] +async fn test_provisioning_without_custom_video_terminates_immediately_with_timeout_configured() { + assert_missing_custom_video_terminates(StreamMode::Provisioning, true).await; +} diff --git a/backend/app/src/api/model/streams/active_client_stream/tests/lifecycle.rs b/backend/app/src/api/model/streams/active_client_stream/tests/lifecycle.rs new file mode 100644 index 000000000..f19092b3a --- /dev/null +++ b/backend/app/src/api/model/streams/active_client_stream/tests/lifecycle.rs @@ -0,0 +1,720 @@ +use super::{ + create_active_client_stream, create_test_app_state, create_test_fingerprint, create_test_stream_channel, + create_test_user, should_use_direct_body_idle_timeout, ActiveClientStreamParams, BufferedStream, +}; +use crate::{ + api::model::{ + AppState, BoxedProviderStream, ProviderHandle, StreamDetails, StreamError, DIRECT_BODY_IDLE_TIMEOUT_SECS, + }, + model::GracePeriodOptions, +}; +use axum::{body::Body, http::HeaderMap}; +use bytes::Bytes; +use futures::{pin_mut, StreamExt}; +use http_body_util::BodyExt; +use shared::{ + model::{PlaylistItemType, StreamChannel, UserConnectionPermission, XtreamCluster}, + utils::Internable, +}; +use std::{ + net::SocketAddr, + pin::Pin, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + task::{Context, Poll}, + time::Duration, +}; +use tokio::sync::{oneshot, Notify}; +use tokio_util::sync::CancellationToken; + +pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_video_stream_channel( + virtual_id: u32, + url: &str, +) -> StreamChannel { + let mut channel = create_test_stream_channel(virtual_id, url); + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + channel.group = "Movies".intern(); + channel.url = url.into(); + channel +} + +pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_series_stream_channel( + virtual_id: u32, + url: &str, +) -> StreamChannel { + let mut channel = create_test_stream_channel(virtual_id, url); + channel.item_type = PlaylistItemType::Series; + channel.cluster = XtreamCluster::Series; + channel.group = "Series".intern(); + channel.url = url.into(); + channel +} + +#[derive(Clone, Default)] +pub(in crate::api::model::streams::active_client_stream::tests) struct DropTracker( + pub(in crate::api::model::streams::active_client_stream::tests) Arc, +); + +impl DropTracker { + pub(in crate::api::model::streams::active_client_stream::tests) fn is_dropped(&self) -> bool { + self.0.load(Ordering::Acquire) + } +} + +pub(in crate::api::model::streams::active_client_stream::tests) struct DropTrackedProviderStream { + pub(in crate::api::model::streams::active_client_stream::tests) inner: BoxedProviderStream, + pub(in crate::api::model::streams::active_client_stream::tests) tracker: DropTracker, +} + +impl futures::Stream for DropTrackedProviderStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.inner.as_mut().poll_next(cx) + } +} + +impl Drop for DropTrackedProviderStream { + fn drop(&mut self) { self.tracker.0.store(true, Ordering::Release); } +} + +pub(in crate::api::model::streams::active_client_stream::tests) fn track_provider_stream( + stream: BoxedProviderStream, +) -> (BoxedProviderStream, DropTracker) { + let tracker = DropTracker::default(); + let tracked_stream = DropTrackedProviderStream { inner: stream, tracker: tracker.clone() }.boxed(); + (tracked_stream, tracker) +} + +pub(in crate::api::model::streams::active_client_stream::tests) struct TestDirectStreamParams<'a> { + pub(in crate::api::model::streams::active_client_stream::tests) app_state: &'a Arc, + pub(in crate::api::model::streams::active_client_stream::tests) username: &'a str, + pub(in crate::api::model::streams::active_client_stream::tests) max_connections: u32, + pub(in crate::api::model::streams::active_client_stream::tests) addr: SocketAddr, + pub(in crate::api::model::streams::active_client_stream::tests) stream_channel: StreamChannel, + pub(in crate::api::model::streams::active_client_stream::tests) provider_stream: BoxedProviderStream, + pub(in crate::api::model::streams::active_client_stream::tests) provider_handle: Option, +} + +pub(in crate::api::model::streams::active_client_stream::tests) struct TestDirectStream { + pub(in crate::api::model::streams::active_client_stream::tests) stream: BoxedProviderStream, + pub(in crate::api::model::streams::active_client_stream::tests) uid: u32, +} + +pub(in crate::api::model::streams::active_client_stream::tests) async fn create_test_active_direct_stream( + params: TestDirectStreamParams<'_>, +) -> TestDirectStream { + let TestDirectStreamParams { + app_state, + username, + max_connections, + addr, + stream_channel, + provider_stream, + provider_handle, + } = params; + let mut user = create_test_user(username); + user.max_connections = max_connections; + let fingerprint = create_test_fingerprint(addr); + let mut stream_details = StreamDetails::from_stream(provider_stream, GracePeriodOptions::default()); + if provider_handle.is_some() { + stream_details.provider_name = Some("provider_1".intern()); + } + stream_details.provider_handle = provider_handle.map(|handle| { + tuliprox_session::ManagedProviderHandle::new(Arc::clone(&app_state.connection_manager.provider_manager), handle) + }); + let virtual_id = stream_channel.virtual_id; + + let stream = create_active_client_stream(ActiveClientStreamParams { + stream_details, + app_state, + user: &user, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: crate::api::model::ConnectionKind::Normal, + fingerprint: &fingerprint, + stream_channel, + socket_bound: false, + session_token: None, + req_headers: &HeaderMap::default(), + meter_uid: 0, + meter_stream: false, + }) + .await + .expect("direct test stream admission should succeed"); + let uid = app_state + .active_users + .active_streams() + .await + .into_iter() + .find(|active| active.username == username && active.addr == addr && active.channel.virtual_id == virtual_id) + .map(|active| active.uid) + .expect("direct test stream should be registered"); + + TestDirectStream { stream, uid } +} + +pub(in crate::api::model::streams::active_client_stream::tests) fn acquire_test_provider_handle( + app_state: &Arc, + addr: SocketAddr, +) -> ProviderHandle { + app_state + .active_provider + .acquire_exact_connection_with_grace( + &"provider_1".intern(), + &addr, + false, + 0, + crate::api::model::ConnectionKind::Normal, + ) + .expect("direct test stream should acquire the provider slot") +} + +#[derive(Debug, Eq, PartialEq)] +pub(in crate::api::model::streams::active_client_stream::tests) struct TestLifecycleSnapshot { + pub(in crate::api::model::streams::active_client_stream::tests) active_counts: (usize, usize), + pub(in crate::api::model::streams::active_client_stream::tests) stream_uids: Vec, + pub(in crate::api::model::streams::active_client_stream::tests) provider_connections: usize, +} + +pub(in crate::api::model::streams::active_client_stream::tests) async fn lifecycle_snapshot( + app_state: &Arc, +) -> TestLifecycleSnapshot { + let active_counts = app_state.active_users.active_users_and_connections().await; + let mut stream_uids = + app_state.active_users.active_streams().await.into_iter().map(|stream| stream.uid).collect::>(); + stream_uids.sort_unstable(); + TestLifecycleSnapshot { + active_counts, + stream_uids, + provider_connections: app_state.active_provider.get_provider_connections_count(), + } +} + +pub(in crate::api::model::streams::active_client_stream::tests) struct ExpectedLifecycle<'a> { + pub(in crate::api::model::streams::active_client_stream::tests) description: &'static str, + pub(in crate::api::model::streams::active_client_stream::tests) active_counts: (usize, usize), + pub(in crate::api::model::streams::active_client_stream::tests) stream_uids: &'a [u32], + pub(in crate::api::model::streams::active_client_stream::tests) provider_connections: usize, + pub(in crate::api::model::streams::active_client_stream::tests) dropped_streams: &'a [&'a DropTracker], +} + +pub(in crate::api::model::streams::active_client_stream::tests) async fn wait_for_lifecycle( + app_state: &Arc, + expected: ExpectedLifecycle<'_>, +) { + let mut expected_stream_uids = expected.stream_uids.to_vec(); + expected_stream_uids.sort_unstable(); + let expected_snapshot = TestLifecycleSnapshot { + active_counts: expected.active_counts, + stream_uids: expected_stream_uids, + provider_connections: expected.provider_connections, + }; + let completed = tokio::time::timeout(Duration::from_secs(1), async { + loop { + let snapshot = lifecycle_snapshot(app_state).await; + if snapshot == expected_snapshot && expected.dropped_streams.iter().all(|tracker| tracker.is_dropped()) { + break; + } + tokio::task::yield_now().await; + } + }) + .await; + + if completed.is_err() { + let snapshot = lifecycle_snapshot(app_state).await; + let dropped = expected.dropped_streams.iter().map(|tracker| tracker.is_dropped()).collect::>(); + panic!( + "{} did not converge: expected={expected_snapshot:?}, actual={snapshot:?}, dropped={dropped:?}", + expected.description + ); + } +} + +#[test] +fn test_direct_body_idle_timeout_only_applies_to_direct_vod_series_streams() { + let video = create_test_video_stream_channel(1, "http://provider-1.example/movie/1.mkv"); + assert!(should_use_direct_body_idle_timeout(&video)); + + let mut series = create_test_video_stream_channel(2, "http://provider-1.example/series/2.mkv"); + series.item_type = PlaylistItemType::Series; + series.cluster = XtreamCluster::Series; + assert!(should_use_direct_body_idle_timeout(&series)); + + let live = create_test_stream_channel(3, "http://provider-1.example/live/3.ts"); + assert!(!should_use_direct_body_idle_timeout(&live)); + + let shared_video = { + let mut channel = video.clone(); + channel.shared = true; + channel + }; + assert!(!should_use_direct_body_idle_timeout(&shared_video)); +} + +#[tokio::test] +async fn direct_series_normal_eof_releases_full_lifecycle() { + let app_state = create_test_app_state(); + let addr = "127.0.0.1:55031".parse().unwrap_or_else(|_| unreachable!()); + let provider_handle = acquire_test_provider_handle(&app_state, addr); + let (provider_stream, tracker) = + track_provider_stream(futures::stream::once(async { Ok(Bytes::from_static(b"series-eof")) }).boxed()); + let direct = create_test_active_direct_stream(TestDirectStreamParams { + app_state: &app_state, + username: "series-eof-user", + max_connections: 1, + addr, + stream_channel: create_test_series_stream_channel(1, "http://provider-1.example/series/1.mkv"), + provider_stream, + provider_handle: Some(provider_handle), + }) + .await; + let active_uids = [direct.uid]; + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "registered Series EOF stream", + active_counts: (1, 1), + stream_uids: &active_uids, + provider_connections: 1, + dropped_streams: &[], + }, + ) + .await; + assert!(!tracker.is_dropped()); + + let body = Body::from_stream(direct.stream); + let collected = body.collect().await.expect("normal provider EOF should complete the response body"); + assert_eq!(collected.to_bytes(), Bytes::from_static(b"series-eof")); + + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "normal Series EOF cleanup", + active_counts: (0, 0), + stream_uids: &[], + provider_connections: 0, + dropped_streams: &[&tracker], + }, + ) + .await; +} + +#[tokio::test] +async fn direct_series_upstream_error_after_registration_releases_full_lifecycle() { + let app_state = create_test_app_state(); + let addr = "127.0.0.1:55032".parse().unwrap_or_else(|_| unreachable!()); + let provider_handle = acquire_test_provider_handle(&app_state, addr); + let provider_items = vec![ + Ok(Bytes::from_static(b"before-error")), + Err(StreamError::Stream("controlled upstream failure".to_string())), + ]; + let (provider_stream, tracker) = track_provider_stream(futures::stream::iter(provider_items).boxed()); + let direct = create_test_active_direct_stream(TestDirectStreamParams { + app_state: &app_state, + username: "series-error-user", + max_connections: 1, + addr, + stream_channel: create_test_series_stream_channel(2, "http://provider-1.example/series/2.mkv"), + provider_stream, + provider_handle: Some(provider_handle), + }) + .await; + let active_uids = [direct.uid]; + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "registered Series error stream", + active_counts: (1, 1), + stream_uids: &active_uids, + provider_connections: 1, + dropped_streams: &[], + }, + ) + .await; + + let body = Body::from_stream(direct.stream); + let collected = body + .collect() + .await + .expect("ActiveClientStream should convert the controlled upstream error into stream termination"); + assert_eq!(collected.to_bytes(), Bytes::from_static(b"before-error")); + + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "Series upstream error cleanup", + active_counts: (0, 0), + stream_uids: &[], + provider_connections: 0, + dropped_streams: &[&tracker], + }, + ) + .await; +} + +#[tokio::test] +async fn closed_buffered_consumer_releases_direct_series_full_lifecycle() { + let app_state = create_test_app_state(); + let addr = "127.0.0.1:55039".parse().unwrap_or_else(|_| unreachable!()); + let provider_handle = acquire_test_provider_handle(&app_state, addr); + let (gate_tx, gate_rx) = oneshot::channel(); + let gated_provider = futures::stream::once(async move { + gate_rx.await.expect("test should release the upstream chunk"); + Ok(Bytes::from_static(b"after-consumer-drop")) + }) + .chain(futures::stream::pending()) + .boxed(); + let (tracked_provider, tracker) = track_provider_stream(gated_provider); + let producer_cancel = CancellationToken::new(); + let buffered_provider = + BufferedStream::new(tracked_provider, 1, 0, producer_cancel.clone(), "controlled-test-stream").boxed(); + let direct = create_test_active_direct_stream(TestDirectStreamParams { + app_state: &app_state, + username: "series-closed-consumer-user", + max_connections: 1, + addr, + stream_channel: create_test_series_stream_channel(10, "http://provider-1.example/series/10.mkv"), + provider_stream: buffered_provider, + provider_handle: Some(provider_handle), + }) + .await; + let active_uids = [direct.uid]; + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "registered buffered Series stream", + active_counts: (1, 1), + stream_uids: &active_uids, + provider_connections: 1, + dropped_streams: &[], + }, + ) + .await; + + drop(Body::from_stream(direct.stream)); + gate_tx.send(()).expect("buffered producer should still own the gated upstream"); + + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "closed buffered Series consumer cleanup", + active_counts: (0, 0), + stream_uids: &[], + provider_connections: 0, + dropped_streams: &[&tracker], + }, + ) + .await; + assert!(producer_cancel.is_cancelled(), "closed consumer must cancel the buffered producer"); +} + +#[tokio::test] +async fn aborting_task_polling_direct_series_body_releases_full_lifecycle() { + let app_state = create_test_app_state(); + let addr = "127.0.0.1:55033".parse().unwrap_or_else(|_| unreachable!()); + let provider_handle = acquire_test_provider_handle(&app_state, addr); + let (provider_stream, tracker) = + track_provider_stream(futures::stream::pending::>().boxed()); + let direct = create_test_active_direct_stream(TestDirectStreamParams { + app_state: &app_state, + username: "series-abort-user", + max_connections: 1, + addr, + stream_channel: create_test_series_stream_channel(3, "http://provider-1.example/series/3.mkv"), + provider_stream, + provider_handle: Some(provider_handle), + }) + .await; + let active_uids = [direct.uid]; + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "registered Series task-abort stream", + active_counts: (1, 1), + stream_uids: &active_uids, + provider_connections: 1, + dropped_streams: &[], + }, + ) + .await; + + let started = Arc::new(Notify::new()); + let started_in_task = Arc::clone(&started); + let task = tokio::spawn(async move { + let mut body = Body::from_stream(direct.stream); + started_in_task.notify_one(); + let _ = body.frame().await; + }); + tokio::time::timeout(Duration::from_secs(1), started.notified()).await.expect("body polling task should start"); + task.abort(); + let join_error = tokio::time::timeout(Duration::from_secs(1), task) + .await + .expect("aborted body polling task should finish") + .expect_err("pending body polling task should be cancelled"); + assert!(join_error.is_cancelled()); + + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "aborted Series body task cleanup", + active_counts: (0, 0), + stream_uids: &[], + provider_connections: 0, + dropped_streams: &[&tracker], + }, + ) + .await; +} + +#[tokio::test] +async fn response_builder_error_after_registration_drops_direct_series_body() { + let app_state = create_test_app_state(); + let addr = "127.0.0.1:55034".parse().unwrap_or_else(|_| unreachable!()); + let provider_handle = acquire_test_provider_handle(&app_state, addr); + let (provider_stream, tracker) = + track_provider_stream(futures::stream::pending::>().boxed()); + let direct = create_test_active_direct_stream(TestDirectStreamParams { + app_state: &app_state, + username: "series-builder-error-user", + max_connections: 1, + addr, + stream_channel: create_test_series_stream_channel(4, "http://provider-1.example/series/4.mkv"), + provider_stream, + provider_handle: Some(provider_handle), + }) + .await; + let active_uids = [direct.uid]; + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "registered Series response-builder stream", + active_counts: (1, 1), + stream_uids: &active_uids, + provider_connections: 1, + dropped_streams: &[], + }, + ) + .await; + + let response = axum::response::Response::builder().status(10_000_u16).body(Body::from_stream(direct.stream)); + assert!(response.is_err(), "invalid status should fail response construction"); + + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "response-builder error cleanup", + active_counts: (0, 0), + stream_uids: &[], + provider_connections: 0, + dropped_streams: &[&tracker], + }, + ) + .await; +} + +#[tokio::test(start_paused = true)] +async fn test_direct_vod_body_idle_timeout_releases_active_stream() { + let app_state = create_test_app_state(); + let addr = "127.0.0.1:55035".parse().unwrap_or_else(|_| unreachable!()); + let provider_handle = acquire_test_provider_handle(&app_state, addr); + let pending_provider = + futures::stream::once(async { Ok(Bytes::from_static(b"vod")) }).chain(futures::stream::pending()).boxed(); + let (provider_stream, tracker) = track_provider_stream(pending_provider); + let direct = create_test_active_direct_stream(TestDirectStreamParams { + app_state: &app_state, + username: "vod-user", + max_connections: 1, + addr, + stream_channel: create_test_video_stream_channel(5, "http://provider-1.example/movie/5.mkv"), + provider_stream, + provider_handle: Some(provider_handle), + }) + .await; + let active_uids = [direct.uid]; + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "registered idle-timeout VOD stream", + active_counts: (1, 1), + stream_uids: &active_uids, + provider_connections: 1, + dropped_streams: &[], + }, + ) + .await; + let stream = direct.stream; + pin_mut!(stream); + + let first_chunk = stream.next().await; + assert!(matches!(first_chunk, Some(Ok(ref bytes)) if bytes.as_ref() == b"vod")); + assert!( + matches!(futures::poll!(stream.next()), std::task::Poll::Pending), + "pending VOD body should wait until the direct body idle timeout elapses" + ); + + tokio::time::advance(Duration::from_secs(DIRECT_BODY_IDLE_TIMEOUT_SECS)).await; + tokio::task::yield_now().await; + assert!(stream.next().await.is_none(), "VOD body idle timeout should terminate the stream"); + tokio::time::resume(); + + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "VOD body idle-timeout cleanup", + active_counts: (0, 0), + stream_uids: &[], + provider_connections: 0, + dropped_streams: &[&tracker], + }, + ) + .await; +} + +#[tokio::test] +async fn dropping_direct_series_body_releases_original_user_after_socket_owner_changes() { + let app_state = create_test_app_state(); + let addr = "127.0.0.1:55036".parse().unwrap_or_else(|_| unreachable!()); + let provider_handle = acquire_test_provider_handle(&app_state, addr); + let (first_provider_stream, first_tracker) = + track_provider_stream(futures::stream::pending::>().boxed()); + let first = create_test_active_direct_stream(TestDirectStreamParams { + app_state: &app_state, + username: "series-user-a", + max_connections: 1, + addr, + stream_channel: create_test_series_stream_channel(6, "http://provider-1.example/series/6.mkv"), + provider_stream: first_provider_stream, + provider_handle: Some(provider_handle), + }) + .await; + let (second_provider_stream, second_tracker) = + track_provider_stream(futures::stream::pending::>().boxed()); + let second = create_test_active_direct_stream(TestDirectStreamParams { + app_state: &app_state, + username: "series-user-b", + max_connections: 1, + addr, + stream_channel: create_test_series_stream_channel(7, "http://provider-1.example/series/7.mkv"), + provider_stream: second_provider_stream, + provider_handle: None, + }) + .await; + let both_uids = [first.uid, second.uid]; + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "two users sharing a reused socket address", + active_counts: (2, 2), + stream_uids: &both_uids, + provider_connections: 1, + dropped_streams: &[], + }, + ) + .await; + + let first_body = Body::from_stream(first.stream); + let second_body = Body::from_stream(second.stream); + drop(first_body); + let second_uid = [second.uid]; + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "original user Body drop after socket owner replacement", + active_counts: (1, 1), + stream_uids: &second_uid, + provider_connections: 0, + dropped_streams: &[&first_tracker], + }, + ) + .await; + assert!(!second_tracker.is_dropped()); + + drop(second_body); + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "replacement user Body drop cleanup", + active_counts: (0, 0), + stream_uids: &[], + provider_connections: 0, + dropped_streams: &[&first_tracker, &second_tracker], + }, + ) + .await; +} + +#[tokio::test] +async fn multiple_same_user_direct_series_connections_release_independently() { + let app_state = create_test_app_state(); + let first_addr = "127.0.0.1:55037".parse().unwrap_or_else(|_| unreachable!()); + let second_addr = "127.0.0.1:55038".parse().unwrap_or_else(|_| unreachable!()); + let (first_provider_stream, first_tracker) = + track_provider_stream(futures::stream::pending::>().boxed()); + let first = create_test_active_direct_stream(TestDirectStreamParams { + app_state: &app_state, + username: "multi-series-user", + max_connections: 2, + addr: first_addr, + stream_channel: create_test_series_stream_channel(8, "http://provider-1.example/series/8.mkv"), + provider_stream: first_provider_stream, + provider_handle: None, + }) + .await; + let (second_provider_stream, second_tracker) = + track_provider_stream(futures::stream::pending::>().boxed()); + let second = create_test_active_direct_stream(TestDirectStreamParams { + app_state: &app_state, + username: "multi-series-user", + max_connections: 2, + addr: second_addr, + stream_channel: create_test_series_stream_channel(9, "http://provider-1.example/series/9.mkv"), + provider_stream: second_provider_stream, + provider_handle: None, + }) + .await; + let both_uids = [first.uid, second.uid]; + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "two connections for one Series user", + active_counts: (1, 2), + stream_uids: &both_uids, + provider_connections: 0, + dropped_streams: &[], + }, + ) + .await; + + let first_body = Body::from_stream(first.stream); + let second_body = Body::from_stream(second.stream); + drop(first_body); + let second_uid = [second.uid]; + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "first same-user Series connection cleanup", + active_counts: (1, 1), + stream_uids: &second_uid, + provider_connections: 0, + dropped_streams: &[&first_tracker], + }, + ) + .await; + assert!(!second_tracker.is_dropped()); + + drop(second_body); + wait_for_lifecycle( + &app_state, + ExpectedLifecycle { + description: "last same-user Series connection cleanup", + active_counts: (0, 0), + stream_uids: &[], + provider_connections: 0, + dropped_streams: &[&first_tracker, &second_tracker], + }, + ) + .await; +} diff --git a/backend/app/src/api/model/streams/active_client_stream/tests/support.rs b/backend/app/src/api/model/streams/active_client_stream/tests/support.rs new file mode 100644 index 000000000..8135f1764 --- /dev/null +++ b/backend/app/src/api/model/streams/active_client_stream/tests/support.rs @@ -0,0 +1,248 @@ +use super::{ActiveClientStream, ActiveClientStreamState, CustomVideoBuffers, DirectBodyIdleTimeout, StreamMode}; +use crate::{ + api::model::{ + connection_manager::PROVIDER_END_NOT_SET, ActiveProviderManager, ActiveUserManager, AppState, CancelTokens, + ConnectionManager, EventManager, MetadataUpdateManager, PlaylistStorageState, + ProviderContentRepresentationMode, RecordingQueue, SharedStreamManager, StreamDetails, + }, + auth::Fingerprint, + model::{ + AppConfig, Config, ConfigInput, GracePeriodOptions, MediaToolCapabilities, ProxyUserCredentials, SourcesConfig, + }, + repository::GeoIp, + utils::FileLockManager, +}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use futures::{pin_mut, StreamExt}; +use shared::{ + model::{ConfigPaths, InputFetchMethod, InputType, PlaylistItemType, StreamChannel, XtreamCluster}, + utils::Internable, +}; +use std::{ + collections::HashMap, + sync::{atomic::AtomicU8, Arc}, +}; + +pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_app_config() -> AppConfig { + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: "http://provider-1.example".to_string(), + username: Some("user1".to_string()), + password: Some("pass1".to_string()), + enabled: true, + priority: 0, + max_connections: 1, + method: InputFetchMethod::default(), + aliases: None, + ..ConfigInput::default() + }); + let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; + + AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(sources)), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + } +} + +pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_connection_manager( +) -> Arc { + let app_cfg = create_test_app_config(); + let event_manager = Arc::new(EventManager::new()); + let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let shared_manager = Arc::new(SharedStreamManager::new(Arc::clone(&provider_manager))); + provider_manager.set_shared_stream_manager(&shared_manager); + + let geo_ip = Arc::new(ArcSwapOption::::default()); + let config = app_cfg.config.load(); + let user_manager = Arc::new(ActiveUserManager::new(&config, &geo_ip, &event_manager)); + + Arc::new(ConnectionManager::new(&user_manager, &provider_manager, &shared_manager, &event_manager, None)) +} + +pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_app_state() -> Arc { + let app_cfg = Arc::new(create_test_app_config()); + let event_manager = Arc::new(EventManager::new()); + let active_provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); + active_provider.set_shared_stream_manager(&shared_stream_manager); + + let geoip = Arc::new(ArcSwapOption::::default()); + let config = app_cfg.config.load(); + let active_users = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); + let connection_manager = + Arc::new(ConnectionManager::new(&active_users, &active_provider, &shared_stream_manager, &event_manager, None)); + + let tokens = CancelTokens::default(); + let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); + + Arc::new(AppState { + recording_capacity: crate::api::model::recording_runtime::ProviderCapacityAdapter::new( + Arc::clone(&active_provider), + Arc::clone(&connection_manager), + ), + app_config: app_cfg, + http_clients: Arc::default(), + recordings: Arc::new(RecordingQueue::new()), + cache: Arc::new(ArcSwapOption::default()), + shared_stream_manager, + hls: crate::api::model::HlsState::new(Arc::new(crate::api::model::HlsProxyManager::new())), + stalker_resolve_coordinator: crate::api::model::StalkerResolveCoordinator::default(), + active_users, + active_provider, + connection_manager, + event_manager, + cancel_tokens: ArcSwap::from_pointee(tokens), + playlists: Arc::new(PlaylistStorageState::new()), + geoip, + metadata_manager, + auth: crate::api::model::AuthState::for_tests(), + playlist_updates: crate::api::model::PlaylistUpdateControl::for_tests(), + }) +} + +pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_user( + username: &str, +) -> ProxyUserCredentials { + let mut user = ProxyUserCredentials::default(); + user.username = username.to_string(); + user.max_connections = 1; + user +} + +pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_fingerprint( + addr: std::net::SocketAddr, +) -> Fingerprint { + Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr) +} + +pub(in crate::api::model::streams::active_client_stream::tests) fn create_test_stream_channel( + virtual_id: u32, + url: &str, +) -> StreamChannel { + StreamChannel { + target_id: 1, + virtual_id, + provider_id: 1, + input_name: "input".intern(), + item_type: PlaylistItemType::Live, + cluster: XtreamCluster::Live, + group: "Live".intern(), + title: "Test Channel".intern(), + url: url.into(), + shared: false, + shared_joined_existing: None, + shared_stream_id: None, + technical: None, + epg_channel_id: None, + epg_reference_ts: None, + upstream_user_agent: None, + } +} + +pub(in crate::api::model::streams::active_client_stream::tests) fn create_deferred_provider_grace_details( + provider_name: &Arc, + provider_handle: tuliprox_session::ManagedProviderHandle, +) -> StreamDetails { + StreamDetails { + shared_subscriber_id: None, + stream: None, + stream_info: None, + provider_name: Some(Arc::clone(provider_name)), + request_url: Some("http://provider-1.example/live/1".intern()), + session_headers: None, + provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), + user_agent_stream_index: None, + grace_period: GracePeriodOptions { period_millis: 100, timeout_secs: 0, hold_stream: true }, + provider_grace_active: true, + disable_provider_grace: false, + reconnect_flag: None, + provider_handle: Some(provider_handle), + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + grace_resolution_context: None, + custom_reason: None, + response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), + session_registration: None, + } +} + +pub(in crate::api::model::streams::active_client_stream::tests) fn custom_video_test_state( + mode: StreamMode, + provisionable: bool, +) -> ActiveClientStreamState { + let connection_manager = create_test_connection_manager(); + let addr = std::net::SocketAddr::from(([127, 0, 0, 1], 55001)); + + ActiveClientStreamState { + response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), + inner: None, + send_custom_stream_flag: Some(Arc::new(AtomicU8::new(mode as u8))), + provider_handle: None, + deferred_provider_open: None, + timed_stream_context: None, + preempt_cancelled: None, + grace_task_handle: None, + provisioning_stop_signal: None, + provisionable, + custom_video: CustomVideoBuffers { + user_exhausted: None, + provider_exhausted: None, + unavailable: None, + provisioning: None, + low_priority_preempted: None, + }, + meter: None, + event_manager: Arc::new(EventManager::new()), + waker: None, + connection_manager, + fingerprint: Arc::new(Fingerprint::new("fp-key".to_string(), "127.0.0.1".to_string(), addr)), + stream_uid: None, + provider_stopped: true, + user_stream_released: true, + provider_handle_released: true, + custom_video_timeout_secs: 5, + custom_video_timeout_mode: None, + custom_video_timeout_sleep: None, + direct_body_idle_timeout: DirectBodyIdleTimeout::disabled(), + provider_end_reason: AtomicU8::new(PROVIDER_END_NOT_SET), + provider_error_class: None, + provider_http_status: None, + provider_reconnect_count: AtomicU8::new(0), + lease_owner: None, + media_started: None, + lease_confirmed: false, + lease_request_id: None, + request_cleanup: None, + } +} + +pub(in crate::api::model::streams::active_client_stream::tests) async fn assert_missing_custom_video_terminates( + mode: StreamMode, + provisionable: bool, +) { + let stream = ActiveClientStream { state: custom_video_test_state(mode, provisionable) }; + pin_mut!(stream); + assert!(stream.next().await.is_none()); +} diff --git a/backend/app/src/api/model/streams/provider_stream_factory.rs b/backend/app/src/api/model/streams/provider_stream_factory.rs index b6c63160b..97be489b9 100644 --- a/backend/app/src/api/model/streams/provider_stream_factory.rs +++ b/backend/app/src/api/model/streams/provider_stream_factory.rs @@ -1,59 +1,21 @@ -use crate::{ - api::model::{ - create_channel_unavailable_stream, get_header_filter_for_item_type, get_response_headers, - streams::client_stream::ClientStream, CustomVideoStreamType, ProviderContentRepresentationMode, - ProviderStreamFactoryResponse, StreamError, STREAM_IDLE_TIMEOUT, - }, - iptv::stalker::client::validate_public_playable_url, - model::{AppConfig, ConfigProvider, ReverseProxyDisabledHeaderConfig}, - utils::{ - content_coding::{ - apply_outbound_content_coding_policy, content_decoding_error_from_io, decode_response_to_identity, - ContentCodingDetection, ContentCodingError, OutboundContentCodingPolicy, - }, - debug_if_enabled, - request::{ - get_request_headers, is_safe_cross_origin_redirect_header, preview_request_diagnostics_for_logging, - preview_request_target_for_logging, send_with_retry_and_provider_policy_with_options, RequestFetchOptions, - }, - }, -}; -use futures::{StreamExt, TryStreamExt}; -use log::{debug, log_enabled, warn}; -use reqwest::{ - header::{HeaderMap, HeaderValue, CONTENT_RANGE, RANGE}, - StatusCode, -}; +use crate::{api::model::ProviderContentRepresentationMode, model::ConfigProvider}; +use reqwest::header::{HeaderMap, HeaderValue}; use shared::{ create_bitset, - defaults::DEFAULT_USER_AGENT, - model::{ConnectFailureReason, FailureStage, PlaylistItemType, StreamChannel, StreamInfo}, - utils::{filter_request_header, is_sanitize_sensitive_info_enabled, sanitize_sensitive_info, Internable}, + model::{FailureStage, PlaylistItemType, StreamChannel}, }; use std::{ - collections::HashMap, - error::Error as StdError, - io, net::SocketAddr, sync::{atomic::AtomicU8, Arc}, - time::{Duration, Instant}, -}; -use tokio_util::{io::ReaderStream, sync::CancellationToken}; -use tuliprox_core::{model::AllocationId, utils::request_headers::get_headers_from_request}; -use tuliprox_hls::api::{ - extract_hls_provider_session_headers, log_hls_origin_content_coding, HlsOriginContentCodingObjectKind, - HlsOriginContentCodingSource, -}; -use tuliprox_session::{ - response_headers::{provider_response_headers, ProviderResponseHeaderError}, - stream_ctx::ProviderStreamCtx, - stream_options::{StreamOptions, StreamResponseMode}, - streams::{ProviderBodyOwner, ProviderBodyOwnerConfig}, - ActiveProviderManager, ManagedProviderHandle, ProviderSessionHeaders, }; +use tokio_util::sync::CancellationToken; +use tuliprox_core::model::AllocationId; +use tuliprox_hls::api::HlsOriginContentCodingObjectKind; +use tuliprox_session::ActiveProviderManager; use url::Url; const RETRY_SECONDS: u64 = 5; + const ERR_MAX_RETRY_COUNT: u32 = 5; /// Describes whether the provider response head is available before Tuliprox commits the client response head. @@ -112,1296 +74,11 @@ pub struct ProviderStreamFactoryOptions { account: Option>, } -pub(crate) struct ProviderStreamFactoryParams<'a> { - pub addr: SocketAddr, - pub item_type: PlaylistItemType, - pub share_stream: bool, - pub stream_options: &'a StreamOptions, - pub stream_url: &'a Url, - pub req_headers: &'a HeaderMap, - pub input_headers: Option<&'a HashMap>, - pub session_headers: Option<&'a HashMap>, - pub disabled_headers: Option<&'a ReverseProxyDisabledHeaderConfig>, - pub default_user_agent: Option<&'a str>, - pub username: Option<&'a str>, - pub client_ip: Option<&'a str>, - pub stream_channel: Option<&'a StreamChannel>, - pub connect_failure_stage: Option, - pub content_representation: ProviderContentRepresentationMode, -} - -impl ProviderStreamFactoryOptions { - pub(crate) fn new(request: &ProviderStreamFactoryParams<'_>) -> Self { - let ProviderStreamFactoryParams { - addr, - item_type, - share_stream, - stream_options, - stream_url, - req_headers, - input_headers, - session_headers, - disabled_headers, - default_user_agent, - username, - client_ip, - stream_channel, - connect_failure_stage, - content_representation, - } = request; - let buffer_size = if stream_options.buffer_enabled { stream_options.buffer_size } else { 0 }; - let buffer_max_bytes = stream_options.buffer_max_bytes; - let user_agent = req_headers - .get(axum::http::header::USER_AGENT) - .and_then(|value| value.to_str().ok()) - .map(ToString::to_string); - let filter_header = if stream_options.response_mode == StreamResponseMode::HlsResource { - None - } else { - get_header_filter_for_item_type(*item_type) - }; - let mut req_headers = get_headers_from_request(req_headers, &filter_header); - let requested_range = req_headers.remove(RANGE.as_str()).and_then(|value| HeaderValue::from_bytes(&value).ok()); - - let merged_input_headers = merge_provider_request_headers(*input_headers, *session_headers); - - // We merge configured input headers with the headers from the request. - let mut headers = get_request_headers( - merged_input_headers.as_ref(), - Some(&req_headers), - *disabled_headers, - *default_user_agent, - ); - crate::utils::request::overlay_upstream_user_agent( - &mut headers, - stream_channel.as_ref().and_then(|channel| channel.upstream_user_agent.as_deref()), - *disabled_headers, - ); - - let default_user_agent = default_user_agent - .and_then(|ua| { - let trimmed = ua.trim(); - (!trimmed.is_empty()).then_some(trimmed) - }) - .and_then(|ua| axum::http::header::HeaderValue::from_str(ua).ok()); - let url = (*stream_url).clone(); - let mut flags = ProviderStreamFactoryFlagsSet::new(); - if stream_options.stream_retry { - flags.set(ProviderStreamFactoryFlags::RetryEnabled); - if !item_type.is_live_adaptive() && stream_options.response_mode != StreamResponseMode::HlsResource { - flags.set(ProviderStreamFactoryFlags::InitialRetryLoopEnabled); - } - } - if stream_options.pipe_provider_stream { - flags.set(ProviderStreamFactoryFlags::PipeStream); - } - if stream_options.buffer_enabled { - flags.set(ProviderStreamFactoryFlags::BufferEnabled); - } - if stream_options.response_mode == StreamResponseMode::HlsResource { - flags.set(ProviderStreamFactoryFlags::HlsResource); - } - if *share_stream { - flags.set(ProviderStreamFactoryFlags::ShareStream); - } - if requested_range.is_some() { - flags.set(ProviderStreamFactoryFlags::RangeRequested); - } - - Self { - item_type: *item_type, - addr: *addr, - flags, - buffer_size, - buffer_max_bytes, - reconnect_flag: CancellationToken::new(), - url, - headers, - default_user_agent, - requested_range, - provider: None, - username: username.map(ToString::to_string), - client_ip: client_ip.map(ToString::to_string), - user_agent, - stream_channel: stream_channel.cloned(), - connect_failure_stage: *connect_failure_stage, - content_representation: *content_representation, - response_head_availability: ProviderResponseHeadAvailability::Available, - hls_content_coding_object_kind: match *content_representation { - ProviderContentRepresentationMode::Identity => Some(HlsOriginContentCodingObjectKind::Other), - ProviderContentRepresentationMode::PreserveOrigin => None, - }, - cancel_token: None, - completion_token: None, - close_reason: None, - account: None, - } - } - - pub fn set_provider_handle_tokens( - &mut self, - cancel_token: Option, - completion_token: Option, - close_reason: Option>, - ) { - self.cancel_token = cancel_token; - self.completion_token = completion_token; - self.close_reason = close_reason; - } - - pub fn get_cancel_token(&self) -> Option { self.cancel_token.clone() } - - pub fn get_completion_token(&self) -> Option { self.completion_token.clone() } - - pub fn get_close_reason(&self) -> Option> { self.close_reason.clone() } - - pub fn set_provider(&mut self, provider: Option>) { self.provider = provider; } - - pub fn apply_user_agent_stream_index(&mut self, stream_index: u64) { - crate::utils::request::append_user_agent_stream_index(&mut self.headers, stream_index); - } - - pub fn require_public_destination(&mut self) { - self.flags.set(ProviderStreamFactoryFlags::PublicDestinationRequired); - } - - /// Applies per-input stream options that the generic factory parameters do not carry. - pub fn apply_input_options(&mut self, input: &crate::model::ConfigInput) { - if input.has_flag(crate::model::ConfigInputFlags::FlussonicHlsAudioTracks) { - self.flags.set(ProviderStreamFactoryFlags::FlussonicAudioTracks); - } - } - - pub fn get_provider(&self) -> Option<&Arc> { self.provider.as_ref() } - - #[inline] - fn is_piped(&self) -> bool { self.flags.contains(ProviderStreamFactoryFlags::PipeStream) } - - #[inline] - fn is_buffer_enabled(&self) -> bool { self.flags.contains(ProviderStreamFactoryFlags::BufferEnabled) } - - #[inline] - pub(crate) fn get_buffer_size(&self) -> usize { self.buffer_size } - - #[inline] - pub(crate) fn get_buffer_max_bytes(&self) -> usize { self.buffer_max_bytes } - - #[inline] - pub fn get_reconnect_flag_clone(&self) -> CancellationToken { self.reconnect_flag.clone() } - - #[inline] - pub fn cancel_reconnect(&self) { self.reconnect_flag.cancel(); } - - #[inline] - pub fn get_url(&self) -> &Url { &self.url } - - #[inline] - pub fn get_url_as_str(&self) -> &str { self.url.as_str() } - - #[inline] - fn get_item_type(&self) -> PlaylistItemType { self.item_type } - - #[inline] - pub fn should_retry_provider_request(&self) -> bool { - self.flags.contains(ProviderStreamFactoryFlags::RetryEnabled) - } - - fn requires_public_destination(&self) -> bool { - self.flags.contains(ProviderStreamFactoryFlags::PublicDestinationRequired) - } - - #[inline] - pub fn should_retry_initial_open_loop(&self) -> bool { - self.flags.contains(ProviderStreamFactoryFlags::InitialRetryLoopEnabled) - } - - #[inline] - pub fn get_headers(&self) -> &HeaderMap { &self.headers } - - #[inline] - pub fn get_requested_range(&self) -> Option<&HeaderValue> { self.requested_range.as_ref() } - - #[inline] - pub fn should_continue(&self) -> bool { !self.reconnect_flag.is_cancelled() } - - #[inline] - pub fn was_range_requested(&self) -> bool { self.flags.contains(ProviderStreamFactoryFlags::RangeRequested) } - - fn get_log_url(&self) -> std::borrow::Cow<'_, str> { - if is_sanitize_sensitive_info_enabled() { - return std::borrow::Cow::Borrowed(self.url.as_str()); - } - - std::borrow::Cow::Owned(preview_request_target_for_logging(&self.url, self.provider.as_ref())) - } - - fn build_connect_failed_stream_info(&self, provider_name: Arc) -> Option { - let username = self.username.as_deref()?; - let client_ip = self.client_ip.as_deref()?; - let stream_channel = self.stream_channel.clone()?; - Some(StreamInfo::new(shared::model::StreamInfoParams { - uid: 0, - meter_uid: 0, - username, - addr: &self.addr, - client_ip, - provider: provider_name, - stream_channel, - user_agent: self.user_agent.clone().unwrap_or_default(), - country_code: None, - session_token: None, - })) - } - - fn get_connect_failure_stage(&self) -> Option { self.connect_failure_stage } - - #[inline] - pub(crate) fn content_representation(&self) -> ProviderContentRepresentationMode { self.content_representation } - - /// Converts an open delayed until body polling to the only representation safe without an origin response head. - pub(crate) fn for_deferred_open(mut self) -> Self { - self.content_representation = ProviderContentRepresentationMode::Identity; - self.response_head_availability = ProviderResponseHeadAvailability::Unavailable; - self - } - - #[cfg(test)] - pub(crate) fn response_head_is_available(&self) -> bool { - matches!(self.response_head_availability, ProviderResponseHeadAvailability::Available) - } - - #[cfg(test)] - pub(crate) fn hls_content_coding_object_kind(&self) -> Option { - self.hls_content_coding_object_kind - } -} - -fn merge_provider_request_headers( - input_headers: Option<&HashMap>, - session_headers: Option<&HashMap>, -) -> Option> { - match (input_headers, session_headers) { - (None, None) => None, - (Some(headers), None) | (None, Some(headers)) => Some(headers.clone()), - (Some(input), Some(session)) => { - let mut merged = input.clone(); - for (key, value) in session { - merged.insert(key.clone(), value.clone()); - } - Some(merged) - } - } -} - -fn record_provider_open_failure( - ctx: &ProviderStreamCtx, - stream_options: &ProviderStreamFactoryOptions, - reason: ConnectFailureReason, - provider_http_status: Option, - provider_error_class: Option<&str>, -) { - let Some(failure_stage) = stream_options.get_connect_failure_stage() else { return }; - let provider_name = - stream_options.get_provider().map_or_else(|| "unknown".intern(), |provider| provider.name.clone()); - let Some(info) = stream_options.build_connect_failed_stream_info(provider_name) else { return }; - // Resolve target_name from target_id using the stable target config name. - let target_name = ctx.app_config.get_target_by_id(info.channel.target_id).as_deref().map(|t| (&t.name).intern()); - ctx.connection_manager.record_connect_failed_with_provider_failure( - &info, - reason, - failure_stage, - provider_http_status.map(|status| status.as_u16()), - provider_error_class, - target_name, - ); -} - -fn classify_provider_status_error(status: StatusCode) -> &'static str { - if status.is_client_error() { - "http_4xx" - } else if status.is_server_error() { - "http_5xx" - } else if status.is_redirection() { - "http_3xx" - } else { - "http_other" - } -} - -fn provider_content_type_looks_like_html(headers: &HeaderMap) -> bool { - headers - .get(reqwest::header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.split(';').next().unwrap_or_default().trim().eq_ignore_ascii_case("text/html")) -} - -fn should_reject_success_response_content_type(item_type: PlaylistItemType, headers: &HeaderMap) -> bool { - !item_type.is_live_adaptive() && provider_content_type_looks_like_html(headers) -} - -#[derive(Debug)] -enum ProviderStreamRequestFailure { - HlsStatus { status: StatusCode, headers: Vec<(String, String)> }, - Status { status: StatusCode, provider_error_class: &'static str, serve_channel_unavailable: bool }, -} - -#[derive(Debug, thiserror::Error)] -enum ProviderStreamPreparationError { - #[error(transparent)] - ContentCoding(#[from] ContentCodingError), - - #[error(transparent)] - ResponseHeader(#[from] ProviderResponseHeaderError), - - #[error("provider response head is unavailable for status {status} or Content-Range={has_content_range}")] - DeferredResponseHead { status: StatusCode, has_content_range: bool }, -} - -impl ProviderStreamPreparationError { - fn provider_error_class(&self) -> &'static str { - match self { - Self::ContentCoding(ContentCodingError::InvalidHeader | ContentCodingError::Unsupported(_)) - | Self::ResponseHeader(ProviderResponseHeaderError::InvalidContentEncoding) => "content_encoding", - Self::ContentCoding(ContentCodingError::EncodedPartialContent) => "encoded_partial_content", - Self::ContentCoding(ContentCodingError::PrefixRead(_)) => "body", - Self::ResponseHeader( - ProviderResponseHeaderError::InvalidContentLength | ProviderResponseHeaderError::InvalidContentRange, - ) - | Self::DeferredResponseHead { .. } => "response_headers", - } - } -} - -impl ProviderStreamRequestFailure { - fn status(&self) -> StatusCode { - match self { - Self::Status { status, .. } | Self::HlsStatus { status, .. } => *status, - } - } - - fn provider_error_class(&self) -> &'static str { - match self { - Self::Status { provider_error_class, .. } => provider_error_class, - Self::HlsStatus { status, .. } => classify_provider_status_error(*status), - } - } - - fn should_serve_channel_unavailable(&self) -> bool { - match self { - Self::Status { serve_channel_unavailable, .. } => *serve_channel_unavailable, - Self::HlsStatus { .. } => false, - } - } -} - -fn classify_provider_io_error(err: &std::io::Error) -> &'static str { - use std::io::ErrorKind; - - match err.kind() { - ErrorKind::TimedOut => "timeout", - ErrorKind::ConnectionRefused - | ErrorKind::ConnectionReset - | ErrorKind::ConnectionAborted - | ErrorKind::NotConnected => "connect", - ErrorKind::AddrNotAvailable => "dns", - _ => { - let lowered = err.to_string().to_ascii_lowercase(); - if lowered.contains("dns") - || lowered.contains("failed to lookup address information") - || lowered.contains("name or service not known") - || lowered.contains("no such host") - || lowered.contains("temporary failure in name resolution") - { - "dns" - } else { - "io" - } - } - } -} - -fn should_wrap_provider_stream_in_buffer(stream_options: &ProviderStreamFactoryOptions) -> bool { - !stream_options.is_piped() - && !stream_options.flags.contains(ProviderStreamFactoryFlags::ShareStream) - && stream_options.is_buffer_enabled() -} - -// fn get_host_and_optional_port(url: &Url) -> Option { -// let host = url.host_str()?; -// match url.port() { -// Some(port) => Some(format!("{host}:{port}")), -// None => Some(host.to_string()), -// } -// } - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum ProviderRequestCredentialState { - OriginalOrigin, - Scrubbed, -} - -impl ProviderRequestCredentialState { - fn observe_target(&mut self, original_url: &Url, target_url: &Url) { - if !same_origin(original_url, target_url) { - *self = Self::Scrubbed; - } - } -} - -fn prepare_client( - request_client: &reqwest::Client, - stream_options: &ProviderStreamFactoryOptions, - url_override: Option<&Url>, - credential_state: ProviderRequestCredentialState, -) -> (reqwest::RequestBuilder, bool) { - let original_url = stream_options.get_url(); - let url = url_override.unwrap_or(original_url); - let requested_range = stream_options.get_requested_range(); - let original_headers = stream_options.get_headers(); - - if log_enabled!(log::Level::Debug) { - let message = format!("original headers {original_headers:?}"); - debug!("{}", sanitize_sensitive_info(&message)); - } - - let mut headers = HeaderMap::default(); - - for (key, value) in original_headers { - if filter_request_header(key.as_str()) { - headers.insert(key.clone(), value.clone()); - } - } - - if matches!(credential_state, ProviderRequestCredentialState::Scrubbed) { - remove_sensitive_headers(&mut headers); - } - prepare_default_headers(&mut headers, stream_options); - let partial = prepare_partial_request_headers(&mut headers, requested_range); - let content_coding_policy = match stream_options.content_representation() { - ProviderContentRepresentationMode::PreserveOrigin => OutboundContentCodingPolicy::Inherit, - ProviderContentRepresentationMode::Identity => OutboundContentCodingPolicy::Identity, - }; - apply_outbound_content_coding_policy(&mut headers, content_coding_policy); - - if log_enabled!(log::Level::Debug) { - let message = format!( - "Stream requested with headers: {:?}", - headers.iter().map(|header| (header.0, String::from_utf8_lossy(header.1.as_ref()))).collect::>() - ); - debug!("{}", sanitize_sensitive_info(&message)); - } - - let request_builder = request_client.get(url.clone()).headers(headers); - - (request_builder, partial) -} - -fn remove_sensitive_headers(headers: &mut axum::http::HeaderMap) { - let names_to_remove = - headers.keys().filter(|name| !is_safe_cross_origin_redirect_header(name.as_str())).cloned().collect::>(); - for name in names_to_remove { - headers.remove(name); - } -} - -fn provider_headers_require_manual_redirects(headers: &HeaderMap) -> bool { - headers.keys().any(|name| !is_safe_cross_origin_redirect_header(name.as_str())) -} - -fn same_origin(lhs: &Url, rhs: &Url) -> bool { - lhs.scheme().eq_ignore_ascii_case(rhs.scheme()) - && lhs.host_str() == rhs.host_str() - && lhs.port_or_known_default() == rhs.port_or_known_default() -} - -fn prepare_default_headers(headers: &mut axum::http::HeaderMap, stream_options: &ProviderStreamFactoryOptions) { - // Force Connection: close so the provider releases its slot immediately when the stream ends. - // This prevents 509 errors from providers counting idle pooled connections against limits. - headers.insert(axum::http::header::CONNECTION, axum::http::header::HeaderValue::from_static("close")); - - if !headers.contains_key(axum::http::header::USER_AGENT) { - headers.insert( - axum::http::header::USER_AGENT, - stream_options - .default_user_agent - .clone() - .unwrap_or_else(|| axum::http::header::HeaderValue::from_static(DEFAULT_USER_AGENT)), - ); - } -} - -fn prepare_partial_request_headers(headers: &mut HeaderMap, requested_range: Option<&HeaderValue>) -> bool { - if let Some(range) = requested_range { - headers.insert(RANGE, range.clone()); - true - } else { - false - } -} - -fn collect_debug_headers(headers: &HeaderMap) -> Vec<(String, String)> { - const HEADER_NAMES: [&str; 8] = - ["proxy-authenticate", "via", "server", "location", "x-cache", "x-cache-status", "x-served-by", "x-proxy-id"]; - - HEADER_NAMES - .iter() - .filter_map(|name| { - headers.get_all(*name).iter().next().map(|value| { - let value = value.to_str().unwrap_or("").to_string(); - ((*name).to_string(), value) - }) - }) - .collect() -} - -async fn send_with_manual_redirects( - request_client: &reqwest::Client, - stream_options: &ProviderStreamFactoryOptions, - app_config: &Arc, -) -> Result { - let mut current_url = stream_options.get_url().clone(); - let mut remaining_redirects = 10u8; - let provider = stream_options.get_provider().cloned(); - let mut credential_state = ProviderRequestCredentialState::OriginalOrigin; - - loop { - if stream_options.requires_public_destination() { - validate_public_playable_url(¤t_url) - .await - .map_err(|err| io::Error::new(io::ErrorKind::PermissionDenied, err))?; - } - let result = send_with_retry_and_provider_policy_with_options( - app_config, - ¤t_url, - provider.as_ref(), - true, - stream_options.should_retry_provider_request(), - RequestFetchOptions::default() - .with_http_error_responses(stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource)), - |resolved_url| prepare_client(request_client, stream_options, Some(resolved_url), credential_state).0, - ) - .await; - - let response = match result { - Ok(resp) => resp, - Err(e) => { - // send_with_retry_and_provider already applies provider failover policy. - // Do not rotate again here, otherwise non-failover errors (e.g. auth) may - // incorrectly switch provider URLs. - debug!("Manual redirect failed: {}", sanitize_sensitive_info(&e.to_string())); - return Err(e); - } - }; - - let status = response.status(); - - if status.is_redirection() { - if remaining_redirects == 0 { - return Ok(response); - } - let location = response.headers().get(reqwest::header::LOCATION); - let Some(location) = location else { - return Ok(response); - }; - let Ok(location_str) = location.to_str() else { - return Ok(response); - }; - let response_url = response.url().clone(); - let next_url = response_url.join(location_str).or_else(|_| Url::parse(location_str)); - let Ok(next_url) = next_url else { - return Ok(response); - }; - credential_state.observe_target(&response_url, &next_url); - current_url = next_url; - remaining_redirects = remaining_redirects.saturating_sub(1); - continue; - } - return Ok(response); - } -} - -#[cfg(test)] -async fn prepare_provider_stream_response( - response: reqwest::Response, - mode: ProviderContentRepresentationMode, - response_head_availability: ProviderResponseHeadAvailability, -) -> Result { - prepare_provider_stream_response_with_context( - response, - ProviderStreamPreparationContext { - representation: mode, - response_head_availability, - range_requested: false, - hls_content_coding_object_kind: matches!(mode, ProviderContentRepresentationMode::Identity) - .then_some(HlsOriginContentCodingObjectKind::Other), - }, - Duration::from_secs(STREAM_IDLE_TIMEOUT), - ) - .await -} - -#[cfg(test)] -async fn prepare_provider_stream_response_with_idle_timeout( - response: reqwest::Response, - mode: ProviderContentRepresentationMode, - response_head_availability: ProviderResponseHeadAvailability, - idle_timeout: Duration, -) -> Result { - prepare_provider_stream_response_with_context( - response, - ProviderStreamPreparationContext { - representation: mode, - response_head_availability, - range_requested: false, - hls_content_coding_object_kind: matches!(mode, ProviderContentRepresentationMode::Identity) - .then_some(HlsOriginContentCodingObjectKind::Other), - }, - idle_timeout, - ) - .await -} - -/// Preserve specific upstream MIME types; repair missing or generic types for MP4 objects. -/// Audio and video share most fMP4 extensions, so audio is only recognized by an `m4a`/`cmfa` -/// extension or, when enabled for the input, by Flussonic's `tracks-a` rendition paths. -fn normalize_hls_resource_content_type(headers: &mut HeaderMap, url: &Url, flussonic_audio_tracks: bool) { - let extension = url.path().rsplit_once('.').map_or("", |(_, ext)| ext); - let mime = crate::utils::request::content_type_from_ext(extension); - if !matches!(mime, "video/mp4" | "audio/mp4") { - return; - } - let needs_type = - headers.get(reqwest::header::CONTENT_TYPE).and_then(|value| value.to_str().ok()).is_none_or(|value| { - let base = value.split(';').next().unwrap_or_default().trim(); - base.is_empty() - || base.eq_ignore_ascii_case("application/octet-stream") - || base.eq_ignore_ascii_case("video/mp2t") - }); - if needs_type { - // Flussonic names separate audio-only renditions tracks-a. - let audio_track = flussonic_audio_tracks - && url.path_segments().is_some_and(|mut segments| { - segments.any(|segment| { - segment - .strip_prefix("tracks-a") - .is_some_and(|index| !index.is_empty() && index.bytes().all(|byte| byte.is_ascii_digit())) - }) - }); - headers.insert( - reqwest::header::CONTENT_TYPE, - HeaderValue::from_static(if audio_track { "audio/mp4" } else { mime }), - ); - } -} - -async fn prepare_provider_stream_response_for_request( - mut response: reqwest::Response, - stream_options: &ProviderStreamFactoryOptions, -) -> Result { - if stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource) { - normalize_hls_resource_content_type( - response.headers_mut(), - stream_options.get_url(), - stream_options.flags.contains(ProviderStreamFactoryFlags::FlussonicAudioTracks), - ); - } - prepare_provider_stream_response_with_context( - response, - ProviderStreamPreparationContext { - representation: stream_options.content_representation(), - response_head_availability: stream_options.response_head_availability, - range_requested: stream_options.was_range_requested(), - hls_content_coding_object_kind: stream_options.hls_content_coding_object_kind, - }, - Duration::from_secs(STREAM_IDLE_TIMEOUT), - ) - .await -} - -async fn prepare_provider_stream_response_with_context( - response: reqwest::Response, - context: ProviderStreamPreparationContext, - idle_timeout: Duration, -) -> Result { - let provider_session_headers = extract_hls_provider_session_headers(response.headers()); - match context.representation { - ProviderContentRepresentationMode::PreserveOrigin => { - if matches!(context.response_head_availability, ProviderResponseHeadAvailability::Unavailable) { - return Err(ProviderStreamPreparationError::DeferredResponseHead { - status: response.status(), - has_content_range: response.headers().contains_key(CONTENT_RANGE), - }); - } - let headers = provider_response_headers(response.headers(), context.representation)?; - let response_info = Some((headers, response.status(), Some(response.url().clone()), None)); - let stream = response.bytes_stream().map_err(|error| StreamError::reqwest(&error)).boxed(); - Ok(ProviderStreamFactoryResponse { - stream, - info: response_info, - provider_session_headers, - has_upstream_owner: false, - }) - } - ProviderContentRepresentationMode::Identity => { - let origin_status = response.status(); - let origin_has_content_range = response.headers().contains_key(CONTENT_RANGE); - // `deflate` decoder selection reads a prefix before the returned stream can enter the - // normal provider body wrappers, so decoder setup must own the same idle bound too. - let decoded = tokio::time::timeout( - idle_timeout, - decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly), - ) - .await - .map_err(|_| { - ContentCodingError::PrefixRead(io::Error::new( - io::ErrorKind::TimedOut, - "provider body idle timeout during content-decoder setup", - )) - })??; - if matches!(context.response_head_availability, ProviderResponseHeadAvailability::Unavailable) - && (origin_status != StatusCode::OK || origin_has_content_range) - { - return Err(ProviderStreamPreparationError::DeferredResponseHead { - status: origin_status, - has_content_range: origin_has_content_range, - }); - } - if let (Some(observation), Some(object_kind)) = - (decoded.content_coding_observation(), context.hls_content_coding_object_kind) - { - log_hls_origin_content_coding( - observation, - object_kind, - context.range_requested, - HlsOriginContentCodingSource::Legacy, - ); - } - let headers = provider_response_headers(&decoded.headers, context.representation)?; - let response_info = Some((headers, decoded.status, Some(decoded.final_url), None)); - let stream = ReaderStream::new(decoded.body).map_err(|error| provider_decoded_body_error(&error)).boxed(); - Ok(ProviderStreamFactoryResponse { - stream, - info: response_info, - provider_session_headers, - has_upstream_owner: false, - }) - } - } -} - -fn provider_decoded_body_error(error: &io::Error) -> StreamError { - if let Some(error) = content_decoding_error_from_io(error) { - return StreamError::ContentDecoding(format!("coding={}", error.coding.as_http_token())); - } - - let mut source = StdError::source(error); - while let Some(current) = source { - if let Some(error) = current.downcast_ref::() { - return StreamError::reqwest(error); - } - source = current.source(); - } - StreamError::StdIo(error.to_string()) -} - -#[allow(clippy::too_many_lines)] -async fn provider_stream_request( - ctx: &ProviderStreamCtx, - request_client: &reqwest::Client, - stream_options: &ProviderStreamFactoryOptions, -) -> Result, ProviderStreamRequestFailure> { - let use_manual_redirects = stream_options.requires_public_destination() - || tuliprox_core::model::should_use_manual_redirects(&ctx.app_config) - || provider_headers_require_manual_redirects(stream_options.get_headers()); - if log_enabled!(log::Level::Debug) { - let diagnostics = - preview_request_diagnostics_for_logging(stream_options.get_url(), stream_options.get_provider()); - debug!( - "Provider request diagnostics: manual_redirects={}, {}", - use_manual_redirects, - sanitize_sensitive_info(&diagnostics) - ); - } - let response_result = if use_manual_redirects { - let client_no_redirect = if stream_options.requires_public_destination() { - ctx.http_clients.public_no_redirect.load() - } else { - ctx.http_clients.no_redirect.load() - }; - send_with_manual_redirects(&client_no_redirect, stream_options, &ctx.app_config).await - } else { - // Use send_with_retry_and_provider for automatic failover support - let url = stream_options.get_url(); - let provider = stream_options.get_provider().cloned(); - - send_with_retry_and_provider_policy_with_options( - &ctx.app_config, - url, - provider.as_ref(), - false, - stream_options.should_retry_provider_request(), - RequestFetchOptions::default() - .with_http_error_responses(stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource)), - |resolved_url| { - let (client, _partial_content) = prepare_client( - request_client, - stream_options, - Some(resolved_url), - ProviderRequestCredentialState::OriginalOrigin, - ); - client - }, - ) - .await - }; - match response_result { - Ok(response) => { - let status = response.status(); - let response_url = response.url().clone(); - if log_enabled!(log::Level::Debug) && !status.is_success() { - let debug_headers = collect_debug_headers(response.headers()); - let diagnostics = - preview_request_diagnostics_for_logging(stream_options.get_url(), stream_options.get_provider()); - let message = - format!( - "Provider response error: status={status}, url={response_url}, headers={debug_headers:?}, {diagnostics}" - ); - debug!("{}", sanitize_sensitive_info(&message)); - } - if status.is_success() { - if should_reject_success_response_content_type(stream_options.get_item_type(), response.headers()) { - debug!( - "Provider returned HTML content for non-adaptive stream {}", - sanitize_sensitive_info(stream_options.get_log_url().as_ref()) - ); - return Err(ProviderStreamRequestFailure::Status { - status: StatusCode::BAD_GATEWAY, - provider_error_class: "unexpected_content_type", - serve_channel_unavailable: true, - }); - } - let response = prepare_provider_stream_response_for_request(response, stream_options).await; - let response = match response { - Ok(response) => response, - Err(error) => { - let provider_error_class = error.provider_error_class(); - return Err(ProviderStreamRequestFailure::Status { - status: StatusCode::BAD_GATEWAY, - provider_error_class, - serve_channel_unavailable: true, - }); - } - }; - if log_enabled!(log::Level::Debug) { - // Unfortunately, the HEAD request does not work, so we need this workaround. - // We need some header information from the provider, we extract the necessary headers and forward them to the client - let message = format!("Provider response info: {:?}", response.info); - debug!("{}", sanitize_sensitive_info(&message)); - } - return Ok(Some(response)); - } - - if stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource) { - // Keep range and retry metadata while discarding the provider's error body. - let headers = response - .headers() - .iter() - .filter(|(name, _)| { - matches!( - **name, - reqwest::header::CONTENT_RANGE - | reqwest::header::ACCEPT_RANGES - | reqwest::header::RETRY_AFTER - ) - }) - .filter_map(|(name, value)| value.to_str().ok().map(|value| (name.to_string(), value.to_string()))) - .collect(); - // Only upstream errors pass through; an unfollowed redirect or other non-success - // status would reach the client without its Location or body. - let status = - if status.is_client_error() || status.is_server_error() { status } else { StatusCode::BAD_GATEWAY }; - return Err(ProviderStreamRequestFailure::HlsStatus { status, headers }); - } - - if status.is_client_error() { - debug!("Client error status response : {status}"); - return match status { - StatusCode::NOT_FOUND - | StatusCode::FORBIDDEN - | StatusCode::UNAUTHORIZED - | StatusCode::PROXY_AUTHENTICATION_REQUIRED - | StatusCode::METHOD_NOT_ALLOWED - | StatusCode::BAD_REQUEST => Err(ProviderStreamRequestFailure::Status { - status, - provider_error_class: classify_provider_status_error(status), - serve_channel_unavailable: true, - }), - _ => Err(ProviderStreamRequestFailure::Status { - status, - provider_error_class: classify_provider_status_error(status), - serve_channel_unavailable: false, - }), - }; - } - if status.is_server_error() { - debug!("Server error status response : {status}"); - return match status { - StatusCode::INTERNAL_SERVER_ERROR - | StatusCode::BAD_GATEWAY - | StatusCode::SERVICE_UNAVAILABLE - | StatusCode::GATEWAY_TIMEOUT => Err(ProviderStreamRequestFailure::Status { - status, - provider_error_class: classify_provider_status_error(status), - serve_channel_unavailable: true, - }), - _ => Err(ProviderStreamRequestFailure::Status { - status, - provider_error_class: classify_provider_status_error(status), - serve_channel_unavailable: false, - }), - }; - } - Err(ProviderStreamRequestFailure::Status { - status, - provider_error_class: classify_provider_status_error(status), - serve_channel_unavailable: false, - }) - } - Err(err) => { - let diagnostics = - preview_request_diagnostics_for_logging(stream_options.get_url(), stream_options.get_provider()); - debug!( - "Provider request failed: {}, {}", - sanitize_sensitive_info(&err.to_string()), - sanitize_sensitive_info(&diagnostics) - ); - Err(ProviderStreamRequestFailure::Status { - status: StatusCode::SERVICE_UNAVAILABLE, - provider_error_class: classify_provider_io_error(&err), - serve_channel_unavailable: true, - }) - } - } -} - -fn failed_stream_account( - input: &crate::model::ConfigInput, - options: &ProviderStreamFactoryOptions, -) -> Option { - let url = &options.url; - let credentials = shared::utils::get_credentials_from_url(url); - let segments: Vec<_> = url.path_segments().into_iter().flatten().collect(); - let matches = |username: Option<&str>, password: Option<&str>| { - username.zip(password).is_some_and(|(username, password)| { - credentials.0.as_deref() == Some(username) && credentials.1.as_deref() == Some(password) - || segments.windows(2).any(|pair| pair[0] == username && pair[1] == password) - }) - }; - let matches_origin = |base: &str| { - input - .resolve_url(base) - .ok() - .and_then(|base| Url::parse(&base).ok()) - .is_some_and(|base| base.origin() == url.origin()) - }; - let account = if let Some(allocated) = &options.account { - if input.name == allocated.name { - input.clone() - } else { - let alias = input.aliases.iter().flatten().find(|alias| alias.name == allocated.name)?; - input.as_input(alias) - } - } else if matches(input.username.as_deref(), input.password.as_deref()) && matches_origin(&input.url) { - input.clone() - } else { - let alias = input.aliases.iter().flatten().find(|alias| { - matches(alias.username.as_deref(), alias.password.as_deref()) && matches_origin(&alias.url) - })?; - input.as_input(alias) - }; - if options.account.as_ref().is_some_and(|allocated| account.account_identity() != allocated.account_identity()) { - return None; - } - Some(account) -} - -/// Flags the failed account for the Xtream expiry worker. No request is sent here: the worker -/// checks the account only when its per-account, per-panel and cooldown throttles allow it, -/// because frequent account queries can get the account banned. -fn request_provider_account_probe(ctx: &ProviderStreamCtx, options: &ProviderStreamFactoryOptions) { - let Some(channel) = options.stream_channel.as_ref() else { - return; - }; - let sources = ctx.app_config.sources.load_full(); - let Some(input) = sources.get_input_by_name(&channel.input_name) else { - return; - }; - if !input.input_type.is_xtream() { - return; - } - if let Some(account) = failed_stream_account(input, options) { - ctx.active_provider.request_account_probe(&account.name); - } -} - -#[allow(clippy::too_many_lines)] -async fn get_provider_stream( - ctx: &ProviderStreamCtx, - client: &reqwest::Client, - stream_options: &ProviderStreamFactoryOptions, -) -> Result, ProviderStreamRequestFailure> { - let log_url = stream_options.get_log_url(); - debug_if_enabled!("stream provider {}", sanitize_sensitive_info(log_url.as_ref())); - let start = Instant::now(); - let mut connect_err: u32 = 1; - - let handle_cancel = stream_options.get_cancel_token(); - while stream_options.should_continue() { - let request_future = provider_stream_request(ctx, client, stream_options); - let request_res = if let Some(ref cancel) = handle_cancel { - tokio::select! { - biased; - () = cancel.cancelled() => { - debug!( - "Provider stream request cancelled during open: {}", - sanitize_sensitive_info(stream_options.get_log_url().as_ref()) - ); - return Err(ProviderStreamRequestFailure::Status { - status: StatusCode::BAD_GATEWAY, - provider_error_class: "cancelled", - serve_channel_unavailable: false, - }); - } - res = request_future => res, - } - } else { - request_future.await - }; - match request_res { - Ok(Some(stream_response)) => { - return Ok(Some(stream_response)); - } - Ok(None) => { - if connect_err > ERR_MAX_RETRY_COUNT { - warn!( - "The stream could be unavailable. {}", - sanitize_sensitive_info(stream_options.get_log_url().as_ref()) - ); - break; - } - } - Err(failure) => { - if matches!(failure.status(), StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) { - request_provider_account_probe(ctx, stream_options); - } - if stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource) - || failure.should_serve_channel_unavailable() - { - return Err(failure); - } - let status = failure.status(); - debug!("Provider stream response error status response : {status}"); - if matches!( - status, - StatusCode::FORBIDDEN - | StatusCode::SERVICE_UNAVAILABLE - | StatusCode::UNAUTHORIZED - | StatusCode::PROXY_AUTHENTICATION_REQUIRED - | StatusCode::RANGE_NOT_SATISFIABLE - ) { - warn!( - "The stream could be unavailable. ({status}) {}", - sanitize_sensitive_info(stream_options.get_log_url().as_ref()) - ); - break; - } - if connect_err > ERR_MAX_RETRY_COUNT { - warn!( - "The stream could be unavailable. ({status}) {}", - sanitize_sensitive_info(stream_options.get_log_url().as_ref()) - ); - break; - } - } - } - if !stream_options.should_continue() || connect_err > ERR_MAX_RETRY_COUNT { - break; - } - if !stream_options.should_retry_initial_open_loop() { - break; - } - if start.elapsed().as_secs() > RETRY_SECONDS { - warn!( - "The stream could be unavailable. Giving up after {RETRY_SECONDS} seconds. {}", - sanitize_sensitive_info(stream_options.get_log_url().as_ref()) - ); - break; - } - connect_err += 1; - if let Some(ref cancel) = handle_cancel { - tokio::select! { - biased; - () = cancel.cancelled() => break, - () = tokio::time::sleep(Duration::from_millis(50)) => {}, - } - } else { - tokio::time::sleep(Duration::from_millis(50)).await; - } - debug_if_enabled!("Reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref())); - } - debug_if_enabled!("Stopped reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref())); - stream_options.cancel_reconnect(); - Err(ProviderStreamRequestFailure::Status { - status: StatusCode::SERVICE_UNAVAILABLE, - provider_error_class: "service_unavailable", - serve_channel_unavailable: true, - }) -} - struct ProviderOpenGuard { token: Option, handed_off: bool, } -impl Drop for ProviderOpenGuard { - fn drop(&mut self) { - if !self.handed_off { - if let Some(token) = self.token.take() { - token.cancel(); - } - } - } -} - -/// Result of opening a provider stream. -pub enum ProviderStreamOpen { - Stream(ProviderStreamFactoryResponse), - /// A finite HLS resource keeps the upstream error status and its range and retry headers - /// instead of receiving a fallback body. - UpstreamStatus { - status: StatusCode, - headers: Vec<(String, String)>, - }, -} - -#[allow(clippy::too_many_lines)] -pub async fn create_provider_stream( - ctx: &ProviderStreamCtx, - client: &reqwest::Client, - stream_options: ProviderStreamFactoryOptions, -) -> Option { - let mut open_guard = ProviderOpenGuard { token: stream_options.get_completion_token(), handed_off: false }; - match get_provider_stream(ctx, client, &stream_options).await { - Ok(Some(ProviderStreamFactoryResponse { stream: init_stream, info, provider_session_headers, .. })) => { - let continue_signal = stream_options.get_reconnect_flag_clone(); - if let Some((_headers, _status, _response_url, Some(custom_video_type))) = &info { - let reason = match custom_video_type { - CustomVideoStreamType::ChannelUnavailable => Some(ConnectFailureReason::ChannelUnavailable), - CustomVideoStreamType::Provisioning => Some(ConnectFailureReason::Provisioning), - CustomVideoStreamType::ProviderConnectionsExhausted => { - Some(ConnectFailureReason::ProviderConnectionsExhausted) - } - _ => None, - }; - if let Some(reason) = reason { - record_provider_open_failure(ctx, &stream_options, reason, None, None); - } - return Some(ProviderStreamOpen::Stream(ProviderStreamFactoryResponse { - stream: ClientStream::new(init_stream, continue_signal, None, stream_options.get_url_as_str()) - .boxed(), - info, - provider_session_headers, - has_upstream_owner: false, - })); - } - let handle_cancel = stream_options.get_cancel_token(); - let completion_token = stream_options.get_completion_token(); - let close_reason = stream_options.get_close_reason(); - - let cancel_token = if let Some(hc) = handle_cancel { - let combined = CancellationToken::new(); - let c1 = combined.clone(); - let c2 = combined.clone(); - let hc_clone = hc.clone(); - let cs_clone = continue_signal.clone(); - tokio::spawn(async move { - tokio::select! { - biased; - () = hc_clone.cancelled() => { - cs_clone.cancel(); - c1.cancel(); - } - () = cs_clone.cancelled() => { - c2.cancel(); - } - } - }); - combined - } else { - continue_signal.clone() - }; - - let body_owner_config = if should_wrap_provider_stream_in_buffer(&stream_options) { - ProviderBodyOwnerConfig::for_buffered( - stream_options.get_buffer_size(), - stream_options.get_buffer_max_bytes(), - ) - } else { - ProviderBodyOwnerConfig::for_direct_body() - }; - - open_guard.handed_off = true; - let stream = ProviderBodyOwner::new( - init_stream.boxed(), - body_owner_config, - cancel_token, - completion_token, - close_reason, - ) - .boxed(); - - Some(ProviderStreamOpen::Stream(ProviderStreamFactoryResponse { - stream: ClientStream::new(stream, continue_signal.clone(), None, stream_options.get_url_as_str()) - .boxed(), - info, - provider_session_headers, - has_upstream_owner: true, - })) - } - Ok(None) => None, - Err(failure) => { - let status = failure.status(); - record_provider_open_failure( - ctx, - &stream_options, - ConnectFailureReason::ChannelUnavailable, - Some(status), - Some(failure.provider_error_class()), - ); - if stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource) { - let headers = match failure { - ProviderStreamRequestFailure::HlsStatus { headers, .. } => headers, - ProviderStreamRequestFailure::Status { .. } => Vec::new(), - }; - return Some(ProviderStreamOpen::UpstreamStatus { status, headers }); - } - if let (Some(boxed_provider_stream), response_info) = create_channel_unavailable_stream( - &ctx.app_config, - &get_response_headers(stream_options.get_headers()), - StatusCode::OK, - ) { - return Some(ProviderStreamOpen::Stream(ProviderStreamFactoryResponse { - stream: boxed_provider_stream, - info: response_info, - provider_session_headers: ProviderSessionHeaders::default(), - has_upstream_owner: false, - })); - } - None - } - } -} - #[derive(Clone)] pub struct ProviderStreamOpenLifecycle { manager: Arc, @@ -1412,1399 +89,22 @@ pub struct ProviderStreamOpenLifecycle { account: Option>, } -impl ProviderStreamOpenLifecycle { - pub fn from_managed(managed: &ManagedProviderHandle) -> Option { - let handle = managed.handle()?; - Some(Self { - manager: Arc::clone(managed.manager()), - allocation_id: handle.allocation_id, - cancel_token: handle.cancel_token.clone(), - completion_token: handle.completion_token.clone(), - close_reason: Some(Arc::clone(&handle.close_reason)), - account: handle.allocation.get_provider_config(), - }) - } - - pub fn mark_opening(&self) { self.manager.mark_opening(self.allocation_id); } - - pub fn register_body_owner(&self) -> bool { self.manager.register_body_owner(self.allocation_id) } -} - -pub async fn open_provider_stream_with_lifecycle( - ctx: &ProviderStreamCtx, - client: &reqwest::Client, - mut stream_options: ProviderStreamFactoryOptions, - lifecycle: Option, -) -> Option { - if let Some(ref lc) = lifecycle { - lc.mark_opening(); - stream_options.account.clone_from(&lc.account); - stream_options.set_provider_handle_tokens( - lc.cancel_token.clone(), - lc.completion_token.clone(), - lc.close_reason.clone(), - ); - } - let open = create_provider_stream(ctx, client, stream_options).await?; - if let (ProviderStreamOpen::Stream(response), Some(lc)) = (&open, lifecycle.as_ref()) { - if response.has_upstream_owner && !lc.register_body_owner() { - return None; - } - } - Some(open) -} - #[cfg(test)] -mod tests { - use super::*; - use crate::{ - api::model::BoxedProviderStream, - model::{Config, MediaToolCapabilities, SourcesConfig}, - utils::FileLockManager, - }; - use arc_swap::{ArcSwap, ArcSwapOption}; - use axum::http::HeaderMap; - use bytes::Bytes; - use futures::TryStreamExt; - use shared::{ - model::{ConfigPaths, PlaylistItemType, StreamChannel, XtreamCluster}, - utils::Internable, - }; - use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }; - use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::{TcpListener, TcpStream}, - }; - - struct RedirectRoundTrip { - entry_url: Url, - first_origin_task: tokio::task::JoinHandle>, - second_origin_task: tokio::task::JoinHandle, - } - - fn test_app_config() -> Arc { - Arc::new(AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config::default())), - sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), - hdhomerun: Arc::new(ArcSwapOption::default()), - api_proxy: Arc::new(ArcSwapOption::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - }) - } - - #[test] - fn hls_mp4_mime_detection_preserves_specific_types_and_repairs_container_defaults( - ) -> Result<(), Box> { - for (path, original, expected) in [ - ("tracks-v1/init-1.hls.mp4", Some("video/MP2T"), "video/mp4"), - ("tracks-v1/init-1.hls.fmp4", Some("application/octet-stream"), "video/mp4"), - ("tracks-a1/init-1.hls.fmp4", None, "audio/mp4"), - ("tracks-a2/init-1.hls.mp4", Some("video/mp2t"), "audio/mp4"), - ("audio/init.mp4", Some("audio/mp4"), "audio/mp4"), - ("tracks-a1/init.mp4", Some("audio/mp4; codecs=mp4a.40.2"), "audio/mp4; codecs=mp4a.40.2"), - ("tracks-v1/segment.fmp4", None, "video/mp4"), - ("tracks-v1/segment.m4s", Some("video/mp2t; charset=binary"), "video/mp4"), - ("audio/segment.cmfa", None, "audio/mp4"), - ("audio/segment.m4a", None, "audio/mp4"), - ("video/segment.CMFV", None, "video/mp4"), - ("tracks-v1/init.mp4", Some("application/mp4"), "application/mp4"), - ("tracks-v1/init.mp4", Some("text/html"), "text/html"), - ("channel/dvr-1.ts", Some("video/MP2T"), "video/MP2T"), - ("key.bin", Some("application/octet-stream"), "application/octet-stream"), - ] { - for flussonic_audio_tracks in [true, false] { - let url = Url::parse(&format!("https://origin.example/{path}?token=signed&file=ignored.ts"))?; - let mut headers = HeaderMap::new(); - if let Some(original) = original { - headers.insert(reqwest::header::CONTENT_TYPE, HeaderValue::try_from(original)?); - } - let mut headers = headers.clone(); - normalize_hls_resource_content_type(&mut headers, &url, flussonic_audio_tracks); - // Without the input option only the extension decides; tracks-a paths stay video. - let expected = if !flussonic_audio_tracks - && path.starts_with("tracks-a") - && original.is_none_or(|o| !o.starts_with("audio")) - { - "video/mp4" - } else { - expected - }; - assert_eq!( - headers[reqwest::header::CONTENT_TYPE], - expected, - "{path} flussonic_audio_tracks={flussonic_audio_tracks}" - ); - } - } - Ok(()) - } - - #[test] - fn auth_failure_selects_origin_and_allocated_account() -> Result<(), Box> { - let dto = shared::model::ConfigInputDto { - name: Arc::from("root"), - url: "http://main.example".to_string(), - username: Some("user".to_string()), - password: Some("pass".to_string()), - aliases: Some(vec![shared::model::ConfigInputAliasDto { - name: Arc::from("alias"), - url: "http://alias.example".to_string(), - username: Some("user".to_string()), - password: Some("pass".to_string()), - ..shared::model::ConfigInputAliasDto::default() - }]), - ..shared::model::ConfigInputDto::default() - }; - let mut input = crate::model::ConfigInput::from(&dto); - let mut options = redirect_test_options(&Url::parse("http://alias.example/live/user/pass/1.ts")?); - assert_eq!(failed_stream_account(&input, &options).map(|account| account.name), Some(Arc::from("alias"))); - options.url = Url::parse("http://main.example/live/user/pass/1.ts")?; - assert_eq!(failed_stream_account(&input, &options).map(|account| account.name), Some(Arc::from("root"))); - options.url = Url::parse("http://cdn.example/token")?; - assert!(failed_stream_account(&input, &options).is_none()); - let alias = input.aliases.as_ref().and_then(|aliases| aliases.first()).ok_or("missing alias")?; - options.account = Some(Arc::new(crate::model::ProviderConfig::new_alias( - &input, - alias, - Arc::new(std::sync::RwLock::new(crate::model::ProviderConfigConnection::default())), - Arc::new(|_, _| {}), - ))); - assert_eq!(failed_stream_account(&input, &options).map(|account| account.name), Some(Arc::from("alias"))); - if let Some(alias) = input.aliases.as_mut().and_then(|aliases| aliases.first_mut()) { - alias.password = Some("renewed".to_string()); - } - assert!(failed_stream_account(&input, &options).is_none()); - Ok(()) - } - - async fn read_http_request(socket: &mut TcpStream) -> String { - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 1024]; - let read = socket.read(&mut chunk).await.expect("test origin reads request"); - if read == 0 { - break; - } - request.extend_from_slice(&chunk[..read]); - } - String::from_utf8_lossy(&request).into_owned() - } - - async fn spawn_redirect_round_trip(encoded_body: Vec) -> RedirectRoundTrip { - let first_listener = TcpListener::bind("127.0.0.1:0").await.expect("first test origin binds"); - let first_addr = first_listener.local_addr().expect("first test origin address"); - let second_listener = TcpListener::bind("127.0.0.1:0").await.expect("second test origin binds"); - let second_addr = second_listener.local_addr().expect("second test origin address"); - - let first_origin_task = tokio::spawn(async move { - let (mut first_socket, _) = first_listener.accept().await.expect("first origin accepts entry request"); - let entry_request = read_http_request(&mut first_socket).await; - let redirect = format!( - "HTTP/1.1 302 Found\r\nLocation: http://localhost:{}/hop\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", - second_addr.port() - ); - first_socket.write_all(redirect.as_bytes()).await.expect("first origin redirects to second origin"); - - let (mut final_socket, _) = first_listener.accept().await.expect("first origin accepts final request"); - let final_request = read_http_request(&mut final_socket).await; - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - encoded_body.len() - ); - final_socket.write_all(response.as_bytes()).await.expect("first origin writes final response head"); - final_socket.write_all(&encoded_body).await.expect("first origin writes final response body"); - vec![entry_request, final_request] - }); - - let second_origin_task = tokio::spawn(async move { - let (mut socket, _) = second_listener.accept().await.expect("second origin accepts redirect request"); - let request = read_http_request(&mut socket).await; - let redirect = format!( - "HTTP/1.1 302 Found\r\nLocation: http://{first_addr}/final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" - ); - socket.write_all(redirect.as_bytes()).await.expect("second origin redirects to first origin"); - request - }); - - RedirectRoundTrip { - entry_url: Url::parse(&format!("http://{first_addr}/entry")).expect("entry URL"), - first_origin_task, - second_origin_task, - } - } - - fn redirect_test_options(stream_url: &Url) -> ProviderStreamFactoryOptions { - let mut req_headers = HeaderMap::new(); - req_headers.insert(reqwest::header::ACCEPT_ENCODING, "br".parse().expect("Accept-Encoding")); - req_headers.insert(reqwest::header::AUTHORIZATION, "Bearer client-secret".parse().expect("Authorization")); - req_headers.insert(reqwest::header::COOKIE, "session=client-secret".parse().expect("Cookie")); - test_options(ProviderContentRepresentationMode::Identity, stream_url, &req_headers, None, None) - } - - fn assert_redirect_request_headers(request: &str, credentials_expected: bool) { - let request = request.to_ascii_lowercase(); - assert!(request.contains("\r\naccept-encoding: identity\r\n")); - assert_eq!(request.contains("\r\nauthorization: bearer client-secret\r\n"), credentials_expected); - assert_eq!(request.contains("\r\ncookie: session=client-secret\r\n"), credentials_expected); - } - - async fn assert_identity_redirect_result( - response: reqwest::Response, - first_origin_task: tokio::task::JoinHandle>, - second_origin_task: tokio::task::JoinHandle, - ) { - const IDENTITY_BODY: &[u8] = b"decoded redirect body"; - - let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::Identity, - ProviderResponseHeadAvailability::Available, - ) - .await - .expect("redirect response is prepared"); - assert_eq!(collect_stream(stream).await.expect("redirect body decodes"), IDENTITY_BODY); - let (headers, status, _, _) = info.expect("redirect response metadata"); - assert_eq!(status, StatusCode::OK); - assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-encoding"))); - - let first_requests = tokio::time::timeout(Duration::from_secs(2), first_origin_task) - .await - .expect("first origin finishes") - .expect("first origin task succeeds"); - let second_request = tokio::time::timeout(Duration::from_secs(2), second_origin_task) - .await - .expect("second origin finishes") - .expect("second origin task succeeds"); - assert_eq!(first_requests.len(), 2); - assert_redirect_request_headers(&first_requests[0], true); - assert_redirect_request_headers(&second_request, false); - assert_redirect_request_headers(&first_requests[1], false); - } - - fn test_options( - mode: ProviderContentRepresentationMode, - stream_url: &Url, - req_headers: &HeaderMap, - input_headers: Option<&HashMap>, - session_headers: Option<&HashMap>, - ) -> ProviderStreamFactoryOptions { - let stream_options = StreamOptions { - stream_retry: true, - buffer_enabled: false, - buffer_size: 0, - buffer_max_bytes: 0, - pipe_provider_stream: false, - response_mode: StreamResponseMode::Stream, - }; - ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { - addr: "127.0.0.1:8080".parse().unwrap(), - item_type: PlaylistItemType::Catchup, - share_stream: false, - stream_options: &stream_options, - stream_url, - req_headers, - input_headers, - session_headers, - disabled_headers: None, - default_user_agent: None, - username: None, - client_ip: None, - stream_channel: None, - connect_failure_stage: None, - content_representation: mode, - }) - } - - async fn local_response( - status: StatusCode, - headers: &[(&str, &str)], - body: Vec, - ) -> (reqwest::Response, Arc) { - use std::fmt::Write as _; - - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - let requests = Arc::new(AtomicUsize::new(0)); - let task_requests = Arc::clone(&requests); - let response_headers = headers.iter().fold(String::new(), |mut headers, (name, value)| { - let _ = write!(headers, "{name}: {value}\r\n"); - headers - }); - tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.unwrap(); - task_requests.fetch_add(1, Ordering::SeqCst); - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 1024]; - let read = socket.read(&mut chunk).await.unwrap(); - if read == 0 { - break; - } - request.extend_from_slice(&chunk[..read]); - } - let reason = status.canonical_reason().unwrap_or("Status"); - let head = format!( - "HTTP/1.1 {} {reason}\r\nContent-Length: {}\r\n{response_headers}Connection: close\r\n\r\n", - status.as_u16(), - body.len() - ); - socket.write_all(head.as_bytes()).await.unwrap(); - socket.write_all(&body).await.unwrap(); - }); - - let response = reqwest::Client::new().get(format!("http://{addr}/resource")).send().await.unwrap(); - (response, requests) - } - - async fn local_stalled_deflate_response() -> (reqwest::Response, tokio::task::JoinHandle<()>) { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - let task = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.unwrap(); - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 1024]; - let read = socket.read(&mut chunk).await.unwrap(); - if read == 0 { - return; - } - request.extend_from_slice(&chunk[..read]); - } - socket - .write_all( - b"HTTP/1.1 200 OK\r\nContent-Encoding: deflate\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n", - ) - .await - .unwrap(); - tokio::time::sleep(Duration::from_secs(5)).await; - }); - let response = reqwest::Client::new().get(format!("http://{addr}/resource")).send().await.unwrap(); - (response, task) - } - - #[derive(Clone, Copy)] - enum TestEncoding { - Gzip, - Zlib, - RawDeflate, - Brotli, - Zstd, - } - - async fn encode(body: &[u8], encoding: TestEncoding) -> Vec { - match encoding { - TestEncoding::Gzip => { - let mut encoder = async_compression::tokio::write::GzipEncoder::new(Vec::new()); - encoder.write_all(body).await.unwrap(); - encoder.shutdown().await.unwrap(); - encoder.into_inner() - } - TestEncoding::Zlib => { - let mut encoder = async_compression::tokio::write::ZlibEncoder::new(Vec::new()); - encoder.write_all(body).await.unwrap(); - encoder.shutdown().await.unwrap(); - encoder.into_inner() - } - TestEncoding::RawDeflate => { - let mut encoder = async_compression::tokio::write::DeflateEncoder::new(Vec::new()); - encoder.write_all(body).await.unwrap(); - encoder.shutdown().await.unwrap(); - encoder.into_inner() - } - TestEncoding::Brotli => { - let mut encoder = async_compression::tokio::write::BrotliEncoder::new(Vec::new()); - encoder.write_all(body).await.unwrap(); - encoder.shutdown().await.unwrap(); - encoder.into_inner() - } - TestEncoding::Zstd => { - let mut encoder = async_compression::tokio::write::ZstdEncoder::new(Vec::new()); - encoder.write_all(body).await.unwrap(); - encoder.shutdown().await.unwrap(); - encoder.into_inner() - } - } - } - - async fn collect_stream(stream: BoxedProviderStream) -> Result, StreamError> { - stream - .try_fold(Vec::new(), |mut bytes, chunk| async move { - bytes.extend_from_slice(&chunk); - Ok(bytes) - }) - .await - } - - #[test] - fn provider_requests_preserve_complete_client_range_header() { - let stream_url = Url::parse("http://provider.example/movie").unwrap(); - let client = reqwest::Client::new(); - - for requested_range in ["bytes=100-199", "bytes=100-", "bytes=-100", "bytes=0-0", "bytes=0-0,200-299"] { - let mut req_headers = HeaderMap::new(); - req_headers.insert(reqwest::header::RANGE, requested_range.parse().unwrap()); - let options = - test_options(ProviderContentRepresentationMode::PreserveOrigin, &stream_url, &req_headers, None, None); - - let (request, partial) = - prepare_client(&client, &options, None, ProviderRequestCredentialState::OriginalOrigin); - let request = request.build().unwrap(); - - assert!(partial, "Range request was not recognized: {requested_range}"); - assert_eq!( - request.headers().get(reqwest::header::RANGE).and_then(|value| value.to_str().ok()), - Some(requested_range) - ); - } - } - - #[test] - fn client_range_takes_precedence_over_configured_provider_range() { - let stream_url = Url::parse("http://provider.example/movie").unwrap(); - let mut req_headers = HeaderMap::new(); - req_headers.insert(reqwest::header::RANGE, "bytes=100-199".parse().unwrap()); - let input_headers = HashMap::from([("Range".to_string(), "bytes=0-".to_string())]); - let options = test_options( - ProviderContentRepresentationMode::PreserveOrigin, - &stream_url, - &req_headers, - Some(&input_headers), - None, - ); - - let request = - prepare_client(&reqwest::Client::new(), &options, None, ProviderRequestCredentialState::OriginalOrigin) - .0 - .build() - .unwrap(); - - assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=100-199"); - } - - #[test] - fn test_provider_stream_factory_options_propagate_buffer_max_bytes() { - let addr = "127.0.0.1:8080".parse().unwrap(); - let stream_url = Url::parse("http://example.com/stream").unwrap(); - let stream_options = StreamOptions { - stream_retry: true, - buffer_enabled: true, - buffer_size: 1024, - buffer_max_bytes: 4096, - pipe_provider_stream: false, - response_mode: StreamResponseMode::Stream, - }; - let req_headers = HeaderMap::new(); - let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { - addr, - item_type: PlaylistItemType::Live, - share_stream: false, - stream_options: &stream_options, - stream_url: &stream_url, - req_headers: &req_headers, - input_headers: None, - session_headers: None, - disabled_headers: None, - default_user_agent: None, - username: None, - client_ip: None, - stream_channel: None, - connect_failure_stage: None, - content_representation: ProviderContentRepresentationMode::PreserveOrigin, - }); - assert_eq!(options.get_buffer_max_bytes(), 4096); - assert_eq!(options.get_buffer_size(), 1024); - } - - #[test] - fn test_provider_stream_factory_options_range_logic() { - let addr = "127.0.0.1:8080".parse().unwrap(); - let stream_url = Url::parse("http://example.com/stream").unwrap(); - let stream_options = StreamOptions { - stream_retry: true, - buffer_enabled: true, - buffer_size: 1024, - buffer_max_bytes: 0, - pipe_provider_stream: false, - response_mode: StreamResponseMode::Stream, - }; - let disabled_headers = None; - - // Case 1: VOD, no initial range requested - let mut req_headers = HeaderMap::new(); - let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { - addr, - item_type: PlaylistItemType::Video, - share_stream: false, - stream_options: &stream_options, - stream_url: &stream_url, - req_headers: &req_headers, - input_headers: None, - session_headers: None, - disabled_headers, - default_user_agent: None, - username: None, - client_ip: None, - stream_channel: None, - connect_failure_stage: None, - content_representation: ProviderContentRepresentationMode::PreserveOrigin, - }); - assert!(!options.was_range_requested()); - assert!(options.get_requested_range().is_none()); - - // Case 2: VOD, range requested - req_headers.insert("Range", "bytes=100-".parse().unwrap()); - let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { - addr, - item_type: PlaylistItemType::Video, - share_stream: false, - stream_options: &stream_options, - stream_url: &stream_url, - req_headers: &req_headers, - input_headers: None, - session_headers: None, - disabled_headers, - default_user_agent: None, - username: None, - client_ip: None, - stream_channel: None, - connect_failure_stage: None, - content_representation: ProviderContentRepresentationMode::PreserveOrigin, - }); - assert!(options.was_range_requested()); - assert_eq!(options.get_requested_range().and_then(|value| value.to_str().ok()), Some("bytes=100-")); - - // Case 3: Live, no initial range requested - let req_headers = HeaderMap::new(); - let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { - addr, - item_type: PlaylistItemType::Live, - share_stream: false, - stream_options: &stream_options, - stream_url: &stream_url, - req_headers: &req_headers, - input_headers: None, - session_headers: None, - disabled_headers, - default_user_agent: None, - username: None, - client_ip: None, - stream_channel: None, - connect_failure_stage: None, - content_representation: ProviderContentRepresentationMode::PreserveOrigin, - }); - assert!(!options.was_range_requested()); - assert!(options.get_requested_range().is_none()); - - // Case 4: Live, range requested (should be stripped) - let mut req_headers = HeaderMap::new(); - req_headers.insert("Range", "bytes=100-".parse().unwrap()); - let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { - addr, - item_type: PlaylistItemType::Live, - share_stream: false, - stream_options: &stream_options, - stream_url: &stream_url, - req_headers: &req_headers, - input_headers: None, - session_headers: None, - disabled_headers, - default_user_agent: None, - username: None, - client_ip: None, - stream_channel: None, - connect_failure_stage: None, - content_representation: ProviderContentRepresentationMode::PreserveOrigin, - }); - assert!(!options.was_range_requested()); // Stripped by filter - assert!(options.get_requested_range().is_none()); - } - - #[test] - fn test_provider_stream_factory_options_keeps_initial_retry_for_live_adaptive_streams() { - let addr = "127.0.0.1:8080".parse().unwrap(); - let stream_url = Url::parse("http://example.com/segment.ts").unwrap(); - let req_headers = HeaderMap::new(); - let stream_options = StreamOptions { - stream_retry: true, - buffer_enabled: true, - buffer_size: 1024, - buffer_max_bytes: 0, - pipe_provider_stream: false, - response_mode: StreamResponseMode::Stream, - }; - - let hls_options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { - addr, - item_type: PlaylistItemType::LiveHls, - share_stream: false, - stream_options: &stream_options, - stream_url: &stream_url, - req_headers: &req_headers, - input_headers: None, - session_headers: None, - disabled_headers: None, - default_user_agent: None, - username: None, - client_ip: None, - stream_channel: None, - connect_failure_stage: None, - content_representation: ProviderContentRepresentationMode::PreserveOrigin, - }); - - let dash_options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { - addr, - item_type: PlaylistItemType::LiveDash, - share_stream: false, - stream_options: &stream_options, - stream_url: &stream_url, - req_headers: &req_headers, - input_headers: None, - session_headers: None, - disabled_headers: None, - default_user_agent: None, - username: None, - client_ip: None, - stream_channel: None, - connect_failure_stage: None, - content_representation: ProviderContentRepresentationMode::PreserveOrigin, - }); - - assert!(hls_options.should_retry_provider_request()); - assert!(dash_options.should_retry_provider_request()); - assert!(!hls_options.should_retry_initial_open_loop()); - assert!(!dash_options.should_retry_initial_open_loop()); - } - - #[test] - fn test_shared_streams_do_not_use_provider_buffer_wrapper() { - let addr = "127.0.0.1:8080".parse().unwrap(); - let stream_url = Url::parse("http://example.com/shared.ts").unwrap(); - let req_headers = HeaderMap::new(); - let stream_options = StreamOptions { - stream_retry: true, - buffer_enabled: true, - buffer_size: 1024, - buffer_max_bytes: 0, - pipe_provider_stream: false, - response_mode: StreamResponseMode::Stream, - }; - - let shared_options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { - addr, - item_type: PlaylistItemType::Live, - share_stream: true, - stream_options: &stream_options, - stream_url: &stream_url, - req_headers: &req_headers, - input_headers: None, - session_headers: None, - disabled_headers: None, - default_user_agent: None, - username: None, - client_ip: None, - stream_channel: None, - connect_failure_stage: None, - content_representation: ProviderContentRepresentationMode::PreserveOrigin, - }); - - assert!( - !should_wrap_provider_stream_in_buffer(&shared_options), - "shared streams must bypass provider-side BufferedStream" - ); - } - - #[test] - fn test_provider_stream_factory_options_builds_connect_failed_stream_info_from_history_context() { - let addr = "127.0.0.1:8080".parse().unwrap(); - let stream_url = Url::parse("http://example.com/stream").unwrap(); - let req_headers = HeaderMap::new(); - let stream_options = StreamOptions { - stream_retry: true, - buffer_enabled: true, - buffer_size: 1024, - buffer_max_bytes: 0, - pipe_provider_stream: false, - response_mode: StreamResponseMode::Stream, - }; - - let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { - addr, - item_type: PlaylistItemType::Live, - share_stream: false, - stream_options: &stream_options, - stream_url: &stream_url, - req_headers: &req_headers, - input_headers: None, - session_headers: None, - disabled_headers: None, - default_user_agent: None, - username: Some("alice"), - client_ip: Some("203.0.113.9"), - stream_channel: Some(&StreamChannel { - target_id: 1, - virtual_id: 77, - provider_id: 3, - input_name: "input-a".intern(), - item_type: PlaylistItemType::Live, - cluster: XtreamCluster::Live, - group: "News".intern(), - title: "Example".intern(), - url: "http://provider.example/live/77".intern(), - shared: false, - shared_joined_existing: None, - shared_stream_id: None, - technical: None, - epg_channel_id: None, - epg_reference_ts: None, - upstream_user_agent: Some("Channel-UA".intern()), - }), - connect_failure_stage: Some(FailureStage::ProviderOpen), - content_representation: ProviderContentRepresentationMode::PreserveOrigin, - }); - - let info = options.build_connect_failed_stream_info("provider-a".intern()).expect("history context"); - - assert_eq!(info.username, "alice"); - assert_eq!(info.client_ip, "203.0.113.9"); - assert_eq!(info.provider.as_ref(), "provider-a"); - assert_eq!(info.channel.input_name.as_ref(), "input-a"); - assert_eq!(info.channel.virtual_id, 77); - assert_eq!( - options.headers.get(reqwest::header::USER_AGENT).and_then(|value| value.to_str().ok()), - Some("Channel-UA") - ); - } - - #[test] - fn html_content_type_is_rejected_for_catchup_streams() { - let mut headers = HeaderMap::new(); - headers.insert(reqwest::header::CONTENT_TYPE, "text/html; charset=UTF-8".parse().unwrap()); - - assert!(should_reject_success_response_content_type(PlaylistItemType::Catchup, &headers)); - assert!(should_reject_success_response_content_type(PlaylistItemType::Video, &headers)); - assert!(should_reject_success_response_content_type(PlaylistItemType::Live, &headers)); - } - - #[test] - fn html_content_type_is_allowed_for_live_adaptive_streams() { - let mut headers = HeaderMap::new(); - headers.insert(reqwest::header::CONTENT_TYPE, "text/html; charset=UTF-8".parse().unwrap()); - - assert!(!should_reject_success_response_content_type(PlaylistItemType::LiveHls, &headers)); - assert!(!should_reject_success_response_content_type(PlaylistItemType::LiveDash, &headers)); - } - - #[test] - fn session_headers_are_forwarded_to_provider_requests() { - let addr = "127.0.0.1:8080".parse().unwrap(); - let stream_url = Url::parse("http://example.com/live/segment.ts").unwrap(); - let req_headers = HeaderMap::new(); - let mut session_headers = HashMap::new(); - session_headers.insert(String::from("cookie"), String::from("sid=abc; pref=1")); - let stream_options = StreamOptions { - stream_retry: true, - buffer_enabled: true, - buffer_size: 1024, - buffer_max_bytes: 0, - pipe_provider_stream: false, - response_mode: StreamResponseMode::Stream, - }; - - let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { - addr, - item_type: PlaylistItemType::LiveHls, - share_stream: false, - stream_options: &stream_options, - stream_url: &stream_url, - req_headers: &req_headers, - input_headers: None, - session_headers: Some(&session_headers), - disabled_headers: None, - default_user_agent: None, - username: None, - client_ip: None, - stream_channel: None, - connect_failure_stage: None, - content_representation: ProviderContentRepresentationMode::PreserveOrigin, - }); - - assert_eq!( - options.get_headers().get(axum::http::header::COOKIE).and_then(|value| value.to_str().ok()), - Some("sid=abc; pref=1") - ); - } - - #[tokio::test] - async fn provider_stream_factory_captures_session_cookies_separately() { - let (response, _) = local_response( - StatusCode::OK, - &[("content-type", "application/vnd.apple.mpegurl"), ("set-cookie", "sid=abc; Path=/")], - b"#EXTM3U\nsegment.ts\n".to_vec(), - ) - .await; - - let result = prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::Identity, - ProviderResponseHeadAvailability::Available, - ) - .await; - - assert!(result.is_ok()); - if let Ok(response) = result { - assert_eq!(response.provider_session_headers.headers.get("cookie").map(String::as_str), Some("sid=abc")); - assert!(response.info.as_ref().is_some_and(|(headers, _, _, _)| headers - .iter() - .all(|(name, _)| !name.eq_ignore_ascii_case("set-cookie")))); - } - } - - #[test] - fn cross_origin_provider_requests_keep_only_redirect_safe_headers() { - let stream_url = Url::parse("http://provider-a.example/live/segment.ts").unwrap(); - let mut req_headers = HeaderMap::new(); - req_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip".parse().unwrap()); - req_headers.insert(reqwest::header::RANGE, "bytes=17-31".parse().unwrap()); - req_headers.insert(reqwest::header::USER_AGENT, "test-agent".parse().unwrap()); - req_headers.insert(reqwest::header::AUTHORIZATION, "Bearer client".parse().unwrap()); - req_headers.insert(reqwest::header::COOKIE, "client=1".parse().unwrap()); - req_headers.insert(reqwest::header::PROXY_AUTHORIZATION, "Basic proxy-secret".parse().unwrap()); - let input_headers = HashMap::from([ - ("X-API-Key".to_string(), "api-secret".to_string()), - ("X-Provider-Token".to_string(), "provider-secret".to_string()), - ]); - let options = test_options( - ProviderContentRepresentationMode::PreserveOrigin, - &stream_url, - &req_headers, - Some(&input_headers), - None, - ); - assert!(provider_headers_require_manual_redirects(options.get_headers())); - - let mut safe_headers = HeaderMap::new(); - safe_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip".parse().unwrap()); - safe_headers.insert(reqwest::header::RANGE, "bytes=17-31".parse().unwrap()); - let safe_options = - test_options(ProviderContentRepresentationMode::PreserveOrigin, &stream_url, &safe_headers, None, None); - assert!(!provider_headers_require_manual_redirects(safe_options.get_headers())); - - let client = reqwest::Client::new(); - let sensitive_headers = ["authorization", "cookie", "proxy-authorization", "x-api-key", "x-provider-token"]; - - let same_origin = Url::parse("http://provider-a.example/live/failover.ts").unwrap(); - let same_origin_request = - prepare_client(&client, &options, Some(&same_origin), ProviderRequestCredentialState::OriginalOrigin) - .0 - .build() - .unwrap(); - for name in sensitive_headers { - assert!(same_origin_request.headers().contains_key(name), "same-origin request lost {name}"); - } - - let cross_origin = Url::parse("http://provider-b.example/live/segment.ts").unwrap(); - for (target, credential_state) in [ - (Some(&cross_origin), ProviderRequestCredentialState::Scrubbed), - (None, ProviderRequestCredentialState::Scrubbed), - ] { - let request = prepare_client(&client, &options, target, credential_state).0.build().unwrap(); - for name in sensitive_headers { - assert!(!request.headers().contains_key(name), "cross-origin request retained {name}"); - } - assert_eq!(request.headers()[reqwest::header::ACCEPT_ENCODING], "gzip"); - assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=17-31"); - assert_eq!(request.headers()[reqwest::header::USER_AGENT], "test-agent"); - } - } - - #[test] - fn provider_scheme_resolution_keeps_configured_headers_before_redirect() { - let stream_url = Url::parse("provider://mirrors/live/segment.ts").unwrap(); - let req_headers = HeaderMap::new(); - let input_headers = HashMap::from([("X-API-Key".to_string(), "api-secret".to_string())]); - let options = test_options( - ProviderContentRepresentationMode::PreserveOrigin, - &stream_url, - &req_headers, - Some(&input_headers), - None, - ); - let resolved_url = Url::parse("https://mirror-a.example/live/segment.ts").unwrap(); - - let request = prepare_client( - &reqwest::Client::new(), - &options, - Some(&resolved_url), - ProviderRequestCredentialState::OriginalOrigin, - ) - .0 - .build() - .unwrap(); - - assert_eq!(request.headers()["x-api-key"], "api-secret"); - } - - #[test] - fn identity_is_enforced_after_header_merges_for_every_request_target() { - let stream_url = Url::parse("http://provider-a.example/live/segment.ts").unwrap(); - let mut req_headers = HeaderMap::new(); - req_headers.insert(reqwest::header::ACCEPT_ENCODING, "br".parse().unwrap()); - req_headers.insert(reqwest::header::RANGE, "bytes=17-31".parse().unwrap()); - req_headers.insert(reqwest::header::AUTHORIZATION, "Bearer client".parse().unwrap()); - req_headers.insert(reqwest::header::COOKIE, "client=1".parse().unwrap()); - let input_headers = HashMap::from([("Accept-Encoding".to_string(), "gzip".to_string())]); - let session_headers = HashMap::from([("Accept-Encoding".to_string(), "zstd".to_string())]); - let options = test_options( - ProviderContentRepresentationMode::Identity, - &stream_url, - &req_headers, - Some(&input_headers), - Some(&session_headers), - ); - let client = reqwest::Client::new(); - - let same_origin = Url::parse("http://provider-a.example/live/failover.ts").unwrap(); - let cross_origin = Url::parse("http://provider-b.example/live/segment.ts").unwrap(); - for target in [None, Some(&same_origin), Some(&cross_origin)] { - let credential_state = if target == Some(&cross_origin) { - ProviderRequestCredentialState::Scrubbed - } else { - ProviderRequestCredentialState::OriginalOrigin - }; - let request = prepare_client(&client, &options, target, credential_state).0.build().unwrap(); - assert_eq!(request.headers()[reqwest::header::ACCEPT_ENCODING], "identity"); - assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=17-31"); - if target == Some(&cross_origin) { - assert!(!request.headers().contains_key(reqwest::header::AUTHORIZATION)); - assert!(!request.headers().contains_key(reqwest::header::COOKIE)); - } - } - - let reconnect_options = options.clone(); - assert_eq!(reconnect_options.content_representation(), ProviderContentRepresentationMode::Identity); - assert_eq!(reconnect_options.hls_content_coding_object_kind(), Some(HlsOriginContentCodingObjectKind::Other)); - assert_eq!( - options.clone().for_deferred_open().hls_content_coding_object_kind(), - Some(HlsOriginContentCodingObjectKind::Other) - ); - } - - #[test] - fn preserve_origin_does_not_override_accept_encoding() { - let stream_url = Url::parse("http://provider.example/movie").unwrap(); - let mut req_headers = HeaderMap::new(); - req_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip, br".parse().unwrap()); - let options = - test_options(ProviderContentRepresentationMode::PreserveOrigin, &stream_url, &req_headers, None, None); - - let request = - prepare_client(&reqwest::Client::new(), &options, None, ProviderRequestCredentialState::OriginalOrigin) - .0 - .build() - .unwrap(); - - assert_eq!(request.headers()[reqwest::header::ACCEPT_ENCODING], "gzip, br"); - } - - #[tokio::test] - async fn automatic_redirect_round_trip_keeps_identity_and_never_restores_credentials() { - let encoded = encode(b"decoded redirect body", TestEncoding::Gzip).await; - let RedirectRoundTrip { entry_url, first_origin_task, second_origin_task } = - spawn_redirect_round_trip(encoded).await; - let options = redirect_test_options(&entry_url); - let client = reqwest::Client::builder() - .no_proxy() - .redirect(reqwest::redirect::Policy::limited(10)) - .build() - .expect("automatic redirect client"); - - let response = prepare_client(&client, &options, None, ProviderRequestCredentialState::OriginalOrigin) - .0 - .send() - .await - .expect("automatic redirect request succeeds"); - - assert_identity_redirect_result(response, first_origin_task, second_origin_task).await; - } - - #[tokio::test] - async fn manual_redirect_round_trip_keeps_identity_and_never_restores_credentials() { - let encoded = encode(b"decoded redirect body", TestEncoding::Gzip).await; - let RedirectRoundTrip { entry_url, first_origin_task, second_origin_task } = - spawn_redirect_round_trip(encoded).await; - let options = redirect_test_options(&entry_url); - let client = reqwest::Client::builder() - .no_proxy() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("manual redirect client"); - - let response = send_with_manual_redirects(&client, &options, &test_app_config()) - .await - .expect("manual redirect request succeeds"); - - assert_identity_redirect_result(response, first_origin_task, second_origin_task).await; - } - - #[tokio::test] - async fn deferred_open_without_origin_head_accepts_only_plain_origin_200() { - const IDENTITY_BODY: &[u8] = b"deferred identity body"; - - let stream_url = Url::parse("http://provider.example/live/deferred.ts").unwrap(); - let mut req_headers = HeaderMap::new(); - req_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip".parse().unwrap()); - let deferred_options = - test_options(ProviderContentRepresentationMode::PreserveOrigin, &stream_url, &req_headers, None, None) - .for_deferred_open(); - let deferred_request = prepare_client( - &reqwest::Client::new(), - &deferred_options, - None, - ProviderRequestCredentialState::OriginalOrigin, - ) - .0 - .build() - .expect("deferred provider request"); - assert_eq!(deferred_options.content_representation(), ProviderContentRepresentationMode::Identity); - assert!(!deferred_options.response_head_is_available()); - assert_eq!(deferred_options.hls_content_coding_object_kind(), None); - assert_eq!(deferred_request.headers()[reqwest::header::ACCEPT_ENCODING], "identity"); - - let encoded = encode(IDENTITY_BODY, TestEncoding::Gzip).await; - let (response, _) = local_response(StatusCode::OK, &[("Content-Encoding", "gzip")], encoded).await; - let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::Identity, - ProviderResponseHeadAvailability::Unavailable, - ) - .await - .expect("plain 200 is safe for deferred identity streaming"); - assert_eq!(collect_stream(stream).await.expect("deferred body decodes"), IDENTITY_BODY); - let (headers, status, _, _) = info.expect("normalized deferred response metadata"); - assert_eq!(status, StatusCode::OK); - assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-encoding"))); - assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-length"))); - - let encoded = encode(IDENTITY_BODY, TestEncoding::Gzip).await; - let (response, _) = local_response( - StatusCode::OK, - &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 0-21/22")], - encoded, - ) - .await; - assert!(matches!( - prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::Identity, - ProviderResponseHeadAvailability::Unavailable, - ) - .await, - Err(ProviderStreamPreparationError::DeferredResponseHead { - status: StatusCode::OK, - has_content_range: true, - }) - )); - - let (response, _) = local_response(StatusCode::PARTIAL_CONTENT, &[], IDENTITY_BODY.to_vec()).await; - assert!(matches!( - prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::Identity, - ProviderResponseHeadAvailability::Unavailable, - ) - .await, - Err(ProviderStreamPreparationError::DeferredResponseHead { - status: StatusCode::PARTIAL_CONTENT, - has_content_range: false, - }) - )); - } - - #[tokio::test] - async fn deferred_open_rejects_preserve_origin_without_origin_head() { - let (response, _) = local_response(StatusCode::OK, &[], b"opaque bytes".to_vec()).await; - - assert!(matches!( - prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::PreserveOrigin, - ProviderResponseHeadAvailability::Unavailable, - ) - .await, - Err(ProviderStreamPreparationError::DeferredResponseHead { - status: StatusCode::OK, - has_content_range: false, - }) - )); - } - - #[tokio::test] - async fn preserve_origin_rejects_invalid_representation_header_before_stream_release() { - let (response, requests) = - local_response(StatusCode::OK, &[("Content-Encoding", "gzip,,br")], b"body must not be released".to_vec()) - .await; - - assert!(matches!( - prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::PreserveOrigin, - ProviderResponseHeadAvailability::Available, - ) - .await, - Err(ProviderStreamPreparationError::ResponseHeader(ProviderResponseHeaderError::InvalidContentEncoding)) - )); - assert_eq!(requests.load(Ordering::SeqCst), 1); - } - - #[tokio::test] - async fn legacy_hls_segment_key_map_part_and_other_stream_as_identity() { - const IDENTITY: &[u8] = b"identity hls resource bytes"; - let cases = [ - ("segment", "gzip", TestEncoding::Gzip), - ("segment-zlib", "deflate", TestEncoding::Zlib), - ("segment-raw-deflate", "deflate", TestEncoding::RawDeflate), - ("key", "br", TestEncoding::Brotli), - ("map", "zstd", TestEncoding::Zstd), - ("part", "gzip", TestEncoding::Gzip), - ("other", "br", TestEncoding::Brotli), - ]; - - for (resource_kind, content_encoding, encoding) in cases { - let encoded = encode(IDENTITY, encoding).await; - let (response, _) = - local_response(StatusCode::OK, &[("Content-Encoding", content_encoding)], encoded).await; - - let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::Identity, - ProviderResponseHeadAvailability::Available, - ) - .await - .unwrap(); - - assert_eq!(collect_stream(stream).await.unwrap(), IDENTITY, "failed for {resource_kind}"); - let (headers, status, _, _) = info.unwrap(); - assert_eq!(status, StatusCode::OK); - assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-encoding"))); - assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-length"))); - } - } - - #[tokio::test] - async fn legacy_hls_declared_only_preserves_headerless_gzip_magic() { - let body = Bytes::from_static(b"\x1f\x8bnot-an-encoded-key"); - let (response, _) = local_response(StatusCode::OK, &[], body.to_vec()).await; - - let ProviderStreamFactoryResponse { stream, .. } = prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::Identity, - ProviderResponseHeadAvailability::Available, - ) - .await - .unwrap(); - - assert_eq!(collect_stream(stream).await.unwrap(), body); - } - - #[tokio::test] - async fn legacy_hls_identity_keeps_unencoded_partial_content_headers() { - const PARTIAL_BODY: &[u8] = b"abc"; - let (response, _) = - local_response(StatusCode::PARTIAL_CONTENT, &[("Content-Range", "bytes 0-2/10")], PARTIAL_BODY.to_vec()) - .await; - - let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::Identity, - ProviderResponseHeadAvailability::Available, - ) - .await - .expect("unencoded partial identity response is safe"); - let (headers, status, _, _) = info.expect("partial response metadata"); - - assert_eq!(collect_stream(stream).await.expect("partial body streams"), PARTIAL_BODY); - assert_eq!(status, StatusCode::PARTIAL_CONTENT); - assert!(headers.iter().any(|(name, value)| name == "content-length" && value == "3")); - assert!(headers.iter().any(|(name, value)| name == "content-range" && value == "bytes 0-2/10")); - assert!(headers.iter().all(|(name, _)| name != "content-encoding")); - } - - #[tokio::test] - async fn legacy_hls_identity_decoding_removes_stale_representation_headers() { - const IDENTITY_BODY: &[u8] = b"decoded identity response"; - let encoded = encode(IDENTITY_BODY, TestEncoding::Gzip).await; - let (response, _) = - local_response(StatusCode::OK, &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 0-9/10")], encoded) - .await; - - let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::Identity, - ProviderResponseHeadAvailability::Available, - ) - .await - .expect("encoded full response decodes"); - let (headers, status, _, _) = info.expect("decoded response metadata"); - - assert_eq!(collect_stream(stream).await.expect("decoded body streams"), IDENTITY_BODY); - assert_eq!(status, StatusCode::OK); - for name in ["content-encoding", "content-length", "content-range"] { - assert!(headers.iter().all(|(key, _)| key != name), "stale header {name}"); - } - } - - #[tokio::test] - async fn legacy_hls_rejects_encoded_partial_content_before_client_streaming() { - let encoded = encode(b"partial identity bytes", TestEncoding::Gzip).await; - let (response, _) = local_response( - StatusCode::PARTIAL_CONTENT, - &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 0-9/10")], - encoded, - ) - .await; - - let result = prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::Identity, - ProviderResponseHeadAvailability::Available, - ) - .await; - - assert!(matches!( - result, - Err(ProviderStreamPreparationError::ContentCoding(ContentCodingError::EncodedPartialContent)) - )); - } - - #[tokio::test] - async fn legacy_hls_decoder_setup_obeys_provider_body_idle_timeout() { - let (response, origin_task) = local_stalled_deflate_response().await; - - let result = prepare_provider_stream_response_with_idle_timeout( - response, - ProviderContentRepresentationMode::Identity, - ProviderResponseHeadAvailability::Available, - Duration::from_millis(25), - ) - .await; - origin_task.abort(); - - assert!(matches!( - result, - Err(ProviderStreamPreparationError::ContentCoding(ContentCodingError::PrefixRead(error))) - if error.kind() == io::ErrorKind::TimedOut - )); - } - - #[tokio::test] - async fn preserve_origin_keeps_encoded_body_and_representation_headers() { - let encoded = encode(b"preserved representation", TestEncoding::Gzip).await; - let expected = encoded.clone(); - let expected_len = encoded.len().to_string(); - let (response, _) = local_response( - StatusCode::PARTIAL_CONTENT, - &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 10-20/100")], - encoded, - ) - .await; - - let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::PreserveOrigin, - ProviderResponseHeadAvailability::Available, - ) - .await - .unwrap(); - let (headers, status, _, _) = info.unwrap(); - - assert_eq!(collect_stream(stream).await.unwrap(), expected); - assert_eq!(status, StatusCode::PARTIAL_CONTENT); - assert!(headers.iter().any(|(name, value)| name == "content-encoding" && value == "gzip")); - assert!(headers.iter().any(|(name, value)| name == "content-length" && value == &expected_len)); - assert!(headers.iter().any(|(name, value)| name == "content-range" && value == "bytes 10-20/100")); - assert!(headers.iter().all(|(name, _)| name != "transfer-encoding")); - } - - #[tokio::test] - async fn preserve_origin_keeps_unknown_content_coding_and_body_unchanged() { - let body = b"opaque provider representation".to_vec(); - let expected = body.clone(); - let expected_len = body.len().to_string(); - let (response, _) = local_response( - StatusCode::PARTIAL_CONTENT, - &[("Content-Encoding", "x-provider-coding"), ("Content-Range", "bytes 0-29/100")], - body, - ) - .await; - - let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::PreserveOrigin, - ProviderResponseHeadAvailability::Available, - ) - .await - .expect("valid unknown coding is preserved"); - let (headers, status, _, _) = info.expect("preserved response metadata"); - - assert_eq!(collect_stream(stream).await.expect("opaque body streams"), expected); - assert_eq!(status, StatusCode::PARTIAL_CONTENT); - assert!(headers.iter().any(|(name, value)| name == "content-encoding" && value == "x-provider-coding")); - assert!(headers.iter().any(|(name, value)| name == "content-length" && value == &expected_len)); - assert!(headers.iter().any(|(name, value)| name == "content-range" && value == "bytes 0-29/100")); - } - - #[tokio::test] - async fn preserve_origin_never_magic_sniffs_headerless_body() { - let body = Bytes::from_static(b"\x1f\x8bopaque non-hls bytes"); - let (response, _) = local_response(StatusCode::OK, &[], body.to_vec()).await; - - let ProviderStreamFactoryResponse { stream, .. } = prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::PreserveOrigin, - ProviderResponseHeadAvailability::Available, - ) - .await - .unwrap(); - - assert_eq!(collect_stream(stream).await.unwrap(), body); - } - - #[tokio::test] - async fn legacy_hls_stream_decoder_failure_aborts_without_retry() { - let mut truncated = encode(b"decoder failure after response headers", TestEncoding::Gzip).await; - truncated.truncate(truncated.len().saturating_sub(5)); - let (response, requests) = local_response(StatusCode::OK, &[("Content-Encoding", "gzip")], truncated).await; - - let ProviderStreamFactoryResponse { stream, .. } = prepare_provider_stream_response( - response, - ProviderContentRepresentationMode::Identity, - ProviderResponseHeadAvailability::Available, - ) - .await - .unwrap(); - let error = collect_stream(stream).await.expect_err("truncated gzip must terminate the body stream"); - - assert!(matches!(error, StreamError::ContentDecoding(_))); - assert_eq!(requests.load(Ordering::SeqCst), 1, "a body-stream failure cannot trigger a new request"); - } -} +mod tests; + +mod error; +mod opening; +mod options; +mod request; +mod response; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use opening::{create_provider_stream, open_provider_stream_with_lifecycle, ProviderStreamOpen}; +pub(crate) use options::ProviderStreamFactoryParams; +#[cfg(test)] +use request::{prepare_client, provider_headers_require_manual_redirects, send_with_manual_redirects}; +#[cfg(test)] +use response::normalize_hls_resource_content_type; +#[cfg(test)] +use response::prepare_provider_stream_response; +#[cfg(test)] +use response::prepare_provider_stream_response_with_idle_timeout; diff --git a/backend/app/src/api/model/streams/provider_stream_factory/error.rs b/backend/app/src/api/model/streams/provider_stream_factory/error.rs new file mode 100644 index 000000000..9f7fd5bbe --- /dev/null +++ b/backend/app/src/api/model/streams/provider_stream_factory/error.rs @@ -0,0 +1,196 @@ +use super::ProviderStreamFactoryOptions; +use crate::{ + api::model::StreamError, + utils::content_coding::{content_decoding_error_from_io, ContentCodingError}, +}; +use reqwest::{header::HeaderMap, StatusCode}; +use shared::{ + model::{ConnectFailureReason, PlaylistItemType}, + utils::Internable, +}; +use std::{error::Error as StdError, io}; +use tuliprox_session::{response_headers::ProviderResponseHeaderError, stream_ctx::ProviderStreamCtx}; +use url::Url; + +pub(super) fn record_provider_open_failure( + ctx: &ProviderStreamCtx, + stream_options: &ProviderStreamFactoryOptions, + reason: ConnectFailureReason, + provider_http_status: Option, + provider_error_class: Option<&str>, +) { + let Some(failure_stage) = stream_options.get_connect_failure_stage() else { return }; + let provider_name = + stream_options.get_provider().map_or_else(|| "unknown".intern(), |provider| provider.name.clone()); + let Some(info) = stream_options.build_connect_failed_stream_info(provider_name) else { return }; + // Resolve target_name from target_id using the stable target config name. + let target_name = ctx.app_config.get_target_by_id(info.channel.target_id).as_deref().map(|t| (&t.name).intern()); + ctx.connection_manager.record_connect_failed_with_provider_failure( + &info, + reason, + failure_stage, + provider_http_status.map(|status| status.as_u16()), + provider_error_class, + target_name, + ); +} + +pub(super) fn classify_provider_status_error(status: StatusCode) -> &'static str { + if status.is_client_error() { + "http_4xx" + } else if status.is_server_error() { + "http_5xx" + } else if status.is_redirection() { + "http_3xx" + } else { + "http_other" + } +} + +fn provider_content_type_looks_like_html(headers: &HeaderMap) -> bool { + headers + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.split(';').next().unwrap_or_default().trim().eq_ignore_ascii_case("text/html")) +} + +pub(super) fn should_reject_success_response_content_type(item_type: PlaylistItemType, headers: &HeaderMap) -> bool { + !item_type.is_live_adaptive() && provider_content_type_looks_like_html(headers) +} + +#[derive(Debug)] +pub(super) enum ProviderStreamRequestFailure { + HlsStatus { status: StatusCode, headers: Vec<(String, String)> }, + Status { status: StatusCode, provider_error_class: &'static str, serve_channel_unavailable: bool }, +} + +#[derive(Debug, thiserror::Error)] +pub(super) enum ProviderStreamPreparationError { + #[error(transparent)] + ContentCoding(#[from] ContentCodingError), + + #[error(transparent)] + ResponseHeader(#[from] ProviderResponseHeaderError), + + #[error("provider response head is unavailable for status {status} or Content-Range={has_content_range}")] + DeferredResponseHead { status: StatusCode, has_content_range: bool }, +} + +impl ProviderStreamPreparationError { + pub(super) fn provider_error_class(&self) -> &'static str { + match self { + Self::ContentCoding(ContentCodingError::InvalidHeader | ContentCodingError::Unsupported(_)) + | Self::ResponseHeader(ProviderResponseHeaderError::InvalidContentEncoding) => "content_encoding", + Self::ContentCoding(ContentCodingError::EncodedPartialContent) => "encoded_partial_content", + Self::ContentCoding(ContentCodingError::PrefixRead(_)) => "body", + Self::ResponseHeader( + ProviderResponseHeaderError::InvalidContentLength | ProviderResponseHeaderError::InvalidContentRange, + ) + | Self::DeferredResponseHead { .. } => "response_headers", + } + } +} + +impl ProviderStreamRequestFailure { + pub(super) fn status(&self) -> StatusCode { + match self { + Self::Status { status, .. } | Self::HlsStatus { status, .. } => *status, + } + } + + pub(super) fn provider_error_class(&self) -> &'static str { + match self { + Self::Status { provider_error_class, .. } => provider_error_class, + Self::HlsStatus { status, .. } => classify_provider_status_error(*status), + } + } + + pub(super) fn should_serve_channel_unavailable(&self) -> bool { + match self { + Self::Status { serve_channel_unavailable, .. } => *serve_channel_unavailable, + Self::HlsStatus { .. } => false, + } + } +} + +pub(super) fn classify_provider_io_error(err: &std::io::Error) -> &'static str { + use std::io::ErrorKind; + + match err.kind() { + ErrorKind::TimedOut => "timeout", + ErrorKind::ConnectionRefused + | ErrorKind::ConnectionReset + | ErrorKind::ConnectionAborted + | ErrorKind::NotConnected => "connect", + ErrorKind::AddrNotAvailable => "dns", + _ => { + let lowered = err.to_string().to_ascii_lowercase(); + if lowered.contains("dns") + || lowered.contains("failed to lookup address information") + || lowered.contains("name or service not known") + || lowered.contains("no such host") + || lowered.contains("temporary failure in name resolution") + { + "dns" + } else { + "io" + } + } + } +} + +pub(super) fn provider_decoded_body_error(error: &io::Error) -> StreamError { + if let Some(error) = content_decoding_error_from_io(error) { + return StreamError::ContentDecoding(format!("coding={}", error.coding.as_http_token())); + } + + let mut source = StdError::source(error); + while let Some(current) = source { + if let Some(error) = current.downcast_ref::() { + return StreamError::reqwest(error); + } + source = current.source(); + } + StreamError::StdIo(error.to_string()) +} + +pub(super) fn failed_stream_account( + input: &crate::model::ConfigInput, + options: &ProviderStreamFactoryOptions, +) -> Option { + let url = &options.url; + let credentials = shared::utils::get_credentials_from_url(url); + let segments: Vec<_> = url.path_segments().into_iter().flatten().collect(); + let matches = |username: Option<&str>, password: Option<&str>| { + username.zip(password).is_some_and(|(username, password)| { + credentials.0.as_deref() == Some(username) && credentials.1.as_deref() == Some(password) + || segments.windows(2).any(|pair| pair[0] == username && pair[1] == password) + }) + }; + let matches_origin = |base: &str| { + input + .resolve_url(base) + .ok() + .and_then(|base| Url::parse(&base).ok()) + .is_some_and(|base| base.origin() == url.origin()) + }; + let account = if let Some(allocated) = &options.account { + if input.name == allocated.name { + input.clone() + } else { + let alias = input.aliases.iter().flatten().find(|alias| alias.name == allocated.name)?; + input.as_input(alias) + } + } else if matches(input.username.as_deref(), input.password.as_deref()) && matches_origin(&input.url) { + input.clone() + } else { + let alias = input.aliases.iter().flatten().find(|alias| { + matches(alias.username.as_deref(), alias.password.as_deref()) && matches_origin(&alias.url) + })?; + input.as_input(alias) + }; + if options.account.as_ref().is_some_and(|allocated| account.account_identity() != allocated.account_identity()) { + return None; + } + Some(account) +} diff --git a/backend/app/src/api/model/streams/provider_stream_factory/opening.rs b/backend/app/src/api/model/streams/provider_stream_factory/opening.rs new file mode 100644 index 000000000..703a28e4a --- /dev/null +++ b/backend/app/src/api/model/streams/provider_stream_factory/opening.rs @@ -0,0 +1,191 @@ +use super::{ + error::{record_provider_open_failure, ProviderStreamRequestFailure}, + options::should_wrap_provider_stream_in_buffer, + request::get_provider_stream, + ProviderOpenGuard, ProviderStreamFactoryFlags, ProviderStreamFactoryOptions, ProviderStreamOpenLifecycle, +}; +use crate::api::model::{ + create_channel_unavailable_stream, get_response_headers, streams::client_stream::ClientStream, + CustomVideoStreamType, ProviderStreamFactoryResponse, +}; +use futures::StreamExt; +use reqwest::StatusCode; +use shared::model::ConnectFailureReason; +use std::sync::Arc; +use tokio_util::sync::CancellationToken; +use tuliprox_session::{ + stream_ctx::ProviderStreamCtx, + streams::{ProviderBodyOwner, ProviderBodyOwnerConfig}, + ManagedProviderHandle, ProviderSessionHeaders, +}; + +/// Result of opening a provider stream. +pub enum ProviderStreamOpen { + Stream(ProviderStreamFactoryResponse), + /// A finite HLS resource keeps the upstream error status and its range and retry headers + /// instead of receiving a fallback body. + UpstreamStatus { + status: StatusCode, + headers: Vec<(String, String)>, + }, +} + +#[allow(clippy::too_many_lines)] +pub async fn create_provider_stream( + ctx: &ProviderStreamCtx, + client: &reqwest::Client, + stream_options: ProviderStreamFactoryOptions, +) -> Option { + let mut open_guard = ProviderOpenGuard { token: stream_options.get_completion_token(), handed_off: false }; + match get_provider_stream(ctx, client, &stream_options).await { + Ok(Some(ProviderStreamFactoryResponse { stream: init_stream, info, provider_session_headers, .. })) => { + let continue_signal = stream_options.get_reconnect_flag_clone(); + if let Some((_headers, _status, _response_url, Some(custom_video_type))) = &info { + let reason = match custom_video_type { + CustomVideoStreamType::ChannelUnavailable => Some(ConnectFailureReason::ChannelUnavailable), + CustomVideoStreamType::Provisioning => Some(ConnectFailureReason::Provisioning), + CustomVideoStreamType::ProviderConnectionsExhausted => { + Some(ConnectFailureReason::ProviderConnectionsExhausted) + } + _ => None, + }; + if let Some(reason) = reason { + record_provider_open_failure(ctx, &stream_options, reason, None, None); + } + return Some(ProviderStreamOpen::Stream(ProviderStreamFactoryResponse { + stream: ClientStream::new(init_stream, continue_signal, None, stream_options.get_url_as_str()) + .boxed(), + info, + provider_session_headers, + has_upstream_owner: false, + })); + } + let handle_cancel = stream_options.get_cancel_token(); + let completion_token = stream_options.get_completion_token(); + let close_reason = stream_options.get_close_reason(); + + let cancel_token = if let Some(hc) = handle_cancel { + let combined = CancellationToken::new(); + let c1 = combined.clone(); + let c2 = combined.clone(); + let hc_clone = hc.clone(); + let cs_clone = continue_signal.clone(); + tokio::spawn(async move { + tokio::select! { + biased; + () = hc_clone.cancelled() => { + cs_clone.cancel(); + c1.cancel(); + } + () = cs_clone.cancelled() => { + c2.cancel(); + } + } + }); + combined + } else { + continue_signal.clone() + }; + + let body_owner_config = if should_wrap_provider_stream_in_buffer(&stream_options) { + ProviderBodyOwnerConfig::for_buffered( + stream_options.get_buffer_size(), + stream_options.get_buffer_max_bytes(), + ) + } else { + ProviderBodyOwnerConfig::for_direct_body() + }; + + open_guard.handed_off = true; + let stream = ProviderBodyOwner::new( + init_stream.boxed(), + body_owner_config, + cancel_token, + completion_token, + close_reason, + ) + .boxed(); + + Some(ProviderStreamOpen::Stream(ProviderStreamFactoryResponse { + stream: ClientStream::new(stream, continue_signal.clone(), None, stream_options.get_url_as_str()) + .boxed(), + info, + provider_session_headers, + has_upstream_owner: true, + })) + } + Ok(None) => None, + Err(failure) => { + let status = failure.status(); + record_provider_open_failure( + ctx, + &stream_options, + ConnectFailureReason::ChannelUnavailable, + Some(status), + Some(failure.provider_error_class()), + ); + if stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource) { + let headers = match failure { + ProviderStreamRequestFailure::HlsStatus { headers, .. } => headers, + ProviderStreamRequestFailure::Status { .. } => Vec::new(), + }; + return Some(ProviderStreamOpen::UpstreamStatus { status, headers }); + } + if let (Some(boxed_provider_stream), response_info) = create_channel_unavailable_stream( + &ctx.app_config, + &get_response_headers(stream_options.get_headers()), + StatusCode::OK, + ) { + return Some(ProviderStreamOpen::Stream(ProviderStreamFactoryResponse { + stream: boxed_provider_stream, + info: response_info, + provider_session_headers: ProviderSessionHeaders::default(), + has_upstream_owner: false, + })); + } + None + } + } +} + +impl ProviderStreamOpenLifecycle { + pub fn from_managed(managed: &ManagedProviderHandle) -> Option { + let handle = managed.handle()?; + Some(Self { + manager: Arc::clone(managed.manager()), + allocation_id: handle.allocation_id, + cancel_token: handle.cancel_token.clone(), + completion_token: handle.completion_token.clone(), + close_reason: Some(Arc::clone(&handle.close_reason)), + account: handle.allocation.get_provider_config(), + }) + } + + pub fn mark_opening(&self) { self.manager.mark_opening(self.allocation_id); } + + pub fn register_body_owner(&self) -> bool { self.manager.register_body_owner(self.allocation_id) } +} + +pub async fn open_provider_stream_with_lifecycle( + ctx: &ProviderStreamCtx, + client: &reqwest::Client, + mut stream_options: ProviderStreamFactoryOptions, + lifecycle: Option, +) -> Option { + if let Some(ref lc) = lifecycle { + lc.mark_opening(); + stream_options.account.clone_from(&lc.account); + stream_options.set_provider_handle_tokens( + lc.cancel_token.clone(), + lc.completion_token.clone(), + lc.close_reason.clone(), + ); + } + let open = create_provider_stream(ctx, client, stream_options).await?; + if let (ProviderStreamOpen::Stream(response), Some(lc)) = (&open, lifecycle.as_ref()) { + if response.has_upstream_owner && !lc.register_body_owner() { + return None; + } + } + Some(open) +} diff --git a/backend/app/src/api/model/streams/provider_stream_factory/options.rs b/backend/app/src/api/model/streams/provider_stream_factory/options.rs new file mode 100644 index 000000000..86713f8d6 --- /dev/null +++ b/backend/app/src/api/model/streams/provider_stream_factory/options.rs @@ -0,0 +1,317 @@ +use super::{ + request::{merge_provider_request_headers, same_origin}, + ProviderResponseHeadAvailability, ProviderStreamFactoryFlags, ProviderStreamFactoryFlagsSet, + ProviderStreamFactoryOptions, +}; +use crate::{ + api::model::{get_header_filter_for_item_type, ProviderContentRepresentationMode}, + model::{ConfigProvider, ReverseProxyDisabledHeaderConfig}, + utils::request::{get_request_headers, preview_request_target_for_logging}, +}; +use reqwest::header::{HeaderMap, HeaderValue, RANGE}; +use shared::{ + model::{FailureStage, PlaylistItemType, StreamChannel, StreamInfo}, + utils::is_sanitize_sensitive_info_enabled, +}; +use std::{ + collections::HashMap, + net::SocketAddr, + sync::{atomic::AtomicU8, Arc}, +}; +use tokio_util::sync::CancellationToken; +use tuliprox_core::utils::request_headers::get_headers_from_request; +use tuliprox_hls::api::HlsOriginContentCodingObjectKind; +use tuliprox_session::stream_options::{StreamOptions, StreamResponseMode}; +use url::Url; + +pub(crate) struct ProviderStreamFactoryParams<'a> { + pub addr: SocketAddr, + pub item_type: PlaylistItemType, + pub share_stream: bool, + pub stream_options: &'a StreamOptions, + pub stream_url: &'a Url, + pub req_headers: &'a HeaderMap, + pub input_headers: Option<&'a HashMap>, + pub session_headers: Option<&'a HashMap>, + pub disabled_headers: Option<&'a ReverseProxyDisabledHeaderConfig>, + pub default_user_agent: Option<&'a str>, + pub username: Option<&'a str>, + pub client_ip: Option<&'a str>, + pub stream_channel: Option<&'a StreamChannel>, + pub connect_failure_stage: Option, + pub content_representation: ProviderContentRepresentationMode, +} + +impl ProviderStreamFactoryOptions { + pub(crate) fn new(request: &ProviderStreamFactoryParams<'_>) -> Self { + let ProviderStreamFactoryParams { + addr, + item_type, + share_stream, + stream_options, + stream_url, + req_headers, + input_headers, + session_headers, + disabled_headers, + default_user_agent, + username, + client_ip, + stream_channel, + connect_failure_stage, + content_representation, + } = request; + let buffer_size = if stream_options.buffer_enabled { stream_options.buffer_size } else { 0 }; + let buffer_max_bytes = stream_options.buffer_max_bytes; + let user_agent = req_headers + .get(axum::http::header::USER_AGENT) + .and_then(|value| value.to_str().ok()) + .map(ToString::to_string); + let filter_header = if stream_options.response_mode == StreamResponseMode::HlsResource { + None + } else { + get_header_filter_for_item_type(*item_type) + }; + let mut req_headers = get_headers_from_request(req_headers, &filter_header); + let requested_range = req_headers.remove(RANGE.as_str()).and_then(|value| HeaderValue::from_bytes(&value).ok()); + + let merged_input_headers = merge_provider_request_headers(*input_headers, *session_headers); + + // We merge configured input headers with the headers from the request. + let mut headers = get_request_headers( + merged_input_headers.as_ref(), + Some(&req_headers), + *disabled_headers, + *default_user_agent, + ); + crate::utils::request::overlay_upstream_user_agent( + &mut headers, + stream_channel.as_ref().and_then(|channel| channel.upstream_user_agent.as_deref()), + *disabled_headers, + ); + + let default_user_agent = default_user_agent + .and_then(|ua| { + let trimmed = ua.trim(); + (!trimmed.is_empty()).then_some(trimmed) + }) + .and_then(|ua| axum::http::header::HeaderValue::from_str(ua).ok()); + let url = (*stream_url).clone(); + let mut flags = ProviderStreamFactoryFlagsSet::new(); + if stream_options.stream_retry { + flags.set(ProviderStreamFactoryFlags::RetryEnabled); + if !item_type.is_live_adaptive() && stream_options.response_mode != StreamResponseMode::HlsResource { + flags.set(ProviderStreamFactoryFlags::InitialRetryLoopEnabled); + } + } + if stream_options.pipe_provider_stream { + flags.set(ProviderStreamFactoryFlags::PipeStream); + } + if stream_options.buffer_enabled { + flags.set(ProviderStreamFactoryFlags::BufferEnabled); + } + if stream_options.response_mode == StreamResponseMode::HlsResource { + flags.set(ProviderStreamFactoryFlags::HlsResource); + } + if *share_stream { + flags.set(ProviderStreamFactoryFlags::ShareStream); + } + if requested_range.is_some() { + flags.set(ProviderStreamFactoryFlags::RangeRequested); + } + + Self { + item_type: *item_type, + addr: *addr, + flags, + buffer_size, + buffer_max_bytes, + reconnect_flag: CancellationToken::new(), + url, + headers, + default_user_agent, + requested_range, + provider: None, + username: username.map(ToString::to_string), + client_ip: client_ip.map(ToString::to_string), + user_agent, + stream_channel: stream_channel.cloned(), + connect_failure_stage: *connect_failure_stage, + content_representation: *content_representation, + response_head_availability: ProviderResponseHeadAvailability::Available, + hls_content_coding_object_kind: match *content_representation { + ProviderContentRepresentationMode::Identity => Some(HlsOriginContentCodingObjectKind::Other), + ProviderContentRepresentationMode::PreserveOrigin => None, + }, + cancel_token: None, + completion_token: None, + close_reason: None, + account: None, + } + } + + pub fn set_provider_handle_tokens( + &mut self, + cancel_token: Option, + completion_token: Option, + close_reason: Option>, + ) { + self.cancel_token = cancel_token; + self.completion_token = completion_token; + self.close_reason = close_reason; + } + + pub fn get_cancel_token(&self) -> Option { self.cancel_token.clone() } + + pub fn get_completion_token(&self) -> Option { self.completion_token.clone() } + + pub fn get_close_reason(&self) -> Option> { self.close_reason.clone() } + + pub fn set_provider(&mut self, provider: Option>) { self.provider = provider; } + + pub fn apply_user_agent_stream_index(&mut self, stream_index: u64) { + crate::utils::request::append_user_agent_stream_index(&mut self.headers, stream_index); + } + + pub fn require_public_destination(&mut self) { + self.flags.set(ProviderStreamFactoryFlags::PublicDestinationRequired); + } + + /// Applies per-input stream options that the generic factory parameters do not carry. + pub fn apply_input_options(&mut self, input: &crate::model::ConfigInput) { + if input.has_flag(crate::model::ConfigInputFlags::FlussonicHlsAudioTracks) { + self.flags.set(ProviderStreamFactoryFlags::FlussonicAudioTracks); + } + } + + pub fn get_provider(&self) -> Option<&Arc> { self.provider.as_ref() } + + #[inline] + pub(super) fn is_piped(&self) -> bool { self.flags.contains(ProviderStreamFactoryFlags::PipeStream) } + + #[inline] + pub(super) fn is_buffer_enabled(&self) -> bool { self.flags.contains(ProviderStreamFactoryFlags::BufferEnabled) } + + #[inline] + pub(crate) fn get_buffer_size(&self) -> usize { self.buffer_size } + + #[inline] + pub(crate) fn get_buffer_max_bytes(&self) -> usize { self.buffer_max_bytes } + + #[inline] + pub fn get_reconnect_flag_clone(&self) -> CancellationToken { self.reconnect_flag.clone() } + + #[inline] + pub fn cancel_reconnect(&self) { self.reconnect_flag.cancel(); } + + #[inline] + pub fn get_url(&self) -> &Url { &self.url } + + #[inline] + pub fn get_url_as_str(&self) -> &str { self.url.as_str() } + + #[inline] + pub(super) fn get_item_type(&self) -> PlaylistItemType { self.item_type } + + #[inline] + pub fn should_retry_provider_request(&self) -> bool { + self.flags.contains(ProviderStreamFactoryFlags::RetryEnabled) + } + + pub(super) fn requires_public_destination(&self) -> bool { + self.flags.contains(ProviderStreamFactoryFlags::PublicDestinationRequired) + } + + #[inline] + pub fn should_retry_initial_open_loop(&self) -> bool { + self.flags.contains(ProviderStreamFactoryFlags::InitialRetryLoopEnabled) + } + + #[inline] + pub fn get_headers(&self) -> &HeaderMap { &self.headers } + + #[inline] + pub fn get_requested_range(&self) -> Option<&HeaderValue> { self.requested_range.as_ref() } + + #[inline] + pub fn should_continue(&self) -> bool { !self.reconnect_flag.is_cancelled() } + + #[inline] + pub fn was_range_requested(&self) -> bool { self.flags.contains(ProviderStreamFactoryFlags::RangeRequested) } + + pub(super) fn get_log_url(&self) -> std::borrow::Cow<'_, str> { + if is_sanitize_sensitive_info_enabled() { + return std::borrow::Cow::Borrowed(self.url.as_str()); + } + + std::borrow::Cow::Owned(preview_request_target_for_logging(&self.url, self.provider.as_ref())) + } + + pub(super) fn build_connect_failed_stream_info(&self, provider_name: Arc) -> Option { + let username = self.username.as_deref()?; + let client_ip = self.client_ip.as_deref()?; + let stream_channel = self.stream_channel.clone()?; + Some(StreamInfo::new(shared::model::StreamInfoParams { + uid: 0, + meter_uid: 0, + username, + addr: &self.addr, + client_ip, + provider: provider_name, + stream_channel, + user_agent: self.user_agent.clone().unwrap_or_default(), + country_code: None, + session_token: None, + })) + } + + pub(super) fn get_connect_failure_stage(&self) -> Option { self.connect_failure_stage } + + #[inline] + pub(crate) fn content_representation(&self) -> ProviderContentRepresentationMode { self.content_representation } + + /// Converts an open delayed until body polling to the only representation safe without an origin response head. + pub(crate) fn for_deferred_open(mut self) -> Self { + self.content_representation = ProviderContentRepresentationMode::Identity; + self.response_head_availability = ProviderResponseHeadAvailability::Unavailable; + self + } + + #[cfg(test)] + pub(crate) fn response_head_is_available(&self) -> bool { + matches!(self.response_head_availability, ProviderResponseHeadAvailability::Available) + } + + #[cfg(test)] + pub(crate) fn hls_content_coding_object_kind(&self) -> Option { + self.hls_content_coding_object_kind + } +} + +pub(super) fn should_wrap_provider_stream_in_buffer(stream_options: &ProviderStreamFactoryOptions) -> bool { + !stream_options.is_piped() + && !stream_options.flags.contains(ProviderStreamFactoryFlags::ShareStream) + && stream_options.is_buffer_enabled() +} + +// fn get_host_and_optional_port(url: &Url) -> Option { +// let host = url.host_str()?; +// match url.port() { +// Some(port) => Some(format!("{host}:{port}")), +// None => Some(host.to_string()), +// } +// } + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum ProviderRequestCredentialState { + OriginalOrigin, + Scrubbed, +} + +impl ProviderRequestCredentialState { + pub(super) fn observe_target(&mut self, original_url: &Url, target_url: &Url) { + if !same_origin(original_url, target_url) { + *self = Self::Scrubbed; + } + } +} diff --git a/backend/app/src/api/model/streams/provider_stream_factory/request.rs b/backend/app/src/api/model/streams/provider_stream_factory/request.rs new file mode 100644 index 000000000..fa58c5281 --- /dev/null +++ b/backend/app/src/api/model/streams/provider_stream_factory/request.rs @@ -0,0 +1,551 @@ +use super::{ + error::{ + classify_provider_io_error, classify_provider_status_error, failed_stream_account, + should_reject_success_response_content_type, ProviderStreamRequestFailure, + }, + options::ProviderRequestCredentialState, + response::prepare_provider_stream_response_for_request, + ProviderOpenGuard, ProviderStreamFactoryFlags, ProviderStreamFactoryOptions, ERR_MAX_RETRY_COUNT, RETRY_SECONDS, +}; +use crate::{ + api::model::{ProviderContentRepresentationMode, ProviderStreamFactoryResponse}, + iptv::stalker::client::validate_public_playable_url, + model::AppConfig, + utils::{ + content_coding::{apply_outbound_content_coding_policy, OutboundContentCodingPolicy}, + debug_if_enabled, + request::{ + is_safe_cross_origin_redirect_header, preview_request_diagnostics_for_logging, + send_with_retry_and_provider_policy_with_options, RequestFetchOptions, + }, + }, +}; +use log::{debug, log_enabled, warn}; +use reqwest::{ + header::{HeaderMap, HeaderValue, RANGE}, + StatusCode, +}; +use shared::{ + defaults::DEFAULT_USER_AGENT, + utils::{filter_request_header, sanitize_sensitive_info}, +}; +use std::{ + collections::HashMap, + io, + sync::Arc, + time::{Duration, Instant}, +}; +use tuliprox_session::stream_ctx::ProviderStreamCtx; +use url::Url; + +pub(super) fn merge_provider_request_headers( + input_headers: Option<&HashMap>, + session_headers: Option<&HashMap>, +) -> Option> { + match (input_headers, session_headers) { + (None, None) => None, + (Some(headers), None) | (None, Some(headers)) => Some(headers.clone()), + (Some(input), Some(session)) => { + let mut merged = input.clone(); + for (key, value) in session { + merged.insert(key.clone(), value.clone()); + } + Some(merged) + } + } +} + +pub(super) fn prepare_client( + request_client: &reqwest::Client, + stream_options: &ProviderStreamFactoryOptions, + url_override: Option<&Url>, + credential_state: ProviderRequestCredentialState, +) -> (reqwest::RequestBuilder, bool) { + let original_url = stream_options.get_url(); + let url = url_override.unwrap_or(original_url); + let requested_range = stream_options.get_requested_range(); + let original_headers = stream_options.get_headers(); + + if log_enabled!(log::Level::Debug) { + let message = format!("original headers {original_headers:?}"); + debug!("{}", sanitize_sensitive_info(&message)); + } + + let mut headers = HeaderMap::default(); + + for (key, value) in original_headers { + if filter_request_header(key.as_str()) { + headers.insert(key.clone(), value.clone()); + } + } + + if matches!(credential_state, ProviderRequestCredentialState::Scrubbed) { + remove_sensitive_headers(&mut headers); + } + prepare_default_headers(&mut headers, stream_options); + let partial = prepare_partial_request_headers(&mut headers, requested_range); + let content_coding_policy = match stream_options.content_representation() { + ProviderContentRepresentationMode::PreserveOrigin => OutboundContentCodingPolicy::Inherit, + ProviderContentRepresentationMode::Identity => OutboundContentCodingPolicy::Identity, + }; + apply_outbound_content_coding_policy(&mut headers, content_coding_policy); + + if log_enabled!(log::Level::Debug) { + let message = format!( + "Stream requested with headers: {:?}", + headers.iter().map(|header| (header.0, String::from_utf8_lossy(header.1.as_ref()))).collect::>() + ); + debug!("{}", sanitize_sensitive_info(&message)); + } + + let request_builder = request_client.get(url.clone()).headers(headers); + + (request_builder, partial) +} + +fn remove_sensitive_headers(headers: &mut axum::http::HeaderMap) { + let names_to_remove = + headers.keys().filter(|name| !is_safe_cross_origin_redirect_header(name.as_str())).cloned().collect::>(); + for name in names_to_remove { + headers.remove(name); + } +} + +pub(super) fn provider_headers_require_manual_redirects(headers: &HeaderMap) -> bool { + headers.keys().any(|name| !is_safe_cross_origin_redirect_header(name.as_str())) +} + +pub(super) fn same_origin(lhs: &Url, rhs: &Url) -> bool { + lhs.scheme().eq_ignore_ascii_case(rhs.scheme()) + && lhs.host_str() == rhs.host_str() + && lhs.port_or_known_default() == rhs.port_or_known_default() +} + +fn prepare_default_headers(headers: &mut axum::http::HeaderMap, stream_options: &ProviderStreamFactoryOptions) { + // Force Connection: close so the provider releases its slot immediately when the stream ends. + // This prevents 509 errors from providers counting idle pooled connections against limits. + headers.insert(axum::http::header::CONNECTION, axum::http::header::HeaderValue::from_static("close")); + + if !headers.contains_key(axum::http::header::USER_AGENT) { + headers.insert( + axum::http::header::USER_AGENT, + stream_options + .default_user_agent + .clone() + .unwrap_or_else(|| axum::http::header::HeaderValue::from_static(DEFAULT_USER_AGENT)), + ); + } +} + +fn prepare_partial_request_headers(headers: &mut HeaderMap, requested_range: Option<&HeaderValue>) -> bool { + if let Some(range) = requested_range { + headers.insert(RANGE, range.clone()); + true + } else { + false + } +} + +fn collect_debug_headers(headers: &HeaderMap) -> Vec<(String, String)> { + const HEADER_NAMES: [&str; 8] = + ["proxy-authenticate", "via", "server", "location", "x-cache", "x-cache-status", "x-served-by", "x-proxy-id"]; + + HEADER_NAMES + .iter() + .filter_map(|name| { + headers.get_all(*name).iter().next().map(|value| { + let value = value.to_str().unwrap_or("").to_string(); + ((*name).to_string(), value) + }) + }) + .collect() +} + +pub(super) async fn send_with_manual_redirects( + request_client: &reqwest::Client, + stream_options: &ProviderStreamFactoryOptions, + app_config: &Arc, +) -> Result { + let mut current_url = stream_options.get_url().clone(); + let mut remaining_redirects = 10u8; + let provider = stream_options.get_provider().cloned(); + let mut credential_state = ProviderRequestCredentialState::OriginalOrigin; + + loop { + if stream_options.requires_public_destination() { + validate_public_playable_url(¤t_url) + .await + .map_err(|err| io::Error::new(io::ErrorKind::PermissionDenied, err))?; + } + let result = send_with_retry_and_provider_policy_with_options( + app_config, + ¤t_url, + provider.as_ref(), + true, + stream_options.should_retry_provider_request(), + RequestFetchOptions::default() + .with_http_error_responses(stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource)), + |resolved_url| prepare_client(request_client, stream_options, Some(resolved_url), credential_state).0, + ) + .await; + + let response = match result { + Ok(resp) => resp, + Err(e) => { + // send_with_retry_and_provider already applies provider failover policy. + // Do not rotate again here, otherwise non-failover errors (e.g. auth) may + // incorrectly switch provider URLs. + debug!("Manual redirect failed: {}", sanitize_sensitive_info(&e.to_string())); + return Err(e); + } + }; + + let status = response.status(); + + if status.is_redirection() { + if remaining_redirects == 0 { + return Ok(response); + } + let location = response.headers().get(reqwest::header::LOCATION); + let Some(location) = location else { + return Ok(response); + }; + let Ok(location_str) = location.to_str() else { + return Ok(response); + }; + let response_url = response.url().clone(); + let next_url = response_url.join(location_str).or_else(|_| Url::parse(location_str)); + let Ok(next_url) = next_url else { + return Ok(response); + }; + credential_state.observe_target(&response_url, &next_url); + current_url = next_url; + remaining_redirects = remaining_redirects.saturating_sub(1); + continue; + } + return Ok(response); + } +} + +#[allow(clippy::too_many_lines)] +async fn provider_stream_request( + ctx: &ProviderStreamCtx, + request_client: &reqwest::Client, + stream_options: &ProviderStreamFactoryOptions, +) -> Result, ProviderStreamRequestFailure> { + let use_manual_redirects = stream_options.requires_public_destination() + || tuliprox_core::model::should_use_manual_redirects(&ctx.app_config) + || provider_headers_require_manual_redirects(stream_options.get_headers()); + if log_enabled!(log::Level::Debug) { + let diagnostics = + preview_request_diagnostics_for_logging(stream_options.get_url(), stream_options.get_provider()); + debug!( + "Provider request diagnostics: manual_redirects={}, {}", + use_manual_redirects, + sanitize_sensitive_info(&diagnostics) + ); + } + let response_result = if use_manual_redirects { + let client_no_redirect = if stream_options.requires_public_destination() { + ctx.http_clients.public_no_redirect.load() + } else { + ctx.http_clients.no_redirect.load() + }; + send_with_manual_redirects(&client_no_redirect, stream_options, &ctx.app_config).await + } else { + // Use send_with_retry_and_provider for automatic failover support + let url = stream_options.get_url(); + let provider = stream_options.get_provider().cloned(); + + send_with_retry_and_provider_policy_with_options( + &ctx.app_config, + url, + provider.as_ref(), + false, + stream_options.should_retry_provider_request(), + RequestFetchOptions::default() + .with_http_error_responses(stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource)), + |resolved_url| { + let (client, _partial_content) = prepare_client( + request_client, + stream_options, + Some(resolved_url), + ProviderRequestCredentialState::OriginalOrigin, + ); + client + }, + ) + .await + }; + match response_result { + Ok(response) => { + let status = response.status(); + let response_url = response.url().clone(); + if log_enabled!(log::Level::Debug) && !status.is_success() { + let debug_headers = collect_debug_headers(response.headers()); + let diagnostics = + preview_request_diagnostics_for_logging(stream_options.get_url(), stream_options.get_provider()); + let message = + format!( + "Provider response error: status={status}, url={response_url}, headers={debug_headers:?}, {diagnostics}" + ); + debug!("{}", sanitize_sensitive_info(&message)); + } + if status.is_success() { + if should_reject_success_response_content_type(stream_options.get_item_type(), response.headers()) { + debug!( + "Provider returned HTML content for non-adaptive stream {}", + sanitize_sensitive_info(stream_options.get_log_url().as_ref()) + ); + return Err(ProviderStreamRequestFailure::Status { + status: StatusCode::BAD_GATEWAY, + provider_error_class: "unexpected_content_type", + serve_channel_unavailable: true, + }); + } + let response = prepare_provider_stream_response_for_request(response, stream_options).await; + let response = match response { + Ok(response) => response, + Err(error) => { + let provider_error_class = error.provider_error_class(); + return Err(ProviderStreamRequestFailure::Status { + status: StatusCode::BAD_GATEWAY, + provider_error_class, + serve_channel_unavailable: true, + }); + } + }; + if log_enabled!(log::Level::Debug) { + // Unfortunately, the HEAD request does not work, so we need this workaround. + // We need some header information from the provider, we extract the necessary headers and forward them to the client + let message = format!("Provider response info: {:?}", response.info); + debug!("{}", sanitize_sensitive_info(&message)); + } + return Ok(Some(response)); + } + + if stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource) { + // Keep range and retry metadata while discarding the provider's error body. + let headers = response + .headers() + .iter() + .filter(|(name, _)| { + matches!( + **name, + reqwest::header::CONTENT_RANGE + | reqwest::header::ACCEPT_RANGES + | reqwest::header::RETRY_AFTER + ) + }) + .filter_map(|(name, value)| value.to_str().ok().map(|value| (name.to_string(), value.to_string()))) + .collect(); + // Only upstream errors pass through; an unfollowed redirect or other non-success + // status would reach the client without its Location or body. + let status = + if status.is_client_error() || status.is_server_error() { status } else { StatusCode::BAD_GATEWAY }; + return Err(ProviderStreamRequestFailure::HlsStatus { status, headers }); + } + + if status.is_client_error() { + debug!("Client error status response : {status}"); + return match status { + StatusCode::NOT_FOUND + | StatusCode::FORBIDDEN + | StatusCode::UNAUTHORIZED + | StatusCode::PROXY_AUTHENTICATION_REQUIRED + | StatusCode::METHOD_NOT_ALLOWED + | StatusCode::BAD_REQUEST => Err(ProviderStreamRequestFailure::Status { + status, + provider_error_class: classify_provider_status_error(status), + serve_channel_unavailable: true, + }), + _ => Err(ProviderStreamRequestFailure::Status { + status, + provider_error_class: classify_provider_status_error(status), + serve_channel_unavailable: false, + }), + }; + } + if status.is_server_error() { + debug!("Server error status response : {status}"); + return match status { + StatusCode::INTERNAL_SERVER_ERROR + | StatusCode::BAD_GATEWAY + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::GATEWAY_TIMEOUT => Err(ProviderStreamRequestFailure::Status { + status, + provider_error_class: classify_provider_status_error(status), + serve_channel_unavailable: true, + }), + _ => Err(ProviderStreamRequestFailure::Status { + status, + provider_error_class: classify_provider_status_error(status), + serve_channel_unavailable: false, + }), + }; + } + Err(ProviderStreamRequestFailure::Status { + status, + provider_error_class: classify_provider_status_error(status), + serve_channel_unavailable: false, + }) + } + Err(err) => { + let diagnostics = + preview_request_diagnostics_for_logging(stream_options.get_url(), stream_options.get_provider()); + debug!( + "Provider request failed: {}, {}", + sanitize_sensitive_info(&err.to_string()), + sanitize_sensitive_info(&diagnostics) + ); + Err(ProviderStreamRequestFailure::Status { + status: StatusCode::SERVICE_UNAVAILABLE, + provider_error_class: classify_provider_io_error(&err), + serve_channel_unavailable: true, + }) + } + } +} + +/// Flags the failed account for the Xtream expiry worker. No request is sent here: the worker +/// checks the account only when its per-account, per-panel and cooldown throttles allow it, +/// because frequent account queries can get the account banned. +fn request_provider_account_probe(ctx: &ProviderStreamCtx, options: &ProviderStreamFactoryOptions) { + let Some(channel) = options.stream_channel.as_ref() else { + return; + }; + let sources = ctx.app_config.sources.load_full(); + let Some(input) = sources.get_input_by_name(&channel.input_name) else { + return; + }; + if !input.input_type.is_xtream() { + return; + } + if let Some(account) = failed_stream_account(input, options) { + ctx.active_provider.request_account_probe(&account.name); + } +} + +#[allow(clippy::too_many_lines)] +pub(super) async fn get_provider_stream( + ctx: &ProviderStreamCtx, + client: &reqwest::Client, + stream_options: &ProviderStreamFactoryOptions, +) -> Result, ProviderStreamRequestFailure> { + let log_url = stream_options.get_log_url(); + debug_if_enabled!("stream provider {}", sanitize_sensitive_info(log_url.as_ref())); + let start = Instant::now(); + let mut connect_err: u32 = 1; + + let handle_cancel = stream_options.get_cancel_token(); + while stream_options.should_continue() { + let request_future = provider_stream_request(ctx, client, stream_options); + let request_res = if let Some(ref cancel) = handle_cancel { + tokio::select! { + biased; + () = cancel.cancelled() => { + debug!( + "Provider stream request cancelled during open: {}", + sanitize_sensitive_info(stream_options.get_log_url().as_ref()) + ); + return Err(ProviderStreamRequestFailure::Status { + status: StatusCode::BAD_GATEWAY, + provider_error_class: "cancelled", + serve_channel_unavailable: false, + }); + } + res = request_future => res, + } + } else { + request_future.await + }; + match request_res { + Ok(Some(stream_response)) => { + return Ok(Some(stream_response)); + } + Ok(None) => { + if connect_err > ERR_MAX_RETRY_COUNT { + warn!( + "The stream could be unavailable. {}", + sanitize_sensitive_info(stream_options.get_log_url().as_ref()) + ); + break; + } + } + Err(failure) => { + if matches!(failure.status(), StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) { + request_provider_account_probe(ctx, stream_options); + } + if stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource) + || failure.should_serve_channel_unavailable() + { + return Err(failure); + } + let status = failure.status(); + debug!("Provider stream response error status response : {status}"); + if matches!( + status, + StatusCode::FORBIDDEN + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::UNAUTHORIZED + | StatusCode::PROXY_AUTHENTICATION_REQUIRED + | StatusCode::RANGE_NOT_SATISFIABLE + ) { + warn!( + "The stream could be unavailable. ({status}) {}", + sanitize_sensitive_info(stream_options.get_log_url().as_ref()) + ); + break; + } + if connect_err > ERR_MAX_RETRY_COUNT { + warn!( + "The stream could be unavailable. ({status}) {}", + sanitize_sensitive_info(stream_options.get_log_url().as_ref()) + ); + break; + } + } + } + if !stream_options.should_continue() || connect_err > ERR_MAX_RETRY_COUNT { + break; + } + if !stream_options.should_retry_initial_open_loop() { + break; + } + if start.elapsed().as_secs() > RETRY_SECONDS { + warn!( + "The stream could be unavailable. Giving up after {RETRY_SECONDS} seconds. {}", + sanitize_sensitive_info(stream_options.get_log_url().as_ref()) + ); + break; + } + connect_err += 1; + if let Some(ref cancel) = handle_cancel { + tokio::select! { + biased; + () = cancel.cancelled() => break, + () = tokio::time::sleep(Duration::from_millis(50)) => {}, + } + } else { + tokio::time::sleep(Duration::from_millis(50)).await; + } + debug_if_enabled!("Reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref())); + } + debug_if_enabled!("Stopped reconnecting stream {}", sanitize_sensitive_info(stream_options.get_log_url().as_ref())); + stream_options.cancel_reconnect(); + Err(ProviderStreamRequestFailure::Status { + status: StatusCode::SERVICE_UNAVAILABLE, + provider_error_class: "service_unavailable", + serve_channel_unavailable: true, + }) +} + +impl Drop for ProviderOpenGuard { + fn drop(&mut self) { + if !self.handed_off { + if let Some(token) = self.token.take() { + token.cancel(); + } + } + } +} diff --git a/backend/app/src/api/model/streams/provider_stream_factory/response.rs b/backend/app/src/api/model/streams/provider_stream_factory/response.rs new file mode 100644 index 000000000..e922bc17c --- /dev/null +++ b/backend/app/src/api/model/streams/provider_stream_factory/response.rs @@ -0,0 +1,192 @@ +use super::{ + error::{provider_decoded_body_error, ProviderStreamPreparationError}, + ProviderResponseHeadAvailability, ProviderStreamFactoryFlags, ProviderStreamFactoryOptions, + ProviderStreamPreparationContext, +}; +use crate::{ + api::model::{ProviderContentRepresentationMode, ProviderStreamFactoryResponse, StreamError, STREAM_IDLE_TIMEOUT}, + utils::content_coding::{decode_response_to_identity, ContentCodingDetection, ContentCodingError}, +}; +use futures::{StreamExt, TryStreamExt}; +use reqwest::{ + header::{HeaderMap, HeaderValue, CONTENT_RANGE}, + StatusCode, +}; +use std::{io, time::Duration}; +use tokio_util::io::ReaderStream; +#[cfg(test)] +use tuliprox_hls::api::HlsOriginContentCodingObjectKind; +use tuliprox_hls::api::{ + extract_hls_provider_session_headers, log_hls_origin_content_coding, HlsOriginContentCodingSource, +}; +use tuliprox_session::response_headers::provider_response_headers; +use url::Url; + +#[cfg(test)] +pub(super) async fn prepare_provider_stream_response( + response: reqwest::Response, + mode: ProviderContentRepresentationMode, + response_head_availability: ProviderResponseHeadAvailability, +) -> Result { + prepare_provider_stream_response_with_context( + response, + ProviderStreamPreparationContext { + representation: mode, + response_head_availability, + range_requested: false, + hls_content_coding_object_kind: matches!(mode, ProviderContentRepresentationMode::Identity) + .then_some(HlsOriginContentCodingObjectKind::Other), + }, + Duration::from_secs(STREAM_IDLE_TIMEOUT), + ) + .await +} + +#[cfg(test)] +pub(super) async fn prepare_provider_stream_response_with_idle_timeout( + response: reqwest::Response, + mode: ProviderContentRepresentationMode, + response_head_availability: ProviderResponseHeadAvailability, + idle_timeout: Duration, +) -> Result { + prepare_provider_stream_response_with_context( + response, + ProviderStreamPreparationContext { + representation: mode, + response_head_availability, + range_requested: false, + hls_content_coding_object_kind: matches!(mode, ProviderContentRepresentationMode::Identity) + .then_some(HlsOriginContentCodingObjectKind::Other), + }, + idle_timeout, + ) + .await +} + +/// Preserve specific upstream MIME types; repair missing or generic types for MP4 objects. +/// Audio and video share most fMP4 extensions, so audio is only recognized by an `m4a`/`cmfa` +/// extension or, when enabled for the input, by Flussonic's `tracks-a` rendition paths. +pub(super) fn normalize_hls_resource_content_type(headers: &mut HeaderMap, url: &Url, flussonic_audio_tracks: bool) { + let extension = url.path().rsplit_once('.').map_or("", |(_, ext)| ext); + let mime = crate::utils::request::content_type_from_ext(extension); + if !matches!(mime, "video/mp4" | "audio/mp4") { + return; + } + let needs_type = + headers.get(reqwest::header::CONTENT_TYPE).and_then(|value| value.to_str().ok()).is_none_or(|value| { + let base = value.split(';').next().unwrap_or_default().trim(); + base.is_empty() + || base.eq_ignore_ascii_case("application/octet-stream") + || base.eq_ignore_ascii_case("video/mp2t") + }); + if needs_type { + // Flussonic names separate audio-only renditions tracks-a. + let audio_track = flussonic_audio_tracks + && url.path_segments().is_some_and(|mut segments| { + segments.any(|segment| { + segment + .strip_prefix("tracks-a") + .is_some_and(|index| !index.is_empty() && index.bytes().all(|byte| byte.is_ascii_digit())) + }) + }); + headers.insert( + reqwest::header::CONTENT_TYPE, + HeaderValue::from_static(if audio_track { "audio/mp4" } else { mime }), + ); + } +} + +pub(super) async fn prepare_provider_stream_response_for_request( + mut response: reqwest::Response, + stream_options: &ProviderStreamFactoryOptions, +) -> Result { + if stream_options.flags.contains(ProviderStreamFactoryFlags::HlsResource) { + normalize_hls_resource_content_type( + response.headers_mut(), + stream_options.get_url(), + stream_options.flags.contains(ProviderStreamFactoryFlags::FlussonicAudioTracks), + ); + } + prepare_provider_stream_response_with_context( + response, + ProviderStreamPreparationContext { + representation: stream_options.content_representation(), + response_head_availability: stream_options.response_head_availability, + range_requested: stream_options.was_range_requested(), + hls_content_coding_object_kind: stream_options.hls_content_coding_object_kind, + }, + Duration::from_secs(STREAM_IDLE_TIMEOUT), + ) + .await +} + +async fn prepare_provider_stream_response_with_context( + response: reqwest::Response, + context: ProviderStreamPreparationContext, + idle_timeout: Duration, +) -> Result { + let provider_session_headers = extract_hls_provider_session_headers(response.headers()); + match context.representation { + ProviderContentRepresentationMode::PreserveOrigin => { + if matches!(context.response_head_availability, ProviderResponseHeadAvailability::Unavailable) { + return Err(ProviderStreamPreparationError::DeferredResponseHead { + status: response.status(), + has_content_range: response.headers().contains_key(CONTENT_RANGE), + }); + } + let headers = provider_response_headers(response.headers(), context.representation)?; + let response_info = Some((headers, response.status(), Some(response.url().clone()), None)); + let stream = response.bytes_stream().map_err(|error| StreamError::reqwest(&error)).boxed(); + Ok(ProviderStreamFactoryResponse { + stream, + info: response_info, + provider_session_headers, + has_upstream_owner: false, + }) + } + ProviderContentRepresentationMode::Identity => { + let origin_status = response.status(); + let origin_has_content_range = response.headers().contains_key(CONTENT_RANGE); + // `deflate` decoder selection reads a prefix before the returned stream can enter the + // normal provider body wrappers, so decoder setup must own the same idle bound too. + let decoded = tokio::time::timeout( + idle_timeout, + decode_response_to_identity(response, ContentCodingDetection::DeclaredOnly), + ) + .await + .map_err(|_| { + ContentCodingError::PrefixRead(io::Error::new( + io::ErrorKind::TimedOut, + "provider body idle timeout during content-decoder setup", + )) + })??; + if matches!(context.response_head_availability, ProviderResponseHeadAvailability::Unavailable) + && (origin_status != StatusCode::OK || origin_has_content_range) + { + return Err(ProviderStreamPreparationError::DeferredResponseHead { + status: origin_status, + has_content_range: origin_has_content_range, + }); + } + if let (Some(observation), Some(object_kind)) = + (decoded.content_coding_observation(), context.hls_content_coding_object_kind) + { + log_hls_origin_content_coding( + observation, + object_kind, + context.range_requested, + HlsOriginContentCodingSource::Legacy, + ); + } + let headers = provider_response_headers(&decoded.headers, context.representation)?; + let response_info = Some((headers, decoded.status, Some(decoded.final_url), None)); + let stream = ReaderStream::new(decoded.body).map_err(|error| provider_decoded_body_error(&error)).boxed(); + Ok(ProviderStreamFactoryResponse { + stream, + info: response_info, + provider_session_headers, + has_upstream_owner: false, + }) + } + } +} diff --git a/backend/app/src/api/model/streams/provider_stream_factory/tests.rs b/backend/app/src/api/model/streams/provider_stream_factory/tests.rs new file mode 100644 index 000000000..0d4b48649 --- /dev/null +++ b/backend/app/src/api/model/streams/provider_stream_factory/tests.rs @@ -0,0 +1,335 @@ +use super::{ + error::{failed_stream_account, should_reject_success_response_content_type, ProviderStreamPreparationError}, + options::{should_wrap_provider_stream_in_buffer, ProviderRequestCredentialState}, + *, +}; +use crate::{ + api::model::{BoxedProviderStream, ProviderStreamFactoryResponse, StreamError}, + model::{AppConfig, Config, MediaToolCapabilities, SourcesConfig}, + utils::{content_coding::ContentCodingError, FileLockManager}, +}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use axum::http::HeaderMap; +use bytes::Bytes; +use futures::TryStreamExt; +use reqwest::StatusCode; +use shared::{ + model::{ConfigPaths, PlaylistItemType, StreamChannel, XtreamCluster}, + utils::Internable, +}; +use std::{ + collections::HashMap, + io, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, +}; +use tuliprox_session::{ + response_headers::ProviderResponseHeaderError, + stream_options::{StreamOptions, StreamResponseMode}, +}; + +struct RedirectRoundTrip { + entry_url: Url, + first_origin_task: tokio::task::JoinHandle>, + second_origin_task: tokio::task::JoinHandle, +} + +fn test_app_config() -> Arc { + Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + }) +} + +async fn read_http_request(socket: &mut TcpStream) -> String { + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let read = socket.read(&mut chunk).await.expect("test origin reads request"); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + String::from_utf8_lossy(&request).into_owned() +} + +async fn spawn_redirect_round_trip(encoded_body: Vec) -> RedirectRoundTrip { + let first_listener = TcpListener::bind("127.0.0.1:0").await.expect("first test origin binds"); + let first_addr = first_listener.local_addr().expect("first test origin address"); + let second_listener = TcpListener::bind("127.0.0.1:0").await.expect("second test origin binds"); + let second_addr = second_listener.local_addr().expect("second test origin address"); + + let first_origin_task = tokio::spawn(async move { + let (mut first_socket, _) = first_listener.accept().await.expect("first origin accepts entry request"); + let entry_request = read_http_request(&mut first_socket).await; + let redirect = format!( + "HTTP/1.1 302 Found\r\nLocation: http://localhost:{}/hop\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + second_addr.port() + ); + first_socket.write_all(redirect.as_bytes()).await.expect("first origin redirects to second origin"); + + let (mut final_socket, _) = first_listener.accept().await.expect("first origin accepts final request"); + let final_request = read_http_request(&mut final_socket).await; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: video/mp2t\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + encoded_body.len() + ); + final_socket.write_all(response.as_bytes()).await.expect("first origin writes final response head"); + final_socket.write_all(&encoded_body).await.expect("first origin writes final response body"); + vec![entry_request, final_request] + }); + + let second_origin_task = tokio::spawn(async move { + let (mut socket, _) = second_listener.accept().await.expect("second origin accepts redirect request"); + let request = read_http_request(&mut socket).await; + let redirect = format!( + "HTTP/1.1 302 Found\r\nLocation: http://{first_addr}/final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + socket.write_all(redirect.as_bytes()).await.expect("second origin redirects to first origin"); + request + }); + + RedirectRoundTrip { + entry_url: Url::parse(&format!("http://{first_addr}/entry")).expect("entry URL"), + first_origin_task, + second_origin_task, + } +} + +fn redirect_test_options(stream_url: &Url) -> ProviderStreamFactoryOptions { + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::ACCEPT_ENCODING, "br".parse().expect("Accept-Encoding")); + req_headers.insert(reqwest::header::AUTHORIZATION, "Bearer client-secret".parse().expect("Authorization")); + req_headers.insert(reqwest::header::COOKIE, "session=client-secret".parse().expect("Cookie")); + test_options(ProviderContentRepresentationMode::Identity, stream_url, &req_headers, None, None) +} + +fn assert_redirect_request_headers(request: &str, credentials_expected: bool) { + let request = request.to_ascii_lowercase(); + assert!(request.contains("\r\naccept-encoding: identity\r\n")); + assert_eq!(request.contains("\r\nauthorization: bearer client-secret\r\n"), credentials_expected); + assert_eq!(request.contains("\r\ncookie: session=client-secret\r\n"), credentials_expected); +} + +async fn assert_identity_redirect_result( + response: reqwest::Response, + first_origin_task: tokio::task::JoinHandle>, + second_origin_task: tokio::task::JoinHandle, +) { + const IDENTITY_BODY: &[u8] = b"decoded redirect body"; + + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await + .expect("redirect response is prepared"); + assert_eq!(collect_stream(stream).await.expect("redirect body decodes"), IDENTITY_BODY); + let (headers, status, _, _) = info.expect("redirect response metadata"); + assert_eq!(status, StatusCode::OK); + assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-encoding"))); + + let first_requests = tokio::time::timeout(Duration::from_secs(2), first_origin_task) + .await + .expect("first origin finishes") + .expect("first origin task succeeds"); + let second_request = tokio::time::timeout(Duration::from_secs(2), second_origin_task) + .await + .expect("second origin finishes") + .expect("second origin task succeeds"); + assert_eq!(first_requests.len(), 2); + assert_redirect_request_headers(&first_requests[0], true); + assert_redirect_request_headers(&second_request, false); + assert_redirect_request_headers(&first_requests[1], false); +} + +fn test_options( + mode: ProviderContentRepresentationMode, + stream_url: &Url, + req_headers: &HeaderMap, + input_headers: Option<&HashMap>, + session_headers: Option<&HashMap>, +) -> ProviderStreamFactoryOptions { + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: false, + buffer_size: 0, + buffer_max_bytes: 0, + pipe_provider_stream: false, + response_mode: StreamResponseMode::Stream, + }; + ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { + addr: "127.0.0.1:8080".parse().unwrap(), + item_type: PlaylistItemType::Catchup, + share_stream: false, + stream_options: &stream_options, + stream_url, + req_headers, + input_headers, + session_headers, + disabled_headers: None, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: mode, + }) +} + +async fn local_response( + status: StatusCode, + headers: &[(&str, &str)], + body: Vec, +) -> (reqwest::Response, Arc) { + use std::fmt::Write as _; + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let requests = Arc::new(AtomicUsize::new(0)); + let task_requests = Arc::clone(&requests); + let response_headers = headers.iter().fold(String::new(), |mut headers, (name, value)| { + let _ = write!(headers, "{name}: {value}\r\n"); + headers + }); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + task_requests.fetch_add(1, Ordering::SeqCst); + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let read = socket.read(&mut chunk).await.unwrap(); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + let reason = status.canonical_reason().unwrap_or("Status"); + let head = format!( + "HTTP/1.1 {} {reason}\r\nContent-Length: {}\r\n{response_headers}Connection: close\r\n\r\n", + status.as_u16(), + body.len() + ); + socket.write_all(head.as_bytes()).await.unwrap(); + socket.write_all(&body).await.unwrap(); + }); + + let response = reqwest::Client::new().get(format!("http://{addr}/resource")).send().await.unwrap(); + (response, requests) +} + +async fn local_stalled_deflate_response() -> (reqwest::Response, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let read = socket.read(&mut chunk).await.unwrap(); + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + socket + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Encoding: deflate\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n", + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_secs(5)).await; + }); + let response = reqwest::Client::new().get(format!("http://{addr}/resource")).send().await.unwrap(); + (response, task) +} + +#[derive(Clone, Copy)] +enum TestEncoding { + Gzip, + Zlib, + RawDeflate, + Brotli, + Zstd, +} + +async fn encode(body: &[u8], encoding: TestEncoding) -> Vec { + match encoding { + TestEncoding::Gzip => { + let mut encoder = async_compression::tokio::write::GzipEncoder::new(Vec::new()); + encoder.write_all(body).await.unwrap(); + encoder.shutdown().await.unwrap(); + encoder.into_inner() + } + TestEncoding::Zlib => { + let mut encoder = async_compression::tokio::write::ZlibEncoder::new(Vec::new()); + encoder.write_all(body).await.unwrap(); + encoder.shutdown().await.unwrap(); + encoder.into_inner() + } + TestEncoding::RawDeflate => { + let mut encoder = async_compression::tokio::write::DeflateEncoder::new(Vec::new()); + encoder.write_all(body).await.unwrap(); + encoder.shutdown().await.unwrap(); + encoder.into_inner() + } + TestEncoding::Brotli => { + let mut encoder = async_compression::tokio::write::BrotliEncoder::new(Vec::new()); + encoder.write_all(body).await.unwrap(); + encoder.shutdown().await.unwrap(); + encoder.into_inner() + } + TestEncoding::Zstd => { + let mut encoder = async_compression::tokio::write::ZstdEncoder::new(Vec::new()); + encoder.write_all(body).await.unwrap(); + encoder.shutdown().await.unwrap(); + encoder.into_inner() + } + } +} + +async fn collect_stream(stream: BoxedProviderStream) -> Result, StreamError> { + stream + .try_fold(Vec::new(), |mut bytes, chunk| async move { + bytes.extend_from_slice(&chunk); + Ok(bytes) + }) + .await +} + +mod http; +mod lifecycle; +mod policy; +mod recovery; +mod storage; +mod streaming; +mod transport; diff --git a/backend/app/src/api/model/streams/provider_stream_factory/tests/http.rs b/backend/app/src/api/model/streams/provider_stream_factory/tests/http.rs new file mode 100644 index 000000000..9d6a5e9e2 --- /dev/null +++ b/backend/app/src/api/model/streams/provider_stream_factory/tests/http.rs @@ -0,0 +1,244 @@ +use super::*; + +#[test] +fn provider_requests_preserve_complete_client_range_header() { + let stream_url = Url::parse("http://provider.example/movie").unwrap(); + let client = reqwest::Client::new(); + + for requested_range in ["bytes=100-199", "bytes=100-", "bytes=-100", "bytes=0-0", "bytes=0-0,200-299"] { + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::RANGE, requested_range.parse().unwrap()); + let options = + test_options(ProviderContentRepresentationMode::PreserveOrigin, &stream_url, &req_headers, None, None); + + let (request, partial) = + prepare_client(&client, &options, None, ProviderRequestCredentialState::OriginalOrigin); + let request = request.build().unwrap(); + + assert!(partial, "Range request was not recognized: {requested_range}"); + assert_eq!( + request.headers().get(reqwest::header::RANGE).and_then(|value| value.to_str().ok()), + Some(requested_range) + ); + } +} + +#[test] +fn client_range_takes_precedence_over_configured_provider_range() { + let stream_url = Url::parse("http://provider.example/movie").unwrap(); + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::RANGE, "bytes=100-199".parse().unwrap()); + let input_headers = HashMap::from([("Range".to_string(), "bytes=0-".to_string())]); + let options = test_options( + ProviderContentRepresentationMode::PreserveOrigin, + &stream_url, + &req_headers, + Some(&input_headers), + None, + ); + + let request = + prepare_client(&reqwest::Client::new(), &options, None, ProviderRequestCredentialState::OriginalOrigin) + .0 + .build() + .unwrap(); + + assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=100-199"); +} + +#[test] +fn test_provider_stream_factory_options_range_logic() { + let addr = "127.0.0.1:8080".parse().unwrap(); + let stream_url = Url::parse("http://example.com/stream").unwrap(); + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: true, + buffer_size: 1024, + buffer_max_bytes: 0, + pipe_provider_stream: false, + response_mode: StreamResponseMode::Stream, + }; + let disabled_headers = None; + + // Case 1: VOD, no initial range requested + let mut req_headers = HeaderMap::new(); + let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { + addr, + item_type: PlaylistItemType::Video, + share_stream: false, + stream_options: &stream_options, + stream_url: &stream_url, + req_headers: &req_headers, + input_headers: None, + session_headers: None, + disabled_headers, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + }); + assert!(!options.was_range_requested()); + assert!(options.get_requested_range().is_none()); + + // Case 2: VOD, range requested + req_headers.insert("Range", "bytes=100-".parse().unwrap()); + let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { + addr, + item_type: PlaylistItemType::Video, + share_stream: false, + stream_options: &stream_options, + stream_url: &stream_url, + req_headers: &req_headers, + input_headers: None, + session_headers: None, + disabled_headers, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + }); + assert!(options.was_range_requested()); + assert_eq!(options.get_requested_range().and_then(|value| value.to_str().ok()), Some("bytes=100-")); + + // Case 3: Live, no initial range requested + let req_headers = HeaderMap::new(); + let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { + addr, + item_type: PlaylistItemType::Live, + share_stream: false, + stream_options: &stream_options, + stream_url: &stream_url, + req_headers: &req_headers, + input_headers: None, + session_headers: None, + disabled_headers, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + }); + assert!(!options.was_range_requested()); + assert!(options.get_requested_range().is_none()); + + // Case 4: Live, range requested (should be stripped) + let mut req_headers = HeaderMap::new(); + req_headers.insert("Range", "bytes=100-".parse().unwrap()); + let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { + addr, + item_type: PlaylistItemType::Live, + share_stream: false, + stream_options: &stream_options, + stream_url: &stream_url, + req_headers: &req_headers, + input_headers: None, + session_headers: None, + disabled_headers, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + }); + assert!(!options.was_range_requested()); // Stripped by filter + assert!(options.get_requested_range().is_none()); +} + +#[tokio::test] +async fn legacy_hls_decoder_setup_obeys_provider_body_idle_timeout() { + let (response, origin_task) = local_stalled_deflate_response().await; + + let result = prepare_provider_stream_response_with_idle_timeout( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + Duration::from_millis(25), + ) + .await; + origin_task.abort(); + + assert!(matches!( + result, + Err(ProviderStreamPreparationError::ContentCoding(ContentCodingError::PrefixRead(error))) + if error.kind() == io::ErrorKind::TimedOut + )); +} + +#[tokio::test] +async fn preserve_origin_keeps_encoded_body_and_representation_headers() { + let encoded = encode(b"preserved representation", TestEncoding::Gzip).await; + let expected = encoded.clone(); + let expected_len = encoded.len().to_string(); + let (response, _) = local_response( + StatusCode::PARTIAL_CONTENT, + &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 10-20/100")], + encoded, + ) + .await; + + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::PreserveOrigin, + ProviderResponseHeadAvailability::Available, + ) + .await + .unwrap(); + let (headers, status, _, _) = info.unwrap(); + + assert_eq!(collect_stream(stream).await.unwrap(), expected); + assert_eq!(status, StatusCode::PARTIAL_CONTENT); + assert!(headers.iter().any(|(name, value)| name == "content-encoding" && value == "gzip")); + assert!(headers.iter().any(|(name, value)| name == "content-length" && value == &expected_len)); + assert!(headers.iter().any(|(name, value)| name == "content-range" && value == "bytes 10-20/100")); + assert!(headers.iter().all(|(name, _)| name != "transfer-encoding")); +} + +#[tokio::test] +async fn preserve_origin_keeps_unknown_content_coding_and_body_unchanged() { + let body = b"opaque provider representation".to_vec(); + let expected = body.clone(); + let expected_len = body.len().to_string(); + let (response, _) = local_response( + StatusCode::PARTIAL_CONTENT, + &[("Content-Encoding", "x-provider-coding"), ("Content-Range", "bytes 0-29/100")], + body, + ) + .await; + + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::PreserveOrigin, + ProviderResponseHeadAvailability::Available, + ) + .await + .expect("valid unknown coding is preserved"); + let (headers, status, _, _) = info.expect("preserved response metadata"); + + assert_eq!(collect_stream(stream).await.expect("opaque body streams"), expected); + assert_eq!(status, StatusCode::PARTIAL_CONTENT); + assert!(headers.iter().any(|(name, value)| name == "content-encoding" && value == "x-provider-coding")); + assert!(headers.iter().any(|(name, value)| name == "content-length" && value == &expected_len)); + assert!(headers.iter().any(|(name, value)| name == "content-range" && value == "bytes 0-29/100")); +} + +#[tokio::test] +async fn preserve_origin_never_magic_sniffs_headerless_body() { + let body = Bytes::from_static(b"\x1f\x8bopaque non-hls bytes"); + let (response, _) = local_response(StatusCode::OK, &[], body.to_vec()).await; + + let ProviderStreamFactoryResponse { stream, .. } = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::PreserveOrigin, + ProviderResponseHeadAvailability::Available, + ) + .await + .unwrap(); + + assert_eq!(collect_stream(stream).await.unwrap(), body); +} diff --git a/backend/app/src/api/model/streams/provider_stream_factory/tests/lifecycle.rs b/backend/app/src/api/model/streams/provider_stream_factory/tests/lifecycle.rs new file mode 100644 index 000000000..31fdab413 --- /dev/null +++ b/backend/app/src/api/model/streams/provider_stream_factory/tests/lifecycle.rs @@ -0,0 +1,43 @@ +use super::*; + +#[tokio::test] +async fn preserve_origin_rejects_invalid_representation_header_before_stream_release() { + let (response, requests) = + local_response(StatusCode::OK, &[("Content-Encoding", "gzip,,br")], b"body must not be released".to_vec()) + .await; + + assert!(matches!( + prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::PreserveOrigin, + ProviderResponseHeadAvailability::Available, + ) + .await, + Err(ProviderStreamPreparationError::ResponseHeader(ProviderResponseHeaderError::InvalidContentEncoding)) + )); + assert_eq!(requests.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn legacy_hls_identity_decoding_removes_stale_representation_headers() { + const IDENTITY_BODY: &[u8] = b"decoded identity response"; + let encoded = encode(IDENTITY_BODY, TestEncoding::Gzip).await; + let (response, _) = + local_response(StatusCode::OK, &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 0-9/10")], encoded) + .await; + + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await + .expect("encoded full response decodes"); + let (headers, status, _, _) = info.expect("decoded response metadata"); + + assert_eq!(collect_stream(stream).await.expect("decoded body streams"), IDENTITY_BODY); + assert_eq!(status, StatusCode::OK); + for name in ["content-encoding", "content-length", "content-range"] { + assert!(headers.iter().all(|(key, _)| key != name), "stale header {name}"); + } +} diff --git a/backend/app/src/api/model/streams/provider_stream_factory/tests/policy.rs b/backend/app/src/api/model/streams/provider_stream_factory/tests/policy.rs new file mode 100644 index 000000000..6e05b8f39 --- /dev/null +++ b/backend/app/src/api/model/streams/provider_stream_factory/tests/policy.rs @@ -0,0 +1,84 @@ +use super::*; + +#[test] +fn session_headers_are_forwarded_to_provider_requests() { + let addr = "127.0.0.1:8080".parse().unwrap(); + let stream_url = Url::parse("http://example.com/live/segment.ts").unwrap(); + let req_headers = HeaderMap::new(); + let mut session_headers = HashMap::new(); + session_headers.insert(String::from("cookie"), String::from("sid=abc; pref=1")); + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: true, + buffer_size: 1024, + buffer_max_bytes: 0, + pipe_provider_stream: false, + response_mode: StreamResponseMode::Stream, + }; + + let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { + addr, + item_type: PlaylistItemType::LiveHls, + share_stream: false, + stream_options: &stream_options, + stream_url: &stream_url, + req_headers: &req_headers, + input_headers: None, + session_headers: Some(&session_headers), + disabled_headers: None, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + }); + + assert_eq!( + options.get_headers().get(axum::http::header::COOKIE).and_then(|value| value.to_str().ok()), + Some("sid=abc; pref=1") + ); +} + +#[test] +fn provider_scheme_resolution_keeps_configured_headers_before_redirect() { + let stream_url = Url::parse("provider://mirrors/live/segment.ts").unwrap(); + let req_headers = HeaderMap::new(); + let input_headers = HashMap::from([("X-API-Key".to_string(), "api-secret".to_string())]); + let options = test_options( + ProviderContentRepresentationMode::PreserveOrigin, + &stream_url, + &req_headers, + Some(&input_headers), + None, + ); + let resolved_url = Url::parse("https://mirror-a.example/live/segment.ts").unwrap(); + + let request = prepare_client( + &reqwest::Client::new(), + &options, + Some(&resolved_url), + ProviderRequestCredentialState::OriginalOrigin, + ) + .0 + .build() + .unwrap(); + + assert_eq!(request.headers()["x-api-key"], "api-secret"); +} + +#[tokio::test] +async fn legacy_hls_declared_only_preserves_headerless_gzip_magic() { + let body = Bytes::from_static(b"\x1f\x8bnot-an-encoded-key"); + let (response, _) = local_response(StatusCode::OK, &[], body.to_vec()).await; + + let ProviderStreamFactoryResponse { stream, .. } = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await + .unwrap(); + + assert_eq!(collect_stream(stream).await.unwrap(), body); +} diff --git a/backend/app/src/api/model/streams/provider_stream_factory/tests/recovery.rs b/backend/app/src/api/model/streams/provider_stream_factory/tests/recovery.rs new file mode 100644 index 000000000..aac89f9c8 --- /dev/null +++ b/backend/app/src/api/model/streams/provider_stream_factory/tests/recovery.rs @@ -0,0 +1,76 @@ +use super::*; + +#[test] +fn test_provider_stream_factory_options_keeps_initial_retry_for_live_adaptive_streams() { + let addr = "127.0.0.1:8080".parse().unwrap(); + let stream_url = Url::parse("http://example.com/segment.ts").unwrap(); + let req_headers = HeaderMap::new(); + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: true, + buffer_size: 1024, + buffer_max_bytes: 0, + pipe_provider_stream: false, + response_mode: StreamResponseMode::Stream, + }; + + let hls_options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { + addr, + item_type: PlaylistItemType::LiveHls, + share_stream: false, + stream_options: &stream_options, + stream_url: &stream_url, + req_headers: &req_headers, + input_headers: None, + session_headers: None, + disabled_headers: None, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + }); + + let dash_options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { + addr, + item_type: PlaylistItemType::LiveDash, + share_stream: false, + stream_options: &stream_options, + stream_url: &stream_url, + req_headers: &req_headers, + input_headers: None, + session_headers: None, + disabled_headers: None, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + }); + + assert!(hls_options.should_retry_provider_request()); + assert!(dash_options.should_retry_provider_request()); + assert!(!hls_options.should_retry_initial_open_loop()); + assert!(!dash_options.should_retry_initial_open_loop()); +} + +#[tokio::test] +async fn legacy_hls_stream_decoder_failure_aborts_without_retry() { + let mut truncated = encode(b"decoder failure after response headers", TestEncoding::Gzip).await; + truncated.truncate(truncated.len().saturating_sub(5)); + let (response, requests) = local_response(StatusCode::OK, &[("Content-Encoding", "gzip")], truncated).await; + + let ProviderStreamFactoryResponse { stream, .. } = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await + .unwrap(); + let error = collect_stream(stream).await.expect_err("truncated gzip must terminate the body stream"); + + assert!(matches!(error, StreamError::ContentDecoding(_))); + assert_eq!(requests.load(Ordering::SeqCst), 1, "a body-stream failure cannot trigger a new request"); +} diff --git a/backend/app/src/api/model/streams/provider_stream_factory/tests/storage.rs b/backend/app/src/api/model/streams/provider_stream_factory/tests/storage.rs new file mode 100644 index 000000000..8b5f74d06 --- /dev/null +++ b/backend/app/src/api/model/streams/provider_stream_factory/tests/storage.rs @@ -0,0 +1,141 @@ +use super::*; + +#[test] +fn identity_is_enforced_after_header_merges_for_every_request_target() { + let stream_url = Url::parse("http://provider-a.example/live/segment.ts").unwrap(); + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::ACCEPT_ENCODING, "br".parse().unwrap()); + req_headers.insert(reqwest::header::RANGE, "bytes=17-31".parse().unwrap()); + req_headers.insert(reqwest::header::AUTHORIZATION, "Bearer client".parse().unwrap()); + req_headers.insert(reqwest::header::COOKIE, "client=1".parse().unwrap()); + let input_headers = HashMap::from([("Accept-Encoding".to_string(), "gzip".to_string())]); + let session_headers = HashMap::from([("Accept-Encoding".to_string(), "zstd".to_string())]); + let options = test_options( + ProviderContentRepresentationMode::Identity, + &stream_url, + &req_headers, + Some(&input_headers), + Some(&session_headers), + ); + let client = reqwest::Client::new(); + + let same_origin = Url::parse("http://provider-a.example/live/failover.ts").unwrap(); + let cross_origin = Url::parse("http://provider-b.example/live/segment.ts").unwrap(); + for target in [None, Some(&same_origin), Some(&cross_origin)] { + let credential_state = if target == Some(&cross_origin) { + ProviderRequestCredentialState::Scrubbed + } else { + ProviderRequestCredentialState::OriginalOrigin + }; + let request = prepare_client(&client, &options, target, credential_state).0.build().unwrap(); + assert_eq!(request.headers()[reqwest::header::ACCEPT_ENCODING], "identity"); + assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=17-31"); + if target == Some(&cross_origin) { + assert!(!request.headers().contains_key(reqwest::header::AUTHORIZATION)); + assert!(!request.headers().contains_key(reqwest::header::COOKIE)); + } + } + + let reconnect_options = options.clone(); + assert_eq!(reconnect_options.content_representation(), ProviderContentRepresentationMode::Identity); + assert_eq!(reconnect_options.hls_content_coding_object_kind(), Some(HlsOriginContentCodingObjectKind::Other)); + assert_eq!( + options.clone().for_deferred_open().hls_content_coding_object_kind(), + Some(HlsOriginContentCodingObjectKind::Other) + ); +} + +#[tokio::test] +async fn automatic_redirect_round_trip_keeps_identity_and_never_restores_credentials() { + let encoded = encode(b"decoded redirect body", TestEncoding::Gzip).await; + let RedirectRoundTrip { entry_url, first_origin_task, second_origin_task } = + spawn_redirect_round_trip(encoded).await; + let options = redirect_test_options(&entry_url); + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::limited(10)) + .build() + .expect("automatic redirect client"); + + let response = prepare_client(&client, &options, None, ProviderRequestCredentialState::OriginalOrigin) + .0 + .send() + .await + .expect("automatic redirect request succeeds"); + + assert_identity_redirect_result(response, first_origin_task, second_origin_task).await; +} + +#[tokio::test] +async fn manual_redirect_round_trip_keeps_identity_and_never_restores_credentials() { + let encoded = encode(b"decoded redirect body", TestEncoding::Gzip).await; + let RedirectRoundTrip { entry_url, first_origin_task, second_origin_task } = + spawn_redirect_round_trip(encoded).await; + let options = redirect_test_options(&entry_url); + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("manual redirect client"); + + let response = send_with_manual_redirects(&client, &options, &test_app_config()) + .await + .expect("manual redirect request succeeds"); + + assert_identity_redirect_result(response, first_origin_task, second_origin_task).await; +} + +#[tokio::test] +async fn legacy_hls_segment_key_map_part_and_other_stream_as_identity() { + const IDENTITY: &[u8] = b"identity hls resource bytes"; + let cases = [ + ("segment", "gzip", TestEncoding::Gzip), + ("segment-zlib", "deflate", TestEncoding::Zlib), + ("segment-raw-deflate", "deflate", TestEncoding::RawDeflate), + ("key", "br", TestEncoding::Brotli), + ("map", "zstd", TestEncoding::Zstd), + ("part", "gzip", TestEncoding::Gzip), + ("other", "br", TestEncoding::Brotli), + ]; + + for (resource_kind, content_encoding, encoding) in cases { + let encoded = encode(IDENTITY, encoding).await; + let (response, _) = local_response(StatusCode::OK, &[("Content-Encoding", content_encoding)], encoded).await; + + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await + .unwrap(); + + assert_eq!(collect_stream(stream).await.unwrap(), IDENTITY, "failed for {resource_kind}"); + let (headers, status, _, _) = info.unwrap(); + assert_eq!(status, StatusCode::OK); + assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-encoding"))); + assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-length"))); + } +} + +#[tokio::test] +async fn legacy_hls_identity_keeps_unencoded_partial_content_headers() { + const PARTIAL_BODY: &[u8] = b"abc"; + let (response, _) = + local_response(StatusCode::PARTIAL_CONTENT, &[("Content-Range", "bytes 0-2/10")], PARTIAL_BODY.to_vec()).await; + + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await + .expect("unencoded partial identity response is safe"); + let (headers, status, _, _) = info.expect("partial response metadata"); + + assert_eq!(collect_stream(stream).await.expect("partial body streams"), PARTIAL_BODY); + assert_eq!(status, StatusCode::PARTIAL_CONTENT); + assert!(headers.iter().any(|(name, value)| name == "content-length" && value == "3")); + assert!(headers.iter().any(|(name, value)| name == "content-range" && value == "bytes 0-2/10")); + assert!(headers.iter().all(|(name, _)| name != "content-encoding")); +} diff --git a/backend/app/src/api/model/streams/provider_stream_factory/tests/streaming.rs b/backend/app/src/api/model/streams/provider_stream_factory/tests/streaming.rs new file mode 100644 index 000000000..b453c5361 --- /dev/null +++ b/backend/app/src/api/model/streams/provider_stream_factory/tests/streaming.rs @@ -0,0 +1,196 @@ +use super::*; + +#[test] +fn test_provider_stream_factory_options_propagate_buffer_max_bytes() { + let addr = "127.0.0.1:8080".parse().unwrap(); + let stream_url = Url::parse("http://example.com/stream").unwrap(); + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: true, + buffer_size: 1024, + buffer_max_bytes: 4096, + pipe_provider_stream: false, + response_mode: StreamResponseMode::Stream, + }; + let req_headers = HeaderMap::new(); + let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { + addr, + item_type: PlaylistItemType::Live, + share_stream: false, + stream_options: &stream_options, + stream_url: &stream_url, + req_headers: &req_headers, + input_headers: None, + session_headers: None, + disabled_headers: None, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + }); + assert_eq!(options.get_buffer_max_bytes(), 4096); + assert_eq!(options.get_buffer_size(), 1024); +} + +#[test] +fn test_shared_streams_do_not_use_provider_buffer_wrapper() { + let addr = "127.0.0.1:8080".parse().unwrap(); + let stream_url = Url::parse("http://example.com/shared.ts").unwrap(); + let req_headers = HeaderMap::new(); + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: true, + buffer_size: 1024, + buffer_max_bytes: 0, + pipe_provider_stream: false, + response_mode: StreamResponseMode::Stream, + }; + + let shared_options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { + addr, + item_type: PlaylistItemType::Live, + share_stream: true, + stream_options: &stream_options, + stream_url: &stream_url, + req_headers: &req_headers, + input_headers: None, + session_headers: None, + disabled_headers: None, + default_user_agent: None, + username: None, + client_ip: None, + stream_channel: None, + connect_failure_stage: None, + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + }); + + assert!( + !should_wrap_provider_stream_in_buffer(&shared_options), + "shared streams must bypass provider-side BufferedStream" + ); +} + +#[test] +fn test_provider_stream_factory_options_builds_connect_failed_stream_info_from_history_context() { + let addr = "127.0.0.1:8080".parse().unwrap(); + let stream_url = Url::parse("http://example.com/stream").unwrap(); + let req_headers = HeaderMap::new(); + let stream_options = StreamOptions { + stream_retry: true, + buffer_enabled: true, + buffer_size: 1024, + buffer_max_bytes: 0, + pipe_provider_stream: false, + response_mode: StreamResponseMode::Stream, + }; + + let options = ProviderStreamFactoryOptions::new(&ProviderStreamFactoryParams { + addr, + item_type: PlaylistItemType::Live, + share_stream: false, + stream_options: &stream_options, + stream_url: &stream_url, + req_headers: &req_headers, + input_headers: None, + session_headers: None, + disabled_headers: None, + default_user_agent: None, + username: Some("alice"), + client_ip: Some("203.0.113.9"), + stream_channel: Some(&StreamChannel { + target_id: 1, + virtual_id: 77, + provider_id: 3, + input_name: "input-a".intern(), + item_type: PlaylistItemType::Live, + cluster: XtreamCluster::Live, + group: "News".intern(), + title: "Example".intern(), + url: "http://provider.example/live/77".intern(), + shared: false, + shared_joined_existing: None, + shared_stream_id: None, + technical: None, + epg_channel_id: None, + epg_reference_ts: None, + upstream_user_agent: Some("Channel-UA".intern()), + }), + connect_failure_stage: Some(FailureStage::ProviderOpen), + content_representation: ProviderContentRepresentationMode::PreserveOrigin, + }); + + let info = options.build_connect_failed_stream_info("provider-a".intern()).expect("history context"); + + assert_eq!(info.username, "alice"); + assert_eq!(info.client_ip, "203.0.113.9"); + assert_eq!(info.provider.as_ref(), "provider-a"); + assert_eq!(info.channel.input_name.as_ref(), "input-a"); + assert_eq!(info.channel.virtual_id, 77); + assert_eq!( + options.headers.get(reqwest::header::USER_AGENT).and_then(|value| value.to_str().ok()), + Some("Channel-UA") + ); +} + +#[test] +fn html_content_type_is_rejected_for_catchup_streams() { + let mut headers = HeaderMap::new(); + headers.insert(reqwest::header::CONTENT_TYPE, "text/html; charset=UTF-8".parse().unwrap()); + + assert!(should_reject_success_response_content_type(PlaylistItemType::Catchup, &headers)); + assert!(should_reject_success_response_content_type(PlaylistItemType::Video, &headers)); + assert!(should_reject_success_response_content_type(PlaylistItemType::Live, &headers)); +} + +#[test] +fn html_content_type_is_allowed_for_live_adaptive_streams() { + let mut headers = HeaderMap::new(); + headers.insert(reqwest::header::CONTENT_TYPE, "text/html; charset=UTF-8".parse().unwrap()); + + assert!(!should_reject_success_response_content_type(PlaylistItemType::LiveHls, &headers)); + assert!(!should_reject_success_response_content_type(PlaylistItemType::LiveDash, &headers)); +} + +#[tokio::test] +async fn provider_stream_factory_captures_session_cookies_separately() { + let (response, _) = local_response( + StatusCode::OK, + &[("content-type", "application/vnd.apple.mpegurl"), ("set-cookie", "sid=abc; Path=/")], + b"#EXTM3U\nsegment.ts\n".to_vec(), + ) + .await; + + let result = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await; + + assert!(result.is_ok()); + if let Ok(response) = result { + assert_eq!(response.provider_session_headers.headers.get("cookie").map(String::as_str), Some("sid=abc")); + assert!(response.info.as_ref().is_some_and(|(headers, _, _, _)| headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case("set-cookie")))); + } +} + +#[test] +fn preserve_origin_does_not_override_accept_encoding() { + let stream_url = Url::parse("http://provider.example/movie").unwrap(); + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip, br".parse().unwrap()); + let options = + test_options(ProviderContentRepresentationMode::PreserveOrigin, &stream_url, &req_headers, None, None); + + let request = + prepare_client(&reqwest::Client::new(), &options, None, ProviderRequestCredentialState::OriginalOrigin) + .0 + .build() + .unwrap(); + + assert_eq!(request.headers()[reqwest::header::ACCEPT_ENCODING], "gzip, br"); +} diff --git a/backend/app/src/api/model/streams/provider_stream_factory/tests/transport.rs b/backend/app/src/api/model/streams/provider_stream_factory/tests/transport.rs new file mode 100644 index 000000000..692b75046 --- /dev/null +++ b/backend/app/src/api/model/streams/provider_stream_factory/tests/transport.rs @@ -0,0 +1,250 @@ +use super::*; + +#[test] +fn hls_mp4_mime_detection_preserves_specific_types_and_repairs_container_defaults( +) -> Result<(), Box> { + for (path, original, expected) in [ + ("tracks-v1/init-1.hls.mp4", Some("video/MP2T"), "video/mp4"), + ("tracks-v1/init-1.hls.fmp4", Some("application/octet-stream"), "video/mp4"), + ("tracks-a1/init-1.hls.fmp4", None, "audio/mp4"), + ("tracks-a2/init-1.hls.mp4", Some("video/mp2t"), "audio/mp4"), + ("audio/init.mp4", Some("audio/mp4"), "audio/mp4"), + ("tracks-a1/init.mp4", Some("audio/mp4; codecs=mp4a.40.2"), "audio/mp4; codecs=mp4a.40.2"), + ("tracks-v1/segment.fmp4", None, "video/mp4"), + ("tracks-v1/segment.m4s", Some("video/mp2t; charset=binary"), "video/mp4"), + ("audio/segment.cmfa", None, "audio/mp4"), + ("audio/segment.m4a", None, "audio/mp4"), + ("video/segment.CMFV", None, "video/mp4"), + ("tracks-v1/init.mp4", Some("application/mp4"), "application/mp4"), + ("tracks-v1/init.mp4", Some("text/html"), "text/html"), + ("channel/dvr-1.ts", Some("video/MP2T"), "video/MP2T"), + ("key.bin", Some("application/octet-stream"), "application/octet-stream"), + ] { + for flussonic_audio_tracks in [true, false] { + let url = Url::parse(&format!("https://origin.example/{path}?token=signed&file=ignored.ts"))?; + let mut headers = HeaderMap::new(); + if let Some(original) = original { + headers.insert(reqwest::header::CONTENT_TYPE, HeaderValue::try_from(original)?); + } + let mut headers = headers.clone(); + normalize_hls_resource_content_type(&mut headers, &url, flussonic_audio_tracks); + // Without the input option only the extension decides; tracks-a paths stay video. + let expected = if !flussonic_audio_tracks + && path.starts_with("tracks-a") + && original.is_none_or(|o| !o.starts_with("audio")) + { + "video/mp4" + } else { + expected + }; + assert_eq!( + headers[reqwest::header::CONTENT_TYPE], + expected, + "{path} flussonic_audio_tracks={flussonic_audio_tracks}" + ); + } + } + Ok(()) +} + +#[test] +fn auth_failure_selects_origin_and_allocated_account() -> Result<(), Box> { + let dto = shared::model::ConfigInputDto { + name: Arc::from("root"), + url: "http://main.example".to_string(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + aliases: Some(vec![shared::model::ConfigInputAliasDto { + name: Arc::from("alias"), + url: "http://alias.example".to_string(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + ..shared::model::ConfigInputAliasDto::default() + }]), + ..shared::model::ConfigInputDto::default() + }; + let mut input = crate::model::ConfigInput::from(&dto); + let mut options = redirect_test_options(&Url::parse("http://alias.example/live/user/pass/1.ts")?); + assert_eq!(failed_stream_account(&input, &options).map(|account| account.name), Some(Arc::from("alias"))); + options.url = Url::parse("http://main.example/live/user/pass/1.ts")?; + assert_eq!(failed_stream_account(&input, &options).map(|account| account.name), Some(Arc::from("root"))); + options.url = Url::parse("http://cdn.example/token")?; + assert!(failed_stream_account(&input, &options).is_none()); + let alias = input.aliases.as_ref().and_then(|aliases| aliases.first()).ok_or("missing alias")?; + options.account = Some(Arc::new(crate::model::ProviderConfig::new_alias( + &input, + alias, + Arc::new(std::sync::RwLock::new(crate::model::ProviderConfigConnection::default())), + Arc::new(|_, _| {}), + ))); + assert_eq!(failed_stream_account(&input, &options).map(|account| account.name), Some(Arc::from("alias"))); + if let Some(alias) = input.aliases.as_mut().and_then(|aliases| aliases.first_mut()) { + alias.password = Some("renewed".to_string()); + } + assert!(failed_stream_account(&input, &options).is_none()); + Ok(()) +} + +#[test] +fn cross_origin_provider_requests_keep_only_redirect_safe_headers() { + let stream_url = Url::parse("http://provider-a.example/live/segment.ts").unwrap(); + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip".parse().unwrap()); + req_headers.insert(reqwest::header::RANGE, "bytes=17-31".parse().unwrap()); + req_headers.insert(reqwest::header::USER_AGENT, "test-agent".parse().unwrap()); + req_headers.insert(reqwest::header::AUTHORIZATION, "Bearer client".parse().unwrap()); + req_headers.insert(reqwest::header::COOKIE, "client=1".parse().unwrap()); + req_headers.insert(reqwest::header::PROXY_AUTHORIZATION, "Basic proxy-secret".parse().unwrap()); + let input_headers = HashMap::from([ + ("X-API-Key".to_string(), "api-secret".to_string()), + ("X-Provider-Token".to_string(), "provider-secret".to_string()), + ]); + let options = test_options( + ProviderContentRepresentationMode::PreserveOrigin, + &stream_url, + &req_headers, + Some(&input_headers), + None, + ); + assert!(provider_headers_require_manual_redirects(options.get_headers())); + + let mut safe_headers = HeaderMap::new(); + safe_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip".parse().unwrap()); + safe_headers.insert(reqwest::header::RANGE, "bytes=17-31".parse().unwrap()); + let safe_options = + test_options(ProviderContentRepresentationMode::PreserveOrigin, &stream_url, &safe_headers, None, None); + assert!(!provider_headers_require_manual_redirects(safe_options.get_headers())); + + let client = reqwest::Client::new(); + let sensitive_headers = ["authorization", "cookie", "proxy-authorization", "x-api-key", "x-provider-token"]; + + let same_origin = Url::parse("http://provider-a.example/live/failover.ts").unwrap(); + let same_origin_request = + prepare_client(&client, &options, Some(&same_origin), ProviderRequestCredentialState::OriginalOrigin) + .0 + .build() + .unwrap(); + for name in sensitive_headers { + assert!(same_origin_request.headers().contains_key(name), "same-origin request lost {name}"); + } + + let cross_origin = Url::parse("http://provider-b.example/live/segment.ts").unwrap(); + for (target, credential_state) in [ + (Some(&cross_origin), ProviderRequestCredentialState::Scrubbed), + (None, ProviderRequestCredentialState::Scrubbed), + ] { + let request = prepare_client(&client, &options, target, credential_state).0.build().unwrap(); + for name in sensitive_headers { + assert!(!request.headers().contains_key(name), "cross-origin request retained {name}"); + } + assert_eq!(request.headers()[reqwest::header::ACCEPT_ENCODING], "gzip"); + assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=17-31"); + assert_eq!(request.headers()[reqwest::header::USER_AGENT], "test-agent"); + } +} + +#[tokio::test] +async fn deferred_open_without_origin_head_accepts_only_plain_origin_200() { + const IDENTITY_BODY: &[u8] = b"deferred identity body"; + + let stream_url = Url::parse("http://provider.example/live/deferred.ts").unwrap(); + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip".parse().unwrap()); + let deferred_options = + test_options(ProviderContentRepresentationMode::PreserveOrigin, &stream_url, &req_headers, None, None) + .for_deferred_open(); + let deferred_request = prepare_client( + &reqwest::Client::new(), + &deferred_options, + None, + ProviderRequestCredentialState::OriginalOrigin, + ) + .0 + .build() + .expect("deferred provider request"); + assert_eq!(deferred_options.content_representation(), ProviderContentRepresentationMode::Identity); + assert!(!deferred_options.response_head_is_available()); + assert_eq!(deferred_options.hls_content_coding_object_kind(), None); + assert_eq!(deferred_request.headers()[reqwest::header::ACCEPT_ENCODING], "identity"); + + let encoded = encode(IDENTITY_BODY, TestEncoding::Gzip).await; + let (response, _) = local_response(StatusCode::OK, &[("Content-Encoding", "gzip")], encoded).await; + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Unavailable, + ) + .await + .expect("plain 200 is safe for deferred identity streaming"); + assert_eq!(collect_stream(stream).await.expect("deferred body decodes"), IDENTITY_BODY); + let (headers, status, _, _) = info.expect("normalized deferred response metadata"); + assert_eq!(status, StatusCode::OK); + assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-encoding"))); + assert!(headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("content-length"))); + + let encoded = encode(IDENTITY_BODY, TestEncoding::Gzip).await; + let (response, _) = + local_response(StatusCode::OK, &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 0-21/22")], encoded) + .await; + assert!(matches!( + prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Unavailable, + ) + .await, + Err(ProviderStreamPreparationError::DeferredResponseHead { status: StatusCode::OK, has_content_range: true }) + )); + + let (response, _) = local_response(StatusCode::PARTIAL_CONTENT, &[], IDENTITY_BODY.to_vec()).await; + assert!(matches!( + prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Unavailable, + ) + .await, + Err(ProviderStreamPreparationError::DeferredResponseHead { + status: StatusCode::PARTIAL_CONTENT, + has_content_range: false, + }) + )); +} + +#[tokio::test] +async fn deferred_open_rejects_preserve_origin_without_origin_head() { + let (response, _) = local_response(StatusCode::OK, &[], b"opaque bytes".to_vec()).await; + + assert!(matches!( + prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::PreserveOrigin, + ProviderResponseHeadAvailability::Unavailable, + ) + .await, + Err(ProviderStreamPreparationError::DeferredResponseHead { status: StatusCode::OK, has_content_range: false }) + )); +} + +#[tokio::test] +async fn legacy_hls_rejects_encoded_partial_content_before_client_streaming() { + let encoded = encode(b"partial identity bytes", TestEncoding::Gzip).await; + let (response, _) = local_response( + StatusCode::PARTIAL_CONTENT, + &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 0-9/10")], + encoded, + ) + .await; + + let result = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await; + + assert!(matches!( + result, + Err(ProviderStreamPreparationError::ContentCoding(ContentCodingError::EncodedPartialContent)) + )); +} diff --git a/backend/app/src/api/panel_api/account.rs b/backend/app/src/api/panel_api/account.rs new file mode 100644 index 000000000..f9be38a94 --- /dev/null +++ b/backend/app/src/api/panel_api/account.rs @@ -0,0 +1,219 @@ +use super::{ + build_panel_url, build_player_api_action_url, extract_account_creds_from_input, extract_stringish, + extract_username_password_from_json, first_json_object, normalize_panel_expire, panel_get_json, parse_boolish, + resolve_query_params, user_api_get_json, validate_account_info_params, validate_client_adult_content_params, + validate_client_info_params, validate_client_new_params, validate_client_renew_params, PanelApiTimeContext, +}; +use crate::{ + api::model::AppState, + model::{ConfigInput, InputSource, PanelApiConfig}, + utils::debug_if_enabled, +}; +use chrono_tz::Tz; +use shared::{ + error::TuliproxError, + utils::{ + get_base_url_from_str, get_credentials_from_url_str, get_i64_from_serde_value, get_string_from_serde_value, + sanitize_sensitive_info, + }, +}; +use std::sync::Arc; + +#[derive(Debug, Clone)] +pub(super) struct UserApiAccountInfo { + pub(super) exp_date: Option, + pub(super) server_now_ts: Option, + pub(super) server_tz: Option, +} + +pub(super) fn build_user_api_account_info_input_source( + input: &ConfigInput, + username: &str, + password: &str, +) -> Result { + let url = build_player_api_action_url(input.url.as_str(), username, password, "account_info").ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!( + "panel_api: invalid user_api base_url: {}", + sanitize_sensitive_info(input.url.as_str()) + )) + })?; + + Ok(InputSource::from(input).with_url(url.to_string())) +} + +pub(super) async fn panel_client_new( + app_state: &AppState, + cfg: &PanelApiConfig, +) -> Result<(String, String, Option), TuliproxError> { + validate_client_new_params(&cfg.query_parameter.client_new)?; + let params = resolve_query_params(&cfg.query_parameter.client_new, cfg.api_key.as_deref(), None)?; + let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; + let json = panel_get_json(app_state, url).await?; + let Some(obj) = first_json_object(&json) else { + return Err(TuliproxError::ConfigPanelApi( + "panel_api: client_new response is not a JSON object/array".to_string(), + )); + }; + let status_ok = obj.get("status").is_some_and(parse_boolish); + if !status_ok { + return Err(TuliproxError::ConfigPanelApi("panel_api: client_new status=false".to_string())); + } + if let Some((u, p)) = extract_username_password_from_json(obj) { + return Ok((u, p, None)); + } + if let Some(url_str) = obj.get("url").and_then(|v| v.as_str()) { + if let (Some(u), Some(p)) = get_credentials_from_url_str(url_str) { + let base = get_base_url_from_str(url_str); + return Ok((u, p, base)); + } + } + Err(TuliproxError::ConfigPanelApi( + "panel_api: client_new response missing username/password (and no parsable url)".to_string(), + )) +} + +pub(super) async fn panel_client_renew( + app_state: &AppState, + cfg: &PanelApiConfig, + username: &str, + password: &str, +) -> Result<(), TuliproxError> { + validate_client_renew_params(&cfg.query_parameter.client_renew)?; + let params = + resolve_query_params(&cfg.query_parameter.client_renew, cfg.api_key.as_deref(), Some((username, password)))?; + let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; + let json = panel_get_json(app_state, url).await?; + let Some(obj) = first_json_object(&json) else { + return Err(TuliproxError::ConfigPanelApi( + "panel_api: client_renew response is not a JSON object/array".to_string(), + )); + }; + let status_ok = obj.get("status").is_some_and(parse_boolish); + if !status_ok { + return Err(TuliproxError::ConfigPanelApi("panel_api: client_renew status=false".to_string())); + } + Ok(()) +} + +pub(super) async fn panel_client_info_raw( + app_state: &AppState, + cfg: &PanelApiConfig, + username: &str, + password: &str, +) -> Result, TuliproxError> { + validate_client_info_params(&cfg.query_parameter.client_info)?; + let params = + resolve_query_params(&cfg.query_parameter.client_info, cfg.api_key.as_deref(), Some((username, password)))?; + let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; + let json = panel_get_json(app_state, url).await?; + let Some(obj) = first_json_object(&json) else { + return Err(TuliproxError::ConfigPanelApi( + "panel_api: client_info response is not a JSON object/array".to_string(), + )); + }; + let status_ok = obj.get("status").is_some_and(parse_boolish); + if !status_ok { + return Err(TuliproxError::ConfigPanelApi("panel_api: client_info status=false".to_string())); + } + let expire = obj.get("expire").and_then(|v| v.as_str()).unwrap_or_default().trim().to_string(); + if expire.is_empty() { + Ok(None) + } else { + Ok(Some(expire)) + } +} + +pub(super) async fn panel_client_info( + app_state: &AppState, + cfg: &PanelApiConfig, + username: &str, + password: &str, + time_ctx: Option<&PanelApiTimeContext>, +) -> Result, TuliproxError> { + let expire = panel_client_info_raw(app_state, cfg, username, password).await?; + Ok(expire.as_deref().and_then(|value| normalize_panel_expire(value, time_ctx))) +} + +pub(super) async fn fetch_root_user_api_info( + app_state: &Arc, + input: &ConfigInput, +) -> Result, TuliproxError> { + let Some((username, password)) = extract_account_creds_from_input(input) else { + return Ok(None); + }; + + let input_source = build_user_api_account_info_input_source(input, username.as_ref(), password.as_ref())?; + + let json = match user_api_get_json(app_state, &input_source).await { + Ok(json) => json, + Err(err) => { + debug_if_enabled!( + "panel_api user_api account_info failed for {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + return Ok(None); + } + }; + + let exp_date = json.get("user_info").and_then(|v| v.get("exp_date")).and_then(get_i64_from_serde_value); + let server_now_ts = json.get("server_info").and_then(|v| v.get("timestamp_now")).and_then(get_i64_from_serde_value); + let server_tz = json + .get("server_info") + .and_then(|v| v.get("timezone")) + .and_then(get_string_from_serde_value) + .and_then(|tz| tz.parse::().ok()); + + Ok(Some(UserApiAccountInfo { exp_date, server_now_ts, server_tz })) +} + +pub(super) async fn panel_account_info( + app_state: &AppState, + cfg: &PanelApiConfig, + creds: Option<(&str, &str)>, +) -> Result, TuliproxError> { + if cfg.query_parameter.account_info.is_empty() { + return Ok(None); + } + validate_account_info_params(&cfg.query_parameter.account_info)?; + let params = resolve_query_params(&cfg.query_parameter.account_info, cfg.api_key.as_deref(), creds)?; + let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; + let json = panel_get_json(app_state, url).await?; + let Some(obj) = first_json_object(&json) else { + return Err(TuliproxError::ConfigPanelApi( + "panel_api: account_info response is not a JSON object/array".to_string(), + )); + }; + let status_ok = obj.get("status").is_some_and(parse_boolish); + if !status_ok { + return Err(TuliproxError::ConfigPanelApi("panel_api: account_info status=false".to_string())); + } + let Some(credits) = obj.get("credits").and_then(extract_stringish) else { + return Err(TuliproxError::ConfigPanelApi("panel_api: account_info response missing credits".to_string())); + }; + Ok(Some(credits)) +} + +pub(super) async fn panel_client_adult_content( + app_state: &AppState, + cfg: &PanelApiConfig, + creds: Option<(&str, &str)>, +) -> Result<(), TuliproxError> { + if cfg.query_parameter.client_adult_content.is_empty() { + return Ok(()); + } + validate_client_adult_content_params(&cfg.query_parameter.client_adult_content)?; + let params = resolve_query_params(&cfg.query_parameter.client_adult_content, cfg.api_key.as_deref(), creds)?; + let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; + let json = panel_get_json(app_state, url).await?; + let Some(obj) = first_json_object(&json) else { + return Err(TuliproxError::ConfigPanelApi( + "panel_api: client_adult_content response is not a JSON object/array".to_string(), + )); + }; + let status_ok = obj.get("status").is_some_and(parse_boolish); + if !status_ok { + return Err(TuliproxError::ConfigPanelApi("panel_api: client_adult_content status=false".to_string())); + } + Ok(()) +} diff --git a/backend/app/src/api/panel_api/alias_pool.rs b/backend/app/src/api/panel_api/alias_pool.rs new file mode 100644 index 000000000..cbdcffced --- /dev/null +++ b/backend/app/src/api/panel_api/alias_pool.rs @@ -0,0 +1,573 @@ +use super::{ + panel_client_adult_content, panel_client_info, panel_client_new, panel_client_renew, refresh_internal_sources, + should_reload_sources_after_internal_write, sync_panel_api_for_input_on_boot, validate_panel_api_config, + AccountCredentials, PanelApiOptionalFlags, PanelApiOptionalFlagsSet, PanelApiTimeContext, +}; +use crate::{ + api::{ + internal_csv::{csv_patch_batch_append, csv_patch_batch_update_exp_date}, + model::AppState, + source_yml_patch::{derive_unique_alias_name_set, resolve_provisioned_account_base_url, SourcesYmlPatch}, + }, + model::{ + is_input_expired, is_input_expired_at, ConfigInput, ConfigInputAlias, PanelApiConfig, ProxyUserCredentials, + }, + utils::debug_if_enabled, +}; +use log::warn; +use shared::{ + error::TuliproxError, + model::{InputType, PanelApiAliasPoolSizeValue, ProxyUserStatus}, + utils::{get_credentials_from_url, sanitize_sensitive_info}, +}; +use smallvec::SmallVec; +use std::{ + cmp::Ordering, + collections::HashSet, + path::{Path, PathBuf}, + sync::Arc, +}; +use url::Url; + +pub(super) fn extract_account_creds_from_input(input: &ConfigInput) -> Option<(String, String)> { + if let (Some(u), Some(p)) = (input.username.as_deref(), input.password.as_deref()) { + if !u.trim().is_empty() && !p.trim().is_empty() { + return Some((u.to_string(), p.to_string())); + } + } + Url::parse(input.url.as_str()).ok().and_then(|u| { + let (uu, pp) = get_credentials_from_url(&u); + match (uu, pp) { + (Some(uu), Some(pp)) if !uu.trim().is_empty() && !pp.trim().is_empty() => Some((uu, pp)), + _ => None, + } + }) +} + +pub(super) fn alias_pool_limit_values( + cfg: &PanelApiConfig, +) -> (Option<&PanelApiAliasPoolSizeValue>, Option<&PanelApiAliasPoolSizeValue>) { + let size = cfg.alias_pool.as_ref().and_then(|p| p.size.as_ref()); + let min = size.and_then(|s| s.min.as_ref()); + let max = size.and_then(|s| s.max.as_ref()); + (min, max) +} + +pub(super) fn alias_pool_has_min(cfg: &PanelApiConfig) -> bool { + let (min, _) = alias_pool_limit_values(cfg); + min.is_some() +} + +pub(super) fn resolve_alias_pool_limit_value( + value: Option<&PanelApiAliasPoolSizeValue>, + auto_value: Option, +) -> Option { + match value { + Some(PanelApiAliasPoolSizeValue::Number(v)) => Some(*v), + Some(PanelApiAliasPoolSizeValue::Auto(_)) => auto_value, + None => None, + } +} + +pub(super) fn is_proxy_user_enabled(user: &ProxyUserCredentials) -> bool { + if let Some(status) = user.status { + if !matches!(status, ProxyUserStatus::Active | ProxyUserStatus::Trial) { + return false; + } + } + !is_input_expired(user.exp_date) +} + +pub(super) fn find_input_target_names(app_state: &AppState, input_name: &Arc) -> Vec { + let sources = app_state.app_config.sources.load(); + for source in &sources.sources { + if source.inputs.iter().any(|name| name == input_name) { + let mut names: SmallVec<[String; 8]> = SmallVec::new(); + for target in &source.targets { + names.push(target.name.clone()); + } + return names.into_vec(); + } + } + Vec::new() +} + +pub(super) fn count_enabled_proxy_users(app_state: &AppState, input_name: &Arc) -> usize { + let api_proxy_guard = app_state.app_config.api_proxy.load(); + let Some(api_proxy) = api_proxy_guard.as_ref() else { + return 0; + }; + let target_names = find_input_target_names(app_state, input_name); + if target_names.is_empty() { + return 0; + } + api_proxy + .user + .iter() + .filter(|target_user| target_names.iter().any(|target| target.eq_ignore_ascii_case(&target_user.target))) + .map(|target_user| target_user.credentials.iter().filter(|cred| is_proxy_user_enabled(cred)).count()) + .sum() +} + +pub(super) fn resolve_alias_pool_auto_value(app_state: &AppState, input_name: &Arc) -> u16 { + let enabled_users = count_enabled_proxy_users(app_state, input_name); + u16::try_from(enabled_users).unwrap_or(u16::MAX) +} + +pub(crate) fn target_has_alias_pool_min(app_state: &AppState, target_name: &str) -> bool { + let sources = app_state.app_config.sources.load(); + for source in &sources.sources { + let target_match = source.targets.iter().any(|target| target.name.eq_ignore_ascii_case(target_name)); + if !target_match { + continue; + } + for input_name in &source.inputs { + let Some(input) = sources.get_input_by_name(input_name) else { + continue; + }; + if let Some(panel_cfg) = input.panel_api.as_ref() { + if !panel_cfg.enabled { + continue; + } + if alias_pool_has_min(panel_cfg) { + return true; + } + } + } + } + false +} + +pub(super) fn resolve_alias_pool_limits( + app_state: &AppState, + input_name: &Arc, + cfg: &PanelApiConfig, +) -> Result<(Option, Option), TuliproxError> { + let (min_val, max_val) = alias_pool_limit_values(cfg); + if min_val.is_none() && max_val.is_none() { + return Ok((None, None)); + } + let min_auto = min_val.is_some_and(PanelApiAliasPoolSizeValue::is_auto); + let auto_value = min_auto.then(|| resolve_alias_pool_auto_value(app_state, input_name)); + let min = resolve_alias_pool_limit_value(min_val, auto_value); + let max = match max_val { + Some(PanelApiAliasPoolSizeValue::Number(value)) => Some(*value), + Some(PanelApiAliasPoolSizeValue::Auto(_)) | None => None, + }; + if let (Some(min), Some(max)) = (min, max) { + if min > max { + return Err(TuliproxError::ConfigPanelApi( + "panel_api.alias_pool.size.min must be <= panel_api.alias_pool.size.max".to_string(), + )); + } + } + Ok((min, max)) +} + +pub(super) fn resolve_alias_pool_min(app_state: &AppState, input_name: &Arc, cfg: &PanelApiConfig) -> Option { + let (min_val, _) = alias_pool_limit_values(cfg); + let min_val = min_val?; + let auto_value = min_val.is_auto().then(|| resolve_alias_pool_auto_value(app_state, input_name)); + resolve_alias_pool_limit_value(Some(min_val), auto_value) +} + +pub(super) fn alias_pool_remove_expired(cfg: &PanelApiConfig) -> bool { + cfg.alias_pool.as_ref().is_some_and(|p| p.remove_expired) +} + +pub(super) fn collect_accounts(input: &ConfigInput) -> Vec { + let mut out = Vec::new(); + if let Some((u, p)) = extract_account_creds_from_input(input) { + out.push(AccountCredentials { name: input.name.clone(), username: u, password: p, exp_date: input.exp_date }); + } + if let Some(aliases) = input.aliases.as_ref() { + for a in aliases { + if let (Some(u), Some(p)) = (a.username.as_deref(), a.password.as_deref()) { + if !u.trim().is_empty() && !p.trim().is_empty() { + out.push(AccountCredentials { + name: a.name.clone(), + username: u.to_string(), + password: p.to_string(), + exp_date: a.exp_date, + }); + } + } + } + } + out +} + +pub(super) fn compare_alias_exp_date_config(a: &ConfigInputAlias, b: &ConfigInputAlias) -> Ordering { + compare_named_exp_date(a.exp_date, a.name.as_ref(), b.exp_date, b.name.as_ref()) +} + +pub(super) fn compare_account_exp_date(a: &AccountCredentials, b: &AccountCredentials) -> Ordering { + compare_named_exp_date(a.exp_date, a.name.as_ref(), b.exp_date, b.name.as_ref()) +} + +pub(super) fn compare_named_exp_date( + a_exp_date: Option, + a_name: &str, + b_exp_date: Option, + b_name: &str, +) -> Ordering { + let a_ts = a_exp_date.unwrap_or(i64::MIN); + let b_ts = b_exp_date.unwrap_or(i64::MIN); + b_ts.cmp(&a_ts).then_with(|| a_name.cmp(b_name)) +} + +pub(super) fn sort_account_aliases_keep_root_first(accounts: &mut Vec, root_name: &str) { + let root = accounts.iter().find(|acct| acct.name.as_ref() == root_name).cloned(); + let mut aliases: Vec = + accounts.iter().filter(|acct| acct.name.as_ref() != root_name).cloned().collect(); + aliases.sort_by(compare_account_exp_date); + accounts.clear(); + if let Some(root) = root { + accounts.push(root); + } + accounts.extend(aliases); +} + +pub(super) fn is_account_valid(exp_date: Option) -> bool { exp_date.is_some() && !is_input_expired(exp_date) } + +pub(super) fn count_valid_accounts(accounts: &[AccountCredentials]) -> usize { + accounts.iter().filter(|acct| is_account_valid(acct.exp_date)).count() +} + +pub(super) fn root_counts_towards_pool(accounts: &[AccountCredentials], input_name: &Arc) -> bool { + accounts.iter().find(|acct| &acct.name == input_name).is_some_and(|acct| is_account_valid(acct.exp_date)) +} + +pub(super) fn count_valid_accounts_at(accounts: &[AccountCredentials], now: u64) -> usize { + accounts.iter().filter(|acct| acct.exp_date.is_some() && !is_input_expired_at(acct.exp_date, now)).count() +} + +pub(super) fn count_valid_alias_accounts_at(accounts: &[AccountCredentials], input_name: &Arc, now: u64) -> usize { + accounts + .iter() + .filter(|acct| &acct.name != input_name && acct.exp_date.is_some() && !is_input_expired_at(acct.exp_date, now)) + .count() +} + +pub(super) fn root_counts_towards_pool_at(accounts: &[AccountCredentials], input_name: &Arc, now: u64) -> bool { + accounts + .iter() + .find(|acct| &acct.name == input_name) + .is_some_and(|acct| acct.exp_date.is_some() && !is_input_expired_at(acct.exp_date, now)) +} + +pub(crate) fn is_alias_pool_max_reached(app_state: &AppState, input: &ConfigInput) -> bool { + let Some(panel_cfg) = input.panel_api.as_ref() else { + return false; + }; + if !panel_cfg.enabled { + return false; + } + if panel_cfg.url.trim().is_empty() { + return false; + } + if validate_panel_api_config(panel_cfg).is_err() { + return false; + } + let Ok((_, max_pool)) = resolve_alias_pool_limits(app_state, &input.name, panel_cfg) else { + return false; + }; + if let Some(max_pool) = max_pool { + let valid_count = count_valid_accounts(&collect_accounts(input)); + if valid_count >= max_pool as usize { + debug_if_enabled!( + "panel_api: alias_pool.size.max reached for input {} (valid_accounts={}, max={})", + sanitize_sensitive_info(&input.name), + valid_count, + max_pool + ); + return true; + } + } + false +} + +#[allow(clippy::too_many_lines, clippy::too_many_arguments)] +pub(super) async fn ensure_alias_pool_min( + app_state: &Arc, + input: &ConfigInput, + panel_cfg: &PanelApiConfig, + accounts: &mut Vec, + min_pool: u16, + csv_path: Option<&Path>, + sources_yml_patches: &mut Vec, + time_ctx: Option<&PanelApiTimeContext>, + effective_now: u64, + optional: PanelApiOptionalFlagsSet, +) -> (bool, u16) { + if min_pool == 0 { + return (false, 0); + } + + let renew_enabled = optional.contains(PanelApiOptionalFlags::ClientRenew); + let new_enabled = optional.contains(PanelApiOptionalFlags::ClientNew); + let adult_enabled = optional.contains(PanelApiOptionalFlags::AdultContent); + if !renew_enabled && !new_enabled { + return (false, 0); + } + + let mut changed = false; + let mut provisioned = 0_u16; + let max_pool = resolve_alias_pool_limits(app_state.as_ref(), &input.name, panel_cfg).ok().and_then(|(_, max)| max); + let mut existing_names: HashSet> = accounts.iter().map(|a| a.name.clone()).collect(); + let max_attempts = usize::from(min_pool).saturating_add(10); + for _ in 0..max_attempts { + let current_valid = count_valid_alias_accounts_at(accounts, &input.name, effective_now); + if current_valid >= usize::from(min_pool) { + break; + } + if let Some(max_pool) = max_pool { + if current_valid >= usize::from(max_pool) { + break; + } + } + + let expired_index = accounts + .iter() + .enumerate() + .filter(|(_, acct)| acct.name != input.name && is_input_expired_at(acct.exp_date, effective_now)) + .min_by_key(|(_, acct)| acct.exp_date.unwrap_or(i64::MAX)) + .map(|(idx, _)| idx); + + if let Some(idx) = expired_index { + let acct = accounts.get(idx).cloned(); + let Some(acct) = acct else { + break; + }; + if renew_enabled { + match panel_client_renew(app_state.as_ref(), panel_cfg, acct.username.as_str(), acct.password.as_str()) + .await + { + Ok(()) => { + provisioned = provisioned.saturating_add(1); + if adult_enabled { + if let Err(err) = panel_client_adult_content( + app_state.as_ref(), + panel_cfg, + Some((acct.username.as_str(), acct.password.as_str())), + ) + .await + { + debug_if_enabled!( + "panel_api client_adult_content failed for {}: {}", + sanitize_sensitive_info(&acct.name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + + let refreshed_exp = panel_client_info( + app_state.as_ref(), + panel_cfg, + acct.username.as_str(), + acct.password.as_str(), + time_ctx, + ) + .await + .ok() + .flatten() + .or(acct.exp_date); + + if let Some(new_exp) = refreshed_exp { + if let Some(acct_mut) = accounts.get_mut(idx) { + acct_mut.exp_date = Some(new_exp); + } + if let Some(csv_path) = csv_path { + let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; + if let Err(err) = csv_patch_batch_update_exp_date( + app_state, + &csv_lock, + input.input_type, + csv_path, + &acct.name, + acct.username.as_str(), + acct.password.as_str(), + new_exp, + ) + .await + { + debug_if_enabled!("panel_api failed to persist renew exp_date to csv: {}", err); + } else { + changed = true; + } + } else { + sources_yml_patches.push(SourcesYmlPatch::UpdatePanelAccountExpiry { + input_name: input.name.clone(), + account_name: acct.name.clone(), + exp_date: new_exp, + }); + changed = true; + } + } + continue; + } + Err(err) => { + debug_if_enabled!( + "panel_api client_renew failed for {}: {}", + sanitize_sensitive_info(&acct.name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + } + } + + if !new_enabled { + break; + } + match panel_client_new(app_state.as_ref(), panel_cfg).await { + Ok((username, password, base_url_from_resp)) => { + let base_url = resolve_provisioned_account_base_url( + input.url.as_str(), + base_url_from_resp.as_deref(), + &username, + &password, + ); + + let alias_name = derive_unique_alias_name_set(&existing_names, &input.name, &username); + existing_names.insert(alias_name.clone().into()); + + if adult_enabled { + if let Err(err) = + panel_client_adult_content(app_state.as_ref(), panel_cfg, Some((&username, &password))).await + { + debug_if_enabled!( + "panel_api client_adult_content failed for {}: {}", + sanitize_sensitive_info(&alias_name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + + let exp_date = panel_client_info(app_state.as_ref(), panel_cfg, &username, &password, time_ctx) + .await + .ok() + .flatten(); + + accounts.push(AccountCredentials { + name: alias_name.clone().into(), + username: username.clone(), + password: password.clone(), + exp_date, + }); + provisioned = provisioned.saturating_add(1); + + if let Some(csv_path) = csv_path { + let batch_type = if input.input_type == InputType::Xtream { + InputType::XtreamBatch + } else if input.input_type == InputType::M3u { + InputType::M3uBatch + } else { + input.input_type + }; + let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; + if let Err(err) = csv_patch_batch_append( + app_state, + &csv_lock, + csv_path, + batch_type, + &alias_name, + &base_url, + &username, + &password, + exp_date, + ) + .await + { + warn!("panel_api failed to append new account to csv: {err}"); + break; + } + changed = true; + } else { + sources_yml_patches.push(SourcesYmlPatch::AddAlias { + input_name: input.name.clone(), + alias_name: alias_name.into(), + base_url, + username, + password, + exp_date, + }); + changed = true; + } + } + Err(err) => { + debug_if_enabled!("panel_api client_new failed: {}", sanitize_sensitive_info(&err.to_string())); + break; + } + } + } + + (changed, provisioned) +} + +pub(crate) async fn sync_panel_api_exp_dates(app_state: &Arc) { + let sources_file_path = app_state.app_config.paths.load().sources_file_path.clone(); + let sources_path = PathBuf::from(&sources_file_path); + let mut any_change = false; + + let sources = app_state.app_config.sources.load(); + for input in &sources.inputs { + if sync_panel_api_for_input_on_boot(app_state, input, sources_path.as_path()).await { + any_change = true; + } + } + + if any_change { + // Account writes already synchronize prepared sources; refresh allocation without restarting services. + refresh_internal_sources(app_state); + } +} + +pub(crate) async fn sync_panel_api_exp_dates_on_boot(app_state: &Arc) { + sync_panel_api_exp_dates(app_state).await; +} + +pub(crate) async fn sync_panel_api_alias_pool_for_target(app_state: &Arc, target_name: &str) { + let sources_file_path = app_state.app_config.paths.load().sources_file_path.clone(); + let sources_path = PathBuf::from(&sources_file_path); + let mut any_change = false; + + let sources = app_state.app_config.sources.load(); + for source in &sources.sources { + let target_match = source.targets.iter().any(|target| target.name.eq_ignore_ascii_case(target_name)); + if !target_match { + continue; + } + + for input_name in &source.inputs { + let Some(input) = sources.get_input_by_name(input_name) else { + continue; + }; + let Some(panel_cfg) = input.panel_api.as_ref() else { + continue; + }; + if !panel_cfg.enabled || panel_cfg.url.trim().is_empty() { + continue; + } + if let Err(err) = validate_panel_api_config(panel_cfg) { + debug_if_enabled!( + "panel_api user sync skipped for {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + continue; + } + if !alias_pool_has_min(panel_cfg) { + continue; + } + + if sync_panel_api_for_input_on_boot(app_state, input, sources_path.as_path()).await { + any_change = true; + } + } + } + + if any_change && should_reload_sources_after_internal_write(app_state.as_ref()) { + refresh_internal_sources(app_state); + } +} diff --git a/backend/app/src/api/panel_api/mod.rs b/backend/app/src/api/panel_api/mod.rs index 1fd906ff9..a69ca5497 100644 --- a/backend/app/src/api/panel_api/mod.rs +++ b/backend/app/src/api/panel_api/mod.rs @@ -19,3774 +19,84 @@ //! mutates the root state directly, would have to move too. That is a //! deliberate not-yet, not an oversight. -use crate::{ - api::{ - internal_csv::{ - csv_patch_batch_append, csv_patch_batch_remove_expired, csv_patch_batch_sort_by_exp_date, - csv_patch_batch_update_credentials, csv_patch_batch_update_exp_date, - }, - model::{ - create_panel_api_provisioning_stream_with_stop, create_provider_connections_exhausted_stream, AppState, - StreamDetails, - }, - source_yml_patch::{ - derive_unique_alias_name, derive_unique_alias_name_set, resolve_provisioned_account_base_url, - SourcesYmlPatch, - }, - }, - model::{ - is_input_expired, is_input_expired_at, ConfigInput, ConfigInputAlias, GracePeriodOptions, InputSource, - PanelApiConfig, PanelApiQueryParam, ProxyUserCredentials, - }, - repository::{get_csv_file_path, AliasExpDateSortOrder}, - utils::{debug_if_enabled, format_http_status, request}, -}; -use axum::http::{header, HeaderMap, Method, StatusCode}; -use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; -use chrono_tz::Tz; -use jsonwebtoken::get_current_timestamp; -use log::{error, warn}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use shared::{ - concat_string, create_bitset, - error::{string_to_io_error, TuliproxError}, - model::{ - DisconnectReason, InputType, PanelApiAliasPoolSizeValue, PanelApiProvisioningMethod, ProxyUserStatus, VirtualId, - }, - utils::{ - get_base_url_from_str, get_credentials_from_url, get_credentials_from_url_str, get_i64_from_serde_value, - get_string_from_serde_value, parse_timestamp, sanitize_sensitive_info, - }, -}; -use smallvec::SmallVec; -use std::{ - cmp::Ordering, - collections::{HashMap, HashSet}, - io, - net::SocketAddr, - path::{Path, PathBuf}, - sync::Arc, - time::{Duration, Instant}, -}; -use tokio_util::sync::CancellationToken; -use url::Url; - -#[derive(Debug, Clone)] -struct AccountCredentials { - name: Arc, - username: String, - password: String, - exp_date: Option, -} +use shared::create_bitset; const PANEL_API_REQUEST_TIMEOUT_SECS: u64 = 30; + const PANEL_API_RETRY_ATTEMPTS: usize = 2; + const PANEL_API_DEFAULT_RETRY_AFTER_SECS: u64 = 1; + const PANEL_API_MAX_RETRY_AFTER_SECS: u64 = 5; -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -enum PanelApiExpireMode { - UtcString, - ServerTzString, -} - -#[derive(Debug, Clone)] -struct PanelApiTimeContext { - expire_mode: PanelApiExpireMode, - server_tz: Option, -} - -#[derive(Debug, Clone)] -struct UserApiAccountInfo { - exp_date: Option, - server_now_ts: Option, - server_tz: Option, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -struct PanelApiTimeCache { - inputs: HashMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct PanelApiTimeCacheEntry { - expire_mode: PanelApiExpireMode, - server_tz: Option, - skew_secs: Option, -} - -fn parse_boolish(value: &Value) -> bool { - match value { - Value::Bool(b) => *b, - Value::Number(n) => n.as_i64().unwrap_or(0) != 0, - Value::String(s) => matches!(s.trim().to_lowercase().as_str(), "true" | "1" | "yes" | "y" | "ok"), - _ => false, - } -} - -fn extract_stringish(value: &Value) -> Option { - match value { - Value::String(s) => { - let trimmed = s.trim(); - if trimmed.is_empty() { - None - } else { - Some(trimmed.to_string()) - } - } - Value::Number(n) => Some(n.to_string()), - Value::Bool(b) => Some(b.to_string()), - _ => None, - } -} - -fn is_date_only_yyyy_mm_dd(value: &str) -> bool { - let value = value.trim(); - if value.len() != 10 { - return false; - } - let bytes = value.as_bytes(); - bytes[4] == b'-' - && bytes[7] == b'-' - && bytes[0..4].iter().all(u8::is_ascii_digit) - && bytes[5..7].iter().all(u8::is_ascii_digit) - && bytes[8..10].iter().all(u8::is_ascii_digit) -} - -fn parse_panel_expire_utc(value: &str) -> Option { - let value = value.trim(); - if value.is_empty() { - return None; - } - let parsed = parse_timestamp(value).ok().flatten(); - if parsed.is_some() { - return parsed; - } - if is_date_only_yyyy_mm_dd(value) { - let normalized = format!("{value} 00:00:00"); - return parse_timestamp(&normalized).ok().flatten(); - } - None -} - -fn parse_panel_expire_with_tz(value: &str, tz: Tz) -> Option { - let value = value.trim(); - if value.is_empty() { - return None; - } - if let Ok(ts) = value.parse::() { - return Some(ts); - } - let value = if is_date_only_yyyy_mm_dd(value) { format!("{value} 00:00:00") } else { value.to_string() }; - let dt = NaiveDateTime::parse_from_str(&value, "%Y-%m-%d %H:%M:%S").ok()?; - match tz.from_local_datetime(&dt) { - chrono::LocalResult::Single(local_dt) => Some(local_dt.timestamp()), - chrono::LocalResult::Ambiguous(first, _) => Some(first.timestamp()), - chrono::LocalResult::None => None, - } -} - -fn normalize_panel_expire(value: &str, ctx: Option<&PanelApiTimeContext>) -> Option { - let Some(ctx) = ctx else { - return parse_panel_expire_utc(value); - }; - if let Ok(ts) = value.trim().parse::() { - return Some(ts); - } - match ctx.expire_mode { - PanelApiExpireMode::UtcString => parse_panel_expire_utc(value), - PanelApiExpireMode::ServerTzString => { - ctx.server_tz.and_then(|tz| parse_panel_expire_with_tz(value, tz)).or_else(|| parse_panel_expire_utc(value)) - } - } -} - -fn is_expiring_with_offset_at(exp_date: Option, offset_secs: u64, now: u64) -> bool { - let Some(exp_date) = exp_date else { - return false; - }; - let Ok(exp_ts) = u64::try_from(exp_date) else { - return true; - }; - if exp_ts <= now { - return false; - } - now.saturating_add(offset_secs) >= exp_ts -} - -fn first_json_object(value: &Value) -> Option<&serde_json::Map> { - match value { - Value::Array(arr) => arr.first().and_then(|v| v.as_object()), - Value::Object(obj) => Some(obj), - _ => None, - } -} - -fn extract_username_password_from_json(obj: &serde_json::Map) -> Option<(String, String)> { - let username = obj.get("username").and_then(|v| v.as_str()).map(str::trim).filter(|s| !s.is_empty()); - let password = obj.get("password").and_then(|v| v.as_str()).map(str::trim).filter(|s| !s.is_empty()); - match (username, password) { - (Some(u), Some(p)) => Some((u.to_string(), p.to_string())), - _ => None, - } -} - -fn validate_type_is_m3u(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { - let typ = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("type")).map(|p| p.value.trim().to_string()); - let typ_str = typ.as_deref().unwrap_or_default(); - if typ_str.trim().eq_ignore_ascii_case("m3u") { - Ok(()) - } else if typ_str.is_empty() { - Err(TuliproxError::ConfigPanelApi("panel_api: missing required query param 'type=m3u'".to_string())) - } else { - Err(TuliproxError::ConfigPanelApi(format!("panel_api: unsupported type={typ_str}, only m3u is supported"))) - } -} - -fn require_api_key_param(params: &[PanelApiQueryParam], section: &str) -> Result<(), TuliproxError> { - let api_key = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("api_key")); - let Some(api_key) = api_key else { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api: {section} must contain query param 'api_key' (use value 'auto')" - ))); - }; - if api_key.value.trim().is_empty() { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api: {section} query param 'api_key' must not be empty (use value 'auto')" - ))); - } - Ok(()) -} - -fn require_username_password_params_auto(params: &[PanelApiQueryParam], section: &str) -> Result<(), TuliproxError> { - let username = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("username")); - let password = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("password")); - if username.is_none() || password.is_none() { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api: {section} must contain query params 'username' and 'password' (use value 'auto')" - ))); - } - if !username.is_some_and(|p| p.value.trim().eq_ignore_ascii_case("auto")) - || !password.is_some_and(|p| p.value.trim().eq_ignore_ascii_case("auto")) - { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api: {section} requires 'username: auto' and 'password: auto' (credentials must not be hardcoded)" - ))); - } - Ok(()) -} - -fn validate_client_new_params(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { - require_api_key_param(params, "query_parameter.client_new")?; - validate_type_is_m3u(params)?; - if params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("user")) { - return Err(TuliproxError::ConfigPanelApi( - "panel_api: client_new must not contain query param 'user'".to_string(), - )); - } - Ok(()) -} - -fn validate_client_renew_params(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { - require_api_key_param(params, "query_parameter.client_renew")?; - validate_type_is_m3u(params)?; - require_username_password_params_auto(params, "query_parameter.client_renew")?; - Ok(()) -} - -fn validate_client_info_params(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { - require_api_key_param(params, "query_parameter.client_info")?; - require_username_password_params_auto(params, "query_parameter.client_info")?; - Ok(()) -} - -fn validate_account_info_params(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { - require_api_key_param(params, "query_parameter.account_info")?; - let has_user = params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("username")); - let has_pass = params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("password")); - if has_user || has_pass { - require_username_password_params_auto(params, "query_parameter.account_info")?; - } - Ok(()) -} - -fn validate_client_adult_content_params(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { - require_api_key_param(params, "query_parameter.client_adult_content")?; - let has_user = params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("username")); - let has_pass = params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("password")); - if has_user || has_pass { - require_username_password_params_auto(params, "query_parameter.client_adult_content")?; - } - Ok(()) -} - create_bitset!(u8, PanelApiOptionalFlags, AccountInfo, ClientNew, ClientRenew, AdultContent); -fn resolve_panel_api_optional_flags(cfg: &PanelApiConfig, input_name: &str) -> PanelApiOptionalFlagsSet { - let mut flags = PanelApiOptionalFlagsSet::new(); - if !cfg.query_parameter.account_info.is_empty() { - flags.set(PanelApiOptionalFlags::AccountInfo); - } - if !cfg.query_parameter.client_new.is_empty() { - flags.set(PanelApiOptionalFlags::ClientNew); - } - if !cfg.query_parameter.client_renew.is_empty() { - flags.set(PanelApiOptionalFlags::ClientRenew); - } - if !cfg.query_parameter.client_adult_content.is_empty() { - flags.set(PanelApiOptionalFlags::AdultContent); - } - - let name = sanitize_sensitive_info(input_name); - if !flags.contains(PanelApiOptionalFlags::ClientRenew) { - debug_if_enabled!("panel_api request for client_renew disabled due to missing arguments for {}", name); - } - if !flags.contains(PanelApiOptionalFlags::ClientNew) { - debug_if_enabled!("panel_api request for client_new disabled due to missing arguments for {}", name); - } - if !flags.contains(PanelApiOptionalFlags::AdultContent) { - debug_if_enabled!("panel_api request for client_adult_content disabled due to missing arguments for {}", name); - } - if !flags.contains(PanelApiOptionalFlags::AccountInfo) { - debug_if_enabled!("panel_api request for account_info disabled due to missing arguments for {}", name); - } - flags -} - -fn parse_panel_api_provisioning_offset_secs(offset: &str) -> Result { - let raw = offset.trim(); - if raw.is_empty() { - return Ok(0); - } - let lower = raw.to_ascii_lowercase(); - let bytes = lower.as_bytes(); - let last = *bytes.last().unwrap_or(&b'\0'); - let (num_part, multiplier) = match last { - b's' => (&lower[..lower.len().saturating_sub(1)], 1_u64), - b'm' => (&lower[..lower.len().saturating_sub(1)], 60_u64), - b'h' => (&lower[..lower.len().saturating_sub(1)], 60_u64 * 60), - b'd' => (&lower[..lower.len().saturating_sub(1)], 60_u64 * 60 * 24), - b'0'..=b'9' => (lower.as_str(), 1_u64), - _ => { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api.provisioning.offset must be a number with optional suffix s/m/h/d (e.g. 30m, 12h), got '{raw}'" - ))); - } - }; - let num_part = num_part.trim(); - if num_part.is_empty() { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api.provisioning.offset must be a number with optional suffix s/m/h/d (e.g. 30m, 12h), got '{raw}'" - ))); - } - let value: u64 = num_part.parse().map_err(|_| { - TuliproxError::ConfigPanelApi(format!("panel_api.provisioning.offset is not a valid number: '{raw}'")) - })?; - value - .checked_mul(multiplier) - .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api.provisioning.offset is too large: '{raw}'"))) -} - -fn validate_panel_api_config(cfg: &PanelApiConfig) -> Result<(), TuliproxError> { - if !cfg.enabled { - return Ok(()); - } - if cfg.url.trim().is_empty() { - return Err(TuliproxError::ConfigPanelApi("panel_api: url is missing".to_string())); - } - if cfg.api_key.as_ref().is_none_or(|k| k.trim().is_empty()) { - return Err(TuliproxError::ConfigPanelApi("panel_api: api_key is missing".to_string())); - } - if cfg.query_parameter.client_info.is_empty() { - return Err(TuliproxError::ConfigPanelApi( - "panel_api: query_parameter.client_info must be configured".to_string(), - )); - } - validate_client_info_params(&cfg.query_parameter.client_info)?; - if !cfg.query_parameter.client_new.is_empty() { - validate_client_new_params(&cfg.query_parameter.client_new)?; - } - if !cfg.query_parameter.client_renew.is_empty() { - validate_client_renew_params(&cfg.query_parameter.client_renew)?; - } - if !cfg.query_parameter.account_info.is_empty() { - validate_account_info_params(&cfg.query_parameter.account_info)?; - } - if !cfg.query_parameter.client_adult_content.is_empty() { - validate_client_adult_content_params(&cfg.query_parameter.client_adult_content)?; - } - let (min_val, max_val) = alias_pool_limit_values(cfg); - if let Some(PanelApiAliasPoolSizeValue::Number(value)) = min_val { - if *value == 0 { - return Err(TuliproxError::ConfigPanelApi( - "panel_api.alias_pool.size.min must be greater than 0".to_string(), - )); - } - } - if let Some(PanelApiAliasPoolSizeValue::Number(value)) = max_val { - if *value == 0 { - return Err(TuliproxError::ConfigPanelApi( - "panel_api.alias_pool.size.max must be greater than 0".to_string(), - )); - } - } - let min = min_val.and_then(PanelApiAliasPoolSizeValue::as_number); - let max = max_val.and_then(PanelApiAliasPoolSizeValue::as_number); - if let (Some(min), Some(max)) = (min, max) { - if min > max { - return Err(TuliproxError::ConfigPanelApi( - "panel_api.alias_pool.size.min must be <= panel_api.alias_pool.size.max".to_string(), - )); - } - } - if cfg.provisioning.probe_interval_sec == 0 { - return Err(TuliproxError::ConfigPanelApi( - "panel_api.provisioning.probe_interval_sec must be greater than 0".to_string(), - )); - } - if let Some(offset) = cfg.provisioning.offset.as_deref() { - let _secs = parse_panel_api_provisioning_offset_secs(offset)?; - } - Ok(()) -} - -fn resolve_query_params( - params: &[PanelApiQueryParam], - api_key: Option<&str>, - creds: Option<(&str, &str)>, -) -> Result, TuliproxError> { - let mut out = Vec::with_capacity(params.len()); - for p in params { - let key = p.key.trim(); - if key.is_empty() { - continue; - } - let mut value = p.value.trim().to_string(); - if value.eq_ignore_ascii_case("auto") { - if key.eq_ignore_ascii_case("api_key") { - let Some(k) = api_key.filter(|s| !s.trim().is_empty()) else { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api: query param {key} uses 'auto' but panel_api.api_key is missing" - ))); - }; - value = k.to_string(); - } else if key.eq_ignore_ascii_case("username") { - let Some((u, _)) = creds else { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api: query param {key} uses 'auto' but no account username is available" - ))); - }; - value = u.to_string(); - } else if key.eq_ignore_ascii_case("password") { - let Some((_, pw)) = creds else { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api: query param {key} uses 'auto' but no account password is available" - ))); - }; - value = pw.to_string(); - } - } - out.push((key.to_string(), value)); - } - Ok(out) -} - -fn build_panel_url(base_url: &str, query_params: &[(String, String)]) -> Result { - let mut url = Url::parse(base_url) - .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: invalid url {base_url}: {e}")))?; - { - let mut pairs = url.query_pairs_mut(); - for (k, v) in query_params { - pairs.append_pair(k, v); - } - } - Ok(url) -} - -fn sanitize_panel_api_json_for_log(value: &Value, sanitize_sensitive: bool) -> Value { - match value { - Value::Array(arr) => { - Value::Array(arr.iter().map(|v| sanitize_panel_api_json_for_log(v, sanitize_sensitive)).collect()) - } - Value::Object(obj) => { - let mut out = serde_json::Map::with_capacity(obj.len()); - for (k, v) in obj { - if sanitize_sensitive { - if k.eq_ignore_ascii_case("api_key") - || k.eq_ignore_ascii_case("apikey") - || k.eq_ignore_ascii_case("token") - { - out.insert(k.clone(), Value::String("***".to_string())); - continue; - } - if k.eq_ignore_ascii_case("username") || k.eq_ignore_ascii_case("password") { - out.insert(k.clone(), Value::String("***".to_string())); - continue; - } - } - if k.eq_ignore_ascii_case("url") { - if let Some(s) = v.as_str() { - out.insert(k.clone(), Value::String(sanitize_sensitive_info(s).into_owned())); - continue; - } - } - out.insert(k.clone(), sanitize_panel_api_json_for_log(v, sanitize_sensitive)); - } - Value::Object(out) - } - other => other.clone(), - } -} - -fn panel_api_retryable_status(status: StatusCode) -> bool { - status.is_server_error() - || matches!(status, StatusCode::TOO_MANY_REQUESTS | StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_EARLY) -} - -fn panel_api_retry_after_from_header_value(raw: &str) -> Option { - let trimmed = raw.trim(); - if trimmed.is_empty() { - return None; - } - if let Ok(secs) = trimmed.parse::() { - return Some(Duration::from_secs( - secs.clamp(PANEL_API_DEFAULT_RETRY_AFTER_SECS, PANEL_API_MAX_RETRY_AFTER_SECS), - )); - } - let retry_at = DateTime::parse_from_rfc2822(trimmed).ok()?.with_timezone(&Utc); - let now = Utc::now(); - let secs = retry_at.signed_duration_since(now).num_seconds(); - let secs = if secs <= 0 { - PANEL_API_DEFAULT_RETRY_AFTER_SECS - } else { - u64::try_from(secs).unwrap_or(PANEL_API_DEFAULT_RETRY_AFTER_SECS) - }; - Some(Duration::from_secs(secs.clamp(PANEL_API_DEFAULT_RETRY_AFTER_SECS, PANEL_API_MAX_RETRY_AFTER_SECS))) -} - -fn panel_api_retry_after_from_headers(headers: &HeaderMap) -> Duration { - headers - .get(header::RETRY_AFTER) - .and_then(|value| value.to_str().ok()) - .and_then(panel_api_retry_after_from_header_value) - .unwrap_or_else(|| Duration::from_secs(PANEL_API_DEFAULT_RETRY_AFTER_SECS)) -} - -async fn panel_get_json(app_state: &AppState, url: Url) -> Result { - let client = app_state.http_clients.default.load(); - let sanitized = sanitize_sensitive_info(url.as_str()); - for attempt in 0..PANEL_API_RETRY_ATTEMPTS { - debug_if_enabled!("panel_api request attempt {} of {}: {}", attempt + 1, PANEL_API_RETRY_ATTEMPTS, sanitized); - let resp = client - .get(url.clone()) - .timeout(Duration::from_secs(PANEL_API_REQUEST_TIMEOUT_SECS)) - .send() - .await - .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api request failed: {e}")))?; - let status = resp.status(); - if !status.is_success() { - if attempt + 1 < PANEL_API_RETRY_ATTEMPTS && panel_api_retryable_status(status) { - let cooldown = panel_api_retry_after_from_headers(resp.headers()); - warn!( - "panel_api request returned {}, retrying after {:.3}s: {}", - format_http_status(status), - cooldown.as_secs_f64(), - sanitized - ); - tokio::time::sleep(cooldown).await; - continue; - } - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api request failed (http {}): {}", - format_http_status(status), - sanitized - ))); - } - let body = resp - .text() - .await - .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api read response failed: {e}")))?; - let json: Value = serde_json::from_str(&body) - .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api invalid json (http {status}): {e}")))?; - let sanitize_sensitive = - app_state.app_config.config.load().log.as_ref().is_none_or(|l| l.sanitize_sensitive_info); - let json_for_log = sanitize_panel_api_json_for_log(&json, sanitize_sensitive); - if let Ok(json_str) = serde_json::to_string(&json_for_log) { - debug_if_enabled!( - "panel_api response (http {}): {}", - format_http_status(status), - sanitize_sensitive_info(&json_str) - ); - } - return Ok(json); - } - Err(TuliproxError::ConfigPanelApi(format!("panel_api request failed: {sanitized}"))) -} - -fn build_user_api_account_info_input_source( - input: &ConfigInput, - username: &str, - password: &str, -) -> Result { - let url = build_player_api_action_url(input.url.as_str(), username, password, "account_info").ok_or_else(|| { - TuliproxError::ConfigPanelApi(format!( - "panel_api: invalid user_api base_url: {}", - sanitize_sensitive_info(input.url.as_str()) - )) - })?; - - Ok(InputSource::from(input).with_url(url.to_string())) -} - -async fn user_api_get_json(app_state: &AppState, input_source: &InputSource) -> Result { - let client = app_state.http_clients.default.load(); - let url = Url::parse(input_source.url.as_str()).map_err(|e| { - TuliproxError::ConfigPanelApi(format!( - "panel_api user_api invalid url {}: {e}", - sanitize_sensitive_info(input_source.url.as_str()) - )) - })?; - debug_if_enabled!( - "panel_api user_api request {}", - sanitize_sensitive_info(&request::preview_request_diagnostics_for_logging(&url, input_source.get_provider())) - ); - - let config = app_state.app_config.config.load(); - let default_user_agent = config.default_user_agent.clone(); - let disabled_headers = config.get_disabled_headers(); - drop(config); - - let headers = request::get_request_headers( - Some(&input_source.headers), - None::<&HashMap>>, - disabled_headers.as_ref(), - default_user_agent.as_deref(), - ); - let resp = request::send_with_retry_and_provider( - &app_state.app_config, - &url, - input_source.get_provider(), - false, - |resolved_url| { - client.get(resolved_url.clone()).headers(headers.clone()).timeout(std::time::Duration::from_secs(30)) - }, - ) - .await - .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api request failed: {e}")))?; - let status = resp.status(); - let body = resp - .text() - .await - .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api read response failed: {e}")))?; - let json: Value = serde_json::from_str(&body) - .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api invalid json (http {status}): {e}")))?; - let sanitize_sensitive = app_state.app_config.config.load().log.as_ref().is_none_or(|l| l.sanitize_sensitive_info); - let json_for_log = sanitize_panel_api_json_for_log(&json, sanitize_sensitive); - if let Ok(json_str) = serde_json::to_string(&json_for_log) { - debug_if_enabled!( - "panel_api user_api response (http {}): {}", - format_http_status(status), - sanitize_sensitive_info(&json_str) - ); - } - Ok(json) -} - -async fn panel_client_new( - app_state: &AppState, - cfg: &PanelApiConfig, -) -> Result<(String, String, Option), TuliproxError> { - validate_client_new_params(&cfg.query_parameter.client_new)?; - let params = resolve_query_params(&cfg.query_parameter.client_new, cfg.api_key.as_deref(), None)?; - let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; - let json = panel_get_json(app_state, url).await?; - let Some(obj) = first_json_object(&json) else { - return Err(TuliproxError::ConfigPanelApi( - "panel_api: client_new response is not a JSON object/array".to_string(), - )); - }; - let status_ok = obj.get("status").is_some_and(parse_boolish); - if !status_ok { - return Err(TuliproxError::ConfigPanelApi("panel_api: client_new status=false".to_string())); - } - if let Some((u, p)) = extract_username_password_from_json(obj) { - return Ok((u, p, None)); - } - if let Some(url_str) = obj.get("url").and_then(|v| v.as_str()) { - if let (Some(u), Some(p)) = get_credentials_from_url_str(url_str) { - let base = get_base_url_from_str(url_str); - return Ok((u, p, base)); - } - } - Err(TuliproxError::ConfigPanelApi( - "panel_api: client_new response missing username/password (and no parsable url)".to_string(), - )) -} - -async fn panel_client_renew( - app_state: &AppState, - cfg: &PanelApiConfig, - username: &str, - password: &str, -) -> Result<(), TuliproxError> { - validate_client_renew_params(&cfg.query_parameter.client_renew)?; - let params = - resolve_query_params(&cfg.query_parameter.client_renew, cfg.api_key.as_deref(), Some((username, password)))?; - let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; - let json = panel_get_json(app_state, url).await?; - let Some(obj) = first_json_object(&json) else { - return Err(TuliproxError::ConfigPanelApi( - "panel_api: client_renew response is not a JSON object/array".to_string(), - )); - }; - let status_ok = obj.get("status").is_some_and(parse_boolish); - if !status_ok { - return Err(TuliproxError::ConfigPanelApi("panel_api: client_renew status=false".to_string())); - } - Ok(()) -} - -async fn panel_client_info_raw( - app_state: &AppState, - cfg: &PanelApiConfig, - username: &str, - password: &str, -) -> Result, TuliproxError> { - validate_client_info_params(&cfg.query_parameter.client_info)?; - let params = - resolve_query_params(&cfg.query_parameter.client_info, cfg.api_key.as_deref(), Some((username, password)))?; - let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; - let json = panel_get_json(app_state, url).await?; - let Some(obj) = first_json_object(&json) else { - return Err(TuliproxError::ConfigPanelApi( - "panel_api: client_info response is not a JSON object/array".to_string(), - )); - }; - let status_ok = obj.get("status").is_some_and(parse_boolish); - if !status_ok { - return Err(TuliproxError::ConfigPanelApi("panel_api: client_info status=false".to_string())); - } - let expire = obj.get("expire").and_then(|v| v.as_str()).unwrap_or_default().trim().to_string(); - if expire.is_empty() { - Ok(None) - } else { - Ok(Some(expire)) - } -} - -async fn panel_client_info( - app_state: &AppState, - cfg: &PanelApiConfig, - username: &str, - password: &str, - time_ctx: Option<&PanelApiTimeContext>, -) -> Result, TuliproxError> { - let expire = panel_client_info_raw(app_state, cfg, username, password).await?; - Ok(expire.as_deref().and_then(|value| normalize_panel_expire(value, time_ctx))) -} - -async fn fetch_root_user_api_info( - app_state: &Arc, - input: &ConfigInput, -) -> Result, TuliproxError> { - let Some((username, password)) = extract_account_creds_from_input(input) else { - return Ok(None); - }; - - let input_source = build_user_api_account_info_input_source(input, username.as_ref(), password.as_ref())?; - - let json = match user_api_get_json(app_state, &input_source).await { - Ok(json) => json, - Err(err) => { - debug_if_enabled!( - "panel_api user_api account_info failed for {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - return Ok(None); - } - }; - - let exp_date = json.get("user_info").and_then(|v| v.get("exp_date")).and_then(get_i64_from_serde_value); - let server_now_ts = json.get("server_info").and_then(|v| v.get("timestamp_now")).and_then(get_i64_from_serde_value); - let server_tz = json - .get("server_info") - .and_then(|v| v.get("timezone")) - .and_then(get_string_from_serde_value) - .and_then(|tz| tz.parse::().ok()); - - Ok(Some(UserApiAccountInfo { exp_date, server_now_ts, server_tz })) -} - -fn resolve_panel_expire_mode( - root_expire: Option, - panel_expire: Option<&str>, - server_tz: Option, -) -> PanelApiExpireMode { - let Some(root_expire) = root_expire else { - return PanelApiExpireMode::UtcString; - }; - let Some(panel_expire) = panel_expire else { - return PanelApiExpireMode::UtcString; - }; - let utc_ts = parse_panel_expire_utc(panel_expire); - let tz_ts = server_tz.and_then(|tz| parse_panel_expire_with_tz(panel_expire, tz)); - let Some(utc_ts) = utc_ts else { - return tz_ts.map_or(PanelApiExpireMode::UtcString, |_| PanelApiExpireMode::ServerTzString); - }; - let Some(tz_ts) = tz_ts else { - return PanelApiExpireMode::UtcString; - }; - let diff_utc = (utc_ts - root_expire).abs(); - let diff_tz = (tz_ts - root_expire).abs(); - let threshold = 120_i64; - match (diff_utc <= threshold, diff_tz <= threshold) { - (true, false) => PanelApiExpireMode::UtcString, - (false, true) => PanelApiExpireMode::ServerTzString, - _ => { - if diff_tz < diff_utc { - PanelApiExpireMode::ServerTzString - } else { - PanelApiExpireMode::UtcString - } - } - } -} - -fn apply_clock_skew(now: u64, skew_secs: i64) -> u64 { - if skew_secs == 0 { - return now; - } - if skew_secs.is_negative() { - now.saturating_sub(skew_secs.unsigned_abs()) - } else { - now.saturating_add(u64::try_from(skew_secs).unwrap_or(0)) - } -} - -fn panel_api_time_cache_path(app_state: &AppState) -> PathBuf { - let paths = app_state.app_config.paths.load(); - PathBuf::from(&paths.config_path).join("panel-api").join("panel_api_time_cache.json") -} - -async fn load_panel_api_time_cache(app_state: &AppState, cache_path: &Path) -> PanelApiTimeCache { - if let Some(parent) = cache_path.parent() { - if tokio::fs::create_dir_all(parent).await.is_err() { - return PanelApiTimeCache::default(); - } - } - let _lock = app_state.app_config.file_locks.read_lock(cache_path).await; - let Ok(content) = tokio::fs::read_to_string(cache_path).await else { - return PanelApiTimeCache::default(); - }; - serde_json::from_str(&content).unwrap_or_default() -} - -async fn persist_panel_api_time_cache( - app_state: &AppState, - cache_path: &Path, - cache: &PanelApiTimeCache, -) -> Result<(), TuliproxError> { - if let Some(parent) = cache_path.parent() { - tokio::fs::create_dir_all(parent) - .await - .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: failed to create cache dir: {e}")))?; - } - let content = - serde_json::to_string_pretty(cache).map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: {e}")))?; - let _lock = app_state.app_config.file_locks.write_lock(cache_path).await; - tokio::fs::write(cache_path, content) - .await - .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: failed to persist time cache: {e}")))?; - Ok(()) -} - -fn parse_cached_tz(tz: Option) -> Option { tz.and_then(|name| name.parse::().ok()) } - -async fn panel_account_info( - app_state: &AppState, - cfg: &PanelApiConfig, - creds: Option<(&str, &str)>, -) -> Result, TuliproxError> { - if cfg.query_parameter.account_info.is_empty() { - return Ok(None); - } - validate_account_info_params(&cfg.query_parameter.account_info)?; - let params = resolve_query_params(&cfg.query_parameter.account_info, cfg.api_key.as_deref(), creds)?; - let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; - let json = panel_get_json(app_state, url).await?; - let Some(obj) = first_json_object(&json) else { - return Err(TuliproxError::ConfigPanelApi( - "panel_api: account_info response is not a JSON object/array".to_string(), - )); - }; - let status_ok = obj.get("status").is_some_and(parse_boolish); - if !status_ok { - return Err(TuliproxError::ConfigPanelApi("panel_api: account_info status=false".to_string())); - } - let Some(credits) = obj.get("credits").and_then(extract_stringish) else { - return Err(TuliproxError::ConfigPanelApi("panel_api: account_info response missing credits".to_string())); - }; - Ok(Some(credits)) -} - -async fn panel_client_adult_content( - app_state: &AppState, - cfg: &PanelApiConfig, - creds: Option<(&str, &str)>, -) -> Result<(), TuliproxError> { - if cfg.query_parameter.client_adult_content.is_empty() { - return Ok(()); - } - validate_client_adult_content_params(&cfg.query_parameter.client_adult_content)?; - let params = resolve_query_params(&cfg.query_parameter.client_adult_content, cfg.api_key.as_deref(), creds)?; - let url = build_panel_url(cfg.url.as_ref(), ¶ms)?; - let json = panel_get_json(app_state, url).await?; - let Some(obj) = first_json_object(&json) else { - return Err(TuliproxError::ConfigPanelApi( - "panel_api: client_adult_content response is not a JSON object/array".to_string(), - )); - }; - let status_ok = obj.get("status").is_some_and(parse_boolish); - if !status_ok { - return Err(TuliproxError::ConfigPanelApi("panel_api: client_adult_content status=false".to_string())); - } - Ok(()) -} - -fn extract_account_creds_from_input(input: &ConfigInput) -> Option<(String, String)> { - if let (Some(u), Some(p)) = (input.username.as_deref(), input.password.as_deref()) { - if !u.trim().is_empty() && !p.trim().is_empty() { - return Some((u.to_string(), p.to_string())); - } - } - Url::parse(input.url.as_str()).ok().and_then(|u| { - let (uu, pp) = get_credentials_from_url(&u); - match (uu, pp) { - (Some(uu), Some(pp)) if !uu.trim().is_empty() && !pp.trim().is_empty() => Some((uu, pp)), - _ => None, - } - }) -} - -fn alias_pool_limit_values( - cfg: &PanelApiConfig, -) -> (Option<&PanelApiAliasPoolSizeValue>, Option<&PanelApiAliasPoolSizeValue>) { - let size = cfg.alias_pool.as_ref().and_then(|p| p.size.as_ref()); - let min = size.and_then(|s| s.min.as_ref()); - let max = size.and_then(|s| s.max.as_ref()); - (min, max) -} - -fn alias_pool_has_min(cfg: &PanelApiConfig) -> bool { - let (min, _) = alias_pool_limit_values(cfg); - min.is_some() -} - -fn resolve_alias_pool_limit_value(value: Option<&PanelApiAliasPoolSizeValue>, auto_value: Option) -> Option { - match value { - Some(PanelApiAliasPoolSizeValue::Number(v)) => Some(*v), - Some(PanelApiAliasPoolSizeValue::Auto(_)) => auto_value, - None => None, - } -} - -fn is_proxy_user_enabled(user: &ProxyUserCredentials) -> bool { - if let Some(status) = user.status { - if !matches!(status, ProxyUserStatus::Active | ProxyUserStatus::Trial) { - return false; - } - } - !is_input_expired(user.exp_date) -} - -fn find_input_target_names(app_state: &AppState, input_name: &Arc) -> Vec { - let sources = app_state.app_config.sources.load(); - for source in &sources.sources { - if source.inputs.iter().any(|name| name == input_name) { - let mut names: SmallVec<[String; 8]> = SmallVec::new(); - for target in &source.targets { - names.push(target.name.clone()); - } - return names.into_vec(); - } - } - Vec::new() -} - -fn count_enabled_proxy_users(app_state: &AppState, input_name: &Arc) -> usize { - let api_proxy_guard = app_state.app_config.api_proxy.load(); - let Some(api_proxy) = api_proxy_guard.as_ref() else { - return 0; - }; - let target_names = find_input_target_names(app_state, input_name); - if target_names.is_empty() { - return 0; - } - api_proxy - .user - .iter() - .filter(|target_user| target_names.iter().any(|target| target.eq_ignore_ascii_case(&target_user.target))) - .map(|target_user| target_user.credentials.iter().filter(|cred| is_proxy_user_enabled(cred)).count()) - .sum() -} - -fn resolve_alias_pool_auto_value(app_state: &AppState, input_name: &Arc) -> u16 { - let enabled_users = count_enabled_proxy_users(app_state, input_name); - u16::try_from(enabled_users).unwrap_or(u16::MAX) -} - -pub(crate) fn target_has_alias_pool_min(app_state: &AppState, target_name: &str) -> bool { - let sources = app_state.app_config.sources.load(); - for source in &sources.sources { - let target_match = source.targets.iter().any(|target| target.name.eq_ignore_ascii_case(target_name)); - if !target_match { - continue; - } - for input_name in &source.inputs { - let Some(input) = sources.get_input_by_name(input_name) else { - continue; - }; - if let Some(panel_cfg) = input.panel_api.as_ref() { - if !panel_cfg.enabled { - continue; - } - if alias_pool_has_min(panel_cfg) { - return true; - } - } - } - } - false -} - -fn resolve_alias_pool_limits( - app_state: &AppState, - input_name: &Arc, - cfg: &PanelApiConfig, -) -> Result<(Option, Option), TuliproxError> { - let (min_val, max_val) = alias_pool_limit_values(cfg); - if min_val.is_none() && max_val.is_none() { - return Ok((None, None)); - } - let min_auto = min_val.is_some_and(PanelApiAliasPoolSizeValue::is_auto); - let auto_value = min_auto.then(|| resolve_alias_pool_auto_value(app_state, input_name)); - let min = resolve_alias_pool_limit_value(min_val, auto_value); - let max = match max_val { - Some(PanelApiAliasPoolSizeValue::Number(value)) => Some(*value), - Some(PanelApiAliasPoolSizeValue::Auto(_)) | None => None, - }; - if let (Some(min), Some(max)) = (min, max) { - if min > max { - return Err(TuliproxError::ConfigPanelApi( - "panel_api.alias_pool.size.min must be <= panel_api.alias_pool.size.max".to_string(), - )); - } - } - Ok((min, max)) -} - -fn resolve_alias_pool_min(app_state: &AppState, input_name: &Arc, cfg: &PanelApiConfig) -> Option { - let (min_val, _) = alias_pool_limit_values(cfg); - let min_val = min_val?; - let auto_value = min_val.is_auto().then(|| resolve_alias_pool_auto_value(app_state, input_name)); - resolve_alias_pool_limit_value(Some(min_val), auto_value) -} - -fn alias_pool_remove_expired(cfg: &PanelApiConfig) -> bool { cfg.alias_pool.as_ref().is_some_and(|p| p.remove_expired) } - -fn collect_accounts(input: &ConfigInput) -> Vec { - let mut out = Vec::new(); - if let Some((u, p)) = extract_account_creds_from_input(input) { - out.push(AccountCredentials { name: input.name.clone(), username: u, password: p, exp_date: input.exp_date }); - } - if let Some(aliases) = input.aliases.as_ref() { - for a in aliases { - if let (Some(u), Some(p)) = (a.username.as_deref(), a.password.as_deref()) { - if !u.trim().is_empty() && !p.trim().is_empty() { - out.push(AccountCredentials { - name: a.name.clone(), - username: u.to_string(), - password: p.to_string(), - exp_date: a.exp_date, - }); - } - } - } - } - out -} - -fn compare_alias_exp_date_config(a: &ConfigInputAlias, b: &ConfigInputAlias) -> Ordering { - compare_named_exp_date(a.exp_date, a.name.as_ref(), b.exp_date, b.name.as_ref()) -} - -fn compare_account_exp_date(a: &AccountCredentials, b: &AccountCredentials) -> Ordering { - compare_named_exp_date(a.exp_date, a.name.as_ref(), b.exp_date, b.name.as_ref()) -} - -fn compare_named_exp_date(a_exp_date: Option, a_name: &str, b_exp_date: Option, b_name: &str) -> Ordering { - let a_ts = a_exp_date.unwrap_or(i64::MIN); - let b_ts = b_exp_date.unwrap_or(i64::MIN); - b_ts.cmp(&a_ts).then_with(|| a_name.cmp(b_name)) -} - -fn aliases_need_sort_config(aliases: &[ConfigInputAlias]) -> bool { - if aliases.len() < 2 { - return false; - } - aliases.windows(2).any(|pair| compare_alias_exp_date_config(&pair[0], &pair[1]) == Ordering::Greater) -} - -fn sort_account_aliases_keep_root_first(accounts: &mut Vec, root_name: &str) { - let root = accounts.iter().find(|acct| acct.name.as_ref() == root_name).cloned(); - let mut aliases: Vec = - accounts.iter().filter(|acct| acct.name.as_ref() != root_name).cloned().collect(); - aliases.sort_by(compare_account_exp_date); - accounts.clear(); - if let Some(root) = root { - accounts.push(root); - } - accounts.extend(aliases); -} - -fn is_account_valid(exp_date: Option) -> bool { exp_date.is_some() && !is_input_expired(exp_date) } - -fn count_valid_accounts(accounts: &[AccountCredentials]) -> usize { - accounts.iter().filter(|acct| is_account_valid(acct.exp_date)).count() -} - -fn root_counts_towards_pool(accounts: &[AccountCredentials], input_name: &Arc) -> bool { - accounts.iter().find(|acct| &acct.name == input_name).is_some_and(|acct| is_account_valid(acct.exp_date)) -} - -fn count_valid_accounts_at(accounts: &[AccountCredentials], now: u64) -> usize { - accounts.iter().filter(|acct| acct.exp_date.is_some() && !is_input_expired_at(acct.exp_date, now)).count() -} - -fn count_valid_alias_accounts_at(accounts: &[AccountCredentials], input_name: &Arc, now: u64) -> usize { - accounts - .iter() - .filter(|acct| &acct.name != input_name && acct.exp_date.is_some() && !is_input_expired_at(acct.exp_date, now)) - .count() -} - -fn root_counts_towards_pool_at(accounts: &[AccountCredentials], input_name: &Arc, now: u64) -> bool { - accounts - .iter() - .find(|acct| &acct.name == input_name) - .is_some_and(|acct| acct.exp_date.is_some() && !is_input_expired_at(acct.exp_date, now)) -} - -fn refresh_internal_sources(app_state: &Arc) { - app_state.active_provider.update_config(&app_state.app_config); -} - -fn should_reload_sources_after_internal_write(app_state: &AppState) -> bool { - !app_state.app_config.config.load().config_hot_reload -} - -fn resolve_batch_alias_path(batch_url: Option<&str>) -> Result, TuliproxError> { - let Some(batch_url) = batch_url.filter(|url| !url.trim().is_empty()) else { - return Ok(None); - }; - get_csv_file_path(batch_url).map(Some).map_err(|err| TuliproxError::ConfigInput(format!("{err}"))) -} - -fn require_batch_alias_path(input: &ConfigInput) -> Result { - resolve_batch_alias_path(input.t_batch_url.as_deref())?.ok_or_else(|| { - TuliproxError::ConfigInput(format!("batch input '{}' does not define a CSV alias path", input.name)) - }) -} - -pub(crate) fn is_alias_pool_max_reached(app_state: &AppState, input: &ConfigInput) -> bool { - let Some(panel_cfg) = input.panel_api.as_ref() else { - return false; - }; - if !panel_cfg.enabled { - return false; - } - if panel_cfg.url.trim().is_empty() { - return false; - } - if validate_panel_api_config(panel_cfg).is_err() { - return false; - } - let Ok((_, max_pool)) = resolve_alias_pool_limits(app_state, &input.name, panel_cfg) else { - return false; - }; - if let Some(max_pool) = max_pool { - let valid_count = count_valid_accounts(&collect_accounts(input)); - if valid_count >= max_pool as usize { - debug_if_enabled!( - "panel_api: alias_pool.size.max reached for input {} (valid_accounts={}, max={})", - sanitize_sensitive_info(&input.name), - valid_count, - max_pool - ); - return true; - } - } - false -} - -pub(crate) fn can_provision_on_exhausted(app_state: &AppState, input: &ConfigInput) -> bool { - let Some(panel_cfg) = input.panel_api.as_ref() else { - return false; - }; - if !panel_cfg.enabled { - return false; - } - if panel_cfg.url.trim().is_empty() { - return false; - } - if let Err(err) = validate_panel_api_config(panel_cfg) { - debug_if_enabled!("panel_api config invalid: {}", sanitize_sensitive_info(&err.to_string())); - return false; - } - !is_alias_pool_max_reached(app_state, input) -} - -pub(crate) fn find_input_by_provider_name(app_state: &AppState, provider_name: &str) -> Option> { - let sources = app_state.app_config.sources.load(); - for input in &sources.inputs { - if &*input.name == provider_name { - return Some(Arc::clone(input)); - } - if input.aliases.as_ref().is_some_and(|aliases| aliases.iter().any(|alias| &*alias.name == provider_name)) { - return Some(Arc::clone(input)); - } - } - None -} - #[derive(Debug, Clone)] pub(crate) enum PanelApiProvisionOutcome { Renewed, Created, } -impl PanelApiProvisionOutcome { - pub(crate) fn kind_label(&self) -> &'static str { - match self { - Self::Renewed => "client_renew", - Self::Created => "client_new", - } - } -} - -#[allow(clippy::too_many_lines)] -#[allow(clippy::too_many_arguments)] -async fn try_renew_expired_account( - app_state: &Arc, - input: &ConfigInput, - panel_cfg: &PanelApiConfig, - is_batch: bool, - sources_path: &Path, - treat_missing_exp_date_as_expired: bool, - include_root: bool, - optional: PanelApiOptionalFlagsSet, -) -> Option { - if !optional.contains(PanelApiOptionalFlags::ClientRenew) { - return None; - } - let adult_enabled = optional.contains(PanelApiOptionalFlags::AdultContent); - let mut candidates = collect_accounts(input); - if !include_root { - candidates.retain(|acct| acct.name != input.name); - } - for acct in &mut candidates { - if treat_missing_exp_date_as_expired && acct.exp_date.is_none() { - acct.exp_date = - panel_client_info(app_state, panel_cfg, acct.username.as_str(), acct.password.as_str(), None) - .await - .ok() - .flatten(); - } - } - candidates.sort_by_key(|a| a.exp_date.unwrap_or(i64::MAX)); - - for acct in &candidates { - // Only attempt renew/new when the account is *known* to be expired. - // If exp_date is missing (even after an optional client_info refresh), we skip renewal. - let expired = is_input_expired(acct.exp_date); - if !expired { - continue; - } - match panel_client_renew(app_state, panel_cfg, acct.username.as_str(), acct.password.as_str()).await { - Ok(()) => { - if adult_enabled { - if let Err(err) = panel_client_adult_content( - app_state, - panel_cfg, - Some((acct.username.as_str(), acct.password.as_str())), - ) - .await - { - debug_if_enabled!( - "panel_api client_adult_content failed for {}: {}", - sanitize_sensitive_info(&acct.name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - let refreshed_exp = - panel_client_info(app_state, panel_cfg, acct.username.as_str(), acct.password.as_str(), None) - .await - .ok() - .flatten(); - - if let Some(new_exp) = refreshed_exp.or(acct.exp_date) { - if is_batch { - match require_batch_alias_path(input) { - Ok(csv_path) => { - let csv_lock = app_state.app_config.file_locks.write_lock(&csv_path).await; - if let Err(err) = csv_patch_batch_update_exp_date( - app_state, - &csv_lock, - input.input_type, - &csv_path, - &acct.name, - &acct.username, - &acct.password, - new_exp, - ) - .await - { - debug_if_enabled!("panel_api failed to persist renew exp_date to csv: {}", err); - } - if let Err(err) = csv_patch_batch_sort_by_exp_date( - app_state, - &csv_lock, - input.input_type, - &csv_path, - AliasExpDateSortOrder::NewestFirst, - ) - .await - { - debug_if_enabled!("panel_api failed to sort csv accounts after renew: {}", err); - } - } - Err(err) => debug_if_enabled!("panel_api cannot resolve batch csv path: {}", err), - } - } else { - let patches = [ - SourcesYmlPatch::UpdatePanelAccountExpiry { - input_name: input.name.clone(), - account_name: Arc::clone(&acct.name), - exp_date: new_exp, - }, - SourcesYmlPatch::SortAliases { - input_name: input.name.clone(), - order: AliasExpDateSortOrder::NewestFirst, - }, - ]; - if let Err(err) = crate::api::source_yml_patch::execute_account_source_patches( - app_state, - sources_path, - &patches, - ) - .await - { - debug_if_enabled!("panel_api failed to persist renew exp_date to source.yml: {}", err); - } - } - } - - if should_reload_sources_after_internal_write(app_state.as_ref()) { - refresh_internal_sources(app_state); - } - return Some(PanelApiProvisionOutcome::Renewed); - } - Err(err) => { - debug_if_enabled!( - "panel_api client_renew failed for {}: {}", - sanitize_sensitive_info(&acct.name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - } - None -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -async fn try_refresh_root_account_on_exhausted( - app_state: &Arc, - input: &ConfigInput, - panel_cfg: &PanelApiConfig, - is_batch: bool, - sources_path: &Path, - optional: PanelApiOptionalFlagsSet, -) -> Option { - let renew_enabled = optional.contains(PanelApiOptionalFlags::ClientRenew); - let new_enabled = optional.contains(PanelApiOptionalFlags::ClientNew); - if !renew_enabled && !new_enabled { - return None; - } - - let now = get_current_timestamp(); - let offset_secs = panel_cfg - .provisioning - .offset - .as_deref() - .and_then(|value| parse_panel_api_provisioning_offset_secs(value).ok()) - .unwrap_or(0); - let root_exp_missing = input.exp_date.is_none(); - let root_expired = is_input_expired_at(input.exp_date, now); - let root_expiring = is_expiring_with_offset_at(input.exp_date, offset_secs, now); - if !root_exp_missing && !root_expired && !root_expiring { - return None; - } - - let old_username = input.username.clone().unwrap_or_default(); - let old_password = input.password.clone().unwrap_or_default(); - let adult_enabled = optional.contains(PanelApiOptionalFlags::AdultContent); - - let (outcome, active_username, active_password, credentials_changed) = - if renew_enabled && !old_username.is_empty() && !old_password.is_empty() { - match panel_client_renew(app_state, panel_cfg, old_username.as_str(), old_password.as_str()).await { - Ok(()) => (PanelApiProvisionOutcome::Renewed, old_username.clone(), old_password.clone(), false), - Err(err) => { - debug_if_enabled!( - "panel_api client_renew failed for root {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - if !new_enabled { - return None; - } - match panel_client_new(app_state, panel_cfg).await { - Ok((username, password, _base_url_from_resp)) => { - (PanelApiProvisionOutcome::Created, username, password, true) - } - Err(err) => { - debug_if_enabled!( - "panel_api client_new failed for root {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - return None; - } - } - } - } - } else if new_enabled { - match panel_client_new(app_state, panel_cfg).await { - Ok((username, password, _base_url_from_resp)) => { - (PanelApiProvisionOutcome::Created, username, password, true) - } - Err(err) => { - debug_if_enabled!( - "panel_api client_new failed for root {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - return None; - } - } - } else { - return None; - }; - - if adult_enabled { - if let Err(err) = - panel_client_adult_content(app_state, panel_cfg, Some((active_username.as_str(), active_password.as_str()))) - .await - { - debug_if_enabled!( - "panel_api client_adult_content failed for root {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - - if !wait_for_panel_api_account_ready( - app_state, - input, - panel_cfg, - input.name.as_ref(), - active_username.as_str(), - active_password.as_str(), - ) - .await - { - debug_if_enabled!( - "panel_api root account not ready after probe/cooldown for {}", - sanitize_sensitive_info(&input.name) - ); - return None; - } - - let refreshed_exp_date = - panel_client_info(app_state, panel_cfg, active_username.as_str(), active_password.as_str(), None) - .await - .ok() - .flatten(); - let exp_date = if credentials_changed { refreshed_exp_date } else { refreshed_exp_date.or(input.exp_date) }; - - if is_batch { - let Ok(csv_path) = require_batch_alias_path(input) else { - return None; - }; - let csv_lock = app_state.app_config.file_locks.write_lock(&csv_path).await; - let result = if credentials_changed { - csv_patch_batch_update_credentials( - app_state, - &csv_lock, - input.input_type, - &csv_path, - &input.name, - old_username.as_str(), - old_password.as_str(), - active_username.as_str(), - active_password.as_str(), - exp_date, - ) - .await - } else if let Some(exp_date) = exp_date { - csv_patch_batch_update_exp_date( - app_state, - &csv_lock, - input.input_type, - &csv_path, - &input.name, - active_username.as_str(), - active_password.as_str(), - exp_date, - ) - .await - } else { - Ok(()) - }; - if let Err(err) = result { - debug_if_enabled!("panel_api failed to persist root provisioning to csv: {}", err); - return None; - } - if let Err(err) = csv_patch_batch_sort_by_exp_date( - app_state, - &csv_lock, - input.input_type, - &csv_path, - AliasExpDateSortOrder::NewestFirst, - ) - .await - { - debug_if_enabled!("panel_api failed to sort csv accounts after root provisioning: {}", err); - return None; - } - } else { - let patch = SourcesYmlPatch::PersistProvisionedAccount { - input_name: input.name.clone(), - username: active_username, - password: active_password, - exp_date, - }; - if let Err(err) = - crate::api::source_yml_patch::execute_account_source_patches(app_state, sources_path, &[patch]).await - { - debug_if_enabled!("panel_api failed to persist root provisioning to source.yml: {}", err); - return None; - } - } - - if should_reload_sources_after_internal_write(app_state.as_ref()) { - refresh_internal_sources(app_state); - } - - Some(outcome) -} - -#[allow(clippy::too_many_lines)] -async fn try_create_new_account( - app_state: &Arc, - input: &ConfigInput, - panel_cfg: &PanelApiConfig, - is_batch: bool, - sources_path: &Path, - optional: PanelApiOptionalFlagsSet, -) -> Option { - if !optional.contains(PanelApiOptionalFlags::ClientNew) { - return None; - } - let adult_enabled = optional.contains(PanelApiOptionalFlags::AdultContent); - match panel_client_new(app_state, panel_cfg).await { - Ok((username, password, base_url_from_resp)) => { - let base_url = resolve_provisioned_account_base_url( - input.url.as_str(), - base_url_from_resp.as_deref(), - &username, - &password, - ); - - let mut existing_names: Vec> = vec![input.name.clone()]; - if let Some(aliases) = input.aliases.as_ref() { - existing_names.extend(aliases.iter().map(|a| a.name.clone())); - } - let alias_name = derive_unique_alias_name(&existing_names, &input.name, &username); - - if adult_enabled { - if let Err(err) = panel_client_adult_content(app_state, panel_cfg, Some((&username, &password))).await { - debug_if_enabled!( - "panel_api client_adult_content failed for {}: {}", - sanitize_sensitive_info(&alias_name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - - let exp_date = panel_client_info(app_state, panel_cfg, &username, &password, None).await.ok().flatten(); - - if !wait_for_panel_api_account_ready( - app_state, - input, - panel_cfg, - alias_name.as_ref(), - username.as_str(), - password.as_str(), - ) - .await - { - debug_if_enabled!( - "panel_api client_new account not ready after probe/cooldown for {}", - sanitize_sensitive_info(alias_name.as_ref()) - ); - return None; - } - - if is_batch { - match require_batch_alias_path(input) { - Ok(csv_path) => { - let batch_type = if input.input_type == InputType::Xtream { - InputType::XtreamBatch - } else { - InputType::M3uBatch - }; - let csv_lock = app_state.app_config.file_locks.write_lock(&csv_path).await; - if let Err(err) = csv_patch_batch_append( - app_state, - &csv_lock, - &csv_path, - batch_type, - &alias_name, - &base_url, - &username, - &password, - exp_date, - ) - .await - { - warn!("panel_api failed to append new account to csv: {err}"); - return None; - } - if let Err(err) = csv_patch_batch_sort_by_exp_date( - app_state, - &csv_lock, - batch_type, - &csv_path, - AliasExpDateSortOrder::NewestFirst, - ) - .await - { - warn!("panel_api failed to sort csv accounts after append: {err}"); - return None; - } - } - Err(err) => { - warn!( - "panel_api cannot resolve batch csv path for {}: {}", - sanitize_sensitive_info(input.name.as_ref()), - err, - ); - return None; - } - } - } else { - let patches = [ - SourcesYmlPatch::AddAlias { - input_name: input.name.clone(), - alias_name: Arc::clone(&alias_name), - base_url, - username, - password, - exp_date, - }, - SourcesYmlPatch::SortAliases { - input_name: input.name.clone(), - order: AliasExpDateSortOrder::NewestFirst, - }, - ]; - if let Err(err) = - crate::api::source_yml_patch::execute_account_source_patches(app_state, sources_path, &patches) - .await - { - warn!("panel_api failed to persist new alias to source.yml: {err}"); - return None; - } - } - - if should_reload_sources_after_internal_write(app_state.as_ref()) { - refresh_internal_sources(app_state); - } - Some(PanelApiProvisionOutcome::Created) - } - Err(err) => { - debug_if_enabled!("panel_api client_new failed: {}", sanitize_sensitive_info(&err.to_string())); - None - } - } -} - -pub async fn try_provision_account_on_exhausted( - app_state: &Arc, - input_name: &Arc, -) -> Option { - let _input_lock = app_state.app_config.file_locks.write_lock_str(format!("panel_api:{input_name}").as_str()).await; - let current_input = find_input_by_provider_name(app_state.as_ref(), input_name.as_ref()); - let Some(input) = current_input.as_deref() else { - debug_if_enabled!( - "panel_api: skipped (input no longer exists) for input {}", - sanitize_sensitive_info(input_name.as_ref()) - ); - return None; - }; - - let Some(panel_cfg) = input.panel_api.as_ref() else { - debug_if_enabled!( - "panel_api: skipped (no panel_api config) for input {}", - sanitize_sensitive_info(&input.name) - ); - return None; - }; - if !panel_cfg.enabled { - debug_if_enabled!( - "panel_api: skipped (panel_api.enabled false) for input {}", - sanitize_sensitive_info(&input.name) - ); - return None; - } - if panel_cfg.url.trim().is_empty() { - debug_if_enabled!( - "panel_api: skipped (panel_api.url empty) for input {}", - sanitize_sensitive_info(&input.name) - ); - return None; - } - - if let Err(err) = validate_panel_api_config(panel_cfg) { - debug_if_enabled!("panel_api config invalid: {}", sanitize_sensitive_info(&err.to_string())); - return None; - } - let optional = resolve_panel_api_optional_flags(panel_cfg, &input.name); - debug_if_enabled!( - "panel_api: exhausted -> provisioning for input {} (aliases={})", - sanitize_sensitive_info(&input.name), - input.aliases.as_ref().map_or(0, Vec::len) - ); - - let is_batch = input.t_batch_url.as_ref().is_some_and(|u| !u.trim().is_empty()); - let sources_file_path = app_state.app_config.paths.load().sources_file_path.clone(); - let sources_path = PathBuf::from(&sources_file_path); - - if let Some(outcome) = - try_refresh_root_account_on_exhausted(app_state, input, panel_cfg, is_batch, sources_path.as_path(), optional) - .await - { - debug_if_enabled!( - "panel_api: provisioning succeeded via root {} for input {}", - outcome.kind_label(), - sanitize_sensitive_info(&input.name) - ); - return Some(outcome); - } - - if is_alias_pool_max_reached(app_state, input) { - return None; - } - - if let Some(outcome) = - try_renew_expired_account(app_state, input, panel_cfg, is_batch, sources_path.as_path(), true, false, optional) - .await - { - debug_if_enabled!( - "panel_api: provisioning succeeded via client_renew for input {}", - sanitize_sensitive_info(&input.name) - ); - return Some(outcome); - } - let created = try_create_new_account(app_state, input, panel_cfg, is_batch, sources_path.as_path(), optional).await; - debug_if_enabled!( - "panel_api: provisioning via client_new for input {} => {}", - sanitize_sensitive_info(&input.name), - if created.is_some() { "success" } else { "failed" } - ); - created -} - -#[allow(clippy::too_many_lines, clippy::too_many_arguments)] -async fn ensure_alias_pool_min( - app_state: &Arc, - input: &ConfigInput, - panel_cfg: &PanelApiConfig, - accounts: &mut Vec, - min_pool: u16, - csv_path: Option<&Path>, - sources_yml_patches: &mut Vec, - time_ctx: Option<&PanelApiTimeContext>, - effective_now: u64, - optional: PanelApiOptionalFlagsSet, -) -> (bool, u16) { - if min_pool == 0 { - return (false, 0); - } - - let renew_enabled = optional.contains(PanelApiOptionalFlags::ClientRenew); - let new_enabled = optional.contains(PanelApiOptionalFlags::ClientNew); - let adult_enabled = optional.contains(PanelApiOptionalFlags::AdultContent); - if !renew_enabled && !new_enabled { - return (false, 0); - } - - let mut changed = false; - let mut provisioned = 0_u16; - let max_pool = resolve_alias_pool_limits(app_state.as_ref(), &input.name, panel_cfg).ok().and_then(|(_, max)| max); - let mut existing_names: HashSet> = accounts.iter().map(|a| a.name.clone()).collect(); - let max_attempts = usize::from(min_pool).saturating_add(10); - for _ in 0..max_attempts { - let current_valid = count_valid_alias_accounts_at(accounts, &input.name, effective_now); - if current_valid >= usize::from(min_pool) { - break; - } - if let Some(max_pool) = max_pool { - if current_valid >= usize::from(max_pool) { - break; - } - } - - let expired_index = accounts - .iter() - .enumerate() - .filter(|(_, acct)| acct.name != input.name && is_input_expired_at(acct.exp_date, effective_now)) - .min_by_key(|(_, acct)| acct.exp_date.unwrap_or(i64::MAX)) - .map(|(idx, _)| idx); - - if let Some(idx) = expired_index { - let acct = accounts.get(idx).cloned(); - let Some(acct) = acct else { - break; - }; - if renew_enabled { - match panel_client_renew(app_state.as_ref(), panel_cfg, acct.username.as_str(), acct.password.as_str()) - .await - { - Ok(()) => { - provisioned = provisioned.saturating_add(1); - if adult_enabled { - if let Err(err) = panel_client_adult_content( - app_state.as_ref(), - panel_cfg, - Some((acct.username.as_str(), acct.password.as_str())), - ) - .await - { - debug_if_enabled!( - "panel_api client_adult_content failed for {}: {}", - sanitize_sensitive_info(&acct.name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - - let refreshed_exp = panel_client_info( - app_state.as_ref(), - panel_cfg, - acct.username.as_str(), - acct.password.as_str(), - time_ctx, - ) - .await - .ok() - .flatten() - .or(acct.exp_date); - - if let Some(new_exp) = refreshed_exp { - if let Some(acct_mut) = accounts.get_mut(idx) { - acct_mut.exp_date = Some(new_exp); - } - if let Some(csv_path) = csv_path { - let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - if let Err(err) = csv_patch_batch_update_exp_date( - app_state, - &csv_lock, - input.input_type, - csv_path, - &acct.name, - acct.username.as_str(), - acct.password.as_str(), - new_exp, - ) - .await - { - debug_if_enabled!("panel_api failed to persist renew exp_date to csv: {}", err); - } else { - changed = true; - } - } else { - sources_yml_patches.push(SourcesYmlPatch::UpdatePanelAccountExpiry { - input_name: input.name.clone(), - account_name: acct.name.clone(), - exp_date: new_exp, - }); - changed = true; - } - } - continue; - } - Err(err) => { - debug_if_enabled!( - "panel_api client_renew failed for {}: {}", - sanitize_sensitive_info(&acct.name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - } - } - - if !new_enabled { - break; - } - match panel_client_new(app_state.as_ref(), panel_cfg).await { - Ok((username, password, base_url_from_resp)) => { - let base_url = resolve_provisioned_account_base_url( - input.url.as_str(), - base_url_from_resp.as_deref(), - &username, - &password, - ); - - let alias_name = derive_unique_alias_name_set(&existing_names, &input.name, &username); - existing_names.insert(alias_name.clone().into()); - - if adult_enabled { - if let Err(err) = - panel_client_adult_content(app_state.as_ref(), panel_cfg, Some((&username, &password))).await - { - debug_if_enabled!( - "panel_api client_adult_content failed for {}: {}", - sanitize_sensitive_info(&alias_name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - - let exp_date = panel_client_info(app_state.as_ref(), panel_cfg, &username, &password, time_ctx) - .await - .ok() - .flatten(); - - accounts.push(AccountCredentials { - name: alias_name.clone().into(), - username: username.clone(), - password: password.clone(), - exp_date, - }); - provisioned = provisioned.saturating_add(1); - - if let Some(csv_path) = csv_path { - let batch_type = if input.input_type == InputType::Xtream { - InputType::XtreamBatch - } else if input.input_type == InputType::M3u { - InputType::M3uBatch - } else { - input.input_type - }; - let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - if let Err(err) = csv_patch_batch_append( - app_state, - &csv_lock, - csv_path, - batch_type, - &alias_name, - &base_url, - &username, - &password, - exp_date, - ) - .await - { - warn!("panel_api failed to append new account to csv: {err}"); - break; - } - changed = true; - } else { - sources_yml_patches.push(SourcesYmlPatch::AddAlias { - input_name: input.name.clone(), - alias_name: alias_name.into(), - base_url, - username, - password, - exp_date, - }); - changed = true; - } - } - Err(err) => { - debug_if_enabled!("panel_api client_new failed: {}", sanitize_sensitive_info(&err.to_string())); - break; - } - } - } - - (changed, provisioned) -} - -#[allow(clippy::too_many_lines)] -async fn sync_panel_api_for_input_on_boot( - app_state: &Arc, - input: &Arc, - sources_path: &Path, -) -> bool { - let Some(panel_cfg) = input.panel_api.as_ref() else { - return false; - }; - if !panel_cfg.enabled || panel_cfg.url.trim().is_empty() { - return false; - } - - if let Err(err) = validate_panel_api_config(panel_cfg) { - debug_if_enabled!( - "panel_api boot sync skipped for {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - return false; - } - let optional = resolve_panel_api_optional_flags(panel_cfg, &input.name); - - let input_name = &input.name; - let _input_lock = app_state.app_config.file_locks.write_lock_str(format!("panel_api:{input_name}").as_str()).await; - - let mut any_change = false; - let csv_path = match resolve_batch_alias_path(input.t_batch_url.as_deref()) { - Ok(path) => path, - Err(err) => { - warn!( - "panel_api boot sync skipped alias mutations for batch input {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - return false; - } - }; - let mut sources_yml_patches: Vec = Vec::new(); - let mut pending_sources_yml = false; - let mut source_yml_sort_aliases_requested = false; - - let mut accounts = collect_accounts(input.as_ref()); - if panel_cfg.alias_pool.is_some() { - sort_account_aliases_keep_root_first(&mut accounts, &input.name); - } - let mut existing_names: HashSet> = accounts.iter().map(|a| a.name.clone()).collect(); - let mut newly_created_accounts: Vec = Vec::new(); - let mut time_ctx: Option = None; - let mut effective_now = get_current_timestamp(); - - let cache_path = panel_api_time_cache_path(app_state.as_ref()); - let mut time_cache = load_panel_api_time_cache(app_state.as_ref(), &cache_path).await; - let cached_entry = time_cache.inputs.get(input.name.as_ref()).cloned(); - - if panel_cfg.alias_pool.is_some() - && csv_path.is_none() - && input.aliases.as_ref().is_some_and(|aliases| aliases_need_sort_config(aliases)) - { - sources_yml_patches.push(SourcesYmlPatch::SortAliases { - input_name: input.name.clone(), - order: AliasExpDateSortOrder::NewestFirst, - }); - pending_sources_yml = true; - source_yml_sort_aliases_requested = true; - } - - if let Some((root_username, root_password)) = extract_account_creds_from_input(input.as_ref()) { - let user_info = fetch_root_user_api_info(app_state, input.as_ref()).await; - let (root_exp_date, server_tz, skew_secs) = if let Ok(Some(info)) = user_info { - let local_now = i64::try_from(get_current_timestamp()).unwrap_or(0); - let skew_secs = info.server_now_ts.unwrap_or(local_now) - local_now; - (info.exp_date, info.server_tz, Some(skew_secs)) - } else { - (None, None, None) - }; - - let panel_expire_raw = - match panel_client_info_raw(app_state.as_ref(), panel_cfg, root_username.as_str(), root_password.as_str()) - .await - { - Ok(expire) => expire, - Err(err) => { - debug_if_enabled!( - "panel_api root client_info (raw) failed for {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - None - } - }; - - let mut expire_mode = resolve_panel_expire_mode(root_exp_date, panel_expire_raw.as_deref(), server_tz); - let mut server_tz = server_tz; - let mut skew_secs = skew_secs.or_else(|| cached_entry.as_ref().and_then(|e| e.skew_secs)); - - if root_exp_date.is_none() { - if let Some(cached) = cached_entry.as_ref() { - expire_mode = cached.expire_mode; - if server_tz.is_none() { - server_tz = parse_cached_tz(cached.server_tz.clone()); - } - if skew_secs.is_none() { - skew_secs = cached.skew_secs; - } - } - } - - time_ctx = Some(PanelApiTimeContext { expire_mode, server_tz }); - effective_now = apply_clock_skew(get_current_timestamp(), skew_secs.unwrap_or_default()); - - time_cache.inputs.insert( - input.name.to_string(), - PanelApiTimeCacheEntry { expire_mode, server_tz: server_tz.map(|tz| tz.name().to_string()), skew_secs }, - ); - if let Err(err) = persist_panel_api_time_cache(app_state.as_ref(), &cache_path, &time_cache).await { - debug_if_enabled!( - "panel_api failed to persist time cache for {}: {}", - sanitize_sensitive_info(&input.name), - err - ); - } - - let server_tz_name = server_tz.as_ref().map_or("none", |tz| Tz::name(*tz)); - debug_if_enabled!( - "panel_api time context for input {}: expire_mode={:?}, tz={}, skew_secs={}", - sanitize_sensitive_info(&input.name), - expire_mode, - server_tz_name, - skew_secs.unwrap_or_default() - ); - } else if let Some(cached) = cached_entry.as_ref() { - let server_tz = parse_cached_tz(cached.server_tz.clone()); - time_ctx = Some(PanelApiTimeContext { expire_mode: cached.expire_mode, server_tz }); - effective_now = apply_clock_skew(get_current_timestamp(), cached.skew_secs.unwrap_or_default()); - let server_tz_name = server_tz.as_ref().map_or("none", |tz| Tz::name(*tz)); - debug_if_enabled!( - "panel_api time context fallback for input {}: expire_mode={:?}, tz={}, skew_secs={}", - sanitize_sensitive_info(&input.name), - cached.expire_mode, - server_tz_name, - cached.skew_secs.unwrap_or_default() - ); - } else { - debug_if_enabled!( - "panel_api time context skipped for input {}: missing root credentials", - sanitize_sensitive_info(&input.name) - ); - } - - for acct in &mut accounts { - let new_exp = - match panel_client_info(app_state.as_ref(), panel_cfg, &acct.username, &acct.password, time_ctx.as_ref()) - .await - { - Ok(v) => v, - Err(err) => { - debug_if_enabled!( - "panel_api client_info failed for {}: {}", - sanitize_sensitive_info(&acct.name), - sanitize_sensitive_info(&err.to_string()) - ); - None - } - }; - let Some(new_exp) = new_exp else { - continue; - }; - if acct.exp_date == Some(new_exp) { - continue; - } - - if let Some(csv_path) = csv_path.as_ref() { - let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - if let Err(err) = csv_patch_batch_update_exp_date( - app_state, - &csv_lock, - input.input_type, - csv_path, - &acct.name, - &acct.username, - &acct.password, - new_exp, - ) - .await - { - debug_if_enabled!("panel_api boot sync failed to persist exp_date to csv: {}", err); - continue; - } - any_change = true; - } else { - sources_yml_patches.push(SourcesYmlPatch::UpdatePanelAccountExpiry { - input_name: input.name.clone(), - account_name: acct.name.clone(), - exp_date: new_exp, - }); - pending_sources_yml = true; - } - acct.exp_date = Some(new_exp); - } - - let offset_secs = panel_cfg - .provisioning - .offset - .as_deref() - .and_then(|v| parse_panel_api_provisioning_offset_secs(v).ok()) - .unwrap_or(0); - - let min_pool = resolve_alias_pool_min(app_state.as_ref(), &input.name, panel_cfg); - let renew_enabled = optional.contains(PanelApiOptionalFlags::ClientRenew); - let new_enabled = optional.contains(PanelApiOptionalFlags::ClientNew); - let adult_enabled = optional.contains(PanelApiOptionalFlags::AdultContent); - let provisioning_enabled = renew_enabled || new_enabled; - // Refresh/provision credentials on boot/update. - // Root is handled first (may affect desired_aliases and avoids over-provisioning). - let mut provisioned_root = 0_u16; - let mut provisioned_aliases = 0_u16; - - if let Some(root_idx) = accounts.iter().position(|a| a.name == input.name) { - let now = effective_now; - let offset_deadline = now.saturating_add(offset_secs); - let root_exp_date = accounts[root_idx].exp_date; - let root_exp_missing = root_exp_date.is_none(); - let root_expired = match root_exp_date { - Some(ts) => u64::try_from(ts).map_or(true, |exp_ts| exp_ts <= now), - None => false, - }; - let root_expiring = match root_exp_date { - Some(ts) => u64::try_from(ts).map_or(true, |exp_ts| exp_ts > now && exp_ts <= offset_deadline), - None => false, - }; - let should_refresh_root = root_exp_missing || root_expired || root_expiring; - - let root_exp_display = root_exp_date.map_or_else(|| "None".to_string(), |ts| ts.to_string()); - debug_if_enabled!( - "panel_api boot/update root status for input {} (offset={}s): exp_date={}, expired={}, expiring(offset)={}", - sanitize_sensitive_info(&input.name), - offset_secs, - root_exp_display, - root_expired, - root_expiring - ); - - if should_refresh_root { - if provisioning_enabled { - let old_username = accounts[root_idx].username.clone(); - let old_password = accounts[root_idx].password.clone(); - - debug_if_enabled!( - "panel_api boot/update refreshing root account {} for input {} (exp_date={}, offset={}s)", - sanitize_sensitive_info(&old_username), - sanitize_sensitive_info(&input.name), - root_exp_display, - offset_secs - ); - - let (active_username, active_password, creds_changed) = if renew_enabled { - match panel_client_renew( - app_state.as_ref(), - panel_cfg, - old_username.as_str(), - old_password.as_str(), - ) - .await - { - Ok(()) => { - provisioned_root = 1; - (old_username.clone(), old_password.clone(), false) - } - Err(err) => { - debug_if_enabled!( - "panel_api client_renew failed for root {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - if new_enabled { - match panel_client_new(app_state.as_ref(), panel_cfg).await { - Ok((new_username, new_password, _base_url_from_resp)) => { - provisioned_root = 1; - // Variant B: if the old root is still valid but within offset window, - // keep it as a new alias entry so we don't lose usable credentials. - let park_old_root_as_alias = - root_expiring && !root_expired && root_exp_date.is_some(); - - if park_old_root_as_alias { - let base_url = resolve_provisioned_account_base_url( - input.url.as_str(), - None, - &old_username, - &old_password, - ); - let alias_name = derive_unique_alias_name_set( - &existing_names, - &input.name, - old_username.as_str(), - ); - existing_names.insert(alias_name.clone().into()); - - if let Some(csv_path) = csv_path.as_ref() { - let batch_type = if input.input_type == InputType::Xtream { - InputType::XtreamBatch - } else if input.input_type == InputType::M3u { - InputType::M3uBatch - } else { - input.input_type - }; - let csv_lock = - app_state.app_config.file_locks.write_lock(csv_path).await; - if let Err(err) = csv_patch_batch_append( - app_state, - &csv_lock, - csv_path, - batch_type, - &alias_name, - &base_url, - &old_username, - &old_password, - root_exp_date, - ) - .await - { - debug_if_enabled!( - "panel_api boot/update failed to park old root as csv alias {}: {}", - sanitize_sensitive_info(&alias_name), - err - ); - } else { - any_change = true; - } - } else { - sources_yml_patches.push(SourcesYmlPatch::AddAlias { - input_name: input.name.clone(), - alias_name: alias_name.clone().into(), - base_url, - username: old_username.clone(), - password: old_password.clone(), - exp_date: root_exp_date, - }); - pending_sources_yml = true; - } - - accounts.push(AccountCredentials { - name: alias_name.into(), - username: old_username.clone(), - password: old_password.clone(), - exp_date: root_exp_date, - }); - - debug_if_enabled!( - "panel_api boot/update parked old root credentials for input {} as new alias (exp_date={:?})", - sanitize_sensitive_info(&input.name), - root_exp_date - ); - } - - if let Some(csv_path) = csv_path.as_ref() { - let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - if let Err(err) = csv_patch_batch_update_credentials( - app_state, - &csv_lock, - input.input_type, - csv_path, - &input.name, - &old_username, - &old_password, - &new_username, - &new_password, - None, - ) - .await - { - debug_if_enabled!( - "panel_api boot/update failed to persist new root credentials to csv for {}: {}", - sanitize_sensitive_info(&input.name), - err - ); - } else { - any_change = true; - } - } else { - sources_yml_patches.push(SourcesYmlPatch::UpdateRootCredentials { - input_name: input.name.clone(), - username: new_username.clone(), - password: new_password.clone(), - exp_date: None, - }); - pending_sources_yml = true; - } - - accounts[root_idx].username.clone_from(&new_username); - accounts[root_idx].password.clone_from(&new_password); - (new_username, new_password, true) - } - Err(err) => { - debug_if_enabled!( - "panel_api client_new failed for root {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - (old_username.clone(), old_password.clone(), false) - } - } - } else { - (old_username.clone(), old_password.clone(), false) - } - } - } - } else if new_enabled { - match panel_client_new(app_state.as_ref(), panel_cfg).await { - Ok((new_username, new_password, _base_url_from_resp)) => { - provisioned_root = 1; - let park_old_root_as_alias = root_expiring && !root_expired && root_exp_date.is_some(); - - if park_old_root_as_alias { - let base_url = resolve_provisioned_account_base_url( - input.url.as_str(), - None, - &old_username, - &old_password, - ); - let alias_name = - derive_unique_alias_name_set(&existing_names, &input.name, old_username.as_str()); - existing_names.insert(alias_name.clone().into()); - - if let Some(csv_path) = csv_path.as_ref() { - let batch_type = if input.input_type == InputType::Xtream { - InputType::XtreamBatch - } else if input.input_type == InputType::M3u { - InputType::M3uBatch - } else { - input.input_type - }; - let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - if let Err(err) = csv_patch_batch_append( - app_state, - &csv_lock, - csv_path, - batch_type, - &alias_name, - &base_url, - &old_username, - &old_password, - root_exp_date, - ) - .await - { - debug_if_enabled!( - "panel_api boot/update failed to park old root as csv alias {}: {}", - sanitize_sensitive_info(&alias_name), - err - ); - } else { - any_change = true; - } - } else { - sources_yml_patches.push(SourcesYmlPatch::AddAlias { - input_name: input.name.clone(), - alias_name: alias_name.clone().into(), - base_url, - username: old_username.clone(), - password: old_password.clone(), - exp_date: root_exp_date, - }); - pending_sources_yml = true; - } - - accounts.push(AccountCredentials { - name: alias_name.into(), - username: old_username.clone(), - password: old_password.clone(), - exp_date: root_exp_date, - }); - - debug_if_enabled!( - "panel_api boot/update parked old root credentials for input {} as new alias (exp_date={:?})", - sanitize_sensitive_info(&input.name), - root_exp_date - ); - } - - if let Some(csv_path) = csv_path.as_ref() { - let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - if let Err(err) = csv_patch_batch_update_credentials( - app_state, - &csv_lock, - input.input_type, - csv_path, - &input.name, - &old_username, - &old_password, - &new_username, - &new_password, - None, - ) - .await - { - debug_if_enabled!( - "panel_api boot/update failed to persist new root credentials to csv for {}: {}", - sanitize_sensitive_info(&input.name), - err - ); - } else { - any_change = true; - } - } else { - sources_yml_patches.push(SourcesYmlPatch::UpdateRootCredentials { - input_name: input.name.clone(), - username: new_username.clone(), - password: new_password.clone(), - exp_date: None, - }); - pending_sources_yml = true; - } - - accounts[root_idx].username.clone_from(&new_username); - accounts[root_idx].password.clone_from(&new_password); - (new_username, new_password, true) - } - Err(err) => { - debug_if_enabled!( - "panel_api client_new failed for root {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - (old_username.clone(), old_password.clone(), false) - } - } - } else { - (old_username.clone(), old_password.clone(), false) - }; - - if adult_enabled { - if let Err(err) = panel_client_adult_content( - app_state.as_ref(), - panel_cfg, - Some((active_username.as_str(), active_password.as_str())), - ) - .await - { - debug_if_enabled!( - "panel_api client_adult_content failed for root {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - - let refreshed_exp = panel_client_info( - app_state.as_ref(), - panel_cfg, - active_username.as_str(), - active_password.as_str(), - time_ctx.as_ref(), - ) - .await - .ok() - .flatten(); - - let ready = wait_for_panel_api_account_ready( - app_state, - input.as_ref(), - panel_cfg, - &input.name, - active_username.as_str(), - active_password.as_str(), - ) - .await; - if !ready { - debug_if_enabled!( - "panel_api boot/update probe timeout for root {}; skipping exp_date refresh", - sanitize_sensitive_info(&input.name) - ); - } else if let Some(new_exp) = refreshed_exp { - if let Some(csv_path) = csv_path.as_ref() { - let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - let result = if creds_changed { - csv_patch_batch_update_credentials( - app_state, - &csv_lock, - input.input_type, - csv_path, - &input.name, - old_username.as_str(), - old_password.as_str(), - active_username.as_str(), - active_password.as_str(), - Some(new_exp), - ) - .await - } else { - csv_patch_batch_update_exp_date( - app_state, - &csv_lock, - input.input_type, - csv_path, - &input.name, - old_username.as_str(), - old_password.as_str(), - new_exp, - ) - .await - }; - if let Err(err) = result { - debug_if_enabled!( - "panel_api boot/update failed to persist root exp_date to csv for {}: {}", - sanitize_sensitive_info(&input.name), - err - ); - } else { - accounts[root_idx].exp_date = Some(new_exp); - any_change = true; - } - } else { - if creds_changed { - sources_yml_patches.push(SourcesYmlPatch::UpdateRootCredentials { - input_name: input.name.clone(), - username: active_username.clone(), - password: active_password.clone(), - exp_date: Some(new_exp), - }); - } else { - sources_yml_patches.push(SourcesYmlPatch::UpdatePanelAccountExpiry { - input_name: input.name.clone(), - account_name: input.name.clone(), - exp_date: new_exp, - }); - } - pending_sources_yml = true; - accounts[root_idx].exp_date = Some(new_exp); - } - } - } else { - debug_if_enabled!( - "panel_api boot/update skipped root refresh for input {}: client_new/client_renew disabled", - sanitize_sensitive_info(&input.name) - ); - } - } - } else { - debug_if_enabled!( - "panel_api boot/update skipped root provisioning for input {}: missing credentials", - sanitize_sensitive_info(&input.name) - ); - } - - // Refresh aliases after the root operation to avoid over-provisioning. - let now = effective_now; - let offset_deadline = now.saturating_add(offset_secs); - let root_valid = root_counts_towards_pool_at(&accounts, &input.name, now); - let desired_aliases = min_pool.filter(|m| *m > 0).map_or_else( - || u16::try_from(accounts.iter().filter(|a| a.name != input.name).count()).unwrap_or(u16::MAX), - |min_pool| min_pool.saturating_sub(u16::from(root_valid)), - ); - - let expiring_aliases = accounts - .iter() - .filter(|a| a.name != input.name && is_expiring_with_offset_at(a.exp_date, offset_secs, now)) - .count(); - let expired_aliases = - accounts.iter().filter(|a| a.name != input.name && is_input_expired_at(a.exp_date, now)).count(); - - let valid_aliases_beyond_offset = accounts - .iter() - .filter(|a| { - if a.name == input.name { - return false; - } - if is_input_expired_at(a.exp_date, now) { - return false; - } - match a.exp_date { - Some(ts) => u64::try_from(ts).is_ok_and(|exp_ts| exp_ts > offset_deadline), - None => false, - } - }) - .count(); - - let desired_aliases_u16 = desired_aliases; - let alias_total = accounts.iter().filter(|a| a.name != input.name).count(); - let alias_total_u16 = u16::try_from(alias_total).unwrap_or(u16::MAX); - let missing_aliases_u16 = desired_aliases_u16.saturating_sub(alias_total_u16); - - let (refresh_plan, planned_refresh_aliases) = if provisioning_enabled { - let mut refresh_candidates: Vec = accounts - .iter() - .enumerate() - .filter(|(_, a)| { - if a.name == input.name { - return false; - } - if is_input_expired_at(a.exp_date, now) { - return false; - } - match a.exp_date { - None => true, - Some(ts) => u64::try_from(ts).map_or(true, |exp_ts| exp_ts <= offset_deadline), - } - }) - .map(|(idx, _)| idx) - .collect(); - - refresh_candidates.sort_by_key(|idx| { - let acct = &accounts[*idx]; - match acct.exp_date { - None => (0_u8, i64::MIN), - Some(ts) => (1_u8, ts), - } - }); - - let valid_aliases_beyond_offset_u16 = u16::try_from(valid_aliases_beyond_offset).unwrap_or(u16::MAX); - let needed_refresh_aliases_u16 = desired_aliases_u16.saturating_sub(valid_aliases_beyond_offset_u16); - let planned_refresh_aliases = refresh_candidates.len().min(usize::from(needed_refresh_aliases_u16)); - let refresh_plan: Vec = refresh_candidates.into_iter().take(planned_refresh_aliases).collect(); - (refresh_plan, planned_refresh_aliases) - } else { - (Vec::new(), 0) - }; - - let log_pool = alias_pool_has_min(panel_cfg); - let enabled_users = if log_pool { count_enabled_proxy_users(app_state.as_ref(), &input.name) } else { 0 }; - if log_pool { - debug_if_enabled!( - "panel_api boot/update provisioning aliases for input {} (offset={}s): desired={}, valid_beyond_offset={}, expiring(offset)={}, expired={}, refresh_planned(offset)={}, missing={}", - sanitize_sensitive_info(&input.name), - offset_secs, - desired_aliases_u16, - valid_aliases_beyond_offset, - expiring_aliases, - expired_aliases, - planned_refresh_aliases, - missing_aliases_u16 - ); - } - - if !refresh_plan.is_empty() { - debug_if_enabled!( - "panel_api boot/update alias refresh plan for input {}: selected={}", - sanitize_sensitive_info(&input.name), - refresh_plan - .iter() - .filter_map(|idx| accounts.get(*idx)) - .map(|acct| sanitize_sensitive_info(&acct.name).to_string()) - .collect::>() - .join(",") - ); - } - - for idx in refresh_plan { - let Some(acct) = accounts.get_mut(idx) else { - continue; - }; - let account_name = acct.name.clone(); - let old_username = acct.username.clone(); - let old_password = acct.password.clone(); - - debug_if_enabled!( - "panel_api boot/update refreshing alias account {} for input {} (exp_date={:?}, offset={}s)", - sanitize_sensitive_info(&account_name), - sanitize_sensitive_info(&input.name), - acct.exp_date, - offset_secs - ); - - let (active_username, active_password, creds_changed) = if renew_enabled { - match panel_client_renew(app_state.as_ref(), panel_cfg, old_username.as_str(), old_password.as_str()).await - { - Ok(()) => { - provisioned_aliases = provisioned_aliases.saturating_add(1); - (old_username.clone(), old_password.clone(), false) - } - Err(err) => { - debug_if_enabled!( - "panel_api client_renew failed for alias {}: {}", - sanitize_sensitive_info(&account_name), - sanitize_sensitive_info(&err.to_string()) - ); - if new_enabled { - match panel_client_new(app_state.as_ref(), panel_cfg).await { - Ok((new_username, new_password, base_url_from_resp)) => { - let base_url = resolve_provisioned_account_base_url( - input.url.as_str(), - base_url_from_resp.as_deref(), - &new_username, - &new_password, - ); - - let alias_name = - derive_unique_alias_name_set(&existing_names, &input.name, &new_username); - existing_names.insert(alias_name.clone().into()); - - if adult_enabled { - if let Err(err) = panel_client_adult_content( - app_state.as_ref(), - panel_cfg, - Some((new_username.as_str(), new_password.as_str())), - ) - .await - { - debug_if_enabled!( - "panel_api client_adult_content failed for {}: {}", - sanitize_sensitive_info(&alias_name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - - let exp_date = panel_client_info( - app_state.as_ref(), - panel_cfg, - new_username.as_str(), - new_password.as_str(), - time_ctx.as_ref(), - ) - .await - .ok() - .flatten(); - - if let Some(csv_path) = csv_path.as_ref() { - let batch_type = if input.input_type == InputType::Xtream { - InputType::XtreamBatch - } else if input.input_type == InputType::M3u { - InputType::M3uBatch - } else { - input.input_type - }; - let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - if let Err(err) = csv_patch_batch_append( - app_state, - &csv_lock, - csv_path, - batch_type, - &alias_name, - &base_url, - &new_username, - &new_password, - exp_date, - ) - .await - { - debug_if_enabled!( - "panel_api boot/update failed to append new csv account for {}: {}", - sanitize_sensitive_info(&alias_name), - err - ); - continue; - } - any_change = true; - } else { - sources_yml_patches.push(SourcesYmlPatch::AddAlias { - input_name: input.name.clone(), - alias_name: alias_name.clone().into(), - base_url, - username: new_username.clone(), - password: new_password.clone(), - exp_date, - }); - pending_sources_yml = true; - } - - newly_created_accounts.push(AccountCredentials { - name: alias_name.into(), - username: new_username, - password: new_password, - exp_date, - }); - provisioned_aliases = provisioned_aliases.saturating_add(1); - continue; - } - Err(err) => { - debug_if_enabled!( - "panel_api client_new failed for input {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - continue; - } - } - } - continue; - } - } - } else if new_enabled { - match panel_client_new(app_state.as_ref(), panel_cfg).await { - Ok((new_username, new_password, base_url_from_resp)) => { - let base_url = resolve_provisioned_account_base_url( - input.url.as_str(), - base_url_from_resp.as_deref(), - &new_username, - &new_password, - ); - - let alias_name = derive_unique_alias_name_set(&existing_names, &input.name, &new_username); - existing_names.insert(alias_name.clone().into()); - - if adult_enabled { - if let Err(err) = panel_client_adult_content( - app_state.as_ref(), - panel_cfg, - Some((new_username.as_str(), new_password.as_str())), - ) - .await - { - debug_if_enabled!( - "panel_api client_adult_content failed for {}: {}", - sanitize_sensitive_info(&alias_name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - - let exp_date = panel_client_info( - app_state.as_ref(), - panel_cfg, - new_username.as_str(), - new_password.as_str(), - time_ctx.as_ref(), - ) - .await - .ok() - .flatten(); - - if let Some(csv_path) = csv_path.as_ref() { - let batch_type = if input.input_type == InputType::Xtream { - InputType::XtreamBatch - } else if input.input_type == InputType::M3u { - InputType::M3uBatch - } else { - input.input_type - }; - let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - if let Err(err) = csv_patch_batch_append( - app_state, - &csv_lock, - csv_path, - batch_type, - &alias_name, - &base_url, - &new_username, - &new_password, - exp_date, - ) - .await - { - debug_if_enabled!( - "panel_api boot/update failed to append new csv account for {}: {}", - sanitize_sensitive_info(&alias_name), - err - ); - continue; - } - any_change = true; - } else { - sources_yml_patches.push(SourcesYmlPatch::AddAlias { - input_name: input.name.clone(), - alias_name: alias_name.clone().into(), - base_url, - username: new_username.clone(), - password: new_password.clone(), - exp_date, - }); - pending_sources_yml = true; - } - - newly_created_accounts.push(AccountCredentials { - name: alias_name.into(), - username: new_username, - password: new_password, - exp_date, - }); - provisioned_aliases = provisioned_aliases.saturating_add(1); - continue; - } - Err(err) => { - debug_if_enabled!( - "panel_api client_new failed for input {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - continue; - } - } - } else { - continue; - }; - - if adult_enabled { - if let Err(err) = panel_client_adult_content( - app_state.as_ref(), - panel_cfg, - Some((active_username.as_str(), active_password.as_str())), - ) - .await - { - debug_if_enabled!( - "panel_api client_adult_content failed for {}: {}", - sanitize_sensitive_info(&account_name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - - let refreshed_exp = panel_client_info( - app_state.as_ref(), - panel_cfg, - active_username.as_str(), - active_password.as_str(), - time_ctx.as_ref(), - ) - .await - .ok() - .flatten(); - - if let Some(new_exp) = refreshed_exp { - if let Some(csv_path) = csv_path.as_ref() { - let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - let result = if creds_changed { - csv_patch_batch_update_credentials( - app_state, - &csv_lock, - input.input_type, - csv_path, - &account_name, - &old_username, - &old_password, - active_username.as_str(), - active_password.as_str(), - Some(new_exp), - ) - .await - } else { - csv_patch_batch_update_exp_date( - app_state, - &csv_lock, - input.input_type, - csv_path, - &account_name, - &old_username, - &old_password, - new_exp, - ) - .await - }; - if let Err(err) = result { - debug_if_enabled!( - "panel_api boot/update failed to persist exp_date to csv for {}: {}", - sanitize_sensitive_info(&account_name), - err - ); - } else { - acct.exp_date = Some(new_exp); - any_change = true; - } - } else { - if creds_changed { - sources_yml_patches.push(SourcesYmlPatch::UpdateAliasCredentials { - input_name: input.name.clone(), - alias_name: account_name.clone(), - username: active_username.clone(), - password: active_password.clone(), - exp_date: Some(new_exp), - }); - } else { - sources_yml_patches.push(SourcesYmlPatch::UpdatePanelAccountExpiry { - input_name: input.name.clone(), - account_name: account_name.clone(), - exp_date: new_exp, - }); - } - pending_sources_yml = true; - acct.exp_date = Some(new_exp); - } - } else { - debug_if_enabled!( - "panel_api boot/update renew/create succeeded but exp_date refresh failed for {}", - sanitize_sensitive_info(&account_name) - ); - } - } - - if !newly_created_accounts.is_empty() { - accounts.extend(newly_created_accounts); - } - - if adult_enabled { - for acct in &accounts { - if let Err(err) = panel_client_adult_content( - app_state.as_ref(), - panel_cfg, - Some((acct.username.as_str(), acct.password.as_str())), - ) - .await - { - debug_if_enabled!( - "panel_api client_adult_content failed for {}: {}", - sanitize_sensitive_info(&acct.name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - } - - let min_pool = min_pool.filter(|m| *m > 0); - if let Some(min_pool) = min_pool { - let root_valid = root_counts_towards_pool(&accounts, &input.name); - let alias_min = min_pool.saturating_sub(u16::from(root_valid)); - let (pool_changed, pool_provisioned) = ensure_alias_pool_min( - app_state, - input.as_ref(), - panel_cfg, - &mut accounts, - alias_min, - csv_path.as_deref(), - &mut sources_yml_patches, - time_ctx.as_ref(), - effective_now, - optional, - ) - .await; - if pool_changed { - if csv_path.is_some() { - any_change = true; - } else { - pending_sources_yml = true; - } - } - provisioned_aliases = provisioned_aliases.saturating_add(pool_provisioned); - } - - if log_pool { - let valid_total = count_valid_accounts_at(&accounts, now); - debug_if_enabled!( - "panel_api boot/update provisioning total for input {} (offset={}s): enabled_users={}, valid_accounts={}, provisioned_root={}, provisioned_aliases={}", - sanitize_sensitive_info(&input.name), - offset_secs, - enabled_users, - valid_total, - provisioned_root, - provisioned_aliases - ); - } - - if alias_pool_remove_expired(panel_cfg) { - if let Some(csv_path) = csv_path.as_ref() { - let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - match csv_patch_batch_remove_expired(app_state, &csv_lock, input.input_type, csv_path).await { - Ok(true) => any_change = true, - Ok(false) => {} - Err(err) => debug_if_enabled!("panel_api boot sync failed to remove expired csv accounts: {}", err), - } - } else { - sources_yml_patches.push(SourcesYmlPatch::RemoveExpiredAliases { input_name: input.name.clone() }); - pending_sources_yml = true; - } - } - - if optional.contains(PanelApiOptionalFlags::AccountInfo) { - let creds = accounts.first().map(|acct| (acct.username.as_str(), acct.password.as_str())); - match panel_account_info(app_state.as_ref(), panel_cfg, creds).await { - Ok(Some(credits)) => { - let normalized = credits.trim().to_string(); - if !normalized.is_empty() - /* && panel_cfg.credits.as_deref().map(str::trim) != Some(normalized.as_str()) */ - { - sources_yml_patches.push(SourcesYmlPatch::UpdatePanelApiCredits { - input_name: input.name.clone(), - credits: normalized, - }); - pending_sources_yml = true; - } - } - Ok(None) => {} - Err(err) => { - debug_if_enabled!( - "panel_api account_info failed for {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - } - } - } - - if panel_cfg.alias_pool.is_some() { - if let Some(csv_path) = csv_path.as_ref() { - let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; - match csv_patch_batch_sort_by_exp_date( - app_state, - &csv_lock, - input.input_type, - csv_path, - AliasExpDateSortOrder::NewestFirst, - ) - .await - { - Ok(true) => any_change = true, - Ok(false) => {} - Err(err) => debug_if_enabled!( - "panel_api boot/update final sort failed for csv alias pool {}: {}", - sanitize_sensitive_info(&input.name), - err - ), - } - } else if pending_sources_yml && !source_yml_sort_aliases_requested { - sources_yml_patches.push(SourcesYmlPatch::SortAliases { - input_name: input.name.clone(), - order: AliasExpDateSortOrder::NewestFirst, - }); - } - } - - if pending_sources_yml { - match crate::api::source_yml_patch::execute_account_source_patches( - app_state, - sources_path, - &sources_yml_patches, - ) - .await - { - Ok(true) => any_change = true, - Ok(false) => {} - Err(err) => debug_if_enabled!("panel_api boot sync failed to persist source.yml patches: {}", err), - } - } - - any_change -} - -pub(crate) async fn sync_panel_api_exp_dates(app_state: &Arc) { - let sources_file_path = app_state.app_config.paths.load().sources_file_path.clone(); - let sources_path = PathBuf::from(&sources_file_path); - let mut any_change = false; - - let sources = app_state.app_config.sources.load(); - for input in &sources.inputs { - if sync_panel_api_for_input_on_boot(app_state, input, sources_path.as_path()).await { - any_change = true; - } - } - - if any_change { - // Account writes already synchronize prepared sources; refresh allocation without restarting services. - refresh_internal_sources(app_state); - } -} - -pub(crate) async fn sync_panel_api_exp_dates_on_boot(app_state: &Arc) { - sync_panel_api_exp_dates(app_state).await; -} - -pub(crate) async fn sync_panel_api_alias_pool_for_target(app_state: &Arc, target_name: &str) { - let sources_file_path = app_state.app_config.paths.load().sources_file_path.clone(); - let sources_path = PathBuf::from(&sources_file_path); - let mut any_change = false; - - let sources = app_state.app_config.sources.load(); - for source in &sources.sources { - let target_match = source.targets.iter().any(|target| target.name.eq_ignore_ascii_case(target_name)); - if !target_match { - continue; - } - - for input_name in &source.inputs { - let Some(input) = sources.get_input_by_name(input_name) else { - continue; - }; - let Some(panel_cfg) = input.panel_api.as_ref() else { - continue; - }; - if !panel_cfg.enabled || panel_cfg.url.trim().is_empty() { - continue; - } - if let Err(err) = validate_panel_api_config(panel_cfg) { - debug_if_enabled!( - "panel_api user sync skipped for {}: {}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&err.to_string()) - ); - continue; - } - if !alias_pool_has_min(panel_cfg) { - continue; - } - - if sync_panel_api_for_input_on_boot(app_state, input, sources_path.as_path()).await { - any_change = true; - } - } - } - - if any_change && should_reload_sources_after_internal_write(app_state.as_ref()) { - refresh_internal_sources(app_state); - } -} - -fn provisioning_method_to_reqwest(method: PanelApiProvisioningMethod) -> Method { - match method { - PanelApiProvisioningMethod::Head => Method::HEAD, - PanelApiProvisioningMethod::Get => Method::GET, - PanelApiProvisioningMethod::Post => Method::POST, - } -} - -fn build_player_api_action_url(base_url: &str, username: &str, password: &str, action: &str) -> Option { - let url = Url::parse(base_url).ok()?; - let host = url.host_str()?; - let scheme = url.scheme(); - let mut base = concat_string!(scheme, "://", host); - if let Some(port) = url.port() { - base.push(':'); - base.push_str(&port.to_string()); - } - base.push_str("/player_api.php"); - let mut test_url = Url::parse(&base).ok()?; - test_url - .query_pairs_mut() - .append_pair("username", username) - .append_pair("password", password) - .append_pair("action", action); - Some(test_url) -} - -enum PanelApiProbeTarget { - PlayerApi { action: &'static str, input_source: InputSource }, -} - -impl PanelApiProbeTarget { - fn action(&self) -> &'static str { - match self { - PanelApiProbeTarget::PlayerApi { action, .. } => action, - } - } -} - -fn format_probe_target_actions(targets: &[PanelApiProbeTarget]) -> String { - let mut result = String::new(); - for (idx, target) in targets.iter().enumerate() { - if idx > 0 { - result.push(','); - } - result.push_str(target.action()); - } - result -} - -fn build_panel_api_probe_targets(input: &ConfigInput, username: &str, password: &str) -> Vec { - let mut targets = Vec::new(); - for action in ["client_info", "get_live_categories", "get_series_categories", "get_vod_categories"] { - if let Some(url) = build_player_api_action_url(input.url.as_str(), username, password, action) { - targets.push(PanelApiProbeTarget::PlayerApi { - action, - input_source: InputSource::from(input).with_url(url.to_string()), - }); - } - } - targets -} - -async fn probe_panel_api_targets( - app_state: &Arc, - probe_method: PanelApiProvisioningMethod, - targets: &[PanelApiProbeTarget], - done: &mut HashSet<&'static str>, -) -> bool { - for target in targets { - let action = target.action(); - if done.contains(action) { - continue; - } - match target { - PanelApiProbeTarget::PlayerApi { action, input_source } => { - match probe_panel_api_test_url(app_state, input_source, probe_method).await { - Ok(status) => { - debug_if_enabled!( - "panel_api probe status: '{}' action={} url: {}", - format_http_status(status), - action, - sanitize_sensitive_info(input_source.url.as_str()) - ); - if status.is_success() { - done.insert(action); - } - } - Err(err) => { - if err.kind() == io::ErrorKind::TimedOut { - debug_if_enabled!( - "panel_api probe timeout action={} url: {}", - action, - sanitize_sensitive_info(input_source.url.as_str()) - ); - } else { - debug_if_enabled!( - "panel_api probe failed action={} url: {}: {err}", - action, - sanitize_sensitive_info(input_source.url.as_str()) - ); - } - } - } - } - } - } - done.len() == targets.len() -} - -async fn probe_panel_api_test_url( - app_state: &Arc, - input_source: &InputSource, - method: PanelApiProvisioningMethod, -) -> Result { - let client = app_state.http_clients.default.load(); - let request_method = provisioning_method_to_reqwest(method); - let test_url = Url::parse(input_source.url.as_str()).map_err(|err| { - string_to_io_error(format!( - "Malformed URL {}: {}", - sanitize_sensitive_info(input_source.url.as_str()), - sanitize_sensitive_info(err.to_string().as_str()) - )) - })?; - - let config = app_state.app_config.config.load(); - let default_user_agent = config.default_user_agent.clone(); - let disabled_headers = config.get_disabled_headers(); - drop(config); - - let headers = request::get_request_headers( - Some(&input_source.headers), - None::<&HashMap>>, - disabled_headers.as_ref(), - default_user_agent.as_deref(), - ); - let response = request::send_with_retry_and_provider( - &app_state.app_config, - &test_url, - input_source.get_provider(), - false, - |resolved_url| client.request(request_method.clone(), resolved_url.clone()).headers(headers.clone()), - ) - .await?; - Ok(response.status()) -} - -async fn apply_provisioning_cooldown(panel_cfg: &PanelApiConfig, account_name: &str, input_name: &Arc) { - let cooldown_secs = panel_cfg.provisioning.cooldown_sec; - if cooldown_secs == 0 { - return; - } - debug_if_enabled!( - "panel_api provisioning cooldown for {} (input={}): {}s", - sanitize_sensitive_info(account_name), - sanitize_sensitive_info(input_name), - cooldown_secs - ); - tokio::time::sleep(Duration::from_secs(cooldown_secs)).await; -} - -pub(crate) async fn wait_for_panel_api_account_ready( - app_state: &Arc, - input: &ConfigInput, - panel_cfg: &PanelApiConfig, - account_name: &str, - username: &str, - password: &str, -) -> bool { - let max_wait_secs = panel_cfg.provisioning.timeout_sec; - let probe_interval_secs = panel_cfg.provisioning.probe_interval_sec.max(1); - let probe_method = panel_cfg.provisioning.method; - - let probe_targets = build_panel_api_probe_targets(input, username, password); - if probe_targets.is_empty() { - debug_if_enabled!( - "panel_api probe skipped for {} (input={}): no probe targets", - sanitize_sensitive_info(account_name), - sanitize_sensitive_info(&input.name) - ); - return false; - } - - let targets_list = format_probe_target_actions(&probe_targets); - debug_if_enabled!( - "panel_api probe start for {} (input={} timeout={}s interval={}s method={}) targets={}", - sanitize_sensitive_info(account_name), - sanitize_sensitive_info(&input.name), - max_wait_secs, - probe_interval_secs, - probe_method, - targets_list - ); - - let deadline = Instant::now() + Duration::from_secs(max_wait_secs); - let probe_delay = Duration::from_secs(probe_interval_secs); - let mut done_targets = HashSet::new(); - let mut attempt = 0u64; - loop { - attempt += 1; - debug_if_enabled!("panel_api probe attempt {}", attempt); - if probe_panel_api_targets(app_state, probe_method, &probe_targets, &mut done_targets).await { - apply_provisioning_cooldown(panel_cfg, account_name, &input.name).await; - return true; - } - - if max_wait_secs == 0 { - return false; - } - let now = Instant::now(); - if now >= deadline { - return false; - } - let remaining = deadline.checked_duration_since(now).unwrap_or_default(); - let sleep_for = if remaining < probe_delay { remaining } else { probe_delay }; - tokio::time::sleep(sleep_for).await; - } -} - -#[allow(clippy::too_many_lines)] -pub(crate) async fn run_panel_api_provisioning_probe( - app_state: Arc, - input_name: Arc, - stop_signal: CancellationToken, - addr: SocketAddr, - virtual_id: VirtualId, -) -> Result<(), TuliproxError> { - if stop_signal.is_cancelled() { - return Ok(()); - } - let provisioning_kick_secs = 0; - let Some(input) = find_input_by_provider_name(app_state.as_ref(), input_name.as_ref()) else { - debug_if_enabled!( - "panel_api provisioning probe skipped (input no longer exists) for input {}", - sanitize_sensitive_info(input_name.as_ref()) - ); - stop_signal.cancel(); - let _ = app_state - .connection_manager - .close_connection_with_reason_and_block( - &addr, - virtual_id, - provisioning_kick_secs, - DisconnectReason::Provisioning, - ) - .await; - return Ok(()); - }; - - let Some(panel_cfg) = input.panel_api.as_ref() else { - debug_if_enabled!( - "panel_api provisioning probe skipped (missing config) for input {}", - sanitize_sensitive_info(&input.name) - ); - stop_signal.cancel(); - let _ = app_state - .connection_manager - .close_connection_with_reason_and_block( - &addr, - virtual_id, - provisioning_kick_secs, - DisconnectReason::Provisioning, - ) - .await; - return Ok(()); - }; - if !panel_cfg.enabled { - debug_if_enabled!( - "panel_api provisioning probe skipped (panel_api.enabled false) for input {}", - sanitize_sensitive_info(&input.name) - ); - stop_signal.cancel(); - let _ = app_state - .connection_manager - .close_connection_with_reason_and_block( - &addr, - virtual_id, - provisioning_kick_secs, - DisconnectReason::Provisioning, - ) - .await; - return Ok(()); - } - if panel_cfg.url.trim().is_empty() { - debug_if_enabled!( - "panel_api provisioning probe skipped (panel_api.url empty) for input {}", - sanitize_sensitive_info(&input.name) - ); - stop_signal.cancel(); - let _ = app_state - .connection_manager - .close_connection_with_reason_and_block( - &addr, - virtual_id, - provisioning_kick_secs, - DisconnectReason::Provisioning, - ) - .await; - return Ok(()); - } - - let max_wait_secs = panel_cfg.provisioning.timeout_sec; - - debug_if_enabled!( - "panel_api provisioning probe start for input {} (timeout={}s)", - sanitize_sensitive_info(&input.name), - max_wait_secs - ); - - let outcome = tokio::select! { - () = stop_signal.cancelled() => return Ok(()), - outcome = try_provision_account_on_exhausted(&app_state, &input_name) => outcome, - }; - - if let Some(outcome) = outcome.as_ref() { - debug_if_enabled!( - "panel_api provisioning {} completed for input {}", - outcome.kind_label(), - sanitize_sensitive_info(&input.name) - ); - } else { - debug_if_enabled!( - "panel_api provisioning failed for input {}; waiting for timeout", - sanitize_sensitive_info(&input.name) - ); - } - - if outcome.is_none() { - if max_wait_secs > 0 { - tokio::select! { - () = stop_signal.cancelled() => return Ok(()), - () = tokio::time::sleep(Duration::from_secs(max_wait_secs)) => {} - } - } - debug_if_enabled!( - "panel_api provisioning probe timeout reached for input {} (no credentials)", - sanitize_sensitive_info(&input.name) - ); - stop_signal.cancel(); - debug_if_enabled!( - "panel_api provisioning closing client connection for input {} addr={}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&addr.to_string()) - ); - let _ = app_state - .connection_manager - .close_connection_with_reason_and_block( - &addr, - virtual_id, - provisioning_kick_secs, - DisconnectReason::Provisioning, - ) - .await; - return Ok(()); - } - - if stop_signal.is_cancelled() { - return Ok(()); - } - stop_signal.cancel(); - debug_if_enabled!( - "panel_api provisioning closing client connection for input {} addr={}", - sanitize_sensitive_info(&input.name), - sanitize_sensitive_info(&addr.to_string()) - ); - let _ = app_state - .connection_manager - .close_connection_with_reason_and_block( - &addr, - virtual_id, - provisioning_kick_secs, - DisconnectReason::Provisioning, - ) - .await; - Ok(()) -} - -pub fn create_panel_api_provisioning_stream_details( - app_state: &Arc, - input: &ConfigInput, - provider_name: Option>, - grace_period_options: &GracePeriodOptions, - addr: SocketAddr, - virtual_id: VirtualId, -) -> StreamDetails { - let stop_signal = CancellationToken::new(); - let headers = [("connection".to_string(), "close".to_string())]; - let (stream, stream_info) = - create_panel_api_provisioning_stream_with_stop(&app_state.app_config, &headers, stop_signal.clone()); - - if stream.is_none() { - debug_if_enabled!( - "panel_api provisioning stream missing; falling back to provider exhausted for input {}", - sanitize_sensitive_info(&input.name) - ); - let (stream, stream_info) = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); - return StreamDetails { - shared_subscriber_id: None, - stream, - stream_info, - provider_name, - request_url: None, - session_headers: None, - provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), - user_agent_stream_index: None, - grace_period: *grace_period_options, - provider_grace_active: false, - disable_provider_grace: true, - reconnect_flag: None, - provider_handle: None, - content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, - grace_resolution_context: None, - custom_reason: None, - response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), - session_registration: None, - }; - } - - let app_state_clone = Arc::clone(app_state); - let input_name = Arc::clone(&input.name); - let stop_clone = stop_signal.clone(); - tokio::spawn(async move { - if let Err(err) = - run_panel_api_provisioning_probe(app_state_clone, input_name, stop_clone, addr, virtual_id).await - { - error!("Error running Probe: {err:?}"); - } - }); - - StreamDetails { - shared_subscriber_id: None, - stream, - stream_info, - provider_name, - request_url: None, - session_headers: None, - provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), - user_agent_stream_index: None, - grace_period: *grace_period_options, - provider_grace_active: false, - disable_provider_grace: true, - reconnect_flag: None, - provider_handle: None, - content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, - grace_resolution_context: None, - custom_reason: None, - response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), - session_registration: None, - } -} - #[cfg(test)] mod tests; + +mod account; +mod alias_pool; +mod params; +mod probe; +mod provisioning; +mod sync; +mod time; +mod transport; + +#[cfg(test)] +use self::account::build_user_api_account_info_input_source; +pub use self::provisioning::{create_panel_api_provisioning_stream_details, try_provision_account_on_exhausted}; +#[cfg(test)] +use self::transport::panel_api_retry_after_from_header_value; +#[cfg(test)] +use self::transport::panel_api_retryable_status; +use self::{ + account::{ + fetch_root_user_api_info, panel_account_info, panel_client_adult_content, panel_client_info, + panel_client_info_raw, panel_client_new, panel_client_renew, + }, + alias_pool::{ + alias_pool_has_min, alias_pool_limit_values, alias_pool_remove_expired, collect_accounts, + compare_alias_exp_date_config, count_enabled_proxy_users, count_valid_accounts_at, ensure_alias_pool_min, + extract_account_creds_from_input, resolve_alias_pool_min, root_counts_towards_pool, + root_counts_towards_pool_at, sort_account_aliases_keep_root_first, + }, + params::{ + build_panel_url, extract_stringish, extract_username_password_from_json, first_json_object, parse_boolish, + require_batch_alias_path, resolve_query_params, validate_account_info_params, + validate_client_adult_content_params, validate_client_info_params, validate_client_new_params, + validate_client_renew_params, validate_panel_api_config, AccountCredentials, + }, + probe::{ + build_panel_api_probe_targets, build_player_api_action_url, format_probe_target_actions, + provisioning_method_to_reqwest, PanelApiProbeTarget, + }, + provisioning::{ + aliases_need_sort_config, is_date_only_yyyy_mm_dd, parse_panel_api_provisioning_offset_secs, + resolve_panel_api_optional_flags, + }, + sync::{ + refresh_internal_sources, resolve_batch_alias_path, should_reload_sources_after_internal_write, + sync_panel_api_for_input_on_boot, + }, + time::{ + apply_clock_skew, is_expiring_with_offset_at, load_panel_api_time_cache, normalize_panel_expire, + panel_api_time_cache_path, parse_cached_tz, persist_panel_api_time_cache, resolve_panel_expire_mode, + PanelApiTimeCacheEntry, PanelApiTimeContext, + }, + transport::{panel_get_json, user_api_get_json}, +}; +pub(crate) use self::{ + alias_pool::{ + is_alias_pool_max_reached, sync_panel_api_alias_pool_for_target, sync_panel_api_exp_dates, + sync_panel_api_exp_dates_on_boot, target_has_alias_pool_min, + }, + probe::run_panel_api_provisioning_probe, + provisioning::{can_provision_on_exhausted, find_input_by_provider_name, wait_for_panel_api_account_ready}, +}; diff --git a/backend/app/src/api/panel_api/params.rs b/backend/app/src/api/panel_api/params.rs new file mode 100644 index 000000000..6a5c3c06f --- /dev/null +++ b/backend/app/src/api/panel_api/params.rs @@ -0,0 +1,270 @@ +use super::{alias_pool_limit_values, parse_panel_api_provisioning_offset_secs, resolve_batch_alias_path}; +use crate::model::{ConfigInput, PanelApiConfig, PanelApiQueryParam}; +use serde_json::Value; +use shared::{error::TuliproxError, model::PanelApiAliasPoolSizeValue}; +use std::{path::PathBuf, sync::Arc}; +use url::Url; + +#[derive(Debug, Clone)] +pub(super) struct AccountCredentials { + pub(super) name: Arc, + pub(super) username: String, + pub(super) password: String, + pub(super) exp_date: Option, +} + +pub(super) fn parse_boolish(value: &Value) -> bool { + match value { + Value::Bool(b) => *b, + Value::Number(n) => n.as_i64().unwrap_or(0) != 0, + Value::String(s) => matches!(s.trim().to_lowercase().as_str(), "true" | "1" | "yes" | "y" | "ok"), + _ => false, + } +} + +pub(super) fn extract_stringish(value: &Value) -> Option { + match value { + Value::String(s) => { + let trimmed = s.trim(); + if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + } + } + Value::Number(n) => Some(n.to_string()), + Value::Bool(b) => Some(b.to_string()), + _ => None, + } +} + +pub(super) fn first_json_object(value: &Value) -> Option<&serde_json::Map> { + match value { + Value::Array(arr) => arr.first().and_then(|v| v.as_object()), + Value::Object(obj) => Some(obj), + _ => None, + } +} + +pub(super) fn extract_username_password_from_json(obj: &serde_json::Map) -> Option<(String, String)> { + let username = obj.get("username").and_then(|v| v.as_str()).map(str::trim).filter(|s| !s.is_empty()); + let password = obj.get("password").and_then(|v| v.as_str()).map(str::trim).filter(|s| !s.is_empty()); + match (username, password) { + (Some(u), Some(p)) => Some((u.to_string(), p.to_string())), + _ => None, + } +} + +pub(super) fn validate_type_is_m3u(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { + let typ = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("type")).map(|p| p.value.trim().to_string()); + let typ_str = typ.as_deref().unwrap_or_default(); + if typ_str.trim().eq_ignore_ascii_case("m3u") { + Ok(()) + } else if typ_str.is_empty() { + Err(TuliproxError::ConfigPanelApi("panel_api: missing required query param 'type=m3u'".to_string())) + } else { + Err(TuliproxError::ConfigPanelApi(format!("panel_api: unsupported type={typ_str}, only m3u is supported"))) + } +} + +pub(super) fn require_api_key_param(params: &[PanelApiQueryParam], section: &str) -> Result<(), TuliproxError> { + let api_key = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("api_key")); + let Some(api_key) = api_key else { + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api: {section} must contain query param 'api_key' (use value 'auto')" + ))); + }; + if api_key.value.trim().is_empty() { + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api: {section} query param 'api_key' must not be empty (use value 'auto')" + ))); + } + Ok(()) +} + +pub(super) fn require_username_password_params_auto( + params: &[PanelApiQueryParam], + section: &str, +) -> Result<(), TuliproxError> { + let username = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("username")); + let password = params.iter().find(|p| p.key.trim().eq_ignore_ascii_case("password")); + if username.is_none() || password.is_none() { + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api: {section} must contain query params 'username' and 'password' (use value 'auto')" + ))); + } + if !username.is_some_and(|p| p.value.trim().eq_ignore_ascii_case("auto")) + || !password.is_some_and(|p| p.value.trim().eq_ignore_ascii_case("auto")) + { + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api: {section} requires 'username: auto' and 'password: auto' (credentials must not be hardcoded)" + ))); + } + Ok(()) +} + +pub(super) fn validate_client_new_params(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { + require_api_key_param(params, "query_parameter.client_new")?; + validate_type_is_m3u(params)?; + if params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("user")) { + return Err(TuliproxError::ConfigPanelApi( + "panel_api: client_new must not contain query param 'user'".to_string(), + )); + } + Ok(()) +} + +pub(super) fn validate_client_renew_params(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { + require_api_key_param(params, "query_parameter.client_renew")?; + validate_type_is_m3u(params)?; + require_username_password_params_auto(params, "query_parameter.client_renew")?; + Ok(()) +} + +pub(super) fn validate_client_info_params(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { + require_api_key_param(params, "query_parameter.client_info")?; + require_username_password_params_auto(params, "query_parameter.client_info")?; + Ok(()) +} + +pub(super) fn validate_account_info_params(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { + require_api_key_param(params, "query_parameter.account_info")?; + let has_user = params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("username")); + let has_pass = params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("password")); + if has_user || has_pass { + require_username_password_params_auto(params, "query_parameter.account_info")?; + } + Ok(()) +} + +pub(super) fn validate_client_adult_content_params(params: &[PanelApiQueryParam]) -> Result<(), TuliproxError> { + require_api_key_param(params, "query_parameter.client_adult_content")?; + let has_user = params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("username")); + let has_pass = params.iter().any(|p| p.key.trim().eq_ignore_ascii_case("password")); + if has_user || has_pass { + require_username_password_params_auto(params, "query_parameter.client_adult_content")?; + } + Ok(()) +} + +pub(super) fn validate_panel_api_config(cfg: &PanelApiConfig) -> Result<(), TuliproxError> { + if !cfg.enabled { + return Ok(()); + } + if cfg.url.trim().is_empty() { + return Err(TuliproxError::ConfigPanelApi("panel_api: url is missing".to_string())); + } + if cfg.api_key.as_ref().is_none_or(|k| k.trim().is_empty()) { + return Err(TuliproxError::ConfigPanelApi("panel_api: api_key is missing".to_string())); + } + if cfg.query_parameter.client_info.is_empty() { + return Err(TuliproxError::ConfigPanelApi( + "panel_api: query_parameter.client_info must be configured".to_string(), + )); + } + validate_client_info_params(&cfg.query_parameter.client_info)?; + if !cfg.query_parameter.client_new.is_empty() { + validate_client_new_params(&cfg.query_parameter.client_new)?; + } + if !cfg.query_parameter.client_renew.is_empty() { + validate_client_renew_params(&cfg.query_parameter.client_renew)?; + } + if !cfg.query_parameter.account_info.is_empty() { + validate_account_info_params(&cfg.query_parameter.account_info)?; + } + if !cfg.query_parameter.client_adult_content.is_empty() { + validate_client_adult_content_params(&cfg.query_parameter.client_adult_content)?; + } + let (min_val, max_val) = alias_pool_limit_values(cfg); + if let Some(PanelApiAliasPoolSizeValue::Number(value)) = min_val { + if *value == 0 { + return Err(TuliproxError::ConfigPanelApi( + "panel_api.alias_pool.size.min must be greater than 0".to_string(), + )); + } + } + if let Some(PanelApiAliasPoolSizeValue::Number(value)) = max_val { + if *value == 0 { + return Err(TuliproxError::ConfigPanelApi( + "panel_api.alias_pool.size.max must be greater than 0".to_string(), + )); + } + } + let min = min_val.and_then(PanelApiAliasPoolSizeValue::as_number); + let max = max_val.and_then(PanelApiAliasPoolSizeValue::as_number); + if let (Some(min), Some(max)) = (min, max) { + if min > max { + return Err(TuliproxError::ConfigPanelApi( + "panel_api.alias_pool.size.min must be <= panel_api.alias_pool.size.max".to_string(), + )); + } + } + if cfg.provisioning.probe_interval_sec == 0 { + return Err(TuliproxError::ConfigPanelApi( + "panel_api.provisioning.probe_interval_sec must be greater than 0".to_string(), + )); + } + if let Some(offset) = cfg.provisioning.offset.as_deref() { + let _secs = parse_panel_api_provisioning_offset_secs(offset)?; + } + Ok(()) +} + +pub(super) fn resolve_query_params( + params: &[PanelApiQueryParam], + api_key: Option<&str>, + creds: Option<(&str, &str)>, +) -> Result, TuliproxError> { + let mut out = Vec::with_capacity(params.len()); + for p in params { + let key = p.key.trim(); + if key.is_empty() { + continue; + } + let mut value = p.value.trim().to_string(); + if value.eq_ignore_ascii_case("auto") { + if key.eq_ignore_ascii_case("api_key") { + let Some(k) = api_key.filter(|s| !s.trim().is_empty()) else { + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api: query param {key} uses 'auto' but panel_api.api_key is missing" + ))); + }; + value = k.to_string(); + } else if key.eq_ignore_ascii_case("username") { + let Some((u, _)) = creds else { + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api: query param {key} uses 'auto' but no account username is available" + ))); + }; + value = u.to_string(); + } else if key.eq_ignore_ascii_case("password") { + let Some((_, pw)) = creds else { + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api: query param {key} uses 'auto' but no account password is available" + ))); + }; + value = pw.to_string(); + } + } + out.push((key.to_string(), value)); + } + Ok(out) +} + +pub(super) fn build_panel_url(base_url: &str, query_params: &[(String, String)]) -> Result { + let mut url = Url::parse(base_url) + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: invalid url {base_url}: {e}")))?; + { + let mut pairs = url.query_pairs_mut(); + for (k, v) in query_params { + pairs.append_pair(k, v); + } + } + Ok(url) +} + +pub(super) fn require_batch_alias_path(input: &ConfigInput) -> Result { + resolve_batch_alias_path(input.t_batch_url.as_deref())?.ok_or_else(|| { + TuliproxError::ConfigInput(format!("batch input '{}' does not define a CSV alias path", input.name)) + }) +} diff --git a/backend/app/src/api/panel_api/probe.rs b/backend/app/src/api/panel_api/probe.rs new file mode 100644 index 000000000..641f9bba7 --- /dev/null +++ b/backend/app/src/api/panel_api/probe.rs @@ -0,0 +1,241 @@ +use super::{find_input_by_provider_name, try_provision_account_on_exhausted}; +use crate::{ + api::model::AppState, + model::{ConfigInput, InputSource}, + utils::debug_if_enabled, +}; +use axum::http::Method; +use shared::{ + concat_string, + error::TuliproxError, + model::{DisconnectReason, PanelApiProvisioningMethod, VirtualId}, + utils::sanitize_sensitive_info, +}; +use std::{net::SocketAddr, sync::Arc, time::Duration}; +use tokio_util::sync::CancellationToken; +use url::Url; + +pub(super) fn provisioning_method_to_reqwest(method: PanelApiProvisioningMethod) -> Method { + match method { + PanelApiProvisioningMethod::Head => Method::HEAD, + PanelApiProvisioningMethod::Get => Method::GET, + PanelApiProvisioningMethod::Post => Method::POST, + } +} + +pub(super) fn build_player_api_action_url(base_url: &str, username: &str, password: &str, action: &str) -> Option { + let url = Url::parse(base_url).ok()?; + let host = url.host_str()?; + let scheme = url.scheme(); + let mut base = concat_string!(scheme, "://", host); + if let Some(port) = url.port() { + base.push(':'); + base.push_str(&port.to_string()); + } + base.push_str("/player_api.php"); + let mut test_url = Url::parse(&base).ok()?; + test_url + .query_pairs_mut() + .append_pair("username", username) + .append_pair("password", password) + .append_pair("action", action); + Some(test_url) +} + +pub(super) enum PanelApiProbeTarget { + PlayerApi { action: &'static str, input_source: InputSource }, +} + +impl PanelApiProbeTarget { + pub(super) fn action(&self) -> &'static str { + match self { + PanelApiProbeTarget::PlayerApi { action, .. } => action, + } + } +} + +pub(super) fn format_probe_target_actions(targets: &[PanelApiProbeTarget]) -> String { + let mut result = String::new(); + for (idx, target) in targets.iter().enumerate() { + if idx > 0 { + result.push(','); + } + result.push_str(target.action()); + } + result +} + +pub(super) fn build_panel_api_probe_targets( + input: &ConfigInput, + username: &str, + password: &str, +) -> Vec { + let mut targets = Vec::new(); + for action in ["client_info", "get_live_categories", "get_series_categories", "get_vod_categories"] { + if let Some(url) = build_player_api_action_url(input.url.as_str(), username, password, action) { + targets.push(PanelApiProbeTarget::PlayerApi { + action, + input_source: InputSource::from(input).with_url(url.to_string()), + }); + } + } + targets +} + +#[allow(clippy::too_many_lines)] +pub(crate) async fn run_panel_api_provisioning_probe( + app_state: Arc, + input_name: Arc, + stop_signal: CancellationToken, + addr: SocketAddr, + virtual_id: VirtualId, +) -> Result<(), TuliproxError> { + if stop_signal.is_cancelled() { + return Ok(()); + } + let provisioning_kick_secs = 0; + let Some(input) = find_input_by_provider_name(app_state.as_ref(), input_name.as_ref()) else { + debug_if_enabled!( + "panel_api provisioning probe skipped (input no longer exists) for input {}", + sanitize_sensitive_info(input_name.as_ref()) + ); + stop_signal.cancel(); + let _ = app_state + .connection_manager + .close_connection_with_reason_and_block( + &addr, + virtual_id, + provisioning_kick_secs, + DisconnectReason::Provisioning, + ) + .await; + return Ok(()); + }; + + let Some(panel_cfg) = input.panel_api.as_ref() else { + debug_if_enabled!( + "panel_api provisioning probe skipped (missing config) for input {}", + sanitize_sensitive_info(&input.name) + ); + stop_signal.cancel(); + let _ = app_state + .connection_manager + .close_connection_with_reason_and_block( + &addr, + virtual_id, + provisioning_kick_secs, + DisconnectReason::Provisioning, + ) + .await; + return Ok(()); + }; + if !panel_cfg.enabled { + debug_if_enabled!( + "panel_api provisioning probe skipped (panel_api.enabled false) for input {}", + sanitize_sensitive_info(&input.name) + ); + stop_signal.cancel(); + let _ = app_state + .connection_manager + .close_connection_with_reason_and_block( + &addr, + virtual_id, + provisioning_kick_secs, + DisconnectReason::Provisioning, + ) + .await; + return Ok(()); + } + if panel_cfg.url.trim().is_empty() { + debug_if_enabled!( + "panel_api provisioning probe skipped (panel_api.url empty) for input {}", + sanitize_sensitive_info(&input.name) + ); + stop_signal.cancel(); + let _ = app_state + .connection_manager + .close_connection_with_reason_and_block( + &addr, + virtual_id, + provisioning_kick_secs, + DisconnectReason::Provisioning, + ) + .await; + return Ok(()); + } + + let max_wait_secs = panel_cfg.provisioning.timeout_sec; + + debug_if_enabled!( + "panel_api provisioning probe start for input {} (timeout={}s)", + sanitize_sensitive_info(&input.name), + max_wait_secs + ); + + let outcome = tokio::select! { + () = stop_signal.cancelled() => return Ok(()), + outcome = try_provision_account_on_exhausted(&app_state, &input_name) => outcome, + }; + + if let Some(outcome) = outcome.as_ref() { + debug_if_enabled!( + "panel_api provisioning {} completed for input {}", + outcome.kind_label(), + sanitize_sensitive_info(&input.name) + ); + } else { + debug_if_enabled!( + "panel_api provisioning failed for input {}; waiting for timeout", + sanitize_sensitive_info(&input.name) + ); + } + + if outcome.is_none() { + if max_wait_secs > 0 { + tokio::select! { + () = stop_signal.cancelled() => return Ok(()), + () = tokio::time::sleep(Duration::from_secs(max_wait_secs)) => {} + } + } + debug_if_enabled!( + "panel_api provisioning probe timeout reached for input {} (no credentials)", + sanitize_sensitive_info(&input.name) + ); + stop_signal.cancel(); + debug_if_enabled!( + "panel_api provisioning closing client connection for input {} addr={}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&addr.to_string()) + ); + let _ = app_state + .connection_manager + .close_connection_with_reason_and_block( + &addr, + virtual_id, + provisioning_kick_secs, + DisconnectReason::Provisioning, + ) + .await; + return Ok(()); + } + + if stop_signal.is_cancelled() { + return Ok(()); + } + stop_signal.cancel(); + debug_if_enabled!( + "panel_api provisioning closing client connection for input {} addr={}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&addr.to_string()) + ); + let _ = app_state + .connection_manager + .close_connection_with_reason_and_block( + &addr, + virtual_id, + provisioning_kick_secs, + DisconnectReason::Provisioning, + ) + .await; + Ok(()) +} diff --git a/backend/app/src/api/panel_api/provisioning.rs b/backend/app/src/api/panel_api/provisioning.rs new file mode 100644 index 000000000..4a6452b4f --- /dev/null +++ b/backend/app/src/api/panel_api/provisioning.rs @@ -0,0 +1,949 @@ +use super::{ + build_panel_api_probe_targets, collect_accounts, compare_alias_exp_date_config, format_probe_target_actions, + is_alias_pool_max_reached, is_expiring_with_offset_at, panel_client_adult_content, panel_client_info, + panel_client_new, panel_client_renew, provisioning_method_to_reqwest, refresh_internal_sources, + require_batch_alias_path, run_panel_api_provisioning_probe, should_reload_sources_after_internal_write, + validate_panel_api_config, PanelApiOptionalFlags, PanelApiOptionalFlagsSet, PanelApiProbeTarget, + PanelApiProvisionOutcome, +}; +use crate::{ + api::{ + internal_csv::{ + csv_patch_batch_append, csv_patch_batch_sort_by_exp_date, csv_patch_batch_update_credentials, + csv_patch_batch_update_exp_date, + }, + model::{ + create_panel_api_provisioning_stream_with_stop, create_provider_connections_exhausted_stream, AppState, + StreamDetails, + }, + source_yml_patch::{derive_unique_alias_name, resolve_provisioned_account_base_url, SourcesYmlPatch}, + }, + model::{ + is_input_expired, is_input_expired_at, ConfigInput, ConfigInputAlias, GracePeriodOptions, InputSource, + PanelApiConfig, + }, + repository::AliasExpDateSortOrder, + utils::{debug_if_enabled, format_http_status, request}, +}; +use axum::http::StatusCode; +use jsonwebtoken::get_current_timestamp; +use log::{error, warn}; +use shared::{ + error::{string_to_io_error, TuliproxError}, + model::{InputType, PanelApiProvisioningMethod, VirtualId}, + utils::sanitize_sensitive_info, +}; +use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, + io, + net::SocketAddr, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio_util::sync::CancellationToken; +use url::Url; + +pub(super) fn is_date_only_yyyy_mm_dd(value: &str) -> bool { + let value = value.trim(); + if value.len() != 10 { + return false; + } + let bytes = value.as_bytes(); + bytes[4] == b'-' + && bytes[7] == b'-' + && bytes[0..4].iter().all(u8::is_ascii_digit) + && bytes[5..7].iter().all(u8::is_ascii_digit) + && bytes[8..10].iter().all(u8::is_ascii_digit) +} + +pub(super) fn resolve_panel_api_optional_flags(cfg: &PanelApiConfig, input_name: &str) -> PanelApiOptionalFlagsSet { + let mut flags = PanelApiOptionalFlagsSet::new(); + if !cfg.query_parameter.account_info.is_empty() { + flags.set(PanelApiOptionalFlags::AccountInfo); + } + if !cfg.query_parameter.client_new.is_empty() { + flags.set(PanelApiOptionalFlags::ClientNew); + } + if !cfg.query_parameter.client_renew.is_empty() { + flags.set(PanelApiOptionalFlags::ClientRenew); + } + if !cfg.query_parameter.client_adult_content.is_empty() { + flags.set(PanelApiOptionalFlags::AdultContent); + } + + let name = sanitize_sensitive_info(input_name); + if !flags.contains(PanelApiOptionalFlags::ClientRenew) { + debug_if_enabled!("panel_api request for client_renew disabled due to missing arguments for {}", name); + } + if !flags.contains(PanelApiOptionalFlags::ClientNew) { + debug_if_enabled!("panel_api request for client_new disabled due to missing arguments for {}", name); + } + if !flags.contains(PanelApiOptionalFlags::AdultContent) { + debug_if_enabled!("panel_api request for client_adult_content disabled due to missing arguments for {}", name); + } + if !flags.contains(PanelApiOptionalFlags::AccountInfo) { + debug_if_enabled!("panel_api request for account_info disabled due to missing arguments for {}", name); + } + flags +} + +pub(super) fn parse_panel_api_provisioning_offset_secs(offset: &str) -> Result { + let raw = offset.trim(); + if raw.is_empty() { + return Ok(0); + } + let lower = raw.to_ascii_lowercase(); + let bytes = lower.as_bytes(); + let last = *bytes.last().unwrap_or(&b'\0'); + let (num_part, multiplier) = match last { + b's' => (&lower[..lower.len().saturating_sub(1)], 1_u64), + b'm' => (&lower[..lower.len().saturating_sub(1)], 60_u64), + b'h' => (&lower[..lower.len().saturating_sub(1)], 60_u64 * 60), + b'd' => (&lower[..lower.len().saturating_sub(1)], 60_u64 * 60 * 24), + b'0'..=b'9' => (lower.as_str(), 1_u64), + _ => { + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api.provisioning.offset must be a number with optional suffix s/m/h/d (e.g. 30m, 12h), got '{raw}'" + ))); + } + }; + let num_part = num_part.trim(); + if num_part.is_empty() { + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api.provisioning.offset must be a number with optional suffix s/m/h/d (e.g. 30m, 12h), got '{raw}'" + ))); + } + let value: u64 = num_part.parse().map_err(|_| { + TuliproxError::ConfigPanelApi(format!("panel_api.provisioning.offset is not a valid number: '{raw}'")) + })?; + value + .checked_mul(multiplier) + .ok_or_else(|| TuliproxError::ConfigPanelApi(format!("panel_api.provisioning.offset is too large: '{raw}'"))) +} + +pub(super) fn aliases_need_sort_config(aliases: &[ConfigInputAlias]) -> bool { + if aliases.len() < 2 { + return false; + } + aliases.windows(2).any(|pair| compare_alias_exp_date_config(&pair[0], &pair[1]) == Ordering::Greater) +} + +pub(crate) fn can_provision_on_exhausted(app_state: &AppState, input: &ConfigInput) -> bool { + let Some(panel_cfg) = input.panel_api.as_ref() else { + return false; + }; + if !panel_cfg.enabled { + return false; + } + if panel_cfg.url.trim().is_empty() { + return false; + } + if let Err(err) = validate_panel_api_config(panel_cfg) { + debug_if_enabled!("panel_api config invalid: {}", sanitize_sensitive_info(&err.to_string())); + return false; + } + !is_alias_pool_max_reached(app_state, input) +} + +pub(crate) fn find_input_by_provider_name(app_state: &AppState, provider_name: &str) -> Option> { + let sources = app_state.app_config.sources.load(); + for input in &sources.inputs { + if &*input.name == provider_name { + return Some(Arc::clone(input)); + } + if input.aliases.as_ref().is_some_and(|aliases| aliases.iter().any(|alias| &*alias.name == provider_name)) { + return Some(Arc::clone(input)); + } + } + None +} + +impl PanelApiProvisionOutcome { + pub(crate) fn kind_label(&self) -> &'static str { + match self { + Self::Renewed => "client_renew", + Self::Created => "client_new", + } + } +} + +#[allow(clippy::too_many_lines)] +#[allow(clippy::too_many_arguments)] +pub(super) async fn try_renew_expired_account( + app_state: &Arc, + input: &ConfigInput, + panel_cfg: &PanelApiConfig, + is_batch: bool, + sources_path: &Path, + treat_missing_exp_date_as_expired: bool, + include_root: bool, + optional: PanelApiOptionalFlagsSet, +) -> Option { + if !optional.contains(PanelApiOptionalFlags::ClientRenew) { + return None; + } + let adult_enabled = optional.contains(PanelApiOptionalFlags::AdultContent); + let mut candidates = collect_accounts(input); + if !include_root { + candidates.retain(|acct| acct.name != input.name); + } + for acct in &mut candidates { + if treat_missing_exp_date_as_expired && acct.exp_date.is_none() { + acct.exp_date = + panel_client_info(app_state, panel_cfg, acct.username.as_str(), acct.password.as_str(), None) + .await + .ok() + .flatten(); + } + } + candidates.sort_by_key(|a| a.exp_date.unwrap_or(i64::MAX)); + + for acct in &candidates { + // Only attempt renew/new when the account is *known* to be expired. + // If exp_date is missing (even after an optional client_info refresh), we skip renewal. + let expired = is_input_expired(acct.exp_date); + if !expired { + continue; + } + match panel_client_renew(app_state, panel_cfg, acct.username.as_str(), acct.password.as_str()).await { + Ok(()) => { + if adult_enabled { + if let Err(err) = panel_client_adult_content( + app_state, + panel_cfg, + Some((acct.username.as_str(), acct.password.as_str())), + ) + .await + { + debug_if_enabled!( + "panel_api client_adult_content failed for {}: {}", + sanitize_sensitive_info(&acct.name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + let refreshed_exp = + panel_client_info(app_state, panel_cfg, acct.username.as_str(), acct.password.as_str(), None) + .await + .ok() + .flatten(); + + if let Some(new_exp) = refreshed_exp.or(acct.exp_date) { + if is_batch { + match require_batch_alias_path(input) { + Ok(csv_path) => { + let csv_lock = app_state.app_config.file_locks.write_lock(&csv_path).await; + if let Err(err) = csv_patch_batch_update_exp_date( + app_state, + &csv_lock, + input.input_type, + &csv_path, + &acct.name, + &acct.username, + &acct.password, + new_exp, + ) + .await + { + debug_if_enabled!("panel_api failed to persist renew exp_date to csv: {}", err); + } + if let Err(err) = csv_patch_batch_sort_by_exp_date( + app_state, + &csv_lock, + input.input_type, + &csv_path, + AliasExpDateSortOrder::NewestFirst, + ) + .await + { + debug_if_enabled!("panel_api failed to sort csv accounts after renew: {}", err); + } + } + Err(err) => debug_if_enabled!("panel_api cannot resolve batch csv path: {}", err), + } + } else { + let patches = [ + SourcesYmlPatch::UpdatePanelAccountExpiry { + input_name: input.name.clone(), + account_name: Arc::clone(&acct.name), + exp_date: new_exp, + }, + SourcesYmlPatch::SortAliases { + input_name: input.name.clone(), + order: AliasExpDateSortOrder::NewestFirst, + }, + ]; + if let Err(err) = crate::api::source_yml_patch::execute_account_source_patches( + app_state, + sources_path, + &patches, + ) + .await + { + debug_if_enabled!("panel_api failed to persist renew exp_date to source.yml: {}", err); + } + } + } + + if should_reload_sources_after_internal_write(app_state.as_ref()) { + refresh_internal_sources(app_state); + } + return Some(PanelApiProvisionOutcome::Renewed); + } + Err(err) => { + debug_if_enabled!( + "panel_api client_renew failed for {}: {}", + sanitize_sensitive_info(&acct.name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + } + None +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +pub(super) async fn try_refresh_root_account_on_exhausted( + app_state: &Arc, + input: &ConfigInput, + panel_cfg: &PanelApiConfig, + is_batch: bool, + sources_path: &Path, + optional: PanelApiOptionalFlagsSet, +) -> Option { + let renew_enabled = optional.contains(PanelApiOptionalFlags::ClientRenew); + let new_enabled = optional.contains(PanelApiOptionalFlags::ClientNew); + if !renew_enabled && !new_enabled { + return None; + } + + let now = get_current_timestamp(); + let offset_secs = panel_cfg + .provisioning + .offset + .as_deref() + .and_then(|value| parse_panel_api_provisioning_offset_secs(value).ok()) + .unwrap_or(0); + let root_exp_missing = input.exp_date.is_none(); + let root_expired = is_input_expired_at(input.exp_date, now); + let root_expiring = is_expiring_with_offset_at(input.exp_date, offset_secs, now); + if !root_exp_missing && !root_expired && !root_expiring { + return None; + } + + let old_username = input.username.clone().unwrap_or_default(); + let old_password = input.password.clone().unwrap_or_default(); + let adult_enabled = optional.contains(PanelApiOptionalFlags::AdultContent); + + let (outcome, active_username, active_password, credentials_changed) = + if renew_enabled && !old_username.is_empty() && !old_password.is_empty() { + match panel_client_renew(app_state, panel_cfg, old_username.as_str(), old_password.as_str()).await { + Ok(()) => (PanelApiProvisionOutcome::Renewed, old_username.clone(), old_password.clone(), false), + Err(err) => { + debug_if_enabled!( + "panel_api client_renew failed for root {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + if !new_enabled { + return None; + } + match panel_client_new(app_state, panel_cfg).await { + Ok((username, password, _base_url_from_resp)) => { + (PanelApiProvisionOutcome::Created, username, password, true) + } + Err(err) => { + debug_if_enabled!( + "panel_api client_new failed for root {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + return None; + } + } + } + } + } else if new_enabled { + match panel_client_new(app_state, panel_cfg).await { + Ok((username, password, _base_url_from_resp)) => { + (PanelApiProvisionOutcome::Created, username, password, true) + } + Err(err) => { + debug_if_enabled!( + "panel_api client_new failed for root {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + return None; + } + } + } else { + return None; + }; + + if adult_enabled { + if let Err(err) = + panel_client_adult_content(app_state, panel_cfg, Some((active_username.as_str(), active_password.as_str()))) + .await + { + debug_if_enabled!( + "panel_api client_adult_content failed for root {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + + if !wait_for_panel_api_account_ready( + app_state, + input, + panel_cfg, + input.name.as_ref(), + active_username.as_str(), + active_password.as_str(), + ) + .await + { + debug_if_enabled!( + "panel_api root account not ready after probe/cooldown for {}", + sanitize_sensitive_info(&input.name) + ); + return None; + } + + let refreshed_exp_date = + panel_client_info(app_state, panel_cfg, active_username.as_str(), active_password.as_str(), None) + .await + .ok() + .flatten(); + let exp_date = if credentials_changed { refreshed_exp_date } else { refreshed_exp_date.or(input.exp_date) }; + + if is_batch { + let Ok(csv_path) = require_batch_alias_path(input) else { + return None; + }; + let csv_lock = app_state.app_config.file_locks.write_lock(&csv_path).await; + let result = if credentials_changed { + csv_patch_batch_update_credentials( + app_state, + &csv_lock, + input.input_type, + &csv_path, + &input.name, + old_username.as_str(), + old_password.as_str(), + active_username.as_str(), + active_password.as_str(), + exp_date, + ) + .await + } else if let Some(exp_date) = exp_date { + csv_patch_batch_update_exp_date( + app_state, + &csv_lock, + input.input_type, + &csv_path, + &input.name, + active_username.as_str(), + active_password.as_str(), + exp_date, + ) + .await + } else { + Ok(()) + }; + if let Err(err) = result { + debug_if_enabled!("panel_api failed to persist root provisioning to csv: {}", err); + return None; + } + if let Err(err) = csv_patch_batch_sort_by_exp_date( + app_state, + &csv_lock, + input.input_type, + &csv_path, + AliasExpDateSortOrder::NewestFirst, + ) + .await + { + debug_if_enabled!("panel_api failed to sort csv accounts after root provisioning: {}", err); + return None; + } + } else { + let patch = SourcesYmlPatch::PersistProvisionedAccount { + input_name: input.name.clone(), + username: active_username, + password: active_password, + exp_date, + }; + if let Err(err) = + crate::api::source_yml_patch::execute_account_source_patches(app_state, sources_path, &[patch]).await + { + debug_if_enabled!("panel_api failed to persist root provisioning to source.yml: {}", err); + return None; + } + } + + if should_reload_sources_after_internal_write(app_state.as_ref()) { + refresh_internal_sources(app_state); + } + + Some(outcome) +} + +#[allow(clippy::too_many_lines)] +pub(super) async fn try_create_new_account( + app_state: &Arc, + input: &ConfigInput, + panel_cfg: &PanelApiConfig, + is_batch: bool, + sources_path: &Path, + optional: PanelApiOptionalFlagsSet, +) -> Option { + if !optional.contains(PanelApiOptionalFlags::ClientNew) { + return None; + } + let adult_enabled = optional.contains(PanelApiOptionalFlags::AdultContent); + match panel_client_new(app_state, panel_cfg).await { + Ok((username, password, base_url_from_resp)) => { + let base_url = resolve_provisioned_account_base_url( + input.url.as_str(), + base_url_from_resp.as_deref(), + &username, + &password, + ); + + let mut existing_names: Vec> = vec![input.name.clone()]; + if let Some(aliases) = input.aliases.as_ref() { + existing_names.extend(aliases.iter().map(|a| a.name.clone())); + } + let alias_name = derive_unique_alias_name(&existing_names, &input.name, &username); + + if adult_enabled { + if let Err(err) = panel_client_adult_content(app_state, panel_cfg, Some((&username, &password))).await { + debug_if_enabled!( + "panel_api client_adult_content failed for {}: {}", + sanitize_sensitive_info(&alias_name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + + let exp_date = panel_client_info(app_state, panel_cfg, &username, &password, None).await.ok().flatten(); + + if !wait_for_panel_api_account_ready( + app_state, + input, + panel_cfg, + alias_name.as_ref(), + username.as_str(), + password.as_str(), + ) + .await + { + debug_if_enabled!( + "panel_api client_new account not ready after probe/cooldown for {}", + sanitize_sensitive_info(alias_name.as_ref()) + ); + return None; + } + + if is_batch { + match require_batch_alias_path(input) { + Ok(csv_path) => { + let batch_type = if input.input_type == InputType::Xtream { + InputType::XtreamBatch + } else { + InputType::M3uBatch + }; + let csv_lock = app_state.app_config.file_locks.write_lock(&csv_path).await; + if let Err(err) = csv_patch_batch_append( + app_state, + &csv_lock, + &csv_path, + batch_type, + &alias_name, + &base_url, + &username, + &password, + exp_date, + ) + .await + { + warn!("panel_api failed to append new account to csv: {err}"); + return None; + } + if let Err(err) = csv_patch_batch_sort_by_exp_date( + app_state, + &csv_lock, + batch_type, + &csv_path, + AliasExpDateSortOrder::NewestFirst, + ) + .await + { + warn!("panel_api failed to sort csv accounts after append: {err}"); + return None; + } + } + Err(err) => { + warn!( + "panel_api cannot resolve batch csv path for {}: {}", + sanitize_sensitive_info(input.name.as_ref()), + err, + ); + return None; + } + } + } else { + let patches = [ + SourcesYmlPatch::AddAlias { + input_name: input.name.clone(), + alias_name: Arc::clone(&alias_name), + base_url, + username, + password, + exp_date, + }, + SourcesYmlPatch::SortAliases { + input_name: input.name.clone(), + order: AliasExpDateSortOrder::NewestFirst, + }, + ]; + if let Err(err) = + crate::api::source_yml_patch::execute_account_source_patches(app_state, sources_path, &patches) + .await + { + warn!("panel_api failed to persist new alias to source.yml: {err}"); + return None; + } + } + + if should_reload_sources_after_internal_write(app_state.as_ref()) { + refresh_internal_sources(app_state); + } + Some(PanelApiProvisionOutcome::Created) + } + Err(err) => { + debug_if_enabled!("panel_api client_new failed: {}", sanitize_sensitive_info(&err.to_string())); + None + } + } +} + +pub async fn try_provision_account_on_exhausted( + app_state: &Arc, + input_name: &Arc, +) -> Option { + let _input_lock = app_state.app_config.file_locks.write_lock_str(format!("panel_api:{input_name}").as_str()).await; + let current_input = find_input_by_provider_name(app_state.as_ref(), input_name.as_ref()); + let Some(input) = current_input.as_deref() else { + debug_if_enabled!( + "panel_api: skipped (input no longer exists) for input {}", + sanitize_sensitive_info(input_name.as_ref()) + ); + return None; + }; + + let Some(panel_cfg) = input.panel_api.as_ref() else { + debug_if_enabled!( + "panel_api: skipped (no panel_api config) for input {}", + sanitize_sensitive_info(&input.name) + ); + return None; + }; + if !panel_cfg.enabled { + debug_if_enabled!( + "panel_api: skipped (panel_api.enabled false) for input {}", + sanitize_sensitive_info(&input.name) + ); + return None; + } + if panel_cfg.url.trim().is_empty() { + debug_if_enabled!( + "panel_api: skipped (panel_api.url empty) for input {}", + sanitize_sensitive_info(&input.name) + ); + return None; + } + + if let Err(err) = validate_panel_api_config(panel_cfg) { + debug_if_enabled!("panel_api config invalid: {}", sanitize_sensitive_info(&err.to_string())); + return None; + } + let optional = resolve_panel_api_optional_flags(panel_cfg, &input.name); + debug_if_enabled!( + "panel_api: exhausted -> provisioning for input {} (aliases={})", + sanitize_sensitive_info(&input.name), + input.aliases.as_ref().map_or(0, Vec::len) + ); + + let is_batch = input.t_batch_url.as_ref().is_some_and(|u| !u.trim().is_empty()); + let sources_file_path = app_state.app_config.paths.load().sources_file_path.clone(); + let sources_path = PathBuf::from(&sources_file_path); + + if let Some(outcome) = + try_refresh_root_account_on_exhausted(app_state, input, panel_cfg, is_batch, sources_path.as_path(), optional) + .await + { + debug_if_enabled!( + "panel_api: provisioning succeeded via root {} for input {}", + outcome.kind_label(), + sanitize_sensitive_info(&input.name) + ); + return Some(outcome); + } + + if is_alias_pool_max_reached(app_state, input) { + return None; + } + + if let Some(outcome) = + try_renew_expired_account(app_state, input, panel_cfg, is_batch, sources_path.as_path(), true, false, optional) + .await + { + debug_if_enabled!( + "panel_api: provisioning succeeded via client_renew for input {}", + sanitize_sensitive_info(&input.name) + ); + return Some(outcome); + } + let created = try_create_new_account(app_state, input, panel_cfg, is_batch, sources_path.as_path(), optional).await; + debug_if_enabled!( + "panel_api: provisioning via client_new for input {} => {}", + sanitize_sensitive_info(&input.name), + if created.is_some() { "success" } else { "failed" } + ); + created +} + +pub(super) async fn probe_panel_api_targets( + app_state: &Arc, + probe_method: PanelApiProvisioningMethod, + targets: &[PanelApiProbeTarget], + done: &mut HashSet<&'static str>, +) -> bool { + for target in targets { + let action = target.action(); + if done.contains(action) { + continue; + } + match target { + PanelApiProbeTarget::PlayerApi { action, input_source } => { + match probe_panel_api_test_url(app_state, input_source, probe_method).await { + Ok(status) => { + debug_if_enabled!( + "panel_api probe status: '{}' action={} url: {}", + format_http_status(status), + action, + sanitize_sensitive_info(input_source.url.as_str()) + ); + if status.is_success() { + done.insert(action); + } + } + Err(err) => { + if err.kind() == io::ErrorKind::TimedOut { + debug_if_enabled!( + "panel_api probe timeout action={} url: {}", + action, + sanitize_sensitive_info(input_source.url.as_str()) + ); + } else { + debug_if_enabled!( + "panel_api probe failed action={} url: {}: {err}", + action, + sanitize_sensitive_info(input_source.url.as_str()) + ); + } + } + } + } + } + } + done.len() == targets.len() +} + +pub(super) async fn probe_panel_api_test_url( + app_state: &Arc, + input_source: &InputSource, + method: PanelApiProvisioningMethod, +) -> Result { + let client = app_state.http_clients.default.load(); + let request_method = provisioning_method_to_reqwest(method); + let test_url = Url::parse(input_source.url.as_str()).map_err(|err| { + string_to_io_error(format!( + "Malformed URL {}: {}", + sanitize_sensitive_info(input_source.url.as_str()), + sanitize_sensitive_info(err.to_string().as_str()) + )) + })?; + + let config = app_state.app_config.config.load(); + let default_user_agent = config.default_user_agent.clone(); + let disabled_headers = config.get_disabled_headers(); + drop(config); + + let headers = request::get_request_headers( + Some(&input_source.headers), + None::<&HashMap>>, + disabled_headers.as_ref(), + default_user_agent.as_deref(), + ); + let response = request::send_with_retry_and_provider( + &app_state.app_config, + &test_url, + input_source.get_provider(), + false, + |resolved_url| client.request(request_method.clone(), resolved_url.clone()).headers(headers.clone()), + ) + .await?; + Ok(response.status()) +} + +pub(super) async fn apply_provisioning_cooldown(panel_cfg: &PanelApiConfig, account_name: &str, input_name: &Arc) { + let cooldown_secs = panel_cfg.provisioning.cooldown_sec; + if cooldown_secs == 0 { + return; + } + debug_if_enabled!( + "panel_api provisioning cooldown for {} (input={}): {}s", + sanitize_sensitive_info(account_name), + sanitize_sensitive_info(input_name), + cooldown_secs + ); + tokio::time::sleep(Duration::from_secs(cooldown_secs)).await; +} + +pub(crate) async fn wait_for_panel_api_account_ready( + app_state: &Arc, + input: &ConfigInput, + panel_cfg: &PanelApiConfig, + account_name: &str, + username: &str, + password: &str, +) -> bool { + let max_wait_secs = panel_cfg.provisioning.timeout_sec; + let probe_interval_secs = panel_cfg.provisioning.probe_interval_sec.max(1); + let probe_method = panel_cfg.provisioning.method; + + let probe_targets = build_panel_api_probe_targets(input, username, password); + if probe_targets.is_empty() { + debug_if_enabled!( + "panel_api probe skipped for {} (input={}): no probe targets", + sanitize_sensitive_info(account_name), + sanitize_sensitive_info(&input.name) + ); + return false; + } + + let targets_list = format_probe_target_actions(&probe_targets); + debug_if_enabled!( + "panel_api probe start for {} (input={} timeout={}s interval={}s method={}) targets={}", + sanitize_sensitive_info(account_name), + sanitize_sensitive_info(&input.name), + max_wait_secs, + probe_interval_secs, + probe_method, + targets_list + ); + + let deadline = Instant::now() + Duration::from_secs(max_wait_secs); + let probe_delay = Duration::from_secs(probe_interval_secs); + let mut done_targets = HashSet::new(); + let mut attempt = 0u64; + loop { + attempt += 1; + debug_if_enabled!("panel_api probe attempt {}", attempt); + if probe_panel_api_targets(app_state, probe_method, &probe_targets, &mut done_targets).await { + apply_provisioning_cooldown(panel_cfg, account_name, &input.name).await; + return true; + } + + if max_wait_secs == 0 { + return false; + } + let now = Instant::now(); + if now >= deadline { + return false; + } + let remaining = deadline.checked_duration_since(now).unwrap_or_default(); + let sleep_for = if remaining < probe_delay { remaining } else { probe_delay }; + tokio::time::sleep(sleep_for).await; + } +} + +pub fn create_panel_api_provisioning_stream_details( + app_state: &Arc, + input: &ConfigInput, + provider_name: Option>, + grace_period_options: &GracePeriodOptions, + addr: SocketAddr, + virtual_id: VirtualId, +) -> StreamDetails { + let stop_signal = CancellationToken::new(); + let headers = [("connection".to_string(), "close".to_string())]; + let (stream, stream_info) = + create_panel_api_provisioning_stream_with_stop(&app_state.app_config, &headers, stop_signal.clone()); + + if stream.is_none() { + debug_if_enabled!( + "panel_api provisioning stream missing; falling back to provider exhausted for input {}", + sanitize_sensitive_info(&input.name) + ); + let (stream, stream_info) = create_provider_connections_exhausted_stream(&app_state.app_config, &[]); + return StreamDetails { + shared_subscriber_id: None, + stream, + stream_info, + provider_name, + request_url: None, + session_headers: None, + provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), + user_agent_stream_index: None, + grace_period: *grace_period_options, + provider_grace_active: false, + disable_provider_grace: true, + reconnect_flag: None, + provider_handle: None, + content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, + grace_resolution_context: None, + custom_reason: None, + response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), + session_registration: None, + }; + } + + let app_state_clone = Arc::clone(app_state); + let input_name = Arc::clone(&input.name); + let stop_clone = stop_signal.clone(); + tokio::spawn(async move { + if let Err(err) = + run_panel_api_provisioning_probe(app_state_clone, input_name, stop_clone, addr, virtual_id).await + { + error!("Error running Probe: {err:?}"); + } + }); + + StreamDetails { + shared_subscriber_id: None, + stream, + stream_info, + provider_name, + request_url: None, + session_headers: None, + provider_session_headers: tuliprox_session::ProviderSessionHeaders::default(), + user_agent_stream_index: None, + grace_period: *grace_period_options, + provider_grace_active: false, + disable_provider_grace: true, + reconnect_flag: None, + provider_handle: None, + content_representation: crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, + grace_resolution_context: None, + custom_reason: None, + response_mode: tuliprox_session::stream_options::StreamResponseMode::default(), + session_registration: None, + } +} diff --git a/backend/app/src/api/panel_api/sync.rs b/backend/app/src/api/panel_api/sync.rs new file mode 100644 index 000000000..ed84dd34a --- /dev/null +++ b/backend/app/src/api/panel_api/sync.rs @@ -0,0 +1,1274 @@ +use super::{ + alias_pool_has_min, alias_pool_remove_expired, aliases_need_sort_config, apply_clock_skew, collect_accounts, + count_enabled_proxy_users, count_valid_accounts_at, ensure_alias_pool_min, extract_account_creds_from_input, + fetch_root_user_api_info, is_expiring_with_offset_at, load_panel_api_time_cache, panel_account_info, + panel_api_time_cache_path, panel_client_adult_content, panel_client_info, panel_client_info_raw, panel_client_new, + panel_client_renew, parse_cached_tz, parse_panel_api_provisioning_offset_secs, persist_panel_api_time_cache, + resolve_alias_pool_min, resolve_panel_api_optional_flags, resolve_panel_expire_mode, root_counts_towards_pool, + root_counts_towards_pool_at, sort_account_aliases_keep_root_first, validate_panel_api_config, + wait_for_panel_api_account_ready, AccountCredentials, PanelApiOptionalFlags, PanelApiTimeCacheEntry, + PanelApiTimeContext, +}; +use crate::{ + api::{ + internal_csv::{ + csv_patch_batch_append, csv_patch_batch_remove_expired, csv_patch_batch_sort_by_exp_date, + csv_patch_batch_update_credentials, csv_patch_batch_update_exp_date, + }, + model::AppState, + source_yml_patch::{derive_unique_alias_name_set, resolve_provisioned_account_base_url, SourcesYmlPatch}, + }, + model::{is_input_expired_at, ConfigInput}, + repository::{get_csv_file_path, AliasExpDateSortOrder}, + utils::debug_if_enabled, +}; +use chrono_tz::Tz; +use jsonwebtoken::get_current_timestamp; +use log::warn; +use shared::{error::TuliproxError, model::InputType, utils::sanitize_sensitive_info}; +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + sync::Arc, +}; + +pub(super) fn refresh_internal_sources(app_state: &Arc) { + app_state.active_provider.update_config(&app_state.app_config); +} + +pub(super) fn should_reload_sources_after_internal_write(app_state: &AppState) -> bool { + !app_state.app_config.config.load().config_hot_reload +} + +pub(super) fn resolve_batch_alias_path(batch_url: Option<&str>) -> Result, TuliproxError> { + let Some(batch_url) = batch_url.filter(|url| !url.trim().is_empty()) else { + return Ok(None); + }; + get_csv_file_path(batch_url).map(Some).map_err(|err| TuliproxError::ConfigInput(format!("{err}"))) +} + +#[allow(clippy::too_many_lines)] +pub(super) async fn sync_panel_api_for_input_on_boot( + app_state: &Arc, + input: &Arc, + sources_path: &Path, +) -> bool { + let Some(panel_cfg) = input.panel_api.as_ref() else { + return false; + }; + if !panel_cfg.enabled || panel_cfg.url.trim().is_empty() { + return false; + } + + if let Err(err) = validate_panel_api_config(panel_cfg) { + debug_if_enabled!( + "panel_api boot sync skipped for {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + return false; + } + let optional = resolve_panel_api_optional_flags(panel_cfg, &input.name); + + let input_name = &input.name; + let _input_lock = app_state.app_config.file_locks.write_lock_str(format!("panel_api:{input_name}").as_str()).await; + + let mut any_change = false; + let csv_path = match resolve_batch_alias_path(input.t_batch_url.as_deref()) { + Ok(path) => path, + Err(err) => { + warn!( + "panel_api boot sync skipped alias mutations for batch input {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + return false; + } + }; + let mut sources_yml_patches: Vec = Vec::new(); + let mut pending_sources_yml = false; + let mut source_yml_sort_aliases_requested = false; + + let mut accounts = collect_accounts(input.as_ref()); + if panel_cfg.alias_pool.is_some() { + sort_account_aliases_keep_root_first(&mut accounts, &input.name); + } + let mut existing_names: HashSet> = accounts.iter().map(|a| a.name.clone()).collect(); + let mut newly_created_accounts: Vec = Vec::new(); + let mut time_ctx: Option = None; + let mut effective_now = get_current_timestamp(); + + let cache_path = panel_api_time_cache_path(app_state.as_ref()); + let mut time_cache = load_panel_api_time_cache(app_state.as_ref(), &cache_path).await; + let cached_entry = time_cache.inputs.get(input.name.as_ref()).cloned(); + + if panel_cfg.alias_pool.is_some() + && csv_path.is_none() + && input.aliases.as_ref().is_some_and(|aliases| aliases_need_sort_config(aliases)) + { + sources_yml_patches.push(SourcesYmlPatch::SortAliases { + input_name: input.name.clone(), + order: AliasExpDateSortOrder::NewestFirst, + }); + pending_sources_yml = true; + source_yml_sort_aliases_requested = true; + } + + if let Some((root_username, root_password)) = extract_account_creds_from_input(input.as_ref()) { + let user_info = fetch_root_user_api_info(app_state, input.as_ref()).await; + let (root_exp_date, server_tz, skew_secs) = if let Ok(Some(info)) = user_info { + let local_now = i64::try_from(get_current_timestamp()).unwrap_or(0); + let skew_secs = info.server_now_ts.unwrap_or(local_now) - local_now; + (info.exp_date, info.server_tz, Some(skew_secs)) + } else { + (None, None, None) + }; + + let panel_expire_raw = + match panel_client_info_raw(app_state.as_ref(), panel_cfg, root_username.as_str(), root_password.as_str()) + .await + { + Ok(expire) => expire, + Err(err) => { + debug_if_enabled!( + "panel_api root client_info (raw) failed for {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + None + } + }; + + let mut expire_mode = resolve_panel_expire_mode(root_exp_date, panel_expire_raw.as_deref(), server_tz); + let mut server_tz = server_tz; + let mut skew_secs = skew_secs.or_else(|| cached_entry.as_ref().and_then(|e| e.skew_secs)); + + if root_exp_date.is_none() { + if let Some(cached) = cached_entry.as_ref() { + expire_mode = cached.expire_mode; + if server_tz.is_none() { + server_tz = parse_cached_tz(cached.server_tz.clone()); + } + if skew_secs.is_none() { + skew_secs = cached.skew_secs; + } + } + } + + time_ctx = Some(PanelApiTimeContext { expire_mode, server_tz }); + effective_now = apply_clock_skew(get_current_timestamp(), skew_secs.unwrap_or_default()); + + time_cache.inputs.insert( + input.name.to_string(), + PanelApiTimeCacheEntry { expire_mode, server_tz: server_tz.map(|tz| tz.name().to_string()), skew_secs }, + ); + if let Err(err) = persist_panel_api_time_cache(app_state.as_ref(), &cache_path, &time_cache).await { + debug_if_enabled!( + "panel_api failed to persist time cache for {}: {}", + sanitize_sensitive_info(&input.name), + err + ); + } + + let server_tz_name = server_tz.as_ref().map_or("none", |tz| Tz::name(*tz)); + debug_if_enabled!( + "panel_api time context for input {}: expire_mode={:?}, tz={}, skew_secs={}", + sanitize_sensitive_info(&input.name), + expire_mode, + server_tz_name, + skew_secs.unwrap_or_default() + ); + } else if let Some(cached) = cached_entry.as_ref() { + let server_tz = parse_cached_tz(cached.server_tz.clone()); + time_ctx = Some(PanelApiTimeContext { expire_mode: cached.expire_mode, server_tz }); + effective_now = apply_clock_skew(get_current_timestamp(), cached.skew_secs.unwrap_or_default()); + let server_tz_name = server_tz.as_ref().map_or("none", |tz| Tz::name(*tz)); + debug_if_enabled!( + "panel_api time context fallback for input {}: expire_mode={:?}, tz={}, skew_secs={}", + sanitize_sensitive_info(&input.name), + cached.expire_mode, + server_tz_name, + cached.skew_secs.unwrap_or_default() + ); + } else { + debug_if_enabled!( + "panel_api time context skipped for input {}: missing root credentials", + sanitize_sensitive_info(&input.name) + ); + } + + for acct in &mut accounts { + let new_exp = + match panel_client_info(app_state.as_ref(), panel_cfg, &acct.username, &acct.password, time_ctx.as_ref()) + .await + { + Ok(v) => v, + Err(err) => { + debug_if_enabled!( + "panel_api client_info failed for {}: {}", + sanitize_sensitive_info(&acct.name), + sanitize_sensitive_info(&err.to_string()) + ); + None + } + }; + let Some(new_exp) = new_exp else { + continue; + }; + if acct.exp_date == Some(new_exp) { + continue; + } + + if let Some(csv_path) = csv_path.as_ref() { + let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; + if let Err(err) = csv_patch_batch_update_exp_date( + app_state, + &csv_lock, + input.input_type, + csv_path, + &acct.name, + &acct.username, + &acct.password, + new_exp, + ) + .await + { + debug_if_enabled!("panel_api boot sync failed to persist exp_date to csv: {}", err); + continue; + } + any_change = true; + } else { + sources_yml_patches.push(SourcesYmlPatch::UpdatePanelAccountExpiry { + input_name: input.name.clone(), + account_name: acct.name.clone(), + exp_date: new_exp, + }); + pending_sources_yml = true; + } + acct.exp_date = Some(new_exp); + } + + let offset_secs = panel_cfg + .provisioning + .offset + .as_deref() + .and_then(|v| parse_panel_api_provisioning_offset_secs(v).ok()) + .unwrap_or(0); + + let min_pool = resolve_alias_pool_min(app_state.as_ref(), &input.name, panel_cfg); + let renew_enabled = optional.contains(PanelApiOptionalFlags::ClientRenew); + let new_enabled = optional.contains(PanelApiOptionalFlags::ClientNew); + let adult_enabled = optional.contains(PanelApiOptionalFlags::AdultContent); + let provisioning_enabled = renew_enabled || new_enabled; + // Refresh/provision credentials on boot/update. + // Root is handled first (may affect desired_aliases and avoids over-provisioning). + let mut provisioned_root = 0_u16; + let mut provisioned_aliases = 0_u16; + + if let Some(root_idx) = accounts.iter().position(|a| a.name == input.name) { + let now = effective_now; + let offset_deadline = now.saturating_add(offset_secs); + let root_exp_date = accounts[root_idx].exp_date; + let root_exp_missing = root_exp_date.is_none(); + let root_expired = match root_exp_date { + Some(ts) => u64::try_from(ts).map_or(true, |exp_ts| exp_ts <= now), + None => false, + }; + let root_expiring = match root_exp_date { + Some(ts) => u64::try_from(ts).map_or(true, |exp_ts| exp_ts > now && exp_ts <= offset_deadline), + None => false, + }; + let should_refresh_root = root_exp_missing || root_expired || root_expiring; + + let root_exp_display = root_exp_date.map_or_else(|| "None".to_string(), |ts| ts.to_string()); + debug_if_enabled!( + "panel_api boot/update root status for input {} (offset={}s): exp_date={}, expired={}, expiring(offset)={}", + sanitize_sensitive_info(&input.name), + offset_secs, + root_exp_display, + root_expired, + root_expiring + ); + + if should_refresh_root { + if provisioning_enabled { + let old_username = accounts[root_idx].username.clone(); + let old_password = accounts[root_idx].password.clone(); + + debug_if_enabled!( + "panel_api boot/update refreshing root account {} for input {} (exp_date={}, offset={}s)", + sanitize_sensitive_info(&old_username), + sanitize_sensitive_info(&input.name), + root_exp_display, + offset_secs + ); + + let (active_username, active_password, creds_changed) = if renew_enabled { + match panel_client_renew( + app_state.as_ref(), + panel_cfg, + old_username.as_str(), + old_password.as_str(), + ) + .await + { + Ok(()) => { + provisioned_root = 1; + (old_username.clone(), old_password.clone(), false) + } + Err(err) => { + debug_if_enabled!( + "panel_api client_renew failed for root {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + if new_enabled { + match panel_client_new(app_state.as_ref(), panel_cfg).await { + Ok((new_username, new_password, _base_url_from_resp)) => { + provisioned_root = 1; + // Variant B: if the old root is still valid but within offset window, + // keep it as a new alias entry so we don't lose usable credentials. + let park_old_root_as_alias = + root_expiring && !root_expired && root_exp_date.is_some(); + + if park_old_root_as_alias { + let base_url = resolve_provisioned_account_base_url( + input.url.as_str(), + None, + &old_username, + &old_password, + ); + let alias_name = derive_unique_alias_name_set( + &existing_names, + &input.name, + old_username.as_str(), + ); + existing_names.insert(alias_name.clone().into()); + + if let Some(csv_path) = csv_path.as_ref() { + let batch_type = if input.input_type == InputType::Xtream { + InputType::XtreamBatch + } else if input.input_type == InputType::M3u { + InputType::M3uBatch + } else { + input.input_type + }; + let csv_lock = + app_state.app_config.file_locks.write_lock(csv_path).await; + if let Err(err) = csv_patch_batch_append( + app_state, + &csv_lock, + csv_path, + batch_type, + &alias_name, + &base_url, + &old_username, + &old_password, + root_exp_date, + ) + .await + { + debug_if_enabled!( + "panel_api boot/update failed to park old root as csv alias {}: {}", + sanitize_sensitive_info(&alias_name), + err + ); + } else { + any_change = true; + } + } else { + sources_yml_patches.push(SourcesYmlPatch::AddAlias { + input_name: input.name.clone(), + alias_name: alias_name.clone().into(), + base_url, + username: old_username.clone(), + password: old_password.clone(), + exp_date: root_exp_date, + }); + pending_sources_yml = true; + } + + accounts.push(AccountCredentials { + name: alias_name.into(), + username: old_username.clone(), + password: old_password.clone(), + exp_date: root_exp_date, + }); + + debug_if_enabled!( + "panel_api boot/update parked old root credentials for input {} as new alias (exp_date={:?})", + sanitize_sensitive_info(&input.name), + root_exp_date + ); + } + + if let Some(csv_path) = csv_path.as_ref() { + let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; + if let Err(err) = csv_patch_batch_update_credentials( + app_state, + &csv_lock, + input.input_type, + csv_path, + &input.name, + &old_username, + &old_password, + &new_username, + &new_password, + None, + ) + .await + { + debug_if_enabled!( + "panel_api boot/update failed to persist new root credentials to csv for {}: {}", + sanitize_sensitive_info(&input.name), + err + ); + } else { + any_change = true; + } + } else { + sources_yml_patches.push(SourcesYmlPatch::UpdateRootCredentials { + input_name: input.name.clone(), + username: new_username.clone(), + password: new_password.clone(), + exp_date: None, + }); + pending_sources_yml = true; + } + + accounts[root_idx].username.clone_from(&new_username); + accounts[root_idx].password.clone_from(&new_password); + (new_username, new_password, true) + } + Err(err) => { + debug_if_enabled!( + "panel_api client_new failed for root {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + (old_username.clone(), old_password.clone(), false) + } + } + } else { + (old_username.clone(), old_password.clone(), false) + } + } + } + } else if new_enabled { + match panel_client_new(app_state.as_ref(), panel_cfg).await { + Ok((new_username, new_password, _base_url_from_resp)) => { + provisioned_root = 1; + let park_old_root_as_alias = root_expiring && !root_expired && root_exp_date.is_some(); + + if park_old_root_as_alias { + let base_url = resolve_provisioned_account_base_url( + input.url.as_str(), + None, + &old_username, + &old_password, + ); + let alias_name = + derive_unique_alias_name_set(&existing_names, &input.name, old_username.as_str()); + existing_names.insert(alias_name.clone().into()); + + if let Some(csv_path) = csv_path.as_ref() { + let batch_type = if input.input_type == InputType::Xtream { + InputType::XtreamBatch + } else if input.input_type == InputType::M3u { + InputType::M3uBatch + } else { + input.input_type + }; + let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; + if let Err(err) = csv_patch_batch_append( + app_state, + &csv_lock, + csv_path, + batch_type, + &alias_name, + &base_url, + &old_username, + &old_password, + root_exp_date, + ) + .await + { + debug_if_enabled!( + "panel_api boot/update failed to park old root as csv alias {}: {}", + sanitize_sensitive_info(&alias_name), + err + ); + } else { + any_change = true; + } + } else { + sources_yml_patches.push(SourcesYmlPatch::AddAlias { + input_name: input.name.clone(), + alias_name: alias_name.clone().into(), + base_url, + username: old_username.clone(), + password: old_password.clone(), + exp_date: root_exp_date, + }); + pending_sources_yml = true; + } + + accounts.push(AccountCredentials { + name: alias_name.into(), + username: old_username.clone(), + password: old_password.clone(), + exp_date: root_exp_date, + }); + + debug_if_enabled!( + "panel_api boot/update parked old root credentials for input {} as new alias (exp_date={:?})", + sanitize_sensitive_info(&input.name), + root_exp_date + ); + } + + if let Some(csv_path) = csv_path.as_ref() { + let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; + if let Err(err) = csv_patch_batch_update_credentials( + app_state, + &csv_lock, + input.input_type, + csv_path, + &input.name, + &old_username, + &old_password, + &new_username, + &new_password, + None, + ) + .await + { + debug_if_enabled!( + "panel_api boot/update failed to persist new root credentials to csv for {}: {}", + sanitize_sensitive_info(&input.name), + err + ); + } else { + any_change = true; + } + } else { + sources_yml_patches.push(SourcesYmlPatch::UpdateRootCredentials { + input_name: input.name.clone(), + username: new_username.clone(), + password: new_password.clone(), + exp_date: None, + }); + pending_sources_yml = true; + } + + accounts[root_idx].username.clone_from(&new_username); + accounts[root_idx].password.clone_from(&new_password); + (new_username, new_password, true) + } + Err(err) => { + debug_if_enabled!( + "panel_api client_new failed for root {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + (old_username.clone(), old_password.clone(), false) + } + } + } else { + (old_username.clone(), old_password.clone(), false) + }; + + if adult_enabled { + if let Err(err) = panel_client_adult_content( + app_state.as_ref(), + panel_cfg, + Some((active_username.as_str(), active_password.as_str())), + ) + .await + { + debug_if_enabled!( + "panel_api client_adult_content failed for root {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + + let refreshed_exp = panel_client_info( + app_state.as_ref(), + panel_cfg, + active_username.as_str(), + active_password.as_str(), + time_ctx.as_ref(), + ) + .await + .ok() + .flatten(); + + let ready = wait_for_panel_api_account_ready( + app_state, + input.as_ref(), + panel_cfg, + &input.name, + active_username.as_str(), + active_password.as_str(), + ) + .await; + if !ready { + debug_if_enabled!( + "panel_api boot/update probe timeout for root {}; skipping exp_date refresh", + sanitize_sensitive_info(&input.name) + ); + } else if let Some(new_exp) = refreshed_exp { + if let Some(csv_path) = csv_path.as_ref() { + let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; + let result = if creds_changed { + csv_patch_batch_update_credentials( + app_state, + &csv_lock, + input.input_type, + csv_path, + &input.name, + old_username.as_str(), + old_password.as_str(), + active_username.as_str(), + active_password.as_str(), + Some(new_exp), + ) + .await + } else { + csv_patch_batch_update_exp_date( + app_state, + &csv_lock, + input.input_type, + csv_path, + &input.name, + old_username.as_str(), + old_password.as_str(), + new_exp, + ) + .await + }; + if let Err(err) = result { + debug_if_enabled!( + "panel_api boot/update failed to persist root exp_date to csv for {}: {}", + sanitize_sensitive_info(&input.name), + err + ); + } else { + accounts[root_idx].exp_date = Some(new_exp); + any_change = true; + } + } else { + if creds_changed { + sources_yml_patches.push(SourcesYmlPatch::UpdateRootCredentials { + input_name: input.name.clone(), + username: active_username.clone(), + password: active_password.clone(), + exp_date: Some(new_exp), + }); + } else { + sources_yml_patches.push(SourcesYmlPatch::UpdatePanelAccountExpiry { + input_name: input.name.clone(), + account_name: input.name.clone(), + exp_date: new_exp, + }); + } + pending_sources_yml = true; + accounts[root_idx].exp_date = Some(new_exp); + } + } + } else { + debug_if_enabled!( + "panel_api boot/update skipped root refresh for input {}: client_new/client_renew disabled", + sanitize_sensitive_info(&input.name) + ); + } + } + } else { + debug_if_enabled!( + "panel_api boot/update skipped root provisioning for input {}: missing credentials", + sanitize_sensitive_info(&input.name) + ); + } + + // Refresh aliases after the root operation to avoid over-provisioning. + let now = effective_now; + let offset_deadline = now.saturating_add(offset_secs); + let root_valid = root_counts_towards_pool_at(&accounts, &input.name, now); + let desired_aliases = min_pool.filter(|m| *m > 0).map_or_else( + || u16::try_from(accounts.iter().filter(|a| a.name != input.name).count()).unwrap_or(u16::MAX), + |min_pool| min_pool.saturating_sub(u16::from(root_valid)), + ); + + let expiring_aliases = accounts + .iter() + .filter(|a| a.name != input.name && is_expiring_with_offset_at(a.exp_date, offset_secs, now)) + .count(); + let expired_aliases = + accounts.iter().filter(|a| a.name != input.name && is_input_expired_at(a.exp_date, now)).count(); + + let valid_aliases_beyond_offset = accounts + .iter() + .filter(|a| { + if a.name == input.name { + return false; + } + if is_input_expired_at(a.exp_date, now) { + return false; + } + match a.exp_date { + Some(ts) => u64::try_from(ts).is_ok_and(|exp_ts| exp_ts > offset_deadline), + None => false, + } + }) + .count(); + + let desired_aliases_u16 = desired_aliases; + let alias_total = accounts.iter().filter(|a| a.name != input.name).count(); + let alias_total_u16 = u16::try_from(alias_total).unwrap_or(u16::MAX); + let missing_aliases_u16 = desired_aliases_u16.saturating_sub(alias_total_u16); + + let (refresh_plan, planned_refresh_aliases) = if provisioning_enabled { + let mut refresh_candidates: Vec = accounts + .iter() + .enumerate() + .filter(|(_, a)| { + if a.name == input.name { + return false; + } + if is_input_expired_at(a.exp_date, now) { + return false; + } + match a.exp_date { + None => true, + Some(ts) => u64::try_from(ts).map_or(true, |exp_ts| exp_ts <= offset_deadline), + } + }) + .map(|(idx, _)| idx) + .collect(); + + refresh_candidates.sort_by_key(|idx| { + let acct = &accounts[*idx]; + match acct.exp_date { + None => (0_u8, i64::MIN), + Some(ts) => (1_u8, ts), + } + }); + + let valid_aliases_beyond_offset_u16 = u16::try_from(valid_aliases_beyond_offset).unwrap_or(u16::MAX); + let needed_refresh_aliases_u16 = desired_aliases_u16.saturating_sub(valid_aliases_beyond_offset_u16); + let planned_refresh_aliases = refresh_candidates.len().min(usize::from(needed_refresh_aliases_u16)); + let refresh_plan: Vec = refresh_candidates.into_iter().take(planned_refresh_aliases).collect(); + (refresh_plan, planned_refresh_aliases) + } else { + (Vec::new(), 0) + }; + + let log_pool = alias_pool_has_min(panel_cfg); + let enabled_users = if log_pool { count_enabled_proxy_users(app_state.as_ref(), &input.name) } else { 0 }; + if log_pool { + debug_if_enabled!( + "panel_api boot/update provisioning aliases for input {} (offset={}s): desired={}, valid_beyond_offset={}, expiring(offset)={}, expired={}, refresh_planned(offset)={}, missing={}", + sanitize_sensitive_info(&input.name), + offset_secs, + desired_aliases_u16, + valid_aliases_beyond_offset, + expiring_aliases, + expired_aliases, + planned_refresh_aliases, + missing_aliases_u16 + ); + } + + if !refresh_plan.is_empty() { + debug_if_enabled!( + "panel_api boot/update alias refresh plan for input {}: selected={}", + sanitize_sensitive_info(&input.name), + refresh_plan + .iter() + .filter_map(|idx| accounts.get(*idx)) + .map(|acct| sanitize_sensitive_info(&acct.name).to_string()) + .collect::>() + .join(",") + ); + } + + for idx in refresh_plan { + let Some(acct) = accounts.get_mut(idx) else { + continue; + }; + let account_name = acct.name.clone(); + let old_username = acct.username.clone(); + let old_password = acct.password.clone(); + + debug_if_enabled!( + "panel_api boot/update refreshing alias account {} for input {} (exp_date={:?}, offset={}s)", + sanitize_sensitive_info(&account_name), + sanitize_sensitive_info(&input.name), + acct.exp_date, + offset_secs + ); + + let (active_username, active_password, creds_changed) = if renew_enabled { + match panel_client_renew(app_state.as_ref(), panel_cfg, old_username.as_str(), old_password.as_str()).await + { + Ok(()) => { + provisioned_aliases = provisioned_aliases.saturating_add(1); + (old_username.clone(), old_password.clone(), false) + } + Err(err) => { + debug_if_enabled!( + "panel_api client_renew failed for alias {}: {}", + sanitize_sensitive_info(&account_name), + sanitize_sensitive_info(&err.to_string()) + ); + if new_enabled { + match panel_client_new(app_state.as_ref(), panel_cfg).await { + Ok((new_username, new_password, base_url_from_resp)) => { + let base_url = resolve_provisioned_account_base_url( + input.url.as_str(), + base_url_from_resp.as_deref(), + &new_username, + &new_password, + ); + + let alias_name = + derive_unique_alias_name_set(&existing_names, &input.name, &new_username); + existing_names.insert(alias_name.clone().into()); + + if adult_enabled { + if let Err(err) = panel_client_adult_content( + app_state.as_ref(), + panel_cfg, + Some((new_username.as_str(), new_password.as_str())), + ) + .await + { + debug_if_enabled!( + "panel_api client_adult_content failed for {}: {}", + sanitize_sensitive_info(&alias_name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + + let exp_date = panel_client_info( + app_state.as_ref(), + panel_cfg, + new_username.as_str(), + new_password.as_str(), + time_ctx.as_ref(), + ) + .await + .ok() + .flatten(); + + if let Some(csv_path) = csv_path.as_ref() { + let batch_type = if input.input_type == InputType::Xtream { + InputType::XtreamBatch + } else if input.input_type == InputType::M3u { + InputType::M3uBatch + } else { + input.input_type + }; + let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; + if let Err(err) = csv_patch_batch_append( + app_state, + &csv_lock, + csv_path, + batch_type, + &alias_name, + &base_url, + &new_username, + &new_password, + exp_date, + ) + .await + { + debug_if_enabled!( + "panel_api boot/update failed to append new csv account for {}: {}", + sanitize_sensitive_info(&alias_name), + err + ); + continue; + } + any_change = true; + } else { + sources_yml_patches.push(SourcesYmlPatch::AddAlias { + input_name: input.name.clone(), + alias_name: alias_name.clone().into(), + base_url, + username: new_username.clone(), + password: new_password.clone(), + exp_date, + }); + pending_sources_yml = true; + } + + newly_created_accounts.push(AccountCredentials { + name: alias_name.into(), + username: new_username, + password: new_password, + exp_date, + }); + provisioned_aliases = provisioned_aliases.saturating_add(1); + continue; + } + Err(err) => { + debug_if_enabled!( + "panel_api client_new failed for input {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + continue; + } + } + } + continue; + } + } + } else if new_enabled { + match panel_client_new(app_state.as_ref(), panel_cfg).await { + Ok((new_username, new_password, base_url_from_resp)) => { + let base_url = resolve_provisioned_account_base_url( + input.url.as_str(), + base_url_from_resp.as_deref(), + &new_username, + &new_password, + ); + + let alias_name = derive_unique_alias_name_set(&existing_names, &input.name, &new_username); + existing_names.insert(alias_name.clone().into()); + + if adult_enabled { + if let Err(err) = panel_client_adult_content( + app_state.as_ref(), + panel_cfg, + Some((new_username.as_str(), new_password.as_str())), + ) + .await + { + debug_if_enabled!( + "panel_api client_adult_content failed for {}: {}", + sanitize_sensitive_info(&alias_name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + + let exp_date = panel_client_info( + app_state.as_ref(), + panel_cfg, + new_username.as_str(), + new_password.as_str(), + time_ctx.as_ref(), + ) + .await + .ok() + .flatten(); + + if let Some(csv_path) = csv_path.as_ref() { + let batch_type = if input.input_type == InputType::Xtream { + InputType::XtreamBatch + } else if input.input_type == InputType::M3u { + InputType::M3uBatch + } else { + input.input_type + }; + let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; + if let Err(err) = csv_patch_batch_append( + app_state, + &csv_lock, + csv_path, + batch_type, + &alias_name, + &base_url, + &new_username, + &new_password, + exp_date, + ) + .await + { + debug_if_enabled!( + "panel_api boot/update failed to append new csv account for {}: {}", + sanitize_sensitive_info(&alias_name), + err + ); + continue; + } + any_change = true; + } else { + sources_yml_patches.push(SourcesYmlPatch::AddAlias { + input_name: input.name.clone(), + alias_name: alias_name.clone().into(), + base_url, + username: new_username.clone(), + password: new_password.clone(), + exp_date, + }); + pending_sources_yml = true; + } + + newly_created_accounts.push(AccountCredentials { + name: alias_name.into(), + username: new_username, + password: new_password, + exp_date, + }); + provisioned_aliases = provisioned_aliases.saturating_add(1); + continue; + } + Err(err) => { + debug_if_enabled!( + "panel_api client_new failed for input {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + continue; + } + } + } else { + continue; + }; + + if adult_enabled { + if let Err(err) = panel_client_adult_content( + app_state.as_ref(), + panel_cfg, + Some((active_username.as_str(), active_password.as_str())), + ) + .await + { + debug_if_enabled!( + "panel_api client_adult_content failed for {}: {}", + sanitize_sensitive_info(&account_name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + + let refreshed_exp = panel_client_info( + app_state.as_ref(), + panel_cfg, + active_username.as_str(), + active_password.as_str(), + time_ctx.as_ref(), + ) + .await + .ok() + .flatten(); + + if let Some(new_exp) = refreshed_exp { + if let Some(csv_path) = csv_path.as_ref() { + let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; + let result = if creds_changed { + csv_patch_batch_update_credentials( + app_state, + &csv_lock, + input.input_type, + csv_path, + &account_name, + &old_username, + &old_password, + active_username.as_str(), + active_password.as_str(), + Some(new_exp), + ) + .await + } else { + csv_patch_batch_update_exp_date( + app_state, + &csv_lock, + input.input_type, + csv_path, + &account_name, + &old_username, + &old_password, + new_exp, + ) + .await + }; + if let Err(err) = result { + debug_if_enabled!( + "panel_api boot/update failed to persist exp_date to csv for {}: {}", + sanitize_sensitive_info(&account_name), + err + ); + } else { + acct.exp_date = Some(new_exp); + any_change = true; + } + } else { + if creds_changed { + sources_yml_patches.push(SourcesYmlPatch::UpdateAliasCredentials { + input_name: input.name.clone(), + alias_name: account_name.clone(), + username: active_username.clone(), + password: active_password.clone(), + exp_date: Some(new_exp), + }); + } else { + sources_yml_patches.push(SourcesYmlPatch::UpdatePanelAccountExpiry { + input_name: input.name.clone(), + account_name: account_name.clone(), + exp_date: new_exp, + }); + } + pending_sources_yml = true; + acct.exp_date = Some(new_exp); + } + } else { + debug_if_enabled!( + "panel_api boot/update renew/create succeeded but exp_date refresh failed for {}", + sanitize_sensitive_info(&account_name) + ); + } + } + + if !newly_created_accounts.is_empty() { + accounts.extend(newly_created_accounts); + } + + if adult_enabled { + for acct in &accounts { + if let Err(err) = panel_client_adult_content( + app_state.as_ref(), + panel_cfg, + Some((acct.username.as_str(), acct.password.as_str())), + ) + .await + { + debug_if_enabled!( + "panel_api client_adult_content failed for {}: {}", + sanitize_sensitive_info(&acct.name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + } + + let min_pool = min_pool.filter(|m| *m > 0); + if let Some(min_pool) = min_pool { + let root_valid = root_counts_towards_pool(&accounts, &input.name); + let alias_min = min_pool.saturating_sub(u16::from(root_valid)); + let (pool_changed, pool_provisioned) = ensure_alias_pool_min( + app_state, + input.as_ref(), + panel_cfg, + &mut accounts, + alias_min, + csv_path.as_deref(), + &mut sources_yml_patches, + time_ctx.as_ref(), + effective_now, + optional, + ) + .await; + if pool_changed { + if csv_path.is_some() { + any_change = true; + } else { + pending_sources_yml = true; + } + } + provisioned_aliases = provisioned_aliases.saturating_add(pool_provisioned); + } + + if log_pool { + let valid_total = count_valid_accounts_at(&accounts, now); + debug_if_enabled!( + "panel_api boot/update provisioning total for input {} (offset={}s): enabled_users={}, valid_accounts={}, provisioned_root={}, provisioned_aliases={}", + sanitize_sensitive_info(&input.name), + offset_secs, + enabled_users, + valid_total, + provisioned_root, + provisioned_aliases + ); + } + + if alias_pool_remove_expired(panel_cfg) { + if let Some(csv_path) = csv_path.as_ref() { + let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; + match csv_patch_batch_remove_expired(app_state, &csv_lock, input.input_type, csv_path).await { + Ok(true) => any_change = true, + Ok(false) => {} + Err(err) => debug_if_enabled!("panel_api boot sync failed to remove expired csv accounts: {}", err), + } + } else { + sources_yml_patches.push(SourcesYmlPatch::RemoveExpiredAliases { input_name: input.name.clone() }); + pending_sources_yml = true; + } + } + + if optional.contains(PanelApiOptionalFlags::AccountInfo) { + let creds = accounts.first().map(|acct| (acct.username.as_str(), acct.password.as_str())); + match panel_account_info(app_state.as_ref(), panel_cfg, creds).await { + Ok(Some(credits)) => { + let normalized = credits.trim().to_string(); + if !normalized.is_empty() + /* && panel_cfg.credits.as_deref().map(str::trim) != Some(normalized.as_str()) */ + { + sources_yml_patches.push(SourcesYmlPatch::UpdatePanelApiCredits { + input_name: input.name.clone(), + credits: normalized, + }); + pending_sources_yml = true; + } + } + Ok(None) => {} + Err(err) => { + debug_if_enabled!( + "panel_api account_info failed for {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + } + } + } + + if panel_cfg.alias_pool.is_some() { + if let Some(csv_path) = csv_path.as_ref() { + let csv_lock = app_state.app_config.file_locks.write_lock(csv_path).await; + match csv_patch_batch_sort_by_exp_date( + app_state, + &csv_lock, + input.input_type, + csv_path, + AliasExpDateSortOrder::NewestFirst, + ) + .await + { + Ok(true) => any_change = true, + Ok(false) => {} + Err(err) => debug_if_enabled!( + "panel_api boot/update final sort failed for csv alias pool {}: {}", + sanitize_sensitive_info(&input.name), + err + ), + } + } else if pending_sources_yml && !source_yml_sort_aliases_requested { + sources_yml_patches.push(SourcesYmlPatch::SortAliases { + input_name: input.name.clone(), + order: AliasExpDateSortOrder::NewestFirst, + }); + } + } + + if pending_sources_yml { + match crate::api::source_yml_patch::execute_account_source_patches( + app_state, + sources_path, + &sources_yml_patches, + ) + .await + { + Ok(true) => any_change = true, + Ok(false) => {} + Err(err) => debug_if_enabled!("panel_api boot sync failed to persist source.yml patches: {}", err), + } + } + + any_change +} diff --git a/backend/app/src/api/panel_api/tests.rs b/backend/app/src/api/panel_api/tests.rs index 222d1b901..f75bfcd02 100644 --- a/backend/app/src/api/panel_api/tests.rs +++ b/backend/app/src/api/panel_api/tests.rs @@ -3,402 +3,12 @@ use super::{ panel_api_retryable_status, resolve_batch_alias_path, PanelApiProbeTarget, PANEL_API_DEFAULT_RETRY_AFTER_SECS, PANEL_API_MAX_RETRY_AFTER_SECS, }; -use crate::{ - api::source_yml_patch::{apply_sources_yml_patches, resolve_provisioned_account_base_url, SourcesYmlPatch}, - model::{ConfigInput, ConfigProvider}, - repository::AliasExpDateSortOrder, -}; -use axum::http::StatusCode; -use shared::model::{ - ConfigInputAliasDto, ConfigInputDto, ConfigProviderDto, InputType, ProviderUrlSelectionPolicy, SourcesConfigDto, -}; -use std::{sync::Arc, time::Duration}; -use url::Url; -fn source_alias(name: &str, exp_date: Option) -> ConfigInputAliasDto { - ConfigInputAliasDto { - id: 0, - name: Arc::from(name), - url: "provider://tivione".to_string(), - username: Some(format!("{name}-user")), - password: Some(format!("{name}-pass")), - priority: 0, - max_connections: 1, - exp_date, - enabled: true, - stalker: None, - } -} +mod behavior; +mod lifecycle; +mod persistence; +mod query; +mod retry; +mod support; -#[test] -fn batch_alias_storage_never_falls_back_to_source_yml() { - assert!(resolve_batch_alias_path(None).expect("non-batch input").is_none()); - assert!(resolve_batch_alias_path(Some("provider://not-a-csv")).is_err()); -} - -fn source_doc_with_aliases(aliases: Vec) -> SourcesConfigDto { - SourcesConfigDto { - inputs: vec![ConfigInputDto { - name: Arc::from("cdn-dev"), - input_type: InputType::Xtream, - url: "provider://tivione".to_string(), - username: Some("root-user".to_string()), - password: Some("root-pass".to_string()), - aliases: Some(aliases), - ..ConfigInputDto::default() - }], - ..SourcesConfigDto::default() - } -} - -#[test] -fn sources_yml_add_alias_appends_without_overwriting_existing_alias() { - let mut doc = source_doc_with_aliases(vec![source_alias("cdn-dev-old", Some(10))]); - doc.inputs[0].aliases.as_mut().expect("aliases")[0].id = 2; - - let changed = apply_sources_yml_patches( - &mut doc, - &[ - SourcesYmlPatch::AddAlias { - input_name: Arc::from("cdn-dev"), - alias_name: Arc::from("cdn-dev-new"), - base_url: "provider://tivione".to_string(), - username: "new-user".to_string(), - password: "new-pass".to_string(), - exp_date: Some(20), - }, - SourcesYmlPatch::SortAliases { - input_name: Arc::from("cdn-dev"), - order: AliasExpDateSortOrder::NewestFirst, - }, - ], - ) - .expect("patches apply"); - - assert!(changed); - let aliases = doc.inputs[0].aliases.as_ref().expect("aliases"); - assert_eq!(aliases.len(), 2); - assert_eq!(aliases[0].name.as_ref(), "cdn-dev-new"); - assert_eq!(aliases[0].id, 3); - assert_eq!(aliases[1].name.as_ref(), "cdn-dev-old"); - assert_eq!(aliases[1].id, 2); -} - -#[test] -fn sources_yml_sort_aliases_oldest_first_is_available_for_maintenance_paths() { - let mut doc = source_doc_with_aliases(vec![ - source_alias("cdn-dev-newest", Some(30)), - source_alias("cdn-dev-oldest", Some(10)), - source_alias("cdn-dev-missing-exp", None), - ]); - - let changed = apply_sources_yml_patches( - &mut doc, - &[SourcesYmlPatch::SortAliases { input_name: Arc::from("cdn-dev"), order: AliasExpDateSortOrder::OldestFirst }], - ) - .expect("patches apply"); - - assert!(changed); - let aliases = doc.inputs[0].aliases.as_ref().expect("aliases"); - assert_eq!(aliases[0].name.as_ref(), "cdn-dev-oldest"); - assert_eq!(aliases[1].name.as_ref(), "cdn-dev-newest"); - assert_eq!(aliases[2].name.as_ref(), "cdn-dev-missing-exp"); -} - -#[test] -fn sources_yml_update_exp_date_keeps_legacy_root_refresh_semantics() { - let mut doc = source_doc_with_aliases(Vec::new()); - doc.inputs[0].exp_date = Some(20); - doc.inputs[0].enabled = false; - doc.inputs[0].max_connections = 0; - - let changed = apply_sources_yml_patches( - &mut doc, - &[SourcesYmlPatch::UpdatePanelAccountExpiry { - input_name: Arc::from("cdn-dev"), - account_name: Arc::from("cdn-dev"), - exp_date: 20, - }], - ) - .expect("patches apply"); - - assert!(changed); - assert_eq!(doc.inputs[0].exp_date, Some(20)); - assert!(doc.inputs[0].enabled); - assert_eq!(doc.inputs[0].max_connections, 1); -} - -#[test] -fn sources_yml_update_exp_date_keeps_legacy_alias_refresh_semantics() { - let mut doc = source_doc_with_aliases(vec![source_alias("cdn-dev-old", Some(20))]); - doc.inputs[0].aliases.as_mut().expect("aliases")[0].max_connections = 0; - - let changed = apply_sources_yml_patches( - &mut doc, - &[SourcesYmlPatch::UpdatePanelAccountExpiry { - input_name: Arc::from("cdn-dev"), - account_name: Arc::from("cdn-dev-old"), - exp_date: 20, - }], - ) - .expect("patches apply"); - - assert!(changed); - let alias = &doc.inputs[0].aliases.as_ref().expect("aliases")[0]; - assert_eq!(alias.exp_date, Some(20)); - assert_eq!(alias.max_connections, 1); -} - -#[test] -fn sources_yml_update_root_credentials_updates_root_directly() { - let mut doc = source_doc_with_aliases(Vec::new()); - - let changed = apply_sources_yml_patches( - &mut doc, - &[SourcesYmlPatch::UpdateRootCredentials { - input_name: Arc::from("cdn-dev"), - username: "new-root".to_string(), - password: "new-pass".to_string(), - exp_date: Some(42), - }], - ) - .expect("patches apply"); - - assert!(changed); - assert_eq!(doc.inputs[0].username.as_deref(), Some("new-root")); - assert_eq!(doc.inputs[0].password.as_deref(), Some("new-pass")); - assert_eq!(doc.inputs[0].exp_date, Some(42)); - assert!(doc.inputs[0].enabled); - assert_eq!(doc.inputs[0].max_connections, 1); -} - -#[test] -fn sources_yml_persist_provisioned_account_adds_alias_when_current_root_is_valid() { - let mut doc = source_doc_with_aliases(Vec::new()); - doc.inputs[0].username = Some("current-root".to_string()); - doc.inputs[0].password = Some("current-pass".to_string()); - doc.inputs[0].exp_date = Some(i64::try_from(jsonwebtoken::get_current_timestamp()).expect("timestamp") + 3600); - - let changed = apply_sources_yml_patches( - &mut doc, - &[SourcesYmlPatch::PersistProvisionedAccount { - input_name: Arc::from("cdn-dev"), - username: "new-root".to_string(), - password: "new-pass".to_string(), - exp_date: Some(42), - }], - ) - .expect("patches apply"); - - assert!(changed); - assert_eq!(doc.inputs[0].username.as_deref(), Some("current-root")); - assert_eq!(doc.inputs[0].password.as_deref(), Some("current-pass")); - assert_ne!(doc.inputs[0].exp_date, Some(42)); - - let aliases = doc.inputs[0].aliases.as_ref().expect("aliases"); - assert_eq!(aliases.len(), 1); - assert_eq!(aliases[0].name.as_ref(), "cdn-dev-new-root"); - assert_eq!(aliases[0].username.as_deref(), Some("new-root")); - assert_eq!(aliases[0].password.as_deref(), Some("new-pass")); - assert_eq!(aliases[0].exp_date, Some(42)); -} - -#[test] -fn sources_yml_persist_provisioned_account_replaces_root_when_current_root_is_expired() { - let mut doc = source_doc_with_aliases(Vec::new()); - doc.inputs[0].username = Some("current-root".to_string()); - doc.inputs[0].password = Some("current-pass".to_string()); - doc.inputs[0].exp_date = Some(i64::try_from(jsonwebtoken::get_current_timestamp()).expect("timestamp") - 1); - - let changed = apply_sources_yml_patches( - &mut doc, - &[SourcesYmlPatch::PersistProvisionedAccount { - input_name: Arc::from("cdn-dev"), - username: "new-root".to_string(), - password: "new-pass".to_string(), - exp_date: Some(42), - }], - ) - .expect("patches apply"); - - assert!(changed); - assert_eq!(doc.inputs[0].username.as_deref(), Some("new-root")); - assert_eq!(doc.inputs[0].password.as_deref(), Some("new-pass")); - assert_eq!(doc.inputs[0].exp_date, Some(42)); - assert!(doc.inputs[0].aliases.as_ref().is_none_or(Vec::is_empty)); -} - -#[test] -fn panel_api_retryable_status_covers_rate_limit_and_temporary_failures() { - assert!(panel_api_retryable_status(StatusCode::TOO_MANY_REQUESTS)); - assert!(panel_api_retryable_status(StatusCode::REQUEST_TIMEOUT)); - assert!(panel_api_retryable_status(StatusCode::TOO_EARLY)); - assert!(panel_api_retryable_status(StatusCode::BAD_GATEWAY)); - assert!(!panel_api_retryable_status(StatusCode::BAD_REQUEST)); - assert!(!panel_api_retryable_status(StatusCode::UNAUTHORIZED)); - assert!(!panel_api_retryable_status(StatusCode::NOT_FOUND)); -} - -#[test] -fn panel_api_retry_after_header_is_short_and_bounded() { - assert_eq!(panel_api_retry_after_from_header_value("2"), Some(Duration::from_secs(2))); - assert_eq!( - panel_api_retry_after_from_header_value("0"), - Some(Duration::from_secs(PANEL_API_DEFAULT_RETRY_AFTER_SECS)) - ); - assert_eq!( - panel_api_retry_after_from_header_value("600"), - Some(Duration::from_secs(PANEL_API_MAX_RETRY_AFTER_SECS)) - ); - assert_eq!(panel_api_retry_after_from_header_value("not-a-delay"), None); -} - -#[test] -fn resolve_base_url_keeps_provider_scheme_for_provisioned_accounts() { - let result = resolve_provisioned_account_base_url( - "provider://demo-provider/live", - Some("http://panel.example.com:8080/get.php?username=new&password=new"), - "new", - "secret", - ); - - assert_eq!(result, "provider://demo-provider/live"); -} - -#[test] -fn resolve_base_url_updates_provider_query_credentials_when_present() { - let result = resolve_provisioned_account_base_url( - "provider://demo-provider/live?foo=bar&username=old&password=oldpw", - Some("http://panel.example.com:8080/get.php?username=new&password=new"), - "new-user", - "new-pass", - ); - - let parsed = Url::parse(result.as_str()).expect("expected valid provider url"); - let pairs: Vec<(String, String)> = parsed.query_pairs().map(|(k, v)| (k.to_string(), v.to_string())).collect(); - assert!(pairs.contains(&("foo".to_string(), "bar".to_string()))); - assert!(pairs.contains(&("username".to_string(), "new-user".to_string()))); - assert!(pairs.contains(&("password".to_string(), "new-pass".to_string()))); -} - -#[test] -fn resolve_base_url_uses_panel_response_origin_for_http_inputs() { - let result = resolve_provisioned_account_base_url( - "http://input.example.org/path?x=1", - Some("http://panel.example.com:8080/get.php?username=new&password=new"), - "new", - "secret", - ); - - assert_eq!(result, "http://panel.example.com:8080"); -} - -#[test] -fn resolve_base_url_falls_back_when_panel_response_is_literal_null() { - let result = - resolve_provisioned_account_base_url("http://input.example.org/path?x=1", Some("null"), "new", "secret"); - - assert_eq!(result, "http://input.example.org/path?x=1"); -} - -#[test] -fn resolve_base_url_avoids_null_origin_for_non_special_schemes() { - let result = resolve_provisioned_account_base_url( - "http://input.example.org/path?x=1", - Some("custom-scheme://panel.example.com/path?username=new&password=new"), - "new", - "secret", - ); - - assert_eq!(result, "custom-scheme://panel.example.com/path?username=new&password=new"); -} - -#[test] -fn panel_api_probe_targets_preserve_provider_failover_context() { - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "tivione".into(), - urls: vec!["http://line-a.example.test".into(), "http://line-b.example.test".into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, - dns: None, - })); - let input = ConfigInput { - name: Arc::from("cdn-dev"), - url: "provider://tivione".to_string(), - provider_configs: Some(vec![Arc::clone(&provider)]), - ..ConfigInput::default() - }; - - let targets = build_panel_api_probe_targets(&input, "probe-user", "probe-pass"); - - assert_eq!(targets.len(), 4); - let PanelApiProbeTarget::PlayerApi { action, input_source } = &targets[0]; - assert_eq!(*action, "client_info"); - assert_eq!(input_source.provider.as_ref().expect("provider context should be preserved").name.as_ref(), "tivione"); - assert!(input_source.url.starts_with("provider://tivione/player_api.php?")); - assert!(input_source.url.contains("username=probe-user")); - assert!(input_source.url.contains("password=probe-pass")); - assert!(input_source.url.contains("action=client_info")); -} - -#[test] -fn panel_api_probe_targets_keep_plain_http_without_provider_context() { - let input = ConfigInput { - name: Arc::from("plain"), - url: "http://origin.example.test/some/path?ignored=1".to_string(), - ..ConfigInput::default() - }; - - let targets = build_panel_api_probe_targets(&input, "probe-user", "probe-pass"); - - assert_eq!(targets.len(), 4); - let PanelApiProbeTarget::PlayerApi { action, input_source } = &targets[0]; - assert_eq!(*action, "client_info"); - assert!(input_source.provider.is_none()); - assert!(input_source.url.starts_with("http://origin.example.test/player_api.php?")); - assert!(input_source.url.contains("username=probe-user")); - assert!(input_source.url.contains("password=probe-pass")); - assert!(input_source.url.contains("action=client_info")); -} - -#[test] -fn user_api_account_info_preserves_provider_failover_context() { - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "tivione".into(), - urls: vec!["http://line-a.example.test".into(), "http://line-b.example.test".into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, - dns: None, - })); - let input = ConfigInput { - name: Arc::from("cdn-dev"), - url: "provider://tivione".to_string(), - provider_configs: Some(vec![Arc::clone(&provider)]), - ..ConfigInput::default() - }; - - let input_source = build_user_api_account_info_input_source(&input, "root-user", "root-pass") - .expect("expected provider account_info input source"); - - assert_eq!(input_source.provider.as_ref().expect("provider context should be preserved").name.as_ref(), "tivione"); - assert!(input_source.url.starts_with("provider://tivione/player_api.php?")); - assert!(input_source.url.contains("username=root-user")); - assert!(input_source.url.contains("password=root-pass")); - assert!(input_source.url.contains("action=account_info")); -} - -#[test] -fn user_api_account_info_keeps_plain_http_without_provider_context() { - let input = ConfigInput { - name: Arc::from("plain"), - url: "http://origin.example.test/some/path?ignored=1".to_string(), - ..ConfigInput::default() - }; - - let input_source = build_user_api_account_info_input_source(&input, "root-user", "root-pass") - .expect("expected plain account_info input source"); - - assert!(input_source.provider.is_none()); - assert!(input_source.url.starts_with("http://origin.example.test/player_api.php?")); - assert!(input_source.url.contains("username=root-user")); - assert!(input_source.url.contains("password=root-pass")); - assert!(input_source.url.contains("action=account_info")); -} +use self::support::source_doc_with_aliases; diff --git a/backend/app/src/api/panel_api/tests/behavior.rs b/backend/app/src/api/panel_api/tests/behavior.rs new file mode 100644 index 000000000..4dafdf2e1 --- /dev/null +++ b/backend/app/src/api/panel_api/tests/behavior.rs @@ -0,0 +1,279 @@ +use super::{ + build_panel_api_probe_targets, build_user_api_account_info_input_source, resolve_batch_alias_path, + source_doc_with_aliases, PanelApiProbeTarget, +}; +use crate::{ + api::source_yml_patch::{apply_sources_yml_patches, resolve_provisioned_account_base_url, SourcesYmlPatch}, + model::{ConfigInput, ConfigProvider}, + repository::AliasExpDateSortOrder, +}; +use shared::model::{ConfigInputAliasDto, ConfigProviderDto, ProviderUrlSelectionPolicy}; +use std::sync::Arc; + +pub(in crate::api::panel_api::tests) fn source_alias(name: &str, exp_date: Option) -> ConfigInputAliasDto { + ConfigInputAliasDto { + id: 0, + name: Arc::from(name), + url: "provider://tivione".to_string(), + username: Some(format!("{name}-user")), + password: Some(format!("{name}-pass")), + priority: 0, + max_connections: 1, + exp_date, + enabled: true, + stalker: None, + } +} + +#[test] +fn batch_alias_storage_never_falls_back_to_source_yml() { + assert!(resolve_batch_alias_path(None).expect("non-batch input").is_none()); + assert!(resolve_batch_alias_path(Some("provider://not-a-csv")).is_err()); +} + +#[test] +fn sources_yml_add_alias_appends_without_overwriting_existing_alias() { + let mut doc = source_doc_with_aliases(vec![source_alias("cdn-dev-old", Some(10))]); + doc.inputs[0].aliases.as_mut().expect("aliases")[0].id = 2; + + let changed = apply_sources_yml_patches( + &mut doc, + &[ + SourcesYmlPatch::AddAlias { + input_name: Arc::from("cdn-dev"), + alias_name: Arc::from("cdn-dev-new"), + base_url: "provider://tivione".to_string(), + username: "new-user".to_string(), + password: "new-pass".to_string(), + exp_date: Some(20), + }, + SourcesYmlPatch::SortAliases { + input_name: Arc::from("cdn-dev"), + order: AliasExpDateSortOrder::NewestFirst, + }, + ], + ) + .expect("patches apply"); + + assert!(changed); + let aliases = doc.inputs[0].aliases.as_ref().expect("aliases"); + assert_eq!(aliases.len(), 2); + assert_eq!(aliases[0].name.as_ref(), "cdn-dev-new"); + assert_eq!(aliases[0].id, 3); + assert_eq!(aliases[1].name.as_ref(), "cdn-dev-old"); + assert_eq!(aliases[1].id, 2); +} + +#[test] +fn sources_yml_sort_aliases_oldest_first_is_available_for_maintenance_paths() { + let mut doc = source_doc_with_aliases(vec![ + source_alias("cdn-dev-newest", Some(30)), + source_alias("cdn-dev-oldest", Some(10)), + source_alias("cdn-dev-missing-exp", None), + ]); + + let changed = apply_sources_yml_patches( + &mut doc, + &[SourcesYmlPatch::SortAliases { input_name: Arc::from("cdn-dev"), order: AliasExpDateSortOrder::OldestFirst }], + ) + .expect("patches apply"); + + assert!(changed); + let aliases = doc.inputs[0].aliases.as_ref().expect("aliases"); + assert_eq!(aliases[0].name.as_ref(), "cdn-dev-oldest"); + assert_eq!(aliases[1].name.as_ref(), "cdn-dev-newest"); + assert_eq!(aliases[2].name.as_ref(), "cdn-dev-missing-exp"); +} + +#[test] +fn sources_yml_update_exp_date_keeps_legacy_root_refresh_semantics() { + let mut doc = source_doc_with_aliases(Vec::new()); + doc.inputs[0].exp_date = Some(20); + doc.inputs[0].enabled = false; + doc.inputs[0].max_connections = 0; + + let changed = apply_sources_yml_patches( + &mut doc, + &[SourcesYmlPatch::UpdatePanelAccountExpiry { + input_name: Arc::from("cdn-dev"), + account_name: Arc::from("cdn-dev"), + exp_date: 20, + }], + ) + .expect("patches apply"); + + assert!(changed); + assert_eq!(doc.inputs[0].exp_date, Some(20)); + assert!(doc.inputs[0].enabled); + assert_eq!(doc.inputs[0].max_connections, 1); +} + +#[test] +fn sources_yml_update_exp_date_keeps_legacy_alias_refresh_semantics() { + let mut doc = source_doc_with_aliases(vec![source_alias("cdn-dev-old", Some(20))]); + doc.inputs[0].aliases.as_mut().expect("aliases")[0].max_connections = 0; + + let changed = apply_sources_yml_patches( + &mut doc, + &[SourcesYmlPatch::UpdatePanelAccountExpiry { + input_name: Arc::from("cdn-dev"), + account_name: Arc::from("cdn-dev-old"), + exp_date: 20, + }], + ) + .expect("patches apply"); + + assert!(changed); + let alias = &doc.inputs[0].aliases.as_ref().expect("aliases")[0]; + assert_eq!(alias.exp_date, Some(20)); + assert_eq!(alias.max_connections, 1); +} + +#[test] +fn sources_yml_update_root_credentials_updates_root_directly() { + let mut doc = source_doc_with_aliases(Vec::new()); + + let changed = apply_sources_yml_patches( + &mut doc, + &[SourcesYmlPatch::UpdateRootCredentials { + input_name: Arc::from("cdn-dev"), + username: "new-root".to_string(), + password: "new-pass".to_string(), + exp_date: Some(42), + }], + ) + .expect("patches apply"); + + assert!(changed); + assert_eq!(doc.inputs[0].username.as_deref(), Some("new-root")); + assert_eq!(doc.inputs[0].password.as_deref(), Some("new-pass")); + assert_eq!(doc.inputs[0].exp_date, Some(42)); + assert!(doc.inputs[0].enabled); + assert_eq!(doc.inputs[0].max_connections, 1); +} + +#[test] +fn resolve_base_url_keeps_provider_scheme_for_provisioned_accounts() { + let result = resolve_provisioned_account_base_url( + "provider://demo-provider/live", + Some("http://panel.example.com:8080/get.php?username=new&password=new"), + "new", + "secret", + ); + + assert_eq!(result, "provider://demo-provider/live"); +} + +#[test] +fn resolve_base_url_uses_panel_response_origin_for_http_inputs() { + let result = resolve_provisioned_account_base_url( + "http://input.example.org/path?x=1", + Some("http://panel.example.com:8080/get.php?username=new&password=new"), + "new", + "secret", + ); + + assert_eq!(result, "http://panel.example.com:8080"); +} + +#[test] +fn resolve_base_url_avoids_null_origin_for_non_special_schemes() { + let result = resolve_provisioned_account_base_url( + "http://input.example.org/path?x=1", + Some("custom-scheme://panel.example.com/path?username=new&password=new"), + "new", + "secret", + ); + + assert_eq!(result, "custom-scheme://panel.example.com/path?username=new&password=new"); +} + +#[test] +fn panel_api_probe_targets_preserve_provider_failover_context() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "tivione".into(), + urls: vec!["http://line-a.example.test".into(), "http://line-b.example.test".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let input = ConfigInput { + name: Arc::from("cdn-dev"), + url: "provider://tivione".to_string(), + provider_configs: Some(vec![Arc::clone(&provider)]), + ..ConfigInput::default() + }; + + let targets = build_panel_api_probe_targets(&input, "probe-user", "probe-pass"); + + assert_eq!(targets.len(), 4); + let PanelApiProbeTarget::PlayerApi { action, input_source } = &targets[0]; + assert_eq!(*action, "client_info"); + assert_eq!(input_source.provider.as_ref().expect("provider context should be preserved").name.as_ref(), "tivione"); + assert!(input_source.url.starts_with("provider://tivione/player_api.php?")); + assert!(input_source.url.contains("username=probe-user")); + assert!(input_source.url.contains("password=probe-pass")); + assert!(input_source.url.contains("action=client_info")); +} + +#[test] +fn panel_api_probe_targets_keep_plain_http_without_provider_context() { + let input = ConfigInput { + name: Arc::from("plain"), + url: "http://origin.example.test/some/path?ignored=1".to_string(), + ..ConfigInput::default() + }; + + let targets = build_panel_api_probe_targets(&input, "probe-user", "probe-pass"); + + assert_eq!(targets.len(), 4); + let PanelApiProbeTarget::PlayerApi { action, input_source } = &targets[0]; + assert_eq!(*action, "client_info"); + assert!(input_source.provider.is_none()); + assert!(input_source.url.starts_with("http://origin.example.test/player_api.php?")); + assert!(input_source.url.contains("username=probe-user")); + assert!(input_source.url.contains("password=probe-pass")); + assert!(input_source.url.contains("action=client_info")); +} + +#[test] +fn user_api_account_info_preserves_provider_failover_context() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "tivione".into(), + urls: vec!["http://line-a.example.test".into(), "http://line-b.example.test".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + })); + let input = ConfigInput { + name: Arc::from("cdn-dev"), + url: "provider://tivione".to_string(), + provider_configs: Some(vec![Arc::clone(&provider)]), + ..ConfigInput::default() + }; + + let input_source = build_user_api_account_info_input_source(&input, "root-user", "root-pass") + .expect("expected provider account_info input source"); + + assert_eq!(input_source.provider.as_ref().expect("provider context should be preserved").name.as_ref(), "tivione"); + assert!(input_source.url.starts_with("provider://tivione/player_api.php?")); + assert!(input_source.url.contains("username=root-user")); + assert!(input_source.url.contains("password=root-pass")); + assert!(input_source.url.contains("action=account_info")); +} + +#[test] +fn user_api_account_info_keeps_plain_http_without_provider_context() { + let input = ConfigInput { + name: Arc::from("plain"), + url: "http://origin.example.test/some/path?ignored=1".to_string(), + ..ConfigInput::default() + }; + + let input_source = build_user_api_account_info_input_source(&input, "root-user", "root-pass") + .expect("expected plain account_info input source"); + + assert!(input_source.provider.is_none()); + assert!(input_source.url.starts_with("http://origin.example.test/player_api.php?")); + assert!(input_source.url.contains("username=root-user")); + assert!(input_source.url.contains("password=root-pass")); + assert!(input_source.url.contains("action=account_info")); +} diff --git a/backend/app/src/api/panel_api/tests/lifecycle.rs b/backend/app/src/api/panel_api/tests/lifecycle.rs new file mode 100644 index 000000000..5bb40c83b --- /dev/null +++ b/backend/app/src/api/panel_api/tests/lifecycle.rs @@ -0,0 +1,28 @@ +use super::source_doc_with_aliases; +use crate::api::source_yml_patch::{apply_sources_yml_patches, SourcesYmlPatch}; +use std::sync::Arc; + +#[test] +fn sources_yml_persist_provisioned_account_replaces_root_when_current_root_is_expired() { + let mut doc = source_doc_with_aliases(Vec::new()); + doc.inputs[0].username = Some("current-root".to_string()); + doc.inputs[0].password = Some("current-pass".to_string()); + doc.inputs[0].exp_date = Some(i64::try_from(jsonwebtoken::get_current_timestamp()).expect("timestamp") - 1); + + let changed = apply_sources_yml_patches( + &mut doc, + &[SourcesYmlPatch::PersistProvisionedAccount { + input_name: Arc::from("cdn-dev"), + username: "new-root".to_string(), + password: "new-pass".to_string(), + exp_date: Some(42), + }], + ) + .expect("patches apply"); + + assert!(changed); + assert_eq!(doc.inputs[0].username.as_deref(), Some("new-root")); + assert_eq!(doc.inputs[0].password.as_deref(), Some("new-pass")); + assert_eq!(doc.inputs[0].exp_date, Some(42)); + assert!(doc.inputs[0].aliases.as_ref().is_none_or(Vec::is_empty)); +} diff --git a/backend/app/src/api/panel_api/tests/persistence.rs b/backend/app/src/api/panel_api/tests/persistence.rs new file mode 100644 index 000000000..4a9021aba --- /dev/null +++ b/backend/app/src/api/panel_api/tests/persistence.rs @@ -0,0 +1,34 @@ +use super::source_doc_with_aliases; +use crate::api::source_yml_patch::{apply_sources_yml_patches, SourcesYmlPatch}; +use std::sync::Arc; + +#[test] +fn sources_yml_persist_provisioned_account_adds_alias_when_current_root_is_valid() { + let mut doc = source_doc_with_aliases(Vec::new()); + doc.inputs[0].username = Some("current-root".to_string()); + doc.inputs[0].password = Some("current-pass".to_string()); + doc.inputs[0].exp_date = Some(i64::try_from(jsonwebtoken::get_current_timestamp()).expect("timestamp") + 3600); + + let changed = apply_sources_yml_patches( + &mut doc, + &[SourcesYmlPatch::PersistProvisionedAccount { + input_name: Arc::from("cdn-dev"), + username: "new-root".to_string(), + password: "new-pass".to_string(), + exp_date: Some(42), + }], + ) + .expect("patches apply"); + + assert!(changed); + assert_eq!(doc.inputs[0].username.as_deref(), Some("current-root")); + assert_eq!(doc.inputs[0].password.as_deref(), Some("current-pass")); + assert_ne!(doc.inputs[0].exp_date, Some(42)); + + let aliases = doc.inputs[0].aliases.as_ref().expect("aliases"); + assert_eq!(aliases.len(), 1); + assert_eq!(aliases[0].name.as_ref(), "cdn-dev-new-root"); + assert_eq!(aliases[0].username.as_deref(), Some("new-root")); + assert_eq!(aliases[0].password.as_deref(), Some("new-pass")); + assert_eq!(aliases[0].exp_date, Some(42)); +} diff --git a/backend/app/src/api/panel_api/tests/query.rs b/backend/app/src/api/panel_api/tests/query.rs new file mode 100644 index 000000000..7c172c879 --- /dev/null +++ b/backend/app/src/api/panel_api/tests/query.rs @@ -0,0 +1,26 @@ +use crate::api::source_yml_patch::resolve_provisioned_account_base_url; +use url::Url; + +#[test] +fn resolve_base_url_updates_provider_query_credentials_when_present() { + let result = resolve_provisioned_account_base_url( + "provider://demo-provider/live?foo=bar&username=old&password=oldpw", + Some("http://panel.example.com:8080/get.php?username=new&password=new"), + "new-user", + "new-pass", + ); + + let parsed = Url::parse(result.as_str()).expect("expected valid provider url"); + let pairs: Vec<(String, String)> = parsed.query_pairs().map(|(k, v)| (k.to_string(), v.to_string())).collect(); + assert!(pairs.contains(&("foo".to_string(), "bar".to_string()))); + assert!(pairs.contains(&("username".to_string(), "new-user".to_string()))); + assert!(pairs.contains(&("password".to_string(), "new-pass".to_string()))); +} + +#[test] +fn resolve_base_url_falls_back_when_panel_response_is_literal_null() { + let result = + resolve_provisioned_account_base_url("http://input.example.org/path?x=1", Some("null"), "new", "secret"); + + assert_eq!(result, "http://input.example.org/path?x=1"); +} diff --git a/backend/app/src/api/panel_api/tests/retry.rs b/backend/app/src/api/panel_api/tests/retry.rs new file mode 100644 index 000000000..99b4243bd --- /dev/null +++ b/backend/app/src/api/panel_api/tests/retry.rs @@ -0,0 +1,31 @@ +use super::{ + panel_api_retry_after_from_header_value, panel_api_retryable_status, PANEL_API_DEFAULT_RETRY_AFTER_SECS, + PANEL_API_MAX_RETRY_AFTER_SECS, +}; +use axum::http::StatusCode; +use std::time::Duration; + +#[test] +fn panel_api_retryable_status_covers_rate_limit_and_temporary_failures() { + assert!(panel_api_retryable_status(StatusCode::TOO_MANY_REQUESTS)); + assert!(panel_api_retryable_status(StatusCode::REQUEST_TIMEOUT)); + assert!(panel_api_retryable_status(StatusCode::TOO_EARLY)); + assert!(panel_api_retryable_status(StatusCode::BAD_GATEWAY)); + assert!(!panel_api_retryable_status(StatusCode::BAD_REQUEST)); + assert!(!panel_api_retryable_status(StatusCode::UNAUTHORIZED)); + assert!(!panel_api_retryable_status(StatusCode::NOT_FOUND)); +} + +#[test] +fn panel_api_retry_after_header_is_short_and_bounded() { + assert_eq!(panel_api_retry_after_from_header_value("2"), Some(Duration::from_secs(2))); + assert_eq!( + panel_api_retry_after_from_header_value("0"), + Some(Duration::from_secs(PANEL_API_DEFAULT_RETRY_AFTER_SECS)) + ); + assert_eq!( + panel_api_retry_after_from_header_value("600"), + Some(Duration::from_secs(PANEL_API_MAX_RETRY_AFTER_SECS)) + ); + assert_eq!(panel_api_retry_after_from_header_value("not-a-delay"), None); +} diff --git a/backend/app/src/api/panel_api/tests/support.rs b/backend/app/src/api/panel_api/tests/support.rs new file mode 100644 index 000000000..c1cb15093 --- /dev/null +++ b/backend/app/src/api/panel_api/tests/support.rs @@ -0,0 +1,17 @@ +use shared::model::{ConfigInputAliasDto, ConfigInputDto, InputType, SourcesConfigDto}; +use std::sync::Arc; + +pub(in crate::api::panel_api::tests) fn source_doc_with_aliases(aliases: Vec) -> SourcesConfigDto { + SourcesConfigDto { + inputs: vec![ConfigInputDto { + name: Arc::from("cdn-dev"), + input_type: InputType::Xtream, + url: "provider://tivione".to_string(), + username: Some("root-user".to_string()), + password: Some("root-pass".to_string()), + aliases: Some(aliases), + ..ConfigInputDto::default() + }], + ..SourcesConfigDto::default() + } +} diff --git a/backend/app/src/api/panel_api/time.rs b/backend/app/src/api/panel_api/time.rs new file mode 100644 index 000000000..33036dbce --- /dev/null +++ b/backend/app/src/api/panel_api/time.rs @@ -0,0 +1,181 @@ +use super::is_date_only_yyyy_mm_dd; +use crate::api::model::AppState; +use chrono::{NaiveDateTime, TimeZone}; +use chrono_tz::Tz; +use serde::{Deserialize, Serialize}; +use shared::{error::TuliproxError, utils::parse_timestamp}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(super) enum PanelApiExpireMode { + UtcString, + ServerTzString, +} + +#[derive(Debug, Clone)] +pub(super) struct PanelApiTimeContext { + pub(super) expire_mode: PanelApiExpireMode, + pub(super) server_tz: Option, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub(super) struct PanelApiTimeCache { + pub(super) inputs: HashMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(super) struct PanelApiTimeCacheEntry { + pub(super) expire_mode: PanelApiExpireMode, + pub(super) server_tz: Option, + pub(super) skew_secs: Option, +} + +pub(super) fn parse_panel_expire_utc(value: &str) -> Option { + let value = value.trim(); + if value.is_empty() { + return None; + } + let parsed = parse_timestamp(value).ok().flatten(); + if parsed.is_some() { + return parsed; + } + if is_date_only_yyyy_mm_dd(value) { + let normalized = format!("{value} 00:00:00"); + return parse_timestamp(&normalized).ok().flatten(); + } + None +} + +pub(super) fn parse_panel_expire_with_tz(value: &str, tz: Tz) -> Option { + let value = value.trim(); + if value.is_empty() { + return None; + } + if let Ok(ts) = value.parse::() { + return Some(ts); + } + let value = if is_date_only_yyyy_mm_dd(value) { format!("{value} 00:00:00") } else { value.to_string() }; + let dt = NaiveDateTime::parse_from_str(&value, "%Y-%m-%d %H:%M:%S").ok()?; + match tz.from_local_datetime(&dt) { + chrono::LocalResult::Single(local_dt) => Some(local_dt.timestamp()), + chrono::LocalResult::Ambiguous(first, _) => Some(first.timestamp()), + chrono::LocalResult::None => None, + } +} + +pub(super) fn normalize_panel_expire(value: &str, ctx: Option<&PanelApiTimeContext>) -> Option { + let Some(ctx) = ctx else { + return parse_panel_expire_utc(value); + }; + if let Ok(ts) = value.trim().parse::() { + return Some(ts); + } + match ctx.expire_mode { + PanelApiExpireMode::UtcString => parse_panel_expire_utc(value), + PanelApiExpireMode::ServerTzString => { + ctx.server_tz.and_then(|tz| parse_panel_expire_with_tz(value, tz)).or_else(|| parse_panel_expire_utc(value)) + } + } +} + +pub(super) fn is_expiring_with_offset_at(exp_date: Option, offset_secs: u64, now: u64) -> bool { + let Some(exp_date) = exp_date else { + return false; + }; + let Ok(exp_ts) = u64::try_from(exp_date) else { + return true; + }; + if exp_ts <= now { + return false; + } + now.saturating_add(offset_secs) >= exp_ts +} + +pub(super) fn resolve_panel_expire_mode( + root_expire: Option, + panel_expire: Option<&str>, + server_tz: Option, +) -> PanelApiExpireMode { + let Some(root_expire) = root_expire else { + return PanelApiExpireMode::UtcString; + }; + let Some(panel_expire) = panel_expire else { + return PanelApiExpireMode::UtcString; + }; + let utc_ts = parse_panel_expire_utc(panel_expire); + let tz_ts = server_tz.and_then(|tz| parse_panel_expire_with_tz(panel_expire, tz)); + let Some(utc_ts) = utc_ts else { + return tz_ts.map_or(PanelApiExpireMode::UtcString, |_| PanelApiExpireMode::ServerTzString); + }; + let Some(tz_ts) = tz_ts else { + return PanelApiExpireMode::UtcString; + }; + let diff_utc = (utc_ts - root_expire).abs(); + let diff_tz = (tz_ts - root_expire).abs(); + let threshold = 120_i64; + match (diff_utc <= threshold, diff_tz <= threshold) { + (true, false) => PanelApiExpireMode::UtcString, + (false, true) => PanelApiExpireMode::ServerTzString, + _ => { + if diff_tz < diff_utc { + PanelApiExpireMode::ServerTzString + } else { + PanelApiExpireMode::UtcString + } + } + } +} + +pub(super) fn apply_clock_skew(now: u64, skew_secs: i64) -> u64 { + if skew_secs == 0 { + return now; + } + if skew_secs.is_negative() { + now.saturating_sub(skew_secs.unsigned_abs()) + } else { + now.saturating_add(u64::try_from(skew_secs).unwrap_or(0)) + } +} + +pub(super) fn panel_api_time_cache_path(app_state: &AppState) -> PathBuf { + let paths = app_state.app_config.paths.load(); + PathBuf::from(&paths.config_path).join("panel-api").join("panel_api_time_cache.json") +} + +pub(super) async fn load_panel_api_time_cache(app_state: &AppState, cache_path: &Path) -> PanelApiTimeCache { + if let Some(parent) = cache_path.parent() { + if tokio::fs::create_dir_all(parent).await.is_err() { + return PanelApiTimeCache::default(); + } + } + let _lock = app_state.app_config.file_locks.read_lock(cache_path).await; + let Ok(content) = tokio::fs::read_to_string(cache_path).await else { + return PanelApiTimeCache::default(); + }; + serde_json::from_str(&content).unwrap_or_default() +} + +pub(super) async fn persist_panel_api_time_cache( + app_state: &AppState, + cache_path: &Path, + cache: &PanelApiTimeCache, +) -> Result<(), TuliproxError> { + if let Some(parent) = cache_path.parent() { + tokio::fs::create_dir_all(parent) + .await + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: failed to create cache dir: {e}")))?; + } + let content = + serde_json::to_string_pretty(cache).map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: {e}")))?; + let _lock = app_state.app_config.file_locks.write_lock(cache_path).await; + tokio::fs::write(cache_path, content) + .await + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: failed to persist time cache: {e}")))?; + Ok(()) +} + +pub(super) fn parse_cached_tz(tz: Option) -> Option { tz.and_then(|name| name.parse::().ok()) } diff --git a/backend/app/src/api/panel_api/transport.rs b/backend/app/src/api/panel_api/transport.rs new file mode 100644 index 000000000..74da97480 --- /dev/null +++ b/backend/app/src/api/panel_api/transport.rs @@ -0,0 +1,193 @@ +use super::{ + PANEL_API_DEFAULT_RETRY_AFTER_SECS, PANEL_API_MAX_RETRY_AFTER_SECS, PANEL_API_REQUEST_TIMEOUT_SECS, + PANEL_API_RETRY_ATTEMPTS, +}; +use crate::{ + api::model::AppState, + model::InputSource, + utils::{debug_if_enabled, format_http_status, request}, +}; +use axum::http::{header, HeaderMap, StatusCode}; +use chrono::{DateTime, Utc}; +use log::warn; +use serde_json::Value; +use shared::{error::TuliproxError, utils::sanitize_sensitive_info}; +use std::{collections::HashMap, time::Duration}; +use url::Url; + +pub(super) fn sanitize_panel_api_json_for_log(value: &Value, sanitize_sensitive: bool) -> Value { + match value { + Value::Array(arr) => { + Value::Array(arr.iter().map(|v| sanitize_panel_api_json_for_log(v, sanitize_sensitive)).collect()) + } + Value::Object(obj) => { + let mut out = serde_json::Map::with_capacity(obj.len()); + for (k, v) in obj { + if sanitize_sensitive { + if k.eq_ignore_ascii_case("api_key") + || k.eq_ignore_ascii_case("apikey") + || k.eq_ignore_ascii_case("token") + { + out.insert(k.clone(), Value::String("***".to_string())); + continue; + } + if k.eq_ignore_ascii_case("username") || k.eq_ignore_ascii_case("password") { + out.insert(k.clone(), Value::String("***".to_string())); + continue; + } + } + if k.eq_ignore_ascii_case("url") { + if let Some(s) = v.as_str() { + out.insert(k.clone(), Value::String(sanitize_sensitive_info(s).into_owned())); + continue; + } + } + out.insert(k.clone(), sanitize_panel_api_json_for_log(v, sanitize_sensitive)); + } + Value::Object(out) + } + other => other.clone(), + } +} + +pub(super) fn panel_api_retryable_status(status: StatusCode) -> bool { + status.is_server_error() + || matches!(status, StatusCode::TOO_MANY_REQUESTS | StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_EARLY) +} + +pub(super) fn panel_api_retry_after_from_header_value(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + if let Ok(secs) = trimmed.parse::() { + return Some(Duration::from_secs( + secs.clamp(PANEL_API_DEFAULT_RETRY_AFTER_SECS, PANEL_API_MAX_RETRY_AFTER_SECS), + )); + } + let retry_at = DateTime::parse_from_rfc2822(trimmed).ok()?.with_timezone(&Utc); + let now = Utc::now(); + let secs = retry_at.signed_duration_since(now).num_seconds(); + let secs = if secs <= 0 { + PANEL_API_DEFAULT_RETRY_AFTER_SECS + } else { + u64::try_from(secs).unwrap_or(PANEL_API_DEFAULT_RETRY_AFTER_SECS) + }; + Some(Duration::from_secs(secs.clamp(PANEL_API_DEFAULT_RETRY_AFTER_SECS, PANEL_API_MAX_RETRY_AFTER_SECS))) +} + +pub(super) fn panel_api_retry_after_from_headers(headers: &HeaderMap) -> Duration { + headers + .get(header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(panel_api_retry_after_from_header_value) + .unwrap_or_else(|| Duration::from_secs(PANEL_API_DEFAULT_RETRY_AFTER_SECS)) +} + +pub(super) async fn panel_get_json(app_state: &AppState, url: Url) -> Result { + let client = app_state.http_clients.default.load(); + let sanitized = sanitize_sensitive_info(url.as_str()); + for attempt in 0..PANEL_API_RETRY_ATTEMPTS { + debug_if_enabled!("panel_api request attempt {} of {}: {}", attempt + 1, PANEL_API_RETRY_ATTEMPTS, sanitized); + let resp = client + .get(url.clone()) + .timeout(Duration::from_secs(PANEL_API_REQUEST_TIMEOUT_SECS)) + .send() + .await + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api request failed: {e}")))?; + let status = resp.status(); + if !status.is_success() { + if attempt + 1 < PANEL_API_RETRY_ATTEMPTS && panel_api_retryable_status(status) { + let cooldown = panel_api_retry_after_from_headers(resp.headers()); + warn!( + "panel_api request returned {}, retrying after {:.3}s: {}", + format_http_status(status), + cooldown.as_secs_f64(), + sanitized + ); + tokio::time::sleep(cooldown).await; + continue; + } + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api request failed (http {}): {}", + format_http_status(status), + sanitized + ))); + } + let body = resp + .text() + .await + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api read response failed: {e}")))?; + let json: Value = serde_json::from_str(&body) + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api invalid json (http {status}): {e}")))?; + let sanitize_sensitive = + app_state.app_config.config.load().log.as_ref().is_none_or(|l| l.sanitize_sensitive_info); + let json_for_log = sanitize_panel_api_json_for_log(&json, sanitize_sensitive); + if let Ok(json_str) = serde_json::to_string(&json_for_log) { + debug_if_enabled!( + "panel_api response (http {}): {}", + format_http_status(status), + sanitize_sensitive_info(&json_str) + ); + } + return Ok(json); + } + Err(TuliproxError::ConfigPanelApi(format!("panel_api request failed: {sanitized}"))) +} + +pub(super) async fn user_api_get_json( + app_state: &AppState, + input_source: &InputSource, +) -> Result { + let client = app_state.http_clients.default.load(); + let url = Url::parse(input_source.url.as_str()).map_err(|e| { + TuliproxError::ConfigPanelApi(format!( + "panel_api user_api invalid url {}: {e}", + sanitize_sensitive_info(input_source.url.as_str()) + )) + })?; + debug_if_enabled!( + "panel_api user_api request {}", + sanitize_sensitive_info(&request::preview_request_diagnostics_for_logging(&url, input_source.get_provider())) + ); + + let config = app_state.app_config.config.load(); + let default_user_agent = config.default_user_agent.clone(); + let disabled_headers = config.get_disabled_headers(); + drop(config); + + let headers = request::get_request_headers( + Some(&input_source.headers), + None::<&HashMap>>, + disabled_headers.as_ref(), + default_user_agent.as_deref(), + ); + let resp = request::send_with_retry_and_provider( + &app_state.app_config, + &url, + input_source.get_provider(), + false, + |resolved_url| { + client.get(resolved_url.clone()).headers(headers.clone()).timeout(std::time::Duration::from_secs(30)) + }, + ) + .await + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api request failed: {e}")))?; + let status = resp.status(); + let body = resp + .text() + .await + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api read response failed: {e}")))?; + let json: Value = serde_json::from_str(&body) + .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api user_api invalid json (http {status}): {e}")))?; + let sanitize_sensitive = app_state.app_config.config.load().log.as_ref().is_none_or(|l| l.sanitize_sensitive_info); + let json_for_log = sanitize_panel_api_json_for_log(&json, sanitize_sensitive); + if let Ok(json_str) = serde_json::to_string(&json_for_log) { + debug_if_enabled!( + "panel_api user_api response (http {}): {}", + format_http_status(status), + sanitize_sensitive_info(&json_str) + ); + } + Ok(json) +} diff --git a/backend/app/src/api/source_yml_patch.rs b/backend/app/src/api/source_yml_patch.rs index 0da1de134..ccdbbe5b3 100644 --- a/backend/app/src/api/source_yml_patch.rs +++ b/backend/app/src/api/source_yml_patch.rs @@ -1221,14 +1221,14 @@ mod tests { let mut edits = Vec::new(); plan_scalar_field_edits(fixture, &doc, &Arc::from("provider"), &before, &expected, &mut edits).expect("plan"); - let patched = edits_to_text(fixture, edits); + let patched_text = edits_to_text(fixture, edits); for line in fixture.lines() { - assert!(patched.contains(line), "line lost after patch: {line}"); + assert!(patched_text.contains(line), "line lost after patch: {line}"); } - assert!(patched.contains("exp_date: 1700000000")); + assert!(patched_text.contains("exp_date: 1700000000")); // The exact bug signature: `pass\n exp_date: ...` glued with no separator. assert!( - !patched.contains("pass1700000000") && !patched.contains("pass exp_date"), + !patched_text.contains("pass1700000000") && !patched_text.contains("pass exp_date"), "next line must not be glued to the inserted value" ); } @@ -1261,9 +1261,9 @@ mod tests { .expect("plan"); assert_eq!(edits.len(), 1, "exactly one replacement edit, no insertion"); - let patched = edits_to_text(fixture, edits); - assert!(patched.contains("enabled: false")); - assert_eq!(patched.matches("enabled:").count(), 1, "no duplicate `enabled:` key"); + let patched_text = edits_to_text(fixture, edits); + assert!(patched_text.contains("enabled: false")); + assert_eq!(patched_text.matches("enabled:").count(), 1, "no duplicate `enabled:` key"); } #[test] @@ -1291,9 +1291,9 @@ mod tests { plan_scalar_field_edits(fixture, &doc, &Arc::from("provider"), &before, &expected, &mut edits).expect("plan"); assert!(find_edit(&edits, "credits").is_some(), "credits edit planned"); - let patched = edits_to_text(fixture, edits); + let patched_text = edits_to_text(fixture, edits); let reparsed: crate::config_loader::source_patch::SourcePatchDocument = - serde_saphyr::from_str(&patched).expect("reparse"); + serde_saphyr::from_str(&patched_text).expect("reparse"); let panel = reparsed.inputs[0].value.panel_api.as_ref().expect("panel"); assert_eq!(panel.value.credits.as_ref().expect("credits").value, "42"); } @@ -1307,12 +1307,12 @@ mod tests { let anchor = span_byte_range(&input.name).expect("name span"); let edit = build_field_insertion_edit(fixture, &anchor, "enabled", "false").expect("build edit"); - let patched = edits_to_text(fixture, vec![edit]); + let patched_text = edits_to_text(fixture, vec![edit]); let reparsed: crate::config_loader::source_patch::SourcePatchDocument = - serde_saphyr::from_str(&patched).expect("reparse"); + serde_saphyr::from_str(&patched_text).expect("reparse"); assert!(!reparsed.inputs[0].value.enabled.as_ref().expect("enabled").value); - assert!(patched.contains("name: provider, enabled: false")); + assert!(patched_text.contains("name: provider, enabled: false")); } // ----------------------------------------------------------------------- @@ -1490,8 +1490,8 @@ sources: [] }, ]; assert!(execute_fixture_patches(&app_cfg, &source_path, &patches).await.expect("patch")); - let patched = std::fs::read_to_string(&source_path).expect("read patched"); - let mut parsed: SourcesConfigDto = serde_saphyr::from_str(&patched).expect("reparse"); + let patched_text = std::fs::read_to_string(&source_path).expect("read patched_text"); + let mut parsed: SourcesConfigDto = serde_saphyr::from_str(&patched_text).expect("reparse"); let input = &parsed.inputs[0]; assert_eq!(input.username.as_deref(), Some("renewed-root")); assert_eq!(input.exp_date, Some(4_102_444_900)); @@ -1504,15 +1504,15 @@ sources: [] ["added", "current"] ); assert_eq!(aliases[1].password.as_deref(), Some("pass # with: {}, commas")); - assert!(patched.contains("# - {name: commented-old, url: provider://example}")); + assert!(patched_text.contains("# - {name: commented-old, url: provider://example}")); let panel_start = fixture.find(" # keep panel settings").expect("panel"); - assert!(patched.ends_with(&fixture[panel_start..])); + assert!(patched_text.ends_with(&fixture[panel_start..])); let current_prefix = if style == "block" { "- name: current" } else { "- {name: current" }; - assert!(patched.contains(current_prefix)); + assert!(patched_text.contains(current_prefix)); // The initial sort makes the older entry the last item. In // mixed lists its block style is the insertion convention. let added_prefix = if style == "flow" { "- {name: added" } else { "- name: added" }; - assert!(patched.contains(added_prefix)); + assert!(patched_text.contains(added_prefix)); assert!(std::fs::read_dir(&backup_dir).expect("backups").any(|entry| { let path = entry.expect("backup entry").path(); std::fs::read(path).is_ok_and(|bytes| bytes == fixture.as_bytes()) @@ -1628,8 +1628,8 @@ sources: [] let written = execute_fixture_patches(&app_cfg, &source_path, &patches).await.expect("patch"); assert!(written, "patch should report a write happened"); - let patched_text = tokio::fs::read_to_string(&source_path).await.expect("read patched"); - assert!(patched_text.contains("exp_date: 1700000000"), "patched file must contain inserted exp_date"); + let patched_text = tokio::fs::read_to_string(&source_path).await.expect("read patched_text"); + assert!(patched_text.contains("exp_date: 1700000000"), "patched_text file must contain inserted exp_date"); for line in FIXTURE.lines() { assert!(patched_text.contains(line), "line lost after patch: {line}"); } @@ -1673,8 +1673,8 @@ sources: [] ]; assert!(execute_fixture_patches(&app_cfg, &source_path, &patches).await.expect("patch")); - let patched = tokio::fs::read_to_string(&source_path).await.expect("read patched"); - let parsed: SourcesConfigDto = serde_saphyr::from_str(&patched).expect("reparse"); + let patched_text = tokio::fs::read_to_string(&source_path).await.expect("read patched_text"); + let parsed: SourcesConfigDto = serde_saphyr::from_str(&patched_text).expect("reparse"); assert_eq!(parsed.inputs[0].exp_date, Some(1_700_000_000)); assert_eq!(parsed.inputs[0].aliases.as_ref().expect("aliases")[0].exp_date, Some(1_800_000_000)); @@ -1840,10 +1840,10 @@ sources: [] ]; assert!(execute_fixture_patches(&app_cfg, &source_path, &patches).await.expect("patch")); - let patched = tokio::fs::read_to_string(&source_path).await.expect("read"); - assert!(patched.find("name: new").expect("new") < patched.find("name: old").expect("old")); - assert_eq!(patched.matches("# old account").count(), 1); - assert!(patched.contains("\r\n")); + let patched_text = tokio::fs::read_to_string(&source_path).await.expect("read"); + assert!(patched_text.find("name: new").expect("new") < patched_text.find("name: old").expect("old")); + assert_eq!(patched_text.matches("# old account").count(), 1); + assert!(patched_text.contains("\r\n")); let _ = std::fs::remove_dir_all(&dir); } @@ -1887,8 +1887,8 @@ sources: [] }]; assert!(execute_fixture_patches(&app_cfg, &source_path, &patches).await.expect("patch")); - let patched = tokio::fs::read_to_string(&source_path).await.expect("read"); - let parsed: SourcesConfigDto = serde_saphyr::from_str(&patched).expect("reparse"); + let patched_text = tokio::fs::read_to_string(&source_path).await.expect("read"); + let parsed: SourcesConfigDto = serde_saphyr::from_str(&patched_text).expect("reparse"); assert_eq!(parsed.inputs.len(), 2); assert!(parsed.inputs[0].panel_api.is_some()); @@ -1896,8 +1896,8 @@ sources: [] assert_eq!(aliases.len(), 1); assert_eq!(aliases[0].name.as_ref(), "provider-new"); assert_eq!(parsed.inputs[1].name.as_ref(), "next"); - assert!(patched.contains(" aliases:\n - name: provider-new")); - assert!(patched.contains(" remove_expired: true")); + assert!(patched_text.contains(" aliases:\n - name: provider-new")); + assert!(patched_text.contains(" remove_expired: true")); let _ = std::fs::remove_dir_all(&dir); } @@ -1937,10 +1937,12 @@ sources: [] ]; assert!(execute_fixture_patches(&app_cfg, &source_path, &patches).await.expect("patch")); - let patched = tokio::fs::read_to_string(&source_path).await.expect("read"); - assert!(patched.find("name: first").expect("first") < patched.find("name: second").expect("second")); - assert!(patched.contains("exp_date: 300")); - assert!(patched.contains("max_connections: 1")); + let patched_text = tokio::fs::read_to_string(&source_path).await.expect("read"); + assert!( + patched_text.find("name: first").expect("first") < patched_text.find("name: second").expect("second") + ); + assert!(patched_text.contains("exp_date: 300")); + assert!(patched_text.contains("max_connections: 1")); let _ = std::fs::remove_dir_all(&dir); } @@ -1967,9 +1969,9 @@ sources: [] let patches = [SourcesYmlPatch::RemoveExpiredAliases { input_name: Arc::from("provider") }]; assert!(execute_fixture_patches(&app_cfg, &source_path, &patches).await.expect("patch")); - let patched = tokio::fs::read_to_string(&source_path).await.expect("read"); - assert!(!patched.contains("name: expired")); - let parsed: SourcesConfigDto = serde_saphyr::from_str(&patched).expect("reparse"); + let patched_text = tokio::fs::read_to_string(&source_path).await.expect("read"); + assert!(!patched_text.contains("name: expired")); + let parsed: SourcesConfigDto = serde_saphyr::from_str(&patched_text).expect("reparse"); assert!(parsed.inputs[0].aliases.as_ref().is_none_or(Vec::is_empty)); let _ = std::fs::remove_dir_all(&dir); diff --git a/backend/app/src/api/sys_usage.rs b/backend/app/src/api/sys_usage.rs index 0c6c3a2f4..f318bb68a 100644 --- a/backend/app/src/api/sys_usage.rs +++ b/backend/app/src/api/sys_usage.rs @@ -115,9 +115,9 @@ impl DiskProbe { let ok = unsafe { windows_sys::Win32::Storage::FileSystem::GetDiskFreeSpaceExW( self.path.0.as_ptr(), - &mut free_bytes_available, - &mut total_bytes, - &mut total_free_bytes, + &raw mut free_bytes_available, + &raw mut total_bytes, + &raw mut total_free_bytes, ) }; if ok == 0 { @@ -149,6 +149,10 @@ impl DiskPath { fn from_path(path: &std::path::Path) -> Option { use std::os::windows::ffi::OsStrExt; let mut wide: Vec = path.as_os_str().encode_wide().collect(); + // Mirror the Unix `CString` check: an embedded NUL would truncate the path. + if wide.contains(&0) { + return None; + } wide.push(0); Some(Self(wide)) } diff --git a/backend/btree/src/v3/tree.rs b/backend/btree/src/v3/tree.rs index dba2f5e92..fd2fe3f2c 100644 --- a/backend/btree/src/v3/tree.rs +++ b/backend/btree/src/v3/tree.rs @@ -13,93 +13,17 @@ use super::{ }, BPlusTreeMetadata, }; -use crate::{ - codec::{binary_deserialize, binary_serialize, binary_serialize_into}, - common::{mmap_with_advice, read_exact_at_offset, write_all_at_offset, Advice, BPlusTreeError}, -}; -use memmap2::Mmap; use parking_lot::Mutex; -use serde::{Deserialize, Serialize}; use std::{ - collections::{BTreeMap, HashMap, HashSet}, - fs::{File, OpenOptions}, - io::{self, Read}, + collections::BTreeMap, + io::{self}, marker::PhantomData, ops::{Bound, Range}, - path::{Path, PathBuf}, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, OnceLock, - }, + path::PathBuf, + sync::{atomic::AtomicBool, Arc}, thread::JoinHandle, - time::Duration, }; -fn invalid_data(message: &'static str) -> io::Error { io::Error::new(io::ErrorKind::InvalidData, message) } - -fn invalid_input(message: &'static str) -> io::Error { io::Error::new(io::ErrorKind::InvalidInput, message) } - -fn checked_page_usage<'a, I>(base: usize, cells: I, maximum_cell_footprint: usize) -> io::Result -where - I: IntoIterator, -{ - cells.into_iter().try_fold(base, |used, cell| { - let footprint = SLOT_LEN.checked_add(cell.len()).ok_or_else(|| invalid_input("cell footprint overflow"))?; - if footprint > maximum_cell_footprint { - return Err(invalid_input("cell footprint exceeds format limit")); - } - used.checked_add(footprint).ok_or_else(|| invalid_input("page usage overflow")) - }) -} - -pub(crate) fn used_leaf_bytes>(cells: &[T]) -> io::Result { - checked_page_usage(PAGE_HEADER_LEN, cells.iter().map(AsRef::as_ref), MAX_CELL_FOOTPRINT) -} - -pub(crate) fn choose_leaf_split>(cells: &[T]) -> io::Result { - if cells.len() < 2 { - return Err(invalid_input("leaf split requires two non-empty outputs")); - } - let mut total = 0usize; - for cell in cells { - let footprint = - SLOT_LEN.checked_add(cell.as_ref().len()).ok_or_else(|| invalid_input("leaf cell footprint overflow"))?; - if footprint > MAX_CELL_FOOTPRINT { - return Err(invalid_input("leaf cell footprint exceeds format limit")); - } - total = total.checked_add(footprint).ok_or_else(|| invalid_input("leaf split usage overflow"))?; - } - - let mut left_payload = 0usize; - let mut best = None; - for boundary in 1..cells.len() { - let cell = cells.get(boundary - 1).ok_or_else(|| invalid_input("leaf split boundary is outside cells"))?; - let footprint = - SLOT_LEN.checked_add(cell.as_ref().len()).ok_or_else(|| invalid_input("leaf cell footprint overflow"))?; - left_payload = left_payload.checked_add(footprint).ok_or_else(|| invalid_input("leaf split usage overflow"))?; - let right_payload = - total.checked_sub(left_payload).ok_or_else(|| invalid_input("leaf split usage underflow"))?; - let left_used = - PAGE_HEADER_LEN.checked_add(left_payload).ok_or_else(|| invalid_input("leaf split usage overflow"))?; - let right_used = - PAGE_HEADER_LEN.checked_add(right_payload).ok_or_else(|| invalid_input("leaf split usage overflow"))?; - if left_used <= PAGE_SIZE && right_used <= PAGE_SIZE { - let imbalance = left_used.abs_diff(right_used); - if best.is_none_or(|(_, best_imbalance)| imbalance < best_imbalance) { - best = Some((boundary, imbalance)); - } - } - } - best.map(|(boundary, _)| boundary).ok_or_else(|| invalid_input("leaf split has no valid boundary")) -} - -fn used_internal_bytes<'a, I>(cells: I) -> io::Result -where - I: IntoIterator, -{ - checked_page_usage(PAGE_HEADER_LEN + 8, cells, SLOT_LEN + 12 + 2004) -} - #[derive(Debug)] pub(crate) struct InternalSplit<'a, T> { #[cfg(test)] @@ -110,210 +34,11 @@ pub(crate) struct InternalSplit<'a, T> { pub(crate) right_cells: &'a [T], } -pub(crate) fn choose_internal_split>( - cells: &[T], - page_id: u64, - next_page_id: u64, -) -> io::Result> { - if cells.len() < 3 { - return Err(invalid_input("internal split requires two non-empty outputs and a promoted separator")); - } - for cell in cells { - InternalCellRef::decode(cell.as_ref(), page_id, next_page_id)?; - } - - let mut best = None; - for promoted_index in 1..cells.len() - 1 { - let left_used = used_internal_bytes(cells[..promoted_index].iter().map(AsRef::as_ref))?; - let right_used = used_internal_bytes(cells[promoted_index + 1..].iter().map(AsRef::as_ref))?; - if left_used <= PAGE_SIZE && right_used <= PAGE_SIZE { - let imbalance = left_used.abs_diff(right_used); - if best.is_none_or(|(_, best_imbalance)| imbalance < best_imbalance) { - best = Some((promoted_index, imbalance)); - } - } - } - let promoted_index = - best.map(|(index, _)| index).ok_or_else(|| invalid_input("internal split has no valid boundary"))?; - let promoted = InternalCellRef::decode( - cells.get(promoted_index).ok_or_else(|| invalid_input("promoted separator is outside cells"))?.as_ref(), - page_id, - next_page_id, - )?; - Ok(InternalSplit { - #[cfg(test)] - promoted_index, - right_leftmost_child: promoted.right_child, - promoted, - left_cells: cells.get(..promoted_index).ok_or_else(|| invalid_input("left split range is outside cells"))?, - right_cells: cells - .get(promoted_index + 1..) - .ok_or_else(|| invalid_input("right split range is outside cells"))?, - }) -} - -fn decode_key(encoded: &[u8]) -> io::Result -where - K: for<'de> Deserialize<'de>, -{ - rmp_serde::from_slice(encoded) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("invalid serialized key: {err}"))) -} - #[cfg(test)] thread_local! { static INTERNAL_KEY_DECODE_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; } -#[cfg(test)] -fn reset_internal_key_decode_count() { INTERNAL_KEY_DECODE_COUNT.set(0); } - -#[cfg(test)] -fn internal_key_decode_count() -> usize { INTERNAL_KEY_DECODE_COUNT.get() } - -fn decode_internal_key(encoded: &[u8]) -> io::Result -where - K: for<'de> Deserialize<'de>, -{ - #[cfg(test)] - INTERNAL_KEY_DECODE_COUNT.with(|count| count.set(count.get().saturating_add(1))); - decode_key(encoded) -} - -pub(crate) fn search_leaf(page: &SlottedPage, target: &K) -> io::Result> -where - K: Ord + for<'de> Deserialize<'de>, - B: AsRef<[u8]>, -{ - if page.header().page_type != PageType::Leaf { - return Err(invalid_data("typed leaf search requires a leaf page")); - } - let mut left = 0usize; - let mut right = usize::from(page.header().cell_count); - while left < right { - let middle = left + (right - left) / 2; - let cell = LeafCellRef::decode(page.cell(middle)?, page.page_id(), page.next_page_id())?; - match decode_key::(cell.key_bytes)?.cmp(target) { - std::cmp::Ordering::Less => left = middle + 1, - std::cmp::Ordering::Greater => right = middle, - std::cmp::Ordering::Equal => return Ok(Ok(middle)), - } - } - Ok(Err(left)) -} - -#[cfg(test)] -pub(crate) fn validate_locator>( - page: &SlottedPage, - locator: Locator, - serialized_primary_key: &[u8], -) -> io::Result<()> { - if page.header().page_type != PageType::Leaf || page.page_id() != locator.leaf_page_id { - return Err(invalid_data("locator does not reference this leaf page")); - } - let cell = LeafCellRef::decode(page.cell(usize::from(locator.slot_index))?, page.page_id(), page.next_page_id())?; - let cell_crc = crc32fast::hash(cell.key_bytes); - if cell_crc != locator.serialized_key_crc32 - || crc32fast::hash(serialized_primary_key) != locator.serialized_key_crc32 - || cell.key_bytes != serialized_primary_key - { - return Err(invalid_data("locator serialized key mismatch")); - } - Ok(()) -} - -#[cfg(test)] -fn database_page(database: &[u8], page_id: u64, next_page_id: u64) -> io::Result<&[u8]> { - if page_id == 0 || page_id >= next_page_id { - return Err(invalid_data("overflow page id is outside database")); - } - let page_id = usize::try_from(page_id).map_err(|_| invalid_data("overflow page id exceeds usize"))?; - let offset = page_id.checked_mul(PAGE_SIZE).ok_or_else(|| invalid_data("overflow page offset overflow"))?; - let end = offset.checked_add(PAGE_SIZE).ok_or_else(|| invalid_data("overflow page end overflow"))?; - database.get(offset..end).ok_or_else(|| invalid_data("truncated overflow page")) -} - -#[cfg(test)] -pub(crate) fn read_leaf_value<'a>( - database: &'a [u8], - value: &LeafValueRef<'a>, - next_page_id: u64, - maximum_length: usize, - scratch: &'a mut Vec, -) -> io::Result> { - match *value { - LeafValueRef::Tombstone => Ok(None), - LeafValueRef::Inline { compression, logical_len, stored, .. } => { - let logical_len = usize::try_from(logical_len).map_err(|_| invalid_data("logical length exceeds usize"))?; - if logical_len > maximum_length { - return Err(invalid_data("logical length exceeds allocation limit")); - } - match compression { - Compression::None => Ok(Some(stored)), - Compression::Lz4 => decompress_value_into( - stored, - u32::try_from(logical_len).map_err(|_| invalid_data("logical length exceeds u32"))?, - maximum_length, - scratch, - ) - .map(Some), - } - } - LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } => { - let logical_length = - usize::try_from(logical_len).map_err(|_| invalid_data("logical length exceeds usize"))?; - let stored_length = usize::try_from(stored_len).map_err(|_| invalid_data("stored length exceeds usize"))?; - if logical_length > maximum_length || stored_length > maximum_length { - return Err(invalid_data("overflow value exceeds allocation limit")); - } - scratch.clear(); - scratch.try_reserve(stored_length).map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; - let mut page_id = head; - let mut visited = HashSet::new(); - while page_id != 0 { - if u64::try_from(visited.len()).map_err(|_| invalid_data("overflow chain length exceeds u64"))? - >= next_page_id - { - return Err(invalid_data("overflow chain contains a cycle")); - } - let page = SlottedPage::open(database_page(database, page_id, next_page_id)?, page_id, next_page_id)?; - if page.header().page_type != PageType::Overflow { - return Err(invalid_data("overflow chain references a non-overflow page")); - } - let payload = overflow_payload(&page)?; - if payload.is_empty() { - return Err(invalid_data("overflow chain contains an empty payload")); - } - visited.try_reserve(1).map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; - if !visited.insert(page_id) { - return Err(invalid_data("overflow chain contains a cycle")); - } - let new_length = scratch - .len() - .checked_add(payload.len()) - .ok_or_else(|| invalid_data("overflow chain length overflow"))?; - if new_length > stored_length { - return Err(invalid_data("overflow chain exceeds stored length")); - } - scratch.extend_from_slice(payload); - page_id = page.header().right; - } - if scratch.len() != stored_length { - return Err(invalid_data("overflow chain stored length mismatch")); - } - if crc32fast::hash(scratch) != crc32 { - return Err(invalid_data("stored value checksum mismatch")); - } - if compression == Compression::Lz4 { - decompress_value_in_place(scratch, logical_len, maximum_length)?; - } else if scratch.len() != logical_length { - return Err(invalid_data("uncompressed overflow length mismatch")); - } - Ok(Some(scratch.as_slice())) - } - } -} - #[derive(Clone, Debug)] pub struct BPlusTree { entries: BTreeMap, @@ -321,777 +46,14 @@ pub struct BPlusTree { dirty: bool, } -impl Default for BPlusTree { - fn default() -> Self { Self::new() } -} - -impl BPlusTree { - pub const fn new() -> Self { Self { entries: BTreeMap::new(), metadata: BPlusTreeMetadata::Empty, dirty: true } } - - pub fn get_metadata(&self) -> &BPlusTreeMetadata { &self.metadata } - - pub fn set_metadata(&mut self, metadata: BPlusTreeMetadata) { - self.metadata = metadata; - self.dirty = true; - } - - pub fn is_empty(&self) -> bool { self.entries.is_empty() } - - pub fn len(&self) -> usize { self.entries.len() } - - pub fn insert(&mut self, key: K, value: V) { - let _ = self.entries.insert(key, value); - self.dirty = true; - } - - pub fn query(&self, key: &K) -> Option<&V> { self.entries.get(key) } - - pub fn find_le(&self, key: &K) -> Option<(&K, &V)> { self.entries.range(..=key).next_back() } - - pub fn iter(&self) -> std::collections::btree_map::Iter<'_, K, V> { self.entries.iter() } -} - -impl<'a, K: Ord, V> IntoIterator for &'a BPlusTree { - type Item = (&'a K, &'a V); - type IntoIter = std::collections::btree_map::Iter<'a, K, V>; - - fn into_iter(self) -> Self::IntoIter { self.entries.iter() } -} - -#[derive(Clone)] -struct NodeInfo { - page_id: u64, - minimum_key: Vec, -} - -/// Sink for the bulk tree builder: hands out page ids and writes each finished page -/// straight to the destination file instead of buffering the whole database in memory. -/// -/// Pages are **not** emitted in ascending id order — a leaf reserves its id before the -/// overflow chain it points at, but is encoded only once all of its cells are known, so -/// it is written after them. Writes must therefore be positional; a sequential writer -/// would silently scramble the file. Every id handed out is written exactly once, so the -/// finished file has no holes, and `verify_full` re-reads it before it is published. -struct PageSink { - file: File, - next_page_id: u64, -} - const PAGE_SIZE_U64: u64 = PAGE_SIZE as u64; -impl PageSink { - const fn new(file: File) -> Self { Self { file, next_page_id: 0 } } - - fn allocate(&mut self) -> io::Result { - let page_id = self.next_page_id; - self.next_page_id = page_id.checked_add(1).ok_or_else(|| invalid_input("page count exceeds u64"))?; - Ok(page_id) - } - - fn allocate_many(&mut self, count: usize) -> io::Result { - let head = self.next_page_id; - let count = u64::try_from(count).map_err(|_| invalid_input("page count exceeds u64"))?; - self.next_page_id = head.checked_add(count).ok_or_else(|| invalid_input("page count exceeds u64"))?; - Ok(head) - } - - fn write(&self, page_id: u64, page: &[u8; PAGE_SIZE]) -> io::Result<()> { - let offset = page_id.checked_mul(PAGE_SIZE_U64).ok_or_else(|| invalid_input("page offset overflow"))?; - write_all_at_offset(&self.file, page, offset) - } -} - -fn leaf_page>(page_id: u64, left: u64, right: u64, cells: &[T]) -> io::Result<[u8; PAGE_SIZE]> { - let mut bytes = [0; PAGE_SIZE]; - PageHeader { - page_type: PageType::Leaf, - cell_count: 0, - free_start: u16::try_from(PAGE_HEADER_LEN).map_err(|_| invalid_input("leaf header exceeds u16"))?, - free_end: u16::try_from(PAGE_SIZE).map_err(|_| invalid_input("page size exceeds u16"))?, - left, - right, - } - .encode_into(&mut bytes, page_id, u64::MAX)?; - SlottedPage::open(bytes.as_mut_slice(), page_id, u64::MAX)?.rebuild_ordered(cells.iter().map(AsRef::as_ref))?; - Ok(bytes) -} - -fn internal_page>(page_id: u64, leftmost_child: u64, cells: &[T]) -> io::Result<[u8; PAGE_SIZE]> { - let first = cells.first().ok_or_else(|| invalid_input("internal page must contain a separator"))?.as_ref(); - let first_offset = PAGE_SIZE.checked_sub(first.len()).ok_or_else(|| invalid_input("internal cell exceeds page"))?; - let mut bytes = [0; PAGE_SIZE]; - InternalPreamble { leftmost_child }.encode_into(&mut bytes, page_id, u64::MAX)?; - bytes.get_mut(40..44).ok_or_else(|| invalid_input("internal slot is outside page"))?.copy_from_slice( - &Slot { - offset: u16::try_from(first_offset).map_err(|_| invalid_input("internal cell offset exceeds u16"))?, - length: u16::try_from(first.len()).map_err(|_| invalid_input("internal cell length exceeds u16"))?, - } - .encode(), - ); - bytes.get_mut(first_offset..).ok_or_else(|| invalid_input("internal cell is outside page"))?.copy_from_slice(first); - PageHeader { - page_type: PageType::Internal, - cell_count: 1, - free_start: 44, - free_end: u16::try_from(first_offset).map_err(|_| invalid_input("internal free_end exceeds u16"))?, - left: 0, - right: 0, - } - .encode_into(&mut bytes, page_id, u64::MAX)?; - SlottedPage::open(bytes.as_mut_slice(), page_id, u64::MAX)?.rebuild_ordered(cells.iter().map(AsRef::as_ref))?; - Ok(bytes) -} - -fn allocate_overflow_chain(sink: &mut PageSink, stored: &[u8]) -> io::Result { - let count = stored.len().div_ceil(OVERFLOW_PAYLOAD_LEN); - if count == 0 { - return Err(invalid_input("overflow value must not be empty")); - } - let head = sink.allocate_many(count)?; - for (index, payload) in stored.chunks(OVERFLOW_PAYLOAD_LEN).enumerate() { - let page_id = head - .checked_add(u64::try_from(index).map_err(|_| invalid_input("overflow index exceeds u64"))?) - .ok_or_else(|| invalid_input("overflow page id overflow"))?; - let next = if index + 1 == count { - 0 - } else { - page_id.checked_add(1).ok_or_else(|| invalid_input("overflow page id overflow"))? - }; - let page = encode_overflow_page(page_id, u64::MAX, next, payload)?; - sink.write(page_id, &page)?; - } - Ok(head) -} - -fn finish_leaf(sink: &PageSink, page_id: u64, left: u64, right: u64, cells: &[Vec]) -> io::Result<()> { - let page = leaf_page(page_id, left, right, cells)?; - sink.write(page_id, &page) -} - -fn build_leaf_level(entries: &BTreeMap, sink: &mut PageSink) -> io::Result> -where - K: Ord + Serialize, - V: Serialize, -{ - let mut leaf_id = sink.allocate()?; - let mut left_leaf = 0; - let mut cells = Vec::>::new(); - let mut leaves = Vec::::new(); - let mut leaf_minimum = None; - let mut compression_scratch = Vec::new(); - let mut cell = Vec::new(); - - for (key, value) in entries { - let key_bytes = binary_serialize(key)?; - let raw_value = binary_serialize(value)?; - let logical_len = u32::try_from(raw_value.len()).map_err(|_| invalid_input("serialized value exceeds u32"))?; - let stored = encode_value(&raw_value, &mut compression_scratch)?; - let stored_bytes = stored.as_slice(); - let inline_footprint = SLOT_LEN - .checked_add(24) - .and_then(|size| size.checked_add(key_bytes.len())) - .and_then(|size| size.checked_add(stored_bytes.len())) - .ok_or_else(|| invalid_input("leaf cell footprint overflow"))?; - if stored_bytes.len() <= MAX_INLINE_STORED_VALUE && inline_footprint <= MAX_CELL_FOOTPRINT { - encode_inline_leaf_cell(&key_bytes, logical_len, stored.compression(), stored_bytes, &mut cell)?; - } else { - let head = allocate_overflow_chain(sink, stored_bytes)?; - encode_overflow_leaf_cell( - &key_bytes, - logical_len, - stored.compression(), - u32::try_from(stored_bytes.len()).map_err(|_| invalid_input("stored value exceeds u32"))?, - head, - stored_value_checksum(stored_bytes), - leaf_id, - u64::MAX, - &mut cell, - )?; - } - - let next_usage = checked_page_usage( - PAGE_HEADER_LEN, - cells.iter().map(Vec::as_slice).chain(std::iter::once(cell.as_slice())), - MAX_CELL_FOOTPRINT, - )?; - if next_usage > PAGE_SIZE { - let next_leaf = sink.allocate()?; - finish_leaf(sink, leaf_id, left_leaf, next_leaf, &cells)?; - leaves.push(NodeInfo { - page_id: leaf_id, - minimum_key: leaf_minimum.take().ok_or_else(|| invalid_input("non-empty leaf has no minimum key"))?, - }); - left_leaf = leaf_id; - leaf_id = next_leaf; - cells.clear(); - } - if cells.is_empty() { - leaf_minimum = Some(key_bytes); - } - cells.try_reserve(1).map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; - cells.push(std::mem::take(&mut cell)); - } - - finish_leaf(sink, leaf_id, left_leaf, 0, &cells)?; - leaves.push(NodeInfo { page_id: leaf_id, minimum_key: leaf_minimum.unwrap_or_default() }); - Ok(leaves) -} - -fn group_internal_children(level: &[NodeInfo]) -> io::Result> { - let mut groups = Vec::<(usize, usize)>::new(); - let mut start = 0usize; - while start < level.len() { - let mut end = start + 1; - let mut used = PAGE_HEADER_LEN + 8; - while end < level.len() { - let child = level.get(end).ok_or_else(|| invalid_input("internal child index is invalid"))?; - let cell_len = 12usize - .checked_add(child.minimum_key.len()) - .ok_or_else(|| invalid_input("internal cell length overflow"))?; - let next = used - .checked_add(SLOT_LEN) - .and_then(|size| size.checked_add(cell_len)) - .ok_or_else(|| invalid_input("internal page usage overflow"))?; - if next > PAGE_SIZE { - break; - } - used = next; - end += 1; - } - groups.push((start, end)); - start = end; - } - if groups.len() > 1 && groups.last().is_some_and(|(from, to)| to - from == 1) { - let last = groups.len() - 1; - let previous = groups - .get(last - 1) - .copied() - .ok_or_else(|| invalid_input("internal grouping is missing its previous group"))?; - let moved = previous.1.checked_sub(1).ok_or_else(|| invalid_input("internal grouping underflow"))?; - if moved <= previous.0 { - return Err(invalid_input("internal page cannot retain two children")); - } - groups.get_mut(last - 1).ok_or_else(|| invalid_input("internal grouping is missing its previous group"))?.1 = - moved; - groups.get_mut(last).ok_or_else(|| invalid_input("internal grouping is missing its last group"))?.0 = moved; - } - Ok(groups) -} - -fn build_parent_level(level: &[NodeInfo], sink: &mut PageSink) -> io::Result> { - let groups = group_internal_children(level)?; - let mut parents = Vec::with_capacity(groups.len()); - for (start, end) in groups { - let children = level.get(start..end).ok_or_else(|| invalid_input("internal child range is invalid"))?; - if children.len() < 2 { - return Err(invalid_input("internal page requires two children")); - } - let page_id = sink.allocate()?; - let (first_child, remaining_children) = - children.split_first().ok_or_else(|| invalid_input("internal page has no children"))?; - let mut encoded_cells = Vec::with_capacity(remaining_children.len()); - for child in remaining_children { - let mut encoded = Vec::new(); - encode_internal_cell(&child.minimum_key, child.page_id, page_id, u64::MAX, &mut encoded)?; - encoded_cells.push(encoded); - } - let encoded = internal_page(page_id, first_child.page_id, &encoded_cells)?; - sink.write(page_id, &encoded)?; - parents.push(NodeInfo { page_id, minimum_key: first_child.minimum_key.clone() }); - } - Ok(parents) -} - -/// Streams the whole tree into `sink` and returns the root page id. Page 0 is reserved -/// for the database header, which the caller writes last once `sink.next_page_id` is final. -fn build_pages(entries: &BTreeMap, sink: &mut PageSink) -> io::Result -where - K: Ord + Serialize, - V: Serialize, -{ - let header_page_id = sink.allocate()?; - if header_page_id != 0 { - return Err(invalid_input("database header must be the first allocated page")); - } - let mut level = build_leaf_level(entries, sink)?; - if entries.is_empty() { - return Ok(level.first().ok_or_else(|| invalid_input("tree has no root page"))?.page_id); - } - while level.len() > 1 { - level = build_parent_level(&level, sink)?; - } - Ok(level.first().ok_or_else(|| invalid_input("tree has no root page"))?.page_id) -} - -fn temporary_path(filepath: &Path) -> io::Result { - let name = filepath.file_name().ok_or_else(|| invalid_input("database path has no file name"))?.to_string_lossy(); - Ok(filepath.with_file_name(format!("{name}.{}.v3.tmp", uuid::Uuid::new_v4()))) -} - -#[derive(Debug)] -pub(super) enum PublishDatabaseError { - NotPublished(io::Error), - PublishedDurabilityUnknown(io::Error), -} - -impl PublishDatabaseError { - pub(super) const fn database_was_published(&self) -> bool { matches!(self, Self::PublishedDurabilityUnknown(_)) } - - fn into_io(self) -> io::Error { - match self { - Self::NotPublished(error) | Self::PublishedDurabilityUnknown(error) => error, - } - } -} - -impl From for io::Error { - fn from(error: PublishDatabaseError) -> Self { error.into_io() } -} - -pub(super) fn publish_database( - temporary: &Path, - destination: &Path, - sync_directory: impl FnOnce(&Path) -> io::Result<()>, -) -> Result<(), PublishDatabaseError> { - let temporary = match tempfile::TempPath::try_from_path(temporary) { - Ok(path) => path, - Err(error) => { - let _ = std::fs::remove_file(temporary); - return Err(PublishDatabaseError::NotPublished(error)); - } - }; - temporary.persist(destination).map_err(io::Error::from).map_err(PublishDatabaseError::NotPublished)?; - sync_directory(destination).map_err(|error| { - PublishDatabaseError::PublishedDurabilityUnknown(io::Error::new( - error.kind(), - format!("database published but directory sync failed; durability unknown: {error}"), - )) - }) -} - -/// Invalidates the sorted index whenever publication made the replacement database visible. -pub(super) fn publish_database_and_invalidate_sorted_index( - temporary: &Path, - destination: &Path, - sync_directory: impl FnOnce(&Path) -> io::Result<()>, -) -> io::Result<()> { - let publish_error = match publish_database(temporary, destination, sync_directory) { - Ok(()) => None, - Err(error) if error.database_was_published() => Some(io::Error::from(error)), - Err(error) => return Err(io::Error::from(error)), - }; - let invalidation_error = invalidate_sorted_index(destination).err(); - match (publish_error, invalidation_error) { - (Some(publish_error), Some(invalidation_error)) => Err(io::Error::new( - publish_error.kind(), - format!( - "database {} was published with unknown directory durability: {publish_error}; its previous sorted index could not be invalidated: {invalidation_error}", - destination.display() - ), - )), - (Some(publish_error), None) => Err(io::Error::new( - publish_error.kind(), - format!( - "database {} was published and its previous sorted index was invalidated, but publication durability remains unknown: {publish_error}", - destination.display() - ), - )), - (None, Some(error)) => Err(io::Error::new( - error.kind(), - format!( - "database {} was published, but its previous sorted index could not be invalidated: {error}", - destination.display() - ), - )), - (None, None) => Ok(()), - } -} - -impl BPlusTree -where - K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, - V: Serialize + for<'de> Deserialize<'de>, -{ - pub fn store(&mut self, filepath: &Path) -> io::Result { - with_exclusive_sidecar(filepath, || { - recover_pending_under_existing_lock(filepath)?; - if self.dirty { - self.store_exclusive(filepath).map(|stored| stored.root_page_id) - } else { - Ok(0) - } - }) - } - - pub(crate) fn store_verified(&mut self, filepath: &Path) -> io::Result { - with_exclusive_sidecar(filepath, || { - recover_pending_under_existing_lock(filepath)?; - if !self.dirty { - return Err(invalid_input("verified store requires a dirty tree")); - } - self.store_exclusive(filepath).map(|stored| stored.verification) - }) - } - - fn store_exclusive(&mut self, filepath: &Path) -> io::Result { - self.store_exclusive_with_directory_sync(filepath, sync_parent_directory) - } - - fn store_exclusive_with_directory_sync( - &mut self, - filepath: &Path, - sync_published_directory: impl FnOnce(&Path) -> io::Result<()>, - ) -> io::Result { - let temp_path = temporary_path(filepath)?; - // The builder streams into the temp file, so it exists from here on — every error - // path below has to remove it again. - let prepared = (|| { - let mut sink = PageSink::new(OpenOptions::new().write(true).create_new(true).open(&temp_path)?); - let root_page_id = build_pages(&self.entries, &mut sink)?; - let header = DatabaseHeader { - root_page_id, - next_page_id: sink.next_page_id, - free_page_head: 0, - generation: 1, - database_id: *uuid::Uuid::new_v4().as_bytes(), - metadata: self.metadata.clone(), - } - .encode()?; - sink.write(0, &header)?; - sink.file.sync_all()?; - drop(sink); - let mut query = BPlusTreeQuery::::from_file_unlocked(File::open(&temp_path)?)?; - let verification = verify_full(&mut query)?; - drop(query); - Ok((root_page_id, verification)) - })(); - let (root_page_id, verification) = match prepared { - Ok(prepared) => prepared, - Err(error) => { - let _ = std::fs::remove_file(&temp_path); - return Err(error); - } - }; - publish_database_and_invalidate_sorted_index(&temp_path, filepath, sync_published_directory)?; - self.dirty = false; - Ok(StoredDatabase { root_page_id, verification }) - } - - pub fn store_with_index(&mut self, filepath: &Path, sort_key_extractor: F) -> io::Result - where - SortKey: Ord + Serialize, - F: Fn(&V) -> SortKey, - { - self.store_with_index_result(filepath, sort_key_extractor) - .map(|stored| stored.map_or(0, |stored| stored.root_page_id)) - } - - pub(crate) fn store_with_index_verified( - &mut self, - filepath: &Path, - sort_key_extractor: F, - ) -> io::Result - where - SortKey: Ord + Serialize, - F: Fn(&V) -> SortKey, - { - self.store_with_index_result(filepath, sort_key_extractor)? - .map(|stored| stored.verification) - .ok_or_else(|| invalid_input("verified indexed store requires a dirty tree")) - } - - fn store_with_index_result( - &mut self, - filepath: &Path, - sort_key_extractor: F, - ) -> io::Result> - where - SortKey: Ord + Serialize, - F: Fn(&V) -> SortKey, - { - with_exclusive_sidecar(filepath, || { - recover_pending_under_existing_lock(filepath)?; - if !self.dirty { - return Ok(None); - } - let stored = self.store_exclusive(filepath)?; - Self::store_index_exclusive(filepath, sort_key_extractor, stored.verification.live_entries)?; - Ok(Some(stored)) - }) - } - - fn store_index_exclusive( - filepath: &Path, - sort_key_extractor: F, - expected_entries: u64, - ) -> io::Result<()> - where - SortKey: Ord + Serialize, - F: Fn(&V) -> SortKey, - { - let mut query = BPlusTreeQuery::::from_file_unlocked(File::open(filepath)?)?; - let (database_id, generation) = query.snapshot_identity(); - let mut entries = query - .collect_with_locators()? - .into_iter() - .map(|(key, value, locator)| (sort_key_extractor(&value), key, locator)) - .collect::>(); - if u64::try_from(entries.len()).map_err(|_| invalid_data("sorted-index entry count exceeds u64"))? - != expected_entries - { - return Err(invalid_data("sorted-index source entry count differs from verified database")); - } - entries.sort_unstable_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1))); - drop(query); - - let index_path = crate::common::get_file_path_for_db_index(filepath); - let temporary = temporary_path(&index_path)?; - let prepared = (|| { - let mut writer = crate::sorted_index::v4::Writer::new(&temporary, database_id, generation)?; - for (sort_key, primary_key, locator) in &entries { - writer.push(sort_key, primary_key, *locator)?; - } - let _ = writer.finish()?; - Ok(()) - })(); - if let Err(error) = prepared { - let _ = std::fs::remove_file(&temporary); - return Err(error); - } - publish_database(&temporary, &index_path, sync_parent_directory).map_err(io::Error::from) - } -} - -struct StoredDatabase { - root_page_id: u64, - verification: VerificationReport, -} - -impl BPlusTree -where - K: Ord + for<'de> Deserialize<'de>, - V: for<'de> Deserialize<'de>, -{ - pub fn load(filepath: &Path) -> io::Result { - let mut query = BPlusTreeQuery::::try_new(filepath)?; - let metadata = query.header.metadata.clone(); - let mut entries = BTreeMap::new(); - for entry in query.iter() { - let (key, value) = entry?; - let _ = entries.insert(key, value); - } - Ok(Self { entries, metadata, dirty: false }) - } -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum FlushPolicy { Immediate, Batch, } -struct DatabaseImage { - mmap: Option, - fallback: Vec, -} - -impl DatabaseImage { - fn open(path: &Path) -> io::Result<(Self, DatabaseHeader)> { - let mut file = File::open(path)?; - let file_len = - usize::try_from(file.metadata()?.len()).map_err(|_| invalid_data("database length exceeds usize"))?; - let mmap = mmap_with_advice(&file, Advice::Normal, "v3 B+Tree update"); - let mut fallback = Vec::new(); - if mmap.is_none() { - fallback.try_reserve_exact(file_len).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - file.read_to_end(&mut fallback)?; - } - let image = Self { mmap, fallback }; - let bytes = image.as_slice(); - let header = DatabaseHeader::decode( - bytes.get(..PAGE_SIZE).ok_or_else(|| invalid_data("database header is truncated"))?, - )?; - let expected = usize::try_from(header.next_page_id) - .map_err(|_| invalid_data("next page id exceeds usize"))? - .checked_mul(PAGE_SIZE) - .ok_or_else(|| invalid_data("database length overflow"))?; - if bytes.len() != expected { - return Err(invalid_data("database file length does not match header")); - } - Ok((image, header)) - } - - fn as_slice(&self) -> &[u8] { self.mmap.as_deref().unwrap_or(&self.fallback) } -} - -struct WriteTransaction { - original_header: DatabaseHeader, - next_header: DatabaseHeader, - original_file_len: u64, - dirty_pages: BTreeMap>, - allocated_pages: HashSet, - freed_pages: HashSet, -} - -impl WriteTransaction { - fn new(header: DatabaseHeader, original_file_len: usize) -> io::Result { - Ok(Self { - original_header: header.clone(), - next_header: header, - original_file_len: u64::try_from(original_file_len) - .map_err(|_| invalid_data("database length exceeds u64"))?, - dirty_pages: BTreeMap::new(), - allocated_pages: HashSet::new(), - freed_pages: HashSet::new(), - }) - } - - fn page<'a>(&'a self, base: &'a [u8], page_id: u64) -> io::Result<&'a [u8]> { - if page_id == 0 || page_id >= self.next_header.next_page_id { - return Err(invalid_data("transaction page id is outside database")); - } - if let Some(page) = self.dirty_pages.get(&page_id) { - return Ok(page.as_slice()); - } - if page_id >= self.original_header.next_page_id { - return Err(invalid_data("transaction appended page is missing")); - } - let range = page_byte_range(page_id, base.len())?; - base.get(range).ok_or_else(|| invalid_data("transaction base page is truncated")) - } - - fn page_copy(&self, base: &[u8], page_id: u64) -> io::Result<[u8; PAGE_SIZE]> { - let mut copied = [0; PAGE_SIZE]; - copied.copy_from_slice(self.page(base, page_id)?); - Ok(copied) - } - - fn page_mut<'a>(&'a mut self, base: &[u8], page_id: u64) -> io::Result<&'a mut [u8; PAGE_SIZE]> { - if page_id == 0 || page_id >= self.next_header.next_page_id { - return Err(invalid_data("transaction page id is outside database")); - } - if self.dirty_pages.contains_key(&page_id) { - return self - .dirty_pages - .get_mut(&page_id) - .map(Box::as_mut) - .ok_or_else(|| invalid_data("transaction page copy is missing")); - } - if page_id >= self.original_header.next_page_id { - return Err(invalid_data("appended transaction page is missing")); - } - let range = page_byte_range(page_id, base.len())?; - let source = base.get(range).ok_or_else(|| invalid_data("transaction base page is truncated"))?; - let mut copied = Box::new([0; PAGE_SIZE]); - copied.copy_from_slice(source); - let _ = self.dirty_pages.insert(page_id, copied); - self.dirty_pages - .get_mut(&page_id) - .map(Box::as_mut) - .ok_or_else(|| invalid_data("transaction page copy is missing")) - } - - #[allow(clippy::large_types_passed_by_value)] - fn write_page(&mut self, page_id: u64, page: [u8; PAGE_SIZE]) -> io::Result<()> { - if page_id == 0 || page_id >= self.next_header.next_page_id { - return Err(invalid_input("written page id is outside transaction bounds")); - } - let _ = self.dirty_pages.insert(page_id, Box::new(page)); - Ok(()) - } - - fn allocate_page(&mut self, base: &[u8]) -> io::Result { - if self.next_header.free_page_head != 0 { - let page_id = self.next_header.free_page_head; - let page = SlottedPage::open(self.page(base, page_id)?, page_id, self.next_header.next_page_id)?; - if page.header().page_type != PageType::Free { - return Err(invalid_data("free list references a non-free page")); - } - self.next_header.free_page_head = page.header().right; - if !self.allocated_pages.insert(page_id) { - return Err(invalid_data("free page was allocated twice")); - } - let _ = self.freed_pages.remove(&page_id); - return Ok(page_id); - } - let page_id = self.next_header.next_page_id; - self.next_header.next_page_id = page_id.checked_add(1).ok_or_else(|| invalid_input("next page id overflow"))?; - if !self.allocated_pages.insert(page_id) { - return Err(invalid_data("appended page was allocated twice")); - } - Ok(page_id) - } - - fn free_page(&mut self, page_id: u64) -> io::Result<()> { - if page_id == 0 || page_id == self.next_header.root_page_id || page_id >= self.next_header.next_page_id { - return Err(invalid_input("page cannot be added to the free list")); - } - if !self.freed_pages.insert(page_id) { - return Err(invalid_data("page was freed twice in one transaction")); - } - let _ = self.allocated_pages.remove(&page_id); - let page = encode_free_page(page_id, self.next_header.next_page_id, self.next_header.free_page_head)?; - self.next_header.free_page_head = page_id; - self.write_page(page_id, page) - } - - fn has_changes(&self) -> bool { - !self.dirty_pages.is_empty() || self.next_header.metadata != self.original_header.metadata - } - - fn prepared_pages(&mut self) -> io::Result> { - if !self.has_changes() { - return Ok(Vec::new()); - } - self.next_header.generation = self - .original_header - .generation - .checked_add(1) - .ok_or_else(|| invalid_input("database generation overflow"))?; - let _ = self.dirty_pages.insert(0, Box::new(self.next_header.encode()?)); - let mut prepared = Vec::new(); - prepared - .try_reserve_exact(self.dirty_pages.len()) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - for (page_id, page) in &self.dirty_pages { - if *page_id == 0 { - DatabaseHeader::decode(page.as_slice())?; - } else { - SlottedPage::open(page.as_slice(), *page_id, self.next_header.next_page_id)?; - } - prepared.push((*page_id, page.as_ref())); - } - Ok(prepared) - } -} - -fn page_byte_range(page_id: u64, database_len: usize) -> io::Result> { - let start = usize::try_from(page_id) - .map_err(|_| invalid_data("page id exceeds usize"))? - .checked_mul(PAGE_SIZE) - .ok_or_else(|| invalid_data("page offset overflow"))?; - let end = start.checked_add(PAGE_SIZE).ok_or_else(|| invalid_data("page end overflow"))?; - if end > database_len { - return Err(invalid_data("page is outside database")); - } - Ok(start..end) -} - -struct ActiveBatch { - base: DatabaseImage, - transaction: WriteTransaction, - _guard: ExclusiveSidecarGuard, -} - -#[derive(Default)] -struct WriteScratch { - key: Vec, - value: Vec, - compression: Vec, - cell: Vec, - read_value: Vec, -} - pub struct BPlusTreeUpdate { filepath: PathBuf, database_id: [u8; 16], @@ -1103,383 +65,6 @@ pub struct BPlusTreeUpdate { _types: PhantomData<(K, V)>, } -impl BPlusTreeUpdate -where - K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, - V: Serialize + for<'de> Deserialize<'de>, -{ - pub fn try_new(filepath: &Path) -> io::Result { - let mut query = BPlusTreeQuery::::try_new(filepath)?; - let _ = verify_full(&mut query)?; - let database_id = query.header.database_id; - let verified_generation = query.header.generation; - let verified_next_page_id = query.header.next_page_id; - drop(query); - Ok(Self { - filepath: filepath.to_path_buf(), - database_id, - verified_generation, - verified_next_page_id, - flush_policy: FlushPolicy::Immediate, - active: None, - scratch: WriteScratch::default(), - _types: PhantomData, - }) - } - - pub fn try_new_with_backoff(filepath: &Path) -> io::Result { Self::try_new(filepath) } - - pub fn try_new_with_backoff_stats(filepath: &Path) -> io::Result<(Self, u64)> { - Self::try_new(filepath).map(|updater| (updater, 0)) - } - - pub fn set_flush_policy(&mut self, policy: FlushPolicy) { self.flush_policy = policy; } - - fn ensure_transaction(&mut self) -> io::Result<()> { - if self.active.is_some() { - return Ok(()); - } - let guard = ExclusiveSidecarGuard::acquire(&self.filepath)?; - recover_pending_under_existing_lock(&self.filepath)?; - let (base, header) = DatabaseImage::open(&self.filepath)?; - if header.database_id != self.database_id - || header.generation != self.verified_generation - || header.next_page_id != self.verified_next_page_id - { - let mut query = BPlusTreeQuery::::from_file_unlocked(File::open(&self.filepath)?)?; - let _ = verify_full(&mut query)?; - self.database_id = query.header.database_id; - self.verified_generation = query.header.generation; - self.verified_next_page_id = query.header.next_page_id; - } - let transaction = WriteTransaction::new(header, base.as_slice().len())?; - self.active = Some(ActiveBatch { base, transaction, _guard: guard }); - Ok(()) - } - - pub fn update(&mut self, key: &K, value: V) -> Result { - self.upsert(key, &value).map_err(Into::into) - } - - pub fn update_batch(&mut self, items: &[(&K, &V)]) -> Result { - self.upsert_batch(items).map_err(Into::into) - } - - pub fn prepare_upsert_batch(items: &[(&K, &V)]) -> io::Result)>> { - let mut prepared = Vec::new(); - prepared.try_reserve_exact(items.len()).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - for (key, value) in items { - prepared.push(((*key).clone(), binary_serialize(value)?)); - } - prepared.sort_by(|left, right| left.0.cmp(&right.0)); - Ok(prepared) - } - - pub fn upsert_batch_prepared_with_backoff(filepath: &Path, items: &[(&K, &V)]) -> io::Result { - let prepared = Self::prepare_upsert_batch(items)?; - let mut updater = Self::try_new_with_backoff(filepath)?; - updater.upsert_batch_encoded(prepared) - } - - pub fn upsert_batch(&mut self, items: &[(&K, &V)]) -> io::Result { - let prepared = match Self::prepare_upsert_batch(items) { - Ok(prepared) => prepared, - Err(error) => { - self.active = None; - return Err(error); - } - }; - self.upsert_batch_encoded(prepared) - } - - pub fn upsert_batch_encoded(&mut self, items: Vec<(K, Vec)>) -> io::Result { - if items.is_empty() { - return self.upsert_batch(&[]); - } - let policy = self.flush_policy; - self.flush_policy = FlushPolicy::Batch; - let mut root = 0; - for (key, value) in items { - match self.upsert_serialized(&key, &value) { - Ok(next_root) => root = next_root, - Err(error) => { - self.active = None; - self.flush_policy = policy; - return Err(error); - } - } - } - self.flush_policy = policy; - if policy == FlushPolicy::Immediate { - self.commit()?; - } - Ok(root) - } - - pub fn upsert(&mut self, key: &K, value: &V) -> io::Result { - let result = self.upsert_inner(key, value); - if result.is_err() { - self.active = None; - } - result - } - - fn upsert_inner(&mut self, key: &K, value: &V) -> io::Result { - self.scratch.key.clear(); - binary_serialize_into(&mut self.scratch.key, key)?; - self.scratch.value.clear(); - binary_serialize_into(&mut self.scratch.value, value)?; - self.ensure_transaction()?; - let stored = match encode_value(&self.scratch.value, &mut self.scratch.compression) { - Ok(stored) => stored, - Err(error) => { - self.active = None; - return Err(error); - } - }; - let active = self.active.as_mut().ok_or_else(|| invalid_data("write transaction is missing"))?; - let staged = stage_upsert::( - &mut active.transaction, - active.base.as_slice(), - key, - &self.scratch.key, - u32::try_from(self.scratch.value.len()).map_err(|_| invalid_input("serialized value exceeds u32"))?, - stored.compression(), - stored.as_slice(), - &mut self.scratch.cell, - ); - if let Err(error) = staged { - self.active = None; - return Err(error); - } - let root = self - .active - .as_ref() - .ok_or_else(|| invalid_data("write transaction is missing"))? - .transaction - .next_header - .root_page_id; - if self.flush_policy == FlushPolicy::Immediate { - self.commit()?; - } - Ok(root) - } - - fn upsert_serialized(&mut self, key: &K, raw_value: &[u8]) -> io::Result { - let result = self.upsert_serialized_inner(key, raw_value); - if result.is_err() { - self.active = None; - } - result - } - - fn upsert_serialized_inner(&mut self, key: &K, raw_value: &[u8]) -> io::Result { - self.scratch.key.clear(); - binary_serialize_into(&mut self.scratch.key, key)?; - self.ensure_transaction()?; - let stored = match encode_value(raw_value, &mut self.scratch.compression) { - Ok(stored) => stored, - Err(error) => { - self.active = None; - return Err(error); - } - }; - let active = self.active.as_mut().ok_or_else(|| invalid_data("write transaction is missing"))?; - let staged = stage_upsert::( - &mut active.transaction, - active.base.as_slice(), - key, - &self.scratch.key, - u32::try_from(raw_value.len()).map_err(|_| invalid_input("serialized value exceeds u32"))?, - stored.compression(), - stored.as_slice(), - &mut self.scratch.cell, - ); - if let Err(error) = staged { - self.active = None; - return Err(error); - } - let root = self - .active - .as_ref() - .ok_or_else(|| invalid_data("write transaction is missing"))? - .transaction - .next_header - .root_page_id; - if self.flush_policy == FlushPolicy::Immediate { - self.commit()?; - } - Ok(root) - } - - pub fn delete(&mut self, key: &K) -> io::Result { - let result = self.delete_inner(key); - if result.is_err() { - self.active = None; - } - result - } - - fn delete_inner(&mut self, key: &K) -> io::Result { - let started = self.active.is_none(); - self.scratch.key.clear(); - binary_serialize_into(&mut self.scratch.key, key)?; - self.ensure_transaction()?; - let active = self.active.as_mut().ok_or_else(|| invalid_data("write transaction is missing"))?; - let deleted = stage_delete::( - &mut active.transaction, - active.base.as_slice(), - key, - &self.scratch.key, - &mut self.scratch.cell, - ); - let deleted = match deleted { - Ok(deleted) => deleted, - Err(error) => { - self.active = None; - return Err(error); - } - }; - if !deleted && started { - self.active = None; - } else if deleted && self.flush_policy == FlushPolicy::Immediate { - self.commit()?; - } - Ok(deleted) - } - - pub fn delete_batch(&mut self, keys: &[&K]) -> io::Result { - if keys.is_empty() { - return Ok(0); - } - let policy = self.flush_policy; - self.flush_policy = FlushPolicy::Batch; - let mut deleted = 0usize; - let mut ordered = Vec::new(); - ordered.try_reserve_exact(keys.len()).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - ordered.extend_from_slice(keys); - ordered.sort(); - for key in ordered { - match self.delete(key) { - Ok(true) => deleted = deleted.checked_add(1).ok_or_else(|| invalid_input("delete count overflow"))?, - Ok(false) => {} - Err(error) => { - self.active = None; - self.flush_policy = policy; - return Err(error); - } - } - } - self.flush_policy = policy; - if policy == FlushPolicy::Immediate { - self.commit()?; - } - Ok(deleted) - } - - pub fn get_metadata(&self) -> io::Result { - if let Some(active) = &self.active { - return Ok(active.transaction.next_header.metadata.clone()); - } - BPlusTreeQuery::::try_new(&self.filepath).map(|query| query.header.metadata) - } - - pub fn set_metadata(&mut self, metadata: &BPlusTreeMetadata) -> io::Result<()> { - let started = self.active.is_none(); - self.ensure_transaction()?; - let active = self.active.as_mut().ok_or_else(|| invalid_data("write transaction is missing"))?; - if active.transaction.next_header.metadata == *metadata { - if started { - self.active = None; - } - return Ok(()); - } - active.transaction.next_header.metadata = metadata.clone(); - if self.flush_policy == FlushPolicy::Immediate { - self.commit()?; - } - Ok(()) - } - - pub fn query(&mut self, key: &K) -> Result, BPlusTreeError> { - if let Some(active) = &mut self.active { - return query_transaction::( - &active.transaction, - active.base.as_slice(), - key, - &mut self.scratch.read_value, - ) - .map_err(Into::into); - } - BPlusTreeQuery::::try_new(&self.filepath).and_then(|mut query| query.query_io(key)).map_err(Into::into) - } - - pub fn commit(&mut self) -> io::Result<()> { - let Some(mut active) = self.active.take() else { return Ok(()) }; - validate_transaction_links::(&active.transaction, active.base.as_slice())?; - let prepared = active.transaction.prepared_pages()?; - if prepared.is_empty() { - return Ok(()); - } - let committed_header = DatabaseHeader::decode(prepared[0].1)?; - let committed_generation = committed_header.generation; - let committed_next_page_id = committed_header.next_page_id; - let result = commit_ordered_page_refs_under_existing_lock(&self.filepath, &prepared); - match result { - Ok(()) => { - self.verified_generation = committed_generation; - self.verified_next_page_id = committed_next_page_id; - Ok(()) - } - Err(error) => { - let outcome = error - .get_ref() - .and_then(|source| source.downcast_ref::()) - .map(WalOperationError::outcome); - if outcome == Some(WalOutcome::CommittedCleanupPending) { - self.verified_generation = committed_generation; - self.verified_next_page_id = committed_next_page_id; - } - if let Err(recovery_error) = recover_pending_under_existing_lock(&self.filepath) { - log::error!( - "B+Tree commit failed and recovery remains pending for {}: {recovery_error}", - self.filepath.display() - ); - } - Err(error) - } - } - } - - pub fn compact(&mut self) -> io::Result<()> { - self.commit()?; - let filepath = self.filepath.clone(); - let header = with_exclusive_sidecar(&filepath, || { - recover_pending_under_existing_lock(&filepath)?; - let mut query = BPlusTreeQuery::::from_file_unlocked(File::open(&filepath)?)?; - let _ = verify_full(&mut query)?; - let metadata = query.header.metadata.clone(); - let mut entries = BTreeMap::new(); - for entry in query.iter() { - let (key, value) = entry?; - let _ = entries.insert(key, value); - } - drop(query); - - let mut replacement = BPlusTree { entries, metadata, dirty: true }; - let _ = replacement.store_exclusive(&filepath)?; - let file = File::open(&filepath)?; - let mut page = [0; PAGE_SIZE]; - read_exact_at_offset(&file, &mut page, 0)?; - DatabaseHeader::decode(&page) - })?; - self.database_id = header.database_id; - self.verified_generation = header.generation; - self.verified_next_page_id = header.next_page_id; - Ok(()) - } -} - pub struct BPlusTreeSerialWriter { updater: Arc>>, flush_policy: FlushPolicy, @@ -1489,805 +74,6 @@ pub struct BPlusTreeSerialWriter { background_handle: Mutex>>, } -impl BPlusTreeSerialWriter -where - K: Ord + Serialize + for<'de> Deserialize<'de> + Clone + Send + 'static, - V: Serialize + for<'de> Deserialize<'de> + Send + 'static, -{ - pub fn new(filepath: &Path, flush_policy: FlushPolicy) -> io::Result { - let mut updater = BPlusTreeUpdate::try_new_with_backoff(filepath)?; - updater.set_flush_policy(flush_policy); - Ok(Self { - updater: Arc::new(Mutex::new(updater)), - flush_policy, - dirty: Arc::new(AtomicBool::new(false)), - shutdown: Arc::new(AtomicBool::new(false)), - background_error: Arc::new(Mutex::new(None)), - background_handle: Mutex::new(None), - }) - } - - pub fn upsert_prepared(&self, items: Vec<(K, Vec)>) -> io::Result { - let result = self.updater.lock().upsert_batch_encoded(items); - if result.is_ok() { - self.dirty.store(self.flush_policy == FlushPolicy::Batch, Ordering::Release); - } - result - } - - pub fn upsert(&self, items: &[(&K, &V)]) -> io::Result { - self.upsert_prepared(BPlusTreeUpdate::::prepare_upsert_batch(items)?) - } - - pub fn start_background_commit(&self, interval: Duration) -> io::Result<()> { - if self.flush_policy != FlushPolicy::Batch { - return Err(invalid_input("background commit requires batch flush policy")); - } - if interval.is_zero() { - return Err(invalid_input("background commit interval must be greater than zero")); - } - let mut slot = self.background_handle.lock(); - if slot.is_some() { - return Ok(()); - } - self.shutdown.store(false, Ordering::Release); - let updater = Arc::clone(&self.updater); - let dirty = Arc::clone(&self.dirty); - let shutdown = Arc::clone(&self.shutdown); - let background_error = Arc::clone(&self.background_error); - *slot = Some( - std::thread::Builder::new() - .name(String::from("bplustree-commit")) - .spawn(move || { - while !shutdown.load(Ordering::Acquire) { - std::thread::park_timeout(interval); - if shutdown.load(Ordering::Acquire) { - break; - } - commit_if_dirty(&updater, &dirty, &background_error); - } - commit_if_dirty(&updater, &dirty, &background_error); - }) - .map_err(io::Error::other)?, - ); - Ok(()) - } - - pub fn stop_background_commit(&self) -> io::Result<()> { - self.shutdown.store(true, Ordering::Release); - if let Some(handle) = self.background_handle.lock().take() { - handle.thread().unpark(); - handle.join().map_err(|_| io::Error::other("background B+Tree commit thread panicked"))?; - } - if let Some(error) = self.background_error.lock().take() { - return Err(error); - } - Ok(()) - } - - pub fn flush_now(&self) -> io::Result<()> { self.commit() } - - pub fn commit(&self) -> io::Result<()> { - self.updater.lock().commit()?; - self.dirty.store(false, Ordering::Release); - Ok(()) - } - - pub fn shutdown(&self) -> io::Result<()> { - self.stop_background_commit()?; - self.commit() - } -} - -fn commit_if_dirty( - updater: &Mutex>, - dirty: &AtomicBool, - background_error: &Mutex>, -) where - K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, - V: Serialize + for<'de> Deserialize<'de>, -{ - if !dirty.swap(false, Ordering::AcqRel) { - return; - } - if let Err(error) = updater.lock().commit() { - dirty.store(true, Ordering::Release); - log::error!("Background B+Tree commit failed: {error}"); - *background_error.lock() = Some(error); - } -} - -impl Drop for BPlusTreeSerialWriter { - fn drop(&mut self) { - self.shutdown.store(true, Ordering::Release); - if self.dirty.load(Ordering::Acquire) { - log::warn!("Dropping dirty B+Tree serial writer without an explicit shutdown"); - } - if let Some(handle) = self.background_handle.lock().take() { - handle.thread().unpark(); - drop(handle); - } - } -} - -fn minimum_tree_key(transaction: &WriteTransaction, base: &[u8], mut page_id: u64) -> io::Result>> { - let mut visited = HashSet::new(); - loop { - if !visited.insert(page_id) { - return Err(invalid_data("minimum-key descent contains a cycle")); - } - let page = SlottedPage::open(transaction.page(base, page_id)?, page_id, transaction.next_header.next_page_id)?; - match page.header().page_type { - PageType::Leaf => { - let Some(cell) = page.cells().next() else { return Ok(None) }; - return LeafCellRef::decode(cell?, page_id, transaction.next_header.next_page_id) - .map(|cell| Some(cell.key_bytes.to_vec())); - } - PageType::Internal => { - page_id = InternalPreamble::decode(page.as_bytes(), page_id, transaction.next_header.next_page_id)? - .leftmost_child; - } - PageType::Overflow | PageType::Free => return Err(invalid_data("tree references a non-tree page")), - } - } -} - -fn validate_leaf_backlink( - transaction: &WriteTransaction, - base: &[u8], - page_id: u64, - sibling_id: u64, - sibling_points_left: bool, -) -> io::Result<()> { - if sibling_id == 0 { - return Ok(()); - } - let sibling = - SlottedPage::open(transaction.page(base, sibling_id)?, sibling_id, transaction.next_header.next_page_id)?; - let backlink = if sibling_points_left { sibling.header().left } else { sibling.header().right }; - if sibling.header().page_type != PageType::Leaf || backlink != page_id { - return Err(invalid_data("asymmetric leaf sibling link")); - } - Ok(()) -} - -fn validate_transaction_free_list(transaction: &WriteTransaction, base: &[u8]) -> io::Result<()> { - let mut free = HashSet::new(); - let mut page_id = transaction.next_header.free_page_head; - while page_id != 0 { - if !free.insert(page_id) { - return Err(invalid_data("free list contains a cycle")); - } - if transaction.allocated_pages.contains(&page_id) { - return Err(invalid_data("allocated page remains linked from free list")); - } - let page = SlottedPage::open(transaction.page(base, page_id)?, page_id, transaction.next_header.next_page_id)?; - if page.header().page_type != PageType::Free { - return Err(invalid_data("free list references a non-free page")); - } - page_id = page.header().right; - } - if !transaction.freed_pages.iter().all(|page| free.contains(page)) { - return Err(invalid_data("freed page is missing from free list")); - } - for page_id in &transaction.allocated_pages { - let page = transaction.page(base, *page_id)?; - if SlottedPage::open(page, *page_id, transaction.next_header.next_page_id)?.header().page_type == PageType::Free - { - return Err(invalid_data("allocated page still has free-page type")); - } - } - Ok(()) -} - -fn validate_transaction_links(transaction: &WriteTransaction, base: &[u8]) -> io::Result<()> -where - K: Ord + for<'de> Deserialize<'de>, -{ - let original_length = transaction - .original_header - .next_page_id - .checked_mul(u64::try_from(PAGE_SIZE).map_err(|_| invalid_data("page size exceeds u64"))?) - .ok_or_else(|| invalid_data("original database length overflow"))?; - if original_length != transaction.original_file_len { - return Err(invalid_data("transaction original length does not match header")); - } - - let mut owned_overflow_pages = HashSet::new(); - for (page_id, bytes) in &transaction.dirty_pages { - if *page_id == 0 { - continue; - } - let page = SlottedPage::open(bytes.as_slice(), *page_id, transaction.next_header.next_page_id)?; - match page.header().page_type { - PageType::Leaf => { - validate_leaf_backlink(transaction, base, *page_id, page.header().left, false)?; - validate_leaf_backlink(transaction, base, *page_id, page.header().right, true)?; - let mut previous_key = None; - for cell in page.cells() { - let cell = LeafCellRef::decode(cell?, *page_id, transaction.next_header.next_page_id)?; - let key = decode_key::(cell.key_bytes)?; - if previous_key.as_ref().is_some_and(|previous| previous >= &key) { - return Err(invalid_data("leaf keys are not strictly increasing")); - } - previous_key = Some(key); - if let LeafValueRef::Overflow { stored_len, head, crc32, .. } = cell.value { - for overflow_page in validated_overflow_chain_pages(transaction, base, head, stored_len, crc32)? - { - if !owned_overflow_pages.insert(overflow_page) { - return Err(invalid_data("overflow page is owned by multiple values")); - } - } - } - } - } - PageType::Internal => { - let preamble = - InternalPreamble::decode(bytes.as_slice(), *page_id, transaction.next_header.next_page_id)?; - let child = SlottedPage::open( - transaction.page(base, preamble.leftmost_child)?, - preamble.leftmost_child, - transaction.next_header.next_page_id, - )?; - if !matches!(child.header().page_type, PageType::Leaf | PageType::Internal) { - return Err(invalid_data("internal page references a non-tree child")); - } - let mut previous_key = None; - for cell in page.cells() { - let cell = InternalCellRef::decode(cell?, *page_id, transaction.next_header.next_page_id)?; - let key = decode_key::(cell.key_bytes)?; - if previous_key.as_ref().is_some_and(|previous| previous >= &key) { - return Err(invalid_data("internal separator keys are not strictly increasing")); - } - previous_key = Some(key); - let minimum = minimum_tree_key(transaction, base, cell.right_child)? - .ok_or_else(|| invalid_data("internal separator references an empty subtree"))?; - if minimum != cell.key_bytes { - return Err(invalid_data("internal separator differs from right subtree minimum")); - } - } - } - PageType::Overflow => { - if page.header().right != 0 { - let next = SlottedPage::open( - transaction.page(base, page.header().right)?, - page.header().right, - transaction.next_header.next_page_id, - )?; - if next.header().page_type != PageType::Overflow { - return Err(invalid_data("overflow page references a non-overflow page")); - } - } - } - PageType::Free => {} - } - } - - validate_transaction_free_list(transaction, base) -} - -#[derive(Debug)] -struct Promotion { - key: Vec, - right_child: u64, -} - -fn internal_child_position Deserialize<'de>, B: AsRef<[u8]>>( - page: &SlottedPage, - key: &K, -) -> io::Result<(usize, u64)> { - let mut left = 0usize; - let mut right = usize::from(page.header().cell_count); - while left < right { - let middle = left + (right - left) / 2; - let cell = InternalCellRef::decode(page.cell(middle)?, page.page_id(), page.next_page_id())?; - if decode_key::(cell.key_bytes)? <= *key { - left = middle + 1; - } else { - right = middle; - } - } - let child = if left == 0 { - InternalPreamble::decode(page.as_bytes(), page.page_id(), page.next_page_id())?.leftmost_child - } else { - InternalCellRef::decode(page.cell(left - 1)?, page.page_id(), page.next_page_id())?.right_child - }; - Ok((left, child)) -} - -fn locate_transaction_leaf Deserialize<'de>>( - transaction: &WriteTransaction, - base: &[u8], - key: &K, -) -> io::Result<(u64, Vec<(u64, usize)>)> { - let mut page_id = transaction.next_header.root_page_id; - let mut path = Vec::new(); - let mut visited = HashSet::new(); - loop { - if !visited.insert(page_id) { - return Err(invalid_data("tree descent contains a cycle")); - } - let page = SlottedPage::open(transaction.page(base, page_id)?, page_id, transaction.next_header.next_page_id)?; - match page.header().page_type { - PageType::Leaf => return Ok((page_id, path)), - PageType::Internal => { - let (position, child) = internal_child_position(&page, key)?; - path.try_reserve(1).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - path.push((page_id, position)); - page_id = child; - } - PageType::Overflow | PageType::Free => return Err(invalid_data("tree references a non-tree page")), - } - } -} - -fn overflow_chain_pages(transaction: &WriteTransaction, base: &[u8], mut page_id: u64) -> io::Result> { - let mut pages = Vec::new(); - let mut visited = HashSet::new(); - while page_id != 0 { - if !visited.insert(page_id) { - return Err(invalid_data("overflow chain contains a cycle")); - } - let page = SlottedPage::open(transaction.page(base, page_id)?, page_id, transaction.next_header.next_page_id)?; - if page.header().page_type != PageType::Overflow || overflow_payload(&page)?.is_empty() { - return Err(invalid_data("invalid overflow chain page")); - } - pages.try_reserve(1).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - pages.push(page_id); - page_id = page.header().right; - } - if pages.is_empty() { - return Err(invalid_data("overflow chain is empty")); - } - Ok(pages) -} - -fn validated_overflow_chain_pages( - transaction: &WriteTransaction, - base: &[u8], - head: u64, - stored_len: u32, - crc32: u32, -) -> io::Result> { - let pages = overflow_chain_pages(transaction, base, head)?; - let expected = usize::try_from(stored_len).map_err(|_| invalid_data("stored length exceeds usize"))?; - let mut actual = 0usize; - let mut hasher = crc32fast::Hasher::new(); - for page_id in &pages { - let page = - SlottedPage::open(transaction.page(base, *page_id)?, *page_id, transaction.next_header.next_page_id)?; - let payload = overflow_payload(&page)?; - actual = actual.checked_add(payload.len()).ok_or_else(|| invalid_data("overflow chain length overflow"))?; - if actual > expected { - return Err(invalid_data("overflow chain exceeds declared length")); - } - hasher.update(payload); - } - if actual != expected || hasher.finalize() != crc32 { - return Err(invalid_data("overflow value checksum or length mismatch")); - } - Ok(pages) -} - -fn free_pages(transaction: &mut WriteTransaction, pages: &[u64]) -> io::Result<()> { - for page_id in pages.iter().rev() { - transaction.free_page(*page_id)?; - } - Ok(()) -} - -fn write_overflow_chain( - transaction: &mut WriteTransaction, - base: &[u8], - stored: &[u8], - old_pages: &[u64], -) -> io::Result { - let required = stored.len().div_ceil(OVERFLOW_PAYLOAD_LEN); - if required == 0 { - return Err(invalid_input("overflow value must not be empty")); - } - let mut pages = Vec::new(); - pages.try_reserve_exact(required).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - if old_pages.len() >= required { - pages.extend_from_slice( - old_pages.get(..required).ok_or_else(|| invalid_data("overflow reuse range is invalid"))?, - ); - free_pages( - transaction, - old_pages.get(required..).ok_or_else(|| invalid_data("overflow tail range is invalid"))?, - )?; - } else { - for _ in 0..required { - pages.push(transaction.allocate_page(base)?); - } - free_pages(transaction, old_pages)?; - } - for (index, payload) in stored.chunks(OVERFLOW_PAYLOAD_LEN).enumerate() { - let page_id = *pages.get(index).ok_or_else(|| invalid_data("overflow page allocation is missing"))?; - let next = pages.get(index + 1).copied().unwrap_or(0); - transaction - .write_page(page_id, encode_overflow_page(page_id, transaction.next_header.next_page_id, next, payload)?)?; - } - pages.first().copied().ok_or_else(|| invalid_data("overflow head is missing")) -} - -fn repair_right_leaf_backlink( - transaction: &mut WriteTransaction, - base: &[u8], - right_page_id: u64, - left_page_id: u64, -) -> io::Result<()> { - if right_page_id == 0 { - return Ok(()); - } - let next_page_id = transaction.next_header.next_page_id; - let page = transaction.page_mut(base, right_page_id)?; - let mut header = SlottedPage::open(page.as_slice(), right_page_id, next_page_id)?.header(); - if header.page_type != PageType::Leaf { - return Err(invalid_data("leaf sibling references a non-leaf page")); - } - header.left = left_page_id; - header.encode_into(page, right_page_id, next_page_id) -} - -fn mutate_leaf( - transaction: &mut WriteTransaction, - base: &[u8], - leaf_id: u64, - index: usize, - replace: bool, - cell: &[u8], -) -> io::Result> { - let snapshot = transaction.page_copy(base, leaf_id)?; - let page = SlottedPage::open(snapshot.as_slice(), leaf_id, transaction.next_header.next_page_id)?; - if page.header().page_type != PageType::Leaf { - return Err(invalid_data("mutation target is not a leaf page")); - } - let count = usize::from(page.header().cell_count); - if (replace && index >= count) || (!replace && index > count) { - return Err(invalid_input("leaf mutation index is outside page")); - } - if replace && page.cell(index)?.len() == cell.len() { - let next_page_id = transaction.next_header.next_page_id; - let dirty = transaction.page_mut(base, leaf_id)?; - return SlottedPage::open(dirty.as_mut_slice(), leaf_id, next_page_id)? - .replace_same_len(index, cell) - .map(|()| None); - } - let mut cells = Vec::<&[u8]>::new(); - cells - .try_reserve_exact(count + usize::from(!replace)) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - for current in 0..count { - if current == index { - cells.push(cell); - if replace { - continue; - } - } - cells.push(page.cell(current)?); - } - if index == count { - cells.push(cell); - } - if used_leaf_bytes(&cells)? <= PAGE_SIZE { - let next_page_id = transaction.next_header.next_page_id; - let dirty = transaction.page_mut(base, leaf_id)?; - SlottedPage::open(dirty.as_mut_slice(), leaf_id, next_page_id)?.rebuild_ordered(cells.iter().copied())?; - return Ok(None); - } - - let new_leaf_id = transaction.allocate_page(base)?; - let boundary = choose_leaf_split(&cells)?; - let (left_cells, right_cells) = cells.split_at(boundary); - let right = page.header().right; - let left_page = leaf_page(leaf_id, page.header().left, new_leaf_id, left_cells)?; - let right_page = leaf_page(new_leaf_id, leaf_id, right, right_cells)?; - let separator = LeafCellRef::decode( - right_cells.first().ok_or_else(|| invalid_data("split right leaf is empty"))?, - new_leaf_id, - transaction.next_header.next_page_id, - )? - .key_bytes - .to_vec(); - transaction.write_page(leaf_id, left_page)?; - transaction.write_page(new_leaf_id, right_page)?; - repair_right_leaf_backlink(transaction, base, right, new_leaf_id)?; - Ok(Some(Promotion { key: separator, right_child: new_leaf_id })) -} - -fn insert_internal_promotion( - transaction: &mut WriteTransaction, - base: &[u8], - page_id: u64, - position: usize, - promotion: &Promotion, - cell_scratch: &mut Vec, -) -> io::Result> { - let snapshot = transaction.page_copy(base, page_id)?; - let page = SlottedPage::open(snapshot.as_slice(), page_id, transaction.next_header.next_page_id)?; - if page.header().page_type != PageType::Internal { - return Err(invalid_data("promotion target is not an internal page")); - } - let count = usize::from(page.header().cell_count); - if position > count { - return Err(invalid_input("internal insertion position is outside page")); - } - encode_internal_cell( - &promotion.key, - promotion.right_child, - page_id, - transaction.next_header.next_page_id, - cell_scratch, - )?; - let mut cells = Vec::<&[u8]>::new(); - cells.try_reserve_exact(count + 1).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - for current in 0..count { - if current == position { - cells.push(cell_scratch); - } - cells.push(page.cell(current)?); - } - if position == count { - cells.push(cell_scratch); - } - if used_internal_bytes(cells.iter().copied())? <= PAGE_SIZE { - let next_page_id = transaction.next_header.next_page_id; - let dirty = transaction.page_mut(base, page_id)?; - SlottedPage::open(dirty.as_mut_slice(), page_id, next_page_id)?.rebuild_ordered(cells.iter().copied())?; - return Ok(None); - } - - let right_page_id = transaction.allocate_page(base)?; - let split = choose_internal_split(&cells, page_id, transaction.next_header.next_page_id)?; - let leftmost = - InternalPreamble::decode(snapshot.as_slice(), page_id, transaction.next_header.next_page_id)?.leftmost_child; - let promoted = Promotion { key: split.promoted.key_bytes.to_vec(), right_child: right_page_id }; - let left_page = internal_page(page_id, leftmost, split.left_cells)?; - let right_page = internal_page(right_page_id, split.right_leftmost_child, split.right_cells)?; - transaction.write_page(page_id, left_page)?; - transaction.write_page(right_page_id, right_page)?; - Ok(Some(promoted)) -} - -fn propagate_promotion( - transaction: &mut WriteTransaction, - base: &[u8], - mut path: Vec<(u64, usize)>, - mut promotion: Promotion, - cell_scratch: &mut Vec, -) -> io::Result<()> { - while let Some((parent, position)) = path.pop() { - let Some(next) = insert_internal_promotion(transaction, base, parent, position, &promotion, cell_scratch)? - else { - return Ok(()); - }; - promotion = next; - } - let old_root = transaction.next_header.root_page_id; - let new_root = transaction.allocate_page(base)?; - encode_internal_cell( - &promotion.key, - promotion.right_child, - new_root, - transaction.next_header.next_page_id, - cell_scratch, - )?; - transaction.write_page(new_root, internal_page(new_root, old_root, &[cell_scratch.as_slice()])?)?; - transaction.next_header.root_page_id = new_root; - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -fn stage_upsert Deserialize<'de>>( - transaction: &mut WriteTransaction, - base: &[u8], - key: &K, - encoded_key: &[u8], - logical_len: u32, - compression: Compression, - stored: &[u8], - cell_scratch: &mut Vec, -) -> io::Result<()> { - let (leaf_id, path) = locate_transaction_leaf(transaction, base, key)?; - let page = SlottedPage::open(transaction.page(base, leaf_id)?, leaf_id, transaction.next_header.next_page_id)?; - let search = search_leaf(&page, key)?; - let (replace, index, old_overflow) = match search { - Ok(index) => { - let old = LeafCellRef::decode(page.cell(index)?, leaf_id, transaction.next_header.next_page_id)?; - let overflow = match old.value { - LeafValueRef::Overflow { stored_len, head, crc32, .. } => { - validated_overflow_chain_pages(transaction, base, head, stored_len, crc32)? - } - LeafValueRef::Inline { .. } | LeafValueRef::Tombstone => Vec::new(), - }; - (true, index, overflow) - } - Err(index) => (false, index, Vec::new()), - }; - let inline_footprint = SLOT_LEN - .checked_add(24) - .and_then(|size| size.checked_add(encoded_key.len())) - .and_then(|size| size.checked_add(stored.len())) - .ok_or_else(|| invalid_input("leaf cell footprint overflow"))?; - if stored.len() <= MAX_INLINE_STORED_VALUE && inline_footprint <= MAX_CELL_FOOTPRINT { - free_pages(transaction, &old_overflow)?; - encode_inline_leaf_cell(encoded_key, logical_len, compression, stored, cell_scratch)?; - } else { - let head = write_overflow_chain(transaction, base, stored, &old_overflow)?; - encode_overflow_leaf_cell( - encoded_key, - logical_len, - compression, - u32::try_from(stored.len()).map_err(|_| invalid_input("stored value exceeds u32"))?, - head, - stored_value_checksum(stored), - leaf_id, - transaction.next_header.next_page_id, - cell_scratch, - )?; - } - if let Some(promotion) = mutate_leaf(transaction, base, leaf_id, index, replace, cell_scratch)? { - propagate_promotion(transaction, base, path, promotion, cell_scratch)?; - } - Ok(()) -} - -fn stage_delete Deserialize<'de>>( - transaction: &mut WriteTransaction, - base: &[u8], - key: &K, - encoded_key: &[u8], - cell_scratch: &mut Vec, -) -> io::Result { - let (leaf_id, _) = locate_transaction_leaf(transaction, base, key)?; - let page = SlottedPage::open(transaction.page(base, leaf_id)?, leaf_id, transaction.next_header.next_page_id)?; - let Ok(index) = search_leaf(&page, key)? else { return Ok(false) }; - let old = LeafCellRef::decode(page.cell(index)?, leaf_id, transaction.next_header.next_page_id)?; - match old.value { - LeafValueRef::Tombstone => return Ok(false), - LeafValueRef::Overflow { stored_len, head, crc32, .. } => { - let pages = validated_overflow_chain_pages(transaction, base, head, stored_len, crc32)?; - free_pages(transaction, &pages)?; - } - LeafValueRef::Inline { .. } => {} - } - encode_tombstone_leaf_cell(encoded_key, cell_scratch)?; - let _ = mutate_leaf(transaction, base, leaf_id, index, true, cell_scratch)?; - Ok(true) -} - -fn query_transaction( - transaction: &WriteTransaction, - base: &[u8], - key: &K, - scratch: &mut Vec, -) -> io::Result> -where - K: Ord + for<'de> Deserialize<'de>, - V: for<'de> Deserialize<'de>, -{ - let (leaf_id, _) = locate_transaction_leaf(transaction, base, key)?; - let page = SlottedPage::open(transaction.page(base, leaf_id)?, leaf_id, transaction.next_header.next_page_id)?; - let Ok(index) = search_leaf(&page, key)? else { return Ok(None) }; - let cell = LeafCellRef::decode(page.cell(index)?, leaf_id, transaction.next_header.next_page_id)?; - let bytes = match cell.value { - LeafValueRef::Tombstone => return Ok(None), - LeafValueRef::Inline { compression: Compression::None, stored, .. } => stored, - LeafValueRef::Inline { compression: Compression::Lz4, logical_len, stored, .. } => { - decompress_value_into(stored, logical_len, transaction_value_limit(transaction)?, scratch)? - } - LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } => { - read_transaction_overflow(transaction, base, compression, logical_len, stored_len, head, crc32, scratch)? - } - }; - binary_deserialize(bytes).map(Some) -} - -fn transaction_value_limit(transaction: &WriteTransaction) -> io::Result { - usize::try_from(transaction.next_header.next_page_id) - .map_err(|_| invalid_data("next page id exceeds usize"))? - .checked_mul(PAGE_SIZE) - .and_then(|size| size.checked_mul(256)) - .map(|size| size.min(usize::try_from(u32::MAX).unwrap_or(usize::MAX))) - .ok_or_else(|| invalid_data("value allocation limit overflow")) -} - -#[allow(clippy::too_many_arguments)] -fn read_transaction_overflow<'a>( - transaction: &WriteTransaction, - base: &[u8], - compression: Compression, - logical_len: u32, - stored_len: u32, - head: u64, - crc32: u32, - scratch: &'a mut Vec, -) -> io::Result<&'a [u8]> { - let stored_len = usize::try_from(stored_len).map_err(|_| invalid_data("stored length exceeds usize"))?; - if stored_len > transaction_value_limit(transaction)? { - return Err(invalid_data("overflow value exceeds allocation limit")); - } - scratch.clear(); - scratch.try_reserve(stored_len).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - for page_id in overflow_chain_pages(transaction, base, head)? { - let page = SlottedPage::open(transaction.page(base, page_id)?, page_id, transaction.next_header.next_page_id)?; - let payload = overflow_payload(&page)?; - if scratch.len().saturating_add(payload.len()) > stored_len { - return Err(invalid_data("overflow chain exceeds declared length")); - } - scratch.extend_from_slice(payload); - } - if scratch.len() != stored_len || crc32fast::hash(scratch) != crc32 { - return Err(invalid_data("overflow value checksum or length mismatch")); - } - if compression == Compression::Lz4 { - decompress_value_in_place(scratch, logical_len, transaction_value_limit(transaction)?)?; - } else if scratch.len() != usize::try_from(logical_len).map_err(|_| invalid_data("logical length exceeds usize"))? { - return Err(invalid_data("uncompressed overflow length mismatch")); - } - Ok(scratch) -} - -enum DecodedEntry { - Ready(K, V), - Overflow(K, Compression, u32, u32, u64, u32), - Tombstone, -} - -enum DecodedValue { - Ready(V), - Overflow(Compression, u32, u32, u64, u32), - Tombstone, -} - -struct InternalRoute { - leftmost_child: u64, - separators: Vec<(K, u64)>, -} - -impl InternalRoute { - fn child_for(&self, target: &K) -> u64 { - let index = self.separators.partition_point(|(key, _)| key <= target); - index.checked_sub(1).map_or(self.leftmost_child, |previous| self.separators[previous].1) - } -} - -fn decode_internal_route(page: &SlottedPage) -> io::Result> -where - K: Ord + for<'de> Deserialize<'de>, - B: AsRef<[u8]>, -{ - let preamble = InternalPreamble::decode(page.as_bytes(), page.page_id(), page.next_page_id())?; - let count = usize::from(page.header().cell_count); - let mut separators = Vec::new(); - separators.try_reserve_exact(count).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - for index in 0..count { - let cell = InternalCellRef::decode(page.cell(index)?, page.page_id(), page.next_page_id())?; - separators.push((decode_internal_key(cell.key_bytes)?, cell.right_child)); - } - Ok(InternalRoute { leftmost_child: preamble.leftmost_child, separators }) -} - -enum LocateCell { - Internal(InternalRoute), - Leaf(Option>), -} - -enum LocateLeaf { - Internal(InternalRoute), - Leaf, -} - -struct QuerySnapshot { - file: Option, - mmap: Option, - filepath: PathBuf, - page_validations: Vec>, - internal_routes: Vec>>, - sidecar_guard: Option, -} - pub struct BPlusTreeQuery { snapshot: Arc>, header: DatabaseHeader, @@ -2299,993 +85,22 @@ pub struct BPlusTreeQuery { _value: PhantomData, } -impl BPlusTreeQuery { - fn from_file_unlocked(file: File) -> io::Result { - let file_len = - usize::try_from(file.metadata()?.len()).map_err(|_| invalid_data("database length exceeds usize"))?; - if file_len < PAGE_SIZE { - return Err(invalid_data("database is shorter than its header page")); - } - let mut header_page = [0; PAGE_SIZE]; - read_exact_at_offset(&file, &mut header_page, 0)?; - let header = DatabaseHeader::decode(&header_page)?; - let expected = usize::try_from(header.next_page_id) - .map_err(|_| invalid_data("next page id exceeds usize"))? - .checked_mul(PAGE_SIZE) - .ok_or_else(|| invalid_data("database length overflow"))?; - if file_len != expected { - return Err(invalid_data("database file length does not match header")); - } - let mmap = mmap_with_advice(&file, Advice::Normal, "v3 B+Tree query"); - let page_count = - usize::try_from(header.next_page_id).map_err(|_| invalid_data("next page id exceeds usize"))?; - let mut page_validations = Vec::new(); - page_validations - .try_reserve_exact(page_count) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - let mut internal_routes = Vec::new(); - internal_routes - .try_reserve_exact(page_count) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - for _ in 0..page_count { - page_validations.push(OnceLock::new()); - internal_routes.push(OnceLock::new()); - } - let file = mmap.is_none().then_some(file); - let mut page_buffer = Vec::new(); - if mmap.is_none() { - page_buffer - .try_reserve_exact(PAGE_SIZE) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - page_buffer.resize(PAGE_SIZE, 0); - } - Ok(Self { - snapshot: Arc::new(QuerySnapshot { - file, - mmap, - filepath: PathBuf::new(), - page_validations, - internal_routes, - sidecar_guard: None, - }), - header, - file_len, - page_buffer, - value_scratch: Vec::new(), - locator_page_id: 0, - locator_cell_ranges: Vec::new(), - _value: PhantomData, - }) - } - - pub fn try_new(filepath: &Path) -> io::Result { - loop { - let sidecar_guard = SharedSidecarGuard::acquire(filepath)?; - let pending = wal_path(filepath).try_exists()? || wal_temporary_path(filepath).try_exists()?; - if !pending { - let mut query = Self::from_file_unlocked(File::open(filepath)?)?; - let snapshot = Arc::get_mut(&mut query.snapshot) - .ok_or_else(|| invalid_data("new query snapshot is unexpectedly shared"))?; - snapshot.filepath = filepath.to_path_buf(); - snapshot.sidecar_guard = Some(sidecar_guard); - return Ok(query); - } - drop(sidecar_guard); - match recover_pending(filepath) { - Ok(()) => {} - Err(error) - if matches!(error.kind(), io::ErrorKind::PermissionDenied | io::ErrorKind::ReadOnlyFilesystem) => - { - return Err(recovery_required(filepath, error)); - } - Err(error) => return Err(error), - } - } - } - - pub fn try_clone(&self) -> io::Result { - if self.snapshot.filepath.as_os_str().is_empty() { - return Err(invalid_input("mapped query without a path cannot be cloned")); - } - let mut page_buffer = Vec::new(); - if self.snapshot.mmap.is_none() { - page_buffer - .try_reserve_exact(PAGE_SIZE) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - page_buffer.resize(PAGE_SIZE, 0); - } - Ok(Self { - snapshot: Arc::clone(&self.snapshot), - header: self.header.clone(), - file_len: self.file_len, - page_buffer, - value_scratch: Vec::new(), - locator_page_id: 0, - locator_cell_ranges: Vec::new(), - _value: PhantomData, - }) - } - - /// A query handle over an empty in-memory snapshot, for tests that need a - /// `BPlusTreeQuery` value without a database behind it. - #[cfg(any(test, feature = "test-support"))] - pub fn clone_error_fixture() -> Self { - Self { - snapshot: Arc::new(QuerySnapshot { - file: None, - mmap: None, - filepath: PathBuf::new(), - page_validations: Vec::new(), - internal_routes: Vec::new(), - sidecar_guard: None, - }), - header: DatabaseHeader { - root_page_id: 1, - next_page_id: 2, - free_page_head: 0, - generation: 1, - database_id: [0; 16], - metadata: BPlusTreeMetadata::Empty, - }, - file_len: 0, - page_buffer: Vec::new(), - value_scratch: Vec::new(), - locator_page_id: 0, - locator_cell_ranges: Vec::new(), - _value: PhantomData, - } - } - - pub fn filepath(&self) -> &Path { &self.snapshot.filepath } - - /// The metadata carried in the database header of this snapshot. - pub fn metadata(&self) -> &BPlusTreeMetadata { &self.header.metadata } - - pub fn snapshot_identity(&self) -> ([u8; 16], u64) { (self.header.database_id, self.header.generation) } - - pub(crate) fn snapshot_metadata(&self) -> &BPlusTreeMetadata { &self.header.metadata } - - fn value_allocation_limit(&self) -> usize { - self.file_len.saturating_mul(256).min(usize::try_from(u32::MAX).unwrap_or(usize::MAX)) - } - - fn assemble_overflow_chain( - &mut self, - compression: Compression, - logical_len: u32, - stored_len: u32, - mut page_id: u64, - crc32: u32, - mut owned_pages: Option<&mut HashSet>, - ) -> io::Result<()> { - let stored_len = usize::try_from(stored_len).map_err(|_| invalid_data("stored length exceeds usize"))?; - let logical_len_usize = - usize::try_from(logical_len).map_err(|_| invalid_data("logical length exceeds usize"))?; - if stored_len > self.file_len || logical_len_usize > self.value_allocation_limit() { - return Err(invalid_data("overflow value exceeds allocation limit")); - } - self.value_scratch.clear(); - self.value_scratch - .try_reserve(stored_len) - .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - let mut chain = HashSet::new(); - while page_id != 0 { - record_page_visit(&mut chain, page_id, "overflow chain contains a cycle")?; - if let Some(pages) = owned_pages.as_deref_mut() { - record_page_visit(pages, page_id, "overflow page is owned by multiple values")?; - } - page_id = self.with_slotted_page(page_id, |page, scratch| { - if page.header().page_type != PageType::Overflow { - return Err(invalid_data("overflow chain references a non-overflow page")); - } - let payload = overflow_payload(page)?; - if payload.is_empty() { - return Err(invalid_data("overflow chain contains an empty payload")); - } - let next_len = scratch - .len() - .checked_add(payload.len()) - .ok_or_else(|| invalid_data("overflow value length overflow"))?; - if next_len > stored_len { - return Err(invalid_data("overflow chain exceeds declared length")); - } - scratch.extend_from_slice(payload); - Ok(page.header().right) - })?; - } - if self.value_scratch.len() != stored_len || crc32fast::hash(&self.value_scratch) != crc32 { - return Err(invalid_data("overflow value checksum or length mismatch")); - } - if compression == Compression::Lz4 { - let allocation_limit = self.value_allocation_limit(); - decompress_value_in_place(&mut self.value_scratch, logical_len, allocation_limit)?; - } else if self.value_scratch.len() != logical_len_usize { - return Err(invalid_data("uncompressed overflow length mismatch")); - } - Ok(()) - } - - fn with_page(&mut self, page_id: u64, read: impl FnOnce(&[u8], &mut Vec) -> io::Result) -> io::Result { - if page_id == 0 || page_id >= self.header.next_page_id { - return Err(invalid_data("page id is outside database")); - } - let offset = usize::try_from(page_id) - .map_err(|_| invalid_data("page id exceeds usize"))? - .checked_mul(PAGE_SIZE) - .ok_or_else(|| invalid_data("page offset overflow"))?; - let end = offset.checked_add(PAGE_SIZE).ok_or_else(|| invalid_data("page end overflow"))?; - if let Some(mmap) = &self.snapshot.mmap { - let page = mmap.get(offset..end).ok_or_else(|| invalid_data("page is truncated"))?; - return read(page, &mut self.value_scratch); - } - let file = self.snapshot.file.as_ref().ok_or_else(|| invalid_data("query has no data source"))?; - if self.page_buffer.len() != PAGE_SIZE { - return Err(invalid_data("query page buffer has invalid length")); - } - read_exact_at_offset( - file, - &mut self.page_buffer, - u64::try_from(offset).map_err(|_| invalid_data("page offset exceeds u64"))?, - )?; - read(&self.page_buffer, &mut self.value_scratch) - } - - fn with_slotted_page( - &mut self, - page_id: u64, - read: impl FnOnce(&SlottedPage<&[u8]>, &mut Vec) -> io::Result, - ) -> io::Result { - let index = usize::try_from(page_id).map_err(|_| invalid_data("page id exceeds usize"))?; - let cached = self - .snapshot - .page_validations - .get(index) - .ok_or_else(|| invalid_data("page id is outside cache"))? - .get() - .copied(); - let next_page_id = self.header.next_page_id; - let (result, validation) = self.with_page(page_id, |bytes, scratch| { - let page = match cached { - Some(validation) => SlottedPage::from_immutable_snapshot(bytes, validation)?, - None => SlottedPage::open(bytes, page_id, next_page_id)?, - }; - let validation = cached.is_none().then(|| page.validation()); - read(&page, scratch).map(|result| (result, validation)) - })?; - if let Some(validation) = validation { - let slot = - self.snapshot.page_validations.get(index).ok_or_else(|| invalid_data("page id is outside cache"))?; - let _ = slot.set(validation); - } - Ok(result) - } - - fn cached_internal_child(&self, page_id: u64, key: &K) -> io::Result> - where - K: Ord, - { - let index = usize::try_from(page_id).map_err(|_| invalid_data("page id exceeds usize"))?; - Ok(self - .snapshot - .internal_routes - .get(index) - .ok_or_else(|| invalid_data("page id is outside route cache"))? - .get() - .map(|route| route.child_for(key))) - } - - fn cache_internal_route(&self, page_id: u64, route: InternalRoute) -> io::Result<()> { - let index = usize::try_from(page_id).map_err(|_| invalid_data("page id exceeds usize"))?; - let slot = - self.snapshot.internal_routes.get(index).ok_or_else(|| invalid_data("page id is outside route cache"))?; - let _ = slot.set(route); - Ok(()) - } - - fn locate_leaf(&mut self, key: &K) -> io::Result - where - K: Ord + for<'de> Deserialize<'de>, - { - let mut page_id = self.header.root_page_id; - let mut depth = 0u64; - loop { - if depth >= self.header.next_page_id { - return Err(invalid_data("tree descent contains a cycle")); - } - if let Some(child) = self.cached_internal_child(page_id, key)? { - page_id = child; - depth += 1; - continue; - } - let step = self.with_slotted_page(page_id, |page, _| match page.header().page_type { - PageType::Leaf => Ok(LocateLeaf::Leaf), - PageType::Internal => decode_internal_route(page).map(LocateLeaf::Internal), - PageType::Overflow | PageType::Free => Err(invalid_data("tree references a non-tree page")), - })?; - match step { - LocateLeaf::Internal(route) => { - let child = route.child_for(key); - self.cache_internal_route(page_id, route)?; - page_id = child; - depth += 1; - } - LocateLeaf::Leaf => return Ok(page_id), - } - } - } - - fn locate_cell(&mut self, key: &K) -> io::Result)>> - where - K: Ord + for<'de> Deserialize<'de>, - { - let mut page_id = self.header.root_page_id; - let mut depth = 0u64; - loop { - if depth >= self.header.next_page_id { - return Err(invalid_data("tree descent contains a cycle")); - } - if let Some(child) = self.cached_internal_child(page_id, key)? { - page_id = child; - depth += 1; - continue; - } - let step = self.with_slotted_page(page_id, |page, _| match page.header().page_type { - PageType::Internal => decode_internal_route(page).map(LocateCell::Internal), - PageType::Leaf => { - search_leaf(page, key)?.ok().map(|index| page.cell_range(index)).transpose().map(LocateCell::Leaf) - } - PageType::Overflow | PageType::Free => Err(invalid_data("tree references a non-tree page")), - })?; - match step { - LocateCell::Internal(route) => { - let child = route.child_for(key); - self.cache_internal_route(page_id, route)?; - page_id = child; - depth += 1; - } - LocateCell::Leaf(range) => return Ok(range.map(|range| (page_id, range))), - } - } - } - - fn leftmost_leaf(&mut self) -> io::Result - where - K: for<'de> Deserialize<'de>, - { - let mut page_id = self.header.root_page_id; - let mut depth = 0u64; - loop { - if depth >= self.header.next_page_id { - return Err(invalid_data("tree descent contains a cycle")); - } - let result = self.with_slotted_page(page_id, |page, _| match page.header().page_type { - PageType::Leaf => Ok(None), - PageType::Internal => InternalPreamble::decode(page.as_bytes(), page_id, page.next_page_id()) - .map(|preamble| Some(preamble.leftmost_child)), - PageType::Overflow | PageType::Free => Err(invalid_data("tree references a non-tree page")), - })?; - let Some(child) = result else { return Ok(page_id) }; - page_id = child; - depth += 1; - } - } - - fn decode_entry(&mut self, leaf_page_id: u64, slot_index: usize) -> io::Result> - where - K: for<'de> Deserialize<'de>, - V: for<'de> Deserialize<'de>, - { - let range = self.with_slotted_page(leaf_page_id, |page, _| { - if page.header().page_type != PageType::Leaf { - return Err(invalid_data("iterator expected a leaf page")); - } - page.cell_range(slot_index) - })?; - self.decode_entry_range(leaf_page_id, range) - } - - fn decode_entry_range(&mut self, leaf_page_id: u64, range: Range) -> io::Result> - where - K: for<'de> Deserialize<'de>, - V: for<'de> Deserialize<'de>, - { - let next_page_id = self.header.next_page_id; - let allocation_limit = self.value_allocation_limit(); - let decoded = self.with_page(leaf_page_id, |bytes, scratch| { - let cell_bytes = bytes.get(range).ok_or_else(|| invalid_data("leaf cell is outside page"))?; - let cell = LeafCellRef::decode(cell_bytes, leaf_page_id, next_page_id)?; - let key = binary_deserialize(cell.key_bytes)?; - match cell.value { - LeafValueRef::Inline { compression: Compression::None, stored, .. } => { - binary_deserialize(stored).map(|value| DecodedEntry::Ready(key, value)) - } - LeafValueRef::Inline { compression: Compression::Lz4, logical_len, stored, .. } => { - let decompressed = decompress_value_into(stored, logical_len, allocation_limit, scratch)?; - binary_deserialize(decompressed).map(|value| DecodedEntry::Ready(key, value)) - } - LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } => { - Ok(DecodedEntry::Overflow(key, compression, logical_len, stored_len, head, crc32)) - } - LeafValueRef::Tombstone => Ok(DecodedEntry::Tombstone), - } - })?; - match decoded { - DecodedEntry::Ready(key, value) => Ok(Some((key, value))), - DecodedEntry::Tombstone => Ok(None), - DecodedEntry::Overflow(key, compression, logical_len, stored_len, page_id, crc32) => { - self.assemble_overflow_chain(compression, logical_len, stored_len, page_id, crc32, None)?; - let value = binary_deserialize(&self.value_scratch)?; - Ok(Some((key, value))) - } - } - } -} - -impl BPlusTreeQuery -where - K: Ord + for<'de> Deserialize<'de>, - V: for<'de> Deserialize<'de>, -{ - fn query_io(&mut self, key: &K) -> io::Result> { - let Some((leaf_id, range)) = self.locate_cell(key)? else { return Ok(None) }; - self.decode_entry_range(leaf_id, range).map(|entry| entry.map(|(_, value)| value)) - } - - pub fn query(&mut self, key: &K) -> Result, BPlusTreeError> { self.query_io(key).map_err(Into::into) } - - pub fn query_zero_copy(&mut self, key: &K) -> Result, BPlusTreeError> { self.query(key) } - - fn locator_cell_range(&mut self, locator: Locator) -> io::Result> { - if self.locator_page_id != locator.leaf_page_id { - let page_id = locator.leaf_page_id; - let mut ranges = std::mem::take(&mut self.locator_cell_ranges); - self.with_slotted_page(page_id, |page, _| { - if page.header().page_type != PageType::Leaf { - return Err(invalid_data("locator does not reference a leaf page")); - } - let count = usize::from(page.header().cell_count); - ranges.clear(); - ranges.try_reserve(count).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - for index in 0..count { - ranges.push(page.cell_range(index)?); - } - Ok(()) - })?; - self.locator_page_id = page_id; - self.locator_cell_ranges = ranges; - } - self.locator_cell_ranges - .get(usize::from(locator.slot_index)) - .cloned() - .ok_or_else(|| invalid_data("locator slot index is outside leaf page")) - } - - pub(crate) fn read_locator_value(&mut self, locator: Locator, primary_key: &[u8]) -> io::Result { - let page_id = locator.leaf_page_id; - let next_page_id = self.header.next_page_id; - let range = self.locator_cell_range(locator)?; - let allocation_limit = self.value_allocation_limit(); - let decoded = self.with_page(page_id, |bytes, scratch| { - let cell = LeafCellRef::decode( - bytes.get(range).ok_or_else(|| invalid_data("locator cell is outside page"))?, - page_id, - next_page_id, - )?; - if crc32fast::hash(cell.key_bytes) != locator.serialized_key_crc32 - || crc32fast::hash(primary_key) != locator.serialized_key_crc32 - || cell.key_bytes != primary_key - { - return Err(invalid_data("locator serialized key mismatch")); - } - match cell.value { - LeafValueRef::Inline { compression: Compression::None, stored, .. } => { - binary_deserialize(stored).map(DecodedValue::Ready) - } - LeafValueRef::Inline { compression: Compression::Lz4, logical_len, stored, .. } => { - let decompressed = decompress_value_into(stored, logical_len, allocation_limit, scratch)?; - binary_deserialize(decompressed).map(DecodedValue::Ready) - } - LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } => { - Ok(DecodedValue::Overflow(compression, logical_len, stored_len, head, crc32)) - } - LeafValueRef::Tombstone => Ok(DecodedValue::Tombstone), - } - })?; - match decoded { - DecodedValue::Ready(value) => Ok(value), - DecodedValue::Overflow(compression, logical_len, stored_len, head, crc32) => { - self.assemble_overflow_chain(compression, logical_len, stored_len, head, crc32, None)?; - binary_deserialize(&self.value_scratch) - } - DecodedValue::Tombstone => Err(invalid_data("locator references a tombstone")), - } - } - - pub(crate) fn collect_with_locators(&mut self) -> io::Result> { - let mut result = Vec::new(); - let mut page_id = self.header.root_page_id; - let mut visited = HashSet::new(); - let mut descending = true; - let mut descent_depth = 0u64; - loop { - let next_page_id = self.header.next_page_id; - let (child, right, locators) = self.with_page(page_id, |bytes, _| { - let page = SlottedPage::open(bytes, page_id, next_page_id)?; - if descending && page.header().page_type == PageType::Internal { - let leftmost = InternalPreamble::decode(bytes, page_id, next_page_id)?.leftmost_child; - return Ok((Some(leftmost), 0, Vec::new())); - } - if page.header().page_type != PageType::Leaf { - return Err(invalid_data("locator scan expected a tree page")); - } - let mut locators = Vec::new(); - for index in 0..usize::from(page.header().cell_count) { - let cell = LeafCellRef::decode(page.cell(index)?, page_id, next_page_id)?; - if !matches!(cell.value, LeafValueRef::Tombstone) { - locators.push(( - Locator::for_key( - page_id, - u16::try_from(index).map_err(|_| invalid_data("slot index exceeds u16"))?, - cell.key_bytes, - )?, - page.cell_range(index)?, - )); - } - } - Ok((None, page.header().right, locators)) - })?; - if let Some(child) = child { - if descent_depth >= self.header.next_page_id { - return Err(invalid_data("locator descent contains a cycle")); - } - page_id = child; - descent_depth += 1; - continue; - } - descending = false; - record_page_visit(&mut visited, page_id, "right sibling chain contains a cycle")?; - result.try_reserve(locators.len()).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - for (locator, range) in locators { - let entry = self - .decode_entry_range(page_id, range)? - .ok_or_else(|| invalid_data("live locator became a tombstone"))?; - result.push((entry.0, entry.1, locator)); - } - if right == 0 { - return Ok(result); - } - page_id = right; - } - } - - pub fn contains_live_key(&mut self, key: &K) -> Result { - let result: io::Result = (|| { - let leaf_id = self.locate_leaf(key)?; - let next_page_id = self.header.next_page_id; - let index = self.with_slotted_page(leaf_id, |page, _| search_leaf(page, key))?; - let Ok(index) = index else { return Ok(false) }; - self.with_slotted_page(leaf_id, |page, _| { - let cell = LeafCellRef::decode(page.cell(index)?, leaf_id, next_page_id)?; - Ok(!matches!(cell.value, LeafValueRef::Tombstone)) - }) - })(); - result.map_err(Into::into) - } - - pub fn query_le(&mut self, key: &K) -> Result, BPlusTreeError> { - self.query_le_io(key).map_err(Into::into) - } - - fn query_le_io(&mut self, key: &K) -> io::Result> { - let mut leaf_id = self.locate_leaf(key)?; - let mut first = true; - let mut visited = HashSet::new(); - record_page_visit(&mut visited, leaf_id, "left sibling chain contains a cycle")?; - loop { - let (left, count, start) = self.with_slotted_page(leaf_id, |page, _| { - if page.header().page_type != PageType::Leaf { - return Err(invalid_data("left sibling is not a leaf")); - } - let count = usize::from(page.header().cell_count); - let start = if first { - match search_leaf(page, key)? { - Ok(index) => Some(index), - Err(0) => None, - Err(index) => index.checked_sub(1), - } - } else { - count.checked_sub(1) - }; - Ok((page.header().left, count, start)) - })?; - if let Some(start) = start { - for index in (0..=start.min(count.saturating_sub(1))).rev() { - if let Some((_, value)) = self.decode_entry(leaf_id, index)? { - return Ok(Some(value)); - } - } - } - if left == 0 { - return Ok(None); - } - let current = leaf_id; - record_page_visit(&mut visited, left, "left sibling chain contains a cycle")?; - leaf_id = left; - self.with_slotted_page(leaf_id, |page, _| { - if page.header().page_type != PageType::Leaf || page.header().right != current { - return Err(invalid_data("asymmetric leaf sibling link")); - } - Ok(()) - })?; - first = false; - } - } - - pub fn len(&mut self) -> Result { - let result: io::Result = (|| { - let mut page_id = self.leftmost_leaf()?; - let mut visited = HashSet::new(); - record_page_visit(&mut visited, page_id, "right sibling chain contains a cycle")?; - let mut total = 0usize; - loop { - let next_page_id = self.header.next_page_id; - let (right, live) = self.with_page(page_id, |bytes, _| { - let page = SlottedPage::open(bytes, page_id, next_page_id)?; - if page.header().page_type != PageType::Leaf { - return Err(invalid_data("length scan expected a leaf page")); - } - let mut live = 0usize; - for index in 0..usize::from(page.header().cell_count) { - let cell = LeafCellRef::decode(page.cell(index)?, page_id, next_page_id)?; - if !matches!(cell.value, LeafValueRef::Tombstone) { - live = live.checked_add(1).ok_or_else(|| invalid_data("entry count overflow"))?; - } - } - Ok((page.header().right, live)) - })?; - total = total.checked_add(live).ok_or_else(|| invalid_data("entry count overflow"))?; - if right == 0 { - return Ok(total); - } - record_page_visit(&mut visited, right, "right sibling chain contains a cycle")?; - self.with_page(right, |bytes, _| { - let page = SlottedPage::open(bytes, right, next_page_id)?; - if page.header().page_type != PageType::Leaf || page.header().left != page_id { - return Err(invalid_data("asymmetric leaf sibling link")); - } - Ok(()) - })?; - page_id = right; - } - })(); - result.map_err(Into::into) - } - - pub fn is_empty(&mut self) -> Result { - let mut iterator = self.iter(); - match iterator.next() { - None => Ok(true), - Some(Ok(_)) => Ok(false), - Some(Err(err)) => Err(BPlusTreeError::Io(err)), - } - } - - pub fn iter(&mut self) -> BPlusTreeDiskIterator<'_, K, V> { BPlusTreeDiskIterator::new(self) } - - pub fn disk_iter(self) -> BPlusTreeDiskIteratorOwned { BPlusTreeDiskIteratorOwned::new(self) } - - pub fn range_iter(&mut self, start: Bound<&K>, end: Bound<&K>) -> BPlusTreeRangeIterator<'_, K, V> - where - K: Clone, - { - let start = start.map(Clone::clone); - let end = end.map(Clone::clone); - BPlusTreeRangeIterator { iterator: BPlusTreeDiskIterator::from_bound(self, start.clone()), start, end } - } - - pub fn range_page( - &mut self, - start: Bound<&K>, - end: Bound<&K>, - offset: usize, - limit: usize, - ) -> Result<(Vec<(K, V)>, bool), BPlusTreeError> - where - K: Clone, - { - let mut iterator = self.range_iter(start, end); - for _ in 0..offset { - if let Some(entry) = iterator.next() { - let _ = entry.map_err(BPlusTreeError::Io)?; - } else { - return Ok((Vec::new(), false)); - } - } - let mut result = Vec::new(); - while result.len() < limit { - match iterator.next() { - Some(Ok(entry)) => { - result - .try_reserve(1) - .map_err(|error| BPlusTreeError::Io(io::Error::new(io::ErrorKind::OutOfMemory, error)))?; - result.push(entry); - } - Some(Err(err)) => return Err(BPlusTreeError::Io(err)), - None => return Ok((result, false)), - } - } - let has_more = match iterator.next() { - Some(Ok(_)) => true, - Some(Err(err)) => return Err(BPlusTreeError::Io(err)), - None => false, - }; - Ok((result, has_more)) - } -} - -struct CursorState { - leaf_page_id: u64, - slot_index: usize, - cell_ranges: Vec>, - right_sibling: u64, - expected_left_sibling: Option, - page_loaded: bool, - start: Bound, - initialized: bool, - finished: bool, - visited_leaves: HashSet, - pending: Option<(K, V)>, -} - -impl CursorState { - fn new(start: Bound) -> Self { - Self { - leaf_page_id: 0, - slot_index: 0, - cell_ranges: Vec::new(), - right_sibling: 0, - expected_left_sibling: None, - page_loaded: false, - start, - initialized: false, - finished: false, - visited_leaves: HashSet::new(), - pending: None, - } - } -} - -fn load_cursor_page(query: &mut BPlusTreeQuery, state: &mut CursorState) -> io::Result<()> { - let page_id = state.leaf_page_id; - let next_page_id = query.header.next_page_id; - let expected_left = state.expected_left_sibling; - let mut cell_ranges = std::mem::take(&mut state.cell_ranges); - let right_sibling = query.with_page(page_id, |bytes, _| { - let page = SlottedPage::open(bytes, page_id, next_page_id)?; - if page.header().page_type != PageType::Leaf { - return Err(invalid_data("iterator sibling is not a leaf")); - } - if expected_left.is_some_and(|left| page.header().left != left) { - return Err(invalid_data("asymmetric leaf sibling link")); - } - let count = usize::from(page.header().cell_count); - cell_ranges.clear(); - cell_ranges.try_reserve(count).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - for index in 0..count { - cell_ranges.push(page.cell_range(index)?); - } - Ok(page.header().right) - })?; - state.cell_ranges = cell_ranges; - state.right_sibling = right_sibling; - state.page_loaded = true; - Ok(()) -} - -fn record_page_visit(visited: &mut HashSet, page_id: u64, cycle_error: &'static str) -> io::Result<()> { - visited.try_reserve(1).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; - if !visited.insert(page_id) { - return Err(invalid_data(cycle_error)); - } - Ok(()) -} - -fn cursor_next(query: &mut BPlusTreeQuery, state: &mut CursorState) -> Option> -where - K: Ord + for<'de> Deserialize<'de>, - V: for<'de> Deserialize<'de>, -{ - if let Some(entry) = state.pending.take() { - return Some(Ok(entry)); - } - if state.finished { - return None; - } - let mut entry_error = false; - let result = (|| { - if !state.initialized { - state.leaf_page_id = match &state.start { - Bound::Included(key) | Bound::Excluded(key) => query.locate_leaf(key)?, - Bound::Unbounded => query.leftmost_leaf()?, - }; - record_page_visit(&mut state.visited_leaves, state.leaf_page_id, "right sibling chain contains a cycle")?; - if let Bound::Included(key) | Bound::Excluded(key) = &state.start { - let page_id = state.leaf_page_id; - let next_page_id = query.header.next_page_id; - state.slot_index = query.with_page(page_id, |bytes, _| { - let page = SlottedPage::open(bytes, page_id, next_page_id)?; - let index = match search_leaf(&page, key)? { - Ok(index) if matches!(state.start, Bound::Excluded(_)) => index + 1, - Ok(index) | Err(index) => index, - }; - Ok(index) - })?; - } - state.initialized = true; - } - loop { - if !state.page_loaded { - load_cursor_page(query, state)?; - } - let page_id = state.leaf_page_id; - while state.slot_index < state.cell_ranges.len() { - let index = state.slot_index; - state.slot_index += 1; - match query.decode_entry_range(page_id, state.cell_ranges[index].clone()) { - Ok(Some(entry)) => return Ok(Some(entry)), - Ok(None) => {} - Err(error) => { - entry_error = true; - return Err(error); - } - } - } - let right = state.right_sibling; - if right == 0 { - return Ok(None); - } - record_page_visit(&mut state.visited_leaves, right, "right sibling chain contains a cycle")?; - state.expected_left_sibling = Some(page_id); - state.leaf_page_id = right; - state.slot_index = 0; - state.page_loaded = false; - state.cell_ranges.clear(); - } - })(); - match result { - Ok(Some(entry)) => Some(Ok(entry)), - Ok(None) => { - state.finished = true; - None - } - Err(err) => { - state.finished = !entry_error; - Some(Err(err)) - } - } -} - pub struct BPlusTreeDiskIterator<'a, K, V> { query: &'a mut BPlusTreeQuery, state: CursorState, } -impl<'a, K, V> BPlusTreeDiskIterator<'a, K, V> { - fn new(query: &'a mut BPlusTreeQuery) -> Self { Self { query, state: CursorState::new(Bound::Unbounded) } } - - fn from_bound(query: &'a mut BPlusTreeQuery, start: Bound) -> Self { - Self { query, state: CursorState::new(start) } - } -} - -impl BPlusTreeDiskIterator<'_, K, V> -where - K: Ord + for<'de> Deserialize<'de>, - V: for<'de> Deserialize<'de>, -{ - pub fn try_is_empty(&mut self) -> io::Result { - match self.next() { - None => Ok(true), - Some(Ok(entry)) => { - self.state.pending = Some(entry); - Ok(false) - } - Some(Err(err)) => Err(err), - } - } -} - -impl Iterator for BPlusTreeDiskIterator<'_, K, V> -where - K: Ord + for<'de> Deserialize<'de>, - V: for<'de> Deserialize<'de>, -{ - type Item = io::Result<(K, V)>; - - fn next(&mut self) -> Option { cursor_next(self.query, &mut self.state) } -} - pub struct BPlusTreeDiskIteratorOwned { query: BPlusTreeQuery, state: CursorState, } -impl BPlusTreeDiskIteratorOwned { - fn new(query: BPlusTreeQuery) -> Self { Self { query, state: CursorState::new(Bound::Unbounded) } } -} - -impl BPlusTreeDiskIteratorOwned -where - K: Ord + for<'de> Deserialize<'de>, - V: for<'de> Deserialize<'de>, -{ - pub fn try_is_empty(&mut self) -> io::Result { - match self.next() { - None => Ok(true), - Some(Ok(entry)) => { - self.state.pending = Some(entry); - Ok(false) - } - Some(Err(err)) => Err(err), - } - } -} - -impl Iterator for BPlusTreeDiskIteratorOwned -where - K: Ord + for<'de> Deserialize<'de>, - V: for<'de> Deserialize<'de>, -{ - type Item = io::Result<(K, V)>; - - fn next(&mut self) -> Option { cursor_next(&mut self.query, &mut self.state) } -} - pub struct BPlusTreeRangeIterator<'a, K, V> { iterator: BPlusTreeDiskIterator<'a, K, V>, start: Bound, end: Bound, } -fn within_start(key: &K, bound: &Bound) -> bool { - match bound { - Bound::Included(start) => key >= start, - Bound::Excluded(start) => key > start, - Bound::Unbounded => true, - } -} - -fn past_end(key: &K, bound: &Bound) -> bool { - match bound { - Bound::Included(end) => key > end, - Bound::Excluded(end) => key >= end, - Bound::Unbounded => false, - } -} - -impl Iterator for BPlusTreeRangeIterator<'_, K, V> -where - K: Ord + for<'de> Deserialize<'de>, - V: for<'de> Deserialize<'de>, -{ - type Item = io::Result<(K, V)>; - - fn next(&mut self) -> Option { - loop { - let entry = self.iterator.next()?; - match entry { - Ok((key, _value)) if past_end(&key, &self.end) => { - self.iterator.state.finished = true; - return None; - } - Ok((key, value)) if within_start(&key, &self.start) => return Some(Ok((key, value))), - Ok(_) => {} - Err(err) => return Some(Err(err)), - } - } - } -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) struct VerificationReport { pub(crate) live_entries: u64, @@ -3294,2638 +109,48 @@ pub(crate) struct VerificationReport { pub(crate) free_pages: u64, } -struct VerifyLeaf { - page_id: u64, - left: u64, - right: u64, - minimum: Option, - maximum: Option, -} +#[cfg(test)] +mod tests; -enum Visit { - Enter(u64, Option, Option), - Exit(u64), -} - -enum VerifyValue { - Inline, - Overflow(Compression, u32, u32, u64, u32), - Tombstone, -} - -fn finish_verified_page( - page_id: u64, - active: &mut HashSet, - internal_children: &mut HashMap)>>, - page_minimum: &mut HashMap>, -) -> io::Result<()> { - if !active.remove(&page_id) { - return Err(invalid_data("tree verifier active set is inconsistent")); - } - let Some(children) = internal_children.remove(&page_id) else { return Ok(()) }; - for (child, separator) in children.iter().skip(1) { - let actual = page_minimum - .get(child) - .and_then(Option::as_ref) - .ok_or_else(|| invalid_data("internal child has no minimum key"))?; - if separator.as_ref() != Some(actual) { - return Err(invalid_data("internal separator is not the right child minimum")); - } - } - let minimum = children - .first() - .and_then(|(child, _)| page_minimum.get(child)) - .cloned() - .ok_or_else(|| invalid_data("internal leftmost child has no verified minimum"))?; - page_minimum.insert(page_id, minimum); - Ok(()) -} - -fn verify_leaf_page( - query: &mut BPlusTreeQuery, - page_id: u64, - lower: Option<&K>, - upper: Option<&K>, - overflow_pages: &mut HashSet, -) -> io::Result<(VerifyLeaf, u64)> -where - K: Ord + for<'de> Deserialize<'de> + Clone, -{ - let next_page_id = query.header.next_page_id; - let (left, right, count) = query.with_page(page_id, |bytes, _| { - let page = SlottedPage::open(bytes, page_id, next_page_id)?; - Ok((page.header().left, page.header().right, usize::from(page.header().cell_count))) - })?; - let mut minimum = None; - let mut maximum = None; - let mut live_entries = 0u64; - for index in 0..count { - let (key, value) = query.with_page(page_id, |bytes, _| { - let page = SlottedPage::open(bytes, page_id, next_page_id)?; - let cell = LeafCellRef::decode(page.cell(index)?, page_id, next_page_id)?; - let value = match cell.value { - LeafValueRef::Inline { .. } => VerifyValue::Inline, - LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } => { - VerifyValue::Overflow(compression, logical_len, stored_len, head, crc32) - } - LeafValueRef::Tombstone => VerifyValue::Tombstone, - }; - Ok((binary_deserialize::(cell.key_bytes)?, value)) - })?; - if maximum.as_ref().is_some_and(|previous| previous >= &key) { - return Err(invalid_data("leaf keys are not strictly ordered")); - } - if lower.is_some_and(|bound| &key < bound) || upper.is_some_and(|bound| &key >= bound) { - return Err(invalid_data("leaf key is outside parent separator range")); - } - minimum.get_or_insert_with(|| key.clone()); - maximum = Some(key); - match value { - VerifyValue::Tombstone => continue, - VerifyValue::Inline => {} - VerifyValue::Overflow(compression, logical_len, stored_len, head, crc32) => query.assemble_overflow_chain( - compression, - logical_len, - stored_len, - head, - crc32, - Some(overflow_pages), - )?, - } - live_entries = live_entries.checked_add(1).ok_or_else(|| invalid_data("live entry count overflow"))?; - } - Ok((VerifyLeaf { page_id, left, right, minimum, maximum }, live_entries)) -} - -fn verify_internal_page(query: &mut BPlusTreeQuery, page_id: u64) -> io::Result)>> -where - K: Ord + for<'de> Deserialize<'de> + Clone, -{ - let next_page_id = query.header.next_page_id; - query.with_page(page_id, |bytes, _| { - let page = SlottedPage::open(bytes, page_id, next_page_id)?; - let mut children = Vec::with_capacity(usize::from(page.header().cell_count) + 1); - children.push((InternalPreamble::decode(bytes, page_id, next_page_id)?.leftmost_child, None)); - let mut previous = None; - for cell in page.cells() { - let cell = InternalCellRef::decode(cell?, page_id, next_page_id)?; - let key = binary_deserialize::(cell.key_bytes)?; - if previous.as_ref().is_some_and(|prior| prior >= &key) { - return Err(invalid_data("internal separators are not strictly ordered")); - } - previous = Some(key.clone()); - children.push((cell.right_child, Some(key))); - } - Ok(children) - }) -} - -fn push_child_visits( - stack: &mut Vec>, - children: &[(u64, Option)], - lower: Option<&K>, - upper: Option<&K>, -) -> io::Result<()> { - for index in (0..children.len()).rev() { - let child_lower = - if index == 0 { lower.cloned() } else { children.get(index).and_then(|(_, key)| key.clone()) }; - let child_upper = children.get(index + 1).and_then(|(_, key)| key.clone()).or_else(|| upper.cloned()); - let child = children - .get(index) - .map(|(child, _)| *child) - .ok_or_else(|| invalid_data("internal child index is invalid"))?; - stack.push(Visit::Enter(child, child_lower, child_upper)); - } - Ok(()) -} - -fn verify_leaf_links(leaves: &[VerifyLeaf]) -> io::Result<()> { - for (index, leaf) in leaves.iter().enumerate() { - let expected_left = index.checked_sub(1).map_or(0, |previous| leaves[previous].page_id); - let expected_right = leaves.get(index + 1).map_or(0, |next| next.page_id); - if leaf.left != expected_left || leaf.right != expected_right { - return Err(invalid_data("leaf sibling links do not match tree order")); - } - if index > 0 - && leaves[index - 1] - .maximum - .as_ref() - .zip(leaf.minimum.as_ref()) - .is_some_and(|(previous, current)| previous >= current) - { - return Err(invalid_data("keys are inverted across leaf siblings")); - } - } - Ok(()) -} - -fn verify_free_pages( - query: &mut BPlusTreeQuery, - tree_pages: &HashSet, - overflow_pages: &HashSet, -) -> io::Result> { - let mut free_pages = HashSet::new(); - let mut free = query.header.free_page_head; - while free != 0 { - record_page_visit(&mut free_pages, free, "free list contains a duplicate or cycle")?; - if tree_pages.contains(&free) || overflow_pages.contains(&free) { - return Err(invalid_data("page is reachable from both live data and free list")); - } - let next_page_id = query.header.next_page_id; - free = query.with_page(free, |bytes, _| { - let page = SlottedPage::open(bytes, free, next_page_id)?; - if page.header().page_type != PageType::Free { - return Err(invalid_data("free list references a non-free page")); - } - Ok(page.header().right) - })?; - } - Ok(free_pages) -} - -fn verify_page_ownership( - next_page_id: u64, - tree_pages: &HashSet, - overflow_pages: &HashSet, - free_pages: &HashSet, -) -> io::Result<()> { - for page_id in 1..next_page_id { - let memberships = u8::from(tree_pages.contains(&page_id)) - + u8::from(overflow_pages.contains(&page_id)) - + u8::from(free_pages.contains(&page_id)); - if memberships != 1 { - return Err(invalid_data("database contains an orphan or multiply-owned page")); - } - } - Ok(()) -} - -pub(crate) fn verify_full(query: &mut BPlusTreeQuery) -> io::Result -where - K: Ord + for<'de> Deserialize<'de> + Clone, -{ - let mut tree_pages = HashSet::new(); - let mut active = HashSet::new(); - let mut overflow_pages = HashSet::new(); - let mut leaves = Vec::>::new(); - let mut page_minimum = HashMap::>::new(); - let mut internal_children = HashMap::)>>::new(); - let mut live_entries = 0u64; - let mut stack = vec![Visit::Enter(query.header.root_page_id, None, None)]; - - while let Some(visit) = stack.pop() { - let (page_id, lower, upper) = match visit { - Visit::Exit(page_id) => { - finish_verified_page(page_id, &mut active, &mut internal_children, &mut page_minimum)?; - continue; - } - Visit::Enter(page_id, lower, upper) => (page_id, lower, upper), - }; - if active.contains(&page_id) { - return Err(invalid_data("tree child graph contains a cycle")); - } - if !tree_pages.insert(page_id) { - return Err(invalid_data("tree page has multiple parents")); - } - active.insert(page_id); - stack.push(Visit::Exit(page_id)); - let next_page_id = query.header.next_page_id; - let page_type = query.with_page(page_id, |bytes, _| { - SlottedPage::open(bytes, page_id, next_page_id).map(|page| page.header().page_type) - })?; - match page_type { - PageType::Leaf => { - let (leaf, page_live_entries) = - verify_leaf_page(query, page_id, lower.as_ref(), upper.as_ref(), &mut overflow_pages)?; - live_entries = live_entries - .checked_add(page_live_entries) - .ok_or_else(|| invalid_data("live entry count overflow"))?; - page_minimum.insert(page_id, leaf.minimum.clone()); - leaves.push(leaf); - } - PageType::Internal => { - let children = verify_internal_page(query, page_id)?; - push_child_visits(&mut stack, &children, lower.as_ref(), upper.as_ref())?; - internal_children.insert(page_id, children); - } - PageType::Overflow | PageType::Free => return Err(invalid_data("tree child has the wrong page type")), - } - } - - verify_leaf_links(&leaves)?; - let free_pages = verify_free_pages(query, &tree_pages, &overflow_pages)?; - verify_page_ownership(query.header.next_page_id, &tree_pages, &overflow_pages, &free_pages)?; - Ok(VerificationReport { - live_entries, - tree_pages: u64::try_from(tree_pages.len()).map_err(|_| invalid_data("tree page count exceeds u64"))?, - overflow_pages: u64::try_from(overflow_pages.len()) - .map_err(|_| invalid_data("overflow page count exceeds u64"))?, - free_pages: u64::try_from(free_pages.len()).map_err(|_| invalid_data("free page count exceeds u64"))?, - }) -} +mod bulk_store; +mod error; +mod iterators; +mod memory; +mod mutation; +mod query; +mod serial_writer; +mod transaction; +mod update; +mod verify; #[cfg(test)] -mod tests { - use super::*; - use crate::{ - codec::binary_serialize, - v3::{ - format::{ - encode_inline_leaf_cell, encode_internal_cell, encode_overflow_leaf_cell, encode_tombstone_leaf_cell, - encode_value, Compression, PageHeader, PageType, OVERFLOW_PAYLOAD_LEN, PAGE_HEADER_LEN, PAGE_SIZE, - }, - page::{encode_free_page, encode_overflow_page, page_open_count, reset_page_open_count, SlottedPage}, - }, - }; - use std::{ - fs, io, - path::{Path, PathBuf}, - process::Command, - sync::mpsc::{self, Receiver, RecvTimeoutError}, - thread::{self, JoinHandle}, - time::Duration, - }; - - const PAGE_ID: u64 = 7; - const NEXT_PAGE_ID: u64 = 20; - - fn invalid_data(result: io::Result) -> io::Result<()> { - match result { - Err(err) if err.kind() == io::ErrorKind::InvalidData => Ok(()), - Err(err) => Err(io::Error::other(format!("expected InvalidData, got {err}"))), - Ok(_) => Err(io::Error::other("expected InvalidData")), - } - } - - fn invalid_input(result: io::Result) -> io::Result<()> { - match result { - Err(err) if err.kind() == io::ErrorKind::InvalidInput => Ok(()), - Err(err) => Err(io::Error::other(format!("expected InvalidInput, got {err}"))), - Ok(_) => Err(io::Error::other("expected InvalidInput")), - } - } - - fn database_header(path: &Path) -> io::Result { - let bytes = fs::read(path)?; - DatabaseHeader::decode( - bytes - .get(..PAGE_SIZE) - .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "database header is truncated"))?, - ) - } - - #[test] - fn point_query_validates_a_single_leaf_once() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("point-query-page-opens.db"); - let mut tree = BPlusTree::new(); - tree.insert(7u32, String::from("value")); - tree.store(&path)?; - - let mut query = BPlusTreeQuery::::try_new(&path)?; - reset_page_open_count(); - assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(String::from("value"))); - assert_eq!(page_open_count(), 1); - Ok(()) - } - - #[test] - fn repeated_point_query_reuses_snapshot_page_validation_and_internal_keys() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("repeated-point-query.db"); - let mut tree = BPlusTree::new(); - for key in 0..2_000u32 { - tree.insert(key, format!("value-{key:04}")); - } - tree.store(&path)?; - - let mut query = BPlusTreeQuery::::try_new(&path)?; - reset_page_open_count(); - reset_internal_key_decode_count(); - assert_eq!(query.query(&1_337).map_err(BPlusTreeError::to_io)?, Some(String::from("value-1337"))); - let first_page_opens = page_open_count(); - let first_internal_decodes = internal_key_decode_count(); - if first_page_opens < 2 || first_internal_decodes == 0 { - return Err(io::Error::other("test fixture must contain internal pages")); - } - - assert_eq!(query.query(&1_337).map_err(BPlusTreeError::to_io)?, Some(String::from("value-1337"))); - assert_eq!(page_open_count(), first_page_opens); - assert_eq!(internal_key_decode_count(), first_internal_decodes); - Ok(()) - } - - #[test] - fn cloned_query_reuses_snapshot_page_validation_and_internal_keys() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("cloned-point-query.db"); - let mut tree = BPlusTree::new(); - for key in 0..2_000u32 { - tree.insert(key, format!("value-{key:04}")); - } - tree.store(&path)?; - - let mut query = BPlusTreeQuery::::try_new(&path)?; - reset_page_open_count(); - reset_internal_key_decode_count(); - assert_eq!(query.query(&1_337).map_err(BPlusTreeError::to_io)?, Some(String::from("value-1337"))); - let first_page_opens = page_open_count(); - let first_internal_decodes = internal_key_decode_count(); - if first_page_opens < 2 || first_internal_decodes == 0 { - return Err(io::Error::other("test fixture must contain internal pages")); - } - - let mut cloned = query.try_clone()?; - assert_eq!(cloned.query(&1_337).map_err(BPlusTreeError::to_io)?, Some(String::from("value-1337"))); - assert_eq!(page_open_count(), first_page_opens); - assert_eq!(internal_key_decode_count(), first_internal_decodes); - Ok(()) - } - - #[test] - fn repeated_query_le_reuses_snapshot_page_validation_and_internal_keys() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("repeated-query-le.db"); - let mut tree = BPlusTree::new(); - for key in 0..2_000u32 { - tree.insert(key, format!("value-{key:04}")); - } - tree.store(&path)?; - - let mut query = BPlusTreeQuery::::try_new(&path)?; - reset_page_open_count(); - reset_internal_key_decode_count(); - assert_eq!(query.query_le(&1_337).map_err(BPlusTreeError::to_io)?, Some(String::from("value-1337"))); - let first_page_opens = page_open_count(); - let first_internal_decodes = internal_key_decode_count(); - if first_page_opens < 2 || first_internal_decodes == 0 { - return Err(io::Error::other("test fixture must contain internal pages")); - } - - assert_eq!(query.query_le(&1_337).map_err(BPlusTreeError::to_io)?, Some(String::from("value-1337"))); - assert_eq!(page_open_count(), first_page_opens); - assert_eq!(internal_key_decode_count(), first_internal_decodes); - Ok(()) - } - - #[test] - fn locator_collection_validates_each_leaf_once() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("locator-page-opens.db"); - let mut tree = BPlusTree::new(); - for key in 0..3u32 { - tree.insert(key, format!("value-{key}")); - } - tree.store(&path)?; - - let mut query = BPlusTreeQuery::::try_new(&path)?; - reset_page_open_count(); - assert_eq!(query.collect_with_locators()?.len(), 3); - assert_eq!(page_open_count(), 1); - Ok(()) - } - - #[test] - fn prepared_upsert_batch_is_key_sorted_and_stable() -> io::Result<()> { - let keys = [3u32, 1, 2, 1]; - let values = ["three", "first", "two", "last"].map(String::from); - let items = keys.iter().zip(&values).collect::>(); - - let prepared = BPlusTreeUpdate::::prepare_upsert_batch(&items)?; - let decoded = prepared - .into_iter() - .map(|(key, value)| binary_deserialize::(&value).map(|value| (key, value))) - .collect::>>()?; - assert_eq!( - decoded, - vec![ - (1, String::from("first")), - (1, String::from("last")), - (2, String::from("two")), - (3, String::from("three")), - ] - ); - Ok(()) - } - - #[test] - fn equal_size_inline_update_rewrites_without_appending_pages() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("equal-inline.db"); - let mut tree = BPlusTree::new(); - tree.insert(7u32, String::from("old")); - tree.store(&path)?; - let before = database_header(&path)?; - let before_len = fs::metadata(&path)?.len(); - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.update(&7, String::from("new")).map_err(BPlusTreeError::to_io)?; - drop(updater); - - let after = database_header(&path)?; - assert_eq!(after.generation, before.generation + 1); - assert_eq!(after.root_page_id, before.root_page_id); - assert_eq!(fs::metadata(&path)?.len(), before_len); - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(String::from("new"))); - let _ = verify_full(&mut query)?; - Ok(()) - } - - #[test] - fn smaller_inline_update_compacts_without_appending_pages() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("smaller-inline.db"); - let mut tree = BPlusTree::new(); - tree.insert(7u32, String::from("a much longer inline value")); - tree.store(&path)?; - let before = database_header(&path)?; - let before_len = fs::metadata(&path)?.len(); - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.update(&7, String::from("x")).map_err(BPlusTreeError::to_io)?; - - let after = database_header(&path)?; - assert_eq!(after.generation, before.generation + 1); - assert_eq!(after.root_page_id, before.root_page_id); - assert_eq!(fs::metadata(&path)?.len(), before_len); - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(String::from("x"))); - let _ = verify_full(&mut query)?; - Ok(()) - } - - #[test] - fn growing_inline_value_uses_page_local_compaction_before_split() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("growing-inline.db"); - let mut tree = BPlusTree::new(); - for key in 0..8u32 { - tree.insert(key, vec![u8::try_from(key).map_err(io::Error::other)?; 8]); - } - tree.store(&path)?; - let before = database_header(&path)?; - let before_len = fs::metadata(&path)?.len(); - let grown = vec![0x5a; 200]; - - let mut updater = BPlusTreeUpdate::>::try_new(&path)?; - updater.update(&4, grown.clone()).map_err(BPlusTreeError::to_io)?; - - let after = database_header(&path)?; - assert_eq!(after.generation, before.generation + 1); - assert_eq!(after.root_page_id, before.root_page_id); - assert_eq!(fs::metadata(&path)?.len(), before_len); - let mut query = BPlusTreeQuery::>::try_new(&path)?; - assert_eq!(query.query(&4).map_err(BPlusTreeError::to_io)?, Some(grown)); - let _ = verify_full(&mut query)?; - Ok(()) - } - - #[test] - fn medium_stored_value_remains_inline() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("medium-inline.db"); - let value = - random_value().get(..300).ok_or_else(|| io::Error::other("random test value is too short"))?.to_vec(); - let mut tree = BPlusTree::new(); - tree.insert(7u32, value.clone()); - - let report = tree.store_verified(&path)?; - - assert_eq!(report.overflow_pages, 0); - let mut query = BPlusTreeQuery::>::try_new(&path)?; - assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(value)); - Ok(()) - } - - fn overflow_head(path: &Path, key: u32) -> io::Result { - let mut query = BPlusTreeQuery::>::try_new(path)?; - let leaf = query.locate_leaf(&key)?; - let next_page_id = query.header.next_page_id; - query.with_page(leaf, |bytes, _| { - let page = SlottedPage::open(bytes, leaf, next_page_id)?; - let index = search_leaf(&page, &key)?.map_err(|_| io::Error::other("test key is missing"))?; - match LeafCellRef::decode(page.cell(index)?, leaf, next_page_id)?.value { - LeafValueRef::Overflow { head, .. } => Ok(head), - LeafValueRef::Inline { .. } | LeafValueRef::Tombstone => { - Err(io::Error::other("test value is not overflow-backed")) - } - } - }) - } - - #[test] - fn overflow_update_reuses_head_and_frees_unused_tail() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("overflow-reuse.db"); - let large = random_value(); - let smaller = large.get(..5_000).ok_or_else(|| io::Error::other("random test value is too short"))?.to_vec(); - let mut tree = BPlusTree::new(); - tree.insert(7u32, large); - tree.store(&path)?; - let before = database_header(&path)?; - let before_len = fs::metadata(&path)?.len(); - let before_head = overflow_head(&path, 7)?; - - let mut updater = BPlusTreeUpdate::>::try_new(&path)?; - updater.update(&7, smaller.clone()).map_err(BPlusTreeError::to_io)?; - - let after = database_header(&path)?; - assert_eq!(after.generation, before.generation + 1); - assert_eq!(fs::metadata(&path)?.len(), before_len); - assert_eq!(overflow_head(&path, 7)?, before_head); - assert_ne!(after.free_page_head, 0); - let mut query = BPlusTreeQuery::>::try_new(&path)?; - assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(smaller)); - let _ = verify_full(&mut query)?; - Ok(()) - } - - #[test] - fn inline_to_overflow_update_allocates_new_pages() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("overflow-allocation.db"); - let mut tree = BPlusTree::new(); - tree.insert(7u32, vec![7]); - tree.store(&path)?; - let before = database_header(&path)?; - let before_len = fs::metadata(&path)?.len(); - let large = random_value(); - - let mut updater = BPlusTreeUpdate::>::try_new(&path)?; - updater.update(&7, large.clone()).map_err(BPlusTreeError::to_io)?; - - let after = database_header(&path)?; - assert_eq!(after.generation, before.generation + 1); - assert!(fs::metadata(&path)?.len() > before_len); - assert_ne!(overflow_head(&path, 7)?, 0); - let mut query = BPlusTreeQuery::>::try_new(&path)?; - assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(large)); - let _ = verify_full(&mut query)?; - Ok(()) - } - - #[test] - fn new_key_is_inserted_into_existing_leaf_without_growth() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("leaf-insert.db"); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("one")); - tree.insert(3u32, String::from("three")); - tree.store(&path)?; - let before = database_header(&path)?; - let before_len = fs::metadata(&path)?.len(); - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.upsert(&2, &String::from("two"))?; - - let after = database_header(&path)?; - assert_eq!(after.generation, before.generation + 1); - assert_eq!(after.root_page_id, before.root_page_id); - assert_eq!(fs::metadata(&path)?.len(), before_len); - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!( - query.iter().collect::>>()?, - vec![(1, "one".into()), (2, "two".into()), (3, "three".into())] - ); - let _ = verify_full(&mut query)?; - Ok(()) - } - - #[test] - fn leaf_split_repairs_siblings_and_creates_a_new_root() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("leaf-split.db"); - let value: String = random_value() - .get(..200) - .ok_or_else(|| io::Error::other("random test value is too short"))? - .iter() - .map(|byte| char::from(33 + byte % 90)) - .collect(); - let mut tree = BPlusTree::new(); - for key in 0..17u32 { - tree.insert(key, value.clone()); - } - tree.store(&path)?; - let before = database_header(&path)?; - let before_len = fs::metadata(&path)?.len(); - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.upsert(&17, &value)?; - - let after = database_header(&path)?; - assert_eq!(after.generation, before.generation + 1); - assert_ne!(after.root_page_id, before.root_page_id); - assert_eq!(fs::metadata(&path)?.len(), before_len + 2 * u64::try_from(PAGE_SIZE).map_err(io::Error::other)?); - let mut query = BPlusTreeQuery::::try_new(&path)?; - let report = verify_full(&mut query)?; - assert_eq!(report.live_entries, 18); - assert_eq!(query.query(&17).map_err(BPlusTreeError::to_io)?, Some(value)); - Ok(()) - } - - #[test] - fn splitting_a_non_rightmost_leaf_repairs_the_former_neighbor() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("middle-leaf-split.db"); - let value = - random_value().get(..200).ok_or_else(|| io::Error::other("random test value is too short"))?.to_vec(); - let mut tree = BPlusTree::new(); - for key in (0..800u32).step_by(2) { - tree.insert(key, value.clone()); - } - tree.store(&path)?; - let before = database_header(&path)?; - let mut query = BPlusTreeQuery::>::try_new(&path)?; - let left_leaf = query.locate_leaf(&0)?; - let former_right = query.with_page(left_leaf, |bytes, _| { - Ok(SlottedPage::open(bytes, left_leaf, before.next_page_id)?.header().right) - })?; - if former_right == 0 { - return Err(io::Error::other("fixture did not create a right leaf neighbor")); - } - drop(query); - - let mut updater = BPlusTreeUpdate::>::try_new(&path)?; - updater.upsert(&1, &value)?; - - let after = database_header(&path)?; - assert_eq!(after.root_page_id, before.root_page_id); - assert!(after.next_page_id > before.next_page_id); - let mut query = BPlusTreeQuery::>::try_new(&path)?; - let new_right = query.with_page(left_leaf, |bytes, _| { - Ok(SlottedPage::open(bytes, left_leaf, after.next_page_id)?.header().right) - })?; - assert_ne!(new_right, former_right); - query.with_page(new_right, |bytes, _| { - let page = SlottedPage::open(bytes, new_right, after.next_page_id)?; - assert_eq!(page.header().left, left_leaf); - assert_eq!(page.header().right, former_right); - Ok(()) - })?; - query.with_page(former_right, |bytes, _| { - let page = SlottedPage::open(bytes, former_right, after.next_page_id)?; - assert_eq!(page.header().left, new_right); - Ok(()) - })?; - let _ = verify_full(&mut query)?; - Ok(()) - } - - fn long_split_key(index: u32, marker: char) -> String { format!("{index:04}{marker}{}", "k".repeat(1_880)) } - - #[test] - fn leaf_promotion_recursively_splits_a_full_internal_root() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("internal-split.db"); - let value = String::from("value"); - let mut tree = BPlusTree::new(); - for index in 0..6u32 { - tree.insert(long_split_key(index, '-'), value.clone()); - } - tree.store(&path)?; - let before = database_header(&path)?; - let before_root = fs::read(&path)?; - let range = page_byte_range(before.root_page_id, before_root.len())?; - let root_page = SlottedPage::open( - before_root.get(range).ok_or_else(|| io::Error::other("root page is missing"))?, - before.root_page_id, - before.next_page_id, - )?; - assert_eq!(root_page.header().page_type, PageType::Internal); - assert_eq!(root_page.header().cell_count, 2); - - let inserted_key = long_split_key(1, 'z'); - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.upsert(&inserted_key, &value)?; - - let after = database_header(&path)?; - assert_eq!(after.generation, before.generation + 1); - assert_ne!(after.root_page_id, before.root_page_id); - let mut query = BPlusTreeQuery::::try_new(&path)?; - let report = verify_full(&mut query)?; - assert_eq!(report.live_entries, 7); - assert_eq!(query.query(&inserted_key).map_err(BPlusTreeError::to_io)?, Some(value)); - Ok(()) - } - - #[test] - fn delete_writes_tombstone_and_reinsert_reuses_the_leaf() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("tombstone.db"); - let mut tree = BPlusTree::new(); - tree.insert(7u32, String::from("original")); - tree.store(&path)?; - let before = database_header(&path)?; - let before_len = fs::metadata(&path)?.len(); - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - assert!(updater.delete(&7)?); - assert_eq!(updater.query(&7).map_err(BPlusTreeError::to_io)?, None); - updater.upsert(&7, &String::from("restored"))?; - - let after = database_header(&path)?; - assert_eq!(after.generation, before.generation + 2); - assert_eq!(after.root_page_id, before.root_page_id); - assert_eq!(fs::metadata(&path)?.len(), before_len); - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(String::from("restored"))); - assert_eq!(verify_full(&mut query)?.live_entries, 1); - Ok(()) - } - - #[test] - fn freed_overflow_pages_are_reused_before_file_growth() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("free-reuse.db"); - let large = random_value(); - let mut tree = BPlusTree::new(); - tree.insert(1u32, large.clone()); - tree.store(&path)?; - let before = database_header(&path)?; - let before_len = fs::metadata(&path)?.len(); - - let mut updater = BPlusTreeUpdate::>::try_new(&path)?; - assert!(updater.delete(&1)?); - assert_ne!(database_header(&path)?.free_page_head, 0); - updater.upsert(&2, &large)?; - - let after = database_header(&path)?; - assert_eq!(after.generation, before.generation + 2); - assert_eq!(fs::metadata(&path)?.len(), before_len); - let mut query = BPlusTreeQuery::>::try_new(&path)?; - assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, None); - assert_eq!(query.query(&2).map_err(BPlusTreeError::to_io)?, Some(large)); - assert_eq!(verify_full(&mut query)?.live_entries, 1); - Ok(()) - } - - #[test] - fn metadata_update_is_a_single_header_transaction() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("metadata.db"); - let mut tree = BPlusTree::::new(); - tree.insert(1, String::from("one")); - tree.store(&path)?; - let before = database_header(&path)?; - let before_len = fs::metadata(&path)?.len(); - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.set_metadata(&BPlusTreeMetadata::TargetIdMapping(42))?; - - let after = database_header(&path)?; - assert_eq!(after.generation, before.generation + 1); - assert_eq!(after.metadata, BPlusTreeMetadata::TargetIdMapping(42)); - assert_eq!(fs::metadata(&path)?.len(), before_len); - assert_eq!(updater.get_metadata()?, BPlusTreeMetadata::TargetIdMapping(42)); - Ok(()) - } - - /// The Xtream cluster import commits after every batch so the transaction's - /// dirty-page map stays bounded. That reopens the write transaction mid-import, - /// so every batch must survive and stay readable — including values large enough - /// to spill into overflow chains, which is what dominated the dirty-page map. - #[test] - fn committing_between_batches_keeps_every_entry_readable() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("batched-commits.db"); - BPlusTree::::new().store(&path)?; - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.set_flush_policy(FlushPolicy::Batch); - for batch in 0..4u32 { - let keys: Vec = (0..8).map(|index| batch * 8 + index).collect(); - let values: Vec = keys.iter().map(|key| format!("{key}").repeat(600)).collect(); - let items = keys.iter().zip(&values).collect::>(); - let prepared = BPlusTreeUpdate::::prepare_upsert_batch(&items)?; - updater.upsert_batch_encoded(prepared)?; - updater.commit()?; - assert!(updater.active.is_none(), "commit must release the transaction after batch {batch}"); - } - - let mut query = BPlusTreeQuery::::try_new(&path)?; - for key in 0..32u32 { - assert_eq!( - query.query(&key).map_err(BPlusTreeError::to_io)?, - Some(format!("{key}").repeat(600)), - "key {key} is missing after a mid-import commit" - ); - } - assert!(!wal_path(&path).try_exists()?); - Ok(()) - } - - /// Throughput benchmark for `BPlusTree::store` across three workload sizes. - #[test] - #[ignore = "benchmark; run explicitly with --ignored --nocapture"] - fn bench_store_throughput() -> io::Result<()> { - let noise = |seed: u32, len: usize| -> Vec { - let mut state = seed.wrapping_mul(2_654_435_761).wrapping_add(1); - (0..len) - .map(|_| { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - u8::try_from(state >> 24).unwrap_or(0) - }) - .collect() - }; - for (label, count, size) in [("klein", 2_000u32, 800usize), ("mittel", 20_000, 2_000), ("gross", 60_000, 2_000)] - { - let dir = tempfile::tempdir()?; - let path = dir.path().join("bench.db"); - let mut tree = BPlusTree::>::new(); - for key in 0..count { - tree.insert(key, noise(key, size)); - } - let start = std::time::Instant::now(); - tree.store(&path)?; - let elapsed = start.elapsed(); - println!( - "BENCH {label:6} entries={count:6} bytes={size:5} -> {:>8.1} ms file={} MiB", - elapsed.as_secs_f64() * 1000.0, - fs::metadata(&path)?.len() / (1024 * 1024) - ); - } - Ok(()) - } - - /// Point-query benchmark for the persisted read path. - /// - /// `bench_store_throughput` only covers writing. The request hot path is - /// `BPlusTreeQuery::query_zero_copy` against an mmapped database, so the - /// extraction gate needs a read-side number too. Everything here is derived - /// from fixed seeds so two runs on the same host compare like for like: the - /// values come from the same LCG the store benchmark uses, and the lookup - /// order is a fixed-stride walk that defeats sequential prefetching without - /// needing a random number generator. - #[test] - #[ignore = "benchmark; run explicitly with --ignored --nocapture"] - fn bench_point_query_throughput() -> io::Result<()> { - // Coprime with every entry count below, so the walk visits each key exactly - // once and never degenerates into a scan. - const STRIDE: u32 = 7_919; - const LOOKUPS: u32 = 200_000; - - let noise = |seed: u32, len: usize| -> Vec { - let mut state = seed.wrapping_mul(2_654_435_761).wrapping_add(1); - (0..len) - .map(|_| { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - u8::try_from(state >> 24).unwrap_or(0) - }) - .collect() - }; - - for (label, count, size) in [("klein", 2_000u32, 800usize), ("mittel", 20_000, 2_000), ("gross", 60_000, 2_000)] - { - let dir = tempfile::tempdir()?; - let path = dir.path().join("bench.db"); - let mut tree = BPlusTree::>::new(); - for key in 0..count { - tree.insert(key, noise(key, size)); - } - tree.store(&path)?; - - let mut query = BPlusTreeQuery::>::try_new(&path)?; - // Fault the mapping in first; this benchmark measures lookup, not page-in. - let mut checksum = 0u64; - for key in 0..count { - checksum += - query.query_zero_copy(&key).map_err(BPlusTreeError::to_io)?.map_or(0, |value| u64::from(value[0])); - } - - let start = std::time::Instant::now(); - let mut key = 0u32; - for _ in 0..LOOKUPS { - key = (key + STRIDE) % count; - checksum += - query.query_zero_copy(&key).map_err(BPlusTreeError::to_io)?.map_or(0, |value| u64::from(value[0])); - } - let elapsed = start.elapsed(); - - assert!(checksum > 0, "lookups must not be optimized away"); - println!( - "BENCH query {label:6} entries={count:6} bytes={size:5} lookups={LOOKUPS} -> {:>8.1} ms {:>9.0} lookups/s", - elapsed.as_secs_f64() * 1000.0, - f64::from(LOOKUPS) / elapsed.as_secs_f64() - ); - } - Ok(()) - } - - #[test] - fn streamed_store_writes_every_page_despite_out_of_order_overflow_chains() -> io::Result<()> { - // The streaming builder hands out a leaf's page id *before* the overflow chain it - // points at, but writes the leaf *after* those pages — so page ids do not reach the - // file in ascending order. Worse, this nests: leaf A is reserved, its chains are - // written, A is written, then leaf B repeats the whole dance at higher ids. - // - // The entry count is sized to produce many leaves, not one: overflow leaf cells are - // tiny (key plus pointers, ~34 bytes), so a few hundred entries would all land in a - // single leaf and never exercise the nesting. Values are incompressible on purpose — - // a repeated string would compress back under `MAX_INLINE_STORED_VALUE` and skip - // overflow entirely. Both properties are asserted below rather than assumed. - // - // The file-length check is the real guard: writing pages sequentially instead of - // positionally would leave the file short or the ids scrambled. - // Deterministic LCG — compresses badly, so values are forced into overflow chains. - let noise = |seed: u32, len: usize| -> Vec { - let mut state = seed.wrapping_mul(2_654_435_761).wrapping_add(1); - (0..len) - .map(|_| { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - u8::try_from(state >> 24).unwrap_or(0) - }) - .collect() - }; - - let dir = tempfile::tempdir()?; - let path = dir.path().join("streamed.db"); - let mut tree = BPlusTree::>::new(); - for key in 0..1_200u32 { - // Spans one, two and three overflow pages (OVERFLOW_PAYLOAD_LEN = 4056). - tree.insert(key, noise(key, 3000 + (key as usize % 3) * 4000)); - } - let report = tree.store_verified(&path)?; - assert!( - report.overflow_pages > 2_000, - "test must force multi-page overflow chains, got {} overflow pages", - report.overflow_pages - ); - assert!( - report.tree_pages > 8, - "test must produce many leaves so the reserve-then-write dance nests, got {} tree pages", - report.tree_pages - ); - - let header = BPlusTreeQuery::>::try_new(&path)?.header; - assert_eq!( - fs::metadata(&path)?.len(), - header.next_page_id * PAGE_SIZE as u64, - "file length must match the allocated page count exactly" - ); - - let mut query = BPlusTreeQuery::>::try_new(&path)?; - for key in 0..1_200u32 { - assert_eq!( - query.query(&key).map_err(BPlusTreeError::to_io)?, - Some(noise(key, 3000 + (key as usize % 3) * 4000)), - "key {key} did not survive the streamed store" - ); - } - Ok(()) - } - - /// Compaction is NOT optional cleanup on an insert-built tree: leaf splits leave - /// pages roughly half full, and `build_pages` repacks them to near-capacity. - /// - /// Note `free_page_head` stays 0 throughout — inserts never free a page — so it is - /// NOT a usable "nothing to compact" signal. Guarding compaction on an empty free - /// list would skip exactly the case that gains the most. - #[test] - fn compaction_repacks_an_insert_built_tree_despite_an_empty_free_list() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("repack.db"); - BPlusTree::::new().store(&path)?; - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.set_flush_policy(FlushPolicy::Batch); - for batch in 0..8u32 { - let keys: Vec = (0..250).map(|index| batch * 250 + index).collect(); - let values: Vec = keys.iter().map(|key| format!("{key:07}").repeat(90)).collect(); - let items = keys.iter().zip(&values).collect::>(); - updater.upsert_batch_encoded(BPlusTreeUpdate::::prepare_upsert_batch(&items)?)?; - updater.commit()?; - } - - let before = BPlusTreeQuery::::try_new(&path)?.header; - assert_eq!(before.free_page_head, 0, "inserts must not free pages"); - - updater.compact()?; - - let after = BPlusTreeQuery::::try_new(&path)?.header; - assert!( - after.next_page_id * 3 < before.next_page_id * 2, - "compaction must reclaim over a third of the pages, got {} -> {}", - before.next_page_id, - after.next_page_id - ); - - let mut query = BPlusTreeQuery::::try_new(&path)?; - for key in 0..2000u32 { - assert_eq!( - query.query(&key).map_err(BPlusTreeError::to_io)?, - Some(format!("{key:07}").repeat(90)), - "key {key} did not survive compaction" - ); - } - Ok(()) - } - - #[test] - fn identical_batch_metadata_is_a_true_noop() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("metadata-noop.db"); - let mut tree = BPlusTree::::new(); - tree.set_metadata(BPlusTreeMetadata::TargetIdMapping(42)); - tree.store(&path)?; - let before = fs::read(&path)?; - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.set_flush_policy(FlushPolicy::Batch); - updater.set_metadata(&BPlusTreeMetadata::TargetIdMapping(42))?; - assert!(updater.active.is_none()); - updater.commit()?; - - assert_eq!(fs::read(&path)?, before); - assert!(!wal_path(&path).try_exists()?); - Ok(()) - } - - #[test] - fn immediate_commit_clears_wal_then_invalidates_sorted_index() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("immediate.db"); - let index_path = crate::common::get_file_path_for_db_index(&path); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("old")); - tree.store(&path)?; - fs::write(&index_path, b"derived")?; - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.upsert(&1, &String::from("new"))?; - - assert!(!wal_path(&path).try_exists()?); - assert!(!wal_temporary_path(&path).try_exists()?); - assert!(!index_path.try_exists()?); - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, Some(String::from("new"))); - Ok(()) - } - - #[test] - fn database_with_index_extension_is_never_deleted_as_derived_data() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("tree.idx"); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("old")); - tree.store(&path)?; - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.upsert(&1, &String::from("new"))?; - - assert!(path.try_exists()?); - assert!(!wal_path(&path).try_exists()?); - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, Some(String::from("new"))); - let _ = verify_full(&mut query)?; - Ok(()) - } - - #[test] - fn missing_delete_and_empty_commit_are_true_noops() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("no-op.db"); - let index_path = crate::common::get_file_path_for_db_index(&path); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("one")); - tree.store(&path)?; - fs::write(&index_path, b"still-valid")?; - let before = fs::read(&path)?; - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - assert!(!updater.delete(&2)?); - updater.commit()?; - - assert_eq!(fs::read(&path)?, before); - assert!(index_path.try_exists()?); - assert!(!wal_path(&path).try_exists()?); - Ok(()) - } - - #[test] - fn serial_writer_commits_a_batch_and_shuts_down_cleanly() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("serial-writer.db"); - BPlusTree::::new().store(&path)?; - - let writer = BPlusTreeSerialWriter::new(&path, FlushPolicy::Batch)?; - let one = String::from("one"); - let two = String::from("two"); - assert_ne!(writer.upsert(&[(&1, &one), (&2, &two)])?, 0); - writer.shutdown()?; - - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.iter().collect::>>()?, vec![(1, one), (2, two)]); - Ok(()) - } - - #[test] - fn store_with_index_publishes_an_identity_bound_sorted_snapshot() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("indexed.db"); - let index_path = crate::common::get_file_path_for_db_index(&path); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("ccc")); - tree.insert(2u32, String::from("a")); - tree.insert(3u32, String::from("bb")); - - assert_ne!(tree.store_with_index(&path, String::len)?, 0); - - let query = BPlusTreeQuery::::try_new(&path)?; - let mut sorted = crate::sorted_index::v4::OwnedIterator::::open(query, &index_path)?; - assert_eq!( - sorted.by_ref().collect::>>()?, - vec![(2, String::from("a")), (3, String::from("bb")), (1, String::from("ccc"))] - ); - assert_eq!(sorted.remaining(), 0); - assert!(!fs::read_dir(dir.path())? - .any(|entry| entry.is_ok_and(|entry| entry.file_name().to_string_lossy().ends_with(".v3.tmp")))); - Ok(()) - } - - #[test] - fn compact_rebuilds_only_live_data_with_a_fresh_identity() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("compact.db"); - let index_path = crate::common::get_file_path_for_db_index(&path); - let mut tree = BPlusTree::new(); - for key in 0..100u32 { - tree.insert( - key, - if key % 10 == 0 { random_value() } else { vec![u8::try_from(key).map_err(io::Error::other)?; 32] }, - ); - } - tree.store(&path)?; - let original_id = database_header(&path)?.database_id; - - let mut updater = BPlusTreeUpdate::>::try_new(&path)?; - updater.set_flush_policy(FlushPolicy::Batch); - for key in (0..100u32).step_by(2) { - assert!(updater.delete(&key)?); - } - updater.commit()?; - let length_before = fs::metadata(&path)?.len(); - fs::write(&index_path, b"stale")?; - - updater.compact()?; - - let header = database_header(&path)?; - assert_ne!(header.database_id, original_id); - assert_eq!(header.generation, 1); - assert!(fs::metadata(&path)?.len() <= length_before); - assert!(!index_path.try_exists()?); - let mut query = BPlusTreeQuery::>::try_new(&path)?; - let entries = query.iter().collect::>>()?; - assert_eq!(entries.len(), 50); - assert!(entries.iter().all(|(key, _)| key % 2 == 1)); - let report = verify_full(&mut query)?; - assert_eq!(report.live_entries, 50); - assert_eq!(report.free_pages, 0); - Ok(()) - } - - #[test] - fn compact_read_failure_preserves_database_and_index() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("compact-corrupt.db"); - let index_path = crate::common::get_file_path_for_db_index(&path); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("one")); - tree.store(&path)?; - let updater = BPlusTreeUpdate::::try_new(&path)?; - - let header = database_header(&path)?; - let mut corrupted = fs::read(&path)?; - let offset = usize::try_from(header.root_page_id) - .map_err(io::Error::other)? - .checked_mul(PAGE_SIZE) - .and_then(|start| start.checked_add(PAGE_HEADER_LEN)) - .ok_or_else(|| io::Error::other("corruption offset overflow"))?; - *corrupted.get_mut(offset).ok_or_else(|| io::Error::other("corruption offset outside database"))? ^= 0xff; - fs::write(&path, &corrupted)?; - fs::write(&index_path, b"still-valid")?; - - let mut updater = updater; - assert!(updater.compact().is_err()); - assert_eq!(fs::read(&path)?, corrupted); - assert_eq!(fs::read(&index_path)?, b"still-valid"); - Ok(()) - } - - #[test] - fn batch_overlay_is_visible_to_updater_and_commits_once() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("batch.db"); - let value: String = random_value() - .get(..200) - .ok_or_else(|| io::Error::other("random test value is too short"))? - .iter() - .map(|byte| char::from(33 + byte % 90)) - .collect(); - let mut tree = BPlusTree::new(); - for key in 0..17u32 { - tree.insert(key, value.clone()); - } - tree.store(&path)?; - let before = database_header(&path)?; - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.set_flush_policy(FlushPolicy::Batch); - updater.upsert(&17, &value)?; - updater.upsert(&18, &value)?; - assert_eq!(updater.query(&18).map_err(BPlusTreeError::to_io)?, Some(value.clone())); - assert_eq!(database_header(&path)?.generation, before.generation); - - updater.commit()?; - - let after = database_header(&path)?; - assert_eq!(after.generation, before.generation + 1); - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.query(&17).map_err(BPlusTreeError::to_io)?, Some(value.clone())); - assert_eq!(query.query(&18).map_err(BPlusTreeError::to_io)?, Some(value)); - assert_eq!(verify_full(&mut query)?.live_entries, 19); - Ok(()) - } - - #[test] - fn dropped_uncommitted_batch_discards_overlay_without_wal() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("dropped-batch.db"); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("old")); - tree.store(&path)?; - let before = fs::read(&path)?; - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.set_flush_policy(FlushPolicy::Batch); - updater.upsert(&1, &String::from("uncommitted"))?; - assert_eq!(updater.query(&1).map_err(BPlusTreeError::to_io)?, Some(String::from("uncommitted"))); - drop(updater); - - assert_eq!(fs::read(&path)?, before); - assert!(!wal_path(&path).try_exists()?); - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, Some(String::from("old"))); - Ok(()) - } - - #[test] - fn failed_batch_mutation_discards_the_whole_overlay() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("poisoned-batch.db"); - let mut tree = BPlusTree::new(); - tree.insert(String::from("key"), String::from("old")); - tree.store(&path)?; - let before = fs::read(&path)?; - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.set_flush_policy(FlushPolicy::Batch); - updater.upsert(&String::from("key"), &String::from("staged"))?; - assert!(updater.upsert(&"x".repeat(2_100), &String::from("invalid")).is_err()); - updater.commit()?; - - assert_eq!(fs::read(&path)?, before); - assert!(!wal_path(&path).try_exists()?); - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.query(&String::from("key")).map_err(BPlusTreeError::to_io)?, Some(String::from("old"))); - Ok(()) - } - - #[test] - fn final_overlay_validation_rejects_unordered_leaf_keys_before_wal() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("unordered-overlay.db"); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("one")); - tree.insert(2u32, String::from("two")); - tree.store(&path)?; - let before = fs::read(&path)?; - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.set_flush_policy(FlushPolicy::Batch); - updater.upsert(&1, &String::from("staged"))?; - let active = updater.active.as_mut().ok_or_else(|| io::Error::other("test transaction is missing"))?; - let leaf_id = active.transaction.next_header.root_page_id; - let snapshot = active.transaction.page_copy(active.base.as_slice(), leaf_id)?; - let page = SlottedPage::open(snapshot.as_slice(), leaf_id, active.transaction.next_header.next_page_id)?; - let mut cells = page.cells().map(|cell| cell.map(<[u8]>::to_vec)).collect::>>()?; - let duplicate = cells.first().cloned().ok_or_else(|| io::Error::other("test leaf has no cells"))?; - *cells.get_mut(1).ok_or_else(|| io::Error::other("test leaf lacks a second cell"))? = duplicate; - let next_page_id = active.transaction.next_header.next_page_id; - let dirty = active.transaction.page_mut(active.base.as_slice(), leaf_id)?; - SlottedPage::open(dirty.as_mut_slice(), leaf_id, next_page_id)? - .rebuild_ordered(cells.iter().map(Vec::as_slice))?; - - let error = updater.commit().err().ok_or_else(|| io::Error::other("unordered overlay was committed"))?; - assert_eq!(error.kind(), io::ErrorKind::InvalidData); - assert_eq!(fs::read(&path)?, before); - assert!(!wal_path(&path).try_exists()?); - Ok(()) - } - - #[test] - fn final_overlay_validation_rejects_shared_overflow_before_wal() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("shared-overflow-overlay.db"); - let first_value = random_value(); - let mut second_value = first_value.clone(); - second_value.reverse(); - let mut tree = BPlusTree::new(); - tree.insert(1u32, first_value); - tree.insert(2u32, second_value); - tree.store(&path)?; - let before = fs::read(&path)?; - - let mut updater = BPlusTreeUpdate::>::try_new(&path)?; - updater.set_flush_policy(FlushPolicy::Batch); - updater.ensure_transaction()?; - let active = updater.active.as_mut().ok_or_else(|| io::Error::other("test transaction is missing"))?; - let leaf_id = active.transaction.next_header.root_page_id; - let next_page_id = active.transaction.next_header.next_page_id; - let snapshot = active.transaction.page_copy(active.base.as_slice(), leaf_id)?; - let page = SlottedPage::open(snapshot.as_slice(), leaf_id, next_page_id)?; - let first = LeafCellRef::decode(page.cell(0)?, leaf_id, next_page_id)?; - let second = LeafCellRef::decode(page.cell(1)?, leaf_id, next_page_id)?; - let LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } = first.value else { - return Err(io::Error::other("first test value is not overflow-backed")); - }; - let mut replacement = Vec::new(); - encode_overflow_leaf_cell( - second.key_bytes, - logical_len, - compression, - stored_len, - head, - crc32, - leaf_id, - next_page_id, - &mut replacement, - )?; - let dirty = active.transaction.page_mut(active.base.as_slice(), leaf_id)?; - SlottedPage::open(dirty.as_mut_slice(), leaf_id, next_page_id)?.replace_same_len(1, &replacement)?; - - let error = updater.commit().err().ok_or_else(|| io::Error::other("shared overflow was committed"))?; - assert_eq!(error.kind(), io::ErrorKind::InvalidData); - assert_eq!(fs::read(&path)?, before); - assert!(!wal_path(&path).try_exists()?); - Ok(()) - } - - #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] - struct ConditionalSerialize { - value: String, - fail: bool, - } - - impl Serialize for ConditionalSerialize { - fn serialize(&self, serializer: S) -> Result { - if self.fail { - return Err(serde::ser::Error::custom("injected serialization failure")); - } - self.value.serialize(serializer) - } - } - - impl<'de> Deserialize<'de> for ConditionalSerialize { - fn deserialize>(deserializer: D) -> Result { - String::deserialize(deserializer).map(|value| Self { value, fail: false }) - } - } - - #[test] - fn batch_serialization_error_discards_earlier_items() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("batch-serialization.db"); - let initial = ConditionalSerialize { value: String::from("initial"), fail: false }; - let mut tree = BPlusTree::new(); - tree.insert(0u32, initial); - tree.store(&path)?; - let before = fs::read(&path)?; - let first = ConditionalSerialize { value: String::from("first"), fail: false }; - let failing = ConditionalSerialize { value: String::from("second"), fail: true }; - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - assert!(updater.upsert_batch(&[(&1, &first), (&2, &failing)]).is_err()); - updater.commit()?; - - assert_eq!(fs::read(&path)?, before); - assert!(!wal_path(&path).try_exists()?); - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, None); - assert_eq!(query.query(&2).map_err(BPlusTreeError::to_io)?, None); - Ok(()) - } - - #[test] - fn direct_serialization_error_aborts_an_existing_batch() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("direct-serialization.db"); - let initial = ConditionalSerialize { value: String::from("initial"), fail: false }; - let mut tree = BPlusTree::new(); - tree.insert(0u32, initial); - tree.store(&path)?; - let before = fs::read(&path)?; - let staged = ConditionalSerialize { value: String::from("staged"), fail: false }; - let failing = ConditionalSerialize { value: String::from("failing"), fail: true }; - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.set_flush_policy(FlushPolicy::Batch); - updater.upsert(&1, &staged)?; - assert!(updater.upsert(&2, &failing).is_err()); - updater.commit()?; - - assert_eq!(fs::read(&path)?, before); - assert!(!wal_path(&path).try_exists()?); - Ok(()) - } - - #[test] - fn delete_key_serialization_error_aborts_an_existing_batch() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("delete-serialization.db"); - let initial = ConditionalSerialize { value: String::from("initial"), fail: false }; - let mut tree = BPlusTree::new(); - tree.insert(initial.clone(), String::from("old")); - tree.store(&path)?; - let before = fs::read(&path)?; - let staged = ConditionalSerialize { value: String::from("staged"), fail: false }; - let failing = ConditionalSerialize { value: String::from("failing"), fail: true }; - - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - updater.set_flush_policy(FlushPolicy::Batch); - updater.upsert(&staged, &String::from("new"))?; - assert!(updater.delete(&failing).is_err()); - updater.commit()?; - - assert_eq!(fs::read(&path)?, before); - assert!(!wal_path(&path).try_exists()?); - Ok(()) - } - - #[test] - fn idle_updater_refreshes_after_full_replacement() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("replacement-refresh.db"); - let mut original = BPlusTree::new(); - original.insert(1u32, String::from("original")); - original.store(&path)?; - let mut updater = BPlusTreeUpdate::::try_new(&path)?; - let original_id = updater.database_id; - - let mut replacement = BPlusTree::new(); - replacement.insert(2u32, String::from("replacement")); - replacement.store(&path)?; - let replacement_id = database_header(&path)?.database_id; - assert_ne!(replacement_id, original_id); - - updater.upsert(&3, &String::from("updated"))?; - - assert_eq!(updater.database_id, replacement_id); - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, None); - assert_eq!(query.query(&2).map_err(BPlusTreeError::to_io)?, Some(String::from("replacement"))); - assert_eq!(query.query(&3).map_err(BPlusTreeError::to_io)?, Some(String::from("updated"))); - let _ = verify_full(&mut query)?; - Ok(()) - } - - fn pending_path(database: &Path, suffix: &str) -> PathBuf { - let mut name = database.as_os_str().to_os_string(); - name.push(suffix); - PathBuf::from(name) - } - - fn spawn_replacement_writer(path: PathBuf) -> io::Result<(Receiver>, JoinHandle<()>)> { - let (started_sender, started_receiver) = mpsc::channel(); - let (result_sender, result_receiver) = mpsc::channel(); - let handle = thread::spawn(move || { - let mut replacement = BPlusTree::new(); - replacement.insert(2u32, String::from("replacement")); - let _ = started_sender.send(()); - let _ = result_sender.send(replacement.store(&path)); - }); - started_receiver - .recv_timeout(Duration::from_secs(5)) - .map_err(|error| io::Error::other(format!("replacement writer did not start: {error}")))?; - Ok((result_receiver, handle)) - } - - fn assert_writer_is_blocked(receiver: &Receiver>) -> io::Result<()> { - match receiver.recv_timeout(Duration::from_millis(100)) { - Err(RecvTimeoutError::Timeout) => Ok(()), - Err(RecvTimeoutError::Disconnected) => Err(io::Error::other("replacement writer disconnected")), - Ok(result) => { - let root = result?; - Err(io::Error::other(format!("replacement writer completed early with root page {root}"))) - } - } - } - - fn finish_writer(receiver: &Receiver>, handle: JoinHandle<()>) -> io::Result<()> { - receiver - .recv_timeout(Duration::from_secs(5)) - .map_err(|error| io::Error::other(format!("replacement writer stayed blocked: {error}")))??; - handle.join().map_err(|_| io::Error::other("replacement writer panicked")) - } - - /// Waits for the sidecar lock to become free. - /// - /// A writer reporting its result does not mean the lock is already released: - /// publication can finish on the engine's background thread, so the release - /// happens on a different thread from the one the test observes. Asserting - /// `try_exclusive_sidecar` immediately races with that hand-off and fails - /// intermittently. Waiting for the lock to become available tests what the - /// engine actually promises - that it is released - without asserting an - /// instantaneous guarantee it never made. - fn wait_for_exclusive_sidecar(database: &Path) -> io::Result { - let deadline = std::time::Instant::now() + Duration::from_secs(5); - loop { - if try_exclusive_sidecar(database)? { - return Ok(true); - } - if std::time::Instant::now() >= deadline { - return Ok(false); - } - thread::sleep(Duration::from_millis(10)); - } - } - - fn try_exclusive_sidecar(database: &Path) -> io::Result { - let file = OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(false) - .open(crate::common::sidecar_lock_path(database))?; - match file.try_lock() { - Ok(()) => { - file.unlock()?; - Ok(true) - } - Err(fs::TryLockError::WouldBlock) => Ok(false), - Err(fs::TryLockError::Error(error)) => Err(error), - } - } - - fn run_exclusive_probe_child(database: &Path, expected: &str) -> io::Result<()> { - let status = Command::new(std::env::current_exe()?) - .arg("--exact") - .arg("v3::tree::tests::exclusive_sidecar_probe_child") - .arg("--nocapture") - .env("TULIPROX_V3_LOCK_PROBE_PATH", database) - .env("TULIPROX_V3_LOCK_PROBE_EXPECTED", expected) - .status()?; - if status.success() { - Ok(()) - } else { - Err(io::Error::other(format!("exclusive sidecar child probe failed with {status}"))) - } - } - - #[test] - fn exclusive_sidecar_probe_child() -> io::Result<()> { - let Some(path) = std::env::var_os("TULIPROX_V3_LOCK_PROBE_PATH") else { - return Ok(()); - }; - let expected = std::env::var("TULIPROX_V3_LOCK_PROBE_EXPECTED") - .map_err(|error| io::Error::other(format!("missing child probe expectation: {error}")))?; - let acquired = try_exclusive_sidecar(Path::new(&path))?; - match expected.as_str() { - "acquired" => assert!(acquired), - "blocked" => assert!(!acquired), - _ => return Err(io::Error::other(format!("unknown child probe expectation: {expected}"))), - } - Ok(()) - } - - #[test] - fn shared_queries_coexist_and_block_replacement_until_both_drop() -> io::Result<()> { - let directory = tempfile::tempdir()?; - let path = directory.path().join("shared-queries.db"); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("original")); - tree.store(&path)?; - - let first = BPlusTreeQuery::::try_new(&path)?; - let second = first.try_clone()?; - assert!(!try_exclusive_sidecar(&path)?); - drop(first); - assert!(!try_exclusive_sidecar(&path)?); - let (receiver, handle) = spawn_replacement_writer(path.clone())?; - assert_writer_is_blocked(&receiver)?; - drop(second); - finish_writer(&receiver, handle)?; - assert!(wait_for_exclusive_sidecar(&path)?); - Ok(()) - } - - #[test] - fn owned_iterator_keeps_shared_guard_after_query_is_consumed() -> io::Result<()> { - let directory = tempfile::tempdir()?; - let path = directory.path().join("owned-iterator.db"); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("original")); - tree.store(&path)?; - - let query = BPlusTreeQuery::::try_new(&path)?; - let iterator = query.disk_iter(); - assert!(!try_exclusive_sidecar(&path)?); - let (receiver, handle) = spawn_replacement_writer(path.clone())?; - assert_writer_is_blocked(&receiver)?; - drop(iterator); - finish_writer(&receiver, handle)?; - assert!(wait_for_exclusive_sidecar(&path)?); - Ok(()) - } - - #[test] - fn shared_query_blocks_exclusive_writer_in_another_process() -> io::Result<()> { - let directory = tempfile::tempdir()?; - let path = directory.path().join("two-process.db"); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("original")); - tree.store(&path)?; - - let query = BPlusTreeQuery::::try_new(&path)?; - run_exclusive_probe_child(&path, "blocked")?; - drop(query); - run_exclusive_probe_child(&path, "acquired") - } - - #[test] - fn query_removes_abandoned_wal_temp_and_rejects_corrupt_active_wal() -> io::Result<()> { - let directory = tempfile::tempdir()?; - let path = directory.path().join("pending-recovery.db"); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("original")); - tree.store(&path)?; - - let temporary = pending_path(&path, ".wal.tmp"); - fs::write(&temporary, b"not activated")?; - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, Some(String::from("original"))); - assert!(!temporary.try_exists()?); - drop(query); - - let active = pending_path(&path, ".wal"); - fs::write(&active, b"corrupt active WAL")?; - let active_before = fs::read(&active)?; - let error = BPlusTreeQuery::::try_new(&path) - .err() - .ok_or_else(|| io::Error::other("query accepted corrupt active WAL"))?; - assert_eq!(error.kind(), io::ErrorKind::InvalidData); - assert_eq!(fs::read(active)?, active_before); - Ok(()) - } - - #[test] - fn replacement_store_removes_wal_temp_and_preserves_corrupt_active_wal() -> io::Result<()> { - let directory = tempfile::tempdir()?; - let path = directory.path().join("pending-store.db"); - let mut original = BPlusTree::new(); - original.insert(1u32, String::from("original")); - original.store(&path)?; - let database_before = fs::read(&path)?; - - let temporary = pending_path(&path, ".wal.tmp"); - fs::write(&temporary, b"not activated")?; - let mut replacement = BPlusTree::new(); - replacement.insert(2u32, String::from("replacement")); - replacement.store(&path)?; - assert!(!temporary.try_exists()?); - - let published_before = fs::read(&path)?; - let active = pending_path(&path, ".wal"); - let active_before = b"corrupt active WAL".to_vec(); - fs::write(&active, &active_before)?; - let mut rejected = BPlusTree::new(); - rejected.insert(3u32, String::from("rejected")); - let error = rejected.store(&path).err().ok_or_else(|| io::Error::other("store accepted corrupt WAL"))?; - assert_eq!(error.kind(), io::ErrorKind::InvalidData); - assert_ne!(published_before, database_before); - assert_eq!(fs::read(&path)?, published_before); - assert_eq!(fs::read(active)?, active_before); - Ok(()) - } - - #[test] - fn clean_loaded_store_recovers_temp_but_rejects_corrupt_active_wal() -> io::Result<()> { - let directory = tempfile::tempdir()?; - let path = directory.path().join("pending-clean-store.db"); - let mut original = BPlusTree::new(); - original.insert(1u32, String::from("original")); - original.store(&path)?; - let database_before = fs::read(&path)?; - let mut loaded = BPlusTree::::load(&path)?; - - let temporary = pending_path(&path, ".wal.tmp"); - fs::write(&temporary, b"not activated")?; - assert_eq!(loaded.store(&path)?, 0); - assert!(!temporary.try_exists()?); - assert_eq!(fs::read(&path)?, database_before); - - let active = pending_path(&path, ".wal"); - let active_before = b"corrupt active WAL".to_vec(); - fs::write(&active, &active_before)?; - let error = loaded.store(&path).err().ok_or_else(|| io::Error::other("clean store accepted corrupt WAL"))?; - assert_eq!(error.kind(), io::ErrorKind::InvalidData); - assert_eq!(fs::read(&path)?, database_before); - assert_eq!(fs::read(active)?, active_before); - Ok(()) - } - - fn empty_leaf(page_id: u64, next_page_id: u64) -> io::Result<[u8; PAGE_SIZE]> { - let mut page = [0; PAGE_SIZE]; - PageHeader { - page_type: PageType::Leaf, - cell_count: 0, - free_start: u16::try_from(PAGE_HEADER_LEN).map_err(io::Error::other)?, - free_end: u16::try_from(PAGE_SIZE).map_err(io::Error::other)?, - left: 0, - right: 0, - } - .encode_into(&mut page, page_id, next_page_id)?; - Ok(page) - } - - fn leaf_cell_of_footprint(footprint: usize) -> io::Result> { - let key_length = footprint.checked_sub(4 + 24).ok_or_else(|| io::Error::other("footprint is too small"))?; - if footprint <= 2032 { - let key = vec![b'k'; key_length]; - let mut cell = Vec::new(); - encode_tombstone_leaf_cell(&key, &mut cell)?; - Ok(cell) - } else { - Ok(vec![0; footprint - 4]) - } - } - - #[test] - fn typed_leaf_cells_round_trip_and_validate_value_crc() -> io::Result<()> { - let mut cell = Vec::new(); - encode_inline_leaf_cell(b"key", 5, Compression::None, b"value", &mut cell)?; - let decoded = LeafCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)?; - assert_eq!(decoded.key_bytes, b"key"); - match decoded.value { - LeafValueRef::Inline { compression, logical_len, stored, crc32 } => { - assert_eq!(compression, Compression::None); - assert_eq!(logical_len, 5); - assert_eq!(stored, b"value"); - assert_eq!(crc32, crc32fast::hash(b"value")); - } - _ => return Err(io::Error::other("expected inline value")), - } - - let last = cell.len().checked_sub(1).ok_or_else(|| io::Error::other("empty test cell"))?; - cell[last] ^= 1; - invalid_data(LeafCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)) - } - - #[test] - fn compressed_inline_tombstone_and_overflow_descriptors_round_trip() -> io::Result<()> { - let raw = [0u8; 128]; - let mut compression_scratch = Vec::new(); - let stored = encode_value(&raw, &mut compression_scratch)?; - assert_eq!(stored.compression(), Compression::Lz4); - - let mut cell = Vec::new(); - encode_inline_leaf_cell( - b"compressed", - u32::try_from(raw.len()).map_err(io::Error::other)?, - stored.compression(), - stored.as_slice(), - &mut cell, - )?; - let decoded = LeafCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)?; - let mut value_scratch = Vec::new(); - assert_eq!(read_leaf_value(&[], &decoded.value, NEXT_PAGE_ID, 1024, &mut value_scratch)?, Some(raw.as_slice())); - - encode_tombstone_leaf_cell(b"deleted", &mut cell)?; - assert!(matches!(LeafCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)?.value, LeafValueRef::Tombstone)); - - encode_overflow_leaf_cell( - b"large", - 5000, - Compression::None, - 5000, - 2, - 0x1234_5678, - PAGE_ID, - NEXT_PAGE_ID, - &mut cell, - )?; - assert!(matches!( - LeafCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)?.value, - LeafValueRef::Overflow { head: 2, .. } - )); - Ok(()) - } - - #[test] - fn internal_separator_and_locator_have_exact_codecs() -> io::Result<()> { - let mut cell = Vec::new(); - encode_internal_cell(b"separator", 9, PAGE_ID, NEXT_PAGE_ID, &mut cell)?; - let decoded = InternalCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)?; - assert_eq!(decoded.key_bytes, b"separator"); - assert_eq!(decoded.right_child, 9); - - let locator = Locator::for_key(7, 3, b"separator")?; - let encoded = locator.encode(); - assert_eq!(encoded.len(), 16); - assert_eq!(Locator::decode(&encoded)?, locator); - let mut corrupt = encoded; - corrupt[10] = 1; - invalid_data(Locator::decode(&corrupt)) - } - - #[test] - fn locator_rejects_wrong_key_crc_and_wrong_primary_key() -> io::Result<()> { - let key = binary_serialize(&42u32)?; - let mut cell = Vec::new(); - encode_inline_leaf_cell(&key, 1, Compression::None, &[7], &mut cell)?; - let mut bytes = empty_leaf(PAGE_ID, NEXT_PAGE_ID)?; - let mut page = SlottedPage::open(bytes.as_mut_slice(), PAGE_ID, NEXT_PAGE_ID)?; - page.rebuild_ordered([cell.as_slice()])?; - - let locator = Locator::for_key(PAGE_ID, 0, &key)?; - validate_locator(&page, locator, &key)?; - let bad_crc = Locator { serialized_key_crc32: locator.serialized_key_crc32 ^ 1, ..locator }; - invalid_data(validate_locator(&page, bad_crc, &key))?; - invalid_data(validate_locator(&page, locator, &binary_serialize(&43u32)?)) - } - - #[test] - fn encoded_key_limit_is_2004_bytes() -> io::Result<()> { - let mut cell = Vec::new(); - encode_tombstone_leaf_cell(&vec![b'k'; 2004], &mut cell)?; - assert_eq!(cell.len() + 4, 2032); - invalid_input(encode_tombstone_leaf_cell(&vec![b'k'; 2005], &mut cell))?; - - encode_internal_cell(&vec![b'k'; 2004], 9, PAGE_ID, NEXT_PAGE_ID, &mut cell)?; - invalid_input(encode_internal_cell(&vec![b'k'; 2005], 9, PAGE_ID, NEXT_PAGE_ID, &mut cell)) - } - - #[test] - fn adversarial_leaf_splits_reject_old_limit_and_accept_capped_cells() -> io::Result<()> { - let first_witness = - [leaf_cell_of_footprint(2026)?, leaf_cell_of_footprint(2040)?, leaf_cell_of_footprint(2038)?]; - invalid_input(choose_leaf_split(&first_witness))?; - - let second_witness = - [leaf_cell_of_footprint(1984)?, leaf_cell_of_footprint(2296)?, leaf_cell_of_footprint(1984)?]; - invalid_input(choose_leaf_split(&second_witness))?; - - let capped = [leaf_cell_of_footprint(2026)?, leaf_cell_of_footprint(2032)?, leaf_cell_of_footprint(2032)?]; - let split = choose_leaf_split(&capped)?; - assert_eq!(split, 2); - assert!(used_leaf_bytes(&capped[..split])? <= PAGE_SIZE); - assert!(used_leaf_bytes(&capped[split..])? <= PAGE_SIZE); - Ok(()) - } - - #[test] - fn leaf_split_selection_is_deterministic_for_variable_sizes() -> io::Result<()> { - let mut seed = 0x5eed_u64; - for count in 3..36 { - let mut cells = Vec::with_capacity(count); - for _ in 0..count { - seed = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); - let footprint = 100 + usize::try_from(seed % 60).map_err(io::Error::other)?; - let insert_at = if cells.is_empty() { - 0 - } else { - usize::try_from(seed).map_err(io::Error::other)? % (cells.len() + 1) - }; - cells.insert(insert_at, leaf_cell_of_footprint(footprint)?); - } - if used_leaf_bytes(&cells)? > PAGE_SIZE { - let first = choose_leaf_split(&cells)?; - let second = choose_leaf_split(&cells)?; - assert_eq!(first, second); - assert!(used_leaf_bytes(&cells[..first])? <= PAGE_SIZE); - assert!(used_leaf_bytes(&cells[first..])? <= PAGE_SIZE); - } - } - Ok(()) - } - - #[test] - fn internal_split_promotes_exact_separator_and_child() -> io::Result<()> { - let mut cells = Vec::new(); - for (key_length, child) in [(1000, 2), (1000, 3), (1000, 4), (1000, 5)] { - let mut cell = Vec::new(); - encode_internal_cell(&vec![b'k'; key_length], child, PAGE_ID, NEXT_PAGE_ID, &mut cell)?; - cells.push(cell); - } - let split = choose_internal_split(&cells, PAGE_ID, NEXT_PAGE_ID)?; - assert_eq!(split.promoted_index, 1); - assert_eq!(split.promoted.key_bytes.len(), 1000); - assert_eq!(split.right_leftmost_child, 3); - assert_eq!(split.left_cells, &cells[..1]); - assert_eq!(split.right_cells, &cells[2..]); - Ok(()) - } - - #[test] - fn typed_search_decodes_only_binary_search_candidates() -> io::Result<()> { - let mut cells = Vec::new(); - for key in 0u8..8 { - let mut cell = Vec::new(); - encode_inline_leaf_cell(&binary_serialize(&key)?, 1, Compression::None, &[key], &mut cell)?; - cells.push(cell); - } - if let Some(byte) = cells.get_mut(0).and_then(|cell| cell.get_mut(24)) { - *byte = 0xc1; - } - - let mut bytes = empty_leaf(PAGE_ID, NEXT_PAGE_ID)?; - let mut mutable = SlottedPage::open(bytes.as_mut_slice(), PAGE_ID, NEXT_PAGE_ID)?; - mutable.rebuild_ordered(cells.iter().map(Vec::as_slice))?; - let page = SlottedPage::open(bytes.as_slice(), PAGE_ID, NEXT_PAGE_ID)?; - assert_eq!(search_leaf(&page, &7u8)?, Ok(7)); - invalid_data(search_leaf(&page, &0u8)) - } - - #[test] - fn overflow_chain_round_trip_is_bounded_and_checksum_validated() -> io::Result<()> { - let stored = vec![0x5a; 5000]; - let mut database = vec![0; PAGE_SIZE * 4]; - let first = encode_overflow_page(2, NEXT_PAGE_ID, 3, &stored[..4056])?; - let second = encode_overflow_page(3, NEXT_PAGE_ID, 0, &stored[4056..])?; - database[PAGE_SIZE * 2..PAGE_SIZE * 3].copy_from_slice(&first); - database[PAGE_SIZE * 3..PAGE_SIZE * 4].copy_from_slice(&second); - - let value = LeafValueRef::Overflow { - compression: Compression::None, - logical_len: 5000, - stored_len: 5000, - head: 2, - crc32: crc32fast::hash(&stored), - }; - let mut scratch = Vec::new(); - assert_eq!(read_leaf_value(&database, &value, NEXT_PAGE_ID, 6000, &mut scratch)?, Some(stored.as_slice())); - - let free = encode_free_page(3, NEXT_PAGE_ID, 0)?; - database[PAGE_SIZE * 3..PAGE_SIZE * 4].copy_from_slice(&free); - invalid_data(read_leaf_value(&database, &value, NEXT_PAGE_ID, 6000, &mut scratch)) - } - - #[test] - fn compressed_overflow_chain_reuses_scratch_and_round_trips() -> io::Result<()> { - let mut raw = Vec::with_capacity(10_000); - let mut state = 0x51a7_9e2d_4c83_b6f0_u64; - for _ in 0..5000 { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - raw.push(state.to_le_bytes()[0]); - } - raw.resize(10_000, 0); - let mut encoded = Vec::new(); - let stored = encode_value(&raw, &mut encoded)?; - if stored.compression() != Compression::Lz4 || stored.as_slice().len() <= OVERFLOW_PAYLOAD_LEN { - return Err(io::Error::other("test value must span compressed overflow pages")); - } - let stored = stored.as_slice().to_vec(); - let mut corrupt = stored.clone(); - corrupt - .get_mut(..4) - .ok_or_else(|| io::Error::other("missing test LZ4 length"))? - .copy_from_slice(&1u32.to_le_bytes()); - invalid_data(decompress_value_in_place( - &mut corrupt, - u32::try_from(raw.len()).map_err(io::Error::other)?, - raw.len(), - ))?; - let page_count = stored.len().div_ceil(OVERFLOW_PAYLOAD_LEN); - assert!(page_count > 1); - let next_page_id = u64::try_from(page_count) - .map_err(io::Error::other)? - .checked_add(2) - .ok_or_else(|| io::Error::other("test page count overflow"))?; - let mut database = vec![0; usize::try_from(next_page_id).map_err(io::Error::other)? * PAGE_SIZE]; - for (index, payload) in stored.chunks(OVERFLOW_PAYLOAD_LEN).enumerate() { - let page_id = u64::try_from(index).map_err(io::Error::other)? + 1; - let next = if index + 1 == page_count { 0 } else { page_id + 1 }; - let page = encode_overflow_page(page_id, next_page_id, next, payload)?; - let start = usize::try_from(page_id).map_err(io::Error::other)? * PAGE_SIZE; - database[start..start + PAGE_SIZE].copy_from_slice(&page); - } - let value = LeafValueRef::Overflow { - compression: Compression::Lz4, - logical_len: u32::try_from(raw.len()).map_err(io::Error::other)?, - stored_len: u32::try_from(stored.len()).map_err(io::Error::other)?, - head: 1, - crc32: crc32fast::hash(&stored), - }; - let mut scratch = Vec::new(); - assert_eq!(read_leaf_value(&database, &value, next_page_id, raw.len(), &mut scratch)?, Some(raw.as_slice())); - Ok(()) - } - - #[test] - fn overflow_chain_rejects_truncation_cycle_oversize_and_bad_crc() -> io::Result<()> { - let stored = vec![0x5a; 5000]; - let value = LeafValueRef::Overflow { - compression: Compression::None, - logical_len: 5000, - stored_len: 5000, - head: 2, - crc32: crc32fast::hash(&stored), - }; - let mut database = vec![0; PAGE_SIZE * 4]; - let first = encode_overflow_page(2, NEXT_PAGE_ID, 3, &stored[..4056])?; - let short = encode_overflow_page(3, NEXT_PAGE_ID, 0, &stored[4056..4999])?; - database[PAGE_SIZE * 2..PAGE_SIZE * 3].copy_from_slice(&first); - database[PAGE_SIZE * 3..PAGE_SIZE * 4].copy_from_slice(&short); - let mut scratch = Vec::new(); - invalid_data(read_leaf_value(&database, &value, NEXT_PAGE_ID, 6000, &mut scratch))?; - - let cycle = encode_overflow_page(3, NEXT_PAGE_ID, 2, &stored[4056..])?; - database[PAGE_SIZE * 3..PAGE_SIZE * 4].copy_from_slice(&cycle); - invalid_data(read_leaf_value(&database, &value, NEXT_PAGE_ID, 6000, &mut scratch))?; - invalid_data(read_leaf_value(&database, &value, NEXT_PAGE_ID, 4999, &mut scratch))?; - - let last = encode_overflow_page(3, NEXT_PAGE_ID, 0, &stored[4056..])?; - database[PAGE_SIZE * 3..PAGE_SIZE * 4].copy_from_slice(&last); - let LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } = value else { - return Err(io::Error::other("expected overflow value")); - }; - let bad_crc = LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32: crc32 ^ 1 }; - invalid_data(read_leaf_value(&database, &bad_crc, NEXT_PAGE_ID, 6000, &mut scratch))?; - - database[PAGE_SIZE * 2 + 40] ^= 1; - invalid_data(read_leaf_value(&database, &value, NEXT_PAGE_ID, 6000, &mut scratch))?; - - let empty = encode_overflow_page(2, NEXT_PAGE_ID, 3, &[])?; - let one_byte = encode_overflow_page(3, NEXT_PAGE_ID, 0, b"x")?; - database[PAGE_SIZE * 2..PAGE_SIZE * 3].copy_from_slice(&empty); - database[PAGE_SIZE * 3..PAGE_SIZE * 4].copy_from_slice(&one_byte); - let empty_chain_value = LeafValueRef::Overflow { - compression: Compression::None, - logical_len: 1, - stored_len: 1, - head: 2, - crc32: crc32fast::hash(b"x"), - }; - invalid_data(read_leaf_value(&database, &empty_chain_value, NEXT_PAGE_ID, 6000, &mut scratch)) - } - - fn page_range(page_id: u64) -> io::Result> { - let start = usize::try_from(page_id) - .map_err(io::Error::other)? - .checked_mul(PAGE_SIZE) - .ok_or_else(|| io::Error::other("test page offset overflow"))?; - Ok(start..start + PAGE_SIZE) - } - - fn rewrite_page_checksum(database: &mut [u8], page_id: u64) -> io::Result<()> { - let range = page_range(page_id)?; - crate::v3::format::write_page_checksum( - database.get_mut(range).ok_or_else(|| io::Error::other("test page missing"))?, - ) - } - - fn set_page_reference(database: &mut [u8], page_id: u64, offset: usize, reference: u64) -> io::Result<()> { - let page = page_range(page_id)?; - let start = page.start + offset; - database - .get_mut(start..start + 8) - .ok_or_else(|| io::Error::other("test reference missing"))? - .copy_from_slice(&reference.to_le_bytes()); - rewrite_page_checksum(database, page_id) - } - - fn verify_rejects(database: &[u8]) -> io::Result<()> - where - K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, - V: Serialize + for<'de> Deserialize<'de> + Clone, - { - let dir = tempfile::tempdir()?; - let path = dir.path().join("corrupt.db"); - fs::write(&path, database)?; - let mut query = BPlusTreeQuery::::try_new(&path)?; - invalid_data(verify_full(&mut query)) - } - - fn stored_tree_fixture() -> io::Result<(Vec, u64, u64, u64, u64)> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("tree.db"); - let mut tree = BPlusTree::new(); - for key in 1_000..1_700u32 { - tree.insert(key, "same-value".to_string()); - } - tree.store(&path)?; - let mut query = BPlusTreeQuery::::try_new(&path)?; - let root = query.header.root_page_id; - let first = query.leftmost_leaf()?; - let next_page_id = query.header.next_page_id; - let second = query.with_page(first, |bytes, _| { - SlottedPage::open(bytes, first, next_page_id).map(|page| page.header().right) - })?; - if second == 0 { - return Err(io::Error::other("test tree did not split")); - } - let mut last = second; - loop { - let right = query.with_page(last, |bytes, _| { - SlottedPage::open(bytes, last, next_page_id).map(|page| page.header().right) - })?; - if right == 0 { - break; - } - last = right; - } - drop(query); - Ok((fs::read(path)?, root, first, second, last)) - } - - #[test] - fn iterator_yields_second_leaf_corruption_once_then_fuses() -> io::Result<()> { - let (mut database, _, _, second, _) = stored_tree_fixture()?; - let byte = page_range(second)?.start + 100; - *database.get_mut(byte).ok_or_else(|| io::Error::other("test corruption byte missing"))? ^= 1; - let dir = tempfile::tempdir()?; - let path = dir.path().join("iterator-corrupt.db"); - fs::write(&path, database)?; - let mut query = BPlusTreeQuery::::try_new(&path)?; - let mut iterator = query.iter(); - let mut yielded = 0usize; - let mut saw_error = false; - for item in iterator.by_ref() { - match item { - Ok(_) => yielded += 1, - Err(err) => { - assert_eq!(err.kind(), io::ErrorKind::InvalidData); - saw_error = true; - break; - } - } - } - if !saw_error { - return Err(io::Error::other("corruption was hidden as end-of-stream")); - } - assert!(iterator.next().is_none()); - assert!(yielded > 0); - Ok(()) - } - - #[test] - fn iterator_skips_corrupt_value_and_continues_with_next_cell() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("iterator-corrupt-value.db"); - let mut tree = BPlusTree::new(); - for key in 1..=3u32 { - tree.insert(key, format!("value-{key}")); - } - tree.store(&path)?; - - let mut query = BPlusTreeQuery::::try_new(&path)?; - let (page_id, range) = query.locate_cell(&2)?.ok_or_else(|| io::Error::other("test key missing"))?; - let next_page_id = query.header.next_page_id; - let stored_offset = query.with_page(page_id, |bytes, _| { - let cell = LeafCellRef::decode( - bytes.get(range).ok_or_else(|| io::Error::other("test cell range missing"))?, - page_id, - next_page_id, - )?; - let LeafValueRef::Inline { stored, .. } = cell.value else { - return Err(io::Error::other("test value is not inline")); - }; - Ok(stored.as_ptr() as usize - bytes.as_ptr() as usize) - })?; - drop(query); - - let mut database = fs::read(&path)?; - let absolute = page_range(page_id)?.start + stored_offset; - *database.get_mut(absolute).ok_or_else(|| io::Error::other("test value byte missing"))? = 0xc1; - rewrite_page_checksum(&mut database, page_id)?; - fs::write(&path, database)?; - - let mut query = BPlusTreeQuery::::try_new(&path)?; - let mut iterator = query.iter(); - assert_eq!(iterator.next().transpose()?, Some((1, String::from("value-1")))); - assert!(iterator.next().is_some_and(|entry| entry.is_err())); - assert_eq!(iterator.next().transpose()?, Some((3, String::from("value-3")))); - assert!(iterator.next().is_none()); - Ok(()) - } - - #[test] - fn iterator_rejects_leaf_cycle_before_yielding_duplicate_entries() -> io::Result<()> { - let (mut database, _, first, second, _) = stored_tree_fixture()?; - set_page_reference(&mut database, second, 16, first)?; - set_page_reference(&mut database, first, 8, second)?; - let dir = tempfile::tempdir()?; - let path = dir.path().join("iterator-cycle.db"); - fs::write(&path, database)?; - - let mut query = BPlusTreeQuery::::try_new(&path)?; - let mut iterator = query.iter(); - let mut seen = HashSet::new(); - let mut errors = 0usize; - for item in iterator.by_ref() { - match item { - Ok((key, _)) => assert!(seen.insert(key), "iterator yielded key {key} twice"), - Err(err) => { - assert_eq!(err.kind(), io::ErrorKind::InvalidData); - errors += 1; - } - } - } - assert_eq!(errors, 1); - assert!(iterator.next().is_none()); - Ok(()) - } - - #[test] - fn range_page_does_not_preallocate_the_requested_limit() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("range-limit.db"); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("one")); - tree.insert(2u32, String::from("two")); - tree.store(&path)?; - - let mut query = BPlusTreeQuery::::try_new(&path)?; - let (entries, has_more) = - query.range_page(Bound::Unbounded, Bound::Unbounded, 0, usize::MAX).map_err(BPlusTreeError::to_io)?; - assert_eq!(entries, vec![(1, String::from("one")), (2, String::from("two"))]); - assert!(!has_more); - Ok(()) - } - - #[derive(Debug)] - struct RejectValueDeserialize; - - impl<'de> Deserialize<'de> for RejectValueDeserialize { - fn deserialize>(_deserializer: D) -> Result { - Err(serde::de::Error::custom("value deserialization must not run")) - } - } - - #[test] - fn contains_and_len_read_only_leaf_descriptors() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("descriptor-only.db"); - let mut tree = BPlusTree::new(); - tree.insert(1u32, String::from("one")); - tree.insert(2u32, String::from("two")); - tree.store(&path)?; - - let mut query = BPlusTreeQuery::::try_new(&path)?; - assert!(query.contains_live_key(&1).map_err(BPlusTreeError::to_io)?); - assert!(!query.contains_live_key(&3).map_err(BPlusTreeError::to_io)?); - assert_eq!(query.len().map_err(BPlusTreeError::to_io)?, 2); - Ok(()) - } - - #[test] - fn full_verifier_rejects_tree_sibling_free_and_orphan_corruption() -> io::Result<()> { - let (database, root, first, second, last) = stored_tree_fixture()?; - - let mut child_cycle = database.clone(); - set_page_reference(&mut child_cycle, root, 32, root)?; - verify_rejects::(&child_cycle)?; - - let mut duplicate_child = database.clone(); - let root_start = page_range(root)?.start; - let leftmost = - u64::from_le_bytes(duplicate_child[root_start + 32..root_start + 40].try_into().map_err(io::Error::other)?); - let first_cell = - u16::from_le_bytes(duplicate_child[root_start + 40..root_start + 42].try_into().map_err(io::Error::other)?); - let child_offset = root_start + usize::from(first_cell) + 4; - duplicate_child[child_offset..child_offset + 8].copy_from_slice(&leftmost.to_le_bytes()); - rewrite_page_checksum(&mut duplicate_child, root)?; - verify_rejects::(&duplicate_child)?; - - let mut wrong_type = database.clone(); - let header = DatabaseHeader::decode(&wrong_type[..PAGE_SIZE])?; - let overflow = encode_overflow_page(leftmost, header.next_page_id, 0, b"wrong type")?; - let range = page_range(leftmost)?; - wrong_type[range].copy_from_slice(&overflow); - verify_rejects::(&wrong_type)?; - - let mut asymmetric = database.clone(); - set_page_reference(&mut asymmetric, second, 8, 0)?; - verify_rejects::(&asymmetric)?; - - let mut sibling_cycle = database.clone(); - set_page_reference(&mut sibling_cycle, last, 16, first)?; - set_page_reference(&mut sibling_cycle, first, 8, last)?; - verify_rejects::(&sibling_cycle)?; - - let mut inverted = database.clone(); - let first_start = page_range(first)?.start; - let second_start = page_range(second)?.start; - let first_count = - u16::from_le_bytes(inverted[first_start + 2..first_start + 4].try_into().map_err(io::Error::other)?); - let last_slot = first_start + PAGE_HEADER_LEN + (usize::from(first_count) - 1) * SLOT_LEN; - let left_cell = - usize::from(u16::from_le_bytes(inverted[last_slot..last_slot + 2].try_into().map_err(io::Error::other)?)); - let right_cell = usize::from(u16::from_le_bytes( - inverted[second_start + PAGE_HEADER_LEN..second_start + PAGE_HEADER_LEN + 2] - .try_into() - .map_err(io::Error::other)?, - )); - for index in 0..3 { - inverted.swap(first_start + left_cell + 24 + index, second_start + right_cell + 24 + index); - } - rewrite_page_checksum(&mut inverted, first)?; - rewrite_page_checksum(&mut inverted, second)?; - verify_rejects::(&inverted)?; - - let mut live_and_free = database.clone(); - let mut header = DatabaseHeader::decode(&live_and_free[..PAGE_SIZE])?; - header.free_page_head = first; - live_and_free[..PAGE_SIZE].copy_from_slice(&header.encode()?); - verify_rejects::(&live_and_free)?; - - let mut orphan = database.clone(); - let mut header = DatabaseHeader::decode(&orphan[..PAGE_SIZE])?; - let orphan_id = header.next_page_id; - header.next_page_id += 1; - orphan[..PAGE_SIZE].copy_from_slice(&header.encode()?); - orphan.extend_from_slice(&encode_free_page(orphan_id, header.next_page_id, 0)?); - verify_rejects::(&orphan)?; - - let mut duplicate_free = database; - let mut header = DatabaseHeader::decode(&duplicate_free[..PAGE_SIZE])?; - let first_free = header.next_page_id; - let second_free = first_free + 1; - header.next_page_id += 2; - header.free_page_head = first_free; - duplicate_free[..PAGE_SIZE].copy_from_slice(&header.encode()?); - duplicate_free.extend_from_slice(&encode_free_page(first_free, header.next_page_id, second_free)?); - duplicate_free.extend_from_slice(&encode_free_page(second_free, header.next_page_id, first_free)?); - verify_rejects::(&duplicate_free) - } - - fn random_value() -> Vec { - let mut state = 0x1234_5678_9abc_def0u64; - (0..12_000) - .map(|_| { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - state.to_le_bytes()[0] - }) - .collect() - } - - fn overflow_heads(query: &mut BPlusTreeQuery>) -> io::Result<(u64, u64, u64)> { - let mut result = Vec::new(); - for key in [1, 2] { - let leaf = query.locate_leaf(&key)?; - let next_page_id = query.header.next_page_id; - let (head, cell_offset) = query.with_page(leaf, |bytes, _| { - let page = SlottedPage::open(bytes, leaf, next_page_id)?; - let index = search_leaf(&page, &key)?.map_err(|_| io::Error::other("test key missing"))?; - let cell = LeafCellRef::decode(page.cell(index)?, leaf, next_page_id)?; - let LeafValueRef::Overflow { head, .. } = cell.value else { - return Err(io::Error::other("test value is not overflow-backed")); - }; - let slot = PAGE_HEADER_LEN + index * SLOT_LEN; - let offset = u16::from_le_bytes(bytes[slot..slot + 2].try_into().map_err(io::Error::other)?); - Ok((head, u64::from(offset))) - })?; - result.push((leaf, head, cell_offset)); - } - let first = result.first().ok_or_else(|| io::Error::other("first test overflow missing"))?; - let second = result.get(1).ok_or_else(|| io::Error::other("second test overflow missing"))?; - Ok((first.1, second.0, second.2)) - } - - #[test] - fn full_verifier_rejects_overflow_cycle_and_shared_chain() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("overflow.db"); - let value = random_value(); - let mut tree = BPlusTree::new(); - tree.insert(1u32, value.clone()); - tree.insert(2u32, value); - tree.store(&path)?; - let mut query = BPlusTreeQuery::>::try_new(&path)?; - let (first_head, second_leaf, second_cell_offset) = overflow_heads(&mut query)?; - let next_page_id = query.header.next_page_id; - drop(query); - let database = fs::read(&path)?; - - let mut cycle = database.clone(); - let mut last = first_head; - loop { - let start = page_range(last)?.start; - let next = u64::from_le_bytes(cycle[start + 16..start + 24].try_into().map_err(io::Error::other)?); - if next == 0 { - break; - } - last = next; - } - set_page_reference(&mut cycle, last, 16, first_head)?; - verify_rejects::>(&cycle)?; - - let mut shared = database; - let start = - page_range(second_leaf)?.start + usize::try_from(second_cell_offset).map_err(io::Error::other)? + 12; - shared[start..start + 8].copy_from_slice(&first_head.to_le_bytes()); - rewrite_page_checksum(&mut shared, second_leaf)?; - let _ = next_page_id; - verify_rejects::>(&shared) - } - - #[test] - fn full_verifier_rejects_oversized_overflow_descriptor_before_reserve() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("oversized-overflow.db"); - let value = random_value(); - let mut tree = BPlusTree::new(); - tree.insert(1u32, value.clone()); - tree.insert(2u32, value); - tree.store(&path)?; - let mut query = BPlusTreeQuery::>::try_new(&path)?; - let (_, second_leaf, second_cell_offset) = overflow_heads(&mut query)?; - drop(query); - - let mut database = fs::read(&path)?; - let descriptor = - page_range(second_leaf)?.start + usize::try_from(second_cell_offset).map_err(io::Error::other)?; - database[descriptor + 4..descriptor + 8].copy_from_slice(&u32::MAX.to_le_bytes()); - database[descriptor + 8..descriptor + 12].copy_from_slice(&u32::MAX.to_le_bytes()); - rewrite_page_checksum(&mut database, second_leaf)?; - fs::write(&path, database)?; - - let mut query = BPlusTreeQuery::>::try_new(&path)?; - let error = verify_full(&mut query).err().ok_or_else(|| io::Error::other("oversized value accepted"))?; - assert_eq!(error.kind(), io::ErrorKind::InvalidData); - assert!(error.to_string().contains("overflow value exceeds allocation limit")); - Ok(()) - } - - #[test] - fn full_verifier_rejects_uncompressed_overflow_length_mismatch() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let path = dir.path().join("overflow-length.db"); - let mut tree = BPlusTree::new(); - tree.insert(1u32, random_value()); - tree.store(&path)?; - let mut query = BPlusTreeQuery::>::try_new(&path)?; - let leaf = query.locate_leaf(&1)?; - let next_page_id = query.header.next_page_id; - let cell_offset = query.with_page(leaf, |bytes, _| { - let page = SlottedPage::open(bytes, leaf, next_page_id)?; - let index = search_leaf(&page, &1)?.map_err(|_| io::Error::other("test key missing"))?; - let slot = PAGE_HEADER_LEN - .checked_add(index.checked_mul(SLOT_LEN).ok_or_else(|| io::Error::other("test slot overflow"))?) - .ok_or_else(|| io::Error::other("test slot overflow"))?; - Ok(u16::from_le_bytes(bytes[slot..slot + 2].try_into().map_err(io::Error::other)?)) - })?; - drop(query); - - let mut database = fs::read(&path)?; - let descriptor = page_range(leaf)?.start + usize::from(cell_offset); - database[descriptor + 4..descriptor + 8].copy_from_slice(&1u32.to_le_bytes()); - rewrite_page_checksum(&mut database, leaf)?; - fs::write(&path, database)?; - let mut query = BPlusTreeQuery::>::try_new(&path)?; - invalid_data(verify_full(&mut query)) - } - - #[test] - fn publish_reports_post_commit_directory_sync_failure() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let destination = dir.path().join("database.db"); - let temporary = dir.path().join("database.tx.v3.tmp"); - let mut old = BPlusTree::new(); - old.insert(1u32, vec![1]); - old.store(&destination)?; - let mut new = BPlusTree::new(); - new.insert(2u32, vec![2]); - new.store(&temporary)?; - - let error = - publish_database(&temporary, &destination, |_| Err(io::Error::other("injected directory sync failure"))) - .err() - .ok_or_else(|| io::Error::other("post-commit sync failure was hidden"))?; - assert!(error.database_was_published()); - let error = io::Error::from(error); - assert!(error.to_string().contains("database published but directory sync failed; durability unknown")); - assert!(!temporary.exists()); - let mut query = BPlusTreeQuery::>::try_new(&destination)?; - assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, None); - assert_eq!(query.query(&2).map_err(BPlusTreeError::to_io)?, Some(vec![2])); - Ok(()) - } - - #[test] - fn store_sync_failure_after_rename_invalidates_previous_sorted_index() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let destination = dir.path().join("database.db"); - let index = crate::common::get_file_path_for_db_index(&destination); - let mut old = BPlusTree::new(); - old.insert(1u32, String::from("old")); - let _ = old.store_with_index(&destination, String::clone)?; - assert!(index.is_file()); - - let mut replacement = BPlusTree::new(); - replacement.insert(2u32, String::from("new")); - let error = replacement - .store_exclusive_with_directory_sync(&destination, |_| { - Err(io::Error::other("injected directory sync failure")) - }) - .err() - .ok_or_else(|| io::Error::other("post-rename sync failure was hidden"))?; - - assert!(error.to_string().contains("publication durability remains unknown")); - assert!(!index.exists()); - assert!(replacement.dirty); - let mut query = BPlusTreeQuery::::try_new(&destination)?; - assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, None); - assert_eq!(query.query(&2).map_err(BPlusTreeError::to_io)?, Some(String::from("new"))); - Ok(()) - } - - #[test] - fn store_reports_sync_and_index_invalidation_failures_together() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let destination = dir.path().join("database.db"); - let index = crate::common::get_file_path_for_db_index(&destination); - let mut old = BPlusTree::new(); - old.insert(1u32, String::from("old")); - let _ = old.store(&destination)?; - fs::create_dir(&index)?; - - let mut replacement = BPlusTree::new(); - replacement.insert(2u32, String::from("new")); - let error = replacement - .store_exclusive_with_directory_sync(&destination, |_| { - Err(io::Error::other("injected directory sync failure")) - }) - .err() - .ok_or_else(|| io::Error::other("publish and index invalidation failures were hidden"))?; - let message = error.to_string(); - - assert!(message.contains("unknown directory durability")); - assert!(message.contains("previous sorted index could not be invalidated")); - assert!(index.is_dir()); - assert!(replacement.dirty); - let mut query = BPlusTreeQuery::::try_new(&destination)?; - assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, None); - assert_eq!(query.query(&2).map_err(BPlusTreeError::to_io)?, Some(String::from("new"))); - Ok(()) - } - - #[test] - fn publish_failure_removes_the_temporary_file() -> io::Result<()> { - let dir = tempfile::tempdir()?; - let destination = dir.path().join("destination-directory"); - fs::create_dir(&destination)?; - let temporary = dir.path().join("database.tx.v3.tmp"); - fs::write(&temporary, b"new")?; - - let error = publish_database(&temporary, &destination, sync_parent_directory) - .expect_err("publishing over a directory must fail"); - assert!(!error.database_was_published()); - assert!(!temporary.exists()); - Ok(()) - } -} +use self::mutation::database_page; +#[allow(unused_imports)] +pub(crate) use self::mutation::{choose_internal_split, choose_leaf_split, used_leaf_bytes}; +#[cfg(test)] +use self::query::{internal_key_decode_count, reset_internal_key_decode_count}; +#[cfg(test)] +pub(crate) use self::query::{read_leaf_value, validate_locator}; +#[cfg(test)] +use self::transaction::page_byte_range; +use self::{ + bulk_store::{ + build_pages, internal_page, leaf_page, temporary_path, verify_internal_page, verify_leaf_page, PageSink, + StoredDatabase, + }, + error::{invalid_data, invalid_input}, + iterators::{record_page_visit, CursorState}, + mutation::{ + checked_page_usage, locate_transaction_leaf, minimum_tree_key, overflow_chain_pages, push_child_visits, + stage_delete, stage_upsert, validate_leaf_backlink, validated_overflow_chain_pages, + }, + query::{decode_key, QuerySnapshot}, + transaction::{query_transaction, DatabaseImage, WriteScratch, WriteTransaction}, + update::ActiveBatch, + verify::{validate_transaction_links, VerifyLeaf, VerifyValue, Visit}, +}; +pub(super) use self::{ + bulk_store::{publish_database, publish_database_and_invalidate_sorted_index}, + error::PublishDatabaseError, +}; +pub(crate) use self::{query::search_leaf, verify::verify_full}; diff --git a/backend/btree/src/v3/tree/bulk_store.rs b/backend/btree/src/v3/tree/bulk_store.rs new file mode 100644 index 000000000..9a897d3cf --- /dev/null +++ b/backend/btree/src/v3/tree/bulk_store.rs @@ -0,0 +1,441 @@ +use super::{ + checked_page_usage, encode_inline_leaf_cell, encode_internal_cell, encode_overflow_leaf_cell, encode_overflow_page, + encode_value, invalid_data, invalid_input, invalidate_sorted_index, stored_value_checksum, BPlusTreeQuery, + InternalCellRef, InternalPreamble, LeafCellRef, LeafValueRef, PageHeader, PageType, PublishDatabaseError, Slot, + SlottedPage, VerificationReport, VerifyLeaf, VerifyValue, MAX_CELL_FOOTPRINT, MAX_INLINE_STORED_VALUE, + OVERFLOW_PAYLOAD_LEN, PAGE_HEADER_LEN, PAGE_SIZE, PAGE_SIZE_U64, SLOT_LEN, +}; +use crate::{ + codec::{binary_deserialize, binary_serialize}, + common::write_all_at_offset, +}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, HashSet}, + fs::File, + io, + path::{Path, PathBuf}, +}; + +#[derive(Clone)] +struct NodeInfo { + page_id: u64, + minimum_key: Vec, +} + +/// Sink for the bulk tree builder: hands out page ids and writes each finished page +/// straight to the destination file instead of buffering the whole database in memory. +/// +/// Pages are **not** emitted in ascending id order — a leaf reserves its id before the +/// overflow chain it points at, but is encoded only once all of its cells are known, so +/// it is written after them. Writes must therefore be positional; a sequential writer +/// would silently scramble the file. Every id handed out is written exactly once, so the +/// finished file has no holes, and `verify_full` re-reads it before it is published. +pub(super) struct PageSink { + pub(super) file: File, + pub(super) next_page_id: u64, +} + +impl PageSink { + pub(super) const fn new(file: File) -> Self { Self { file, next_page_id: 0 } } + + fn allocate(&mut self) -> io::Result { + let page_id = self.next_page_id; + self.next_page_id = page_id.checked_add(1).ok_or_else(|| invalid_input("page count exceeds u64"))?; + Ok(page_id) + } + + fn allocate_many(&mut self, count: usize) -> io::Result { + let head = self.next_page_id; + let count = u64::try_from(count).map_err(|_| invalid_input("page count exceeds u64"))?; + self.next_page_id = head.checked_add(count).ok_or_else(|| invalid_input("page count exceeds u64"))?; + Ok(head) + } + + pub(super) fn write(&self, page_id: u64, page: &[u8; PAGE_SIZE]) -> io::Result<()> { + let offset = page_id.checked_mul(PAGE_SIZE_U64).ok_or_else(|| invalid_input("page offset overflow"))?; + write_all_at_offset(&self.file, page, offset) + } +} + +pub(super) fn leaf_page>( + page_id: u64, + left: u64, + right: u64, + cells: &[T], +) -> io::Result<[u8; PAGE_SIZE]> { + let mut bytes = [0; PAGE_SIZE]; + PageHeader { + page_type: PageType::Leaf, + cell_count: 0, + free_start: u16::try_from(PAGE_HEADER_LEN).map_err(|_| invalid_input("leaf header exceeds u16"))?, + free_end: u16::try_from(PAGE_SIZE).map_err(|_| invalid_input("page size exceeds u16"))?, + left, + right, + } + .encode_into(&mut bytes, page_id, u64::MAX)?; + SlottedPage::open(bytes.as_mut_slice(), page_id, u64::MAX)?.rebuild_ordered(cells.iter().map(AsRef::as_ref))?; + Ok(bytes) +} + +pub(super) fn internal_page>( + page_id: u64, + leftmost_child: u64, + cells: &[T], +) -> io::Result<[u8; PAGE_SIZE]> { + let first = cells.first().ok_or_else(|| invalid_input("internal page must contain a separator"))?.as_ref(); + let first_offset = PAGE_SIZE.checked_sub(first.len()).ok_or_else(|| invalid_input("internal cell exceeds page"))?; + let mut bytes = [0; PAGE_SIZE]; + InternalPreamble { leftmost_child }.encode_into(&mut bytes, page_id, u64::MAX)?; + bytes.get_mut(40..44).ok_or_else(|| invalid_input("internal slot is outside page"))?.copy_from_slice( + &Slot { + offset: u16::try_from(first_offset).map_err(|_| invalid_input("internal cell offset exceeds u16"))?, + length: u16::try_from(first.len()).map_err(|_| invalid_input("internal cell length exceeds u16"))?, + } + .encode(), + ); + bytes.get_mut(first_offset..).ok_or_else(|| invalid_input("internal cell is outside page"))?.copy_from_slice(first); + PageHeader { + page_type: PageType::Internal, + cell_count: 1, + free_start: 44, + free_end: u16::try_from(first_offset).map_err(|_| invalid_input("internal free_end exceeds u16"))?, + left: 0, + right: 0, + } + .encode_into(&mut bytes, page_id, u64::MAX)?; + SlottedPage::open(bytes.as_mut_slice(), page_id, u64::MAX)?.rebuild_ordered(cells.iter().map(AsRef::as_ref))?; + Ok(bytes) +} + +fn allocate_overflow_chain(sink: &mut PageSink, stored: &[u8]) -> io::Result { + let count = stored.len().div_ceil(OVERFLOW_PAYLOAD_LEN); + if count == 0 { + return Err(invalid_input("overflow value must not be empty")); + } + let head = sink.allocate_many(count)?; + for (index, payload) in stored.chunks(OVERFLOW_PAYLOAD_LEN).enumerate() { + let page_id = head + .checked_add(u64::try_from(index).map_err(|_| invalid_input("overflow index exceeds u64"))?) + .ok_or_else(|| invalid_input("overflow page id overflow"))?; + let next = if index + 1 == count { + 0 + } else { + page_id.checked_add(1).ok_or_else(|| invalid_input("overflow page id overflow"))? + }; + let page = encode_overflow_page(page_id, u64::MAX, next, payload)?; + sink.write(page_id, &page)?; + } + Ok(head) +} + +fn finish_leaf(sink: &PageSink, page_id: u64, left: u64, right: u64, cells: &[Vec]) -> io::Result<()> { + let page = leaf_page(page_id, left, right, cells)?; + sink.write(page_id, &page) +} + +fn build_leaf_level(entries: &BTreeMap, sink: &mut PageSink) -> io::Result> +where + K: Ord + Serialize, + V: Serialize, +{ + let mut leaf_id = sink.allocate()?; + let mut left_leaf = 0; + let mut cells = Vec::>::new(); + let mut leaves = Vec::::new(); + let mut leaf_minimum = None; + let mut compression_scratch = Vec::new(); + let mut cell = Vec::new(); + + for (key, value) in entries { + let key_bytes = binary_serialize(key)?; + let raw_value = binary_serialize(value)?; + let logical_len = u32::try_from(raw_value.len()).map_err(|_| invalid_input("serialized value exceeds u32"))?; + let stored = encode_value(&raw_value, &mut compression_scratch)?; + let stored_bytes = stored.as_slice(); + let inline_footprint = SLOT_LEN + .checked_add(24) + .and_then(|size| size.checked_add(key_bytes.len())) + .and_then(|size| size.checked_add(stored_bytes.len())) + .ok_or_else(|| invalid_input("leaf cell footprint overflow"))?; + if stored_bytes.len() <= MAX_INLINE_STORED_VALUE && inline_footprint <= MAX_CELL_FOOTPRINT { + encode_inline_leaf_cell(&key_bytes, logical_len, stored.compression(), stored_bytes, &mut cell)?; + } else { + let head = allocate_overflow_chain(sink, stored_bytes)?; + encode_overflow_leaf_cell( + &key_bytes, + logical_len, + stored.compression(), + u32::try_from(stored_bytes.len()).map_err(|_| invalid_input("stored value exceeds u32"))?, + head, + stored_value_checksum(stored_bytes), + leaf_id, + u64::MAX, + &mut cell, + )?; + } + + let next_usage = checked_page_usage( + PAGE_HEADER_LEN, + cells.iter().map(Vec::as_slice).chain(std::iter::once(cell.as_slice())), + MAX_CELL_FOOTPRINT, + )?; + if next_usage > PAGE_SIZE { + let next_leaf = sink.allocate()?; + finish_leaf(sink, leaf_id, left_leaf, next_leaf, &cells)?; + leaves.push(NodeInfo { + page_id: leaf_id, + minimum_key: leaf_minimum.take().ok_or_else(|| invalid_input("non-empty leaf has no minimum key"))?, + }); + left_leaf = leaf_id; + leaf_id = next_leaf; + cells.clear(); + } + if cells.is_empty() { + leaf_minimum = Some(key_bytes); + } + cells.try_reserve(1).map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; + cells.push(std::mem::take(&mut cell)); + } + + finish_leaf(sink, leaf_id, left_leaf, 0, &cells)?; + leaves.push(NodeInfo { page_id: leaf_id, minimum_key: leaf_minimum.unwrap_or_default() }); + Ok(leaves) +} + +fn group_internal_children(level: &[NodeInfo]) -> io::Result> { + let mut groups = Vec::<(usize, usize)>::new(); + let mut start = 0usize; + while start < level.len() { + let mut end = start + 1; + let mut used = PAGE_HEADER_LEN + 8; + while end < level.len() { + let child = level.get(end).ok_or_else(|| invalid_input("internal child index is invalid"))?; + let cell_len = 12usize + .checked_add(child.minimum_key.len()) + .ok_or_else(|| invalid_input("internal cell length overflow"))?; + let next = used + .checked_add(SLOT_LEN) + .and_then(|size| size.checked_add(cell_len)) + .ok_or_else(|| invalid_input("internal page usage overflow"))?; + if next > PAGE_SIZE { + break; + } + used = next; + end += 1; + } + groups.push((start, end)); + start = end; + } + if groups.len() > 1 && groups.last().is_some_and(|(from, to)| to - from == 1) { + let last = groups.len() - 1; + let previous = groups + .get(last - 1) + .copied() + .ok_or_else(|| invalid_input("internal grouping is missing its previous group"))?; + let moved = previous.1.checked_sub(1).ok_or_else(|| invalid_input("internal grouping underflow"))?; + if moved <= previous.0 { + return Err(invalid_input("internal page cannot retain two children")); + } + groups.get_mut(last - 1).ok_or_else(|| invalid_input("internal grouping is missing its previous group"))?.1 = + moved; + groups.get_mut(last).ok_or_else(|| invalid_input("internal grouping is missing its last group"))?.0 = moved; + } + Ok(groups) +} + +fn build_parent_level(level: &[NodeInfo], sink: &mut PageSink) -> io::Result> { + let groups = group_internal_children(level)?; + let mut parents = Vec::with_capacity(groups.len()); + for (start, end) in groups { + let children = level.get(start..end).ok_or_else(|| invalid_input("internal child range is invalid"))?; + if children.len() < 2 { + return Err(invalid_input("internal page requires two children")); + } + let page_id = sink.allocate()?; + let (first_child, remaining_children) = + children.split_first().ok_or_else(|| invalid_input("internal page has no children"))?; + let mut encoded_cells = Vec::with_capacity(remaining_children.len()); + for child in remaining_children { + let mut encoded = Vec::new(); + encode_internal_cell(&child.minimum_key, child.page_id, page_id, u64::MAX, &mut encoded)?; + encoded_cells.push(encoded); + } + let encoded = internal_page(page_id, first_child.page_id, &encoded_cells)?; + sink.write(page_id, &encoded)?; + parents.push(NodeInfo { page_id, minimum_key: first_child.minimum_key.clone() }); + } + Ok(parents) +} + +/// Streams the whole tree into `sink` and returns the root page id. Page 0 is reserved +/// for the database header, which the caller writes last once `sink.next_page_id` is final. +pub(super) fn build_pages(entries: &BTreeMap, sink: &mut PageSink) -> io::Result +where + K: Ord + Serialize, + V: Serialize, +{ + let header_page_id = sink.allocate()?; + if header_page_id != 0 { + return Err(invalid_input("database header must be the first allocated page")); + } + let mut level = build_leaf_level(entries, sink)?; + if entries.is_empty() { + return Ok(level.first().ok_or_else(|| invalid_input("tree has no root page"))?.page_id); + } + while level.len() > 1 { + level = build_parent_level(&level, sink)?; + } + Ok(level.first().ok_or_else(|| invalid_input("tree has no root page"))?.page_id) +} + +pub(super) fn temporary_path(filepath: &Path) -> io::Result { + let name = filepath.file_name().ok_or_else(|| invalid_input("database path has no file name"))?.to_string_lossy(); + Ok(filepath.with_file_name(format!("{name}.{}.v3.tmp", uuid::Uuid::new_v4()))) +} + +pub(in crate::v3) fn publish_database( + temporary: &Path, + destination: &Path, + sync_directory: impl FnOnce(&Path) -> io::Result<()>, +) -> Result<(), PublishDatabaseError> { + let temporary = match tempfile::TempPath::try_from_path(temporary) { + Ok(path) => path, + Err(error) => { + let _ = std::fs::remove_file(temporary); + return Err(PublishDatabaseError::NotPublished(error)); + } + }; + temporary.persist(destination).map_err(io::Error::from).map_err(PublishDatabaseError::NotPublished)?; + sync_directory(destination).map_err(|error| { + PublishDatabaseError::PublishedDurabilityUnknown(io::Error::new( + error.kind(), + format!("database published but directory sync failed; durability unknown: {error}"), + )) + }) +} + +/// Invalidates the sorted index whenever publication made the replacement database visible. +pub(in crate::v3) fn publish_database_and_invalidate_sorted_index( + temporary: &Path, + destination: &Path, + sync_directory: impl FnOnce(&Path) -> io::Result<()>, +) -> io::Result<()> { + let publish_error = match publish_database(temporary, destination, sync_directory) { + Ok(()) => None, + Err(error) if error.database_was_published() => Some(io::Error::from(error)), + Err(error) => return Err(io::Error::from(error)), + }; + let invalidation_error = invalidate_sorted_index(destination).err(); + match (publish_error, invalidation_error) { + (Some(publish_error), Some(invalidation_error)) => Err(io::Error::new( + publish_error.kind(), + format!( + "database {} was published with unknown directory durability: {publish_error}; its previous sorted index could not be invalidated: {invalidation_error}", + destination.display() + ), + )), + (Some(publish_error), None) => Err(io::Error::new( + publish_error.kind(), + format!( + "database {} was published and its previous sorted index was invalidated, but publication durability remains unknown: {publish_error}", + destination.display() + ), + )), + (None, Some(error)) => Err(io::Error::new( + error.kind(), + format!( + "database {} was published, but its previous sorted index could not be invalidated: {error}", + destination.display() + ), + )), + (None, None) => Ok(()), + } +} + +pub(super) struct StoredDatabase { + pub(super) root_page_id: u64, + pub(super) verification: VerificationReport, +} + +pub(super) fn verify_leaf_page( + query: &mut BPlusTreeQuery, + page_id: u64, + lower: Option<&K>, + upper: Option<&K>, + overflow_pages: &mut HashSet, +) -> io::Result<(VerifyLeaf, u64)> +where + K: Ord + for<'de> Deserialize<'de> + Clone, +{ + let next_page_id = query.header.next_page_id; + let (left, right, count) = query.with_page(page_id, |bytes, _| { + let page = SlottedPage::open(bytes, page_id, next_page_id)?; + Ok((page.header().left, page.header().right, usize::from(page.header().cell_count))) + })?; + let mut minimum = None; + let mut maximum = None; + let mut live_entries = 0u64; + for index in 0..count { + let (key, value) = query.with_page(page_id, |bytes, _| { + let page = SlottedPage::open(bytes, page_id, next_page_id)?; + let cell = LeafCellRef::decode(page.cell(index)?, page_id, next_page_id)?; + let value = match cell.value { + LeafValueRef::Inline { .. } => VerifyValue::Inline, + LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } => { + VerifyValue::Overflow(compression, logical_len, stored_len, head, crc32) + } + LeafValueRef::Tombstone => VerifyValue::Tombstone, + }; + Ok((binary_deserialize::(cell.key_bytes)?, value)) + })?; + if maximum.as_ref().is_some_and(|previous| previous >= &key) { + return Err(invalid_data("leaf keys are not strictly ordered")); + } + if lower.is_some_and(|bound| &key < bound) || upper.is_some_and(|bound| &key >= bound) { + return Err(invalid_data("leaf key is outside parent separator range")); + } + minimum.get_or_insert_with(|| key.clone()); + maximum = Some(key); + match value { + VerifyValue::Tombstone => continue, + VerifyValue::Inline => {} + VerifyValue::Overflow(compression, logical_len, stored_len, head, crc32) => query.assemble_overflow_chain( + compression, + logical_len, + stored_len, + head, + crc32, + Some(overflow_pages), + )?, + } + live_entries = live_entries.checked_add(1).ok_or_else(|| invalid_data("live entry count overflow"))?; + } + Ok((VerifyLeaf { page_id, left, right, minimum, maximum }, live_entries)) +} + +pub(super) fn verify_internal_page( + query: &mut BPlusTreeQuery, + page_id: u64, +) -> io::Result)>> +where + K: Ord + for<'de> Deserialize<'de> + Clone, +{ + let next_page_id = query.header.next_page_id; + query.with_page(page_id, |bytes, _| { + let page = SlottedPage::open(bytes, page_id, next_page_id)?; + let mut children = Vec::with_capacity(usize::from(page.header().cell_count) + 1); + children.push((InternalPreamble::decode(bytes, page_id, next_page_id)?.leftmost_child, None)); + let mut previous = None; + for cell in page.cells() { + let cell = InternalCellRef::decode(cell?, page_id, next_page_id)?; + let key = binary_deserialize::(cell.key_bytes)?; + if previous.as_ref().is_some_and(|prior| prior >= &key) { + return Err(invalid_data("internal separators are not strictly ordered")); + } + previous = Some(key.clone()); + children.push((cell.right_child, Some(key))); + } + Ok(children) + }) +} diff --git a/backend/btree/src/v3/tree/error.rs b/backend/btree/src/v3/tree/error.rs new file mode 100644 index 000000000..dbd09d140 --- /dev/null +++ b/backend/btree/src/v3/tree/error.rs @@ -0,0 +1,27 @@ +#[derive(Debug)] +pub(in crate::v3) enum PublishDatabaseError { + NotPublished(io::Error), + PublishedDurabilityUnknown(io::Error), +} + +use std::io; + +pub(super) fn invalid_data(message: &'static str) -> io::Error { io::Error::new(io::ErrorKind::InvalidData, message) } + +pub(super) fn invalid_input(message: &'static str) -> io::Error { io::Error::new(io::ErrorKind::InvalidInput, message) } + +impl PublishDatabaseError { + pub(in crate::v3) const fn database_was_published(&self) -> bool { + matches!(self, Self::PublishedDurabilityUnknown(_)) + } + + fn into_io(self) -> io::Error { + match self { + Self::NotPublished(error) | Self::PublishedDurabilityUnknown(error) => error, + } + } +} + +impl From for io::Error { + fn from(error: PublishDatabaseError) -> Self { error.into_io() } +} diff --git a/backend/btree/src/v3/tree/iterators.rs b/backend/btree/src/v3/tree/iterators.rs new file mode 100644 index 000000000..471ee80e5 --- /dev/null +++ b/backend/btree/src/v3/tree/iterators.rs @@ -0,0 +1,259 @@ +use super::{ + invalid_data, search_leaf, BPlusTreeDiskIterator, BPlusTreeDiskIteratorOwned, BPlusTreeQuery, + BPlusTreeRangeIterator, PageType, SlottedPage, +}; +use serde::Deserialize; +use std::{ + collections::HashSet, + io, + ops::{Bound, Range}, +}; + +pub(super) struct CursorState { + pub(super) leaf_page_id: u64, + pub(super) slot_index: usize, + pub(super) cell_ranges: Vec>, + pub(super) right_sibling: u64, + pub(super) expected_left_sibling: Option, + pub(super) page_loaded: bool, + pub(super) start: Bound, + pub(super) initialized: bool, + pub(super) finished: bool, + pub(super) visited_leaves: HashSet, + pub(super) pending: Option<(K, V)>, +} + +impl CursorState { + pub(super) fn new(start: Bound) -> Self { + Self { + leaf_page_id: 0, + slot_index: 0, + cell_ranges: Vec::new(), + right_sibling: 0, + expected_left_sibling: None, + page_loaded: false, + start, + initialized: false, + finished: false, + visited_leaves: HashSet::new(), + pending: None, + } + } +} + +fn load_cursor_page(query: &mut BPlusTreeQuery, state: &mut CursorState) -> io::Result<()> { + let page_id = state.leaf_page_id; + let next_page_id = query.header.next_page_id; + let expected_left = state.expected_left_sibling; + let mut cell_ranges = std::mem::take(&mut state.cell_ranges); + let right_sibling = query.with_page(page_id, |bytes, _| { + let page = SlottedPage::open(bytes, page_id, next_page_id)?; + if page.header().page_type != PageType::Leaf { + return Err(invalid_data("iterator sibling is not a leaf")); + } + if expected_left.is_some_and(|left| page.header().left != left) { + return Err(invalid_data("asymmetric leaf sibling link")); + } + let count = usize::from(page.header().cell_count); + cell_ranges.clear(); + cell_ranges.try_reserve(count).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + for index in 0..count { + cell_ranges.push(page.cell_range(index)?); + } + Ok(page.header().right) + })?; + state.cell_ranges = cell_ranges; + state.right_sibling = right_sibling; + state.page_loaded = true; + Ok(()) +} + +pub(super) fn record_page_visit(visited: &mut HashSet, page_id: u64, cycle_error: &'static str) -> io::Result<()> { + visited.try_reserve(1).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + if !visited.insert(page_id) { + return Err(invalid_data(cycle_error)); + } + Ok(()) +} + +fn cursor_next(query: &mut BPlusTreeQuery, state: &mut CursorState) -> Option> +where + K: Ord + for<'de> Deserialize<'de>, + V: for<'de> Deserialize<'de>, +{ + if let Some(entry) = state.pending.take() { + return Some(Ok(entry)); + } + if state.finished { + return None; + } + let mut entry_error = false; + let result = (|| { + if !state.initialized { + state.leaf_page_id = match &state.start { + Bound::Included(key) | Bound::Excluded(key) => query.locate_leaf(key)?, + Bound::Unbounded => query.leftmost_leaf()?, + }; + record_page_visit(&mut state.visited_leaves, state.leaf_page_id, "right sibling chain contains a cycle")?; + if let Bound::Included(key) | Bound::Excluded(key) = &state.start { + let page_id = state.leaf_page_id; + let next_page_id = query.header.next_page_id; + state.slot_index = query.with_page(page_id, |bytes, _| { + let page = SlottedPage::open(bytes, page_id, next_page_id)?; + let index = match search_leaf(&page, key)? { + Ok(index) if matches!(state.start, Bound::Excluded(_)) => index + 1, + Ok(index) | Err(index) => index, + }; + Ok(index) + })?; + } + state.initialized = true; + } + loop { + if !state.page_loaded { + load_cursor_page(query, state)?; + } + let page_id = state.leaf_page_id; + while state.slot_index < state.cell_ranges.len() { + let index = state.slot_index; + state.slot_index += 1; + match query.decode_entry_range(page_id, state.cell_ranges[index].clone()) { + Ok(Some(entry)) => return Ok(Some(entry)), + Ok(None) => {} + Err(error) => { + entry_error = true; + return Err(error); + } + } + } + let right = state.right_sibling; + if right == 0 { + return Ok(None); + } + record_page_visit(&mut state.visited_leaves, right, "right sibling chain contains a cycle")?; + state.expected_left_sibling = Some(page_id); + state.leaf_page_id = right; + state.slot_index = 0; + state.page_loaded = false; + state.cell_ranges.clear(); + } + })(); + match result { + Ok(Some(entry)) => Some(Ok(entry)), + Ok(None) => { + state.finished = true; + None + } + Err(err) => { + state.finished = !entry_error; + Some(Err(err)) + } + } +} + +impl<'a, K, V> BPlusTreeDiskIterator<'a, K, V> { + pub(super) fn new(query: &'a mut BPlusTreeQuery) -> Self { + Self { query, state: CursorState::new(Bound::Unbounded) } + } + + pub(super) fn from_bound(query: &'a mut BPlusTreeQuery, start: Bound) -> Self { + Self { query, state: CursorState::new(start) } + } +} + +impl BPlusTreeDiskIterator<'_, K, V> +where + K: Ord + for<'de> Deserialize<'de>, + V: for<'de> Deserialize<'de>, +{ + pub fn try_is_empty(&mut self) -> io::Result { + match self.next() { + None => Ok(true), + Some(Ok(entry)) => { + self.state.pending = Some(entry); + Ok(false) + } + Some(Err(err)) => Err(err), + } + } +} + +impl Iterator for BPlusTreeDiskIterator<'_, K, V> +where + K: Ord + for<'de> Deserialize<'de>, + V: for<'de> Deserialize<'de>, +{ + type Item = io::Result<(K, V)>; + + fn next(&mut self) -> Option { cursor_next(self.query, &mut self.state) } +} + +impl BPlusTreeDiskIteratorOwned { + pub(super) fn new(query: BPlusTreeQuery) -> Self { Self { query, state: CursorState::new(Bound::Unbounded) } } +} + +impl BPlusTreeDiskIteratorOwned +where + K: Ord + for<'de> Deserialize<'de>, + V: for<'de> Deserialize<'de>, +{ + pub fn try_is_empty(&mut self) -> io::Result { + match self.next() { + None => Ok(true), + Some(Ok(entry)) => { + self.state.pending = Some(entry); + Ok(false) + } + Some(Err(err)) => Err(err), + } + } +} + +impl Iterator for BPlusTreeDiskIteratorOwned +where + K: Ord + for<'de> Deserialize<'de>, + V: for<'de> Deserialize<'de>, +{ + type Item = io::Result<(K, V)>; + + fn next(&mut self) -> Option { cursor_next(&mut self.query, &mut self.state) } +} + +fn within_start(key: &K, bound: &Bound) -> bool { + match bound { + Bound::Included(start) => key >= start, + Bound::Excluded(start) => key > start, + Bound::Unbounded => true, + } +} + +fn past_end(key: &K, bound: &Bound) -> bool { + match bound { + Bound::Included(end) => key > end, + Bound::Excluded(end) => key >= end, + Bound::Unbounded => false, + } +} + +impl Iterator for BPlusTreeRangeIterator<'_, K, V> +where + K: Ord + for<'de> Deserialize<'de>, + V: for<'de> Deserialize<'de>, +{ + type Item = io::Result<(K, V)>; + + fn next(&mut self) -> Option { + loop { + let entry = self.iterator.next()?; + match entry { + Ok((key, _value)) if past_end(&key, &self.end) => { + self.iterator.state.finished = true; + return None; + } + Ok((key, value)) if within_start(&key, &self.start) => return Some(Ok((key, value))), + Ok(_) => {} + Err(err) => return Some(Err(err)), + } + } + } +} diff --git a/backend/btree/src/v3/tree/memory.rs b/backend/btree/src/v3/tree/memory.rs new file mode 100644 index 000000000..8da92e89e --- /dev/null +++ b/backend/btree/src/v3/tree/memory.rs @@ -0,0 +1,221 @@ +use super::{ + build_pages, invalid_data, invalid_input, publish_database, publish_database_and_invalidate_sorted_index, + recover_pending_under_existing_lock, sync_parent_directory, temporary_path, verify_full, with_exclusive_sidecar, + BPlusTree, BPlusTreeMetadata, BPlusTreeQuery, DatabaseHeader, PageSink, StoredDatabase, VerificationReport, +}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::BTreeMap, + fs::{File, OpenOptions}, + io, + path::Path, +}; + +impl Default for BPlusTree { + fn default() -> Self { Self::new() } +} + +impl BPlusTree { + pub const fn new() -> Self { Self { entries: BTreeMap::new(), metadata: BPlusTreeMetadata::Empty, dirty: true } } + + pub fn get_metadata(&self) -> &BPlusTreeMetadata { &self.metadata } + + pub fn set_metadata(&mut self, metadata: BPlusTreeMetadata) { + self.metadata = metadata; + self.dirty = true; + } + + pub fn is_empty(&self) -> bool { self.entries.is_empty() } + + pub fn len(&self) -> usize { self.entries.len() } + + pub fn insert(&mut self, key: K, value: V) { + let _ = self.entries.insert(key, value); + self.dirty = true; + } + + pub fn query(&self, key: &K) -> Option<&V> { self.entries.get(key) } + + pub fn find_le(&self, key: &K) -> Option<(&K, &V)> { self.entries.range(..=key).next_back() } + + pub fn iter(&self) -> std::collections::btree_map::Iter<'_, K, V> { self.entries.iter() } +} + +impl<'a, K: Ord, V> IntoIterator for &'a BPlusTree { + type Item = (&'a K, &'a V); + type IntoIter = std::collections::btree_map::Iter<'a, K, V>; + + fn into_iter(self) -> Self::IntoIter { self.entries.iter() } +} + +impl BPlusTree +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de>, +{ + pub fn store(&mut self, filepath: &Path) -> io::Result { + with_exclusive_sidecar(filepath, || { + recover_pending_under_existing_lock(filepath)?; + if self.dirty { + self.store_exclusive(filepath).map(|stored| stored.root_page_id) + } else { + Ok(0) + } + }) + } + + pub(crate) fn store_verified(&mut self, filepath: &Path) -> io::Result { + with_exclusive_sidecar(filepath, || { + recover_pending_under_existing_lock(filepath)?; + if !self.dirty { + return Err(invalid_input("verified store requires a dirty tree")); + } + self.store_exclusive(filepath).map(|stored| stored.verification) + }) + } + + pub(super) fn store_exclusive(&mut self, filepath: &Path) -> io::Result { + self.store_exclusive_with_directory_sync(filepath, sync_parent_directory) + } + + pub(super) fn store_exclusive_with_directory_sync( + &mut self, + filepath: &Path, + sync_published_directory: impl FnOnce(&Path) -> io::Result<()>, + ) -> io::Result { + let temp_path = temporary_path(filepath)?; + // The builder streams into the temp file, so it exists from here on — every error + // path below has to remove it again. + let prepared = (|| { + let mut sink = PageSink::new(OpenOptions::new().write(true).create_new(true).open(&temp_path)?); + let root_page_id = build_pages(&self.entries, &mut sink)?; + let header = DatabaseHeader { + root_page_id, + next_page_id: sink.next_page_id, + free_page_head: 0, + generation: 1, + database_id: *uuid::Uuid::new_v4().as_bytes(), + metadata: self.metadata.clone(), + } + .encode()?; + sink.write(0, &header)?; + sink.file.sync_all()?; + drop(sink); + let mut query = BPlusTreeQuery::::from_file_unlocked(File::open(&temp_path)?)?; + let verification = verify_full(&mut query)?; + drop(query); + Ok((root_page_id, verification)) + })(); + let (root_page_id, verification) = match prepared { + Ok(prepared) => prepared, + Err(error) => { + let _ = std::fs::remove_file(&temp_path); + return Err(error); + } + }; + publish_database_and_invalidate_sorted_index(&temp_path, filepath, sync_published_directory)?; + self.dirty = false; + Ok(StoredDatabase { root_page_id, verification }) + } + + pub fn store_with_index(&mut self, filepath: &Path, sort_key_extractor: F) -> io::Result + where + SortKey: Ord + Serialize, + F: Fn(&V) -> SortKey, + { + self.store_with_index_result(filepath, sort_key_extractor) + .map(|stored| stored.map_or(0, |stored| stored.root_page_id)) + } + + pub(crate) fn store_with_index_verified( + &mut self, + filepath: &Path, + sort_key_extractor: F, + ) -> io::Result + where + SortKey: Ord + Serialize, + F: Fn(&V) -> SortKey, + { + self.store_with_index_result(filepath, sort_key_extractor)? + .map(|stored| stored.verification) + .ok_or_else(|| invalid_input("verified indexed store requires a dirty tree")) + } + + fn store_with_index_result( + &mut self, + filepath: &Path, + sort_key_extractor: F, + ) -> io::Result> + where + SortKey: Ord + Serialize, + F: Fn(&V) -> SortKey, + { + with_exclusive_sidecar(filepath, || { + recover_pending_under_existing_lock(filepath)?; + if !self.dirty { + return Ok(None); + } + let stored = self.store_exclusive(filepath)?; + Self::store_index_exclusive(filepath, sort_key_extractor, stored.verification.live_entries)?; + Ok(Some(stored)) + }) + } + + fn store_index_exclusive( + filepath: &Path, + sort_key_extractor: F, + expected_entries: u64, + ) -> io::Result<()> + where + SortKey: Ord + Serialize, + F: Fn(&V) -> SortKey, + { + let mut query = BPlusTreeQuery::::from_file_unlocked(File::open(filepath)?)?; + let (database_id, generation) = query.snapshot_identity(); + let mut entries = query + .collect_with_locators()? + .into_iter() + .map(|(key, value, locator)| (sort_key_extractor(&value), key, locator)) + .collect::>(); + if u64::try_from(entries.len()).map_err(|_| invalid_data("sorted-index entry count exceeds u64"))? + != expected_entries + { + return Err(invalid_data("sorted-index source entry count differs from verified database")); + } + entries.sort_unstable_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1))); + drop(query); + + let index_path = crate::common::get_file_path_for_db_index(filepath); + let temporary = temporary_path(&index_path)?; + let prepared = (|| { + let mut writer = crate::sorted_index::v4::Writer::new(&temporary, database_id, generation)?; + for (sort_key, primary_key, locator) in &entries { + writer.push(sort_key, primary_key, *locator)?; + } + let _ = writer.finish()?; + Ok(()) + })(); + if let Err(error) = prepared { + let _ = std::fs::remove_file(&temporary); + return Err(error); + } + publish_database(&temporary, &index_path, sync_parent_directory).map_err(io::Error::from) + } +} + +impl BPlusTree +where + K: Ord + for<'de> Deserialize<'de>, + V: for<'de> Deserialize<'de>, +{ + pub fn load(filepath: &Path) -> io::Result { + let mut query = BPlusTreeQuery::::try_new(filepath)?; + let metadata = query.header.metadata.clone(); + let mut entries = BTreeMap::new(); + for entry in query.iter() { + let (key, value) = entry?; + let _ = entries.insert(key, value); + } + Ok(Self { entries, metadata, dirty: false }) + } +} diff --git a/backend/btree/src/v3/tree/mutation.rs b/backend/btree/src/v3/tree/mutation.rs new file mode 100644 index 000000000..bee31f573 --- /dev/null +++ b/backend/btree/src/v3/tree/mutation.rs @@ -0,0 +1,582 @@ +use super::{ + decode_key, encode_inline_leaf_cell, encode_internal_cell, encode_overflow_leaf_cell, encode_overflow_page, + encode_tombstone_leaf_cell, internal_page, invalid_data, invalid_input, leaf_page, overflow_payload, search_leaf, + stored_value_checksum, Compression, InternalCellRef, InternalPreamble, InternalSplit, LeafCellRef, LeafValueRef, + PageType, SlottedPage, Visit, WriteTransaction, MAX_CELL_FOOTPRINT, MAX_INLINE_STORED_VALUE, OVERFLOW_PAYLOAD_LEN, + PAGE_HEADER_LEN, PAGE_SIZE, SLOT_LEN, +}; +use serde::Deserialize; +use std::{collections::HashSet, io}; + +pub(super) fn checked_page_usage<'a, I>(base: usize, cells: I, maximum_cell_footprint: usize) -> io::Result +where + I: IntoIterator, +{ + cells.into_iter().try_fold(base, |used, cell| { + let footprint = SLOT_LEN.checked_add(cell.len()).ok_or_else(|| invalid_input("cell footprint overflow"))?; + if footprint > maximum_cell_footprint { + return Err(invalid_input("cell footprint exceeds format limit")); + } + used.checked_add(footprint).ok_or_else(|| invalid_input("page usage overflow")) + }) +} + +pub(crate) fn used_leaf_bytes>(cells: &[T]) -> io::Result { + checked_page_usage(PAGE_HEADER_LEN, cells.iter().map(AsRef::as_ref), MAX_CELL_FOOTPRINT) +} + +pub(crate) fn choose_leaf_split>(cells: &[T]) -> io::Result { + if cells.len() < 2 { + return Err(invalid_input("leaf split requires two non-empty outputs")); + } + let mut total = 0usize; + for cell in cells { + let footprint = + SLOT_LEN.checked_add(cell.as_ref().len()).ok_or_else(|| invalid_input("leaf cell footprint overflow"))?; + if footprint > MAX_CELL_FOOTPRINT { + return Err(invalid_input("leaf cell footprint exceeds format limit")); + } + total = total.checked_add(footprint).ok_or_else(|| invalid_input("leaf split usage overflow"))?; + } + + let mut left_payload = 0usize; + let mut best = None; + for boundary in 1..cells.len() { + let cell = cells.get(boundary - 1).ok_or_else(|| invalid_input("leaf split boundary is outside cells"))?; + let footprint = + SLOT_LEN.checked_add(cell.as_ref().len()).ok_or_else(|| invalid_input("leaf cell footprint overflow"))?; + left_payload = left_payload.checked_add(footprint).ok_or_else(|| invalid_input("leaf split usage overflow"))?; + let right_payload = + total.checked_sub(left_payload).ok_or_else(|| invalid_input("leaf split usage underflow"))?; + let left_used = + PAGE_HEADER_LEN.checked_add(left_payload).ok_or_else(|| invalid_input("leaf split usage overflow"))?; + let right_used = + PAGE_HEADER_LEN.checked_add(right_payload).ok_or_else(|| invalid_input("leaf split usage overflow"))?; + if left_used <= PAGE_SIZE && right_used <= PAGE_SIZE { + let imbalance = left_used.abs_diff(right_used); + if best.is_none_or(|(_, best_imbalance)| imbalance < best_imbalance) { + best = Some((boundary, imbalance)); + } + } + } + best.map(|(boundary, _)| boundary).ok_or_else(|| invalid_input("leaf split has no valid boundary")) +} + +fn used_internal_bytes<'a, I>(cells: I) -> io::Result +where + I: IntoIterator, +{ + checked_page_usage(PAGE_HEADER_LEN + 8, cells, SLOT_LEN + 12 + 2004) +} + +pub(crate) fn choose_internal_split>( + cells: &[T], + page_id: u64, + next_page_id: u64, +) -> io::Result> { + if cells.len() < 3 { + return Err(invalid_input("internal split requires two non-empty outputs and a promoted separator")); + } + for cell in cells { + InternalCellRef::decode(cell.as_ref(), page_id, next_page_id)?; + } + + let mut best = None; + for promoted_index in 1..cells.len() - 1 { + let left_used = used_internal_bytes(cells[..promoted_index].iter().map(AsRef::as_ref))?; + let right_used = used_internal_bytes(cells[promoted_index + 1..].iter().map(AsRef::as_ref))?; + if left_used <= PAGE_SIZE && right_used <= PAGE_SIZE { + let imbalance = left_used.abs_diff(right_used); + if best.is_none_or(|(_, best_imbalance)| imbalance < best_imbalance) { + best = Some((promoted_index, imbalance)); + } + } + } + let promoted_index = + best.map(|(index, _)| index).ok_or_else(|| invalid_input("internal split has no valid boundary"))?; + let promoted = InternalCellRef::decode( + cells.get(promoted_index).ok_or_else(|| invalid_input("promoted separator is outside cells"))?.as_ref(), + page_id, + next_page_id, + )?; + Ok(InternalSplit { + #[cfg(test)] + promoted_index, + right_leftmost_child: promoted.right_child, + promoted, + left_cells: cells.get(..promoted_index).ok_or_else(|| invalid_input("left split range is outside cells"))?, + right_cells: cells + .get(promoted_index + 1..) + .ok_or_else(|| invalid_input("right split range is outside cells"))?, + }) +} + +#[cfg(test)] +pub(super) fn database_page(database: &[u8], page_id: u64, next_page_id: u64) -> io::Result<&[u8]> { + if page_id == 0 || page_id >= next_page_id { + return Err(invalid_data("overflow page id is outside database")); + } + let page_id = usize::try_from(page_id).map_err(|_| invalid_data("overflow page id exceeds usize"))?; + let offset = page_id.checked_mul(PAGE_SIZE).ok_or_else(|| invalid_data("overflow page offset overflow"))?; + let end = offset.checked_add(PAGE_SIZE).ok_or_else(|| invalid_data("overflow page end overflow"))?; + database.get(offset..end).ok_or_else(|| invalid_data("truncated overflow page")) +} + +pub(super) fn minimum_tree_key( + transaction: &WriteTransaction, + base: &[u8], + mut page_id: u64, +) -> io::Result>> { + let mut visited = HashSet::new(); + loop { + if !visited.insert(page_id) { + return Err(invalid_data("minimum-key descent contains a cycle")); + } + let page = SlottedPage::open(transaction.page(base, page_id)?, page_id, transaction.next_header.next_page_id)?; + match page.header().page_type { + PageType::Leaf => { + let Some(cell) = page.cells().next() else { return Ok(None) }; + return LeafCellRef::decode(cell?, page_id, transaction.next_header.next_page_id) + .map(|cell| Some(cell.key_bytes.to_vec())); + } + PageType::Internal => { + page_id = InternalPreamble::decode(page.as_bytes(), page_id, transaction.next_header.next_page_id)? + .leftmost_child; + } + PageType::Overflow | PageType::Free => return Err(invalid_data("tree references a non-tree page")), + } + } +} + +pub(super) fn validate_leaf_backlink( + transaction: &WriteTransaction, + base: &[u8], + page_id: u64, + sibling_id: u64, + sibling_points_left: bool, +) -> io::Result<()> { + if sibling_id == 0 { + return Ok(()); + } + let sibling = + SlottedPage::open(transaction.page(base, sibling_id)?, sibling_id, transaction.next_header.next_page_id)?; + let backlink = if sibling_points_left { sibling.header().left } else { sibling.header().right }; + if sibling.header().page_type != PageType::Leaf || backlink != page_id { + return Err(invalid_data("asymmetric leaf sibling link")); + } + Ok(()) +} + +#[derive(Debug)] +struct Promotion { + key: Vec, + right_child: u64, +} + +fn internal_child_position Deserialize<'de>, B: AsRef<[u8]>>( + page: &SlottedPage, + key: &K, +) -> io::Result<(usize, u64)> { + let mut left = 0usize; + let mut right = usize::from(page.header().cell_count); + while left < right { + let middle = left + (right - left) / 2; + let cell = InternalCellRef::decode(page.cell(middle)?, page.page_id(), page.next_page_id())?; + if decode_key::(cell.key_bytes)? <= *key { + left = middle + 1; + } else { + right = middle; + } + } + let child = if left == 0 { + InternalPreamble::decode(page.as_bytes(), page.page_id(), page.next_page_id())?.leftmost_child + } else { + InternalCellRef::decode(page.cell(left - 1)?, page.page_id(), page.next_page_id())?.right_child + }; + Ok((left, child)) +} + +pub(super) fn locate_transaction_leaf Deserialize<'de>>( + transaction: &WriteTransaction, + base: &[u8], + key: &K, +) -> io::Result<(u64, Vec<(u64, usize)>)> { + let mut page_id = transaction.next_header.root_page_id; + let mut path = Vec::new(); + let mut visited = HashSet::new(); + loop { + if !visited.insert(page_id) { + return Err(invalid_data("tree descent contains a cycle")); + } + let page = SlottedPage::open(transaction.page(base, page_id)?, page_id, transaction.next_header.next_page_id)?; + match page.header().page_type { + PageType::Leaf => return Ok((page_id, path)), + PageType::Internal => { + let (position, child) = internal_child_position(&page, key)?; + path.try_reserve(1).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + path.push((page_id, position)); + page_id = child; + } + PageType::Overflow | PageType::Free => return Err(invalid_data("tree references a non-tree page")), + } + } +} + +pub(super) fn overflow_chain_pages( + transaction: &WriteTransaction, + base: &[u8], + mut page_id: u64, +) -> io::Result> { + let mut pages = Vec::new(); + let mut visited = HashSet::new(); + while page_id != 0 { + if !visited.insert(page_id) { + return Err(invalid_data("overflow chain contains a cycle")); + } + let page = SlottedPage::open(transaction.page(base, page_id)?, page_id, transaction.next_header.next_page_id)?; + if page.header().page_type != PageType::Overflow || overflow_payload(&page)?.is_empty() { + return Err(invalid_data("invalid overflow chain page")); + } + pages.try_reserve(1).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + pages.push(page_id); + page_id = page.header().right; + } + if pages.is_empty() { + return Err(invalid_data("overflow chain is empty")); + } + Ok(pages) +} + +pub(super) fn validated_overflow_chain_pages( + transaction: &WriteTransaction, + base: &[u8], + head: u64, + stored_len: u32, + crc32: u32, +) -> io::Result> { + let pages = overflow_chain_pages(transaction, base, head)?; + let expected = usize::try_from(stored_len).map_err(|_| invalid_data("stored length exceeds usize"))?; + let mut actual = 0usize; + let mut hasher = crc32fast::Hasher::new(); + for page_id in &pages { + let page = + SlottedPage::open(transaction.page(base, *page_id)?, *page_id, transaction.next_header.next_page_id)?; + let payload = overflow_payload(&page)?; + actual = actual.checked_add(payload.len()).ok_or_else(|| invalid_data("overflow chain length overflow"))?; + if actual > expected { + return Err(invalid_data("overflow chain exceeds declared length")); + } + hasher.update(payload); + } + if actual != expected || hasher.finalize() != crc32 { + return Err(invalid_data("overflow value checksum or length mismatch")); + } + Ok(pages) +} + +pub(super) fn free_pages(transaction: &mut WriteTransaction, pages: &[u64]) -> io::Result<()> { + for page_id in pages.iter().rev() { + transaction.free_page(*page_id)?; + } + Ok(()) +} + +fn write_overflow_chain( + transaction: &mut WriteTransaction, + base: &[u8], + stored: &[u8], + old_pages: &[u64], +) -> io::Result { + let required = stored.len().div_ceil(OVERFLOW_PAYLOAD_LEN); + if required == 0 { + return Err(invalid_input("overflow value must not be empty")); + } + let mut pages = Vec::new(); + pages.try_reserve_exact(required).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + if old_pages.len() >= required { + pages.extend_from_slice( + old_pages.get(..required).ok_or_else(|| invalid_data("overflow reuse range is invalid"))?, + ); + free_pages( + transaction, + old_pages.get(required..).ok_or_else(|| invalid_data("overflow tail range is invalid"))?, + )?; + } else { + for _ in 0..required { + pages.push(transaction.allocate_page(base)?); + } + free_pages(transaction, old_pages)?; + } + for (index, payload) in stored.chunks(OVERFLOW_PAYLOAD_LEN).enumerate() { + let page_id = *pages.get(index).ok_or_else(|| invalid_data("overflow page allocation is missing"))?; + let next = pages.get(index + 1).copied().unwrap_or(0); + transaction + .write_page(page_id, encode_overflow_page(page_id, transaction.next_header.next_page_id, next, payload)?)?; + } + pages.first().copied().ok_or_else(|| invalid_data("overflow head is missing")) +} + +fn repair_right_leaf_backlink( + transaction: &mut WriteTransaction, + base: &[u8], + right_page_id: u64, + left_page_id: u64, +) -> io::Result<()> { + if right_page_id == 0 { + return Ok(()); + } + let next_page_id = transaction.next_header.next_page_id; + let page = transaction.page_mut(base, right_page_id)?; + let mut header = SlottedPage::open(page.as_slice(), right_page_id, next_page_id)?.header(); + if header.page_type != PageType::Leaf { + return Err(invalid_data("leaf sibling references a non-leaf page")); + } + header.left = left_page_id; + header.encode_into(page, right_page_id, next_page_id) +} + +fn mutate_leaf( + transaction: &mut WriteTransaction, + base: &[u8], + leaf_id: u64, + index: usize, + replace: bool, + cell: &[u8], +) -> io::Result> { + let snapshot = transaction.page_copy(base, leaf_id)?; + let page = SlottedPage::open(snapshot.as_slice(), leaf_id, transaction.next_header.next_page_id)?; + if page.header().page_type != PageType::Leaf { + return Err(invalid_data("mutation target is not a leaf page")); + } + let count = usize::from(page.header().cell_count); + if (replace && index >= count) || (!replace && index > count) { + return Err(invalid_input("leaf mutation index is outside page")); + } + if replace && page.cell(index)?.len() == cell.len() { + let next_page_id = transaction.next_header.next_page_id; + let dirty = transaction.page_mut(base, leaf_id)?; + return SlottedPage::open(dirty.as_mut_slice(), leaf_id, next_page_id)? + .replace_same_len(index, cell) + .map(|()| None); + } + let mut cells = Vec::<&[u8]>::new(); + cells + .try_reserve_exact(count + usize::from(!replace)) + .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + for current in 0..count { + if current == index { + cells.push(cell); + if replace { + continue; + } + } + cells.push(page.cell(current)?); + } + if index == count { + cells.push(cell); + } + if used_leaf_bytes(&cells)? <= PAGE_SIZE { + let next_page_id = transaction.next_header.next_page_id; + let dirty = transaction.page_mut(base, leaf_id)?; + SlottedPage::open(dirty.as_mut_slice(), leaf_id, next_page_id)?.rebuild_ordered(cells.iter().copied())?; + return Ok(None); + } + + let new_leaf_id = transaction.allocate_page(base)?; + let boundary = choose_leaf_split(&cells)?; + let (left_cells, right_cells) = cells.split_at(boundary); + let right = page.header().right; + let left_page = leaf_page(leaf_id, page.header().left, new_leaf_id, left_cells)?; + let right_page = leaf_page(new_leaf_id, leaf_id, right, right_cells)?; + let separator = LeafCellRef::decode( + right_cells.first().ok_or_else(|| invalid_data("split right leaf is empty"))?, + new_leaf_id, + transaction.next_header.next_page_id, + )? + .key_bytes + .to_vec(); + transaction.write_page(leaf_id, left_page)?; + transaction.write_page(new_leaf_id, right_page)?; + repair_right_leaf_backlink(transaction, base, right, new_leaf_id)?; + Ok(Some(Promotion { key: separator, right_child: new_leaf_id })) +} + +fn insert_internal_promotion( + transaction: &mut WriteTransaction, + base: &[u8], + page_id: u64, + position: usize, + promotion: &Promotion, + cell_scratch: &mut Vec, +) -> io::Result> { + let snapshot = transaction.page_copy(base, page_id)?; + let page = SlottedPage::open(snapshot.as_slice(), page_id, transaction.next_header.next_page_id)?; + if page.header().page_type != PageType::Internal { + return Err(invalid_data("promotion target is not an internal page")); + } + let count = usize::from(page.header().cell_count); + if position > count { + return Err(invalid_input("internal insertion position is outside page")); + } + encode_internal_cell( + &promotion.key, + promotion.right_child, + page_id, + transaction.next_header.next_page_id, + cell_scratch, + )?; + let mut cells = Vec::<&[u8]>::new(); + cells.try_reserve_exact(count + 1).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + for current in 0..count { + if current == position { + cells.push(cell_scratch); + } + cells.push(page.cell(current)?); + } + if position == count { + cells.push(cell_scratch); + } + if used_internal_bytes(cells.iter().copied())? <= PAGE_SIZE { + let next_page_id = transaction.next_header.next_page_id; + let dirty = transaction.page_mut(base, page_id)?; + SlottedPage::open(dirty.as_mut_slice(), page_id, next_page_id)?.rebuild_ordered(cells.iter().copied())?; + return Ok(None); + } + + let right_page_id = transaction.allocate_page(base)?; + let split = choose_internal_split(&cells, page_id, transaction.next_header.next_page_id)?; + let leftmost = + InternalPreamble::decode(snapshot.as_slice(), page_id, transaction.next_header.next_page_id)?.leftmost_child; + let promoted = Promotion { key: split.promoted.key_bytes.to_vec(), right_child: right_page_id }; + let left_page = internal_page(page_id, leftmost, split.left_cells)?; + let right_page = internal_page(right_page_id, split.right_leftmost_child, split.right_cells)?; + transaction.write_page(page_id, left_page)?; + transaction.write_page(right_page_id, right_page)?; + Ok(Some(promoted)) +} + +fn propagate_promotion( + transaction: &mut WriteTransaction, + base: &[u8], + mut path: Vec<(u64, usize)>, + mut promotion: Promotion, + cell_scratch: &mut Vec, +) -> io::Result<()> { + while let Some((parent, position)) = path.pop() { + let Some(next) = insert_internal_promotion(transaction, base, parent, position, &promotion, cell_scratch)? + else { + return Ok(()); + }; + promotion = next; + } + let old_root = transaction.next_header.root_page_id; + let new_root = transaction.allocate_page(base)?; + encode_internal_cell( + &promotion.key, + promotion.right_child, + new_root, + transaction.next_header.next_page_id, + cell_scratch, + )?; + transaction.write_page(new_root, internal_page(new_root, old_root, &[cell_scratch.as_slice()])?)?; + transaction.next_header.root_page_id = new_root; + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn stage_upsert Deserialize<'de>>( + transaction: &mut WriteTransaction, + base: &[u8], + key: &K, + encoded_key: &[u8], + logical_len: u32, + compression: Compression, + stored: &[u8], + cell_scratch: &mut Vec, +) -> io::Result<()> { + let (leaf_id, path) = locate_transaction_leaf(transaction, base, key)?; + let page = SlottedPage::open(transaction.page(base, leaf_id)?, leaf_id, transaction.next_header.next_page_id)?; + let search = search_leaf(&page, key)?; + let (replace, index, old_overflow) = match search { + Ok(index) => { + let old = LeafCellRef::decode(page.cell(index)?, leaf_id, transaction.next_header.next_page_id)?; + let overflow = match old.value { + LeafValueRef::Overflow { stored_len, head, crc32, .. } => { + validated_overflow_chain_pages(transaction, base, head, stored_len, crc32)? + } + LeafValueRef::Inline { .. } | LeafValueRef::Tombstone => Vec::new(), + }; + (true, index, overflow) + } + Err(index) => (false, index, Vec::new()), + }; + let inline_footprint = SLOT_LEN + .checked_add(24) + .and_then(|size| size.checked_add(encoded_key.len())) + .and_then(|size| size.checked_add(stored.len())) + .ok_or_else(|| invalid_input("leaf cell footprint overflow"))?; + if stored.len() <= MAX_INLINE_STORED_VALUE && inline_footprint <= MAX_CELL_FOOTPRINT { + free_pages(transaction, &old_overflow)?; + encode_inline_leaf_cell(encoded_key, logical_len, compression, stored, cell_scratch)?; + } else { + let head = write_overflow_chain(transaction, base, stored, &old_overflow)?; + encode_overflow_leaf_cell( + encoded_key, + logical_len, + compression, + u32::try_from(stored.len()).map_err(|_| invalid_input("stored value exceeds u32"))?, + head, + stored_value_checksum(stored), + leaf_id, + transaction.next_header.next_page_id, + cell_scratch, + )?; + } + if let Some(promotion) = mutate_leaf(transaction, base, leaf_id, index, replace, cell_scratch)? { + propagate_promotion(transaction, base, path, promotion, cell_scratch)?; + } + Ok(()) +} + +pub(super) fn stage_delete Deserialize<'de>>( + transaction: &mut WriteTransaction, + base: &[u8], + key: &K, + encoded_key: &[u8], + cell_scratch: &mut Vec, +) -> io::Result { + let (leaf_id, _) = locate_transaction_leaf(transaction, base, key)?; + let page = SlottedPage::open(transaction.page(base, leaf_id)?, leaf_id, transaction.next_header.next_page_id)?; + let Ok(index) = search_leaf(&page, key)? else { return Ok(false) }; + let old = LeafCellRef::decode(page.cell(index)?, leaf_id, transaction.next_header.next_page_id)?; + match old.value { + LeafValueRef::Tombstone => return Ok(false), + LeafValueRef::Overflow { stored_len, head, crc32, .. } => { + let pages = validated_overflow_chain_pages(transaction, base, head, stored_len, crc32)?; + free_pages(transaction, &pages)?; + } + LeafValueRef::Inline { .. } => {} + } + encode_tombstone_leaf_cell(encoded_key, cell_scratch)?; + let _ = mutate_leaf(transaction, base, leaf_id, index, true, cell_scratch)?; + Ok(true) +} + +pub(super) fn push_child_visits( + stack: &mut Vec>, + children: &[(u64, Option)], + lower: Option<&K>, + upper: Option<&K>, +) -> io::Result<()> { + for index in (0..children.len()).rev() { + let child_lower = + if index == 0 { lower.cloned() } else { children.get(index).and_then(|(_, key)| key.clone()) }; + let child_upper = children.get(index + 1).and_then(|(_, key)| key.clone()).or_else(|| upper.cloned()); + let child = children + .get(index) + .map(|(child, _)| *child) + .ok_or_else(|| invalid_data("internal child index is invalid"))?; + stack.push(Visit::Enter(child, child_lower, child_upper)); + } + Ok(()) +} diff --git a/backend/btree/src/v3/tree/query.rs b/backend/btree/src/v3/tree/query.rs new file mode 100644 index 000000000..198c95a53 --- /dev/null +++ b/backend/btree/src/v3/tree/query.rs @@ -0,0 +1,957 @@ +#[cfg(test)] +use super::{database_page, INTERNAL_KEY_DECODE_COUNT}; +use super::{ + decompress_value_in_place, decompress_value_into, invalid_data, invalid_input, overflow_payload, record_page_visit, + recover_pending, recovery_required, wal_path, wal_temporary_path, BPlusTreeDiskIterator, + BPlusTreeDiskIteratorOwned, BPlusTreeMetadata, BPlusTreeQuery, BPlusTreeRangeIterator, Compression, DatabaseHeader, + InternalCellRef, InternalPreamble, LeafCellRef, LeafValueRef, Locator, PageType, PageValidation, + SharedSidecarGuard, SlottedPage, PAGE_SIZE, +}; +use crate::{ + codec::binary_deserialize, + common::{mmap_with_advice, read_exact_at_offset, Advice, BPlusTreeError}, +}; +use memmap2::Mmap; +use serde::Deserialize; +use std::{ + collections::HashSet, + fs::File, + io, + marker::PhantomData, + ops::{Bound, Range}, + path::{Path, PathBuf}, + sync::{Arc, OnceLock}, +}; + +pub(super) fn decode_key(encoded: &[u8]) -> io::Result +where + K: for<'de> Deserialize<'de>, +{ + rmp_serde::from_slice(encoded) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("invalid serialized key: {err}"))) +} + +#[cfg(test)] +pub(super) fn reset_internal_key_decode_count() { INTERNAL_KEY_DECODE_COUNT.set(0); } + +#[cfg(test)] +pub(super) fn internal_key_decode_count() -> usize { INTERNAL_KEY_DECODE_COUNT.get() } + +fn decode_internal_key(encoded: &[u8]) -> io::Result +where + K: for<'de> Deserialize<'de>, +{ + #[cfg(test)] + INTERNAL_KEY_DECODE_COUNT.with(|count| count.set(count.get().saturating_add(1))); + decode_key(encoded) +} + +pub(crate) fn search_leaf(page: &SlottedPage, target: &K) -> io::Result> +where + K: Ord + for<'de> Deserialize<'de>, + B: AsRef<[u8]>, +{ + if page.header().page_type != PageType::Leaf { + return Err(invalid_data("typed leaf search requires a leaf page")); + } + let mut left = 0usize; + let mut right = usize::from(page.header().cell_count); + while left < right { + let middle = left + (right - left) / 2; + let cell = LeafCellRef::decode(page.cell(middle)?, page.page_id(), page.next_page_id())?; + match decode_key::(cell.key_bytes)?.cmp(target) { + std::cmp::Ordering::Less => left = middle + 1, + std::cmp::Ordering::Greater => right = middle, + std::cmp::Ordering::Equal => return Ok(Ok(middle)), + } + } + Ok(Err(left)) +} + +#[cfg(test)] +pub(crate) fn validate_locator>( + page: &SlottedPage, + locator: Locator, + serialized_primary_key: &[u8], +) -> io::Result<()> { + if page.header().page_type != PageType::Leaf || page.page_id() != locator.leaf_page_id { + return Err(invalid_data("locator does not reference this leaf page")); + } + let cell = LeafCellRef::decode(page.cell(usize::from(locator.slot_index))?, page.page_id(), page.next_page_id())?; + let cell_crc = crc32fast::hash(cell.key_bytes); + if cell_crc != locator.serialized_key_crc32 + || crc32fast::hash(serialized_primary_key) != locator.serialized_key_crc32 + || cell.key_bytes != serialized_primary_key + { + return Err(invalid_data("locator serialized key mismatch")); + } + Ok(()) +} + +#[cfg(test)] +pub(crate) fn read_leaf_value<'a>( + database: &'a [u8], + value: &LeafValueRef<'a>, + next_page_id: u64, + maximum_length: usize, + scratch: &'a mut Vec, +) -> io::Result> { + match *value { + LeafValueRef::Tombstone => Ok(None), + LeafValueRef::Inline { compression, logical_len, stored, .. } => { + let logical_len = usize::try_from(logical_len).map_err(|_| invalid_data("logical length exceeds usize"))?; + if logical_len > maximum_length { + return Err(invalid_data("logical length exceeds allocation limit")); + } + match compression { + Compression::None => Ok(Some(stored)), + Compression::Lz4 => decompress_value_into( + stored, + u32::try_from(logical_len).map_err(|_| invalid_data("logical length exceeds u32"))?, + maximum_length, + scratch, + ) + .map(Some), + } + } + LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } => { + let logical_length = + usize::try_from(logical_len).map_err(|_| invalid_data("logical length exceeds usize"))?; + let stored_length = usize::try_from(stored_len).map_err(|_| invalid_data("stored length exceeds usize"))?; + if logical_length > maximum_length || stored_length > maximum_length { + return Err(invalid_data("overflow value exceeds allocation limit")); + } + scratch.clear(); + scratch.try_reserve(stored_length).map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; + let mut page_id = head; + let mut visited = HashSet::new(); + while page_id != 0 { + if u64::try_from(visited.len()).map_err(|_| invalid_data("overflow chain length exceeds u64"))? + >= next_page_id + { + return Err(invalid_data("overflow chain contains a cycle")); + } + let page = SlottedPage::open(database_page(database, page_id, next_page_id)?, page_id, next_page_id)?; + if page.header().page_type != PageType::Overflow { + return Err(invalid_data("overflow chain references a non-overflow page")); + } + let payload = overflow_payload(&page)?; + if payload.is_empty() { + return Err(invalid_data("overflow chain contains an empty payload")); + } + visited.try_reserve(1).map_err(|err| io::Error::new(io::ErrorKind::OutOfMemory, err))?; + if !visited.insert(page_id) { + return Err(invalid_data("overflow chain contains a cycle")); + } + let new_length = scratch + .len() + .checked_add(payload.len()) + .ok_or_else(|| invalid_data("overflow chain length overflow"))?; + if new_length > stored_length { + return Err(invalid_data("overflow chain exceeds stored length")); + } + scratch.extend_from_slice(payload); + page_id = page.header().right; + } + if scratch.len() != stored_length { + return Err(invalid_data("overflow chain stored length mismatch")); + } + if crc32fast::hash(scratch) != crc32 { + return Err(invalid_data("stored value checksum mismatch")); + } + if compression == Compression::Lz4 { + decompress_value_in_place(scratch, logical_len, maximum_length)?; + } else if scratch.len() != logical_length { + return Err(invalid_data("uncompressed overflow length mismatch")); + } + Ok(Some(scratch.as_slice())) + } + } +} + +enum DecodedEntry { + Ready(K, V), + Overflow(K, Compression, u32, u32, u64, u32), + Tombstone, +} + +enum DecodedValue { + Ready(V), + Overflow(Compression, u32, u32, u64, u32), + Tombstone, +} + +struct InternalRoute { + leftmost_child: u64, + separators: Vec<(K, u64)>, +} + +impl InternalRoute { + fn child_for(&self, target: &K) -> u64 { + let index = self.separators.partition_point(|(key, _)| key <= target); + index.checked_sub(1).map_or(self.leftmost_child, |previous| self.separators[previous].1) + } +} + +fn decode_internal_route(page: &SlottedPage) -> io::Result> +where + K: Ord + for<'de> Deserialize<'de>, + B: AsRef<[u8]>, +{ + let preamble = InternalPreamble::decode(page.as_bytes(), page.page_id(), page.next_page_id())?; + let count = usize::from(page.header().cell_count); + let mut separators = Vec::new(); + separators.try_reserve_exact(count).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + for index in 0..count { + let cell = InternalCellRef::decode(page.cell(index)?, page.page_id(), page.next_page_id())?; + separators.push((decode_internal_key(cell.key_bytes)?, cell.right_child)); + } + Ok(InternalRoute { leftmost_child: preamble.leftmost_child, separators }) +} + +enum LocateCell { + Internal(InternalRoute), + Leaf(Option>), +} + +enum LocateLeaf { + Internal(InternalRoute), + Leaf, +} + +pub(super) struct QuerySnapshot { + pub(super) file: Option, + pub(super) mmap: Option, + pub(super) filepath: PathBuf, + pub(super) page_validations: Vec>, + internal_routes: Vec>>, + pub(super) sidecar_guard: Option, +} + +impl BPlusTreeQuery { + pub(super) fn from_file_unlocked(file: File) -> io::Result { + let file_len = + usize::try_from(file.metadata()?.len()).map_err(|_| invalid_data("database length exceeds usize"))?; + if file_len < PAGE_SIZE { + return Err(invalid_data("database is shorter than its header page")); + } + let mut header_page = [0; PAGE_SIZE]; + read_exact_at_offset(&file, &mut header_page, 0)?; + let header = DatabaseHeader::decode(&header_page)?; + let expected = usize::try_from(header.next_page_id) + .map_err(|_| invalid_data("next page id exceeds usize"))? + .checked_mul(PAGE_SIZE) + .ok_or_else(|| invalid_data("database length overflow"))?; + if file_len != expected { + return Err(invalid_data("database file length does not match header")); + } + let mmap = mmap_with_advice(&file, Advice::Normal, "v3 B+Tree query"); + let page_count = + usize::try_from(header.next_page_id).map_err(|_| invalid_data("next page id exceeds usize"))?; + let mut page_validations = Vec::new(); + page_validations + .try_reserve_exact(page_count) + .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + let mut internal_routes = Vec::new(); + internal_routes + .try_reserve_exact(page_count) + .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + for _ in 0..page_count { + page_validations.push(OnceLock::new()); + internal_routes.push(OnceLock::new()); + } + let file = mmap.is_none().then_some(file); + let mut page_buffer = Vec::new(); + if mmap.is_none() { + page_buffer + .try_reserve_exact(PAGE_SIZE) + .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + page_buffer.resize(PAGE_SIZE, 0); + } + Ok(Self { + snapshot: Arc::new(QuerySnapshot { + file, + mmap, + filepath: PathBuf::new(), + page_validations, + internal_routes, + sidecar_guard: None, + }), + header, + file_len, + page_buffer, + value_scratch: Vec::new(), + locator_page_id: 0, + locator_cell_ranges: Vec::new(), + _value: PhantomData, + }) + } + + pub fn try_new(filepath: &Path) -> io::Result { + loop { + let sidecar_guard = SharedSidecarGuard::acquire(filepath)?; + let pending = wal_path(filepath).try_exists()? || wal_temporary_path(filepath).try_exists()?; + if !pending { + let mut query = Self::from_file_unlocked(File::open(filepath)?)?; + let snapshot = Arc::get_mut(&mut query.snapshot) + .ok_or_else(|| invalid_data("new query snapshot is unexpectedly shared"))?; + snapshot.filepath = filepath.to_path_buf(); + snapshot.sidecar_guard = Some(sidecar_guard); + return Ok(query); + } + drop(sidecar_guard); + match recover_pending(filepath) { + Ok(()) => {} + Err(error) + if matches!(error.kind(), io::ErrorKind::PermissionDenied | io::ErrorKind::ReadOnlyFilesystem) => + { + return Err(recovery_required(filepath, error)); + } + Err(error) => return Err(error), + } + } + } + + pub fn try_clone(&self) -> io::Result { + if self.snapshot.filepath.as_os_str().is_empty() { + return Err(invalid_input("mapped query without a path cannot be cloned")); + } + let mut page_buffer = Vec::new(); + if self.snapshot.mmap.is_none() { + page_buffer + .try_reserve_exact(PAGE_SIZE) + .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + page_buffer.resize(PAGE_SIZE, 0); + } + Ok(Self { + snapshot: Arc::clone(&self.snapshot), + header: self.header.clone(), + file_len: self.file_len, + page_buffer, + value_scratch: Vec::new(), + locator_page_id: 0, + locator_cell_ranges: Vec::new(), + _value: PhantomData, + }) + } + + /// A query handle over an empty in-memory snapshot, for tests that need a + /// `BPlusTreeQuery` value without a database behind it. + #[cfg(any(test, feature = "test-support"))] + pub fn clone_error_fixture() -> Self { + Self { + snapshot: Arc::new(QuerySnapshot { + file: None, + mmap: None, + filepath: PathBuf::new(), + page_validations: Vec::new(), + internal_routes: Vec::new(), + sidecar_guard: None, + }), + header: DatabaseHeader { + root_page_id: 1, + next_page_id: 2, + free_page_head: 0, + generation: 1, + database_id: [0; 16], + metadata: BPlusTreeMetadata::Empty, + }, + file_len: 0, + page_buffer: Vec::new(), + value_scratch: Vec::new(), + locator_page_id: 0, + locator_cell_ranges: Vec::new(), + _value: PhantomData, + } + } + + pub fn filepath(&self) -> &Path { &self.snapshot.filepath } + + /// The metadata carried in the database header of this snapshot. + pub fn metadata(&self) -> &BPlusTreeMetadata { &self.header.metadata } + + pub fn snapshot_identity(&self) -> ([u8; 16], u64) { (self.header.database_id, self.header.generation) } + + pub(crate) fn snapshot_metadata(&self) -> &BPlusTreeMetadata { &self.header.metadata } + + fn value_allocation_limit(&self) -> usize { + self.file_len.saturating_mul(256).min(usize::try_from(u32::MAX).unwrap_or(usize::MAX)) + } + + pub(super) fn assemble_overflow_chain( + &mut self, + compression: Compression, + logical_len: u32, + stored_len: u32, + mut page_id: u64, + crc32: u32, + mut owned_pages: Option<&mut HashSet>, + ) -> io::Result<()> { + let stored_len = usize::try_from(stored_len).map_err(|_| invalid_data("stored length exceeds usize"))?; + let logical_len_usize = + usize::try_from(logical_len).map_err(|_| invalid_data("logical length exceeds usize"))?; + if stored_len > self.file_len || logical_len_usize > self.value_allocation_limit() { + return Err(invalid_data("overflow value exceeds allocation limit")); + } + self.value_scratch.clear(); + self.value_scratch + .try_reserve(stored_len) + .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + let mut chain = HashSet::new(); + while page_id != 0 { + record_page_visit(&mut chain, page_id, "overflow chain contains a cycle")?; + if let Some(pages) = owned_pages.as_deref_mut() { + record_page_visit(pages, page_id, "overflow page is owned by multiple values")?; + } + page_id = self.with_slotted_page(page_id, |page, scratch| { + if page.header().page_type != PageType::Overflow { + return Err(invalid_data("overflow chain references a non-overflow page")); + } + let payload = overflow_payload(page)?; + if payload.is_empty() { + return Err(invalid_data("overflow chain contains an empty payload")); + } + let next_len = scratch + .len() + .checked_add(payload.len()) + .ok_or_else(|| invalid_data("overflow value length overflow"))?; + if next_len > stored_len { + return Err(invalid_data("overflow chain exceeds declared length")); + } + scratch.extend_from_slice(payload); + Ok(page.header().right) + })?; + } + if self.value_scratch.len() != stored_len || crc32fast::hash(&self.value_scratch) != crc32 { + return Err(invalid_data("overflow value checksum or length mismatch")); + } + if compression == Compression::Lz4 { + let allocation_limit = self.value_allocation_limit(); + decompress_value_in_place(&mut self.value_scratch, logical_len, allocation_limit)?; + } else if self.value_scratch.len() != logical_len_usize { + return Err(invalid_data("uncompressed overflow length mismatch")); + } + Ok(()) + } + + pub(super) fn with_page( + &mut self, + page_id: u64, + read: impl FnOnce(&[u8], &mut Vec) -> io::Result, + ) -> io::Result { + if page_id == 0 || page_id >= self.header.next_page_id { + return Err(invalid_data("page id is outside database")); + } + let offset = usize::try_from(page_id) + .map_err(|_| invalid_data("page id exceeds usize"))? + .checked_mul(PAGE_SIZE) + .ok_or_else(|| invalid_data("page offset overflow"))?; + let end = offset.checked_add(PAGE_SIZE).ok_or_else(|| invalid_data("page end overflow"))?; + if let Some(mmap) = &self.snapshot.mmap { + let page = mmap.get(offset..end).ok_or_else(|| invalid_data("page is truncated"))?; + return read(page, &mut self.value_scratch); + } + let file = self.snapshot.file.as_ref().ok_or_else(|| invalid_data("query has no data source"))?; + if self.page_buffer.len() != PAGE_SIZE { + return Err(invalid_data("query page buffer has invalid length")); + } + read_exact_at_offset( + file, + &mut self.page_buffer, + u64::try_from(offset).map_err(|_| invalid_data("page offset exceeds u64"))?, + )?; + read(&self.page_buffer, &mut self.value_scratch) + } + + fn with_slotted_page( + &mut self, + page_id: u64, + read: impl FnOnce(&SlottedPage<&[u8]>, &mut Vec) -> io::Result, + ) -> io::Result { + let index = usize::try_from(page_id).map_err(|_| invalid_data("page id exceeds usize"))?; + let cached = self + .snapshot + .page_validations + .get(index) + .ok_or_else(|| invalid_data("page id is outside cache"))? + .get() + .copied(); + let next_page_id = self.header.next_page_id; + let (result, validation) = self.with_page(page_id, |bytes, scratch| { + let page = match cached { + Some(validation) => SlottedPage::from_immutable_snapshot(bytes, validation)?, + None => SlottedPage::open(bytes, page_id, next_page_id)?, + }; + let validation = cached.is_none().then(|| page.validation()); + read(&page, scratch).map(|result| (result, validation)) + })?; + if let Some(validation) = validation { + let slot = + self.snapshot.page_validations.get(index).ok_or_else(|| invalid_data("page id is outside cache"))?; + let _ = slot.set(validation); + } + Ok(result) + } + + fn cached_internal_child(&self, page_id: u64, key: &K) -> io::Result> + where + K: Ord, + { + let index = usize::try_from(page_id).map_err(|_| invalid_data("page id exceeds usize"))?; + Ok(self + .snapshot + .internal_routes + .get(index) + .ok_or_else(|| invalid_data("page id is outside route cache"))? + .get() + .map(|route| route.child_for(key))) + } + + fn cache_internal_route(&self, page_id: u64, route: InternalRoute) -> io::Result<()> { + let index = usize::try_from(page_id).map_err(|_| invalid_data("page id exceeds usize"))?; + let slot = + self.snapshot.internal_routes.get(index).ok_or_else(|| invalid_data("page id is outside route cache"))?; + let _ = slot.set(route); + Ok(()) + } + + pub(super) fn locate_leaf(&mut self, key: &K) -> io::Result + where + K: Ord + for<'de> Deserialize<'de>, + { + let mut page_id = self.header.root_page_id; + let mut depth = 0u64; + loop { + if depth >= self.header.next_page_id { + return Err(invalid_data("tree descent contains a cycle")); + } + if let Some(child) = self.cached_internal_child(page_id, key)? { + page_id = child; + depth += 1; + continue; + } + let step = self.with_slotted_page(page_id, |page, _| match page.header().page_type { + PageType::Leaf => Ok(LocateLeaf::Leaf), + PageType::Internal => decode_internal_route(page).map(LocateLeaf::Internal), + PageType::Overflow | PageType::Free => Err(invalid_data("tree references a non-tree page")), + })?; + match step { + LocateLeaf::Internal(route) => { + let child = route.child_for(key); + self.cache_internal_route(page_id, route)?; + page_id = child; + depth += 1; + } + LocateLeaf::Leaf => return Ok(page_id), + } + } + } + + pub(super) fn locate_cell(&mut self, key: &K) -> io::Result)>> + where + K: Ord + for<'de> Deserialize<'de>, + { + let mut page_id = self.header.root_page_id; + let mut depth = 0u64; + loop { + if depth >= self.header.next_page_id { + return Err(invalid_data("tree descent contains a cycle")); + } + if let Some(child) = self.cached_internal_child(page_id, key)? { + page_id = child; + depth += 1; + continue; + } + let step = self.with_slotted_page(page_id, |page, _| match page.header().page_type { + PageType::Internal => decode_internal_route(page).map(LocateCell::Internal), + PageType::Leaf => { + search_leaf(page, key)?.ok().map(|index| page.cell_range(index)).transpose().map(LocateCell::Leaf) + } + PageType::Overflow | PageType::Free => Err(invalid_data("tree references a non-tree page")), + })?; + match step { + LocateCell::Internal(route) => { + let child = route.child_for(key); + self.cache_internal_route(page_id, route)?; + page_id = child; + depth += 1; + } + LocateCell::Leaf(range) => return Ok(range.map(|range| (page_id, range))), + } + } + } + + pub(super) fn leftmost_leaf(&mut self) -> io::Result + where + K: for<'de> Deserialize<'de>, + { + let mut page_id = self.header.root_page_id; + let mut depth = 0u64; + loop { + if depth >= self.header.next_page_id { + return Err(invalid_data("tree descent contains a cycle")); + } + let result = self.with_slotted_page(page_id, |page, _| match page.header().page_type { + PageType::Leaf => Ok(None), + PageType::Internal => InternalPreamble::decode(page.as_bytes(), page_id, page.next_page_id()) + .map(|preamble| Some(preamble.leftmost_child)), + PageType::Overflow | PageType::Free => Err(invalid_data("tree references a non-tree page")), + })?; + let Some(child) = result else { return Ok(page_id) }; + page_id = child; + depth += 1; + } + } + + fn decode_entry(&mut self, leaf_page_id: u64, slot_index: usize) -> io::Result> + where + K: for<'de> Deserialize<'de>, + V: for<'de> Deserialize<'de>, + { + let range = self.with_slotted_page(leaf_page_id, |page, _| { + if page.header().page_type != PageType::Leaf { + return Err(invalid_data("iterator expected a leaf page")); + } + page.cell_range(slot_index) + })?; + self.decode_entry_range(leaf_page_id, range) + } + + pub(super) fn decode_entry_range(&mut self, leaf_page_id: u64, range: Range) -> io::Result> + where + K: for<'de> Deserialize<'de>, + V: for<'de> Deserialize<'de>, + { + let next_page_id = self.header.next_page_id; + let allocation_limit = self.value_allocation_limit(); + let decoded = self.with_page(leaf_page_id, |bytes, scratch| { + let cell_bytes = bytes.get(range).ok_or_else(|| invalid_data("leaf cell is outside page"))?; + let cell = LeafCellRef::decode(cell_bytes, leaf_page_id, next_page_id)?; + let key = binary_deserialize(cell.key_bytes)?; + match cell.value { + LeafValueRef::Inline { compression: Compression::None, stored, .. } => { + binary_deserialize(stored).map(|value| DecodedEntry::Ready(key, value)) + } + LeafValueRef::Inline { compression: Compression::Lz4, logical_len, stored, .. } => { + let decompressed = decompress_value_into(stored, logical_len, allocation_limit, scratch)?; + binary_deserialize(decompressed).map(|value| DecodedEntry::Ready(key, value)) + } + LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } => { + Ok(DecodedEntry::Overflow(key, compression, logical_len, stored_len, head, crc32)) + } + LeafValueRef::Tombstone => Ok(DecodedEntry::Tombstone), + } + })?; + match decoded { + DecodedEntry::Ready(key, value) => Ok(Some((key, value))), + DecodedEntry::Tombstone => Ok(None), + DecodedEntry::Overflow(key, compression, logical_len, stored_len, page_id, crc32) => { + self.assemble_overflow_chain(compression, logical_len, stored_len, page_id, crc32, None)?; + let value = binary_deserialize(&self.value_scratch)?; + Ok(Some((key, value))) + } + } + } +} + +impl BPlusTreeQuery +where + K: Ord + for<'de> Deserialize<'de>, + V: for<'de> Deserialize<'de>, +{ + pub(super) fn query_io(&mut self, key: &K) -> io::Result> { + let Some((leaf_id, range)) = self.locate_cell(key)? else { return Ok(None) }; + self.decode_entry_range(leaf_id, range).map(|entry| entry.map(|(_, value)| value)) + } + + pub fn query(&mut self, key: &K) -> Result, BPlusTreeError> { self.query_io(key).map_err(Into::into) } + + pub fn query_zero_copy(&mut self, key: &K) -> Result, BPlusTreeError> { self.query(key) } + + fn locator_cell_range(&mut self, locator: Locator) -> io::Result> { + if self.locator_page_id != locator.leaf_page_id { + let page_id = locator.leaf_page_id; + let mut ranges = std::mem::take(&mut self.locator_cell_ranges); + self.with_slotted_page(page_id, |page, _| { + if page.header().page_type != PageType::Leaf { + return Err(invalid_data("locator does not reference a leaf page")); + } + let count = usize::from(page.header().cell_count); + ranges.clear(); + ranges.try_reserve(count).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + for index in 0..count { + ranges.push(page.cell_range(index)?); + } + Ok(()) + })?; + self.locator_page_id = page_id; + self.locator_cell_ranges = ranges; + } + self.locator_cell_ranges + .get(usize::from(locator.slot_index)) + .cloned() + .ok_or_else(|| invalid_data("locator slot index is outside leaf page")) + } + + pub(crate) fn read_locator_value(&mut self, locator: Locator, primary_key: &[u8]) -> io::Result { + let page_id = locator.leaf_page_id; + let next_page_id = self.header.next_page_id; + let range = self.locator_cell_range(locator)?; + let allocation_limit = self.value_allocation_limit(); + let decoded = self.with_page(page_id, |bytes, scratch| { + let cell = LeafCellRef::decode( + bytes.get(range).ok_or_else(|| invalid_data("locator cell is outside page"))?, + page_id, + next_page_id, + )?; + if crc32fast::hash(cell.key_bytes) != locator.serialized_key_crc32 + || crc32fast::hash(primary_key) != locator.serialized_key_crc32 + || cell.key_bytes != primary_key + { + return Err(invalid_data("locator serialized key mismatch")); + } + match cell.value { + LeafValueRef::Inline { compression: Compression::None, stored, .. } => { + binary_deserialize(stored).map(DecodedValue::Ready) + } + LeafValueRef::Inline { compression: Compression::Lz4, logical_len, stored, .. } => { + let decompressed = decompress_value_into(stored, logical_len, allocation_limit, scratch)?; + binary_deserialize(decompressed).map(DecodedValue::Ready) + } + LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } => { + Ok(DecodedValue::Overflow(compression, logical_len, stored_len, head, crc32)) + } + LeafValueRef::Tombstone => Ok(DecodedValue::Tombstone), + } + })?; + match decoded { + DecodedValue::Ready(value) => Ok(value), + DecodedValue::Overflow(compression, logical_len, stored_len, head, crc32) => { + self.assemble_overflow_chain(compression, logical_len, stored_len, head, crc32, None)?; + binary_deserialize(&self.value_scratch) + } + DecodedValue::Tombstone => Err(invalid_data("locator references a tombstone")), + } + } + + pub(crate) fn collect_with_locators(&mut self) -> io::Result> { + let mut result = Vec::new(); + let mut page_id = self.header.root_page_id; + let mut visited = HashSet::new(); + let mut descending = true; + let mut descent_depth = 0u64; + loop { + let next_page_id = self.header.next_page_id; + let (child, right, locators) = self.with_page(page_id, |bytes, _| { + let page = SlottedPage::open(bytes, page_id, next_page_id)?; + if descending && page.header().page_type == PageType::Internal { + let leftmost = InternalPreamble::decode(bytes, page_id, next_page_id)?.leftmost_child; + return Ok((Some(leftmost), 0, Vec::new())); + } + if page.header().page_type != PageType::Leaf { + return Err(invalid_data("locator scan expected a tree page")); + } + let mut locators = Vec::new(); + for index in 0..usize::from(page.header().cell_count) { + let cell = LeafCellRef::decode(page.cell(index)?, page_id, next_page_id)?; + if !matches!(cell.value, LeafValueRef::Tombstone) { + locators.push(( + Locator::for_key( + page_id, + u16::try_from(index).map_err(|_| invalid_data("slot index exceeds u16"))?, + cell.key_bytes, + )?, + page.cell_range(index)?, + )); + } + } + Ok((None, page.header().right, locators)) + })?; + if let Some(child) = child { + if descent_depth >= self.header.next_page_id { + return Err(invalid_data("locator descent contains a cycle")); + } + page_id = child; + descent_depth += 1; + continue; + } + descending = false; + record_page_visit(&mut visited, page_id, "right sibling chain contains a cycle")?; + result.try_reserve(locators.len()).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + for (locator, range) in locators { + let entry = self + .decode_entry_range(page_id, range)? + .ok_or_else(|| invalid_data("live locator became a tombstone"))?; + result.push((entry.0, entry.1, locator)); + } + if right == 0 { + return Ok(result); + } + page_id = right; + } + } + + pub fn contains_live_key(&mut self, key: &K) -> Result { + let result: io::Result = (|| { + let leaf_id = self.locate_leaf(key)?; + let next_page_id = self.header.next_page_id; + let index = self.with_slotted_page(leaf_id, |page, _| search_leaf(page, key))?; + let Ok(index) = index else { return Ok(false) }; + self.with_slotted_page(leaf_id, |page, _| { + let cell = LeafCellRef::decode(page.cell(index)?, leaf_id, next_page_id)?; + Ok(!matches!(cell.value, LeafValueRef::Tombstone)) + }) + })(); + result.map_err(Into::into) + } + + pub fn query_le(&mut self, key: &K) -> Result, BPlusTreeError> { + self.query_le_io(key).map_err(Into::into) + } + + fn query_le_io(&mut self, key: &K) -> io::Result> { + let mut leaf_id = self.locate_leaf(key)?; + let mut first = true; + let mut visited = HashSet::new(); + record_page_visit(&mut visited, leaf_id, "left sibling chain contains a cycle")?; + loop { + let (left, count, start) = self.with_slotted_page(leaf_id, |page, _| { + if page.header().page_type != PageType::Leaf { + return Err(invalid_data("left sibling is not a leaf")); + } + let count = usize::from(page.header().cell_count); + let start = if first { + match search_leaf(page, key)? { + Ok(index) => Some(index), + Err(0) => None, + Err(index) => index.checked_sub(1), + } + } else { + count.checked_sub(1) + }; + Ok((page.header().left, count, start)) + })?; + if let Some(start) = start { + for index in (0..=start.min(count.saturating_sub(1))).rev() { + if let Some((_, value)) = self.decode_entry(leaf_id, index)? { + return Ok(Some(value)); + } + } + } + if left == 0 { + return Ok(None); + } + let current = leaf_id; + record_page_visit(&mut visited, left, "left sibling chain contains a cycle")?; + leaf_id = left; + self.with_slotted_page(leaf_id, |page, _| { + if page.header().page_type != PageType::Leaf || page.header().right != current { + return Err(invalid_data("asymmetric leaf sibling link")); + } + Ok(()) + })?; + first = false; + } + } + + pub fn len(&mut self) -> Result { + let result: io::Result = (|| { + let mut page_id = self.leftmost_leaf()?; + let mut visited = HashSet::new(); + record_page_visit(&mut visited, page_id, "right sibling chain contains a cycle")?; + let mut total = 0usize; + loop { + let next_page_id = self.header.next_page_id; + let (right, live) = self.with_page(page_id, |bytes, _| { + let page = SlottedPage::open(bytes, page_id, next_page_id)?; + if page.header().page_type != PageType::Leaf { + return Err(invalid_data("length scan expected a leaf page")); + } + let mut live = 0usize; + for index in 0..usize::from(page.header().cell_count) { + let cell = LeafCellRef::decode(page.cell(index)?, page_id, next_page_id)?; + if !matches!(cell.value, LeafValueRef::Tombstone) { + live = live.checked_add(1).ok_or_else(|| invalid_data("entry count overflow"))?; + } + } + Ok((page.header().right, live)) + })?; + total = total.checked_add(live).ok_or_else(|| invalid_data("entry count overflow"))?; + if right == 0 { + return Ok(total); + } + record_page_visit(&mut visited, right, "right sibling chain contains a cycle")?; + self.with_page(right, |bytes, _| { + let page = SlottedPage::open(bytes, right, next_page_id)?; + if page.header().page_type != PageType::Leaf || page.header().left != page_id { + return Err(invalid_data("asymmetric leaf sibling link")); + } + Ok(()) + })?; + page_id = right; + } + })(); + result.map_err(Into::into) + } + + pub fn is_empty(&mut self) -> Result { + let mut iterator = self.iter(); + match iterator.next() { + None => Ok(true), + Some(Ok(_)) => Ok(false), + Some(Err(err)) => Err(BPlusTreeError::Io(err)), + } + } + + pub fn iter(&mut self) -> BPlusTreeDiskIterator<'_, K, V> { BPlusTreeDiskIterator::new(self) } + + pub fn disk_iter(self) -> BPlusTreeDiskIteratorOwned { BPlusTreeDiskIteratorOwned::new(self) } + + pub fn range_iter(&mut self, start: Bound<&K>, end: Bound<&K>) -> BPlusTreeRangeIterator<'_, K, V> + where + K: Clone, + { + let start = start.map(Clone::clone); + let end = end.map(Clone::clone); + BPlusTreeRangeIterator { iterator: BPlusTreeDiskIterator::from_bound(self, start.clone()), start, end } + } + + pub fn range_page( + &mut self, + start: Bound<&K>, + end: Bound<&K>, + offset: usize, + limit: usize, + ) -> Result<(Vec<(K, V)>, bool), BPlusTreeError> + where + K: Clone, + { + let mut iterator = self.range_iter(start, end); + for _ in 0..offset { + if let Some(entry) = iterator.next() { + let _ = entry.map_err(BPlusTreeError::Io)?; + } else { + return Ok((Vec::new(), false)); + } + } + let mut result = Vec::new(); + while result.len() < limit { + match iterator.next() { + Some(Ok(entry)) => { + result + .try_reserve(1) + .map_err(|error| BPlusTreeError::Io(io::Error::new(io::ErrorKind::OutOfMemory, error)))?; + result.push(entry); + } + Some(Err(err)) => return Err(BPlusTreeError::Io(err)), + None => return Ok((result, false)), + } + } + let has_more = match iterator.next() { + Some(Ok(_)) => true, + Some(Err(err)) => return Err(BPlusTreeError::Io(err)), + None => false, + }; + Ok((result, has_more)) + } +} diff --git a/backend/btree/src/v3/tree/serial_writer.rs b/backend/btree/src/v3/tree/serial_writer.rs new file mode 100644 index 000000000..b09a5fec1 --- /dev/null +++ b/backend/btree/src/v3/tree/serial_writer.rs @@ -0,0 +1,133 @@ +use super::{invalid_input, BPlusTreeSerialWriter, BPlusTreeUpdate, FlushPolicy}; +use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; +use std::{ + io, + path::Path, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; + +impl BPlusTreeSerialWriter +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone + Send + 'static, + V: Serialize + for<'de> Deserialize<'de> + Send + 'static, +{ + pub fn new(filepath: &Path, flush_policy: FlushPolicy) -> io::Result { + let mut updater = BPlusTreeUpdate::try_new_with_backoff(filepath)?; + updater.set_flush_policy(flush_policy); + Ok(Self { + updater: Arc::new(Mutex::new(updater)), + flush_policy, + dirty: Arc::new(AtomicBool::new(false)), + shutdown: Arc::new(AtomicBool::new(false)), + background_error: Arc::new(Mutex::new(None)), + background_handle: Mutex::new(None), + }) + } + + pub fn upsert_prepared(&self, items: Vec<(K, Vec)>) -> io::Result { + let result = self.updater.lock().upsert_batch_encoded(items); + if result.is_ok() { + self.dirty.store(self.flush_policy == FlushPolicy::Batch, Ordering::Release); + } + result + } + + pub fn upsert(&self, items: &[(&K, &V)]) -> io::Result { + self.upsert_prepared(BPlusTreeUpdate::::prepare_upsert_batch(items)?) + } + + pub fn start_background_commit(&self, interval: Duration) -> io::Result<()> { + if self.flush_policy != FlushPolicy::Batch { + return Err(invalid_input("background commit requires batch flush policy")); + } + if interval.is_zero() { + return Err(invalid_input("background commit interval must be greater than zero")); + } + let mut slot = self.background_handle.lock(); + if slot.is_some() { + return Ok(()); + } + self.shutdown.store(false, Ordering::Release); + let updater = Arc::clone(&self.updater); + let dirty = Arc::clone(&self.dirty); + let shutdown = Arc::clone(&self.shutdown); + let background_error = Arc::clone(&self.background_error); + *slot = Some( + std::thread::Builder::new() + .name(String::from("bplustree-commit")) + .spawn(move || { + while !shutdown.load(Ordering::Acquire) { + std::thread::park_timeout(interval); + if shutdown.load(Ordering::Acquire) { + break; + } + commit_if_dirty(&updater, &dirty, &background_error); + } + commit_if_dirty(&updater, &dirty, &background_error); + }) + .map_err(io::Error::other)?, + ); + Ok(()) + } + + pub fn stop_background_commit(&self) -> io::Result<()> { + self.shutdown.store(true, Ordering::Release); + if let Some(handle) = self.background_handle.lock().take() { + handle.thread().unpark(); + handle.join().map_err(|_| io::Error::other("background B+Tree commit thread panicked"))?; + } + if let Some(error) = self.background_error.lock().take() { + return Err(error); + } + Ok(()) + } + + pub fn flush_now(&self) -> io::Result<()> { self.commit() } + + pub fn commit(&self) -> io::Result<()> { + self.updater.lock().commit()?; + self.dirty.store(false, Ordering::Release); + Ok(()) + } + + pub fn shutdown(&self) -> io::Result<()> { + self.stop_background_commit()?; + self.commit() + } +} + +fn commit_if_dirty( + updater: &Mutex>, + dirty: &AtomicBool, + background_error: &Mutex>, +) where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de>, +{ + if !dirty.swap(false, Ordering::AcqRel) { + return; + } + if let Err(error) = updater.lock().commit() { + dirty.store(true, Ordering::Release); + log::error!("Background B+Tree commit failed: {error}"); + *background_error.lock() = Some(error); + } +} + +impl Drop for BPlusTreeSerialWriter { + fn drop(&mut self) { + self.shutdown.store(true, Ordering::Release); + if self.dirty.load(Ordering::Acquire) { + log::warn!("Dropping dirty B+Tree serial writer without an explicit shutdown"); + } + if let Some(handle) = self.background_handle.lock().take() { + handle.thread().unpark(); + drop(handle); + } + } +} diff --git a/backend/btree/src/v3/tree/tests.rs b/backend/btree/src/v3/tree/tests.rs new file mode 100644 index 000000000..8e4279320 --- /dev/null +++ b/backend/btree/src/v3/tree/tests.rs @@ -0,0 +1,15 @@ +use super::*; + +mod behavior; +mod configuration; +mod lifecycle; +mod persistence; +mod protocol; +mod query; +mod support; + +use self::support::{ + assert_writer_is_blocked, database_header, empty_leaf, finish_writer, invalid_data, invalid_input, random_value, + spawn_replacement_writer, try_exclusive_sidecar, wait_for_exclusive_sidecar, ConditionalSerialize, NEXT_PAGE_ID, + PAGE_ID, +}; diff --git a/backend/btree/src/v3/tree/tests/behavior.rs b/backend/btree/src/v3/tree/tests/behavior.rs new file mode 100644 index 000000000..6780bd914 --- /dev/null +++ b/backend/btree/src/v3/tree/tests/behavior.rs @@ -0,0 +1,260 @@ +use super::{ + database_header, empty_leaf, invalid_data, invalid_input, publish_database, search_leaf, sync_parent_directory, + try_exclusive_sidecar, verify_full, wal_path, BPlusTree, BPlusTreeMetadata, BPlusTreeQuery, BPlusTreeUpdate, + ConditionalSerialize, FlushPolicy, LeafCellRef, LeafValueRef, NEXT_PAGE_ID, PAGE_ID, +}; +use crate::{ + codec::binary_serialize, + common::BPlusTreeError, + v3::{ + format::{encode_inline_leaf_cell, encode_internal_cell, encode_tombstone_leaf_cell, Compression}, + page::SlottedPage, + }, +}; +use serde::Deserialize; +use std::{fs, io, path::Path}; + +#[test] +fn new_key_is_inserted_into_existing_leaf_without_growth() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("leaf-insert.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("one")); + tree.insert(3u32, String::from("three")); + tree.store(&path)?; + let before = database_header(&path)?; + let before_len = fs::metadata(&path)?.len(); + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.upsert(&2, &String::from("two"))?; + + let after = database_header(&path)?; + assert_eq!(after.generation, before.generation + 1); + assert_eq!(after.root_page_id, before.root_page_id); + assert_eq!(fs::metadata(&path)?.len(), before_len); + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert_eq!( + query.iter().collect::>>()?, + vec![(1, "one".into()), (2, "two".into()), (3, "three".into())] + ); + let _ = verify_full(&mut query)?; + Ok(()) +} + +#[test] +fn delete_writes_tombstone_and_reinsert_reuses_the_leaf() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("tombstone.db"); + let mut tree = BPlusTree::new(); + tree.insert(7u32, String::from("original")); + tree.store(&path)?; + let before = database_header(&path)?; + let before_len = fs::metadata(&path)?.len(); + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + assert!(updater.delete(&7)?); + assert_eq!(updater.query(&7).map_err(BPlusTreeError::to_io)?, None); + updater.upsert(&7, &String::from("restored"))?; + + let after = database_header(&path)?; + assert_eq!(after.generation, before.generation + 2); + assert_eq!(after.root_page_id, before.root_page_id); + assert_eq!(fs::metadata(&path)?.len(), before_len); + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(String::from("restored"))); + assert_eq!(verify_full(&mut query)?.live_entries, 1); + Ok(()) +} + +#[test] +fn identical_batch_metadata_is_a_true_noop() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("metadata-noop.db"); + let mut tree = BPlusTree::::new(); + tree.set_metadata(BPlusTreeMetadata::TargetIdMapping(42)); + tree.store(&path)?; + let before = fs::read(&path)?; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.set_flush_policy(FlushPolicy::Batch); + updater.set_metadata(&BPlusTreeMetadata::TargetIdMapping(42))?; + assert!(updater.active.is_none()); + updater.commit()?; + + assert_eq!(fs::read(&path)?, before); + assert!(!wal_path(&path).try_exists()?); + Ok(()) +} + +#[test] +fn database_with_index_extension_is_never_deleted_as_derived_data() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("tree.idx"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("old")); + tree.store(&path)?; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.upsert(&1, &String::from("new"))?; + + assert!(path.try_exists()?); + assert!(!wal_path(&path).try_exists()?); + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, Some(String::from("new"))); + let _ = verify_full(&mut query)?; + Ok(()) +} + +#[test] +fn failed_batch_mutation_discards_the_whole_overlay() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("poisoned-batch.db"); + let mut tree = BPlusTree::new(); + tree.insert(String::from("key"), String::from("old")); + tree.store(&path)?; + let before = fs::read(&path)?; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.set_flush_policy(FlushPolicy::Batch); + updater.upsert(&String::from("key"), &String::from("staged"))?; + assert!(updater.upsert(&"x".repeat(2_100), &String::from("invalid")).is_err()); + updater.commit()?; + + assert_eq!(fs::read(&path)?, before); + assert!(!wal_path(&path).try_exists()?); + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert_eq!(query.query(&String::from("key")).map_err(BPlusTreeError::to_io)?, Some(String::from("old"))); + Ok(()) +} + +#[test] +fn batch_serialization_error_discards_earlier_items() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("batch-serialization.db"); + let initial = ConditionalSerialize { value: String::from("initial"), fail: false }; + let mut tree = BPlusTree::new(); + tree.insert(0u32, initial); + tree.store(&path)?; + let before = fs::read(&path)?; + let first = ConditionalSerialize { value: String::from("first"), fail: false }; + let failing = ConditionalSerialize { value: String::from("second"), fail: true }; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + assert!(updater.upsert_batch(&[(&1, &first), (&2, &failing)]).is_err()); + updater.commit()?; + + assert_eq!(fs::read(&path)?, before); + assert!(!wal_path(&path).try_exists()?); + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, None); + assert_eq!(query.query(&2).map_err(BPlusTreeError::to_io)?, None); + Ok(()) +} + +#[test] +fn exclusive_sidecar_probe_child() -> io::Result<()> { + let Some(path) = std::env::var_os("TULIPROX_V3_LOCK_PROBE_PATH") else { + return Ok(()); + }; + let expected = std::env::var("TULIPROX_V3_LOCK_PROBE_EXPECTED") + .map_err(|error| io::Error::other(format!("missing child probe expectation: {error}")))?; + let acquired = try_exclusive_sidecar(Path::new(&path))?; + match expected.as_str() { + "acquired" => assert!(acquired), + "blocked" => assert!(!acquired), + _ => return Err(io::Error::other(format!("unknown child probe expectation: {expected}"))), + } + Ok(()) +} + +#[test] +fn typed_leaf_cells_round_trip_and_validate_value_crc() -> io::Result<()> { + let mut cell = Vec::new(); + encode_inline_leaf_cell(b"key", 5, Compression::None, b"value", &mut cell)?; + let decoded = LeafCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)?; + assert_eq!(decoded.key_bytes, b"key"); + match decoded.value { + LeafValueRef::Inline { compression, logical_len, stored, crc32 } => { + assert_eq!(compression, Compression::None); + assert_eq!(logical_len, 5); + assert_eq!(stored, b"value"); + assert_eq!(crc32, crc32fast::hash(b"value")); + } + _ => return Err(io::Error::other("expected inline value")), + } + + let last = cell.len().checked_sub(1).ok_or_else(|| io::Error::other("empty test cell"))?; + cell[last] ^= 1; + invalid_data(LeafCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)) +} + +#[test] +fn encoded_key_limit_is_2004_bytes() -> io::Result<()> { + let mut cell = Vec::new(); + encode_tombstone_leaf_cell(&vec![b'k'; 2004], &mut cell)?; + assert_eq!(cell.len() + 4, 2032); + invalid_input(encode_tombstone_leaf_cell(&vec![b'k'; 2005], &mut cell))?; + + encode_internal_cell(&vec![b'k'; 2004], 9, PAGE_ID, NEXT_PAGE_ID, &mut cell)?; + invalid_input(encode_internal_cell(&vec![b'k'; 2005], 9, PAGE_ID, NEXT_PAGE_ID, &mut cell)) +} + +#[test] +fn typed_search_decodes_only_binary_search_candidates() -> io::Result<()> { + let mut cells = Vec::new(); + for key in 0u8..8 { + let mut cell = Vec::new(); + encode_inline_leaf_cell(&binary_serialize(&key)?, 1, Compression::None, &[key], &mut cell)?; + cells.push(cell); + } + if let Some(byte) = cells.get_mut(0).and_then(|cell| cell.get_mut(24)) { + *byte = 0xc1; + } + + let mut bytes = empty_leaf(PAGE_ID, NEXT_PAGE_ID)?; + let mut mutable = SlottedPage::open(bytes.as_mut_slice(), PAGE_ID, NEXT_PAGE_ID)?; + mutable.rebuild_ordered(cells.iter().map(Vec::as_slice))?; + let page = SlottedPage::open(bytes.as_slice(), PAGE_ID, NEXT_PAGE_ID)?; + assert_eq!(search_leaf(&page, &7u8)?, Ok(7)); + invalid_data(search_leaf(&page, &0u8)) +} + +#[derive(Debug)] +pub(in crate::v3::tree::tests) struct RejectValueDeserialize; + +impl<'de> Deserialize<'de> for RejectValueDeserialize { + fn deserialize>(_deserializer: D) -> Result { + Err(serde::de::Error::custom("value deserialization must not run")) + } +} + +#[test] +fn contains_and_len_read_only_leaf_descriptors() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("descriptor-only.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("one")); + tree.insert(2u32, String::from("two")); + tree.store(&path)?; + + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert!(query.contains_live_key(&1).map_err(BPlusTreeError::to_io)?); + assert!(!query.contains_live_key(&3).map_err(BPlusTreeError::to_io)?); + assert_eq!(query.len().map_err(BPlusTreeError::to_io)?, 2); + Ok(()) +} + +#[test] +fn publish_failure_removes_the_temporary_file() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let destination = dir.path().join("destination-directory"); + fs::create_dir(&destination)?; + let temporary = dir.path().join("database.tx.v3.tmp"); + fs::write(&temporary, b"new")?; + + let error = publish_database(&temporary, &destination, sync_parent_directory) + .expect_err("publishing over a directory must fail"); + assert!(!error.database_was_published()); + assert!(!temporary.exists()); + Ok(()) +} diff --git a/backend/btree/src/v3/tree/tests/configuration.rs b/backend/btree/src/v3/tree/tests/configuration.rs new file mode 100644 index 000000000..f21e8faf8 --- /dev/null +++ b/backend/btree/src/v3/tree/tests/configuration.rs @@ -0,0 +1,26 @@ +use super::BPlusTreeUpdate; +use crate::codec::binary_deserialize; +use std::io; + +#[test] +fn prepared_upsert_batch_is_key_sorted_and_stable() -> io::Result<()> { + let keys = [3u32, 1, 2, 1]; + let values = ["three", "first", "two", "last"].map(String::from); + let items = keys.iter().zip(&values).collect::>(); + + let prepared = BPlusTreeUpdate::::prepare_upsert_batch(&items)?; + let decoded = prepared + .into_iter() + .map(|(key, value)| binary_deserialize::(&value).map(|value| (key, value))) + .collect::>>()?; + assert_eq!( + decoded, + vec![ + (1, String::from("first")), + (1, String::from("last")), + (2, String::from("two")), + (3, String::from("three")), + ] + ); + Ok(()) +} diff --git a/backend/btree/src/v3/tree/tests/lifecycle.rs b/backend/btree/src/v3/tree/tests/lifecycle.rs new file mode 100644 index 000000000..41e347321 --- /dev/null +++ b/backend/btree/src/v3/tree/tests/lifecycle.rs @@ -0,0 +1,123 @@ +use super::{ + assert_writer_is_blocked, database_header, finish_writer, spawn_replacement_writer, try_exclusive_sidecar, + verify_full, wait_for_exclusive_sidecar, wal_path, BPlusTree, BPlusTreeQuery, BPlusTreeUpdate, + ConditionalSerialize, FlushPolicy, +}; +use crate::common::BPlusTreeError; +use std::{fs, io}; + +#[test] +fn dropped_uncommitted_batch_discards_overlay_without_wal() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("dropped-batch.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("old")); + tree.store(&path)?; + let before = fs::read(&path)?; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.set_flush_policy(FlushPolicy::Batch); + updater.upsert(&1, &String::from("uncommitted"))?; + assert_eq!(updater.query(&1).map_err(BPlusTreeError::to_io)?, Some(String::from("uncommitted"))); + drop(updater); + + assert_eq!(fs::read(&path)?, before); + assert!(!wal_path(&path).try_exists()?); + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, Some(String::from("old"))); + Ok(()) +} + +#[test] +fn direct_serialization_error_aborts_an_existing_batch() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("direct-serialization.db"); + let initial = ConditionalSerialize { value: String::from("initial"), fail: false }; + let mut tree = BPlusTree::new(); + tree.insert(0u32, initial); + tree.store(&path)?; + let before = fs::read(&path)?; + let staged = ConditionalSerialize { value: String::from("staged"), fail: false }; + let failing = ConditionalSerialize { value: String::from("failing"), fail: true }; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.set_flush_policy(FlushPolicy::Batch); + updater.upsert(&1, &staged)?; + assert!(updater.upsert(&2, &failing).is_err()); + updater.commit()?; + + assert_eq!(fs::read(&path)?, before); + assert!(!wal_path(&path).try_exists()?); + Ok(()) +} + +#[test] +fn delete_key_serialization_error_aborts_an_existing_batch() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("delete-serialization.db"); + let initial = ConditionalSerialize { value: String::from("initial"), fail: false }; + let mut tree = BPlusTree::new(); + tree.insert(initial.clone(), String::from("old")); + tree.store(&path)?; + let before = fs::read(&path)?; + let staged = ConditionalSerialize { value: String::from("staged"), fail: false }; + let failing = ConditionalSerialize { value: String::from("failing"), fail: true }; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.set_flush_policy(FlushPolicy::Batch); + updater.upsert(&staged, &String::from("new"))?; + assert!(updater.delete(&failing).is_err()); + updater.commit()?; + + assert_eq!(fs::read(&path)?, before); + assert!(!wal_path(&path).try_exists()?); + Ok(()) +} + +#[test] +fn idle_updater_refreshes_after_full_replacement() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("replacement-refresh.db"); + let mut original = BPlusTree::new(); + original.insert(1u32, String::from("original")); + original.store(&path)?; + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + let original_id = updater.database_id; + + let mut replacement = BPlusTree::new(); + replacement.insert(2u32, String::from("replacement")); + replacement.store(&path)?; + let replacement_id = database_header(&path)?.database_id; + assert_ne!(replacement_id, original_id); + + updater.upsert(&3, &String::from("updated"))?; + + assert_eq!(updater.database_id, replacement_id); + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, None); + assert_eq!(query.query(&2).map_err(BPlusTreeError::to_io)?, Some(String::from("replacement"))); + assert_eq!(query.query(&3).map_err(BPlusTreeError::to_io)?, Some(String::from("updated"))); + let _ = verify_full(&mut query)?; + Ok(()) +} + +#[test] +fn shared_queries_coexist_and_block_replacement_until_both_drop() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("shared-queries.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("original")); + tree.store(&path)?; + + let first = BPlusTreeQuery::::try_new(&path)?; + let second = first.try_clone()?; + assert!(!try_exclusive_sidecar(&path)?); + drop(first); + assert!(!try_exclusive_sidecar(&path)?); + let (receiver, handle) = spawn_replacement_writer(path.clone())?; + assert_writer_is_blocked(&receiver)?; + drop(second); + finish_writer(&receiver, handle)?; + assert!(wait_for_exclusive_sidecar(&path)?); + Ok(()) +} diff --git a/backend/btree/src/v3/tree/tests/persistence.rs b/backend/btree/src/v3/tree/tests/persistence.rs new file mode 100644 index 000000000..a8b58cc98 --- /dev/null +++ b/backend/btree/src/v3/tree/tests/persistence.rs @@ -0,0 +1,689 @@ +use super::{ + database_header, publish_database, random_value, verify_full, wal_path, wal_temporary_path, BPlusTree, + BPlusTreeQuery, BPlusTreeSerialWriter, BPlusTreeUpdate, FlushPolicy, LeafCellRef, LeafValueRef, +}; +use crate::{ + common::BPlusTreeError, + v3::{ + format::{encode_overflow_leaf_cell, PAGE_HEADER_LEN, PAGE_SIZE}, + page::SlottedPage, + }, +}; +use std::{ + fs, io, + path::{Path, PathBuf}, +}; + +#[test] +fn smaller_inline_update_compacts_without_appending_pages() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("smaller-inline.db"); + let mut tree = BPlusTree::new(); + tree.insert(7u32, String::from("a much longer inline value")); + tree.store(&path)?; + let before = database_header(&path)?; + let before_len = fs::metadata(&path)?.len(); + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.update(&7, String::from("x")).map_err(BPlusTreeError::to_io)?; + + let after = database_header(&path)?; + assert_eq!(after.generation, before.generation + 1); + assert_eq!(after.root_page_id, before.root_page_id); + assert_eq!(fs::metadata(&path)?.len(), before_len); + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(String::from("x"))); + let _ = verify_full(&mut query)?; + Ok(()) +} + +#[test] +fn growing_inline_value_uses_page_local_compaction_before_split() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("growing-inline.db"); + let mut tree = BPlusTree::new(); + for key in 0..8u32 { + tree.insert(key, vec![u8::try_from(key).map_err(io::Error::other)?; 8]); + } + tree.store(&path)?; + let before = database_header(&path)?; + let before_len = fs::metadata(&path)?.len(); + let grown = vec![0x5a; 200]; + + let mut updater = BPlusTreeUpdate::>::try_new(&path)?; + updater.update(&4, grown.clone()).map_err(BPlusTreeError::to_io)?; + + let after = database_header(&path)?; + assert_eq!(after.generation, before.generation + 1); + assert_eq!(after.root_page_id, before.root_page_id); + assert_eq!(fs::metadata(&path)?.len(), before_len); + let mut query = BPlusTreeQuery::>::try_new(&path)?; + assert_eq!(query.query(&4).map_err(BPlusTreeError::to_io)?, Some(grown)); + let _ = verify_full(&mut query)?; + Ok(()) +} + +#[test] +fn medium_stored_value_remains_inline() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("medium-inline.db"); + let value = random_value().get(..300).ok_or_else(|| io::Error::other("random test value is too short"))?.to_vec(); + let mut tree = BPlusTree::new(); + tree.insert(7u32, value.clone()); + + let report = tree.store_verified(&path)?; + + assert_eq!(report.overflow_pages, 0); + let mut query = BPlusTreeQuery::>::try_new(&path)?; + assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(value)); + Ok(()) +} + +/// The Xtream cluster import commits after every batch so the transaction's +/// dirty-page map stays bounded. That reopens the write transaction mid-import, +/// so every batch must survive and stay readable — including values large enough +/// to spill into overflow chains, which is what dominated the dirty-page map. +#[test] +fn committing_between_batches_keeps_every_entry_readable() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("batched-commits.db"); + BPlusTree::::new().store(&path)?; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.set_flush_policy(FlushPolicy::Batch); + for batch in 0..4u32 { + let keys: Vec = (0..8).map(|index| batch * 8 + index).collect(); + let values: Vec = keys.iter().map(|key| format!("{key}").repeat(600)).collect(); + let items = keys.iter().zip(&values).collect::>(); + let prepared = BPlusTreeUpdate::::prepare_upsert_batch(&items)?; + updater.upsert_batch_encoded(prepared)?; + updater.commit()?; + assert!(updater.active.is_none(), "commit must release the transaction after batch {batch}"); + } + + let mut query = BPlusTreeQuery::::try_new(&path)?; + for key in 0..32u32 { + assert_eq!( + query.query(&key).map_err(BPlusTreeError::to_io)?, + Some(format!("{key}").repeat(600)), + "key {key} is missing after a mid-import commit" + ); + } + assert!(!wal_path(&path).try_exists()?); + Ok(()) +} + +/// Throughput benchmark for `BPlusTree::store` across three workload sizes. +#[test] +#[ignore = "benchmark; run explicitly with --ignored --nocapture"] +fn bench_store_throughput() -> io::Result<()> { + let noise = |seed: u32, len: usize| -> Vec { + let mut state = seed.wrapping_mul(2_654_435_761).wrapping_add(1); + (0..len) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + u8::try_from(state >> 24).unwrap_or(0) + }) + .collect() + }; + for (label, count, size) in [("klein", 2_000u32, 800usize), ("mittel", 20_000, 2_000), ("gross", 60_000, 2_000)] { + let dir = tempfile::tempdir()?; + let path = dir.path().join("bench.db"); + let mut tree = BPlusTree::>::new(); + for key in 0..count { + tree.insert(key, noise(key, size)); + } + let start = std::time::Instant::now(); + tree.store(&path)?; + let elapsed = start.elapsed(); + println!( + "BENCH {label:6} entries={count:6} bytes={size:5} -> {:>8.1} ms file={} MiB", + elapsed.as_secs_f64() * 1000.0, + fs::metadata(&path)?.len() / (1024 * 1024) + ); + } + Ok(()) +} + +#[test] +fn streamed_store_writes_every_page_despite_out_of_order_overflow_chains() -> io::Result<()> { + // The streaming builder hands out a leaf's page id *before* the overflow chain it + // points at, but writes the leaf *after* those pages — so page ids do not reach the + // file in ascending order. Worse, this nests: leaf A is reserved, its chains are + // written, A is written, then leaf B repeats the whole dance at higher ids. + // + // The entry count is sized to produce many leaves, not one: overflow leaf cells are + // tiny (key plus pointers, ~34 bytes), so a few hundred entries would all land in a + // single leaf and never exercise the nesting. Values are incompressible on purpose — + // a repeated string would compress back under `MAX_INLINE_STORED_VALUE` and skip + // overflow entirely. Both properties are asserted below rather than assumed. + // + // The file-length check is the real guard: writing pages sequentially instead of + // positionally would leave the file short or the ids scrambled. + // Deterministic LCG — compresses badly, so values are forced into overflow chains. + let noise = |seed: u32, len: usize| -> Vec { + let mut state = seed.wrapping_mul(2_654_435_761).wrapping_add(1); + (0..len) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + u8::try_from(state >> 24).unwrap_or(0) + }) + .collect() + }; + + let dir = tempfile::tempdir()?; + let path = dir.path().join("streamed.db"); + let mut tree = BPlusTree::>::new(); + for key in 0..1_200u32 { + // Spans one, two and three overflow pages (OVERFLOW_PAYLOAD_LEN = 4056). + tree.insert(key, noise(key, 3000 + (key as usize % 3) * 4000)); + } + let report = tree.store_verified(&path)?; + assert!( + report.overflow_pages > 2_000, + "test must force multi-page overflow chains, got {} overflow pages", + report.overflow_pages + ); + assert!( + report.tree_pages > 8, + "test must produce many leaves so the reserve-then-write dance nests, got {} tree pages", + report.tree_pages + ); + + let header = BPlusTreeQuery::>::try_new(&path)?.header; + assert_eq!( + fs::metadata(&path)?.len(), + header.next_page_id * PAGE_SIZE as u64, + "file length must match the allocated page count exactly" + ); + + let mut query = BPlusTreeQuery::>::try_new(&path)?; + for key in 0..1_200u32 { + assert_eq!( + query.query(&key).map_err(BPlusTreeError::to_io)?, + Some(noise(key, 3000 + (key as usize % 3) * 4000)), + "key {key} did not survive the streamed store" + ); + } + Ok(()) +} + +/// Compaction is NOT optional cleanup on an insert-built tree: leaf splits leave +/// pages roughly half full, and `build_pages` repacks them to near-capacity. +/// +/// Note `free_page_head` stays 0 throughout — inserts never free a page — so it is +/// NOT a usable "nothing to compact" signal. Guarding compaction on an empty free +/// list would skip exactly the case that gains the most. +#[test] +fn compaction_repacks_an_insert_built_tree_despite_an_empty_free_list() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("repack.db"); + BPlusTree::::new().store(&path)?; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.set_flush_policy(FlushPolicy::Batch); + for batch in 0..8u32 { + let keys: Vec = (0..250).map(|index| batch * 250 + index).collect(); + let values: Vec = keys.iter().map(|key| format!("{key:07}").repeat(90)).collect(); + let items = keys.iter().zip(&values).collect::>(); + updater.upsert_batch_encoded(BPlusTreeUpdate::::prepare_upsert_batch(&items)?)?; + updater.commit()?; + } + + let before = BPlusTreeQuery::::try_new(&path)?.header; + assert_eq!(before.free_page_head, 0, "inserts must not free pages"); + + updater.compact()?; + + let after = BPlusTreeQuery::::try_new(&path)?.header; + assert!( + after.next_page_id * 3 < before.next_page_id * 2, + "compaction must reclaim over a third of the pages, got {} -> {}", + before.next_page_id, + after.next_page_id + ); + + let mut query = BPlusTreeQuery::::try_new(&path)?; + for key in 0..2000u32 { + assert_eq!( + query.query(&key).map_err(BPlusTreeError::to_io)?, + Some(format!("{key:07}").repeat(90)), + "key {key} did not survive compaction" + ); + } + Ok(()) +} + +#[test] +fn immediate_commit_clears_wal_then_invalidates_sorted_index() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("immediate.db"); + let index_path = crate::common::get_file_path_for_db_index(&path); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("old")); + tree.store(&path)?; + fs::write(&index_path, b"derived")?; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.upsert(&1, &String::from("new"))?; + + assert!(!wal_path(&path).try_exists()?); + assert!(!wal_temporary_path(&path).try_exists()?); + assert!(!index_path.try_exists()?); + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, Some(String::from("new"))); + Ok(()) +} + +#[test] +fn missing_delete_and_empty_commit_are_true_noops() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("no-op.db"); + let index_path = crate::common::get_file_path_for_db_index(&path); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("one")); + tree.store(&path)?; + fs::write(&index_path, b"still-valid")?; + let before = fs::read(&path)?; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + assert!(!updater.delete(&2)?); + updater.commit()?; + + assert_eq!(fs::read(&path)?, before); + assert!(index_path.try_exists()?); + assert!(!wal_path(&path).try_exists()?); + Ok(()) +} + +#[test] +fn serial_writer_commits_a_batch_and_shuts_down_cleanly() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("serial-writer.db"); + BPlusTree::::new().store(&path)?; + + let writer = BPlusTreeSerialWriter::new(&path, FlushPolicy::Batch)?; + let one = String::from("one"); + let two = String::from("two"); + assert_ne!(writer.upsert(&[(&1, &one), (&2, &two)])?, 0); + writer.shutdown()?; + + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert_eq!(query.iter().collect::>>()?, vec![(1, one), (2, two)]); + Ok(()) +} + +#[test] +fn store_with_index_publishes_an_identity_bound_sorted_snapshot() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("indexed.db"); + let index_path = crate::common::get_file_path_for_db_index(&path); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("ccc")); + tree.insert(2u32, String::from("a")); + tree.insert(3u32, String::from("bb")); + + assert_ne!(tree.store_with_index(&path, String::len)?, 0); + + let query = BPlusTreeQuery::::try_new(&path)?; + let mut sorted = crate::sorted_index::v4::OwnedIterator::::open(query, &index_path)?; + assert_eq!( + sorted.by_ref().collect::>>()?, + vec![(2, String::from("a")), (3, String::from("bb")), (1, String::from("ccc"))] + ); + assert_eq!(sorted.remaining(), 0); + assert!(!fs::read_dir(dir.path())? + .any(|entry| entry.is_ok_and(|entry| entry.file_name().to_string_lossy().ends_with(".v3.tmp")))); + Ok(()) +} + +#[test] +fn compact_rebuilds_only_live_data_with_a_fresh_identity() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("compact.db"); + let index_path = crate::common::get_file_path_for_db_index(&path); + let mut tree = BPlusTree::new(); + for key in 0..100u32 { + tree.insert( + key, + if key % 10 == 0 { random_value() } else { vec![u8::try_from(key).map_err(io::Error::other)?; 32] }, + ); + } + tree.store(&path)?; + let original_id = database_header(&path)?.database_id; + + let mut updater = BPlusTreeUpdate::>::try_new(&path)?; + updater.set_flush_policy(FlushPolicy::Batch); + for key in (0..100u32).step_by(2) { + assert!(updater.delete(&key)?); + } + updater.commit()?; + let length_before = fs::metadata(&path)?.len(); + fs::write(&index_path, b"stale")?; + + updater.compact()?; + + let header = database_header(&path)?; + assert_ne!(header.database_id, original_id); + assert_eq!(header.generation, 1); + assert!(fs::metadata(&path)?.len() <= length_before); + assert!(!index_path.try_exists()?); + let mut query = BPlusTreeQuery::>::try_new(&path)?; + let entries = query.iter().collect::>>()?; + assert_eq!(entries.len(), 50); + assert!(entries.iter().all(|(key, _)| key % 2 == 1)); + let report = verify_full(&mut query)?; + assert_eq!(report.live_entries, 50); + assert_eq!(report.free_pages, 0); + Ok(()) +} + +#[test] +fn compact_read_failure_preserves_database_and_index() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("compact-corrupt.db"); + let index_path = crate::common::get_file_path_for_db_index(&path); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("one")); + tree.store(&path)?; + let updater = BPlusTreeUpdate::::try_new(&path)?; + + let header = database_header(&path)?; + let mut corrupted = fs::read(&path)?; + let offset = usize::try_from(header.root_page_id) + .map_err(io::Error::other)? + .checked_mul(PAGE_SIZE) + .and_then(|start| start.checked_add(PAGE_HEADER_LEN)) + .ok_or_else(|| io::Error::other("corruption offset overflow"))?; + *corrupted.get_mut(offset).ok_or_else(|| io::Error::other("corruption offset outside database"))? ^= 0xff; + fs::write(&path, &corrupted)?; + fs::write(&index_path, b"still-valid")?; + + let mut updater = updater; + assert!(updater.compact().is_err()); + assert_eq!(fs::read(&path)?, corrupted); + assert_eq!(fs::read(&index_path)?, b"still-valid"); + Ok(()) +} + +#[test] +fn batch_overlay_is_visible_to_updater_and_commits_once() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("batch.db"); + let value: String = random_value() + .get(..200) + .ok_or_else(|| io::Error::other("random test value is too short"))? + .iter() + .map(|byte| char::from(33 + byte % 90)) + .collect(); + let mut tree = BPlusTree::new(); + for key in 0..17u32 { + tree.insert(key, value.clone()); + } + tree.store(&path)?; + let before = database_header(&path)?; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.set_flush_policy(FlushPolicy::Batch); + updater.upsert(&17, &value)?; + updater.upsert(&18, &value)?; + assert_eq!(updater.query(&18).map_err(BPlusTreeError::to_io)?, Some(value.clone())); + assert_eq!(database_header(&path)?.generation, before.generation); + + updater.commit()?; + + let after = database_header(&path)?; + assert_eq!(after.generation, before.generation + 1); + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert_eq!(query.query(&17).map_err(BPlusTreeError::to_io)?, Some(value.clone())); + assert_eq!(query.query(&18).map_err(BPlusTreeError::to_io)?, Some(value)); + assert_eq!(verify_full(&mut query)?.live_entries, 19); + Ok(()) +} + +#[test] +fn final_overlay_validation_rejects_unordered_leaf_keys_before_wal() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("unordered-overlay.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("one")); + tree.insert(2u32, String::from("two")); + tree.store(&path)?; + let before = fs::read(&path)?; + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.set_flush_policy(FlushPolicy::Batch); + updater.upsert(&1, &String::from("staged"))?; + let active = updater.active.as_mut().ok_or_else(|| io::Error::other("test transaction is missing"))?; + let leaf_id = active.transaction.next_header.root_page_id; + let snapshot = active.transaction.page_copy(active.base.as_slice(), leaf_id)?; + let page = SlottedPage::open(snapshot.as_slice(), leaf_id, active.transaction.next_header.next_page_id)?; + let mut cells = page.cells().map(|cell| cell.map(<[u8]>::to_vec)).collect::>>()?; + let duplicate = cells.first().cloned().ok_or_else(|| io::Error::other("test leaf has no cells"))?; + *cells.get_mut(1).ok_or_else(|| io::Error::other("test leaf lacks a second cell"))? = duplicate; + let next_page_id = active.transaction.next_header.next_page_id; + let dirty = active.transaction.page_mut(active.base.as_slice(), leaf_id)?; + SlottedPage::open(dirty.as_mut_slice(), leaf_id, next_page_id)?.rebuild_ordered(cells.iter().map(Vec::as_slice))?; + + let error = updater.commit().err().ok_or_else(|| io::Error::other("unordered overlay was committed"))?; + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(fs::read(&path)?, before); + assert!(!wal_path(&path).try_exists()?); + Ok(()) +} + +#[test] +fn final_overlay_validation_rejects_shared_overflow_before_wal() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("shared-overflow-overlay.db"); + let first_value = random_value(); + let mut second_value = first_value.clone(); + second_value.reverse(); + let mut tree = BPlusTree::new(); + tree.insert(1u32, first_value); + tree.insert(2u32, second_value); + tree.store(&path)?; + let before = fs::read(&path)?; + + let mut updater = BPlusTreeUpdate::>::try_new(&path)?; + updater.set_flush_policy(FlushPolicy::Batch); + updater.ensure_transaction()?; + let active = updater.active.as_mut().ok_or_else(|| io::Error::other("test transaction is missing"))?; + let leaf_id = active.transaction.next_header.root_page_id; + let next_page_id = active.transaction.next_header.next_page_id; + let snapshot = active.transaction.page_copy(active.base.as_slice(), leaf_id)?; + let page = SlottedPage::open(snapshot.as_slice(), leaf_id, next_page_id)?; + let first = LeafCellRef::decode(page.cell(0)?, leaf_id, next_page_id)?; + let second = LeafCellRef::decode(page.cell(1)?, leaf_id, next_page_id)?; + let LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } = first.value else { + return Err(io::Error::other("first test value is not overflow-backed")); + }; + let mut replacement = Vec::new(); + encode_overflow_leaf_cell( + second.key_bytes, + logical_len, + compression, + stored_len, + head, + crc32, + leaf_id, + next_page_id, + &mut replacement, + )?; + let dirty = active.transaction.page_mut(active.base.as_slice(), leaf_id)?; + SlottedPage::open(dirty.as_mut_slice(), leaf_id, next_page_id)?.replace_same_len(1, &replacement)?; + + let error = updater.commit().err().ok_or_else(|| io::Error::other("shared overflow was committed"))?; + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(fs::read(&path)?, before); + assert!(!wal_path(&path).try_exists()?); + Ok(()) +} + +pub(in crate::v3::tree::tests) fn pending_path(database: &Path, suffix: &str) -> PathBuf { + let mut name = database.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +#[test] +fn query_removes_abandoned_wal_temp_and_rejects_corrupt_active_wal() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("pending-recovery.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("original")); + tree.store(&path)?; + + let temporary = pending_path(&path, ".wal.tmp"); + fs::write(&temporary, b"not activated")?; + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, Some(String::from("original"))); + assert!(!temporary.try_exists()?); + drop(query); + + let active = pending_path(&path, ".wal"); + fs::write(&active, b"corrupt active WAL")?; + let active_before = fs::read(&active)?; + let error = BPlusTreeQuery::::try_new(&path) + .err() + .ok_or_else(|| io::Error::other("query accepted corrupt active WAL"))?; + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(fs::read(active)?, active_before); + Ok(()) +} + +#[test] +fn replacement_store_removes_wal_temp_and_preserves_corrupt_active_wal() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("pending-store.db"); + let mut original = BPlusTree::new(); + original.insert(1u32, String::from("original")); + original.store(&path)?; + let database_before = fs::read(&path)?; + + let temporary = pending_path(&path, ".wal.tmp"); + fs::write(&temporary, b"not activated")?; + let mut replacement = BPlusTree::new(); + replacement.insert(2u32, String::from("replacement")); + replacement.store(&path)?; + assert!(!temporary.try_exists()?); + + let published_before = fs::read(&path)?; + let active = pending_path(&path, ".wal"); + let active_before = b"corrupt active WAL".to_vec(); + fs::write(&active, &active_before)?; + let mut rejected = BPlusTree::new(); + rejected.insert(3u32, String::from("rejected")); + let error = rejected.store(&path).err().ok_or_else(|| io::Error::other("store accepted corrupt WAL"))?; + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_ne!(published_before, database_before); + assert_eq!(fs::read(&path)?, published_before); + assert_eq!(fs::read(active)?, active_before); + Ok(()) +} + +#[test] +fn clean_loaded_store_recovers_temp_but_rejects_corrupt_active_wal() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("pending-clean-store.db"); + let mut original = BPlusTree::new(); + original.insert(1u32, String::from("original")); + original.store(&path)?; + let database_before = fs::read(&path)?; + let mut loaded = BPlusTree::::load(&path)?; + + let temporary = pending_path(&path, ".wal.tmp"); + fs::write(&temporary, b"not activated")?; + assert_eq!(loaded.store(&path)?, 0); + assert!(!temporary.try_exists()?); + assert_eq!(fs::read(&path)?, database_before); + + let active = pending_path(&path, ".wal"); + let active_before = b"corrupt active WAL".to_vec(); + fs::write(&active, &active_before)?; + let error = loaded.store(&path).err().ok_or_else(|| io::Error::other("clean store accepted corrupt WAL"))?; + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(fs::read(&path)?, database_before); + assert_eq!(fs::read(active)?, active_before); + Ok(()) +} + +#[test] +fn publish_reports_post_commit_directory_sync_failure() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let destination = dir.path().join("database.db"); + let temporary = dir.path().join("database.tx.v3.tmp"); + let mut old = BPlusTree::new(); + old.insert(1u32, vec![1]); + old.store(&destination)?; + let mut new = BPlusTree::new(); + new.insert(2u32, vec![2]); + new.store(&temporary)?; + + let error = + publish_database(&temporary, &destination, |_| Err(io::Error::other("injected directory sync failure"))) + .err() + .ok_or_else(|| io::Error::other("post-commit sync failure was hidden"))?; + assert!(error.database_was_published()); + let error = io::Error::from(error); + assert!(error.to_string().contains("database published but directory sync failed; durability unknown")); + assert!(!temporary.exists()); + let mut query = BPlusTreeQuery::>::try_new(&destination)?; + assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, None); + assert_eq!(query.query(&2).map_err(BPlusTreeError::to_io)?, Some(vec![2])); + Ok(()) +} + +#[test] +fn store_sync_failure_after_rename_invalidates_previous_sorted_index() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let destination = dir.path().join("database.db"); + let index = crate::common::get_file_path_for_db_index(&destination); + let mut old = BPlusTree::new(); + old.insert(1u32, String::from("old")); + let _ = old.store_with_index(&destination, String::clone)?; + assert!(index.is_file()); + + let mut replacement = BPlusTree::new(); + replacement.insert(2u32, String::from("new")); + let error = replacement + .store_exclusive_with_directory_sync(&destination, |_| Err(io::Error::other("injected directory sync failure"))) + .err() + .ok_or_else(|| io::Error::other("post-rename sync failure was hidden"))?; + + assert!(error.to_string().contains("publication durability remains unknown")); + assert!(!index.exists()); + assert!(replacement.dirty); + let mut query = BPlusTreeQuery::::try_new(&destination)?; + assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, None); + assert_eq!(query.query(&2).map_err(BPlusTreeError::to_io)?, Some(String::from("new"))); + Ok(()) +} + +#[test] +fn store_reports_sync_and_index_invalidation_failures_together() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let destination = dir.path().join("database.db"); + let index = crate::common::get_file_path_for_db_index(&destination); + let mut old = BPlusTree::new(); + old.insert(1u32, String::from("old")); + let _ = old.store(&destination)?; + fs::create_dir(&index)?; + + let mut replacement = BPlusTree::new(); + replacement.insert(2u32, String::from("new")); + let error = replacement + .store_exclusive_with_directory_sync(&destination, |_| Err(io::Error::other("injected directory sync failure"))) + .err() + .ok_or_else(|| io::Error::other("publish and index invalidation failures were hidden"))?; + let message = error.to_string(); + + assert!(message.contains("unknown directory durability")); + assert!(message.contains("previous sorted index could not be invalidated")); + assert!(index.is_dir()); + assert!(replacement.dirty); + let mut query = BPlusTreeQuery::::try_new(&destination)?; + assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, None); + assert_eq!(query.query(&2).map_err(BPlusTreeError::to_io)?, Some(String::from("new"))); + Ok(()) +} diff --git a/backend/btree/src/v3/tree/tests/protocol.rs b/backend/btree/src/v3/tree/tests/protocol.rs new file mode 100644 index 000000000..f8107e652 --- /dev/null +++ b/backend/btree/src/v3/tree/tests/protocol.rs @@ -0,0 +1,41 @@ +use super::{database_header, BPlusTree, BPlusTreeMetadata, BPlusTreeQuery, BPlusTreeUpdate}; +use crate::common::BPlusTreeError; +use std::{fs, io, ops::Bound}; + +#[test] +fn metadata_update_is_a_single_header_transaction() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("metadata.db"); + let mut tree = BPlusTree::::new(); + tree.insert(1, String::from("one")); + tree.store(&path)?; + let before = database_header(&path)?; + let before_len = fs::metadata(&path)?.len(); + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.set_metadata(&BPlusTreeMetadata::TargetIdMapping(42))?; + + let after = database_header(&path)?; + assert_eq!(after.generation, before.generation + 1); + assert_eq!(after.metadata, BPlusTreeMetadata::TargetIdMapping(42)); + assert_eq!(fs::metadata(&path)?.len(), before_len); + assert_eq!(updater.get_metadata()?, BPlusTreeMetadata::TargetIdMapping(42)); + Ok(()) +} + +#[test] +fn range_page_does_not_preallocate_the_requested_limit() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("range-limit.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("one")); + tree.insert(2u32, String::from("two")); + tree.store(&path)?; + + let mut query = BPlusTreeQuery::::try_new(&path)?; + let (entries, has_more) = + query.range_page(Bound::Unbounded, Bound::Unbounded, 0, usize::MAX).map_err(BPlusTreeError::to_io)?; + assert_eq!(entries, vec![(1, String::from("one")), (2, String::from("two"))]); + assert!(!has_more); + Ok(()) +} diff --git a/backend/btree/src/v3/tree/tests/query.rs b/backend/btree/src/v3/tree/tests/query.rs new file mode 100644 index 000000000..9c5da8789 --- /dev/null +++ b/backend/btree/src/v3/tree/tests/query.rs @@ -0,0 +1,1108 @@ +use super::{ + assert_writer_is_blocked, choose_internal_split, choose_leaf_split, database_header, decompress_value_in_place, + empty_leaf, finish_writer, internal_key_decode_count, invalid_data, invalid_input, page_byte_range, random_value, + read_leaf_value, reset_internal_key_decode_count, search_leaf, spawn_replacement_writer, try_exclusive_sidecar, + used_leaf_bytes, validate_locator, verify_full, wait_for_exclusive_sidecar, BPlusTree, BPlusTreeQuery, + BPlusTreeUpdate, DatabaseHeader, InternalCellRef, LeafCellRef, LeafValueRef, Locator, NEXT_PAGE_ID, PAGE_ID, + SLOT_LEN, +}; +use crate::{ + codec::binary_serialize, + common::BPlusTreeError, + v3::{ + format::{ + encode_inline_leaf_cell, encode_internal_cell, encode_overflow_leaf_cell, encode_tombstone_leaf_cell, + encode_value, Compression, PageType, OVERFLOW_PAYLOAD_LEN, PAGE_HEADER_LEN, PAGE_SIZE, + }, + page::{encode_free_page, encode_overflow_page, page_open_count, reset_page_open_count, SlottedPage}, + }, +}; +use serde::{Deserialize, Serialize}; +use std::{collections::HashSet, fs, io, path::Path, process::Command}; + +#[test] +fn point_query_validates_a_single_leaf_once() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("point-query-page-opens.db"); + let mut tree = BPlusTree::new(); + tree.insert(7u32, String::from("value")); + tree.store(&path)?; + + let mut query = BPlusTreeQuery::::try_new(&path)?; + reset_page_open_count(); + assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(String::from("value"))); + assert_eq!(page_open_count(), 1); + Ok(()) +} + +#[test] +fn repeated_point_query_reuses_snapshot_page_validation_and_internal_keys() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("repeated-point-query.db"); + let mut tree = BPlusTree::new(); + for key in 0..2_000u32 { + tree.insert(key, format!("value-{key:04}")); + } + tree.store(&path)?; + + let mut query = BPlusTreeQuery::::try_new(&path)?; + reset_page_open_count(); + reset_internal_key_decode_count(); + assert_eq!(query.query(&1_337).map_err(BPlusTreeError::to_io)?, Some(String::from("value-1337"))); + let first_page_opens = page_open_count(); + let first_internal_decodes = internal_key_decode_count(); + if first_page_opens < 2 || first_internal_decodes == 0 { + return Err(io::Error::other("test fixture must contain internal pages")); + } + + assert_eq!(query.query(&1_337).map_err(BPlusTreeError::to_io)?, Some(String::from("value-1337"))); + assert_eq!(page_open_count(), first_page_opens); + assert_eq!(internal_key_decode_count(), first_internal_decodes); + Ok(()) +} + +#[test] +fn cloned_query_reuses_snapshot_page_validation_and_internal_keys() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("cloned-point-query.db"); + let mut tree = BPlusTree::new(); + for key in 0..2_000u32 { + tree.insert(key, format!("value-{key:04}")); + } + tree.store(&path)?; + + let mut query = BPlusTreeQuery::::try_new(&path)?; + reset_page_open_count(); + reset_internal_key_decode_count(); + assert_eq!(query.query(&1_337).map_err(BPlusTreeError::to_io)?, Some(String::from("value-1337"))); + let first_page_opens = page_open_count(); + let first_internal_decodes = internal_key_decode_count(); + if first_page_opens < 2 || first_internal_decodes == 0 { + return Err(io::Error::other("test fixture must contain internal pages")); + } + + let mut cloned = query.try_clone()?; + assert_eq!(cloned.query(&1_337).map_err(BPlusTreeError::to_io)?, Some(String::from("value-1337"))); + assert_eq!(page_open_count(), first_page_opens); + assert_eq!(internal_key_decode_count(), first_internal_decodes); + Ok(()) +} + +#[test] +fn repeated_query_le_reuses_snapshot_page_validation_and_internal_keys() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("repeated-query-le.db"); + let mut tree = BPlusTree::new(); + for key in 0..2_000u32 { + tree.insert(key, format!("value-{key:04}")); + } + tree.store(&path)?; + + let mut query = BPlusTreeQuery::::try_new(&path)?; + reset_page_open_count(); + reset_internal_key_decode_count(); + assert_eq!(query.query_le(&1_337).map_err(BPlusTreeError::to_io)?, Some(String::from("value-1337"))); + let first_page_opens = page_open_count(); + let first_internal_decodes = internal_key_decode_count(); + if first_page_opens < 2 || first_internal_decodes == 0 { + return Err(io::Error::other("test fixture must contain internal pages")); + } + + assert_eq!(query.query_le(&1_337).map_err(BPlusTreeError::to_io)?, Some(String::from("value-1337"))); + assert_eq!(page_open_count(), first_page_opens); + assert_eq!(internal_key_decode_count(), first_internal_decodes); + Ok(()) +} + +#[test] +fn locator_collection_validates_each_leaf_once() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("locator-page-opens.db"); + let mut tree = BPlusTree::new(); + for key in 0..3u32 { + tree.insert(key, format!("value-{key}")); + } + tree.store(&path)?; + + let mut query = BPlusTreeQuery::::try_new(&path)?; + reset_page_open_count(); + assert_eq!(query.collect_with_locators()?.len(), 3); + assert_eq!(page_open_count(), 1); + Ok(()) +} + +#[test] +fn equal_size_inline_update_rewrites_without_appending_pages() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("equal-inline.db"); + let mut tree = BPlusTree::new(); + tree.insert(7u32, String::from("old")); + tree.store(&path)?; + let before = database_header(&path)?; + let before_len = fs::metadata(&path)?.len(); + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.update(&7, String::from("new")).map_err(BPlusTreeError::to_io)?; + drop(updater); + + let after = database_header(&path)?; + assert_eq!(after.generation, before.generation + 1); + assert_eq!(after.root_page_id, before.root_page_id); + assert_eq!(fs::metadata(&path)?.len(), before_len); + let mut query = BPlusTreeQuery::::try_new(&path)?; + assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(String::from("new"))); + let _ = verify_full(&mut query)?; + Ok(()) +} + +pub(in crate::v3::tree::tests) fn overflow_head(path: &Path, key: u32) -> io::Result { + let mut query = BPlusTreeQuery::>::try_new(path)?; + let leaf = query.locate_leaf(&key)?; + let next_page_id = query.header.next_page_id; + query.with_page(leaf, |bytes, _| { + let page = SlottedPage::open(bytes, leaf, next_page_id)?; + let index = search_leaf(&page, &key)?.map_err(|_| io::Error::other("test key is missing"))?; + match LeafCellRef::decode(page.cell(index)?, leaf, next_page_id)?.value { + LeafValueRef::Overflow { head, .. } => Ok(head), + LeafValueRef::Inline { .. } | LeafValueRef::Tombstone => { + Err(io::Error::other("test value is not overflow-backed")) + } + } + }) +} + +#[test] +fn overflow_update_reuses_head_and_frees_unused_tail() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("overflow-reuse.db"); + let large = random_value(); + let smaller = large.get(..5_000).ok_or_else(|| io::Error::other("random test value is too short"))?.to_vec(); + let mut tree = BPlusTree::new(); + tree.insert(7u32, large); + tree.store(&path)?; + let before = database_header(&path)?; + let before_len = fs::metadata(&path)?.len(); + let before_head = overflow_head(&path, 7)?; + + let mut updater = BPlusTreeUpdate::>::try_new(&path)?; + updater.update(&7, smaller.clone()).map_err(BPlusTreeError::to_io)?; + + let after = database_header(&path)?; + assert_eq!(after.generation, before.generation + 1); + assert_eq!(fs::metadata(&path)?.len(), before_len); + assert_eq!(overflow_head(&path, 7)?, before_head); + assert_ne!(after.free_page_head, 0); + let mut query = BPlusTreeQuery::>::try_new(&path)?; + assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(smaller)); + let _ = verify_full(&mut query)?; + Ok(()) +} + +#[test] +fn inline_to_overflow_update_allocates_new_pages() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("overflow-allocation.db"); + let mut tree = BPlusTree::new(); + tree.insert(7u32, vec![7]); + tree.store(&path)?; + let before = database_header(&path)?; + let before_len = fs::metadata(&path)?.len(); + let large = random_value(); + + let mut updater = BPlusTreeUpdate::>::try_new(&path)?; + updater.update(&7, large.clone()).map_err(BPlusTreeError::to_io)?; + + let after = database_header(&path)?; + assert_eq!(after.generation, before.generation + 1); + assert!(fs::metadata(&path)?.len() > before_len); + assert_ne!(overflow_head(&path, 7)?, 0); + let mut query = BPlusTreeQuery::>::try_new(&path)?; + assert_eq!(query.query(&7).map_err(BPlusTreeError::to_io)?, Some(large)); + let _ = verify_full(&mut query)?; + Ok(()) +} + +#[test] +fn leaf_split_repairs_siblings_and_creates_a_new_root() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("leaf-split.db"); + let value: String = random_value() + .get(..200) + .ok_or_else(|| io::Error::other("random test value is too short"))? + .iter() + .map(|byte| char::from(33 + byte % 90)) + .collect(); + let mut tree = BPlusTree::new(); + for key in 0..17u32 { + tree.insert(key, value.clone()); + } + tree.store(&path)?; + let before = database_header(&path)?; + let before_len = fs::metadata(&path)?.len(); + + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.upsert(&17, &value)?; + + let after = database_header(&path)?; + assert_eq!(after.generation, before.generation + 1); + assert_ne!(after.root_page_id, before.root_page_id); + assert_eq!(fs::metadata(&path)?.len(), before_len + 2 * u64::try_from(PAGE_SIZE).map_err(io::Error::other)?); + let mut query = BPlusTreeQuery::::try_new(&path)?; + let report = verify_full(&mut query)?; + assert_eq!(report.live_entries, 18); + assert_eq!(query.query(&17).map_err(BPlusTreeError::to_io)?, Some(value)); + Ok(()) +} + +#[test] +fn splitting_a_non_rightmost_leaf_repairs_the_former_neighbor() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("middle-leaf-split.db"); + let value = random_value().get(..200).ok_or_else(|| io::Error::other("random test value is too short"))?.to_vec(); + let mut tree = BPlusTree::new(); + for key in (0..800u32).step_by(2) { + tree.insert(key, value.clone()); + } + tree.store(&path)?; + let before = database_header(&path)?; + let mut query = BPlusTreeQuery::>::try_new(&path)?; + let left_leaf = query.locate_leaf(&0)?; + let former_right = query.with_page(left_leaf, |bytes, _| { + Ok(SlottedPage::open(bytes, left_leaf, before.next_page_id)?.header().right) + })?; + if former_right == 0 { + return Err(io::Error::other("fixture did not create a right leaf neighbor")); + } + drop(query); + + let mut updater = BPlusTreeUpdate::>::try_new(&path)?; + updater.upsert(&1, &value)?; + + let after = database_header(&path)?; + assert_eq!(after.root_page_id, before.root_page_id); + assert!(after.next_page_id > before.next_page_id); + let mut query = BPlusTreeQuery::>::try_new(&path)?; + let new_right = query + .with_page(left_leaf, |bytes, _| Ok(SlottedPage::open(bytes, left_leaf, after.next_page_id)?.header().right))?; + assert_ne!(new_right, former_right); + query.with_page(new_right, |bytes, _| { + let page = SlottedPage::open(bytes, new_right, after.next_page_id)?; + assert_eq!(page.header().left, left_leaf); + assert_eq!(page.header().right, former_right); + Ok(()) + })?; + query.with_page(former_right, |bytes, _| { + let page = SlottedPage::open(bytes, former_right, after.next_page_id)?; + assert_eq!(page.header().left, new_right); + Ok(()) + })?; + let _ = verify_full(&mut query)?; + Ok(()) +} + +pub(in crate::v3::tree::tests) fn long_split_key(index: u32, marker: char) -> String { + format!("{index:04}{marker}{}", "k".repeat(1_880)) +} + +#[test] +fn leaf_promotion_recursively_splits_a_full_internal_root() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("internal-split.db"); + let value = String::from("value"); + let mut tree = BPlusTree::new(); + for index in 0..6u32 { + tree.insert(long_split_key(index, '-'), value.clone()); + } + tree.store(&path)?; + let before = database_header(&path)?; + let before_root = fs::read(&path)?; + let range = page_byte_range(before.root_page_id, before_root.len())?; + let root_page = SlottedPage::open( + before_root.get(range).ok_or_else(|| io::Error::other("root page is missing"))?, + before.root_page_id, + before.next_page_id, + )?; + assert_eq!(root_page.header().page_type, PageType::Internal); + assert_eq!(root_page.header().cell_count, 2); + + let inserted_key = long_split_key(1, 'z'); + let mut updater = BPlusTreeUpdate::::try_new(&path)?; + updater.upsert(&inserted_key, &value)?; + + let after = database_header(&path)?; + assert_eq!(after.generation, before.generation + 1); + assert_ne!(after.root_page_id, before.root_page_id); + let mut query = BPlusTreeQuery::::try_new(&path)?; + let report = verify_full(&mut query)?; + assert_eq!(report.live_entries, 7); + assert_eq!(query.query(&inserted_key).map_err(BPlusTreeError::to_io)?, Some(value)); + Ok(()) +} + +#[test] +fn freed_overflow_pages_are_reused_before_file_growth() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("free-reuse.db"); + let large = random_value(); + let mut tree = BPlusTree::new(); + tree.insert(1u32, large.clone()); + tree.store(&path)?; + let before = database_header(&path)?; + let before_len = fs::metadata(&path)?.len(); + + let mut updater = BPlusTreeUpdate::>::try_new(&path)?; + assert!(updater.delete(&1)?); + assert_ne!(database_header(&path)?.free_page_head, 0); + updater.upsert(&2, &large)?; + + let after = database_header(&path)?; + assert_eq!(after.generation, before.generation + 2); + assert_eq!(fs::metadata(&path)?.len(), before_len); + let mut query = BPlusTreeQuery::>::try_new(&path)?; + assert_eq!(query.query(&1).map_err(BPlusTreeError::to_io)?, None); + assert_eq!(query.query(&2).map_err(BPlusTreeError::to_io)?, Some(large)); + assert_eq!(verify_full(&mut query)?.live_entries, 1); + Ok(()) +} + +/// Point-query benchmark for the persisted read path. +/// +/// `bench_store_throughput` only covers writing. The request hot path is +/// `BPlusTreeQuery::query_zero_copy` against an mmapped database, so the +/// extraction gate needs a read-side number too. Everything here is derived +/// from fixed seeds so two runs on the same host compare like for like: the +/// values come from the same LCG the store benchmark uses, and the lookup +/// order is a fixed-stride walk that defeats sequential prefetching without +/// needing a random number generator. +#[test] +#[ignore = "benchmark; run explicitly with --ignored --nocapture"] +fn bench_point_query_throughput() -> io::Result<()> { + // Coprime with every entry count below, so the walk visits each key exactly + // once and never degenerates into a scan. + const STRIDE: u32 = 7_919; + const LOOKUPS: u32 = 200_000; + + let noise = |seed: u32, len: usize| -> Vec { + let mut state = seed.wrapping_mul(2_654_435_761).wrapping_add(1); + (0..len) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + u8::try_from(state >> 24).unwrap_or(0) + }) + .collect() + }; + + for (label, count, size) in [("klein", 2_000u32, 800usize), ("mittel", 20_000, 2_000), ("gross", 60_000, 2_000)] { + let dir = tempfile::tempdir()?; + let path = dir.path().join("bench.db"); + let mut tree = BPlusTree::>::new(); + for key in 0..count { + tree.insert(key, noise(key, size)); + } + tree.store(&path)?; + + let mut query = BPlusTreeQuery::>::try_new(&path)?; + // Fault the mapping in first; this benchmark measures lookup, not page-in. + let mut checksum = 0u64; + for key in 0..count { + checksum += + query.query_zero_copy(&key).map_err(BPlusTreeError::to_io)?.map_or(0, |value| u64::from(value[0])); + } + + let start = std::time::Instant::now(); + let mut key = 0u32; + for _ in 0..LOOKUPS { + key = (key + STRIDE) % count; + checksum += + query.query_zero_copy(&key).map_err(BPlusTreeError::to_io)?.map_or(0, |value| u64::from(value[0])); + } + let elapsed = start.elapsed(); + + assert!(checksum > 0, "lookups must not be optimized away"); + println!( + "BENCH query {label:6} entries={count:6} bytes={size:5} lookups={LOOKUPS} -> {:>8.1} ms {:>9.0} lookups/s", + elapsed.as_secs_f64() * 1000.0, + f64::from(LOOKUPS) / elapsed.as_secs_f64() + ); + } + Ok(()) +} + +pub(in crate::v3::tree::tests) fn run_exclusive_probe_child(database: &Path, expected: &str) -> io::Result<()> { + let status = Command::new(std::env::current_exe()?) + .arg("--exact") + .arg("v3::tree::tests::exclusive_sidecar_probe_child") + .arg("--nocapture") + .env("TULIPROX_V3_LOCK_PROBE_PATH", database) + .env("TULIPROX_V3_LOCK_PROBE_EXPECTED", expected) + .status()?; + if status.success() { + Ok(()) + } else { + Err(io::Error::other(format!("exclusive sidecar child probe failed with {status}"))) + } +} + +#[test] +fn owned_iterator_keeps_shared_guard_after_query_is_consumed() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("owned-iterator.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("original")); + tree.store(&path)?; + + let query = BPlusTreeQuery::::try_new(&path)?; + let iterator = query.disk_iter(); + assert!(!try_exclusive_sidecar(&path)?); + let (receiver, handle) = spawn_replacement_writer(path.clone())?; + assert_writer_is_blocked(&receiver)?; + drop(iterator); + finish_writer(&receiver, handle)?; + assert!(wait_for_exclusive_sidecar(&path)?); + Ok(()) +} + +#[test] +fn shared_query_blocks_exclusive_writer_in_another_process() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("two-process.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, String::from("original")); + tree.store(&path)?; + + let query = BPlusTreeQuery::::try_new(&path)?; + run_exclusive_probe_child(&path, "blocked")?; + drop(query); + run_exclusive_probe_child(&path, "acquired") +} + +pub(in crate::v3::tree::tests) fn leaf_cell_of_footprint(footprint: usize) -> io::Result> { + let key_length = footprint.checked_sub(4 + 24).ok_or_else(|| io::Error::other("footprint is too small"))?; + if footprint <= 2032 { + let key = vec![b'k'; key_length]; + let mut cell = Vec::new(); + encode_tombstone_leaf_cell(&key, &mut cell)?; + Ok(cell) + } else { + Ok(vec![0; footprint - 4]) + } +} + +#[test] +fn compressed_inline_tombstone_and_overflow_descriptors_round_trip() -> io::Result<()> { + let raw = [0u8; 128]; + let mut compression_scratch = Vec::new(); + let stored = encode_value(&raw, &mut compression_scratch)?; + assert_eq!(stored.compression(), Compression::Lz4); + + let mut cell = Vec::new(); + encode_inline_leaf_cell( + b"compressed", + u32::try_from(raw.len()).map_err(io::Error::other)?, + stored.compression(), + stored.as_slice(), + &mut cell, + )?; + let decoded = LeafCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)?; + let mut value_scratch = Vec::new(); + assert_eq!(read_leaf_value(&[], &decoded.value, NEXT_PAGE_ID, 1024, &mut value_scratch)?, Some(raw.as_slice())); + + encode_tombstone_leaf_cell(b"deleted", &mut cell)?; + assert!(matches!(LeafCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)?.value, LeafValueRef::Tombstone)); + + encode_overflow_leaf_cell( + b"large", + 5000, + Compression::None, + 5000, + 2, + 0x1234_5678, + PAGE_ID, + NEXT_PAGE_ID, + &mut cell, + )?; + assert!(matches!(LeafCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)?.value, LeafValueRef::Overflow { head: 2, .. })); + Ok(()) +} + +#[test] +fn internal_separator_and_locator_have_exact_codecs() -> io::Result<()> { + let mut cell = Vec::new(); + encode_internal_cell(b"separator", 9, PAGE_ID, NEXT_PAGE_ID, &mut cell)?; + let decoded = InternalCellRef::decode(&cell, PAGE_ID, NEXT_PAGE_ID)?; + assert_eq!(decoded.key_bytes, b"separator"); + assert_eq!(decoded.right_child, 9); + + let locator = Locator::for_key(7, 3, b"separator")?; + let encoded = locator.encode(); + assert_eq!(encoded.len(), 16); + assert_eq!(Locator::decode(&encoded)?, locator); + let mut corrupt = encoded; + corrupt[10] = 1; + invalid_data(Locator::decode(&corrupt)) +} + +#[test] +fn locator_rejects_wrong_key_crc_and_wrong_primary_key() -> io::Result<()> { + let key = binary_serialize(&42u32)?; + let mut cell = Vec::new(); + encode_inline_leaf_cell(&key, 1, Compression::None, &[7], &mut cell)?; + let mut bytes = empty_leaf(PAGE_ID, NEXT_PAGE_ID)?; + let mut page = SlottedPage::open(bytes.as_mut_slice(), PAGE_ID, NEXT_PAGE_ID)?; + page.rebuild_ordered([cell.as_slice()])?; + + let locator = Locator::for_key(PAGE_ID, 0, &key)?; + validate_locator(&page, locator, &key)?; + let bad_crc = Locator { serialized_key_crc32: locator.serialized_key_crc32 ^ 1, ..locator }; + invalid_data(validate_locator(&page, bad_crc, &key))?; + invalid_data(validate_locator(&page, locator, &binary_serialize(&43u32)?)) +} + +#[test] +fn adversarial_leaf_splits_reject_old_limit_and_accept_capped_cells() -> io::Result<()> { + let first_witness = [leaf_cell_of_footprint(2026)?, leaf_cell_of_footprint(2040)?, leaf_cell_of_footprint(2038)?]; + invalid_input(choose_leaf_split(&first_witness))?; + + let second_witness = [leaf_cell_of_footprint(1984)?, leaf_cell_of_footprint(2296)?, leaf_cell_of_footprint(1984)?]; + invalid_input(choose_leaf_split(&second_witness))?; + + let capped = [leaf_cell_of_footprint(2026)?, leaf_cell_of_footprint(2032)?, leaf_cell_of_footprint(2032)?]; + let split = choose_leaf_split(&capped)?; + assert_eq!(split, 2); + assert!(used_leaf_bytes(&capped[..split])? <= PAGE_SIZE); + assert!(used_leaf_bytes(&capped[split..])? <= PAGE_SIZE); + Ok(()) +} + +#[test] +fn leaf_split_selection_is_deterministic_for_variable_sizes() -> io::Result<()> { + let mut seed = 0x5eed_u64; + for count in 3..36 { + let mut cells = Vec::with_capacity(count); + for _ in 0..count { + seed = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + let footprint = 100 + usize::try_from(seed % 60).map_err(io::Error::other)?; + let insert_at = + if cells.is_empty() { 0 } else { usize::try_from(seed).map_err(io::Error::other)? % (cells.len() + 1) }; + cells.insert(insert_at, leaf_cell_of_footprint(footprint)?); + } + if used_leaf_bytes(&cells)? > PAGE_SIZE { + let first = choose_leaf_split(&cells)?; + let second = choose_leaf_split(&cells)?; + assert_eq!(first, second); + assert!(used_leaf_bytes(&cells[..first])? <= PAGE_SIZE); + assert!(used_leaf_bytes(&cells[first..])? <= PAGE_SIZE); + } + } + Ok(()) +} + +#[test] +fn internal_split_promotes_exact_separator_and_child() -> io::Result<()> { + let mut cells = Vec::new(); + for (key_length, child) in [(1000, 2), (1000, 3), (1000, 4), (1000, 5)] { + let mut cell = Vec::new(); + encode_internal_cell(&vec![b'k'; key_length], child, PAGE_ID, NEXT_PAGE_ID, &mut cell)?; + cells.push(cell); + } + let split = choose_internal_split(&cells, PAGE_ID, NEXT_PAGE_ID)?; + assert_eq!(split.promoted_index, 1); + assert_eq!(split.promoted.key_bytes.len(), 1000); + assert_eq!(split.right_leftmost_child, 3); + assert_eq!(split.left_cells, &cells[..1]); + assert_eq!(split.right_cells, &cells[2..]); + Ok(()) +} + +#[test] +fn overflow_chain_round_trip_is_bounded_and_checksum_validated() -> io::Result<()> { + let stored = vec![0x5a; 5000]; + let mut database = vec![0; PAGE_SIZE * 4]; + let first = encode_overflow_page(2, NEXT_PAGE_ID, 3, &stored[..4056])?; + let second = encode_overflow_page(3, NEXT_PAGE_ID, 0, &stored[4056..])?; + database[PAGE_SIZE * 2..PAGE_SIZE * 3].copy_from_slice(&first); + database[PAGE_SIZE * 3..PAGE_SIZE * 4].copy_from_slice(&second); + + let value = LeafValueRef::Overflow { + compression: Compression::None, + logical_len: 5000, + stored_len: 5000, + head: 2, + crc32: crc32fast::hash(&stored), + }; + let mut scratch = Vec::new(); + assert_eq!(read_leaf_value(&database, &value, NEXT_PAGE_ID, 6000, &mut scratch)?, Some(stored.as_slice())); + + let free = encode_free_page(3, NEXT_PAGE_ID, 0)?; + database[PAGE_SIZE * 3..PAGE_SIZE * 4].copy_from_slice(&free); + invalid_data(read_leaf_value(&database, &value, NEXT_PAGE_ID, 6000, &mut scratch)) +} + +#[test] +fn compressed_overflow_chain_reuses_scratch_and_round_trips() -> io::Result<()> { + let mut raw = Vec::with_capacity(10_000); + let mut state = 0x51a7_9e2d_4c83_b6f0_u64; + for _ in 0..5000 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + raw.push(state.to_le_bytes()[0]); + } + raw.resize(10_000, 0); + let mut encoded = Vec::new(); + let stored = encode_value(&raw, &mut encoded)?; + if stored.compression() != Compression::Lz4 || stored.as_slice().len() <= OVERFLOW_PAYLOAD_LEN { + return Err(io::Error::other("test value must span compressed overflow pages")); + } + let stored = stored.as_slice().to_vec(); + let mut corrupt = stored.clone(); + corrupt + .get_mut(..4) + .ok_or_else(|| io::Error::other("missing test LZ4 length"))? + .copy_from_slice(&1u32.to_le_bytes()); + invalid_data(decompress_value_in_place( + &mut corrupt, + u32::try_from(raw.len()).map_err(io::Error::other)?, + raw.len(), + ))?; + let page_count = stored.len().div_ceil(OVERFLOW_PAYLOAD_LEN); + assert!(page_count > 1); + let next_page_id = u64::try_from(page_count) + .map_err(io::Error::other)? + .checked_add(2) + .ok_or_else(|| io::Error::other("test page count overflow"))?; + let mut database = vec![0; usize::try_from(next_page_id).map_err(io::Error::other)? * PAGE_SIZE]; + for (index, payload) in stored.chunks(OVERFLOW_PAYLOAD_LEN).enumerate() { + let page_id = u64::try_from(index).map_err(io::Error::other)? + 1; + let next = if index + 1 == page_count { 0 } else { page_id + 1 }; + let page = encode_overflow_page(page_id, next_page_id, next, payload)?; + let start = usize::try_from(page_id).map_err(io::Error::other)? * PAGE_SIZE; + database[start..start + PAGE_SIZE].copy_from_slice(&page); + } + let value = LeafValueRef::Overflow { + compression: Compression::Lz4, + logical_len: u32::try_from(raw.len()).map_err(io::Error::other)?, + stored_len: u32::try_from(stored.len()).map_err(io::Error::other)?, + head: 1, + crc32: crc32fast::hash(&stored), + }; + let mut scratch = Vec::new(); + assert_eq!(read_leaf_value(&database, &value, next_page_id, raw.len(), &mut scratch)?, Some(raw.as_slice())); + Ok(()) +} + +#[test] +fn overflow_chain_rejects_truncation_cycle_oversize_and_bad_crc() -> io::Result<()> { + let stored = vec![0x5a; 5000]; + let value = LeafValueRef::Overflow { + compression: Compression::None, + logical_len: 5000, + stored_len: 5000, + head: 2, + crc32: crc32fast::hash(&stored), + }; + let mut database = vec![0; PAGE_SIZE * 4]; + let first = encode_overflow_page(2, NEXT_PAGE_ID, 3, &stored[..4056])?; + let short = encode_overflow_page(3, NEXT_PAGE_ID, 0, &stored[4056..4999])?; + database[PAGE_SIZE * 2..PAGE_SIZE * 3].copy_from_slice(&first); + database[PAGE_SIZE * 3..PAGE_SIZE * 4].copy_from_slice(&short); + let mut scratch = Vec::new(); + invalid_data(read_leaf_value(&database, &value, NEXT_PAGE_ID, 6000, &mut scratch))?; + + let cycle = encode_overflow_page(3, NEXT_PAGE_ID, 2, &stored[4056..])?; + database[PAGE_SIZE * 3..PAGE_SIZE * 4].copy_from_slice(&cycle); + invalid_data(read_leaf_value(&database, &value, NEXT_PAGE_ID, 6000, &mut scratch))?; + invalid_data(read_leaf_value(&database, &value, NEXT_PAGE_ID, 4999, &mut scratch))?; + + let last = encode_overflow_page(3, NEXT_PAGE_ID, 0, &stored[4056..])?; + database[PAGE_SIZE * 3..PAGE_SIZE * 4].copy_from_slice(&last); + let LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } = value else { + return Err(io::Error::other("expected overflow value")); + }; + let bad_crc = LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32: crc32 ^ 1 }; + invalid_data(read_leaf_value(&database, &bad_crc, NEXT_PAGE_ID, 6000, &mut scratch))?; + + database[PAGE_SIZE * 2 + 40] ^= 1; + invalid_data(read_leaf_value(&database, &value, NEXT_PAGE_ID, 6000, &mut scratch))?; + + let empty = encode_overflow_page(2, NEXT_PAGE_ID, 3, &[])?; + let one_byte = encode_overflow_page(3, NEXT_PAGE_ID, 0, b"x")?; + database[PAGE_SIZE * 2..PAGE_SIZE * 3].copy_from_slice(&empty); + database[PAGE_SIZE * 3..PAGE_SIZE * 4].copy_from_slice(&one_byte); + let empty_chain_value = LeafValueRef::Overflow { + compression: Compression::None, + logical_len: 1, + stored_len: 1, + head: 2, + crc32: crc32fast::hash(b"x"), + }; + invalid_data(read_leaf_value(&database, &empty_chain_value, NEXT_PAGE_ID, 6000, &mut scratch)) +} + +pub(in crate::v3::tree::tests) fn page_range(page_id: u64) -> io::Result> { + let start = usize::try_from(page_id) + .map_err(io::Error::other)? + .checked_mul(PAGE_SIZE) + .ok_or_else(|| io::Error::other("test page offset overflow"))?; + Ok(start..start + PAGE_SIZE) +} + +pub(in crate::v3::tree::tests) fn rewrite_page_checksum(database: &mut [u8], page_id: u64) -> io::Result<()> { + let range = page_range(page_id)?; + crate::v3::format::write_page_checksum( + database.get_mut(range).ok_or_else(|| io::Error::other("test page missing"))?, + ) +} + +pub(in crate::v3::tree::tests) fn set_page_reference( + database: &mut [u8], + page_id: u64, + offset: usize, + reference: u64, +) -> io::Result<()> { + let page = page_range(page_id)?; + let start = page.start + offset; + database + .get_mut(start..start + 8) + .ok_or_else(|| io::Error::other("test reference missing"))? + .copy_from_slice(&reference.to_le_bytes()); + rewrite_page_checksum(database, page_id) +} + +pub(in crate::v3::tree::tests) fn verify_rejects(database: &[u8]) -> io::Result<()> +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de> + Clone, +{ + let dir = tempfile::tempdir()?; + let path = dir.path().join("corrupt.db"); + fs::write(&path, database)?; + let mut query = BPlusTreeQuery::::try_new(&path)?; + invalid_data(verify_full(&mut query)) +} + +pub(in crate::v3::tree::tests) fn stored_tree_fixture() -> io::Result<(Vec, u64, u64, u64, u64)> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("tree.db"); + let mut tree = BPlusTree::new(); + for key in 1_000..1_700u32 { + tree.insert(key, "same-value".to_string()); + } + tree.store(&path)?; + let mut query = BPlusTreeQuery::::try_new(&path)?; + let root = query.header.root_page_id; + let first = query.leftmost_leaf()?; + let next_page_id = query.header.next_page_id; + let second = query + .with_page(first, |bytes, _| SlottedPage::open(bytes, first, next_page_id).map(|page| page.header().right))?; + if second == 0 { + return Err(io::Error::other("test tree did not split")); + } + let mut last = second; + loop { + let right = query + .with_page(last, |bytes, _| SlottedPage::open(bytes, last, next_page_id).map(|page| page.header().right))?; + if right == 0 { + break; + } + last = right; + } + drop(query); + Ok((fs::read(path)?, root, first, second, last)) +} + +#[test] +fn iterator_yields_second_leaf_corruption_once_then_fuses() -> io::Result<()> { + let (mut database, _, _, second, _) = stored_tree_fixture()?; + let byte = page_range(second)?.start + 100; + *database.get_mut(byte).ok_or_else(|| io::Error::other("test corruption byte missing"))? ^= 1; + let dir = tempfile::tempdir()?; + let path = dir.path().join("iterator-corrupt.db"); + fs::write(&path, database)?; + let mut query = BPlusTreeQuery::::try_new(&path)?; + let mut iterator = query.iter(); + let mut yielded = 0usize; + let mut saw_error = false; + for item in iterator.by_ref() { + match item { + Ok(_) => yielded += 1, + Err(err) => { + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + saw_error = true; + break; + } + } + } + if !saw_error { + return Err(io::Error::other("corruption was hidden as end-of-stream")); + } + assert!(iterator.next().is_none()); + assert!(yielded > 0); + Ok(()) +} + +#[test] +fn iterator_skips_corrupt_value_and_continues_with_next_cell() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("iterator-corrupt-value.db"); + let mut tree = BPlusTree::new(); + for key in 1..=3u32 { + tree.insert(key, format!("value-{key}")); + } + tree.store(&path)?; + + let mut query = BPlusTreeQuery::::try_new(&path)?; + let (page_id, range) = query.locate_cell(&2)?.ok_or_else(|| io::Error::other("test key missing"))?; + let next_page_id = query.header.next_page_id; + let stored_offset = query.with_page(page_id, |bytes, _| { + let cell = LeafCellRef::decode( + bytes.get(range).ok_or_else(|| io::Error::other("test cell range missing"))?, + page_id, + next_page_id, + )?; + let LeafValueRef::Inline { stored, .. } = cell.value else { + return Err(io::Error::other("test value is not inline")); + }; + Ok(stored.as_ptr() as usize - bytes.as_ptr() as usize) + })?; + drop(query); + + let mut database = fs::read(&path)?; + let absolute = page_range(page_id)?.start + stored_offset; + *database.get_mut(absolute).ok_or_else(|| io::Error::other("test value byte missing"))? = 0xc1; + rewrite_page_checksum(&mut database, page_id)?; + fs::write(&path, database)?; + + let mut query = BPlusTreeQuery::::try_new(&path)?; + let mut iterator = query.iter(); + assert_eq!(iterator.next().transpose()?, Some((1, String::from("value-1")))); + assert!(iterator.next().is_some_and(|entry| entry.is_err())); + assert_eq!(iterator.next().transpose()?, Some((3, String::from("value-3")))); + assert!(iterator.next().is_none()); + Ok(()) +} + +#[test] +fn iterator_rejects_leaf_cycle_before_yielding_duplicate_entries() -> io::Result<()> { + let (mut database, _, first, second, _) = stored_tree_fixture()?; + set_page_reference(&mut database, second, 16, first)?; + set_page_reference(&mut database, first, 8, second)?; + let dir = tempfile::tempdir()?; + let path = dir.path().join("iterator-cycle.db"); + fs::write(&path, database)?; + + let mut query = BPlusTreeQuery::::try_new(&path)?; + let mut iterator = query.iter(); + let mut seen = HashSet::new(); + let mut errors = 0usize; + for item in iterator.by_ref() { + match item { + Ok((key, _)) => assert!(seen.insert(key), "iterator yielded key {key} twice"), + Err(err) => { + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + errors += 1; + } + } + } + assert_eq!(errors, 1); + assert!(iterator.next().is_none()); + Ok(()) +} + +#[test] +fn full_verifier_rejects_tree_sibling_free_and_orphan_corruption() -> io::Result<()> { + let (database, root, first, second, last) = stored_tree_fixture()?; + + let mut child_cycle = database.clone(); + set_page_reference(&mut child_cycle, root, 32, root)?; + verify_rejects::(&child_cycle)?; + + let mut duplicate_child = database.clone(); + let root_start = page_range(root)?.start; + let leftmost = + u64::from_le_bytes(duplicate_child[root_start + 32..root_start + 40].try_into().map_err(io::Error::other)?); + let first_cell = + u16::from_le_bytes(duplicate_child[root_start + 40..root_start + 42].try_into().map_err(io::Error::other)?); + let child_offset = root_start + usize::from(first_cell) + 4; + duplicate_child[child_offset..child_offset + 8].copy_from_slice(&leftmost.to_le_bytes()); + rewrite_page_checksum(&mut duplicate_child, root)?; + verify_rejects::(&duplicate_child)?; + + let mut wrong_type = database.clone(); + let header = DatabaseHeader::decode(&wrong_type[..PAGE_SIZE])?; + let overflow = encode_overflow_page(leftmost, header.next_page_id, 0, b"wrong type")?; + let range = page_range(leftmost)?; + wrong_type[range].copy_from_slice(&overflow); + verify_rejects::(&wrong_type)?; + + let mut asymmetric = database.clone(); + set_page_reference(&mut asymmetric, second, 8, 0)?; + verify_rejects::(&asymmetric)?; + + let mut sibling_cycle = database.clone(); + set_page_reference(&mut sibling_cycle, last, 16, first)?; + set_page_reference(&mut sibling_cycle, first, 8, last)?; + verify_rejects::(&sibling_cycle)?; + + let mut inverted = database.clone(); + let first_start = page_range(first)?.start; + let second_start = page_range(second)?.start; + let first_count = + u16::from_le_bytes(inverted[first_start + 2..first_start + 4].try_into().map_err(io::Error::other)?); + let last_slot = first_start + PAGE_HEADER_LEN + (usize::from(first_count) - 1) * SLOT_LEN; + let left_cell = + usize::from(u16::from_le_bytes(inverted[last_slot..last_slot + 2].try_into().map_err(io::Error::other)?)); + let right_cell = usize::from(u16::from_le_bytes( + inverted[second_start + PAGE_HEADER_LEN..second_start + PAGE_HEADER_LEN + 2] + .try_into() + .map_err(io::Error::other)?, + )); + for index in 0..3 { + inverted.swap(first_start + left_cell + 24 + index, second_start + right_cell + 24 + index); + } + rewrite_page_checksum(&mut inverted, first)?; + rewrite_page_checksum(&mut inverted, second)?; + verify_rejects::(&inverted)?; + + let mut live_and_free = database.clone(); + let mut header = DatabaseHeader::decode(&live_and_free[..PAGE_SIZE])?; + header.free_page_head = first; + live_and_free[..PAGE_SIZE].copy_from_slice(&header.encode()?); + verify_rejects::(&live_and_free)?; + + let mut orphan = database.clone(); + let mut header = DatabaseHeader::decode(&orphan[..PAGE_SIZE])?; + let orphan_id = header.next_page_id; + header.next_page_id += 1; + orphan[..PAGE_SIZE].copy_from_slice(&header.encode()?); + orphan.extend_from_slice(&encode_free_page(orphan_id, header.next_page_id, 0)?); + verify_rejects::(&orphan)?; + + let mut duplicate_free = database; + let mut header = DatabaseHeader::decode(&duplicate_free[..PAGE_SIZE])?; + let first_free = header.next_page_id; + let second_free = first_free + 1; + header.next_page_id += 2; + header.free_page_head = first_free; + duplicate_free[..PAGE_SIZE].copy_from_slice(&header.encode()?); + duplicate_free.extend_from_slice(&encode_free_page(first_free, header.next_page_id, second_free)?); + duplicate_free.extend_from_slice(&encode_free_page(second_free, header.next_page_id, first_free)?); + verify_rejects::(&duplicate_free) +} + +pub(in crate::v3::tree::tests) fn overflow_heads( + query: &mut BPlusTreeQuery>, +) -> io::Result<(u64, u64, u64)> { + let mut result = Vec::new(); + for key in [1, 2] { + let leaf = query.locate_leaf(&key)?; + let next_page_id = query.header.next_page_id; + let (head, cell_offset) = query.with_page(leaf, |bytes, _| { + let page = SlottedPage::open(bytes, leaf, next_page_id)?; + let index = search_leaf(&page, &key)?.map_err(|_| io::Error::other("test key missing"))?; + let cell = LeafCellRef::decode(page.cell(index)?, leaf, next_page_id)?; + let LeafValueRef::Overflow { head, .. } = cell.value else { + return Err(io::Error::other("test value is not overflow-backed")); + }; + let slot = PAGE_HEADER_LEN + index * SLOT_LEN; + let offset = u16::from_le_bytes(bytes[slot..slot + 2].try_into().map_err(io::Error::other)?); + Ok((head, u64::from(offset))) + })?; + result.push((leaf, head, cell_offset)); + } + let first = result.first().ok_or_else(|| io::Error::other("first test overflow missing"))?; + let second = result.get(1).ok_or_else(|| io::Error::other("second test overflow missing"))?; + Ok((first.1, second.0, second.2)) +} + +#[test] +fn full_verifier_rejects_overflow_cycle_and_shared_chain() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("overflow.db"); + let value = random_value(); + let mut tree = BPlusTree::new(); + tree.insert(1u32, value.clone()); + tree.insert(2u32, value); + tree.store(&path)?; + let mut query = BPlusTreeQuery::>::try_new(&path)?; + let (first_head, second_leaf, second_cell_offset) = overflow_heads(&mut query)?; + let next_page_id = query.header.next_page_id; + drop(query); + let database = fs::read(&path)?; + + let mut cycle = database.clone(); + let mut last = first_head; + loop { + let start = page_range(last)?.start; + let next = u64::from_le_bytes(cycle[start + 16..start + 24].try_into().map_err(io::Error::other)?); + if next == 0 { + break; + } + last = next; + } + set_page_reference(&mut cycle, last, 16, first_head)?; + verify_rejects::>(&cycle)?; + + let mut shared = database; + let start = page_range(second_leaf)?.start + usize::try_from(second_cell_offset).map_err(io::Error::other)? + 12; + shared[start..start + 8].copy_from_slice(&first_head.to_le_bytes()); + rewrite_page_checksum(&mut shared, second_leaf)?; + let _ = next_page_id; + verify_rejects::>(&shared) +} + +#[test] +fn full_verifier_rejects_oversized_overflow_descriptor_before_reserve() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("oversized-overflow.db"); + let value = random_value(); + let mut tree = BPlusTree::new(); + tree.insert(1u32, value.clone()); + tree.insert(2u32, value); + tree.store(&path)?; + let mut query = BPlusTreeQuery::>::try_new(&path)?; + let (_, second_leaf, second_cell_offset) = overflow_heads(&mut query)?; + drop(query); + + let mut database = fs::read(&path)?; + let descriptor = page_range(second_leaf)?.start + usize::try_from(second_cell_offset).map_err(io::Error::other)?; + database[descriptor + 4..descriptor + 8].copy_from_slice(&u32::MAX.to_le_bytes()); + database[descriptor + 8..descriptor + 12].copy_from_slice(&u32::MAX.to_le_bytes()); + rewrite_page_checksum(&mut database, second_leaf)?; + fs::write(&path, database)?; + + let mut query = BPlusTreeQuery::>::try_new(&path)?; + let error = verify_full(&mut query).err().ok_or_else(|| io::Error::other("oversized value accepted"))?; + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("overflow value exceeds allocation limit")); + Ok(()) +} + +#[test] +fn full_verifier_rejects_uncompressed_overflow_length_mismatch() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("overflow-length.db"); + let mut tree = BPlusTree::new(); + tree.insert(1u32, random_value()); + tree.store(&path)?; + let mut query = BPlusTreeQuery::>::try_new(&path)?; + let leaf = query.locate_leaf(&1)?; + let next_page_id = query.header.next_page_id; + let cell_offset = query.with_page(leaf, |bytes, _| { + let page = SlottedPage::open(bytes, leaf, next_page_id)?; + let index = search_leaf(&page, &1)?.map_err(|_| io::Error::other("test key missing"))?; + let slot = PAGE_HEADER_LEN + .checked_add(index.checked_mul(SLOT_LEN).ok_or_else(|| io::Error::other("test slot overflow"))?) + .ok_or_else(|| io::Error::other("test slot overflow"))?; + Ok(u16::from_le_bytes(bytes[slot..slot + 2].try_into().map_err(io::Error::other)?)) + })?; + drop(query); + + let mut database = fs::read(&path)?; + let descriptor = page_range(leaf)?.start + usize::from(cell_offset); + database[descriptor + 4..descriptor + 8].copy_from_slice(&1u32.to_le_bytes()); + rewrite_page_checksum(&mut database, leaf)?; + fs::write(&path, database)?; + let mut query = BPlusTreeQuery::>::try_new(&path)?; + invalid_data(verify_full(&mut query)) +} diff --git a/backend/btree/src/v3/tree/tests/support.rs b/backend/btree/src/v3/tree/tests/support.rs new file mode 100644 index 000000000..10a6b7c3a --- /dev/null +++ b/backend/btree/src/v3/tree/tests/support.rs @@ -0,0 +1,169 @@ +use super::{BPlusTree, DatabaseHeader}; +use crate::v3::format::{PageHeader, PageType, PAGE_HEADER_LEN, PAGE_SIZE}; +use serde::{Deserialize, Serialize}; +use std::{ + fs, + fs::OpenOptions, + io, + path::{Path, PathBuf}, + sync::{ + mpsc, + mpsc::{Receiver, RecvTimeoutError}, + }, + thread, + thread::JoinHandle, + time::Duration, +}; + +pub(in crate::v3::tree::tests) const PAGE_ID: u64 = 7; + +pub(in crate::v3::tree::tests) const NEXT_PAGE_ID: u64 = 20; + +pub(in crate::v3::tree::tests) fn invalid_data(result: io::Result) -> io::Result<()> { + match result { + Err(err) if err.kind() == io::ErrorKind::InvalidData => Ok(()), + Err(err) => Err(io::Error::other(format!("expected InvalidData, got {err}"))), + Ok(_) => Err(io::Error::other("expected InvalidData")), + } +} + +pub(in crate::v3::tree::tests) fn invalid_input(result: io::Result) -> io::Result<()> { + match result { + Err(err) if err.kind() == io::ErrorKind::InvalidInput => Ok(()), + Err(err) => Err(io::Error::other(format!("expected InvalidInput, got {err}"))), + Ok(_) => Err(io::Error::other("expected InvalidInput")), + } +} + +pub(in crate::v3::tree::tests) fn database_header(path: &Path) -> io::Result { + let bytes = fs::read(path)?; + DatabaseHeader::decode( + bytes + .get(..PAGE_SIZE) + .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "database header is truncated"))?, + ) +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(in crate::v3::tree::tests) struct ConditionalSerialize { + pub(in crate::v3::tree::tests) value: String, + pub(in crate::v3::tree::tests) fail: bool, +} + +impl Serialize for ConditionalSerialize { + fn serialize(&self, serializer: S) -> Result { + if self.fail { + return Err(serde::ser::Error::custom("injected serialization failure")); + } + self.value.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for ConditionalSerialize { + fn deserialize>(deserializer: D) -> Result { + String::deserialize(deserializer).map(|value| Self { value, fail: false }) + } +} + +pub(in crate::v3::tree::tests) fn spawn_replacement_writer( + path: PathBuf, +) -> io::Result<(Receiver>, JoinHandle<()>)> { + let (started_sender, started_receiver) = mpsc::channel(); + let (result_sender, result_receiver) = mpsc::channel(); + let handle = thread::spawn(move || { + let mut replacement = BPlusTree::new(); + replacement.insert(2u32, String::from("replacement")); + let _ = started_sender.send(()); + let _ = result_sender.send(replacement.store(&path)); + }); + started_receiver + .recv_timeout(Duration::from_secs(5)) + .map_err(|error| io::Error::other(format!("replacement writer did not start: {error}")))?; + Ok((result_receiver, handle)) +} + +pub(in crate::v3::tree::tests) fn assert_writer_is_blocked(receiver: &Receiver>) -> io::Result<()> { + match receiver.recv_timeout(Duration::from_millis(100)) { + Err(RecvTimeoutError::Timeout) => Ok(()), + Err(RecvTimeoutError::Disconnected) => Err(io::Error::other("replacement writer disconnected")), + Ok(result) => { + let root = result?; + Err(io::Error::other(format!("replacement writer completed early with root page {root}"))) + } + } +} + +pub(in crate::v3::tree::tests) fn finish_writer( + receiver: &Receiver>, + handle: JoinHandle<()>, +) -> io::Result<()> { + receiver + .recv_timeout(Duration::from_secs(5)) + .map_err(|error| io::Error::other(format!("replacement writer stayed blocked: {error}")))??; + handle.join().map_err(|_| io::Error::other("replacement writer panicked")) +} + +/// Waits for the sidecar lock to become free. +/// +/// A writer reporting its result does not mean the lock is already released: +/// publication can finish on the engine's background thread, so the release +/// happens on a different thread from the one the test observes. Asserting +/// `try_exclusive_sidecar` immediately races with that hand-off and fails +/// intermittently. Waiting for the lock to become available tests what the +/// engine actually promises - that it is released - without asserting an +/// instantaneous guarantee it never made. +pub(in crate::v3::tree::tests) fn wait_for_exclusive_sidecar(database: &Path) -> io::Result { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + if try_exclusive_sidecar(database)? { + return Ok(true); + } + if std::time::Instant::now() >= deadline { + return Ok(false); + } + thread::sleep(Duration::from_millis(10)); + } +} + +pub(in crate::v3::tree::tests) fn try_exclusive_sidecar(database: &Path) -> io::Result { + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(crate::common::sidecar_lock_path(database))?; + match file.try_lock() { + Ok(()) => { + file.unlock()?; + Ok(true) + } + Err(fs::TryLockError::WouldBlock) => Ok(false), + Err(fs::TryLockError::Error(error)) => Err(error), + } +} + +pub(in crate::v3::tree::tests) fn empty_leaf(page_id: u64, next_page_id: u64) -> io::Result<[u8; PAGE_SIZE]> { + let mut page = [0; PAGE_SIZE]; + PageHeader { + page_type: PageType::Leaf, + cell_count: 0, + free_start: u16::try_from(PAGE_HEADER_LEN).map_err(io::Error::other)?, + free_end: u16::try_from(PAGE_SIZE).map_err(io::Error::other)?, + left: 0, + right: 0, + } + .encode_into(&mut page, page_id, next_page_id)?; + Ok(page) +} + +pub(in crate::v3::tree::tests) fn random_value() -> Vec { + let mut state = 0x1234_5678_9abc_def0u64; + (0..12_000) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state.to_le_bytes()[0] + }) + .collect() +} diff --git a/backend/btree/src/v3/tree/transaction.rs b/backend/btree/src/v3/tree/transaction.rs new file mode 100644 index 000000000..fd92e4b74 --- /dev/null +++ b/backend/btree/src/v3/tree/transaction.rs @@ -0,0 +1,286 @@ +use super::{ + decompress_value_in_place, decompress_value_into, encode_free_page, invalid_data, invalid_input, + locate_transaction_leaf, overflow_chain_pages, overflow_payload, search_leaf, Compression, DatabaseHeader, + LeafCellRef, LeafValueRef, PageType, SlottedPage, PAGE_SIZE, +}; +use crate::{ + codec::binary_deserialize, + common::{mmap_with_advice, Advice}, +}; +use memmap2::Mmap; +use serde::Deserialize; +use std::{ + collections::{BTreeMap, HashSet}, + fs::File, + io, + io::Read, + path::Path, +}; + +pub(super) struct DatabaseImage { + pub(super) mmap: Option, + pub(super) fallback: Vec, +} + +impl DatabaseImage { + pub(super) fn open(path: &Path) -> io::Result<(Self, DatabaseHeader)> { + let mut file = File::open(path)?; + let file_len = + usize::try_from(file.metadata()?.len()).map_err(|_| invalid_data("database length exceeds usize"))?; + let mmap = mmap_with_advice(&file, Advice::Normal, "v3 B+Tree update"); + let mut fallback = Vec::new(); + if mmap.is_none() { + fallback.try_reserve_exact(file_len).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + file.read_to_end(&mut fallback)?; + } + let image = Self { mmap, fallback }; + let bytes = image.as_slice(); + let header = DatabaseHeader::decode( + bytes.get(..PAGE_SIZE).ok_or_else(|| invalid_data("database header is truncated"))?, + )?; + let expected = usize::try_from(header.next_page_id) + .map_err(|_| invalid_data("next page id exceeds usize"))? + .checked_mul(PAGE_SIZE) + .ok_or_else(|| invalid_data("database length overflow"))?; + if bytes.len() != expected { + return Err(invalid_data("database file length does not match header")); + } + Ok((image, header)) + } + + pub(super) fn as_slice(&self) -> &[u8] { self.mmap.as_deref().unwrap_or(&self.fallback) } +} + +pub(super) struct WriteTransaction { + pub(super) original_header: DatabaseHeader, + pub(super) next_header: DatabaseHeader, + pub(super) original_file_len: u64, + pub(super) dirty_pages: BTreeMap>, + pub(super) allocated_pages: HashSet, + pub(super) freed_pages: HashSet, +} + +impl WriteTransaction { + pub(super) fn new(header: DatabaseHeader, original_file_len: usize) -> io::Result { + Ok(Self { + original_header: header.clone(), + next_header: header, + original_file_len: u64::try_from(original_file_len) + .map_err(|_| invalid_data("database length exceeds u64"))?, + dirty_pages: BTreeMap::new(), + allocated_pages: HashSet::new(), + freed_pages: HashSet::new(), + }) + } + + pub(super) fn page<'a>(&'a self, base: &'a [u8], page_id: u64) -> io::Result<&'a [u8]> { + if page_id == 0 || page_id >= self.next_header.next_page_id { + return Err(invalid_data("transaction page id is outside database")); + } + if let Some(page) = self.dirty_pages.get(&page_id) { + return Ok(page.as_slice()); + } + if page_id >= self.original_header.next_page_id { + return Err(invalid_data("transaction appended page is missing")); + } + let range = page_byte_range(page_id, base.len())?; + base.get(range).ok_or_else(|| invalid_data("transaction base page is truncated")) + } + + pub(super) fn page_copy(&self, base: &[u8], page_id: u64) -> io::Result<[u8; PAGE_SIZE]> { + let mut copied = [0; PAGE_SIZE]; + copied.copy_from_slice(self.page(base, page_id)?); + Ok(copied) + } + + pub(super) fn page_mut<'a>(&'a mut self, base: &[u8], page_id: u64) -> io::Result<&'a mut [u8; PAGE_SIZE]> { + if page_id == 0 || page_id >= self.next_header.next_page_id { + return Err(invalid_data("transaction page id is outside database")); + } + if self.dirty_pages.contains_key(&page_id) { + return self + .dirty_pages + .get_mut(&page_id) + .map(Box::as_mut) + .ok_or_else(|| invalid_data("transaction page copy is missing")); + } + if page_id >= self.original_header.next_page_id { + return Err(invalid_data("appended transaction page is missing")); + } + let range = page_byte_range(page_id, base.len())?; + let source = base.get(range).ok_or_else(|| invalid_data("transaction base page is truncated"))?; + let mut copied = Box::new([0; PAGE_SIZE]); + copied.copy_from_slice(source); + let _ = self.dirty_pages.insert(page_id, copied); + self.dirty_pages + .get_mut(&page_id) + .map(Box::as_mut) + .ok_or_else(|| invalid_data("transaction page copy is missing")) + } + + #[allow(clippy::large_types_passed_by_value)] + pub(super) fn write_page(&mut self, page_id: u64, page: [u8; PAGE_SIZE]) -> io::Result<()> { + if page_id == 0 || page_id >= self.next_header.next_page_id { + return Err(invalid_input("written page id is outside transaction bounds")); + } + let _ = self.dirty_pages.insert(page_id, Box::new(page)); + Ok(()) + } + + pub(super) fn allocate_page(&mut self, base: &[u8]) -> io::Result { + if self.next_header.free_page_head != 0 { + let page_id = self.next_header.free_page_head; + let page = SlottedPage::open(self.page(base, page_id)?, page_id, self.next_header.next_page_id)?; + if page.header().page_type != PageType::Free { + return Err(invalid_data("free list references a non-free page")); + } + self.next_header.free_page_head = page.header().right; + if !self.allocated_pages.insert(page_id) { + return Err(invalid_data("free page was allocated twice")); + } + let _ = self.freed_pages.remove(&page_id); + return Ok(page_id); + } + let page_id = self.next_header.next_page_id; + self.next_header.next_page_id = page_id.checked_add(1).ok_or_else(|| invalid_input("next page id overflow"))?; + if !self.allocated_pages.insert(page_id) { + return Err(invalid_data("appended page was allocated twice")); + } + Ok(page_id) + } + + pub(super) fn free_page(&mut self, page_id: u64) -> io::Result<()> { + if page_id == 0 || page_id == self.next_header.root_page_id || page_id >= self.next_header.next_page_id { + return Err(invalid_input("page cannot be added to the free list")); + } + if !self.freed_pages.insert(page_id) { + return Err(invalid_data("page was freed twice in one transaction")); + } + let _ = self.allocated_pages.remove(&page_id); + let page = encode_free_page(page_id, self.next_header.next_page_id, self.next_header.free_page_head)?; + self.next_header.free_page_head = page_id; + self.write_page(page_id, page) + } + + fn has_changes(&self) -> bool { + !self.dirty_pages.is_empty() || self.next_header.metadata != self.original_header.metadata + } + + pub(super) fn prepared_pages(&mut self) -> io::Result> { + if !self.has_changes() { + return Ok(Vec::new()); + } + self.next_header.generation = self + .original_header + .generation + .checked_add(1) + .ok_or_else(|| invalid_input("database generation overflow"))?; + let _ = self.dirty_pages.insert(0, Box::new(self.next_header.encode()?)); + let mut prepared = Vec::new(); + prepared + .try_reserve_exact(self.dirty_pages.len()) + .map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + for (page_id, page) in &self.dirty_pages { + if *page_id == 0 { + DatabaseHeader::decode(page.as_slice())?; + } else { + SlottedPage::open(page.as_slice(), *page_id, self.next_header.next_page_id)?; + } + prepared.push((*page_id, page.as_ref())); + } + Ok(prepared) + } +} + +pub(super) fn page_byte_range(page_id: u64, database_len: usize) -> io::Result> { + let start = usize::try_from(page_id) + .map_err(|_| invalid_data("page id exceeds usize"))? + .checked_mul(PAGE_SIZE) + .ok_or_else(|| invalid_data("page offset overflow"))?; + let end = start.checked_add(PAGE_SIZE).ok_or_else(|| invalid_data("page end overflow"))?; + if end > database_len { + return Err(invalid_data("page is outside database")); + } + Ok(start..end) +} + +#[derive(Default)] +pub(super) struct WriteScratch { + pub(super) key: Vec, + pub(super) value: Vec, + pub(super) compression: Vec, + pub(super) cell: Vec, + pub(super) read_value: Vec, +} + +pub(super) fn query_transaction( + transaction: &WriteTransaction, + base: &[u8], + key: &K, + scratch: &mut Vec, +) -> io::Result> +where + K: Ord + for<'de> Deserialize<'de>, + V: for<'de> Deserialize<'de>, +{ + let (leaf_id, _) = locate_transaction_leaf(transaction, base, key)?; + let page = SlottedPage::open(transaction.page(base, leaf_id)?, leaf_id, transaction.next_header.next_page_id)?; + let Ok(index) = search_leaf(&page, key)? else { return Ok(None) }; + let cell = LeafCellRef::decode(page.cell(index)?, leaf_id, transaction.next_header.next_page_id)?; + let bytes = match cell.value { + LeafValueRef::Tombstone => return Ok(None), + LeafValueRef::Inline { compression: Compression::None, stored, .. } => stored, + LeafValueRef::Inline { compression: Compression::Lz4, logical_len, stored, .. } => { + decompress_value_into(stored, logical_len, transaction_value_limit(transaction)?, scratch)? + } + LeafValueRef::Overflow { compression, logical_len, stored_len, head, crc32 } => { + read_transaction_overflow(transaction, base, compression, logical_len, stored_len, head, crc32, scratch)? + } + }; + binary_deserialize(bytes).map(Some) +} + +fn transaction_value_limit(transaction: &WriteTransaction) -> io::Result { + usize::try_from(transaction.next_header.next_page_id) + .map_err(|_| invalid_data("next page id exceeds usize"))? + .checked_mul(PAGE_SIZE) + .and_then(|size| size.checked_mul(256)) + .map(|size| size.min(usize::try_from(u32::MAX).unwrap_or(usize::MAX))) + .ok_or_else(|| invalid_data("value allocation limit overflow")) +} + +#[allow(clippy::too_many_arguments)] +fn read_transaction_overflow<'a>( + transaction: &WriteTransaction, + base: &[u8], + compression: Compression, + logical_len: u32, + stored_len: u32, + head: u64, + crc32: u32, + scratch: &'a mut Vec, +) -> io::Result<&'a [u8]> { + let stored_len = usize::try_from(stored_len).map_err(|_| invalid_data("stored length exceeds usize"))?; + if stored_len > transaction_value_limit(transaction)? { + return Err(invalid_data("overflow value exceeds allocation limit")); + } + scratch.clear(); + scratch.try_reserve(stored_len).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + for page_id in overflow_chain_pages(transaction, base, head)? { + let page = SlottedPage::open(transaction.page(base, page_id)?, page_id, transaction.next_header.next_page_id)?; + let payload = overflow_payload(&page)?; + if scratch.len().saturating_add(payload.len()) > stored_len { + return Err(invalid_data("overflow chain exceeds declared length")); + } + scratch.extend_from_slice(payload); + } + if scratch.len() != stored_len || crc32fast::hash(scratch) != crc32 { + return Err(invalid_data("overflow value checksum or length mismatch")); + } + if compression == Compression::Lz4 { + decompress_value_in_place(scratch, logical_len, transaction_value_limit(transaction)?)?; + } else if scratch.len() != usize::try_from(logical_len).map_err(|_| invalid_data("logical length exceeds usize"))? { + return Err(invalid_data("uncompressed overflow length mismatch")); + } + Ok(scratch) +} diff --git a/backend/btree/src/v3/tree/update.rs b/backend/btree/src/v3/tree/update.rs new file mode 100644 index 000000000..74382de57 --- /dev/null +++ b/backend/btree/src/v3/tree/update.rs @@ -0,0 +1,396 @@ +use super::{ + commit_ordered_page_refs_under_existing_lock, encode_value, invalid_data, invalid_input, query_transaction, + recover_pending_under_existing_lock, stage_delete, stage_upsert, validate_transaction_links, verify_full, + with_exclusive_sidecar, BPlusTree, BPlusTreeMetadata, BPlusTreeQuery, BPlusTreeUpdate, DatabaseHeader, + DatabaseImage, ExclusiveSidecarGuard, FlushPolicy, WalOperationError, WalOutcome, WriteScratch, WriteTransaction, + PAGE_SIZE, +}; +use crate::{ + codec::{binary_serialize, binary_serialize_into}, + common::{read_exact_at_offset, BPlusTreeError}, +}; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, fs::File, io, marker::PhantomData, path::Path}; + +pub(super) struct ActiveBatch { + pub(super) base: DatabaseImage, + pub(super) transaction: WriteTransaction, + pub(super) _guard: ExclusiveSidecarGuard, +} + +impl BPlusTreeUpdate +where + K: Ord + Serialize + for<'de> Deserialize<'de> + Clone, + V: Serialize + for<'de> Deserialize<'de>, +{ + pub fn try_new(filepath: &Path) -> io::Result { + let mut query = BPlusTreeQuery::::try_new(filepath)?; + let _ = verify_full(&mut query)?; + let database_id = query.header.database_id; + let verified_generation = query.header.generation; + let verified_next_page_id = query.header.next_page_id; + drop(query); + Ok(Self { + filepath: filepath.to_path_buf(), + database_id, + verified_generation, + verified_next_page_id, + flush_policy: FlushPolicy::Immediate, + active: None, + scratch: WriteScratch::default(), + _types: PhantomData, + }) + } + + pub fn try_new_with_backoff(filepath: &Path) -> io::Result { Self::try_new(filepath) } + + pub fn try_new_with_backoff_stats(filepath: &Path) -> io::Result<(Self, u64)> { + Self::try_new(filepath).map(|updater| (updater, 0)) + } + + pub fn set_flush_policy(&mut self, policy: FlushPolicy) { self.flush_policy = policy; } + + pub(super) fn ensure_transaction(&mut self) -> io::Result<()> { + if self.active.is_some() { + return Ok(()); + } + let guard = ExclusiveSidecarGuard::acquire(&self.filepath)?; + recover_pending_under_existing_lock(&self.filepath)?; + let (base, header) = DatabaseImage::open(&self.filepath)?; + if header.database_id != self.database_id + || header.generation != self.verified_generation + || header.next_page_id != self.verified_next_page_id + { + let mut query = BPlusTreeQuery::::from_file_unlocked(File::open(&self.filepath)?)?; + let _ = verify_full(&mut query)?; + self.database_id = query.header.database_id; + self.verified_generation = query.header.generation; + self.verified_next_page_id = query.header.next_page_id; + } + let transaction = WriteTransaction::new(header, base.as_slice().len())?; + self.active = Some(ActiveBatch { base, transaction, _guard: guard }); + Ok(()) + } + + pub fn update(&mut self, key: &K, value: V) -> Result { + self.upsert(key, &value).map_err(Into::into) + } + + pub fn update_batch(&mut self, items: &[(&K, &V)]) -> Result { + self.upsert_batch(items).map_err(Into::into) + } + + pub fn prepare_upsert_batch(items: &[(&K, &V)]) -> io::Result)>> { + let mut prepared = Vec::new(); + prepared.try_reserve_exact(items.len()).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + for (key, value) in items { + prepared.push(((*key).clone(), binary_serialize(value)?)); + } + prepared.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(prepared) + } + + pub fn upsert_batch_prepared_with_backoff(filepath: &Path, items: &[(&K, &V)]) -> io::Result { + let prepared = Self::prepare_upsert_batch(items)?; + let mut updater = Self::try_new_with_backoff(filepath)?; + updater.upsert_batch_encoded(prepared) + } + + pub fn upsert_batch(&mut self, items: &[(&K, &V)]) -> io::Result { + let prepared = match Self::prepare_upsert_batch(items) { + Ok(prepared) => prepared, + Err(error) => { + self.active = None; + return Err(error); + } + }; + self.upsert_batch_encoded(prepared) + } + + pub fn upsert_batch_encoded(&mut self, items: Vec<(K, Vec)>) -> io::Result { + if items.is_empty() { + return self.upsert_batch(&[]); + } + let policy = self.flush_policy; + self.flush_policy = FlushPolicy::Batch; + let mut root = 0; + for (key, value) in items { + match self.upsert_serialized(&key, &value) { + Ok(next_root) => root = next_root, + Err(error) => { + self.active = None; + self.flush_policy = policy; + return Err(error); + } + } + } + self.flush_policy = policy; + if policy == FlushPolicy::Immediate { + self.commit()?; + } + Ok(root) + } + + pub fn upsert(&mut self, key: &K, value: &V) -> io::Result { + let result = self.upsert_inner(key, value); + if result.is_err() { + self.active = None; + } + result + } + + fn upsert_inner(&mut self, key: &K, value: &V) -> io::Result { + self.scratch.key.clear(); + binary_serialize_into(&mut self.scratch.key, key)?; + self.scratch.value.clear(); + binary_serialize_into(&mut self.scratch.value, value)?; + self.ensure_transaction()?; + let stored = match encode_value(&self.scratch.value, &mut self.scratch.compression) { + Ok(stored) => stored, + Err(error) => { + self.active = None; + return Err(error); + } + }; + let active = self.active.as_mut().ok_or_else(|| invalid_data("write transaction is missing"))?; + let staged = stage_upsert::( + &mut active.transaction, + active.base.as_slice(), + key, + &self.scratch.key, + u32::try_from(self.scratch.value.len()).map_err(|_| invalid_input("serialized value exceeds u32"))?, + stored.compression(), + stored.as_slice(), + &mut self.scratch.cell, + ); + if let Err(error) = staged { + self.active = None; + return Err(error); + } + let root = self + .active + .as_ref() + .ok_or_else(|| invalid_data("write transaction is missing"))? + .transaction + .next_header + .root_page_id; + if self.flush_policy == FlushPolicy::Immediate { + self.commit()?; + } + Ok(root) + } + + fn upsert_serialized(&mut self, key: &K, raw_value: &[u8]) -> io::Result { + let result = self.upsert_serialized_inner(key, raw_value); + if result.is_err() { + self.active = None; + } + result + } + + fn upsert_serialized_inner(&mut self, key: &K, raw_value: &[u8]) -> io::Result { + self.scratch.key.clear(); + binary_serialize_into(&mut self.scratch.key, key)?; + self.ensure_transaction()?; + let stored = match encode_value(raw_value, &mut self.scratch.compression) { + Ok(stored) => stored, + Err(error) => { + self.active = None; + return Err(error); + } + }; + let active = self.active.as_mut().ok_or_else(|| invalid_data("write transaction is missing"))?; + let staged = stage_upsert::( + &mut active.transaction, + active.base.as_slice(), + key, + &self.scratch.key, + u32::try_from(raw_value.len()).map_err(|_| invalid_input("serialized value exceeds u32"))?, + stored.compression(), + stored.as_slice(), + &mut self.scratch.cell, + ); + if let Err(error) = staged { + self.active = None; + return Err(error); + } + let root = self + .active + .as_ref() + .ok_or_else(|| invalid_data("write transaction is missing"))? + .transaction + .next_header + .root_page_id; + if self.flush_policy == FlushPolicy::Immediate { + self.commit()?; + } + Ok(root) + } + + pub fn delete(&mut self, key: &K) -> io::Result { + let result = self.delete_inner(key); + if result.is_err() { + self.active = None; + } + result + } + + fn delete_inner(&mut self, key: &K) -> io::Result { + let started = self.active.is_none(); + self.scratch.key.clear(); + binary_serialize_into(&mut self.scratch.key, key)?; + self.ensure_transaction()?; + let active = self.active.as_mut().ok_or_else(|| invalid_data("write transaction is missing"))?; + let deleted = stage_delete::( + &mut active.transaction, + active.base.as_slice(), + key, + &self.scratch.key, + &mut self.scratch.cell, + ); + let deleted = match deleted { + Ok(deleted) => deleted, + Err(error) => { + self.active = None; + return Err(error); + } + }; + if !deleted && started { + self.active = None; + } else if deleted && self.flush_policy == FlushPolicy::Immediate { + self.commit()?; + } + Ok(deleted) + } + + pub fn delete_batch(&mut self, keys: &[&K]) -> io::Result { + if keys.is_empty() { + return Ok(0); + } + let policy = self.flush_policy; + self.flush_policy = FlushPolicy::Batch; + let mut deleted = 0usize; + let mut ordered = Vec::new(); + ordered.try_reserve_exact(keys.len()).map_err(|error| io::Error::new(io::ErrorKind::OutOfMemory, error))?; + ordered.extend_from_slice(keys); + ordered.sort(); + for key in ordered { + match self.delete(key) { + Ok(true) => deleted = deleted.checked_add(1).ok_or_else(|| invalid_input("delete count overflow"))?, + Ok(false) => {} + Err(error) => { + self.active = None; + self.flush_policy = policy; + return Err(error); + } + } + } + self.flush_policy = policy; + if policy == FlushPolicy::Immediate { + self.commit()?; + } + Ok(deleted) + } + + pub fn get_metadata(&self) -> io::Result { + if let Some(active) = &self.active { + return Ok(active.transaction.next_header.metadata.clone()); + } + BPlusTreeQuery::::try_new(&self.filepath).map(|query| query.header.metadata) + } + + pub fn set_metadata(&mut self, metadata: &BPlusTreeMetadata) -> io::Result<()> { + let started = self.active.is_none(); + self.ensure_transaction()?; + let active = self.active.as_mut().ok_or_else(|| invalid_data("write transaction is missing"))?; + if active.transaction.next_header.metadata == *metadata { + if started { + self.active = None; + } + return Ok(()); + } + active.transaction.next_header.metadata = metadata.clone(); + if self.flush_policy == FlushPolicy::Immediate { + self.commit()?; + } + Ok(()) + } + + pub fn query(&mut self, key: &K) -> Result, BPlusTreeError> { + if let Some(active) = &mut self.active { + return query_transaction::( + &active.transaction, + active.base.as_slice(), + key, + &mut self.scratch.read_value, + ) + .map_err(Into::into); + } + BPlusTreeQuery::::try_new(&self.filepath).and_then(|mut query| query.query_io(key)).map_err(Into::into) + } + + pub fn commit(&mut self) -> io::Result<()> { + let Some(mut active) = self.active.take() else { return Ok(()) }; + validate_transaction_links::(&active.transaction, active.base.as_slice())?; + let prepared = active.transaction.prepared_pages()?; + if prepared.is_empty() { + return Ok(()); + } + let committed_header = DatabaseHeader::decode(prepared[0].1)?; + let committed_generation = committed_header.generation; + let committed_next_page_id = committed_header.next_page_id; + let result = commit_ordered_page_refs_under_existing_lock(&self.filepath, &prepared); + match result { + Ok(()) => { + self.verified_generation = committed_generation; + self.verified_next_page_id = committed_next_page_id; + Ok(()) + } + Err(error) => { + let outcome = error + .get_ref() + .and_then(|source| source.downcast_ref::()) + .map(WalOperationError::outcome); + if outcome == Some(WalOutcome::CommittedCleanupPending) { + self.verified_generation = committed_generation; + self.verified_next_page_id = committed_next_page_id; + } + if let Err(recovery_error) = recover_pending_under_existing_lock(&self.filepath) { + log::error!( + "B+Tree commit failed and recovery remains pending for {}: {recovery_error}", + self.filepath.display() + ); + } + Err(error) + } + } + } + + pub fn compact(&mut self) -> io::Result<()> { + self.commit()?; + let filepath = self.filepath.clone(); + let header = with_exclusive_sidecar(&filepath, || { + recover_pending_under_existing_lock(&filepath)?; + let mut query = BPlusTreeQuery::::from_file_unlocked(File::open(&filepath)?)?; + let _ = verify_full(&mut query)?; + let metadata = query.header.metadata.clone(); + let mut entries = BTreeMap::new(); + for entry in query.iter() { + let (key, value) = entry?; + let _ = entries.insert(key, value); + } + drop(query); + + let mut replacement = BPlusTree { entries, metadata, dirty: true }; + let _ = replacement.store_exclusive(&filepath)?; + let file = File::open(&filepath)?; + let mut page = [0; PAGE_SIZE]; + read_exact_at_offset(&file, &mut page, 0)?; + DatabaseHeader::decode(&page) + })?; + self.database_id = header.database_id; + self.verified_generation = header.generation; + self.verified_next_page_id = header.next_page_id; + Ok(()) + } +} diff --git a/backend/btree/src/v3/tree/verify.rs b/backend/btree/src/v3/tree/verify.rs new file mode 100644 index 000000000..78e4462bd --- /dev/null +++ b/backend/btree/src/v3/tree/verify.rs @@ -0,0 +1,298 @@ +use super::{ + decode_key, invalid_data, minimum_tree_key, push_child_visits, record_page_visit, validate_leaf_backlink, + validated_overflow_chain_pages, verify_internal_page, verify_leaf_page, BPlusTreeQuery, Compression, + InternalCellRef, InternalPreamble, LeafCellRef, LeafValueRef, PageType, SlottedPage, VerificationReport, + WriteTransaction, PAGE_SIZE, +}; +use serde::Deserialize; +use std::{ + collections::{HashMap, HashSet}, + io, +}; + +fn validate_transaction_free_list(transaction: &WriteTransaction, base: &[u8]) -> io::Result<()> { + let mut free = HashSet::new(); + let mut page_id = transaction.next_header.free_page_head; + while page_id != 0 { + if !free.insert(page_id) { + return Err(invalid_data("free list contains a cycle")); + } + if transaction.allocated_pages.contains(&page_id) { + return Err(invalid_data("allocated page remains linked from free list")); + } + let page = SlottedPage::open(transaction.page(base, page_id)?, page_id, transaction.next_header.next_page_id)?; + if page.header().page_type != PageType::Free { + return Err(invalid_data("free list references a non-free page")); + } + page_id = page.header().right; + } + if !transaction.freed_pages.iter().all(|page| free.contains(page)) { + return Err(invalid_data("freed page is missing from free list")); + } + for page_id in &transaction.allocated_pages { + let page = transaction.page(base, *page_id)?; + if SlottedPage::open(page, *page_id, transaction.next_header.next_page_id)?.header().page_type == PageType::Free + { + return Err(invalid_data("allocated page still has free-page type")); + } + } + Ok(()) +} + +pub(super) fn validate_transaction_links(transaction: &WriteTransaction, base: &[u8]) -> io::Result<()> +where + K: Ord + for<'de> Deserialize<'de>, +{ + let original_length = transaction + .original_header + .next_page_id + .checked_mul(u64::try_from(PAGE_SIZE).map_err(|_| invalid_data("page size exceeds u64"))?) + .ok_or_else(|| invalid_data("original database length overflow"))?; + if original_length != transaction.original_file_len { + return Err(invalid_data("transaction original length does not match header")); + } + + let mut owned_overflow_pages = HashSet::new(); + for (page_id, bytes) in &transaction.dirty_pages { + if *page_id == 0 { + continue; + } + let page = SlottedPage::open(bytes.as_slice(), *page_id, transaction.next_header.next_page_id)?; + match page.header().page_type { + PageType::Leaf => { + validate_leaf_backlink(transaction, base, *page_id, page.header().left, false)?; + validate_leaf_backlink(transaction, base, *page_id, page.header().right, true)?; + let mut previous_key = None; + for cell in page.cells() { + let cell = LeafCellRef::decode(cell?, *page_id, transaction.next_header.next_page_id)?; + let key = decode_key::(cell.key_bytes)?; + if previous_key.as_ref().is_some_and(|previous| previous >= &key) { + return Err(invalid_data("leaf keys are not strictly increasing")); + } + previous_key = Some(key); + if let LeafValueRef::Overflow { stored_len, head, crc32, .. } = cell.value { + for overflow_page in validated_overflow_chain_pages(transaction, base, head, stored_len, crc32)? + { + if !owned_overflow_pages.insert(overflow_page) { + return Err(invalid_data("overflow page is owned by multiple values")); + } + } + } + } + } + PageType::Internal => { + let preamble = + InternalPreamble::decode(bytes.as_slice(), *page_id, transaction.next_header.next_page_id)?; + let child = SlottedPage::open( + transaction.page(base, preamble.leftmost_child)?, + preamble.leftmost_child, + transaction.next_header.next_page_id, + )?; + if !matches!(child.header().page_type, PageType::Leaf | PageType::Internal) { + return Err(invalid_data("internal page references a non-tree child")); + } + let mut previous_key = None; + for cell in page.cells() { + let cell = InternalCellRef::decode(cell?, *page_id, transaction.next_header.next_page_id)?; + let key = decode_key::(cell.key_bytes)?; + if previous_key.as_ref().is_some_and(|previous| previous >= &key) { + return Err(invalid_data("internal separator keys are not strictly increasing")); + } + previous_key = Some(key); + let minimum = minimum_tree_key(transaction, base, cell.right_child)? + .ok_or_else(|| invalid_data("internal separator references an empty subtree"))?; + if minimum != cell.key_bytes { + return Err(invalid_data("internal separator differs from right subtree minimum")); + } + } + } + PageType::Overflow => { + if page.header().right != 0 { + let next = SlottedPage::open( + transaction.page(base, page.header().right)?, + page.header().right, + transaction.next_header.next_page_id, + )?; + if next.header().page_type != PageType::Overflow { + return Err(invalid_data("overflow page references a non-overflow page")); + } + } + } + PageType::Free => {} + } + } + + validate_transaction_free_list(transaction, base) +} + +pub(super) struct VerifyLeaf { + pub(super) page_id: u64, + pub(super) left: u64, + pub(super) right: u64, + pub(super) minimum: Option, + pub(super) maximum: Option, +} + +pub(super) enum Visit { + Enter(u64, Option, Option), + Exit(u64), +} + +pub(super) enum VerifyValue { + Inline, + Overflow(Compression, u32, u32, u64, u32), + Tombstone, +} + +fn finish_verified_page( + page_id: u64, + active: &mut HashSet, + internal_children: &mut HashMap)>>, + page_minimum: &mut HashMap>, +) -> io::Result<()> { + if !active.remove(&page_id) { + return Err(invalid_data("tree verifier active set is inconsistent")); + } + let Some(children) = internal_children.remove(&page_id) else { return Ok(()) }; + for (child, separator) in children.iter().skip(1) { + let actual = page_minimum + .get(child) + .and_then(Option::as_ref) + .ok_or_else(|| invalid_data("internal child has no minimum key"))?; + if separator.as_ref() != Some(actual) { + return Err(invalid_data("internal separator is not the right child minimum")); + } + } + let minimum = children + .first() + .and_then(|(child, _)| page_minimum.get(child)) + .cloned() + .ok_or_else(|| invalid_data("internal leftmost child has no verified minimum"))?; + page_minimum.insert(page_id, minimum); + Ok(()) +} + +fn verify_leaf_links(leaves: &[VerifyLeaf]) -> io::Result<()> { + for (index, leaf) in leaves.iter().enumerate() { + let expected_left = index.checked_sub(1).map_or(0, |previous| leaves[previous].page_id); + let expected_right = leaves.get(index + 1).map_or(0, |next| next.page_id); + if leaf.left != expected_left || leaf.right != expected_right { + return Err(invalid_data("leaf sibling links do not match tree order")); + } + if index > 0 + && leaves[index - 1] + .maximum + .as_ref() + .zip(leaf.minimum.as_ref()) + .is_some_and(|(previous, current)| previous >= current) + { + return Err(invalid_data("keys are inverted across leaf siblings")); + } + } + Ok(()) +} + +fn verify_free_pages( + query: &mut BPlusTreeQuery, + tree_pages: &HashSet, + overflow_pages: &HashSet, +) -> io::Result> { + let mut free_pages = HashSet::new(); + let mut free = query.header.free_page_head; + while free != 0 { + record_page_visit(&mut free_pages, free, "free list contains a duplicate or cycle")?; + if tree_pages.contains(&free) || overflow_pages.contains(&free) { + return Err(invalid_data("page is reachable from both live data and free list")); + } + let next_page_id = query.header.next_page_id; + free = query.with_page(free, |bytes, _| { + let page = SlottedPage::open(bytes, free, next_page_id)?; + if page.header().page_type != PageType::Free { + return Err(invalid_data("free list references a non-free page")); + } + Ok(page.header().right) + })?; + } + Ok(free_pages) +} + +fn verify_page_ownership( + next_page_id: u64, + tree_pages: &HashSet, + overflow_pages: &HashSet, + free_pages: &HashSet, +) -> io::Result<()> { + for page_id in 1..next_page_id { + let memberships = u8::from(tree_pages.contains(&page_id)) + + u8::from(overflow_pages.contains(&page_id)) + + u8::from(free_pages.contains(&page_id)); + if memberships != 1 { + return Err(invalid_data("database contains an orphan or multiply-owned page")); + } + } + Ok(()) +} + +pub(crate) fn verify_full(query: &mut BPlusTreeQuery) -> io::Result +where + K: Ord + for<'de> Deserialize<'de> + Clone, +{ + let mut tree_pages = HashSet::new(); + let mut active = HashSet::new(); + let mut overflow_pages = HashSet::new(); + let mut leaves = Vec::>::new(); + let mut page_minimum = HashMap::>::new(); + let mut internal_children = HashMap::)>>::new(); + let mut live_entries = 0u64; + let mut stack = vec![Visit::Enter(query.header.root_page_id, None, None)]; + + while let Some(visit) = stack.pop() { + let (page_id, lower, upper) = match visit { + Visit::Exit(page_id) => { + finish_verified_page(page_id, &mut active, &mut internal_children, &mut page_minimum)?; + continue; + } + Visit::Enter(page_id, lower, upper) => (page_id, lower, upper), + }; + if active.contains(&page_id) { + return Err(invalid_data("tree child graph contains a cycle")); + } + if !tree_pages.insert(page_id) { + return Err(invalid_data("tree page has multiple parents")); + } + active.insert(page_id); + stack.push(Visit::Exit(page_id)); + let next_page_id = query.header.next_page_id; + let page_type = query.with_page(page_id, |bytes, _| { + SlottedPage::open(bytes, page_id, next_page_id).map(|page| page.header().page_type) + })?; + match page_type { + PageType::Leaf => { + let (leaf, page_live_entries) = + verify_leaf_page(query, page_id, lower.as_ref(), upper.as_ref(), &mut overflow_pages)?; + live_entries = live_entries + .checked_add(page_live_entries) + .ok_or_else(|| invalid_data("live entry count overflow"))?; + page_minimum.insert(page_id, leaf.minimum.clone()); + leaves.push(leaf); + } + PageType::Internal => { + let children = verify_internal_page(query, page_id)?; + push_child_visits(&mut stack, &children, lower.as_ref(), upper.as_ref())?; + internal_children.insert(page_id, children); + } + PageType::Overflow | PageType::Free => return Err(invalid_data("tree child has the wrong page type")), + } + } + + verify_leaf_links(&leaves)?; + let free_pages = verify_free_pages(query, &tree_pages, &overflow_pages)?; + verify_page_ownership(query.header.next_page_id, &tree_pages, &overflow_pages, &free_pages)?; + Ok(VerificationReport { + live_entries, + tree_pages: u64::try_from(tree_pages.len()).map_err(|_| invalid_data("tree page count exceeds u64"))?, + overflow_pages: u64::try_from(overflow_pages.len()) + .map_err(|_| invalid_data("overflow page count exceeds u64"))?, + free_pages: u64::try_from(free_pages.len()).map_err(|_| invalid_data("free page count exceeds u64"))?, + }) +} diff --git a/backend/config-loader/src/lib.rs b/backend/config-loader/src/lib.rs index 9162a58b2..41388a750 100644 --- a/backend/config-loader/src/lib.rs +++ b/backend/config-loader/src/lib.rs @@ -1460,35 +1460,35 @@ mod tests { let template_path = dir.path().join("template.yml"); tokio::fs::write( &sources_path, - r#" + r" inputs: [] sources: [] templates: - name: source_group value: '${env:PATH}' -"#, +", ) .await?; tokio::fs::write( &mapping_path, - r#" + r" mappings: templates: - name: mapping_group value: '${env:PATH}' mapping: [] -"#, +", ) .await?; tokio::fs::write( &template_path, - r#" + r" templates: - name: file_group value: '${env:PATH}' - name: combined value: '!source_group!' -"#, +", ) .await?; let paths = shared::model::ConfigPaths { @@ -1528,7 +1528,7 @@ templates: async fn curation_item_limits_survive_source_load_sanitize_save_reload() { let dir = tempdir().unwrap(); let path = dir.path().join("source.yml"); - let document = r#" + let document = r" inputs: [] sources: - inputs: [] @@ -1554,7 +1554,7 @@ sources: limit: 100 - kind: movie time_window: day -"#; +"; tokio::fs::write(&path, document).await.unwrap(); let dto = super::parse_sources_file_from_path(&path, false).await.unwrap(); let expected = dto.sources[0].targets[0].curation.clone(); diff --git a/backend/core/src/model/config/reverse_proxy.rs b/backend/core/src/model/config/reverse_proxy.rs index 7cf176e4b..f602bfb91 100644 --- a/backend/core/src/model/config/reverse_proxy.rs +++ b/backend/core/src/model/config/reverse_proxy.rs @@ -256,8 +256,61 @@ impl From<&HlsCorruptSegmentWatchdogConfig> for HlsCorruptSegmentWatchdogConfigD } } +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsStartupConfig { + pub mode: shared::model::HlsStartupMode, + pub max_progressive_segments: usize, + pub max_progressive_bytes_per_segment: Bytes, + pub max_progressive_bytes_per_segment_str: String, + pub max_progressive_bytes_total: Bytes, + pub max_progressive_bytes_total_str: String, + pub max_progressive_reader_lifetime_secs: Secs, + pub max_deferred_repairs: usize, +} + +impl From<&shared::model::HlsStartupConfigDto> for HlsStartupConfig { + fn from(dto: &shared::model::HlsStartupConfigDto) -> Self { + Self { + mode: dto.mode, + max_progressive_segments: dto.max_progressive_segments, + max_progressive_bytes_per_segment: parse_hls_byte_size_or_default( + &dto.max_progressive_bytes_per_segment, + shared::model::DEFAULT_HLS_PROGRESSIVE_BYTES_PER_SEGMENT, + ), + max_progressive_bytes_per_segment_str: dto.max_progressive_bytes_per_segment.as_str().to_string(), + max_progressive_bytes_total: parse_hls_byte_size_or_default( + &dto.max_progressive_bytes_total, + shared::model::DEFAULT_HLS_PROGRESSIVE_BYTES_TOTAL, + ), + max_progressive_bytes_total_str: dto.max_progressive_bytes_total.as_str().to_string(), + max_progressive_reader_lifetime_secs: dto.max_progressive_reader_lifetime_secs, + max_deferred_repairs: dto.max_deferred_repairs, + } + } +} + +impl From<&HlsStartupConfig> for shared::model::HlsStartupConfigDto { + fn from(config: &HlsStartupConfig) -> Self { + Self { + mode: config.mode, + max_progressive_segments: config.max_progressive_segments, + max_progressive_bytes_per_segment: shared::model::ByteSize::new( + config.max_progressive_bytes_per_segment_str.clone(), + ), + max_progressive_bytes_total: shared::model::ByteSize::new(config.max_progressive_bytes_total_str.clone()), + max_progressive_reader_lifetime_secs: config.max_progressive_reader_lifetime_secs, + max_deferred_repairs: config.max_deferred_repairs, + } + } +} + +impl Default for HlsStartupConfig { + fn default() -> Self { Self::from(&shared::model::HlsStartupConfigDto::default()) } +} + #[derive(Debug, Clone, Eq, PartialEq)] pub struct HlsCacheConfig { + pub startup: HlsStartupConfig, pub cache_path: String, pub strip: StripConfig, pub cache_duration: Secs, @@ -289,6 +342,7 @@ pub fn default_hls_cache_path() -> String { impl From<&HlsCacheConfigDto> for HlsCacheConfig { fn from(dto: &HlsCacheConfigDto) -> Self { Self { + startup: HlsStartupConfig::from(&dto.startup), cache_path: dto.cache_path.as_ref().map_or_else(default_hls_cache_path, |path: &String| Clone::clone(path)), strip: StripConfig::from(&dto.strip), cache_duration: dto.cache_duration, @@ -312,6 +366,7 @@ impl From<&HlsCacheConfigDto> for HlsCacheConfig { impl From<&HlsCacheConfig> for HlsCacheConfigDto { fn from(config: &HlsCacheConfig) -> Self { Self { + startup: shared::model::HlsStartupConfigDto::from(&config.startup), cache_path: Some(config.cache_path.clone()), strip: shared::model::HlsStripConfigDto::from(&config.strip), cache_duration: config.cache_duration, @@ -399,6 +454,21 @@ mod tests { default_hls_cache_path, HlsCacheConfig, HlsManifestRecoveryBurstConfig, HlsSegmentRepairConfig, ReverseProxyConfig, }; + + #[test] + fn startup_config_preserves_mode_and_custom_budgets_across_dto_conversion() -> Result<(), Box> + { + let mut dto: shared::model::HlsCacheConfigDto = serde_json::from_str( + r#"{"startup":{"mode":"progressive","max_progressive_segments":3,"max_progressive_bytes_per_segment":"8MB","max_progressive_bytes_total":"24MB","max_progressive_reader_lifetime_secs":40,"max_deferred_repairs":7}}"#, + )?; + dto.prepare()?; + let domain = super::HlsCacheConfig::from(&dto); + let restored = shared::model::HlsCacheConfigDto::from(&domain); + assert_eq!(restored.startup, dto.startup); + assert_eq!(domain.startup.max_progressive_segments, 3); + assert_eq!(domain.startup.max_deferred_repairs, 7); + Ok(()) + } use shared::model::{ ByteSize, Bytes, HlsCacheConfigDto, HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripMode, Millis, QosAggregationConfigDto, ReverseProxyConfigDto, Secs, StreamHistoryConfigDto, @@ -475,6 +545,7 @@ mod tests { cache_bytes_str: "10GB".to_string(), cache_bytes_per_session: Bytes::new(536_870_912), // 512 * 1024^2 (was 512 * 1e6 under SI decimal before consolidation) cache_bytes_per_session_str: "512MB".to_string(), + startup: super::HlsStartupConfig::default(), max_segments_prefetch: 6, max_concurrent_segment_fetches_per_session: 2, max_concurrent_segment_fetches_global: 64, diff --git a/backend/core/src/utils/file/config_backup.rs b/backend/core/src/utils/file/config_backup.rs index b66f3f289..f4d9dd56d 100644 --- a/backend/core/src/utils/file/config_backup.rs +++ b/backend/core/src/utils/file/config_backup.rs @@ -61,6 +61,8 @@ async fn remove_snapshot(snapshot: &Path) -> io::Result<()> { { let mut permissions = tokio::fs::metadata(snapshot).await?.permissions(); if permissions.readonly() { + // Windows maps this to clearing FILE_ATTRIBUTE_READONLY; it does not grant world-write. + #[allow(clippy::permissions_set_readonly_false)] permissions.set_readonly(false); tokio::fs::set_permissions(snapshot, permissions).await?; } diff --git a/backend/core/src/utils/network/request.rs b/backend/core/src/utils/network/request.rs index 5e2a7f41e..d914568a6 100644 --- a/backend/core/src/utils/network/request.rs +++ b/backend/core/src/utils/network/request.rs @@ -5,137 +5,20 @@ pub use super::DynReader; /// Owned here because this is the client that enforces it; the streaming layer /// re-exports it so both sides cannot drift apart. pub const STREAM_IDLE_TIMEOUT: u64 = 60; -use crate::{ - model::{ - resolve_provider_scheme_url_with_provider_index, AppConfig, Config, ConfigInput, ConfigProvider, InputSource, - ResourceRetryConfig, ReverseProxyDisabledHeaderConfig, - }, - utils::{ - async_file_reader, async_file_writer, - compression::compression_utils::is_gzip, - content_coding::{ - apply_outbound_content_coding_policy, content_decoding_error_from_io, decode_response_to_identity, - is_http_body_transport_error, log_hls_origin_content_coding, read_utf8_limited, ContentBodyReadError, - ContentCodingDetection, ContentCodingError, HlsOriginContentCodingObjectKind, HlsOriginContentCodingSource, - OutboundContentCodingPolicy, - }, - debug_if_enabled, get_file_path, - network::persist_pipe::tee_dyn_reader, - persist_file, - }, -}; -use futures::StreamExt; -use log::{debug, error, log_enabled, trace, warn, Level}; -use lru::LruCache; -use regex::Regex; -use reqwest::{ - header::{HeaderMap, HeaderName, HeaderValue, HOST}, - redirect::Policy, - StatusCode, -}; -use shared::{ - defaults::DEFAULT_USER_AGENT, - error::{string_to_io_error, TuliproxError}, - model::{format_elapsed_time, InputFetchMethod, OnConnectErrorPolicy, ProviderUrlSelectionPolicy}, - utils::{filter_request_header, human_readable_byte_size, sanitize_sensitive_info, CONTENT_TYPE_JSON}, -}; + +use crate::utils::content_coding::{ContentCodingDetection, OutboundContentCodingPolicy}; +use reqwest::header::HeaderMap; use std::{ - collections::{HashMap, HashSet}, - io::{Error, ErrorKind}, - net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, - num::NonZeroUsize, - path::{Path, PathBuf}, - sync::{Arc, Mutex, Once, OnceLock, PoisonError}, - time::{Duration, Instant}, + path::Path, + sync::{Arc, Once}, + time::Duration, }; -use tokio::{ - fs::File, - io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}, - time::{sleep, timeout}, -}; -use url::Url; static PROXY_DIAGNOSTICS_ONCE: Once = Once::new(); #[derive(Debug, Clone, Copy, Default)] pub struct PublicIpResolver; -impl reqwest::dns::Resolve for PublicIpResolver { - fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { - let host = name.as_str().to_string(); - Box::pin(async move { - let addresses = resolve_public_socket_addrs(&host, 0) - .await - .map_err(|err| Box::new(err) as Box)?; - Ok(Box::new(addresses.into_iter()) as reqwest::dns::Addrs) - }) - } -} - -pub async fn resolve_public_socket_addrs(host: &str, port: u16) -> std::io::Result> { - let addresses = lookup_socket_addrs(host, port).await?; - if addresses.is_empty() || addresses.iter().any(|address| !is_public_ip(address.ip())) { - return Err(Error::new(ErrorKind::PermissionDenied, "destination resolves to a non-public address")); - } - Ok(addresses) -} - -/// Resolves a resource destination, refusing addresses that are local to this host. -/// -/// Private network destinations stay allowed: self-hosted services are a normal resource source. -pub async fn resolve_resource_socket_addrs(host: &str, port: u16) -> std::io::Result> { - let addresses = lookup_socket_addrs(host, port).await?; - if addresses.is_empty() { - return Err(Error::new(ErrorKind::PermissionDenied, "destination does not resolve")); - } - if addresses.iter().any(|address| classify_ip(address.ip()) == ResourceDestination::Blocked) { - return Err(Error::new(ErrorKind::PermissionDenied, "destination is local to this host")); - } - Ok(addresses) -} - -/// Resolves a host name or IP literal, bracketed IPv6 included. An unresolvable name yields an empty -/// result so that every policy decides on its own how to report it. -async fn lookup_socket_addrs(host: &str, port: u16) -> std::io::Result> { - if let Ok(address) = host_literal(host).parse::() { - return Ok(vec![SocketAddr::new(address, port)]); - } - Ok(tokio::net::lookup_host((host, port)).await?.collect()) -} - -/// Hosts of the proxies an outbound request may be routed through. -/// -/// A resource client that honours the configured proxy asks its resolver about the proxy host as well, -/// and a proxy commonly lives on the loopback interface (`http://localhost:8118`). That name is exempt -/// from the local-address guard, because a proxy on this host is a valid destination for this host. -fn proxy_hosts(app_config: &AppConfig) -> Vec> { - const ENV_KEYS: [&str; 3] = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"]; - let mut hosts = Vec::new(); - let config = app_config.config.load(); - if let Some(proxy) = config.proxy.as_ref() { - if let Some(host) = - crate::model::parse_proxy_url_with_http_fallback(&proxy.url).and_then(|url| url.host_str().map(Arc::from)) - { - hosts.push(host); - } - } - drop(config); - for (key, value) in std::env::vars_os() { - let (Some(key), Some(value)) = (key.to_str(), value.to_str()) else { - continue; - }; - if !ENV_KEYS.iter().any(|candidate| candidate.eq_ignore_ascii_case(key)) { - continue; - } - if let Some(host) = - crate::model::parse_proxy_url_with_http_fallback(value).and_then(|url| url.host_str().map(Arc::from)) - { - hosts.push(host); - } - } - hosts -} - /// DNS layer of the proxied resource client. /// /// Classifying a destination leaves a window: a name can resolve to a public address while it is @@ -150,29 +33,6 @@ pub struct ResourceDestinationResolver { allowed_hosts: Arc<[Arc]>, } -impl ResourceDestinationResolver { - /// Resolver that resolves the proxy hosts of this configuration and guards every other name. - pub fn allowing_proxy_hosts(app_config: &AppConfig) -> Self { - Self { allowed_hosts: proxy_hosts(app_config).into() } - } -} - -impl reqwest::dns::Resolve for ResourceDestinationResolver { - fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { - let host = name.as_str().to_owned(); - let allowed = self.allowed_hosts.iter().any(|allowed| allowed.as_ref().eq_ignore_ascii_case(&host)); - Box::pin(async move { - let addresses = if allowed { - lookup_socket_addrs(&host, 0).await - } else { - resolve_resource_socket_addrs(&host, 0).await - } - .map_err(|err| Box::new(err) as Box)?; - Ok(Box::new(addresses.into_iter()) as reqwest::dns::Addrs) - }) - } -} - /// Reachability class of a resource destination. /// /// Resource URLs are supplied by providers and third-party EPG data, so they are untrusted input. @@ -191,213 +51,26 @@ pub enum ResourceDestination { Blocked, } -/// Reachability class of an address. Single place that decides which range belongs to which class. -pub fn classify_ip(address: IpAddr) -> ResourceDestination { - match address { - IpAddr::V4(address) => classify_ipv4(address), - IpAddr::V6(address) => classify_ipv6(address), - } -} - -fn classify_ipv4(address: Ipv4Addr) -> ResourceDestination { - let [first, second, _, _] = address.octets(); - if address.is_loopback() - || address.is_link_local() - || address.is_unspecified() - || address.is_multicast() - || address.is_broadcast() - || first == 0 - { - ResourceDestination::Blocked - } else if address.is_private() - || address.is_documentation() - || (first == 100 && (64..=127).contains(&second)) - || (first == 198 && matches!(second, 18 | 19)) - || first >= 240 - { - ResourceDestination::Private - } else { - ResourceDestination::Public - } -} - -/// IPv4 address carried inside an IPv6 address: IPv4-compatible, NAT64 (`64:ff9b::/96`) or 6to4. -fn embedded_ipv4(address: Ipv6Addr) -> Option { - let segments = address.segments(); - let (high, low) = if segments[..6] == [0, 0, 0, 0, 0, 0] || segments[..6] == [0x0064, 0xff9b, 0, 0, 0, 0] { - (segments[6], segments[7]) - } else if segments[0] == 0x2002 { - (segments[1], segments[2]) - } else { - return None; - }; - let [a, b] = high.to_be_bytes(); - let [c, d] = low.to_be_bytes(); - Some(Ipv4Addr::new(a, b, c, d)) -} - -fn classify_ipv6(address: Ipv6Addr) -> ResourceDestination { - let segments = address.segments(); - let embedded = address.to_ipv4_mapped().or_else(|| embedded_ipv4(address)); - if address.is_loopback() - || address.is_unspecified() - || address.is_multicast() - || segments[0] & 0xffc0 == 0xfe80 - || embedded.is_some_and(|mapped| classify_ipv4(mapped) == ResourceDestination::Blocked) - { - ResourceDestination::Blocked - } else if segments[0] & 0xfe00 == 0xfc00 - || segments[0] & 0xffc0 == 0xfec0 - || (segments[0] == 0x2001 && segments[1] == 0x0db8) - || embedded.is_some_and(|mapped| classify_ipv4(mapped) == ResourceDestination::Private) - { - ResourceDestination::Private - } else { - ResourceDestination::Public - } -} - -/// Whether an address is publicly routable. -/// -/// Derived from [`classify_ip`] so classification and this answer cannot drift apart. -pub fn is_public_ip(address: IpAddr) -> bool { classify_ip(address) == ResourceDestination::Public } - -/// Strips the brackets of an IPv6 literal so that it parses as an address. -fn host_literal(host: &str) -> &str { host.strip_prefix('[').and_then(|value| value.strip_suffix(']')).unwrap_or(host) } - /// How long a verdict that allows a URL to be handed to a client is remembered. const RESOURCE_DESTINATION_PUBLIC_TTL: Duration = Duration::from_secs(60); + /// How long a verdict that keeps a destination away from clients is remembered. Both classes fail /// closed, so they can be cached longer than the one that decides exposure. const RESOURCE_DESTINATION_PRIVATE_TTL: Duration = Duration::from_secs(300); + /// How long a name that produced no answer within the resolution budget is remembered. /// /// The cause - a resolver that is slow, overloaded, or briefly unreachable - is transient, so such a /// name must not be pinned to a non-public verdict for minutes. It stays non-public while it is /// remembered, because nothing was proven. const RESOURCE_DESTINATION_UNANSWERED_TTL: Duration = Duration::from_secs(10); + /// Upper bound for remembered verdicts, evicted least-recently-used. const RESOURCE_DESTINATION_CACHE_CAPACITY: usize = 4096; + /// Resolution budget when classifying a destination given as a name. const RESOURCE_DESTINATION_LOOKUP_TIMEOUT: Duration = Duration::from_millis(250); -struct RememberedDestination { - verdict: ResourceDestination, - /// Whether the verdict came from a resolved answer. A name whose lookup produced none stays - /// non-public, but nothing is known about it either. - answered: bool, - expires_at: Instant, -} - -/// Memoizes destination verdicts so that rendering a playlist or EPG does not resolve the same host -/// per icon. -/// -/// The verdict is a property of the host, so one memo serves the whole process: every route and -/// every rendering path has to reach the same answer, and a per-request memo would put a DNS lookup -/// on the playlist path for each request instead of once per host. -/// -/// This is a hot-path memo, not policy state: the verdict decides whether a URL may be exposed, and -/// the resource client resolves and validates again while the connection is built, so a stale entry -/// can never open a connection to a local address. -struct DestinationCache { - destinations: Mutex>, -} - -impl DestinationCache { - fn new() -> Self { - Self { - destinations: Mutex::new(LruCache::new( - NonZeroUsize::new(RESOURCE_DESTINATION_CACHE_CAPACITY).unwrap_or(NonZeroUsize::MIN), - )), - } - } - - /// The process-wide memo. - fn shared() -> &'static Self { - static SHARED: OnceLock = OnceLock::new(); - SHARED.get_or_init(DestinationCache::new) - } - - fn verdict_and_answer(&self, host: &str) -> Option<(ResourceDestination, bool)> { - let mut destinations = self.destinations.lock().unwrap_or_else(PoisonError::into_inner); - match destinations.get(host) { - Some(entry) if entry.expires_at > Instant::now() => Some((entry.verdict, entry.answered)), - Some(_) => { - destinations.pop(host); - None - } - None => None, - } - } - - fn remember(&self, host: &str, verdict: ResourceDestination, answered: bool, ttl: Duration) { - let mut destinations = self.destinations.lock().unwrap_or_else(PoisonError::into_inner); - let entry = RememberedDestination { verdict, answered, expires_at: Instant::now() + ttl }; - destinations.put(host.to_owned(), entry); - } -} - -/// Classifies a resource destination given as host name or IP literal, bracketed IPv6 included. -/// -/// An IP literal is decided by its address alone, without resolving anything. Names are resolved once -/// per verdict lifetime and remembered in the process-wide memo. A name that cannot be resolved is -/// reported as [`ResourceDestination::Private`]: callers must not hand a possibly internal name to a -/// client, and the fetch attempt itself decides whether the destination is reachable. -pub async fn classify_resource_destination(host: &str) -> ResourceDestination { classify_host(host).await.0 } - -/// Classifies a destination and reports whether the verdict came from a resolved answer. -/// -/// The second value separates "proven to be local" from "nothing was proven": both keep a client away -/// from the destination, but only the first says the destination cannot be reached from outside. -pub(crate) async fn classify_host(host: &str) -> (ResourceDestination, bool) { - if let Ok(address) = host_literal(host).parse::() { - // An IP literal is decided by its address alone, so the answer is factual either way. - return (classify_ip(address), true); - } - let cache = DestinationCache::shared(); - if let Some(verdict) = cache.verdict_and_answer(host) { - return verdict; - } - - let mut resolved = Vec::new(); - if let Ok(Ok(addresses)) = timeout(RESOURCE_DESTINATION_LOOKUP_TIMEOUT, tokio::net::lookup_host((host, 0))).await { - resolved.extend(addresses.map(|address| address.ip())); - } - let (verdict, ttl) = verdict_for_addresses(&resolved); - let answered = !resolved.is_empty(); - cache.remember(host, verdict, answered, ttl); - (verdict, answered) -} - -/// Aggregates the addresses a name resolved to into a single verdict, and the time that verdict may be -/// remembered. -/// -/// One local-only address blocks the whole destination, and one private address keeps it non-public: -/// a client must not be able to reach the destination by any of its addresses. An empty answer is -/// non-public, because nothing was proven, and is remembered only briefly: unlike a resolved address, -/// it does not prove anything about the destination, so the name may well be reachable once the -/// resolver answers again. -fn verdict_for_addresses(addresses: &[IpAddr]) -> (ResourceDestination, Duration) { - if addresses.is_empty() { - return (ResourceDestination::Private, RESOURCE_DESTINATION_UNANSWERED_TTL); - } - let mut verdict = ResourceDestination::Public; - for address in addresses { - match classify_ip(*address) { - ResourceDestination::Blocked => { - return (ResourceDestination::Blocked, RESOURCE_DESTINATION_PRIVATE_TTL); - } - ResourceDestination::Private => verdict = ResourceDestination::Private, - ResourceDestination::Public => {} - } - } - let ttl = match verdict { - ResourceDestination::Public => RESOURCE_DESTINATION_PUBLIC_TTL, - ResourceDestination::Private | ResourceDestination::Blocked => RESOURCE_DESTINATION_PRIVATE_TTL, - }; - (verdict, ttl) -} - /// Options applied at the final boundary of every physical request attempt. #[derive(Debug, Clone, Copy, Default)] pub struct RequestFetchOptions { @@ -407,65 +80,6 @@ pub struct RequestFetchOptions { return_http_errors: bool, } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -enum ResourceRetryExecution { - #[default] - Configured, - ProviderFailoverOnly, -} - -impl RequestFetchOptions { - pub fn with_attempt_idle_timeout(timeout: Duration) -> Self { - Self { attempt_idle_timeout: Some(timeout.max(Duration::from_millis(1))), ..Self::default() } - } - - pub const fn with_content_coding(mut self, content_coding: OutboundContentCodingPolicy) -> Self { - self.content_coding = content_coding; - self - } - - /// Returns the final HTTP error response after the configured retry and failover policy. - pub const fn with_http_error_responses(mut self, enabled: bool) -> Self { - self.return_http_errors = enabled; - self - } - - /// Leaves bounded provider failover intact while assigning retry rounds to the caller. - pub const fn without_resource_retries(mut self) -> Self { - self.resource_retry = ResourceRetryExecution::ProviderFailoverOnly; - self - } - - fn attempt_idle_timeout_or_default(self) -> Duration { - self.attempt_idle_timeout.unwrap_or_else(|| Duration::from_secs(STREAM_IDLE_TIMEOUT)) - } - - const fn uses_provider_failover_only(self) -> bool { - matches!(self.resource_retry, ResourceRetryExecution::ProviderFailoverOnly) - } -} - -fn apply_request_fetch_options(request: &mut reqwest::Request, options: RequestFetchOptions) { - if let Some(timeout) = options.attempt_idle_timeout { - *request.timeout_mut() = Some(timeout); - } - apply_outbound_content_coding_policy(request.headers_mut(), options.content_coding); -} - -fn prepare_physical_request_attempt( - request_builder: reqwest::RequestBuilder, - target: &AttemptTarget, - options: RequestFetchOptions, -) -> Result<(reqwest::Client, reqwest::Request), std::io::Error> { - let (base_client, request_result) = request_builder.build_split(); - let mut request = request_result.map_err(|error| { - string_to_io_error(format!("Failed to build request: {}", sanitize_sensitive_info(error.to_string().as_str()))) - })?; - apply_attempt_to_request(&mut request, target)?; - apply_request_fetch_options(&mut request, options); - Ok((base_client, request)) -} - #[derive(Debug, Clone, Copy, Default)] pub struct RecordingTaskOptions { pub max_bytes: Option, @@ -480,54 +94,6 @@ pub struct InputEpgFileRequest<'a> { pub max_bytes: Option, } -fn log_proxy_diagnostics(config: &Config) { - PROXY_DIAGNOSTICS_ONCE.call_once(|| { - if let Some(proxy_cfg) = config.proxy.as_ref() { - let sanitized_url = sanitize_sensitive_info(proxy_cfg.url.as_str()); - let has_inline_credentials = proxy_cfg - .url - .contains('@') - || proxy_cfg.url.contains("://") - && proxy_cfg - .url - .split("://") - .nth(1) - .is_some_and(|part| part.contains('@')); - let has_explicit_credentials = - proxy_cfg.username.as_ref().is_some() || proxy_cfg.password.as_ref().is_some(); - debug!( - "Proxy config enabled: url={sanitized_url}, credentials_inline={has_inline_credentials}, credentials_fields={has_explicit_credentials}" - ); - } else { - debug!("Proxy config disabled (config.yml)"); - } - - let env_keys = [ - "HTTP_PROXY", - "HTTPS_PROXY", - "ALL_PROXY", - "NO_PROXY", - "http_proxy", - "https_proxy", - "all_proxy", - "no_proxy", - ]; - let mut env_values = Vec::new(); - for key in env_keys { - if let Ok(value) = std::env::var(key) { - if !value.trim().is_empty() { - env_values.push((key, sanitize_sensitive_info(value.as_str()).to_string())); - } - } - } - if env_values.is_empty() { - debug!("Proxy env vars not set"); - } else { - debug!("Proxy env vars present: {env_values:?}"); - } - }); -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum MimeCategory { Unknown, @@ -540,1772 +106,12 @@ pub enum MimeCategory { Unclassified, } -pub fn classify_content_type(headers: &[(String, String)]) -> MimeCategory { - headers.iter().find_map(|(k, v)| (k == axum::http::header::CONTENT_TYPE.as_str()).then_some(v)).map_or( - MimeCategory::Unknown, - |v| match v.to_lowercase().as_str() { - v if v.starts_with("video/") || v == "application/octet-stream" => MimeCategory::Video, - v if v.contains("mpegurl") => MimeCategory::M3U8, - v if v.starts_with("image/") => MimeCategory::Image, - v if v.starts_with(CONTENT_TYPE_JSON) || v.ends_with("+json") => MimeCategory::Json, - v if v.starts_with("application/xml") || v.ends_with("+xml") || v == "text/xml" => MimeCategory::Xml, - v if v.starts_with("text/") => MimeCategory::Text, - _ => MimeCategory::Unclassified, - }, - ) -} - -pub fn format_http_status(status: StatusCode) -> String { - let code = status.as_u16(); - match status.canonical_reason() { - Some(reason) => format!("{code} {reason}"), - None => code.to_string(), - } -} - -pub fn content_type_from_ext(ext: &str) -> &'static str { - match ext.to_ascii_lowercase().as_str() { - "mp4" | "fmp4" | "m4s" | "m4v" | "cmfv" => "video/mp4", - "m4a" | "cmfa" => "audio/mp4", - "mkv" => "video/x-matroska", - "avi" => "video/x-msvideo", - "mov" => "video/quicktime", - "webm" => "video/webm", - "ts" => "video/mp2t", - _ => "application/octet-stream", - } -} - -fn resolve_provider_url_for_attempt( - url: &Url, - provider: Option<&Arc>, - provider_url_index: usize, -) -> Url { - let Some(provider) = provider else { - return url.clone(); - }; - - match resolve_provider_scheme_url_with_provider_index(url.as_str(), Some(provider.clone()), provider_url_index) { - Ok((_provider, resolved)) => { - if resolved.as_ref() == url.as_str() { - return url.clone(); - } - Url::parse(resolved.as_ref()).unwrap_or_else(|_| url.clone()) - } - Err(err) => { - debug!("Failed to resolve provider URL: {err}"); - url.clone() - } - } -} - -#[derive(Debug, Clone)] -struct AttemptTarget { - request_url: Url, - effective_url: Url, - host_header: Option, - sni_host: Option, - connect_ip: Option, - dns_host: Option, -} - -impl AttemptTarget { - fn new(url: Url) -> Self { - Self { - request_url: url.clone(), - effective_url: url, - host_header: None, - sni_host: None, - connect_ip: None, - dns_host: None, - } - } -} - -fn is_ip_literal(host: &str) -> bool { host.parse::().is_ok() } - -fn format_host_header_with_port(host: &str, port: Option) -> String { - match port { - Some(port) => format!("{host}:{port}"), - None => host.to_string(), - } -} - -fn format_ip_host_header_with_port(ip: IpAddr, port: Option) -> String { - match (ip, port) { - (IpAddr::V4(addr), Some(port)) => format!("{addr}:{port}"), - (IpAddr::V4(addr), None) => addr.to_string(), - (IpAddr::V6(addr), Some(port)) => format!("[{addr}]:{port}"), - (IpAddr::V6(addr), None) => format!("[{addr}]"), - } -} - -fn resolve_attempt_target_with_dns_mode( - url: &Url, - provider: Option<&Arc>, - preview_dns_selection: bool, - provider_url_index: usize, -) -> AttemptTarget { - let resolved_url = resolve_provider_url_for_attempt(url, provider, provider_url_index); - let Some(provider) = provider else { - return AttemptTarget::new(resolved_url); - }; - - let mut target = AttemptTarget::new(resolved_url.clone()); - let scheme = resolved_url.scheme(); - if !provider.dns_enabled_for_scheme(scheme) { - return target; - } - - let Some(host) = resolved_url.host_str() else { - return target; - }; - if is_ip_literal(host) { - return target; - } - - let connect_ip = - if preview_dns_selection { provider.preview_ip_for_host(host) } else { provider.select_ip_for_host(host) }; - let Some(connect_ip) = connect_ip else { - return target; - }; - let keep_vhost = provider.get_dns_config().is_some_and(|dns| dns.keep_vhost); - let host_header = if keep_vhost { - format_host_header_with_port(host, resolved_url.port()) - } else { - format_ip_host_header_with_port(connect_ip, resolved_url.port()) - }; - - target.host_header = Some(host_header); - target.connect_ip = Some(connect_ip); - target.dns_host = Some(host.to_ascii_lowercase()); - - if scheme.eq_ignore_ascii_case("https") { - target.sni_host = Some(host.to_string()); - return target; - } - - if scheme.eq_ignore_ascii_case("http") { - let mut effective = resolved_url.clone(); - if effective.set_ip_host(connect_ip).is_ok() { - target.effective_url = effective; - } - } - - target -} - -#[cfg(test)] -fn resolve_attempt_target(url: &Url, provider: Option<&Arc>) -> AttemptTarget { - resolve_attempt_target_with_dns_mode(url, provider, false, 0) -} - -fn resolve_attempt_target_at_provider_index( - url: &Url, - provider: Option<&Arc>, - provider_url_index: usize, -) -> AttemptTarget { - resolve_attempt_target_with_dns_mode(url, provider, false, provider_url_index) -} - -fn preview_attempt_target(url: &Url, provider: Option<&Arc>) -> AttemptTarget { - resolve_attempt_target_with_dns_mode(url, provider, true, provider_start_index(provider)) -} - -fn provider_start_index(provider: Option<&Arc>) -> usize { - provider.map_or(0, |provider| match provider.provider_url_selection_policy() { - ProviderUrlSelectionPolicy::ResumeLastWorking => provider.get_current_index(), - ProviderUrlSelectionPolicy::RestartFromFirst => 0, - }) -} - -fn next_provider_url_index(current_index: usize, provider_url_count: usize, start_index: usize) -> Option { - if provider_url_count <= 1 { - return None; - } - - let next_index = (current_index + 1) % provider_url_count; - (next_index != start_index).then_some(next_index) -} - -fn provider_cycle_exhausted(provider: &ConfigProvider, current_index: usize, start_index: usize) -> bool { - next_provider_url_index(current_index, provider.urls.len(), start_index).is_none() -} - -fn log_provider_cycle_exhausted( - provider: &ConfigProvider, - start_index: usize, - current_index: usize, - last_failure: &str, -) { - error!( - "Provider '{}' exhausted all {} URL(s) after one full cycle starting at preferred index {} and ending at index {}: {}", - provider.name, - provider.urls.len(), - start_index, - current_index, - sanitize_sensitive_info(last_failure) - ); -} - -fn rotate_to_next_provider_url( - provider: &ConfigProvider, - provider_url_index: &mut usize, - start_provider_index: usize, - reason: &str, -) -> bool { - let Some(next_index) = next_provider_url_index(*provider_url_index, provider.urls.len(), start_provider_index) - else { - return false; - }; - - warn!( - "Provider '{}' failover: {} -> switching from URL index {} to {}", - provider.name, - sanitize_sensitive_info(reason), - *provider_url_index, - next_index - ); - *provider_url_index = next_index; - true -} - -fn format_request_target_for_logging(target: &AttemptTarget) -> String { - if target.effective_url.scheme().eq_ignore_ascii_case("https") { - if let Some(connect_ip) = target.connect_ip { - format!("{} (connect_ip={connect_ip})", target.request_url) - } else { - target.request_url.to_string() - } - } else { - target.effective_url.to_string() - } -} - -pub fn preview_request_target_for_logging(url: &Url, provider: Option<&Arc>) -> String { - let target = preview_attempt_target(url, provider); - format_request_target_for_logging(&target) -} - -pub fn preview_request_diagnostics_for_logging(url: &Url, provider: Option<&Arc>) -> String { - let target = preview_attempt_target(url, provider); - let mut parts = vec![ - format!("request_url={}", sanitize_sensitive_info(target.request_url.as_str())), - format!("effective_url={}", sanitize_sensitive_info(target.effective_url.as_str())), - ]; - - if let Some(host_header) = target.host_header.as_ref() { - parts.push(format!("host_header={}", sanitize_sensitive_info(host_header))); - } - if let Some(connect_ip) = target.connect_ip { - parts.push(format!("connect_ip={}", sanitize_sensitive_info(&connect_ip.to_string()))); - } - if let Some(sni_host) = target.sni_host.as_ref() { - parts.push(format!("sni_host={}", sanitize_sensitive_info(sni_host))); - } - - parts.join(", ") -} - -fn should_try_next_ip_on_connect_error( - provider: Option<&Arc>, - target: &AttemptTarget, - attempted_ips: &mut HashSet, -) -> bool { - let Some(provider) = provider else { - return false; - }; - let Some(connect_ip) = target.connect_ip else { - return false; - }; - let Some(dns_host) = target.dns_host.as_ref() else { - return false; - }; - let Some(dns_cfg) = provider.get_dns_config() else { - return false; - }; - if dns_cfg.on_connect_error != OnConnectErrorPolicy::TryNextIp { - return false; - } - - let inserted = attempted_ips.insert(connect_ip); - if !inserted { - return false; - } - - let total_ips = provider.ip_count_for_host(dns_host); - total_ips > attempted_ips.len() -} - -fn apply_attempt_to_request(request: &mut reqwest::Request, target: &AttemptTarget) -> Result<(), std::io::Error> { - if request.url().as_str() != target.effective_url.as_str() { - *request.url_mut() = target.effective_url.clone(); - } - if let Some(host_header) = target.host_header.as_ref() { - let host = HeaderValue::from_str(host_header) - .map_err(|err| string_to_io_error(format!("Invalid host header '{host_header}': {err}")))?; - request.headers_mut().insert(HOST, host); - } - Ok(()) -} - -fn build_https_attempt_client( - app_config: &Arc, - sni_host: &str, - connect_ip: IpAddr, - connect_port: u16, -) -> Result { - let config = app_config.config.load(); - let mut builder = create_client(app_config).http1_only(); - if config.connect_timeout_secs > 0 { - builder = builder.connect_timeout(Duration::from_secs(u64::from(config.connect_timeout_secs))); - } - drop(config); - builder = builder.resolve_to_addrs(sni_host, &[SocketAddr::new(connect_ip, connect_port)]); - builder.build() -} - -async fn execute_attempt_request( - app_config: &Arc, - base_client: reqwest::Client, - request: reqwest::Request, - target: &AttemptTarget, -) -> Result { - if target.effective_url.scheme().eq_ignore_ascii_case("https") { - if let (Some(sni_host), Some(connect_ip)) = (target.sni_host.as_ref(), target.connect_ip) { - let connect_port = target.effective_url.port_or_known_default().unwrap_or(443); - let https_client = build_https_attempt_client(app_config, sni_host.as_str(), connect_ip, connect_port)?; - return https_client.execute(request).await; - } - } - base_client.execute(request).await -} - /// Response returned after applying provider URL failover without applying the generic resource retry policy. pub struct ProviderFailoverResponse { pub response: reqwest::Response, pub provider_url_index: Option, } -#[allow(clippy::too_many_lines)] -async fn send_with_provider_failover_only_with_options( - app_config: &Arc, - url: &Url, - provider: Option<&Arc>, - allow_redirects: bool, - options: RequestFetchOptions, - mut send: impl FnMut(&Url) -> reqwest::RequestBuilder, -) -> Result { - let failover_patterns = app_config.config.load().reverse_proxy.as_ref().map_or_else( - || ResourceRetryConfig::default().failover_redirect_patterns, - |rp| rp.resource_retry.failover_redirect_patterns.clone(), - ); - - let start_provider_index = provider_start_index(provider); - let mut provider_url_index = start_provider_index; - let idle_timeout = options.attempt_idle_timeout_or_default(); - let idle = sleep(idle_timeout); - tokio::pin!(idle); - - 'provider_loop: loop { - let mut attempted_dns_ips = HashSet::new(); - - 'ip_loop: loop { - let attempt_target = resolve_attempt_target_at_provider_index(url, provider, provider_url_index); - if log_enabled!(Level::Debug) { - if let Some(current_provider) = provider { - let attempt_target_log = format_request_target_for_logging(&attempt_target); - debug!( - "Provider '{}' acquiring URL index {} of {}: {}", - current_provider.name, - provider_url_index, - current_provider.urls.len(), - sanitize_sensitive_info(attempt_target_log.as_str()) - ); - } - } - - idle.as_mut().reset(tokio::time::Instant::now() + idle_timeout); - let (base_client, request) = - prepare_physical_request_attempt(send(&attempt_target.request_url), &attempt_target, options)?; - - tokio::select! { - () = &mut idle => { - if should_try_next_ip_on_connect_error(provider, &attempt_target, &mut attempted_dns_ips) { - continue 'ip_loop; - } - - let last_provider_failure = format!( - "idle timeout while trying {}", - sanitize_sensitive_info(attempt_target.request_url.as_str()) - ); - if let Some(current_provider) = provider { - if rotate_to_next_provider_url( - current_provider.as_ref(), - &mut provider_url_index, - start_provider_index, - "idle timeout", - ) { - continue 'provider_loop; - } - log_provider_cycle_exhausted( - current_provider.as_ref(), - start_provider_index, - provider_url_index, - &last_provider_failure, - ); - } - - return Err(Error::new( - ErrorKind::TimedOut, - format!("Request timed out: {}", sanitize_sensitive_info(url.as_str())), - )); - } - result = execute_attempt_request(app_config, base_client, request, &attempt_target) => match result { - Ok(response) => { - let status = response.status(); - if allow_redirects && status.is_redirection() { - if let Some(current_provider) = provider { - current_provider.set_current_index(provider_url_index); - } - return Ok(ProviderFailoverResponse { - response, - provider_url_index: provider.map(|_| provider_url_index), - }); - } - - let is_failover = is_failover_redirect(response.url(), &failover_patterns); - if !is_failover && !should_trigger_failover(status) { - if status.is_success() { - if let Some(current_provider) = provider { - current_provider.set_current_index(provider_url_index); - } - } - return Ok(ProviderFailoverResponse { - response, - provider_url_index: provider.map(|_| provider_url_index), - }); - } - - let last_provider_failure = format!( - "status {} while trying {}", - format_http_status(status), - sanitize_sensitive_info(attempt_target.request_url.as_str()) - ); - - if let Some(current_provider) = provider { - let reason = format!("status {}", format_http_status(status)); - if rotate_to_next_provider_url( - current_provider.as_ref(), - &mut provider_url_index, - start_provider_index, - reason.as_str(), - ) { - continue 'provider_loop; - } - log_provider_cycle_exhausted( - current_provider.as_ref(), - start_provider_index, - provider_url_index, - &last_provider_failure, - ); - } - - return Ok(ProviderFailoverResponse { - response, - provider_url_index: provider.map(|_| provider_url_index), - }); - }, - Err(err) => { - if (err.is_timeout() || err.is_connect()) - && should_try_next_ip_on_connect_error(provider, &attempt_target, &mut attempted_dns_ips) - { - continue 'ip_loop; - } - - let last_provider_failure = format!( - "connection error while trying {}: {}", - sanitize_sensitive_info(attempt_target.request_url.as_str()), - sanitize_sensitive_info(err.to_string().as_str()) - ); - - if err.is_timeout() || err.is_connect() { - if let Some(current_provider) = provider { - if rotate_to_next_provider_url( - current_provider.as_ref(), - &mut provider_url_index, - start_provider_index, - "connection error", - ) { - continue 'provider_loop; - } - log_provider_cycle_exhausted( - current_provider.as_ref(), - start_provider_index, - provider_url_index, - &last_provider_failure, - ); - } - } - - let message = format!("Request error: {}", sanitize_sensitive_info(err.to_string().as_str())); - return Err(if err.is_timeout() { - Error::new(ErrorKind::TimedOut, message) - } else if err.is_connect() { - Error::new(ErrorKind::ConnectionRefused, message) - } else { - string_to_io_error(message) - }); - }, - } - } - } - } -} - -#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss, clippy::cast_precision_loss)] -pub fn calculate_retry_backoff(base_delay_ms: u64, multiplier: f64, attempt: u32) -> u64 { - let base = base_delay_ms.max(1); - if multiplier <= 1.0 { - return base; - } - let delay = (base as f64) * multiplier.powi(i32::try_from(attempt).unwrap_or(i32::MAX)); - if !delay.is_finite() || delay < 1.0 { - base - } else if delay >= u64::MAX as f64 { - u64::MAX - } else { - delay as u64 - } -} - -/// Sends a request with retry logic and optional provider failover support. -pub async fn send_with_retry_and_provider( - app_config: &Arc, - url: &Url, // Used primarily for logging/context - provider: Option<&Arc>, - allow_redirects: bool, - send: impl FnMut(&Url) -> reqwest::RequestBuilder, -) -> Result { - send_with_retry_and_provider_policy_with_options( - app_config, - url, - provider, - allow_redirects, - true, - RequestFetchOptions::default(), - send, - ) - .await -} - -/// Canonical retry and provider-failover entry point for outbound resource requests. -/// -/// `send_with_retry_and_provider` is a thin wrapper that enables the standard retry policy. Retry attempt counts, -/// backoff values, and failover redirect patterns are sourced from `AppConfig` (`reverse_proxy.resource_retry`). The -/// `url` argument is used as the stable logging/context URL; callers should pass the original request target rather -/// than an already-rotated provider URL. -/// -/// When `retry_enabled` is `false`, this function forces `max_attempts` to 1, disables provider URL rotation for idle -/// timeouts, retryable HTTP statuses, and connection/timeout errors, and skips the final fallback provider rotation -/// after attempts are exhausted. -#[allow(clippy::too_many_lines)] -pub async fn send_with_retry_and_provider_policy( - app_config: &Arc, - url: &Url, // Used primarily for logging/context - provider: Option<&Arc>, - allow_redirects: bool, - retry_enabled: bool, - send: impl FnMut(&Url) -> reqwest::RequestBuilder, -) -> Result { - send_with_retry_and_provider_policy_with_options( - app_config, - url, - provider, - allow_redirects, - retry_enabled, - RequestFetchOptions::default(), - send, - ) - .await -} - -#[allow(clippy::too_many_lines)] -pub async fn send_with_retry_and_provider_policy_with_options( - app_config: &Arc, - url: &Url, // Used primarily for logging/context - provider: Option<&Arc>, - allow_redirects: bool, - retry_enabled: bool, - options: RequestFetchOptions, - send: impl FnMut(&Url) -> reqwest::RequestBuilder, -) -> Result { - send_with_retry_and_provider_policy_with_options_result( - app_config, - url, - provider, - allow_redirects, - retry_enabled, - options, - send, - ) - .await - .map(|result| result.response) -} - -#[allow(clippy::too_many_lines)] -async fn send_with_retry_and_provider_policy_with_options_result( - app_config: &Arc, - url: &Url, // Used primarily for logging/context - provider: Option<&Arc>, - allow_redirects: bool, - retry_enabled: bool, - options: RequestFetchOptions, - mut send: impl FnMut(&Url) -> reqwest::RequestBuilder, -) -> Result { - if options.uses_provider_failover_only() { - return send_with_provider_failover_only_with_options( - app_config, - url, - provider, - allow_redirects, - options, - send, - ) - .await; - } - - let config = app_config.config.load(); - let (max_attempts, backoff_ms, backoff_multiplier, failover_patterns) = config.reverse_proxy.as_ref().map_or_else( - || { - let (a, b, c) = ResourceRetryConfig::get_default_retry_values(); - (a, b, c, ResourceRetryConfig::default().failover_redirect_patterns) - }, - |rp| { - let (a, b, c) = rp.resource_retry.get_retry_values(); - (a, b, c, rp.resource_retry.failover_redirect_patterns.clone()) - }, - ); - let max_attempts = if retry_enabled { max_attempts } else { 1 }; - drop(config); - - let idle_timeout = options.attempt_idle_timeout_or_default(); - let idle = sleep(idle_timeout); - tokio::pin!(idle); - - let max_provider_attempts = provider.as_ref().map_or(0, |p| p.urls.len()); - let start_provider_index = provider_start_index(provider); - let mut provider_url_index = start_provider_index; - let mut last_provider_failure: Option = None; - - 'provider_loop: loop { - // 2. Retry loop for the current URL - 'attempt_loop: for attempt in 0..max_attempts { - let mut attempted_dns_ips = HashSet::new(); - - 'ip_loop: loop { - let attempt_target = resolve_attempt_target_at_provider_index(url, provider, provider_url_index); - if log_enabled!(Level::Debug) { - if let Some(current_provider) = provider { - let attempt_target_log = format_request_target_for_logging(&attempt_target); - debug!( - "Provider '{}' attempting URL index {} of {}: {}", - current_provider.name, - provider_url_index, - max_provider_attempts, - sanitize_sensitive_info(attempt_target_log.as_str()) - ); - } - } - // Reset the idle timer for a new attempt - idle.as_mut().reset(tokio::time::Instant::now() + idle_timeout); - - let (base_client, request) = - prepare_physical_request_attempt(send(&attempt_target.request_url), &attempt_target, options)?; - - tokio::select! { - () = &mut idle => { - warn!("Request idle for too long: {}", sanitize_sensitive_info(url.as_str())); - last_provider_failure = Some(format!( - "idle timeout while trying {}", - sanitize_sensitive_info(attempt_target.request_url.as_str()) - )); - // 1. Try Provider Failover first - let mut provider_failover_exhausted = false; - if retry_enabled { - if let Some(current_provider) = provider { - if rotate_to_next_provider_url( - current_provider.as_ref(), - &mut provider_url_index, - start_provider_index, - "idle timeout", - ) { - continue 'provider_loop; - } - provider_failover_exhausted = - max_provider_attempts > 0 && provider_cycle_exhausted(current_provider.as_ref(), provider_url_index, start_provider_index); - } - } - - // 2. If no provider or rotation failed, check if we can retry the same URL - if attempt < max_attempts - 1 { - let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); - warn!("Idle timeout, retrying same URL in {}ms (attempt {})", delay, attempt + 1); - tokio::time::sleep(Duration::from_millis(delay)).await; - continue 'attempt_loop; - } - - if provider_failover_exhausted { - if let Some(current_provider) = provider { - log_provider_cycle_exhausted( - current_provider.as_ref(), - start_provider_index, - provider_url_index, - last_provider_failure.as_deref().unwrap_or("idle timeout"), - ); - } - } - - return Err(Error::new( - ErrorKind::TimedOut, - format!( - "Request timed out and no retries left: {}", - sanitize_sensitive_info(url.as_str()) - ), - )); - } - - result = execute_attempt_request(app_config, base_client, request, &attempt_target) => { - match result { - Ok(response) => { - let status = response.status(); - if allow_redirects && status.is_redirection() { - if let Some(current_provider) = provider { - current_provider.set_current_index(provider_url_index); - } - return Ok(ProviderFailoverResponse { - response, - provider_url_index: provider.map(|_| provider_url_index), - }); - } - let is_failover = is_failover_redirect(response.url(), &failover_patterns); - if !is_failover && status.is_success() { - if let Some(current_provider) = provider { - current_provider.set_current_index(provider_url_index); - } - return Ok(ProviderFailoverResponse { - response, - provider_url_index: provider.map(|_| provider_url_index), - }); - } - - last_provider_failure = Some(format!( - "status {} while trying {}", - format_http_status(status), - sanitize_sensitive_info(attempt_target.request_url.as_str()) - )); - - // Failover check: Should we switch to the next provider URL? - let provider_failover_exhausted = retry_enabled - && (is_failover || should_trigger_failover(status)) - && provider.is_some_and(|current_provider| { - provider_cycle_exhausted(current_provider.as_ref(), provider_url_index, start_provider_index) - }); - if retry_enabled && (is_failover || should_trigger_failover(status)) { - if let Some(current_provider) = provider { - let reason = format!("status {}", format_http_status(status)); - if rotate_to_next_provider_url( - current_provider.as_ref(), - &mut provider_url_index, - start_provider_index, - reason.as_str(), - ) { - continue 'provider_loop; - } - } - } - - // Standard retry check for the same URL - let is_retryable = status.is_server_error() - || matches!(status, StatusCode::TOO_MANY_REQUESTS | StatusCode::REQUEST_TIMEOUT); - - if attempt < max_attempts - 1 && is_retryable { - perform_backoff(attempt, backoff_ms, backoff_multiplier, &response).await; - continue 'attempt_loop; - } - - if provider_failover_exhausted { - if let Some(current_provider) = provider { - log_provider_cycle_exhausted( - current_provider.as_ref(), - start_provider_index, - provider_url_index, - last_provider_failure.as_deref().unwrap_or("request failed"), - ); - } - } - - if options.return_http_errors && (status.is_client_error() || status.is_server_error()) { - return Ok(ProviderFailoverResponse { - response, - provider_url_index: provider.map(|_| provider_url_index), - }); - } - - return Err(string_to_io_error(format!("Request failed ({}): {}", - format_http_status(status), sanitize_sensitive_info(url.as_str())))); - } - - Err(err) => { - // For DNS IP-connect policy, attempt next IP before provider URL rotation. - if retry_enabled - && (err.is_timeout() || err.is_connect()) - && should_try_next_ip_on_connect_error(provider, &attempt_target, &mut attempted_dns_ips) - { - continue 'ip_loop; - } - - last_provider_failure = Some(format!( - "connection error while trying {}: {}", - sanitize_sensitive_info(attempt_target.request_url.as_str()), - sanitize_sensitive_info(&err.to_string()) - )); - - // Connection errors (Timeout/Connect) trigger failover if provider exists - let provider_failover_exhausted = retry_enabled - && (err.is_timeout() || err.is_connect()) - && provider.is_some_and(|current_provider| { - provider_cycle_exhausted(current_provider.as_ref(), provider_url_index, start_provider_index) - }); - if retry_enabled && (err.is_timeout() || err.is_connect()) { - if let Some(current_provider) = provider { - if rotate_to_next_provider_url( - current_provider.as_ref(), - &mut provider_url_index, - start_provider_index, - "connection error", - ) { - continue 'provider_loop; - } - } - } - - // If not a provider or rotation failed, try standard retry - if (err.is_timeout() || err.is_connect()) && attempt < max_attempts - 1 { - let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); - tokio::time::sleep(Duration::from_millis(delay)).await; - continue 'attempt_loop; - } - - if provider_failover_exhausted { - if let Some(current_provider) = provider { - log_provider_cycle_exhausted( - current_provider.as_ref(), - start_provider_index, - provider_url_index, - last_provider_failure.as_deref().unwrap_or("request error"), - ); - } - } - - let error_message = format!( - "Request error: {}", - sanitize_sensitive_info(&err.to_string()) - ); - return Err(if err.is_timeout() { - Error::new(ErrorKind::TimedOut, error_message) - } else { - string_to_io_error(error_message) - }); - } - } - } - } - } - } - - // 2. If per-URL retries are exhausted, try next provider URL as a last resort - if retry_enabled { - if let Some(current_provider) = provider { - if rotate_to_next_provider_url( - current_provider.as_ref(), - &mut provider_url_index, - start_provider_index, - "retries exhausted for current URL", - ) { - continue 'provider_loop; - } - - if max_provider_attempts > 0 { - let last_failure = - last_provider_failure.as_deref().unwrap_or("all attempts and providers exhausted"); - log_provider_cycle_exhausted( - current_provider.as_ref(), - start_provider_index, - provider_url_index, - last_failure, - ); - } - } - } - - break; - } - - Err(string_to_io_error("All attempts and providers exhausted")) -} - -fn prepare_input_request_headers( - app_config: &Arc, - input: &InputSource, - headers: Option<&HeaderMap>, -) -> (HashMap, Option) { - let custom_headers = headers - .map(|h| h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::>()); - - let config = app_config.config.load(); - let default_user_agent = config.default_user_agent.clone(); - let disabled_headers = config.get_disabled_headers(); - drop(config); - - let merged = get_request_headers( - Some(&input.headers), - custom_headers.as_ref(), - disabled_headers.as_ref(), - default_user_agent.as_deref(), - ); - - let request_headers: HashMap = merged - .iter() - .map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())) - .collect(); - - (request_headers, default_user_agent) -} - -#[allow(clippy::implicit_hasher)] -pub async fn send_input_with_retry_and_provider_policy_with_options_result( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - url: &Url, - options: RequestFetchOptions, -) -> Result { - let (request_headers, default_user_agent) = prepare_input_request_headers(app_config, input, headers); - send_with_retry_and_provider_policy_with_options_result( - app_config, - url, - input.get_provider(), - false, - true, - options, - |resolved_url| { - get_client_request( - client, - input.method, - Some(&request_headers), - resolved_url, - None, - None, - default_user_agent.as_deref(), - ) - }, - ) - .await -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines, clippy::implicit_hasher)] -pub async fn send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - url: &Url, - max_redirects: usize, - options: RequestFetchOptions, -) -> Result { - let config = app_config.config.load(); - let (configured_max_attempts, backoff_ms, backoff_multiplier, failover_patterns) = - config.reverse_proxy.as_ref().map_or_else( - || { - let (a, b, c) = ResourceRetryConfig::get_default_retry_values(); - (a, b, c, ResourceRetryConfig::default().failover_redirect_patterns) - }, - |rp| { - let (a, b, c) = rp.resource_retry.get_retry_values(); - (a, b, c, rp.resource_retry.failover_redirect_patterns.clone()) - }, - ); - drop(config); - let provider_failover_only = options.uses_provider_failover_only(); - let max_attempts = if provider_failover_only { 1 } else { configured_max_attempts }; - - let (base_headers, default_user_agent) = prepare_input_request_headers(app_config, input, headers); - let provider = input.get_provider(); - let max_provider_attempts = provider.as_ref().map_or(0, |p| p.urls.len()); - let start_provider_index = provider_start_index(provider); - let mut provider_url_index = start_provider_index; - let mut last_provider_failure: Option = None; - let idle_timeout = options.attempt_idle_timeout_or_default(); - let idle = sleep(idle_timeout); - tokio::pin!(idle); - - 'provider_loop: loop { - 'attempt_loop: for attempt in 0..max_attempts { - let mut current_url = url.clone(); - let mut current_headers = base_headers.clone(); - let mut remaining_redirects = max_redirects; - let mut attempted_dns_ips = HashSet::new(); - - 'redirect_loop: loop { - let attempt_target = - resolve_attempt_target_at_provider_index(¤t_url, provider, provider_url_index); - if log_enabled!(Level::Debug) { - if let Some(current_provider) = provider { - let attempt_target_log = format_request_target_for_logging(&attempt_target); - debug!( - "Provider '{}' attempting URL index {} of {}: {}", - current_provider.name, - provider_url_index, - max_provider_attempts, - sanitize_sensitive_info(attempt_target_log.as_str()) - ); - } - } - idle.as_mut().reset(tokio::time::Instant::now() + idle_timeout); - - let request_builder = get_client_request( - client, - input.method, - Some(¤t_headers), - &attempt_target.request_url, - None, - None, - default_user_agent.as_deref(), - ); - let (base_client, request) = - prepare_physical_request_attempt(request_builder, &attempt_target, options)?; - - tokio::select! { - () = &mut idle => { - warn!("Request idle for too long: {}", sanitize_sensitive_info(url.as_str())); - last_provider_failure = Some(format!( - "idle timeout while trying {}", - sanitize_sensitive_info(attempt_target.request_url.as_str()) - )); - if let Some(current_provider) = provider { - if rotate_to_next_provider_url( - current_provider.as_ref(), - &mut provider_url_index, - start_provider_index, - "idle timeout", - ) { - continue 'provider_loop; - } - if max_provider_attempts > 0 { - log_provider_cycle_exhausted( - current_provider.as_ref(), - start_provider_index, - provider_url_index, - last_provider_failure.as_deref().unwrap_or("idle timeout"), - ); - } - } - - if attempt < max_attempts - 1 { - let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); - warn!("Idle timeout, retrying same URL in {}ms (attempt {})", delay, attempt + 1); - tokio::time::sleep(Duration::from_millis(delay)).await; - continue 'attempt_loop; - } - - return Err(Error::new( - ErrorKind::TimedOut, - format!( - "Request timed out and no retries left: {}", - sanitize_sensitive_info(url.as_str()) - ), - )); - } - - result = execute_attempt_request(app_config, base_client, request, &attempt_target) => { - match result { - Ok(response) => { - if response.status().is_redirection() { - if remaining_redirects == 0 { - return Err(string_to_io_error(format!( - "Too many redirects while requesting {}", - sanitize_sensitive_info(url.as_str()) - ))); - } - - let response_base_url = response.url().clone(); - let Some(location) = response.headers().get(reqwest::header::LOCATION) else { - return Err(string_to_io_error(format!( - "Redirect response missing location header for {}", - sanitize_sensitive_info(current_url.as_str()) - ))); - }; - let Ok(location_str) = location.to_str() else { - return Err(string_to_io_error(format!( - "Redirect response contains invalid location header for {}", - sanitize_sensitive_info(current_url.as_str()) - ))); - }; - let next_url = response_base_url - .join(location_str) - .or_else(|_| Url::parse(location_str)) - .map_err(|_| { - string_to_io_error(format!( - "Redirect response contains invalid location URL for {}", - sanitize_sensitive_info(current_url.as_str()) - )) - })?; - - if !same_origin(&response_base_url, &next_url) { - strip_sensitive_headers_for_cross_origin_redirect(&mut current_headers); - } - current_url = next_url; - remaining_redirects = remaining_redirects.saturating_sub(1); - continue 'redirect_loop; - } - - let status = response.status(); - let is_failover = is_failover_redirect(response.url(), &failover_patterns); - if !is_failover && status.is_success() { - if let Some(current_provider) = provider { - current_provider.set_current_index(provider_url_index); - } - return Ok(ProviderFailoverResponse { - response, - provider_url_index: provider.map(|_| provider_url_index), - }); - } - - last_provider_failure = Some(format!( - "status {} while trying {}", - format_http_status(status), - sanitize_sensitive_info(attempt_target.request_url.as_str()) - )); - - let provider_failover_exhausted = (is_failover || should_trigger_failover(status)) - && provider.is_some_and(|current_provider| { - provider_cycle_exhausted(current_provider.as_ref(), provider_url_index, start_provider_index) - }); - if is_failover || should_trigger_failover(status) { - if let Some(current_provider) = provider { - let reason = format!("status {}", format_http_status(status)); - if rotate_to_next_provider_url( - current_provider.as_ref(), - &mut provider_url_index, - start_provider_index, - reason.as_str(), - ) { - continue 'provider_loop; - } - } - } - - let is_retryable = status.is_server_error() - || matches!(status, StatusCode::TOO_MANY_REQUESTS | StatusCode::REQUEST_TIMEOUT); - if attempt < max_attempts - 1 && is_retryable { - perform_backoff(attempt, backoff_ms, backoff_multiplier, &response).await; - continue 'attempt_loop; - } - - if provider_failover_exhausted { - if let Some(current_provider) = provider { - log_provider_cycle_exhausted( - current_provider.as_ref(), - start_provider_index, - provider_url_index, - last_provider_failure.as_deref().unwrap_or("request failed"), - ); - } - } - - if provider_failover_only - || (options.return_http_errors && (status.is_client_error() || status.is_server_error())) - { - return Ok(ProviderFailoverResponse { - response, - provider_url_index: provider.map(|_| provider_url_index), - }); - } - - return Err(string_to_io_error(format!( - "Request failed ({}): {}", - format_http_status(status), - sanitize_sensitive_info(url.as_str()) - ))); - } - Err(err) => { - if (err.is_timeout() || err.is_connect()) - && should_try_next_ip_on_connect_error(provider, &attempt_target, &mut attempted_dns_ips) - { - continue 'redirect_loop; - } - - last_provider_failure = Some(format!( - "connection error while trying {}: {}", - sanitize_sensitive_info(attempt_target.request_url.as_str()), - sanitize_sensitive_info(err.to_string().as_str()) - )); - - let provider_failover_exhausted = (err.is_timeout() || err.is_connect()) - && provider.is_some_and(|current_provider| { - provider_cycle_exhausted(current_provider.as_ref(), provider_url_index, start_provider_index) - }); - if err.is_timeout() || err.is_connect() { - if let Some(current_provider) = provider { - if rotate_to_next_provider_url( - current_provider.as_ref(), - &mut provider_url_index, - start_provider_index, - "connection error", - ) { - continue 'provider_loop; - } - } - } - - if (err.is_timeout() || err.is_connect()) && attempt < max_attempts - 1 { - let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); - tokio::time::sleep(Duration::from_millis(delay)).await; - continue 'attempt_loop; - } - - if provider_failover_exhausted { - if let Some(current_provider) = provider { - log_provider_cycle_exhausted( - current_provider.as_ref(), - start_provider_index, - provider_url_index, - last_provider_failure.as_deref().unwrap_or("request error"), - ); - } - } - - let error_message = format!( - "Request error: {}", - sanitize_sensitive_info(err.to_string().as_str()) - ); - return Err(if err.is_timeout() { - Error::new(ErrorKind::TimedOut, error_message) - } else if err.is_connect() { - Error::new(ErrorKind::ConnectionRefused, error_message) - } else { - string_to_io_error(error_message) - }); - } - } - } - } - } - } - - if let Some(current_provider) = provider { - if rotate_to_next_provider_url( - current_provider.as_ref(), - &mut provider_url_index, - start_provider_index, - "retries exhausted for current URL", - ) { - continue 'provider_loop; - } - - if max_provider_attempts > 0 { - let last_failure = last_provider_failure.as_deref().unwrap_or("all attempts and providers exhausted"); - log_provider_cycle_exhausted( - current_provider.as_ref(), - start_provider_index, - provider_url_index, - last_failure, - ); - } - } - - break; - } - - Err(string_to_io_error("All attempts and providers exhausted")) -} - -fn is_failover_redirect(url: &Url, patterns: &[Arc]) -> bool { - let redirect_url = url.as_str(); - patterns.iter().any(|pattern| pattern.is_match(redirect_url)) -} - -/// Helper to handle sleep duration for retries, respecting Retry-After headers -async fn perform_backoff(attempt: u32, ms: u64, mult: f64, response: &reqwest::Response) { - let wait_dur = response - .headers() - .get(reqwest::header::RETRY_AFTER) - .and_then(|h| h.to_str().ok()) - .and_then(|s| s.parse::().ok()) - .map_or_else(|| Duration::from_millis(calculate_retry_backoff(ms, mult, attempt)), Duration::from_secs); - - tokio::time::sleep(wait_dur).await; -} - -pub async fn get_input_epg_content_as_file( - app_config: &Arc, - client: &reqwest::Client, - input: &ConfigInput, - request: InputEpgFileRequest<'_>, -) -> Result { - let InputEpgFileRequest { headers, storage_dir, url: url_str, persist_path, max_bytes } = request; - debug_if_enabled!( - "getting input epg content storage_dir: {}, url: {}", - storage_dir, - sanitize_sensitive_info(url_str) - ); - - // This is the single write-lock boundary for EPG cache population. Callers must - // not hold a lock for `persist_path` while invoking this function. - let _persist_lock = app_config.file_locks.write_lock(persist_path).await; - - // On Windows, drive-letter paths also parse as URLs (with `c` as the scheme). - // Interpret an absolute platform path before attempting URL parsing. - if !Path::new(url_str).is_absolute() && url_str.parse::().is_ok() { - match download_epg_content_as_file(app_config, client, input, headers, url_str, persist_path, max_bytes).await { - Ok(content) => Ok(content), - Err(e) => { - error!( - "can't download input {} epg url: {} => {}", - input.name, - sanitize_sensitive_info(url_str), - sanitize_sensitive_info(&e.to_string()) - ); - Err(TuliproxError::RepositoryNetwork(format!( - "can't download input {} epg url: {} => {}", - input.name, - sanitize_sensitive_info(url_str), - sanitize_sensitive_info(&e.to_string()) - ))) - } - } - } else { - let Some(file_path) = get_file_path(storage_dir, Some(PathBuf::from(url_str))) else { - let msg = format!("can't read input url: {}", sanitize_sensitive_info(url_str)); - error!("{msg}"); - return Err(TuliproxError::RepositoryNetwork(msg)); - }; - if !file_path.exists() { - let msg = format!("can't read input url: {}", sanitize_sensitive_info(url_str)); - error!("{msg}"); - return Err(TuliproxError::RepositoryNetwork(msg)); - } - - copy_local_epg_file_to_persist(&file_path, persist_path, max_bytes).await.map_err(|err| { - error!("can't persist to: {} => {}", persist_path.display(), err); - TuliproxError::RepositoryNetwork(format!("Failed to persist: {} => {err}", persist_path.display())) - }) - } -} - -pub async fn get_input_text_content_as_stream( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - storage_dir: &str, - persist_filepath: Option, -) -> Result { - debug_if_enabled!( - "getting input text content storage_dir: {}, url: {}", - storage_dir, - sanitize_sensitive_info(&input.url) - ); - - if input.url.parse::().is_ok() { - match download_text_content_as_stream(app_config, client, input, persist_filepath).await { - Ok((content, _response_url)) => Ok(content), - Err(e) => { - error!("Failed to download input '{}': {}", input.name, sanitize_sensitive_info(&e.to_string())); - Err(TuliproxError::RepositoryNetwork(format!( - "Failed to download input '{}': {}", - input.name, - sanitize_sensitive_info(&e.to_string()) - ))) - } - } - } else { - let result = match get_file_path(storage_dir, Some(PathBuf::from(&input.url))) { - Some(filepath) => { - if filepath.exists() { - match get_local_file_content_as_stream(&filepath).await { - Ok(content) => { - if let Some(path) = persist_filepath { - let tee = tee_dyn_reader( - content, - &path, - Some(Arc::new(|size| { - debug_if_enabled!("Persisted {} bytes", human_readable_byte_size(size as u64)); - })), - ) - .await; - Some(tee) - } else { - Some(content) - } - } - Err(err) => { - return Err(TuliproxError::RepositoryNetwork(format!("Failed : {err}"))); - } - } - } else { - None - } - } - None => None, - }; - result.map_or_else( - || { - let msg = format!("can't read input url: {}", sanitize_sensitive_info(&input.url)); - error!("{msg}"); - Err(TuliproxError::RepositoryNetwork(msg)) - }, - Ok, - ) - } -} - -pub fn get_client_request( - client: &reqwest::Client, - method: InputFetchMethod, - headers: Option<&HashMap>, - url: &Url, - custom_headers: Option<&HashMap, S>>, - disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, - default_user_agent: Option<&str>, -) -> reqwest::RequestBuilder { - let request = match method { - InputFetchMethod::GET => client.get(url.clone()), - InputFetchMethod::POST => { - // let base_url = url[..url::Position::BeforePath].to_string() + url.path(); - let mut params: HashMap = HashMap::default(); - for (key, value) in url.query_pairs() { - params.insert(key.to_string(), value.to_string()); - } - // we could cut the params but we leave them as query and add them as form. - client.post(url.clone()).form(¶ms) - } - }; - let headers = get_request_headers(headers, custom_headers, disabled_headers, default_user_agent); - request.headers(headers) -} - -pub fn get_request_headers( - request_headers: Option<&HashMap>, - custom_headers: Option<&HashMap, S>>, - disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, - default_user_agent: Option<&str>, -) -> HeaderMap { - let mut headers = HeaderMap::default(); - let mut has_user_agent = false; - - // 1. First, we process the configured request headers (from input config). - // These should have the highest priority. - if let Some(req_headers) = request_headers { - for (key, value) in req_headers { - if let (Ok(key), Ok(value)) = - (HeaderName::from_bytes(key.as_bytes()), HeaderValue::from_bytes(value.as_bytes())) - { - if filter_request_header(key.as_str()) { - if disabled_headers.as_ref().is_some_and(|d| d.should_remove(key.as_str())) { - continue; - } - if key == axum::http::header::USER_AGENT { - has_user_agent = true; - } - headers.insert(key, value); - } - } - } - } - - // 2. Next, we process custom headers (from the client request). - // These are only added if they don't already exist in the headers map (i.e., not overridden by config). - if let Some(custom) = custom_headers { - for (key, value) in custom { - let key_lc = key.to_lowercase(); - if filter_request_header(key_lc.as_str()) { - if disabled_headers.as_ref().is_some_and(|d| d.should_remove(key_lc.as_str())) { - continue; - } - if let (Ok(name), Ok(val)) = (HeaderName::from_bytes(key.as_bytes()), HeaderValue::from_bytes(value)) { - // Only insert if not already present (config takes precedence) - if !headers.contains_key(&name) { - if name == axum::http::header::USER_AGENT { - has_user_agent = true; - } - headers.insert(name, val); - } - } - } - } - } - - if log_enabled!(Level::Trace) { - let he: HashMap = - headers.iter().map(|(k, v)| (k.to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())).collect(); - if !he.is_empty() { - trace!("Request headers {he:?}"); - } - } - - // 3. Finally, if no User-Agent was provided by config OR client, use the default. - if !has_user_agent - && !disabled_headers.is_some_and(|disabled| disabled.should_remove(axum::http::header::USER_AGENT.as_str())) - { - let config_ua = default_user_agent - .and_then(|ua| { - let trimmed = ua.trim(); - (!trimmed.is_empty()).then_some(trimmed) - }) - .and_then(|ua| HeaderValue::from_str(ua).ok()); - - headers.insert( - axum::http::header::USER_AGENT, - config_ua.unwrap_or_else(|| HeaderValue::from_static(DEFAULT_USER_AGENT)), - ); - } - - headers -} - -pub fn overlay_upstream_user_agent( - headers: &mut HeaderMap, - upstream_user_agent: Option<&str>, - disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, -) { - if disabled_headers.is_some_and(|disabled| disabled.should_remove(axum::http::header::USER_AGENT.as_str())) { - return; - } - if let Some(value) = upstream_user_agent - .map(str::trim) - .filter(|value| !value.is_empty()) - .and_then(|value| HeaderValue::from_str(value).ok()) - { - headers.insert(axum::http::header::USER_AGENT, value); - } -} - -/// Appends a stable playback-session index to an already resolved User-Agent. -pub fn append_user_agent_stream_index(headers: &mut HeaderMap, stream_index: u64) { - let Some(user_agent) = headers.get(axum::http::header::USER_AGENT).map(HeaderValue::as_bytes) else { - return; - }; - let Some(first) = user_agent.iter().position(|byte| !byte.is_ascii_whitespace()) else { - return; - }; - let last = user_agent.iter().rposition(|byte| !byte.is_ascii_whitespace()).map_or(first, |index| index + 1); - let user_agent = &user_agent[first..last]; - - let mut digits = [0_u8; 20]; - let mut cursor = digits.len(); - let mut remaining = stream_index; - loop { - cursor -= 1; - digits[cursor] = b'0' + (remaining % 10) as u8; - remaining /= 10; - if remaining == 0 { - break; - } - } - let suffix = &digits[cursor..]; - if user_agent.strip_suffix(suffix).is_some_and(|prefix| prefix.ends_with(b" ")) { - return; - } - - let mut indexed_user_agent = Vec::with_capacity(user_agent.len() + 1 + suffix.len()); - indexed_user_agent.extend_from_slice(user_agent); - indexed_user_agent.push(b' '); - indexed_user_agent.extend_from_slice(suffix); - if let Ok(value) = HeaderValue::from_bytes(&indexed_user_agent) { - headers.insert(axum::http::header::USER_AGENT, value); - } -} - -// read local file content and return it as a string. -// Gzipped file content is supported. -pub async fn get_local_file_content(file_path: &Path) -> Result { - // open file - let file = File::open(file_path).await.map_err(|err| { - std::io::Error::new(ErrorKind::NotFound, format!("Failed to open file: {}, {err:?}", file_path.display())) - })?; - - let mut buf_reader = async_file_reader(file); - - // Peek first 2 bytes to detect gzip encoding - let buffer = buf_reader.fill_buf().await?; - let is_gzipped = buffer.len() >= 2 && is_gzip(&buffer[0..2]); - - let mut decoded = String::new(); - - if is_gzipped { - // Use async gzip decoder - let mut gzip_decoder = async_compression::tokio::bufread::GzipDecoder::new(buf_reader); - gzip_decoder - .read_to_string(&mut decoded) - .await - .map_err(|e| std::io::Error::other(format!("Failed to decode gzip content: {e}")))?; - } else { - // read plaintext - buf_reader - .read_to_string(&mut decoded) - .await - .map_err(|e| std::io::Error::other(format!("Failed to read file: {e}")))?; - } - - Ok(decoded) -} - -pub async fn get_local_file_content_as_stream(file_path: &Path) -> Result { - // open file - let file = File::open(file_path).await.map_err(|err| { - std::io::Error::new(ErrorKind::NotFound, format!("Failed to open file: {}, {err:?}", file_path.display())) - })?; - - let mut buf_reader = async_file_reader(file); - - // Peek first 2 Bytes, for gzip detection - let buffer = buf_reader.fill_buf().await?; - let is_gzipped = buffer.len() >= 2 && is_gzip(&buffer[0..2]); - - if is_gzipped { - // use Async Gzip Decoder - Ok(Box::pin(async_compression::tokio::bufread::GzipDecoder::new(buf_reader))) - } else { - Ok(Box::pin(buf_reader)) - } -} - -pub async fn get_remote_content_as_file( - app_config: &Arc, - client: &reqwest::Client, - input: &ConfigInput, - headers: Option<&HeaderMap>, - url: &Url, - file_path: &Path, -) -> Result { - get_remote_content_as_file_with_options( - app_config, - client, - input, - headers, - url, - file_path, - RecordingTaskOptions::default(), - ) - .await -} - -pub async fn get_remote_content_as_file_with_options( - app_config: &Arc, - client: &reqwest::Client, - input: &ConfigInput, - headers: Option<&HeaderMap>, - url: &Url, - file_path: &Path, - options: RecordingTaskOptions, -) -> Result { - let input_source = InputSource { - name: input.name.clone(), - url: url.to_string(), - provider: input.get_resolve_provider(url.as_str()), - username: input.username.clone(), - password: input.password.clone(), - method: input.method, - headers: input.headers.clone(), - }; - - let response = send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( - app_config, - client, - &input_source, - headers, - url, - 10, - RequestFetchOptions::default(), - ) - .await? - .response; - - let start_time = tokio::time::Instant::now(); - let (temp_file, output_file) = if options.atomic_write { - let (temp_file, output_file) = create_atomic_download_file(file_path)?; - (Some(temp_file), output_file) - } else { - (None, File::create(file_path).await?) - }; - let mut writer = async_file_writer(output_file); - - let mut stream = response.bytes_stream(); - let mut downloaded = 0_u64; - - let idle_timeout = tokio::time::Duration::from_secs(STREAM_IDLE_TIMEOUT); - let idle = sleep(idle_timeout); - tokio::pin!(idle); - - loop { - tokio::select! { - () = &mut idle => { - warn!("Stream idle for request, closing {}", sanitize_sensitive_info(url.as_ref())); - return Err(string_to_io_error(format!( - "Download timed out for {}", - sanitize_sensitive_info(url.as_ref()) - ))); - } - - chunk = stream.next() => { - idle.as_mut().reset(tokio::time::Instant::now() + idle_timeout); - - match chunk { - Some(Ok(bytes)) => { - downloaded = downloaded.checked_add(bytes.len() as u64).ok_or_else(|| { - string_to_io_error(format!( - "Download size overflow for {}", - sanitize_sensitive_info(url.as_ref()) - )) - })?; - if options.max_bytes.is_some_and(|max| downloaded > max) { - return Err(string_to_io_error(format!( - "Download exceeds configured limit for {}", - sanitize_sensitive_info(url.as_ref()) - ))); - } - writer.write_all(&bytes).await?; - } - Some(Err(e)) => { - return Err(string_to_io_error(format!("Failed to read chunk: {e}"))); - } - None => { - break; - } - } - } - } - } - - writer.flush().await?; - writer.shutdown().await?; - drop(writer); - - if let Some(temp_file) = temp_file { - persist_atomic_download_file(temp_file, file_path)?; - } - - debug!( - "File downloaded successfully to {}, took {}", - file_path.display(), - format_elapsed_time(start_time.elapsed().as_secs()) - ); - - Ok(file_path.to_path_buf()) -} - /// Controls decoding and bounded consumption of a fully buffered text response. #[derive(Debug, Clone, Copy)] pub struct TextContentBodyOptions { @@ -2315,40 +121,6 @@ pub struct TextContentBodyOptions { retry_owner: TextContentRetryOwner, } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -enum TextContentRetryOwner { - #[default] - RequestStack, - DecodedBodyConsumer, -} - -impl Default for TextContentBodyOptions { - fn default() -> Self { - Self { - detection: ContentCodingDetection::DeclaredOrLegacyTextMagic, - max_decoded_bytes: None, - deadline: None, - retry_owner: TextContentRetryOwner::RequestStack, - } - } -} - -impl TextContentBodyOptions { - /// Selects the narrowly scoped HLS-manifest fallback detection and decoded-size limit. - pub fn hls_manifest(max_decoded_bytes: usize, deadline: Duration) -> Self { - Self { - detection: ContentCodingDetection::DeclaredOrKnownHlsManifestMagic, - max_decoded_bytes: Some(max_decoded_bytes), - deadline: Some(deadline.max(Duration::from_millis(1))), - retry_owner: TextContentRetryOwner::DecodedBodyConsumer, - } - } - - fn legacy_text_with_deadline(deadline: Option) -> Self { - Self { deadline: deadline.map(|value| value.max(Duration::from_millis(1))), ..Self::default() } - } -} - /// Groups request-boundary and decoded-text-consumer options for one text fetch. #[derive(Debug, Clone, Copy, Default)] pub struct TextContentFetchOptions { @@ -2356,3063 +128,70 @@ pub struct TextContentFetchOptions { body: TextContentBodyOptions, } -impl TextContentFetchOptions { - pub const fn new(request: RequestFetchOptions, body: TextContentBodyOptions) -> Self { Self { request, body } } - - fn with_request_options(request: RequestFetchOptions) -> Self { - Self { body: TextContentBodyOptions::legacy_text_with_deadline(request.attempt_idle_timeout), request } - } -} - -async fn build_decoded_stream_reader(response: reqwest::Response) -> Result { - decode_response_to_identity(response, ContentCodingDetection::DeclaredOrLegacyTextMagic) - .await - .map(|decoded| decoded.body) - .map_err(content_coding_error_to_io) -} - -fn content_coding_error_to_io(error: ContentCodingError) -> Error { - let kind = match &error { - ContentCodingError::PrefixRead(source) => source.kind(), - ContentCodingError::InvalidHeader - | ContentCodingError::Unsupported(_) - | ContentCodingError::EncodedPartialContent => ErrorKind::Other, - }; - Error::new(kind, error) -} - -fn content_body_read_error_to_io(error: ContentBodyReadError) -> Error { - match error { - ContentBodyReadError::Io(error) => error, - error @ ContentBodyReadError::LimitExceeded { .. } => Error::other(error), - error @ ContentBodyReadError::InvalidUtf8 { .. } => Error::new(ErrorKind::InvalidData, error), - } -} - -/// Builds a fixed/numeric text-response diagnostic without origin-controlled content. -pub fn text_response_error_log_label(error: &Error) -> String { - if let Some(error) = content_decoding_error_from_io(error) { - return format!("content_decoding coding={}", error.coding.as_http_token()); - } - if let Some(error) = error.get_ref().and_then(|source| source.downcast_ref::()) { - return match error { - ContentBodyReadError::LimitExceeded { limit } => format!("decoded_body_limit limit={limit}"), - ContentBodyReadError::InvalidUtf8 { valid_up_to, error_len } => { - format!("invalid_utf8 valid_up_to={valid_up_to} error_len={error_len:?}") - } - ContentBodyReadError::Io(error) => format!("io kind={:?}", error.kind()), - }; - } - if let Some(error) = error.get_ref().and_then(|source| source.downcast_ref::()) { - return match error { - ContentCodingError::InvalidHeader => "content_coding class=invalid_header".to_string(), - ContentCodingError::Unsupported(_) => "content_coding class=unsupported".to_string(), - ContentCodingError::EncodedPartialContent => "content_coding class=encoded_partial_content".to_string(), - ContentCodingError::PrefixRead(_) => "content_coding class=prefix_read".to_string(), - }; - } - if error.kind() == ErrorKind::TimedOut { - return "timeout".to_string(); - } - if is_http_body_transport_error(error) { - return "transport".to_string(); - } - format!("io kind={:?}", error.kind()) -} - -async fn read_text_response_with_body_options( - response: reqwest::Response, - body_options: TextContentBodyOptions, -) -> Result<(String, String, HeaderMap), Error> { - let request_url = response.url().to_string(); - let read = async move { - let mut decoded = - decode_response_to_identity(response, body_options.detection).await.map_err(content_coding_error_to_io)?; - if let (ContentCodingDetection::DeclaredOrKnownHlsManifestMagic, Some(observation)) = - (body_options.detection, decoded.content_coding_observation()) - { - log_hls_origin_content_coding( - observation, - HlsOriginContentCodingObjectKind::Manifest, - false, - HlsOriginContentCodingSource::Legacy, - ); - } - let content = if let Some(max_decoded_bytes) = body_options.max_decoded_bytes { - read_utf8_limited(&mut decoded.body, max_decoded_bytes).await.map_err(content_body_read_error_to_io)? - } else { - let mut content = String::new(); - decoded.body.read_to_string(&mut content).await.map_err(|error| Error::new(error.kind(), error))?; - content - }; - Ok((content, decoded.final_url.to_string(), decoded.headers)) - }; - - if let Some(deadline) = body_options.deadline { - tokio::time::timeout(deadline, read).await.map_err(|_| { - Error::new( - ErrorKind::TimedOut, - format!("Timed out reading content body: {}", sanitize_sensitive_info(&request_url)), - ) - })? - } else { - read.await - } -} - -#[allow(clippy::implicit_hasher)] -pub async fn get_remote_content_as_stream( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - url: &Url, -) -> Result<(DynReader, String), Error> { - get_remote_content_as_stream_with_options(app_config, client, input, headers, url, RequestFetchOptions::default()) - .await -} - -#[allow(clippy::implicit_hasher)] -async fn get_remote_content_as_stream_with_options( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - url: &Url, - options: RequestFetchOptions, -) -> Result<(DynReader, String), Error> { - let response = - send_input_with_retry_and_provider_policy_with_options_result(app_config, client, input, headers, url, options) - .await? - .response; - let response_url = response.url().to_string(); - - let reader = build_decoded_stream_reader(response).await?; - Ok((reader, response_url)) -} - -fn text_body_retry_values(app_config: &Arc, body_options: TextContentBodyOptions) -> (u32, u64, f64) { - if body_options.retry_owner != TextContentRetryOwner::DecodedBodyConsumer { - return (1, 0, 1.0); - } - let config = app_config.config.load(); - let values = config - .reverse_proxy - .as_ref() - .map_or_else(ResourceRetryConfig::get_default_retry_values, |rp| rp.resource_retry.get_retry_values()); - drop(config); - values -} - -fn should_retry_text_body_error(error: &Error) -> bool { - matches!(error.kind(), ErrorKind::TimedOut | ErrorKind::ConnectionRefused) - || content_decoding_error_from_io(error).is_some() - || is_http_body_transport_error(error) - || error - .get_ref() - .and_then(|source| source.downcast_ref::()) - .is_some_and(|error| matches!(error, ContentCodingError::PrefixRead(_))) -} - -async fn sleep_before_text_body_retry(attempt: u32, backoff_ms: u64, backoff_multiplier: f64, err: &Error) { - let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); - warn!( - "Text response body failed retryably, retrying in {}ms (attempt {}): {}", - delay, - attempt + 1, - text_response_error_log_label(err) - ); - tokio::time::sleep(Duration::from_millis(delay)).await; -} - -fn is_retryable_text_response_status(status: StatusCode) -> bool { - status.is_server_error() - || matches!( - status, - StatusCode::PROXY_AUTHENTICATION_REQUIRED - | StatusCode::REQUEST_TIMEOUT - | StatusCode::TOO_EARLY - | StatusCode::TOO_MANY_REQUESTS - ) -} - -fn text_response_status_error(status: StatusCode, url: &Url) -> Error { - string_to_io_error(format!( - "Request failed ({}): {}", - format_http_status(status), - sanitize_sensitive_info(url.as_str()) - )) -} - -async fn get_remote_content_with_options( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - url: &Url, - options: RequestFetchOptions, -) -> Result<(String, String), Error> { - get_remote_content_with_headers_and_options( - app_config, - client, - input, - headers, - url, - TextContentFetchOptions::with_request_options(options), - ) - .await - .map(|(content, response_url, _)| (content, response_url)) -} - -async fn get_remote_content_with_headers_and_options( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - url: &Url, - options: TextContentFetchOptions, -) -> Result<(String, String, HeaderMap), Error> { - let (max_attempts, backoff_ms, backoff_multiplier) = text_body_retry_values(app_config, options.body); - let attempt_options = if max_attempts > 1 { options.request.without_resource_retries() } else { options.request }; - - // This consumer owns the logical attempt budget whenever decoded-body retries are enabled. Provider failover - // and redirect hops remain bounded subrequests, but the configured retry count is not applied again below it. - for attempt in 0..max_attempts { - let response = match send_input_with_retry_and_provider_policy_with_options_result( - app_config, - client, - input, - headers, - url, - attempt_options, - ) - .await - { - Ok(result) => result.response, - Err(error) if should_retry_text_body_error(&error) && attempt + 1 < max_attempts => { - sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &error).await; - continue; - } - Err(error) => return Err(error), - }; - let status = response.status(); - if !status.is_success() { - if is_retryable_text_response_status(status) && attempt + 1 < max_attempts { - perform_backoff(attempt, backoff_ms, backoff_multiplier, &response).await; - continue; - } - return Err(text_response_status_error(status, url)); - } - match read_text_response_with_body_options(response, options.body).await { - Ok(result) => return Ok(result), - Err(error) if should_retry_text_body_error(&error) && attempt + 1 < max_attempts => { - sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &error).await; - } - Err(error) => return Err(error), - } - } - Err(string_to_io_error("Text response body retry attempts exhausted")) -} - -#[allow(clippy::too_many_lines)] -async fn get_remote_content_with_manual_redirects_and_options( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - url: &Url, - max_redirects: usize, - options: RequestFetchOptions, -) -> Result<(String, String), Error> { - get_remote_content_with_manual_redirects_and_headers_and_options( - app_config, - client, - input, - headers, - url, - max_redirects, - TextContentFetchOptions::with_request_options(options), - ) - .await - .map(|(content, response_url, _)| (content, response_url)) -} - -async fn get_remote_content_with_manual_redirects_and_headers_and_options( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - url: &Url, - max_redirects: usize, - options: TextContentFetchOptions, -) -> Result<(String, String, HeaderMap), Error> { - let (max_attempts, backoff_ms, backoff_multiplier) = text_body_retry_values(app_config, options.body); - let attempt_options = if max_attempts > 1 { options.request.without_resource_retries() } else { options.request }; - - // Manual redirects retain their own credential-scrubbing loop inside each caller-owned logical attempt. - for attempt in 0..max_attempts { - let response = match send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( - app_config, - client, - input, - headers, - url, - max_redirects, - attempt_options, - ) - .await - { - Ok(result) => result.response, - Err(error) if should_retry_text_body_error(&error) && attempt + 1 < max_attempts => { - sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &error).await; - continue; - } - Err(error) => return Err(error), - }; - let status = response.status(); - if !status.is_success() { - if is_retryable_text_response_status(status) && attempt + 1 < max_attempts { - perform_backoff(attempt, backoff_ms, backoff_multiplier, &response).await; - continue; - } - return Err(text_response_status_error(status, url)); - } - match read_text_response_with_body_options(response, options.body).await { - Ok(result) => return Ok(result), - Err(error) if should_retry_text_body_error(&error) && attempt + 1 < max_attempts => { - sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &error).await; - } - Err(error) => return Err(error), - } - } - Err(string_to_io_error("Text response body retry attempts exhausted")) -} - -fn same_origin(lhs: &Url, rhs: &Url) -> bool { - lhs.scheme().eq_ignore_ascii_case(rhs.scheme()) - && lhs.host_str() == rhs.host_str() - && lhs.port_or_known_default() == rhs.port_or_known_default() -} - -/// Reports whether a request header may be retained when the target origin changes. -pub fn is_safe_cross_origin_redirect_header(key: &str) -> bool { - key.eq_ignore_ascii_case("accept") - || key.eq_ignore_ascii_case("accept-encoding") - || key.eq_ignore_ascii_case("accept-language") - || key.eq_ignore_ascii_case("user-agent") - || key.eq_ignore_ascii_case("range") - || key.eq_ignore_ascii_case("if-range") - || key.eq_ignore_ascii_case("icy-metadata") -} - -fn strip_sensitive_headers_for_cross_origin_redirect(headers: &mut HashMap) { - headers.retain(|key, _| is_safe_cross_origin_redirect_header(key)); -} - -fn create_atomic_download_file(file_path: &Path) -> Result<(tempfile::NamedTempFile, File), Error> { - let parent = file_path.parent().filter(|path| !path.as_os_str().is_empty()).unwrap_or_else(|| Path::new(".")); - let temp_file = tempfile::Builder::new().prefix(".tuliprox-download-").suffix(".tmp").tempfile_in(parent)?; - let output_file = File::from_std(temp_file.reopen()?); - Ok((temp_file, output_file)) -} - -fn persist_atomic_download_file(temp_file: tempfile::NamedTempFile, file_path: &Path) -> Result<(), Error> { - match temp_file.persist(file_path) { - Ok(_) => Ok(()), - Err(err) => Err(err.error), - } -} - -async fn copy_local_epg_file_to_persist( - file_path: &Path, - persist_filepath: &Path, - max_bytes: Option, -) -> Result { - let mut reader = File::open(file_path).await?; - let (temp_file, output_file) = create_atomic_download_file(persist_filepath)?; - let mut writer = async_file_writer(output_file); - let mut copied = 0_u64; - let mut buffer = vec![0_u8; 64 * 1024].into_boxed_slice(); - - loop { - let read = reader.read(&mut buffer).await?; - if read == 0 { - break; - } - copied = copied - .checked_add(read as u64) - .ok_or_else(|| string_to_io_error(format!("Local EPG file size overflow for {}", file_path.display())))?; - if max_bytes.is_some_and(|max| copied > max) { - return Err(string_to_io_error(format!("Local EPG file {} exceeds configured limit", file_path.display()))); - } - writer.write_all(&buffer[..read]).await?; - } - - writer.flush().await?; - writer.shutdown().await?; - drop(writer); - drop(reader); - persist_atomic_download_file(temp_file, persist_filepath)?; - Ok(persist_filepath.to_path_buf()) -} - -async fn download_epg_content_as_file( - app_config: &Arc, - client: &reqwest::Client, - input: &ConfigInput, - headers: Option<&HeaderMap>, - url_str: &str, - persist_filepath: &Path, - max_bytes: Option, -) -> Result { - if let Ok(url) = url_str.parse::() { - match url.scheme() { - "file" => { - let file_path = url.to_file_path().map_err(|()| { - Error::new(ErrorKind::Unsupported, format!("Unknown file {}", sanitize_sensitive_info(url_str))) - })?; - if file_path.exists() { - copy_local_epg_file_to_persist(&file_path, persist_filepath, max_bytes).await - } else { - Err(Error::new(ErrorKind::NotFound, format!("Unknown file {}", file_path.display()))) - } - } - "http" | "https" | "provider" => { - get_remote_content_as_file_with_options( - app_config, - client, - input, - headers, - &url, - persist_filepath, - RecordingTaskOptions { max_bytes, atomic_write: true }, - ) - .await - } - scheme => Err(Error::new( - ErrorKind::Unsupported, - format!("Unsupported EPG URL scheme '{scheme}' for {}", sanitize_sensitive_info(url_str)), - )), - } - } else { - Err(Error::new(ErrorKind::Unsupported, format!("Malformed URL {}", sanitize_sensitive_info(url_str)))) - } -} - -pub async fn download_text_content( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - persist_filepath: Option, - trace_log: bool, -) -> Result<(String, String), Error> { - Box::pin(download_text_content_with_options( - app_config, - client, - input, - headers, - persist_filepath, - trace_log, - RequestFetchOptions::default(), - )) - .await -} - -pub async fn download_text_content_with_options( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - persist_filepath: Option, - trace_log: bool, - options: RequestFetchOptions, -) -> Result<(String, String), Error> { - let start_time = tokio::time::Instant::now(); - let result = if let Ok(url) = input.url.parse::() { - let result = if url.scheme() == "file" { - match url.to_file_path() { - Ok(file_path) => get_local_file_content(&file_path).await.map(|content| (content, url.to_string())), - Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), - } - } else { - get_remote_content_with_options(app_config, client, input, headers, &url, options).await - }; - match result { - Ok((content, response_url)) => { - if persist_filepath.is_some() { - persist_file(persist_filepath, &content).await; - } - Ok((content, response_url)) - } - Err(err) => Err(err), - } - } else { - Err(string_to_io_error(format!("Malformed URL {}", sanitize_sensitive_info(&input.url)))) - }; - - let level = if trace_log { log::Level::Trace } else { log::Level::Debug }; - if log_enabled!(level) { - if let Ok((_content, response_url)) = result.as_ref() { - log::log!( - level, - "Request took: {} {}", - format_elapsed_time(start_time.elapsed().as_secs()), - sanitize_sensitive_info(response_url.as_str()) - ); - } - } - - result -} - -pub async fn download_text_content_with_headers( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - trace_log: bool, -) -> Result<(String, String, HeaderMap), Error> { - Box::pin(download_text_content_with_headers_and_options( - app_config, - client, - input, - headers, - trace_log, - TextContentFetchOptions::default(), - )) - .await -} - -pub async fn download_text_content_with_headers_and_options( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - trace_log: bool, - options: TextContentFetchOptions, -) -> Result<(String, String, HeaderMap), Error> { - let start_time = tokio::time::Instant::now(); - let result = if let Ok(url) = input.url.parse::() { - let result = if url.scheme() == "file" { - match url.to_file_path() { - Ok(file_path) => { - get_local_file_content(&file_path).await.map(|content| (content, url.to_string(), HeaderMap::new())) - } - Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), - } - } else { - get_remote_content_with_headers_and_options(app_config, client, input, headers, &url, options).await - }; - result - } else { - Err(string_to_io_error(format!("Malformed URL {}", sanitize_sensitive_info(&input.url)))) - }; - - let level = if trace_log { log::Level::Trace } else { log::Level::Debug }; - if log_enabled!(level) { - if let Ok((_, response_url, _)) = result.as_ref() { - log::log!( - level, - "Request took: {} {}", - format_elapsed_time(start_time.elapsed().as_secs()), - sanitize_sensitive_info(response_url.as_str()) - ); - } - } - - result -} - -pub async fn download_text_content_with_manual_redirects( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - persist_filepath: Option, - trace_log: bool, - max_redirects: usize, -) -> Result<(String, String), Error> { - Box::pin(download_text_content_with_manual_redirects_and_options( - app_config, - client, - input, - headers, - persist_filepath, - trace_log, - max_redirects, - RequestFetchOptions::default(), - )) - .await -} - -#[allow(clippy::too_many_arguments)] -pub async fn download_text_content_with_manual_redirects_and_options( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - persist_filepath: Option, - trace_log: bool, - max_redirects: usize, - options: RequestFetchOptions, -) -> Result<(String, String), Error> { - let start_time = tokio::time::Instant::now(); - let result = if let Ok(url) = input.url.parse::() { - let result = if url.scheme() == "file" { - match url.to_file_path() { - Ok(file_path) => get_local_file_content(&file_path).await.map(|content| (content, url.to_string())), - Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), - } - } else { - get_remote_content_with_manual_redirects_and_options( - app_config, - client, - input, - headers, - &url, - max_redirects, - options, - ) - .await - }; - match result { - Ok((content, response_url)) => { - if persist_filepath.is_some() { - persist_file(persist_filepath, &content).await; - } - Ok((content, response_url)) - } - Err(err) => Err(err), - } - } else { - Err(string_to_io_error(format!("Malformed URL {}", sanitize_sensitive_info(&input.url)))) - }; - - let level = if trace_log { log::Level::Trace } else { log::Level::Debug }; - if log_enabled!(level) { - if let Ok((_content, response_url)) = result.as_ref() { - log::log!( - level, - "Request took: {} {}", - format_elapsed_time(start_time.elapsed().as_secs()), - sanitize_sensitive_info(response_url.as_str()) - ); - } - } - - result -} - -pub async fn download_text_content_with_manual_redirects_and_headers( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - trace_log: bool, - max_redirects: usize, -) -> Result<(String, String, HeaderMap), Error> { - Box::pin(download_text_content_with_manual_redirects_and_headers_and_options( - app_config, - client, - input, - headers, - trace_log, - max_redirects, - TextContentFetchOptions::default(), - )) - .await -} - -pub async fn download_text_content_with_manual_redirects_and_headers_and_options( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - headers: Option<&HeaderMap>, - trace_log: bool, - max_redirects: usize, - options: TextContentFetchOptions, -) -> Result<(String, String, HeaderMap), Error> { - let start_time = tokio::time::Instant::now(); - let result = if let Ok(url) = input.url.parse::() { - let result = if url.scheme() == "file" { - match url.to_file_path() { - Ok(file_path) => { - get_local_file_content(&file_path).await.map(|content| (content, url.to_string(), HeaderMap::new())) - } - Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), - } - } else { - get_remote_content_with_manual_redirects_and_headers_and_options( - app_config, - client, - input, - headers, - &url, - max_redirects, - options, - ) - .await - }; - result - } else { - Err(string_to_io_error(format!("Malformed URL {}", sanitize_sensitive_info(&input.url)))) - }; - - let level = if trace_log { log::Level::Trace } else { log::Level::Debug }; - if log_enabled!(level) { - if let Ok((_, response_url, _)) = result.as_ref() { - log::log!( - level, - "Request took: {} {}", - format_elapsed_time(start_time.elapsed().as_secs()), - sanitize_sensitive_info(response_url.as_str()) - ); - } - } - - result -} - -pub async fn download_text_content_as_stream( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - persist_filepath: Option, -) -> Result<(DynReader, String), Error> { - if let Ok(url) = input.url.parse::() { - let result = if url.scheme() == "file" { - match url.to_file_path() { - Ok(file_path) => get_local_file_content_as_stream(&file_path).await.map(|c| (c, url.to_string())), - Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), - } - } else { - get_remote_content_as_stream(app_config, client, input, None, &url).await - }; - match result { - Ok((content, response_url)) => { - if let Some(path) = persist_filepath { - let tee_reader: DynReader = tee_dyn_reader( - content, - &path, - Some(Arc::new(|size| { - debug!("Persisted {size} bytes"); - })), - ) - .await; - Ok((tee_reader, response_url)) - } else { - Ok((content, response_url)) - } - } - Err(err) => Err(err), - } - } else { - Err(string_to_io_error(format!("Malformed URL {}", sanitize_sensitive_info(&input.url)))) - } -} - -async fn download_json_content( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - persist_filepath: Option, - trace_log: bool, -) -> Result { - debug_if_enabled!("Downloading json content from {}", sanitize_sensitive_info(&input.url)); - match download_text_content(app_config, client, input, None, persist_filepath, trace_log).await { - Ok((content, _response_url)) => match serde_json::from_str::(&content) { - Ok(value) => Ok(value), - Err(err) => Err(string_to_io_error(format!("Failed to parse json {err}"))), - }, - Err(err) => Err(err), - } -} - -pub async fn get_input_json_content( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - persist_filepath: Option, - trace_log: bool, -) -> Result { - match download_json_content(app_config, client, input, persist_filepath, trace_log).await { - Ok(content) => Ok(content), - Err(e) => Err(TuliproxError::RepositoryNetwork(format!( - "can't download input {input} => {sanitized}", - input = input.name, - sanitized = sanitize_sensitive_info(&e.to_string()) - ))), - } -} - -async fn download_json_content_as_stream( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - persist_filepath: Option, -) -> Result { - debug_if_enabled!("Downloading json content as stream from {}", sanitize_sensitive_info(&input.url)); - match download_text_content_as_stream(app_config, client, input, persist_filepath).await { - Ok((reader, _response_url)) => Ok(reader), - Err(err) => Err(err), - } -} - -pub async fn get_input_json_content_as_stream( - app_config: &Arc, - client: &reqwest::Client, - input: &InputSource, - persist_filepath: Option, -) -> Result { - match download_json_content_as_stream(app_config, client, input, persist_filepath).await { - Ok(stream) => Ok(stream), - Err(e) => Err(TuliproxError::RepositoryNetwork(format!( - "can't download input {input} => {sanitized}", - input = input.name, - sanitized = sanitize_sensitive_info(&e.to_string()) - ))), - } -} - -pub fn create_client_with_redirect(cfg: &AppConfig, redirect_policy: Policy) -> reqwest::ClientBuilder { - configured_client(&cfg.config.load(), redirect_policy) -} - -fn configured_client(config: &Config, redirect_policy: Policy) -> reqwest::ClientBuilder { - log_proxy_diagnostics(config); - let mut client = reqwest::Client::builder() - .redirect(redirect_policy) - .pool_idle_timeout(Duration::from_secs(30)) - .pool_max_idle_per_host(10) - .danger_accept_invalid_certs(config.accept_insecure_ssl_certificates); - - if let Some(proxy_cfg) = config.proxy.as_ref() { - match Url::parse(&proxy_cfg.url) { - Ok(mut url) => { - let scheme = url.scheme().to_ascii_lowercase(); - - match scheme.as_str() { - "socks5" | "socks5h" => { - if let Some(user) = &proxy_cfg.username { - let _ = url.set_username(user); - } - if let Some(pass) = &proxy_cfg.password { - let _ = url.set_password(Some(pass)); - } - match reqwest::Proxy::all(url.as_str()) { - Ok(p) => { - client = client.proxy(p); - } - Err(err) => error!("Failed to create SOCKS proxy {url}: {err}"), - } - } - "http" | "https" => match reqwest::Proxy::all(url.as_str()) { - Ok(p) => { - if let (Some(username), Some(password)) = (&proxy_cfg.username, &proxy_cfg.password) { - client = client.proxy(p.basic_auth(username, password)); - } else { - client = client.proxy(p); - } - } - Err(err) => error!("Failed to create HTTP proxy {url}: {err}"), - }, - _ => { - error!("Unsupported proxy scheme '{scheme}' in URL: {url}"); - } - } - } - Err(e) => { - error!("Invalid proxy URL '{}': {e}", proxy_cfg.url); - } - } - } - - if let Some(rp_config) = config.reverse_proxy.as_ref() { - if rp_config.disabled_header.as_ref().is_some_and(|d| d.referer_header) { - client = client.referer(false); - } - } - - client -} - -pub fn create_client(cfg: &AppConfig) -> reqwest::ClientBuilder { - create_client_with_redirect(cfg, Policy::limited(10)) -} - -/// Dedicated discovery profile. Callers must propagate construction failure, never -/// fall back to an unconfigured client. Builder access permits local CA/DNS fixtures. -pub fn create_tmdb_client(cfg: &AppConfig) -> Result { - let config = cfg.config.load(); - let invalid_proxy = || TuliproxError::Config("TMDB HTTP profile has invalid proxy configuration".to_string()); - // The general factory historically logs malformed proxies and continues. TMDB - // must fail closed, using the same snapshot and proxy transformations instead. - if let Some(proxy) = &config.proxy { - let mut url = Url::parse(&proxy.url).map_err(|_| invalid_proxy())?; - match url.scheme() { - "socks5" | "socks5h" => { - if let Some(user) = &proxy.username { - url.set_username(user).map_err(|()| invalid_proxy())?; - } - if let Some(pass) = &proxy.password { - url.set_password(Some(pass)).map_err(|()| invalid_proxy())?; - } - } - "http" | "https" => {} - _ => return Err(invalid_proxy()), - } - reqwest::Proxy::all(url.as_str()).map_err(|_| invalid_proxy())?; - } - // Discovery carries its own Bearer credential: never inherit the generic TLS bypass. - // Keep configured proxy/auth and trusted roots, including hostname verification. - let mut builder = configured_client(&config, Policy::none()) - .danger_accept_invalid_certs(false) - .retry(reqwest::retry::never()) - .http1_only(); - if config.connect_timeout_secs > 0 { - builder = builder.connect_timeout(Duration::from_secs(u64::from(config.connect_timeout_secs))); - } - Ok(builder) -} - -pub fn parse_range(range: &str) -> Option<(u64, Option)> { - // expect: "bytes=START-END" - if !range.starts_with("bytes=") { - return None; - } - - let range = &range[6..]; - let mut parts = range.split('-'); - - let start = parts.next()?.parse().ok()?; - let end = parts.next().and_then(|s| s.parse().ok()); - - Some((start, end)) -} - -pub fn is_file_url(url: &str) -> bool { Url::parse(url).is_ok_and(|u| u.scheme().eq_ignore_ascii_case("file")) } - -pub fn is_uri(url: &str) -> bool { - Url::parse(url).is_ok_and(|u| { - u.scheme().eq_ignore_ascii_case("file") - || u.scheme().eq_ignore_ascii_case("http") - || u.scheme().eq_ignore_ascii_case("https") - }) -} - -/// Checks if a status code or error indicates a need for failover -/// -/// Returns true for server-side errors that might be resolved by trying another URL. -/// Returns false for client-side errors (401, 403, etc.) where the problem is with -/// credentials or permissions, not the server availability. -pub fn should_trigger_failover(status: StatusCode) -> bool { - matches!( - status, - StatusCode::NOT_FOUND - | StatusCode::GONE - | StatusCode::SERVICE_UNAVAILABLE - | StatusCode::BAD_GATEWAY - | StatusCode::GATEWAY_TIMEOUT - | StatusCode::INTERNAL_SERVER_ERROR - | StatusCode::TOO_MANY_REQUESTS - | StatusCode::REQUEST_TIMEOUT - | StatusCode::PROXY_AUTHENTICATION_REQUIRED - ) - // Explicitly NOT triggering failover for: - // - 401 Unauthorized (wrong credentials) - // - 403 Forbidden (permission issue) - // - 402 Payment Required (subscription issue) - // - 451 Unavailable For Legal Reasons (geo-blocking) - // - // Note: DO triggering failover for: - // - 429 Too Many Requests - // - 408 Request Timeout - // - 407 Proxy Authentication Required -} +#[cfg(test)] +mod tests; + +mod attempt; +mod body; +mod client; +mod destination; +mod download; +mod headers; +mod options; +mod protocol; +mod redirect; +mod retry; #[cfg(test)] -mod tests { - mod tmdb_profile; - - use super::{ - classify_resource_destination, download_text_content, download_text_content_with_headers_and_options, - get_input_epg_content_as_file, get_remote_content_as_stream, is_safe_cross_origin_redirect_header, - next_provider_url_index, preview_request_diagnostics_for_logging, preview_request_target_for_logging, - resolve_attempt_target, resolve_resource_socket_addrs, same_origin, - send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, +use self::attempt::resolve_attempt_target; +pub(crate) use self::destination::classify_host; +#[cfg(test)] +use self::destination::{proxy_hosts, verdict_for_addresses}; +#[cfg(test)] +use self::download::create_atomic_download_file; +#[cfg(test)] +use self::retry::{next_provider_url_index, should_try_next_ip_on_connect_error}; +use self::{ + attempt::{ + execute_attempt_request, prepare_physical_request_attempt, preview_attempt_target, + resolve_attempt_target_at_provider_index, AttemptTarget, + }, + body::{ + get_remote_content_with_headers_and_options, get_remote_content_with_options, + read_text_response_with_body_options, text_response_status_error, + }, + headers::{format_request_target_for_logging, log_proxy_diagnostics, prepare_input_request_headers}, + options::{apply_request_fetch_options, ResourceRetryExecution, TextContentRetryOwner}, + redirect::{is_failover_redirect, same_origin, strip_sensitive_headers_for_cross_origin_redirect}, + retry::{ + is_retryable_text_response_status, perform_backoff, provider_start_index, should_retry_text_body_error, + sleep_before_text_body_retry, text_body_retry_values, + }, +}; +pub use self::{ + body::{get_remote_content_as_stream, text_response_error_log_label}, + client::{create_client, create_client_with_redirect, create_tmdb_client, get_client_request}, + destination::{classify_ip, is_public_ip, resolve_public_socket_addrs, resolve_resource_socket_addrs}, + download::{ + classify_resource_destination, download_text_content, download_text_content_as_stream, + download_text_content_with_headers, download_text_content_with_headers_and_options, + download_text_content_with_options, get_input_epg_content_as_file, get_input_json_content, + get_input_json_content_as_stream, get_input_text_content_as_stream, get_local_file_content, + get_local_file_content_as_stream, get_remote_content_as_file, get_remote_content_as_file_with_options, + }, + headers::{ + append_user_agent_stream_index, get_request_headers, overlay_upstream_user_agent, + preview_request_diagnostics_for_logging, preview_request_target_for_logging, + }, + protocol::{classify_content_type, content_type_from_ext, format_http_status, is_file_url, is_uri, parse_range}, + redirect::{ + download_text_content_with_manual_redirects, download_text_content_with_manual_redirects_and_headers, + download_text_content_with_manual_redirects_and_headers_and_options, + download_text_content_with_manual_redirects_and_options, is_safe_cross_origin_redirect_header, + }, + retry::{ + calculate_retry_backoff, send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, send_input_with_retry_and_provider_policy_with_options_result, send_with_retry_and_provider, - send_with_retry_and_provider_policy, should_retry_text_body_error, should_try_next_ip_on_connect_error, - strip_sensitive_headers_for_cross_origin_redirect, text_response_error_log_label, InputEpgFileRequest, - PublicIpResolver, RequestFetchOptions, ResourceDestination, ResourceDestinationResolver, - TextContentBodyOptions, TextContentFetchOptions, STREAM_IDLE_TIMEOUT, - }; - use crate::{ - model::{ - AppConfig, Config, ConfigInput, ConfigProvider, InputSource, MediaToolCapabilities, ResourceRetryConfig, - ReverseProxyConfig, ReverseProxyDisabledHeaderConfig, SourcesConfig, - }, - utils::{ - content_coding::{ContentCoding, ContentCodingError, ContentDecodingIoError, OutboundContentCodingPolicy}, - FileLockManager, - }, - }; - use arc_swap::{ArcSwap, ArcSwapOption}; - use flate2::{ - write::{GzEncoder, ZlibEncoder}, - Compression, - }; - use reqwest::header::{HeaderMap, HeaderValue, ACCEPT_ENCODING, COOKIE}; - use shared::{ - defaults::DEFAULT_USER_AGENT, - model::{ - ConfigPaths, ConfigProviderDto, DnsScheme, InputFetchMethod, OnConnectErrorPolicy, ProviderDnsDto, - ProviderUrlSelectionPolicy, - }, - utils::{get_base_url_from_str, replace_url_extension, sanitize_sensitive_info}, - }; - use std::{ - collections::{HashMap, HashSet}, - io::{Error, ErrorKind, Write}, - net::{IpAddr, SocketAddr}, - path::{Path, PathBuf}, - sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, - Arc, - }, - time::Duration, - }; - use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpListener, - sync::{oneshot, Mutex}, - }; - use url::Url; - - #[test] - fn ipv6_addresses_embedding_private_ipv4_are_not_public() { - for address in ["::10.0.0.1", "64:ff9b::a00:1", "2002:0a00:0001::", "::ffff:10.0.0.1"] { - assert!(address.parse::().is_ok_and(|ip| !super::is_public_ip(ip)), "{address}"); - } - } - - #[test] - fn resource_destinations_are_classified_by_reachability() { - for (address, expected) in [ - ("1.1.1.1", ResourceDestination::Public), - ("8.8.8.8", ResourceDestination::Public), - ("64:ff9b::808:808", ResourceDestination::Public), - ("2002:808:808::", ResourceDestination::Public), - ("::808:808", ResourceDestination::Public), - ("10.0.0.1", ResourceDestination::Private), - ("172.16.5.5", ResourceDestination::Private), - ("192.168.1.20", ResourceDestination::Private), - ("100.64.0.1", ResourceDestination::Private), - ("fc00::1", ResourceDestination::Private), - ("64:ff9b::a00:1", ResourceDestination::Private), - ("2002:0a00:0001::", ResourceDestination::Private), - ("::a00:1", ResourceDestination::Private), - ("::ffff:10.0.0.1", ResourceDestination::Private), - ("0.0.0.0", ResourceDestination::Blocked), - ("127.0.0.1", ResourceDestination::Blocked), - ("169.254.1.1", ResourceDestination::Blocked), - ("169.254.169.254", ResourceDestination::Blocked), - ("255.255.255.255", ResourceDestination::Blocked), - ("224.0.0.1", ResourceDestination::Blocked), - ("::", ResourceDestination::Blocked), - ("::1", ResourceDestination::Blocked), - ("fe80::1", ResourceDestination::Blocked), - ("ff02::1", ResourceDestination::Blocked), - ("::ffff:127.0.0.1", ResourceDestination::Blocked), - ("64:ff9b::7f00:1", ResourceDestination::Blocked), - ("2002:7f00:1::", ResourceDestination::Blocked), - ] { - let address = address.parse::().expect("valid IP address"); - assert_eq!(super::classify_ip(address), expected, "{address}"); - } - } - - #[tokio::test] - async fn resource_destination_names_are_classified_without_resolving_ip_literals() { - assert_eq!(classify_resource_destination("192.168.1.20").await, ResourceDestination::Private); - assert_eq!(classify_resource_destination("[::1]").await, ResourceDestination::Blocked); - assert_eq!(classify_resource_destination("[64:ff9b::7f00:1]").await, ResourceDestination::Blocked); - assert_eq!(classify_resource_destination("8.8.8.8").await, ResourceDestination::Public); - } - - #[tokio::test] - async fn resource_destination_resolution_refuses_addresses_local_to_this_host() { - let private = resolve_resource_socket_addrs("10.0.0.1", 80).await.expect("private destination is allowed"); - assert_eq!(private.len(), 1); - - for local_only in ["127.0.0.1", "169.254.169.254", "[::1]", "fe80::1", "0.0.0.0"] { - let error = resolve_resource_socket_addrs(local_only, 80) - .await - .expect_err("local-only destination must be refused"); - assert_eq!(error.kind(), ErrorKind::PermissionDenied, "{local_only}"); - } - } - - #[test] - fn proxy_hosts_are_exempt_from_the_connect_time_local_address_guard() { - use crate::{model::Config, utils::request::proxy_hosts}; - - // `localhost` is what a proxy on this host is usually configured as, and reqwest asks the - // resolver for the proxy host as well: guarding it would break every fetch through such a proxy. - let app_config = make_test_app_config(Config { - proxy: Some(crate::model::ProxyConfig { - url: "http://localhost:8118".to_string(), - username: None, - password: None, - }), - ..Config::default() - }); - - assert_eq!(proxy_hosts(&app_config), vec![Arc::from("localhost")]); - } - - #[test] - fn resolved_address_sets_are_aggregated_for_exposure() { - use super::{ - verdict_for_addresses, RESOURCE_DESTINATION_PRIVATE_TTL, RESOURCE_DESTINATION_PUBLIC_TTL, - RESOURCE_DESTINATION_UNANSWERED_TTL, - }; - - let public: IpAddr = "8.8.8.8".parse().expect("valid public address"); - let private: IpAddr = "10.0.0.1".parse().expect("valid private address"); - let blocked: IpAddr = "127.0.0.1".parse().expect("valid loopback address"); - - assert_eq!(verdict_for_addresses(&[public]), (ResourceDestination::Public, RESOURCE_DESTINATION_PUBLIC_TTL)); - assert_eq!( - verdict_for_addresses(&[public, public]), - (ResourceDestination::Public, RESOURCE_DESTINATION_PUBLIC_TTL) - ); - assert_eq!(verdict_for_addresses(&[private]), (ResourceDestination::Private, RESOURCE_DESTINATION_PRIVATE_TTL)); - assert_eq!( - verdict_for_addresses(&[public, private]), - (ResourceDestination::Private, RESOURCE_DESTINATION_PRIVATE_TTL) - ); - assert_eq!( - verdict_for_addresses(&[public, blocked]), - (ResourceDestination::Blocked, RESOURCE_DESTINATION_PRIVATE_TTL) - ); - assert_eq!( - verdict_for_addresses(&[private, blocked]), - (ResourceDestination::Blocked, RESOURCE_DESTINATION_PRIVATE_TTL) - ); - // An unanswered name stays non-public, but only briefly: a resolver hiccup must not pin the - // destination to the proxy for minutes. - assert_eq!(verdict_for_addresses(&[]), (ResourceDestination::Private, RESOURCE_DESTINATION_UNANSWERED_TTL)); - assert!(RESOURCE_DESTINATION_UNANSWERED_TTL < RESOURCE_DESTINATION_PRIVATE_TTL); - } - - #[tokio::test] - async fn resource_destination_resolver_refuses_local_only_names() { - use reqwest::dns::Resolve; - use std::str::FromStr; - - let resolver = ResourceDestinationResolver::default(); - let name = reqwest::dns::Name::from_str("127.0.0.1").expect("valid destination name"); - let result = resolver.resolve(name).await; - - match result { - Ok(_) => panic!("local-only destination must be refused"), - Err(error) => assert!(error.to_string().contains("local to this host"), "{error}"), - } - - // Only the proxy hosts are exempt, so a proxy on the loopback interface stays usable while a - // destination that resolves to the loopback interface is still refused. - let resolver = ResourceDestinationResolver { allowed_hosts: vec![Arc::from("localhost")].into() }; - let name = reqwest::dns::Name::from_str("localhost").expect("valid proxy name"); - assert!(resolver.resolve(name).await.is_ok(), "the configured proxy host must resolve"); - let name = reqwest::dns::Name::from_str("127.0.0.1").expect("valid destination name"); - assert!(resolver.resolve(name).await.is_err(), "a destination may not use the exemption"); - } - - fn make_test_app_config(config: Config) -> Arc { - Arc::new(AppConfig { - config: Arc::new(ArcSwap::from_pointee(config)), - sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), - hdhomerun: Arc::new(ArcSwapOption::default()), - api_proxy: Arc::new(ArcSwapOption::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - }) - } - - fn make_epg_test_client() -> reqwest::Client { - reqwest::Client::builder() - .no_proxy() - .redirect(reqwest::redirect::Policy::none()) - .timeout(Duration::from_secs(2)) - .build() - .expect("test client") - } - - async fn atomic_download_temp_files(directory: &Path) -> Vec { - let mut result = Vec::new(); - let mut entries = tokio::fs::read_dir(directory).await.expect("read temp directory"); - while let Some(entry) = entries.next_entry().await.expect("read temp directory entry") { - if entry.file_name().to_string_lossy().starts_with(".tuliprox-download-") { - result.push(entry.path()); - } - } - result - } - - #[tokio::test] - async fn atomic_download_temp_files_are_unique_and_use_target_directory() { - let dir = tempfile::tempdir().expect("temp dir"); - let target = dir.path().join("cache.ics"); - let (first_temp, first_output) = super::create_atomic_download_file(&target).expect("first temp file"); - let (second_temp, second_output) = super::create_atomic_download_file(&target).expect("second temp file"); - - assert_ne!(first_temp.path(), second_temp.path()); - assert_eq!(first_temp.path().parent(), Some(dir.path())); - assert_eq!(second_temp.path().parent(), Some(dir.path())); - - drop(first_output); - drop(second_output); - drop(first_temp); - drop(second_temp); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); - } - - fn make_provider_with_dns( - keep_vhost: bool, - on_connect_error: OnConnectErrorPolicy, - ips: Vec<&str>, - ) -> Arc { - let parsed_ips = ips.into_iter().map(|raw| raw.parse().expect("ip must parse")).collect::>(); - let dto = ConfigProviderDto { - name: "provider-a".into(), - urls: vec!["http://example.com".into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), - dns: Some(ProviderDnsDto { - enabled: true, - schemes: Some(vec![DnsScheme::Http, DnsScheme::Https]), - keep_vhost, - overrides: Some(HashMap::from([("example.com".to_string(), parsed_ips)])), - on_connect_error, - ..ProviderDnsDto::default() - }), - }; - Arc::new(ConfigProvider::from(&dto)) - } - - #[test] - fn test_url_mask() { - // Replace with "***" - let query = "https://bubblegum.tv/live/username/password/2344"; - let masked = sanitize_sensitive_info(query); - println!("{masked}"); - } - - #[test] - fn test_replace_ext() { - let tests = [ - ("http://hello.world.com", "http://hello.world.com"), - ("http://hello.world.com/123", "http://hello.world.com/123.mp4"), - ("http://hello.world.com/123.ts?hello=world", "http://hello.world.com/123.mp4?hello=world"), - ("http://hello.world.com/123?hello=world", "http://hello.world.com/123.mp4?hello=world"), - ("http://hello.world.com/123#hello=world", "http://hello.world.com/123.mp4#hello=world"), - ]; - - for (test, expect) in &tests { - assert_eq!(replace_url_extension(test, ".mp4"), *expect); - } - } - - #[test] - fn tes_base_url() { - let url = "http://my.provider.com:8080/xmltv?username=hello"; - let expected = "http://my.provider.com:8080"; - assert_eq!(get_base_url_from_str(url).unwrap(), expected); - } - - #[test] - fn test_get_request_headers_prioritization() { - use super::{append_user_agent_stream_index, get_request_headers, overlay_upstream_user_agent}; - use axum::http::header::USER_AGENT; - - // Case 1: No headers provided -> Default UA - let headers = get_request_headers::(None, None, None, None); - assert_eq!(headers.get(USER_AGENT).unwrap(), DEFAULT_USER_AGENT); - - // Case 2: No headers provided but config default UA set -> Config default UA - let headers = - get_request_headers::(None, None, None, Some("Config-Default-UA")); - assert_eq!(headers.get(USER_AGENT).unwrap(), "Config-Default-UA"); - - // Case 3: Only client header -> Client UA (overrides config default UA) - let mut client_headers = HashMap::new(); - client_headers.insert("User-Agent".to_string(), b"Client-UA".to_vec()); - let headers = get_request_headers(None, Some(&client_headers), None, Some("Config-Default-UA")); - assert_eq!(headers.get(USER_AGENT).unwrap(), "Client-UA"); - - // Case 4: Both config and client -> Config UA overrides - let mut config_headers = HashMap::new(); - config_headers.insert("User-Agent".to_string(), "Config-UA".to_string()); - let headers = - get_request_headers(Some(&config_headers), Some(&client_headers), None, Some("Config-Default-UA")); - assert_eq!(headers.get(USER_AGENT).unwrap(), "Config-UA"); - - // Case 5: Other headers also prioritized - config_headers.insert("X-Test".to_string(), "From-Config".to_string()); - let mut client_headers = HashMap::new(); - client_headers.insert("X-Test".to_string(), b"From-Client".to_vec()); - let headers = - get_request_headers(Some(&config_headers), Some(&client_headers), None, Some("Config-Default-UA")); - assert_eq!(headers.get("X-Test").unwrap(), "From-Config"); - - let mut headers = - get_request_headers(Some(&config_headers), Some(&client_headers), None, Some("Config-Default-UA")); - overlay_upstream_user_agent(&mut headers, Some("Channel-UA"), None); - assert_eq!(headers.get(USER_AGENT).unwrap(), "Channel-UA"); - - let disabled = ReverseProxyDisabledHeaderConfig { - referer_header: false, - x_header: false, - cloudflare_header: false, - custom_header: vec!["User-Agent".to_string()], - }; - let mut headers = get_request_headers( - Some(&config_headers), - Some(&client_headers), - Some(&disabled), - Some("Config-Default-UA"), - ); - overlay_upstream_user_agent(&mut headers, Some("Blocked-Channel-UA"), Some(&disabled)); - assert!(!headers.contains_key(USER_AGENT)); - - let mut headers = - get_request_headers::(None, None, None, Some("VLC/3.0")); - append_user_agent_stream_index(&mut headers, 42); - assert_eq!(headers.get(USER_AGENT).and_then(|value| value.to_str().ok()), Some("VLC/3.0 42")); - append_user_agent_stream_index(&mut headers, 42); - assert_eq!(headers.get(USER_AGENT).and_then(|value| value.to_str().ok()), Some("VLC/3.0 42")); - - let mut headers = HeaderMap::new(); - let extended_user_agent = HeaderValue::from_bytes(b"Receiver/1.0 \xE4"); - assert!(extended_user_agent.is_ok()); - if let Ok(value) = extended_user_agent { - headers.insert(USER_AGENT, value); - } - append_user_agent_stream_index(&mut headers, 7); - assert_eq!(headers.get(USER_AGENT).map(HeaderValue::as_bytes), Some(b"Receiver/1.0 \xE4 7".as_slice())); - } - - #[test] - fn test_same_origin_checks_scheme_host_and_port() { - let a = Url::parse("https://example.com/path").expect("url parse should work"); - let b = Url::parse("https://example.com/other").expect("url parse should work"); - let c = Url::parse("http://example.com/other").expect("url parse should work"); - let d = Url::parse("https://example.com:8443/other").expect("url parse should work"); - - assert!(same_origin(&a, &b)); - assert!(!same_origin(&a, &c)); - assert!(!same_origin(&a, &d)); - } - - #[test] - fn test_cross_origin_redirect_strips_sensitive_headers() { - let mut headers = HashMap::new(); - headers.insert("Authorization".to_string(), "Bearer test".to_string()); - headers.insert("Cookie".to_string(), "sid=123".to_string()); - headers.insert("Proxy-Authorization".to_string(), "Basic abc".to_string()); - headers.insert("Host".to_string(), "old.host".to_string()); - headers.insert("X-API-Key".to_string(), "secret".to_string()); - headers.insert("Accept".to_string(), "application/x-mpegurl".to_string()); - headers.insert("User-Agent".to_string(), "mpv".to_string()); - - strip_sensitive_headers_for_cross_origin_redirect(&mut headers); - - assert!(!headers.contains_key("Authorization")); - assert!(!headers.contains_key("Cookie")); - assert!(!headers.contains_key("Proxy-Authorization")); - assert!(!headers.contains_key("Host")); - assert!(!headers.contains_key("X-API-Key")); - assert_eq!(headers.get("Accept").map(String::as_str), Some("application/x-mpegurl")); - assert_eq!(headers.get("User-Agent").map(String::as_str), Some("mpv")); - } - - #[test] - fn test_cross_origin_redirect_header_allowlist_is_minimal() { - assert!(is_safe_cross_origin_redirect_header("accept")); - assert!(is_safe_cross_origin_redirect_header("user-agent")); - assert!(is_safe_cross_origin_redirect_header("icy-metadata")); - assert!(!is_safe_cross_origin_redirect_header("authorization")); - assert!(!is_safe_cross_origin_redirect_header("cookie")); - assert!(!is_safe_cross_origin_redirect_header("x-api-key")); - assert!(!is_safe_cross_origin_redirect_header("x-auth-token")); - } - - #[tokio::test] - async fn local_epg_file_respects_max_download_bytes() { - let dir = tempfile::tempdir().expect("temp dir"); - let source = dir.path().join("large.ics"); - let persist = dir.path().join("cache.ics"); - tokio::fs::write(&source, b"0123456789").await.expect("write source"); - tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); - let app_config = make_test_app_config(Config::default()); - let client = make_epg_test_client(); - let input = ConfigInput::default(); - - let err = get_input_epg_content_as_file( - &app_config, - &client, - &input, - InputEpgFileRequest { - headers: None, - storage_dir: dir.path().to_string_lossy().as_ref(), - url: source.to_string_lossy().as_ref(), - persist_path: &persist, - max_bytes: Some(4), - }, - ) - .await - .expect_err("size limit should fail"); - - assert!(err.to_string().contains("exceeds configured limit")); - assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); - } - - #[tokio::test] - async fn file_url_epg_source_is_copied_to_persist_path() { - let dir = tempfile::tempdir().expect("temp dir"); - let source = dir.path().join("source.ics"); - let persist = dir.path().join("cache.ics"); - tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n").await.expect("write source"); - tokio::fs::write(&persist, b"old cache").await.expect("write old cache"); - let source_url = Url::from_file_path(&source).expect("file url"); - let app_config = make_test_app_config(Config::default()); - let client = make_epg_test_client(); - let input = ConfigInput::default(); - - let result = get_input_epg_content_as_file( - &app_config, - &client, - &input, - InputEpgFileRequest { - headers: None, - storage_dir: dir.path().to_string_lossy().as_ref(), - url: source_url.as_str(), - persist_path: &persist, - max_bytes: Some(1024), - }, - ) - .await - .expect("download"); - - assert_eq!(result, persist); - assert_eq!( - tokio::fs::read_to_string(&persist).await.expect("persisted content"), - "BEGIN:VCALENDAR\nEND:VCALENDAR\n" - ); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); - } - - #[tokio::test] - async fn file_url_epg_source_respects_streamed_size_limit() { - let dir = tempfile::tempdir().expect("temp dir"); - let source = dir.path().join("large.ics"); - let persist = dir.path().join("cache.ics"); - tokio::fs::write(&source, b"0123456789").await.expect("write source"); - tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); - let source_url = Url::from_file_path(&source).expect("file url"); - let app_config = make_test_app_config(Config::default()); - let client = make_epg_test_client(); - let input = ConfigInput::default(); - - let err = get_input_epg_content_as_file( - &app_config, - &client, - &input, - InputEpgFileRequest { - headers: None, - storage_dir: dir.path().to_string_lossy().as_ref(), - url: source_url.as_str(), - persist_path: &persist, - max_bytes: Some(4), - }, - ) - .await - .expect_err("size limit should fail"); - - assert!(err.to_string().contains("exceeds configured limit")); - assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); - } - - #[tokio::test] - async fn local_epg_replace_error_cleans_temp_file() { - let dir = tempfile::tempdir().expect("temp dir"); - let source = dir.path().join("source.ics"); - let persist = dir.path().join("cache.ics"); - tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n").await.expect("write source"); - tokio::fs::create_dir(&persist).await.expect("create conflicting destination"); - let app_config = make_test_app_config(Config::default()); - let client = make_epg_test_client(); - let input = ConfigInput::default(); - - get_input_epg_content_as_file( - &app_config, - &client, - &input, - InputEpgFileRequest { - headers: None, - storage_dir: dir.path().to_string_lossy().as_ref(), - url: source.to_string_lossy().as_ref(), - persist_path: &persist, - max_bytes: Some(1024), - }, - ) - .await - .expect_err("replacing a directory should fail"); - - assert!(persist.is_dir()); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); - } - - #[tokio::test] - async fn local_epg_read_error_preserves_existing_cache_and_cleans_temp_file() { - let dir = tempfile::tempdir().expect("temp dir"); - let source = dir.path().join("source-directory"); - let persist = dir.path().join("cache.ics"); - tokio::fs::create_dir(&source).await.expect("create source directory"); - tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); - let app_config = make_test_app_config(Config::default()); - let client = make_epg_test_client(); - let input = ConfigInput::default(); - - get_input_epg_content_as_file( - &app_config, - &client, - &input, - InputEpgFileRequest { - headers: None, - storage_dir: dir.path().to_string_lossy().as_ref(), - url: source.to_string_lossy().as_ref(), - persist_path: &persist, - max_bytes: Some(1024), - }, - ) - .await - .expect_err("reading a directory as an EPG file should fail"); - - assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); - } - - #[tokio::test] - async fn unsupported_epg_url_scheme_is_rejected_without_network_access() { - let dir = tempfile::tempdir().expect("temp dir"); - let persist = dir.path().join("cache.ics"); - let app_config = make_test_app_config(Config::default()); - let client = make_epg_test_client(); - let input = ConfigInput::default(); - - let err = get_input_epg_content_as_file( - &app_config, - &client, - &input, - InputEpgFileRequest { - headers: None, - storage_dir: dir.path().to_string_lossy().as_ref(), - url: "ftp://example.com/calendar.ics", - persist_path: &persist, - max_bytes: Some(1024), - }, - ) - .await - .expect_err("unsupported scheme should fail"); - - assert!(err.to_string().contains("Unsupported EPG URL scheme 'ftp'")); - assert!(!persist.exists()); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); - } - - #[cfg(windows)] - #[tokio::test] - async fn absolute_windows_epg_path_is_dispatched_as_local_file() { - let dir = tempfile::tempdir().expect("temp dir"); - let source = dir.path().join("source.ics"); - let persist = dir.path().join("cache.ics"); - tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n").await.expect("write source"); - let app_config = make_test_app_config(Config::default()); - let client = make_epg_test_client(); - let input = ConfigInput::default(); - let source_path = source.to_str().expect("Windows temp path should be UTF-8"); - assert!(source_path.contains(':')); - - get_input_epg_content_as_file( - &app_config, - &client, - &input, - InputEpgFileRequest { - headers: None, - storage_dir: dir.path().to_string_lossy().as_ref(), - url: source_path, - persist_path: &persist, - max_bytes: Some(1024), - }, - ) - .await - .expect("absolute Windows path should be copied as a local file"); - - assert_eq!(tokio::fs::read(&persist).await.expect("read cache"), b"BEGIN:VCALENDAR\nEND:VCALENDAR\n"); - } - - #[tokio::test] - async fn remote_epg_size_limit_preserves_existing_cache_and_cleans_temp_file() { - let (addr, _accepted, server_handle) = match start_plain_http_server_with_body(b"0123456789").await { - Ok(server) => server, - Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { - eprintln!("skipping remote_epg_size_limit_preserves_existing_cache_and_cleans_temp_file: {err}"); - return; - } - Err(err) => panic!("failed to start test server: {err}"), - }; - let dir = tempfile::tempdir().expect("temp dir"); - let persist = dir.path().join("cache.ics"); - tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); - let url = format!("http://{addr}/calendar.ics"); - let app_config = make_test_app_config(Config::default()); - let client = make_epg_test_client(); - let input = ConfigInput::default(); - - let err = get_input_epg_content_as_file( - &app_config, - &client, - &input, - InputEpgFileRequest { - headers: None, - storage_dir: dir.path().to_string_lossy().as_ref(), - url: &url, - persist_path: &persist, - max_bytes: Some(4), - }, - ) - .await - .expect_err("remote size limit should fail"); - - assert!(err.to_string().contains("exceeds configured limit")); - assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); - server_handle.abort(); - } - - #[tokio::test] - async fn remote_epg_stream_error_preserves_existing_cache_and_cleans_temp_file() { - let response = "HTTP/1.1 200 OK\r\nContent-Length: 64\r\nConnection: close\r\n\r\ntruncated".to_string(); - let (addr, _accepted, server_handle) = match start_plain_http_server_with_response(response).await { - Ok(server) => server, - Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { - eprintln!("skipping remote_epg_stream_error_preserves_existing_cache_and_cleans_temp_file: {err}"); - return; - } - Err(err) => panic!("failed to start test server: {err}"), - }; - let dir = tempfile::tempdir().expect("temp dir"); - let persist = dir.path().join("cache.ics"); - tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); - let url = format!("http://{addr}/calendar.ics"); - let app_config = make_test_app_config(Config::default()); - let client = make_epg_test_client(); - let input = ConfigInput::default(); - - get_input_epg_content_as_file( - &app_config, - &client, - &input, - InputEpgFileRequest { - headers: None, - storage_dir: dir.path().to_string_lossy().as_ref(), - url: &url, - persist_path: &persist, - max_bytes: Some(1024), - }, - ) - .await - .expect_err("truncated response body should fail"); - - assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); - server_handle.abort(); - } - - #[tokio::test] - async fn parallel_epg_downloads_to_same_target_are_serialized_and_replace_atomically() { - let (addr, accepted, max_active, server_handle) = - match start_delayed_http_server_with_body(b"BEGIN:VCALENDAR\nEND:VCALENDAR\n").await { - Ok(server) => server, - Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { - eprintln!( - "skipping parallel_epg_downloads_to_same_target_are_serialized_and_replace_atomically: {err}" - ); - return; - } - Err(err) => panic!("failed to start test server: {err}"), - }; - let dir = tempfile::tempdir().expect("temp dir"); - let persist = dir.path().join("cache.ics"); - tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); - let url = format!("http://{addr}/calendar.ics"); - let app_config = make_test_app_config(Config::default()); - let client = make_epg_test_client(); - let input = ConfigInput::default(); - let storage_dir = dir.path().to_string_lossy(); - - let first = get_input_epg_content_as_file( - &app_config, - &client, - &input, - InputEpgFileRequest { - headers: None, - storage_dir: storage_dir.as_ref(), - url: &url, - persist_path: &persist, - max_bytes: Some(1024), - }, - ); - let second = get_input_epg_content_as_file( - &app_config, - &client, - &input, - InputEpgFileRequest { - headers: None, - storage_dir: storage_dir.as_ref(), - url: &url, - persist_path: &persist, - max_bytes: Some(1024), - }, - ); - let (first_result, second_result) = tokio::join!(first, second); - - assert_eq!(first_result.expect("first download"), persist); - assert_eq!(second_result.expect("second download"), persist); - assert_eq!(accepted.load(Ordering::SeqCst), 2); - assert_eq!(max_active.load(Ordering::SeqCst), 1); - assert_eq!(tokio::fs::read(&persist).await.expect("read refreshed cache"), b"BEGIN:VCALENDAR\nEND:VCALENDAR\n"); - assert!(atomic_download_temp_files(dir.path()).await.is_empty()); - server_handle.abort(); - } - - #[test] - fn test_keep_vhost_false_uses_ip_host_header_for_http() { - let provider = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1"]); - let url = Url::parse("http://example.com:8080/stream").expect("url parse should work"); - - let target = resolve_attempt_target(&url, Some(&provider)); - assert_eq!(target.effective_url.host_str(), Some("192.168.0.1")); - assert_eq!(target.host_header.as_deref(), Some("192.168.0.1:8080")); - } - - #[test] - fn test_keep_vhost_true_uses_hostname_host_header_for_http() { - let provider = make_provider_with_dns(true, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1"]); - let url = Url::parse("http://example.com:8080/stream").expect("url parse should work"); - - let target = resolve_attempt_target(&url, Some(&provider)); - assert_eq!(target.effective_url.host_str(), Some("192.168.0.1")); - assert_eq!(target.host_header.as_deref(), Some("example.com:8080")); - } - - #[test] - fn test_preview_request_diagnostics_for_logging_includes_effective_target_and_host_details() { - let provider = make_provider_with_dns(true, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1"]); - let url = Url::parse("http://example.com:8080/stream").expect("url parse should work"); - - let diagnostics = preview_request_diagnostics_for_logging(&url, Some(&provider)); - - assert_eq!( - diagnostics, - "request_url=http://***/stream, effective_url=http://***/stream, host_header=example.com:8080, connect_ip=0.***" - ); - } - - #[test] - fn test_preview_request_diagnostics_for_logging_sanitizes_each_stream_url() -> Result<(), url::ParseError> { - let provider = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1"]); - let url = Url::parse("http://example.com/live/abcd/efgh/1092671.ts")?; - - let diagnostics = preview_request_diagnostics_for_logging(&url, Some(&provider)); - - assert!(!diagnostics.contains("example.com")); - assert!(!diagnostics.contains("abcd")); - assert!(!diagnostics.contains("efgh")); - assert_eq!( - diagnostics, - "request_url=http://***/live/***/1092671.ts, effective_url=http://***/live/***/1092671.ts, host_header=0.***, connect_ip=0.***" - ); - Ok(()) - } - - #[test] - fn test_http_attempt_uses_bracketed_ipv6_target_for_logging_and_request_url() { - let provider = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["2a06:98c1:3121::3"]); - let url = Url::parse("http://example.com/live/stream.ts").expect("url parse should work"); - - let preview = preview_request_target_for_logging(&url, Some(&provider)); - let target = resolve_attempt_target(&url, Some(&provider)); - - assert_eq!(preview, "http://[2a06:98c1:3121::3]/live/stream.ts"); - assert_eq!(target.effective_url.as_str(), "http://[2a06:98c1:3121::3]/live/stream.ts"); - assert_eq!(target.host_header.as_deref(), Some("[2a06:98c1:3121::3]")); - } - - #[test] - fn test_https_attempt_keeps_hostname_and_sets_sni() { - let provider = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1"]); - let url = Url::parse("https://example.com/live").expect("url parse should work"); - - let target = resolve_attempt_target(&url, Some(&provider)); - assert_eq!(target.effective_url.host_str(), Some("example.com")); - assert_eq!(target.sni_host.as_deref(), Some("example.com")); - assert_eq!(target.connect_ip.map(|ip| ip.to_string()), Some("192.168.0.1".to_string())); - assert_eq!(target.host_header.as_deref(), Some("192.168.0.1")); - } - - #[test] - fn test_try_next_ip_policy_uses_next_ip_until_exhausted() { - let provider = - make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "192.168.0.2"]); - let url = Url::parse("http://example.com/live").expect("url parse should work"); - let mut tried = HashSet::new(); - - let first = resolve_attempt_target(&url, Some(&provider)); - let second = resolve_attempt_target(&url, Some(&provider)); - - assert!(should_try_next_ip_on_connect_error(Some(&provider), &first, &mut tried)); - assert!(!should_try_next_ip_on_connect_error(Some(&provider), &second, &mut tried)); - } - - #[test] - fn test_preview_request_target_for_logging_does_not_advance_dns_rotation() { - let provider = - make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "192.168.0.2"]); - let url = Url::parse("http://example.com/live").expect("url parse should work"); - - let preview = preview_request_target_for_logging(&url, Some(&provider)); - let first = resolve_attempt_target(&url, Some(&provider)); - let second = resolve_attempt_target(&url, Some(&provider)); - - assert_eq!(preview, "http://192.168.0.1/live"); - assert_eq!(first.connect_ip.map(|ip| ip.to_string()), Some("192.168.0.1".to_string())); - assert_eq!(second.connect_ip.map(|ip| ip.to_string()), Some("192.168.0.2".to_string())); - - let provider_https = - make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "192.168.0.2"]); - let https_url = Url::parse("https://example.com/live").expect("url parse should work"); - - let https_preview = preview_request_target_for_logging(&https_url, Some(&provider_https)); - let https_first = resolve_attempt_target(&https_url, Some(&provider_https)); - let https_second = resolve_attempt_target(&https_url, Some(&provider_https)); - - assert_eq!(https_preview, "https://example.com/live (connect_ip=192.168.0.1)"); - assert_eq!(https_first.connect_ip.map(|ip| ip.to_string()), Some("192.168.0.1".to_string())); - assert_eq!(https_second.connect_ip.map(|ip| ip.to_string()), Some("192.168.0.2".to_string())); - } - - #[test] - fn test_preview_request_target_for_logging_uses_preferred_provider_index() { - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "provider-a".into(), - urls: vec!["http://provider-a.example".into(), "http://provider-b.example".into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), - dns: None, - })); - provider.set_current_index(1); - - let url = Url::parse("provider://provider-a/live").expect("provider url should parse"); - let preview = preview_request_target_for_logging(&url, Some(&provider)); - - assert_eq!(preview, "http://provider-b.example/live"); - } - - async fn start_plain_http_server_with_body( - body: &'static [u8], - ) -> std::io::Result<(SocketAddr, Arc, tokio::task::JoinHandle<()>)> { - let listener = TcpListener::bind("127.0.0.1:0").await?; - let addr = listener.local_addr()?; - let accepted = Arc::new(AtomicUsize::new(0)); - let accepted_clone = Arc::clone(&accepted); - let content_length = body.len(); - - let handle = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - continue; - }; - accepted_clone.fetch_add(1, Ordering::SeqCst); - let body = body; - tokio::spawn(async move { - let mut buf = vec![0_u8; 2048]; - let _ = socket.read(&mut buf).await; - let response_head = - format!("HTTP/1.1 200 OK\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n"); - let _ = socket.write_all(response_head.as_bytes()).await; - let _ = socket.write_all(body).await; - let _ = socket.shutdown().await; - }); - } - }); - - Ok((addr, accepted, handle)) - } - - async fn start_delayed_http_server_with_body( - body: &'static [u8], - ) -> std::io::Result<(SocketAddr, Arc, Arc, tokio::task::JoinHandle<()>)> { - let listener = TcpListener::bind("127.0.0.1:0").await?; - let addr = listener.local_addr()?; - let accepted = Arc::new(AtomicUsize::new(0)); - let active = Arc::new(AtomicUsize::new(0)); - let max_active = Arc::new(AtomicUsize::new(0)); - let accepted_clone = Arc::clone(&accepted); - let active_clone = Arc::clone(&active); - let max_active_clone = Arc::clone(&max_active); - let content_length = body.len(); - - let handle = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - continue; - }; - accepted_clone.fetch_add(1, Ordering::SeqCst); - let active = Arc::clone(&active_clone); - let max_active = Arc::clone(&max_active_clone); - tokio::spawn(async move { - let active_count = active.fetch_add(1, Ordering::SeqCst) + 1; - max_active.fetch_max(active_count, Ordering::SeqCst); - let mut buf = vec![0_u8; 2048]; - let _ = socket.read(&mut buf).await; - tokio::time::sleep(Duration::from_millis(100)).await; - let response_head = - format!("HTTP/1.1 200 OK\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n"); - let _ = socket.write_all(response_head.as_bytes()).await; - let _ = socket.write_all(body).await; - let _ = socket.shutdown().await; - active.fetch_sub(1, Ordering::SeqCst); - }); - } - }); - - Ok((addr, accepted, max_active, handle)) - } - - async fn start_plain_http_server_with_response( - response: String, - ) -> std::io::Result<(SocketAddr, Arc, tokio::task::JoinHandle<()>)> { - let listener = TcpListener::bind("127.0.0.1:0").await?; - let addr = listener.local_addr()?; - let accepted = Arc::new(AtomicUsize::new(0)); - let accepted_clone = Arc::clone(&accepted); - - let handle = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - continue; - }; - accepted_clone.fetch_add(1, Ordering::SeqCst); - let response = response.clone(); - tokio::spawn(async move { - let mut buf = vec![0_u8; 2048]; - let _ = socket.read(&mut buf).await; - let _ = socket.write_all(response.as_bytes()).await; - let _ = socket.shutdown().await; - }); - } - }); - - Ok((addr, accepted, handle)) - } - - async fn start_plain_http_server() -> std::io::Result<(SocketAddr, Arc, tokio::task::JoinHandle<()>)> { - start_plain_http_server_with_body(b"ok").await - } - - #[tokio::test] - async fn public_resolver_rejects_loopback_at_connection_time() { - let (address, accepted, server) = match start_plain_http_server().await { - Ok(server) => server, - Err(err) if err.kind() == ErrorKind::PermissionDenied => return, - Err(err) => panic!("failed to start test server: {err}"), - }; - let client = reqwest::Client::builder() - .no_proxy() - .dns_resolver(PublicIpResolver) - .build() - .expect("public-only client should build"); - - let result = client.get(format!("http://localhost:{}/", address.port())).send().await; - - assert!(result.is_err()); - assert_eq!(accepted.load(Ordering::SeqCst), 0); - server.abort(); - } - - async fn start_recording_http_server( - responses: Vec, - ) -> std::io::Result<(SocketAddr, Arc>>, tokio::task::JoinHandle<()>)> { - start_recording_http_byte_server(responses.into_iter().map(String::into_bytes).collect()).await - } - - async fn start_recording_http_byte_server( - responses: Vec>, - ) -> std::io::Result<(SocketAddr, Arc>>, tokio::task::JoinHandle<()>)> { - let listener = TcpListener::bind("127.0.0.1:0").await?; - let addr = listener.local_addr()?; - let requests = Arc::new(Mutex::new(Vec::new())); - let task_requests = Arc::clone(&requests); - let responses = Arc::new(responses); - let response_index = Arc::new(AtomicUsize::new(0)); - - let handle = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let requests = Arc::clone(&task_requests); - let responses = Arc::clone(&responses); - let response_index = Arc::clone(&response_index); - tokio::spawn(async move { - let mut request = Vec::new(); - loop { - let mut chunk = [0_u8; 2048]; - let Ok(read) = socket.read(&mut chunk).await else { - return; - }; - if read == 0 { - return; - } - request.extend_from_slice(&chunk[..read]); - if request.windows(4).any(|window| window == b"\r\n\r\n") || request.len() >= 16 * 1024 { - break; - } - } - requests.lock().await.push(String::from_utf8_lossy(&request).into_owned()); - let index = response_index.fetch_add(1, Ordering::SeqCst); - let Some(response) = responses.get(index).or_else(|| responses.last()) else { - return; - }; - let _ = socket.write_all(response).await; - let _ = socket.shutdown().await; - }); - } - }); - - Ok((addr, requests, handle)) - } - - async fn start_hanging_http_server( - ) -> std::io::Result<(SocketAddr, oneshot::Receiver<()>, tokio::task::JoinHandle<()>)> { - let listener = TcpListener::bind("127.0.0.1:0").await?; - let addr = listener.local_addr()?; - let (request_seen_tx, request_seen_rx) = oneshot::channel(); - let handle = tokio::spawn(async move { - let Ok((mut socket, _)) = listener.accept().await else { - return; - }; - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 2048]; - let Ok(read) = socket.read(&mut chunk).await else { - return; - }; - if read == 0 { - return; - } - request.extend_from_slice(&chunk[..read]); - } - let _ = request_seen_tx.send(()); - std::future::pending::<()>().await; - drop(socket); - }); - - Ok((addr, request_seen_rx, handle)) - } - - fn response_with_body(status: &str, body: &str) -> String { - format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()) - } - - fn response_with_byte_body(status: &str, body: &[u8]) -> Vec { - let mut response = - format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).into_bytes(); - response.extend_from_slice(body); - response - } - - fn gzip_encoded(body: &[u8]) -> Vec { - let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); - encoder.write_all(body).expect("gzip input"); - encoder.finish().expect("gzip output") - } - - fn zlib_encoded(body: &[u8]) -> Vec { - let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); - encoder.write_all(body).expect("zlib input"); - encoder.finish().expect("zlib output") - } - - fn request_header_value<'a>(request: &'a str, expected_name: &str) -> Option<&'a str> { - request.lines().find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case(expected_name).then_some(value.trim()) - }) - } - - fn test_retry_config(max_attempts: u32) -> Arc { - let mut config = Config { connect_timeout_secs: 1, ..Config::default() }; - config.reverse_proxy = Some(ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: ResourceRetryConfig { - max_attempts, - backoff_millis: 1, - backoff_multiplier: 1.0, - ..ResourceRetryConfig::default() - }, - disabled_header: None, - stream: None, - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }); - make_test_app_config(config) - } - - #[test] - fn content_type_from_ext_maps_fragmented_mp4_extensions_case_insensitively() { - for (ext, expected) in [ - ("mp4", "video/mp4"), - ("FMP4", "video/mp4"), - ("m4s", "video/mp4"), - ("m4v", "video/mp4"), - ("cmfv", "video/mp4"), - ("m4a", "audio/mp4"), - ("CMFA", "audio/mp4"), - ("ts", "video/mp2t"), - ("mkv", "video/x-matroska"), - ("bin", "application/octet-stream"), - ("", "application/octet-stream"), - ] { - assert_eq!(super::content_type_from_ext(ext), expected, "{ext}"); - } - } - - #[tokio::test] - async fn http_error_response_option_preserves_final_status_headers_and_configured_retries( - ) -> Result<(), Box> { - let app_config = test_retry_config(3); - let client = reqwest::Client::builder().no_proxy().build()?; - for failover_only in [false, true] { - for status in [reqwest::StatusCode::FORBIDDEN, reqwest::StatusCode::SERVICE_UNAVAILABLE] { - let response = format!( - "HTTP/1.1 {}\r\nRetry-After: 0\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", - super::format_http_status(status) - ); - let (addr, accepted, server) = start_plain_http_server_with_response(response).await?; - let url = Url::parse(&format!("http://{addr}/init.hls.fmp4"))?; - let mut options = RequestFetchOptions::default().with_http_error_responses(true); - if failover_only { - options = options.without_resource_retries(); - } - let response = super::send_with_retry_and_provider_policy_with_options( - &app_config, - &url, - None, - false, - true, - options, - |resolved| client.get(resolved.clone()), - ) - .await?; - assert_eq!(response.status(), status); - assert_eq!(response.headers()[reqwest::header::RETRY_AFTER], "0"); - assert_eq!( - accepted.load(Ordering::SeqCst), - if !failover_only && status.is_server_error() { 3 } else { 1 } - ); - server.abort(); - } - } - Ok(()) - } - - #[tokio::test] - async fn http_error_response_option_recovers_transient_statuses_in_both_request_paths( - ) -> Result<(), Box> { - let client = reqwest::Client::builder().no_proxy().build()?; - for manual_redirects in [false, true] { - for status in ["503 Service Unavailable", "429 Too Many Requests", "408 Request Timeout"] { - let (addr, requests, server) = start_recording_http_server(vec![ - response_with_body(status, ""), - response_with_body("200 OK", "media"), - ]) - .await?; - let url = Url::parse(&format!("http://{addr}/init.hls.fmp4"))?; - let app_config = test_retry_config(2); - let options = RequestFetchOptions::default().with_http_error_responses(true); - let response = if manual_redirects { - send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( - &app_config, - &client, - &test_input_source(url.to_string(), None), - None, - &url, - 10, - options, - ) - .await? - .response - } else { - super::send_with_retry_and_provider_policy_with_options( - &app_config, - &url, - None, - false, - true, - options, - |resolved| client.get(resolved.clone()), - ) - .await? - }; - assert_eq!(response.status(), reqwest::StatusCode::OK); - assert_eq!(response.bytes().await?.as_ref(), b"media"); - assert_eq!(requests.lock().await.len(), 2); - server.abort(); - } - } - Ok(()) - } - - fn identity_fetch_options() -> RequestFetchOptions { - RequestFetchOptions::with_attempt_idle_timeout(Duration::from_secs(1)) - .with_content_coding(OutboundContentCodingPolicy::Identity) - } - - #[test] - fn physical_request_attempt_applies_target_and_final_content_coding_options() { - let client = reqwest::Client::builder().no_proxy().build().expect("test client"); - let request_url = Url::parse("http://origin.example/manifest.m3u8").expect("request URL"); - let effective_url = Url::parse("http://127.0.0.1/manifest.m3u8").expect("effective URL"); - let target = super::AttemptTarget { - request_url: request_url.clone(), - effective_url: effective_url.clone(), - host_header: Some("origin.example".to_string()), - sni_host: None, - connect_ip: Some("127.0.0.1".parse().expect("connect IP")), - dns_host: Some("origin.example".to_string()), - }; - let builder = - client.get(request_url).header(ACCEPT_ENCODING, "gzip").header(reqwest::header::HOST, "wrong.example"); - - let (_, request) = super::prepare_physical_request_attempt(builder, &target, identity_fetch_options()) - .expect("physical request should build"); - - assert_eq!(request.url(), &effective_url); - assert_eq!(request.headers()[reqwest::header::HOST], "origin.example"); - assert_eq!(request.headers()[ACCEPT_ENCODING], "identity"); - assert_eq!(request.timeout(), Some(&Duration::from_secs(1))); - } - - #[tokio::test(start_paused = true)] - async fn provider_failover_only_default_options_keep_default_idle_timeout() { - let (addr, request_seen, server) = start_hanging_http_server().await.expect("hanging origin should start"); - let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); - let input = test_input_source(url.to_string(), None); - let app_config = test_retry_config(5); - let client = reqwest::Client::builder().no_proxy().build().expect("test client"); - let request_url = url.clone(); - let request = tokio::spawn(async move { - send_input_with_retry_and_provider_policy_with_options_result( - &app_config, - &client, - &input, - None, - &request_url, - RequestFetchOptions::default().without_resource_retries(), - ) - .await - }); - - // A ready task prevents Tokio's paused clock from auto-advancing through the default timeout while the local - // TCP handshake is still in progress. After the request arrives, this guard stops and the test advances time - // explicitly to the production deadline. - let hold_virtual_time = Arc::new(AtomicBool::new(true)); - let hold_virtual_time_for_task = Arc::clone(&hold_virtual_time); - let virtual_time_guard = tokio::spawn(async move { - while hold_virtual_time_for_task.load(Ordering::SeqCst) { - tokio::task::yield_now().await; - } - }); - let request_seen = request_seen.await; - hold_virtual_time.store(false, Ordering::SeqCst); - virtual_time_guard.await.expect("virtual time guard should stop"); - request_seen.expect("origin should receive the request"); - tokio::time::advance(Duration::from_secs(STREAM_IDLE_TIMEOUT + 1)).await; - tokio::task::yield_now().await; - if !request.is_finished() { - request.abort(); - panic!("provider-only request lost the default idle-timeout guard"); - } - - let Err(error) = request.await.expect("request task should join") else { - panic!("hanging request must time out"); - }; - assert_eq!(error.kind(), ErrorKind::TimedOut); - server.abort(); - } - - #[test] - fn content_coding_prefix_read_error_is_retryable_but_unsupported_coding_is_not() { - let prefix_read = - Error::other(ContentCodingError::PrefixRead(Error::new(ErrorKind::UnexpectedEof, "prefix truncated"))); - let unsupported = Error::other(ContentCodingError::Unsupported("compress".to_string())); - - assert!(should_retry_text_body_error(&prefix_read)); - assert!(!should_retry_text_body_error(&unsupported)); - } - - #[test] - fn text_response_error_log_labels_never_expose_origin_controlled_details() { - let unsupported = Error::other(ContentCodingError::Unsupported("signed-token-secret".to_string())); - assert_eq!(text_response_error_log_label(&unsupported), "content_coding class=unsupported"); - assert!(!text_response_error_log_label(&unsupported).contains("signed-token-secret")); - - let decoding = Error::new(ErrorKind::InvalidData, ContentDecodingIoError { coding: ContentCoding::Zstd }); - assert_eq!(text_response_error_log_label(&decoding), "content_decoding coding=zstd"); - } - - fn test_input_source(url: String, provider: Option>) -> InputSource { - InputSource { - name: Arc::from("test"), - url, - provider, - username: None, - password: None, - method: InputFetchMethod::GET, - headers: HashMap::new(), - } - } - - #[tokio::test] - async fn text_content_download_decodes_headerless_gzip_and_zlib() { - const TEXT: &[u8] = b"headerless legacy text\n"; - - for (label, encoded) in [("gzip", gzip_encoded(TEXT)), ("zlib", zlib_encoded(TEXT))] { - let (addr, requests, handle) = - start_recording_http_byte_server(vec![response_with_byte_body("200 OK", &encoded)]) - .await - .expect("recording origin should start"); - let url = Url::parse(&format!("http://{addr}/guide.xml")).expect("origin URL"); - let input = test_input_source(url.to_string(), None); - - let (content, _) = - download_text_content(&test_retry_config(1), &make_epg_test_client(), &input, None, None, false) - .await - .unwrap_or_else(|error| panic!("headerless {label} text should decode: {error}")); - - assert_eq!(content.as_bytes(), TEXT, "failed for {label}"); - assert_eq!(requests.lock().await.len(), 1); - handle.abort(); - } - } - - #[tokio::test] - async fn text_content_stream_decodes_headerless_gzip_and_zlib() { - const TEXT: &[u8] = b"streamed legacy text\n"; - - for (label, encoded) in [("gzip", gzip_encoded(TEXT)), ("zlib", zlib_encoded(TEXT))] { - let (addr, requests, handle) = - start_recording_http_byte_server(vec![response_with_byte_body("200 OK", &encoded)]) - .await - .expect("recording origin should start"); - let url = Url::parse(&format!("http://{addr}/guide.xml")).expect("origin URL"); - let input = test_input_source(url.to_string(), None); - - let (mut reader, _) = - get_remote_content_as_stream(&test_retry_config(1), &make_epg_test_client(), &input, None, &url) - .await - .unwrap_or_else(|error| panic!("headerless {label} stream should decode: {error}")); - let mut content = Vec::new(); - reader.read_to_end(&mut content).await.expect("read decoded stream"); - - assert_eq!(content, TEXT, "failed for {label}"); - assert_eq!(requests.lock().await.len(), 1); - handle.abort(); - } - } - - #[tokio::test] - async fn content_coding_identity_wins_after_input_and_client_header_merge() { - let (addr, requests, handle) = start_recording_http_server(vec![response_with_body("200 OK", "ok")]) - .await - .expect("recording origin should start"); - let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); - let mut input = test_input_source(url.to_string(), None); - input.headers.insert("Accept-Encoding".to_string(), "gzip".to_string()); - let mut client_headers = HeaderMap::new(); - client_headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("br")); - - let response = send_input_with_retry_and_provider_policy_with_options_result( - &test_retry_config(1), - &make_epg_test_client(), - &input, - Some(&client_headers), - &url, - identity_fetch_options(), - ) - .await - .expect("origin request should succeed"); - drop(response); - - let captured = requests.lock().await; - assert_eq!(captured.len(), 1); - assert_eq!(request_header_value(&captured[0], "accept-encoding"), Some("identity")); - handle.abort(); - } - - #[tokio::test] - async fn content_coding_identity_is_reapplied_for_retry() { - let responses = vec![response_with_body("500 Internal Server Error", ""), response_with_body("200 OK", "ok")]; - let (addr, requests, handle) = - start_recording_http_server(responses).await.expect("recording origin should start"); - let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); - let input = test_input_source(url.to_string(), None); - - send_input_with_retry_and_provider_policy_with_options_result( - &test_retry_config(2), - &make_epg_test_client(), - &input, - None, - &url, - identity_fetch_options(), - ) - .await - .expect("retry should reach successful response"); - - let captured = requests.lock().await; - assert_eq!(captured.len(), 2); - assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); - handle.abort(); - } - - #[tokio::test] - async fn content_coding_identity_manifest_retries_temporary_body_transport_error() { - let truncated = "HTTP/1.1 200 OK\r\nContent-Length: 64\r\nConnection: close\r\n\r\n#EXTM3U\n"; - let responses = vec![truncated.to_string(), response_with_body("200 OK", "#EXTM3U\nsegment.ts\n")]; - let (addr, requests, handle) = - start_recording_http_server(responses).await.expect("recording origin should start"); - let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); - let input = test_input_source(url.to_string(), None); - - let (manifest, _, _) = download_text_content_with_headers_and_options( - &test_retry_config(2), - &make_epg_test_client(), - &input, - None, - false, - TextContentFetchOptions::new( - identity_fetch_options(), - TextContentBodyOptions::hls_manifest(1024, Duration::from_secs(1)), - ), - ) - .await - .expect("temporary body transport failure should retry"); - - assert_eq!(manifest, "#EXTM3U\nsegment.ts\n"); - let captured = requests.lock().await; - assert_eq!(captured.len(), 2); - assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); - handle.abort(); - } - - #[tokio::test] - async fn text_content_manifest_uses_one_budget_for_status_decoder_and_success() { - let corrupt_gzip = - "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: 7\r\nConnection: close\r\n\r\ncorrupt"; - let responses = vec![ - response_with_body("503 Service Unavailable", ""), - corrupt_gzip.to_string(), - response_with_body("200 OK", "#EXTM3U\nsegment.ts\n"), - ]; - let (addr, requests, handle) = - start_recording_http_server(responses).await.expect("recording origin should start"); - let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); - let input = test_input_source(url.to_string(), None); - let options = TextContentFetchOptions::new( - identity_fetch_options(), - TextContentBodyOptions::hls_manifest(1024, Duration::from_secs(1)), - ); - - let (manifest, _, _) = download_text_content_with_headers_and_options( - &test_retry_config(3), - &make_epg_test_client(), - &input, - None, - false, - options, - ) - .await - .expect("third logical attempt should succeed"); - - assert_eq!(manifest, "#EXTM3U\nsegment.ts\n"); - let captured = requests.lock().await; - assert_eq!(captured.len(), 3); - assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); - handle.abort(); - } - - #[tokio::test] - async fn text_content_manifest_retry_budget_is_not_multiplied_by_inner_status_retries() { - let corrupt_gzip = - "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: 7\r\nConnection: close\r\n\r\ncorrupt"; - let responses = vec![ - response_with_body("503 Service Unavailable", ""), - response_with_body("502 Bad Gateway", ""), - corrupt_gzip.to_string(), - response_with_body("200 OK", "#EXTM3U\nsegment.ts\n"), - ]; - let (addr, requests, handle) = - start_recording_http_server(responses).await.expect("recording origin should start"); - let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); - let input = test_input_source(url.to_string(), None); - let options = TextContentFetchOptions::new( - identity_fetch_options(), - TextContentBodyOptions::hls_manifest(1024, Duration::from_secs(1)), - ); - - download_text_content_with_headers_and_options( - &test_retry_config(3), - &make_epg_test_client(), - &input, - None, - false, - options, - ) - .await - .expect_err("three logical failures must exhaust the configured budget"); - - let captured = requests.lock().await; - assert_eq!(captured.len(), 3, "the fourth success response must not be requested"); - assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); - handle.abort(); - } - - #[tokio::test] - async fn text_content_manifest_repeated_decoder_failures_stop_at_configured_budget() { - let corrupt_gzip = - "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: 7\r\nConnection: close\r\n\r\ncorrupt"; - let (addr, requests, handle) = - start_recording_http_server(vec![corrupt_gzip.to_string()]).await.expect("recording origin should start"); - let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); - let input = test_input_source(url.to_string(), None); - let options = TextContentFetchOptions::new( - identity_fetch_options(), - TextContentBodyOptions::hls_manifest(1024, Duration::from_secs(1)), - ); - - download_text_content_with_headers_and_options( - &test_retry_config(3), - &make_epg_test_client(), - &input, - None, - false, - options, - ) - .await - .expect_err("decoder failures must exhaust the configured budget"); - - let captured = requests.lock().await; - assert_eq!(captured.len(), 3); - assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); - handle.abort(); - } - - #[tokio::test] - async fn content_coding_identity_is_reapplied_after_provider_url_switch() { - let (first_addr, first_requests, first_handle) = - start_recording_http_server(vec![response_with_body("502 Bad Gateway", "")]) - .await - .expect("first provider origin should start"); - let (second_addr, second_requests, second_handle) = - start_recording_http_server(vec![response_with_body("200 OK", "ok")]) - .await - .expect("second provider origin should start"); - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "provider-a".into(), - urls: vec![format!("http://{first_addr}").into(), format!("http://{second_addr}").into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, - dns: None, - })); - let url = Url::parse("provider://provider-a/manifest.m3u8").expect("provider URL"); - let input = test_input_source(url.to_string(), Some(provider)); - - send_input_with_retry_and_provider_policy_with_options_result( - &test_retry_config(1), - &make_epg_test_client(), - &input, - None, - &url, - identity_fetch_options(), - ) - .await - .expect("provider failover should reach successful response"); - - let first = first_requests.lock().await; - let second = second_requests.lock().await; - assert_eq!(first.len(), 1); - assert_eq!(second.len(), 1); - assert_eq!(request_header_value(&first[0], "accept-encoding"), Some("identity")); - assert_eq!(request_header_value(&second[0], "accept-encoding"), Some("identity")); - first_handle.abort(); - second_handle.abort(); - } - - #[tokio::test] - async fn content_coding_identity_is_reapplied_for_same_origin_manual_redirect() { - let redirect = "HTTP/1.1 302 Found\r\nLocation: /final.m3u8\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; - let (addr, requests, handle) = - start_recording_http_server(vec![redirect.to_string(), response_with_body("200 OK", "ok")]) - .await - .expect("recording origin should start"); - let url = Url::parse(&format!("http://{addr}/entry.m3u8")).expect("origin URL"); - let input = test_input_source(url.to_string(), None); - - send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( - &test_retry_config(1), - &make_epg_test_client(), - &input, - None, - &url, - 2, - identity_fetch_options(), - ) - .await - .expect("same-origin redirect should succeed"); - - let captured = requests.lock().await; - assert_eq!(captured.len(), 2); - assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); - handle.abort(); - } - - #[tokio::test] - async fn content_coding_identity_survives_cross_origin_redirect_credential_scrubbing() { - let (target_addr, target_requests, target_handle) = - start_recording_http_server(vec![response_with_body("200 OK", "ok")]) - .await - .expect("redirect target should start"); - let redirect = format!( - "HTTP/1.1 302 Found\r\nLocation: http://{target_addr}/final.m3u8\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" - ); - let (entry_addr, entry_requests, entry_handle) = - start_recording_http_server(vec![redirect]).await.expect("redirect entry should start"); - let url = Url::parse(&format!("http://{entry_addr}/entry.m3u8")).expect("entry URL"); - let mut input = test_input_source(url.to_string(), None); - input.headers.insert("Authorization".to_string(), "Bearer input-secret".to_string()); - let mut client_headers = HeaderMap::new(); - client_headers.insert(COOKIE, HeaderValue::from_static("sid=client-secret")); - - send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( - &test_retry_config(1), - &make_epg_test_client(), - &input, - Some(&client_headers), - &url, - 2, - identity_fetch_options(), - ) - .await - .expect("cross-origin redirect should succeed"); - - let entry = entry_requests.lock().await; - assert_eq!(entry.len(), 1); - assert_eq!(request_header_value(&entry[0], "authorization"), Some("Bearer input-secret")); - assert_eq!(request_header_value(&entry[0], "cookie"), Some("sid=client-secret")); - drop(entry); - let target = target_requests.lock().await; - assert_eq!(target.len(), 1); - assert_eq!(request_header_value(&target[0], "accept-encoding"), Some("identity")); - assert!(request_header_value(&target[0], "authorization").is_none()); - assert!(request_header_value(&target[0], "cookie").is_none()); - entry_handle.abort(); - target_handle.abort(); - } - - #[tokio::test] - async fn manual_redirect_provider_failover_restarts_from_provider_entry() { - let (redirect_addr, redirect_hits, redirect_handle) = match start_plain_http_server_with_response( - "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(), - ) - .await - { - Ok(server) => server, - Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { - eprintln!("skipping manual_redirect_provider_failover_restarts_from_provider_entry: {err}"); - return; - } - Err(err) => panic!("failed to start redirect target server: {err}"), - }; - let redirect_url = format!("http://127.0.0.1:{}/redirected", redirect_addr.port()); - let provider_a_response = - format!("HTTP/1.1 302 Found\r\nLocation: {redirect_url}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); - let provider_entrypoint = start_plain_http_server_with_response(provider_a_response) - .await - .expect("provider a test server should start"); - let successful_mirror = - start_plain_http_server_with_body(b"provider-b").await.expect("provider b test server should start"); - - let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; - cfg.reverse_proxy = Some(ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: ResourceRetryConfig { max_attempts: 1, ..ResourceRetryConfig::default() }, - disabled_header: None, - stream: None, - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }); - let app_config = make_test_app_config(cfg); - let client = reqwest::Client::builder() - .no_proxy() - .redirect(reqwest::redirect::Policy::none()) - .connect_timeout(Duration::from_millis(400)) - .timeout(Duration::from_secs(2)) - .build() - .expect("http client should build"); - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "provider-a".into(), - urls: vec![ - format!("http://127.0.0.1:{}", provider_entrypoint.0.port()).into(), - format!("http://127.0.0.1:{}", successful_mirror.0.port()).into(), - ], - provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, - dns: None, - })); - let input = InputSource { - name: Arc::::from("test"), - url: "provider://provider-a/live/u/p/1.m3u8".to_string(), - provider: Some(provider), - username: None, - password: None, - method: InputFetchMethod::GET, - headers: HashMap::default(), - }; - let entry_url = Url::parse(input.url.as_str()).expect("provider URL should parse"); - - let response = send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( - &app_config, - &client, - &input, - None, - &entry_url, - 5, - RequestFetchOptions::with_attempt_idle_timeout(Duration::from_secs(1)), - ) - .await - .expect("request should fail over from redirected target to next provider entry"); - let provider_url_index = response.provider_url_index; - let body = response.response.text().await.expect("body should be readable"); - - assert_eq!(body, "provider-b"); - assert_eq!(provider_url_index, Some(1)); - assert_eq!(provider_entrypoint.1.load(Ordering::SeqCst), 1); - assert_eq!(redirect_hits.load(Ordering::SeqCst), 1); - assert_eq!(successful_mirror.1.load(Ordering::SeqCst), 1); - - provider_entrypoint.2.abort(); - successful_mirror.2.abort(); - redirect_handle.abort(); - } - - #[tokio::test] - async fn test_provider_request_chain_starts_from_last_successful_url() { - let (addr_b, accepted_b, handle_b) = match start_plain_http_server_with_body(b"b").await { - Ok(server) => server, - Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { - eprintln!("skipping test_provider_request_chain_starts_from_last_successful_url: {err}"); - return; - } - Err(err) => panic!("failed to start test http server: {err}"), - }; - - let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; - cfg.accept_insecure_ssl_certificates = true; - cfg.reverse_proxy = Some(ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: ResourceRetryConfig { max_attempts: 1, ..ResourceRetryConfig::default() }, - disabled_header: None, - stream: None, - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }); - let app_config = make_test_app_config(cfg); - let client = reqwest::Client::builder() - .no_proxy() - .connect_timeout(Duration::from_millis(400)) - .timeout(Duration::from_secs(2)) - .build() - .expect("http client should build"); - let dead_addr = SocketAddr::from(([127, 0, 0, 1], 1)); - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "provider-a".into(), - urls: vec![ - format!("http://127.0.0.1:{}", dead_addr.port()).into(), - format!("http://127.0.0.1:{}", addr_b.port()).into(), - ], - provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), - dns: None, - })); - - let url = Url::parse("provider://provider-a/live").expect("provider url should parse"); - let first_response = send_with_retry_and_provider(&app_config, &url, Some(&provider), false, |resolved_url| { - client.get(resolved_url.clone()) - }) - .await - .expect("request should fail over to the second provider url"); - let first_body = first_response.text().await.expect("response body should be readable"); - - assert_eq!(first_body, "b"); - assert_eq!(provider.get_current_index(), 1); - - let second_response = send_with_retry_and_provider(&app_config, &url, Some(&provider), false, |resolved_url| { - client.get(resolved_url.clone()) - }) - .await - .expect("next request should start from the last successful provider url"); - let second_body = second_response.text().await.expect("response body should be readable"); - - assert_eq!(second_body, "b"); - assert_eq!(accepted_b.load(Ordering::SeqCst), 2); - - handle_b.abort(); - } - - #[tokio::test] - async fn send_with_retry_policy_false_does_not_fail_over_provider_urls() { - let (addr_b, accepted_b, handle_b) = match start_plain_http_server_with_body(b"b").await { - Ok(server) => server, - Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { - eprintln!("skipping send_with_retry_policy_false_does_not_fail_over_provider_urls: {err}"); - return; - } - Err(err) => panic!("failed to start test http server: {err}"), - }; - - let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; - cfg.accept_insecure_ssl_certificates = true; - cfg.reverse_proxy = Some(ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: ResourceRetryConfig { - max_attempts: 3, - backoff_millis: 1, - ..ResourceRetryConfig::default() - }, - disabled_header: None, - stream: None, - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }); - let app_config = make_test_app_config(cfg); - let client = reqwest::Client::builder() - .no_proxy() - .connect_timeout(Duration::from_millis(200)) - .timeout(Duration::from_secs(2)) - .build() - .expect("http client should build"); - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "provider-a".into(), - urls: vec!["http://127.0.0.1:1".into(), format!("http://127.0.0.1:{}", addr_b.port()).into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), - dns: None, - })); - - let url = Url::parse("provider://provider-a/live").expect("provider url should parse"); - let result = - send_with_retry_and_provider_policy(&app_config, &url, Some(&provider), false, false, |resolved_url| { - client.get(resolved_url.clone()) - }) - .await; - - assert!(result.is_err(), "retry disabled must not fail over to the second provider URL"); - assert_eq!(accepted_b.load(Ordering::SeqCst), 0, "fallback provider URL must not be contacted"); - assert_eq!(provider.get_current_index(), 0, "retry disabled must not advance provider URL selection"); - - handle_b.abort(); - } - - #[tokio::test] - async fn test_provider_request_chain_restarts_from_first_url_when_policy_requests_it() { - let (addr_b, accepted_b, handle_b) = match start_plain_http_server_with_body(b"b").await { - Ok(server) => server, - Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { - eprintln!( - "skipping test_provider_request_chain_restarts_from_first_url_when_policy_requests_it: {err}" - ); - return; - } - Err(err) => panic!("failed to start test http server: {err}"), - }; - - let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; - cfg.accept_insecure_ssl_certificates = true; - cfg.reverse_proxy = Some(ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: ResourceRetryConfig { max_attempts: 1, ..ResourceRetryConfig::default() }, - disabled_header: None, - stream: None, - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }); - let app_config = make_test_app_config(cfg); - let client = reqwest::Client::builder() - .no_proxy() - .connect_timeout(Duration::from_millis(400)) - .timeout(Duration::from_secs(2)) - .build() - .expect("http client should build"); - let dead_addr = SocketAddr::from(([127, 0, 0, 1], 1)); - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "provider-a".into(), - urls: vec![ - format!("http://127.0.0.1:{}", dead_addr.port()).into(), - format!("http://127.0.0.1:{}", addr_b.port()).into(), - ], - provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, - dns: None, - })); - - let url = Url::parse("provider://provider-a/live").expect("provider url should parse"); - let first_response = send_with_retry_and_provider(&app_config, &url, Some(&provider), false, |resolved_url| { - client.get(resolved_url.clone()) - }) - .await - .expect("request should fail over to the second provider url"); - let first_body = first_response.text().await.expect("response body should be readable"); - - assert_eq!(first_body, "b"); - assert_eq!(provider.get_current_index(), 1); - - let second_response = send_with_retry_and_provider(&app_config, &url, Some(&provider), false, |resolved_url| { - client.get(resolved_url.clone()) - }) - .await - .expect("next request should restart from the first provider url and fail over again"); - let second_body = second_response.text().await.expect("response body should be readable"); - - assert_eq!(second_body, "b"); - assert_eq!(provider.get_current_index(), 1); - assert_eq!(accepted_b.load(Ordering::SeqCst), 2); - - handle_b.abort(); - } - - #[test] - fn test_next_provider_url_index_wraps_once_then_stops() { - assert_eq!(next_provider_url_index(2, 4, 2), Some(3)); - assert_eq!(next_provider_url_index(3, 4, 2), Some(0)); - assert_eq!(next_provider_url_index(0, 4, 2), Some(1)); - assert_eq!(next_provider_url_index(1, 4, 2), None); - assert_eq!(next_provider_url_index(0, 1, 0), None); - } - - #[tokio::test] - async fn test_on_connect_error_try_next_ip_before_provider_rotation() { - let (addr, accepted, server_handle) = match start_plain_http_server().await { - Ok(server) => server, - Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { - eprintln!("skipping test_on_connect_error_try_next_ip_before_provider_rotation: {err}"); - return; - } - Err(err) => panic!("failed to start test http server: {err}"), - }; - - let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; - cfg.accept_insecure_ssl_certificates = true; - cfg.reverse_proxy = Some(ReverseProxyConfig { - resource_rewrite_disabled: false, - rewrite_secret: [0; 16], - resource_retry: ResourceRetryConfig { max_attempts: 1, ..ResourceRetryConfig::default() }, - disabled_header: None, - stream: None, - cache: None, - rate_limit: None, - geoip: None, - stream_history: None, - qos_aggregation: None, - hls_cache: None, - }); - let app_config = make_test_app_config(cfg); - let client = reqwest::Client::builder() - .no_proxy() - .connect_timeout(Duration::from_millis(400)) - .timeout(Duration::from_secs(2)) - .build() - .expect("http client should build"); - let url = Url::parse(format!("http://example.com:{}/ok", addr.port()).as_str()).expect("url parse should work"); - - let provider_rotate = - make_provider_with_dns(false, OnConnectErrorPolicy::RotateProviderUrl, vec!["192.168.0.1", "127.0.0.1"]); - let result_rotate = - send_with_retry_and_provider(&app_config, &url, Some(&provider_rotate), false, |resolved_url| { - client.get(resolved_url.clone()) - }) - .await; - assert!(result_rotate.is_err(), "without try_next_ip policy the request should fail"); - - let provider_try_next = - make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "127.0.0.1"]); - let result_try_next = - send_with_retry_and_provider(&app_config, &url, Some(&provider_try_next), false, |resolved_url| { - client.get(resolved_url.clone()) - }) - .await; - assert!(result_try_next.is_ok(), "try_next_ip should succeed by trying the second IP"); - assert_eq!(accepted.load(Ordering::SeqCst), 1, "server should be reached exactly once"); - - server_handle.abort(); - } -} + send_with_retry_and_provider_policy, send_with_retry_and_provider_policy_with_options, should_trigger_failover, + }, +}; diff --git a/backend/core/src/utils/network/request/attempt.rs b/backend/core/src/utils/network/request/attempt.rs new file mode 100644 index 000000000..8875b2c7a --- /dev/null +++ b/backend/core/src/utils/network/request/attempt.rs @@ -0,0 +1,205 @@ +use super::{apply_request_fetch_options, create_client, provider_start_index, RequestFetchOptions}; +use crate::model::{resolve_provider_scheme_url_with_provider_index, AppConfig, ConfigProvider}; +use log::debug; +use reqwest::header::{HeaderValue, HOST}; +use shared::{error::string_to_io_error, utils::sanitize_sensitive_info}; +use std::{ + net::{IpAddr, SocketAddr}, + sync::Arc, + time::Duration, +}; +use url::Url; + +pub(super) fn prepare_physical_request_attempt( + request_builder: reqwest::RequestBuilder, + target: &AttemptTarget, + options: RequestFetchOptions, +) -> Result<(reqwest::Client, reqwest::Request), std::io::Error> { + let (base_client, request_result) = request_builder.build_split(); + let mut request = request_result.map_err(|error| { + string_to_io_error(format!("Failed to build request: {}", sanitize_sensitive_info(error.to_string().as_str()))) + })?; + apply_attempt_to_request(&mut request, target)?; + apply_request_fetch_options(&mut request, options); + Ok((base_client, request)) +} + +fn resolve_provider_url_for_attempt( + url: &Url, + provider: Option<&Arc>, + provider_url_index: usize, +) -> Url { + let Some(provider) = provider else { + return url.clone(); + }; + + match resolve_provider_scheme_url_with_provider_index(url.as_str(), Some(provider.clone()), provider_url_index) { + Ok((_provider, resolved)) => { + if resolved.as_ref() == url.as_str() { + return url.clone(); + } + Url::parse(resolved.as_ref()).unwrap_or_else(|_| url.clone()) + } + Err(err) => { + debug!("Failed to resolve provider URL: {err}"); + url.clone() + } + } +} + +#[derive(Debug, Clone)] +pub(super) struct AttemptTarget { + pub(super) request_url: Url, + pub(super) effective_url: Url, + pub(super) host_header: Option, + pub(super) sni_host: Option, + pub(super) connect_ip: Option, + pub(super) dns_host: Option, +} + +impl AttemptTarget { + pub(super) fn new(url: Url) -> Self { + Self { + request_url: url.clone(), + effective_url: url, + host_header: None, + sni_host: None, + connect_ip: None, + dns_host: None, + } + } +} + +fn is_ip_literal(host: &str) -> bool { host.parse::().is_ok() } + +fn format_host_header_with_port(host: &str, port: Option) -> String { + match port { + Some(port) => format!("{host}:{port}"), + None => host.to_string(), + } +} + +fn format_ip_host_header_with_port(ip: IpAddr, port: Option) -> String { + match (ip, port) { + (IpAddr::V4(addr), Some(port)) => format!("{addr}:{port}"), + (IpAddr::V4(addr), None) => addr.to_string(), + (IpAddr::V6(addr), Some(port)) => format!("[{addr}]:{port}"), + (IpAddr::V6(addr), None) => format!("[{addr}]"), + } +} + +fn resolve_attempt_target_with_dns_mode( + url: &Url, + provider: Option<&Arc>, + preview_dns_selection: bool, + provider_url_index: usize, +) -> AttemptTarget { + let resolved_url = resolve_provider_url_for_attempt(url, provider, provider_url_index); + let Some(provider) = provider else { + return AttemptTarget::new(resolved_url); + }; + + let mut target = AttemptTarget::new(resolved_url.clone()); + let scheme = resolved_url.scheme(); + if !provider.dns_enabled_for_scheme(scheme) { + return target; + } + + let Some(host) = resolved_url.host_str() else { + return target; + }; + if is_ip_literal(host) { + return target; + } + + let connect_ip = + if preview_dns_selection { provider.preview_ip_for_host(host) } else { provider.select_ip_for_host(host) }; + let Some(connect_ip) = connect_ip else { + return target; + }; + let keep_vhost = provider.get_dns_config().is_some_and(|dns| dns.keep_vhost); + let host_header = if keep_vhost { + format_host_header_with_port(host, resolved_url.port()) + } else { + format_ip_host_header_with_port(connect_ip, resolved_url.port()) + }; + + target.host_header = Some(host_header); + target.connect_ip = Some(connect_ip); + target.dns_host = Some(host.to_ascii_lowercase()); + + if scheme.eq_ignore_ascii_case("https") { + target.sni_host = Some(host.to_string()); + return target; + } + + if scheme.eq_ignore_ascii_case("http") { + let mut effective = resolved_url.clone(); + if effective.set_ip_host(connect_ip).is_ok() { + target.effective_url = effective; + } + } + + target +} + +#[cfg(test)] +pub(super) fn resolve_attempt_target(url: &Url, provider: Option<&Arc>) -> AttemptTarget { + resolve_attempt_target_with_dns_mode(url, provider, false, 0) +} + +pub(super) fn resolve_attempt_target_at_provider_index( + url: &Url, + provider: Option<&Arc>, + provider_url_index: usize, +) -> AttemptTarget { + resolve_attempt_target_with_dns_mode(url, provider, false, provider_url_index) +} + +pub(super) fn preview_attempt_target(url: &Url, provider: Option<&Arc>) -> AttemptTarget { + resolve_attempt_target_with_dns_mode(url, provider, true, provider_start_index(provider)) +} + +fn apply_attempt_to_request(request: &mut reqwest::Request, target: &AttemptTarget) -> Result<(), std::io::Error> { + if request.url().as_str() != target.effective_url.as_str() { + *request.url_mut() = target.effective_url.clone(); + } + if let Some(host_header) = target.host_header.as_ref() { + let host = HeaderValue::from_str(host_header) + .map_err(|err| string_to_io_error(format!("Invalid host header '{host_header}': {err}")))?; + request.headers_mut().insert(HOST, host); + } + Ok(()) +} + +fn build_https_attempt_client( + app_config: &Arc, + sni_host: &str, + connect_ip: IpAddr, + connect_port: u16, +) -> Result { + let config = app_config.config.load(); + let mut builder = create_client(app_config).http1_only(); + if config.connect_timeout_secs > 0 { + builder = builder.connect_timeout(Duration::from_secs(u64::from(config.connect_timeout_secs))); + } + drop(config); + builder = builder.resolve_to_addrs(sni_host, &[SocketAddr::new(connect_ip, connect_port)]); + builder.build() +} + +pub(super) async fn execute_attempt_request( + app_config: &Arc, + base_client: reqwest::Client, + request: reqwest::Request, + target: &AttemptTarget, +) -> Result { + if target.effective_url.scheme().eq_ignore_ascii_case("https") { + if let (Some(sni_host), Some(connect_ip)) = (target.sni_host.as_ref(), target.connect_ip) { + let connect_port = target.effective_url.port_or_known_default().unwrap_or(443); + let https_client = build_https_attempt_client(app_config, sni_host.as_str(), connect_ip, connect_port)?; + return https_client.execute(request).await; + } + } + base_client.execute(request).await +} diff --git a/backend/core/src/utils/network/request/body.rs b/backend/core/src/utils/network/request/body.rs new file mode 100644 index 000000000..c5ed04e7a --- /dev/null +++ b/backend/core/src/utils/network/request/body.rs @@ -0,0 +1,227 @@ +use super::{ + format_http_status, is_retryable_text_response_status, perform_backoff, + send_input_with_retry_and_provider_policy_with_options_result, should_retry_text_body_error, + sleep_before_text_body_retry, text_body_retry_values, DynReader, RequestFetchOptions, TextContentBodyOptions, + TextContentFetchOptions, +}; +use crate::{ + model::{AppConfig, InputSource}, + utils::content_coding::{ + content_decoding_error_from_io, decode_response_to_identity, is_http_body_transport_error, + log_hls_origin_content_coding, read_utf8_limited, ContentBodyReadError, ContentCodingDetection, + ContentCodingError, HlsOriginContentCodingObjectKind, HlsOriginContentCodingSource, + }, +}; +use reqwest::{header::HeaderMap, StatusCode}; +use shared::{error::string_to_io_error, utils::sanitize_sensitive_info}; +use std::{ + io::{Error, ErrorKind}, + sync::Arc, +}; +use tokio::io::AsyncReadExt; +use url::Url; + +async fn build_decoded_stream_reader(response: reqwest::Response) -> Result { + decode_response_to_identity(response, ContentCodingDetection::DeclaredOrLegacyTextMagic) + .await + .map(|decoded| decoded.body) + .map_err(content_coding_error_to_io) +} + +fn content_coding_error_to_io(error: ContentCodingError) -> Error { + let kind = match &error { + ContentCodingError::PrefixRead(source) => source.kind(), + ContentCodingError::InvalidHeader + | ContentCodingError::Unsupported(_) + | ContentCodingError::EncodedPartialContent => ErrorKind::Other, + }; + Error::new(kind, error) +} + +fn content_body_read_error_to_io(error: ContentBodyReadError) -> Error { + match error { + ContentBodyReadError::Io(error) => error, + error @ ContentBodyReadError::LimitExceeded { .. } => Error::other(error), + error @ ContentBodyReadError::InvalidUtf8 { .. } => Error::new(ErrorKind::InvalidData, error), + } +} + +/// Builds a fixed/numeric text-response diagnostic without origin-controlled content. +pub fn text_response_error_log_label(error: &Error) -> String { + if let Some(error) = content_decoding_error_from_io(error) { + return format!("content_decoding coding={}", error.coding.as_http_token()); + } + if let Some(error) = error.get_ref().and_then(|source| source.downcast_ref::()) { + return match error { + ContentBodyReadError::LimitExceeded { limit } => format!("decoded_body_limit limit={limit}"), + ContentBodyReadError::InvalidUtf8 { valid_up_to, error_len } => { + format!("invalid_utf8 valid_up_to={valid_up_to} error_len={error_len:?}") + } + ContentBodyReadError::Io(error) => format!("io kind={:?}", error.kind()), + }; + } + if let Some(error) = error.get_ref().and_then(|source| source.downcast_ref::()) { + return match error { + ContentCodingError::InvalidHeader => "content_coding class=invalid_header".to_string(), + ContentCodingError::Unsupported(_) => "content_coding class=unsupported".to_string(), + ContentCodingError::EncodedPartialContent => "content_coding class=encoded_partial_content".to_string(), + ContentCodingError::PrefixRead(_) => "content_coding class=prefix_read".to_string(), + }; + } + if error.kind() == ErrorKind::TimedOut { + return "timeout".to_string(); + } + if is_http_body_transport_error(error) { + return "transport".to_string(); + } + format!("io kind={:?}", error.kind()) +} + +pub(super) async fn read_text_response_with_body_options( + response: reqwest::Response, + body_options: TextContentBodyOptions, +) -> Result<(String, String, HeaderMap), Error> { + let request_url = response.url().to_string(); + let read = async move { + let mut decoded = + decode_response_to_identity(response, body_options.detection).await.map_err(content_coding_error_to_io)?; + if let (ContentCodingDetection::DeclaredOrKnownHlsManifestMagic, Some(observation)) = + (body_options.detection, decoded.content_coding_observation()) + { + log_hls_origin_content_coding( + observation, + HlsOriginContentCodingObjectKind::Manifest, + false, + HlsOriginContentCodingSource::Legacy, + ); + } + let content = if let Some(max_decoded_bytes) = body_options.max_decoded_bytes { + read_utf8_limited(&mut decoded.body, max_decoded_bytes).await.map_err(content_body_read_error_to_io)? + } else { + let mut content = String::new(); + decoded.body.read_to_string(&mut content).await.map_err(|error| Error::new(error.kind(), error))?; + content + }; + Ok((content, decoded.final_url.to_string(), decoded.headers)) + }; + + if let Some(deadline) = body_options.deadline { + tokio::time::timeout(deadline, read).await.map_err(|_| { + Error::new( + ErrorKind::TimedOut, + format!("Timed out reading content body: {}", sanitize_sensitive_info(&request_url)), + ) + })? + } else { + read.await + } +} + +#[allow(clippy::implicit_hasher)] +pub async fn get_remote_content_as_stream( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + url: &Url, +) -> Result<(DynReader, String), Error> { + get_remote_content_as_stream_with_options(app_config, client, input, headers, url, RequestFetchOptions::default()) + .await +} + +#[allow(clippy::implicit_hasher)] +async fn get_remote_content_as_stream_with_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + url: &Url, + options: RequestFetchOptions, +) -> Result<(DynReader, String), Error> { + let response = + send_input_with_retry_and_provider_policy_with_options_result(app_config, client, input, headers, url, options) + .await? + .response; + let response_url = response.url().to_string(); + + let reader = build_decoded_stream_reader(response).await?; + Ok((reader, response_url)) +} + +pub(super) fn text_response_status_error(status: StatusCode, url: &Url) -> Error { + string_to_io_error(format!( + "Request failed ({}): {}", + format_http_status(status), + sanitize_sensitive_info(url.as_str()) + )) +} + +pub(super) async fn get_remote_content_with_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + url: &Url, + options: RequestFetchOptions, +) -> Result<(String, String), Error> { + get_remote_content_with_headers_and_options( + app_config, + client, + input, + headers, + url, + TextContentFetchOptions::with_request_options(options), + ) + .await + .map(|(content, response_url, _)| (content, response_url)) +} + +pub(super) async fn get_remote_content_with_headers_and_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + url: &Url, + options: TextContentFetchOptions, +) -> Result<(String, String, HeaderMap), Error> { + let (max_attempts, backoff_ms, backoff_multiplier) = text_body_retry_values(app_config, options.body); + let attempt_options = if max_attempts > 1 { options.request.without_resource_retries() } else { options.request }; + + // This consumer owns the logical attempt budget whenever decoded-body retries are enabled. Provider failover + // and redirect hops remain bounded subrequests, but the configured retry count is not applied again below it. + for attempt in 0..max_attempts { + let response = match send_input_with_retry_and_provider_policy_with_options_result( + app_config, + client, + input, + headers, + url, + attempt_options, + ) + .await + { + Ok(result) => result.response, + Err(error) if should_retry_text_body_error(&error) && attempt + 1 < max_attempts => { + sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &error).await; + continue; + } + Err(error) => return Err(error), + }; + let status = response.status(); + if !status.is_success() { + if is_retryable_text_response_status(status) && attempt + 1 < max_attempts { + perform_backoff(attempt, backoff_ms, backoff_multiplier, &response).await; + continue; + } + return Err(text_response_status_error(status, url)); + } + match read_text_response_with_body_options(response, options.body).await { + Ok(result) => return Ok(result), + Err(error) if should_retry_text_body_error(&error) && attempt + 1 < max_attempts => { + sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &error).await; + } + Err(error) => return Err(error), + } + } + Err(string_to_io_error("Text response body retry attempts exhausted")) +} diff --git a/backend/core/src/utils/network/request/client.rs b/backend/core/src/utils/network/request/client.rs new file mode 100644 index 000000000..8eb530271 --- /dev/null +++ b/backend/core/src/utils/network/request/client.rs @@ -0,0 +1,133 @@ +use super::{get_request_headers, log_proxy_diagnostics}; +use crate::model::{AppConfig, Config, ReverseProxyDisabledHeaderConfig}; +use log::error; +use reqwest::redirect::Policy; +use shared::{error::TuliproxError, model::InputFetchMethod}; +use std::{collections::HashMap, time::Duration}; +use url::Url; + +pub fn get_client_request( + client: &reqwest::Client, + method: InputFetchMethod, + headers: Option<&HashMap>, + url: &Url, + custom_headers: Option<&HashMap, S>>, + disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, + default_user_agent: Option<&str>, +) -> reqwest::RequestBuilder { + let request = match method { + InputFetchMethod::GET => client.get(url.clone()), + InputFetchMethod::POST => { + // let base_url = url[..url::Position::BeforePath].to_string() + url.path(); + let mut params: HashMap = HashMap::default(); + for (key, value) in url.query_pairs() { + params.insert(key.to_string(), value.to_string()); + } + // we could cut the params but we leave them as query and add them as form. + client.post(url.clone()).form(¶ms) + } + }; + let headers = get_request_headers(headers, custom_headers, disabled_headers, default_user_agent); + request.headers(headers) +} + +pub fn create_client_with_redirect(cfg: &AppConfig, redirect_policy: Policy) -> reqwest::ClientBuilder { + configured_client(&cfg.config.load(), redirect_policy) +} + +fn configured_client(config: &Config, redirect_policy: Policy) -> reqwest::ClientBuilder { + log_proxy_diagnostics(config); + let mut client = reqwest::Client::builder() + .redirect(redirect_policy) + .pool_idle_timeout(Duration::from_secs(30)) + .pool_max_idle_per_host(10) + .danger_accept_invalid_certs(config.accept_insecure_ssl_certificates); + + if let Some(proxy_cfg) = config.proxy.as_ref() { + match Url::parse(&proxy_cfg.url) { + Ok(mut url) => { + let scheme = url.scheme().to_ascii_lowercase(); + + match scheme.as_str() { + "socks5" | "socks5h" => { + if let Some(user) = &proxy_cfg.username { + let _ = url.set_username(user); + } + if let Some(pass) = &proxy_cfg.password { + let _ = url.set_password(Some(pass)); + } + match reqwest::Proxy::all(url.as_str()) { + Ok(p) => { + client = client.proxy(p); + } + Err(err) => error!("Failed to create SOCKS proxy {url}: {err}"), + } + } + "http" | "https" => match reqwest::Proxy::all(url.as_str()) { + Ok(p) => { + if let (Some(username), Some(password)) = (&proxy_cfg.username, &proxy_cfg.password) { + client = client.proxy(p.basic_auth(username, password)); + } else { + client = client.proxy(p); + } + } + Err(err) => error!("Failed to create HTTP proxy {url}: {err}"), + }, + _ => { + error!("Unsupported proxy scheme '{scheme}' in URL: {url}"); + } + } + } + Err(e) => { + error!("Invalid proxy URL '{}': {e}", proxy_cfg.url); + } + } + } + + if let Some(rp_config) = config.reverse_proxy.as_ref() { + if rp_config.disabled_header.as_ref().is_some_and(|d| d.referer_header) { + client = client.referer(false); + } + } + + client +} + +pub fn create_client(cfg: &AppConfig) -> reqwest::ClientBuilder { + create_client_with_redirect(cfg, Policy::limited(10)) +} + +/// Dedicated discovery profile. Callers must propagate construction failure, never +/// fall back to an unconfigured client. Builder access permits local CA/DNS fixtures. +pub fn create_tmdb_client(cfg: &AppConfig) -> Result { + let config = cfg.config.load(); + let invalid_proxy = || TuliproxError::Config("TMDB HTTP profile has invalid proxy configuration".to_string()); + // The general factory historically logs malformed proxies and continues. TMDB + // must fail closed, using the same snapshot and proxy transformations instead. + if let Some(proxy) = &config.proxy { + let mut url = Url::parse(&proxy.url).map_err(|_| invalid_proxy())?; + match url.scheme() { + "socks5" | "socks5h" => { + if let Some(user) = &proxy.username { + url.set_username(user).map_err(|()| invalid_proxy())?; + } + if let Some(pass) = &proxy.password { + url.set_password(Some(pass)).map_err(|()| invalid_proxy())?; + } + } + "http" | "https" => {} + _ => return Err(invalid_proxy()), + } + reqwest::Proxy::all(url.as_str()).map_err(|_| invalid_proxy())?; + } + // Discovery carries its own Bearer credential: never inherit the generic TLS bypass. + // Keep configured proxy/auth and trusted roots, including hostname verification. + let mut builder = configured_client(&config, Policy::none()) + .danger_accept_invalid_certs(false) + .retry(reqwest::retry::never()) + .http1_only(); + if config.connect_timeout_secs > 0 { + builder = builder.connect_timeout(Duration::from_secs(u64::from(config.connect_timeout_secs))); + } + Ok(builder) +} diff --git a/backend/core/src/utils/network/request/destination.rs b/backend/core/src/utils/network/request/destination.rs new file mode 100644 index 000000000..de1c8f1c7 --- /dev/null +++ b/backend/core/src/utils/network/request/destination.rs @@ -0,0 +1,297 @@ +use super::{ + PublicIpResolver, ResourceDestination, ResourceDestinationResolver, RESOURCE_DESTINATION_CACHE_CAPACITY, + RESOURCE_DESTINATION_LOOKUP_TIMEOUT, RESOURCE_DESTINATION_PRIVATE_TTL, RESOURCE_DESTINATION_PUBLIC_TTL, + RESOURCE_DESTINATION_UNANSWERED_TTL, +}; +use crate::model::AppConfig; +use lru::LruCache; +use std::{ + io::{Error, ErrorKind}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, + num::NonZeroUsize, + sync::{Arc, Mutex, OnceLock, PoisonError}, + time::{Duration, Instant}, +}; +use tokio::time::timeout; + +impl reqwest::dns::Resolve for PublicIpResolver { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { + let host = name.as_str().to_string(); + Box::pin(async move { + let addresses = resolve_public_socket_addrs(&host, 0) + .await + .map_err(|err| Box::new(err) as Box)?; + Ok(Box::new(addresses.into_iter()) as reqwest::dns::Addrs) + }) + } +} + +pub async fn resolve_public_socket_addrs(host: &str, port: u16) -> std::io::Result> { + let addresses = lookup_socket_addrs(host, port).await?; + if addresses.is_empty() || addresses.iter().any(|address| !is_public_ip(address.ip())) { + return Err(Error::new(ErrorKind::PermissionDenied, "destination resolves to a non-public address")); + } + Ok(addresses) +} + +/// Resolves a resource destination, refusing addresses that are local to this host. +/// +/// Private network destinations stay allowed: self-hosted services are a normal resource source. +pub async fn resolve_resource_socket_addrs(host: &str, port: u16) -> std::io::Result> { + let addresses = lookup_socket_addrs(host, port).await?; + if addresses.is_empty() { + return Err(Error::new(ErrorKind::PermissionDenied, "destination does not resolve")); + } + if addresses.iter().any(|address| classify_ip(address.ip()) == ResourceDestination::Blocked) { + return Err(Error::new(ErrorKind::PermissionDenied, "destination is local to this host")); + } + Ok(addresses) +} + +/// Resolves a host name or IP literal, bracketed IPv6 included. An unresolvable name yields an empty +/// result so that every policy decides on its own how to report it. +async fn lookup_socket_addrs(host: &str, port: u16) -> std::io::Result> { + if let Ok(address) = host_literal(host).parse::() { + return Ok(vec![SocketAddr::new(address, port)]); + } + Ok(tokio::net::lookup_host((host, port)).await?.collect()) +} + +/// Hosts of the proxies an outbound request may be routed through. +/// +/// A resource client that honours the configured proxy asks its resolver about the proxy host as well, +/// and a proxy commonly lives on the loopback interface (`http://localhost:8118`). That name is exempt +/// from the local-address guard, because a proxy on this host is a valid destination for this host. +pub(super) fn proxy_hosts(app_config: &AppConfig) -> Vec> { + const ENV_KEYS: [&str; 3] = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"]; + let mut hosts = Vec::new(); + let config = app_config.config.load(); + if let Some(proxy) = config.proxy.as_ref() { + if let Some(host) = + crate::model::parse_proxy_url_with_http_fallback(&proxy.url).and_then(|url| url.host_str().map(Arc::from)) + { + hosts.push(host); + } + } + drop(config); + for (key, value) in std::env::vars_os() { + let (Some(key), Some(value)) = (key.to_str(), value.to_str()) else { + continue; + }; + if !ENV_KEYS.iter().any(|candidate| candidate.eq_ignore_ascii_case(key)) { + continue; + } + if let Some(host) = + crate::model::parse_proxy_url_with_http_fallback(value).and_then(|url| url.host_str().map(Arc::from)) + { + hosts.push(host); + } + } + hosts +} + +impl ResourceDestinationResolver { + /// Resolver that resolves the proxy hosts of this configuration and guards every other name. + pub fn allowing_proxy_hosts(app_config: &AppConfig) -> Self { + Self { allowed_hosts: proxy_hosts(app_config).into() } + } +} + +impl reqwest::dns::Resolve for ResourceDestinationResolver { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { + let host = name.as_str().to_owned(); + let allowed = self.allowed_hosts.iter().any(|allowed| allowed.as_ref().eq_ignore_ascii_case(&host)); + Box::pin(async move { + let addresses = if allowed { + lookup_socket_addrs(&host, 0).await + } else { + resolve_resource_socket_addrs(&host, 0).await + } + .map_err(|err| Box::new(err) as Box)?; + Ok(Box::new(addresses.into_iter()) as reqwest::dns::Addrs) + }) + } +} + +/// Reachability class of an address. Single place that decides which range belongs to which class. +pub fn classify_ip(address: IpAddr) -> ResourceDestination { + match address { + IpAddr::V4(address) => classify_ipv4(address), + IpAddr::V6(address) => classify_ipv6(address), + } +} + +fn classify_ipv4(address: Ipv4Addr) -> ResourceDestination { + let [first, second, _, _] = address.octets(); + if address.is_loopback() + || address.is_link_local() + || address.is_unspecified() + || address.is_multicast() + || address.is_broadcast() + || first == 0 + { + ResourceDestination::Blocked + } else if address.is_private() + || address.is_documentation() + || (first == 100 && (64..=127).contains(&second)) + || (first == 198 && matches!(second, 18 | 19)) + || first >= 240 + { + ResourceDestination::Private + } else { + ResourceDestination::Public + } +} + +/// IPv4 address carried inside an IPv6 address: IPv4-compatible, NAT64 (`64:ff9b::/96`) or 6to4. +fn embedded_ipv4(address: Ipv6Addr) -> Option { + let segments = address.segments(); + let (high, low) = if segments[..6] == [0, 0, 0, 0, 0, 0] || segments[..6] == [0x0064, 0xff9b, 0, 0, 0, 0] { + (segments[6], segments[7]) + } else if segments[0] == 0x2002 { + (segments[1], segments[2]) + } else { + return None; + }; + let [a, b] = high.to_be_bytes(); + let [c, d] = low.to_be_bytes(); + Some(Ipv4Addr::new(a, b, c, d)) +} + +fn classify_ipv6(address: Ipv6Addr) -> ResourceDestination { + let segments = address.segments(); + let embedded = address.to_ipv4_mapped().or_else(|| embedded_ipv4(address)); + if address.is_loopback() + || address.is_unspecified() + || address.is_multicast() + || segments[0] & 0xffc0 == 0xfe80 + || embedded.is_some_and(|mapped| classify_ipv4(mapped) == ResourceDestination::Blocked) + { + ResourceDestination::Blocked + } else if segments[0] & 0xfe00 == 0xfc00 + || segments[0] & 0xffc0 == 0xfec0 + || (segments[0] == 0x2001 && segments[1] == 0x0db8) + || embedded.is_some_and(|mapped| classify_ipv4(mapped) == ResourceDestination::Private) + { + ResourceDestination::Private + } else { + ResourceDestination::Public + } +} + +/// Whether an address is publicly routable. +/// +/// Derived from [`classify_ip`] so classification and this answer cannot drift apart. +pub fn is_public_ip(address: IpAddr) -> bool { classify_ip(address) == ResourceDestination::Public } + +/// Strips the brackets of an IPv6 literal so that it parses as an address. +fn host_literal(host: &str) -> &str { host.strip_prefix('[').and_then(|value| value.strip_suffix(']')).unwrap_or(host) } + +struct RememberedDestination { + verdict: ResourceDestination, + /// Whether the verdict came from a resolved answer. A name whose lookup produced none stays + /// non-public, but nothing is known about it either. + answered: bool, + expires_at: Instant, +} + +/// Memoizes destination verdicts so that rendering a playlist or EPG does not resolve the same host +/// per icon. +/// +/// The verdict is a property of the host, so one memo serves the whole process: every route and +/// every rendering path has to reach the same answer, and a per-request memo would put a DNS lookup +/// on the playlist path for each request instead of once per host. +/// +/// This is a hot-path memo, not policy state: the verdict decides whether a URL may be exposed, and +/// the resource client resolves and validates again while the connection is built, so a stale entry +/// can never open a connection to a local address. +struct DestinationCache { + destinations: Mutex>, +} + +impl DestinationCache { + pub(super) fn new() -> Self { + Self { + destinations: Mutex::new(LruCache::new( + NonZeroUsize::new(RESOURCE_DESTINATION_CACHE_CAPACITY).unwrap_or(NonZeroUsize::MIN), + )), + } + } + + /// The process-wide memo. + pub(super) fn shared() -> &'static Self { + static SHARED: OnceLock = OnceLock::new(); + SHARED.get_or_init(DestinationCache::new) + } + + fn verdict_and_answer(&self, host: &str) -> Option<(ResourceDestination, bool)> { + let mut destinations = self.destinations.lock().unwrap_or_else(PoisonError::into_inner); + match destinations.get(host) { + Some(entry) if entry.expires_at > Instant::now() => Some((entry.verdict, entry.answered)), + Some(_) => { + destinations.pop(host); + None + } + None => None, + } + } + + fn remember(&self, host: &str, verdict: ResourceDestination, answered: bool, ttl: Duration) { + let mut destinations = self.destinations.lock().unwrap_or_else(PoisonError::into_inner); + let entry = RememberedDestination { verdict, answered, expires_at: Instant::now() + ttl }; + destinations.put(host.to_owned(), entry); + } +} + +/// Classifies a destination and reports whether the verdict came from a resolved answer. +/// +/// The second value separates "proven to be local" from "nothing was proven": both keep a client away +/// from the destination, but only the first says the destination cannot be reached from outside. +pub(crate) async fn classify_host(host: &str) -> (ResourceDestination, bool) { + if let Ok(address) = host_literal(host).parse::() { + // An IP literal is decided by its address alone, so the answer is factual either way. + return (classify_ip(address), true); + } + let cache = DestinationCache::shared(); + if let Some(verdict) = cache.verdict_and_answer(host) { + return verdict; + } + + let mut resolved = Vec::new(); + if let Ok(Ok(addresses)) = timeout(RESOURCE_DESTINATION_LOOKUP_TIMEOUT, tokio::net::lookup_host((host, 0))).await { + resolved.extend(addresses.map(|address| address.ip())); + } + let (verdict, ttl) = verdict_for_addresses(&resolved); + let answered = !resolved.is_empty(); + cache.remember(host, verdict, answered, ttl); + (verdict, answered) +} + +/// Aggregates the addresses a name resolved to into a single verdict, and the time that verdict may be +/// remembered. +/// +/// One local-only address blocks the whole destination, and one private address keeps it non-public: +/// a client must not be able to reach the destination by any of its addresses. An empty answer is +/// non-public, because nothing was proven, and is remembered only briefly: unlike a resolved address, +/// it does not prove anything about the destination, so the name may well be reachable once the +/// resolver answers again. +pub(super) fn verdict_for_addresses(addresses: &[IpAddr]) -> (ResourceDestination, Duration) { + if addresses.is_empty() { + return (ResourceDestination::Private, RESOURCE_DESTINATION_UNANSWERED_TTL); + } + let mut verdict = ResourceDestination::Public; + for address in addresses { + match classify_ip(*address) { + ResourceDestination::Blocked => { + return (ResourceDestination::Blocked, RESOURCE_DESTINATION_PRIVATE_TTL); + } + ResourceDestination::Private => verdict = ResourceDestination::Private, + ResourceDestination::Public => {} + } + } + let ttl = match verdict { + ResourceDestination::Public => RESOURCE_DESTINATION_PUBLIC_TTL, + ResourceDestination::Private | ResourceDestination::Blocked => RESOURCE_DESTINATION_PRIVATE_TTL, + }; + (verdict, ttl) +} diff --git a/backend/core/src/utils/network/request/download.rs b/backend/core/src/utils/network/request/download.rs new file mode 100644 index 000000000..63b9db770 --- /dev/null +++ b/backend/core/src/utils/network/request/download.rs @@ -0,0 +1,717 @@ +use super::{ + classify_host, get_remote_content_as_stream, get_remote_content_with_headers_and_options, + get_remote_content_with_options, + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, DynReader, InputEpgFileRequest, + RecordingTaskOptions, RequestFetchOptions, ResourceDestination, ResourceRetryExecution, TextContentBodyOptions, + TextContentFetchOptions, TextContentRetryOwner, STREAM_IDLE_TIMEOUT, +}; +use crate::{ + model::{AppConfig, ConfigInput, InputSource}, + utils::{ + async_file_reader, async_file_writer, + compression::compression_utils::is_gzip, + content_coding::{ContentCodingDetection, OutboundContentCodingPolicy}, + debug_if_enabled, get_file_path, + network::persist_pipe::tee_dyn_reader, + persist_file, + }, +}; +use futures::StreamExt; +use log::{debug, error, log_enabled, warn}; +use reqwest::header::HeaderMap; +use shared::{ + error::{string_to_io_error, TuliproxError}, + model::format_elapsed_time, + utils::{human_readable_byte_size, sanitize_sensitive_info}, +}; +use std::{ + io::{Error, ErrorKind}, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; +use tokio::{ + fs::File, + io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}, + time::sleep, +}; +use url::Url; + +/// Classifies a resource destination given as host name or IP literal, bracketed IPv6 included. +/// +/// An IP literal is decided by its address alone, without resolving anything. Names are resolved once +/// per verdict lifetime and remembered in the process-wide memo. A name that cannot be resolved is +/// reported as [`ResourceDestination::Private`]: callers must not hand a possibly internal name to a +/// client, and the fetch attempt itself decides whether the destination is reachable. +pub async fn classify_resource_destination(host: &str) -> ResourceDestination { classify_host(host).await.0 } + +impl RequestFetchOptions { + pub fn with_attempt_idle_timeout(timeout: Duration) -> Self { + Self { attempt_idle_timeout: Some(timeout.max(Duration::from_millis(1))), ..Self::default() } + } + + pub const fn with_content_coding(mut self, content_coding: OutboundContentCodingPolicy) -> Self { + self.content_coding = content_coding; + self + } + + /// Returns the final HTTP error response after the configured retry and failover policy. + pub const fn with_http_error_responses(mut self, enabled: bool) -> Self { + self.return_http_errors = enabled; + self + } + + /// Leaves bounded provider failover intact while assigning retry rounds to the caller. + pub const fn without_resource_retries(mut self) -> Self { + self.resource_retry = ResourceRetryExecution::ProviderFailoverOnly; + self + } + + pub(super) fn attempt_idle_timeout_or_default(self) -> Duration { + self.attempt_idle_timeout.unwrap_or_else(|| Duration::from_secs(STREAM_IDLE_TIMEOUT)) + } + + pub(super) const fn uses_provider_failover_only(self) -> bool { + matches!(self.resource_retry, ResourceRetryExecution::ProviderFailoverOnly) + } +} + +pub async fn get_input_epg_content_as_file( + app_config: &Arc, + client: &reqwest::Client, + input: &ConfigInput, + request: InputEpgFileRequest<'_>, +) -> Result { + let InputEpgFileRequest { headers, storage_dir, url: url_str, persist_path, max_bytes } = request; + debug_if_enabled!( + "getting input epg content storage_dir: {}, url: {}", + storage_dir, + sanitize_sensitive_info(url_str) + ); + + // This is the single write-lock boundary for EPG cache population. Callers must + // not hold a lock for `persist_path` while invoking this function. + let _persist_lock = app_config.file_locks.write_lock(persist_path).await; + + // On Windows, drive-letter paths also parse as URLs (with `c` as the scheme). + // Interpret an absolute platform path before attempting URL parsing. + if !Path::new(url_str).is_absolute() && url_str.parse::().is_ok() { + match download_epg_content_as_file(app_config, client, input, headers, url_str, persist_path, max_bytes).await { + Ok(content) => Ok(content), + Err(e) => { + error!( + "can't download input {} epg url: {} => {}", + input.name, + sanitize_sensitive_info(url_str), + sanitize_sensitive_info(&e.to_string()) + ); + Err(TuliproxError::RepositoryNetwork(format!( + "can't download input {} epg url: {} => {}", + input.name, + sanitize_sensitive_info(url_str), + sanitize_sensitive_info(&e.to_string()) + ))) + } + } + } else { + let Some(file_path) = get_file_path(storage_dir, Some(PathBuf::from(url_str))) else { + let msg = format!("can't read input url: {}", sanitize_sensitive_info(url_str)); + error!("{msg}"); + return Err(TuliproxError::RepositoryNetwork(msg)); + }; + if !file_path.exists() { + let msg = format!("can't read input url: {}", sanitize_sensitive_info(url_str)); + error!("{msg}"); + return Err(TuliproxError::RepositoryNetwork(msg)); + } + + copy_local_epg_file_to_persist(&file_path, persist_path, max_bytes).await.map_err(|err| { + error!("can't persist to: {} => {}", persist_path.display(), err); + TuliproxError::RepositoryNetwork(format!("Failed to persist: {} => {err}", persist_path.display())) + }) + } +} + +pub async fn get_input_text_content_as_stream( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + storage_dir: &str, + persist_filepath: Option, +) -> Result { + debug_if_enabled!( + "getting input text content storage_dir: {}, url: {}", + storage_dir, + sanitize_sensitive_info(&input.url) + ); + + if input.url.parse::().is_ok() { + match download_text_content_as_stream(app_config, client, input, persist_filepath).await { + Ok((content, _response_url)) => Ok(content), + Err(e) => { + error!("Failed to download input '{}': {}", input.name, sanitize_sensitive_info(&e.to_string())); + Err(TuliproxError::RepositoryNetwork(format!( + "Failed to download input '{}': {}", + input.name, + sanitize_sensitive_info(&e.to_string()) + ))) + } + } + } else { + let result = match get_file_path(storage_dir, Some(PathBuf::from(&input.url))) { + Some(filepath) => { + if filepath.exists() { + match get_local_file_content_as_stream(&filepath).await { + Ok(content) => { + if let Some(path) = persist_filepath { + let tee = tee_dyn_reader( + content, + &path, + Some(Arc::new(|size| { + debug_if_enabled!("Persisted {} bytes", human_readable_byte_size(size as u64)); + })), + ) + .await; + Some(tee) + } else { + Some(content) + } + } + Err(err) => { + return Err(TuliproxError::RepositoryNetwork(format!("Failed : {err}"))); + } + } + } else { + None + } + } + None => None, + }; + result.map_or_else( + || { + let msg = format!("can't read input url: {}", sanitize_sensitive_info(&input.url)); + error!("{msg}"); + Err(TuliproxError::RepositoryNetwork(msg)) + }, + Ok, + ) + } +} + +// read local file content and return it as a string. +// Gzipped file content is supported. +pub async fn get_local_file_content(file_path: &Path) -> Result { + // open file + let file = File::open(file_path).await.map_err(|err| { + std::io::Error::new(ErrorKind::NotFound, format!("Failed to open file: {}, {err:?}", file_path.display())) + })?; + + let mut buf_reader = async_file_reader(file); + + // Peek first 2 bytes to detect gzip encoding + let buffer = buf_reader.fill_buf().await?; + let is_gzipped = buffer.len() >= 2 && is_gzip(&buffer[0..2]); + + let mut decoded = String::new(); + + if is_gzipped { + // Use async gzip decoder + let mut gzip_decoder = async_compression::tokio::bufread::GzipDecoder::new(buf_reader); + gzip_decoder + .read_to_string(&mut decoded) + .await + .map_err(|e| std::io::Error::other(format!("Failed to decode gzip content: {e}")))?; + } else { + // read plaintext + buf_reader + .read_to_string(&mut decoded) + .await + .map_err(|e| std::io::Error::other(format!("Failed to read file: {e}")))?; + } + + Ok(decoded) +} + +pub async fn get_local_file_content_as_stream(file_path: &Path) -> Result { + // open file + let file = File::open(file_path).await.map_err(|err| { + std::io::Error::new(ErrorKind::NotFound, format!("Failed to open file: {}, {err:?}", file_path.display())) + })?; + + let mut buf_reader = async_file_reader(file); + + // Peek first 2 Bytes, for gzip detection + let buffer = buf_reader.fill_buf().await?; + let is_gzipped = buffer.len() >= 2 && is_gzip(&buffer[0..2]); + + if is_gzipped { + // use Async Gzip Decoder + Ok(Box::pin(async_compression::tokio::bufread::GzipDecoder::new(buf_reader))) + } else { + Ok(Box::pin(buf_reader)) + } +} + +pub async fn get_remote_content_as_file( + app_config: &Arc, + client: &reqwest::Client, + input: &ConfigInput, + headers: Option<&HeaderMap>, + url: &Url, + file_path: &Path, +) -> Result { + get_remote_content_as_file_with_options( + app_config, + client, + input, + headers, + url, + file_path, + RecordingTaskOptions::default(), + ) + .await +} + +pub async fn get_remote_content_as_file_with_options( + app_config: &Arc, + client: &reqwest::Client, + input: &ConfigInput, + headers: Option<&HeaderMap>, + url: &Url, + file_path: &Path, + options: RecordingTaskOptions, +) -> Result { + let input_source = InputSource { + name: input.name.clone(), + url: url.to_string(), + provider: input.get_resolve_provider(url.as_str()), + username: input.username.clone(), + password: input.password.clone(), + method: input.method, + headers: input.headers.clone(), + }; + + let response = send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + app_config, + client, + &input_source, + headers, + url, + 10, + RequestFetchOptions::default(), + ) + .await? + .response; + + let start_time = tokio::time::Instant::now(); + let (temp_file, output_file) = if options.atomic_write { + let (temp_file, output_file) = create_atomic_download_file(file_path)?; + (Some(temp_file), output_file) + } else { + (None, File::create(file_path).await?) + }; + let mut writer = async_file_writer(output_file); + + let mut stream = response.bytes_stream(); + let mut downloaded = 0_u64; + + let idle_timeout = tokio::time::Duration::from_secs(STREAM_IDLE_TIMEOUT); + let idle = sleep(idle_timeout); + tokio::pin!(idle); + + loop { + tokio::select! { + () = &mut idle => { + warn!("Stream idle for request, closing {}", sanitize_sensitive_info(url.as_ref())); + return Err(string_to_io_error(format!( + "Download timed out for {}", + sanitize_sensitive_info(url.as_ref()) + ))); + } + + chunk = stream.next() => { + idle.as_mut().reset(tokio::time::Instant::now() + idle_timeout); + + match chunk { + Some(Ok(bytes)) => { + downloaded = downloaded.checked_add(bytes.len() as u64).ok_or_else(|| { + string_to_io_error(format!( + "Download size overflow for {}", + sanitize_sensitive_info(url.as_ref()) + )) + })?; + if options.max_bytes.is_some_and(|max| downloaded > max) { + return Err(string_to_io_error(format!( + "Download exceeds configured limit for {}", + sanitize_sensitive_info(url.as_ref()) + ))); + } + writer.write_all(&bytes).await?; + } + Some(Err(e)) => { + return Err(string_to_io_error(format!("Failed to read chunk: {e}"))); + } + None => { + break; + } + } + } + } + } + + writer.flush().await?; + writer.shutdown().await?; + drop(writer); + + if let Some(temp_file) = temp_file { + persist_atomic_download_file(temp_file, file_path)?; + } + + debug!( + "File downloaded successfully to {}, took {}", + file_path.display(), + format_elapsed_time(start_time.elapsed().as_secs()) + ); + + Ok(file_path.to_path_buf()) +} + +impl TextContentBodyOptions { + /// Selects the narrowly scoped HLS-manifest fallback detection and decoded-size limit. + pub fn hls_manifest(max_decoded_bytes: usize, deadline: Duration) -> Self { + Self { + detection: ContentCodingDetection::DeclaredOrKnownHlsManifestMagic, + max_decoded_bytes: Some(max_decoded_bytes), + deadline: Some(deadline.max(Duration::from_millis(1))), + retry_owner: TextContentRetryOwner::DecodedBodyConsumer, + } + } + + fn legacy_text_with_deadline(deadline: Option) -> Self { + Self { deadline: deadline.map(|value| value.max(Duration::from_millis(1))), ..Self::default() } + } +} + +impl TextContentFetchOptions { + pub const fn new(request: RequestFetchOptions, body: TextContentBodyOptions) -> Self { Self { request, body } } + + pub(super) fn with_request_options(request: RequestFetchOptions) -> Self { + Self { body: TextContentBodyOptions::legacy_text_with_deadline(request.attempt_idle_timeout), request } + } +} + +pub(super) fn create_atomic_download_file(file_path: &Path) -> Result<(tempfile::NamedTempFile, File), Error> { + let parent = file_path.parent().filter(|path| !path.as_os_str().is_empty()).unwrap_or_else(|| Path::new(".")); + let temp_file = tempfile::Builder::new().prefix(".tuliprox-download-").suffix(".tmp").tempfile_in(parent)?; + let output_file = File::from_std(temp_file.reopen()?); + Ok((temp_file, output_file)) +} + +fn persist_atomic_download_file(temp_file: tempfile::NamedTempFile, file_path: &Path) -> Result<(), Error> { + match temp_file.persist(file_path) { + Ok(_) => Ok(()), + Err(err) => Err(err.error), + } +} + +async fn copy_local_epg_file_to_persist( + file_path: &Path, + persist_filepath: &Path, + max_bytes: Option, +) -> Result { + let mut reader = File::open(file_path).await?; + let (temp_file, output_file) = create_atomic_download_file(persist_filepath)?; + let mut writer = async_file_writer(output_file); + let mut copied = 0_u64; + let mut buffer = vec![0_u8; 64 * 1024].into_boxed_slice(); + + loop { + let read = reader.read(&mut buffer).await?; + if read == 0 { + break; + } + copied = copied + .checked_add(read as u64) + .ok_or_else(|| string_to_io_error(format!("Local EPG file size overflow for {}", file_path.display())))?; + if max_bytes.is_some_and(|max| copied > max) { + return Err(string_to_io_error(format!("Local EPG file {} exceeds configured limit", file_path.display()))); + } + writer.write_all(&buffer[..read]).await?; + } + + writer.flush().await?; + writer.shutdown().await?; + drop(writer); + drop(reader); + persist_atomic_download_file(temp_file, persist_filepath)?; + Ok(persist_filepath.to_path_buf()) +} + +async fn download_epg_content_as_file( + app_config: &Arc, + client: &reqwest::Client, + input: &ConfigInput, + headers: Option<&HeaderMap>, + url_str: &str, + persist_filepath: &Path, + max_bytes: Option, +) -> Result { + if let Ok(url) = url_str.parse::() { + match url.scheme() { + "file" => { + let file_path = url.to_file_path().map_err(|()| { + Error::new(ErrorKind::Unsupported, format!("Unknown file {}", sanitize_sensitive_info(url_str))) + })?; + if file_path.exists() { + copy_local_epg_file_to_persist(&file_path, persist_filepath, max_bytes).await + } else { + Err(Error::new(ErrorKind::NotFound, format!("Unknown file {}", file_path.display()))) + } + } + "http" | "https" | "provider" => { + get_remote_content_as_file_with_options( + app_config, + client, + input, + headers, + &url, + persist_filepath, + RecordingTaskOptions { max_bytes, atomic_write: true }, + ) + .await + } + scheme => Err(Error::new( + ErrorKind::Unsupported, + format!("Unsupported EPG URL scheme '{scheme}' for {}", sanitize_sensitive_info(url_str)), + )), + } + } else { + Err(Error::new(ErrorKind::Unsupported, format!("Malformed URL {}", sanitize_sensitive_info(url_str)))) + } +} + +pub async fn download_text_content( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + persist_filepath: Option, + trace_log: bool, +) -> Result<(String, String), Error> { + Box::pin(download_text_content_with_options( + app_config, + client, + input, + headers, + persist_filepath, + trace_log, + RequestFetchOptions::default(), + )) + .await +} + +pub async fn download_text_content_with_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + persist_filepath: Option, + trace_log: bool, + options: RequestFetchOptions, +) -> Result<(String, String), Error> { + let start_time = tokio::time::Instant::now(); + let result = if let Ok(url) = input.url.parse::() { + let result = if url.scheme() == "file" { + match url.to_file_path() { + Ok(file_path) => get_local_file_content(&file_path).await.map(|content| (content, url.to_string())), + Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), + } + } else { + get_remote_content_with_options(app_config, client, input, headers, &url, options).await + }; + match result { + Ok((content, response_url)) => { + if persist_filepath.is_some() { + persist_file(persist_filepath, &content).await; + } + Ok((content, response_url)) + } + Err(err) => Err(err), + } + } else { + Err(string_to_io_error(format!("Malformed URL {}", sanitize_sensitive_info(&input.url)))) + }; + + let level = if trace_log { log::Level::Trace } else { log::Level::Debug }; + if log_enabled!(level) { + if let Ok((_content, response_url)) = result.as_ref() { + log::log!( + level, + "Request took: {} {}", + format_elapsed_time(start_time.elapsed().as_secs()), + sanitize_sensitive_info(response_url.as_str()) + ); + } + } + + result +} + +pub async fn download_text_content_with_headers( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + trace_log: bool, +) -> Result<(String, String, HeaderMap), Error> { + Box::pin(download_text_content_with_headers_and_options( + app_config, + client, + input, + headers, + trace_log, + TextContentFetchOptions::default(), + )) + .await +} + +pub async fn download_text_content_with_headers_and_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + trace_log: bool, + options: TextContentFetchOptions, +) -> Result<(String, String, HeaderMap), Error> { + let start_time = tokio::time::Instant::now(); + let result = if let Ok(url) = input.url.parse::() { + let result = if url.scheme() == "file" { + match url.to_file_path() { + Ok(file_path) => { + get_local_file_content(&file_path).await.map(|content| (content, url.to_string(), HeaderMap::new())) + } + Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), + } + } else { + get_remote_content_with_headers_and_options(app_config, client, input, headers, &url, options).await + }; + result + } else { + Err(string_to_io_error(format!("Malformed URL {}", sanitize_sensitive_info(&input.url)))) + }; + + let level = if trace_log { log::Level::Trace } else { log::Level::Debug }; + if log_enabled!(level) { + if let Ok((_, response_url, _)) = result.as_ref() { + log::log!( + level, + "Request took: {} {}", + format_elapsed_time(start_time.elapsed().as_secs()), + sanitize_sensitive_info(response_url.as_str()) + ); + } + } + + result +} + +pub async fn download_text_content_as_stream( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + persist_filepath: Option, +) -> Result<(DynReader, String), Error> { + if let Ok(url) = input.url.parse::() { + let result = if url.scheme() == "file" { + match url.to_file_path() { + Ok(file_path) => get_local_file_content_as_stream(&file_path).await.map(|c| (c, url.to_string())), + Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), + } + } else { + get_remote_content_as_stream(app_config, client, input, None, &url).await + }; + match result { + Ok((content, response_url)) => { + if let Some(path) = persist_filepath { + let tee_reader: DynReader = tee_dyn_reader( + content, + &path, + Some(Arc::new(|size| { + debug!("Persisted {size} bytes"); + })), + ) + .await; + Ok((tee_reader, response_url)) + } else { + Ok((content, response_url)) + } + } + Err(err) => Err(err), + } + } else { + Err(string_to_io_error(format!("Malformed URL {}", sanitize_sensitive_info(&input.url)))) + } +} + +async fn download_json_content( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + persist_filepath: Option, + trace_log: bool, +) -> Result { + debug_if_enabled!("Downloading json content from {}", sanitize_sensitive_info(&input.url)); + match download_text_content(app_config, client, input, None, persist_filepath, trace_log).await { + Ok((content, _response_url)) => match serde_json::from_str::(&content) { + Ok(value) => Ok(value), + Err(err) => Err(string_to_io_error(format!("Failed to parse json {err}"))), + }, + Err(err) => Err(err), + } +} + +pub async fn get_input_json_content( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + persist_filepath: Option, + trace_log: bool, +) -> Result { + match download_json_content(app_config, client, input, persist_filepath, trace_log).await { + Ok(content) => Ok(content), + Err(e) => Err(TuliproxError::RepositoryNetwork(format!( + "can't download input {input} => {sanitized}", + input = input.name, + sanitized = sanitize_sensitive_info(&e.to_string()) + ))), + } +} + +async fn download_json_content_as_stream( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + persist_filepath: Option, +) -> Result { + debug_if_enabled!("Downloading json content as stream from {}", sanitize_sensitive_info(&input.url)); + match download_text_content_as_stream(app_config, client, input, persist_filepath).await { + Ok((reader, _response_url)) => Ok(reader), + Err(err) => Err(err), + } +} + +pub async fn get_input_json_content_as_stream( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + persist_filepath: Option, +) -> Result { + match download_json_content_as_stream(app_config, client, input, persist_filepath).await { + Ok(stream) => Ok(stream), + Err(e) => Err(TuliproxError::RepositoryNetwork(format!( + "can't download input {input} => {sanitized}", + input = input.name, + sanitized = sanitize_sensitive_info(&e.to_string()) + ))), + } +} diff --git a/backend/core/src/utils/network/request/headers.rs b/backend/core/src/utils/network/request/headers.rs new file mode 100644 index 000000000..efcf05748 --- /dev/null +++ b/backend/core/src/utils/network/request/headers.rs @@ -0,0 +1,255 @@ +use super::{preview_attempt_target, AttemptTarget, PROXY_DIAGNOSTICS_ONCE}; +use crate::model::{AppConfig, Config, ConfigProvider, InputSource, ReverseProxyDisabledHeaderConfig}; +use log::{debug, log_enabled, trace, Level}; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; +use shared::{ + defaults::DEFAULT_USER_AGENT, + utils::{filter_request_header, sanitize_sensitive_info}, +}; +use std::{collections::HashMap, sync::Arc}; +use url::Url; + +pub(super) fn log_proxy_diagnostics(config: &Config) { + PROXY_DIAGNOSTICS_ONCE.call_once(|| { + if let Some(proxy_cfg) = config.proxy.as_ref() { + let sanitized_url = sanitize_sensitive_info(proxy_cfg.url.as_str()); + let has_inline_credentials = proxy_cfg + .url + .contains('@') + || proxy_cfg.url.contains("://") + && proxy_cfg + .url + .split("://") + .nth(1) + .is_some_and(|part| part.contains('@')); + let has_explicit_credentials = + proxy_cfg.username.as_ref().is_some() || proxy_cfg.password.as_ref().is_some(); + debug!( + "Proxy config enabled: url={sanitized_url}, credentials_inline={has_inline_credentials}, credentials_fields={has_explicit_credentials}" + ); + } else { + debug!("Proxy config disabled (config.yml)"); + } + + let env_keys = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + "no_proxy", + ]; + let mut env_values = Vec::new(); + for key in env_keys { + if let Ok(value) = std::env::var(key) { + if !value.trim().is_empty() { + env_values.push((key, sanitize_sensitive_info(value.as_str()).to_string())); + } + } + } + if env_values.is_empty() { + debug!("Proxy env vars not set"); + } else { + debug!("Proxy env vars present: {env_values:?}"); + } + }); +} + +pub(super) fn format_request_target_for_logging(target: &AttemptTarget) -> String { + if target.effective_url.scheme().eq_ignore_ascii_case("https") { + if let Some(connect_ip) = target.connect_ip { + format!("{} (connect_ip={connect_ip})", target.request_url) + } else { + target.request_url.to_string() + } + } else { + target.effective_url.to_string() + } +} + +pub fn preview_request_target_for_logging(url: &Url, provider: Option<&Arc>) -> String { + let target = preview_attempt_target(url, provider); + format_request_target_for_logging(&target) +} + +pub fn preview_request_diagnostics_for_logging(url: &Url, provider: Option<&Arc>) -> String { + let target = preview_attempt_target(url, provider); + let mut parts = vec![ + format!("request_url={}", sanitize_sensitive_info(target.request_url.as_str())), + format!("effective_url={}", sanitize_sensitive_info(target.effective_url.as_str())), + ]; + + if let Some(host_header) = target.host_header.as_ref() { + parts.push(format!("host_header={}", sanitize_sensitive_info(host_header))); + } + if let Some(connect_ip) = target.connect_ip { + parts.push(format!("connect_ip={}", sanitize_sensitive_info(&connect_ip.to_string()))); + } + if let Some(sni_host) = target.sni_host.as_ref() { + parts.push(format!("sni_host={}", sanitize_sensitive_info(sni_host))); + } + + parts.join(", ") +} + +pub(super) fn prepare_input_request_headers( + app_config: &Arc, + input: &InputSource, + headers: Option<&HeaderMap>, +) -> (HashMap, Option) { + let custom_headers = headers + .map(|h| h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::>()); + + let config = app_config.config.load(); + let default_user_agent = config.default_user_agent.clone(); + let disabled_headers = config.get_disabled_headers(); + drop(config); + + let merged = get_request_headers( + Some(&input.headers), + custom_headers.as_ref(), + disabled_headers.as_ref(), + default_user_agent.as_deref(), + ); + + let request_headers: HashMap = merged + .iter() + .map(|(k, v)| (k.as_str().to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())) + .collect(); + + (request_headers, default_user_agent) +} + +pub fn get_request_headers( + request_headers: Option<&HashMap>, + custom_headers: Option<&HashMap, S>>, + disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, + default_user_agent: Option<&str>, +) -> HeaderMap { + let mut headers = HeaderMap::default(); + let mut has_user_agent = false; + + // 1. First, we process the configured request headers (from input config). + // These should have the highest priority. + if let Some(req_headers) = request_headers { + for (key, value) in req_headers { + if let (Ok(key), Ok(value)) = + (HeaderName::from_bytes(key.as_bytes()), HeaderValue::from_bytes(value.as_bytes())) + { + if filter_request_header(key.as_str()) { + if disabled_headers.as_ref().is_some_and(|d| d.should_remove(key.as_str())) { + continue; + } + if key == axum::http::header::USER_AGENT { + has_user_agent = true; + } + headers.insert(key, value); + } + } + } + } + + // 2. Next, we process custom headers (from the client request). + // These are only added if they don't already exist in the headers map (i.e., not overridden by config). + if let Some(custom) = custom_headers { + for (key, value) in custom { + let key_lc = key.to_lowercase(); + if filter_request_header(key_lc.as_str()) { + if disabled_headers.as_ref().is_some_and(|d| d.should_remove(key_lc.as_str())) { + continue; + } + if let (Ok(name), Ok(val)) = (HeaderName::from_bytes(key.as_bytes()), HeaderValue::from_bytes(value)) { + // Only insert if not already present (config takes precedence) + if !headers.contains_key(&name) { + if name == axum::http::header::USER_AGENT { + has_user_agent = true; + } + headers.insert(name, val); + } + } + } + } + } + + if log_enabled!(Level::Trace) { + let he: HashMap = + headers.iter().map(|(k, v)| (k.to_string(), String::from_utf8_lossy(v.as_bytes()).to_string())).collect(); + if !he.is_empty() { + trace!("Request headers {he:?}"); + } + } + + // 3. Finally, if no User-Agent was provided by config OR client, use the default. + if !has_user_agent + && !disabled_headers.is_some_and(|disabled| disabled.should_remove(axum::http::header::USER_AGENT.as_str())) + { + let config_ua = default_user_agent + .and_then(|ua| { + let trimmed = ua.trim(); + (!trimmed.is_empty()).then_some(trimmed) + }) + .and_then(|ua| HeaderValue::from_str(ua).ok()); + + headers.insert( + axum::http::header::USER_AGENT, + config_ua.unwrap_or_else(|| HeaderValue::from_static(DEFAULT_USER_AGENT)), + ); + } + + headers +} + +pub fn overlay_upstream_user_agent( + headers: &mut HeaderMap, + upstream_user_agent: Option<&str>, + disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, +) { + if disabled_headers.is_some_and(|disabled| disabled.should_remove(axum::http::header::USER_AGENT.as_str())) { + return; + } + if let Some(value) = upstream_user_agent + .map(str::trim) + .filter(|value| !value.is_empty()) + .and_then(|value| HeaderValue::from_str(value).ok()) + { + headers.insert(axum::http::header::USER_AGENT, value); + } +} + +/// Appends a stable playback-session index to an already resolved User-Agent. +pub fn append_user_agent_stream_index(headers: &mut HeaderMap, stream_index: u64) { + let Some(user_agent) = headers.get(axum::http::header::USER_AGENT).map(HeaderValue::as_bytes) else { + return; + }; + let Some(first) = user_agent.iter().position(|byte| !byte.is_ascii_whitespace()) else { + return; + }; + let last = user_agent.iter().rposition(|byte| !byte.is_ascii_whitespace()).map_or(first, |index| index + 1); + let user_agent = &user_agent[first..last]; + + let mut digits = [0_u8; 20]; + let mut cursor = digits.len(); + let mut remaining = stream_index; + loop { + cursor -= 1; + digits[cursor] = b'0' + (remaining % 10) as u8; + remaining /= 10; + if remaining == 0 { + break; + } + } + let suffix = &digits[cursor..]; + if user_agent.strip_suffix(suffix).is_some_and(|prefix| prefix.ends_with(b" ")) { + return; + } + + let mut indexed_user_agent = Vec::with_capacity(user_agent.len() + 1 + suffix.len()); + indexed_user_agent.extend_from_slice(user_agent); + indexed_user_agent.push(b' '); + indexed_user_agent.extend_from_slice(suffix); + if let Ok(value) = HeaderValue::from_bytes(&indexed_user_agent) { + headers.insert(axum::http::header::USER_AGENT, value); + } +} diff --git a/backend/core/src/utils/network/request/options.rs b/backend/core/src/utils/network/request/options.rs new file mode 100644 index 000000000..350627d82 --- /dev/null +++ b/backend/core/src/utils/network/request/options.rs @@ -0,0 +1,34 @@ +use super::{RequestFetchOptions, TextContentBodyOptions}; +use crate::utils::content_coding::{apply_outbound_content_coding_policy, ContentCodingDetection}; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(super) enum ResourceRetryExecution { + #[default] + Configured, + ProviderFailoverOnly, +} + +pub(super) fn apply_request_fetch_options(request: &mut reqwest::Request, options: RequestFetchOptions) { + if let Some(timeout) = options.attempt_idle_timeout { + *request.timeout_mut() = Some(timeout); + } + apply_outbound_content_coding_policy(request.headers_mut(), options.content_coding); +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(super) enum TextContentRetryOwner { + #[default] + RequestStack, + DecodedBodyConsumer, +} + +impl Default for TextContentBodyOptions { + fn default() -> Self { + Self { + detection: ContentCodingDetection::DeclaredOrLegacyTextMagic, + max_decoded_bytes: None, + deadline: None, + retry_owner: TextContentRetryOwner::RequestStack, + } + } +} diff --git a/backend/core/src/utils/network/request/protocol.rs b/backend/core/src/utils/network/request/protocol.rs new file mode 100644 index 000000000..b17de11c4 --- /dev/null +++ b/backend/core/src/utils/network/request/protocol.rs @@ -0,0 +1,65 @@ +use super::MimeCategory; +use reqwest::StatusCode; +use shared::utils::CONTENT_TYPE_JSON; +use url::Url; + +pub fn classify_content_type(headers: &[(String, String)]) -> MimeCategory { + headers.iter().find_map(|(k, v)| (k == axum::http::header::CONTENT_TYPE.as_str()).then_some(v)).map_or( + MimeCategory::Unknown, + |v| match v.to_lowercase().as_str() { + v if v.starts_with("video/") || v == "application/octet-stream" => MimeCategory::Video, + v if v.contains("mpegurl") => MimeCategory::M3U8, + v if v.starts_with("image/") => MimeCategory::Image, + v if v.starts_with(CONTENT_TYPE_JSON) || v.ends_with("+json") => MimeCategory::Json, + v if v.starts_with("application/xml") || v.ends_with("+xml") || v == "text/xml" => MimeCategory::Xml, + v if v.starts_with("text/") => MimeCategory::Text, + _ => MimeCategory::Unclassified, + }, + ) +} + +pub fn format_http_status(status: StatusCode) -> String { + let code = status.as_u16(); + match status.canonical_reason() { + Some(reason) => format!("{code} {reason}"), + None => code.to_string(), + } +} + +pub fn content_type_from_ext(ext: &str) -> &'static str { + match ext.to_ascii_lowercase().as_str() { + "mp4" | "fmp4" | "m4s" | "m4v" | "cmfv" => "video/mp4", + "m4a" | "cmfa" => "audio/mp4", + "mkv" => "video/x-matroska", + "avi" => "video/x-msvideo", + "mov" => "video/quicktime", + "webm" => "video/webm", + "ts" => "video/mp2t", + _ => "application/octet-stream", + } +} + +pub fn parse_range(range: &str) -> Option<(u64, Option)> { + // expect: "bytes=START-END" + if !range.starts_with("bytes=") { + return None; + } + + let range = &range[6..]; + let mut parts = range.split('-'); + + let start = parts.next()?.parse().ok()?; + let end = parts.next().and_then(|s| s.parse().ok()); + + Some((start, end)) +} + +pub fn is_file_url(url: &str) -> bool { Url::parse(url).is_ok_and(|u| u.scheme().eq_ignore_ascii_case("file")) } + +pub fn is_uri(url: &str) -> bool { + Url::parse(url).is_ok_and(|u| { + u.scheme().eq_ignore_ascii_case("file") + || u.scheme().eq_ignore_ascii_case("http") + || u.scheme().eq_ignore_ascii_case("https") + }) +} diff --git a/backend/core/src/utils/network/request/redirect.rs b/backend/core/src/utils/network/request/redirect.rs new file mode 100644 index 000000000..29557c51f --- /dev/null +++ b/backend/core/src/utils/network/request/redirect.rs @@ -0,0 +1,266 @@ +use super::{ + get_local_file_content, is_retryable_text_response_status, perform_backoff, read_text_response_with_body_options, + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, should_retry_text_body_error, + sleep_before_text_body_retry, text_body_retry_values, text_response_status_error, RequestFetchOptions, + TextContentFetchOptions, +}; +use crate::{ + model::{AppConfig, InputSource}, + utils::persist_file, +}; +use log::log_enabled; +use regex::Regex; +use reqwest::header::HeaderMap; +use shared::{error::string_to_io_error, model::format_elapsed_time, utils::sanitize_sensitive_info}; +use std::{collections::HashMap, io::Error, path::PathBuf, sync::Arc}; +use url::Url; + +pub(super) fn is_failover_redirect(url: &Url, patterns: &[Arc]) -> bool { + let redirect_url = url.as_str(); + patterns.iter().any(|pattern| pattern.is_match(redirect_url)) +} + +#[allow(clippy::too_many_lines)] +async fn get_remote_content_with_manual_redirects_and_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + url: &Url, + max_redirects: usize, + options: RequestFetchOptions, +) -> Result<(String, String), Error> { + get_remote_content_with_manual_redirects_and_headers_and_options( + app_config, + client, + input, + headers, + url, + max_redirects, + TextContentFetchOptions::with_request_options(options), + ) + .await + .map(|(content, response_url, _)| (content, response_url)) +} + +async fn get_remote_content_with_manual_redirects_and_headers_and_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + url: &Url, + max_redirects: usize, + options: TextContentFetchOptions, +) -> Result<(String, String, HeaderMap), Error> { + let (max_attempts, backoff_ms, backoff_multiplier) = text_body_retry_values(app_config, options.body); + let attempt_options = if max_attempts > 1 { options.request.without_resource_retries() } else { options.request }; + + // Manual redirects retain their own credential-scrubbing loop inside each caller-owned logical attempt. + for attempt in 0..max_attempts { + let response = match send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + app_config, + client, + input, + headers, + url, + max_redirects, + attempt_options, + ) + .await + { + Ok(result) => result.response, + Err(error) if should_retry_text_body_error(&error) && attempt + 1 < max_attempts => { + sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &error).await; + continue; + } + Err(error) => return Err(error), + }; + let status = response.status(); + if !status.is_success() { + if is_retryable_text_response_status(status) && attempt + 1 < max_attempts { + perform_backoff(attempt, backoff_ms, backoff_multiplier, &response).await; + continue; + } + return Err(text_response_status_error(status, url)); + } + match read_text_response_with_body_options(response, options.body).await { + Ok(result) => return Ok(result), + Err(error) if should_retry_text_body_error(&error) && attempt + 1 < max_attempts => { + sleep_before_text_body_retry(attempt, backoff_ms, backoff_multiplier, &error).await; + } + Err(error) => return Err(error), + } + } + Err(string_to_io_error("Text response body retry attempts exhausted")) +} + +pub(super) fn same_origin(lhs: &Url, rhs: &Url) -> bool { + lhs.scheme().eq_ignore_ascii_case(rhs.scheme()) + && lhs.host_str() == rhs.host_str() + && lhs.port_or_known_default() == rhs.port_or_known_default() +} + +/// Reports whether a request header may be retained when the target origin changes. +pub fn is_safe_cross_origin_redirect_header(key: &str) -> bool { + key.eq_ignore_ascii_case("accept") + || key.eq_ignore_ascii_case("accept-encoding") + || key.eq_ignore_ascii_case("accept-language") + || key.eq_ignore_ascii_case("user-agent") + || key.eq_ignore_ascii_case("range") + || key.eq_ignore_ascii_case("if-range") + || key.eq_ignore_ascii_case("icy-metadata") +} + +pub(super) fn strip_sensitive_headers_for_cross_origin_redirect(headers: &mut HashMap) { + headers.retain(|key, _| is_safe_cross_origin_redirect_header(key)); +} + +pub async fn download_text_content_with_manual_redirects( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + persist_filepath: Option, + trace_log: bool, + max_redirects: usize, +) -> Result<(String, String), Error> { + Box::pin(download_text_content_with_manual_redirects_and_options( + app_config, + client, + input, + headers, + persist_filepath, + trace_log, + max_redirects, + RequestFetchOptions::default(), + )) + .await +} + +#[allow(clippy::too_many_arguments)] +pub async fn download_text_content_with_manual_redirects_and_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + persist_filepath: Option, + trace_log: bool, + max_redirects: usize, + options: RequestFetchOptions, +) -> Result<(String, String), Error> { + let start_time = tokio::time::Instant::now(); + let result = if let Ok(url) = input.url.parse::() { + let result = if url.scheme() == "file" { + match url.to_file_path() { + Ok(file_path) => get_local_file_content(&file_path).await.map(|content| (content, url.to_string())), + Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), + } + } else { + get_remote_content_with_manual_redirects_and_options( + app_config, + client, + input, + headers, + &url, + max_redirects, + options, + ) + .await + }; + match result { + Ok((content, response_url)) => { + if persist_filepath.is_some() { + persist_file(persist_filepath, &content).await; + } + Ok((content, response_url)) + } + Err(err) => Err(err), + } + } else { + Err(string_to_io_error(format!("Malformed URL {}", sanitize_sensitive_info(&input.url)))) + }; + + let level = if trace_log { log::Level::Trace } else { log::Level::Debug }; + if log_enabled!(level) { + if let Ok((_content, response_url)) = result.as_ref() { + log::log!( + level, + "Request took: {} {}", + format_elapsed_time(start_time.elapsed().as_secs()), + sanitize_sensitive_info(response_url.as_str()) + ); + } + } + + result +} + +pub async fn download_text_content_with_manual_redirects_and_headers( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + trace_log: bool, + max_redirects: usize, +) -> Result<(String, String, HeaderMap), Error> { + Box::pin(download_text_content_with_manual_redirects_and_headers_and_options( + app_config, + client, + input, + headers, + trace_log, + max_redirects, + TextContentFetchOptions::default(), + )) + .await +} + +pub async fn download_text_content_with_manual_redirects_and_headers_and_options( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + trace_log: bool, + max_redirects: usize, + options: TextContentFetchOptions, +) -> Result<(String, String, HeaderMap), Error> { + let start_time = tokio::time::Instant::now(); + let result = if let Ok(url) = input.url.parse::() { + let result = if url.scheme() == "file" { + match url.to_file_path() { + Ok(file_path) => { + get_local_file_content(&file_path).await.map(|content| (content, url.to_string(), HeaderMap::new())) + } + Err(()) => Err(string_to_io_error(format!("Unknown file {}", sanitize_sensitive_info(&input.url)))), + } + } else { + get_remote_content_with_manual_redirects_and_headers_and_options( + app_config, + client, + input, + headers, + &url, + max_redirects, + options, + ) + .await + }; + result + } else { + Err(string_to_io_error(format!("Malformed URL {}", sanitize_sensitive_info(&input.url)))) + }; + + let level = if trace_log { log::Level::Trace } else { log::Level::Debug }; + if log_enabled!(level) { + if let Ok((_, response_url, _)) = result.as_ref() { + log::log!( + level, + "Request took: {} {}", + format_elapsed_time(start_time.elapsed().as_secs()), + sanitize_sensitive_info(response_url.as_str()) + ); + } + } + + result +} diff --git a/backend/core/src/utils/network/request/retry.rs b/backend/core/src/utils/network/request/retry.rs new file mode 100644 index 000000000..3ca55f19d --- /dev/null +++ b/backend/core/src/utils/network/request/retry.rs @@ -0,0 +1,1126 @@ +use super::{ + execute_attempt_request, format_http_status, format_request_target_for_logging, get_client_request, + is_failover_redirect, prepare_input_request_headers, prepare_physical_request_attempt, + resolve_attempt_target_at_provider_index, same_origin, strip_sensitive_headers_for_cross_origin_redirect, + text_response_error_log_label, AttemptTarget, ProviderFailoverResponse, RequestFetchOptions, + TextContentBodyOptions, TextContentRetryOwner, +}; +use crate::{ + model::{AppConfig, ConfigProvider, InputSource, ResourceRetryConfig}, + utils::content_coding::{content_decoding_error_from_io, is_http_body_transport_error, ContentCodingError}, +}; +use log::{debug, error, log_enabled, warn, Level}; +use reqwest::{header::HeaderMap, StatusCode}; +use shared::{ + error::string_to_io_error, + model::{OnConnectErrorPolicy, ProviderUrlSelectionPolicy}, + utils::sanitize_sensitive_info, +}; +use std::{ + collections::HashSet, + io::{Error, ErrorKind}, + net::IpAddr, + sync::Arc, + time::Duration, +}; +use tokio::time::sleep; +use url::Url; + +pub(super) fn provider_start_index(provider: Option<&Arc>) -> usize { + provider.map_or(0, |provider| match provider.provider_url_selection_policy() { + ProviderUrlSelectionPolicy::ResumeLastWorking => provider.get_current_index(), + ProviderUrlSelectionPolicy::RestartFromFirst => 0, + }) +} + +pub(super) fn next_provider_url_index( + current_index: usize, + provider_url_count: usize, + start_index: usize, +) -> Option { + if provider_url_count <= 1 { + return None; + } + + let next_index = (current_index + 1) % provider_url_count; + (next_index != start_index).then_some(next_index) +} + +fn provider_cycle_exhausted(provider: &ConfigProvider, current_index: usize, start_index: usize) -> bool { + next_provider_url_index(current_index, provider.urls.len(), start_index).is_none() +} + +fn log_provider_cycle_exhausted( + provider: &ConfigProvider, + start_index: usize, + current_index: usize, + last_failure: &str, +) { + error!( + "Provider '{}' exhausted all {} URL(s) after one full cycle starting at preferred index {} and ending at index {}: {}", + provider.name, + provider.urls.len(), + start_index, + current_index, + sanitize_sensitive_info(last_failure) + ); +} + +fn rotate_to_next_provider_url( + provider: &ConfigProvider, + provider_url_index: &mut usize, + start_provider_index: usize, + reason: &str, +) -> bool { + let Some(next_index) = next_provider_url_index(*provider_url_index, provider.urls.len(), start_provider_index) + else { + return false; + }; + + warn!( + "Provider '{}' failover: {} -> switching from URL index {} to {}", + provider.name, + sanitize_sensitive_info(reason), + *provider_url_index, + next_index + ); + *provider_url_index = next_index; + true +} + +pub(super) fn should_try_next_ip_on_connect_error( + provider: Option<&Arc>, + target: &AttemptTarget, + attempted_ips: &mut HashSet, +) -> bool { + let Some(provider) = provider else { + return false; + }; + let Some(connect_ip) = target.connect_ip else { + return false; + }; + let Some(dns_host) = target.dns_host.as_ref() else { + return false; + }; + let Some(dns_cfg) = provider.get_dns_config() else { + return false; + }; + if dns_cfg.on_connect_error != OnConnectErrorPolicy::TryNextIp { + return false; + } + + let inserted = attempted_ips.insert(connect_ip); + if !inserted { + return false; + } + + let total_ips = provider.ip_count_for_host(dns_host); + total_ips > attempted_ips.len() +} + +#[allow(clippy::too_many_lines)] +async fn send_with_provider_failover_only_with_options( + app_config: &Arc, + url: &Url, + provider: Option<&Arc>, + allow_redirects: bool, + options: RequestFetchOptions, + mut send: impl FnMut(&Url) -> reqwest::RequestBuilder, +) -> Result { + let failover_patterns = app_config.config.load().reverse_proxy.as_ref().map_or_else( + || ResourceRetryConfig::default().failover_redirect_patterns, + |rp| rp.resource_retry.failover_redirect_patterns.clone(), + ); + + let start_provider_index = provider_start_index(provider); + let mut provider_url_index = start_provider_index; + let idle_timeout = options.attempt_idle_timeout_or_default(); + let idle = sleep(idle_timeout); + tokio::pin!(idle); + + 'provider_loop: loop { + let mut attempted_dns_ips = HashSet::new(); + + 'ip_loop: loop { + let attempt_target = resolve_attempt_target_at_provider_index(url, provider, provider_url_index); + if log_enabled!(Level::Debug) { + if let Some(current_provider) = provider { + let attempt_target_log = format_request_target_for_logging(&attempt_target); + debug!( + "Provider '{}' acquiring URL index {} of {}: {}", + current_provider.name, + provider_url_index, + current_provider.urls.len(), + sanitize_sensitive_info(attempt_target_log.as_str()) + ); + } + } + + idle.as_mut().reset(tokio::time::Instant::now() + idle_timeout); + let (base_client, request) = + prepare_physical_request_attempt(send(&attempt_target.request_url), &attempt_target, options)?; + + tokio::select! { + () = &mut idle => { + if should_try_next_ip_on_connect_error(provider, &attempt_target, &mut attempted_dns_ips) { + continue 'ip_loop; + } + + let last_provider_failure = format!( + "idle timeout while trying {}", + sanitize_sensitive_info(attempt_target.request_url.as_str()) + ); + if let Some(current_provider) = provider { + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + "idle timeout", + ) { + continue 'provider_loop; + } + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + &last_provider_failure, + ); + } + + return Err(Error::new( + ErrorKind::TimedOut, + format!("Request timed out: {}", sanitize_sensitive_info(url.as_str())), + )); + } + result = execute_attempt_request(app_config, base_client, request, &attempt_target) => match result { + Ok(response) => { + let status = response.status(); + if allow_redirects && status.is_redirection() { + if let Some(current_provider) = provider { + current_provider.set_current_index(provider_url_index); + } + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); + } + + let is_failover = is_failover_redirect(response.url(), &failover_patterns); + if !is_failover && !should_trigger_failover(status) { + if status.is_success() { + if let Some(current_provider) = provider { + current_provider.set_current_index(provider_url_index); + } + } + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); + } + + let last_provider_failure = format!( + "status {} while trying {}", + format_http_status(status), + sanitize_sensitive_info(attempt_target.request_url.as_str()) + ); + + if let Some(current_provider) = provider { + let reason = format!("status {}", format_http_status(status)); + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + reason.as_str(), + ) { + continue 'provider_loop; + } + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + &last_provider_failure, + ); + } + + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); + }, + Err(err) => { + if (err.is_timeout() || err.is_connect()) + && should_try_next_ip_on_connect_error(provider, &attempt_target, &mut attempted_dns_ips) + { + continue 'ip_loop; + } + + let last_provider_failure = format!( + "connection error while trying {}: {}", + sanitize_sensitive_info(attempt_target.request_url.as_str()), + sanitize_sensitive_info(err.to_string().as_str()) + ); + + if err.is_timeout() || err.is_connect() { + if let Some(current_provider) = provider { + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + "connection error", + ) { + continue 'provider_loop; + } + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + &last_provider_failure, + ); + } + } + + let message = format!("Request error: {}", sanitize_sensitive_info(err.to_string().as_str())); + return Err(if err.is_timeout() { + Error::new(ErrorKind::TimedOut, message) + } else if err.is_connect() { + Error::new(ErrorKind::ConnectionRefused, message) + } else { + string_to_io_error(message) + }); + }, + } + } + } + } +} + +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss, clippy::cast_precision_loss)] +pub fn calculate_retry_backoff(base_delay_ms: u64, multiplier: f64, attempt: u32) -> u64 { + let base = base_delay_ms.max(1); + if multiplier <= 1.0 { + return base; + } + let delay = (base as f64) * multiplier.powi(i32::try_from(attempt).unwrap_or(i32::MAX)); + if !delay.is_finite() || delay < 1.0 { + base + } else if delay >= u64::MAX as f64 { + u64::MAX + } else { + delay as u64 + } +} + +/// Sends a request with retry logic and optional provider failover support. +pub async fn send_with_retry_and_provider( + app_config: &Arc, + url: &Url, // Used primarily for logging/context + provider: Option<&Arc>, + allow_redirects: bool, + send: impl FnMut(&Url) -> reqwest::RequestBuilder, +) -> Result { + send_with_retry_and_provider_policy_with_options( + app_config, + url, + provider, + allow_redirects, + true, + RequestFetchOptions::default(), + send, + ) + .await +} + +/// Canonical retry and provider-failover entry point for outbound resource requests. +/// +/// `send_with_retry_and_provider` is a thin wrapper that enables the standard retry policy. Retry attempt counts, +/// backoff values, and failover redirect patterns are sourced from `AppConfig` (`reverse_proxy.resource_retry`). The +/// `url` argument is used as the stable logging/context URL; callers should pass the original request target rather +/// than an already-rotated provider URL. +/// +/// When `retry_enabled` is `false`, this function forces `max_attempts` to 1, disables provider URL rotation for idle +/// timeouts, retryable HTTP statuses, and connection/timeout errors, and skips the final fallback provider rotation +/// after attempts are exhausted. +#[allow(clippy::too_many_lines)] +pub async fn send_with_retry_and_provider_policy( + app_config: &Arc, + url: &Url, // Used primarily for logging/context + provider: Option<&Arc>, + allow_redirects: bool, + retry_enabled: bool, + send: impl FnMut(&Url) -> reqwest::RequestBuilder, +) -> Result { + send_with_retry_and_provider_policy_with_options( + app_config, + url, + provider, + allow_redirects, + retry_enabled, + RequestFetchOptions::default(), + send, + ) + .await +} + +#[allow(clippy::too_many_lines)] +pub async fn send_with_retry_and_provider_policy_with_options( + app_config: &Arc, + url: &Url, // Used primarily for logging/context + provider: Option<&Arc>, + allow_redirects: bool, + retry_enabled: bool, + options: RequestFetchOptions, + send: impl FnMut(&Url) -> reqwest::RequestBuilder, +) -> Result { + send_with_retry_and_provider_policy_with_options_result( + app_config, + url, + provider, + allow_redirects, + retry_enabled, + options, + send, + ) + .await + .map(|result| result.response) +} + +#[allow(clippy::too_many_lines)] +async fn send_with_retry_and_provider_policy_with_options_result( + app_config: &Arc, + url: &Url, // Used primarily for logging/context + provider: Option<&Arc>, + allow_redirects: bool, + retry_enabled: bool, + options: RequestFetchOptions, + mut send: impl FnMut(&Url) -> reqwest::RequestBuilder, +) -> Result { + if options.uses_provider_failover_only() { + return send_with_provider_failover_only_with_options( + app_config, + url, + provider, + allow_redirects, + options, + send, + ) + .await; + } + + let config = app_config.config.load(); + let (max_attempts, backoff_ms, backoff_multiplier, failover_patterns) = config.reverse_proxy.as_ref().map_or_else( + || { + let (a, b, c) = ResourceRetryConfig::get_default_retry_values(); + (a, b, c, ResourceRetryConfig::default().failover_redirect_patterns) + }, + |rp| { + let (a, b, c) = rp.resource_retry.get_retry_values(); + (a, b, c, rp.resource_retry.failover_redirect_patterns.clone()) + }, + ); + let max_attempts = if retry_enabled { max_attempts } else { 1 }; + drop(config); + + let idle_timeout = options.attempt_idle_timeout_or_default(); + let idle = sleep(idle_timeout); + tokio::pin!(idle); + + let max_provider_attempts = provider.as_ref().map_or(0, |p| p.urls.len()); + let start_provider_index = provider_start_index(provider); + let mut provider_url_index = start_provider_index; + let mut last_provider_failure: Option = None; + + 'provider_loop: loop { + // 2. Retry loop for the current URL + 'attempt_loop: for attempt in 0..max_attempts { + let mut attempted_dns_ips = HashSet::new(); + + 'ip_loop: loop { + let attempt_target = resolve_attempt_target_at_provider_index(url, provider, provider_url_index); + if log_enabled!(Level::Debug) { + if let Some(current_provider) = provider { + let attempt_target_log = format_request_target_for_logging(&attempt_target); + debug!( + "Provider '{}' attempting URL index {} of {}: {}", + current_provider.name, + provider_url_index, + max_provider_attempts, + sanitize_sensitive_info(attempt_target_log.as_str()) + ); + } + } + // Reset the idle timer for a new attempt + idle.as_mut().reset(tokio::time::Instant::now() + idle_timeout); + + let (base_client, request) = + prepare_physical_request_attempt(send(&attempt_target.request_url), &attempt_target, options)?; + + tokio::select! { + () = &mut idle => { + warn!("Request idle for too long: {}", sanitize_sensitive_info(url.as_str())); + last_provider_failure = Some(format!( + "idle timeout while trying {}", + sanitize_sensitive_info(attempt_target.request_url.as_str()) + )); + // 1. Try Provider Failover first + let mut provider_failover_exhausted = false; + if retry_enabled { + if let Some(current_provider) = provider { + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + "idle timeout", + ) { + continue 'provider_loop; + } + provider_failover_exhausted = + max_provider_attempts > 0 && provider_cycle_exhausted(current_provider.as_ref(), provider_url_index, start_provider_index); + } + } + + // 2. If no provider or rotation failed, check if we can retry the same URL + if attempt < max_attempts - 1 { + let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); + warn!("Idle timeout, retrying same URL in {}ms (attempt {})", delay, attempt + 1); + tokio::time::sleep(Duration::from_millis(delay)).await; + continue 'attempt_loop; + } + + if provider_failover_exhausted { + if let Some(current_provider) = provider { + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + last_provider_failure.as_deref().unwrap_or("idle timeout"), + ); + } + } + + return Err(Error::new( + ErrorKind::TimedOut, + format!( + "Request timed out and no retries left: {}", + sanitize_sensitive_info(url.as_str()) + ), + )); + } + + result = execute_attempt_request(app_config, base_client, request, &attempt_target) => { + match result { + Ok(response) => { + let status = response.status(); + if allow_redirects && status.is_redirection() { + if let Some(current_provider) = provider { + current_provider.set_current_index(provider_url_index); + } + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); + } + let is_failover = is_failover_redirect(response.url(), &failover_patterns); + if !is_failover && status.is_success() { + if let Some(current_provider) = provider { + current_provider.set_current_index(provider_url_index); + } + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); + } + + last_provider_failure = Some(format!( + "status {} while trying {}", + format_http_status(status), + sanitize_sensitive_info(attempt_target.request_url.as_str()) + )); + + // Failover check: Should we switch to the next provider URL? + let provider_failover_exhausted = retry_enabled + && (is_failover || should_trigger_failover(status)) + && provider.is_some_and(|current_provider| { + provider_cycle_exhausted(current_provider.as_ref(), provider_url_index, start_provider_index) + }); + if retry_enabled && (is_failover || should_trigger_failover(status)) { + if let Some(current_provider) = provider { + let reason = format!("status {}", format_http_status(status)); + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + reason.as_str(), + ) { + continue 'provider_loop; + } + } + } + + // Standard retry check for the same URL + let is_retryable = status.is_server_error() + || matches!(status, StatusCode::TOO_MANY_REQUESTS | StatusCode::REQUEST_TIMEOUT); + + if attempt < max_attempts - 1 && is_retryable { + perform_backoff(attempt, backoff_ms, backoff_multiplier, &response).await; + continue 'attempt_loop; + } + + if provider_failover_exhausted { + if let Some(current_provider) = provider { + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + last_provider_failure.as_deref().unwrap_or("request failed"), + ); + } + } + + if options.return_http_errors && (status.is_client_error() || status.is_server_error()) { + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); + } + + return Err(string_to_io_error(format!("Request failed ({}): {}", + format_http_status(status), sanitize_sensitive_info(url.as_str())))); + } + + Err(err) => { + // For DNS IP-connect policy, attempt next IP before provider URL rotation. + if retry_enabled + && (err.is_timeout() || err.is_connect()) + && should_try_next_ip_on_connect_error(provider, &attempt_target, &mut attempted_dns_ips) + { + continue 'ip_loop; + } + + last_provider_failure = Some(format!( + "connection error while trying {}: {}", + sanitize_sensitive_info(attempt_target.request_url.as_str()), + sanitize_sensitive_info(&err.to_string()) + )); + + // Connection errors (Timeout/Connect) trigger failover if provider exists + let provider_failover_exhausted = retry_enabled + && (err.is_timeout() || err.is_connect()) + && provider.is_some_and(|current_provider| { + provider_cycle_exhausted(current_provider.as_ref(), provider_url_index, start_provider_index) + }); + if retry_enabled && (err.is_timeout() || err.is_connect()) { + if let Some(current_provider) = provider { + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + "connection error", + ) { + continue 'provider_loop; + } + } + } + + // If not a provider or rotation failed, try standard retry + if (err.is_timeout() || err.is_connect()) && attempt < max_attempts - 1 { + let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); + tokio::time::sleep(Duration::from_millis(delay)).await; + continue 'attempt_loop; + } + + if provider_failover_exhausted { + if let Some(current_provider) = provider { + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + last_provider_failure.as_deref().unwrap_or("request error"), + ); + } + } + + let error_message = format!( + "Request error: {}", + sanitize_sensitive_info(&err.to_string()) + ); + return Err(if err.is_timeout() { + Error::new(ErrorKind::TimedOut, error_message) + } else { + string_to_io_error(error_message) + }); + } + } + } + } + } + } + + // 2. If per-URL retries are exhausted, try next provider URL as a last resort + if retry_enabled { + if let Some(current_provider) = provider { + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + "retries exhausted for current URL", + ) { + continue 'provider_loop; + } + + if max_provider_attempts > 0 { + let last_failure = + last_provider_failure.as_deref().unwrap_or("all attempts and providers exhausted"); + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + last_failure, + ); + } + } + } + + break; + } + + Err(string_to_io_error("All attempts and providers exhausted")) +} + +#[allow(clippy::implicit_hasher)] +pub async fn send_input_with_retry_and_provider_policy_with_options_result( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + url: &Url, + options: RequestFetchOptions, +) -> Result { + let (request_headers, default_user_agent) = prepare_input_request_headers(app_config, input, headers); + send_with_retry_and_provider_policy_with_options_result( + app_config, + url, + input.get_provider(), + false, + true, + options, + |resolved_url| { + get_client_request( + client, + input.method, + Some(&request_headers), + resolved_url, + None, + None, + default_user_agent.as_deref(), + ) + }, + ) + .await +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines, clippy::implicit_hasher)] +pub async fn send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + app_config: &Arc, + client: &reqwest::Client, + input: &InputSource, + headers: Option<&HeaderMap>, + url: &Url, + max_redirects: usize, + options: RequestFetchOptions, +) -> Result { + let config = app_config.config.load(); + let (configured_max_attempts, backoff_ms, backoff_multiplier, failover_patterns) = + config.reverse_proxy.as_ref().map_or_else( + || { + let (a, b, c) = ResourceRetryConfig::get_default_retry_values(); + (a, b, c, ResourceRetryConfig::default().failover_redirect_patterns) + }, + |rp| { + let (a, b, c) = rp.resource_retry.get_retry_values(); + (a, b, c, rp.resource_retry.failover_redirect_patterns.clone()) + }, + ); + drop(config); + let provider_failover_only = options.uses_provider_failover_only(); + let max_attempts = if provider_failover_only { 1 } else { configured_max_attempts }; + + let (base_headers, default_user_agent) = prepare_input_request_headers(app_config, input, headers); + let provider = input.get_provider(); + let max_provider_attempts = provider.as_ref().map_or(0, |p| p.urls.len()); + let start_provider_index = provider_start_index(provider); + let mut provider_url_index = start_provider_index; + let mut last_provider_failure: Option = None; + let idle_timeout = options.attempt_idle_timeout_or_default(); + let idle = sleep(idle_timeout); + tokio::pin!(idle); + + 'provider_loop: loop { + 'attempt_loop: for attempt in 0..max_attempts { + let mut current_url = url.clone(); + let mut current_headers = base_headers.clone(); + let mut remaining_redirects = max_redirects; + let mut attempted_dns_ips = HashSet::new(); + + 'redirect_loop: loop { + let attempt_target = + resolve_attempt_target_at_provider_index(¤t_url, provider, provider_url_index); + if log_enabled!(Level::Debug) { + if let Some(current_provider) = provider { + let attempt_target_log = format_request_target_for_logging(&attempt_target); + debug!( + "Provider '{}' attempting URL index {} of {}: {}", + current_provider.name, + provider_url_index, + max_provider_attempts, + sanitize_sensitive_info(attempt_target_log.as_str()) + ); + } + } + idle.as_mut().reset(tokio::time::Instant::now() + idle_timeout); + + let request_builder = get_client_request( + client, + input.method, + Some(¤t_headers), + &attempt_target.request_url, + None, + None, + default_user_agent.as_deref(), + ); + let (base_client, request) = + prepare_physical_request_attempt(request_builder, &attempt_target, options)?; + + tokio::select! { + () = &mut idle => { + warn!("Request idle for too long: {}", sanitize_sensitive_info(url.as_str())); + last_provider_failure = Some(format!( + "idle timeout while trying {}", + sanitize_sensitive_info(attempt_target.request_url.as_str()) + )); + if let Some(current_provider) = provider { + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + "idle timeout", + ) { + continue 'provider_loop; + } + if max_provider_attempts > 0 { + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + last_provider_failure.as_deref().unwrap_or("idle timeout"), + ); + } + } + + if attempt < max_attempts - 1 { + let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); + warn!("Idle timeout, retrying same URL in {}ms (attempt {})", delay, attempt + 1); + tokio::time::sleep(Duration::from_millis(delay)).await; + continue 'attempt_loop; + } + + return Err(Error::new( + ErrorKind::TimedOut, + format!( + "Request timed out and no retries left: {}", + sanitize_sensitive_info(url.as_str()) + ), + )); + } + + result = execute_attempt_request(app_config, base_client, request, &attempt_target) => { + match result { + Ok(response) => { + if response.status().is_redirection() { + if remaining_redirects == 0 { + return Err(string_to_io_error(format!( + "Too many redirects while requesting {}", + sanitize_sensitive_info(url.as_str()) + ))); + } + + let response_base_url = response.url().clone(); + let Some(location) = response.headers().get(reqwest::header::LOCATION) else { + return Err(string_to_io_error(format!( + "Redirect response missing location header for {}", + sanitize_sensitive_info(current_url.as_str()) + ))); + }; + let Ok(location_str) = location.to_str() else { + return Err(string_to_io_error(format!( + "Redirect response contains invalid location header for {}", + sanitize_sensitive_info(current_url.as_str()) + ))); + }; + let next_url = response_base_url + .join(location_str) + .or_else(|_| Url::parse(location_str)) + .map_err(|_| { + string_to_io_error(format!( + "Redirect response contains invalid location URL for {}", + sanitize_sensitive_info(current_url.as_str()) + )) + })?; + + if !same_origin(&response_base_url, &next_url) { + strip_sensitive_headers_for_cross_origin_redirect(&mut current_headers); + } + current_url = next_url; + remaining_redirects = remaining_redirects.saturating_sub(1); + continue 'redirect_loop; + } + + let status = response.status(); + let is_failover = is_failover_redirect(response.url(), &failover_patterns); + if !is_failover && status.is_success() { + if let Some(current_provider) = provider { + current_provider.set_current_index(provider_url_index); + } + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); + } + + last_provider_failure = Some(format!( + "status {} while trying {}", + format_http_status(status), + sanitize_sensitive_info(attempt_target.request_url.as_str()) + )); + + let provider_failover_exhausted = (is_failover || should_trigger_failover(status)) + && provider.is_some_and(|current_provider| { + provider_cycle_exhausted(current_provider.as_ref(), provider_url_index, start_provider_index) + }); + if is_failover || should_trigger_failover(status) { + if let Some(current_provider) = provider { + let reason = format!("status {}", format_http_status(status)); + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + reason.as_str(), + ) { + continue 'provider_loop; + } + } + } + + let is_retryable = status.is_server_error() + || matches!(status, StatusCode::TOO_MANY_REQUESTS | StatusCode::REQUEST_TIMEOUT); + if attempt < max_attempts - 1 && is_retryable { + perform_backoff(attempt, backoff_ms, backoff_multiplier, &response).await; + continue 'attempt_loop; + } + + if provider_failover_exhausted { + if let Some(current_provider) = provider { + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + last_provider_failure.as_deref().unwrap_or("request failed"), + ); + } + } + + if provider_failover_only + || (options.return_http_errors && (status.is_client_error() || status.is_server_error())) + { + return Ok(ProviderFailoverResponse { + response, + provider_url_index: provider.map(|_| provider_url_index), + }); + } + + return Err(string_to_io_error(format!( + "Request failed ({}): {}", + format_http_status(status), + sanitize_sensitive_info(url.as_str()) + ))); + } + Err(err) => { + if (err.is_timeout() || err.is_connect()) + && should_try_next_ip_on_connect_error(provider, &attempt_target, &mut attempted_dns_ips) + { + continue 'redirect_loop; + } + + last_provider_failure = Some(format!( + "connection error while trying {}: {}", + sanitize_sensitive_info(attempt_target.request_url.as_str()), + sanitize_sensitive_info(err.to_string().as_str()) + )); + + let provider_failover_exhausted = (err.is_timeout() || err.is_connect()) + && provider.is_some_and(|current_provider| { + provider_cycle_exhausted(current_provider.as_ref(), provider_url_index, start_provider_index) + }); + if err.is_timeout() || err.is_connect() { + if let Some(current_provider) = provider { + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + "connection error", + ) { + continue 'provider_loop; + } + } + } + + if (err.is_timeout() || err.is_connect()) && attempt < max_attempts - 1 { + let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); + tokio::time::sleep(Duration::from_millis(delay)).await; + continue 'attempt_loop; + } + + if provider_failover_exhausted { + if let Some(current_provider) = provider { + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + last_provider_failure.as_deref().unwrap_or("request error"), + ); + } + } + + let error_message = format!( + "Request error: {}", + sanitize_sensitive_info(err.to_string().as_str()) + ); + return Err(if err.is_timeout() { + Error::new(ErrorKind::TimedOut, error_message) + } else if err.is_connect() { + Error::new(ErrorKind::ConnectionRefused, error_message) + } else { + string_to_io_error(error_message) + }); + } + } + } + } + } + } + + if let Some(current_provider) = provider { + if rotate_to_next_provider_url( + current_provider.as_ref(), + &mut provider_url_index, + start_provider_index, + "retries exhausted for current URL", + ) { + continue 'provider_loop; + } + + if max_provider_attempts > 0 { + let last_failure = last_provider_failure.as_deref().unwrap_or("all attempts and providers exhausted"); + log_provider_cycle_exhausted( + current_provider.as_ref(), + start_provider_index, + provider_url_index, + last_failure, + ); + } + } + + break; + } + + Err(string_to_io_error("All attempts and providers exhausted")) +} + +/// Helper to handle sleep duration for retries, respecting Retry-After headers +pub(super) async fn perform_backoff(attempt: u32, ms: u64, mult: f64, response: &reqwest::Response) { + let wait_dur = response + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|h| h.to_str().ok()) + .and_then(|s| s.parse::().ok()) + .map_or_else(|| Duration::from_millis(calculate_retry_backoff(ms, mult, attempt)), Duration::from_secs); + + tokio::time::sleep(wait_dur).await; +} + +pub(super) fn text_body_retry_values( + app_config: &Arc, + body_options: TextContentBodyOptions, +) -> (u32, u64, f64) { + if body_options.retry_owner != TextContentRetryOwner::DecodedBodyConsumer { + return (1, 0, 1.0); + } + let config = app_config.config.load(); + let values = config + .reverse_proxy + .as_ref() + .map_or_else(ResourceRetryConfig::get_default_retry_values, |rp| rp.resource_retry.get_retry_values()); + drop(config); + values +} + +pub(super) fn should_retry_text_body_error(error: &Error) -> bool { + matches!(error.kind(), ErrorKind::TimedOut | ErrorKind::ConnectionRefused) + || content_decoding_error_from_io(error).is_some() + || is_http_body_transport_error(error) + || error + .get_ref() + .and_then(|source| source.downcast_ref::()) + .is_some_and(|error| matches!(error, ContentCodingError::PrefixRead(_))) +} + +pub(super) async fn sleep_before_text_body_retry(attempt: u32, backoff_ms: u64, backoff_multiplier: f64, err: &Error) { + let delay = calculate_retry_backoff(backoff_ms, backoff_multiplier, attempt); + warn!( + "Text response body failed retryably, retrying in {}ms (attempt {}): {}", + delay, + attempt + 1, + text_response_error_log_label(err) + ); + tokio::time::sleep(Duration::from_millis(delay)).await; +} + +pub(super) fn is_retryable_text_response_status(status: StatusCode) -> bool { + status.is_server_error() + || matches!( + status, + StatusCode::PROXY_AUTHENTICATION_REQUIRED + | StatusCode::REQUEST_TIMEOUT + | StatusCode::TOO_EARLY + | StatusCode::TOO_MANY_REQUESTS + ) +} + +/// Checks if a status code or error indicates a need for failover +/// +/// Returns true for server-side errors that might be resolved by trying another URL. +/// Returns false for client-side errors (401, 403, etc.) where the problem is with +/// credentials or permissions, not the server availability. +pub fn should_trigger_failover(status: StatusCode) -> bool { + matches!( + status, + StatusCode::NOT_FOUND + | StatusCode::GONE + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::BAD_GATEWAY + | StatusCode::GATEWAY_TIMEOUT + | StatusCode::INTERNAL_SERVER_ERROR + | StatusCode::TOO_MANY_REQUESTS + | StatusCode::REQUEST_TIMEOUT + | StatusCode::PROXY_AUTHENTICATION_REQUIRED + ) + // Explicitly NOT triggering failover for: + // - 401 Unauthorized (wrong credentials) + // - 403 Forbidden (permission issue) + // - 402 Payment Required (subscription issue) + // - 451 Unavailable For Legal Reasons (geo-blocking) + // + // Note: DO triggering failover for: + // - 429 Too Many Requests + // - 408 Request Timeout + // - 407 Proxy Authentication Required +} diff --git a/backend/core/src/utils/network/request/tests.rs b/backend/core/src/utils/network/request/tests.rs new file mode 100644 index 000000000..e92f144e0 --- /dev/null +++ b/backend/core/src/utils/network/request/tests.rs @@ -0,0 +1,32 @@ +mod tmdb_profile; + +use super::{ + classify_resource_destination, download_text_content, download_text_content_with_headers_and_options, + get_input_epg_content_as_file, get_remote_content_as_stream, is_safe_cross_origin_redirect_header, + next_provider_url_index, preview_request_diagnostics_for_logging, preview_request_target_for_logging, + resolve_attempt_target, resolve_resource_socket_addrs, same_origin, + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, + send_input_with_retry_and_provider_policy_with_options_result, send_with_retry_and_provider, + send_with_retry_and_provider_policy, should_retry_text_body_error, should_try_next_ip_on_connect_error, + strip_sensitive_headers_for_cross_origin_redirect, text_response_error_log_label, InputEpgFileRequest, + PublicIpResolver, RequestFetchOptions, ResourceDestination, ResourceDestinationResolver, TextContentBodyOptions, + TextContentFetchOptions, STREAM_IDLE_TIMEOUT, +}; + +mod behavior; +mod configuration; +mod lifecycle; +mod persistence; +mod protocol; +mod retry; +mod support; + +use self::{ + lifecycle::start_hanging_http_server, + support::{ + atomic_download_temp_files, identity_fetch_options, make_epg_test_client, make_provider_with_dns, + make_test_app_config, request_header_value, response_with_body, start_plain_http_server_with_body, + start_plain_http_server_with_response, start_recording_http_byte_server, start_recording_http_server, + test_input_source, test_retry_config, + }, +}; diff --git a/backend/core/src/utils/network/request/tests/behavior.rs b/backend/core/src/utils/network/request/tests/behavior.rs new file mode 100644 index 000000000..2a529adf3 --- /dev/null +++ b/backend/core/src/utils/network/request/tests/behavior.rs @@ -0,0 +1,934 @@ +use super::{ + atomic_download_temp_files, download_text_content_with_headers_and_options, get_input_epg_content_as_file, + identity_fetch_options, make_epg_test_client, make_provider_with_dns, make_test_app_config, + next_provider_url_index, preview_request_diagnostics_for_logging, preview_request_target_for_logging, + request_header_value, resolve_attempt_target, response_with_body, same_origin, + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, send_with_retry_and_provider, + should_try_next_ip_on_connect_error, start_plain_http_server_with_body, start_plain_http_server_with_response, + start_recording_http_server, test_input_source, test_retry_config, text_response_error_log_label, + InputEpgFileRequest, PublicIpResolver, RequestFetchOptions, ResourceDestination, TextContentBodyOptions, + TextContentFetchOptions, +}; +use crate::{ + model::{Config, ConfigInput, ConfigProvider, ResourceRetryConfig, ReverseProxyConfig}, + utils::content_coding::{ContentCoding, ContentCodingError, ContentDecodingIoError}, +}; +use shared::{ + model::{ConfigProviderDto, OnConnectErrorPolicy, ProviderUrlSelectionPolicy}, + utils::{get_base_url_from_str, replace_url_extension, sanitize_sensitive_info}, +}; +use std::{ + collections::HashSet, + io::{Error, ErrorKind}, + net::{IpAddr, SocketAddr}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, +}; +use url::Url; + +#[test] +fn ipv6_addresses_embedding_private_ipv4_are_not_public() { + for address in ["::10.0.0.1", "64:ff9b::a00:1", "2002:0a00:0001::", "::ffff:10.0.0.1"] { + assert!(address.parse::().is_ok_and(|ip| !super::super::is_public_ip(ip)), "{address}"); + } +} + +#[test] +fn proxy_hosts_are_exempt_from_the_connect_time_local_address_guard() { + use crate::{model::Config, utils::request::proxy_hosts}; + + // `localhost` is what a proxy on this host is usually configured as, and reqwest asks the + // resolver for the proxy host as well: guarding it would break every fetch through such a proxy. + let app_config = make_test_app_config(Config { + proxy: Some(crate::model::ProxyConfig { + url: "http://localhost:8118".to_string(), + username: None, + password: None, + }), + ..Config::default() + }); + + assert_eq!(proxy_hosts(&app_config), vec![Arc::from("localhost")]); +} + +#[test] +fn resolved_address_sets_are_aggregated_for_exposure() { + use super::super::{ + verdict_for_addresses, RESOURCE_DESTINATION_PRIVATE_TTL, RESOURCE_DESTINATION_PUBLIC_TTL, + RESOURCE_DESTINATION_UNANSWERED_TTL, + }; + + let public: IpAddr = "8.8.8.8".parse().expect("valid public address"); + let private: IpAddr = "10.0.0.1".parse().expect("valid private address"); + let blocked: IpAddr = "127.0.0.1".parse().expect("valid loopback address"); + + assert_eq!(verdict_for_addresses(&[public]), (ResourceDestination::Public, RESOURCE_DESTINATION_PUBLIC_TTL)); + assert_eq!( + verdict_for_addresses(&[public, public]), + (ResourceDestination::Public, RESOURCE_DESTINATION_PUBLIC_TTL) + ); + assert_eq!(verdict_for_addresses(&[private]), (ResourceDestination::Private, RESOURCE_DESTINATION_PRIVATE_TTL)); + assert_eq!( + verdict_for_addresses(&[public, private]), + (ResourceDestination::Private, RESOURCE_DESTINATION_PRIVATE_TTL) + ); + assert_eq!( + verdict_for_addresses(&[public, blocked]), + (ResourceDestination::Blocked, RESOURCE_DESTINATION_PRIVATE_TTL) + ); + assert_eq!( + verdict_for_addresses(&[private, blocked]), + (ResourceDestination::Blocked, RESOURCE_DESTINATION_PRIVATE_TTL) + ); + // An unanswered name stays non-public, but only briefly: a resolver hiccup must not pin the + // destination to the proxy for minutes. + assert_eq!(verdict_for_addresses(&[]), (ResourceDestination::Private, RESOURCE_DESTINATION_UNANSWERED_TTL)); + assert!(RESOURCE_DESTINATION_UNANSWERED_TTL < RESOURCE_DESTINATION_PRIVATE_TTL); +} + +#[tokio::test] +async fn atomic_download_temp_files_are_unique_and_use_target_directory() { + let dir = tempfile::tempdir().expect("temp dir"); + let target = dir.path().join("cache.ics"); + let (first_temp, first_output) = super::super::create_atomic_download_file(&target).expect("first temp file"); + let (second_temp, second_output) = super::super::create_atomic_download_file(&target).expect("second temp file"); + + assert_ne!(first_temp.path(), second_temp.path()); + assert_eq!(first_temp.path().parent(), Some(dir.path())); + assert_eq!(second_temp.path().parent(), Some(dir.path())); + + drop(first_output); + drop(second_output); + drop(first_temp); + drop(second_temp); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); +} + +#[test] +fn test_url_mask() { + // Replace with "***" + let query = "https://bubblegum.tv/live/username/password/2344"; + let masked = sanitize_sensitive_info(query); + println!("{masked}"); +} + +#[test] +fn test_replace_ext() { + let tests = [ + ("http://hello.world.com", "http://hello.world.com"), + ("http://hello.world.com/123", "http://hello.world.com/123.mp4"), + ("http://hello.world.com/123.ts?hello=world", "http://hello.world.com/123.mp4?hello=world"), + ("http://hello.world.com/123?hello=world", "http://hello.world.com/123.mp4?hello=world"), + ("http://hello.world.com/123#hello=world", "http://hello.world.com/123.mp4#hello=world"), + ]; + + for (test, expect) in &tests { + assert_eq!(replace_url_extension(test, ".mp4"), *expect); + } +} + +#[test] +fn tes_base_url() { + let url = "http://my.provider.com:8080/xmltv?username=hello"; + let expected = "http://my.provider.com:8080"; + assert_eq!(get_base_url_from_str(url).unwrap(), expected); +} + +#[test] +fn test_same_origin_checks_scheme_host_and_port() { + let a = Url::parse("https://example.com/path").expect("url parse should work"); + let b = Url::parse("https://example.com/other").expect("url parse should work"); + let c = Url::parse("http://example.com/other").expect("url parse should work"); + let d = Url::parse("https://example.com:8443/other").expect("url parse should work"); + + assert!(same_origin(&a, &b)); + assert!(!same_origin(&a, &c)); + assert!(!same_origin(&a, &d)); +} + +#[tokio::test] +async fn local_epg_file_respects_max_download_bytes() { + let dir = tempfile::tempdir().expect("temp dir"); + let source = dir.path().join("large.ics"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&source, b"0123456789").await.expect("write source"); + tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + let err = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: source.to_string_lossy().as_ref(), + persist_path: &persist, + max_bytes: Some(4), + }, + ) + .await + .expect_err("size limit should fail"); + + assert!(err.to_string().contains("exceeds configured limit")); + assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); +} + +#[tokio::test] +async fn file_url_epg_source_respects_streamed_size_limit() { + let dir = tempfile::tempdir().expect("temp dir"); + let source = dir.path().join("large.ics"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&source, b"0123456789").await.expect("write source"); + tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); + let source_url = Url::from_file_path(&source).expect("file url"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + let err = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: source_url.as_str(), + persist_path: &persist, + max_bytes: Some(4), + }, + ) + .await + .expect_err("size limit should fail"); + + assert!(err.to_string().contains("exceeds configured limit")); + assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); +} + +#[tokio::test] +async fn local_epg_replace_error_cleans_temp_file() { + let dir = tempfile::tempdir().expect("temp dir"); + let source = dir.path().join("source.ics"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n").await.expect("write source"); + tokio::fs::create_dir(&persist).await.expect("create conflicting destination"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: source.to_string_lossy().as_ref(), + persist_path: &persist, + max_bytes: Some(1024), + }, + ) + .await + .expect_err("replacing a directory should fail"); + + assert!(persist.is_dir()); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); +} + +#[tokio::test] +async fn local_epg_read_error_preserves_existing_cache_and_cleans_temp_file() { + let dir = tempfile::tempdir().expect("temp dir"); + let source = dir.path().join("source-directory"); + let persist = dir.path().join("cache.ics"); + tokio::fs::create_dir(&source).await.expect("create source directory"); + tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: source.to_string_lossy().as_ref(), + persist_path: &persist, + max_bytes: Some(1024), + }, + ) + .await + .expect_err("reading a directory as an EPG file should fail"); + + assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); +} + +#[tokio::test] +async fn unsupported_epg_url_scheme_is_rejected_without_network_access() { + let dir = tempfile::tempdir().expect("temp dir"); + let persist = dir.path().join("cache.ics"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + let err = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: "ftp://example.com/calendar.ics", + persist_path: &persist, + max_bytes: Some(1024), + }, + ) + .await + .expect_err("unsupported scheme should fail"); + + assert!(err.to_string().contains("Unsupported EPG URL scheme 'ftp'")); + assert!(!persist.exists()); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); +} + +#[cfg(windows)] +#[tokio::test] +async fn absolute_windows_epg_path_is_dispatched_as_local_file() { + let dir = tempfile::tempdir().expect("temp dir"); + let source = dir.path().join("source.ics"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n").await.expect("write source"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + let source_path = source.to_str().expect("Windows temp path should be UTF-8"); + assert!(source_path.contains(':')); + + get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: source_path, + persist_path: &persist, + max_bytes: Some(1024), + }, + ) + .await + .expect("absolute Windows path should be copied as a local file"); + + assert_eq!(tokio::fs::read(&persist).await.expect("read cache"), b"BEGIN:VCALENDAR\nEND:VCALENDAR\n"); +} + +#[tokio::test] +async fn remote_epg_size_limit_preserves_existing_cache_and_cleans_temp_file() { + let (addr, _accepted, server_handle) = match start_plain_http_server_with_body(b"0123456789").await { + Ok(server) => server, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!("skipping remote_epg_size_limit_preserves_existing_cache_and_cleans_temp_file: {err}"); + return; + } + Err(err) => panic!("failed to start test server: {err}"), + }; + let dir = tempfile::tempdir().expect("temp dir"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); + let url = format!("http://{addr}/calendar.ics"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + let err = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: &url, + persist_path: &persist, + max_bytes: Some(4), + }, + ) + .await + .expect_err("remote size limit should fail"); + + assert!(err.to_string().contains("exceeds configured limit")); + assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + server_handle.abort(); +} + +#[tokio::test] +async fn remote_epg_stream_error_preserves_existing_cache_and_cleans_temp_file() { + let response = "HTTP/1.1 200 OK\r\nContent-Length: 64\r\nConnection: close\r\n\r\ntruncated".to_string(); + let (addr, _accepted, server_handle) = match start_plain_http_server_with_response(response).await { + Ok(server) => server, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!("skipping remote_epg_stream_error_preserves_existing_cache_and_cleans_temp_file: {err}"); + return; + } + Err(err) => panic!("failed to start test server: {err}"), + }; + let dir = tempfile::tempdir().expect("temp dir"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); + let url = format!("http://{addr}/calendar.ics"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: &url, + persist_path: &persist, + max_bytes: Some(1024), + }, + ) + .await + .expect_err("truncated response body should fail"); + + assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + server_handle.abort(); +} + +#[tokio::test] +async fn parallel_epg_downloads_to_same_target_are_serialized_and_replace_atomically() { + let (addr, accepted, max_active, server_handle) = + match start_delayed_http_server_with_body(b"BEGIN:VCALENDAR\nEND:VCALENDAR\n").await { + Ok(server) => server, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!( + "skipping parallel_epg_downloads_to_same_target_are_serialized_and_replace_atomically: {err}" + ); + return; + } + Err(err) => panic!("failed to start test server: {err}"), + }; + let dir = tempfile::tempdir().expect("temp dir"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); + let url = format!("http://{addr}/calendar.ics"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + let storage_dir = dir.path().to_string_lossy(); + + let first = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: storage_dir.as_ref(), + url: &url, + persist_path: &persist, + max_bytes: Some(1024), + }, + ); + let second = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: storage_dir.as_ref(), + url: &url, + persist_path: &persist, + max_bytes: Some(1024), + }, + ); + let (first_result, second_result) = tokio::join!(first, second); + + assert_eq!(first_result.expect("first download"), persist); + assert_eq!(second_result.expect("second download"), persist); + assert_eq!(accepted.load(Ordering::SeqCst), 2); + assert_eq!(max_active.load(Ordering::SeqCst), 1); + assert_eq!(tokio::fs::read(&persist).await.expect("read refreshed cache"), b"BEGIN:VCALENDAR\nEND:VCALENDAR\n"); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + server_handle.abort(); +} + +#[test] +fn test_preview_request_diagnostics_for_logging_includes_effective_target_and_host_details() { + let provider = make_provider_with_dns(true, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1"]); + let url = Url::parse("http://example.com:8080/stream").expect("url parse should work"); + + let diagnostics = preview_request_diagnostics_for_logging(&url, Some(&provider)); + + assert_eq!( + diagnostics, + "request_url=http://***/stream, effective_url=http://***/stream, host_header=example.com:8080, connect_ip=0.***" + ); +} + +#[test] +fn test_preview_request_diagnostics_for_logging_sanitizes_each_stream_url() -> Result<(), url::ParseError> { + let provider = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1"]); + let url = Url::parse("http://example.com/live/abcd/efgh/1092671.ts")?; + + let diagnostics = preview_request_diagnostics_for_logging(&url, Some(&provider)); + + assert!(!diagnostics.contains("example.com")); + assert!(!diagnostics.contains("abcd")); + assert!(!diagnostics.contains("efgh")); + assert_eq!( + diagnostics, + "request_url=http://***/live/***/1092671.ts, effective_url=http://***/live/***/1092671.ts, host_header=0.***, connect_ip=0.***" + ); + Ok(()) +} + +#[test] +fn test_http_attempt_uses_bracketed_ipv6_target_for_logging_and_request_url() { + let provider = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["2a06:98c1:3121::3"]); + let url = Url::parse("http://example.com/live/stream.ts").expect("url parse should work"); + + let preview = preview_request_target_for_logging(&url, Some(&provider)); + let target = resolve_attempt_target(&url, Some(&provider)); + + assert_eq!(preview, "http://[2a06:98c1:3121::3]/live/stream.ts"); + assert_eq!(target.effective_url.as_str(), "http://[2a06:98c1:3121::3]/live/stream.ts"); + assert_eq!(target.host_header.as_deref(), Some("[2a06:98c1:3121::3]")); +} + +#[test] +fn test_https_attempt_keeps_hostname_and_sets_sni() { + let provider = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1"]); + let url = Url::parse("https://example.com/live").expect("url parse should work"); + + let target = resolve_attempt_target(&url, Some(&provider)); + assert_eq!(target.effective_url.host_str(), Some("example.com")); + assert_eq!(target.sni_host.as_deref(), Some("example.com")); + assert_eq!(target.connect_ip.map(|ip| ip.to_string()), Some("192.168.0.1".to_string())); + assert_eq!(target.host_header.as_deref(), Some("192.168.0.1")); +} + +#[test] +fn test_try_next_ip_policy_uses_next_ip_until_exhausted() { + let provider = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "192.168.0.2"]); + let url = Url::parse("http://example.com/live").expect("url parse should work"); + let mut tried = HashSet::new(); + + let first = resolve_attempt_target(&url, Some(&provider)); + let second = resolve_attempt_target(&url, Some(&provider)); + + assert!(should_try_next_ip_on_connect_error(Some(&provider), &first, &mut tried)); + assert!(!should_try_next_ip_on_connect_error(Some(&provider), &second, &mut tried)); +} + +#[test] +fn test_preview_request_target_for_logging_uses_preferred_provider_index() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "provider-a".into(), + urls: vec!["http://provider-a.example".into(), "http://provider-b.example".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), + dns: None, + })); + provider.set_current_index(1); + + let url = Url::parse("provider://provider-a/live").expect("provider url should parse"); + let preview = preview_request_target_for_logging(&url, Some(&provider)); + + assert_eq!(preview, "http://provider-b.example/live"); +} + +pub(in crate::utils::network::request::tests) async fn start_delayed_http_server_with_body( + body: &'static [u8], +) -> std::io::Result<(SocketAddr, Arc, Arc, tokio::task::JoinHandle<()>)> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let accepted = Arc::new(AtomicUsize::new(0)); + let active = Arc::new(AtomicUsize::new(0)); + let max_active = Arc::new(AtomicUsize::new(0)); + let accepted_clone = Arc::clone(&accepted); + let active_clone = Arc::clone(&active); + let max_active_clone = Arc::clone(&max_active); + let content_length = body.len(); + + let handle = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + continue; + }; + accepted_clone.fetch_add(1, Ordering::SeqCst); + let active = Arc::clone(&active_clone); + let max_active = Arc::clone(&max_active_clone); + tokio::spawn(async move { + let active_count = active.fetch_add(1, Ordering::SeqCst) + 1; + max_active.fetch_max(active_count, Ordering::SeqCst); + let mut buf = vec![0_u8; 2048]; + let _ = socket.read(&mut buf).await; + tokio::time::sleep(Duration::from_millis(100)).await; + let response_head = + format!("HTTP/1.1 200 OK\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n"); + let _ = socket.write_all(response_head.as_bytes()).await; + let _ = socket.write_all(body).await; + let _ = socket.shutdown().await; + active.fetch_sub(1, Ordering::SeqCst); + }); + } + }); + + Ok((addr, accepted, max_active, handle)) +} + +pub(in crate::utils::network::request::tests) async fn start_plain_http_server( +) -> std::io::Result<(SocketAddr, Arc, tokio::task::JoinHandle<()>)> { + start_plain_http_server_with_body(b"ok").await +} + +#[tokio::test] +async fn public_resolver_rejects_loopback_at_connection_time() { + let (address, accepted, server) = match start_plain_http_server().await { + Ok(server) => server, + Err(err) if err.kind() == ErrorKind::PermissionDenied => return, + Err(err) => panic!("failed to start test server: {err}"), + }; + let client = reqwest::Client::builder() + .no_proxy() + .dns_resolver(PublicIpResolver) + .build() + .expect("public-only client should build"); + + let result = client.get(format!("http://localhost:{}/", address.port())).send().await; + + assert!(result.is_err()); + assert_eq!(accepted.load(Ordering::SeqCst), 0); + server.abort(); +} + +#[test] +fn content_type_from_ext_maps_fragmented_mp4_extensions_case_insensitively() { + for (ext, expected) in [ + ("mp4", "video/mp4"), + ("FMP4", "video/mp4"), + ("m4s", "video/mp4"), + ("m4v", "video/mp4"), + ("cmfv", "video/mp4"), + ("m4a", "audio/mp4"), + ("CMFA", "audio/mp4"), + ("ts", "video/mp2t"), + ("mkv", "video/x-matroska"), + ("bin", "application/octet-stream"), + ("", "application/octet-stream"), + ] { + assert_eq!(super::super::content_type_from_ext(ext), expected, "{ext}"); + } +} + +#[tokio::test] +async fn http_error_response_option_recovers_transient_statuses_in_both_request_paths( +) -> Result<(), Box> { + let client = reqwest::Client::builder().no_proxy().build()?; + for manual_redirects in [false, true] { + for status in ["503 Service Unavailable", "429 Too Many Requests", "408 Request Timeout"] { + let (addr, requests, server) = start_recording_http_server(vec![ + response_with_body(status, ""), + response_with_body("200 OK", "media"), + ]) + .await?; + let url = Url::parse(&format!("http://{addr}/init.hls.fmp4"))?; + let app_config = test_retry_config(2); + let options = RequestFetchOptions::default().with_http_error_responses(true); + let response = if manual_redirects { + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + &app_config, + &client, + &test_input_source(url.to_string(), None), + None, + &url, + 10, + options, + ) + .await? + .response + } else { + super::super::send_with_retry_and_provider_policy_with_options( + &app_config, + &url, + None, + false, + true, + options, + |resolved| client.get(resolved.clone()), + ) + .await? + }; + assert_eq!(response.status(), reqwest::StatusCode::OK); + assert_eq!(response.bytes().await?.as_ref(), b"media"); + assert_eq!(requests.lock().await.len(), 2); + server.abort(); + } + } + Ok(()) +} + +#[test] +fn text_response_error_log_labels_never_expose_origin_controlled_details() { + let unsupported = Error::other(ContentCodingError::Unsupported("signed-token-secret".to_string())); + assert_eq!(text_response_error_log_label(&unsupported), "content_coding class=unsupported"); + assert!(!text_response_error_log_label(&unsupported).contains("signed-token-secret")); + + let decoding = Error::new(ErrorKind::InvalidData, ContentDecodingIoError { coding: ContentCoding::Zstd }); + assert_eq!(text_response_error_log_label(&decoding), "content_decoding coding=zstd"); +} + +#[tokio::test] +async fn text_content_manifest_uses_one_budget_for_status_decoder_and_success() { + let corrupt_gzip = + "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: 7\r\nConnection: close\r\n\r\ncorrupt"; + let responses = vec![ + response_with_body("503 Service Unavailable", ""), + corrupt_gzip.to_string(), + response_with_body("200 OK", "#EXTM3U\nsegment.ts\n"), + ]; + let (addr, requests, handle) = start_recording_http_server(responses).await.expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + let options = TextContentFetchOptions::new( + identity_fetch_options(), + TextContentBodyOptions::hls_manifest(1024, Duration::from_secs(1)), + ); + + let (manifest, _, _) = download_text_content_with_headers_and_options( + &test_retry_config(3), + &make_epg_test_client(), + &input, + None, + false, + options, + ) + .await + .expect("third logical attempt should succeed"); + + assert_eq!(manifest, "#EXTM3U\nsegment.ts\n"); + let captured = requests.lock().await; + assert_eq!(captured.len(), 3); + assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + handle.abort(); +} + +#[tokio::test] +async fn test_provider_request_chain_starts_from_last_successful_url() { + let (addr_b, accepted_b, handle_b) = match start_plain_http_server_with_body(b"b").await { + Ok(server) => server, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!("skipping test_provider_request_chain_starts_from_last_successful_url: {err}"); + return; + } + Err(err) => panic!("failed to start test http server: {err}"), + }; + + let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; + cfg.accept_insecure_ssl_certificates = true; + cfg.reverse_proxy = Some(ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: ResourceRetryConfig { max_attempts: 1, ..ResourceRetryConfig::default() }, + disabled_header: None, + stream: None, + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }); + let app_config = make_test_app_config(cfg); + let client = reqwest::Client::builder() + .no_proxy() + .connect_timeout(Duration::from_millis(400)) + .timeout(Duration::from_secs(2)) + .build() + .expect("http client should build"); + let dead_addr = SocketAddr::from(([127, 0, 0, 1], 1)); + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "provider-a".into(), + urls: vec![ + format!("http://127.0.0.1:{}", dead_addr.port()).into(), + format!("http://127.0.0.1:{}", addr_b.port()).into(), + ], + provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), + dns: None, + })); + + let url = Url::parse("provider://provider-a/live").expect("provider url should parse"); + let first_response = send_with_retry_and_provider(&app_config, &url, Some(&provider), false, |resolved_url| { + client.get(resolved_url.clone()) + }) + .await + .expect("request should fail over to the second provider url"); + let first_body = first_response.text().await.expect("response body should be readable"); + + assert_eq!(first_body, "b"); + assert_eq!(provider.get_current_index(), 1); + + let second_response = send_with_retry_and_provider(&app_config, &url, Some(&provider), false, |resolved_url| { + client.get(resolved_url.clone()) + }) + .await + .expect("next request should start from the last successful provider url"); + let second_body = second_response.text().await.expect("response body should be readable"); + + assert_eq!(second_body, "b"); + assert_eq!(accepted_b.load(Ordering::SeqCst), 2); + + handle_b.abort(); +} + +#[tokio::test] +async fn test_provider_request_chain_restarts_from_first_url_when_policy_requests_it() { + let (addr_b, accepted_b, handle_b) = match start_plain_http_server_with_body(b"b").await { + Ok(server) => server, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!("skipping test_provider_request_chain_restarts_from_first_url_when_policy_requests_it: {err}"); + return; + } + Err(err) => panic!("failed to start test http server: {err}"), + }; + + let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; + cfg.accept_insecure_ssl_certificates = true; + cfg.reverse_proxy = Some(ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: ResourceRetryConfig { max_attempts: 1, ..ResourceRetryConfig::default() }, + disabled_header: None, + stream: None, + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }); + let app_config = make_test_app_config(cfg); + let client = reqwest::Client::builder() + .no_proxy() + .connect_timeout(Duration::from_millis(400)) + .timeout(Duration::from_secs(2)) + .build() + .expect("http client should build"); + let dead_addr = SocketAddr::from(([127, 0, 0, 1], 1)); + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "provider-a".into(), + urls: vec![ + format!("http://127.0.0.1:{}", dead_addr.port()).into(), + format!("http://127.0.0.1:{}", addr_b.port()).into(), + ], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + })); + + let url = Url::parse("provider://provider-a/live").expect("provider url should parse"); + let first_response = send_with_retry_and_provider(&app_config, &url, Some(&provider), false, |resolved_url| { + client.get(resolved_url.clone()) + }) + .await + .expect("request should fail over to the second provider url"); + let first_body = first_response.text().await.expect("response body should be readable"); + + assert_eq!(first_body, "b"); + assert_eq!(provider.get_current_index(), 1); + + let second_response = send_with_retry_and_provider(&app_config, &url, Some(&provider), false, |resolved_url| { + client.get(resolved_url.clone()) + }) + .await + .expect("next request should restart from the first provider url and fail over again"); + let second_body = second_response.text().await.expect("response body should be readable"); + + assert_eq!(second_body, "b"); + assert_eq!(provider.get_current_index(), 1); + assert_eq!(accepted_b.load(Ordering::SeqCst), 2); + + handle_b.abort(); +} + +#[test] +fn test_next_provider_url_index_wraps_once_then_stops() { + assert_eq!(next_provider_url_index(2, 4, 2), Some(3)); + assert_eq!(next_provider_url_index(3, 4, 2), Some(0)); + assert_eq!(next_provider_url_index(0, 4, 2), Some(1)); + assert_eq!(next_provider_url_index(1, 4, 2), None); + assert_eq!(next_provider_url_index(0, 1, 0), None); +} + +#[tokio::test] +async fn test_on_connect_error_try_next_ip_before_provider_rotation() { + let (addr, accepted, server_handle) = match start_plain_http_server().await { + Ok(server) => server, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!("skipping test_on_connect_error_try_next_ip_before_provider_rotation: {err}"); + return; + } + Err(err) => panic!("failed to start test http server: {err}"), + }; + + let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; + cfg.accept_insecure_ssl_certificates = true; + cfg.reverse_proxy = Some(ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: ResourceRetryConfig { max_attempts: 1, ..ResourceRetryConfig::default() }, + disabled_header: None, + stream: None, + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }); + let app_config = make_test_app_config(cfg); + let client = reqwest::Client::builder() + .no_proxy() + .connect_timeout(Duration::from_millis(400)) + .timeout(Duration::from_secs(2)) + .build() + .expect("http client should build"); + let url = Url::parse(format!("http://example.com:{}/ok", addr.port()).as_str()).expect("url parse should work"); + + let provider_rotate = + make_provider_with_dns(false, OnConnectErrorPolicy::RotateProviderUrl, vec!["192.168.0.1", "127.0.0.1"]); + let result_rotate = + send_with_retry_and_provider(&app_config, &url, Some(&provider_rotate), false, |resolved_url| { + client.get(resolved_url.clone()) + }) + .await; + assert!(result_rotate.is_err(), "without try_next_ip policy the request should fail"); + + let provider_try_next = + make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "127.0.0.1"]); + let result_try_next = + send_with_retry_and_provider(&app_config, &url, Some(&provider_try_next), false, |resolved_url| { + client.get(resolved_url.clone()) + }) + .await; + assert!(result_try_next.is_ok(), "try_next_ip should succeed by trying the second IP"); + assert_eq!(accepted.load(Ordering::SeqCst), 1, "server should be reached exactly once"); + + server_handle.abort(); +} diff --git a/backend/core/src/utils/network/request/tests/configuration.rs b/backend/core/src/utils/network/request/tests/configuration.rs new file mode 100644 index 000000000..f262640dd --- /dev/null +++ b/backend/core/src/utils/network/request/tests/configuration.rs @@ -0,0 +1,36 @@ +use super::{ + download_text_content_with_headers_and_options, identity_fetch_options, make_epg_test_client, request_header_value, + start_recording_http_server, test_input_source, test_retry_config, TextContentBodyOptions, TextContentFetchOptions, +}; +use std::time::Duration; +use url::Url; + +#[tokio::test] +async fn text_content_manifest_repeated_decoder_failures_stop_at_configured_budget() { + let corrupt_gzip = + "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: 7\r\nConnection: close\r\n\r\ncorrupt"; + let (addr, requests, handle) = + start_recording_http_server(vec![corrupt_gzip.to_string()]).await.expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + let options = TextContentFetchOptions::new( + identity_fetch_options(), + TextContentBodyOptions::hls_manifest(1024, Duration::from_secs(1)), + ); + + download_text_content_with_headers_and_options( + &test_retry_config(3), + &make_epg_test_client(), + &input, + None, + false, + options, + ) + .await + .expect_err("decoder failures must exhaust the configured budget"); + + let captured = requests.lock().await; + assert_eq!(captured.len(), 3); + assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + handle.abort(); +} diff --git a/backend/core/src/utils/network/request/tests/lifecycle.rs b/backend/core/src/utils/network/request/tests/lifecycle.rs new file mode 100644 index 000000000..6402b8278 --- /dev/null +++ b/backend/core/src/utils/network/request/tests/lifecycle.rs @@ -0,0 +1,91 @@ +use super::{ + send_input_with_retry_and_provider_policy_with_options_result, test_input_source, test_retry_config, + RequestFetchOptions, STREAM_IDLE_TIMEOUT, +}; +use std::{ + io::ErrorKind, + net::SocketAddr, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{io::AsyncReadExt, net::TcpListener, sync::oneshot}; +use url::Url; + +pub(in crate::utils::network::request::tests) async fn start_hanging_http_server( +) -> std::io::Result<(SocketAddr, oneshot::Receiver<()>, tokio::task::JoinHandle<()>)> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let (request_seen_tx, request_seen_rx) = oneshot::channel(); + let handle = tokio::spawn(async move { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 2048]; + let Ok(read) = socket.read(&mut chunk).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + let _ = request_seen_tx.send(()); + std::future::pending::<()>().await; + drop(socket); + }); + + Ok((addr, request_seen_rx, handle)) +} + +#[tokio::test(start_paused = true)] +async fn provider_failover_only_default_options_keep_default_idle_timeout() { + let (addr, request_seen, server) = start_hanging_http_server().await.expect("hanging origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + let app_config = test_retry_config(5); + let client = reqwest::Client::builder().no_proxy().build().expect("test client"); + let request_url = url.clone(); + let request = tokio::spawn(async move { + send_input_with_retry_and_provider_policy_with_options_result( + &app_config, + &client, + &input, + None, + &request_url, + RequestFetchOptions::default().without_resource_retries(), + ) + .await + }); + + // A ready task prevents Tokio's paused clock from auto-advancing through the default timeout while the local + // TCP handshake is still in progress. After the request arrives, this guard stops and the test advances time + // explicitly to the production deadline. + let hold_virtual_time = Arc::new(AtomicBool::new(true)); + let hold_virtual_time_for_task = Arc::clone(&hold_virtual_time); + let virtual_time_guard = tokio::spawn(async move { + while hold_virtual_time_for_task.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + }); + let request_seen = request_seen.await; + hold_virtual_time.store(false, Ordering::SeqCst); + virtual_time_guard.await.expect("virtual time guard should stop"); + request_seen.expect("origin should receive the request"); + tokio::time::advance(Duration::from_secs(STREAM_IDLE_TIMEOUT + 1)).await; + tokio::task::yield_now().await; + if !request.is_finished() { + request.abort(); + panic!("provider-only request lost the default idle-timeout guard"); + } + + let Err(error) = request.await.expect("request task should join") else { + panic!("hanging request must time out"); + }; + assert_eq!(error.kind(), ErrorKind::TimedOut); + server.abort(); +} diff --git a/backend/core/src/utils/network/request/tests/persistence.rs b/backend/core/src/utils/network/request/tests/persistence.rs new file mode 100644 index 000000000..e4b1fa9a0 --- /dev/null +++ b/backend/core/src/utils/network/request/tests/persistence.rs @@ -0,0 +1,41 @@ +use super::{ + atomic_download_temp_files, get_input_epg_content_as_file, make_epg_test_client, make_test_app_config, + InputEpgFileRequest, +}; +use crate::model::{Config, ConfigInput}; +use url::Url; + +#[tokio::test] +async fn file_url_epg_source_is_copied_to_persist_path() { + let dir = tempfile::tempdir().expect("temp dir"); + let source = dir.path().join("source.ics"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n").await.expect("write source"); + tokio::fs::write(&persist, b"old cache").await.expect("write old cache"); + let source_url = Url::from_file_path(&source).expect("file url"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + let result = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: source_url.as_str(), + persist_path: &persist, + max_bytes: Some(1024), + }, + ) + .await + .expect("download"); + + assert_eq!(result, persist); + assert_eq!( + tokio::fs::read_to_string(&persist).await.expect("persisted content"), + "BEGIN:VCALENDAR\nEND:VCALENDAR\n" + ); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); +} diff --git a/backend/core/src/utils/network/request/tests/protocol.rs b/backend/core/src/utils/network/request/tests/protocol.rs new file mode 100644 index 000000000..bd5e173f3 --- /dev/null +++ b/backend/core/src/utils/network/request/tests/protocol.rs @@ -0,0 +1,641 @@ +use super::{ + classify_resource_destination, download_text_content, download_text_content_with_headers_and_options, + get_remote_content_as_stream, identity_fetch_options, is_safe_cross_origin_redirect_header, make_epg_test_client, + make_provider_with_dns, make_test_app_config, preview_request_target_for_logging, request_header_value, + resolve_attempt_target, resolve_resource_socket_addrs, response_with_body, + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, + send_input_with_retry_and_provider_policy_with_options_result, start_plain_http_server_with_body, + start_plain_http_server_with_response, start_recording_http_byte_server, start_recording_http_server, + strip_sensitive_headers_for_cross_origin_redirect, test_input_source, test_retry_config, RequestFetchOptions, + ResourceDestination, ResourceDestinationResolver, TextContentBodyOptions, TextContentFetchOptions, +}; +use crate::model::{ + Config, ConfigProvider, InputSource, ResourceRetryConfig, ReverseProxyConfig, ReverseProxyDisabledHeaderConfig, +}; +use flate2::{ + write::{GzEncoder, ZlibEncoder}, + Compression, +}; +use reqwest::header::{HeaderMap, HeaderValue, ACCEPT_ENCODING, COOKIE}; +use shared::{ + defaults::DEFAULT_USER_AGENT, + model::{ConfigProviderDto, InputFetchMethod, OnConnectErrorPolicy, ProviderUrlSelectionPolicy}, +}; +use std::{ + collections::HashMap, + io::{ErrorKind, Write}, + net::IpAddr, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; +use tokio::io::AsyncReadExt; +use url::Url; + +#[test] +fn resource_destinations_are_classified_by_reachability() { + for (address, expected) in [ + ("1.1.1.1", ResourceDestination::Public), + ("8.8.8.8", ResourceDestination::Public), + ("64:ff9b::808:808", ResourceDestination::Public), + ("2002:808:808::", ResourceDestination::Public), + ("::808:808", ResourceDestination::Public), + ("10.0.0.1", ResourceDestination::Private), + ("172.16.5.5", ResourceDestination::Private), + ("192.168.1.20", ResourceDestination::Private), + ("100.64.0.1", ResourceDestination::Private), + ("fc00::1", ResourceDestination::Private), + ("64:ff9b::a00:1", ResourceDestination::Private), + ("2002:0a00:0001::", ResourceDestination::Private), + ("::a00:1", ResourceDestination::Private), + ("::ffff:10.0.0.1", ResourceDestination::Private), + ("0.0.0.0", ResourceDestination::Blocked), + ("127.0.0.1", ResourceDestination::Blocked), + ("169.254.1.1", ResourceDestination::Blocked), + ("169.254.169.254", ResourceDestination::Blocked), + ("255.255.255.255", ResourceDestination::Blocked), + ("224.0.0.1", ResourceDestination::Blocked), + ("::", ResourceDestination::Blocked), + ("::1", ResourceDestination::Blocked), + ("fe80::1", ResourceDestination::Blocked), + ("ff02::1", ResourceDestination::Blocked), + ("::ffff:127.0.0.1", ResourceDestination::Blocked), + ("64:ff9b::7f00:1", ResourceDestination::Blocked), + ("2002:7f00:1::", ResourceDestination::Blocked), + ] { + let address = address.parse::().expect("valid IP address"); + assert_eq!(super::super::classify_ip(address), expected, "{address}"); + } +} + +#[tokio::test] +async fn resource_destination_names_are_classified_without_resolving_ip_literals() { + assert_eq!(classify_resource_destination("192.168.1.20").await, ResourceDestination::Private); + assert_eq!(classify_resource_destination("[::1]").await, ResourceDestination::Blocked); + assert_eq!(classify_resource_destination("[64:ff9b::7f00:1]").await, ResourceDestination::Blocked); + assert_eq!(classify_resource_destination("8.8.8.8").await, ResourceDestination::Public); +} + +#[tokio::test] +async fn resource_destination_resolution_refuses_addresses_local_to_this_host() { + let private = resolve_resource_socket_addrs("10.0.0.1", 80).await.expect("private destination is allowed"); + assert_eq!(private.len(), 1); + + for local_only in ["127.0.0.1", "169.254.169.254", "[::1]", "fe80::1", "0.0.0.0"] { + let error = + resolve_resource_socket_addrs(local_only, 80).await.expect_err("local-only destination must be refused"); + assert_eq!(error.kind(), ErrorKind::PermissionDenied, "{local_only}"); + } +} + +#[tokio::test] +async fn resource_destination_resolver_refuses_local_only_names() { + use reqwest::dns::Resolve; + use std::str::FromStr; + + let resolver = ResourceDestinationResolver::default(); + let name = reqwest::dns::Name::from_str("127.0.0.1").expect("valid destination name"); + let result = resolver.resolve(name).await; + + match result { + Ok(_) => panic!("local-only destination must be refused"), + Err(error) => assert!(error.to_string().contains("local to this host"), "{error}"), + } + + // Only the proxy hosts are exempt, so a proxy on the loopback interface stays usable while a + // destination that resolves to the loopback interface is still refused. + let resolver = ResourceDestinationResolver { allowed_hosts: vec![Arc::from("localhost")].into() }; + let name = reqwest::dns::Name::from_str("localhost").expect("valid proxy name"); + assert!(resolver.resolve(name).await.is_ok(), "the configured proxy host must resolve"); + let name = reqwest::dns::Name::from_str("127.0.0.1").expect("valid destination name"); + assert!(resolver.resolve(name).await.is_err(), "a destination may not use the exemption"); +} + +#[test] +fn test_get_request_headers_prioritization() { + use super::super::{append_user_agent_stream_index, get_request_headers, overlay_upstream_user_agent}; + use axum::http::header::USER_AGENT; + + // Case 1: No headers provided -> Default UA + let headers = get_request_headers::(None, None, None, None); + assert_eq!(headers.get(USER_AGENT).unwrap(), DEFAULT_USER_AGENT); + + // Case 2: No headers provided but config default UA set -> Config default UA + let headers = + get_request_headers::(None, None, None, Some("Config-Default-UA")); + assert_eq!(headers.get(USER_AGENT).unwrap(), "Config-Default-UA"); + + // Case 3: Only client header -> Client UA (overrides config default UA) + let mut client_headers = HashMap::new(); + client_headers.insert("User-Agent".to_string(), b"Client-UA".to_vec()); + let headers = get_request_headers(None, Some(&client_headers), None, Some("Config-Default-UA")); + assert_eq!(headers.get(USER_AGENT).unwrap(), "Client-UA"); + + // Case 4: Both config and client -> Config UA overrides + let mut config_headers = HashMap::new(); + config_headers.insert("User-Agent".to_string(), "Config-UA".to_string()); + let headers = get_request_headers(Some(&config_headers), Some(&client_headers), None, Some("Config-Default-UA")); + assert_eq!(headers.get(USER_AGENT).unwrap(), "Config-UA"); + + // Case 5: Other headers also prioritized + config_headers.insert("X-Test".to_string(), "From-Config".to_string()); + let mut client_headers = HashMap::new(); + client_headers.insert("X-Test".to_string(), b"From-Client".to_vec()); + let headers = get_request_headers(Some(&config_headers), Some(&client_headers), None, Some("Config-Default-UA")); + assert_eq!(headers.get("X-Test").unwrap(), "From-Config"); + + let mut headers = + get_request_headers(Some(&config_headers), Some(&client_headers), None, Some("Config-Default-UA")); + overlay_upstream_user_agent(&mut headers, Some("Channel-UA"), None); + assert_eq!(headers.get(USER_AGENT).unwrap(), "Channel-UA"); + + let disabled = ReverseProxyDisabledHeaderConfig { + referer_header: false, + x_header: false, + cloudflare_header: false, + custom_header: vec!["User-Agent".to_string()], + }; + let mut headers = + get_request_headers(Some(&config_headers), Some(&client_headers), Some(&disabled), Some("Config-Default-UA")); + overlay_upstream_user_agent(&mut headers, Some("Blocked-Channel-UA"), Some(&disabled)); + assert!(!headers.contains_key(USER_AGENT)); + + let mut headers = get_request_headers::(None, None, None, Some("VLC/3.0")); + append_user_agent_stream_index(&mut headers, 42); + assert_eq!(headers.get(USER_AGENT).and_then(|value| value.to_str().ok()), Some("VLC/3.0 42")); + append_user_agent_stream_index(&mut headers, 42); + assert_eq!(headers.get(USER_AGENT).and_then(|value| value.to_str().ok()), Some("VLC/3.0 42")); + + let mut headers = HeaderMap::new(); + let extended_user_agent = HeaderValue::from_bytes(b"Receiver/1.0 \xE4"); + assert!(extended_user_agent.is_ok()); + if let Ok(value) = extended_user_agent { + headers.insert(USER_AGENT, value); + } + append_user_agent_stream_index(&mut headers, 7); + assert_eq!(headers.get(USER_AGENT).map(HeaderValue::as_bytes), Some(b"Receiver/1.0 \xE4 7".as_slice())); +} + +#[test] +fn test_cross_origin_redirect_strips_sensitive_headers() { + let mut headers = HashMap::new(); + headers.insert("Authorization".to_string(), "Bearer test".to_string()); + headers.insert("Cookie".to_string(), "sid=123".to_string()); + headers.insert("Proxy-Authorization".to_string(), "Basic abc".to_string()); + headers.insert("Host".to_string(), "old.host".to_string()); + headers.insert("X-API-Key".to_string(), "secret".to_string()); + headers.insert("Accept".to_string(), "application/x-mpegurl".to_string()); + headers.insert("User-Agent".to_string(), "mpv".to_string()); + + strip_sensitive_headers_for_cross_origin_redirect(&mut headers); + + assert!(!headers.contains_key("Authorization")); + assert!(!headers.contains_key("Cookie")); + assert!(!headers.contains_key("Proxy-Authorization")); + assert!(!headers.contains_key("Host")); + assert!(!headers.contains_key("X-API-Key")); + assert_eq!(headers.get("Accept").map(String::as_str), Some("application/x-mpegurl")); + assert_eq!(headers.get("User-Agent").map(String::as_str), Some("mpv")); +} + +#[test] +fn test_cross_origin_redirect_header_allowlist_is_minimal() { + assert!(is_safe_cross_origin_redirect_header("accept")); + assert!(is_safe_cross_origin_redirect_header("user-agent")); + assert!(is_safe_cross_origin_redirect_header("icy-metadata")); + assert!(!is_safe_cross_origin_redirect_header("authorization")); + assert!(!is_safe_cross_origin_redirect_header("cookie")); + assert!(!is_safe_cross_origin_redirect_header("x-api-key")); + assert!(!is_safe_cross_origin_redirect_header("x-auth-token")); +} + +#[test] +fn test_keep_vhost_false_uses_ip_host_header_for_http() { + let provider = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1"]); + let url = Url::parse("http://example.com:8080/stream").expect("url parse should work"); + + let target = resolve_attempt_target(&url, Some(&provider)); + assert_eq!(target.effective_url.host_str(), Some("192.168.0.1")); + assert_eq!(target.host_header.as_deref(), Some("192.168.0.1:8080")); +} + +#[test] +fn test_keep_vhost_true_uses_hostname_host_header_for_http() { + let provider = make_provider_with_dns(true, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1"]); + let url = Url::parse("http://example.com:8080/stream").expect("url parse should work"); + + let target = resolve_attempt_target(&url, Some(&provider)); + assert_eq!(target.effective_url.host_str(), Some("192.168.0.1")); + assert_eq!(target.host_header.as_deref(), Some("example.com:8080")); +} + +#[test] +fn test_preview_request_target_for_logging_does_not_advance_dns_rotation() { + let provider = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "192.168.0.2"]); + let url = Url::parse("http://example.com/live").expect("url parse should work"); + + let preview = preview_request_target_for_logging(&url, Some(&provider)); + let first = resolve_attempt_target(&url, Some(&provider)); + let second = resolve_attempt_target(&url, Some(&provider)); + + assert_eq!(preview, "http://192.168.0.1/live"); + assert_eq!(first.connect_ip.map(|ip| ip.to_string()), Some("192.168.0.1".to_string())); + assert_eq!(second.connect_ip.map(|ip| ip.to_string()), Some("192.168.0.2".to_string())); + + let provider_https = + make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1", "192.168.0.2"]); + let https_url = Url::parse("https://example.com/live").expect("url parse should work"); + + let https_preview = preview_request_target_for_logging(&https_url, Some(&provider_https)); + let https_first = resolve_attempt_target(&https_url, Some(&provider_https)); + let https_second = resolve_attempt_target(&https_url, Some(&provider_https)); + + assert_eq!(https_preview, "https://example.com/live (connect_ip=192.168.0.1)"); + assert_eq!(https_first.connect_ip.map(|ip| ip.to_string()), Some("192.168.0.1".to_string())); + assert_eq!(https_second.connect_ip.map(|ip| ip.to_string()), Some("192.168.0.2".to_string())); +} + +pub(in crate::utils::network::request::tests) fn response_with_byte_body(status: &str, body: &[u8]) -> Vec { + let mut response = + format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).into_bytes(); + response.extend_from_slice(body); + response +} + +pub(in crate::utils::network::request::tests) fn gzip_encoded(body: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).expect("gzip input"); + encoder.finish().expect("gzip output") +} + +pub(in crate::utils::network::request::tests) fn zlib_encoded(body: &[u8]) -> Vec { + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).expect("zlib input"); + encoder.finish().expect("zlib output") +} + +#[tokio::test] +async fn http_error_response_option_preserves_final_status_headers_and_configured_retries( +) -> Result<(), Box> { + let app_config = test_retry_config(3); + let client = reqwest::Client::builder().no_proxy().build()?; + for failover_only in [false, true] { + for status in [reqwest::StatusCode::FORBIDDEN, reqwest::StatusCode::SERVICE_UNAVAILABLE] { + let response = format!( + "HTTP/1.1 {}\r\nRetry-After: 0\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + super::super::format_http_status(status) + ); + let (addr, accepted, server) = start_plain_http_server_with_response(response).await?; + let url = Url::parse(&format!("http://{addr}/init.hls.fmp4"))?; + let mut options = RequestFetchOptions::default().with_http_error_responses(true); + if failover_only { + options = options.without_resource_retries(); + } + let response = super::super::send_with_retry_and_provider_policy_with_options( + &app_config, + &url, + None, + false, + true, + options, + |resolved| client.get(resolved.clone()), + ) + .await?; + assert_eq!(response.status(), status); + assert_eq!(response.headers()[reqwest::header::RETRY_AFTER], "0"); + assert_eq!(accepted.load(Ordering::SeqCst), if !failover_only && status.is_server_error() { 3 } else { 1 }); + server.abort(); + } + } + Ok(()) +} + +#[test] +fn physical_request_attempt_applies_target_and_final_content_coding_options() { + let client = reqwest::Client::builder().no_proxy().build().expect("test client"); + let request_url = Url::parse("http://origin.example/manifest.m3u8").expect("request URL"); + let effective_url = Url::parse("http://127.0.0.1/manifest.m3u8").expect("effective URL"); + let target = super::super::AttemptTarget { + request_url: request_url.clone(), + effective_url: effective_url.clone(), + host_header: Some("origin.example".to_string()), + sni_host: None, + connect_ip: Some("127.0.0.1".parse().expect("connect IP")), + dns_host: Some("origin.example".to_string()), + }; + let builder = + client.get(request_url).header(ACCEPT_ENCODING, "gzip").header(reqwest::header::HOST, "wrong.example"); + + let (_, request) = super::super::prepare_physical_request_attempt(builder, &target, identity_fetch_options()) + .expect("physical request should build"); + + assert_eq!(request.url(), &effective_url); + assert_eq!(request.headers()[reqwest::header::HOST], "origin.example"); + assert_eq!(request.headers()[ACCEPT_ENCODING], "identity"); + assert_eq!(request.timeout(), Some(&Duration::from_secs(1))); +} + +#[tokio::test] +async fn text_content_download_decodes_headerless_gzip_and_zlib() { + const TEXT: &[u8] = b"headerless legacy text\n"; + + for (label, encoded) in [("gzip", gzip_encoded(TEXT)), ("zlib", zlib_encoded(TEXT))] { + let (addr, requests, handle) = + start_recording_http_byte_server(vec![response_with_byte_body("200 OK", &encoded)]) + .await + .expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/guide.xml")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + + let (content, _) = + download_text_content(&test_retry_config(1), &make_epg_test_client(), &input, None, None, false) + .await + .unwrap_or_else(|error| panic!("headerless {label} text should decode: {error}")); + + assert_eq!(content.as_bytes(), TEXT, "failed for {label}"); + assert_eq!(requests.lock().await.len(), 1); + handle.abort(); + } +} + +#[tokio::test] +async fn text_content_stream_decodes_headerless_gzip_and_zlib() { + const TEXT: &[u8] = b"streamed legacy text\n"; + + for (label, encoded) in [("gzip", gzip_encoded(TEXT)), ("zlib", zlib_encoded(TEXT))] { + let (addr, requests, handle) = + start_recording_http_byte_server(vec![response_with_byte_body("200 OK", &encoded)]) + .await + .expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/guide.xml")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + + let (mut reader, _) = + get_remote_content_as_stream(&test_retry_config(1), &make_epg_test_client(), &input, None, &url) + .await + .unwrap_or_else(|error| panic!("headerless {label} stream should decode: {error}")); + let mut content = Vec::new(); + reader.read_to_end(&mut content).await.expect("read decoded stream"); + + assert_eq!(content, TEXT, "failed for {label}"); + assert_eq!(requests.lock().await.len(), 1); + handle.abort(); + } +} + +#[tokio::test] +async fn content_coding_identity_wins_after_input_and_client_header_merge() { + let (addr, requests, handle) = start_recording_http_server(vec![response_with_body("200 OK", "ok")]) + .await + .expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let mut input = test_input_source(url.to_string(), None); + input.headers.insert("Accept-Encoding".to_string(), "gzip".to_string()); + let mut client_headers = HeaderMap::new(); + client_headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("br")); + + let response = send_input_with_retry_and_provider_policy_with_options_result( + &test_retry_config(1), + &make_epg_test_client(), + &input, + Some(&client_headers), + &url, + identity_fetch_options(), + ) + .await + .expect("origin request should succeed"); + drop(response); + + let captured = requests.lock().await; + assert_eq!(captured.len(), 1); + assert_eq!(request_header_value(&captured[0], "accept-encoding"), Some("identity")); + handle.abort(); +} + +#[tokio::test] +async fn content_coding_identity_manifest_retries_temporary_body_transport_error() { + let truncated = "HTTP/1.1 200 OK\r\nContent-Length: 64\r\nConnection: close\r\n\r\n#EXTM3U\n"; + let responses = vec![truncated.to_string(), response_with_body("200 OK", "#EXTM3U\nsegment.ts\n")]; + let (addr, requests, handle) = start_recording_http_server(responses).await.expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + + let (manifest, _, _) = download_text_content_with_headers_and_options( + &test_retry_config(2), + &make_epg_test_client(), + &input, + None, + false, + TextContentFetchOptions::new( + identity_fetch_options(), + TextContentBodyOptions::hls_manifest(1024, Duration::from_secs(1)), + ), + ) + .await + .expect("temporary body transport failure should retry"); + + assert_eq!(manifest, "#EXTM3U\nsegment.ts\n"); + let captured = requests.lock().await; + assert_eq!(captured.len(), 2); + assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + handle.abort(); +} + +#[tokio::test] +async fn content_coding_identity_is_reapplied_after_provider_url_switch() { + let (first_addr, first_requests, first_handle) = + start_recording_http_server(vec![response_with_body("502 Bad Gateway", "")]) + .await + .expect("first provider origin should start"); + let (second_addr, second_requests, second_handle) = + start_recording_http_server(vec![response_with_body("200 OK", "ok")]) + .await + .expect("second provider origin should start"); + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "provider-a".into(), + urls: vec![format!("http://{first_addr}").into(), format!("http://{second_addr}").into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + })); + let url = Url::parse("provider://provider-a/manifest.m3u8").expect("provider URL"); + let input = test_input_source(url.to_string(), Some(provider)); + + send_input_with_retry_and_provider_policy_with_options_result( + &test_retry_config(1), + &make_epg_test_client(), + &input, + None, + &url, + identity_fetch_options(), + ) + .await + .expect("provider failover should reach successful response"); + + let first = first_requests.lock().await; + let second = second_requests.lock().await; + assert_eq!(first.len(), 1); + assert_eq!(second.len(), 1); + assert_eq!(request_header_value(&first[0], "accept-encoding"), Some("identity")); + assert_eq!(request_header_value(&second[0], "accept-encoding"), Some("identity")); + first_handle.abort(); + second_handle.abort(); +} + +#[tokio::test] +async fn content_coding_identity_is_reapplied_for_same_origin_manual_redirect() { + let redirect = "HTTP/1.1 302 Found\r\nLocation: /final.m3u8\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let (addr, requests, handle) = + start_recording_http_server(vec![redirect.to_string(), response_with_body("200 OK", "ok")]) + .await + .expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/entry.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + &test_retry_config(1), + &make_epg_test_client(), + &input, + None, + &url, + 2, + identity_fetch_options(), + ) + .await + .expect("same-origin redirect should succeed"); + + let captured = requests.lock().await; + assert_eq!(captured.len(), 2); + assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + handle.abort(); +} + +#[tokio::test] +async fn content_coding_identity_survives_cross_origin_redirect_credential_scrubbing() { + let (target_addr, target_requests, target_handle) = + start_recording_http_server(vec![response_with_body("200 OK", "ok")]) + .await + .expect("redirect target should start"); + let redirect = format!( + "HTTP/1.1 302 Found\r\nLocation: http://{target_addr}/final.m3u8\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + let (entry_addr, entry_requests, entry_handle) = + start_recording_http_server(vec![redirect]).await.expect("redirect entry should start"); + let url = Url::parse(&format!("http://{entry_addr}/entry.m3u8")).expect("entry URL"); + let mut input = test_input_source(url.to_string(), None); + input.headers.insert("Authorization".to_string(), "Bearer input-secret".to_string()); + let mut client_headers = HeaderMap::new(); + client_headers.insert(COOKIE, HeaderValue::from_static("sid=client-secret")); + + send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + &test_retry_config(1), + &make_epg_test_client(), + &input, + Some(&client_headers), + &url, + 2, + identity_fetch_options(), + ) + .await + .expect("cross-origin redirect should succeed"); + + let entry = entry_requests.lock().await; + assert_eq!(entry.len(), 1); + assert_eq!(request_header_value(&entry[0], "authorization"), Some("Bearer input-secret")); + assert_eq!(request_header_value(&entry[0], "cookie"), Some("sid=client-secret")); + drop(entry); + let target = target_requests.lock().await; + assert_eq!(target.len(), 1); + assert_eq!(request_header_value(&target[0], "accept-encoding"), Some("identity")); + assert!(request_header_value(&target[0], "authorization").is_none()); + assert!(request_header_value(&target[0], "cookie").is_none()); + entry_handle.abort(); + target_handle.abort(); +} + +#[tokio::test] +async fn manual_redirect_provider_failover_restarts_from_provider_entry() { + let (redirect_addr, redirect_hits, redirect_handle) = match start_plain_http_server_with_response( + "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(), + ) + .await + { + Ok(server) => server, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!("skipping manual_redirect_provider_failover_restarts_from_provider_entry: {err}"); + return; + } + Err(err) => panic!("failed to start redirect target server: {err}"), + }; + let redirect_url = format!("http://127.0.0.1:{}/redirected", redirect_addr.port()); + let provider_a_response = + format!("HTTP/1.1 302 Found\r\nLocation: {redirect_url}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + let provider_entrypoint = + start_plain_http_server_with_response(provider_a_response).await.expect("provider a test server should start"); + let successful_mirror = + start_plain_http_server_with_body(b"provider-b").await.expect("provider b test server should start"); + + let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; + cfg.reverse_proxy = Some(ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: ResourceRetryConfig { max_attempts: 1, ..ResourceRetryConfig::default() }, + disabled_header: None, + stream: None, + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }); + let app_config = make_test_app_config(cfg); + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_millis(400)) + .timeout(Duration::from_secs(2)) + .build() + .expect("http client should build"); + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "provider-a".into(), + urls: vec![ + format!("http://127.0.0.1:{}", provider_entrypoint.0.port()).into(), + format!("http://127.0.0.1:{}", successful_mirror.0.port()).into(), + ], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + })); + let input = InputSource { + name: Arc::::from("test"), + url: "provider://provider-a/live/u/p/1.m3u8".to_string(), + provider: Some(provider), + username: None, + password: None, + method: InputFetchMethod::GET, + headers: HashMap::default(), + }; + let entry_url = Url::parse(input.url.as_str()).expect("provider URL should parse"); + + let response = send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + &app_config, + &client, + &input, + None, + &entry_url, + 5, + RequestFetchOptions::with_attempt_idle_timeout(Duration::from_secs(1)), + ) + .await + .expect("request should fail over from redirected target to next provider entry"); + let provider_url_index = response.provider_url_index; + let body = response.response.text().await.expect("body should be readable"); + + assert_eq!(body, "provider-b"); + assert_eq!(provider_url_index, Some(1)); + assert_eq!(provider_entrypoint.1.load(Ordering::SeqCst), 1); + assert_eq!(redirect_hits.load(Ordering::SeqCst), 1); + assert_eq!(successful_mirror.1.load(Ordering::SeqCst), 1); + + provider_entrypoint.2.abort(); + successful_mirror.2.abort(); + redirect_handle.abort(); +} diff --git a/backend/core/src/utils/network/request/tests/retry.rs b/backend/core/src/utils/network/request/tests/retry.rs new file mode 100644 index 000000000..515a8795c --- /dev/null +++ b/backend/core/src/utils/network/request/tests/retry.rs @@ -0,0 +1,140 @@ +use super::{ + download_text_content_with_headers_and_options, identity_fetch_options, make_epg_test_client, make_test_app_config, + request_header_value, response_with_body, send_input_with_retry_and_provider_policy_with_options_result, + send_with_retry_and_provider_policy, should_retry_text_body_error, start_plain_http_server_with_body, + start_recording_http_server, test_input_source, test_retry_config, TextContentBodyOptions, TextContentFetchOptions, +}; +use crate::{ + model::{Config, ConfigProvider, ResourceRetryConfig, ReverseProxyConfig}, + utils::content_coding::ContentCodingError, +}; +use shared::model::{ConfigProviderDto, ProviderUrlSelectionPolicy}; +use std::{ + io::{Error, ErrorKind}, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; +use url::Url; + +#[test] +fn content_coding_prefix_read_error_is_retryable_but_unsupported_coding_is_not() { + let prefix_read = + Error::other(ContentCodingError::PrefixRead(Error::new(ErrorKind::UnexpectedEof, "prefix truncated"))); + let unsupported = Error::other(ContentCodingError::Unsupported("compress".to_string())); + + assert!(should_retry_text_body_error(&prefix_read)); + assert!(!should_retry_text_body_error(&unsupported)); +} + +#[tokio::test] +async fn content_coding_identity_is_reapplied_for_retry() { + let responses = vec![response_with_body("500 Internal Server Error", ""), response_with_body("200 OK", "ok")]; + let (addr, requests, handle) = start_recording_http_server(responses).await.expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + + send_input_with_retry_and_provider_policy_with_options_result( + &test_retry_config(2), + &make_epg_test_client(), + &input, + None, + &url, + identity_fetch_options(), + ) + .await + .expect("retry should reach successful response"); + + let captured = requests.lock().await; + assert_eq!(captured.len(), 2); + assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + handle.abort(); +} + +#[tokio::test] +async fn text_content_manifest_retry_budget_is_not_multiplied_by_inner_status_retries() { + let corrupt_gzip = + "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: 7\r\nConnection: close\r\n\r\ncorrupt"; + let responses = vec![ + response_with_body("503 Service Unavailable", ""), + response_with_body("502 Bad Gateway", ""), + corrupt_gzip.to_string(), + response_with_body("200 OK", "#EXTM3U\nsegment.ts\n"), + ]; + let (addr, requests, handle) = start_recording_http_server(responses).await.expect("recording origin should start"); + let url = Url::parse(&format!("http://{addr}/manifest.m3u8")).expect("origin URL"); + let input = test_input_source(url.to_string(), None); + let options = TextContentFetchOptions::new( + identity_fetch_options(), + TextContentBodyOptions::hls_manifest(1024, Duration::from_secs(1)), + ); + + download_text_content_with_headers_and_options( + &test_retry_config(3), + &make_epg_test_client(), + &input, + None, + false, + options, + ) + .await + .expect_err("three logical failures must exhaust the configured budget"); + + let captured = requests.lock().await; + assert_eq!(captured.len(), 3, "the fourth success response must not be requested"); + assert!(captured.iter().all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); + handle.abort(); +} + +#[tokio::test] +async fn send_with_retry_policy_false_does_not_fail_over_provider_urls() { + let (addr_b, accepted_b, handle_b) = match start_plain_http_server_with_body(b"b").await { + Ok(server) => server, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!("skipping send_with_retry_policy_false_does_not_fail_over_provider_urls: {err}"); + return; + } + Err(err) => panic!("failed to start test http server: {err}"), + }; + + let mut cfg = Config { connect_timeout_secs: 1, ..Config::default() }; + cfg.accept_insecure_ssl_certificates = true; + cfg.reverse_proxy = Some(ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: ResourceRetryConfig { max_attempts: 3, backoff_millis: 1, ..ResourceRetryConfig::default() }, + disabled_header: None, + stream: None, + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }); + let app_config = make_test_app_config(cfg); + let client = reqwest::Client::builder() + .no_proxy() + .connect_timeout(Duration::from_millis(200)) + .timeout(Duration::from_secs(2)) + .build() + .expect("http client should build"); + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "provider-a".into(), + urls: vec!["http://127.0.0.1:1".into(), format!("http://127.0.0.1:{}", addr_b.port()).into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), + dns: None, + })); + + let url = Url::parse("provider://provider-a/live").expect("provider url should parse"); + let result = + send_with_retry_and_provider_policy(&app_config, &url, Some(&provider), false, false, |resolved_url| { + client.get(resolved_url.clone()) + }) + .await; + + assert!(result.is_err(), "retry disabled must not fail over to the second provider URL"); + assert_eq!(accepted_b.load(Ordering::SeqCst), 0, "fallback provider URL must not be contacted"); + assert_eq!(provider.get_current_index(), 0, "retry disabled must not advance provider URL selection"); + + handle_b.abort(); +} diff --git a/backend/core/src/utils/network/request/tests/support.rs b/backend/core/src/utils/network/request/tests/support.rs new file mode 100644 index 000000000..f684ab6df --- /dev/null +++ b/backend/core/src/utils/network/request/tests/support.rs @@ -0,0 +1,265 @@ +use super::RequestFetchOptions; +use crate::{ + model::{ + AppConfig, Config, ConfigProvider, InputSource, MediaToolCapabilities, ResourceRetryConfig, ReverseProxyConfig, + SourcesConfig, + }, + utils::{content_coding::OutboundContentCodingPolicy, FileLockManager}, +}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use shared::model::{ + ConfigPaths, ConfigProviderDto, DnsScheme, InputFetchMethod, OnConnectErrorPolicy, ProviderDnsDto, + ProviderUrlSelectionPolicy, +}; +use std::{ + collections::HashMap, + net::SocketAddr, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::Mutex, +}; + +pub(in crate::utils::network::request::tests) fn make_test_app_config(config: Config) -> Arc { + Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(config)), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + }) +} + +pub(in crate::utils::network::request::tests) fn make_epg_test_client() -> reqwest::Client { + reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(2)) + .build() + .expect("test client") +} + +pub(in crate::utils::network::request::tests) async fn atomic_download_temp_files(directory: &Path) -> Vec { + let mut result = Vec::new(); + let mut entries = tokio::fs::read_dir(directory).await.expect("read temp directory"); + while let Some(entry) = entries.next_entry().await.expect("read temp directory entry") { + if entry.file_name().to_string_lossy().starts_with(".tuliprox-download-") { + result.push(entry.path()); + } + } + result +} + +pub(in crate::utils::network::request::tests) fn make_provider_with_dns( + keep_vhost: bool, + on_connect_error: OnConnectErrorPolicy, + ips: Vec<&str>, +) -> Arc { + let parsed_ips = ips.into_iter().map(|raw| raw.parse().expect("ip must parse")).collect::>(); + let dto = ConfigProviderDto { + name: "provider-a".into(), + urls: vec!["http://example.com".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), + dns: Some(ProviderDnsDto { + enabled: true, + schemes: Some(vec![DnsScheme::Http, DnsScheme::Https]), + keep_vhost, + overrides: Some(HashMap::from([("example.com".to_string(), parsed_ips)])), + on_connect_error, + ..ProviderDnsDto::default() + }), + }; + Arc::new(ConfigProvider::from(&dto)) +} + +pub(in crate::utils::network::request::tests) async fn start_plain_http_server_with_body( + body: &'static [u8], +) -> std::io::Result<(SocketAddr, Arc, tokio::task::JoinHandle<()>)> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let accepted = Arc::new(AtomicUsize::new(0)); + let accepted_clone = Arc::clone(&accepted); + let content_length = body.len(); + + let handle = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + continue; + }; + accepted_clone.fetch_add(1, Ordering::SeqCst); + let body = body; + tokio::spawn(async move { + let mut buf = vec![0_u8; 2048]; + let _ = socket.read(&mut buf).await; + let response_head = + format!("HTTP/1.1 200 OK\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n"); + let _ = socket.write_all(response_head.as_bytes()).await; + let _ = socket.write_all(body).await; + let _ = socket.shutdown().await; + }); + } + }); + + Ok((addr, accepted, handle)) +} + +pub(in crate::utils::network::request::tests) async fn start_plain_http_server_with_response( + response: String, +) -> std::io::Result<(SocketAddr, Arc, tokio::task::JoinHandle<()>)> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let accepted = Arc::new(AtomicUsize::new(0)); + let accepted_clone = Arc::clone(&accepted); + + let handle = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + continue; + }; + accepted_clone.fetch_add(1, Ordering::SeqCst); + let response = response.clone(); + tokio::spawn(async move { + let mut buf = vec![0_u8; 2048]; + let _ = socket.read(&mut buf).await; + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.shutdown().await; + }); + } + }); + + Ok((addr, accepted, handle)) +} + +pub(in crate::utils::network::request::tests) async fn start_recording_http_server( + responses: Vec, +) -> std::io::Result<(SocketAddr, Arc>>, tokio::task::JoinHandle<()>)> { + start_recording_http_byte_server(responses.into_iter().map(String::into_bytes).collect()).await +} + +pub(in crate::utils::network::request::tests) async fn start_recording_http_byte_server( + responses: Vec>, +) -> std::io::Result<(SocketAddr, Arc>>, tokio::task::JoinHandle<()>)> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let responses = Arc::new(responses); + let response_index = Arc::new(AtomicUsize::new(0)); + + let handle = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + let responses = Arc::clone(&responses); + let response_index = Arc::clone(&response_index); + tokio::spawn(async move { + let mut request = Vec::new(); + loop { + let mut chunk = [0_u8; 2048]; + let Ok(read) = socket.read(&mut chunk).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") || request.len() >= 16 * 1024 { + break; + } + } + requests.lock().await.push(String::from_utf8_lossy(&request).into_owned()); + let index = response_index.fetch_add(1, Ordering::SeqCst); + let Some(response) = responses.get(index).or_else(|| responses.last()) else { + return; + }; + let _ = socket.write_all(response).await; + let _ = socket.shutdown().await; + }); + } + }); + + Ok((addr, requests, handle)) +} + +pub(in crate::utils::network::request::tests) fn response_with_body(status: &str, body: &str) -> String { + format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()) +} + +pub(in crate::utils::network::request::tests) fn request_header_value<'a>( + request: &'a str, + expected_name: &str, +) -> Option<&'a str> { + request.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case(expected_name).then_some(value.trim()) + }) +} + +pub(in crate::utils::network::request::tests) fn test_retry_config(max_attempts: u32) -> Arc { + let mut config = Config { connect_timeout_secs: 1, ..Config::default() }; + config.reverse_proxy = Some(ReverseProxyConfig { + resource_rewrite_disabled: false, + rewrite_secret: [0; 16], + resource_retry: ResourceRetryConfig { + max_attempts, + backoff_millis: 1, + backoff_multiplier: 1.0, + ..ResourceRetryConfig::default() + }, + disabled_header: None, + stream: None, + cache: None, + rate_limit: None, + geoip: None, + stream_history: None, + qos_aggregation: None, + hls_cache: None, + }); + make_test_app_config(config) +} + +pub(in crate::utils::network::request::tests) fn identity_fetch_options() -> RequestFetchOptions { + RequestFetchOptions::with_attempt_idle_timeout(Duration::from_secs(1)) + .with_content_coding(OutboundContentCodingPolicy::Identity) +} + +pub(in crate::utils::network::request::tests) fn test_input_source( + url: String, + provider: Option>, +) -> InputSource { + InputSource { + name: Arc::from("test"), + url, + provider, + username: None, + password: None, + method: InputFetchMethod::GET, + headers: HashMap::new(), + } +} diff --git a/backend/core/src/utils/network/request/tests/tmdb_profile.rs b/backend/core/src/utils/network/request/tests/tmdb_profile.rs index 6142c88c1..dd1a94dd0 100644 --- a/backend/core/src/utils/network/request/tests/tmdb_profile.rs +++ b/backend/core/src/utils/network/request/tests/tmdb_profile.rs @@ -96,13 +96,14 @@ async fn tmdb_profile_connect_timeout_covers_a_hanging_proxy_tunnel() { } #[test] + fn tmdb_profile_construction_errors_are_not_unconfigured_fallbacks() { for proxy in ["not a url", "file:///tmp/proxy", "http://[invalid"] { let cfg = make_test_app_config(Config { proxy: Some(ProxyConfig { url: proxy.into(), username: None, password: None }), ..Config::default() }); - let error = create_tmdb_client(&cfg).err().expect("malformed proxy must fail closed"); + let error = create_tmdb_client(&cfg).expect_err("malformed proxy must fail closed"); assert!(!error.to_string().contains(proxy)); } let cfg = make_test_app_config(Config::default()); diff --git a/backend/curation/src/tmdb/model.rs b/backend/curation/src/tmdb/model.rs index 9d0c0cd55..2a0ff02a9 100644 --- a/backend/curation/src/tmdb/model.rs +++ b/backend/curation/src/tmdb/model.rs @@ -90,7 +90,7 @@ mod tests { use super::*; use serde_json::{json, Value}; - fn translate(results: Value, kind: TmdbTrendingKind, preceding: u32) -> Result { + fn translate(results: &Value, kind: TmdbTrendingKind, preceding: u32) -> Result { translate_page( &serde_json::to_vec(&json!({"page":2,"total_pages":3,"total_results":100,"results":results})).unwrap(), kind, @@ -102,7 +102,7 @@ mod tests { #[test] fn tmdb_wire_translation_preserves_every_row_and_cumulative_rank() { let page = translate( - json!([{"id":7,"title":"First","release_date":"2024-01-02"},{"id":7,"title":"Duplicate"},{"id":u32::MAX}]), + &json!([{"id":7,"title":"First","release_date":"2024-01-02"},{"id":7,"title":"Duplicate"},{"id":u32::MAX}]), TmdbTrendingKind::Movie, 3, ) @@ -114,7 +114,7 @@ mod tests { assert_eq!(page.rows[2].rank, Some(6)); assert_eq!(page.rows[2].tmdb_id, Some(u32::MAX)); let page = translate( - json!([{"id":7,"media_type":"tv","name":"Show","first_air_date":"unknown"},{"id":8,"name":null}]), + &json!([{"id":7,"media_type":"tv","name":"Show","first_air_date":"unknown"},{"id":8,"name":null}]), TmdbTrendingKind::Tv, 0, ) @@ -123,7 +123,7 @@ mod tests { assert_eq!(page.rows[0].title, "Show"); assert_eq!(page.rows[0].year, None); assert_eq!(page.rows[1].title, ""); - assert!(translate(json!([{"id":7}]), TmdbTrendingKind::Movie, u32::MAX).is_err()); + assert!(translate(&json!([{"id":7}]), TmdbTrendingKind::Movie, u32::MAX).is_err()); } #[test] @@ -132,7 +132,7 @@ mod tests { json!({}), json!({"id":0}), json!({"id":-1}), - json!({"id":4294967296_u64}), + json!({"id":4_294_967_296_u64}), json!({"id":"7"}), json!({"id":7.0}), json!({"id":7,"media_type":"tv"}), @@ -140,7 +140,7 @@ mod tests { json!({"id":7,"title":42}), json!({"id":7,"release_date":false}), ] { - assert!(translate(json!([{"id":7},bad]), TmdbTrendingKind::Movie, 0).is_err()); + assert!(translate(&json!([{"id":7},bad]), TmdbTrendingKind::Movie, 0).is_err()); } } } diff --git a/backend/dvr/Cargo.toml b/backend/dvr/Cargo.toml index a9f0b3db0..2ae885116 100644 --- a/backend/dvr/Cargo.toml +++ b/backend/dvr/Cargo.toml @@ -18,7 +18,6 @@ arc-swap.workspace = true chrono.workspace = true fastrand.workspace = true chrono-tz = "0.10.4" -bytes.workspace = true futures.workspace = true libc = "0.2" log.workspace = true diff --git a/backend/dvr/src/recording/recording_disk.rs b/backend/dvr/src/recording/recording_disk.rs index ad1622141..54f691475 100644 --- a/backend/dvr/src/recording/recording_disk.rs +++ b/backend/dvr/src/recording/recording_disk.rs @@ -143,9 +143,9 @@ pub fn filesystem_capacity_for(path: &Path) -> Option<(u64, u64)> { let ok = unsafe { windows_sys::Win32::Storage::FileSystem::GetDiskFreeSpaceExW( wide.as_ptr(), - &mut free_bytes_available, - &mut total_bytes, - &mut total_free_bytes, + &raw mut free_bytes_available, + &raw mut total_bytes, + &raw mut total_free_bytes, ) }; if ok == 0 { @@ -197,9 +197,9 @@ pub fn free_bytes_for(path: &Path) -> Option { let ok = unsafe { windows_sys::Win32::Storage::FileSystem::GetDiskFreeSpaceExW( wide.as_ptr(), - &mut free_bytes_available, - &mut total_bytes, - &mut total_free_bytes, + &raw mut free_bytes_available, + &raw mut total_bytes, + &raw mut total_free_bytes, ) }; if ok == 0 { diff --git a/backend/dvr/src/recording/recording_queue.rs b/backend/dvr/src/recording/recording_queue.rs index b61819d7c..f77bd681b 100644 --- a/backend/dvr/src/recording/recording_queue.rs +++ b/backend/dvr/src/recording/recording_queue.rs @@ -11,31 +11,14 @@ //! * [`shared::model::RecordingTaskDto`] — the owner-safe public projection, //! produced only through [`RecordingTask::to_owner_view`]. -use crate::recording::{ - recording_path, - recording_transition::{self, RecordingCommand}, -}; -use chrono::Utc; -use log::error; -use serde::{Deserialize, Serialize}; pub use shared::model::RecordingTaskState; -use shared::{ - model::{Claims, QueueRevision, RecordingKind, RecordingMetadata, RecordingTaskDto, TaskPriorityDto, UserId}, - utils::{sanitize_filename_chars, CONSTANTS, FILENAME_TRIM_PATTERNS}, -}; use std::{ - collections::{HashMap, HashSet, VecDeque}, - ffi::OsStr, - path::{Path, PathBuf}, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, Mutex as StdMutex, - }, - time::{SystemTime, UNIX_EPOCH}, + collections::{HashMap, VecDeque}, + sync::{atomic::AtomicU64, Arc, Mutex as StdMutex}, }; use tokio::sync::{Mutex, Notify, RwLock}; -use tuliprox_core::model::RecordingConfig; use tuliprox_repository::recording_repository::RecordingRepository; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] pub use tuliprox_repository::recording_repository::{ IdempotencyOutcome, PersistedIdempotency, PersistedRecordingTask, RecordingPartition, }; @@ -45,753 +28,15 @@ fn poisoned_repository() -> std::io::Error { } const RECORDING_WINDOW_EXPIRED_ERR: &str = "Recording window already expired"; + const RECORDING_INTERRUPTED_ERR: &str = "Recording was interrupted by a restart and cannot be resumed"; + static RECORDING_TASK_ID_COUNTER: AtomicU64 = AtomicU64::new(1); + /// How many `_N` names are tried for a file already on disk before the /// request is refused as having no usable path. const MAX_COLLISION_PROBES: usize = 1000; -/// Reason a persisted entry cannot be converted back to its in-memory form -/// during the commit step. Surfaced to the caller so a corrupt persisted file -/// fails closed instead of silently dropping entries. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum PersistedError { - /// The persisted URL could not be parsed. - InvalidUrl(String), -} - -impl std::fmt::Display for PersistedError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::InvalidUrl(s) => write!(f, "persisted url is invalid: {s}"), - } - } -} - -impl std::error::Error for PersistedError {} - -/// Typed error returned from the queue mutation boundary. Every -/// `mutate` closure that fails must return a known variant; the -/// `Other` variant is an escape hatch for dynamically-formatted -/// messages that have no stable wire code. -#[derive(Debug)] -pub enum QueueMutationError { - UnknownRecording, - StateNotEditable, - Forbidden, - InvalidInterval, - InvalidQuotaPool, - InvalidPath, - PaddingLimitExceeded, - QuotaExceeded, - Duplicate, - NotInTerminalState, - DiskFull, - MutationSkipped, - /// The idempotency key was accepted before, for the same request. - IdempotentReplay { - recording_id: String, - }, - /// The idempotency key was accepted before, for a different request. - IdempotencyConflict, - /// Escape hatch for dynamically-formatted validation messages - /// that have no stable wire code. Prefer the typed variants. - Other(String), - Io(std::io::Error), -} - -impl QueueMutationError { - /// Escape-hatch constructor for messages that cannot be expressed - /// as a typed variant. Prefer the typed `Self::X` constructors. - pub fn new(message: impl Into) -> Self { Self::Other(message.into()) } - - pub fn from_io(err: std::io::Error) -> Self { Self::Io(err) } - - /// Stable display message for logging and HTTP error rendering. - pub fn message(&self) -> &'static str { - match self { - Self::UnknownRecording => "recording unknown", - Self::StateNotEditable => "recording state not editable", - Self::Forbidden => "recording forbidden", - Self::InvalidInterval => "recording invalid interval", - Self::InvalidQuotaPool => "recording invalid quota pool", - Self::InvalidPath => "recording invalid path", - Self::PaddingLimitExceeded => "recording_padding_limit_exceeded", - Self::QuotaExceeded => "recording quota exceeded", - Self::Duplicate => "recording duplicate", - Self::NotInTerminalState => "recording not in terminal state", - Self::DiskFull => "disk full", - Self::MutationSkipped => "mutation unexpectedly skipped", - Self::IdempotentReplay { .. } => "recording idempotent replay", - Self::IdempotencyConflict => "recording idempotency conflict", - Self::Other(_) => "queue mutation failed", - Self::Io(_) => "queue mutation persistence failed", - } - } - - pub fn source_io(&self) -> Option<&std::io::Error> { - match self { - Self::Io(err) => Some(err), - _ => None, - } - } -} - -impl std::fmt::Display for QueueMutationError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Other(s) => f.write_str(s), - Self::Io(e) => std::fmt::Display::fmt(e, f), - other => f.write_str(other.message()), - } - } -} - -impl std::error::Error for QueueMutationError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - Self::Io(err) => Some(err as &(dyn std::error::Error + 'static)), - _ => None, - } - } -} - -/// Lock ordering for the queue mutation boundary: -/// -/// 1. `mutation_guard` (`Mutex`) — outermost for persisted mutations and ordered control publication. -/// 2. `queue` (`Mutex`) — always taken before persisted state locks. -/// 3. `scheduled` / `active` / `finished` (`RwLock`, write) — taken after the queue. -/// 4. `control_signal` (`RwLock`) and `control_notify` (`Notify`) — taken after the -/// queue locks; they signal runtime state, not persisted state. -/// 5. `revision` (`AtomicU64`) — no lock; swapped atomically with the -/// commit step. -/// -/// The queue mutation boundary (`mutate`) holds the queue locks only while -/// building the candidate snapshot. It does **not** hold any queue lock -/// while running the user closure or while persisting. Persisting holds -/// the recording repository mutex only. -/// -/// Rule repository mutations are acquired strictly after the queue boundary -/// has committed; never inside it. -/// -/// Apply a single transactional queue mutation. The closure receives an -/// owned `PersistedRecordingQueue` candidate cloned from the current state -/// and returns either a value or a [`QueueMutationError`]. On success the -/// candidate is persisted atomically, then swapped into the in-memory state, -/// then the `QueueRevision` is incremented. On any failure — closure error -/// or persist error — the in-memory state, the repository, and the -/// revision are all unchanged. -pub async fn mutate(this: &RecordingQueue, op: F) -> Result -where - F: FnOnce(&mut PersistedRecordingQueue) -> Result, -{ - match mutate_optional(this, |candidate| op(candidate).map(Some)).await? { - Some(result) => Ok(result), - None => Err(QueueMutationError::MutationSkipped), - } -} - -/// [`mutate`], recording an accepted idempotency key in the same commit. -/// -/// The key has to land with the recording it describes. Writing it -/// afterwards would leave a window in which the recording exists and the -/// record that suppresses its retry does not. -pub async fn mutate_with_idempotency( - this: &RecordingQueue, - idempotency: PersistedIdempotency, - op: F, -) -> Result -where - F: FnOnce(&mut PersistedRecordingQueue) -> Result, -{ - let _mutation = this.mutation_guard.lock().await; - // Checked again under the guard: the caller's lookup ran without it, so - // another request with the same key may have committed in between. - let outcome = this - .lookup_idempotency(&idempotency.principal, &idempotency.key, &idempotency.request_fingerprint) - .await - .map_err(QueueMutationError::from_io)?; - match outcome { - IdempotencyOutcome::Fresh => {} - IdempotencyOutcome::Replay { recording_id } => { - return Err(QueueMutationError::IdempotentReplay { recording_id }); - } - IdempotencyOutcome::Conflict => return Err(QueueMutationError::IdempotencyConflict), - } - match mutate_optional_locked_with(this, Some(idempotency), |candidate| op(candidate).map(Some)).await? { - Some(result) => Ok(result), - None => Err(QueueMutationError::MutationSkipped), - } -} - -/// [`mutate`], then `after` with its result, before the mutation guard is -/// released. -/// -/// For filesystem work that must not race the next mutation: once the guard -/// is released a new admission or a worker start can claim the path the -/// committed state just gave up. -pub async fn mutate_then(this: &RecordingQueue, op: F, after: A) -> Result -where - F: FnOnce(&mut PersistedRecordingQueue) -> Result, - A: FnOnce(&R) -> Fut, - Fut: std::future::Future, -{ - let _mutation = this.mutation_guard.lock().await; - let result = mutate_optional_locked(this, |candidate| op(candidate).map(Some)) - .await? - .ok_or(QueueMutationError::MutationSkipped)?; - after(&result).await; - Ok(result) -} - -/// `prepare`, then [`mutate`] with its result, under one mutation guard. -/// -/// For filesystem work that has to happen before a mutation commits, without -/// blocking a runtime thread inside the mutation closure and without another -/// mutation slipping in between. `prepare` reads the committed in-memory -/// state, which cannot change while the guard is held. -pub async fn mutate_prepared(this: &RecordingQueue, prepare: P, op: F) -> Result -where - P: FnOnce() -> Fut, - Fut: std::future::Future>, - F: FnOnce(&mut PersistedRecordingQueue, T) -> Result, -{ - let _mutation = this.mutation_guard.lock().await; - let prepared = prepare().await?; - mutate_optional_locked(this, |candidate| op(candidate, prepared).map(Some)) - .await? - .ok_or(QueueMutationError::MutationSkipped) -} - -pub async fn mutate_optional(this: &RecordingQueue, op: F) -> Result, QueueMutationError> -where - F: FnOnce(&mut PersistedRecordingQueue) -> Result, QueueMutationError>, -{ - let _mutation = this.mutation_guard.lock().await; - mutate_optional_locked(this, op).await -} - -async fn mutate_optional_locked(this: &RecordingQueue, op: F) -> Result, QueueMutationError> -where - F: FnOnce(&mut PersistedRecordingQueue) -> Result, QueueMutationError>, -{ - mutate_optional_locked_with(this, None, op).await -} - -async fn mutate_optional_locked_with( - this: &RecordingQueue, - idempotency: Option, - op: F, -) -> Result, QueueMutationError> -where - F: FnOnce(&mut PersistedRecordingQueue) -> Result, QueueMutationError>, -{ - let next_revision = this.revision.load(Ordering::SeqCst).saturating_add(1); - - // 2. Build candidate under the queue locks. - let mut candidate = this.snapshot_current(QueueRevision(next_revision)).await; - - // 3. Apply the mutation to a candidate snapshot. The closure can do - // arbitrary validation and refer back to the candidate's prior - // state. - let Some(result) = op(&mut candidate)? else { - return Ok(None); - }; - - let records = candidate.to_records(); - - let PersistedRecordingQueue { - queue: candidate_queue, - scheduled: candidate_scheduled, - active: candidate_active, - finished: candidate_finished, - revision: _, - } = candidate; - - // 4. Validate every persisted entry into its in-memory form before - // swapping. A single corrupt entry must abort the commit so the - // persisted file and the in-memory state stay identical. Without this, - // a bad URL would be silently dropped from memory while the file still - // listed it, desyncing the two. - let mut queue: VecDeque = VecDeque::with_capacity(candidate_queue.len()); - for p in candidate_queue { - queue.push_back( - RecordingQueue::from_persisted(p) - .map_err(|e| QueueMutationError::new(format!("persisted queue entry invalid: {e}")))?, - ); - } - let mut scheduled: Vec = Vec::with_capacity(candidate_scheduled.len()); - for p in candidate_scheduled { - scheduled.push( - RecordingQueue::from_persisted(p) - .map_err(|e| QueueMutationError::new(format!("persisted scheduled entry invalid: {e}")))?, - ); - } - let active = candidate_active - .into_iter() - .map(|task| { - RecordingQueue::from_persisted(task) - .map_err(|e| QueueMutationError::new(format!("persisted active entry invalid: {e}"))) - }) - .collect::, _>>()?; - let mut finished: Vec = Vec::with_capacity(candidate_finished.len()); - for p in candidate_finished { - finished.push( - RecordingQueue::from_persisted(p) - .map_err(|e| QueueMutationError::new(format!("persisted finished entry invalid: {e}")))?, - ); - } - - // 5. Persist only after the complete candidate has been validated. - this.persist_records(next_revision, records, idempotency).await?; - - // 6. Commit. Swap the validated in-memory state from the persisted - // candidate. - let mut queue_lock = this.queue.lock().await; - let mut scheduled_lock = this.scheduled.write().await; - let mut active_lock = this.active.write().await; - let mut finished_lock = this.finished.write().await; - *queue_lock = queue; - *scheduled_lock = scheduled; - *active_lock = active; - *finished_lock = finished; - this.prune_idle_workers(&queue_lock, &scheduled_lock, &active_lock); - this.revision.store(next_revision, Ordering::SeqCst); - this.queue_changed.notify_one(); - Ok(Some(result)) -} - -/// A recording task in memory. Internal shape — never serialized to a -/// client. Use [`RecordingTask::to_owner_view`] for the public projection. -#[derive(Clone, Debug)] -pub struct RecordingTask { - /// uuid of the task for identification. - pub uuid: String, - /// Server-resolved media kind. Decides the execution strategy. - pub kind: RecordingKind, - /// `file_dir` is the directory where the file should be placed. - pub file_dir: PathBuf, - /// `file_path` is the complete path including the filename. - pub file_path: PathBuf, - /// filename is the filename. - pub filename: String, - /// Server-resolved source url. - pub url: reqwest::Url, - /// finished is true when the task reached a terminal state. - pub finished: bool, - /// Bytes transferred so far. - pub size: u64, - /// Total size in bytes (from the Content-Length header), when known. - pub total_size: Option, - /// Paused state. Only VOD/Series can be paused. - pub paused: bool, - /// Optional error if something goes wrong while running the task. - pub error: Option, - /// Task state. - pub state: RecordingTaskState, - /// The input source name used to acquire a provider connection. - pub input_name: Option>, - /// Priority for provider connection preemption (lower = higher priority). - pub priority: i8, - /// Consecutive retry attempts for transient failures. - pub retry_attempts: u8, - /// Unix timestamp of the next retry attempt while waiting. - pub next_retry_at: Option, - /// Recording metadata. Every task carries it; the programme window is - /// populated for `RecordingKind::Live` only. - pub recording: RecordingMetadata, -} - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct PersistedRecordingQueue { - pub queue: Vec, - pub scheduled: Vec, - pub active: Vec, - pub finished: Vec, - /// Monotonic revision. Increments once per committed queue mutation. - /// The in-memory `RecordingQueue` mirrors this counter via an `AtomicU64`. - #[serde(default)] - pub revision: QueueRevision, -} - -impl PersistedRecordingQueue { - /// Flatten the four partitions into the repository's record set, tagging - /// each task with the partition it must be restored into. - fn to_records(&self) -> Vec { - let mut records = - Vec::with_capacity(self.queue.len() + self.scheduled.len() + self.finished.len() + self.active.len()); - let partitions: [(&[PersistedRecordingTask], RecordingPartition); 4] = [ - (&self.queue, RecordingPartition::Queued), - (&self.scheduled, RecordingPartition::Scheduled), - (&self.active, RecordingPartition::Active), - (&self.finished, RecordingPartition::Finished), - ]; - for (tasks, partition) in partitions { - for task in tasks { - let mut task = task.clone(); - task.partition = partition; - records.push(task); - } - } - records - } - - /// Rebuild the four partitions from a repository snapshot. - fn from_records(records: Vec, revision: QueueRevision) -> Self { - let mut restored = Self { revision, ..Self::default() }; - for task in records { - match task.partition { - RecordingPartition::Scheduled => restored.scheduled.push(task), - RecordingPartition::Finished => restored.finished.push(task), - RecordingPartition::Active => restored.active.push(task), - RecordingPartition::Queued => restored.queue.push(task), - } - } - restored - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] -pub enum RecordingControl { - #[default] - None, - Pause, - Cancel, - Restart, -} - -/// What the organised layout groups this recording under. -/// -/// Every kind is filed under its playlist group. Without one, Live groups by -/// channel and VOD by title. A series groups by its series name, which falls -/// back to the programme title and then to the episode pattern applied to the -/// filename stem, which is how the layout was derived before the metadata -/// carried it. -fn recording_grouping( - kind: RecordingKind, - recording: &RecordingMetadata, - recording_cfg: &RecordingConfig, - file_stem: &str, -) -> recording_path::RecordingGrouping { - let grouping = match kind { - RecordingKind::Live => recording_path::RecordingGrouping::live( - recording.channel_name.clone().unwrap_or_else(|| stem_group(recording_cfg, file_stem)), - ), - RecordingKind::Vod => recording_path::RecordingGrouping::vod( - recording.program_title.clone().unwrap_or_else(|| stem_group(recording_cfg, file_stem)), - ), - RecordingKind::Series => recording_path::RecordingGrouping::series( - recording - .series_name - .clone() - .or_else(|| recording.program_title.clone()) - .unwrap_or_else(|| stem_group(recording_cfg, file_stem)), - ), - }; - grouping.in_group(recording.group.clone()) -} - -/// The pre-metadata grouping rule: strip the episode marker and any trailing -/// filename decoration from the stem. -fn stem_group(recording_cfg: &RecordingConfig, file_stem: &str) -> String { - let mut stem = file_stem; - if let Some(re) = &recording_cfg.episode_pattern { - if let Some(captures) = re.captures(stem) { - if let Some(episode) = captures.name("episode") { - if !episode.as_str().is_empty() { - stem = &stem[..episode.start()]; - } - } - } - } - CONSTANTS.re_remove_filename_ending.replace(stem, "").into_owned() -} - -/// What to do with a queued task whose media another entry may already hold. -/// -/// Several library entries can reference one physical file. Promoting each of -/// them would start a second transfer to the same path, so a queued entry that -/// is not the one producing the file must attach to it or wait for it. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum PromotionDecision { - /// Nothing else holds this media; run the transfer. - Execute, - /// A completed entry at this index in `finished` already holds it. - AttachTo(usize), - /// Another active entry is producing this media; wait for its file. - Wait, -} - -/// Two tasks refer to the same media when they carry the same identity. -/// -/// An empty identity never matches, including another empty one: it means the -/// identity could not be resolved, and guessing that two unidentified requests -/// are the same recording would merge two different files. -fn same_media(left: &PersistedRecordingTask, right: &PersistedRecordingTask) -> bool { - !left.media_identity.is_empty() && left.media_identity == right.media_identity -} - -/// Every task in the candidate, whichever partition it sits in. -fn all_tasks(candidate: &PersistedRecordingQueue) -> impl Iterator { - candidate - .queue - .iter() - .chain(candidate.scheduled.iter()) - .chain(candidate.active.iter()) - .chain(candidate.finished.iter()) -} - -/// Whether an entry other than `uuid` still holds the same file. -/// -/// Entries already mid-deletion are not counted. Two concurrent deletions -/// that each saw the other as a holder would both decline to unlink and -/// leave the file behind with nothing pointing at it. -pub fn media_is_still_referenced(candidate: &PersistedRecordingQueue, uuid: &str) -> bool { - let Some(subject) = all_tasks(candidate).find(|task| task.uuid == uuid) else { - return false; - }; - media_held_by_another( - all_tasks(candidate).map(|task| { - (task.uuid.as_str(), task.media_identity.as_str(), task.recording.deleting_previous_state.is_some()) - }), - uuid, - &subject.media_identity, - ) -} - -/// The reference rule itself, over `(uuid, media identity, being deleted)` -/// entries, so the persisted queue and an in-memory snapshot apply the same -/// one. An entry being deleted holds nothing, and an empty identity matches -/// nothing. -pub fn media_held_by_another<'a>( - entries: impl IntoIterator, - uuid: &str, - media: &str, -) -> bool { - !media.is_empty() - && entries - .into_iter() - .any(|(other_uuid, other_media, deleting)| other_uuid != uuid && !deleting && other_media == media) -} - -pub fn promotion_decision(candidate: &PersistedRecordingQueue, task: &PersistedRecordingTask) -> PromotionDecision { - if candidate.active.iter().any(|active| same_media(active, task)) { - return PromotionDecision::Wait; - } - if let Some(index) = - candidate.finished.iter().position(|done| done.state == RecordingTaskState::Completed && same_media(done, task)) - { - return PromotionDecision::AttachTo(index); - } - PromotionDecision::Execute -} - -/// Add the next runnable entry to the active tasks, attaching any entry whose -/// file another entry already produced. -/// -/// Every path that activates a task goes through here. Taking the head of -/// the queue directly would re-download a file that was just completed by the -/// entry ahead of it. -pub fn promote_from_queue(candidate: &mut PersistedRecordingQueue) -> Option<(String, String)> { - let mut index = 0; - while index < candidate.queue.len() { - match promotion_decision(candidate, &candidate.queue[index]) { - PromotionDecision::Execute => { - // Bulk transfers stay serial; live captures can run alongside them. - if candidate.queue[index].kind != RecordingKind::Live - && candidate.active.iter().any(|task| task.kind != RecordingKind::Live) - { - index += 1; - continue; - } - let next = candidate.queue.remove(index); - let promoted = (next.uuid.clone(), next.filename.clone()); - candidate.active.push(next); - return Some(promoted); - } - PromotionDecision::AttachTo(completed) => { - let source = candidate.finished[completed].clone(); - let mut attached = candidate.queue.remove(index); - attach_to_completed(&mut attached, &source); - candidate.finished.push(attached); - // The queue shrank; re-examine this position. - } - PromotionDecision::Wait => index += 1, - } - } - None -} - -/// Adopt an already-produced file instead of transferring it again. -/// -/// Only the physical result is copied. The owner, visibility and quota belong -/// to the entry and must survive: this is one user's link to a file another -/// user's request happened to produce. -pub fn attach_to_completed(task: &mut PersistedRecordingTask, source: &PersistedRecordingTask) { - task.file_dir.clone_from(&source.file_dir); - task.file_path.clone_from(&source.file_path); - task.filename.clone_from(&source.filename); - task.size = source.size; - task.total_size = source.total_size; - task.finished = true; - task.paused = false; - task.error = None; - task.state = RecordingTaskState::Completed; - task.next_retry_at = None; - task.retry_attempts = 0; - task.recording.relative_path.clone_from(&source.recording.relative_path); - task.recording.partial_relative_path = None; - task.recording.measured_bytes = source.recording.measured_bytes; - task.recording.completed_at = source.recording.completed_at; - // The reservation is released: the bytes are already on disk and charged - // to this entry as measured, not reserved. - task.recording.reserved_bytes = 0; -} - -fn generate_recording_task_id() -> String { - let now_nanos = SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |duration| duration.as_nanos()); - let counter = RECORDING_TASK_ID_COUNTER.fetch_add(1, Ordering::Relaxed); - format!("{now_nanos:032x}{counter:016x}") -} - -impl RecordingTask { - /// Build a task for a server-resolved source. Live tasks start in - /// `Scheduled`; VOD/Series tasks start in `Queued`. - pub fn new( - kind: RecordingKind, - req_url: &str, - req_filename: &str, - recording_cfg: &RecordingConfig, - input_name: Option>, - priority: i8, - recording: RecordingMetadata, - ) -> Option { - let url = reqwest::Url::parse(req_url).ok()?; - let tmp_filename = sanitize_filename_chars(req_filename, true).replace("__", "_").replace("_-_", "-"); - let filename_path = Path::new(&tmp_filename); - let file_stem = - filename_path.file_stem().and_then(OsStr::to_str).unwrap_or("").trim_matches(FILENAME_TRIM_PATTERNS); - let file_ext = filename_path.extension().and_then(OsStr::to_str).unwrap_or(""); - - let base_filename = if file_ext.is_empty() { file_stem.to_string() } else { format!("{file_stem}.{file_ext}") }; - let root = Path::new(recording_cfg.directory.as_str()); - let grouping = recording_grouping(kind, &recording, recording_cfg, file_stem); - let mut relative = recording_path::build_relative_path( - kind, - recording_cfg.organize_into_directories, - &grouping, - &base_filename, - ) - .ok()?; - // A file already on disk under this name belongs to an earlier - // recording; the repository reservation resolves logical collisions, - // this only avoids clobbering something already written. Bounded: each - // probe is a filesystem call on the caller's thread. - let base = relative.clone(); - let mut index = 0; - while root.join(&relative).is_file() { - index += 1; - if index > MAX_COLLISION_PROBES { - return None; - } - relative = recording_path::with_collision_suffix(&base, index); - } - let file_path = recording_path::resolve_under_root(root, &relative).ok()?; - let file_dir = file_path.parent().map(Path::to_path_buf)?; - let filename = relative.file_name().and_then(|name| name.to_str())?.to_string(); - - file_path.to_str()?; - let mut recording = recording; - recording.relative_path = Some(relative.to_string_lossy().into_owned()); - - Some(Self { - uuid: generate_recording_task_id(), - kind, - file_dir, - file_path, - filename, - url, - finished: false, - size: 0, - total_size: None, - paused: false, - error: None, - state: if kind.is_scheduled() { RecordingTaskState::Scheduled } else { RecordingTaskState::Queued }, - input_name, - priority, - retry_attempts: 0, - next_retry_at: None, - recording, - }) - } - - /// Padded start of a scheduled Live window. - pub fn scheduled_start(&self) -> Option { self.recording.scheduled_start } - - /// Padded end of a scheduled Live window. - pub fn scheduled_end(&self) -> Option { self.recording.scheduled_end } - - /// Length of the padded Live window, when both bounds are known. - pub fn scheduled_duration_secs(&self) -> Option { - let (start, end) = self.recording.scheduled_start.zip(self.recording.scheduled_end)?; - u64::try_from(end.saturating_sub(start)).ok() - } - - pub fn owner_id(&self) -> &UserId { self.recording.owner_id() } - - /// Public projection for one viewer. Returns `None` when the viewer is - /// neither the owner nor entitled to see a shared task; the owner id is - /// disclosed only to the owner itself. - pub fn to_owner_view(&self, claims: &Claims, shared_visible: bool) -> Option { - let is_owner = claims.subject_id.as_ref().is_some_and(|subject| subject == self.owner_id()); - if !is_owner && !shared_visible { - return None; - } - Some(self.to_view(is_owner)) - } - - /// Projection without a viewer check. Callers that already authorized the - /// viewer pass `is_owner` explicitly. - pub fn to_view(&self, is_owner: bool) -> RecordingTaskDto { - let meta = &self.recording; - RecordingTaskDto { - id: self.uuid.clone(), - title: meta.program_title.clone().unwrap_or_else(|| self.filename.clone()), - kind: self.kind, - priority: match self.priority.cmp(&0) { - std::cmp::Ordering::Less => TaskPriorityDto::High, - std::cmp::Ordering::Equal => TaskPriorityDto::Normal, - std::cmp::Ordering::Greater => TaskPriorityDto::Background, - }, - status: self.state.into(), - retry_attempts: self.retry_attempts, - transferred_bytes: self.size, - total_bytes: self.total_size, - next_retry_at: self.next_retry_at, - error: self.error.clone(), - restart_from_beginning_required: self.state == RecordingTaskState::Failed - && self.error.as_deref() == Some(super::recording_transfer::RANGE_UNSUPPORTED_ERROR), - owner_id: is_owner.then(|| meta.owner_id().clone()), - visibility: meta.visibility, - channel_id: meta.channel_id.clone(), - channel_name: meta.channel_name.clone(), - program_title: meta.program_title.clone(), - program_start: meta.program_start, - program_end: meta.program_end, - scheduled_start: meta.scheduled_start, - scheduled_end: meta.scheduled_end, - pre_roll_secs: meta.pre_roll_secs, - post_roll_secs: meta.post_roll_secs, - completed_at: meta.completed_at, - filename: meta.filename().map(str::to_string), - epg: meta.epg.clone(), - rule_id: meta.provenance.rule_id.clone(), - occurrence_key: meta.provenance.occurrence_key.clone(), - allowed_actions: recording_transition::allowed_actions(self.kind, self.state), - } - } -} - /// Priority-aware wait queue for provider connection slots. /// When the provider is at capacity, tasks register here and are /// woken one-at-a-time in descending priority order (lowest i8 = highest priority). @@ -802,152 +47,17 @@ struct RecordingWaiter { notify: Arc, } -type RecordingWaiters = Arc>>; - struct RecordingWaitRegistration { waiters: RecordingWaiters, id: u64, } -impl Drop for RecordingWaitRegistration { - fn drop(&mut self) { lock_unpoisoned(&self.waiters).retain(|waiter| waiter.id != self.id); } -} - -#[derive(Clone)] -pub struct RecordingWaiterSnapshot { - pub id: u64, - pub input_name: Option>, - pub priority: i8, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum RecordingWaitOutcome { - Signalled, - Paused, - Cancelled, - Restarted, -} - pub struct RecordingSlotWaitQueue { waiters: RecordingWaiters, next_waiter_id: AtomicU64, pub registration_changed: Notify, } -impl Default for RecordingSlotWaitQueue { - fn default() -> Self { Self::new() } -} - -impl RecordingSlotWaitQueue { - pub fn new() -> Self { - Self { - waiters: Arc::new(StdMutex::new(Vec::new())), - next_waiter_id: AtomicU64::new(1), - registration_changed: Notify::new(), - } - } - - fn remove_waiter(&self, waiter_id: u64) { lock_unpoisoned(&self.waiters).retain(|waiter| waiter.id != waiter_id); } - - /// Register and block until this task is signalled or control flow requests pause/cancel. - pub async fn wait( - &self, - input_name: Option>, - priority: i8, - control_signal: &RwLock, - control_notify: &Notify, - ) -> RecordingWaitOutcome { - let waiter_id = self.next_waiter_id.fetch_add(1, Ordering::Relaxed); - let notify = Arc::new(Notify::new()); - lock_unpoisoned(&self.waiters).push(RecordingWaiter { - id: waiter_id, - input_name, - priority, - notify: Arc::clone(¬ify), - }); - - let _registration = RecordingWaitRegistration { waiters: Arc::clone(&self.waiters), id: waiter_id }; - self.registration_changed.notify_one(); - - if let Some(outcome) = self.control_outcome(waiter_id, control_signal).await { - return outcome; - } - - loop { - let controlled = control_notify.notified(); - tokio::pin!(controlled); - controlled.as_mut().enable(); - if let Some(outcome) = self.control_outcome(waiter_id, control_signal).await { - return outcome; - } - tokio::select! { - () = notify.notified() => return RecordingWaitOutcome::Signalled, - () = &mut controlled => {} - } - } - } - - /// Map the current control signal to a wait outcome, deregistering the - /// waiter when the wait ends. - async fn control_outcome( - &self, - waiter_id: u64, - control_signal: &RwLock, - ) -> Option { - let outcome = match *control_signal.read().await { - RecordingControl::Pause => RecordingWaitOutcome::Paused, - RecordingControl::Cancel => RecordingWaitOutcome::Cancelled, - RecordingControl::Restart => RecordingWaitOutcome::Restarted, - RecordingControl::None => return None, - }; - self.remove_waiter(waiter_id); - Some(outcome) - } - - pub fn snapshots(&self) -> Vec { - lock_unpoisoned(&self.waiters) - .iter() - .map(|waiter| RecordingWaiterSnapshot { - id: waiter.id, - input_name: waiter.input_name.clone(), - priority: waiter.priority, - }) - .collect() - } - - /// Wake a specific waiter by id. - pub fn signal_waiter(&self, waiter_id: u64) -> bool { - let mut waiters = lock_unpoisoned(&self.waiters); - if let Some(idx) = waiters.iter().position(|waiter| waiter.id == waiter_id) { - let notify = Arc::clone(&waiters[idx].notify); - waiters.remove(idx); - notify.notify_one(); - true - } else { - false - } - } -} - -fn lock_unpoisoned(mutex: &StdMutex) -> std::sync::MutexGuard<'_, T> { - mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner) -} - -/// Runtime state belonging to one recording. Control signals never cross task boundaries. -/// -/// Queue mutations and worker release take `mutation_guard` first. The worker-map -/// mutex is held only for lookup/removal, never across an await. The `running` -/// lock is released before acquiring queue/active locks or entering a mutation. -/// Control publication follows persisted mutations under `mutation_guard`. -/// Idle-map pruning uses `try_read` under partition locks and retains contested -/// entries for a later worker release. -#[derive(Default)] -pub struct RecordingWorkerState { - pub control_signal: Arc>, - pub control_notify: Arc, - pub running: Arc>, -} - pub struct RecordingQueue { pub queue: Arc>>, pub scheduled: Arc>>, @@ -972,2549 +82,26 @@ impl Default for RecordingQueue { fn default() -> Self { Self::new() } } -impl RecordingQueue { - pub fn worker(&self, uuid: &str) -> Arc { - let mut workers = lock_unpoisoned(&self.workers); - if let Some(worker) = workers.get(uuid) { - return Arc::clone(worker); - } - workers.entry(uuid.to_owned()).or_default().clone() - } - - pub fn existing_worker(&self, uuid: &str) -> Option> { - lock_unpoisoned(&self.workers).get(uuid).cloned() - } - - fn prune_idle_workers( - &self, - queued: &VecDeque, - scheduled: &[RecordingTask], - active: &[RecordingTask], - ) { - lock_unpoisoned(&self.workers).retain(|uuid, worker| { - queued.iter().chain(scheduled).chain(active).any(|task| task.uuid == *uuid) - || !worker.running.try_read().is_ok_and(|running| !*running) - }); - } - - /// Check promotion eligibility without cloning persisted partitions. - pub(crate) async fn has_promotable_queued(&self) -> bool { - use crate::recording::recording_service::recording_identity; - let queued = self.queue.lock().await; - if queued.is_empty() { - return false; - } - let active = self.active.read().await; - let transfer_running = active.iter().any(|task| task.kind != RecordingKind::Live); - let active_media: HashSet<_> = - active.iter().map(|task| recording_identity(&task.recording, task.url.as_str())).collect(); - let mut waiting_media = HashSet::new(); - for task in queued.iter() { - let identity = recording_identity(&task.recording, task.url.as_str()); - if active_media.contains(&identity) { - continue; - } - if task.kind == RecordingKind::Live || !transfer_running { - return true; - } - waiting_media.insert(identity); - } - if waiting_media.is_empty() { - return false; - } - self.finished.read().await.iter().any(|task| { - task.state == RecordingTaskState::Completed - && waiting_media.contains(&recording_identity(&task.recording, task.url.as_str())) - }) - } - - pub(crate) async fn claim_worker(&self, uuid: &str) -> Option> { - let _mutation = self.mutation_guard.lock().await; - if !self.active.read().await.iter().any(|task| task.uuid == uuid && !task.paused && !task.finished) { - return None; - } - let worker = self.worker(uuid); - let mut running = worker.running.write().await; - if *running { - return None; - } - *running = true; - drop(running); - Some(worker) - } - - pub async fn release_worker(&self, uuid: &str) { - let _mutation = self.mutation_guard.lock().await; - if let Some(worker) = self.existing_worker(uuid) { - *worker.running.write().await = false; - let queued = self.queue.lock().await.iter().any(|task| task.uuid == uuid); - let active = self.active.read().await.iter().any(|task| task.uuid == uuid); - if !queued && !active { - lock_unpoisoned(&self.workers).remove(uuid); - } - } - // A resume/requeue may have raced the previous worker's final iteration. - self.queue_changed.notify_one(); - } - - pub async fn workers_running(&self) -> bool { - let workers: Vec<_> = lock_unpoisoned(&self.workers).values().cloned().collect(); - for worker in workers { - if *worker.running.read().await { - return true; - } - } - false - } - - pub async fn mutate_optional_and_clear_control( - &self, - uuid: &str, - expected: RecordingControl, - op: F, - ) -> Result, QueueMutationError> - where - F: FnOnce(&mut PersistedRecordingQueue) -> Result, QueueMutationError>, - { - let _mutation = self.mutation_guard.lock().await; - let worker = self.existing_worker(uuid); - let result = mutate_optional_locked(self, op).await?; - if result.is_some() { - let Some(worker) = worker else { return Ok(result) }; - let mut control = worker.control_signal.write().await; - if *control == expected { - *control = RecordingControl::None; - } - } - Ok(result) - } - - async fn snapshot_current(&self, revision: QueueRevision) -> PersistedRecordingQueue { - let queue = self.queue.lock().await; - let scheduled = self.scheduled.read().await; - let active = self.active.read().await; - let finished = self.finished.read().await; - - PersistedRecordingQueue { - queue: queue.iter().map(Self::to_persisted).collect(), - scheduled: scheduled.iter().map(Self::to_persisted).collect(), - active: active.iter().map(Self::to_persisted).collect(), - finished: finished.iter().map(Self::to_persisted).collect(), - revision, - } - } - - /// The provider input and the priority the active transfer should run at. - /// - /// One transfer serves every entry pointing at its file, so it runs at the - /// strongest priority any of them asked for. Taking the active entry's own - /// value lets a background request hold back somebody else's foreground one - /// for the same media. Lower is stronger. - /// - /// Takes the mutation guard first, like `committed_snapshot`, so the two - /// cannot interleave their inner locks. - pub async fn active_scheduling_priority(&self, uuid: &str) -> Option<(Option>, i8)> { - let _mutation = self.mutation_guard.lock().await; - let active = self.active.read().await; - let active = active.iter().find(|task| task.uuid == uuid)?; - let media = crate::recording::recording_service::recording_identity_key(&active.recording, active.url.as_str()); - let queue = self.queue.lock().await; - let scheduled = self.scheduled.read().await; - let finished = self.finished.read().await; - let strongest = queue - .iter() - .chain(scheduled.iter()) - .chain(finished.iter()) - .filter(|task| { - task.uuid != active.uuid - && crate::recording::recording_service::recording_identity_key(&task.recording, task.url.as_str()) - == media - }) - .fold(active.priority, |strongest, task| strongest.min(task.priority)); - Some((active.input_name.clone(), strongest)) - } - - pub async fn committed_snapshot(&self) -> (QueueRevision, Vec) { - let _mutation = self.mutation_guard.lock().await; - let revision = QueueRevision(self.revision.load(Ordering::SeqCst)); - let queue = self.queue.lock().await; - let scheduled = self.scheduled.read().await; - let active = self.active.read().await; - let finished = self.finished.read().await; - let mut tasks = Vec::with_capacity(queue.len() + scheduled.len() + finished.len() + active.len()); - tasks.extend(queue.iter().cloned()); - tasks.extend(scheduled.iter().cloned()); - tasks.extend(active.iter().cloned()); - tasks.extend(finished.iter().cloned()); - (revision, tasks) - } - - pub async fn committed_partitioned_snapshot(&self) -> (Vec, Vec, Vec) { - let _mutation = self.mutation_guard.lock().await; - let queue = self.queue.lock().await; - let scheduled = self.scheduled.read().await; - let active = self.active.read().await; - let finished = self.finished.read().await; - let mut queued = Vec::with_capacity(queue.len() + scheduled.len()); - queued.extend(queue.iter().cloned()); - queued.extend(scheduled.iter().cloned()); - (queued, active.clone(), finished.clone()) - } - - fn finalize_missed_recording(mut task: RecordingTask) -> RecordingTask { - task.finished = true; - task.paused = false; - task.state = RecordingTaskState::Failed; - task.error = Some(RECORDING_WINDOW_EXPIRED_ERR.to_string()); - task.recording.reserved_bytes = 0; - task - } - - fn recording_start_missed_window(task: &RecordingTask, now_ts: i64) -> bool { - task.kind == RecordingKind::Live - && task.scheduled_start().zip(task.scheduled_duration_secs()).is_some_and(|(start_at, duration_secs)| { - shared::model::recording_math::window_elapsed(start_at, duration_secs, now_ts) - }) - } - - pub fn new() -> Self { Self::new_with_repository(None) } - - /// Opens the recoverable store and builds a queue backed by it. - /// - /// `recovery_root` may point at a different filesystem than `storage_dir`; - /// that is what lets the history outlive the loss of the database volume. - pub fn new_persistent(storage_dir: &Path, recovery_root: &Path) -> std::io::Result { - let (repository, _) = RecordingRepository::open(storage_dir, recovery_root)?; - Ok(Self::new_with_repository(Some(Arc::new(StdMutex::new(repository))))) - } - - pub fn new_with_repository(repository: Option>>) -> Self { - Self { - queue: Arc::from(Mutex::new(VecDeque::new())), - scheduled: Arc::from(RwLock::new(Vec::new())), - active: Arc::from(RwLock::new(Vec::new())), - finished: Arc::from(RwLock::new(Vec::new())), - workers: StdMutex::new(HashMap::new()), - capacity_guard: Mutex::new(()), - queue_changed: Notify::new(), - repository, - slot_waiters: Arc::new(RecordingSlotWaitQueue::new()), - revision: Arc::new(AtomicU64::new(0)), - mutation_guard: Arc::new(Mutex::new(())), - } - } - - pub fn to_persisted(task: &RecordingTask) -> PersistedRecordingTask { - PersistedRecordingTask { - media_identity: crate::recording::recording_service::recording_identity_key( - &task.recording, - task.url.as_str(), - ), - // Overwritten from the owning partition when the candidate is flattened. - partition: RecordingPartition::default(), - uuid: task.uuid.clone(), - kind: task.kind, - file_dir: task.file_dir.clone(), - file_path: task.file_path.clone(), - filename: task.filename.clone(), - url: task.url.to_string(), - finished: task.finished, - size: task.size, - total_size: task.total_size, - paused: task.paused, - error: task.error.clone(), - state: task.state, - input_name: task.input_name.as_ref().map(std::string::ToString::to_string), - priority: task.priority, - retry_attempts: task.retry_attempts, - next_retry_at: task.next_retry_at, - recording: task.recording.clone(), - } - } - - pub fn from_persisted(task: PersistedRecordingTask) -> Result { - let url = reqwest::Url::parse(&task.url).map_err(|e| PersistedError::InvalidUrl(e.to_string()))?; - Ok(RecordingTask { - uuid: task.uuid, - kind: task.kind, - file_dir: task.file_dir, - file_path: task.file_path, - filename: task.filename, - url, - finished: task.finished, - size: task.size, - total_size: task.total_size, - paused: task.paused, - error: task.error, - state: task.state, - input_name: task.input_name.map(|s| Arc::from(s.as_str())), - priority: task.priority, - retry_attempts: task.retry_attempts, - next_retry_at: task.next_retry_at, - recording: task.recording, - }) - } - - pub async fn persist_to_disk(&self) -> std::io::Result<()> { - // `snapshot_current` takes the partition locks in mutation order; the - // guard is what keeps that order from meeting a reader that takes them - // differently. - let _mutation = self.mutation_guard.lock().await; - let revision = self.revision.load(Ordering::SeqCst); - let records = self.snapshot_current(QueueRevision(revision)).await.to_records(); - let result = self.commit_records(revision, records, None).await; - // Callers discard the result; log here so persistence failures are never silent - if let Err(err) = &result { - error!("Failed to persist recording queue: {err}"); - } - result - } - - /// Commit a record set to the repository, off the async executor. - /// - /// The repository fsyncs both its journal and its B+Tree, so it must not - /// run on a runtime worker thread. - async fn commit_records( - &self, - revision: u64, - records: Vec, - idempotency: Option, - ) -> std::io::Result<()> { - let Some(repository) = self.repository.clone() else { - return Ok(()); - }; - let now = chrono::Utc::now().timestamp(); - tokio::task::spawn_blocking(move || { - let mut guard = repository.lock().map_err(|_| poisoned_repository())?; - guard.commit_with_idempotency(revision, &records, idempotency, now) - }) - .await - .map_err(std::io::Error::other)? - } - - async fn persist_records( - &self, - revision: u64, - records: Vec, - idempotency: Option, - ) -> Result<(), QueueMutationError> { - self.commit_records(revision, records, idempotency).await.map_err(QueueMutationError::from_io) - } - - /// Load the canonical recording state from the repository. - /// - /// An empty repository is a fresh install. A record that cannot be - /// converted back to its in-memory form is an error the caller must - /// propagate: the repository is never reset or rebuilt from memory. - /// The recovery contract for an operator tool. - /// - /// Read-only: it reports, and never opens the B+Tree for mutation. - /// `None` when the queue is not repository backed. - pub async fn recovery_health(&self) -> Option { - let repository = self.repository.clone()?; - tokio::task::spawn_blocking(move || repository.lock().ok().map(|guard| guard.health())).await.ok().flatten() - } - - /// Whether a request carrying this idempotency key has already been - /// accepted. Without a repository there is nothing to remember, so every - /// request is fresh. - pub async fn lookup_idempotency( - &self, - principal: &str, - key: &str, - request_fingerprint: &str, - ) -> std::io::Result { - let Some(repository) = self.repository.clone() else { - return Ok(IdempotencyOutcome::Fresh); - }; - let (principal, key, fingerprint) = (principal.to_owned(), key.to_owned(), request_fingerprint.to_owned()); - let now = Utc::now().timestamp(); - tokio::task::spawn_blocking(move || { - let mut guard = repository.lock().map_err(|_| poisoned_repository())?; - guard.lookup_idempotency(&principal, &key, &fingerprint, now) - }) - .await - .map_err(std::io::Error::other)? - } - - pub async fn load_from_disk(&self) -> std::io::Result<()> { - let Some(repository) = self.repository.clone() else { - return Ok(()); - }; - let snapshot = tokio::task::spawn_blocking(move || { - let mut guard = repository.lock().map_err(|_| poisoned_repository())?; - guard.load() - }) - .await - .map_err(std::io::Error::other)??; - - let persisted = PersistedRecordingQueue::from_records(snapshot.tasks, QueueRevision(snapshot.queue_revision)); - - let invalid = |err: PersistedError| std::io::Error::new(std::io::ErrorKind::InvalidData, err); - let mut queue = VecDeque::with_capacity(persisted.queue.len()); - for task in persisted.queue { - queue.push_back(Self::recover_loaded_task(Self::from_persisted(task).map_err(invalid)?)); - } - let now_ts = Utc::now().timestamp(); - let mut scheduled = Vec::with_capacity(persisted.scheduled.len()); - let mut missed_scheduled = Vec::new(); - for task in persisted.scheduled { - let task = Self::recover_loaded_task(Self::from_persisted(task).map_err(invalid)?); - if Self::recording_start_missed_window(&task, now_ts) { - missed_scheduled.push(task); - } else { - scheduled.push(task); - } - } - let active = persisted - .active - .into_iter() - .map(|task| Self::from_persisted(task).map(Self::recover_loaded_task).map_err(invalid)) - .collect::, _>>()?; - let mut finished = Vec::with_capacity(persisted.finished.len()); - for task in persisted.finished { - finished.push(Self::from_persisted(task).map_err(invalid)?); - } - finished.extend(missed_scheduled.into_iter().map(Self::finalize_missed_recording)); - - *self.queue.lock().await = queue; - *self.scheduled.write().await = scheduled; - *self.finished.write().await = finished; - self.revision.store(persisted.revision.0, Ordering::SeqCst); - let mut paused = Vec::new(); - for task in active { - if task.paused || task.state == RecordingTaskState::Paused { - paused.push(task); - } else if !task.finished && task.state != RecordingTaskState::Cancelled { - self.queue.lock().await.push_front(task); - } else { - self.finished.write().await.push(task); - } - } - *self.active.write().await = paused; - lock_unpoisoned(&self.workers).clear(); - Ok(()) - } - - fn recover_loaded_task(mut task: RecordingTask) -> RecordingTask { - if task.paused || task.state == RecordingTaskState::Paused { - task.paused = true; - task.finished = false; - task.state = RecordingTaskState::Paused; - return task; - } - // The worker that was going to acknowledge the cancellation died with - // the process. Nothing will ever finish it, and it is not `finished`, - // so the requeue below would restart work the user cancelled. - if task.state == RecordingTaskState::Cancelling { - task.paused = false; - task.finished = true; - task.state = RecordingTaskState::Cancelled; - task.error.get_or_insert_with(|| "Cancelled by user".to_string()); - task.next_retry_at = None; - task.recording.reserved_bytes = 0; - return task; - } - if task.state == RecordingTaskState::Scheduled { - task.paused = false; - task.finished = false; - return task; - } - // A live capture cannot be picked up again: whatever was broadcast while - // the process was down is gone. Requeueing would restart it and write a - // recording silently missing everything up to now. - if task.kind == RecordingKind::Live - && matches!(task.state, RecordingTaskState::Running | RecordingTaskState::WaitingForCapacity) - { - task.paused = false; - task.finished = true; - task.state = RecordingTaskState::Failed; - task.error = Some(RECORDING_INTERRUPTED_ERR.to_string()); - task.next_retry_at = None; - task.recording.reserved_bytes = 0; - return task; - } - if !task.finished { - task.paused = false; - task.state = RecordingTaskState::Queued; - task.error = None; - task.retry_attempts = 0; - task.next_retry_at = None; - } - task - } - - /// Pause the active task. Persists the new state through the - /// transactional boundary. The runtime-only control signal is published - /// after the commit while the mutation guard still preserves ordering. - /// Live captures are not pausable and are rejected here. - pub async fn pause_active(&self, uuid: &str) -> Result { - let _mutation = self.mutation_guard.lock().await; - let changed = mutate_optional_locked(self, |candidate| { - let Some(active) = candidate.active.iter_mut().find(|active| active.uuid == uuid) else { - return Ok(None); - }; - active.state = recording_transition::transition(active.kind, active.state, RecordingCommand::Pause) - .map_err(|_| QueueMutationError::StateNotEditable)?; - active.paused = true; - active.next_retry_at = None; - Ok(Some(true)) - }) - .await? - .unwrap_or(false); - if !changed { - return Ok(false); - } - let worker = self.worker(uuid); - *worker.control_signal.write().await = RecordingControl::Pause; - worker.control_notify.notify_waiters(); - Ok(true) - } - - /// Resume the active task. Persists the new state through the - /// transactional boundary. - pub async fn resume_active(&self, uuid: &str) -> Result { - let _mutation = self.mutation_guard.lock().await; - let changed = mutate_optional_locked(self, |candidate| { - let Some(active) = candidate.active.iter_mut().find(|active| active.uuid == uuid && active.paused) else { - return Ok(None); - }; - active.state = recording_transition::transition(active.kind, active.state, RecordingCommand::Resume) - .map_err(|_| QueueMutationError::StateNotEditable)?; - active.paused = false; - active.next_retry_at = None; - Ok(Some(true)) - }) - .await? - .unwrap_or(false); - if !changed { - return Ok(false); - } - let worker = self.worker(uuid); - *worker.control_signal.write().await = RecordingControl::None; - worker.control_notify.notify_waiters(); - Ok(true) - } - - /// Cancel the active task `uuid`, if it is still the active one. - /// - /// A paused task with no running worker is filed as `Cancelled` immediately. - /// A worker still closing a paused stream is asked to cancel just like a - /// running one: it commits `Cancelled` after releasing its file and slot. - /// Returns whether cancellation finished immediately, or `None` when - /// `uuid` is not active. - pub async fn cancel_requested(&self, uuid: &str) -> Result, QueueMutationError> { - let _mutation = self.mutation_guard.lock().await; - let worker_running = match self.existing_worker(uuid) { - Some(worker) => *worker.running.read().await, - None => false, - }; - let cancelled_immediately = mutate_optional_locked(self, |candidate| { - let Some(active) = candidate.active.iter().find(|active| active.uuid == uuid) else { - return Ok(None); - }; - let cancelled_immediately = active.paused && !worker_running; - if cancelled_immediately { - let Some(index) = candidate.active.iter().position(|task| task.uuid == uuid) else { - return Ok(None); - }; - let mut cancelled = candidate.active.remove(index); - cancelled.finished = true; - cancelled.paused = false; - cancelled.next_retry_at = None; - cancelled.error.get_or_insert_with(|| "Cancelled by user".to_string()); - cancelled.state = RecordingTaskState::Cancelled; - cancelled.recording.reserved_bytes = 0; - candidate.finished.push(cancelled); - promote_from_queue(candidate); - } else if let Some(active) = candidate.active.iter_mut().find(|task| task.uuid == uuid) { - // The worker still owns the file and the provider slot; it - // commits `Cancelled` once it has let go. - active.state = if active.paused { - RecordingTaskState::Cancelling - } else { - recording_transition::transition( - active.kind, - active.state, - recording_transition::RecordingCommand::Cancel, - ) - .unwrap_or(RecordingTaskState::Cancelling) - }; - active.paused = false; - active.error = Some("Cancelled by user".to_string()); - active.next_retry_at = None; - } - Ok(Some(cancelled_immediately)) - }) - .await?; - - if let Some(cancelled_immediately) = cancelled_immediately { - let worker = if cancelled_immediately { self.existing_worker(uuid) } else { Some(self.worker(uuid)) }; - if let Some(worker) = worker { - *worker.control_signal.write().await = - if cancelled_immediately { RecordingControl::None } else { RecordingControl::Cancel }; - worker.control_notify.notify_waiters(); - if cancelled_immediately { - lock_unpoisoned(&self.workers).remove(uuid); - } - } - } - Ok(cancelled_immediately) - } - - /// Ask the worker to restart so it picks up a reloaded configuration. - /// - /// A restart never replaces a pending pause or cancel. The fallback below - /// runs later than the caller, so it may land after a user's pause; it - /// must not turn that pause into a requeue. - pub fn request_worker_restart(&self) { - fn request(control: &mut RecordingControl) -> bool { - if *control == RecordingControl::None { - *control = RecordingControl::Restart; - } - *control == RecordingControl::Restart - } - let workers: Vec<_> = lock_unpoisoned(&self.workers).values().cloned().collect(); - for worker in workers { - if let Ok(mut control) = worker.control_signal.try_write() { - if request(&mut control) { - worker.control_notify.notify_waiters(); - } - } else { - let worker = Arc::clone(&worker); - tokio::spawn(async move { - let mut control = worker.control_signal.write().await; - if request(&mut control) { - worker.control_notify.notify_waiters(); - } - }); - } - } - } - - pub async fn remove_from_queue(&self, uuid: &str) -> Result { - Ok(mutate_optional(self, |candidate| { - let queue_len = candidate.queue.len(); - candidate.queue.retain(|task| task.uuid != uuid); - if candidate.queue.len() != queue_len { - return Ok(Some(true)); - } - let scheduled_len = candidate.scheduled.len(); - candidate.scheduled.retain(|task| task.uuid != uuid); - Ok((candidate.scheduled.len() != scheduled_len).then_some(true)) - }) - .await? - .unwrap_or(false)) - } - - pub async fn remove_finished(&self, uuid: &str) -> Result { - Ok(mutate_optional(self, |candidate| { - let initial_len = candidate.finished.len(); - candidate.finished.retain(|task| task.uuid != uuid); - Ok((candidate.finished.len() != initial_len).then_some(true)) - }) - .await? - .unwrap_or(false)) - } - - pub async fn remove(&self, uuid: &str) -> Result { - Ok(mutate_optional(self, |candidate| { - let original_len = candidate.queue.len() + candidate.scheduled.len() + candidate.finished.len(); - candidate.queue.retain(|task| task.uuid != uuid); - candidate.scheduled.retain(|task| task.uuid != uuid); - candidate.finished.retain(|task| task.uuid != uuid); - let current_len = candidate.queue.len() + candidate.scheduled.len() + candidate.finished.len(); - Ok((current_len != original_len).then_some(true)) - }) - .await? - .unwrap_or(false)) - } - - /// Requeue a finished VOD/Series transfer. A Live capture cannot be - /// retried: its programme window is gone. - pub async fn retry_finished(&self, uuid: &str) -> Result { - Ok(mutate_optional(self, |candidate| { - if let Some(pos) = candidate.finished.iter().position(|task| task.uuid == uuid) { - let mut task = candidate.finished.remove(pos); - let Ok(next) = recording_transition::transition(task.kind, task.state, RecordingCommand::Retry) else { - candidate.finished.insert(pos, task); - return Ok(None); - }; - task.finished = false; - task.size = 0; - task.paused = false; - task.error = None; - task.state = next; - task.retry_attempts = 0; - task.next_retry_at = None; - candidate.queue.push(task); - Ok(Some(true)) - } else { - Ok(None) - } - }) - .await? - .unwrap_or(false)) - } - - pub async fn promote_due_scheduled(&self, now_ts: i64) -> usize { - let due = self.scheduled.read().await.iter().any(|task| { - task.recording.scheduled_start.is_some_and(|start| start <= now_ts) - || (task.kind == RecordingKind::Live && task.recording.scheduled_end.is_some_and(|end| now_ts >= end)) - }); - if !due { - return 0; - } - let result = mutate_optional(self, |candidate| { - let mut due_tasks = Vec::new(); - let mut missed_recordings = Vec::new(); - candidate.scheduled.retain(|task| { - let scheduled_start = task.recording.scheduled_start; - let is_missed = - task.kind == RecordingKind::Live && task.recording.scheduled_end.is_some_and(|end| now_ts >= end); - if is_missed { - let mut missed = task.clone(); - missed.finished = true; - missed.paused = false; - missed.state = RecordingTaskState::Failed; - missed.error = Some(RECORDING_WINDOW_EXPIRED_ERR.to_string()); - missed.recording.reserved_bytes = 0; - missed_recordings.push(missed); - return false; - } - let is_due = scheduled_start.is_some_and(|start_at| start_at <= now_ts); - if is_due { - let mut queued = task.clone(); - queued.state = RecordingTaskState::Queued; - queued.paused = false; - queued.finished = false; - queued.error = None; - queued.size = 0; - queued.total_size = None; - queued.retry_attempts = 0; - queued.next_retry_at = None; - due_tasks.push(queued); - } - !is_due - }); - - if due_tasks.is_empty() && missed_recordings.is_empty() { - return Ok(None); - } - - let due_count = due_tasks.len(); - let missed_count = missed_recordings.len(); - candidate.finished.extend(missed_recordings); - candidate.queue.splice(0..0, due_tasks); - Ok(Some(if due_count == 0 { missed_count } else { due_count })) - }) - .await; - - match result { - Ok(Some(promoted)) => promoted, - Ok(None) => 0, - Err(err) => { - error!("Promoting due scheduled recordings failed: {err}"); - 0 - } - } - } - - pub async fn promote_due_scheduled_now(&self) -> usize { self.promote_due_scheduled(Utc::now().timestamp()).await } -} #[cfg(test)] -mod tests { - use super::*; - use shared::model::recording::{RecordingOwner, RecordingSource, RecordingVisibility}; - use std::time::{SystemTime, UNIX_EPOCH}; - use tokio::time::{timeout, Duration}; - - fn temp_state_file(name: &str) -> PathBuf { - let nanos = SystemTime::now().duration_since(UNIX_EPOCH).expect("time").as_nanos(); - let dir = std::env::temp_dir().join(format!("tuliprox_{name}_{nanos}")); - std::fs::create_dir_all(&dir).expect("create state dir"); - dir - } - - fn live_meta(owner: &str, start: i64, duration: u64) -> RecordingMetadata { - RecordingMetadata::new_live( - RecordingOwner::User(UserId::from(owner)), - RecordingVisibility::Private, - RecordingSource::new("target", "v", "input-a"), - start, - start.saturating_add(duration.cast_signed()), - 0, - 0, - ) - } - - fn media_meta(owner: &str) -> RecordingMetadata { - RecordingMetadata::new_media( - RecordingOwner::User(UserId::from(owner)), - RecordingVisibility::Private, - RecordingSource::new("target", "v", "input-a"), - String::new(), - ) - } - - fn task(uuid: &str, kind: RecordingKind, state: RecordingTaskState) -> RecordingTask { - RecordingTask { - uuid: uuid.to_string(), - kind, - file_dir: PathBuf::from("/tmp"), - file_path: PathBuf::from(format!("/tmp/{uuid}.ts")), - filename: format!("{uuid}.ts"), - url: reqwest::Url::parse(&format!("https://example.com/{uuid}")).expect("valid url"), - finished: false, - size: 0, - total_size: None, - paused: false, - error: None, - state, - input_name: None, - priority: 0, - retry_attempts: 0, - next_retry_at: None, - recording: match kind { - RecordingKind::Live => live_meta("web:alice", 1_700_000_000, 3_600), - _ => media_meta("web:alice"), - }, - } - } - - #[tokio::test] - async fn recovering_multiple_active_recordings_preserves_every_task_and_its_recovery_policy( - ) -> Result<(), Box> { - let dir = tempfile::tempdir()?; - let queue = RecordingQueue::new_persistent(dir.path(), dir.path())?; - let mut tasks = Vec::new(); - for index in 0..2 { - tasks.push(RecordingQueue::to_persisted(&task( - &format!("live-{index}"), - RecordingKind::Live, - RecordingTaskState::Running, - ))); - let mut paused = task(&format!("paused-{index}"), RecordingKind::Vod, RecordingTaskState::Paused); - paused.paused = true; - tasks.push(RecordingQueue::to_persisted(&paused)); - tasks.push(RecordingQueue::to_persisted(&task( - &format!("vod-{index}"), - RecordingKind::Vod, - RecordingTaskState::Running, - ))); - } - mutate(&queue, |candidate| { - candidate.active = tasks; - Ok(()) - }) - .await?; - let restored = RecordingQueue::new_persistent(dir.path(), dir.path())?; - restored.load_from_disk().await?; - assert_eq!(restored.active.read().await.len(), 2); - assert!(restored.active.read().await.iter().all(|task| task.state == RecordingTaskState::Paused)); - assert_eq!(restored.queue.lock().await.len(), 2); - assert!(restored.queue.lock().await.iter().all(|task| task.state == RecordingTaskState::Queued)); - assert_eq!(restored.finished.read().await.len(), 2); - assert!(restored - .finished - .read() - .await - .iter() - .all(|task| task.state == RecordingTaskState::Failed && task.kind == RecordingKind::Live)); - let (_, tasks) = restored.committed_snapshot().await; - assert_eq!(tasks.len(), 6); - assert!(!restored.workers_running().await); - Ok(()) - } - - #[tokio::test] - async fn pause_and_resume_keep_active_download_resumable() { - let queue = RecordingQueue::new(); - let active = RecordingTask { - size: 42, - total_size: Some(100), - state: RecordingTaskState::Running, - ..task("id", RecordingKind::Vod, RecordingTaskState::Running) - }; - - *queue.active.write().await = vec![active]; - queue.pause_active("id").await.expect("pause active"); - - let paused = queue.active.read().await.first().cloned().expect("active download"); - assert_eq!(paused.state, RecordingTaskState::Paused); - assert!(paused.paused); - assert!(!paused.finished); - - queue.resume_active("id").await.expect("resume active"); - - let resumed = queue.active.read().await.first().cloned().expect("active download"); - assert_eq!(resumed.state, RecordingTaskState::Running); - assert!(!resumed.paused); - assert!(!resumed.finished); - } - - #[tokio::test] - async fn cancel_leaves_a_running_task_cancelling_until_the_worker_lets_go() { - let queue = RecordingQueue::new(); - let worker = queue.worker("id"); - let active = task("id", RecordingKind::Vod, RecordingTaskState::Running); - - *queue.active.write().await = vec![active]; - assert_eq!(queue.cancel_requested("id").await.expect("cancel active"), Some(false)); - - let cancelled = queue.active.read().await.first().cloned().expect("active download"); - assert_eq!(cancelled.state, RecordingTaskState::Cancelling); - assert_eq!(*worker.control_signal.read().await, RecordingControl::Cancel); - assert!(!cancelled.finished); - assert_eq!(cancelled.error.as_deref(), Some("Cancelled by user")); - assert!(queue.finished.read().await.is_empty()); - } - - #[tokio::test] - async fn cancelling_a_paused_task_releases_idle_worker_state() { - for running in [None, Some(false), Some(true)] { - let queue = RecordingQueue::new(); - let mut paused = task("paused", RecordingKind::Vod, RecordingTaskState::Paused); - paused.paused = true; - queue.active.write().await.push(paused); - if let Some(running) = running { - *queue.worker("paused").running.write().await = running; - } - let immediate = running != Some(true); - assert_eq!(queue.cancel_requested("paused").await.expect("cancel paused"), Some(immediate)); - if immediate { - assert!(queue.active.read().await.is_empty()); - assert_eq!(queue.finished.read().await[0].state, RecordingTaskState::Cancelled); - } else { - assert!(queue.finished.read().await.is_empty(), "the stream has not closed yet"); - assert_eq!(queue.active.read().await[0].state, RecordingTaskState::Cancelling); - assert_eq!(*queue.worker("paused").control_signal.read().await, RecordingControl::Cancel); - mutate(&queue, |candidate| { - let mut task = candidate.active.remove(0); - task.state = RecordingTaskState::Cancelled; - task.finished = true; - candidate.finished.push(task); - Ok(()) - }) - .await - .expect("worker closes and commits"); - } - let retained = queue.workers.lock().expect("worker map").contains_key("paused"); - assert_eq!(retained, running == Some(true), "only an exiting worker retains its state"); - queue.release_worker("paused").await; - assert!(!queue.workers.lock().expect("worker map").contains_key("paused")); - } - } - - #[tokio::test] - async fn removing_requeued_tasks_prunes_only_idle_workers() { - for running in [false, true] { - let queue = RecordingQueue::new(); - queue.queue.lock().await.push_back(task("requeued", RecordingKind::Vod, RecordingTaskState::Queued)); - let worker = queue.worker("requeued"); - *worker.running.write().await = running; - assert!(queue.remove_from_queue("requeued").await.expect("remove")); - assert_eq!(queue.existing_worker("requeued").is_some(), running); - queue.request_worker_restart(); - if !running { - assert_eq!(*worker.control_signal.read().await, RecordingControl::None); - } - queue.release_worker("requeued").await; - assert!(queue.existing_worker("requeued").is_none()); - } - } - - #[tokio::test] - async fn attaching_requeued_tasks_prunes_their_idle_worker_state() { - let queue = RecordingQueue::new(); - let completed = task("completed", RecordingKind::Vod, RecordingTaskState::Completed); - let mut waiting = task("waiting", RecordingKind::Vod, RecordingTaskState::Queued); - waiting.url = completed.url.clone(); - queue.queue.lock().await.push_back(waiting); - queue.finished.write().await.push(completed); - queue.worker("waiting"); - mutate(&queue, |candidate| { - assert!(promote_from_queue(candidate).is_none()); - Ok(()) - }) - .await - .expect("attach existing recording"); - assert!(queue.queue.lock().await.is_empty()); - assert_eq!(queue.finished.read().await.len(), 2); - assert!(queue.existing_worker("waiting").is_none()); - } - - #[tokio::test] - async fn promotion_precheck_keeps_attachments_and_live_tasks_eligible_while_vod_is_running() { - let queue = RecordingQueue::new(); - let active = task("active", RecordingKind::Vod, RecordingTaskState::Running); - let mut duplicate = task("duplicate", RecordingKind::Vod, RecordingTaskState::Queued); - duplicate.url = active.url.clone(); - let completed = task("completed", RecordingKind::Series, RecordingTaskState::Completed); - let mut attached = task("attached", RecordingKind::Series, RecordingTaskState::Queued); - attached.url = completed.url.clone(); - queue.active.write().await.push(active); - queue.finished.write().await.push(completed); - queue.queue.lock().await.extend([duplicate, attached]); - assert!(queue.has_promotable_queued().await); - mutate(&queue, |candidate| { - assert!(promote_from_queue(candidate).is_none()); - Ok(()) - }) - .await - .expect("attach without starting a second transfer"); - assert_eq!(queue.finished.read().await.len(), 2); - assert!(!queue.has_promotable_queued().await, "the duplicate waits for its active transfer"); - queue.queue.lock().await.push_back(task("live", RecordingKind::Live, RecordingTaskState::Queued)); - assert!(queue.has_promotable_queued().await); - mutate(&queue, |candidate| { - assert_eq!(promote_from_queue(candidate).map(|(uuid, _)| uuid).as_deref(), Some("live")); - Ok(()) - }) - .await - .expect("start live alongside the transfer"); - assert_eq!(queue.active.read().await.len(), 2); - assert_eq!(queue.queue.lock().await.len(), 1); - } - - #[tokio::test] - async fn promotion_precheck_matches_promotion_for_mixed_queues() { - fn fixture(uuid: &str, media: usize, kind: RecordingKind, state: RecordingTaskState) -> RecordingTask { - let mut entry = task(&format!("media-{media}"), kind, state); - entry.uuid = uuid.to_string(); - entry.recording = media_meta("web:alice"); - entry.recording.source.virtual_id = media.to_string(); - match media % 3 { - 1 => { - entry.recording.program_start = Some(10); - entry.recording.program_end = Some(20); - } - 2 => { - entry.recording.provenance.rule_id = Some("rule".to_string()); - entry.recording.provenance.occurrence_key = Some(media.to_string()); - } - _ => {} - } - entry.paused = state == RecordingTaskState::Paused; - entry.finished = matches!( - state, - RecordingTaskState::Completed | RecordingTaskState::Failed | RecordingTaskState::Cancelled - ); - entry - } - let kinds = [RecordingKind::Live, RecordingKind::Vod, RecordingKind::Series]; - let active_states = [ - RecordingTaskState::Running, - RecordingTaskState::Paused, - RecordingTaskState::WaitingForCapacity, - RecordingTaskState::RetryWaiting, - ]; - let finished_states = - [RecordingTaskState::Completed, RecordingTaskState::Failed, RecordingTaskState::Cancelled]; - let mut random = fastrand::Rng::with_seed(0x7265_636f_7264); - let (mut promotable, mut blocked, mut attachment_only) = (0, 0, 0); - for case in 0..512 { - let queue = RecordingQueue::new(); - for index in 0..random.usize(0..12) { - queue.queue.lock().await.push_back(fixture( - &format!("queued-{index}"), - random.usize(0..6), - kinds[random.usize(0..kinds.len())], - RecordingTaskState::Queued, - )); - } - for index in 0..random.usize(0..4) { - queue.active.write().await.push(fixture( - &format!("active-{index}"), - random.usize(0..6), - kinds[random.usize(0..kinds.len())], - active_states[random.usize(0..active_states.len())], - )); - } - for index in 0..random.usize(0..6) { - queue.finished.write().await.push(fixture( - &format!("finished-{index}"), - random.usize(0..6), - kinds[random.usize(0..kinds.len())], - finished_states[random.usize(0..finished_states.len())], - )); - } - let precheck = queue.has_promotable_queued().await; - let mut snapshot = queue.snapshot_current(QueueRevision(0)).await; - let queued_before = snapshot.queue.len(); - let promoted = promote_from_queue(&mut snapshot).is_some(); - let attached = snapshot.queue.len() < queued_before; - assert_eq!(precheck, promoted || attached, "case {case}: {snapshot:?}"); - if precheck { - promotable += 1; - } else { - blocked += 1; - } - if !promoted && attached { - attachment_only += 1; - } - } - assert!(promotable > 0 && blocked > 0 && attachment_only > 0); - } - - #[tokio::test] - async fn worker_claims_do_not_create_state_for_missing_or_paused_tasks() { - let queue = RecordingQueue::new(); - assert!(queue.claim_worker("missing").await.is_none()); - let mut paused = task("paused", RecordingKind::Vod, RecordingTaskState::Paused); - paused.paused = true; - queue.active.write().await.push(paused); - assert!(queue.claim_worker("paused").await.is_none()); - assert!(queue.workers.lock().expect("worker map").is_empty()); - assert!(queue.resume_active("paused").await.expect("resume")); - assert!(queue.claim_worker("paused").await.is_some()); - assert!(queue.claim_worker("paused").await.is_none(), "a worker can be claimed only once"); - } - - #[test] - fn bulk_transfers_stay_serial_while_live_tasks_promote_in_parallel() { - let mut candidate = PersistedRecordingQueue::default(); - for index in 0..50 { - let mut transfer = - identified(&format!("transfer-{index}"), &format!("media-{index}"), RecordingTaskState::Queued); - transfer.kind = if index % 2 == 0 { RecordingKind::Vod } else { RecordingKind::Series }; - transfer.input_name = None; - candidate.queue.push(transfer); - } - for index in 0..2 { - let mut live = - identified(&format!("live-{index}"), &format!("live-media-{index}"), RecordingTaskState::Queued); - live.kind = RecordingKind::Live; - candidate.queue.push(live); - } - while promote_from_queue(&mut candidate).is_some() {} - assert_eq!(candidate.active.len(), 3); - assert_eq!(candidate.active.iter().filter(|task| task.kind != RecordingKind::Live).count(), 1); - assert_eq!(candidate.queue.len(), 49); - candidate.active.retain(|task| task.kind == RecordingKind::Live); - assert_eq!(promote_from_queue(&mut candidate).map(|(uuid, _)| uuid).as_deref(), Some("transfer-1")); - } - - fn identified(uuid: &str, identity: &str, state: RecordingTaskState) -> PersistedRecordingTask { - let mut persisted = RecordingQueue::to_persisted(&task(uuid, RecordingKind::Vod, state)); - persisted.media_identity = identity.to_owned(); - persisted - } - - #[test] - fn the_dto_offers_exactly_what_the_transition_graph_permits() { - // The frontend renders its buttons straight from this set, so a - // mismatch here is a button that errors when pressed. - for kind in [RecordingKind::Live, RecordingKind::Vod, RecordingKind::Series] { - for state in [ - RecordingTaskState::Scheduled, - RecordingTaskState::Queued, - RecordingTaskState::WaitingForCapacity, - RecordingTaskState::Running, - RecordingTaskState::Paused, - RecordingTaskState::RetryWaiting, - RecordingTaskState::Completed, - RecordingTaskState::Failed, - RecordingTaskState::Cancelled, - ] { - let dto = task("t", kind, state).to_view(true); - assert_eq!( - dto.allowed_actions, - recording_transition::allowed_actions(kind, state), - "{kind} in {} disagrees with the graph", - state.label() - ); - } - } - } - - #[tokio::test] - async fn a_transfer_runs_at_the_strongest_priority_anyone_attached_asked_for() { - // Alice's background request is producing the file; Bob wants the same - // media urgently. One transfer serves both, so it must not sit in the - // background queue while Bob waits for bytes already being fetched. - let dir = tempfile::TempDir::new().expect("tempdir"); - let queue = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open recording repository"); - let mut background = task("alice", RecordingKind::Vod, RecordingTaskState::Running); - background.priority = 5; - let mut foreground = task("bob", RecordingKind::Vod, RecordingTaskState::Queued); - foreground.priority = -3; - // Same media: a VOD identity keys on the url, so both entries name one. - foreground.url = background.url.clone(); - let (background, foreground) = - (RecordingQueue::to_persisted(&background), RecordingQueue::to_persisted(&foreground)); - assert_eq!(background.media_identity, foreground.media_identity, "fixture must share one media"); - mutate(&queue, move |candidate| { - candidate.active = vec![background.clone()]; - candidate.queue.push(foreground.clone()); - Ok(()) - }) - .await - .expect("seed"); - - let (_input, priority) = queue.active_scheduling_priority("alice").await.expect("an active transfer"); - assert_eq!(priority, -3, "the transfer inherits Bob's urgency"); - } - - #[tokio::test] - async fn an_unrelated_urgent_recording_does_not_lift_this_one() { - let dir = tempfile::TempDir::new().expect("tempdir"); - let queue = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open recording repository"); - let mut background = task("alice", RecordingKind::Vod, RecordingTaskState::Running); - background.priority = 5; - let mut other = task("bob", RecordingKind::Vod, RecordingTaskState::Queued); - other.priority = -3; - let (background, other) = (RecordingQueue::to_persisted(&background), RecordingQueue::to_persisted(&other)); - assert_ne!(background.media_identity, other.media_identity, "fixture must be different media"); - mutate(&queue, move |candidate| { - candidate.active = vec![background.clone()]; - candidate.queue.push(other.clone()); - Ok(()) - }) - .await - .expect("seed"); - - let (_input, priority) = queue.active_scheduling_priority("alice").await.expect("an active transfer"); - assert_eq!(priority, 5, "priority is shared only with entries on the same file"); - } - - #[test] - fn a_queued_entry_runs_when_nothing_else_holds_its_media() { - let candidate = PersistedRecordingQueue::default(); - let queued = identified("a", "film-42", RecordingTaskState::Queued); - assert_eq!(promotion_decision(&candidate, &queued), PromotionDecision::Execute); - } - - #[test] - fn a_queued_entry_waits_while_another_entry_produces_the_file() { - // Promoting it would start a second transfer to the same path. - let candidate = PersistedRecordingQueue { - active: vec![identified("a", "film-42", RecordingTaskState::Running)], - ..PersistedRecordingQueue::default() - }; - let queued = identified("b", "film-42", RecordingTaskState::Queued); - assert_eq!(promotion_decision(&candidate, &queued), PromotionDecision::Wait); - } - - #[test] - fn a_queued_entry_attaches_to_a_file_another_entry_finished() { - let candidate = PersistedRecordingQueue { - finished: vec![identified("a", "film-42", RecordingTaskState::Completed)], - ..PersistedRecordingQueue::default() - }; - let queued = identified("b", "film-42", RecordingTaskState::Queued); - assert_eq!(promotion_decision(&candidate, &queued), PromotionDecision::AttachTo(0)); - } - - #[test] - fn a_failed_sibling_is_not_attached_to() { - // There is no file to adopt, so the entry has to run. - let candidate = PersistedRecordingQueue { - finished: vec![identified("a", "film-42", RecordingTaskState::Failed)], - ..PersistedRecordingQueue::default() - }; - let queued = identified("b", "film-42", RecordingTaskState::Queued); - assert_eq!(promotion_decision(&candidate, &queued), PromotionDecision::Execute); - } - - #[test] - fn an_unidentified_entry_never_matches_another() { - // An empty identity means the identity could not be resolved. Treating - // two of them as the same recording would merge two different files. - let candidate = PersistedRecordingQueue { - finished: vec![identified("a", "", RecordingTaskState::Completed)], - active: vec![identified("c", "", RecordingTaskState::Running)], - ..PersistedRecordingQueue::default() - }; - let queued = identified("b", "", RecordingTaskState::Queued); - assert_eq!(promotion_decision(&candidate, &queued), PromotionDecision::Execute); - } - - #[test] - fn different_media_does_not_attach() { - let candidate = PersistedRecordingQueue { - finished: vec![identified("a", "film-42", RecordingTaskState::Completed)], - ..PersistedRecordingQueue::default() - }; - let queued = identified("b", "film-99", RecordingTaskState::Queued); - assert_eq!(promotion_decision(&candidate, &queued), PromotionDecision::Execute); - } - - #[test] - fn the_entry_behind_a_finished_transfer_attaches_instead_of_downloading_again() { - // The live path: Alice's transfer completes and Bob is next in the - // queue for the same film. Taking the queue head blindly would start a - // second transfer over the file Alice just produced. - let mut candidate = PersistedRecordingQueue { - finished: vec![identified("alice", "film-42", RecordingTaskState::Completed)], - queue: vec![identified("bob", "film-42", RecordingTaskState::Queued)], - ..PersistedRecordingQueue::default() - }; - let promoted = promote_from_queue(&mut candidate); - assert!(promoted.is_none(), "nothing needs to run"); - assert!(candidate.active.is_empty()); - assert!(candidate.queue.is_empty(), "bob left the queue"); - let bob = candidate.finished.iter().find(|task| task.uuid == "bob").expect("bob is filed"); - assert_eq!(bob.state, RecordingTaskState::Completed); - } - - #[test] - fn an_entry_for_other_media_behind_a_finished_transfer_still_runs() { - let mut candidate = PersistedRecordingQueue { - finished: vec![identified("alice", "film-42", RecordingTaskState::Completed)], - queue: vec![identified("bob", "film-99", RecordingTaskState::Queued)], - ..PersistedRecordingQueue::default() - }; - let promoted = promote_from_queue(&mut candidate); - assert_eq!(promoted.map(|(uuid, _)| uuid), Some("bob".to_string())); - assert_eq!(candidate.active.first().map(|task| task.uuid.as_str()), Some("bob")); - } - - #[test] - fn attachable_entries_are_skipped_to_reach_one_that_must_run() { - let mut candidate = PersistedRecordingQueue { - finished: vec![identified("alice", "film-42", RecordingTaskState::Completed)], - queue: vec![ - identified("bob", "film-42", RecordingTaskState::Queued), - identified("carol", "film-42", RecordingTaskState::Queued), - identified("dave", "film-99", RecordingTaskState::Queued), - ], - ..PersistedRecordingQueue::default() - }; - let promoted = promote_from_queue(&mut candidate); - assert_eq!(promoted.map(|(uuid, _)| uuid), Some("dave".to_string())); - assert!(candidate.queue.is_empty(), "every entry was dispatched"); - // Bob and Carol were filed against Alice's file without running. - assert_eq!(candidate.finished.len(), 3); - } - - #[test] - fn attaching_adopts_the_file_but_keeps_the_entry_its_own() { - let mut source = identified("a", "film-42", RecordingTaskState::Completed); - source.file_path = PathBuf::from("/rec/film.mp4"); - source.filename = "film.mp4".to_string(); - source.size = 4_096; - source.recording.measured_bytes = 4_096; - source.recording.completed_at = Some(1_700_000_000); - source.recording.relative_path = Some("film.mp4".to_string()); - - let mut attached = identified("b", "film-42", RecordingTaskState::Queued); - attached.recording.owner = RecordingOwner::User(UserId::from("web:bob")); - attached.recording.reserved_bytes = 9_999; - attach_to_completed(&mut attached, &source); - - // The bytes are shared. - assert_eq!(attached.file_path, source.file_path); - assert_eq!(attached.recording.measured_bytes, 4_096); - assert_eq!(attached.state, RecordingTaskState::Completed); - // The link is not: this is Bob's entry onto Alice's file. - assert_eq!(attached.uuid, "b"); - assert_eq!(attached.recording.owner_id().0, "web:bob"); - // Nothing is reserved any more; the bytes are already on disk. - assert_eq!(attached.recording.reserved_bytes, 0); - } - - #[tokio::test] - async fn persisted_queue_round_trips_and_requeues_running_downloads() { - let state_file = temp_state_file("download_state"); - let queue = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); - let queued = RecordingTask { - size: 10, - total_size: Some(100), - ..task("queued", RecordingKind::Vod, RecordingTaskState::Queued) - }; - let active = RecordingTask { - size: 20, - total_size: Some(200), - ..task("active", RecordingKind::Vod, RecordingTaskState::Running) - }; - let paused = RecordingTask { - size: 30, - total_size: Some(300), - paused: true, - state: RecordingTaskState::Paused, - ..task("paused", RecordingKind::Vod, RecordingTaskState::Paused) - }; - - queue.queue.lock().await.push_back(queued); - *queue.active.write().await = vec![active]; - queue.finished.write().await.push(paused.clone()); - queue.persist_to_disk().await.expect("persist state"); - - let restored = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); - restored.load_from_disk().await.expect("load state"); - - assert_eq!(restored.queue.lock().await.len(), 2); - let restored_active = restored.active.read().await.clone(); - assert!(restored_active.is_empty()); - let restored_finished = restored.finished.read().await.clone(); - assert_eq!(restored_finished.len(), 1); - assert_eq!(restored_finished[0].uuid, paused.uuid); - - let queued_items = restored.queue.lock().await.iter().map(|d| d.uuid.clone()).collect::>(); - assert!(queued_items.iter().any(|id| id == "queued")); - assert!(queued_items.iter().any(|id| id == "active")); - - let _ = std::fs::remove_dir_all(state_file); - } - - #[tokio::test] - async fn persisted_scheduled_recordings_round_trip_without_becoming_active() { - let state_file = temp_state_file("record_state"); - let queue = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); - let future_start = Utc::now().timestamp().saturating_add(3_600); - let scheduled = RecordingTask { - url: reqwest::Url::parse("https://example.com/live/1").expect("valid url"), - state: RecordingTaskState::Scheduled, - recording: live_meta("web:alice", future_start, 5_400), - ..task("recording", RecordingKind::Live, RecordingTaskState::Scheduled) - }; - - queue.scheduled.write().await.push(scheduled.clone()); - queue.persist_to_disk().await.expect("persist state"); - - let restored = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); - restored.load_from_disk().await.expect("load state"); - - assert!(restored.active.read().await.is_empty()); - assert_eq!(restored.queue.lock().await.len(), 0); - let restored_scheduled = restored.scheduled.read().await.clone(); - assert_eq!(restored_scheduled.len(), 1); - assert_eq!(restored_scheduled[0].uuid, scheduled.uuid); - assert_eq!(restored_scheduled[0].state, RecordingTaskState::Scheduled); - assert_eq!(restored_scheduled[0].scheduled_start(), Some(future_start)); - assert_eq!(restored_scheduled[0].scheduled_end(), Some(future_start + 5_400)); - assert_eq!(restored_scheduled[0].scheduled_duration_secs(), Some(5_400)); - assert_eq!(restored_scheduled[0].kind, RecordingKind::Live); - - let _ = std::fs::remove_dir_all(state_file); - } - - #[tokio::test] - async fn a_live_window_that_has_not_opened_acquires_nothing() { - // Promotion is what puts a capture in front of the provider. A window - // that starts in an hour must not reach it, or the DVR holds a slot for - // an hour of nothing and records the wrong programme. - let dir = tempfile::TempDir::new().expect("tempdir"); - let queue = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open recording repository"); - let now = 1_700_000_000; - let mut upcoming = task("live", RecordingKind::Live, RecordingTaskState::Scheduled); - upcoming.recording.scheduled_start = Some(now + 3_600); - upcoming.recording.scheduled_end = Some(now + 7_200); - let upcoming = RecordingQueue::to_persisted(&upcoming); - mutate(&queue, move |candidate| { - candidate.scheduled.push(upcoming.clone()); - Ok(()) - }) - .await - .expect("seed"); - - assert_eq!(queue.promote_due_scheduled(now).await, 0, "an hour early is not due"); - assert!(queue.queue.lock().await.is_empty(), "and nothing is queued for a provider"); - - // At the padded start it becomes work. - assert_eq!(queue.promote_due_scheduled(now + 3_600).await, 1, "due at its padded start"); - assert_eq!(queue.queue.lock().await.len(), 1); - } - - #[test] - fn an_interrupted_live_capture_fails_instead_of_restarting() { - // The broadcast that played while the process was down is gone. A - // resumed live capture would write a recording with a hole at the - // front and report it as complete. - for state in [RecordingTaskState::Running, RecordingTaskState::WaitingForCapacity] { - let interrupted = RecordingTask { state, ..task("live", RecordingKind::Live, state) }; - let recovered = RecordingQueue::recover_loaded_task(interrupted); - assert_eq!(recovered.state, RecordingTaskState::Failed, "live in {}", state.label()); - assert!(recovered.finished); - assert!(recovered.error.is_some(), "the operator is told why"); - } - } - - #[test] - fn an_interrupted_transfer_is_still_resumed() { - // The counterpart: a file on a server is still there after a restart, - // so VOD and series pick up where they left off. - for kind in [RecordingKind::Vod, RecordingKind::Series] { - let interrupted = task("film", kind, RecordingTaskState::Running); - let recovered = RecordingQueue::recover_loaded_task(interrupted); - assert_eq!(recovered.state, RecordingTaskState::Queued, "{kind} must resume"); - assert!(!recovered.finished); - } - } - - #[test] - fn a_future_live_recording_survives_a_restart_still_scheduled() { - let scheduled = task("live", RecordingKind::Live, RecordingTaskState::Scheduled); - let recovered = RecordingQueue::recover_loaded_task(scheduled); - assert_eq!(recovered.state, RecordingTaskState::Scheduled, "a window that has not opened is untouched"); - assert!(!recovered.finished); - } - - #[test] - fn recover_loaded_task_requeues_waiting_states() { - let waiting_for_capacity = RecordingTask { - size: 77, - total_size: Some(99), - error: Some("old error".to_string()), - state: RecordingTaskState::WaitingForCapacity, - ..task("capacity", RecordingKind::Vod, RecordingTaskState::WaitingForCapacity) - }; - let retry_waiting = RecordingTask { state: RecordingTaskState::RetryWaiting, ..waiting_for_capacity.clone() }; - - let restored_waiting_for_capacity = RecordingQueue::recover_loaded_task(waiting_for_capacity); - let restored_retry_waiting = RecordingQueue::recover_loaded_task(retry_waiting); - - assert_eq!(restored_waiting_for_capacity.state, RecordingTaskState::Queued); - assert!(!restored_waiting_for_capacity.paused); - assert!(restored_waiting_for_capacity.error.is_none()); - - assert_eq!(restored_retry_waiting.state, RecordingTaskState::Queued); - assert!(!restored_retry_waiting.paused); - assert!(restored_retry_waiting.error.is_none()); - } - - #[test] - fn recover_loaded_task_clears_pending_retry_timestamp() { - let retry_waiting = RecordingTask { - size: 12, - total_size: Some(20), - error: Some("retrying".to_string()), - state: RecordingTaskState::RetryWaiting, - retry_attempts: 2, - next_retry_at: Some(1_700_000_000), - ..task("retry", RecordingKind::Vod, RecordingTaskState::RetryWaiting) - }; - - let restored = RecordingQueue::recover_loaded_task(retry_waiting); - assert_eq!(restored.state, RecordingTaskState::Queued); - assert_eq!(restored.retry_attempts, 0); - assert!(restored.next_retry_at.is_none()); - } - - #[tokio::test] - async fn retry_finished_clears_retry_metadata() { - let queue = RecordingQueue::new(); - queue.finished.write().await.push(RecordingTask { - finished: true, - error: Some("Retry limit reached".to_string()), - state: RecordingTaskState::Failed, - retry_attempts: 5, - next_retry_at: Some(1_700_000_000), - ..task("done", RecordingKind::Vod, RecordingTaskState::Failed) - }); - - assert!(queue.retry_finished("done").await.expect("retry finished")); - let queued = queue.queue.lock().await.front().cloned().expect("queued download"); - assert_eq!(queued.state, RecordingTaskState::Queued); - assert_eq!(queued.retry_attempts, 0); - assert!(queued.next_retry_at.is_none()); - assert!(queued.error.is_none()); - } - - #[tokio::test] - async fn retry_finished_rejects_recordings() { - let queue = RecordingQueue::new(); - queue.finished.write().await.push(RecordingTask { - finished: true, - error: Some("Cancelled by user".to_string()), - state: RecordingTaskState::Cancelled, - recording: live_meta("web:alice", 1_700_000_000, 300), - ..task("recording", RecordingKind::Live, RecordingTaskState::Cancelled) - }); - - assert!(!queue.retry_finished("recording").await.expect("reject recording retry")); - assert!(queue.queue.lock().await.is_empty()); - assert_eq!(queue.finished.read().await.len(), 1); - } - - #[tokio::test] - async fn promote_due_scheduled_moves_only_ready_recordings_to_queue() { - let queue = RecordingQueue::new(); - let due = RecordingTask { - size: 123, - total_size: Some(999), - error: Some("old error".to_string()), - state: RecordingTaskState::Scheduled, - recording: live_meta("web:alice", 100, 60), - ..task("due", RecordingKind::Live, RecordingTaskState::Scheduled) - }; - let future = RecordingTask { - state: RecordingTaskState::Scheduled, - recording: live_meta("web:alice", 200, 60), - ..task("future", RecordingKind::Live, RecordingTaskState::Scheduled) - }; - - queue.scheduled.write().await.extend([due, future]); - let revision = queue.revision.load(Ordering::SeqCst); - - let promoted = queue.promote_due_scheduled(150).await; - - assert_eq!(promoted, 1); - assert_eq!(queue.revision.load(Ordering::SeqCst), revision + 1); - let queued_items = queue.queue.lock().await.iter().cloned().collect::>(); - assert_eq!(queued_items.len(), 1); - assert_eq!(queued_items[0].uuid, "due"); - assert_eq!(queued_items[0].state, RecordingTaskState::Queued); - assert_eq!(queued_items[0].size, 0); - assert!(queued_items[0].error.is_none()); - let scheduled_items = queue.scheduled.read().await.clone(); - assert_eq!(scheduled_items.len(), 1); - assert_eq!(scheduled_items[0].uuid, "future"); - } - - #[tokio::test] - async fn promote_due_scheduled_marks_expired_recordings_failed() { - let queue = RecordingQueue::new(); - let expired = RecordingTask { - state: RecordingTaskState::Scheduled, - recording: live_meta("web:alice", 100, 60), - ..task("expired", RecordingKind::Live, RecordingTaskState::Scheduled) - }; - - queue.scheduled.write().await.push(expired); - let promoted = queue.promote_due_scheduled(200).await; - - assert_eq!(promoted, 1); - assert!(queue.queue.lock().await.is_empty()); - let finished = queue.finished.read().await.clone(); - assert_eq!(finished.len(), 1); - assert_eq!(finished[0].uuid, "expired"); - assert_eq!(finished[0].state, RecordingTaskState::Failed); - assert!(finished[0].finished); - assert_eq!(finished[0].error.as_deref(), Some("Recording window already expired")); - } - - #[tokio::test] - async fn load_from_disk_moves_expired_scheduled_recordings_to_finished() { - let state_file = temp_state_file("expired_record_state"); - let queue = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); - let expired = RecordingTask { - state: RecordingTaskState::Scheduled, - recording: live_meta("web:alice", 100, 60), - ..task("expired", RecordingKind::Live, RecordingTaskState::Scheduled) - }; - - queue.scheduled.write().await.push(expired); - queue.persist_to_disk().await.expect("persist state"); - - let restored = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); - restored.load_from_disk().await.expect("load state"); - - assert!(restored.scheduled.read().await.is_empty()); - let finished = restored.finished.read().await.clone(); - assert_eq!(finished.len(), 1); - assert_eq!(finished[0].uuid, "expired"); - assert_eq!(finished[0].state, RecordingTaskState::Failed); - assert_eq!(finished[0].error.as_deref(), Some("Recording window already expired")); - - let _ = std::fs::remove_dir_all(state_file); - } - - #[test] - fn recording_uuid_differs_for_same_url_with_different_start_times() { - let cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { - directory: Some("/tmp".to_string()), - ..Default::default() - }); - - let first = RecordingTask::new( - RecordingKind::Live, - "https://example.com/live/1", - "recording_1.ts", - &cfg, - None, - 0, - live_meta("web:alice", 1_700_000_000, 5_400), - ) - .expect("first recording"); - let second = RecordingTask::new( - RecordingKind::Live, - "https://example.com/live/1", - "recording_2.ts", - &cfg, - None, - 0, - live_meta("web:alice", 1_700_005_400, 5_400), - ) - .expect("second recording"); - - assert_ne!(first.uuid, second.uuid); - } - - #[test] - fn download_uuid_differs_for_same_url_with_different_filenames() { - let cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { - directory: Some("/tmp".to_string()), - ..Default::default() - }); - - let first = RecordingTask::new( - RecordingKind::Vod, - "https://example.com/video.mp4", - "first.mp4", - &cfg, - None, - 0, - media_meta("web:alice"), - ) - .expect("first download"); - let second = RecordingTask::new( - RecordingKind::Vod, - "https://example.com/video.mp4", - "second.mp4", - &cfg, - None, - 0, - media_meta("web:alice"), - ) - .expect("second download"); - - assert_ne!(first.uuid, second.uuid); - } - - #[test] - fn a_name_taken_on_disk_is_numbered_from_the_original_stem() { - // Collision suffixes number the original stem: the third copy is - // `film_2.mp4`, not `film_1_2.mp4`. - let dir = tempfile::TempDir::new().expect("tempdir"); - std::fs::write(dir.path().join("film.mp4"), b"").expect("first copy"); - std::fs::write(dir.path().join("film_1.mp4"), b"").expect("second copy"); - let cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { - directory: Some(dir.path().to_string_lossy().into_owned()), - ..Default::default() - }); - - let task = RecordingTask::new( - RecordingKind::Vod, - "https://example.com/film.mp4", - "film.mp4", - &cfg, - None, - 0, - media_meta("web:alice"), - ) - .expect("a free name"); - - assert_eq!(task.filename, "film_2.mp4"); - } - - #[test] - fn download_new_omits_trailing_dot_when_filename_has_no_extension() { - let cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { - directory: Some("/tmp".to_string()), - ..Default::default() - }); - - let task = RecordingTask::new( - RecordingKind::Vod, - "https://example.com/live", - "title with trailing dot.", - &cfg, - None, - 0, - media_meta("web:alice"), - ) - .expect("download"); - - assert_eq!(task.filename, "title_with_trailing_dot"); - assert!(!task.filename.ends_with('.')); - } - - fn organized_task(kind: RecordingKind, filename: &str, recording: RecordingMetadata) -> RecordingTask { - let dir = tempfile::tempdir().expect("tempdir"); - let cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { - directory: Some(dir.path().to_string_lossy().into_owned()), - organize_into_directories: true, - ..Default::default() - }); - RecordingTask::new(kind, "https://example.com/item", filename, &cfg, None, 0, recording).expect("task") - } - - #[test] - fn organized_live_recording_lands_in_its_group() { - let mut meta = live_meta("web:alice", 1_700_000_000, 1_800); - meta.channel_name = Some("SBS 6 HD".to_string()); - meta.group = Some("NEDERLAND".to_string()); - let task = organized_task(RecordingKind::Live, "news.ts", meta); - assert_eq!(task.recording.relative_path.as_deref(), Some("NEDERLAND/news.ts")); - } - - #[test] - fn task_filename_keeps_extension_dots_and_letters_of_every_script() { - let mut meta = live_meta("web:alice", 1_700_000_000, 1_800); - meta.group = Some("Новости".to_string()); - let task = organized_task(RecordingKind::Live, "Mr. Robot — Çalıkuşu 🎬.ts", meta); - assert_eq!(task.recording.relative_path.as_deref(), Some("Новости/Mr._Robot_Çalıkuşu.ts")); - } - - #[test] - fn organized_series_episodes_share_the_series_folder_in_their_group() { - let episode = |title: &str| { - let mut meta = media_meta("web:alice"); - meta.program_title = Some(title.to_string()); - meta.series_name = Some("The Show".to_string()); - meta.group = Some("Crime".to_string()); - meta - }; - let first = organized_task(RecordingKind::Series, "The Show S01E01.mkv", episode("Pilot")); - let second = organized_task(RecordingKind::Series, "The Show S02E05.mkv", episode("Finale")); - assert_eq!(first.recording.relative_path.as_deref(), Some("Crime/The Show/The_Show_S01E01.mkv")); - assert_eq!(second.recording.relative_path.as_deref(), Some("Crime/The Show/The_Show_S02E05.mkv")); - } - - #[tokio::test] - async fn promote_due_scheduled_places_due_recordings_ahead_of_existing_queue_items() { - let queue = RecordingQueue::new(); - queue.queue.lock().await.push_back(task("existing", RecordingKind::Vod, RecordingTaskState::Queued)); - queue.scheduled.write().await.extend([ - RecordingTask { - state: RecordingTaskState::Scheduled, - recording: live_meta("web:alice", 100, 60), - ..task("due-first", RecordingKind::Live, RecordingTaskState::Scheduled) - }, - RecordingTask { - state: RecordingTaskState::Scheduled, - recording: live_meta("web:alice", 110, 60), - ..task("due-second", RecordingKind::Live, RecordingTaskState::Scheduled) - }, - ]); - - let promoted = queue.promote_due_scheduled(150).await; - - assert_eq!(promoted, 2); - let queued = queue.queue.lock().await.iter().map(|download| download.uuid.clone()).collect::>(); - assert_eq!(queued, vec!["due-first", "due-second", "existing"]); - } - - #[tokio::test] - async fn download_slot_wait_queue_signals_matching_waiter_by_id() { - let queue = Arc::new(RecordingSlotWaitQueue::new()); - let control_signal = Arc::new(RwLock::new(RecordingControl::None)); - let control_notify = Arc::new(Notify::new()); - - let queue_for_a = Arc::clone(&queue); - let control_signal_for_a = Arc::clone(&control_signal); - let control_notify_for_a = Arc::clone(&control_notify); - let waiter_a = tokio::spawn(async move { - queue_for_a - .wait(Some(Arc::from("input-a")), 1, control_signal_for_a.as_ref(), control_notify_for_a.as_ref()) - .await - }); - - let queue_for_b = Arc::clone(&queue); - let control_signal_for_b = Arc::clone(&control_signal); - let control_notify_for_b = Arc::clone(&control_notify); - let waiter_b = tokio::spawn(async move { - queue_for_b - .wait(Some(Arc::from("input-b")), 0, control_signal_for_b.as_ref(), control_notify_for_b.as_ref()) - .await - }); - - let waiter_b_id = loop { - let snapshots = queue.snapshots(); - if snapshots.len() == 2 { - break snapshots - .into_iter() - .find(|waiter| waiter.input_name.as_deref() == Some("input-b")) - .map(|waiter| waiter.id) - .expect("waiter id for input-b"); - } - tokio::task::yield_now().await; - }; - - assert!(queue.signal_waiter(waiter_b_id)); - assert_eq!( - timeout(Duration::from_millis(100), waiter_b).await.expect("waiter_b finished").expect("join ok"), - RecordingWaitOutcome::Signalled - ); - - *control_signal.write().await = RecordingControl::Cancel; - control_notify.notify_waiters(); - assert_eq!( - timeout(Duration::from_millis(100), waiter_a).await.expect("waiter_a finished").expect("join ok"), - RecordingWaitOutcome::Cancelled - ); - } - - #[tokio::test] - async fn request_worker_restart_sets_restart_control_and_notifies_waiters() { - let queue = RecordingQueue::new(); - let worker = queue.worker("task"); - let waiter_queue = Arc::clone(&queue.slot_waiters); - let control_signal = Arc::clone(&worker.control_signal); - let control_notify = Arc::clone(&worker.control_notify); - - let waiter = - tokio::spawn( - async move { waiter_queue.wait(None, 0, control_signal.as_ref(), control_notify.as_ref()).await }, - ); - - tokio::task::yield_now().await; - queue.request_worker_restart(); - - assert_eq!( - timeout(Duration::from_millis(100), waiter).await.expect("waiter finished").expect("join ok"), - RecordingWaitOutcome::Restarted - ); - assert_eq!(*worker.control_signal.read().await, RecordingControl::Restart); - } - - /// Age the first recovery generation so the next commit wants a - /// checkpoint, and block the directory that checkpoint would be written - /// to: the checkpoint fails after the commit itself succeeded. - fn force_a_failing_checkpoint(dir: &Path) { - let recovery = dir.join("recordings_library_recovery"); - let manifest = recovery.join("gen-00000000000000000001/manifest.bin"); - let bytes = std::fs::read(&manifest).expect("manifest"); - let mut payload: serde_json::Value = serde_json::from_slice(&bytes[40..]).expect("manifest payload"); - payload["created_at_unix"] = serde_json::json!(0); - let payload = serde_json::to_vec(&payload).expect("encode"); - let mut framed = b"TRJ1".to_vec(); - framed.extend_from_slice(&u32::try_from(payload.len()).expect("length").to_le_bytes()); - framed.extend_from_slice(blake3::hash(&payload).as_bytes()); - framed.extend_from_slice(&payload); - std::fs::write(&manifest, framed).expect("rewrite manifest"); - std::fs::write(recovery.join("gen-00000000000000000002"), b"blocks the next generation").expect("block"); - } - - #[tokio::test] - async fn a_commit_whose_checkpoint_fails_is_kept_in_memory_and_on_disk() { - // The checkpoint runs after the batch is durable. The queue keeps the - // committed record, and the next commit does not delete it again. - let dir = tempfile::TempDir::new().expect("tempdir"); - drop(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - force_a_failing_checkpoint(dir.path()); - let queue = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("reopen repository"); - - let committed = RecordingQueue::to_persisted(&task("kept", RecordingKind::Vod, RecordingTaskState::Completed)); - mutate(&queue, move |candidate| { - candidate.finished.push(committed.clone()); - Ok(()) - }) - .await - .expect("the commit succeeds although its checkpoint does not"); - assert_eq!(queue.finished.read().await.len(), 1); - - // A second commit diffs against that state rather than deleting it. - let another = RecordingQueue::to_persisted(&task("next", RecordingKind::Vod, RecordingTaskState::Completed)); - mutate(&queue, move |candidate| { - candidate.finished.push(another.clone()); - Ok(()) - }) - .await - .expect("second commit"); - drop(queue); - - let reopened = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("reopen repository"); - reopened.load_from_disk().await.expect("load"); - let mut uuids: Vec = reopened.finished.read().await.iter().map(|task| task.uuid.clone()).collect(); - uuids.sort(); - assert_eq!(uuids, ["kept", "next"]); - } - - #[tokio::test] - async fn a_second_request_on_an_accepted_key_is_refused_and_leaves_the_key_alone() { - let dir = tempfile::TempDir::new().expect("tempdir"); - let queue = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository"); - let record = |fingerprint: &str, recording_id: &str| PersistedIdempotency { - principal: "web:alice".to_string(), - key: "same-key".to_string(), - request_fingerprint: fingerprint.to_string(), - recording_id: recording_id.to_string(), - accepted_at: Utc::now().timestamp(), - }; - - let first = RecordingQueue::to_persisted(&task("first", RecordingKind::Vod, RecordingTaskState::Queued)); - mutate_with_idempotency(&queue, record("first-body", "first"), move |candidate| { - candidate.queue.push(first.clone()); - Ok(()) - }) - .await - .expect("first request"); - - let second = RecordingQueue::to_persisted(&task("second", RecordingKind::Vod, RecordingTaskState::Queued)); - let refused = mutate_with_idempotency(&queue, record("different-body", "second"), move |candidate| { - candidate.queue.push(second.clone()); - Ok(()) - }) - .await; - assert!(matches!(refused, Err(QueueMutationError::IdempotencyConflict)), "got {refused:?}"); - assert_eq!(queue.queue.lock().await.len(), 1, "the second request admitted nothing"); - assert_eq!( - queue.lookup_idempotency("web:alice", "same-key", "first-body").await.expect("lookup"), - IdempotencyOutcome::Replay { recording_id: "first".to_string() }, - "the accepted key still answers for the first request" - ); - - let replay = RecordingQueue::to_persisted(&task("again", RecordingKind::Vod, RecordingTaskState::Queued)); - let replayed = mutate_with_idempotency(&queue, record("first-body", "again"), move |candidate| { - candidate.queue.push(replay.clone()); - Ok(()) - }) - .await; - assert!( - matches!(&replayed, Err(QueueMutationError::IdempotentReplay { recording_id }) if recording_id == "first"), - "got {replayed:?}" - ); - } - - #[tokio::test] - async fn a_restart_never_replaces_a_pending_pause_or_cancel() { - // A configuration reload must not turn a user's pause into a requeue - // or swallow a cancel, whichever of the two writes lands first. - for pending in [RecordingControl::Pause, RecordingControl::Cancel] { - let queue = RecordingQueue::new(); - let worker = queue.worker("task"); - *worker.control_signal.write().await = pending; - queue.request_worker_restart(); - assert_eq!(*worker.control_signal.read().await, pending); - - // The deferred path, taken while another writer holds the lock. - let held = worker.control_signal.write().await; - queue.request_worker_restart(); - drop(held); - tokio::task::yield_now().await; - tokio::task::yield_now().await; - assert_eq!(*worker.control_signal.read().await, pending); - } - } - - #[tokio::test] - async fn wait_observes_preexisting_control_before_selecting() { - let queue = RecordingQueue::new(); - let worker = queue.worker("task"); - *worker.control_signal.write().await = RecordingControl::Pause; - - let outcome = - queue.slot_waiters.wait(None, 0, worker.control_signal.as_ref(), worker.control_notify.as_ref()).await; - - assert_eq!(outcome, RecordingWaitOutcome::Paused); - assert!(queue.slot_waiters.snapshots().is_empty()); - } - - #[test] - fn from_persisted_rejects_invalid_url() { - let mut p = RecordingQueue::to_persisted(&task("bad-url", RecordingKind::Vod, RecordingTaskState::Queued)); - p.url = "not a url at all".to_string(); - let result = RecordingQueue::from_persisted(p); - assert!(result.is_err(), "invalid url must surface as an error"); - assert!(matches!(result.unwrap_err(), PersistedError::InvalidUrl(_)), "must surface the parse error"); - } - - // --- Transactional queue mutation boundary --- - - fn make_test_recording_task(uuid: &str, file_path: PathBuf) -> RecordingTask { - make_test_task_of_kind(uuid, file_path, RecordingKind::Live) - } - - /// Pause/resume are VOD/Series-only, so those tests need a resumable task. - fn make_test_transfer_task(uuid: &str, file_path: PathBuf) -> RecordingTask { - make_test_task_of_kind(uuid, file_path, RecordingKind::Vod) - } - - fn make_test_task_of_kind(uuid: &str, file_path: PathBuf, kind: RecordingKind) -> RecordingTask { - let mut task = task(uuid, kind, RecordingTaskState::Running); - task.file_dir = file_path.parent().unwrap_or(Path::new("/")).to_path_buf(); - task.file_path = file_path; - task.filename = format!("{uuid}.ts"); - task - } - - /// Read back what the repository actually committed, rather than - /// trusting the in-memory mirror. - async fn committed(queue: &RecordingQueue) -> (u64, Vec) { - let repository = queue.repository.clone().expect("queue is repository backed"); - let snapshot = tokio::task::spawn_blocking(move || { - let mut guard = repository.lock().expect("repository lock"); - guard.load() - }) - .await - .expect("join") - .expect("load"); - (snapshot.queue_revision, snapshot.tasks.iter().map(|task| task.uuid.clone()).collect()) - } - - #[tokio::test] - async fn mutate_persists_and_increments_revision_on_success() { - let dir = tempfile::tempdir().expect("tempdir"); - let state_dir = dir.path().to_path_buf(); - let queue = RecordingQueue::new_persistent(&state_dir, &state_dir).expect("open recording repository"); - assert_eq!(queue.revision.load(Ordering::SeqCst), 0); - - // Insert one recording so a candidate is non-empty. - let task = make_test_recording_task("rec-1", dir.path().join("a.ts")); - queue.queue.lock().await.push_back(task); - - let result: Result<(), QueueMutationError> = mutate(&queue, |_candidate| Ok(())).await; - assert!(result.is_ok(), "mutate should succeed: {result:?}"); - // The first committed mutation publishes revision 1, both in - // memory and in the repository. - assert_eq!(queue.revision.load(Ordering::SeqCst), 1, "counter must store the new value"); - let (revision, uuids) = committed(&queue).await; - assert_eq!(revision, 1, "repository carries the candidate's revision"); - assert_eq!(uuids, vec!["rec-1".to_string()]); - } - - #[tokio::test] - async fn mutate_keeps_state_when_closure_errors() { - let dir = tempfile::tempdir().expect("tempdir"); - let state_dir = dir.path().to_path_buf(); - let queue = RecordingQueue::new_persistent(&state_dir, &state_dir).expect("open recording repository"); - let original_revision = queue.revision.load(Ordering::SeqCst); - let original_len = queue.queue.lock().await.len(); - - let result: Result<(), QueueMutationError> = - mutate(&queue, |_candidate| Err(QueueMutationError::new("validation failed"))).await; - assert!(result.is_err(), "closure error should propagate"); - assert_eq!(queue.queue.lock().await.len(), original_len, "queue must stay unchanged"); - assert_eq!(queue.revision.load(Ordering::SeqCst), original_revision); - let (revision, uuids) = committed(&queue).await; - assert_eq!(revision, 0, "nothing may be committed on closure error"); - assert_eq!(uuids.len(), 0); - } - - #[tokio::test] - async fn mutate_invalid_candidate_keeps_existing_repository_memory_and_revision() { - let dir = tempfile::tempdir().expect("tempdir"); - let state_dir = dir.path().to_path_buf(); - let queue = RecordingQueue::new_persistent(&state_dir, &state_dir).expect("open recording repository"); - let initial = make_test_recording_task("rec-1", dir.path().join("a.ts")); - let persisted = RecordingQueue::to_persisted(&initial); - mutate(&queue, |candidate| { - candidate.queue.push(persisted); - Ok(()) - }) - .await - .expect("initial commit"); - let committed_before = committed(&queue).await; - - let result: Result<(), QueueMutationError> = mutate(&queue, |candidate| { - if let Some(download) = candidate.queue.first_mut() { - download.url = "not a url".to_string(); - } - Ok(()) - }) - .await; - - assert!(result.is_err()); - assert_eq!(queue.revision.load(Ordering::SeqCst), 1); - // A candidate that cannot be converted back is rejected before the - // repository is touched, so the committed set is byte-identical. - assert_eq!(committed(&queue).await, committed_before); - assert_eq!(queue.queue.lock().await.front().map(|download| download.uuid.as_str()), Some("rec-1")); - } - - #[tokio::test] - async fn control_uuid_mismatch_is_noop_and_preserves_next_task() { - let queue = RecordingQueue::new(); - let worker = queue.worker("active"); - *queue.active.write().await = vec![make_test_recording_task("active", PathBuf::from("/tmp/active.ts"))]; - queue.queue.lock().await.push_back(make_test_recording_task("next", PathBuf::from("/tmp/next.ts"))); - - assert!(!queue.pause_active("next").await.expect("uuid mismatch")); - - assert_eq!(queue.revision.load(Ordering::SeqCst), 0); - assert_eq!(queue.active.read().await.first().map(|download| download.uuid.as_str()), Some("active")); - assert_eq!(queue.queue.lock().await.front().map(|download| download.uuid.as_str()), Some("next")); - assert_eq!(*worker.control_signal.read().await, RecordingControl::None); - } - - #[tokio::test] - async fn retry_finished_after_failed_state_commits_one_revision() { - let queue = RecordingQueue::new(); - let finished = RecordingTask { - finished: true, - state: RecordingTaskState::Failed, - ..task("done", RecordingKind::Vod, RecordingTaskState::Failed) - }; - queue.finished.write().await.push(finished); - - assert!(queue.retry_finished("done").await.expect("retry commit")); - - assert_eq!(queue.revision.load(Ordering::SeqCst), 1); - assert!(queue.finished.read().await.is_empty()); - assert_eq!(queue.queue.lock().await.front().map(|download| download.uuid.as_str()), Some("done")); - } - - #[tokio::test] - async fn concurrent_writers_serialize_and_increment_each_revision() { - let queue = Arc::new(RecordingQueue::new()); - queue.queue.lock().await.push_back(make_test_recording_task("remove", PathBuf::from("/tmp/remove.ts"))); - let finished = RecordingTask { - finished: true, - state: RecordingTaskState::Failed, - ..task("retry", RecordingKind::Vod, RecordingTaskState::Failed) - }; - queue.finished.write().await.push(finished); - - let remove_queue = Arc::clone(&queue); - let retry_queue = Arc::clone(&queue); - let (removed, retried) = - tokio::join!(async move { remove_queue.remove_from_queue("remove").await }, async move { - retry_queue.retry_finished("retry").await - },); - - assert!(removed.expect("remove commit")); - assert!(retried.expect("retry commit")); - assert_eq!(queue.revision.load(Ordering::SeqCst), 2); - assert_eq!(queue.queue.lock().await.front().map(|download| download.uuid.as_str()), Some("retry")); - assert!(queue.finished.read().await.is_empty()); - } - - #[tokio::test] - async fn mutate_serializes_concurrent_calls() { - let dir = tempfile::tempdir().expect("tempdir"); - let state_file = dir.path().to_path_buf(); - let queue = std::sync::Arc::new( - RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"), - ); - let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(5)); - - let mut handles = Vec::new(); - for _ in 0..5 { - let q = std::sync::Arc::clone(&queue); - let b = std::sync::Arc::clone(&barrier); - handles.push(tokio::spawn(async move { - b.wait().await; - mutate(&q, |_candidate| Ok(())).await - })); - } - for h in handles { - assert!(h.await.expect("join").is_ok(), "all mutations should succeed"); - } - let final_rev = queue.revision.load(Ordering::SeqCst); - assert_eq!(final_rev, 5); - } - - #[tokio::test] - async fn committed_snapshot_waits_for_mutation_boundary() { - let queue = std::sync::Arc::new(RecordingQueue::new()); - let task = make_test_recording_task("rec-1", PathBuf::from("/tmp/rec-1.ts")); - queue.queue.lock().await.push_back(task); - - let mutation_guard = queue.mutation_guard.lock().await; - let snapshot_queue = std::sync::Arc::clone(&queue); - let snapshot = tokio::spawn(async move { snapshot_queue.committed_snapshot().await }); - - tokio::task::yield_now().await; - assert!(!snapshot.is_finished(), "snapshot must wait for the mutation boundary"); - - drop(mutation_guard); - let Ok((revision, tasks)) = snapshot.await else { - unreachable!("snapshot task failed"); - }; - assert_eq!(revision, QueueRevision(0)); - assert_eq!(tasks.len(), 1); - assert_eq!(tasks.first().map(|task| task.uuid.as_str()), Some("rec-1")); - } - - #[tokio::test] - async fn committed_partitioned_snapshot_waits_for_mutation_boundary() { - let queue = Arc::new(RecordingQueue::new()); - let mutation_guard = queue.mutation_guard.lock().await; - let snapshot_queue = Arc::clone(&queue); - let snapshot = tokio::spawn(async move { snapshot_queue.committed_partitioned_snapshot().await }); - - tokio::task::yield_now().await; - assert!(!snapshot.is_finished()); - - drop(mutation_guard); - let Ok((queued, active, finished)) = snapshot.await else { - unreachable!("snapshot task failed"); - }; - assert!(queued.is_empty()); - assert!(active.is_empty()); - assert!(finished.is_empty()); - } - - #[tokio::test] - async fn control_signal_is_ordered_inside_mutation_guard() { - let queue = Arc::new(RecordingQueue::new()); - let worker = queue.worker("active"); - *queue.active.write().await = vec![make_test_transfer_task("active", PathBuf::from("/tmp/active.ts"))]; - let control_lock = worker.control_signal.write().await; - let pause_queue = Arc::clone(&queue); - let pause = tokio::spawn(async move { pause_queue.pause_active("active").await }); - - for _ in 0..100 { - if queue.revision.load(Ordering::SeqCst) == 1 { - break; - } - tokio::task::yield_now().await; - } - assert_eq!(queue.revision.load(Ordering::SeqCst), 1); - let snapshot_queue = Arc::clone(&queue); - let snapshot = tokio::spawn(async move { snapshot_queue.committed_snapshot().await }); - tokio::task::yield_now().await; - assert!(!snapshot.is_finished(), "mutation guard must remain held until control publication"); - - drop(control_lock); - assert!(pause.await.is_ok_and(|result| result.is_ok())); - assert!(snapshot.await.is_ok()); - assert_eq!(*worker.control_signal.read().await, RecordingControl::Pause); - } - - #[tokio::test] - async fn same_value_control_is_published_after_worker_commit_and_clear() { - let queue = Arc::new(RecordingQueue::new()); - let worker = queue.worker("task-a"); - *queue.active.write().await = vec![make_test_recording_task("task-a", PathBuf::from("/tmp/task-a.ts"))]; - queue.queue.lock().await.push_back(make_test_recording_task("task-b", PathBuf::from("/tmp/task-b.ts"))); - let mut control_lock = worker.control_signal.write().await; - *control_lock = RecordingControl::Cancel; - let worker_queue = Arc::clone(&queue); - let worker_commit = tokio::spawn(async move { - worker_queue - .mutate_optional_and_clear_control("task-a", RecordingControl::Cancel, |candidate| { - let Some(mut active) = candidate.active.pop() else { - return Ok(None); - }; - active.finished = true; - candidate.finished.push(active); - if !candidate.queue.is_empty() { - candidate.active = vec![candidate.queue.remove(0)]; - } - Ok(Some(true)) - }) - .await - }); - - for _ in 0..100 { - if queue.revision.load(Ordering::SeqCst) == 1 { - break; - } - tokio::task::yield_now().await; - } - assert_eq!(queue.revision.load(Ordering::SeqCst), 1); - let api_queue = Arc::clone(&queue); - let newer_cancel = tokio::spawn(async move { api_queue.cancel_requested("task-b").await }); - tokio::task::yield_now().await; - assert!(!newer_cancel.is_finished()); - - drop(control_lock); - - assert!(worker_commit.await.expect("worker task").expect("worker commit").is_some()); - assert_eq!(newer_cancel.await.expect("cancel task").expect("cancel commit"), Some(false)); - assert_eq!(*queue.worker("task-b").control_signal.read().await, RecordingControl::Cancel); - } - - #[tokio::test] - async fn mutate_swap_restores_in_memory_state_from_persisted_candidate() { - let dir = tempfile::tempdir().expect("tempdir"); - let state_file = dir.path().to_path_buf(); - let queue = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); - - let persisted = RecordingQueue::to_persisted(&make_test_recording_task("rec-1", dir.path().join("a.ts"))); - mutate(&queue, |candidate| { - candidate.queue.push(persisted); - Ok(()) - }) - .await - .expect("first mutate"); - - let len_after_commit = queue.queue.lock().await.len(); - assert_eq!(len_after_commit, 1, "committed task must be in memory"); - - // Now mutate again to remove that task. The candidate should be - // re-built from the just-committed state, not from the empty - // pre-mutate in-memory state. - mutate(&queue, |candidate| { - candidate.queue.retain(|d| d.uuid != "rec-1"); - Ok(()) - }) - .await - .expect("second mutate"); - - assert!(queue.queue.lock().await.is_empty(), "remove must propagate to in-memory state"); - assert_eq!(queue.revision.load(Ordering::SeqCst), 2); - } - - // --- load_from_disk rejection paths --- - - #[tokio::test] - async fn load_from_disk_rejects_invalid_url_in_persisted_entry() { - let dir = tempfile::tempdir().expect("tempdir"); - let state_dir = dir.path().to_path_buf(); - let mut persisted = - RecordingQueue::to_persisted(&task("rec-1", RecordingKind::Vod, RecordingTaskState::Queued)); - persisted.url = "not a url".to_string(); - - let queue = RecordingQueue::new_persistent(&state_dir, &state_dir).expect("open recording repository"); - queue.commit_records(1, vec![persisted], None).await.expect("commit unparseable record"); - - let result = queue.load_from_disk().await; - - assert!(result.is_err(), "invalid url must surface as an error"); - assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidData); - // Rejection happens before assignment, so memory stays empty and the - // repository is left for an operator to inspect. - assert!(queue.queue.lock().await.is_empty()); - assert!(queue.scheduled.read().await.is_empty()); - assert!(queue.finished.read().await.is_empty()); - assert!(queue.active.read().await.is_empty()); - } - - // --- Filename rendering + collision reservation --- - - fn collect_existing_relative_paths(candidate: &PersistedRecordingQueue) -> Vec { - let mut out = Vec::new(); - for d in &candidate.queue { - if let Some(p) = &d.recording.relative_path { - out.push(p.clone()); - } - } - for d in &candidate.scheduled { - if let Some(p) = &d.recording.relative_path { - out.push(p.clone()); - } - } - for d in &candidate.active { - if let Some(p) = &d.recording.relative_path { - out.push(p.clone()); - } - } - for d in &candidate.finished { - if let Some(p) = &d.recording.relative_path { - out.push(p.clone()); - } - } - out - } - - /// Reserve a unique relative path for a new recording inside the - /// queue mutation boundary. The candidate is the in-memory - /// `PersistedRecordingQueue` the closure is building; the helper - /// collects all already-reserved `relative_path` values, applies - /// the supplied stem, and appends a numbered collision suffix - /// (`_1`, `_2`, …) until the result is unique. The reserved path - /// is also written to the recording metadata so a later collision - /// created externally is detected by the worker at execute time. - fn reserve_recording_relative_path( - candidate: &mut PersistedRecordingQueue, - stem: &str, - recording_uuid: &str, - ) -> String { - // If this recording already has a reserved path (e.g., a retry - // or an edit), keep it. Re-reserving on a re-entrant call must - // not bump the suffix because of the caller's own previous - // entry. - let existing_self = find_recording_relative_path(candidate, recording_uuid); - if let Some(prior) = existing_self { - return prior; - } - let existing = collect_existing_relative_paths(candidate); - let reserved = shared::utils::next_collision_suffix(stem, &existing); - for d in &mut candidate.queue { - if d.uuid == recording_uuid { - d.recording.relative_path = Some(reserved.clone()); - } - } - for d in &mut candidate.scheduled { - if d.uuid == recording_uuid { - d.recording.relative_path = Some(reserved.clone()); - } - } - if let Some(d) = candidate.active.iter_mut().find(|task| task.uuid == recording_uuid) { - d.recording.relative_path = Some(reserved.clone()); - } - for d in &mut candidate.finished { - if d.uuid == recording_uuid { - d.recording.relative_path = Some(reserved.clone()); - } - } - reserved - } - - fn find_recording_relative_path(candidate: &PersistedRecordingQueue, recording_uuid: &str) -> Option { - for d in &candidate.queue { - if d.uuid == recording_uuid { - return d.recording.relative_path.clone(); - } - } - for d in &candidate.scheduled { - if d.uuid == recording_uuid { - return d.recording.relative_path.clone(); - } - } - for d in &candidate.active { - if d.uuid == recording_uuid { - return d.recording.relative_path.clone(); - } - } - for d in &candidate.finished { - if d.uuid == recording_uuid { - return d.recording.relative_path.clone(); - } - } - None - } - - fn persisted_recording(uuid: &str, meta: RecordingMetadata) -> PersistedRecordingTask { - PersistedRecordingTask { - media_identity: String::new(), - partition: RecordingPartition::default(), - uuid: uuid.to_string(), - kind: RecordingKind::Live, - file_dir: PathBuf::from("/tmp"), - file_path: PathBuf::from(format!("/tmp/{uuid}.ts")), - filename: format!("{uuid}.ts"), - url: format!("https://example.com/{uuid}"), - finished: false, - size: 0, - total_size: None, - paused: false, - error: None, - state: RecordingTaskState::Scheduled, - input_name: None, - priority: 0, - retry_attempts: 0, - next_retry_at: None, - recording: meta, - } - } - - #[test] - fn reserve_recording_relative_path_returns_numbered_stem_when_no_collision() { - let mut candidate = PersistedRecordingQueue::default(); - let reserved = reserve_recording_relative_path(&mut candidate, "pilot", "rec-1"); - assert_eq!(reserved, "pilot_1", "numbered suffix is the first candidate"); - } - - #[test] - fn reserve_recording_relative_path_skips_existing_paths() { - let mut candidate = PersistedRecordingQueue::default(); - let occupied = RecordingMetadata { relative_path: Some("pilot_1".to_string()), ..media_meta("web:alice") }; - candidate.queue.push(persisted_recording("other", occupied)); - let reserved = reserve_recording_relative_path(&mut candidate, "pilot", "rec-1"); - assert_eq!(reserved, "pilot_2"); - } - - #[test] - fn reserve_recording_relative_path_does_not_double_bump_for_self() { - let mut candidate = PersistedRecordingQueue::default(); - let existing = RecordingMetadata { relative_path: Some("pilot_3".to_string()), ..media_meta("web:alice") }; - candidate.queue.push(persisted_recording("rec-1", existing)); - let reserved = reserve_recording_relative_path(&mut candidate, "pilot", "rec-1"); - assert_eq!(reserved, "pilot_3", "must not bump because of self"); - } - - #[tokio::test] - async fn pause_active_routes_through_mutate() { - let dir = tempfile::tempdir().expect("tempdir"); - let state_file = dir.path().to_path_buf(); - let queue = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); - let task = make_test_transfer_task("rec-1", dir.path().join("a.ts")); - { - let mut active = queue.active.write().await; - *active = vec![task]; - } - let prior_revision = queue.revision.load(Ordering::SeqCst); - queue.pause_active("rec-1").await.expect("pause_active"); - let active = queue.active.read().await.first().cloned().expect("active"); - assert!(active.paused); - assert_eq!(active.state, RecordingTaskState::Paused); - assert!(active.next_retry_at.is_none()); - assert_eq!(queue.revision.load(Ordering::SeqCst), prior_revision + 1); - } -} +mod tests; + +mod control; +mod error; +mod mutation; +mod persistence; +mod promotion; +mod task; +mod waiters; +mod workers; +pub use control::RecordingControl; +pub use error::{PersistedError, QueueMutationError}; +pub use mutation::{mutate, mutate_optional, mutate_prepared, mutate_then, mutate_with_idempotency}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use promotion::{ + attach_to_completed, media_held_by_another, media_is_still_referenced, promote_from_queue, promotion_decision, + PromotionDecision, +}; +pub use task::{PersistedRecordingQueue, RecordingTask}; +use waiters::RecordingWaiters; +pub use waiters::{RecordingWaitOutcome, RecordingWaiterSnapshot}; +pub use workers::RecordingWorkerState; diff --git a/backend/dvr/src/recording/recording_queue/control.rs b/backend/dvr/src/recording/recording_queue/control.rs new file mode 100644 index 000000000..5bf01a7b5 --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/control.rs @@ -0,0 +1,320 @@ +use super::{ + mutate_optional, mutation::mutate_optional_locked, promote_from_queue, waiters::lock_unpoisoned, + PersistedRecordingQueue, QueueMutationError, RecordingQueue, RecordingTask, RecordingTaskState, + RECORDING_WINDOW_EXPIRED_ERR, +}; +use crate::recording::{recording_transition, recording_transition::RecordingCommand}; +use shared::model::{QueueRevision, RecordingKind}; +use std::sync::{atomic::Ordering, Arc}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum RecordingControl { + #[default] + None, + Pause, + Cancel, + Restart, +} + +impl RecordingQueue { + pub(super) async fn snapshot_current(&self, revision: QueueRevision) -> PersistedRecordingQueue { + let queue = self.queue.lock().await; + let scheduled = self.scheduled.read().await; + let active = self.active.read().await; + let finished = self.finished.read().await; + + PersistedRecordingQueue { + queue: queue.iter().map(Self::to_persisted).collect(), + scheduled: scheduled.iter().map(Self::to_persisted).collect(), + active: active.iter().map(Self::to_persisted).collect(), + finished: finished.iter().map(Self::to_persisted).collect(), + revision, + } + } + + /// The provider input and the priority the active transfer should run at. + /// + /// One transfer serves every entry pointing at its file, so it runs at the + /// strongest priority any of them asked for. Taking the active entry's own + /// value lets a background request hold back somebody else's foreground one + /// for the same media. Lower is stronger. + /// + /// Takes the mutation guard first, like `committed_snapshot`, so the two + /// cannot interleave their inner locks. + pub async fn active_scheduling_priority(&self, uuid: &str) -> Option<(Option>, i8)> { + let _mutation = self.mutation_guard.lock().await; + let active = self.active.read().await; + let active = active.iter().find(|task| task.uuid == uuid)?; + let media = crate::recording::recording_service::recording_identity_key(&active.recording, active.url.as_str()); + let queue = self.queue.lock().await; + let scheduled = self.scheduled.read().await; + let finished = self.finished.read().await; + let strongest = queue + .iter() + .chain(scheduled.iter()) + .chain(finished.iter()) + .filter(|task| { + task.uuid != active.uuid + && crate::recording::recording_service::recording_identity_key(&task.recording, task.url.as_str()) + == media + }) + .fold(active.priority, |strongest, task| strongest.min(task.priority)); + Some((active.input_name.clone(), strongest)) + } + + pub async fn committed_snapshot(&self) -> (QueueRevision, Vec) { + let _mutation = self.mutation_guard.lock().await; + let revision = QueueRevision(self.revision.load(Ordering::SeqCst)); + let queue = self.queue.lock().await; + let scheduled = self.scheduled.read().await; + let active = self.active.read().await; + let finished = self.finished.read().await; + let mut tasks = Vec::with_capacity(queue.len() + scheduled.len() + finished.len() + active.len()); + tasks.extend(queue.iter().cloned()); + tasks.extend(scheduled.iter().cloned()); + tasks.extend(active.iter().cloned()); + tasks.extend(finished.iter().cloned()); + (revision, tasks) + } + + pub async fn committed_partitioned_snapshot(&self) -> (Vec, Vec, Vec) { + let _mutation = self.mutation_guard.lock().await; + let queue = self.queue.lock().await; + let scheduled = self.scheduled.read().await; + let active = self.active.read().await; + let finished = self.finished.read().await; + let mut queued = Vec::with_capacity(queue.len() + scheduled.len()); + queued.extend(queue.iter().cloned()); + queued.extend(scheduled.iter().cloned()); + (queued, active.clone(), finished.clone()) + } + + pub(super) fn finalize_missed_recording(mut task: RecordingTask) -> RecordingTask { + task.finished = true; + task.paused = false; + task.state = RecordingTaskState::Failed; + task.error = Some(RECORDING_WINDOW_EXPIRED_ERR.to_string()); + task.recording.reserved_bytes = 0; + task + } + + pub(super) fn recording_start_missed_window(task: &RecordingTask, now_ts: i64) -> bool { + task.kind == RecordingKind::Live + && task.scheduled_start().zip(task.scheduled_duration_secs()).is_some_and(|(start_at, duration_secs)| { + shared::model::recording_math::window_elapsed(start_at, duration_secs, now_ts) + }) + } + + /// Pause the active task. Persists the new state through the + /// transactional boundary. The runtime-only control signal is published + /// after the commit while the mutation guard still preserves ordering. + /// Live captures are not pausable and are rejected here. + pub async fn pause_active(&self, uuid: &str) -> Result { + let _mutation = self.mutation_guard.lock().await; + let changed = mutate_optional_locked(self, |candidate| { + let Some(active) = candidate.active.iter_mut().find(|active| active.uuid == uuid) else { + return Ok(None); + }; + active.state = recording_transition::transition(active.kind, active.state, RecordingCommand::Pause) + .map_err(|_| QueueMutationError::StateNotEditable)?; + active.paused = true; + active.next_retry_at = None; + Ok(Some(true)) + }) + .await? + .unwrap_or(false); + if !changed { + return Ok(false); + } + let worker = self.worker(uuid); + *worker.control_signal.write().await = RecordingControl::Pause; + worker.control_notify.notify_waiters(); + Ok(true) + } + + /// Resume the active task. Persists the new state through the + /// transactional boundary. + pub async fn resume_active(&self, uuid: &str) -> Result { + let _mutation = self.mutation_guard.lock().await; + let changed = mutate_optional_locked(self, |candidate| { + let Some(active) = candidate.active.iter_mut().find(|active| active.uuid == uuid && active.paused) else { + return Ok(None); + }; + active.state = recording_transition::transition(active.kind, active.state, RecordingCommand::Resume) + .map_err(|_| QueueMutationError::StateNotEditable)?; + active.paused = false; + active.next_retry_at = None; + Ok(Some(true)) + }) + .await? + .unwrap_or(false); + if !changed { + return Ok(false); + } + let worker = self.worker(uuid); + *worker.control_signal.write().await = RecordingControl::None; + worker.control_notify.notify_waiters(); + Ok(true) + } + + /// Cancel the active task `uuid`, if it is still the active one. + /// + /// A paused task with no running worker is filed as `Cancelled` immediately. + /// A worker still closing a paused stream is asked to cancel just like a + /// running one: it commits `Cancelled` after releasing its file and slot. + /// Returns whether cancellation finished immediately, or `None` when + /// `uuid` is not active. + pub async fn cancel_requested(&self, uuid: &str) -> Result, QueueMutationError> { + let _mutation = self.mutation_guard.lock().await; + let worker_running = match self.existing_worker(uuid) { + Some(worker) => *worker.running.read().await, + None => false, + }; + let cancelled_immediately = mutate_optional_locked(self, |candidate| { + let Some(active) = candidate.active.iter().find(|active| active.uuid == uuid) else { + return Ok(None); + }; + let cancelled_immediately = active.paused && !worker_running; + if cancelled_immediately { + let Some(index) = candidate.active.iter().position(|task| task.uuid == uuid) else { + return Ok(None); + }; + let mut cancelled = candidate.active.remove(index); + cancelled.finished = true; + cancelled.paused = false; + cancelled.next_retry_at = None; + cancelled.error.get_or_insert_with(|| "Cancelled by user".to_string()); + cancelled.state = RecordingTaskState::Cancelled; + cancelled.recording.reserved_bytes = 0; + candidate.finished.push(cancelled); + promote_from_queue(candidate); + } else if let Some(active) = candidate.active.iter_mut().find(|task| task.uuid == uuid) { + // The worker still owns the file and the provider slot; it + // commits `Cancelled` once it has let go. + active.state = if active.paused { + RecordingTaskState::Cancelling + } else { + recording_transition::transition( + active.kind, + active.state, + recording_transition::RecordingCommand::Cancel, + ) + .unwrap_or(RecordingTaskState::Cancelling) + }; + active.paused = false; + active.error = Some("Cancelled by user".to_string()); + active.next_retry_at = None; + } + Ok(Some(cancelled_immediately)) + }) + .await?; + + if let Some(cancelled_immediately) = cancelled_immediately { + let worker = if cancelled_immediately { self.existing_worker(uuid) } else { Some(self.worker(uuid)) }; + if let Some(worker) = worker { + *worker.control_signal.write().await = + if cancelled_immediately { RecordingControl::None } else { RecordingControl::Cancel }; + worker.control_notify.notify_waiters(); + if cancelled_immediately { + lock_unpoisoned(&self.workers).remove(uuid); + } + } + } + Ok(cancelled_immediately) + } + + /// Ask the worker to restart so it picks up a reloaded configuration. + /// + /// A restart never replaces a pending pause or cancel. The fallback below + /// runs later than the caller, so it may land after a user's pause; it + /// must not turn that pause into a requeue. + pub fn request_worker_restart(&self) { + fn request(control: &mut RecordingControl) -> bool { + if *control == RecordingControl::None { + *control = RecordingControl::Restart; + } + *control == RecordingControl::Restart + } + let workers: Vec<_> = lock_unpoisoned(&self.workers).values().cloned().collect(); + for worker in workers { + if let Ok(mut control) = worker.control_signal.try_write() { + if request(&mut control) { + worker.control_notify.notify_waiters(); + } + } else { + let worker = Arc::clone(&worker); + tokio::spawn(async move { + let mut control = worker.control_signal.write().await; + if request(&mut control) { + worker.control_notify.notify_waiters(); + } + }); + } + } + } + + pub async fn remove_from_queue(&self, uuid: &str) -> Result { + Ok(mutate_optional(self, |candidate| { + let queue_len = candidate.queue.len(); + candidate.queue.retain(|task| task.uuid != uuid); + if candidate.queue.len() != queue_len { + return Ok(Some(true)); + } + let scheduled_len = candidate.scheduled.len(); + candidate.scheduled.retain(|task| task.uuid != uuid); + Ok((candidate.scheduled.len() != scheduled_len).then_some(true)) + }) + .await? + .unwrap_or(false)) + } + + pub async fn remove_finished(&self, uuid: &str) -> Result { + Ok(mutate_optional(self, |candidate| { + let initial_len = candidate.finished.len(); + candidate.finished.retain(|task| task.uuid != uuid); + Ok((candidate.finished.len() != initial_len).then_some(true)) + }) + .await? + .unwrap_or(false)) + } + + pub async fn remove(&self, uuid: &str) -> Result { + Ok(mutate_optional(self, |candidate| { + let original_len = candidate.queue.len() + candidate.scheduled.len() + candidate.finished.len(); + candidate.queue.retain(|task| task.uuid != uuid); + candidate.scheduled.retain(|task| task.uuid != uuid); + candidate.finished.retain(|task| task.uuid != uuid); + let current_len = candidate.queue.len() + candidate.scheduled.len() + candidate.finished.len(); + Ok((current_len != original_len).then_some(true)) + }) + .await? + .unwrap_or(false)) + } + + /// Requeue a finished VOD/Series transfer. A Live capture cannot be + /// retried: its programme window is gone. + pub async fn retry_finished(&self, uuid: &str) -> Result { + Ok(mutate_optional(self, |candidate| { + if let Some(pos) = candidate.finished.iter().position(|task| task.uuid == uuid) { + let mut task = candidate.finished.remove(pos); + let Ok(next) = recording_transition::transition(task.kind, task.state, RecordingCommand::Retry) else { + candidate.finished.insert(pos, task); + return Ok(None); + }; + task.finished = false; + task.size = 0; + task.paused = false; + task.error = None; + task.state = next; + task.retry_attempts = 0; + task.next_retry_at = None; + candidate.queue.push(task); + Ok(Some(true)) + } else { + Ok(None) + } + }) + .await? + .unwrap_or(false)) + } +} diff --git a/backend/dvr/src/recording/recording_queue/error.rs b/backend/dvr/src/recording/recording_queue/error.rs new file mode 100644 index 000000000..3639ec8f0 --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/error.rs @@ -0,0 +1,104 @@ +/// Reason a persisted entry cannot be converted back to its in-memory form +/// during the commit step. Surfaced to the caller so a corrupt persisted file +/// fails closed instead of silently dropping entries. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PersistedError { + /// The persisted URL could not be parsed. + InvalidUrl(String), +} + +impl std::fmt::Display for PersistedError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidUrl(s) => write!(f, "persisted url is invalid: {s}"), + } + } +} + +impl std::error::Error for PersistedError {} + +/// Typed error returned from the queue mutation boundary. Every +/// `mutate` closure that fails must return a known variant; the +/// `Other` variant is an escape hatch for dynamically-formatted +/// messages that have no stable wire code. +#[derive(Debug)] +pub enum QueueMutationError { + UnknownRecording, + StateNotEditable, + Forbidden, + InvalidInterval, + InvalidQuotaPool, + InvalidPath, + PaddingLimitExceeded, + QuotaExceeded, + Duplicate, + NotInTerminalState, + DiskFull, + MutationSkipped, + /// The idempotency key was accepted before, for the same request. + IdempotentReplay { + recording_id: String, + }, + /// The idempotency key was accepted before, for a different request. + IdempotencyConflict, + /// Escape hatch for dynamically-formatted validation messages + /// that have no stable wire code. Prefer the typed variants. + Other(String), + Io(std::io::Error), +} + +impl QueueMutationError { + /// Escape-hatch constructor for messages that cannot be expressed + /// as a typed variant. Prefer the typed `Self::X` constructors. + pub fn new(message: impl Into) -> Self { Self::Other(message.into()) } + + pub fn from_io(err: std::io::Error) -> Self { Self::Io(err) } + + /// Stable display message for logging and HTTP error rendering. + pub fn message(&self) -> &'static str { + match self { + Self::UnknownRecording => "recording unknown", + Self::StateNotEditable => "recording state not editable", + Self::Forbidden => "recording forbidden", + Self::InvalidInterval => "recording invalid interval", + Self::InvalidQuotaPool => "recording invalid quota pool", + Self::InvalidPath => "recording invalid path", + Self::PaddingLimitExceeded => "recording_padding_limit_exceeded", + Self::QuotaExceeded => "recording quota exceeded", + Self::Duplicate => "recording duplicate", + Self::NotInTerminalState => "recording not in terminal state", + Self::DiskFull => "disk full", + Self::MutationSkipped => "mutation unexpectedly skipped", + Self::IdempotentReplay { .. } => "recording idempotent replay", + Self::IdempotencyConflict => "recording idempotency conflict", + Self::Other(_) => "queue mutation failed", + Self::Io(_) => "queue mutation persistence failed", + } + } + + pub fn source_io(&self) -> Option<&std::io::Error> { + match self { + Self::Io(err) => Some(err), + _ => None, + } + } +} + +impl std::fmt::Display for QueueMutationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Other(s) => f.write_str(s), + Self::Io(e) => std::fmt::Display::fmt(e, f), + other => f.write_str(other.message()), + } + } +} + +impl std::error::Error for QueueMutationError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Io(err) => Some(err as &(dyn std::error::Error + 'static)), + _ => None, + } + } +} diff --git a/backend/dvr/src/recording/recording_queue/mutation.rs b/backend/dvr/src/recording/recording_queue/mutation.rs new file mode 100644 index 000000000..6d137ebcb --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/mutation.rs @@ -0,0 +1,235 @@ +use super::{ + IdempotencyOutcome, PersistedIdempotency, PersistedRecordingQueue, QueueMutationError, RecordingControl, + RecordingQueue, RecordingTask, +}; +use shared::model::QueueRevision; +use std::{collections::VecDeque, sync::atomic::Ordering}; + +/// Lock ordering for the queue mutation boundary: +/// +/// 1. `mutation_guard` (`Mutex`) — outermost for persisted mutations and ordered control publication. +/// 2. `queue` (`Mutex`) — always taken before persisted state locks. +/// 3. `scheduled` / `active` / `finished` (`RwLock`, write) — taken after the queue. +/// 4. `control_signal` (`RwLock`) and `control_notify` (`Notify`) — taken after the +/// queue locks; they signal runtime state, not persisted state. +/// 5. `revision` (`AtomicU64`) — no lock; swapped atomically with the +/// commit step. +/// +/// The queue mutation boundary (`mutate`) holds the queue locks only while +/// building the candidate snapshot. It does **not** hold any queue lock +/// while running the user closure or while persisting. Persisting holds +/// the recording repository mutex only. +/// +/// Rule repository mutations are acquired strictly after the queue boundary +/// has committed; never inside it. +/// +/// Apply a single transactional queue mutation. The closure receives an +/// owned `PersistedRecordingQueue` candidate cloned from the current state +/// and returns either a value or a [`QueueMutationError`]. On success the +/// candidate is persisted atomically, then swapped into the in-memory state, +/// then the `QueueRevision` is incremented. On any failure — closure error +/// or persist error — the in-memory state, the repository, and the +/// revision are all unchanged. +pub async fn mutate(this: &RecordingQueue, op: F) -> Result +where + F: FnOnce(&mut PersistedRecordingQueue) -> Result, +{ + match mutate_optional(this, |candidate| op(candidate).map(Some)).await? { + Some(result) => Ok(result), + None => Err(QueueMutationError::MutationSkipped), + } +} + +/// [`mutate`], recording an accepted idempotency key in the same commit. +/// +/// The key has to land with the recording it describes. Writing it +/// afterwards would leave a window in which the recording exists and the +/// record that suppresses its retry does not. +pub async fn mutate_with_idempotency( + this: &RecordingQueue, + idempotency: PersistedIdempotency, + op: F, +) -> Result +where + F: FnOnce(&mut PersistedRecordingQueue) -> Result, +{ + let _mutation = this.mutation_guard.lock().await; + // Checked again under the guard: the caller's lookup ran without it, so + // another request with the same key may have committed in between. + let outcome = this + .lookup_idempotency(&idempotency.principal, &idempotency.key, &idempotency.request_fingerprint) + .await + .map_err(QueueMutationError::from_io)?; + match outcome { + IdempotencyOutcome::Fresh => {} + IdempotencyOutcome::Replay { recording_id } => { + return Err(QueueMutationError::IdempotentReplay { recording_id }); + } + IdempotencyOutcome::Conflict => return Err(QueueMutationError::IdempotencyConflict), + } + match mutate_optional_locked_with(this, Some(idempotency), |candidate| op(candidate).map(Some)).await? { + Some(result) => Ok(result), + None => Err(QueueMutationError::MutationSkipped), + } +} + +/// [`mutate`], then `after` with its result, before the mutation guard is +/// released. +/// +/// For filesystem work that must not race the next mutation: once the guard +/// is released a new admission or a worker start can claim the path the +/// committed state just gave up. +pub async fn mutate_then(this: &RecordingQueue, op: F, after: A) -> Result +where + F: FnOnce(&mut PersistedRecordingQueue) -> Result, + A: FnOnce(&R) -> Fut, + Fut: std::future::Future, +{ + let _mutation = this.mutation_guard.lock().await; + let result = mutate_optional_locked(this, |candidate| op(candidate).map(Some)) + .await? + .ok_or(QueueMutationError::MutationSkipped)?; + after(&result).await; + Ok(result) +} + +/// `prepare`, then [`mutate`] with its result, under one mutation guard. +/// +/// For filesystem work that has to happen before a mutation commits, without +/// blocking a runtime thread inside the mutation closure and without another +/// mutation slipping in between. `prepare` reads the committed in-memory +/// state, which cannot change while the guard is held. +pub async fn mutate_prepared(this: &RecordingQueue, prepare: P, op: F) -> Result +where + P: FnOnce() -> Fut, + Fut: std::future::Future>, + F: FnOnce(&mut PersistedRecordingQueue, T) -> Result, +{ + let _mutation = this.mutation_guard.lock().await; + let prepared = prepare().await?; + mutate_optional_locked(this, |candidate| op(candidate, prepared).map(Some)) + .await? + .ok_or(QueueMutationError::MutationSkipped) +} + +pub async fn mutate_optional(this: &RecordingQueue, op: F) -> Result, QueueMutationError> +where + F: FnOnce(&mut PersistedRecordingQueue) -> Result, QueueMutationError>, +{ + let _mutation = this.mutation_guard.lock().await; + mutate_optional_locked(this, op).await +} + +pub(super) async fn mutate_optional_locked(this: &RecordingQueue, op: F) -> Result, QueueMutationError> +where + F: FnOnce(&mut PersistedRecordingQueue) -> Result, QueueMutationError>, +{ + mutate_optional_locked_with(this, None, op).await +} + +async fn mutate_optional_locked_with( + this: &RecordingQueue, + idempotency: Option, + op: F, +) -> Result, QueueMutationError> +where + F: FnOnce(&mut PersistedRecordingQueue) -> Result, QueueMutationError>, +{ + let next_revision = this.revision.load(Ordering::SeqCst).saturating_add(1); + + // 2. Build candidate under the queue locks. + let mut candidate = this.snapshot_current(QueueRevision(next_revision)).await; + + // 3. Apply the mutation to a candidate snapshot. The closure can do + // arbitrary validation and refer back to the candidate's prior + // state. + let Some(result) = op(&mut candidate)? else { + return Ok(None); + }; + + let records = candidate.to_records(); + + let PersistedRecordingQueue { + queue: candidate_queue, + scheduled: candidate_scheduled, + active: candidate_active, + finished: candidate_finished, + revision: _, + } = candidate; + + // 4. Validate every persisted entry into its in-memory form before + // swapping. A single corrupt entry must abort the commit so the + // persisted file and the in-memory state stay identical. Without this, + // a bad URL would be silently dropped from memory while the file still + // listed it, desyncing the two. + let mut queue: VecDeque = VecDeque::with_capacity(candidate_queue.len()); + for p in candidate_queue { + queue.push_back( + RecordingQueue::from_persisted(p) + .map_err(|e| QueueMutationError::new(format!("persisted queue entry invalid: {e}")))?, + ); + } + let mut scheduled: Vec = Vec::with_capacity(candidate_scheduled.len()); + for p in candidate_scheduled { + scheduled.push( + RecordingQueue::from_persisted(p) + .map_err(|e| QueueMutationError::new(format!("persisted scheduled entry invalid: {e}")))?, + ); + } + let active = candidate_active + .into_iter() + .map(|task| { + RecordingQueue::from_persisted(task) + .map_err(|e| QueueMutationError::new(format!("persisted active entry invalid: {e}"))) + }) + .collect::, _>>()?; + let mut finished: Vec = Vec::with_capacity(candidate_finished.len()); + for p in candidate_finished { + finished.push( + RecordingQueue::from_persisted(p) + .map_err(|e| QueueMutationError::new(format!("persisted finished entry invalid: {e}")))?, + ); + } + + // 5. Persist only after the complete candidate has been validated. + this.persist_records(next_revision, records, idempotency).await?; + + // 6. Commit. Swap the validated in-memory state from the persisted + // candidate. + let mut queue_lock = this.queue.lock().await; + let mut scheduled_lock = this.scheduled.write().await; + let mut active_lock = this.active.write().await; + let mut finished_lock = this.finished.write().await; + *queue_lock = queue; + *scheduled_lock = scheduled; + *active_lock = active; + *finished_lock = finished; + this.prune_idle_workers(&queue_lock, &scheduled_lock, &active_lock); + this.revision.store(next_revision, Ordering::SeqCst); + this.queue_changed.notify_one(); + Ok(Some(result)) +} + +impl RecordingQueue { + pub async fn mutate_optional_and_clear_control( + &self, + uuid: &str, + expected: RecordingControl, + op: F, + ) -> Result, QueueMutationError> + where + F: FnOnce(&mut PersistedRecordingQueue) -> Result, QueueMutationError>, + { + let _mutation = self.mutation_guard.lock().await; + let worker = self.existing_worker(uuid); + let result = mutate_optional_locked(self, op).await?; + if result.is_some() { + let Some(worker) = worker else { return Ok(result) }; + let mut control = worker.control_signal.write().await; + if *control == expected { + *control = RecordingControl::None; + } + } + Ok(result) + } +} diff --git a/backend/dvr/src/recording/recording_queue/persistence.rs b/backend/dvr/src/recording/recording_queue/persistence.rs new file mode 100644 index 000000000..7f00764cc --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/persistence.rs @@ -0,0 +1,288 @@ +use super::{ + poisoned_repository, waiters::lock_unpoisoned, PersistedError, PersistedIdempotency, PersistedRecordingQueue, + PersistedRecordingTask, QueueMutationError, RecordingPartition, RecordingQueue, RecordingSlotWaitQueue, + RecordingTask, RecordingTaskState, RECORDING_INTERRUPTED_ERR, +}; +use chrono::Utc; +use log::error; +use shared::model::{QueueRevision, RecordingKind}; +use std::{ + collections::{HashMap, VecDeque}, + path::Path, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex as StdMutex, + }, +}; +use tokio::sync::{Mutex, Notify, RwLock}; +use tuliprox_repository::recording_repository::RecordingRepository; + +impl PersistedRecordingQueue { + /// Flatten the four partitions into the repository's record set, tagging + /// each task with the partition it must be restored into. + pub(super) fn to_records(&self) -> Vec { + let mut records = + Vec::with_capacity(self.queue.len() + self.scheduled.len() + self.finished.len() + self.active.len()); + let partitions: [(&[PersistedRecordingTask], RecordingPartition); 4] = [ + (&self.queue, RecordingPartition::Queued), + (&self.scheduled, RecordingPartition::Scheduled), + (&self.active, RecordingPartition::Active), + (&self.finished, RecordingPartition::Finished), + ]; + for (tasks, partition) in partitions { + for task in tasks { + let mut task = task.clone(); + task.partition = partition; + records.push(task); + } + } + records + } + + /// Rebuild the four partitions from a repository snapshot. + pub(super) fn from_records(records: Vec, revision: QueueRevision) -> Self { + let mut restored = Self { revision, ..Self::default() }; + for task in records { + match task.partition { + RecordingPartition::Scheduled => restored.scheduled.push(task), + RecordingPartition::Finished => restored.finished.push(task), + RecordingPartition::Active => restored.active.push(task), + RecordingPartition::Queued => restored.queue.push(task), + } + } + restored + } +} + +impl RecordingQueue { + pub fn new() -> Self { Self::new_with_repository(None) } + + /// Opens the recoverable store and builds a queue backed by it. + /// + /// `recovery_root` may point at a different filesystem than `storage_dir`; + /// that is what lets the history outlive the loss of the database volume. + pub fn new_persistent(storage_dir: &Path, recovery_root: &Path) -> std::io::Result { + let (repository, _) = RecordingRepository::open(storage_dir, recovery_root)?; + Ok(Self::new_with_repository(Some(Arc::new(StdMutex::new(repository))))) + } + + pub fn new_with_repository(repository: Option>>) -> Self { + Self { + queue: Arc::from(Mutex::new(VecDeque::new())), + scheduled: Arc::from(RwLock::new(Vec::new())), + active: Arc::from(RwLock::new(Vec::new())), + finished: Arc::from(RwLock::new(Vec::new())), + workers: StdMutex::new(HashMap::new()), + capacity_guard: Mutex::new(()), + queue_changed: Notify::new(), + repository, + slot_waiters: Arc::new(RecordingSlotWaitQueue::new()), + revision: Arc::new(AtomicU64::new(0)), + mutation_guard: Arc::new(Mutex::new(())), + } + } + + pub fn to_persisted(task: &RecordingTask) -> PersistedRecordingTask { + PersistedRecordingTask { + media_identity: crate::recording::recording_service::recording_identity_key( + &task.recording, + task.url.as_str(), + ), + // Overwritten from the owning partition when the candidate is flattened. + partition: RecordingPartition::default(), + uuid: task.uuid.clone(), + kind: task.kind, + file_dir: task.file_dir.clone(), + file_path: task.file_path.clone(), + filename: task.filename.clone(), + url: task.url.to_string(), + finished: task.finished, + size: task.size, + total_size: task.total_size, + paused: task.paused, + error: task.error.clone(), + state: task.state, + input_name: task.input_name.as_ref().map(std::string::ToString::to_string), + priority: task.priority, + retry_attempts: task.retry_attempts, + next_retry_at: task.next_retry_at, + recording: task.recording.clone(), + } + } + + pub fn from_persisted(task: PersistedRecordingTask) -> Result { + let url = reqwest::Url::parse(&task.url).map_err(|e| PersistedError::InvalidUrl(e.to_string()))?; + Ok(RecordingTask { + uuid: task.uuid, + kind: task.kind, + file_dir: task.file_dir, + file_path: task.file_path, + filename: task.filename, + url, + finished: task.finished, + size: task.size, + total_size: task.total_size, + paused: task.paused, + error: task.error, + state: task.state, + input_name: task.input_name.map(|s| Arc::from(s.as_str())), + priority: task.priority, + retry_attempts: task.retry_attempts, + next_retry_at: task.next_retry_at, + recording: task.recording, + }) + } + + pub async fn persist_to_disk(&self) -> std::io::Result<()> { + // `snapshot_current` takes the partition locks in mutation order; the + // guard is what keeps that order from meeting a reader that takes them + // differently. + let _mutation = self.mutation_guard.lock().await; + let revision = self.revision.load(Ordering::SeqCst); + let records = self.snapshot_current(QueueRevision(revision)).await.to_records(); + let result = self.commit_records(revision, records, None).await; + // Callers discard the result; log here so persistence failures are never silent + if let Err(err) = &result { + error!("Failed to persist recording queue: {err}"); + } + result + } + + /// Commit a record set to the repository, off the async executor. + /// + /// The repository fsyncs both its journal and its B+Tree, so it must not + /// run on a runtime worker thread. + pub(super) async fn commit_records( + &self, + revision: u64, + records: Vec, + idempotency: Option, + ) -> std::io::Result<()> { + let Some(repository) = self.repository.clone() else { + return Ok(()); + }; + let now = chrono::Utc::now().timestamp(); + tokio::task::spawn_blocking(move || { + let mut guard = repository.lock().map_err(|_| poisoned_repository())?; + guard.commit_with_idempotency(revision, &records, idempotency, now) + }) + .await + .map_err(std::io::Error::other)? + } + + pub(super) async fn persist_records( + &self, + revision: u64, + records: Vec, + idempotency: Option, + ) -> Result<(), QueueMutationError> { + self.commit_records(revision, records, idempotency).await.map_err(QueueMutationError::from_io) + } + + pub async fn load_from_disk(&self) -> std::io::Result<()> { + let Some(repository) = self.repository.clone() else { + return Ok(()); + }; + let snapshot = tokio::task::spawn_blocking(move || { + let mut guard = repository.lock().map_err(|_| poisoned_repository())?; + guard.load() + }) + .await + .map_err(std::io::Error::other)??; + + let persisted = PersistedRecordingQueue::from_records(snapshot.tasks, QueueRevision(snapshot.queue_revision)); + + let invalid = |err: PersistedError| std::io::Error::new(std::io::ErrorKind::InvalidData, err); + let mut queue = VecDeque::with_capacity(persisted.queue.len()); + for task in persisted.queue { + queue.push_back(Self::recover_loaded_task(Self::from_persisted(task).map_err(invalid)?)); + } + let now_ts = Utc::now().timestamp(); + let mut scheduled = Vec::with_capacity(persisted.scheduled.len()); + let mut missed_scheduled = Vec::new(); + for task in persisted.scheduled { + let task = Self::recover_loaded_task(Self::from_persisted(task).map_err(invalid)?); + if Self::recording_start_missed_window(&task, now_ts) { + missed_scheduled.push(task); + } else { + scheduled.push(task); + } + } + let active = persisted + .active + .into_iter() + .map(|task| Self::from_persisted(task).map(Self::recover_loaded_task).map_err(invalid)) + .collect::, _>>()?; + let mut finished = Vec::with_capacity(persisted.finished.len()); + for task in persisted.finished { + finished.push(Self::from_persisted(task).map_err(invalid)?); + } + finished.extend(missed_scheduled.into_iter().map(Self::finalize_missed_recording)); + + *self.queue.lock().await = queue; + *self.scheduled.write().await = scheduled; + *self.finished.write().await = finished; + self.revision.store(persisted.revision.0, Ordering::SeqCst); + let mut paused = Vec::new(); + for task in active { + if task.paused || task.state == RecordingTaskState::Paused { + paused.push(task); + } else if !task.finished && task.state != RecordingTaskState::Cancelled { + self.queue.lock().await.push_front(task); + } else { + self.finished.write().await.push(task); + } + } + *self.active.write().await = paused; + lock_unpoisoned(&self.workers).clear(); + Ok(()) + } + + pub(super) fn recover_loaded_task(mut task: RecordingTask) -> RecordingTask { + if task.paused || task.state == RecordingTaskState::Paused { + task.paused = true; + task.finished = false; + task.state = RecordingTaskState::Paused; + return task; + } + // The worker that was going to acknowledge the cancellation died with + // the process. Nothing will ever finish it, and it is not `finished`, + // so the requeue below would restart work the user cancelled. + if task.state == RecordingTaskState::Cancelling { + task.paused = false; + task.finished = true; + task.state = RecordingTaskState::Cancelled; + task.error.get_or_insert_with(|| "Cancelled by user".to_string()); + task.next_retry_at = None; + task.recording.reserved_bytes = 0; + return task; + } + if task.state == RecordingTaskState::Scheduled { + task.paused = false; + task.finished = false; + return task; + } + // A live capture cannot be picked up again: whatever was broadcast while + // the process was down is gone. Requeueing would restart it and write a + // recording silently missing everything up to now. + if task.kind == RecordingKind::Live + && matches!(task.state, RecordingTaskState::Running | RecordingTaskState::WaitingForCapacity) + { + task.paused = false; + task.finished = true; + task.state = RecordingTaskState::Failed; + task.error = Some(RECORDING_INTERRUPTED_ERR.to_string()); + task.next_retry_at = None; + task.recording.reserved_bytes = 0; + return task; + } + if !task.finished { + task.paused = false; + task.state = RecordingTaskState::Queued; + task.error = None; + task.retry_attempts = 0; + task.next_retry_at = None; + } + task + } +} diff --git a/backend/dvr/src/recording/recording_queue/promotion.rs b/backend/dvr/src/recording/recording_queue/promotion.rs new file mode 100644 index 000000000..3bab2e44b --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/promotion.rs @@ -0,0 +1,267 @@ +use super::{ + mutate_optional, PersistedRecordingQueue, PersistedRecordingTask, RecordingQueue, RecordingTaskState, + RECORDING_WINDOW_EXPIRED_ERR, +}; +use crate::recording::recording_path; +use chrono::Utc; +use log::error; +use shared::{ + model::{RecordingKind, RecordingMetadata}, + utils::CONSTANTS, +}; +use tuliprox_core::model::RecordingConfig; + +/// What the organised layout groups this recording under. +/// +/// Every kind is filed under its playlist group. Without one, Live groups by +/// channel and VOD by title. A series groups by its series name, which falls +/// back to the programme title and then to the episode pattern applied to the +/// filename stem, which is how the layout was derived before the metadata +/// carried it. +pub(super) fn recording_grouping( + kind: RecordingKind, + recording: &RecordingMetadata, + recording_cfg: &RecordingConfig, + file_stem: &str, +) -> recording_path::RecordingGrouping { + let grouping = match kind { + RecordingKind::Live => recording_path::RecordingGrouping::live( + recording.channel_name.clone().unwrap_or_else(|| stem_group(recording_cfg, file_stem)), + ), + RecordingKind::Vod => recording_path::RecordingGrouping::vod( + recording.program_title.clone().unwrap_or_else(|| stem_group(recording_cfg, file_stem)), + ), + RecordingKind::Series => recording_path::RecordingGrouping::series( + recording + .series_name + .clone() + .or_else(|| recording.program_title.clone()) + .unwrap_or_else(|| stem_group(recording_cfg, file_stem)), + ), + }; + grouping.in_group(recording.group.clone()) +} + +/// The pre-metadata grouping rule: strip the episode marker and any trailing +/// filename decoration from the stem. +fn stem_group(recording_cfg: &RecordingConfig, file_stem: &str) -> String { + let mut stem = file_stem; + if let Some(re) = &recording_cfg.episode_pattern { + if let Some(captures) = re.captures(stem) { + if let Some(episode) = captures.name("episode") { + if !episode.as_str().is_empty() { + stem = &stem[..episode.start()]; + } + } + } + } + CONSTANTS.re_remove_filename_ending.replace(stem, "").into_owned() +} + +/// What to do with a queued task whose media another entry may already hold. +/// +/// Several library entries can reference one physical file. Promoting each of +/// them would start a second transfer to the same path, so a queued entry that +/// is not the one producing the file must attach to it or wait for it. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PromotionDecision { + /// Nothing else holds this media; run the transfer. + Execute, + /// A completed entry at this index in `finished` already holds it. + AttachTo(usize), + /// Another active entry is producing this media; wait for its file. + Wait, +} + +/// Two tasks refer to the same media when they carry the same identity. +/// +/// An empty identity never matches, including another empty one: it means the +/// identity could not be resolved, and guessing that two unidentified requests +/// are the same recording would merge two different files. +fn same_media(left: &PersistedRecordingTask, right: &PersistedRecordingTask) -> bool { + !left.media_identity.is_empty() && left.media_identity == right.media_identity +} + +/// Every task in the candidate, whichever partition it sits in. +fn all_tasks(candidate: &PersistedRecordingQueue) -> impl Iterator { + candidate + .queue + .iter() + .chain(candidate.scheduled.iter()) + .chain(candidate.active.iter()) + .chain(candidate.finished.iter()) +} + +/// Whether an entry other than `uuid` still holds the same file. +/// +/// Entries already mid-deletion are not counted. Two concurrent deletions +/// that each saw the other as a holder would both decline to unlink and +/// leave the file behind with nothing pointing at it. +pub fn media_is_still_referenced(candidate: &PersistedRecordingQueue, uuid: &str) -> bool { + let Some(subject) = all_tasks(candidate).find(|task| task.uuid == uuid) else { + return false; + }; + media_held_by_another( + all_tasks(candidate).map(|task| { + (task.uuid.as_str(), task.media_identity.as_str(), task.recording.deleting_previous_state.is_some()) + }), + uuid, + &subject.media_identity, + ) +} + +/// The reference rule itself, over `(uuid, media identity, being deleted)` +/// entries, so the persisted queue and an in-memory snapshot apply the same +/// one. An entry being deleted holds nothing, and an empty identity matches +/// nothing. +pub fn media_held_by_another<'a>( + entries: impl IntoIterator, + uuid: &str, + media: &str, +) -> bool { + !media.is_empty() + && entries + .into_iter() + .any(|(other_uuid, other_media, deleting)| other_uuid != uuid && !deleting && other_media == media) +} + +pub fn promotion_decision(candidate: &PersistedRecordingQueue, task: &PersistedRecordingTask) -> PromotionDecision { + if candidate.active.iter().any(|active| same_media(active, task)) { + return PromotionDecision::Wait; + } + if let Some(index) = + candidate.finished.iter().position(|done| done.state == RecordingTaskState::Completed && same_media(done, task)) + { + return PromotionDecision::AttachTo(index); + } + PromotionDecision::Execute +} + +/// Add the next runnable entry to the active tasks, attaching any entry whose +/// file another entry already produced. +/// +/// Every path that activates a task goes through here. Taking the head of +/// the queue directly would re-download a file that was just completed by the +/// entry ahead of it. +pub fn promote_from_queue(candidate: &mut PersistedRecordingQueue) -> Option<(String, String)> { + let mut index = 0; + while index < candidate.queue.len() { + match promotion_decision(candidate, &candidate.queue[index]) { + PromotionDecision::Execute => { + // Bulk transfers stay serial; live captures can run alongside them. + if candidate.queue[index].kind != RecordingKind::Live + && candidate.active.iter().any(|task| task.kind != RecordingKind::Live) + { + index += 1; + continue; + } + let next = candidate.queue.remove(index); + let promoted = (next.uuid.clone(), next.filename.clone()); + candidate.active.push(next); + return Some(promoted); + } + PromotionDecision::AttachTo(completed) => { + let source = candidate.finished[completed].clone(); + let mut attached = candidate.queue.remove(index); + attach_to_completed(&mut attached, &source); + candidate.finished.push(attached); + // The queue shrank; re-examine this position. + } + PromotionDecision::Wait => index += 1, + } + } + None +} + +/// Adopt an already-produced file instead of transferring it again. +/// +/// Only the physical result is copied. The owner, visibility and quota belong +/// to the entry and must survive: this is one user's link to a file another +/// user's request happened to produce. +pub fn attach_to_completed(task: &mut PersistedRecordingTask, source: &PersistedRecordingTask) { + task.file_dir.clone_from(&source.file_dir); + task.file_path.clone_from(&source.file_path); + task.filename.clone_from(&source.filename); + task.size = source.size; + task.total_size = source.total_size; + task.finished = true; + task.paused = false; + task.error = None; + task.state = RecordingTaskState::Completed; + task.next_retry_at = None; + task.retry_attempts = 0; + task.recording.relative_path.clone_from(&source.recording.relative_path); + task.recording.partial_relative_path = None; + task.recording.measured_bytes = source.recording.measured_bytes; + task.recording.completed_at = source.recording.completed_at; + // The reservation is released: the bytes are already on disk and charged + // to this entry as measured, not reserved. + task.recording.reserved_bytes = 0; +} + +impl RecordingQueue { + pub async fn promote_due_scheduled(&self, now_ts: i64) -> usize { + let due = self.scheduled.read().await.iter().any(|task| { + task.recording.scheduled_start.is_some_and(|start| start <= now_ts) + || (task.kind == RecordingKind::Live && task.recording.scheduled_end.is_some_and(|end| now_ts >= end)) + }); + if !due { + return 0; + } + let result = mutate_optional(self, |candidate| { + let mut due_tasks = Vec::new(); + let mut missed_recordings = Vec::new(); + candidate.scheduled.retain(|task| { + let scheduled_start = task.recording.scheduled_start; + let is_missed = + task.kind == RecordingKind::Live && task.recording.scheduled_end.is_some_and(|end| now_ts >= end); + if is_missed { + let mut missed = task.clone(); + missed.finished = true; + missed.paused = false; + missed.state = RecordingTaskState::Failed; + missed.error = Some(RECORDING_WINDOW_EXPIRED_ERR.to_string()); + missed.recording.reserved_bytes = 0; + missed_recordings.push(missed); + return false; + } + let is_due = scheduled_start.is_some_and(|start_at| start_at <= now_ts); + if is_due { + let mut queued = task.clone(); + queued.state = RecordingTaskState::Queued; + queued.paused = false; + queued.finished = false; + queued.error = None; + queued.size = 0; + queued.total_size = None; + queued.retry_attempts = 0; + queued.next_retry_at = None; + due_tasks.push(queued); + } + !is_due + }); + + if due_tasks.is_empty() && missed_recordings.is_empty() { + return Ok(None); + } + + let due_count = due_tasks.len(); + let missed_count = missed_recordings.len(); + candidate.finished.extend(missed_recordings); + candidate.queue.splice(0..0, due_tasks); + Ok(Some(if due_count == 0 { missed_count } else { due_count })) + }) + .await; + + match result { + Ok(Some(promoted)) => promoted, + Ok(None) => 0, + Err(err) => { + error!("Promoting due scheduled recordings failed: {err}"); + 0 + } + } + } + + pub async fn promote_due_scheduled_now(&self) -> usize { self.promote_due_scheduled(Utc::now().timestamp()).await } +} diff --git a/backend/dvr/src/recording/recording_queue/task.rs b/backend/dvr/src/recording/recording_queue/task.rs new file mode 100644 index 000000000..ffa8b9ed5 --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/task.rs @@ -0,0 +1,214 @@ +use super::{ + promotion::recording_grouping, PersistedRecordingTask, RecordingTaskState, MAX_COLLISION_PROBES, + RECORDING_TASK_ID_COUNTER, +}; +use crate::recording::{recording_path, recording_transition}; +use serde::{Deserialize, Serialize}; +use shared::{ + model::{Claims, QueueRevision, RecordingKind, RecordingMetadata, RecordingTaskDto, TaskPriorityDto, UserId}, + utils::{sanitize_filename_chars, FILENAME_TRIM_PATTERNS}, +}; +use std::{ + ffi::OsStr, + path::{Path, PathBuf}, + sync::{atomic::Ordering, Arc}, + time::{SystemTime, UNIX_EPOCH}, +}; +use tuliprox_core::model::RecordingConfig; + +/// A recording task in memory. Internal shape — never serialized to a +/// client. Use [`RecordingTask::to_owner_view`] for the public projection. +#[derive(Clone, Debug)] +pub struct RecordingTask { + /// uuid of the task for identification. + pub uuid: String, + /// Server-resolved media kind. Decides the execution strategy. + pub kind: RecordingKind, + /// `file_dir` is the directory where the file should be placed. + pub file_dir: PathBuf, + /// `file_path` is the complete path including the filename. + pub file_path: PathBuf, + /// filename is the filename. + pub filename: String, + /// Server-resolved source url. + pub url: reqwest::Url, + /// finished is true when the task reached a terminal state. + pub finished: bool, + /// Bytes transferred so far. + pub size: u64, + /// Total size in bytes (from the Content-Length header), when known. + pub total_size: Option, + /// Paused state. Only VOD/Series can be paused. + pub paused: bool, + /// Optional error if something goes wrong while running the task. + pub error: Option, + /// Task state. + pub state: RecordingTaskState, + /// The input source name used to acquire a provider connection. + pub input_name: Option>, + /// Priority for provider connection preemption (lower = higher priority). + pub priority: i8, + /// Consecutive retry attempts for transient failures. + pub retry_attempts: u8, + /// Unix timestamp of the next retry attempt while waiting. + pub next_retry_at: Option, + /// Recording metadata. Every task carries it; the programme window is + /// populated for `RecordingKind::Live` only. + pub recording: RecordingMetadata, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct PersistedRecordingQueue { + pub queue: Vec, + pub scheduled: Vec, + pub active: Vec, + pub finished: Vec, + /// Monotonic revision. Increments once per committed queue mutation. + /// The in-memory `RecordingQueue` mirrors this counter via an `AtomicU64`. + #[serde(default)] + pub revision: QueueRevision, +} + +fn generate_recording_task_id() -> String { + let now_nanos = SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |duration| duration.as_nanos()); + let counter = RECORDING_TASK_ID_COUNTER.fetch_add(1, Ordering::Relaxed); + format!("{now_nanos:032x}{counter:016x}") +} + +impl RecordingTask { + /// Build a task for a server-resolved source. Live tasks start in + /// `Scheduled`; VOD/Series tasks start in `Queued`. + pub fn new( + kind: RecordingKind, + req_url: &str, + req_filename: &str, + recording_cfg: &RecordingConfig, + input_name: Option>, + priority: i8, + recording: RecordingMetadata, + ) -> Option { + let url = reqwest::Url::parse(req_url).ok()?; + let tmp_filename = sanitize_filename_chars(req_filename, true).replace("__", "_").replace("_-_", "-"); + let filename_path = Path::new(&tmp_filename); + let file_stem = + filename_path.file_stem().and_then(OsStr::to_str).unwrap_or("").trim_matches(FILENAME_TRIM_PATTERNS); + let file_ext = filename_path.extension().and_then(OsStr::to_str).unwrap_or(""); + + let base_filename = if file_ext.is_empty() { file_stem.to_string() } else { format!("{file_stem}.{file_ext}") }; + let root = Path::new(recording_cfg.directory.as_str()); + let grouping = recording_grouping(kind, &recording, recording_cfg, file_stem); + let mut relative = recording_path::build_relative_path( + kind, + recording_cfg.organize_into_directories, + &grouping, + &base_filename, + ) + .ok()?; + // A file already on disk under this name belongs to an earlier + // recording; the repository reservation resolves logical collisions, + // this only avoids clobbering something already written. Bounded: each + // probe is a filesystem call on the caller's thread. + let base = relative.clone(); + let mut index = 0; + while root.join(&relative).is_file() { + index += 1; + if index > MAX_COLLISION_PROBES { + return None; + } + relative = recording_path::with_collision_suffix(&base, index); + } + let file_path = recording_path::resolve_under_root(root, &relative).ok()?; + let file_dir = file_path.parent().map(Path::to_path_buf)?; + let filename = relative.file_name().and_then(|name| name.to_str())?.to_string(); + + file_path.to_str()?; + let mut recording = recording; + recording.relative_path = Some(relative.to_string_lossy().into_owned()); + + Some(Self { + uuid: generate_recording_task_id(), + kind, + file_dir, + file_path, + filename, + url, + finished: false, + size: 0, + total_size: None, + paused: false, + error: None, + state: if kind.is_scheduled() { RecordingTaskState::Scheduled } else { RecordingTaskState::Queued }, + input_name, + priority, + retry_attempts: 0, + next_retry_at: None, + recording, + }) + } + + /// Padded start of a scheduled Live window. + pub fn scheduled_start(&self) -> Option { self.recording.scheduled_start } + + /// Padded end of a scheduled Live window. + pub fn scheduled_end(&self) -> Option { self.recording.scheduled_end } + + /// Length of the padded Live window, when both bounds are known. + pub fn scheduled_duration_secs(&self) -> Option { + let (start, end) = self.recording.scheduled_start.zip(self.recording.scheduled_end)?; + u64::try_from(end.saturating_sub(start)).ok() + } + + pub fn owner_id(&self) -> &UserId { self.recording.owner_id() } + + /// Public projection for one viewer. Returns `None` when the viewer is + /// neither the owner nor entitled to see a shared task; the owner id is + /// disclosed only to the owner itself. + pub fn to_owner_view(&self, claims: &Claims, shared_visible: bool) -> Option { + let is_owner = claims.subject_id.as_ref().is_some_and(|subject| subject == self.owner_id()); + if !is_owner && !shared_visible { + return None; + } + Some(self.to_view(is_owner)) + } + + /// Projection without a viewer check. Callers that already authorized the + /// viewer pass `is_owner` explicitly. + pub fn to_view(&self, is_owner: bool) -> RecordingTaskDto { + let meta = &self.recording; + RecordingTaskDto { + id: self.uuid.clone(), + title: meta.program_title.clone().unwrap_or_else(|| self.filename.clone()), + kind: self.kind, + priority: match self.priority.cmp(&0) { + std::cmp::Ordering::Less => TaskPriorityDto::High, + std::cmp::Ordering::Equal => TaskPriorityDto::Normal, + std::cmp::Ordering::Greater => TaskPriorityDto::Background, + }, + status: self.state.into(), + retry_attempts: self.retry_attempts, + transferred_bytes: self.size, + total_bytes: self.total_size, + next_retry_at: self.next_retry_at, + error: self.error.clone(), + restart_from_beginning_required: self.state == RecordingTaskState::Failed + && self.error.as_deref() == Some(super::super::recording_transfer::RANGE_UNSUPPORTED_ERROR), + owner_id: is_owner.then(|| meta.owner_id().clone()), + visibility: meta.visibility, + channel_id: meta.channel_id.clone(), + channel_name: meta.channel_name.clone(), + program_title: meta.program_title.clone(), + program_start: meta.program_start, + program_end: meta.program_end, + scheduled_start: meta.scheduled_start, + scheduled_end: meta.scheduled_end, + pre_roll_secs: meta.pre_roll_secs, + post_roll_secs: meta.post_roll_secs, + completed_at: meta.completed_at, + filename: meta.filename().map(str::to_string), + epg: meta.epg.clone(), + rule_id: meta.provenance.rule_id.clone(), + occurrence_key: meta.provenance.occurrence_key.clone(), + allowed_actions: recording_transition::allowed_actions(self.kind, self.state), + } + } +} diff --git a/backend/dvr/src/recording/recording_queue/tests.rs b/backend/dvr/src/recording/recording_queue/tests.rs new file mode 100644 index 000000000..fafe9bc18 --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/tests.rs @@ -0,0 +1,262 @@ +use super::*; +use crate::recording::recording_transition; +use chrono::Utc; +use shared::model::{ + recording::{RecordingOwner, RecordingSource, RecordingVisibility}, + QueueRevision, RecordingKind, RecordingMetadata, UserId, +}; +use std::{ + path::{Path, PathBuf}, + sync::atomic::Ordering, + time::{SystemTime, UNIX_EPOCH}, +}; +use tokio::time::{timeout, Duration}; +use tuliprox_core::model::RecordingConfig; + +fn temp_state_file(name: &str) -> PathBuf { + let nanos = SystemTime::now().duration_since(UNIX_EPOCH).expect("time").as_nanos(); + let dir = std::env::temp_dir().join(format!("tuliprox_{name}_{nanos}")); + std::fs::create_dir_all(&dir).expect("create state dir"); + dir +} + +fn live_meta(owner: &str, start: i64, duration: u64) -> RecordingMetadata { + RecordingMetadata::new_live( + RecordingOwner::User(UserId::from(owner)), + RecordingVisibility::Private, + RecordingSource::new("target", "v", "input-a"), + start, + start.saturating_add(duration.cast_signed()), + 0, + 0, + ) +} + +fn media_meta(owner: &str) -> RecordingMetadata { + RecordingMetadata::new_media( + RecordingOwner::User(UserId::from(owner)), + RecordingVisibility::Private, + RecordingSource::new("target", "v", "input-a"), + String::new(), + ) +} + +fn task(uuid: &str, kind: RecordingKind, state: RecordingTaskState) -> RecordingTask { + RecordingTask { + uuid: uuid.to_string(), + kind, + file_dir: PathBuf::from("/tmp"), + file_path: PathBuf::from(format!("/tmp/{uuid}.ts")), + filename: format!("{uuid}.ts"), + url: reqwest::Url::parse(&format!("https://example.com/{uuid}")).expect("valid url"), + finished: false, + size: 0, + total_size: None, + paused: false, + error: None, + state, + input_name: None, + priority: 0, + retry_attempts: 0, + next_retry_at: None, + recording: match kind { + RecordingKind::Live => live_meta("web:alice", 1_700_000_000, 3_600), + _ => media_meta("web:alice"), + }, + } +} + +fn identified(uuid: &str, identity: &str, state: RecordingTaskState) -> PersistedRecordingTask { + let mut persisted = RecordingQueue::to_persisted(&task(uuid, RecordingKind::Vod, state)); + persisted.media_identity = identity.to_owned(); + persisted +} + +fn organized_task(kind: RecordingKind, filename: &str, recording: RecordingMetadata) -> RecordingTask { + let dir = tempfile::tempdir().expect("tempdir"); + let cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { + directory: Some(dir.path().to_string_lossy().into_owned()), + organize_into_directories: true, + ..Default::default() + }); + RecordingTask::new(kind, "https://example.com/item", filename, &cfg, None, 0, recording).expect("task") +} + +/// Age the first recovery generation so the next commit wants a +/// checkpoint, and block the directory that checkpoint would be written +/// to: the checkpoint fails after the commit itself succeeded. +fn force_a_failing_checkpoint(dir: &Path) { + let recovery = dir.join("recordings_library_recovery"); + let manifest = recovery.join("gen-00000000000000000001/manifest.bin"); + let bytes = std::fs::read(&manifest).expect("manifest"); + let mut payload: serde_json::Value = serde_json::from_slice(&bytes[40..]).expect("manifest payload"); + payload["created_at_unix"] = serde_json::json!(0); + let payload = serde_json::to_vec(&payload).expect("encode"); + let mut framed = b"TRJ1".to_vec(); + framed.extend_from_slice(&u32::try_from(payload.len()).expect("length").to_le_bytes()); + framed.extend_from_slice(blake3::hash(&payload).as_bytes()); + framed.extend_from_slice(&payload); + std::fs::write(&manifest, framed).expect("rewrite manifest"); + std::fs::write(recovery.join("gen-00000000000000000002"), b"blocks the next generation").expect("block"); +} + +// --- Transactional queue mutation boundary --- + +fn make_test_recording_task(uuid: &str, file_path: PathBuf) -> RecordingTask { + make_test_task_of_kind(uuid, file_path, RecordingKind::Live) +} + +/// Pause/resume are VOD/Series-only, so those tests need a resumable task. +fn make_test_transfer_task(uuid: &str, file_path: PathBuf) -> RecordingTask { + make_test_task_of_kind(uuid, file_path, RecordingKind::Vod) +} + +fn make_test_task_of_kind(uuid: &str, file_path: PathBuf, kind: RecordingKind) -> RecordingTask { + let mut task = task(uuid, kind, RecordingTaskState::Running); + task.file_dir = file_path.parent().unwrap_or(Path::new("/")).to_path_buf(); + task.file_path = file_path; + task.filename = format!("{uuid}.ts"); + task +} + +/// Read back what the repository actually committed, rather than +/// trusting the in-memory mirror. +async fn committed(queue: &RecordingQueue) -> (u64, Vec) { + let repository = queue.repository.clone().expect("queue is repository backed"); + let snapshot = tokio::task::spawn_blocking(move || { + let mut guard = repository.lock().expect("repository lock"); + guard.load() + }) + .await + .expect("join") + .expect("load"); + (snapshot.queue_revision, snapshot.tasks.iter().map(|task| task.uuid.clone()).collect()) +} + +// --- Filename rendering + collision reservation --- + +fn collect_existing_relative_paths(candidate: &PersistedRecordingQueue) -> Vec { + let mut out = Vec::new(); + for d in &candidate.queue { + if let Some(p) = &d.recording.relative_path { + out.push(p.clone()); + } + } + for d in &candidate.scheduled { + if let Some(p) = &d.recording.relative_path { + out.push(p.clone()); + } + } + for d in &candidate.active { + if let Some(p) = &d.recording.relative_path { + out.push(p.clone()); + } + } + for d in &candidate.finished { + if let Some(p) = &d.recording.relative_path { + out.push(p.clone()); + } + } + out +} + +/// Reserve a unique relative path for a new recording inside the +/// queue mutation boundary. The candidate is the in-memory +/// `PersistedRecordingQueue` the closure is building; the helper +/// collects all already-reserved `relative_path` values, applies +/// the supplied stem, and appends a numbered collision suffix +/// (`_1`, `_2`, …) until the result is unique. The reserved path +/// is also written to the recording metadata so a later collision +/// created externally is detected by the worker at execute time. +fn reserve_recording_relative_path( + candidate: &mut PersistedRecordingQueue, + stem: &str, + recording_uuid: &str, +) -> String { + // If this recording already has a reserved path (e.g., a retry + // or an edit), keep it. Re-reserving on a re-entrant call must + // not bump the suffix because of the caller's own previous + // entry. + let existing_self = find_recording_relative_path(candidate, recording_uuid); + if let Some(prior) = existing_self { + return prior; + } + let existing = collect_existing_relative_paths(candidate); + let reserved = shared::utils::next_collision_suffix(stem, &existing); + for d in &mut candidate.queue { + if d.uuid == recording_uuid { + d.recording.relative_path = Some(reserved.clone()); + } + } + for d in &mut candidate.scheduled { + if d.uuid == recording_uuid { + d.recording.relative_path = Some(reserved.clone()); + } + } + if let Some(d) = candidate.active.iter_mut().find(|task| task.uuid == recording_uuid) { + d.recording.relative_path = Some(reserved.clone()); + } + for d in &mut candidate.finished { + if d.uuid == recording_uuid { + d.recording.relative_path = Some(reserved.clone()); + } + } + reserved +} + +fn find_recording_relative_path(candidate: &PersistedRecordingQueue, recording_uuid: &str) -> Option { + for d in &candidate.queue { + if d.uuid == recording_uuid { + return d.recording.relative_path.clone(); + } + } + for d in &candidate.scheduled { + if d.uuid == recording_uuid { + return d.recording.relative_path.clone(); + } + } + for d in &candidate.active { + if d.uuid == recording_uuid { + return d.recording.relative_path.clone(); + } + } + for d in &candidate.finished { + if d.uuid == recording_uuid { + return d.recording.relative_path.clone(); + } + } + None +} + +fn persisted_recording(uuid: &str, meta: RecordingMetadata) -> PersistedRecordingTask { + PersistedRecordingTask { + media_identity: String::new(), + partition: RecordingPartition::default(), + uuid: uuid.to_string(), + kind: RecordingKind::Live, + file_dir: PathBuf::from("/tmp"), + file_path: PathBuf::from(format!("/tmp/{uuid}.ts")), + filename: format!("{uuid}.ts"), + url: format!("https://example.com/{uuid}"), + finished: false, + size: 0, + total_size: None, + paused: false, + error: None, + state: RecordingTaskState::Scheduled, + input_name: None, + priority: 0, + retry_attempts: 0, + next_retry_at: None, + recording: meta, + } +} + +mod admission; +mod lifecycle; +mod policy; +mod publication; +mod recovery; +mod storage; +mod streaming; +mod transport; diff --git a/backend/dvr/src/recording/recording_queue/tests/admission.rs b/backend/dvr/src/recording/recording_queue/tests/admission.rs new file mode 100644 index 000000000..c317365c9 --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/tests/admission.rs @@ -0,0 +1,26 @@ +use super::*; + +#[tokio::test] +async fn mutate_serializes_concurrent_calls() { + let dir = tempfile::tempdir().expect("tempdir"); + let state_file = dir.path().to_path_buf(); + let queue = std::sync::Arc::new( + RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"), + ); + let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(5)); + + let mut handles = Vec::new(); + for _ in 0..5 { + let q = std::sync::Arc::clone(&queue); + let b = std::sync::Arc::clone(&barrier); + handles.push(tokio::spawn(async move { + b.wait().await; + mutate(&q, |_candidate| Ok(())).await + })); + } + for h in handles { + assert!(h.await.expect("join").is_ok(), "all mutations should succeed"); + } + let final_rev = queue.revision.load(Ordering::SeqCst); + assert_eq!(final_rev, 5); +} diff --git a/backend/dvr/src/recording/recording_queue/tests/lifecycle.rs b/backend/dvr/src/recording/recording_queue/tests/lifecycle.rs new file mode 100644 index 000000000..77b49826a --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/tests/lifecycle.rs @@ -0,0 +1,113 @@ +use super::*; + +#[tokio::test] +async fn pause_and_resume_keep_active_download_resumable() { + let queue = RecordingQueue::new(); + let active = RecordingTask { + size: 42, + total_size: Some(100), + state: RecordingTaskState::Running, + ..task("id", RecordingKind::Vod, RecordingTaskState::Running) + }; + + *queue.active.write().await = vec![active]; + queue.pause_active("id").await.expect("pause active"); + + let paused = queue.active.read().await.first().cloned().expect("active download"); + assert_eq!(paused.state, RecordingTaskState::Paused); + assert!(paused.paused); + assert!(!paused.finished); + + queue.resume_active("id").await.expect("resume active"); + + let resumed = queue.active.read().await.first().cloned().expect("active download"); + assert_eq!(resumed.state, RecordingTaskState::Running); + assert!(!resumed.paused); + assert!(!resumed.finished); +} + +#[tokio::test] +async fn cancel_leaves_a_running_task_cancelling_until_the_worker_lets_go() { + let queue = RecordingQueue::new(); + let worker = queue.worker("id"); + let active = task("id", RecordingKind::Vod, RecordingTaskState::Running); + + *queue.active.write().await = vec![active]; + assert_eq!(queue.cancel_requested("id").await.expect("cancel active"), Some(false)); + + let cancelled = queue.active.read().await.first().cloned().expect("active download"); + assert_eq!(cancelled.state, RecordingTaskState::Cancelling); + assert_eq!(*worker.control_signal.read().await, RecordingControl::Cancel); + assert!(!cancelled.finished); + assert_eq!(cancelled.error.as_deref(), Some("Cancelled by user")); + assert!(queue.finished.read().await.is_empty()); +} + +#[tokio::test] +async fn cancelling_a_paused_task_releases_idle_worker_state() { + for running in [None, Some(false), Some(true)] { + let queue = RecordingQueue::new(); + let mut paused = task("paused", RecordingKind::Vod, RecordingTaskState::Paused); + paused.paused = true; + queue.active.write().await.push(paused); + if let Some(running) = running { + *queue.worker("paused").running.write().await = running; + } + let immediate = running != Some(true); + assert_eq!(queue.cancel_requested("paused").await.expect("cancel paused"), Some(immediate)); + if immediate { + assert!(queue.active.read().await.is_empty()); + assert_eq!(queue.finished.read().await[0].state, RecordingTaskState::Cancelled); + } else { + assert!(queue.finished.read().await.is_empty(), "the stream has not closed yet"); + assert_eq!(queue.active.read().await[0].state, RecordingTaskState::Cancelling); + assert_eq!(*queue.worker("paused").control_signal.read().await, RecordingControl::Cancel); + mutate(&queue, |candidate| { + let mut task = candidate.active.remove(0); + task.state = RecordingTaskState::Cancelled; + task.finished = true; + candidate.finished.push(task); + Ok(()) + }) + .await + .expect("worker closes and commits"); + } + let retained = queue.workers.lock().expect("worker map").contains_key("paused"); + assert_eq!(retained, running == Some(true), "only an exiting worker retains its state"); + queue.release_worker("paused").await; + assert!(!queue.workers.lock().expect("worker map").contains_key("paused")); + } +} + +#[tokio::test] +async fn worker_claims_do_not_create_state_for_missing_or_paused_tasks() { + let queue = RecordingQueue::new(); + assert!(queue.claim_worker("missing").await.is_none()); + let mut paused = task("paused", RecordingKind::Vod, RecordingTaskState::Paused); + paused.paused = true; + queue.active.write().await.push(paused); + assert!(queue.claim_worker("paused").await.is_none()); + assert!(queue.workers.lock().expect("worker map").is_empty()); + assert!(queue.resume_active("paused").await.expect("resume")); + assert!(queue.claim_worker("paused").await.is_some()); + assert!(queue.claim_worker("paused").await.is_none(), "a worker can be claimed only once"); +} + +#[tokio::test] +async fn pause_active_routes_through_mutate() { + let dir = tempfile::tempdir().expect("tempdir"); + let state_file = dir.path().to_path_buf(); + let queue = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); + let task = make_test_transfer_task("rec-1", dir.path().join("a.ts")); + { + let mut active = queue.active.write().await; + *active = vec![task]; + } + let prior_revision = queue.revision.load(Ordering::SeqCst); + queue.pause_active("rec-1").await.expect("pause_active"); + let active = queue.active.read().await.first().cloned().expect("active"); + assert!(active.paused); + assert_eq!(active.state, RecordingTaskState::Paused); + assert!(active.next_retry_at.is_none()); + assert_eq!(queue.revision.load(Ordering::SeqCst), prior_revision + 1); +} diff --git a/backend/dvr/src/recording/recording_queue/tests/policy.rs b/backend/dvr/src/recording/recording_queue/tests/policy.rs new file mode 100644 index 000000000..feb57ac94 --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/tests/policy.rs @@ -0,0 +1,548 @@ +use super::*; + +#[tokio::test] +async fn promotion_precheck_matches_promotion_for_mixed_queues() { + fn fixture(uuid: &str, media: usize, kind: RecordingKind, state: RecordingTaskState) -> RecordingTask { + let mut entry = task(&format!("media-{media}"), kind, state); + entry.uuid = uuid.to_string(); + entry.recording = media_meta("web:alice"); + entry.recording.source.virtual_id = media.to_string(); + match media % 3 { + 1 => { + entry.recording.program_start = Some(10); + entry.recording.program_end = Some(20); + } + 2 => { + entry.recording.provenance.rule_id = Some("rule".to_string()); + entry.recording.provenance.occurrence_key = Some(media.to_string()); + } + _ => {} + } + entry.paused = state == RecordingTaskState::Paused; + entry.finished = + matches!(state, RecordingTaskState::Completed | RecordingTaskState::Failed | RecordingTaskState::Cancelled); + entry + } + let kinds = [RecordingKind::Live, RecordingKind::Vod, RecordingKind::Series]; + let active_states = [ + RecordingTaskState::Running, + RecordingTaskState::Paused, + RecordingTaskState::WaitingForCapacity, + RecordingTaskState::RetryWaiting, + ]; + let finished_states = [RecordingTaskState::Completed, RecordingTaskState::Failed, RecordingTaskState::Cancelled]; + let mut random = fastrand::Rng::with_seed(0x7265_636f_7264); + let (mut promotable, mut blocked, mut attachment_only) = (0, 0, 0); + for case in 0..512 { + let queue = RecordingQueue::new(); + for index in 0..random.usize(0..12) { + queue.queue.lock().await.push_back(fixture( + &format!("queued-{index}"), + random.usize(0..6), + kinds[random.usize(0..kinds.len())], + RecordingTaskState::Queued, + )); + } + for index in 0..random.usize(0..4) { + queue.active.write().await.push(fixture( + &format!("active-{index}"), + random.usize(0..6), + kinds[random.usize(0..kinds.len())], + active_states[random.usize(0..active_states.len())], + )); + } + for index in 0..random.usize(0..6) { + queue.finished.write().await.push(fixture( + &format!("finished-{index}"), + random.usize(0..6), + kinds[random.usize(0..kinds.len())], + finished_states[random.usize(0..finished_states.len())], + )); + } + let precheck = queue.has_promotable_queued().await; + let mut snapshot = queue.snapshot_current(QueueRevision(0)).await; + let queued_before = snapshot.queue.len(); + let promoted = promote_from_queue(&mut snapshot).is_some(); + let attached = snapshot.queue.len() < queued_before; + assert_eq!(precheck, promoted || attached, "case {case}: {snapshot:?}"); + if precheck { + promotable += 1; + } else { + blocked += 1; + } + if !promoted && attached { + attachment_only += 1; + } + } + assert!(promotable > 0 && blocked > 0 && attachment_only > 0); +} + +#[test] +fn bulk_transfers_stay_serial_while_live_tasks_promote_in_parallel() { + let mut candidate = PersistedRecordingQueue::default(); + for index in 0..50 { + let mut transfer = + identified(&format!("transfer-{index}"), &format!("media-{index}"), RecordingTaskState::Queued); + transfer.kind = if index % 2 == 0 { RecordingKind::Vod } else { RecordingKind::Series }; + transfer.input_name = None; + candidate.queue.push(transfer); + } + for index in 0..2 { + let mut live = identified(&format!("live-{index}"), &format!("live-media-{index}"), RecordingTaskState::Queued); + live.kind = RecordingKind::Live; + candidate.queue.push(live); + } + while promote_from_queue(&mut candidate).is_some() {} + assert_eq!(candidate.active.len(), 3); + assert_eq!(candidate.active.iter().filter(|task| task.kind != RecordingKind::Live).count(), 1); + assert_eq!(candidate.queue.len(), 49); + candidate.active.retain(|task| task.kind == RecordingKind::Live); + assert_eq!(promote_from_queue(&mut candidate).map(|(uuid, _)| uuid).as_deref(), Some("transfer-1")); +} + +#[test] +fn the_dto_offers_exactly_what_the_transition_graph_permits() { + // The frontend renders its buttons straight from this set, so a + // mismatch here is a button that errors when pressed. + for kind in [RecordingKind::Live, RecordingKind::Vod, RecordingKind::Series] { + for state in [ + RecordingTaskState::Scheduled, + RecordingTaskState::Queued, + RecordingTaskState::WaitingForCapacity, + RecordingTaskState::Running, + RecordingTaskState::Paused, + RecordingTaskState::RetryWaiting, + RecordingTaskState::Completed, + RecordingTaskState::Failed, + RecordingTaskState::Cancelled, + ] { + let dto = task("t", kind, state).to_view(true); + assert_eq!( + dto.allowed_actions, + recording_transition::allowed_actions(kind, state), + "{kind} in {} disagrees with the graph", + state.label() + ); + } + } +} + +#[tokio::test] +async fn a_transfer_runs_at_the_strongest_priority_anyone_attached_asked_for() { + // Alice's background request is producing the file; Bob wants the same + // media urgently. One transfer serves both, so it must not sit in the + // background queue while Bob waits for bytes already being fetched. + let dir = tempfile::TempDir::new().expect("tempdir"); + let queue = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open recording repository"); + let mut background = task("alice", RecordingKind::Vod, RecordingTaskState::Running); + background.priority = 5; + let mut foreground = task("bob", RecordingKind::Vod, RecordingTaskState::Queued); + foreground.priority = -3; + // Same media: a VOD identity keys on the url, so both entries name one. + foreground.url = background.url.clone(); + let (background, foreground) = + (RecordingQueue::to_persisted(&background), RecordingQueue::to_persisted(&foreground)); + assert_eq!(background.media_identity, foreground.media_identity, "fixture must share one media"); + mutate(&queue, move |candidate| { + candidate.active = vec![background.clone()]; + candidate.queue.push(foreground.clone()); + Ok(()) + }) + .await + .expect("seed"); + + let (_input, priority) = queue.active_scheduling_priority("alice").await.expect("an active transfer"); + assert_eq!(priority, -3, "the transfer inherits Bob's urgency"); +} + +#[tokio::test] +async fn an_unrelated_urgent_recording_does_not_lift_this_one() { + let dir = tempfile::TempDir::new().expect("tempdir"); + let queue = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open recording repository"); + let mut background = task("alice", RecordingKind::Vod, RecordingTaskState::Running); + background.priority = 5; + let mut other = task("bob", RecordingKind::Vod, RecordingTaskState::Queued); + other.priority = -3; + let (background, other) = (RecordingQueue::to_persisted(&background), RecordingQueue::to_persisted(&other)); + assert_ne!(background.media_identity, other.media_identity, "fixture must be different media"); + mutate(&queue, move |candidate| { + candidate.active = vec![background.clone()]; + candidate.queue.push(other.clone()); + Ok(()) + }) + .await + .expect("seed"); + + let (_input, priority) = queue.active_scheduling_priority("alice").await.expect("an active transfer"); + assert_eq!(priority, 5, "priority is shared only with entries on the same file"); +} + +#[test] +fn a_queued_entry_attaches_to_a_file_another_entry_finished() { + let candidate = PersistedRecordingQueue { + finished: vec![identified("a", "film-42", RecordingTaskState::Completed)], + ..PersistedRecordingQueue::default() + }; + let queued = identified("b", "film-42", RecordingTaskState::Queued); + assert_eq!(promotion_decision(&candidate, &queued), PromotionDecision::AttachTo(0)); +} + +#[test] +fn a_failed_sibling_is_not_attached_to() { + // There is no file to adopt, so the entry has to run. + let candidate = PersistedRecordingQueue { + finished: vec![identified("a", "film-42", RecordingTaskState::Failed)], + ..PersistedRecordingQueue::default() + }; + let queued = identified("b", "film-42", RecordingTaskState::Queued); + assert_eq!(promotion_decision(&candidate, &queued), PromotionDecision::Execute); +} + +#[test] +fn an_unidentified_entry_never_matches_another() { + // An empty identity means the identity could not be resolved. Treating + // two of them as the same recording would merge two different files. + let candidate = PersistedRecordingQueue { + finished: vec![identified("a", "", RecordingTaskState::Completed)], + active: vec![identified("c", "", RecordingTaskState::Running)], + ..PersistedRecordingQueue::default() + }; + let queued = identified("b", "", RecordingTaskState::Queued); + assert_eq!(promotion_decision(&candidate, &queued), PromotionDecision::Execute); +} + +#[test] +fn different_media_does_not_attach() { + let candidate = PersistedRecordingQueue { + finished: vec![identified("a", "film-42", RecordingTaskState::Completed)], + ..PersistedRecordingQueue::default() + }; + let queued = identified("b", "film-99", RecordingTaskState::Queued); + assert_eq!(promotion_decision(&candidate, &queued), PromotionDecision::Execute); +} + +#[test] +fn the_entry_behind_a_finished_transfer_attaches_instead_of_downloading_again() { + // The live path: Alice's transfer completes and Bob is next in the + // queue for the same film. Taking the queue head blindly would start a + // second transfer over the file Alice just produced. + let mut candidate = PersistedRecordingQueue { + finished: vec![identified("alice", "film-42", RecordingTaskState::Completed)], + queue: vec![identified("bob", "film-42", RecordingTaskState::Queued)], + ..PersistedRecordingQueue::default() + }; + let promoted = promote_from_queue(&mut candidate); + assert!(promoted.is_none(), "nothing needs to run"); + assert!(candidate.active.is_empty()); + assert!(candidate.queue.is_empty(), "bob left the queue"); + let bob = candidate.finished.iter().find(|task| task.uuid == "bob").expect("bob is filed"); + assert_eq!(bob.state, RecordingTaskState::Completed); +} + +#[test] +fn an_entry_for_other_media_behind_a_finished_transfer_still_runs() { + let mut candidate = PersistedRecordingQueue { + finished: vec![identified("alice", "film-42", RecordingTaskState::Completed)], + queue: vec![identified("bob", "film-99", RecordingTaskState::Queued)], + ..PersistedRecordingQueue::default() + }; + let promoted = promote_from_queue(&mut candidate); + assert_eq!(promoted.map(|(uuid, _)| uuid), Some("bob".to_string())); + assert_eq!(candidate.active.first().map(|task| task.uuid.as_str()), Some("bob")); +} + +#[test] +fn attachable_entries_are_skipped_to_reach_one_that_must_run() { + let mut candidate = PersistedRecordingQueue { + finished: vec![identified("alice", "film-42", RecordingTaskState::Completed)], + queue: vec![ + identified("bob", "film-42", RecordingTaskState::Queued), + identified("carol", "film-42", RecordingTaskState::Queued), + identified("dave", "film-99", RecordingTaskState::Queued), + ], + ..PersistedRecordingQueue::default() + }; + let promoted = promote_from_queue(&mut candidate); + assert_eq!(promoted.map(|(uuid, _)| uuid), Some("dave".to_string())); + assert!(candidate.queue.is_empty(), "every entry was dispatched"); + // Bob and Carol were filed against Alice's file without running. + assert_eq!(candidate.finished.len(), 3); +} + +#[tokio::test] +async fn a_live_window_that_has_not_opened_acquires_nothing() { + // Promotion is what puts a capture in front of the provider. A window + // that starts in an hour must not reach it, or the DVR holds a slot for + // an hour of nothing and records the wrong programme. + let dir = tempfile::TempDir::new().expect("tempdir"); + let queue = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open recording repository"); + let now = 1_700_000_000; + let mut upcoming = task("live", RecordingKind::Live, RecordingTaskState::Scheduled); + upcoming.recording.scheduled_start = Some(now + 3_600); + upcoming.recording.scheduled_end = Some(now + 7_200); + let upcoming = RecordingQueue::to_persisted(&upcoming); + mutate(&queue, move |candidate| { + candidate.scheduled.push(upcoming.clone()); + Ok(()) + }) + .await + .expect("seed"); + + assert_eq!(queue.promote_due_scheduled(now).await, 0, "an hour early is not due"); + assert!(queue.queue.lock().await.is_empty(), "and nothing is queued for a provider"); + + // At the padded start it becomes work. + assert_eq!(queue.promote_due_scheduled(now + 3_600).await, 1, "due at its padded start"); + assert_eq!(queue.queue.lock().await.len(), 1); +} + +#[tokio::test] +async fn promote_due_scheduled_moves_only_ready_recordings_to_queue() { + let queue = RecordingQueue::new(); + let due = RecordingTask { + size: 123, + total_size: Some(999), + error: Some("old error".to_string()), + state: RecordingTaskState::Scheduled, + recording: live_meta("web:alice", 100, 60), + ..task("due", RecordingKind::Live, RecordingTaskState::Scheduled) + }; + let future = RecordingTask { + state: RecordingTaskState::Scheduled, + recording: live_meta("web:alice", 200, 60), + ..task("future", RecordingKind::Live, RecordingTaskState::Scheduled) + }; + + queue.scheduled.write().await.extend([due, future]); + let revision = queue.revision.load(Ordering::SeqCst); + + let promoted = queue.promote_due_scheduled(150).await; + + assert_eq!(promoted, 1); + assert_eq!(queue.revision.load(Ordering::SeqCst), revision + 1); + let queued_items = queue.queue.lock().await.iter().cloned().collect::>(); + assert_eq!(queued_items.len(), 1); + assert_eq!(queued_items[0].uuid, "due"); + assert_eq!(queued_items[0].state, RecordingTaskState::Queued); + assert_eq!(queued_items[0].size, 0); + assert!(queued_items[0].error.is_none()); + let scheduled_items = queue.scheduled.read().await.clone(); + assert_eq!(scheduled_items.len(), 1); + assert_eq!(scheduled_items[0].uuid, "future"); +} + +#[tokio::test] +async fn promote_due_scheduled_marks_expired_recordings_failed() { + let queue = RecordingQueue::new(); + let expired = RecordingTask { + state: RecordingTaskState::Scheduled, + recording: live_meta("web:alice", 100, 60), + ..task("expired", RecordingKind::Live, RecordingTaskState::Scheduled) + }; + + queue.scheduled.write().await.push(expired); + let promoted = queue.promote_due_scheduled(200).await; + + assert_eq!(promoted, 1); + assert!(queue.queue.lock().await.is_empty()); + let finished = queue.finished.read().await.clone(); + assert_eq!(finished.len(), 1); + assert_eq!(finished[0].uuid, "expired"); + assert_eq!(finished[0].state, RecordingTaskState::Failed); + assert!(finished[0].finished); + assert_eq!(finished[0].error.as_deref(), Some("Recording window already expired")); +} + +#[test] +fn recording_uuid_differs_for_same_url_with_different_start_times() { + let cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { + directory: Some("/tmp".to_string()), + ..Default::default() + }); + + let first = RecordingTask::new( + RecordingKind::Live, + "https://example.com/live/1", + "recording_1.ts", + &cfg, + None, + 0, + live_meta("web:alice", 1_700_000_000, 5_400), + ) + .expect("first recording"); + let second = RecordingTask::new( + RecordingKind::Live, + "https://example.com/live/1", + "recording_2.ts", + &cfg, + None, + 0, + live_meta("web:alice", 1_700_005_400, 5_400), + ) + .expect("second recording"); + + assert_ne!(first.uuid, second.uuid); +} + +#[test] +fn organized_series_episodes_share_the_series_folder_in_their_group() { + let episode = |title: &str| { + let mut meta = media_meta("web:alice"); + meta.program_title = Some(title.to_string()); + meta.series_name = Some("The Show".to_string()); + meta.group = Some("Crime".to_string()); + meta + }; + let first = organized_task(RecordingKind::Series, "The Show S01E01.mkv", episode("Pilot")); + let second = organized_task(RecordingKind::Series, "The Show S02E05.mkv", episode("Finale")); + assert_eq!(first.recording.relative_path.as_deref(), Some("Crime/The Show/The_Show_S01E01.mkv")); + assert_eq!(second.recording.relative_path.as_deref(), Some("Crime/The Show/The_Show_S02E05.mkv")); +} + +#[tokio::test] +async fn promote_due_scheduled_places_due_recordings_ahead_of_existing_queue_items() { + let queue = RecordingQueue::new(); + queue.queue.lock().await.push_back(task("existing", RecordingKind::Vod, RecordingTaskState::Queued)); + queue.scheduled.write().await.extend([ + RecordingTask { + state: RecordingTaskState::Scheduled, + recording: live_meta("web:alice", 100, 60), + ..task("due-first", RecordingKind::Live, RecordingTaskState::Scheduled) + }, + RecordingTask { + state: RecordingTaskState::Scheduled, + recording: live_meta("web:alice", 110, 60), + ..task("due-second", RecordingKind::Live, RecordingTaskState::Scheduled) + }, + ]); + + let promoted = queue.promote_due_scheduled(150).await; + + assert_eq!(promoted, 2); + let queued = queue.queue.lock().await.iter().map(|download| download.uuid.clone()).collect::>(); + assert_eq!(queued, vec!["due-first", "due-second", "existing"]); +} + +#[tokio::test] +async fn download_slot_wait_queue_signals_matching_waiter_by_id() { + let queue = Arc::new(RecordingSlotWaitQueue::new()); + let control_signal = Arc::new(RwLock::new(RecordingControl::None)); + let control_notify = Arc::new(Notify::new()); + + let queue_for_a = Arc::clone(&queue); + let control_signal_for_a = Arc::clone(&control_signal); + let control_notify_for_a = Arc::clone(&control_notify); + let waiter_a = tokio::spawn(async move { + queue_for_a + .wait(Some(Arc::from("input-a")), 1, control_signal_for_a.as_ref(), control_notify_for_a.as_ref()) + .await + }); + + let queue_for_b = Arc::clone(&queue); + let control_signal_for_b = Arc::clone(&control_signal); + let control_notify_for_b = Arc::clone(&control_notify); + let waiter_b = tokio::spawn(async move { + queue_for_b + .wait(Some(Arc::from("input-b")), 0, control_signal_for_b.as_ref(), control_notify_for_b.as_ref()) + .await + }); + + let waiter_b_id = loop { + let snapshots = queue.snapshots(); + if snapshots.len() == 2 { + break snapshots + .into_iter() + .find(|waiter| waiter.input_name.as_deref() == Some("input-b")) + .map(|waiter| waiter.id) + .expect("waiter id for input-b"); + } + tokio::task::yield_now().await; + }; + + assert!(queue.signal_waiter(waiter_b_id)); + assert_eq!( + timeout(Duration::from_millis(100), waiter_b).await.expect("waiter_b finished").expect("join ok"), + RecordingWaitOutcome::Signalled + ); + + *control_signal.write().await = RecordingControl::Cancel; + control_notify.notify_waiters(); + assert_eq!( + timeout(Duration::from_millis(100), waiter_a).await.expect("waiter_a finished").expect("join ok"), + RecordingWaitOutcome::Cancelled + ); +} + +#[tokio::test] +async fn a_second_request_on_an_accepted_key_is_refused_and_leaves_the_key_alone() { + let dir = tempfile::TempDir::new().expect("tempdir"); + let queue = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository"); + let record = |fingerprint: &str, recording_id: &str| PersistedIdempotency { + principal: "web:alice".to_string(), + key: "same-key".to_string(), + request_fingerprint: fingerprint.to_string(), + recording_id: recording_id.to_string(), + accepted_at: Utc::now().timestamp(), + }; + + let first = RecordingQueue::to_persisted(&task("first", RecordingKind::Vod, RecordingTaskState::Queued)); + mutate_with_idempotency(&queue, record("first-body", "first"), move |candidate| { + candidate.queue.push(first.clone()); + Ok(()) + }) + .await + .expect("first request"); + + let second = RecordingQueue::to_persisted(&task("second", RecordingKind::Vod, RecordingTaskState::Queued)); + let refused = mutate_with_idempotency(&queue, record("different-body", "second"), move |candidate| { + candidate.queue.push(second.clone()); + Ok(()) + }) + .await; + assert!(matches!(refused, Err(QueueMutationError::IdempotencyConflict)), "got {refused:?}"); + assert_eq!(queue.queue.lock().await.len(), 1, "the second request admitted nothing"); + assert_eq!( + queue.lookup_idempotency("web:alice", "same-key", "first-body").await.expect("lookup"), + IdempotencyOutcome::Replay { recording_id: "first".to_string() }, + "the accepted key still answers for the first request" + ); + + let replay = RecordingQueue::to_persisted(&task("again", RecordingKind::Vod, RecordingTaskState::Queued)); + let replayed = mutate_with_idempotency(&queue, record("first-body", "again"), move |candidate| { + candidate.queue.push(replay.clone()); + Ok(()) + }) + .await; + assert!( + matches!(&replayed, Err(QueueMutationError::IdempotentReplay { recording_id }) if recording_id == "first"), + "got {replayed:?}" + ); +} + +#[tokio::test] +async fn wait_observes_preexisting_control_before_selecting() { + let queue = RecordingQueue::new(); + let worker = queue.worker("task"); + *worker.control_signal.write().await = RecordingControl::Pause; + + let outcome = + queue.slot_waiters.wait(None, 0, worker.control_signal.as_ref(), worker.control_notify.as_ref()).await; + + assert_eq!(outcome, RecordingWaitOutcome::Paused); + assert!(queue.slot_waiters.snapshots().is_empty()); +} + +#[tokio::test] +async fn control_uuid_mismatch_is_noop_and_preserves_next_task() { + let queue = RecordingQueue::new(); + let worker = queue.worker("active"); + *queue.active.write().await = vec![make_test_recording_task("active", PathBuf::from("/tmp/active.ts"))]; + queue.queue.lock().await.push_back(make_test_recording_task("next", PathBuf::from("/tmp/next.ts"))); + + assert!(!queue.pause_active("next").await.expect("uuid mismatch")); + + assert_eq!(queue.revision.load(Ordering::SeqCst), 0); + assert_eq!(queue.active.read().await.first().map(|download| download.uuid.as_str()), Some("active")); + assert_eq!(queue.queue.lock().await.front().map(|download| download.uuid.as_str()), Some("next")); + assert_eq!(*worker.control_signal.read().await, RecordingControl::None); +} diff --git a/backend/dvr/src/recording/recording_queue/tests/publication.rs b/backend/dvr/src/recording/recording_queue/tests/publication.rs new file mode 100644 index 000000000..f84fd2e37 --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/tests/publication.rs @@ -0,0 +1,121 @@ +use super::*; + +#[tokio::test] +async fn mutate_persists_and_increments_revision_on_success() { + let dir = tempfile::tempdir().expect("tempdir"); + let state_dir = dir.path().to_path_buf(); + let queue = RecordingQueue::new_persistent(&state_dir, &state_dir).expect("open recording repository"); + assert_eq!(queue.revision.load(Ordering::SeqCst), 0); + + // Insert one recording so a candidate is non-empty. + let task = make_test_recording_task("rec-1", dir.path().join("a.ts")); + queue.queue.lock().await.push_back(task); + + let result: Result<(), QueueMutationError> = mutate(&queue, |_candidate| Ok(())).await; + assert!(result.is_ok(), "mutate should succeed: {result:?}"); + // The first committed mutation publishes revision 1, both in + // memory and in the repository. + assert_eq!(queue.revision.load(Ordering::SeqCst), 1, "counter must store the new value"); + let (revision, uuids) = committed(&queue).await; + assert_eq!(revision, 1, "repository carries the candidate's revision"); + assert_eq!(uuids, vec!["rec-1".to_string()]); +} + +#[tokio::test] +async fn mutate_invalid_candidate_keeps_existing_repository_memory_and_revision() { + let dir = tempfile::tempdir().expect("tempdir"); + let state_dir = dir.path().to_path_buf(); + let queue = RecordingQueue::new_persistent(&state_dir, &state_dir).expect("open recording repository"); + let initial = make_test_recording_task("rec-1", dir.path().join("a.ts")); + let persisted = RecordingQueue::to_persisted(&initial); + mutate(&queue, |candidate| { + candidate.queue.push(persisted); + Ok(()) + }) + .await + .expect("initial commit"); + let committed_before = committed(&queue).await; + + let result: Result<(), QueueMutationError> = mutate(&queue, |candidate| { + if let Some(download) = candidate.queue.first_mut() { + download.url = "not a url".to_string(); + } + Ok(()) + }) + .await; + + assert!(result.is_err()); + assert_eq!(queue.revision.load(Ordering::SeqCst), 1); + // A candidate that cannot be converted back is rejected before the + // repository is touched, so the committed set is byte-identical. + assert_eq!(committed(&queue).await, committed_before); + assert_eq!(queue.queue.lock().await.front().map(|download| download.uuid.as_str()), Some("rec-1")); +} + +#[tokio::test] +async fn concurrent_writers_serialize_and_increment_each_revision() { + let queue = Arc::new(RecordingQueue::new()); + queue.queue.lock().await.push_back(make_test_recording_task("remove", PathBuf::from("/tmp/remove.ts"))); + let finished = RecordingTask { + finished: true, + state: RecordingTaskState::Failed, + ..task("retry", RecordingKind::Vod, RecordingTaskState::Failed) + }; + queue.finished.write().await.push(finished); + + let remove_queue = Arc::clone(&queue); + let retry_queue = Arc::clone(&queue); + let (removed, retried) = tokio::join!(async move { remove_queue.remove_from_queue("remove").await }, async move { + retry_queue.retry_finished("retry").await + },); + + assert!(removed.expect("remove commit")); + assert!(retried.expect("retry commit")); + assert_eq!(queue.revision.load(Ordering::SeqCst), 2); + assert_eq!(queue.queue.lock().await.front().map(|download| download.uuid.as_str()), Some("retry")); + assert!(queue.finished.read().await.is_empty()); +} + +#[tokio::test] +async fn same_value_control_is_published_after_worker_commit_and_clear() { + let queue = Arc::new(RecordingQueue::new()); + let worker = queue.worker("task-a"); + *queue.active.write().await = vec![make_test_recording_task("task-a", PathBuf::from("/tmp/task-a.ts"))]; + queue.queue.lock().await.push_back(make_test_recording_task("task-b", PathBuf::from("/tmp/task-b.ts"))); + let mut control_lock = worker.control_signal.write().await; + *control_lock = RecordingControl::Cancel; + let worker_queue = Arc::clone(&queue); + let worker_commit = tokio::spawn(async move { + worker_queue + .mutate_optional_and_clear_control("task-a", RecordingControl::Cancel, |candidate| { + let Some(mut active) = candidate.active.pop() else { + return Ok(None); + }; + active.finished = true; + candidate.finished.push(active); + if !candidate.queue.is_empty() { + candidate.active = vec![candidate.queue.remove(0)]; + } + Ok(Some(true)) + }) + .await + }); + + for _ in 0..100 { + if queue.revision.load(Ordering::SeqCst) == 1 { + break; + } + tokio::task::yield_now().await; + } + assert_eq!(queue.revision.load(Ordering::SeqCst), 1); + let api_queue = Arc::clone(&queue); + let newer_cancel = tokio::spawn(async move { api_queue.cancel_requested("task-b").await }); + tokio::task::yield_now().await; + assert!(!newer_cancel.is_finished()); + + drop(control_lock); + + assert!(worker_commit.await.expect("worker task").expect("worker commit").is_some()); + assert_eq!(newer_cancel.await.expect("cancel task").expect("cancel commit"), Some(false)); + assert_eq!(*queue.worker("task-b").control_signal.read().await, RecordingControl::Cancel); +} diff --git a/backend/dvr/src/recording/recording_queue/tests/recovery.rs b/backend/dvr/src/recording/recording_queue/tests/recovery.rs new file mode 100644 index 000000000..8b5058e2c --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/tests/recovery.rs @@ -0,0 +1,216 @@ +use super::*; + +#[tokio::test] +async fn recovering_multiple_active_recordings_preserves_every_task_and_its_recovery_policy( +) -> Result<(), Box> { + let dir = tempfile::tempdir()?; + let queue = RecordingQueue::new_persistent(dir.path(), dir.path())?; + let mut tasks = Vec::new(); + for index in 0..2 { + tasks.push(RecordingQueue::to_persisted(&task( + &format!("live-{index}"), + RecordingKind::Live, + RecordingTaskState::Running, + ))); + let mut paused = task(&format!("paused-{index}"), RecordingKind::Vod, RecordingTaskState::Paused); + paused.paused = true; + tasks.push(RecordingQueue::to_persisted(&paused)); + tasks.push(RecordingQueue::to_persisted(&task( + &format!("vod-{index}"), + RecordingKind::Vod, + RecordingTaskState::Running, + ))); + } + mutate(&queue, |candidate| { + candidate.active = tasks; + Ok(()) + }) + .await?; + let restored = RecordingQueue::new_persistent(dir.path(), dir.path())?; + restored.load_from_disk().await?; + assert_eq!(restored.active.read().await.len(), 2); + assert!(restored.active.read().await.iter().all(|task| task.state == RecordingTaskState::Paused)); + assert_eq!(restored.queue.lock().await.len(), 2); + assert!(restored.queue.lock().await.iter().all(|task| task.state == RecordingTaskState::Queued)); + assert_eq!(restored.finished.read().await.len(), 2); + assert!(restored + .finished + .read() + .await + .iter() + .all(|task| task.state == RecordingTaskState::Failed && task.kind == RecordingKind::Live)); + let (_, tasks) = restored.committed_snapshot().await; + assert_eq!(tasks.len(), 6); + assert!(!restored.workers_running().await); + Ok(()) +} + +#[test] +fn an_interrupted_live_capture_fails_instead_of_restarting() { + // The broadcast that played while the process was down is gone. A + // resumed live capture would write a recording with a hole at the + // front and report it as complete. + for state in [RecordingTaskState::Running, RecordingTaskState::WaitingForCapacity] { + let interrupted = RecordingTask { state, ..task("live", RecordingKind::Live, state) }; + let recovered = RecordingQueue::recover_loaded_task(interrupted); + assert_eq!(recovered.state, RecordingTaskState::Failed, "live in {}", state.label()); + assert!(recovered.finished); + assert!(recovered.error.is_some(), "the operator is told why"); + } +} + +#[test] +fn an_interrupted_transfer_is_still_resumed() { + // The counterpart: a file on a server is still there after a restart, + // so VOD and series pick up where they left off. + for kind in [RecordingKind::Vod, RecordingKind::Series] { + let interrupted = task("film", kind, RecordingTaskState::Running); + let recovered = RecordingQueue::recover_loaded_task(interrupted); + assert_eq!(recovered.state, RecordingTaskState::Queued, "{kind} must resume"); + assert!(!recovered.finished); + } +} + +#[test] +fn a_future_live_recording_survives_a_restart_still_scheduled() { + let scheduled = task("live", RecordingKind::Live, RecordingTaskState::Scheduled); + let recovered = RecordingQueue::recover_loaded_task(scheduled); + assert_eq!(recovered.state, RecordingTaskState::Scheduled, "a window that has not opened is untouched"); + assert!(!recovered.finished); +} + +#[test] +fn recover_loaded_task_requeues_waiting_states() { + let waiting_for_capacity = RecordingTask { + size: 77, + total_size: Some(99), + error: Some("old error".to_string()), + state: RecordingTaskState::WaitingForCapacity, + ..task("capacity", RecordingKind::Vod, RecordingTaskState::WaitingForCapacity) + }; + let retry_waiting = RecordingTask { state: RecordingTaskState::RetryWaiting, ..waiting_for_capacity.clone() }; + + let restored_waiting_for_capacity = RecordingQueue::recover_loaded_task(waiting_for_capacity); + let restored_retry_waiting = RecordingQueue::recover_loaded_task(retry_waiting); + + assert_eq!(restored_waiting_for_capacity.state, RecordingTaskState::Queued); + assert!(!restored_waiting_for_capacity.paused); + assert!(restored_waiting_for_capacity.error.is_none()); + + assert_eq!(restored_retry_waiting.state, RecordingTaskState::Queued); + assert!(!restored_retry_waiting.paused); + assert!(restored_retry_waiting.error.is_none()); +} + +#[test] +fn recover_loaded_task_clears_pending_retry_timestamp() { + let retry_waiting = RecordingTask { + size: 12, + total_size: Some(20), + error: Some("retrying".to_string()), + state: RecordingTaskState::RetryWaiting, + retry_attempts: 2, + next_retry_at: Some(1_700_000_000), + ..task("retry", RecordingKind::Vod, RecordingTaskState::RetryWaiting) + }; + + let restored = RecordingQueue::recover_loaded_task(retry_waiting); + assert_eq!(restored.state, RecordingTaskState::Queued); + assert_eq!(restored.retry_attempts, 0); + assert!(restored.next_retry_at.is_none()); +} + +#[tokio::test] +async fn retry_finished_clears_retry_metadata() { + let queue = RecordingQueue::new(); + queue.finished.write().await.push(RecordingTask { + finished: true, + error: Some("Retry limit reached".to_string()), + state: RecordingTaskState::Failed, + retry_attempts: 5, + next_retry_at: Some(1_700_000_000), + ..task("done", RecordingKind::Vod, RecordingTaskState::Failed) + }); + + assert!(queue.retry_finished("done").await.expect("retry finished")); + let queued = queue.queue.lock().await.front().cloned().expect("queued download"); + assert_eq!(queued.state, RecordingTaskState::Queued); + assert_eq!(queued.retry_attempts, 0); + assert!(queued.next_retry_at.is_none()); + assert!(queued.error.is_none()); +} + +#[tokio::test] +async fn retry_finished_rejects_recordings() { + let queue = RecordingQueue::new(); + queue.finished.write().await.push(RecordingTask { + finished: true, + error: Some("Cancelled by user".to_string()), + state: RecordingTaskState::Cancelled, + recording: live_meta("web:alice", 1_700_000_000, 300), + ..task("recording", RecordingKind::Live, RecordingTaskState::Cancelled) + }); + + assert!(!queue.retry_finished("recording").await.expect("reject recording retry")); + assert!(queue.queue.lock().await.is_empty()); + assert_eq!(queue.finished.read().await.len(), 1); +} + +#[tokio::test] +async fn request_worker_restart_sets_restart_control_and_notifies_waiters() { + let queue = RecordingQueue::new(); + let worker = queue.worker("task"); + let waiter_queue = Arc::clone(&queue.slot_waiters); + let control_signal = Arc::clone(&worker.control_signal); + let control_notify = Arc::clone(&worker.control_notify); + + let waiter = + tokio::spawn(async move { waiter_queue.wait(None, 0, control_signal.as_ref(), control_notify.as_ref()).await }); + + tokio::task::yield_now().await; + queue.request_worker_restart(); + + assert_eq!( + timeout(Duration::from_millis(100), waiter).await.expect("waiter finished").expect("join ok"), + RecordingWaitOutcome::Restarted + ); + assert_eq!(*worker.control_signal.read().await, RecordingControl::Restart); +} + +#[tokio::test] +async fn a_restart_never_replaces_a_pending_pause_or_cancel() { + // A configuration reload must not turn a user's pause into a requeue + // or swallow a cancel, whichever of the two writes lands first. + for pending in [RecordingControl::Pause, RecordingControl::Cancel] { + let queue = RecordingQueue::new(); + let worker = queue.worker("task"); + *worker.control_signal.write().await = pending; + queue.request_worker_restart(); + assert_eq!(*worker.control_signal.read().await, pending); + + // The deferred path, taken while another writer holds the lock. + let held = worker.control_signal.write().await; + queue.request_worker_restart(); + drop(held); + tokio::task::yield_now().await; + tokio::task::yield_now().await; + assert_eq!(*worker.control_signal.read().await, pending); + } +} + +#[tokio::test] +async fn retry_finished_after_failed_state_commits_one_revision() { + let queue = RecordingQueue::new(); + let finished = RecordingTask { + finished: true, + state: RecordingTaskState::Failed, + ..task("done", RecordingKind::Vod, RecordingTaskState::Failed) + }; + queue.finished.write().await.push(finished); + + assert!(queue.retry_finished("done").await.expect("retry commit")); + + assert_eq!(queue.revision.load(Ordering::SeqCst), 1); + assert!(queue.finished.read().await.is_empty()); + assert_eq!(queue.queue.lock().await.front().map(|download| download.uuid.as_str()), Some("done")); +} diff --git a/backend/dvr/src/recording/recording_queue/tests/storage.rs b/backend/dvr/src/recording/recording_queue/tests/storage.rs new file mode 100644 index 000000000..39913a52c --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/tests/storage.rs @@ -0,0 +1,400 @@ +use super::*; + +#[tokio::test] +async fn persisted_queue_round_trips_and_requeues_running_downloads() { + let state_file = temp_state_file("download_state"); + let queue = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); + let queued = RecordingTask { + size: 10, + total_size: Some(100), + ..task("queued", RecordingKind::Vod, RecordingTaskState::Queued) + }; + let active = RecordingTask { + size: 20, + total_size: Some(200), + ..task("active", RecordingKind::Vod, RecordingTaskState::Running) + }; + let paused = RecordingTask { + size: 30, + total_size: Some(300), + paused: true, + state: RecordingTaskState::Paused, + ..task("paused", RecordingKind::Vod, RecordingTaskState::Paused) + }; + + queue.queue.lock().await.push_back(queued); + *queue.active.write().await = vec![active]; + queue.finished.write().await.push(paused.clone()); + queue.persist_to_disk().await.expect("persist state"); + + let restored = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); + restored.load_from_disk().await.expect("load state"); + + assert_eq!(restored.queue.lock().await.len(), 2); + let restored_active = restored.active.read().await.clone(); + assert!(restored_active.is_empty()); + let restored_finished = restored.finished.read().await.clone(); + assert_eq!(restored_finished.len(), 1); + assert_eq!(restored_finished[0].uuid, paused.uuid); + + let queued_items = restored.queue.lock().await.iter().map(|d| d.uuid.clone()).collect::>(); + assert!(queued_items.iter().any(|id| id == "queued")); + assert!(queued_items.iter().any(|id| id == "active")); + + let _ = std::fs::remove_dir_all(state_file); +} + +#[tokio::test] +async fn persisted_scheduled_recordings_round_trip_without_becoming_active() { + let state_file = temp_state_file("record_state"); + let queue = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); + let future_start = Utc::now().timestamp().saturating_add(3_600); + let scheduled = RecordingTask { + url: reqwest::Url::parse("https://example.com/live/1").expect("valid url"), + state: RecordingTaskState::Scheduled, + recording: live_meta("web:alice", future_start, 5_400), + ..task("recording", RecordingKind::Live, RecordingTaskState::Scheduled) + }; + + queue.scheduled.write().await.push(scheduled.clone()); + queue.persist_to_disk().await.expect("persist state"); + + let restored = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); + restored.load_from_disk().await.expect("load state"); + + assert!(restored.active.read().await.is_empty()); + assert_eq!(restored.queue.lock().await.len(), 0); + let restored_scheduled = restored.scheduled.read().await.clone(); + assert_eq!(restored_scheduled.len(), 1); + assert_eq!(restored_scheduled[0].uuid, scheduled.uuid); + assert_eq!(restored_scheduled[0].state, RecordingTaskState::Scheduled); + assert_eq!(restored_scheduled[0].scheduled_start(), Some(future_start)); + assert_eq!(restored_scheduled[0].scheduled_end(), Some(future_start + 5_400)); + assert_eq!(restored_scheduled[0].scheduled_duration_secs(), Some(5_400)); + assert_eq!(restored_scheduled[0].kind, RecordingKind::Live); + + let _ = std::fs::remove_dir_all(state_file); +} + +#[tokio::test] +async fn load_from_disk_moves_expired_scheduled_recordings_to_finished() { + let state_file = temp_state_file("expired_record_state"); + let queue = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); + let expired = RecordingTask { + state: RecordingTaskState::Scheduled, + recording: live_meta("web:alice", 100, 60), + ..task("expired", RecordingKind::Live, RecordingTaskState::Scheduled) + }; + + queue.scheduled.write().await.push(expired); + queue.persist_to_disk().await.expect("persist state"); + + let restored = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); + restored.load_from_disk().await.expect("load state"); + + assert!(restored.scheduled.read().await.is_empty()); + let finished = restored.finished.read().await.clone(); + assert_eq!(finished.len(), 1); + assert_eq!(finished[0].uuid, "expired"); + assert_eq!(finished[0].state, RecordingTaskState::Failed); + assert_eq!(finished[0].error.as_deref(), Some("Recording window already expired")); + + let _ = std::fs::remove_dir_all(state_file); +} + +#[test] +fn download_uuid_differs_for_same_url_with_different_filenames() { + let cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { + directory: Some("/tmp".to_string()), + ..Default::default() + }); + + let first = RecordingTask::new( + RecordingKind::Vod, + "https://example.com/video.mp4", + "first.mp4", + &cfg, + None, + 0, + media_meta("web:alice"), + ) + .expect("first download"); + let second = RecordingTask::new( + RecordingKind::Vod, + "https://example.com/video.mp4", + "second.mp4", + &cfg, + None, + 0, + media_meta("web:alice"), + ) + .expect("second download"); + + assert_ne!(first.uuid, second.uuid); +} + +#[test] +fn a_name_taken_on_disk_is_numbered_from_the_original_stem() { + // Collision suffixes number the original stem: the third copy is + // `film_2.mp4`, not `film_1_2.mp4`. + let dir = tempfile::TempDir::new().expect("tempdir"); + std::fs::write(dir.path().join("film.mp4"), b"").expect("first copy"); + std::fs::write(dir.path().join("film_1.mp4"), b"").expect("second copy"); + let cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { + directory: Some(dir.path().to_string_lossy().into_owned()), + ..Default::default() + }); + + let task = RecordingTask::new( + RecordingKind::Vod, + "https://example.com/film.mp4", + "film.mp4", + &cfg, + None, + 0, + media_meta("web:alice"), + ) + .expect("a free name"); + + assert_eq!(task.filename, "film_2.mp4"); +} + +#[test] +fn download_new_omits_trailing_dot_when_filename_has_no_extension() { + let cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { + directory: Some("/tmp".to_string()), + ..Default::default() + }); + + let task = RecordingTask::new( + RecordingKind::Vod, + "https://example.com/live", + "title with trailing dot.", + &cfg, + None, + 0, + media_meta("web:alice"), + ) + .expect("download"); + + assert_eq!(task.filename, "title_with_trailing_dot"); + assert!(!task.filename.ends_with('.')); +} + +#[test] +fn task_filename_keeps_extension_dots_and_letters_of_every_script() { + let mut meta = live_meta("web:alice", 1_700_000_000, 1_800); + meta.group = Some("Новости".to_string()); + let task = organized_task(RecordingKind::Live, "Mr. Robot — Çalıkuşu 🎬.ts", meta); + assert_eq!(task.recording.relative_path.as_deref(), Some("Новости/Mr._Robot_Çalıkuşu.ts")); +} + +#[tokio::test] +async fn a_commit_whose_checkpoint_fails_is_kept_in_memory_and_on_disk() { + // The checkpoint runs after the batch is durable. The queue keeps the + // committed record, and the next commit does not delete it again. + let dir = tempfile::TempDir::new().expect("tempdir"); + drop(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + force_a_failing_checkpoint(dir.path()); + let queue = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("reopen repository"); + + let committed = RecordingQueue::to_persisted(&task("kept", RecordingKind::Vod, RecordingTaskState::Completed)); + mutate(&queue, move |candidate| { + candidate.finished.push(committed.clone()); + Ok(()) + }) + .await + .expect("the commit succeeds although its checkpoint does not"); + assert_eq!(queue.finished.read().await.len(), 1); + + // A second commit diffs against that state rather than deleting it. + let another = RecordingQueue::to_persisted(&task("next", RecordingKind::Vod, RecordingTaskState::Completed)); + mutate(&queue, move |candidate| { + candidate.finished.push(another.clone()); + Ok(()) + }) + .await + .expect("second commit"); + drop(queue); + + let reopened = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("reopen repository"); + reopened.load_from_disk().await.expect("load"); + let mut uuids: Vec = reopened.finished.read().await.iter().map(|task| task.uuid.clone()).collect(); + uuids.sort(); + assert_eq!(uuids, ["kept", "next"]); +} + +#[test] +fn from_persisted_rejects_invalid_url() { + let mut p = RecordingQueue::to_persisted(&task("bad-url", RecordingKind::Vod, RecordingTaskState::Queued)); + p.url = "not a url at all".to_string(); + let result = RecordingQueue::from_persisted(p); + assert!(result.is_err(), "invalid url must surface as an error"); + assert!(matches!(result.unwrap_err(), PersistedError::InvalidUrl(_)), "must surface the parse error"); +} + +#[tokio::test] +async fn mutate_keeps_state_when_closure_errors() { + let dir = tempfile::tempdir().expect("tempdir"); + let state_dir = dir.path().to_path_buf(); + let queue = RecordingQueue::new_persistent(&state_dir, &state_dir).expect("open recording repository"); + let original_revision = queue.revision.load(Ordering::SeqCst); + let original_len = queue.queue.lock().await.len(); + + let result: Result<(), QueueMutationError> = + mutate(&queue, |_candidate| Err(QueueMutationError::new("validation failed"))).await; + assert!(result.is_err(), "closure error should propagate"); + assert_eq!(queue.queue.lock().await.len(), original_len, "queue must stay unchanged"); + assert_eq!(queue.revision.load(Ordering::SeqCst), original_revision); + let (revision, uuids) = committed(&queue).await; + assert_eq!(revision, 0, "nothing may be committed on closure error"); + assert_eq!(uuids.len(), 0); +} + +#[tokio::test] +async fn committed_snapshot_waits_for_mutation_boundary() { + let queue = std::sync::Arc::new(RecordingQueue::new()); + let task = make_test_recording_task("rec-1", PathBuf::from("/tmp/rec-1.ts")); + queue.queue.lock().await.push_back(task); + + let mutation_guard = queue.mutation_guard.lock().await; + let snapshot_queue = std::sync::Arc::clone(&queue); + let snapshot = tokio::spawn(async move { snapshot_queue.committed_snapshot().await }); + + tokio::task::yield_now().await; + assert!(!snapshot.is_finished(), "snapshot must wait for the mutation boundary"); + + drop(mutation_guard); + let Ok((revision, tasks)) = snapshot.await else { + unreachable!("snapshot task failed"); + }; + assert_eq!(revision, QueueRevision(0)); + assert_eq!(tasks.len(), 1); + assert_eq!(tasks.first().map(|task| task.uuid.as_str()), Some("rec-1")); +} + +#[tokio::test] +async fn committed_partitioned_snapshot_waits_for_mutation_boundary() { + let queue = Arc::new(RecordingQueue::new()); + let mutation_guard = queue.mutation_guard.lock().await; + let snapshot_queue = Arc::clone(&queue); + let snapshot = tokio::spawn(async move { snapshot_queue.committed_partitioned_snapshot().await }); + + tokio::task::yield_now().await; + assert!(!snapshot.is_finished()); + + drop(mutation_guard); + let Ok((queued, active, finished)) = snapshot.await else { + unreachable!("snapshot task failed"); + }; + assert!(queued.is_empty()); + assert!(active.is_empty()); + assert!(finished.is_empty()); +} + +#[tokio::test] +async fn control_signal_is_ordered_inside_mutation_guard() { + let queue = Arc::new(RecordingQueue::new()); + let worker = queue.worker("active"); + *queue.active.write().await = vec![make_test_transfer_task("active", PathBuf::from("/tmp/active.ts"))]; + let control_lock = worker.control_signal.write().await; + let pause_queue = Arc::clone(&queue); + let pause = tokio::spawn(async move { pause_queue.pause_active("active").await }); + + for _ in 0..100 { + if queue.revision.load(Ordering::SeqCst) == 1 { + break; + } + tokio::task::yield_now().await; + } + assert_eq!(queue.revision.load(Ordering::SeqCst), 1); + let snapshot_queue = Arc::clone(&queue); + let snapshot = tokio::spawn(async move { snapshot_queue.committed_snapshot().await }); + tokio::task::yield_now().await; + assert!(!snapshot.is_finished(), "mutation guard must remain held until control publication"); + + drop(control_lock); + assert!(pause.await.is_ok_and(|result| result.is_ok())); + assert!(snapshot.await.is_ok()); + assert_eq!(*worker.control_signal.read().await, RecordingControl::Pause); +} + +#[tokio::test] +async fn mutate_swap_restores_in_memory_state_from_persisted_candidate() { + let dir = tempfile::tempdir().expect("tempdir"); + let state_file = dir.path().to_path_buf(); + let queue = RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository"); + + let persisted = RecordingQueue::to_persisted(&make_test_recording_task("rec-1", dir.path().join("a.ts"))); + mutate(&queue, |candidate| { + candidate.queue.push(persisted); + Ok(()) + }) + .await + .expect("first mutate"); + + let len_after_commit = queue.queue.lock().await.len(); + assert_eq!(len_after_commit, 1, "committed task must be in memory"); + + // Now mutate again to remove that task. The candidate should be + // re-built from the just-committed state, not from the empty + // pre-mutate in-memory state. + mutate(&queue, |candidate| { + candidate.queue.retain(|d| d.uuid != "rec-1"); + Ok(()) + }) + .await + .expect("second mutate"); + + assert!(queue.queue.lock().await.is_empty(), "remove must propagate to in-memory state"); + assert_eq!(queue.revision.load(Ordering::SeqCst), 2); +} + +// --- load_from_disk rejection paths --- + +#[tokio::test] +async fn load_from_disk_rejects_invalid_url_in_persisted_entry() { + let dir = tempfile::tempdir().expect("tempdir"); + let state_dir = dir.path().to_path_buf(); + let mut persisted = RecordingQueue::to_persisted(&task("rec-1", RecordingKind::Vod, RecordingTaskState::Queued)); + persisted.url = "not a url".to_string(); + + let queue = RecordingQueue::new_persistent(&state_dir, &state_dir).expect("open recording repository"); + queue.commit_records(1, vec![persisted], None).await.expect("commit unparseable record"); + + let result = queue.load_from_disk().await; + + assert!(result.is_err(), "invalid url must surface as an error"); + assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidData); + // Rejection happens before assignment, so memory stays empty and the + // repository is left for an operator to inspect. + assert!(queue.queue.lock().await.is_empty()); + assert!(queue.scheduled.read().await.is_empty()); + assert!(queue.finished.read().await.is_empty()); + assert!(queue.active.read().await.is_empty()); +} + +#[test] +fn reserve_recording_relative_path_returns_numbered_stem_when_no_collision() { + let mut candidate = PersistedRecordingQueue::default(); + let reserved = reserve_recording_relative_path(&mut candidate, "pilot", "rec-1"); + assert_eq!(reserved, "pilot_1", "numbered suffix is the first candidate"); +} + +#[test] +fn reserve_recording_relative_path_skips_existing_paths() { + let mut candidate = PersistedRecordingQueue::default(); + let occupied = RecordingMetadata { relative_path: Some("pilot_1".to_string()), ..media_meta("web:alice") }; + candidate.queue.push(persisted_recording("other", occupied)); + let reserved = reserve_recording_relative_path(&mut candidate, "pilot", "rec-1"); + assert_eq!(reserved, "pilot_2"); +} + +#[test] +fn reserve_recording_relative_path_does_not_double_bump_for_self() { + let mut candidate = PersistedRecordingQueue::default(); + let existing = RecordingMetadata { relative_path: Some("pilot_3".to_string()), ..media_meta("web:alice") }; + candidate.queue.push(persisted_recording("rec-1", existing)); + let reserved = reserve_recording_relative_path(&mut candidate, "pilot", "rec-1"); + assert_eq!(reserved, "pilot_3", "must not bump because of self"); +} diff --git a/backend/dvr/src/recording/recording_queue/tests/streaming.rs b/backend/dvr/src/recording/recording_queue/tests/streaming.rs new file mode 100644 index 000000000..f3769fb3f --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/tests/streaming.rs @@ -0,0 +1,39 @@ +use super::*; + +#[tokio::test] +async fn removing_requeued_tasks_prunes_only_idle_workers() { + for running in [false, true] { + let queue = RecordingQueue::new(); + queue.queue.lock().await.push_back(task("requeued", RecordingKind::Vod, RecordingTaskState::Queued)); + let worker = queue.worker("requeued"); + *worker.running.write().await = running; + assert!(queue.remove_from_queue("requeued").await.expect("remove")); + assert_eq!(queue.existing_worker("requeued").is_some(), running); + queue.request_worker_restart(); + if !running { + assert_eq!(*worker.control_signal.read().await, RecordingControl::None); + } + queue.release_worker("requeued").await; + assert!(queue.existing_worker("requeued").is_none()); + } +} + +#[tokio::test] +async fn attaching_requeued_tasks_prunes_their_idle_worker_state() { + let queue = RecordingQueue::new(); + let completed = task("completed", RecordingKind::Vod, RecordingTaskState::Completed); + let mut waiting = task("waiting", RecordingKind::Vod, RecordingTaskState::Queued); + waiting.url = completed.url.clone(); + queue.queue.lock().await.push_back(waiting); + queue.finished.write().await.push(completed); + queue.worker("waiting"); + mutate(&queue, |candidate| { + assert!(promote_from_queue(candidate).is_none()); + Ok(()) + }) + .await + .expect("attach existing recording"); + assert!(queue.queue.lock().await.is_empty()); + assert_eq!(queue.finished.read().await.len(), 2); + assert!(queue.existing_worker("waiting").is_none()); +} diff --git a/backend/dvr/src/recording/recording_queue/tests/transport.rs b/backend/dvr/src/recording/recording_queue/tests/transport.rs new file mode 100644 index 000000000..5ab61f9d9 --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/tests/transport.rs @@ -0,0 +1,87 @@ +use super::*; + +#[tokio::test] +async fn promotion_precheck_keeps_attachments_and_live_tasks_eligible_while_vod_is_running() { + let queue = RecordingQueue::new(); + let active = task("active", RecordingKind::Vod, RecordingTaskState::Running); + let mut duplicate = task("duplicate", RecordingKind::Vod, RecordingTaskState::Queued); + duplicate.url = active.url.clone(); + let completed = task("completed", RecordingKind::Series, RecordingTaskState::Completed); + let mut attached = task("attached", RecordingKind::Series, RecordingTaskState::Queued); + attached.url = completed.url.clone(); + queue.active.write().await.push(active); + queue.finished.write().await.push(completed); + queue.queue.lock().await.extend([duplicate, attached]); + assert!(queue.has_promotable_queued().await); + mutate(&queue, |candidate| { + assert!(promote_from_queue(candidate).is_none()); + Ok(()) + }) + .await + .expect("attach without starting a second transfer"); + assert_eq!(queue.finished.read().await.len(), 2); + assert!(!queue.has_promotable_queued().await, "the duplicate waits for its active transfer"); + queue.queue.lock().await.push_back(task("live", RecordingKind::Live, RecordingTaskState::Queued)); + assert!(queue.has_promotable_queued().await); + mutate(&queue, |candidate| { + assert_eq!(promote_from_queue(candidate).map(|(uuid, _)| uuid).as_deref(), Some("live")); + Ok(()) + }) + .await + .expect("start live alongside the transfer"); + assert_eq!(queue.active.read().await.len(), 2); + assert_eq!(queue.queue.lock().await.len(), 1); +} + +#[test] +fn a_queued_entry_runs_when_nothing_else_holds_its_media() { + let candidate = PersistedRecordingQueue::default(); + let queued = identified("a", "film-42", RecordingTaskState::Queued); + assert_eq!(promotion_decision(&candidate, &queued), PromotionDecision::Execute); +} + +#[test] +fn a_queued_entry_waits_while_another_entry_produces_the_file() { + // Promoting it would start a second transfer to the same path. + let candidate = PersistedRecordingQueue { + active: vec![identified("a", "film-42", RecordingTaskState::Running)], + ..PersistedRecordingQueue::default() + }; + let queued = identified("b", "film-42", RecordingTaskState::Queued); + assert_eq!(promotion_decision(&candidate, &queued), PromotionDecision::Wait); +} + +#[test] +fn attaching_adopts_the_file_but_keeps_the_entry_its_own() { + let mut source = identified("a", "film-42", RecordingTaskState::Completed); + source.file_path = PathBuf::from("/rec/film.mp4"); + source.filename = "film.mp4".to_string(); + source.size = 4_096; + source.recording.measured_bytes = 4_096; + source.recording.completed_at = Some(1_700_000_000); + source.recording.relative_path = Some("film.mp4".to_string()); + + let mut attached = identified("b", "film-42", RecordingTaskState::Queued); + attached.recording.owner = RecordingOwner::User(UserId::from("web:bob")); + attached.recording.reserved_bytes = 9_999; + attach_to_completed(&mut attached, &source); + + // The bytes are shared. + assert_eq!(attached.file_path, source.file_path); + assert_eq!(attached.recording.measured_bytes, 4_096); + assert_eq!(attached.state, RecordingTaskState::Completed); + // The link is not: this is Bob's entry onto Alice's file. + assert_eq!(attached.uuid, "b"); + assert_eq!(attached.recording.owner_id().0, "web:bob"); + // Nothing is reserved any more; the bytes are already on disk. + assert_eq!(attached.recording.reserved_bytes, 0); +} + +#[test] +fn organized_live_recording_lands_in_its_group() { + let mut meta = live_meta("web:alice", 1_700_000_000, 1_800); + meta.channel_name = Some("SBS 6 HD".to_string()); + meta.group = Some("NEDERLAND".to_string()); + let task = organized_task(RecordingKind::Live, "news.ts", meta); + assert_eq!(task.recording.relative_path.as_deref(), Some("NEDERLAND/news.ts")); +} diff --git a/backend/dvr/src/recording/recording_queue/waiters.rs b/backend/dvr/src/recording/recording_queue/waiters.rs new file mode 100644 index 000000000..5e8b60d31 --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/waiters.rs @@ -0,0 +1,128 @@ +use super::{RecordingControl, RecordingSlotWaitQueue, RecordingWaitRegistration, RecordingWaiter}; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex as StdMutex, +}; +use tokio::sync::{Notify, RwLock}; + +pub(super) type RecordingWaiters = Arc>>; + +impl Drop for RecordingWaitRegistration { + fn drop(&mut self) { lock_unpoisoned(&self.waiters).retain(|waiter| waiter.id != self.id); } +} + +#[derive(Clone)] +pub struct RecordingWaiterSnapshot { + pub id: u64, + pub input_name: Option>, + pub priority: i8, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecordingWaitOutcome { + Signalled, + Paused, + Cancelled, + Restarted, +} + +impl Default for RecordingSlotWaitQueue { + fn default() -> Self { Self::new() } +} + +impl RecordingSlotWaitQueue { + pub fn new() -> Self { + Self { + waiters: Arc::new(StdMutex::new(Vec::new())), + next_waiter_id: AtomicU64::new(1), + registration_changed: Notify::new(), + } + } + + pub(super) fn remove_waiter(&self, waiter_id: u64) { + lock_unpoisoned(&self.waiters).retain(|waiter| waiter.id != waiter_id); + } + + /// Register and block until this task is signalled or control flow requests pause/cancel. + pub async fn wait( + &self, + input_name: Option>, + priority: i8, + control_signal: &RwLock, + control_notify: &Notify, + ) -> RecordingWaitOutcome { + let waiter_id = self.next_waiter_id.fetch_add(1, Ordering::Relaxed); + let notify = Arc::new(Notify::new()); + lock_unpoisoned(&self.waiters).push(RecordingWaiter { + id: waiter_id, + input_name, + priority, + notify: Arc::clone(¬ify), + }); + + let _registration = RecordingWaitRegistration { waiters: Arc::clone(&self.waiters), id: waiter_id }; + self.registration_changed.notify_one(); + + if let Some(outcome) = self.control_outcome(waiter_id, control_signal).await { + return outcome; + } + + loop { + let controlled = control_notify.notified(); + tokio::pin!(controlled); + controlled.as_mut().enable(); + if let Some(outcome) = self.control_outcome(waiter_id, control_signal).await { + return outcome; + } + tokio::select! { + () = notify.notified() => return RecordingWaitOutcome::Signalled, + () = &mut controlled => {} + } + } + } + + /// Map the current control signal to a wait outcome, deregistering the + /// waiter when the wait ends. + pub(super) async fn control_outcome( + &self, + waiter_id: u64, + control_signal: &RwLock, + ) -> Option { + let outcome = match *control_signal.read().await { + RecordingControl::Pause => RecordingWaitOutcome::Paused, + RecordingControl::Cancel => RecordingWaitOutcome::Cancelled, + RecordingControl::Restart => RecordingWaitOutcome::Restarted, + RecordingControl::None => return None, + }; + self.remove_waiter(waiter_id); + Some(outcome) + } + + pub fn snapshots(&self) -> Vec { + lock_unpoisoned(&self.waiters) + .iter() + .map(|waiter| RecordingWaiterSnapshot { + id: waiter.id, + input_name: waiter.input_name.clone(), + priority: waiter.priority, + }) + .collect() + } + + /// Wake a specific waiter by id. + pub fn signal_waiter(&self, waiter_id: u64) -> bool { + let mut waiters = lock_unpoisoned(&self.waiters); + if let Some(idx) = waiters.iter().position(|waiter| waiter.id == waiter_id) { + let notify = Arc::clone(&waiters[idx].notify); + waiters.remove(idx); + notify.notify_one(); + true + } else { + false + } + } +} + +pub(super) fn lock_unpoisoned(mutex: &StdMutex) -> std::sync::MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} diff --git a/backend/dvr/src/recording/recording_queue/workers.rs b/backend/dvr/src/recording/recording_queue/workers.rs new file mode 100644 index 000000000..9525bad16 --- /dev/null +++ b/backend/dvr/src/recording/recording_queue/workers.rs @@ -0,0 +1,158 @@ +use super::{ + poisoned_repository, waiters::lock_unpoisoned, IdempotencyOutcome, RecordingControl, RecordingQueue, RecordingTask, + RecordingTaskState, +}; +use chrono::Utc; +use shared::model::RecordingKind; +use std::{ + collections::{HashSet, VecDeque}, + sync::Arc, +}; +use tokio::sync::{Notify, RwLock}; + +/// Runtime state belonging to one recording. Control signals never cross task boundaries. +/// +/// Queue mutations and worker release take `mutation_guard` first. The worker-map +/// mutex is held only for lookup/removal, never across an await. The `running` +/// lock is released before acquiring queue/active locks or entering a mutation. +/// Control publication follows persisted mutations under `mutation_guard`. +/// Idle-map pruning uses `try_read` under partition locks and retains contested +/// entries for a later worker release. +#[derive(Default)] +pub struct RecordingWorkerState { + pub control_signal: Arc>, + pub control_notify: Arc, + pub running: Arc>, +} + +impl RecordingQueue { + pub fn worker(&self, uuid: &str) -> Arc { + let mut workers = lock_unpoisoned(&self.workers); + if let Some(worker) = workers.get(uuid) { + return Arc::clone(worker); + } + workers.entry(uuid.to_owned()).or_default().clone() + } + + pub fn existing_worker(&self, uuid: &str) -> Option> { + lock_unpoisoned(&self.workers).get(uuid).cloned() + } + + pub(super) fn prune_idle_workers( + &self, + queued: &VecDeque, + scheduled: &[RecordingTask], + active: &[RecordingTask], + ) { + lock_unpoisoned(&self.workers).retain(|uuid, worker| { + queued.iter().chain(scheduled).chain(active).any(|task| task.uuid == *uuid) + || !worker.running.try_read().is_ok_and(|running| !*running) + }); + } + + /// Check promotion eligibility without cloning persisted partitions. + pub(crate) async fn has_promotable_queued(&self) -> bool { + use crate::recording::recording_service::recording_identity; + let queued = self.queue.lock().await; + if queued.is_empty() { + return false; + } + let active = self.active.read().await; + let transfer_running = active.iter().any(|task| task.kind != RecordingKind::Live); + let active_media: HashSet<_> = + active.iter().map(|task| recording_identity(&task.recording, task.url.as_str())).collect(); + let mut waiting_media = HashSet::new(); + for task in queued.iter() { + let identity = recording_identity(&task.recording, task.url.as_str()); + if active_media.contains(&identity) { + continue; + } + if task.kind == RecordingKind::Live || !transfer_running { + return true; + } + waiting_media.insert(identity); + } + if waiting_media.is_empty() { + return false; + } + self.finished.read().await.iter().any(|task| { + task.state == RecordingTaskState::Completed + && waiting_media.contains(&recording_identity(&task.recording, task.url.as_str())) + }) + } + + pub(crate) async fn claim_worker(&self, uuid: &str) -> Option> { + let _mutation = self.mutation_guard.lock().await; + if !self.active.read().await.iter().any(|task| task.uuid == uuid && !task.paused && !task.finished) { + return None; + } + let worker = self.worker(uuid); + let mut running = worker.running.write().await; + if *running { + return None; + } + *running = true; + drop(running); + Some(worker) + } + + pub async fn release_worker(&self, uuid: &str) { + let _mutation = self.mutation_guard.lock().await; + if let Some(worker) = self.existing_worker(uuid) { + *worker.running.write().await = false; + let queued = self.queue.lock().await.iter().any(|task| task.uuid == uuid); + let active = self.active.read().await.iter().any(|task| task.uuid == uuid); + if !queued && !active { + lock_unpoisoned(&self.workers).remove(uuid); + } + } + // A resume/requeue may have raced the previous worker's final iteration. + self.queue_changed.notify_one(); + } + + pub async fn workers_running(&self) -> bool { + let workers: Vec<_> = lock_unpoisoned(&self.workers).values().cloned().collect(); + for worker in workers { + if *worker.running.read().await { + return true; + } + } + false + } + + /// Load the canonical recording state from the repository. + /// + /// An empty repository is a fresh install. A record that cannot be + /// converted back to its in-memory form is an error the caller must + /// propagate: the repository is never reset or rebuilt from memory. + /// The recovery contract for an operator tool. + /// + /// Read-only: it reports, and never opens the B+Tree for mutation. + /// `None` when the queue is not repository backed. + pub async fn recovery_health(&self) -> Option { + let repository = self.repository.clone()?; + tokio::task::spawn_blocking(move || repository.lock().ok().map(|guard| guard.health())).await.ok().flatten() + } + + /// Whether a request carrying this idempotency key has already been + /// accepted. Without a repository there is nothing to remember, so every + /// request is fresh. + pub async fn lookup_idempotency( + &self, + principal: &str, + key: &str, + request_fingerprint: &str, + ) -> std::io::Result { + let Some(repository) = self.repository.clone() else { + return Ok(IdempotencyOutcome::Fresh); + }; + let (principal, key, fingerprint) = (principal.to_owned(), key.to_owned(), request_fingerprint.to_owned()); + let now = Utc::now().timestamp(); + tokio::task::spawn_blocking(move || { + let mut guard = repository.lock().map_err(|_| poisoned_repository())?; + guard.lookup_idempotency(&principal, &key, &fingerprint, now) + }) + .await + .map_err(std::io::Error::other)? + } +} diff --git a/backend/dvr/src/recording/recording_service.rs b/backend/dvr/src/recording/recording_service.rs index 9f8a322ac..472fdad8f 100644 --- a/backend/dvr/src/recording/recording_service.rs +++ b/backend/dvr/src/recording/recording_service.rs @@ -2,203 +2,12 @@ use super::{recording_ctx::RecordingCtx, recording_source_resolution as source_resolution}; use crate::{ - recording::{ - recording_disk, recording_path, - recording_queue::{ - mutate, mutate_prepared, mutate_then, mutate_with_idempotency, IdempotencyOutcome, PersistedIdempotency, - PersistedRecordingQueue, PersistedRecordingTask, QueueMutationError, RecordingQueue, RecordingTask, - RecordingTaskState, - }, - }, - recording_deletion::{ - begin_deletion_authorized, execute_deletion_target, finalize_deletion, rollback_deletion, DeletionError, - }, - recording_edit::{self, EditError, PaddingBounds}, - recording_quota::{self, AdmissionOutcome, QuotaLimits, QuotaPool}, + recording::recording_queue::{PersistedRecordingTask, RecordingQueue}, + recording_quota::QuotaPool, }; -use shared::model::{ - recording::{RecordingMetadata, RecordingOwner, RecordingProvenance, RecordingSource, RecordingVisibility}, - EventSink, RecordingKind, UserId, XtreamCluster, -}; -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::Arc, -}; -use tuliprox_auth::{authorize, authorize_orphan, RecordingAction, RecordingDecision, RecordingSubject, TerminalState}; +use std::sync::Arc; use tuliprox_core::model::AppConfig; -/// Server-resolved identifiers that the recording system needs. The -/// caller never sees the URL; the service resolves it from these. -#[derive(Debug, Clone)] -pub struct RecordingSourceInput { - pub target_id: String, - pub virtual_id: String, - pub cluster: XtreamCluster, - pub input_name: String, -} - -impl RecordingSourceInput { - pub fn validate(&self) -> Result<(), ServiceError> { - if self.target_id.trim().is_empty() || self.virtual_id.trim().is_empty() || self.input_name.trim().is_empty() { - return Err(ServiceError::InvalidSource); - } - Ok(()) - } -} - -/// Input for `RecordingService::create_recording`. -#[derive(Debug, Clone)] -pub struct CreateRecordingInput { - pub source: RecordingSourceInput, - pub program_title: String, - pub program_start: i64, - pub program_end: i64, - pub pre_roll_secs: u64, - pub post_roll_secs: u64, - pub visibility: RecordingVisibility, - pub channel_id: Option, - pub channel_name: Option, - /// Playlist group of the channel, when the caller already resolved it. - /// `None` makes the service look it up for organised layouts. - pub group: Option, - pub provenance: RecordingProvenance, - pub epg: Option, -} - -impl CreateRecordingInput { - pub fn validate(&self) -> Result<(), ServiceError> { - self.source.validate()?; - if self.program_end.checked_sub(self.program_start).is_none_or(|duration| duration <= 0) { - return Err(ServiceError::InvalidInterval); - } - Ok(()) - } -} - -/// Input for `RecordingService::create_media_recording_idempotent`: an -/// immediate VOD or series transfer. Title, extension and grouping come from -/// the playlist entry the caller already resolved. -#[derive(Debug, Clone)] -pub struct CreateMediaRecordingInput { - pub source: RecordingSourceInput, - pub title: String, - pub extension: String, - pub visibility: RecordingVisibility, - pub group: Option, - pub series_name: Option, -} - -impl CreateMediaRecordingInput { - pub fn validate(&self) -> Result { - self.source.validate()?; - if self.extension.trim_start_matches('.').trim().is_empty() { - return Err(ServiceError::InvalidPath); - } - match self.source.cluster { - XtreamCluster::Video => Ok(RecordingKind::Vod), - XtreamCluster::Series => Ok(RecordingKind::Series), - XtreamCluster::Live => Err(ServiceError::InvalidSource), - } - } -} - -/// Stable service-layer errors. The HTTP layer maps each variant to a -/// stable status; the frontend maps each variant to a localized -/// message. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ServiceError { - /// The recording owner could not be resolved from the authenticated - /// claims (`subject_id` missing or invalid). - UnknownOwner, - /// The supplied `RecordingSource` does not match a configured - /// target/input combination. - InvalidSource, - /// Caller asked for an action the principal cannot perform. - Forbidden, - /// Caller asked for shared creation but is not an administrator. - SharedCreationNotAdministrator, - /// Caller asked to act on a recording that is in an ineligible - /// state (e.g., edit on a recording marked `Deleting`). - InvalidState, - /// `program_end - program_start` overflows or is non-positive. - InvalidInterval, - /// Requested padding exceeds the configured recording maximum. - PaddingLimitExceeded, - /// uuid not in the queue. - UnknownRecording, - /// `mutate`'s persist step failed and the in-memory state was - /// kept unchanged. - PersistenceFailed, - /// IO error during physical deletion. - IoError(String), - /// Configured recording quota would be exceeded. - QuotaExceeded, - /// The patch would clear `rule_id` / `occurrence_key`. Both are - /// immutable provenance; surfacing this as `InvalidState` hid the - /// real reason from the client. - ProvenanceImmutable, - /// Caller tried to create a recording that already exists in the - /// queue (same target / window). Distinct from `InvalidState` so - /// the client can render a specific "duplicate" message. - Duplicate, - /// The recording's filesystem path is not within the configured - /// storage root, or otherwise violates the path policy. - InvalidPath, - /// The recording cannot fit on disk; reservation would exceed - /// available space. - DiskFull, - /// The server has no download engine: the `video.recording` block - /// is missing from the configuration. Distinct from - /// `InvalidSource` because the caller's identifiers may be - /// perfectly valid — nothing on the server can execute them. - Disabled, - /// This exact request was already accepted under this idempotency key. - /// Not a failure: the caller gets the original outcome. - IdempotentReplay { recording_id: String }, - /// Same idempotency key, different request body. - IdempotencyConflict, -} - -/// An `Idempotency-Key` and a digest of the body it arrived with. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct IdempotencyRequest { - pub key: String, - pub fingerprint: String, -} - -impl std::fmt::Display for ServiceError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.code()) } -} - -impl std::error::Error for ServiceError {} - -impl ServiceError { - /// Stable wire-level code. - pub fn code(&self) -> &'static str { - match self { - Self::UnknownOwner => "recording_unknown_owner", - Self::InvalidSource => "recording_invalid_source", - Self::Forbidden => "recording_forbidden", - Self::SharedCreationNotAdministrator => "recording_shared_not_administrator", - Self::InvalidState => "recording_invalid_state", - Self::InvalidInterval => "recording_invalid_interval", - Self::PaddingLimitExceeded => "recording_padding_limit_exceeded", - Self::UnknownRecording => "recording_unknown", - Self::PersistenceFailed => "recording_persistence_failed", - Self::IoError(_) => "recording_io_error", - Self::QuotaExceeded => "recording_quota_exceeded", - Self::ProvenanceImmutable => "recording_provenance_immutable", - Self::Duplicate => "recording_duplicate", - Self::InvalidPath => "recording_invalid_path", - Self::DiskFull => "recording_disk_full", - Self::Disabled => "recording_disabled", - Self::IdempotentReplay { .. } => "recording_idempotent_replay", - Self::IdempotencyConflict => "recording_idempotency_conflict", - } - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct EffectiveRecordingWindow { scheduled_start: i64, @@ -207,95 +16,6 @@ struct EffectiveRecordingWindow { remaining_duration_secs: u64, } -fn effective_recording_window( - program_start: i64, - program_end: i64, - pre_roll_secs: u64, - post_roll_secs: u64, - now: i64, -) -> Result { - if program_end <= program_start { - return Err(ServiceError::InvalidInterval); - } - let pre_roll = i64::try_from(pre_roll_secs).map_err(|_| ServiceError::PaddingLimitExceeded)?; - let post_roll = i64::try_from(post_roll_secs).map_err(|_| ServiceError::PaddingLimitExceeded)?; - let scheduled_start = program_start.saturating_sub(pre_roll); - let scheduled_end = program_end.saturating_add(post_roll); - let execution_start = now.max(scheduled_start); - // `scheduled_end >= execution_start` because both come from - // saturating arithmetic on a non-empty interval, so the cast is - // safe and the only remaining error is the degenerate - // already-finished window. - let remaining = scheduled_end.saturating_sub(execution_start); - if remaining <= 0 { - return Err(ServiceError::InvalidInterval); - } - let remaining_duration_secs = remaining.cast_unsigned(); - Ok(EffectiveRecordingWindow { scheduled_start, scheduled_end, execution_start, remaining_duration_secs }) -} - -fn padding_bounds(recording: Option<&tuliprox_core::model::RecordingConfig>) -> PaddingBounds { - recording.map_or( - PaddingBounds { - max_pre_roll_secs: shared::model::default_recording_max_pre_roll_secs(), - max_post_roll_secs: shared::model::default_recording_max_post_roll_secs(), - }, - |config| PaddingBounds { - max_pre_roll_secs: config.max_pre_roll_secs, - max_post_roll_secs: config.max_post_roll_secs, - }, - ) -} - -fn map_edit_validation_error(error: &EditError) -> ServiceError { - match error { - EditError::InvalidInterval => ServiceError::InvalidInterval, - EditError::PaddingLimitExceeded => ServiceError::PaddingLimitExceeded, - EditError::ProvenanceCleared => ServiceError::ProvenanceImmutable, - EditError::StateNotEditable | EditError::ChannelChangedWithoutProgramme => ServiceError::InvalidState, - } -} - -fn map_deletion_error(error: DeletionError) -> ServiceError { - match error { - DeletionError::Forbidden => ServiceError::Forbidden, - DeletionError::NotTerminal => ServiceError::InvalidState, - DeletionError::UnknownTask | DeletionError::NotARecording => ServiceError::UnknownRecording, - DeletionError::DeleteFailed(err) => ServiceError::IoError(err.to_string()), - DeletionError::BeginFailed(err) | DeletionError::FinalizeFailed(err) => { - if err.source_io().is_some() { - ServiceError::PersistenceFailed - } else { - ServiceError::UnknownRecording - } - } - } -} - -/// Output of `RecordingService::create_recording` and friends. -#[derive(Debug, Clone)] -pub struct RecordingTaskView { - pub uuid: String, - pub owner_id: UserId, - pub visibility: RecordingVisibility, - pub filename_preview: String, - pub start_at: Option, - pub duration_secs: Option, - pub state: RecordingTaskState, -} - -/// Input for `RecordingService::edit_recording`. -#[derive(Debug, Clone, Default)] -pub struct EditRecordingPatch { - pub program_start: Option, - pub program_end: Option, - pub pre_roll_secs: Option, - pub post_roll_secs: Option, - pub program_title: Option, - pub channel_id: Option, - pub channel_name: Option, -} - /// Recording mutation boundary. Holds the queue and app config directly /// so the server's root state does not carry a back-reference to the service. pub struct RecordingService { @@ -303,1411 +23,6 @@ pub struct RecordingService { app_config: Arc, } -impl RecordingService { - /// Construct from the queue and app config. - pub fn new(recordings: Arc, app_config: Arc) -> Self { Self { recordings, app_config } } - - /// Convenience constructor from the DVR's context. - pub fn from_ctx(ctx: &RecordingCtx) -> Self { - Self::new(ctx.recordings.clone(), ctx.app_config.clone()) - } - - fn subject_id(claims: &shared::model::Claims) -> Result { - claims.subject_id.clone().ok_or(ServiceError::UnknownOwner) - } - - fn recording_url(&self, source: &RecordingSourceInput) -> Option { - let virtual_id = source.virtual_id.parse::().ok()?; - source_resolution::resolve_recording_target(&self.app_config, &source.target_id, &source.input_name)?; - source_resolution::build_recording_source_descriptor( - &source.target_id, - &source.input_name, - virtual_id, - source.cluster, - ) - } - - /// Create a new recording. Enforces well-formedness, source - /// validity, owner from claims, and the authorization matrix. - #[allow(clippy::too_many_lines)] - pub async fn create_recording( - &self, - claims: &shared::model::Claims, - input: &CreateRecordingInput, - ) -> Result { - self.create_recording_idempotent(claims, input, None).await - } - - /// Builds the task a live request describes, and the space it reserves. - /// - /// Split out of `create_recording_idempotent` purely for length: this is - /// the request-to-task translation, with no admission decisions in it. - fn build_live_recording( - owner_id: &UserId, - owner_display: &str, - input: &CreateRecordingInput, - group: Option, - recording_cfg: &tuliprox_core::model::RecordingConfig, - window: &EffectiveRecordingWindow, - url: &str, - ) -> Result<(RecordingTask, u64), ServiceError> { - let duration_secs = window.remaining_duration_secs; - let filename = render_live_filename(input, recording_cfg, window, owner_display); - let input_name: Option> = - (!input.source.input_name.trim().is_empty()).then(|| Arc::from(input.source.input_name.as_str())); - let source = RecordingSource::new( - input.source.target_id.clone(), - input.source.virtual_id.clone(), - input.source.input_name.clone(), - ) - .with_cluster(input.source.cluster); - let mut meta = RecordingMetadata::new_live( - RecordingOwner::User(owner_id.clone()), - input.visibility, - source, - input.program_start, - input.program_end, - input.pre_roll_secs, - input.post_roll_secs, - ); - meta.scheduled_start = Some(window.scheduled_start); - meta.scheduled_end = Some(window.scheduled_end); - meta.channel_id.clone_from(&input.channel_id); - meta.channel_name.clone_from(&input.channel_name); - meta.program_title = Some(input.program_title.clone()); - meta.group = group; - meta.provenance = input.provenance.clone(); - meta.epg.clone_from(&input.epg); - let (reserved_bytes, _) = - recording_quota::estimate_reservation(duration_secs, 0, recording_cfg.fallback_bytes_per_minute); - meta.reserved_bytes = reserved_bytes; - let recording = RecordingTask::new( - RecordingKind::Live, - url, - &filename, - recording_cfg, - input_name, - recording_cfg.priority, - meta, - ) - .ok_or(ServiceError::InvalidSource)?; - Ok((recording, reserved_bytes)) - } - - /// Playlist group the live recording is filed under. - /// - /// Callers that did not resolve it (the rule scheduler) still get the - /// organised layout the operator asked for. Without organisation the - /// group is irrelevant, so the playlist is not read for it. - async fn live_group( - &self, - input: &CreateRecordingInput, - recording_cfg: &tuliprox_core::model::RecordingConfig, - ) -> Option { - if input.group.is_some() || !recording_cfg.organize_into_directories { - return input.group.clone(); - } - source_resolution::resolve_live_group( - &self.app_config, - &input.source.target_id, - &input.source.input_name, - &input.source.virtual_id, - ) - .await - } - - /// `create_recording`, honouring an `Idempotency-Key`. - /// - /// A replay of an accepted request is answered from the stored record - /// rather than run again; the same key with a different body is a - /// conflict, because answering it with the first request's result would - /// hide a caller bug. - pub async fn create_recording_idempotent( - &self, - claims: &shared::model::Claims, - input: &CreateRecordingInput, - idempotency: Option, - ) -> Result { - input.validate()?; - let owner_id = Self::subject_id(claims)?; - self.check_idempotency(&owner_id, idempotency.as_ref()).await?; - let config = self.enabled_config()?; - let Some(recording_cfg) = config.recording() else { - return Err(ServiceError::Disabled); - }; - recording_edit::validate_padding( - input.pre_roll_secs, - input.post_roll_secs, - padding_bounds(Some(recording_cfg)), - ) - .map_err(|error: EditError| map_edit_validation_error(&error))?; - let window = effective_recording_window( - input.program_start, - input.program_end, - input.pre_roll_secs, - input.post_roll_secs, - chrono::Utc::now().timestamp(), - )?; - let url = self.recording_url(&input.source).ok_or(ServiceError::InvalidSource)?; - - // Shared creation requires admin. - authorize_create_recording(claims, &owner_id, input.visibility)?; - - // The layout is fixed when the task is built, so the group has to be - // known first. - let group = self.live_group(input, recording_cfg).await; - let duration_secs = window.remaining_duration_secs; - let (recording, reserved_bytes) = - Self::build_live_recording(&owner_id, &claims.username, input, group, recording_cfg, &window, &url)?; - self.admit(&recording, reserved_bytes, &owner_id, idempotency.as_ref(), recording_cfg).await?; - - Ok(RecordingTaskView { - uuid: recording.uuid, - owner_id, - visibility: input.visibility, - filename_preview: recording.filename, - start_at: Some(window.execution_start), - duration_secs: Some(duration_secs), - state: RecordingTaskState::Scheduled, - }) - } - - /// Queue an immediate VOD or series transfer, honouring an - /// `Idempotency-Key`. - /// - /// Runs through the same admission as a live request: path reservation, - /// per-principal duplicate check, quota and disk. The size of a transfer - /// is unknown until the provider answers, so nothing is reserved up front; - /// the worker re-checks quota and disk before it opens the destination. - pub async fn create_media_recording_idempotent( - &self, - claims: &shared::model::Claims, - input: &CreateMediaRecordingInput, - idempotency: Option, - ) -> Result { - let kind = input.validate()?; - let owner_id = Self::subject_id(claims)?; - self.check_idempotency(&owner_id, idempotency.as_ref()).await?; - let config = self.enabled_config()?; - let Some(recording_cfg) = config.recording() else { - return Err(ServiceError::Disabled); - }; - let url = self.recording_url(&input.source).ok_or(ServiceError::InvalidSource)?; - authorize_create_recording(claims, &owner_id, input.visibility)?; - - let source = RecordingSource::new( - input.source.target_id.clone(), - input.source.virtual_id.clone(), - input.source.input_name.clone(), - ) - .with_cluster(input.source.cluster); - let mut meta = RecordingMetadata::new_media( - RecordingOwner::User(owner_id.clone()), - input.visibility, - source, - input.title.clone(), - ); - meta.group.clone_from(&input.group); - meta.series_name.clone_from(&input.series_name); - let filename = format!("{}.{}", input.title, input.extension.trim_start_matches('.')); - let recording = RecordingTask::new( - kind, - &url, - &filename, - recording_cfg, - Some(Arc::from(input.source.input_name.as_str())), - recording_cfg.priority, - meta, - ) - .ok_or(ServiceError::InvalidPath)?; - self.admit(&recording, 0, &owner_id, idempotency.as_ref(), recording_cfg).await?; - - Ok(RecordingTaskView { - uuid: recording.uuid, - owner_id, - visibility: input.visibility, - filename_preview: recording.filename, - start_at: None, - duration_secs: None, - state: RecordingTaskState::Queued, - }) - } - - /// Answer a replayed or conflicting `Idempotency-Key` before any work: - /// a replay must not resolve sources, reserve a path or touch quota. - async fn check_idempotency( - &self, - owner_id: &UserId, - idempotency: Option<&IdempotencyRequest>, - ) -> Result<(), ServiceError> { - let Some(request) = idempotency else { - return Ok(()); - }; - match self - .recordings - .lookup_idempotency(owner_id.0.as_str(), &request.key, &request.fingerprint) - .await - .map_err(|err| ServiceError::IoError(err.to_string()))? - { - IdempotencyOutcome::Fresh => Ok(()), - IdempotencyOutcome::Replay { recording_id } => Err(ServiceError::IdempotentReplay { recording_id }), - IdempotencyOutcome::Conflict => Err(ServiceError::IdempotencyConflict), - } - } - - fn enabled_config(&self) -> Result, ServiceError> { - if !crate::recording::recording_supervisor::recording_enabled(&self.app_config) { - return Err(ServiceError::Disabled); - } - Ok(self.app_config.config.load_full()) - } - - /// Admit a freshly built task into the queue: reserve its path, refuse a - /// duplicate from the same principal, then check quota and disk, all in - /// one mutation together with the idempotency record. - /// - /// Scheduled kinds land in `scheduled`, everything else in `queue`. - async fn admit( - &self, - recording: &RecordingTask, - reserved_bytes: u64, - owner_id: &UserId, - idempotency: Option<&IdempotencyRequest>, - recording_cfg: &tuliprox_core::model::RecordingConfig, - ) -> Result<(), ServiceError> { - let mut persisted = RecordingQueue::to_persisted(recording); - let quota_limits = quota_limits_from_config(recording_cfg.quota.as_ref()); - // Measured before the mutation: it is a syscall, and the lock is - // held for the whole closure. `None` means the root could not be - // measured, and an unmeasurable disk is not grounds to refuse. - let disk_safety_bytes = recording_cfg.disk.as_ref().and_then(|disk| disk.safety_bytes).unwrap_or(0); - let free_bytes = recording_disk::free_bytes_for(std::path::Path::new(&recording_cfg.directory)); - let idempotency_record = idempotency.map(|request| PersistedIdempotency { - principal: owner_id.0.clone(), - key: request.key.clone(), - request_fingerprint: request.fingerprint.clone(), - recording_id: recording.uuid.clone(), - accepted_at: chrono::Utc::now().timestamp(), - }); - - let admit = |candidate: &mut PersistedRecordingQueue| -> Result<(), QueueMutationError> { - reserve_recording_relative_path(candidate, &mut persisted)?; - if candidate_has_duplicate_recording(candidate, recording) { - return Err(QueueMutationError::Duplicate); - } - // Media another entry finished, or is fetching with a known size, - // is charged at that size now: attaching copies the whole file - // into this entry's charge, so admitting it at zero would let it - // past the quota. - let charge = reserved_bytes.max(known_media_size(candidate, &persisted)); - persisted.recording.reserved_bytes = charge; - let pool = recording_quota::quota_pool_for_task(&persisted); - let used = used_bytes_for_pool(candidate, &pool); - if matches!( - recording_quota::would_exceed(&pool, used, charge, "a_limits), - AdmissionOutcome::OverLimit { .. } - ) { - return Err(QueueMutationError::QuotaExceeded); - } - // Quota is per-owner and logical; this is the physical - // question, and one can pass while the other fails. - if let Some(free_bytes) = free_bytes { - let active = recording_disk::active_disk_reservations(candidate_tasks(candidate)); - if matches!( - recording_disk::would_fit_on_disk(free_bytes, disk_safety_bytes, active, reserved_bytes), - recording_disk::DiskAdmission::Insufficient { .. } - ) { - return Err(QueueMutationError::DiskFull); - } - } - if recording.kind.is_scheduled() { - candidate.scheduled.push(persisted.clone()); - } else { - candidate.queue.push(persisted.clone()); - } - Ok(()) - }; - - match idempotency_record { - Some(record) => mutate_with_idempotency(&self.recordings, record, admit).await, - None => mutate(&self.recordings, admit).await, - } - .map_err(|e| map_queue_error(&e)) - } - - /// Edit an existing recording. - #[allow(clippy::too_many_lines)] - pub async fn edit_recording( - &self, - claims: &shared::model::Claims, - uuid: &str, - patch: EditRecordingPatch, - ) -> Result { - let owner_id = Self::subject_id(claims)?; - let config = self.app_config.config.load(); - // Edit does not re-resolve a URL. The recording config is - // only needed for padding bounds, the unknown-bitrate fallback, - // and quota limits. When the recording block is absent the - // helper falls back to the shared-model defaults, so a - // configured-without-recording deployment still validates - // edits. - let recording_cfg = config.recording(); - let bounds = padding_bounds(recording_cfg); - let fallback_bytes_per_minute = recording_cfg.map_or(8 * 1024 * 1024, |cfg| cfg.fallback_bytes_per_minute); - let quota_limits = quota_limits_from_config(recording_cfg.and_then(|cfg| cfg.quota.as_ref())); - let mut out = None; - mutate(&self.recordings, |candidate| { - // Single linear scan: locate the recording, snapshot the - // primitives we need for the immutable analysis, drop the - // borrow, run the checks, then re-acquire the same task - // via the remembered location for the O(1) write phase. - let location = locate_recording(candidate, uuid).ok_or(QueueMutationError::UnknownRecording)?; - let snapshot = { - // `Active` and `Finished` are not in `scheduled` / - // `queue`, but `locate_recording` returns them anyway — - // short-circuit with `StateNotEditable` so the match - // arms below stay narrowed to the editable lists. - let task = match location { - RecordingLocation::Scheduled(i) => &candidate.scheduled[i], - RecordingLocation::Queue(i) => &candidate.queue[i], - RecordingLocation::Active(_) | RecordingLocation::Finished(_) => { - return Err(QueueMutationError::StateNotEditable); - } - }; - if !recording_edit::state_is_editable(task.state) { - return Err(QueueMutationError::StateNotEditable); - } - let meta_snapshot = &task.recording; - let pool = recording_quota::quota_pool_for_task(task); - let subject = RecordingSubject::new(Some(meta_snapshot), TerminalState::Active, true); - if !matches!(authorize(claims, &owner_id, RecordingAction::Edit, &subject), RecordingDecision::Allow) { - return Err(QueueMutationError::Forbidden); - } - let merged_pre = patch.pre_roll_secs.unwrap_or(meta_snapshot.pre_roll_secs); - let merged_post = patch.post_roll_secs.unwrap_or(meta_snapshot.post_roll_secs); - recording_edit::validate_padding(merged_pre, merged_post, bounds).map_err(|err| match err { - EditError::PaddingLimitExceeded => QueueMutationError::PaddingLimitExceeded, - EditError::InvalidInterval => QueueMutationError::InvalidInterval, - EditError::StateNotEditable | EditError::ChannelChangedWithoutProgramme => { - QueueMutationError::StateNotEditable - } - EditError::ProvenanceCleared => QueueMutationError::Forbidden, - })?; - let channel_changed_now = recording_edit::channel_changed( - patch.channel_id.as_deref(), - patch.channel_name.as_deref(), - meta_snapshot.channel_id.as_deref(), - meta_snapshot.channel_name.as_deref(), - ); - EditSnapshot { - pool, - merged_pre, - merged_post, - channel_changed_now, - current_start: meta_snapshot.program_start, - current_end: meta_snapshot.program_end, - current_reserved: meta_snapshot.reserved_bytes, - is_live: task.kind == RecordingKind::Live, - shares_media: crate::recording::recording_queue::media_is_still_referenced(candidate, uuid), - } - }; - // A programme window and its padding exist for live captures - // only; the create path refuses them for VOD and series too. - let edits_window = patch.program_start.is_some() - || patch.program_end.is_some() - || patch.pre_roll_secs.is_some() - || patch.post_roll_secs.is_some(); - if !snapshot.is_live && edits_window { - return Err(QueueMutationError::InvalidInterval); - } - // A changed window is derived the way the create path derives - // it: padded, and reserved for what is left of it. An edit that - // leaves the window alone leaves its schedule and reservation - // alone too. - let window = if edits_window { - let start = - patch.program_start.or(snapshot.current_start).ok_or(QueueMutationError::InvalidInterval)?; - let end = patch.program_end.or(snapshot.current_end).ok_or(QueueMutationError::InvalidInterval)?; - // Same rule as a new request: the interval itself must be - // representable, not only its padded ends. - if end.checked_sub(start).is_none_or(|duration| duration <= 0) { - return Err(QueueMutationError::InvalidInterval); - } - let window = effective_recording_window( - start, - end, - snapshot.merged_pre, - snapshot.merged_post, - chrono::Utc::now().timestamp(), - ) - .map_err(|error| match error { - ServiceError::PaddingLimitExceeded => QueueMutationError::PaddingLimitExceeded, - _ => QueueMutationError::InvalidInterval, - })?; - let (reserved, _) = - recording_quota::estimate_reservation(window.remaining_duration_secs, 0, fallback_bytes_per_minute); - let pool_used = used_bytes_for_pool(candidate, &snapshot.pool); - let pool_used_minus_this = pool_used.saturating_sub(snapshot.current_reserved); - if matches!( - recording_quota::would_exceed(&snapshot.pool, pool_used_minus_this, reserved, "a_limits), - AdmissionOutcome::OverLimit { .. } - ) { - return Err(QueueMutationError::QuotaExceeded); - } - Some((start, end, window, reserved)) - } else { - None - }; - - // All immutable borrows are out of scope. Re-acquire the - // same task via the remembered location (O(1)) for the - // actual edit and apply every field write here. If any - // earlier step returned `Err`, none of these writes run, - // so the candidate is rolled back atomically. - let Some(task) = recording_mut_at(candidate, location) else { - return Err(QueueMutationError::UnknownRecording); - }; - let meta = &mut task.recording; - if let Some(title) = patch.program_title { - meta.program_title = Some(title); - } - if let Some(channel_id) = patch.channel_id { - meta.channel_id = Some(channel_id); - } - if let Some(channel_name) = patch.channel_name { - meta.channel_name = Some(channel_name); - } - if let Some((start, end, window, reserved)) = window { - meta.pre_roll_secs = snapshot.merged_pre; - meta.post_roll_secs = snapshot.merged_post; - meta.program_start = Some(start); - meta.program_end = Some(end); - meta.scheduled_start = Some(window.scheduled_start); - meta.scheduled_end = Some(window.scheduled_end); - meta.reserved_bytes = reserved; - } - if snapshot.channel_changed_now { - meta.epg = None; - } - - // The window is part of what makes two requests the same media. - // An edit that changes it leaves the media this entry shared, - // so it must not keep the shared file's path either: two - // captures would write one file, and deleting either would - // take the other's. - let identity = recording_identity_key(&task.recording, &task.url); - let leaves_shared_media = identity != task.media_identity && snapshot.shares_media; - task.media_identity = identity; - if leaves_shared_media { - let mut detached = task.clone(); - reserve_recording_relative_path(candidate, &mut detached)?; - let Some(task) = recording_mut_at(candidate, location) else { - return Err(QueueMutationError::UnknownRecording); - }; - *task = detached; - } - - let Some(task) = recording_mut_at(candidate, location) else { - return Err(QueueMutationError::UnknownRecording); - }; - out = Some(RecordingTaskView { - uuid: task.uuid.clone(), - owner_id: owner_id.clone(), - visibility: task.recording.visibility, - filename_preview: task.filename.clone(), - start_at: task.recording.scheduled_start, - duration_secs: window.map(|(_, _, window, _)| window.remaining_duration_secs), - state: task.state, - }); - Ok(()) - }) - .await - .map_err(|e| map_queue_error(&e))?; - out.ok_or(ServiceError::UnknownRecording) - } - - /// Cancel an in-flight or scheduled recording. - /// - /// An active recording is asked to stop, not marked stopped: the worker - /// still owns the file and the provider slot, so it must release them - /// first and is the one that commits `Cancelled`. Marking it terminal - /// here would show it as a finished recording while it is still writing, - /// and a later remove would then find it in the active slot, leave it - /// there, and report success. - pub async fn cancel_recording(&self, claims: &shared::model::Claims, uuid: &str) -> Result<(), ServiceError> { - let owner_id = Self::subject_id(claims)?; - let active = self.recordings.active.read().await.iter().find(|task| task.uuid == uuid).cloned(); - if let Some(active) = active { - let meta = active.recording.clone(); - let subject = RecordingSubject::new(Some(&meta), TerminalState::Active, true); - if !matches!(authorize(claims, &owner_id, RecordingAction::Cancel, &subject), RecordingDecision::Allow) { - return Err(ServiceError::Forbidden); - } - // Cancel by uuid. Between the read above and this call ffmpeg - // can finish and the queue can promote a *different* recording - // into the active slot; cancelling whatever is active would then - // kill that innocent recording. - match self.recordings.cancel_requested(uuid).await { - // The task is active (running, waiting, or paused). A paused - // one has already been moved to `finished`; a worker-owned one - // is now `Cancelling` and the worker will finish it. - Ok(Some(_)) => return Ok(()), - // The task left the active slot in the meantime. Fall - // through to the inactive path: it either finds the task - // in `scheduled`/`queue` (a re-promotion) or reports - // `UnknownRecording`, which is the truthful answer. - Ok(None) => {} - Err(err) => { - log::error!("cancel_requested failed for {uuid}: {err}"); - return Err(ServiceError::PersistenceFailed); - } - } - } - mutate(&self.recordings, |candidate| { - let Some(task) = remove_inactive_recording(candidate, uuid) else { - return Err(QueueMutationError::UnknownRecording); - }; - let meta = &task.recording; - let subject = RecordingSubject::new(Some(meta), TerminalState::Active, true); - if !matches!(authorize(claims, &owner_id, RecordingAction::Cancel, &subject), RecordingDecision::Allow) { - return Err(QueueMutationError::Forbidden); - } - let mut cancelled = task; - cancelled.state = RecordingTaskState::Cancelled; - cancelled.finished = true; - cancelled.error = Some("cancelled".to_string()); - cancelled.recording.reserved_bytes = 0; - candidate.finished.push(cancelled); - Ok(()) - }) - .await - .map_err(|e| map_queue_error(&e)) - } - - /// Cancel future inactive recordings that were materialized from a - /// recurring rule. Active recordings are intentionally left untouched. - /// Returns the pre-cancel snapshots of everything it cancelled. The - /// caller is mid-way through a two-store operation (cancel the - /// occurrences, then delete the rule) that cannot be made atomic, so - /// it keeps these to undo the queue side if the rule store fails — - /// see [`Self::restore_cancelled_rule_recordings`]. - pub async fn pause_recording(&self, claims: &shared::model::Claims, uuid: &str) -> Result<(), ServiceError> { - let owner_id = Self::subject_id(claims)?; - let active = self.recordings.active.read().await.iter().find(|task| task.uuid == uuid).cloned(); - if let Some(active) = active { - let meta = active.recording.clone(); - if !active.kind.is_resumable() { - return Err(ServiceError::InvalidState); // Live cannot be paused - } - let subject = RecordingSubject::new(Some(&meta), TerminalState::Active, true); - if !matches!(authorize(claims, &owner_id, RecordingAction::Edit, &subject), RecordingDecision::Allow) { - return Err(ServiceError::Forbidden); - } - self.recordings.pause_active(uuid).await.map_err(|_| ServiceError::PersistenceFailed)?; - return Ok(()); - } - Err(ServiceError::UnknownRecording) - } - - pub async fn resume_recording(&self, claims: &shared::model::Claims, uuid: &str) -> Result { - let owner_id = Self::subject_id(claims)?; - let active = self.recordings.active.read().await.iter().find(|task| task.uuid == uuid).cloned(); - if let Some(active) = active { - let meta = active.recording.clone(); - if !active.kind.is_resumable() { - return Err(ServiceError::InvalidState); - } - let subject = RecordingSubject::new(Some(&meta), TerminalState::Active, true); - if !matches!(authorize(claims, &owner_id, RecordingAction::Edit, &subject), RecordingDecision::Allow) { - return Err(ServiceError::Forbidden); - } - return self.recordings.resume_active(uuid).await.map_err(|_| ServiceError::PersistenceFailed); - } - Err(ServiceError::UnknownRecording) - } - - /// Remove a task from the queue. The ownership check runs inside the - /// same mutation that removes it, so a task cannot change hands - /// between the check and the write. - pub async fn remove_recording_task( - &self, - claims: &shared::model::Claims, - uuid: &str, - ) -> Result { - let owner_id = Self::subject_id(claims)?; - let (removed, _) = mutate_then( - &self.recordings, - |candidate| { - authorize_task_in_candidate(candidate, uuid, claims, &owner_id, RecordingAction::Delete)?; - // The active slot is owned by the worker. Removing a task from - // it here would leave the worker writing to a file no entry - // names, so refuse instead of silently doing nothing (the retain - // below cannot reach the active slot). - if candidate.active.iter().any(|active| active.uuid == uuid) { - return Err(QueueMutationError::StateNotEditable); - } - // Removing an entry keeps a finished recording on disk; that is - // what separates it from deleting the file. A partial is - // different: with no entry and no worker left on its media, it - // is never resumed. A worker on the same media holds the active - // slot, so it counts as a reference here. - let orphaned_partial = candidate_tasks(candidate) - .find(|task| task.uuid == uuid) - .filter(|_| !crate::recording::recording_queue::media_is_still_referenced(candidate, uuid)) - .map(|task| task.file_path.clone()); - let original = candidate.queue.len() + candidate.scheduled.len() + candidate.finished.len(); - candidate.queue.retain(|task| task.uuid != uuid); - candidate.scheduled.retain(|task| task.uuid != uuid); - candidate.finished.retain(|task| task.uuid != uuid); - let current = candidate.queue.len() + candidate.scheduled.len() + candidate.finished.len(); - let removed = current != original; - Ok((removed, orphaned_partial.filter(|_| removed))) - }, - // Still under the guard that decided the partial is orphaned: a - // request admitted after it may reuse the path, and must find it - // either still claimed or already empty, never emptied under it. - |(_, orphaned_partial)| { - let orphaned_partial = orphaned_partial.clone(); - async move { - let Some(file_path) = orphaned_partial else { - return; - }; - if let Err(err) = crate::recording_deletion::remove_orphaned_partial(&file_path).await { - log::warn!("Removed a recording, but could not remove its leftover partial: {err}"); - } - } - }, - ) - .await - .map_err(|e| map_queue_error(&e))?; - Ok(removed) - } - - /// Requeue a finished VOD/Series transfer. Live is rejected by the - /// queue itself: its programme window is gone. - pub async fn retry_recording(&self, claims: &shared::model::Claims, uuid: &str) -> Result { - self.retry_recording_inner(claims, uuid, false).await - } - - /// Discard a partial only after the caller confirmed that a provider - /// without byte-range support should download the file from the start. - pub async fn restart_recording(&self, claims: &shared::model::Claims, uuid: &str) -> Result { - self.retry_recording_inner(claims, uuid, true).await - } - - async fn retry_recording_inner( - &self, - claims: &shared::model::Claims, - uuid: &str, - restart_from_beginning: bool, - ) -> Result { - let owner_id = Self::subject_id(claims)?; - let recordings = &self.recordings; - let requester = &owner_id; - mutate_prepared( - recordings, - // A restart discards the partial before the task is queued - // again, so a worker can never resume from bytes the provider - // cannot continue. It runs once the request is known to be - // allowed, and before the mutation, under the same guard. - || async move { - if !restart_from_beginning { - return Ok(()); - } - let partial = { - let finished = recordings.finished.read().await; - let Some(task) = finished.iter().find(|task| task.uuid == uuid) else { - return Ok(()); - }; - let subject = RecordingSubject::new(Some(&task.recording), TerminalState::Active, true); - if !matches!( - authorize(claims, requester, RecordingAction::Edit, &subject), - RecordingDecision::Allow - ) { - return Err(QueueMutationError::Forbidden); - } - if task.error.as_deref() != Some(super::recording_transfer::RANGE_UNSUPPORTED_ERROR) { - return Err(QueueMutationError::StateNotEditable); - } - crate::recording::recording_worker::recording_partial_path(&task.file_path) - }; - match tokio::fs::remove_file(&partial).await { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(QueueMutationError::from_io(error)), - } - }, - |candidate, ()| { - authorize_task_in_candidate(candidate, uuid, claims, &owner_id, RecordingAction::Edit)?; - let Some(pos) = candidate.finished.iter().position(|task| task.uuid == uuid) else { - return Ok(false); - }; - if !candidate.finished[pos].kind.is_resumable() { - return Err(QueueMutationError::StateNotEditable); - } - let range_unsupported = candidate.finished[pos].error.as_deref() - == Some(super::recording_transfer::RANGE_UNSUPPORTED_ERROR); - if range_unsupported != restart_from_beginning { - return Err(QueueMutationError::StateNotEditable); - } - let mut task = candidate.finished.remove(pos); - task.finished = false; - task.size = 0; - if restart_from_beginning { - task.total_size = None; - task.recording.resume_etag = None; - task.recording.resume_last_modified = None; - } - task.paused = false; - task.error = None; - task.state = RecordingTaskState::Queued; - task.retry_attempts = 0; - task.next_retry_at = None; - candidate.queue.push(task); - Ok(true) - }, - ) - .await - .map_err(|e| map_queue_error(&e)) - } - - pub async fn cancel_future_rule_recordings( - &self, - claims: &shared::model::Claims, - rule_id: &str, - now_secs: i64, - ) -> Result, ServiceError> { - let _ = Self::subject_id(claims)?; - if !claims.permissions.contains(shared::model::Permission::RecordingManage) { - return Err(ServiceError::Forbidden); - } - let mut cancelled = Vec::new(); - mutate(&self.recordings, |candidate| { - cancelled = cancel_future_rule_recordings_in_candidate(candidate, rule_id, now_secs); - Ok(()) - }) - .await - .map_err(|e| map_queue_error(&e))?; - Ok(cancelled) - } - - /// Compensating transaction for [`Self::cancel_future_rule_recordings`]. - /// - /// Moves each task back out of `finished` into the list it came from, - /// restoring the exact record that was captured before the cancel - /// (including `reserved_bytes`, which the cancel zeroed). A uuid that - /// something else has since claimed is left alone: a real - /// create/edit always wins over an undo. - pub async fn restore_cancelled_rule_recordings( - &self, - cancelled: &[CancelledRuleRecording], - ) -> Result<(), ServiceError> { - if cancelled.is_empty() { - return Ok(()); - } - mutate(&self.recordings, |candidate| { - for entry in cancelled { - let uuid = entry.task.uuid.as_str(); - candidate.finished.retain(|task| task.uuid != uuid); - if locate_recording(candidate, uuid).is_some() { - continue; - } - match entry.origin { - CancelOrigin::Scheduled => candidate.scheduled.push(entry.task.clone()), - CancelOrigin::Queue => candidate.queue.push(entry.task.clone()), - } - } - Ok(()) - }) - .await - .map_err(|e| map_queue_error(&e)) - } - - /// Delete a finished recording via the three-step service. - /// Marks the task as `Deleting` (atomic), unlinks the file - /// (outside the boundary), then removes the task (atomic). - pub async fn delete_recording(&self, claims: &shared::model::Claims, uuid: &str) -> Result<(), ServiceError> { - let owner_id = Self::subject_id(claims)?; - self.run_deletion(uuid, |meta| { - let subject = RecordingSubject::new(Some(meta), TerminalState::Completed, true); - matches!(authorize(claims, &owner_id, RecordingAction::Delete, &subject), RecordingDecision::Allow) - }) - .await - .map(|_| ()) - } - - /// The three-phase deletion, shared by the user-facing delete and the - /// retention worker. `permit` runs *inside* the same mutation - /// boundary that stamps the task as deleting, so there is no window - /// in which the authorized metadata and the stamped task can differ: - /// the previous implementation looked the task up, authorized it, - /// stamped it, then looked it up a second time and could act on a - /// stale copy. - /// Returns `true` when the file was actually unlinked. `false` means the - /// entry is gone but another library entry still holds its bytes, so the - /// caller must not count the space as reclaimed. - async fn run_deletion(&self, uuid: &str, permit: F) -> Result - where - F: FnOnce(&RecordingMetadata) -> bool, - { - let queue = self.recordings.clone(); - let target = begin_deletion_authorized(&queue, uuid, permit).await.map_err(map_deletion_error)?; - if let Err(err) = execute_deletion_target(&target).await { - // File removal failed: undo the deletion transition so the - // recording stays visible in its prior state instead of - // being silently lost when finalize_deletion runs. - let uuid_owned = uuid.to_string(); - let _ = mutate(&self.recordings, |candidate| { - rollback_deletion(candidate, &uuid_owned); - Ok(()) - }) - .await; - return Err(ServiceError::IoError(err.to_string())); - } - finalize_deletion(&queue, uuid).await.map_err(|_| ServiceError::UnknownRecording)?; - Ok(!target.still_referenced) - } - - /// Internal retention-delete entrypoint used by the retention - /// worker. Bypasses user ownership but enforces state/kind/path - pub async fn system_retention_delete( - &self, - claims: &shared::model::Claims, - uuid: &str, - ) -> Result { - let owner_id = Self::subject_id(claims)?; - self.run_deletion(uuid, |meta| { - let subject = RecordingSubject::new(Some(meta), TerminalState::Completed, true); - matches!( - authorize(claims, &owner_id, RecordingAction::SystemRetentionDelete, &subject,), - RecordingDecision::Allow - ) - }) - .await - } - - /// Re-export the orphan policy for callers that need it. - pub fn authorize_orphan_read(&self, claims: &shared::model::Claims) -> Result<(), ServiceError> { - match authorize_orphan(claims) { - RecordingDecision::Allow => Ok(()), - RecordingDecision::Deny(_) => Err(ServiceError::Forbidden), - } - } - - /// Server-side conflict preview. The caller submits a candidate - /// padded interval plus server-owned source identifiers. The - /// server enumerates the committed queue state for that - /// provider/input, builds the demand points itself, resolves the - /// effective capacity from config, and runs the deterministic - /// analyzer. The request never carries another user's - /// `others`, capacity, or provider identifier. - pub async fn preview_conflicts( - &self, - claims: &shared::model::Claims, - request: &ConflictPreviewRequest, - ) -> Result { - // Require an authenticated principal. The owner id is not - // needed for the analyzer (the privacy contract applies to - // the response), but a missing / invalid claim must reject. - Self::subject_id(claims)?; - // Reject malformed input up front so the analyzer never sees - // garbage. The endpoint enforces the same bounds; this is the - // service-layer defense in depth. - let bounds = padding_bounds(self.app_config.config.load().recording()); - recording_edit::validate_padding(request.pre_roll_secs, request.post_roll_secs, bounds) - .map_err(|error: EditError| map_edit_validation_error(&error))?; - if request.padded_start >= request.padded_end { - return Err(ServiceError::InvalidInterval); - } - // Server resolves the source. The candidate must map to a - // configured provider; anything else is `InvalidSource`. - if self.recording_url(&request.source).is_none() { - return Err(ServiceError::InvalidSource); - } - // Capacity comes from config, not the caller. The runtime - // slot model is the source of truth. - let capacity = effective_capacity_from_config(&self.app_config.config.load()); - // Demand points come from the committed queue state. The - // privacy contract from `recording_conflict.rs` still applies - // — the response only carries anonymized segments. - let others = - collect_demand_points_for_provider(&self.recordings, &request.source.target_id, &request.source.input_name) - .await; - let candidate = crate::recording_conflict::DemandPoint { - task_id: String::new(), - padded_start: request.padded_start, - padded_end: request.padded_end, - priority: request.priority, - }; - let provider_scope = Some(request.source.target_id.clone()); - Ok(crate::recording_conflict::preview_conflict(&candidate, &others, capacity, provider_scope)) - } -} - -/// Maximum bytes in a single sanitized filename component. Well under -/// the 255-byte limit every supported filesystem enforces, leaving room -/// for the `_N` disambiguation suffix and the `.partial` extension the -/// worker appends. -const MAX_FILENAME_COMPONENT_BYTES: usize = 200; - -/// Substitute for a title that sanitizes down to nothing. -const FILENAME_FALLBACK: &str = "recording"; - -/// Characters that are illegal in a path component on at least one -/// supported platform. `/` and `\` are separators where it matters; the -/// rest are Windows-reserved but are equally unwelcome in a -/// URL-addressed media path. -const FILENAME_FORBIDDEN_CHARS: &[char] = &['<', '>', ':', '"', '/', '\\', '|', '?', '*']; - -/// Windows reserved device names. A component whose stem matches one of -/// these (case-insensitively) cannot be created on Windows, with or -/// without an extension. -const WINDOWS_RESERVED_STEMS: &[&str] = &[ - "con", "prn", "aux", "nul", "com1", "com2", "com3", "com4", "com5", "com6", "com7", "com8", "com9", "lpt1", "lpt2", - "lpt3", "lpt4", "lpt5", "lpt6", "lpt7", "lpt8", "lpt9", -]; - -/// Turn arbitrary programme text into one safe path component. -/// -/// The previous implementation replaced only the two path separators, -/// which let control characters, Windows-reserved characters, trailing -/// dots/spaces, and `BiDi` override codepoints through into the path the -/// muxer opens and the media API re-validates. This is the single -/// chokepoint: everything that lands in `filename` goes through here. -/// -/// Guarantees on the returned string: -/// - exactly one path component (no separator survives), -/// - no ASCII control characters and no Unicode `BiDi` / invisible -/// formatting codepoints, -/// - no leading or trailing whitespace or `.`, -/// - never empty, never `.` or `..`, never a Windows device name, -/// - at most `MAX_FILENAME_COMPONENT_BYTES` bytes, truncated on a -/// character boundary, -/// - idempotent: sanitizing an already-sanitized value is a no-op. -pub fn sanitize_filename_component(raw: &str) -> String { - let mut out = String::with_capacity(raw.len()); - let mut last_was_underscore = false; - for ch in raw.chars() { - // Invisible formatting codepoints can reorder the rendered - // filename so it does not match the bytes on disk. Drop them - // outright rather than substituting, so they leave no trace. - if is_invisible_formatting(ch) { - continue; - } - if ch.is_control() || FILENAME_FORBIDDEN_CHARS.contains(&ch) { - // Collapse runs so `a///b` becomes `a_b`, not `a___b`. - if !last_was_underscore { - out.push('_'); - last_was_underscore = true; - } - continue; - } - out.push(ch); - last_was_underscore = ch == '_'; - } - - // Trailing dots and spaces are silently stripped by Windows, which - // would desync the persisted `relative_path` from the real file. - let trimmed = out.trim_matches(|ch: char| ch.is_whitespace() || ch == '.'); - let mut result = truncate_on_char_boundary(trimmed, MAX_FILENAME_COMPONENT_BYTES) - .trim_end_matches(|ch: char| ch.is_whitespace() || ch == '.') - .to_string(); - - if result.is_empty() || is_windows_reserved_stem(&result) { - result = FILENAME_FALLBACK.to_string(); - } - result -} - -/// `BiDi` controls, zero-width characters, and the other invisible -/// formatting codepoints that make a filename render differently from -/// what it actually contains. -fn is_invisible_formatting(ch: char) -> bool { - matches!( - ch, - '\u{200b}'..='\u{200f}' // zero-width space .. RLM - | '\u{202a}'..='\u{202e}' // embedding / override - | '\u{2060}'..='\u{2064}' // word joiner, invisible operators - | '\u{2066}'..='\u{2069}' // directional isolates - | '\u{feff}' // BOM / zero-width no-break space - ) -} - -/// Truncate to at most `max_bytes`, never splitting a character. -fn truncate_on_char_boundary(value: &str, max_bytes: usize) -> &str { - if value.len() <= max_bytes { - return value; - } - let mut end = max_bytes; - while end > 0 && !value.is_char_boundary(end) { - end -= 1; - } - &value[..end] -} - -/// `true` when the component's stem is a Windows device name. -fn is_windows_reserved_stem(value: &str) -> bool { - let stem = value.split('.').next().unwrap_or(value); - WINDOWS_RESERVED_STEMS.iter().any(|reserved| stem.eq_ignore_ascii_case(reserved)) -} - -/// Live recording filename: `recording.filename_template` rendered for this -/// programme, plus the extension of the container the worker muxes into, so -/// players and the media API can tell the format from the name. -/// -/// The sanitized programme title stands in when the template is unset or -/// cannot be rendered. -fn render_live_filename( - input: &CreateRecordingInput, - recording_cfg: &tuliprox_core::model::RecordingConfig, - window: &EffectiveRecordingWindow, - owner_display: &str, -) -> String { - let title_stem = sanitize_filename_component(&input.program_title); - let context = shared::utils::RecordingFilenameContext { - task_id: title_stem.clone(), - channel_id: input.channel_id.clone(), - channel_name: input.channel_name.clone(), - program_title: Some(input.program_title.clone()), - episode_season: input.epg.as_ref().and_then(|epg| epg.season), - episode_number: input.epg.as_ref().and_then(|epg| epg.episode), - owner_display: (!owner_display.trim().is_empty()).then(|| owner_display.to_string()), - program_start: Some(input.program_start), - program_end: Some(input.program_end), - scheduled_start: Some(window.scheduled_start), - scheduled_end: Some(window.scheduled_end), - }; - let stem = Some(recording_cfg.filename_template.as_str()) - .filter(|template| !template.trim().is_empty()) - .and_then(|template| { - shared::utils::render_recording_stem(template, &context, &recording_cfg.timezone) - .map_err(|err| log::warn!("Recording filename template could not be rendered: {err}")) - .ok() - }) - .unwrap_or(title_stem); - let extension = recording_cfg.container_format.file_extension(); - // A template that already ends in the container extension must not double it. - let stem_path = Path::new(&stem); - let stem = match (stem_path.file_stem(), stem_path.extension()) { - (Some(base), Some(ext)) if ext.eq_ignore_ascii_case(extension) => base.to_string_lossy(), - _ => std::borrow::Cow::Borrowed(stem.as_str()), - }; - format!("{stem}.{extension}") -} - -fn authorize_create_recording( - claims: &shared::model::Claims, - owner_id: &UserId, - visibility: RecordingVisibility, -) -> Result<(), ServiceError> { - let action = match visibility { - RecordingVisibility::Private => RecordingAction::CreatePrivate, - RecordingVisibility::Shared => RecordingAction::CreateShared, - }; - match authorize(claims, owner_id, action, &RecordingSubject::new(None, TerminalState::Active, true)) { - RecordingDecision::Allow => Ok(()), - RecordingDecision::Deny(tuliprox_auth::DenyReason::NotAdministrator) => { - Err(ServiceError::SharedCreationNotAdministrator) - } - RecordingDecision::Deny(_) => Err(ServiceError::Forbidden), - } -} - -/// Map a queue-mutation failure onto the service error surface. -/// -/// Every call site used to enumerate all twelve `QueueMutationError` -/// variants inline, so adding a variant meant editing four or more -/// matches. The variants that carry no site-specific meaning collapse -/// here; a site that needs a different mapping for one variant still -/// handles it before delegating. -fn map_queue_error(err: &QueueMutationError) -> ServiceError { - match err { - QueueMutationError::Io(_) => ServiceError::PersistenceFailed, - QueueMutationError::UnknownRecording => ServiceError::UnknownRecording, - QueueMutationError::Forbidden => ServiceError::Forbidden, - QueueMutationError::InvalidInterval => ServiceError::InvalidInterval, - QueueMutationError::PaddingLimitExceeded => ServiceError::PaddingLimitExceeded, - QueueMutationError::QuotaExceeded => ServiceError::QuotaExceeded, - QueueMutationError::Duplicate => ServiceError::Duplicate, - QueueMutationError::InvalidPath => ServiceError::InvalidPath, - QueueMutationError::DiskFull => ServiceError::DiskFull, - QueueMutationError::IdempotentReplay { recording_id } => { - ServiceError::IdempotentReplay { recording_id: recording_id.clone() } - } - QueueMutationError::IdempotencyConflict => ServiceError::IdempotencyConflict, - QueueMutationError::StateNotEditable - | QueueMutationError::InvalidQuotaPool - | QueueMutationError::NotInTerminalState - | QueueMutationError::MutationSkipped - | QueueMutationError::Other(_) => ServiceError::InvalidState, - } -} - -pub fn quota_limits_from_config(config: Option<&tuliprox_core::model::RecordingQuotaConfig>) -> QuotaLimits { - let mut per_user_bytes = HashMap::new(); - if let Some(config) = config { - for (user_id, bytes) in &config.per_user_bytes { - per_user_bytes.insert(UserId::from(user_id.clone()), *bytes); - } - QuotaLimits { - default_private_bytes: config.default_private_bytes, - per_user_bytes, - shared_bytes: config.shared_bytes, - } - } else { - QuotaLimits::default() - } -} - -/// Every task in the candidate snapshot, borrowed. Admission checks run -/// inside `mutate`, so this must not allocate a clone per task — the -/// previous implementation built a `Vec` of the -/// entire queue on every create and every edit. -/// Authorize an action against a task found in the candidate snapshot. -/// Called inside the queue mutation so the ownership decision and the -/// state change commit together. -fn authorize_task_in_candidate( - candidate: &PersistedRecordingQueue, - uuid: &str, - claims: &shared::model::Claims, - subject_id: &UserId, - action: RecordingAction, -) -> Result<(), QueueMutationError> { - let Some(task) = candidate_tasks(candidate).find(|task| task.uuid == uuid) else { - return Err(QueueMutationError::UnknownRecording); - }; - let subject = RecordingSubject::new(Some(&task.recording), TerminalState::Active, true); - match authorize(claims, subject_id, action, &subject) { - RecordingDecision::Allow => Ok(()), - RecordingDecision::Deny(_) => Err(QueueMutationError::Forbidden), - } -} - -fn candidate_tasks(candidate: &PersistedRecordingQueue) -> impl Iterator + '_ { - candidate - .queue - .iter() - .chain(candidate.scheduled.iter()) - .chain(candidate.active.iter()) - .chain(candidate.finished.iter()) -} - -/// Bytes charged against a single quota pool. Only the pool the caller -/// asked about is summed; the previous implementation built the full -/// per-user `HashMap` and then read one entry out of it. -fn used_bytes_for_pool(candidate: &PersistedRecordingQueue, pool: &QuotaPool) -> u64 { - recording_quota::used_bytes_in_pool(candidate_tasks(candidate), pool) -} - -fn reserve_recording_relative_path( - candidate: &PersistedRecordingQueue, - task: &mut PersistedRecordingTask, -) -> Result<(), QueueMutationError> { - // Borrowed set, built once. The old code walked a `Vec` of - // cloned filenames once per `_N` candidate, so reserving the - // (N+1)-th recording of a title cost O(N^2) string comparisons. - let existing: std::collections::HashSet<&str> = collect_existing_relative_paths(candidate).collect(); - // The reservation is over the whole root-relative path, not the bare - // filename: two series can legitimately hold an `e01.mkv` in different - // season directories. - let base = task.recording.relative_path.clone().unwrap_or_else(|| task.filename.clone()); - let mut relative = PathBuf::from(&base); - if existing.contains(base.as_str()) { - // Linear probe over indices; each probe is one hash lookup. - for index in 1.. { - relative = recording_path::with_collision_suffix(Path::new(&base), index); - if !existing.contains(relative.to_string_lossy().as_ref()) { - break; - } - } - } - let filename = - relative.file_name().and_then(|name| name.to_str()).ok_or(QueueMutationError::InvalidPath)?.to_string(); - validate_reserved_filename(&filename).map_err(|_| QueueMutationError::InvalidPath)?; - if !recording_path::is_contained_relative_path(&relative) { - return Err(QueueMutationError::InvalidPath); - } - task.file_path = task.file_dir.join(&filename); - task.filename = filename; - task.recording.relative_path = Some(relative.to_string_lossy().into_owned()); - Ok(()) -} - -/// The size of the media `task` refers to, as far as another entry already -/// knows it: the measured size of a completed file, or the total a transfer -/// learned from its provider. `0` when nobody knows yet. -fn known_media_size(candidate: &PersistedRecordingQueue, task: &PersistedRecordingTask) -> u64 { - if task.media_identity.is_empty() { - return 0; - } - candidate_tasks(candidate) - .filter(|other| other.media_identity == task.media_identity) - .filter_map(|other| { - if other.state == RecordingTaskState::Completed { - Some(other.recording.measured_bytes.max(other.size)) - } else { - other.total_size - } - }) - .max() - .unwrap_or(0) -} - -fn collect_existing_relative_paths(candidate: &PersistedRecordingQueue) -> impl Iterator + '_ { - candidate_tasks(candidate).map(task_relative_path) -} - -fn task_relative_path(task: &PersistedRecordingTask) -> &str { - task.recording.relative_path.as_deref().unwrap_or(task.filename.as_str()) -} - -/// What makes two recording requests "the same thing". -/// -/// The previous key was `(url, start_at, duration_secs)` OR `file_path`, -/// and neither half worked: -/// - `file_path` is disambiguated with a `_N` suffix by -/// `reserve_recording_relative_path`, so it *never* matches an -/// existing task and the whole disjunct was dead. -/// - `start_at` is `now.max(scheduled_start)`, so for a -/// currently-airing programme every request inside the window -/// produces a different value and the same programme could be -/// booked over and over. -/// -/// The identity is now derived from what the user actually asked for. -#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize)] -pub enum RecordingIdentity { - /// Materialization of one rule occurrence. Two tasks with the same - /// `(rule_id, occurrence_key)` are the same recording by - /// definition, whatever their window looks like. - Occurrence { rule_id: S, occurrence_key: S }, - /// A concrete programme on a concrete source. Deliberately free of any - /// owner: two users asking for the same programme are asking for the same - /// file, and it is recorded once and linked twice. - Programme { target_id: S, virtual_id: S, program_start: i64, program_end: i64 }, - /// No programme metadata at all: every VOD and series transfer, and a - /// Live capture whose programme window is unknown. Falls back to the - /// resolved URL plus the *scheduled* (padded) window, which — unlike - /// `start_at` — is stable across requests inside a currently-airing - /// window. - /// - /// Owner-free for the same reason `Programme` is: the same URL over the - /// same window is the same bytes, whoever asked for them. - Url { url: S, scheduled_start: Option, scheduled_end: Option }, -} - -/// A stable, field-named key for the media a request refers to. -/// -/// Two tasks with the same key are the same recording, so they share one -/// physical file. The key is persisted, so it has to be stable across builds: -/// that is why it is serialised rather than formatted with `Debug`. -pub fn recording_identity_key(meta: &RecordingMetadata, url: &str) -> String { - serde_json::to_string(&recording_identity(meta, url)).unwrap_or_else(|_| format!("url:{url}")) -} - -pub(crate) fn recording_identity<'a>(meta: &'a RecordingMetadata, url: &'a str) -> RecordingIdentity<&'a str> { - if let (Some(rule_id), Some(occurrence_key)) = - (meta.provenance.rule_id.as_deref(), meta.provenance.occurrence_key.as_deref()) - { - return RecordingIdentity::Occurrence { rule_id, occurrence_key }; - } - if let (Some(program_start), Some(program_end)) = (meta.program_start, meta.program_end) { - return RecordingIdentity::Programme { - target_id: &meta.source.target_id, - virtual_id: &meta.source.virtual_id, - program_start, - program_end, - }; - } - RecordingIdentity::Url { url, scheduled_start: meta.scheduled_start, scheduled_end: meta.scheduled_end } -} - -fn persisted_recording_identity(task: &PersistedRecordingTask) -> RecordingIdentity<&str> { - recording_identity(&task.recording, &task.url) -} - -fn candidate_has_duplicate_recording(candidate: &PersistedRecordingQueue, task: &RecordingTask) -> bool { - let meta = &task.recording; - let identity = recording_identity(meta, task.url.as_str()); - let pool = recording_quota::quota_pool_for_task(task); - // Only the *same* principal asking twice is a duplicate. A different - // principal asking for the same media gets their own library entry, which - // attaches to the one physical file rather than producing a second. - let pending_match = candidate - .queue - .iter() - .chain(candidate.scheduled.iter()) - .chain(candidate.active.iter()) - .filter(|existing| recording_quota::quota_pool_for_task(*existing) == pool) - .map(persisted_recording_identity) - .any(|existing| existing == identity); - if pending_match { - return true; - } - // Terminal tasks do not block a fresh request: after a failed or - // cancelled attempt the user must be able to try again, and after a - // successful one they may legitimately want a second copy. The one - // exception is a rule occurrence — re-materializing an occurrence - // that already ran would duplicate it on every scheduler tick. - if !matches!(identity, RecordingIdentity::Occurrence { .. }) { - return false; - } - candidate - .finished - .iter() - .filter(|existing| recording_quota::quota_pool_for_task(*existing) == pool) - .map(persisted_recording_identity) - .any(|existing| existing == identity) -} - -/// Where a recording lives in the candidate snapshot. The first scan -/// produces one of these so the second access (mut borrow for writes) -/// is O(1) instead of repeating the linear search. -#[derive(Debug, Clone, Copy)] -enum RecordingLocation { - Scheduled(usize), - Queue(usize), - Active(usize), - Finished(usize), -} - -/// Single linear scan that locates a recording anywhere in the -/// candidate snapshot. The returned `RecordingLocation` lets the -/// caller re-acquire the same task for a mutable borrow without a -/// second search. -fn locate_recording(candidate: &PersistedRecordingQueue, uuid: &str) -> Option { - let matches_uuid = |task: &PersistedRecordingTask| task.uuid == uuid; - if let Some(i) = candidate.scheduled.iter().position(matches_uuid) { - return Some(RecordingLocation::Scheduled(i)); - } - if let Some(i) = candidate.queue.iter().position(matches_uuid) { - return Some(RecordingLocation::Queue(i)); - } - if let Some(i) = candidate.active.iter().position(matches_uuid) { - return Some(RecordingLocation::Active(i)); - } - if let Some(i) = candidate.finished.iter().position(matches_uuid) { - return Some(RecordingLocation::Finished(i)); - } - None -} - -/// Resolve a recording to a mutable borrow using a remembered -/// location. The location must have come from the same candidate; -/// callers obtain it via [`locate_recording`]. -fn recording_mut_at( - candidate: &mut PersistedRecordingQueue, - location: RecordingLocation, -) -> Option<&mut PersistedRecordingTask> { - match location { - RecordingLocation::Scheduled(i) => candidate.scheduled.get_mut(i), - RecordingLocation::Queue(i) => candidate.queue.get_mut(i), - RecordingLocation::Active(i) => candidate.active.get_mut(i), - // Must be the located index, not element 0: returning the first - // finished task would silently edit an unrelated recording. - RecordingLocation::Finished(i) => candidate.finished.get_mut(i), - } -} - /// Primitives extracted from `RecordingMetadata` during the immutable /// analysis pass. Carries only the fields the post-borrow code needs /// so we never clone the full `RecordingMetadata` (which holds @@ -1724,88 +39,6 @@ struct EditSnapshot { /// Another entry holds the same media, and with it the same file path. shares_media: bool, } -/// Server-owned input for the conflict preview. The caller never -/// supplies another recording's padded interval, capacity, or -/// provider identifier — those are derived server-side. -#[derive(Debug, Clone)] -pub struct ConflictPreviewRequest { - pub source: RecordingSourceInput, - pub padded_start: i64, - pub padded_end: i64, - pub pre_roll_secs: u64, - pub post_roll_secs: u64, - pub priority: i32, -} - -fn effective_capacity_from_config( - config: &tuliprox_core::model::Config, -) -> crate::recording_conflict::EffectiveCapacity { - let background_slots = config.recording().map_or(0, |cfg| u32::from(cfg.max_background_per_provider)); - let reserved = config.recording().map_or(0, |cfg| u32::from(cfg.reserve_slots_for_users)); - crate::recording_conflict::EffectiveCapacity { background_slots, reserved_interactive_slots: reserved } -} - -async fn collect_demand_points_for_provider( - queue: &Arc, - target_id: &str, - input_name: &str, -) -> Vec { - use crate::recording_conflict::DemandPoint; - fn matches(task: &RecordingTask, target_id: &str, input_name: &str) -> bool { - task.kind == RecordingKind::Live - && task.recording.source.target_id == target_id - && task.recording.source.input_name == input_name - } - fn to_demand_point(task: &RecordingTask) -> Option { - let meta = &task.recording; - let start = meta.scheduled_start?; - let end = meta.scheduled_end?; - if end <= start { - return None; - } - Some(DemandPoint { - task_id: task.uuid.clone(), - padded_start: start, - padded_end: end, - priority: i32::from(task.priority), - }) - } - // Pending and active recordings are real capacity consumers. - // Finished recordings no longer claim slots, so they would only - // inflate the conflict preview's `peak_demand`. - fn claims_a_slot(task: &RecordingTask) -> bool { - !matches!( - task.state, - RecordingTaskState::Completed | RecordingTaskState::Failed | RecordingTaskState::Cancelled - ) - } - // One committed snapshot rather than three sequential guards. Reading - // `scheduled`, then `queue`, then `active` in turn let a task that - // moved between two of those reads be counted twice or not at all, - // which silently shifted the reported severity. - let (_revision, tasks) = queue.committed_snapshot().await; - tasks - .iter() - .filter(|task| claims_a_slot(task) && matches(task, target_id, input_name)) - .filter_map(to_demand_point) - .collect() -} - -fn remove_inactive_recording(candidate: &mut PersistedRecordingQueue, uuid: &str) -> Option { - if let Some(index) = candidate.scheduled.iter().position(|task| task.uuid == uuid) { - return Some(candidate.scheduled.remove(index)); - } - let index = candidate.queue.iter().position(|task| task.uuid == uuid)?; - Some(candidate.queue.remove(index)) -} - -/// Which pending list a cancelled rule recording came from, so the -/// compensating restore puts it back where it belongs. -#[derive(Debug, Clone, Copy)] -enum CancelOrigin { - Scheduled, - Queue, -} /// A rule-materialized recording exactly as it was before the cancel. #[derive(Debug, Clone)] @@ -1814,80 +47,6 @@ pub struct CancelledRuleRecording { task: PersistedRecordingTask, } -fn cancel_future_rule_recordings_in_candidate( - candidate: &mut PersistedRecordingQueue, - rule_id: &str, - now_secs: i64, -) -> Vec { - let mut undo = Vec::new(); - let mut moved = Vec::new(); - drain_future_rule_recordings( - &mut candidate.scheduled, - CancelOrigin::Scheduled, - rule_id, - now_secs, - &mut undo, - &mut moved, - ); - drain_future_rule_recordings(&mut candidate.queue, CancelOrigin::Queue, rule_id, now_secs, &mut undo, &mut moved); - candidate.finished.extend(moved); - undo -} - -fn drain_future_rule_recordings( - tasks: &mut Vec, - origin: CancelOrigin, - rule_id: &str, - now_secs: i64, - undo: &mut Vec, - out: &mut Vec, -) { - let mut index = 0; - while index < tasks.len() { - if is_future_rule_recording(&tasks[index], rule_id, now_secs) { - let mut task = tasks.remove(index); - // Snapshot before the cancel mutates it: the undo has to - // restore `reserved_bytes`, which is zeroed just below. - undo.push(CancelledRuleRecording { origin, task: task.clone() }); - task.state = RecordingTaskState::Cancelled; - task.finished = true; - task.error = Some("cancelled".to_string()); - task.recording.reserved_bytes = 0; - out.push(task); - } else { - index += 1; - } - } -} - -fn is_future_rule_recording(task: &PersistedRecordingTask, rule_id: &str, now_secs: i64) -> bool { - if task.kind != RecordingKind::Live { - return false; - } - let meta = &task.recording; - let Some(start_at) = meta.scheduled_start else { - return false; - }; - if start_at <= now_secs { - return false; - } - if meta.provenance.rule_id.as_deref() != Some(rule_id) { - return false; - } - recording_edit::state_is_editable(task.state) -} - -fn validate_reserved_filename(filename: &str) -> Result<(), &'static str> { - use std::path::Component; - let path = Path::new(filename); - let single_normal_component = - path.components().next().is_some_and(|c| matches!(c, Component::Normal(_))) && path.components().count() == 1; - if filename.is_empty() || path.is_absolute() || !single_normal_component || filename.as_bytes().contains(&0) { - return Err("recording invalid path"); - } - Ok(()) -} - impl std::fmt::Debug for RecordingService { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RecordingService").finish_non_exhaustive() @@ -1895,1656 +54,33 @@ impl std::fmt::Debug for RecordingService { } #[cfg(test)] -mod tests { - use super::*; - use shared::model::{Permission, RecordingContainerFormat, XtreamCluster}; - use tuliprox_core::model::{RecordingConfig, RecordingNotificationConfig}; - - fn source(target_name: &str, virtual_id: &str, input_name: &str) -> RecordingSourceInput { - RecordingSourceInput { - target_id: target_name.to_string(), - virtual_id: virtual_id.to_string(), - cluster: XtreamCluster::Live, - input_name: input_name.to_string(), - } - } - - fn create_input() -> CreateRecordingInput { - CreateRecordingInput { - source: source("target", "1", "input-a"), - program_title: "Pilot".to_string(), - program_start: 1_700_000_000, - program_end: 1_700_003_600, - pre_roll_secs: 0, - post_roll_secs: 0, - visibility: RecordingVisibility::Private, - channel_id: None, - channel_name: None, - group: None, - provenance: RecordingProvenance::default(), - epg: None, - } - } - - /// A VOD transfer for `owner`, resolved to `url`. - #[test] - fn borrowed_media_identity_preserves_persisted_keys() { - let url = "http://provider/film.mp4"; - let mut meta = persisted_media("key", "web:alice", RecordingVisibility::Private, url).recording; - assert_eq!( - recording_identity_key(&meta, url), - r#"{"Url":{"url":"http://provider/film.mp4","scheduled_start":null,"scheduled_end":null}}"# - ); - meta.source.target_id = "target".to_string(); - meta.source.virtual_id = "channel".to_string(); - meta.program_start = Some(10); - meta.program_end = Some(20); - assert_eq!( - recording_identity_key(&meta, url), - r#"{"Programme":{"target_id":"target","virtual_id":"channel","program_start":10,"program_end":20}}"# - ); - meta.provenance.rule_id = Some("rule".to_string()); - meta.provenance.occurrence_key = Some("occurrence".to_string()); - assert_eq!( - recording_identity_key(&meta, url), - r#"{"Occurrence":{"rule_id":"rule","occurrence_key":"occurrence"}}"# - ); - } - - fn persisted_media(uuid: &str, owner: &str, visibility: RecordingVisibility, url: &str) -> PersistedRecordingTask { - let meta = RecordingMetadata::new_media( - RecordingOwner::User(UserId::from(owner)), - visibility, - RecordingSource::new("1", "1", "input-a"), - "Film".to_string(), - ); - PersistedRecordingTask { - media_identity: String::new(), - partition: crate::recording::recording_queue::RecordingPartition::default(), - uuid: uuid.to_string(), - kind: RecordingKind::Vod, - file_dir: std::path::PathBuf::from("/tmp"), - file_path: std::path::PathBuf::from(format!("/tmp/{uuid}.mp4")), - filename: format!("{uuid}.mp4"), - url: url.to_string(), - finished: false, - size: 0, - total_size: None, - paused: false, - error: None, - state: RecordingTaskState::Queued, - input_name: Some("input-a".to_string()), - priority: 0, - retry_attempts: 0, - next_retry_at: None, - recording: meta, - } - } - - fn queued_candidate(tasks: Vec) -> PersistedRecordingQueue { - PersistedRecordingQueue { queue: tasks, ..PersistedRecordingQueue::default() } - } - - #[test] - fn a_live_window_must_have_time_left_in_it() { - // A window whose padded end is not after its padded start cannot - // produce a recording, and admitting one means a scheduled capture that - // can only ever fail. - let now = 1_700_000_000; - // Zero-length and inverted programmes. - assert!(matches!( - effective_recording_window(now + 100, now + 100, 0, 0, now), - Err(ServiceError::InvalidInterval) - )); - assert!(matches!( - effective_recording_window(now + 200, now + 100, 0, 0, now), - Err(ServiceError::InvalidInterval) - )); - // A programme that finished before it was asked for. - assert!(matches!( - effective_recording_window(now - 7_200, now - 3_600, 0, 0, now), - Err(ServiceError::InvalidInterval) - )); - } - - #[test] - fn a_live_window_already_underway_records_only_what_is_left() { - // Joining late is legal; it just cannot rewind. The padded bounds stay - // as planned so the stop time is still the programme's. - let now = 1_700_000_000; - let window = effective_recording_window(now - 600, now + 600, 60, 120, now).expect("still running"); - - assert_eq!(window.scheduled_start, now - 660, "the padded start is history, not moved to now"); - assert_eq!(window.scheduled_end, now + 720); - assert_eq!(window.execution_start, now, "but recording starts now"); - assert_eq!(window.remaining_duration_secs, 720, "and runs to the padded end"); - } - - #[test] - fn a_repeated_vod_request_is_a_duplicate() { - // Regression: duplicate detection only produced an identity for Live, - // so a user who asked for the same film twice got two downloads of it - // written side by side as `film.mp4` and `film_1.mp4`. - let existing = persisted_media("a", "web:alice", RecordingVisibility::Private, "http://p/film.mp4"); - let repeat = RecordingQueue::from_persisted(persisted_media( - "b", - "web:alice", - RecordingVisibility::Private, - "http://p/film.mp4", - )) - .expect("valid task"); - assert!(candidate_has_duplicate_recording(&queued_candidate(vec![existing]), &repeat)); - } - - #[test] - fn a_different_film_is_not_a_duplicate() { - let existing = persisted_media("a", "web:alice", RecordingVisibility::Private, "http://p/film.mp4"); - let other = RecordingQueue::from_persisted(persisted_media( - "b", - "web:alice", - RecordingVisibility::Private, - "http://p/other.mp4", - )) - .expect("valid task"); - assert!(!candidate_has_duplicate_recording(&queued_candidate(vec![existing]), &other)); - } - - #[test] - fn another_user_asking_for_the_same_film_is_not_refused() { - // Until one physical file can carry several library entries, treating - // this as a duplicate would tell the second user "already recording" - // and leave them with nothing. - let existing = persisted_media("a", "web:alice", RecordingVisibility::Private, "http://p/film.mp4"); - let other_user = RecordingQueue::from_persisted(persisted_media( - "b", - "web:bob", - RecordingVisibility::Private, - "http://p/film.mp4", - )) - .expect("valid task"); - assert!(!candidate_has_duplicate_recording(&queued_candidate(vec![existing]), &other_user)); - } - - #[test] - fn a_shared_copy_does_not_collide_with_a_private_one() { - // The two charge different quota pools, so they are two recordings. - let existing = persisted_media("a", "web:alice", RecordingVisibility::Private, "http://p/film.mp4"); - let shared = RecordingQueue::from_persisted(persisted_media( - "b", - "web:alice", - RecordingVisibility::Shared, - "http://p/film.mp4", - )) - .expect("valid task"); - assert!(!candidate_has_duplicate_recording(&queued_candidate(vec![existing]), &shared)); - } - - #[test] - fn a_finished_transfer_does_not_block_a_fresh_request() { - // After a completed or failed attempt the user may legitimately want - // another copy. - let mut finished = persisted_media("a", "web:alice", RecordingVisibility::Private, "http://p/film.mp4"); - finished.finished = true; - finished.state = RecordingTaskState::Completed; - let candidate = PersistedRecordingQueue { finished: vec![finished], ..PersistedRecordingQueue::default() }; - let again = RecordingQueue::from_persisted(persisted_media( - "b", - "web:alice", - RecordingVisibility::Private, - "http://p/film.mp4", - )) - .expect("valid task"); - assert!(!candidate_has_duplicate_recording(&candidate, &again)); - } - - #[test] - fn a_completed_rule_occurrence_still_blocks_a_repeat() { - // The scheduler re-evaluates rules on every tick; without this an - // occurrence would be re-materialized forever. - let mut finished = persisted_rule_recording("a", Some("rule-1"), 100); - finished.recording.provenance.occurrence_key = Some("occ-1".to_string()); - finished.finished = true; - finished.state = RecordingTaskState::Completed; - let candidate = PersistedRecordingQueue { finished: vec![finished], ..PersistedRecordingQueue::default() }; - - let mut repeat_persisted = persisted_rule_recording("b", Some("rule-1"), 100); - repeat_persisted.recording.provenance.occurrence_key = Some("occ-1".to_string()); - let repeat = RecordingQueue::from_persisted(repeat_persisted).expect("valid task"); - assert!(candidate_has_duplicate_recording(&candidate, &repeat)); - } - - fn persisted_rule_recording(uuid: &str, rule_id: Option<&str>, start_at: i64) -> PersistedRecordingTask { - let mut meta = RecordingMetadata::new_live( - RecordingOwner::User(UserId::from("web:alice")), - RecordingVisibility::Private, - RecordingSource::new("1", "1", "input-a"), - start_at, - start_at + 3_600, - 0, - 0, - ); - meta.reserved_bytes = 123; - meta.provenance.rule_id = rule_id.map(str::to_string); - PersistedRecordingTask { - media_identity: String::new(), - partition: crate::recording::recording_queue::RecordingPartition::default(), - uuid: uuid.to_string(), - file_dir: std::path::PathBuf::from("/tmp"), - file_path: std::path::PathBuf::from(format!("/tmp/{uuid}.ts")), - filename: format!("{uuid}.ts"), - url: "http://example.test/live.ts".to_string(), - finished: false, - size: 0, - total_size: None, - paused: false, - error: None, - state: RecordingTaskState::Scheduled, - kind: RecordingKind::Live, - input_name: Some("input-a".to_string()), - priority: 0, - retry_attempts: 0, - next_retry_at: None, - recording: meta, - } - } - - #[test] - fn source_input_rejects_empty_virtual_id() { - let input = source("target", "", "i"); - assert!(matches!(input.validate(), Err(ServiceError::InvalidSource))); - } - - #[test] - fn source_input_rejects_empty_input_name() { - let input = source("target", "1", ""); - assert!(matches!(input.validate(), Err(ServiceError::InvalidSource))); - } - - #[test] - fn source_input_accepts_non_empty_identifiers() { - let input = source("target", "1", "input-a"); - assert!(input.validate().is_ok()); - } - - #[test] - fn create_recording_input_rejects_zero_or_negative_interval() { - let mut input = create_input(); - input.program_end = input.program_start; - assert!(matches!(input.validate(), Err(ServiceError::InvalidInterval))); - input.program_end = input.program_start - 1; - assert!(matches!(input.validate(), Err(ServiceError::InvalidInterval))); - } - - #[test] - fn create_recording_input_accepts_valid_interval() { - let input = create_input(); - assert!(input.validate().is_ok()); - } - - #[test] - fn create_recording_input_rejects_overflowing_interval() { - let mut input = create_input(); - input.program_start = i64::MIN; - input.program_end = i64::MAX; - - assert!(matches!(input.validate(), Err(ServiceError::InvalidInterval))); - } - - #[test] - fn effective_window_applies_padding_and_remaining_duration() { - let window = effective_recording_window(1_000, 2_000, 100, 200, 1_500).expect("valid effective window"); - - assert_eq!(window.scheduled_start, 900); - assert_eq!(window.scheduled_end, 2_200); - assert_eq!(window.execution_start, 1_500); - assert_eq!(window.remaining_duration_secs, 700); - } - - #[test] - fn effective_window_rejects_exact_or_past_end_boundary() { - assert!(matches!( - effective_recording_window(1_000, 2_000, 100, 200, 2_200), - Err(ServiceError::InvalidInterval) - )); - assert!(matches!( - effective_recording_window(1_000, 2_000, 100, 200, 2_201), - Err(ServiceError::InvalidInterval) - )); - } - - #[test] - fn effective_window_is_panic_free_at_integer_boundaries() { - let window = - effective_recording_window(i64::MIN + 1, i64::MAX - 1, 10, 10, 0).expect("saturated effective window"); - - assert_eq!(window.scheduled_start, i64::MIN); - assert_eq!(window.scheduled_end, i64::MAX); - assert_eq!(window.execution_start, 0); - assert_eq!(window.remaining_duration_secs, i64::MAX as u64); - } - - #[tokio::test] - async fn edit_recording_rejects_padding_above_max_without_persisting_mutation() { - let dir = tempfile::tempdir().expect("tempdir"); - let state_file = dir.path().join("downloads.json"); - let downloads = - Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); - let task = RecordingQueue::from_persisted(persisted_rule_recording("recording", None, 100)) - .expect("valid recording task"); - downloads.scheduled.write().await.push(task); - downloads.persist_to_disk().await.expect("persist initial queue"); - let persisted_before = committed_records(&downloads).await; - let service = RecordingService::new(Arc::clone(&downloads), test_app_config()); - let claims = shared::model::Claims { - username: "alice".to_string(), - iss: "tuliprox".to_string(), - iat: 0, - exp: 0, - roles: shared::model::RoleSet::new(), - permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, - pwd_version: 0, - subject_id: Some(UserId::from("web:alice")), - permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, - }; - let patch = EditRecordingPatch { pre_roll_secs: Some(901), ..EditRecordingPatch::default() }; - - let result = service.edit_recording(&claims, "recording", patch).await; - - assert!(matches!(result, Err(ServiceError::PaddingLimitExceeded))); - assert_eq!(committed_records(&downloads).await, persisted_before); - let scheduled = downloads.scheduled.read().await; - assert_eq!(scheduled[0].recording.pre_roll_secs, 0); - } - - #[tokio::test] - async fn edit_recording_rejects_active_state_with_invalid_state_error() { - let dir = tempfile::tempdir().expect("tempdir"); - let state_file = dir.path().join("downloads.json"); - let downloads = - Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); - let mut task = RecordingQueue::from_persisted(persisted_rule_recording("recording", None, 100)) - .expect("valid recording task"); - task.state = RecordingTaskState::Running; - downloads.scheduled.write().await.push(task); - downloads.persist_to_disk().await.expect("persist initial queue"); - let persisted_before = committed_records(&downloads).await; - let service = RecordingService::new(Arc::clone(&downloads), test_app_config()); - let claims = shared::model::Claims { - username: "alice".to_string(), - iss: "tuliprox".to_string(), - iat: 0, - exp: 0, - roles: shared::model::RoleSet::new(), - permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, - pwd_version: 0, - subject_id: Some(UserId::from("web:alice")), - permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, - }; - let patch = - EditRecordingPatch { program_title: Some("must not persist".to_string()), ..EditRecordingPatch::default() }; - - let result = service.edit_recording(&claims, "recording", patch).await; - - assert!(matches!(result, Err(ServiceError::InvalidState))); - assert_eq!(committed_records(&downloads).await, persisted_before); - } - - #[tokio::test] - async fn unsupported_range_requires_confirmation_before_vod_or_series_partial_is_discarded() { - for kind in [RecordingKind::Vod, RecordingKind::Series] { - let dir = tempfile::tempdir().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("recording repository")); - let mut task = - persisted_media("recording", "web:alice", RecordingVisibility::Private, "http://provider/film.mp4"); - task.kind = kind; - task.file_dir = dir.path().to_path_buf(); - task.file_path = dir.path().join("recording.mp4"); - task.state = RecordingTaskState::Failed; - task.finished = true; - task.size = 4; - task.total_size = Some(10); - task.error = Some(super::super::recording_transfer::RANGE_UNSUPPORTED_ERROR.to_string()); - task.recording.resume_etag = Some("\"old-etag\"".to_string()); - let partial = crate::recording::recording_worker::recording_partial_path(&task.file_path); - std::fs::write(&partial, b"0123").expect("saved partial"); - mutate(&queue, move |candidate| { - candidate.finished.push(task.clone()); - Ok(()) - }) - .await - .expect("seed failed recording"); - let service = RecordingService::new(Arc::clone(&queue), test_app_config()); - let claims = shared::model::Claims { - username: "alice".to_string(), - iss: "tuliprox".to_string(), - iat: 0, - exp: 0, - roles: shared::model::RoleSet::new(), - permissions: Permission::RecordingManage.into(), - pwd_version: 0, - subject_id: Some(UserId::from("web:alice")), - permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, - }; - - assert!(matches!(service.retry_recording(&claims, "recording").await, Err(ServiceError::InvalidState))); - assert_eq!(std::fs::read(&partial).expect("partial kept without consent"), b"0123"); - assert!(queue.finished.read().await[0].to_view(true).restart_from_beginning_required); - - assert!(service.restart_recording(&claims, "recording").await.expect("confirmed restart")); - assert!(!partial.exists()); - assert!(queue.finished.read().await.is_empty()); - let queued = queue.queue.lock().await.front().cloned().expect("queued transfer"); - assert_eq!(queued.state, RecordingTaskState::Queued); - assert_eq!(queued.size, 0); - assert_eq!(queued.total_size, None); - assert_eq!(queued.recording.resume_etag, None); - } - } - - fn deleting_claims() -> shared::model::Claims { - shared::model::Claims { - username: "alice".to_string(), - iss: "tuliprox".to_string(), - iat: 0, - exp: 0, - roles: shared::model::RoleSet::new(), - permissions: Permission::RecordingDelete.into(), - pwd_version: 0, - subject_id: Some(UserId::from("web:alice")), - permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, - } - } - - /// A finished entry `uuid` of `owner` on the shared film, with its file in - /// `dir`. Both the final file and the partial are written, so a test sees - /// exactly which one a removal touched. - async fn finished_film( - queue: &RecordingQueue, - dir: &std::path::Path, - uuid: &str, - owner: &str, - state: RecordingTaskState, - ) -> (std::path::PathBuf, std::path::PathBuf) { - let mut task = persisted_media(uuid, owner, RecordingVisibility::Private, "http://provider/film.mp4"); - task.file_dir = dir.to_path_buf(); - task.file_path = dir.join("recording.mp4"); - task.state = state; - task.finished = true; - let final_path = task.file_path.clone(); - let partial = crate::recording::recording_worker::recording_partial_path(&final_path); - std::fs::write(&final_path, b"recorded bytes").expect("write final file"); - std::fs::write(&partial, b"partial bytes").expect("write partial"); - let task = RecordingQueue::to_persisted(&RecordingQueue::from_persisted(task).expect("valid task")); - mutate(queue, move |candidate| { - candidate.finished.push(task.clone()); - Ok(()) - }) - .await - .expect("seed recording"); - (final_path, partial) - } - - #[tokio::test] - async fn removing_a_completed_entry_keeps_its_file() { - // Remove takes the entry off the list; the recording stays on disk - // for whoever reads the directory. Deleting the file is its own action. - let dir = tempfile::tempdir().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let (final_path, _) = - finished_film(&queue, dir.path(), "recording", "web:alice", RecordingTaskState::Completed).await; - let service = RecordingService::new(Arc::clone(&queue), test_app_config()); - - assert!(service.remove_recording_task(&deleting_claims(), "recording").await.expect("remove")); - assert!(final_path.exists(), "the completed file was deleted while only removing the entry"); - assert!(queue.finished.read().await.is_empty()); - } - - #[tokio::test] - async fn removing_the_last_failed_or_cancelled_entry_removes_its_partial() { - for state in [RecordingTaskState::Failed, RecordingTaskState::Cancelled] { - let dir = tempfile::tempdir().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let (final_path, partial) = finished_film(&queue, dir.path(), "recording", "web:alice", state).await; - let service = RecordingService::new(Arc::clone(&queue), test_app_config()); - - assert!(service.remove_recording_task(&deleting_claims(), "recording").await.expect("remove")); - assert!(!partial.exists(), "{state:?}: a partial nothing will resume is not kept"); - assert!(final_path.exists(), "{state:?}: a final file is never removed by Remove"); - } - } - - #[test] - fn a_download_claiming_the_path_during_the_cleanup_keeps_its_partial() { - // The orphan check and the unlink happen under one mutation guard. A - // download admitted after the removal may take over the path, and - // must find its partial intact. - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .max_blocking_threads(1) - .build() - .expect("runtime"); - runtime.block_on(async { - let dir = tempfile::tempdir().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new()); - let (final_path, partial) = - finished_film(&queue, dir.path(), "recording", "web:alice", RecordingTaskState::Failed).await; - std::fs::remove_file(&final_path).expect("only the partial exists"); - let mut newcomer = - persisted_media("newcomer", "web:alice", RecordingVisibility::Private, "http://provider/film.mp4"); - newcomer.file_path.clone_from(&final_path); - newcomer.state = RecordingTaskState::Running; - - // Park the filesystem: the cleanup's unlink waits on the only - // blocking thread until released. - let (parked_tx, parked_rx) = std::sync::mpsc::channel(); - let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); - let blocker = tokio::task::spawn_blocking(move || { - parked_tx.send(()).expect("parked"); - let _ = release_rx.recv(); - }); - parked_rx.recv().expect("blocking thread taken"); - - let service = RecordingService::new(Arc::clone(&queue), test_app_config()); - let removal = - tokio::spawn(async move { service.remove_recording_task(&deleting_claims(), "recording").await }); - while !queue.finished.read().await.is_empty() { - tokio::task::yield_now().await; - } - // The newcomer is admitted and starts writing, as soon as it can. - let claim_queue = Arc::clone(&queue); - let claim_partial = partial.clone(); - let claim = tokio::spawn(async move { - mutate(&claim_queue, move |candidate| { - candidate.active = vec![newcomer.clone()]; - Ok(()) - }) - .await - .expect("admit newcomer"); - tokio::fs::write(&claim_partial, b"newcomer's bytes").await.expect("newcomer writes"); - }); - for _ in 0..10 { - tokio::task::yield_now().await; - } - release_tx.send(()).expect("release"); - blocker.await.expect("blocker"); - assert!(removal.await.expect("join").expect("remove")); - claim.await.expect("claim"); - - assert_eq!(queue.active.read().await.first().map(|task| task.uuid.clone()).as_deref(), Some("newcomer")); - assert!(partial.exists(), "the newcomer's partial must survive the cleanup of the old entry"); - }); - } - - #[tokio::test] - async fn removing_an_entry_keeps_a_partial_another_entry_still_holds() { - let dir = tempfile::tempdir().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let (_, partial) = - finished_film(&queue, dir.path(), "recording", "web:alice", RecordingTaskState::Cancelled).await; - let _ = finished_film(&queue, dir.path(), "bob-recording", "web:bob", RecordingTaskState::Failed).await; - let service = RecordingService::new(Arc::clone(&queue), test_app_config()); - - assert!(service.remove_recording_task(&deleting_claims(), "recording").await.expect("remove")); - assert!(partial.exists(), "Bob's entry still points at this media"); - assert_eq!(queue.finished.read().await.len(), 1); - } - - #[tokio::test] - async fn cancelling_an_active_recording_leaves_it_worker_owned_until_the_worker_finishes() { - // A cancel request leaves the active task to its worker: it must not - // show as finished while the worker is still writing, and removing it - // in that state must be refused rather than reported as done. - let dir = tempfile::tempdir().expect("tempdir"); - let state_file = dir.path().join("downloads.json"); - let downloads = - Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); - let mut task = RecordingQueue::from_persisted(persisted_media( - "recording", - "web:alice", - RecordingVisibility::Private, - "http://provider/film.mp4", - )) - .expect("valid recording task"); - task.state = RecordingTaskState::Running; - *downloads.active.write().await = vec![task]; - - let service = RecordingService::new(Arc::clone(&downloads), test_app_config()); - let claims = shared::model::Claims { - username: "alice".to_string(), - iss: "tuliprox".to_string(), - iat: 0, - exp: 0, - roles: shared::model::RoleSet::new(), - permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, - pwd_version: 0, - subject_id: Some(UserId::from("web:alice")), - permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, - }; - - service.cancel_recording(&claims, "recording").await.expect("cancel request accepted"); - - let active = downloads.active.read().await.first().cloned().expect("the worker still owns it"); - assert_eq!(active.state, RecordingTaskState::Cancelling, "the worker commits the terminal state"); - assert!(downloads.finished.read().await.is_empty(), "nothing terminal yet"); - assert!( - matches!(service.remove_recording_task(&claims, "recording").await, Err(ServiceError::InvalidState)), - "removing a worker-owned recording must be refused, not silently ignored" - ); - } - - #[tokio::test] - async fn edit_recording_clears_epg_when_channel_changes_without_programme() { - let dir = tempfile::tempdir().expect("tempdir"); - let state_file = dir.path().join("downloads.json"); - let downloads = - Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); - let mut task = RecordingQueue::from_persisted(persisted_rule_recording("recording", None, 100)) - .expect("valid recording task"); - { - let meta = &mut task.recording; - meta.channel_id = Some("a".into()); - meta.channel_name = Some("A".into()); - meta.epg = Some(shared::model::recording::EpgEpisodeMetadata { - programme_id: Some("p-1".into()), - series_id: None, - episode_id: None, - season: None, - episode: None, - airing: shared::model::recording::AiringStatus::New, - }); - } - downloads.scheduled.write().await.push(task); - downloads.persist_to_disk().await.expect("persist initial queue"); - let service = RecordingService::new(Arc::clone(&downloads), test_app_config()); - let claims = shared::model::Claims { - username: "alice".to_string(), - iss: "tuliprox".to_string(), - iat: 0, - exp: 0, - roles: shared::model::RoleSet::new(), - permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, - pwd_version: 0, - subject_id: Some(UserId::from("web:alice")), - permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, - }; - let patch = EditRecordingPatch { channel_id: Some("b".into()), ..EditRecordingPatch::default() }; - - let result = service.edit_recording(&claims, "recording", patch).await; - assert!(result.is_ok()); - let scheduled = downloads.scheduled.read().await; - let meta = &scheduled[0].recording; - assert_eq!(meta.channel_id.as_deref(), Some("b")); - assert!(meta.epg.is_none(), "epg metadata must be cleared when channel changed without a fresh programme"); - } - - #[tokio::test] - #[allow(clippy::too_many_lines)] - async fn edit_recording_re_validates_quota_against_new_duration_atomically() { - // Owner already has 800 reserved. New duration would push the - // reservation to 1100 against a 1000-byte quota. Edit must - // fail with QuotaExceeded and persist nothing. - let dir = tempfile::tempdir().expect("tempdir"); - let state_file = dir.path().join("downloads.json"); - let downloads = - Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); - let start = chrono::Utc::now().timestamp() + 3_600; - let mut task = RecordingQueue::from_persisted(persisted_rule_recording("recording", None, start)) - .expect("valid recording task"); - { - let meta = &mut task.recording; - meta.reserved_bytes = 800; - } - downloads.scheduled.write().await.push(task); - downloads.persist_to_disk().await.expect("persist initial queue"); - let persisted_before = committed_records(&downloads).await; - let quota = tuliprox_core::model::RecordingQuotaConfig { - default_private_bytes: Some(1_000), - per_user_bytes: HashMap::new(), - shared_bytes: None, - }; - let rec_cfg = RecordingConfig { - headers: HashMap::new(), - t_origin_headers: tuliprox_core::model::RecordingOriginHeaders::default(), - organize_into_directories: false, - episode_pattern: None, - priority: 0, - reserve_slots_for_users: 0, - max_background_per_provider: 0, - retry_backoff_initial_secs: 1, - retry_backoff_multiplier: 1.0, - retry_backoff_max_secs: 1, - retry_backoff_jitter_percent: 0, - retry_max_attempts: 1, - enabled: true, - container_format: RecordingContainerFormat::default(), - directory: String::new(), - timezone: "UTC".parse().expect("UTC must parse"), - filename_template: String::new(), - default_pre_roll_secs: 0, - max_pre_roll_secs: 900, - default_post_roll_secs: 0, - max_post_roll_secs: 1800, - retention: None, - disk: None, - quota: Some(quota), - notifications: RecordingNotificationConfig::default(), - fallback_bytes_per_minute: 60, - }; - let config = tuliprox_core::model::Config { - video: Some(tuliprox_core::model::VideoConfig { - extensions: Vec::new(), - web_search: None, - recording: Some(rec_cfg.clone()), - }), - ..tuliprox_core::model::Config::default() - }; - let app_config = Arc::new(AppConfig { - config: Arc::new(arc_swap::ArcSwap::from_pointee(config)), - sources: Arc::new(arc_swap::ArcSwap::from_pointee(tuliprox_core::model::SourcesConfig::default())), - hdhomerun: Arc::new(arc_swap::ArcSwapOption::empty()), - api_proxy: Arc::new(arc_swap::ArcSwapOption::empty()), - file_locks: Arc::new(tuliprox_core::utils::FileLockManager::default()), - paths: Arc::new(arc_swap::ArcSwap::from_pointee(shared::model::ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(arc_swap::ArcSwapOption::empty()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(tuliprox_core::model::MediaToolCapabilities::default()), - }); - let service = RecordingService::new(Arc::clone(&downloads), app_config); - let claims = shared::model::Claims { - username: "alice".to_string(), - iss: "tuliprox".to_string(), - iat: 0, - exp: 0, - roles: shared::model::RoleSet::new(), - permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, - pwd_version: 0, - subject_id: Some(UserId::from("web:alice")), - permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, - }; - let patch = EditRecordingPatch { program_end: Some(start + 1_200), ..EditRecordingPatch::default() }; - - let result = service.edit_recording(&claims, "recording", patch).await; - - assert!(matches!(result, Err(ServiceError::QuotaExceeded)), "got {result:?}"); - assert_eq!(committed_records(&downloads).await, persisted_before); - let scheduled = downloads.scheduled.read().await; - assert_eq!(scheduled[0].scheduled_start(), Some(start)); - } - - fn editing_claims() -> shared::model::Claims { - shared::model::Claims { - username: "alice".to_string(), - iss: "tuliprox".to_string(), - iat: 0, - exp: 0, - roles: shared::model::RoleSet::new(), - permissions: Permission::RecordingManage.into(), - pwd_version: 0, - subject_id: Some(UserId::from("web:alice")), - permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, - } - } - - /// A scheduled live recording an hour from now, for `owner`. - fn upcoming_live(uuid: &str, owner: &str, start: i64) -> PersistedRecordingTask { - let mut task = persisted_rule_recording(uuid, None, start); - task.recording.owner = RecordingOwner::User(UserId::from(owner)); - task.file_path = std::path::PathBuf::from("/tmp/shared-programme.ts"); - task.filename = "shared-programme.ts".to_string(); - task.recording.relative_path = Some("shared-programme.ts".to_string()); - RecordingQueue::to_persisted(&RecordingQueue::from_persisted(task).expect("valid task")) - } - - async fn scheduled_queue(tasks: Vec) -> Arc { - let queue = Arc::new(RecordingQueue::new()); - mutate(&queue, move |candidate| { - candidate.scheduled.clone_from(&tasks); - Ok(()) - }) - .await - .expect("seed"); - queue - } - - #[tokio::test] - async fn an_edited_window_keeps_its_padding() { - // An edited window is scheduled with its pre- and post-roll, like a - // new one. - let start = chrono::Utc::now().timestamp() + 3_600; - let queue = scheduled_queue(vec![upcoming_live("recording", "web:alice", start)]).await; - let service = RecordingService::new(Arc::clone(&queue), test_app_config()); - - let patch = EditRecordingPatch { - program_end: Some(start + 1_800), - pre_roll_secs: Some(120), - post_roll_secs: Some(300), - ..EditRecordingPatch::default() - }; - service.edit_recording(&editing_claims(), "recording", patch).await.expect("edit"); - - let scheduled = queue.scheduled.read().await; - let meta = &scheduled[0].recording; - assert_eq!(meta.program_start, Some(start)); - assert_eq!(meta.scheduled_start, Some(start - 120)); - assert_eq!(meta.scheduled_end, Some(start + 1_800 + 300)); - } - - #[tokio::test] - async fn a_vod_takes_no_window_but_can_still_be_retitled() { - let queue = scheduled_queue(Vec::new()).await; - let mut vod = persisted_media("film", "web:alice", RecordingVisibility::Private, "http://provider/film.mp4"); - vod.state = RecordingTaskState::Queued; - mutate(&queue, move |candidate| { - candidate.queue.push(vod.clone()); - Ok(()) - }) - .await - .expect("seed"); - let service = RecordingService::new(Arc::clone(&queue), test_app_config()); - - for patch in [ - EditRecordingPatch { program_start: Some(1), program_end: Some(2), ..EditRecordingPatch::default() }, - EditRecordingPatch { pre_roll_secs: Some(60), ..EditRecordingPatch::default() }, - ] { - let refused = service.edit_recording(&editing_claims(), "film", patch).await; - assert!(matches!(refused, Err(ServiceError::InvalidInterval)), "got {refused:?}"); - } - let retitled = - EditRecordingPatch { program_title: Some("Better title".into()), ..EditRecordingPatch::default() }; - service.edit_recording(&editing_claims(), "film", retitled).await.expect("a title edit is fine"); - let queued = queue.queue.lock().await; - assert_eq!(queued[0].recording.program_title.as_deref(), Some("Better title")); - assert_eq!(queued[0].recording.program_start, None, "and it stays a transfer without a window"); - } - - #[tokio::test] - async fn an_edit_that_leaves_shared_media_moves_to_a_path_of_its_own() { - // Alice and Bob scheduled the same programme and hold one file path. - // Alice moving her window makes it different media; keeping the path - // would have two captures write one file and let deleting hers remove - // Bob's. - let start = chrono::Utc::now().timestamp() + 3_600; - let queue = scheduled_queue(vec![ - upcoming_live("alice-entry", "web:alice", start), - upcoming_live("bob-entry", "web:bob", start), - ]) - .await; - let service = RecordingService::new(Arc::clone(&queue), test_app_config()); - - let patch = EditRecordingPatch { program_end: Some(start + 7_200), ..EditRecordingPatch::default() }; - service.edit_recording(&editing_claims(), "alice-entry", patch).await.expect("edit"); - - let (_, tasks) = queue.committed_snapshot().await; - let task = |uuid: &str| tasks.iter().find(|task| task.uuid == uuid).cloned().expect("entry"); - let (alice, bob) = (task("alice-entry"), task("bob-entry")); - assert_ne!(alice.file_path, bob.file_path, "the edited entry no longer writes Bob's file"); - assert_eq!(bob.file_path, std::path::PathBuf::from("/tmp/shared-programme.ts"), "Bob's entry is untouched"); - let identity = |task: &RecordingTask| recording_identity_key(&task.recording, task.url.as_str()); - assert_ne!(identity(&alice), identity(&bob)); - } - - #[tokio::test] - async fn padding_alone_does_not_split_shared_media() { - // The programme window decides the media, its padding does not. - let start = chrono::Utc::now().timestamp() + 3_600; - let queue = scheduled_queue(vec![ - upcoming_live("alice-entry", "web:alice", start), - upcoming_live("bob-entry", "web:bob", start), - ]) - .await; - let service = RecordingService::new(Arc::clone(&queue), test_app_config()); - - let patch = EditRecordingPatch { post_roll_secs: Some(600), ..EditRecordingPatch::default() }; - service.edit_recording(&editing_claims(), "alice-entry", patch).await.expect("edit"); - - let (_, tasks) = queue.committed_snapshot().await; - let paths: Vec<_> = tasks.iter().map(|task| task.file_path.clone()).collect(); - assert_eq!(paths[0], paths[1], "still one file for one programme"); - } - - #[tokio::test] - async fn edit_recording_rejects_overflowing_interval_without_persisting_mutation() { - let dir = tempfile::tempdir().expect("tempdir"); - let state_file = dir.path().join("downloads.json"); - let downloads = - Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); - let task = RecordingQueue::from_persisted(persisted_rule_recording("recording", None, 100)) - .expect("valid recording task"); - downloads.scheduled.write().await.push(task); - downloads.persist_to_disk().await.expect("persist initial queue"); - let persisted_before = committed_records(&downloads).await; - let revision_before = downloads.revision.load(std::sync::atomic::Ordering::SeqCst); - let service = RecordingService::new(Arc::clone(&downloads), test_app_config()); - let claims = shared::model::Claims { - username: "alice".to_string(), - iss: "tuliprox".to_string(), - iat: 0, - exp: 0, - roles: shared::model::RoleSet::new(), - permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, - pwd_version: 0, - subject_id: Some(UserId::from("web:alice")), - permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, - }; - let patch = EditRecordingPatch { - program_start: Some(i64::MIN), - program_end: Some(i64::MAX), - program_title: Some("must not persist".to_string()), - ..EditRecordingPatch::default() - }; - - let result = service.edit_recording(&claims, "recording", patch).await; - - assert!(matches!(result, Err(ServiceError::InvalidInterval))); - assert_eq!(downloads.revision.load(std::sync::atomic::Ordering::SeqCst), revision_before); - assert_eq!(committed_records(&downloads).await, persisted_before); - let scheduled = downloads.scheduled.read().await; - assert_eq!(scheduled[0].scheduled_start(), Some(100)); - assert_ne!(scheduled[0].recording.program_title.as_deref(), Some("must not persist")); - } - - #[tokio::test] - async fn create_recording_without_download_config_reports_disabled() { - // A missing `video.recording` block means the server has no - // download engine at all — the caller's source identifiers are - // not wrong. Reporting `InvalidSource` here sent clients - // hunting for a misconfiguration that does not exist. - let dir = tempfile::tempdir().expect("tempdir"); - let downloads = - Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open recording repository")); - let service = RecordingService::new(Arc::clone(&downloads), test_app_config()); - let claims = shared::model::Claims { - username: "alice".to_string(), - iss: "tuliprox".to_string(), - iat: 0, - exp: 0, - roles: shared::model::RoleSet::new(), - permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, - pwd_version: 0, - subject_id: Some(UserId::from("web:alice")), - permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, - }; - - let result = service.create_recording(&claims, &create_input()).await; - - assert!(matches!(result, Err(ServiceError::Disabled))); - } - - /// The record set the repository actually holds, for tests that - /// assert a rejected mutation left persistence untouched. - async fn committed_records(queue: &RecordingQueue) -> Vec { - let repository = queue.repository.clone().expect("queue is repository backed"); - tokio::task::spawn_blocking(move || { - let mut guard = repository.lock().expect("repository lock"); - guard.load() - }) - .await - .expect("join") - .expect("load") - .tasks - } - - fn test_app_config() -> Arc { - Arc::new(AppConfig { - config: Arc::new(arc_swap::ArcSwap::from_pointee(tuliprox_core::model::Config::default())), - sources: Arc::new(arc_swap::ArcSwap::from_pointee(tuliprox_core::model::SourcesConfig::default())), - hdhomerun: Arc::new(arc_swap::ArcSwapOption::empty()), - api_proxy: Arc::new(arc_swap::ArcSwapOption::empty()), - file_locks: Arc::new(tuliprox_core::utils::FileLockManager::default()), - paths: Arc::new(arc_swap::ArcSwap::from_pointee(shared::model::ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(arc_swap::ArcSwapOption::empty()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(tuliprox_core::model::MediaToolCapabilities::default()), - }) - } - - #[test] - fn service_error_code_is_stable_string() { - assert_eq!(ServiceError::UnknownOwner.code(), "recording_unknown_owner"); - assert_eq!(ServiceError::InvalidSource.code(), "recording_invalid_source"); - assert_eq!(ServiceError::Forbidden.code(), "recording_forbidden"); - assert_eq!(ServiceError::SharedCreationNotAdministrator.code(), "recording_shared_not_administrator"); - assert_eq!(ServiceError::InvalidState.code(), "recording_invalid_state"); - assert_eq!(ServiceError::InvalidInterval.code(), "recording_invalid_interval"); - assert_eq!(ServiceError::UnknownRecording.code(), "recording_unknown"); - assert_eq!(ServiceError::PersistenceFailed.code(), "recording_persistence_failed"); - assert_eq!(ServiceError::ProvenanceImmutable.code(), "recording_provenance_immutable"); - assert_eq!(ServiceError::Disabled.code(), "recording_disabled"); - } - - #[test] - fn provenance_cleared_does_not_masquerade_as_invalid_state() { - assert_eq!(map_edit_validation_error(&EditError::ProvenanceCleared), ServiceError::ProvenanceImmutable); - } - - fn filename_config(template: &str, container_format: RecordingContainerFormat) -> RecordingConfig { - let mut cfg = RecordingConfig::from(&shared::model::RecordingConfigDto::default()); - cfg.filename_template = template.to_string(); - cfg.container_format = container_format; - cfg.timezone = "Europe/Berlin".parse().expect("timezone parses"); - cfg - } - - fn filename_window(input: &CreateRecordingInput) -> EffectiveRecordingWindow { - EffectiveRecordingWindow { - scheduled_start: input.program_start, - scheduled_end: input.program_end, - execution_start: input.program_start, - remaining_duration_secs: 0, - } - } - - fn filename_input() -> CreateRecordingInput { - let mut input = create_input(); - input.program_title = "Hart van Nederland".to_string(); - input.channel_name = Some("┃NLZIET┃ SBS 6 HD".to_string()); - // 2023-11-14 22:13:20 UTC = 23:13 in Berlin. - input.program_start = 1_700_000_000; - input.program_end = 1_700_001_800; - input - } - - #[test] - fn live_filename_renders_the_configured_template() { - let input = filename_input(); - let cfg = filename_config("{channel}_{program_title}_{start_time}", RecordingContainerFormat::Mpegts); - assert_eq!( - render_live_filename(&input, &cfg, &filename_window(&input), "alice"), - "NLZIET_SBS_6_HD_Hart_van_Nederland_2023-11-14_23-13.ts" - ); - } - - #[test] - fn live_filename_collapses_empty_placeholders() { - let mut input = filename_input(); - input.channel_name = None; - let cfg = filename_config("{channel}_{program_title}_{episode}", RecordingContainerFormat::Matroska); - assert_eq!(render_live_filename(&input, &cfg, &filename_window(&input), "alice"), "Hart_van_Nederland.mkv"); - } - - #[test] - fn live_filename_renders_episode_and_owner() { - let mut input = filename_input(); - input.epg = Some(shared::model::EpgEpisodeMetadata { - programme_id: None, - series_id: None, - episode_id: None, - season: Some(3), - episode: Some(9), - airing: shared::model::AiringStatus::default(), - }); - let cfg = filename_config("{owner}-{program_title}-{episode}", RecordingContainerFormat::Mp4); - assert_eq!( - render_live_filename(&input, &cfg, &filename_window(&input), "alice"), - "alice-Hart_van_Nederland-S03E09.mp4" - ); - } - - #[test] - fn live_filename_keeps_dots_and_does_not_double_the_extension() { - let mut input = filename_input(); - input.program_title = "Mr. Robot".to_string(); - let cfg = filename_config("{program_title}.ts", RecordingContainerFormat::Mpegts); - assert_eq!(render_live_filename(&input, &cfg, &filename_window(&input), "alice"), "Mr.Robot.ts"); - let cfg = filename_config("{program_title}", RecordingContainerFormat::Matroska); - assert_eq!(render_live_filename(&input, &cfg, &filename_window(&input), "alice"), "Mr.Robot.mkv"); - } - - #[test] - fn live_filename_without_template_uses_the_title() { - let input = filename_input(); - let cfg = filename_config("", RecordingContainerFormat::Mpegts); - assert_eq!(render_live_filename(&input, &cfg, &filename_window(&input), "alice"), "Hart van Nederland.ts"); - } - - #[test] - fn sanitize_filename_strips_separators_and_reserved_characters() { - assert_eq!(sanitize_filename_component("a/b\\c:d*e?f\"gi|j"), "a_b_c_d_e_f_g_h_i_j"); - } - - #[test] - fn sanitize_filename_collapses_runs_and_drops_control_chars() { - assert_eq!(sanitize_filename_component("a///b"), "a_b"); - assert_eq!(sanitize_filename_component("a\u{7}\u{1}b"), "a_b"); - } - - #[test] - fn sanitize_filename_drops_invisible_formatting() { - // A right-to-left override renders the name differently from the - // bytes on disk; it must leave no trace at all. - assert_eq!(sanitize_filename_component("news\u{202e}sj.ts"), "newssj.ts"); - assert_eq!(sanitize_filename_component("a\u{200b}b"), "ab"); - } - - #[test] - fn sanitize_filename_rejects_traversal_and_empty_results() { - assert_eq!(sanitize_filename_component(""), "recording"); - assert_eq!(sanitize_filename_component("."), "recording"); - assert_eq!(sanitize_filename_component(".."), "recording"); - assert_eq!(sanitize_filename_component(" "), "recording"); - // A lone separator becomes the substitute character, which is - // itself a perfectly valid component. - assert_eq!(sanitize_filename_component("/"), "_"); - } - - #[test] - fn sanitize_filename_rejects_windows_device_names() { - assert_eq!(sanitize_filename_component("CON"), "recording"); - assert_eq!(sanitize_filename_component("nul.ts"), "recording"); - assert_eq!(sanitize_filename_component("lpt9"), "recording"); - // Not reserved: only an exact stem match counts. - assert_eq!(sanitize_filename_component("console"), "console"); - } - - #[test] - fn sanitize_filename_trims_trailing_dots_and_spaces() { - assert_eq!(sanitize_filename_component("Show. "), "Show"); - assert_eq!(sanitize_filename_component(" .Show"), "Show"); - } - - #[test] - fn sanitize_filename_is_idempotent_and_bounded() { - let long = "\u{e9}".repeat(400); - let once = sanitize_filename_component(&long); - assert!(once.len() <= MAX_FILENAME_COMPONENT_BYTES); - // Truncation never splits a character. - assert!(once.chars().all(|ch| ch == '\u{e9}')); - assert_eq!(sanitize_filename_component(&once), once); - for raw in ["a/b", "CON", "", "Show. ", "news\u{202e}sj.ts"] { - let first = sanitize_filename_component(raw); - assert_eq!(sanitize_filename_component(&first), first, "not idempotent: {raw}"); - } - } - - #[test] - fn sanitized_filename_is_always_a_single_valid_component() { - let very_long = "x".repeat(500); - let cases = ["a/b/c", "..", "\u{0}x", "CON", " ", "../../etc/passwd", &very_long]; - for raw in cases { - let sanitized = sanitize_filename_component(raw); - validate_reserved_filename(&sanitized) - .unwrap_or_else(|err| panic!("{raw:?} sanitized to invalid component: {err}")); - } - } - - #[test] - fn cancel_future_rule_recordings_moves_only_matching_future_tasks() { - let now = 1_700_000_000; - let mut queue = PersistedRecordingQueue::default(); - queue.scheduled.push(persisted_rule_recording("future-match", Some("rule-1"), now + 60)); - queue.scheduled.push(persisted_rule_recording("past-match", Some("rule-1"), now - 60)); - queue.queue.push(persisted_rule_recording("other-rule", Some("rule-2"), now + 60)); - - let cancelled = cancel_future_rule_recordings_in_candidate(&mut queue, "rule-1", now); - - assert_eq!(cancelled.len(), 1); - assert_eq!(queue.scheduled.len(), 1); - assert_eq!(queue.queue.len(), 1); - assert_eq!(queue.finished.len(), 1); - let task = &queue.finished[0]; - assert_eq!(task.uuid, "future-match"); - assert_eq!(task.state, RecordingTaskState::Cancelled); - assert!(task.finished); - assert_eq!(task.recording.reserved_bytes, 0); - } - - #[tokio::test] - async fn preview_conflict_collects_demand_points_from_queue_state() { - // The server-side preview must build its own demand points from - // the committed queue state. A queued recording on the same - // target/input pair must show up as `others` even when the - // caller submits no `others` payload. - let dir = tempfile::tempdir().expect("tempdir"); - let state_file = dir.path().join("downloads.json"); - let downloads = - Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); - let mut existing = persisted_rule_recording("existing", None, 100); - // Place a padded window that overlaps 100..200. - existing.recording.scheduled_start = Some(100); - existing.recording.scheduled_end = Some(200); - let existing = RecordingQueue::from_persisted(existing).expect("valid recording task"); - downloads.queue.lock().await.push_back(existing); - let points = collect_demand_points_for_provider(&downloads, "1", "input-a").await; - assert_eq!(points.len(), 1, "queue entry must surface as a demand point"); - assert_eq!(points[0].padded_start, 100); - assert_eq!(points[0].padded_end, 200); - } - - #[tokio::test] - async fn preview_conflict_ignores_other_target_or_input() { - let dir = tempfile::tempdir().expect("tempdir"); - let state_file = dir.path().join("downloads.json"); - let downloads = - Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); - let mut other_target = persisted_rule_recording("other-target", None, 100); - other_target.recording.source = shared::model::recording::RecordingSource::new("other-target", "9", "input-a"); - let mut other_input = persisted_rule_recording("other-input", None, 100); - other_input.recording.source = shared::model::recording::RecordingSource::new("1", "9", "input-b"); - let other_target_task = RecordingQueue::from_persisted(other_target).expect("valid task"); - let other_input_task = RecordingQueue::from_persisted(other_input).expect("valid task"); - downloads.queue.lock().await.push_back(other_target_task); - downloads.queue.lock().await.push_back(other_input_task); - let points = collect_demand_points_for_provider(&downloads, "1", "input-a").await; - assert!(points.is_empty(), "foreign target or input must not leak into the demand set"); - } - - #[test] - fn validate_reserved_filename_rejects_parent_and_curdir_components() { - assert!(validate_reserved_filename("..").is_err()); - assert!(validate_reserved_filename(".").is_err()); - assert!(validate_reserved_filename("a/..").is_err()); - assert!(validate_reserved_filename("normal.ts").is_ok()); - } - - #[test] - fn is_future_rule_recording_rejects_non_editable_states() { - // Old implementation passed `cancel_targets_task(false, true)` - // literally — that always returned `true`, so the rule cancel - // path would happily tear down a task whose state was already - // terminal. Both terminal and non-editable-but-active states - // must be skipped now. - let mut cancelled_task = persisted_rule_recording("uuid-c", Some("rule-1"), 1_900_000_000); - cancelled_task.state = RecordingTaskState::Cancelled; - assert!(!is_future_rule_recording(&cancelled_task, "rule-1", 1_800_000_000)); - - let mut paused_task = persisted_rule_recording("uuid-p", Some("rule-1"), 1_900_000_000); - paused_task.state = RecordingTaskState::Paused; - assert!(!is_future_rule_recording(&paused_task, "rule-1", 1_800_000_000)); - - // Sanity: the happy path still accepts editable future tasks. - let scheduled_task = persisted_rule_recording("uuid-s", Some("rule-1"), 1_900_000_000); - assert!(is_future_rule_recording(&scheduled_task, "rule-1", 1_800_000_000)); - } - - #[tokio::test] - async fn create_recording_rejects_absent_recording_config() { - let config = tuliprox_core::model::Config::default(); - let app_config = Arc::new(AppConfig { - config: Arc::new(arc_swap::ArcSwap::from_pointee(config)), - sources: Arc::new(arc_swap::ArcSwap::from_pointee(tuliprox_core::model::SourcesConfig::default())), - hdhomerun: Arc::new(arc_swap::ArcSwapOption::empty()), - api_proxy: Arc::new(arc_swap::ArcSwapOption::empty()), - file_locks: Arc::new(tuliprox_core::utils::FileLockManager::default()), - paths: Arc::new(arc_swap::ArcSwap::from_pointee(shared::model::ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(arc_swap::ArcSwapOption::empty()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(tuliprox_core::model::MediaToolCapabilities::default()), - }); - let downloads = Arc::new(RecordingQueue::new()); - let service = RecordingService::new(Arc::clone(&downloads), app_config); - let claims = shared::model::Claims { - username: "alice".to_string(), - iss: "tuliprox".to_string(), - iat: 0, - exp: 0, - roles: shared::model::RoleSet::new(), - permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, - pwd_version: 0, - subject_id: Some(UserId::from("web:alice")), - permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, - }; - let input = CreateRecordingInput { - source: RecordingSourceInput { - target_id: "1".to_string(), - virtual_id: "1".to_string(), - cluster: XtreamCluster::Live, - input_name: "input-a".to_string(), - }, - program_title: "title".to_string(), - program_start: 0, - program_end: 60, - pre_roll_secs: 0, - post_roll_secs: 0, - visibility: RecordingVisibility::Private, - channel_id: None, - channel_name: None, - group: None, - provenance: RecordingProvenance::default(), - epg: None, - }; - - let result = service.create_recording(&claims, &input).await; - - assert!( - matches!(result, Err(ServiceError::Disabled)), - "absent recording config must fail closed with Disabled, got: {result:?}" - ); - } - - /// A service rooted at `dir` with the supplied disk block, and the - /// source and server configuration a recording needs to resolve its - /// own target and URL. - fn service_with_disk( - dir: &std::path::Path, - queue: &Arc, - disk: Option, - ) -> RecordingService { - let mut rec_cfg = - RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }); - rec_cfg.directory = dir.to_string_lossy().into_owned(); - rec_cfg.disk = disk; - let config = tuliprox_core::model::Config { - video: Some(tuliprox_core::model::VideoConfig { - extensions: Vec::new(), - web_search: None, - recording: Some(rec_cfg), - }), - ..tuliprox_core::model::Config::default() - }; - - let input = Arc::new(tuliprox_core::model::ConfigInput { id: 7, name: "input-a".into(), ..Default::default() }); - let target = Arc::new(tuliprox_core::model::ConfigTarget { - id: 11, - enabled: true, - name: "1".to_string(), - options: None, - sort: None, - filter: tuliprox_core::model::StagedFilter::default(), - output: vec![], - rename: None, - mapping_ids: None, - mapping: Arc::default(), - favourites: None, - processing_order: shared::model::ProcessingOrder::default(), - curation: None, - execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - }); - let sources = tuliprox_core::model::SourcesConfig { - inputs: vec![Arc::clone(&input)], - sources: vec![tuliprox_core::model::ConfigSource { inputs: vec!["input-a".into()], targets: vec![target] }], - ..tuliprox_core::model::SourcesConfig::default() - }; - - let app_config = test_app_config(); - app_config.config.store(Arc::new(config)); - app_config.sources.store(Arc::new(sources)); - RecordingService::new(Arc::clone(queue), app_config) - } - - fn creating_claims() -> shared::model::Claims { - shared::model::Claims { - username: "alice".to_string(), - iss: "tuliprox".to_string(), - iat: 0, - exp: 0, - roles: shared::model::RoleSet::new(), - permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, - pwd_version: 0, - subject_id: Some(UserId::from("web:alice")), - permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, - } - } - - fn claims_for(user: &str, permissions: shared::model::permission::PermissionSet) -> shared::model::Claims { - shared::model::Claims { - username: user.to_string(), - subject_id: Some(UserId::from(format!("web:{user}"))), - permissions, - ..creating_claims() - } - } - - fn media_input() -> CreateMediaRecordingInput { - CreateMediaRecordingInput { - source: RecordingSourceInput { - target_id: "1".to_string(), - virtual_id: "77".to_string(), - cluster: XtreamCluster::Video, - input_name: "input-a".to_string(), - }, - title: "The Film".to_string(), - extension: "mp4".to_string(), - visibility: RecordingVisibility::Private, - group: None, - series_name: None, - } - } - - #[tokio::test] - async fn a_media_request_needs_the_create_permission_not_manage() { - let dir = tempfile::tempdir().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let service = service_with_disk(dir.path(), &queue, None); - - let manage_only = claims_for("alice", Permission::RecordingManage.into()); - let refused = service.create_media_recording_idempotent(&manage_only, &media_input(), None).await; - assert!(matches!(refused, Err(ServiceError::Forbidden)), "got {refused:?}"); - assert!(queue.queue.lock().await.is_empty()); - - let create_only = claims_for("alice", Permission::RecordingCreate.into()); - let admitted = service.create_media_recording_idempotent(&create_only, &media_input(), None).await; - assert!(admitted.is_ok(), "got {admitted:?}"); - let queued = queue.queue.lock().await; - assert_eq!(queued.len(), 1, "a transfer is queued, not scheduled"); - assert_eq!(queued[0].kind, RecordingKind::Vod); - } - - #[tokio::test] - async fn a_second_user_gets_an_own_entry_and_a_repeat_is_a_duplicate() { - // Another user asking for the same film must neither be handed the - // first user's entry nor be refused; the same user asking twice is - // a duplicate. - let dir = tempfile::tempdir().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let service = service_with_disk(dir.path(), &queue, None); - let alice = claims_for("alice", Permission::RecordingCreate.into()); - let bob = claims_for("bob", Permission::RecordingCreate.into()); - - let first = service.create_media_recording_idempotent(&alice, &media_input(), None).await.expect("alice"); - let second = service.create_media_recording_idempotent(&bob, &media_input(), None).await.expect("bob"); - assert_ne!(first.uuid, second.uuid); - assert_eq!(second.owner_id, UserId::from("web:bob")); - - let repeat = service.create_media_recording_idempotent(&alice, &media_input(), None).await; - assert!(matches!(repeat, Err(ServiceError::Duplicate)), "got {repeat:?}"); - assert_eq!(queue.queue.lock().await.len(), 2); - } - - #[tokio::test] - async fn a_replayed_media_request_is_answered_without_queueing_again() { - let dir = tempfile::tempdir().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let service = service_with_disk(dir.path(), &queue, None); - let alice = claims_for("alice", Permission::RecordingCreate.into()); - let request = IdempotencyRequest { key: "k1".to_string(), fingerprint: "f1".to_string() }; - - let first = service - .create_media_recording_idempotent(&alice, &media_input(), Some(request.clone())) - .await - .expect("first request"); - let replay = service.create_media_recording_idempotent(&alice, &media_input(), Some(request)).await; - assert_eq!(replay.err(), Some(ServiceError::IdempotentReplay { recording_id: first.uuid })); - assert_eq!(queue.queue.lock().await.len(), 1); - } - - fn with_private_quota(service: &RecordingService, bytes: u64) { - let mut config = tuliprox_core::model::Config::clone(&service.app_config.config.load()); - if let Some(recording) = config.video.as_mut().and_then(|video| video.recording.as_mut()) { - recording.quota = Some(tuliprox_core::model::RecordingQuotaConfig { - default_private_bytes: Some(bytes), - ..Default::default() - }); - } - service.app_config.config.store(Arc::new(config)); - } - - /// Alice's request for the film, finished at `bytes`. - async fn completed_film(service: &RecordingService, queue: &RecordingQueue, bytes: u64) { - service - .create_media_recording_idempotent(&creating_claims(), &media_input(), None) - .await - .expect("alice's request"); - mutate(queue, |candidate| { - let mut done = candidate.queue.remove(0); - done.state = RecordingTaskState::Completed; - done.finished = true; - done.size = bytes; - done.total_size = Some(bytes); - done.recording.measured_bytes = bytes; - candidate.finished.push(done); - Ok(()) - }) - .await - .expect("complete"); - } - - #[tokio::test] - async fn asking_for_a_finished_file_is_charged_its_size_at_admission() { - // Attaching copies the whole file into the new entry's charge, so a - // 16 byte quota must not let a 4096 byte file in. - let dir = tempfile::tempdir().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let service = service_with_disk(dir.path(), &queue, None); - completed_film(&service, &queue, 4096).await; - with_private_quota(&service, 16); - - let bob = claims_for("bob", Permission::RecordingCreate.into()); - let refused = service.create_media_recording_idempotent(&bob, &media_input(), None).await; - assert!(matches!(refused, Err(ServiceError::QuotaExceeded)), "got {refused:?}"); - - with_private_quota(&service, 8192); - service.create_media_recording_idempotent(&bob, &media_input(), None).await.expect("fits now"); - let queued = queue.queue.lock().await; - assert_eq!(queued.len(), 1); - assert_eq!(queued[0].recording.reserved_bytes, 4096, "reserved at the known size until it attaches"); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn two_concurrent_requests_with_one_key_admit_only_one() { - // Both requests pass the first key lookup before either commits; the - // lookup under the mutation guard admits only one of them. - let dir = tempfile::tempdir().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let service = Arc::new(service_with_disk(dir.path(), &queue, None)); - let held = queue.queue.lock().await; - let handles: Vec<_> = [("77", "body-a"), ("78", "body-b")] - .into_iter() - .map(|(virtual_id, fingerprint)| { - let service = Arc::clone(&service); - let mut input = media_input(); - input.source.virtual_id = virtual_id.to_string(); - let request = IdempotencyRequest { key: "same-key".to_string(), fingerprint: fingerprint.to_string() }; - tokio::spawn(async move { - service.create_media_recording_idempotent(&creating_claims(), &input, Some(request)).await - }) - }) - .collect(); - // Both are past their first lookup and waiting to commit. - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - drop(held); - - let mut admitted = 0; - for handle in handles { - match handle.await.expect("join") { - Ok(_) => admitted += 1, - Err(error) => assert_eq!(error, ServiceError::IdempotencyConflict), - } - } - assert_eq!(admitted, 1); - assert_eq!(queue.queue.lock().await.len(), 1); - } - - fn disk_test_input() -> CreateRecordingInput { - let now = chrono::Utc::now().timestamp(); - CreateRecordingInput { - source: RecordingSourceInput { - target_id: "1".to_string(), - virtual_id: "42".to_string(), - cluster: XtreamCluster::Live, - input_name: "input-a".to_string(), - }, - program_title: "title".to_string(), - program_start: now, - program_end: now + 600, - pre_roll_secs: 0, - post_roll_secs: 0, - visibility: RecordingVisibility::Private, - channel_id: None, - channel_name: None, - group: None, - provenance: RecordingProvenance::default(), - epg: None, - } - } - - #[tokio::test] - async fn a_recording_with_no_room_on_disk_is_refused() { - // Logical quota and physical space are different questions, and - // admission asks both: a full disk refuses the recording up front - // instead of letting ffmpeg fail on ENOSPC. The safety margin drives - // headroom to zero here rather than actually filling a filesystem. - let dir = tempfile::tempdir().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let service = service_with_disk( - dir.path(), - &queue, - Some(tuliprox_core::model::RecordingDiskConfig { - high_water_percent: None, - low_water_percent: None, - cleanup_interval_secs: None, - safety_bytes: Some(u64::MAX), - }), - ); - - let result = service.create_recording(&creating_claims(), &disk_test_input()).await; - - assert!(matches!(result, Err(ServiceError::DiskFull)), "got {result:?}"); - assert!(queue.scheduled.read().await.is_empty(), "a refused admission must not leave a recording behind"); - } - - #[tokio::test] - async fn the_same_recording_is_admitted_when_the_disk_has_room() { - // The counterpart: without the safety margin the identical request - // succeeds, so the refusal above is the disk rule and not the - // fixture failing for some unrelated reason. - let dir = tempfile::tempdir().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let service = service_with_disk(dir.path(), &queue, None); - - let result = service.create_recording(&creating_claims(), &disk_test_input()).await; - - assert!(result.is_ok(), "got {result:?}"); - assert_eq!(queue.scheduled.read().await.len(), 1); - } -} +mod tests; + +mod conflicts; +mod control; +mod create; +mod deletion; +mod edit; +mod error; +mod filenames; +mod identity; +mod input; +mod rules; +mod window; +#[cfg(test)] +use conflicts::collect_demand_points_for_provider; +pub use conflicts::{quota_limits_from_config, ConflictPreviewRequest}; +#[cfg(test)] +use control::is_future_rule_recording; +use control::CancelOrigin; +pub use create::IdempotencyRequest; +pub use error::ServiceError; +pub use filenames::sanitize_filename_component; +#[cfg(test)] +use filenames::{validate_reserved_filename, MAX_FILENAME_COMPONENT_BYTES}; +pub(crate) use identity::recording_identity; +pub use identity::{recording_identity_key, RecordingIdentity}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use input::{ + CreateMediaRecordingInput, CreateRecordingInput, EditRecordingPatch, RecordingSourceInput, RecordingTaskView, +}; diff --git a/backend/dvr/src/recording/recording_service/conflicts.rs b/backend/dvr/src/recording/recording_service/conflicts.rs new file mode 100644 index 000000000..1d69f8d7c --- /dev/null +++ b/backend/dvr/src/recording/recording_service/conflicts.rs @@ -0,0 +1,191 @@ +use super::{ + error::map_edit_validation_error, window::padding_bounds, RecordingService, RecordingSourceInput, ServiceError, +}; +use crate::{ + recording::recording_queue::{ + PersistedRecordingQueue, PersistedRecordingTask, QueueMutationError, RecordingTask, RecordingTaskState, + }, + recording_edit, + recording_edit::EditError, + recording_quota, + recording_quota::{QuotaLimits, QuotaPool}, +}; +use shared::model::{RecordingKind, UserId}; +use std::{collections::HashMap, sync::Arc}; +use tuliprox_auth::{authorize, RecordingAction, RecordingDecision, RecordingSubject, TerminalState}; + +impl RecordingService { + /// Server-side conflict preview. The caller submits a candidate + /// padded interval plus server-owned source identifiers. The + /// server enumerates the committed queue state for that + /// provider/input, builds the demand points itself, resolves the + /// effective capacity from config, and runs the deterministic + /// analyzer. The request never carries another user's + /// `others`, capacity, or provider identifier. + pub async fn preview_conflicts( + &self, + claims: &shared::model::Claims, + request: &ConflictPreviewRequest, + ) -> Result { + // Require an authenticated principal. The owner id is not + // needed for the analyzer (the privacy contract applies to + // the response), but a missing / invalid claim must reject. + Self::subject_id(claims)?; + // Reject malformed input up front so the analyzer never sees + // garbage. The endpoint enforces the same bounds; this is the + // service-layer defense in depth. + let bounds = padding_bounds(self.app_config.config.load().recording()); + recording_edit::validate_padding(request.pre_roll_secs, request.post_roll_secs, bounds) + .map_err(|error: EditError| map_edit_validation_error(&error))?; + if request.padded_start >= request.padded_end { + return Err(ServiceError::InvalidInterval); + } + // Server resolves the source. The candidate must map to a + // configured provider; anything else is `InvalidSource`. + if self.recording_url(&request.source).is_none() { + return Err(ServiceError::InvalidSource); + } + // Capacity comes from config, not the caller. The runtime + // slot model is the source of truth. + let capacity = effective_capacity_from_config(&self.app_config.config.load()); + // Demand points come from the committed queue state. The + // privacy contract from `recording_conflict.rs` still applies + // — the response only carries anonymized segments. + let others = + collect_demand_points_for_provider(&self.recordings, &request.source.target_id, &request.source.input_name) + .await; + let candidate = crate::recording_conflict::DemandPoint { + task_id: String::new(), + padded_start: request.padded_start, + padded_end: request.padded_end, + priority: request.priority, + }; + let provider_scope = Some(request.source.target_id.clone()); + Ok(crate::recording_conflict::preview_conflict(&candidate, &others, capacity, provider_scope)) + } +} + +pub fn quota_limits_from_config(config: Option<&tuliprox_core::model::RecordingQuotaConfig>) -> QuotaLimits { + let mut per_user_bytes = HashMap::new(); + if let Some(config) = config { + for (user_id, bytes) in &config.per_user_bytes { + per_user_bytes.insert(UserId::from(user_id.clone()), *bytes); + } + QuotaLimits { + default_private_bytes: config.default_private_bytes, + per_user_bytes, + shared_bytes: config.shared_bytes, + } + } else { + QuotaLimits::default() + } +} + +/// Every task in the candidate snapshot, borrowed. Admission checks run +/// inside `mutate`, so this must not allocate a clone per task — the +/// previous implementation built a `Vec` of the +/// entire queue on every create and every edit. +/// Authorize an action against a task found in the candidate snapshot. +/// Called inside the queue mutation so the ownership decision and the +/// state change commit together. +pub(super) fn authorize_task_in_candidate( + candidate: &PersistedRecordingQueue, + uuid: &str, + claims: &shared::model::Claims, + subject_id: &UserId, + action: RecordingAction, +) -> Result<(), QueueMutationError> { + let Some(task) = candidate_tasks(candidate).find(|task| task.uuid == uuid) else { + return Err(QueueMutationError::UnknownRecording); + }; + let subject = RecordingSubject::new(Some(&task.recording), TerminalState::Active, true); + match authorize(claims, subject_id, action, &subject) { + RecordingDecision::Allow => Ok(()), + RecordingDecision::Deny(_) => Err(QueueMutationError::Forbidden), + } +} + +pub(super) fn candidate_tasks( + candidate: &PersistedRecordingQueue, +) -> impl Iterator + '_ { + candidate + .queue + .iter() + .chain(candidate.scheduled.iter()) + .chain(candidate.active.iter()) + .chain(candidate.finished.iter()) +} + +/// Bytes charged against a single quota pool. Only the pool the caller +/// asked about is summed; the previous implementation built the full +/// per-user `HashMap` and then read one entry out of it. +pub(super) fn used_bytes_for_pool(candidate: &PersistedRecordingQueue, pool: &QuotaPool) -> u64 { + recording_quota::used_bytes_in_pool(candidate_tasks(candidate), pool) +} + +/// Server-owned input for the conflict preview. The caller never +/// supplies another recording's padded interval, capacity, or +/// provider identifier — those are derived server-side. +#[derive(Debug, Clone)] +pub struct ConflictPreviewRequest { + pub source: RecordingSourceInput, + pub padded_start: i64, + pub padded_end: i64, + pub pre_roll_secs: u64, + pub post_roll_secs: u64, + pub priority: i32, +} + +fn effective_capacity_from_config( + config: &tuliprox_core::model::Config, +) -> crate::recording_conflict::EffectiveCapacity { + let background_slots = config.recording().map_or(0, |cfg| u32::from(cfg.max_background_per_provider)); + let reserved = config.recording().map_or(0, |cfg| u32::from(cfg.reserve_slots_for_users)); + crate::recording_conflict::EffectiveCapacity { background_slots, reserved_interactive_slots: reserved } +} + +pub(super) async fn collect_demand_points_for_provider( + queue: &Arc, + target_id: &str, + input_name: &str, +) -> Vec { + use crate::recording_conflict::DemandPoint; + fn matches(task: &RecordingTask, target_id: &str, input_name: &str) -> bool { + task.kind == RecordingKind::Live + && task.recording.source.target_id == target_id + && task.recording.source.input_name == input_name + } + fn to_demand_point(task: &RecordingTask) -> Option { + let meta = &task.recording; + let start = meta.scheduled_start?; + let end = meta.scheduled_end?; + if end <= start { + return None; + } + Some(DemandPoint { + task_id: task.uuid.clone(), + padded_start: start, + padded_end: end, + priority: i32::from(task.priority), + }) + } + // Pending and active recordings are real capacity consumers. + // Finished recordings no longer claim slots, so they would only + // inflate the conflict preview's `peak_demand`. + fn claims_a_slot(task: &RecordingTask) -> bool { + !matches!( + task.state, + RecordingTaskState::Completed | RecordingTaskState::Failed | RecordingTaskState::Cancelled + ) + } + // One committed snapshot rather than three sequential guards. Reading + // `scheduled`, then `queue`, then `active` in turn let a task that + // moved between two of those reads be counted twice or not at all, + // which silently shifted the reported severity. + let (_revision, tasks) = queue.committed_snapshot().await; + tasks + .iter() + .filter(|task| claims_a_slot(task) && matches(task, target_id, input_name)) + .filter_map(to_demand_point) + .collect() +} diff --git a/backend/dvr/src/recording/recording_service/control.rs b/backend/dvr/src/recording/recording_service/control.rs new file mode 100644 index 000000000..b7ddbcefd --- /dev/null +++ b/backend/dvr/src/recording/recording_service/control.rs @@ -0,0 +1,387 @@ +use super::{ + conflicts::{authorize_task_in_candidate, candidate_tasks}, + CancelledRuleRecording, RecordingService, ServiceError, +}; +use crate::{ + recording::recording_queue::{ + mutate, mutate_prepared, mutate_then, PersistedRecordingQueue, PersistedRecordingTask, QueueMutationError, + RecordingTaskState, + }, + recording_edit, +}; +use shared::model::{recording::RecordingVisibility, RecordingKind, UserId}; +use tuliprox_auth::{authorize, RecordingAction, RecordingDecision, RecordingSubject, TerminalState}; + +impl RecordingService { + /// Cancel an in-flight or scheduled recording. + /// + /// An active recording is asked to stop, not marked stopped: the worker + /// still owns the file and the provider slot, so it must release them + /// first and is the one that commits `Cancelled`. Marking it terminal + /// here would show it as a finished recording while it is still writing, + /// and a later remove would then find it in the active slot, leave it + /// there, and report success. + pub async fn cancel_recording(&self, claims: &shared::model::Claims, uuid: &str) -> Result<(), ServiceError> { + let owner_id = Self::subject_id(claims)?; + let active = self.recordings.active.read().await.iter().find(|task| task.uuid == uuid).cloned(); + if let Some(active) = active { + let meta = active.recording.clone(); + let subject = RecordingSubject::new(Some(&meta), TerminalState::Active, true); + if !matches!(authorize(claims, &owner_id, RecordingAction::Cancel, &subject), RecordingDecision::Allow) { + return Err(ServiceError::Forbidden); + } + // Cancel by uuid. Between the read above and this call ffmpeg + // can finish and the queue can promote a *different* recording + // into the active slot; cancelling whatever is active would then + // kill that innocent recording. + match self.recordings.cancel_requested(uuid).await { + // The task is active (running, waiting, or paused). A paused + // one has already been moved to `finished`; a worker-owned one + // is now `Cancelling` and the worker will finish it. + Ok(Some(_)) => return Ok(()), + // The task left the active slot in the meantime. Fall + // through to the inactive path: it either finds the task + // in `scheduled`/`queue` (a re-promotion) or reports + // `UnknownRecording`, which is the truthful answer. + Ok(None) => {} + Err(err) => { + log::error!("cancel_requested failed for {uuid}: {err}"); + return Err(ServiceError::PersistenceFailed); + } + } + } + mutate(&self.recordings, |candidate| { + let Some(task) = remove_inactive_recording(candidate, uuid) else { + return Err(QueueMutationError::UnknownRecording); + }; + let meta = &task.recording; + let subject = RecordingSubject::new(Some(meta), TerminalState::Active, true); + if !matches!(authorize(claims, &owner_id, RecordingAction::Cancel, &subject), RecordingDecision::Allow) { + return Err(QueueMutationError::Forbidden); + } + let mut cancelled = task; + cancelled.state = RecordingTaskState::Cancelled; + cancelled.finished = true; + cancelled.error = Some("cancelled".to_string()); + cancelled.recording.reserved_bytes = 0; + candidate.finished.push(cancelled); + Ok(()) + }) + .await + .map_err(|e| map_queue_error(&e)) + } + + /// Cancel future inactive recordings that were materialized from a + /// recurring rule. Active recordings are intentionally left untouched. + /// Returns the pre-cancel snapshots of everything it cancelled. The + /// caller is mid-way through a two-store operation (cancel the + /// occurrences, then delete the rule) that cannot be made atomic, so + /// it keeps these to undo the queue side if the rule store fails — + /// see [`Self::restore_cancelled_rule_recordings`]. + pub async fn pause_recording(&self, claims: &shared::model::Claims, uuid: &str) -> Result<(), ServiceError> { + let owner_id = Self::subject_id(claims)?; + let active = self.recordings.active.read().await.iter().find(|task| task.uuid == uuid).cloned(); + if let Some(active) = active { + let meta = active.recording.clone(); + if !active.kind.is_resumable() { + return Err(ServiceError::InvalidState); // Live cannot be paused + } + let subject = RecordingSubject::new(Some(&meta), TerminalState::Active, true); + if !matches!(authorize(claims, &owner_id, RecordingAction::Edit, &subject), RecordingDecision::Allow) { + return Err(ServiceError::Forbidden); + } + self.recordings.pause_active(uuid).await.map_err(|_| ServiceError::PersistenceFailed)?; + return Ok(()); + } + Err(ServiceError::UnknownRecording) + } + + pub async fn resume_recording(&self, claims: &shared::model::Claims, uuid: &str) -> Result { + let owner_id = Self::subject_id(claims)?; + let active = self.recordings.active.read().await.iter().find(|task| task.uuid == uuid).cloned(); + if let Some(active) = active { + let meta = active.recording.clone(); + if !active.kind.is_resumable() { + return Err(ServiceError::InvalidState); + } + let subject = RecordingSubject::new(Some(&meta), TerminalState::Active, true); + if !matches!(authorize(claims, &owner_id, RecordingAction::Edit, &subject), RecordingDecision::Allow) { + return Err(ServiceError::Forbidden); + } + return self.recordings.resume_active(uuid).await.map_err(|_| ServiceError::PersistenceFailed); + } + Err(ServiceError::UnknownRecording) + } + + /// Remove a task from the queue. The ownership check runs inside the + /// same mutation that removes it, so a task cannot change hands + /// between the check and the write. + pub async fn remove_recording_task( + &self, + claims: &shared::model::Claims, + uuid: &str, + ) -> Result { + let owner_id = Self::subject_id(claims)?; + let (removed, _) = mutate_then( + &self.recordings, + |candidate| { + authorize_task_in_candidate(candidate, uuid, claims, &owner_id, RecordingAction::Delete)?; + // The active slot is owned by the worker. Removing a task from + // it here would leave the worker writing to a file no entry + // names, so refuse instead of silently doing nothing (the retain + // below cannot reach the active slot). + if candidate.active.iter().any(|active| active.uuid == uuid) { + return Err(QueueMutationError::StateNotEditable); + } + // Removing an entry keeps a finished recording on disk; that is + // what separates it from deleting the file. A partial is + // different: with no entry and no worker left on its media, it + // is never resumed. A worker on the same media holds the active + // slot, so it counts as a reference here. + let orphaned_partial = candidate_tasks(candidate) + .find(|task| task.uuid == uuid) + .filter(|_| !crate::recording::recording_queue::media_is_still_referenced(candidate, uuid)) + .map(|task| task.file_path.clone()); + let original = candidate.queue.len() + candidate.scheduled.len() + candidate.finished.len(); + candidate.queue.retain(|task| task.uuid != uuid); + candidate.scheduled.retain(|task| task.uuid != uuid); + candidate.finished.retain(|task| task.uuid != uuid); + let current = candidate.queue.len() + candidate.scheduled.len() + candidate.finished.len(); + let removed = current != original; + Ok((removed, orphaned_partial.filter(|_| removed))) + }, + // Still under the guard that decided the partial is orphaned: a + // request admitted after it may reuse the path, and must find it + // either still claimed or already empty, never emptied under it. + |(_, orphaned_partial)| { + let orphaned_partial = orphaned_partial.clone(); + async move { + let Some(file_path) = orphaned_partial else { + return; + }; + if let Err(err) = crate::recording_deletion::remove_orphaned_partial(&file_path).await { + log::warn!("Removed a recording, but could not remove its leftover partial: {err}"); + } + } + }, + ) + .await + .map_err(|e| map_queue_error(&e))?; + Ok(removed) + } + + /// Requeue a finished VOD/Series transfer. Live is rejected by the + /// queue itself: its programme window is gone. + pub async fn retry_recording(&self, claims: &shared::model::Claims, uuid: &str) -> Result { + self.retry_recording_inner(claims, uuid, false).await + } + + /// Discard a partial only after the caller confirmed that a provider + /// without byte-range support should download the file from the start. + pub async fn restart_recording(&self, claims: &shared::model::Claims, uuid: &str) -> Result { + self.retry_recording_inner(claims, uuid, true).await + } + + pub(super) async fn retry_recording_inner( + &self, + claims: &shared::model::Claims, + uuid: &str, + restart_from_beginning: bool, + ) -> Result { + let owner_id = Self::subject_id(claims)?; + let recordings = &self.recordings; + let requester = &owner_id; + mutate_prepared( + recordings, + // A restart discards the partial before the task is queued + // again, so a worker can never resume from bytes the provider + // cannot continue. It runs once the request is known to be + // allowed, and before the mutation, under the same guard. + || async move { + if !restart_from_beginning { + return Ok(()); + } + let partial = { + let finished = recordings.finished.read().await; + let Some(task) = finished.iter().find(|task| task.uuid == uuid) else { + return Ok(()); + }; + let subject = RecordingSubject::new(Some(&task.recording), TerminalState::Active, true); + if !matches!( + authorize(claims, requester, RecordingAction::Edit, &subject), + RecordingDecision::Allow + ) { + return Err(QueueMutationError::Forbidden); + } + if task.error.as_deref() != Some(super::super::recording_transfer::RANGE_UNSUPPORTED_ERROR) { + return Err(QueueMutationError::StateNotEditable); + } + crate::recording::recording_worker::recording_partial_path(&task.file_path) + }; + match tokio::fs::remove_file(&partial).await { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(QueueMutationError::from_io(error)), + } + }, + |candidate, ()| { + authorize_task_in_candidate(candidate, uuid, claims, &owner_id, RecordingAction::Edit)?; + let Some(pos) = candidate.finished.iter().position(|task| task.uuid == uuid) else { + return Ok(false); + }; + if !candidate.finished[pos].kind.is_resumable() { + return Err(QueueMutationError::StateNotEditable); + } + let range_unsupported = candidate.finished[pos].error.as_deref() + == Some(super::super::recording_transfer::RANGE_UNSUPPORTED_ERROR); + if range_unsupported != restart_from_beginning { + return Err(QueueMutationError::StateNotEditable); + } + let mut task = candidate.finished.remove(pos); + task.finished = false; + task.size = 0; + if restart_from_beginning { + task.total_size = None; + task.recording.resume_etag = None; + task.recording.resume_last_modified = None; + } + task.paused = false; + task.error = None; + task.state = RecordingTaskState::Queued; + task.retry_attempts = 0; + task.next_retry_at = None; + candidate.queue.push(task); + Ok(true) + }, + ) + .await + .map_err(|e| map_queue_error(&e)) + } +} + +pub(super) fn authorize_create_recording( + claims: &shared::model::Claims, + owner_id: &UserId, + visibility: RecordingVisibility, +) -> Result<(), ServiceError> { + let action = match visibility { + RecordingVisibility::Private => RecordingAction::CreatePrivate, + RecordingVisibility::Shared => RecordingAction::CreateShared, + }; + match authorize(claims, owner_id, action, &RecordingSubject::new(None, TerminalState::Active, true)) { + RecordingDecision::Allow => Ok(()), + RecordingDecision::Deny(tuliprox_auth::DenyReason::NotAdministrator) => { + Err(ServiceError::SharedCreationNotAdministrator) + } + RecordingDecision::Deny(_) => Err(ServiceError::Forbidden), + } +} + +/// Map a queue-mutation failure onto the service error surface. +/// +/// Every call site used to enumerate all twelve `QueueMutationError` +/// variants inline, so adding a variant meant editing four or more +/// matches. The variants that carry no site-specific meaning collapse +/// here; a site that needs a different mapping for one variant still +/// handles it before delegating. +pub(super) fn map_queue_error(err: &QueueMutationError) -> ServiceError { + match err { + QueueMutationError::Io(_) => ServiceError::PersistenceFailed, + QueueMutationError::UnknownRecording => ServiceError::UnknownRecording, + QueueMutationError::Forbidden => ServiceError::Forbidden, + QueueMutationError::InvalidInterval => ServiceError::InvalidInterval, + QueueMutationError::PaddingLimitExceeded => ServiceError::PaddingLimitExceeded, + QueueMutationError::QuotaExceeded => ServiceError::QuotaExceeded, + QueueMutationError::Duplicate => ServiceError::Duplicate, + QueueMutationError::InvalidPath => ServiceError::InvalidPath, + QueueMutationError::DiskFull => ServiceError::DiskFull, + QueueMutationError::IdempotentReplay { recording_id } => { + ServiceError::IdempotentReplay { recording_id: recording_id.clone() } + } + QueueMutationError::IdempotencyConflict => ServiceError::IdempotencyConflict, + QueueMutationError::StateNotEditable + | QueueMutationError::InvalidQuotaPool + | QueueMutationError::NotInTerminalState + | QueueMutationError::MutationSkipped + | QueueMutationError::Other(_) => ServiceError::InvalidState, + } +} + +fn remove_inactive_recording(candidate: &mut PersistedRecordingQueue, uuid: &str) -> Option { + if let Some(index) = candidate.scheduled.iter().position(|task| task.uuid == uuid) { + return Some(candidate.scheduled.remove(index)); + } + let index = candidate.queue.iter().position(|task| task.uuid == uuid)?; + Some(candidate.queue.remove(index)) +} + +/// Which pending list a cancelled rule recording came from, so the +/// compensating restore puts it back where it belongs. +#[derive(Debug, Clone, Copy)] +pub(super) enum CancelOrigin { + Scheduled, + Queue, +} + +pub(super) fn cancel_future_rule_recordings_in_candidate( + candidate: &mut PersistedRecordingQueue, + rule_id: &str, + now_secs: i64, +) -> Vec { + let mut undo = Vec::new(); + let mut moved = Vec::new(); + drain_future_rule_recordings( + &mut candidate.scheduled, + CancelOrigin::Scheduled, + rule_id, + now_secs, + &mut undo, + &mut moved, + ); + drain_future_rule_recordings(&mut candidate.queue, CancelOrigin::Queue, rule_id, now_secs, &mut undo, &mut moved); + candidate.finished.extend(moved); + undo +} + +fn drain_future_rule_recordings( + tasks: &mut Vec, + origin: CancelOrigin, + rule_id: &str, + now_secs: i64, + undo: &mut Vec, + out: &mut Vec, +) { + let mut index = 0; + while index < tasks.len() { + if is_future_rule_recording(&tasks[index], rule_id, now_secs) { + let mut task = tasks.remove(index); + // Snapshot before the cancel mutates it: the undo has to + // restore `reserved_bytes`, which is zeroed just below. + undo.push(CancelledRuleRecording { origin, task: task.clone() }); + task.state = RecordingTaskState::Cancelled; + task.finished = true; + task.error = Some("cancelled".to_string()); + task.recording.reserved_bytes = 0; + out.push(task); + } else { + index += 1; + } + } +} + +pub(super) fn is_future_rule_recording(task: &PersistedRecordingTask, rule_id: &str, now_secs: i64) -> bool { + if task.kind != RecordingKind::Live { + return false; + } + let meta = &task.recording; + let Some(start_at) = meta.scheduled_start else { + return false; + }; + if start_at <= now_secs { + return false; + } + if meta.provenance.rule_id.as_deref() != Some(rule_id) { + return false; + } + recording_edit::state_is_editable(task.state) +} diff --git a/backend/dvr/src/recording/recording_service/create.rs b/backend/dvr/src/recording/recording_service/create.rs new file mode 100644 index 000000000..a2bb32016 --- /dev/null +++ b/backend/dvr/src/recording/recording_service/create.rs @@ -0,0 +1,371 @@ +use super::{ + conflicts::{candidate_tasks, used_bytes_for_pool}, + control::{authorize_create_recording, map_queue_error}, + deletion::known_media_size, + error::map_edit_validation_error, + filenames::{render_live_filename, reserve_recording_relative_path}, + identity::candidate_has_duplicate_recording, + quota_limits_from_config, source_resolution, + window::{effective_recording_window, padding_bounds}, + CreateMediaRecordingInput, CreateRecordingInput, EffectiveRecordingWindow, RecordingCtx, RecordingService, + RecordingSourceInput, RecordingTaskView, ServiceError, +}; +use crate::{ + recording::{ + recording_disk, + recording_queue::{ + mutate, mutate_with_idempotency, IdempotencyOutcome, PersistedIdempotency, PersistedRecordingQueue, + QueueMutationError, RecordingQueue, RecordingTask, RecordingTaskState, + }, + }, + recording_edit, + recording_edit::EditError, + recording_quota, + recording_quota::AdmissionOutcome, +}; +use shared::model::{ + recording::{RecordingMetadata, RecordingOwner, RecordingSource}, + EventSink, RecordingKind, UserId, +}; +use std::sync::Arc; +use tuliprox_core::model::AppConfig; + +/// An `Idempotency-Key` and a digest of the body it arrived with. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IdempotencyRequest { + pub key: String, + pub fingerprint: String, +} + +impl RecordingService { + /// Construct from the queue and app config. + pub fn new(recordings: Arc, app_config: Arc) -> Self { Self { recordings, app_config } } + + /// Convenience constructor from the DVR's context. + pub fn from_ctx(ctx: &RecordingCtx) -> Self { + Self::new(ctx.recordings.clone(), ctx.app_config.clone()) + } + + pub(super) fn subject_id(claims: &shared::model::Claims) -> Result { + claims.subject_id.clone().ok_or(ServiceError::UnknownOwner) + } + + pub(super) fn recording_url(&self, source: &RecordingSourceInput) -> Option { + let virtual_id = source.virtual_id.parse::().ok()?; + source_resolution::resolve_recording_target(&self.app_config, &source.target_id, &source.input_name)?; + source_resolution::build_recording_source_descriptor( + &source.target_id, + &source.input_name, + virtual_id, + source.cluster, + ) + } + + /// Create a new recording. Enforces well-formedness, source + /// validity, owner from claims, and the authorization matrix. + #[allow(clippy::too_many_lines)] + pub async fn create_recording( + &self, + claims: &shared::model::Claims, + input: &CreateRecordingInput, + ) -> Result { + self.create_recording_idempotent(claims, input, None).await + } + + /// Builds the task a live request describes, and the space it reserves. + /// + /// Split out of `create_recording_idempotent` purely for length: this is + /// the request-to-task translation, with no admission decisions in it. + pub(super) fn build_live_recording( + owner_id: &UserId, + owner_display: &str, + input: &CreateRecordingInput, + group: Option, + recording_cfg: &tuliprox_core::model::RecordingConfig, + window: &EffectiveRecordingWindow, + url: &str, + ) -> Result<(RecordingTask, u64), ServiceError> { + let duration_secs = window.remaining_duration_secs; + let filename = render_live_filename(input, recording_cfg, window, owner_display); + let input_name: Option> = + (!input.source.input_name.trim().is_empty()).then(|| Arc::from(input.source.input_name.as_str())); + let source = RecordingSource::new( + input.source.target_id.clone(), + input.source.virtual_id.clone(), + input.source.input_name.clone(), + ) + .with_cluster(input.source.cluster); + let mut meta = RecordingMetadata::new_live( + RecordingOwner::User(owner_id.clone()), + input.visibility, + source, + input.program_start, + input.program_end, + input.pre_roll_secs, + input.post_roll_secs, + ); + meta.scheduled_start = Some(window.scheduled_start); + meta.scheduled_end = Some(window.scheduled_end); + meta.channel_id.clone_from(&input.channel_id); + meta.channel_name.clone_from(&input.channel_name); + meta.program_title = Some(input.program_title.clone()); + meta.group = group; + meta.provenance = input.provenance.clone(); + meta.epg.clone_from(&input.epg); + let (reserved_bytes, _) = + recording_quota::estimate_reservation(duration_secs, 0, recording_cfg.fallback_bytes_per_minute); + meta.reserved_bytes = reserved_bytes; + let recording = RecordingTask::new( + RecordingKind::Live, + url, + &filename, + recording_cfg, + input_name, + recording_cfg.priority, + meta, + ) + .ok_or(ServiceError::InvalidSource)?; + Ok((recording, reserved_bytes)) + } + + /// Playlist group the live recording is filed under. + /// + /// Callers that did not resolve it (the rule scheduler) still get the + /// organised layout the operator asked for. Without organisation the + /// group is irrelevant, so the playlist is not read for it. + pub(super) async fn live_group( + &self, + input: &CreateRecordingInput, + recording_cfg: &tuliprox_core::model::RecordingConfig, + ) -> Option { + if input.group.is_some() || !recording_cfg.organize_into_directories { + return input.group.clone(); + } + source_resolution::resolve_live_group( + &self.app_config, + &input.source.target_id, + &input.source.input_name, + &input.source.virtual_id, + ) + .await + } + + /// `create_recording`, honouring an `Idempotency-Key`. + /// + /// A replay of an accepted request is answered from the stored record + /// rather than run again; the same key with a different body is a + /// conflict, because answering it with the first request's result would + /// hide a caller bug. + pub async fn create_recording_idempotent( + &self, + claims: &shared::model::Claims, + input: &CreateRecordingInput, + idempotency: Option, + ) -> Result { + input.validate()?; + let owner_id = Self::subject_id(claims)?; + self.check_idempotency(&owner_id, idempotency.as_ref()).await?; + let config = self.enabled_config()?; + let Some(recording_cfg) = config.recording() else { + return Err(ServiceError::Disabled); + }; + recording_edit::validate_padding( + input.pre_roll_secs, + input.post_roll_secs, + padding_bounds(Some(recording_cfg)), + ) + .map_err(|error: EditError| map_edit_validation_error(&error))?; + let window = effective_recording_window( + input.program_start, + input.program_end, + input.pre_roll_secs, + input.post_roll_secs, + chrono::Utc::now().timestamp(), + )?; + let url = self.recording_url(&input.source).ok_or(ServiceError::InvalidSource)?; + + // Shared creation requires admin. + authorize_create_recording(claims, &owner_id, input.visibility)?; + + // The layout is fixed when the task is built, so the group has to be + // known first. + let group = self.live_group(input, recording_cfg).await; + let duration_secs = window.remaining_duration_secs; + let (recording, reserved_bytes) = + Self::build_live_recording(&owner_id, &claims.username, input, group, recording_cfg, &window, &url)?; + self.admit(&recording, reserved_bytes, &owner_id, idempotency.as_ref(), recording_cfg).await?; + + Ok(RecordingTaskView { + uuid: recording.uuid, + owner_id, + visibility: input.visibility, + filename_preview: recording.filename, + start_at: Some(window.execution_start), + duration_secs: Some(duration_secs), + state: RecordingTaskState::Scheduled, + }) + } + + /// Queue an immediate VOD or series transfer, honouring an + /// `Idempotency-Key`. + /// + /// Runs through the same admission as a live request: path reservation, + /// per-principal duplicate check, quota and disk. The size of a transfer + /// is unknown until the provider answers, so nothing is reserved up front; + /// the worker re-checks quota and disk before it opens the destination. + pub async fn create_media_recording_idempotent( + &self, + claims: &shared::model::Claims, + input: &CreateMediaRecordingInput, + idempotency: Option, + ) -> Result { + let kind = input.validate()?; + let owner_id = Self::subject_id(claims)?; + self.check_idempotency(&owner_id, idempotency.as_ref()).await?; + let config = self.enabled_config()?; + let Some(recording_cfg) = config.recording() else { + return Err(ServiceError::Disabled); + }; + let url = self.recording_url(&input.source).ok_or(ServiceError::InvalidSource)?; + authorize_create_recording(claims, &owner_id, input.visibility)?; + + let source = RecordingSource::new( + input.source.target_id.clone(), + input.source.virtual_id.clone(), + input.source.input_name.clone(), + ) + .with_cluster(input.source.cluster); + let mut meta = RecordingMetadata::new_media( + RecordingOwner::User(owner_id.clone()), + input.visibility, + source, + input.title.clone(), + ); + meta.group.clone_from(&input.group); + meta.series_name.clone_from(&input.series_name); + let filename = format!("{}.{}", input.title, input.extension.trim_start_matches('.')); + let recording = RecordingTask::new( + kind, + &url, + &filename, + recording_cfg, + Some(Arc::from(input.source.input_name.as_str())), + recording_cfg.priority, + meta, + ) + .ok_or(ServiceError::InvalidPath)?; + self.admit(&recording, 0, &owner_id, idempotency.as_ref(), recording_cfg).await?; + + Ok(RecordingTaskView { + uuid: recording.uuid, + owner_id, + visibility: input.visibility, + filename_preview: recording.filename, + start_at: None, + duration_secs: None, + state: RecordingTaskState::Queued, + }) + } + + /// Answer a replayed or conflicting `Idempotency-Key` before any work: + /// a replay must not resolve sources, reserve a path or touch quota. + pub(super) async fn check_idempotency( + &self, + owner_id: &UserId, + idempotency: Option<&IdempotencyRequest>, + ) -> Result<(), ServiceError> { + let Some(request) = idempotency else { + return Ok(()); + }; + match self + .recordings + .lookup_idempotency(owner_id.0.as_str(), &request.key, &request.fingerprint) + .await + .map_err(|err| ServiceError::IoError(err.to_string()))? + { + IdempotencyOutcome::Fresh => Ok(()), + IdempotencyOutcome::Replay { recording_id } => Err(ServiceError::IdempotentReplay { recording_id }), + IdempotencyOutcome::Conflict => Err(ServiceError::IdempotencyConflict), + } + } + + pub(super) fn enabled_config(&self) -> Result, ServiceError> { + if !crate::recording::recording_supervisor::recording_enabled(&self.app_config) { + return Err(ServiceError::Disabled); + } + Ok(self.app_config.config.load_full()) + } + + /// Admit a freshly built task into the queue: reserve its path, refuse a + /// duplicate from the same principal, then check quota and disk, all in + /// one mutation together with the idempotency record. + /// + /// Scheduled kinds land in `scheduled`, everything else in `queue`. + pub(super) async fn admit( + &self, + recording: &RecordingTask, + reserved_bytes: u64, + owner_id: &UserId, + idempotency: Option<&IdempotencyRequest>, + recording_cfg: &tuliprox_core::model::RecordingConfig, + ) -> Result<(), ServiceError> { + let mut persisted = RecordingQueue::to_persisted(recording); + let quota_limits = quota_limits_from_config(recording_cfg.quota.as_ref()); + // Measured before the mutation: it is a syscall, and the lock is + // held for the whole closure. `None` means the root could not be + // measured, and an unmeasurable disk is not grounds to refuse. + let disk_safety_bytes = recording_cfg.disk.as_ref().and_then(|disk| disk.safety_bytes).unwrap_or(0); + let free_bytes = recording_disk::free_bytes_for(std::path::Path::new(&recording_cfg.directory)); + let idempotency_record = idempotency.map(|request| PersistedIdempotency { + principal: owner_id.0.clone(), + key: request.key.clone(), + request_fingerprint: request.fingerprint.clone(), + recording_id: recording.uuid.clone(), + accepted_at: chrono::Utc::now().timestamp(), + }); + + let admit = |candidate: &mut PersistedRecordingQueue| -> Result<(), QueueMutationError> { + reserve_recording_relative_path(candidate, &mut persisted)?; + if candidate_has_duplicate_recording(candidate, recording) { + return Err(QueueMutationError::Duplicate); + } + // Media another entry finished, or is fetching with a known size, + // is charged at that size now: attaching copies the whole file + // into this entry's charge, so admitting it at zero would let it + // past the quota. + let charge = reserved_bytes.max(known_media_size(candidate, &persisted)); + persisted.recording.reserved_bytes = charge; + let pool = recording_quota::quota_pool_for_task(&persisted); + let used = used_bytes_for_pool(candidate, &pool); + if matches!( + recording_quota::would_exceed(&pool, used, charge, "a_limits), + AdmissionOutcome::OverLimit { .. } + ) { + return Err(QueueMutationError::QuotaExceeded); + } + // Quota is per-owner and logical; this is the physical + // question, and one can pass while the other fails. + if let Some(free_bytes) = free_bytes { + let active = recording_disk::active_disk_reservations(candidate_tasks(candidate)); + if matches!( + recording_disk::would_fit_on_disk(free_bytes, disk_safety_bytes, active, reserved_bytes), + recording_disk::DiskAdmission::Insufficient { .. } + ) { + return Err(QueueMutationError::DiskFull); + } + } + if recording.kind.is_scheduled() { + candidate.scheduled.push(persisted.clone()); + } else { + candidate.queue.push(persisted.clone()); + } + Ok(()) + }; + + match idempotency_record { + Some(record) => mutate_with_idempotency(&self.recordings, record, admit).await, + None => mutate(&self.recordings, admit).await, + } + .map_err(|e| map_queue_error(&e)) + } +} diff --git a/backend/dvr/src/recording/recording_service/deletion.rs b/backend/dvr/src/recording/recording_service/deletion.rs new file mode 100644 index 000000000..ee39f703a --- /dev/null +++ b/backend/dvr/src/recording/recording_service/deletion.rs @@ -0,0 +1,108 @@ +use super::{conflicts::candidate_tasks, error::map_deletion_error, RecordingService, ServiceError}; +use crate::{ + recording::recording_queue::{mutate, PersistedRecordingQueue, PersistedRecordingTask, RecordingTaskState}, + recording_deletion::{begin_deletion_authorized, execute_deletion_target, finalize_deletion, rollback_deletion}, +}; +use shared::model::recording::RecordingMetadata; +use tuliprox_auth::{authorize, authorize_orphan, RecordingAction, RecordingDecision, RecordingSubject, TerminalState}; + +impl RecordingService { + /// Delete a finished recording via the three-step service. + /// Marks the task as `Deleting` (atomic), unlinks the file + /// (outside the boundary), then removes the task (atomic). + pub async fn delete_recording(&self, claims: &shared::model::Claims, uuid: &str) -> Result<(), ServiceError> { + let owner_id = Self::subject_id(claims)?; + self.run_deletion(uuid, |meta| { + let subject = RecordingSubject::new(Some(meta), TerminalState::Completed, true); + matches!(authorize(claims, &owner_id, RecordingAction::Delete, &subject), RecordingDecision::Allow) + }) + .await + .map(|_| ()) + } + + /// The three-phase deletion, shared by the user-facing delete and the + /// retention worker. `permit` runs *inside* the same mutation + /// boundary that stamps the task as deleting, so there is no window + /// in which the authorized metadata and the stamped task can differ: + /// the previous implementation looked the task up, authorized it, + /// stamped it, then looked it up a second time and could act on a + /// stale copy. + /// Returns `true` when the file was actually unlinked. `false` means the + /// entry is gone but another library entry still holds its bytes, so the + /// caller must not count the space as reclaimed. + pub(super) async fn run_deletion(&self, uuid: &str, permit: F) -> Result + where + F: FnOnce(&RecordingMetadata) -> bool, + { + let queue = self.recordings.clone(); + let target = begin_deletion_authorized(&queue, uuid, permit).await.map_err(map_deletion_error)?; + if let Err(err) = execute_deletion_target(&target).await { + // File removal failed: undo the deletion transition so the + // recording stays visible in its prior state instead of + // being silently lost when finalize_deletion runs. + let uuid_owned = uuid.to_string(); + let _ = mutate(&self.recordings, |candidate| { + rollback_deletion(candidate, &uuid_owned); + Ok(()) + }) + .await; + return Err(ServiceError::IoError(err.to_string())); + } + finalize_deletion(&queue, uuid).await.map_err(|_| ServiceError::UnknownRecording)?; + Ok(!target.still_referenced) + } + + /// Internal retention-delete entrypoint used by the retention + /// worker. Bypasses user ownership but enforces state/kind/path + pub async fn system_retention_delete( + &self, + claims: &shared::model::Claims, + uuid: &str, + ) -> Result { + let owner_id = Self::subject_id(claims)?; + self.run_deletion(uuid, |meta| { + let subject = RecordingSubject::new(Some(meta), TerminalState::Completed, true); + matches!( + authorize(claims, &owner_id, RecordingAction::SystemRetentionDelete, &subject,), + RecordingDecision::Allow + ) + }) + .await + } + + /// Re-export the orphan policy for callers that need it. + pub fn authorize_orphan_read(&self, claims: &shared::model::Claims) -> Result<(), ServiceError> { + match authorize_orphan(claims) { + RecordingDecision::Allow => Ok(()), + RecordingDecision::Deny(_) => Err(ServiceError::Forbidden), + } + } +} + +/// The size of the media `task` refers to, as far as another entry already +/// knows it: the measured size of a completed file, or the total a transfer +/// learned from its provider. `0` when nobody knows yet. +pub(super) fn known_media_size(candidate: &PersistedRecordingQueue, task: &PersistedRecordingTask) -> u64 { + if task.media_identity.is_empty() { + return 0; + } + candidate_tasks(candidate) + .filter(|other| other.media_identity == task.media_identity) + .filter_map(|other| { + if other.state == RecordingTaskState::Completed { + Some(other.recording.measured_bytes.max(other.size)) + } else { + other.total_size + } + }) + .max() + .unwrap_or(0) +} + +pub(super) fn collect_existing_relative_paths(candidate: &PersistedRecordingQueue) -> impl Iterator + '_ { + candidate_tasks(candidate).map(task_relative_path) +} + +fn task_relative_path(task: &PersistedRecordingTask) -> &str { + task.recording.relative_path.as_deref().unwrap_or(task.filename.as_str()) +} diff --git a/backend/dvr/src/recording/recording_service/edit.rs b/backend/dvr/src/recording/recording_service/edit.rs new file mode 100644 index 000000000..e19ffb02a --- /dev/null +++ b/backend/dvr/src/recording/recording_service/edit.rs @@ -0,0 +1,211 @@ +use super::{ + conflicts::used_bytes_for_pool, + control::map_queue_error, + filenames::reserve_recording_relative_path, + identity::{locate_recording, recording_mut_at, RecordingLocation}, + quota_limits_from_config, recording_identity_key, + window::{effective_recording_window, padding_bounds}, + EditRecordingPatch, EditSnapshot, RecordingService, RecordingTaskView, ServiceError, +}; +use crate::{ + recording::recording_queue::{mutate, QueueMutationError}, + recording_edit, + recording_edit::EditError, + recording_quota, + recording_quota::AdmissionOutcome, +}; +use shared::model::RecordingKind; +use tuliprox_auth::{authorize, RecordingAction, RecordingDecision, RecordingSubject, TerminalState}; + +impl RecordingService { + /// Edit an existing recording. + #[allow(clippy::too_many_lines)] + pub async fn edit_recording( + &self, + claims: &shared::model::Claims, + uuid: &str, + patch: EditRecordingPatch, + ) -> Result { + let owner_id = Self::subject_id(claims)?; + let config = self.app_config.config.load(); + // Edit does not re-resolve a URL. The recording config is + // only needed for padding bounds, the unknown-bitrate fallback, + // and quota limits. When the recording block is absent the + // helper falls back to the shared-model defaults, so a + // configured-without-recording deployment still validates + // edits. + let recording_cfg = config.recording(); + let bounds = padding_bounds(recording_cfg); + let fallback_bytes_per_minute = recording_cfg.map_or(8 * 1024 * 1024, |cfg| cfg.fallback_bytes_per_minute); + let quota_limits = quota_limits_from_config(recording_cfg.and_then(|cfg| cfg.quota.as_ref())); + let mut out = None; + mutate(&self.recordings, |candidate| { + // Single linear scan: locate the recording, snapshot the + // primitives we need for the immutable analysis, drop the + // borrow, run the checks, then re-acquire the same task + // via the remembered location for the O(1) write phase. + let location = locate_recording(candidate, uuid).ok_or(QueueMutationError::UnknownRecording)?; + let snapshot = { + // `Active` and `Finished` are not in `scheduled` / + // `queue`, but `locate_recording` returns them anyway — + // short-circuit with `StateNotEditable` so the match + // arms below stay narrowed to the editable lists. + let task = match location { + RecordingLocation::Scheduled(i) => &candidate.scheduled[i], + RecordingLocation::Queue(i) => &candidate.queue[i], + RecordingLocation::Active(_) | RecordingLocation::Finished(_) => { + return Err(QueueMutationError::StateNotEditable); + } + }; + if !recording_edit::state_is_editable(task.state) { + return Err(QueueMutationError::StateNotEditable); + } + let meta_snapshot = &task.recording; + let pool = recording_quota::quota_pool_for_task(task); + let subject = RecordingSubject::new(Some(meta_snapshot), TerminalState::Active, true); + if !matches!(authorize(claims, &owner_id, RecordingAction::Edit, &subject), RecordingDecision::Allow) { + return Err(QueueMutationError::Forbidden); + } + let merged_pre = patch.pre_roll_secs.unwrap_or(meta_snapshot.pre_roll_secs); + let merged_post = patch.post_roll_secs.unwrap_or(meta_snapshot.post_roll_secs); + recording_edit::validate_padding(merged_pre, merged_post, bounds).map_err(|err| match err { + EditError::PaddingLimitExceeded => QueueMutationError::PaddingLimitExceeded, + EditError::InvalidInterval => QueueMutationError::InvalidInterval, + EditError::StateNotEditable | EditError::ChannelChangedWithoutProgramme => { + QueueMutationError::StateNotEditable + } + EditError::ProvenanceCleared => QueueMutationError::Forbidden, + })?; + let channel_changed_now = recording_edit::channel_changed( + patch.channel_id.as_deref(), + patch.channel_name.as_deref(), + meta_snapshot.channel_id.as_deref(), + meta_snapshot.channel_name.as_deref(), + ); + EditSnapshot { + pool, + merged_pre, + merged_post, + channel_changed_now, + current_start: meta_snapshot.program_start, + current_end: meta_snapshot.program_end, + current_reserved: meta_snapshot.reserved_bytes, + is_live: task.kind == RecordingKind::Live, + shares_media: crate::recording::recording_queue::media_is_still_referenced(candidate, uuid), + } + }; + // A programme window and its padding exist for live captures + // only; the create path refuses them for VOD and series too. + let edits_window = patch.program_start.is_some() + || patch.program_end.is_some() + || patch.pre_roll_secs.is_some() + || patch.post_roll_secs.is_some(); + if !snapshot.is_live && edits_window { + return Err(QueueMutationError::InvalidInterval); + } + // A changed window is derived the way the create path derives + // it: padded, and reserved for what is left of it. An edit that + // leaves the window alone leaves its schedule and reservation + // alone too. + let window = if edits_window { + let start = + patch.program_start.or(snapshot.current_start).ok_or(QueueMutationError::InvalidInterval)?; + let end = patch.program_end.or(snapshot.current_end).ok_or(QueueMutationError::InvalidInterval)?; + // Same rule as a new request: the interval itself must be + // representable, not only its padded ends. + if end.checked_sub(start).is_none_or(|duration| duration <= 0) { + return Err(QueueMutationError::InvalidInterval); + } + let window = effective_recording_window( + start, + end, + snapshot.merged_pre, + snapshot.merged_post, + chrono::Utc::now().timestamp(), + ) + .map_err(|error| match error { + ServiceError::PaddingLimitExceeded => QueueMutationError::PaddingLimitExceeded, + _ => QueueMutationError::InvalidInterval, + })?; + let (reserved, _) = + recording_quota::estimate_reservation(window.remaining_duration_secs, 0, fallback_bytes_per_minute); + let pool_used = used_bytes_for_pool(candidate, &snapshot.pool); + let pool_used_minus_this = pool_used.saturating_sub(snapshot.current_reserved); + if matches!( + recording_quota::would_exceed(&snapshot.pool, pool_used_minus_this, reserved, "a_limits), + AdmissionOutcome::OverLimit { .. } + ) { + return Err(QueueMutationError::QuotaExceeded); + } + Some((start, end, window, reserved)) + } else { + None + }; + + // All immutable borrows are out of scope. Re-acquire the + // same task via the remembered location (O(1)) for the + // actual edit and apply every field write here. If any + // earlier step returned `Err`, none of these writes run, + // so the candidate is rolled back atomically. + let Some(task) = recording_mut_at(candidate, location) else { + return Err(QueueMutationError::UnknownRecording); + }; + let meta = &mut task.recording; + if let Some(title) = patch.program_title { + meta.program_title = Some(title); + } + if let Some(channel_id) = patch.channel_id { + meta.channel_id = Some(channel_id); + } + if let Some(channel_name) = patch.channel_name { + meta.channel_name = Some(channel_name); + } + if let Some((start, end, window, reserved)) = window { + meta.pre_roll_secs = snapshot.merged_pre; + meta.post_roll_secs = snapshot.merged_post; + meta.program_start = Some(start); + meta.program_end = Some(end); + meta.scheduled_start = Some(window.scheduled_start); + meta.scheduled_end = Some(window.scheduled_end); + meta.reserved_bytes = reserved; + } + if snapshot.channel_changed_now { + meta.epg = None; + } + + // The window is part of what makes two requests the same media. + // An edit that changes it leaves the media this entry shared, + // so it must not keep the shared file's path either: two + // captures would write one file, and deleting either would + // take the other's. + let identity = recording_identity_key(&task.recording, &task.url); + let leaves_shared_media = identity != task.media_identity && snapshot.shares_media; + task.media_identity = identity; + if leaves_shared_media { + let mut detached = task.clone(); + reserve_recording_relative_path(candidate, &mut detached)?; + let Some(task) = recording_mut_at(candidate, location) else { + return Err(QueueMutationError::UnknownRecording); + }; + *task = detached; + } + + let Some(task) = recording_mut_at(candidate, location) else { + return Err(QueueMutationError::UnknownRecording); + }; + out = Some(RecordingTaskView { + uuid: task.uuid.clone(), + owner_id: owner_id.clone(), + visibility: task.recording.visibility, + filename_preview: task.filename.clone(), + start_at: task.recording.scheduled_start, + duration_secs: window.map(|(_, _, window, _)| window.remaining_duration_secs), + state: task.state, + }); + Ok(()) + }) + .await + .map_err(|e| map_queue_error(&e))?; + out.ok_or(ServiceError::UnknownRecording) + } +} diff --git a/backend/dvr/src/recording/recording_service/error.rs b/backend/dvr/src/recording/recording_service/error.rs new file mode 100644 index 000000000..d40e7ce59 --- /dev/null +++ b/backend/dvr/src/recording/recording_service/error.rs @@ -0,0 +1,115 @@ +use crate::{recording_deletion::DeletionError, recording_edit::EditError}; + +/// Stable service-layer errors. The HTTP layer maps each variant to a +/// stable status; the frontend maps each variant to a localized +/// message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ServiceError { + /// The recording owner could not be resolved from the authenticated + /// claims (`subject_id` missing or invalid). + UnknownOwner, + /// The supplied `RecordingSource` does not match a configured + /// target/input combination. + InvalidSource, + /// Caller asked for an action the principal cannot perform. + Forbidden, + /// Caller asked for shared creation but is not an administrator. + SharedCreationNotAdministrator, + /// Caller asked to act on a recording that is in an ineligible + /// state (e.g., edit on a recording marked `Deleting`). + InvalidState, + /// `program_end - program_start` overflows or is non-positive. + InvalidInterval, + /// Requested padding exceeds the configured recording maximum. + PaddingLimitExceeded, + /// uuid not in the queue. + UnknownRecording, + /// `mutate`'s persist step failed and the in-memory state was + /// kept unchanged. + PersistenceFailed, + /// IO error during physical deletion. + IoError(String), + /// Configured recording quota would be exceeded. + QuotaExceeded, + /// The patch would clear `rule_id` / `occurrence_key`. Both are + /// immutable provenance; surfacing this as `InvalidState` hid the + /// real reason from the client. + ProvenanceImmutable, + /// Caller tried to create a recording that already exists in the + /// queue (same target / window). Distinct from `InvalidState` so + /// the client can render a specific "duplicate" message. + Duplicate, + /// The recording's filesystem path is not within the configured + /// storage root, or otherwise violates the path policy. + InvalidPath, + /// The recording cannot fit on disk; reservation would exceed + /// available space. + DiskFull, + /// The server has no download engine: the `video.recording` block + /// is missing from the configuration. Distinct from + /// `InvalidSource` because the caller's identifiers may be + /// perfectly valid — nothing on the server can execute them. + Disabled, + /// This exact request was already accepted under this idempotency key. + /// Not a failure: the caller gets the original outcome. + IdempotentReplay { recording_id: String }, + /// Same idempotency key, different request body. + IdempotencyConflict, +} + +impl std::fmt::Display for ServiceError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.code()) } +} + +impl std::error::Error for ServiceError {} + +impl ServiceError { + /// Stable wire-level code. + pub fn code(&self) -> &'static str { + match self { + Self::UnknownOwner => "recording_unknown_owner", + Self::InvalidSource => "recording_invalid_source", + Self::Forbidden => "recording_forbidden", + Self::SharedCreationNotAdministrator => "recording_shared_not_administrator", + Self::InvalidState => "recording_invalid_state", + Self::InvalidInterval => "recording_invalid_interval", + Self::PaddingLimitExceeded => "recording_padding_limit_exceeded", + Self::UnknownRecording => "recording_unknown", + Self::PersistenceFailed => "recording_persistence_failed", + Self::IoError(_) => "recording_io_error", + Self::QuotaExceeded => "recording_quota_exceeded", + Self::ProvenanceImmutable => "recording_provenance_immutable", + Self::Duplicate => "recording_duplicate", + Self::InvalidPath => "recording_invalid_path", + Self::DiskFull => "recording_disk_full", + Self::Disabled => "recording_disabled", + Self::IdempotentReplay { .. } => "recording_idempotent_replay", + Self::IdempotencyConflict => "recording_idempotency_conflict", + } + } +} + +pub(super) fn map_edit_validation_error(error: &EditError) -> ServiceError { + match error { + EditError::InvalidInterval => ServiceError::InvalidInterval, + EditError::PaddingLimitExceeded => ServiceError::PaddingLimitExceeded, + EditError::ProvenanceCleared => ServiceError::ProvenanceImmutable, + EditError::StateNotEditable | EditError::ChannelChangedWithoutProgramme => ServiceError::InvalidState, + } +} + +pub(super) fn map_deletion_error(error: DeletionError) -> ServiceError { + match error { + DeletionError::Forbidden => ServiceError::Forbidden, + DeletionError::NotTerminal => ServiceError::InvalidState, + DeletionError::UnknownTask | DeletionError::NotARecording => ServiceError::UnknownRecording, + DeletionError::DeleteFailed(err) => ServiceError::IoError(err.to_string()), + DeletionError::BeginFailed(err) | DeletionError::FinalizeFailed(err) => { + if err.source_io().is_some() { + ServiceError::PersistenceFailed + } else { + ServiceError::UnknownRecording + } + } + } +} diff --git a/backend/dvr/src/recording/recording_service/filenames.rs b/backend/dvr/src/recording/recording_service/filenames.rs new file mode 100644 index 000000000..d502c953b --- /dev/null +++ b/backend/dvr/src/recording/recording_service/filenames.rs @@ -0,0 +1,202 @@ +use super::{deletion::collect_existing_relative_paths, CreateRecordingInput, EffectiveRecordingWindow}; +use crate::recording::{ + recording_path, + recording_queue::{PersistedRecordingQueue, PersistedRecordingTask, QueueMutationError}, +}; +use std::path::{Path, PathBuf}; + +/// Maximum bytes in a single sanitized filename component. Well under +/// the 255-byte limit every supported filesystem enforces, leaving room +/// for the `_N` disambiguation suffix and the `.partial` extension the +/// worker appends. +pub(super) const MAX_FILENAME_COMPONENT_BYTES: usize = 200; + +/// Substitute for a title that sanitizes down to nothing. +const FILENAME_FALLBACK: &str = "recording"; + +/// Characters that are illegal in a path component on at least one +/// supported platform. `/` and `\` are separators where it matters; the +/// rest are Windows-reserved but are equally unwelcome in a +/// URL-addressed media path. +const FILENAME_FORBIDDEN_CHARS: &[char] = &['<', '>', ':', '"', '/', '\\', '|', '?', '*']; + +/// Windows reserved device names. A component whose stem matches one of +/// these (case-insensitively) cannot be created on Windows, with or +/// without an extension. +const WINDOWS_RESERVED_STEMS: &[&str] = &[ + "con", "prn", "aux", "nul", "com1", "com2", "com3", "com4", "com5", "com6", "com7", "com8", "com9", "lpt1", "lpt2", + "lpt3", "lpt4", "lpt5", "lpt6", "lpt7", "lpt8", "lpt9", +]; + +/// Turn arbitrary programme text into one safe path component. +/// +/// The previous implementation replaced only the two path separators, +/// which let control characters, Windows-reserved characters, trailing +/// dots/spaces, and `BiDi` override codepoints through into the path the +/// muxer opens and the media API re-validates. This is the single +/// chokepoint: everything that lands in `filename` goes through here. +/// +/// Guarantees on the returned string: +/// - exactly one path component (no separator survives), +/// - no ASCII control characters and no Unicode `BiDi` / invisible +/// formatting codepoints, +/// - no leading or trailing whitespace or `.`, +/// - never empty, never `.` or `..`, never a Windows device name, +/// - at most `MAX_FILENAME_COMPONENT_BYTES` bytes, truncated on a +/// character boundary, +/// - idempotent: sanitizing an already-sanitized value is a no-op. +pub fn sanitize_filename_component(raw: &str) -> String { + let mut out = String::with_capacity(raw.len()); + let mut last_was_underscore = false; + for ch in raw.chars() { + // Invisible formatting codepoints can reorder the rendered + // filename so it does not match the bytes on disk. Drop them + // outright rather than substituting, so they leave no trace. + if is_invisible_formatting(ch) { + continue; + } + if ch.is_control() || FILENAME_FORBIDDEN_CHARS.contains(&ch) { + // Collapse runs so `a///b` becomes `a_b`, not `a___b`. + if !last_was_underscore { + out.push('_'); + last_was_underscore = true; + } + continue; + } + out.push(ch); + last_was_underscore = ch == '_'; + } + + // Trailing dots and spaces are silently stripped by Windows, which + // would desync the persisted `relative_path` from the real file. + let trimmed = out.trim_matches(|ch: char| ch.is_whitespace() || ch == '.'); + let mut result = truncate_on_char_boundary(trimmed, MAX_FILENAME_COMPONENT_BYTES) + .trim_end_matches(|ch: char| ch.is_whitespace() || ch == '.') + .to_string(); + + if result.is_empty() || is_windows_reserved_stem(&result) { + result = FILENAME_FALLBACK.to_string(); + } + result +} + +/// `BiDi` controls, zero-width characters, and the other invisible +/// formatting codepoints that make a filename render differently from +/// what it actually contains. +fn is_invisible_formatting(ch: char) -> bool { + matches!( + ch, + '\u{200b}'..='\u{200f}' // zero-width space .. RLM + | '\u{202a}'..='\u{202e}' // embedding / override + | '\u{2060}'..='\u{2064}' // word joiner, invisible operators + | '\u{2066}'..='\u{2069}' // directional isolates + | '\u{feff}' // BOM / zero-width no-break space + ) +} + +/// Truncate to at most `max_bytes`, never splitting a character. +fn truncate_on_char_boundary(value: &str, max_bytes: usize) -> &str { + if value.len() <= max_bytes { + return value; + } + let mut end = max_bytes; + while end > 0 && !value.is_char_boundary(end) { + end -= 1; + } + &value[..end] +} + +/// `true` when the component's stem is a Windows device name. +fn is_windows_reserved_stem(value: &str) -> bool { + let stem = value.split('.').next().unwrap_or(value); + WINDOWS_RESERVED_STEMS.iter().any(|reserved| stem.eq_ignore_ascii_case(reserved)) +} + +/// Live recording filename: `recording.filename_template` rendered for this +/// programme, plus the extension of the container the worker muxes into, so +/// players and the media API can tell the format from the name. +/// +/// The sanitized programme title stands in when the template is unset or +/// cannot be rendered. +pub(super) fn render_live_filename( + input: &CreateRecordingInput, + recording_cfg: &tuliprox_core::model::RecordingConfig, + window: &EffectiveRecordingWindow, + owner_display: &str, +) -> String { + let title_stem = sanitize_filename_component(&input.program_title); + let context = shared::utils::RecordingFilenameContext { + task_id: title_stem.clone(), + channel_id: input.channel_id.clone(), + channel_name: input.channel_name.clone(), + program_title: Some(input.program_title.clone()), + episode_season: input.epg.as_ref().and_then(|epg| epg.season), + episode_number: input.epg.as_ref().and_then(|epg| epg.episode), + owner_display: (!owner_display.trim().is_empty()).then(|| owner_display.to_string()), + program_start: Some(input.program_start), + program_end: Some(input.program_end), + scheduled_start: Some(window.scheduled_start), + scheduled_end: Some(window.scheduled_end), + }; + let stem = Some(recording_cfg.filename_template.as_str()) + .filter(|template| !template.trim().is_empty()) + .and_then(|template| { + shared::utils::render_recording_stem(template, &context, &recording_cfg.timezone) + .map_err(|err| log::warn!("Recording filename template could not be rendered: {err}")) + .ok() + }) + .unwrap_or(title_stem); + let extension = recording_cfg.container_format.file_extension(); + // A template that already ends in the container extension must not double it. + let stem_path = Path::new(&stem); + let stem = match (stem_path.file_stem(), stem_path.extension()) { + (Some(base), Some(ext)) if ext.eq_ignore_ascii_case(extension) => base.to_string_lossy(), + _ => std::borrow::Cow::Borrowed(stem.as_str()), + }; + format!("{stem}.{extension}") +} + +pub(super) fn reserve_recording_relative_path( + candidate: &PersistedRecordingQueue, + task: &mut PersistedRecordingTask, +) -> Result<(), QueueMutationError> { + // Borrowed set, built once. The old code walked a `Vec` of + // cloned filenames once per `_N` candidate, so reserving the + // (N+1)-th recording of a title cost O(N^2) string comparisons. + let existing: std::collections::HashSet<&str> = collect_existing_relative_paths(candidate).collect(); + // The reservation is over the whole root-relative path, not the bare + // filename: two series can legitimately hold an `e01.mkv` in different + // season directories. + let base = task.recording.relative_path.clone().unwrap_or_else(|| task.filename.clone()); + let mut relative = PathBuf::from(&base); + if existing.contains(base.as_str()) { + // Linear probe over indices; each probe is one hash lookup. + for index in 1.. { + relative = recording_path::with_collision_suffix(Path::new(&base), index); + if !existing.contains(relative.to_string_lossy().as_ref()) { + break; + } + } + } + let filename = + relative.file_name().and_then(|name| name.to_str()).ok_or(QueueMutationError::InvalidPath)?.to_string(); + validate_reserved_filename(&filename).map_err(|_| QueueMutationError::InvalidPath)?; + if !recording_path::is_contained_relative_path(&relative) { + return Err(QueueMutationError::InvalidPath); + } + task.file_path = task.file_dir.join(&filename); + task.filename = filename; + task.recording.relative_path = Some(relative.to_string_lossy().into_owned()); + Ok(()) +} + +pub(super) fn validate_reserved_filename(filename: &str) -> Result<(), &'static str> { + use std::path::Component; + let path = Path::new(filename); + let single_normal_component = + path.components().next().is_some_and(|c| matches!(c, Component::Normal(_))) && path.components().count() == 1; + if filename.is_empty() || path.is_absolute() || !single_normal_component || filename.as_bytes().contains(&0) { + return Err("recording invalid path"); + } + Ok(()) +} diff --git a/backend/dvr/src/recording/recording_service/identity.rs b/backend/dvr/src/recording/recording_service/identity.rs new file mode 100644 index 000000000..d8663fe01 --- /dev/null +++ b/backend/dvr/src/recording/recording_service/identity.rs @@ -0,0 +1,152 @@ +use crate::{ + recording::recording_queue::{PersistedRecordingQueue, PersistedRecordingTask, RecordingTask}, + recording_quota, +}; +use shared::model::recording::RecordingMetadata; + +/// What makes two recording requests "the same thing". +/// +/// The previous key was `(url, start_at, duration_secs)` OR `file_path`, +/// and neither half worked: +/// - `file_path` is disambiguated with a `_N` suffix by +/// `reserve_recording_relative_path`, so it *never* matches an +/// existing task and the whole disjunct was dead. +/// - `start_at` is `now.max(scheduled_start)`, so for a +/// currently-airing programme every request inside the window +/// produces a different value and the same programme could be +/// booked over and over. +/// +/// The identity is now derived from what the user actually asked for. +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize)] +pub enum RecordingIdentity { + /// Materialization of one rule occurrence. Two tasks with the same + /// `(rule_id, occurrence_key)` are the same recording by + /// definition, whatever their window looks like. + Occurrence { rule_id: S, occurrence_key: S }, + /// A concrete programme on a concrete source. Deliberately free of any + /// owner: two users asking for the same programme are asking for the same + /// file, and it is recorded once and linked twice. + Programme { target_id: S, virtual_id: S, program_start: i64, program_end: i64 }, + /// No programme metadata at all: every VOD and series transfer, and a + /// Live capture whose programme window is unknown. Falls back to the + /// resolved URL plus the *scheduled* (padded) window, which — unlike + /// `start_at` — is stable across requests inside a currently-airing + /// window. + /// + /// Owner-free for the same reason `Programme` is: the same URL over the + /// same window is the same bytes, whoever asked for them. + Url { url: S, scheduled_start: Option, scheduled_end: Option }, +} + +/// A stable, field-named key for the media a request refers to. +/// +/// Two tasks with the same key are the same recording, so they share one +/// physical file. The key is persisted, so it has to be stable across builds: +/// that is why it is serialised rather than formatted with `Debug`. +pub fn recording_identity_key(meta: &RecordingMetadata, url: &str) -> String { + serde_json::to_string(&recording_identity(meta, url)).unwrap_or_else(|_| format!("url:{url}")) +} + +pub(crate) fn recording_identity<'a>(meta: &'a RecordingMetadata, url: &'a str) -> RecordingIdentity<&'a str> { + if let (Some(rule_id), Some(occurrence_key)) = + (meta.provenance.rule_id.as_deref(), meta.provenance.occurrence_key.as_deref()) + { + return RecordingIdentity::Occurrence { rule_id, occurrence_key }; + } + if let (Some(program_start), Some(program_end)) = (meta.program_start, meta.program_end) { + return RecordingIdentity::Programme { + target_id: &meta.source.target_id, + virtual_id: &meta.source.virtual_id, + program_start, + program_end, + }; + } + RecordingIdentity::Url { url, scheduled_start: meta.scheduled_start, scheduled_end: meta.scheduled_end } +} + +fn persisted_recording_identity(task: &PersistedRecordingTask) -> RecordingIdentity<&str> { + recording_identity(&task.recording, &task.url) +} + +pub(super) fn candidate_has_duplicate_recording(candidate: &PersistedRecordingQueue, task: &RecordingTask) -> bool { + let meta = &task.recording; + let identity = recording_identity(meta, task.url.as_str()); + let pool = recording_quota::quota_pool_for_task(task); + // Only the *same* principal asking twice is a duplicate. A different + // principal asking for the same media gets their own library entry, which + // attaches to the one physical file rather than producing a second. + let pending_match = candidate + .queue + .iter() + .chain(candidate.scheduled.iter()) + .chain(candidate.active.iter()) + .filter(|existing| recording_quota::quota_pool_for_task(*existing) == pool) + .map(persisted_recording_identity) + .any(|existing| existing == identity); + if pending_match { + return true; + } + // Terminal tasks do not block a fresh request: after a failed or + // cancelled attempt the user must be able to try again, and after a + // successful one they may legitimately want a second copy. The one + // exception is a rule occurrence — re-materializing an occurrence + // that already ran would duplicate it on every scheduler tick. + if !matches!(identity, RecordingIdentity::Occurrence { .. }) { + return false; + } + candidate + .finished + .iter() + .filter(|existing| recording_quota::quota_pool_for_task(*existing) == pool) + .map(persisted_recording_identity) + .any(|existing| existing == identity) +} + +/// Where a recording lives in the candidate snapshot. The first scan +/// produces one of these so the second access (mut borrow for writes) +/// is O(1) instead of repeating the linear search. +#[derive(Debug, Clone, Copy)] +pub(super) enum RecordingLocation { + Scheduled(usize), + Queue(usize), + Active(usize), + Finished(usize), +} + +/// Single linear scan that locates a recording anywhere in the +/// candidate snapshot. The returned `RecordingLocation` lets the +/// caller re-acquire the same task for a mutable borrow without a +/// second search. +pub(super) fn locate_recording(candidate: &PersistedRecordingQueue, uuid: &str) -> Option { + let matches_uuid = |task: &PersistedRecordingTask| task.uuid == uuid; + if let Some(i) = candidate.scheduled.iter().position(matches_uuid) { + return Some(RecordingLocation::Scheduled(i)); + } + if let Some(i) = candidate.queue.iter().position(matches_uuid) { + return Some(RecordingLocation::Queue(i)); + } + if let Some(i) = candidate.active.iter().position(matches_uuid) { + return Some(RecordingLocation::Active(i)); + } + if let Some(i) = candidate.finished.iter().position(matches_uuid) { + return Some(RecordingLocation::Finished(i)); + } + None +} + +/// Resolve a recording to a mutable borrow using a remembered +/// location. The location must have come from the same candidate; +/// callers obtain it via [`locate_recording`]. +pub(super) fn recording_mut_at( + candidate: &mut PersistedRecordingQueue, + location: RecordingLocation, +) -> Option<&mut PersistedRecordingTask> { + match location { + RecordingLocation::Scheduled(i) => candidate.scheduled.get_mut(i), + RecordingLocation::Queue(i) => candidate.queue.get_mut(i), + RecordingLocation::Active(i) => candidate.active.get_mut(i), + // Must be the located index, not element 0: returning the first + // finished task would silently edit an unrelated recording. + RecordingLocation::Finished(i) => candidate.finished.get_mut(i), + } +} diff --git a/backend/dvr/src/recording/recording_service/input.rs b/backend/dvr/src/recording/recording_service/input.rs new file mode 100644 index 000000000..27670d89a --- /dev/null +++ b/backend/dvr/src/recording/recording_service/input.rs @@ -0,0 +1,105 @@ +use super::ServiceError; +use crate::recording::recording_queue::RecordingTaskState; +use shared::model::{ + recording::{RecordingProvenance, RecordingVisibility}, + RecordingKind, UserId, XtreamCluster, +}; + +/// Server-resolved identifiers that the recording system needs. The +/// caller never sees the URL; the service resolves it from these. +#[derive(Debug, Clone)] +pub struct RecordingSourceInput { + pub target_id: String, + pub virtual_id: String, + pub cluster: XtreamCluster, + pub input_name: String, +} + +impl RecordingSourceInput { + pub fn validate(&self) -> Result<(), ServiceError> { + if self.target_id.trim().is_empty() || self.virtual_id.trim().is_empty() || self.input_name.trim().is_empty() { + return Err(ServiceError::InvalidSource); + } + Ok(()) + } +} + +/// Input for `RecordingService::create_recording`. +#[derive(Debug, Clone)] +pub struct CreateRecordingInput { + pub source: RecordingSourceInput, + pub program_title: String, + pub program_start: i64, + pub program_end: i64, + pub pre_roll_secs: u64, + pub post_roll_secs: u64, + pub visibility: RecordingVisibility, + pub channel_id: Option, + pub channel_name: Option, + /// Playlist group of the channel, when the caller already resolved it. + /// `None` makes the service look it up for organised layouts. + pub group: Option, + pub provenance: RecordingProvenance, + pub epg: Option, +} + +impl CreateRecordingInput { + pub fn validate(&self) -> Result<(), ServiceError> { + self.source.validate()?; + if self.program_end.checked_sub(self.program_start).is_none_or(|duration| duration <= 0) { + return Err(ServiceError::InvalidInterval); + } + Ok(()) + } +} + +/// Input for `RecordingService::create_media_recording_idempotent`: an +/// immediate VOD or series transfer. Title, extension and grouping come from +/// the playlist entry the caller already resolved. +#[derive(Debug, Clone)] +pub struct CreateMediaRecordingInput { + pub source: RecordingSourceInput, + pub title: String, + pub extension: String, + pub visibility: RecordingVisibility, + pub group: Option, + pub series_name: Option, +} + +impl CreateMediaRecordingInput { + pub fn validate(&self) -> Result { + self.source.validate()?; + if self.extension.trim_start_matches('.').trim().is_empty() { + return Err(ServiceError::InvalidPath); + } + match self.source.cluster { + XtreamCluster::Video => Ok(RecordingKind::Vod), + XtreamCluster::Series => Ok(RecordingKind::Series), + XtreamCluster::Live => Err(ServiceError::InvalidSource), + } + } +} + +/// Output of `RecordingService::create_recording` and friends. +#[derive(Debug, Clone)] +pub struct RecordingTaskView { + pub uuid: String, + pub owner_id: UserId, + pub visibility: RecordingVisibility, + pub filename_preview: String, + pub start_at: Option, + pub duration_secs: Option, + pub state: RecordingTaskState, +} + +/// Input for `RecordingService::edit_recording`. +#[derive(Debug, Clone, Default)] +pub struct EditRecordingPatch { + pub program_start: Option, + pub program_end: Option, + pub pre_roll_secs: Option, + pub post_roll_secs: Option, + pub program_title: Option, + pub channel_id: Option, + pub channel_name: Option, +} diff --git a/backend/dvr/src/recording/recording_service/rules.rs b/backend/dvr/src/recording/recording_service/rules.rs new file mode 100644 index 000000000..cc517d38c --- /dev/null +++ b/backend/dvr/src/recording/recording_service/rules.rs @@ -0,0 +1,60 @@ +use super::{ + control::{cancel_future_rule_recordings_in_candidate, map_queue_error, CancelOrigin}, + identity::locate_recording, + CancelledRuleRecording, RecordingService, ServiceError, +}; +use crate::recording::recording_queue::mutate; + +impl RecordingService { + pub async fn cancel_future_rule_recordings( + &self, + claims: &shared::model::Claims, + rule_id: &str, + now_secs: i64, + ) -> Result, ServiceError> { + let _ = Self::subject_id(claims)?; + if !claims.permissions.contains(shared::model::Permission::RecordingManage) { + return Err(ServiceError::Forbidden); + } + let mut cancelled = Vec::new(); + mutate(&self.recordings, |candidate| { + cancelled = cancel_future_rule_recordings_in_candidate(candidate, rule_id, now_secs); + Ok(()) + }) + .await + .map_err(|e| map_queue_error(&e))?; + Ok(cancelled) + } + + /// Compensating transaction for [`Self::cancel_future_rule_recordings`]. + /// + /// Moves each task back out of `finished` into the list it came from, + /// restoring the exact record that was captured before the cancel + /// (including `reserved_bytes`, which the cancel zeroed). A uuid that + /// something else has since claimed is left alone: a real + /// create/edit always wins over an undo. + pub async fn restore_cancelled_rule_recordings( + &self, + cancelled: &[CancelledRuleRecording], + ) -> Result<(), ServiceError> { + if cancelled.is_empty() { + return Ok(()); + } + mutate(&self.recordings, |candidate| { + for entry in cancelled { + let uuid = entry.task.uuid.as_str(); + candidate.finished.retain(|task| task.uuid != uuid); + if locate_recording(candidate, uuid).is_some() { + continue; + } + match entry.origin { + CancelOrigin::Scheduled => candidate.scheduled.push(entry.task.clone()), + CancelOrigin::Queue => candidate.queue.push(entry.task.clone()), + } + } + Ok(()) + }) + .await + .map_err(|e| map_queue_error(&e)) + } +} diff --git a/backend/dvr/src/recording/recording_service/tests.rs b/backend/dvr/src/recording/recording_service/tests.rs new file mode 100644 index 000000000..cb7898071 --- /dev/null +++ b/backend/dvr/src/recording/recording_service/tests.rs @@ -0,0 +1,405 @@ +use super::{ + control::cancel_future_rule_recordings_in_candidate, error::map_edit_validation_error, + filenames::render_live_filename, identity::candidate_has_duplicate_recording, window::effective_recording_window, + *, +}; +use crate::{ + recording::recording_queue::{mutate, PersistedRecordingQueue, RecordingTask, RecordingTaskState}, + recording_edit::EditError, +}; +use shared::model::{ + recording::{RecordingMetadata, RecordingOwner, RecordingProvenance, RecordingSource, RecordingVisibility}, + Permission, RecordingContainerFormat, RecordingKind, UserId, XtreamCluster, +}; +use std::collections::HashMap; +use tuliprox_core::model::{RecordingConfig, RecordingNotificationConfig}; + +fn source(target_name: &str, virtual_id: &str, input_name: &str) -> RecordingSourceInput { + RecordingSourceInput { + target_id: target_name.to_string(), + virtual_id: virtual_id.to_string(), + cluster: XtreamCluster::Live, + input_name: input_name.to_string(), + } +} + +fn create_input() -> CreateRecordingInput { + CreateRecordingInput { + source: source("target", "1", "input-a"), + program_title: "Pilot".to_string(), + program_start: 1_700_000_000, + program_end: 1_700_003_600, + pre_roll_secs: 0, + post_roll_secs: 0, + visibility: RecordingVisibility::Private, + channel_id: None, + channel_name: None, + group: None, + provenance: RecordingProvenance::default(), + epg: None, + } +} + +fn persisted_media(uuid: &str, owner: &str, visibility: RecordingVisibility, url: &str) -> PersistedRecordingTask { + let meta = RecordingMetadata::new_media( + RecordingOwner::User(UserId::from(owner)), + visibility, + RecordingSource::new("1", "1", "input-a"), + "Film".to_string(), + ); + PersistedRecordingTask { + media_identity: String::new(), + partition: crate::recording::recording_queue::RecordingPartition::default(), + uuid: uuid.to_string(), + kind: RecordingKind::Vod, + file_dir: std::path::PathBuf::from("/tmp"), + file_path: std::path::PathBuf::from(format!("/tmp/{uuid}.mp4")), + filename: format!("{uuid}.mp4"), + url: url.to_string(), + finished: false, + size: 0, + total_size: None, + paused: false, + error: None, + state: RecordingTaskState::Queued, + input_name: Some("input-a".to_string()), + priority: 0, + retry_attempts: 0, + next_retry_at: None, + recording: meta, + } +} + +fn queued_candidate(tasks: Vec) -> PersistedRecordingQueue { + PersistedRecordingQueue { queue: tasks, ..PersistedRecordingQueue::default() } +} + +fn persisted_rule_recording(uuid: &str, rule_id: Option<&str>, start_at: i64) -> PersistedRecordingTask { + let mut meta = RecordingMetadata::new_live( + RecordingOwner::User(UserId::from("web:alice")), + RecordingVisibility::Private, + RecordingSource::new("1", "1", "input-a"), + start_at, + start_at + 3_600, + 0, + 0, + ); + meta.reserved_bytes = 123; + meta.provenance.rule_id = rule_id.map(str::to_string); + PersistedRecordingTask { + media_identity: String::new(), + partition: crate::recording::recording_queue::RecordingPartition::default(), + uuid: uuid.to_string(), + file_dir: std::path::PathBuf::from("/tmp"), + file_path: std::path::PathBuf::from(format!("/tmp/{uuid}.ts")), + filename: format!("{uuid}.ts"), + url: "http://example.test/live.ts".to_string(), + finished: false, + size: 0, + total_size: None, + paused: false, + error: None, + state: RecordingTaskState::Scheduled, + kind: RecordingKind::Live, + input_name: Some("input-a".to_string()), + priority: 0, + retry_attempts: 0, + next_retry_at: None, + recording: meta, + } +} + +fn deleting_claims() -> shared::model::Claims { + shared::model::Claims { + username: "alice".to_string(), + iss: "tuliprox".to_string(), + iat: 0, + exp: 0, + roles: shared::model::RoleSet::new(), + permissions: Permission::RecordingDelete.into(), + pwd_version: 0, + subject_id: Some(UserId::from("web:alice")), + permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, + } +} + +/// A finished entry `uuid` of `owner` on the shared film, with its file in +/// `dir`. Both the final file and the partial are written, so a test sees +/// exactly which one a removal touched. +async fn finished_film( + queue: &RecordingQueue, + dir: &std::path::Path, + uuid: &str, + owner: &str, + state: RecordingTaskState, +) -> (std::path::PathBuf, std::path::PathBuf) { + let mut task = persisted_media(uuid, owner, RecordingVisibility::Private, "http://provider/film.mp4"); + task.file_dir = dir.to_path_buf(); + task.file_path = dir.join("recording.mp4"); + task.state = state; + task.finished = true; + let final_path = task.file_path.clone(); + let partial = crate::recording::recording_worker::recording_partial_path(&final_path); + std::fs::write(&final_path, b"recorded bytes").expect("write final file"); + std::fs::write(&partial, b"partial bytes").expect("write partial"); + let task = RecordingQueue::to_persisted(&RecordingQueue::from_persisted(task).expect("valid task")); + mutate(queue, move |candidate| { + candidate.finished.push(task.clone()); + Ok(()) + }) + .await + .expect("seed recording"); + (final_path, partial) +} + +fn editing_claims() -> shared::model::Claims { + shared::model::Claims { + username: "alice".to_string(), + iss: "tuliprox".to_string(), + iat: 0, + exp: 0, + roles: shared::model::RoleSet::new(), + permissions: Permission::RecordingManage.into(), + pwd_version: 0, + subject_id: Some(UserId::from("web:alice")), + permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, + } +} + +/// A scheduled live recording an hour from now, for `owner`. +fn upcoming_live(uuid: &str, owner: &str, start: i64) -> PersistedRecordingTask { + let mut task = persisted_rule_recording(uuid, None, start); + task.recording.owner = RecordingOwner::User(UserId::from(owner)); + task.file_path = std::path::PathBuf::from("/tmp/shared-programme.ts"); + task.filename = "shared-programme.ts".to_string(); + task.recording.relative_path = Some("shared-programme.ts".to_string()); + RecordingQueue::to_persisted(&RecordingQueue::from_persisted(task).expect("valid task")) +} + +async fn scheduled_queue(tasks: Vec) -> Arc { + let queue = Arc::new(RecordingQueue::new()); + mutate(&queue, move |candidate| { + candidate.scheduled.clone_from(&tasks); + Ok(()) + }) + .await + .expect("seed"); + queue +} + +/// The record set the repository actually holds, for tests that +/// assert a rejected mutation left persistence untouched. +async fn committed_records(queue: &RecordingQueue) -> Vec { + let repository = queue.repository.clone().expect("queue is repository backed"); + tokio::task::spawn_blocking(move || { + let mut guard = repository.lock().expect("repository lock"); + guard.load() + }) + .await + .expect("join") + .expect("load") + .tasks +} + +fn test_app_config() -> Arc { + Arc::new(AppConfig { + config: Arc::new(arc_swap::ArcSwap::from_pointee(tuliprox_core::model::Config::default())), + sources: Arc::new(arc_swap::ArcSwap::from_pointee(tuliprox_core::model::SourcesConfig::default())), + hdhomerun: Arc::new(arc_swap::ArcSwapOption::empty()), + api_proxy: Arc::new(arc_swap::ArcSwapOption::empty()), + file_locks: Arc::new(tuliprox_core::utils::FileLockManager::default()), + paths: Arc::new(arc_swap::ArcSwap::from_pointee(shared::model::ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(arc_swap::ArcSwapOption::empty()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(tuliprox_core::model::MediaToolCapabilities::default()), + }) +} + +fn filename_config(template: &str, container_format: RecordingContainerFormat) -> RecordingConfig { + let mut cfg = RecordingConfig::from(&shared::model::RecordingConfigDto::default()); + cfg.filename_template = template.to_string(); + cfg.container_format = container_format; + cfg.timezone = "Europe/Berlin".parse().expect("timezone parses"); + cfg +} + +fn filename_window(input: &CreateRecordingInput) -> EffectiveRecordingWindow { + EffectiveRecordingWindow { + scheduled_start: input.program_start, + scheduled_end: input.program_end, + execution_start: input.program_start, + remaining_duration_secs: 0, + } +} + +fn filename_input() -> CreateRecordingInput { + let mut input = create_input(); + input.program_title = "Hart van Nederland".to_string(); + input.channel_name = Some("┃NLZIET┃ SBS 6 HD".to_string()); + // 2023-11-14 22:13:20 UTC = 23:13 in Berlin. + input.program_start = 1_700_000_000; + input.program_end = 1_700_001_800; + input +} + +/// A service rooted at `dir` with the supplied disk block, and the +/// source and server configuration a recording needs to resolve its +/// own target and URL. +fn service_with_disk( + dir: &std::path::Path, + queue: &Arc, + disk: Option, +) -> RecordingService { + let mut rec_cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }); + rec_cfg.directory = dir.to_string_lossy().into_owned(); + rec_cfg.disk = disk; + let config = tuliprox_core::model::Config { + video: Some(tuliprox_core::model::VideoConfig { + extensions: Vec::new(), + web_search: None, + recording: Some(rec_cfg), + }), + ..tuliprox_core::model::Config::default() + }; + + let input = Arc::new(tuliprox_core::model::ConfigInput { id: 7, name: "input-a".into(), ..Default::default() }); + let target = Arc::new(tuliprox_core::model::ConfigTarget { + id: 11, + enabled: true, + name: "1".to_string(), + options: None, + sort: None, + filter: tuliprox_core::model::StagedFilter::default(), + output: vec![], + rename: None, + mapping_ids: None, + mapping: Arc::default(), + favourites: None, + processing_order: shared::model::ProcessingOrder::default(), + curation: None, + execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + }); + let sources = tuliprox_core::model::SourcesConfig { + inputs: vec![Arc::clone(&input)], + sources: vec![tuliprox_core::model::ConfigSource { inputs: vec!["input-a".into()], targets: vec![target] }], + ..tuliprox_core::model::SourcesConfig::default() + }; + + let app_config = test_app_config(); + app_config.config.store(Arc::new(config)); + app_config.sources.store(Arc::new(sources)); + RecordingService::new(Arc::clone(queue), app_config) +} + +fn creating_claims() -> shared::model::Claims { + shared::model::Claims { + username: "alice".to_string(), + iss: "tuliprox".to_string(), + iat: 0, + exp: 0, + roles: shared::model::RoleSet::new(), + permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, + pwd_version: 0, + subject_id: Some(UserId::from("web:alice")), + permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, + } +} + +fn claims_for(user: &str, permissions: shared::model::permission::PermissionSet) -> shared::model::Claims { + shared::model::Claims { + username: user.to_string(), + subject_id: Some(UserId::from(format!("web:{user}"))), + permissions, + ..creating_claims() + } +} + +fn media_input() -> CreateMediaRecordingInput { + CreateMediaRecordingInput { + source: RecordingSourceInput { + target_id: "1".to_string(), + virtual_id: "77".to_string(), + cluster: XtreamCluster::Video, + input_name: "input-a".to_string(), + }, + title: "The Film".to_string(), + extension: "mp4".to_string(), + visibility: RecordingVisibility::Private, + group: None, + series_name: None, + } +} + +fn with_private_quota(service: &RecordingService, bytes: u64) { + let mut config = tuliprox_core::model::Config::clone(&service.app_config.config.load()); + if let Some(recording) = config.video.as_mut().and_then(|video| video.recording.as_mut()) { + recording.quota = Some(tuliprox_core::model::RecordingQuotaConfig { + default_private_bytes: Some(bytes), + ..Default::default() + }); + } + service.app_config.config.store(Arc::new(config)); +} + +/// Alice's request for the film, finished at `bytes`. +async fn completed_film(service: &RecordingService, queue: &RecordingQueue, bytes: u64) { + service.create_media_recording_idempotent(&creating_claims(), &media_input(), None).await.expect("alice's request"); + mutate(queue, |candidate| { + let mut done = candidate.queue.remove(0); + done.state = RecordingTaskState::Completed; + done.finished = true; + done.size = bytes; + done.total_size = Some(bytes); + done.recording.measured_bytes = bytes; + candidate.finished.push(done); + Ok(()) + }) + .await + .expect("complete"); +} + +fn disk_test_input() -> CreateRecordingInput { + let now = chrono::Utc::now().timestamp(); + CreateRecordingInput { + source: RecordingSourceInput { + target_id: "1".to_string(), + virtual_id: "42".to_string(), + cluster: XtreamCluster::Live, + input_name: "input-a".to_string(), + }, + program_title: "title".to_string(), + program_start: now, + program_end: now + 600, + pre_roll_secs: 0, + post_roll_secs: 0, + visibility: RecordingVisibility::Private, + channel_id: None, + channel_name: None, + group: None, + provenance: RecordingProvenance::default(), + epg: None, + } +} + +mod admission; +mod http; +mod lifecycle; +mod playlist; +mod policy; +mod storage; +mod transport; diff --git a/backend/dvr/src/recording/recording_service/tests/admission.rs b/backend/dvr/src/recording/recording_service/tests/admission.rs new file mode 100644 index 000000000..f9551f265 --- /dev/null +++ b/backend/dvr/src/recording/recording_service/tests/admission.rs @@ -0,0 +1,179 @@ +use super::*; + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn edit_recording_re_validates_quota_against_new_duration_atomically() { + // Owner already has 800 reserved. New duration would push the + // reservation to 1100 against a 1000-byte quota. Edit must + // fail with QuotaExceeded and persist nothing. + let dir = tempfile::tempdir().expect("tempdir"); + let state_file = dir.path().join("downloads.json"); + let downloads = + Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); + let start = chrono::Utc::now().timestamp() + 3_600; + let mut task = RecordingQueue::from_persisted(persisted_rule_recording("recording", None, start)) + .expect("valid recording task"); + { + let meta = &mut task.recording; + meta.reserved_bytes = 800; + } + downloads.scheduled.write().await.push(task); + downloads.persist_to_disk().await.expect("persist initial queue"); + let persisted_before = committed_records(&downloads).await; + let quota = tuliprox_core::model::RecordingQuotaConfig { + default_private_bytes: Some(1_000), + per_user_bytes: HashMap::new(), + shared_bytes: None, + }; + let rec_cfg = RecordingConfig { + headers: HashMap::new(), + t_origin_headers: tuliprox_core::model::RecordingOriginHeaders::default(), + organize_into_directories: false, + episode_pattern: None, + priority: 0, + reserve_slots_for_users: 0, + max_background_per_provider: 0, + retry_backoff_initial_secs: 1, + retry_backoff_multiplier: 1.0, + retry_backoff_max_secs: 1, + retry_backoff_jitter_percent: 0, + retry_max_attempts: 1, + enabled: true, + container_format: RecordingContainerFormat::default(), + directory: String::new(), + timezone: "UTC".parse().expect("UTC must parse"), + filename_template: String::new(), + default_pre_roll_secs: 0, + max_pre_roll_secs: 900, + default_post_roll_secs: 0, + max_post_roll_secs: 1800, + retention: None, + disk: None, + quota: Some(quota), + notifications: RecordingNotificationConfig::default(), + fallback_bytes_per_minute: 60, + }; + let config = tuliprox_core::model::Config { + video: Some(tuliprox_core::model::VideoConfig { + extensions: Vec::new(), + web_search: None, + recording: Some(rec_cfg.clone()), + }), + ..tuliprox_core::model::Config::default() + }; + let app_config = Arc::new(AppConfig { + config: Arc::new(arc_swap::ArcSwap::from_pointee(config)), + sources: Arc::new(arc_swap::ArcSwap::from_pointee(tuliprox_core::model::SourcesConfig::default())), + hdhomerun: Arc::new(arc_swap::ArcSwapOption::empty()), + api_proxy: Arc::new(arc_swap::ArcSwapOption::empty()), + file_locks: Arc::new(tuliprox_core::utils::FileLockManager::default()), + paths: Arc::new(arc_swap::ArcSwap::from_pointee(shared::model::ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(arc_swap::ArcSwapOption::empty()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(tuliprox_core::model::MediaToolCapabilities::default()), + }); + let service = RecordingService::new(Arc::clone(&downloads), app_config); + let claims = shared::model::Claims { + username: "alice".to_string(), + iss: "tuliprox".to_string(), + iat: 0, + exp: 0, + roles: shared::model::RoleSet::new(), + permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, + pwd_version: 0, + subject_id: Some(UserId::from("web:alice")), + permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, + }; + let patch = EditRecordingPatch { program_end: Some(start + 1_200), ..EditRecordingPatch::default() }; + + let result = service.edit_recording(&claims, "recording", patch).await; + + assert!(matches!(result, Err(ServiceError::QuotaExceeded)), "got {result:?}"); + assert_eq!(committed_records(&downloads).await, persisted_before); + let scheduled = downloads.scheduled.read().await; + assert_eq!(scheduled[0].scheduled_start(), Some(start)); +} + +#[tokio::test] +async fn asking_for_a_finished_file_is_charged_its_size_at_admission() { + // Attaching copies the whole file into the new entry's charge, so a + // 16 byte quota must not let a 4096 byte file in. + let dir = tempfile::tempdir().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let service = service_with_disk(dir.path(), &queue, None); + completed_film(&service, &queue, 4096).await; + with_private_quota(&service, 16); + + let bob = claims_for("bob", Permission::RecordingCreate.into()); + let refused = service.create_media_recording_idempotent(&bob, &media_input(), None).await; + assert!(matches!(refused, Err(ServiceError::QuotaExceeded)), "got {refused:?}"); + + with_private_quota(&service, 8192); + service.create_media_recording_idempotent(&bob, &media_input(), None).await.expect("fits now"); + let queued = queue.queue.lock().await; + assert_eq!(queued.len(), 1); + assert_eq!(queued[0].recording.reserved_bytes, 4096, "reserved at the known size until it attaches"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn two_concurrent_requests_with_one_key_admit_only_one() { + // Both requests pass the first key lookup before either commits; the + // lookup under the mutation guard admits only one of them. + let dir = tempfile::tempdir().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let service = Arc::new(service_with_disk(dir.path(), &queue, None)); + let held = queue.queue.lock().await; + let handles: Vec<_> = [("77", "body-a"), ("78", "body-b")] + .into_iter() + .map(|(virtual_id, fingerprint)| { + let service = Arc::clone(&service); + let mut input = media_input(); + input.source.virtual_id = virtual_id.to_string(); + let request = IdempotencyRequest { key: "same-key".to_string(), fingerprint: fingerprint.to_string() }; + tokio::spawn(async move { + service.create_media_recording_idempotent(&creating_claims(), &input, Some(request)).await + }) + }) + .collect(); + // Both are past their first lookup and waiting to commit. + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + drop(held); + + let mut admitted = 0; + for handle in handles { + match handle.await.expect("join") { + Ok(_) => admitted += 1, + Err(error) => assert_eq!(error, ServiceError::IdempotencyConflict), + } + } + assert_eq!(admitted, 1); + assert_eq!(queue.queue.lock().await.len(), 1); +} + +#[tokio::test] +async fn the_same_recording_is_admitted_when_the_disk_has_room() { + // The counterpart: without the safety margin the identical request + // succeeds, so the refusal above is the disk rule and not the + // fixture failing for some unrelated reason. + let dir = tempfile::tempdir().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let service = service_with_disk(dir.path(), &queue, None); + + let result = service.create_recording(&creating_claims(), &disk_test_input()).await; + + assert!(result.is_ok(), "got {result:?}"); + assert_eq!(queue.scheduled.read().await.len(), 1); +} diff --git a/backend/dvr/src/recording/recording_service/tests/http.rs b/backend/dvr/src/recording/recording_service/tests/http.rs new file mode 100644 index 000000000..d16e1461a --- /dev/null +++ b/backend/dvr/src/recording/recording_service/tests/http.rs @@ -0,0 +1,53 @@ +use super::*; + +#[tokio::test] +async fn unsupported_range_requires_confirmation_before_vod_or_series_partial_is_discarded() { + for kind in [RecordingKind::Vod, RecordingKind::Series] { + let dir = tempfile::tempdir().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("recording repository")); + let mut task = + persisted_media("recording", "web:alice", RecordingVisibility::Private, "http://provider/film.mp4"); + task.kind = kind; + task.file_dir = dir.path().to_path_buf(); + task.file_path = dir.path().join("recording.mp4"); + task.state = RecordingTaskState::Failed; + task.finished = true; + task.size = 4; + task.total_size = Some(10); + task.error = Some(super::super::super::recording_transfer::RANGE_UNSUPPORTED_ERROR.to_string()); + task.recording.resume_etag = Some("\"old-etag\"".to_string()); + let partial = crate::recording::recording_worker::recording_partial_path(&task.file_path); + std::fs::write(&partial, b"0123").expect("saved partial"); + mutate(&queue, move |candidate| { + candidate.finished.push(task.clone()); + Ok(()) + }) + .await + .expect("seed failed recording"); + let service = RecordingService::new(Arc::clone(&queue), test_app_config()); + let claims = shared::model::Claims { + username: "alice".to_string(), + iss: "tuliprox".to_string(), + iat: 0, + exp: 0, + roles: shared::model::RoleSet::new(), + permissions: Permission::RecordingManage.into(), + pwd_version: 0, + subject_id: Some(UserId::from("web:alice")), + permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, + }; + + assert!(matches!(service.retry_recording(&claims, "recording").await, Err(ServiceError::InvalidState))); + assert_eq!(std::fs::read(&partial).expect("partial kept without consent"), b"0123"); + assert!(queue.finished.read().await[0].to_view(true).restart_from_beginning_required); + + assert!(service.restart_recording(&claims, "recording").await.expect("confirmed restart")); + assert!(!partial.exists()); + assert!(queue.finished.read().await.is_empty()); + let queued = queue.queue.lock().await.front().cloned().expect("queued transfer"); + assert_eq!(queued.state, RecordingTaskState::Queued); + assert_eq!(queued.size, 0); + assert_eq!(queued.total_size, None); + assert_eq!(queued.recording.resume_etag, None); + } +} diff --git a/backend/dvr/src/recording/recording_service/tests/lifecycle.rs b/backend/dvr/src/recording/recording_service/tests/lifecycle.rs new file mode 100644 index 000000000..8dbf1e8b2 --- /dev/null +++ b/backend/dvr/src/recording/recording_service/tests/lifecycle.rs @@ -0,0 +1,151 @@ +use super::*; + +#[tokio::test] +async fn removing_the_last_failed_or_cancelled_entry_removes_its_partial() { + for state in [RecordingTaskState::Failed, RecordingTaskState::Cancelled] { + let dir = tempfile::tempdir().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let (final_path, partial) = finished_film(&queue, dir.path(), "recording", "web:alice", state).await; + let service = RecordingService::new(Arc::clone(&queue), test_app_config()); + + assert!(service.remove_recording_task(&deleting_claims(), "recording").await.expect("remove")); + assert!(!partial.exists(), "{state:?}: a partial nothing will resume is not kept"); + assert!(final_path.exists(), "{state:?}: a final file is never removed by Remove"); + } +} + +#[test] +fn a_download_claiming_the_path_during_the_cleanup_keeps_its_partial() { + // The orphan check and the unlink happen under one mutation guard. A + // download admitted after the removal may take over the path, and + // must find its partial intact. + let runtime = + tokio::runtime::Builder::new_current_thread().enable_all().max_blocking_threads(1).build().expect("runtime"); + runtime.block_on(async { + let dir = tempfile::tempdir().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new()); + let (final_path, partial) = + finished_film(&queue, dir.path(), "recording", "web:alice", RecordingTaskState::Failed).await; + std::fs::remove_file(&final_path).expect("only the partial exists"); + let mut newcomer = + persisted_media("newcomer", "web:alice", RecordingVisibility::Private, "http://provider/film.mp4"); + newcomer.file_path.clone_from(&final_path); + newcomer.state = RecordingTaskState::Running; + + // Park the filesystem: the cleanup's unlink waits on the only + // blocking thread until released. + let (parked_tx, parked_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let blocker = tokio::task::spawn_blocking(move || { + parked_tx.send(()).expect("parked"); + let _ = release_rx.recv(); + }); + parked_rx.recv().expect("blocking thread taken"); + + let service = RecordingService::new(Arc::clone(&queue), test_app_config()); + let removal = tokio::spawn(async move { service.remove_recording_task(&deleting_claims(), "recording").await }); + while !queue.finished.read().await.is_empty() { + tokio::task::yield_now().await; + } + // The newcomer is admitted and starts writing, as soon as it can. + let claim_queue = Arc::clone(&queue); + let claim_partial = partial.clone(); + let claim = tokio::spawn(async move { + mutate(&claim_queue, move |candidate| { + candidate.active = vec![newcomer.clone()]; + Ok(()) + }) + .await + .expect("admit newcomer"); + tokio::fs::write(&claim_partial, b"newcomer's bytes").await.expect("newcomer writes"); + }); + for _ in 0..10 { + tokio::task::yield_now().await; + } + release_tx.send(()).expect("release"); + blocker.await.expect("blocker"); + assert!(removal.await.expect("join").expect("remove")); + claim.await.expect("claim"); + + assert_eq!(queue.active.read().await.first().map(|task| task.uuid.clone()).as_deref(), Some("newcomer")); + assert!(partial.exists(), "the newcomer's partial must survive the cleanup of the old entry"); + }); +} + +#[tokio::test] +async fn cancelling_an_active_recording_leaves_it_worker_owned_until_the_worker_finishes() { + // A cancel request leaves the active task to its worker: it must not + // show as finished while the worker is still writing, and removing it + // in that state must be refused rather than reported as done. + let dir = tempfile::tempdir().expect("tempdir"); + let state_file = dir.path().join("downloads.json"); + let downloads = + Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); + let mut task = RecordingQueue::from_persisted(persisted_media( + "recording", + "web:alice", + RecordingVisibility::Private, + "http://provider/film.mp4", + )) + .expect("valid recording task"); + task.state = RecordingTaskState::Running; + *downloads.active.write().await = vec![task]; + + let service = RecordingService::new(Arc::clone(&downloads), test_app_config()); + let claims = shared::model::Claims { + username: "alice".to_string(), + iss: "tuliprox".to_string(), + iat: 0, + exp: 0, + roles: shared::model::RoleSet::new(), + permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, + pwd_version: 0, + subject_id: Some(UserId::from("web:alice")), + permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, + }; + + service.cancel_recording(&claims, "recording").await.expect("cancel request accepted"); + + let active = downloads.active.read().await.first().cloned().expect("the worker still owns it"); + assert_eq!(active.state, RecordingTaskState::Cancelling, "the worker commits the terminal state"); + assert!(downloads.finished.read().await.is_empty(), "nothing terminal yet"); + assert!( + matches!(service.remove_recording_task(&claims, "recording").await, Err(ServiceError::InvalidState)), + "removing a worker-owned recording must be refused, not silently ignored" + ); +} + +#[test] +fn sanitize_filename_collapses_runs_and_drops_control_chars() { + assert_eq!(sanitize_filename_component("a///b"), "a_b"); + assert_eq!(sanitize_filename_component("a\u{7}\u{1}b"), "a_b"); +} + +#[test] +fn sanitize_filename_drops_invisible_formatting() { + // A right-to-left override renders the name differently from the + // bytes on disk; it must leave no trace at all. + assert_eq!(sanitize_filename_component("news\u{202e}sj.ts"), "newssj.ts"); + assert_eq!(sanitize_filename_component("a\u{200b}b"), "ab"); +} + +#[test] +fn cancel_future_rule_recordings_moves_only_matching_future_tasks() { + let now = 1_700_000_000; + let mut queue = PersistedRecordingQueue::default(); + queue.scheduled.push(persisted_rule_recording("future-match", Some("rule-1"), now + 60)); + queue.scheduled.push(persisted_rule_recording("past-match", Some("rule-1"), now - 60)); + queue.queue.push(persisted_rule_recording("other-rule", Some("rule-2"), now + 60)); + + let cancelled = cancel_future_rule_recordings_in_candidate(&mut queue, "rule-1", now); + + assert_eq!(cancelled.len(), 1); + assert_eq!(queue.scheduled.len(), 1); + assert_eq!(queue.queue.len(), 1); + assert_eq!(queue.finished.len(), 1); + let task = &queue.finished[0]; + assert_eq!(task.uuid, "future-match"); + assert_eq!(task.state, RecordingTaskState::Cancelled); + assert!(task.finished); + assert_eq!(task.recording.reserved_bytes, 0); +} diff --git a/backend/dvr/src/recording/recording_service/tests/playlist.rs b/backend/dvr/src/recording/recording_service/tests/playlist.rs new file mode 100644 index 000000000..69024e1dc --- /dev/null +++ b/backend/dvr/src/recording/recording_service/tests/playlist.rs @@ -0,0 +1,64 @@ +use super::*; + +#[tokio::test] +async fn edit_recording_clears_epg_when_channel_changes_without_programme() { + let dir = tempfile::tempdir().expect("tempdir"); + let state_file = dir.path().join("downloads.json"); + let downloads = + Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); + let mut task = + RecordingQueue::from_persisted(persisted_rule_recording("recording", None, 100)).expect("valid recording task"); + { + let meta = &mut task.recording; + meta.channel_id = Some("a".into()); + meta.channel_name = Some("A".into()); + meta.epg = Some(shared::model::recording::EpgEpisodeMetadata { + programme_id: Some("p-1".into()), + series_id: None, + episode_id: None, + season: None, + episode: None, + airing: shared::model::recording::AiringStatus::New, + }); + } + downloads.scheduled.write().await.push(task); + downloads.persist_to_disk().await.expect("persist initial queue"); + let service = RecordingService::new(Arc::clone(&downloads), test_app_config()); + let claims = shared::model::Claims { + username: "alice".to_string(), + iss: "tuliprox".to_string(), + iat: 0, + exp: 0, + roles: shared::model::RoleSet::new(), + permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, + pwd_version: 0, + subject_id: Some(UserId::from("web:alice")), + permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, + }; + let patch = EditRecordingPatch { channel_id: Some("b".into()), ..EditRecordingPatch::default() }; + + let result = service.edit_recording(&claims, "recording", patch).await; + assert!(result.is_ok()); + let scheduled = downloads.scheduled.read().await; + let meta = &scheduled[0].recording; + assert_eq!(meta.channel_id.as_deref(), Some("b")); + assert!(meta.epg.is_none(), "epg metadata must be cleared when channel changed without a fresh programme"); +} + +#[tokio::test] +async fn preview_conflict_ignores_other_target_or_input() { + let dir = tempfile::tempdir().expect("tempdir"); + let state_file = dir.path().join("downloads.json"); + let downloads = + Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); + let mut other_target = persisted_rule_recording("other-target", None, 100); + other_target.recording.source = shared::model::recording::RecordingSource::new("other-target", "9", "input-a"); + let mut other_input = persisted_rule_recording("other-input", None, 100); + other_input.recording.source = shared::model::recording::RecordingSource::new("1", "9", "input-b"); + let other_target_task = RecordingQueue::from_persisted(other_target).expect("valid task"); + let other_input_task = RecordingQueue::from_persisted(other_input).expect("valid task"); + downloads.queue.lock().await.push_back(other_target_task); + downloads.queue.lock().await.push_back(other_input_task); + let points = collect_demand_points_for_provider(&downloads, "1", "input-a").await; + assert!(points.is_empty(), "foreign target or input must not leak into the demand set"); +} diff --git a/backend/dvr/src/recording/recording_service/tests/policy.rs b/backend/dvr/src/recording/recording_service/tests/policy.rs new file mode 100644 index 000000000..d7332148e --- /dev/null +++ b/backend/dvr/src/recording/recording_service/tests/policy.rs @@ -0,0 +1,228 @@ +use super::*; + +#[test] +fn a_live_window_must_have_time_left_in_it() { + // A window whose padded end is not after its padded start cannot + // produce a recording, and admitting one means a scheduled capture that + // can only ever fail. + let now = 1_700_000_000; + // Zero-length and inverted programmes. + assert!(matches!(effective_recording_window(now + 100, now + 100, 0, 0, now), Err(ServiceError::InvalidInterval))); + assert!(matches!(effective_recording_window(now + 200, now + 100, 0, 0, now), Err(ServiceError::InvalidInterval))); + // A programme that finished before it was asked for. + assert!(matches!( + effective_recording_window(now - 7_200, now - 3_600, 0, 0, now), + Err(ServiceError::InvalidInterval) + )); +} + +#[test] +fn a_live_window_already_underway_records_only_what_is_left() { + // Joining late is legal; it just cannot rewind. The padded bounds stay + // as planned so the stop time is still the programme's. + let now = 1_700_000_000; + let window = effective_recording_window(now - 600, now + 600, 60, 120, now).expect("still running"); + + assert_eq!(window.scheduled_start, now - 660, "the padded start is history, not moved to now"); + assert_eq!(window.scheduled_end, now + 720); + assert_eq!(window.execution_start, now, "but recording starts now"); + assert_eq!(window.remaining_duration_secs, 720, "and runs to the padded end"); +} + +#[test] +fn another_user_asking_for_the_same_film_is_not_refused() { + // Until one physical file can carry several library entries, treating + // this as a duplicate would tell the second user "already recording" + // and leave them with nothing. + let existing = persisted_media("a", "web:alice", RecordingVisibility::Private, "http://p/film.mp4"); + let other_user = RecordingQueue::from_persisted(persisted_media( + "b", + "web:bob", + RecordingVisibility::Private, + "http://p/film.mp4", + )) + .expect("valid task"); + assert!(!candidate_has_duplicate_recording(&queued_candidate(vec![existing]), &other_user)); +} + +#[test] +fn a_shared_copy_does_not_collide_with_a_private_one() { + // The two charge different quota pools, so they are two recordings. + let existing = persisted_media("a", "web:alice", RecordingVisibility::Private, "http://p/film.mp4"); + let shared = RecordingQueue::from_persisted(persisted_media( + "b", + "web:alice", + RecordingVisibility::Shared, + "http://p/film.mp4", + )) + .expect("valid task"); + assert!(!candidate_has_duplicate_recording(&queued_candidate(vec![existing]), &shared)); +} + +#[test] +fn a_finished_transfer_does_not_block_a_fresh_request() { + // After a completed or failed attempt the user may legitimately want + // another copy. + let mut finished = persisted_media("a", "web:alice", RecordingVisibility::Private, "http://p/film.mp4"); + finished.finished = true; + finished.state = RecordingTaskState::Completed; + let candidate = PersistedRecordingQueue { finished: vec![finished], ..PersistedRecordingQueue::default() }; + let again = RecordingQueue::from_persisted(persisted_media( + "b", + "web:alice", + RecordingVisibility::Private, + "http://p/film.mp4", + )) + .expect("valid task"); + assert!(!candidate_has_duplicate_recording(&candidate, &again)); +} + +#[test] +fn a_completed_rule_occurrence_still_blocks_a_repeat() { + // The scheduler re-evaluates rules on every tick; without this an + // occurrence would be re-materialized forever. + let mut finished = persisted_rule_recording("a", Some("rule-1"), 100); + finished.recording.provenance.occurrence_key = Some("occ-1".to_string()); + finished.finished = true; + finished.state = RecordingTaskState::Completed; + let candidate = PersistedRecordingQueue { finished: vec![finished], ..PersistedRecordingQueue::default() }; + + let mut repeat_persisted = persisted_rule_recording("b", Some("rule-1"), 100); + repeat_persisted.recording.provenance.occurrence_key = Some("occ-1".to_string()); + let repeat = RecordingQueue::from_persisted(repeat_persisted).expect("valid task"); + assert!(candidate_has_duplicate_recording(&candidate, &repeat)); +} + +#[test] +fn effective_window_applies_padding_and_remaining_duration() { + let window = effective_recording_window(1_000, 2_000, 100, 200, 1_500).expect("valid effective window"); + + assert_eq!(window.scheduled_start, 900); + assert_eq!(window.scheduled_end, 2_200); + assert_eq!(window.execution_start, 1_500); + assert_eq!(window.remaining_duration_secs, 700); +} + +#[test] +fn effective_window_is_panic_free_at_integer_boundaries() { + let window = effective_recording_window(i64::MIN + 1, i64::MAX - 1, 10, 10, 0).expect("saturated effective window"); + + assert_eq!(window.scheduled_start, i64::MIN); + assert_eq!(window.scheduled_end, i64::MAX); + assert_eq!(window.execution_start, 0); + assert_eq!(window.remaining_duration_secs, i64::MAX as u64); +} + +#[tokio::test] +async fn removing_an_entry_keeps_a_partial_another_entry_still_holds() { + let dir = tempfile::tempdir().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let (_, partial) = finished_film(&queue, dir.path(), "recording", "web:alice", RecordingTaskState::Cancelled).await; + let _ = finished_film(&queue, dir.path(), "bob-recording", "web:bob", RecordingTaskState::Failed).await; + let service = RecordingService::new(Arc::clone(&queue), test_app_config()); + + assert!(service.remove_recording_task(&deleting_claims(), "recording").await.expect("remove")); + assert!(partial.exists(), "Bob's entry still points at this media"); + assert_eq!(queue.finished.read().await.len(), 1); +} + +#[tokio::test] +async fn a_vod_takes_no_window_but_can_still_be_retitled() { + let queue = scheduled_queue(Vec::new()).await; + let mut vod = persisted_media("film", "web:alice", RecordingVisibility::Private, "http://provider/film.mp4"); + vod.state = RecordingTaskState::Queued; + mutate(&queue, move |candidate| { + candidate.queue.push(vod.clone()); + Ok(()) + }) + .await + .expect("seed"); + let service = RecordingService::new(Arc::clone(&queue), test_app_config()); + + for patch in [ + EditRecordingPatch { program_start: Some(1), program_end: Some(2), ..EditRecordingPatch::default() }, + EditRecordingPatch { pre_roll_secs: Some(60), ..EditRecordingPatch::default() }, + ] { + let refused = service.edit_recording(&editing_claims(), "film", patch).await; + assert!(matches!(refused, Err(ServiceError::InvalidInterval)), "got {refused:?}"); + } + let retitled = EditRecordingPatch { program_title: Some("Better title".into()), ..EditRecordingPatch::default() }; + service.edit_recording(&editing_claims(), "film", retitled).await.expect("a title edit is fine"); + let queued = queue.queue.lock().await; + assert_eq!(queued[0].recording.program_title.as_deref(), Some("Better title")); + assert_eq!(queued[0].recording.program_start, None, "and it stays a transfer without a window"); +} + +#[tokio::test] +async fn padding_alone_does_not_split_shared_media() { + // The programme window decides the media, its padding does not. + let start = chrono::Utc::now().timestamp() + 3_600; + let queue = scheduled_queue(vec![ + upcoming_live("alice-entry", "web:alice", start), + upcoming_live("bob-entry", "web:bob", start), + ]) + .await; + let service = RecordingService::new(Arc::clone(&queue), test_app_config()); + + let patch = EditRecordingPatch { post_roll_secs: Some(600), ..EditRecordingPatch::default() }; + service.edit_recording(&editing_claims(), "alice-entry", patch).await.expect("edit"); + + let (_, tasks) = queue.committed_snapshot().await; + let paths: Vec<_> = tasks.iter().map(|task| task.file_path.clone()).collect(); + assert_eq!(paths[0], paths[1], "still one file for one programme"); +} + +#[test] +fn service_error_code_is_stable_string() { + assert_eq!(ServiceError::UnknownOwner.code(), "recording_unknown_owner"); + assert_eq!(ServiceError::InvalidSource.code(), "recording_invalid_source"); + assert_eq!(ServiceError::Forbidden.code(), "recording_forbidden"); + assert_eq!(ServiceError::SharedCreationNotAdministrator.code(), "recording_shared_not_administrator"); + assert_eq!(ServiceError::InvalidState.code(), "recording_invalid_state"); + assert_eq!(ServiceError::InvalidInterval.code(), "recording_invalid_interval"); + assert_eq!(ServiceError::UnknownRecording.code(), "recording_unknown"); + assert_eq!(ServiceError::PersistenceFailed.code(), "recording_persistence_failed"); + assert_eq!(ServiceError::ProvenanceImmutable.code(), "recording_provenance_immutable"); + assert_eq!(ServiceError::Disabled.code(), "recording_disabled"); +} + +#[test] +fn provenance_cleared_does_not_masquerade_as_invalid_state() { + assert_eq!(map_edit_validation_error(&EditError::ProvenanceCleared), ServiceError::ProvenanceImmutable); +} + +#[tokio::test] +async fn a_media_request_needs_the_create_permission_not_manage() { + let dir = tempfile::tempdir().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let service = service_with_disk(dir.path(), &queue, None); + + let manage_only = claims_for("alice", Permission::RecordingManage.into()); + let refused = service.create_media_recording_idempotent(&manage_only, &media_input(), None).await; + assert!(matches!(refused, Err(ServiceError::Forbidden)), "got {refused:?}"); + assert!(queue.queue.lock().await.is_empty()); + + let create_only = claims_for("alice", Permission::RecordingCreate.into()); + let admitted = service.create_media_recording_idempotent(&create_only, &media_input(), None).await; + assert!(admitted.is_ok(), "got {admitted:?}"); + let queued = queue.queue.lock().await; + assert_eq!(queued.len(), 1, "a transfer is queued, not scheduled"); + assert_eq!(queued[0].kind, RecordingKind::Vod); +} + +#[tokio::test] +async fn a_replayed_media_request_is_answered_without_queueing_again() { + let dir = tempfile::tempdir().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let service = service_with_disk(dir.path(), &queue, None); + let alice = claims_for("alice", Permission::RecordingCreate.into()); + let request = IdempotencyRequest { key: "k1".to_string(), fingerprint: "f1".to_string() }; + + let first = service + .create_media_recording_idempotent(&alice, &media_input(), Some(request.clone())) + .await + .expect("first request"); + let replay = service.create_media_recording_idempotent(&alice, &media_input(), Some(request)).await; + assert_eq!(replay.err(), Some(ServiceError::IdempotentReplay { recording_id: first.uuid })); + assert_eq!(queue.queue.lock().await.len(), 1); +} diff --git a/backend/dvr/src/recording/recording_service/tests/storage.rs b/backend/dvr/src/recording/recording_service/tests/storage.rs new file mode 100644 index 000000000..cde6d0fa6 --- /dev/null +++ b/backend/dvr/src/recording/recording_service/tests/storage.rs @@ -0,0 +1,318 @@ +use super::*; + +/// A VOD transfer for `owner`, resolved to `url`. +#[test] +fn borrowed_media_identity_preserves_persisted_keys() { + let url = "http://provider/film.mp4"; + let mut meta = persisted_media("key", "web:alice", RecordingVisibility::Private, url).recording; + assert_eq!( + recording_identity_key(&meta, url), + r#"{"Url":{"url":"http://provider/film.mp4","scheduled_start":null,"scheduled_end":null}}"# + ); + meta.source.target_id = "target".to_string(); + meta.source.virtual_id = "channel".to_string(); + meta.program_start = Some(10); + meta.program_end = Some(20); + assert_eq!( + recording_identity_key(&meta, url), + r#"{"Programme":{"target_id":"target","virtual_id":"channel","program_start":10,"program_end":20}}"# + ); + meta.provenance.rule_id = Some("rule".to_string()); + meta.provenance.occurrence_key = Some("occurrence".to_string()); + assert_eq!( + recording_identity_key(&meta, url), + r#"{"Occurrence":{"rule_id":"rule","occurrence_key":"occurrence"}}"# + ); +} + +#[test] +fn a_repeated_vod_request_is_a_duplicate() { + // Regression: duplicate detection only produced an identity for Live, + // so a user who asked for the same film twice got two downloads of it + // written side by side as `film.mp4` and `film_1.mp4`. + let existing = persisted_media("a", "web:alice", RecordingVisibility::Private, "http://p/film.mp4"); + let repeat = RecordingQueue::from_persisted(persisted_media( + "b", + "web:alice", + RecordingVisibility::Private, + "http://p/film.mp4", + )) + .expect("valid task"); + assert!(candidate_has_duplicate_recording(&queued_candidate(vec![existing]), &repeat)); +} + +#[test] +fn a_different_film_is_not_a_duplicate() { + let existing = persisted_media("a", "web:alice", RecordingVisibility::Private, "http://p/film.mp4"); + let other = RecordingQueue::from_persisted(persisted_media( + "b", + "web:alice", + RecordingVisibility::Private, + "http://p/other.mp4", + )) + .expect("valid task"); + assert!(!candidate_has_duplicate_recording(&queued_candidate(vec![existing]), &other)); +} + +#[tokio::test] +async fn edit_recording_rejects_padding_above_max_without_persisting_mutation() { + let dir = tempfile::tempdir().expect("tempdir"); + let state_file = dir.path().join("downloads.json"); + let downloads = + Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); + let task = + RecordingQueue::from_persisted(persisted_rule_recording("recording", None, 100)).expect("valid recording task"); + downloads.scheduled.write().await.push(task); + downloads.persist_to_disk().await.expect("persist initial queue"); + let persisted_before = committed_records(&downloads).await; + let service = RecordingService::new(Arc::clone(&downloads), test_app_config()); + let claims = shared::model::Claims { + username: "alice".to_string(), + iss: "tuliprox".to_string(), + iat: 0, + exp: 0, + roles: shared::model::RoleSet::new(), + permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, + pwd_version: 0, + subject_id: Some(UserId::from("web:alice")), + permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, + }; + let patch = EditRecordingPatch { pre_roll_secs: Some(901), ..EditRecordingPatch::default() }; + + let result = service.edit_recording(&claims, "recording", patch).await; + + assert!(matches!(result, Err(ServiceError::PaddingLimitExceeded))); + assert_eq!(committed_records(&downloads).await, persisted_before); + let scheduled = downloads.scheduled.read().await; + assert_eq!(scheduled[0].recording.pre_roll_secs, 0); +} + +#[tokio::test] +async fn an_edit_that_leaves_shared_media_moves_to_a_path_of_its_own() { + // Alice and Bob scheduled the same programme and hold one file path. + // Alice moving her window makes it different media; keeping the path + // would have two captures write one file and let deleting hers remove + // Bob's. + let start = chrono::Utc::now().timestamp() + 3_600; + let queue = scheduled_queue(vec![ + upcoming_live("alice-entry", "web:alice", start), + upcoming_live("bob-entry", "web:bob", start), + ]) + .await; + let service = RecordingService::new(Arc::clone(&queue), test_app_config()); + + let patch = EditRecordingPatch { program_end: Some(start + 7_200), ..EditRecordingPatch::default() }; + service.edit_recording(&editing_claims(), "alice-entry", patch).await.expect("edit"); + + let (_, tasks) = queue.committed_snapshot().await; + let task = |uuid: &str| tasks.iter().find(|task| task.uuid == uuid).cloned().expect("entry"); + let (alice, bob) = (task("alice-entry"), task("bob-entry")); + assert_ne!(alice.file_path, bob.file_path, "the edited entry no longer writes Bob's file"); + assert_eq!(bob.file_path, std::path::PathBuf::from("/tmp/shared-programme.ts"), "Bob's entry is untouched"); + let identity = |task: &RecordingTask| recording_identity_key(&task.recording, task.url.as_str()); + assert_ne!(identity(&alice), identity(&bob)); +} + +#[tokio::test] +async fn edit_recording_rejects_overflowing_interval_without_persisting_mutation() { + let dir = tempfile::tempdir().expect("tempdir"); + let state_file = dir.path().join("downloads.json"); + let downloads = + Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); + let task = + RecordingQueue::from_persisted(persisted_rule_recording("recording", None, 100)).expect("valid recording task"); + downloads.scheduled.write().await.push(task); + downloads.persist_to_disk().await.expect("persist initial queue"); + let persisted_before = committed_records(&downloads).await; + let revision_before = downloads.revision.load(std::sync::atomic::Ordering::SeqCst); + let service = RecordingService::new(Arc::clone(&downloads), test_app_config()); + let claims = shared::model::Claims { + username: "alice".to_string(), + iss: "tuliprox".to_string(), + iat: 0, + exp: 0, + roles: shared::model::RoleSet::new(), + permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, + pwd_version: 0, + subject_id: Some(UserId::from("web:alice")), + permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, + }; + let patch = EditRecordingPatch { + program_start: Some(i64::MIN), + program_end: Some(i64::MAX), + program_title: Some("must not persist".to_string()), + ..EditRecordingPatch::default() + }; + + let result = service.edit_recording(&claims, "recording", patch).await; + + assert!(matches!(result, Err(ServiceError::InvalidInterval))); + assert_eq!(downloads.revision.load(std::sync::atomic::Ordering::SeqCst), revision_before); + assert_eq!(committed_records(&downloads).await, persisted_before); + let scheduled = downloads.scheduled.read().await; + assert_eq!(scheduled[0].scheduled_start(), Some(100)); + assert_ne!(scheduled[0].recording.program_title.as_deref(), Some("must not persist")); +} + +#[test] +fn live_filename_renders_the_configured_template() { + let input = filename_input(); + let cfg = filename_config("{channel}_{program_title}_{start_time}", RecordingContainerFormat::Mpegts); + assert_eq!( + render_live_filename(&input, &cfg, &filename_window(&input), "alice"), + "NLZIET_SBS_6_HD_Hart_van_Nederland_2023-11-14_23-13.ts" + ); +} + +#[test] +fn live_filename_collapses_empty_placeholders() { + let mut input = filename_input(); + input.channel_name = None; + let cfg = filename_config("{channel}_{program_title}_{episode}", RecordingContainerFormat::Matroska); + assert_eq!(render_live_filename(&input, &cfg, &filename_window(&input), "alice"), "Hart_van_Nederland.mkv"); +} + +#[test] +fn live_filename_renders_episode_and_owner() { + let mut input = filename_input(); + input.epg = Some(shared::model::EpgEpisodeMetadata { + programme_id: None, + series_id: None, + episode_id: None, + season: Some(3), + episode: Some(9), + airing: shared::model::AiringStatus::default(), + }); + let cfg = filename_config("{owner}-{program_title}-{episode}", RecordingContainerFormat::Mp4); + assert_eq!( + render_live_filename(&input, &cfg, &filename_window(&input), "alice"), + "alice-Hart_van_Nederland-S03E09.mp4" + ); +} + +#[test] +fn live_filename_keeps_dots_and_does_not_double_the_extension() { + let mut input = filename_input(); + input.program_title = "Mr. Robot".to_string(); + let cfg = filename_config("{program_title}.ts", RecordingContainerFormat::Mpegts); + assert_eq!(render_live_filename(&input, &cfg, &filename_window(&input), "alice"), "Mr.Robot.ts"); + let cfg = filename_config("{program_title}", RecordingContainerFormat::Matroska); + assert_eq!(render_live_filename(&input, &cfg, &filename_window(&input), "alice"), "Mr.Robot.mkv"); +} + +#[test] +fn live_filename_without_template_uses_the_title() { + let input = filename_input(); + let cfg = filename_config("", RecordingContainerFormat::Mpegts); + assert_eq!(render_live_filename(&input, &cfg, &filename_window(&input), "alice"), "Hart van Nederland.ts"); +} + +#[test] +fn sanitize_filename_strips_separators_and_reserved_characters() { + assert_eq!(sanitize_filename_component("a/b\\c:d*e?f\"gi|j"), "a_b_c_d_e_f_g_h_i_j"); +} + +#[test] +fn sanitize_filename_rejects_traversal_and_empty_results() { + assert_eq!(sanitize_filename_component(""), "recording"); + assert_eq!(sanitize_filename_component("."), "recording"); + assert_eq!(sanitize_filename_component(".."), "recording"); + assert_eq!(sanitize_filename_component(" "), "recording"); + // A lone separator becomes the substitute character, which is + // itself a perfectly valid component. + assert_eq!(sanitize_filename_component("/"), "_"); +} + +#[test] +fn sanitize_filename_rejects_windows_device_names() { + assert_eq!(sanitize_filename_component("CON"), "recording"); + assert_eq!(sanitize_filename_component("nul.ts"), "recording"); + assert_eq!(sanitize_filename_component("lpt9"), "recording"); + // Not reserved: only an exact stem match counts. + assert_eq!(sanitize_filename_component("console"), "console"); +} + +#[test] +fn sanitize_filename_trims_trailing_dots_and_spaces() { + assert_eq!(sanitize_filename_component("Show. "), "Show"); + assert_eq!(sanitize_filename_component(" .Show"), "Show"); +} + +#[test] +fn sanitize_filename_is_idempotent_and_bounded() { + let long = "\u{e9}".repeat(400); + let once = sanitize_filename_component(&long); + assert!(once.len() <= MAX_FILENAME_COMPONENT_BYTES); + // Truncation never splits a character. + assert!(once.chars().all(|ch| ch == '\u{e9}')); + assert_eq!(sanitize_filename_component(&once), once); + for raw in ["a/b", "CON", "", "Show. ", "news\u{202e}sj.ts"] { + let first = sanitize_filename_component(raw); + assert_eq!(sanitize_filename_component(&first), first, "not idempotent: {raw}"); + } +} + +#[test] +fn sanitized_filename_is_always_a_single_valid_component() { + let very_long = "x".repeat(500); + let cases = ["a/b/c", "..", "\u{0}x", "CON", " ", "../../etc/passwd", &very_long]; + for raw in cases { + let sanitized = sanitize_filename_component(raw); + validate_reserved_filename(&sanitized) + .unwrap_or_else(|err| panic!("{raw:?} sanitized to invalid component: {err}")); + } +} + +#[test] +fn validate_reserved_filename_rejects_parent_and_curdir_components() { + assert!(validate_reserved_filename("..").is_err()); + assert!(validate_reserved_filename(".").is_err()); + assert!(validate_reserved_filename("a/..").is_err()); + assert!(validate_reserved_filename("normal.ts").is_ok()); +} + +#[tokio::test] +async fn a_second_user_gets_an_own_entry_and_a_repeat_is_a_duplicate() { + // Another user asking for the same film must neither be handed the + // first user's entry nor be refused; the same user asking twice is + // a duplicate. + let dir = tempfile::tempdir().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let service = service_with_disk(dir.path(), &queue, None); + let alice = claims_for("alice", Permission::RecordingCreate.into()); + let bob = claims_for("bob", Permission::RecordingCreate.into()); + + let first = service.create_media_recording_idempotent(&alice, &media_input(), None).await.expect("alice"); + let second = service.create_media_recording_idempotent(&bob, &media_input(), None).await.expect("bob"); + assert_ne!(first.uuid, second.uuid); + assert_eq!(second.owner_id, UserId::from("web:bob")); + + let repeat = service.create_media_recording_idempotent(&alice, &media_input(), None).await; + assert!(matches!(repeat, Err(ServiceError::Duplicate)), "got {repeat:?}"); + assert_eq!(queue.queue.lock().await.len(), 2); +} + +#[tokio::test] +async fn a_recording_with_no_room_on_disk_is_refused() { + // Logical quota and physical space are different questions, and + // admission asks both: a full disk refuses the recording up front + // instead of letting ffmpeg fail on ENOSPC. The safety margin drives + // headroom to zero here rather than actually filling a filesystem. + let dir = tempfile::tempdir().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let service = service_with_disk( + dir.path(), + &queue, + Some(tuliprox_core::model::RecordingDiskConfig { + high_water_percent: None, + low_water_percent: None, + cleanup_interval_secs: None, + safety_bytes: Some(u64::MAX), + }), + ); + + let result = service.create_recording(&creating_claims(), &disk_test_input()).await; + + assert!(matches!(result, Err(ServiceError::DiskFull)), "got {result:?}"); + assert!(queue.scheduled.read().await.is_empty(), "a refused admission must not leave a recording behind"); +} diff --git a/backend/dvr/src/recording/recording_service/tests/transport.rs b/backend/dvr/src/recording/recording_service/tests/transport.rs new file mode 100644 index 000000000..469259b62 --- /dev/null +++ b/backend/dvr/src/recording/recording_service/tests/transport.rs @@ -0,0 +1,257 @@ +use super::*; + +#[test] +fn source_input_rejects_empty_virtual_id() { + let input = source("target", "", "i"); + assert!(matches!(input.validate(), Err(ServiceError::InvalidSource))); +} + +#[test] +fn source_input_rejects_empty_input_name() { + let input = source("target", "1", ""); + assert!(matches!(input.validate(), Err(ServiceError::InvalidSource))); +} + +#[test] +fn source_input_accepts_non_empty_identifiers() { + let input = source("target", "1", "input-a"); + assert!(input.validate().is_ok()); +} + +#[test] +fn create_recording_input_rejects_zero_or_negative_interval() { + let mut input = create_input(); + input.program_end = input.program_start; + assert!(matches!(input.validate(), Err(ServiceError::InvalidInterval))); + input.program_end = input.program_start - 1; + assert!(matches!(input.validate(), Err(ServiceError::InvalidInterval))); +} + +#[test] +fn create_recording_input_accepts_valid_interval() { + let input = create_input(); + assert!(input.validate().is_ok()); +} + +#[test] +fn create_recording_input_rejects_overflowing_interval() { + let mut input = create_input(); + input.program_start = i64::MIN; + input.program_end = i64::MAX; + + assert!(matches!(input.validate(), Err(ServiceError::InvalidInterval))); +} + +#[test] +fn effective_window_rejects_exact_or_past_end_boundary() { + assert!(matches!(effective_recording_window(1_000, 2_000, 100, 200, 2_200), Err(ServiceError::InvalidInterval))); + assert!(matches!(effective_recording_window(1_000, 2_000, 100, 200, 2_201), Err(ServiceError::InvalidInterval))); +} + +#[tokio::test] +async fn edit_recording_rejects_active_state_with_invalid_state_error() { + let dir = tempfile::tempdir().expect("tempdir"); + let state_file = dir.path().join("downloads.json"); + let downloads = + Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); + let mut task = + RecordingQueue::from_persisted(persisted_rule_recording("recording", None, 100)).expect("valid recording task"); + task.state = RecordingTaskState::Running; + downloads.scheduled.write().await.push(task); + downloads.persist_to_disk().await.expect("persist initial queue"); + let persisted_before = committed_records(&downloads).await; + let service = RecordingService::new(Arc::clone(&downloads), test_app_config()); + let claims = shared::model::Claims { + username: "alice".to_string(), + iss: "tuliprox".to_string(), + iat: 0, + exp: 0, + roles: shared::model::RoleSet::new(), + permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, + pwd_version: 0, + subject_id: Some(UserId::from("web:alice")), + permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, + }; + let patch = + EditRecordingPatch { program_title: Some("must not persist".to_string()), ..EditRecordingPatch::default() }; + + let result = service.edit_recording(&claims, "recording", patch).await; + + assert!(matches!(result, Err(ServiceError::InvalidState))); + assert_eq!(committed_records(&downloads).await, persisted_before); +} + +#[tokio::test] +async fn removing_a_completed_entry_keeps_its_file() { + // Remove takes the entry off the list; the recording stays on disk + // for whoever reads the directory. Deleting the file is its own action. + let dir = tempfile::tempdir().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let (final_path, _) = + finished_film(&queue, dir.path(), "recording", "web:alice", RecordingTaskState::Completed).await; + let service = RecordingService::new(Arc::clone(&queue), test_app_config()); + + assert!(service.remove_recording_task(&deleting_claims(), "recording").await.expect("remove")); + assert!(final_path.exists(), "the completed file was deleted while only removing the entry"); + assert!(queue.finished.read().await.is_empty()); +} + +#[tokio::test] +async fn an_edited_window_keeps_its_padding() { + // An edited window is scheduled with its pre- and post-roll, like a + // new one. + let start = chrono::Utc::now().timestamp() + 3_600; + let queue = scheduled_queue(vec![upcoming_live("recording", "web:alice", start)]).await; + let service = RecordingService::new(Arc::clone(&queue), test_app_config()); + + let patch = EditRecordingPatch { + program_end: Some(start + 1_800), + pre_roll_secs: Some(120), + post_roll_secs: Some(300), + ..EditRecordingPatch::default() + }; + service.edit_recording(&editing_claims(), "recording", patch).await.expect("edit"); + + let scheduled = queue.scheduled.read().await; + let meta = &scheduled[0].recording; + assert_eq!(meta.program_start, Some(start)); + assert_eq!(meta.scheduled_start, Some(start - 120)); + assert_eq!(meta.scheduled_end, Some(start + 1_800 + 300)); +} + +#[tokio::test] +async fn create_recording_without_download_config_reports_disabled() { + // A missing `video.recording` block means the server has no + // download engine at all — the caller's source identifiers are + // not wrong. Reporting `InvalidSource` here sent clients + // hunting for a misconfiguration that does not exist. + let dir = tempfile::tempdir().expect("tempdir"); + let downloads = + Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open recording repository")); + let service = RecordingService::new(Arc::clone(&downloads), test_app_config()); + let claims = shared::model::Claims { + username: "alice".to_string(), + iss: "tuliprox".to_string(), + iat: 0, + exp: 0, + roles: shared::model::RoleSet::new(), + permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, + pwd_version: 0, + subject_id: Some(UserId::from("web:alice")), + permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, + }; + + let result = service.create_recording(&claims, &create_input()).await; + + assert!(matches!(result, Err(ServiceError::Disabled))); +} + +#[tokio::test] +async fn preview_conflict_collects_demand_points_from_queue_state() { + // The server-side preview must build its own demand points from + // the committed queue state. A queued recording on the same + // target/input pair must show up as `others` even when the + // caller submits no `others` payload. + let dir = tempfile::tempdir().expect("tempdir"); + let state_file = dir.path().join("downloads.json"); + let downloads = + Arc::new(RecordingQueue::new_persistent(&state_file, &state_file).expect("open recording repository")); + let mut existing = persisted_rule_recording("existing", None, 100); + // Place a padded window that overlaps 100..200. + existing.recording.scheduled_start = Some(100); + existing.recording.scheduled_end = Some(200); + let existing = RecordingQueue::from_persisted(existing).expect("valid recording task"); + downloads.queue.lock().await.push_back(existing); + let points = collect_demand_points_for_provider(&downloads, "1", "input-a").await; + assert_eq!(points.len(), 1, "queue entry must surface as a demand point"); + assert_eq!(points[0].padded_start, 100); + assert_eq!(points[0].padded_end, 200); +} + +#[test] +fn is_future_rule_recording_rejects_non_editable_states() { + // Old implementation passed `cancel_targets_task(false, true)` + // literally — that always returned `true`, so the rule cancel + // path would happily tear down a task whose state was already + // terminal. Both terminal and non-editable-but-active states + // must be skipped now. + let mut cancelled_task = persisted_rule_recording("uuid-c", Some("rule-1"), 1_900_000_000); + cancelled_task.state = RecordingTaskState::Cancelled; + assert!(!is_future_rule_recording(&cancelled_task, "rule-1", 1_800_000_000)); + + let mut paused_task = persisted_rule_recording("uuid-p", Some("rule-1"), 1_900_000_000); + paused_task.state = RecordingTaskState::Paused; + assert!(!is_future_rule_recording(&paused_task, "rule-1", 1_800_000_000)); + + // Sanity: the happy path still accepts editable future tasks. + let scheduled_task = persisted_rule_recording("uuid-s", Some("rule-1"), 1_900_000_000); + assert!(is_future_rule_recording(&scheduled_task, "rule-1", 1_800_000_000)); +} + +#[tokio::test] +async fn create_recording_rejects_absent_recording_config() { + let config = tuliprox_core::model::Config::default(); + let app_config = Arc::new(AppConfig { + config: Arc::new(arc_swap::ArcSwap::from_pointee(config)), + sources: Arc::new(arc_swap::ArcSwap::from_pointee(tuliprox_core::model::SourcesConfig::default())), + hdhomerun: Arc::new(arc_swap::ArcSwapOption::empty()), + api_proxy: Arc::new(arc_swap::ArcSwapOption::empty()), + file_locks: Arc::new(tuliprox_core::utils::FileLockManager::default()), + paths: Arc::new(arc_swap::ArcSwap::from_pointee(shared::model::ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(arc_swap::ArcSwapOption::empty()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(tuliprox_core::model::MediaToolCapabilities::default()), + }); + let downloads = Arc::new(RecordingQueue::new()); + let service = RecordingService::new(Arc::clone(&downloads), app_config); + let claims = shared::model::Claims { + username: "alice".to_string(), + iss: "tuliprox".to_string(), + iat: 0, + exp: 0, + roles: shared::model::RoleSet::new(), + permissions: Permission::RecordingCreate | Permission::RecordingManage | Permission::RecordingDelete, + pwd_version: 0, + subject_id: Some(UserId::from("web:alice")), + permission_schema_version: shared::model::CURRENT_PERMISSION_SCHEMA_VERSION, + }; + let input = CreateRecordingInput { + source: RecordingSourceInput { + target_id: "1".to_string(), + virtual_id: "1".to_string(), + cluster: XtreamCluster::Live, + input_name: "input-a".to_string(), + }, + program_title: "title".to_string(), + program_start: 0, + program_end: 60, + pre_roll_secs: 0, + post_roll_secs: 0, + visibility: RecordingVisibility::Private, + channel_id: None, + channel_name: None, + group: None, + provenance: RecordingProvenance::default(), + epg: None, + }; + + let result = service.create_recording(&claims, &input).await; + + assert!( + matches!(result, Err(ServiceError::Disabled)), + "absent recording config must fail closed with Disabled, got: {result:?}" + ); +} diff --git a/backend/dvr/src/recording/recording_service/window.rs b/backend/dvr/src/recording/recording_service/window.rs new file mode 100644 index 000000000..9c8b9bae5 --- /dev/null +++ b/backend/dvr/src/recording/recording_service/window.rs @@ -0,0 +1,42 @@ +use super::{EffectiveRecordingWindow, ServiceError}; +use crate::recording_edit::PaddingBounds; + +pub(super) fn effective_recording_window( + program_start: i64, + program_end: i64, + pre_roll_secs: u64, + post_roll_secs: u64, + now: i64, +) -> Result { + if program_end <= program_start { + return Err(ServiceError::InvalidInterval); + } + let pre_roll = i64::try_from(pre_roll_secs).map_err(|_| ServiceError::PaddingLimitExceeded)?; + let post_roll = i64::try_from(post_roll_secs).map_err(|_| ServiceError::PaddingLimitExceeded)?; + let scheduled_start = program_start.saturating_sub(pre_roll); + let scheduled_end = program_end.saturating_add(post_roll); + let execution_start = now.max(scheduled_start); + // `scheduled_end >= execution_start` because both come from + // saturating arithmetic on a non-empty interval, so the cast is + // safe and the only remaining error is the degenerate + // already-finished window. + let remaining = scheduled_end.saturating_sub(execution_start); + if remaining <= 0 { + return Err(ServiceError::InvalidInterval); + } + let remaining_duration_secs = remaining.cast_unsigned(); + Ok(EffectiveRecordingWindow { scheduled_start, scheduled_end, execution_start, remaining_duration_secs }) +} + +pub(super) fn padding_bounds(recording: Option<&tuliprox_core::model::RecordingConfig>) -> PaddingBounds { + recording.map_or( + PaddingBounds { + max_pre_roll_secs: shared::model::default_recording_max_pre_roll_secs(), + max_post_roll_secs: shared::model::default_recording_max_post_roll_secs(), + }, + |config| PaddingBounds { + max_pre_roll_secs: config.max_pre_roll_secs, + max_post_roll_secs: config.max_post_roll_secs, + }, + ) +} diff --git a/backend/dvr/src/recording/recording_transfer.rs b/backend/dvr/src/recording/recording_transfer.rs index 5f0606df5..f4f9fa340 100644 --- a/backend/dvr/src/recording/recording_transfer.rs +++ b/backend/dvr/src/recording/recording_transfer.rs @@ -6,4007 +6,91 @@ //! notifications and progress publication. The HTTP layer only queues tasks //! and reads the queue; it never drives execution. -use crate::{ - http_transfer::{ - compute_total_size, is_retryable_error, is_retryable_status, retryable_transport_error_message, - validate_resume_response, ResponseSnapshot, ResumeValidationError, ResumeValidator, - }, - recording::{ - recording_capacity::RecordingCapacityPort, - recording_ctx::RecordingCtx, - recording_notification::LifecycleEvent, - recording_notification_adapter::{build_marker, decide, message_for, DispatchDecision}, - recording_queue::{ - mutate_optional, PersistedRecordingQueue, PersistedRecordingTask, QueueMutationError, RecordingControl, - RecordingQueue, RecordingTask, RecordingTaskState, RecordingWaitOutcome, - }, - recording_sidecar, - recording_url::build_stable_recording_url, - recording_worker::{ - recording_partial_path, redact_url_tokens, run_recording_with_binary, RecordingExecutionResult, - }, - }, -}; -use futures::stream::TryStreamExt; -use log::{debug, error, info, warn}; -use shared::{ - error::to_io_error, - model::{EventMessage, EventSink, RecordingKind, RecordingMetadata}, - utils::bytes_to_megabytes, -}; -use std::{collections::HashMap, path::Path, sync::Arc}; -use tokio::{ - fs, - io::{AsyncWrite, AsyncWriteExt}, - sync::{Notify, RwLock}, - time::{self, Duration, Instant}, -}; -use tokio_util::sync::CancellationToken; -use tuliprox_core::{ - model::{AppConfig, MessageContent, RecordingConfig}, - utils::{async_file_writer, request, request::create_client, IO_BUFFER_SIZE}, -}; +use tokio::time::Duration; const DOWNLOAD_PROGRESS_LOG_INTERVAL: Duration = Duration::from_secs(5); + const DOWNLOAD_PROGRESS_LOG_BYTES: u64 = 16 * 1024 * 1024; + const DOWNLOAD_SNAPSHOT_UPDATE_INTERVAL: Duration = Duration::from_secs(2); + const DOWNLOAD_SNAPSHOT_UPDATE_BYTES: u64 = 4 * 1024 * 1024; + // Pause/cancel/restart are delivered immediately via `control_notify` while the // worker is parked in the `select!`. This poll is only a fallback for the rare // race where a control change fires while a chunk is being written (notify is not // persisted), so it does not need to run on every chunk. const DOWNLOAD_CONTROL_POLL_INTERVAL: Duration = Duration::from_millis(200); + pub(crate) const RANGE_UNSUPPORTED_ERROR: &str = "range_unsupported"; + const RECORDING_PROGRESS_UPDATE_INTERVAL: Duration = Duration::from_secs(5); -type ProviderCapacities = Vec<(Arc, usize, usize)>; - -#[derive(Debug)] -enum DownloadExecutionResult { - Completed, - Paused, - Cancelled, - Preempted, - Retryable(String), - Failed(String), -} - -enum ProviderAcquireResult { - Acquired(Option), - Paused, - Cancelled, - Preempted, - /// A live capture waited for capacity until its broadcast window closed. - WindowClosed, -} - -fn recording_execution_download( - app_config: &AppConfig, - task: &RecordingTask, - provider_handle: Option<&tuliprox_core::model::ProviderHandle>, -) -> Result { - let source = &task.recording.source; - let virtual_id = source.virtual_id.parse::().map_err(|_| "Recording source virtual id invalid".to_string())?; - let url = build_stable_recording_url( - app_config, - &source.target_id, - &source.input_name, - virtual_id, - source.cluster, - provider_handle.map(|handle| handle.allocation_id), - ) - .ok_or_else(|| "Recording execution URL unavailable".to_string())?; - let mut execution = task.clone(); - execution.url = reqwest::Url::parse(&url).map_err(|_| "Recording execution URL invalid".to_string())?; - Ok(execution) -} - -fn classify_download_open_error(url: &reqwest::Url, err: &reqwest::Error) -> DownloadExecutionResult { - if is_retryable_error(err) { - DownloadExecutionResult::Retryable(format!("Error while opening url: {url} {err}")) - } else { - DownloadExecutionResult::Failed(format!("Error while opening url: {url} {err}")) - } -} - -fn classify_download_stream_io_error(file_path_str: &str, err: &std::io::Error) -> DownloadExecutionResult { - if retryable_transport_error_message(&err.to_string()) { - DownloadExecutionResult::Retryable(format!("Error while downloading file: {file_path_str} {err}")) - } else { - DownloadExecutionResult::Failed(format!("Error while downloading file: {file_path_str} {err}")) - } -} - -fn apply_download_retry_jitter(base_secs: u64, jitter_percent: u8) -> u64 { - let jitter_percent = i64::from(jitter_percent.min(95)); - if jitter_percent == 0 { - return base_secs.max(1); - } - let jitter_percent = fastrand::i64(-jitter_percent..=jitter_percent); - let base_i64 = i64::try_from(base_secs.max(1)).unwrap_or(i64::MAX); - let jitter_delta = base_i64.saturating_mul(jitter_percent).saturating_div(100); - let jittered = base_i64.saturating_add(jitter_delta); - u64::try_from(jittered.max(1)).unwrap_or(1) -} - -#[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation, clippy::cast_sign_loss)] -fn compute_download_retry_backoff_secs(attempts: u8, download_cfg: &RecordingConfig) -> u64 { - let exponent = i32::from(attempts.saturating_sub(1)); - let scaled_secs = - (download_cfg.retry_backoff_initial_secs as f64) * download_cfg.retry_backoff_multiplier.powi(exponent); - let clamped_secs = - scaled_secs.clamp(download_cfg.retry_backoff_initial_secs as f64, download_cfg.retry_backoff_max_secs as f64); - let base_secs = clamped_secs.round() as u64; - apply_download_retry_jitter(base_secs, download_cfg.retry_backoff_jitter_percent) -} - -fn background_download_should_wait( - priority: i8, - capacities: &[(Arc, usize, usize)], - download_cfg: &RecordingConfig, -) -> bool { - if priority <= 0 || capacities.is_empty() { - return false; - } - - let background_limit = usize::from(download_cfg.max_background_per_provider); - let reserve_slots = usize::from(download_cfg.reserve_slots_for_users); - - let blocked_by_background_limit = - background_limit > 0 && capacities.iter().all(|(_, current, _)| *current >= background_limit); - let blocked_by_reserved_slots = reserve_slots > 0 - && capacities.iter().all(|(_, current, max)| *max > 0 && current.saturating_add(reserve_slots) >= *max); - - blocked_by_background_limit || blocked_by_reserved_slots -} - -fn capacities_have_free_slot(capacities: &[(Arc, usize, usize)]) -> bool { - capacities.iter().any(|(_, current, max)| *max == 0 || current < max) -} - -async fn active_download_snapshot_for_worker( - active: &RwLock>, - worker_uuid: &str, -) -> Option { - active.read().await.iter().find(|task| task.uuid == worker_uuid).cloned() -} - -async fn update_active_download_for_worker(active: &RwLock>, worker_uuid: &str, update: F) -> bool -where - F: FnOnce(&mut RecordingTask) -> bool, -{ - let mut active = active.write().await; - let Some(task) = active.iter_mut().find(|task| task.uuid == worker_uuid) else { - return false; - }; - update(task) -} - -/// Publish a queue change. Sessions answer it by pulling an owner-filtered -/// snapshot; no task data is broadcast globally, so a session can never see -/// another user's recording. -fn publish_recording_change(event_manager: &E) { event_manager.emit(EventMessage::RecordingChanged); } - -/// Announce that a running recording has grown. -/// -/// A separate event from [`publish_recording_change`]: a capture can produce -/// hundreds of these a second, and a session throttles them. A state -/// transition must never be throttled, so it does not come through here. -fn publish_recording_progress(event_manager: &E) { event_manager.emit(EventMessage::RecordingProgress); } - -fn broadcast_worker_mutation( - event_manager: &E, - result: Result, - action: &str, -) -> Result { - match result { - Ok(true) => { - publish_recording_change(event_manager); - Ok(true) - } - Ok(false) => Ok(false), - Err(err) => Err(QueueMutationError::new(format!("{action}: {err}"))), - } -} - -fn broadcast_required_worker_mutation( - event_manager: &E, - result: Result, - action: &str, -) -> Result<(), QueueMutationError> { - if broadcast_worker_mutation(event_manager, result, action)? { - Ok(()) - } else { - Err(QueueMutationError::new(format!("{action}: active task changed"))) - } -} - -async fn refresh_recording_progress( - active: &RwLock>, - worker_uuid: &str, - file_path: &std::path::Path, - event_manager: &E, -) { - let current_size = tokio::fs::metadata(file_path).await.map_or(0, |metadata| metadata.len()); - let changed = update_active_download_for_worker(active, worker_uuid, |task| { - if task.kind == RecordingKind::Live && task.size != current_size { - task.size = current_size; - true - } else { - false - } - }) - .await; - if changed { - publish_recording_progress(event_manager); - } -} - -/// The validators a later resume can be checked against. -/// -/// A weak `ETag` is discarded: it only promises semantic equivalence, so it -/// cannot prove the bytes on the far side of an interruption are the same -/// ones. `Last-Modified` is the fallback, and is only meaningful when no -/// strong tag was offered. -fn capture_resume_validators(response: &reqwest::Response) -> (Option, Option) { - let headers = response.headers(); - let etag = headers - .get(reqwest::header::ETAG) - .and_then(|value| value.to_str().ok()) - .and_then(crate::http_transfer::strong_etag) - .map(str::to_owned); - if etag.is_some() { - return (etag, None); - } - let last_modified = headers - .get(reqwest::header::LAST_MODIFIED) - .and_then(|value| value.to_str().ok()) - .filter(|value| !value.trim().is_empty()) - .map(str::to_owned); - (None, last_modified) -} - -async fn send_download_request( - client: &reqwest::Client, - upstream_client: Option<&reqwest::Client>, - url: &reqwest::Url, - offset: u64, - control_signal: &RwLock, - control_notify: &Notify, -) -> Result { - if let Some(result) = handle_download_control_without_writer(current_download_control(control_signal)) { - return Err(result); - } - - let send = async { - let mut destination = url.clone(); - let mut upstream_origin = None; - for redirects in 0..=10 { - let hop_client = - if destination.origin() == url.origin() { client } else { upstream_client.unwrap_or(client) }; - // Resume offsets refer to bytes on disk, so every hop must preserve encoding. - let mut request = hop_client - .get(destination.clone()) - .header(reqwest::header::USER_AGENT, shared::model::RECORDING_STREAM_USER_AGENT) - .header(reqwest::header::ACCEPT_ENCODING, "identity"); - if offset > 0 { - request = request.header(reqwest::header::RANGE, format!("bytes={offset}-")); - } - if destination.origin() != url.origin() { - let trusted_origin = upstream_origin.get_or_insert_with(|| destination.origin()); - if upstream_client.is_none() || destination.origin() != *trusted_origin { - // Listener credentials stay local; provider credentials stay - // on the first upstream origin throughout the redirect chain. - request = request.header(reqwest::header::AUTHORIZATION, "").header(reqwest::header::COOKIE, ""); - } - } - let response = request.send().await.map_err(|error| classify_download_open_error(&destination, &error))?; - if !matches!(response.status().as_u16(), 301 | 302 | 303 | 307 | 308) { - return Ok(response); - } - let Some(location) = response.headers().get(reqwest::header::LOCATION) else { - return Ok(response); - }; - let next = location - .to_str() - .ok() - .and_then(|value| response.url().join(value).ok()) - .filter(|value| matches!(value.scheme(), "http" | "https")); - let Some(next) = next else { - return Err(DownloadExecutionResult::Failed("Invalid recording redirect location".to_string())); - }; - if redirects == 10 { - return Err(DownloadExecutionResult::Failed("Recording redirect limit exceeded".to_string())); - } - destination = next; - } - Err(DownloadExecutionResult::Failed("Recording redirect limit exceeded".to_string())) - }; - tokio::pin!(send); - loop { - tokio::select! { - biased; - () = control_notify.notified() => { - if let Some(result) = handle_download_control_without_writer(*control_signal.read().await) { - return Err(result); - } - } - response = &mut send => return response, - } - } -} - -/// Wait for a provider slot, giving up when a live window closes. -/// -/// Waiting past `scheduled_end` cannot produce the recording that was asked -/// for: the programme has finished. Only live work has a deadline; a transfer -/// waits as long as it takes. -async fn wait_for_provider_slot( - download_queue: &RecordingQueue, - input_name: &Arc, - priority: i8, - control_signal: &RwLock, - control_notify: &Notify, - deadline: Option, -) -> Option { - let waiting = - download_queue.slot_waiters.wait(Some(Arc::clone(input_name)), priority, control_signal, control_notify); - match deadline { - Some(deadline) => time::timeout_at(deadline, waiting).await.ok(), - None => Some(waiting.await), - } -} - -/// What a control signal observed while acquiring a slot means; `None` keeps -/// trying. -fn acquire_result_for_control(control: RecordingControl) -> Option { - match control { - RecordingControl::Cancel => Some(ProviderAcquireResult::Cancelled), - RecordingControl::Pause => Some(ProviderAcquireResult::Paused), - RecordingControl::Restart => Some(ProviderAcquireResult::Preempted), - RecordingControl::None => None, - } -} - -/// What the end of a provider-slot wait means; `None` (signalled) tries to -/// acquire again. A wait without an outcome ran into the live window's end. -fn acquire_result_after_wait(outcome: Option) -> Option { - match outcome { - None => Some(ProviderAcquireResult::WindowClosed), - Some(RecordingWaitOutcome::Signalled) => None, - Some(RecordingWaitOutcome::Paused) => Some(ProviderAcquireResult::Paused), - Some(RecordingWaitOutcome::Cancelled) => Some(ProviderAcquireResult::Cancelled), - Some(RecordingWaitOutcome::Restarted) => Some(ProviderAcquireResult::Preempted), - } -} - -fn http_transfer_path(task: &RecordingTask) -> std::path::PathBuf { - if task.kind.is_resumable() { - recording_partial_path(&task.file_path) - } else { - task.file_path.clone() - } -} - -/// Publish a completed transfer at its final path. -/// -/// Idempotent: linking is the only step that is not naturally repeatable. A -/// crash between the link and the partial's removal leaves the final file in -/// place, and finalizing again accepts it instead of failing the recording. -async fn finalize_http_transfer(final_path: &std::path::Path, transfer_path: &std::path::Path) -> std::io::Result<()> { - if transfer_path == final_path { - return Ok(()); - } - fs::File::open(transfer_path).await?.sync_all().await?; - match fs::hard_link(transfer_path, final_path).await { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { - // Path reservation gives this recording sole claim on `final_path`, - // so something already there is this task's own earlier link. A - // size mismatch means that reasoning does not hold; say so rather - // than publish bytes that were never checked. - let published = fs::metadata(final_path).await?.len(); - let staged = fs::metadata(transfer_path).await?.len(); - if published != staged { - return Err(std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - format!( - "{} already exists with {published} bytes, but the transfer staged {staged}", - final_path.display() - ), - )); - } - } - Err(error) => return Err(error), - } - if let Err(error) = fs::remove_file(transfer_path).await { - warn!("Finalized {} but could not remove partial {}: {error}", final_path.display(), transfer_path.display()); - } - Ok(()) -} - -#[allow(clippy::too_many_lines, clippy::too_many_arguments)] -async fn download_file( - active: Arc>>, - file_download: RecordingTask, - client: &reqwest::Client, - upstream_client: Option<&reqwest::Client>, - control_signal: Arc>, - control_notify: Arc, - event_manager: Option<&E>, - quota_gate: Option>, -) -> DownloadExecutionResult { - let worker_uuid = file_download.uuid.as_str(); - let url = file_download.url.clone(); - let file_path = http_transfer_path(&file_download); - let existing_size = tokio::fs::metadata(&file_path).await.map_or(0, |metadata| metadata.len()); - let response_result = - send_download_request(client, upstream_client, &url, existing_size, &control_signal, &control_notify).await; - - match response_result { - Ok(response) => { - if existing_size > 0 { - // The validators were captured when the first byte was written. - // Without them a provider that replaces the file between an - // interruption and the resume returns a well-formed 206 whose - // bytes belong to a different resource, and they are appended - // to the old partial. - let validator = ResumeValidator { - expected_offset: existing_size, - expected_total: file_download.total_size, - expected_etag: file_download.recording.resume_etag.clone(), - expected_last_modified: file_download.recording.resume_last_modified.clone(), - }; - match validate_resume_response(&ResponseSnapshot::from_response(&response), &validator) { - Ok(()) => {} - Err(ResumeValidationError::Unsatisfiable { complete: true }) => { - return match finalize_http_transfer(&file_download.file_path, &file_path).await { - Ok(()) => DownloadExecutionResult::Completed, - Err(error) => DownloadExecutionResult::Failed(format!( - "Could not finalize {}: {error}", - file_download.file_path.display() - )), - }; - } - Err(ResumeValidationError::IgnoredRange) => { - return DownloadExecutionResult::Failed(RANGE_UNSUPPORTED_ERROR.to_string()); - } - Err(error) => { - return DownloadExecutionResult::Failed(format!( - "Cannot resume download at byte {existing_size} for {url}: {error}; partial file preserved" - )); - } - } - } - let status = response.status(); - if !status.is_success() && status != reqwest::StatusCode::PARTIAL_CONTENT { - if is_retryable_status(status) { - return DownloadExecutionResult::Retryable(format!( - "Download request failed for {url} with transient HTTP {status}" - )); - } - return DownloadExecutionResult::Failed(format!( - "Download request failed for {url} with HTTP {status}" - )); - } - let is_resume = status == reqwest::StatusCode::PARTIAL_CONTENT; - - let total_size = compute_total_size(&response, existing_size); - let captured = capture_resume_validators(&response); - - if total_size.is_some() || existing_size == 0 { - update_active_download_for_worker(&active, worker_uuid, |download| { - if let Some(total) = total_size { - download.total_size = Some(total); - } - // Only a fresh transfer may set the validators; a resume - // must keep the ones its partial was written against. - if existing_size == 0 { - download.recording.resume_etag.clone_from(&captured.0); - download.recording.resume_last_modified.clone_from(&captured.1); - } - true - }) - .await; - } - - // The size is known from here on, or known to be unknown. - let byte_cap = match quota_gate { - Some(gate) => match gate.admit_size(worker_uuid, total_size, existing_size).await { - Ok(admitted) => { - if admitted.siblings_refused { - if let Some(event_manager) = event_manager { - publish_recording_change(event_manager); - } - } - admitted.byte_cap - } - Err(reason) => return DownloadExecutionResult::Failed(reason), - }, - None => None, - }; - - match fs::create_dir_all(&file_download.file_dir).await { - Ok(()) => { - let mut open_options = tokio::fs::OpenOptions::new(); - let file_mode = if existing_size > 0 && is_resume { - open_options.append(true) - } else { - open_options.write(true).create(true).truncate(true) - }; - - if let Some(file_path_str) = file_path.to_str() { - info!("{} {}", if is_resume { "Resuming" } else { "Downloading" }, file_path_str); - match file_mode.open(&file_path).await { - Ok(file) => { - let mut buf_writer = async_file_writer(file); - let mut downloaded: u64 = if is_resume { existing_size } else { 0 }; - let mut stream = response.bytes_stream(); - let mut write_counter = 0; - let mut saw_first_chunk = existing_size > 0; - let mut last_progress_log_at = Instant::now(); - let mut last_progress_logged_bytes = downloaded; - let mut last_snapshot_update_at = Instant::now(); - let mut last_snapshot_update_bytes = downloaded; - let mut control_poll = time::interval(DOWNLOAD_CONTROL_POLL_INTERVAL); - control_poll.set_missed_tick_behavior(time::MissedTickBehavior::Skip); - control_poll.tick().await; - loop { - let next_item = tokio::select! { - biased; - () = control_notify.notified() => { - if let Some(result) = - handle_download_control(*control_signal.read().await, &mut buf_writer) - .await - { - return result; - } - continue; - } - _ = control_poll.tick() => { - if let Some(result) = handle_download_control( - current_download_control(&control_signal), - &mut buf_writer, - ) - .await - { - return result; - } - continue; - } - next_item = stream.try_next() => next_item.map_err(to_io_error), - }; - - match next_item { - Ok(item) => { - if let Some(chunk) = item { - match buf_writer.write_all(&chunk).await { - Ok(()) => { - write_counter += chunk.len(); - if write_counter >= IO_BUFFER_SIZE { - if let Err(err) = buf_writer.flush().await { - return DownloadExecutionResult::Failed( - err.to_string(), - ); - } - write_counter = 0; - } - - downloaded += chunk.len() as u64; - if byte_cap.is_some_and(|cap| downloaded > cap) { - // Unknown size, so the quota is enforced as - // the bytes arrive. - let _ = buf_writer.flush().await; - let _ = buf_writer.shutdown().await; - return DownloadExecutionResult::Failed( - QUOTA_EXCEEDED_DURING_TRANSFER.to_string(), - ); - } - if saw_first_chunk { - let now = Instant::now(); - let should_log_progress = now - .duration_since(last_progress_log_at) - >= DOWNLOAD_PROGRESS_LOG_INTERVAL - || downloaded - .saturating_sub(last_progress_logged_bytes) - >= DOWNLOAD_PROGRESS_LOG_BYTES; - if should_log_progress { - match total_size { - Some(total) if total > 0 => { - let percent = downloaded - .saturating_mul(100) - .checked_div(total) - .unwrap_or(0) - .min(100); - debug!( - "Download progress for {file_path_str}: {}MB / {}MB ({}%)", - bytes_to_megabytes(downloaded), - bytes_to_megabytes(total), - percent - ); - } - _ => { - debug!( - "Download progress for {file_path_str}: {}MB received", - bytes_to_megabytes(downloaded) - ); - } - } - last_progress_log_at = now; - last_progress_logged_bytes = downloaded; - } - } else { - saw_first_chunk = true; - info!( - "Receiving download data for {file_path_str}: {}MB received", - bytes_to_megabytes(downloaded) - ); - last_progress_log_at = Instant::now(); - last_progress_logged_bytes = downloaded; - } - let should_update_snapshot = (last_snapshot_update_bytes == 0 - && downloaded > 0) - || downloaded.saturating_sub(last_snapshot_update_bytes) - >= DOWNLOAD_SNAPSHOT_UPDATE_BYTES - || Instant::now().duration_since(last_snapshot_update_at) - >= DOWNLOAD_SNAPSHOT_UPDATE_INTERVAL; - if should_update_snapshot { - let changed = update_active_download_for_worker( - &active, - worker_uuid, - |download| { - download.size = downloaded; - true - }, - ) - .await; - if changed { - if let Some(event_manager) = event_manager { - publish_recording_progress(event_manager); - } - } - last_snapshot_update_at = Instant::now(); - last_snapshot_update_bytes = downloaded; - } - } - Err(err) => { - return DownloadExecutionResult::Failed(format!( - "Error while writing to file: {file_path_str} {err}" - )); - } - } - } else { - let megabytes = bytes_to_megabytes(downloaded); - info!("Downloaded {file_path_str}, filesize: {megabytes}MB"); - update_active_download_for_worker(&active, worker_uuid, |download| { - download.size = downloaded; - true - }) - .await; - if let Err(err) = buf_writer.flush().await { - return DownloadExecutionResult::Failed(err.to_string()); - } - if let Err(err) = buf_writer.shutdown().await { - return DownloadExecutionResult::Failed(err.to_string()); - } - return match finalize_http_transfer( - &file_download.file_path, - &file_path, - ) - .await - { - Ok(()) => DownloadExecutionResult::Completed, - Err(error) => DownloadExecutionResult::Failed(format!( - "Could not finalize {}: {error}", - file_download.file_path.display() - )), - }; - } - } - Err(err) => return classify_download_stream_io_error(file_path_str, &err), - } - } - } - Err(err) => DownloadExecutionResult::Failed(format!( - "Error while opening file: {file_path_str} {err}" - )), - } - } else { - DownloadExecutionResult::Failed("Error file-download file-path unknown".to_string()) - } - } - Err(err) => DownloadExecutionResult::Failed(format!( - "Error while creating directory for file: {} {}", - file_download.file_dir.to_str().unwrap_or("?"), - err - )), - } - } - Err(result) => result, - } -} - -fn current_download_control(control_signal: &RwLock) -> RecordingControl { - control_signal.try_read().map_or(RecordingControl::None, |control| *control) -} - -fn should_exit_worker_after_preempt(control: RecordingControl) -> bool { control == RecordingControl::Restart } - -/// Close the writer for a pause, cancel or restart, so everything received -/// so far is on disk before the worker reports the outcome. -async fn handle_download_control(control: RecordingControl, buf_writer: &mut W) -> Option -where - W: AsyncWrite + Unpin, -{ - let outcome = handle_download_control_without_writer(control)?; - let closed = async { - buf_writer.flush().await?; - buf_writer.shutdown().await - }; - if let Err(err) = closed.await { - return Some(DownloadExecutionResult::Failed(err.to_string())); - } - Some(outcome) -} - -fn handle_download_control_without_writer(control: RecordingControl) -> Option { - match control { - RecordingControl::Pause => Some(DownloadExecutionResult::Paused), - RecordingControl::Cancel => Some(DownloadExecutionResult::Cancelled), - RecordingControl::Restart => Some(DownloadExecutionResult::Preempted), - RecordingControl::None => None, - } -} - -fn recording_deadline_instant(task: &RecordingTask) -> Option { - if task.kind != RecordingKind::Live { - return None; - } - let (start_at, duration_secs) = task.scheduled_start().zip(task.scheduled_duration_secs())?; - let deadline_ts = start_at.saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX)); - let now_ts = chrono::Utc::now().timestamp(); - if now_ts >= deadline_ts { - return Some(Instant::now()); - } - let remaining_secs = u64::try_from(deadline_ts.saturating_sub(now_ts)).ok()?; - Some(Instant::now() + Duration::from_secs(remaining_secs)) -} - -async fn set_active_download_state( - download_queue: &RecordingQueue, - uuid: &str, - state: RecordingTaskState, - error: Option, - paused: bool, -) -> Result { - Ok(mutate_optional(download_queue, |candidate| { - let Some(task) = candidate.active.iter_mut().find(|active| active.uuid == uuid) else { - return Ok(None); - }; - if task.state == state && task.error == error && task.paused == paused && !task.finished { - return Ok(None); - } - task.state = state; - task.error = error; - task.paused = paused; - task.finished = false; - Ok(Some(true)) - }) - .await? - .unwrap_or(false)) -} - -async fn continue_after_pause(download_queue: &RecordingQueue, uuid: &str) -> bool { - download_queue.active.read().await.iter().any(|task| task.uuid == uuid && !task.paused && !task.finished) -} - -async fn commit_acquired_download( - download_queue: &RecordingQueue, - uuid: &str, -) -> Result, QueueMutationError> { - mutate_optional(download_queue, |candidate| { - let Some(active) = candidate.active.iter_mut().find(|active| active.uuid == uuid) else { - return Ok(None); - }; - active.state = RecordingTaskState::Running; - active.error = None; - active.paused = false; - active.finished = false; - let notification = mark_recording_metadata_notification(&mut active.recording, LifecycleEvent::Started, None); - Ok(Some(notification)) - }) - .await -} - -fn mark_recording_metadata_notification( - meta: &mut RecordingMetadata, - event: LifecycleEvent, - failure_reason: Option, -) -> RecordingNotificationPlan { - let is_admin_owner = meta.owner.user_id().is_builtin_admin(); - match decide(meta, event, chrono::Utc::now().timestamp(), is_admin_owner, failure_reason) { - DispatchDecision::PersistAndDeliver { payload, kind, attempted_at } => { - meta.notification_markers.push(build_marker(kind.clone(), attempted_at)); - RecordingNotificationPlan { - message: Some(MessageContent::RecordingLifecycle(message_for(event, &payload))), - } - } - DispatchDecision::AlreadyDelivered { .. } | DispatchDecision::Suppressed { .. } => { - RecordingNotificationPlan::empty() - } - } -} - -struct RecordingNotificationPlan { - message: Option, -} - -impl RecordingNotificationPlan { - fn empty() -> Self { Self { message: None } } -} - -/// Hand a lifecycle notification off for delivery once its marker has -/// been persisted. -/// -/// The marker is written inside the queue-mutation boundary, so it is -/// durable before this runs; delivery must be durable too. The -/// notification outbox owns that: it persists the entry, retries per -/// channel with backoff, and dead-letters what it cannot deliver. -/// -/// A direct send is the fallback for paths that run before the supervisor -/// installs the outbox (notably unit tests), and for an outbox that refuses -/// the entry. -fn spawn_recording_notification_after_persist( - app_config: &Arc, - client: &reqwest::Client, - plan: RecordingNotificationPlan, - persisted: bool, -) { - if !persisted { - return; - } - let Some(message) = plan.message else { - return; - }; - let event = tuliprox_core::model::NotificationEvent::from_content(&message); - // A full or closed outbox hands the event back; fall through to the - // direct send rather than dropping it outright. - let event = match crate::recording::recording_supervisor::notification_outbox() { - Some(outbox) => match outbox.enqueue(event) { - None => return, - Some(rejected) => rejected, - }, - None => event, - }; - let app_config = Arc::clone(app_config); - let client = client.clone(); - tokio::spawn(async move { - tuliprox_messaging::send_event(&app_config, &client, event).await; - }); -} - -/// Take the active task out of the candidate, but only when it is still the -/// one this worker is executing. -fn take_active(candidate: &mut PersistedRecordingQueue, uuid: &str) -> Option { - candidate.active.iter().position(|active| active.uuid == uuid).map(|index| candidate.active.remove(index)) -} - -async fn requeue_active_download_for_retry( - download_queue: &RecordingQueue, - uuid: &str, - promote: bool, -) -> Result { - Ok(mutate_optional(download_queue, |candidate| { - let Some(mut download) = take_active(candidate, uuid) else { - return Ok(None); - }; - download.finished = false; - download.paused = false; - download.error = None; - download.state = RecordingTaskState::Queued; - download.next_retry_at = None; - candidate.queue.insert(0, download); - if promote { - crate::recording::recording_queue::promote_from_queue(candidate); - } - Ok(Some(true)) - }) - .await? - .unwrap_or(false)) -} - -async fn requeue_active_download_for_capacity_wait( - download_queue: &RecordingQueue, - uuid: &str, - reason: &str, - promote: bool, - consumed_control: Option, -) -> Result { - let mutation = |candidate: &mut PersistedRecordingQueue| { - let Some(mut download) = take_active(candidate, uuid) else { - return Ok(None); - }; - download.finished = false; - download.paused = false; - download.error = Some(reason.to_string()); - download.state = RecordingTaskState::WaitingForCapacity; - download.next_retry_at = None; - candidate.queue.insert(0, download); - if promote { - crate::recording::recording_queue::promote_from_queue(candidate); - } - Ok(Some(true)) - }; - let result = if let Some(control) = consumed_control { - download_queue.mutate_optional_and_clear_control(uuid, control, mutation).await? - } else { - mutate_optional(download_queue, mutation).await? - }; - Ok(result.unwrap_or(false)) -} - -/// Write the sidecar for the recording that just finished. -/// -/// Best effort by design: the recording is on disk and committed either way, -/// and refusing to complete a transfer because a descriptive file could not be -/// written would trade a real recording for a diagnostic. -async fn write_completion_sidecar(download_queue: &RecordingQueue, uuid: &str, measured_bytes: u64) { - let Some(task) = download_queue.active.read().await.iter().find(|task| task.uuid == uuid).cloned() else { - return; - }; - let Some(relative_path) = task.recording.relative_path.clone() else { - return; - }; - let persisted = RecordingQueue::to_persisted(&task); - let sidecar = recording_sidecar::RecordingSidecar { - materialization_id: tuliprox_repository::recording_repository::materialization_id_for(&persisted), - media_identity: persisted.media_identity, - kind: task.kind, - relative_path, - size_bytes: measured_bytes, - completed_at: chrono::Utc::now().timestamp(), - }; - if let Err(error) = recording_sidecar::write_sidecar(&task.file_path, &sidecar).await { - warn!("Could not write the sidecar for {}: {error}", task.file_path.display()); - } -} const LIVE_CAPACITY_WINDOW_CLOSED: &str = "No provider capacity became available before the recording window closed"; -/// Commit a terminal failure for the active task and move on. -async fn fail_active_download( - download_queue: &RecordingQueue, - uuid: &str, - reason: &str, -) -> Result { - Ok(mutate_optional(download_queue, |candidate| { - let Some(mut failed) = take_active(candidate, uuid) else { - return Ok(None); - }; - failed.finished = true; - failed.paused = false; - failed.next_retry_at = None; - failed.state = RecordingTaskState::Failed; - failed.error = Some(reason.to_string()); - failed.recording.reserved_bytes = 0; - candidate.finished.push(failed); - crate::recording::recording_queue::promote_from_queue(candidate); - Ok(Some(true)) - }) - .await? - .unwrap_or(false)) -} - -async fn promote_ready_downloads(download_queue: &RecordingQueue) -> Result { - if !download_queue.has_promotable_queued().await { - return Ok(false); - } - Ok(mutate_optional(download_queue, |candidate| { - let queued = candidate.queue.len(); - while crate::recording::recording_queue::promote_from_queue(candidate).is_some() {} - Ok((candidate.queue.len() != queued).then_some(true)) - }) - .await? - .unwrap_or(false)) -} - -/// Move this worker's task to `finished` and promote the next runnable entry. -/// -/// With `waiting_quota`, the task finished a file other entries may be -/// waiting to attach to. Its measured size is checked against each of their -/// quotas before promotion attaches them: a transfer of unknown size cannot -/// be checked when it starts, and attaching charges the whole file. -async fn finish_active_and_promote( - download_queue: &RecordingQueue, - uuid: &str, - waiting_quota: Option<&crate::recording::recording_quota::QuotaLimits>, - finish: F, -) -> Result, QueueMutationError> -where - F: FnOnce(&mut PersistedRecordingTask) -> RecordingNotificationPlan, -{ - download_queue - .mutate_optional_and_clear_control(uuid, RecordingControl::Cancel, |candidate| { - let Some(mut active) = take_active(candidate, uuid) else { - return Ok(None); - }; - let notification = finish(&mut active); - let media = active.media_identity.clone(); - let measured = active.recording.measured_bytes; - candidate.finished.push(active); - if let Some(limits) = waiting_quota { - let _ = charge_waiting_siblings(candidate, &media, measured, limits); - } - crate::recording::recording_queue::promote_from_queue(candidate); - Ok(Some(notification)) - }) - .await -} - -async fn cancel_active_and_promote(download_queue: &RecordingQueue, uuid: &str) -> Result { - Ok(download_queue - .mutate_optional_and_clear_control(uuid, RecordingControl::Cancel, |candidate| { - let Some(mut active) = take_active(candidate, uuid) else { - return Ok(None); - }; - active.finished = true; - active.paused = false; - active.next_retry_at = None; - active.error.get_or_insert_with(|| "Cancelled by user".to_string()); - active.state = RecordingTaskState::Cancelled; - // Nothing more will be written, so the space stops being spoken for. - active.recording.reserved_bytes = 0; - candidate.finished.push(active); - crate::recording::recording_queue::promote_from_queue(candidate); - Ok(Some(true)) - }) - .await? - .unwrap_or(false)) -} - -enum RetryCommit { - Waiting { delay_secs: u64, attempts: u8 }, - Failed(RecordingNotificationPlan), -} - -async fn prepare_active_retry( - download_queue: &RecordingQueue, - uuid: &str, - download_cfg: &RecordingConfig, - cause: &str, -) -> Result, QueueMutationError> { - mutate_optional(download_queue, |candidate| { - let Some(active) = candidate.active.iter_mut().find(|active| active.uuid == uuid) else { - return Ok(None); - }; - // A live broadcast cannot be retried: whatever played during the - // backoff is gone, so a second attempt records a different part of the - // programme and calls it the same recording. `RetryWaiting` is not a - // state a live capture can legally be in either. - let retryable_kind = active.kind.is_resumable(); - active.retry_attempts = active.retry_attempts.saturating_add(1); - let attempts = active.retry_attempts; - if !retryable_kind || attempts > download_cfg.retry_max_attempts { - let Some(mut failed) = take_active(candidate, uuid) else { - return Ok(None); - }; - let error = if retryable_kind { - format!("Retry limit reached after {} attempts: {cause}", download_cfg.retry_max_attempts) - } else { - format!("Live recording failed; restarting a live capture is not supported: {cause}") - }; - failed.finished = true; - failed.paused = false; - failed.next_retry_at = None; - failed.state = RecordingTaskState::Failed; - failed.error = Some(error.clone()); - failed.recording.reserved_bytes = 0; - let notification = - mark_recording_metadata_notification(&mut failed.recording, LifecycleEvent::Failed, Some(error)); - candidate.finished.push(failed); - crate::recording::recording_queue::promote_from_queue(candidate); - return Ok(Some(RetryCommit::Failed(notification))); - } - - let delay_secs = compute_download_retry_backoff_secs(attempts, download_cfg); - let next_retry_at = - chrono::Utc::now().timestamp().saturating_add(i64::try_from(delay_secs).unwrap_or(i64::MAX)); - active.next_retry_at = Some(next_retry_at); - active.state = RecordingTaskState::RetryWaiting; - active.paused = false; - active.finished = false; - active.error = Some(format!( - "Retrying after transient failure in {delay_secs}s (attempt {attempts}/{}): {cause}", - download_cfg.retry_max_attempts - )); - Ok(Some(RetryCommit::Waiting { delay_secs, attempts })) - }) - .await -} - const DOWNLOAD_PREEMPTED_REASON: &str = "Preempted by higher-priority foreground stream"; + const RECORDING_PREEMPTED_REASON: &str = "Recording preempted by higher-priority foreground stream; waiting to resume within the remaining window"; -fn preemption_reason_for(download: &RecordingTask) -> &'static str { - match download.kind { - RecordingKind::Vod | RecordingKind::Series => DOWNLOAD_PREEMPTED_REASON, - RecordingKind::Live => RECORDING_PREEMPTED_REASON, - } -} - pub(crate) const QUOTA_EXCEEDED_DURING_TRANSFER: &str = "Quota exceeded: the recording is larger than the quota that is left"; + const SIBLING_QUOTA_EXCEEDED: &str = "Quota exceeded: the recording is larger than this entry's quota allows"; -/// What the size check at the start of a transfer decided. -struct SizeAdmission { - /// Byte count the transfer may not exceed when its total is unknown. - byte_cap: Option, - /// Waiting entries for the same media were refused for their own quota. - siblings_refused: bool, -} - -/// Quota and disk admission once a transfer knows how large it is. -/// -/// A VOD or series request is admitted before its size is known, so it -/// reserves nothing up front. The first response carries the size, and this -/// is the first point where the charge can be checked: against the -/// transfer's own pool, against every entry waiting to attach to the same -/// file, and against the disk. -struct QuotaGate<'a> { - queue: &'a RecordingQueue, - app_config: &'a AppConfig, -} - -impl QuotaGate<'_> { - async fn admit_size(&self, uuid: &str, total: Option, on_disk: u64) -> Result { - let config = self.app_config.config.load(); - let Some(recording_cfg) = config.recording() else { - return Ok(SizeAdmission { byte_cap: None, siblings_refused: false }); - }; - let limits = crate::recording::recording_service::quota_limits_from_config(recording_cfg.quota.as_ref()); - // Measured before the mutation: it is a syscall. - let free = crate::recording::recording_disk::free_bytes_for(Path::new(&recording_cfg.directory)); - let safety = recording_cfg.disk.as_ref().and_then(|disk| disk.safety_bytes).unwrap_or(0); - - let decided = mutate_optional(self.queue, |candidate| { - let Some(subject) = candidate.active.iter().find(|active| active.uuid == uuid) else { - return Ok(None); - }; - let pool = crate::recording::recording_quota::quota_pool_for_task(subject); - let media = subject.media_identity.clone(); - let used = used_by_others(candidate, uuid, &pool); - let Some(total) = total else { - let byte_cap = crate::recording::recording_quota::limit_for_pool(&pool, &limits) - .map(|limit| limit.saturating_sub(used)); - return Ok(Some(Ok(SizeAdmission { byte_cap, siblings_refused: false }))); - }; - if over_limit(&pool, used, total, &limits) { - return Ok(Some(Err(QUOTA_EXCEEDED_DURING_TRANSFER.to_string()))); - } - if let Some(free) = free { - let running = crate::recording::recording_disk::active_disk_reservations( - all_tasks(candidate).filter(|task| task.uuid != uuid), - ); - if matches!( - crate::recording::recording_disk::would_fit_on_disk( - free, - safety, - running, - total.saturating_sub(on_disk) - ), - crate::recording::recording_disk::DiskAdmission::Insufficient { .. } - ) { - return Ok(Some(Err(DISK_GONE_BEFORE_START.to_string()))); - } - } - if let Some(active) = candidate.active.iter_mut().find(|task| task.uuid == uuid) { - active.recording.reserved_bytes = active.recording.reserved_bytes.max(total); - } - let siblings_refused = charge_waiting_siblings(candidate, &media, total, &limits); - Ok(Some(Ok(SizeAdmission { byte_cap: None, siblings_refused }))) - }) - .await - .map_err(|err| format!("Could not record the transfer size: {err}"))?; - // The task left the active slot meanwhile; the worker finds out next. - decided.unwrap_or(Ok(SizeAdmission { byte_cap: None, siblings_refused: false })) - } -} - -fn all_tasks(candidate: &PersistedRecordingQueue) -> impl Iterator { - candidate - .queue - .iter() - .chain(candidate.scheduled.iter()) - .chain(candidate.active.iter()) - .chain(candidate.finished.iter()) -} - -fn used_by_others( - candidate: &PersistedRecordingQueue, - uuid: &str, - pool: &crate::recording::recording_quota::QuotaPool, -) -> u64 { - crate::recording::recording_quota::used_bytes_in_pool(all_tasks(candidate).filter(|task| task.uuid != uuid), pool) -} - -fn over_limit( - pool: &crate::recording::recording_quota::QuotaPool, - used: u64, - charge: u64, - limits: &crate::recording::recording_quota::QuotaLimits, -) -> bool { - matches!( - crate::recording::recording_quota::would_exceed(pool, used, charge, limits), - crate::recording::recording_quota::AdmissionOutcome::OverLimit { .. } - ) -} - -/// Charge every queued entry that will attach to this file with its size, -/// and refuse the ones whose own quota cannot take it. Returns whether any -/// entry was refused. -/// -/// They were admitted while the size was unknown. Attaching later copies the -/// whole file into their charge, so this is the last point where their quota -/// can still say no. -fn charge_waiting_siblings( - candidate: &mut PersistedRecordingQueue, - media: &str, - total: u64, - limits: &crate::recording::recording_quota::QuotaLimits, -) -> bool { - if media.is_empty() { - return false; - } - let mut refused = Vec::new(); - let waiting: Vec = - candidate.queue.iter().filter(|task| task.media_identity == media).map(|task| task.uuid.clone()).collect(); - for uuid in waiting { - let Some(sibling) = candidate.queue.iter().find(|task| task.uuid == uuid) else { - continue; - }; - let pool = crate::recording::recording_quota::quota_pool_for_task(sibling); - let used = used_by_others(candidate, &uuid, &pool); - if over_limit(&pool, used, total, limits) { - refused.push(uuid); - } else if let Some(sibling) = candidate.queue.iter_mut().find(|task| task.uuid == uuid) { - sibling.recording.reserved_bytes = sibling.recording.reserved_bytes.max(total); - } - } - for uuid in &refused { - if let Some(index) = candidate.queue.iter().position(|task| &task.uuid == uuid) { - let mut failed = candidate.queue.remove(index); - failed.finished = true; - failed.paused = false; - failed.next_retry_at = None; - failed.state = RecordingTaskState::Failed; - failed.error = Some(SIBLING_QUOTA_EXCEEDED.to_string()); - failed.recording.reserved_bytes = 0; - candidate.finished.push(failed); - } - } - !refused.is_empty() -} - pub(crate) const QUOTA_GONE_BEFORE_START: &str = "Quota was exhausted while this recording waited to start"; + pub(crate) const DISK_GONE_BEFORE_START: &str = "Disk space was exhausted while this recording waited to start"; -/// Re-run admission for the recording that is about to open its -/// destination, returning the reason it can no longer start. -/// -/// Admission happened when the request was accepted, which can be a long -/// time before this point: the recording may have waited for provider -/// capacity, sat through a retry backoff, or been scheduled hours ahead. -/// The quota and the free space it was admitted against are not the ones -/// it is about to consume. Without this the first sign of a full disk is -/// a write failure part-way through a recording. -/// -/// The subject is excluded from both sums and then added back as the -/// candidate charge, so it is not counted twice. -async fn refused_before_start( - download_queue: &Arc, - app_config: &AppConfig, - uuid: &str, -) -> Option<&'static str> { - let config = app_config.config.load(); - let recording_cfg = config.recording()?; - let (_, tasks) = download_queue.committed_snapshot().await; - let subject = tasks.iter().find(|task| task.uuid == uuid)?; - let charge = crate::recording::recording_quota::charge_for_task(subject); - let others = || tasks.iter().filter(|task| task.uuid != uuid); +#[cfg(test)] +mod tests; - let limits = crate::recording::recording_service::quota_limits_from_config(recording_cfg.quota.as_ref()); - let pool = crate::recording::recording_quota::quota_pool_for_task(subject); - let used = crate::recording::recording_quota::used_bytes_in_pool(others(), &pool); - if matches!( - crate::recording::recording_quota::would_exceed(&pool, used, charge, &limits), - crate::recording::recording_quota::AdmissionOutcome::OverLimit { .. } - ) { - return Some(QUOTA_GONE_BEFORE_START); - } - - // An unmeasurable root is not grounds to refuse, exactly as at admission. - let free = crate::recording::recording_disk::free_bytes_for(Path::new(&recording_cfg.directory))?; - let safety = recording_cfg.disk.as_ref().and_then(|disk| disk.safety_bytes).unwrap_or(0); - let active = crate::recording::recording_disk::active_disk_reservations(others()); - if matches!( - crate::recording::recording_disk::would_fit_on_disk(free, safety, active, charge), - crate::recording::recording_disk::DiskAdmission::Insufficient { .. } - ) { - return Some(DISK_GONE_BEFORE_START); - } - None -} - -pub async fn ensure_recording_worker_running( - cfg: &AppConfig, - download_cfg: &RecordingConfig, - download_queue: &Arc, - event_manager: &E, - capacity: &Arc, - recording_binary: &Path, -) -> Result<(), String> { - if promote_ready_downloads(download_queue).await.map_err(|err| err.to_string())? { - publish_recording_change(event_manager); - } - let ready: Vec<_> = download_queue - .active - .read() - .await - .iter() - .filter(|task| !task.paused && !task.finished) - .map(|task| task.uuid.clone()) - .collect(); - for uuid in ready { - start_recording_worker(cfg, download_cfg, download_queue, event_manager, capacity, recording_binary, uuid) - .await?; - } - Ok(()) -} - -#[allow(clippy::too_many_lines, clippy::too_many_arguments)] -async fn start_recording_worker( - cfg: &AppConfig, - download_cfg: &RecordingConfig, - download_queue: &Arc, - event_manager: &E, - capacity: &Arc, - recording_binary: &Path, - worker_uuid: String, -) -> Result<(), String> { - let Some(worker) = download_queue.claim_worker(&worker_uuid).await else { - return Ok(()); - }; - - if download_queue.active.read().await.iter().any(|task| task.uuid == worker_uuid) { - let config = cfg.config.load(); - let disabled_headers = cfg.get_disabled_headers(); - let upstream_headers = request::get_request_headers( - Some(&download_cfg.headers), - None, - disabled_headers.as_ref(), - config.default_user_agent.as_deref(), - ); - let dq = Arc::clone(download_queue); - let control_signal = Arc::clone(&worker.control_signal); - let control_notify = Arc::clone(&worker.control_notify); - let event_manager = event_manager.clone(); - let capacity = Arc::clone(capacity); - let download_cfg = download_cfg.clone(); - let recording_binary = recording_binary.to_path_buf(); - let app_config = Arc::new(cfg.clone()); - - // The listener authenticates through its recording URL and bypasses proxies. - // Provider headers stay on the upstream client; proxied listener requests - // receive them through the recording input configuration. - // Notifications retain the proxy without the recording headers. - let clients = ( - create_client(cfg).no_proxy().redirect(reqwest::redirect::Policy::none()).build(), - create_client(cfg).redirect(reqwest::redirect::Policy::none()).default_headers(upstream_headers).build(), - create_client(cfg).build(), - ); - if let (Ok(transfer_client), Ok(upstream_transfer_client), Ok(client)) = clients { - if let Some(active) = dq.active.read().await.iter().find(|task| task.uuid == worker_uuid) { - info!("Starting download worker for active download {} ({})", active.uuid, active.filename); - } - tokio::spawn(async move { - 'worker: loop { - // One read: the uuid, pause flag and live window must describe - // the same task. - let active_head = dq - .active - .read() - .await - .iter() - .find(|task| task.uuid == worker_uuid) - .map(|download| (download.paused, recording_deadline_instant(download))); - if let Some((paused, window_deadline)) = active_head { - if paused { - break; - } - - // Acquire a provider connection slot for this download. - // If the provider is at capacity, wait in the priority queue until signalled. - // Never proceeds without a slot when input_name is set — account bans otherwise. - let provider_acquire_result = { - let (input_name, priority) = - dq.active_scheduling_priority(&worker_uuid).await.unwrap_or((None, 0i8)); - // Only live work has a window deadline; a transfer waits as long as it takes. - if let Some(input_name) = input_name { - loop { - if let Some(stopped) = acquire_result_for_control(*control_signal.read().await) { - break stopped; - } - let handle = { - // Keep the policy check and allocation together so - // concurrent recording starts respect background limits. - let _capacity = dq.capacity_guard.lock().await; - let capacities = capacity.capacities_for_input(&input_name).await; - let live_config = app_config.config.load(); - let capacity_cfg = live_config.recording().unwrap_or(&download_cfg); - if background_download_should_wait(priority, &capacities, capacity_cfg) { - None - } else { - capacity.acquire(&input_name, priority).await - } - }; - if let Some(handle) = handle { - break ProviderAcquireResult::Acquired(Some(handle)); - } - if let Err(err) = broadcast_worker_mutation( - &event_manager, - set_active_download_state( - &dq, - &worker_uuid, - RecordingTaskState::WaitingForCapacity, - None, - false, - ) - .await, - "waiting-for-capacity state", - ) { - error!("Download worker commit failed: {err}"); - break 'worker; - } - // Wait for highest-priority signal — no sleep, no polling. - let waited = wait_for_provider_slot( - &dq, - &input_name, - priority, - control_signal.as_ref(), - control_notify.as_ref(), - window_deadline, - ) - .await; - if let Some(stopped) = acquire_result_after_wait(waited) { - break stopped; - } - } - } else { - ProviderAcquireResult::Acquired(None) - } - }; - - let provider_handle = match provider_acquire_result { - ProviderAcquireResult::Acquired(handle) => { - match commit_acquired_download(&dq, &worker_uuid).await { - Ok(Some(notification)) => { - spawn_recording_notification_after_persist( - &app_config, - &client, - notification, - true, - ); - publish_recording_change(&event_manager); - } - Ok(None) => { - capacity.release(handle).await; - error!("Download worker active task changed after provider acquire"); - break 'worker; - } - Err(err) => { - capacity.release(handle).await; - error!("Download worker commit failed after provider acquire: {err}"); - break 'worker; - } - } - handle - } - ProviderAcquireResult::WindowClosed => { - // No slot was ever acquired, so there is nothing - // to release and nothing was written. - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - fail_active_download(&dq, &worker_uuid, LIVE_CAPACITY_WINDOW_CLOSED).await, - "live capacity window closed", - ) { - error!("Download worker commit failed: {err}"); - break 'worker; - } - continue; - } - ProviderAcquireResult::Paused => { - break; - } - ProviderAcquireResult::Cancelled => { - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - cancel_active_and_promote(&dq, &worker_uuid).await, - "provider-wait cancellation", - ) { - error!("Download worker commit failed: {err}"); - break 'worker; - } - continue; - } - ProviderAcquireResult::Preempted => { - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - requeue_active_download_for_capacity_wait( - &dq, - &worker_uuid, - "Reloading download service configuration", - true, - Some(RecordingControl::Restart), - ) - .await, - "configuration-reload requeue", - ) { - error!("Download worker commit failed: {err}"); - break 'worker; - } - break; - } - }; - - let mut execution_result = { - let Some(download) = active_download_snapshot_for_worker(&dq.active, &worker_uuid).await - else { - capacity.release(provider_handle).await; - break 'worker; - }; - // Last point before a destination is opened. - if let Some(reason) = refused_before_start(&dq, &app_config, &worker_uuid).await { - capacity.release(provider_handle).await; - match fail_active_download(&dq, &worker_uuid, reason).await { - Ok(_) => publish_recording_change(&event_manager), - Err(err) => { - error!("Download worker commit failed: {err}"); - break 'worker; - } - } - continue; - } - match download.kind { - RecordingKind::Vod | RecordingKind::Series => 'http_execution: { - let execution_download = match recording_execution_download( - &app_config, - &download, - provider_handle.as_ref(), - ) { - Ok(execution) => execution, - Err(error) => break 'http_execution DownloadExecutionResult::Failed(error), - }; - download_file( - Arc::clone(&dq.active), - execution_download, - &transfer_client, - Some(&upstream_transfer_client), - Arc::clone(&control_signal), - Arc::clone(&control_notify), - Some(&event_manager), - Some(QuotaGate { queue: &dq, app_config: &app_config }), - ) - .await - } - RecordingKind::Live => 'recording_execution: { - let execution_download = match recording_execution_download( - &app_config, - &download, - provider_handle.as_ref(), - ) { - Ok(execution_download) => execution_download, - Err(err) => break 'recording_execution DownloadExecutionResult::Failed(err), - }; - let progress_path = recording_partial_path(&execution_download.file_path); - let container_format = - app_config.config.load().recording().map_or_else( - shared::model::RecordingContainerFormat::default, - |recording| recording.container_format, - ); - let mut recording_future = Box::pin(run_recording_with_binary( - &recording_binary, - &execution_download, - &control_signal, - &control_notify, - None, - container_format, - )); - let mut progress_tick = time::interval(RECORDING_PROGRESS_UPDATE_INTERVAL); - progress_tick.set_missed_tick_behavior(time::MissedTickBehavior::Skip); - - let result = loop { - tokio::select! { - recording_result = &mut recording_future => break recording_result, - _ = progress_tick.tick() => { - refresh_recording_progress( - &dq.active, - &worker_uuid, - &progress_path, - &event_manager, - ) - .await; - } - } - }; - let final_progress_path = if matches!(&result, RecordingExecutionResult::Completed) - { - &execution_download.file_path - } else { - &progress_path - }; - refresh_recording_progress( - &dq.active, - &worker_uuid, - final_progress_path, - &event_manager, - ) - .await; - - match result { - RecordingExecutionResult::Completed => DownloadExecutionResult::Completed, - RecordingExecutionResult::Paused => DownloadExecutionResult::Paused, - RecordingExecutionResult::Cancelled => DownloadExecutionResult::Cancelled, - RecordingExecutionResult::Preempted => DownloadExecutionResult::Preempted, - RecordingExecutionResult::Retryable(err) => { - DownloadExecutionResult::Retryable(err) - } - RecordingExecutionResult::Failed(err) => DownloadExecutionResult::Failed(err), - } - } - } - }; - - // The internal HTTP stream owns the same provider allocation, and its - // normal EOF/client-close cleanup cancels the allocation token too, so - // cancellation alone does not mean foreground preemption. The manager - // records an explicit close reason before a real priority eviction; - // only that outcome requeues the recording. - if provider_handle.as_ref().is_some_and(|handle| { - handle.get_close_reason() == tuliprox_core::model::ProviderCloseReason::PriorityPreempted - }) { - execution_result = DownloadExecutionResult::Preempted; - } - - match execution_result { - DownloadExecutionResult::Completed => { - capacity.release(provider_handle).await; - // Path copied out first: the lock must not be held across - // the filesystem call, or progress writers and commits stall. - let finished_file = dq - .active - .read() - .await - .iter() - .find(|task| task.uuid == worker_uuid) - .map(|fd| (fd.file_path.clone(), fd.size)); - let measured_bytes = match finished_file { - Some((path, fallback)) => { - tokio::fs::metadata(&path).await.map_or(fallback, |metadata| metadata.len()) - } - None => 0, - }; - // Beside the file, before the repository is told - // it is complete: an operator finding an orphan - // needs the record even if the commit is what - // failed. - write_completion_sidecar(&dq, &worker_uuid, measured_bytes).await; - let completion_limits = app_config.config.load().recording().map(|recording| { - crate::recording::recording_service::quota_limits_from_config( - recording.quota.as_ref(), - ) - }); - let committed = - finish_active_and_promote(&dq, &worker_uuid, completion_limits.as_ref(), |fd| { - fd.finished = true; - fd.paused = false; - fd.state = RecordingTaskState::Completed; - fd.size = measured_bytes; - fd.error = None; - fd.next_retry_at = None; - let meta = &mut fd.recording; - meta.measured_bytes = measured_bytes; - meta.reserved_bytes = 0; - meta.completed_at = Some(chrono::Utc::now().timestamp()); - meta.partial_relative_path = None; - mark_recording_metadata_notification( - &mut fd.recording, - LifecycleEvent::Completed, - None, - ) - }) - .await; - match committed { - Ok(Some(notification)) => { - spawn_recording_notification_after_persist( - &app_config, - &client, - notification, - true, - ); - publish_recording_change(&event_manager); - } - Ok(None) => {} - Err(err) => { - error!("Failed to persist completed download state: {err}"); - break 'worker; - } - } - } - DownloadExecutionResult::Paused => { - capacity.release(provider_handle).await; - // A resume observed here continues this worker. A later resume - // is picked up after release_worker clears the claim and wakes - // the scheduler to start a new worker. - if continue_after_pause(&dq, &worker_uuid).await { - continue 'worker; - } - break 'worker; - } - DownloadExecutionResult::Cancelled => { - capacity.release(provider_handle).await; - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - cancel_active_and_promote(&dq, &worker_uuid).await, - "cancelled state", - ) { - error!("Download worker commit failed: {err}"); - break 'worker; - } - } - DownloadExecutionResult::Preempted => { - capacity.release(provider_handle).await; - let control = *control_signal.read().await; - match control { - RecordingControl::Restart => warn!( - "Active transfer is restarting to apply updated download service configuration" - ), - _ => warn!("Active transfer was preempted by a higher-priority stream"), - } - let reason = { - let active = dq.active.read().await; - if control == RecordingControl::Restart { - "Reloading download service configuration" - } else { - active - .iter() - .find(|task| task.uuid == worker_uuid) - .map_or(DOWNLOAD_PREEMPTED_REASON, preemption_reason_for) - } - }; - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - requeue_active_download_for_capacity_wait( - &dq, - &worker_uuid, - reason, - true, - (control == RecordingControl::Restart).then_some(RecordingControl::Restart), - ) - .await, - "preempted requeue", - ) { - error!("Download worker commit failed: {err}"); - break 'worker; - } - if should_exit_worker_after_preempt(control) { - break; - } - } - DownloadExecutionResult::Retryable(err) => { - capacity.release(provider_handle).await; - let err = redact_url_tokens(&err); - warn!("Recording transfer encountered a transient failure: {err}"); - let retry_commit = prepare_active_retry(&dq, &worker_uuid, &download_cfg, &err).await; - let retry_delay_secs = match retry_commit { - Ok(Some(RetryCommit::Waiting { delay_secs, attempts })) => { - debug!("Download retry attempt {attempts} scheduled in {delay_secs}s"); - publish_recording_change(&event_manager); - delay_secs - } - Ok(Some(RetryCommit::Failed(notification))) => { - spawn_recording_notification_after_persist( - &app_config, - &client, - notification, - true, - ); - publish_recording_change(&event_manager); - if dq.active.read().await.iter().any(|task| task.uuid == worker_uuid) { - continue; - } - break; - } - Ok(None) => break, - Err(err) => { - error!("Failed to persist retry state: {err}"); - break; - } - }; - let mut retry_sleep = Box::pin(time::sleep(Duration::from_secs(retry_delay_secs))); - let retry_wait_outcome = loop { - tokio::select! { - () = &mut retry_sleep => break DownloadExecutionResult::Retryable(String::new()), - () = control_notify.notified() => { - match *control_signal.read().await { - RecordingControl::Pause => break DownloadExecutionResult::Paused, - RecordingControl::Cancel => break DownloadExecutionResult::Cancelled, - RecordingControl::Restart => break DownloadExecutionResult::Preempted, - RecordingControl::None => {} - } - } - } - }; - - match retry_wait_outcome { - DownloadExecutionResult::Retryable(_) => { - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - requeue_active_download_for_retry(&dq, &worker_uuid, true).await, - "retry requeue", - ) { - error!("Download worker commit failed: {err}"); - break 'worker; - } - } - DownloadExecutionResult::Paused => { - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - set_active_download_state( - &dq, - &worker_uuid, - RecordingTaskState::Paused, - None, - true, - ) - .await, - "paused retry state", - ) { - error!("Download worker commit failed: {err}"); - break 'worker; - } - break; - } - DownloadExecutionResult::Cancelled => { - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - cancel_active_and_promote(&dq, &worker_uuid).await, - "cancelled retry state", - ) { - error!("Download worker commit failed: {err}"); - break 'worker; - } - } - DownloadExecutionResult::Completed | DownloadExecutionResult::Failed(_) => {} - DownloadExecutionResult::Preempted => { - if let Err(err) = broadcast_required_worker_mutation( - &event_manager, - requeue_active_download_for_capacity_wait( - &dq, - &worker_uuid, - "Reloading download service configuration", - true, - Some(RecordingControl::Restart), - ) - .await, - "configuration-reload retry requeue", - ) { - error!("Download worker commit failed: {err}"); - break 'worker; - } - break; - } - } - } - DownloadExecutionResult::Failed(err) => { - capacity.release(provider_handle).await; - let err = redact_url_tokens(&err); - warn!("Download failed permanently: {err}"); - let committed = finish_active_and_promote(&dq, &worker_uuid, None, |fd| { - fd.finished = true; - fd.paused = false; - fd.next_retry_at = None; - fd.error = Some(err.clone()); - fd.state = RecordingTaskState::Failed; - fd.recording.reserved_bytes = 0; - mark_recording_metadata_notification( - &mut fd.recording, - LifecycleEvent::Failed, - Some(err), - ) - }) - .await; - match committed { - Ok(Some(notification)) => { - spawn_recording_notification_after_persist( - &app_config, - &client, - notification, - true, - ); - publish_recording_change(&event_manager); - } - Ok(None) => {} - Err(commit_err) => { - error!("Failed to persist failed download state: {commit_err}"); - break 'worker; - } - } - } - } - } else { - break; - } - } - dq.release_worker(&worker_uuid).await; - }); - } else { - *worker.running.write().await = false; - return Err("Failed to build http client".to_string()); - } - } else { - download_queue.release_worker(&worker_uuid).await; - } - Ok(()) -} - -pub fn spawn_recording_services( - ctx: &RecordingCtx, - cancel_token: &CancellationToken, -) { - let config = ctx.app_config.config.load(); - let Some(recording_cfg) = config.recording().cloned() else { - return; - }; - - start_recording_scheduler( - Arc::clone(&ctx.app_config), - recording_cfg, - &ctx.recordings, - ctx.events.clone(), - Arc::clone(&ctx.recording_capacity), - cancel_token.clone(), - Path::new(crate::recording::recording_worker::FFMPEG_BINARY).to_path_buf(), - ); -} - -pub async fn resume_recording_worker_if_needed( - ctx: &RecordingCtx, - recording_cfg: &RecordingConfig, -) -> Result<(), String> { - if ctx.recordings.queue.lock().await.is_empty() && ctx.recordings.active.read().await.is_empty() { - return Ok(()); - } - - ensure_recording_worker_running( - &ctx.app_config, - recording_cfg, - &ctx.recordings, - &ctx.events, - &ctx.recording_capacity, - Path::new(crate::recording::recording_worker::FFMPEG_BINARY), - ) - .await -} - -fn start_recording_scheduler( - app_config: Arc, - recording_cfg: RecordingConfig, - recordings: &Arc, - event_manager: E, - capacity: Arc, - cancel_token: CancellationToken, - recording_binary: std::path::PathBuf, -) { - let capacity_notify = capacity.capacity_changed(); - let slot_waiters = Arc::clone(&recordings.slot_waiters); - let bridge_capacity = Arc::clone(&capacity); - let bridge_recording_cfg = recording_cfg.clone(); - let bridge_app_config = Arc::clone(&app_config); - let bridge_cancel_token = cancel_token.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - () = bridge_cancel_token.cancelled() => break, - () = capacity_notify.notified() => {}, - () = slot_waiters.registration_changed.notified() => {} - } - let mut capacities_by_input: HashMap, ProviderCapacities> = HashMap::new(); - let live_config = bridge_app_config.config.load(); - let waiter_cfg = live_config.recording().unwrap_or(&bridge_recording_cfg); - let mut waiters = slot_waiters.snapshots(); - waiters.sort_by_key(|waiter| waiter.priority); - for waiter in waiters { - let Some(input_name) = waiter.input_name.as_ref() else { - let _ = slot_waiters.signal_waiter(waiter.id); - continue; - }; - let capacities = match capacities_by_input.entry(Arc::clone(input_name)) { - std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(), - std::collections::hash_map::Entry::Vacant(entry) => { - entry.insert(bridge_capacity.capacities_for_input(input_name).await) - } - }; - if !capacities_have_free_slot(capacities) - || background_download_should_wait(waiter.priority, capacities, waiter_cfg) - { - continue; - } - if slot_waiters.signal_waiter(waiter.id) { - // Reserve a wake-up against this snapshot so one signal wakes - // as many eligible recordings as there are free connections. - if let Some((_, used, _)) = - capacities.iter_mut().find(|(_, used, limit)| *limit == 0 || *used < *limit) - { - *used = used.saturating_add(1); - } - } - } - } - }); - - let scheduler_recordings = Arc::clone(recordings); - let scheduler_cancel_token = cancel_token; - tokio::spawn(async move { - let mut interval = time::interval(Duration::from_secs(1)); - interval.set_missed_tick_behavior(time::MissedTickBehavior::Skip); - loop { - tokio::select! { - () = scheduler_cancel_token.cancelled() => break, - _ = interval.tick() => {}, - () = scheduler_recordings.queue_changed.notified() => {} - } - let promoted = scheduler_recordings.promote_due_scheduled_now().await; - if promoted > 0 { - publish_recording_change(&event_manager); - } - // Commits and worker release wake the scheduler immediately. The - // interval also checks scheduled windows and retries failed starts. - if let Err(error) = ensure_recording_worker_running( - &app_config, - &recording_cfg, - &scheduler_recordings, - &event_manager, - &capacity, - &recording_binary, - ) - .await - { - error!("Could not start recording workers: {error}"); - } - } - }); -} +mod completion; +mod control; +mod http; +mod notifications; +mod provider; +mod quota; +mod retry; +mod worker; #[cfg(test)] -mod tests { - use super::{ - acquire_result_after_wait, acquire_result_for_control, continue_after_pause, download_file, - ensure_recording_worker_running, finalize_http_transfer, finish_active_and_promote, http_transfer_path, - recording_deadline_instant, refresh_recording_progress, requeue_active_download_for_capacity_wait, - start_recording_scheduler, wait_for_provider_slot, DownloadExecutionResult, ProviderAcquireResult, QuotaGate, - RecordingNotificationPlan, DISK_GONE_BEFORE_START, DOWNLOAD_PREEMPTED_REASON, LIVE_CAPACITY_WINDOW_CLOSED, - QUOTA_EXCEEDED_DURING_TRANSFER, - }; - use crate::recording::{ - recording_capacity::{stub::StubCapacity, RecordingCapacityPort}, - recording_queue::{ - PersistedRecordingTask, RecordingControl, RecordingPartition, RecordingQueue, RecordingTask, - RecordingWaitOutcome, - }, - }; - use shared::model::{ - EventMessage, EventSink, NoopSink, RecordingKind, RecordingMetadata, RecordingOwner, RecordingSource, - RecordingTaskState, RecordingVisibility, UserId, - }; - use std::{ - io::{Read, Write}, - net::TcpListener, - path::{Path, PathBuf}, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }, - time::Duration, - }; - use tempfile::TempDir; - use tokio::sync::{Notify, RwLock}; - use tuliprox_core::model::RecordingConfig; - - #[derive(Clone, Default)] - struct ProgressSink(Arc); - - impl EventSink for ProgressSink { - fn emit(&self, event: EventMessage) { - if matches!(event, EventMessage::RecordingProgress) { - self.0.fetch_add(1, Ordering::Relaxed); - } - } - } - - /// A task whose Live window runs from `program_start` for `duration_secs`. - fn scheduled_task(kind: RecordingKind, program_start: i64, duration_secs: i64) -> RecordingTask { - let meta = RecordingMetadata::new_live( - RecordingOwner::User(UserId::from("web:alice")), - RecordingVisibility::Private, - RecordingSource::new("t1", "v1", "in1"), - program_start, - program_start + duration_secs, - 0, - 0, - ); - RecordingQueue::from_persisted(PersistedRecordingTask { - media_identity: String::new(), - partition: RecordingPartition::default(), - uuid: "task".to_string(), - kind, - file_dir: PathBuf::from("/tmp"), - file_path: PathBuf::from("/tmp/capture.ts"), - filename: "capture.ts".to_string(), - url: "https://example.com/stream".to_string(), - finished: false, - size: 0, - total_size: None, - paused: false, - error: None, - state: RecordingTaskState::Running, - input_name: None, - priority: 0, - retry_attempts: 0, - next_retry_at: None, - recording: meta, - }) - .expect("valid fixture") - } - - fn serve_range_fixture(ignore_range: bool) -> (reqwest::Url, std::thread::JoinHandle) { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind fixture server"); - let url = reqwest::Url::parse(&format!("http://{}/download", listener.local_addr().expect("server address"))) - .expect("fixture URL"); - let server = std::thread::spawn(move || { - let (mut socket, _) = listener.accept().expect("accept download request"); - socket.set_read_timeout(Some(Duration::from_secs(5))).expect("read timeout"); - let mut request = Vec::new(); - let mut chunk = [0_u8; 1024]; - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let read = socket.read(&mut chunk).expect("read download request"); - assert!(read > 0, "request ended before headers"); - request.extend_from_slice(&chunk[..read]); - } - let request = String::from_utf8(request).expect("request headers"); - let (status, headers, body) = if ignore_range { - ("200 OK", "", b"0123456789".as_slice()) - } else { - ("206 Partial Content", "Content-Range: bytes 4-9/10\r\n", b"456789".as_slice()) - }; - let response = - format!("HTTP/1.1 {status}\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n", body.len()); - socket.write_all(response.as_bytes()).expect("write response headers"); - socket.write_all(body).expect("write response body"); - request - }); - (url, server) - } - - #[tokio::test] - async fn vod_and_series_resume_at_the_saved_byte_offset() { - for kind in [RecordingKind::Vod, RecordingKind::Series] { - let dir = TempDir::new().expect("recording directory"); - let (url, server) = serve_range_fixture(false); - let mut task = scheduled_task(kind, chrono::Utc::now().timestamp(), 900); - task.file_dir = dir.path().to_path_buf(); - task.file_path = dir.path().join("recording.mp4"); - task.url = url; - task.total_size = Some(10); - let partial = http_transfer_path(&task); - tokio::fs::write(&partial, b"0123").await.expect("saved partial"); - let active = Arc::new(RwLock::new(vec![task.clone()])); - let result = download_file::( - active, - task.clone(), - &reqwest::Client::new(), - None, - Arc::new(RwLock::new(RecordingControl::None)), - Arc::new(Notify::new()), - None, - None, - ) - .await; - assert!(matches!(result, DownloadExecutionResult::Completed)); - assert_eq!(tokio::fs::read(&task.file_path).await.expect("completed recording"), b"0123456789"); - assert!(!partial.exists()); - let request = server.join().expect("fixture server").to_ascii_lowercase(); - assert!(request.contains("range: bytes=4-\r\n"), "missing resume Range: {request}"); - assert!(request.contains("accept-encoding: identity\r\n"), "missing identity encoding: {request}"); - } - } - - #[tokio::test] - async fn resume_during_pause_shutdown_keeps_a_worker_for_vod_and_series() { - for kind in [RecordingKind::Vod, RecordingKind::Series] { - let queue = RecordingQueue::new(); - *queue.active.write().await = vec![scheduled_task(kind, 0, 900)]; - *queue.worker("task").running.write().await = true; - - assert!(queue.pause_active("task").await.expect("pause")); - assert!(queue.resume_active("task").await.expect("resume before old worker exits")); - assert!(continue_after_pause(&queue, "task").await); - assert!(queue.workers_running().await); - assert_eq!(queue.active.read().await.first().map(|task| task.state), Some(RecordingTaskState::Running)); - - assert!(queue.pause_active("task").await.expect("second pause")); - assert!(!continue_after_pause(&queue, "task").await); - assert!(queue.workers_running().await, "the claim lasts until worker exit"); - queue.release_worker("task").await; - assert!(!queue.workers_running().await); - assert!(queue.resume_active("task").await.expect("resume after old worker exits")); - assert!(!queue.active.read().await.first().is_some_and(|task| task.paused)); - } - } - - #[tokio::test] - async fn ignored_range_preserves_vod_and_series_partials() { - for kind in [RecordingKind::Vod, RecordingKind::Series] { - let dir = TempDir::new().expect("recording directory"); - let (url, server) = serve_range_fixture(true); - let mut task = scheduled_task(kind, chrono::Utc::now().timestamp(), 900); - task.file_dir = dir.path().to_path_buf(); - task.file_path = dir.path().join("recording.mp4"); - task.url = url; - task.total_size = Some(10); - let partial = http_transfer_path(&task); - tokio::fs::write(&partial, b"0123").await.expect("saved partial"); - let result = download_file::( - Arc::new(RwLock::new(vec![task.clone()])), - task.clone(), - &reqwest::Client::new(), - None, - Arc::new(RwLock::new(RecordingControl::None)), - Arc::new(Notify::new()), - None, - None, - ) - .await; - assert!( - matches!(result, DownloadExecutionResult::Failed(error) if error == super::RANGE_UNSUPPORTED_ERROR) - ); - assert_eq!(tokio::fs::read(&partial).await.expect("partial preserved"), b"0123"); - assert!(!task.file_path.exists()); - assert!(server.join().expect("fixture server").to_ascii_lowercase().contains("range: bytes=4-\r\n")); - } - } - - #[tokio::test] - async fn live_byte_progress_is_visible_without_a_queue_revision_change() { - let dir = TempDir::new().expect("tempdir"); - let partial = dir.path().join("capture.ts.partial"); - std::fs::write(&partial, b"growing capture").expect("write partial recording"); - let queue = RecordingQueue::new(); - *queue.active.write().await = vec![scheduled_task(RecordingKind::Live, 0, 60)]; - let events = ProgressSink::default(); - - refresh_recording_progress(&queue.active, "task", &partial, &events).await; - let (revision, tasks) = queue.committed_snapshot().await; - - assert_eq!(revision.0, 0); - assert_eq!(tasks[0].size, b"growing capture".len() as u64); - assert_eq!(events.0.load(Ordering::Relaxed), 1); - - refresh_recording_progress(&queue.active, "task", &partial, &events).await; - assert_eq!(events.0.load(Ordering::Relaxed), 1, "unchanged bytes need no second event"); - } - - #[tokio::test] - async fn a_restart_requeue_consumes_the_restart_signal() { - // A restart that survives its own requeue makes every new worker - // preempt itself on start, spinning until the live window closes. - let queue = RecordingQueue::new(); - *queue.active.write().await = vec![scheduled_task(RecordingKind::Live, 0, 60)]; - *queue.worker("task").control_signal.write().await = RecordingControl::Restart; - - let requeued = requeue_active_download_for_capacity_wait( - &queue, - "task", - "Reloading download service configuration", - false, - Some(RecordingControl::Restart), - ) - .await - .expect("requeue"); - - assert!(requeued); - assert_eq!(*queue.worker("task").control_signal.read().await, RecordingControl::None); - } - - #[tokio::test] - async fn a_preemption_requeue_leaves_a_pending_control_untouched() { - let queue = RecordingQueue::new(); - *queue.active.write().await = vec![scheduled_task(RecordingKind::Live, 0, 60)]; - *queue.worker("task").control_signal.write().await = RecordingControl::Pause; - - requeue_active_download_for_capacity_wait(&queue, "task", DOWNLOAD_PREEMPTED_REASON, false, None) - .await - .expect("requeue"); - - assert_eq!(*queue.worker("task").control_signal.read().await, RecordingControl::Pause); - } - - /// One VOD on one media for `owner`, in the given partition and state. - fn vod_entry(uuid: &str, owner: &str, state: RecordingTaskState) -> PersistedRecordingTask { - let mut task = scheduled_task(RecordingKind::Vod, 0, 0); - task.uuid = uuid.to_string(); - task.state = state; - task.recording = RecordingMetadata::new_media( - RecordingOwner::User(UserId::from(owner)), - RecordingVisibility::Private, - RecordingSource::new("t1", "v1", "in1"), - "Film".to_string(), - ); - let mut persisted = RecordingQueue::to_persisted(&task); - persisted.media_identity = "film".to_string(); - persisted - } - - #[tokio::test] - async fn starting_workers_commits_entries_attached_to_an_already_completed_file( - ) -> Result<(), Box> { - let queue = Arc::new(RecordingQueue::new()); - let mut done = vod_entry("done", "alice", RecordingTaskState::Completed); - done.finished = true; - done.size = 123; - done.recording.measured_bytes = 123; - let waiting = vod_entry("waiting", "bob", RecordingTaskState::Queued); - crate::recording::recording_queue::mutate(&queue, |candidate| { - candidate.finished.push(done); - candidate.queue.push(waiting); - Ok(()) - }) - .await?; - let stub = StubCapacity::with_room(); - let capacity: Arc = stub.clone(); - ensure_recording_worker_running( - &app_config_with_listener(), - &RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }), - &queue, - &NoopSink, - &capacity, - Path::new("unused-encoder"), - ) - .await?; - assert!(queue.queue.lock().await.is_empty()); - assert!(queue.active.read().await.is_empty()); - assert_eq!(queue.finished.read().await.len(), 2); - let finished = queue.finished.read().await; - let attached = finished.iter().find(|task| task.uuid == "waiting").ok_or("missing attached entry")?; - assert_eq!(attached.state, RecordingTaskState::Completed); - assert_eq!(attached.size, 123); - assert_eq!(stub.acquire_count(), 0); - Ok(()) - } - - /// Recording enabled under `dir` with a private quota of `quota` bytes. - fn app_config_with_quota(dir: &Path, quota: u64) -> tuliprox_core::model::AppConfig { - let app_config = bare_app_config(); - let mut rec_cfg = - RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }); - rec_cfg.directory = dir.to_string_lossy().into_owned(); - rec_cfg.quota = Some(tuliprox_core::model::RecordingQuotaConfig { - default_private_bytes: Some(quota), - ..Default::default() - }); - app_config.config.store(Arc::new(tuliprox_core::model::Config { - video: Some(tuliprox_core::model::VideoConfig { - extensions: Vec::new(), - web_search: None, - recording: Some(rec_cfg), - }), - ..tuliprox_core::model::Config::default() - })); - app_config - } - - async fn queue_with( - dir: &TempDir, - active: PersistedRecordingTask, - queued: Vec, - ) -> RecordingQueue { - let queue = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository"); - crate::recording::recording_queue::mutate(&queue, move |candidate| { - candidate.active = vec![active.clone()]; - candidate.queue.clone_from(&queued); - Ok(()) - }) - .await - .expect("seed"); - queue - } - - #[tokio::test] - async fn a_transfer_larger_than_the_quota_left_is_refused_once_its_size_is_known() { - // Admitted at zero because the size was unknown; the first response - // says 4096 bytes against a 16 byte quota. - let dir = TempDir::new().expect("tempdir"); - let queue = queue_with(&dir, vod_entry("alice", "web:alice", RecordingTaskState::Running), Vec::new()).await; - let app_config = app_config_with_quota(dir.path(), 16); - let gate = QuotaGate { queue: &queue, app_config: &app_config }; - - let refused = gate.admit_size("alice", Some(4096), 0).await.err(); - assert_eq!(refused.as_deref(), Some(QUOTA_EXCEEDED_DURING_TRANSFER)); - - let admitted = gate.admit_size("alice", Some(16), 0).await.expect("fits"); - assert_eq!(admitted.byte_cap, None); - let reserved = queue.active.read().await.first().map(|active| active.recording.reserved_bytes); - assert_eq!(reserved, Some(16), "the known size is now reserved"); - } - - #[tokio::test] - async fn a_download_over_the_quota_stops_before_writing_its_file() { - let dir = TempDir::new().expect("tempdir"); - let (url, server) = serve_range_fixture(true); - let mut task = scheduled_task(RecordingKind::Vod, chrono::Utc::now().timestamp(), 900); - task.file_dir = dir.path().to_path_buf(); - task.file_path = dir.path().join("film.mp4"); - task.url = url; - task.recording = RecordingMetadata::new_media( - RecordingOwner::User(UserId::from("web:alice")), - RecordingVisibility::Private, - RecordingSource::new("t1", "v1", "in1"), - "Film".to_string(), - ); - let queue = queue_with(&dir, RecordingQueue::to_persisted(&task), Vec::new()).await; - let app_config = app_config_with_quota(dir.path(), 4); - - let result = download_file::( - Arc::clone(&queue.active), - task.clone(), - &reqwest::Client::new(), - None, - Arc::new(RwLock::new(RecordingControl::None)), - Arc::new(Notify::new()), - None, - Some(QuotaGate { queue: &queue, app_config: &app_config }), - ) - .await; - let _ = server.join(); - - assert!( - matches!(&result, DownloadExecutionResult::Failed(reason) if reason == QUOTA_EXCEEDED_DURING_TRANSFER), - "a 10 byte file against a 4 byte quota" - ); - assert!(!task.file_path.exists(), "nothing was published"); - } - - #[tokio::test] - async fn a_file_of_unknown_size_is_checked_against_waiting_quotas_when_it_completes() { - // Without a Content-Length nothing can be charged when the transfer - // starts. Alice has room; Bob, waiting to attach, has 16 bytes. The - // finished size is the first moment his quota can be asked. - let dir = TempDir::new().expect("tempdir"); - let queue = queue_with( - &dir, - vod_entry("alice", "web:alice", RecordingTaskState::Running), - vec![vod_entry("bob", "web:bob", RecordingTaskState::Queued)], - ) - .await; - let app_config = app_config_with_quota(dir.path(), 1 << 20); - let gate = QuotaGate { queue: &queue, app_config: &app_config }; - let started = gate.admit_size("alice", None, 0).await.expect("alice starts"); - assert!(!started.siblings_refused, "nothing is known about the size yet"); - - let limits = crate::recording::recording_quota::QuotaLimits { - default_private_bytes: Some(16), - per_user_bytes: [(UserId::from("web:alice"), 1 << 20)].into_iter().collect(), - shared_bytes: None, - }; - finish_active_and_promote(&queue, "alice", Some(&limits), |done| { - done.finished = true; - done.state = RecordingTaskState::Completed; - done.size = 4096; - done.recording.measured_bytes = 4096; - done.recording.reserved_bytes = 0; - RecordingNotificationPlan::empty() - }) - .await - .expect("commit"); - - let finished = queue.finished.read().await.clone(); - let state_of = |uuid: &str| finished.iter().find(|task| task.uuid == uuid).map(|task| task.state); - assert_eq!(state_of("alice"), Some(RecordingTaskState::Completed)); - assert_eq!(state_of("bob"), Some(RecordingTaskState::Failed), "not attached past his quota"); - assert!(queue.queue.lock().await.is_empty()); - } - - #[tokio::test] - async fn a_transfer_of_unknown_size_is_capped_at_the_quota_left() { - let dir = TempDir::new().expect("tempdir"); - let queue = queue_with(&dir, vod_entry("alice", "web:alice", RecordingTaskState::Running), Vec::new()).await; - let app_config = app_config_with_quota(dir.path(), 1000); - let gate = QuotaGate { queue: &queue, app_config: &app_config }; - - let admitted = gate.admit_size("alice", None, 0).await.expect("admitted"); - assert_eq!(admitted.byte_cap, Some(1000)); - } - - #[tokio::test] - async fn an_entry_waiting_to_attach_is_refused_for_its_own_quota() { - // Bob queued behind Alice's transfer while its size was unknown. - // Attaching later would charge him the whole file, so his quota has - // to be asked now. - let dir = TempDir::new().expect("tempdir"); - let queue = queue_with( - &dir, - vod_entry("alice", "web:alice", RecordingTaskState::Running), - vec![vod_entry("bob", "web:bob", RecordingTaskState::Queued)], - ) - .await; - let app_config = app_config_with_quota(dir.path(), 4096); - // Bob already holds something else that leaves him less than the file. - crate::recording::recording_queue::mutate(&queue, |candidate| { - let mut other = vod_entry("bob-other", "web:bob", RecordingTaskState::Completed); - other.media_identity = "other".to_string(); - other.recording.measured_bytes = 1000; - other.finished = true; - candidate.finished.push(other); - Ok(()) - }) - .await - .expect("seed"); - let gate = QuotaGate { queue: &queue, app_config: &app_config }; - - let admitted = gate.admit_size("alice", Some(4096), 0).await.expect("alice fits"); - assert!(admitted.siblings_refused); - assert!(queue.queue.lock().await.is_empty(), "bob no longer waits"); - let bob = queue.finished.read().await.iter().find(|task| task.uuid == "bob").cloned().expect("bob is filed"); - assert_eq!(bob.state, RecordingTaskState::Failed); - assert_eq!(bob.recording.reserved_bytes, 0); - } - - #[test] - fn a_slot_wait_maps_onto_the_acquisition_outcome() { - // A wait that ends without an outcome ran into the live window's end; - // being signalled means trying to acquire again. - assert!(matches!(acquire_result_after_wait(None), Some(ProviderAcquireResult::WindowClosed))); - assert!(acquire_result_after_wait(Some(RecordingWaitOutcome::Signalled)).is_none()); - assert!(matches!( - acquire_result_after_wait(Some(RecordingWaitOutcome::Paused)), - Some(ProviderAcquireResult::Paused) - )); - assert!(matches!( - acquire_result_after_wait(Some(RecordingWaitOutcome::Cancelled)), - Some(ProviderAcquireResult::Cancelled) - )); - assert!(matches!( - acquire_result_after_wait(Some(RecordingWaitOutcome::Restarted)), - Some(ProviderAcquireResult::Preempted) - )); - assert!(acquire_result_for_control(RecordingControl::None).is_none()); - assert!(matches!(acquire_result_for_control(RecordingControl::Cancel), Some(ProviderAcquireResult::Cancelled))); - assert!(matches!(acquire_result_for_control(RecordingControl::Pause), Some(ProviderAcquireResult::Paused))); - assert!(matches!( - acquire_result_for_control(RecordingControl::Restart), - Some(ProviderAcquireResult::Preempted) - )); - } - - #[test] - fn a_live_window_is_measured_in_wall_clock_not_in_time_spent_recording() { - // A capture that was preempted or waiting for capacity does not get - // that time back: the broadcast ran regardless. The deadline is - // anchored to the programme, so an interruption cannot push a capture - // past the window and into the next programme. - let now = chrono::Utc::now().timestamp(); - let started_ten_minutes_ago = scheduled_task(RecordingKind::Live, now - 600, 900); - let remaining = recording_deadline_instant(&started_ten_minutes_ago) - .expect("a scheduled live capture has a deadline") - .saturating_duration_since(tokio::time::Instant::now()) - .as_secs(); - // 900s window, 600s already elapsed: about 300 left, never a fresh 900. - assert!((295..=305).contains(&remaining), "expected roughly 300s left, got {remaining}"); - } - - #[test] - fn a_live_window_that_has_already_closed_stops_immediately() { - let now = chrono::Utc::now().timestamp(); - let long_over = scheduled_task(RecordingKind::Live, now - 7_200, 900); - let deadline = recording_deadline_instant(&long_over).expect("still has a deadline"); - assert!(deadline <= tokio::time::Instant::now(), "a closed window must not keep recording"); - } - - #[test] - fn only_live_captures_are_bounded_by_a_window() { - // A film does not stop being downloadable because a clock ran out. - let now = chrono::Utc::now().timestamp(); - for kind in [RecordingKind::Vod, RecordingKind::Series] { - assert!(recording_deadline_instant(&scheduled_task(kind, now, 900)).is_none()); - } - } - - #[test] - fn resumable_kinds_stage_through_a_partial_and_live_writes_in_place() { - // The strategy split: an HTTP transfer can be interrupted and resumed, - // so it stages. ffmpeg owns its output file for the whole capture and - // has nothing to resume into. - let now = chrono::Utc::now().timestamp(); - let live = scheduled_task(RecordingKind::Live, now, 900); - assert_eq!(http_transfer_path(&live), live.file_path, "a live capture writes straight to its file"); - - for kind in [RecordingKind::Vod, RecordingKind::Series] { - let task = scheduled_task(kind, now, 900); - let staged = http_transfer_path(&task); - assert_ne!(staged, task.file_path); - assert!( - staged.to_string_lossy().ends_with(".partial"), - "a resumable transfer stages beside its final file, got {}", - staged.display() - ); - } - } - - #[tokio::test] - async fn stopping_live_capture_clears_cancel_before_starting_a_due_recording( - ) -> Result<(), Box> { - for terminal in [RecordingTaskState::Completed, RecordingTaskState::Failed] { - for has_next in [false, true] { - let dir = TempDir::new()?; - let queue = RecordingQueue::new_persistent(dir.path(), dir.path())?; - let now = chrono::Utc::now().timestamp(); - let mut active = scheduled_task(RecordingKind::Live, now - 60, 900); - active.uuid = "immediate".to_string(); - let mut scheduled = scheduled_task(RecordingKind::Live, now, 900); - scheduled.uuid = "scheduled".to_string(); - scheduled.state = RecordingTaskState::Scheduled; - let active = RecordingQueue::to_persisted(&active); - let scheduled = RecordingQueue::to_persisted(&scheduled); - crate::recording::recording_queue::mutate(&queue, |candidate| { - candidate.active = vec![active]; - if has_next { - candidate.scheduled.push(scheduled); - } - Ok(()) - }) - .await?; - assert_eq!(queue.promote_due_scheduled(now).await, usize::from(has_next)); - assert_eq!(queue.cancel_requested("immediate").await?, Some(false)); - assert_eq!(*queue.worker("immediate").control_signal.read().await, RecordingControl::Cancel); - - let committed = finish_active_and_promote(&queue, "immediate", None, |task| { - task.state = terminal; - task.finished = true; - task.error = - (terminal == RecordingTaskState::Failed).then(|| "Stopped recording has no data".to_string()); - task.recording.reserved_bytes = 0; - RecordingNotificationPlan::empty() - }) - .await?; - assert!(committed.is_some()); - assert_eq!(*queue.worker("immediate").control_signal.read().await, RecordingControl::None); - assert_eq!( - queue.active.read().await.first().map(|task| task.uuid.as_str()), - has_next.then_some("scheduled") - ); - let finished = queue.finished.read().await; - assert_eq!(finished.len(), 1); - assert_eq!(finished[0].state, terminal); - assert!(finished[0].to_view(true).is_terminal()); - drop(finished); - - let restored = RecordingQueue::new_persistent(dir.path(), dir.path())?; - restored.load_from_disk().await?; - assert_eq!(restored.finished.read().await[0].state, terminal); - } - } - Ok(()) - } - - #[tokio::test] - async fn finalizing_publishes_the_staged_bytes_and_clears_the_partial() { - let dir = TempDir::new().expect("tempdir"); - let partial = dir.path().join("film.mp4.partial"); - let final_path = dir.path().join("film.mp4"); - std::fs::write(&partial, b"complete recording").expect("write"); - - finalize_http_transfer(&final_path, &partial).await.expect("finalize"); - - assert!(final_path.exists()); - assert!(!partial.exists(), "the partial is cleared once published"); - } - - #[tokio::test] - async fn finalizing_twice_succeeds_instead_of_failing_a_complete_recording() { - // A crash between the link and the partial's removal leaves both files. - // The resumed transfer sees a complete file, finalizes again, and used - // to get AlreadyExists back -- reporting a recording whose bytes are - // entirely on disk as failed, on every retry. - let dir = TempDir::new().expect("tempdir"); - let partial = dir.path().join("film.mp4.partial"); - let final_path = dir.path().join("film.mp4"); - std::fs::write(&partial, b"complete recording").expect("write"); - std::fs::hard_link(&partial, &final_path).expect("simulate the interrupted finalize"); - - finalize_http_transfer(&final_path, &partial).await.expect("finalizing again must succeed"); - - assert!(final_path.exists()); - assert!(!partial.exists(), "the second attempt finishes the job the first did not"); - } - - #[tokio::test] - async fn a_different_file_at_the_final_path_is_not_published_over() { - // Path reservation should make this unreachable. If it ever happens the - // staged bytes were never verified against what is already there, so - // the transfer must not claim success. - let dir = TempDir::new().expect("tempdir"); - let partial = dir.path().join("film.mp4.partial"); - let final_path = dir.path().join("film.mp4"); - std::fs::write(&partial, b"the recording we just made").expect("write"); - std::fs::write(&final_path, b"something else entirely, of another size").expect("write"); - - let error = finalize_http_transfer(&final_path, &partial).await.expect_err("must refuse"); - assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); - assert!(partial.exists(), "the staged bytes are kept for inspection"); - } - - #[tokio::test] - async fn a_transfer_written_straight_to_its_final_path_needs_no_finalization() { - let dir = TempDir::new().expect("tempdir"); - let final_path = dir.path().join("stream.ts"); - std::fs::write(&final_path, b"live capture").expect("write"); - - finalize_http_transfer(&final_path, &final_path).await.expect("finalize"); - - assert!(final_path.exists()); - } - /// A queue with nothing in it; the wait queue is what these exercise. - fn slot_queue(dir: &TempDir) -> RecordingQueue { - RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open recording repository") - } - - #[tokio::test(start_paused = true)] - async fn a_live_capture_stops_waiting_for_capacity_when_its_window_closes() { - // Waiting past the padded end cannot produce the recording that was - // asked for -- the programme has finished. Before this the wait had no - // bound at all, so a live capture with no free slot sat in the queue - // through its own broadcast and out the other side. - let dir = TempDir::new().expect("tempdir"); - let queue = slot_queue(&dir); - let control = RwLock::new(RecordingControl::None); - let notify = Notify::new(); - let window_ends = tokio::time::Instant::now() + Duration::from_mins(30); - - let outcome = - wait_for_provider_slot(&queue, &Arc::from("provider"), 0, &control, ¬ify, Some(window_ends)).await; - - assert!(outcome.is_none(), "the wait ends when the programme does"); - assert!(queue.slot_waiters.snapshots().is_empty(), "expired waits must deregister"); - assert!(tokio::time::Instant::now() >= window_ends, "and only then"); - } - - #[tokio::test(start_paused = true)] - async fn a_transfer_waits_for_a_slot_however_long_it_takes() { - // The counterpart, and the reason the bound is not applied to - // everything: a file on a server is still there in an hour. - let dir = TempDir::new().expect("tempdir"); - let queue = slot_queue(&dir); - let control = RwLock::new(RecordingControl::None); - let notify = Notify::new(); - - let waiters = Arc::clone(&queue.slot_waiters); - let freed = tokio::spawn(async move { - tokio::time::sleep(Duration::from_mins(60)).await; - let queued = waiters.snapshots(); - let waiter = queued.first().expect("someone is waiting an hour later"); - waiters.signal_waiter(waiter.id) - }); - - let outcome = wait_for_provider_slot(&queue, &Arc::from("provider"), 0, &control, ¬ify, None).await; - - assert!(freed.await.expect("signaller"), "the waiter was still there to signal"); - assert!(matches!(outcome, Some(RecordingWaitOutcome::Signalled)), "it took the slot when one appeared"); - } - - #[tokio::test(start_paused = true)] - async fn a_capacity_wait_still_answers_a_cancellation_before_its_deadline() { - // The deadline must not swallow the control signals: a user cancelling - // a live capture that is waiting for capacity should not have to wait - // out the programme. - let dir = TempDir::new().expect("tempdir"); - let queue = slot_queue(&dir); - let control = RwLock::new(RecordingControl::None); - let notify = Notify::new(); - let window_ends = tokio::time::Instant::now() + Duration::from_mins(30); - - let provider: Arc = Arc::from("provider"); - let waiting = wait_for_provider_slot(&queue, &provider, 0, &control, ¬ify, Some(window_ends)); - let cancelling = async { - tokio::time::sleep(Duration::from_secs(5)).await; - *control.write().await = RecordingControl::Cancel; - notify.notify_waiters(); - }; - let (outcome, ()) = tokio::join!(waiting, cancelling); - - assert!(matches!(outcome, Some(RecordingWaitOutcome::Cancelled)), "the cancel is answered, not the deadline"); - assert!(tokio::time::Instant::now() < window_ends, "and long before the window closes"); - } - fn bare_app_config() -> tuliprox_core::model::AppConfig { - tuliprox_core::model::AppConfig { - config: Arc::new(arc_swap::ArcSwap::from_pointee(tuliprox_core::model::Config::default())), - sources: Arc::new(arc_swap::ArcSwap::from_pointee(tuliprox_core::model::SourcesConfig::default())), - hdhomerun: Arc::new(arc_swap::ArcSwapOption::default()), - api_proxy: Arc::new(arc_swap::ArcSwapOption::default()), - file_locks: Arc::new(tuliprox_core::utils::FileLockManager::default()), - paths: Arc::new(arc_swap::ArcSwap::from_pointee(shared::model::ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(arc_swap::ArcSwapOption::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(tuliprox_core::model::MediaToolCapabilities::new()), - } - } - - /// Configured local API listener with no public playback server. - fn app_config_with_listener() -> tuliprox_core::model::AppConfig { - let app_config = bare_app_config(); - let mut config = (*app_config.config.load_full()).clone(); - config.api.host = "127.0.0.1".to_string(); - config.api.port = 8901; - app_config.config.store(Arc::new(config)); - app_config - } - - /// An executable standing in for ffmpeg that appends a line to `log` - /// every time it is run, then writes its output argument and exits - /// cleanly. The log is what lets a test say "once" rather than "at - /// least once". - fn counting_ffmpeg(dir: &Path, log: &Path) -> PathBuf { - let script_path = dir.join("fake-ffmpeg"); - std::fs::write( - &script_path, - format!( - "#!/bin/sh\necho run >> \"{}\"\nfor arg in \"$@\"; do output=\"$arg\"; done\nprintf 'recorded' > \"$output\"\nexit 0\n", - log.display() - ), - ) - .expect("write fake ffmpeg"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mut perms = std::fs::metadata(&script_path).expect("metadata").permissions(); - perms.set_mode(0o755); - std::fs::set_permissions(&script_path, perms).expect("chmod"); - } - script_path - } - - fn spawn_count(log: &Path) -> usize { - std::fs::read_to_string(log).map_or(0, |text| text.lines().filter(|line| !line.is_empty()).count()) - } - - #[cfg(unix)] - struct LimitedCapacity { - limit: usize, - in_use: AtomicUsize, - peak: AtomicUsize, - releases: AtomicUsize, - notify: Arc, - } - - #[cfg(unix)] - impl RecordingCapacityPort for LimitedCapacity { - fn capacities_for_input<'a>( - &'a self, - input: &'a Arc, - ) -> futures::future::BoxFuture<'a, Vec> { - Box::pin(async move { - let in_use = self.in_use.load(Ordering::SeqCst); - tokio::task::yield_now().await; - vec![(input.clone(), in_use, self.limit)] - }) - } - - fn acquire<'a>( - &'a self, - _input: &'a Arc, - _priority: i8, - ) -> futures::future::BoxFuture<'a, Option> { - Box::pin(async move { - let previous = self - .in_use - .try_update(Ordering::SeqCst, Ordering::SeqCst, |used| { - (self.limit == 0 || used < self.limit).then_some(used + 1) - }) - .ok()?; - self.peak.fetch_max(previous + 1, Ordering::SeqCst); - Some(tuliprox_core::model::ProviderHandle::new( - std::net::SocketAddr::from(([127, 0, 0, 1], 0)), - 1, - tuliprox_core::model::ProviderAllocation::Exhausted, - Some(tokio_util::sync::CancellationToken::new()), - )) - }) - } - - fn release(&self, handle: Option) -> futures::future::BoxFuture<'_, ()> { - Box::pin(async move { - if handle.is_some() { - self.in_use.fetch_sub(1, Ordering::SeqCst); - self.releases.fetch_add(1, Ordering::SeqCst); - self.notify.notify_one(); - } - }) - } - - fn capacity_changed(&self) -> Arc { self.notify.clone() } - } - - #[cfg(unix)] - struct ConcurrentLiveFixture { - dir: TempDir, - queue: Arc, - script: PathBuf, - capacity: Arc, - app: Arc, - config: RecordingConfig, - cancel: tokio_util::sync::CancellationToken, - } - - #[cfg(unix)] - impl ConcurrentLiveFixture { - async fn new(count: u32, limit: usize) -> Result> { - Self::with_background_limit(count, limit, 0).await - } - - async fn with_background_limit( - count: u32, - limit: usize, - background_limit: u8, - ) -> Result> { - let dir = TempDir::new()?; - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path())?); - let script = counting_ffmpeg(dir.path(), &dir.path().join("spawns.log")); - std::fs::write(&script, "#!/bin/sh\nfor arg in \"$@\"; do output=\"$arg\"; done\nprintf recorded > \"$output\"\ntouch \"$output.ready\"\nread -r control\nexit 0\n")?; - let now = chrono::Utc::now().timestamp(); - let tasks = (0..count) - .map(|index| { - let mut task = scheduled_task(RecordingKind::Live, now, 300); - task.uuid = format!("live-{index}"); - task.state = RecordingTaskState::Queued; - task.priority = 5; - task.input_name = Some(Arc::from("provider")); - task.recording.source.virtual_id = (42 + index).to_string(); - task.recording.source.input_name = "provider".to_string(); - task.file_dir = dir.path().to_path_buf(); - task.file_path = dir.path().join(format!("live-{index}.ts")); - RecordingQueue::to_persisted(&task) - }) - .collect::>(); - crate::recording::recording_queue::mutate(&queue, |candidate| { - candidate.queue.extend(tasks); - Ok(()) - }) - .await?; - let fixture = Self { - dir, - queue, - script, - capacity: Arc::new(LimitedCapacity { - limit, - in_use: AtomicUsize::new(0), - peak: AtomicUsize::new(0), - releases: AtomicUsize::new(0), - notify: Arc::new(Notify::new()), - }), - app: Arc::new(app_config_with_listener()), - config: RecordingConfig::from(&shared::model::RecordingConfigDto { - enabled: true, - max_background_per_provider: background_limit, - ..Default::default() - }), - cancel: tokio_util::sync::CancellationToken::new(), - }; - start_recording_scheduler( - fixture.app.clone(), - fixture.config.clone(), - &fixture.queue, - NoopSink, - fixture.capacity.clone(), - fixture.cancel.clone(), - fixture.script.clone(), - ); - Ok(fixture) - } - - async fn start(&self) -> Result<(), String> { - let capacity: Arc = self.capacity.clone(); - ensure_recording_worker_running(&self.app, &self.config, &self.queue, &NoopSink, &capacity, &self.script) - .await - } - - async fn wait_for_running(&self, count: usize) -> Result, tokio::time::error::Elapsed> { - tokio::time::timeout(Duration::from_secs(5), async { - loop { - let ready = self - .queue - .active - .read() - .await - .iter() - .filter(|task| { - task.state == RecordingTaskState::Running - && self.dir.path().join(format!("{}.ts.partial.ready", task.uuid)).exists() - }) - .map(|task| task.uuid.clone()) - .collect::>(); - if ready.len() == count { - break ready; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - } - - async fn stop_all(&self, count: usize) -> Result<(), Box> { - let active = self.queue.active.read().await.iter().map(|task| task.uuid.clone()).collect::>(); - for uuid in active { - self.queue.cancel_requested(&uuid).await?; - } - tokio::time::timeout(Duration::from_secs(5), async { - while self.queue.finished.read().await.len() != count || self.queue.workers_running().await { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await?; - assert_eq!(self.capacity.in_use.load(Ordering::SeqCst), 0); - Ok(()) - } - } - - #[cfg(unix)] - impl Drop for ConcurrentLiveFixture { - fn drop(&mut self) { self.cancel.cancel(); } - } - - #[cfg(unix)] - #[tokio::test] - async fn independent_live_workers_fill_provider_capacity_and_stop_only_the_selected_recording( - ) -> Result<(), Box> { - for limit in [0, 4] { - let fixture = ConcurrentLiveFixture::new(4, limit).await?; - let (first, second) = tokio::join!(fixture.start(), fixture.start()); - first?; - second?; - fixture.wait_for_running(4).await?; - assert_eq!(fixture.capacity.peak.load(Ordering::SeqCst), 4); - fixture.queue.cancel_requested("live-2").await?; - let remaining = fixture.wait_for_running(3).await?; - assert!(!remaining.iter().any(|uuid| uuid == "live-2")); - assert!(fixture - .queue - .active - .read() - .await - .iter() - .filter(|task| task.uuid != "live-2") - .all(|task| task.state == RecordingTaskState::Running)); - fixture.stop_all(4).await?; - assert_eq!( - fixture.capacity.releases.load(Ordering::SeqCst), - 4, - "one allocation per recording despite concurrent starts" - ); - assert!(fixture.queue.finished.read().await.iter().all(|task| task.state == RecordingTaskState::Completed)); - } - Ok(()) - } - - #[cfg(unix)] - #[tokio::test] - async fn live_workers_respect_capacity_and_start_waiting_recordings_when_a_slot_is_released( - ) -> Result<(), Box> { - let fixture = ConcurrentLiveFixture::new(4, 2).await?; - fixture.start().await?; - let running = fixture.wait_for_running(2).await?; - tokio::time::timeout(Duration::from_secs(5), async { - while fixture.queue.slot_waiters.snapshots().len() != 2 { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await?; - assert_eq!(fixture.capacity.in_use.load(Ordering::SeqCst), 2); - fixture.queue.cancel_requested(&running[0]).await?; - tokio::time::timeout(Duration::from_secs(5), async { - loop { - let now_running = fixture.wait_for_running(2).await?; - if now_running.iter().any(|uuid| !running.contains(uuid)) { - break Ok::<_, tokio::time::error::Elapsed>(()); - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await??; - assert_eq!(fixture.capacity.peak.load(Ordering::SeqCst), 2); - assert!(fixture - .queue - .active - .read() - .await - .iter() - .any(|task| task.uuid == running[1] && task.state == RecordingTaskState::Running)); - fixture.stop_all(4).await?; - assert!(fixture.queue.finished.read().await.iter().all(|task| task.to_view(true).is_terminal())); - assert!(fixture.queue.slot_waiters.snapshots().is_empty()); - Ok(()) - } - - #[cfg(unix)] - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn simultaneous_starts_respect_the_configured_background_limit() -> Result<(), Box> { - let fixture = ConcurrentLiveFixture::with_background_limit(3, 3, 1).await?; - fixture.start().await?; - let running = fixture.wait_for_running(1).await?; - tokio::time::timeout(Duration::from_secs(5), async { - while fixture.queue.slot_waiters.snapshots().len() != 2 { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await?; - assert_eq!(fixture.capacity.peak.load(Ordering::SeqCst), 1); - fixture.queue.cancel_requested(&running[0]).await?; - tokio::time::timeout(Duration::from_secs(5), async { - loop { - let now_running = fixture.wait_for_running(1).await?; - if now_running != running { - break Ok::<_, tokio::time::error::Elapsed>(()); - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await??; - assert_eq!(fixture.capacity.peak.load(Ordering::SeqCst), 1); - fixture.stop_all(3).await?; - Ok(()) - } - - #[cfg(unix)] - #[tokio::test] - async fn scheduler_starts_a_due_live_capture_while_another_capture_is_running( - ) -> Result<(), Box> { - for has_data in [false, true] { - let dir = TempDir::new()?; - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path())?); - let now = chrono::Utc::now().timestamp(); - let mut immediate = scheduled_task(RecordingKind::Live, now - 60, 900); - immediate.uuid = "immediate".to_string(); - immediate.state = RecordingTaskState::Queued; - immediate.input_name = Some(Arc::from("provider")); - immediate.recording.source.virtual_id = "42".to_string(); - immediate.file_dir = dir.path().to_path_buf(); - immediate.file_path = dir.path().join("immediate.ts"); - let mut scheduled = immediate.clone(); - scheduled.uuid = "scheduled".to_string(); - scheduled.state = RecordingTaskState::Scheduled; - scheduled.file_path = dir.path().join("scheduled.ts"); - scheduled.recording.program_start = Some(now); - scheduled.recording.scheduled_start = Some(now); - let immediate = RecordingQueue::to_persisted(&immediate); - let scheduled = RecordingQueue::to_persisted(&scheduled); - crate::recording::recording_queue::mutate(&queue, |candidate| { - candidate.queue.push(immediate); - candidate.scheduled.push(scheduled); - Ok(()) - }) - .await?; - let script = counting_ffmpeg(dir.path(), &dir.path().join("spawns.log")); - let write_initial = if has_data { "printf recorded > \"$output\"" } else { ":" }; - std::fs::write(&script, format!( - "#!/bin/sh\nfor arg in \"$@\"; do output=\"$arg\"; done\ncase \"$output\" in\n*immediate.ts.partial) {write_initial}; touch \"$output.ready\"; read -r control ;;\n*) printf recorded > \"$output\" ;;\nesac\nexit 0\n" - ))?; - let stub = StubCapacity::with_room(); - let capacity: Arc = stub.clone(); - let config = - RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }); - ensure_recording_worker_running( - &app_config_with_listener(), - &config, - &queue, - &NoopSink, - &capacity, - &script, - ) - .await?; - tokio::time::timeout(Duration::from_secs(5), async { - while !dir.path().join("immediate.ts.partial.ready").exists() { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await?; - let cancel = tokio_util::sync::CancellationToken::new(); - start_recording_scheduler( - Arc::new(app_config_with_listener()), - config.clone(), - &queue, - NoopSink, - capacity.clone(), - cancel.clone(), - script.clone(), - ); - tokio::time::timeout(Duration::from_secs(5), async { - while !queue.finished.read().await.iter().any(|task| task.uuid == "scheduled") { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await?; - assert!(queue - .active - .read() - .await - .iter() - .any(|task| task.uuid == "immediate" && task.state == RecordingTaskState::Running)); - assert!(queue.finished.read().await.iter().all(|task| task.uuid != "immediate")); - queue.cancel_requested("immediate").await?; - tokio::time::timeout(Duration::from_secs(5), async { - while queue.finished.read().await.len() != 2 || queue.workers_running().await { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await?; - cancel.cancel(); - assert_eq!(stub.acquire_count(), 2); - let finished = queue.finished.read().await; - let stopped = finished.iter().find(|task| task.uuid == "immediate").ok_or("missing stopped task")?; - let next = finished.iter().find(|task| task.uuid == "scheduled").ok_or("missing scheduled task")?; - assert_eq!( - stopped.state, - if has_data { RecordingTaskState::Completed } else { RecordingTaskState::Failed } - ); - assert_eq!(next.state, RecordingTaskState::Completed, "{:?}", next.error); - assert_eq!(tokio::fs::read(&next.file_path).await?, b"recorded"); - assert!(stopped.to_view(true).is_terminal()); - assert!(next.to_view(true).is_terminal()); - assert!(queue.active.read().await.is_empty()); - assert_eq!(*queue.worker("immediate").control_signal.read().await, RecordingControl::None); - assert_eq!(stub.release_count(), 2); - } - Ok(()) - } - - #[tokio::test] - async fn live_timeout_keeps_the_transfer_cause_without_claiming_the_window_expired( - ) -> Result<(), Box> { - let dir = tempfile::tempdir()?; - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path())?); - let mut capture = scheduled_task(RecordingKind::Live, chrono::Utc::now().timestamp(), 300); - capture.recording.source.virtual_id = "42".to_string(); - capture.file_dir = dir.path().to_path_buf(); - capture.file_path = dir.path().join("capture.ts"); - capture.state = RecordingTaskState::Queued; - capture.input_name = Some(Arc::from("provider")); - let persisted = RecordingQueue::to_persisted(&capture); - crate::recording::recording_queue::mutate(&queue, move |candidate| { - candidate.queue.push(persisted.clone()); - Ok(()) - }) - .await?; - let script = counting_ffmpeg(dir.path(), &dir.path().join("spawns.log")); - std::fs::write(&script, "#!/bin/sh\necho 'Error opening input files: Operation timed out' >&2\nexit 1\n")?; - let stub = StubCapacity::with_room(); - let capacity: Arc = Arc::clone(&stub) as Arc; - ensure_recording_worker_running( - &app_config_with_listener(), - &RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }), - &queue, - &NoopSink, - &capacity, - &script, - ) - .await?; - let settled = tokio::time::timeout(Duration::from_secs(5), async { - loop { - if let Some(task) = queue.finished.read().await.first().cloned() { - break task; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await?; - assert_eq!(settled.state, RecordingTaskState::Failed); - let error = settled.error.as_deref().ok_or("missing transfer cause")?; - assert!(error.contains("Error opening input files: Operation timed out"), "{error}"); - assert!(!error.contains("window has moved on"), "{error}"); - assert_eq!(settled.retry_attempts, 1); - assert_eq!(settled.recording.reserved_bytes, 0); - assert_eq!(stub.acquire_count(), 1); - assert_eq!(stub.release_count(), 1); - assert!(queue.active.read().await.is_empty()); - assert_eq!(queue.finished.read().await.len(), 1); - Ok(()) - } - - #[tokio::test] - async fn media_retry_waiting_and_limit_keep_the_transfer_cause() -> Result<(), Box> { - for kind in [RecordingKind::Vod, RecordingKind::Series] { - let queue = RecordingQueue::new(); - let task = scheduled_task(kind, chrono::Utc::now().timestamp(), 300); - let persisted = RecordingQueue::to_persisted(&task); - crate::recording::recording_queue::mutate(&queue, move |candidate| { - candidate.active = vec![persisted.clone()]; - Ok(()) - }) - .await?; - let config = RecordingConfig::from(&shared::model::RecordingConfigDto { - retry_max_attempts: 1, - retry_backoff_initial_secs: 1, - retry_backoff_max_secs: 1, - retry_backoff_jitter_percent: 0, - ..Default::default() - }); - let cause = "Error while opening stream: (http://user:password@upstream.example/live/1?token=secret) Operation timed out"; - let public_cause = crate::recording::recording_worker::redact_url_tokens(cause); - assert_eq!(public_cause, "Error while opening stream: [stream URL] Operation timed out"); - let retry = super::prepare_active_retry(&queue, "task", &config, &public_cause).await?; - assert!(matches!(retry, Some(super::RetryCommit::Waiting { delay_secs: 1, attempts: 1 }))); - { - let active = queue.active.read().await; - let active = active.first().ok_or("active transfer missing")?; - assert_eq!(active.state, RecordingTaskState::RetryWaiting); - assert!(active.error.as_deref().is_some_and(|error| error.contains(&public_cause))); - assert!(active.next_retry_at.is_some()); - } - let retry = super::prepare_active_retry(&queue, "task", &config, &public_cause).await?; - assert!(matches!(retry, Some(super::RetryCommit::Failed(_)))); - let finished = queue.finished.read().await; - let failed = finished.first().ok_or("terminal transfer missing")?; - assert_eq!(failed.state, RecordingTaskState::Failed); - assert_eq!( - failed.error.as_deref(), - Some(format!("Retry limit reached after 1 attempts: {public_cause}").as_str()) - ); - assert_eq!(failed.recording.reserved_bytes, 0); - assert!(failed.next_retry_at.is_none()); - } - Ok(()) - } - - #[tokio::test] - async fn scheduler_starts_a_queued_transfer_after_worker_exit() { - let dir = tempfile::TempDir::new().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let mut transfer = scheduled_task(RecordingKind::Vod, chrono::Utc::now().timestamp(), 300); - transfer.state = RecordingTaskState::Queued; - transfer.input_name = Some(Arc::from("provider")); - let persisted = RecordingQueue::to_persisted(&transfer); - crate::recording::recording_queue::mutate(&queue, move |candidate| { - candidate.queue.push(persisted.clone()); - Ok(()) - }) - .await - .expect("seed queue"); - - let capacity: Arc = StubCapacity::full(); - let cancel = tokio_util::sync::CancellationToken::new(); - start_recording_scheduler( - Arc::new(bare_app_config()), - RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }), - &queue, - NoopSink, - capacity, - cancel.clone(), - Path::new(crate::recording::recording_worker::FFMPEG_BINARY).to_path_buf(), - ); - - tokio::time::timeout(Duration::from_secs(3), async { - loop { - if queue - .active - .read() - .await - .first() - .is_some_and(|task| task.state == RecordingTaskState::WaitingForCapacity) - { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("scheduler did not restart the stranded transfer"); - cancel.cancel(); - } - - #[tokio::test(start_paused = true)] - async fn a_live_capture_that_never_gets_capacity_fails_at_its_window_without_recording() { - // The whole path, driven against a provider that is permanently full: - // the worker asks for a slot, is refused, waits, and gives up when the - // programme it was going to record has finished. Until the capacity - // port existed this could only be produced by a real provider actually - // being busy for the length of a broadcast. - let dir = tempfile::TempDir::new().expect("tempdir"); - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let now = chrono::Utc::now().timestamp(); - let mut capture = scheduled_task(RecordingKind::Live, now, 1_800); - capture.uuid = "live".to_string(); - capture.state = RecordingTaskState::Queued; - capture.input_name = Some(Arc::from("provider")); - let capture = RecordingQueue::to_persisted(&capture); - crate::recording::recording_queue::mutate(&queue, move |candidate| { - candidate.queue.push(capture.clone()); - Ok(()) - }) - .await - .expect("seed"); - - let stub = StubCapacity::full(); - let capacity: Arc = Arc::clone(&stub) as Arc; - let app_config = bare_app_config(); - ensure_recording_worker_running( - &app_config, - &RecordingConfig::from(&shared::model::RecordingConfigDto { - enabled: true, - ..shared::model::RecordingConfigDto::default() - }), - &queue, - &NoopSink, - &capacity, - Path::new(crate::recording::recording_worker::FFMPEG_BINARY), - ) - .await - .expect("worker started"); - - // Sleeping rather than spinning: with a paused clock the runtime only - // advances time when nothing is runnable, so a busy wait would stop the - // window from ever closing. - tokio::time::sleep(Duration::from_mins(31)).await; - - let finished = queue.finished.read().await; - let settled = finished.first().expect("the capture reached a terminal state"); - assert_eq!(settled.state, RecordingTaskState::Failed, "no capacity ever came"); - assert_eq!(settled.error.as_deref(), Some(LIVE_CAPACITY_WINDOW_CLOSED), "and it says why"); - assert_eq!(settled.size, 0, "nothing was ever recorded"); - assert_eq!(settled.recording.reserved_bytes, 0, "and it is not still holding disk"); - assert_eq!( - stub.acquire_count(), - 1, - "it asked for a slot once and then waited, rather than spinning on the provider" - ); - assert_eq!(stub.release_count(), 0, "and it never held one to give back"); - } - - #[tokio::test] - async fn a_live_capture_asks_for_capacity_once_runs_the_encoder_once_and_completes_once() { - // The other half of the window contract. A real clock here on - // purpose: the worker spawns an actual child process, and pausing - // time would let unrelated timers fire between the spawn and the - // exit while the process itself still ran in wall-clock. - let dir = tempfile::TempDir::new().expect("tempdir"); - let recordings_dir = dir.path().join("recordings"); - std::fs::create_dir_all(&recordings_dir).expect("create recording dir"); - let log = dir.path().join("spawns.log"); - let script = counting_ffmpeg(dir.path(), &log); - - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let now = chrono::Utc::now().timestamp(); - let mut capture = scheduled_task(RecordingKind::Live, now, 1_800); - capture.uuid = "live".to_string(); - capture.state = RecordingTaskState::Queued; - capture.input_name = Some(Arc::from("provider")); - // The execution URL is derived from the source, so the virtual id has - // to be a real one rather than the placeholder the other fixtures use. - capture.recording.source.virtual_id = "42".to_string(); - capture.file_dir.clone_from(&recordings_dir); - capture.file_path = recordings_dir.join("capture.ts"); - let persisted = RecordingQueue::to_persisted(&capture); - crate::recording::recording_queue::mutate(&queue, move |candidate| { - candidate.queue.push(persisted.clone()); - Ok(()) - }) - .await - .expect("seed"); - - let stub = StubCapacity::with_room(); - let capacity: Arc = Arc::clone(&stub) as Arc; - ensure_recording_worker_running( - &app_config_with_listener(), - &RecordingConfig::from(&shared::model::RecordingConfigDto { - enabled: true, - ..shared::model::RecordingConfigDto::default() - }), - &queue, - &NoopSink, - &capacity, - &script, - ) - .await - .expect("worker started"); - - let settled = tokio::time::timeout(Duration::from_secs(10), async { - loop { - if let Some(done) = queue.finished.read().await.first().cloned() { - break done; - } - tokio::time::sleep(Duration::from_millis(20)).await; - } - }) - .await; - // A bare timeout here says nothing about why. Where the capture got - // stuck is the whole diagnosis, so report it. - let Ok(settled) = settled else { - panic!( - "never settled: active={:?} queued={} spawns={} acquires={}", - queue.active.read().await.first().map(|active| ( - active.uuid.clone(), - active.state, - active.error.clone() - )), - queue.queue.lock().await.len(), - spawn_count(&log), - stub.acquire_count(), - ); - }; - - assert_eq!(settled.state, RecordingTaskState::Completed, "{:?}", settled.error); - assert_eq!(spawn_count(&log), 1, "the encoder ran once, not once per retry or per promotion pass"); - assert_eq!(stub.acquire_count(), 1, "and one slot was asked for"); - assert_eq!(stub.release_count(), 1, "and given back exactly once"); - assert_eq!(queue.finished.read().await.len(), 1, "the recording was committed once"); - assert_eq!( - tokio::fs::read(&capture.file_path).await.expect("the recording is where playback will look"), - b"recorded" - ); - assert!(!crate::recording_partial_path(&capture.file_path).exists(), "and nothing was left staged"); - assert_eq!(settled.recording.reserved_bytes, 0, "and it is not still holding disk"); - } - - async fn read_request(socket: &mut tokio::net::TcpStream) -> std::io::Result { - use tokio::io::AsyncReadExt; - let mut request = Vec::new(); - let mut byte = [0; 1]; - while !request.ends_with(b"\r\n\r\n") { - if socket.read(&mut byte).await? == 0 { - return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "incomplete request headers")); - } - request.push(byte[0]); - } - Ok(String::from_utf8_lossy(&request).to_ascii_lowercase()) - } - - #[tokio::test] - async fn recording_redirects_keep_the_proxy_and_resume_headers() -> Result<(), Box> { - use tokio::io::AsyncWriteExt; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let local_url = reqwest::Url::parse(&format!("http://{}/capture", listener.local_addr()?))?; - let local_task = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await?; - let request = read_request(&mut socket).await?; - socket.write_all(b"HTTP/1.1 302 Found\r\nLocation: http://recording-origin.invalid/start\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await?; - Ok::<_, std::io::Error>(request) - }); - let proxy = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let proxy_url = format!("http://{}", proxy.local_addr()?); - let proxy_task = tokio::spawn(async move { - let mut requests = Vec::new(); - for response in [ - "HTTP/1.1 307 Temporary Redirect\r\nLocation: /finish\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", - "HTTP/1.1 302 Found\r\nLocation: http://recording-cdn.invalid/end\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", - "HTTP/1.1 206 Partial Content\r\nContent-Range: bytes 4-9/10\r\nContent-Length: 6\r\nConnection: close\r\n\r\n456789", - ] { - let (mut socket, _) = proxy.accept().await?; - let request = read_request(&mut socket).await?; - socket.write_all(response.as_bytes()).await?; - requests.push(request); - } - Ok::<_, std::io::Error>(requests) - }); - let config = app_config_with_listener(); - let mut updated = (*config.config.load_full()).clone(); - updated.proxy = Some(tuliprox_core::model::ProxyConfig { url: proxy_url, username: None, password: None }); - config.config.store(Arc::new(updated)); - let mut sensitive = reqwest::header::HeaderMap::new(); - sensitive.insert( - reqwest::header::AUTHORIZATION, - reqwest::header::HeaderValue::from_static("Bearer listener-secret"), - ); - sensitive.insert(reqwest::header::COOKIE, reqwest::header::HeaderValue::from_static("session=listener-secret")); - let local = tuliprox_core::utils::request::create_client(&config) - .no_proxy() - .redirect(reqwest::redirect::Policy::none()) - .default_headers(sensitive) - .build()?; - let recording_headers = std::collections::HashMap::from([ - ("Authorization".to_string(), "Bearer recording-secret".to_string()), - ("Cookie".to_string(), "session=recording-secret".to_string()), - ("X-Recording-Test".to_string(), "custom".to_string()), - ]); - let upstream = tuliprox_core::utils::request::create_client(&config) - .redirect(reqwest::redirect::Policy::none()) - .default_headers(tuliprox_core::utils::request::get_request_headers( - Some(&recording_headers), - None, - None, - None, - )) - .build()?; - let response = tokio::time::timeout( - Duration::from_secs(5), - super::send_download_request( - &local, - Some(&upstream), - &local_url, - 4, - &RwLock::new(RecordingControl::None), - &Notify::new(), - ), - ) - .await? - .map_err(|result| format!("request failed: {result:?}"))?; - assert_eq!(response.status(), reqwest::StatusCode::PARTIAL_CONTENT); - assert_eq!(response.bytes().await?.as_ref(), b"456789"); - let local_request = local_task.await??; - let upstream_requests = proxy_task.await??; - assert!(local_request.starts_with("get /capture ")); - assert!(upstream_requests[0].starts_with("get http://recording-origin.invalid/start ")); - assert!(upstream_requests[1].starts_with("get http://recording-origin.invalid/finish ")); - assert!(local_request.contains("authorization: bearer listener-secret"), "{local_request}"); - assert!(local_request.contains("cookie: session=listener-secret"), "{local_request}"); - assert!(!local_request.contains("recording-secret"), "{local_request}"); - assert!(upstream_requests[2].starts_with("get http://recording-cdn.invalid/end ")); - for (index, request) in upstream_requests.iter().enumerate() { - assert!(!request.contains("listener-secret"), "{request}"); - if index < 2 { - assert!(request.contains("authorization: bearer recording-secret"), "{request}"); - assert!(request.contains("cookie: session=recording-secret"), "{request}"); - } else { - assert!(!request.contains("recording-secret"), "{request}"); - } - assert!(request.contains("x-recording-test: custom"), "{request}"); - } - for request in std::iter::once(&local_request).chain(&upstream_requests) { - assert!(request.contains("range: bytes=4-"), "{request}"); - assert!(request.contains("accept-encoding: identity"), "{request}"); - assert!(request.contains(&shared::model::RECORDING_STREAM_USER_AGENT.to_ascii_lowercase()), "{request}"); - } - Ok(()) - } - - #[tokio::test] - async fn recording_redirects_without_upstream_client_strip_listener_credentials( - ) -> Result<(), Box> { - use tokio::io::AsyncWriteExt; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let upstream = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let local_url = reqwest::Url::parse(&format!("http://{}/capture", listener.local_addr()?))?; - let redirect = format!( - "HTTP/1.1 302 Found\r\nLocation: http://{}/capture\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", - upstream.local_addr()? - ); - let server = tokio::spawn(async move { - let mut requests = Vec::new(); - for (listener, response) in [ - (listener, redirect), - (upstream, "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string()), - ] { - let (mut socket, _) = listener.accept().await?; - let request = read_request(&mut socket).await?; - socket.write_all(response.as_bytes()).await?; - requests.push(request); - } - Ok::<_, std::io::Error>(requests) - }); - let mut headers = reqwest::header::HeaderMap::new(); - headers.insert( - reqwest::header::AUTHORIZATION, - reqwest::header::HeaderValue::from_static("Bearer listener-secret"), - ); - headers.insert(reqwest::header::COOKIE, reqwest::header::HeaderValue::from_static("session=listener-secret")); - headers.insert("x-recording-test", reqwest::header::HeaderValue::from_static("custom")); - let client = reqwest::Client::builder() - .no_proxy() - .redirect(reqwest::redirect::Policy::none()) - .default_headers(headers) - .build()?; - let response = tokio::time::timeout( - Duration::from_secs(5), - super::send_download_request( - &client, - None, - &local_url, - 0, - &RwLock::new(RecordingControl::None), - &Notify::new(), - ), - ) - .await? - .map_err(|result| format!("request failed: {result:?}"))?; - assert_eq!(response.status(), reqwest::StatusCode::OK); - let requests = server.await??; - assert!(requests[0].contains("authorization: bearer listener-secret"), "{}", requests[0]); - assert!(requests[0].contains("cookie: session=listener-secret"), "{}", requests[0]); - assert!(!requests[1].contains("listener-secret"), "{}", requests[1]); - assert!(requests[1].contains("x-recording-test: custom"), "{}", requests[1]); - Ok(()) - } - - #[tokio::test] - async fn serial_transfers_start_the_next_worker_without_waiting_for_a_tick( - ) -> Result<(), Box> { - use tokio::io::AsyncWriteExt; - let dir = TempDir::new()?; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let app = Arc::new(app_config_with_listener()); - let mut config = (*app.config.load_full()).clone(); - config.api.port = listener.local_addr()?.port(); - app.config.store(Arc::new(config)); - let queue = Arc::new(RecordingQueue::new()); - for index in 0..2 { - let mut task = scheduled_task(RecordingKind::Vod, chrono::Utc::now().timestamp(), 300); - task.uuid = format!("transfer-{index}"); - task.state = RecordingTaskState::Queued; - task.recording.source.virtual_id = (42 + index).to_string(); - task.file_dir = dir.path().to_path_buf(); - task.file_path = dir.path().join(format!("transfer-{index}.mp4")); - queue.queue.lock().await.push_back(task); - } - let (first_seen, first_received) = tokio::sync::oneshot::channel(); - let (release, released) = tokio::sync::oneshot::channel(); - let (second_seen, second_received) = tokio::sync::oneshot::channel(); - let server = tokio::spawn(async move { - for (seen, gate) in [(first_seen, Some(released)), (second_seen, None)] { - let (mut socket, _) = listener.accept().await?; - read_request(&mut socket).await?; - socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\nConnection: close\r\n\r\n").await?; - let _ = seen.send(()); - if let Some(gate) = gate { - let _ = gate.await; - } - socket.write_all(b"0123456789").await?; - } - Ok::<_, std::io::Error>(()) - }); - let cancellation = tokio_util::sync::CancellationToken::new(); - let capacity: Arc = StubCapacity::with_room(); - start_recording_scheduler( - app, - RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }), - &queue, - NoopSink, - capacity, - cancellation.clone(), - PathBuf::from("unused"), - ); - let result = async { - tokio::time::timeout(Duration::from_secs(5), first_received).await??; - assert_eq!(queue.active.read().await.len(), 1); - assert_eq!(queue.queue.lock().await.len(), 1); - tokio::time::sleep(Duration::from_millis(50)).await; - release.send(()).map_err(|()| "fixture stopped")?; - tokio::time::timeout(Duration::from_millis(500), second_received).await??; - tokio::time::timeout(Duration::from_secs(5), async { - while queue.finished.read().await.len() != 2 || queue.workers_running().await { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await?; - assert!(queue.finished.read().await.iter().all(|task| task.state == RecordingTaskState::Completed)); - server.await??; - Ok::<_, Box>(()) - } - .await; - cancellation.cancel(); - result - } - - #[tokio::test] - async fn blocked_serial_queue_does_not_clone_partitions_on_each_commit() -> Result<(), Box> { - let queue = RecordingQueue::new(); - queue.active.write().await.push(scheduled_task(RecordingKind::Vod, 0, 300)); - for index in 0..49 { - let mut task = scheduled_task(RecordingKind::Vod, 0, 300); - task.uuid = format!("waiting-{index}"); - task.recording.source.virtual_id = index.to_string(); - task.state = RecordingTaskState::Queued; - queue.queue.lock().await.push_back(task); - } - for index in 0..100 { - let mut done = scheduled_task(RecordingKind::Vod, 0, 300); - done.uuid = format!("completed-{index}"); - done.recording.source.virtual_id = (100 + index).to_string(); - done.state = RecordingTaskState::Completed; - queue.finished.write().await.push(done); - } - for size in 0..3 { - crate::recording::recording_queue::mutate(&queue, |candidate| { - candidate.active[0].size = size; - Ok(()) - }) - .await?; - // Promotion has no reason to read this partition; a full snapshot does. - let scheduled = queue.scheduled.write().await; - assert!(!tokio::time::timeout(Duration::from_millis(200), super::promote_ready_downloads(&queue)).await??); - drop(scheduled); - } - assert_eq!(queue.queue.lock().await.len(), 49); - Ok(()) - } - - #[tokio::test] - async fn idle_scheduler_does_not_snapshot_recording_history() -> Result<(), Box> { - let queue = Arc::new(RecordingQueue::new()); - let history = queue.finished.write().await; - let capacity: Arc = StubCapacity::with_room(); - let config = RecordingConfig::from(&shared::model::RecordingConfigDto::default()); - tokio::time::timeout(Duration::from_millis(200), async { - queue.promote_due_scheduled_now().await; - ensure_recording_worker_running( - &app_config_with_listener(), - &config, - &queue, - &NoopSink, - &capacity, - Path::new("unused"), - ) - .await - }) - .await??; - drop(history); - Ok(()) - } - - #[tokio::test] - async fn vod_and_series_transfers_reach_the_local_listener_past_a_configured_proxy() { - for kind in [RecordingKind::Vod, RecordingKind::Series] { - let dir = tempfile::TempDir::new().expect("tempdir"); - let (fixture_url, server) = serve_range_fixture(true); - let app_config = app_config_with_listener(); - let mut config = (*app_config.config.load_full()).clone(); - config.api.port = fixture_url.port().expect("fixture port"); - // Nothing listens here: a transfer routed through the proxy fails. - config.proxy = Some(tuliprox_core::model::ProxyConfig { - url: "http://127.0.0.1:9".to_string(), - username: None, - password: None, - }); - app_config.config.store(Arc::new(config)); - - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let mut transfer = scheduled_task(kind, chrono::Utc::now().timestamp(), 300); - transfer.state = RecordingTaskState::Queued; - transfer.input_name = Some(Arc::from("provider")); - transfer.recording.source.virtual_id = "42".to_string(); - transfer.file_dir = dir.path().to_path_buf(); - transfer.file_path = dir.path().join("transfer.mp4"); - let persisted = RecordingQueue::to_persisted(&transfer); - crate::recording::recording_queue::mutate(&queue, move |candidate| { - candidate.queue.push(persisted.clone()); - Ok(()) - }) - .await - .expect("seed"); - - let capacity: Arc = StubCapacity::with_room() as Arc; - ensure_recording_worker_running( - &app_config, - &RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }), - &queue, - &NoopSink, - &capacity, - Path::new(crate::recording::recording_worker::FFMPEG_BINARY), - ) - .await - .expect("worker started"); - - let settled = tokio::time::timeout(Duration::from_secs(10), async { - loop { - if let Some(done) = queue.finished.read().await.first().cloned() { - break done; - } - tokio::time::sleep(Duration::from_millis(20)).await; - } - }) - .await - .expect("transfer settled"); - assert_eq!(settled.state, RecordingTaskState::Completed, "{kind:?}: {:?}", settled.error); - let request = server.join().expect("fixture server"); - assert!(request.starts_with("GET /api/v1/playlist/recording/"), "{request}"); - } - } - - #[tokio::test] - async fn a_recording_whose_disk_filled_while_it_waited_never_opens_a_destination() { - // Admission happened when the request was accepted; this recording - // then waited. By the time it reaches the front of the queue the - // space it was admitted against is gone. Without the recheck the - // first sign of that is a write failure part-way through. - let dir = tempfile::TempDir::new().expect("tempdir"); - let recordings_dir = dir.path().join("recordings"); - std::fs::create_dir_all(&recordings_dir).expect("create recording dir"); - let log = dir.path().join("spawns.log"); - let script = counting_ffmpeg(dir.path(), &log); - - let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); - let now = chrono::Utc::now().timestamp(); - let mut capture = scheduled_task(RecordingKind::Live, now, 1_800); - capture.uuid = "live".to_string(); - capture.state = RecordingTaskState::Queued; - capture.input_name = Some(Arc::from("provider")); - capture.recording.source.virtual_id = "42".to_string(); - capture.recording.reserved_bytes = 1_024; - capture.file_dir.clone_from(&recordings_dir); - capture.file_path = recordings_dir.join("capture.ts"); - let persisted = RecordingQueue::to_persisted(&capture); - crate::recording::recording_queue::mutate(&queue, move |candidate| { - candidate.queue.push(persisted.clone()); - Ok(()) - }) - .await - .expect("seed"); - - // The safety margin stands in for a disk that filled up: it drives - // headroom to zero without needing a real full filesystem. - let app_config = app_config_with_listener(); - let mut rec_cfg = - RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }); - rec_cfg.directory = recordings_dir.to_string_lossy().into_owned(); - rec_cfg.disk = Some(tuliprox_core::model::RecordingDiskConfig { - high_water_percent: None, - low_water_percent: None, - cleanup_interval_secs: None, - safety_bytes: Some(u64::MAX), - }); - let mut config = tuliprox_core::model::Config::clone(&app_config.config.load()); - config.video = Some(tuliprox_core::model::VideoConfig { - extensions: Vec::new(), - web_search: None, - recording: Some(rec_cfg.clone()), - }); - app_config.config.store(Arc::new(config)); - - let stub = StubCapacity::with_room(); - let capacity: Arc = Arc::clone(&stub) as Arc; - ensure_recording_worker_running(&app_config, &rec_cfg, &queue, &NoopSink, &capacity, &script) - .await - .expect("worker started"); - - let settled = tokio::time::timeout(Duration::from_secs(10), async { - loop { - if let Some(done) = queue.finished.read().await.first().cloned() { - break done; - } - tokio::time::sleep(Duration::from_millis(20)).await; - } - }) - .await; - let Ok(settled) = settled else { - panic!( - "never settled: active={:?} spawns={}", - queue.active.read().await.first().map(|active| (active.state, active.error.clone())), - spawn_count(&log), - ); - }; - - assert_eq!(settled.state, RecordingTaskState::Failed, "{:?}", settled.error); - assert_eq!(settled.error.as_deref(), Some(DISK_GONE_BEFORE_START), "and it says which resource ran out"); - assert_eq!(spawn_count(&log), 0, "the encoder was never started"); - assert!(!capture.file_path.exists(), "and no destination was opened"); - assert_eq!(stub.release_count(), 1, "the provider slot it had was given back"); - assert_eq!(settled.recording.reserved_bytes, 0, "and it is not still holding disk"); - } -} +use self::http::finalize_http_transfer; +#[cfg(test)] +use self::http::http_transfer_path; +#[cfg(test)] +use self::http::send_download_request; +#[cfg(test)] +use self::worker::requeue_active_download_for_capacity_wait; +#[cfg(test)] +use self::worker::start_recording_scheduler; +pub use self::worker::{ensure_recording_worker_running, resume_recording_worker_if_needed, spawn_recording_services}; +use self::{ + completion::{ + active_download_snapshot_for_worker, cancel_active_and_promote, fail_active_download, + finish_active_and_promote, promote_ready_downloads, set_active_download_state, take_active, + update_active_download_for_worker, write_completion_sidecar, + }, + control::{ + continue_after_pause, current_download_control, handle_download_control, + handle_download_control_without_writer, preemption_reason_for, recording_deadline_instant, + should_exit_worker_after_preempt, + }, + http::{download_file, recording_execution_download}, + notifications::{ + broadcast_required_worker_mutation, broadcast_worker_mutation, mark_recording_metadata_notification, + publish_recording_change, publish_recording_progress, refresh_recording_progress, + spawn_recording_notification_after_persist, RecordingNotificationPlan, + }, + provider::{ + acquire_result_after_wait, acquire_result_for_control, background_download_should_wait, + capacities_have_free_slot, commit_acquired_download, wait_for_provider_slot, ProviderAcquireResult, + ProviderCapacities, + }, + quota::{charge_waiting_siblings, refused_before_start, QuotaGate}, + retry::{ + classify_download_open_error, classify_download_stream_io_error, prepare_active_retry, + requeue_active_download_for_retry, RetryCommit, + }, + worker::DownloadExecutionResult, +}; diff --git a/backend/dvr/src/recording/recording_transfer/completion.rs b/backend/dvr/src/recording/recording_transfer/completion.rs new file mode 100644 index 000000000..3857d117a --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/completion.rs @@ -0,0 +1,182 @@ +use super::{charge_waiting_siblings, RecordingNotificationPlan}; +use crate::recording::{ + recording_queue::{ + mutate_optional, PersistedRecordingQueue, PersistedRecordingTask, QueueMutationError, RecordingControl, + RecordingQueue, RecordingTask, RecordingTaskState, + }, + recording_sidecar, +}; +use log::warn; +use tokio::sync::RwLock; + +pub(super) async fn active_download_snapshot_for_worker( + active: &RwLock>, + worker_uuid: &str, +) -> Option { + active.read().await.iter().find(|task| task.uuid == worker_uuid).cloned() +} + +pub(super) async fn update_active_download_for_worker( + active: &RwLock>, + worker_uuid: &str, + update: F, +) -> bool +where + F: FnOnce(&mut RecordingTask) -> bool, +{ + let mut active = active.write().await; + let Some(task) = active.iter_mut().find(|task| task.uuid == worker_uuid) else { + return false; + }; + update(task) +} + +pub(super) async fn set_active_download_state( + download_queue: &RecordingQueue, + uuid: &str, + state: RecordingTaskState, + error: Option, + paused: bool, +) -> Result { + Ok(mutate_optional(download_queue, |candidate| { + let Some(task) = candidate.active.iter_mut().find(|active| active.uuid == uuid) else { + return Ok(None); + }; + if task.state == state && task.error == error && task.paused == paused && !task.finished { + return Ok(None); + } + task.state = state; + task.error = error; + task.paused = paused; + task.finished = false; + Ok(Some(true)) + }) + .await? + .unwrap_or(false)) +} + +/// Take the active task out of the candidate, but only when it is still the +/// one this worker is executing. +pub(super) fn take_active(candidate: &mut PersistedRecordingQueue, uuid: &str) -> Option { + candidate.active.iter().position(|active| active.uuid == uuid).map(|index| candidate.active.remove(index)) +} + +/// Write the sidecar for the recording that just finished. +/// +/// Best effort by design: the recording is on disk and committed either way, +/// and refusing to complete a transfer because a descriptive file could not be +/// written would trade a real recording for a diagnostic. +pub(super) async fn write_completion_sidecar(download_queue: &RecordingQueue, uuid: &str, measured_bytes: u64) { + let Some(task) = download_queue.active.read().await.iter().find(|task| task.uuid == uuid).cloned() else { + return; + }; + let Some(relative_path) = task.recording.relative_path.clone() else { + return; + }; + let persisted = RecordingQueue::to_persisted(&task); + let sidecar = recording_sidecar::RecordingSidecar { + materialization_id: tuliprox_repository::recording_repository::materialization_id_for(&persisted), + media_identity: persisted.media_identity, + kind: task.kind, + relative_path, + size_bytes: measured_bytes, + completed_at: chrono::Utc::now().timestamp(), + }; + if let Err(error) = recording_sidecar::write_sidecar(&task.file_path, &sidecar).await { + warn!("Could not write the sidecar for {}: {error}", task.file_path.display()); + } +} + +/// Commit a terminal failure for the active task and move on. +pub(super) async fn fail_active_download( + download_queue: &RecordingQueue, + uuid: &str, + reason: &str, +) -> Result { + Ok(mutate_optional(download_queue, |candidate| { + let Some(mut failed) = take_active(candidate, uuid) else { + return Ok(None); + }; + failed.finished = true; + failed.paused = false; + failed.next_retry_at = None; + failed.state = RecordingTaskState::Failed; + failed.error = Some(reason.to_string()); + failed.recording.reserved_bytes = 0; + candidate.finished.push(failed); + crate::recording::recording_queue::promote_from_queue(candidate); + Ok(Some(true)) + }) + .await? + .unwrap_or(false)) +} + +pub(super) async fn promote_ready_downloads(download_queue: &RecordingQueue) -> Result { + if !download_queue.has_promotable_queued().await { + return Ok(false); + } + Ok(mutate_optional(download_queue, |candidate| { + let queued = candidate.queue.len(); + while crate::recording::recording_queue::promote_from_queue(candidate).is_some() {} + Ok((candidate.queue.len() != queued).then_some(true)) + }) + .await? + .unwrap_or(false)) +} + +/// Move this worker's task to `finished` and promote the next runnable entry. +/// +/// With `waiting_quota`, the task finished a file other entries may be +/// waiting to attach to. Its measured size is checked against each of their +/// quotas before promotion attaches them: a transfer of unknown size cannot +/// be checked when it starts, and attaching charges the whole file. +pub(super) async fn finish_active_and_promote( + download_queue: &RecordingQueue, + uuid: &str, + waiting_quota: Option<&crate::recording::recording_quota::QuotaLimits>, + finish: F, +) -> Result, QueueMutationError> +where + F: FnOnce(&mut PersistedRecordingTask) -> RecordingNotificationPlan, +{ + download_queue + .mutate_optional_and_clear_control(uuid, RecordingControl::Cancel, |candidate| { + let Some(mut active) = take_active(candidate, uuid) else { + return Ok(None); + }; + let notification = finish(&mut active); + let media = active.media_identity.clone(); + let measured = active.recording.measured_bytes; + candidate.finished.push(active); + if let Some(limits) = waiting_quota { + let _ = charge_waiting_siblings(candidate, &media, measured, limits); + } + crate::recording::recording_queue::promote_from_queue(candidate); + Ok(Some(notification)) + }) + .await +} + +pub(super) async fn cancel_active_and_promote( + download_queue: &RecordingQueue, + uuid: &str, +) -> Result { + Ok(download_queue + .mutate_optional_and_clear_control(uuid, RecordingControl::Cancel, |candidate| { + let Some(mut active) = take_active(candidate, uuid) else { + return Ok(None); + }; + active.finished = true; + active.paused = false; + active.next_retry_at = None; + active.error.get_or_insert_with(|| "Cancelled by user".to_string()); + active.state = RecordingTaskState::Cancelled; + // Nothing more will be written, so the space stops being spoken for. + active.recording.reserved_bytes = 0; + candidate.finished.push(active); + crate::recording::recording_queue::promote_from_queue(candidate); + Ok(Some(true)) + }) + .await? + .unwrap_or(false)) +} diff --git a/backend/dvr/src/recording/recording_transfer/control.rs b/backend/dvr/src/recording/recording_transfer/control.rs new file mode 100644 index 000000000..57cbc622b --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/control.rs @@ -0,0 +1,70 @@ +use super::{DownloadExecutionResult, DOWNLOAD_PREEMPTED_REASON, RECORDING_PREEMPTED_REASON}; +use crate::recording::recording_queue::{RecordingControl, RecordingQueue, RecordingTask}; +use shared::model::RecordingKind; +use tokio::{ + io::{AsyncWrite, AsyncWriteExt}, + sync::RwLock, + time::{Duration, Instant}, +}; + +pub(super) fn current_download_control(control_signal: &RwLock) -> RecordingControl { + control_signal.try_read().map_or(RecordingControl::None, |control| *control) +} + +pub(super) fn should_exit_worker_after_preempt(control: RecordingControl) -> bool { + control == RecordingControl::Restart +} + +/// Close the writer for a pause, cancel or restart, so everything received +/// so far is on disk before the worker reports the outcome. +pub(super) async fn handle_download_control( + control: RecordingControl, + buf_writer: &mut W, +) -> Option +where + W: AsyncWrite + Unpin, +{ + let outcome = handle_download_control_without_writer(control)?; + let closed = async { + buf_writer.flush().await?; + buf_writer.shutdown().await + }; + if let Err(err) = closed.await { + return Some(DownloadExecutionResult::Failed(err.to_string())); + } + Some(outcome) +} + +pub(super) fn handle_download_control_without_writer(control: RecordingControl) -> Option { + match control { + RecordingControl::Pause => Some(DownloadExecutionResult::Paused), + RecordingControl::Cancel => Some(DownloadExecutionResult::Cancelled), + RecordingControl::Restart => Some(DownloadExecutionResult::Preempted), + RecordingControl::None => None, + } +} + +pub(super) fn recording_deadline_instant(task: &RecordingTask) -> Option { + if task.kind != RecordingKind::Live { + return None; + } + let (start_at, duration_secs) = task.scheduled_start().zip(task.scheduled_duration_secs())?; + let deadline_ts = start_at.saturating_add(i64::try_from(duration_secs).unwrap_or(i64::MAX)); + let now_ts = chrono::Utc::now().timestamp(); + if now_ts >= deadline_ts { + return Some(Instant::now()); + } + let remaining_secs = u64::try_from(deadline_ts.saturating_sub(now_ts)).ok()?; + Some(Instant::now() + Duration::from_secs(remaining_secs)) +} + +pub(super) async fn continue_after_pause(download_queue: &RecordingQueue, uuid: &str) -> bool { + download_queue.active.read().await.iter().any(|task| task.uuid == uuid && !task.paused && !task.finished) +} + +pub(super) fn preemption_reason_for(download: &RecordingTask) -> &'static str { + match download.kind { + RecordingKind::Vod | RecordingKind::Series => DOWNLOAD_PREEMPTED_REASON, + RecordingKind::Live => RECORDING_PREEMPTED_REASON, + } +} diff --git a/backend/dvr/src/recording/recording_transfer/http.rs b/backend/dvr/src/recording/recording_transfer/http.rs new file mode 100644 index 000000000..ae8c5bd94 --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/http.rs @@ -0,0 +1,497 @@ +use super::{ + classify_download_open_error, classify_download_stream_io_error, current_download_control, handle_download_control, + handle_download_control_without_writer, publish_recording_change, publish_recording_progress, + update_active_download_for_worker, DownloadExecutionResult, QuotaGate, DOWNLOAD_CONTROL_POLL_INTERVAL, + DOWNLOAD_PROGRESS_LOG_BYTES, DOWNLOAD_PROGRESS_LOG_INTERVAL, DOWNLOAD_SNAPSHOT_UPDATE_BYTES, + DOWNLOAD_SNAPSHOT_UPDATE_INTERVAL, QUOTA_EXCEEDED_DURING_TRANSFER, RANGE_UNSUPPORTED_ERROR, +}; +use crate::{ + http_transfer::{ + compute_total_size, is_retryable_status, validate_resume_response, ResponseSnapshot, ResumeValidationError, + ResumeValidator, + }, + recording::{ + recording_queue::{RecordingControl, RecordingTask}, + recording_url::build_stable_recording_url, + recording_worker::recording_partial_path, + }, +}; +use futures::stream::TryStreamExt; +use log::{debug, info, warn}; +use shared::{error::to_io_error, model::EventSink, utils::bytes_to_megabytes}; +use std::sync::Arc; +use tokio::{ + fs, + io::AsyncWriteExt, + sync::{Notify, RwLock}, + time, + time::Instant, +}; +use tuliprox_core::{ + model::AppConfig, + utils::{async_file_writer, IO_BUFFER_SIZE}, +}; + +pub(super) fn recording_execution_download( + app_config: &AppConfig, + task: &RecordingTask, + provider_handle: Option<&tuliprox_core::model::ProviderHandle>, +) -> Result { + let source = &task.recording.source; + let virtual_id = source.virtual_id.parse::().map_err(|_| "Recording source virtual id invalid".to_string())?; + let url = build_stable_recording_url( + app_config, + &source.target_id, + &source.input_name, + virtual_id, + source.cluster, + provider_handle.map(|handle| handle.allocation_id), + ) + .ok_or_else(|| "Recording execution URL unavailable".to_string())?; + let mut execution = task.clone(); + execution.url = reqwest::Url::parse(&url).map_err(|_| "Recording execution URL invalid".to_string())?; + Ok(execution) +} + +/// The validators a later resume can be checked against. +/// +/// A weak `ETag` is discarded: it only promises semantic equivalence, so it +/// cannot prove the bytes on the far side of an interruption are the same +/// ones. `Last-Modified` is the fallback, and is only meaningful when no +/// strong tag was offered. +fn capture_resume_validators(response: &reqwest::Response) -> (Option, Option) { + let headers = response.headers(); + let etag = headers + .get(reqwest::header::ETAG) + .and_then(|value| value.to_str().ok()) + .and_then(crate::http_transfer::strong_etag) + .map(str::to_owned); + if etag.is_some() { + return (etag, None); + } + let last_modified = headers + .get(reqwest::header::LAST_MODIFIED) + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.trim().is_empty()) + .map(str::to_owned); + (None, last_modified) +} + +pub(super) async fn send_download_request( + client: &reqwest::Client, + upstream_client: Option<&reqwest::Client>, + url: &reqwest::Url, + offset: u64, + control_signal: &RwLock, + control_notify: &Notify, +) -> Result { + if let Some(result) = handle_download_control_without_writer(current_download_control(control_signal)) { + return Err(result); + } + + let send = async { + let mut destination = url.clone(); + let mut upstream_origin = None; + for redirects in 0..=10 { + let hop_client = + if destination.origin() == url.origin() { client } else { upstream_client.unwrap_or(client) }; + // Resume offsets refer to bytes on disk, so every hop must preserve encoding. + let mut request = hop_client + .get(destination.clone()) + .header(reqwest::header::USER_AGENT, shared::model::RECORDING_STREAM_USER_AGENT) + .header(reqwest::header::ACCEPT_ENCODING, "identity"); + if offset > 0 { + request = request.header(reqwest::header::RANGE, format!("bytes={offset}-")); + } + if destination.origin() != url.origin() { + let trusted_origin = upstream_origin.get_or_insert_with(|| destination.origin()); + if upstream_client.is_none() || destination.origin() != *trusted_origin { + // Listener credentials stay local; provider credentials stay + // on the first upstream origin throughout the redirect chain. + request = request.header(reqwest::header::AUTHORIZATION, "").header(reqwest::header::COOKIE, ""); + } + } + let response = request.send().await.map_err(|error| classify_download_open_error(&destination, &error))?; + if !matches!(response.status().as_u16(), 301 | 302 | 303 | 307 | 308) { + return Ok(response); + } + let Some(location) = response.headers().get(reqwest::header::LOCATION) else { + return Ok(response); + }; + let next = location + .to_str() + .ok() + .and_then(|value| response.url().join(value).ok()) + .filter(|value| matches!(value.scheme(), "http" | "https")); + let Some(next) = next else { + return Err(DownloadExecutionResult::Failed("Invalid recording redirect location".to_string())); + }; + if redirects == 10 { + return Err(DownloadExecutionResult::Failed("Recording redirect limit exceeded".to_string())); + } + destination = next; + } + Err(DownloadExecutionResult::Failed("Recording redirect limit exceeded".to_string())) + }; + tokio::pin!(send); + loop { + tokio::select! { + biased; + () = control_notify.notified() => { + if let Some(result) = handle_download_control_without_writer(*control_signal.read().await) { + return Err(result); + } + } + response = &mut send => return response, + } + } +} + +pub(super) fn http_transfer_path(task: &RecordingTask) -> std::path::PathBuf { + if task.kind.is_resumable() { + recording_partial_path(&task.file_path) + } else { + task.file_path.clone() + } +} + +/// Publish a completed transfer at its final path. +/// +/// Idempotent: linking is the only step that is not naturally repeatable. A +/// crash between the link and the partial's removal leaves the final file in +/// place, and finalizing again accepts it instead of failing the recording. +pub(super) async fn finalize_http_transfer( + final_path: &std::path::Path, + transfer_path: &std::path::Path, +) -> std::io::Result<()> { + if transfer_path == final_path { + return Ok(()); + } + fs::File::open(transfer_path).await?.sync_all().await?; + match fs::hard_link(transfer_path, final_path).await { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + // Path reservation gives this recording sole claim on `final_path`, + // so something already there is this task's own earlier link. A + // size mismatch means that reasoning does not hold; say so rather + // than publish bytes that were never checked. + let published = fs::metadata(final_path).await?.len(); + let staged = fs::metadata(transfer_path).await?.len(); + if published != staged { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!( + "{} already exists with {published} bytes, but the transfer staged {staged}", + final_path.display() + ), + )); + } + } + Err(error) => return Err(error), + } + if let Err(error) = fs::remove_file(transfer_path).await { + warn!("Finalized {} but could not remove partial {}: {error}", final_path.display(), transfer_path.display()); + } + Ok(()) +} + +#[allow(clippy::too_many_lines, clippy::too_many_arguments)] +pub(super) async fn download_file( + active: Arc>>, + file_download: RecordingTask, + client: &reqwest::Client, + upstream_client: Option<&reqwest::Client>, + control_signal: Arc>, + control_notify: Arc, + event_manager: Option<&E>, + quota_gate: Option>, +) -> DownloadExecutionResult { + let worker_uuid = file_download.uuid.as_str(); + let url = file_download.url.clone(); + let file_path = http_transfer_path(&file_download); + let existing_size = tokio::fs::metadata(&file_path).await.map_or(0, |metadata| metadata.len()); + let response_result = + send_download_request(client, upstream_client, &url, existing_size, &control_signal, &control_notify).await; + + match response_result { + Ok(response) => { + if existing_size > 0 { + // The validators were captured when the first byte was written. + // Without them a provider that replaces the file between an + // interruption and the resume returns a well-formed 206 whose + // bytes belong to a different resource, and they are appended + // to the old partial. + let validator = ResumeValidator { + expected_offset: existing_size, + expected_total: file_download.total_size, + expected_etag: file_download.recording.resume_etag.clone(), + expected_last_modified: file_download.recording.resume_last_modified.clone(), + }; + match validate_resume_response(&ResponseSnapshot::from_response(&response), &validator) { + Ok(()) => {} + Err(ResumeValidationError::Unsatisfiable { complete: true }) => { + return match finalize_http_transfer(&file_download.file_path, &file_path).await { + Ok(()) => DownloadExecutionResult::Completed, + Err(error) => DownloadExecutionResult::Failed(format!( + "Could not finalize {}: {error}", + file_download.file_path.display() + )), + }; + } + Err(ResumeValidationError::IgnoredRange) => { + return DownloadExecutionResult::Failed(RANGE_UNSUPPORTED_ERROR.to_string()); + } + Err(error) => { + return DownloadExecutionResult::Failed(format!( + "Cannot resume download at byte {existing_size} for {url}: {error}; partial file preserved" + )); + } + } + } + let status = response.status(); + if !status.is_success() && status != reqwest::StatusCode::PARTIAL_CONTENT { + if is_retryable_status(status) { + return DownloadExecutionResult::Retryable(format!( + "Download request failed for {url} with transient HTTP {status}" + )); + } + return DownloadExecutionResult::Failed(format!( + "Download request failed for {url} with HTTP {status}" + )); + } + let is_resume = status == reqwest::StatusCode::PARTIAL_CONTENT; + + let total_size = compute_total_size(&response, existing_size); + let captured = capture_resume_validators(&response); + + if total_size.is_some() || existing_size == 0 { + update_active_download_for_worker(&active, worker_uuid, |download| { + if let Some(total) = total_size { + download.total_size = Some(total); + } + // Only a fresh transfer may set the validators; a resume + // must keep the ones its partial was written against. + if existing_size == 0 { + download.recording.resume_etag.clone_from(&captured.0); + download.recording.resume_last_modified.clone_from(&captured.1); + } + true + }) + .await; + } + + // The size is known from here on, or known to be unknown. + let byte_cap = match quota_gate { + Some(gate) => match gate.admit_size(worker_uuid, total_size, existing_size).await { + Ok(admitted) => { + if admitted.siblings_refused { + if let Some(event_manager) = event_manager { + publish_recording_change(event_manager); + } + } + admitted.byte_cap + } + Err(reason) => return DownloadExecutionResult::Failed(reason), + }, + None => None, + }; + + match fs::create_dir_all(&file_download.file_dir).await { + Ok(()) => { + let mut open_options = tokio::fs::OpenOptions::new(); + let file_mode = if existing_size > 0 && is_resume { + open_options.append(true) + } else { + open_options.write(true).create(true).truncate(true) + }; + + if let Some(file_path_str) = file_path.to_str() { + info!("{} {}", if is_resume { "Resuming" } else { "Downloading" }, file_path_str); + match file_mode.open(&file_path).await { + Ok(file) => { + let mut buf_writer = async_file_writer(file); + let mut downloaded: u64 = if is_resume { existing_size } else { 0 }; + let mut stream = response.bytes_stream(); + let mut write_counter = 0; + let mut saw_first_chunk = existing_size > 0; + let mut last_progress_log_at = Instant::now(); + let mut last_progress_logged_bytes = downloaded; + let mut last_snapshot_update_at = Instant::now(); + let mut last_snapshot_update_bytes = downloaded; + let mut control_poll = time::interval(DOWNLOAD_CONTROL_POLL_INTERVAL); + control_poll.set_missed_tick_behavior(time::MissedTickBehavior::Skip); + control_poll.tick().await; + loop { + let next_item = tokio::select! { + biased; + () = control_notify.notified() => { + if let Some(result) = + handle_download_control(*control_signal.read().await, &mut buf_writer) + .await + { + return result; + } + continue; + } + _ = control_poll.tick() => { + if let Some(result) = handle_download_control( + current_download_control(&control_signal), + &mut buf_writer, + ) + .await + { + return result; + } + continue; + } + next_item = stream.try_next() => next_item.map_err(to_io_error), + }; + + match next_item { + Ok(item) => { + if let Some(chunk) = item { + match buf_writer.write_all(&chunk).await { + Ok(()) => { + write_counter += chunk.len(); + if write_counter >= IO_BUFFER_SIZE { + if let Err(err) = buf_writer.flush().await { + return DownloadExecutionResult::Failed( + err.to_string(), + ); + } + write_counter = 0; + } + + downloaded += chunk.len() as u64; + if byte_cap.is_some_and(|cap| downloaded > cap) { + // Unknown size, so the quota is enforced as + // the bytes arrive. + let _ = buf_writer.flush().await; + let _ = buf_writer.shutdown().await; + return DownloadExecutionResult::Failed( + QUOTA_EXCEEDED_DURING_TRANSFER.to_string(), + ); + } + if saw_first_chunk { + let now = Instant::now(); + let should_log_progress = now + .duration_since(last_progress_log_at) + >= DOWNLOAD_PROGRESS_LOG_INTERVAL + || downloaded + .saturating_sub(last_progress_logged_bytes) + >= DOWNLOAD_PROGRESS_LOG_BYTES; + if should_log_progress { + match total_size { + Some(total) if total > 0 => { + let percent = downloaded + .saturating_mul(100) + .checked_div(total) + .unwrap_or(0) + .min(100); + debug!( + "Download progress for {file_path_str}: {}MB / {}MB ({}%)", + bytes_to_megabytes(downloaded), + bytes_to_megabytes(total), + percent + ); + } + _ => { + debug!( + "Download progress for {file_path_str}: {}MB received", + bytes_to_megabytes(downloaded) + ); + } + } + last_progress_log_at = now; + last_progress_logged_bytes = downloaded; + } + } else { + saw_first_chunk = true; + info!( + "Receiving download data for {file_path_str}: {}MB received", + bytes_to_megabytes(downloaded) + ); + last_progress_log_at = Instant::now(); + last_progress_logged_bytes = downloaded; + } + let should_update_snapshot = (last_snapshot_update_bytes == 0 + && downloaded > 0) + || downloaded.saturating_sub(last_snapshot_update_bytes) + >= DOWNLOAD_SNAPSHOT_UPDATE_BYTES + || Instant::now().duration_since(last_snapshot_update_at) + >= DOWNLOAD_SNAPSHOT_UPDATE_INTERVAL; + if should_update_snapshot { + let changed = update_active_download_for_worker( + &active, + worker_uuid, + |download| { + download.size = downloaded; + true + }, + ) + .await; + if changed { + if let Some(event_manager) = event_manager { + publish_recording_progress(event_manager); + } + } + last_snapshot_update_at = Instant::now(); + last_snapshot_update_bytes = downloaded; + } + } + Err(err) => { + return DownloadExecutionResult::Failed(format!( + "Error while writing to file: {file_path_str} {err}" + )); + } + } + } else { + let megabytes = bytes_to_megabytes(downloaded); + info!("Downloaded {file_path_str}, filesize: {megabytes}MB"); + update_active_download_for_worker(&active, worker_uuid, |download| { + download.size = downloaded; + true + }) + .await; + if let Err(err) = buf_writer.flush().await { + return DownloadExecutionResult::Failed(err.to_string()); + } + if let Err(err) = buf_writer.shutdown().await { + return DownloadExecutionResult::Failed(err.to_string()); + } + return match finalize_http_transfer( + &file_download.file_path, + &file_path, + ) + .await + { + Ok(()) => DownloadExecutionResult::Completed, + Err(error) => DownloadExecutionResult::Failed(format!( + "Could not finalize {}: {error}", + file_download.file_path.display() + )), + }; + } + } + Err(err) => return classify_download_stream_io_error(file_path_str, &err), + } + } + } + Err(err) => DownloadExecutionResult::Failed(format!( + "Error while opening file: {file_path_str} {err}" + )), + } + } else { + DownloadExecutionResult::Failed("Error file-download file-path unknown".to_string()) + } + } + Err(err) => DownloadExecutionResult::Failed(format!( + "Error while creating directory for file: {} {}", + file_download.file_dir.to_str().unwrap_or("?"), + err + )), + } + } + Err(result) => result, + } +} diff --git a/backend/dvr/src/recording/recording_transfer/notifications.rs b/backend/dvr/src/recording/recording_transfer/notifications.rs new file mode 100644 index 000000000..11c2e781a --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/notifications.rs @@ -0,0 +1,141 @@ +use super::update_active_download_for_worker; +use crate::recording::{ + recording_notification::LifecycleEvent, + recording_notification_adapter::{build_marker, decide, message_for, DispatchDecision}, + recording_queue::{QueueMutationError, RecordingTask}, +}; +use shared::model::{EventMessage, EventSink, RecordingKind, RecordingMetadata}; +use std::sync::Arc; +use tokio::sync::RwLock; +use tuliprox_core::model::{AppConfig, MessageContent}; + +/// Publish a queue change. Sessions answer it by pulling an owner-filtered +/// snapshot; no task data is broadcast globally, so a session can never see +/// another user's recording. +pub(super) fn publish_recording_change(event_manager: &E) { + event_manager.emit(EventMessage::RecordingChanged); +} + +/// Announce that a running recording has grown. +/// +/// A separate event from [`publish_recording_change`]: a capture can produce +/// hundreds of these a second, and a session throttles them. A state +/// transition must never be throttled, so it does not come through here. +pub(super) fn publish_recording_progress(event_manager: &E) { + event_manager.emit(EventMessage::RecordingProgress); +} + +pub(super) fn broadcast_worker_mutation( + event_manager: &E, + result: Result, + action: &str, +) -> Result { + match result { + Ok(true) => { + publish_recording_change(event_manager); + Ok(true) + } + Ok(false) => Ok(false), + Err(err) => Err(QueueMutationError::new(format!("{action}: {err}"))), + } +} + +pub(super) fn broadcast_required_worker_mutation( + event_manager: &E, + result: Result, + action: &str, +) -> Result<(), QueueMutationError> { + if broadcast_worker_mutation(event_manager, result, action)? { + Ok(()) + } else { + Err(QueueMutationError::new(format!("{action}: active task changed"))) + } +} + +pub(super) async fn refresh_recording_progress( + active: &RwLock>, + worker_uuid: &str, + file_path: &std::path::Path, + event_manager: &E, +) { + let current_size = tokio::fs::metadata(file_path).await.map_or(0, |metadata| metadata.len()); + let changed = update_active_download_for_worker(active, worker_uuid, |task| { + if task.kind == RecordingKind::Live && task.size != current_size { + task.size = current_size; + true + } else { + false + } + }) + .await; + if changed { + publish_recording_progress(event_manager); + } +} + +pub(super) fn mark_recording_metadata_notification( + meta: &mut RecordingMetadata, + event: LifecycleEvent, + failure_reason: Option, +) -> RecordingNotificationPlan { + let is_admin_owner = meta.owner.user_id().is_builtin_admin(); + match decide(meta, event, chrono::Utc::now().timestamp(), is_admin_owner, failure_reason) { + DispatchDecision::PersistAndDeliver { payload, kind, attempted_at } => { + meta.notification_markers.push(build_marker(kind.clone(), attempted_at)); + RecordingNotificationPlan { + message: Some(MessageContent::RecordingLifecycle(message_for(event, &payload))), + } + } + DispatchDecision::AlreadyDelivered { .. } | DispatchDecision::Suppressed { .. } => { + RecordingNotificationPlan::empty() + } + } +} + +pub(super) struct RecordingNotificationPlan { + pub(super) message: Option, +} + +impl RecordingNotificationPlan { + pub(super) fn empty() -> Self { Self { message: None } } +} + +/// Hand a lifecycle notification off for delivery once its marker has +/// been persisted. +/// +/// The marker is written inside the queue-mutation boundary, so it is +/// durable before this runs; delivery must be durable too. The +/// notification outbox owns that: it persists the entry, retries per +/// channel with backoff, and dead-letters what it cannot deliver. +/// +/// A direct send is the fallback for paths that run before the supervisor +/// installs the outbox (notably unit tests), and for an outbox that refuses +/// the entry. +pub(super) fn spawn_recording_notification_after_persist( + app_config: &Arc, + client: &reqwest::Client, + plan: RecordingNotificationPlan, + persisted: bool, +) { + if !persisted { + return; + } + let Some(message) = plan.message else { + return; + }; + let event = tuliprox_core::model::NotificationEvent::from_content(&message); + // A full or closed outbox hands the event back; fall through to the + // direct send rather than dropping it outright. + let event = match crate::recording::recording_supervisor::notification_outbox() { + Some(outbox) => match outbox.enqueue(event) { + None => return, + Some(rejected) => rejected, + }, + None => event, + }; + let app_config = Arc::clone(app_config); + let client = client.clone(); + tokio::spawn(async move { + tuliprox_messaging::send_event(&app_config, &client, event).await; + }); +} diff --git a/backend/dvr/src/recording/recording_transfer/provider.rs b/backend/dvr/src/recording/recording_transfer/provider.rs new file mode 100644 index 000000000..1783f551e --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/provider.rs @@ -0,0 +1,111 @@ +use super::{mark_recording_metadata_notification, RecordingNotificationPlan}; +use crate::recording::{ + recording_notification::LifecycleEvent, + recording_queue::{ + mutate_optional, QueueMutationError, RecordingControl, RecordingQueue, RecordingTaskState, RecordingWaitOutcome, + }, +}; +use std::sync::Arc; +use tokio::{ + sync::{Notify, RwLock}, + time, + time::Instant, +}; +use tuliprox_core::model::RecordingConfig; + +pub(super) type ProviderCapacities = Vec<(Arc, usize, usize)>; + +pub(super) enum ProviderAcquireResult { + Acquired(Option), + Paused, + Cancelled, + Preempted, + /// A live capture waited for capacity until its broadcast window closed. + WindowClosed, +} + +pub(super) fn background_download_should_wait( + priority: i8, + capacities: &[(Arc, usize, usize)], + download_cfg: &RecordingConfig, +) -> bool { + if priority <= 0 || capacities.is_empty() { + return false; + } + + let background_limit = usize::from(download_cfg.max_background_per_provider); + let reserve_slots = usize::from(download_cfg.reserve_slots_for_users); + + let blocked_by_background_limit = + background_limit > 0 && capacities.iter().all(|(_, current, _)| *current >= background_limit); + let blocked_by_reserved_slots = reserve_slots > 0 + && capacities.iter().all(|(_, current, max)| *max > 0 && current.saturating_add(reserve_slots) >= *max); + + blocked_by_background_limit || blocked_by_reserved_slots +} + +pub(super) fn capacities_have_free_slot(capacities: &[(Arc, usize, usize)]) -> bool { + capacities.iter().any(|(_, current, max)| *max == 0 || current < max) +} + +/// Wait for a provider slot, giving up when a live window closes. +/// +/// Waiting past `scheduled_end` cannot produce the recording that was asked +/// for: the programme has finished. Only live work has a deadline; a transfer +/// waits as long as it takes. +pub(super) async fn wait_for_provider_slot( + download_queue: &RecordingQueue, + input_name: &Arc, + priority: i8, + control_signal: &RwLock, + control_notify: &Notify, + deadline: Option, +) -> Option { + let waiting = + download_queue.slot_waiters.wait(Some(Arc::clone(input_name)), priority, control_signal, control_notify); + match deadline { + Some(deadline) => time::timeout_at(deadline, waiting).await.ok(), + None => Some(waiting.await), + } +} + +/// What a control signal observed while acquiring a slot means; `None` keeps +/// trying. +pub(super) fn acquire_result_for_control(control: RecordingControl) -> Option { + match control { + RecordingControl::Cancel => Some(ProviderAcquireResult::Cancelled), + RecordingControl::Pause => Some(ProviderAcquireResult::Paused), + RecordingControl::Restart => Some(ProviderAcquireResult::Preempted), + RecordingControl::None => None, + } +} + +/// What the end of a provider-slot wait means; `None` (signalled) tries to +/// acquire again. A wait without an outcome ran into the live window's end. +pub(super) fn acquire_result_after_wait(outcome: Option) -> Option { + match outcome { + None => Some(ProviderAcquireResult::WindowClosed), + Some(RecordingWaitOutcome::Signalled) => None, + Some(RecordingWaitOutcome::Paused) => Some(ProviderAcquireResult::Paused), + Some(RecordingWaitOutcome::Cancelled) => Some(ProviderAcquireResult::Cancelled), + Some(RecordingWaitOutcome::Restarted) => Some(ProviderAcquireResult::Preempted), + } +} + +pub(super) async fn commit_acquired_download( + download_queue: &RecordingQueue, + uuid: &str, +) -> Result, QueueMutationError> { + mutate_optional(download_queue, |candidate| { + let Some(active) = candidate.active.iter_mut().find(|active| active.uuid == uuid) else { + return Ok(None); + }; + active.state = RecordingTaskState::Running; + active.error = None; + active.paused = false; + active.finished = false; + let notification = mark_recording_metadata_notification(&mut active.recording, LifecycleEvent::Started, None); + Ok(Some(notification)) + }) + .await +} diff --git a/backend/dvr/src/recording/recording_transfer/quota.rs b/backend/dvr/src/recording/recording_transfer/quota.rs new file mode 100644 index 000000000..8a271ceab --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/quota.rs @@ -0,0 +1,208 @@ +use super::{DISK_GONE_BEFORE_START, QUOTA_EXCEEDED_DURING_TRANSFER, QUOTA_GONE_BEFORE_START, SIBLING_QUOTA_EXCEEDED}; +use crate::recording::recording_queue::{ + mutate_optional, PersistedRecordingQueue, PersistedRecordingTask, RecordingQueue, RecordingTaskState, +}; +use std::{path::Path, sync::Arc}; +use tuliprox_core::model::AppConfig; + +/// What the size check at the start of a transfer decided. +pub(super) struct SizeAdmission { + /// Byte count the transfer may not exceed when its total is unknown. + pub(super) byte_cap: Option, + /// Waiting entries for the same media were refused for their own quota. + pub(super) siblings_refused: bool, +} + +/// Quota and disk admission once a transfer knows how large it is. +/// +/// A VOD or series request is admitted before its size is known, so it +/// reserves nothing up front. The first response carries the size, and this +/// is the first point where the charge can be checked: against the +/// transfer's own pool, against every entry waiting to attach to the same +/// file, and against the disk. +pub(super) struct QuotaGate<'a> { + pub(super) queue: &'a RecordingQueue, + pub(super) app_config: &'a AppConfig, +} + +impl QuotaGate<'_> { + pub(super) async fn admit_size( + &self, + uuid: &str, + total: Option, + on_disk: u64, + ) -> Result { + let config = self.app_config.config.load(); + let Some(recording_cfg) = config.recording() else { + return Ok(SizeAdmission { byte_cap: None, siblings_refused: false }); + }; + let limits = crate::recording::recording_service::quota_limits_from_config(recording_cfg.quota.as_ref()); + // Measured before the mutation: it is a syscall. + let free = crate::recording::recording_disk::free_bytes_for(Path::new(&recording_cfg.directory)); + let safety = recording_cfg.disk.as_ref().and_then(|disk| disk.safety_bytes).unwrap_or(0); + + let decided = mutate_optional(self.queue, |candidate| { + let Some(subject) = candidate.active.iter().find(|active| active.uuid == uuid) else { + return Ok(None); + }; + let pool = crate::recording::recording_quota::quota_pool_for_task(subject); + let media = subject.media_identity.clone(); + let used = used_by_others(candidate, uuid, &pool); + let Some(total) = total else { + let byte_cap = crate::recording::recording_quota::limit_for_pool(&pool, &limits) + .map(|limit| limit.saturating_sub(used)); + return Ok(Some(Ok(SizeAdmission { byte_cap, siblings_refused: false }))); + }; + if over_limit(&pool, used, total, &limits) { + return Ok(Some(Err(QUOTA_EXCEEDED_DURING_TRANSFER.to_string()))); + } + if let Some(free) = free { + let running = crate::recording::recording_disk::active_disk_reservations( + all_tasks(candidate).filter(|task| task.uuid != uuid), + ); + if matches!( + crate::recording::recording_disk::would_fit_on_disk( + free, + safety, + running, + total.saturating_sub(on_disk) + ), + crate::recording::recording_disk::DiskAdmission::Insufficient { .. } + ) { + return Ok(Some(Err(DISK_GONE_BEFORE_START.to_string()))); + } + } + if let Some(active) = candidate.active.iter_mut().find(|task| task.uuid == uuid) { + active.recording.reserved_bytes = active.recording.reserved_bytes.max(total); + } + let siblings_refused = charge_waiting_siblings(candidate, &media, total, &limits); + Ok(Some(Ok(SizeAdmission { byte_cap: None, siblings_refused }))) + }) + .await + .map_err(|err| format!("Could not record the transfer size: {err}"))?; + // The task left the active slot meanwhile; the worker finds out next. + decided.unwrap_or(Ok(SizeAdmission { byte_cap: None, siblings_refused: false })) + } +} + +fn all_tasks(candidate: &PersistedRecordingQueue) -> impl Iterator { + candidate + .queue + .iter() + .chain(candidate.scheduled.iter()) + .chain(candidate.active.iter()) + .chain(candidate.finished.iter()) +} + +fn used_by_others( + candidate: &PersistedRecordingQueue, + uuid: &str, + pool: &crate::recording::recording_quota::QuotaPool, +) -> u64 { + crate::recording::recording_quota::used_bytes_in_pool(all_tasks(candidate).filter(|task| task.uuid != uuid), pool) +} + +fn over_limit( + pool: &crate::recording::recording_quota::QuotaPool, + used: u64, + charge: u64, + limits: &crate::recording::recording_quota::QuotaLimits, +) -> bool { + matches!( + crate::recording::recording_quota::would_exceed(pool, used, charge, limits), + crate::recording::recording_quota::AdmissionOutcome::OverLimit { .. } + ) +} + +/// Charge every queued entry that will attach to this file with its size, +/// and refuse the ones whose own quota cannot take it. Returns whether any +/// entry was refused. +/// +/// They were admitted while the size was unknown. Attaching later copies the +/// whole file into their charge, so this is the last point where their quota +/// can still say no. +pub(super) fn charge_waiting_siblings( + candidate: &mut PersistedRecordingQueue, + media: &str, + total: u64, + limits: &crate::recording::recording_quota::QuotaLimits, +) -> bool { + if media.is_empty() { + return false; + } + let mut refused = Vec::new(); + let waiting: Vec = + candidate.queue.iter().filter(|task| task.media_identity == media).map(|task| task.uuid.clone()).collect(); + for uuid in waiting { + let Some(sibling) = candidate.queue.iter().find(|task| task.uuid == uuid) else { + continue; + }; + let pool = crate::recording::recording_quota::quota_pool_for_task(sibling); + let used = used_by_others(candidate, &uuid, &pool); + if over_limit(&pool, used, total, limits) { + refused.push(uuid); + } else if let Some(sibling) = candidate.queue.iter_mut().find(|task| task.uuid == uuid) { + sibling.recording.reserved_bytes = sibling.recording.reserved_bytes.max(total); + } + } + for uuid in &refused { + if let Some(index) = candidate.queue.iter().position(|task| &task.uuid == uuid) { + let mut failed = candidate.queue.remove(index); + failed.finished = true; + failed.paused = false; + failed.next_retry_at = None; + failed.state = RecordingTaskState::Failed; + failed.error = Some(SIBLING_QUOTA_EXCEEDED.to_string()); + failed.recording.reserved_bytes = 0; + candidate.finished.push(failed); + } + } + !refused.is_empty() +} + +/// Re-run admission for the recording that is about to open its +/// destination, returning the reason it can no longer start. +/// +/// Admission happened when the request was accepted, which can be a long +/// time before this point: the recording may have waited for provider +/// capacity, sat through a retry backoff, or been scheduled hours ahead. +/// The quota and the free space it was admitted against are not the ones +/// it is about to consume. Without this the first sign of a full disk is +/// a write failure part-way through a recording. +/// +/// The subject is excluded from both sums and then added back as the +/// candidate charge, so it is not counted twice. +pub(super) async fn refused_before_start( + download_queue: &Arc, + app_config: &AppConfig, + uuid: &str, +) -> Option<&'static str> { + let config = app_config.config.load(); + let recording_cfg = config.recording()?; + let (_, tasks) = download_queue.committed_snapshot().await; + let subject = tasks.iter().find(|task| task.uuid == uuid)?; + let charge = crate::recording::recording_quota::charge_for_task(subject); + let others = || tasks.iter().filter(|task| task.uuid != uuid); + + let limits = crate::recording::recording_service::quota_limits_from_config(recording_cfg.quota.as_ref()); + let pool = crate::recording::recording_quota::quota_pool_for_task(subject); + let used = crate::recording::recording_quota::used_bytes_in_pool(others(), &pool); + if matches!( + crate::recording::recording_quota::would_exceed(&pool, used, charge, &limits), + crate::recording::recording_quota::AdmissionOutcome::OverLimit { .. } + ) { + return Some(QUOTA_GONE_BEFORE_START); + } + + // An unmeasurable root is not grounds to refuse, exactly as at admission. + let free = crate::recording::recording_disk::free_bytes_for(Path::new(&recording_cfg.directory))?; + let safety = recording_cfg.disk.as_ref().and_then(|disk| disk.safety_bytes).unwrap_or(0); + let active = crate::recording::recording_disk::active_disk_reservations(others()); + if matches!( + crate::recording::recording_disk::would_fit_on_disk(free, safety, active, charge), + crate::recording::recording_disk::DiskAdmission::Insufficient { .. } + ) { + return Some(DISK_GONE_BEFORE_START); + } + None +} diff --git a/backend/dvr/src/recording/recording_transfer/retry.rs b/backend/dvr/src/recording/recording_transfer/retry.rs new file mode 100644 index 000000000..494b39d1c --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/retry.rs @@ -0,0 +1,132 @@ +use super::{mark_recording_metadata_notification, take_active, DownloadExecutionResult, RecordingNotificationPlan}; +use crate::{ + http_transfer::{is_retryable_error, retryable_transport_error_message}, + recording::{ + recording_notification::LifecycleEvent, + recording_queue::{mutate_optional, QueueMutationError, RecordingQueue, RecordingTaskState}, + }, +}; +use tuliprox_core::model::RecordingConfig; + +pub(super) fn classify_download_open_error(url: &reqwest::Url, err: &reqwest::Error) -> DownloadExecutionResult { + if is_retryable_error(err) { + DownloadExecutionResult::Retryable(format!("Error while opening url: {url} {err}")) + } else { + DownloadExecutionResult::Failed(format!("Error while opening url: {url} {err}")) + } +} + +pub(super) fn classify_download_stream_io_error(file_path_str: &str, err: &std::io::Error) -> DownloadExecutionResult { + if retryable_transport_error_message(&err.to_string()) { + DownloadExecutionResult::Retryable(format!("Error while downloading file: {file_path_str} {err}")) + } else { + DownloadExecutionResult::Failed(format!("Error while downloading file: {file_path_str} {err}")) + } +} + +fn apply_download_retry_jitter(base_secs: u64, jitter_percent: u8) -> u64 { + let jitter_percent = i64::from(jitter_percent.min(95)); + if jitter_percent == 0 { + return base_secs.max(1); + } + let jitter_percent = fastrand::i64(-jitter_percent..=jitter_percent); + let base_i64 = i64::try_from(base_secs.max(1)).unwrap_or(i64::MAX); + let jitter_delta = base_i64.saturating_mul(jitter_percent).saturating_div(100); + let jittered = base_i64.saturating_add(jitter_delta); + u64::try_from(jittered.max(1)).unwrap_or(1) +} + +#[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation, clippy::cast_sign_loss)] +fn compute_download_retry_backoff_secs(attempts: u8, download_cfg: &RecordingConfig) -> u64 { + let exponent = i32::from(attempts.saturating_sub(1)); + let scaled_secs = + (download_cfg.retry_backoff_initial_secs as f64) * download_cfg.retry_backoff_multiplier.powi(exponent); + let clamped_secs = + scaled_secs.clamp(download_cfg.retry_backoff_initial_secs as f64, download_cfg.retry_backoff_max_secs as f64); + let base_secs = clamped_secs.round() as u64; + apply_download_retry_jitter(base_secs, download_cfg.retry_backoff_jitter_percent) +} + +pub(super) async fn requeue_active_download_for_retry( + download_queue: &RecordingQueue, + uuid: &str, + promote: bool, +) -> Result { + Ok(mutate_optional(download_queue, |candidate| { + let Some(mut download) = take_active(candidate, uuid) else { + return Ok(None); + }; + download.finished = false; + download.paused = false; + download.error = None; + download.state = RecordingTaskState::Queued; + download.next_retry_at = None; + candidate.queue.insert(0, download); + if promote { + crate::recording::recording_queue::promote_from_queue(candidate); + } + Ok(Some(true)) + }) + .await? + .unwrap_or(false)) +} + +pub(super) enum RetryCommit { + Waiting { delay_secs: u64, attempts: u8 }, + Failed(RecordingNotificationPlan), +} + +pub(super) async fn prepare_active_retry( + download_queue: &RecordingQueue, + uuid: &str, + download_cfg: &RecordingConfig, + cause: &str, +) -> Result, QueueMutationError> { + mutate_optional(download_queue, |candidate| { + let Some(active) = candidate.active.iter_mut().find(|active| active.uuid == uuid) else { + return Ok(None); + }; + // A live broadcast cannot be retried: whatever played during the + // backoff is gone, so a second attempt records a different part of the + // programme and calls it the same recording. `RetryWaiting` is not a + // state a live capture can legally be in either. + let retryable_kind = active.kind.is_resumable(); + active.retry_attempts = active.retry_attempts.saturating_add(1); + let attempts = active.retry_attempts; + if !retryable_kind || attempts > download_cfg.retry_max_attempts { + let Some(mut failed) = take_active(candidate, uuid) else { + return Ok(None); + }; + let error = if retryable_kind { + format!("Retry limit reached after {} attempts: {cause}", download_cfg.retry_max_attempts) + } else { + format!("Live recording failed; restarting a live capture is not supported: {cause}") + }; + failed.finished = true; + failed.paused = false; + failed.next_retry_at = None; + failed.state = RecordingTaskState::Failed; + failed.error = Some(error.clone()); + failed.recording.reserved_bytes = 0; + let notification = + mark_recording_metadata_notification(&mut failed.recording, LifecycleEvent::Failed, Some(error)); + candidate.finished.push(failed); + crate::recording::recording_queue::promote_from_queue(candidate); + return Ok(Some(RetryCommit::Failed(notification))); + } + + let delay_secs = compute_download_retry_backoff_secs(attempts, download_cfg); + let next_retry_at = + chrono::Utc::now().timestamp().saturating_add(i64::try_from(delay_secs).unwrap_or(i64::MAX)); + active.next_retry_at = Some(next_retry_at); + active.state = RecordingTaskState::RetryWaiting; + active.paused = false; + active.finished = false; + active.error = Some(format!( + "Retrying after transient failure in {delay_secs}s (attempt {attempts}/{}): {cause}", + download_cfg.retry_max_attempts + )); + Ok(Some(RetryCommit::Waiting { delay_secs, attempts })) + }) + .await +} diff --git a/backend/dvr/src/recording/recording_transfer/tests.rs b/backend/dvr/src/recording/recording_transfer/tests.rs new file mode 100644 index 000000000..31a587d81 --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/tests.rs @@ -0,0 +1,25 @@ +use super::{ + acquire_result_after_wait, acquire_result_for_control, continue_after_pause, download_file, + ensure_recording_worker_running, finalize_http_transfer, finish_active_and_promote, http_transfer_path, + recording_deadline_instant, refresh_recording_progress, requeue_active_download_for_capacity_wait, + start_recording_scheduler, wait_for_provider_slot, DownloadExecutionResult, ProviderAcquireResult, QuotaGate, + RecordingNotificationPlan, DISK_GONE_BEFORE_START, DOWNLOAD_PREEMPTED_REASON, LIVE_CAPACITY_WINDOW_CLOSED, + QUOTA_EXCEEDED_DURING_TRANSFER, +}; + +mod admission; +mod behavior; +mod configuration; +mod lifecycle; +mod persistence; +mod protocol; +mod retry; +mod startup; +mod support; + +#[cfg(unix)] +use self::support::ConcurrentLiveFixture; +use self::support::{ + app_config_with_listener, bare_app_config, counting_ffmpeg, read_request, scheduled_task, serve_range_fixture, + slot_queue, spawn_count, vod_entry, +}; diff --git a/backend/dvr/src/recording/recording_transfer/tests/admission.rs b/backend/dvr/src/recording/recording_transfer/tests/admission.rs new file mode 100644 index 000000000..5e095b9cb --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/tests/admission.rs @@ -0,0 +1,427 @@ +#[cfg(unix)] +use super::ConcurrentLiveFixture; +use super::{ + app_config_with_listener, bare_app_config, counting_ffmpeg, download_file, ensure_recording_worker_running, + finish_active_and_promote, requeue_active_download_for_capacity_wait, scheduled_task, serve_range_fixture, + slot_queue, spawn_count, vod_entry, wait_for_provider_slot, DownloadExecutionResult, QuotaGate, + RecordingNotificationPlan, DOWNLOAD_PREEMPTED_REASON, LIVE_CAPACITY_WINDOW_CLOSED, QUOTA_EXCEEDED_DURING_TRANSFER, +}; +use crate::recording::{ + recording_capacity::{stub::StubCapacity, RecordingCapacityPort}, + recording_queue::{PersistedRecordingTask, RecordingControl, RecordingQueue, RecordingWaitOutcome}, +}; +use shared::model::{ + NoopSink, RecordingKind, RecordingMetadata, RecordingOwner, RecordingSource, RecordingTaskState, + RecordingVisibility, UserId, +}; +#[cfg(unix)] +use std::sync::atomic::Ordering; +use std::{path::Path, sync::Arc, time::Duration}; +use tempfile::TempDir; +use tokio::sync::{Notify, RwLock}; +use tuliprox_core::model::RecordingConfig; + +#[tokio::test] +async fn a_preemption_requeue_leaves_a_pending_control_untouched() { + let queue = RecordingQueue::new(); + *queue.active.write().await = vec![scheduled_task(RecordingKind::Live, 0, 60)]; + *queue.worker("task").control_signal.write().await = RecordingControl::Pause; + + requeue_active_download_for_capacity_wait(&queue, "task", DOWNLOAD_PREEMPTED_REASON, false, None) + .await + .expect("requeue"); + + assert_eq!(*queue.worker("task").control_signal.read().await, RecordingControl::Pause); +} + +/// Recording enabled under `dir` with a private quota of `quota` bytes. +pub(in crate::recording::recording_transfer::tests) fn app_config_with_quota( + dir: &Path, + quota: u64, +) -> tuliprox_core::model::AppConfig { + let app_config = bare_app_config(); + let mut rec_cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }); + rec_cfg.directory = dir.to_string_lossy().into_owned(); + rec_cfg.quota = + Some(tuliprox_core::model::RecordingQuotaConfig { default_private_bytes: Some(quota), ..Default::default() }); + app_config.config.store(Arc::new(tuliprox_core::model::Config { + video: Some(tuliprox_core::model::VideoConfig { + extensions: Vec::new(), + web_search: None, + recording: Some(rec_cfg), + }), + ..tuliprox_core::model::Config::default() + })); + app_config +} + +pub(in crate::recording::recording_transfer::tests) async fn queue_with( + dir: &TempDir, + active: PersistedRecordingTask, + queued: Vec, +) -> RecordingQueue { + let queue = RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository"); + crate::recording::recording_queue::mutate(&queue, move |candidate| { + candidate.active = vec![active.clone()]; + candidate.queue.clone_from(&queued); + Ok(()) + }) + .await + .expect("seed"); + queue +} + +#[tokio::test] +async fn a_transfer_larger_than_the_quota_left_is_refused_once_its_size_is_known() { + // Admitted at zero because the size was unknown; the first response + // says 4096 bytes against a 16 byte quota. + let dir = TempDir::new().expect("tempdir"); + let queue = queue_with(&dir, vod_entry("alice", "web:alice", RecordingTaskState::Running), Vec::new()).await; + let app_config = app_config_with_quota(dir.path(), 16); + let gate = QuotaGate { queue: &queue, app_config: &app_config }; + + let refused = gate.admit_size("alice", Some(4096), 0).await.err(); + assert_eq!(refused.as_deref(), Some(QUOTA_EXCEEDED_DURING_TRANSFER)); + + let admitted = gate.admit_size("alice", Some(16), 0).await.expect("fits"); + assert_eq!(admitted.byte_cap, None); + let reserved = queue.active.read().await.first().map(|active| active.recording.reserved_bytes); + assert_eq!(reserved, Some(16), "the known size is now reserved"); +} + +#[tokio::test] +async fn a_download_over_the_quota_stops_before_writing_its_file() { + let dir = TempDir::new().expect("tempdir"); + let (url, server) = serve_range_fixture(true); + let mut task = scheduled_task(RecordingKind::Vod, chrono::Utc::now().timestamp(), 900); + task.file_dir = dir.path().to_path_buf(); + task.file_path = dir.path().join("film.mp4"); + task.url = url; + task.recording = RecordingMetadata::new_media( + RecordingOwner::User(UserId::from("web:alice")), + RecordingVisibility::Private, + RecordingSource::new("t1", "v1", "in1"), + "Film".to_string(), + ); + let queue = queue_with(&dir, RecordingQueue::to_persisted(&task), Vec::new()).await; + let app_config = app_config_with_quota(dir.path(), 4); + + let result = download_file::( + Arc::clone(&queue.active), + task.clone(), + &reqwest::Client::new(), + None, + Arc::new(RwLock::new(RecordingControl::None)), + Arc::new(Notify::new()), + None, + Some(QuotaGate { queue: &queue, app_config: &app_config }), + ) + .await; + let _ = server.join(); + + assert!( + matches!(&result, DownloadExecutionResult::Failed(reason) if reason == QUOTA_EXCEEDED_DURING_TRANSFER), + "a 10 byte file against a 4 byte quota" + ); + assert!(!task.file_path.exists(), "nothing was published"); +} + +#[tokio::test] +async fn a_file_of_unknown_size_is_checked_against_waiting_quotas_when_it_completes() { + // Without a Content-Length nothing can be charged when the transfer + // starts. Alice has room; Bob, waiting to attach, has 16 bytes. The + // finished size is the first moment his quota can be asked. + let dir = TempDir::new().expect("tempdir"); + let queue = queue_with( + &dir, + vod_entry("alice", "web:alice", RecordingTaskState::Running), + vec![vod_entry("bob", "web:bob", RecordingTaskState::Queued)], + ) + .await; + let app_config = app_config_with_quota(dir.path(), 1 << 20); + let gate = QuotaGate { queue: &queue, app_config: &app_config }; + let started = gate.admit_size("alice", None, 0).await.expect("alice starts"); + assert!(!started.siblings_refused, "nothing is known about the size yet"); + + let limits = crate::recording::recording_quota::QuotaLimits { + default_private_bytes: Some(16), + per_user_bytes: [(UserId::from("web:alice"), 1 << 20)].into_iter().collect(), + shared_bytes: None, + }; + finish_active_and_promote(&queue, "alice", Some(&limits), |done| { + done.finished = true; + done.state = RecordingTaskState::Completed; + done.size = 4096; + done.recording.measured_bytes = 4096; + done.recording.reserved_bytes = 0; + RecordingNotificationPlan::empty() + }) + .await + .expect("commit"); + + let finished = queue.finished.read().await.clone(); + let state_of = |uuid: &str| finished.iter().find(|task| task.uuid == uuid).map(|task| task.state); + assert_eq!(state_of("alice"), Some(RecordingTaskState::Completed)); + assert_eq!(state_of("bob"), Some(RecordingTaskState::Failed), "not attached past his quota"); + assert!(queue.queue.lock().await.is_empty()); +} + +#[tokio::test] +async fn a_transfer_of_unknown_size_is_capped_at_the_quota_left() { + let dir = TempDir::new().expect("tempdir"); + let queue = queue_with(&dir, vod_entry("alice", "web:alice", RecordingTaskState::Running), Vec::new()).await; + let app_config = app_config_with_quota(dir.path(), 1000); + let gate = QuotaGate { queue: &queue, app_config: &app_config }; + + let admitted = gate.admit_size("alice", None, 0).await.expect("admitted"); + assert_eq!(admitted.byte_cap, Some(1000)); +} + +#[tokio::test] +async fn an_entry_waiting_to_attach_is_refused_for_its_own_quota() { + // Bob queued behind Alice's transfer while its size was unknown. + // Attaching later would charge him the whole file, so his quota has + // to be asked now. + let dir = TempDir::new().expect("tempdir"); + let queue = queue_with( + &dir, + vod_entry("alice", "web:alice", RecordingTaskState::Running), + vec![vod_entry("bob", "web:bob", RecordingTaskState::Queued)], + ) + .await; + let app_config = app_config_with_quota(dir.path(), 4096); + // Bob already holds something else that leaves him less than the file. + crate::recording::recording_queue::mutate(&queue, |candidate| { + let mut other = vod_entry("bob-other", "web:bob", RecordingTaskState::Completed); + other.media_identity = "other".to_string(); + other.recording.measured_bytes = 1000; + other.finished = true; + candidate.finished.push(other); + Ok(()) + }) + .await + .expect("seed"); + let gate = QuotaGate { queue: &queue, app_config: &app_config }; + + let admitted = gate.admit_size("alice", Some(4096), 0).await.expect("alice fits"); + assert!(admitted.siblings_refused); + assert!(queue.queue.lock().await.is_empty(), "bob no longer waits"); + let bob = queue.finished.read().await.iter().find(|task| task.uuid == "bob").cloned().expect("bob is filed"); + assert_eq!(bob.state, RecordingTaskState::Failed); + assert_eq!(bob.recording.reserved_bytes, 0); +} + +#[tokio::test(start_paused = true)] +async fn a_live_capture_stops_waiting_for_capacity_when_its_window_closes() { + // Waiting past the padded end cannot produce the recording that was + // asked for -- the programme has finished. Before this the wait had no + // bound at all, so a live capture with no free slot sat in the queue + // through its own broadcast and out the other side. + let dir = TempDir::new().expect("tempdir"); + let queue = slot_queue(&dir); + let control = RwLock::new(RecordingControl::None); + let notify = Notify::new(); + let window_ends = tokio::time::Instant::now() + Duration::from_mins(30); + + let outcome = wait_for_provider_slot(&queue, &Arc::from("provider"), 0, &control, ¬ify, Some(window_ends)).await; + + assert!(outcome.is_none(), "the wait ends when the programme does"); + assert!(queue.slot_waiters.snapshots().is_empty(), "expired waits must deregister"); + assert!(tokio::time::Instant::now() >= window_ends, "and only then"); +} + +#[tokio::test(start_paused = true)] +async fn a_capacity_wait_still_answers_a_cancellation_before_its_deadline() { + // The deadline must not swallow the control signals: a user cancelling + // a live capture that is waiting for capacity should not have to wait + // out the programme. + let dir = TempDir::new().expect("tempdir"); + let queue = slot_queue(&dir); + let control = RwLock::new(RecordingControl::None); + let notify = Notify::new(); + let window_ends = tokio::time::Instant::now() + Duration::from_mins(30); + + let provider: Arc = Arc::from("provider"); + let waiting = wait_for_provider_slot(&queue, &provider, 0, &control, ¬ify, Some(window_ends)); + let cancelling = async { + tokio::time::sleep(Duration::from_secs(5)).await; + *control.write().await = RecordingControl::Cancel; + notify.notify_waiters(); + }; + let (outcome, ()) = tokio::join!(waiting, cancelling); + + assert!(matches!(outcome, Some(RecordingWaitOutcome::Cancelled)), "the cancel is answered, not the deadline"); + assert!(tokio::time::Instant::now() < window_ends, "and long before the window closes"); +} + +#[cfg(unix)] +#[tokio::test] +async fn independent_live_workers_fill_provider_capacity_and_stop_only_the_selected_recording( +) -> Result<(), Box> { + for limit in [0, 4] { + let fixture = ConcurrentLiveFixture::new(4, limit).await?; + let (first, second) = tokio::join!(fixture.start(), fixture.start()); + first?; + second?; + fixture.wait_for_running(4).await?; + assert_eq!(fixture.capacity.peak.load(Ordering::SeqCst), 4); + fixture.queue.cancel_requested("live-2").await?; + let remaining = fixture.wait_for_running(3).await?; + assert!(!remaining.iter().any(|uuid| uuid == "live-2")); + assert!(fixture + .queue + .active + .read() + .await + .iter() + .filter(|task| task.uuid != "live-2") + .all(|task| task.state == RecordingTaskState::Running)); + fixture.stop_all(4).await?; + assert_eq!( + fixture.capacity.releases.load(Ordering::SeqCst), + 4, + "one allocation per recording despite concurrent starts" + ); + assert!(fixture.queue.finished.read().await.iter().all(|task| task.state == RecordingTaskState::Completed)); + } + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn a_live_capture_that_never_gets_capacity_fails_at_its_window_without_recording() { + // The whole path, driven against a provider that is permanently full: + // the worker asks for a slot, is refused, waits, and gives up when the + // programme it was going to record has finished. Until the capacity + // port existed this could only be produced by a real provider actually + // being busy for the length of a broadcast. + let dir = tempfile::TempDir::new().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let now = chrono::Utc::now().timestamp(); + let mut capture = scheduled_task(RecordingKind::Live, now, 1_800); + capture.uuid = "live".to_string(); + capture.state = RecordingTaskState::Queued; + capture.input_name = Some(Arc::from("provider")); + let capture = RecordingQueue::to_persisted(&capture); + crate::recording::recording_queue::mutate(&queue, move |candidate| { + candidate.queue.push(capture.clone()); + Ok(()) + }) + .await + .expect("seed"); + + let stub = StubCapacity::full(); + let capacity: Arc = Arc::clone(&stub) as Arc; + let app_config = bare_app_config(); + ensure_recording_worker_running( + &app_config, + &RecordingConfig::from(&shared::model::RecordingConfigDto { + enabled: true, + ..shared::model::RecordingConfigDto::default() + }), + &queue, + &NoopSink, + &capacity, + Path::new(crate::recording::recording_worker::FFMPEG_BINARY), + ) + .await + .expect("worker started"); + + // Sleeping rather than spinning: with a paused clock the runtime only + // advances time when nothing is runnable, so a busy wait would stop the + // window from ever closing. + tokio::time::sleep(Duration::from_mins(31)).await; + + let finished = queue.finished.read().await; + let settled = finished.first().expect("the capture reached a terminal state"); + assert_eq!(settled.state, RecordingTaskState::Failed, "no capacity ever came"); + assert_eq!(settled.error.as_deref(), Some(LIVE_CAPACITY_WINDOW_CLOSED), "and it says why"); + assert_eq!(settled.size, 0, "nothing was ever recorded"); + assert_eq!(settled.recording.reserved_bytes, 0, "and it is not still holding disk"); + assert_eq!( + stub.acquire_count(), + 1, + "it asked for a slot once and then waited, rather than spinning on the provider" + ); + assert_eq!(stub.release_count(), 0, "and it never held one to give back"); +} + +#[tokio::test] +async fn a_live_capture_asks_for_capacity_once_runs_the_encoder_once_and_completes_once() { + // The other half of the window contract. A real clock here on + // purpose: the worker spawns an actual child process, and pausing + // time would let unrelated timers fire between the spawn and the + // exit while the process itself still ran in wall-clock. + let dir = tempfile::TempDir::new().expect("tempdir"); + let recordings_dir = dir.path().join("recordings"); + std::fs::create_dir_all(&recordings_dir).expect("create recording dir"); + let log = dir.path().join("spawns.log"); + let script = counting_ffmpeg(dir.path(), &log); + + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let now = chrono::Utc::now().timestamp(); + let mut capture = scheduled_task(RecordingKind::Live, now, 1_800); + capture.uuid = "live".to_string(); + capture.state = RecordingTaskState::Queued; + capture.input_name = Some(Arc::from("provider")); + // The execution URL is derived from the source, so the virtual id has + // to be a real one rather than the placeholder the other fixtures use. + capture.recording.source.virtual_id = "42".to_string(); + capture.file_dir.clone_from(&recordings_dir); + capture.file_path = recordings_dir.join("capture.ts"); + let persisted = RecordingQueue::to_persisted(&capture); + crate::recording::recording_queue::mutate(&queue, move |candidate| { + candidate.queue.push(persisted.clone()); + Ok(()) + }) + .await + .expect("seed"); + + let stub = StubCapacity::with_room(); + let capacity: Arc = Arc::clone(&stub) as Arc; + ensure_recording_worker_running( + &app_config_with_listener(), + &RecordingConfig::from(&shared::model::RecordingConfigDto { + enabled: true, + ..shared::model::RecordingConfigDto::default() + }), + &queue, + &NoopSink, + &capacity, + &script, + ) + .await + .expect("worker started"); + + let settled = tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Some(done) = queue.finished.read().await.first().cloned() { + break done; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + // A bare timeout here says nothing about why. Where the capture got + // stuck is the whole diagnosis, so report it. + let Ok(settled) = settled else { + panic!( + "never settled: active={:?} queued={} spawns={} acquires={}", + queue.active.read().await.first().map(|active| (active.uuid.clone(), active.state, active.error.clone())), + queue.queue.lock().await.len(), + spawn_count(&log), + stub.acquire_count(), + ); + }; + + assert_eq!(settled.state, RecordingTaskState::Completed, "{:?}", settled.error); + assert_eq!(spawn_count(&log), 1, "the encoder ran once, not once per retry or per promotion pass"); + assert_eq!(stub.acquire_count(), 1, "and one slot was asked for"); + assert_eq!(stub.release_count(), 1, "and given back exactly once"); + assert_eq!(queue.finished.read().await.len(), 1, "the recording was committed once"); + assert_eq!( + tokio::fs::read(&capture.file_path).await.expect("the recording is where playback will look"), + b"recorded" + ); + assert!(!crate::recording_partial_path(&capture.file_path).exists(), "and nothing was left staged"); + assert_eq!(settled.recording.reserved_bytes, 0, "and it is not still holding disk"); +} diff --git a/backend/dvr/src/recording/recording_transfer/tests/behavior.rs b/backend/dvr/src/recording/recording_transfer/tests/behavior.rs new file mode 100644 index 000000000..a957942ab --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/tests/behavior.rs @@ -0,0 +1,505 @@ +use super::{ + acquire_result_after_wait, acquire_result_for_control, app_config_with_listener, bare_app_config, + continue_after_pause, download_file, finalize_http_transfer, finish_active_and_promote, http_transfer_path, + read_request, recording_deadline_instant, requeue_active_download_for_capacity_wait, scheduled_task, + serve_range_fixture, slot_queue, start_recording_scheduler, wait_for_provider_slot, DownloadExecutionResult, + ProviderAcquireResult, RecordingNotificationPlan, +}; +#[cfg(unix)] +use super::{counting_ffmpeg, ensure_recording_worker_running}; +use crate::recording::{ + recording_capacity::{stub::StubCapacity, RecordingCapacityPort}, + recording_queue::{RecordingControl, RecordingQueue, RecordingWaitOutcome}, +}; +use shared::model::{NoopSink, RecordingKind, RecordingTaskState}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; +use tempfile::TempDir; +use tokio::sync::{Notify, RwLock}; +use tuliprox_core::model::RecordingConfig; + +#[tokio::test] +async fn vod_and_series_resume_at_the_saved_byte_offset() { + for kind in [RecordingKind::Vod, RecordingKind::Series] { + let dir = TempDir::new().expect("recording directory"); + let (url, server) = serve_range_fixture(false); + let mut task = scheduled_task(kind, chrono::Utc::now().timestamp(), 900); + task.file_dir = dir.path().to_path_buf(); + task.file_path = dir.path().join("recording.mp4"); + task.url = url; + task.total_size = Some(10); + let partial = http_transfer_path(&task); + tokio::fs::write(&partial, b"0123").await.expect("saved partial"); + let active = Arc::new(RwLock::new(vec![task.clone()])); + let result = download_file::( + active, + task.clone(), + &reqwest::Client::new(), + None, + Arc::new(RwLock::new(RecordingControl::None)), + Arc::new(Notify::new()), + None, + None, + ) + .await; + assert!(matches!(result, DownloadExecutionResult::Completed)); + assert_eq!(tokio::fs::read(&task.file_path).await.expect("completed recording"), b"0123456789"); + assert!(!partial.exists()); + let request = server.join().expect("fixture server").to_ascii_lowercase(); + assert!(request.contains("range: bytes=4-\r\n"), "missing resume Range: {request}"); + assert!(request.contains("accept-encoding: identity\r\n"), "missing identity encoding: {request}"); + } +} + +#[tokio::test] +async fn resume_during_pause_shutdown_keeps_a_worker_for_vod_and_series() { + for kind in [RecordingKind::Vod, RecordingKind::Series] { + let queue = RecordingQueue::new(); + *queue.active.write().await = vec![scheduled_task(kind, 0, 900)]; + *queue.worker("task").running.write().await = true; + + assert!(queue.pause_active("task").await.expect("pause")); + assert!(queue.resume_active("task").await.expect("resume before old worker exits")); + assert!(continue_after_pause(&queue, "task").await); + assert!(queue.workers_running().await); + assert_eq!(queue.active.read().await.first().map(|task| task.state), Some(RecordingTaskState::Running)); + + assert!(queue.pause_active("task").await.expect("second pause")); + assert!(!continue_after_pause(&queue, "task").await); + assert!(queue.workers_running().await, "the claim lasts until worker exit"); + queue.release_worker("task").await; + assert!(!queue.workers_running().await); + assert!(queue.resume_active("task").await.expect("resume after old worker exits")); + assert!(!queue.active.read().await.first().is_some_and(|task| task.paused)); + } +} + +#[tokio::test] +async fn a_restart_requeue_consumes_the_restart_signal() { + // A restart that survives its own requeue makes every new worker + // preempt itself on start, spinning until the live window closes. + let queue = RecordingQueue::new(); + *queue.active.write().await = vec![scheduled_task(RecordingKind::Live, 0, 60)]; + *queue.worker("task").control_signal.write().await = RecordingControl::Restart; + + let requeued = requeue_active_download_for_capacity_wait( + &queue, + "task", + "Reloading download service configuration", + false, + Some(RecordingControl::Restart), + ) + .await + .expect("requeue"); + + assert!(requeued); + assert_eq!(*queue.worker("task").control_signal.read().await, RecordingControl::None); +} + +#[test] +fn a_slot_wait_maps_onto_the_acquisition_outcome() { + // A wait that ends without an outcome ran into the live window's end; + // being signalled means trying to acquire again. + assert!(matches!(acquire_result_after_wait(None), Some(ProviderAcquireResult::WindowClosed))); + assert!(acquire_result_after_wait(Some(RecordingWaitOutcome::Signalled)).is_none()); + assert!(matches!( + acquire_result_after_wait(Some(RecordingWaitOutcome::Paused)), + Some(ProviderAcquireResult::Paused) + )); + assert!(matches!( + acquire_result_after_wait(Some(RecordingWaitOutcome::Cancelled)), + Some(ProviderAcquireResult::Cancelled) + )); + assert!(matches!( + acquire_result_after_wait(Some(RecordingWaitOutcome::Restarted)), + Some(ProviderAcquireResult::Preempted) + )); + assert!(acquire_result_for_control(RecordingControl::None).is_none()); + assert!(matches!(acquire_result_for_control(RecordingControl::Cancel), Some(ProviderAcquireResult::Cancelled))); + assert!(matches!(acquire_result_for_control(RecordingControl::Pause), Some(ProviderAcquireResult::Paused))); + assert!(matches!(acquire_result_for_control(RecordingControl::Restart), Some(ProviderAcquireResult::Preempted))); +} + +#[test] +fn a_live_window_that_has_already_closed_stops_immediately() { + let now = chrono::Utc::now().timestamp(); + let long_over = scheduled_task(RecordingKind::Live, now - 7_200, 900); + let deadline = recording_deadline_instant(&long_over).expect("still has a deadline"); + assert!(deadline <= tokio::time::Instant::now(), "a closed window must not keep recording"); +} + +#[test] +fn only_live_captures_are_bounded_by_a_window() { + // A film does not stop being downloadable because a clock ran out. + let now = chrono::Utc::now().timestamp(); + for kind in [RecordingKind::Vod, RecordingKind::Series] { + assert!(recording_deadline_instant(&scheduled_task(kind, now, 900)).is_none()); + } +} + +#[test] +fn resumable_kinds_stage_through_a_partial_and_live_writes_in_place() { + // The strategy split: an HTTP transfer can be interrupted and resumed, + // so it stages. ffmpeg owns its output file for the whole capture and + // has nothing to resume into. + let now = chrono::Utc::now().timestamp(); + let live = scheduled_task(RecordingKind::Live, now, 900); + assert_eq!(http_transfer_path(&live), live.file_path, "a live capture writes straight to its file"); + + for kind in [RecordingKind::Vod, RecordingKind::Series] { + let task = scheduled_task(kind, now, 900); + let staged = http_transfer_path(&task); + assert_ne!(staged, task.file_path); + assert!( + staged.to_string_lossy().ends_with(".partial"), + "a resumable transfer stages beside its final file, got {}", + staged.display() + ); + } +} + +#[tokio::test] +async fn stopping_live_capture_clears_cancel_before_starting_a_due_recording() -> Result<(), Box> +{ + for terminal in [RecordingTaskState::Completed, RecordingTaskState::Failed] { + for has_next in [false, true] { + let dir = TempDir::new()?; + let queue = RecordingQueue::new_persistent(dir.path(), dir.path())?; + let now = chrono::Utc::now().timestamp(); + let mut active = scheduled_task(RecordingKind::Live, now - 60, 900); + active.uuid = "immediate".to_string(); + let mut scheduled = scheduled_task(RecordingKind::Live, now, 900); + scheduled.uuid = "scheduled".to_string(); + scheduled.state = RecordingTaskState::Scheduled; + let active = RecordingQueue::to_persisted(&active); + let scheduled = RecordingQueue::to_persisted(&scheduled); + crate::recording::recording_queue::mutate(&queue, |candidate| { + candidate.active = vec![active]; + if has_next { + candidate.scheduled.push(scheduled); + } + Ok(()) + }) + .await?; + assert_eq!(queue.promote_due_scheduled(now).await, usize::from(has_next)); + assert_eq!(queue.cancel_requested("immediate").await?, Some(false)); + assert_eq!(*queue.worker("immediate").control_signal.read().await, RecordingControl::Cancel); + + let committed = finish_active_and_promote(&queue, "immediate", None, |task| { + task.state = terminal; + task.finished = true; + task.error = + (terminal == RecordingTaskState::Failed).then(|| "Stopped recording has no data".to_string()); + task.recording.reserved_bytes = 0; + RecordingNotificationPlan::empty() + }) + .await?; + assert!(committed.is_some()); + assert_eq!(*queue.worker("immediate").control_signal.read().await, RecordingControl::None); + assert_eq!( + queue.active.read().await.first().map(|task| task.uuid.as_str()), + has_next.then_some("scheduled") + ); + let finished = queue.finished.read().await; + assert_eq!(finished.len(), 1); + assert_eq!(finished[0].state, terminal); + assert!(finished[0].to_view(true).is_terminal()); + drop(finished); + + let restored = RecordingQueue::new_persistent(dir.path(), dir.path())?; + restored.load_from_disk().await?; + assert_eq!(restored.finished.read().await[0].state, terminal); + } + } + Ok(()) +} + +#[tokio::test] +async fn finalizing_publishes_the_staged_bytes_and_clears_the_partial() { + let dir = TempDir::new().expect("tempdir"); + let partial = dir.path().join("film.mp4.partial"); + let final_path = dir.path().join("film.mp4"); + std::fs::write(&partial, b"complete recording").expect("write"); + + finalize_http_transfer(&final_path, &partial).await.expect("finalize"); + + assert!(final_path.exists()); + assert!(!partial.exists(), "the partial is cleared once published"); +} + +#[tokio::test] +async fn finalizing_twice_succeeds_instead_of_failing_a_complete_recording() { + // A crash between the link and the partial's removal leaves both files. + // The resumed transfer sees a complete file, finalizes again, and used + // to get AlreadyExists back -- reporting a recording whose bytes are + // entirely on disk as failed, on every retry. + let dir = TempDir::new().expect("tempdir"); + let partial = dir.path().join("film.mp4.partial"); + let final_path = dir.path().join("film.mp4"); + std::fs::write(&partial, b"complete recording").expect("write"); + std::fs::hard_link(&partial, &final_path).expect("simulate the interrupted finalize"); + + finalize_http_transfer(&final_path, &partial).await.expect("finalizing again must succeed"); + + assert!(final_path.exists()); + assert!(!partial.exists(), "the second attempt finishes the job the first did not"); +} + +#[tokio::test] +async fn a_different_file_at_the_final_path_is_not_published_over() { + // Path reservation should make this unreachable. If it ever happens the + // staged bytes were never verified against what is already there, so + // the transfer must not claim success. + let dir = TempDir::new().expect("tempdir"); + let partial = dir.path().join("film.mp4.partial"); + let final_path = dir.path().join("film.mp4"); + std::fs::write(&partial, b"the recording we just made").expect("write"); + std::fs::write(&final_path, b"something else entirely, of another size").expect("write"); + + let error = finalize_http_transfer(&final_path, &partial).await.expect_err("must refuse"); + assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); + assert!(partial.exists(), "the staged bytes are kept for inspection"); +} + +#[tokio::test] +async fn a_transfer_written_straight_to_its_final_path_needs_no_finalization() { + let dir = TempDir::new().expect("tempdir"); + let final_path = dir.path().join("stream.ts"); + std::fs::write(&final_path, b"live capture").expect("write"); + + finalize_http_transfer(&final_path, &final_path).await.expect("finalize"); + + assert!(final_path.exists()); +} + +#[tokio::test(start_paused = true)] +async fn a_transfer_waits_for_a_slot_however_long_it_takes() { + // The counterpart, and the reason the bound is not applied to + // everything: a file on a server is still there in an hour. + let dir = TempDir::new().expect("tempdir"); + let queue = slot_queue(&dir); + let control = RwLock::new(RecordingControl::None); + let notify = Notify::new(); + + let waiters = Arc::clone(&queue.slot_waiters); + let freed = tokio::spawn(async move { + tokio::time::sleep(Duration::from_mins(60)).await; + let queued = waiters.snapshots(); + let waiter = queued.first().expect("someone is waiting an hour later"); + waiters.signal_waiter(waiter.id) + }); + + let outcome = wait_for_provider_slot(&queue, &Arc::from("provider"), 0, &control, ¬ify, None).await; + + assert!(freed.await.expect("signaller"), "the waiter was still there to signal"); + assert!(matches!(outcome, Some(RecordingWaitOutcome::Signalled)), "it took the slot when one appeared"); +} + +#[cfg(unix)] +#[tokio::test] +async fn scheduler_starts_a_due_live_capture_while_another_capture_is_running() -> Result<(), Box> +{ + for has_data in [false, true] { + let dir = TempDir::new()?; + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path())?); + let now = chrono::Utc::now().timestamp(); + let mut immediate = scheduled_task(RecordingKind::Live, now - 60, 900); + immediate.uuid = "immediate".to_string(); + immediate.state = RecordingTaskState::Queued; + immediate.input_name = Some(Arc::from("provider")); + immediate.recording.source.virtual_id = "42".to_string(); + immediate.file_dir = dir.path().to_path_buf(); + immediate.file_path = dir.path().join("immediate.ts"); + let mut scheduled = immediate.clone(); + scheduled.uuid = "scheduled".to_string(); + scheduled.state = RecordingTaskState::Scheduled; + scheduled.file_path = dir.path().join("scheduled.ts"); + scheduled.recording.program_start = Some(now); + scheduled.recording.scheduled_start = Some(now); + let immediate = RecordingQueue::to_persisted(&immediate); + let scheduled = RecordingQueue::to_persisted(&scheduled); + crate::recording::recording_queue::mutate(&queue, |candidate| { + candidate.queue.push(immediate); + candidate.scheduled.push(scheduled); + Ok(()) + }) + .await?; + let script = counting_ffmpeg(dir.path(), &dir.path().join("spawns.log")); + let write_initial = if has_data { "printf recorded > \"$output\"" } else { ":" }; + std::fs::write(&script, format!( + "#!/bin/sh\nfor arg in \"$@\"; do output=\"$arg\"; done\ncase \"$output\" in\n*immediate.ts.partial) {write_initial}; touch \"$output.ready\"; read -r control ;;\n*) printf recorded > \"$output\" ;;\nesac\nexit 0\n" + ))?; + let stub = StubCapacity::with_room(); + let capacity: Arc = stub.clone(); + let config = RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }); + ensure_recording_worker_running(&app_config_with_listener(), &config, &queue, &NoopSink, &capacity, &script) + .await?; + tokio::time::timeout(Duration::from_secs(5), async { + while !dir.path().join("immediate.ts.partial.ready").exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await?; + let cancel = tokio_util::sync::CancellationToken::new(); + start_recording_scheduler( + Arc::new(app_config_with_listener()), + config.clone(), + &queue, + NoopSink, + capacity.clone(), + cancel.clone(), + script.clone(), + ); + tokio::time::timeout(Duration::from_secs(5), async { + while !queue.finished.read().await.iter().any(|task| task.uuid == "scheduled") { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await?; + assert!(queue + .active + .read() + .await + .iter() + .any(|task| task.uuid == "immediate" && task.state == RecordingTaskState::Running)); + assert!(queue.finished.read().await.iter().all(|task| task.uuid != "immediate")); + queue.cancel_requested("immediate").await?; + tokio::time::timeout(Duration::from_secs(5), async { + while queue.finished.read().await.len() != 2 || queue.workers_running().await { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await?; + cancel.cancel(); + assert_eq!(stub.acquire_count(), 2); + let finished = queue.finished.read().await; + let stopped = finished.iter().find(|task| task.uuid == "immediate").ok_or("missing stopped task")?; + let next = finished.iter().find(|task| task.uuid == "scheduled").ok_or("missing scheduled task")?; + assert_eq!(stopped.state, if has_data { RecordingTaskState::Completed } else { RecordingTaskState::Failed }); + assert_eq!(next.state, RecordingTaskState::Completed, "{:?}", next.error); + assert_eq!(tokio::fs::read(&next.file_path).await?, b"recorded"); + assert!(stopped.to_view(true).is_terminal()); + assert!(next.to_view(true).is_terminal()); + assert!(queue.active.read().await.is_empty()); + assert_eq!(*queue.worker("immediate").control_signal.read().await, RecordingControl::None); + assert_eq!(stub.release_count(), 2); + } + Ok(()) +} + +#[tokio::test] +async fn scheduler_starts_a_queued_transfer_after_worker_exit() { + let dir = tempfile::TempDir::new().expect("tempdir"); + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let mut transfer = scheduled_task(RecordingKind::Vod, chrono::Utc::now().timestamp(), 300); + transfer.state = RecordingTaskState::Queued; + transfer.input_name = Some(Arc::from("provider")); + let persisted = RecordingQueue::to_persisted(&transfer); + crate::recording::recording_queue::mutate(&queue, move |candidate| { + candidate.queue.push(persisted.clone()); + Ok(()) + }) + .await + .expect("seed queue"); + + let capacity: Arc = StubCapacity::full(); + let cancel = tokio_util::sync::CancellationToken::new(); + start_recording_scheduler( + Arc::new(bare_app_config()), + RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }), + &queue, + NoopSink, + capacity, + cancel.clone(), + Path::new(crate::recording::recording_worker::FFMPEG_BINARY).to_path_buf(), + ); + + tokio::time::timeout(Duration::from_secs(3), async { + loop { + if queue + .active + .read() + .await + .first() + .is_some_and(|task| task.state == RecordingTaskState::WaitingForCapacity) + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("scheduler did not restart the stranded transfer"); + cancel.cancel(); +} + +#[tokio::test] +async fn serial_transfers_start_the_next_worker_without_waiting_for_a_tick() -> Result<(), Box> { + use tokio::io::AsyncWriteExt; + let dir = TempDir::new()?; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let app = Arc::new(app_config_with_listener()); + let mut config = (*app.config.load_full()).clone(); + config.api.port = listener.local_addr()?.port(); + app.config.store(Arc::new(config)); + let queue = Arc::new(RecordingQueue::new()); + for index in 0..2 { + let mut task = scheduled_task(RecordingKind::Vod, chrono::Utc::now().timestamp(), 300); + task.uuid = format!("transfer-{index}"); + task.state = RecordingTaskState::Queued; + task.recording.source.virtual_id = (42 + index).to_string(); + task.file_dir = dir.path().to_path_buf(); + task.file_path = dir.path().join(format!("transfer-{index}.mp4")); + queue.queue.lock().await.push_back(task); + } + let (first_seen, first_received) = tokio::sync::oneshot::channel(); + let (release, released) = tokio::sync::oneshot::channel(); + let (second_seen, second_received) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + for (seen, gate) in [(first_seen, Some(released)), (second_seen, None)] { + let (mut socket, _) = listener.accept().await?; + read_request(&mut socket).await?; + socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\nConnection: close\r\n\r\n").await?; + let _ = seen.send(()); + if let Some(gate) = gate { + let _ = gate.await; + } + socket.write_all(b"0123456789").await?; + } + Ok::<_, std::io::Error>(()) + }); + let cancellation = tokio_util::sync::CancellationToken::new(); + let capacity: Arc = StubCapacity::with_room(); + start_recording_scheduler( + app, + RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }), + &queue, + NoopSink, + capacity, + cancellation.clone(), + PathBuf::from("unused"), + ); + let result = async { + tokio::time::timeout(Duration::from_secs(5), first_received).await??; + assert_eq!(queue.active.read().await.len(), 1); + assert_eq!(queue.queue.lock().await.len(), 1); + tokio::time::sleep(Duration::from_millis(50)).await; + release.send(()).map_err(|()| "fixture stopped")?; + tokio::time::timeout(Duration::from_millis(500), second_received).await??; + tokio::time::timeout(Duration::from_secs(5), async { + while queue.finished.read().await.len() != 2 || queue.workers_running().await { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await?; + assert!(queue.finished.read().await.iter().all(|task| task.state == RecordingTaskState::Completed)); + server.await??; + Ok::<_, Box>(()) + } + .await; + cancellation.cancel(); + result +} diff --git a/backend/dvr/src/recording/recording_transfer/tests/configuration.rs b/backend/dvr/src/recording/recording_transfer/tests/configuration.rs new file mode 100644 index 000000000..eee6f411a --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/tests/configuration.rs @@ -0,0 +1,100 @@ +#[cfg(unix)] +use super::ConcurrentLiveFixture; +use super::{app_config_with_listener, ensure_recording_worker_running, scheduled_task, serve_range_fixture}; +use crate::recording::{ + recording_capacity::{stub::StubCapacity, RecordingCapacityPort}, + recording_queue::RecordingQueue, +}; +use shared::model::{NoopSink, RecordingKind, RecordingTaskState}; +#[cfg(unix)] +use std::sync::atomic::Ordering; +use std::{path::Path, sync::Arc, time::Duration}; +use tuliprox_core::model::RecordingConfig; + +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn simultaneous_starts_respect_the_configured_background_limit() -> Result<(), Box> { + let fixture = ConcurrentLiveFixture::with_background_limit(3, 3, 1).await?; + fixture.start().await?; + let running = fixture.wait_for_running(1).await?; + tokio::time::timeout(Duration::from_secs(5), async { + while fixture.queue.slot_waiters.snapshots().len() != 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await?; + assert_eq!(fixture.capacity.peak.load(Ordering::SeqCst), 1); + fixture.queue.cancel_requested(&running[0]).await?; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let now_running = fixture.wait_for_running(1).await?; + if now_running != running { + break Ok::<_, tokio::time::error::Elapsed>(()); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await??; + assert_eq!(fixture.capacity.peak.load(Ordering::SeqCst), 1); + fixture.stop_all(3).await?; + Ok(()) +} + +#[tokio::test] +async fn vod_and_series_transfers_reach_the_local_listener_past_a_configured_proxy() { + for kind in [RecordingKind::Vod, RecordingKind::Series] { + let dir = tempfile::TempDir::new().expect("tempdir"); + let (fixture_url, server) = serve_range_fixture(true); + let app_config = app_config_with_listener(); + let mut config = (*app_config.config.load_full()).clone(); + config.api.port = fixture_url.port().expect("fixture port"); + // Nothing listens here: a transfer routed through the proxy fails. + config.proxy = Some(tuliprox_core::model::ProxyConfig { + url: "http://127.0.0.1:9".to_string(), + username: None, + password: None, + }); + app_config.config.store(Arc::new(config)); + + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let mut transfer = scheduled_task(kind, chrono::Utc::now().timestamp(), 300); + transfer.state = RecordingTaskState::Queued; + transfer.input_name = Some(Arc::from("provider")); + transfer.recording.source.virtual_id = "42".to_string(); + transfer.file_dir = dir.path().to_path_buf(); + transfer.file_path = dir.path().join("transfer.mp4"); + let persisted = RecordingQueue::to_persisted(&transfer); + crate::recording::recording_queue::mutate(&queue, move |candidate| { + candidate.queue.push(persisted.clone()); + Ok(()) + }) + .await + .expect("seed"); + + let capacity: Arc = StubCapacity::with_room() as Arc; + ensure_recording_worker_running( + &app_config, + &RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }), + &queue, + &NoopSink, + &capacity, + Path::new(crate::recording::recording_worker::FFMPEG_BINARY), + ) + .await + .expect("worker started"); + + let settled = tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Some(done) = queue.finished.read().await.first().cloned() { + break done; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("transfer settled"); + assert_eq!(settled.state, RecordingTaskState::Completed, "{kind:?}: {:?}", settled.error); + let request = server.join().expect("fixture server"); + assert!(request.starts_with("GET /api/v1/playlist/recording/"), "{request}"); + } +} diff --git a/backend/dvr/src/recording/recording_transfer/tests/lifecycle.rs b/backend/dvr/src/recording/recording_transfer/tests/lifecycle.rs new file mode 100644 index 000000000..a52c9f174 --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/tests/lifecycle.rs @@ -0,0 +1,126 @@ +#[cfg(unix)] +use super::ConcurrentLiveFixture; +use super::{app_config_with_listener, counting_ffmpeg, ensure_recording_worker_running, scheduled_task}; +use crate::recording::{ + recording_capacity::{stub::StubCapacity, RecordingCapacityPort}, + recording_queue::RecordingQueue, +}; +use shared::model::{NoopSink, RecordingKind, RecordingTaskState}; +#[cfg(unix)] +use std::sync::atomic::Ordering; +use std::{path::Path, sync::Arc, time::Duration}; +use tuliprox_core::model::RecordingConfig; + +#[cfg(unix)] +#[tokio::test] +async fn live_workers_respect_capacity_and_start_waiting_recordings_when_a_slot_is_released( +) -> Result<(), Box> { + let fixture = ConcurrentLiveFixture::new(4, 2).await?; + fixture.start().await?; + let running = fixture.wait_for_running(2).await?; + tokio::time::timeout(Duration::from_secs(5), async { + while fixture.queue.slot_waiters.snapshots().len() != 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await?; + assert_eq!(fixture.capacity.in_use.load(Ordering::SeqCst), 2); + fixture.queue.cancel_requested(&running[0]).await?; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let now_running = fixture.wait_for_running(2).await?; + if now_running.iter().any(|uuid| !running.contains(uuid)) { + break Ok::<_, tokio::time::error::Elapsed>(()); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await??; + assert_eq!(fixture.capacity.peak.load(Ordering::SeqCst), 2); + assert!(fixture + .queue + .active + .read() + .await + .iter() + .any(|task| task.uuid == running[1] && task.state == RecordingTaskState::Running)); + fixture.stop_all(4).await?; + assert!(fixture.queue.finished.read().await.iter().all(|task| task.to_view(true).is_terminal())); + assert!(fixture.queue.slot_waiters.snapshots().is_empty()); + Ok(()) +} + +#[tokio::test] +async fn live_timeout_keeps_the_transfer_cause_without_claiming_the_window_expired( +) -> Result<(), Box> { + let dir = tempfile::tempdir()?; + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path())?); + let mut capture = scheduled_task(RecordingKind::Live, chrono::Utc::now().timestamp(), 300); + capture.recording.source.virtual_id = "42".to_string(); + capture.file_dir = dir.path().to_path_buf(); + capture.file_path = dir.path().join("capture.ts"); + capture.state = RecordingTaskState::Queued; + capture.input_name = Some(Arc::from("provider")); + let persisted = RecordingQueue::to_persisted(&capture); + crate::recording::recording_queue::mutate(&queue, move |candidate| { + candidate.queue.push(persisted.clone()); + Ok(()) + }) + .await?; + let script = counting_ffmpeg(dir.path(), &dir.path().join("spawns.log")); + std::fs::write(&script, "#!/bin/sh\necho 'Error opening input files: Operation timed out' >&2\nexit 1\n")?; + let stub = StubCapacity::with_room(); + let capacity: Arc = Arc::clone(&stub) as Arc; + ensure_recording_worker_running( + &app_config_with_listener(), + &RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }), + &queue, + &NoopSink, + &capacity, + &script, + ) + .await?; + let settled = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Some(task) = queue.finished.read().await.first().cloned() { + break task; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await?; + assert_eq!(settled.state, RecordingTaskState::Failed); + let error = settled.error.as_deref().ok_or("missing transfer cause")?; + assert!(error.contains("Error opening input files: Operation timed out"), "{error}"); + assert!(!error.contains("window has moved on"), "{error}"); + assert_eq!(settled.retry_attempts, 1); + assert_eq!(settled.recording.reserved_bytes, 0); + assert_eq!(stub.acquire_count(), 1); + assert_eq!(stub.release_count(), 1); + assert!(queue.active.read().await.is_empty()); + assert_eq!(queue.finished.read().await.len(), 1); + Ok(()) +} + +#[tokio::test] +async fn idle_scheduler_does_not_snapshot_recording_history() -> Result<(), Box> { + let queue = Arc::new(RecordingQueue::new()); + let history = queue.finished.write().await; + let capacity: Arc = StubCapacity::with_room(); + let config = RecordingConfig::from(&shared::model::RecordingConfigDto::default()); + tokio::time::timeout(Duration::from_millis(200), async { + queue.promote_due_scheduled_now().await; + ensure_recording_worker_running( + &app_config_with_listener(), + &config, + &queue, + &NoopSink, + &capacity, + Path::new("unused"), + ) + .await + }) + .await??; + drop(history); + Ok(()) +} diff --git a/backend/dvr/src/recording/recording_transfer/tests/persistence.rs b/backend/dvr/src/recording/recording_transfer/tests/persistence.rs new file mode 100644 index 000000000..b87fb7cb1 --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/tests/persistence.rs @@ -0,0 +1,98 @@ +use super::{ + app_config_with_listener, ensure_recording_worker_running, recording_deadline_instant, scheduled_task, vod_entry, +}; +use crate::recording::{ + recording_capacity::{stub::StubCapacity, RecordingCapacityPort}, + recording_queue::RecordingQueue, +}; +use shared::model::{NoopSink, RecordingKind, RecordingTaskState}; +use std::{path::Path, sync::Arc, time::Duration}; +use tuliprox_core::model::RecordingConfig; + +#[tokio::test] +async fn starting_workers_commits_entries_attached_to_an_already_completed_file( +) -> Result<(), Box> { + let queue = Arc::new(RecordingQueue::new()); + let mut done = vod_entry("done", "alice", RecordingTaskState::Completed); + done.finished = true; + done.size = 123; + done.recording.measured_bytes = 123; + let waiting = vod_entry("waiting", "bob", RecordingTaskState::Queued); + crate::recording::recording_queue::mutate(&queue, |candidate| { + candidate.finished.push(done); + candidate.queue.push(waiting); + Ok(()) + }) + .await?; + let stub = StubCapacity::with_room(); + let capacity: Arc = stub.clone(); + ensure_recording_worker_running( + &app_config_with_listener(), + &RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }), + &queue, + &NoopSink, + &capacity, + Path::new("unused-encoder"), + ) + .await?; + assert!(queue.queue.lock().await.is_empty()); + assert!(queue.active.read().await.is_empty()); + assert_eq!(queue.finished.read().await.len(), 2); + let finished = queue.finished.read().await; + let attached = finished.iter().find(|task| task.uuid == "waiting").ok_or("missing attached entry")?; + assert_eq!(attached.state, RecordingTaskState::Completed); + assert_eq!(attached.size, 123); + assert_eq!(stub.acquire_count(), 0); + Ok(()) +} + +#[test] +fn a_live_window_is_measured_in_wall_clock_not_in_time_spent_recording() { + // A capture that was preempted or waiting for capacity does not get + // that time back: the broadcast ran regardless. The deadline is + // anchored to the programme, so an interruption cannot push a capture + // past the window and into the next programme. + let now = chrono::Utc::now().timestamp(); + let started_ten_minutes_ago = scheduled_task(RecordingKind::Live, now - 600, 900); + let remaining = recording_deadline_instant(&started_ten_minutes_ago) + .expect("a scheduled live capture has a deadline") + .saturating_duration_since(tokio::time::Instant::now()) + .as_secs(); + // 900s window, 600s already elapsed: about 300 left, never a fresh 900. + assert!((295..=305).contains(&remaining), "expected roughly 300s left, got {remaining}"); +} + +#[tokio::test] +async fn blocked_serial_queue_does_not_clone_partitions_on_each_commit() -> Result<(), Box> { + let queue = RecordingQueue::new(); + queue.active.write().await.push(scheduled_task(RecordingKind::Vod, 0, 300)); + for index in 0..49 { + let mut task = scheduled_task(RecordingKind::Vod, 0, 300); + task.uuid = format!("waiting-{index}"); + task.recording.source.virtual_id = index.to_string(); + task.state = RecordingTaskState::Queued; + queue.queue.lock().await.push_back(task); + } + for index in 0..100 { + let mut done = scheduled_task(RecordingKind::Vod, 0, 300); + done.uuid = format!("completed-{index}"); + done.recording.source.virtual_id = (100 + index).to_string(); + done.state = RecordingTaskState::Completed; + queue.finished.write().await.push(done); + } + for size in 0..3 { + crate::recording::recording_queue::mutate(&queue, |candidate| { + candidate.active[0].size = size; + Ok(()) + }) + .await?; + // Promotion has no reason to read this partition; a full snapshot does. + let scheduled = queue.scheduled.write().await; + assert!( + !tokio::time::timeout(Duration::from_millis(200), super::super::promote_ready_downloads(&queue)).await?? + ); + drop(scheduled); + } + assert_eq!(queue.queue.lock().await.len(), 49); + Ok(()) +} diff --git a/backend/dvr/src/recording/recording_transfer/tests/protocol.rs b/backend/dvr/src/recording/recording_transfer/tests/protocol.rs new file mode 100644 index 000000000..e8ab4aa92 --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/tests/protocol.rs @@ -0,0 +1,271 @@ +use super::{ + app_config_with_listener, counting_ffmpeg, download_file, ensure_recording_worker_running, http_transfer_path, + read_request, scheduled_task, serve_range_fixture, spawn_count, DownloadExecutionResult, DISK_GONE_BEFORE_START, +}; +use crate::recording::{ + recording_capacity::{stub::StubCapacity, RecordingCapacityPort}, + recording_queue::{RecordingControl, RecordingQueue}, +}; +use shared::model::{NoopSink, RecordingKind, RecordingTaskState}; +use std::{sync::Arc, time::Duration}; +use tempfile::TempDir; +use tokio::sync::{Notify, RwLock}; +use tuliprox_core::model::RecordingConfig; + +#[tokio::test] +async fn ignored_range_preserves_vod_and_series_partials() { + for kind in [RecordingKind::Vod, RecordingKind::Series] { + let dir = TempDir::new().expect("recording directory"); + let (url, server) = serve_range_fixture(true); + let mut task = scheduled_task(kind, chrono::Utc::now().timestamp(), 900); + task.file_dir = dir.path().to_path_buf(); + task.file_path = dir.path().join("recording.mp4"); + task.url = url; + task.total_size = Some(10); + let partial = http_transfer_path(&task); + tokio::fs::write(&partial, b"0123").await.expect("saved partial"); + let result = download_file::( + Arc::new(RwLock::new(vec![task.clone()])), + task.clone(), + &reqwest::Client::new(), + None, + Arc::new(RwLock::new(RecordingControl::None)), + Arc::new(Notify::new()), + None, + None, + ) + .await; + assert!( + matches!(result, DownloadExecutionResult::Failed(error) if error == super::super::RANGE_UNSUPPORTED_ERROR) + ); + assert_eq!(tokio::fs::read(&partial).await.expect("partial preserved"), b"0123"); + assert!(!task.file_path.exists()); + assert!(server.join().expect("fixture server").to_ascii_lowercase().contains("range: bytes=4-\r\n")); + } +} + +#[tokio::test] +async fn recording_redirects_keep_the_proxy_and_resume_headers() -> Result<(), Box> { + use tokio::io::AsyncWriteExt; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let local_url = reqwest::Url::parse(&format!("http://{}/capture", listener.local_addr()?))?; + let local_task = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + let request = read_request(&mut socket).await?; + socket.write_all(b"HTTP/1.1 302 Found\r\nLocation: http://recording-origin.invalid/start\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await?; + Ok::<_, std::io::Error>(request) + }); + let proxy = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let proxy_url = format!("http://{}", proxy.local_addr()?); + let proxy_task = tokio::spawn(async move { + let mut requests = Vec::new(); + for response in [ + "HTTP/1.1 307 Temporary Redirect\r\nLocation: /finish\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + "HTTP/1.1 302 Found\r\nLocation: http://recording-cdn.invalid/end\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + "HTTP/1.1 206 Partial Content\r\nContent-Range: bytes 4-9/10\r\nContent-Length: 6\r\nConnection: close\r\n\r\n456789", + ] { + let (mut socket, _) = proxy.accept().await?; + let request = read_request(&mut socket).await?; + socket.write_all(response.as_bytes()).await?; + requests.push(request); + } + Ok::<_, std::io::Error>(requests) + }); + let config = app_config_with_listener(); + let mut updated = (*config.config.load_full()).clone(); + updated.proxy = Some(tuliprox_core::model::ProxyConfig { url: proxy_url, username: None, password: None }); + config.config.store(Arc::new(updated)); + let mut sensitive = reqwest::header::HeaderMap::new(); + sensitive + .insert(reqwest::header::AUTHORIZATION, reqwest::header::HeaderValue::from_static("Bearer listener-secret")); + sensitive.insert(reqwest::header::COOKIE, reqwest::header::HeaderValue::from_static("session=listener-secret")); + let local = tuliprox_core::utils::request::create_client(&config) + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .default_headers(sensitive) + .build()?; + let recording_headers = std::collections::HashMap::from([ + ("Authorization".to_string(), "Bearer recording-secret".to_string()), + ("Cookie".to_string(), "session=recording-secret".to_string()), + ("X-Recording-Test".to_string(), "custom".to_string()), + ]); + let upstream = tuliprox_core::utils::request::create_client(&config) + .redirect(reqwest::redirect::Policy::none()) + .default_headers(tuliprox_core::utils::request::get_request_headers(Some(&recording_headers), None, None, None)) + .build()?; + let response = tokio::time::timeout( + Duration::from_secs(5), + super::super::send_download_request( + &local, + Some(&upstream), + &local_url, + 4, + &RwLock::new(RecordingControl::None), + &Notify::new(), + ), + ) + .await? + .map_err(|result| format!("request failed: {result:?}"))?; + assert_eq!(response.status(), reqwest::StatusCode::PARTIAL_CONTENT); + assert_eq!(response.bytes().await?.as_ref(), b"456789"); + let local_request = local_task.await??; + let upstream_requests = proxy_task.await??; + assert!(local_request.starts_with("get /capture ")); + assert!(upstream_requests[0].starts_with("get http://recording-origin.invalid/start ")); + assert!(upstream_requests[1].starts_with("get http://recording-origin.invalid/finish ")); + assert!(local_request.contains("authorization: bearer listener-secret"), "{local_request}"); + assert!(local_request.contains("cookie: session=listener-secret"), "{local_request}"); + assert!(!local_request.contains("recording-secret"), "{local_request}"); + assert!(upstream_requests[2].starts_with("get http://recording-cdn.invalid/end ")); + for (index, request) in upstream_requests.iter().enumerate() { + assert!(!request.contains("listener-secret"), "{request}"); + if index < 2 { + assert!(request.contains("authorization: bearer recording-secret"), "{request}"); + assert!(request.contains("cookie: session=recording-secret"), "{request}"); + } else { + assert!(!request.contains("recording-secret"), "{request}"); + } + assert!(request.contains("x-recording-test: custom"), "{request}"); + } + for request in std::iter::once(&local_request).chain(&upstream_requests) { + assert!(request.contains("range: bytes=4-"), "{request}"); + assert!(request.contains("accept-encoding: identity"), "{request}"); + assert!(request.contains(&shared::model::RECORDING_STREAM_USER_AGENT.to_ascii_lowercase()), "{request}"); + } + Ok(()) +} + +#[tokio::test] +async fn recording_redirects_without_upstream_client_strip_listener_credentials( +) -> Result<(), Box> { + use tokio::io::AsyncWriteExt; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let upstream = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let local_url = reqwest::Url::parse(&format!("http://{}/capture", listener.local_addr()?))?; + let redirect = format!( + "HTTP/1.1 302 Found\r\nLocation: http://{}/capture\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + upstream.local_addr()? + ); + let server = tokio::spawn(async move { + let mut requests = Vec::new(); + for (listener, response) in [ + (listener, redirect), + (upstream, "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string()), + ] { + let (mut socket, _) = listener.accept().await?; + let request = read_request(&mut socket).await?; + socket.write_all(response.as_bytes()).await?; + requests.push(request); + } + Ok::<_, std::io::Error>(requests) + }); + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert(reqwest::header::AUTHORIZATION, reqwest::header::HeaderValue::from_static("Bearer listener-secret")); + headers.insert(reqwest::header::COOKIE, reqwest::header::HeaderValue::from_static("session=listener-secret")); + headers.insert("x-recording-test", reqwest::header::HeaderValue::from_static("custom")); + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .default_headers(headers) + .build()?; + let response = tokio::time::timeout( + Duration::from_secs(5), + super::super::send_download_request( + &client, + None, + &local_url, + 0, + &RwLock::new(RecordingControl::None), + &Notify::new(), + ), + ) + .await? + .map_err(|result| format!("request failed: {result:?}"))?; + assert_eq!(response.status(), reqwest::StatusCode::OK); + let requests = server.await??; + assert!(requests[0].contains("authorization: bearer listener-secret"), "{}", requests[0]); + assert!(requests[0].contains("cookie: session=listener-secret"), "{}", requests[0]); + assert!(!requests[1].contains("listener-secret"), "{}", requests[1]); + assert!(requests[1].contains("x-recording-test: custom"), "{}", requests[1]); + Ok(()) +} + +#[tokio::test] +async fn a_recording_whose_disk_filled_while_it_waited_never_opens_a_destination() { + // Admission happened when the request was accepted; this recording + // then waited. By the time it reaches the front of the queue the + // space it was admitted against is gone. Without the recheck the + // first sign of that is a write failure part-way through. + let dir = tempfile::TempDir::new().expect("tempdir"); + let recordings_dir = dir.path().join("recordings"); + std::fs::create_dir_all(&recordings_dir).expect("create recording dir"); + let log = dir.path().join("spawns.log"); + let script = counting_ffmpeg(dir.path(), &log); + + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open repository")); + let now = chrono::Utc::now().timestamp(); + let mut capture = scheduled_task(RecordingKind::Live, now, 1_800); + capture.uuid = "live".to_string(); + capture.state = RecordingTaskState::Queued; + capture.input_name = Some(Arc::from("provider")); + capture.recording.source.virtual_id = "42".to_string(); + capture.recording.reserved_bytes = 1_024; + capture.file_dir.clone_from(&recordings_dir); + capture.file_path = recordings_dir.join("capture.ts"); + let persisted = RecordingQueue::to_persisted(&capture); + crate::recording::recording_queue::mutate(&queue, move |candidate| { + candidate.queue.push(persisted.clone()); + Ok(()) + }) + .await + .expect("seed"); + + // The safety margin stands in for a disk that filled up: it drives + // headroom to zero without needing a real full filesystem. + let app_config = app_config_with_listener(); + let mut rec_cfg = RecordingConfig::from(&shared::model::RecordingConfigDto { enabled: true, ..Default::default() }); + rec_cfg.directory = recordings_dir.to_string_lossy().into_owned(); + rec_cfg.disk = Some(tuliprox_core::model::RecordingDiskConfig { + high_water_percent: None, + low_water_percent: None, + cleanup_interval_secs: None, + safety_bytes: Some(u64::MAX), + }); + let mut config = tuliprox_core::model::Config::clone(&app_config.config.load()); + config.video = Some(tuliprox_core::model::VideoConfig { + extensions: Vec::new(), + web_search: None, + recording: Some(rec_cfg.clone()), + }); + app_config.config.store(Arc::new(config)); + + let stub = StubCapacity::with_room(); + let capacity: Arc = Arc::clone(&stub) as Arc; + ensure_recording_worker_running(&app_config, &rec_cfg, &queue, &NoopSink, &capacity, &script) + .await + .expect("worker started"); + + let settled = tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Some(done) = queue.finished.read().await.first().cloned() { + break done; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + let Ok(settled) = settled else { + panic!( + "never settled: active={:?} spawns={}", + queue.active.read().await.first().map(|active| (active.state, active.error.clone())), + spawn_count(&log), + ); + }; + + assert_eq!(settled.state, RecordingTaskState::Failed, "{:?}", settled.error); + assert_eq!(settled.error.as_deref(), Some(DISK_GONE_BEFORE_START), "and it says which resource ran out"); + assert_eq!(spawn_count(&log), 0, "the encoder was never started"); + assert!(!capture.file_path.exists(), "and no destination was opened"); + assert_eq!(stub.release_count(), 1, "the provider slot it had was given back"); + assert_eq!(settled.recording.reserved_bytes, 0, "and it is not still holding disk"); +} diff --git a/backend/dvr/src/recording/recording_transfer/tests/retry.rs b/backend/dvr/src/recording/recording_transfer/tests/retry.rs new file mode 100644 index 000000000..af6c7cce6 --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/tests/retry.rs @@ -0,0 +1,49 @@ +use super::scheduled_task; +use crate::recording::recording_queue::RecordingQueue; +use shared::model::{RecordingKind, RecordingTaskState}; +use tuliprox_core::model::RecordingConfig; + +#[tokio::test] +async fn media_retry_waiting_and_limit_keep_the_transfer_cause() -> Result<(), Box> { + for kind in [RecordingKind::Vod, RecordingKind::Series] { + let queue = RecordingQueue::new(); + let task = scheduled_task(kind, chrono::Utc::now().timestamp(), 300); + let persisted = RecordingQueue::to_persisted(&task); + crate::recording::recording_queue::mutate(&queue, move |candidate| { + candidate.active = vec![persisted.clone()]; + Ok(()) + }) + .await?; + let config = RecordingConfig::from(&shared::model::RecordingConfigDto { + retry_max_attempts: 1, + retry_backoff_initial_secs: 1, + retry_backoff_max_secs: 1, + retry_backoff_jitter_percent: 0, + ..Default::default() + }); + let cause = "Error while opening stream: (http://user:password@upstream.example/live/1?token=secret) Operation timed out"; + let public_cause = crate::recording::recording_worker::redact_url_tokens(cause); + assert_eq!(public_cause, "Error while opening stream: [stream URL] Operation timed out"); + let retry = super::super::prepare_active_retry(&queue, "task", &config, &public_cause).await?; + assert!(matches!(retry, Some(super::super::RetryCommit::Waiting { delay_secs: 1, attempts: 1 }))); + { + let active = queue.active.read().await; + let active = active.first().ok_or("active transfer missing")?; + assert_eq!(active.state, RecordingTaskState::RetryWaiting); + assert!(active.error.as_deref().is_some_and(|error| error.contains(&public_cause))); + assert!(active.next_retry_at.is_some()); + } + let retry = super::super::prepare_active_retry(&queue, "task", &config, &public_cause).await?; + assert!(matches!(retry, Some(super::super::RetryCommit::Failed(_)))); + let finished = queue.finished.read().await; + let failed = finished.first().ok_or("terminal transfer missing")?; + assert_eq!(failed.state, RecordingTaskState::Failed); + assert_eq!( + failed.error.as_deref(), + Some(format!("Retry limit reached after 1 attempts: {public_cause}").as_str()) + ); + assert_eq!(failed.recording.reserved_bytes, 0); + assert!(failed.next_retry_at.is_none()); + } + Ok(()) +} diff --git a/backend/dvr/src/recording/recording_transfer/tests/startup.rs b/backend/dvr/src/recording/recording_transfer/tests/startup.rs new file mode 100644 index 000000000..8b9d48bbd --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/tests/startup.rs @@ -0,0 +1,41 @@ +use super::{refresh_recording_progress, scheduled_task}; +use crate::recording::recording_queue::RecordingQueue; +use shared::model::{EventMessage, EventSink, RecordingKind}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; +use tempfile::TempDir; + +#[derive(Clone, Default)] +pub(in crate::recording::recording_transfer::tests) struct ProgressSink( + pub(in crate::recording::recording_transfer::tests) Arc, +); + +impl EventSink for ProgressSink { + fn emit(&self, event: EventMessage) { + if matches!(event, EventMessage::RecordingProgress) { + self.0.fetch_add(1, Ordering::Relaxed); + } + } +} + +#[tokio::test] +async fn live_byte_progress_is_visible_without_a_queue_revision_change() { + let dir = TempDir::new().expect("tempdir"); + let partial = dir.path().join("capture.ts.partial"); + std::fs::write(&partial, b"growing capture").expect("write partial recording"); + let queue = RecordingQueue::new(); + *queue.active.write().await = vec![scheduled_task(RecordingKind::Live, 0, 60)]; + let events = ProgressSink::default(); + + refresh_recording_progress(&queue.active, "task", &partial, &events).await; + let (revision, tasks) = queue.committed_snapshot().await; + + assert_eq!(revision.0, 0); + assert_eq!(tasks[0].size, b"growing capture".len() as u64); + assert_eq!(events.0.load(Ordering::Relaxed), 1); + + refresh_recording_progress(&queue.active, "task", &partial, &events).await; + assert_eq!(events.0.load(Ordering::Relaxed), 1, "unchanged bytes need no second event"); +} diff --git a/backend/dvr/src/recording/recording_transfer/tests/support.rs b/backend/dvr/src/recording/recording_transfer/tests/support.rs new file mode 100644 index 000000000..407315d9a --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/tests/support.rs @@ -0,0 +1,392 @@ +#[cfg(unix)] +use super::{ensure_recording_worker_running, start_recording_scheduler}; +#[cfg(unix)] +use crate::recording::recording_capacity::RecordingCapacityPort; +use crate::recording::recording_queue::{PersistedRecordingTask, RecordingPartition, RecordingQueue, RecordingTask}; +#[cfg(unix)] +use shared::model::NoopSink; +use shared::model::{ + RecordingKind, RecordingMetadata, RecordingOwner, RecordingSource, RecordingTaskState, RecordingVisibility, UserId, +}; +#[cfg(unix)] +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::{ + io::{Read, Write}, + net::TcpListener, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; +use tempfile::TempDir; +#[cfg(unix)] +use tokio::sync::Notify; +#[cfg(unix)] +use tuliprox_core::model::RecordingConfig; + +/// A task whose Live window runs from `program_start` for `duration_secs`. +pub(in crate::recording::recording_transfer::tests) fn scheduled_task( + kind: RecordingKind, + program_start: i64, + duration_secs: i64, +) -> RecordingTask { + let meta = RecordingMetadata::new_live( + RecordingOwner::User(UserId::from("web:alice")), + RecordingVisibility::Private, + RecordingSource::new("t1", "v1", "in1"), + program_start, + program_start + duration_secs, + 0, + 0, + ); + RecordingQueue::from_persisted(PersistedRecordingTask { + media_identity: String::new(), + partition: RecordingPartition::default(), + uuid: "task".to_string(), + kind, + file_dir: PathBuf::from("/tmp"), + file_path: PathBuf::from("/tmp/capture.ts"), + filename: "capture.ts".to_string(), + url: "https://example.com/stream".to_string(), + finished: false, + size: 0, + total_size: None, + paused: false, + error: None, + state: RecordingTaskState::Running, + input_name: None, + priority: 0, + retry_attempts: 0, + next_retry_at: None, + recording: meta, + }) + .expect("valid fixture") +} + +pub(in crate::recording::recording_transfer::tests) fn serve_range_fixture( + ignore_range: bool, +) -> (reqwest::Url, std::thread::JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind fixture server"); + let url = reqwest::Url::parse(&format!("http://{}/download", listener.local_addr().expect("server address"))) + .expect("fixture URL"); + let server = std::thread::spawn(move || { + let (mut socket, _) = listener.accept().expect("accept download request"); + socket.set_read_timeout(Some(Duration::from_secs(5))).expect("read timeout"); + let mut request = Vec::new(); + let mut chunk = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = socket.read(&mut chunk).expect("read download request"); + assert!(read > 0, "request ended before headers"); + request.extend_from_slice(&chunk[..read]); + } + let request = String::from_utf8(request).expect("request headers"); + let (status, headers, body) = if ignore_range { + ("200 OK", "", b"0123456789".as_slice()) + } else { + ("206 Partial Content", "Content-Range: bytes 4-9/10\r\n", b"456789".as_slice()) + }; + let response = + format!("HTTP/1.1 {status}\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + socket.write_all(response.as_bytes()).expect("write response headers"); + socket.write_all(body).expect("write response body"); + request + }); + (url, server) +} + +/// One VOD on one media for `owner`, in the given partition and state. +pub(in crate::recording::recording_transfer::tests) fn vod_entry( + uuid: &str, + owner: &str, + state: RecordingTaskState, +) -> PersistedRecordingTask { + let mut task = scheduled_task(RecordingKind::Vod, 0, 0); + task.uuid = uuid.to_string(); + task.state = state; + task.recording = RecordingMetadata::new_media( + RecordingOwner::User(UserId::from(owner)), + RecordingVisibility::Private, + RecordingSource::new("t1", "v1", "in1"), + "Film".to_string(), + ); + let mut persisted = RecordingQueue::to_persisted(&task); + persisted.media_identity = "film".to_string(); + persisted +} + +/// A queue with nothing in it; the wait queue is what these exercise. +pub(in crate::recording::recording_transfer::tests) fn slot_queue(dir: &TempDir) -> RecordingQueue { + RecordingQueue::new_persistent(dir.path(), dir.path()).expect("open recording repository") +} + +pub(in crate::recording::recording_transfer::tests) fn bare_app_config() -> tuliprox_core::model::AppConfig { + tuliprox_core::model::AppConfig { + config: Arc::new(arc_swap::ArcSwap::from_pointee(tuliprox_core::model::Config::default())), + sources: Arc::new(arc_swap::ArcSwap::from_pointee(tuliprox_core::model::SourcesConfig::default())), + hdhomerun: Arc::new(arc_swap::ArcSwapOption::default()), + api_proxy: Arc::new(arc_swap::ArcSwapOption::default()), + file_locks: Arc::new(tuliprox_core::utils::FileLockManager::default()), + paths: Arc::new(arc_swap::ArcSwap::from_pointee(shared::model::ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(arc_swap::ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(tuliprox_core::model::MediaToolCapabilities::new()), + } +} + +/// Configured local API listener with no public playback server. +pub(in crate::recording::recording_transfer::tests) fn app_config_with_listener() -> tuliprox_core::model::AppConfig { + let app_config = bare_app_config(); + let mut config = (*app_config.config.load_full()).clone(); + config.api.host = "127.0.0.1".to_string(); + config.api.port = 8901; + app_config.config.store(Arc::new(config)); + app_config +} + +/// An executable standing in for ffmpeg that appends a line to `log` +/// every time it is run, then writes its output argument and exits +/// cleanly. The log is what lets a test say "once" rather than "at +/// least once". +pub(in crate::recording::recording_transfer::tests) fn counting_ffmpeg(dir: &Path, log: &Path) -> PathBuf { + let script_path = dir.join("fake-ffmpeg"); + std::fs::write( + &script_path, + format!( + "#!/bin/sh\necho run >> \"{}\"\nfor arg in \"$@\"; do output=\"$arg\"; done\nprintf 'recorded' > \"$output\"\nexit 0\n", + log.display() + ), + ) + .expect("write fake ffmpeg"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = std::fs::metadata(&script_path).expect("metadata").permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&script_path, perms).expect("chmod"); + } + script_path +} + +pub(in crate::recording::recording_transfer::tests) fn spawn_count(log: &Path) -> usize { + std::fs::read_to_string(log).map_or(0, |text| text.lines().filter(|line| !line.is_empty()).count()) +} + +#[cfg(unix)] +pub(in crate::recording::recording_transfer::tests) struct LimitedCapacity { + pub(in crate::recording::recording_transfer::tests) limit: usize, + pub(in crate::recording::recording_transfer::tests) in_use: AtomicUsize, + pub(in crate::recording::recording_transfer::tests) peak: AtomicUsize, + pub(in crate::recording::recording_transfer::tests) releases: AtomicUsize, + pub(in crate::recording::recording_transfer::tests) notify: Arc, +} + +#[cfg(unix)] +impl RecordingCapacityPort for LimitedCapacity { + fn capacities_for_input<'a>( + &'a self, + input: &'a Arc, + ) -> futures::future::BoxFuture<'a, Vec> { + Box::pin(async move { + let in_use = self.in_use.load(Ordering::SeqCst); + tokio::task::yield_now().await; + vec![(input.clone(), in_use, self.limit)] + }) + } + + fn acquire<'a>( + &'a self, + _input: &'a Arc, + _priority: i8, + ) -> futures::future::BoxFuture<'a, Option> { + Box::pin(async move { + let previous = self + .in_use + .try_update(Ordering::SeqCst, Ordering::SeqCst, |used| { + (self.limit == 0 || used < self.limit).then_some(used + 1) + }) + .ok()?; + self.peak.fetch_max(previous + 1, Ordering::SeqCst); + Some(tuliprox_core::model::ProviderHandle::new( + std::net::SocketAddr::from(([127, 0, 0, 1], 0)), + 1, + tuliprox_core::model::ProviderAllocation::Exhausted, + Some(tokio_util::sync::CancellationToken::new()), + )) + }) + } + + fn release(&self, handle: Option) -> futures::future::BoxFuture<'_, ()> { + Box::pin(async move { + if handle.is_some() { + self.in_use.fetch_sub(1, Ordering::SeqCst); + self.releases.fetch_add(1, Ordering::SeqCst); + self.notify.notify_one(); + } + }) + } + + fn capacity_changed(&self) -> Arc { self.notify.clone() } +} + +#[cfg(unix)] +pub(in crate::recording::recording_transfer::tests) struct ConcurrentLiveFixture { + pub(in crate::recording::recording_transfer::tests) dir: TempDir, + pub(in crate::recording::recording_transfer::tests) queue: Arc, + pub(in crate::recording::recording_transfer::tests) script: PathBuf, + pub(in crate::recording::recording_transfer::tests) capacity: Arc, + pub(in crate::recording::recording_transfer::tests) app: Arc, + pub(in crate::recording::recording_transfer::tests) config: RecordingConfig, + pub(in crate::recording::recording_transfer::tests) cancel: tokio_util::sync::CancellationToken, +} + +#[cfg(unix)] +impl ConcurrentLiveFixture { + pub(in crate::recording::recording_transfer::tests) async fn new( + count: u32, + limit: usize, + ) -> Result> { + Self::with_background_limit(count, limit, 0).await + } + + pub(in crate::recording::recording_transfer::tests) async fn with_background_limit( + count: u32, + limit: usize, + background_limit: u8, + ) -> Result> { + let dir = TempDir::new()?; + let queue = Arc::new(RecordingQueue::new_persistent(dir.path(), dir.path())?); + let script = counting_ffmpeg(dir.path(), &dir.path().join("spawns.log")); + std::fs::write(&script, "#!/bin/sh\nfor arg in \"$@\"; do output=\"$arg\"; done\nprintf recorded > \"$output\"\ntouch \"$output.ready\"\nread -r control\nexit 0\n")?; + let now = chrono::Utc::now().timestamp(); + let tasks = (0..count) + .map(|index| { + let mut task = scheduled_task(RecordingKind::Live, now, 300); + task.uuid = format!("live-{index}"); + task.state = RecordingTaskState::Queued; + task.priority = 5; + task.input_name = Some(Arc::from("provider")); + task.recording.source.virtual_id = (42 + index).to_string(); + task.recording.source.input_name = "provider".to_string(); + task.file_dir = dir.path().to_path_buf(); + task.file_path = dir.path().join(format!("live-{index}.ts")); + RecordingQueue::to_persisted(&task) + }) + .collect::>(); + crate::recording::recording_queue::mutate(&queue, |candidate| { + candidate.queue.extend(tasks); + Ok(()) + }) + .await?; + let fixture = Self { + dir, + queue, + script, + capacity: Arc::new(LimitedCapacity { + limit, + in_use: AtomicUsize::new(0), + peak: AtomicUsize::new(0), + releases: AtomicUsize::new(0), + notify: Arc::new(Notify::new()), + }), + app: Arc::new(app_config_with_listener()), + config: RecordingConfig::from(&shared::model::RecordingConfigDto { + enabled: true, + max_background_per_provider: background_limit, + ..Default::default() + }), + cancel: tokio_util::sync::CancellationToken::new(), + }; + start_recording_scheduler( + fixture.app.clone(), + fixture.config.clone(), + &fixture.queue, + NoopSink, + fixture.capacity.clone(), + fixture.cancel.clone(), + fixture.script.clone(), + ); + Ok(fixture) + } + + pub(in crate::recording::recording_transfer::tests) async fn start(&self) -> Result<(), String> { + let capacity: Arc = self.capacity.clone(); + ensure_recording_worker_running(&self.app, &self.config, &self.queue, &NoopSink, &capacity, &self.script).await + } + + pub(in crate::recording::recording_transfer::tests) async fn wait_for_running( + &self, + count: usize, + ) -> Result, tokio::time::error::Elapsed> { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let ready = self + .queue + .active + .read() + .await + .iter() + .filter(|task| { + task.state == RecordingTaskState::Running + && self.dir.path().join(format!("{}.ts.partial.ready", task.uuid)).exists() + }) + .map(|task| task.uuid.clone()) + .collect::>(); + if ready.len() == count { + break ready; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + } + + pub(in crate::recording::recording_transfer::tests) async fn stop_all( + &self, + count: usize, + ) -> Result<(), Box> { + let active = self.queue.active.read().await.iter().map(|task| task.uuid.clone()).collect::>(); + for uuid in active { + self.queue.cancel_requested(&uuid).await?; + } + tokio::time::timeout(Duration::from_secs(5), async { + while self.queue.finished.read().await.len() != count || self.queue.workers_running().await { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await?; + assert_eq!(self.capacity.in_use.load(Ordering::SeqCst), 0); + Ok(()) + } +} + +#[cfg(unix)] +impl Drop for ConcurrentLiveFixture { + fn drop(&mut self) { self.cancel.cancel(); } +} + +pub(in crate::recording::recording_transfer::tests) async fn read_request( + socket: &mut tokio::net::TcpStream, +) -> std::io::Result { + use tokio::io::AsyncReadExt; + let mut request = Vec::new(); + let mut byte = [0; 1]; + while !request.ends_with(b"\r\n\r\n") { + if socket.read(&mut byte).await? == 0 { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "incomplete request headers")); + } + request.push(byte[0]); + } + Ok(String::from_utf8_lossy(&request).to_ascii_lowercase()) +} diff --git a/backend/dvr/src/recording/recording_transfer/worker.rs b/backend/dvr/src/recording/recording_transfer/worker.rs new file mode 100644 index 000000000..212dc1477 --- /dev/null +++ b/backend/dvr/src/recording/recording_transfer/worker.rs @@ -0,0 +1,823 @@ +use super::{ + acquire_result_after_wait, acquire_result_for_control, active_download_snapshot_for_worker, + background_download_should_wait, broadcast_required_worker_mutation, broadcast_worker_mutation, + cancel_active_and_promote, capacities_have_free_slot, commit_acquired_download, continue_after_pause, + download_file, fail_active_download, finish_active_and_promote, mark_recording_metadata_notification, + preemption_reason_for, prepare_active_retry, promote_ready_downloads, publish_recording_change, + recording_deadline_instant, recording_execution_download, refresh_recording_progress, refused_before_start, + requeue_active_download_for_retry, set_active_download_state, should_exit_worker_after_preempt, + spawn_recording_notification_after_persist, take_active, wait_for_provider_slot, write_completion_sidecar, + ProviderAcquireResult, ProviderCapacities, QuotaGate, RetryCommit, DOWNLOAD_PREEMPTED_REASON, + LIVE_CAPACITY_WINDOW_CLOSED, RECORDING_PROGRESS_UPDATE_INTERVAL, +}; +use crate::recording::{ + recording_capacity::RecordingCapacityPort, + recording_ctx::RecordingCtx, + recording_notification::LifecycleEvent, + recording_queue::{ + mutate_optional, PersistedRecordingQueue, QueueMutationError, RecordingControl, RecordingQueue, + RecordingTaskState, + }, + recording_worker::{ + recording_partial_path, redact_url_tokens, run_recording_with_binary, RecordingExecutionResult, + }, +}; +use log::{debug, error, info, warn}; +use shared::model::{EventSink, RecordingKind}; +use std::{collections::HashMap, path::Path, sync::Arc}; +use tokio::{time, time::Duration}; +use tokio_util::sync::CancellationToken; +use tuliprox_core::{ + model::{AppConfig, RecordingConfig}, + utils::{request, request::create_client}, +}; + +#[derive(Debug)] +pub(super) enum DownloadExecutionResult { + Completed, + Paused, + Cancelled, + Preempted, + Retryable(String), + Failed(String), +} + +pub(super) async fn requeue_active_download_for_capacity_wait( + download_queue: &RecordingQueue, + uuid: &str, + reason: &str, + promote: bool, + consumed_control: Option, +) -> Result { + let mutation = |candidate: &mut PersistedRecordingQueue| { + let Some(mut download) = take_active(candidate, uuid) else { + return Ok(None); + }; + download.finished = false; + download.paused = false; + download.error = Some(reason.to_string()); + download.state = RecordingTaskState::WaitingForCapacity; + download.next_retry_at = None; + candidate.queue.insert(0, download); + if promote { + crate::recording::recording_queue::promote_from_queue(candidate); + } + Ok(Some(true)) + }; + let result = if let Some(control) = consumed_control { + download_queue.mutate_optional_and_clear_control(uuid, control, mutation).await? + } else { + mutate_optional(download_queue, mutation).await? + }; + Ok(result.unwrap_or(false)) +} + +pub async fn ensure_recording_worker_running( + cfg: &AppConfig, + download_cfg: &RecordingConfig, + download_queue: &Arc, + event_manager: &E, + capacity: &Arc, + recording_binary: &Path, +) -> Result<(), String> { + if promote_ready_downloads(download_queue).await.map_err(|err| err.to_string())? { + publish_recording_change(event_manager); + } + let ready: Vec<_> = download_queue + .active + .read() + .await + .iter() + .filter(|task| !task.paused && !task.finished) + .map(|task| task.uuid.clone()) + .collect(); + for uuid in ready { + start_recording_worker(cfg, download_cfg, download_queue, event_manager, capacity, recording_binary, uuid) + .await?; + } + Ok(()) +} + +#[allow(clippy::too_many_lines, clippy::too_many_arguments)] +async fn start_recording_worker( + cfg: &AppConfig, + download_cfg: &RecordingConfig, + download_queue: &Arc, + event_manager: &E, + capacity: &Arc, + recording_binary: &Path, + worker_uuid: String, +) -> Result<(), String> { + let Some(worker) = download_queue.claim_worker(&worker_uuid).await else { + return Ok(()); + }; + + if download_queue.active.read().await.iter().any(|task| task.uuid == worker_uuid) { + let config = cfg.config.load(); + let disabled_headers = cfg.get_disabled_headers(); + let upstream_headers = request::get_request_headers( + Some(&download_cfg.headers), + None, + disabled_headers.as_ref(), + config.default_user_agent.as_deref(), + ); + let dq = Arc::clone(download_queue); + let control_signal = Arc::clone(&worker.control_signal); + let control_notify = Arc::clone(&worker.control_notify); + let event_manager = event_manager.clone(); + let capacity = Arc::clone(capacity); + let download_cfg = download_cfg.clone(); + let recording_binary = recording_binary.to_path_buf(); + let app_config = Arc::new(cfg.clone()); + + // The listener authenticates through its recording URL and bypasses proxies. + // Provider headers stay on the upstream client; proxied listener requests + // receive them through the recording input configuration. + // Notifications retain the proxy without the recording headers. + let clients = ( + create_client(cfg).no_proxy().redirect(reqwest::redirect::Policy::none()).build(), + create_client(cfg).redirect(reqwest::redirect::Policy::none()).default_headers(upstream_headers).build(), + create_client(cfg).build(), + ); + if let (Ok(transfer_client), Ok(upstream_transfer_client), Ok(client)) = clients { + if let Some(active) = dq.active.read().await.iter().find(|task| task.uuid == worker_uuid) { + info!("Starting download worker for active download {} ({})", active.uuid, active.filename); + } + tokio::spawn(async move { + 'worker: loop { + // One read: the uuid, pause flag and live window must describe + // the same task. + let active_head = dq + .active + .read() + .await + .iter() + .find(|task| task.uuid == worker_uuid) + .map(|download| (download.paused, recording_deadline_instant(download))); + if let Some((paused, window_deadline)) = active_head { + if paused { + break; + } + + // Acquire a provider connection slot for this download. + // If the provider is at capacity, wait in the priority queue until signalled. + // Never proceeds without a slot when input_name is set — account bans otherwise. + let provider_acquire_result = { + let (input_name, priority) = + dq.active_scheduling_priority(&worker_uuid).await.unwrap_or((None, 0i8)); + // Only live work has a window deadline; a transfer waits as long as it takes. + if let Some(input_name) = input_name { + loop { + if let Some(stopped) = acquire_result_for_control(*control_signal.read().await) { + break stopped; + } + let handle = { + // Keep the policy check and allocation together so + // concurrent recording starts respect background limits. + let _capacity = dq.capacity_guard.lock().await; + let capacities = capacity.capacities_for_input(&input_name).await; + let live_config = app_config.config.load(); + let capacity_cfg = live_config.recording().unwrap_or(&download_cfg); + if background_download_should_wait(priority, &capacities, capacity_cfg) { + None + } else { + capacity.acquire(&input_name, priority).await + } + }; + if let Some(handle) = handle { + break ProviderAcquireResult::Acquired(Some(handle)); + } + if let Err(err) = broadcast_worker_mutation( + &event_manager, + set_active_download_state( + &dq, + &worker_uuid, + RecordingTaskState::WaitingForCapacity, + None, + false, + ) + .await, + "waiting-for-capacity state", + ) { + error!("Download worker commit failed: {err}"); + break 'worker; + } + // Wait for highest-priority signal — no sleep, no polling. + let waited = wait_for_provider_slot( + &dq, + &input_name, + priority, + control_signal.as_ref(), + control_notify.as_ref(), + window_deadline, + ) + .await; + if let Some(stopped) = acquire_result_after_wait(waited) { + break stopped; + } + } + } else { + ProviderAcquireResult::Acquired(None) + } + }; + + let provider_handle = match provider_acquire_result { + ProviderAcquireResult::Acquired(handle) => { + match commit_acquired_download(&dq, &worker_uuid).await { + Ok(Some(notification)) => { + spawn_recording_notification_after_persist( + &app_config, + &client, + notification, + true, + ); + publish_recording_change(&event_manager); + } + Ok(None) => { + capacity.release(handle).await; + error!("Download worker active task changed after provider acquire"); + break 'worker; + } + Err(err) => { + capacity.release(handle).await; + error!("Download worker commit failed after provider acquire: {err}"); + break 'worker; + } + } + handle + } + ProviderAcquireResult::WindowClosed => { + // No slot was ever acquired, so there is nothing + // to release and nothing was written. + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + fail_active_download(&dq, &worker_uuid, LIVE_CAPACITY_WINDOW_CLOSED).await, + "live capacity window closed", + ) { + error!("Download worker commit failed: {err}"); + break 'worker; + } + continue; + } + ProviderAcquireResult::Paused => { + break; + } + ProviderAcquireResult::Cancelled => { + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + cancel_active_and_promote(&dq, &worker_uuid).await, + "provider-wait cancellation", + ) { + error!("Download worker commit failed: {err}"); + break 'worker; + } + continue; + } + ProviderAcquireResult::Preempted => { + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + requeue_active_download_for_capacity_wait( + &dq, + &worker_uuid, + "Reloading download service configuration", + true, + Some(RecordingControl::Restart), + ) + .await, + "configuration-reload requeue", + ) { + error!("Download worker commit failed: {err}"); + break 'worker; + } + break; + } + }; + + let mut execution_result = { + let Some(download) = active_download_snapshot_for_worker(&dq.active, &worker_uuid).await + else { + capacity.release(provider_handle).await; + break 'worker; + }; + // Last point before a destination is opened. + if let Some(reason) = refused_before_start(&dq, &app_config, &worker_uuid).await { + capacity.release(provider_handle).await; + match fail_active_download(&dq, &worker_uuid, reason).await { + Ok(_) => publish_recording_change(&event_manager), + Err(err) => { + error!("Download worker commit failed: {err}"); + break 'worker; + } + } + continue; + } + match download.kind { + RecordingKind::Vod | RecordingKind::Series => 'http_execution: { + let execution_download = match recording_execution_download( + &app_config, + &download, + provider_handle.as_ref(), + ) { + Ok(execution) => execution, + Err(error) => break 'http_execution DownloadExecutionResult::Failed(error), + }; + download_file( + Arc::clone(&dq.active), + execution_download, + &transfer_client, + Some(&upstream_transfer_client), + Arc::clone(&control_signal), + Arc::clone(&control_notify), + Some(&event_manager), + Some(QuotaGate { queue: &dq, app_config: &app_config }), + ) + .await + } + RecordingKind::Live => 'recording_execution: { + let execution_download = match recording_execution_download( + &app_config, + &download, + provider_handle.as_ref(), + ) { + Ok(execution_download) => execution_download, + Err(err) => break 'recording_execution DownloadExecutionResult::Failed(err), + }; + let progress_path = recording_partial_path(&execution_download.file_path); + let container_format = + app_config.config.load().recording().map_or_else( + shared::model::RecordingContainerFormat::default, + |recording| recording.container_format, + ); + let mut recording_future = Box::pin(run_recording_with_binary( + &recording_binary, + &execution_download, + &control_signal, + &control_notify, + None, + container_format, + )); + let mut progress_tick = time::interval(RECORDING_PROGRESS_UPDATE_INTERVAL); + progress_tick.set_missed_tick_behavior(time::MissedTickBehavior::Skip); + + let result = loop { + tokio::select! { + recording_result = &mut recording_future => break recording_result, + _ = progress_tick.tick() => { + refresh_recording_progress( + &dq.active, + &worker_uuid, + &progress_path, + &event_manager, + ) + .await; + } + } + }; + let final_progress_path = if matches!(&result, RecordingExecutionResult::Completed) + { + &execution_download.file_path + } else { + &progress_path + }; + refresh_recording_progress( + &dq.active, + &worker_uuid, + final_progress_path, + &event_manager, + ) + .await; + + match result { + RecordingExecutionResult::Completed => DownloadExecutionResult::Completed, + RecordingExecutionResult::Paused => DownloadExecutionResult::Paused, + RecordingExecutionResult::Cancelled => DownloadExecutionResult::Cancelled, + RecordingExecutionResult::Preempted => DownloadExecutionResult::Preempted, + RecordingExecutionResult::Retryable(err) => { + DownloadExecutionResult::Retryable(err) + } + RecordingExecutionResult::Failed(err) => DownloadExecutionResult::Failed(err), + } + } + } + }; + + // The internal HTTP stream owns the same provider allocation, and its + // normal EOF/client-close cleanup cancels the allocation token too, so + // cancellation alone does not mean foreground preemption. The manager + // records an explicit close reason before a real priority eviction; + // only that outcome requeues the recording. + if provider_handle.as_ref().is_some_and(|handle| { + handle.get_close_reason() == tuliprox_core::model::ProviderCloseReason::PriorityPreempted + }) { + execution_result = DownloadExecutionResult::Preempted; + } + + match execution_result { + DownloadExecutionResult::Completed => { + capacity.release(provider_handle).await; + // Path copied out first: the lock must not be held across + // the filesystem call, or progress writers and commits stall. + let finished_file = dq + .active + .read() + .await + .iter() + .find(|task| task.uuid == worker_uuid) + .map(|fd| (fd.file_path.clone(), fd.size)); + let measured_bytes = match finished_file { + Some((path, fallback)) => { + tokio::fs::metadata(&path).await.map_or(fallback, |metadata| metadata.len()) + } + None => 0, + }; + // Beside the file, before the repository is told + // it is complete: an operator finding an orphan + // needs the record even if the commit is what + // failed. + write_completion_sidecar(&dq, &worker_uuid, measured_bytes).await; + let completion_limits = app_config.config.load().recording().map(|recording| { + crate::recording::recording_service::quota_limits_from_config( + recording.quota.as_ref(), + ) + }); + let committed = + finish_active_and_promote(&dq, &worker_uuid, completion_limits.as_ref(), |fd| { + fd.finished = true; + fd.paused = false; + fd.state = RecordingTaskState::Completed; + fd.size = measured_bytes; + fd.error = None; + fd.next_retry_at = None; + let meta = &mut fd.recording; + meta.measured_bytes = measured_bytes; + meta.reserved_bytes = 0; + meta.completed_at = Some(chrono::Utc::now().timestamp()); + meta.partial_relative_path = None; + mark_recording_metadata_notification( + &mut fd.recording, + LifecycleEvent::Completed, + None, + ) + }) + .await; + match committed { + Ok(Some(notification)) => { + spawn_recording_notification_after_persist( + &app_config, + &client, + notification, + true, + ); + publish_recording_change(&event_manager); + } + Ok(None) => {} + Err(err) => { + error!("Failed to persist completed download state: {err}"); + break 'worker; + } + } + } + DownloadExecutionResult::Paused => { + capacity.release(provider_handle).await; + // A resume observed here continues this worker. A later resume + // is picked up after release_worker clears the claim and wakes + // the scheduler to start a new worker. + if continue_after_pause(&dq, &worker_uuid).await { + continue 'worker; + } + break 'worker; + } + DownloadExecutionResult::Cancelled => { + capacity.release(provider_handle).await; + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + cancel_active_and_promote(&dq, &worker_uuid).await, + "cancelled state", + ) { + error!("Download worker commit failed: {err}"); + break 'worker; + } + } + DownloadExecutionResult::Preempted => { + capacity.release(provider_handle).await; + let control = *control_signal.read().await; + match control { + RecordingControl::Restart => warn!( + "Active transfer is restarting to apply updated download service configuration" + ), + _ => warn!("Active transfer was preempted by a higher-priority stream"), + } + let reason = { + let active = dq.active.read().await; + if control == RecordingControl::Restart { + "Reloading download service configuration" + } else { + active + .iter() + .find(|task| task.uuid == worker_uuid) + .map_or(DOWNLOAD_PREEMPTED_REASON, preemption_reason_for) + } + }; + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + requeue_active_download_for_capacity_wait( + &dq, + &worker_uuid, + reason, + true, + (control == RecordingControl::Restart).then_some(RecordingControl::Restart), + ) + .await, + "preempted requeue", + ) { + error!("Download worker commit failed: {err}"); + break 'worker; + } + if should_exit_worker_after_preempt(control) { + break; + } + } + DownloadExecutionResult::Retryable(err) => { + capacity.release(provider_handle).await; + let err = redact_url_tokens(&err); + warn!("Recording transfer encountered a transient failure: {err}"); + let retry_commit = prepare_active_retry(&dq, &worker_uuid, &download_cfg, &err).await; + let retry_delay_secs = match retry_commit { + Ok(Some(RetryCommit::Waiting { delay_secs, attempts })) => { + debug!("Download retry attempt {attempts} scheduled in {delay_secs}s"); + publish_recording_change(&event_manager); + delay_secs + } + Ok(Some(RetryCommit::Failed(notification))) => { + spawn_recording_notification_after_persist( + &app_config, + &client, + notification, + true, + ); + publish_recording_change(&event_manager); + if dq.active.read().await.iter().any(|task| task.uuid == worker_uuid) { + continue; + } + break; + } + Ok(None) => break, + Err(err) => { + error!("Failed to persist retry state: {err}"); + break; + } + }; + let mut retry_sleep = Box::pin(time::sleep(Duration::from_secs(retry_delay_secs))); + let retry_wait_outcome = loop { + tokio::select! { + () = &mut retry_sleep => break DownloadExecutionResult::Retryable(String::new()), + () = control_notify.notified() => { + match *control_signal.read().await { + RecordingControl::Pause => break DownloadExecutionResult::Paused, + RecordingControl::Cancel => break DownloadExecutionResult::Cancelled, + RecordingControl::Restart => break DownloadExecutionResult::Preempted, + RecordingControl::None => {} + } + } + } + }; + + match retry_wait_outcome { + DownloadExecutionResult::Retryable(_) => { + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + requeue_active_download_for_retry(&dq, &worker_uuid, true).await, + "retry requeue", + ) { + error!("Download worker commit failed: {err}"); + break 'worker; + } + } + DownloadExecutionResult::Paused => { + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + set_active_download_state( + &dq, + &worker_uuid, + RecordingTaskState::Paused, + None, + true, + ) + .await, + "paused retry state", + ) { + error!("Download worker commit failed: {err}"); + break 'worker; + } + break; + } + DownloadExecutionResult::Cancelled => { + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + cancel_active_and_promote(&dq, &worker_uuid).await, + "cancelled retry state", + ) { + error!("Download worker commit failed: {err}"); + break 'worker; + } + } + DownloadExecutionResult::Completed | DownloadExecutionResult::Failed(_) => {} + DownloadExecutionResult::Preempted => { + if let Err(err) = broadcast_required_worker_mutation( + &event_manager, + requeue_active_download_for_capacity_wait( + &dq, + &worker_uuid, + "Reloading download service configuration", + true, + Some(RecordingControl::Restart), + ) + .await, + "configuration-reload retry requeue", + ) { + error!("Download worker commit failed: {err}"); + break 'worker; + } + break; + } + } + } + DownloadExecutionResult::Failed(err) => { + capacity.release(provider_handle).await; + let err = redact_url_tokens(&err); + warn!("Download failed permanently: {err}"); + let committed = finish_active_and_promote(&dq, &worker_uuid, None, |fd| { + fd.finished = true; + fd.paused = false; + fd.next_retry_at = None; + fd.error = Some(err.clone()); + fd.state = RecordingTaskState::Failed; + fd.recording.reserved_bytes = 0; + mark_recording_metadata_notification( + &mut fd.recording, + LifecycleEvent::Failed, + Some(err), + ) + }) + .await; + match committed { + Ok(Some(notification)) => { + spawn_recording_notification_after_persist( + &app_config, + &client, + notification, + true, + ); + publish_recording_change(&event_manager); + } + Ok(None) => {} + Err(commit_err) => { + error!("Failed to persist failed download state: {commit_err}"); + break 'worker; + } + } + } + } + } else { + break; + } + } + dq.release_worker(&worker_uuid).await; + }); + } else { + *worker.running.write().await = false; + return Err("Failed to build http client".to_string()); + } + } else { + download_queue.release_worker(&worker_uuid).await; + } + Ok(()) +} + +pub fn spawn_recording_services( + ctx: &RecordingCtx, + cancel_token: &CancellationToken, +) { + let config = ctx.app_config.config.load(); + let Some(recording_cfg) = config.recording().cloned() else { + return; + }; + + start_recording_scheduler( + Arc::clone(&ctx.app_config), + recording_cfg, + &ctx.recordings, + ctx.events.clone(), + Arc::clone(&ctx.recording_capacity), + cancel_token.clone(), + Path::new(crate::recording::recording_worker::FFMPEG_BINARY).to_path_buf(), + ); +} + +pub async fn resume_recording_worker_if_needed( + ctx: &RecordingCtx, + recording_cfg: &RecordingConfig, +) -> Result<(), String> { + if ctx.recordings.queue.lock().await.is_empty() && ctx.recordings.active.read().await.is_empty() { + return Ok(()); + } + + ensure_recording_worker_running( + &ctx.app_config, + recording_cfg, + &ctx.recordings, + &ctx.events, + &ctx.recording_capacity, + Path::new(crate::recording::recording_worker::FFMPEG_BINARY), + ) + .await +} + +pub(super) fn start_recording_scheduler( + app_config: Arc, + recording_cfg: RecordingConfig, + recordings: &Arc, + event_manager: E, + capacity: Arc, + cancel_token: CancellationToken, + recording_binary: std::path::PathBuf, +) { + let capacity_notify = capacity.capacity_changed(); + let slot_waiters = Arc::clone(&recordings.slot_waiters); + let bridge_capacity = Arc::clone(&capacity); + let bridge_recording_cfg = recording_cfg.clone(); + let bridge_app_config = Arc::clone(&app_config); + let bridge_cancel_token = cancel_token.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + () = bridge_cancel_token.cancelled() => break, + () = capacity_notify.notified() => {}, + () = slot_waiters.registration_changed.notified() => {} + } + let mut capacities_by_input: HashMap, ProviderCapacities> = HashMap::new(); + let live_config = bridge_app_config.config.load(); + let waiter_cfg = live_config.recording().unwrap_or(&bridge_recording_cfg); + let mut waiters = slot_waiters.snapshots(); + waiters.sort_by_key(|waiter| waiter.priority); + for waiter in waiters { + let Some(input_name) = waiter.input_name.as_ref() else { + let _ = slot_waiters.signal_waiter(waiter.id); + continue; + }; + let capacities = match capacities_by_input.entry(Arc::clone(input_name)) { + std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(bridge_capacity.capacities_for_input(input_name).await) + } + }; + if !capacities_have_free_slot(capacities) + || background_download_should_wait(waiter.priority, capacities, waiter_cfg) + { + continue; + } + if slot_waiters.signal_waiter(waiter.id) { + // Reserve a wake-up against this snapshot so one signal wakes + // as many eligible recordings as there are free connections. + if let Some((_, used, _)) = + capacities.iter_mut().find(|(_, used, limit)| *limit == 0 || *used < *limit) + { + *used = used.saturating_add(1); + } + } + } + } + }); + + let scheduler_recordings = Arc::clone(recordings); + let scheduler_cancel_token = cancel_token; + tokio::spawn(async move { + let mut interval = time::interval(Duration::from_secs(1)); + interval.set_missed_tick_behavior(time::MissedTickBehavior::Skip); + loop { + tokio::select! { + () = scheduler_cancel_token.cancelled() => break, + _ = interval.tick() => {}, + () = scheduler_recordings.queue_changed.notified() => {} + } + let promoted = scheduler_recordings.promote_due_scheduled_now().await; + if promoted > 0 { + publish_recording_change(&event_manager); + } + // Commits and worker release wake the scheduler immediately. The + // interval also checks scheduled windows and retries failed starts. + if let Err(error) = ensure_recording_worker_running( + &app_config, + &recording_cfg, + &scheduler_recordings, + &event_manager, + &capacity, + &recording_binary, + ) + .await + { + error!("Could not start recording workers: {error}"); + } + } + }); +} diff --git a/backend/dvr/src/recording/recording_worker.rs b/backend/dvr/src/recording/recording_worker.rs index 2ac46d69f..6a5eba473 100644 --- a/backend/dvr/src/recording/recording_worker.rs +++ b/backend/dvr/src/recording/recording_worker.rs @@ -405,11 +405,13 @@ pub async fn recovery_decision_for(final_path: &Path, partial: &Path) -> Recover #[cfg(test)] mod tests { + #[cfg(unix)] + use super::spawn_retrying_busy_executable; use super::{ build_recording_args, classify_ffmpeg_failure, recording_partial_path, recording_resume_or_retry_is_unsupported, recording_start_missed_window, recovery_decision_for, - redact_url_tokens, remaining_recording_duration_secs, run_recording_with_binary, - spawn_retrying_busy_executable, strip_url_tokens, RecordingExecutionResult, RecoveryDecision, + redact_url_tokens, remaining_recording_duration_secs, run_recording_with_binary, strip_url_tokens, + RecordingExecutionResult, RecoveryDecision, }; use crate::{ recording::recording_queue::{RecordingControl, RecordingTask, RecordingTaskState}, diff --git a/backend/hls/src/availability/tests.rs b/backend/hls/src/availability/tests.rs index fdee11339..6bd1385ee 100644 --- a/backend/hls/src/availability/tests.rs +++ b/backend/hls/src/availability/tests.rs @@ -89,45 +89,22 @@ use tuliprox_parser::hls::origin_manifest::{ const TERMINAL_ASSET_BYTES: &[u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); + const LOW_PRIORITY_ASSET_BYTES: &[u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/low_priority_preempted.ts")); + const PROVIDER_EXHAUSTED_ASSET_BYTES: &[u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/provider_connections_exhausted.ts")); + const USER_EXHAUSTED_ASSET_BYTES: &[u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/user_connections_exhausted.ts")); + const ACCOUNT_EXPIRED_ASSET_BYTES: &[u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/user_account_expired.ts")); + const SESSION_EXPIRED_ASSET_BYTES: &[u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/hls_session_or_lease_expired.ts")); -#[test] -fn availability_refresh_waits_for_completion_instead_of_polling_in_flight() { - for outcome in [HlsOriginRefreshTriggerOutcome::Started, HlsOriginRefreshTriggerOutcome::InFlight] { - assert_eq!( - availability_refresh_trigger_decision(outcome, true), - HlsAvailabilityRefreshTriggerDecision::Wait(HlsAvailabilityAttemptSchedule::RefreshCompletion) - ); - } - assert_eq!(HlsAvailabilityAttemptSchedule::RefreshCompletion.wake_at_ms(100, 1, 2_100), Some(2_100)); -} - -#[test] -fn availability_refresh_waits_until_concrete_debounce_boundary() { - let schedule = HlsAvailabilityAttemptSchedule::DebouncedUntil { retry_at_ms: 1_100 }; - assert_eq!( - availability_refresh_trigger_decision( - HlsOriginRefreshTriggerOutcome::DebouncedUntil { retry_at_ms: 1_100 }, - true, - ), - HlsAvailabilityRefreshTriggerDecision::Wait(schedule) - ); - assert_eq!(schedule.wake_at_ms(100, 1, 2_100), Some(1_100)); - assert_eq!( - HlsAvailabilityAttemptSchedule::DebouncedUntil { retry_at_ms: 3_000 }.wake_at_ms(100, 1, 2_100), - Some(2_100) - ); -} - fn runtime_custom_responses() -> Arc { runtime_custom_responses_with_low_priority(LOW_PRIORITY_ASSET_BYTES) } @@ -145,15 +122,6 @@ fn runtime_custom_responses_with_low_priority(low_priority_bytes: &[u8]) -> Arc< }) } -#[tokio::test] -async fn disabled_custom_responses_do_not_seed_terminal_media_preparation() { - let hls_ctx = crate::HlsCtx::for_test(Config { custom_stream_response_enabled: false, ..Config::default() }); - let ctx = &hls_ctx; - ctx.app_config.custom_stream_response.store(Some(runtime_custom_responses())); - - assert_eq!(terminal_media_timing_seed(ctx, 10_000), (None, HlsTerminalMediaPreparationState::Failed { key: None })); -} - fn publication_late_decision(reserve_ms: u64) -> HlsOriginProgressDecision { evaluate_origin_progress(HlsOriginProgressSnapshot { phase: HlsOriginProgressPhase::Fresh, @@ -177,65 +145,6 @@ fn lease_timing_seed() -> HlsAcceptanceEpisodeTimingSeed { } } -#[test] -fn hls_terminal_commit_outcomes_never_turn_failures_into_live_serving() { - for (outcome, reason) in [ - (HlsTerminalCommitOutcome::BundleNotReady, HlsTerminalFailedClosedReason::BundleNotReadyWithoutOwner), - (HlsTerminalCommitOutcome::BundleIncompatible, HlsTerminalFailedClosedReason::BundleIncompatible), - (HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed, HlsTerminalFailedClosedReason::SafeCommitDeadlineElapsed), - (HlsTerminalCommitOutcome::RetryCapacityExceeded, HlsTerminalFailedClosedReason::RetryCapacityExceeded), - (HlsTerminalCommitOutcome::RetryAttemptsExhausted, HlsTerminalFailedClosedReason::RetryAttemptsExhausted), - (HlsTerminalCommitOutcome::RetryWorkerUnavailable, HlsTerminalFailedClosedReason::RuntimeUnavailable), - ] { - assert_eq!( - terminal_resolution_for_commit_outcome(outcome, 1_000), - HlsTerminalResolution::FailedClosed { reason } - ); - } - assert_eq!( - terminal_resolution_for_commit_outcome(HlsTerminalCommitOutcome::LockBusy { retry_before_ms: 1_025 }, 1_000,), - HlsTerminalResolution::Pending { retry_after_ms: 25 } - ); -} - -#[test] -fn autonomous_terminal_live_allowed_means_no_cutover_is_required() { - assert_eq!( - classify_autonomous_terminal_resolution(HlsTerminalResolution::LiveAllowed), - HlsAutonomousTerminalObservation::NoCutoverRequired, - ); -} - -#[test] -fn hls_terminal_commit_pending_fallback_preserves_a_retryable_fail_closed_handoff() { - let safe_deadline_ms = 10_000; - let handoff_budget_ms = HlsTerminalCommitAcquisitionBudgetMs::fail_closed_handoff_from_retry_policy().as_millis(); - - assert_eq!( - terminal_pending_fallback_commit_at_ms(safe_deadline_ms), - safe_deadline_ms.saturating_sub(handoff_budget_ms) - ); - assert_eq!( - safe_deadline_ms.saturating_sub(terminal_pending_fallback_commit_at_ms(safe_deadline_ms)), - handoff_budget_ms - ); - assert_eq!(terminal_pending_fallback_commit_at_ms(handoff_budget_ms.saturating_sub(1)), 0); -} - -#[test] -fn live_reserve_wake_is_strictly_before_safe_terminal_deadline() { - let now_ms = 1_000; - let mut reserve = terminal_pending_commit_reserve(); - reserve.guaranteed_reserve_ms = reserve.guaranteed_reserve_ms.saturating_add(5_000); - reserve.guaranteed_media_horizon_ms = reserve.guaranteed_reserve_ms; - let cutover_timing = - HlsLeaseCutoverTiming::from_reserve(now_ms, reserve.guaranteed_reserve_ms, reserve.transition_margin, None); - - let deadline = live_reserve_deadline(now_ms, reserve, cutover_timing).expect("future acquisition wake"); - assert_eq!(deadline.next_reevaluation_at_ms, now_ms.saturating_add(5_000)); - assert!(deadline.next_reevaluation_at_ms < deadline.latest_safe_terminal_commit_at_ms); -} - fn pending_decision_bundle_key() -> HlsPreparedTerminalBundleKey { HlsPreparedTerminalBundleKey { asset: HlsTerminalAssetIdentity { revision: 7, fingerprint: [7; 32] }, @@ -288,767 +197,6 @@ struct HlsTerminalPendingCommitFixture { now_ms: u64, } -impl HlsTerminalPendingCommitFixture { - fn owner_key(&self) -> HlsTerminalPendingOwnerKey { - HlsTerminalPendingOwnerKey { - session_incarnation: self - .ctx - .hls_proxy - .sessions() - .session_incarnation(&self.session) - .expect("fixture session has a current incarnation"), - proxy_session_id: self.proxy_session_id.clone(), - lease_id: self.lease_id.clone(), - lease_issued_at_ms: self.preparation.lease_issued_at_ms, - expected_admission_generation: self.preparation.expected_admission_generation, - manifest_snapshot_generation: self.preparation.manifest_snapshot_generation, - cursor_generation: self.preparation.cursor_generation, - decision_generation: self.preparation.decision_generation, - reason: self.expected_asset.reason, - bundle_key: self.bundle_key, - latest_safe_commit_at_ms: self - .preparation - .cutover_timing - .latest_safe_terminal_commit_at - .as_millis_since_epoch(), - } - } - - fn ready_bundle(&self) -> Arc { - build_prepared_terminal_bundle(&self.asset, self.bundle_key).expect("fixture relative terminal bundle") - } - - fn register_owner(&self, ticket: HlsPreparedTerminalBundleCompletionTicket) -> oneshot::Receiver<()> { - let coordinator = self.ctx.hls_proxy.terminal_pending(); - let owner_key = self.owner_key(); - let ctx = self.ctx.clone(); - let session = Arc::clone(&self.session); - let proxy_session_id = self.proxy_session_id.clone(); - let lease_id = self.lease_id.clone(); - let preparation = self.preparation.clone(); - let asset = Arc::clone(&self.asset); - let expected_asset = self.expected_asset; - let bundle_key = self.bundle_key; - let (completed_tx, completed_rx) = oneshot::channel(); - let asset_guard = terminal_asset_revision_guard(&self.ctx, expected_asset.reason, Some(expected_asset)); - - assert_eq!( - coordinator.register(owner_key, &asset_guard, move |ownership| async move { - run_terminal_pending_owner( - ctx, - session, - proxy_session_id, - lease_id, - preparation, - asset, - expected_asset, - bundle_key, - ticket, - ownership, - ) - .await; - assert!(completed_tx.send(()).is_ok()); - }), - HlsTerminalPendingRegistration::Scheduled - ); - completed_rx - } -} - -fn terminal_pending_commit_reserve() -> HlsLeaseReserveSnapshot { - let transition_margin = HlsTransitionMarginMs::from_millis(12_000); - let guaranteed_reserve_ms = transition_margin - .as_millis() - .saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()); - HlsLeaseReserveSnapshot { - availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, - guaranteed_media_horizon_ms: guaranteed_reserve_ms, - conservative_playback_position_ms: 0, - guaranteed_reserve_ms, - initial_hidden_ready_duration_ms: 0, - transition_margin, - key_readiness_valid_until_ms: None, - recovery_required: true, - cutover_required: false, - } -} - -fn terminal_pending_commit_manifest( - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, - duration_ms: u64, -) -> HlsLeaseManifestSnapshot { - HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(1), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 0, - delivered_at_ms: 0, - first_proxy_seq: 40, - last_proxy_seq: 40, - visible_segments: Arc::from([HlsLeaseManifestSegment { - proxy_seq: 40, - duration_ms, - uri: format!("/hls/shared/live/{}/{}/40.ts", proxy_session_id.0, lease_id.0).into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }]), - discontinuity_sequence: 0, - target_duration_ms: 12_000, - playlist_duration_ms: duration_ms, - last_visible_media_end_ms: duration_ms, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - } -} - -async fn terminal_pending_commit_fixture(name: &str) -> HlsTerminalPendingCommitFixture { - terminal_pending_commit_fixture_with_base(name, TERMINAL_ASSET_BYTES).await -} - -async fn install_terminal_pending_ready_base( - ctx: &HlsCtx, - session: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - asset: &super::super::terminal_tail::HlsTerminalMediaAsset, - base_segment_bytes: &[u8], - now_ms: u64, -) { - let cache_key = SegmentCacheKey::new(proxy_session_id.clone(), 40, "ts"); - ctx.hls_proxy - .segment_cache() - .write_bytes_and_commit(&cache_key, base_segment_bytes) - .await - .expect("READY terminal-base bytes commit"); - let mut session = session.write().await; - let origin_epoch = session.origin_control.origin_epoch; - session.segments.insert( - 40, - SegmentEntry { - origin_key: OriginSegmentKey { - origin_epoch, - effective_host_id: 1, - host_local_sequence: 40, - host_local_index: 0, - }, - proxy_seq: 40, - duration_ms: asset.duration_ms(), - proxy_file_ext: "ts".to_string(), - content_type: "video/mp2t".to_string(), - cache_key, - discontinuity_before: false, - program_date_time: None, - daterange_tags_before: Vec::new(), - origin_byte_range: None, - map_ref: None, - encryption: None, - origin_fetch_ref: None, - status: SegmentCacheStatus::Ready { - content_length: u64::try_from(base_segment_bytes.len()).unwrap_or(u64::MAX), - ready_at_ms: now_ms, - }, - last_rendered_at_ms: Some(now_ms), - access: Arc::new(CacheAccessState::new()), - }, - ); -} - -async fn publish_terminal_pending_lease( - ctx: &HlsCtx, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, - name: &str, - duration_ms: u64, - now_ms: u64, -) { - ctx.hls_proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("pending-owner", name), - proxy_session_id.clone(), - "pending-owner".to_string(), - name.to_string(), - 1, - name.to_string(), - 1, - now_ms, - 60_000, - )) - .await; - let publication = ctx - .hls_proxy - .prepare_access_lease_manifest_publication(lease_id, proxy_session_id, now_ms) - .await - .expect("manifest publication guard"); - assert!(ctx - .hls_proxy - .commit_access_lease_manifest_publication( - lease_id, - proxy_session_id, - publication, - terminal_pending_commit_manifest(proxy_session_id, lease_id, duration_ms), - now_ms, - ) - .await - .is_committed()); -} - -async fn terminal_pending_commit_fixture_with_base( - name: &str, - base_segment_bytes: &[u8], -) -> HlsTerminalPendingCommitFixture { - let hls_ctx = crate::HlsCtx::for_test(Config { custom_stream_response_enabled: true, ..Config::default() }); - let ctx = &hls_ctx; - let terminal_buffer = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); - let asset = snapshot_terminal_media_asset(&terminal_buffer).expect("valid terminal asset fixture"); - ctx.app_config.custom_stream_response.store(Some(runtime_custom_responses())); - let now_ms = ctx.hls_proxy.terminal_commit_now_ms(); - let (session, _) = - ctx.hls_proxy.get_or_create_session_with_outcome(HlsSessionKey::new(1, name), b"secret", now_ms).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId(name.to_string()); - install_terminal_pending_ready_base(ctx, &session, &proxy_session_id, &asset, base_segment_bytes, now_ms).await; - publish_terminal_pending_lease(ctx, &proxy_session_id, &lease_id, name, asset.duration_ms(), now_ms).await; - let (origin_progress_generation, media_readiness_generation, last_media_progress_at_ms) = { - let session = session.read().await; - ( - session.origin_control.progress_generation, - session.activity.media_readiness_generation, - session.origin_control.last_media_progress_at_ms, - ) - }; - let reserve = terminal_pending_commit_reserve(); - let cutover_timing = - HlsLeaseCutoverTiming::from_reserve(now_ms, reserve.guaranteed_reserve_ms, reserve.transition_margin, None); - let preparation = ctx - .hls_proxy - .prepare_access_lease_terminal_tail(HlsTerminalTailPreparationRequest { - lease_id: &lease_id, - proxy_session_id: &proxy_session_id, - manifest_snapshot_generation: 1, - cursor_generation: 0, - reserve, - cutover_timing, - commit_window: HlsTerminalCommitWindow::AcquisitionOpen, - now_ms, - origin_progress_generation, - media_readiness_generation, - last_media_progress_at_ms, - }) - .await - .expect("cutover-local terminal preparation"); - let expected_asset = - HlsRuntimeCustomTailAssetIdentity::channel_unavailable(HlsTerminalAssetIdentity::from_asset(&asset)); - let bundle_key = prepared_terminal_bundle_key( - &asset, - preparation.manifest_snapshot.target_duration_ms, - HLS_TERMINAL_TAIL_SEGMENT_COUNT, - ); - - HlsTerminalPendingCommitFixture { - ctx: ctx.clone(), - session, - proxy_session_id, - lease_id, - preparation, - asset, - expected_asset, - bundle_key, - now_ms, - } -} - -#[tokio::test] -async fn hls_terminal_commit_pending_ready_completion_selects_tail_without_a_client_retry() { - let coordinator = Arc::new(HlsTerminalPendingCoordinator::default()); - let bundle_key = pending_decision_bundle_key(); - let bundle = pending_decision_ready_bundle(bundle_key); - let (ticket, publisher) = prepared_terminal_bundle_completion_channel_for_test(bundle_key); - let (_fallback_tx, fallback_rx) = oneshot::channel::<()>(); - let (decision_tx, decision_rx) = oneshot::channel(); - let asset_guard = HlsTerminalAssetRevisionGuard::matching_for_test(Some(bundle_key.asset)); - - assert_eq!( - coordinator.register(pending_decision_owner_key(bundle_key), &asset_guard, move |ownership| async move { - let decision = await_terminal_pending_decision(ticket, &ownership, bundle_key, async move { - assert!(fallback_rx.await.is_ok()); - }) - .await; - assert!(decision_tx.send(decision).is_ok()); - },), - HlsTerminalPendingRegistration::Scheduled - ); - publisher.publish(HlsPreparedTerminalBundleCompletion::Ready { bundle: Arc::clone(&bundle) }); - - let decision = decision_rx.await.expect("pending decision completes"); - assert!(matches!(decision, Some(HlsTerminalPendingDecision::Ready(actual)) if Arc::ptr_eq(&actual, &bundle))); - assert_eq!(coordinator.owner_count(), 0); -} - -#[tokio::test] -async fn hls_terminal_commit_pending_fallback_selects_terminal_unavailable_without_a_client_retry() { - let coordinator = Arc::new(HlsTerminalPendingCoordinator::default()); - let bundle_key = pending_decision_bundle_key(); - let (ticket, _publisher) = prepared_terminal_bundle_completion_channel_for_test(bundle_key); - let (fallback_tx, fallback_rx) = oneshot::channel::<()>(); - let (decision_tx, decision_rx) = oneshot::channel(); - let asset_guard = HlsTerminalAssetRevisionGuard::matching_for_test(Some(bundle_key.asset)); - - assert_eq!( - coordinator.register(pending_decision_owner_key(bundle_key), &asset_guard, move |ownership| async move { - let decision = await_terminal_pending_decision(ticket, &ownership, bundle_key, async move { - assert!(fallback_rx.await.is_ok()); - }) - .await; - assert!(decision_tx.send(decision).is_ok()); - },), - HlsTerminalPendingRegistration::Scheduled - ); - assert!(fallback_tx.send(()).is_ok()); - - assert!(matches!( - decision_rx.await, - Ok(Some(HlsTerminalPendingDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady))) - )); - assert_eq!(coordinator.owner_count(), 0); -} - -#[tokio::test] -async fn hls_terminal_commit_pending_owner_store_ready_completion_commits_terminal_tail() { - let fixture = terminal_pending_commit_fixture("pending-owner-ready-store").await; - let bundle = fixture.ready_bundle(); - let (ticket, publisher) = prepared_terminal_bundle_completion_channel_for_test(fixture.bundle_key); - let completed = fixture.register_owner(ticket); - - publisher.publish(HlsPreparedTerminalBundleCompletion::Ready { bundle }); - completed.await.expect("productive pending owner completes"); - - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("terminal lease remains stored"); - assert!(matches!( - lease.playback_mode, - HlsLeasePlaybackMode::TerminalTail(ref plan) - if plan.generation.0 == fixture.preparation.decision_generation - )); - assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 0); -} - -fn terminal_base_without_timestamps() -> Vec { - let mut bytes = TERMINAL_ASSET_BYTES.to_vec(); - for packet in bytes.as_chunks_mut::<188>().0 { - let adaptation_field_control = (packet[3] >> 4) & 0b11; - if matches!(adaptation_field_control, 0b10 | 0b11) && packet[4] > 0 { - packet[5] &= !0x10; - } - if packet[1] & 0x40 == 0 { - continue; - } - let payload_offset = match adaptation_field_control { - 0b01 => 4, - 0b11 => 5usize.saturating_add(usize::from(packet[4])), - _ => continue, - }; - let Some(payload) = packet.get_mut(payload_offset..) else { - continue; - }; - if payload.len() >= 9 && payload.starts_with(&[0x00, 0x00, 0x01]) { - payload[7] &= 0x3F; - } - } - bytes -} - -#[tokio::test] -async fn missing_terminal_base_timestamp_profile_commits_terminal_unavailable() { - let base_bytes = terminal_base_without_timestamps(); - let fixture = terminal_pending_commit_fixture_with_base("pending-owner-missing-timestamp", &base_bytes).await; - let bundle = fixture.ready_bundle(); - let (ticket, publisher) = prepared_terminal_bundle_completion_channel_for_test(fixture.bundle_key); - let completed = fixture.register_owner(ticket); - - publisher.publish(HlsPreparedTerminalBundleCompletion::Ready { bundle }); - completed.await.expect("terminal owner completes fail-closed decision"); - - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("terminal unavailable lease remains stored"); - assert!(matches!( - lease.playback_mode, - HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::MissingTimestampAnchor, .. } - )); -} - -#[tokio::test(start_paused = true)] -async fn hls_terminal_commit_pending_owner_store_fallback_commits_terminal_unavailable() { - let fixture = terminal_pending_commit_fixture("pending-owner-fallback-store").await; - let (ticket, _publisher) = prepared_terminal_bundle_completion_channel_for_test(fixture.bundle_key); - let completed = fixture.register_owner(ticket); - - tokio::task::yield_now().await; - tokio::time::advance(Duration::from_millis(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis())) - .await; - completed.await.expect("productive fallback owner completes"); - - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("terminal lease remains stored"); - assert!(matches!( - lease.playback_mode, - HlsLeasePlaybackMode::TerminalUnavailable { - decision_generation, - reason: HlsTerminalTailCompatibility::TerminalMediaNotReady, - } if decision_generation == fixture.preparation.decision_generation - )); - assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 0); -} - -#[tokio::test] -async fn hls_terminal_commit_pending_owner_store_progress_supersession_keeps_lease_live() { - let fixture = terminal_pending_commit_fixture("pending-owner-progress-store").await; - let bundle = fixture.ready_bundle(); - let (ticket, publisher) = prepared_terminal_bundle_completion_channel_for_test(fixture.bundle_key); - let completed = fixture.register_owner(ticket); - tokio::task::yield_now().await; - - { - let mut session = fixture.session.write().await; - session.origin_control.progress_generation = session.origin_control.progress_generation.saturating_add(1); - session.origin_control.last_media_progress_at_ms = Some(fixture.now_ms.saturating_add(1)); - } - fixture.ctx.hls_proxy.cancel_superseded_terminal_work_for_session(&fixture.proxy_session_id); - publisher.publish(HlsPreparedTerminalBundleCompletion::Ready { bundle }); - completed.await.expect("cancelled pending owner completes"); - - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("superseded lease remains stored"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - assert!(!fixture.session.read().await.has_terminal_tail_protections()); - assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 0); -} - -async fn assert_terminal_pending_registration_failure_commits_unavailable( - failure: HlsTerminalPendingRegistration, - name: &str, -) { - let fixture = terminal_pending_commit_fixture(name).await; - let resolution = terminal_resolution_for_pending_registration( - HlsTerminalCommitContext { - ctx: &fixture.ctx, - session: &fixture.session, - proxy_session_id: &fixture.proxy_session_id, - lease_id: &fixture.lease_id, - preparation: &fixture.preparation, - now_ms: fixture.now_ms, - }, - fixture.expected_asset, - fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), - failure, - ); - - assert_eq!(resolution, HlsTerminalResolution::Committed); - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("terminal unavailable lease remains stored"); - assert!(matches!( - lease.playback_mode, - HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::TerminalMediaNotReady, .. } - )); -} - -#[tokio::test] -async fn terminal_pending_capacity_failure_commits_terminal_unavailable() { - assert_terminal_pending_registration_failure_commits_unavailable( - HlsTerminalPendingRegistration::CapacityExceeded, - "pending-capacity-failure", - ) - .await; -} - -#[tokio::test] -async fn terminal_pending_runtime_failure_commits_terminal_unavailable() { - assert_terminal_pending_registration_failure_commits_unavailable( - HlsTerminalPendingRegistration::RuntimeUnavailable, - "pending-runtime-failure", - ) - .await; -} - -async fn assert_post_refresh_registration_failure_leaves_no_unowned_live_lease( - failure: HlsAvailabilityReevaluationRegistration, - name: &str, -) { - let fixture = terminal_pending_commit_fixture(name).await; - let (origin_progress_generation, media_readiness_generation) = { - let mut session = fixture.session.write().await; - session.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict; - let origin_epoch = session.origin_control.origin_epoch; - for proxy_seq in 41_u64..=44 { - session.segments.insert( - proxy_seq, - SegmentEntry { - origin_key: OriginSegmentKey { - origin_epoch, - effective_host_id: 1, - host_local_sequence: proxy_seq, - host_local_index: u32::try_from(proxy_seq.saturating_sub(40)).unwrap_or(u32::MAX), - }, - proxy_seq, - duration_ms: 20_000, - proxy_file_ext: "ts".to_string(), - content_type: "video/mp2t".to_string(), - cache_key: SegmentCacheKey::new(fixture.proxy_session_id.clone(), proxy_seq, "ts"), - discontinuity_before: false, - program_date_time: None, - daterange_tags_before: Vec::new(), - origin_byte_range: None, - map_ref: None, - encryption: None, - origin_fetch_ref: None, - status: SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: fixture.now_ms }, - last_rendered_at_ms: None, - access: Arc::new(CacheAccessState::new()), - }, - ); - } - (session.origin_control.progress_generation, session.activity.media_readiness_generation) - }; - assert!(fixture - .ctx - .hls_proxy - .activate_access_lease( - &fixture.lease_id, - &fixture.proxy_session_id, - fixture.now_ms, - HlsAccessLeaseTiming { active_window_ms: 60_000, valid_window_ms: 60_000 }, - ) - .await - .is_activated()); - let outcome = commit_post_refresh_terminal_fallback( - fixture.ctx.clone(), - Arc::clone(&fixture.session), - HlsPostRefreshAvailabilityAction::Reevaluate { - reason: super::super::refresh::HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, - origin_progress_generation, - media_readiness_generation, - }, - failure, - ) - .await; - - assert_eq!(outcome, HlsPostRefreshFallbackOutcome::TerminalCommitted); - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("fallback lease remains stored"); - assert!(matches!( - lease.playback_mode, - HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::TerminalMediaNotReady, .. } - )); -} - -#[tokio::test] -async fn post_refresh_coordinator_capacity_failure_leaves_no_unowned_live_lease() { - assert_post_refresh_registration_failure_leaves_no_unowned_live_lease( - HlsAvailabilityReevaluationRegistration::CapacityExceeded, - "post-refresh-capacity-failure", - ) - .await; -} - -#[tokio::test] -async fn post_refresh_runtime_failure_leaves_no_unowned_live_lease() { - assert_post_refresh_registration_failure_leaves_no_unowned_live_lease( - HlsAvailabilityReevaluationRegistration::RuntimeUnavailable, - "post-refresh-runtime-failure", - ) - .await; -} - -fn pressure_manifest(target_duration_ms: u64) -> HlsLeaseManifestSnapshot { - HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(1), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 1, - delivered_at_ms: 1, - first_proxy_seq: 0, - last_proxy_seq: 0, - visible_segments: Arc::from([HlsLeaseManifestSegment { - proxy_seq: 0, - duration_ms: 4_000, - uri: "0.ts".into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }]), - discontinuity_sequence: 0, - target_duration_ms, - playlist_duration_ms: 4_000, - last_visible_media_end_ms: 4_000, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - } -} - -fn pressure_manifest_at(proxy_seq: u64, target_duration_ms: u64) -> HlsLeaseManifestSnapshot { - let mut manifest = pressure_manifest(target_duration_ms); - manifest.first_proxy_seq = proxy_seq; - manifest.last_proxy_seq = proxy_seq; - Arc::make_mut(&mut manifest.visible_segments)[0].proxy_seq = proxy_seq; - manifest -} - -fn pressure_timeline(hidden_durations_ms: &[u64]) -> HlsReadyTimelineSnapshot { - let mut start_ms = 0_u64; - let mut units = Vec::with_capacity(hidden_durations_ms.len().saturating_add(1)); - for (index, duration_ms) in std::iter::once(4_000_u64).chain(hidden_durations_ms.iter().copied()).enumerate() { - units.push(HlsReadyTimelineUnit { - proxy_seq: u64::try_from(index).unwrap_or(u64::MAX), - start_ms, - duration_ms, - state: HlsReadyMediaState::Ready, - required_map_ready: true, - required_key_ready: true, - key_ready_valid_until_ms: None, - }); - start_ms = start_ms.saturating_add(duration_ms); - } - HlsReadyTimelineSnapshot { units: units.into() } -} - -fn evaluated_pressure( - lease_id: &str, - target_duration_ms: u64, - hidden_durations_ms: &[u64], - recovery_budget_ms: u64, -) -> HlsLeaseRecoveryEvidence { - let manifest = pressure_manifest(target_duration_ms); - let ready_timeline = pressure_timeline(hidden_durations_ms); - let recovery_trigger_budget = HlsRecoveryTriggerBudgetMs::from_millis(recovery_budget_ms); - let reserve = evaluate_lease_reserve(HlsLeaseReserveInput { - manifest: &manifest, - cursor: &HlsLeasePlaybackCursor::default(), - ready_timeline: &ready_timeline, - now_ms: 100, - playback_rate_guard_milli: HLS_PLAYBACK_RATE_GUARD_MILLI, - recovery_trigger_budget, - origin_path_degraded: true, - recovery_committed: false, - }); - let boundary_ms = recovery_trigger_budget.as_millis().saturating_add(reserve.transition_margin.as_millis()); - let cutover_timing = - HlsLeaseCutoverTiming::from_reserve(100, reserve.guaranteed_reserve_ms, reserve.transition_margin, None); - HlsLeaseRecoveryEvidence { - lease_id: HlsAccessLeaseId(lease_id.to_string()), - reserve, - cursor: HlsLeasePlaybackCursor::default(), - workload: HlsRecoveryWorkloadEnvelope::acceptance_policy().ceiling(), - target_duration_ms, - latest_safe_terminal_commit_at: cutover_timing.latest_safe_terminal_commit_at, - recovery_boundary_slack_ms: HlsRecoveryBoundarySlackMs::from_reserve_and_boundary( - reserve.guaranteed_reserve_ms, - boundary_ms, - ), - } -} - -fn atomic_pressure_policy() -> HlsRecoveryPressurePolicy { - HlsRecoveryPressurePolicy { - burst_plan: shared::model::HlsManifestRecoveryBurstPlan { slots: 1, lanes_per_slot: 1 }, - timing: HlsRecoveryTimingPolicy::new( - HlsOperationTimeoutMs::from_millis(1_000), - HlsOperationTimeoutMs::from_millis(1_000), - HlsRecoveryEtaMs::from_millis(0), - HlsRecoveryEtaMs::from_millis(0), - ), - } -} - -fn atomic_pressure_session() -> HlsSession { - let mut session = HlsSession::new(HlsSessionKey::new(1, "pressure"), b"secret", 0); - let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-TARGETDURATION:8\n\ - #EXTINF:4.0,\n0.ts\n#EXTINF:8.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n"; - let OriginManifestParseOutcome::Normal(manifest) = - parse_origin_media_manifest(body, "http://origin.example/live/index.m3u8") - else { - panic!("pressure manifest parses"); - }; - session.apply_origin_manifest(&manifest).expect("pressure timeline applies"); - for segment in session.segments.values_mut() { - segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1 }; - } - session.origin_control.path_condition = HlsOriginPathCondition::RetryableFetchFailure; - session.origin_control.last_media_progress_at_ms = Some(90); - session.origin_control.target_duration_snapshot_ms = Some(8_000); - session -} - -#[tokio::test] -async fn capacity_deferred_ready_boundary_keeps_affected_lease_live() { - let hls_ctx = crate::HlsCtx::for_test(Config::default()); - let ctx = &hls_ctx; - let now_ms = ctx.hls_proxy.terminal_commit_now_ms(); - let mut session = atomic_pressure_session(); - session.segments.get_mut(&1).expect("second segment").status = - SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Prefetch, deferred_at_ms: 100 }; - let proxy_session_id = session.proxy_session_id.clone(); - let session = Arc::new(tokio::sync::RwLock::new(session)); - let lease_id = HlsAccessLeaseId("capacity-deferred-live".to_string()); - let mut lease = HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("capacity-user", "capacity-client"), - proxy_session_id.clone(), - "capacity-user".to_string(), - "capacity-session".to_string(), - 1, - "capacity-stream".to_string(), - 1, - now_ms, - 60_000, - ); - lease.state = HlsAccessLeaseState::Activated; - lease.active_until_ms = Some(now_ms.saturating_add(60_000)); - lease.pending_deadline = None; - lease.last_manifest_snapshot = Some(pressure_manifest_at(0, 8_000)); - ctx.hls_proxy.prepare_access_lease(lease.clone()).await; - - let resolution = - commit_terminal_tail_if_lease_reserve_requires_cutover(ctx, &session, &proxy_session_id, &lease, now_ms).await; - - assert_eq!(resolution, HlsTerminalResolution::LiveAllowed); - let current = ctx - .hls_proxy - .access_lease_response_snapshot(&lease_id, &proxy_session_id, now_ms) - .await - .expect("capacity-deferred lease remains available"); - assert_eq!(current.state, HlsAccessLeaseState::Activated); - assert_eq!(current.playback_mode, HlsLeasePlaybackMode::Live); -} - struct PostRefreshTerminalFixture { ctx: HlsCtx, session: HlsSessionHandle, @@ -1057,1607 +205,42 @@ struct PostRefreshTerminalFixture { now_ms: u64, } -fn post_refresh_owner_request(fixture: &PostRefreshTerminalFixture) -> OriginRefreshRequest { - let origin_entry = - super::super::manifest_fetch::LiveHlsOriginEntry::parse("http://127.0.0.1:9/live/user/pass/12345.m3u8") - .expect("test origin entry parses"); - OriginRefreshRequest { - app_config: Arc::clone(&fixture.ctx.app_config), - session: Arc::clone(&fixture.session), - origin_entry, - headers: axum::http::HeaderMap::new(), - origin_provider_session_headers: axum::http::HeaderMap::new(), - disabled_headers: None, - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("test client builds"), - use_manual_redirects: false, - segment_cache: Arc::clone(fixture.ctx.hls_proxy.segment_cache()), - hls_proxy: Arc::clone(&fixture.ctx.hls_proxy), - segment_repair: Arc::clone(fixture.ctx.hls_proxy.segment_repair()), - segment_worker_pool: Arc::clone(fixture.ctx.hls_proxy.segment_worker_pool()), - map_worker_pool: Arc::clone(fixture.ctx.hls_proxy.map_worker_pool()), - origin_manifest_timeout_ms: fixture.ctx.hls_proxy.origin_manifest_timeout_ms(), - manifest_recovery_burst: fixture.ctx.hls_proxy.manifest_recovery_burst(), - strip: fixture.ctx.hls_proxy.strip(), - retry_policy: super::super::manifest_fetch::RetryPolicy { delays_ms: [0; 5], jitter_max_ms: 0 }, - reverse_proxy_rewrite_secret: b"secret".to_vec(), - transient_resource_ttl_ms: 300_000, - manifest_commit_requirement: super::super::refresh::HlsManifestCommitRequirement::CommittedManifestAllowed, - fresh_manifest_requirement_generation: None, - acceptance_directive: HlsManifestAcceptanceDirective::none(), - access_lease_id: None, - now_ms: fixture.now_ms, - origin_io: None, - post_refresh_runtime: Some(super::super::refresh::HlsPostRefreshRuntime { ctx: fixture.ctx.downgrade() }), - } -} - -async fn register_real_post_refresh_owner( - fixture: &PostRefreshTerminalFixture, - reason: super::super::refresh::HlsPostRefreshAvailabilityReason, -) { - let (origin_progress_generation, media_readiness_generation) = { - let session = fixture.session.read().await; - (session.origin_control.progress_generation, session.activity.media_readiness_generation) - }; - assert_eq!( - register_post_refresh_availability_reevaluation( - fixture.ctx.clone(), - Arc::clone(&fixture.session), - post_refresh_owner_request(fixture), - HlsPostRefreshAvailabilityAction::Reevaluate { - reason, - origin_progress_generation, - media_readiness_generation, - }, - ) - .await, - HlsAvailabilityReevaluationRegistration::Scheduled - ); -} - -fn assert_availability_owner_registered(fixture: &PostRefreshTerminalFixture) { - assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1); -} - -/// How long the owner gets to finish before the wait gives up. -/// -/// These tests run on a paused clock, so this is logical time and costs -/// nothing when the owner completes normally. -const AVAILABILITY_OWNER_COMPLETION_TIMEOUT: Duration = Duration::from_secs(30); - -/// Number of observer revisions to accept before concluding the owner is -/// cycling without converging. -const AVAILABILITY_OWNER_MAX_REVISIONS: usize = 1_024; - -async fn wait_for_availability_owner_completion(fixture: &PostRefreshTerminalFixture) { - let coordinator = fixture.ctx.hls_proxy.availability_reevaluations(); - // Bounded twice over, because the two ways this can fail to converge - // need different guards. A parked owner never wakes the observer, and - // on a paused clock the runtime is idle, so the timeout fires. An owner - // that keeps re-arming a successor wakes the observer forever and keeps - // the runtime busy, so the clock never advances and only the revision - // count catches it. Either way the test fails with a diagnostic instead - // of hanging the suite. - let wait = async { - let mut revisions = 0usize; - while let Some(mut observer) = coordinator.observe_owner(&fixture.proxy_session_id) { - if matches!( - observer.changed().await, - crate::availability_reevaluation::HlsAvailabilityReevaluationObservation::OwnerFinished - ) { - // The owner is gone; re-check the map and leave the loop. - continue; - } - revisions += 1; - assert!( - revisions <= AVAILABILITY_OWNER_MAX_REVISIONS, - "availability owner for {:?} produced {revisions} revisions without completing", - fixture.proxy_session_id - ); - } - }; - assert!( - tokio::time::timeout(AVAILABILITY_OWNER_COMPLETION_TIMEOUT, wait).await.is_ok(), - "availability owner for {:?} did not complete within {AVAILABILITY_OWNER_COMPLETION_TIMEOUT:?}; \ - {} owner(s) still registered", - fixture.proxy_session_id, - coordinator.owner_count() - ); - assert_eq!(coordinator.owner_count(), 0); -} - -async fn assert_post_refresh_owner_checks_refresh_gate_once(fixture: &PostRefreshTerminalFixture) { - let refresh_skipped_before = fixture.ctx.hls_proxy.metrics().snapshot().refresh_skipped; - register_real_post_refresh_owner( - fixture, - super::super::refresh::HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, - ) - .await; - for _ in 0..256 { - if fixture.ctx.hls_proxy.metrics().snapshot().refresh_skipped == refresh_skipped_before.saturating_add(1) { - break; - } - tokio::task::yield_now().await; - } - assert_eq!( - fixture.ctx.hls_proxy.metrics().snapshot().refresh_skipped, - refresh_skipped_before.saturating_add(1), - "the owner must observe the unavailable refresh gate once" - ); - - for _ in 0..10 { - tokio::time::advance(Duration::from_millis(50)).await; - tokio::task::yield_now().await; - } - assert_eq!( - fixture.ctx.hls_proxy.metrics().snapshot().refresh_skipped, - refresh_skipped_before.saturating_add(1), - "unchanged refresh evidence must not produce repeated gate attempts" - ); - - fixture.ctx.hls_proxy.availability_reevaluations().cancel_session(&fixture.proxy_session_id); - wait_for_availability_owner_completion(fixture).await; -} - -async fn post_refresh_terminal_fixture(name: &str, terminal_asset: bool) -> PostRefreshTerminalFixture { - post_refresh_terminal_fixture_with_progress(name, terminal_asset, true).await -} - -async fn post_refresh_terminal_fixture_with_progress( - name: &str, - terminal_asset: bool, - complete_playback: bool, -) -> PostRefreshTerminalFixture { - post_refresh_terminal_fixture_with_bundle_state(name, terminal_asset, complete_playback, true).await -} - -fn post_refresh_origin_manifest(terminal_asset: bool) -> (u64, ParsedOriginManifest) { - let (target_duration_ms, body) = if terminal_asset { - ( - 12_000, - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-TARGETDURATION:12\n\ - #EXTINF:12.0,\n0.ts\n#EXTINF:12.0,\n1.ts\n#EXTINF:12.0,\n2.ts\n", - ) - } else { - ( - 8_000, - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-TARGETDURATION:8\n\ - #EXTINF:4.0,\n0.ts\n#EXTINF:8.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n", - ) - }; - let OriginManifestParseOutcome::Normal(manifest) = - parse_origin_media_manifest(body, "http://origin.example/live/index.m3u8") - else { - panic!("post-refresh terminal fixture parses"); - }; - (target_duration_ms, manifest) -} - -async fn prepare_post_refresh_terminal_base( - ctx: &HlsCtx, - session: &HlsSessionHandle, - manifest: &ParsedOriginManifest, - terminal_asset: bool, - prepare_terminal_bundle: bool, - target_duration_ms: u64, - now_ms: u64, -) { - let terminal_base_cache_key = { - let mut session = session.write().await; - session.apply_origin_manifest(manifest).expect("fixture timeline applies"); - for segment in session.segments.values_mut() { - segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: now_ms }; - } - session.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict; - session.origin_control.last_media_progress_at_ms = Some(now_ms); - session.origin_control.target_duration_snapshot_ms = Some(target_duration_ms); - session.segments.get(&0).expect("fixture terminal-base segment").cache_key.clone() - }; - if !terminal_asset { - return; - } - ctx.hls_proxy - .segment_cache() - .write_bytes_and_commit(&terminal_base_cache_key, TERMINAL_ASSET_BYTES) - .await - .expect("terminal-compatible READY base bytes commit"); - session.write().await.segments.get_mut(&0).expect("fixture terminal-base segment").status = - SegmentCacheStatus::Ready { - content_length: u64::try_from(TERMINAL_ASSET_BYTES.len()).unwrap_or(u64::MAX), - ready_at_ms: now_ms, - }; - if prepare_terminal_bundle { - let asset = snapshot_terminal_media_asset(&TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec())) - .expect("fixture terminal asset parses"); - let key = prepared_terminal_bundle_key(&asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); - let state = ctx.hls_proxy.start_prepared_terminal_bundle( - Arc::clone(&asset), - target_duration_ms, - HLS_TERMINAL_TAIL_SEGMENT_COUNT, - ); - let state = match state { - HlsPreparedTerminalBundleState::Preparing { .. } => { - ctx.hls_proxy.wait_for_prepared_terminal_bundle(key).await - } - HlsPreparedTerminalBundleState::Ready { .. } - | HlsPreparedTerminalBundleState::Failed { .. } - | HlsPreparedTerminalBundleState::Incompatible { .. } => Some(state), - }; - assert!( - matches!(state, Some(HlsPreparedTerminalBundleState::Ready { .. })), - "terminal preparation must be ready: {state:?}" - ); - } -} - -async fn publish_post_refresh_terminal_lease( - ctx: &HlsCtx, - proxy_session_id: &ProxySessionId, - name: &str, - terminal_asset: bool, - target_duration_ms: u64, - now_ms: u64, -) -> HlsAccessLeaseId { - let lease_id = HlsAccessLeaseId(format!("{name}-lease")); - ctx.hls_proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new(name, name), - proxy_session_id.clone(), - name.to_string(), - "token".to_string(), - 1, - "stream".to_string(), - 1, - now_ms, - 60_000, - )) - .await; - let publication = ctx - .hls_proxy - .prepare_access_lease_manifest_publication(&lease_id, proxy_session_id, now_ms) - .await - .expect("terminal fixture publication guard"); - let mut manifest_snapshot = pressure_manifest(target_duration_ms); - if terminal_asset { - Arc::make_mut(&mut manifest_snapshot.visible_segments)[0].duration_ms = target_duration_ms; - manifest_snapshot.playlist_duration_ms = target_duration_ms; - manifest_snapshot.last_visible_media_end_ms = target_duration_ms; - } - Arc::make_mut(&mut manifest_snapshot.visible_segments)[0].uri = - format!("/hls/shared/live/{}/{}/0.ts", proxy_session_id.0, lease_id.0).into(); - assert!(ctx - .hls_proxy - .commit_access_lease_manifest_publication(&lease_id, proxy_session_id, publication, manifest_snapshot, now_ms,) - .await - .is_committed()); - assert!(ctx - .hls_proxy - .activate_access_lease( - &lease_id, - proxy_session_id, - now_ms, - HlsAccessLeaseTiming { active_window_ms: 60_000, valid_window_ms: 60_000 }, - ) - .await - .is_activated()); - assert!( - ctx.hls_proxy.access_lease_response_snapshot(&lease_id, proxy_session_id, now_ms).await.is_some(), - "activated terminal fixture lease remains available" - ); - lease_id -} - -async fn post_refresh_terminal_fixture_with_bundle_state( - name: &str, - terminal_asset: bool, - complete_playback: bool, - prepare_terminal_bundle: bool, -) -> PostRefreshTerminalFixture { - let hls_ctx = crate::HlsCtx::for_test(Config { custom_stream_response_enabled: true, ..Config::default() }); - let ctx = &hls_ctx; - if terminal_asset { - ctx.app_config.custom_stream_response.store(Some(runtime_custom_responses())); - } - let now_ms = ctx.hls_proxy.terminal_commit_now_ms(); - // These tests drive time with `tokio::time::advance`. Anchor the scheduling - // clock to tokio's so the owner's deadlines move with it; otherwise it - // sleeps towards a wall-clock deadline the paused clock never reaches. - ctx.hls_proxy.follow_tokio_clock_for_test(now_ms); - let (session, _) = - ctx.hls_proxy.get_or_create_session_with_outcome(HlsSessionKey::new(1, name), b"secret", now_ms).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let (target_duration_ms, manifest) = post_refresh_origin_manifest(terminal_asset); - prepare_post_refresh_terminal_base( - ctx, - &session, - &manifest, - terminal_asset, - prepare_terminal_bundle, - target_duration_ms, - now_ms, - ) - .await; - let lease_id = - publish_post_refresh_terminal_lease(ctx, &proxy_session_id, name, terminal_asset, target_duration_ms, now_ms) - .await; - if complete_playback { - advance_post_refresh_fixture_playback( - ctx, - &session, - &proxy_session_id, - &lease_id, - if terminal_asset { 22_000 } else { 7_000 }, - now_ms, - ) - .await; - } - PostRefreshTerminalFixture { ctx: ctx.clone(), session, proxy_session_id, lease_id, now_ms } -} - -async fn prepare_runtime_custom_bundle(fixture: &PostRefreshTerminalFixture, reason: HlsRuntimeCustomTailReason) { - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("runtime custom-tail fixture lease"); - let target_duration_ms = - lease.last_manifest_snapshot.as_ref().expect("published runtime custom-tail manifest").target_duration_ms; - let asset = - snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, reason).expect("configured runtime custom-tail asset"); - let key = prepared_terminal_bundle_key(&asset.asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); - let state = fixture.ctx.hls_proxy.start_prepared_terminal_bundle( - Arc::clone(&asset.asset), - target_duration_ms, - HLS_TERMINAL_TAIL_SEGMENT_COUNT, - ); - let state = match state { - HlsPreparedTerminalBundleState::Preparing { .. } => fixture - .ctx - .hls_proxy - .wait_for_prepared_terminal_bundle(key) - .await - .expect("runtime custom-tail bundle completion"), - state => state, - }; - assert!( - matches!(state, HlsPreparedTerminalBundleState::Ready { ref bundle } if bundle.matches_key_and_shape(key)), - "runtime custom-tail bundle must be READY: {state:?}" - ); -} - -async fn wait_for_runtime_custom_plan(fixture: &PostRefreshTerminalFixture) -> Arc { - let completed = tokio::time::timeout(Duration::from_secs(10), async { - loop { - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("runtime custom-tail lease remains stored"); - if let HlsLeasePlaybackMode::TerminalTail(plan) = lease.playback_mode { - return plan; - } - tokio::task::yield_now().await; - } - }) - .await; - if let Ok(plan) = completed { - return plan; - } - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await; - let state = lease.as_ref().map_or("missing", |lease| lease.state.as_log_value()); - let playback = lease.as_ref().map_or("missing", |lease| match lease.playback_mode { - HlsLeasePlaybackMode::Live => "live", - HlsLeasePlaybackMode::TerminalTail(_) => "terminal-tail", - HlsLeasePlaybackMode::TerminalUnavailable { .. } => "terminal-unavailable", - HlsLeasePlaybackMode::Ended => "ended", - }); - panic!( - "runtime custom-tail owner deadline: state={state} playback={playback} owners={}", - fixture.ctx.hls_proxy.terminal_pending().owner_count() - ); -} - -async fn commit_runtime_custom_reason( - fixture: &PostRefreshTerminalFixture, - reason: HlsRuntimeCustomTailReason, - prewarm: bool, -) -> (HlsRuntimeCustomTailOutcome, Arc) { - if prewarm { - prepare_runtime_custom_bundle(fixture, reason).await; - } - let outcome = commit_hls_runtime_custom_tail( - fixture.ctx.clone(), - HlsRuntimeCustomTailRequest { - session: Arc::clone(&fixture.session), - proxy_session_id: fixture.proxy_session_id.clone(), - lease_id: fixture.lease_id.clone(), - reason, - now_ms: fixture.now_ms, - }, - ) - .await; - assert!(matches!( - outcome, - HlsRuntimeCustomTailOutcome::Committed - | HlsRuntimeCustomTailOutcome::AlreadyCommitted - | HlsRuntimeCustomTailOutcome::PendingOwnerRegistered - )); - let plan = wait_for_runtime_custom_plan(fixture).await; - (outcome, plan) -} - -fn configured_runtime_custom_buffer( - fixture: &PostRefreshTerminalFixture, - reason: HlsRuntimeCustomTailReason, -) -> TransportStreamBuffer { - let responses = fixture.ctx.app_config.custom_stream_response.load_full().expect("runtime custom responses"); - match reason { - HlsRuntimeCustomTailReason::ChannelUnavailable => responses.channel_unavailable.as_ref(), - HlsRuntimeCustomTailReason::LowPriorityPreempted => responses.low_priority_preempted.as_ref(), - HlsRuntimeCustomTailReason::UserConnectionsExhausted => responses.user_connections_exhausted.as_ref(), - HlsRuntimeCustomTailReason::ProviderConnectionsExhausted => responses.provider_connections_exhausted.as_ref(), - HlsRuntimeCustomTailReason::UserAccountExpired => responses.user_account_expired.as_ref(), - HlsRuntimeCustomTailReason::SessionOrLeaseExpired => responses.hls_session_or_lease_expired.as_ref(), - } - .expect("reason-specific runtime buffer") - .clone() -} - -fn segment_bytes(plan: &HlsTerminalTailPlan, index: u16) -> Bytes { - plan.segment_bytes(HlsTerminalSegmentPath { generation: plan.generation, index }) - .expect("committed immutable custom-tail segment") -} - -fn payload_continuity_bounds(bytes: &[u8]) -> std::collections::HashMap { - let mut payload_bounds = std::collections::HashMap::::new(); - let mut first_packet_discontinuity = std::collections::HashMap::::new(); - for packet in bytes.as_chunks::<188>().0.iter().filter(|packet| packet[0] == 0x47) { - let adaptation_field_control = (packet[3] >> 4) & 0b11; - let pid = (u16::from(packet[1] & 0x1f) << 8) | u16::from(packet[2]); - let discontinuity = matches!(adaptation_field_control, 0b10 | 0b11) - && packet[4] > 0 - && packet.get(5).is_some_and(|flags| flags & 0x80 != 0); - first_packet_discontinuity.entry(pid).or_insert(discontinuity); - if !matches!(adaptation_field_control, 0b01 | 0b11) { - continue; - } - let counter = packet[3] & 0x0f; - payload_bounds.entry(pid).and_modify(|entry| entry.1 = counter).or_insert((counter, counter)); - } - payload_bounds - .into_iter() - .map(|(pid, (first, last))| { - let discontinuity = first_packet_discontinuity.get(&pid).copied().unwrap_or(false); - (pid, (first, last, discontinuity)) - }) - .collect() -} - -fn with_internal_payload_continuity_jump(bytes: &[u8]) -> Vec { - let mut corrupted = bytes.to_vec(); - let mut first_payload_pid = None; - for packet in corrupted.as_chunks_mut::<188>().0.iter_mut().filter(|packet| packet[0] == 0x47) { - let adaptation_field_control = (packet[3] >> 4) & 0b11; - if !matches!(adaptation_field_control, 0b01 | 0b11) { - continue; - } - let pid = (u16::from(packet[1] & 0x1f) << 8) | u16::from(packet[2]); - if pid == 0x1fff { - continue; - } - match first_payload_pid { - None => first_payload_pid = Some(pid), - Some(first_pid) if first_pid == pid => { - let continuity_counter = packet[3] & 0x0f; - packet[3] = (packet[3] & 0xf0) | (continuity_counter.wrapping_add(3) & 0x0f); - return corrupted; - } - Some(_) => {} - } - } - panic!("terminal fixture must contain two payload packets for one PID"); -} - -#[tokio::test] -async fn active_hls_preemption_commits_low_priority_preempted_tail_without_redirect() { - let fixture = post_refresh_terminal_fixture("runtime-preemption", true).await; - - let (_, plan) = - commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted, true).await; - let manifest = terminal_tail_manifest_body(&plan, &fixture.proxy_session_id, &fixture.lease_id) - .expect("preemption plan route binding"); - - assert_eq!(plan.reason, HlsRuntimeCustomTailReason::LowPriorityPreempted); - assert!(manifest.ends_with("#EXT-X-ENDLIST\n")); - assert!(!manifest.contains("/cvs/hls/")); - assert!(matches!( - fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("committed preemption lease") - .playback_mode, - HlsLeasePlaybackMode::TerminalTail(_) - )); -} - -#[tokio::test] -async fn unsafe_live_transport_evidence_commits_unavailable_without_terminal_bytes() { - let fixture = post_refresh_terminal_fixture("runtime-unsafe-live-splice", true).await; - prepare_runtime_custom_bundle(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted).await; - let cache_key = fixture.session.read().await.segments.get(&0).expect("terminal-base segment").cache_key.clone(); - let metadata = fixture - .ctx - .hls_proxy - .segment_cache() - .metadata(&cache_key) - .await - .expect("cache metadata lookup") - .expect("terminal-base metadata"); - tokio::fs::write(&metadata.path, with_internal_payload_continuity_jump(TERMINAL_ASSET_BYTES)) - .await - .expect("replace test fixture with same-size unsafe bytes"); - - let outcome = commit_hls_runtime_custom_tail( - fixture.ctx.clone(), - HlsRuntimeCustomTailRequest { - session: Arc::clone(&fixture.session), - proxy_session_id: fixture.proxy_session_id.clone(), - lease_id: fixture.lease_id.clone(), - reason: HlsRuntimeCustomTailReason::LowPriorityPreempted, - now_ms: fixture.now_ms, - }, - ) - .await; - wait_for_terminal_pending_owners(&fixture, 0).await; - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("unsafe splice lease remains stored"); - - assert_eq!(outcome, HlsRuntimeCustomTailOutcome::PendingOwnerRegistered); - assert!(matches!( - lease.playback_mode, - HlsLeasePlaybackMode::TerminalUnavailable { - reason: HlsTerminalTailCompatibility::SpliceTransportFailure( - super::super::HlsTsSpliceIncompatibility::ContinuityFailure { .. } - ), - .. - } - )); - assert!(fixture.session.read().await.terminal_tail_protection(&fixture.lease_id).is_none()); -} - -#[tokio::test] -async fn preemption_tail_uses_low_priority_asset_not_channel_unavailable_asset() { - let fixture = post_refresh_terminal_fixture("runtime-preemption-asset", true).await; - let low_priority = - snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, HlsRuntimeCustomTailReason::LowPriorityPreempted) - .expect("low-priority asset"); - let channel = snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, HlsRuntimeCustomTailReason::ChannelUnavailable) - .expect("channel-unavailable asset"); - - let (_, plan) = - commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted, true).await; - - assert_eq!(plan.asset_identity, HlsRuntimeCustomTailAssetIdentity::from_asset(&low_priority)); - assert_ne!(plan.asset_identity.media, HlsRuntimeCustomTailAssetIdentity::from_asset(&channel).media); -} - -#[tokio::test] -async fn preemption_tail_preserves_live_to_custom_pts_dts_pcr_and_cc() { - let fixture = post_refresh_terminal_fixture("runtime-preemption-splice", true).await; - let low_priority = configured_runtime_custom_buffer(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted); - let live = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); - let expected_anchor = HlsTsSpliceAnchor::between( - live.finite_hls_timestamp_profile().expect("live timestamp profile"), - low_priority.finite_hls_timestamp_profile().expect("custom timestamp profile"), - ) - .expect("compatible live-to-custom splice"); - - let (_, plan) = - commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted, true).await; - let first = segment_bytes(&plan, 0); - let second = segment_bytes(&plan, 1); - let first_profile = TransportStreamBuffer::new(first.to_vec()) - .finite_hls_timestamp_profile() - .expect("first anchored custom profile"); - let second_profile = TransportStreamBuffer::new(second.to_vec()) - .finite_hls_timestamp_profile() - .expect("second anchored custom profile"); - assert_eq!(first_profile.first_clock_90khz, expected_anchor.terminal_first_clock); - assert!(first_profile.observed_pts_or_dts && first_profile.observed_pcr); - assert!(second_profile.observed_pts_or_dts && second_profile.observed_pcr); - assert_eq!( - second_profile.first_clock_90khz.wrapping_add(1_u64 << 33).wrapping_sub(first_profile.first_clock_90khz) - % (1_u64 << 33), - 902_400 - ); - let first_cc = payload_continuity_bounds(&first); - let second_cc = payload_continuity_bounds(&second); - assert!(!first_cc.is_empty()); - assert!(first_cc.iter().all(|(pid, (_, _, discontinuity))| *pid == 0x1fff || *discontinuity), "{first_cc:?}"); - for (pid, (_, last, _)) in first_cc { - let (next, _, _) = second_cc.get(&pid).expect("PID continues into second custom segment"); - assert_eq!(*next, last.wrapping_add(1) & 0x0f, "PID {pid} continuity"); - } -} - -#[tokio::test] -async fn preemption_tail_is_committed_at_safe_segment_boundary_without_waiting_for_reserve_cutover() { - let fixture = post_refresh_terminal_fixture_with_progress("runtime-preemption-immediate", true, false).await; - let lease_before = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("live lease before immediate cutover"); - assert_eq!(lease_before.playback_mode, HlsLeasePlaybackMode::Live); - - let (outcome, plan) = - commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted, true).await; - - assert!(matches!( - outcome, - HlsRuntimeCustomTailOutcome::Committed | HlsRuntimeCustomTailOutcome::PendingOwnerRegistered - )); - assert_eq!(plan.base_manifest.last_proxy_seq, lease_before.last_manifest_snapshot.unwrap().last_proxy_seq); -} - -#[tokio::test] -async fn preemption_does_not_fetch_another_origin_manifest() { - let fixture = post_refresh_terminal_fixture("runtime-preemption-no-origin", true).await; - let refresh_before = fixture.session.read().await.origin_refresh.clone(); - - let _ = commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted, true).await; - - assert_eq!(fixture.session.read().await.origin_refresh, refresh_before); -} - -async fn assert_active_policy_reason_commits(name: &str, reason: HlsRuntimeCustomTailReason) { - let fixture = post_refresh_terminal_fixture(name, true).await; - let (_, plan) = commit_runtime_custom_reason(&fixture, reason, true).await; - assert_eq!(plan.reason, reason); - assert_eq!(plan.segment_duration_ms, 10_027); - assert_eq!( - plan.asset_identity, - HlsRuntimeCustomTailAssetIdentity::from_asset( - &snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, reason).expect("reason-specific configured asset") - ) - ); -} - -#[tokio::test] -async fn active_provider_exhaustion_commits_provider_exhausted_tail_after_grace() { - assert_active_policy_reason_commits( - "runtime-provider-exhausted", - HlsRuntimeCustomTailReason::ProviderConnectionsExhausted, - ) - .await; -} - -#[tokio::test] -async fn active_user_exhaustion_commits_user_exhausted_tail() { - assert_active_policy_reason_commits("runtime-user-exhausted", HlsRuntimeCustomTailReason::UserConnectionsExhausted) - .await; -} - -#[tokio::test] -async fn active_user_account_expiry_commits_account_expired_tail() { - assert_active_policy_reason_commits("runtime-account-expired", HlsRuntimeCustomTailReason::UserAccountExpired) - .await; -} - -#[tokio::test] -async fn first_committed_custom_reason_is_immutable() { - let fixture = post_refresh_terminal_fixture("runtime-first-reason", true).await; - let (_, first) = - commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted, true).await; - - let outcome = commit_hls_runtime_custom_tail( - fixture.ctx.clone(), - HlsRuntimeCustomTailRequest { - session: Arc::clone(&fixture.session), - proxy_session_id: fixture.proxy_session_id.clone(), - lease_id: fixture.lease_id.clone(), - reason: HlsRuntimeCustomTailReason::ProviderConnectionsExhausted, - now_ms: fixture.now_ms.saturating_add(1), - }, - ) - .await; - let replay = wait_for_runtime_custom_plan(&fixture).await; - - assert_eq!(outcome, HlsRuntimeCustomTailOutcome::AlreadyCommitted); - assert_eq!(replay.reason, HlsRuntimeCustomTailReason::LowPriorityPreempted); - assert_eq!(replay.generation, first.generation); - assert_eq!(segment_bytes(&replay, 0), segment_bytes(&first, 0)); -} - -#[tokio::test] -async fn different_late_reason_cannot_replace_committed_plan() { - let fixture = post_refresh_terminal_fixture("runtime-late-reason", true).await; - let (_, first) = - commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::UserConnectionsExhausted, true).await; - - let (outcome, replay) = - commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::SessionOrLeaseExpired, false).await; - - assert_eq!(outcome, HlsRuntimeCustomTailOutcome::AlreadyCommitted); - assert_eq!(replay.reason, HlsRuntimeCustomTailReason::UserConnectionsExhausted); - assert_eq!(replay.asset_identity, first.asset_identity); -} - -async fn wait_for_terminal_pending_owners(fixture: &PostRefreshTerminalFixture, expected: usize) { - tokio::time::timeout(Duration::from_secs(10), async { - loop { - if fixture.ctx.hls_proxy.terminal_pending().owner_count() == expected { - return; - } - tokio::task::yield_now().await; - } - }) - .await - .unwrap_or_else(|_| { - panic!( - "terminal-pending owner deadline: actual={} expected={expected}", - fixture.ctx.hls_proxy.terminal_pending().owner_count() - ) - }); -} - -#[tokio::test] -async fn asset_reload_supersedes_pending_custom_tail_but_not_committed_bytes() { - let fixture = post_refresh_terminal_fixture("runtime-asset-reload", true).await; - let reason = HlsRuntimeCustomTailReason::LowPriorityPreempted; - let old_asset = snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, reason).expect("old low-priority asset"); - let target_duration_ms = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .and_then(|lease| lease.last_manifest_snapshot.map(|manifest| manifest.target_duration_ms)) - .expect("published target duration"); - let old_key = prepared_terminal_bundle_key(&old_asset.asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); - let publisher = fixture - .ctx - .hls_proxy - .install_controlled_terminal_bundle_flight_for_test(old_key) - .expect("controlled old-asset preparation"); - let pending = commit_hls_runtime_custom_tail( - fixture.ctx.clone(), - HlsRuntimeCustomTailRequest { - session: Arc::clone(&fixture.session), - proxy_session_id: fixture.proxy_session_id.clone(), - lease_id: fixture.lease_id.clone(), - reason, - now_ms: fixture.now_ms, - }, - ) - .await; - assert_eq!(pending, HlsRuntimeCustomTailOutcome::PendingOwnerRegistered); - wait_for_terminal_pending_owners(&fixture, 1).await; - - let mut revised_bytes = LOW_PRIORITY_ASSET_BYTES.to_vec(); - *revised_bytes.last_mut().expect("non-empty low-priority asset") ^= 1; - fixture - .ctx - .app_config - .custom_stream_response - .store(Some(runtime_custom_responses_with_low_priority(&revised_bytes))); - let old_bundle = build_prepared_terminal_bundle(&old_asset.asset, old_key).expect("old controlled relative bundle"); - publisher.publish(HlsPreparedTerminalBundleCompletion::Ready { bundle: old_bundle }); - wait_for_terminal_pending_owners(&fixture, 0).await; - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("reloaded lease remains stored"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - - let (_, committed) = commit_runtime_custom_reason(&fixture, reason, true).await; - let committed_zero = segment_bytes(&committed, 0); - fixture.ctx.app_config.custom_stream_response.store(Some(runtime_custom_responses())); - let replay = wait_for_runtime_custom_plan(&fixture).await; - - assert_eq!(replay.asset_identity, committed.asset_identity); - assert_eq!(segment_bytes(&replay, 0), committed_zero); -} - -#[tokio::test] -async fn same_reason_singleflight_has_one_media_finalizer() { - let fixture = post_refresh_terminal_fixture("runtime-same-reason-singleflight", true).await; - let reason = HlsRuntimeCustomTailReason::LowPriorityPreempted; - let buffer = configured_runtime_custom_buffer(&fixture, reason); - let asset = snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, reason).expect("singleflight asset"); - let target_duration_ms = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .and_then(|lease| lease.last_manifest_snapshot.map(|manifest| manifest.target_duration_ms)) - .expect("published target duration"); - let key = prepared_terminal_bundle_key(&asset.asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); - let publisher = fixture - .ctx - .hls_proxy - .install_controlled_terminal_bundle_flight_for_test(key) - .expect("controlled singleflight preparation"); - let request = || HlsRuntimeCustomTailRequest { - session: Arc::clone(&fixture.session), - proxy_session_id: fixture.proxy_session_id.clone(), - lease_id: fixture.lease_id.clone(), - reason, - now_ms: fixture.now_ms, - }; - - let (first, second) = tokio::join!( - commit_hls_runtime_custom_tail(fixture.ctx.clone(), request()), - commit_hls_runtime_custom_tail(fixture.ctx.clone(), request()), - ); - assert_eq!(first, HlsRuntimeCustomTailOutcome::PendingOwnerRegistered); - assert_eq!(second, HlsRuntimeCustomTailOutcome::PendingOwnerRegistered); - assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 1); - let bundle = build_prepared_terminal_bundle(&asset.asset, key).expect("single relative bundle"); - publisher.publish(HlsPreparedTerminalBundleCompletion::Ready { bundle }); - let plan = wait_for_runtime_custom_plan(&fixture).await; - - assert_eq!(plan.reason, reason); - assert_eq!(buffer.finite_hls_render_count(), usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); - assert_eq!(buffer.finite_hls_finalize_count(), usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); -} - -async fn add_active_post_refresh_lease( - fixture: &PostRefreshTerminalFixture, - lease_id: HlsAccessLeaseId, - active_map: Option, -) { - let lease = HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("multi-lease", &lease_id.0), - fixture.proxy_session_id.clone(), - "multi-lease".to_string(), - lease_id.0.clone(), - 1, - "stream".to_string(), - 1, - fixture.now_ms, - 60_000, - ); - fixture.ctx.hls_proxy.prepare_access_lease(lease).await; - let publication = fixture - .ctx - .hls_proxy - .prepare_access_lease_manifest_publication(&lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("second lease publication guard"); - let mut manifest = pressure_manifest(12_000); - manifest.active_map = active_map; - Arc::make_mut(&mut manifest.visible_segments)[0].uri = - format!("/hls/shared/live/{}/{}/0.ts", fixture.proxy_session_id.0, lease_id.0).into(); - assert!(fixture - .ctx - .hls_proxy - .commit_access_lease_manifest_publication( - &lease_id, - &fixture.proxy_session_id, - publication, - manifest, - fixture.now_ms, - ) - .await - .is_committed()); - assert!(fixture - .ctx - .hls_proxy - .activate_access_lease( - &lease_id, - &fixture.proxy_session_id, - fixture.now_ms, - HlsAccessLeaseTiming { active_window_ms: 60_000, valid_window_ms: 60_000 }, - ) - .await - .is_activated()); -} - -async fn advance_post_refresh_fixture_playback( - ctx: &HlsCtx, - session: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, - playback_elapsed_ms: u64, - now_ms: u64, -) { - let lease = ctx - .hls_proxy - .access_lease_response_snapshot(lease_id, proxy_session_id, now_ms) - .await - .expect("live terminal fixture remains available"); - let identity = lease.media_identity().expect("live terminal fixture identity"); - let playback_at_ms = now_ms.saturating_sub(playback_elapsed_ms); - for proxy_seq in [0_u64, 1] { - let token = ctx - .hls_proxy - .record_access_lease_segment_request_started_if_identity_matches( - lease_id, - proxy_session_id, - identity, - proxy_seq, - playback_at_ms, - ) - .await - .expect("fixture segment request starts"); - assert_eq!( - ctx.hls_proxy - .record_access_lease_segment_request_completed_and_mark_media_if_identity_matches( - session, - lease_id, - proxy_session_id, - identity, - token, - playback_at_ms, - ) - .await, - super::super::manager::HlsMediaActivityCommitOutcome::Committed - ); - } -} - -#[tokio::test(start_paused = true)] -async fn in_flight_post_refresh_owner_does_not_poll_refresh_gate() { - let fixture = post_refresh_terminal_fixture_with_progress("in-flight-refresh-wait", true, false).await; - fixture.session.write().await.origin_refresh.mark_started(fixture.now_ms); - assert_post_refresh_owner_checks_refresh_gate_once(&fixture).await; -} - -#[tokio::test(start_paused = true)] -async fn debounced_post_refresh_owner_does_not_poll_refresh_gate() { - let fixture = post_refresh_terminal_fixture_with_progress("debounced-refresh-wait", true, false).await; - fixture.session.write().await.origin_refresh.next_fetch_allowed_at_ms = fixture.now_ms.saturating_add(10_000); - assert_post_refresh_owner_checks_refresh_gate_once(&fixture).await; -} - -#[tokio::test(start_paused = true)] -async fn persistent_conflict_real_owner_commits_before_exclusive_deadline() { - let fixture = post_refresh_terminal_fixture_with_progress("real-owner-terminal", true, false).await; - register_real_post_refresh_owner( - &fixture, - super::super::refresh::HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, - ) - .await; - assert_availability_owner_registered(&fixture); - - tokio::time::advance(Duration::from_millis(2_100)).await; - assert_eq!( - fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), - 1, - "the real owner must survive its rapid evaluation budget while reserve remains" - ); - advance_post_refresh_fixture_playback( - &fixture.ctx, - &fixture.session, - &fixture.proxy_session_id, - &fixture.lease_id, - 20_100, - fixture.now_ms, - ) - .await; - wait_for_availability_owner_completion(&fixture).await; - - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("terminal lease remains stored"); - assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalTail(_))); -} - -#[tokio::test(start_paused = true)] -async fn hard_failure_real_owner_commits_unavailable_before_exclusive_deadline() { - let fixture = post_refresh_terminal_fixture_with_progress("real-owner-unavailable", false, false).await; - register_real_post_refresh_owner( - &fixture, - super::super::refresh::HlsPostRefreshAvailabilityReason::HardManifestFailure, - ) - .await; - assert_availability_owner_registered(&fixture); - - tokio::time::advance(Duration::from_millis(2_100)).await; - assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1); - // Playback evidence is relative to the clock as it stands now, not to the - // fixture's base: the advance above really does move the scheduling clock. - advance_post_refresh_fixture_playback( - &fixture.ctx, - &fixture.session, - &fixture.proxy_session_id, - &fixture.lease_id, - 7_000, - fixture.ctx.hls_proxy.now_ms(), - ) - .await; - wait_for_availability_owner_completion(&fixture).await; - - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("unavailable lease remains stored"); - assert!(matches!( - lease.playback_mode, - HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::MissingAsset, .. } - )); -} - -#[tokio::test(start_paused = true)] -async fn failed_closed_retry_capacity_retains_owner() { - let fixture = post_refresh_terminal_fixture("capacity-retained-owner", false).await; - fixture.ctx.hls_proxy.set_terminal_commit_retry_capacity_for_test(0); - register_real_post_refresh_owner( - &fixture, - super::super::refresh::HlsPostRefreshAvailabilityReason::HardManifestFailure, - ) - .await; - assert_availability_owner_registered(&fixture); - for _ in 0..32 { - tokio::task::yield_now().await; - } - assert_eq!( - fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), - 1, - "retry-capacity pressure must not drop the last Availability owner" - ); - - fixture - .ctx - .hls_proxy - .set_terminal_commit_retry_capacity_for_test(super::super::terminal_commit::HLS_TERMINAL_COMMIT_RETRY_CAPACITY); - fixture.ctx.hls_proxy.notify_session_evidence_changed(&fixture.proxy_session_id); - wait_for_availability_owner_completion(&fixture).await; - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("capacity retry resolves the live lease"); - assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); -} - -#[tokio::test(start_paused = true)] -async fn failed_closed_lock_contention_retains_owner() { - let fixture = post_refresh_terminal_fixture("lock-retained-owner", false).await; - let owner_key = fixture - .ctx - .hls_proxy - .availability_reevaluation_owner_key(&fixture.session, &fixture.proxy_session_id) - .await - .expect("lock-contention owner key"); - let lease_guard = fixture.ctx.hls_proxy.hold_access_lease_store_for_test().await; - assert_eq!( - register_hls_availability_reevaluation_with_mode( - fixture.ctx.clone(), - Arc::clone(&fixture.session), - owner_key, - post_refresh_owner_request(&fixture), - HlsAvailabilityReevaluationMode::PostRefresh( - super::super::refresh::HlsPostRefreshAvailabilityReason::HardManifestFailure, - ), - ), - HlsAvailabilityReevaluationRegistration::Scheduled - ); - assert_availability_owner_registered(&fixture); - for _ in 0..32 { - tokio::task::yield_now().await; - } - assert_eq!( - fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), - 1, - "lease-store contention must retain session ownership" - ); - - drop(lease_guard); - wait_for_availability_owner_completion(&fixture).await; - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("lock release resolves the live lease"); - assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); -} - -async fn assert_multi_lease_fallback_handles_pending_and_unavailable(reverse_insertion: bool) { - let fixture_name = if reverse_insertion { "multi-lease-fallback-reverse" } else { "multi-lease-fallback-forward" }; - let fixture = post_refresh_terminal_fixture_with_bundle_state(fixture_name, true, true, false).await; - let incompatible_lease_id = HlsAccessLeaseId("multi-lease-incompatible".to_string()); - add_active_post_refresh_lease( - &fixture, - incompatible_lease_id.clone(), - Some(HlsMapSignature { fingerprint: [0x5a; 32], container: HlsMediaContainer::FragmentedMp4 }), - ) - .await; - if reverse_insertion { - let mut leases = fixture.ctx.hls_proxy.hold_access_lease_store_for_test().await; - let primary = leases.remove_access_lease(&fixture.lease_id).expect("primary live lease remains stored"); - leases.prepare_access_lease(primary); - } - let asset = snapshot_terminal_media_asset(&TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec())) - .expect("controlled terminal asset parses"); - let bundle_key = prepared_terminal_bundle_key(&asset, 12_000, HLS_TERMINAL_TAIL_SEGMENT_COUNT); - let _controlled_flight = fixture - .ctx - .hls_proxy - .install_controlled_terminal_bundle_flight_for_test(bundle_key) - .expect("controlled terminal preparation is unique"); - let evaluation_now_ms = fixture.ctx.hls_proxy.terminal_commit_now_ms(); - - let aggregate = - evaluate_owner_failure_fallback(&fixture.ctx, &fixture.session, &fixture.proxy_session_id, evaluation_now_ms) - .await; - - assert_eq!(aggregate.total, 2); - assert_eq!(aggregate.pending_owned, 1); - assert_eq!(aggregate.terminal_committed, 1, "{aggregate:?}"); - assert!(aggregate.unresolved.is_empty()); - assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 1); - let unavailable = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&incompatible_lease_id, &fixture.proxy_session_id, evaluation_now_ms) - .await - .expect("incompatible lease remains stored"); - assert!(matches!(unavailable.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); - fixture.ctx.hls_proxy.terminal_pending().cancel_session(&fixture.proxy_session_id); -} - -#[tokio::test(start_paused = true)] -async fn multi_lease_fallback_keeps_pending_owner_and_commits_other_unavailable() { - assert_multi_lease_fallback_handles_pending_and_unavailable(false).await; -} - -#[tokio::test(start_paused = true)] -async fn multi_lease_fallback_handles_reverse_insertion_without_early_return() { - assert_multi_lease_fallback_handles_pending_and_unavailable(true).await; -} - -#[tokio::test] -async fn replay_conflict_commits_prepared_terminal_tail_before_safe_deadline() { - let fixture = post_refresh_terminal_fixture("post-refresh-terminal", true).await; - let safe_deadline = - HlsLeaseCutoverTiming::from_reserve(fixture.now_ms, 12_900, HlsTransitionMarginMs::from_millis(12_000), None) - .latest_safe_terminal_commit_at - .as_millis_since_epoch(); - - let evaluation = evaluate_active_terminal_leases_for_reevaluation( - &fixture.ctx, - &fixture.session, - &fixture.proxy_session_id, - fixture.now_ms, - ) - .await; - - assert_eq!(evaluation, HlsPostRefreshTerminalEvaluation::TerminalCommitted); - assert!(fixture.now_ms < safe_deadline); - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("terminal lease remains available"); - assert!( - matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalTail(_)), - "prepared compatible asset must commit a terminal tail: {:?}", - lease.playback_mode - ); - assert_eq!( - commit_terminal_tail_if_lease_reserve_requires_cutover( - &fixture.ctx, - &fixture.session, - &fixture.proxy_session_id, - &lease, - safe_deadline.saturating_add(1), - ) - .await, - HlsTerminalResolution::Committed, - "a later client observation sees the immutable terminal decision" - ); -} - -#[tokio::test] -async fn incompatible_terminal_asset_commits_terminal_unavailable_without_client_request() { - let fixture = post_refresh_terminal_fixture("post-refresh-unavailable", false).await; - - let evaluation = evaluate_active_terminal_leases_for_reevaluation( - &fixture.ctx, - &fixture.session, - &fixture.proxy_session_id, - fixture.now_ms, - ) - .await; - - assert_eq!(evaluation, HlsPostRefreshTerminalEvaluation::TerminalCommitted); - let lease = fixture - .ctx - .hls_proxy - .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) - .await - .expect("terminal-unavailable lease remains available"); - assert!(matches!( - lease.playback_mode, - HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::MissingAsset, .. } - )); - assert_eq!( - commit_terminal_tail_if_lease_reserve_requires_cutover( - &fixture.ctx, - &fixture.session, - &fixture.proxy_session_id, - &lease, - fixture.now_ms.saturating_add(60_000), - ) - .await, - HlsTerminalResolution::Committed, - "a later client observation cannot reopen safe-deadline failure" - ); -} - -fn install_atomic_pressure_lease( - store: &mut HlsAccessLeaseStore, - proxy_session_id: &ProxySessionId, - lease_id: &str, - manifest: HlsLeaseManifestSnapshot, - valid_window_ms: u64, -) -> HlsAccessLeaseId { - let lease_id = HlsAccessLeaseId(lease_id.to_string()); - store.prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new(lease_id.0.clone(), lease_id.0.clone()), - proxy_session_id.clone(), - lease_id.0.clone(), - "token".to_string(), - 1, - "stream".to_string(), - 1, - 0, - valid_window_ms, - )); - let guard = store.prepare_manifest_publication(&lease_id, proxy_session_id, 1).expect("manifest publication guard"); - assert!(store.commit_manifest_publication(&lease_id, proxy_session_id, guard, manifest, 1).is_committed()); - assert!(store - .activate_access_lease( - &lease_id, - proxy_session_id, - 2, - HlsAccessLeaseTiming { active_window_ms: valid_window_ms, valid_window_ms }, - ) - .is_activated()); - lease_id -} - -#[tokio::test] -async fn hls_manifest_acceptance_directive_transient_lock_busy_retries_regular_evaluation() { - let mut session = atomic_pressure_session(); - let proxy_session_id = session.proxy_session_id.clone(); - let mut leases = HlsAccessLeaseStore::default(); - install_atomic_pressure_lease( - &mut leases, - &proxy_session_id, - "availability-transient", - pressure_manifest_at(0, 8_000), - 1_000, - ); - let attempts = std::cell::Cell::new(0_usize); - let clock_calls = std::cell::Cell::new(0_usize); - - let access = retry_availability_state_access(|| { - let attempt = attempts.get(); - attempts.set(attempt.saturating_add(1)); - let access = if attempt == 0 { - HlsCriticalHandoffStateAccess::LockBusy - } else { - HlsCriticalHandoffStateAccess::Acquired(evaluate_and_commit_session_recovery_pressure_in_snapshot( - &mut leases, - &mut session, - &proxy_session_id, - atomic_pressure_policy(), - || { - assert_eq!(attempts.get(), 2); - clock_calls.set(clock_calls.get().saturating_add(1)); - 101 - }, - )) - }; - std::future::ready(access) - }) - .await; - - let evidence = availability_snapshot_or_contention(access) - .expect("transient contention must reach the regular snapshot evaluation") - .expect("the active lease supplies recovery evidence"); - assert_eq!(evidence.timing_seed.target_duration_ms, 8_000); - assert_eq!(attempts.get(), 2); - assert_eq!(clock_calls.get(), 1); -} - -#[tokio::test] -async fn hls_manifest_acceptance_directive_exhausted_contention_is_typed() { - let attempts = std::cell::Cell::new(0_usize); - let access: HlsCriticalHandoffStateAccess = retry_availability_state_access(|| { - attempts.set(attempts.get().saturating_add(1)); - std::future::ready(HlsCriticalHandoffStateAccess::LockBusy) - }) - .await; - - let outcome = availability_snapshot_or_contention(access) - .expect_err("exhausted contention must remain a typed endpoint outcome"); - - assert_eq!(attempts.get(), HLS_AVAILABILITY_STATE_ACCESS_ATTEMPTS); - assert_eq!(outcome, HlsAvailabilitySnapshotAccessError::StateContention); -} - -#[test] -fn hls_manifest_acceptance_directive_samples_time_inside_snapshot_scope() { - let mut session = atomic_pressure_session(); - let proxy_session_id = session.proxy_session_id.clone(); - let mut leases = HlsAccessLeaseStore::default(); - install_atomic_pressure_lease( - &mut leases, - &proxy_session_id, - "availability-clock", - pressure_manifest_at(0, 8_000), - 150, - ); - let clock_calls = std::cell::Cell::new(0_usize); - - let evidence = evaluate_and_commit_session_recovery_pressure_in_snapshot( - &mut leases, - &mut session, - &proxy_session_id, - atomic_pressure_policy(), - || { - clock_calls.set(clock_calls.get().saturating_add(1)); - 200 - }, - ); - - assert_eq!(clock_calls.get(), 1); - assert!(evidence.is_none(), "the snapshot-local clock must exclude the lease expired at evaluation time"); -} - -#[test] -fn hls_recovery_timing_publication_late_with_large_reserve_keeps_evidence_without_starting_burst() { - assert_eq!( - recovery_trigger_source(HlsOriginPathCondition::ProgressExpected, true, false, false), - HlsRecoveryTriggerSource::PublicationLate - ); - let directive = acceptance_directive_for_progress( - publication_late_decision(30_000), - lease_timing_seed(), - None, - HlsRecoveryTriggerDiagnostic::new(HlsRecoveryTriggerSource::PublicationLate), - ); - - assert_eq!(directive.trigger, HlsManifestAcceptanceTrigger::None); - assert_eq!(directive.timing_seed, Some(lease_timing_seed())); -} - -#[test] -fn hls_recovery_timing_publication_late_with_narrow_reserve_keeps_full_burst_pending() { - let directive = acceptance_directive_for_progress( - publication_late_decision(14_000), - lease_timing_seed(), - None, - HlsRecoveryTriggerDiagnostic::new(HlsRecoveryTriggerSource::ReservePressure), - ); - - assert_eq!(directive.trigger, HlsManifestAcceptanceTrigger::RecoveryRequired); - assert_eq!(directive.timing_seed.map(|seed| seed.workload.burst), Some(HlsRecoveryBurstWorkload::FullBurstPending)); - - let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); - let seed = directive.timing_seed.expect("narrow reserve keeps timing evidence"); - let timing = HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput { - started_at_ms: 1_000, - burst_plan: plan, - target_duration_ms: seed.target_duration_ms, - transition_margin: seed.transition_margin, - workload: seed.workload, - observed_latency: HlsObservedRecoveryLatency::default(), - required_terminal_media_key: seed.required_terminal_media_key, - terminal_media_preparation: seed.terminal_media_preparation, - policy: HlsRecoveryTimingPolicy::new( - HlsOperationTimeoutMs::from_millis(3_000), - HlsOperationTimeoutMs::from_millis(30_000), - HlsRecoveryEtaMs::from_millis(3_000), - HlsRecoveryEtaMs::from_millis(13_000), - ), - }); - let mut episode = super::super::manifest_acceptance::HlsManifestAcceptanceEpisode::new( - super::super::manifest_acceptance::HlsManifestAcceptanceGeneration(1), - 1_000, - plan, - directive.trigger, - &timing, - ); - assert_eq!(episode.required_candidates(), plan.total_candidates()); - episode.record_full_burst_candidates(plan.total_candidates()); - assert!(episode.full_burst_completed); - assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); -} - -#[test] -fn hls_recovery_timing_unbound_candidate_covers_key_and_map_independent_of_old_manifest() { - let unknown = HlsRecoveryWorkloadEnvelope::acceptance_policy().ceiling(); - - assert_eq!(unknown.burst, HlsRecoveryBurstWorkload::FullBurstPending); - assert_eq!(unknown.segment, HlsRecoverySegmentWorkload::Aes128SegmentFetchWithKeyFetch); - assert_eq!(unknown.map, HlsRecoveryMapWorkload::Fetch); -} - -#[test] -fn hls_session_recovery_pressure_selects_required_lease_over_smaller_raw_reserve() { - let smaller_not_required = evaluated_pressure("lease-a", 4_000, &[1_000, 9_000], 2_000); - let larger_required = evaluated_pressure("lease-b", 8_000, &[8_000, 4_000], 5_000); - - assert!(smaller_not_required.reserve.guaranteed_reserve_ms < larger_required.reserve.guaranteed_reserve_ms); - assert!(!smaller_not_required.reserve.recovery_required); - assert!(larger_required.reserve.recovery_required); - let pressure = aggregate_session_recovery_pressure([smaller_not_required, larger_required.clone()]) - .expect("active lease pressure"); - - assert!(pressure.any_recovery_required); - assert!(!pressure.any_cutover_required); - assert_eq!(pressure.controlling.lease_id, larger_required.lease_id); - let seed = acceptance_timing_seed_for_pressure(&pressure.controlling); - assert_eq!(seed.target_duration_ms, 8_000); - assert_eq!(seed.transition_margin.as_millis(), 8_000); -} - -#[test] -fn hls_session_recovery_pressure_tie_break_is_stable_by_lease_id() { - let lease_b = evaluated_pressure("lease-b", 4_000, &[2_000, 2_000], 4_000); - let mut lease_a = lease_b.clone(); - lease_a.lease_id = HlsAccessLeaseId("lease-a".to_string()); - - let forward = aggregate_session_recovery_pressure([lease_b.clone(), lease_a.clone()]).expect("forward pressure"); - let reverse = aggregate_session_recovery_pressure([lease_a, lease_b]).expect("reverse pressure"); - - assert_eq!(forward.controlling.lease_id.0, "lease-a"); - assert_eq!(reverse.controlling.lease_id.0, "lease-a"); -} - -#[test] -fn hls_session_recovery_pressure_cursor_change_is_observed_by_atomic_commit() { - let mut session = atomic_pressure_session(); - let proxy_session_id = session.proxy_session_id.clone(); - let mut leases = HlsAccessLeaseStore::default(); - let lease_id = - install_atomic_pressure_lease(&mut leases, &proxy_session_id, "lease-a", pressure_manifest_at(0, 8_000), 1_000); - let before = evaluate_and_commit_session_recovery_pressure( - &mut leases, - &mut session, - &proxy_session_id, - 100, - atomic_pressure_policy(), - ) - .expect("initial pressure"); - assert!(!before.decision.evaluate_lease_cutovers); - let identity = leases - .response_snapshot(&lease_id, &proxy_session_id, 100) - .and_then(|lease| lease.media_identity()) - .expect("live media identity"); - assert!(leases - .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 2, 101,) - .is_some()); - - let after = evaluate_and_commit_session_recovery_pressure( - &mut leases, - &mut session, - &proxy_session_id, - 101, - atomic_pressure_policy(), - ) - .expect("cursor pressure"); - - assert!(after.decision.evaluate_lease_cutovers); -} - -#[test] -fn hls_session_recovery_pressure_new_urgent_lease_controls_atomic_commit() { - let mut session = atomic_pressure_session(); - let proxy_session_id = session.proxy_session_id.clone(); - let mut leases = HlsAccessLeaseStore::default(); - install_atomic_pressure_lease(&mut leases, &proxy_session_id, "lease-a", pressure_manifest_at(0, 8_000), 1_000); - let before = evaluate_and_commit_session_recovery_pressure( - &mut leases, - &mut session, - &proxy_session_id, - 100, - atomic_pressure_policy(), - ) - .expect("initial pressure"); - assert!(!before.decision.evaluate_lease_cutovers); - install_atomic_pressure_lease(&mut leases, &proxy_session_id, "lease-b", pressure_manifest_at(2, 10_000), 1_000); - - let after = evaluate_and_commit_session_recovery_pressure( - &mut leases, - &mut session, - &proxy_session_id, - 101, - atomic_pressure_policy(), - ) - .expect("urgent pressure"); - - assert!(after.decision.evaluate_lease_cutovers); - assert_eq!(after.timing_seed.target_duration_ms, 10_000); -} - -#[test] -fn hls_session_recovery_pressure_expired_controller_is_excluded_atomically() { - let mut session = atomic_pressure_session(); - let proxy_session_id = session.proxy_session_id.clone(); - let mut leases = HlsAccessLeaseStore::default(); - install_atomic_pressure_lease(&mut leases, &proxy_session_id, "lease-a", pressure_manifest_at(0, 8_000), 1_000); - install_atomic_pressure_lease(&mut leases, &proxy_session_id, "lease-b", pressure_manifest_at(2, 10_000), 150); - let urgent = evaluate_and_commit_session_recovery_pressure( - &mut leases, - &mut session, - &proxy_session_id, - 100, - atomic_pressure_policy(), - ) - .expect("urgent pressure"); - assert_eq!(urgent.timing_seed.target_duration_ms, 10_000); - - let after_expiry = evaluate_and_commit_session_recovery_pressure( - &mut leases, - &mut session, - &proxy_session_id, - 200, - atomic_pressure_policy(), - ) - .expect("remaining pressure"); - - assert_eq!(after_expiry.timing_seed.target_duration_ms, 8_000); - assert!(!after_expiry.decision.evaluate_lease_cutovers); -} - -#[test] -fn hls_session_recovery_pressure_new_publication_lateness_uses_current_reserve_evidence() { - let mut session = atomic_pressure_session(); - session.origin_control.path_condition = HlsOriginPathCondition::ProgressExpected; - session.origin_control.last_media_progress_at_ms = Some(0); - let proxy_session_id = session.proxy_session_id.clone(); - let mut leases = HlsAccessLeaseStore::default(); - install_atomic_pressure_lease(&mut leases, &proxy_session_id, "lease-a", pressure_manifest_at(0, 8_000), 20_000); - let policy = HlsRecoveryPressurePolicy { - burst_plan: shared::model::HlsManifestRecoveryBurstPlan { slots: 1, lanes_per_slot: 1 }, - timing: HlsRecoveryTimingPolicy::new( - HlsOperationTimeoutMs::from_millis(1_000), - HlsOperationTimeoutMs::from_millis(10_000), - HlsRecoveryEtaMs::from_millis(0), - HlsRecoveryEtaMs::from_millis(2_000), - ), - }; - - let pressure = - evaluate_and_commit_session_recovery_pressure(&mut leases, &mut session, &proxy_session_id, 12_000, policy) - .expect("publication-late pressure"); - - assert!(pressure.decision.start_acceptance_episode); - assert!(pressure.decision.close_admission); - assert!(!pressure.decision.evaluate_lease_cutovers); -} +mod admission; + +mod policy; + +mod publication; + +mod recovery; + +mod storage; + +mod terminal; + +mod transport; + +mod terminal_fixtures; +use terminal_fixtures::{ + assert_post_refresh_registration_failure_leaves_no_unowned_live_lease, + assert_terminal_pending_registration_failure_commits_unavailable, terminal_base_without_timestamps, + terminal_pending_commit_fixture, terminal_pending_commit_fixture_with_base, terminal_pending_commit_reserve, +}; +mod pressure_fixtures; +use pressure_fixtures::{ + atomic_pressure_policy, atomic_pressure_session, evaluated_pressure, install_atomic_pressure_lease, + pressure_manifest, pressure_manifest_at, +}; +mod owner_fixtures; +use owner_fixtures::{ + assert_availability_owner_registered, assert_post_refresh_owner_checks_refresh_gate_once, + post_refresh_owner_request, post_refresh_terminal_fixture, post_refresh_terminal_fixture_with_bundle_state, + post_refresh_terminal_fixture_with_progress, register_real_post_refresh_owner, + wait_for_availability_owner_completion, +}; +mod runtime_fixtures; +use runtime_fixtures::{ + advance_post_refresh_fixture_playback, assert_active_policy_reason_commits, + assert_multi_lease_fallback_handles_pending_and_unavailable, commit_runtime_custom_reason, + configured_runtime_custom_buffer, payload_continuity_bounds, prepare_runtime_custom_bundle, segment_bytes, + wait_for_runtime_custom_plan, wait_for_terminal_pending_owners, with_internal_payload_continuity_jump, +}; diff --git a/backend/hls/src/availability/tests/admission.rs b/backend/hls/src/availability/tests/admission.rs new file mode 100644 index 000000000..35c53fdab --- /dev/null +++ b/backend/hls/src/availability/tests/admission.rs @@ -0,0 +1,104 @@ +use super::*; + +#[tokio::test] +async fn terminal_pending_capacity_failure_commits_terminal_unavailable() { + assert_terminal_pending_registration_failure_commits_unavailable( + HlsTerminalPendingRegistration::CapacityExceeded, + "pending-capacity-failure", + ) + .await; +} + +#[tokio::test] +async fn post_refresh_coordinator_capacity_failure_leaves_no_unowned_live_lease() { + assert_post_refresh_registration_failure_leaves_no_unowned_live_lease( + HlsAvailabilityReevaluationRegistration::CapacityExceeded, + "post-refresh-capacity-failure", + ) + .await; +} + +#[tokio::test] +async fn capacity_deferred_ready_boundary_keeps_affected_lease_live() { + let hls_ctx = crate::HlsCtx::for_test(Config::default()); + let ctx = &hls_ctx; + let now_ms = ctx.hls_proxy.terminal_commit_now_ms(); + let mut session = atomic_pressure_session(); + session.segments.get_mut(&1).expect("second segment").status = + SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Prefetch, deferred_at_ms: 100 }; + let proxy_session_id = session.proxy_session_id.clone(); + let session = Arc::new(tokio::sync::RwLock::new(session)); + let lease_id = HlsAccessLeaseId("capacity-deferred-live".to_string()); + let mut lease = HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("capacity-user", "capacity-client"), + proxy_session_id.clone(), + "capacity-user".to_string(), + "capacity-session".to_string(), + 1, + "capacity-stream".to_string(), + 1, + now_ms, + 60_000, + ); + lease.state = HlsAccessLeaseState::Activated; + lease.active_until_ms = Some(now_ms.saturating_add(60_000)); + lease.pending_deadline = None; + lease.last_manifest_snapshot = Some(pressure_manifest_at(0, 8_000)); + ctx.hls_proxy.prepare_access_lease(lease.clone()).await; + + let resolution = + commit_terminal_tail_if_lease_reserve_requires_cutover(ctx, &session, &proxy_session_id, &lease, now_ms).await; + + assert_eq!(resolution, HlsTerminalResolution::LiveAllowed); + let current = ctx + .hls_proxy + .access_lease_response_snapshot(&lease_id, &proxy_session_id, now_ms) + .await + .expect("capacity-deferred lease remains available"); + assert_eq!(current.state, HlsAccessLeaseState::Activated); + assert_eq!(current.playback_mode, HlsLeasePlaybackMode::Live); +} + +#[tokio::test(start_paused = true)] +async fn failed_closed_lock_contention_retains_owner() { + let fixture = post_refresh_terminal_fixture("lock-retained-owner", false).await; + let owner_key = fixture + .ctx + .hls_proxy + .availability_reevaluation_owner_key(&fixture.session, &fixture.proxy_session_id) + .await + .expect("lock-contention owner key"); + let lease_guard = fixture.ctx.hls_proxy.hold_access_lease_store_for_test().await; + assert_eq!( + register_hls_availability_reevaluation_with_mode( + fixture.ctx.clone(), + Arc::clone(&fixture.session), + owner_key, + post_refresh_owner_request(&fixture), + HlsAvailabilityReevaluationMode::PostRefresh( + super::super::super::refresh::HlsPostRefreshAvailabilityReason::HardManifestFailure, + ), + ), + HlsAvailabilityReevaluationRegistration::Scheduled + ); + assert_availability_owner_registered(&fixture); + for _ in 0..32 { + tokio::task::yield_now().await; + } + assert_eq!( + fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), + 1, + "lease-store contention must retain session ownership" + ); + + drop(lease_guard); + wait_for_availability_owner_completion(&fixture).await; + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("lock release resolves the live lease"); + assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); +} diff --git a/backend/hls/src/availability/tests/owner_fixtures.rs b/backend/hls/src/availability/tests/owner_fixtures.rs new file mode 100644 index 000000000..3ae6a2e53 --- /dev/null +++ b/backend/hls/src/availability/tests/owner_fixtures.rs @@ -0,0 +1,346 @@ +use super::*; + +pub(super) fn post_refresh_owner_request(fixture: &PostRefreshTerminalFixture) -> OriginRefreshRequest { + let origin_entry = + super::super::super::manifest_fetch::LiveHlsOriginEntry::parse("http://127.0.0.1:9/live/user/pass/12345.m3u8") + .expect("test origin entry parses"); + OriginRefreshRequest { + app_config: Arc::clone(&fixture.ctx.app_config), + session: Arc::clone(&fixture.session), + origin_entry, + headers: axum::http::HeaderMap::new(), + origin_provider_session_headers: axum::http::HeaderMap::new(), + disabled_headers: None, + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("test client builds"), + use_manual_redirects: false, + segment_cache: Arc::clone(fixture.ctx.hls_proxy.segment_cache()), + hls_proxy: Arc::clone(&fixture.ctx.hls_proxy), + segment_repair: Arc::clone(fixture.ctx.hls_proxy.segment_repair()), + segment_worker_pool: Arc::clone(fixture.ctx.hls_proxy.segment_worker_pool()), + map_worker_pool: Arc::clone(fixture.ctx.hls_proxy.map_worker_pool()), + origin_manifest_timeout_ms: fixture.ctx.hls_proxy.origin_manifest_timeout_ms(), + manifest_recovery_burst: fixture.ctx.hls_proxy.manifest_recovery_burst(), + strip: fixture.ctx.hls_proxy.strip(), + retry_policy: super::super::super::manifest_fetch::RetryPolicy { delays_ms: [0; 5], jitter_max_ms: 0 }, + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: + super::super::super::refresh::HlsManifestCommitRequirement::CommittedManifestAllowed, + fresh_manifest_requirement_generation: None, + acceptance_directive: HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + now_ms: fixture.now_ms, + origin_io: None, + post_refresh_runtime: Some(super::super::super::refresh::HlsPostRefreshRuntime { + ctx: fixture.ctx.downgrade(), + }), + } +} + +pub(super) async fn register_real_post_refresh_owner( + fixture: &PostRefreshTerminalFixture, + reason: super::super::super::refresh::HlsPostRefreshAvailabilityReason, +) { + let (origin_progress_generation, media_readiness_generation) = { + let session = fixture.session.read().await; + (session.origin_control.progress_generation, session.activity.media_readiness_generation) + }; + assert_eq!( + register_post_refresh_availability_reevaluation( + fixture.ctx.clone(), + Arc::clone(&fixture.session), + post_refresh_owner_request(fixture), + HlsPostRefreshAvailabilityAction::Reevaluate { + reason, + origin_progress_generation, + media_readiness_generation, + }, + ) + .await, + HlsAvailabilityReevaluationRegistration::Scheduled + ); +} + +pub(super) fn assert_availability_owner_registered(fixture: &PostRefreshTerminalFixture) { + assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1); +} + +/// How long the owner gets to finish before the wait gives up. +/// +/// These tests run on a paused clock, so this is logical time and costs +/// nothing when the owner completes normally. +const AVAILABILITY_OWNER_COMPLETION_TIMEOUT: Duration = Duration::from_secs(30); + +/// Number of observer revisions to accept before concluding the owner is +/// cycling without converging. +const AVAILABILITY_OWNER_MAX_REVISIONS: usize = 1_024; + +pub(super) async fn wait_for_availability_owner_completion(fixture: &PostRefreshTerminalFixture) { + let coordinator = fixture.ctx.hls_proxy.availability_reevaluations(); + // Bounded twice over, because the two ways this can fail to converge + // need different guards. A parked owner never wakes the observer, and + // on a paused clock the runtime is idle, so the timeout fires. An owner + // that keeps re-arming a successor wakes the observer forever and keeps + // the runtime busy, so the clock never advances and only the revision + // count catches it. Either way the test fails with a diagnostic instead + // of hanging the suite. + let wait = async { + let mut revisions = 0usize; + while let Some(mut observer) = coordinator.observe_owner(&fixture.proxy_session_id) { + if matches!( + observer.changed().await, + crate::availability_reevaluation::HlsAvailabilityReevaluationObservation::OwnerFinished + ) { + // The owner is gone; re-check the map and leave the loop. + continue; + } + revisions += 1; + assert!( + revisions <= AVAILABILITY_OWNER_MAX_REVISIONS, + "availability owner for {:?} produced {revisions} revisions without completing", + fixture.proxy_session_id + ); + } + }; + assert!( + tokio::time::timeout(AVAILABILITY_OWNER_COMPLETION_TIMEOUT, wait).await.is_ok(), + "availability owner for {:?} did not complete within {AVAILABILITY_OWNER_COMPLETION_TIMEOUT:?}; \ + {} owner(s) still registered", + fixture.proxy_session_id, + coordinator.owner_count() + ); + assert_eq!(coordinator.owner_count(), 0); +} + +pub(super) async fn assert_post_refresh_owner_checks_refresh_gate_once(fixture: &PostRefreshTerminalFixture) { + let refresh_skipped_before = fixture.ctx.hls_proxy.metrics().snapshot().refresh_skipped; + register_real_post_refresh_owner( + fixture, + super::super::super::refresh::HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, + ) + .await; + for _ in 0..256 { + if fixture.ctx.hls_proxy.metrics().snapshot().refresh_skipped == refresh_skipped_before.saturating_add(1) { + break; + } + tokio::task::yield_now().await; + } + assert_eq!( + fixture.ctx.hls_proxy.metrics().snapshot().refresh_skipped, + refresh_skipped_before.saturating_add(1), + "the owner must observe the unavailable refresh gate once" + ); + + for _ in 0..10 { + tokio::time::advance(Duration::from_millis(50)).await; + tokio::task::yield_now().await; + } + assert_eq!( + fixture.ctx.hls_proxy.metrics().snapshot().refresh_skipped, + refresh_skipped_before.saturating_add(1), + "unchanged refresh evidence must not produce repeated gate attempts" + ); + + fixture.ctx.hls_proxy.availability_reevaluations().cancel_session(&fixture.proxy_session_id); + wait_for_availability_owner_completion(fixture).await; +} + +pub(super) async fn post_refresh_terminal_fixture(name: &str, terminal_asset: bool) -> PostRefreshTerminalFixture { + post_refresh_terminal_fixture_with_progress(name, terminal_asset, true).await +} + +pub(super) async fn post_refresh_terminal_fixture_with_progress( + name: &str, + terminal_asset: bool, + complete_playback: bool, +) -> PostRefreshTerminalFixture { + post_refresh_terminal_fixture_with_bundle_state(name, terminal_asset, complete_playback, true).await +} + +pub(super) fn post_refresh_origin_manifest(terminal_asset: bool) -> (u64, ParsedOriginManifest) { + let (target_duration_ms, body) = if terminal_asset { + ( + 12_000, + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-TARGETDURATION:12\n\ + #EXTINF:12.0,\n0.ts\n#EXTINF:12.0,\n1.ts\n#EXTINF:12.0,\n2.ts\n", + ) + } else { + ( + 8_000, + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-TARGETDURATION:8\n\ + #EXTINF:4.0,\n0.ts\n#EXTINF:8.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n", + ) + }; + let OriginManifestParseOutcome::Normal(manifest) = + parse_origin_media_manifest(body, "http://origin.example/live/index.m3u8") + else { + panic!("post-refresh terminal fixture parses"); + }; + (target_duration_ms, manifest) +} + +pub(super) async fn prepare_post_refresh_terminal_base( + ctx: &HlsCtx, + session: &HlsSessionHandle, + manifest: &ParsedOriginManifest, + terminal_asset: bool, + prepare_terminal_bundle: bool, + target_duration_ms: u64, + now_ms: u64, +) { + let terminal_base_cache_key = { + let mut session = session.write().await; + session.apply_origin_manifest(manifest).expect("fixture timeline applies"); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: now_ms }; + } + session.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict; + session.origin_control.last_media_progress_at_ms = Some(now_ms); + session.origin_control.target_duration_snapshot_ms = Some(target_duration_ms); + session.segments.get(&0).expect("fixture terminal-base segment").cache_key.clone() + }; + if !terminal_asset { + return; + } + ctx.hls_proxy + .segment_cache() + .write_bytes_and_commit(&terminal_base_cache_key, TERMINAL_ASSET_BYTES) + .await + .expect("terminal-compatible READY base bytes commit"); + session.write().await.segments.get_mut(&0).expect("fixture terminal-base segment").status = + SegmentCacheStatus::Ready { + content_length: u64::try_from(TERMINAL_ASSET_BYTES.len()).unwrap_or(u64::MAX), + ready_at_ms: now_ms, + }; + if prepare_terminal_bundle { + let asset = snapshot_terminal_media_asset(&TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec())) + .expect("fixture terminal asset parses"); + let key = prepared_terminal_bundle_key(&asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); + let state = ctx.hls_proxy.start_prepared_terminal_bundle( + Arc::clone(&asset), + target_duration_ms, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, + ); + let state = match state { + HlsPreparedTerminalBundleState::Preparing { .. } => { + ctx.hls_proxy.wait_for_prepared_terminal_bundle(key).await + } + HlsPreparedTerminalBundleState::Ready { .. } + | HlsPreparedTerminalBundleState::Failed { .. } + | HlsPreparedTerminalBundleState::Incompatible { .. } => Some(state), + }; + assert!( + matches!(state, Some(HlsPreparedTerminalBundleState::Ready { .. })), + "terminal preparation must be ready: {state:?}" + ); + } +} + +pub(super) async fn publish_post_refresh_terminal_lease( + ctx: &HlsCtx, + proxy_session_id: &ProxySessionId, + name: &str, + terminal_asset: bool, + target_duration_ms: u64, + now_ms: u64, +) -> HlsAccessLeaseId { + let lease_id = HlsAccessLeaseId(format!("{name}-lease")); + ctx.hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new(name, name), + proxy_session_id.clone(), + name.to_string(), + "token".to_string(), + 1, + "stream".to_string(), + 1, + now_ms, + 60_000, + )) + .await; + let publication = ctx + .hls_proxy + .prepare_access_lease_manifest_publication(&lease_id, proxy_session_id, now_ms) + .await + .expect("terminal fixture publication guard"); + let mut manifest_snapshot = pressure_manifest(target_duration_ms); + if terminal_asset { + Arc::make_mut(&mut manifest_snapshot.visible_segments)[0].duration_ms = target_duration_ms; + manifest_snapshot.playlist_duration_ms = target_duration_ms; + manifest_snapshot.last_visible_media_end_ms = target_duration_ms; + } + Arc::make_mut(&mut manifest_snapshot.visible_segments)[0].uri = + format!("/hls/shared/live/{}/{}/0.ts", proxy_session_id.0, lease_id.0).into(); + assert!(ctx + .hls_proxy + .commit_access_lease_manifest_publication(&lease_id, proxy_session_id, publication, manifest_snapshot, now_ms,) + .await + .is_committed()); + assert!(ctx + .hls_proxy + .activate_access_lease( + &lease_id, + proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 60_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + assert!( + ctx.hls_proxy.access_lease_response_snapshot(&lease_id, proxy_session_id, now_ms).await.is_some(), + "activated terminal fixture lease remains available" + ); + lease_id +} + +pub(super) async fn post_refresh_terminal_fixture_with_bundle_state( + name: &str, + terminal_asset: bool, + complete_playback: bool, + prepare_terminal_bundle: bool, +) -> PostRefreshTerminalFixture { + let hls_ctx = crate::HlsCtx::for_test(Config { custom_stream_response_enabled: true, ..Config::default() }); + let ctx = &hls_ctx; + if terminal_asset { + ctx.app_config.custom_stream_response.store(Some(runtime_custom_responses())); + } + let now_ms = ctx.hls_proxy.terminal_commit_now_ms(); + // These tests drive time with `tokio::time::advance`. Anchor the scheduling + // clock to tokio's so the owner's deadlines move with it; otherwise it + // sleeps towards a wall-clock deadline the paused clock never reaches. + ctx.hls_proxy.follow_tokio_clock_for_test(now_ms); + let (session, _) = + ctx.hls_proxy.get_or_create_session_with_outcome(HlsSessionKey::new(1, name), b"secret", now_ms).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let (target_duration_ms, manifest) = post_refresh_origin_manifest(terminal_asset); + prepare_post_refresh_terminal_base( + ctx, + &session, + &manifest, + terminal_asset, + prepare_terminal_bundle, + target_duration_ms, + now_ms, + ) + .await; + let lease_id = + publish_post_refresh_terminal_lease(ctx, &proxy_session_id, name, terminal_asset, target_duration_ms, now_ms) + .await; + if complete_playback { + advance_post_refresh_fixture_playback( + ctx, + &session, + &proxy_session_id, + &lease_id, + if terminal_asset { 22_000 } else { 7_000 }, + now_ms, + ) + .await; + } + PostRefreshTerminalFixture { ctx: ctx.clone(), session, proxy_session_id, lease_id, now_ms } +} diff --git a/backend/hls/src/availability/tests/policy.rs b/backend/hls/src/availability/tests/policy.rs new file mode 100644 index 000000000..2072b70d9 --- /dev/null +++ b/backend/hls/src/availability/tests/policy.rs @@ -0,0 +1,86 @@ +use super::*; + +#[tokio::test] +async fn post_refresh_runtime_failure_leaves_no_unowned_live_lease() { + assert_post_refresh_registration_failure_leaves_no_unowned_live_lease( + HlsAvailabilityReevaluationRegistration::RuntimeUnavailable, + "post-refresh-runtime-failure", + ) + .await; +} + +#[tokio::test] +async fn different_late_reason_cannot_replace_committed_plan() { + let fixture = post_refresh_terminal_fixture("runtime-late-reason", true).await; + let (_, first) = + commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::UserConnectionsExhausted, true).await; + + let (outcome, replay) = + commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::SessionOrLeaseExpired, false).await; + + assert_eq!(outcome, HlsRuntimeCustomTailOutcome::AlreadyCommitted); + assert_eq!(replay.reason, HlsRuntimeCustomTailReason::UserConnectionsExhausted); + assert_eq!(replay.asset_identity, first.asset_identity); +} + +#[tokio::test] +async fn same_reason_singleflight_has_one_media_finalizer() { + let fixture = post_refresh_terminal_fixture("runtime-same-reason-singleflight", true).await; + let reason = HlsRuntimeCustomTailReason::LowPriorityPreempted; + let buffer = configured_runtime_custom_buffer(&fixture, reason); + let asset = snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, reason).expect("singleflight asset"); + let target_duration_ms = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .and_then(|lease| lease.last_manifest_snapshot.map(|manifest| manifest.target_duration_ms)) + .expect("published target duration"); + let key = prepared_terminal_bundle_key(&asset.asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); + let publisher = fixture + .ctx + .hls_proxy + .install_controlled_terminal_bundle_flight_for_test(key) + .expect("controlled singleflight preparation"); + let request = || HlsRuntimeCustomTailRequest { + session: Arc::clone(&fixture.session), + proxy_session_id: fixture.proxy_session_id.clone(), + lease_id: fixture.lease_id.clone(), + reason, + now_ms: fixture.now_ms, + }; + + let (first, second) = tokio::join!( + commit_hls_runtime_custom_tail(fixture.ctx.clone(), request()), + commit_hls_runtime_custom_tail(fixture.ctx.clone(), request()), + ); + assert_eq!(first, HlsRuntimeCustomTailOutcome::PendingOwnerRegistered); + assert_eq!(second, HlsRuntimeCustomTailOutcome::PendingOwnerRegistered); + assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 1); + let bundle = build_prepared_terminal_bundle(&asset.asset, key).expect("single relative bundle"); + publisher.publish(HlsPreparedTerminalBundleCompletion::Ready { bundle }); + let plan = wait_for_runtime_custom_plan(&fixture).await; + + assert_eq!(plan.reason, reason); + assert_eq!(buffer.finite_hls_render_count(), usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); + assert_eq!(buffer.finite_hls_finalize_count(), usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT)); +} + +#[tokio::test(start_paused = true)] +async fn in_flight_post_refresh_owner_does_not_poll_refresh_gate() { + let fixture = post_refresh_terminal_fixture_with_progress("in-flight-refresh-wait", true, false).await; + fixture.session.write().await.origin_refresh.mark_started(fixture.now_ms); + assert_post_refresh_owner_checks_refresh_gate_once(&fixture).await; +} + +#[tokio::test(start_paused = true)] +async fn debounced_post_refresh_owner_does_not_poll_refresh_gate() { + let fixture = post_refresh_terminal_fixture_with_progress("debounced-refresh-wait", true, false).await; + fixture.session.write().await.origin_refresh.next_fetch_allowed_at_ms = fixture.now_ms.saturating_add(10_000); + assert_post_refresh_owner_checks_refresh_gate_once(&fixture).await; +} + +#[tokio::test(start_paused = true)] +async fn multi_lease_fallback_handles_reverse_insertion_without_early_return() { + assert_multi_lease_fallback_handles_pending_and_unavailable(true).await; +} diff --git a/backend/hls/src/availability/tests/pressure_fixtures.rs b/backend/hls/src/availability/tests/pressure_fixtures.rs new file mode 100644 index 000000000..243712dd8 --- /dev/null +++ b/backend/hls/src/availability/tests/pressure_fixtures.rs @@ -0,0 +1,156 @@ +use super::*; + +pub(super) fn pressure_manifest(target_duration_ms: u64) -> HlsLeaseManifestSnapshot { + HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(1), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 1, + delivered_at_ms: 1, + first_proxy_seq: 0, + last_proxy_seq: 0, + visible_segments: Arc::from([HlsLeaseManifestSegment { + proxy_seq: 0, + duration_ms: 4_000, + uri: "0.ts".into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }]), + discontinuity_sequence: 0, + target_duration_ms, + playlist_duration_ms: 4_000, + last_visible_media_end_ms: 4_000, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + } +} + +pub(super) fn pressure_manifest_at(proxy_seq: u64, target_duration_ms: u64) -> HlsLeaseManifestSnapshot { + let mut manifest = pressure_manifest(target_duration_ms); + manifest.first_proxy_seq = proxy_seq; + manifest.last_proxy_seq = proxy_seq; + Arc::make_mut(&mut manifest.visible_segments)[0].proxy_seq = proxy_seq; + manifest +} + +pub(super) fn pressure_timeline(hidden_durations_ms: &[u64]) -> HlsReadyTimelineSnapshot { + let mut start_ms = 0_u64; + let mut units = Vec::with_capacity(hidden_durations_ms.len().saturating_add(1)); + for (index, duration_ms) in std::iter::once(4_000_u64).chain(hidden_durations_ms.iter().copied()).enumerate() { + units.push(HlsReadyTimelineUnit { + proxy_seq: u64::try_from(index).unwrap_or(u64::MAX), + start_ms, + duration_ms, + state: HlsReadyMediaState::Ready, + required_map_ready: true, + required_key_ready: true, + key_ready_valid_until_ms: None, + }); + start_ms = start_ms.saturating_add(duration_ms); + } + HlsReadyTimelineSnapshot { units: units.into() } +} + +pub(super) fn evaluated_pressure( + lease_id: &str, + target_duration_ms: u64, + hidden_durations_ms: &[u64], + recovery_budget_ms: u64, +) -> HlsLeaseRecoveryEvidence { + let manifest = pressure_manifest(target_duration_ms); + let ready_timeline = pressure_timeline(hidden_durations_ms); + let recovery_trigger_budget = HlsRecoveryTriggerBudgetMs::from_millis(recovery_budget_ms); + let reserve = evaluate_lease_reserve(HlsLeaseReserveInput { + manifest: &manifest, + cursor: &HlsLeasePlaybackCursor::default(), + ready_timeline: &ready_timeline, + now_ms: 100, + playback_rate_guard_milli: HLS_PLAYBACK_RATE_GUARD_MILLI, + recovery_trigger_budget, + origin_path_degraded: true, + recovery_committed: false, + }); + let boundary_ms = recovery_trigger_budget.as_millis().saturating_add(reserve.transition_margin.as_millis()); + let cutover_timing = + HlsLeaseCutoverTiming::from_reserve(100, reserve.guaranteed_reserve_ms, reserve.transition_margin, None); + HlsLeaseRecoveryEvidence { + lease_id: HlsAccessLeaseId(lease_id.to_string()), + reserve, + cursor: HlsLeasePlaybackCursor::default(), + workload: HlsRecoveryWorkloadEnvelope::acceptance_policy().ceiling(), + target_duration_ms, + latest_safe_terminal_commit_at: cutover_timing.latest_safe_terminal_commit_at, + recovery_boundary_slack_ms: HlsRecoveryBoundarySlackMs::from_reserve_and_boundary( + reserve.guaranteed_reserve_ms, + boundary_ms, + ), + } +} + +pub(super) fn atomic_pressure_policy() -> HlsRecoveryPressurePolicy { + HlsRecoveryPressurePolicy { + burst_plan: shared::model::HlsManifestRecoveryBurstPlan { slots: 1, lanes_per_slot: 1 }, + timing: HlsRecoveryTimingPolicy::new( + HlsOperationTimeoutMs::from_millis(1_000), + HlsOperationTimeoutMs::from_millis(1_000), + HlsRecoveryEtaMs::from_millis(0), + HlsRecoveryEtaMs::from_millis(0), + ), + } +} + +pub(super) fn atomic_pressure_session() -> HlsSession { + let mut session = HlsSession::new(HlsSessionKey::new(1, "pressure"), b"secret", 0); + let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-TARGETDURATION:8\n\ + #EXTINF:4.0,\n0.ts\n#EXTINF:8.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n"; + let OriginManifestParseOutcome::Normal(manifest) = + parse_origin_media_manifest(body, "http://origin.example/live/index.m3u8") + else { + panic!("pressure manifest parses"); + }; + session.apply_origin_manifest(&manifest).expect("pressure timeline applies"); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1 }; + } + session.origin_control.path_condition = HlsOriginPathCondition::RetryableFetchFailure; + session.origin_control.last_media_progress_at_ms = Some(90); + session.origin_control.target_duration_snapshot_ms = Some(8_000); + session +} + +pub(super) fn install_atomic_pressure_lease( + store: &mut HlsAccessLeaseStore, + proxy_session_id: &ProxySessionId, + lease_id: &str, + manifest: HlsLeaseManifestSnapshot, + valid_window_ms: u64, +) -> HlsAccessLeaseId { + let lease_id = HlsAccessLeaseId(lease_id.to_string()); + store.prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new(lease_id.0.clone(), lease_id.0.clone()), + proxy_session_id.clone(), + lease_id.0.clone(), + "token".to_string(), + 1, + "stream".to_string(), + 1, + 0, + valid_window_ms, + )); + let guard = store.prepare_manifest_publication(&lease_id, proxy_session_id, 1).expect("manifest publication guard"); + assert!(store.commit_manifest_publication(&lease_id, proxy_session_id, guard, manifest, 1).is_committed()); + assert!(store + .activate_access_lease( + &lease_id, + proxy_session_id, + 2, + HlsAccessLeaseTiming { active_window_ms: valid_window_ms, valid_window_ms }, + ) + .is_activated()); + lease_id +} diff --git a/backend/hls/src/availability/tests/publication.rs b/backend/hls/src/availability/tests/publication.rs new file mode 100644 index 000000000..49670a8c6 --- /dev/null +++ b/backend/hls/src/availability/tests/publication.rs @@ -0,0 +1,126 @@ +use super::*; + +#[tokio::test] +async fn preemption_does_not_fetch_another_origin_manifest() { + let fixture = post_refresh_terminal_fixture("runtime-preemption-no-origin", true).await; + let refresh_before = fixture.session.read().await.origin_refresh.clone(); + + let _ = commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted, true).await; + + assert_eq!(fixture.session.read().await.origin_refresh, refresh_before); +} + +#[tokio::test] +async fn first_committed_custom_reason_is_immutable() { + let fixture = post_refresh_terminal_fixture("runtime-first-reason", true).await; + let (_, first) = + commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted, true).await; + + let outcome = commit_hls_runtime_custom_tail( + fixture.ctx.clone(), + HlsRuntimeCustomTailRequest { + session: Arc::clone(&fixture.session), + proxy_session_id: fixture.proxy_session_id.clone(), + lease_id: fixture.lease_id.clone(), + reason: HlsRuntimeCustomTailReason::ProviderConnectionsExhausted, + now_ms: fixture.now_ms.saturating_add(1), + }, + ) + .await; + let replay = wait_for_runtime_custom_plan(&fixture).await; + + assert_eq!(outcome, HlsRuntimeCustomTailOutcome::AlreadyCommitted); + assert_eq!(replay.reason, HlsRuntimeCustomTailReason::LowPriorityPreempted); + assert_eq!(replay.generation, first.generation); + assert_eq!(segment_bytes(&replay, 0), segment_bytes(&first, 0)); +} + +#[tokio::test] +async fn hls_manifest_acceptance_directive_transient_lock_busy_retries_regular_evaluation() { + let mut session = atomic_pressure_session(); + let proxy_session_id = session.proxy_session_id.clone(); + let mut leases = HlsAccessLeaseStore::default(); + install_atomic_pressure_lease( + &mut leases, + &proxy_session_id, + "availability-transient", + pressure_manifest_at(0, 8_000), + 1_000, + ); + let attempts = std::cell::Cell::new(0_usize); + let clock_calls = std::cell::Cell::new(0_usize); + + let access = retry_availability_state_access(|| { + let attempt = attempts.get(); + attempts.set(attempt.saturating_add(1)); + let access = if attempt == 0 { + HlsCriticalHandoffStateAccess::LockBusy + } else { + HlsCriticalHandoffStateAccess::Acquired(evaluate_and_commit_session_recovery_pressure_in_snapshot( + &mut leases, + &mut session, + &proxy_session_id, + atomic_pressure_policy(), + || { + assert_eq!(attempts.get(), 2); + clock_calls.set(clock_calls.get().saturating_add(1)); + 101 + }, + )) + }; + std::future::ready(access) + }) + .await; + + let evidence = availability_snapshot_or_contention(access) + .expect("transient contention must reach the regular snapshot evaluation") + .expect("the active lease supplies recovery evidence"); + assert_eq!(evidence.timing_seed.target_duration_ms, 8_000); + assert_eq!(attempts.get(), 2); + assert_eq!(clock_calls.get(), 1); +} + +#[tokio::test] +async fn hls_manifest_acceptance_directive_exhausted_contention_is_typed() { + let attempts = std::cell::Cell::new(0_usize); + let access: HlsCriticalHandoffStateAccess = retry_availability_state_access(|| { + attempts.set(attempts.get().saturating_add(1)); + std::future::ready(HlsCriticalHandoffStateAccess::LockBusy) + }) + .await; + + let outcome = availability_snapshot_or_contention(access) + .expect_err("exhausted contention must remain a typed endpoint outcome"); + + assert_eq!(attempts.get(), HLS_AVAILABILITY_STATE_ACCESS_ATTEMPTS); + assert_eq!(outcome, HlsAvailabilitySnapshotAccessError::StateContention); +} + +#[test] +fn hls_manifest_acceptance_directive_samples_time_inside_snapshot_scope() { + let mut session = atomic_pressure_session(); + let proxy_session_id = session.proxy_session_id.clone(); + let mut leases = HlsAccessLeaseStore::default(); + install_atomic_pressure_lease( + &mut leases, + &proxy_session_id, + "availability-clock", + pressure_manifest_at(0, 8_000), + 150, + ); + let clock_calls = std::cell::Cell::new(0_usize); + + let evidence = evaluate_and_commit_session_recovery_pressure_in_snapshot( + &mut leases, + &mut session, + &proxy_session_id, + atomic_pressure_policy(), + || { + clock_calls.set(clock_calls.get().saturating_add(1)); + 200 + }, + ); + + assert_eq!(clock_calls.get(), 1); + assert!(evidence.is_none(), "the snapshot-local clock must exclude the lease expired at evaluation time"); +} diff --git a/backend/hls/src/availability/tests/recovery.rs b/backend/hls/src/availability/tests/recovery.rs new file mode 100644 index 000000000..218a42c31 --- /dev/null +++ b/backend/hls/src/availability/tests/recovery.rs @@ -0,0 +1,388 @@ +use super::*; + +#[test] +fn hls_terminal_commit_pending_fallback_preserves_a_retryable_fail_closed_handoff() { + let safe_deadline_ms = 10_000; + let handoff_budget_ms = HlsTerminalCommitAcquisitionBudgetMs::fail_closed_handoff_from_retry_policy().as_millis(); + + assert_eq!( + terminal_pending_fallback_commit_at_ms(safe_deadline_ms), + safe_deadline_ms.saturating_sub(handoff_budget_ms) + ); + assert_eq!( + safe_deadline_ms.saturating_sub(terminal_pending_fallback_commit_at_ms(safe_deadline_ms)), + handoff_budget_ms + ); + assert_eq!(terminal_pending_fallback_commit_at_ms(handoff_budget_ms.saturating_sub(1)), 0); +} + +#[tokio::test] +async fn hls_terminal_commit_pending_ready_completion_selects_tail_without_a_client_retry() { + let coordinator = Arc::new(HlsTerminalPendingCoordinator::default()); + let bundle_key = pending_decision_bundle_key(); + let bundle = pending_decision_ready_bundle(bundle_key); + let (ticket, publisher) = prepared_terminal_bundle_completion_channel_for_test(bundle_key); + let (_fallback_tx, fallback_rx) = oneshot::channel::<()>(); + let (decision_tx, decision_rx) = oneshot::channel(); + let asset_guard = HlsTerminalAssetRevisionGuard::matching_for_test(Some(bundle_key.asset)); + + assert_eq!( + coordinator.register(pending_decision_owner_key(bundle_key), &asset_guard, move |ownership| async move { + let decision = await_terminal_pending_decision(ticket, &ownership, bundle_key, async move { + assert!(fallback_rx.await.is_ok()); + }) + .await; + assert!(decision_tx.send(decision).is_ok()); + },), + HlsTerminalPendingRegistration::Scheduled + ); + publisher.publish(HlsPreparedTerminalBundleCompletion::Ready { bundle: Arc::clone(&bundle) }); + + let decision = decision_rx.await.expect("pending decision completes"); + assert!(matches!(decision, Some(HlsTerminalPendingDecision::Ready(actual)) if Arc::ptr_eq(&actual, &bundle))); + assert_eq!(coordinator.owner_count(), 0); +} + +#[tokio::test] +async fn hls_terminal_commit_pending_fallback_selects_terminal_unavailable_without_a_client_retry() { + let coordinator = Arc::new(HlsTerminalPendingCoordinator::default()); + let bundle_key = pending_decision_bundle_key(); + let (ticket, _publisher) = prepared_terminal_bundle_completion_channel_for_test(bundle_key); + let (fallback_tx, fallback_rx) = oneshot::channel::<()>(); + let (decision_tx, decision_rx) = oneshot::channel(); + let asset_guard = HlsTerminalAssetRevisionGuard::matching_for_test(Some(bundle_key.asset)); + + assert_eq!( + coordinator.register(pending_decision_owner_key(bundle_key), &asset_guard, move |ownership| async move { + let decision = await_terminal_pending_decision(ticket, &ownership, bundle_key, async move { + assert!(fallback_rx.await.is_ok()); + }) + .await; + assert!(decision_tx.send(decision).is_ok()); + },), + HlsTerminalPendingRegistration::Scheduled + ); + assert!(fallback_tx.send(()).is_ok()); + + assert!(matches!( + decision_rx.await, + Ok(Some(HlsTerminalPendingDecision::Unavailable(HlsTerminalTailCompatibility::TerminalMediaNotReady))) + )); + assert_eq!(coordinator.owner_count(), 0); +} + +#[tokio::test] +async fn asset_reload_supersedes_pending_custom_tail_but_not_committed_bytes() { + let fixture = post_refresh_terminal_fixture("runtime-asset-reload", true).await; + let reason = HlsRuntimeCustomTailReason::LowPriorityPreempted; + let old_asset = snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, reason).expect("old low-priority asset"); + let target_duration_ms = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .and_then(|lease| lease.last_manifest_snapshot.map(|manifest| manifest.target_duration_ms)) + .expect("published target duration"); + let old_key = prepared_terminal_bundle_key(&old_asset.asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); + let publisher = fixture + .ctx + .hls_proxy + .install_controlled_terminal_bundle_flight_for_test(old_key) + .expect("controlled old-asset preparation"); + let pending = commit_hls_runtime_custom_tail( + fixture.ctx.clone(), + HlsRuntimeCustomTailRequest { + session: Arc::clone(&fixture.session), + proxy_session_id: fixture.proxy_session_id.clone(), + lease_id: fixture.lease_id.clone(), + reason, + now_ms: fixture.now_ms, + }, + ) + .await; + assert_eq!(pending, HlsRuntimeCustomTailOutcome::PendingOwnerRegistered); + wait_for_terminal_pending_owners(&fixture, 1).await; + + let mut revised_bytes = LOW_PRIORITY_ASSET_BYTES.to_vec(); + *revised_bytes.last_mut().expect("non-empty low-priority asset") ^= 1; + fixture + .ctx + .app_config + .custom_stream_response + .store(Some(runtime_custom_responses_with_low_priority(&revised_bytes))); + let old_bundle = build_prepared_terminal_bundle(&old_asset.asset, old_key).expect("old controlled relative bundle"); + publisher.publish(HlsPreparedTerminalBundleCompletion::Ready { bundle: old_bundle }); + wait_for_terminal_pending_owners(&fixture, 0).await; + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("reloaded lease remains stored"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + + let (_, committed) = commit_runtime_custom_reason(&fixture, reason, true).await; + let committed_zero = segment_bytes(&committed, 0); + fixture.ctx.app_config.custom_stream_response.store(Some(runtime_custom_responses())); + let replay = wait_for_runtime_custom_plan(&fixture).await; + + assert_eq!(replay.asset_identity, committed.asset_identity); + assert_eq!(segment_bytes(&replay, 0), committed_zero); +} + +#[tokio::test(start_paused = true)] +async fn failed_closed_retry_capacity_retains_owner() { + let fixture = post_refresh_terminal_fixture("capacity-retained-owner", false).await; + fixture.ctx.hls_proxy.set_terminal_commit_retry_capacity_for_test(0); + register_real_post_refresh_owner( + &fixture, + super::super::super::refresh::HlsPostRefreshAvailabilityReason::HardManifestFailure, + ) + .await; + assert_availability_owner_registered(&fixture); + for _ in 0..32 { + tokio::task::yield_now().await; + } + assert_eq!( + fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), + 1, + "retry-capacity pressure must not drop the last Availability owner" + ); + + fixture.ctx.hls_proxy.set_terminal_commit_retry_capacity_for_test( + super::super::super::terminal_commit::HLS_TERMINAL_COMMIT_RETRY_CAPACITY, + ); + fixture.ctx.hls_proxy.notify_session_evidence_changed(&fixture.proxy_session_id); + wait_for_availability_owner_completion(&fixture).await; + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("capacity retry resolves the live lease"); + assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); +} + +#[test] +fn hls_recovery_timing_publication_late_with_large_reserve_keeps_evidence_without_starting_burst() { + assert_eq!( + recovery_trigger_source(HlsOriginPathCondition::ProgressExpected, true, false, false), + HlsRecoveryTriggerSource::PublicationLate + ); + let directive = acceptance_directive_for_progress( + publication_late_decision(30_000), + lease_timing_seed(), + None, + HlsRecoveryTriggerDiagnostic::new(HlsRecoveryTriggerSource::PublicationLate), + ); + + assert_eq!(directive.trigger, HlsManifestAcceptanceTrigger::None); + assert_eq!(directive.timing_seed, Some(lease_timing_seed())); +} + +#[test] +fn hls_recovery_timing_publication_late_with_narrow_reserve_keeps_full_burst_pending() { + let directive = acceptance_directive_for_progress( + publication_late_decision(14_000), + lease_timing_seed(), + None, + HlsRecoveryTriggerDiagnostic::new(HlsRecoveryTriggerSource::ReservePressure), + ); + + assert_eq!(directive.trigger, HlsManifestAcceptanceTrigger::RecoveryRequired); + assert_eq!(directive.timing_seed.map(|seed| seed.workload.burst), Some(HlsRecoveryBurstWorkload::FullBurstPending)); + + let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); + let seed = directive.timing_seed.expect("narrow reserve keeps timing evidence"); + let timing = HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput { + started_at_ms: 1_000, + burst_plan: plan, + target_duration_ms: seed.target_duration_ms, + transition_margin: seed.transition_margin, + workload: seed.workload, + observed_latency: HlsObservedRecoveryLatency::default(), + required_terminal_media_key: seed.required_terminal_media_key, + terminal_media_preparation: seed.terminal_media_preparation, + policy: HlsRecoveryTimingPolicy::new( + HlsOperationTimeoutMs::from_millis(3_000), + HlsOperationTimeoutMs::from_millis(30_000), + HlsRecoveryEtaMs::from_millis(3_000), + HlsRecoveryEtaMs::from_millis(13_000), + ), + }); + let mut episode = super::super::super::manifest_acceptance::HlsManifestAcceptanceEpisode::new( + super::super::super::manifest_acceptance::HlsManifestAcceptanceGeneration(1), + 1_000, + plan, + directive.trigger, + &timing, + ); + assert_eq!(episode.required_candidates(), plan.total_candidates()); + episode.record_full_burst_candidates(plan.total_candidates()); + assert!(episode.full_burst_completed); + assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); +} + +#[test] +fn hls_recovery_timing_unbound_candidate_covers_key_and_map_independent_of_old_manifest() { + let unknown = HlsRecoveryWorkloadEnvelope::acceptance_policy().ceiling(); + + assert_eq!(unknown.burst, HlsRecoveryBurstWorkload::FullBurstPending); + assert_eq!(unknown.segment, HlsRecoverySegmentWorkload::Aes128SegmentFetchWithKeyFetch); + assert_eq!(unknown.map, HlsRecoveryMapWorkload::Fetch); +} + +#[test] +fn hls_session_recovery_pressure_selects_required_lease_over_smaller_raw_reserve() { + let smaller_not_required = evaluated_pressure("lease-a", 4_000, &[1_000, 9_000], 2_000); + let larger_required = evaluated_pressure("lease-b", 8_000, &[8_000, 4_000], 5_000); + + assert!(smaller_not_required.reserve.guaranteed_reserve_ms < larger_required.reserve.guaranteed_reserve_ms); + assert!(!smaller_not_required.reserve.recovery_required); + assert!(larger_required.reserve.recovery_required); + let pressure = aggregate_session_recovery_pressure([smaller_not_required, larger_required.clone()]) + .expect("active lease pressure"); + + assert!(pressure.any_recovery_required); + assert!(!pressure.any_cutover_required); + assert_eq!(pressure.controlling.lease_id, larger_required.lease_id); + let seed = acceptance_timing_seed_for_pressure(&pressure.controlling); + assert_eq!(seed.target_duration_ms, 8_000); + assert_eq!(seed.transition_margin.as_millis(), 8_000); +} + +#[test] +fn hls_session_recovery_pressure_tie_break_is_stable_by_lease_id() { + let lease_b = evaluated_pressure("lease-b", 4_000, &[2_000, 2_000], 4_000); + let mut lease_a = lease_b.clone(); + lease_a.lease_id = HlsAccessLeaseId("lease-a".to_string()); + + let forward = aggregate_session_recovery_pressure([lease_b.clone(), lease_a.clone()]).expect("forward pressure"); + let reverse = aggregate_session_recovery_pressure([lease_a, lease_b]).expect("reverse pressure"); + + assert_eq!(forward.controlling.lease_id.0, "lease-a"); + assert_eq!(reverse.controlling.lease_id.0, "lease-a"); +} + +#[test] +fn hls_session_recovery_pressure_cursor_change_is_observed_by_atomic_commit() { + let mut session = atomic_pressure_session(); + let proxy_session_id = session.proxy_session_id.clone(); + let mut leases = HlsAccessLeaseStore::default(); + let lease_id = + install_atomic_pressure_lease(&mut leases, &proxy_session_id, "lease-a", pressure_manifest_at(0, 8_000), 1_000); + let before = evaluate_and_commit_session_recovery_pressure( + &mut leases, + &mut session, + &proxy_session_id, + 100, + atomic_pressure_policy(), + ) + .expect("initial pressure"); + assert!(!before.decision.evaluate_lease_cutovers); + let identity = leases + .response_snapshot(&lease_id, &proxy_session_id, 100) + .and_then(|lease| lease.media_identity()) + .expect("live media identity"); + assert!(leases + .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 2, 101,) + .is_some()); + + let after = evaluate_and_commit_session_recovery_pressure( + &mut leases, + &mut session, + &proxy_session_id, + 101, + atomic_pressure_policy(), + ) + .expect("cursor pressure"); + + assert!(after.decision.evaluate_lease_cutovers); +} + +#[test] +fn hls_session_recovery_pressure_new_urgent_lease_controls_atomic_commit() { + let mut session = atomic_pressure_session(); + let proxy_session_id = session.proxy_session_id.clone(); + let mut leases = HlsAccessLeaseStore::default(); + install_atomic_pressure_lease(&mut leases, &proxy_session_id, "lease-a", pressure_manifest_at(0, 8_000), 1_000); + let before = evaluate_and_commit_session_recovery_pressure( + &mut leases, + &mut session, + &proxy_session_id, + 100, + atomic_pressure_policy(), + ) + .expect("initial pressure"); + assert!(!before.decision.evaluate_lease_cutovers); + install_atomic_pressure_lease(&mut leases, &proxy_session_id, "lease-b", pressure_manifest_at(2, 10_000), 1_000); + + let after = evaluate_and_commit_session_recovery_pressure( + &mut leases, + &mut session, + &proxy_session_id, + 101, + atomic_pressure_policy(), + ) + .expect("urgent pressure"); + + assert!(after.decision.evaluate_lease_cutovers); + assert_eq!(after.timing_seed.target_duration_ms, 10_000); +} + +#[test] +fn hls_session_recovery_pressure_expired_controller_is_excluded_atomically() { + let mut session = atomic_pressure_session(); + let proxy_session_id = session.proxy_session_id.clone(); + let mut leases = HlsAccessLeaseStore::default(); + install_atomic_pressure_lease(&mut leases, &proxy_session_id, "lease-a", pressure_manifest_at(0, 8_000), 1_000); + install_atomic_pressure_lease(&mut leases, &proxy_session_id, "lease-b", pressure_manifest_at(2, 10_000), 150); + let urgent = evaluate_and_commit_session_recovery_pressure( + &mut leases, + &mut session, + &proxy_session_id, + 100, + atomic_pressure_policy(), + ) + .expect("urgent pressure"); + assert_eq!(urgent.timing_seed.target_duration_ms, 10_000); + + let after_expiry = evaluate_and_commit_session_recovery_pressure( + &mut leases, + &mut session, + &proxy_session_id, + 200, + atomic_pressure_policy(), + ) + .expect("remaining pressure"); + + assert_eq!(after_expiry.timing_seed.target_duration_ms, 8_000); + assert!(!after_expiry.decision.evaluate_lease_cutovers); +} + +#[test] +fn hls_session_recovery_pressure_new_publication_lateness_uses_current_reserve_evidence() { + let mut session = atomic_pressure_session(); + session.origin_control.path_condition = HlsOriginPathCondition::ProgressExpected; + session.origin_control.last_media_progress_at_ms = Some(0); + let proxy_session_id = session.proxy_session_id.clone(); + let mut leases = HlsAccessLeaseStore::default(); + install_atomic_pressure_lease(&mut leases, &proxy_session_id, "lease-a", pressure_manifest_at(0, 8_000), 20_000); + let policy = HlsRecoveryPressurePolicy { + burst_plan: shared::model::HlsManifestRecoveryBurstPlan { slots: 1, lanes_per_slot: 1 }, + timing: HlsRecoveryTimingPolicy::new( + HlsOperationTimeoutMs::from_millis(1_000), + HlsOperationTimeoutMs::from_millis(10_000), + HlsRecoveryEtaMs::from_millis(0), + HlsRecoveryEtaMs::from_millis(2_000), + ), + }; + + let pressure = + evaluate_and_commit_session_recovery_pressure(&mut leases, &mut session, &proxy_session_id, 12_000, policy) + .expect("publication-late pressure"); + + assert!(pressure.decision.start_acceptance_episode); + assert!(pressure.decision.close_admission); + assert!(!pressure.decision.evaluate_lease_cutovers); +} diff --git a/backend/hls/src/availability/tests/runtime_fixtures.rs b/backend/hls/src/availability/tests/runtime_fixtures.rs new file mode 100644 index 000000000..4305fb8e9 --- /dev/null +++ b/backend/hls/src/availability/tests/runtime_fixtures.rs @@ -0,0 +1,345 @@ +use super::*; + +pub(super) async fn prepare_runtime_custom_bundle( + fixture: &PostRefreshTerminalFixture, + reason: HlsRuntimeCustomTailReason, +) { + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("runtime custom-tail fixture lease"); + let target_duration_ms = + lease.last_manifest_snapshot.as_ref().expect("published runtime custom-tail manifest").target_duration_ms; + let asset = + snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, reason).expect("configured runtime custom-tail asset"); + let key = prepared_terminal_bundle_key(&asset.asset, target_duration_ms, HLS_TERMINAL_TAIL_SEGMENT_COUNT); + let state = fixture.ctx.hls_proxy.start_prepared_terminal_bundle( + Arc::clone(&asset.asset), + target_duration_ms, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, + ); + let state = match state { + HlsPreparedTerminalBundleState::Preparing { .. } => fixture + .ctx + .hls_proxy + .wait_for_prepared_terminal_bundle(key) + .await + .expect("runtime custom-tail bundle completion"), + state => state, + }; + assert!( + matches!(state, HlsPreparedTerminalBundleState::Ready { ref bundle } if bundle.matches_key_and_shape(key)), + "runtime custom-tail bundle must be READY: {state:?}" + ); +} + +pub(super) async fn wait_for_runtime_custom_plan(fixture: &PostRefreshTerminalFixture) -> Arc { + let completed = tokio::time::timeout(Duration::from_secs(10), async { + loop { + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("runtime custom-tail lease remains stored"); + if let HlsLeasePlaybackMode::TerminalTail(plan) = lease.playback_mode { + return plan; + } + tokio::task::yield_now().await; + } + }) + .await; + if let Ok(plan) = completed { + return plan; + } + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await; + let state = lease.as_ref().map_or("missing", |lease| lease.state.as_log_value()); + let playback = lease.as_ref().map_or("missing", |lease| match lease.playback_mode { + HlsLeasePlaybackMode::Live => "live", + HlsLeasePlaybackMode::TerminalTail(_) => "terminal-tail", + HlsLeasePlaybackMode::TerminalUnavailable { .. } => "terminal-unavailable", + HlsLeasePlaybackMode::Ended => "ended", + }); + panic!( + "runtime custom-tail owner deadline: state={state} playback={playback} owners={}", + fixture.ctx.hls_proxy.terminal_pending().owner_count() + ); +} + +pub(super) async fn commit_runtime_custom_reason( + fixture: &PostRefreshTerminalFixture, + reason: HlsRuntimeCustomTailReason, + prewarm: bool, +) -> (HlsRuntimeCustomTailOutcome, Arc) { + if prewarm { + prepare_runtime_custom_bundle(fixture, reason).await; + } + let outcome = commit_hls_runtime_custom_tail( + fixture.ctx.clone(), + HlsRuntimeCustomTailRequest { + session: Arc::clone(&fixture.session), + proxy_session_id: fixture.proxy_session_id.clone(), + lease_id: fixture.lease_id.clone(), + reason, + now_ms: fixture.now_ms, + }, + ) + .await; + assert!(matches!( + outcome, + HlsRuntimeCustomTailOutcome::Committed + | HlsRuntimeCustomTailOutcome::AlreadyCommitted + | HlsRuntimeCustomTailOutcome::PendingOwnerRegistered + )); + let plan = wait_for_runtime_custom_plan(fixture).await; + (outcome, plan) +} + +pub(super) fn configured_runtime_custom_buffer( + fixture: &PostRefreshTerminalFixture, + reason: HlsRuntimeCustomTailReason, +) -> TransportStreamBuffer { + let responses = fixture.ctx.app_config.custom_stream_response.load_full().expect("runtime custom responses"); + match reason { + HlsRuntimeCustomTailReason::ChannelUnavailable => responses.channel_unavailable.as_ref(), + HlsRuntimeCustomTailReason::LowPriorityPreempted => responses.low_priority_preempted.as_ref(), + HlsRuntimeCustomTailReason::UserConnectionsExhausted => responses.user_connections_exhausted.as_ref(), + HlsRuntimeCustomTailReason::ProviderConnectionsExhausted => responses.provider_connections_exhausted.as_ref(), + HlsRuntimeCustomTailReason::UserAccountExpired => responses.user_account_expired.as_ref(), + HlsRuntimeCustomTailReason::SessionOrLeaseExpired => responses.hls_session_or_lease_expired.as_ref(), + } + .expect("reason-specific runtime buffer") + .clone() +} + +pub(super) fn segment_bytes(plan: &HlsTerminalTailPlan, index: u16) -> Bytes { + plan.segment_bytes(HlsTerminalSegmentPath { generation: plan.generation, index }) + .expect("committed immutable custom-tail segment") +} + +pub(super) fn payload_continuity_bounds(bytes: &[u8]) -> std::collections::HashMap { + let mut payload_bounds = std::collections::HashMap::::new(); + let mut first_packet_discontinuity = std::collections::HashMap::::new(); + for packet in bytes.as_chunks::<188>().0.iter().filter(|packet| packet[0] == 0x47) { + let adaptation_field_control = (packet[3] >> 4) & 0b11; + let pid = (u16::from(packet[1] & 0x1f) << 8) | u16::from(packet[2]); + let discontinuity = matches!(adaptation_field_control, 0b10 | 0b11) + && packet[4] > 0 + && packet.get(5).is_some_and(|flags| flags & 0x80 != 0); + first_packet_discontinuity.entry(pid).or_insert(discontinuity); + if !matches!(adaptation_field_control, 0b01 | 0b11) { + continue; + } + let counter = packet[3] & 0x0f; + payload_bounds.entry(pid).and_modify(|entry| entry.1 = counter).or_insert((counter, counter)); + } + payload_bounds + .into_iter() + .map(|(pid, (first, last))| { + let discontinuity = first_packet_discontinuity.get(&pid).copied().unwrap_or(false); + (pid, (first, last, discontinuity)) + }) + .collect() +} + +pub(super) fn with_internal_payload_continuity_jump(bytes: &[u8]) -> Vec { + let mut corrupted = bytes.to_vec(); + let mut first_payload_pid = None; + for packet in corrupted.as_chunks_mut::<188>().0.iter_mut().filter(|packet| packet[0] == 0x47) { + let adaptation_field_control = (packet[3] >> 4) & 0b11; + if !matches!(adaptation_field_control, 0b01 | 0b11) { + continue; + } + let pid = (u16::from(packet[1] & 0x1f) << 8) | u16::from(packet[2]); + if pid == 0x1fff { + continue; + } + match first_payload_pid { + None => first_payload_pid = Some(pid), + Some(first_pid) if first_pid == pid => { + let continuity_counter = packet[3] & 0x0f; + packet[3] = (packet[3] & 0xf0) | (continuity_counter.wrapping_add(3) & 0x0f); + return corrupted; + } + Some(_) => {} + } + } + panic!("terminal fixture must contain two payload packets for one PID"); +} + +pub(super) async fn assert_active_policy_reason_commits(name: &str, reason: HlsRuntimeCustomTailReason) { + let fixture = post_refresh_terminal_fixture(name, true).await; + let (_, plan) = commit_runtime_custom_reason(&fixture, reason, true).await; + assert_eq!(plan.reason, reason); + assert_eq!(plan.segment_duration_ms, 10_027); + assert_eq!( + plan.asset_identity, + HlsRuntimeCustomTailAssetIdentity::from_asset( + &snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, reason).expect("reason-specific configured asset") + ) + ); +} + +pub(super) async fn wait_for_terminal_pending_owners(fixture: &PostRefreshTerminalFixture, expected: usize) { + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if fixture.ctx.hls_proxy.terminal_pending().owner_count() == expected { + return; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap_or_else(|_| { + panic!( + "terminal-pending owner deadline: actual={} expected={expected}", + fixture.ctx.hls_proxy.terminal_pending().owner_count() + ) + }); +} + +pub(super) async fn add_active_post_refresh_lease( + fixture: &PostRefreshTerminalFixture, + lease_id: HlsAccessLeaseId, + active_map: Option, +) { + let lease = HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("multi-lease", &lease_id.0), + fixture.proxy_session_id.clone(), + "multi-lease".to_string(), + lease_id.0.clone(), + 1, + "stream".to_string(), + 1, + fixture.now_ms, + 60_000, + ); + fixture.ctx.hls_proxy.prepare_access_lease(lease).await; + let publication = fixture + .ctx + .hls_proxy + .prepare_access_lease_manifest_publication(&lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("second lease publication guard"); + let mut manifest = pressure_manifest(12_000); + manifest.active_map = active_map; + Arc::make_mut(&mut manifest.visible_segments)[0].uri = + format!("/hls/shared/live/{}/{}/0.ts", fixture.proxy_session_id.0, lease_id.0).into(); + assert!(fixture + .ctx + .hls_proxy + .commit_access_lease_manifest_publication( + &lease_id, + &fixture.proxy_session_id, + publication, + manifest, + fixture.now_ms, + ) + .await + .is_committed()); + assert!(fixture + .ctx + .hls_proxy + .activate_access_lease( + &lease_id, + &fixture.proxy_session_id, + fixture.now_ms, + HlsAccessLeaseTiming { active_window_ms: 60_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); +} + +pub(super) async fn advance_post_refresh_fixture_playback( + ctx: &HlsCtx, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + playback_elapsed_ms: u64, + now_ms: u64, +) { + let lease = ctx + .hls_proxy + .access_lease_response_snapshot(lease_id, proxy_session_id, now_ms) + .await + .expect("live terminal fixture remains available"); + let identity = lease.media_identity().expect("live terminal fixture identity"); + let playback_at_ms = now_ms.saturating_sub(playback_elapsed_ms); + for proxy_seq in [0_u64, 1] { + let token = ctx + .hls_proxy + .record_access_lease_segment_request_started_if_identity_matches( + lease_id, + proxy_session_id, + identity, + proxy_seq, + playback_at_ms, + ) + .await + .expect("fixture segment request starts"); + assert_eq!( + ctx.hls_proxy + .record_access_lease_segment_request_completed_and_mark_media_if_identity_matches( + session, + lease_id, + proxy_session_id, + identity, + token, + playback_at_ms, + ) + .await, + super::super::super::manager::HlsMediaActivityCommitOutcome::Committed + ); + } +} + +pub(super) async fn assert_multi_lease_fallback_handles_pending_and_unavailable(reverse_insertion: bool) { + let fixture_name = if reverse_insertion { "multi-lease-fallback-reverse" } else { "multi-lease-fallback-forward" }; + let fixture = post_refresh_terminal_fixture_with_bundle_state(fixture_name, true, true, false).await; + let incompatible_lease_id = HlsAccessLeaseId("multi-lease-incompatible".to_string()); + add_active_post_refresh_lease( + &fixture, + incompatible_lease_id.clone(), + Some(HlsMapSignature { fingerprint: [0x5a; 32], container: HlsMediaContainer::FragmentedMp4 }), + ) + .await; + if reverse_insertion { + let mut leases = fixture.ctx.hls_proxy.hold_access_lease_store_for_test().await; + let primary = leases.remove_access_lease(&fixture.lease_id).expect("primary live lease remains stored"); + leases.prepare_access_lease(primary); + } + let asset = snapshot_terminal_media_asset(&TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec())) + .expect("controlled terminal asset parses"); + let bundle_key = prepared_terminal_bundle_key(&asset, 12_000, HLS_TERMINAL_TAIL_SEGMENT_COUNT); + let _controlled_flight = fixture + .ctx + .hls_proxy + .install_controlled_terminal_bundle_flight_for_test(bundle_key) + .expect("controlled terminal preparation is unique"); + let evaluation_now_ms = fixture.ctx.hls_proxy.terminal_commit_now_ms(); + + let aggregate = + evaluate_owner_failure_fallback(&fixture.ctx, &fixture.session, &fixture.proxy_session_id, evaluation_now_ms) + .await; + + assert_eq!(aggregate.total, 2); + assert_eq!(aggregate.pending_owned, 1); + assert_eq!(aggregate.terminal_committed, 1, "{aggregate:?}"); + assert!(aggregate.unresolved.is_empty()); + assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 1); + let unavailable = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&incompatible_lease_id, &fixture.proxy_session_id, evaluation_now_ms) + .await + .expect("incompatible lease remains stored"); + assert!(matches!(unavailable.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); + fixture.ctx.hls_proxy.terminal_pending().cancel_session(&fixture.proxy_session_id); +} diff --git a/backend/hls/src/availability/tests/storage.rs b/backend/hls/src/availability/tests/storage.rs new file mode 100644 index 000000000..b36b62d75 --- /dev/null +++ b/backend/hls/src/availability/tests/storage.rs @@ -0,0 +1,37 @@ +use super::*; + +#[tokio::test(start_paused = true)] +async fn persistent_conflict_real_owner_commits_before_exclusive_deadline() { + let fixture = post_refresh_terminal_fixture_with_progress("real-owner-terminal", true, false).await; + register_real_post_refresh_owner( + &fixture, + super::super::super::refresh::HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, + ) + .await; + assert_availability_owner_registered(&fixture); + + tokio::time::advance(Duration::from_millis(2_100)).await; + assert_eq!( + fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), + 1, + "the real owner must survive its rapid evaluation budget while reserve remains" + ); + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + 20_100, + fixture.now_ms, + ) + .await; + wait_for_availability_owner_completion(&fixture).await; + + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("terminal lease remains stored"); + assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalTail(_))); +} diff --git a/backend/hls/src/availability/tests/terminal.rs b/backend/hls/src/availability/tests/terminal.rs new file mode 100644 index 000000000..98642ace8 --- /dev/null +++ b/backend/hls/src/availability/tests/terminal.rs @@ -0,0 +1,414 @@ +use super::*; + +#[tokio::test] +async fn disabled_custom_responses_do_not_seed_terminal_media_preparation() { + let hls_ctx = crate::HlsCtx::for_test(Config { custom_stream_response_enabled: false, ..Config::default() }); + let ctx = &hls_ctx; + ctx.app_config.custom_stream_response.store(Some(runtime_custom_responses())); + + assert_eq!(terminal_media_timing_seed(ctx, 10_000), (None, HlsTerminalMediaPreparationState::Failed { key: None })); +} + +#[test] +fn hls_terminal_commit_outcomes_never_turn_failures_into_live_serving() { + for (outcome, reason) in [ + (HlsTerminalCommitOutcome::BundleNotReady, HlsTerminalFailedClosedReason::BundleNotReadyWithoutOwner), + (HlsTerminalCommitOutcome::BundleIncompatible, HlsTerminalFailedClosedReason::BundleIncompatible), + (HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed, HlsTerminalFailedClosedReason::SafeCommitDeadlineElapsed), + (HlsTerminalCommitOutcome::RetryCapacityExceeded, HlsTerminalFailedClosedReason::RetryCapacityExceeded), + (HlsTerminalCommitOutcome::RetryAttemptsExhausted, HlsTerminalFailedClosedReason::RetryAttemptsExhausted), + (HlsTerminalCommitOutcome::RetryWorkerUnavailable, HlsTerminalFailedClosedReason::RuntimeUnavailable), + ] { + assert_eq!( + terminal_resolution_for_commit_outcome(outcome, 1_000), + HlsTerminalResolution::FailedClosed { reason } + ); + } + assert_eq!( + terminal_resolution_for_commit_outcome(HlsTerminalCommitOutcome::LockBusy { retry_before_ms: 1_025 }, 1_000,), + HlsTerminalResolution::Pending { retry_after_ms: 25 } + ); +} + +#[test] +fn autonomous_terminal_live_allowed_means_no_cutover_is_required() { + assert_eq!( + classify_autonomous_terminal_resolution(HlsTerminalResolution::LiveAllowed), + HlsAutonomousTerminalObservation::NoCutoverRequired, + ); +} + +#[test] +fn live_reserve_wake_is_strictly_before_safe_terminal_deadline() { + let now_ms = 1_000; + let mut reserve = terminal_pending_commit_reserve(); + reserve.guaranteed_reserve_ms = reserve.guaranteed_reserve_ms.saturating_add(5_000); + reserve.guaranteed_media_horizon_ms = reserve.guaranteed_reserve_ms; + let cutover_timing = + HlsLeaseCutoverTiming::from_reserve(now_ms, reserve.guaranteed_reserve_ms, reserve.transition_margin, None); + + let deadline = live_reserve_deadline(now_ms, reserve, cutover_timing).expect("future acquisition wake"); + assert_eq!(deadline.next_reevaluation_at_ms, now_ms.saturating_add(5_000)); + assert!(deadline.next_reevaluation_at_ms < deadline.latest_safe_terminal_commit_at_ms); +} + +#[tokio::test] +async fn hls_terminal_commit_pending_owner_store_ready_completion_commits_terminal_tail() { + let fixture = terminal_pending_commit_fixture("pending-owner-ready-store").await; + let bundle = fixture.ready_bundle(); + let (ticket, publisher) = prepared_terminal_bundle_completion_channel_for_test(fixture.bundle_key); + let completed = fixture.register_owner(ticket); + + publisher.publish(HlsPreparedTerminalBundleCompletion::Ready { bundle }); + completed.await.expect("productive pending owner completes"); + + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("terminal lease remains stored"); + assert!(matches!( + lease.playback_mode, + HlsLeasePlaybackMode::TerminalTail(ref plan) + if plan.generation.0 == fixture.preparation.decision_generation + )); + assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 0); +} + +#[tokio::test] +async fn missing_terminal_base_timestamp_profile_commits_terminal_unavailable() { + let base_bytes = terminal_base_without_timestamps(); + let fixture = terminal_pending_commit_fixture_with_base("pending-owner-missing-timestamp", &base_bytes).await; + let bundle = fixture.ready_bundle(); + let (ticket, publisher) = prepared_terminal_bundle_completion_channel_for_test(fixture.bundle_key); + let completed = fixture.register_owner(ticket); + + publisher.publish(HlsPreparedTerminalBundleCompletion::Ready { bundle }); + completed.await.expect("terminal owner completes fail-closed decision"); + + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("terminal unavailable lease remains stored"); + assert!(matches!( + lease.playback_mode, + HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::MissingTimestampAnchor, .. } + )); +} + +#[tokio::test(start_paused = true)] +async fn hls_terminal_commit_pending_owner_store_fallback_commits_terminal_unavailable() { + let fixture = terminal_pending_commit_fixture("pending-owner-fallback-store").await; + let (ticket, _publisher) = prepared_terminal_bundle_completion_channel_for_test(fixture.bundle_key); + let completed = fixture.register_owner(ticket); + + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_millis(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis())) + .await; + completed.await.expect("productive fallback owner completes"); + + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("terminal lease remains stored"); + assert!(matches!( + lease.playback_mode, + HlsLeasePlaybackMode::TerminalUnavailable { + decision_generation, + reason: HlsTerminalTailCompatibility::TerminalMediaNotReady, + } if decision_generation == fixture.preparation.decision_generation + )); + assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 0); +} + +#[tokio::test] +async fn hls_terminal_commit_pending_owner_store_progress_supersession_keeps_lease_live() { + let fixture = terminal_pending_commit_fixture("pending-owner-progress-store").await; + let bundle = fixture.ready_bundle(); + let (ticket, publisher) = prepared_terminal_bundle_completion_channel_for_test(fixture.bundle_key); + let completed = fixture.register_owner(ticket); + tokio::task::yield_now().await; + + { + let mut session = fixture.session.write().await; + session.origin_control.progress_generation = session.origin_control.progress_generation.saturating_add(1); + session.origin_control.last_media_progress_at_ms = Some(fixture.now_ms.saturating_add(1)); + } + fixture.ctx.hls_proxy.cancel_superseded_terminal_work_for_session(&fixture.proxy_session_id); + publisher.publish(HlsPreparedTerminalBundleCompletion::Ready { bundle }); + completed.await.expect("cancelled pending owner completes"); + + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("superseded lease remains stored"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + assert!(!fixture.session.read().await.has_terminal_tail_protections()); + assert_eq!(fixture.ctx.hls_proxy.terminal_pending().owner_count(), 0); +} + +#[tokio::test] +async fn terminal_pending_runtime_failure_commits_terminal_unavailable() { + assert_terminal_pending_registration_failure_commits_unavailable( + HlsTerminalPendingRegistration::RuntimeUnavailable, + "pending-runtime-failure", + ) + .await; +} + +#[tokio::test] +async fn active_hls_preemption_commits_low_priority_preempted_tail_without_redirect() { + let fixture = post_refresh_terminal_fixture("runtime-preemption", true).await; + + let (_, plan) = + commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted, true).await; + let manifest = terminal_tail_manifest_body(&plan, &fixture.proxy_session_id, &fixture.lease_id) + .expect("preemption plan route binding"); + + assert_eq!(plan.reason, HlsRuntimeCustomTailReason::LowPriorityPreempted); + assert!(manifest.ends_with("#EXT-X-ENDLIST\n")); + assert!(!manifest.contains("/cvs/hls/")); + assert!(matches!( + fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("committed preemption lease") + .playback_mode, + HlsLeasePlaybackMode::TerminalTail(_) + )); +} + +#[tokio::test] +async fn unsafe_live_transport_evidence_commits_unavailable_without_terminal_bytes() { + let fixture = post_refresh_terminal_fixture("runtime-unsafe-live-splice", true).await; + prepare_runtime_custom_bundle(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted).await; + let cache_key = fixture.session.read().await.segments.get(&0).expect("terminal-base segment").cache_key.clone(); + let metadata = fixture + .ctx + .hls_proxy + .segment_cache() + .metadata(&cache_key) + .await + .expect("cache metadata lookup") + .expect("terminal-base metadata"); + tokio::fs::write(&metadata.path, with_internal_payload_continuity_jump(TERMINAL_ASSET_BYTES)) + .await + .expect("replace test fixture with same-size unsafe bytes"); + + let outcome = commit_hls_runtime_custom_tail( + fixture.ctx.clone(), + HlsRuntimeCustomTailRequest { + session: Arc::clone(&fixture.session), + proxy_session_id: fixture.proxy_session_id.clone(), + lease_id: fixture.lease_id.clone(), + reason: HlsRuntimeCustomTailReason::LowPriorityPreempted, + now_ms: fixture.now_ms, + }, + ) + .await; + wait_for_terminal_pending_owners(&fixture, 0).await; + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("unsafe splice lease remains stored"); + + assert_eq!(outcome, HlsRuntimeCustomTailOutcome::PendingOwnerRegistered); + assert!(matches!( + lease.playback_mode, + HlsLeasePlaybackMode::TerminalUnavailable { + reason: HlsTerminalTailCompatibility::SpliceTransportFailure( + super::super::super::HlsTsSpliceIncompatibility::ContinuityFailure { .. } + ), + .. + } + )); + assert!(fixture.session.read().await.terminal_tail_protection(&fixture.lease_id).is_none()); +} + +#[tokio::test] +async fn preemption_tail_uses_low_priority_asset_not_channel_unavailable_asset() { + let fixture = post_refresh_terminal_fixture("runtime-preemption-asset", true).await; + let low_priority = + snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, HlsRuntimeCustomTailReason::LowPriorityPreempted) + .expect("low-priority asset"); + let channel = snapshot_hls_runtime_custom_tail_asset(&fixture.ctx, HlsRuntimeCustomTailReason::ChannelUnavailable) + .expect("channel-unavailable asset"); + + let (_, plan) = + commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted, true).await; + + assert_eq!(plan.asset_identity, HlsRuntimeCustomTailAssetIdentity::from_asset(&low_priority)); + assert_ne!(plan.asset_identity.media, HlsRuntimeCustomTailAssetIdentity::from_asset(&channel).media); +} + +#[tokio::test] +async fn preemption_tail_preserves_live_to_custom_pts_dts_pcr_and_cc() { + let fixture = post_refresh_terminal_fixture("runtime-preemption-splice", true).await; + let low_priority = configured_runtime_custom_buffer(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted); + let live = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); + let expected_anchor = HlsTsSpliceAnchor::between( + live.finite_hls_timestamp_profile().expect("live timestamp profile"), + low_priority.finite_hls_timestamp_profile().expect("custom timestamp profile"), + ) + .expect("compatible live-to-custom splice"); + + let (_, plan) = + commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted, true).await; + let first = segment_bytes(&plan, 0); + let second = segment_bytes(&plan, 1); + let first_profile = TransportStreamBuffer::new(first.to_vec()) + .finite_hls_timestamp_profile() + .expect("first anchored custom profile"); + let second_profile = TransportStreamBuffer::new(second.to_vec()) + .finite_hls_timestamp_profile() + .expect("second anchored custom profile"); + assert_eq!(first_profile.first_clock_90khz, expected_anchor.terminal_first_clock); + assert!(first_profile.observed_pts_or_dts && first_profile.observed_pcr); + assert!(second_profile.observed_pts_or_dts && second_profile.observed_pcr); + assert_eq!( + second_profile.first_clock_90khz.wrapping_add(1_u64 << 33).wrapping_sub(first_profile.first_clock_90khz) + % (1_u64 << 33), + 902_400 + ); + let first_cc = payload_continuity_bounds(&first); + let second_cc = payload_continuity_bounds(&second); + assert!(!first_cc.is_empty()); + assert!(first_cc.iter().all(|(pid, (_, _, discontinuity))| *pid == 0x1fff || *discontinuity), "{first_cc:?}"); + for (pid, (_, last, _)) in first_cc { + let (next, _, _) = second_cc.get(&pid).expect("PID continues into second custom segment"); + assert_eq!(*next, last.wrapping_add(1) & 0x0f, "PID {pid} continuity"); + } +} + +#[tokio::test] +async fn preemption_tail_is_committed_at_safe_segment_boundary_without_waiting_for_reserve_cutover() { + let fixture = post_refresh_terminal_fixture_with_progress("runtime-preemption-immediate", true, false).await; + let lease_before = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("live lease before immediate cutover"); + assert_eq!(lease_before.playback_mode, HlsLeasePlaybackMode::Live); + + let (outcome, plan) = + commit_runtime_custom_reason(&fixture, HlsRuntimeCustomTailReason::LowPriorityPreempted, true).await; + + assert!(matches!( + outcome, + HlsRuntimeCustomTailOutcome::Committed | HlsRuntimeCustomTailOutcome::PendingOwnerRegistered + )); + assert_eq!(plan.base_manifest.last_proxy_seq, lease_before.last_manifest_snapshot.unwrap().last_proxy_seq); +} + +#[tokio::test] +async fn active_provider_exhaustion_commits_provider_exhausted_tail_after_grace() { + assert_active_policy_reason_commits( + "runtime-provider-exhausted", + HlsRuntimeCustomTailReason::ProviderConnectionsExhausted, + ) + .await; +} + +#[tokio::test] +async fn active_user_exhaustion_commits_user_exhausted_tail() { + assert_active_policy_reason_commits("runtime-user-exhausted", HlsRuntimeCustomTailReason::UserConnectionsExhausted) + .await; +} + +#[tokio::test] +async fn active_user_account_expiry_commits_account_expired_tail() { + assert_active_policy_reason_commits("runtime-account-expired", HlsRuntimeCustomTailReason::UserAccountExpired) + .await; +} + +#[tokio::test] +async fn replay_conflict_commits_prepared_terminal_tail_before_safe_deadline() { + let fixture = post_refresh_terminal_fixture("post-refresh-terminal", true).await; + let safe_deadline = + HlsLeaseCutoverTiming::from_reserve(fixture.now_ms, 12_900, HlsTransitionMarginMs::from_millis(12_000), None) + .latest_safe_terminal_commit_at + .as_millis_since_epoch(); + + let evaluation = evaluate_active_terminal_leases_for_reevaluation( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + fixture.now_ms, + ) + .await; + + assert_eq!(evaluation, HlsPostRefreshTerminalEvaluation::TerminalCommitted); + assert!(fixture.now_ms < safe_deadline); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("terminal lease remains available"); + assert!( + matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalTail(_)), + "prepared compatible asset must commit a terminal tail: {:?}", + lease.playback_mode + ); + assert_eq!( + commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &lease, + safe_deadline.saturating_add(1), + ) + .await, + HlsTerminalResolution::Committed, + "a later client observation sees the immutable terminal decision" + ); +} + +#[tokio::test] +async fn incompatible_terminal_asset_commits_terminal_unavailable_without_client_request() { + let fixture = post_refresh_terminal_fixture("post-refresh-unavailable", false).await; + + let evaluation = evaluate_active_terminal_leases_for_reevaluation( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + fixture.now_ms, + ) + .await; + + assert_eq!(evaluation, HlsPostRefreshTerminalEvaluation::TerminalCommitted); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("terminal-unavailable lease remains available"); + assert!(matches!( + lease.playback_mode, + HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::MissingAsset, .. } + )); + assert_eq!( + commit_terminal_tail_if_lease_reserve_requires_cutover( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &lease, + fixture.now_ms.saturating_add(60_000), + ) + .await, + HlsTerminalResolution::Committed, + "a later client observation cannot reopen safe-deadline failure" + ); +} diff --git a/backend/hls/src/availability/tests/terminal_fixtures.rs b/backend/hls/src/availability/tests/terminal_fixtures.rs new file mode 100644 index 000000000..6d0167694 --- /dev/null +++ b/backend/hls/src/availability/tests/terminal_fixtures.rs @@ -0,0 +1,407 @@ +use super::*; + +impl HlsTerminalPendingCommitFixture { + pub(super) fn owner_key(&self) -> HlsTerminalPendingOwnerKey { + HlsTerminalPendingOwnerKey { + session_incarnation: self + .ctx + .hls_proxy + .sessions() + .session_incarnation(&self.session) + .expect("fixture session has a current incarnation"), + proxy_session_id: self.proxy_session_id.clone(), + lease_id: self.lease_id.clone(), + lease_issued_at_ms: self.preparation.lease_issued_at_ms, + expected_admission_generation: self.preparation.expected_admission_generation, + manifest_snapshot_generation: self.preparation.manifest_snapshot_generation, + cursor_generation: self.preparation.cursor_generation, + decision_generation: self.preparation.decision_generation, + reason: self.expected_asset.reason, + bundle_key: self.bundle_key, + latest_safe_commit_at_ms: self + .preparation + .cutover_timing + .latest_safe_terminal_commit_at + .as_millis_since_epoch(), + } + } + + pub(super) fn ready_bundle(&self) -> Arc { + build_prepared_terminal_bundle(&self.asset, self.bundle_key).expect("fixture relative terminal bundle") + } + + pub(super) fn register_owner(&self, ticket: HlsPreparedTerminalBundleCompletionTicket) -> oneshot::Receiver<()> { + let coordinator = self.ctx.hls_proxy.terminal_pending(); + let owner_key = self.owner_key(); + let ctx = self.ctx.clone(); + let session = Arc::clone(&self.session); + let proxy_session_id = self.proxy_session_id.clone(); + let lease_id = self.lease_id.clone(); + let preparation = self.preparation.clone(); + let asset = Arc::clone(&self.asset); + let expected_asset = self.expected_asset; + let bundle_key = self.bundle_key; + let (completed_tx, completed_rx) = oneshot::channel(); + let asset_guard = terminal_asset_revision_guard(&self.ctx, expected_asset.reason, Some(expected_asset)); + + assert_eq!( + coordinator.register(owner_key, &asset_guard, move |ownership| async move { + run_terminal_pending_owner( + ctx, + session, + proxy_session_id, + lease_id, + preparation, + asset, + expected_asset, + bundle_key, + ticket, + ownership, + ) + .await; + assert!(completed_tx.send(()).is_ok()); + }), + HlsTerminalPendingRegistration::Scheduled + ); + completed_rx + } +} + +pub(super) fn terminal_pending_commit_reserve() -> HlsLeaseReserveSnapshot { + let transition_margin = HlsTransitionMarginMs::from_millis(12_000); + let guaranteed_reserve_ms = transition_margin + .as_millis() + .saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()); + HlsLeaseReserveSnapshot { + availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, + guaranteed_media_horizon_ms: guaranteed_reserve_ms, + conservative_playback_position_ms: 0, + guaranteed_reserve_ms, + initial_hidden_ready_duration_ms: 0, + transition_margin, + key_readiness_valid_until_ms: None, + recovery_required: true, + cutover_required: false, + } +} + +pub(super) fn terminal_pending_commit_manifest( + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + duration_ms: u64, +) -> HlsLeaseManifestSnapshot { + HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(1), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 0, + delivered_at_ms: 0, + first_proxy_seq: 40, + last_proxy_seq: 40, + visible_segments: Arc::from([HlsLeaseManifestSegment { + proxy_seq: 40, + duration_ms, + uri: format!("/hls/shared/live/{}/{}/40.ts", proxy_session_id.0, lease_id.0).into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }]), + discontinuity_sequence: 0, + target_duration_ms: 12_000, + playlist_duration_ms: duration_ms, + last_visible_media_end_ms: duration_ms, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + } +} + +pub(super) async fn terminal_pending_commit_fixture(name: &str) -> HlsTerminalPendingCommitFixture { + terminal_pending_commit_fixture_with_base(name, TERMINAL_ASSET_BYTES).await +} + +pub(super) async fn install_terminal_pending_ready_base( + ctx: &HlsCtx, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + asset: &super::super::super::terminal_tail::HlsTerminalMediaAsset, + base_segment_bytes: &[u8], + now_ms: u64, +) { + let cache_key = SegmentCacheKey::new(proxy_session_id.clone(), 40, "ts"); + ctx.hls_proxy + .segment_cache() + .write_bytes_and_commit(&cache_key, base_segment_bytes) + .await + .expect("READY terminal-base bytes commit"); + let mut session = session.write().await; + let origin_epoch = session.origin_control.origin_epoch; + session.segments.insert( + 40, + SegmentEntry { + origin_key: OriginSegmentKey { + origin_epoch, + effective_host_id: 1, + host_local_sequence: 40, + host_local_index: 0, + }, + proxy_seq: 40, + duration_ms: asset.duration_ms(), + proxy_file_ext: "ts".to_string(), + content_type: "video/mp2t".to_string(), + cache_key, + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + encryption: None, + origin_fetch_ref: None, + status: SegmentCacheStatus::Ready { + content_length: u64::try_from(base_segment_bytes.len()).unwrap_or(u64::MAX), + ready_at_ms: now_ms, + }, + last_rendered_at_ms: Some(now_ms), + access: Arc::new(CacheAccessState::new()), + }, + ); +} + +pub(super) async fn publish_terminal_pending_lease( + ctx: &HlsCtx, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + name: &str, + duration_ms: u64, + now_ms: u64, +) { + ctx.hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("pending-owner", name), + proxy_session_id.clone(), + "pending-owner".to_string(), + name.to_string(), + 1, + name.to_string(), + 1, + now_ms, + 60_000, + )) + .await; + let publication = ctx + .hls_proxy + .prepare_access_lease_manifest_publication(lease_id, proxy_session_id, now_ms) + .await + .expect("manifest publication guard"); + assert!(ctx + .hls_proxy + .commit_access_lease_manifest_publication( + lease_id, + proxy_session_id, + publication, + terminal_pending_commit_manifest(proxy_session_id, lease_id, duration_ms), + now_ms, + ) + .await + .is_committed()); +} + +pub(super) async fn terminal_pending_commit_fixture_with_base( + name: &str, + base_segment_bytes: &[u8], +) -> HlsTerminalPendingCommitFixture { + let hls_ctx = crate::HlsCtx::for_test(Config { custom_stream_response_enabled: true, ..Config::default() }); + let ctx = &hls_ctx; + let terminal_buffer = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); + let asset = snapshot_terminal_media_asset(&terminal_buffer).expect("valid terminal asset fixture"); + ctx.app_config.custom_stream_response.store(Some(runtime_custom_responses())); + let now_ms = ctx.hls_proxy.terminal_commit_now_ms(); + let (session, _) = + ctx.hls_proxy.get_or_create_session_with_outcome(HlsSessionKey::new(1, name), b"secret", now_ms).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId(name.to_string()); + install_terminal_pending_ready_base(ctx, &session, &proxy_session_id, &asset, base_segment_bytes, now_ms).await; + publish_terminal_pending_lease(ctx, &proxy_session_id, &lease_id, name, asset.duration_ms(), now_ms).await; + let (origin_progress_generation, media_readiness_generation, last_media_progress_at_ms) = { + let session = session.read().await; + ( + session.origin_control.progress_generation, + session.activity.media_readiness_generation, + session.origin_control.last_media_progress_at_ms, + ) + }; + let reserve = terminal_pending_commit_reserve(); + let cutover_timing = + HlsLeaseCutoverTiming::from_reserve(now_ms, reserve.guaranteed_reserve_ms, reserve.transition_margin, None); + let preparation = ctx + .hls_proxy + .prepare_access_lease_terminal_tail(HlsTerminalTailPreparationRequest { + lease_id: &lease_id, + proxy_session_id: &proxy_session_id, + manifest_snapshot_generation: 1, + cursor_generation: 0, + reserve, + cutover_timing, + commit_window: HlsTerminalCommitWindow::AcquisitionOpen, + now_ms, + origin_progress_generation, + media_readiness_generation, + last_media_progress_at_ms, + }) + .await + .expect("cutover-local terminal preparation"); + let expected_asset = + HlsRuntimeCustomTailAssetIdentity::channel_unavailable(HlsTerminalAssetIdentity::from_asset(&asset)); + let bundle_key = prepared_terminal_bundle_key( + &asset, + preparation.manifest_snapshot.target_duration_ms, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, + ); + + HlsTerminalPendingCommitFixture { + ctx: ctx.clone(), + session, + proxy_session_id, + lease_id, + preparation, + asset, + expected_asset, + bundle_key, + now_ms, + } +} + +pub(super) fn terminal_base_without_timestamps() -> Vec { + let mut bytes = TERMINAL_ASSET_BYTES.to_vec(); + for packet in bytes.as_chunks_mut::<188>().0 { + let adaptation_field_control = (packet[3] >> 4) & 0b11; + if matches!(adaptation_field_control, 0b10 | 0b11) && packet[4] > 0 { + packet[5] &= !0x10; + } + if packet[1] & 0x40 == 0 { + continue; + } + let payload_offset = match adaptation_field_control { + 0b01 => 4, + 0b11 => 5usize.saturating_add(usize::from(packet[4])), + _ => continue, + }; + let Some(payload) = packet.get_mut(payload_offset..) else { + continue; + }; + if payload.len() >= 9 && payload.starts_with(&[0x00, 0x00, 0x01]) { + payload[7] &= 0x3F; + } + } + bytes +} + +pub(super) async fn assert_terminal_pending_registration_failure_commits_unavailable( + failure: HlsTerminalPendingRegistration, + name: &str, +) { + let fixture = terminal_pending_commit_fixture(name).await; + let resolution = terminal_resolution_for_pending_registration( + HlsTerminalCommitContext { + ctx: &fixture.ctx, + session: &fixture.session, + proxy_session_id: &fixture.proxy_session_id, + lease_id: &fixture.lease_id, + preparation: &fixture.preparation, + now_ms: fixture.now_ms, + }, + fixture.expected_asset, + fixture.preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), + failure, + ); + + assert_eq!(resolution, HlsTerminalResolution::Committed); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("terminal unavailable lease remains stored"); + assert!(matches!( + lease.playback_mode, + HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::TerminalMediaNotReady, .. } + )); +} + +pub(super) async fn assert_post_refresh_registration_failure_leaves_no_unowned_live_lease( + failure: HlsAvailabilityReevaluationRegistration, + name: &str, +) { + let fixture = terminal_pending_commit_fixture(name).await; + let (origin_progress_generation, media_readiness_generation) = { + let mut session = fixture.session.write().await; + session.origin_control.path_condition = HlsOriginPathCondition::AcceptanceConflict; + let origin_epoch = session.origin_control.origin_epoch; + for proxy_seq in 41_u64..=44 { + session.segments.insert( + proxy_seq, + SegmentEntry { + origin_key: OriginSegmentKey { + origin_epoch, + effective_host_id: 1, + host_local_sequence: proxy_seq, + host_local_index: u32::try_from(proxy_seq.saturating_sub(40)).unwrap_or(u32::MAX), + }, + proxy_seq, + duration_ms: 20_000, + proxy_file_ext: "ts".to_string(), + content_type: "video/mp2t".to_string(), + cache_key: SegmentCacheKey::new(fixture.proxy_session_id.clone(), proxy_seq, "ts"), + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + encryption: None, + origin_fetch_ref: None, + status: SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: fixture.now_ms }, + last_rendered_at_ms: None, + access: Arc::new(CacheAccessState::new()), + }, + ); + } + (session.origin_control.progress_generation, session.activity.media_readiness_generation) + }; + assert!(fixture + .ctx + .hls_proxy + .activate_access_lease( + &fixture.lease_id, + &fixture.proxy_session_id, + fixture.now_ms, + HlsAccessLeaseTiming { active_window_ms: 60_000, valid_window_ms: 60_000 }, + ) + .await + .is_activated()); + let outcome = commit_post_refresh_terminal_fallback( + fixture.ctx.clone(), + Arc::clone(&fixture.session), + HlsPostRefreshAvailabilityAction::Reevaluate { + reason: super::super::super::refresh::HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, + origin_progress_generation, + media_readiness_generation, + }, + failure, + ) + .await; + + assert_eq!(outcome, HlsPostRefreshFallbackOutcome::TerminalCommitted); + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("fallback lease remains stored"); + assert!(matches!( + lease.playback_mode, + HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::TerminalMediaNotReady, .. } + )); +} diff --git a/backend/hls/src/availability/tests/transport.rs b/backend/hls/src/availability/tests/transport.rs new file mode 100644 index 000000000..f5fc81a14 --- /dev/null +++ b/backend/hls/src/availability/tests/transport.rs @@ -0,0 +1,71 @@ +use super::*; + +#[test] +fn availability_refresh_waits_for_completion_instead_of_polling_in_flight() { + for outcome in [HlsOriginRefreshTriggerOutcome::Started, HlsOriginRefreshTriggerOutcome::InFlight] { + assert_eq!( + availability_refresh_trigger_decision(outcome, true), + HlsAvailabilityRefreshTriggerDecision::Wait(HlsAvailabilityAttemptSchedule::RefreshCompletion) + ); + } + assert_eq!(HlsAvailabilityAttemptSchedule::RefreshCompletion.wake_at_ms(100, 1, 2_100), Some(2_100)); +} + +#[test] +fn availability_refresh_waits_until_concrete_debounce_boundary() { + let schedule = HlsAvailabilityAttemptSchedule::DebouncedUntil { retry_at_ms: 1_100 }; + assert_eq!( + availability_refresh_trigger_decision( + HlsOriginRefreshTriggerOutcome::DebouncedUntil { retry_at_ms: 1_100 }, + true, + ), + HlsAvailabilityRefreshTriggerDecision::Wait(schedule) + ); + assert_eq!(schedule.wake_at_ms(100, 1, 2_100), Some(1_100)); + assert_eq!( + HlsAvailabilityAttemptSchedule::DebouncedUntil { retry_at_ms: 3_000 }.wake_at_ms(100, 1, 2_100), + Some(2_100) + ); +} + +#[tokio::test(start_paused = true)] +async fn hard_failure_real_owner_commits_unavailable_before_exclusive_deadline() { + let fixture = post_refresh_terminal_fixture_with_progress("real-owner-unavailable", false, false).await; + register_real_post_refresh_owner( + &fixture, + super::super::super::refresh::HlsPostRefreshAvailabilityReason::HardManifestFailure, + ) + .await; + assert_availability_owner_registered(&fixture); + + tokio::time::advance(Duration::from_millis(2_100)).await; + assert_eq!(fixture.ctx.hls_proxy.availability_reevaluations().owner_count(), 1); + // Playback evidence is relative to the clock as it stands now, not to the + // fixture's base: the advance above really does move the scheduling clock. + advance_post_refresh_fixture_playback( + &fixture.ctx, + &fixture.session, + &fixture.proxy_session_id, + &fixture.lease_id, + 7_000, + fixture.ctx.hls_proxy.now_ms(), + ) + .await; + wait_for_availability_owner_completion(&fixture).await; + + let lease = fixture + .ctx + .hls_proxy + .access_lease_response_snapshot(&fixture.lease_id, &fixture.proxy_session_id, fixture.now_ms) + .await + .expect("unavailable lease remains stored"); + assert!(matches!( + lease.playback_mode, + HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::MissingAsset, .. } + )); +} + +#[tokio::test(start_paused = true)] +async fn multi_lease_fallback_keeps_pending_owner_and_commits_other_unavailable() { + assert_multi_lease_fallback_handles_pending_and_unavailable(false).await; +} diff --git a/backend/hls/src/cache.rs b/backend/hls/src/cache.rs index 129580c21..48099555f 100644 --- a/backend/hls/src/cache.rs +++ b/backend/hls/src/cache.rs @@ -1,29 +1,18 @@ use super::{safe_proxy_session_id, ProxyMapId, ProxySessionId, TransientResourceId}; -use futures::future::BoxFuture; -use log::warn; use std::{ collections::{HashMap, HashSet}, - fmt, - future::Future, - io, - path::{Path, PathBuf}, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, Mutex as StdMutex, RwLock as StdRwLock, Weak, - }, - time::SystemTime, -}; -use tokio::{ - fs::{self, File, OpenOptions}, - io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt, SeekFrom}, - sync::{Mutex as AsyncMutex, Notify, RwLock, Semaphore}, - time::{timeout_at, Instant}, + path::PathBuf, + sync::{atomic::AtomicU64, Arc, Mutex as StdMutex, RwLock as StdRwLock, Weak}, }; +use tokio::sync::{Mutex as AsyncMutex, Notify, RwLock, Semaphore}; pub const DEFAULT_HLS_CACHE_PATH: &str = "/tmp/tuliprox/cache/hls"; + const TEMP_CREATE_ATTEMPTS: usize = 8; + const MAX_CONCURRENT_OWNED_CACHE_OPERATIONS: usize = 64; -const MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN: usize = 128; + +pub(crate) const MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN: usize = 128; /// Stable cache key for one proxy-visible HLS segment. #[derive(Clone, Eq, PartialEq, Hash)] @@ -33,28 +22,6 @@ pub struct SegmentCacheKey { file_ext: String, } -impl SegmentCacheKey { - pub fn new(proxy_session_id: ProxySessionId, proxy_seq: u64, proxy_file_ext: impl Into) -> Self { - Self { session_id: proxy_session_id, seq: proxy_seq, file_ext: proxy_file_ext.into() } - } - - pub fn stable_value(&self) -> String { format!("hls:{}:{:020}", self.session_id.0, self.seq) } - - pub fn proxy_session_id(&self) -> &ProxySessionId { &self.session_id } - - pub fn proxy_seq(&self) -> u64 { self.seq } -} - -impl fmt::Debug for SegmentCacheKey { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SegmentCacheKey") - .field("session_id", &"") - .field("seq", &self.seq) - .field("file_ext", &self.file_ext) - .finish() - } -} - /// Stable cache key for one proxy-visible HLS EXT-X-MAP object. #[derive(Clone, Eq, PartialEq, Hash)] pub struct MapCacheKey { @@ -63,30 +30,6 @@ pub struct MapCacheKey { file_ext: String, } -impl MapCacheKey { - pub fn new( - proxy_session_id: ProxySessionId, - proxy_map_id: impl Into, - proxy_file_ext: impl Into, - ) -> Self { - Self { session_id: proxy_session_id, map_id: proxy_map_id.into(), file_ext: proxy_file_ext.into() } - } - - pub fn stable_value(&self) -> String { format!("hls-map:{}:{:020}", self.session_id.0, self.map_id.0) } - - pub fn proxy_map_id(&self) -> ProxyMapId { self.map_id } -} - -impl fmt::Debug for MapCacheKey { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("MapCacheKey") - .field("session_id", &"") - .field("map_id", &self.map_id.0) - .field("file_ext", &self.file_ext) - .finish() - } -} - /// Stable cache key for one demand-cached transient passthrough object. /// /// This key is not a provider-source identity. It keys the proxy-visible object for a concrete transient resource. @@ -99,86 +42,6 @@ pub struct TransientObjectCacheKey { file_ext: String, } -impl TransientObjectCacheKey { - pub fn new( - proxy_session_id: ProxySessionId, - transient_resource_id: TransientResourceId, - proxy_file_ext: impl Into, - ) -> Self { - Self { session_id: proxy_session_id, resource_id: transient_resource_id, file_ext: proxy_file_ext.into() } - } - - pub fn stable_value(&self) -> String { format!("hls-transient:{}:{}", self.session_id.0, self.resource_id.0) } - - pub fn transient_resource_id(&self) -> &TransientResourceId { &self.resource_id } -} - -impl fmt::Debug for TransientObjectCacheKey { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("TransientObjectCacheKey") - .field("session_id", &"") - .field("resource_id", &self.resource_id) - .field("file_ext", &self.file_ext) - .finish() - } -} - -/// Resolves a proxy cache object to a safe path below the configured HLS cache root. -pub trait HlsCacheObjectKey { - fn proxy_session_id(&self) -> &ProxySessionId; - fn session_path_component(&self) -> String; - fn file_name(&self) -> String; -} - -impl HlsCacheObjectKey for SegmentCacheKey { - fn proxy_session_id(&self) -> &ProxySessionId { &self.session_id } - - fn session_path_component(&self) -> String { - let value = &self.session_id.0; - if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) { - return value.clone(); - } - blake3::hash(value.as_bytes()).to_hex().to_string() - } - - fn file_name(&self) -> String { format!("{:06}.{}", self.seq, self.file_ext) } -} - -impl HlsCacheObjectKey for MapCacheKey { - fn proxy_session_id(&self) -> &ProxySessionId { &self.session_id } - - fn session_path_component(&self) -> String { - let value = &self.session_id.0; - if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) { - return value.clone(); - } - blake3::hash(value.as_bytes()).to_hex().to_string() - } - - fn file_name(&self) -> String { format!("map/{:06}.{}", self.map_id.0, self.file_ext) } -} - -impl HlsCacheObjectKey for TransientObjectCacheKey { - fn proxy_session_id(&self) -> &ProxySessionId { &self.session_id } - - fn session_path_component(&self) -> String { - let value = &self.session_id.0; - if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) { - return value.clone(); - } - blake3::hash(value.as_bytes()).to_hex().to_string() - } - - fn file_name(&self) -> String { format!("r/{}.{}", self.resource_id.0, self.file_ext) } -} - -/// Filesystem metadata for a committed HLS cache object. -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct CachedSegmentMetadata { - pub path: PathBuf, - pub size: u64, -} - #[derive(Clone)] pub struct StagedCacheObject { pub path: PathBuf, @@ -187,37 +50,6 @@ pub struct StagedCacheObject { _registration: Arc, } -impl StagedCacheObject { - fn registered( - path: PathBuf, - size: u64, - cache_path_generation: Arc, - registration: Arc, - ) -> Self { - Self { path, size, cache_path_generation, _registration: registration } - } -} - -impl fmt::Debug for StagedCacheObject { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("StagedCacheObject") - .field("path", &self.path) - .field("size", &self.size) - .finish_non_exhaustive() - } -} - -impl PartialEq for StagedCacheObject { - fn eq(&self, other: &Self) -> bool { - self.path == other.path - && self.size == other.size - && Arc::ptr_eq(&self.cache_path_generation, &other.cache_path_generation) - } -} - -impl Eq for StagedCacheObject {} - /// Typed source used to distinguish a decoded-object size violation from a filesystem failure. #[derive(Debug, thiserror::Error)] #[error("hls cache object exceeds configured size limit {limit}")] @@ -225,24 +57,6 @@ pub struct HlsCacheObjectLimitError { limit: u64, } -impl HlsCacheObjectLimitError { - pub fn limit(&self) -> u64 { self.limit } -} - -pub fn hls_cache_object_limit_from_io(error: &io::Error) -> Option<&HlsCacheObjectLimitError> { - let mut source: &(dyn std::error::Error + 'static) = error.get_ref()?; - loop { - if let Some(limit_error) = source.downcast_ref::() { - return Some(limit_error); - } - source = source.source()?; - } -} - -fn cache_object_limit_error(limit: u64) -> io::Error { - io::Error::new(io::ErrorKind::InvalidData, HlsCacheObjectLimitError { limit }) -} - /// Revision of cache accounting or of the protected working set. /// /// Deferred segment work waits for this opaque token to become stale before it is @@ -251,17 +65,6 @@ fn cache_object_limit_error(limit: u64) -> io::Error { #[derive(Clone)] pub struct HlsCacheCapacityRevision(Arc); -impl fmt::Debug for HlsCacheCapacityRevision { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("HlsCacheCapacityRevision()") - } -} - -impl HlsCacheCapacityRevision { - #[cfg(any(test, feature = "test-support"))] - pub fn for_test() -> Self { Self(Arc::new(CapacityRevision)) } -} - /// Typed local budget deferral kept distinct from filesystem capacity failures. #[derive(Debug, Clone, thiserror::Error)] #[error( @@ -280,90 +83,6 @@ pub struct HlsCacheCapacityError { revision: HlsCacheCapacityRevision, } -impl HlsCacheCapacityError { - pub fn configured_session_bytes(&self) -> u64 { self.configured_session_bytes } - - pub fn configured_global_bytes(&self) -> u64 { self.configured_global_bytes } - - pub fn current_session_bytes(&self) -> u64 { self.current_session_bytes } - - pub fn current_global_bytes(&self) -> u64 { self.current_global_bytes } - - pub fn staged_bytes(&self) -> u64 { self.staged_bytes } - - pub fn required_session_bytes(&self) -> u64 { self.required_session_bytes } - - pub fn required_global_bytes(&self) -> u64 { self.required_global_bytes } - - pub fn revision(&self) -> &HlsCacheCapacityRevision { &self.revision } - - #[cfg(any(test, feature = "test-support"))] - pub fn protected_working_set_bytes(&self) -> u64 { self.protected_working_set_bytes } - - #[cfg(any(test, feature = "test-support"))] - pub fn reclaimable_bytes(&self) -> u64 { self.reclaimable_bytes } - - fn pressure(&self) -> CacheCapacityPressure { - CacheCapacityPressure { - configured_session_bytes: self.configured_session_bytes, - configured_global_bytes: self.configured_global_bytes, - current_session_bytes: self.current_session_bytes, - current_global_bytes: self.current_global_bytes, - staged_bytes: self.staged_bytes, - required_session_bytes: self.required_session_bytes, - required_global_bytes: self.required_global_bytes, - } - } -} - -pub fn hls_cache_capacity_from_io(error: &io::Error) -> Option<&HlsCacheCapacityError> { - let mut source: &(dyn std::error::Error + 'static) = error.get_ref()?; - loop { - if let Some(capacity_error) = source.downcast_ref::() { - return Some(capacity_error); - } - source = source.source()?; - } -} - -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct HlsCacheCapacityReclaimRequest { - pub proxy_session_id: ProxySessionId, - pub target_path: PathBuf, - pub required_session_bytes: u64, - pub required_global_bytes: u64, - pub staged_bytes: u64, -} - -#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] -#[allow(clippy::struct_field_names)] -pub struct HlsCacheCapacityReclaimOutcome { - pub reclaimed_session_bytes: u64, - pub reclaimed_global_bytes: u64, - pub protected_working_set_bytes: u64, - pub reclaimable_bytes: u64, -} - -#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] -pub struct HlsCacheCapacityUsage { - pub session_bytes: u64, - pub global_bytes: u64, -} - -/// Existing GC integration used to make room for one already-staged cache object. -pub trait HlsCacheCapacityReclaimer: Send + Sync { - fn reclaim_capacity( - &self, - request: HlsCacheCapacityReclaimRequest, - ) -> BoxFuture<'_, io::Result>; -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum CacheInvalidationOutcome { - Invalidated, - DeferredActiveTempFiles, -} - /// File-backed cache for committed HLS segment objects. /// /// Lock order: the capacity-admission gate serializes authoritative admission, projected reclamation, and the following @@ -405,6 +124,9 @@ struct CacheCapacityState { session_bytes: HashMap, revision: Arc, active_mutations: HashSet, + revision_reservations: HashMap, + /// Revision file bytes found by the last completed usage scan. + revision_bytes: u64, } #[derive(Default)] @@ -437,10 +159,6 @@ struct CapacityInvalidationGuard { changed: Arc, } -impl Drop for CapacityInvalidationGuard { - fn drop(&mut self) { invalidate_capacity_state(&self.capacity, &self.changed); } -} - impl Drop for CachePathDeletionReservation { fn drop(&mut self) { self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner).deletion_reservations.remove(&self.path); @@ -457,130 +175,6 @@ struct CapacityMutationReservation { changed: Arc, } -impl CapacityMutationReservation { - fn projected_usage(&self) -> HlsCacheCapacityUsage { - let capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - HlsCacheCapacityUsage { - session_bytes: capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(), - global_bytes: capacity.total_bytes, - } - } - - fn reserve_replacement( - &mut self, - old_size: u64, - new_size: u64, - max_cache_bytes: u64, - max_session_bytes: u64, - ) -> Result<(), CapacityReservationError> { - let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if !capacity.initialized || capacity.cache_path != self.cache_path { - return Err(CapacityReservationError::Retry); - } - if !capacity.active_mutations.contains(&self.path) { - return Err(CapacityReservationError::Invalidated); - } - let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); - let projected_total = capacity.total_bytes.saturating_sub(old_size).saturating_add(new_size); - let projected_session = session_size.saturating_sub(old_size).saturating_add(new_size); - if projected_total > max_cache_bytes || projected_session > max_session_bytes { - return Err(CapacityReservationError::Exceeded { - pressure: CacheCapacityPressure { - configured_session_bytes: max_session_bytes, - configured_global_bytes: max_cache_bytes, - current_session_bytes: session_size, - current_global_bytes: capacity.total_bytes, - staged_bytes: new_size, - required_session_bytes: projected_session.saturating_sub(max_session_bytes), - required_global_bytes: projected_total.saturating_sub(max_cache_bytes), - }, - // Capture the opaque token in the same critical section as the - // failed admission decision. A later accounting mutation must - // make this token stale instead of being missed by the waiter. - revision: HlsCacheCapacityRevision(Arc::clone(&capacity.revision)), - }); - } - capacity.total_bytes = projected_total; - store_session_bytes(&mut capacity, &self.session_component, projected_session); - self.replacement = Some((old_size, new_size)); - Ok(()) - } - - fn mark_filesystem_mutation_started(&mut self) { self.filesystem_mutation_started = true; } - - fn finish_replacement(mut self) { - self.replacement = None; - self.finish_mutation(None); - } - - fn finish_delete(mut self, deleted_size: u64) { - self.finish_mutation(Some(deleted_size)); - self.replacement = None; - } - - fn finish_mutation(&mut self, deleted_size: Option) { - let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if !capacity.active_mutations.contains(&self.path) { - return; - } - capacity.active_mutations.remove(&self.path); - if capacity.initialized && capacity.cache_path == self.cache_path { - if let Some(deleted_size) = deleted_size { - capacity.total_bytes = capacity.total_bytes.saturating_sub(deleted_size); - let session_bytes = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); - store_session_bytes(&mut capacity, &self.session_component, session_bytes.saturating_sub(deleted_size)); - } - } - capacity.revision = Arc::new(CapacityRevision); - drop(capacity); - self.changed.notify_waiters(); - } -} - -impl Drop for CapacityMutationReservation { - fn drop(&mut self) { - let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if !capacity.active_mutations.contains(&self.path) { - return; - } - capacity.active_mutations.remove(&self.path); - let revision_changed = if self.filesystem_mutation_started { - capacity.initialized = false; - true - } else if capacity.initialized && capacity.cache_path == self.cache_path { - if let Some((old_size, new_size)) = self.replacement.take() { - capacity.total_bytes = capacity.total_bytes.saturating_sub(new_size).saturating_add(old_size); - let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); - store_session_bytes( - &mut capacity, - &self.session_component, - session_size.saturating_sub(new_size).saturating_add(old_size), - ); - true - } else { - false - } - } else { - false - }; - if revision_changed { - capacity.revision = Arc::new(CapacityRevision); - } - drop(capacity); - // Mutation waiters share this notifier with revision waiters. A no-op - // release must wake the former, while the latter observe the unchanged - // opaque token and continue waiting. - self.changed.notify_waiters(); - } -} - -#[derive(Debug, Clone)] -enum CapacityReservationError { - Retry, - Invalidated, - Exceeded { pressure: CacheCapacityPressure, revision: HlsCacheCapacityRevision }, -} - #[derive(Debug, Clone, Copy, Eq, PartialEq)] #[allow(clippy::struct_field_names)] struct CacheCapacityPressure { @@ -593,2004 +187,42 @@ struct CacheCapacityPressure { required_global_bytes: u64, } -fn store_session_bytes(capacity: &mut CacheCapacityState, session_component: &str, bytes: u64) { - if bytes == 0 { - capacity.session_bytes.remove(session_component); - } else { - capacity.session_bytes.insert(session_component.to_string(), bytes); - } -} - -impl HlsSegmentCache { - pub fn new() -> Self { Self::with_cache_path(DEFAULT_HLS_CACHE_PATH) } - - pub fn with_cache_path(cache_path: impl Into) -> Self { - Self { - cache_path: StdRwLock::new(CachePathState { - path: cache_path.into(), - generation: Arc::new(CachePathGeneration), - }), - cache_path_commit_gate: Arc::new(RwLock::new(())), - temp_files: Arc::new(StdMutex::new(TempFileState::default())), - max_object_bytes: AtomicU64::new(u64::MAX), - max_cache_bytes: AtomicU64::new(u64::MAX), - max_session_bytes: AtomicU64::new(u64::MAX), - marker_path: StdRwLock::new(None), - capacity: Arc::new(StdMutex::new(CacheCapacityState::default())), - capacity_changed: Arc::new(Notify::new()), - capacity_reclaimer: StdRwLock::new(None), - capacity_admission_gate: AsyncMutex::new(()), - owned_operation_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_OWNED_CACHE_OPERATIONS)), - } - } - - pub fn install_capacity_reclaimer(&self, reclaimer: &Arc) - where - T: HlsCacheCapacityReclaimer + 'static, - { - let reclaimer: Arc = reclaimer.clone(); - *self.capacity_reclaimer.write().unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(Arc::downgrade(&reclaimer)); - } - - #[cfg(any(test, feature = "test-support"))] - pub fn capacity_revision(&self) -> HlsCacheCapacityRevision { - let capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - HlsCacheCapacityRevision(Arc::clone(&capacity.revision)) - } - - /// Waits until accounting or playback protection has changed since a - /// capacity deferral. Registering with `Notify` before comparing tokens - /// prevents a missed wake between the failed commit and this wait. - pub async fn wait_for_capacity_change(&self, revision: &HlsCacheCapacityRevision) { - loop { - let notified = self.capacity_changed.notified(); - let changed = { - let capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - !Arc::ptr_eq(&capacity.revision, &revision.0) - }; - if changed { - return; - } - notified.await; - } - } - - /// Announces a cursor/window change which may release protected cache - /// objects. No accounting totals are modified by this operation. - pub fn notify_capacity_protection_changed(&self) { - let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - capacity.revision = Arc::new(CapacityRevision); - drop(capacity); - self.capacity_changed.notify_waiters(); - } - - pub fn cache_path(&self) -> PathBuf { self.cache_path_snapshot().path } - - pub async fn update_cache_path(&self, cache_path: impl Into) -> bool { - let cache_path = cache_path.into(); - // Commits hold a read lease from their final generation check through the atomic rename. Taking the write - // lease makes a cache-root transition linearizable without holding a filesystem/accounting mutex over I/O. - let _transition = self.cache_path_commit_gate.write().await; - let mut current = self.cache_path.write().unwrap_or_else(std::sync::PoisonError::into_inner); - if current.path == cache_path { - return false; - } - current.path = cache_path; - current.generation = Arc::new(CachePathGeneration); - drop(current); - *self.marker_path.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; - self.invalidate_capacity_accounting(); - true - } - - pub fn update_cache_limits(&self, max_cache_bytes: u64, max_session_bytes: u64) { - let max_cache_bytes = max_cache_bytes.max(1); - let max_session_bytes = max_session_bytes.max(1); - self.max_cache_bytes.store(max_cache_bytes, Ordering::Release); - self.max_session_bytes.store(max_session_bytes, Ordering::Release); - self.max_object_bytes.store(max_cache_bytes.min(max_session_bytes), Ordering::Release); - } - - pub async fn metadata(&self, key: &K) -> io::Result> { - let path = self.path_for_key(key); - match fs::metadata(&path).await { - Ok(metadata) => Ok(Some(CachedSegmentMetadata { path, size: metadata.len() })), - Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), - Err(err) => Err(err), - } - } - - /// Performs capacity admission from reliable response metadata before the - /// decoded body is consumed. The authoritative staged commit repeats the - /// same reservation after download, so concurrent mutations cannot create - /// an overshoot. - pub async fn ensure_projected_write_capacity(&self, key: &K, content_length: u64) -> io::Result<()> - where - K: HlsCacheObjectKey, - { - let max_object_bytes = self.max_object_bytes.load(Ordering::Acquire); - if content_length > max_object_bytes { - let configured_session_bytes = self.max_session_bytes.load(Ordering::Acquire); - let configured_global_bytes = self.max_cache_bytes.load(Ordering::Acquire); - let usage = self.capacity_usage(key.proxy_session_id()).await.unwrap_or_default(); - log::warn!( - "HLS cache capacity decision: proxy_session={} resource={} configured_session_bytes={} configured_global_bytes={} current_session_bytes={} current_global_bytes={} staged_bytes={} protected_working_set_bytes=0 reclaimable_bytes=0 required_session_bytes={} required_global_bytes={} outcome=permanently-infeasible", - safe_proxy_session_id(key.proxy_session_id()), - key.file_name(), - configured_session_bytes, - configured_global_bytes, - usage.session_bytes, - usage.global_bytes, - content_length, - content_length.saturating_sub(configured_session_bytes), - content_length.saturating_sub(configured_global_bytes), - ); - return Err(cache_object_limit_error(max_object_bytes)); - } - - let _admission_gate = self.capacity_admission_gate.lock().await; - let mut reclamation_attempted = false; - let mut reclamation_outcome = HlsCacheCapacityReclaimOutcome::default(); - let mut reclamation_pressure: Option = None; - loop { - let cache_path = self.cache_path_snapshot(); - let final_path = Self::path_for_key_in(&cache_path.path, key); - match self - .prepare_capacity_replacement( - &cache_path.path, - &final_path, - key.session_path_component(), - content_length, - self.max_cache_bytes.load(Ordering::Acquire), - self.max_session_bytes.load(Ordering::Acquire), - ) - .await - { - Ok(reservation) => { - drop(reservation); - if let Some(pressure) = reclamation_pressure { - let usage = self.capacity_usage(key.proxy_session_id()).await.unwrap_or_default(); - log::info!( - "HLS cache capacity decision: proxy_session={} resource={} configured_session_bytes={} configured_global_bytes={} current_session_bytes={} current_global_bytes={} staged_bytes={} protected_working_set_bytes={} reclaimable_bytes={} required_session_bytes={} required_global_bytes={} outcome=reclaimed", - safe_proxy_session_id(key.proxy_session_id()), - key.file_name(), - pressure.configured_session_bytes, - pressure.configured_global_bytes, - usage.session_bytes, - usage.global_bytes, - content_length, - reclamation_outcome.protected_working_set_bytes, - reclamation_outcome.reclaimable_bytes, - pressure.required_session_bytes, - pressure.required_global_bytes, - ); - } - return Ok(()); - } - Err(error) => { - let Some(capacity) = hls_cache_capacity_from_io(&error) else { - return Err(error); - }; - if reclamation_attempted { - log::warn!( - "HLS cache capacity decision: proxy_session={} resource={} configured_session_bytes={} configured_global_bytes={} current_session_bytes={} current_global_bytes={} staged_bytes={} protected_working_set_bytes={} reclaimable_bytes={} required_session_bytes={} required_global_bytes={} outcome=deferred-protected wake_reason=capacity-or-protection-revision", - safe_proxy_session_id(key.proxy_session_id()), - key.file_name(), - capacity.configured_session_bytes(), - capacity.configured_global_bytes(), - capacity.current_session_bytes(), - capacity.current_global_bytes(), - capacity.staged_bytes(), - reclamation_outcome.protected_working_set_bytes, - reclamation_outcome.reclaimable_bytes, - capacity.required_session_bytes(), - capacity.required_global_bytes(), - ); - return Err(capacity_error( - capacity.pressure(), - reclamation_outcome, - capacity.revision().clone(), - )); - } - reclamation_attempted = true; - reclamation_pressure = Some(capacity.pressure()); - reclamation_outcome = self - .reclaim_capacity(HlsCacheCapacityReclaimRequest { - proxy_session_id: key.proxy_session_id().clone(), - target_path: final_path, - required_session_bytes: capacity.required_session_bytes(), - required_global_bytes: capacity.required_global_bytes(), - staged_bytes: content_length, - }) - .await?; - } - } - } - } - - pub async fn open_range(&self, key: &K, start: u64) -> io::Result { - let mut file = File::open(self.path_for_key(key)).await?; - file.seek(SeekFrom::Start(start)).await?; - Ok(file) - } - - pub async fn write_temp_and_commit(&self, key: &K, mut reader: R) -> io::Result - where - K: HlsCacheObjectKey, - R: AsyncRead + Unpin, - { - self.write_temp_and_commit_inner(key, &mut reader, None).await - } - - pub async fn write_temp_and_commit_with_deadline( - &self, - key: &K, - mut reader: R, - deadline: Instant, - ) -> io::Result - where - K: HlsCacheObjectKey, - R: AsyncRead + Unpin, - { - self.write_temp_and_commit_inner(key, &mut reader, Some(deadline)).await - } - - pub async fn stage_temp_with_deadline( - &self, - key: &K, - mut reader: R, - deadline: Instant, - ) -> io::Result - where - K: HlsCacheObjectKey, - R: AsyncRead + Unpin, - { - self.stage_temp_inner(key, &mut reader, Some(deadline)).await - } - - pub async fn commit_staged(&self, key: &K, staged: StagedCacheObject) -> io::Result - where - K: HlsCacheObjectKey, - { - self.commit_staged_with_guard(key, staged, ()).await.map(|(metadata, ())| metadata) - } - - /// Commits a staged object while transferring a cancellation cleanup guard through the owned rename task. - pub async fn commit_staged_with_guard( - &self, - key: &K, - staged: StagedCacheObject, - guard: G, - ) -> io::Result<(CachedSegmentMetadata, G)> - where - K: HlsCacheObjectKey, - G: Send + 'static, - { - let staged_path = staged.path.clone(); - let result = self.commit_staged_inner(key, staged, guard).await; - if result.is_err() { - remove_temp_file_after_error(&staged_path, "commit").await; - } - result - } - - #[allow(clippy::too_many_lines)] - async fn commit_staged_inner( - &self, - key: &K, - staged: StagedCacheObject, - guard: G, - ) -> io::Result<(CachedSegmentMetadata, G)> - where - K: HlsCacheObjectKey, - G: Send + 'static, - { - // Every authoritative admission participates in this gate. Otherwise a new writer that happens to fit after - // reclamation can consume the reclaimed bytes before the reclaiming writer performs its retry. - let admission_gate = self.capacity_admission_gate.lock().await; - let mut reclamation_attempted = false; - let mut reclamation_outcome = HlsCacheCapacityReclaimOutcome::default(); - let mut reclamation_pressure: Option = None; - loop { - let commit_gate = Arc::clone(&self.cache_path_commit_gate).read_owned().await; - let cache_path = self.cache_path_snapshot(); - if !Arc::ptr_eq(&staged.cache_path_generation, &cache_path.generation) - || !staged.path.starts_with(&cache_path.path) - { - return Err(io::Error::new( - io::ErrorKind::Interrupted, - "hls cache path changed before staged object commit", - )); - } - self.ensure_cache_root_marker_for(&cache_path.path).await?; - let max_object_bytes = self.max_object_bytes.load(Ordering::Acquire); - if staged.size > max_object_bytes { - return Err(cache_object_limit_error(max_object_bytes)); - } - let final_path = Self::path_for_key_in(&cache_path.path, key); - let Some(parent) = final_path.parent() else { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "cache path has no parent")); - }; - fs::create_dir_all(parent).await?; - let reservation = self - .prepare_capacity_replacement( - &cache_path.path, - &final_path, - key.session_path_component(), - staged.size, - self.max_cache_bytes.load(Ordering::Acquire), - self.max_session_bytes.load(Ordering::Acquire), - ) - .await; - let reservation = match reservation { - Ok(reservation) => { - if let Some(pressure) = reclamation_pressure { - let usage = reservation.projected_usage(); - log::info!( - "HLS cache capacity decision: proxy_session={} resource={} configured_session_bytes={} configured_global_bytes={} current_session_bytes={} current_global_bytes={} staged_bytes={} protected_working_set_bytes={} reclaimable_bytes={} required_session_bytes={} required_global_bytes={} outcome=reclaimed", - safe_proxy_session_id(key.proxy_session_id()), - key.file_name(), - pressure.configured_session_bytes, - pressure.configured_global_bytes, - usage.session_bytes, - usage.global_bytes, - staged.size, - reclamation_outcome.protected_working_set_bytes, - reclamation_outcome.reclaimable_bytes, - pressure.required_session_bytes, - pressure.required_global_bytes, - ); - } - reservation - } - Err(error) => { - let Some(capacity) = hls_cache_capacity_from_io(&error) else { - return Err(error); - }; - if reclamation_attempted { - log::warn!( - "HLS cache capacity decision: proxy_session={} resource={} configured_session_bytes={} configured_global_bytes={} current_session_bytes={} current_global_bytes={} staged_bytes={} protected_working_set_bytes={} reclaimable_bytes={} required_session_bytes={} required_global_bytes={} outcome=deferred-protected wake_reason=capacity-or-protection-revision", - safe_proxy_session_id(key.proxy_session_id()), - key.file_name(), - capacity.configured_session_bytes(), - capacity.configured_global_bytes(), - capacity.current_session_bytes(), - capacity.current_global_bytes(), - capacity.staged_bytes(), - reclamation_outcome.protected_working_set_bytes, - reclamation_outcome.reclaimable_bytes, - capacity.required_session_bytes(), - capacity.required_global_bytes(), - ); - return Err(capacity_error( - capacity.pressure(), - reclamation_outcome, - capacity.revision().clone(), - )); - } - let request = HlsCacheCapacityReclaimRequest { - proxy_session_id: key.proxy_session_id().clone(), - target_path: final_path, - required_session_bytes: capacity.required_session_bytes(), - required_global_bytes: capacity.required_global_bytes(), - staged_bytes: staged.size, - }; - log::warn!( - "HLS cache capacity pressure detected: proxy_session={} staged_bytes={} required_session_bytes={} required_global_bytes={}", - safe_proxy_session_id(key.proxy_session_id()), - staged.size, - capacity.required_session_bytes(), - capacity.required_global_bytes(), - ); - // The admission gate remains held through reclamation and the next authoritative reservation. The - // path read lease cannot: cache-root handoff uses run-gate -> path-write ordering. - drop(commit_gate); - reclamation_attempted = true; - reclamation_pressure = Some(capacity.pressure()); - match self.reclaim_capacity(request).await { - Ok(outcome) => { - reclamation_outcome = outcome; - log::info!( - "HLS cache capacity reclamation completed: proxy_session={} reclaimed_session_bytes={} reclaimed_global_bytes={} protected_working_set_bytes={} reclaimable_bytes={}", - safe_proxy_session_id(key.proxy_session_id()), - outcome.reclaimed_session_bytes, - outcome.reclaimed_global_bytes, - outcome.protected_working_set_bytes, - outcome.reclaimable_bytes, - ); - } - Err(reclaim_error) => { - log::warn!( - "HLS cache capacity reclamation failed: proxy_session={} error_kind={:?}", - safe_proxy_session_id(key.proxy_session_id()), - reclaim_error.kind(), - ); - return Err(reclaim_error); - } - } - continue; - } - }; - drop(admission_gate); - let staged_path = staged.path.clone(); - let committed_size = staged.size; - // Once spawned, the owned mutation deliberately outlives cancellation of the requesting future. This - // keeps the path lease, temp registration, and accounting reservation alive until the atomic filesystem - // operation has a definite result. A runtime shutdown may abort the task, but no cache work can follow - // that shutdown; panic/abort drops the reservation and forces accounting rollback/invalidation. - return run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "commit", async move { - let mut reservation = reservation; - reservation.mark_filesystem_mutation_started(); - if let Err(err) = fs::rename(&staged_path, &final_path).await { - remove_temp_file_after_error(&staged_path, "rename").await; - return Err(err); - } - reservation.finish_replacement(); - drop(staged); - drop(commit_gate); - Ok((CachedSegmentMetadata { path: final_path, size: committed_size }, guard)) - }) - .await; - } - } - - async fn reclaim_capacity( - &self, - request: HlsCacheCapacityReclaimRequest, - ) -> io::Result { - let reclaimer = self - .capacity_reclaimer - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_ref() - .and_then(Weak::upgrade); - match reclaimer { - Some(reclaimer) => reclaimer.reclaim_capacity(request).await, - // Standalone cache users have no session graph to reclaim. This is - // a zero-reclamation capacity result, not a storage I/O failure. - None => Ok(HlsCacheCapacityReclaimOutcome::default()), - } - } - - pub async fn remove_staged(&self, staged: StagedCacheObject) -> io::Result<()> { - let result = match fs::remove_file(&staged.path).await { - Ok(()) => Ok(()), - Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()), - Err(err) => Err(err), - }; - drop(staged); - result - } - - async fn write_temp_and_commit_inner( - &self, - key: &K, - reader: &mut R, - deadline: Option, - ) -> io::Result - where - K: HlsCacheObjectKey, - R: AsyncRead + Unpin, - { - let staged = self.stage_temp_inner(key, reader, deadline).await?; - self.commit_staged(key, staged).await - } - - async fn stage_temp_inner( - &self, - key: &K, - reader: &mut R, - deadline: Option, - ) -> io::Result - where - K: HlsCacheObjectKey, - R: AsyncRead + Unpin, - { - if deadline.is_some_and(|deadline| deadline <= Instant::now()) { - return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); - } - let cache_path = self.cache_path_snapshot(); - self.ensure_cache_root_marker_for(&cache_path.path).await?; - let final_path = Self::path_for_key_in(&cache_path.path, key); - let Some(parent) = final_path.parent() else { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "cache path has no parent")); - }; - fs::create_dir_all(parent).await?; - - let (temp_path, mut temp_file, registration) = self.create_temp_file(key, &cache_path).await?; - if deadline.is_some_and(|deadline| deadline <= Instant::now()) { - drop(temp_file); - remove_temp_file_after_error(&temp_path, "expired-write").await; - return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); - } - let copy = async { - let max_object_bytes = self.max_object_bytes.load(Ordering::Acquire); - let mut limited = reader.take(max_object_bytes.saturating_add(1)); - let size = tokio::io::copy(&mut limited, &mut temp_file).await?; - if size > max_object_bytes { - return Err(cache_object_limit_error(max_object_bytes)); - } - temp_file.flush().await?; - drop(temp_file); - Ok::(size) - }; - let copy_result = if let Some(deadline) = deadline { - if let Ok(result) = timeout_at(deadline, copy).await { - result - } else { - remove_temp_file_after_error(&temp_path, "timed-out-write").await; - return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); - } - } else { - copy.await - }; - match copy_result { - Ok(size) => { - Ok(StagedCacheObject::registered(temp_path, size, Arc::clone(&cache_path.generation), registration)) - } - Err(err) => { - remove_temp_file_after_error(&temp_path, "failed-write").await; - Err(err) - } - } - } - - pub async fn write_bytes_and_commit(&self, key: &K, bytes: &[u8]) -> io::Result - where - K: HlsCacheObjectKey, - { - self.write_temp_and_commit(key, bytes).await - } - - pub async fn delete(&self, key: &K) -> io::Result<()> { - let cache_path = self.cache_path_snapshot(); - let path = Self::path_for_key_in(&cache_path.path, key); - let reservation = self.begin_capacity_mutation(&cache_path.path, &path, key.session_path_component()).await; - self.delete_with_reservation(path, reservation).await - } - - pub async fn delete_if_inactive(&self, key: &K) -> io::Result<()> { - let cache_path = self.cache_path_snapshot(); - let path = Self::path_for_key_in(&cache_path.path, key); - let reservation = self - .try_begin_capacity_mutation(&cache_path.path, &path, key.session_path_component()) - .ok_or_else(|| io::Error::new(io::ErrorKind::WouldBlock, "hls cache object has an active mutation"))?; - self.delete_with_reservation(path, reservation).await - } - - async fn delete_with_reservation(&self, path: PathBuf, reservation: CapacityMutationReservation) -> io::Result<()> { - run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "delete", async move { - let size = match fs::metadata(&path).await { - Ok(metadata) => metadata.len(), - Err(err) if err.kind() == io::ErrorKind::NotFound => 0, - Err(err) => return Err(err), - }; - let mut reservation = reservation; - reservation.mark_filesystem_mutation_started(); - match fs::remove_file(&path).await { - Ok(()) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => {} - Err(err) => return Err(err), - } - reservation.finish_delete(size); - Ok(()) - }) - .await - } - - pub fn object_path(&self, key: &K) -> PathBuf { self.path_for_key(key) } - - pub fn contains_current_cache_path(&self, path: &Path) -> bool { path.starts_with(self.cache_path_snapshot().path) } - - pub fn has_active_mutation(&self, path: &Path) -> bool { - self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner).active_mutations.contains(path) - } - - pub async fn capacity_usage(&self, proxy_session_id: &ProxySessionId) -> io::Result { - let cache_path = self.cache_path_snapshot(); - self.ensure_capacity_initialized(&cache_path.path).await?; - let capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if !capacity.initialized || capacity.cache_path != cache_path.path { - return Err(io::Error::new(io::ErrorKind::Interrupted, "hls cache usage snapshot invalidated")); - } - Ok(HlsCacheCapacityUsage { - session_bytes: capacity - .session_bytes - .get(&safe_session_path_component(proxy_session_id)) - .copied() - .unwrap_or_default(), - global_bytes: capacity.total_bytes, - }) - } - - pub async fn delete_temp_files_older_than(&self, cutoff: SystemTime) -> io::Result { - let cache_path = self.cache_path_snapshot(); - let candidates = old_temp_files(&cache_path.path, cutoff, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN).await?; - let mut deleted = 0_usize; - for path in candidates { - let Some(deletion) = self.reserve_path_deletion(&path) else { - continue; - }; - let result = run_owned_cache_operation( - Arc::clone(&self.owned_operation_permits), - "temporary-file-delete", - async move { - let result = fs::remove_file(path).await; - drop(deletion); - result - }, - ) - .await; - match result { - Ok(()) => deleted = deleted.saturating_add(1), - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => warn!("HLS temporary cache file cleanup deferred: error_kind={:?}", error.kind()), - } - } - Ok(deleted) - } - - pub async fn delete_session_dir(&self, proxy_session_id: &ProxySessionId) -> io::Result<()> { - let cache_path = self.cache_path_snapshot(); - let path = cache_path.path.join(safe_session_path_component(proxy_session_id)); - let deletion = self.reserve_path_deletion(&path).ok_or_else(|| { - io::Error::new(io::ErrorKind::WouldBlock, "hls cache session directory still has active temp files") - })?; - let capacity_invalidation = self.capacity_invalidation_guard(); - run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "session-delete", async move { - let _capacity_invalidation = capacity_invalidation; - let result = match fs::remove_dir_all(path).await { - Ok(()) => Ok(()), - Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()), - Err(err) => Err(err), - }; - drop(deletion); - result - }) - .await - } - - pub async fn delete_orphan_session_dirs( - &self, - active_session_ids: &HashSet, - freshness_cutoff: SystemTime, - ) -> io::Result { - let cache_path = self.cache_path_snapshot(); - ensure_safe_cache_root(&cache_path.path).await?; - let active_paths = active_session_ids - .iter() - .map(|id| cache_path.path.join(safe_session_path_component(id))) - .collect::>(); - let mut entries = fs::read_dir(&cache_path.path).await?; - let mut removed = 0_usize; - while let Some(entry) = entries.next_entry().await? { - let path = entry.path(); - let file_type = match entry.file_type().await { - Ok(file_type) => file_type, - Err(error) if error.kind() == io::ErrorKind::NotFound => continue, - Err(error) => { - warn!("HLS orphan session entry inspection deferred: error_kind={:?}", error.kind()); - continue; - } - }; - if !file_type.is_dir() || active_paths.contains(&path) { - continue; - } - // Freshness guard: skip directories committed after the GC took its - // in-memory session snapshot. Their owning session may not yet be - // visible in `active_session_ids`, so deleting them would race with - // a concurrent segment write that just created the directory. - let metadata = match entry.metadata().await { - Ok(metadata) => metadata, - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(error) => { - warn!("HLS orphan session metadata inspection deferred: error_kind={:?}", error.kind()); - continue; - } - }; - if let Ok(modified) = metadata.modified() { - if modified > freshness_cutoff { - continue; - } - } - let Some(deletion) = self.reserve_path_deletion(&path) else { - continue; - }; - let capacity_invalidation = self.capacity_invalidation_guard(); - let result = run_owned_cache_operation( - Arc::clone(&self.owned_operation_permits), - "orphan-session-delete", - async move { - let _capacity_invalidation = capacity_invalidation; - let result = fs::remove_dir_all(&path).await; - drop(deletion); - result - }, - ) - .await; - match result { - Ok(()) => removed = removed.saturating_add(1), - Err(err) if err.kind() == io::ErrorKind::NotFound => {} - Err(err) => { - warn!("HLS orphan session directory cleanup deferred: error_kind={:?}", err.kind()); - } - } - } - Ok(removed) - } - - pub fn has_active_temp_files_for_session(&self, proxy_session_id: &ProxySessionId) -> bool { - let session_path = self.cache_path_snapshot().path.join(safe_session_path_component(proxy_session_id)); - self.temp_files - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .active_files - .iter() - .any(|path| path.starts_with(&session_path)) - } - - pub fn has_active_temp_files(&self) -> bool { - !self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner).active_files.is_empty() - } - - pub async fn invalidate_all_if_no_active_temp_files(&self) -> io::Result { - let cache_path = self.cache_path_snapshot(); - let Some(deletion) = self.reserve_path_deletion(&cache_path.path) else { - return Ok(CacheInvalidationOutcome::DeferredActiveTempFiles); - }; - self.invalidate_all_unchecked(cache_path.path, deletion).await?; - Ok(CacheInvalidationOutcome::Invalidated) - } - - pub async fn invalidate_all(&self) -> io::Result<()> { - let cache_path = self.cache_path_snapshot(); - let deletion = self.reserve_path_deletion(&cache_path.path).ok_or_else(|| { - io::Error::new(io::ErrorKind::WouldBlock, "hls cache invalidation conflicts with an active object write") - })?; - self.invalidate_all_unchecked(cache_path.path, deletion).await - } - - async fn invalidate_all_unchecked( - &self, - cache_path: PathBuf, - deletion: CachePathDeletionReservation, - ) -> io::Result<()> { - let capacity_invalidation = self.capacity_invalidation_guard(); - run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "invalidate", async move { - let _capacity_invalidation = capacity_invalidation; - let result = async { - ensure_safe_cache_root(&cache_path).await?; - fs::create_dir_all(&cache_path).await?; - let mut entries = fs::read_dir(&cache_path).await?; - while let Some(entry) = entries.next_entry().await? { - if entry.file_name() == REWRITE_SECRET_FINGERPRINT_FILE - || entry.file_name() == HLS_CACHE_ROOT_MARKER_FILE - { - continue; - } - let file_type = entry.file_type().await?; - if file_type.is_dir() { - fs::remove_dir_all(entry.path()).await?; - } else { - fs::remove_file(entry.path()).await?; - } - } - Ok(()) - } - .await; - drop(deletion); - result - }) - .await - } - - pub async fn read_rewrite_secret_fingerprint(&self) -> io::Result> { - match fs::read_to_string(self.rewrite_secret_fingerprint_path()).await { - Ok(value) => Ok(Some(value.trim().to_string())), - Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), - Err(err) => Err(err), - } - } - - pub async fn write_rewrite_secret_fingerprint(&self, fingerprint: &str) -> io::Result<()> { - self.ensure_cache_root_marker().await?; - fs::write(self.rewrite_secret_fingerprint_path(), fingerprint).await - } - - fn path_for_key(&self, key: &K) -> PathBuf { - let cache_path = self.cache_path_snapshot(); - Self::path_for_key_in(&cache_path.path, key) - } - - fn path_for_key_in(cache_path: &Path, key: &K) -> PathBuf { - cache_path.join(key.session_path_component()).join(key.file_name()) - } - - async fn create_temp_file( - &self, - key: &K, - cache_path: &CachePathState, - ) -> io::Result<(PathBuf, File, Arc)> { - for _ in 0..TEMP_CREATE_ATTEMPTS { - let temp_path = Self::temp_path_for_key(key, &cache_path.path); - let registration = self.register_active_temp_path(&temp_path)?; - match OpenOptions::new().write(true).create_new(true).open(&temp_path).await { - Ok(file) => { - return Ok((temp_path, file, registration)); - } - Err(err) if err.kind() == io::ErrorKind::AlreadyExists => { - drop(registration); - } - Err(err) => return Err(err), - } - } - Err(io::Error::new(io::ErrorKind::AlreadyExists, "could not create unique hls cache temp file")) - } - - pub fn adopt_staged_file(&self, path: PathBuf, size: u64) -> io::Result { - let cache_path = self.cache_path_snapshot(); - if !path.starts_with(&cache_path.path) { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "staged hls cache file is outside the cache root")); - } - let registration = self.register_active_temp_path(&path)?; - Ok(StagedCacheObject::registered(path, size, Arc::clone(&cache_path.generation), registration)) - } - - fn temp_path_for_key(key: &K, cache_path: &Path) -> PathBuf { - let suffix = fastrand::u64(..); - cache_path.join(key.session_path_component()).join(format!("{}.tmp.{suffix:016x}", key.file_name())) - } - - fn register_active_temp_path(&self, path: &Path) -> io::Result> { - let mut state = self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if state.deletion_reservations.iter().any(|reserved| path.starts_with(reserved)) { - return Err(io::Error::new(io::ErrorKind::Interrupted, "hls cache path is reserved for invalidation")); - } - if !state.active_files.insert(path.to_path_buf()) { - return Err(io::Error::new(io::ErrorKind::AlreadyExists, "hls cache temp path is already active")); - } - Ok(Arc::new(ActiveTempFileRegistration { path: path.to_path_buf(), state: Arc::clone(&self.temp_files) })) - } - - fn reserve_path_deletion(&self, path: &Path) -> Option { - let mut state = self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let conflicts_with_active = state.active_files.iter().any(|active| active.starts_with(path)); - let conflicts_with_delete = - state.deletion_reservations.iter().any(|reserved| reserved.starts_with(path) || path.starts_with(reserved)); - if conflicts_with_active || conflicts_with_delete { - return None; - } - let path = path.to_path_buf(); - state.deletion_reservations.insert(path.clone()); - Some(CachePathDeletionReservation { path, state: Arc::clone(&self.temp_files) }) - } - - async fn ensure_capacity_initialized(&self, cache_path: &Path) -> io::Result<()> { - loop { - let notified = self.capacity_changed.notified(); - let revision = { - let capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if capacity.initialized && capacity.cache_path == cache_path { - return Ok(()); - } - if capacity.active_mutations.iter().any(|path| path.starts_with(cache_path)) { - None - } else { - Some(Arc::clone(&capacity.revision)) - } - }; - let Some(revision) = revision else { - notified.await; - continue; - }; - let (total_bytes, session_bytes) = scan_committed_cache_usage(cache_path).await?; - if self.cache_path_snapshot().path != cache_path { - return Err(io::Error::new(io::ErrorKind::Interrupted, "hls cache path changed during usage scan")); - } - let scan_invalidated = { - let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if !Arc::ptr_eq(&capacity.revision, &revision) - || capacity.active_mutations.iter().any(|path| path.starts_with(cache_path)) - { - true - } else { - capacity.cache_path = cache_path.to_path_buf(); - capacity.initialized = true; - capacity.total_bytes = total_bytes; - capacity.session_bytes = session_bytes; - false - } - }; - if scan_invalidated { - // The scan result was invalidated by a real accounting or - // mutation transition. Wait on the notification registered - // before the scan instead of immediately walking the complete - // cache tree again under sustained commit churn. - notified.await; - continue; - } - return Ok(()); - } - } - - async fn begin_capacity_mutation( - &self, - cache_path: &Path, - path: &Path, - session_component: String, - ) -> CapacityMutationReservation { - loop { - let notified = self.capacity_changed.notified(); - let Some(reservation) = self.try_begin_capacity_mutation(cache_path, path, session_component.clone()) - else { - notified.await; - continue; - }; - return reservation; - } - } - - fn try_begin_capacity_mutation( - &self, - cache_path: &Path, - path: &Path, - session_component: String, - ) -> Option { - let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if !capacity.active_mutations.insert(path.to_path_buf()) { - return None; - } - drop(capacity); - Some(CapacityMutationReservation { - path: path.to_path_buf(), - cache_path: cache_path.to_path_buf(), - session_component, - replacement: None, - filesystem_mutation_started: false, - capacity: Arc::clone(&self.capacity), - changed: Arc::clone(&self.capacity_changed), - }) - } - - async fn prepare_capacity_replacement( - &self, - cache_path: &Path, - final_path: &Path, - session_component: String, - new_size: u64, - max_cache_bytes: u64, - max_session_bytes: u64, - ) -> io::Result { - loop { - self.ensure_capacity_initialized(cache_path).await?; - let mut reservation = self.begin_capacity_mutation(cache_path, final_path, session_component.clone()).await; - let old_size = match fs::metadata(final_path).await { - Ok(metadata) => metadata.len(), - Err(err) if err.kind() == io::ErrorKind::NotFound => 0, - Err(err) => return Err(err), - }; - if self.cache_path_snapshot().path != cache_path { - return Err(io::Error::new(io::ErrorKind::Interrupted, "hls cache path changed during object write")); - } - match reservation.reserve_replacement(old_size, new_size, max_cache_bytes, max_session_bytes) { - Ok(()) => return Ok(reservation), - Err(CapacityReservationError::Retry | CapacityReservationError::Invalidated) => { - drop(reservation); - } - Err(CapacityReservationError::Exceeded { pressure, revision }) => { - return Err(capacity_error(pressure, HlsCacheCapacityReclaimOutcome::default(), revision)); - } - } - } - } - - fn invalidate_capacity_accounting(&self) { invalidate_capacity_state(&self.capacity, &self.capacity_changed); } - - fn capacity_invalidation_guard(&self) -> CapacityInvalidationGuard { - CapacityInvalidationGuard { capacity: Arc::clone(&self.capacity), changed: Arc::clone(&self.capacity_changed) } - } - - fn rewrite_secret_fingerprint_path(&self) -> PathBuf { - self.cache_path_snapshot().path.join(REWRITE_SECRET_FINGERPRINT_FILE) - } - - async fn ensure_cache_root_marker(&self) -> io::Result<()> { - let cache_path = self.cache_path_snapshot(); - self.ensure_cache_root_marker_for(&cache_path.path).await - } - - async fn ensure_cache_root_marker_for(&self, cache_path: &Path) -> io::Result<()> { - let marker_matches = { - let marker_path = self.marker_path.read().unwrap_or_else(std::sync::PoisonError::into_inner); - marker_path.as_deref() == Some(cache_path) - }; - if marker_matches { - return Ok(()); - } - ensure_not_root_like_cache_path(cache_path)?; - fs::create_dir_all(cache_path).await?; - fs::write(cache_path.join(HLS_CACHE_ROOT_MARKER_FILE), b"tuliprox-hls-cache\n").await?; - *self.marker_path.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(cache_path.to_path_buf()); - Ok(()) - } - - fn cache_path_snapshot(&self) -> CachePathState { - self.cache_path.read().unwrap_or_else(std::sync::PoisonError::into_inner).clone() - } -} - -fn capacity_error( - pressure: CacheCapacityPressure, - reclaim: HlsCacheCapacityReclaimOutcome, - revision: HlsCacheCapacityRevision, -) -> io::Error { - io::Error::other(HlsCacheCapacityError { - configured_session_bytes: pressure.configured_session_bytes, - configured_global_bytes: pressure.configured_global_bytes, - current_session_bytes: pressure.current_session_bytes, - current_global_bytes: pressure.current_global_bytes, - staged_bytes: pressure.staged_bytes, - required_session_bytes: pressure.required_session_bytes, - required_global_bytes: pressure.required_global_bytes, - protected_working_set_bytes: reclaim.protected_working_set_bytes, - reclaimable_bytes: reclaim.reclaimable_bytes, - revision, - }) -} - -fn invalidate_capacity_state(capacity: &StdMutex, changed: &Notify) { - let mut capacity = capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - capacity.initialized = false; - capacity.revision = Arc::new(CapacityRevision); - drop(capacity); - changed.notify_waiters(); -} - -async fn remove_temp_file_after_error(path: &Path, operation: &'static str) { - match fs::remove_file(path).await { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => { - warn!("HLS cache temporary file cleanup failed: operation={operation} error_kind={:?}", error.kind()); - } - } -} - -/// Runs one filesystem mutation with all of its reservations owned by the spawned task. -/// -/// A per-cache permit is acquired without waiting before the spawn. Saturation returns `WouldBlock` and drops the -/// supplied future synchronously, which releases any reservations it owns without creating a detached task. -/// -/// Dropping the caller future detaches, rather than cancels, the mutation. A returned `JoinError` therefore denotes a -/// task panic/abort; unwinding drops the owned guards. Runtime shutdown is the only case where completion is not -/// observed, and no later cache accounting or root transition can run after that shutdown. -async fn run_owned_cache_operation(permits: Arc, operation: &'static str, future: F) -> io::Result -where - T: Send + 'static, - F: Future> + Send + 'static, -{ - let permit = permits - .try_acquire_owned() - .map_err(|_| io::Error::new(io::ErrorKind::WouldBlock, "hls cache owned operation capacity exhausted"))?; - tokio::spawn(async move { - let _permit = permit; - future.await - }) - .await - .map_err(|error| io::Error::other(format!("hls cache {operation} task failed: {error}")))? -} - -const REWRITE_SECRET_FINGERPRINT_FILE: &str = ".rewrite_secret_fingerprint"; -const HLS_CACHE_ROOT_MARKER_FILE: &str = ".tuliprox-hls-cache-root"; - -fn safe_session_path_component(proxy_session_id: &ProxySessionId) -> String { - let value = &proxy_session_id.0; - if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) { - return value.clone(); - } - blake3::hash(value.as_bytes()).to_hex().to_string() -} - -fn ensure_not_root_like_cache_path(cache_path: &Path) -> io::Result<()> { - if cache_path.as_os_str().is_empty() || cache_path.parent().is_none() { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "refusing to invalidate unsafe hls cache root")); - } - Ok(()) -} - -async fn ensure_safe_cache_root(cache_path: &Path) -> io::Result<()> { - ensure_not_root_like_cache_path(cache_path)?; - match fs::metadata(cache_path.join(HLS_CACHE_ROOT_MARKER_FILE)).await { - Ok(metadata) if metadata.is_file() => Ok(()), - Ok(_) => Err(io::Error::new( - io::ErrorKind::InvalidInput, - "refusing to invalidate hls cache root without marker file", - )), - Err(err) if err.kind() == io::ErrorKind::NotFound => Err(io::Error::new( - io::ErrorKind::InvalidInput, - "refusing to invalidate hls cache root without marker file", - )), - Err(err) => Err(err), - } -} - -async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result<(u64, HashMap)> { - let mut total_bytes = 0_u64; - let mut session_bytes = HashMap::new(); - let mut root_entries = match fs::read_dir(cache_path).await { - Ok(entries) => entries, - Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok((0, session_bytes)), - Err(err) => return Err(err), - }; - while let Some(root_entry) = root_entries.next_entry().await? { - if !root_entry.file_type().await?.is_dir() { - continue; - } - let session_component = root_entry.file_name().to_string_lossy().into_owned(); - let mut bytes = 0_u64; - let mut pending_dirs = vec![root_entry.path()]; - while let Some(dir) = pending_dirs.pop() { - let mut entries = fs::read_dir(dir).await?; - while let Some(entry) = entries.next_entry().await? { - let file_type = entry.file_type().await?; - if file_type.is_dir() { - pending_dirs.push(entry.path()); - } else if file_type.is_file() && !is_temp_cache_file(&entry.path()) { - bytes = bytes.saturating_add(entry.metadata().await?.len()); - } - } - } - total_bytes = total_bytes.saturating_add(bytes); - if bytes > 0 { - session_bytes.insert(session_component, bytes); - } - } - Ok((total_bytes, session_bytes)) -} - -async fn old_temp_files(root: &Path, cutoff: SystemTime, limit: usize) -> io::Result> { - let mut candidates = Vec::with_capacity(limit); - if limit == 0 { - return Ok(candidates); - } - let mut root_entries = match fs::read_dir(root).await { - Ok(entries) => entries, - Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(candidates), - Err(err) => return Err(err), - }; - while candidates.len() < limit { - let Some(entry) = root_entries.next_entry().await? else { - break; - }; - let path = entry.path(); - let file_type = match entry.file_type().await { - Ok(file_type) => file_type, - Err(error) if error.kind() == io::ErrorKind::NotFound => continue, - Err(error) => { - warn!("HLS temporary cache entry inspection deferred: error_kind={:?}", error.kind()); - continue; - } - }; - if file_type.is_dir() { - append_old_temp_files_in_session(&path, cutoff, limit, &mut candidates).await?; - } else { - append_old_temp_file_candidate(&entry, cutoff, limit, &mut candidates).await; - } - } - Ok(candidates) -} - -async fn append_old_temp_files_in_session( - session_root: &Path, - cutoff: SystemTime, - limit: usize, - candidates: &mut Vec, -) -> io::Result<()> { - let mut pending_dirs = vec![session_root.to_path_buf()]; - while let Some(dir) = pending_dirs.pop() { - if candidates.len() >= limit { - break; - } - let mut entries = match fs::read_dir(&dir).await { - Ok(entries) => entries, - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(err) => return Err(err), - }; - while candidates.len() < limit { - let Some(entry) = entries.next_entry().await? else { - break; - }; - let path = entry.path(); - let file_type = match entry.file_type().await { - Ok(file_type) => file_type, - Err(error) if error.kind() == io::ErrorKind::NotFound => continue, - Err(error) => { - warn!("HLS temporary cache entry inspection deferred: error_kind={:?}", error.kind()); - continue; - } - }; - if file_type.is_dir() { - pending_dirs.push(path); - continue; - } - append_old_temp_file_candidate(&entry, cutoff, limit, candidates).await; - } - } - Ok(()) -} - -async fn append_old_temp_file_candidate( - entry: &fs::DirEntry, - cutoff: SystemTime, - limit: usize, - candidates: &mut Vec, -) { - let path = entry.path(); - if candidates.len() >= limit || !is_temp_cache_file(&path) { - return; - } - let metadata = match entry.metadata().await { - Ok(metadata) => metadata, - Err(error) if error.kind() == io::ErrorKind::NotFound => return, - Err(error) => { - warn!("HLS temporary cache metadata inspection deferred: error_kind={:?}", error.kind()); - return; - } - }; - if metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH) < cutoff { - candidates.push(path); - } -} - -fn is_temp_cache_file(path: &Path) -> bool { - path.file_name().and_then(|file_name| file_name.to_str()).is_some_and(|file_name| file_name.contains(".tmp.")) -} - impl Default for HlsSegmentCache { fn default() -> Self { Self::new() } } -#[cfg(test)] -mod tests { - use super::{ - hls_cache_capacity_from_io, hls_cache_object_limit_from_io, run_owned_cache_operation, - CacheInvalidationOutcome, HlsCacheCapacityReclaimOutcome, HlsCacheCapacityReclaimRequest, - HlsCacheCapacityReclaimer, HlsCacheObjectKey, HlsSegmentCache, MapCacheKey, SegmentCacheKey, - TransientObjectCacheKey, MAX_CONCURRENT_OWNED_CACHE_OPERATIONS, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN, - }; - use crate::{transient::build_transient_resource_id, HlsOriginResourceFetchError, ProxySessionId}; - use futures::{future::BoxFuture, FutureExt}; - use std::{ - collections::{HashSet, VecDeque}, - future::Future, - io, - pin::Pin, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, Mutex, - }, - task::{Context, Poll}, - time::{Duration, SystemTime}, - }; - use tokio::{ - io::{AsyncRead, AsyncReadExt, ReadBuf}, - sync::{oneshot, Semaphore}, - }; - - fn cache_key() -> SegmentCacheKey { SegmentCacheKey::new(ProxySessionId("proxy_session".to_string()), 123, "ts") } - - async fn cache_with_projected_session_pressure( - ) -> (tempfile::TempDir, Arc, SegmentCacheKey, SegmentCacheKey) { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let proxy_session_id = ProxySessionId("capacity-session".to_string()); - let resident = SegmentCacheKey::new(proxy_session_id.clone(), 0, "ts"); - let first = SegmentCacheKey::new(proxy_session_id.clone(), 1, "ts"); - let second = SegmentCacheKey::new(proxy_session_id, 2, "ts"); - cache.update_cache_limits(5, 5); - cache.write_bytes_and_commit(&resident, b"123").await.expect("resident object commits"); - (temp_dir, cache, first, second) - } - - struct PausingCapacityReclaimer { - cache: Arc, - victims: Mutex>, - first_reclaimed: Mutex>>, - resume_first: Mutex>>, - } - - impl HlsCacheCapacityReclaimer for PausingCapacityReclaimer { - fn reclaim_capacity( - &self, - _request: HlsCacheCapacityReclaimRequest, - ) -> BoxFuture<'_, io::Result> { - async move { - let victim = self.victims.lock().unwrap_or_else(std::sync::PoisonError::into_inner).pop_front(); - let Some(victim) = victim else { - return Ok(HlsCacheCapacityReclaimOutcome::default()); - }; - self.cache.delete_if_inactive(&victim).await?; - let first_reclaimed = - self.first_reclaimed.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); - if let Some(first_reclaimed) = first_reclaimed { - let resume_first = - self.resume_first.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); - let _ = first_reclaimed.send(()); - if let Some(resume_first) = resume_first { - let _ = resume_first.await; - } - } - Ok(HlsCacheCapacityReclaimOutcome { - reclaimed_session_bytes: 10, - reclaimed_global_bytes: 10, - protected_working_set_bytes: 0, - reclaimable_bytes: 10, - }) - } - .boxed() - } - } - - struct ControlledReader { - started: Option>, - release: oneshot::Receiver>, - body: Option>>, - } - - impl Unpin for ControlledReader {} - - impl AsyncRead for ControlledReader { - fn poll_read( - mut self: Pin<&mut Self>, - context: &mut Context<'_>, - buffer: &mut ReadBuf<'_>, - ) -> Poll> { - if let Some(started) = self.started.take() { - let _send_result = started.send(()); - } - if self.body.is_none() { - match Pin::new(&mut self.release).poll(context) { - Poll::Ready(Ok(body)) => self.body = Some(std::io::Cursor::new(body)), - Poll::Ready(Err(_)) => { - return Poll::Ready(Err(io::Error::new( - io::ErrorKind::BrokenPipe, - "controlled cache reader release dropped", - ))) - } - Poll::Pending => return Poll::Pending, - } - } - let body = self.body.as_mut().expect("controlled reader body is initialized"); - let position = usize::try_from(body.position()).unwrap_or(body.get_ref().len()); - let remaining = &body.get_ref()[position.min(body.get_ref().len())..]; - let copied = remaining.len().min(buffer.remaining()); - buffer.put_slice(&remaining[..copied]); - let next_position = body.position().saturating_add(u64::try_from(copied).unwrap_or_default()); - body.set_position(next_position); - Poll::Ready(Ok(())) - } - } - - #[test] - fn segment_cache_key_contains_no_origin_data() { - let key = cache_key(); - - assert_eq!(key.stable_value(), "hls:proxy_session:00000000000000000123"); - } - - #[test] - fn cache_key_debug_redacts_proxy_session_id() { - let key = SegmentCacheKey::new(ProxySessionId("secretToken".to_string()), 123, "ts"); - let debug = format!("{key:?}"); - - assert!(debug.contains("")); - assert!(!debug.contains("secretToken")); - assert!(!debug.contains(&key.stable_value())); - } - - #[test] - fn transient_object_cache_key_keeps_redirect_hosts_distinct_without_leaking_urls() { - let proxy_session_id = ProxySessionId("proxy_session".to_string()); - let first_resource = - build_transient_resource_id("https://cdn-a.example.net/live/redirected/seg001.ts", b"secret"); - let second_resource = - build_transient_resource_id("https://cdn-b.example.net/live/redirected/seg001.ts", b"secret"); - - let first = TransientObjectCacheKey::new(proxy_session_id.clone(), first_resource, "ts"); - let second = TransientObjectCacheKey::new(proxy_session_id, second_resource, "ts"); - - assert_ne!(first, second); - assert_ne!(first.stable_value(), second.stable_value()); - for value in [first.stable_value(), second.stable_value()] { - assert!(!value.contains("provider://")); - assert!(!value.contains("cdn-a.example.net")); - assert!(!value.contains("cdn-b.example.net")); - assert!(!value.contains("/live/redirected/seg001.ts")); - } - } - - #[tokio::test] - async fn write_temp_and_commit_creates_final_segment_cache_file_with_proxy_layout() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let key = cache_key(); - - let metadata = cache.write_bytes_and_commit(&key, b"segment-body").await.expect("commit should succeed"); - - assert_eq!(metadata.size, 12); - assert!(metadata.path.exists()); - assert_eq!(cache.metadata(&key).await.expect("metadata should read"), Some(metadata.clone())); - assert!(metadata.path.ends_with("proxy_session/000123.ts")); - } - - #[tokio::test] - async fn write_temp_and_commit_creates_final_map_cache_file_with_proxy_layout() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let key = MapCacheKey::new(ProxySessionId("proxy_session".to_string()), 0, "mp4"); - - let metadata = cache.write_bytes_and_commit(&key, b"map-body").await.expect("commit should succeed"); - - assert_eq!(metadata.size, 8); - assert!(metadata.path.ends_with("proxy_session/map/000000.mp4")); - assert_eq!(cache.metadata(&key).await.expect("metadata should read"), Some(metadata)); - } - - #[tokio::test] - async fn write_temp_and_commit_with_deadline_cleans_active_temp_file() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let key = cache_key(); - let (_writer, reader) = tokio::io::duplex(64); - - let result = cache - .write_temp_and_commit_with_deadline(&key, reader, tokio::time::Instant::now() + Duration::from_millis(1)) - .await; - - assert_eq!(result.expect_err("commit should time out").kind(), io::ErrorKind::TimedOut); - assert!(!cache.has_active_temp_files()); - assert_eq!(cache.metadata(&key).await.expect("metadata should read"), None); - } - - #[tokio::test] - async fn exhausted_write_deadline_rejects_even_an_immediately_available_body() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let key = cache_key(); - - let result = cache.write_temp_and_commit_with_deadline(&key, &b"ready"[..], tokio::time::Instant::now()).await; - - assert_eq!(result.expect_err("expired deadline must time out").kind(), io::ErrorKind::TimedOut); - assert!(!cache.has_active_temp_files()); - assert_eq!(cache.metadata(&key).await.expect("metadata should read"), None); - assert_eq!(std::fs::read_dir(temp_dir.path()).expect("cache root reads").count(), 0); - } - - #[tokio::test] - async fn cache_object_write_rejects_bytes_above_the_configured_limit() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - cache.update_cache_limits(3, 3); - let key = SegmentCacheKey::new(ProxySessionId("session".to_string()), 1, "ts"); - - let result = cache.write_bytes_and_commit(&key, b"four").await; - - let error = result.expect_err("decoded object above the configured limit must fail"); - let limit_error = hls_cache_object_limit_from_io(&error).expect("typed object-limit source"); - assert_eq!(limit_error.limit(), 3); - assert!(matches!( - HlsOriginResourceFetchError::cache_body(&error), - HlsOriginResourceFetchError::CacheObjectLimit { limit: 3 } - )); - assert!(cache.metadata(&key).await.expect("metadata").is_none()); - assert!(!cache.has_active_temp_files()); - } - - #[tokio::test] - async fn orphan_session_cleanup_preserves_active_session_directories() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - cache.write_rewrite_secret_fingerprint("secret").await.expect("marker"); - let active = ProxySessionId("active".to_string()); - let orphan = ProxySessionId("orphan".to_string()); - tokio::fs::create_dir_all(temp_dir.path().join("active")).await.expect("active dir"); - tokio::fs::create_dir_all(temp_dir.path().join("orphan")).await.expect("orphan dir"); - let cutoff = SystemTime::now(); - - let removed = cache.delete_orphan_session_dirs(&HashSet::from([active]), cutoff).await.expect("cleanup"); - - assert_eq!(removed, 1); - assert!(temp_dir.path().join("active").exists()); - assert!(!temp_dir.path().join(orphan.0).exists()); - } - - #[tokio::test] - async fn orphan_session_cleanup_skips_directories_newer_than_freshness_cutoff() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - cache.write_rewrite_secret_fingerprint("secret").await.expect("marker"); - let stale = ProxySessionId("stale".to_string()); - let fresh = ProxySessionId("fresh".to_string()); - tokio::fs::create_dir_all(temp_dir.path().join(stale.0.clone())).await.expect("stale dir"); - tokio::fs::create_dir_all(temp_dir.path().join(fresh.0.clone())).await.expect("fresh dir"); - - // Set the fresh dir's mtime to the future relative to the cutoff. - let cutoff = SystemTime::now(); - let future = cutoff + std::time::Duration::from_mins(1); - filetime::set_file_mtime(temp_dir.path().join(fresh.0.clone()), filetime::FileTime::from_system_time(future)) - .expect("set fresh mtime"); - - let removed = cache.delete_orphan_session_dirs(&HashSet::new(), cutoff).await.expect("cleanup"); - - assert_eq!(removed, 1, "stale orphan dir should be removed"); - assert!(!temp_dir.path().join(stale.0).exists()); - assert!( - temp_dir.path().join(fresh.0).exists(), - "directory newer than the freshness cutoff must be preserved to avoid racing concurrent session creation" - ); - } - - #[tokio::test] - async fn cache_commits_enforce_the_global_budget_across_sessions() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - cache.update_cache_limits(5, 5); - let first = SegmentCacheKey::new(ProxySessionId("first".to_string()), 1, "ts"); - let second = SegmentCacheKey::new(ProxySessionId("second".to_string()), 1, "ts"); - - assert!(cache.write_bytes_and_commit(&first, b"123").await.is_ok()); - let error = cache.write_bytes_and_commit(&second, b"456").await.expect_err("global limit rejects commit"); - let capacity = hls_cache_capacity_from_io(&error).expect("typed local capacity error"); - assert_eq!(capacity.required_session_bytes(), 0); - assert_eq!(capacity.required_global_bytes(), 1); - assert!(cache.metadata(&second).await.expect("metadata").is_none()); - assert!(!cache.has_active_temp_files()); - let first_usage = cache.capacity_usage(first.proxy_session_id()).await.expect("usage"); - assert_eq!(first_usage.global_bytes, 3); - assert_eq!(first_usage.session_bytes, 3); - } - - #[tokio::test] - async fn failed_projected_capacity_admission_returns_a_live_revision_token() { - let (_temp_dir, cache, first, _) = cache_with_projected_session_pressure().await; - let error = cache - .ensure_projected_write_capacity(&first, 3) - .await - .expect_err("projected object exceeds the remaining session budget"); - let revision = hls_cache_capacity_from_io(&error).expect("typed capacity error").revision().clone(); - let wait = cache.wait_for_capacity_change(&revision); - tokio::pin!(wait); - - assert!(matches!(futures::poll!(wait.as_mut()), Poll::Pending)); - - cache.notify_capacity_protection_changed(); - assert!(matches!(futures::poll!(wait.as_mut()), Poll::Ready(()))); - } - - #[tokio::test] - async fn failed_admission_revision_is_captured_with_the_capacity_decision() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let path = temp_dir.path().join("session/000001.ts"); - { - let mut capacity = cache.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - capacity.cache_path = temp_dir.path().to_path_buf(); - capacity.initialized = true; - capacity.total_bytes = 2; - capacity.session_bytes.insert("session".to_string(), 2); - } - let mut reservation = cache - .try_begin_capacity_mutation(temp_dir.path(), &path, "session".to_string()) - .expect("mutation reservation"); - let error = reservation.reserve_replacement(0, 3, 2, 2).expect_err("capacity is exceeded"); - - cache.notify_capacity_protection_changed(); - drop(reservation); - - let super::CapacityReservationError::Exceeded { revision, .. } = error else { - panic!("expected typed capacity pressure"); - }; - let wait = cache.wait_for_capacity_change(&revision); - tokio::pin!(wait); - assert!(matches!(futures::poll!(wait.as_mut()), Poll::Ready(()))); - } - - #[tokio::test] - async fn concurrent_noop_capacity_failures_do_not_wake_each_other() { - let (_temp_dir, cache, first, second) = cache_with_projected_session_pressure().await; - let (first_error, second_error) = tokio::join!( - cache.ensure_projected_write_capacity(&first, 3), - cache.ensure_projected_write_capacity(&second, 3), - ); - let first_error = first_error.expect_err("first projected object exceeds the remaining budget"); - let second_error = second_error.expect_err("second projected object exceeds the remaining budget"); - let first_revision = - hls_cache_capacity_from_io(&first_error).expect("first typed capacity error").revision().clone(); - let second_revision = - hls_cache_capacity_from_io(&second_error).expect("second typed capacity error").revision().clone(); - let first_wait = cache.wait_for_capacity_change(&first_revision); - let second_wait = cache.wait_for_capacity_change(&second_revision); - tokio::pin!(first_wait, second_wait); - - assert!(matches!(futures::poll!(first_wait.as_mut()), Poll::Pending)); - assert!(matches!(futures::poll!(second_wait.as_mut()), Poll::Pending)); - - cache.notify_capacity_protection_changed(); - assert!(matches!(futures::poll!(first_wait.as_mut()), Poll::Ready(()))); - assert!(matches!(futures::poll!(second_wait.as_mut()), Poll::Ready(()))); - } - - #[tokio::test] - async fn abandoned_capacity_reservation_rolls_back_before_filesystem_mutation() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let path = temp_dir.path().join("session/000001.ts"); - { - let mut capacity = cache.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - capacity.cache_path = temp_dir.path().to_path_buf(); - capacity.initialized = true; - capacity.active_mutations.insert(path.clone()); - } - let mut reservation = super::CapacityMutationReservation { - path: path.clone(), - cache_path: temp_dir.path().to_path_buf(), - session_component: "session".to_string(), - replacement: None, - filesystem_mutation_started: false, - capacity: Arc::clone(&cache.capacity), - changed: Arc::clone(&cache.capacity_changed), - }; - reservation.reserve_replacement(0, 3, 10, 10).expect("capacity reserves"); - let revision = cache.capacity_revision(); - - drop(reservation); - - { - let capacity = cache.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - assert!(capacity.initialized); - assert_eq!(capacity.total_bytes, 0); - assert!(!capacity.session_bytes.contains_key("session")); - assert!(!capacity.active_mutations.contains(&path)); - } - let wait = cache.wait_for_capacity_change(&revision); - tokio::pin!(wait); - assert!(matches!(futures::poll!(wait.as_mut()), Poll::Ready(()))); - } - - #[tokio::test] - async fn abandoned_started_filesystem_mutation_invalidates_capacity_snapshot() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let path = temp_dir.path().join("session/000001.ts"); - { - let mut capacity = cache.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - capacity.cache_path = temp_dir.path().to_path_buf(); - capacity.initialized = true; - capacity.active_mutations.insert(path.clone()); - } - let mut reservation = super::CapacityMutationReservation { - path: path.clone(), - cache_path: temp_dir.path().to_path_buf(), - session_component: "session".to_string(), - replacement: None, - filesystem_mutation_started: false, - capacity: Arc::clone(&cache.capacity), - changed: Arc::clone(&cache.capacity_changed), - }; - reservation.reserve_replacement(0, 3, 10, 10).expect("capacity reserves"); - reservation.mark_filesystem_mutation_started(); - let revision = cache.capacity_revision(); - - drop(reservation); - - { - let capacity = cache.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - assert!(!capacity.initialized); - assert!(!capacity.active_mutations.contains(&path)); - } - let wait = cache.wait_for_capacity_change(&revision); - tokio::pin!(wait); - assert!(matches!(futures::poll!(wait.as_mut()), Poll::Ready(()))); - } - - #[tokio::test] - async fn concurrent_commits_reserve_the_global_budget_exactly_once() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - cache.update_cache_limits(5, 5); - let first = SegmentCacheKey::new(ProxySessionId("first".to_string()), 1, "ts"); - let second = SegmentCacheKey::new(ProxySessionId("second".to_string()), 1, "ts"); - let first_cache = Arc::clone(&cache); - let first_task = tokio::spawn(async move { first_cache.write_bytes_and_commit(&first, b"123").await }); - let second_cache = Arc::clone(&cache); - let second_task = tokio::spawn(async move { second_cache.write_bytes_and_commit(&second, b"456").await }); - - let (first_result, second_result) = tokio::join!(first_task, second_task); - let results = [first_result.expect("first task joins"), second_result.expect("second task joins")]; - - assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1); - assert_eq!(results.iter().filter(|result| result.is_err()).count(), 1); - } - - #[tokio::test] - async fn concurrent_commit_cannot_consume_bytes_reclaimed_for_an_in_flight_writer() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let proxy_session_id = ProxySessionId("serialized-reclamation".to_string()); - let first_resident = SegmentCacheKey::new(proxy_session_id.clone(), 1, "ts"); - let second_resident = SegmentCacheKey::new(proxy_session_id.clone(), 2, "ts"); - let first_target = SegmentCacheKey::new(proxy_session_id.clone(), 3, "ts"); - let second_target = SegmentCacheKey::new(proxy_session_id.clone(), 4, "ts"); - cache.update_cache_limits(100, 100); - cache.write_bytes_and_commit(&first_resident, b"0123456789").await.expect("first resident commits"); - cache.write_bytes_and_commit(&second_resident, b"0123456789").await.expect("second resident commits"); - cache.update_cache_limits(25, 25); - - let first_staged = cache - .stage_temp_with_deadline( - &first_target, - &b"abcdefghij"[..], - tokio::time::Instant::now() + Duration::from_secs(5), - ) - .await - .expect("first target stages"); - let second_staged = cache - .stage_temp_with_deadline( - &second_target, - &b"klmnopqrst"[..], - tokio::time::Instant::now() + Duration::from_secs(5), - ) - .await - .expect("second target stages"); - let (first_reclaimed_tx, first_reclaimed_rx) = oneshot::channel(); - let (resume_first_tx, resume_first_rx) = oneshot::channel(); - let reclaimer = Arc::new(PausingCapacityReclaimer { - cache: Arc::clone(&cache), - victims: Mutex::new(VecDeque::from([first_resident, second_resident])), - first_reclaimed: Mutex::new(Some(first_reclaimed_tx)), - resume_first: Mutex::new(Some(resume_first_rx)), - }); - cache.install_capacity_reclaimer(&reclaimer); - - let first_cache = Arc::clone(&cache); - let first_task = tokio::spawn(async move { first_cache.commit_staged(&first_target, first_staged).await }); - first_reclaimed_rx.await.expect("first reclamation pauses after deleting one resident"); - - let second_cache = Arc::clone(&cache); - let second_task = tokio::spawn(async move { second_cache.commit_staged(&second_target, second_staged).await }); - for _ in 0..100 { - tokio::task::yield_now().await; - } - let paused_usage = cache.capacity_usage(&proxy_session_id).await.expect("paused capacity usage"); - assert_eq!(paused_usage.session_bytes, 10, "a later commit must wait for the reclaiming writer's retry"); - - assert!(resume_first_tx.send(()).is_ok()); - let (first_result, second_result) = tokio::join!(first_task, second_task); - assert!(first_result.expect("first task joins").is_ok()); - assert!(second_result.expect("second task joins").is_ok()); - let usage = cache.capacity_usage(&proxy_session_id).await.expect("final capacity usage"); - assert_eq!(usage.session_bytes, 20); - assert!(usage.global_bytes <= 25); - } - - #[tokio::test] - async fn gc_style_delete_defers_while_the_object_has_an_active_mutation() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let key = cache_key(); - cache.write_bytes_and_commit(&key, b"segment-body").await.expect("fixture commit"); - let cache_path = cache.cache_path_snapshot(); - let path = cache.object_path(&key); - let reservation = cache - .try_begin_capacity_mutation(&cache_path.path, &path, key.session_path_component()) - .expect("test mutation reserves"); - - let error = cache.delete_if_inactive(&key).await.expect_err("active mutation protects object"); - - assert_eq!(error.kind(), io::ErrorKind::WouldBlock); - assert!(cache.metadata(&key).await.expect("metadata").is_some()); - drop(reservation); - cache.delete_if_inactive(&key).await.expect("delete resumes after mutation"); - assert!(cache.metadata(&key).await.expect("metadata").is_none()); - } - - #[tokio::test] - async fn cache_path_change_during_write_rejects_old_generation_commit() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let old_root = temp_dir.path().join("old"); - let new_root = temp_dir.path().join("new"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(&old_root)); - let key = cache_key(); - let old_final_path = old_root.join("proxy_session/000123.ts"); - let (started_tx, started_rx) = oneshot::channel(); - let (release_tx, release_rx) = oneshot::channel(); - let reader = ControlledReader { started: Some(started_tx), release: release_rx, body: None }; - let task_cache = Arc::clone(&cache); - let task_key = key.clone(); - let write_task = tokio::spawn(async move { task_cache.write_temp_and_commit(&task_key, reader).await }); - started_rx.await.expect("write reaches controlled reader"); - - assert!(cache.update_cache_path(&new_root).await); - release_tx.send(b"segment-body".to_vec()).expect("release write"); - let error = write_task.await.expect("write task joins").expect_err("old generation commit must be rejected"); - - assert_eq!(error.kind(), io::ErrorKind::Interrupted); - assert!(!old_final_path.exists()); - assert!(cache.metadata(&key).await.expect("new-root metadata reads").is_none()); - assert!(!cache.has_active_temp_files()); - } - - #[tokio::test] - async fn owned_cache_operation_survives_caller_cancellation() { - let (started_tx, started_rx) = oneshot::channel(); - let (release_tx, release_rx) = oneshot::channel(); - let (completed_tx, completed_rx) = oneshot::channel(); - let permits = Arc::new(Semaphore::new(1)); - let caller = tokio::spawn(async move { - run_owned_cache_operation(permits, "test", async move { - started_tx.send(()).map_err(|()| io::Error::other("start receiver dropped"))?; - release_rx.await.map_err(|_| io::Error::other("release sender dropped"))?; - completed_tx.send(()).map_err(|()| io::Error::other("completion receiver dropped"))?; - Ok(()) - }) - .await - }); - started_rx.await.expect("owned operation starts"); - caller.abort(); - release_tx.send(()).expect("release owned operation"); - - completed_rx.await.expect("detached owned operation completes"); - assert!(caller.await.expect_err("caller is cancelled").is_cancelled()); - } - - #[tokio::test] - async fn saturated_owned_operation_limit_prevents_spawn_and_mutation_until_permit_release() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let key = cache_key(); - let permit_count = - u32::try_from(MAX_CONCURRENT_OWNED_CACHE_OPERATIONS).expect("owned operation limit fits u32"); - let held_permits = Arc::clone(&cache.owned_operation_permits) - .try_acquire_many_owned(permit_count) - .expect("test exclusively holds owned operation permits"); - let operation_started = Arc::new(AtomicBool::new(false)); - let operation_started_in_task = Arc::clone(&operation_started); - - let spawn_error = - run_owned_cache_operation(Arc::clone(&cache.owned_operation_permits), "saturated-test", async move { - operation_started_in_task.store(true, Ordering::SeqCst); - Ok(()) - }) - .await - .expect_err("saturated helper must reject before spawning"); - - let error = cache - .write_bytes_and_commit(&key, b"segment-body") - .await - .expect_err("saturated cache operation must be rejected"); - - assert_eq!(spawn_error.kind(), io::ErrorKind::WouldBlock); - assert!(!operation_started.load(Ordering::SeqCst)); - assert_eq!(error.kind(), io::ErrorKind::WouldBlock); - assert!(cache.metadata(&key).await.expect("metadata reads after rejection").is_none()); - assert!(!cache.has_active_temp_files()); - { - let capacity = cache.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - assert_eq!(capacity.total_bytes, 0); - assert!(!capacity.session_bytes.contains_key("proxy_session")); - assert!(capacity.active_mutations.is_empty(), "pre-spawn rejection must roll back the reservation"); - } - - drop(held_permits); - cache.write_bytes_and_commit(&key, b"segment-body").await.expect("released permits allow a later mutation"); - assert!(cache.metadata(&key).await.expect("metadata reads after commit").is_some()); - } - - #[tokio::test] - async fn open_range_reads_from_requested_offset() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let key = cache_key(); - cache.write_bytes_and_commit(&key, b"0123456789").await.expect("commit should succeed"); - - let file = cache.open_range(&key, 4).await.expect("range should open"); - let mut body = Vec::new(); - file.take(3).read_to_end(&mut body).await.expect("range should read"); - - assert_eq!(body, b"456"); - } - - #[tokio::test] - async fn temp_file_collision_does_not_overwrite_existing_temp_file() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let key = cache_key(); - let parent = temp_dir.path().join("proxy_session"); - tokio::fs::create_dir_all(&parent).await.expect("parent should be created"); - let existing_temp = parent.join("000123.ts.tmp.0000000000000000"); - tokio::fs::write(&existing_temp, b"existing").await.expect("temp fixture should write"); - - cache.write_bytes_and_commit(&key, b"segment-body").await.expect("commit should succeed"); - - assert_eq!(tokio::fs::read(&existing_temp).await.expect("existing temp should remain"), b"existing"); - } - - #[tokio::test] - async fn old_temp_file_cleanup_processes_a_bounded_batch_per_run() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let total_files = MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN.saturating_add(3); - for index in 0..total_files { - let session = if index % 2 == 0 { "session-a" } else { "session-b" }; - let directory = temp_dir.path().join(session).join(if index % 3 == 0 { "map" } else { "r" }); - tokio::fs::create_dir_all(&directory).await.expect("fixture directory writes"); - tokio::fs::write(directory.join(format!("object-{index}.ts.tmp.fixture")), b"stale") - .await - .expect("temporary fixture writes"); - } - let cutoff = SystemTime::now() + Duration::from_mins(1); - - let first = cache.delete_temp_files_older_than(cutoff).await.expect("first cleanup run succeeds"); - let second = cache.delete_temp_files_older_than(cutoff).await.expect("second cleanup run succeeds"); - let third = cache.delete_temp_files_older_than(cutoff).await.expect("third cleanup run succeeds"); - - assert_eq!(first, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN); - assert_eq!(second, total_files.saturating_sub(MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN)); - assert_eq!(third, 0); - } - - #[tokio::test] - async fn invalidate_all_if_no_active_temp_files_deletes_only_when_no_temp_write_is_active() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let committed_key = cache_key(); - cache.write_bytes_and_commit(&committed_key, b"segment-body").await.expect("commit should succeed"); - - let outcome = cache.invalidate_all_if_no_active_temp_files().await.expect("invalidation should succeed"); - - assert_eq!(outcome, CacheInvalidationOutcome::Invalidated); - assert_eq!(cache.metadata(&committed_key).await.expect("metadata should read"), None); - - cache.write_bytes_and_commit(&committed_key, b"segment-body").await.expect("second commit should succeed"); - let active_key = SegmentCacheKey::new(ProxySessionId("proxy_session".to_string()), 124, "ts"); - let staged = cache - .stage_temp_with_deadline(&active_key, &b"done"[..], tokio::time::Instant::now() + Duration::from_mins(1)) - .await - .expect("active object stages"); - assert!(cache.has_active_temp_files()); - - let outcome = - cache.invalidate_all_if_no_active_temp_files().await.expect("deferred invalidation should succeed"); - - assert_eq!(outcome, CacheInvalidationOutcome::DeferredActiveTempFiles); - assert!(cache.metadata(&committed_key).await.expect("metadata should read").is_some()); - cache.commit_staged(&active_key, staged).await.expect("staged object commits"); - } - - #[tokio::test] - async fn invalidate_all_refuses_unmarked_cache_root() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let unrelated_file = temp_dir.path().join("unrelated"); - tokio::fs::write(&unrelated_file, b"keep").await.expect("fixture should write"); - - let err = cache.invalidate_all().await.expect_err("unmarked root should be refused"); - - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - assert!(unrelated_file.exists()); - } - - #[tokio::test] - async fn delete_removes_committed_file_idempotently() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let key = cache_key(); - cache.write_bytes_and_commit(&key, b"segment-body").await.expect("commit should succeed"); - - cache.delete(&key).await.expect("delete should succeed"); - cache.delete(&key).await.expect("second delete should be idempotent"); - - assert_eq!(cache.metadata(&key).await.expect("metadata should read"), None); - } +/// A peak reservation includes raw data, cache staging, repair output and immutable revisions. +pub struct HlsRevisionDiskReservation { + id: u64, + capacity: Arc>, + changed: Arc, + /// Set when bytes were written under this reservation outside the staged commit accounting. + unaccounted_writes: bool, } + +pub struct HlsRevisionFilePin { + _registration: Arc, +} + +#[cfg(test)] +mod tests; + +#[cfg(test)] +mod revision_capacity_tests; + +mod capacity; +mod error; +mod keys; +mod lifecycle; +mod revisions; +mod staging; +#[cfg(test)] +use capacity::CapacityReservationError; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use capacity::{ + CacheInvalidationOutcome, HlsCacheCapacityReclaimOutcome, HlsCacheCapacityReclaimRequest, + HlsCacheCapacityReclaimer, HlsCacheCapacityUsage, +}; +pub use error::{hls_cache_capacity_from_io, hls_cache_object_limit_from_io}; +pub use keys::HlsCacheObjectKey; +pub use staging::CachedSegmentMetadata; diff --git a/backend/hls/src/cache/capacity.rs b/backend/hls/src/cache/capacity.rs new file mode 100644 index 000000000..dbb503968 --- /dev/null +++ b/backend/hls/src/cache/capacity.rs @@ -0,0 +1,508 @@ +use super::{ + error::{cache_object_limit_error, capacity_error}, + hls_cache_capacity_from_io, + lifecycle::{is_temp_cache_file, REWRITE_SECRET_FINGERPRINT_FILE}, + revisions::{is_revision_cache_path, revision_reserved_bytes}, + safe_proxy_session_id, CacheCapacityPressure, CacheCapacityState, CapacityInvalidationGuard, + CapacityMutationReservation, CapacityRevision, HlsCacheCapacityRevision, HlsCacheObjectKey, HlsSegmentCache, + ProxySessionId, +}; +use futures::future::BoxFuture; +use std::{ + collections::HashMap, + io, + path::{Path, PathBuf}, + sync::{atomic::Ordering, Arc, Mutex as StdMutex, Weak}, +}; +use tokio::{fs, sync::Notify}; + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsCacheCapacityReclaimRequest { + pub proxy_session_id: ProxySessionId, + pub target_path: PathBuf, + pub required_session_bytes: u64, + pub required_global_bytes: u64, + pub staged_bytes: u64, +} + +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] +#[allow(clippy::struct_field_names)] +pub struct HlsCacheCapacityReclaimOutcome { + pub reclaimed_session_bytes: u64, + pub reclaimed_global_bytes: u64, + pub protected_working_set_bytes: u64, + pub reclaimable_bytes: u64, +} + +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] +pub struct HlsCacheCapacityUsage { + pub session_bytes: u64, + pub global_bytes: u64, +} + +/// Existing GC integration used to make room for one already-staged cache object. +pub trait HlsCacheCapacityReclaimer: Send + Sync { + fn reclaim_capacity( + &self, + request: HlsCacheCapacityReclaimRequest, + ) -> BoxFuture<'_, io::Result>; +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum CacheInvalidationOutcome { + Invalidated, + DeferredActiveTempFiles, +} + +impl Drop for CapacityInvalidationGuard { + fn drop(&mut self) { invalidate_capacity_state(&self.capacity, &self.changed); } +} + +impl CapacityMutationReservation { + pub(super) fn projected_usage(&self) -> HlsCacheCapacityUsage { + let capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + HlsCacheCapacityUsage { + session_bytes: capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(), + global_bytes: capacity.total_bytes, + } + } + + pub(super) fn reserve_replacement( + &mut self, + old_size: u64, + new_size: u64, + max_cache_bytes: u64, + max_session_bytes: u64, + ) -> Result<(), CapacityReservationError> { + let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if !capacity.initialized || capacity.cache_path != self.cache_path { + return Err(CapacityReservationError::Retry); + } + if !capacity.active_mutations.contains(&self.path) { + return Err(CapacityReservationError::Invalidated); + } + let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); + let projected_total = capacity.total_bytes.saturating_sub(old_size).saturating_add(new_size); + let projected_session = session_size.saturating_sub(old_size).saturating_add(new_size); + let (reserved_global, reserved_session) = revision_reserved_bytes(&capacity, &self.session_component); + if projected_total.saturating_add(reserved_global) > max_cache_bytes + || projected_session.saturating_add(reserved_session) > max_session_bytes + { + return Err(CapacityReservationError::Exceeded { + pressure: CacheCapacityPressure { + configured_session_bytes: max_session_bytes, + configured_global_bytes: max_cache_bytes, + current_session_bytes: session_size, + current_global_bytes: capacity.total_bytes, + staged_bytes: new_size, + required_session_bytes: projected_session + .saturating_add(reserved_session) + .saturating_sub(max_session_bytes), + required_global_bytes: projected_total + .saturating_add(reserved_global) + .saturating_sub(max_cache_bytes), + }, + // Capture the opaque token in the same critical section as the + // failed admission decision. A later accounting mutation must + // make this token stale instead of being missed by the waiter. + revision: HlsCacheCapacityRevision(Arc::clone(&capacity.revision)), + }); + } + capacity.total_bytes = projected_total; + store_session_bytes(&mut capacity, &self.session_component, projected_session); + self.replacement = Some((old_size, new_size)); + Ok(()) + } + + pub(super) fn mark_filesystem_mutation_started(&mut self) { self.filesystem_mutation_started = true; } + + pub(super) fn finish_replacement(mut self) { + self.replacement = None; + self.finish_mutation(None); + } + + pub(super) fn finish_delete(mut self, deleted_size: u64) { + self.finish_mutation(Some(deleted_size)); + self.replacement = None; + } + + pub(super) fn finish_mutation(&mut self, deleted_size: Option) { + let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if !capacity.active_mutations.contains(&self.path) { + return; + } + capacity.active_mutations.remove(&self.path); + if capacity.initialized && capacity.cache_path == self.cache_path { + if let Some(deleted_size) = deleted_size { + capacity.total_bytes = capacity.total_bytes.saturating_sub(deleted_size); + let session_bytes = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); + store_session_bytes(&mut capacity, &self.session_component, session_bytes.saturating_sub(deleted_size)); + } + } + capacity.revision = Arc::new(CapacityRevision); + drop(capacity); + self.changed.notify_waiters(); + } +} + +impl Drop for CapacityMutationReservation { + fn drop(&mut self) { + let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if !capacity.active_mutations.contains(&self.path) { + return; + } + capacity.active_mutations.remove(&self.path); + let revision_changed = if self.filesystem_mutation_started { + capacity.initialized = false; + true + } else if capacity.initialized && capacity.cache_path == self.cache_path { + if let Some((old_size, new_size)) = self.replacement.take() { + capacity.total_bytes = capacity.total_bytes.saturating_sub(new_size).saturating_add(old_size); + let session_size = capacity.session_bytes.get(&self.session_component).copied().unwrap_or_default(); + store_session_bytes( + &mut capacity, + &self.session_component, + session_size.saturating_sub(new_size).saturating_add(old_size), + ); + true + } else { + false + } + } else { + false + }; + if revision_changed { + capacity.revision = Arc::new(CapacityRevision); + } + drop(capacity); + // Mutation waiters share this notifier with revision waiters. A no-op + // release must wake the former, while the latter observe the unchanged + // opaque token and continue waiting. + self.changed.notify_waiters(); + } +} + +#[derive(Debug, Clone)] +pub(super) enum CapacityReservationError { + Retry, + Invalidated, + Exceeded { pressure: CacheCapacityPressure, revision: HlsCacheCapacityRevision }, +} + +fn store_session_bytes(capacity: &mut CacheCapacityState, session_component: &str, bytes: u64) { + if bytes == 0 { + capacity.session_bytes.remove(session_component); + } else { + capacity.session_bytes.insert(session_component.to_string(), bytes); + } +} + +impl HlsSegmentCache { + /// Performs capacity admission from reliable response metadata before the + /// decoded body is consumed. The authoritative staged commit repeats the + /// same reservation after download, so concurrent mutations cannot create + /// an overshoot. + pub async fn ensure_projected_write_capacity(&self, key: &K, content_length: u64) -> io::Result<()> + where + K: HlsCacheObjectKey, + { + let max_object_bytes = self.max_object_bytes.load(Ordering::Acquire); + if content_length > max_object_bytes { + let configured_session_bytes = self.max_session_bytes.load(Ordering::Acquire); + let configured_global_bytes = self.max_cache_bytes.load(Ordering::Acquire); + let usage = self.capacity_usage(key.proxy_session_id()).await.unwrap_or_default(); + log::warn!( + "HLS cache capacity decision: proxy_session={} resource={} configured_session_bytes={} configured_global_bytes={} current_session_bytes={} current_global_bytes={} staged_bytes={} protected_working_set_bytes=0 reclaimable_bytes=0 required_session_bytes={} required_global_bytes={} outcome=permanently-infeasible", + safe_proxy_session_id(key.proxy_session_id()), + key.file_name(), + configured_session_bytes, + configured_global_bytes, + usage.session_bytes, + usage.global_bytes, + content_length, + content_length.saturating_sub(configured_session_bytes), + content_length.saturating_sub(configured_global_bytes), + ); + return Err(cache_object_limit_error(max_object_bytes)); + } + + let _admission_gate = self.capacity_admission_gate.lock().await; + let mut reclamation_attempted = false; + let mut reclamation_outcome = HlsCacheCapacityReclaimOutcome::default(); + let mut reclamation_pressure: Option = None; + loop { + let cache_path = self.cache_path_snapshot(); + let final_path = Self::path_for_key_in(&cache_path.path, key); + match self + .prepare_capacity_replacement( + &cache_path.path, + &final_path, + key.session_path_component(), + content_length, + self.max_cache_bytes.load(Ordering::Acquire), + self.max_session_bytes.load(Ordering::Acquire), + ) + .await + { + Ok(reservation) => { + drop(reservation); + if let Some(pressure) = reclamation_pressure { + let usage = self.capacity_usage(key.proxy_session_id()).await.unwrap_or_default(); + log::info!( + "HLS cache capacity decision: proxy_session={} resource={} configured_session_bytes={} configured_global_bytes={} current_session_bytes={} current_global_bytes={} staged_bytes={} protected_working_set_bytes={} reclaimable_bytes={} required_session_bytes={} required_global_bytes={} outcome=reclaimed", + safe_proxy_session_id(key.proxy_session_id()), + key.file_name(), + pressure.configured_session_bytes, + pressure.configured_global_bytes, + usage.session_bytes, + usage.global_bytes, + content_length, + reclamation_outcome.protected_working_set_bytes, + reclamation_outcome.reclaimable_bytes, + pressure.required_session_bytes, + pressure.required_global_bytes, + ); + } + return Ok(()); + } + Err(error) => { + let Some(capacity) = hls_cache_capacity_from_io(&error) else { + return Err(error); + }; + if reclamation_attempted { + log::warn!( + "HLS cache capacity decision: proxy_session={} resource={} configured_session_bytes={} configured_global_bytes={} current_session_bytes={} current_global_bytes={} staged_bytes={} protected_working_set_bytes={} reclaimable_bytes={} required_session_bytes={} required_global_bytes={} outcome=deferred-protected wake_reason=capacity-or-protection-revision", + safe_proxy_session_id(key.proxy_session_id()), + key.file_name(), + capacity.configured_session_bytes(), + capacity.configured_global_bytes(), + capacity.current_session_bytes(), + capacity.current_global_bytes(), + capacity.staged_bytes(), + reclamation_outcome.protected_working_set_bytes, + reclamation_outcome.reclaimable_bytes, + capacity.required_session_bytes(), + capacity.required_global_bytes(), + ); + return Err(capacity_error( + capacity.pressure(), + reclamation_outcome, + capacity.revision().clone(), + )); + } + reclamation_attempted = true; + reclamation_pressure = Some(capacity.pressure()); + reclamation_outcome = self + .reclaim_capacity(HlsCacheCapacityReclaimRequest { + proxy_session_id: key.proxy_session_id().clone(), + target_path: final_path, + required_session_bytes: capacity.required_session_bytes(), + required_global_bytes: capacity.required_global_bytes(), + staged_bytes: content_length, + }) + .await?; + } + } + } + } + + pub(super) async fn reclaim_capacity( + &self, + request: HlsCacheCapacityReclaimRequest, + ) -> io::Result { + let reclaimer = self + .capacity_reclaimer + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .and_then(Weak::upgrade); + match reclaimer { + Some(reclaimer) => reclaimer.reclaim_capacity(request).await, + // Standalone cache users have no session graph to reclaim. This is + // a zero-reclamation capacity result, not a storage I/O failure. + None => Ok(HlsCacheCapacityReclaimOutcome::default()), + } + } + + pub(super) async fn ensure_capacity_initialized(&self, cache_path: &Path) -> io::Result<()> { + loop { + let notified = self.capacity_changed.notified(); + let revision = { + let capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if capacity.initialized && capacity.cache_path == cache_path { + return Ok(()); + } + if capacity.active_mutations.iter().any(|path| path.starts_with(cache_path)) { + None + } else { + Some(Arc::clone(&capacity.revision)) + } + }; + let Some(revision) = revision else { + notified.await; + continue; + }; + let (total_bytes, session_bytes, revision_bytes) = scan_committed_cache_usage(cache_path).await?; + if self.cache_path_snapshot().path != cache_path { + return Err(io::Error::new(io::ErrorKind::Interrupted, "hls cache path changed during usage scan")); + } + let scan_invalidated = { + let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if !Arc::ptr_eq(&capacity.revision, &revision) + || capacity.active_mutations.iter().any(|path| path.starts_with(cache_path)) + { + true + } else { + capacity.cache_path = cache_path.to_path_buf(); + capacity.initialized = true; + capacity.total_bytes = total_bytes; + capacity.session_bytes = session_bytes; + capacity.revision_bytes = revision_bytes; + false + } + }; + if scan_invalidated { + // The scan result was invalidated by a real accounting or + // mutation transition. Wait on the notification registered + // before the scan instead of immediately walking the complete + // cache tree again under sustained commit churn. + notified.await; + continue; + } + return Ok(()); + } + } + + pub(super) async fn begin_capacity_mutation( + &self, + cache_path: &Path, + path: &Path, + session_component: String, + ) -> CapacityMutationReservation { + loop { + let notified = self.capacity_changed.notified(); + let Some(reservation) = self.try_begin_capacity_mutation(cache_path, path, session_component.clone()) + else { + notified.await; + continue; + }; + return reservation; + } + } + + pub(super) fn try_begin_capacity_mutation( + &self, + cache_path: &Path, + path: &Path, + session_component: String, + ) -> Option { + let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if !capacity.active_mutations.insert(path.to_path_buf()) { + return None; + } + drop(capacity); + Some(CapacityMutationReservation { + path: path.to_path_buf(), + cache_path: cache_path.to_path_buf(), + session_component, + replacement: None, + filesystem_mutation_started: false, + capacity: Arc::clone(&self.capacity), + changed: Arc::clone(&self.capacity_changed), + }) + } + + pub(super) async fn prepare_capacity_replacement( + &self, + cache_path: &Path, + final_path: &Path, + session_component: String, + new_size: u64, + max_cache_bytes: u64, + max_session_bytes: u64, + ) -> io::Result { + loop { + self.ensure_capacity_initialized(cache_path).await?; + let mut reservation = self.begin_capacity_mutation(cache_path, final_path, session_component.clone()).await; + let old_size = match fs::metadata(final_path).await { + Ok(metadata) => metadata.len(), + Err(err) if err.kind() == io::ErrorKind::NotFound => 0, + Err(err) => return Err(err), + }; + if self.cache_path_snapshot().path != cache_path { + return Err(io::Error::new(io::ErrorKind::Interrupted, "hls cache path changed during object write")); + } + match reservation.reserve_replacement(old_size, new_size, max_cache_bytes, max_session_bytes) { + Ok(()) => return Ok(reservation), + Err(CapacityReservationError::Retry | CapacityReservationError::Invalidated) => { + drop(reservation); + } + Err(CapacityReservationError::Exceeded { pressure, revision }) => { + return Err(capacity_error(pressure, HlsCacheCapacityReclaimOutcome::default(), revision)); + } + } + } + } + + pub(super) fn invalidate_capacity_accounting(&self) { + invalidate_capacity_state(&self.capacity, &self.capacity_changed); + } + + pub(super) fn capacity_invalidation_guard(&self) -> CapacityInvalidationGuard { + CapacityInvalidationGuard { capacity: Arc::clone(&self.capacity), changed: Arc::clone(&self.capacity_changed) } + } + + pub(super) fn rewrite_secret_fingerprint_path(&self) -> PathBuf { + self.cache_path_snapshot().path.join(REWRITE_SECRET_FINGERPRINT_FILE) + } +} + +fn invalidate_capacity_state(capacity: &StdMutex, changed: &Notify) { + let mut capacity = capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + capacity.initialized = false; + capacity.revision = Arc::new(CapacityRevision); + drop(capacity); + changed.notify_waiters(); +} + +pub(super) async fn scan_committed_cache_usage(cache_path: &Path) -> io::Result<(u64, HashMap, u64)> { + let mut total_bytes = 0_u64; + let mut revision_bytes = 0_u64; + let mut session_bytes = HashMap::new(); + let mut root_entries = match fs::read_dir(cache_path).await { + Ok(entries) => entries, + Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok((0, session_bytes, 0)), + Err(err) => return Err(err), + }; + while let Some(root_entry) = root_entries.next_entry().await? { + if !root_entry.file_type().await?.is_dir() { + continue; + } + let session_component = root_entry.file_name().to_string_lossy().into_owned(); + let mut bytes = 0_u64; + let mut pending_dirs = vec![root_entry.path()]; + while let Some(dir) = pending_dirs.pop() { + let mut entries = fs::read_dir(dir).await?; + while let Some(entry) = entries.next_entry().await? { + let file_type = entry.file_type().await?; + if file_type.is_dir() { + pending_dirs.push(entry.path()); + } else if file_type.is_file() { + let revision = is_revision_cache_path(&entry.path()); + if revision || !is_temp_cache_file(&entry.path()) { + let size = entry.metadata().await?.len(); + bytes = bytes.saturating_add(size); + if revision { + revision_bytes = revision_bytes.saturating_add(size); + } + } + } + } + } + total_bytes = total_bytes.saturating_add(bytes); + if bytes > 0 { + session_bytes.insert(session_component, bytes); + } + } + Ok((total_bytes, session_bytes, revision_bytes)) +} diff --git a/backend/hls/src/cache/error.rs b/backend/hls/src/cache/error.rs new file mode 100644 index 000000000..32f01cbf3 --- /dev/null +++ b/backend/hls/src/cache/error.rs @@ -0,0 +1,103 @@ +#[cfg(any(test, feature = "test-support"))] +use super::CapacityRevision; +use super::{ + CacheCapacityPressure, HlsCacheCapacityError, HlsCacheCapacityReclaimOutcome, HlsCacheCapacityRevision, + HlsCacheObjectLimitError, +}; +#[cfg(any(test, feature = "test-support"))] +use std::sync::Arc; +use std::{fmt, io}; + +impl HlsCacheObjectLimitError { + pub fn limit(&self) -> u64 { self.limit } +} + +pub fn hls_cache_object_limit_from_io(error: &io::Error) -> Option<&HlsCacheObjectLimitError> { + let mut source: &(dyn std::error::Error + 'static) = error.get_ref()?; + loop { + if let Some(limit_error) = source.downcast_ref::() { + return Some(limit_error); + } + source = source.source()?; + } +} + +pub(super) fn cache_object_limit_error(limit: u64) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, HlsCacheObjectLimitError { limit }) +} + +impl fmt::Debug for HlsCacheCapacityRevision { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("HlsCacheCapacityRevision()") + } +} + +impl HlsCacheCapacityRevision { + #[cfg(any(test, feature = "test-support"))] + pub fn for_test() -> Self { Self(Arc::new(CapacityRevision)) } +} + +impl HlsCacheCapacityError { + pub fn configured_session_bytes(&self) -> u64 { self.configured_session_bytes } + + pub fn configured_global_bytes(&self) -> u64 { self.configured_global_bytes } + + pub fn current_session_bytes(&self) -> u64 { self.current_session_bytes } + + pub fn current_global_bytes(&self) -> u64 { self.current_global_bytes } + + pub fn staged_bytes(&self) -> u64 { self.staged_bytes } + + pub fn required_session_bytes(&self) -> u64 { self.required_session_bytes } + + pub fn required_global_bytes(&self) -> u64 { self.required_global_bytes } + + pub fn revision(&self) -> &HlsCacheCapacityRevision { &self.revision } + + #[cfg(any(test, feature = "test-support"))] + pub fn protected_working_set_bytes(&self) -> u64 { self.protected_working_set_bytes } + + #[cfg(any(test, feature = "test-support"))] + pub fn reclaimable_bytes(&self) -> u64 { self.reclaimable_bytes } + + pub(super) fn pressure(&self) -> CacheCapacityPressure { + CacheCapacityPressure { + configured_session_bytes: self.configured_session_bytes, + configured_global_bytes: self.configured_global_bytes, + current_session_bytes: self.current_session_bytes, + current_global_bytes: self.current_global_bytes, + staged_bytes: self.staged_bytes, + required_session_bytes: self.required_session_bytes, + required_global_bytes: self.required_global_bytes, + } + } +} + +pub fn hls_cache_capacity_from_io(error: &io::Error) -> Option<&HlsCacheCapacityError> { + let mut source: &(dyn std::error::Error + 'static) = error.get_ref()?; + loop { + if let Some(capacity_error) = source.downcast_ref::() { + return Some(capacity_error); + } + source = source.source()?; + } +} + +pub(super) fn capacity_error( + pressure: CacheCapacityPressure, + reclaim: HlsCacheCapacityReclaimOutcome, + revision: HlsCacheCapacityRevision, +) -> io::Error { + io::Error::other(HlsCacheCapacityError { + configured_session_bytes: pressure.configured_session_bytes, + configured_global_bytes: pressure.configured_global_bytes, + current_session_bytes: pressure.current_session_bytes, + current_global_bytes: pressure.current_global_bytes, + staged_bytes: pressure.staged_bytes, + required_session_bytes: pressure.required_session_bytes, + required_global_bytes: pressure.required_global_bytes, + protected_working_set_bytes: reclaim.protected_working_set_bytes, + reclaimable_bytes: reclaim.reclaimable_bytes, + revision, + }) +} diff --git a/backend/hls/src/cache/keys.rs b/backend/hls/src/cache/keys.rs new file mode 100644 index 000000000..6df3e71d6 --- /dev/null +++ b/backend/hls/src/cache/keys.rs @@ -0,0 +1,121 @@ +use super::{MapCacheKey, ProxyMapId, ProxySessionId, SegmentCacheKey, TransientObjectCacheKey, TransientResourceId}; +use std::fmt; + +impl SegmentCacheKey { + pub fn new(proxy_session_id: ProxySessionId, proxy_seq: u64, proxy_file_ext: impl Into) -> Self { + Self { session_id: proxy_session_id, seq: proxy_seq, file_ext: proxy_file_ext.into() } + } + + pub fn stable_value(&self) -> String { format!("hls:{}:{:020}", self.session_id.0, self.seq) } + + pub fn proxy_session_id(&self) -> &ProxySessionId { &self.session_id } + + pub fn proxy_seq(&self) -> u64 { self.seq } +} + +impl fmt::Debug for SegmentCacheKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SegmentCacheKey") + .field("session_id", &"") + .field("seq", &self.seq) + .field("file_ext", &self.file_ext) + .finish() + } +} + +impl MapCacheKey { + pub fn new( + proxy_session_id: ProxySessionId, + proxy_map_id: impl Into, + proxy_file_ext: impl Into, + ) -> Self { + Self { session_id: proxy_session_id, map_id: proxy_map_id.into(), file_ext: proxy_file_ext.into() } + } + + pub fn stable_value(&self) -> String { format!("hls-map:{}:{:020}", self.session_id.0, self.map_id.0) } + + pub fn proxy_map_id(&self) -> ProxyMapId { self.map_id } +} + +impl fmt::Debug for MapCacheKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MapCacheKey") + .field("session_id", &"") + .field("map_id", &self.map_id.0) + .field("file_ext", &self.file_ext) + .finish() + } +} + +impl TransientObjectCacheKey { + pub fn new( + proxy_session_id: ProxySessionId, + transient_resource_id: TransientResourceId, + proxy_file_ext: impl Into, + ) -> Self { + Self { session_id: proxy_session_id, resource_id: transient_resource_id, file_ext: proxy_file_ext.into() } + } + + pub fn stable_value(&self) -> String { format!("hls-transient:{}:{}", self.session_id.0, self.resource_id.0) } + + pub fn transient_resource_id(&self) -> &TransientResourceId { &self.resource_id } +} + +impl fmt::Debug for TransientObjectCacheKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TransientObjectCacheKey") + .field("session_id", &"") + .field("resource_id", &self.resource_id) + .field("file_ext", &self.file_ext) + .finish() + } +} + +/// Resolves a proxy cache object to a safe path below the configured HLS cache root. +pub trait HlsCacheObjectKey { + fn proxy_session_id(&self) -> &ProxySessionId; + fn session_path_component(&self) -> String; + fn file_name(&self) -> String; +} + +impl HlsCacheObjectKey for SegmentCacheKey { + fn proxy_session_id(&self) -> &ProxySessionId { &self.session_id } + + fn session_path_component(&self) -> String { + let value = &self.session_id.0; + if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) { + return value.clone(); + } + blake3::hash(value.as_bytes()).to_hex().to_string() + } + + fn file_name(&self) -> String { format!("{:06}.{}", self.seq, self.file_ext) } +} + +impl HlsCacheObjectKey for MapCacheKey { + fn proxy_session_id(&self) -> &ProxySessionId { &self.session_id } + + fn session_path_component(&self) -> String { + let value = &self.session_id.0; + if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) { + return value.clone(); + } + blake3::hash(value.as_bytes()).to_hex().to_string() + } + + fn file_name(&self) -> String { format!("map/{:06}.{}", self.map_id.0, self.file_ext) } +} + +impl HlsCacheObjectKey for TransientObjectCacheKey { + fn proxy_session_id(&self) -> &ProxySessionId { &self.session_id } + + fn session_path_component(&self) -> String { + let value = &self.session_id.0; + if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) { + return value.clone(); + } + blake3::hash(value.as_bytes()).to_hex().to_string() + } + + fn file_name(&self) -> String { format!("r/{}.{}", self.resource_id.0, self.file_ext) } +} diff --git a/backend/hls/src/cache/lifecycle.rs b/backend/hls/src/cache/lifecycle.rs new file mode 100644 index 000000000..60a6a6d3b --- /dev/null +++ b/backend/hls/src/cache/lifecycle.rs @@ -0,0 +1,580 @@ +use super::{ + revisions::is_revision_cache_path, staging::run_owned_cache_operation, ActiveTempFileRegistration, + CacheCapacityState, CacheInvalidationOutcome, CachePathDeletionReservation, CachePathGeneration, CachePathState, + CachedSegmentMetadata, CapacityMutationReservation, CapacityRevision, HlsCacheCapacityReclaimer, + HlsCacheCapacityRevision, HlsCacheCapacityUsage, HlsCacheObjectKey, HlsSegmentCache, ProxySessionId, TempFileState, + DEFAULT_HLS_CACHE_PATH, MAX_CONCURRENT_OWNED_CACHE_OPERATIONS, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN, +}; +use log::warn; +use std::{ + collections::HashSet, + io, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex as StdMutex, RwLock as StdRwLock, + }, + time::SystemTime, +}; +use tokio::{ + fs, + sync::{Mutex as AsyncMutex, Notify, RwLock, Semaphore}, +}; + +impl HlsSegmentCache { + pub fn new() -> Self { Self::with_cache_path(DEFAULT_HLS_CACHE_PATH) } + + pub fn with_cache_path(cache_path: impl Into) -> Self { + Self { + cache_path: StdRwLock::new(CachePathState { + path: cache_path.into(), + generation: Arc::new(CachePathGeneration), + }), + cache_path_commit_gate: Arc::new(RwLock::new(())), + temp_files: Arc::new(StdMutex::new(TempFileState::default())), + max_object_bytes: AtomicU64::new(u64::MAX), + max_cache_bytes: AtomicU64::new(u64::MAX), + max_session_bytes: AtomicU64::new(u64::MAX), + marker_path: StdRwLock::new(None), + capacity: Arc::new(StdMutex::new(CacheCapacityState::default())), + capacity_changed: Arc::new(Notify::new()), + capacity_reclaimer: StdRwLock::new(None), + capacity_admission_gate: AsyncMutex::new(()), + owned_operation_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_OWNED_CACHE_OPERATIONS)), + } + } + + pub fn install_capacity_reclaimer(&self, reclaimer: &Arc) + where + T: HlsCacheCapacityReclaimer + 'static, + { + let reclaimer: Arc = reclaimer.clone(); + *self.capacity_reclaimer.write().unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Arc::downgrade(&reclaimer)); + } + + #[cfg(any(test, feature = "test-support"))] + pub fn capacity_revision(&self) -> HlsCacheCapacityRevision { + let capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + HlsCacheCapacityRevision(Arc::clone(&capacity.revision)) + } + + /// Waits until accounting or playback protection has changed since a + /// capacity deferral. Registering with `Notify` before comparing tokens + /// prevents a missed wake between the failed commit and this wait. + pub async fn wait_for_capacity_change(&self, revision: &HlsCacheCapacityRevision) { + loop { + let notified = self.capacity_changed.notified(); + let changed = { + let capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + !Arc::ptr_eq(&capacity.revision, &revision.0) + }; + if changed { + return; + } + notified.await; + } + } + + /// Announces a cursor/window change which may release protected cache + /// objects. No accounting totals are modified by this operation. + pub fn notify_capacity_protection_changed(&self) { + let mut capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + capacity.revision = Arc::new(CapacityRevision); + drop(capacity); + self.capacity_changed.notify_waiters(); + } + + pub fn cache_path(&self) -> PathBuf { self.cache_path_snapshot().path } + + pub async fn update_cache_path(&self, cache_path: impl Into) -> bool { + let cache_path = cache_path.into(); + // Commits hold a read lease from their final generation check through the atomic rename. Taking the write + // lease makes a cache-root transition linearizable without holding a filesystem/accounting mutex over I/O. + let _transition = self.cache_path_commit_gate.write().await; + let mut current = self.cache_path.write().unwrap_or_else(std::sync::PoisonError::into_inner); + if current.path == cache_path { + return false; + } + current.path = cache_path; + current.generation = Arc::new(CachePathGeneration); + drop(current); + *self.marker_path.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + self.invalidate_capacity_accounting(); + true + } + + pub fn update_cache_limits(&self, max_cache_bytes: u64, max_session_bytes: u64) { + let max_cache_bytes = max_cache_bytes.max(1); + let max_session_bytes = max_session_bytes.max(1); + self.max_cache_bytes.store(max_cache_bytes, Ordering::Release); + self.max_session_bytes.store(max_session_bytes, Ordering::Release); + self.max_object_bytes.store(max_cache_bytes.min(max_session_bytes), Ordering::Release); + } + + pub async fn metadata(&self, key: &K) -> io::Result> { + let path = self.path_for_key(key); + match fs::metadata(&path).await { + Ok(metadata) => Ok(Some(CachedSegmentMetadata { path, size: metadata.len() })), + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + } + } + + pub async fn delete(&self, key: &K) -> io::Result<()> { + let cache_path = self.cache_path_snapshot(); + let path = Self::path_for_key_in(&cache_path.path, key); + let reservation = self.begin_capacity_mutation(&cache_path.path, &path, key.session_path_component()).await; + self.delete_with_reservation(path, reservation).await + } + + pub async fn delete_if_inactive(&self, key: &K) -> io::Result<()> { + let cache_path = self.cache_path_snapshot(); + let path = Self::path_for_key_in(&cache_path.path, key); + let reservation = self + .try_begin_capacity_mutation(&cache_path.path, &path, key.session_path_component()) + .ok_or_else(|| io::Error::new(io::ErrorKind::WouldBlock, "hls cache object has an active mutation"))?; + self.delete_with_reservation(path, reservation).await + } + + pub(super) async fn delete_with_reservation( + &self, + path: PathBuf, + reservation: CapacityMutationReservation, + ) -> io::Result<()> { + run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "delete", async move { + let size = match fs::metadata(&path).await { + Ok(metadata) => metadata.len(), + Err(err) if err.kind() == io::ErrorKind::NotFound => 0, + Err(err) => return Err(err), + }; + let mut reservation = reservation; + reservation.mark_filesystem_mutation_started(); + match fs::remove_file(&path).await { + Ok(()) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => return Err(err), + } + reservation.finish_delete(size); + Ok(()) + }) + .await + } + + pub fn object_path(&self, key: &K) -> PathBuf { self.path_for_key(key) } + + pub fn contains_current_cache_path(&self, path: &Path) -> bool { path.starts_with(self.cache_path_snapshot().path) } + + pub fn has_active_mutation(&self, path: &Path) -> bool { + self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner).active_mutations.contains(path) + } + + pub async fn capacity_usage(&self, proxy_session_id: &ProxySessionId) -> io::Result { + let cache_path = self.cache_path_snapshot(); + self.ensure_capacity_initialized(&cache_path.path).await?; + let capacity = self.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if !capacity.initialized || capacity.cache_path != cache_path.path { + return Err(io::Error::new(io::ErrorKind::Interrupted, "hls cache usage snapshot invalidated")); + } + Ok(HlsCacheCapacityUsage { + session_bytes: capacity + .session_bytes + .get(&safe_session_path_component(proxy_session_id)) + .copied() + .unwrap_or_default(), + global_bytes: capacity.total_bytes, + }) + } + + pub async fn delete_temp_files_older_than(&self, cutoff: SystemTime) -> io::Result { + let cache_path = self.cache_path_snapshot(); + let candidates = old_temp_files(&cache_path.path, cutoff, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN).await?; + let mut deleted = 0_usize; + let mut removed_revision = false; + for path in candidates { + let revision = is_revision_cache_path(&path); + let Some(deletion) = self.reserve_path_deletion(&path) else { + continue; + }; + let result = run_owned_cache_operation( + Arc::clone(&self.owned_operation_permits), + "temporary-file-delete", + async move { + let result = fs::remove_file(path).await; + drop(deletion); + result + }, + ) + .await; + match result { + Ok(()) => { + deleted = deleted.saturating_add(1); + removed_revision |= revision; + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => warn!("HLS temporary cache file cleanup deferred: error_kind={:?}", error.kind()), + } + } + if removed_revision { + drop(self.capacity_invalidation_guard()); + } + Ok(deleted) + } + + pub async fn delete_session_dir(&self, proxy_session_id: &ProxySessionId) -> io::Result<()> { + let cache_path = self.cache_path_snapshot(); + let path = cache_path.path.join(safe_session_path_component(proxy_session_id)); + let deletion = self.reserve_path_deletion(&path).ok_or_else(|| { + io::Error::new(io::ErrorKind::WouldBlock, "hls cache session directory still has active temp files") + })?; + let capacity_invalidation = self.capacity_invalidation_guard(); + run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "session-delete", async move { + let _capacity_invalidation = capacity_invalidation; + let result = match fs::remove_dir_all(path).await { + Ok(()) => Ok(()), + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(err), + }; + drop(deletion); + result + }) + .await + } + + pub async fn delete_orphan_session_dirs( + &self, + active_session_ids: &HashSet, + freshness_cutoff: SystemTime, + ) -> io::Result { + let cache_path = self.cache_path_snapshot(); + ensure_safe_cache_root(&cache_path.path).await?; + let active_paths = active_session_ids + .iter() + .map(|id| cache_path.path.join(safe_session_path_component(id))) + .collect::>(); + let mut entries = fs::read_dir(&cache_path.path).await?; + let mut removed = 0_usize; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + let file_type = match entry.file_type().await { + Ok(file_type) => file_type, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => { + warn!("HLS orphan session entry inspection deferred: error_kind={:?}", error.kind()); + continue; + } + }; + if !file_type.is_dir() || active_paths.contains(&path) { + continue; + } + // Freshness guard: skip directories committed after the GC took its + // in-memory session snapshot. Their owning session may not yet be + // visible in `active_session_ids`, so deleting them would race with + // a concurrent segment write that just created the directory. + let metadata = match entry.metadata().await { + Ok(metadata) => metadata, + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(error) => { + warn!("HLS orphan session metadata inspection deferred: error_kind={:?}", error.kind()); + continue; + } + }; + if let Ok(modified) = metadata.modified() { + if modified > freshness_cutoff { + continue; + } + } + let Some(deletion) = self.reserve_path_deletion(&path) else { + continue; + }; + let capacity_invalidation = self.capacity_invalidation_guard(); + let result = run_owned_cache_operation( + Arc::clone(&self.owned_operation_permits), + "orphan-session-delete", + async move { + let _capacity_invalidation = capacity_invalidation; + let result = fs::remove_dir_all(&path).await; + drop(deletion); + result + }, + ) + .await; + match result { + Ok(()) => removed = removed.saturating_add(1), + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => { + warn!("HLS orphan session directory cleanup deferred: error_kind={:?}", err.kind()); + } + } + } + Ok(removed) + } + + pub fn has_active_temp_files_for_session(&self, proxy_session_id: &ProxySessionId) -> bool { + let session_path = self.cache_path_snapshot().path.join(safe_session_path_component(proxy_session_id)); + self.temp_files + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .active_files + .iter() + .any(|path| path.starts_with(&session_path)) + } + + pub fn has_active_temp_files(&self) -> bool { + !self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner).active_files.is_empty() + } + + pub async fn invalidate_all_if_no_active_temp_files(&self) -> io::Result { + let cache_path = self.cache_path_snapshot(); + let Some(deletion) = self.reserve_path_deletion(&cache_path.path) else { + return Ok(CacheInvalidationOutcome::DeferredActiveTempFiles); + }; + self.invalidate_all_unchecked(cache_path.path, deletion).await?; + Ok(CacheInvalidationOutcome::Invalidated) + } + + pub async fn invalidate_all(&self) -> io::Result<()> { + let cache_path = self.cache_path_snapshot(); + let deletion = self.reserve_path_deletion(&cache_path.path).ok_or_else(|| { + io::Error::new(io::ErrorKind::WouldBlock, "hls cache invalidation conflicts with an active object write") + })?; + self.invalidate_all_unchecked(cache_path.path, deletion).await + } + + pub(super) async fn invalidate_all_unchecked( + &self, + cache_path: PathBuf, + deletion: CachePathDeletionReservation, + ) -> io::Result<()> { + let capacity_invalidation = self.capacity_invalidation_guard(); + run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "invalidate", async move { + let _capacity_invalidation = capacity_invalidation; + let result = async { + ensure_safe_cache_root(&cache_path).await?; + fs::create_dir_all(&cache_path).await?; + let mut entries = fs::read_dir(&cache_path).await?; + while let Some(entry) = entries.next_entry().await? { + if entry.file_name() == REWRITE_SECRET_FINGERPRINT_FILE + || entry.file_name() == HLS_CACHE_ROOT_MARKER_FILE + { + continue; + } + let file_type = entry.file_type().await?; + if file_type.is_dir() { + fs::remove_dir_all(entry.path()).await?; + } else { + fs::remove_file(entry.path()).await?; + } + } + Ok(()) + } + .await; + drop(deletion); + result + }) + .await + } + + pub async fn read_rewrite_secret_fingerprint(&self) -> io::Result> { + match fs::read_to_string(self.rewrite_secret_fingerprint_path()).await { + Ok(value) => Ok(Some(value.trim().to_string())), + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + } + } + + pub async fn write_rewrite_secret_fingerprint(&self, fingerprint: &str) -> io::Result<()> { + self.ensure_cache_root_marker().await?; + fs::write(self.rewrite_secret_fingerprint_path(), fingerprint).await + } + + pub(super) fn path_for_key(&self, key: &K) -> PathBuf { + let cache_path = self.cache_path_snapshot(); + Self::path_for_key_in(&cache_path.path, key) + } + + pub(super) fn path_for_key_in(cache_path: &Path, key: &K) -> PathBuf { + cache_path.join(key.session_path_component()).join(key.file_name()) + } + + pub(super) fn register_active_temp_path(&self, path: &Path) -> io::Result> { + let mut state = self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if state.deletion_reservations.iter().any(|reserved| path.starts_with(reserved)) { + return Err(io::Error::new(io::ErrorKind::Interrupted, "hls cache path is reserved for invalidation")); + } + if !state.active_files.insert(path.to_path_buf()) { + return Err(io::Error::new(io::ErrorKind::AlreadyExists, "hls cache temp path is already active")); + } + Ok(Arc::new(ActiveTempFileRegistration { path: path.to_path_buf(), state: Arc::clone(&self.temp_files) })) + } + + pub(super) fn reserve_path_deletion(&self, path: &Path) -> Option { + let mut state = self.temp_files.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let conflicts_with_active = state.active_files.iter().any(|active| active.starts_with(path)); + let conflicts_with_delete = + state.deletion_reservations.iter().any(|reserved| reserved.starts_with(path) || path.starts_with(reserved)); + if conflicts_with_active || conflicts_with_delete { + return None; + } + let path = path.to_path_buf(); + state.deletion_reservations.insert(path.clone()); + Some(CachePathDeletionReservation { path, state: Arc::clone(&self.temp_files) }) + } + + pub(super) async fn ensure_cache_root_marker(&self) -> io::Result<()> { + let cache_path = self.cache_path_snapshot(); + self.ensure_cache_root_marker_for(&cache_path.path).await + } + + pub(super) async fn ensure_cache_root_marker_for(&self, cache_path: &Path) -> io::Result<()> { + let marker_matches = { + let marker_path = self.marker_path.read().unwrap_or_else(std::sync::PoisonError::into_inner); + marker_path.as_deref() == Some(cache_path) + }; + if marker_matches { + return Ok(()); + } + ensure_not_root_like_cache_path(cache_path)?; + fs::create_dir_all(cache_path).await?; + fs::write(cache_path.join(HLS_CACHE_ROOT_MARKER_FILE), b"tuliprox-hls-cache\n").await?; + *self.marker_path.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(cache_path.to_path_buf()); + Ok(()) + } + + pub(super) fn cache_path_snapshot(&self) -> CachePathState { + self.cache_path.read().unwrap_or_else(std::sync::PoisonError::into_inner).clone() + } +} + +pub(super) const REWRITE_SECRET_FINGERPRINT_FILE: &str = ".rewrite_secret_fingerprint"; + +const HLS_CACHE_ROOT_MARKER_FILE: &str = ".tuliprox-hls-cache-root"; + +fn safe_session_path_component(proxy_session_id: &ProxySessionId) -> String { + let value = &proxy_session_id.0; + if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) { + return value.clone(); + } + blake3::hash(value.as_bytes()).to_hex().to_string() +} + +fn ensure_not_root_like_cache_path(cache_path: &Path) -> io::Result<()> { + if cache_path.as_os_str().is_empty() || cache_path.parent().is_none() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "refusing to invalidate unsafe hls cache root")); + } + Ok(()) +} + +async fn ensure_safe_cache_root(cache_path: &Path) -> io::Result<()> { + ensure_not_root_like_cache_path(cache_path)?; + match fs::metadata(cache_path.join(HLS_CACHE_ROOT_MARKER_FILE)).await { + Ok(metadata) if metadata.is_file() => Ok(()), + Ok(_) => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "refusing to invalidate hls cache root without marker file", + )), + Err(err) if err.kind() == io::ErrorKind::NotFound => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "refusing to invalidate hls cache root without marker file", + )), + Err(err) => Err(err), + } +} + +async fn old_temp_files(root: &Path, cutoff: SystemTime, limit: usize) -> io::Result> { + let mut candidates = Vec::with_capacity(limit); + if limit == 0 { + return Ok(candidates); + } + let mut root_entries = match fs::read_dir(root).await { + Ok(entries) => entries, + Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(candidates), + Err(err) => return Err(err), + }; + while candidates.len() < limit { + let Some(entry) = root_entries.next_entry().await? else { + break; + }; + let path = entry.path(); + let file_type = match entry.file_type().await { + Ok(file_type) => file_type, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => { + warn!("HLS temporary cache entry inspection deferred: error_kind={:?}", error.kind()); + continue; + } + }; + if file_type.is_dir() { + append_old_temp_files_in_session(&path, cutoff, limit, &mut candidates).await?; + } else { + append_old_temp_file_candidate(&entry, cutoff, limit, &mut candidates).await; + } + } + Ok(candidates) +} + +async fn append_old_temp_files_in_session( + session_root: &Path, + cutoff: SystemTime, + limit: usize, + candidates: &mut Vec, +) -> io::Result<()> { + let mut pending_dirs = vec![session_root.to_path_buf()]; + while let Some(dir) = pending_dirs.pop() { + if candidates.len() >= limit { + break; + } + let mut entries = match fs::read_dir(&dir).await { + Ok(entries) => entries, + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + }; + while candidates.len() < limit { + let Some(entry) = entries.next_entry().await? else { + break; + }; + let path = entry.path(); + let file_type = match entry.file_type().await { + Ok(file_type) => file_type, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => { + warn!("HLS temporary cache entry inspection deferred: error_kind={:?}", error.kind()); + continue; + } + }; + if file_type.is_dir() { + pending_dirs.push(path); + continue; + } + append_old_temp_file_candidate(&entry, cutoff, limit, candidates).await; + } + } + Ok(()) +} + +async fn append_old_temp_file_candidate( + entry: &fs::DirEntry, + cutoff: SystemTime, + limit: usize, + candidates: &mut Vec, +) { + let path = entry.path(); + if candidates.len() >= limit || !is_temp_cache_file(&path) { + return; + } + let metadata = match entry.metadata().await { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return, + Err(error) => { + warn!("HLS temporary cache metadata inspection deferred: error_kind={:?}", error.kind()); + return; + } + }; + if metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH) < cutoff { + candidates.push(path); + } +} + +pub(super) fn is_temp_cache_file(path: &Path) -> bool { + path.file_name().and_then(|file_name| file_name.to_str()).is_some_and(|file_name| file_name.contains(".tmp.")) +} diff --git a/backend/hls/src/cache/revision_capacity_tests.rs b/backend/hls/src/cache/revision_capacity_tests.rs new file mode 100644 index 000000000..f98e713c5 --- /dev/null +++ b/backend/hls/src/cache/revision_capacity_tests.rs @@ -0,0 +1,59 @@ +use super::*; +use std::{io, path::Path, time::SystemTime}; +use tokio::{io::AsyncWriteExt, time::Instant}; + +mod publication; +mod recovery; + +#[tokio::test] +#[ignore = "subprocess checkpoint helper, invoked by the crash recovery test"] +async fn revision_crash_checkpoint() -> Result<(), Box> { + let root = std::env::var("TULIPROX_REVISION_CRASH_ROOT")?; + let phase = std::env::var("TULIPROX_REVISION_CRASH_PHASE")?; + let cache = HlsSegmentCache::with_cache_path(Path::new(&root).join("cache")); + let legacy = SegmentCacheKey::new(ProxySessionId("same-session".into()), 0, "ts"); + cache.write_bytes_and_commit(&legacy, b"valid-processed-media").await?; + let revisions = crate::SegmentRevisionStore::default(); + let kind = + if phase.starts_with("raw") { crate::SegmentRevisionKind::Raw } else { crate::SegmentRevisionKind::Processed }; + let revision = revisions.create(legacy.proxy_session_id().clone(), 0, kind)?; + let mut _staged = None; + let _pin = if phase.starts_with("raw") { + let (mut spool, pin) = cache.create_revision_spool(&revision.revision().key).await?; + spool.write_all(b"unbound-revision-checkpoint").await?; + spool.flush().await?; + if phase == "raw_eof" { + revision.revision().complete(CachedSegmentMetadata { + path: cache.object_path(&revision.revision().key), + size: u64::try_from(b"unbound-revision-checkpoint".len())?, + }); + } + pin + } else if phase == "processed_staged" { + let pin = cache.pin_revision_file(&cache.object_path(&revision.revision().key))?; + _staged = Some( + cache + .stage_temp_with_deadline( + &revision.revision().key, + b"repair-output".as_slice(), + Instant::now() + std::time::Duration::from_secs(5), + ) + .await?, + ); + pin + } else { + let pin = cache.pin_revision_file(&cache.object_path(&revision.revision().key))?; + cache + .publish_processed_revision( + &revision.revision().key, + &cache.object_path(&legacy), + Instant::now() + std::time::Duration::from_secs(5), + ) + .await?; + pin + }; + std::fs::write(Path::new(&root).join("ready"), b"ready")?; + loop { + tokio::time::sleep(std::time::Duration::from_secs(60)).await; + } +} diff --git a/backend/hls/src/cache/revision_capacity_tests/publication.rs b/backend/hls/src/cache/revision_capacity_tests/publication.rs new file mode 100644 index 000000000..f41d09c9c --- /dev/null +++ b/backend/hls/src/cache/revision_capacity_tests/publication.rs @@ -0,0 +1,56 @@ +use super::*; + +#[tokio::test] +async fn revision_peak_and_legacy_cache_share_the_global_and_session_quota() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.update_cache_limits(1000, 500); + let first = SegmentCacheKey::new(ProxySessionId("one".into()), 0, "ts"); + let other = SegmentCacheKey::new(ProxySessionId("two".into()), 0, "ts"); + let reservation = cache.reserve_revision_peak(&first, 400).await?; + assert!(cache.reserve_revision_peak(&first, 200).await.is_err()); + let other_reservation = cache.reserve_revision_peak(&other, 400).await?; + assert!(cache.write_bytes_and_commit(&first, &[1; 300]).await.is_err()); + drop(other_reservation); + drop(reservation); + cache.write_bytes_and_commit(&first, &[1; 300]).await?; + assert_eq!(cache.capacity_usage(first.proxy_session_id()).await?.global_bytes, 300); + Ok(()) +} + +#[tokio::test] +async fn processed_revision_retries_and_ranges_preserve_bytes_across_cache_replacement() -> io::Result<()> { + use futures::StreamExt; + let directory = tempfile::tempdir()?; + let cache = HlsSegmentCache::with_cache_path(directory.path()); + let legacy = SegmentCacheKey::new(ProxySessionId("range".into()), 0, "ts"); + cache.write_bytes_and_commit(&legacy, b"original-segment").await?; + let store = crate::SegmentRevisionStore::default(); + let owner = store.create(legacy.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Processed)?; + let pin = cache.pin_revision_file(&cache.object_path(&owner.revision().key))?; + *owner.revision().file_pin.lock().map_err(|_| io::Error::other("file pin"))? = Some(pin); + let metadata = cache + .publish_processed_revision( + &owner.revision().key, + &cache.object_path(&legacy), + Instant::now() + std::time::Duration::from_secs(5), + ) + .await?; + owner.revision().complete(metadata); + cache.write_bytes_and_commit(&legacy, b"longer-repaired-replacement").await?; + for (start, end, expected) in [ + (0, None, b"original-segment".as_slice()), + (3, Some(11), b"ginal-se".as_slice()), + (0, None, b"original-segment".as_slice()), + ] { + let mut body = + crate::progressive_startup::revision_body(owner.clone(), std::time::Duration::from_secs(5), start, end); + let mut bytes = Vec::new(); + while let Some(chunk) = body.next().await { + bytes.extend_from_slice(&chunk?); + } + assert_eq!(bytes, expected); + } + assert_eq!(cache.delete_orphan_revision_files(SystemTime::now()).await?, 0); + Ok(()) +} diff --git a/backend/hls/src/cache/revision_capacity_tests/recovery.rs b/backend/hls/src/cache/revision_capacity_tests/recovery.rs new file mode 100644 index 000000000..61a3428dc --- /dev/null +++ b/backend/hls/src/cache/revision_capacity_tests/recovery.rs @@ -0,0 +1,61 @@ +use super::*; + +#[tokio::test] +async fn restart_discards_unbound_revisions_and_preserves_legacy_cache() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let legacy = SegmentCacheKey::new(ProxySessionId("same-session".into()), 0, "ts"); + let cache = HlsSegmentCache::with_cache_path(directory.path()); + cache.write_bytes_and_commit(&legacy, b"valid-processed-media").await?; + let revisions = crate::SegmentRevisionStore::default(); + let revision = revisions.create(legacy.proxy_session_id().clone(), 0, crate::SegmentRevisionKind::Raw)?; + let (mut spool, pin) = cache.create_revision_spool(&revision.revision().key).await?; + spool.write_all(b"partial-origin-body").await?; + spool.flush().await?; + assert_eq!(cache.delete_orphan_revision_files(SystemTime::now()).await?, 0); + drop(spool); + drop(pin); + drop(cache); + let restarted = HlsSegmentCache::with_cache_path(directory.path()); + assert_eq!(restarted.delete_orphan_revision_files(SystemTime::now()).await?, 1); + assert_eq!(tokio::fs::read(restarted.object_path(&legacy)).await?, b"valid-processed-media"); + assert_eq!(restarted.capacity_usage(legacy.proxy_session_id()).await?.global_bytes, 21); + Ok(()) +} + +#[tokio::test] +async fn killed_process_revisions_are_reconciled_after_restart() -> Result<(), Box> { + for phase in ["raw", "raw_eof", "processed_staged", "processed"] { + let directory = tempfile::tempdir()?; + let mut child = std::process::Command::new(std::env::current_exe()?) + .args(["--ignored", "--exact", "cache::revision_capacity_tests::revision_crash_checkpoint"]) + .env("TULIPROX_REVISION_CRASH_ROOT", directory.path()) + .env("TULIPROX_REVISION_CRASH_PHASE", phase) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn()?; + let checkpoint = directory.path().join("ready"); + let deadline = Instant::now() + std::time::Duration::from_secs(5); + while !checkpoint.exists() && Instant::now() < deadline { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + let reached = checkpoint.exists(); + let killed = child.kill(); + let _ = child.wait(); + killed?; + assert!(reached, "child reached the durable crash checkpoint"); + let restarted = HlsSegmentCache::with_cache_path(directory.path().join("cache")); + assert_eq!( + restarted.delete_orphan_revision_files(SystemTime::now()).await?, + usize::from(phase != "processed_staged") + ); + assert_eq!( + restarted.delete_temp_files_older_than(SystemTime::now()).await?, + usize::from(phase == "processed_staged") + ); + assert_eq!(restarted.delete_orphan_revision_files(SystemTime::now()).await?, 0); + let legacy = SegmentCacheKey::new(ProxySessionId("same-session".into()), 0, "ts"); + assert_eq!(tokio::fs::read(restarted.object_path(&legacy)).await?, b"valid-processed-media"); + assert_eq!(restarted.capacity_usage(legacy.proxy_session_id()).await?.global_bytes, 21); + } + Ok(()) +} diff --git a/backend/hls/src/cache/revisions.rs b/backend/hls/src/cache/revisions.rs new file mode 100644 index 000000000..c2661068a --- /dev/null +++ b/backend/hls/src/cache/revisions.rs @@ -0,0 +1,217 @@ +use super::{ + error::capacity_error, CacheCapacityPressure, CacheCapacityState, CachedSegmentMetadata, CapacityRevision, + HlsCacheCapacityReclaimOutcome, HlsCacheCapacityRevision, HlsCacheObjectKey, HlsRevisionDiskReservation, + HlsRevisionFilePin, HlsSegmentCache, StagedCacheObject, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN, +}; +use crate::sync_ext::MutexExt; +use std::{ + fmt, io, + path::Path, + sync::{atomic::Ordering, Arc}, + time::SystemTime, +}; +use tokio::{ + fs, + fs::{File, OpenOptions}, + time::Instant, +}; + +impl HlsRevisionDiskReservation { + /// Records that a revision spool was written directly under this reservation. + /// + /// Those bytes bypass the staged commit accounting, so releasing the + /// reservation must rescan the cache to pick them up. + pub fn mark_unaccounted_writes(&mut self) { self.unaccounted_writes = true; } +} + +impl Drop for HlsRevisionDiskReservation { + fn drop(&mut self) { + let mut capacity = self.capacity.lock_unpoisoned(); + capacity.revision_reservations.remove(&self.id); + if self.unaccounted_writes { + capacity.initialized = false; + capacity.revision = Arc::new(CapacityRevision); + } + drop(capacity); + self.changed.notify_waiters(); + } +} + +impl fmt::Debug for HlsRevisionFilePin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("HlsRevisionFilePin") } +} + +impl HlsSegmentCache { + pub async fn reserve_revision_peak( + &self, + key: &K, + bytes: u64, + ) -> io::Result { + let _admission = self.capacity_admission_gate.lock().await; + let root = self.cache_path_snapshot(); + self.ensure_capacity_initialized(&root.path).await?; + let session = key.session_path_component(); + let mut capacity = self.capacity.lock_unpoisoned(); + let (reserved_global, reserved_session) = revision_reserved_bytes(&capacity, &session); + let global = capacity.total_bytes.saturating_add(reserved_global); + let local = capacity.session_bytes.get(&session).copied().unwrap_or_default().saturating_add(reserved_session); + let global_limit = self.max_cache_bytes.load(Ordering::Acquire); + let local_limit = self.max_session_bytes.load(Ordering::Acquire); + let required_global = global.saturating_add(bytes).saturating_sub(global_limit); + let required_session = local.saturating_add(bytes).saturating_sub(local_limit); + if !capacity.initialized || capacity.cache_path != root.path { + return Err(io::Error::new(io::ErrorKind::Interrupted, "revision disk reservation invalidated")); + } + if required_global > 0 || required_session > 0 { + return Err(capacity_error( + CacheCapacityPressure { + configured_session_bytes: local_limit, + configured_global_bytes: global_limit, + current_session_bytes: local, + current_global_bytes: global, + staged_bytes: bytes, + required_session_bytes: required_session, + required_global_bytes: required_global, + }, + HlsCacheCapacityReclaimOutcome::default(), + HlsCacheCapacityRevision(Arc::clone(&capacity.revision)), + )); + } + let id = loop { + let id = fastrand::u64(..); + if !capacity.revision_reservations.contains_key(&id) { + break id; + } + }; + capacity.revision_reservations.insert(id, (session, bytes)); + capacity.revision = Arc::new(CapacityRevision); + Ok(HlsRevisionDiskReservation { + id, + capacity: Arc::clone(&self.capacity), + changed: Arc::clone(&self.capacity_changed), + unaccounted_writes: false, + }) + } + + pub fn pin_revision_file(&self, path: &Path) -> io::Result { + if !self.contains_current_cache_path(path) { + return Err(io::Error::new(io::ErrorKind::Interrupted, "revision outside current cache root")); + } + Ok(HlsRevisionFilePin { _registration: self.register_active_temp_path(path)? }) + } + + pub async fn create_revision_spool(&self, key: &K) -> io::Result<(File, HlsRevisionFilePin)> { + let root = self.cache_path_snapshot(); + self.ensure_cache_root_marker_for(&root.path).await?; + let path = Self::path_for_key_in(&root.path, key); + let pin = self.pin_revision_file(&path)?; + let parent = path.parent().ok_or_else(|| io::Error::other("revision path has no parent"))?; + fs::create_dir_all(parent).await?; + let file = OpenOptions::new().write(true).create_new(true).open(&path).await?; + Ok((file, pin)) + } + + pub async fn delete_orphan_revision_files(&self, cutoff: SystemTime) -> io::Result { + let root = self.cache_path_snapshot(); + let Some(mut sessions) = skip_not_found(fs::read_dir(&root.path).await)? else { return Ok(0) }; + let mut deleted = 0usize; + while let Some(session) = sessions.next_entry().await? { + let Some(session_type) = skip_not_found(session.file_type().await)? else { continue }; + if !session_type.is_dir() { + continue; + } + let Some(mut entries) = skip_not_found(fs::read_dir(session.path()).await)? else { continue }; + while let Some(entry) = entries.next_entry().await? { + if deleted >= MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN { + return Ok(deleted); + } + let Some(kind) = skip_not_found(entry.file_type().await)? else { continue }; + if !kind.is_file() || !is_revision_file_name(&entry.file_name()) { + continue; + } + let Some(metadata) = skip_not_found(entry.metadata().await)? else { continue }; + if metadata.modified().is_ok_and(|modified| modified > cutoff) { + continue; + } + let path = entry.path(); + let Some(deletion) = self.reserve_path_deletion(&path) else { + continue; + }; + let reservation = self + .begin_capacity_mutation(&root.path, &path, session.file_name().to_string_lossy().into_owned()) + .await; + if self.delete_with_reservation(path, reservation).await.is_ok() { + deleted += 1; + } + drop(deletion); + } + } + Ok(deleted) + } + + pub async fn publish_processed_revision( + &self, + key: &K, + source: &Path, + deadline: Instant, + ) -> io::Result { + let root = self.cache_path_snapshot(); + let staged_path = Self::temp_path_for_key(key, &root.path); + let registration = self.register_active_temp_path(&staged_path)?; + if fs::hard_link(source, &staged_path).await.is_ok() { + let size = fs::metadata(&staged_path).await?.len(); + let staged = StagedCacheObject::registered(staged_path, size, Arc::clone(&root.generation), registration); + self.commit_staged(key, staged).await + } else { + drop(registration); + let file = File::open(source).await?; + let staged = self.stage_temp_with_deadline(key, file, deadline).await?; + self.commit_staged(key, staged).await + } + } + + /// Revision file bytes as of the last completed capacity scan. + pub fn revision_disk_bytes(&self) -> u64 { self.capacity.lock_unpoisoned().revision_bytes } + + /// Walks the cache tree for the current revision file bytes. + #[cfg(test)] + pub async fn scan_revision_disk_bytes(&self) -> io::Result { + super::capacity::scan_committed_cache_usage(&self.cache_path_snapshot().path).await.map(|(_, _, bytes)| bytes) + } +} + +pub(super) fn revision_reserved_bytes(capacity: &CacheCapacityState, session: &str) -> (u64, u64) { + capacity.revision_reservations.values().fold((0u64, 0u64), |(global, local), (owner, bytes)| { + (global.saturating_add(*bytes), local.saturating_add(if owner == session { *bytes } else { 0 })) + }) +} + +/// Maps a vanished filesystem entry to `None`; other errors still fail. +fn skip_not_found(result: io::Result) -> io::Result> { + match result { + Ok(value) => Ok(Some(value)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } +} + +fn is_revision_file_name(name: &std::ffi::OsStr) -> bool { + let Some(name) = name.to_str().and_then(|name| name.strip_prefix("revision-")) else { + return false; + }; + let Some((generation, tail)) = name.split_once('-') else { + return false; + }; + let Some((ordinal, suffix)) = tail.split_once('-') else { + return false; + }; + [generation, ordinal].iter().all(|part| part.len() == 16 && part.bytes().all(|byte| byte.is_ascii_hexdigit())) + && matches!(suffix, "raw.ts" | "processed.ts") +} + +pub(super) fn is_revision_cache_path(path: &Path) -> bool { + path.file_name().and_then(|name| name.to_str()).is_some_and(|name| { + let name = name.split_once(".tmp.").map_or(name, |(name, _)| name); + is_revision_file_name(std::ffi::OsStr::new(name)) + }) +} diff --git a/backend/hls/src/cache/staging.rs b/backend/hls/src/cache/staging.rs new file mode 100644 index 000000000..70938876d --- /dev/null +++ b/backend/hls/src/cache/staging.rs @@ -0,0 +1,450 @@ +use super::{ + error::{cache_object_limit_error, capacity_error}, + hls_cache_capacity_from_io, safe_proxy_session_id, ActiveTempFileRegistration, CacheCapacityPressure, + CachePathGeneration, CachePathState, HlsCacheCapacityReclaimOutcome, HlsCacheCapacityReclaimRequest, + HlsCacheObjectKey, HlsSegmentCache, StagedCacheObject, TEMP_CREATE_ATTEMPTS, +}; +use log::warn; +use std::{ + fmt, + future::Future, + io, + path::{Path, PathBuf}, + sync::{atomic::Ordering, Arc}, +}; +use tokio::{ + fs, + fs::{File, OpenOptions}, + io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt, SeekFrom}, + sync::Semaphore, + time::{timeout_at, Instant}, +}; + +/// Filesystem metadata for a committed HLS cache object. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct CachedSegmentMetadata { + pub path: PathBuf, + pub size: u64, +} + +impl StagedCacheObject { + pub(super) fn registered( + path: PathBuf, + size: u64, + cache_path_generation: Arc, + registration: Arc, + ) -> Self { + Self { path, size, cache_path_generation, _registration: registration } + } +} + +impl fmt::Debug for StagedCacheObject { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("StagedCacheObject") + .field("path", &self.path) + .field("size", &self.size) + .finish_non_exhaustive() + } +} + +impl PartialEq for StagedCacheObject { + fn eq(&self, other: &Self) -> bool { + self.path == other.path + && self.size == other.size + && Arc::ptr_eq(&self.cache_path_generation, &other.cache_path_generation) + } +} + +impl Eq for StagedCacheObject {} + +impl HlsSegmentCache { + pub async fn open_range(&self, key: &K, start: u64) -> io::Result { + let mut file = File::open(self.path_for_key(key)).await?; + file.seek(SeekFrom::Start(start)).await?; + Ok(file) + } + + pub async fn write_temp_and_commit(&self, key: &K, mut reader: R) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + self.write_temp_and_commit_inner(key, &mut reader, None).await + } + + pub async fn write_temp_and_commit_with_deadline( + &self, + key: &K, + mut reader: R, + deadline: Instant, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + self.write_temp_and_commit_inner(key, &mut reader, Some(deadline)).await + } + + pub async fn stage_temp_with_deadline( + &self, + key: &K, + mut reader: R, + deadline: Instant, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + self.stage_temp_inner(key, &mut reader, Some(deadline)).await + } + + pub async fn commit_staged(&self, key: &K, staged: StagedCacheObject) -> io::Result + where + K: HlsCacheObjectKey, + { + self.commit_staged_with_guard(key, staged, ()).await.map(|(metadata, ())| metadata) + } + + /// Commits a staged object while transferring a cancellation cleanup guard through the owned rename task. + pub async fn commit_staged_with_guard( + &self, + key: &K, + staged: StagedCacheObject, + guard: G, + ) -> io::Result<(CachedSegmentMetadata, G)> + where + K: HlsCacheObjectKey, + G: Send + 'static, + { + let staged_path = staged.path.clone(); + let result = self.commit_staged_inner(key, staged, guard).await; + if result.is_err() { + remove_temp_file_after_error(&staged_path, "commit").await; + } + result + } + + #[allow(clippy::too_many_lines)] + pub(super) async fn commit_staged_inner( + &self, + key: &K, + staged: StagedCacheObject, + guard: G, + ) -> io::Result<(CachedSegmentMetadata, G)> + where + K: HlsCacheObjectKey, + G: Send + 'static, + { + // Every authoritative admission participates in this gate. Otherwise a new writer that happens to fit after + // reclamation can consume the reclaimed bytes before the reclaiming writer performs its retry. + let admission_gate = self.capacity_admission_gate.lock().await; + let mut reclamation_attempted = false; + let mut reclamation_outcome = HlsCacheCapacityReclaimOutcome::default(); + let mut reclamation_pressure: Option = None; + loop { + let commit_gate = Arc::clone(&self.cache_path_commit_gate).read_owned().await; + let cache_path = self.cache_path_snapshot(); + if !Arc::ptr_eq(&staged.cache_path_generation, &cache_path.generation) + || !staged.path.starts_with(&cache_path.path) + { + return Err(io::Error::new( + io::ErrorKind::Interrupted, + "hls cache path changed before staged object commit", + )); + } + self.ensure_cache_root_marker_for(&cache_path.path).await?; + let max_object_bytes = self.max_object_bytes.load(Ordering::Acquire); + if staged.size > max_object_bytes { + return Err(cache_object_limit_error(max_object_bytes)); + } + let final_path = Self::path_for_key_in(&cache_path.path, key); + let Some(parent) = final_path.parent() else { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "cache path has no parent")); + }; + fs::create_dir_all(parent).await?; + let reservation = self + .prepare_capacity_replacement( + &cache_path.path, + &final_path, + key.session_path_component(), + staged.size, + self.max_cache_bytes.load(Ordering::Acquire), + self.max_session_bytes.load(Ordering::Acquire), + ) + .await; + let reservation = match reservation { + Ok(reservation) => { + if let Some(pressure) = reclamation_pressure { + let usage = reservation.projected_usage(); + log::info!( + "HLS cache capacity decision: proxy_session={} resource={} configured_session_bytes={} configured_global_bytes={} current_session_bytes={} current_global_bytes={} staged_bytes={} protected_working_set_bytes={} reclaimable_bytes={} required_session_bytes={} required_global_bytes={} outcome=reclaimed", + safe_proxy_session_id(key.proxy_session_id()), + key.file_name(), + pressure.configured_session_bytes, + pressure.configured_global_bytes, + usage.session_bytes, + usage.global_bytes, + staged.size, + reclamation_outcome.protected_working_set_bytes, + reclamation_outcome.reclaimable_bytes, + pressure.required_session_bytes, + pressure.required_global_bytes, + ); + } + reservation + } + Err(error) => { + let Some(capacity) = hls_cache_capacity_from_io(&error) else { + return Err(error); + }; + if reclamation_attempted { + log::warn!( + "HLS cache capacity decision: proxy_session={} resource={} configured_session_bytes={} configured_global_bytes={} current_session_bytes={} current_global_bytes={} staged_bytes={} protected_working_set_bytes={} reclaimable_bytes={} required_session_bytes={} required_global_bytes={} outcome=deferred-protected wake_reason=capacity-or-protection-revision", + safe_proxy_session_id(key.proxy_session_id()), + key.file_name(), + capacity.configured_session_bytes(), + capacity.configured_global_bytes(), + capacity.current_session_bytes(), + capacity.current_global_bytes(), + capacity.staged_bytes(), + reclamation_outcome.protected_working_set_bytes, + reclamation_outcome.reclaimable_bytes, + capacity.required_session_bytes(), + capacity.required_global_bytes(), + ); + return Err(capacity_error( + capacity.pressure(), + reclamation_outcome, + capacity.revision().clone(), + )); + } + let request = HlsCacheCapacityReclaimRequest { + proxy_session_id: key.proxy_session_id().clone(), + target_path: final_path, + required_session_bytes: capacity.required_session_bytes(), + required_global_bytes: capacity.required_global_bytes(), + staged_bytes: staged.size, + }; + log::warn!( + "HLS cache capacity pressure detected: proxy_session={} staged_bytes={} required_session_bytes={} required_global_bytes={}", + safe_proxy_session_id(key.proxy_session_id()), + staged.size, + capacity.required_session_bytes(), + capacity.required_global_bytes(), + ); + // The admission gate remains held through reclamation and the next authoritative reservation. The + // path read lease cannot: cache-root handoff uses run-gate -> path-write ordering. + drop(commit_gate); + reclamation_attempted = true; + reclamation_pressure = Some(capacity.pressure()); + match self.reclaim_capacity(request).await { + Ok(outcome) => { + reclamation_outcome = outcome; + log::info!( + "HLS cache capacity reclamation completed: proxy_session={} reclaimed_session_bytes={} reclaimed_global_bytes={} protected_working_set_bytes={} reclaimable_bytes={}", + safe_proxy_session_id(key.proxy_session_id()), + outcome.reclaimed_session_bytes, + outcome.reclaimed_global_bytes, + outcome.protected_working_set_bytes, + outcome.reclaimable_bytes, + ); + } + Err(reclaim_error) => { + log::warn!( + "HLS cache capacity reclamation failed: proxy_session={} error_kind={:?}", + safe_proxy_session_id(key.proxy_session_id()), + reclaim_error.kind(), + ); + return Err(reclaim_error); + } + } + continue; + } + }; + drop(admission_gate); + let staged_path = staged.path.clone(); + let committed_size = staged.size; + // Once spawned, the owned mutation deliberately outlives cancellation of the requesting future. This + // keeps the path lease, temp registration, and accounting reservation alive until the atomic filesystem + // operation has a definite result. A runtime shutdown may abort the task, but no cache work can follow + // that shutdown; panic/abort drops the reservation and forces accounting rollback/invalidation. + return run_owned_cache_operation(Arc::clone(&self.owned_operation_permits), "commit", async move { + let mut reservation = reservation; + reservation.mark_filesystem_mutation_started(); + if let Err(err) = fs::rename(&staged_path, &final_path).await { + remove_temp_file_after_error(&staged_path, "rename").await; + return Err(err); + } + reservation.finish_replacement(); + drop(staged); + drop(commit_gate); + Ok((CachedSegmentMetadata { path: final_path, size: committed_size }, guard)) + }) + .await; + } + } + + pub async fn remove_staged(&self, staged: StagedCacheObject) -> io::Result<()> { + let result = match fs::remove_file(&staged.path).await { + Ok(()) => Ok(()), + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(err), + }; + drop(staged); + result + } + + pub(super) async fn write_temp_and_commit_inner( + &self, + key: &K, + reader: &mut R, + deadline: Option, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + let staged = self.stage_temp_inner(key, reader, deadline).await?; + self.commit_staged(key, staged).await + } + + pub(super) async fn stage_temp_inner( + &self, + key: &K, + reader: &mut R, + deadline: Option, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + if deadline.is_some_and(|deadline| deadline <= Instant::now()) { + return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); + } + let cache_path = self.cache_path_snapshot(); + self.ensure_cache_root_marker_for(&cache_path.path).await?; + let final_path = Self::path_for_key_in(&cache_path.path, key); + let Some(parent) = final_path.parent() else { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "cache path has no parent")); + }; + fs::create_dir_all(parent).await?; + + let (temp_path, mut temp_file, registration) = self.create_temp_file(key, &cache_path).await?; + if deadline.is_some_and(|deadline| deadline <= Instant::now()) { + drop(temp_file); + remove_temp_file_after_error(&temp_path, "expired-write").await; + return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); + } + let copy = async { + let max_object_bytes = self.max_object_bytes.load(Ordering::Acquire); + let mut limited = reader.take(max_object_bytes.saturating_add(1)); + let size = tokio::io::copy(&mut limited, &mut temp_file).await?; + if size > max_object_bytes { + return Err(cache_object_limit_error(max_object_bytes)); + } + temp_file.flush().await?; + drop(temp_file); + Ok::(size) + }; + let copy_result = if let Some(deadline) = deadline { + if let Ok(result) = timeout_at(deadline, copy).await { + result + } else { + remove_temp_file_after_error(&temp_path, "timed-out-write").await; + return Err(io::Error::new(io::ErrorKind::TimedOut, "hls cache object write timed out")); + } + } else { + copy.await + }; + match copy_result { + Ok(size) => { + Ok(StagedCacheObject::registered(temp_path, size, Arc::clone(&cache_path.generation), registration)) + } + Err(err) => { + remove_temp_file_after_error(&temp_path, "failed-write").await; + Err(err) + } + } + } + + pub async fn write_bytes_and_commit(&self, key: &K, bytes: &[u8]) -> io::Result + where + K: HlsCacheObjectKey, + { + self.write_temp_and_commit(key, bytes).await + } + + pub(super) async fn create_temp_file( + &self, + key: &K, + cache_path: &CachePathState, + ) -> io::Result<(PathBuf, File, Arc)> { + for _ in 0..TEMP_CREATE_ATTEMPTS { + let temp_path = Self::temp_path_for_key(key, &cache_path.path); + let registration = self.register_active_temp_path(&temp_path)?; + match OpenOptions::new().write(true).create_new(true).open(&temp_path).await { + Ok(file) => { + return Ok((temp_path, file, registration)); + } + Err(err) if err.kind() == io::ErrorKind::AlreadyExists => { + drop(registration); + } + Err(err) => return Err(err), + } + } + Err(io::Error::new(io::ErrorKind::AlreadyExists, "could not create unique hls cache temp file")) + } + + pub fn adopt_staged_file(&self, path: PathBuf, size: u64) -> io::Result { + let cache_path = self.cache_path_snapshot(); + if !path.starts_with(&cache_path.path) { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "staged hls cache file is outside the cache root")); + } + let registration = self.register_active_temp_path(&path)?; + Ok(StagedCacheObject::registered(path, size, Arc::clone(&cache_path.generation), registration)) + } + + pub(super) fn temp_path_for_key(key: &K, cache_path: &Path) -> PathBuf { + let suffix = fastrand::u64(..); + cache_path.join(key.session_path_component()).join(format!("{}.tmp.{suffix:016x}", key.file_name())) + } +} + +async fn remove_temp_file_after_error(path: &Path, operation: &'static str) { + match fs::remove_file(path).await { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + warn!("HLS cache temporary file cleanup failed: operation={operation} error_kind={:?}", error.kind()); + } + } +} + +/// Runs one filesystem mutation with all of its reservations owned by the spawned task. +/// +/// A per-cache permit is acquired without waiting before the spawn. Saturation returns `WouldBlock` and drops the +/// supplied future synchronously, which releases any reservations it owns without creating a detached task. +/// +/// Dropping the caller future detaches, rather than cancels, the mutation. A returned `JoinError` therefore denotes a +/// task panic/abort; unwinding drops the owned guards. Runtime shutdown is the only case where completion is not +/// observed, and no later cache accounting or root transition can run after that shutdown. +pub(super) async fn run_owned_cache_operation( + permits: Arc, + operation: &'static str, + future: F, +) -> io::Result +where + T: Send + 'static, + F: Future> + Send + 'static, +{ + let permit = permits + .try_acquire_owned() + .map_err(|_| io::Error::new(io::ErrorKind::WouldBlock, "hls cache owned operation capacity exhausted"))?; + tokio::spawn(async move { + let _permit = permit; + future.await + }) + .await + .map_err(|error| io::Error::other(format!("hls cache {operation} task failed: {error}")))? +} diff --git a/backend/hls/src/cache/tests.rs b/backend/hls/src/cache/tests.rs new file mode 100644 index 000000000..a5e73a561 --- /dev/null +++ b/backend/hls/src/cache/tests.rs @@ -0,0 +1,126 @@ +use super::{ + hls_cache_capacity_from_io, hls_cache_object_limit_from_io, staging::run_owned_cache_operation, + CacheInvalidationOutcome, HlsCacheCapacityReclaimOutcome, HlsCacheCapacityReclaimRequest, + HlsCacheCapacityReclaimer, HlsCacheObjectKey, HlsSegmentCache, MapCacheKey, SegmentCacheKey, + TransientObjectCacheKey, MAX_CONCURRENT_OWNED_CACHE_OPERATIONS, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN, +}; +use crate::{transient::build_transient_resource_id, HlsOriginResourceFetchError, ProxySessionId}; +use futures::{future::BoxFuture, FutureExt}; +use std::{ + collections::{HashSet, VecDeque}, + future::Future, + io, + pin::Pin, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, + task::{Context, Poll}, + time::{Duration, SystemTime}, +}; +use tokio::{ + io::{AsyncRead, AsyncReadExt, ReadBuf}, + sync::{oneshot, Semaphore}, +}; + +fn cache_key() -> SegmentCacheKey { SegmentCacheKey::new(ProxySessionId("proxy_session".to_string()), 123, "ts") } + +async fn cache_with_projected_session_pressure( +) -> (tempfile::TempDir, Arc, SegmentCacheKey, SegmentCacheKey) { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let proxy_session_id = ProxySessionId("capacity-session".to_string()); + let resident = SegmentCacheKey::new(proxy_session_id.clone(), 0, "ts"); + let first = SegmentCacheKey::new(proxy_session_id.clone(), 1, "ts"); + let second = SegmentCacheKey::new(proxy_session_id, 2, "ts"); + cache.update_cache_limits(5, 5); + cache.write_bytes_and_commit(&resident, b"123").await.expect("resident object commits"); + (temp_dir, cache, first, second) +} + +struct PausingCapacityReclaimer { + cache: Arc, + victims: Mutex>, + first_reclaimed: Mutex>>, + resume_first: Mutex>>, +} + +impl HlsCacheCapacityReclaimer for PausingCapacityReclaimer { + fn reclaim_capacity( + &self, + _request: HlsCacheCapacityReclaimRequest, + ) -> BoxFuture<'_, io::Result> { + async move { + let victim = self.victims.lock().unwrap_or_else(std::sync::PoisonError::into_inner).pop_front(); + let Some(victim) = victim else { + return Ok(HlsCacheCapacityReclaimOutcome::default()); + }; + self.cache.delete_if_inactive(&victim).await?; + let first_reclaimed = self.first_reclaimed.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + if let Some(first_reclaimed) = first_reclaimed { + let resume_first = self.resume_first.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + let _ = first_reclaimed.send(()); + if let Some(resume_first) = resume_first { + let _ = resume_first.await; + } + } + Ok(HlsCacheCapacityReclaimOutcome { + reclaimed_session_bytes: 10, + reclaimed_global_bytes: 10, + protected_working_set_bytes: 0, + reclaimable_bytes: 10, + }) + } + .boxed() + } +} + +struct ControlledReader { + started: Option>, + release: oneshot::Receiver>, + body: Option>>, +} + +impl Unpin for ControlledReader {} + +impl AsyncRead for ControlledReader { + fn poll_read( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &mut ReadBuf<'_>, + ) -> Poll> { + if let Some(started) = self.started.take() { + let _send_result = started.send(()); + } + if self.body.is_none() { + match Pin::new(&mut self.release).poll(context) { + Poll::Ready(Ok(body)) => self.body = Some(std::io::Cursor::new(body)), + Poll::Ready(Err(_)) => { + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "controlled cache reader release dropped", + ))) + } + Poll::Pending => return Poll::Pending, + } + } + let body = self.body.as_mut().expect("controlled reader body is initialized"); + let position = usize::try_from(body.position()).unwrap_or(body.get_ref().len()); + let remaining = &body.get_ref()[position.min(body.get_ref().len())..]; + let copied = remaining.len().min(buffer.remaining()); + buffer.put_slice(&remaining[..copied]); + let next_position = body.position().saturating_add(u64::try_from(copied).unwrap_or_default()); + body.set_position(next_position); + Poll::Ready(Ok(())) + } +} + +mod admission; +mod http; +mod lifecycle; +mod policy; +mod publication; +mod storage; +mod streaming; +mod terminal; +mod transport; diff --git a/backend/hls/src/cache/tests/admission.rs b/backend/hls/src/cache/tests/admission.rs new file mode 100644 index 000000000..9bb0758f7 --- /dev/null +++ b/backend/hls/src/cache/tests/admission.rs @@ -0,0 +1,179 @@ +use super::*; + +#[tokio::test] +async fn concurrent_noop_capacity_failures_do_not_wake_each_other() { + let (_temp_dir, cache, first, second) = cache_with_projected_session_pressure().await; + let (first_error, second_error) = tokio::join!( + cache.ensure_projected_write_capacity(&first, 3), + cache.ensure_projected_write_capacity(&second, 3), + ); + let first_error = first_error.expect_err("first projected object exceeds the remaining budget"); + let second_error = second_error.expect_err("second projected object exceeds the remaining budget"); + let first_revision = + hls_cache_capacity_from_io(&first_error).expect("first typed capacity error").revision().clone(); + let second_revision = + hls_cache_capacity_from_io(&second_error).expect("second typed capacity error").revision().clone(); + let first_wait = cache.wait_for_capacity_change(&first_revision); + let second_wait = cache.wait_for_capacity_change(&second_revision); + tokio::pin!(first_wait, second_wait); + + assert!(matches!(futures::poll!(first_wait.as_mut()), Poll::Pending)); + assert!(matches!(futures::poll!(second_wait.as_mut()), Poll::Pending)); + + cache.notify_capacity_protection_changed(); + assert!(matches!(futures::poll!(first_wait.as_mut()), Poll::Ready(()))); + assert!(matches!(futures::poll!(second_wait.as_mut()), Poll::Ready(()))); +} + +#[tokio::test] +async fn abandoned_capacity_reservation_rolls_back_before_filesystem_mutation() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let path = temp_dir.path().join("session/000001.ts"); + { + let mut capacity = cache.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + capacity.cache_path = temp_dir.path().to_path_buf(); + capacity.initialized = true; + capacity.active_mutations.insert(path.clone()); + } + let mut reservation = super::super::CapacityMutationReservation { + path: path.clone(), + cache_path: temp_dir.path().to_path_buf(), + session_component: "session".to_string(), + replacement: None, + filesystem_mutation_started: false, + capacity: Arc::clone(&cache.capacity), + changed: Arc::clone(&cache.capacity_changed), + }; + reservation.reserve_replacement(0, 3, 10, 10).expect("capacity reserves"); + let revision = cache.capacity_revision(); + + drop(reservation); + + { + let capacity = cache.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + assert!(capacity.initialized); + assert_eq!(capacity.total_bytes, 0); + assert!(!capacity.session_bytes.contains_key("session")); + assert!(!capacity.active_mutations.contains(&path)); + } + let wait = cache.wait_for_capacity_change(&revision); + tokio::pin!(wait); + assert!(matches!(futures::poll!(wait.as_mut()), Poll::Ready(()))); +} + +#[tokio::test] +async fn abandoned_started_filesystem_mutation_invalidates_capacity_snapshot() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let path = temp_dir.path().join("session/000001.ts"); + { + let mut capacity = cache.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + capacity.cache_path = temp_dir.path().to_path_buf(); + capacity.initialized = true; + capacity.active_mutations.insert(path.clone()); + } + let mut reservation = super::super::CapacityMutationReservation { + path: path.clone(), + cache_path: temp_dir.path().to_path_buf(), + session_component: "session".to_string(), + replacement: None, + filesystem_mutation_started: false, + capacity: Arc::clone(&cache.capacity), + changed: Arc::clone(&cache.capacity_changed), + }; + reservation.reserve_replacement(0, 3, 10, 10).expect("capacity reserves"); + reservation.mark_filesystem_mutation_started(); + let revision = cache.capacity_revision(); + + drop(reservation); + + { + let capacity = cache.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + assert!(!capacity.initialized); + assert!(!capacity.active_mutations.contains(&path)); + } + let wait = cache.wait_for_capacity_change(&revision); + tokio::pin!(wait); + assert!(matches!(futures::poll!(wait.as_mut()), Poll::Ready(()))); +} + +#[tokio::test] +async fn concurrent_commits_reserve_the_global_budget_exactly_once() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + cache.update_cache_limits(5, 5); + let first = SegmentCacheKey::new(ProxySessionId("first".to_string()), 1, "ts"); + let second = SegmentCacheKey::new(ProxySessionId("second".to_string()), 1, "ts"); + let first_cache = Arc::clone(&cache); + let first_task = tokio::spawn(async move { first_cache.write_bytes_and_commit(&first, b"123").await }); + let second_cache = Arc::clone(&cache); + let second_task = tokio::spawn(async move { second_cache.write_bytes_and_commit(&second, b"456").await }); + + let (first_result, second_result) = tokio::join!(first_task, second_task); + let results = [first_result.expect("first task joins"), second_result.expect("second task joins")]; + + assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1); + assert_eq!(results.iter().filter(|result| result.is_err()).count(), 1); +} + +#[tokio::test] +async fn concurrent_commit_cannot_consume_bytes_reclaimed_for_an_in_flight_writer() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let proxy_session_id = ProxySessionId("serialized-reclamation".to_string()); + let first_resident = SegmentCacheKey::new(proxy_session_id.clone(), 1, "ts"); + let second_resident = SegmentCacheKey::new(proxy_session_id.clone(), 2, "ts"); + let first_target = SegmentCacheKey::new(proxy_session_id.clone(), 3, "ts"); + let second_target = SegmentCacheKey::new(proxy_session_id.clone(), 4, "ts"); + cache.update_cache_limits(100, 100); + cache.write_bytes_and_commit(&first_resident, b"0123456789").await.expect("first resident commits"); + cache.write_bytes_and_commit(&second_resident, b"0123456789").await.expect("second resident commits"); + cache.update_cache_limits(25, 25); + + let first_staged = cache + .stage_temp_with_deadline( + &first_target, + &b"abcdefghij"[..], + tokio::time::Instant::now() + Duration::from_secs(5), + ) + .await + .expect("first target stages"); + let second_staged = cache + .stage_temp_with_deadline( + &second_target, + &b"klmnopqrst"[..], + tokio::time::Instant::now() + Duration::from_secs(5), + ) + .await + .expect("second target stages"); + let (first_reclaimed_tx, first_reclaimed_rx) = oneshot::channel(); + let (resume_first_tx, resume_first_rx) = oneshot::channel(); + let reclaimer = Arc::new(PausingCapacityReclaimer { + cache: Arc::clone(&cache), + victims: Mutex::new(VecDeque::from([first_resident, second_resident])), + first_reclaimed: Mutex::new(Some(first_reclaimed_tx)), + resume_first: Mutex::new(Some(resume_first_rx)), + }); + cache.install_capacity_reclaimer(&reclaimer); + + let first_cache = Arc::clone(&cache); + let first_task = tokio::spawn(async move { first_cache.commit_staged(&first_target, first_staged).await }); + first_reclaimed_rx.await.expect("first reclamation pauses after deleting one resident"); + + let second_cache = Arc::clone(&cache); + let second_task = tokio::spawn(async move { second_cache.commit_staged(&second_target, second_staged).await }); + for _ in 0..100 { + tokio::task::yield_now().await; + } + let paused_usage = cache.capacity_usage(&proxy_session_id).await.expect("paused capacity usage"); + assert_eq!(paused_usage.session_bytes, 10, "a later commit must wait for the reclaiming writer's retry"); + + assert!(resume_first_tx.send(()).is_ok()); + let (first_result, second_result) = tokio::join!(first_task, second_task); + assert!(first_result.expect("first task joins").is_ok()); + assert!(second_result.expect("second task joins").is_ok()); + let usage = cache.capacity_usage(&proxy_session_id).await.expect("final capacity usage"); + assert_eq!(usage.session_bytes, 20); + assert!(usage.global_bytes <= 25); +} diff --git a/backend/hls/src/cache/tests/http.rs b/backend/hls/src/cache/tests/http.rs new file mode 100644 index 000000000..1169bac5e --- /dev/null +++ b/backend/hls/src/cache/tests/http.rs @@ -0,0 +1,15 @@ +use super::*; + +#[tokio::test] +async fn open_range_reads_from_requested_offset() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + cache.write_bytes_and_commit(&key, b"0123456789").await.expect("commit should succeed"); + + let file = cache.open_range(&key, 4).await.expect("range should open"); + let mut body = Vec::new(); + file.take(3).read_to_end(&mut body).await.expect("range should read"); + + assert_eq!(body, b"456"); +} diff --git a/backend/hls/src/cache/tests/lifecycle.rs b/backend/hls/src/cache/tests/lifecycle.rs new file mode 100644 index 000000000..d1d4bde81 --- /dev/null +++ b/backend/hls/src/cache/tests/lifecycle.rs @@ -0,0 +1,147 @@ +use super::*; + +#[tokio::test] +async fn orphan_session_cleanup_preserves_active_session_directories() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + cache.write_rewrite_secret_fingerprint("secret").await.expect("marker"); + let active = ProxySessionId("active".to_string()); + let orphan = ProxySessionId("orphan".to_string()); + tokio::fs::create_dir_all(temp_dir.path().join("active")).await.expect("active dir"); + tokio::fs::create_dir_all(temp_dir.path().join("orphan")).await.expect("orphan dir"); + let cutoff = SystemTime::now(); + + let removed = cache.delete_orphan_session_dirs(&HashSet::from([active]), cutoff).await.expect("cleanup"); + + assert_eq!(removed, 1); + assert!(temp_dir.path().join("active").exists()); + assert!(!temp_dir.path().join(orphan.0).exists()); +} + +#[tokio::test] +async fn orphan_session_cleanup_skips_directories_newer_than_freshness_cutoff() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + cache.write_rewrite_secret_fingerprint("secret").await.expect("marker"); + let stale = ProxySessionId("stale".to_string()); + let fresh = ProxySessionId("fresh".to_string()); + tokio::fs::create_dir_all(temp_dir.path().join(stale.0.clone())).await.expect("stale dir"); + tokio::fs::create_dir_all(temp_dir.path().join(fresh.0.clone())).await.expect("fresh dir"); + + // Set the fresh dir's mtime to the future relative to the cutoff. + let cutoff = SystemTime::now(); + let future = cutoff + std::time::Duration::from_mins(1); + filetime::set_file_mtime(temp_dir.path().join(fresh.0.clone()), filetime::FileTime::from_system_time(future)) + .expect("set fresh mtime"); + + let removed = cache.delete_orphan_session_dirs(&HashSet::new(), cutoff).await.expect("cleanup"); + + assert_eq!(removed, 1, "stale orphan dir should be removed"); + assert!(!temp_dir.path().join(stale.0).exists()); + assert!( + temp_dir.path().join(fresh.0).exists(), + "directory newer than the freshness cutoff must be preserved to avoid racing concurrent session creation" + ); +} + +#[tokio::test] +async fn owned_cache_operation_survives_caller_cancellation() { + let (started_tx, started_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + let (completed_tx, completed_rx) = oneshot::channel(); + let permits = Arc::new(Semaphore::new(1)); + let caller = tokio::spawn(async move { + run_owned_cache_operation(permits, "test", async move { + started_tx.send(()).map_err(|()| io::Error::other("start receiver dropped"))?; + release_rx.await.map_err(|_| io::Error::other("release sender dropped"))?; + completed_tx.send(()).map_err(|()| io::Error::other("completion receiver dropped"))?; + Ok(()) + }) + .await + }); + started_rx.await.expect("owned operation starts"); + caller.abort(); + release_tx.send(()).expect("release owned operation"); + + completed_rx.await.expect("detached owned operation completes"); + assert!(caller.await.expect_err("caller is cancelled").is_cancelled()); +} + +#[tokio::test] +async fn saturated_owned_operation_limit_prevents_spawn_and_mutation_until_permit_release() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + let permit_count = u32::try_from(MAX_CONCURRENT_OWNED_CACHE_OPERATIONS).expect("owned operation limit fits u32"); + let held_permits = Arc::clone(&cache.owned_operation_permits) + .try_acquire_many_owned(permit_count) + .expect("test exclusively holds owned operation permits"); + let operation_started = Arc::new(AtomicBool::new(false)); + let operation_started_in_task = Arc::clone(&operation_started); + + let spawn_error = + run_owned_cache_operation(Arc::clone(&cache.owned_operation_permits), "saturated-test", async move { + operation_started_in_task.store(true, Ordering::SeqCst); + Ok(()) + }) + .await + .expect_err("saturated helper must reject before spawning"); + + let error = cache + .write_bytes_and_commit(&key, b"segment-body") + .await + .expect_err("saturated cache operation must be rejected"); + + assert_eq!(spawn_error.kind(), io::ErrorKind::WouldBlock); + assert!(!operation_started.load(Ordering::SeqCst)); + assert_eq!(error.kind(), io::ErrorKind::WouldBlock); + assert!(cache.metadata(&key).await.expect("metadata reads after rejection").is_none()); + assert!(!cache.has_active_temp_files()); + { + let capacity = cache.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!(capacity.total_bytes, 0); + assert!(!capacity.session_bytes.contains_key("proxy_session")); + assert!(capacity.active_mutations.is_empty(), "pre-spawn rejection must roll back the reservation"); + } + + drop(held_permits); + cache.write_bytes_and_commit(&key, b"segment-body").await.expect("released permits allow a later mutation"); + assert!(cache.metadata(&key).await.expect("metadata reads after commit").is_some()); +} + +#[tokio::test] +async fn old_temp_file_cleanup_processes_a_bounded_batch_per_run() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let total_files = MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN.saturating_add(3); + for index in 0..total_files { + let session = if index % 2 == 0 { "session-a" } else { "session-b" }; + let directory = temp_dir.path().join(session).join(if index % 3 == 0 { "map" } else { "r" }); + tokio::fs::create_dir_all(&directory).await.expect("fixture directory writes"); + tokio::fs::write(directory.join(format!("object-{index}.ts.tmp.fixture")), b"stale") + .await + .expect("temporary fixture writes"); + } + let cutoff = SystemTime::now() + Duration::from_mins(1); + + let first = cache.delete_temp_files_older_than(cutoff).await.expect("first cleanup run succeeds"); + let second = cache.delete_temp_files_older_than(cutoff).await.expect("second cleanup run succeeds"); + let third = cache.delete_temp_files_older_than(cutoff).await.expect("third cleanup run succeeds"); + + assert_eq!(first, MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN); + assert_eq!(second, total_files.saturating_sub(MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN)); + assert_eq!(third, 0); +} + +#[tokio::test] +async fn delete_removes_committed_file_idempotently() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + cache.write_bytes_and_commit(&key, b"segment-body").await.expect("commit should succeed"); + + cache.delete(&key).await.expect("delete should succeed"); + cache.delete(&key).await.expect("second delete should be idempotent"); + + assert_eq!(cache.metadata(&key).await.expect("metadata should read"), None); +} diff --git a/backend/hls/src/cache/tests/policy.rs b/backend/hls/src/cache/tests/policy.rs new file mode 100644 index 000000000..220be851d --- /dev/null +++ b/backend/hls/src/cache/tests/policy.rs @@ -0,0 +1,99 @@ +use super::*; + +#[tokio::test] +async fn write_temp_and_commit_creates_final_segment_cache_file_with_proxy_layout() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + + let metadata = cache.write_bytes_and_commit(&key, b"segment-body").await.expect("commit should succeed"); + + assert_eq!(metadata.size, 12); + assert!(metadata.path.exists()); + assert_eq!(cache.metadata(&key).await.expect("metadata should read"), Some(metadata.clone())); + assert!(metadata.path.ends_with("proxy_session/000123.ts")); +} + +#[tokio::test] +async fn write_temp_and_commit_creates_final_map_cache_file_with_proxy_layout() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = MapCacheKey::new(ProxySessionId("proxy_session".to_string()), 0, "mp4"); + + let metadata = cache.write_bytes_and_commit(&key, b"map-body").await.expect("commit should succeed"); + + assert_eq!(metadata.size, 8); + assert!(metadata.path.ends_with("proxy_session/map/000000.mp4")); + assert_eq!(cache.metadata(&key).await.expect("metadata should read"), Some(metadata)); +} + +#[tokio::test] +async fn write_temp_and_commit_with_deadline_cleans_active_temp_file() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + let (_writer, reader) = tokio::io::duplex(64); + + let result = cache + .write_temp_and_commit_with_deadline(&key, reader, tokio::time::Instant::now() + Duration::from_millis(1)) + .await; + + assert_eq!(result.expect_err("commit should time out").kind(), io::ErrorKind::TimedOut); + assert!(!cache.has_active_temp_files()); + assert_eq!(cache.metadata(&key).await.expect("metadata should read"), None); +} + +#[tokio::test] +async fn temp_file_collision_does_not_overwrite_existing_temp_file() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + let parent = temp_dir.path().join("proxy_session"); + tokio::fs::create_dir_all(&parent).await.expect("parent should be created"); + let existing_temp = parent.join("000123.ts.tmp.0000000000000000"); + tokio::fs::write(&existing_temp, b"existing").await.expect("temp fixture should write"); + + cache.write_bytes_and_commit(&key, b"segment-body").await.expect("commit should succeed"); + + assert_eq!(tokio::fs::read(&existing_temp).await.expect("existing temp should remain"), b"existing"); +} + +#[tokio::test] +async fn invalidate_all_if_no_active_temp_files_deletes_only_when_no_temp_write_is_active() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let committed_key = cache_key(); + cache.write_bytes_and_commit(&committed_key, b"segment-body").await.expect("commit should succeed"); + + let outcome = cache.invalidate_all_if_no_active_temp_files().await.expect("invalidation should succeed"); + + assert_eq!(outcome, CacheInvalidationOutcome::Invalidated); + assert_eq!(cache.metadata(&committed_key).await.expect("metadata should read"), None); + + cache.write_bytes_and_commit(&committed_key, b"segment-body").await.expect("second commit should succeed"); + let active_key = SegmentCacheKey::new(ProxySessionId("proxy_session".to_string()), 124, "ts"); + let staged = cache + .stage_temp_with_deadline(&active_key, &b"done"[..], tokio::time::Instant::now() + Duration::from_mins(1)) + .await + .expect("active object stages"); + assert!(cache.has_active_temp_files()); + + let outcome = cache.invalidate_all_if_no_active_temp_files().await.expect("deferred invalidation should succeed"); + + assert_eq!(outcome, CacheInvalidationOutcome::DeferredActiveTempFiles); + assert!(cache.metadata(&committed_key).await.expect("metadata should read").is_some()); + cache.commit_staged(&active_key, staged).await.expect("staged object commits"); +} + +#[tokio::test] +async fn invalidate_all_refuses_unmarked_cache_root() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let unrelated_file = temp_dir.path().join("unrelated"); + tokio::fs::write(&unrelated_file, b"keep").await.expect("fixture should write"); + + let err = cache.invalidate_all().await.expect_err("unmarked root should be refused"); + + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); + assert!(unrelated_file.exists()); +} diff --git a/backend/hls/src/cache/tests/publication.rs b/backend/hls/src/cache/tests/publication.rs new file mode 100644 index 000000000..a765361f4 --- /dev/null +++ b/backend/hls/src/cache/tests/publication.rs @@ -0,0 +1,71 @@ +use super::*; + +#[tokio::test] +async fn failed_projected_capacity_admission_returns_a_live_revision_token() { + let (_temp_dir, cache, first, _) = cache_with_projected_session_pressure().await; + let error = cache + .ensure_projected_write_capacity(&first, 3) + .await + .expect_err("projected object exceeds the remaining session budget"); + let revision = hls_cache_capacity_from_io(&error).expect("typed capacity error").revision().clone(); + let wait = cache.wait_for_capacity_change(&revision); + tokio::pin!(wait); + + assert!(matches!(futures::poll!(wait.as_mut()), Poll::Pending)); + + cache.notify_capacity_protection_changed(); + assert!(matches!(futures::poll!(wait.as_mut()), Poll::Ready(()))); +} + +#[tokio::test] +async fn failed_admission_revision_is_captured_with_the_capacity_decision() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let path = temp_dir.path().join("session/000001.ts"); + { + let mut capacity = cache.capacity.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + capacity.cache_path = temp_dir.path().to_path_buf(); + capacity.initialized = true; + capacity.total_bytes = 2; + capacity.session_bytes.insert("session".to_string(), 2); + } + let mut reservation = + cache.try_begin_capacity_mutation(temp_dir.path(), &path, "session".to_string()).expect("mutation reservation"); + let error = reservation.reserve_replacement(0, 3, 2, 2).expect_err("capacity is exceeded"); + + cache.notify_capacity_protection_changed(); + drop(reservation); + + let super::super::CapacityReservationError::Exceeded { revision, .. } = error else { + panic!("expected typed capacity pressure"); + }; + let wait = cache.wait_for_capacity_change(&revision); + tokio::pin!(wait); + assert!(matches!(futures::poll!(wait.as_mut()), Poll::Ready(()))); +} + +#[tokio::test] +async fn cache_path_change_during_write_rejects_old_generation_commit() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let old_root = temp_dir.path().join("old"); + let new_root = temp_dir.path().join("new"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(&old_root)); + let key = cache_key(); + let old_final_path = old_root.join("proxy_session/000123.ts"); + let (started_tx, started_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + let reader = ControlledReader { started: Some(started_tx), release: release_rx, body: None }; + let task_cache = Arc::clone(&cache); + let task_key = key.clone(); + let write_task = tokio::spawn(async move { task_cache.write_temp_and_commit(&task_key, reader).await }); + started_rx.await.expect("write reaches controlled reader"); + + assert!(cache.update_cache_path(&new_root).await); + release_tx.send(b"segment-body".to_vec()).expect("release write"); + let error = write_task.await.expect("write task joins").expect_err("old generation commit must be rejected"); + + assert_eq!(error.kind(), io::ErrorKind::Interrupted); + assert!(!old_final_path.exists()); + assert!(cache.metadata(&key).await.expect("new-root metadata reads").is_none()); + assert!(!cache.has_active_temp_files()); +} diff --git a/backend/hls/src/cache/tests/storage.rs b/backend/hls/src/cache/tests/storage.rs new file mode 100644 index 000000000..c402cba47 --- /dev/null +++ b/backend/hls/src/cache/tests/storage.rs @@ -0,0 +1,22 @@ +use super::*; + +#[tokio::test] +async fn gc_style_delete_defers_while_the_object_has_an_active_mutation() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + cache.write_bytes_and_commit(&key, b"segment-body").await.expect("fixture commit"); + let cache_path = cache.cache_path_snapshot(); + let path = cache.object_path(&key); + let reservation = cache + .try_begin_capacity_mutation(&cache_path.path, &path, key.session_path_component()) + .expect("test mutation reserves"); + + let error = cache.delete_if_inactive(&key).await.expect_err("active mutation protects object"); + + assert_eq!(error.kind(), io::ErrorKind::WouldBlock); + assert!(cache.metadata(&key).await.expect("metadata").is_some()); + drop(reservation); + cache.delete_if_inactive(&key).await.expect("delete resumes after mutation"); + assert!(cache.metadata(&key).await.expect("metadata").is_none()); +} diff --git a/backend/hls/src/cache/tests/streaming.rs b/backend/hls/src/cache/tests/streaming.rs new file mode 100644 index 000000000..e58f56a7b --- /dev/null +++ b/backend/hls/src/cache/tests/streaming.rs @@ -0,0 +1,8 @@ +use super::*; + +#[test] +fn segment_cache_key_contains_no_origin_data() { + let key = cache_key(); + + assert_eq!(key.stable_value(), "hls:proxy_session:00000000000000000123"); +} diff --git a/backend/hls/src/cache/tests/terminal.rs b/backend/hls/src/cache/tests/terminal.rs new file mode 100644 index 000000000..6cba67692 --- /dev/null +++ b/backend/hls/src/cache/tests/terminal.rs @@ -0,0 +1,15 @@ +use super::*; + +#[tokio::test] +async fn exhausted_write_deadline_rejects_even_an_immediately_available_body() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let key = cache_key(); + + let result = cache.write_temp_and_commit_with_deadline(&key, &b"ready"[..], tokio::time::Instant::now()).await; + + assert_eq!(result.expect_err("expired deadline must time out").kind(), io::ErrorKind::TimedOut); + assert!(!cache.has_active_temp_files()); + assert_eq!(cache.metadata(&key).await.expect("metadata should read"), None); + assert_eq!(std::fs::read_dir(temp_dir.path()).expect("cache root reads").count(), 0); +} diff --git a/backend/hls/src/cache/tests/transport.rs b/backend/hls/src/cache/tests/transport.rs new file mode 100644 index 000000000..ad3cfec25 --- /dev/null +++ b/backend/hls/src/cache/tests/transport.rs @@ -0,0 +1,70 @@ +use super::*; + +#[test] +fn cache_key_debug_redacts_proxy_session_id() { + let key = SegmentCacheKey::new(ProxySessionId("secretToken".to_string()), 123, "ts"); + let debug = format!("{key:?}"); + + assert!(debug.contains("")); + assert!(!debug.contains("secretToken")); + assert!(!debug.contains(&key.stable_value())); +} + +#[test] +fn transient_object_cache_key_keeps_redirect_hosts_distinct_without_leaking_urls() { + let proxy_session_id = ProxySessionId("proxy_session".to_string()); + let first_resource = build_transient_resource_id("https://cdn-a.example.net/live/redirected/seg001.ts", b"secret"); + let second_resource = build_transient_resource_id("https://cdn-b.example.net/live/redirected/seg001.ts", b"secret"); + + let first = TransientObjectCacheKey::new(proxy_session_id.clone(), first_resource, "ts"); + let second = TransientObjectCacheKey::new(proxy_session_id, second_resource, "ts"); + + assert_ne!(first, second); + assert_ne!(first.stable_value(), second.stable_value()); + for value in [first.stable_value(), second.stable_value()] { + assert!(!value.contains("provider://")); + assert!(!value.contains("cdn-a.example.net")); + assert!(!value.contains("cdn-b.example.net")); + assert!(!value.contains("/live/redirected/seg001.ts")); + } +} + +#[tokio::test] +async fn cache_object_write_rejects_bytes_above_the_configured_limit() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + cache.update_cache_limits(3, 3); + let key = SegmentCacheKey::new(ProxySessionId("session".to_string()), 1, "ts"); + + let result = cache.write_bytes_and_commit(&key, b"four").await; + + let error = result.expect_err("decoded object above the configured limit must fail"); + let limit_error = hls_cache_object_limit_from_io(&error).expect("typed object-limit source"); + assert_eq!(limit_error.limit(), 3); + assert!(matches!( + HlsOriginResourceFetchError::cache_body(&error), + HlsOriginResourceFetchError::CacheObjectLimit { limit: 3 } + )); + assert!(cache.metadata(&key).await.expect("metadata").is_none()); + assert!(!cache.has_active_temp_files()); +} + +#[tokio::test] +async fn cache_commits_enforce_the_global_budget_across_sessions() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + cache.update_cache_limits(5, 5); + let first = SegmentCacheKey::new(ProxySessionId("first".to_string()), 1, "ts"); + let second = SegmentCacheKey::new(ProxySessionId("second".to_string()), 1, "ts"); + + assert!(cache.write_bytes_and_commit(&first, b"123").await.is_ok()); + let error = cache.write_bytes_and_commit(&second, b"456").await.expect_err("global limit rejects commit"); + let capacity = hls_cache_capacity_from_io(&error).expect("typed local capacity error"); + assert_eq!(capacity.required_session_bytes(), 0); + assert_eq!(capacity.required_global_bytes(), 1); + assert!(cache.metadata(&second).await.expect("metadata").is_none()); + assert!(!cache.has_active_temp_files()); + let first_usage = cache.capacity_usage(first.proxy_session_id()).await.expect("usage"); + assert_eq!(first_usage.global_bytes, 3); + assert_eq!(first_usage.session_bytes, 3); +} diff --git a/backend/hls/src/gc.rs b/backend/hls/src/gc.rs index b2116c27d..76eca2369 100644 --- a/backend/hls/src/gc.rs +++ b/backend/hls/src/gc.rs @@ -7,207 +7,27 @@ use super::{ TransientResourceId, }; use arc_swap::ArcSwap; -use futures::{future::BoxFuture, FutureExt}; -use log::{debug, error, info, warn}; -use sha2::{Digest, Sha256}; use std::{ - collections::{HashSet, VecDeque}, - io, - path::PathBuf, + collections::VecDeque, sync::{Arc, Mutex as StdMutex, RwLock as StdRwLock, Weak}, - time::{Duration, SystemTime}, + time::Duration, }; use tokio::sync::{Mutex as AsyncMutex, RwLock}; -use tokio_util::sync::CancellationToken; -use tuliprox_core::model::HlsCacheConfig; const HLS_CACHE_GC_INTERVAL: Duration = Duration::from_secs(30); + const DEFAULT_TEMP_FILE_RETENTION_MS: u64 = 30_000; + const DEFAULT_FAILED_SEGMENT_RETENTION_MS: u64 = 10_000; + const MAX_PENDING_CACHE_DELETIONS: usize = 1_024; + const MAX_CACHE_DELETE_RETRIES_PER_RUN: usize = 128; + // A switch can own one segment and one MAP rollback. Keep capacity for 64 concurrent handoffs even while a GC batch // is selecting ordinary deletions; the shared hard bound remains MAX_PENDING_CACHE_DELETIONS. const SWITCH_CACHE_CLEANUP_HEADROOM: usize = 128; -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct GarbageCollectionPolicy { - pub cache_duration_ms: u64, - pub cache_bytes_global: u64, - pub cache_bytes_per_session: u64, - pub session_idle_timeout_ms: u64, - pub temp_file_retention_ms: u64, - pub failed_segment_retention_ms: u64, -} - -impl GarbageCollectionPolicy { - pub fn from_config(config: &HlsCacheConfig) -> Self { - Self { - cache_duration_ms: config.cache_duration.as_millis().get(), - cache_bytes_global: config.cache_bytes.get(), - cache_bytes_per_session: config.cache_bytes_per_session.get(), - session_idle_timeout_ms: config.session_idle_timeout.as_millis().get(), - temp_file_retention_ms: DEFAULT_TEMP_FILE_RETENTION_MS, - failed_segment_retention_ms: DEFAULT_FAILED_SEGMENT_RETENTION_MS, - } - } -} - -impl Default for GarbageCollectionPolicy { - fn default() -> Self { - let default_config = HlsCacheConfig::from(&shared::model::HlsCacheConfigDto::default()); - Self::from_config(&default_config) - } -} - -#[derive(Debug, Default, Clone, Eq, PartialEq)] -pub struct ProtectedSet { - pub segment_proxy_seqs: HashSet, - pub map_ids: HashSet, - pub key_resource_ids: HashSet, - pub transient_object_ids: HashSet, -} - -impl ProtectedSet { - #[cfg(test)] - pub fn from_session(session: &HlsSession) -> Self { Self::from_session_for_capacity(session, None) } - - fn from_session_for_capacity(session: &HlsSession, release_through: Option) -> Self { - let mut protected = Self::default(); - - if let Some(rendered) = &session.last_rendered_manifest { - protected.segment_proxy_seqs.extend( - rendered - .segment_proxy_seqs - .iter() - .copied() - .filter(|proxy_seq| release_through.is_none_or(|release| *proxy_seq > release)), - ); - } - if let Some(rendered) = &session.last_rendered_manifest { - protected.key_resource_ids.extend(extract_transient_resource_ids(&rendered.body)); - } - for (_, protection) in session.terminal_tail_protections() { - protected.segment_proxy_seqs.extend(protection.base_proxy_seqs.iter().copied()); - } - protected.segment_proxy_seqs.extend( - renderer_candidate_window_proxy_seqs(session) - .into_iter() - .filter(|proxy_seq| release_through.is_none_or(|release| *proxy_seq > release)), - ); - - for (proxy_seq, entry) in &session.segments { - if entry.access.active_readers() > 0 - || matches!(entry.status, SegmentCacheStatus::Fetching { .. }) - || (entry.origin_fetch_ref.is_some() - && (matches!(entry.status, SegmentCacheStatus::Queued { .. }) - || session.segment_prefetch_queue.contains(*proxy_seq))) - { - protected.segment_proxy_seqs.insert(*proxy_seq); - } - } - - for proxy_seq in &protected.segment_proxy_seqs { - if let Some(segment) = session.segments.get(proxy_seq) { - if let Some(map_ref) = segment.map_ref { - protected.map_ids.insert(map_ref); - } - if let Some(encryption) = &segment.encryption { - protected.key_resource_ids.insert(encryption.resource_id.clone()); - } - } - } - for (map_id, map) in &session.maps { - if map.access.active_readers() > 0 - || matches!(map.status, MapCacheStatus::Queued { .. } | MapCacheStatus::Fetching { .. }) - { - protected.map_ids.insert(*map_id); - } - } - for entry in session.transient.object_cache.values() { - if entry.access.active_readers() > 0 { - protected.transient_object_ids.insert(entry.key.transient_resource_id().clone()); - } - } - protected.transient_object_ids.extend(protected.key_resource_ids.iter().cloned()); - - protected - } -} - -fn protected_set_for_cache_reclamation( - session: &HlsSession, - cache: &HlsSegmentCache, - excluded_path: Option<&std::path::Path>, - release_through: Option, -) -> ProtectedSet { - let mut protected = ProtectedSet::from_session_for_capacity(session, release_through); - for (proxy_seq, segment) in &session.segments { - let path = cache.object_path(&segment.cache_key); - if cache.has_active_mutation(&path) || excluded_path == Some(path.as_path()) { - protected.segment_proxy_seqs.insert(*proxy_seq); - } - } - for (map_id, map) in &session.maps { - let path = cache.object_path(&map.cache_key); - if cache.has_active_mutation(&path) || excluded_path == Some(path.as_path()) { - protected.map_ids.insert(*map_id); - } - } - for entry in session.transient.object_cache.values() { - let path = cache.object_path(&entry.key); - if cache.has_active_mutation(&path) || excluded_path == Some(path.as_path()) { - protected.transient_object_ids.insert(entry.key.transient_resource_id().clone()); - } - } - protected -} - -#[derive(Debug, Default, Clone, Eq, PartialEq)] -pub struct GarbageCollectionReport { - pub secret_cache_invalidated: bool, - pub secret_cache_invalidation_deferred: bool, - pub temp_files_deleted: usize, - pub orphan_session_dirs_deleted: usize, - pub stale_queue_entries_removed: usize, - pub segments_deleted_duration: usize, - pub segments_deleted_size_session: usize, - pub segments_deleted_size_global: usize, - pub maps_deleted: usize, - pub sessions_deleted: usize, - pub removed_session_ids: Vec, - pub transient_resources_pruned: usize, - pub transient_objects_deleted: usize, - pub transient_object_bytes_deleted: u64, - pub cache_object_deletions_planned: usize, - pub cache_object_deletions_succeeded: usize, - pub cache_object_deletions_deferred: usize, -} - -impl GarbageCollectionReport { - fn segments_deleted(&self) -> usize { - self.segments_deleted_duration - .saturating_add(self.segments_deleted_size_session) - .saturating_add(self.segments_deleted_size_global) - } - - pub fn did_cleanup_or_invalidate(&self) -> bool { - self.secret_cache_invalidated - || self.secret_cache_invalidation_deferred - || self.temp_files_deleted > 0 - || self.orphan_session_dirs_deleted > 0 - || self.stale_queue_entries_removed > 0 - || self.segments_deleted() > 0 - || self.maps_deleted > 0 - || self.sessions_deleted > 0 - || self.transient_resources_pruned > 0 - || self.transient_objects_deleted > 0 - || self.cache_object_deletions_planned > 0 - || self.cache_object_deletions_succeeded > 0 - || self.cache_object_deletions_deferred > 0 - } -} - pub struct HlsGarbageCollector { sessions: Arc, cache: Arc, @@ -219,839 +39,18 @@ pub struct HlsGarbageCollector { run_once_gate: AsyncMutex<()>, } -impl HlsGarbageCollector { - pub fn new( - sessions: Arc, - cache: Arc, - policy: GarbageCollectionPolicy, - rewrite_secret_fingerprint: String, - ) -> Self { - Self::new_with_metrics( - sessions, - cache, - policy, - rewrite_secret_fingerprint, - Arc::new(HlsCacheMetrics::default()), - ) - } - - pub fn new_with_metrics( - sessions: Arc, - cache: Arc, - policy: GarbageCollectionPolicy, - rewrite_secret_fingerprint: String, - metrics: Arc, - ) -> Self { - Self { - sessions, - cache, - policy: ArcSwap::from_pointee(policy), - rewrite_secret_fingerprint: ArcSwap::from_pointee(rewrite_secret_fingerprint), - metrics, - pending_cache_deletions: Arc::new(StdMutex::new(CacheDeletionQueueState::default())), - access_leases: StdRwLock::new(None), - run_once_gate: AsyncMutex::new(()), - } - } - - pub fn install_access_leases(&self, access_leases: &Arc>) { - *self.access_leases.write().unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(Arc::downgrade(access_leases)); - } - - pub fn update_config(&self, policy: GarbageCollectionPolicy, rewrite_secret_fingerprint: String) { - self.policy.store(Arc::new(policy)); - self.rewrite_secret_fingerprint.store(Arc::new(rewrite_secret_fingerprint)); - } - - /// Serializes a cache-root handoff with GC and discards logical deletion tickets bound to the previous root. - pub async fn update_cache_path(&self, cache_path: impl Into) -> bool { - let _run_once = self.run_once_gate.lock().await; - let changed = self.cache.update_cache_path(cache_path).await; - if changed { - let mut queue = self.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - *queue = CacheDeletionQueueState::default(); - } - changed - } - - pub fn policy(&self) -> Arc { self.policy.load_full() } - - pub fn rewrite_secret_fingerprint(&self) -> String { self.rewrite_secret_fingerprint.load().to_string() } - - #[allow(clippy::too_many_lines)] - async fn reclaim_for_projected_write( - &self, - request: HlsCacheCapacityReclaimRequest, - ) -> io::Result { - let _run_once = self.run_once_gate.lock().await; - if !self.cache.contains_current_cache_path(&request.target_path) { - return Err(io::Error::new( - io::ErrorKind::Interrupted, - "hls cache path changed before capacity reclamation", - )); - } - let before = self.cache.capacity_usage(&request.proxy_session_id).await?; - let mut report = GarbageCollectionReport::default(); - let mut deletion_attempt_budget = MAX_CACHE_DELETE_RETRIES_PER_RUN; - let mut protected_working_set_bytes = 0_u64; - let mut reclaimable_bytes = 0_u64; - - let mut pending_attempt_budget = deletion_attempt_budget / 2; - let pending_attempts_before = pending_attempt_budget; - self.retry_pending_cache_deletions(&mut report, &mut pending_attempt_budget).await; - deletion_attempt_budget = - deletion_attempt_budget.saturating_sub(pending_attempts_before.saturating_sub(pending_attempt_budget)); - let after_pending = self.cache.capacity_usage(&request.proxy_session_id).await?; - let required_session_bytes = request - .required_session_bytes - .saturating_sub(before.session_bytes.saturating_sub(after_pending.session_bytes)); - if required_session_bytes > 0 { - if let Some(session) = self.sessions.get_by_proxy_session_id(&request.proxy_session_id).await { - let mut deletions = self.reserve_cache_deletion_batch(); - let access_leases = self - .access_leases - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_ref() - .and_then(Weak::upgrade); - let lease_guard = match &access_leases { - Some(access_leases) => Some(access_leases.read().await), - None => None, - }; - // Projected-capacity selection follows the runtime lock order - // lease store -> session. The session lock is non-blocking - // while the lease snapshot is frozen, and neither guard - // crosses filesystem I/O or an await. Contention installs a - // bounded wake task instead of reversing the lock order. - let release_through = - lease_guard.as_ref().and_then(|leases| leases.capacity_release_through(&request.proxy_session_id)); - let mut session_lock_busy = false; - if let Ok(mut session) = session.try_write() { - let protected = protected_set_for_cache_reclamation( - &session, - &self.cache, - Some(&request.target_path), - release_through, - ); - let evidence = capacity_reclamation_evidence(&session, &protected); - protected_working_set_bytes = evidence.protected_working_set_bytes; - reclaimable_bytes = evidence.reclaimable_bytes; - collect_projected_session_reclamation( - &mut session, - &self.cache, - &request.target_path, - release_through, - required_session_bytes, - deletion_attempt_budget, - &mut deletions, - ); - if !deletions.deletions.is_empty() { - let rendered_at_ms = session - .last_rendered_manifest - .as_ref() - .map_or(session.last_client_access_at_ms, |rendered| { - rendered.rendered_at_ms.saturating_add(1) - }); - let _ = session.render_and_store_manifest(rendered_at_ms); - } - } else { - protected_working_set_bytes = after_pending.session_bytes; - session_lock_busy = true; - } - drop(lease_guard); - if session_lock_busy { - let wake_session = Arc::clone(&session); - let wake_cache = Arc::clone(&self.cache); - tokio::spawn(async move { - let guard = wake_session.write().await; - drop(guard); - wake_cache.notify_capacity_protection_changed(); - }); - } - deletions.execute_prioritized(&self.cache, &mut report, &mut deletion_attempt_budget).await; - } - } - - let after_session = self.cache.capacity_usage(&request.proxy_session_id).await?; - let remaining_session_bytes = request - .required_session_bytes - .saturating_sub(before.session_bytes.saturating_sub(after_session.session_bytes)); - if remaining_session_bytes > 0 { - return Ok(HlsCacheCapacityReclaimOutcome { - reclaimed_session_bytes: before.session_bytes.saturating_sub(after_session.session_bytes), - reclaimed_global_bytes: before.global_bytes.saturating_sub(after_session.global_bytes), - protected_working_set_bytes, - reclaimable_bytes, - }); - } - let required_global_bytes = request - .required_global_bytes - .saturating_sub(before.global_bytes.saturating_sub(after_session.global_bytes)); - if required_global_bytes > 0 { - let sessions = self.sessions.list_sessions().await; - let mut deletions = self.reserve_cache_deletion_batch(); - self.collect_projected_global_reclamation( - &sessions, - &request.target_path, - required_global_bytes, - deletion_attempt_budget, - &mut deletions, - ) - .await; - deletions.execute_prioritized(&self.cache, &mut report, &mut deletion_attempt_budget).await; - } - - let after = self.cache.capacity_usage(&request.proxy_session_id).await?; - Ok(HlsCacheCapacityReclaimOutcome { - reclaimed_session_bytes: before.session_bytes.saturating_sub(after.session_bytes), - reclaimed_global_bytes: before.global_bytes.saturating_sub(after.global_bytes), - protected_working_set_bytes, - reclaimable_bytes, - }) - } - - pub async fn run_once(&self, now_ms: u64) -> io::Result { - let _run_once = self.run_once_gate.lock().await; - let policy = self.policy.load_full(); - let mut report = GarbageCollectionReport::default(); - let mut deletion_attempt_budget = MAX_CACHE_DELETE_RETRIES_PER_RUN; - self.retry_pending_cache_deletions(&mut report, &mut deletion_attempt_budget).await; - if self.ensure_cache_marker(&mut report).await? { - report.cache_object_deletions_deferred = self.pending_cache_deletion_count(); - self.record_report_metrics(&report); - return Ok(report); - } - - // Captured before the in-memory session snapshot so any directory committed - // after this instant is treated as a potential concurrent create and is - // skipped by the orphan cleanup freshness guard. - let gc_start = SystemTime::now(); - let cutoff = gc_start - .checked_sub(Duration::from_millis(policy.temp_file_retention_ms)) - .unwrap_or(SystemTime::UNIX_EPOCH); - report.temp_files_deleted = self.cache.delete_temp_files_older_than(cutoff).await?; - - let sessions = self.sessions.list_sessions().await; - let mut active_session_ids = HashSet::with_capacity(sessions.len()); - for session in &sessions { - active_session_ids.insert(session.read().await.proxy_session_id.clone()); - } - report.orphan_session_dirs_deleted = - self.cache.delete_orphan_session_dirs(&active_session_ids, gc_start).await?; - for session in &sessions { - let mut session_deletions = self.reserve_cache_deletion_batch(); - let mut session = session.write().await; - Self::collect_session_deletions( - &mut session, - &self.cache, - now_ms, - &policy, - &mut report, - &mut session_deletions, - ); - session_deletions.persist(&mut report); - } - - let mut global_deletions = self.reserve_cache_deletion_batch(); - self.collect_global_size_deletions(&sessions, &policy, &mut report, &mut global_deletions).await; - global_deletions.persist(&mut report); - self.retry_pending_cache_deletions(&mut report, &mut deletion_attempt_budget).await; - - for session in &sessions { - self.remove_idle_session_if_still_idle(session, now_ms, &policy, &mut report).await; - } - - report.cache_object_deletions_deferred = self.pending_cache_deletion_count(); - self.record_report_metrics(&report); - if report.did_cleanup_or_invalidate() { - info!( - "HLS session garbage collection completed: temp_files_deleted={} orphan_session_dirs_deleted={} stale_queue_entries_removed={} cache_deletes_planned={} cache_deletes_succeeded={} cache_deletes_deferred={} segments_deleted={} maps_deleted={} transient_resources_pruned={} transient_objects_deleted={} transient_object_bytes_deleted={} sessions_deleted={}", - report.temp_files_deleted, - report.orphan_session_dirs_deleted, - report.stale_queue_entries_removed, - report.cache_object_deletions_planned, - report.cache_object_deletions_succeeded, - report.cache_object_deletions_deferred, - report.segments_deleted(), - report.maps_deleted, - report.transient_resources_pruned, - report.transient_objects_deleted, - report.transient_object_bytes_deleted, - report.sessions_deleted, - ); - } - Ok(report) - } - - async fn ensure_cache_marker(&self, report: &mut GarbageCollectionReport) -> io::Result { - let rewrite_secret_fingerprint = self.rewrite_secret_fingerprint.load_full(); - match self.cache.read_rewrite_secret_fingerprint().await? { - Some(current) if current == *rewrite_secret_fingerprint => Ok(false), - Some(_) => { - self.metrics.record_secret_marker_mismatch(); - warn!("HLS rewrite secret changed or cache marker mismatch detected: action=validate-cache-marker"); - match self.cache.invalidate_all_if_no_active_temp_files().await? { - CacheInvalidationOutcome::Invalidated => { - self.sessions.clear().await; - self.cache.write_rewrite_secret_fingerprint(&rewrite_secret_fingerprint).await?; - report.secret_cache_invalidated = true; - info!("HLS rewrite secret changed or cache marker mismatch detected: action=cache-invalidated"); - Ok(true) - } - CacheInvalidationOutcome::DeferredActiveTempFiles => { - report.secret_cache_invalidation_deferred = true; - self.metrics.record_secret_invalidation_deferred(); - warn!( - "HLS rewrite secret changed or cache marker mismatch detected: action=deferred-active-temp-files" - ); - Ok(true) - } - } - } - None => { - self.cache.write_rewrite_secret_fingerprint(&rewrite_secret_fingerprint).await?; - Ok(false) - } - } - } - - fn collect_session_deletions( - session: &mut HlsSession, - cache: &HlsSegmentCache, - now_ms: u64, - policy: &GarbageCollectionPolicy, - report: &mut GarbageCollectionReport, - deletions: &mut CacheDeletionBatch, - ) { - report.stale_queue_entries_removed = - report.stale_queue_entries_removed.saturating_add(remove_stale_queue_entries(session)); - - let transient_before = session.transient.resources.len(); - let protected = protected_set_for_cache_reclamation(session, cache, None, None); - session.transient.prune_expired_except(now_ms, &protected.key_resource_ids); - let transient_resources_pruned = transient_before.saturating_sub(session.transient.resources.len()); - let mut transient_readiness_changed = transient_resources_pruned > 0; - report.transient_resources_pruned = - report.transient_resources_pruned.saturating_add(transient_resources_pruned); - - let expired_transient_objects = take_expired_transient_objects( - session, - now_ms, - &protected.transient_object_ids, - deletions.remaining_capacity(), - ); - transient_readiness_changed |= !expired_transient_objects.is_empty(); - for (key, content_length) in expired_transient_objects { - deletions.push(CacheObjectDeletion::TransientObject { key, content_length }); - } - - while deletions.has_capacity() { - let Some(proxy_seq) = duration_expired_head_segment( - session, - &protected_set_for_cache_reclamation(session, cache, None, None), - policy, - now_ms, - ) else { - break; - }; - if let Some(deletion) = remove_segment_entry(session, proxy_seq) { - deletions - .push(CacheObjectDeletion::Segment { key: deletion, reason: SegmentCacheDeletionReason::Duration }); - } - } - - let mut session_size = session_cache_size(session); - while session_size > policy.cache_bytes_per_session && deletions.has_capacity() { - let protected = protected_set_for_cache_reclamation(session, cache, None, None); - let Some(removal) = session.transient.remove_oldest_ready_object_except(&protected.transient_object_ids) - else { - break; - }; - session_size = session_size.saturating_sub(removal.content_length); - transient_readiness_changed = true; - deletions.push(CacheObjectDeletion::TransientObject { - key: removal.key, - content_length: removal.content_length, - }); - } - while session_size > policy.cache_bytes_per_session && deletions.has_capacity() { - let Some(candidate) = - fifo_head_size_candidate(session, &protected_set_for_cache_reclamation(session, cache, None, None)) - else { - break; - }; - session_size = session_size.saturating_sub(candidate.content_length); - if let Some(deletion) = remove_segment_entry(session, candidate.proxy_seq) { - deletions.push(CacheObjectDeletion::Segment { - key: deletion, - reason: SegmentCacheDeletionReason::SessionSize, - }); - } - } - - for map_id in - unprotected_unreferenced_map_ids(session, &protected_set_for_cache_reclamation(session, cache, None, None)) - { - if !deletions.has_capacity() { - break; - } - if let Some(deletion) = remove_map_entry(session, map_id) { - session_size = session_size.saturating_sub(deletion.content_length); - deletions.push(CacheObjectDeletion::Map(deletion.key)); - } - } - if transient_readiness_changed { - session.advance_media_readiness_generation(); - } - } - - async fn collect_global_size_deletions( - &self, - sessions: &[HlsSessionHandle], - policy: &GarbageCollectionPolicy, - report: &mut GarbageCollectionReport, - deletions: &mut CacheDeletionBatch, - ) { - let mut total_size = total_sessions_cache_size(sessions).await; - let mut transient_readiness_advanced = HashSet::new(); - let mut skipped_transient_sessions = HashSet::new(); - - loop { - if total_size <= policy.cache_bytes_global || !deletions.has_capacity() { - break; - } - let Some(candidate) = - oldest_global_transient_object_candidate(sessions, &self.cache, None, &skipped_transient_sessions) - .await - else { - break; - }; - let mut session = candidate.session.write().await; - let protected = protected_set_for_cache_reclamation(&session, &self.cache, None, None); - let Some(removal) = session.transient.remove_oldest_ready_object_except(&protected.transient_object_ids) - else { - skipped_transient_sessions.insert(candidate.proxy_session_id); - continue; - }; - total_size = total_size.saturating_sub(removal.content_length); - if transient_readiness_advanced.insert(session.proxy_session_id.clone()) { - session.advance_media_readiness_generation(); - } - deletions.push(CacheObjectDeletion::TransientObject { - key: removal.key, - content_length: removal.content_length, - }); - deletions.persist_pending(report); - } - - let mut skipped_segment_sessions = HashSet::new(); - loop { - if total_size <= policy.cache_bytes_global || !deletions.has_capacity() { - break; - } - let Some(candidate) = - oldest_global_fifo_head_candidate(sessions, &self.cache, None, &skipped_segment_sessions).await - else { - break; - }; - let mut session = candidate.session.write().await; - let Some(current_head) = fifo_head_size_candidate( - &session, - &protected_set_for_cache_reclamation(&session, &self.cache, None, None), - ) else { - skipped_segment_sessions.insert(candidate.proxy_session_id); - continue; - }; - if current_head.proxy_seq != candidate.proxy_seq { - skipped_segment_sessions.insert(candidate.proxy_session_id); - continue; - } - let Some(deletion) = remove_segment_entry(&mut session, candidate.proxy_seq) else { - skipped_segment_sessions.insert(candidate.proxy_session_id); - continue; - }; - total_size = total_size.saturating_sub(candidate.content_length); - deletions - .push(CacheObjectDeletion::Segment { key: deletion, reason: SegmentCacheDeletionReason::GlobalSize }); - deletions.persist_pending(report); - - for map_id in unprotected_unreferenced_map_ids( - &session, - &protected_set_for_cache_reclamation(&session, &self.cache, None, None), - ) { - if !deletions.has_capacity() { - break; - } - if let Some(deletion) = remove_map_entry(&mut session, map_id) { - total_size = total_size.saturating_sub(deletion.content_length); - deletions.push(CacheObjectDeletion::Map(deletion.key)); - deletions.persist_pending(report); - } - } - } - } - - async fn collect_projected_global_reclamation( - &self, - sessions: &[HlsSessionHandle], - excluded_path: &std::path::Path, - required_bytes: u64, - max_deletions: usize, - deletions: &mut CacheDeletionBatch, - ) { - let mut planned_bytes = 0_u64; - let mut planned_deletions = 0_usize; - let mut transient_readiness_advanced = HashSet::new(); - let mut skipped_transient_sessions = HashSet::new(); - while planned_bytes < required_bytes && planned_deletions < max_deletions && deletions.has_capacity() { - let Some(candidate) = oldest_global_transient_object_candidate( - sessions, - &self.cache, - Some(excluded_path), - &skipped_transient_sessions, - ) - .await - else { - break; - }; - let mut session = candidate.session.write().await; - let protected = protected_set_for_cache_reclamation(&session, &self.cache, Some(excluded_path), None); - let Some(removal) = session.transient.remove_oldest_ready_object_except(&protected.transient_object_ids) - else { - skipped_transient_sessions.insert(candidate.proxy_session_id); - continue; - }; - planned_bytes = planned_bytes.saturating_add(removal.content_length); - if transient_readiness_advanced.insert(session.proxy_session_id.clone()) { - session.advance_media_readiness_generation(); - } - deletions.push(CacheObjectDeletion::TransientObject { - key: removal.key, - content_length: removal.content_length, - }); - planned_deletions = planned_deletions.saturating_add(1); - } - - let mut skipped_segment_sessions = HashSet::new(); - while planned_bytes < required_bytes && planned_deletions < max_deletions && deletions.has_capacity() { - let Some(candidate) = oldest_global_fifo_head_candidate( - sessions, - &self.cache, - Some(excluded_path), - &skipped_segment_sessions, - ) - .await - else { - break; - }; - let mut session = candidate.session.write().await; - let protected = protected_set_for_cache_reclamation(&session, &self.cache, Some(excluded_path), None); - let Some(current_head) = fifo_head_size_candidate(&session, &protected) else { - skipped_segment_sessions.insert(candidate.proxy_session_id); - continue; - }; - if current_head.proxy_seq != candidate.proxy_seq { - skipped_segment_sessions.insert(candidate.proxy_session_id); - continue; - } - let Some(deletion) = remove_segment_entry(&mut session, candidate.proxy_seq) else { - skipped_segment_sessions.insert(candidate.proxy_session_id); - continue; - }; - planned_bytes = planned_bytes.saturating_add(candidate.content_length); - deletions - .push(CacheObjectDeletion::Segment { key: deletion, reason: SegmentCacheDeletionReason::GlobalSize }); - planned_deletions = planned_deletions.saturating_add(1); - - for map_id in unprotected_unreferenced_map_ids( - &session, - &protected_set_for_cache_reclamation(&session, &self.cache, Some(excluded_path), None), - ) { - if planned_bytes >= required_bytes || planned_deletions >= max_deletions || !deletions.has_capacity() { - break; - } - if let Some(deletion) = remove_map_entry(&mut session, map_id) { - planned_bytes = planned_bytes.saturating_add(deletion.content_length); - deletions.push(CacheObjectDeletion::Map(deletion.key)); - planned_deletions = planned_deletions.saturating_add(1); - } - } - } - } - - async fn remove_idle_session_if_still_idle( - &self, - session: &HlsSessionHandle, - now_ms: u64, - policy: &GarbageCollectionPolicy, - report: &mut GarbageCollectionReport, - ) { - let (key, proxy_session_id) = { - let mut session = session.write().await; - if !Self::should_remove_idle_session(&session, now_ms, policy) { - return; - } - session.mark_for_gc_removal(); - (session.key.clone(), session.proxy_session_id.clone()) - }; - - if self.cache.has_active_temp_files_for_session(&proxy_session_id) { - session.write().await.clear_gc_removal_mark(); - return; - } - - if self - .sessions - .remove_session_marking_expired( - &key, - &proxy_session_id, - now_ms, - HlsExpiredSessionReason::SessionIdleTimeout, - None, - ) - .await - .is_some() - { - report.sessions_deleted = report.sessions_deleted.saturating_add(1); - report.removed_session_ids.push(proxy_session_id.clone()); - if let Err(error) = self.cache.delete_session_dir(&proxy_session_id).await { - warn!( - "HLS idle session cache directory cleanup deferred: session={} error_kind={:?}", - safe_proxy_session_id(&proxy_session_id), - error.kind(), - ); - } - } else { - session.write().await.clear_gc_removal_mark(); - } - } - - fn reserve_cache_deletion_batch(&self) -> CacheDeletionBatch { - CacheDeletionBatch::reserve(Arc::clone(&self.pending_cache_deletions)) - } - - pub fn reserve_switch_segment_cleanup(&self, key: SegmentCacheKey) -> Option { - HlsSwitchCacheCleanupReservation::reserve( - Arc::clone(&self.pending_cache_deletions), - CacheObjectDeletion::UncommittedSwitchSegment(key), - ) - } - - pub fn reserve_switch_map_cleanup(&self, key: MapCacheKey) -> Option { - HlsSwitchCacheCleanupReservation::reserve( - Arc::clone(&self.pending_cache_deletions), - CacheObjectDeletion::UncommittedSwitchMap(key), - ) - } - - pub fn has_pending_switch_cleanup(&self, segment_key: &SegmentCacheKey, map_key: Option<&MapCacheKey>) -> bool { - let state = self.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - state.pending.iter().any(|pending| match &pending.deletion { - CacheObjectDeletion::UncommittedSwitchSegment(key) => key == segment_key, - CacheObjectDeletion::UncommittedSwitchMap(key) => map_key == Some(key), - CacheObjectDeletion::Segment { .. } - | CacheObjectDeletion::Map(_) - | CacheObjectDeletion::TransientObject { .. } => false, - }) - } - - fn pending_cache_deletion_count(&self) -> usize { - self.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner).pending.len() - } - - #[cfg(any(test, feature = "test-support"))] - pub fn cache_deletion_queue_usage(&self) -> (usize, usize) { - let state = self.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - (state.pending.len(), state.reserved_slots) - } - - async fn retry_pending_cache_deletions(&self, report: &mut GarbageCollectionReport, attempt_budget: &mut usize) { - let attempts = self.pending_cache_deletion_count().min(*attempt_budget); - for _ in 0..attempts { - let pending = { - let queue = self.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - queue.pending.front().cloned() - }; - let Some(pending) = pending else { - break; - }; - *attempt_budget = attempt_budget.saturating_sub(1); - let result = pending.deletion.delete_from(&self.cache).await; - let mut queue = self.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let Some(mut completed) = queue.pending.pop_front() else { - continue; - }; - if result.is_ok() { - drop(queue); - completed.deletion.record_success(report); - completed.deletion.log_success(); - } else { - completed.attempts = completed.attempts.saturating_add(1); - let attempts = completed.attempts; - let kind = result.as_ref().err().map(io::Error::kind); - completed.deletion.log_deferred(attempts, kind); - queue.pending.push_back(completed); - } - } - } - - fn record_report_metrics(&self, report: &GarbageCollectionReport) { - self.metrics.record_gc_run(); - self.metrics.record_segments_removed(report.segments_deleted()); - self.metrics.record_maps_removed(report.maps_deleted); - } - - fn should_remove_idle_session(session: &HlsSession, now_ms: u64, policy: &GarbageCollectionPolicy) -> bool { - session.can_expire_idle_session(now_ms, policy.session_idle_timeout_ms) - } -} - -impl HlsCacheCapacityReclaimer for HlsGarbageCollector { - fn reclaim_capacity( - &self, - request: HlsCacheCapacityReclaimRequest, - ) -> BoxFuture<'_, io::Result> { - async move { self.reclaim_for_projected_write(request).await }.boxed() - } -} - -pub fn build_rewrite_secret_fingerprint(rewrite_secret: &[u8]) -> String { - let digest = Sha256::digest(rewrite_secret); - let value = digest.iter().take(8).fold(0_u64, |value, byte| (value << 8) | u64::from(*byte)); - format!("{value:016x}") -} - -pub fn exec_hls_cache_gc(ctx: &HlsCtx, cancel_token: &CancellationToken) { - let hls_proxy = Arc::clone(&ctx.hls_proxy); - let active_users = Arc::clone(&ctx.active_users); - let active_provider = Arc::clone(&ctx.active_provider); - let cancel_token = cancel_token.clone(); - tokio::spawn(async move { - loop { - let now_ms = chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default(); - hls_proxy - .sync_all_session_access_leases_and_detach_if_needed(&active_users, &active_provider, now_ms) - .await; - match hls_proxy.run_garbage_collection_once(now_ms).await { - Ok(report) if report.did_cleanup_or_invalidate() => { - debug!( - "HLS cache state snapshot after garbage collection: {}", - hls_proxy.debug_state_summary().await - ); - } - Ok(_) => {} - Err(err) => { - error!("HLS cache garbage collection failed: {err}"); - } - } - tokio::select! { - () = cancel_token.cancelled() => break, - () = tokio::time::sleep(HLS_CACHE_GC_INTERVAL) => {} - } - } - }); -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum SegmentCacheDeletionReason { - Duration, - SessionSize, - GlobalSize, -} - -#[derive(Debug, Clone, Eq, PartialEq)] -enum CacheObjectDeletion { - Segment { key: SegmentCacheKey, reason: SegmentCacheDeletionReason }, - Map(MapCacheKey), - TransientObject { key: TransientObjectCacheKey, content_length: u64 }, - UncommittedSwitchSegment(SegmentCacheKey), - UncommittedSwitchMap(MapCacheKey), -} - -impl CacheObjectDeletion { - async fn delete_from(&self, cache: &HlsSegmentCache) -> io::Result<()> { - match self { - Self::Segment { key, .. } | Self::UncommittedSwitchSegment(key) => cache.delete_if_inactive(key).await, - Self::Map(key) | Self::UncommittedSwitchMap(key) => cache.delete_if_inactive(key).await, - Self::TransientObject { key, .. } => cache.delete_if_inactive(key).await, - } - } - - fn record_success(&self, report: &mut GarbageCollectionReport) { - report.cache_object_deletions_succeeded = report.cache_object_deletions_succeeded.saturating_add(1); - match self { - Self::Segment { reason: SegmentCacheDeletionReason::Duration, .. } => { - report.segments_deleted_duration = report.segments_deleted_duration.saturating_add(1); - } - Self::Segment { reason: SegmentCacheDeletionReason::SessionSize, .. } => { - report.segments_deleted_size_session = report.segments_deleted_size_session.saturating_add(1); - } - Self::Segment { reason: SegmentCacheDeletionReason::GlobalSize, .. } => { - report.segments_deleted_size_global = report.segments_deleted_size_global.saturating_add(1); - } - Self::Map(_) => { - report.maps_deleted = report.maps_deleted.saturating_add(1); - } - Self::TransientObject { content_length, .. } => { - report.transient_objects_deleted = report.transient_objects_deleted.saturating_add(1); - report.transient_object_bytes_deleted = - report.transient_object_bytes_deleted.saturating_add(*content_length); - } - Self::UncommittedSwitchSegment(_) | Self::UncommittedSwitchMap(_) => {} - } - } - - fn log_success(&self) { - match self { - Self::Segment { key, .. } => { - info!( - "Segment '{:06}' removed: session={} source=normal", - key.proxy_seq(), - safe_proxy_session_id(key.proxy_session_id()), - ); - } - Self::UncommittedSwitchSegment(_) | Self::UncommittedSwitchMap(_) => { - debug!("HLS uncommitted switch cache object removed"); - } - Self::Map(_) | Self::TransientObject { .. } => {} - } - } - - fn log_deferred(&self, attempts: u16, error_kind: Option) { - let object_kind = match self { - Self::Segment { .. } => "segment", - Self::Map(_) => "map", - Self::TransientObject { .. } => "transient-object", - Self::UncommittedSwitchSegment(_) => "uncommitted-switch-segment", - Self::UncommittedSwitchMap(_) => "uncommitted-switch-map", - }; - warn!( - "HLS cache object deletion deferred: object_kind={object_kind} attempts={attempts} error_kind={error_kind:?}" - ); - } -} - #[derive(Debug, Clone, Eq, PartialEq)] struct PendingCacheObjectDeletion { deletion: CacheObjectDeletion, attempts: u16, } -struct CacheDeletionQueueGeneration; - struct CacheDeletionQueueState { pending: VecDeque, reserved_slots: usize, generation: Arc, } -impl Default for CacheDeletionQueueState { - fn default() -> Self { - Self { pending: VecDeque::new(), reserved_slots: 0, generation: Arc::new(CacheDeletionQueueGeneration) } - } -} - /// Bounded rollback ownership for one physically committed but not yet timeline-committed switch object. pub struct HlsSwitchCacheCleanupReservation { queue: Arc>, @@ -1060,62 +59,6 @@ pub struct HlsSwitchCacheCleanupReservation { slot_reserved: bool, } -impl HlsSwitchCacheCleanupReservation { - fn reserve(queue: Arc>, deletion: CacheObjectDeletion) -> Option { - let queue_generation = { - let mut state = queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if state.pending.len().saturating_add(state.reserved_slots) >= MAX_PENDING_CACHE_DELETIONS { - return None; - } - state.reserved_slots = state.reserved_slots.saturating_add(1); - Arc::clone(&state.generation) - }; - Some(Self { queue, queue_generation, deletion: Some(deletion), slot_reserved: true }) - } - - pub fn disarm(&mut self) { - self.deletion = None; - self.release_slot(); - } - - fn release_slot(&mut self) { - if !self.slot_reserved { - return; - } - let mut state = self.queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if Arc::ptr_eq(&state.generation, &self.queue_generation) { - state.reserved_slots = state.reserved_slots.saturating_sub(1); - } - self.slot_reserved = false; - } -} - -impl std::fmt::Debug for HlsSwitchCacheCleanupReservation { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("HlsSwitchCacheCleanupReservation") - .field("armed", &self.deletion.is_some()) - .field("slot_reserved", &self.slot_reserved) - .finish_non_exhaustive() - } -} - -impl Drop for HlsSwitchCacheCleanupReservation { - fn drop(&mut self) { - if !self.slot_reserved { - return; - } - let mut state = self.queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if Arc::ptr_eq(&state.generation, &self.queue_generation) { - state.reserved_slots = state.reserved_slots.saturating_sub(1); - if let Some(deletion) = self.deletion.take() { - state.pending.push_back(PendingCacheObjectDeletion { deletion, attempts: 0 }); - } - } - self.slot_reserved = false; - } -} - /// Owns a bounded reservation in the retry queue while session metadata is changed. /// /// Lock order: callers may hold a session write lock while appending to this in-memory batch, but the queue mutex is @@ -1127,100 +70,6 @@ struct CacheDeletionBatch { deletions: Vec, } -impl CacheDeletionBatch { - fn reserve(queue: Arc>) -> Self { - let reserved_slots = { - let mut state = queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let batch_limit = MAX_PENDING_CACHE_DELETIONS.saturating_sub(SWITCH_CACHE_CLEANUP_HEADROOM); - let available = batch_limit.saturating_sub(state.pending.len()).saturating_sub(state.reserved_slots); - state.reserved_slots = state.reserved_slots.saturating_add(available); - available - }; - Self { queue, reserved_slots, deletions: Vec::new() } - } - - fn has_capacity(&self) -> bool { self.deletions.len() < self.reserved_slots } - - fn remaining_capacity(&self) -> usize { self.reserved_slots.saturating_sub(self.deletions.len()) } - - fn push(&mut self, deletion: CacheObjectDeletion) { self.deletions.push(deletion); } - - fn persist_pending(&mut self, report: &mut GarbageCollectionReport) { - let planned = self.deletions.len(); - if planned == 0 { - return; - } - { - let mut state = self.queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - state.reserved_slots = state.reserved_slots.saturating_sub(planned); - state - .pending - .extend(self.deletions.drain(..).map(|deletion| PendingCacheObjectDeletion { deletion, attempts: 0 })); - } - self.reserved_slots = self.reserved_slots.saturating_sub(planned); - report.cache_object_deletions_planned = report.cache_object_deletions_planned.saturating_add(planned); - } - - fn persist(mut self, report: &mut GarbageCollectionReport) { self.persist_pending(report); } - - async fn execute_prioritized( - mut self, - cache: &HlsSegmentCache, - report: &mut GarbageCollectionReport, - attempt_budget: &mut usize, - ) { - while *attempt_budget > 0 { - let Some(deletion) = self.deletions.first().cloned() else { - break; - }; - *attempt_budget = attempt_budget.saturating_sub(1); - let result = deletion.delete_from(cache).await; - let completed = self.deletions.remove(0); - self.release_reserved_slot(); - report.cache_object_deletions_planned = report.cache_object_deletions_planned.saturating_add(1); - match result { - Ok(()) => { - completed.record_success(report); - completed.log_success(); - } - Err(error) => { - completed.log_deferred(1, Some(error.kind())); - self.persist_failed(completed); - } - } - } - self.persist_pending(report); - } - - fn release_reserved_slot(&mut self) { - let mut state = self.queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - state.reserved_slots = state.reserved_slots.saturating_sub(1); - self.reserved_slots = self.reserved_slots.saturating_sub(1); - } - - fn persist_failed(&mut self, deletion: CacheObjectDeletion) { - self.queue - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .pending - .push_back(PendingCacheObjectDeletion { deletion, attempts: 1 }); - } -} - -impl Drop for CacheDeletionBatch { - fn drop(&mut self) { - if self.reserved_slots == 0 && self.deletions.is_empty() { - return; - } - let mut state = self.queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - state.reserved_slots = state.reserved_slots.saturating_sub(self.reserved_slots); - state - .pending - .extend(self.deletions.drain(..).map(|deletion| PendingCacheObjectDeletion { deletion, attempts: 0 })); - self.reserved_slots = 0; - } -} - #[derive(Debug, Clone, Eq, PartialEq)] struct SegmentDeleteCandidate { proxy_seq: u64, @@ -1244,2193 +93,27 @@ struct GlobalTransientObjectCandidate { last_accessed_at_ms: u64, } -async fn total_sessions_cache_size(sessions: &[HlsSessionHandle]) -> u64 { - let mut total_size = 0_u64; - for session in sessions { - let session = session.read().await; - total_size = total_size.saturating_add(session_cache_size(&session)); - } - total_size -} - -fn collect_projected_session_reclamation( - session: &mut HlsSession, - cache: &HlsSegmentCache, - excluded_path: &std::path::Path, - release_through: Option, - required_bytes: u64, - max_deletions: usize, - deletions: &mut CacheDeletionBatch, -) { - let mut planned_bytes = 0_u64; - let mut planned_deletions = 0_usize; - let mut transient_readiness_changed = false; - while planned_bytes < required_bytes && planned_deletions < max_deletions && deletions.has_capacity() { - let protected = protected_set_for_cache_reclamation(session, cache, Some(excluded_path), release_through); - let Some(removal) = session.transient.remove_oldest_ready_object_except(&protected.transient_object_ids) else { - break; - }; - planned_bytes = planned_bytes.saturating_add(removal.content_length); - transient_readiness_changed = true; - deletions - .push(CacheObjectDeletion::TransientObject { key: removal.key, content_length: removal.content_length }); - planned_deletions = planned_deletions.saturating_add(1); - } - while planned_bytes < required_bytes && planned_deletions < max_deletions && deletions.has_capacity() { - let protected = protected_set_for_cache_reclamation(session, cache, Some(excluded_path), release_through); - let Some(candidate) = fifo_head_size_candidate(session, &protected) else { - break; - }; - if let Some(deletion) = remove_segment_entry(session, candidate.proxy_seq) { - planned_bytes = planned_bytes.saturating_add(candidate.content_length); - deletions - .push(CacheObjectDeletion::Segment { key: deletion, reason: SegmentCacheDeletionReason::SessionSize }); - planned_deletions = planned_deletions.saturating_add(1); - } - } - for map_id in unprotected_unreferenced_map_ids( - session, - &protected_set_for_cache_reclamation(session, cache, Some(excluded_path), release_through), - ) { - if planned_bytes >= required_bytes || planned_deletions >= max_deletions || !deletions.has_capacity() { - break; - } - if let Some(deletion) = remove_map_entry(session, map_id) { - planned_bytes = planned_bytes.saturating_add(deletion.content_length); - deletions.push(CacheObjectDeletion::Map(deletion.key)); - planned_deletions = planned_deletions.saturating_add(1); - } - } - if transient_readiness_changed { - session.advance_media_readiness_generation(); - } -} - -async fn oldest_global_fifo_head_candidate( - sessions: &[HlsSessionHandle], - cache: &HlsSegmentCache, - excluded_path: Option<&std::path::Path>, - skipped_sessions: &HashSet, -) -> Option { - let mut candidates = Vec::new(); - for session in sessions { - let session_guard = session.read().await; - if skipped_sessions.contains(&session_guard.proxy_session_id) { - continue; - } - if let Some(candidate) = fifo_head_size_candidate( - &session_guard, - &protected_set_for_cache_reclamation(&session_guard, cache, excluded_path, None), - ) { - candidates.push(GlobalSegmentCandidate { - session: Arc::clone(session), - proxy_session_id: session_guard.proxy_session_id.clone(), - proxy_seq: candidate.proxy_seq, - content_length: candidate.content_length, - last_relevant_at_ms: candidate.last_relevant_at_ms, - }); - } - } - candidates.into_iter().min_by_key(|candidate| (candidate.last_relevant_at_ms, candidate.proxy_seq)) -} - -async fn oldest_global_transient_object_candidate( - sessions: &[HlsSessionHandle], - cache: &HlsSegmentCache, - excluded_path: Option<&std::path::Path>, - skipped_sessions: &HashSet, -) -> Option { - let mut candidates = Vec::new(); - for session in sessions { - let session_guard = session.read().await; - if skipped_sessions.contains(&session_guard.proxy_session_id) { - continue; - } - let protected = protected_set_for_cache_reclamation(&session_guard, cache, excluded_path, None); - candidates.extend(session_guard.transient.object_cache.values().filter_map(|entry| { - entry.ready_content_length()?; - if entry.access.active_readers() > 0 - || protected.transient_object_ids.contains(entry.key.transient_resource_id()) - { - return None; - } - Some(GlobalTransientObjectCandidate { - session: Arc::clone(session), - proxy_session_id: session_guard.proxy_session_id.clone(), - last_accessed_at_ms: entry.last_accessed_at_ms, - }) - })); - } - candidates.into_iter().min_by_key(|candidate| candidate.last_accessed_at_ms) -} - -fn take_expired_transient_objects( - session: &mut HlsSession, - now_ms: u64, - protected: &HashSet, - limit: usize, -) -> Vec<(TransientObjectCacheKey, u64)> { - session - .transient - .take_expired_object_removals_except(now_ms, protected, limit) - .into_iter() - .map(|removal| (removal.key, removal.content_length)) - .collect() -} - -fn remove_stale_queue_entries(session: &mut HlsSession) -> usize { - let mut removed = 0_usize; - for proxy_seq in session.segment_prefetch_queue.proxy_seqs() { - let stale = session.segments.get(&proxy_seq).is_none_or(|segment| { - segment.origin_fetch_ref.is_none() || !matches!(segment.status, SegmentCacheStatus::Queued { .. }) - }); - if stale && session.segment_prefetch_queue.remove(proxy_seq).is_some() { - removed = removed.saturating_add(1); - } - } - removed -} - -fn duration_expired_head_segment( - session: &HlsSession, - protected: &ProtectedSet, - policy: &GarbageCollectionPolicy, - now_ms: u64, -) -> Option { - let (proxy_seq, segment) = session.segments.iter().next()?; - if protected.segment_proxy_seqs.contains(proxy_seq) { - return None; - } - let last_relevant_at_ms = segment_last_relevant_at_ms(segment)?; - let retention_ms = match segment.status { - SegmentCacheStatus::FailedRetryable { .. } - | SegmentCacheStatus::FailedPermanent { .. } - | SegmentCacheStatus::Expired => policy.failed_segment_retention_ms, - SegmentCacheStatus::Ready { .. } => policy - .cache_duration_ms - .max(segment.duration_ms.saturating_add(session.longest_rendered_playlist_duration_ms)), - SegmentCacheStatus::Discovered - | SegmentCacheStatus::Queued { .. } - | SegmentCacheStatus::Fetching { .. } - | SegmentCacheStatus::CapacityDeferred { .. } => return None, - }; - (now_ms.saturating_sub(last_relevant_at_ms) >= retention_ms).then_some(*proxy_seq) -} - -fn fifo_head_size_candidate(session: &HlsSession, protected: &ProtectedSet) -> Option { - let (proxy_seq, segment) = session.segments.iter().next()?; - if protected.segment_proxy_seqs.contains(proxy_seq) { - return None; - } - let SegmentCacheStatus::Ready { content_length, .. } = segment.status else { - return None; - }; - Some(SegmentDeleteCandidate { - proxy_seq: *proxy_seq, - content_length, - last_relevant_at_ms: segment_last_relevant_at_ms(segment).unwrap_or_default(), - }) -} - -fn segment_last_relevant_at_ms(segment: &super::SegmentEntry) -> Option { - let status_at = match segment.status { - SegmentCacheStatus::Ready { ready_at_ms, .. } => Some(ready_at_ms), - SegmentCacheStatus::FailedRetryable { failed_at_ms, .. } - | SegmentCacheStatus::FailedPermanent { failed_at_ms, .. } => Some(failed_at_ms), - SegmentCacheStatus::Expired => segment.last_rendered_at_ms, - SegmentCacheStatus::Discovered - | SegmentCacheStatus::Queued { .. } - | SegmentCacheStatus::Fetching { .. } - | SegmentCacheStatus::CapacityDeferred { .. } => None, - }; - [status_at, segment.last_rendered_at_ms, Some(segment.access.last_accessed_at_ms()).filter(|value| *value > 0)] - .into_iter() - .flatten() - .max() -} - -fn session_cache_size(session: &HlsSession) -> u64 { - let segment_bytes = session - .segments - .values() - .map(|segment| match segment.status { - SegmentCacheStatus::Ready { content_length, .. } => content_length, - _ => 0, - }) - .sum::(); - let map_bytes = session - .maps - .values() - .map(|map| match map.status { - MapCacheStatus::Ready { content_length, .. } => content_length, - _ => 0, - }) - .sum::(); - segment_bytes.saturating_add(map_bytes).saturating_add(session.transient.ready_object_cache_size()) -} - #[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] struct CapacityReclamationEvidence { protected_working_set_bytes: u64, reclaimable_bytes: u64, } -fn capacity_reclamation_evidence(session: &HlsSession, protected: &ProtectedSet) -> CapacityReclamationEvidence { - let protected_segment_bytes = session.segments.iter().fold(0_u64, |bytes, (proxy_seq, segment)| { - if !protected.segment_proxy_seqs.contains(proxy_seq) { - return bytes; - } - match segment.status { - SegmentCacheStatus::Ready { content_length, .. } => bytes.saturating_add(content_length), - _ => bytes, - } - }); - let protected_map_bytes = session.maps.iter().fold(0_u64, |bytes, (map_id, map)| { - if !protected.map_ids.contains(map_id) { - return bytes; - } - match map.status { - MapCacheStatus::Ready { content_length, .. } => bytes.saturating_add(content_length), - _ => bytes, - } - }); - let (protected_transient_object_bytes, reclaimable_transient_object_bytes) = - session.transient.object_cache.values().fold((0_u64, 0_u64), |(protected_bytes, reclaimable_bytes), entry| { - let Some(content_length) = entry.ready_content_length() else { - return (protected_bytes, reclaimable_bytes); - }; - if protected.transient_object_ids.contains(entry.key.transient_resource_id()) - || entry.access.active_readers() > 0 - { - (protected_bytes.saturating_add(content_length), reclaimable_bytes) - } else { - (protected_bytes, reclaimable_bytes.saturating_add(content_length)) - } - }); - let reclaimable_bytes = fifo_head_size_candidate(session, protected) - .map_or(0, |candidate| candidate.content_length) - .saturating_add(reclaimable_transient_object_bytes); - CapacityReclamationEvidence { - protected_working_set_bytes: protected_segment_bytes - .saturating_add(protected_map_bytes) - .saturating_add(protected_transient_object_bytes), - reclaimable_bytes, - } -} - -fn remove_segment_entry(session: &mut HlsSession, proxy_seq: u64) -> Option { - let segment = session.segments.remove(&proxy_seq)?; - let removed_ready_media = matches!(segment.status, SegmentCacheStatus::Ready { .. }); - session.segment_prefetch_queue.remove(proxy_seq); - session.origin_to_proxy.retain(|_, mapped_seq| *mapped_seq != proxy_seq); - if segment.discontinuity_before { - session.discontinuity_sequence = session.discontinuity_sequence.saturating_add(1); - } - if removed_ready_media { - session.advance_media_readiness_generation(); - } - if session.publishable_origin_head_proxy_seq == Some(proxy_seq) { - session.publishable_origin_head_proxy_seq = session - .segments - .range(proxy_seq.saturating_add(1)..) - .find_map(|(next_proxy_seq, segment)| segment.origin_fetch_ref.as_ref().map(|_| *next_proxy_seq)); - } - Some(segment.cache_key) -} - -fn unprotected_unreferenced_map_ids(session: &HlsSession, protected: &ProtectedSet) -> Vec { - let referenced = session.segments.values().filter_map(|segment| segment.map_ref).collect::>(); - session - .maps - .iter() - .filter_map(|(map_id, map)| { - if referenced.contains(map_id) - || protected.map_ids.contains(map_id) - || map.access.active_readers() > 0 - || matches!(map.status, MapCacheStatus::Queued { .. } | MapCacheStatus::Fetching { .. }) - { - return None; - } - Some(*map_id) - }) - .collect() -} - struct MapEntryDeletion { key: MapCacheKey, content_length: u64, } -fn remove_map_entry(session: &mut HlsSession, map_id: ProxyMapId) -> Option { - let map = session.maps.remove(&map_id)?; - let (content_length, removed_ready_media) = match map.status { - MapCacheStatus::Ready { content_length, .. } => (content_length, true), - _ => (0, false), - }; - if removed_ready_media { - session.advance_media_readiness_generation(); - } - session.origin_map_to_proxy.retain(|_, mapped_map_id| *mapped_map_id != map_id); - Some(MapEntryDeletion { key: map.cache_key, content_length }) -} - #[cfg(test)] -mod tests { - use super::{ - super::{ - media_reserve::{ - HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsManifestCommitIdentity, HlsManifestDeliveryMode, - }, - terminal_tail::{HlsEncryptionSignature, HlsMediaContainer, HlsTerminalTailGeneration}, - }, - build_rewrite_secret_fingerprint, oldest_global_fifo_head_candidate, remove_segment_entry, CacheObjectDeletion, - GarbageCollectionPolicy, GarbageCollectionReport, HlsGarbageCollector, PendingCacheObjectDeletion, - ProtectedSet, SegmentCacheDeletionReason, MAX_PENDING_CACHE_DELETIONS, SWITCH_CACHE_CLEANUP_HEADROOM, - }; - use crate::{ - prepare_terminal_base_evidence, transient_manifest::TransientManifestRewriter, HlsAccessLease, - HlsAccessLeaseId, HlsAccessLeaseStore, HlsOriginResourceFetchError, HlsPlaybackFamilyKey, HlsSegmentCache, - HlsSegmentEncryption, HlsSession, HlsSessionKey, HlsSessionStore, HlsTerminalTailProtection, MapCacheStatus, - OriginMapKey, ProxyMapId, ProxySessionId, SegmentCacheStatus, SegmentFetchPriority, - TransientObjectFetchDecision, TransientPassthroughState, TransientResourceId, TransientResourceKind, - TransientResourceRef, - }; - use std::{collections::HashSet, fmt::Write as _, sync::Arc, task::Poll, time::Duration}; - use tokio::sync::{Barrier, RwLock}; - use tuliprox_parser::hls::origin_manifest::{ - parse_manifest_semantics, parse_origin_media_manifest, OriginManifestParseOutcome, - }; - - const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; - - fn normal_manifest(body: &str) -> tuliprox_parser::hls::origin_manifest::ParsedOriginManifest { - match parse_origin_media_manifest(body, BASE_URL) { - OriginManifestParseOutcome::Normal(manifest) => manifest, - OriginManifestParseOutcome::TransientPassthrough { reason } => { - panic!("expected normal manifest: {reason:?}") - } - } - } - - fn test_policy() -> GarbageCollectionPolicy { - GarbageCollectionPolicy { - cache_duration_ms: 300, - cache_bytes_global: 10_000, - cache_bytes_per_session: 10_000, - session_idle_timeout_ms: 1_000, - temp_file_retention_ms: 30_000, - failed_segment_retention_ms: 10, - } - } - - async fn gc_with_session(temp_dir: &tempfile::TempDir) -> (Arc, crate::HlsSessionHandle) { - let sessions = Arc::new(HlsSessionStore::new()); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let session = sessions.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; - let gc = Arc::new(HlsGarbageCollector::new( - sessions, - Arc::clone(&cache), - test_policy(), - build_rewrite_secret_fingerprint(b"secret"), - )); - cache.install_capacity_reclaimer(&gc); - (gc, session) - } - - fn update_gc_policy(gc: &HlsGarbageCollector, update: impl FnOnce(&mut GarbageCollectionPolicy)) { - let mut policy = gc.policy().as_ref().clone(); - update(&mut policy); - gc.update_config(policy, gc.rewrite_secret_fingerprint()); - } - - fn six_segment_manifest() -> tuliprox_parser::hls::origin_manifest::ParsedOriginManifest { - normal_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n#EXTINF:4.0,\n4.ts\n#EXTINF:4.0,\n5.ts\n#EXTINF:4.0,\n6.ts\n", - ) - } - - fn apply_six_segment_manifest_for_gc(session: &mut super::HlsSession) { - session.proxy_next_seq = Some(1); - session.apply_origin_manifest(&six_segment_manifest()).expect("manifest should map"); - } - - fn install_finalized_transient_manifest( - session: &mut HlsSession, - segment_count: usize, - resource_ttl_ms: u64, - ) -> Vec { - let mut body = - String::from("#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:1\n"); - for index in 0..segment_count { - writeln!(body, "#EXTINF:6.000,\n{index}.ts").expect("synthetic finalized manifest renders"); - } - body.push_str("#EXT-X-ENDLIST\n"); - let rewritten = TransientManifestRewriter::rewrite( - &body, - BASE_URL, - &session.proxy_session_id, - b"secret", - 0, - resource_ttl_ms, - ); - let resources = rewritten.resources.clone(); - session - .transient - .commit_rewritten_manifest_with_semantics( - rewritten.body, - rewritten.resources, - 0, - Some(u64::try_from(segment_count).expect("segment count fits u64").saturating_mul(6_000)), - parse_manifest_semantics(&body), - ) - .expect("synthetic finalized manifest stays within representation limits"); - resources - } - - #[test] - fn gc_report_is_quiet_when_nothing_changed() { - let report = GarbageCollectionReport::default(); - - assert!(!report.did_cleanup_or_invalidate()); - } - - #[tokio::test] - async fn published_resource_history_survives_production_segment_removal_and_file_deletion() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let baseline = normal_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:480\n\ - #EXTINF:4,\n480.ts\n#EXTINF:4,\n481.ts\n#EXTINF:4,\n482.ts\n", - ); - let replay_then_new = normal_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:490\n\ - #EXTINF:4,\n480.ts\n#EXTINF:4,\n490.ts\n#EXTINF:4,\n491.ts\n#EXTINF:4,\n492.ts\n", - ); - let removed_key = { - let mut session = session.write().await; - session.apply_origin_manifest(&baseline).expect("baseline maps"); - for segment in session.segments.values_mut() { - segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1 }; - } - session.render_and_store_manifest(1).expect("baseline publishes"); - session.segments.get(&0).expect("published head").cache_key.clone() - }; - gc.cache.write_bytes_and_commit(&removed_key, b"x").await.expect("cache object writes"); - { - let mut session = session.write().await; - assert_eq!(remove_segment_entry(&mut session, 0), Some(removed_key.clone())); - } - gc.cache.delete(&removed_key).await.expect("production cache file deletion succeeds"); - { - let mut session = session.write().await; - session.apply_origin_manifest(&replay_then_new).expect("history trims removed resource replay"); - assert!(!session.segments.contains_key(&0)); - assert_eq!(session.proxy_next_seq, Some(6)); - assert!(session.segments.get(&3).expect("first new media").discontinuity_before); - } - assert!(gc.cache.metadata(&removed_key).await.expect("metadata lookup succeeds").is_none()); - } - - #[test] - fn terminal_tail_protects_live_base_until_lease_protection_is_released() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - session.install_terminal_tail_protection( - lease_id.clone(), - HlsTerminalTailProtection { - generation: HlsTerminalTailGeneration(1), - base_proxy_seqs: Arc::from([41, 42]), - key_bindings: Arc::from([]), - }, - ); - - let protected = ProtectedSet::from_session(&session); - assert!(protected.segment_proxy_seqs.contains(&41)); - assert!(protected.segment_proxy_seqs.contains(&42)); - - assert!(session.remove_terminal_tail_protection(&lease_id).is_some()); - let released = ProtectedSet::from_session(&session); - assert!(!released.segment_proxy_seqs.contains(&41)); - assert!(!released.segment_proxy_seqs.contains(&42)); - } - - #[test] - fn protected_encrypted_segment_also_protects_its_key_resource() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - let key_resource = TransientResourceRef::new( - TransientResourceKind::Key, - "http://origin.example.com/live/final/key.bin", - b"secret", - 0, - u64::MAX, - Some("bin".to_string()), - ); - let mut manifest = normal_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\n1.ts\n", - ); - for encryption in manifest.segments.iter_mut().filter_map(|segment| segment.encryption.as_mut()) { - encryption.proxy_resource_id = Some(key_resource.id.0.clone()); - encryption.proxy_resource_extension = Some("bin".to_string()); - } - session.transient.upsert_resources([key_resource]); - session.apply_origin_manifest(&manifest).expect("manifest maps"); - let segment = session.segments.get_mut(&0).expect("segment"); - segment.access.reader_started(1); - let resource_id = segment.encryption.as_ref().expect("encryption").resource_id.clone(); - - let protected = ProtectedSet::from_session(&session); - - assert!(protected.segment_proxy_seqs.contains(&0)); - assert!(protected.key_resource_ids.contains(&resource_id)); - } - - #[tokio::test] - async fn gc_reclaims_terminal_base_only_after_lease_protection_release() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let lease_id = HlsAccessLeaseId("terminal-lease".to_string()); - let cache_key = { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - let segment = session.segments.get_mut(&1).expect("segment"); - gc.cache - .write_bytes_and_commit(&segment.cache_key, b"segment-body") - .await - .expect("cache write should succeed"); - segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; - let cache_key = segment.cache_key.clone(); - session.install_terminal_tail_protection( - lease_id.clone(), - HlsTerminalTailProtection { - generation: HlsTerminalTailGeneration(1), - base_proxy_seqs: Arc::from([1_u64]), - key_bindings: Arc::from([]), - }, - ); - cache_key - }; - - let protected_report = gc.run_once(10_000).await.expect("protected gc should run"); - assert_eq!(protected_report.segments_deleted_duration, 0); - assert!(session.read().await.segments.contains_key(&1)); - assert!(gc.cache.metadata(&cache_key).await.expect("metadata reads").is_some()); - - assert!(session.write().await.remove_terminal_tail_protection(&lease_id).is_some()); - let released_report = gc.run_once(10_001).await.expect("released gc should run"); - - assert_eq!(released_report.segments_deleted_duration, 1); - assert!(gc.cache.metadata(&cache_key).await.expect("metadata reads").is_none()); - } - - #[test] - fn gc_report_logs_when_cleanup_or_invalidation_happened() { - let mut report = GarbageCollectionReport { temp_files_deleted: 1, ..GarbageCollectionReport::default() }; - assert!(report.did_cleanup_or_invalidate()); - - report = GarbageCollectionReport { stale_queue_entries_removed: 1, ..GarbageCollectionReport::default() }; - assert!(report.did_cleanup_or_invalidate()); - - report = GarbageCollectionReport { segments_deleted_duration: 1, ..GarbageCollectionReport::default() }; - assert!(report.did_cleanup_or_invalidate()); - - report = GarbageCollectionReport { maps_deleted: 1, ..GarbageCollectionReport::default() }; - assert!(report.did_cleanup_or_invalidate()); - - report = GarbageCollectionReport { sessions_deleted: 1, ..GarbageCollectionReport::default() }; - assert!(report.did_cleanup_or_invalidate()); - - report = GarbageCollectionReport { transient_resources_pruned: 1, ..GarbageCollectionReport::default() }; - assert!(report.did_cleanup_or_invalidate()); - - report = GarbageCollectionReport { secret_cache_invalidated: true, ..GarbageCollectionReport::default() }; - assert!(report.did_cleanup_or_invalidate()); - - report = - GarbageCollectionReport { secret_cache_invalidation_deferred: true, ..GarbageCollectionReport::default() }; - assert!(report.did_cleanup_or_invalidate()); - } - - async fn populate_ready_segments(gc: &HlsGarbageCollector, session: &crate::HlsSessionHandle, ready_at_ms: u64) { - { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - for segment in session.segments.values_mut() { - gc.cache - .write_bytes_and_commit(&segment.cache_key, b"segment-body") - .await - .expect("cache write should succeed"); - segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; - } - session.render_and_store_manifest(ready_at_ms).expect("ready manifest should render"); - } - } - - async fn cache_selected_ready_segments( - gc: &HlsGarbageCollector, - session: &crate::HlsSessionHandle, - proxy_seqs: &[u64], - body: &[u8], - ) { - let keys = { - let session = session.read().await; - proxy_seqs - .iter() - .map(|proxy_seq| (*proxy_seq, session.segments.get(proxy_seq).expect("segment").cache_key.clone())) - .collect::>() - }; - for (_, key) in &keys { - gc.cache.write_bytes_and_commit(key, body).await.expect("cache fixture writes"); - } - let mut session = session.write().await; - for (proxy_seq, _) in keys { - session.segments.get_mut(&proxy_seq).expect("segment").status = SegmentCacheStatus::Ready { - content_length: u64::try_from(body.len()).unwrap_or(u64::MAX), - ready_at_ms: proxy_seq, - }; - } - } - - async fn commit_sparse_segment(gc: &HlsGarbageCollector, key: &crate::SegmentCacheKey, size: u64) { - let final_path = gc.cache.object_path(key); - let parent = final_path.parent().expect("cache object parent"); - tokio::fs::create_dir_all(parent).await.expect("cache object parent creates"); - let temp_path = final_path.with_extension(format!("ts.tmp.fixture-{}", key.proxy_seq())); - let file = tokio::fs::File::create(&temp_path).await.expect("sparse temp creates"); - file.set_len(size).await.expect("sparse temp length"); - drop(file); - let staged = gc.cache.adopt_staged_file(temp_path, size).expect("sparse staged object adopts"); - gc.cache.commit_staged(key, staged).await.expect("sparse object commits"); - } - - #[tokio::test] - async fn gc_keeps_last_rendered_manifest_segments() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - populate_ready_segments(&gc, &session, 0).await; - - let report = gc.run_once(10_000).await.expect("gc should run"); - - assert_eq!(report.segments_deleted_duration, 0); - assert_eq!(session.read().await.segments.len(), 6); - } - - #[tokio::test] - async fn gc_keeps_active_readers() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - let segment = session.segments.get_mut(&1).expect("segment"); - gc.cache - .write_bytes_and_commit(&segment.cache_key, b"segment-body") - .await - .expect("cache write should succeed"); - segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; - segment.access.reader_started(1); - } - - let report = gc.run_once(10_000).await.expect("gc should run"); - - assert_eq!(report.segments_deleted_duration, 0); - assert!(session.read().await.segments.contains_key(&1)); - } - - #[tokio::test] - async fn gc_keeps_fetching_segments() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - session.segments.get_mut(&1).expect("segment").status = - SegmentCacheStatus::Fetching { priority: SegmentFetchPriority::Prefetch, started_at_ms: 1 }; - } - - let report = gc.run_once(10_000).await.expect("gc should run"); - - assert_eq!(report.segments_deleted_duration, 0); - assert!(session.read().await.segments.contains_key(&1)); - } - - #[tokio::test] - async fn duration_gc_deletes_old_unprotected_segments() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - let segment = session.segments.get_mut(&1).expect("segment"); - gc.cache - .write_bytes_and_commit(&segment.cache_key, b"segment-body") - .await - .expect("cache write should succeed"); - segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; - } - - let report = gc.run_once(10_000).await.expect("gc should run"); - - assert_eq!(report.segments_deleted_duration, 1); - assert!(!session.read().await.segments.contains_key(&1)); - } - - #[tokio::test] - async fn cache_delete_failure_does_not_block_later_deletes_and_retries_in_active_session() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let (failed_key, successful_key) = { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - session.last_client_access_at_ms = 10_000; - for proxy_seq in [1, 2] { - session.segments.get_mut(&proxy_seq).expect("segment").status = - SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; - } - ( - session.segments.get(&1).expect("failed segment").cache_key.clone(), - session.segments.get(&2).expect("successful segment").cache_key.clone(), - ) - }; - gc.cache.write_bytes_and_commit(&failed_key, b"segment-body").await.expect("failed fixture writes"); - gc.cache.write_bytes_and_commit(&successful_key, b"segment-body").await.expect("successful fixture writes"); - let failed_path = gc.cache.object_path(&failed_key); - tokio::fs::remove_file(&failed_path).await.expect("replace failed fixture"); - tokio::fs::create_dir(&failed_path).await.expect("directory makes remove_file fail"); - - let first_report = gc.run_once(10_000).await.expect("first gc run"); - - assert_eq!(first_report.cache_object_deletions_planned, 2); - assert_eq!(first_report.cache_object_deletions_succeeded, 1); - assert_eq!(first_report.cache_object_deletions_deferred, 1); - assert_eq!(first_report.segments_deleted_duration, 1); - assert!(gc.cache.metadata(&successful_key).await.expect("metadata reads").is_none()); - assert_eq!(gc.pending_cache_deletion_count(), 1); - assert!(!session.read().await.segments.contains_key(&1)); - assert!(!session.read().await.segments.contains_key(&2)); - - tokio::fs::remove_dir(&failed_path).await.expect("remove failing fixture"); - tokio::fs::write(&failed_path, b"replacement-body").await.expect("replacement fixture writes"); - - let retry_report = gc.run_once(10_001).await.expect("retry gc run"); - - assert_eq!(retry_report.cache_object_deletions_planned, 0); - assert_eq!(retry_report.cache_object_deletions_succeeded, 1); - assert_eq!(retry_report.cache_object_deletions_deferred, 0); - assert_eq!(retry_report.segments_deleted_duration, 1); - assert_eq!(gc.pending_cache_deletion_count(), 0); - assert!(gc.cache.metadata(&failed_key).await.expect("metadata reads").is_none()); - } - - #[tokio::test] - async fn full_retry_queue_preserves_session_metadata_until_a_slot_is_reserved() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - session.segments.get_mut(&1).expect("segment").status = - SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; - } - { - let mut queue = gc.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - for proxy_seq in 0..MAX_PENDING_CACHE_DELETIONS { - queue.pending.push_back(PendingCacheObjectDeletion { - deletion: CacheObjectDeletion::Segment { - key: crate::SegmentCacheKey::new( - crate::ProxySessionId("queued".to_string()), - u64::try_from(proxy_seq).unwrap_or_default(), - "ts", - ), - reason: SegmentCacheDeletionReason::Duration, - }, - attempts: 0, - }); - } - } - let mut report = GarbageCollectionReport::default(); - let policy = gc.policy(); - let mut batch = gc.reserve_cache_deletion_batch(); - assert_eq!(batch.remaining_capacity(), 0); - - { - let mut session = session.write().await; - HlsGarbageCollector::collect_session_deletions( - &mut session, - &gc.cache, - 10_000, - &policy, - &mut report, - &mut batch, - ); - assert!(session.segments.contains_key(&1)); - } - batch.persist(&mut report); - - assert_eq!(report.cache_object_deletions_planned, 0); - assert_eq!(gc.pending_cache_deletion_count(), MAX_PENDING_CACHE_DELETIONS); - } - - #[tokio::test] - async fn dropping_unpersisted_deletion_batch_keeps_reserved_deletion_retryable() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, _) = gc_with_session(&temp_dir).await; - let key = crate::SegmentCacheKey::new(crate::ProxySessionId("cancelled".to_string()), 1, "ts"); - { - let mut batch = gc.reserve_cache_deletion_batch(); - batch.push(CacheObjectDeletion::Segment { key: key.clone(), reason: SegmentCacheDeletionReason::Duration }); - } - - let queue = gc.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - assert_eq!(queue.reserved_slots, 0); - assert_eq!(queue.pending.len(), 1); - assert_eq!( - queue.pending.front().map(|pending| &pending.deletion), - Some(&CacheObjectDeletion::Segment { key, reason: SegmentCacheDeletionReason::Duration }) - ); - } - - #[tokio::test] - async fn ordinary_gc_batch_preserves_bounded_switch_rollback_headroom() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let batch = gc.reserve_cache_deletion_batch(); - assert_eq!(batch.remaining_capacity(), MAX_PENDING_CACHE_DELETIONS - SWITCH_CACHE_CLEANUP_HEADROOM); - - let segment = gc.reserve_switch_segment_cleanup(crate::SegmentCacheKey::new(proxy_session_id.clone(), 1, "ts")); - let map = gc.reserve_switch_map_cleanup(crate::MapCacheKey::new(proxy_session_id, ProxyMapId(1), "mp4")); - - assert!(segment.is_some()); - assert!(map.is_some()); - let queue = gc.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - assert!(queue.pending.len().saturating_add(queue.reserved_slots) <= MAX_PENDING_CACHE_DELETIONS); - } - - #[tokio::test] - async fn cache_root_handoff_discards_old_logical_deletion_tickets_before_new_root_use() { - let old_root = tempfile::tempdir().expect("old cache root"); - let new_root = tempfile::tempdir().expect("new cache root"); - let (gc, session) = gc_with_session(&old_root).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let key = crate::SegmentCacheKey::new(proxy_session_id, 1, "ts"); - { - let mut queue = gc.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - queue.pending.push_back(PendingCacheObjectDeletion { - deletion: CacheObjectDeletion::Segment { - key: key.clone(), - reason: SegmentCacheDeletionReason::Duration, - }, - attempts: 1, - }); - } - - assert!(gc.update_cache_path(new_root.path()).await); - gc.cache.write_bytes_and_commit(&key, b"new-root-sentinel").await.expect("new-root sentinel writes"); - let report = gc.run_once(0).await.expect("new-root gc runs"); - - assert_eq!(gc.pending_cache_deletion_count(), 0); - assert_eq!(report.cache_object_deletions_succeeded, 0); - let metadata = gc.cache.metadata(&key).await.expect("new-root metadata reads").expect("sentinel remains"); - assert_eq!(tokio::fs::read(metadata.path).await.expect("sentinel reads"), b"new-root-sentinel"); - } - - #[tokio::test] - async fn cache_root_handoff_invalidates_armed_switch_cleanup_reservations() { - let old_root = tempfile::tempdir().expect("old cache root"); - let new_root = tempfile::tempdir().expect("new cache root"); - let (gc, session) = gc_with_session(&old_root).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let key = crate::SegmentCacheKey::new(proxy_session_id, 1, "ts"); - let cleanup = gc.reserve_switch_segment_cleanup(key.clone()).expect("switch cleanup reservation"); - assert_eq!(gc.cache_deletion_queue_usage(), (0, 1)); - - assert!(gc.update_cache_path(new_root.path()).await); - assert_eq!(gc.cache_deletion_queue_usage(), (0, 0)); - drop(cleanup); - assert_eq!(gc.cache_deletion_queue_usage(), (0, 0)); - - gc.cache.write_bytes_and_commit(&key, b"new-root-sentinel").await.expect("new-root sentinel writes"); - let report = gc.run_once(0).await.expect("new-root gc runs"); - - assert_eq!(report.cache_object_deletions_succeeded, 0); - let metadata = gc.cache.metadata(&key).await.expect("new-root metadata reads").expect("sentinel remains"); - assert_eq!(tokio::fs::read(metadata.path).await.expect("sentinel reads"), b"new-root-sentinel"); - } - - #[tokio::test] - async fn failed_switch_rollback_blocks_same_key_until_gc_retry_succeeds() { - let cache_root = tempfile::tempdir().expect("cache root"); - let (gc, session) = gc_with_session(&cache_root).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let key = crate::SegmentCacheKey::new(proxy_session_id, 1, "ts"); - gc.cache.write_bytes_and_commit(&key, b"uncommitted-switch").await.expect("switch fixture writes"); - let object_path = gc.cache.object_path(&key); - tokio::fs::remove_file(&object_path).await.expect("switch fixture file removes"); - tokio::fs::create_dir(&object_path).await.expect("directory forces remove-file failure"); - let cleanup = gc.reserve_switch_segment_cleanup(key.clone()).expect("switch cleanup reservation"); - drop(cleanup); - - assert!(gc.has_pending_switch_cleanup(&key, None)); - let deferred = gc.run_once(0).await.expect("deferred rollback GC runs"); - assert_eq!(deferred.cache_object_deletions_succeeded, 0); - assert_eq!(deferred.cache_object_deletions_deferred, 1); - assert_eq!(gc.cache_deletion_queue_usage(), (1, 0)); - assert!(gc.has_pending_switch_cleanup(&key, None)); - - tokio::fs::remove_dir(&object_path).await.expect("failing directory removes"); - tokio::fs::write(&object_path, b"uncommitted-switch").await.expect("retry fixture writes"); - let retried = gc.run_once(1).await.expect("rollback retry GC runs"); - - assert_eq!(retried.cache_object_deletions_succeeded, 1); - assert_eq!(gc.cache_deletion_queue_usage(), (0, 0)); - assert!(!gc.has_pending_switch_cleanup(&key, None)); - assert!(gc.cache.metadata(&key).await.expect("rolled-back metadata reads").is_none()); - } - - #[tokio::test] - async fn duration_gc_stops_at_protected_fifo_head() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - for proxy_seq in [1, 2] { - let segment = session.segments.get_mut(&proxy_seq).expect("segment"); - gc.cache - .write_bytes_and_commit(&segment.cache_key, b"segment-body") - .await - .expect("cache write should succeed"); - segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; - } - session.segments.get_mut(&1).expect("head segment").access.reader_started(1); - } - - let report = gc.run_once(10_000).await.expect("gc should run"); - - assert_eq!(report.segments_deleted_duration, 0); - let session = session.read().await; - assert!(session.segments.contains_key(&1)); - assert!(session.segments.contains_key(&2)); - } - - #[tokio::test] - async fn duration_gc_stops_at_not_expired_fifo_head() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - for (proxy_seq, ready_at_ms) in [(1, 9_950), (2, 0)] { - let segment = session.segments.get_mut(&proxy_seq).expect("segment"); - gc.cache - .write_bytes_and_commit(&segment.cache_key, b"segment-body") - .await - .expect("cache write should succeed"); - segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; - } - } - - let report = gc.run_once(10_000).await.expect("gc should run"); - - assert_eq!(report.segments_deleted_duration, 0); - let session = session.read().await; - assert!(session.segments.contains_key(&1)); - assert!(session.segments.contains_key(&2)); - } - - #[tokio::test] - async fn session_size_gc_deletes_oldest_unprotected_segment() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - update_gc_policy(&gc, |policy| policy.cache_bytes_per_session = 20); - { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - for proxy_seq in [1, 2, 3] { - let segment = session.segments.get_mut(&proxy_seq).expect("segment"); - gc.cache - .write_bytes_and_commit(&segment.cache_key, b"segment-body") - .await - .expect("cache write should succeed"); - segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: proxy_seq }; - } - } - - let report = gc.run_once(100).await.expect("gc should run"); - - assert_eq!(report.segments_deleted_size_session, 2); - let session = session.read().await; - assert!(!session.segments.contains_key(&1)); - assert!(!session.segments.contains_key(&2)); - assert!(session.segments.contains_key(&3)); - } - - #[tokio::test] - async fn projected_session_pressure_reclaims_before_the_limit_is_crossed() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - } - cache_selected_ready_segments(&gc, &session, &[1, 2], b"0123456789").await; - gc.cache.update_cache_limits(100, 25); - let target = session.read().await.segments.get(&3).expect("target").cache_key.clone(); - - let committed = gc.cache.write_bytes_and_commit(&target, b"abcdefghij").await.expect("projected reclaim"); - - assert_eq!(committed.size, 10); - let session_guard = session.read().await; - assert!(!session_guard.segments.contains_key(&1), "oldest unprotected FIFO head is reclaimed"); - assert!(session_guard.segments.contains_key(&2)); - let proxy_session_id = session_guard.proxy_session_id.clone(); - drop(session_guard); - assert!(gc.cache.metadata(&target).await.expect("target metadata").is_some()); - let usage = gc.cache.capacity_usage(&proxy_session_id).await.expect("capacity usage"); - assert_eq!(usage.session_bytes, 20); - assert!(usage.session_bytes <= 25); - assert!(!gc.cache.has_active_temp_files()); - } - - fn activated_startup_lease(proxy_session_id: &ProxySessionId) -> HlsAccessLease { - let snapshot = HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(10), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 1, - delivered_at_ms: 10, - first_proxy_seq: 1, - last_proxy_seq: 3, - visible_segments: Arc::from( - (1_u64..=3) - .map(|proxy_seq| HlsLeaseManifestSegment { - proxy_seq, - duration_ms: 4_000, - uri: format!("/hls/shared/live/session/lease/{proxy_seq:06}.ts").into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }) - .collect::>(), - ), - discontinuity_sequence: 0, - target_duration_ms: 4_000, - playlist_duration_ms: 12_000, - last_visible_media_end_ms: 12_000, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - }; - let mut lease = HlsAccessLease::pending( - HlsAccessLeaseId("startup-lease".to_string()), - HlsPlaybackFamilyKey::new("user", "player"), - proxy_session_id.clone(), - "user".to_string(), - "user-session".to_string(), - 1, - "12345".to_string(), - 1, - 10, - 60_000, - ); - lease.state = super::super::HlsAccessLeaseState::Activated; - lease.active_until_ms = Some(60_000); - lease.last_manifest_snapshot = Some(snapshot); - lease - } - - #[tokio::test] - async fn protected_startup_tail_defers_until_playback_releases_fifo_head() { - const SESSION_LIMIT: u64 = 125 * 1_024 * 1_024; - const RESIDENT_SIZES: [u64; 5] = [21_000_000, 22_000_000, 21_500_000, 22_500_000, 22_221_044]; - const STAGED_SIZE: u64 = 22_551_164; - - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let (proxy_session_id, keys) = { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - let keys = (1_u64..=6) - .map(|proxy_seq| session.segments.get(&proxy_seq).expect("segment").cache_key.clone()) - .collect::>(); - (session.proxy_session_id.clone(), keys) - }; - for (key, size) in keys.iter().take(5).zip(RESIDENT_SIZES) { - commit_sparse_segment(&gc, key, size).await; - } - { - let mut session = session.write().await; - for (proxy_seq, content_length) in (1_u64..=5).zip(RESIDENT_SIZES) { - session.segments.get_mut(&proxy_seq).expect("resident segment").status = - SegmentCacheStatus::Ready { content_length, ready_at_ms: proxy_seq }; - } - session.segments.get_mut(&6).expect("tail segment").status = - SegmentCacheStatus::Queued { priority: SegmentFetchPriority::Prefetch, queued_at_ms: 1 }; - session.render_and_store_manifest(10).expect("six-segment canonical render"); - } - - let mut lease = activated_startup_lease(&proxy_session_id); - let access_leases = Arc::new(RwLock::new(HlsAccessLeaseStore::default())); - assert!(access_leases.write().await.prepare_access_lease(lease.clone())); - gc.install_access_leases(&access_leases); - gc.cache.update_cache_limits(512 * 1_024 * 1_024, SESSION_LIMIT); - - for key in keys.iter().take(3) { - assert!(gc.cache.metadata(key).await.expect("visible metadata").is_some()); - } - let deferred = gc - .cache - .ensure_projected_write_capacity(&keys[5], STAGED_SIZE) - .await - .expect_err("fully protected startup window defers the tail"); - let capacity = super::super::cache::hls_cache_capacity_from_io(&deferred).expect("typed capacity deferral"); - assert_eq!(capacity.configured_session_bytes(), SESSION_LIMIT); - assert_eq!(capacity.current_session_bytes(), RESIDENT_SIZES.into_iter().sum::()); - assert_eq!(capacity.staged_bytes(), STAGED_SIZE); - assert_eq!(capacity.required_session_bytes(), 700_208); - assert_eq!(capacity.protected_working_set_bytes(), RESIDENT_SIZES.into_iter().sum::()); - assert_eq!(capacity.reclaimable_bytes(), 0); - assert!(gc.cache.metadata(&keys[5]).await.expect("tail metadata").is_none()); - - { - let mut session = session.write().await; - session.segments.get_mut(&6).expect("tail segment").status = - SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Prefetch, deferred_at_ms: 11 }; - let safe_render = session.render_and_store_manifest(11).expect("deferred tail truncates safely"); - assert_eq!(safe_render.last_proxy_seq, 5); - assert!(!safe_render.body.contains("000006.ts")); - } - - lease.playback_cursor.highest_contiguous_completed_proxy_seq = Some(1); - assert!(access_leases.write().await.prepare_access_lease(lease)); - gc.cache.notify_capacity_protection_changed(); - commit_sparse_segment(&gc, &keys[5], STAGED_SIZE).await; - { - let mut session = session.write().await; - session.segments.get_mut(&6).expect("tail segment").status = - SegmentCacheStatus::Ready { content_length: STAGED_SIZE, ready_at_ms: 12 }; - let resumed = session.render_and_store_manifest(12).expect("window resumes after reclamation"); - assert_eq!(resumed.first_proxy_seq, 2); - assert_eq!(resumed.last_proxy_seq, 6); - } - assert!(gc.cache.metadata(&keys[0]).await.expect("released head metadata").is_none()); - assert!(gc.cache.metadata(&keys[5]).await.expect("resumed tail metadata").is_some()); - let usage = gc.cache.capacity_usage(&proxy_session_id).await.expect("bounded usage"); - assert!(usage.session_bytes <= SESSION_LIMIT); - } - - #[tokio::test] - async fn concurrent_projected_pressure_is_serialized_without_over_deletion() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - } - cache_selected_ready_segments(&gc, &session, &[1, 2], b"0123456789").await; - gc.cache.update_cache_limits(100, 25); - let (third, fourth, proxy_session_id) = { - let session = session.read().await; - ( - session.segments.get(&3).expect("third").cache_key.clone(), - session.segments.get(&4).expect("fourth").cache_key.clone(), - session.proxy_session_id.clone(), - ) - }; - let first_staged = gc - .cache - .stage_temp_with_deadline(&third, &b"abcdefghij"[..], tokio::time::Instant::now() + Duration::from_mins(1)) - .await - .expect("first object stages"); - let second_staged = gc - .cache - .stage_temp_with_deadline(&fourth, &b"klmnopqrst"[..], tokio::time::Instant::now() + Duration::from_mins(1)) - .await - .expect("second object stages"); - let commit_barrier = Arc::new(Barrier::new(2)); - let first_cache = Arc::clone(&gc.cache); - let first_barrier = Arc::clone(&commit_barrier); - let first = tokio::spawn(async move { - first_barrier.wait().await; - first_cache.commit_staged(&third, first_staged).await - }); - let second_cache = Arc::clone(&gc.cache); - let second = tokio::spawn(async move { - commit_barrier.wait().await; - second_cache.commit_staged(&fourth, second_staged).await - }); - - let (first, second) = tokio::join!(first, second); - - let first = first.expect("first task"); - let second = second.expect("second task"); - - assert!(first.is_ok(), "first cache write failed: {first:?}"); - assert!(second.is_ok(), "second cache write failed: {second:?}"); - let session = session.read().await; - assert!(!session.segments.contains_key(&1)); - assert!(!session.segments.contains_key(&2)); - assert!(session.segments.contains_key(&3)); - assert!(session.segments.contains_key(&4)); - drop(session); - let usage = gc.cache.capacity_usage(&proxy_session_id).await.expect("capacity usage"); - assert_eq!(usage.session_bytes, 20); - assert!(usage.global_bytes <= 25); - } - - #[tokio::test] - async fn projected_pressure_never_skips_a_protected_fifo_head() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - } - cache_selected_ready_segments(&gc, &session, &[1, 2], b"0123456789").await; - { - let mut session = session.write().await; - session.segments.get_mut(&1).expect("head").access.reader_started(1); - } - gc.cache.update_cache_limits(100, 25); - let target = session.read().await.segments.get(&3).expect("target").cache_key.clone(); - - let error = gc.cache.write_bytes_and_commit(&target, b"abcdefghij").await.expect_err("head is protected"); - - assert!(matches!( - HlsOriginResourceFetchError::cache_commit(&error), - HlsOriginResourceFetchError::LocalCacheCapacity { .. } - )); - let session = session.read().await; - assert!(session.segments.contains_key(&1)); - assert!(session.segments.contains_key(&2)); - drop(session); - assert!(gc.cache.metadata(&target).await.expect("target metadata").is_none()); - assert!(!gc.cache.has_active_temp_files()); - } - - #[tokio::test] - async fn projected_global_pressure_uses_oldest_session_fifo_head() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, first_session) = gc_with_session(&temp_dir).await; - let second_session = gc.sessions.get_or_create_session(HlsSessionKey::new(1, "67890"), b"secret", 0).await; - for session in [&first_session, &second_session] { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - } - cache_selected_ready_segments(&gc, &first_session, &[1], b"12345678").await; - cache_selected_ready_segments(&gc, &second_session, &[1], b"abcdefgh").await; - { - let mut first = first_session.write().await; - let SegmentCacheStatus::Ready { ready_at_ms, .. } = - &mut first.segments.get_mut(&1).expect("first head").status - else { - panic!("first head must be ready"); - }; - *ready_at_ms = 1; - } - { - let mut second = second_session.write().await; - let SegmentCacheStatus::Ready { ready_at_ms, .. } = - &mut second.segments.get_mut(&1).expect("second head").status - else { - panic!("second head must be ready"); - }; - *ready_at_ms = 2; - } - gc.cache.update_cache_limits(20, 100); - let (target, proxy_session_id) = { - let session = second_session.read().await; - (session.segments.get(&2).expect("target").cache_key.clone(), session.proxy_session_id.clone()) - }; - - gc.cache.write_bytes_and_commit(&target, b"ijklmnop").await.expect("global projected reclaim"); - - assert!(!first_session.read().await.segments.contains_key(&1)); - assert!(second_session.read().await.segments.contains_key(&1)); - let usage = gc.cache.capacity_usage(&proxy_session_id).await.expect("capacity usage"); - assert_eq!(usage.global_bytes, 16); - assert!(usage.global_bytes <= 20); - } - - #[tokio::test] - async fn high_bitrate_sequence_advances_beyond_the_512_mib_aggregate_wall() { - use std::fmt::Write as _; - - const SESSION_LIMIT: u64 = 512 * 1024 * 1024; - const OLD_SEGMENT_BYTES: u64 = 21_825_922; - const NEW_SEGMENT_BYTES: u64 = 22_000_000; - const READY_SEGMENTS: u64 = 24; - - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let mut manifest = String::from("#EXTM3U\n#EXT-X-TARGETDURATION:10\n#EXT-X-MEDIA-SEQUENCE:1\n"); - for origin_seq in 1..=READY_SEGMENTS.saturating_add(1) { - writeln!(manifest, "#EXTINF:10.0,\n{origin_seq}.ts").expect("writing to a String cannot fail"); - } - { - let mut session = session.write().await; - session.proxy_next_seq = Some(1); - session.apply_origin_manifest(&normal_manifest(&manifest)).expect("high-bitrate manifest maps"); - } - let keys = { - let session = session.read().await; - (1..=READY_SEGMENTS.saturating_add(1)) - .map(|proxy_seq| session.segments.get(&proxy_seq).expect("segment").cache_key.clone()) - .collect::>() - }; - for key in keys.iter().take(usize::try_from(READY_SEGMENTS).unwrap_or(usize::MAX)) { - commit_sparse_segment(&gc, key, OLD_SEGMENT_BYTES).await; - } - { - let mut session = session.write().await; - for proxy_seq in 1..=READY_SEGMENTS { - session.segments.get_mut(&proxy_seq).expect("ready segment").status = - SegmentCacheStatus::Ready { content_length: OLD_SEGMENT_BYTES, ready_at_ms: proxy_seq }; - } - } - gc.cache.update_cache_limits(2 * SESSION_LIMIT, SESSION_LIMIT); - let target = keys.last().expect("target key"); - - commit_sparse_segment(&gc, target, NEW_SEGMENT_BYTES).await; - { - let mut session = session.write().await; - session.segments.get_mut(&READY_SEGMENTS.saturating_add(1)).expect("newest segment").status = - SegmentCacheStatus::Ready { content_length: NEW_SEGMENT_BYTES, ready_at_ms: READY_SEGMENTS + 1 }; - } - - let aggregate_sequence_bytes = - READY_SEGMENTS.saturating_mul(OLD_SEGMENT_BYTES).saturating_add(NEW_SEGMENT_BYTES); - assert!(aggregate_sequence_bytes > SESSION_LIMIT); - let session_guard = session.read().await; - assert!(!session_guard.segments.contains_key(&1)); - assert!(session_guard.segments.contains_key(&READY_SEGMENTS.saturating_add(1))); - let proxy_session_id = session_guard.proxy_session_id.clone(); - drop(session_guard); - let usage = gc.cache.capacity_usage(&proxy_session_id).await.expect("capacity usage"); - assert_eq!(usage.session_bytes, READY_SEGMENTS.saturating_sub(1) * OLD_SEGMENT_BYTES + NEW_SEGMENT_BYTES); - assert!(usage.session_bytes <= SESSION_LIMIT); - } - - #[tokio::test] - async fn session_size_gc_stops_at_protected_fifo_head() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - update_gc_policy(&gc, |policy| policy.cache_bytes_per_session = 20); - { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - for proxy_seq in [1, 2, 3] { - let segment = session.segments.get_mut(&proxy_seq).expect("segment"); - gc.cache - .write_bytes_and_commit(&segment.cache_key, b"segment-body") - .await - .expect("cache write should succeed"); - segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: proxy_seq }; - } - session.segments.get_mut(&1).expect("head segment").access.reader_started(1); - } - - let report = gc.run_once(100).await.expect("gc should run"); - - assert_eq!(report.segments_deleted_size_session, 0); - let session = session.read().await; - assert!(session.segments.contains_key(&1)); - assert!(session.segments.contains_key(&2)); - assert!(session.segments.contains_key(&3)); - } - - #[tokio::test] - async fn protected_map_remains_and_unreferenced_map_is_deleted() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let manifest = normal_manifest( - "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n1.m4s\n#EXTINF:4.0,\n2.m4s\n#EXTINF:4.0,\n3.m4s\n", - ); - { - let mut session = session.write().await; - session.apply_origin_manifest(&manifest).expect("manifest should map"); - for segment in session.segments.values_mut() { - segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; - } - let protected_map_id = ProxyMapId(0); - session.maps.get_mut(&protected_map_id).expect("map").status = - MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; - let unreferenced_key = OriginMapKey { - origin_epoch: 0, - resolved_origin_uri: "http://origin.example.com/live/unused.mp4".to_string(), - byte_range: None, - }; - let unreferenced_map = crate::MapEntry::new( - &session.proxy_session_id, - ProxyMapId(1), - unreferenced_key.clone(), - "mp4".to_string(), - ); - session.maps.insert(ProxyMapId(1), unreferenced_map); - session.origin_map_to_proxy.insert(unreferenced_key, ProxyMapId(1)); - session.maps.get_mut(&ProxyMapId(1)).expect("map").status = - MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; - session.render_and_store_manifest(1).expect("manifest should render"); - } - - let report = gc.run_once(10_000).await.expect("gc should run"); - - assert_eq!(report.maps_deleted, 1); - let session = session.read().await; - assert!(session.maps.contains_key(&ProxyMapId(0))); - assert!(!session.maps.contains_key(&ProxyMapId(1))); - } - - #[tokio::test] - async fn map_referenced_by_remaining_segment_survives_fifo_gc() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let manifest = - normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n0.m4s\n#EXTINF:4.0,\n1.m4s\n"); - { - let mut session = session.write().await; - session.apply_origin_manifest(&manifest).expect("manifest should map"); - session.maps.get_mut(&ProxyMapId(0)).expect("map").status = - MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; - for (proxy_seq, ready_at_ms) in [(0, 0), (1, 9_950)] { - let segment = session.segments.get_mut(&proxy_seq).expect("segment"); - segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; - } - } - - let report = gc.run_once(10_000).await.expect("gc should run"); - - assert_eq!(report.segments_deleted_duration, 1); - assert_eq!(report.maps_deleted, 0); - let session = session.read().await; - assert!(!session.segments.contains_key(&0)); - assert!(session.segments.contains_key(&1)); - assert!(session.maps.contains_key(&ProxyMapId(0))); - } - - #[tokio::test] - async fn global_size_gc_subtracts_map_bytes_after_segment_removal() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - update_gc_policy(&gc, |policy| policy.cache_bytes_global = 25); - let manifest = normal_manifest( - "#EXTM3U\n#EXT-X-MAP:URI=\"init0.mp4\"\n#EXTINF:4.0,\n0.m4s\n#EXT-X-MAP:URI=\"init1.mp4\"\n#EXTINF:4.0,\n1.m4s\n", - ); - { - let mut session = session.write().await; - session.apply_origin_manifest(&manifest).expect("manifest should map"); - for (proxy_seq, ready_at_ms) in [(0, 0), (1, 1)] { - let segment = session.segments.get_mut(&proxy_seq).expect("segment"); - segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; - } - session.maps.get_mut(&ProxyMapId(0)).expect("first map").status = - MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; - session.maps.get_mut(&ProxyMapId(1)).expect("second map").status = - MapCacheStatus::Ready { content_length: 10, ready_at_ms: 1 }; - } - - let report = gc.run_once(100).await.expect("gc should run"); - - assert_eq!(report.segments_deleted_size_global, 1); - assert_eq!(report.maps_deleted, 1); - let session = session.read().await; - assert!(!session.segments.contains_key(&0)); - assert!(session.segments.contains_key(&1)); - assert!(!session.maps.contains_key(&ProxyMapId(0))); - assert!(session.maps.contains_key(&ProxyMapId(1))); - } - - #[tokio::test] - async fn transient_resource_mappings_expire_unless_active() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let resource_id = { - let mut session = session.write().await; - let resource = crate::TransientResourceRef::new( - crate::TransientResourceKind::Segment, - "http://origin.example.com/live/seg.ts", - b"secret", - 0, - 10, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - session.transient.upsert_resources([resource]); - resource_id - }; - - let report = gc.run_once(20).await.expect("gc should run"); - - assert_eq!(report.transient_resources_pruned, 1); - assert!(!session.read().await.transient.resources.contains_key(&resource_id)); - } - - #[tokio::test] - async fn finalized_transient_resource_mappings_in_current_manifest_are_protected() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let resource_id = { - let mut session = session.write().await; - let resource = crate::TransientResourceRef::new( - crate::TransientResourceKind::Segment, - "http://origin.example.com/live/seg.ts", - b"secret", - 0, - 10, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - session.transient.upsert_resources([resource]); - session.transient.replace_manifest_with_semantics( - format!("#EXTM3U\n#EXTINF:1,\n/hls/shared/live/session/lease/r/{}.ts\n#EXT-X-ENDLIST\n", resource_id.0), - 0, - Some(1_000), - ); - resource_id - }; - - let report = gc.run_once(20).await.expect("gc should run"); - - assert_eq!(report.transient_resources_pruned, 0); - assert!(session.read().await.transient.resources.contains_key(&resource_id)); - } - - #[test] - fn finalized_manifest_mapping_count_does_not_become_object_pin_count() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "large-finalized"), b"secret", 0); - let resources = install_finalized_transient_manifest(&mut session, 1_643, 10); - - let protected = ProtectedSet::from_session(&session); - - assert_eq!(resources.len(), 1_643); - assert_eq!(session.transient.current_manifest_resource_ids().len(), 1_643); - assert!(protected.transient_object_ids.is_empty()); - assert!(resources - .iter() - .all(|resource| session.transient.resolve_current_resource(&resource.id, 20).is_some())); - } - - #[tokio::test] - async fn large_finalized_manifest_keeps_mappings_without_pinning_all_objects() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let (resource, lookup_key, fetch_token) = { - let mut session = session.write().await; - let resources = install_finalized_transient_manifest(&mut session, 1_643, 10); - let resource = resources.into_iter().next().expect("large manifest resource"); - let lookup_key = - TransientPassthroughState::transient_object_key(&session.proxy_session_id, &resource.id, "ts"); - let proxy_session_id = session.proxy_session_id.clone(); - let fetch_token = match session.transient.begin_object_fetch(&proxy_session_id, &resource, "ts", 0, 10) { - TransientObjectFetchDecision::Fetch(token) => token, - TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { - panic!("first archive object starts a cache fill") - } - }; - (resource, lookup_key, fetch_token) - }; - gc.cache - .write_bytes_and_commit(fetch_token.cache_key(), b"archive-object") - .await - .expect("archive object writes"); - { - let mut session = session.write().await; - assert!(session.commit_transient_object_ready_if_current( - resource.kind, - &fetch_token, - "video/mp2t".to_string(), - 14, - 0, - 10, - )); - let protected = ProtectedSet::from_session(&session); - assert_eq!(session.transient.current_manifest_resource_ids().len(), 1_643); - assert!(!protected.transient_object_ids.contains(&resource.id)); - } - - let report = gc.run_once(20).await.expect("finalized archive GC runs"); - - assert_eq!(report.transient_resources_pruned, 0); - assert_eq!(report.transient_objects_deleted, 1); - assert!(gc.cache.metadata(fetch_token.cache_key()).await.expect("object metadata reads").is_none()); - let mut session = session.write().await; - assert_eq!(session.transient.resources.len(), 1_643); - let current = - session.transient.resolve_current_resource(&resource.id, 20).expect("mapping survives byte eviction"); - assert!(!session.transient.object_cache.contains_key(&lookup_key)); - let proxy_session_id = session.proxy_session_id.clone(); - assert!(matches!( - session.transient.begin_object_fetch(&proxy_session_id, ¤t, "ts", 20, 10,), - TransientObjectFetchDecision::Fetch(_) - )); - } - - fn encrypted_terminal_evidence_manifest( - proxy_session_id: &ProxySessionId, - resource_id: &TransientResourceId, - ) -> HlsLeaseManifestSnapshot { - let encryption = Arc::new(HlsEncryptionSignature { - method: "AES-128".to_string(), - key_uri: Some(format!("/hls/shared/live/{}/lease/r/{}.key", proxy_session_id.0, resource_id.0)), - iv: Some("0x00000000000000000000000000000001".to_string()), - key_format: Some("identity".to_string()), - key_format_versions: Some("1".to_string()), - can_reset_to_clear: true, - }); - HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(1), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 1, - delivered_at_ms: 0, - first_proxy_seq: 1, - last_proxy_seq: 1, - visible_segments: Arc::from([HlsLeaseManifestSegment { - proxy_seq: 1, - duration_ms: 4_000, - uri: "/hls/shared/live/session/lease/1.ts".to_string().into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: Some(encryption.clone()), - }]), - discontinuity_sequence: 0, - target_duration_ms: 4_000, - playlist_duration_ms: 4_000, - last_visible_media_end_ms: 4_000, - active_map: None, - active_encryption: Some(encryption), - container: HlsMediaContainer::MpegTs, - } - } - - #[tokio::test] - async fn terminal_evidence_pins_ready_key_object_and_mapping_across_gc_until_release() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let (segment_cache_key, proxy_session_id, resource, resource_id, object_lookup_key, object_fetch_token) = { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - let segment_cache_key = session.segments.get(&1).expect("segment").cache_key.clone(); - let resource = TransientResourceRef::new( - TransientResourceKind::Key, - "http://origin.example.com/live/key.bin", - b"secret", - 0, - 10, - Some("key".to_string()), - ); - let resource_id = resource.id.clone(); - session.transient.upsert_resources([resource]); - let resource = session.transient.resources.get(&resource_id).expect("registered key resource").clone(); - session.segments.get_mut(&1).expect("encrypted segment").encryption = Some(HlsSegmentEncryption { - resource_id: resource_id.clone(), - resource_extension: "key".to_string(), - iv: Some("0x00000000000000000000000000000001".to_string()), - key_format: Some("identity".to_string()), - key_format_versions: Some("1".to_string()), - }); - let proxy_session_id = session.proxy_session_id.clone(); - let object_lookup_key = - TransientPassthroughState::transient_object_key(&proxy_session_id, &resource_id, "key"); - let object_fetch_token = - match session.transient.begin_object_fetch(&proxy_session_id, &resource, "key", 0, 10) { - TransientObjectFetchDecision::Fetch(token) => token, - TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { - panic!("new key object starts one physical cache fill") - } - }; - (segment_cache_key, proxy_session_id, resource, resource_id, object_lookup_key, object_fetch_token) - }; - gc.cache.write_bytes_and_commit(&segment_cache_key, b"ready-media").await.expect("media cache write"); - gc.cache - .write_bytes_and_commit(object_fetch_token.cache_key(), b"0123456789abcdef") - .await - .expect("key cache write"); - { - let mut session = session.write().await; - session.segments.get_mut(&1).expect("segment").status = - SegmentCacheStatus::Ready { content_length: 11, ready_at_ms: 0 }; - assert!(session.commit_transient_object_ready_if_current( - resource.kind, - &object_fetch_token, - "application/octet-stream".to_string(), - 16, - 0, - 10, - )); - } - let manifest = encrypted_terminal_evidence_manifest(&proxy_session_id, &resource_id); - - let evidence = prepare_terminal_base_evidence(&session, &gc.cache, &manifest, 5).await; - assert!(evidence.availability()[0].required_key_ready); - { - let mut session = session.write().await; - session.transient.resources.get_mut(&resource_id).expect("key mapping").expires_at_ms = 0; - session.transient.object_cache.get_mut(&object_lookup_key).expect("key object").expires_at_ms = 0; - } - - let pinned_report = gc.run_once(20).await.expect("pinned GC"); - assert_eq!(pinned_report.transient_resources_pruned, 0); - assert_eq!(pinned_report.transient_objects_deleted, 0); - { - let session = session.read().await; - assert!(session.transient.resources.contains_key(&resource_id)); - assert!(session.transient.object_cache.contains_key(&object_lookup_key)); - } - - evidence.release(); - let generation_before_release_gc = session.read().await.activity.media_readiness_generation; - let released_report = gc.run_once(21).await.expect("released GC"); - assert_eq!(released_report.transient_resources_pruned, 1); - assert_eq!(released_report.transient_objects_deleted, 1); - let session = session.read().await; - assert_eq!( - session.activity.media_readiness_generation, - generation_before_release_gc.saturating_add(1), - "mapping and READY key object removal advance readiness once per session sweep" - ); - assert!(!session.transient.resources.contains_key(&resource_id)); - assert!(!session.transient.object_cache.contains_key(&object_lookup_key)); - } - - #[tokio::test] - async fn transient_object_gc_deletes_the_physical_fill_generation_instead_of_the_logical_lookup_path() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let (lookup_key, fetch_token, resource_kind) = { - let mut session = session.write().await; - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/object1.ts", - b"secret", - 0, - 10, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - session.transient.upsert_resources([resource]); - let resource = session.transient.resources.get(&resource_id).expect("registered resource").clone(); - let proxy_session_id = session.proxy_session_id.clone(); - let lookup_key = TransientPassthroughState::transient_object_key(&proxy_session_id, &resource_id, "ts"); - let fetch_token = match session.transient.begin_object_fetch(&proxy_session_id, &resource, "ts", 0, 10) { - TransientObjectFetchDecision::Fetch(token) => token, - TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { - panic!("new transient object starts one physical cache fill") - } - }; - (lookup_key, fetch_token, resource.kind) - }; - assert_ne!(&lookup_key, fetch_token.cache_key()); - gc.cache - .write_bytes_and_commit(fetch_token.cache_key(), b"transient-body") - .await - .expect("physical generation writes"); - gc.cache.write_bytes_and_commit(&lookup_key, b"logical-decoy").await.expect("logical lookup decoy writes"); - assert!(session.write().await.commit_transient_object_ready_if_current( - resource_kind, - &fetch_token, - "video/mp2t".to_string(), - 14, - 0, - 10, - )); - - let report = gc.run_once(20).await.expect("gc should run"); - - assert_eq!(report.transient_objects_deleted, 1); - assert_eq!(report.transient_object_bytes_deleted, 14); - assert!(gc.cache.metadata(fetch_token.cache_key()).await.expect("physical metadata read").is_none()); - assert!(gc.cache.metadata(&lookup_key).await.expect("logical metadata read").is_some()); - } - - #[tokio::test] - async fn transient_object_gc_expires_stale_fetching_metadata_deletes_its_fill_and_wakes_waiters() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let (lookup_key, fetch_token, notifier) = { - let mut session = session.write().await; - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/stale.ts", - b"secret", - 0, - 10, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - session.transient.upsert_resources([resource]); - let resource = session.transient.resources.get(&resource_id).expect("registered resource").clone(); - let proxy_session_id = session.proxy_session_id.clone(); - let lookup_key = TransientPassthroughState::transient_object_key(&proxy_session_id, &resource_id, "ts"); - let fetch_token = match session.transient.begin_object_fetch(&proxy_session_id, &resource, "ts", 0, 10) { - TransientObjectFetchDecision::Fetch(token) => token, - TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { - panic!("new transient object starts one physical cache fill") - } - }; - let notifier = match session.transient.begin_object_fetch(&proxy_session_id, &resource, "ts", 1, 10) { - TransientObjectFetchDecision::Wait(notifier) => notifier, - TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Fetch(_) => { - panic!("a concurrent request waits for the current cache fill") - } - }; - (lookup_key, fetch_token, notifier) - }; - gc.cache - .write_bytes_and_commit(fetch_token.cache_key(), b"partial") - .await - .expect("stale physical fill fixture writes"); - let waiter = notifier.notified(); - tokio::pin!(waiter); - assert!(matches!(futures::poll!(&mut waiter), Poll::Pending)); - - let report = gc.run_once(20).await.expect("gc should run"); - - assert!(matches!(futures::poll!(&mut waiter), Poll::Ready(()))); - assert_eq!(report.transient_objects_deleted, 1); - assert_eq!(report.transient_object_bytes_deleted, 0); - assert!(!session.read().await.transient.object_cache.contains_key(&lookup_key)); - assert!(gc.cache.metadata(fetch_token.cache_key()).await.expect("metadata reads").is_none()); - } - - #[tokio::test] - async fn session_size_gc_deletes_transient_objects_before_timeline_segments() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - update_gc_policy(&gc, |policy| policy.cache_bytes_per_session = 20); - let (segment_key, object_lookup_key, object_fetch_token, resource_kind) = { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - let segment_key = session.segments.get(&1).expect("segment").cache_key.clone(); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/object1.ts", - b"secret", - 100, - 10_000, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - session.transient.upsert_resources([resource]); - let resource = session.transient.resources.get(&resource_id).expect("registered resource").clone(); - let proxy_session_id = session.proxy_session_id.clone(); - let object_lookup_key = - TransientPassthroughState::transient_object_key(&proxy_session_id, &resource_id, "ts"); - let object_fetch_token = - match session.transient.begin_object_fetch(&proxy_session_id, &resource, "ts", 100, 10_000) { - TransientObjectFetchDecision::Fetch(token) => token, - TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { - panic!("new transient object starts one physical cache fill") - } - }; - (segment_key, object_lookup_key, object_fetch_token, resource.kind) - }; - gc.cache.write_bytes_and_commit(&segment_key, b"segment-body").await.expect("segment writes"); - gc.cache - .write_bytes_and_commit(object_fetch_token.cache_key(), b"transient-body") - .await - .expect("transient object writes"); - { - let mut session = session.write().await; - session.segments.get_mut(&1).expect("segment").status = - SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 100 }; - assert!(session.commit_transient_object_ready_if_current( - resource_kind, - &object_fetch_token, - "video/mp2t".to_string(), - 14, - 100, - 10_000, - )); - } - - let report = gc.run_once(100).await.expect("gc should run"); - - assert_eq!(report.transient_objects_deleted, 1); - assert_eq!(report.segments_deleted_size_session, 0); - let session = session.read().await; - assert!(session.segments.contains_key(&1)); - assert!(!session.transient.object_cache.contains_key(&object_lookup_key)); - drop(session); - assert!(gc.cache.metadata(object_fetch_token.cache_key()).await.expect("metadata reads").is_none()); - } - - #[tokio::test] - async fn secret_fingerprint_mismatch_invalidates_cache_and_sessions() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - let segment = session.segments.get_mut(&1).expect("segment"); - gc.cache - .write_bytes_and_commit(&segment.cache_key, b"segment-body") - .await - .expect("cache write should succeed"); - } - gc.cache.write_rewrite_secret_fingerprint("mismatch").await.expect("marker write"); - - let report = gc.run_once(1).await.expect("gc should run"); - - assert!(report.secret_cache_invalidated); - assert!(!report.secret_cache_invalidation_deferred); - assert_eq!(gc.sessions.len().await, 0); - let rewrite_secret_fingerprint = gc.rewrite_secret_fingerprint(); - assert_eq!( - gc.cache.read_rewrite_secret_fingerprint().await.expect("marker read").as_deref(), - Some(rewrite_secret_fingerprint.as_str()) - ); - } - - #[tokio::test] - async fn secret_fingerprint_mismatch_with_active_temp_defers_invalidation() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let cache_key = { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - session.segments.get(&1).expect("segment").cache_key.clone() - }; - gc.cache.write_rewrite_secret_fingerprint("mismatch").await.expect("marker write"); - let staged = gc - .cache - .stage_temp_with_deadline(&cache_key, &b"done"[..], tokio::time::Instant::now() + Duration::from_mins(1)) - .await - .expect("cache object stages"); - assert!(gc.cache.has_active_temp_files()); - - let report = gc.run_once(1).await.expect("gc should run"); - - assert!(!report.secret_cache_invalidated); - assert!(report.secret_cache_invalidation_deferred); - assert_eq!(gc.sessions.len().await, 1); - assert_eq!(gc.cache.read_rewrite_secret_fingerprint().await.expect("marker read").as_deref(), Some("mismatch")); - assert!(gc.cache.has_active_temp_files()); - gc.cache.commit_staged(&cache_key, staged).await.expect("staged object commits"); - assert!(gc.cache.metadata(&cache_key).await.expect("metadata should read").is_some()); - } - - #[tokio::test] - async fn deferred_secret_fingerprint_invalidation_runs_after_temp_commit() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let cache_key = { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - session.segments.get(&1).expect("segment").cache_key.clone() - }; - gc.cache.write_rewrite_secret_fingerprint("mismatch").await.expect("marker write"); - let staged = gc - .cache - .stage_temp_with_deadline(&cache_key, &b"done"[..], tokio::time::Instant::now() + Duration::from_mins(1)) - .await - .expect("cache object stages"); - assert!(gc.run_once(1).await.expect("first gc should run").secret_cache_invalidation_deferred); - gc.cache.commit_staged(&cache_key, staged).await.expect("staged object commits"); - - let report = gc.run_once(2).await.expect("second gc should run"); - - assert!(report.secret_cache_invalidated); - assert!(!report.secret_cache_invalidation_deferred); - assert_eq!(gc.sessions.len().await, 0); - assert_eq!(gc.cache.metadata(&cache_key).await.expect("metadata should read"), None); - let rewrite_secret_fingerprint = gc.rewrite_secret_fingerprint(); - assert_eq!( - gc.cache.read_rewrite_secret_fingerprint().await.expect("marker read").as_deref(), - Some(rewrite_secret_fingerprint.as_str()) - ); - } - - #[tokio::test] - async fn global_candidate_selection_skips_a_session_that_failed_revalidation() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, first_session) = gc_with_session(&temp_dir).await; - let second_session = gc.sessions.get_or_create_session(HlsSessionKey::new(2, "12345"), b"secret", 0).await; - - for (session, ready_at_ms) in [(&first_session, 1), (&second_session, 2)] { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - session.segments.get_mut(&1).expect("head").status = - SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; - } - let first_proxy_session_id = first_session.read().await.proxy_session_id.clone(); - let sessions = [Arc::clone(&first_session), Arc::clone(&second_session)]; - - let first = - oldest_global_fifo_head_candidate(&sessions, &gc.cache, None, &HashSet::new()).await.expect("candidate"); - assert_eq!(first.proxy_session_id, first_proxy_session_id); - - let skipped = HashSet::from([first_proxy_session_id]); - let next = - oldest_global_fifo_head_candidate(&sessions, &gc.cache, None, &skipped).await.expect("next candidate"); - assert_eq!(next.proxy_session_id, second_session.read().await.proxy_session_id); - } - - #[tokio::test] - async fn global_size_gc_deletes_oldest_unprotected_segments() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, first_session) = gc_with_session(&temp_dir).await; - update_gc_policy(&gc, |policy| policy.cache_bytes_global = 24); - let second_session = gc.sessions.get_or_create_session(HlsSessionKey::new(2, "12345"), b"secret", 0).await; - - for session in [&first_session, &second_session] { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - for proxy_seq in [1, 2, 3] { - let segment = session.segments.get_mut(&proxy_seq).expect("segment"); - gc.cache - .write_bytes_and_commit(&segment.cache_key, b"segment-body") - .await - .expect("cache write should succeed"); - segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: proxy_seq }; - } - } - - let report = gc.run_once(100).await.expect("gc should run"); - - assert_eq!(report.segments_deleted_size_global, 4); - let remaining_size = { - let first = first_session.read().await; - let second = second_session.read().await; - super::session_cache_size(&first) + super::session_cache_size(&second) - }; - assert!(remaining_size <= 24); - } - - #[tokio::test] - async fn global_size_gc_uses_only_current_fifo_heads() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, first_session) = gc_with_session(&temp_dir).await; - update_gc_policy(&gc, |policy| policy.cache_bytes_global = 24); - let second_session = gc.sessions.get_or_create_session(HlsSessionKey::new(2, "12345"), b"secret", 0).await; - - for session in [&first_session, &second_session] { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - for proxy_seq in [1, 2, 3] { - let segment = session.segments.get_mut(&proxy_seq).expect("segment"); - gc.cache - .write_bytes_and_commit(&segment.cache_key, b"segment-body") - .await - .expect("cache write should succeed"); - segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: proxy_seq }; - } - } - first_session.write().await.segments.get_mut(&1).expect("head segment").access.reader_started(1); - - let report = gc.run_once(100).await.expect("gc should run"); - - assert_eq!(report.segments_deleted_size_global, 3); - let first = first_session.read().await; - assert!(first.segments.contains_key(&1)); - assert!(first.segments.contains_key(&2)); - assert!(first.segments.contains_key(&3)); - let second = second_session.read().await; - assert!(!second.segments.contains_key(&1)); - assert!(!second.segments.contains_key(&2)); - assert!(!second.segments.contains_key(&3)); - } - - #[tokio::test] - async fn temp_file_gc_deletes_old_tmp_files() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let temp_path = { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - let path = gc.cache.object_path(&session.segments.get(&1).expect("segment").cache_key); - let parent = path.parent().expect("cache object has parent"); - tokio::fs::create_dir_all(parent).await.expect("parent dir"); - parent.join("000001.ts.tmp.old") - }; - tokio::fs::write(&temp_path, b"partial").await.expect("temp write"); - let old_time = filetime::FileTime::from_unix_time(1, 0); - filetime::set_file_mtime(&temp_path, old_time).expect("set mtime"); - - let report = gc.run_once(1).await.expect("gc should run"); - - assert_eq!(report.temp_files_deleted, 1); - assert!(!temp_path.exists()); - } - - #[tokio::test] - async fn session_gc_keeps_idle_session_with_active_temp_file() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let (cache_key, proxy_session_id) = { - let mut session = session.write().await; - apply_six_segment_manifest_for_gc(&mut session); - (session.segments.get(&1).expect("segment").cache_key.clone(), session.proxy_session_id.clone()) - }; - let staged = gc - .cache - .stage_temp_with_deadline(&cache_key, &b"done"[..], tokio::time::Instant::now() + Duration::from_mins(1)) - .await - .expect("cache object stages"); - assert!(gc.cache.has_active_temp_files_for_session(&proxy_session_id)); - - let report = gc.run_once(2_000).await.expect("gc should run"); - - assert_eq!(report.sessions_deleted, 0); - assert!(!session.read().await.is_gc_marked_for_removal()); - assert_eq!(gc.sessions.len().await, 1); - gc.cache.commit_staged(&cache_key, staged).await.expect("staged object commits"); - } - - #[tokio::test] - async fn session_gc_removes_idle_session_without_activity() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - let (key, proxy_session_id) = { - let session = session.read().await; - (session.key.clone(), session.proxy_session_id.clone()) - }; - - let report = gc.run_once(2_000).await.expect("gc should run"); - - assert_eq!(report.sessions_deleted, 1); - assert!(gc.sessions.get_by_key(&key).await.is_none()); - assert!(gc.sessions.get_by_proxy_session_id(&proxy_session_id).await.is_none()); - } - - #[tokio::test] - async fn session_gc_final_recheck_keeps_new_activity() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - session.write().await.last_client_access_at_ms = 1_500; - let mut report = super::GarbageCollectionReport::default(); - let policy = gc.policy(); - - gc.remove_idle_session_if_still_idle(&session, 2_000, &policy, &mut report).await; - - assert_eq!(report.sessions_deleted, 0); - assert_eq!(gc.sessions.len().await, 1); - assert!(!session.read().await.is_gc_marked_for_removal()); - } - - #[tokio::test] - async fn active_transient_resource_reader_protects_idle_session() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let (gc, session) = gc_with_session(&temp_dir).await; - { - let mut session = session.write().await; - let resource = crate::TransientResourceRef::new( - crate::TransientResourceKind::Segment, - "http://origin.example.com/live/seg.ts", - b"secret", - 0, - 10, - Some("ts".to_string()), - ); - resource.access.reader_started(1); - session.transient.upsert_resources([resource]); - } - - let report = gc.run_once(2_000).await.expect("gc should run"); - - assert_eq!(report.sessions_deleted, 0); - assert_eq!(gc.sessions.len().await, 1); - } -} +mod tests; + +mod collector; +mod deletion; +mod policy; +mod reclaim; +mod report; +mod selection; +pub use collector::{build_rewrite_secret_fingerprint, exec_hls_cache_gc}; +use deletion::{CacheDeletionQueueGeneration, CacheObjectDeletion}; +pub use policy::{GarbageCollectionPolicy, ProtectedSet}; +pub use report::GarbageCollectionReport; diff --git a/backend/hls/src/gc/collector.rs b/backend/hls/src/gc/collector.rs new file mode 100644 index 000000000..f81513a29 --- /dev/null +++ b/backend/hls/src/gc/collector.rs @@ -0,0 +1,367 @@ +use super::{ + deletion::CacheObjectDeletion, safe_proxy_session_id, CacheDeletionBatch, CacheDeletionQueueState, + CacheInvalidationOutcome, GarbageCollectionPolicy, GarbageCollectionReport, HlsAccessLeaseStore, + HlsCacheCapacityReclaimOutcome, HlsCacheCapacityReclaimRequest, HlsCacheCapacityReclaimer, HlsCacheMetrics, HlsCtx, + HlsExpiredSessionReason, HlsGarbageCollector, HlsSegmentCache, HlsSession, HlsSessionHandle, HlsSessionStore, + HlsSwitchCacheCleanupReservation, MapCacheKey, SegmentCacheKey, HLS_CACHE_GC_INTERVAL, + MAX_CACHE_DELETE_RETRIES_PER_RUN, +}; +use arc_swap::ArcSwap; +use futures::{future::BoxFuture, FutureExt}; +use log::{debug, error, info, warn}; +use sha2::{Digest, Sha256}; +use std::{ + collections::HashSet, + io, + path::PathBuf, + sync::{Arc, Mutex as StdMutex, RwLock as StdRwLock}, + time::{Duration, SystemTime}, +}; +use tokio::sync::{Mutex as AsyncMutex, RwLock}; +use tokio_util::sync::CancellationToken; + +impl HlsGarbageCollector { + pub fn new( + sessions: Arc, + cache: Arc, + policy: GarbageCollectionPolicy, + rewrite_secret_fingerprint: String, + ) -> Self { + Self::new_with_metrics( + sessions, + cache, + policy, + rewrite_secret_fingerprint, + Arc::new(HlsCacheMetrics::default()), + ) + } + + pub fn new_with_metrics( + sessions: Arc, + cache: Arc, + policy: GarbageCollectionPolicy, + rewrite_secret_fingerprint: String, + metrics: Arc, + ) -> Self { + Self { + sessions, + cache, + policy: ArcSwap::from_pointee(policy), + rewrite_secret_fingerprint: ArcSwap::from_pointee(rewrite_secret_fingerprint), + metrics, + pending_cache_deletions: Arc::new(StdMutex::new(CacheDeletionQueueState::default())), + access_leases: StdRwLock::new(None), + run_once_gate: AsyncMutex::new(()), + } + } + + pub fn install_access_leases(&self, access_leases: &Arc>) { + *self.access_leases.write().unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Arc::downgrade(access_leases)); + } + + pub fn update_config(&self, policy: GarbageCollectionPolicy, rewrite_secret_fingerprint: String) { + self.policy.store(Arc::new(policy)); + self.rewrite_secret_fingerprint.store(Arc::new(rewrite_secret_fingerprint)); + } + + /// Serializes a cache-root handoff with GC and discards logical deletion tickets bound to the previous root. + pub async fn update_cache_path(&self, cache_path: impl Into) -> bool { + let _run_once = self.run_once_gate.lock().await; + let changed = self.cache.update_cache_path(cache_path).await; + if changed { + let mut queue = self.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + *queue = CacheDeletionQueueState::default(); + } + changed + } + + pub fn policy(&self) -> Arc { self.policy.load_full() } + + pub fn rewrite_secret_fingerprint(&self) -> String { self.rewrite_secret_fingerprint.load().to_string() } + + pub async fn run_once(&self, now_ms: u64) -> io::Result { + let _run_once = self.run_once_gate.lock().await; + let policy = self.policy.load_full(); + let mut report = GarbageCollectionReport::default(); + let mut deletion_attempt_budget = MAX_CACHE_DELETE_RETRIES_PER_RUN; + self.retry_pending_cache_deletions(&mut report, &mut deletion_attempt_budget).await; + if self.ensure_cache_marker(&mut report).await? { + report.cache_object_deletions_deferred = self.pending_cache_deletion_count(); + self.record_report_metrics(&report); + return Ok(report); + } + + // Captured before the in-memory session snapshot so any directory committed + // after this instant is treated as a potential concurrent create and is + // skipped by the orphan cleanup freshness guard. + let gc_start = SystemTime::now(); + let cutoff = gc_start + .checked_sub(Duration::from_millis(policy.temp_file_retention_ms)) + .unwrap_or(SystemTime::UNIX_EPOCH); + report.temp_files_deleted = self.cache.delete_temp_files_older_than(cutoff).await?; + + let sessions = self.sessions.list_sessions().await; + let mut active_session_ids = HashSet::with_capacity(sessions.len()); + for session in &sessions { + active_session_ids.insert(session.read().await.proxy_session_id.clone()); + } + report.orphan_session_dirs_deleted = + self.cache.delete_orphan_session_dirs(&active_session_ids, gc_start).await?; + for session in &sessions { + let mut session_deletions = self.reserve_cache_deletion_batch(); + let mut session = session.write().await; + Self::collect_session_deletions( + &mut session, + &self.cache, + now_ms, + &policy, + &mut report, + &mut session_deletions, + ); + session_deletions.persist(&mut report); + } + + let mut global_deletions = self.reserve_cache_deletion_batch(); + self.collect_global_size_deletions(&sessions, &policy, &mut report, &mut global_deletions).await; + global_deletions.persist(&mut report); + self.retry_pending_cache_deletions(&mut report, &mut deletion_attempt_budget).await; + + for session in &sessions { + self.remove_idle_session_if_still_idle(session, now_ms, &policy, &mut report).await; + } + + report.cache_object_deletions_deferred = self.pending_cache_deletion_count(); + self.record_report_metrics(&report); + if report.did_cleanup_or_invalidate() { + info!( + "HLS session garbage collection completed: temp_files_deleted={} orphan_session_dirs_deleted={} stale_queue_entries_removed={} cache_deletes_planned={} cache_deletes_succeeded={} cache_deletes_deferred={} segments_deleted={} maps_deleted={} transient_resources_pruned={} transient_objects_deleted={} transient_object_bytes_deleted={} sessions_deleted={}", + report.temp_files_deleted, + report.orphan_session_dirs_deleted, + report.stale_queue_entries_removed, + report.cache_object_deletions_planned, + report.cache_object_deletions_succeeded, + report.cache_object_deletions_deferred, + report.segments_deleted(), + report.maps_deleted, + report.transient_resources_pruned, + report.transient_objects_deleted, + report.transient_object_bytes_deleted, + report.sessions_deleted, + ); + } + Ok(report) + } + + pub(super) async fn ensure_cache_marker(&self, report: &mut GarbageCollectionReport) -> io::Result { + let rewrite_secret_fingerprint = self.rewrite_secret_fingerprint.load_full(); + match self.cache.read_rewrite_secret_fingerprint().await? { + Some(current) if current == *rewrite_secret_fingerprint => Ok(false), + Some(_) => { + self.metrics.record_secret_marker_mismatch(); + warn!("HLS rewrite secret changed or cache marker mismatch detected: action=validate-cache-marker"); + match self.cache.invalidate_all_if_no_active_temp_files().await? { + CacheInvalidationOutcome::Invalidated => { + self.sessions.clear().await; + self.cache.write_rewrite_secret_fingerprint(&rewrite_secret_fingerprint).await?; + report.secret_cache_invalidated = true; + info!("HLS rewrite secret changed or cache marker mismatch detected: action=cache-invalidated"); + Ok(true) + } + CacheInvalidationOutcome::DeferredActiveTempFiles => { + report.secret_cache_invalidation_deferred = true; + self.metrics.record_secret_invalidation_deferred(); + warn!( + "HLS rewrite secret changed or cache marker mismatch detected: action=deferred-active-temp-files" + ); + Ok(true) + } + } + } + None => { + self.cache.write_rewrite_secret_fingerprint(&rewrite_secret_fingerprint).await?; + Ok(false) + } + } + } + + pub(super) async fn remove_idle_session_if_still_idle( + &self, + session: &HlsSessionHandle, + now_ms: u64, + policy: &GarbageCollectionPolicy, + report: &mut GarbageCollectionReport, + ) { + let (key, proxy_session_id) = { + let mut session = session.write().await; + if !Self::should_remove_idle_session(&session, now_ms, policy) { + return; + } + session.mark_for_gc_removal(); + (session.key.clone(), session.proxy_session_id.clone()) + }; + + if self.cache.has_active_temp_files_for_session(&proxy_session_id) { + session.write().await.clear_gc_removal_mark(); + return; + } + + if self + .sessions + .remove_session_marking_expired( + &key, + &proxy_session_id, + now_ms, + HlsExpiredSessionReason::SessionIdleTimeout, + None, + ) + .await + .is_some() + { + report.sessions_deleted = report.sessions_deleted.saturating_add(1); + report.removed_session_ids.push(proxy_session_id.clone()); + if let Err(error) = self.cache.delete_session_dir(&proxy_session_id).await { + warn!( + "HLS idle session cache directory cleanup deferred: session={} error_kind={:?}", + safe_proxy_session_id(&proxy_session_id), + error.kind(), + ); + } + } else { + session.write().await.clear_gc_removal_mark(); + } + } + + pub(super) fn reserve_cache_deletion_batch(&self) -> CacheDeletionBatch { + CacheDeletionBatch::reserve(Arc::clone(&self.pending_cache_deletions)) + } + + pub fn reserve_switch_segment_cleanup(&self, key: SegmentCacheKey) -> Option { + HlsSwitchCacheCleanupReservation::reserve( + Arc::clone(&self.pending_cache_deletions), + CacheObjectDeletion::UncommittedSwitchSegment(key), + ) + } + + pub fn reserve_switch_map_cleanup(&self, key: MapCacheKey) -> Option { + HlsSwitchCacheCleanupReservation::reserve( + Arc::clone(&self.pending_cache_deletions), + CacheObjectDeletion::UncommittedSwitchMap(key), + ) + } + + pub fn has_pending_switch_cleanup(&self, segment_key: &SegmentCacheKey, map_key: Option<&MapCacheKey>) -> bool { + let state = self.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + state.pending.iter().any(|pending| match &pending.deletion { + CacheObjectDeletion::UncommittedSwitchSegment(key) => key == segment_key, + CacheObjectDeletion::UncommittedSwitchMap(key) => map_key == Some(key), + CacheObjectDeletion::Segment { .. } + | CacheObjectDeletion::Map(_) + | CacheObjectDeletion::TransientObject { .. } => false, + }) + } + + pub(super) fn pending_cache_deletion_count(&self) -> usize { + self.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner).pending.len() + } + + #[cfg(any(test, feature = "test-support"))] + pub fn cache_deletion_queue_usage(&self) -> (usize, usize) { + let state = self.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + (state.pending.len(), state.reserved_slots) + } + + pub(super) async fn retry_pending_cache_deletions( + &self, + report: &mut GarbageCollectionReport, + attempt_budget: &mut usize, + ) { + let attempts = self.pending_cache_deletion_count().min(*attempt_budget); + for _ in 0..attempts { + let pending = { + let queue = self.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + queue.pending.front().cloned() + }; + let Some(pending) = pending else { + break; + }; + *attempt_budget = attempt_budget.saturating_sub(1); + let result = pending.deletion.delete_from(&self.cache).await; + let mut queue = self.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(mut completed) = queue.pending.pop_front() else { + continue; + }; + if result.is_ok() { + drop(queue); + completed.deletion.record_success(report); + completed.deletion.log_success(); + } else { + completed.attempts = completed.attempts.saturating_add(1); + let attempts = completed.attempts; + let kind = result.as_ref().err().map(io::Error::kind); + completed.deletion.log_deferred(attempts, kind); + queue.pending.push_back(completed); + } + } + } + + pub(super) fn record_report_metrics(&self, report: &GarbageCollectionReport) { + self.metrics.record_gc_run(); + self.metrics.record_segments_removed(report.segments_deleted()); + self.metrics.record_maps_removed(report.maps_deleted); + } + + pub(super) fn should_remove_idle_session( + session: &HlsSession, + now_ms: u64, + policy: &GarbageCollectionPolicy, + ) -> bool { + session.can_expire_idle_session(now_ms, policy.session_idle_timeout_ms) + } +} + +impl HlsCacheCapacityReclaimer for HlsGarbageCollector { + fn reclaim_capacity( + &self, + request: HlsCacheCapacityReclaimRequest, + ) -> BoxFuture<'_, io::Result> { + async move { self.reclaim_for_projected_write(request).await }.boxed() + } +} + +pub fn build_rewrite_secret_fingerprint(rewrite_secret: &[u8]) -> String { + let digest = Sha256::digest(rewrite_secret); + let value = digest.iter().take(8).fold(0_u64, |value, byte| (value << 8) | u64::from(*byte)); + format!("{value:016x}") +} + +pub fn exec_hls_cache_gc(ctx: &HlsCtx, cancel_token: &CancellationToken) { + let hls_proxy = Arc::clone(&ctx.hls_proxy); + let active_users = Arc::clone(&ctx.active_users); + let active_provider = Arc::clone(&ctx.active_provider); + let cancel_token = cancel_token.clone(); + tokio::spawn(async move { + loop { + let now_ms = chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default(); + hls_proxy + .sync_all_session_access_leases_and_detach_if_needed(&active_users, &active_provider, now_ms) + .await; + match hls_proxy.run_garbage_collection_once(now_ms).await { + Ok(report) if report.did_cleanup_or_invalidate() => { + debug!( + "HLS cache state snapshot after garbage collection: {}", + hls_proxy.debug_state_summary().await + ); + } + Ok(_) => {} + Err(err) => { + error!("HLS cache garbage collection failed: {err}"); + } + } + tokio::select! { + () = cancel_token.cancelled() => break, + () = tokio::time::sleep(HLS_CACHE_GC_INTERVAL) => {} + } + } + }); +} diff --git a/backend/hls/src/gc/deletion.rs b/backend/hls/src/gc/deletion.rs new file mode 100644 index 000000000..f9b40d2c8 --- /dev/null +++ b/backend/hls/src/gc/deletion.rs @@ -0,0 +1,251 @@ +use super::{ + safe_proxy_session_id, CacheDeletionBatch, CacheDeletionQueueState, GarbageCollectionReport, HlsSegmentCache, + HlsSwitchCacheCleanupReservation, MapCacheKey, PendingCacheObjectDeletion, SegmentCacheKey, + TransientObjectCacheKey, MAX_PENDING_CACHE_DELETIONS, SWITCH_CACHE_CLEANUP_HEADROOM, +}; +use log::{debug, info, warn}; +use std::{ + collections::VecDeque, + io, + sync::{Arc, Mutex as StdMutex}, +}; + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum SegmentCacheDeletionReason { + Duration, + SessionSize, + GlobalSize, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum CacheObjectDeletion { + Segment { key: SegmentCacheKey, reason: SegmentCacheDeletionReason }, + Map(MapCacheKey), + TransientObject { key: TransientObjectCacheKey, content_length: u64 }, + UncommittedSwitchSegment(SegmentCacheKey), + UncommittedSwitchMap(MapCacheKey), +} + +impl CacheObjectDeletion { + pub(super) async fn delete_from(&self, cache: &HlsSegmentCache) -> io::Result<()> { + match self { + Self::Segment { key, .. } | Self::UncommittedSwitchSegment(key) => cache.delete_if_inactive(key).await, + Self::Map(key) | Self::UncommittedSwitchMap(key) => cache.delete_if_inactive(key).await, + Self::TransientObject { key, .. } => cache.delete_if_inactive(key).await, + } + } + + pub(super) fn record_success(&self, report: &mut GarbageCollectionReport) { + report.cache_object_deletions_succeeded = report.cache_object_deletions_succeeded.saturating_add(1); + match self { + Self::Segment { reason: SegmentCacheDeletionReason::Duration, .. } => { + report.segments_deleted_duration = report.segments_deleted_duration.saturating_add(1); + } + Self::Segment { reason: SegmentCacheDeletionReason::SessionSize, .. } => { + report.segments_deleted_size_session = report.segments_deleted_size_session.saturating_add(1); + } + Self::Segment { reason: SegmentCacheDeletionReason::GlobalSize, .. } => { + report.segments_deleted_size_global = report.segments_deleted_size_global.saturating_add(1); + } + Self::Map(_) => { + report.maps_deleted = report.maps_deleted.saturating_add(1); + } + Self::TransientObject { content_length, .. } => { + report.transient_objects_deleted = report.transient_objects_deleted.saturating_add(1); + report.transient_object_bytes_deleted = + report.transient_object_bytes_deleted.saturating_add(*content_length); + } + Self::UncommittedSwitchSegment(_) | Self::UncommittedSwitchMap(_) => {} + } + } + + pub(super) fn log_success(&self) { + match self { + Self::Segment { key, .. } => { + info!( + "Segment '{:06}' removed: session={} source=normal", + key.proxy_seq(), + safe_proxy_session_id(key.proxy_session_id()), + ); + } + Self::UncommittedSwitchSegment(_) | Self::UncommittedSwitchMap(_) => { + debug!("HLS uncommitted switch cache object removed"); + } + Self::Map(_) | Self::TransientObject { .. } => {} + } + } + + pub(super) fn log_deferred(&self, attempts: u16, error_kind: Option) { + let object_kind = match self { + Self::Segment { .. } => "segment", + Self::Map(_) => "map", + Self::TransientObject { .. } => "transient-object", + Self::UncommittedSwitchSegment(_) => "uncommitted-switch-segment", + Self::UncommittedSwitchMap(_) => "uncommitted-switch-map", + }; + warn!( + "HLS cache object deletion deferred: object_kind={object_kind} attempts={attempts} error_kind={error_kind:?}" + ); + } +} + +pub(super) struct CacheDeletionQueueGeneration; + +impl Default for CacheDeletionQueueState { + fn default() -> Self { + Self { pending: VecDeque::new(), reserved_slots: 0, generation: Arc::new(CacheDeletionQueueGeneration) } + } +} + +impl HlsSwitchCacheCleanupReservation { + pub(super) fn reserve( + queue: Arc>, + deletion: CacheObjectDeletion, + ) -> Option { + let queue_generation = { + let mut state = queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if state.pending.len().saturating_add(state.reserved_slots) >= MAX_PENDING_CACHE_DELETIONS { + return None; + } + state.reserved_slots = state.reserved_slots.saturating_add(1); + Arc::clone(&state.generation) + }; + Some(Self { queue, queue_generation, deletion: Some(deletion), slot_reserved: true }) + } + + pub fn disarm(&mut self) { + self.deletion = None; + self.release_slot(); + } + + pub(super) fn release_slot(&mut self) { + if !self.slot_reserved { + return; + } + let mut state = self.queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if Arc::ptr_eq(&state.generation, &self.queue_generation) { + state.reserved_slots = state.reserved_slots.saturating_sub(1); + } + self.slot_reserved = false; + } +} + +impl std::fmt::Debug for HlsSwitchCacheCleanupReservation { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("HlsSwitchCacheCleanupReservation") + .field("armed", &self.deletion.is_some()) + .field("slot_reserved", &self.slot_reserved) + .finish_non_exhaustive() + } +} + +impl Drop for HlsSwitchCacheCleanupReservation { + fn drop(&mut self) { + if !self.slot_reserved { + return; + } + let mut state = self.queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if Arc::ptr_eq(&state.generation, &self.queue_generation) { + state.reserved_slots = state.reserved_slots.saturating_sub(1); + if let Some(deletion) = self.deletion.take() { + state.pending.push_back(PendingCacheObjectDeletion { deletion, attempts: 0 }); + } + } + self.slot_reserved = false; + } +} + +impl CacheDeletionBatch { + pub(super) fn reserve(queue: Arc>) -> Self { + let reserved_slots = { + let mut state = queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let batch_limit = MAX_PENDING_CACHE_DELETIONS.saturating_sub(SWITCH_CACHE_CLEANUP_HEADROOM); + let available = batch_limit.saturating_sub(state.pending.len()).saturating_sub(state.reserved_slots); + state.reserved_slots = state.reserved_slots.saturating_add(available); + available + }; + Self { queue, reserved_slots, deletions: Vec::new() } + } + + pub(super) fn has_capacity(&self) -> bool { self.deletions.len() < self.reserved_slots } + + pub(super) fn remaining_capacity(&self) -> usize { self.reserved_slots.saturating_sub(self.deletions.len()) } + + pub(super) fn push(&mut self, deletion: CacheObjectDeletion) { self.deletions.push(deletion); } + + pub(super) fn persist_pending(&mut self, report: &mut GarbageCollectionReport) { + let planned = self.deletions.len(); + if planned == 0 { + return; + } + { + let mut state = self.queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + state.reserved_slots = state.reserved_slots.saturating_sub(planned); + state + .pending + .extend(self.deletions.drain(..).map(|deletion| PendingCacheObjectDeletion { deletion, attempts: 0 })); + } + self.reserved_slots = self.reserved_slots.saturating_sub(planned); + report.cache_object_deletions_planned = report.cache_object_deletions_planned.saturating_add(planned); + } + + pub(super) fn persist(mut self, report: &mut GarbageCollectionReport) { self.persist_pending(report); } + + pub(super) async fn execute_prioritized( + mut self, + cache: &HlsSegmentCache, + report: &mut GarbageCollectionReport, + attempt_budget: &mut usize, + ) { + while *attempt_budget > 0 { + let Some(deletion) = self.deletions.first().cloned() else { + break; + }; + *attempt_budget = attempt_budget.saturating_sub(1); + let result = deletion.delete_from(cache).await; + let completed = self.deletions.remove(0); + self.release_reserved_slot(); + report.cache_object_deletions_planned = report.cache_object_deletions_planned.saturating_add(1); + match result { + Ok(()) => { + completed.record_success(report); + completed.log_success(); + } + Err(error) => { + completed.log_deferred(1, Some(error.kind())); + self.persist_failed(completed); + } + } + } + self.persist_pending(report); + } + + pub(super) fn release_reserved_slot(&mut self) { + let mut state = self.queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + state.reserved_slots = state.reserved_slots.saturating_sub(1); + self.reserved_slots = self.reserved_slots.saturating_sub(1); + } + + pub(super) fn persist_failed(&mut self, deletion: CacheObjectDeletion) { + self.queue + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .pending + .push_back(PendingCacheObjectDeletion { deletion, attempts: 1 }); + } +} + +impl Drop for CacheDeletionBatch { + fn drop(&mut self) { + if self.reserved_slots == 0 && self.deletions.is_empty() { + return; + } + let mut state = self.queue.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + state.reserved_slots = state.reserved_slots.saturating_sub(self.reserved_slots); + state + .pending + .extend(self.deletions.drain(..).map(|deletion| PendingCacheObjectDeletion { deletion, attempts: 0 })); + self.reserved_slots = 0; + } +} diff --git a/backend/hls/src/gc/policy.rs b/backend/hls/src/gc/policy.rs new file mode 100644 index 000000000..2b7a87d9a --- /dev/null +++ b/backend/hls/src/gc/policy.rs @@ -0,0 +1,140 @@ +use super::{ + extract_transient_resource_ids, renderer_candidate_window_proxy_seqs, HlsSegmentCache, HlsSession, MapCacheStatus, + ProxyMapId, SegmentCacheStatus, TransientResourceId, DEFAULT_FAILED_SEGMENT_RETENTION_MS, + DEFAULT_TEMP_FILE_RETENTION_MS, +}; +use std::collections::HashSet; +use tuliprox_core::model::HlsCacheConfig; + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct GarbageCollectionPolicy { + pub cache_duration_ms: u64, + pub cache_bytes_global: u64, + pub cache_bytes_per_session: u64, + pub session_idle_timeout_ms: u64, + pub temp_file_retention_ms: u64, + pub failed_segment_retention_ms: u64, +} + +impl GarbageCollectionPolicy { + pub fn from_config(config: &HlsCacheConfig) -> Self { + Self { + cache_duration_ms: config.cache_duration.as_millis().get(), + cache_bytes_global: config.cache_bytes.get(), + cache_bytes_per_session: config.cache_bytes_per_session.get(), + session_idle_timeout_ms: config.session_idle_timeout.as_millis().get(), + temp_file_retention_ms: DEFAULT_TEMP_FILE_RETENTION_MS, + failed_segment_retention_ms: DEFAULT_FAILED_SEGMENT_RETENTION_MS, + } + } +} + +impl Default for GarbageCollectionPolicy { + fn default() -> Self { + let default_config = HlsCacheConfig::from(&shared::model::HlsCacheConfigDto::default()); + Self::from_config(&default_config) + } +} + +#[derive(Debug, Default, Clone, Eq, PartialEq)] +pub struct ProtectedSet { + pub segment_proxy_seqs: HashSet, + pub map_ids: HashSet, + pub key_resource_ids: HashSet, + pub transient_object_ids: HashSet, +} + +impl ProtectedSet { + #[cfg(test)] + pub fn from_session(session: &HlsSession) -> Self { Self::from_session_for_capacity(session, None) } + + pub(super) fn from_session_for_capacity(session: &HlsSession, release_through: Option) -> Self { + let mut protected = Self::default(); + + if let Some(rendered) = &session.last_rendered_manifest { + protected.segment_proxy_seqs.extend( + rendered + .segment_proxy_seqs + .iter() + .copied() + .filter(|proxy_seq| release_through.is_none_or(|release| *proxy_seq > release)), + ); + } + if let Some(rendered) = &session.last_rendered_manifest { + protected.key_resource_ids.extend(extract_transient_resource_ids(&rendered.body)); + } + for (_, protection) in session.terminal_tail_protections() { + protected.segment_proxy_seqs.extend(protection.base_proxy_seqs.iter().copied()); + } + protected.segment_proxy_seqs.extend( + renderer_candidate_window_proxy_seqs(session) + .into_iter() + .filter(|proxy_seq| release_through.is_none_or(|release| *proxy_seq > release)), + ); + + for (proxy_seq, entry) in &session.segments { + if entry.access.active_readers() > 0 + || matches!(entry.status, SegmentCacheStatus::Fetching { .. }) + || (entry.origin_fetch_ref.is_some() + && (matches!(entry.status, SegmentCacheStatus::Queued { .. }) + || session.segment_prefetch_queue.contains(*proxy_seq))) + { + protected.segment_proxy_seqs.insert(*proxy_seq); + } + } + + for proxy_seq in &protected.segment_proxy_seqs { + if let Some(segment) = session.segments.get(proxy_seq) { + if let Some(map_ref) = segment.map_ref { + protected.map_ids.insert(map_ref); + } + if let Some(encryption) = &segment.encryption { + protected.key_resource_ids.insert(encryption.resource_id.clone()); + } + } + } + for (map_id, map) in &session.maps { + if map.access.active_readers() > 0 + || matches!(map.status, MapCacheStatus::Queued { .. } | MapCacheStatus::Fetching { .. }) + { + protected.map_ids.insert(*map_id); + } + } + for entry in session.transient.object_cache.values() { + if entry.access.active_readers() > 0 { + protected.transient_object_ids.insert(entry.key.transient_resource_id().clone()); + } + } + protected.transient_object_ids.extend(protected.key_resource_ids.iter().cloned()); + + protected + } +} + +pub(super) fn protected_set_for_cache_reclamation( + session: &HlsSession, + cache: &HlsSegmentCache, + excluded_path: Option<&std::path::Path>, + release_through: Option, +) -> ProtectedSet { + let mut protected = ProtectedSet::from_session_for_capacity(session, release_through); + for (proxy_seq, segment) in &session.segments { + let path = cache.object_path(&segment.cache_key); + if cache.has_active_mutation(&path) || excluded_path == Some(path.as_path()) { + protected.segment_proxy_seqs.insert(*proxy_seq); + } + } + for (map_id, map) in &session.maps { + let path = cache.object_path(&map.cache_key); + if cache.has_active_mutation(&path) || excluded_path == Some(path.as_path()) { + protected.map_ids.insert(*map_id); + } + } + for entry in session.transient.object_cache.values() { + let path = cache.object_path(&entry.key); + if cache.has_active_mutation(&path) || excluded_path == Some(path.as_path()) { + protected.transient_object_ids.insert(entry.key.transient_resource_id().clone()); + } + } + protected +} diff --git a/backend/hls/src/gc/reclaim.rs b/backend/hls/src/gc/reclaim.rs new file mode 100644 index 000000000..ae07f378c --- /dev/null +++ b/backend/hls/src/gc/reclaim.rs @@ -0,0 +1,416 @@ +use super::{ + deletion::{CacheObjectDeletion, SegmentCacheDeletionReason}, + policy::protected_set_for_cache_reclamation, + selection::{ + capacity_reclamation_evidence, collect_projected_session_reclamation, duration_expired_head_segment, + fifo_head_size_candidate, oldest_global_fifo_head_candidate, oldest_global_transient_object_candidate, + remove_map_entry, remove_segment_entry, remove_stale_queue_entries, session_cache_size, + take_expired_transient_objects, total_sessions_cache_size, unprotected_unreferenced_map_ids, + }, + CacheDeletionBatch, GarbageCollectionPolicy, GarbageCollectionReport, HlsCacheCapacityReclaimOutcome, + HlsCacheCapacityReclaimRequest, HlsGarbageCollector, HlsSegmentCache, HlsSession, HlsSessionHandle, + MAX_CACHE_DELETE_RETRIES_PER_RUN, +}; +use std::{ + collections::HashSet, + io, + sync::{Arc, Weak}, +}; + +impl HlsGarbageCollector { + #[allow(clippy::too_many_lines)] + pub(super) async fn reclaim_for_projected_write( + &self, + request: HlsCacheCapacityReclaimRequest, + ) -> io::Result { + let _run_once = self.run_once_gate.lock().await; + if !self.cache.contains_current_cache_path(&request.target_path) { + return Err(io::Error::new( + io::ErrorKind::Interrupted, + "hls cache path changed before capacity reclamation", + )); + } + let before = self.cache.capacity_usage(&request.proxy_session_id).await?; + let mut report = GarbageCollectionReport::default(); + let mut deletion_attempt_budget = MAX_CACHE_DELETE_RETRIES_PER_RUN; + let mut protected_working_set_bytes = 0_u64; + let mut reclaimable_bytes = 0_u64; + + let mut pending_attempt_budget = deletion_attempt_budget / 2; + let pending_attempts_before = pending_attempt_budget; + self.retry_pending_cache_deletions(&mut report, &mut pending_attempt_budget).await; + deletion_attempt_budget = + deletion_attempt_budget.saturating_sub(pending_attempts_before.saturating_sub(pending_attempt_budget)); + let after_pending = self.cache.capacity_usage(&request.proxy_session_id).await?; + let required_session_bytes = request + .required_session_bytes + .saturating_sub(before.session_bytes.saturating_sub(after_pending.session_bytes)); + if required_session_bytes > 0 { + if let Some(session) = self.sessions.get_by_proxy_session_id(&request.proxy_session_id).await { + let mut deletions = self.reserve_cache_deletion_batch(); + let access_leases = self + .access_leases + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .and_then(Weak::upgrade); + let lease_guard = match &access_leases { + Some(access_leases) => Some(access_leases.read().await), + None => None, + }; + // Projected-capacity selection follows the runtime lock order + // lease store -> session. The session lock is non-blocking + // while the lease snapshot is frozen, and neither guard + // crosses filesystem I/O or an await. Contention installs a + // bounded wake task instead of reversing the lock order. + let release_through = + lease_guard.as_ref().and_then(|leases| leases.capacity_release_through(&request.proxy_session_id)); + let mut session_lock_busy = false; + if let Ok(mut session) = session.try_write() { + let protected = protected_set_for_cache_reclamation( + &session, + &self.cache, + Some(&request.target_path), + release_through, + ); + let evidence = capacity_reclamation_evidence(&session, &protected); + protected_working_set_bytes = evidence.protected_working_set_bytes; + reclaimable_bytes = evidence.reclaimable_bytes; + collect_projected_session_reclamation( + &mut session, + &self.cache, + &request.target_path, + release_through, + required_session_bytes, + deletion_attempt_budget, + &mut deletions, + ); + if !deletions.deletions.is_empty() { + let rendered_at_ms = session + .last_rendered_manifest + .as_ref() + .map_or(session.last_client_access_at_ms, |rendered| { + rendered.rendered_at_ms.saturating_add(1) + }); + let _ = session.render_and_store_manifest(rendered_at_ms); + } + } else { + protected_working_set_bytes = after_pending.session_bytes; + session_lock_busy = true; + } + drop(lease_guard); + if session_lock_busy { + let wake_session = Arc::clone(&session); + let wake_cache = Arc::clone(&self.cache); + tokio::spawn(async move { + let guard = wake_session.write().await; + drop(guard); + wake_cache.notify_capacity_protection_changed(); + }); + } + deletions.execute_prioritized(&self.cache, &mut report, &mut deletion_attempt_budget).await; + } + } + + let after_session = self.cache.capacity_usage(&request.proxy_session_id).await?; + let remaining_session_bytes = request + .required_session_bytes + .saturating_sub(before.session_bytes.saturating_sub(after_session.session_bytes)); + if remaining_session_bytes > 0 { + return Ok(HlsCacheCapacityReclaimOutcome { + reclaimed_session_bytes: before.session_bytes.saturating_sub(after_session.session_bytes), + reclaimed_global_bytes: before.global_bytes.saturating_sub(after_session.global_bytes), + protected_working_set_bytes, + reclaimable_bytes, + }); + } + let required_global_bytes = request + .required_global_bytes + .saturating_sub(before.global_bytes.saturating_sub(after_session.global_bytes)); + if required_global_bytes > 0 { + let sessions = self.sessions.list_sessions().await; + let mut deletions = self.reserve_cache_deletion_batch(); + self.collect_projected_global_reclamation( + &sessions, + &request.target_path, + required_global_bytes, + deletion_attempt_budget, + &mut deletions, + ) + .await; + deletions.execute_prioritized(&self.cache, &mut report, &mut deletion_attempt_budget).await; + } + + let after = self.cache.capacity_usage(&request.proxy_session_id).await?; + Ok(HlsCacheCapacityReclaimOutcome { + reclaimed_session_bytes: before.session_bytes.saturating_sub(after.session_bytes), + reclaimed_global_bytes: before.global_bytes.saturating_sub(after.global_bytes), + protected_working_set_bytes, + reclaimable_bytes, + }) + } + + pub(super) fn collect_session_deletions( + session: &mut HlsSession, + cache: &HlsSegmentCache, + now_ms: u64, + policy: &GarbageCollectionPolicy, + report: &mut GarbageCollectionReport, + deletions: &mut CacheDeletionBatch, + ) { + report.stale_queue_entries_removed = + report.stale_queue_entries_removed.saturating_add(remove_stale_queue_entries(session)); + + let transient_before = session.transient.resources.len(); + let protected = protected_set_for_cache_reclamation(session, cache, None, None); + session.transient.prune_expired_except(now_ms, &protected.key_resource_ids); + let transient_resources_pruned = transient_before.saturating_sub(session.transient.resources.len()); + let mut transient_readiness_changed = transient_resources_pruned > 0; + report.transient_resources_pruned = + report.transient_resources_pruned.saturating_add(transient_resources_pruned); + + let expired_transient_objects = take_expired_transient_objects( + session, + now_ms, + &protected.transient_object_ids, + deletions.remaining_capacity(), + ); + transient_readiness_changed |= !expired_transient_objects.is_empty(); + for (key, content_length) in expired_transient_objects { + deletions.push(CacheObjectDeletion::TransientObject { key, content_length }); + } + + while deletions.has_capacity() { + let Some(proxy_seq) = duration_expired_head_segment( + session, + &protected_set_for_cache_reclamation(session, cache, None, None), + policy, + now_ms, + ) else { + break; + }; + if let Some(deletion) = remove_segment_entry(session, proxy_seq) { + deletions + .push(CacheObjectDeletion::Segment { key: deletion, reason: SegmentCacheDeletionReason::Duration }); + } + } + + let mut session_size = session_cache_size(session); + while session_size > policy.cache_bytes_per_session && deletions.has_capacity() { + let protected = protected_set_for_cache_reclamation(session, cache, None, None); + let Some(removal) = session.transient.remove_oldest_ready_object_except(&protected.transient_object_ids) + else { + break; + }; + session_size = session_size.saturating_sub(removal.content_length); + transient_readiness_changed = true; + deletions.push(CacheObjectDeletion::TransientObject { + key: removal.key, + content_length: removal.content_length, + }); + } + while session_size > policy.cache_bytes_per_session && deletions.has_capacity() { + let Some(candidate) = + fifo_head_size_candidate(session, &protected_set_for_cache_reclamation(session, cache, None, None)) + else { + break; + }; + session_size = session_size.saturating_sub(candidate.content_length); + if let Some(deletion) = remove_segment_entry(session, candidate.proxy_seq) { + deletions.push(CacheObjectDeletion::Segment { + key: deletion, + reason: SegmentCacheDeletionReason::SessionSize, + }); + } + } + + for map_id in + unprotected_unreferenced_map_ids(session, &protected_set_for_cache_reclamation(session, cache, None, None)) + { + if !deletions.has_capacity() { + break; + } + if let Some(deletion) = remove_map_entry(session, map_id) { + session_size = session_size.saturating_sub(deletion.content_length); + deletions.push(CacheObjectDeletion::Map(deletion.key)); + } + } + if transient_readiness_changed { + session.advance_media_readiness_generation(); + } + } + + pub(super) async fn collect_global_size_deletions( + &self, + sessions: &[HlsSessionHandle], + policy: &GarbageCollectionPolicy, + report: &mut GarbageCollectionReport, + deletions: &mut CacheDeletionBatch, + ) { + let mut total_size = total_sessions_cache_size(sessions).await; + let mut transient_readiness_advanced = HashSet::new(); + let mut skipped_transient_sessions = HashSet::new(); + + loop { + if total_size <= policy.cache_bytes_global || !deletions.has_capacity() { + break; + } + let Some(candidate) = + oldest_global_transient_object_candidate(sessions, &self.cache, None, &skipped_transient_sessions) + .await + else { + break; + }; + let mut session = candidate.session.write().await; + let protected = protected_set_for_cache_reclamation(&session, &self.cache, None, None); + let Some(removal) = session.transient.remove_oldest_ready_object_except(&protected.transient_object_ids) + else { + skipped_transient_sessions.insert(candidate.proxy_session_id); + continue; + }; + total_size = total_size.saturating_sub(removal.content_length); + if transient_readiness_advanced.insert(session.proxy_session_id.clone()) { + session.advance_media_readiness_generation(); + } + deletions.push(CacheObjectDeletion::TransientObject { + key: removal.key, + content_length: removal.content_length, + }); + deletions.persist_pending(report); + } + + let mut skipped_segment_sessions = HashSet::new(); + loop { + if total_size <= policy.cache_bytes_global || !deletions.has_capacity() { + break; + } + let Some(candidate) = + oldest_global_fifo_head_candidate(sessions, &self.cache, None, &skipped_segment_sessions).await + else { + break; + }; + let mut session = candidate.session.write().await; + let Some(current_head) = fifo_head_size_candidate( + &session, + &protected_set_for_cache_reclamation(&session, &self.cache, None, None), + ) else { + skipped_segment_sessions.insert(candidate.proxy_session_id); + continue; + }; + if current_head.proxy_seq != candidate.proxy_seq { + skipped_segment_sessions.insert(candidate.proxy_session_id); + continue; + } + let Some(deletion) = remove_segment_entry(&mut session, candidate.proxy_seq) else { + skipped_segment_sessions.insert(candidate.proxy_session_id); + continue; + }; + total_size = total_size.saturating_sub(candidate.content_length); + deletions + .push(CacheObjectDeletion::Segment { key: deletion, reason: SegmentCacheDeletionReason::GlobalSize }); + deletions.persist_pending(report); + + for map_id in unprotected_unreferenced_map_ids( + &session, + &protected_set_for_cache_reclamation(&session, &self.cache, None, None), + ) { + if !deletions.has_capacity() { + break; + } + if let Some(deletion) = remove_map_entry(&mut session, map_id) { + total_size = total_size.saturating_sub(deletion.content_length); + deletions.push(CacheObjectDeletion::Map(deletion.key)); + deletions.persist_pending(report); + } + } + } + } + + pub(super) async fn collect_projected_global_reclamation( + &self, + sessions: &[HlsSessionHandle], + excluded_path: &std::path::Path, + required_bytes: u64, + max_deletions: usize, + deletions: &mut CacheDeletionBatch, + ) { + let mut planned_bytes = 0_u64; + let mut planned_deletions = 0_usize; + let mut transient_readiness_advanced = HashSet::new(); + let mut skipped_transient_sessions = HashSet::new(); + while planned_bytes < required_bytes && planned_deletions < max_deletions && deletions.has_capacity() { + let Some(candidate) = oldest_global_transient_object_candidate( + sessions, + &self.cache, + Some(excluded_path), + &skipped_transient_sessions, + ) + .await + else { + break; + }; + let mut session = candidate.session.write().await; + let protected = protected_set_for_cache_reclamation(&session, &self.cache, Some(excluded_path), None); + let Some(removal) = session.transient.remove_oldest_ready_object_except(&protected.transient_object_ids) + else { + skipped_transient_sessions.insert(candidate.proxy_session_id); + continue; + }; + planned_bytes = planned_bytes.saturating_add(removal.content_length); + if transient_readiness_advanced.insert(session.proxy_session_id.clone()) { + session.advance_media_readiness_generation(); + } + deletions.push(CacheObjectDeletion::TransientObject { + key: removal.key, + content_length: removal.content_length, + }); + planned_deletions = planned_deletions.saturating_add(1); + } + + let mut skipped_segment_sessions = HashSet::new(); + while planned_bytes < required_bytes && planned_deletions < max_deletions && deletions.has_capacity() { + let Some(candidate) = oldest_global_fifo_head_candidate( + sessions, + &self.cache, + Some(excluded_path), + &skipped_segment_sessions, + ) + .await + else { + break; + }; + let mut session = candidate.session.write().await; + let protected = protected_set_for_cache_reclamation(&session, &self.cache, Some(excluded_path), None); + let Some(current_head) = fifo_head_size_candidate(&session, &protected) else { + skipped_segment_sessions.insert(candidate.proxy_session_id); + continue; + }; + if current_head.proxy_seq != candidate.proxy_seq { + skipped_segment_sessions.insert(candidate.proxy_session_id); + continue; + } + let Some(deletion) = remove_segment_entry(&mut session, candidate.proxy_seq) else { + skipped_segment_sessions.insert(candidate.proxy_session_id); + continue; + }; + planned_bytes = planned_bytes.saturating_add(candidate.content_length); + deletions + .push(CacheObjectDeletion::Segment { key: deletion, reason: SegmentCacheDeletionReason::GlobalSize }); + planned_deletions = planned_deletions.saturating_add(1); + + for map_id in unprotected_unreferenced_map_ids( + &session, + &protected_set_for_cache_reclamation(&session, &self.cache, Some(excluded_path), None), + ) { + if planned_bytes >= required_bytes || planned_deletions >= max_deletions || !deletions.has_capacity() { + break; + } + if let Some(deletion) = remove_map_entry(&mut session, map_id) { + planned_bytes = planned_bytes.saturating_add(deletion.content_length); + deletions.push(CacheObjectDeletion::Map(deletion.key)); + planned_deletions = planned_deletions.saturating_add(1); + } + } + } + } +} diff --git a/backend/hls/src/gc/report.rs b/backend/hls/src/gc/report.rs new file mode 100644 index 000000000..261ce8660 --- /dev/null +++ b/backend/hls/src/gc/report.rs @@ -0,0 +1,46 @@ +use super::ProxySessionId; + +#[derive(Debug, Default, Clone, Eq, PartialEq)] +pub struct GarbageCollectionReport { + pub secret_cache_invalidated: bool, + pub secret_cache_invalidation_deferred: bool, + pub temp_files_deleted: usize, + pub orphan_session_dirs_deleted: usize, + pub stale_queue_entries_removed: usize, + pub segments_deleted_duration: usize, + pub segments_deleted_size_session: usize, + pub segments_deleted_size_global: usize, + pub maps_deleted: usize, + pub sessions_deleted: usize, + pub removed_session_ids: Vec, + pub transient_resources_pruned: usize, + pub transient_objects_deleted: usize, + pub transient_object_bytes_deleted: u64, + pub cache_object_deletions_planned: usize, + pub cache_object_deletions_succeeded: usize, + pub cache_object_deletions_deferred: usize, +} + +impl GarbageCollectionReport { + pub(super) fn segments_deleted(&self) -> usize { + self.segments_deleted_duration + .saturating_add(self.segments_deleted_size_session) + .saturating_add(self.segments_deleted_size_global) + } + + pub fn did_cleanup_or_invalidate(&self) -> bool { + self.secret_cache_invalidated + || self.secret_cache_invalidation_deferred + || self.temp_files_deleted > 0 + || self.orphan_session_dirs_deleted > 0 + || self.stale_queue_entries_removed > 0 + || self.segments_deleted() > 0 + || self.maps_deleted > 0 + || self.sessions_deleted > 0 + || self.transient_resources_pruned > 0 + || self.transient_objects_deleted > 0 + || self.cache_object_deletions_planned > 0 + || self.cache_object_deletions_succeeded > 0 + || self.cache_object_deletions_deferred > 0 + } +} diff --git a/backend/hls/src/gc/selection.rs b/backend/hls/src/gc/selection.rs new file mode 100644 index 000000000..4c694e8de --- /dev/null +++ b/backend/hls/src/gc/selection.rs @@ -0,0 +1,337 @@ +use super::{ + deletion::{CacheObjectDeletion, SegmentCacheDeletionReason}, + policy::protected_set_for_cache_reclamation, + CacheDeletionBatch, CapacityReclamationEvidence, GarbageCollectionPolicy, GlobalSegmentCandidate, + GlobalTransientObjectCandidate, HlsSegmentCache, HlsSession, HlsSessionHandle, MapCacheStatus, MapEntryDeletion, + ProtectedSet, ProxyMapId, ProxySessionId, SegmentCacheKey, SegmentCacheStatus, SegmentDeleteCandidate, + TransientObjectCacheKey, TransientResourceId, +}; +use std::{collections::HashSet, sync::Arc}; + +pub(super) async fn total_sessions_cache_size(sessions: &[HlsSessionHandle]) -> u64 { + let mut total_size = 0_u64; + for session in sessions { + let session = session.read().await; + total_size = total_size.saturating_add(session_cache_size(&session)); + } + total_size +} + +pub(super) fn collect_projected_session_reclamation( + session: &mut HlsSession, + cache: &HlsSegmentCache, + excluded_path: &std::path::Path, + release_through: Option, + required_bytes: u64, + max_deletions: usize, + deletions: &mut CacheDeletionBatch, +) { + let mut planned_bytes = 0_u64; + let mut planned_deletions = 0_usize; + let mut transient_readiness_changed = false; + while planned_bytes < required_bytes && planned_deletions < max_deletions && deletions.has_capacity() { + let protected = protected_set_for_cache_reclamation(session, cache, Some(excluded_path), release_through); + let Some(removal) = session.transient.remove_oldest_ready_object_except(&protected.transient_object_ids) else { + break; + }; + planned_bytes = planned_bytes.saturating_add(removal.content_length); + transient_readiness_changed = true; + deletions + .push(CacheObjectDeletion::TransientObject { key: removal.key, content_length: removal.content_length }); + planned_deletions = planned_deletions.saturating_add(1); + } + while planned_bytes < required_bytes && planned_deletions < max_deletions && deletions.has_capacity() { + let protected = protected_set_for_cache_reclamation(session, cache, Some(excluded_path), release_through); + let Some(candidate) = fifo_head_size_candidate(session, &protected) else { + break; + }; + if let Some(deletion) = remove_segment_entry(session, candidate.proxy_seq) { + planned_bytes = planned_bytes.saturating_add(candidate.content_length); + deletions + .push(CacheObjectDeletion::Segment { key: deletion, reason: SegmentCacheDeletionReason::SessionSize }); + planned_deletions = planned_deletions.saturating_add(1); + } + } + for map_id in unprotected_unreferenced_map_ids( + session, + &protected_set_for_cache_reclamation(session, cache, Some(excluded_path), release_through), + ) { + if planned_bytes >= required_bytes || planned_deletions >= max_deletions || !deletions.has_capacity() { + break; + } + if let Some(deletion) = remove_map_entry(session, map_id) { + planned_bytes = planned_bytes.saturating_add(deletion.content_length); + deletions.push(CacheObjectDeletion::Map(deletion.key)); + planned_deletions = planned_deletions.saturating_add(1); + } + } + if transient_readiness_changed { + session.advance_media_readiness_generation(); + } +} + +pub(super) async fn oldest_global_fifo_head_candidate( + sessions: &[HlsSessionHandle], + cache: &HlsSegmentCache, + excluded_path: Option<&std::path::Path>, + skipped_sessions: &HashSet, +) -> Option { + let mut candidates = Vec::new(); + for session in sessions { + let session_guard = session.read().await; + if skipped_sessions.contains(&session_guard.proxy_session_id) { + continue; + } + if let Some(candidate) = fifo_head_size_candidate( + &session_guard, + &protected_set_for_cache_reclamation(&session_guard, cache, excluded_path, None), + ) { + candidates.push(GlobalSegmentCandidate { + session: Arc::clone(session), + proxy_session_id: session_guard.proxy_session_id.clone(), + proxy_seq: candidate.proxy_seq, + content_length: candidate.content_length, + last_relevant_at_ms: candidate.last_relevant_at_ms, + }); + } + } + candidates.into_iter().min_by_key(|candidate| (candidate.last_relevant_at_ms, candidate.proxy_seq)) +} + +pub(super) async fn oldest_global_transient_object_candidate( + sessions: &[HlsSessionHandle], + cache: &HlsSegmentCache, + excluded_path: Option<&std::path::Path>, + skipped_sessions: &HashSet, +) -> Option { + let mut candidates = Vec::new(); + for session in sessions { + let session_guard = session.read().await; + if skipped_sessions.contains(&session_guard.proxy_session_id) { + continue; + } + let protected = protected_set_for_cache_reclamation(&session_guard, cache, excluded_path, None); + candidates.extend(session_guard.transient.object_cache.values().filter_map(|entry| { + entry.ready_content_length()?; + if entry.access.active_readers() > 0 + || protected.transient_object_ids.contains(entry.key.transient_resource_id()) + { + return None; + } + Some(GlobalTransientObjectCandidate { + session: Arc::clone(session), + proxy_session_id: session_guard.proxy_session_id.clone(), + last_accessed_at_ms: entry.last_accessed_at_ms, + }) + })); + } + candidates.into_iter().min_by_key(|candidate| candidate.last_accessed_at_ms) +} + +pub(super) fn take_expired_transient_objects( + session: &mut HlsSession, + now_ms: u64, + protected: &HashSet, + limit: usize, +) -> Vec<(TransientObjectCacheKey, u64)> { + session + .transient + .take_expired_object_removals_except(now_ms, protected, limit) + .into_iter() + .map(|removal| (removal.key, removal.content_length)) + .collect() +} + +pub(super) fn remove_stale_queue_entries(session: &mut HlsSession) -> usize { + let mut removed = 0_usize; + for proxy_seq in session.segment_prefetch_queue.proxy_seqs() { + let stale = session.segments.get(&proxy_seq).is_none_or(|segment| { + segment.origin_fetch_ref.is_none() || !matches!(segment.status, SegmentCacheStatus::Queued { .. }) + }); + if stale && session.segment_prefetch_queue.remove(proxy_seq).is_some() { + removed = removed.saturating_add(1); + } + } + removed +} + +pub(super) fn duration_expired_head_segment( + session: &HlsSession, + protected: &ProtectedSet, + policy: &GarbageCollectionPolicy, + now_ms: u64, +) -> Option { + let (proxy_seq, segment) = session.segments.iter().next()?; + if protected.segment_proxy_seqs.contains(proxy_seq) { + return None; + } + let last_relevant_at_ms = segment_last_relevant_at_ms(segment)?; + let retention_ms = match segment.status { + SegmentCacheStatus::FailedRetryable { .. } + | SegmentCacheStatus::FailedPermanent { .. } + | SegmentCacheStatus::Expired => policy.failed_segment_retention_ms, + SegmentCacheStatus::Ready { .. } => policy + .cache_duration_ms + .max(segment.duration_ms.saturating_add(session.longest_rendered_playlist_duration_ms)), + SegmentCacheStatus::Discovered + | SegmentCacheStatus::Queued { .. } + | SegmentCacheStatus::Fetching { .. } + | SegmentCacheStatus::CapacityDeferred { .. } => return None, + }; + (now_ms.saturating_sub(last_relevant_at_ms) >= retention_ms).then_some(*proxy_seq) +} + +pub(super) fn fifo_head_size_candidate( + session: &HlsSession, + protected: &ProtectedSet, +) -> Option { + let (proxy_seq, segment) = session.segments.iter().next()?; + if protected.segment_proxy_seqs.contains(proxy_seq) { + return None; + } + let SegmentCacheStatus::Ready { content_length, .. } = segment.status else { + return None; + }; + Some(SegmentDeleteCandidate { + proxy_seq: *proxy_seq, + content_length, + last_relevant_at_ms: segment_last_relevant_at_ms(segment).unwrap_or_default(), + }) +} + +fn segment_last_relevant_at_ms(segment: &super::super::SegmentEntry) -> Option { + let status_at = match segment.status { + SegmentCacheStatus::Ready { ready_at_ms, .. } => Some(ready_at_ms), + SegmentCacheStatus::FailedRetryable { failed_at_ms, .. } + | SegmentCacheStatus::FailedPermanent { failed_at_ms, .. } => Some(failed_at_ms), + SegmentCacheStatus::Expired => segment.last_rendered_at_ms, + SegmentCacheStatus::Discovered + | SegmentCacheStatus::Queued { .. } + | SegmentCacheStatus::Fetching { .. } + | SegmentCacheStatus::CapacityDeferred { .. } => None, + }; + [status_at, segment.last_rendered_at_ms, Some(segment.access.last_accessed_at_ms()).filter(|value| *value > 0)] + .into_iter() + .flatten() + .max() +} + +pub(super) fn session_cache_size(session: &HlsSession) -> u64 { + let segment_bytes = session + .segments + .values() + .map(|segment| match segment.status { + SegmentCacheStatus::Ready { content_length, .. } => content_length, + _ => 0, + }) + .sum::(); + let map_bytes = session + .maps + .values() + .map(|map| match map.status { + MapCacheStatus::Ready { content_length, .. } => content_length, + _ => 0, + }) + .sum::(); + segment_bytes.saturating_add(map_bytes).saturating_add(session.transient.ready_object_cache_size()) +} + +pub(super) fn capacity_reclamation_evidence( + session: &HlsSession, + protected: &ProtectedSet, +) -> CapacityReclamationEvidence { + let protected_segment_bytes = session.segments.iter().fold(0_u64, |bytes, (proxy_seq, segment)| { + if !protected.segment_proxy_seqs.contains(proxy_seq) { + return bytes; + } + match segment.status { + SegmentCacheStatus::Ready { content_length, .. } => bytes.saturating_add(content_length), + _ => bytes, + } + }); + let protected_map_bytes = session.maps.iter().fold(0_u64, |bytes, (map_id, map)| { + if !protected.map_ids.contains(map_id) { + return bytes; + } + match map.status { + MapCacheStatus::Ready { content_length, .. } => bytes.saturating_add(content_length), + _ => bytes, + } + }); + let (protected_transient_object_bytes, reclaimable_transient_object_bytes) = + session.transient.object_cache.values().fold((0_u64, 0_u64), |(protected_bytes, reclaimable_bytes), entry| { + let Some(content_length) = entry.ready_content_length() else { + return (protected_bytes, reclaimable_bytes); + }; + if protected.transient_object_ids.contains(entry.key.transient_resource_id()) + || entry.access.active_readers() > 0 + { + (protected_bytes.saturating_add(content_length), reclaimable_bytes) + } else { + (protected_bytes, reclaimable_bytes.saturating_add(content_length)) + } + }); + let reclaimable_bytes = fifo_head_size_candidate(session, protected) + .map_or(0, |candidate| candidate.content_length) + .saturating_add(reclaimable_transient_object_bytes); + CapacityReclamationEvidence { + protected_working_set_bytes: protected_segment_bytes + .saturating_add(protected_map_bytes) + .saturating_add(protected_transient_object_bytes), + reclaimable_bytes, + } +} + +pub(super) fn remove_segment_entry(session: &mut HlsSession, proxy_seq: u64) -> Option { + let segment = session.segments.remove(&proxy_seq)?; + if let Some(startup) = &mut session.startup { + startup.revisions.remove(&proxy_seq); + } + let removed_ready_media = matches!(segment.status, SegmentCacheStatus::Ready { .. }); + session.segment_prefetch_queue.remove(proxy_seq); + session.origin_to_proxy.retain(|_, mapped_seq| *mapped_seq != proxy_seq); + if segment.discontinuity_before { + session.discontinuity_sequence = session.discontinuity_sequence.saturating_add(1); + } + if removed_ready_media { + session.advance_media_readiness_generation(); + } + if session.publishable_origin_head_proxy_seq == Some(proxy_seq) { + session.publishable_origin_head_proxy_seq = session + .segments + .range(proxy_seq.saturating_add(1)..) + .find_map(|(next_proxy_seq, segment)| segment.origin_fetch_ref.as_ref().map(|_| *next_proxy_seq)); + } + Some(segment.cache_key) +} + +pub(super) fn unprotected_unreferenced_map_ids(session: &HlsSession, protected: &ProtectedSet) -> Vec { + let referenced = session.segments.values().filter_map(|segment| segment.map_ref).collect::>(); + session + .maps + .iter() + .filter_map(|(map_id, map)| { + if referenced.contains(map_id) + || protected.map_ids.contains(map_id) + || map.access.active_readers() > 0 + || matches!(map.status, MapCacheStatus::Queued { .. } | MapCacheStatus::Fetching { .. }) + { + return None; + } + Some(*map_id) + }) + .collect() +} + +pub(super) fn remove_map_entry(session: &mut HlsSession, map_id: ProxyMapId) -> Option { + let map = session.maps.remove(&map_id)?; + let (content_length, removed_ready_media) = match map.status { + MapCacheStatus::Ready { content_length, .. } => (content_length, true), + _ => (0, false), + }; + if removed_ready_media { + session.advance_media_readiness_generation(); + } + session.origin_map_to_proxy.retain(|_, mapped_map_id| *mapped_map_id != map_id); + Some(MapEntryDeletion { key: map.cache_key, content_length }) +} diff --git a/backend/hls/src/gc/tests.rs b/backend/hls/src/gc/tests.rs new file mode 100644 index 000000000..45ddd1f92 --- /dev/null +++ b/backend/hls/src/gc/tests.rs @@ -0,0 +1,255 @@ +use super::{ + super::{ + media_reserve::{ + HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsManifestCommitIdentity, HlsManifestDeliveryMode, + }, + terminal_tail::{HlsEncryptionSignature, HlsMediaContainer, HlsTerminalTailGeneration}, + }, + build_rewrite_secret_fingerprint, + deletion::{CacheObjectDeletion, SegmentCacheDeletionReason}, + selection::{oldest_global_fifo_head_candidate, remove_segment_entry}, + GarbageCollectionPolicy, GarbageCollectionReport, HlsGarbageCollector, PendingCacheObjectDeletion, ProtectedSet, + MAX_PENDING_CACHE_DELETIONS, SWITCH_CACHE_CLEANUP_HEADROOM, +}; +use crate::{ + prepare_terminal_base_evidence, transient_manifest::TransientManifestRewriter, HlsAccessLease, HlsAccessLeaseId, + HlsAccessLeaseStore, HlsOriginResourceFetchError, HlsPlaybackFamilyKey, HlsSegmentCache, HlsSegmentEncryption, + HlsSession, HlsSessionKey, HlsSessionStore, HlsTerminalTailProtection, MapCacheStatus, OriginMapKey, ProxyMapId, + ProxySessionId, SegmentCacheStatus, SegmentFetchPriority, TransientObjectFetchDecision, TransientPassthroughState, + TransientResourceId, TransientResourceKind, TransientResourceRef, +}; +use std::{collections::HashSet, fmt::Write as _, sync::Arc, task::Poll, time::Duration}; +use tokio::sync::{Barrier, RwLock}; +use tuliprox_parser::hls::origin_manifest::{ + parse_manifest_semantics, parse_origin_media_manifest, OriginManifestParseOutcome, +}; + +const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; + +fn normal_manifest(body: &str) -> tuliprox_parser::hls::origin_manifest::ParsedOriginManifest { + match parse_origin_media_manifest(body, BASE_URL) { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + } +} + +fn test_policy() -> GarbageCollectionPolicy { + GarbageCollectionPolicy { + cache_duration_ms: 300, + cache_bytes_global: 10_000, + cache_bytes_per_session: 10_000, + session_idle_timeout_ms: 1_000, + temp_file_retention_ms: 30_000, + failed_segment_retention_ms: 10, + } +} + +async fn gc_with_session(temp_dir: &tempfile::TempDir) -> (Arc, crate::HlsSessionHandle) { + let sessions = Arc::new(HlsSessionStore::new()); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let session = sessions.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; + let gc = Arc::new(HlsGarbageCollector::new( + sessions, + Arc::clone(&cache), + test_policy(), + build_rewrite_secret_fingerprint(b"secret"), + )); + cache.install_capacity_reclaimer(&gc); + (gc, session) +} + +fn update_gc_policy(gc: &HlsGarbageCollector, update: impl FnOnce(&mut GarbageCollectionPolicy)) { + let mut policy = gc.policy().as_ref().clone(); + update(&mut policy); + gc.update_config(policy, gc.rewrite_secret_fingerprint()); +} + +fn six_segment_manifest() -> tuliprox_parser::hls::origin_manifest::ParsedOriginManifest { + normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n#EXTINF:4.0,\n4.ts\n#EXTINF:4.0,\n5.ts\n#EXTINF:4.0,\n6.ts\n", + ) +} + +fn apply_six_segment_manifest_for_gc(session: &mut super::HlsSession) { + session.proxy_next_seq = Some(1); + session.apply_origin_manifest(&six_segment_manifest()).expect("manifest should map"); +} + +fn install_finalized_transient_manifest( + session: &mut HlsSession, + segment_count: usize, + resource_ttl_ms: u64, +) -> Vec { + let mut body = + String::from("#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:1\n"); + for index in 0..segment_count { + writeln!(body, "#EXTINF:6.000,\n{index}.ts").expect("synthetic finalized manifest renders"); + } + body.push_str("#EXT-X-ENDLIST\n"); + let rewritten = + TransientManifestRewriter::rewrite(&body, BASE_URL, &session.proxy_session_id, b"secret", 0, resource_ttl_ms); + let resources = rewritten.resources.clone(); + session + .transient + .commit_rewritten_manifest_with_semantics( + rewritten.body, + rewritten.resources, + 0, + Some(u64::try_from(segment_count).expect("segment count fits u64").saturating_mul(6_000)), + parse_manifest_semantics(&body), + ) + .expect("synthetic finalized manifest stays within representation limits"); + resources +} + +async fn populate_ready_segments(gc: &HlsGarbageCollector, session: &crate::HlsSessionHandle, ready_at_ms: u64) { + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for segment in session.segments.values_mut() { + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; + } + session.render_and_store_manifest(ready_at_ms).expect("ready manifest should render"); + } +} + +async fn cache_selected_ready_segments( + gc: &HlsGarbageCollector, + session: &crate::HlsSessionHandle, + proxy_seqs: &[u64], + body: &[u8], +) { + let keys = { + let session = session.read().await; + proxy_seqs + .iter() + .map(|proxy_seq| (*proxy_seq, session.segments.get(proxy_seq).expect("segment").cache_key.clone())) + .collect::>() + }; + for (_, key) in &keys { + gc.cache.write_bytes_and_commit(key, body).await.expect("cache fixture writes"); + } + let mut session = session.write().await; + for (proxy_seq, _) in keys { + session.segments.get_mut(&proxy_seq).expect("segment").status = SegmentCacheStatus::Ready { + content_length: u64::try_from(body.len()).unwrap_or(u64::MAX), + ready_at_ms: proxy_seq, + }; + } +} + +async fn commit_sparse_segment(gc: &HlsGarbageCollector, key: &crate::SegmentCacheKey, size: u64) { + let final_path = gc.cache.object_path(key); + let parent = final_path.parent().expect("cache object parent"); + tokio::fs::create_dir_all(parent).await.expect("cache object parent creates"); + let temp_path = final_path.with_extension(format!("ts.tmp.fixture-{}", key.proxy_seq())); + let file = tokio::fs::File::create(&temp_path).await.expect("sparse temp creates"); + file.set_len(size).await.expect("sparse temp length"); + drop(file); + let staged = gc.cache.adopt_staged_file(temp_path, size).expect("sparse staged object adopts"); + gc.cache.commit_staged(key, staged).await.expect("sparse object commits"); +} + +fn activated_startup_lease(proxy_session_id: &ProxySessionId) -> HlsAccessLease { + let snapshot = HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(10), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 1, + delivered_at_ms: 10, + first_proxy_seq: 1, + last_proxy_seq: 3, + visible_segments: Arc::from( + (1_u64..=3) + .map(|proxy_seq| HlsLeaseManifestSegment { + proxy_seq, + duration_ms: 4_000, + uri: format!("/hls/shared/live/session/lease/{proxy_seq:06}.ts").into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }) + .collect::>(), + ), + discontinuity_sequence: 0, + target_duration_ms: 4_000, + playlist_duration_ms: 12_000, + last_visible_media_end_ms: 12_000, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + }; + let mut lease = HlsAccessLease::pending( + HlsAccessLeaseId("startup-lease".to_string()), + HlsPlaybackFamilyKey::new("user", "player"), + proxy_session_id.clone(), + "user".to_string(), + "user-session".to_string(), + 1, + "12345".to_string(), + 1, + 10, + 60_000, + ); + lease.state = super::super::HlsAccessLeaseState::Activated; + lease.active_until_ms = Some(60_000); + lease.last_manifest_snapshot = Some(snapshot); + lease +} + +fn encrypted_terminal_evidence_manifest( + proxy_session_id: &ProxySessionId, + resource_id: &TransientResourceId, +) -> HlsLeaseManifestSnapshot { + let encryption = Arc::new(HlsEncryptionSignature { + method: "AES-128".to_string(), + key_uri: Some(format!("/hls/shared/live/{}/lease/r/{}.key", proxy_session_id.0, resource_id.0)), + iv: Some("0x00000000000000000000000000000001".to_string()), + key_format: Some("identity".to_string()), + key_format_versions: Some("1".to_string()), + can_reset_to_clear: true, + }); + HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(1), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 1, + delivered_at_ms: 0, + first_proxy_seq: 1, + last_proxy_seq: 1, + visible_segments: Arc::from([HlsLeaseManifestSegment { + proxy_seq: 1, + duration_ms: 4_000, + uri: "/hls/shared/live/session/lease/1.ts".to_string().into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: Some(encryption.clone()), + }]), + discontinuity_sequence: 0, + target_duration_ms: 4_000, + playlist_duration_ms: 4_000, + last_visible_media_end_ms: 4_000, + active_map: None, + active_encryption: Some(encryption), + container: HlsMediaContainer::MpegTs, + } +} + +mod admission; +mod lifecycle; +mod playlist; +mod policy; +mod publication; +mod recovery; +mod streaming; +mod transport; diff --git a/backend/hls/src/gc/tests/admission.rs b/backend/hls/src/gc/tests/admission.rs new file mode 100644 index 000000000..8a2354afa --- /dev/null +++ b/backend/hls/src/gc/tests/admission.rs @@ -0,0 +1,83 @@ +use super::*; + +#[tokio::test] +async fn cache_root_handoff_invalidates_armed_switch_cleanup_reservations() { + let old_root = tempfile::tempdir().expect("old cache root"); + let new_root = tempfile::tempdir().expect("new cache root"); + let (gc, session) = gc_with_session(&old_root).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let key = crate::SegmentCacheKey::new(proxy_session_id, 1, "ts"); + let cleanup = gc.reserve_switch_segment_cleanup(key.clone()).expect("switch cleanup reservation"); + assert_eq!(gc.cache_deletion_queue_usage(), (0, 1)); + + assert!(gc.update_cache_path(new_root.path()).await); + assert_eq!(gc.cache_deletion_queue_usage(), (0, 0)); + drop(cleanup); + assert_eq!(gc.cache_deletion_queue_usage(), (0, 0)); + + gc.cache.write_bytes_and_commit(&key, b"new-root-sentinel").await.expect("new-root sentinel writes"); + let report = gc.run_once(0).await.expect("new-root gc runs"); + + assert_eq!(report.cache_object_deletions_succeeded, 0); + let metadata = gc.cache.metadata(&key).await.expect("new-root metadata reads").expect("sentinel remains"); + assert_eq!(tokio::fs::read(metadata.path).await.expect("sentinel reads"), b"new-root-sentinel"); +} + +#[tokio::test] +async fn concurrent_projected_pressure_is_serialized_without_over_deletion() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + } + cache_selected_ready_segments(&gc, &session, &[1, 2], b"0123456789").await; + gc.cache.update_cache_limits(100, 25); + let (third, fourth, proxy_session_id) = { + let session = session.read().await; + ( + session.segments.get(&3).expect("third").cache_key.clone(), + session.segments.get(&4).expect("fourth").cache_key.clone(), + session.proxy_session_id.clone(), + ) + }; + let first_staged = gc + .cache + .stage_temp_with_deadline(&third, &b"abcdefghij"[..], tokio::time::Instant::now() + Duration::from_mins(1)) + .await + .expect("first object stages"); + let second_staged = gc + .cache + .stage_temp_with_deadline(&fourth, &b"klmnopqrst"[..], tokio::time::Instant::now() + Duration::from_mins(1)) + .await + .expect("second object stages"); + let commit_barrier = Arc::new(Barrier::new(2)); + let first_cache = Arc::clone(&gc.cache); + let first_barrier = Arc::clone(&commit_barrier); + let first = tokio::spawn(async move { + first_barrier.wait().await; + first_cache.commit_staged(&third, first_staged).await + }); + let second_cache = Arc::clone(&gc.cache); + let second = tokio::spawn(async move { + commit_barrier.wait().await; + second_cache.commit_staged(&fourth, second_staged).await + }); + + let (first, second) = tokio::join!(first, second); + + let first = first.expect("first task"); + let second = second.expect("second task"); + + assert!(first.is_ok(), "first cache write failed: {first:?}"); + assert!(second.is_ok(), "second cache write failed: {second:?}"); + let session = session.read().await; + assert!(!session.segments.contains_key(&1)); + assert!(!session.segments.contains_key(&2)); + assert!(session.segments.contains_key(&3)); + assert!(session.segments.contains_key(&4)); + drop(session); + let usage = gc.cache.capacity_usage(&proxy_session_id).await.expect("capacity usage"); + assert_eq!(usage.session_bytes, 20); + assert!(usage.global_bytes <= 25); +} diff --git a/backend/hls/src/gc/tests/lifecycle.rs b/backend/hls/src/gc/tests/lifecycle.rs new file mode 100644 index 000000000..af6503d9f --- /dev/null +++ b/backend/hls/src/gc/tests/lifecycle.rs @@ -0,0 +1,271 @@ +use super::*; + +#[test] +fn terminal_tail_protects_live_base_until_lease_protection_is_released() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + session.install_terminal_tail_protection( + lease_id.clone(), + HlsTerminalTailProtection { + generation: HlsTerminalTailGeneration(1), + base_proxy_seqs: Arc::from([41, 42]), + key_bindings: Arc::from([]), + }, + ); + + let protected = ProtectedSet::from_session(&session); + assert!(protected.segment_proxy_seqs.contains(&41)); + assert!(protected.segment_proxy_seqs.contains(&42)); + + assert!(session.remove_terminal_tail_protection(&lease_id).is_some()); + let released = ProtectedSet::from_session(&session); + assert!(!released.segment_proxy_seqs.contains(&41)); + assert!(!released.segment_proxy_seqs.contains(&42)); +} + +#[tokio::test] +async fn gc_reclaims_terminal_base_only_after_lease_protection_release() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let lease_id = HlsAccessLeaseId("terminal-lease".to_string()); + let cache_key = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let segment = session.segments.get_mut(&1).expect("segment"); + gc.cache.write_bytes_and_commit(&segment.cache_key, b"segment-body").await.expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + let cache_key = segment.cache_key.clone(); + session.install_terminal_tail_protection( + lease_id.clone(), + HlsTerminalTailProtection { + generation: HlsTerminalTailGeneration(1), + base_proxy_seqs: Arc::from([1_u64]), + key_bindings: Arc::from([]), + }, + ); + cache_key + }; + + let protected_report = gc.run_once(10_000).await.expect("protected gc should run"); + assert_eq!(protected_report.segments_deleted_duration, 0); + assert!(session.read().await.segments.contains_key(&1)); + assert!(gc.cache.metadata(&cache_key).await.expect("metadata reads").is_some()); + + assert!(session.write().await.remove_terminal_tail_protection(&lease_id).is_some()); + let released_report = gc.run_once(10_001).await.expect("released gc should run"); + + assert_eq!(released_report.segments_deleted_duration, 1); + assert!(gc.cache.metadata(&cache_key).await.expect("metadata reads").is_none()); +} + +#[test] +fn gc_report_logs_when_cleanup_or_invalidation_happened() { + let mut report = GarbageCollectionReport { temp_files_deleted: 1, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = GarbageCollectionReport { stale_queue_entries_removed: 1, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = GarbageCollectionReport { segments_deleted_duration: 1, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = GarbageCollectionReport { maps_deleted: 1, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = GarbageCollectionReport { sessions_deleted: 1, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = GarbageCollectionReport { transient_resources_pruned: 1, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = GarbageCollectionReport { secret_cache_invalidated: true, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); + + report = GarbageCollectionReport { secret_cache_invalidation_deferred: true, ..GarbageCollectionReport::default() }; + assert!(report.did_cleanup_or_invalidate()); +} + +#[tokio::test] +async fn protected_startup_tail_defers_until_playback_releases_fifo_head() { + const SESSION_LIMIT: u64 = 125 * 1_024 * 1_024; + const RESIDENT_SIZES: [u64; 5] = [21_000_000, 22_000_000, 21_500_000, 22_500_000, 22_221_044]; + const STAGED_SIZE: u64 = 22_551_164; + + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let (proxy_session_id, keys) = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let keys = (1_u64..=6) + .map(|proxy_seq| session.segments.get(&proxy_seq).expect("segment").cache_key.clone()) + .collect::>(); + (session.proxy_session_id.clone(), keys) + }; + for (key, size) in keys.iter().take(5).zip(RESIDENT_SIZES) { + commit_sparse_segment(&gc, key, size).await; + } + { + let mut session = session.write().await; + for (proxy_seq, content_length) in (1_u64..=5).zip(RESIDENT_SIZES) { + session.segments.get_mut(&proxy_seq).expect("resident segment").status = + SegmentCacheStatus::Ready { content_length, ready_at_ms: proxy_seq }; + } + session.segments.get_mut(&6).expect("tail segment").status = + SegmentCacheStatus::Queued { priority: SegmentFetchPriority::Prefetch, queued_at_ms: 1 }; + session.render_and_store_manifest(10).expect("six-segment canonical render"); + } + + let mut lease = activated_startup_lease(&proxy_session_id); + let access_leases = Arc::new(RwLock::new(HlsAccessLeaseStore::default())); + assert!(access_leases.write().await.prepare_access_lease(lease.clone())); + gc.install_access_leases(&access_leases); + gc.cache.update_cache_limits(512 * 1_024 * 1_024, SESSION_LIMIT); + + for key in keys.iter().take(3) { + assert!(gc.cache.metadata(key).await.expect("visible metadata").is_some()); + } + let deferred = gc + .cache + .ensure_projected_write_capacity(&keys[5], STAGED_SIZE) + .await + .expect_err("fully protected startup window defers the tail"); + let capacity = super::super::super::cache::hls_cache_capacity_from_io(&deferred).expect("typed capacity deferral"); + assert_eq!(capacity.configured_session_bytes(), SESSION_LIMIT); + assert_eq!(capacity.current_session_bytes(), RESIDENT_SIZES.into_iter().sum::()); + assert_eq!(capacity.staged_bytes(), STAGED_SIZE); + assert_eq!(capacity.required_session_bytes(), 700_208); + assert_eq!(capacity.protected_working_set_bytes(), RESIDENT_SIZES.into_iter().sum::()); + assert_eq!(capacity.reclaimable_bytes(), 0); + assert!(gc.cache.metadata(&keys[5]).await.expect("tail metadata").is_none()); + + { + let mut session = session.write().await; + session.segments.get_mut(&6).expect("tail segment").status = + SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Prefetch, deferred_at_ms: 11 }; + let safe_render = session.render_and_store_manifest(11).expect("deferred tail truncates safely"); + assert_eq!(safe_render.last_proxy_seq, 5); + assert!(!safe_render.body.contains("000006.ts")); + } + + lease.playback_cursor.highest_contiguous_completed_proxy_seq = Some(1); + assert!(access_leases.write().await.prepare_access_lease(lease)); + gc.cache.notify_capacity_protection_changed(); + commit_sparse_segment(&gc, &keys[5], STAGED_SIZE).await; + { + let mut session = session.write().await; + session.segments.get_mut(&6).expect("tail segment").status = + SegmentCacheStatus::Ready { content_length: STAGED_SIZE, ready_at_ms: 12 }; + let resumed = session.render_and_store_manifest(12).expect("window resumes after reclamation"); + assert_eq!(resumed.first_proxy_seq, 2); + assert_eq!(resumed.last_proxy_seq, 6); + } + assert!(gc.cache.metadata(&keys[0]).await.expect("released head metadata").is_none()); + assert!(gc.cache.metadata(&keys[5]).await.expect("resumed tail metadata").is_some()); + let usage = gc.cache.capacity_usage(&proxy_session_id).await.expect("bounded usage"); + assert!(usage.session_bytes <= SESSION_LIMIT); +} + +#[tokio::test] +async fn terminal_evidence_pins_ready_key_object_and_mapping_across_gc_until_release() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let (segment_cache_key, proxy_session_id, resource, resource_id, object_lookup_key, object_fetch_token) = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let segment_cache_key = session.segments.get(&1).expect("segment").cache_key.clone(); + let resource = TransientResourceRef::new( + TransientResourceKind::Key, + "http://origin.example.com/live/key.bin", + b"secret", + 0, + 10, + Some("key".to_string()), + ); + let resource_id = resource.id.clone(); + session.transient.upsert_resources([resource]); + let resource = session.transient.resources.get(&resource_id).expect("registered key resource").clone(); + session.segments.get_mut(&1).expect("encrypted segment").encryption = Some(HlsSegmentEncryption { + resource_id: resource_id.clone(), + resource_extension: "key".to_string(), + iv: Some("0x00000000000000000000000000000001".to_string()), + key_format: Some("identity".to_string()), + key_format_versions: Some("1".to_string()), + }); + let proxy_session_id = session.proxy_session_id.clone(); + let object_lookup_key = TransientPassthroughState::transient_object_key(&proxy_session_id, &resource_id, "key"); + let object_fetch_token = match session.transient.begin_object_fetch(&proxy_session_id, &resource, "key", 0, 10) + { + TransientObjectFetchDecision::Fetch(token) => token, + TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { + panic!("new key object starts one physical cache fill") + } + }; + (segment_cache_key, proxy_session_id, resource, resource_id, object_lookup_key, object_fetch_token) + }; + gc.cache.write_bytes_and_commit(&segment_cache_key, b"ready-media").await.expect("media cache write"); + gc.cache + .write_bytes_and_commit(object_fetch_token.cache_key(), b"0123456789abcdef") + .await + .expect("key cache write"); + { + let mut session = session.write().await; + session.segments.get_mut(&1).expect("segment").status = + SegmentCacheStatus::Ready { content_length: 11, ready_at_ms: 0 }; + assert!(session.commit_transient_object_ready_if_current( + resource.kind, + &object_fetch_token, + "application/octet-stream".to_string(), + 16, + 0, + 10, + )); + } + let manifest = encrypted_terminal_evidence_manifest(&proxy_session_id, &resource_id); + + let evidence = prepare_terminal_base_evidence(&session, &gc.cache, &manifest, 5).await; + assert!(evidence.availability()[0].required_key_ready); + { + let mut session = session.write().await; + session.transient.resources.get_mut(&resource_id).expect("key mapping").expires_at_ms = 0; + session.transient.object_cache.get_mut(&object_lookup_key).expect("key object").expires_at_ms = 0; + } + + let pinned_report = gc.run_once(20).await.expect("pinned GC"); + assert_eq!(pinned_report.transient_resources_pruned, 0); + assert_eq!(pinned_report.transient_objects_deleted, 0); + { + let session = session.read().await; + assert!(session.transient.resources.contains_key(&resource_id)); + assert!(session.transient.object_cache.contains_key(&object_lookup_key)); + } + + evidence.release(); + let generation_before_release_gc = session.read().await.activity.media_readiness_generation; + let released_report = gc.run_once(21).await.expect("released GC"); + assert_eq!(released_report.transient_resources_pruned, 1); + assert_eq!(released_report.transient_objects_deleted, 1); + let session = session.read().await; + assert_eq!( + session.activity.media_readiness_generation, + generation_before_release_gc.saturating_add(1), + "mapping and READY key object removal advance readiness once per session sweep" + ); + assert!(!session.transient.resources.contains_key(&resource_id)); + assert!(!session.transient.object_cache.contains_key(&object_lookup_key)); +} + +#[tokio::test] +async fn session_gc_removes_idle_session_without_activity() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let (key, proxy_session_id) = { + let session = session.read().await; + (session.key.clone(), session.proxy_session_id.clone()) + }; + + let report = gc.run_once(2_000).await.expect("gc should run"); + + assert_eq!(report.sessions_deleted, 1); + assert!(gc.sessions.get_by_key(&key).await.is_none()); + assert!(gc.sessions.get_by_proxy_session_id(&proxy_session_id).await.is_none()); +} diff --git a/backend/hls/src/gc/tests/playlist.rs b/backend/hls/src/gc/tests/playlist.rs new file mode 100644 index 000000000..2c9cd5bea --- /dev/null +++ b/backend/hls/src/gc/tests/playlist.rs @@ -0,0 +1,26 @@ +use super::*; + +#[tokio::test] +async fn transient_resource_mappings_expire_unless_active() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let resource_id = { + let mut session = session.write().await; + let resource = crate::TransientResourceRef::new( + crate::TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 0, + 10, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + session.transient.upsert_resources([resource]); + resource_id + }; + + let report = gc.run_once(20).await.expect("gc should run"); + + assert_eq!(report.transient_resources_pruned, 1); + assert!(!session.read().await.transient.resources.contains_key(&resource_id)); +} diff --git a/backend/hls/src/gc/tests/policy.rs b/backend/hls/src/gc/tests/policy.rs new file mode 100644 index 000000000..ef5d20780 --- /dev/null +++ b/backend/hls/src/gc/tests/policy.rs @@ -0,0 +1,670 @@ +use super::*; + +#[test] +fn gc_report_is_quiet_when_nothing_changed() { + let report = GarbageCollectionReport::default(); + + assert!(!report.did_cleanup_or_invalidate()); +} + +#[tokio::test] +async fn gc_keeps_active_readers() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let segment = session.segments.get_mut(&1).expect("segment"); + gc.cache.write_bytes_and_commit(&segment.cache_key, b"segment-body").await.expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + segment.access.reader_started(1); + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 0); + assert!(session.read().await.segments.contains_key(&1)); +} + +#[tokio::test] +async fn duration_gc_deletes_old_unprotected_segments() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let segment = session.segments.get_mut(&1).expect("segment"); + gc.cache.write_bytes_and_commit(&segment.cache_key, b"segment-body").await.expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 1); + assert!(!session.read().await.segments.contains_key(&1)); +} + +#[tokio::test] +async fn cache_delete_failure_does_not_block_later_deletes_and_retries_in_active_session() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let (failed_key, successful_key) = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + session.last_client_access_at_ms = 10_000; + for proxy_seq in [1, 2] { + session.segments.get_mut(&proxy_seq).expect("segment").status = + SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + } + ( + session.segments.get(&1).expect("failed segment").cache_key.clone(), + session.segments.get(&2).expect("successful segment").cache_key.clone(), + ) + }; + gc.cache.write_bytes_and_commit(&failed_key, b"segment-body").await.expect("failed fixture writes"); + gc.cache.write_bytes_and_commit(&successful_key, b"segment-body").await.expect("successful fixture writes"); + let failed_path = gc.cache.object_path(&failed_key); + tokio::fs::remove_file(&failed_path).await.expect("replace failed fixture"); + tokio::fs::create_dir(&failed_path).await.expect("directory makes remove_file fail"); + + let first_report = gc.run_once(10_000).await.expect("first gc run"); + + assert_eq!(first_report.cache_object_deletions_planned, 2); + assert_eq!(first_report.cache_object_deletions_succeeded, 1); + assert_eq!(first_report.cache_object_deletions_deferred, 1); + assert_eq!(first_report.segments_deleted_duration, 1); + assert!(gc.cache.metadata(&successful_key).await.expect("metadata reads").is_none()); + assert_eq!(gc.pending_cache_deletion_count(), 1); + assert!(!session.read().await.segments.contains_key(&1)); + assert!(!session.read().await.segments.contains_key(&2)); + + tokio::fs::remove_dir(&failed_path).await.expect("remove failing fixture"); + tokio::fs::write(&failed_path, b"replacement-body").await.expect("replacement fixture writes"); + + let retry_report = gc.run_once(10_001).await.expect("retry gc run"); + + assert_eq!(retry_report.cache_object_deletions_planned, 0); + assert_eq!(retry_report.cache_object_deletions_succeeded, 1); + assert_eq!(retry_report.cache_object_deletions_deferred, 0); + assert_eq!(retry_report.segments_deleted_duration, 1); + assert_eq!(gc.pending_cache_deletion_count(), 0); + assert!(gc.cache.metadata(&failed_key).await.expect("metadata reads").is_none()); +} + +#[tokio::test] +async fn ordinary_gc_batch_preserves_bounded_switch_rollback_headroom() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let batch = gc.reserve_cache_deletion_batch(); + assert_eq!(batch.remaining_capacity(), MAX_PENDING_CACHE_DELETIONS - SWITCH_CACHE_CLEANUP_HEADROOM); + + let segment = gc.reserve_switch_segment_cleanup(crate::SegmentCacheKey::new(proxy_session_id.clone(), 1, "ts")); + let map = gc.reserve_switch_map_cleanup(crate::MapCacheKey::new(proxy_session_id, ProxyMapId(1), "mp4")); + + assert!(segment.is_some()); + assert!(map.is_some()); + let queue = gc.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + assert!(queue.pending.len().saturating_add(queue.reserved_slots) <= MAX_PENDING_CACHE_DELETIONS); +} + +#[tokio::test] +async fn duration_gc_stops_at_protected_fifo_head() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for proxy_seq in [1, 2] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + } + session.segments.get_mut(&1).expect("head segment").access.reader_started(1); + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 0); + let session = session.read().await; + assert!(session.segments.contains_key(&1)); + assert!(session.segments.contains_key(&2)); +} + +#[tokio::test] +async fn duration_gc_stops_at_not_expired_fifo_head() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for (proxy_seq, ready_at_ms) in [(1, 9_950), (2, 0)] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; + } + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 0); + let session = session.read().await; + assert!(session.segments.contains_key(&1)); + assert!(session.segments.contains_key(&2)); +} + +#[tokio::test] +async fn session_size_gc_deletes_oldest_unprotected_segment() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + update_gc_policy(&gc, |policy| policy.cache_bytes_per_session = 20); + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for proxy_seq in [1, 2, 3] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: proxy_seq }; + } + } + + let report = gc.run_once(100).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_size_session, 2); + let session = session.read().await; + assert!(!session.segments.contains_key(&1)); + assert!(!session.segments.contains_key(&2)); + assert!(session.segments.contains_key(&3)); +} + +#[tokio::test] +async fn projected_session_pressure_reclaims_before_the_limit_is_crossed() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + } + cache_selected_ready_segments(&gc, &session, &[1, 2], b"0123456789").await; + gc.cache.update_cache_limits(100, 25); + let target = session.read().await.segments.get(&3).expect("target").cache_key.clone(); + + let committed = gc.cache.write_bytes_and_commit(&target, b"abcdefghij").await.expect("projected reclaim"); + + assert_eq!(committed.size, 10); + let session_guard = session.read().await; + assert!(!session_guard.segments.contains_key(&1), "oldest unprotected FIFO head is reclaimed"); + assert!(session_guard.segments.contains_key(&2)); + let proxy_session_id = session_guard.proxy_session_id.clone(); + drop(session_guard); + assert!(gc.cache.metadata(&target).await.expect("target metadata").is_some()); + let usage = gc.cache.capacity_usage(&proxy_session_id).await.expect("capacity usage"); + assert_eq!(usage.session_bytes, 20); + assert!(usage.session_bytes <= 25); + assert!(!gc.cache.has_active_temp_files()); +} + +#[tokio::test] +async fn projected_pressure_never_skips_a_protected_fifo_head() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + } + cache_selected_ready_segments(&gc, &session, &[1, 2], b"0123456789").await; + { + let mut session = session.write().await; + session.segments.get_mut(&1).expect("head").access.reader_started(1); + } + gc.cache.update_cache_limits(100, 25); + let target = session.read().await.segments.get(&3).expect("target").cache_key.clone(); + + let error = gc.cache.write_bytes_and_commit(&target, b"abcdefghij").await.expect_err("head is protected"); + + assert!(matches!( + HlsOriginResourceFetchError::cache_commit(&error), + HlsOriginResourceFetchError::LocalCacheCapacity { .. } + )); + let session = session.read().await; + assert!(session.segments.contains_key(&1)); + assert!(session.segments.contains_key(&2)); + drop(session); + assert!(gc.cache.metadata(&target).await.expect("target metadata").is_none()); + assert!(!gc.cache.has_active_temp_files()); +} + +#[tokio::test] +async fn projected_global_pressure_uses_oldest_session_fifo_head() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, first_session) = gc_with_session(&temp_dir).await; + let second_session = gc.sessions.get_or_create_session(HlsSessionKey::new(1, "67890"), b"secret", 0).await; + for session in [&first_session, &second_session] { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + } + cache_selected_ready_segments(&gc, &first_session, &[1], b"12345678").await; + cache_selected_ready_segments(&gc, &second_session, &[1], b"abcdefgh").await; + { + let mut first = first_session.write().await; + let SegmentCacheStatus::Ready { ready_at_ms, .. } = &mut first.segments.get_mut(&1).expect("first head").status + else { + panic!("first head must be ready"); + }; + *ready_at_ms = 1; + } + { + let mut second = second_session.write().await; + let SegmentCacheStatus::Ready { ready_at_ms, .. } = + &mut second.segments.get_mut(&1).expect("second head").status + else { + panic!("second head must be ready"); + }; + *ready_at_ms = 2; + } + gc.cache.update_cache_limits(20, 100); + let (target, proxy_session_id) = { + let session = second_session.read().await; + (session.segments.get(&2).expect("target").cache_key.clone(), session.proxy_session_id.clone()) + }; + + gc.cache.write_bytes_and_commit(&target, b"ijklmnop").await.expect("global projected reclaim"); + + assert!(!first_session.read().await.segments.contains_key(&1)); + assert!(second_session.read().await.segments.contains_key(&1)); + let usage = gc.cache.capacity_usage(&proxy_session_id).await.expect("capacity usage"); + assert_eq!(usage.global_bytes, 16); + assert!(usage.global_bytes <= 20); +} + +#[tokio::test] +async fn high_bitrate_sequence_advances_beyond_the_512_mib_aggregate_wall() { + use std::fmt::Write as _; + + const SESSION_LIMIT: u64 = 512 * 1024 * 1024; + const OLD_SEGMENT_BYTES: u64 = 21_825_922; + const NEW_SEGMENT_BYTES: u64 = 22_000_000; + const READY_SEGMENTS: u64 = 24; + + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let mut manifest = String::from("#EXTM3U\n#EXT-X-TARGETDURATION:10\n#EXT-X-MEDIA-SEQUENCE:1\n"); + for origin_seq in 1..=READY_SEGMENTS.saturating_add(1) { + writeln!(manifest, "#EXTINF:10.0,\n{origin_seq}.ts").expect("writing to a String cannot fail"); + } + { + let mut session = session.write().await; + session.proxy_next_seq = Some(1); + session.apply_origin_manifest(&normal_manifest(&manifest)).expect("high-bitrate manifest maps"); + } + let keys = { + let session = session.read().await; + (1..=READY_SEGMENTS.saturating_add(1)) + .map(|proxy_seq| session.segments.get(&proxy_seq).expect("segment").cache_key.clone()) + .collect::>() + }; + for key in keys.iter().take(usize::try_from(READY_SEGMENTS).unwrap_or(usize::MAX)) { + commit_sparse_segment(&gc, key, OLD_SEGMENT_BYTES).await; + } + { + let mut session = session.write().await; + for proxy_seq in 1..=READY_SEGMENTS { + session.segments.get_mut(&proxy_seq).expect("ready segment").status = + SegmentCacheStatus::Ready { content_length: OLD_SEGMENT_BYTES, ready_at_ms: proxy_seq }; + } + } + gc.cache.update_cache_limits(2 * SESSION_LIMIT, SESSION_LIMIT); + let target = keys.last().expect("target key"); + + commit_sparse_segment(&gc, target, NEW_SEGMENT_BYTES).await; + { + let mut session = session.write().await; + session.segments.get_mut(&READY_SEGMENTS.saturating_add(1)).expect("newest segment").status = + SegmentCacheStatus::Ready { content_length: NEW_SEGMENT_BYTES, ready_at_ms: READY_SEGMENTS + 1 }; + } + + let aggregate_sequence_bytes = READY_SEGMENTS.saturating_mul(OLD_SEGMENT_BYTES).saturating_add(NEW_SEGMENT_BYTES); + assert!(aggregate_sequence_bytes > SESSION_LIMIT); + let session_guard = session.read().await; + assert!(!session_guard.segments.contains_key(&1)); + assert!(session_guard.segments.contains_key(&READY_SEGMENTS.saturating_add(1))); + let proxy_session_id = session_guard.proxy_session_id.clone(); + drop(session_guard); + let usage = gc.cache.capacity_usage(&proxy_session_id).await.expect("capacity usage"); + assert_eq!(usage.session_bytes, READY_SEGMENTS.saturating_sub(1) * OLD_SEGMENT_BYTES + NEW_SEGMENT_BYTES); + assert!(usage.session_bytes <= SESSION_LIMIT); +} + +#[tokio::test] +async fn session_size_gc_stops_at_protected_fifo_head() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + update_gc_policy(&gc, |policy| policy.cache_bytes_per_session = 20); + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for proxy_seq in [1, 2, 3] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: proxy_seq }; + } + session.segments.get_mut(&1).expect("head segment").access.reader_started(1); + } + + let report = gc.run_once(100).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_size_session, 0); + let session = session.read().await; + assert!(session.segments.contains_key(&1)); + assert!(session.segments.contains_key(&2)); + assert!(session.segments.contains_key(&3)); +} + +#[tokio::test] +async fn protected_map_remains_and_unreferenced_map_is_deleted() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n1.m4s\n#EXTINF:4.0,\n2.m4s\n#EXTINF:4.0,\n3.m4s\n", + ); + { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest should map"); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + } + let protected_map_id = ProxyMapId(0); + session.maps.get_mut(&protected_map_id).expect("map").status = + MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; + let unreferenced_key = OriginMapKey { + origin_epoch: 0, + resolved_origin_uri: "http://origin.example.com/live/unused.mp4".to_string(), + byte_range: None, + }; + let unreferenced_map = + crate::MapEntry::new(&session.proxy_session_id, ProxyMapId(1), unreferenced_key.clone(), "mp4".to_string()); + session.maps.insert(ProxyMapId(1), unreferenced_map); + session.origin_map_to_proxy.insert(unreferenced_key, ProxyMapId(1)); + session.maps.get_mut(&ProxyMapId(1)).expect("map").status = + MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; + session.render_and_store_manifest(1).expect("manifest should render"); + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.maps_deleted, 1); + let session = session.read().await; + assert!(session.maps.contains_key(&ProxyMapId(0))); + assert!(!session.maps.contains_key(&ProxyMapId(1))); +} + +#[tokio::test] +async fn map_referenced_by_remaining_segment_survives_fifo_gc() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let manifest = normal_manifest("#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\n0.m4s\n#EXTINF:4.0,\n1.m4s\n"); + { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest should map"); + session.maps.get_mut(&ProxyMapId(0)).expect("map").status = + MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; + for (proxy_seq, ready_at_ms) in [(0, 0), (1, 9_950)] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; + } + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 1); + assert_eq!(report.maps_deleted, 0); + let session = session.read().await; + assert!(!session.segments.contains_key(&0)); + assert!(session.segments.contains_key(&1)); + assert!(session.maps.contains_key(&ProxyMapId(0))); +} + +#[tokio::test] +async fn secret_fingerprint_mismatch_invalidates_cache_and_sessions() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let segment = session.segments.get_mut(&1).expect("segment"); + gc.cache.write_bytes_and_commit(&segment.cache_key, b"segment-body").await.expect("cache write should succeed"); + } + gc.cache.write_rewrite_secret_fingerprint("mismatch").await.expect("marker write"); + + let report = gc.run_once(1).await.expect("gc should run"); + + assert!(report.secret_cache_invalidated); + assert!(!report.secret_cache_invalidation_deferred); + assert_eq!(gc.sessions.len().await, 0); + let rewrite_secret_fingerprint = gc.rewrite_secret_fingerprint(); + assert_eq!( + gc.cache.read_rewrite_secret_fingerprint().await.expect("marker read").as_deref(), + Some(rewrite_secret_fingerprint.as_str()) + ); +} + +#[tokio::test] +async fn secret_fingerprint_mismatch_with_active_temp_defers_invalidation() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let cache_key = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + session.segments.get(&1).expect("segment").cache_key.clone() + }; + gc.cache.write_rewrite_secret_fingerprint("mismatch").await.expect("marker write"); + let staged = gc + .cache + .stage_temp_with_deadline(&cache_key, &b"done"[..], tokio::time::Instant::now() + Duration::from_mins(1)) + .await + .expect("cache object stages"); + assert!(gc.cache.has_active_temp_files()); + + let report = gc.run_once(1).await.expect("gc should run"); + + assert!(!report.secret_cache_invalidated); + assert!(report.secret_cache_invalidation_deferred); + assert_eq!(gc.sessions.len().await, 1); + assert_eq!(gc.cache.read_rewrite_secret_fingerprint().await.expect("marker read").as_deref(), Some("mismatch")); + assert!(gc.cache.has_active_temp_files()); + gc.cache.commit_staged(&cache_key, staged).await.expect("staged object commits"); + assert!(gc.cache.metadata(&cache_key).await.expect("metadata should read").is_some()); +} + +#[tokio::test] +async fn deferred_secret_fingerprint_invalidation_runs_after_temp_commit() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let cache_key = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + session.segments.get(&1).expect("segment").cache_key.clone() + }; + gc.cache.write_rewrite_secret_fingerprint("mismatch").await.expect("marker write"); + let staged = gc + .cache + .stage_temp_with_deadline(&cache_key, &b"done"[..], tokio::time::Instant::now() + Duration::from_mins(1)) + .await + .expect("cache object stages"); + assert!(gc.run_once(1).await.expect("first gc should run").secret_cache_invalidation_deferred); + gc.cache.commit_staged(&cache_key, staged).await.expect("staged object commits"); + + let report = gc.run_once(2).await.expect("second gc should run"); + + assert!(report.secret_cache_invalidated); + assert!(!report.secret_cache_invalidation_deferred); + assert_eq!(gc.sessions.len().await, 0); + assert_eq!(gc.cache.metadata(&cache_key).await.expect("metadata should read"), None); + let rewrite_secret_fingerprint = gc.rewrite_secret_fingerprint(); + assert_eq!( + gc.cache.read_rewrite_secret_fingerprint().await.expect("marker read").as_deref(), + Some(rewrite_secret_fingerprint.as_str()) + ); +} + +#[tokio::test] +async fn global_candidate_selection_skips_a_session_that_failed_revalidation() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, first_session) = gc_with_session(&temp_dir).await; + let second_session = gc.sessions.get_or_create_session(HlsSessionKey::new(2, "12345"), b"secret", 0).await; + + for (session, ready_at_ms) in [(&first_session, 1), (&second_session, 2)] { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + session.segments.get_mut(&1).expect("head").status = + SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; + } + let first_proxy_session_id = first_session.read().await.proxy_session_id.clone(); + let sessions = [Arc::clone(&first_session), Arc::clone(&second_session)]; + + let first = + oldest_global_fifo_head_candidate(&sessions, &gc.cache, None, &HashSet::new()).await.expect("candidate"); + assert_eq!(first.proxy_session_id, first_proxy_session_id); + + let skipped = HashSet::from([first_proxy_session_id]); + let next = oldest_global_fifo_head_candidate(&sessions, &gc.cache, None, &skipped).await.expect("next candidate"); + assert_eq!(next.proxy_session_id, second_session.read().await.proxy_session_id); +} + +#[tokio::test] +async fn global_size_gc_deletes_oldest_unprotected_segments() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, first_session) = gc_with_session(&temp_dir).await; + update_gc_policy(&gc, |policy| policy.cache_bytes_global = 24); + let second_session = gc.sessions.get_or_create_session(HlsSessionKey::new(2, "12345"), b"secret", 0).await; + + for session in [&first_session, &second_session] { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for proxy_seq in [1, 2, 3] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: proxy_seq }; + } + } + + let report = gc.run_once(100).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_size_global, 4); + let remaining_size = { + let first = first_session.read().await; + let second = second_session.read().await; + super::super::selection::session_cache_size(&first) + super::super::selection::session_cache_size(&second) + }; + assert!(remaining_size <= 24); +} + +#[tokio::test] +async fn global_size_gc_uses_only_current_fifo_heads() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, first_session) = gc_with_session(&temp_dir).await; + update_gc_policy(&gc, |policy| policy.cache_bytes_global = 24); + let second_session = gc.sessions.get_or_create_session(HlsSessionKey::new(2, "12345"), b"secret", 0).await; + + for session in [&first_session, &second_session] { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + for proxy_seq in [1, 2, 3] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + gc.cache + .write_bytes_and_commit(&segment.cache_key, b"segment-body") + .await + .expect("cache write should succeed"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: proxy_seq }; + } + } + first_session.write().await.segments.get_mut(&1).expect("head segment").access.reader_started(1); + + let report = gc.run_once(100).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_size_global, 3); + let first = first_session.read().await; + assert!(first.segments.contains_key(&1)); + assert!(first.segments.contains_key(&2)); + assert!(first.segments.contains_key(&3)); + let second = second_session.read().await; + assert!(!second.segments.contains_key(&1)); + assert!(!second.segments.contains_key(&2)); + assert!(!second.segments.contains_key(&3)); +} + +#[tokio::test] +async fn temp_file_gc_deletes_old_tmp_files() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let temp_path = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let path = gc.cache.object_path(&session.segments.get(&1).expect("segment").cache_key); + let parent = path.parent().expect("cache object has parent"); + tokio::fs::create_dir_all(parent).await.expect("parent dir"); + parent.join("000001.ts.tmp.old") + }; + tokio::fs::write(&temp_path, b"partial").await.expect("temp write"); + let old_time = filetime::FileTime::from_unix_time(1, 0); + filetime::set_file_mtime(&temp_path, old_time).expect("set mtime"); + + let report = gc.run_once(1).await.expect("gc should run"); + + assert_eq!(report.temp_files_deleted, 1); + assert!(!temp_path.exists()); +} + +#[tokio::test] +async fn session_gc_keeps_idle_session_with_active_temp_file() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let (cache_key, proxy_session_id) = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + (session.segments.get(&1).expect("segment").cache_key.clone(), session.proxy_session_id.clone()) + }; + let staged = gc + .cache + .stage_temp_with_deadline(&cache_key, &b"done"[..], tokio::time::Instant::now() + Duration::from_mins(1)) + .await + .expect("cache object stages"); + assert!(gc.cache.has_active_temp_files_for_session(&proxy_session_id)); + + let report = gc.run_once(2_000).await.expect("gc should run"); + + assert_eq!(report.sessions_deleted, 0); + assert!(!session.read().await.is_gc_marked_for_removal()); + assert_eq!(gc.sessions.len().await, 1); + gc.cache.commit_staged(&cache_key, staged).await.expect("staged object commits"); +} + +#[tokio::test] +async fn session_gc_final_recheck_keeps_new_activity() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + session.write().await.last_client_access_at_ms = 1_500; + let mut report = super::super::GarbageCollectionReport::default(); + let policy = gc.policy(); + + gc.remove_idle_session_if_still_idle(&session, 2_000, &policy, &mut report).await; + + assert_eq!(report.sessions_deleted, 0); + assert_eq!(gc.sessions.len().await, 1); + assert!(!session.read().await.is_gc_marked_for_removal()); +} diff --git a/backend/hls/src/gc/tests/publication.rs b/backend/hls/src/gc/tests/publication.rs new file mode 100644 index 000000000..8f23ac22c --- /dev/null +++ b/backend/hls/src/gc/tests/publication.rs @@ -0,0 +1,193 @@ +use super::*; + +#[tokio::test] +async fn published_resource_history_survives_production_segment_removal_and_file_deletion() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let baseline = normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:480\n\ + #EXTINF:4,\n480.ts\n#EXTINF:4,\n481.ts\n#EXTINF:4,\n482.ts\n", + ); + let replay_then_new = normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:490\n\ + #EXTINF:4,\n480.ts\n#EXTINF:4,\n490.ts\n#EXTINF:4,\n491.ts\n#EXTINF:4,\n492.ts\n", + ); + let removed_key = { + let mut session = session.write().await; + session.apply_origin_manifest(&baseline).expect("baseline maps"); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1 }; + } + session.render_and_store_manifest(1).expect("baseline publishes"); + session.segments.get(&0).expect("published head").cache_key.clone() + }; + gc.cache.write_bytes_and_commit(&removed_key, b"x").await.expect("cache object writes"); + { + let mut session = session.write().await; + assert_eq!(remove_segment_entry(&mut session, 0), Some(removed_key.clone())); + } + gc.cache.delete(&removed_key).await.expect("production cache file deletion succeeds"); + { + let mut session = session.write().await; + session.apply_origin_manifest(&replay_then_new).expect("history trims removed resource replay"); + assert!(!session.segments.contains_key(&0)); + assert_eq!(session.proxy_next_seq, Some(6)); + assert!(session.segments.get(&3).expect("first new media").discontinuity_before); + } + assert!(gc.cache.metadata(&removed_key).await.expect("metadata lookup succeeds").is_none()); +} + +#[tokio::test] +async fn gc_keeps_last_rendered_manifest_segments() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + populate_ready_segments(&gc, &session, 0).await; + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 0); + assert_eq!(session.read().await.segments.len(), 6); +} + +#[tokio::test] +async fn finalized_transient_resource_mappings_in_current_manifest_are_protected() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let resource_id = { + let mut session = session.write().await; + let resource = crate::TransientResourceRef::new( + crate::TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 0, + 10, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + session.transient.upsert_resources([resource]); + session.transient.replace_manifest_with_semantics( + format!("#EXTM3U\n#EXTINF:1,\n/hls/shared/live/session/lease/r/{}.ts\n#EXT-X-ENDLIST\n", resource_id.0), + 0, + Some(1_000), + ); + resource_id + }; + + let report = gc.run_once(20).await.expect("gc should run"); + + assert_eq!(report.transient_resources_pruned, 0); + assert!(session.read().await.transient.resources.contains_key(&resource_id)); +} + +#[test] +fn finalized_manifest_mapping_count_does_not_become_object_pin_count() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "large-finalized"), b"secret", 0); + let resources = install_finalized_transient_manifest(&mut session, 1_643, 10); + + let protected = ProtectedSet::from_session(&session); + + assert_eq!(resources.len(), 1_643); + assert_eq!(session.transient.current_manifest_resource_ids().len(), 1_643); + assert!(protected.transient_object_ids.is_empty()); + assert!(resources.iter().all(|resource| session.transient.resolve_current_resource(&resource.id, 20).is_some())); +} + +#[tokio::test] +async fn large_finalized_manifest_keeps_mappings_without_pinning_all_objects() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let (resource, lookup_key, fetch_token) = { + let mut session = session.write().await; + let resources = install_finalized_transient_manifest(&mut session, 1_643, 10); + let resource = resources.into_iter().next().expect("large manifest resource"); + let lookup_key = TransientPassthroughState::transient_object_key(&session.proxy_session_id, &resource.id, "ts"); + let proxy_session_id = session.proxy_session_id.clone(); + let fetch_token = match session.transient.begin_object_fetch(&proxy_session_id, &resource, "ts", 0, 10) { + TransientObjectFetchDecision::Fetch(token) => token, + TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { + panic!("first archive object starts a cache fill") + } + }; + (resource, lookup_key, fetch_token) + }; + gc.cache.write_bytes_and_commit(fetch_token.cache_key(), b"archive-object").await.expect("archive object writes"); + { + let mut session = session.write().await; + assert!(session.commit_transient_object_ready_if_current( + resource.kind, + &fetch_token, + "video/mp2t".to_string(), + 14, + 0, + 10, + )); + let protected = ProtectedSet::from_session(&session); + assert_eq!(session.transient.current_manifest_resource_ids().len(), 1_643); + assert!(!protected.transient_object_ids.contains(&resource.id)); + } + + let report = gc.run_once(20).await.expect("finalized archive GC runs"); + + assert_eq!(report.transient_resources_pruned, 0); + assert_eq!(report.transient_objects_deleted, 1); + assert!(gc.cache.metadata(fetch_token.cache_key()).await.expect("object metadata reads").is_none()); + let mut session = session.write().await; + assert_eq!(session.transient.resources.len(), 1_643); + let current = session.transient.resolve_current_resource(&resource.id, 20).expect("mapping survives byte eviction"); + assert!(!session.transient.object_cache.contains_key(&lookup_key)); + let proxy_session_id = session.proxy_session_id.clone(); + assert!(matches!( + session.transient.begin_object_fetch(&proxy_session_id, ¤t, "ts", 20, 10,), + TransientObjectFetchDecision::Fetch(_) + )); +} + +#[tokio::test] +async fn transient_object_gc_deletes_the_physical_fill_generation_instead_of_the_logical_lookup_path() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let (lookup_key, fetch_token, resource_kind) = { + let mut session = session.write().await; + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/object1.ts", + b"secret", + 0, + 10, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + session.transient.upsert_resources([resource]); + let resource = session.transient.resources.get(&resource_id).expect("registered resource").clone(); + let proxy_session_id = session.proxy_session_id.clone(); + let lookup_key = TransientPassthroughState::transient_object_key(&proxy_session_id, &resource_id, "ts"); + let fetch_token = match session.transient.begin_object_fetch(&proxy_session_id, &resource, "ts", 0, 10) { + TransientObjectFetchDecision::Fetch(token) => token, + TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { + panic!("new transient object starts one physical cache fill") + } + }; + (lookup_key, fetch_token, resource.kind) + }; + assert_ne!(&lookup_key, fetch_token.cache_key()); + gc.cache + .write_bytes_and_commit(fetch_token.cache_key(), b"transient-body") + .await + .expect("physical generation writes"); + gc.cache.write_bytes_and_commit(&lookup_key, b"logical-decoy").await.expect("logical lookup decoy writes"); + assert!(session.write().await.commit_transient_object_ready_if_current( + resource_kind, + &fetch_token, + "video/mp2t".to_string(), + 14, + 0, + 10, + )); + + let report = gc.run_once(20).await.expect("gc should run"); + + assert_eq!(report.transient_objects_deleted, 1); + assert_eq!(report.transient_object_bytes_deleted, 14); + assert!(gc.cache.metadata(fetch_token.cache_key()).await.expect("physical metadata read").is_none()); + assert!(gc.cache.metadata(&lookup_key).await.expect("logical metadata read").is_some()); +} diff --git a/backend/hls/src/gc/tests/recovery.rs b/backend/hls/src/gc/tests/recovery.rs new file mode 100644 index 000000000..8bf513b4f --- /dev/null +++ b/backend/hls/src/gc/tests/recovery.rs @@ -0,0 +1,99 @@ +use super::*; + +#[tokio::test] +async fn full_retry_queue_preserves_session_metadata_until_a_slot_is_reserved() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + session.segments.get_mut(&1).expect("segment").status = + SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 0 }; + } + { + let mut queue = gc.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + for proxy_seq in 0..MAX_PENDING_CACHE_DELETIONS { + queue.pending.push_back(PendingCacheObjectDeletion { + deletion: CacheObjectDeletion::Segment { + key: crate::SegmentCacheKey::new( + crate::ProxySessionId("queued".to_string()), + u64::try_from(proxy_seq).unwrap_or_default(), + "ts", + ), + reason: SegmentCacheDeletionReason::Duration, + }, + attempts: 0, + }); + } + } + let mut report = GarbageCollectionReport::default(); + let policy = gc.policy(); + let mut batch = gc.reserve_cache_deletion_batch(); + assert_eq!(batch.remaining_capacity(), 0); + + { + let mut session = session.write().await; + HlsGarbageCollector::collect_session_deletions( + &mut session, + &gc.cache, + 10_000, + &policy, + &mut report, + &mut batch, + ); + assert!(session.segments.contains_key(&1)); + } + batch.persist(&mut report); + + assert_eq!(report.cache_object_deletions_planned, 0); + assert_eq!(gc.pending_cache_deletion_count(), MAX_PENDING_CACHE_DELETIONS); +} + +#[tokio::test] +async fn dropping_unpersisted_deletion_batch_keeps_reserved_deletion_retryable() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, _) = gc_with_session(&temp_dir).await; + let key = crate::SegmentCacheKey::new(crate::ProxySessionId("cancelled".to_string()), 1, "ts"); + { + let mut batch = gc.reserve_cache_deletion_batch(); + batch.push(CacheObjectDeletion::Segment { key: key.clone(), reason: SegmentCacheDeletionReason::Duration }); + } + + let queue = gc.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!(queue.reserved_slots, 0); + assert_eq!(queue.pending.len(), 1); + assert_eq!( + queue.pending.front().map(|pending| &pending.deletion), + Some(&CacheObjectDeletion::Segment { key, reason: SegmentCacheDeletionReason::Duration }) + ); +} + +#[tokio::test] +async fn failed_switch_rollback_blocks_same_key_until_gc_retry_succeeds() { + let cache_root = tempfile::tempdir().expect("cache root"); + let (gc, session) = gc_with_session(&cache_root).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let key = crate::SegmentCacheKey::new(proxy_session_id, 1, "ts"); + gc.cache.write_bytes_and_commit(&key, b"uncommitted-switch").await.expect("switch fixture writes"); + let object_path = gc.cache.object_path(&key); + tokio::fs::remove_file(&object_path).await.expect("switch fixture file removes"); + tokio::fs::create_dir(&object_path).await.expect("directory forces remove-file failure"); + let cleanup = gc.reserve_switch_segment_cleanup(key.clone()).expect("switch cleanup reservation"); + drop(cleanup); + + assert!(gc.has_pending_switch_cleanup(&key, None)); + let deferred = gc.run_once(0).await.expect("deferred rollback GC runs"); + assert_eq!(deferred.cache_object_deletions_succeeded, 0); + assert_eq!(deferred.cache_object_deletions_deferred, 1); + assert_eq!(gc.cache_deletion_queue_usage(), (1, 0)); + assert!(gc.has_pending_switch_cleanup(&key, None)); + + tokio::fs::remove_dir(&object_path).await.expect("failing directory removes"); + tokio::fs::write(&object_path, b"uncommitted-switch").await.expect("retry fixture writes"); + let retried = gc.run_once(1).await.expect("rollback retry GC runs"); + + assert_eq!(retried.cache_object_deletions_succeeded, 1); + assert_eq!(gc.cache_deletion_queue_usage(), (0, 0)); + assert!(!gc.has_pending_switch_cleanup(&key, None)); + assert!(gc.cache.metadata(&key).await.expect("rolled-back metadata reads").is_none()); +} diff --git a/backend/hls/src/gc/tests/streaming.rs b/backend/hls/src/gc/tests/streaming.rs new file mode 100644 index 000000000..14103da9e --- /dev/null +++ b/backend/hls/src/gc/tests/streaming.rs @@ -0,0 +1,18 @@ +use super::*; + +#[tokio::test] +async fn gc_keeps_fetching_segments() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + session.segments.get_mut(&1).expect("segment").status = + SegmentCacheStatus::Fetching { priority: SegmentFetchPriority::Prefetch, started_at_ms: 1 }; + } + + let report = gc.run_once(10_000).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_duration, 0); + assert!(session.read().await.segments.contains_key(&1)); +} diff --git a/backend/hls/src/gc/tests/transport.rs b/backend/hls/src/gc/tests/transport.rs new file mode 100644 index 000000000..4dd4c2d0a --- /dev/null +++ b/backend/hls/src/gc/tests/transport.rs @@ -0,0 +1,223 @@ +use super::*; + +#[test] +fn protected_encrypted_segment_also_protects_its_key_resource() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let key_resource = TransientResourceRef::new( + TransientResourceKind::Key, + "http://origin.example.com/live/final/key.bin", + b"secret", + 0, + u64::MAX, + Some("bin".to_string()), + ); + let mut manifest = normal_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\n1.ts\n", + ); + for encryption in manifest.segments.iter_mut().filter_map(|segment| segment.encryption.as_mut()) { + encryption.proxy_resource_id = Some(key_resource.id.0.clone()); + encryption.proxy_resource_extension = Some("bin".to_string()); + } + session.transient.upsert_resources([key_resource]); + session.apply_origin_manifest(&manifest).expect("manifest maps"); + let segment = session.segments.get_mut(&0).expect("segment"); + segment.access.reader_started(1); + let resource_id = segment.encryption.as_ref().expect("encryption").resource_id.clone(); + + let protected = ProtectedSet::from_session(&session); + + assert!(protected.segment_proxy_seqs.contains(&0)); + assert!(protected.key_resource_ids.contains(&resource_id)); +} + +#[tokio::test] +async fn cache_root_handoff_discards_old_logical_deletion_tickets_before_new_root_use() { + let old_root = tempfile::tempdir().expect("old cache root"); + let new_root = tempfile::tempdir().expect("new cache root"); + let (gc, session) = gc_with_session(&old_root).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let key = crate::SegmentCacheKey::new(proxy_session_id, 1, "ts"); + { + let mut queue = gc.pending_cache_deletions.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + queue.pending.push_back(PendingCacheObjectDeletion { + deletion: CacheObjectDeletion::Segment { key: key.clone(), reason: SegmentCacheDeletionReason::Duration }, + attempts: 1, + }); + } + + assert!(gc.update_cache_path(new_root.path()).await); + gc.cache.write_bytes_and_commit(&key, b"new-root-sentinel").await.expect("new-root sentinel writes"); + let report = gc.run_once(0).await.expect("new-root gc runs"); + + assert_eq!(gc.pending_cache_deletion_count(), 0); + assert_eq!(report.cache_object_deletions_succeeded, 0); + let metadata = gc.cache.metadata(&key).await.expect("new-root metadata reads").expect("sentinel remains"); + assert_eq!(tokio::fs::read(metadata.path).await.expect("sentinel reads"), b"new-root-sentinel"); +} + +#[tokio::test] +async fn global_size_gc_subtracts_map_bytes_after_segment_removal() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + update_gc_policy(&gc, |policy| policy.cache_bytes_global = 25); + let manifest = normal_manifest( + "#EXTM3U\n#EXT-X-MAP:URI=\"init0.mp4\"\n#EXTINF:4.0,\n0.m4s\n#EXT-X-MAP:URI=\"init1.mp4\"\n#EXTINF:4.0,\n1.m4s\n", + ); + { + let mut session = session.write().await; + session.apply_origin_manifest(&manifest).expect("manifest should map"); + for (proxy_seq, ready_at_ms) in [(0, 0), (1, 1)] { + let segment = session.segments.get_mut(&proxy_seq).expect("segment"); + segment.status = SegmentCacheStatus::Ready { content_length: 12, ready_at_ms }; + } + session.maps.get_mut(&ProxyMapId(0)).expect("first map").status = + MapCacheStatus::Ready { content_length: 10, ready_at_ms: 0 }; + session.maps.get_mut(&ProxyMapId(1)).expect("second map").status = + MapCacheStatus::Ready { content_length: 10, ready_at_ms: 1 }; + } + + let report = gc.run_once(100).await.expect("gc should run"); + + assert_eq!(report.segments_deleted_size_global, 1); + assert_eq!(report.maps_deleted, 1); + let session = session.read().await; + assert!(!session.segments.contains_key(&0)); + assert!(session.segments.contains_key(&1)); + assert!(!session.maps.contains_key(&ProxyMapId(0))); + assert!(session.maps.contains_key(&ProxyMapId(1))); +} + +#[tokio::test] +async fn transient_object_gc_expires_stale_fetching_metadata_deletes_its_fill_and_wakes_waiters() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + let (lookup_key, fetch_token, notifier) = { + let mut session = session.write().await; + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/stale.ts", + b"secret", + 0, + 10, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + session.transient.upsert_resources([resource]); + let resource = session.transient.resources.get(&resource_id).expect("registered resource").clone(); + let proxy_session_id = session.proxy_session_id.clone(); + let lookup_key = TransientPassthroughState::transient_object_key(&proxy_session_id, &resource_id, "ts"); + let fetch_token = match session.transient.begin_object_fetch(&proxy_session_id, &resource, "ts", 0, 10) { + TransientObjectFetchDecision::Fetch(token) => token, + TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { + panic!("new transient object starts one physical cache fill") + } + }; + let notifier = match session.transient.begin_object_fetch(&proxy_session_id, &resource, "ts", 1, 10) { + TransientObjectFetchDecision::Wait(notifier) => notifier, + TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Fetch(_) => { + panic!("a concurrent request waits for the current cache fill") + } + }; + (lookup_key, fetch_token, notifier) + }; + gc.cache + .write_bytes_and_commit(fetch_token.cache_key(), b"partial") + .await + .expect("stale physical fill fixture writes"); + let waiter = notifier.notified(); + tokio::pin!(waiter); + assert!(matches!(futures::poll!(&mut waiter), Poll::Pending)); + + let report = gc.run_once(20).await.expect("gc should run"); + + assert!(matches!(futures::poll!(&mut waiter), Poll::Ready(()))); + assert_eq!(report.transient_objects_deleted, 1); + assert_eq!(report.transient_object_bytes_deleted, 0); + assert!(!session.read().await.transient.object_cache.contains_key(&lookup_key)); + assert!(gc.cache.metadata(fetch_token.cache_key()).await.expect("metadata reads").is_none()); +} + +#[tokio::test] +async fn session_size_gc_deletes_transient_objects_before_timeline_segments() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + update_gc_policy(&gc, |policy| policy.cache_bytes_per_session = 20); + let (segment_key, object_lookup_key, object_fetch_token, resource_kind) = { + let mut session = session.write().await; + apply_six_segment_manifest_for_gc(&mut session); + let segment_key = session.segments.get(&1).expect("segment").cache_key.clone(); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/object1.ts", + b"secret", + 100, + 10_000, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + session.transient.upsert_resources([resource]); + let resource = session.transient.resources.get(&resource_id).expect("registered resource").clone(); + let proxy_session_id = session.proxy_session_id.clone(); + let object_lookup_key = TransientPassthroughState::transient_object_key(&proxy_session_id, &resource_id, "ts"); + let object_fetch_token = + match session.transient.begin_object_fetch(&proxy_session_id, &resource, "ts", 100, 10_000) { + TransientObjectFetchDecision::Fetch(token) => token, + TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { + panic!("new transient object starts one physical cache fill") + } + }; + (segment_key, object_lookup_key, object_fetch_token, resource.kind) + }; + gc.cache.write_bytes_and_commit(&segment_key, b"segment-body").await.expect("segment writes"); + gc.cache + .write_bytes_and_commit(object_fetch_token.cache_key(), b"transient-body") + .await + .expect("transient object writes"); + { + let mut session = session.write().await; + session.segments.get_mut(&1).expect("segment").status = + SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 100 }; + assert!(session.commit_transient_object_ready_if_current( + resource_kind, + &object_fetch_token, + "video/mp2t".to_string(), + 14, + 100, + 10_000, + )); + } + + let report = gc.run_once(100).await.expect("gc should run"); + + assert_eq!(report.transient_objects_deleted, 1); + assert_eq!(report.segments_deleted_size_session, 0); + let session = session.read().await; + assert!(session.segments.contains_key(&1)); + assert!(!session.transient.object_cache.contains_key(&object_lookup_key)); + drop(session); + assert!(gc.cache.metadata(object_fetch_token.cache_key()).await.expect("metadata reads").is_none()); +} + +#[tokio::test] +async fn active_transient_resource_reader_protects_idle_session() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let (gc, session) = gc_with_session(&temp_dir).await; + { + let mut session = session.write().await; + let resource = crate::TransientResourceRef::new( + crate::TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 0, + 10, + Some("ts".to_string()), + ); + resource.access.reader_started(1); + session.transient.upsert_resources([resource]); + } + + let report = gc.run_once(2_000).await.expect("gc should run"); + + assert_eq!(report.sessions_deleted, 0); + assert_eq!(gc.sessions.len().await, 1); +} diff --git a/backend/hls/src/lease.rs b/backend/hls/src/lease.rs index 96e295aed..0945204e8 100644 --- a/backend/hls/src/lease.rs +++ b/backend/hls/src/lease.rs @@ -16,11 +16,8 @@ use super::{ }, HlsEffectiveOriginAcquirePolicy, HlsPublishedTransientResourceIds, ProxySessionId, TransientManifestLeaseBinding, }; -use base64::{engine::general_purpose, Engine as _}; -use rand::{rngs::OsRng, RngCore, TryRngCore}; use std::{ collections::{BTreeSet, HashMap}, - fmt, sync::Arc, }; use tuliprox_session::ConnectionKind; @@ -33,31 +30,10 @@ const HLS_ACCESS_LEASE_ID_BYTES: usize = 16; #[derive(Debug, Clone, Copy, Default, Eq, PartialEq, Ord, PartialOrd)] pub struct HlsAvailabilityEvidenceGeneration(u64); -impl HlsAvailabilityEvidenceGeneration { - const NONE: Self = Self(0); - - #[cfg(any(test, feature = "test-support"))] - pub const fn for_test(generation: u64) -> Self { Self(generation) } - - #[cfg(any(test, feature = "test-support"))] - pub const fn as_u64(self) -> u64 { self.0 } -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum HlsAvailabilityEvidenceAdvanceError { - Exhausted, -} - /// Short opaque lookup key for a server-side HLS access lease. #[derive(Clone, Eq, PartialEq, Hash)] pub struct HlsAccessLeaseId(pub String); -impl fmt::Debug for HlsAccessLeaseId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_tuple("HlsAccessLeaseId").field(&"").finish() - } -} - /// Stable user/player family used only for diagnostics or future UX grouping. #[derive(Debug, Clone, Eq, PartialEq, Hash)] pub struct HlsPlaybackFamilyKey { @@ -65,12 +41,6 @@ pub struct HlsPlaybackFamilyKey { pub client_fingerprint: String, } -impl HlsPlaybackFamilyKey { - pub fn new(username: impl Into, client_fingerprint: impl Into) -> Self { - Self { username: username.into(), client_fingerprint: client_fingerprint.into() } - } -} - #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum HlsAccessLeaseState { Pending, @@ -87,19 +57,6 @@ pub enum HlsLeaseStartupAdmissionState { Admitted, } -impl HlsAccessLeaseState { - pub const fn as_log_value(self) -> &'static str { - match self { - Self::Pending => "Pending", - Self::Activated => "Activated", - Self::Idle => "Idle", - Self::PolicyRevoking => "PolicyRevoking", - Self::Expired => "Expired", - Self::Denied => "Denied", - } - } -} - /// Immutable authorization evidence frozen before an active HLS entitlement is revoked. #[derive(Clone, Debug, Eq, PartialEq)] pub struct HlsRuntimePolicyRevocation { @@ -159,23 +116,6 @@ pub enum HlsAccessLeasePendingDeadline { FollowUp { deadline_ms: u64 }, } -impl HlsAccessLeasePendingDeadline { - pub const fn deadline_ms(self) -> u64 { - match self { - Self::Bootstrap { deadline_ms } | Self::FollowUp { deadline_ms } => deadline_ms, - } - } - - const fn tightened_with(self, candidate: Self) -> Self { - let deadline_ms = - if self.deadline_ms() <= candidate.deadline_ms() { self.deadline_ms() } else { candidate.deadline_ms() }; - match (self, candidate) { - (Self::FollowUp { .. }, _) | (_, Self::FollowUp { .. }) => Self::FollowUp { deadline_ms }, - (Self::Bootstrap { .. }, Self::Bootstrap { .. }) => Self::Bootstrap { deadline_ms }, - } - } -} - #[derive(Debug, Clone, Eq, PartialEq)] pub struct HlsAccessLease { pub lease_id: HlsAccessLeaseId, @@ -201,6 +141,8 @@ pub struct HlsAccessLease { pub startup_admission: HlsLeaseStartupAdmissionState, pub playback_cursor: HlsLeasePlaybackCursor, pub last_manifest_snapshot: Option, + revision_bindings: Option>, + progressive_startup_claimed: bool, published_transient_resource_ids: HlsPublishedTransientResourceIds, published_finalized_manifest_generations: BTreeSet, manifest_snapshot_generation: u64, @@ -220,18 +162,6 @@ pub struct HlsMediaLeaseIdentity { playback: HlsMediaLeasePlaybackIdentity, } -impl HlsMediaLeaseIdentity { - pub const fn is_live(self) -> bool { matches!(self.playback, HlsMediaLeasePlaybackIdentity::Live { .. }) } - - pub const fn lease_issued_at_ms(self) -> u64 { self.issued_at_ms } -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum HlsMediaLeasePlaybackIdentity { - Live { admission_generation: u64 }, - TerminalTail { generation: HlsTerminalTailGeneration }, -} - /// Lease incarnation and admission generation observed before building a manifest response. /// /// The store accepts the corresponding snapshot only while this exact live lease identity is @@ -242,10 +172,6 @@ pub struct HlsLeaseManifestPublicationGuard { admission_generation: u64, } -impl HlsLeaseManifestPublicationGuard { - pub(crate) const fn lease_issued_at_ms(self) -> u64 { self.issued_at_ms } -} - #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum HlsLeaseManifestPublicationRejectReason { UnknownLease, @@ -257,6 +183,7 @@ pub enum HlsLeaseManifestPublicationRejectReason { LeaseNotLive, SourceRegressive, ManifestGenerationUnavailable, + RevisionConflict, SnapshotGenerationExhausted, } @@ -267,300 +194,6 @@ pub enum HlsLeaseManifestPublicationOutcome { Rejected(HlsLeaseManifestPublicationRejectReason), } -impl HlsLeaseManifestPublicationOutcome { - pub const fn is_committed(self) -> bool { matches!(self, Self::Committed { .. }) } - - pub const fn snapshot_generation(self) -> Option { - match self { - Self::Committed { snapshot_generation } => Some(snapshot_generation), - Self::Rejected(_) => None, - } - } -} - -impl HlsAccessLease { - #[allow(clippy::too_many_arguments)] - pub fn pending( - lease_id: HlsAccessLeaseId, - family_key: HlsPlaybackFamilyKey, - proxy_session_id: ProxySessionId, - username: String, - user_session_token: String, - input_id: u16, - stream_ref: String, - virtual_id: u32, - now_ms: u64, - valid_window_ms: u64, - ) -> Self { - Self { - lease_id, - family_key, - proxy_session_id, - username, - user_session_token, - input_id, - stream_ref, - virtual_id, - known_bitrate_bps: None, - origin_connection_kind: ConnectionKind::Normal, - origin_priority: 0, - state: HlsAccessLeaseState::Pending, - issued_at_ms: now_ms, - last_seen_at_ms: now_ms, - active_until_ms: None, - pending_deadline: Some(HlsAccessLeasePendingDeadline::Bootstrap { - deadline_ms: now_ms.saturating_add(valid_window_ms), - }), - valid_until_ms: now_ms.saturating_add(valid_window_ms), - epg_reference_ts: None, - archive_origin_url: None, - playback_mode: HlsLeasePlaybackMode::Live, - startup_admission: HlsLeaseStartupAdmissionState::Pending, - playback_cursor: HlsLeasePlaybackCursor::default(), - last_manifest_snapshot: None, - published_transient_resource_ids: HlsPublishedTransientResourceIds::default(), - published_finalized_manifest_generations: BTreeSet::new(), - manifest_snapshot_generation: 0, - admission_generation: 0, - runtime_policy_revocation: None, - runtime_policy_denial_reason: None, - pending_terminal_protection_release: None, - } - } - - pub fn published_transient_resource_ids(&self) -> &HlsPublishedTransientResourceIds { - &self.published_transient_resource_ids - } - - pub fn with_archive_playback(mut self, epg_reference_ts: Option, archive_origin_url: Option) -> Self { - self.epg_reference_ts = epg_reference_ts; - self.archive_origin_url = archive_origin_url; - self - } - - pub const fn with_origin_acquire_policy(mut self, connection_kind: ConnectionKind, priority: i8) -> Self { - self.origin_connection_kind = connection_kind; - self.origin_priority = priority; - self - } - - pub const fn with_known_bitrate_bps(mut self, known_bitrate_bps: Option) -> Self { - self.known_bitrate_bps = match known_bitrate_bps { - Some(0) | None => None, - Some(bitrate_bps) => Some(bitrate_bps), - }; - self - } - - pub fn update_origin_acquire_policy(&mut self, connection_kind: ConnectionKind, priority: i8) { - self.origin_connection_kind = connection_kind; - self.origin_priority = priority; - } - - pub fn age_ms(&self, now_ms: u64) -> u64 { now_ms.saturating_sub(self.issued_at_ms) } - - pub fn media_identity(&self) -> Option { - let playback = match &self.playback_mode { - HlsLeasePlaybackMode::Live => { - HlsMediaLeasePlaybackIdentity::Live { admission_generation: self.admission_generation } - } - HlsLeasePlaybackMode::TerminalTail(plan) => { - HlsMediaLeasePlaybackIdentity::TerminalTail { generation: plan.generation } - } - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => return None, - }; - Some(HlsMediaLeaseIdentity { issued_at_ms: self.issued_at_ms, playback }) - } - - pub fn runtime_policy_revocation_outcome(&self) -> Option { - match &self.playback_mode { - HlsLeasePlaybackMode::TerminalTail(plan) => { - Some(HlsRuntimePolicyRevocationOutcome::AlreadyCommitted { plan: Arc::clone(plan) }) - } - HlsLeasePlaybackMode::Live => self - .runtime_policy_revocation - .clone() - .map(|token| HlsRuntimePolicyRevocationOutcome::AlreadyPending { token }), - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => None, - } - } - - pub fn runtime_policy_denial_reason(&self) -> Option { - match &self.playback_mode { - HlsLeasePlaybackMode::TerminalTail(plan) => Some(plan.reason), - HlsLeasePlaybackMode::Live => self.runtime_policy_revocation.as_ref().map(|token| token.reason), - HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => { - self.runtime_policy_denial_reason - } - } - } - - pub fn permits_unpublished_standalone_tail(&self, reason: HlsRuntimeCustomTailReason) -> bool { - reason.permits_unpublished_lease_standalone_tail() - && self.state == HlsAccessLeaseState::Pending - && self.playback_mode == HlsLeasePlaybackMode::Live - && self.startup_admission == HlsLeaseStartupAdmissionState::Pending - && self.last_manifest_snapshot.is_none() - && self.runtime_policy_revocation.is_none() - } - - fn manifest_publication_guard(&self) -> Option { - (lease_state_allows_use(self.state) && self.playback_mode == HlsLeasePlaybackMode::Live).then_some( - HlsLeaseManifestPublicationGuard { - issued_at_ms: self.issued_at_ms, - admission_generation: self.admission_generation, - }, - ) - } - - pub fn pending_deadline_ms(&self) -> Option { - self.pending_deadline.map(HlsAccessLeasePendingDeadline::deadline_ms) - } - - fn validity_due_at_ms(&self) -> u64 { - if self.state == HlsAccessLeaseState::Pending { - self.pending_deadline_ms().unwrap_or(self.valid_until_ms) - } else { - self.valid_until_ms - } - } - - fn apply_pending_deadline(&mut self, deadline: HlsAccessLeasePendingDeadline) -> bool { - let previous = self.pending_deadline; - let deadline = self.pending_deadline.map_or(deadline, |current| current.tightened_with(deadline)); - self.pending_deadline = Some(deadline); - self.valid_until_ms = deadline.deadline_ms(); - previous != self.pending_deadline - } - - fn refresh_validity(&mut self, now_ms: u64) -> bool { - if self.state != HlsAccessLeaseState::Expired && self.validity_due_at_ms() <= now_ms { - self.state = HlsAccessLeaseState::Expired; - if self.playback_mode == HlsLeasePlaybackMode::Live { - self.end_playback(); - } - return true; - } - false - } - - fn end_playback(&mut self) { - self.runtime_policy_revocation = None; - let release_generation = match std::mem::replace(&mut self.playback_mode, HlsLeasePlaybackMode::Ended) { - HlsLeasePlaybackMode::TerminalTail(plan) => Some(plan.generation), - HlsLeasePlaybackMode::TerminalUnavailable { decision_generation, .. } => { - Some(HlsTerminalTailGeneration(decision_generation)) - } - HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::Ended => None, - }; - if self.pending_terminal_protection_release.is_none() { - self.pending_terminal_protection_release = release_generation; - } - } - - fn terminal_commit_preconditions_match( - &mut self, - proxy_session_id: &ProxySessionId, - expected_issued_at_ms: u64, - expected_admission_generation: u64, - manifest_snapshot_generation: u64, - cursor_generation: u64, - _now_ms: u64, - ) -> Result<(), HlsTerminalCommitOutcome> { - if !lease_state_allows_use(self.state) - || self.proxy_session_id != *proxy_session_id - || self.issued_at_ms != expected_issued_at_ms - || self.playback_mode != HlsLeasePlaybackMode::Live - { - return Err(HlsTerminalCommitOutcome::LeaseNoLongerEligible); - } - if self.admission_generation != expected_admission_generation - || self.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation) - != Some(manifest_snapshot_generation) - || self.playback_cursor.cursor_generation != cursor_generation - { - return Err(HlsTerminalCommitOutcome::SupersededGeneration); - } - Ok(()) - } - - fn runtime_policy_tail_commit_preconditions_match( - &mut self, - proxy_session_id: &ProxySessionId, - preparation: &HlsTerminalTailPreparation, - revocation: &HlsRuntimePolicyRevocation, - ) -> Result<(), HlsTerminalCommitOutcome> { - if self.state != HlsAccessLeaseState::PolicyRevoking - || self.proxy_session_id != *proxy_session_id - || self.issued_at_ms != revocation.lease_issued_at_ms - || self.playback_mode != HlsLeasePlaybackMode::Live - || self.runtime_policy_revocation.as_ref() != Some(revocation) - || preparation.runtime_policy_revocation.as_ref() != Some(revocation) - || preparation.trigger != HlsFiniteTailTrigger::RuntimePolicy(revocation.reason) - { - return Err(HlsTerminalCommitOutcome::LeaseNoLongerEligible); - } - if self.admission_generation != revocation.expected_admission_generation - || preparation.expected_admission_generation != revocation.expected_admission_generation - || self.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation) - != Some(revocation.manifest_snapshot_generation) - || preparation.manifest_snapshot_generation != revocation.manifest_snapshot_generation - || self.playback_cursor.cursor_generation != revocation.cursor_generation - || preparation.cursor_generation != revocation.cursor_generation - { - return Err(HlsTerminalCommitOutcome::SupersededGeneration); - } - Ok(()) - } - - fn terminal_decision_commit_preconditions_match( - &mut self, - proxy_session_id: &ProxySessionId, - preparation: &HlsTerminalTailPreparation, - now_ms: u64, - ) -> Result<(), HlsTerminalCommitOutcome> { - match preparation.runtime_policy_revocation.as_ref() { - Some(revocation) => { - self.runtime_policy_tail_commit_preconditions_match(proxy_session_id, preparation, revocation) - } - None => self.terminal_commit_preconditions_match( - proxy_session_id, - preparation.lease_issued_at_ms, - preparation.expected_admission_generation, - preparation.manifest_snapshot_generation, - preparation.cursor_generation, - now_ms, - ), - } - } -} - -fn terminal_commit_existing_outcome( - lease: &mut HlsAccessLease, - proxy_session_id: &ProxySessionId, - preparation: &HlsTerminalTailPreparation, - _now_ms: u64, -) -> Option { - if lease.proxy_session_id != *proxy_session_id || lease.issued_at_ms != preparation.lease_issued_at_ms { - return Some(HlsTerminalCommitOutcome::LeaseNoLongerEligible); - } - match &lease.playback_mode { - HlsLeasePlaybackMode::Live => None, - HlsLeasePlaybackMode::TerminalTail(current) if current.generation.0 == preparation.decision_generation => { - Some(HlsTerminalCommitOutcome::AlreadyCommitted) - } - HlsLeasePlaybackMode::TerminalUnavailable { decision_generation, .. } - if *decision_generation == preparation.decision_generation => - { - Some(HlsTerminalCommitOutcome::AlreadyCommitted) - } - HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } => { - Some(HlsTerminalCommitOutcome::SupersededGeneration) - } - HlsLeasePlaybackMode::Ended => Some(HlsTerminalCommitOutcome::LeaseNoLongerEligible), - } -} - #[derive(Debug, Clone, PartialEq, Eq)] pub struct HlsTerminalTailPreparation { pub trigger: HlsFiniteTailTrigger, @@ -584,40 +217,6 @@ pub struct HlsTerminalTailPreparation { pub manifest_snapshot: HlsLeaseManifestSnapshot, } -impl HlsTerminalTailPreparation { - pub fn bind_ready_terminal_media_requirement( - &mut self, - prepared_key: HlsTerminalMediaPreparationKey, - ) -> Result<(), HlsTerminalTailCompatibility> { - let source = match self.terminal_media_requirement_source { - HlsTerminalMediaRequirementSource::AcceptanceEpisode { .. } => { - if self.required_terminal_media_key != Some(prepared_key) { - return Err(HlsTerminalTailCompatibility::AssetRevisionMismatch); - } - self.terminal_media_requirement_source - } - HlsTerminalMediaRequirementSource::CutoverSnapshotPending { decision_generation } - if decision_generation == self.decision_generation => - { - HlsTerminalMediaRequirementSource::CutoverSnapshot { decision_generation, asset: prepared_key.asset } - } - HlsTerminalMediaRequirementSource::CutoverSnapshot { decision_generation, asset } - if decision_generation == self.decision_generation && asset == prepared_key.asset => - { - self.terminal_media_requirement_source - } - HlsTerminalMediaRequirementSource::CutoverSnapshotPending { .. } - | HlsTerminalMediaRequirementSource::CutoverSnapshot { .. } => { - return Err(HlsTerminalTailCompatibility::AssetRevisionMismatch); - } - }; - self.terminal_media_requirement_source = source; - self.required_terminal_media_key = Some(prepared_key); - self.terminal_media_preparation = HlsTerminalMediaPreparationState::Ready { key: prepared_key }; - Ok(()) - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HlsTerminalMediaRequirementOrigin { AcceptanceEpisode { generation: HlsManifestAcceptanceGeneration }, @@ -631,18 +230,6 @@ pub enum HlsTerminalMediaRequirementSource { CutoverSnapshot { decision_generation: u64, asset: HlsTerminalAssetIdentity }, } -impl HlsTerminalMediaRequirementSource { - pub fn authorizes_tail(self, decision_generation: u64, prepared_key: HlsTerminalMediaPreparationKey) -> bool { - match self { - Self::AcceptanceEpisode { .. } => true, - Self::CutoverSnapshotPending { .. } => false, - Self::CutoverSnapshot { decision_generation: source_generation, asset } => { - source_generation == decision_generation && asset == prepared_key.asset - } - } - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct HlsTerminalTailPreparationInput { pub trigger: HlsFiniteTailTrigger, @@ -684,10 +271,6 @@ pub enum HlsAccessLeaseActivation { SessionMismatch, } -impl HlsAccessLeaseActivation { - pub const fn is_activated(&self) -> bool { matches!(self, Self::Activated { .. }) } -} - #[derive(Debug, Clone, Eq, PartialEq)] pub enum HlsAccessLeaseTouch { Touched { lease: Box }, @@ -731,3015 +314,23 @@ pub struct HlsAccessLeaseSessionSnapshot { pub(crate) finalized_transient_manifest_bindings: Vec, } -impl HlsAccessLeaseStore { - pub fn repair_prewarm_is_current( - &self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - issued_at_ms: u64, - snapshot_generation: u64, - ) -> bool { - self.by_lease_id.get(lease_id).is_some_and(|lease| { - lease.proxy_session_id == *proxy_session_id - && lease.issued_at_ms == issued_at_ms - && lease.playback_mode == HlsLeasePlaybackMode::Live - && lease - .last_manifest_snapshot - .as_ref() - .is_some_and(|snapshot| snapshot.snapshot_generation == snapshot_generation) - }) - } - - pub fn availability_evidence_generation( - &self, - proxy_session_id: &ProxySessionId, - ) -> HlsAvailabilityEvidenceGeneration { - self.availability_generation_by_proxy_session - .get(proxy_session_id) - .copied() - .unwrap_or(HlsAvailabilityEvidenceGeneration::NONE) - } - - fn availability_evidence_can_advance(&self) -> Result<(), HlsAvailabilityEvidenceAdvanceError> { - self.last_availability_evidence_generation - .0 - .checked_add(1) - .map(|_| ()) - .ok_or(HlsAvailabilityEvidenceAdvanceError::Exhausted) - } - - fn advance_availability_evidence( - &mut self, - proxy_session_id: &ProxySessionId, - ) -> Result { - let generation = self - .last_availability_evidence_generation - .0 - .checked_add(1) - .map(HlsAvailabilityEvidenceGeneration) - .ok_or(HlsAvailabilityEvidenceAdvanceError::Exhausted)?; - self.last_availability_evidence_generation = generation; - self.availability_generation_by_proxy_session.insert(proxy_session_id.clone(), generation); - Ok(generation) - } - - fn refresh_access_lease_validity( - &mut self, - lease_id: &HlsAccessLeaseId, - now_ms: u64, - ) -> Result { - let Some(lease) = self.by_lease_id.get(lease_id) else { - return Ok(false); - }; - if lease.state == HlsAccessLeaseState::Expired || lease.validity_due_at_ms() > now_ms { - return Ok(false); - } - let proxy_session_id = lease.proxy_session_id.clone(); - self.availability_evidence_can_advance()?; - let changed = self.by_lease_id.get_mut(lease_id).is_some_and(|lease| lease.refresh_validity(now_ms)); - if changed { - self.advance_availability_evidence(&proxy_session_id)?; - } - Ok(changed) - } - - fn refresh_access_lease_validities_for_session( - &mut self, - proxy_session_id: &ProxySessionId, - now_ms: u64, - ) -> Result<(), HlsAvailabilityEvidenceAdvanceError> { - let lease_ids = self - .by_lease_id - .values() - .filter(|lease| lease.proxy_session_id == *proxy_session_id) - .map(|lease| lease.lease_id.clone()) - .collect::>(); - for lease_id in lease_ids { - self.refresh_access_lease_validity(&lease_id, now_ms)?; - } - Ok(()) - } - - fn refresh_access_lease_activity( - &mut self, - lease_id: &HlsAccessLeaseId, - now_ms: u64, - ) -> Result, HlsAvailabilityEvidenceAdvanceError> { - let previous_state = match self.by_lease_id.get(lease_id) { - Some(lease) => lease.state, - None => return Ok(None), - }; - self.refresh_access_lease_validity(lease_id, now_ms)?; - let Some(lease) = self.by_lease_id.get(lease_id) else { - return Ok(None); - }; - if lease.state == HlsAccessLeaseState::Expired - && matches!(previous_state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated) - { - return Ok(Some(HlsAccessLeaseIdleRelease { - lease_id: lease.lease_id.clone(), - username: lease.username.clone(), - user_session_token: lease.user_session_token.clone(), - })); - } - if lease.state != HlsAccessLeaseState::Activated - || lease.active_until_ms.is_none_or(|active_until| active_until > now_ms) - { - return Ok(None); - } - let proxy_session_id = lease.proxy_session_id.clone(); - let release = HlsAccessLeaseIdleRelease { - lease_id: lease.lease_id.clone(), - username: lease.username.clone(), - user_session_token: lease.user_session_token.clone(), - }; - self.availability_evidence_can_advance()?; - if let Some(lease) = self.by_lease_id.get_mut(lease_id) { - lease.state = HlsAccessLeaseState::Idle; - } - self.advance_availability_evidence(&proxy_session_id)?; - Ok(Some(release)) - } - - /// Returns false only when the process-lifetime evidence generation is - /// exhausted. In that case no lease replacement is published. - pub fn prepare_access_lease(&mut self, lease: HlsAccessLease) -> bool { - if self.by_lease_id.get(&lease.lease_id) == Some(&lease) { - return true; - } - let new_proxy_session_id = lease.proxy_session_id.clone(); - let replaced_proxy_session_id = self - .by_lease_id - .get(&lease.lease_id) - .map(|current| current.proxy_session_id.clone()) - .filter(|current| *current != new_proxy_session_id); - let advances = 1_u64.saturating_add(u64::from(replaced_proxy_session_id.is_some())); - if self.last_availability_evidence_generation.0.checked_add(advances).is_none() { - return false; - } - self.by_lease_id.insert(lease.lease_id.clone(), lease); - if self.advance_availability_evidence(&new_proxy_session_id).is_err() { - return false; - } - if let Some(old_proxy_session_id) = replaced_proxy_session_id { - if self.advance_availability_evidence(&old_proxy_session_id).is_err() { - return false; - } - } - true - } - - pub fn begin_runtime_policy_revocation( - &mut self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - reason: HlsRuntimeCustomTailReason, - now_ms: u64, - ) -> HlsRuntimePolicyRevocationOutcome { - let Some(lease) = self.by_lease_id.get(lease_id) else { - return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; - }; - if lease.proxy_session_id != *proxy_session_id { - return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; - } - if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { - return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; - } - let Some(lease) = self.by_lease_id.get(lease_id) else { - return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; - }; - if let HlsLeasePlaybackMode::TerminalTail(plan) = &lease.playback_mode { - return HlsRuntimePolicyRevocationOutcome::AlreadyCommitted { plan: Arc::clone(plan) }; - } - if lease.state == HlsAccessLeaseState::PolicyRevoking { - return lease - .runtime_policy_revocation - .clone() - .filter(|token| token.reason == reason) - .map_or(HlsRuntimePolicyRevocationOutcome::NoLongerEligible, |token| { - HlsRuntimePolicyRevocationOutcome::AlreadyPending { token } - }); - } - let Some(manifest) = lease.last_manifest_snapshot.as_ref() else { - return HlsRuntimePolicyRevocationOutcome::NoPublishedManifest; - }; - if lease.state != HlsAccessLeaseState::Activated || lease.playback_mode != HlsLeasePlaybackMode::Live { - return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; - } - let token = HlsRuntimePolicyRevocation { - reason, - lease_issued_at_ms: lease.issued_at_ms, - expected_admission_generation: lease.admission_generation, - manifest_snapshot_generation: manifest.snapshot_generation, - cursor_generation: lease.playback_cursor.cursor_generation, - started_at_ms: now_ms, - }; - if self.availability_evidence_can_advance().is_err() { - return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; - } - let lease_proxy_session_id = lease.proxy_session_id.clone(); - let Some(lease) = self.by_lease_id.get_mut(lease_id) else { - return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; - }; - lease.state = HlsAccessLeaseState::PolicyRevoking; - lease.runtime_policy_revocation = Some(token.clone()); - lease.runtime_policy_denial_reason = None; - if self.advance_availability_evidence(&lease_proxy_session_id).is_err() { - return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; - } - HlsRuntimePolicyRevocationOutcome::Started { token } - } - - pub fn fail_runtime_policy_revocation( - &mut self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - token: &HlsRuntimePolicyRevocation, - ) -> HlsAccessLeaseDenialOutcome { - let Some(lease) = self.by_lease_id.get(lease_id) else { - return HlsAccessLeaseDenialOutcome::UnknownLease; - }; - if lease.proxy_session_id != *proxy_session_id - || lease.state != HlsAccessLeaseState::PolicyRevoking - || lease.runtime_policy_revocation.as_ref() != Some(token) - { - return HlsAccessLeaseDenialOutcome::UnknownLease; - } - if self.availability_evidence_can_advance().is_err() { - return HlsAccessLeaseDenialOutcome::UnknownLease; - } - let lease_proxy_session_id = lease.proxy_session_id.clone(); - let Some(lease) = self.by_lease_id.get_mut(lease_id) else { - return HlsAccessLeaseDenialOutcome::UnknownLease; - }; - lease.state = HlsAccessLeaseState::Denied; - lease.runtime_policy_denial_reason = Some(token.reason); - lease.end_playback(); - if self.advance_availability_evidence(&lease_proxy_session_id).is_err() { - return HlsAccessLeaseDenialOutcome::UnknownLease; - } - HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } - } - - pub fn remove_access_lease(&mut self, lease_id: &HlsAccessLeaseId) -> Option { - let proxy_session_id = self.by_lease_id.get(lease_id)?.proxy_session_id.clone(); - if self.availability_evidence_can_advance().is_err() { - return None; - } - self.advance_availability_evidence(&proxy_session_id).ok()?; - let removed = self.by_lease_id.remove(lease_id)?; - Some(removed) - } - - /// Snapshots cleanup state without deleting the persistent cancellation-recovery ticket. - pub fn prepare_access_lease_removal( - &self, - lease_id: &HlsAccessLeaseId, - ) -> Option { - let lease = self.by_lease_id.get(lease_id)?; - Some(HlsAccessLeaseRemovalPreparation { - proxy_session_id: lease.proxy_session_id.clone(), - issued_at_ms: lease.issued_at_ms, - terminal_protection_generation: lease.pending_terminal_protection_release.or_else(|| { - if let HlsLeasePlaybackMode::TerminalTail(plan) = &lease.playback_mode { - Some(plan.generation) - } else { - None - } - }), - }) - } - - /// Deletes only the lease instance observed before its protection cleanup. - pub fn remove_access_lease_if_preparation_matches( - &mut self, - lease_id: &HlsAccessLeaseId, - preparation: &HlsAccessLeaseRemovalPreparation, - ) -> Option { - let lease = self.by_lease_id.get(lease_id)?; - if lease.proxy_session_id != preparation.proxy_session_id || lease.issued_at_ms != preparation.issued_at_ms { - return None; - } - self.remove_access_lease(lease_id) - } - - pub fn remove_access_leases_for_session(&mut self, proxy_session_id: &ProxySessionId) -> Vec { - let lease_ids = self - .by_lease_id - .values() - .filter(|lease| lease.proxy_session_id == *proxy_session_id) - .map(|lease| lease.lease_id.clone()) - .collect::>(); - if lease_ids.is_empty() || self.availability_evidence_can_advance().is_err() { - return Vec::new(); - } - if self.advance_availability_evidence(proxy_session_id).is_err() { - return Vec::new(); - } - lease_ids.into_iter().filter_map(|lease_id| self.by_lease_id.remove(&lease_id)).collect() - } - - pub fn clear(&mut self) -> usize { - let removed = self.by_lease_id.len(); - self.by_lease_id.clear(); - self.availability_generation_by_proxy_session.clear(); - removed - } - - pub fn len(&self) -> usize { self.by_lease_id.len() } - - pub fn is_empty(&self) -> bool { self.by_lease_id.is_empty() } - - pub fn first_username_for_session(&self, proxy_session_id: &ProxySessionId) -> Option { - self.by_lease_id - .values() - .find(|lease| lease.proxy_session_id == *proxy_session_id) - .map(|lease| lease.username.clone()) - } - - pub fn all_live_leases_terminal_for_session(&self, proxy_session_id: &ProxySessionId) -> bool { - let mut found = false; - for lease in self.by_lease_id.values().filter(|lease| { - lease.proxy_session_id == *proxy_session_id - && matches!( - lease.state, - HlsAccessLeaseState::Pending - | HlsAccessLeaseState::Activated - | HlsAccessLeaseState::Idle - | HlsAccessLeaseState::PolicyRevoking - | HlsAccessLeaseState::Denied - ) - }) { - found = true; - if !matches!( - lease.playback_mode, - HlsLeasePlaybackMode::TerminalTail(_) - | HlsLeasePlaybackMode::TerminalUnavailable { .. } - | HlsLeasePlaybackMode::Ended - ) { - return false; - } - } - found - } - - /// Returns the oldest sequence which every usable live lease has consumed. - /// - /// `None` deliberately keeps the canonical render protected when a lease - /// has not published or completed media yet. Terminal leases are excluded - /// here because their exact base media is protected by the terminal-tail - /// generation stored on the shared session. - pub fn capacity_release_through(&self, proxy_session_id: &ProxySessionId) -> Option { - let mut found_live_lease = false; - let mut release_through = None; - for lease in self.by_lease_id.values().filter(|lease| { - lease.proxy_session_id == *proxy_session_id - && lease_state_protects_live_evidence(lease.state) - && lease.playback_mode == HlsLeasePlaybackMode::Live - }) { - found_live_lease = true; - let snapshot = lease.last_manifest_snapshot.as_ref()?; - if snapshot.delivery_mode != HlsManifestDeliveryMode::NormalCacheTimeline { - return None; - } - let completed = lease.playback_cursor.highest_contiguous_completed_proxy_seq?; - let completed = completed.min(snapshot.last_proxy_seq); - release_through = Some(release_through.map_or(completed, |current: u64| current.min(completed))); - } - if found_live_lease { - release_through - } else { - None - } - } - - pub fn response_snapshot( - &mut self, - lease_id: &HlsAccessLeaseId, - path_proxy_session_id: &ProxySessionId, - now_ms: u64, - ) -> Option { - let lease = self.by_lease_id.get(lease_id)?; - if &lease.proxy_session_id != path_proxy_session_id { - return None; - } - if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { - return None; - } - self.by_lease_id.get(lease_id).cloned() - } - - /// Returns immutable playback evidence for leases that are actively consuming this shared session. - pub fn active_live_playback_snapshots_for_session( - &mut self, - proxy_session_id: &ProxySessionId, - now_ms: u64, - ) -> Vec { - if self.refresh_access_lease_validities_for_session(proxy_session_id, now_ms).is_err() { - return Vec::new(); - } - self.by_lease_id - .values() - .filter_map(|lease| { - if lease.proxy_session_id != *proxy_session_id { - return None; - } - (lease.state == HlsAccessLeaseState::Activated - && lease.active_until_ms.is_some_and(|active_until_ms| active_until_ms > now_ms) - && lease.playback_mode == HlsLeasePlaybackMode::Live - && lease.last_manifest_snapshot.is_some()) - .then(|| lease.clone()) - }) - .collect() - } - - pub fn prepare_manifest_publication( - &mut self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - now_ms: u64, - ) -> Option { - let lease = self.by_lease_id.get(lease_id)?; - if lease.proxy_session_id != *proxy_session_id { - return None; - } - if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { - return None; - } - self.by_lease_id.get(lease_id)?.manifest_publication_guard() - } - - pub(crate) fn published_transient_resource_ids( - &self, - lease_id: &HlsAccessLeaseId, - ) -> Option<&HlsPublishedTransientResourceIds> { - self.by_lease_id.get(lease_id).map(HlsAccessLease::published_transient_resource_ids) - } - - pub(crate) fn commit_manifest_publication_with_resources( - &mut self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - expected: HlsLeaseManifestPublicationGuard, - mut snapshot: HlsLeaseManifestSnapshot, - published_transient_resource_ids: HlsPublishedTransientResourceIds, - now_ms: u64, - ) -> HlsLeaseManifestPublicationOutcome { - let Some(lease) = self.by_lease_id.get(lease_id) else { - return HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::UnknownLease); - }; - if lease.proxy_session_id != *proxy_session_id { - return HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::SessionMismatch, - ); - } - if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { - return HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::SnapshotGenerationExhausted, - ); - } - let Some(lease) = self.by_lease_id.get(lease_id) else { - return HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::UnknownLease); - }; - if lease.state == HlsAccessLeaseState::Expired { - return HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::LeaseExpired); - } - if !lease_state_allows_use(lease.state) { - return HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::LeaseUnavailable, - ); - } - if lease.issued_at_ms != expected.issued_at_ms { - return HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::LeaseIncarnationChanged, - ); - } - if lease.admission_generation != expected.admission_generation { - return HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::AdmissionGenerationChanged, - ); - } - if lease.playback_mode != HlsLeasePlaybackMode::Live { - return HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::LeaseNotLive); - } - if lease - .last_manifest_snapshot - .as_ref() - .is_some_and(|current| manifest_source_is_regressive(current, &snapshot)) - { - return HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::SourceRegressive, - ); - } - if self.availability_evidence_can_advance().is_err() { - return HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::SnapshotGenerationExhausted, - ); - } - let Some(lease) = self.by_lease_id.get_mut(lease_id) else { - return HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::UnknownLease); - }; - let next_generation = lease.manifest_snapshot_generation.saturating_add(1); - if next_generation == lease.manifest_snapshot_generation { - return HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::SnapshotGenerationExhausted, - ); - } - lease.manifest_snapshot_generation = next_generation; - snapshot.snapshot_generation = next_generation; - let finalized_manifest_generation = snapshot.finalized_transient_manifest_generation; - lease.last_manifest_snapshot = Some(snapshot); - lease.published_transient_resource_ids = published_transient_resource_ids; - if let Some(manifest_generation) = finalized_manifest_generation { - lease.published_finalized_manifest_generations.insert(manifest_generation); - } - lease.startup_admission = HlsLeaseStartupAdmissionState::Admitted; - if self.advance_availability_evidence(proxy_session_id).is_err() { - return HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::SnapshotGenerationExhausted, - ); - } - HlsLeaseManifestPublicationOutcome::Committed { snapshot_generation: next_generation } - } - - #[cfg(any(test, feature = "test-support"))] - pub fn commit_manifest_publication( - &mut self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - expected: HlsLeaseManifestPublicationGuard, - snapshot: HlsLeaseManifestSnapshot, - now_ms: u64, - ) -> HlsLeaseManifestPublicationOutcome { - self.commit_manifest_publication_with_resources( - lease_id, - proxy_session_id, - expected, - snapshot, - HlsPublishedTransientResourceIds::default(), - now_ms, - ) - } - - pub fn media_identity_is_current( - &mut self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - expected: HlsMediaLeaseIdentity, - now_ms: u64, - ) -> bool { - let Some(lease) = self.by_lease_id.get(lease_id) else { - return false; - }; - if lease.proxy_session_id != *proxy_session_id || self.refresh_access_lease_validity(lease_id, now_ms).is_err() - { - return false; - } - self.by_lease_id.get(lease_id).is_some_and(|lease| { - lease_state_allows_use(lease.state) - && lease.issued_at_ms == expected.issued_at_ms - && lease.media_identity() == Some(expected) - }) - } - - pub fn record_segment_request_started_if_identity_matches( - &mut self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - expected: HlsMediaLeaseIdentity, - proxy_seq: u64, - requested_at_ms: u64, - ) -> Option { - let lease = self.by_lease_id.get(lease_id)?; - if lease.proxy_session_id != *proxy_session_id { - return None; - } - if self.refresh_access_lease_validity(lease_id, requested_at_ms).is_err() { - return None; - } - let lease = self.by_lease_id.get(lease_id)?; - if !lease_state_allows_use(lease.state) - || lease.issued_at_ms != expected.issued_at_ms - || lease.media_identity() != Some(expected) - || !matches!(expected.playback, HlsMediaLeasePlaybackIdentity::Live { .. }) - || lease.playback_cursor.cursor_generation == u64::MAX - || self.availability_evidence_can_advance().is_err() - { - return None; - } - let token = - self.by_lease_id.get_mut(lease_id)?.playback_cursor.record_request_started(proxy_seq, requested_at_ms); - self.advance_availability_evidence(proxy_session_id).ok()?; - Some(token) - } - - pub fn record_segment_request_completed_if_identity_matches( - &mut self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - expected: HlsMediaLeaseIdentity, - token: HlsPlaybackRequestToken, - completed_at_ms: u64, - ) -> Option { - let lease = self.by_lease_id.get(lease_id)?; - if lease.proxy_session_id != *proxy_session_id { - return None; - } - if self.refresh_access_lease_validity(lease_id, completed_at_ms).is_err() { - return None; - } - let lease = self.by_lease_id.get(lease_id)?; - if !lease_state_allows_use(lease.state) - || lease.issued_at_ms != expected.issued_at_ms - || lease.media_identity() != Some(expected) - || !matches!(expected.playback, HlsMediaLeasePlaybackIdentity::Live { .. }) - { - return None; - } - let mut cursor = lease.playback_cursor.clone(); - let outcome = cursor.record_request_completed(token, completed_at_ms); - if cursor.cursor_generation == lease.playback_cursor.cursor_generation { - return Some(outcome); - } - if self.availability_evidence_can_advance().is_err() { - return None; - } - self.by_lease_id.get_mut(lease_id)?.playback_cursor = cursor; - self.advance_availability_evidence(proxy_session_id).ok()?; - Some(outcome) - } - - pub fn prepare_terminal_tail( - &self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - input: &HlsTerminalTailPreparationInput, - ) -> Option { - let lease = self.by_lease_id.get(lease_id)?; - if lease.proxy_session_id != *proxy_session_id || lease.playback_mode != HlsLeasePlaybackMode::Live { - return None; - } - let mut manifest_snapshot = lease.last_manifest_snapshot.clone()?; - if manifest_snapshot.snapshot_generation != input.expected_manifest_snapshot_generation - || lease.playback_cursor.cursor_generation != input.expected_cursor_generation - { - return None; - } - let runtime_policy_revocation = match input.trigger { - HlsFiniteTailTrigger::AvailabilityReserve => None, - HlsFiniteTailTrigger::RuntimePolicy(reason) => { - let revocation = lease.runtime_policy_revocation.clone()?; - if lease.state != HlsAccessLeaseState::PolicyRevoking - || revocation.reason != reason - || revocation.lease_issued_at_ms != lease.issued_at_ms - || revocation.expected_admission_generation != lease.admission_generation - || revocation.manifest_snapshot_generation != manifest_snapshot.snapshot_generation - || revocation.cursor_generation != lease.playback_cursor.cursor_generation - { - return None; - } - if reason.base_policy() == HlsRuntimeCustomTailBasePolicy::PreserveCompletedOrInFlightPrefix { - manifest_snapshot = - runtime_policy_authorized_manifest_prefix(manifest_snapshot, &lease.playback_cursor)?; - } - Some(revocation) - } - }; - let expected_admission_generation = lease.admission_generation; - let decision_generation = expected_admission_generation.saturating_add(1); - if decision_generation == expected_admission_generation { - return None; - } - let terminal_media_requirement_source = match input.terminal_media_requirement_origin { - HlsTerminalMediaRequirementOrigin::AcceptanceEpisode { generation } - if generation == input.expected_acceptance_generation => - { - HlsTerminalMediaRequirementSource::AcceptanceEpisode { generation } - } - HlsTerminalMediaRequirementOrigin::CutoverSnapshot => { - HlsTerminalMediaRequirementSource::CutoverSnapshotPending { decision_generation } - } - HlsTerminalMediaRequirementOrigin::AcceptanceEpisode { .. } => return None, - }; - Some(HlsTerminalTailPreparation { - trigger: input.trigger, - runtime_policy_revocation, - lease_issued_at_ms: lease.issued_at_ms, - decision_generation, - expected_admission_generation, - manifest_snapshot_generation: manifest_snapshot.snapshot_generation, - cursor_generation: lease.playback_cursor.cursor_generation, - origin_progress_generation: input.origin_progress_generation, - media_readiness_generation: input.media_readiness_generation, - origin_epoch: input.origin_epoch, - last_media_progress_at_ms: input.last_media_progress_at_ms, - expected_acceptance_generation: input.expected_acceptance_generation, - terminal_media_requirement_source, - cutover_timing: input.cutover_timing, - commit_window: input.commit_window, - required_terminal_media_key: input.required_terminal_media_key, - terminal_media_preparation: input.terminal_media_preparation, - reserve: input.reserve, - manifest_snapshot, - }) - } - - pub fn commit_terminal_tail_if_generation_matches( - &mut self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - preparation: &HlsTerminalTailPreparation, - now_ms: u64, - plan: std::sync::Arc, - ) -> HlsTerminalCommitOutcome { - if !plan.matches_route(proxy_session_id, lease_id) - || plan.generation.0 != preparation.decision_generation - || plan.base_manifest.snapshot_generation != preparation.manifest_snapshot_generation - { - return HlsTerminalCommitOutcome::SupersededGeneration; - } - if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { - return HlsTerminalCommitOutcome::SupersededGeneration; - } - let Some(lease) = self.by_lease_id.get_mut(lease_id) else { - return HlsTerminalCommitOutcome::LeaseNoLongerEligible; - }; - if let Some(outcome) = terminal_commit_existing_outcome(lease, proxy_session_id, preparation, now_ms) { - return outcome; - } - if let Err(outcome) = lease.terminal_decision_commit_preconditions_match(proxy_session_id, preparation, now_ms) - { - return outcome; - } - if self.availability_evidence_can_advance().is_err() { - return HlsTerminalCommitOutcome::SupersededGeneration; - } - let Some(lease) = self.by_lease_id.get_mut(lease_id) else { - return HlsTerminalCommitOutcome::LeaseNoLongerEligible; - }; - lease.admission_generation = preparation.decision_generation; - lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(plan); - if let Some(revocation) = preparation.runtime_policy_revocation.as_ref() { - lease.state = HlsAccessLeaseState::Denied; - lease.runtime_policy_denial_reason = Some(revocation.reason); - lease.runtime_policy_revocation = None; - } - if self.advance_availability_evidence(proxy_session_id).is_err() { - return HlsTerminalCommitOutcome::SupersededGeneration; - } - HlsTerminalCommitOutcome::Committed - } - - pub fn terminal_tail_replay_outcome( - &mut self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - preparation: &HlsTerminalTailPreparation, - now_ms: u64, - ) -> Option { - self.refresh_access_lease_validity(lease_id, now_ms).ok()?; - let lease = self.by_lease_id.get_mut(lease_id)?; - terminal_commit_existing_outcome(lease, proxy_session_id, preparation, now_ms) - } - - pub fn commit_terminal_unavailable_if_generation_matches( - &mut self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - preparation: &HlsTerminalTailPreparation, - now_ms: u64, - reason: HlsTerminalTailCompatibility, - ) -> HlsTerminalCommitOutcome { - if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { - return HlsTerminalCommitOutcome::SupersededGeneration; - } - let Some(lease) = self.by_lease_id.get_mut(lease_id) else { - return HlsTerminalCommitOutcome::LeaseNoLongerEligible; - }; - if let Some(outcome) = terminal_commit_existing_outcome(lease, proxy_session_id, preparation, now_ms) { - return outcome; - } - if let Err(outcome) = lease.terminal_decision_commit_preconditions_match(proxy_session_id, preparation, now_ms) - { - return outcome; - } - if self.availability_evidence_can_advance().is_err() { - return HlsTerminalCommitOutcome::SupersededGeneration; - } - let Some(lease) = self.by_lease_id.get_mut(lease_id) else { - return HlsTerminalCommitOutcome::LeaseNoLongerEligible; - }; - lease.admission_generation = preparation.decision_generation; - lease.playback_mode = - HlsLeasePlaybackMode::TerminalUnavailable { decision_generation: preparation.decision_generation, reason }; - if let Some(revocation) = preparation.runtime_policy_revocation.as_ref() { - lease.state = HlsAccessLeaseState::Denied; - lease.runtime_policy_denial_reason = Some(revocation.reason); - lease.runtime_policy_revocation = None; - } - if self.advance_availability_evidence(proxy_session_id).is_err() { - return HlsTerminalCommitOutcome::SupersededGeneration; - } - HlsTerminalCommitOutcome::Committed - } - - pub fn terminal_unavailable_replay_outcome( - &mut self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - preparation: &HlsTerminalTailPreparation, - now_ms: u64, - ) -> Option { - self.refresh_access_lease_validity(lease_id, now_ms).ok()?; - let lease = self.by_lease_id.get_mut(lease_id)?; - terminal_commit_existing_outcome(lease, proxy_session_id, preparation, now_ms) - } - - pub fn access_lease( - &mut self, - lease_id: &HlsAccessLeaseId, - path_proxy_session_id: &ProxySessionId, - now_ms: u64, - ) -> Option { - let lease = self.by_lease_id.get(lease_id)?; - if &lease.proxy_session_id != path_proxy_session_id { - return None; - } - self.refresh_access_lease_validity(lease_id, now_ms).ok()?; - let state = self.by_lease_id.get(lease_id)?.state; - if state == HlsAccessLeaseState::Expired { - return None; - } - if matches!(state, HlsAccessLeaseState::PolicyRevoking | HlsAccessLeaseState::Denied) { - return self.by_lease_id.get(lease_id).cloned(); - } - if !lease_state_allows_use(state) { - return None; - } - self.by_lease_id.get(lease_id).cloned() - } - - pub fn update_origin_acquire_policy( - &mut self, - lease_id: &HlsAccessLeaseId, - connection_kind: ConnectionKind, - priority: i8, - ) -> Option { - let lease = self.by_lease_id.get_mut(lease_id)?; - if !lease_state_allows_use(lease.state) { - return None; - } - lease.update_origin_acquire_policy(connection_kind, priority); - Some(lease.clone()) - } - - pub fn activate_access_lease( - &mut self, - lease_id: &HlsAccessLeaseId, - path_proxy_session_id: &ProxySessionId, - now_ms: u64, - timing: HlsAccessLeaseTiming, - ) -> HlsAccessLeaseActivation { - let Some(new_lease) = self.by_lease_id.get(lease_id) else { - return HlsAccessLeaseActivation::UnknownLease; - }; - if &new_lease.proxy_session_id != path_proxy_session_id { - return HlsAccessLeaseActivation::SessionMismatch; - } - if !lease_state_allows_use(new_lease.state) { - return activation_for_state(new_lease.state); - } - if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { - return HlsAccessLeaseActivation::Expired; - } - let Some(new_lease) = self.by_lease_id.get(lease_id) else { - return HlsAccessLeaseActivation::UnknownLease; - }; - if new_lease.state == HlsAccessLeaseState::Expired { - return HlsAccessLeaseActivation::Expired; - } - let previous_state = new_lease.state; - let active_until_ms = Some(now_ms.saturating_add(timing.active_window_ms)); - let valid_until_ms = now_ms.saturating_add(timing.valid_window_ms); - let evidence_changed = new_lease.state != HlsAccessLeaseState::Activated - || new_lease.active_until_ms != active_until_ms - || new_lease.pending_deadline.is_some() - || new_lease.valid_until_ms != valid_until_ms; - if evidence_changed && self.availability_evidence_can_advance().is_err() { - return HlsAccessLeaseActivation::Expired; - } - let proxy_session_id = new_lease.proxy_session_id.clone(); - let Some(new_lease) = self.by_lease_id.get_mut(lease_id) else { - return HlsAccessLeaseActivation::UnknownLease; - }; - new_lease.state = HlsAccessLeaseState::Activated; - new_lease.last_seen_at_ms = now_ms; - new_lease.active_until_ms = active_until_ms; - new_lease.pending_deadline = None; - new_lease.valid_until_ms = valid_until_ms; - let lease = new_lease.clone(); - if evidence_changed && self.advance_availability_evidence(&proxy_session_id).is_err() { - return HlsAccessLeaseActivation::Expired; - } - - HlsAccessLeaseActivation::Activated { lease: Box::new(lease), previous_state } - } - - pub fn touch_manifest_access_lease( - &mut self, - lease_id: &HlsAccessLeaseId, - path_proxy_session_id: &ProxySessionId, - now_ms: u64, - active_timing: Option, - pending_deadline: Option, - valid_window_ms: u64, - ) -> HlsAccessLeaseTouch { - let Some(lease) = self.by_lease_id.get(lease_id) else { - return HlsAccessLeaseTouch::UnknownLease; - }; - if &lease.proxy_session_id != path_proxy_session_id { - return HlsAccessLeaseTouch::SessionMismatch; - } - if !lease_state_allows_use(lease.state) { - return touch_for_state(lease.state); - } - if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { - return HlsAccessLeaseTouch::Expired; - } - let Some(lease) = self.by_lease_id.get(lease_id) else { - return HlsAccessLeaseTouch::UnknownLease; - }; - if lease.state == HlsAccessLeaseState::Expired { - return HlsAccessLeaseTouch::Expired; - } - let before = (lease.active_until_ms, lease.pending_deadline, lease.valid_until_ms); - let proxy_session_id = lease.proxy_session_id.clone(); - if self.availability_evidence_can_advance().is_err() { - return HlsAccessLeaseTouch::Expired; - } - let Some(lease) = self.by_lease_id.get_mut(lease_id) else { - return HlsAccessLeaseTouch::UnknownLease; - }; - lease.last_seen_at_ms = now_ms; - match lease.state { - HlsAccessLeaseState::Pending => { - if let Some(pending_deadline) = pending_deadline { - lease.apply_pending_deadline(pending_deadline); - } - } - HlsAccessLeaseState::Activated => { - if let Some(timing) = active_timing { - lease.active_until_ms = Some(now_ms.saturating_add(timing.active_window_ms)); - lease.valid_until_ms = now_ms.saturating_add(timing.valid_window_ms); - } else { - lease.valid_until_ms = now_ms.saturating_add(valid_window_ms); - } - } - HlsAccessLeaseState::Idle => { - lease.valid_until_ms = now_ms.saturating_add(valid_window_ms); - } - HlsAccessLeaseState::PolicyRevoking | HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied => {} - } - let evidence_changed = before != (lease.active_until_ms, lease.pending_deadline, lease.valid_until_ms); - let lease = lease.clone(); - if evidence_changed && self.advance_availability_evidence(&proxy_session_id).is_err() { - return HlsAccessLeaseTouch::Expired; - } - HlsAccessLeaseTouch::Touched { lease: Box::new(lease) } - } - - pub fn mark_pending_manifest_follow_up_for_lease( - &mut self, - lease_id: &HlsAccessLeaseId, - path_proxy_session_id: &ProxySessionId, - now_ms: u64, - deadline: HlsAccessLeasePendingDeadline, - ) -> Option { - let lease = self.by_lease_id.get(lease_id)?; - if &lease.proxy_session_id != path_proxy_session_id { - return None; - } - self.refresh_access_lease_validity(lease_id, now_ms).ok()?; - let lease = self.by_lease_id.get(lease_id)?; - if lease.state != HlsAccessLeaseState::Pending { - return None; - } - let next_deadline = lease.pending_deadline.map_or(deadline, |current| current.tightened_with(deadline)); - if lease.pending_deadline == Some(next_deadline) { - return None; - } - if self.availability_evidence_can_advance().is_err() { - return None; - } - let proxy_session_id = lease.proxy_session_id.clone(); - let lease = self.by_lease_id.get_mut(lease_id)?; - lease.last_seen_at_ms = now_ms; - if !lease.apply_pending_deadline(deadline) { - return None; - } - let lease = lease.clone(); - self.advance_availability_evidence(&proxy_session_id).ok()?; - Some(lease) - } - - pub fn mark_pending_manifest_follow_up_for_session( - &mut self, - proxy_session_id: &ProxySessionId, - now_ms: u64, - deadline: HlsAccessLeasePendingDeadline, - ) -> Vec { - if self.refresh_access_lease_validities_for_session(proxy_session_id, now_ms).is_err() { - return Vec::new(); - } - if self.availability_evidence_can_advance().is_err() { - return Vec::new(); - } - let mut leases = Vec::new(); - for lease in self.by_lease_id.values_mut() { - if lease.proxy_session_id != *proxy_session_id { - continue; - } - if lease.state != HlsAccessLeaseState::Pending { - continue; - } - lease.last_seen_at_ms = now_ms; - if lease.apply_pending_deadline(deadline) { - leases.push(lease.clone()); - } - } - if !leases.is_empty() && self.advance_availability_evidence(proxy_session_id).is_err() { - return Vec::new(); - } - leases - } - - pub fn touch_access_lease( - &mut self, - lease_id: &HlsAccessLeaseId, - now_ms: u64, - timing: HlsAccessLeaseTiming, - ) -> bool { - self.touch_access_lease_snapshot(lease_id, now_ms, timing).is_some() - } - - pub fn touch_access_lease_snapshot( - &mut self, - lease_id: &HlsAccessLeaseId, - now_ms: u64, - timing: HlsAccessLeaseTiming, - ) -> Option { - let lease = self.by_lease_id.get(lease_id)?; - if lease.state != HlsAccessLeaseState::Activated { - return None; - } - self.refresh_access_lease_validity(lease_id, now_ms).ok()?; - let lease = self.by_lease_id.get(lease_id)?; - if lease.state == HlsAccessLeaseState::Expired { - return None; - } - let active_until_ms = Some(now_ms.saturating_add(timing.active_window_ms)); - let valid_until_ms = now_ms.saturating_add(timing.valid_window_ms); - let evidence_changed = lease.active_until_ms != active_until_ms || lease.valid_until_ms != valid_until_ms; - if evidence_changed && self.availability_evidence_can_advance().is_err() { - return None; - } - let proxy_session_id = lease.proxy_session_id.clone(); - let lease = self.by_lease_id.get_mut(lease_id)?; - lease.last_seen_at_ms = now_ms; - lease.active_until_ms = active_until_ms; - lease.valid_until_ms = valid_until_ms; - let lease = lease.clone(); - if evidence_changed { - self.advance_availability_evidence(&proxy_session_id).ok()?; - } - Some(lease) - } - - pub fn deny_access_lease( - &mut self, - lease_id: &HlsAccessLeaseId, - mode: HlsAccessLeaseDenialMode, - ) -> HlsAccessLeaseDenialOutcome { - let Some(lease) = self.by_lease_id.get(lease_id) else { - return HlsAccessLeaseDenialOutcome::UnknownLease; - }; - if lease.state == HlsAccessLeaseState::PolicyRevoking { - return HlsAccessLeaseDenialOutcome::PolicyRevocationPending; - } - let evidence_changed = lease.state != HlsAccessLeaseState::Denied; - if evidence_changed && self.availability_evidence_can_advance().is_err() { - return HlsAccessLeaseDenialOutcome::UnknownLease; - } - let proxy_session_id = lease.proxy_session_id.clone(); - let Some(lease) = self.by_lease_id.get_mut(lease_id) else { - return HlsAccessLeaseDenialOutcome::UnknownLease; - }; - if evidence_changed { - lease.state = HlsAccessLeaseState::Denied; - } - let preserve_finite_decision = matches!( - lease.playback_mode, - HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } - ); - if preserve_finite_decision { - if evidence_changed { - lease.runtime_policy_denial_reason = match &lease.playback_mode { - HlsLeasePlaybackMode::TerminalTail(plan) => Some(plan.reason), - HlsLeasePlaybackMode::TerminalUnavailable { .. } => lease.runtime_policy_denial_reason, - HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::Ended => None, - }; - } - if evidence_changed && self.advance_availability_evidence(&proxy_session_id).is_err() { - return HlsAccessLeaseDenialOutcome::UnknownLease; - } - return HlsAccessLeaseDenialOutcome::FiniteDecisionPreserved; - } - if evidence_changed { - lease.end_playback(); - lease.runtime_policy_denial_reason = match mode { - HlsAccessLeaseDenialMode::ImmediateRuntimePolicyEnd { reason } => Some(reason), - HlsAccessLeaseDenialMode::PreserveCommittedFiniteTail => None, - #[cfg(test)] - HlsAccessLeaseDenialMode::ImmediateEnd => None, - }; - } - let generation = lease.pending_terminal_protection_release; - if evidence_changed && self.advance_availability_evidence(&proxy_session_id).is_err() { - return HlsAccessLeaseDenialOutcome::UnknownLease; - } - HlsAccessLeaseDenialOutcome::Ended { - terminal_release: generation - .map(|generation| HlsDeniedTerminalTailRelease { proxy_session_id, generation }), - } - } - - /// Clears a persisted cleanup ticket only for the same ended lease/session/generation. - pub fn acknowledge_terminal_protection_release( - &mut self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - generation: HlsTerminalTailGeneration, - ) -> bool { - let Some(lease) = self.by_lease_id.get_mut(lease_id) else { - return false; - }; - if !matches!(lease.state, HlsAccessLeaseState::Denied | HlsAccessLeaseState::Expired) - || lease.proxy_session_id != *proxy_session_id - || lease.playback_mode != HlsLeasePlaybackMode::Ended - || lease.pending_terminal_protection_release != Some(generation) - { - return false; - } - lease.pending_terminal_protection_release = None; - true - } - - pub fn lease_state(&self, lease_id: &HlsAccessLeaseId, now_ms: u64) -> Option { - self.by_lease_id.get(lease_id).map(|lease| { - if lease.validity_due_at_ms() <= now_ms { - HlsAccessLeaseState::Expired - } else { - lease.state - } - }) - } - - pub fn active_access_lease_count_for_session(&mut self, proxy_session_id: &ProxySessionId, now_ms: u64) -> usize { - if self.refresh_access_lease_validities_for_session(proxy_session_id, now_ms).is_err() { - return 0; - } - let mut active_count = 0; - for lease in self.by_lease_id.values() { - if lease.proxy_session_id == *proxy_session_id - && lease.state == HlsAccessLeaseState::Activated - && lease.active_until_ms.is_some_and(|active_until| active_until > now_ms) - { - active_count += 1; - } - } - active_count - } - - pub fn has_usable_access_lease_for_session(&mut self, proxy_session_id: &ProxySessionId, now_ms: u64) -> bool { - if self.refresh_access_lease_validities_for_session(proxy_session_id, now_ms).is_err() { - return false; - } - let mut has_usable_lease = false; - for lease in self.by_lease_id.values() { - if lease.proxy_session_id == *proxy_session_id - && (lease.state == HlsAccessLeaseState::Pending - || lease.state == HlsAccessLeaseState::Idle - || (lease.state == HlsAccessLeaseState::Activated && lease.valid_until_ms > now_ms)) - { - has_usable_lease = true; - } - } - has_usable_lease - } - - pub fn session_snapshot( - &mut self, - proxy_session_id: &ProxySessionId, - now_ms: u64, - ) -> HlsAccessLeaseSessionSnapshot { - let mut active_count = 0; - let mut effective_origin_policy = None; - let mut idle_releases = Vec::new(); - let mut finalized_transient_manifest_bindings = Vec::new(); - let lease_ids = self - .by_lease_id - .values() - .filter(|lease| lease.proxy_session_id == *proxy_session_id) - .map(|lease| lease.lease_id.clone()) - .collect::>(); - for lease_id in lease_ids { - match self.refresh_access_lease_activity(&lease_id, now_ms) { - Ok(Some(release)) => idle_releases.push(release), - Ok(None) => {} - Err(HlsAvailabilityEvidenceAdvanceError::Exhausted) => continue, - } - let Some(lease) = self.by_lease_id.get(&lease_id) else { - continue; - }; - if lease.state == HlsAccessLeaseState::Activated { - active_count += 1; - } - if matches!(lease.state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated) { - let candidate = - HlsEffectiveOriginAcquirePolicy::new(lease.origin_connection_kind, lease.origin_priority, now_ms); - effective_origin_policy = Some(effective_origin_policy.map_or(candidate, |current| { - if candidate.is_better_than(current) { - candidate - } else { - current - } - })); - } - if lease_state_allows_use(lease.state) && lease.playback_mode == HlsLeasePlaybackMode::Live { - finalized_transient_manifest_bindings.extend( - lease.published_finalized_manifest_generations.iter().map(|manifest_generation| { - TransientManifestLeaseBinding::new( - lease.lease_id.clone(), - lease.issued_at_ms, - *manifest_generation, - ) - }), - ); - } - } - HlsAccessLeaseSessionSnapshot { - active_count, - effective_origin_policy, - idle_releases, - finalized_transient_manifest_bindings, - } - } - - pub fn lifecycle_snapshot( - &mut self, - lease_id: &HlsAccessLeaseId, - now_ms: u64, - ) -> Option { - let idle_release = self.refresh_access_lease_activity(lease_id, now_ms).ok()?; - let lease = self.by_lease_id.get(lease_id)?; - Some(HlsAccessLeaseLifecycleSnapshot { - lease_id: lease.lease_id.clone(), - proxy_session_id: lease.proxy_session_id.clone(), - state: lease.state, - active_until_ms: lease.active_until_ms, - pending_deadline: lease.pending_deadline, - valid_until_ms: lease.valid_until_ms, - idle_release, - }) - } -} - -pub fn new_hls_access_lease_id() -> HlsAccessLeaseId { - let mut bytes = [0u8; HLS_ACCESS_LEASE_ID_BYTES]; - if OsRng.try_fill_bytes(&mut bytes).is_err() { - rand::rng().fill_bytes(&mut bytes); - } - HlsAccessLeaseId(general_purpose::URL_SAFE_NO_PAD.encode(bytes)) -} - -fn manifest_source_is_regressive(current: &HlsLeaseManifestSnapshot, candidate: &HlsLeaseManifestSnapshot) -> bool { - candidate.source_commit_identity.commit_generation() < current.source_commit_identity.commit_generation() -} - -const fn lease_state_allows_use(state: HlsAccessLeaseState) -> bool { - matches!(state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle) -} - -const fn lease_state_protects_live_evidence(state: HlsAccessLeaseState) -> bool { - lease_state_allows_use(state) || matches!(state, HlsAccessLeaseState::PolicyRevoking) -} - -const fn activation_for_state(state: HlsAccessLeaseState) -> HlsAccessLeaseActivation { - match state { - HlsAccessLeaseState::Expired => HlsAccessLeaseActivation::Expired, - HlsAccessLeaseState::PolicyRevoking | HlsAccessLeaseState::Denied => HlsAccessLeaseActivation::Denied, - HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle => { - HlsAccessLeaseActivation::UnknownLease - } - } -} - -const fn touch_for_state(state: HlsAccessLeaseState) -> HlsAccessLeaseTouch { - match state { - HlsAccessLeaseState::Expired => HlsAccessLeaseTouch::Expired, - HlsAccessLeaseState::PolicyRevoking | HlsAccessLeaseState::Denied => HlsAccessLeaseTouch::Denied, - HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle => { - HlsAccessLeaseTouch::UnknownLease - } - } -} - -fn runtime_policy_authorized_manifest_prefix( - mut manifest: HlsLeaseManifestSnapshot, - cursor: &HlsLeasePlaybackCursor, -) -> Option { - let authorized_last_proxy_seq = - match (cursor.highest_contiguous_completed_proxy_seq, cursor.last_requested_proxy_seq) { - (Some(completed), Some(requested)) => completed.max(requested), - (Some(completed), None) => completed, - (None, Some(requested)) => requested, - (None, None) => return None, - } - .min(manifest.last_proxy_seq); - let end = manifest - .visible_segments - .iter() - .position(|segment| segment.proxy_seq > authorized_last_proxy_seq) - .unwrap_or(manifest.visible_segments.len()); - let selected = manifest.visible_segments.get(..end)?.to_vec(); - let first = selected.first()?; - let last = selected.last()?; - let first_proxy_seq = first.proxy_seq; - let last_proxy_seq = last.proxy_seq; - let active_encryption = last.encryption.clone(); - let playlist_duration_ms = selected.iter().fold(0_u64, |total, segment| total.saturating_add(segment.duration_ms)); - manifest.first_proxy_seq = first_proxy_seq; - manifest.last_proxy_seq = last_proxy_seq; - manifest.visible_segments = Arc::from(selected); - manifest.playlist_duration_ms = playlist_duration_ms; - manifest.last_visible_media_end_ms = playlist_duration_ms; - manifest.active_encryption = active_encryption; - Some(manifest) -} - #[cfg(test)] -mod tests { - use super::{ - super::{ - build_terminal_tail_plan, - manifest_acceptance::HlsManifestAcceptanceGeneration, - media_reserve::{ - HlsLeaseReserveAvailabilityBasis, HlsLeaseReserveSnapshot, HlsManifestCommitIdentity, - HlsPlaybackCompletionOutcome, - }, - recovery_timing::{ - HlsLeaseCutoverTiming, HlsTerminalCommitWindow, HlsTerminalMediaPreparationState, HlsTransitionMarginMs, - }, - runtime_custom_tail::{ - HlsFiniteTailTrigger, HlsRuntimeCustomTailAssetIdentity, HlsRuntimeCustomTailReason, - }, - terminal_commit::HlsTerminalCommitOutcome, - terminal_tail::{ - snapshot_terminal_media_asset, HlsLeasePlaybackMode, HlsTerminalTailCompatibility, HlsTerminalTailPlan, - }, - HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsManifestDeliveryMode, HlsMediaContainer, - HlsTerminalAssetIdentity, HlsTerminalBaseMediaState, HlsTerminalBaseProtection, - HlsTerminalBaseSegmentAvailability, HlsTerminalTailBuildInput, HlsTerminalTailGeneration, - }, - new_hls_access_lease_id, HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseDenialMode, - HlsAccessLeaseDenialOutcome, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseState, - HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsAvailabilityEvidenceGeneration, - HlsLeaseManifestPublicationOutcome, HlsLeaseManifestPublicationRejectReason, HlsPlaybackFamilyKey, - HlsRuntimePolicyRevocationOutcome, HlsTerminalMediaRequirementOrigin, HlsTerminalTailPreparationInput, - }; - use crate::ProxySessionId; - use std::sync::Arc; - use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer; - use tuliprox_session::ConnectionKind; - - const TERMINAL_ASSET_BYTES: &[u8] = - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); - - fn cutover_reserve() -> HlsLeaseReserveSnapshot { - HlsLeaseReserveSnapshot { - availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, - guaranteed_media_horizon_ms: 12_000, - conservative_playback_position_ms: 12_000, - guaranteed_reserve_ms: 0, - initial_hidden_ready_duration_ms: 0, - transition_margin: HlsTransitionMarginMs::from_millis(12_000), - key_readiness_valid_until_ms: None, - recovery_required: true, - cutover_required: true, - } - } - - fn cutover_timing() -> HlsLeaseCutoverTiming { - HlsLeaseCutoverTiming::from_reserve(2_000, 0, HlsTransitionMarginMs::from_millis(12_000), None) - } - - fn lease(lease_id: HlsAccessLeaseId, proxy_session_id: &str, now_ms: u64) -> HlsAccessLease { - HlsAccessLease::pending( - lease_id, - HlsPlaybackFamilyKey::new("alice", "client-a"), - ProxySessionId(proxy_session_id.to_string()), - "alice".to_string(), - "session-a".to_string(), - 1, - "12345".to_string(), - 12345, - now_ms, - 15_000, - ) - } - - const fn timing(active_window_ms: u64, valid_window_ms: u64) -> HlsAccessLeaseTiming { - HlsAccessLeaseTiming { active_window_ms, valid_window_ms } - } - - #[test] - fn known_bitrate_builder_normalizes_zero_and_survives_response_snapshot() { - let proxy_session_id = ProxySessionId("proxy-a".to_string()); - let zero_lease_id = HlsAccessLeaseId("lease-zero".to_string()); - let measured_lease_id = HlsAccessLeaseId("lease-measured".to_string()); - let mut store = HlsAccessLeaseStore::default(); - assert!(store.prepare_access_lease( - lease(zero_lease_id.clone(), &proxy_session_id.0, 1_000).with_known_bitrate_bps(Some(0)) - )); - assert!(store.prepare_access_lease( - lease(measured_lease_id.clone(), &proxy_session_id.0, 1_000).with_known_bitrate_bps(Some(2_500_000)), - )); - - assert_eq!( - store - .response_snapshot(&zero_lease_id, &proxy_session_id, 2_000) - .expect("zero bitrate lease") - .known_bitrate_bps, - None - ); - assert_eq!( - store - .response_snapshot(&measured_lease_id, &proxy_session_id, 2_000) - .expect("measured bitrate lease") - .known_bitrate_bps, - Some(2_500_000) - ); - } - - #[test] - fn standalone_tail_policy_is_limited_to_pending_unpublished_lease() { - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let mut candidate = lease(lease_id, "proxy-a", 1_000); - - assert!(candidate.permits_unpublished_standalone_tail(HlsRuntimeCustomTailReason::ChannelUnavailable)); - assert!(!candidate.permits_unpublished_standalone_tail(HlsRuntimeCustomTailReason::SessionOrLeaseExpired)); - - candidate.last_manifest_snapshot = Some(manifest_snapshot(1)); - candidate.startup_admission = super::HlsLeaseStartupAdmissionState::Admitted; - assert!(!candidate.permits_unpublished_standalone_tail(HlsRuntimeCustomTailReason::ChannelUnavailable)); - - let mut activated = lease(HlsAccessLeaseId("lease-b".to_string()), "proxy-a", 1_000); - activated.state = HlsAccessLeaseState::Activated; - assert!(!activated.permits_unpublished_standalone_tail(HlsRuntimeCustomTailReason::ChannelUnavailable)); - - let mut terminal = lease(HlsAccessLeaseId("lease-c".to_string()), "proxy-a", 1_000); - terminal.playback_mode = HlsLeasePlaybackMode::TerminalUnavailable { - decision_generation: 1, - reason: HlsTerminalTailCompatibility::MissingAsset, - }; - assert!(!terminal.permits_unpublished_standalone_tail(HlsRuntimeCustomTailReason::ChannelUnavailable)); - } - - fn manifest_snapshot(source_rendered_at_ms: u64) -> HlsLeaseManifestSnapshot { - HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(source_rendered_at_ms), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 0, - delivered_at_ms: 2_000, - first_proxy_seq: 40, - last_proxy_seq: 41, - visible_segments: Arc::from([ - HlsLeaseManifestSegment { - proxy_seq: 40, - duration_ms: 6_000, - uri: "/live/40.ts".to_string().into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }, - HlsLeaseManifestSegment { - proxy_seq: 41, - duration_ms: 6_000, - uri: "/live/41.ts".to_string().into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }, - ]), - discontinuity_sequence: 3, - target_duration_ms: 12_000, - playlist_duration_ms: 12_000, - last_visible_media_end_ms: 12_000, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - } - } - - fn manifest_snapshot_for_route( - source_rendered_at_ms: u64, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, - ) -> HlsLeaseManifestSnapshot { - let mut snapshot = manifest_snapshot(source_rendered_at_ms); - for segment in Arc::make_mut(&mut snapshot.visible_segments) { - segment.uri = - format!("/hls/shared/live/{}/{}/{}.ts", proxy_session_id.0, lease_id.0, segment.proxy_seq).into(); - } - snapshot - } - - fn publish_manifest_snapshot( - store: &mut HlsAccessLeaseStore, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - snapshot: HlsLeaseManifestSnapshot, - now_ms: u64, - ) -> HlsLeaseManifestPublicationOutcome { - let guard = - store.prepare_manifest_publication(lease_id, proxy_session_id, now_ms).expect("live manifest publication"); - store.commit_manifest_publication(lease_id, proxy_session_id, guard, snapshot, now_ms) - } - - #[test] - fn repair_prewarm_guard_rejects_newer_publication_and_terminal_lease() { - let proxy_session_id = ProxySessionId("proxy-a".to_string()); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let mut store = HlsAccessLeaseStore::default(); - assert!(store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000))); - let first_generation = - publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000) - .snapshot_generation() - .expect("first publication"); - assert!(store.repair_prewarm_is_current(&lease_id, &proxy_session_id, 1_000, first_generation,)); - - let second_generation = - publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(2), 2_100) - .snapshot_generation() - .expect("second publication"); - assert!(!store.repair_prewarm_is_current(&lease_id, &proxy_session_id, 1_000, first_generation,)); - assert!(store.repair_prewarm_is_current(&lease_id, &proxy_session_id, 1_000, second_generation,)); - - store.by_lease_id.get_mut(&lease_id).expect("lease").playback_mode = HlsLeasePlaybackMode::Ended; - assert!(!store.repair_prewarm_is_current(&lease_id, &proxy_session_id, 1_000, second_generation,)); - } - - fn terminal_plan( - generation: u64, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, - ) -> Arc { - terminal_plan_at(generation, proxy_session_id, lease_id, 3_000) - } - - fn terminal_plan_at( - generation: u64, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, - created_at_ms: u64, - ) -> Arc { - let mut base_manifest = manifest_snapshot_for_route(1, proxy_session_id, lease_id); - base_manifest.snapshot_generation = 1; - let transport_stream = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); - let asset = snapshot_terminal_media_asset(&transport_stream).expect("terminal asset"); - let expected_asset = - HlsRuntimeCustomTailAssetIdentity::channel_unavailable(HlsTerminalAssetIdentity::from_asset(&asset)); - let availability = Arc::from( - base_manifest - .visible_proxy_seqs() - .map(|proxy_seq| HlsTerminalBaseSegmentAvailability { - proxy_seq, - media_state: HlsTerminalBaseMediaState::Ready, - required_map_ready: true, - required_key_ready: true, - protection: HlsTerminalBaseProtection::Protectable, - }) - .collect::>(), - ); - let base_track_signature = Some(asset.track_signature().clone()); - let anchored_bundle = - HlsTerminalTailBuildInput::anchored_bundle_for_test(&asset, base_manifest.target_duration_ms); - let base_timing = Some(HlsTerminalTailBuildInput::base_timing_for_test(&asset, &base_manifest)); - let base_splice_evidence = Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); - let terminal_splice_evidence = base_splice_evidence.clone(); - Arc::new( - build_terminal_tail_plan(HlsTerminalTailBuildInput { - generation: HlsTerminalTailGeneration(generation), - created_at_ms, - base_manifest, - base_availability: availability, - base_track_signature, - base_splice_evidence, - terminal_splice_evidence, - base_timing, - base_key_bindings: Arc::from([]), - expected_asset, - asset, - anchored_bundle, - }) - .expect("compatible terminal plan"), - ) - } - - fn prepare_terminal( - store: &mut HlsAccessLeaseStore, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - origin_progress_generation: u64, - last_media_progress_at_ms: Option, - ) -> super::HlsTerminalTailPreparation { - let lease = store.response_snapshot(lease_id, proxy_session_id, 2_000).expect("test lease snapshot"); - let snapshot_generation = - lease.last_manifest_snapshot.as_ref().expect("test manifest snapshot").snapshot_generation; - store - .prepare_terminal_tail( - lease_id, - proxy_session_id, - &HlsTerminalTailPreparationInput { - trigger: HlsFiniteTailTrigger::AvailabilityReserve, - expected_manifest_snapshot_generation: snapshot_generation, - expected_cursor_generation: lease.playback_cursor.cursor_generation, - origin_progress_generation, - media_readiness_generation: 0, - origin_epoch: 0, - last_media_progress_at_ms, - expected_acceptance_generation: HlsManifestAcceptanceGeneration(origin_progress_generation), - terminal_media_requirement_origin: HlsTerminalMediaRequirementOrigin::AcceptanceEpisode { - generation: HlsManifestAcceptanceGeneration(origin_progress_generation), - }, - cutover_timing: cutover_timing(), - commit_window: HlsTerminalCommitWindow::CutoverDue, - required_terminal_media_key: None, - terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, - reserve: cutover_reserve(), - }, - ) - .expect("terminal preparation") - } - - fn prepare_runtime_policy_terminal( - store: &HlsAccessLeaseStore, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - reason: HlsRuntimeCustomTailReason, - ) -> super::HlsTerminalTailPreparation { - let lease = store.by_lease_id.get(lease_id).expect("runtime policy lease"); - let manifest = lease.last_manifest_snapshot.as_ref().expect("published runtime policy manifest"); - store - .prepare_terminal_tail( - lease_id, - proxy_session_id, - &HlsTerminalTailPreparationInput { - trigger: HlsFiniteTailTrigger::RuntimePolicy(reason), - expected_manifest_snapshot_generation: manifest.snapshot_generation, - expected_cursor_generation: lease.playback_cursor.cursor_generation, - origin_progress_generation: 4, - media_readiness_generation: 5, - origin_epoch: 6, - last_media_progress_at_ms: Some(2_000), - expected_acceptance_generation: HlsManifestAcceptanceGeneration(4), - terminal_media_requirement_origin: HlsTerminalMediaRequirementOrigin::CutoverSnapshot, - cutover_timing: cutover_timing(), - commit_window: HlsTerminalCommitWindow::CutoverDue, - required_terminal_media_key: None, - terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, - reserve: cutover_reserve(), - }, - ) - .expect("runtime policy terminal preparation") - } - - fn commit_prepared_terminal( - store: &mut HlsAccessLeaseStore, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - preparation: &super::HlsTerminalTailPreparation, - ) -> HlsTerminalCommitOutcome { - store.commit_terminal_tail_if_generation_matches( - lease_id, - proxy_session_id, - preparation, - 3_000, - terminal_plan(preparation.decision_generation, proxy_session_id, lease_id), - ) - } - - #[test] - fn access_lease_id_is_short_and_opaque() { - let lease_id = new_hls_access_lease_id(); - - assert_eq!(lease_id.0.len(), 22); - assert!(!lease_id.0.contains("alice")); - assert!(!lease_id.0.contains("session")); - } - - #[test] - fn archive_playback_context_is_opt_in() { - let live = lease(HlsAccessLeaseId("live".to_string()), "proxy-live", 1_000); - assert_eq!(live.epg_reference_ts, None); - assert_eq!(live.archive_origin_url, None); - - let archive = lease(HlsAccessLeaseId("archive".to_string()), "proxy-archive", 1_000).with_archive_playback( - Some(1_784_898_000), - Some("http://provider/channel/timeshift_abs-1784898000.m3u8".to_string()), - ); - assert_eq!(archive.epg_reference_ts, Some(1_784_898_000)); - assert_eq!( - archive.archive_origin_url.as_deref(), - Some("http://provider/channel/timeshift_abs-1784898000.m3u8") - ); - } - - #[test] - fn policy_revocation_token_is_reason_and_generation_bound() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("policy-cas".to_string()); - let proxy_session_id = ProxySessionId("proxy-policy-cas".to_string()); - assert!(store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000))); - assert!(publish_manifest_snapshot( - &mut store, - &lease_id, - &proxy_session_id, - manifest_snapshot_for_route(1, &proxy_session_id, &lease_id), - 2_000, - ) - .is_committed()); - assert!(store - .activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(10_000, 15_000)) - .is_activated()); - let identity = store - .response_snapshot(&lease_id, &proxy_session_id, 2_000) - .and_then(|lease| lease.media_identity()) - .expect("live identity"); - assert!(store - .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 40, 2_100,) - .is_some()); - - let started = store.begin_runtime_policy_revocation( - &lease_id, - &proxy_session_id, - HlsRuntimeCustomTailReason::UserConnectionsExhausted, - 2_200, - ); - let HlsRuntimePolicyRevocationOutcome::Started { token } = started else { - panic!("expected a new policy revocation, got {started:?}"); - }; - assert_eq!( - store.begin_runtime_policy_revocation( - &lease_id, - &proxy_session_id, - HlsRuntimeCustomTailReason::UserConnectionsExhausted, - 2_201, - ), - HlsRuntimePolicyRevocationOutcome::AlreadyPending { token: token.clone() } - ); - assert_eq!( - store.begin_runtime_policy_revocation( - &lease_id, - &proxy_session_id, - HlsRuntimeCustomTailReason::UserAccountExpired, - 2_202, - ), - HlsRuntimePolicyRevocationOutcome::NoLongerEligible - ); - let preparation = prepare_runtime_policy_terminal( - &store, - &lease_id, - &proxy_session_id, - HlsRuntimeCustomTailReason::UserConnectionsExhausted, - ); - assert_eq!(preparation.runtime_policy_revocation, Some(token)); - - store.by_lease_id.get_mut(&lease_id).expect("policy lease").admission_generation = - preparation.expected_admission_generation.saturating_add(1); - assert_eq!( - store.commit_terminal_unavailable_if_generation_matches( - &lease_id, - &proxy_session_id, - &preparation, - 2_300, - HlsTerminalTailCompatibility::MissingAsset, - ), - HlsTerminalCommitOutcome::SupersededGeneration - ); - let lease = store.response_snapshot(&lease_id, &proxy_session_id, 2_300).expect("revoking lease"); - assert_eq!(lease.state, HlsAccessLeaseState::PolicyRevoking); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - } - - #[test] - fn unpublished_runtime_policy_denial_retains_first_reason_without_live_access() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("cold-policy-denial".to_string()); - let proxy_session_id = ProxySessionId("proxy-cold-policy-denial".to_string()); - assert!(store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000))); - assert!(store - .activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(10_000, 15_000)) - .is_activated()); - assert_eq!( - store.begin_runtime_policy_revocation( - &lease_id, - &proxy_session_id, - HlsRuntimeCustomTailReason::UserConnectionsExhausted, - 2_100, - ), - HlsRuntimePolicyRevocationOutcome::NoPublishedManifest - ); - assert_eq!( - store.deny_access_lease( - &lease_id, - HlsAccessLeaseDenialMode::ImmediateRuntimePolicyEnd { - reason: HlsRuntimeCustomTailReason::UserConnectionsExhausted, - }, - ), - HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } - ); - let denied = store.response_snapshot(&lease_id, &proxy_session_id, 2_100).expect("denied lease"); - assert_eq!(denied.state, HlsAccessLeaseState::Denied); - assert_eq!(denied.playback_mode, HlsLeasePlaybackMode::Ended); - assert_eq!(denied.runtime_policy_denial_reason(), Some(HlsRuntimeCustomTailReason::UserConnectionsExhausted)); - - assert_eq!( - store.deny_access_lease( - &lease_id, - HlsAccessLeaseDenialMode::ImmediateRuntimePolicyEnd { - reason: HlsRuntimeCustomTailReason::UserAccountExpired, - }, - ), - HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } - ); - let repeated = store.response_snapshot(&lease_id, &proxy_session_id, 2_100).expect("retained denied lease"); - assert_eq!(repeated.runtime_policy_denial_reason(), Some(HlsRuntimeCustomTailReason::UserConnectionsExhausted)); - } - - #[test] - fn user_revocation_tail_does_not_authorize_unfetched_live_suffix() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("policy-prefix".to_string()); - let proxy_session_id = ProxySessionId("proxy-policy-prefix".to_string()); - let mut snapshot = manifest_snapshot_for_route(1, &proxy_session_id, &lease_id); - let mut segments = snapshot.visible_segments.to_vec(); - for proxy_seq in [42_u64, 43] { - segments.push(HlsLeaseManifestSegment { - proxy_seq, - duration_ms: 6_000, - uri: format!("/hls/shared/live/{}/{}/{}.ts", proxy_session_id.0, lease_id.0, proxy_seq).into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }); - } - snapshot.visible_segments = Arc::from(segments); - snapshot.last_proxy_seq = 43; - snapshot.playlist_duration_ms = 24_000; - snapshot.last_visible_media_end_ms = 24_000; - assert!(store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000))); - assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, snapshot, 2_000).is_committed()); - assert!(store - .activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(10_000, 15_000)) - .is_activated()); - let identity = store - .response_snapshot(&lease_id, &proxy_session_id, 2_000) - .and_then(|lease| lease.media_identity()) - .expect("live identity"); - let completed = store - .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 40, 2_100) - .expect("completed request token"); - assert_eq!( - store.record_segment_request_completed_if_identity_matches( - &lease_id, - &proxy_session_id, - identity, - completed, - 2_150, - ), - Some(HlsPlaybackCompletionOutcome::Advanced) - ); - assert!(store - .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 41, 2_175,) - .is_some()); - assert!(matches!( - store.begin_runtime_policy_revocation( - &lease_id, - &proxy_session_id, - HlsRuntimeCustomTailReason::UserConnectionsExhausted, - 2_200, - ), - HlsRuntimePolicyRevocationOutcome::Started { .. } - )); - assert!(store - .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 42, 2_201,) - .is_none()); - - let preparation = prepare_runtime_policy_terminal( - &store, - &lease_id, - &proxy_session_id, - HlsRuntimeCustomTailReason::UserConnectionsExhausted, - ); - assert_eq!(preparation.manifest_snapshot.first_proxy_seq, 40); - assert_eq!(preparation.manifest_snapshot.last_proxy_seq, 41); - assert_eq!(preparation.manifest_snapshot.visible_proxy_seqs().collect::>(), vec![40, 41]); - } - - #[test] - fn access_lease_activates_and_slides_validity() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - - assert!(store - .activate_access_lease(&lease_id, &proxy_session_id, 10_000, timing(5_000, 30_000)) - .is_activated()); - assert_eq!(store.lease_state(&lease_id, 24_999), Some(HlsAccessLeaseState::Activated)); - assert!(store.touch_access_lease(&lease_id, 24_000, timing(5_000, 30_000))); - assert_eq!(store.lease_state(&lease_id, 53_999), Some(HlsAccessLeaseState::Activated)); - } - - #[test] - fn active_playback_snapshot_excludes_pending_idle_and_terminal_leases() { - let mut store = HlsAccessLeaseStore::default(); - let proxy_session_id = ProxySessionId("proxy".to_string()); - let active_id = HlsAccessLeaseId("active".to_string()); - let pending_id = HlsAccessLeaseId("pending".to_string()); - let terminal_id = HlsAccessLeaseId("terminal".to_string()); - for lease_id in [&active_id, &pending_id, &terminal_id] { - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - assert!(publish_manifest_snapshot(&mut store, lease_id, &proxy_session_id, manifest_snapshot(1), 2_000) - .is_committed()); - } - assert!(store - .activate_access_lease(&active_id, &proxy_session_id, 2_000, timing(5_000, 30_000)) - .is_activated()); - assert!(store - .activate_access_lease(&terminal_id, &proxy_session_id, 2_000, timing(5_000, 30_000)) - .is_activated()); - let preparation = prepare_terminal(&mut store, &terminal_id, &proxy_session_id, 1, Some(1_000)); - assert_eq!( - commit_prepared_terminal(&mut store, &terminal_id, &proxy_session_id, &preparation), - HlsTerminalCommitOutcome::Committed - ); - - let snapshots = store.active_live_playback_snapshots_for_session(&proxy_session_id, 2_100); - - assert_eq!(snapshots.len(), 1); - assert_eq!(snapshots[0].lease_id, active_id); - } - - #[test] - fn access_lease_idles_at_exact_active_until_boundary() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 30_000)).is_activated()); - - let snapshot = store.lifecycle_snapshot(&lease_id, 7_000).expect("lease should exist"); - - assert_eq!(snapshot.state, HlsAccessLeaseState::Idle); - assert!(snapshot.idle_release.is_some()); - assert_eq!(store.active_access_lease_count_for_session(&proxy_session_id, 7_000), 0); - } - - #[test] - fn access_lease_expires_at_exact_valid_until_boundary() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 15_000)).is_activated()); - - let snapshot = store.lifecycle_snapshot(&lease_id, 17_000).expect("lease should exist"); - - assert_eq!(snapshot.state, HlsAccessLeaseState::Expired); - assert!(snapshot.idle_release.is_some()); - assert_eq!(store.lease_state(&lease_id, 17_000), Some(HlsAccessLeaseState::Expired)); - } - - #[test] - fn access_lease_expires_without_activity() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - - assert_eq!( - store.activate_access_lease(&lease_id, &proxy_session_id, 17_000, timing(5_000, 15_000)), - HlsAccessLeaseActivation::Expired - ); - } - - #[test] - fn same_family_leases_remain_independently_valid() { - let mut store = HlsAccessLeaseStore::default(); - let old_lease_id = HlsAccessLeaseId("old".to_string()); - let new_lease_id = HlsAccessLeaseId("new".to_string()); - let proxy_a = ProxySessionId("proxy-a".to_string()); - let proxy_b = ProxySessionId("proxy-b".to_string()); - let family = HlsPlaybackFamilyKey::new("alice", "client-a"); - - store.prepare_access_lease(HlsAccessLease::pending( - old_lease_id.clone(), - family.clone(), - proxy_a.clone(), - "alice".to_string(), - "session-a".to_string(), - 1, - "12345".to_string(), - 12345, - 1_000, - 15_000, - )); - assert!(store.activate_access_lease(&old_lease_id, &proxy_a, 2_000, timing(5_000, 15_000)).is_activated()); - store.prepare_access_lease(HlsAccessLease::pending( - new_lease_id.clone(), - family, - proxy_b.clone(), - "alice".to_string(), - "session-b".to_string(), - 1, - "67890".to_string(), - 67890, - 3_000, - 15_000, - )); - - let activation = store.activate_access_lease(&new_lease_id, &proxy_b, 4_000, timing(5_000, 15_000)); - assert!(activation.is_activated()); - assert_eq!(store.lease_state(&old_lease_id, 4_000), Some(HlsAccessLeaseState::Activated)); - assert!(store.touch_access_lease(&old_lease_id, 5_000, timing(5_000, 15_000))); - assert_eq!(store.lease_state(&old_lease_id, 19_999), Some(HlsAccessLeaseState::Activated)); - } - - #[test] - fn manifest_touch_extends_activated_lease_active_window() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 15_000)).is_activated()); - - assert!(matches!( - store.touch_manifest_access_lease( - &lease_id, - &proxy_session_id, - 6_000, - Some(timing(10_000, 30_000)), - None, - 15_000, - ), - HlsAccessLeaseTouch::Touched { .. } - )); - let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); - assert_eq!(lease.state, HlsAccessLeaseState::Activated); - assert_eq!(lease.last_seen_at_ms, 6_000); - assert_eq!(lease.pending_deadline, None); - assert_eq!(lease.active_until_ms, Some(16_000)); - assert_eq!(lease.valid_until_ms, 36_000); - } - - #[test] - fn pending_lease_expires_at_pending_deadline_even_when_valid_window_is_longer() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - let mut lease = lease(lease_id.clone(), &proxy_session_id.0, 1_000); - lease.pending_deadline = Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 6_000 }); - lease.valid_until_ms = 31_000; - store.prepare_access_lease(lease); - - assert_eq!(store.lease_state(&lease_id, 5_999), Some(HlsAccessLeaseState::Pending)); - let snapshot = store.lifecycle_snapshot(&lease_id, 6_000).expect("lease should exist"); - assert_eq!(snapshot.state, HlsAccessLeaseState::Expired); - assert!(snapshot.idle_release.is_some(), "pending expiry must release counted user admission"); - assert_eq!(store.lease_state(&lease_id, 6_000), Some(HlsAccessLeaseState::Expired)); - } - - #[test] - fn manifest_touch_can_shorten_pending_lease_to_follow_up_deadline() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - - assert!(matches!( - store.touch_manifest_access_lease( - &lease_id, - &proxy_session_id, - 2_000, - None, - Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 }), - 300_000, - ), - HlsAccessLeaseTouch::Touched { .. } - )); - - let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); - assert_eq!(lease.state, HlsAccessLeaseState::Pending); - assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); - assert_eq!(lease.valid_until_ms, 12_000); - assert_eq!(store.lease_state(&lease_id, 11_999), Some(HlsAccessLeaseState::Pending)); - assert_eq!(store.lease_state(&lease_id, 12_000), Some(HlsAccessLeaseState::Expired)); - } - - #[test] - fn bootstrap_touch_cannot_extend_existing_follow_up_pending_deadline() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - - assert!(matches!( - store.touch_manifest_access_lease( - &lease_id, - &proxy_session_id, - 2_000, - None, - Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 }), - 300_000, - ), - HlsAccessLeaseTouch::Touched { .. } - )); - assert!(matches!( - store.touch_manifest_access_lease( - &lease_id, - &proxy_session_id, - 3_000, - None, - Some(HlsAccessLeasePendingDeadline::Bootstrap { deadline_ms: 100_000 }), - 300_000, - ), - HlsAccessLeaseTouch::Touched { .. } - )); - - let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); - assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); - assert_eq!(lease.valid_until_ms, 12_000); - } - - #[test] - fn repeated_follow_up_touch_cannot_extend_existing_pending_deadline() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - - assert!(matches!( - store.touch_manifest_access_lease( - &lease_id, - &proxy_session_id, - 2_000, - None, - Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 }), - 300_000, - ), - HlsAccessLeaseTouch::Touched { .. } - )); - assert!(matches!( - store.touch_manifest_access_lease( - &lease_id, - &proxy_session_id, - 3_000, - None, - Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 30_000 }), - 300_000, - ), - HlsAccessLeaseTouch::Touched { .. } - )); - - let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); - assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); - assert_eq!(lease.valid_until_ms, 12_000); - } - - #[test] - fn session_follow_up_shortens_pending_lease_once() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - - let shortened = store.mark_pending_manifest_follow_up_for_session( - &proxy_session_id, - 2_000, - HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 }, - ); - assert_eq!(shortened.len(), 1); - let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); - assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); - assert_eq!(lease.valid_until_ms, 12_000); - - let unchanged = store.mark_pending_manifest_follow_up_for_session( - &proxy_session_id, - 3_000, - HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 30_000 }, - ); - assert!(unchanged.is_empty()); - let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); - assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); - assert_eq!(lease.valid_until_ms, 12_000); - } - - #[test] - fn activated_lease_remains_valid_after_media_touch() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 15_000)).is_activated()); - - assert!(store.touch_access_lease(&lease_id, 3_000, timing(5_000, 15_000))); - assert_eq!(store.lease_state(&lease_id, 17_999), Some(HlsAccessLeaseState::Activated)); - } - - #[test] - fn activated_lease_becomes_idle_after_active_window_but_remains_reactivatable() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - - assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 30_000)).is_activated()); - - let snapshot = store.session_snapshot(&proxy_session_id, 8_000); - assert_eq!(snapshot.active_count, 0); - assert_eq!(snapshot.idle_releases.len(), 1); - assert_eq!(snapshot.idle_releases[0].lease_id, lease_id); - assert_eq!(store.lease_state(&lease_id, 8_000), Some(HlsAccessLeaseState::Idle)); - assert!(store.has_usable_access_lease_for_session(&proxy_session_id, 8_000)); - assert!(store.access_lease(&lease_id, &proxy_session_id, 8_000).is_some()); - - assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 8_000, timing(5_000, 30_000)).is_activated()); - assert_eq!(store.session_snapshot(&proxy_session_id, 8_000).active_count, 1); - } - - #[test] - fn activated_lease_validity_expiry_reports_idle_release() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - - assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(30_000, 5_000)).is_activated()); - - let snapshot = store.session_snapshot(&proxy_session_id, 8_000); - assert_eq!(snapshot.active_count, 0); - assert_eq!(snapshot.idle_releases.len(), 1); - assert_eq!(snapshot.idle_releases[0].lease_id, lease_id); - assert_eq!(store.lease_state(&lease_id, 8_000), Some(HlsAccessLeaseState::Expired)); - } - - #[test] - fn usable_access_lease_query_accepts_pending_idle_and_active_activated_only() { - let mut store = HlsAccessLeaseStore::default(); - let proxy_session_id = ProxySessionId("proxy".to_string()); - let pending_id = HlsAccessLeaseId("pending".to_string()); - let idle_id = HlsAccessLeaseId("idle".to_string()); - let activated_id = HlsAccessLeaseId("activated".to_string()); - let denied_id = HlsAccessLeaseId("denied".to_string()); - let expired_id = HlsAccessLeaseId("expired".to_string()); - - store.prepare_access_lease(lease(pending_id.clone(), &proxy_session_id.0, 1_000)); - store.prepare_access_lease(lease(idle_id.clone(), &proxy_session_id.0, 1_000)); - store.prepare_access_lease(lease(activated_id.clone(), &proxy_session_id.0, 1_000)); - store.prepare_access_lease(lease(denied_id.clone(), &proxy_session_id.0, 1_000)); - store.prepare_access_lease(lease(expired_id.clone(), &proxy_session_id.0, 1_000)); - assert!(store.activate_access_lease(&idle_id, &proxy_session_id, 2_000, timing(1_000, 15_000)).is_activated()); - assert!(store - .activate_access_lease(&activated_id, &proxy_session_id, 2_000, timing(5_000, 15_000)) - .is_activated()); - assert!(matches!( - store.deny_access_lease(&denied_id, HlsAccessLeaseDenialMode::ImmediateEnd), - HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } - )); - - assert!(store.has_usable_access_lease_for_session(&proxy_session_id, 2_000)); - let snapshot = store.session_snapshot(&proxy_session_id, 3_000); - assert_eq!(snapshot.active_count, 1); - assert_eq!(snapshot.idle_releases.len(), 1); - assert_eq!(snapshot.idle_releases[0].lease_id, idle_id); - assert_eq!(store.lease_state(&idle_id, 3_000), Some(HlsAccessLeaseState::Idle)); - assert_eq!(store.active_access_lease_count_for_session(&proxy_session_id, 3_000), 1); - assert!(store.has_usable_access_lease_for_session(&proxy_session_id, 3_000)); - - assert!(matches!( - store.deny_access_lease(&pending_id, HlsAccessLeaseDenialMode::ImmediateEnd), - HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } - )); - assert!(matches!( - store.deny_access_lease(&activated_id, HlsAccessLeaseDenialMode::ImmediateEnd), - HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } - )); - assert!(matches!( - store.deny_access_lease(&expired_id, HlsAccessLeaseDenialMode::ImmediateEnd), - HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } - )); - assert!(store.has_usable_access_lease_for_session(&proxy_session_id, 3_000)); - assert!(matches!( - store.deny_access_lease(&idle_id, HlsAccessLeaseDenialMode::ImmediateEnd), - HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } - )); - assert!(!store.has_usable_access_lease_for_session(&proxy_session_id, 2_000)); - assert!(!store.has_usable_access_lease_for_session(&proxy_session_id, 17_000)); - assert_eq!(store.lease_state(&expired_id, 17_000), Some(HlsAccessLeaseState::Expired)); - } - - #[test] - fn session_snapshot_prefers_normal_origin_policy_over_soft() { - let mut store = HlsAccessLeaseStore::default(); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease( - lease(HlsAccessLeaseId("soft".to_string()), &proxy_session_id.0, 1_000) - .with_origin_acquire_policy(ConnectionKind::Soft, -20), - ); - store.prepare_access_lease( - lease(HlsAccessLeaseId("normal".to_string()), &proxy_session_id.0, 1_000) - .with_origin_acquire_policy(ConnectionKind::Normal, 50), - ); - - let snapshot = store.session_snapshot(&proxy_session_id, 2_000); - let policy = snapshot.effective_origin_policy.expect("usable lease policy"); - assert_eq!(policy.connection_kind, ConnectionKind::Normal); - assert_eq!(policy.priority, 50); - } - - #[test] - fn origin_policy_update_reclassifies_existing_access_lease() { - let mut store = HlsAccessLeaseStore::default(); - let proxy_session_id = ProxySessionId("proxy".to_string()); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - store.prepare_access_lease( - lease(lease_id.clone(), &proxy_session_id.0, 1_000).with_origin_acquire_policy(ConnectionKind::Soft, 20), - ); - - let updated = - store.update_origin_acquire_policy(&lease_id, ConnectionKind::Normal, -5).expect("lease should update"); - assert_eq!(updated.origin_connection_kind, ConnectionKind::Normal); - assert_eq!(updated.origin_priority, -5); - - let snapshot = store.session_snapshot(&proxy_session_id, 2_000); - let policy = snapshot.effective_origin_policy.expect("updated policy"); - assert_eq!(policy.connection_kind, ConnectionKind::Normal); - assert_eq!(policy.priority, -5); - } - - #[test] - fn session_snapshot_uses_best_priority_within_same_origin_policy_kind() { - let mut store = HlsAccessLeaseStore::default(); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease( - lease(HlsAccessLeaseId("low-priority".to_string()), &proxy_session_id.0, 1_000) - .with_origin_acquire_policy(ConnectionKind::Normal, 30), - ); - store.prepare_access_lease( - lease(HlsAccessLeaseId("high-priority".to_string()), &proxy_session_id.0, 1_000) - .with_origin_acquire_policy(ConnectionKind::Normal, -5), - ); - - let snapshot = store.session_snapshot(&proxy_session_id, 2_000); - let policy = snapshot.effective_origin_policy.expect("usable lease policy"); - assert_eq!(policy.connection_kind, ConnectionKind::Normal); - assert_eq!(policy.priority, -5); - } - - #[test] - fn session_snapshot_ignores_expired_and_denied_origin_policies() { - let mut store = HlsAccessLeaseStore::default(); - let proxy_session_id = ProxySessionId("proxy".to_string()); - let denied_id = HlsAccessLeaseId("denied".to_string()); - store.prepare_access_lease( - lease(denied_id.clone(), &proxy_session_id.0, 1_000) - .with_origin_acquire_policy(ConnectionKind::Normal, -100), - ); - store.prepare_access_lease( - lease(HlsAccessLeaseId("expired".to_string()), &proxy_session_id.0, 1_000) - .with_origin_acquire_policy(ConnectionKind::Normal, -50), - ); - store.prepare_access_lease( - lease(HlsAccessLeaseId("active-soft".to_string()), &proxy_session_id.0, 10_000) - .with_origin_acquire_policy(ConnectionKind::Soft, 10), - ); - assert!(matches!( - store.deny_access_lease(&denied_id, HlsAccessLeaseDenialMode::ImmediateEnd), - HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } - )); - - let snapshot = store.session_snapshot(&proxy_session_id, 17_000); - let policy = snapshot.effective_origin_policy.expect("usable lease policy"); - assert_eq!(policy.connection_kind, ConnectionKind::Soft); - assert_eq!(policy.priority, 10); - } - - #[test] - fn lease_manifest_generation_advances_for_snapshots_rendered_in_the_same_millisecond() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - - assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(2_000), 2_000) - .is_committed()); - let first = store.response_snapshot(&lease_id, &proxy_session_id, 2_000).expect("first manifest snapshot"); - assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(2_000), 2_000) - .is_committed()); - let second = store.response_snapshot(&lease_id, &proxy_session_id, 2_000).expect("second manifest snapshot"); - - assert_eq!(first.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation), Some(1)); - assert_eq!(second.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation), Some(2)); - assert_eq!( - first.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.delivered_at_ms), - second.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.delivered_at_ms) - ); - } - - #[test] - fn newer_source_publication_wins_before_delayed_older_request() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - let older_request = - store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("older request guard"); - let newer_request = - store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("newer request guard"); - - assert_eq!( - store.commit_manifest_publication( - &lease_id, - &proxy_session_id, - newer_request, - manifest_snapshot(20), - 2_100, - ), - HlsLeaseManifestPublicationOutcome::Committed { snapshot_generation: 1 } - ); - assert_eq!( - store.commit_manifest_publication( - &lease_id, - &proxy_session_id, - older_request, - manifest_snapshot(10), - 2_200, - ), - HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::SourceRegressive) - ); - let current = store - .response_snapshot(&lease_id, &proxy_session_id, 2_200) - .and_then(|lease| lease.last_manifest_snapshot) - .expect("newer snapshot remains current"); - assert_eq!(current.source_commit_identity, HlsManifestCommitIdentity::new(20)); - assert_eq!(current.snapshot_generation, 1); - } - - #[test] - fn commit_generation_orders_cross_mode_sources_rendered_in_the_same_millisecond() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - let older_request = - store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("older request guard"); - let newer_request = - store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("newer request guard"); - let mut newer = manifest_snapshot(20); - newer.delivery_mode = HlsManifestDeliveryMode::TransientPassthrough; - newer.source_commit_identity = HlsManifestCommitIdentity::committed(2, 20); - newer.finalized_transient_manifest_generation = Some(super::super::TransientManifestGeneration::for_test(2)); - let mut older = manifest_snapshot(20); - older.delivery_mode = HlsManifestDeliveryMode::NormalCacheTimeline; - older.source_commit_identity = HlsManifestCommitIdentity::committed(1, 20); - older.finalized_transient_manifest_generation = None; - - assert!(store - .commit_manifest_publication(&lease_id, &proxy_session_id, newer_request, newer, 2_100) - .is_committed()); - assert_eq!( - store.commit_manifest_publication(&lease_id, &proxy_session_id, older_request, older, 2_200), - HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::SourceRegressive) - ); - - let lease_id = HlsAccessLeaseId("lease-b".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - let older_request = - store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("older request guard"); - let newer_request = - store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("newer request guard"); - let mut newer = manifest_snapshot(20); - newer.source_commit_identity = HlsManifestCommitIdentity::committed(4, 20); - let mut older = manifest_snapshot(20); - older.delivery_mode = HlsManifestDeliveryMode::TransientPassthrough; - older.source_commit_identity = HlsManifestCommitIdentity::committed(3, 20); - older.finalized_transient_manifest_generation = Some(super::super::TransientManifestGeneration::for_test(1)); - - assert!(store - .commit_manifest_publication(&lease_id, &proxy_session_id, newer_request, newer, 2_100) - .is_committed()); - assert_eq!( - store.commit_manifest_publication(&lease_id, &proxy_session_id, older_request, older, 2_200), - HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::SourceRegressive) - ); - } - - #[test] - fn lease_expiry_between_manifest_publication_prepare_and_commit_rejects_snapshot() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - let guard = - store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("live publication guard"); - - assert_eq!( - store.commit_manifest_publication(&lease_id, &proxy_session_id, guard, manifest_snapshot(10), 16_000,), - HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::LeaseExpired) - ); - let lease = store.response_snapshot(&lease_id, &proxy_session_id, 16_000).expect("expired lease retained"); - assert_eq!(lease.state, HlsAccessLeaseState::Expired); - assert!(lease.last_manifest_snapshot.is_none()); - } - - #[test] - fn replacement_lease_incarnation_rejects_older_manifest_publication() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - let guard = store - .prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000) - .expect("original incarnation guard"); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 3_000)); - - assert_eq!( - store.commit_manifest_publication(&lease_id, &proxy_session_id, guard, manifest_snapshot(10), 3_100,), - HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::LeaseIncarnationChanged - ) - ); - } - - #[test] - fn changed_admission_generation_rejects_manifest_publication() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - let guard = - store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("original admission guard"); - let lease = store.by_lease_id.get_mut(&lease_id).expect("stored lease"); - lease.admission_generation = lease.admission_generation.saturating_add(1); - - assert_eq!( - store.commit_manifest_publication(&lease_id, &proxy_session_id, guard, manifest_snapshot(10), 2_100,), - HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::AdmissionGenerationChanged - ) - ); - } - - #[test] - fn terminal_preparation_is_read_only() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(7), 2_000) - .is_committed()); - let before = store.response_snapshot(&lease_id, &proxy_session_id, 2_000).expect("live lease"); - let manifest_generation = - before.last_manifest_snapshot.as_ref().expect("manifest snapshot").snapshot_generation; - - let preparation = store - .prepare_terminal_tail( - &lease_id, - &proxy_session_id, - &HlsTerminalTailPreparationInput { - trigger: HlsFiniteTailTrigger::AvailabilityReserve, - expected_manifest_snapshot_generation: manifest_generation, - expected_cursor_generation: before.playback_cursor.cursor_generation, - origin_progress_generation: 4, - media_readiness_generation: 9, - origin_epoch: 0, - last_media_progress_at_ms: Some(1_900), - expected_acceptance_generation: HlsManifestAcceptanceGeneration(4), - terminal_media_requirement_origin: HlsTerminalMediaRequirementOrigin::AcceptanceEpisode { - generation: HlsManifestAcceptanceGeneration(4), - }, - cutover_timing: cutover_timing(), - commit_window: HlsTerminalCommitWindow::CutoverDue, - required_terminal_media_key: None, - terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, - reserve: cutover_reserve(), - }, - ) - .expect("terminal preparation"); - let after = store.response_snapshot(&lease_id, &proxy_session_id, 2_000).expect("live lease"); - - assert_eq!(before, after); - assert_eq!(preparation.expected_admission_generation, before.admission_generation); - assert_eq!(preparation.media_readiness_generation, 9); - assert_eq!(preparation.manifest_snapshot_generation, manifest_generation); - assert_eq!(preparation.cutover_timing, cutover_timing()); - assert_eq!(preparation.terminal_media_preparation, HlsTerminalMediaPreparationState::Failed { key: None }); - } - - #[test] - fn expired_lease_lookup_rejects_stale_entry_without_removing_before_lifecycle() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - - assert!(store.access_lease(&lease_id, &proxy_session_id, 17_000).is_none()); - assert_eq!(store.lease_state(&lease_id, 17_000), Some(HlsAccessLeaseState::Expired)); - } - - #[test] - fn hls_terminal_commit_stale_snapshot_cannot_replace_live_generation() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000) - .is_committed()); - - let stale = prepare_terminal(&mut store, &lease_id, &proxy_session_id, 4, Some(2_000)); - assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000) - .is_committed()); - - assert_eq!( - store.commit_terminal_tail_if_generation_matches( - &lease_id, - &proxy_session_id, - &stale, - 3_000, - terminal_plan(stale.decision_generation, &proxy_session_id, &lease_id), - ), - HlsTerminalCommitOutcome::SupersededGeneration - ); - let lease = store.response_snapshot(&lease_id, &proxy_session_id, 3_000).expect("lease remains stored"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - assert_eq!(lease.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation), Some(2)); - } - - #[test] - fn hls_terminal_commit_plan_bound_to_another_route_is_superseded() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000) - .is_committed()); - let preparation = prepare_terminal(&mut store, &lease_id, &proxy_session_id, 4, Some(2_000)); - let other_lease_id = HlsAccessLeaseId("other-lease".to_string()); - - assert_eq!( - store.commit_terminal_tail_if_generation_matches( - &lease_id, - &proxy_session_id, - &preparation, - 3_000, - terminal_plan(preparation.decision_generation, &proxy_session_id, &other_lease_id), - ), - HlsTerminalCommitOutcome::SupersededGeneration - ); - assert_eq!( - store.response_snapshot(&lease_id, &proxy_session_id, 3_000).expect("live lease").playback_mode, - HlsLeasePlaybackMode::Live - ); - } - - #[test] - fn hls_terminal_commit_lease_expiry_wins_the_race() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000) - .is_committed()); - let preparation = prepare_terminal(&mut store, &lease_id, &proxy_session_id, 4, Some(2_000)); - - assert_eq!( - store.commit_terminal_tail_if_generation_matches( - &lease_id, - &proxy_session_id, - &preparation, - 16_000, - terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), - ), - HlsTerminalCommitOutcome::LeaseNoLongerEligible - ); - let lease = - store.response_snapshot(&lease_id, &proxy_session_id, 16_000).expect("expired lease remains stored"); - assert_eq!(lease.state, HlsAccessLeaseState::Expired); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Ended); - assert_eq!(lease.admission_generation, preparation.expected_admission_generation); - } - - #[test] - fn hls_terminal_commit_exact_replay_is_idempotent_and_sticky() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000) - .is_committed()); - let preparation = prepare_terminal(&mut store, &lease_id, &proxy_session_id, 4, Some(2_000)); - let plan = terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id); - let concurrent_plan = terminal_plan_at(preparation.decision_generation, &proxy_session_id, &lease_id, 3_100); - - assert_eq!( - store.commit_terminal_tail_if_generation_matches( - &lease_id, - &proxy_session_id, - &preparation, - 3_000, - Arc::clone(&plan), - ), - HlsTerminalCommitOutcome::Committed - ); - assert_eq!( - store.commit_terminal_tail_if_generation_matches( - &lease_id, - &proxy_session_id, - &preparation, - 3_100, - concurrent_plan, - ), - HlsTerminalCommitOutcome::AlreadyCommitted - ); - assert!(store.prepare_manifest_publication(&lease_id, &proxy_session_id, 3_100).is_none()); - } - - #[test] - fn hls_terminal_commit_unavailable_replay_is_idempotent_for_the_same_decision_generation() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000) - .is_committed()); - let preparation = prepare_terminal(&mut store, &lease_id, &proxy_session_id, 4, Some(2_000)); - - assert_eq!( - store.commit_terminal_unavailable_if_generation_matches( - &lease_id, - &proxy_session_id, - &preparation, - 3_000, - HlsTerminalTailCompatibility::MissingAsset, - ), - HlsTerminalCommitOutcome::Committed - ); - assert_eq!( - store.commit_terminal_unavailable_if_generation_matches( - &lease_id, - &proxy_session_id, - &preparation, - 3_100, - HlsTerminalTailCompatibility::InvalidAsset, - ), - HlsTerminalCommitOutcome::AlreadyCommitted - ); - } - - #[test] - fn hls_terminal_commit_terminal_lease_rejects_recovery_while_other_lease_remains_live() { - let mut store = HlsAccessLeaseStore::default(); - let terminal_lease_id = HlsAccessLeaseId("terminal".to_string()); - let live_lease_id = HlsAccessLeaseId("live".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(terminal_lease_id.clone(), &proxy_session_id.0, 1_000)); - store.prepare_access_lease(lease(live_lease_id.clone(), &proxy_session_id.0, 1_000)); - assert!(publish_manifest_snapshot( - &mut store, - &terminal_lease_id, - &proxy_session_id, - manifest_snapshot(1), - 2_000, - ) - .is_committed()); - assert!(publish_manifest_snapshot(&mut store, &live_lease_id, &proxy_session_id, manifest_snapshot(1), 2_000,) - .is_committed()); - let stale_terminal_publication = store - .prepare_manifest_publication(&terminal_lease_id, &proxy_session_id, 2_000) - .expect("pre-terminal publication guard"); - let preparation = prepare_terminal(&mut store, &terminal_lease_id, &proxy_session_id, 7, Some(2_000)); - assert_eq!( - commit_prepared_terminal(&mut store, &terminal_lease_id, &proxy_session_id, &preparation), - HlsTerminalCommitOutcome::Committed - ); - - assert_eq!( - store.commit_manifest_publication( - &terminal_lease_id, - &proxy_session_id, - stale_terminal_publication, - manifest_snapshot(2), - 3_000, - ), - HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::AdmissionGenerationChanged - ) - ); - assert!(store.prepare_manifest_publication(&terminal_lease_id, &proxy_session_id, 3_000).is_none()); - assert!(publish_manifest_snapshot(&mut store, &live_lease_id, &proxy_session_id, manifest_snapshot(2), 3_000,) - .is_committed()); - assert!(matches!( - store - .response_snapshot(&terminal_lease_id, &proxy_session_id, 3_000) - .expect("terminal lease") - .playback_mode, - HlsLeasePlaybackMode::TerminalTail(_) - )); - assert_eq!( - store.response_snapshot(&live_lease_id, &proxy_session_id, 3_000).expect("live lease").playback_mode, - HlsLeasePlaybackMode::Live - ); - } - - #[test] - fn late_segment_completion_after_terminal_transition_does_not_advance_cursor() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000) - .is_committed()); - let identity = store - .response_snapshot(&lease_id, &proxy_session_id, 2_100) - .and_then(|lease| lease.media_identity()) - .expect("live identity"); - let token = store - .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 40, 2_100) - .expect("live request token"); - let preparation = prepare_terminal(&mut store, &lease_id, &proxy_session_id, 8, Some(2_000)); - assert_eq!( - commit_prepared_terminal(&mut store, &lease_id, &proxy_session_id, &preparation), - HlsTerminalCommitOutcome::Committed - ); - - assert_eq!( - store.record_segment_request_completed_if_identity_matches( - &lease_id, - &proxy_session_id, - identity, - token, - 2_200, - ), - None - ); - let lease = store.response_snapshot(&lease_id, &proxy_session_id, 2_300).expect("terminal lease"); - assert_eq!(lease.playback_cursor.highest_contiguous_completed_proxy_seq, None); - assert_eq!(lease.playback_cursor.first_segment_completed_at_ms, None); - } - - #[test] - fn late_segment_completion_after_forward_seek_is_stale_and_does_not_advance_cursor() { - let mut store = HlsAccessLeaseStore::default(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let proxy_session_id = ProxySessionId("proxy".to_string()); - store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); - let identity = store - .response_snapshot(&lease_id, &proxy_session_id, 2_000) - .and_then(|lease| lease.media_identity()) - .expect("live identity"); - let stale_token = store - .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 40, 2_000) - .expect("first request token"); - let _current_token = store - .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 50, 2_100) - .expect("forward-seek request token"); - - assert_eq!( - store.record_segment_request_completed_if_identity_matches( - &lease_id, - &proxy_session_id, - identity, - stale_token, - 2_200, - ), - Some(HlsPlaybackCompletionOutcome::StaleRequest) - ); - let lease = store.response_snapshot(&lease_id, &proxy_session_id, 2_300).expect("live lease"); - assert_eq!(lease.playback_cursor.first_requested_proxy_seq, Some(50)); - assert_eq!(lease.playback_cursor.highest_contiguous_completed_proxy_seq, None); - assert_eq!(lease.playback_cursor.first_segment_completed_at_ms, None); - } - - #[test] - fn hls_availability_reevaluation_evidence_generation_is_per_session_and_noop_stable() { - let mut store = HlsAccessLeaseStore::default(); - let proxy_a = ProxySessionId("availability-evidence-a".to_string()); - let proxy_b = ProxySessionId("availability-evidence-b".to_string()); - let primary_lease_id = HlsAccessLeaseId("availability-evidence-a".to_string()); - let secondary_lease_id = HlsAccessLeaseId("availability-evidence-b".to_string()); - let lease_a = lease(primary_lease_id.clone(), &proxy_a.0, 1_000); - assert!(store.prepare_access_lease(lease_a.clone())); - let added_a = store.availability_evidence_generation(&proxy_a); - assert!(added_a.as_u64() > 0); - - assert!(store.prepare_access_lease(lease_a)); - assert_eq!(store.availability_evidence_generation(&proxy_a), added_a); - assert!(matches!( - store.activate_access_lease(&primary_lease_id, &proxy_b, 2_000, timing(5_000, 30_000)), - HlsAccessLeaseActivation::SessionMismatch - )); - assert_eq!(store.availability_evidence_generation(&proxy_a), added_a); - - assert!(store.prepare_access_lease(lease(secondary_lease_id, &proxy_b.0, 1_000))); - assert_eq!(store.availability_evidence_generation(&proxy_a), added_a); - assert!(store.availability_evidence_generation(&proxy_b).as_u64() > added_a.as_u64()); - - assert!(store.activate_access_lease(&primary_lease_id, &proxy_a, 2_000, timing(5_000, 30_000)).is_activated()); - let activated_a = store.availability_evidence_generation(&proxy_a); - assert!(activated_a > added_a); - assert_eq!(store.availability_evidence_generation(&proxy_b).as_u64(), added_a.as_u64().saturating_add(1)); - - let identity = store - .response_snapshot(&primary_lease_id, &proxy_a, 2_000) - .and_then(|lease| lease.media_identity()) - .expect("activated live lease identity"); - let token = store - .record_segment_request_started_if_identity_matches(&primary_lease_id, &proxy_a, identity, 7, 2_100) - .expect("cursor request starts"); - let requested_a = store.availability_evidence_generation(&proxy_a); - assert!(requested_a > activated_a); - assert_eq!( - store.record_segment_request_completed_if_identity_matches( - &primary_lease_id, - &proxy_a, - identity, - token, - 2_200, - ), - Some(HlsPlaybackCompletionOutcome::Advanced) - ); - let completed_a = store.availability_evidence_generation(&proxy_a); - assert!(completed_a > requested_a); - assert_eq!( - store.record_segment_request_completed_if_identity_matches( - &primary_lease_id, - &proxy_a, - identity, - token, - 2_300, - ), - Some(HlsPlaybackCompletionOutcome::Duplicate) - ); - assert_eq!(store.availability_evidence_generation(&proxy_a), completed_a); - - assert!(store.remove_access_lease(&primary_lease_id).is_some()); - let removed_a = store.availability_evidence_generation(&proxy_a); - assert!(removed_a > completed_a); - assert_eq!(store.availability_evidence_generation(&proxy_b).as_u64(), added_a.as_u64().saturating_add(1)); - assert!(store.prepare_access_lease(lease(primary_lease_id, &proxy_a.0, 3_000))); - assert!(store.availability_evidence_generation(&proxy_a) > removed_a); - } - - #[test] - fn hls_availability_reevaluation_evidence_generation_fails_closed_on_overflow() { - let mut store = HlsAccessLeaseStore::default(); - let proxy_session_id = ProxySessionId("availability-evidence-overflow".to_string()); - let lease_id = HlsAccessLeaseId("availability-evidence-overflow".to_string()); - store.last_availability_evidence_generation = HlsAvailabilityEvidenceGeneration::for_test(u64::MAX); - - assert!(!store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000))); - assert!(store.response_snapshot(&lease_id, &proxy_session_id, 1_000).is_none()); - assert_eq!(store.availability_evidence_generation(&proxy_session_id).as_u64(), 0); - } - - #[test] - fn bulk_lease_removal_advances_and_preserves_session_evidence_generation() { - let mut store = HlsAccessLeaseStore::default(); - let proxy_session_id = ProxySessionId("bulk-removal".to_string()); - let first = HlsAccessLeaseId("bulk-removal-a".to_string()); - let second = HlsAccessLeaseId("bulk-removal-b".to_string()); - assert!(store.prepare_access_lease(lease(first, &proxy_session_id.0, 1_000))); - assert!(store.prepare_access_lease(lease(second, &proxy_session_id.0, 1_001))); - let before_removal = store.availability_evidence_generation(&proxy_session_id); - - assert_eq!(store.remove_access_leases_for_session(&proxy_session_id).len(), 2); - let after_removal = store.availability_evidence_generation(&proxy_session_id); - assert!(after_removal > before_removal); - assert!(store.remove_access_leases_for_session(&proxy_session_id).is_empty()); - assert_eq!(store.availability_evidence_generation(&proxy_session_id), after_removal); - } -} +mod tests; + +mod error; +mod identity; +mod lifecycle; +mod publication; +mod revision_bindings; +mod runtime_policy; +mod terminal; +mod timing; + +pub use self::identity::new_hls_access_lease_id; +use self::{ + error::HlsAvailabilityEvidenceAdvanceError, + identity::HlsMediaLeasePlaybackIdentity, + lifecycle::lease_state_allows_use, + revision_bindings::{commit_lease_revision_bindings, HlsLeaseRevisionBindings}, + runtime_policy::runtime_policy_authorized_manifest_prefix, +}; diff --git a/backend/hls/src/lease/error.rs b/backend/hls/src/lease/error.rs new file mode 100644 index 000000000..f35d1c599 --- /dev/null +++ b/backend/hls/src/lease/error.rs @@ -0,0 +1,4 @@ +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum HlsAvailabilityEvidenceAdvanceError { + Exhausted, +} diff --git a/backend/hls/src/lease/identity.rs b/backend/hls/src/lease/identity.rs new file mode 100644 index 000000000..6fcde43a7 --- /dev/null +++ b/backend/hls/src/lease/identity.rs @@ -0,0 +1,75 @@ +use super::{ + lease_state_allows_use, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseStore, HlsLeaseManifestPublicationGuard, + HlsLeasePlaybackMode, HlsMediaLeaseIdentity, HlsTerminalTailGeneration, ProxySessionId, HLS_ACCESS_LEASE_ID_BYTES, +}; +use base64::{engine::general_purpose, Engine as _}; +use rand::{rngs::OsRng, RngCore, TryRngCore}; +use std::fmt; + +impl fmt::Debug for HlsAccessLeaseId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("HlsAccessLeaseId").field(&"").finish() + } +} + +impl HlsMediaLeaseIdentity { + pub const fn is_live(self) -> bool { matches!(self.playback, HlsMediaLeasePlaybackIdentity::Live { .. }) } + + pub const fn lease_issued_at_ms(self) -> u64 { self.issued_at_ms } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum HlsMediaLeasePlaybackIdentity { + Live { admission_generation: u64 }, + TerminalTail { generation: HlsTerminalTailGeneration }, +} + +impl HlsLeaseManifestPublicationGuard { + pub(crate) const fn lease_issued_at_ms(self) -> u64 { self.issued_at_ms } +} + +impl HlsAccessLease { + pub fn media_identity(&self) -> Option { + let playback = match &self.playback_mode { + HlsLeasePlaybackMode::Live => { + HlsMediaLeasePlaybackIdentity::Live { admission_generation: self.admission_generation } + } + HlsLeasePlaybackMode::TerminalTail(plan) => { + HlsMediaLeasePlaybackIdentity::TerminalTail { generation: plan.generation } + } + HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => return None, + }; + Some(HlsMediaLeaseIdentity { issued_at_ms: self.issued_at_ms, playback }) + } +} + +impl HlsAccessLeaseStore { + pub fn media_identity_is_current( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + expected: HlsMediaLeaseIdentity, + now_ms: u64, + ) -> bool { + let Some(lease) = self.by_lease_id.get(lease_id) else { + return false; + }; + if lease.proxy_session_id != *proxy_session_id || self.refresh_access_lease_validity(lease_id, now_ms).is_err() + { + return false; + } + self.by_lease_id.get(lease_id).is_some_and(|lease| { + lease_state_allows_use(lease.state) + && lease.issued_at_ms == expected.issued_at_ms + && lease.media_identity() == Some(expected) + }) + } +} + +pub fn new_hls_access_lease_id() -> HlsAccessLeaseId { + let mut bytes = [0u8; HLS_ACCESS_LEASE_ID_BYTES]; + if OsRng.try_fill_bytes(&mut bytes).is_err() { + rand::rng().fill_bytes(&mut bytes); + } + HlsAccessLeaseId(general_purpose::URL_SAFE_NO_PAD.encode(bytes)) +} diff --git a/backend/hls/src/lease/lifecycle.rs b/backend/hls/src/lease/lifecycle.rs new file mode 100644 index 000000000..696d78b6f --- /dev/null +++ b/backend/hls/src/lease/lifecycle.rs @@ -0,0 +1,837 @@ +use super::{ + HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseId, HlsAccessLeaseIdleRelease, + HlsAccessLeaseLifecycleSnapshot, HlsAccessLeasePendingDeadline, HlsAccessLeaseRemovalPreparation, + HlsAccessLeaseSessionSnapshot, HlsAccessLeaseState, HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsAccessLeaseTouch, + HlsAvailabilityEvidenceAdvanceError, HlsAvailabilityEvidenceGeneration, HlsEffectiveOriginAcquirePolicy, + HlsLeaseManifestPublicationOutcome, HlsLeasePlaybackMode, HlsManifestDeliveryMode, HlsMediaLeaseIdentity, + HlsMediaLeasePlaybackIdentity, HlsPlaybackCompletionOutcome, HlsPlaybackFamilyKey, HlsPlaybackRequestToken, + HlsTerminalTailGeneration, ProxySessionId, TransientManifestLeaseBinding, +}; +use tuliprox_session::ConnectionKind; + +impl HlsAvailabilityEvidenceGeneration { + pub(super) const NONE: Self = Self(0); + + #[cfg(any(test, feature = "test-support"))] + pub const fn for_test(generation: u64) -> Self { Self(generation) } + + #[cfg(any(test, feature = "test-support"))] + pub const fn as_u64(self) -> u64 { self.0 } +} + +impl HlsPlaybackFamilyKey { + pub fn new(username: impl Into, client_fingerprint: impl Into) -> Self { + Self { username: username.into(), client_fingerprint: client_fingerprint.into() } + } +} + +impl HlsAccessLeaseState { + pub const fn as_log_value(self) -> &'static str { + match self { + Self::Pending => "Pending", + Self::Activated => "Activated", + Self::Idle => "Idle", + Self::PolicyRevoking => "PolicyRevoking", + Self::Expired => "Expired", + Self::Denied => "Denied", + } + } +} + +impl HlsAccessLeasePendingDeadline { + pub(super) const fn tightened_with(self, candidate: Self) -> Self { + let deadline_ms = + if self.deadline_ms() <= candidate.deadline_ms() { self.deadline_ms() } else { candidate.deadline_ms() }; + match (self, candidate) { + (Self::FollowUp { .. }, _) | (_, Self::FollowUp { .. }) => Self::FollowUp { deadline_ms }, + (Self::Bootstrap { .. }, Self::Bootstrap { .. }) => Self::Bootstrap { deadline_ms }, + } + } +} + +impl HlsLeaseManifestPublicationOutcome { + pub const fn is_committed(self) -> bool { matches!(self, Self::Committed { .. }) } + + pub const fn snapshot_generation(self) -> Option { + match self { + Self::Committed { snapshot_generation } => Some(snapshot_generation), + Self::Rejected(_) => None, + } + } +} + +impl HlsAccessLease { + pub fn with_archive_playback(mut self, epg_reference_ts: Option, archive_origin_url: Option) -> Self { + self.epg_reference_ts = epg_reference_ts; + self.archive_origin_url = archive_origin_url; + self + } + + pub const fn with_origin_acquire_policy(mut self, connection_kind: ConnectionKind, priority: i8) -> Self { + self.origin_connection_kind = connection_kind; + self.origin_priority = priority; + self + } + + pub const fn with_known_bitrate_bps(mut self, known_bitrate_bps: Option) -> Self { + self.known_bitrate_bps = match known_bitrate_bps { + Some(0) | None => None, + Some(bitrate_bps) => Some(bitrate_bps), + }; + self + } + + pub fn update_origin_acquire_policy(&mut self, connection_kind: ConnectionKind, priority: i8) { + self.origin_connection_kind = connection_kind; + self.origin_priority = priority; + } + + pub(super) fn end_playback(&mut self) { + self.runtime_policy_revocation = None; + let release_generation = match std::mem::replace(&mut self.playback_mode, HlsLeasePlaybackMode::Ended) { + HlsLeasePlaybackMode::TerminalTail(plan) => Some(plan.generation), + HlsLeasePlaybackMode::TerminalUnavailable { decision_generation, .. } => { + Some(HlsTerminalTailGeneration(decision_generation)) + } + HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::Ended => None, + }; + if self.pending_terminal_protection_release.is_none() { + self.pending_terminal_protection_release = release_generation; + } + } +} + +impl HlsAccessLeaseActivation { + pub const fn is_activated(&self) -> bool { matches!(self, Self::Activated { .. }) } +} + +impl HlsAccessLeaseStore { + pub fn repair_prewarm_is_current( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + issued_at_ms: u64, + snapshot_generation: u64, + ) -> bool { + self.by_lease_id.get(lease_id).is_some_and(|lease| { + lease.proxy_session_id == *proxy_session_id + && lease.issued_at_ms == issued_at_ms + && lease.playback_mode == HlsLeasePlaybackMode::Live + && lease + .last_manifest_snapshot + .as_ref() + .is_some_and(|snapshot| snapshot.snapshot_generation == snapshot_generation) + }) + } + + fn refresh_access_lease_activity( + &mut self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + ) -> Result, HlsAvailabilityEvidenceAdvanceError> { + let previous_state = match self.by_lease_id.get(lease_id) { + Some(lease) => lease.state, + None => return Ok(None), + }; + self.refresh_access_lease_validity(lease_id, now_ms)?; + let Some(lease) = self.by_lease_id.get(lease_id) else { + return Ok(None); + }; + if lease.state == HlsAccessLeaseState::Expired + && matches!(previous_state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated) + { + return Ok(Some(HlsAccessLeaseIdleRelease { + lease_id: lease.lease_id.clone(), + username: lease.username.clone(), + user_session_token: lease.user_session_token.clone(), + })); + } + if lease.state != HlsAccessLeaseState::Activated + || lease.active_until_ms.is_none_or(|active_until| active_until > now_ms) + { + return Ok(None); + } + let proxy_session_id = lease.proxy_session_id.clone(); + let release = HlsAccessLeaseIdleRelease { + lease_id: lease.lease_id.clone(), + username: lease.username.clone(), + user_session_token: lease.user_session_token.clone(), + }; + self.availability_evidence_can_advance()?; + if let Some(lease) = self.by_lease_id.get_mut(lease_id) { + lease.state = HlsAccessLeaseState::Idle; + } + self.advance_availability_evidence(&proxy_session_id)?; + Ok(Some(release)) + } + + /// Returns false only when the process-lifetime evidence generation is + /// exhausted. In that case no lease replacement is published. + pub fn prepare_access_lease(&mut self, lease: HlsAccessLease) -> bool { + if self.by_lease_id.get(&lease.lease_id) == Some(&lease) { + return true; + } + let new_proxy_session_id = lease.proxy_session_id.clone(); + let replaced_proxy_session_id = self + .by_lease_id + .get(&lease.lease_id) + .map(|current| current.proxy_session_id.clone()) + .filter(|current| *current != new_proxy_session_id); + let advances = 1_u64.saturating_add(u64::from(replaced_proxy_session_id.is_some())); + if self.last_availability_evidence_generation.0.checked_add(advances).is_none() { + return false; + } + self.by_lease_id.insert(lease.lease_id.clone(), lease); + if self.advance_availability_evidence(&new_proxy_session_id).is_err() { + return false; + } + if let Some(old_proxy_session_id) = replaced_proxy_session_id { + if self.advance_availability_evidence(&old_proxy_session_id).is_err() { + return false; + } + } + true + } + + pub fn remove_access_lease(&mut self, lease_id: &HlsAccessLeaseId) -> Option { + let proxy_session_id = self.by_lease_id.get(lease_id)?.proxy_session_id.clone(); + if self.availability_evidence_can_advance().is_err() { + return None; + } + self.advance_availability_evidence(&proxy_session_id).ok()?; + let removed = self.by_lease_id.remove(lease_id)?; + Some(removed) + } + + /// Snapshots cleanup state without deleting the persistent cancellation-recovery ticket. + pub fn prepare_access_lease_removal( + &self, + lease_id: &HlsAccessLeaseId, + ) -> Option { + let lease = self.by_lease_id.get(lease_id)?; + Some(HlsAccessLeaseRemovalPreparation { + proxy_session_id: lease.proxy_session_id.clone(), + issued_at_ms: lease.issued_at_ms, + terminal_protection_generation: lease.pending_terminal_protection_release.or_else(|| { + if let HlsLeasePlaybackMode::TerminalTail(plan) = &lease.playback_mode { + Some(plan.generation) + } else { + None + } + }), + }) + } + + /// Deletes only the lease instance observed before its protection cleanup. + pub fn remove_access_lease_if_preparation_matches( + &mut self, + lease_id: &HlsAccessLeaseId, + preparation: &HlsAccessLeaseRemovalPreparation, + ) -> Option { + let lease = self.by_lease_id.get(lease_id)?; + if lease.proxy_session_id != preparation.proxy_session_id || lease.issued_at_ms != preparation.issued_at_ms { + return None; + } + self.remove_access_lease(lease_id) + } + + pub fn remove_access_leases_for_session(&mut self, proxy_session_id: &ProxySessionId) -> Vec { + let lease_ids = self + .by_lease_id + .values() + .filter(|lease| lease.proxy_session_id == *proxy_session_id) + .map(|lease| lease.lease_id.clone()) + .collect::>(); + if lease_ids.is_empty() || self.availability_evidence_can_advance().is_err() { + return Vec::new(); + } + if self.advance_availability_evidence(proxy_session_id).is_err() { + return Vec::new(); + } + lease_ids.into_iter().filter_map(|lease_id| self.by_lease_id.remove(&lease_id)).collect() + } + + pub fn clear(&mut self) -> usize { + let removed = self.by_lease_id.len(); + self.by_lease_id.clear(); + self.availability_generation_by_proxy_session.clear(); + removed + } + + pub fn len(&self) -> usize { self.by_lease_id.len() } + + pub fn is_empty(&self) -> bool { self.by_lease_id.is_empty() } + + pub fn first_username_for_session(&self, proxy_session_id: &ProxySessionId) -> Option { + self.by_lease_id + .values() + .find(|lease| lease.proxy_session_id == *proxy_session_id) + .map(|lease| lease.username.clone()) + } + + /// Returns the oldest sequence which every usable live lease has consumed. + /// + /// `None` deliberately keeps the canonical render protected when a lease + /// has not published or completed media yet. Terminal leases are excluded + /// here because their exact base media is protected by the terminal-tail + /// generation stored on the shared session. + pub fn capacity_release_through(&self, proxy_session_id: &ProxySessionId) -> Option { + let mut found_live_lease = false; + let mut release_through = None; + for lease in self.by_lease_id.values().filter(|lease| { + lease.proxy_session_id == *proxy_session_id + && lease_state_protects_live_evidence(lease.state) + && lease.playback_mode == HlsLeasePlaybackMode::Live + }) { + found_live_lease = true; + let snapshot = lease.last_manifest_snapshot.as_ref()?; + if snapshot.delivery_mode != HlsManifestDeliveryMode::NormalCacheTimeline { + return None; + } + let completed = lease.playback_cursor.highest_contiguous_completed_proxy_seq?; + let completed = completed.min(snapshot.last_proxy_seq); + release_through = Some(release_through.map_or(completed, |current: u64| current.min(completed))); + } + if found_live_lease { + release_through + } else { + None + } + } + + pub fn response_snapshot( + &mut self, + lease_id: &HlsAccessLeaseId, + path_proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Option { + let lease = self.by_lease_id.get(lease_id)?; + if &lease.proxy_session_id != path_proxy_session_id { + return None; + } + if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { + return None; + } + self.by_lease_id.get(lease_id).cloned() + } + + /// Returns immutable playback evidence for leases that are actively consuming this shared session. + pub fn active_live_playback_snapshots_for_session( + &mut self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Vec { + if self.refresh_access_lease_validities_for_session(proxy_session_id, now_ms).is_err() { + return Vec::new(); + } + self.by_lease_id + .values() + .filter_map(|lease| { + if lease.proxy_session_id != *proxy_session_id { + return None; + } + (lease.state == HlsAccessLeaseState::Activated + && lease.active_until_ms.is_some_and(|active_until_ms| active_until_ms > now_ms) + && lease.playback_mode == HlsLeasePlaybackMode::Live + && lease.last_manifest_snapshot.is_some()) + .then(|| lease.clone()) + }) + .collect() + } + + pub fn record_segment_request_started_if_identity_matches( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + expected: HlsMediaLeaseIdentity, + proxy_seq: u64, + requested_at_ms: u64, + ) -> Option { + let lease = self.by_lease_id.get(lease_id)?; + if lease.proxy_session_id != *proxy_session_id { + return None; + } + if self.refresh_access_lease_validity(lease_id, requested_at_ms).is_err() { + return None; + } + let lease = self.by_lease_id.get(lease_id)?; + if !lease_state_allows_use(lease.state) + || lease.issued_at_ms != expected.issued_at_ms + || lease.media_identity() != Some(expected) + || !matches!(expected.playback, HlsMediaLeasePlaybackIdentity::Live { .. }) + || lease.playback_cursor.cursor_generation == u64::MAX + || self.availability_evidence_can_advance().is_err() + { + return None; + } + let token = + self.by_lease_id.get_mut(lease_id)?.playback_cursor.record_request_started(proxy_seq, requested_at_ms); + self.advance_availability_evidence(proxy_session_id).ok()?; + Some(token) + } + + pub fn record_segment_request_completed_if_identity_matches( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + expected: HlsMediaLeaseIdentity, + token: HlsPlaybackRequestToken, + completed_at_ms: u64, + ) -> Option { + let lease = self.by_lease_id.get(lease_id)?; + if lease.proxy_session_id != *proxy_session_id { + return None; + } + if self.refresh_access_lease_validity(lease_id, completed_at_ms).is_err() { + return None; + } + let lease = self.by_lease_id.get(lease_id)?; + if !lease_state_allows_use(lease.state) + || lease.issued_at_ms != expected.issued_at_ms + || lease.media_identity() != Some(expected) + || !matches!(expected.playback, HlsMediaLeasePlaybackIdentity::Live { .. }) + { + return None; + } + let mut cursor = lease.playback_cursor.clone(); + let outcome = cursor.record_request_completed(token, completed_at_ms); + if cursor.cursor_generation == lease.playback_cursor.cursor_generation { + return Some(outcome); + } + if self.availability_evidence_can_advance().is_err() { + return None; + } + self.by_lease_id.get_mut(lease_id)?.playback_cursor = cursor; + self.advance_availability_evidence(proxy_session_id).ok()?; + Some(outcome) + } + + pub fn access_lease( + &mut self, + lease_id: &HlsAccessLeaseId, + path_proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Option { + let lease = self.by_lease_id.get(lease_id)?; + if &lease.proxy_session_id != path_proxy_session_id { + return None; + } + self.refresh_access_lease_validity(lease_id, now_ms).ok()?; + let state = self.by_lease_id.get(lease_id)?.state; + if state == HlsAccessLeaseState::Expired { + return None; + } + if matches!(state, HlsAccessLeaseState::PolicyRevoking | HlsAccessLeaseState::Denied) { + return self.by_lease_id.get(lease_id).cloned(); + } + if !lease_state_allows_use(state) { + return None; + } + self.by_lease_id.get(lease_id).cloned() + } + + pub fn update_origin_acquire_policy( + &mut self, + lease_id: &HlsAccessLeaseId, + connection_kind: ConnectionKind, + priority: i8, + ) -> Option { + let lease = self.by_lease_id.get_mut(lease_id)?; + if !lease_state_allows_use(lease.state) { + return None; + } + lease.update_origin_acquire_policy(connection_kind, priority); + Some(lease.clone()) + } + + pub fn activate_access_lease( + &mut self, + lease_id: &HlsAccessLeaseId, + path_proxy_session_id: &ProxySessionId, + now_ms: u64, + timing: HlsAccessLeaseTiming, + ) -> HlsAccessLeaseActivation { + let Some(new_lease) = self.by_lease_id.get(lease_id) else { + return HlsAccessLeaseActivation::UnknownLease; + }; + if &new_lease.proxy_session_id != path_proxy_session_id { + return HlsAccessLeaseActivation::SessionMismatch; + } + if !lease_state_allows_use(new_lease.state) { + return activation_for_state(new_lease.state); + } + if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { + return HlsAccessLeaseActivation::Expired; + } + let Some(new_lease) = self.by_lease_id.get(lease_id) else { + return HlsAccessLeaseActivation::UnknownLease; + }; + if new_lease.state == HlsAccessLeaseState::Expired { + return HlsAccessLeaseActivation::Expired; + } + let previous_state = new_lease.state; + let active_until_ms = Some(now_ms.saturating_add(timing.active_window_ms)); + let valid_until_ms = now_ms.saturating_add(timing.valid_window_ms); + let evidence_changed = new_lease.state != HlsAccessLeaseState::Activated + || new_lease.active_until_ms != active_until_ms + || new_lease.pending_deadline.is_some() + || new_lease.valid_until_ms != valid_until_ms; + if evidence_changed && self.availability_evidence_can_advance().is_err() { + return HlsAccessLeaseActivation::Expired; + } + let proxy_session_id = new_lease.proxy_session_id.clone(); + let Some(new_lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsAccessLeaseActivation::UnknownLease; + }; + new_lease.state = HlsAccessLeaseState::Activated; + new_lease.last_seen_at_ms = now_ms; + new_lease.active_until_ms = active_until_ms; + new_lease.pending_deadline = None; + new_lease.valid_until_ms = valid_until_ms; + let lease = new_lease.clone(); + if evidence_changed && self.advance_availability_evidence(&proxy_session_id).is_err() { + return HlsAccessLeaseActivation::Expired; + } + + HlsAccessLeaseActivation::Activated { lease: Box::new(lease), previous_state } + } + + pub fn touch_manifest_access_lease( + &mut self, + lease_id: &HlsAccessLeaseId, + path_proxy_session_id: &ProxySessionId, + now_ms: u64, + active_timing: Option, + pending_deadline: Option, + valid_window_ms: u64, + ) -> HlsAccessLeaseTouch { + let Some(lease) = self.by_lease_id.get(lease_id) else { + return HlsAccessLeaseTouch::UnknownLease; + }; + if &lease.proxy_session_id != path_proxy_session_id { + return HlsAccessLeaseTouch::SessionMismatch; + } + if !lease_state_allows_use(lease.state) { + return touch_for_state(lease.state); + } + if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { + return HlsAccessLeaseTouch::Expired; + } + let Some(lease) = self.by_lease_id.get(lease_id) else { + return HlsAccessLeaseTouch::UnknownLease; + }; + if lease.state == HlsAccessLeaseState::Expired { + return HlsAccessLeaseTouch::Expired; + } + let before = (lease.active_until_ms, lease.pending_deadline, lease.valid_until_ms); + let proxy_session_id = lease.proxy_session_id.clone(); + if self.availability_evidence_can_advance().is_err() { + return HlsAccessLeaseTouch::Expired; + } + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsAccessLeaseTouch::UnknownLease; + }; + lease.last_seen_at_ms = now_ms; + match lease.state { + HlsAccessLeaseState::Pending => { + if let Some(pending_deadline) = pending_deadline { + lease.apply_pending_deadline(pending_deadline); + } + } + HlsAccessLeaseState::Activated => { + if let Some(timing) = active_timing { + lease.active_until_ms = Some(now_ms.saturating_add(timing.active_window_ms)); + lease.valid_until_ms = now_ms.saturating_add(timing.valid_window_ms); + } else { + lease.valid_until_ms = now_ms.saturating_add(valid_window_ms); + } + } + HlsAccessLeaseState::Idle => { + lease.valid_until_ms = now_ms.saturating_add(valid_window_ms); + } + HlsAccessLeaseState::PolicyRevoking | HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied => {} + } + let evidence_changed = before != (lease.active_until_ms, lease.pending_deadline, lease.valid_until_ms); + let lease = lease.clone(); + if evidence_changed && self.advance_availability_evidence(&proxy_session_id).is_err() { + return HlsAccessLeaseTouch::Expired; + } + HlsAccessLeaseTouch::Touched { lease: Box::new(lease) } + } + + pub fn mark_pending_manifest_follow_up_for_lease( + &mut self, + lease_id: &HlsAccessLeaseId, + path_proxy_session_id: &ProxySessionId, + now_ms: u64, + deadline: HlsAccessLeasePendingDeadline, + ) -> Option { + let lease = self.by_lease_id.get(lease_id)?; + if &lease.proxy_session_id != path_proxy_session_id { + return None; + } + self.refresh_access_lease_validity(lease_id, now_ms).ok()?; + let lease = self.by_lease_id.get(lease_id)?; + if lease.state != HlsAccessLeaseState::Pending { + return None; + } + let next_deadline = lease.pending_deadline.map_or(deadline, |current| current.tightened_with(deadline)); + if lease.pending_deadline == Some(next_deadline) { + return None; + } + if self.availability_evidence_can_advance().is_err() { + return None; + } + let proxy_session_id = lease.proxy_session_id.clone(); + let lease = self.by_lease_id.get_mut(lease_id)?; + lease.last_seen_at_ms = now_ms; + if !lease.apply_pending_deadline(deadline) { + return None; + } + let lease = lease.clone(); + self.advance_availability_evidence(&proxy_session_id).ok()?; + Some(lease) + } + + pub fn mark_pending_manifest_follow_up_for_session( + &mut self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + deadline: HlsAccessLeasePendingDeadline, + ) -> Vec { + if self.refresh_access_lease_validities_for_session(proxy_session_id, now_ms).is_err() { + return Vec::new(); + } + if self.availability_evidence_can_advance().is_err() { + return Vec::new(); + } + let mut leases = Vec::new(); + for lease in self.by_lease_id.values_mut() { + if lease.proxy_session_id != *proxy_session_id { + continue; + } + if lease.state != HlsAccessLeaseState::Pending { + continue; + } + lease.last_seen_at_ms = now_ms; + if lease.apply_pending_deadline(deadline) { + leases.push(lease.clone()); + } + } + if !leases.is_empty() && self.advance_availability_evidence(proxy_session_id).is_err() { + return Vec::new(); + } + leases + } + + pub fn touch_access_lease( + &mut self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + timing: HlsAccessLeaseTiming, + ) -> bool { + self.touch_access_lease_snapshot(lease_id, now_ms, timing).is_some() + } + + pub fn touch_access_lease_snapshot( + &mut self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + timing: HlsAccessLeaseTiming, + ) -> Option { + let lease = self.by_lease_id.get(lease_id)?; + if lease.state != HlsAccessLeaseState::Activated { + return None; + } + self.refresh_access_lease_validity(lease_id, now_ms).ok()?; + let lease = self.by_lease_id.get(lease_id)?; + if lease.state == HlsAccessLeaseState::Expired { + return None; + } + let active_until_ms = Some(now_ms.saturating_add(timing.active_window_ms)); + let valid_until_ms = now_ms.saturating_add(timing.valid_window_ms); + let evidence_changed = lease.active_until_ms != active_until_ms || lease.valid_until_ms != valid_until_ms; + if evidence_changed && self.availability_evidence_can_advance().is_err() { + return None; + } + let proxy_session_id = lease.proxy_session_id.clone(); + let lease = self.by_lease_id.get_mut(lease_id)?; + lease.last_seen_at_ms = now_ms; + lease.active_until_ms = active_until_ms; + lease.valid_until_ms = valid_until_ms; + let lease = lease.clone(); + if evidence_changed { + self.advance_availability_evidence(&proxy_session_id).ok()?; + } + Some(lease) + } + + pub fn lease_state(&self, lease_id: &HlsAccessLeaseId, now_ms: u64) -> Option { + self.by_lease_id.get(lease_id).map(|lease| { + if lease.validity_due_at_ms() <= now_ms { + HlsAccessLeaseState::Expired + } else { + lease.state + } + }) + } + + pub fn active_access_lease_count_for_session(&mut self, proxy_session_id: &ProxySessionId, now_ms: u64) -> usize { + if self.refresh_access_lease_validities_for_session(proxy_session_id, now_ms).is_err() { + return 0; + } + let mut active_count = 0; + for lease in self.by_lease_id.values() { + if lease.proxy_session_id == *proxy_session_id + && lease.state == HlsAccessLeaseState::Activated + && lease.active_until_ms.is_some_and(|active_until| active_until > now_ms) + { + active_count += 1; + } + } + active_count + } + + pub(crate) fn select_startup_repair_lease( + &mut self, + session: &ProxySessionId, + now_ms: u64, + ) -> Option { + self.refresh_access_lease_validities_for_session(session, now_ms).ok()?; + self.by_lease_id + .values() + .filter(|lease| { + lease.proxy_session_id == *session + && matches!(lease.playback_mode, HlsLeasePlaybackMode::Live) + && matches!( + lease.state, + HlsAccessLeaseState::Pending | HlsAccessLeaseState::Idle | HlsAccessLeaseState::Activated + ) + }) + .min_by(|left, right| { + left.issued_at_ms.cmp(&right.issued_at_ms).then_with(|| left.lease_id.0.cmp(&right.lease_id.0)) + }) + .map(|lease| lease.lease_id.clone()) + } + + pub fn has_usable_access_lease_for_session(&mut self, proxy_session_id: &ProxySessionId, now_ms: u64) -> bool { + if self.refresh_access_lease_validities_for_session(proxy_session_id, now_ms).is_err() { + return false; + } + let mut has_usable_lease = false; + for lease in self.by_lease_id.values() { + if lease.proxy_session_id == *proxy_session_id + && (lease.state == HlsAccessLeaseState::Pending + || lease.state == HlsAccessLeaseState::Idle + || (lease.state == HlsAccessLeaseState::Activated && lease.valid_until_ms > now_ms)) + { + has_usable_lease = true; + } + } + has_usable_lease + } + + pub fn session_snapshot( + &mut self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> HlsAccessLeaseSessionSnapshot { + let mut active_count = 0; + let mut effective_origin_policy = None; + let mut idle_releases = Vec::new(); + let mut finalized_transient_manifest_bindings = Vec::new(); + let lease_ids = self + .by_lease_id + .values() + .filter(|lease| lease.proxy_session_id == *proxy_session_id) + .map(|lease| lease.lease_id.clone()) + .collect::>(); + for lease_id in lease_ids { + match self.refresh_access_lease_activity(&lease_id, now_ms) { + Ok(Some(release)) => idle_releases.push(release), + Ok(None) => {} + Err(HlsAvailabilityEvidenceAdvanceError::Exhausted) => continue, + } + let Some(lease) = self.by_lease_id.get(&lease_id) else { + continue; + }; + if lease.state == HlsAccessLeaseState::Activated { + active_count += 1; + } + if matches!(lease.state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated) { + let candidate = + HlsEffectiveOriginAcquirePolicy::new(lease.origin_connection_kind, lease.origin_priority, now_ms); + effective_origin_policy = Some(effective_origin_policy.map_or(candidate, |current| { + if candidate.is_better_than(current) { + candidate + } else { + current + } + })); + } + if lease_state_allows_use(lease.state) && lease.playback_mode == HlsLeasePlaybackMode::Live { + finalized_transient_manifest_bindings.extend( + lease.published_finalized_manifest_generations.iter().map(|manifest_generation| { + TransientManifestLeaseBinding::new( + lease.lease_id.clone(), + lease.issued_at_ms, + *manifest_generation, + ) + }), + ); + } + } + HlsAccessLeaseSessionSnapshot { + active_count, + effective_origin_policy, + idle_releases, + finalized_transient_manifest_bindings, + } + } + + pub fn lifecycle_snapshot( + &mut self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + ) -> Option { + let idle_release = self.refresh_access_lease_activity(lease_id, now_ms).ok()?; + let lease = self.by_lease_id.get(lease_id)?; + Some(HlsAccessLeaseLifecycleSnapshot { + lease_id: lease.lease_id.clone(), + proxy_session_id: lease.proxy_session_id.clone(), + state: lease.state, + active_until_ms: lease.active_until_ms, + pending_deadline: lease.pending_deadline, + valid_until_ms: lease.valid_until_ms, + idle_release, + }) + } +} + +pub(super) const fn lease_state_allows_use(state: HlsAccessLeaseState) -> bool { + matches!(state, HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle) +} + +const fn lease_state_protects_live_evidence(state: HlsAccessLeaseState) -> bool { + lease_state_allows_use(state) || matches!(state, HlsAccessLeaseState::PolicyRevoking) +} + +const fn activation_for_state(state: HlsAccessLeaseState) -> HlsAccessLeaseActivation { + match state { + HlsAccessLeaseState::Expired => HlsAccessLeaseActivation::Expired, + HlsAccessLeaseState::PolicyRevoking | HlsAccessLeaseState::Denied => HlsAccessLeaseActivation::Denied, + HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle => { + HlsAccessLeaseActivation::UnknownLease + } + } +} + +const fn touch_for_state(state: HlsAccessLeaseState) -> HlsAccessLeaseTouch { + match state { + HlsAccessLeaseState::Expired => HlsAccessLeaseTouch::Expired, + HlsAccessLeaseState::PolicyRevoking | HlsAccessLeaseState::Denied => HlsAccessLeaseTouch::Denied, + HlsAccessLeaseState::Pending | HlsAccessLeaseState::Activated | HlsAccessLeaseState::Idle => { + HlsAccessLeaseTouch::UnknownLease + } + } +} diff --git a/backend/hls/src/lease/publication.rs b/backend/hls/src/lease/publication.rs new file mode 100644 index 000000000..d0a8ad68d --- /dev/null +++ b/backend/hls/src/lease/publication.rs @@ -0,0 +1,194 @@ +use super::{ + commit_lease_revision_bindings, lease_state_allows_use, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, + HlsAccessLeaseStore, HlsAvailabilityEvidenceAdvanceError, HlsAvailabilityEvidenceGeneration, + HlsLeaseManifestPublicationGuard, HlsLeaseManifestPublicationOutcome, HlsLeaseManifestPublicationRejectReason, + HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, HlsLeaseStartupAdmissionState, HlsPublishedTransientResourceIds, + ProxySessionId, +}; + +impl HlsAccessLease { + pub fn published_transient_resource_ids(&self) -> &HlsPublishedTransientResourceIds { + &self.published_transient_resource_ids + } + + fn manifest_publication_guard(&self) -> Option { + (lease_state_allows_use(self.state) && self.playback_mode == HlsLeasePlaybackMode::Live).then_some( + HlsLeaseManifestPublicationGuard { + issued_at_ms: self.issued_at_ms, + admission_generation: self.admission_generation, + }, + ) + } +} + +impl HlsAccessLeaseStore { + pub fn availability_evidence_generation( + &self, + proxy_session_id: &ProxySessionId, + ) -> HlsAvailabilityEvidenceGeneration { + self.availability_generation_by_proxy_session + .get(proxy_session_id) + .copied() + .unwrap_or(HlsAvailabilityEvidenceGeneration::NONE) + } + + pub(super) fn availability_evidence_can_advance(&self) -> Result<(), HlsAvailabilityEvidenceAdvanceError> { + self.last_availability_evidence_generation + .0 + .checked_add(1) + .map(|_| ()) + .ok_or(HlsAvailabilityEvidenceAdvanceError::Exhausted) + } + + pub(super) fn advance_availability_evidence( + &mut self, + proxy_session_id: &ProxySessionId, + ) -> Result { + let generation = self + .last_availability_evidence_generation + .0 + .checked_add(1) + .map(HlsAvailabilityEvidenceGeneration) + .ok_or(HlsAvailabilityEvidenceAdvanceError::Exhausted)?; + self.last_availability_evidence_generation = generation; + self.availability_generation_by_proxy_session.insert(proxy_session_id.clone(), generation); + Ok(generation) + } + + pub fn prepare_manifest_publication( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Option { + let lease = self.by_lease_id.get(lease_id)?; + if lease.proxy_session_id != *proxy_session_id { + return None; + } + if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { + return None; + } + self.by_lease_id.get(lease_id)?.manifest_publication_guard() + } + + pub(crate) fn published_transient_resource_ids( + &self, + lease_id: &HlsAccessLeaseId, + ) -> Option<&HlsPublishedTransientResourceIds> { + self.by_lease_id.get(lease_id).map(HlsAccessLease::published_transient_resource_ids) + } + + pub(crate) fn commit_manifest_publication_with_resources( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + expected: HlsLeaseManifestPublicationGuard, + mut snapshot: HlsLeaseManifestSnapshot, + published_transient_resource_ids: HlsPublishedTransientResourceIds, + now_ms: u64, + ) -> HlsLeaseManifestPublicationOutcome { + let Some(lease) = self.by_lease_id.get(lease_id) else { + return HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::UnknownLease); + }; + if lease.proxy_session_id != *proxy_session_id { + return HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::SessionMismatch, + ); + } + if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { + return HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::SnapshotGenerationExhausted, + ); + } + let Some(lease) = self.by_lease_id.get(lease_id) else { + return HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::UnknownLease); + }; + if lease.state == HlsAccessLeaseState::Expired { + return HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::LeaseExpired); + } + if !lease_state_allows_use(lease.state) { + return HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::LeaseUnavailable, + ); + } + if lease.issued_at_ms != expected.issued_at_ms { + return HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::LeaseIncarnationChanged, + ); + } + if lease.admission_generation != expected.admission_generation { + return HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::AdmissionGenerationChanged, + ); + } + if lease.playback_mode != HlsLeasePlaybackMode::Live { + return HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::LeaseNotLive); + } + if lease + .last_manifest_snapshot + .as_ref() + .is_some_and(|current| manifest_source_is_regressive(current, &snapshot)) + { + return HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::SourceRegressive, + ); + } + if self.availability_evidence_can_advance().is_err() { + return HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::SnapshotGenerationExhausted, + ); + } + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::UnknownLease); + }; + let next_generation = lease.manifest_snapshot_generation.saturating_add(1); + if next_generation == lease.manifest_snapshot_generation { + return HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::SnapshotGenerationExhausted, + ); + } + if !commit_lease_revision_bindings(lease, &snapshot) { + return HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::RevisionConflict, + ); + } + lease.manifest_snapshot_generation = next_generation; + snapshot.snapshot_generation = next_generation; + let finalized_manifest_generation = snapshot.finalized_transient_manifest_generation; + lease.last_manifest_snapshot = Some(snapshot); + lease.published_transient_resource_ids = published_transient_resource_ids; + if let Some(manifest_generation) = finalized_manifest_generation { + lease.published_finalized_manifest_generations.insert(manifest_generation); + } + lease.startup_admission = HlsLeaseStartupAdmissionState::Admitted; + if self.advance_availability_evidence(proxy_session_id).is_err() { + return HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::SnapshotGenerationExhausted, + ); + } + HlsLeaseManifestPublicationOutcome::Committed { snapshot_generation: next_generation } + } + + #[cfg(any(test, feature = "test-support"))] + pub fn commit_manifest_publication( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + expected: HlsLeaseManifestPublicationGuard, + snapshot: HlsLeaseManifestSnapshot, + now_ms: u64, + ) -> HlsLeaseManifestPublicationOutcome { + self.commit_manifest_publication_with_resources( + lease_id, + proxy_session_id, + expected, + snapshot, + HlsPublishedTransientResourceIds::default(), + now_ms, + ) + } +} + +fn manifest_source_is_regressive(current: &HlsLeaseManifestSnapshot, candidate: &HlsLeaseManifestSnapshot) -> bool { + candidate.source_commit_identity.commit_generation() < current.source_commit_identity.commit_generation() +} diff --git a/backend/hls/src/lease/revision_bindings.rs b/backend/hls/src/lease/revision_bindings.rs new file mode 100644 index 000000000..c582b9ce8 --- /dev/null +++ b/backend/hls/src/lease/revision_bindings.rs @@ -0,0 +1,80 @@ +use super::{ + HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseStore, HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, + ProxySessionId, +}; +use std::sync::Arc; + +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub(super) struct HlsLeaseRevisionBindings { + pub(super) revisions: std::collections::BTreeMap, + pub(super) retired_through: Option, +} + +/// Upper bound of revisions one lease keeps bound, independent of the retained window. +const MAX_LEASE_REVISION_BINDINGS: usize = 1024; + +impl HlsLeaseRevisionBindings { + /// Pops the oldest bindings while `retire(seq, current_len)` holds and remembers the newest retired sequence. + fn retire_while(&mut self, mut retire: impl FnMut(u64, usize) -> bool) { + loop { + let len = self.revisions.len(); + let Some(entry) = self.revisions.first_entry() else { break }; + if !retire(*entry.key(), len) { + break; + } + self.retired_through = Some(entry.remove_entry().0); + } + } +} + +impl HlsAccessLeaseStore { + pub fn prune_revision_bindings(&mut self, session: &ProxySessionId, first_retained_seq: u64) { + for lease in self.by_lease_id.values_mut().filter(|lease| lease.proxy_session_id == *session) { + let Some(bindings) = &mut lease.revision_bindings else { continue }; + if bindings.revisions.first_key_value().is_some_and(|(seq, _)| *seq < first_retained_seq) { + Arc::make_mut(bindings).retire_while(|seq, _| seq < first_retained_seq); + } + } + } + pub fn published_segment_revision( + &self, + lease_id: &HlsAccessLeaseId, + seq: u64, + ) -> Option { + self.by_lease_id.get(lease_id)?.revision_bindings.as_ref()?.revisions.get(&seq).cloned() + } + pub fn try_claim_progressive_startup(&mut self, lease_id: &HlsAccessLeaseId, seq: u64, now_ms: u64) -> bool { + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return false; + }; + if lease.playback_mode != HlsLeasePlaybackMode::Live + || lease.progressive_startup_claimed + || lease.validity_due_at_ms() <= now_ms + || lease.last_manifest_snapshot.as_ref().is_none_or(|snapshot| snapshot.first_proxy_seq != seq) + { + return false; + } + lease.progressive_startup_claimed = true; + true + } +} + +pub(super) fn commit_lease_revision_bindings(lease: &mut HlsAccessLease, snapshot: &HlsLeaseManifestSnapshot) -> bool { + if let Some(publication) = &snapshot.startup_revisions { + let current = lease.revision_bindings.as_ref(); + if publication.revisions.iter().any(|(seq, revision)| { + !revision.revision().is_publishable() + || current.is_some_and(|current| { + current.retired_through.is_some_and(|retired| *seq <= retired) + || current.revisions.get(seq).is_some_and(|bound| bound != revision) + }) + }) { + return false; + } + let bindings = lease.revision_bindings.get_or_insert_with(|| Arc::new(HlsLeaseRevisionBindings::default())); + let bindings = Arc::make_mut(bindings); + bindings.revisions.extend(publication.revisions.iter().map(|(seq, revision)| (*seq, revision.clone()))); + bindings.retire_while(|seq, len| len > MAX_LEASE_REVISION_BINDINGS || seq < publication.retained_start_seq); + } + true +} diff --git a/backend/hls/src/lease/runtime_policy.rs b/backend/hls/src/lease/runtime_policy.rs new file mode 100644 index 000000000..a3794c35d --- /dev/null +++ b/backend/hls/src/lease/runtime_policy.rs @@ -0,0 +1,246 @@ +use super::{ + HlsAccessLease, HlsAccessLeaseDenialMode, HlsAccessLeaseDenialOutcome, HlsAccessLeaseId, HlsAccessLeaseState, + HlsAccessLeaseStore, HlsDeniedTerminalTailRelease, HlsFiniteTailTrigger, HlsLeaseManifestSnapshot, + HlsLeasePlaybackCursor, HlsLeasePlaybackMode, HlsRuntimeCustomTailReason, HlsRuntimePolicyRevocation, + HlsRuntimePolicyRevocationOutcome, HlsTerminalCommitOutcome, HlsTerminalTailPreparation, ProxySessionId, +}; +use std::sync::Arc; + +impl HlsAccessLease { + pub fn runtime_policy_revocation_outcome(&self) -> Option { + match &self.playback_mode { + HlsLeasePlaybackMode::TerminalTail(plan) => { + Some(HlsRuntimePolicyRevocationOutcome::AlreadyCommitted { plan: Arc::clone(plan) }) + } + HlsLeasePlaybackMode::Live => self + .runtime_policy_revocation + .clone() + .map(|token| HlsRuntimePolicyRevocationOutcome::AlreadyPending { token }), + HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => None, + } + } + + pub fn runtime_policy_denial_reason(&self) -> Option { + match &self.playback_mode { + HlsLeasePlaybackMode::TerminalTail(plan) => Some(plan.reason), + HlsLeasePlaybackMode::Live => self.runtime_policy_revocation.as_ref().map(|token| token.reason), + HlsLeasePlaybackMode::TerminalUnavailable { .. } | HlsLeasePlaybackMode::Ended => { + self.runtime_policy_denial_reason + } + } + } + + pub(super) fn runtime_policy_tail_commit_preconditions_match( + &mut self, + proxy_session_id: &ProxySessionId, + preparation: &HlsTerminalTailPreparation, + revocation: &HlsRuntimePolicyRevocation, + ) -> Result<(), HlsTerminalCommitOutcome> { + if self.state != HlsAccessLeaseState::PolicyRevoking + || self.proxy_session_id != *proxy_session_id + || self.issued_at_ms != revocation.lease_issued_at_ms + || self.playback_mode != HlsLeasePlaybackMode::Live + || self.runtime_policy_revocation.as_ref() != Some(revocation) + || preparation.runtime_policy_revocation.as_ref() != Some(revocation) + || preparation.trigger != HlsFiniteTailTrigger::RuntimePolicy(revocation.reason) + { + return Err(HlsTerminalCommitOutcome::LeaseNoLongerEligible); + } + if self.admission_generation != revocation.expected_admission_generation + || preparation.expected_admission_generation != revocation.expected_admission_generation + || self.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation) + != Some(revocation.manifest_snapshot_generation) + || preparation.manifest_snapshot_generation != revocation.manifest_snapshot_generation + || self.playback_cursor.cursor_generation != revocation.cursor_generation + || preparation.cursor_generation != revocation.cursor_generation + { + return Err(HlsTerminalCommitOutcome::SupersededGeneration); + } + Ok(()) + } +} + +impl HlsAccessLeaseStore { + pub fn begin_runtime_policy_revocation( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + reason: HlsRuntimeCustomTailReason, + now_ms: u64, + ) -> HlsRuntimePolicyRevocationOutcome { + let Some(lease) = self.by_lease_id.get(lease_id) else { + return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; + }; + if lease.proxy_session_id != *proxy_session_id { + return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; + } + if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { + return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; + } + let Some(lease) = self.by_lease_id.get(lease_id) else { + return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; + }; + if let HlsLeasePlaybackMode::TerminalTail(plan) = &lease.playback_mode { + return HlsRuntimePolicyRevocationOutcome::AlreadyCommitted { plan: Arc::clone(plan) }; + } + if lease.state == HlsAccessLeaseState::PolicyRevoking { + return lease + .runtime_policy_revocation + .clone() + .filter(|token| token.reason == reason) + .map_or(HlsRuntimePolicyRevocationOutcome::NoLongerEligible, |token| { + HlsRuntimePolicyRevocationOutcome::AlreadyPending { token } + }); + } + let Some(manifest) = lease.last_manifest_snapshot.as_ref() else { + return HlsRuntimePolicyRevocationOutcome::NoPublishedManifest; + }; + if lease.state != HlsAccessLeaseState::Activated || lease.playback_mode != HlsLeasePlaybackMode::Live { + return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; + } + let token = HlsRuntimePolicyRevocation { + reason, + lease_issued_at_ms: lease.issued_at_ms, + expected_admission_generation: lease.admission_generation, + manifest_snapshot_generation: manifest.snapshot_generation, + cursor_generation: lease.playback_cursor.cursor_generation, + started_at_ms: now_ms, + }; + if self.availability_evidence_can_advance().is_err() { + return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; + } + let lease_proxy_session_id = lease.proxy_session_id.clone(); + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; + }; + lease.state = HlsAccessLeaseState::PolicyRevoking; + lease.runtime_policy_revocation = Some(token.clone()); + lease.runtime_policy_denial_reason = None; + if self.advance_availability_evidence(&lease_proxy_session_id).is_err() { + return HlsRuntimePolicyRevocationOutcome::NoLongerEligible; + } + HlsRuntimePolicyRevocationOutcome::Started { token } + } + + pub fn fail_runtime_policy_revocation( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + token: &HlsRuntimePolicyRevocation, + ) -> HlsAccessLeaseDenialOutcome { + let Some(lease) = self.by_lease_id.get(lease_id) else { + return HlsAccessLeaseDenialOutcome::UnknownLease; + }; + if lease.proxy_session_id != *proxy_session_id + || lease.state != HlsAccessLeaseState::PolicyRevoking + || lease.runtime_policy_revocation.as_ref() != Some(token) + { + return HlsAccessLeaseDenialOutcome::UnknownLease; + } + if self.availability_evidence_can_advance().is_err() { + return HlsAccessLeaseDenialOutcome::UnknownLease; + } + let lease_proxy_session_id = lease.proxy_session_id.clone(); + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsAccessLeaseDenialOutcome::UnknownLease; + }; + lease.state = HlsAccessLeaseState::Denied; + lease.runtime_policy_denial_reason = Some(token.reason); + lease.end_playback(); + if self.advance_availability_evidence(&lease_proxy_session_id).is_err() { + return HlsAccessLeaseDenialOutcome::UnknownLease; + } + HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } + } + + pub fn deny_access_lease( + &mut self, + lease_id: &HlsAccessLeaseId, + mode: HlsAccessLeaseDenialMode, + ) -> HlsAccessLeaseDenialOutcome { + let Some(lease) = self.by_lease_id.get(lease_id) else { + return HlsAccessLeaseDenialOutcome::UnknownLease; + }; + if lease.state == HlsAccessLeaseState::PolicyRevoking { + return HlsAccessLeaseDenialOutcome::PolicyRevocationPending; + } + let evidence_changed = lease.state != HlsAccessLeaseState::Denied; + if evidence_changed && self.availability_evidence_can_advance().is_err() { + return HlsAccessLeaseDenialOutcome::UnknownLease; + } + let proxy_session_id = lease.proxy_session_id.clone(); + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsAccessLeaseDenialOutcome::UnknownLease; + }; + if evidence_changed { + lease.state = HlsAccessLeaseState::Denied; + } + let preserve_finite_decision = matches!( + lease.playback_mode, + HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } + ); + if preserve_finite_decision { + if evidence_changed { + lease.runtime_policy_denial_reason = match &lease.playback_mode { + HlsLeasePlaybackMode::TerminalTail(plan) => Some(plan.reason), + HlsLeasePlaybackMode::TerminalUnavailable { .. } => lease.runtime_policy_denial_reason, + HlsLeasePlaybackMode::Live | HlsLeasePlaybackMode::Ended => None, + }; + } + if evidence_changed && self.advance_availability_evidence(&proxy_session_id).is_err() { + return HlsAccessLeaseDenialOutcome::UnknownLease; + } + return HlsAccessLeaseDenialOutcome::FiniteDecisionPreserved; + } + if evidence_changed { + lease.end_playback(); + lease.runtime_policy_denial_reason = match mode { + HlsAccessLeaseDenialMode::ImmediateRuntimePolicyEnd { reason } => Some(reason), + HlsAccessLeaseDenialMode::PreserveCommittedFiniteTail => None, + #[cfg(test)] + HlsAccessLeaseDenialMode::ImmediateEnd => None, + }; + } + let generation = lease.pending_terminal_protection_release; + if evidence_changed && self.advance_availability_evidence(&proxy_session_id).is_err() { + return HlsAccessLeaseDenialOutcome::UnknownLease; + } + HlsAccessLeaseDenialOutcome::Ended { + terminal_release: generation + .map(|generation| HlsDeniedTerminalTailRelease { proxy_session_id, generation }), + } + } +} + +pub(super) fn runtime_policy_authorized_manifest_prefix( + mut manifest: HlsLeaseManifestSnapshot, + cursor: &HlsLeasePlaybackCursor, +) -> Option { + let authorized_last_proxy_seq = + match (cursor.highest_contiguous_completed_proxy_seq, cursor.last_requested_proxy_seq) { + (Some(completed), Some(requested)) => completed.max(requested), + (Some(completed), None) => completed, + (None, Some(requested)) => requested, + (None, None) => return None, + } + .min(manifest.last_proxy_seq); + let end = manifest + .visible_segments + .iter() + .position(|segment| segment.proxy_seq > authorized_last_proxy_seq) + .unwrap_or(manifest.visible_segments.len()); + let selected = manifest.visible_segments.get(..end)?.to_vec(); + let first = selected.first()?; + let last = selected.last()?; + let first_proxy_seq = first.proxy_seq; + let last_proxy_seq = last.proxy_seq; + let active_encryption = last.encryption.clone(); + let playlist_duration_ms = selected.iter().fold(0_u64, |total, segment| total.saturating_add(segment.duration_ms)); + manifest.first_proxy_seq = first_proxy_seq; + manifest.last_proxy_seq = last_proxy_seq; + manifest.visible_segments = Arc::from(selected); + manifest.playlist_duration_ms = playlist_duration_ms; + manifest.last_visible_media_end_ms = playlist_duration_ms; + manifest.active_encryption = active_encryption; + Some(manifest) +} diff --git a/backend/hls/src/lease/terminal.rs b/backend/hls/src/lease/terminal.rs new file mode 100644 index 000000000..63a202782 --- /dev/null +++ b/backend/hls/src/lease/terminal.rs @@ -0,0 +1,374 @@ +use super::{ + lease_state_allows_use, runtime_policy_authorized_manifest_prefix, HlsAccessLease, HlsAccessLeaseId, + HlsAccessLeaseState, HlsAccessLeaseStore, HlsFiniteTailTrigger, HlsLeasePlaybackMode, + HlsLeaseStartupAdmissionState, HlsRuntimeCustomTailBasePolicy, HlsRuntimeCustomTailReason, + HlsTerminalCommitOutcome, HlsTerminalMediaPreparationKey, HlsTerminalMediaPreparationState, + HlsTerminalMediaRequirementOrigin, HlsTerminalMediaRequirementSource, HlsTerminalTailCompatibility, + HlsTerminalTailGeneration, HlsTerminalTailPlan, HlsTerminalTailPreparation, HlsTerminalTailPreparationInput, + ProxySessionId, +}; + +impl HlsAccessLease { + pub fn permits_unpublished_standalone_tail(&self, reason: HlsRuntimeCustomTailReason) -> bool { + reason.permits_unpublished_lease_standalone_tail() + && self.state == HlsAccessLeaseState::Pending + && self.playback_mode == HlsLeasePlaybackMode::Live + && self.startup_admission == HlsLeaseStartupAdmissionState::Pending + && self.last_manifest_snapshot.is_none() + && self.runtime_policy_revocation.is_none() + } + + fn terminal_commit_preconditions_match( + &mut self, + proxy_session_id: &ProxySessionId, + expected_issued_at_ms: u64, + expected_admission_generation: u64, + manifest_snapshot_generation: u64, + cursor_generation: u64, + _now_ms: u64, + ) -> Result<(), HlsTerminalCommitOutcome> { + if !lease_state_allows_use(self.state) + || self.proxy_session_id != *proxy_session_id + || self.issued_at_ms != expected_issued_at_ms + || self.playback_mode != HlsLeasePlaybackMode::Live + { + return Err(HlsTerminalCommitOutcome::LeaseNoLongerEligible); + } + if self.admission_generation != expected_admission_generation + || self.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation) + != Some(manifest_snapshot_generation) + || self.playback_cursor.cursor_generation != cursor_generation + { + return Err(HlsTerminalCommitOutcome::SupersededGeneration); + } + Ok(()) + } + + fn terminal_decision_commit_preconditions_match( + &mut self, + proxy_session_id: &ProxySessionId, + preparation: &HlsTerminalTailPreparation, + now_ms: u64, + ) -> Result<(), HlsTerminalCommitOutcome> { + match preparation.runtime_policy_revocation.as_ref() { + Some(revocation) => { + self.runtime_policy_tail_commit_preconditions_match(proxy_session_id, preparation, revocation) + } + None => self.terminal_commit_preconditions_match( + proxy_session_id, + preparation.lease_issued_at_ms, + preparation.expected_admission_generation, + preparation.manifest_snapshot_generation, + preparation.cursor_generation, + now_ms, + ), + } + } +} + +fn terminal_commit_existing_outcome( + lease: &mut HlsAccessLease, + proxy_session_id: &ProxySessionId, + preparation: &HlsTerminalTailPreparation, + _now_ms: u64, +) -> Option { + if lease.proxy_session_id != *proxy_session_id || lease.issued_at_ms != preparation.lease_issued_at_ms { + return Some(HlsTerminalCommitOutcome::LeaseNoLongerEligible); + } + match &lease.playback_mode { + HlsLeasePlaybackMode::Live => None, + HlsLeasePlaybackMode::TerminalTail(current) if current.generation.0 == preparation.decision_generation => { + Some(HlsTerminalCommitOutcome::AlreadyCommitted) + } + HlsLeasePlaybackMode::TerminalUnavailable { decision_generation, .. } + if *decision_generation == preparation.decision_generation => + { + Some(HlsTerminalCommitOutcome::AlreadyCommitted) + } + HlsLeasePlaybackMode::TerminalTail(_) | HlsLeasePlaybackMode::TerminalUnavailable { .. } => { + Some(HlsTerminalCommitOutcome::SupersededGeneration) + } + HlsLeasePlaybackMode::Ended => Some(HlsTerminalCommitOutcome::LeaseNoLongerEligible), + } +} + +impl HlsTerminalTailPreparation { + pub fn bind_ready_terminal_media_requirement( + &mut self, + prepared_key: HlsTerminalMediaPreparationKey, + ) -> Result<(), HlsTerminalTailCompatibility> { + let source = match self.terminal_media_requirement_source { + HlsTerminalMediaRequirementSource::AcceptanceEpisode { .. } => { + if self.required_terminal_media_key != Some(prepared_key) { + return Err(HlsTerminalTailCompatibility::AssetRevisionMismatch); + } + self.terminal_media_requirement_source + } + HlsTerminalMediaRequirementSource::CutoverSnapshotPending { decision_generation } + if decision_generation == self.decision_generation => + { + HlsTerminalMediaRequirementSource::CutoverSnapshot { decision_generation, asset: prepared_key.asset } + } + HlsTerminalMediaRequirementSource::CutoverSnapshot { decision_generation, asset } + if decision_generation == self.decision_generation && asset == prepared_key.asset => + { + self.terminal_media_requirement_source + } + HlsTerminalMediaRequirementSource::CutoverSnapshotPending { .. } + | HlsTerminalMediaRequirementSource::CutoverSnapshot { .. } => { + return Err(HlsTerminalTailCompatibility::AssetRevisionMismatch); + } + }; + self.terminal_media_requirement_source = source; + self.required_terminal_media_key = Some(prepared_key); + self.terminal_media_preparation = HlsTerminalMediaPreparationState::Ready { key: prepared_key }; + Ok(()) + } +} + +impl HlsTerminalMediaRequirementSource { + pub fn authorizes_tail(self, decision_generation: u64, prepared_key: HlsTerminalMediaPreparationKey) -> bool { + match self { + Self::AcceptanceEpisode { .. } => true, + Self::CutoverSnapshotPending { .. } => false, + Self::CutoverSnapshot { decision_generation: source_generation, asset } => { + source_generation == decision_generation && asset == prepared_key.asset + } + } + } +} + +impl HlsAccessLeaseStore { + pub fn all_live_leases_terminal_for_session(&self, proxy_session_id: &ProxySessionId) -> bool { + let mut found = false; + for lease in self.by_lease_id.values().filter(|lease| { + lease.proxy_session_id == *proxy_session_id + && matches!( + lease.state, + HlsAccessLeaseState::Pending + | HlsAccessLeaseState::Activated + | HlsAccessLeaseState::Idle + | HlsAccessLeaseState::PolicyRevoking + | HlsAccessLeaseState::Denied + ) + }) { + found = true; + if !matches!( + lease.playback_mode, + HlsLeasePlaybackMode::TerminalTail(_) + | HlsLeasePlaybackMode::TerminalUnavailable { .. } + | HlsLeasePlaybackMode::Ended + ) { + return false; + } + } + found + } + + pub fn prepare_terminal_tail( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + input: &HlsTerminalTailPreparationInput, + ) -> Option { + let lease = self.by_lease_id.get(lease_id)?; + if lease.proxy_session_id != *proxy_session_id || lease.playback_mode != HlsLeasePlaybackMode::Live { + return None; + } + let mut manifest_snapshot = lease.last_manifest_snapshot.clone()?; + if manifest_snapshot.snapshot_generation != input.expected_manifest_snapshot_generation + || lease.playback_cursor.cursor_generation != input.expected_cursor_generation + { + return None; + } + let runtime_policy_revocation = match input.trigger { + HlsFiniteTailTrigger::AvailabilityReserve => None, + HlsFiniteTailTrigger::RuntimePolicy(reason) => { + let revocation = lease.runtime_policy_revocation.clone()?; + if lease.state != HlsAccessLeaseState::PolicyRevoking + || revocation.reason != reason + || revocation.lease_issued_at_ms != lease.issued_at_ms + || revocation.expected_admission_generation != lease.admission_generation + || revocation.manifest_snapshot_generation != manifest_snapshot.snapshot_generation + || revocation.cursor_generation != lease.playback_cursor.cursor_generation + { + return None; + } + if reason.base_policy() == HlsRuntimeCustomTailBasePolicy::PreserveCompletedOrInFlightPrefix { + manifest_snapshot = + runtime_policy_authorized_manifest_prefix(manifest_snapshot, &lease.playback_cursor)?; + } + Some(revocation) + } + }; + let expected_admission_generation = lease.admission_generation; + let decision_generation = expected_admission_generation.saturating_add(1); + if decision_generation == expected_admission_generation { + return None; + } + let terminal_media_requirement_source = match input.terminal_media_requirement_origin { + HlsTerminalMediaRequirementOrigin::AcceptanceEpisode { generation } + if generation == input.expected_acceptance_generation => + { + HlsTerminalMediaRequirementSource::AcceptanceEpisode { generation } + } + HlsTerminalMediaRequirementOrigin::CutoverSnapshot => { + HlsTerminalMediaRequirementSource::CutoverSnapshotPending { decision_generation } + } + HlsTerminalMediaRequirementOrigin::AcceptanceEpisode { .. } => return None, + }; + Some(HlsTerminalTailPreparation { + trigger: input.trigger, + runtime_policy_revocation, + lease_issued_at_ms: lease.issued_at_ms, + decision_generation, + expected_admission_generation, + manifest_snapshot_generation: manifest_snapshot.snapshot_generation, + cursor_generation: lease.playback_cursor.cursor_generation, + origin_progress_generation: input.origin_progress_generation, + media_readiness_generation: input.media_readiness_generation, + origin_epoch: input.origin_epoch, + last_media_progress_at_ms: input.last_media_progress_at_ms, + expected_acceptance_generation: input.expected_acceptance_generation, + terminal_media_requirement_source, + cutover_timing: input.cutover_timing, + commit_window: input.commit_window, + required_terminal_media_key: input.required_terminal_media_key, + terminal_media_preparation: input.terminal_media_preparation, + reserve: input.reserve, + manifest_snapshot, + }) + } + + pub fn commit_terminal_tail_if_generation_matches( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + preparation: &HlsTerminalTailPreparation, + now_ms: u64, + plan: std::sync::Arc, + ) -> HlsTerminalCommitOutcome { + if !plan.matches_route(proxy_session_id, lease_id) + || plan.generation.0 != preparation.decision_generation + || plan.base_manifest.snapshot_generation != preparation.manifest_snapshot_generation + { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsTerminalCommitOutcome::LeaseNoLongerEligible; + }; + if let Some(outcome) = terminal_commit_existing_outcome(lease, proxy_session_id, preparation, now_ms) { + return outcome; + } + if let Err(outcome) = lease.terminal_decision_commit_preconditions_match(proxy_session_id, preparation, now_ms) + { + return outcome; + } + if self.availability_evidence_can_advance().is_err() { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsTerminalCommitOutcome::LeaseNoLongerEligible; + }; + lease.admission_generation = preparation.decision_generation; + lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(plan); + if let Some(revocation) = preparation.runtime_policy_revocation.as_ref() { + lease.state = HlsAccessLeaseState::Denied; + lease.runtime_policy_denial_reason = Some(revocation.reason); + lease.runtime_policy_revocation = None; + } + if self.advance_availability_evidence(proxy_session_id).is_err() { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + HlsTerminalCommitOutcome::Committed + } + + pub fn terminal_tail_replay_outcome( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + preparation: &HlsTerminalTailPreparation, + now_ms: u64, + ) -> Option { + self.refresh_access_lease_validity(lease_id, now_ms).ok()?; + let lease = self.by_lease_id.get_mut(lease_id)?; + terminal_commit_existing_outcome(lease, proxy_session_id, preparation, now_ms) + } + + pub fn commit_terminal_unavailable_if_generation_matches( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + preparation: &HlsTerminalTailPreparation, + now_ms: u64, + reason: HlsTerminalTailCompatibility, + ) -> HlsTerminalCommitOutcome { + if self.refresh_access_lease_validity(lease_id, now_ms).is_err() { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsTerminalCommitOutcome::LeaseNoLongerEligible; + }; + if let Some(outcome) = terminal_commit_existing_outcome(lease, proxy_session_id, preparation, now_ms) { + return outcome; + } + if let Err(outcome) = lease.terminal_decision_commit_preconditions_match(proxy_session_id, preparation, now_ms) + { + return outcome; + } + if self.availability_evidence_can_advance().is_err() { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return HlsTerminalCommitOutcome::LeaseNoLongerEligible; + }; + lease.admission_generation = preparation.decision_generation; + lease.playback_mode = + HlsLeasePlaybackMode::TerminalUnavailable { decision_generation: preparation.decision_generation, reason }; + if let Some(revocation) = preparation.runtime_policy_revocation.as_ref() { + lease.state = HlsAccessLeaseState::Denied; + lease.runtime_policy_denial_reason = Some(revocation.reason); + lease.runtime_policy_revocation = None; + } + if self.advance_availability_evidence(proxy_session_id).is_err() { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + HlsTerminalCommitOutcome::Committed + } + + pub fn terminal_unavailable_replay_outcome( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + preparation: &HlsTerminalTailPreparation, + now_ms: u64, + ) -> Option { + self.refresh_access_lease_validity(lease_id, now_ms).ok()?; + let lease = self.by_lease_id.get_mut(lease_id)?; + terminal_commit_existing_outcome(lease, proxy_session_id, preparation, now_ms) + } + + /// Clears a persisted cleanup ticket only for the same ended lease/session/generation. + pub fn acknowledge_terminal_protection_release( + &mut self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + generation: HlsTerminalTailGeneration, + ) -> bool { + let Some(lease) = self.by_lease_id.get_mut(lease_id) else { + return false; + }; + if !matches!(lease.state, HlsAccessLeaseState::Denied | HlsAccessLeaseState::Expired) + || lease.proxy_session_id != *proxy_session_id + || lease.playback_mode != HlsLeasePlaybackMode::Ended + || lease.pending_terminal_protection_release != Some(generation) + { + return false; + } + lease.pending_terminal_protection_release = None; + true + } +} diff --git a/backend/hls/src/lease/tests.rs b/backend/hls/src/lease/tests.rs new file mode 100644 index 000000000..515c26975 --- /dev/null +++ b/backend/hls/src/lease/tests.rs @@ -0,0 +1,37 @@ +use super::{ + super::{ + build_terminal_tail_plan, + manifest_acceptance::HlsManifestAcceptanceGeneration, + media_reserve::{ + HlsLeaseReserveAvailabilityBasis, HlsLeaseReserveSnapshot, HlsManifestCommitIdentity, + HlsPlaybackCompletionOutcome, + }, + recovery_timing::{ + HlsLeaseCutoverTiming, HlsTerminalCommitWindow, HlsTerminalMediaPreparationState, HlsTransitionMarginMs, + }, + runtime_custom_tail::{HlsFiniteTailTrigger, HlsRuntimeCustomTailAssetIdentity, HlsRuntimeCustomTailReason}, + terminal_commit::HlsTerminalCommitOutcome, + terminal_tail::{ + snapshot_terminal_media_asset, HlsLeasePlaybackMode, HlsTerminalTailCompatibility, HlsTerminalTailPlan, + }, + HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsManifestDeliveryMode, HlsMediaContainer, + HlsTerminalAssetIdentity, HlsTerminalBaseMediaState, HlsTerminalBaseProtection, + HlsTerminalBaseSegmentAvailability, HlsTerminalTailBuildInput, HlsTerminalTailGeneration, + }, + new_hls_access_lease_id, HlsAccessLease, HlsAccessLeaseActivation, HlsAccessLeaseDenialMode, + HlsAccessLeaseDenialOutcome, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseState, + HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsAccessLeaseTouch, HlsAvailabilityEvidenceGeneration, + HlsLeaseManifestPublicationOutcome, HlsLeaseManifestPublicationRejectReason, HlsPlaybackFamilyKey, + HlsRuntimePolicyRevocationOutcome, HlsTerminalMediaRequirementOrigin, HlsTerminalTailPreparationInput, +}; + +mod admission; +mod behavior; +mod lifecycle; +mod persistence; +mod query; +mod startup; +mod support; +mod terminal; + +use self::support::{lease, manifest_snapshot, publish_manifest_snapshot, timing}; diff --git a/backend/hls/src/lease/tests/admission.rs b/backend/hls/src/lease/tests/admission.rs new file mode 100644 index 000000000..190a9d19d --- /dev/null +++ b/backend/hls/src/lease/tests/admission.rs @@ -0,0 +1,44 @@ +use super::{ + lease, manifest_snapshot, HlsAccessLeaseId, HlsAccessLeaseStore, HlsLeaseManifestPublicationOutcome, + HlsLeaseManifestPublicationRejectReason, +}; +use crate::ProxySessionId; +use tuliprox_session::ConnectionKind; + +#[test] +fn session_snapshot_uses_best_priority_within_same_origin_policy_kind() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease( + lease(HlsAccessLeaseId("low-priority".to_string()), &proxy_session_id.0, 1_000) + .with_origin_acquire_policy(ConnectionKind::Normal, 30), + ); + store.prepare_access_lease( + lease(HlsAccessLeaseId("high-priority".to_string()), &proxy_session_id.0, 1_000) + .with_origin_acquire_policy(ConnectionKind::Normal, -5), + ); + + let snapshot = store.session_snapshot(&proxy_session_id, 2_000); + let policy = snapshot.effective_origin_policy.expect("usable lease policy"); + assert_eq!(policy.connection_kind, ConnectionKind::Normal); + assert_eq!(policy.priority, -5); +} + +#[test] +fn changed_admission_generation_rejects_manifest_publication() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + let guard = + store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("original admission guard"); + let lease = store.by_lease_id.get_mut(&lease_id).expect("stored lease"); + lease.admission_generation = lease.admission_generation.saturating_add(1); + + assert_eq!( + store.commit_manifest_publication(&lease_id, &proxy_session_id, guard, manifest_snapshot(10), 2_100,), + HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::AdmissionGenerationChanged + ) + ); +} diff --git a/backend/hls/src/lease/tests/behavior.rs b/backend/hls/src/lease/tests/behavior.rs new file mode 100644 index 000000000..f66f78368 --- /dev/null +++ b/backend/hls/src/lease/tests/behavior.rs @@ -0,0 +1,563 @@ +use super::{ + lease, manifest_snapshot, new_hls_access_lease_id, publish_manifest_snapshot, timing, HlsAccessLease, + HlsAccessLeaseActivation, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseState, + HlsAccessLeaseStore, HlsAccessLeaseTouch, HlsLeaseManifestPublicationOutcome, + HlsLeaseManifestPublicationRejectReason, HlsLeasePlaybackMode, HlsManifestCommitIdentity, + HlsPlaybackCompletionOutcome, HlsPlaybackFamilyKey, HlsRuntimeCustomTailReason, HlsTerminalTailCompatibility, +}; +use crate::ProxySessionId; +use std::sync::Arc; +use tuliprox_session::ConnectionKind; + +#[test] +fn known_bitrate_builder_normalizes_zero_and_survives_response_snapshot() { + let proxy_session_id = ProxySessionId("proxy-a".to_string()); + let zero_lease_id = HlsAccessLeaseId("lease-zero".to_string()); + let measured_lease_id = HlsAccessLeaseId("lease-measured".to_string()); + let mut store = HlsAccessLeaseStore::default(); + assert!(store.prepare_access_lease( + lease(zero_lease_id.clone(), &proxy_session_id.0, 1_000).with_known_bitrate_bps(Some(0)) + )); + assert!(store.prepare_access_lease( + lease(measured_lease_id.clone(), &proxy_session_id.0, 1_000).with_known_bitrate_bps(Some(2_500_000)), + )); + + assert_eq!( + store + .response_snapshot(&zero_lease_id, &proxy_session_id, 2_000) + .expect("zero bitrate lease") + .known_bitrate_bps, + None + ); + assert_eq!( + store + .response_snapshot(&measured_lease_id, &proxy_session_id, 2_000) + .expect("measured bitrate lease") + .known_bitrate_bps, + Some(2_500_000) + ); +} + +#[test] +fn standalone_tail_policy_is_limited_to_pending_unpublished_lease() { + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let mut candidate = lease(lease_id, "proxy-a", 1_000); + + assert!(candidate.permits_unpublished_standalone_tail(HlsRuntimeCustomTailReason::ChannelUnavailable)); + assert!(!candidate.permits_unpublished_standalone_tail(HlsRuntimeCustomTailReason::SessionOrLeaseExpired)); + + candidate.last_manifest_snapshot = Some(manifest_snapshot(1)); + candidate.startup_admission = super::super::HlsLeaseStartupAdmissionState::Admitted; + assert!(!candidate.permits_unpublished_standalone_tail(HlsRuntimeCustomTailReason::ChannelUnavailable)); + + let mut activated = lease(HlsAccessLeaseId("lease-b".to_string()), "proxy-a", 1_000); + activated.state = HlsAccessLeaseState::Activated; + assert!(!activated.permits_unpublished_standalone_tail(HlsRuntimeCustomTailReason::ChannelUnavailable)); + + let mut terminal = lease(HlsAccessLeaseId("lease-c".to_string()), "proxy-a", 1_000); + terminal.playback_mode = HlsLeasePlaybackMode::TerminalUnavailable { + decision_generation: 1, + reason: HlsTerminalTailCompatibility::MissingAsset, + }; + assert!(!terminal.permits_unpublished_standalone_tail(HlsRuntimeCustomTailReason::ChannelUnavailable)); +} + +#[test] +fn access_lease_id_is_short_and_opaque() { + let lease_id = new_hls_access_lease_id(); + + assert_eq!(lease_id.0.len(), 22); + assert!(!lease_id.0.contains("alice")); + assert!(!lease_id.0.contains("session")); +} + +#[test] +fn archive_playback_context_is_opt_in() { + let live = lease(HlsAccessLeaseId("live".to_string()), "proxy-live", 1_000); + assert_eq!(live.epg_reference_ts, None); + assert_eq!(live.archive_origin_url, None); + + let archive = lease(HlsAccessLeaseId("archive".to_string()), "proxy-archive", 1_000).with_archive_playback( + Some(1_784_898_000), + Some("http://provider/channel/timeshift_abs-1784898000.m3u8".to_string()), + ); + assert_eq!(archive.epg_reference_ts, Some(1_784_898_000)); + assert_eq!(archive.archive_origin_url.as_deref(), Some("http://provider/channel/timeshift_abs-1784898000.m3u8")); +} + +#[test] +fn access_lease_activates_and_slides_validity() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 10_000, timing(5_000, 30_000)).is_activated()); + assert_eq!(store.lease_state(&lease_id, 24_999), Some(HlsAccessLeaseState::Activated)); + assert!(store.touch_access_lease(&lease_id, 24_000, timing(5_000, 30_000))); + assert_eq!(store.lease_state(&lease_id, 53_999), Some(HlsAccessLeaseState::Activated)); +} + +#[test] +fn same_family_leases_remain_independently_valid() { + let mut store = HlsAccessLeaseStore::default(); + let old_lease_id = HlsAccessLeaseId("old".to_string()); + let new_lease_id = HlsAccessLeaseId("new".to_string()); + let proxy_a = ProxySessionId("proxy-a".to_string()); + let proxy_b = ProxySessionId("proxy-b".to_string()); + let family = HlsPlaybackFamilyKey::new("alice", "client-a"); + + store.prepare_access_lease(HlsAccessLease::pending( + old_lease_id.clone(), + family.clone(), + proxy_a.clone(), + "alice".to_string(), + "session-a".to_string(), + 1, + "12345".to_string(), + 12345, + 1_000, + 15_000, + )); + assert!(store.activate_access_lease(&old_lease_id, &proxy_a, 2_000, timing(5_000, 15_000)).is_activated()); + store.prepare_access_lease(HlsAccessLease::pending( + new_lease_id.clone(), + family, + proxy_b.clone(), + "alice".to_string(), + "session-b".to_string(), + 1, + "67890".to_string(), + 67890, + 3_000, + 15_000, + )); + + let activation = store.activate_access_lease(&new_lease_id, &proxy_b, 4_000, timing(5_000, 15_000)); + assert!(activation.is_activated()); + assert_eq!(store.lease_state(&old_lease_id, 4_000), Some(HlsAccessLeaseState::Activated)); + assert!(store.touch_access_lease(&old_lease_id, 5_000, timing(5_000, 15_000))); + assert_eq!(store.lease_state(&old_lease_id, 19_999), Some(HlsAccessLeaseState::Activated)); +} + +#[test] +fn manifest_touch_extends_activated_lease_active_window() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 15_000)).is_activated()); + + assert!(matches!( + store.touch_manifest_access_lease( + &lease_id, + &proxy_session_id, + 6_000, + Some(timing(10_000, 30_000)), + None, + 15_000, + ), + HlsAccessLeaseTouch::Touched { .. } + )); + let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); + assert_eq!(lease.state, HlsAccessLeaseState::Activated); + assert_eq!(lease.last_seen_at_ms, 6_000); + assert_eq!(lease.pending_deadline, None); + assert_eq!(lease.active_until_ms, Some(16_000)); + assert_eq!(lease.valid_until_ms, 36_000); +} + +#[test] +fn manifest_touch_can_shorten_pending_lease_to_follow_up_deadline() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(matches!( + store.touch_manifest_access_lease( + &lease_id, + &proxy_session_id, + 2_000, + None, + Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 }), + 300_000, + ), + HlsAccessLeaseTouch::Touched { .. } + )); + + let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); + assert_eq!(lease.state, HlsAccessLeaseState::Pending); + assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); + assert_eq!(lease.valid_until_ms, 12_000); + assert_eq!(store.lease_state(&lease_id, 11_999), Some(HlsAccessLeaseState::Pending)); + assert_eq!(store.lease_state(&lease_id, 12_000), Some(HlsAccessLeaseState::Expired)); +} + +#[test] +fn bootstrap_touch_cannot_extend_existing_follow_up_pending_deadline() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(matches!( + store.touch_manifest_access_lease( + &lease_id, + &proxy_session_id, + 2_000, + None, + Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 }), + 300_000, + ), + HlsAccessLeaseTouch::Touched { .. } + )); + assert!(matches!( + store.touch_manifest_access_lease( + &lease_id, + &proxy_session_id, + 3_000, + None, + Some(HlsAccessLeasePendingDeadline::Bootstrap { deadline_ms: 100_000 }), + 300_000, + ), + HlsAccessLeaseTouch::Touched { .. } + )); + + let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); + assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); + assert_eq!(lease.valid_until_ms, 12_000); +} + +#[test] +fn repeated_follow_up_touch_cannot_extend_existing_pending_deadline() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(matches!( + store.touch_manifest_access_lease( + &lease_id, + &proxy_session_id, + 2_000, + None, + Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 }), + 300_000, + ), + HlsAccessLeaseTouch::Touched { .. } + )); + assert!(matches!( + store.touch_manifest_access_lease( + &lease_id, + &proxy_session_id, + 3_000, + None, + Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 30_000 }), + 300_000, + ), + HlsAccessLeaseTouch::Touched { .. } + )); + + let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); + assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); + assert_eq!(lease.valid_until_ms, 12_000); +} + +#[test] +fn session_follow_up_shortens_pending_lease_once() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + let shortened = store.mark_pending_manifest_follow_up_for_session( + &proxy_session_id, + 2_000, + HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 }, + ); + assert_eq!(shortened.len(), 1); + let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); + assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); + assert_eq!(lease.valid_until_ms, 12_000); + + let unchanged = store.mark_pending_manifest_follow_up_for_session( + &proxy_session_id, + 3_000, + HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 30_000 }, + ); + assert!(unchanged.is_empty()); + let lease = store.by_lease_id.get(&lease_id).expect("lease should remain stored"); + assert_eq!(lease.pending_deadline, Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 12_000 })); + assert_eq!(lease.valid_until_ms, 12_000); +} + +#[test] +fn activated_lease_remains_valid_after_media_touch() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 15_000)).is_activated()); + + assert!(store.touch_access_lease(&lease_id, 3_000, timing(5_000, 15_000))); + assert_eq!(store.lease_state(&lease_id, 17_999), Some(HlsAccessLeaseState::Activated)); +} + +#[test] +fn session_snapshot_prefers_normal_origin_policy_over_soft() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease( + lease(HlsAccessLeaseId("soft".to_string()), &proxy_session_id.0, 1_000) + .with_origin_acquire_policy(ConnectionKind::Soft, -20), + ); + store.prepare_access_lease( + lease(HlsAccessLeaseId("normal".to_string()), &proxy_session_id.0, 1_000) + .with_origin_acquire_policy(ConnectionKind::Normal, 50), + ); + + let snapshot = store.session_snapshot(&proxy_session_id, 2_000); + let policy = snapshot.effective_origin_policy.expect("usable lease policy"); + assert_eq!(policy.connection_kind, ConnectionKind::Normal); + assert_eq!(policy.priority, 50); +} + +#[test] +fn origin_policy_update_reclassifies_existing_access_lease() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + store.prepare_access_lease( + lease(lease_id.clone(), &proxy_session_id.0, 1_000).with_origin_acquire_policy(ConnectionKind::Soft, 20), + ); + + let updated = + store.update_origin_acquire_policy(&lease_id, ConnectionKind::Normal, -5).expect("lease should update"); + assert_eq!(updated.origin_connection_kind, ConnectionKind::Normal); + assert_eq!(updated.origin_priority, -5); + + let snapshot = store.session_snapshot(&proxy_session_id, 2_000); + let policy = snapshot.effective_origin_policy.expect("updated policy"); + assert_eq!(policy.connection_kind, ConnectionKind::Normal); + assert_eq!(policy.priority, -5); +} + +#[test] +fn lease_manifest_generation_advances_for_snapshots_rendered_in_the_same_millisecond() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(2_000), 2_000) + .is_committed()); + let first = store.response_snapshot(&lease_id, &proxy_session_id, 2_000).expect("first manifest snapshot"); + assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(2_000), 2_000) + .is_committed()); + let second = store.response_snapshot(&lease_id, &proxy_session_id, 2_000).expect("second manifest snapshot"); + + assert_eq!(first.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation), Some(1)); + assert_eq!(second.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation), Some(2)); + assert_eq!( + first.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.delivered_at_ms), + second.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.delivered_at_ms) + ); +} + +#[test] +fn newer_source_publication_wins_before_delayed_older_request() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + let older_request = + store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("older request guard"); + let newer_request = + store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("newer request guard"); + + assert_eq!( + store.commit_manifest_publication(&lease_id, &proxy_session_id, newer_request, manifest_snapshot(20), 2_100,), + HlsLeaseManifestPublicationOutcome::Committed { snapshot_generation: 1 } + ); + assert_eq!( + store.commit_manifest_publication(&lease_id, &proxy_session_id, older_request, manifest_snapshot(10), 2_200,), + HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::SourceRegressive) + ); + let current = store + .response_snapshot(&lease_id, &proxy_session_id, 2_200) + .and_then(|lease| lease.last_manifest_snapshot) + .expect("newer snapshot remains current"); + assert_eq!(current.source_commit_identity, HlsManifestCommitIdentity::new(20)); + assert_eq!(current.snapshot_generation, 1); +} + +#[test] +fn replacement_lease_incarnation_rejects_older_manifest_publication() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + let guard = + store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("original incarnation guard"); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 3_000)); + + assert_eq!( + store.commit_manifest_publication(&lease_id, &proxy_session_id, guard, manifest_snapshot(10), 3_100,), + HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::LeaseIncarnationChanged) + ); +} + +#[test] +fn late_segment_completion_after_forward_seek_is_stale_and_does_not_advance_cursor() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + let identity = store + .response_snapshot(&lease_id, &proxy_session_id, 2_000) + .and_then(|lease| lease.media_identity()) + .expect("live identity"); + let stale_token = store + .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 40, 2_000) + .expect("first request token"); + let _current_token = store + .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 50, 2_100) + .expect("forward-seek request token"); + + assert_eq!( + store.record_segment_request_completed_if_identity_matches( + &lease_id, + &proxy_session_id, + identity, + stale_token, + 2_200, + ), + Some(HlsPlaybackCompletionOutcome::StaleRequest) + ); + let lease = store.response_snapshot(&lease_id, &proxy_session_id, 2_300).expect("live lease"); + assert_eq!(lease.playback_cursor.first_requested_proxy_seq, Some(50)); + assert_eq!(lease.playback_cursor.highest_contiguous_completed_proxy_seq, None); + assert_eq!(lease.playback_cursor.first_segment_completed_at_ms, None); +} + +#[test] +fn hls_availability_reevaluation_evidence_generation_is_per_session_and_noop_stable() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_a = ProxySessionId("availability-evidence-a".to_string()); + let proxy_b = ProxySessionId("availability-evidence-b".to_string()); + let primary_lease_id = HlsAccessLeaseId("availability-evidence-a".to_string()); + let secondary_lease_id = HlsAccessLeaseId("availability-evidence-b".to_string()); + let lease_a = lease(primary_lease_id.clone(), &proxy_a.0, 1_000); + assert!(store.prepare_access_lease(lease_a.clone())); + let added_a = store.availability_evidence_generation(&proxy_a); + assert!(added_a.as_u64() > 0); + + assert!(store.prepare_access_lease(lease_a)); + assert_eq!(store.availability_evidence_generation(&proxy_a), added_a); + assert!(matches!( + store.activate_access_lease(&primary_lease_id, &proxy_b, 2_000, timing(5_000, 30_000)), + HlsAccessLeaseActivation::SessionMismatch + )); + assert_eq!(store.availability_evidence_generation(&proxy_a), added_a); + + assert!(store.prepare_access_lease(lease(secondary_lease_id, &proxy_b.0, 1_000))); + assert_eq!(store.availability_evidence_generation(&proxy_a), added_a); + assert!(store.availability_evidence_generation(&proxy_b).as_u64() > added_a.as_u64()); + + assert!(store.activate_access_lease(&primary_lease_id, &proxy_a, 2_000, timing(5_000, 30_000)).is_activated()); + let activated_a = store.availability_evidence_generation(&proxy_a); + assert!(activated_a > added_a); + assert_eq!(store.availability_evidence_generation(&proxy_b).as_u64(), added_a.as_u64().saturating_add(1)); + + let identity = store + .response_snapshot(&primary_lease_id, &proxy_a, 2_000) + .and_then(|lease| lease.media_identity()) + .expect("activated live lease identity"); + let token = store + .record_segment_request_started_if_identity_matches(&primary_lease_id, &proxy_a, identity, 7, 2_100) + .expect("cursor request starts"); + let requested_a = store.availability_evidence_generation(&proxy_a); + assert!(requested_a > activated_a); + assert_eq!( + store + .record_segment_request_completed_if_identity_matches(&primary_lease_id, &proxy_a, identity, token, 2_200,), + Some(HlsPlaybackCompletionOutcome::Advanced) + ); + let completed_a = store.availability_evidence_generation(&proxy_a); + assert!(completed_a > requested_a); + assert_eq!( + store + .record_segment_request_completed_if_identity_matches(&primary_lease_id, &proxy_a, identity, token, 2_300,), + Some(HlsPlaybackCompletionOutcome::Duplicate) + ); + assert_eq!(store.availability_evidence_generation(&proxy_a), completed_a); + + assert!(store.remove_access_lease(&primary_lease_id).is_some()); + let removed_a = store.availability_evidence_generation(&proxy_a); + assert!(removed_a > completed_a); + assert_eq!(store.availability_evidence_generation(&proxy_b).as_u64(), added_a.as_u64().saturating_add(1)); + assert!(store.prepare_access_lease(lease(primary_lease_id, &proxy_a.0, 3_000))); + assert!(store.availability_evidence_generation(&proxy_a) > removed_a); +} + +#[test] +fn bulk_lease_removal_advances_and_preserves_session_evidence_generation() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("bulk-removal".to_string()); + let first = HlsAccessLeaseId("bulk-removal-a".to_string()); + let second = HlsAccessLeaseId("bulk-removal-b".to_string()); + assert!(store.prepare_access_lease(lease(first, &proxy_session_id.0, 1_000))); + assert!(store.prepare_access_lease(lease(second, &proxy_session_id.0, 1_001))); + let before_removal = store.availability_evidence_generation(&proxy_session_id); + + assert_eq!(store.remove_access_leases_for_session(&proxy_session_id).len(), 2); + let after_removal = store.availability_evidence_generation(&proxy_session_id); + assert!(after_removal > before_removal); + assert!(store.remove_access_leases_for_session(&proxy_session_id).is_empty()); + assert_eq!(store.availability_evidence_generation(&proxy_session_id), after_removal); +} + +#[test] +fn published_uri_rejects_a_new_attempt_before_the_first_media_request() -> std::io::Result<()> { + let session = ProxySessionId("proxy".into()); + let lease_id = HlsAccessLeaseId("revision-lease".into()); + let mut leases = HlsAccessLeaseStore::default(); + leases.prepare_access_lease(lease(lease_id.clone(), &session.0, 1000)); + let revisions = crate::SegmentRevisionStore::default(); + let published = revisions.create(session.clone(), 40, crate::SegmentRevisionKind::Raw)?; + let tail = revisions.create(session.clone(), 41, crate::SegmentRevisionKind::Raw)?; + for revision in [&published, &tail] { + revision.revision().prefix_available.store(8, std::sync::atomic::Ordering::Release); + } + let mut first = manifest_snapshot(10); + first.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + mode: shared::model::HlsStartupMode::Progressive, + retained_start_seq: 40, + revisions: [(40, published.clone()), (41, tail.clone())].into_iter().collect(), + })); + let guard = leases + .prepare_manifest_publication(&lease_id, &session, 2000) + .ok_or_else(|| std::io::Error::other("publication guard"))?; + assert!(leases.commit_manifest_publication(&lease_id, &session, guard, first, 2000).is_committed()); + published.revision().fail(); + let next = revisions.create(session.clone(), 40, crate::SegmentRevisionKind::Raw)?; + next.revision().prefix_available.store(8, std::sync::atomic::Ordering::Release); + let mut retry = manifest_snapshot(11); + retry.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + mode: shared::model::HlsStartupMode::Progressive, + retained_start_seq: 40, + revisions: [(40, next), (41, tail)].into_iter().collect(), + })); + let guard = leases + .prepare_manifest_publication(&lease_id, &session, 2100) + .ok_or_else(|| std::io::Error::other("publication guard"))?; + assert_eq!( + leases.commit_manifest_publication(&lease_id, &session, guard, retry, 2100), + HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::RevisionConflict) + ); + let bound = leases.published_segment_revision(&lease_id, 40).ok_or_else(|| std::io::Error::other("binding"))?; + assert_eq!(bound, published); + assert!(matches!(*bound.revision().subscribe().borrow(), crate::SegmentRevisionState::Failed)); + Ok(()) +} diff --git a/backend/hls/src/lease/tests/lifecycle.rs b/backend/hls/src/lease/tests/lifecycle.rs new file mode 100644 index 000000000..4a75bb356 --- /dev/null +++ b/backend/hls/src/lease/tests/lifecycle.rs @@ -0,0 +1,213 @@ +use super::{ + lease, manifest_snapshot, timing, HlsAccessLeaseActivation, HlsAccessLeaseDenialMode, HlsAccessLeaseDenialOutcome, + HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseState, HlsAccessLeaseStore, + HlsLeaseManifestPublicationOutcome, HlsLeaseManifestPublicationRejectReason, +}; +use crate::ProxySessionId; +use tuliprox_session::ConnectionKind; + +#[test] +fn access_lease_idles_at_exact_active_until_boundary() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 30_000)).is_activated()); + + let snapshot = store.lifecycle_snapshot(&lease_id, 7_000).expect("lease should exist"); + + assert_eq!(snapshot.state, HlsAccessLeaseState::Idle); + assert!(snapshot.idle_release.is_some()); + assert_eq!(store.active_access_lease_count_for_session(&proxy_session_id, 7_000), 0); +} + +#[test] +fn access_lease_expires_at_exact_valid_until_boundary() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 15_000)).is_activated()); + + let snapshot = store.lifecycle_snapshot(&lease_id, 17_000).expect("lease should exist"); + + assert_eq!(snapshot.state, HlsAccessLeaseState::Expired); + assert!(snapshot.idle_release.is_some()); + assert_eq!(store.lease_state(&lease_id, 17_000), Some(HlsAccessLeaseState::Expired)); +} + +#[test] +fn access_lease_expires_without_activity() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert_eq!( + store.activate_access_lease(&lease_id, &proxy_session_id, 17_000, timing(5_000, 15_000)), + HlsAccessLeaseActivation::Expired + ); +} + +#[test] +fn pending_lease_expires_at_pending_deadline_even_when_valid_window_is_longer() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + let mut lease = lease(lease_id.clone(), &proxy_session_id.0, 1_000); + lease.pending_deadline = Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms: 6_000 }); + lease.valid_until_ms = 31_000; + store.prepare_access_lease(lease); + + assert_eq!(store.lease_state(&lease_id, 5_999), Some(HlsAccessLeaseState::Pending)); + let snapshot = store.lifecycle_snapshot(&lease_id, 6_000).expect("lease should exist"); + assert_eq!(snapshot.state, HlsAccessLeaseState::Expired); + assert!(snapshot.idle_release.is_some(), "pending expiry must release counted user admission"); + assert_eq!(store.lease_state(&lease_id, 6_000), Some(HlsAccessLeaseState::Expired)); +} + +#[test] +fn activated_lease_becomes_idle_after_active_window_but_remains_reactivatable() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(5_000, 30_000)).is_activated()); + + let snapshot = store.session_snapshot(&proxy_session_id, 8_000); + assert_eq!(snapshot.active_count, 0); + assert_eq!(snapshot.idle_releases.len(), 1); + assert_eq!(snapshot.idle_releases[0].lease_id, lease_id); + assert_eq!(store.lease_state(&lease_id, 8_000), Some(HlsAccessLeaseState::Idle)); + assert!(store.has_usable_access_lease_for_session(&proxy_session_id, 8_000)); + assert!(store.access_lease(&lease_id, &proxy_session_id, 8_000).is_some()); + + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 8_000, timing(5_000, 30_000)).is_activated()); + assert_eq!(store.session_snapshot(&proxy_session_id, 8_000).active_count, 1); +} + +#[test] +fn activated_lease_validity_expiry_reports_idle_release() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(30_000, 5_000)).is_activated()); + + let snapshot = store.session_snapshot(&proxy_session_id, 8_000); + assert_eq!(snapshot.active_count, 0); + assert_eq!(snapshot.idle_releases.len(), 1); + assert_eq!(snapshot.idle_releases[0].lease_id, lease_id); + assert_eq!(store.lease_state(&lease_id, 8_000), Some(HlsAccessLeaseState::Expired)); +} + +#[test] +fn usable_access_lease_query_accepts_pending_idle_and_active_activated_only() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + let pending_id = HlsAccessLeaseId("pending".to_string()); + let idle_id = HlsAccessLeaseId("idle".to_string()); + let activated_id = HlsAccessLeaseId("activated".to_string()); + let denied_id = HlsAccessLeaseId("denied".to_string()); + let expired_id = HlsAccessLeaseId("expired".to_string()); + + store.prepare_access_lease(lease(pending_id.clone(), &proxy_session_id.0, 1_000)); + store.prepare_access_lease(lease(idle_id.clone(), &proxy_session_id.0, 1_000)); + store.prepare_access_lease(lease(activated_id.clone(), &proxy_session_id.0, 1_000)); + store.prepare_access_lease(lease(denied_id.clone(), &proxy_session_id.0, 1_000)); + store.prepare_access_lease(lease(expired_id.clone(), &proxy_session_id.0, 1_000)); + assert!(store.activate_access_lease(&idle_id, &proxy_session_id, 2_000, timing(1_000, 15_000)).is_activated()); + assert!(store.activate_access_lease(&activated_id, &proxy_session_id, 2_000, timing(5_000, 15_000)).is_activated()); + assert!(matches!( + store.deny_access_lease(&denied_id, HlsAccessLeaseDenialMode::ImmediateEnd), + HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } + )); + + assert!(store.has_usable_access_lease_for_session(&proxy_session_id, 2_000)); + let snapshot = store.session_snapshot(&proxy_session_id, 3_000); + assert_eq!(snapshot.active_count, 1); + assert_eq!(snapshot.idle_releases.len(), 1); + assert_eq!(snapshot.idle_releases[0].lease_id, idle_id); + assert_eq!(store.lease_state(&idle_id, 3_000), Some(HlsAccessLeaseState::Idle)); + assert_eq!(store.active_access_lease_count_for_session(&proxy_session_id, 3_000), 1); + assert!(store.has_usable_access_lease_for_session(&proxy_session_id, 3_000)); + + assert!(matches!( + store.deny_access_lease(&pending_id, HlsAccessLeaseDenialMode::ImmediateEnd), + HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } + )); + assert!(matches!( + store.deny_access_lease(&activated_id, HlsAccessLeaseDenialMode::ImmediateEnd), + HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } + )); + assert!(matches!( + store.deny_access_lease(&expired_id, HlsAccessLeaseDenialMode::ImmediateEnd), + HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } + )); + assert!(store.has_usable_access_lease_for_session(&proxy_session_id, 3_000)); + assert!(matches!( + store.deny_access_lease(&idle_id, HlsAccessLeaseDenialMode::ImmediateEnd), + HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } + )); + assert!(!store.has_usable_access_lease_for_session(&proxy_session_id, 2_000)); + assert!(!store.has_usable_access_lease_for_session(&proxy_session_id, 17_000)); + assert_eq!(store.lease_state(&expired_id, 17_000), Some(HlsAccessLeaseState::Expired)); +} + +#[test] +fn session_snapshot_ignores_expired_and_denied_origin_policies() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + let denied_id = HlsAccessLeaseId("denied".to_string()); + store.prepare_access_lease( + lease(denied_id.clone(), &proxy_session_id.0, 1_000).with_origin_acquire_policy(ConnectionKind::Normal, -100), + ); + store.prepare_access_lease( + lease(HlsAccessLeaseId("expired".to_string()), &proxy_session_id.0, 1_000) + .with_origin_acquire_policy(ConnectionKind::Normal, -50), + ); + store.prepare_access_lease( + lease(HlsAccessLeaseId("active-soft".to_string()), &proxy_session_id.0, 10_000) + .with_origin_acquire_policy(ConnectionKind::Soft, 10), + ); + assert!(matches!( + store.deny_access_lease(&denied_id, HlsAccessLeaseDenialMode::ImmediateEnd), + HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } + )); + + let snapshot = store.session_snapshot(&proxy_session_id, 17_000); + let policy = snapshot.effective_origin_policy.expect("usable lease policy"); + assert_eq!(policy.connection_kind, ConnectionKind::Soft); + assert_eq!(policy.priority, 10); +} + +#[test] +fn lease_expiry_between_manifest_publication_prepare_and_commit_rejects_snapshot() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + let guard = + store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("live publication guard"); + + assert_eq!( + store.commit_manifest_publication(&lease_id, &proxy_session_id, guard, manifest_snapshot(10), 16_000,), + HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::LeaseExpired) + ); + let lease = store.response_snapshot(&lease_id, &proxy_session_id, 16_000).expect("expired lease retained"); + assert_eq!(lease.state, HlsAccessLeaseState::Expired); + assert!(lease.last_manifest_snapshot.is_none()); +} + +#[test] +fn expired_lease_lookup_rejects_stale_entry_without_removing_before_lifecycle() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + + assert!(store.access_lease(&lease_id, &proxy_session_id, 17_000).is_none()); + assert_eq!(store.lease_state(&lease_id, 17_000), Some(HlsAccessLeaseState::Expired)); +} diff --git a/backend/hls/src/lease/tests/persistence.rs b/backend/hls/src/lease/tests/persistence.rs new file mode 100644 index 000000000..5ed6a68c9 --- /dev/null +++ b/backend/hls/src/lease/tests/persistence.rs @@ -0,0 +1,54 @@ +use super::{ + lease, manifest_snapshot, HlsAccessLeaseId, HlsAccessLeaseStore, HlsLeaseManifestPublicationOutcome, + HlsLeaseManifestPublicationRejectReason, HlsManifestCommitIdentity, HlsManifestDeliveryMode, +}; +use crate::ProxySessionId; + +#[test] +fn commit_generation_orders_cross_mode_sources_rendered_in_the_same_millisecond() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + let older_request = + store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("older request guard"); + let newer_request = + store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("newer request guard"); + let mut newer = manifest_snapshot(20); + newer.delivery_mode = HlsManifestDeliveryMode::TransientPassthrough; + newer.source_commit_identity = HlsManifestCommitIdentity::committed(2, 20); + newer.finalized_transient_manifest_generation = Some(super::super::super::TransientManifestGeneration::for_test(2)); + let mut older = manifest_snapshot(20); + older.delivery_mode = HlsManifestDeliveryMode::NormalCacheTimeline; + older.source_commit_identity = HlsManifestCommitIdentity::committed(1, 20); + older.finalized_transient_manifest_generation = None; + + assert!(store + .commit_manifest_publication(&lease_id, &proxy_session_id, newer_request, newer, 2_100) + .is_committed()); + assert_eq!( + store.commit_manifest_publication(&lease_id, &proxy_session_id, older_request, older, 2_200), + HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::SourceRegressive) + ); + + let lease_id = HlsAccessLeaseId("lease-b".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + let older_request = + store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("older request guard"); + let newer_request = + store.prepare_manifest_publication(&lease_id, &proxy_session_id, 2_000).expect("newer request guard"); + let mut newer = manifest_snapshot(20); + newer.source_commit_identity = HlsManifestCommitIdentity::committed(4, 20); + let mut older = manifest_snapshot(20); + older.delivery_mode = HlsManifestDeliveryMode::TransientPassthrough; + older.source_commit_identity = HlsManifestCommitIdentity::committed(3, 20); + older.finalized_transient_manifest_generation = Some(super::super::super::TransientManifestGeneration::for_test(1)); + + assert!(store + .commit_manifest_publication(&lease_id, &proxy_session_id, newer_request, newer, 2_100) + .is_committed()); + assert_eq!( + store.commit_manifest_publication(&lease_id, &proxy_session_id, older_request, older, 2_200), + HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::SourceRegressive) + ); +} diff --git a/backend/hls/src/lease/tests/query.rs b/backend/hls/src/lease/tests/query.rs new file mode 100644 index 000000000..18a02885c --- /dev/null +++ b/backend/hls/src/lease/tests/query.rs @@ -0,0 +1,14 @@ +use super::{lease, HlsAccessLeaseId, HlsAccessLeaseStore, HlsAvailabilityEvidenceGeneration}; +use crate::ProxySessionId; + +#[test] +fn hls_availability_reevaluation_evidence_generation_fails_closed_on_overflow() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("availability-evidence-overflow".to_string()); + let lease_id = HlsAccessLeaseId("availability-evidence-overflow".to_string()); + store.last_availability_evidence_generation = HlsAvailabilityEvidenceGeneration::for_test(u64::MAX); + + assert!(!store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000))); + assert!(store.response_snapshot(&lease_id, &proxy_session_id, 1_000).is_none()); + assert_eq!(store.availability_evidence_generation(&proxy_session_id).as_u64(), 0); +} diff --git a/backend/hls/src/lease/tests/startup.rs b/backend/hls/src/lease/tests/startup.rs new file mode 100644 index 000000000..3812bd2f1 --- /dev/null +++ b/backend/hls/src/lease/tests/startup.rs @@ -0,0 +1,83 @@ +use super::{lease, manifest_snapshot, publish_manifest_snapshot, HlsAccessLeaseId, HlsAccessLeaseStore}; +use crate::ProxySessionId; +use std::sync::Arc; + +#[test] +fn retired_revision_bindings_cannot_be_rebound_after_the_hard_limit() -> std::io::Result<()> { + let store = crate::SegmentRevisionStore::default(); + let session = ProxySessionId("limit".into()); + let mut lease = lease(HlsAccessLeaseId("bounded".into()), &session.0, 1000); + for seq in 0..1030 { + let revision = store.create(session.clone(), seq, crate::SegmentRevisionKind::Raw)?; + revision.revision().prefix_available.store(1, std::sync::atomic::Ordering::Release); + let mut snapshot = manifest_snapshot(seq); + snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + mode: shared::model::HlsStartupMode::Progressive, + retained_start_seq: 0, + revisions: [(seq, revision)].into_iter().collect(), + })); + assert!(super::super::commit_lease_revision_bindings(&mut lease, &snapshot)); + } + let bindings = lease.revision_bindings.as_ref().ok_or_else(|| std::io::Error::other("bindings"))?; + assert_eq!(bindings.revisions.len(), 1024); + assert_eq!(bindings.retired_through, Some(5)); + let new_attempt = store.create(session, 0, crate::SegmentRevisionKind::Raw)?; + new_attempt.revision().prefix_available.store(1, std::sync::atomic::Ordering::Release); + let mut old_uri = manifest_snapshot(1031); + old_uri.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + mode: shared::model::HlsStartupMode::Progressive, + retained_start_seq: 0, + revisions: [(0, new_attempt)].into_iter().collect(), + })); + assert!(!super::super::commit_lease_revision_bindings(&mut lease, &old_uri)); + Ok(()) +} + +#[test] +fn repeated_lease_retirement_releases_publication_revision_guards() -> std::io::Result<()> { + let session = ProxySessionId("lease-churn".into()); + let revisions = crate::SegmentRevisionStore::default(); + let mut leases = HlsAccessLeaseStore::default(); + for generation in 0..1000 { + let lease_id = HlsAccessLeaseId(format!("lease-{generation}")); + assert!(leases.prepare_access_lease(lease(lease_id.clone(), &session.0, 1000))); + let mut owners = std::collections::BTreeMap::new(); + for seq in [40, 41] { + let owner = revisions.create(session.clone(), seq, crate::SegmentRevisionKind::Raw)?; + owner.revision().prefix_available.store(8, std::sync::atomic::Ordering::Release); + owners.insert(seq, owner); + } + let mut snapshot = manifest_snapshot(generation + 1); + snapshot.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + mode: shared::model::HlsStartupMode::Progressive, + retained_start_seq: 40, + revisions: owners, + })); + let guard = leases + .prepare_manifest_publication(&lease_id, &session, 2000) + .ok_or_else(|| std::io::Error::other("publication guard"))?; + assert!(leases.commit_manifest_publication(&lease_id, &session, guard, snapshot, 2000).is_committed()); + assert!(revisions.retire_unpinned()?.is_empty()); + assert!(leases.remove_access_lease(&lease_id).is_some()); + assert_eq!(revisions.retire_unpinned()?.len(), 2); + } + assert!(leases.by_lease_id.is_empty()); + assert!(revisions.retire_unpinned()?.is_empty()); + Ok(()) +} + +#[test] +fn startup_claim_only_accepts_the_published_head_once() { + let session = ProxySessionId("proxy".into()); + let lease_id = HlsAccessLeaseId("claim-lease".into()); + let mut leases = HlsAccessLeaseStore::default(); + leases.prepare_access_lease(lease(lease_id.clone(), &session.0, 1000)); + assert!(publish_manifest_snapshot(&mut leases, &lease_id, &session, manifest_snapshot(10), 2000).is_committed()); + assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2100)); + if let Some(lease) = leases.by_lease_id.get_mut(&lease_id) { + lease.playback_cursor.first_requested_proxy_seq = Some(41); + } + assert!(leases.try_claim_progressive_startup(&lease_id, 40, 2101)); + assert!(!leases.try_claim_progressive_startup(&lease_id, 40, 2102)); + assert!(!leases.try_claim_progressive_startup(&lease_id, 41, 2102)); +} diff --git a/backend/hls/src/lease/tests/support.rs b/backend/hls/src/lease/tests/support.rs new file mode 100644 index 000000000..ccc6b6a1d --- /dev/null +++ b/backend/hls/src/lease/tests/support.rs @@ -0,0 +1,81 @@ +use super::{ + HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsLeaseManifestPublicationOutcome, + HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsManifestCommitIdentity, HlsManifestDeliveryMode, + HlsMediaContainer, HlsPlaybackFamilyKey, +}; +use crate::ProxySessionId; +use std::sync::Arc; + +pub(in crate::lease::tests) fn lease( + lease_id: HlsAccessLeaseId, + proxy_session_id: &str, + now_ms: u64, +) -> HlsAccessLease { + HlsAccessLease::pending( + lease_id, + HlsPlaybackFamilyKey::new("alice", "client-a"), + ProxySessionId(proxy_session_id.to_string()), + "alice".to_string(), + "session-a".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + 15_000, + ) +} + +pub(in crate::lease::tests) const fn timing(active_window_ms: u64, valid_window_ms: u64) -> HlsAccessLeaseTiming { + HlsAccessLeaseTiming { active_window_ms, valid_window_ms } +} + +pub(in crate::lease::tests) fn manifest_snapshot(source_rendered_at_ms: u64) -> HlsLeaseManifestSnapshot { + HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(source_rendered_at_ms), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 0, + delivered_at_ms: 2_000, + first_proxy_seq: 40, + last_proxy_seq: 41, + visible_segments: Arc::from([ + HlsLeaseManifestSegment { + proxy_seq: 40, + duration_ms: 6_000, + uri: "/live/40.ts".to_string().into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }, + HlsLeaseManifestSegment { + proxy_seq: 41, + duration_ms: 6_000, + uri: "/live/41.ts".to_string().into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }, + ]), + discontinuity_sequence: 3, + target_duration_ms: 12_000, + playlist_duration_ms: 12_000, + last_visible_media_end_ms: 12_000, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + } +} + +pub(in crate::lease::tests) fn publish_manifest_snapshot( + store: &mut HlsAccessLeaseStore, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + snapshot: HlsLeaseManifestSnapshot, + now_ms: u64, +) -> HlsLeaseManifestPublicationOutcome { + let guard = + store.prepare_manifest_publication(lease_id, proxy_session_id, now_ms).expect("live manifest publication"); + store.commit_manifest_publication(lease_id, proxy_session_id, guard, snapshot, now_ms) +} diff --git a/backend/hls/src/lease/tests/terminal.rs b/backend/hls/src/lease/tests/terminal.rs new file mode 100644 index 000000000..996a43fca --- /dev/null +++ b/backend/hls/src/lease/tests/terminal.rs @@ -0,0 +1,706 @@ +use super::{ + build_terminal_tail_plan, lease, manifest_snapshot, publish_manifest_snapshot, snapshot_terminal_media_asset, + timing, HlsAccessLeaseDenialMode, HlsAccessLeaseDenialOutcome, HlsAccessLeaseId, HlsAccessLeaseState, + HlsAccessLeaseStore, HlsFiniteTailTrigger, HlsLeaseCutoverTiming, HlsLeaseManifestPublicationOutcome, + HlsLeaseManifestPublicationRejectReason, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsLeasePlaybackMode, + HlsLeaseReserveAvailabilityBasis, HlsLeaseReserveSnapshot, HlsManifestAcceptanceGeneration, + HlsPlaybackCompletionOutcome, HlsRuntimeCustomTailAssetIdentity, HlsRuntimeCustomTailReason, + HlsRuntimePolicyRevocationOutcome, HlsTerminalAssetIdentity, HlsTerminalBaseMediaState, HlsTerminalBaseProtection, + HlsTerminalBaseSegmentAvailability, HlsTerminalCommitOutcome, HlsTerminalCommitWindow, + HlsTerminalMediaPreparationState, HlsTerminalMediaRequirementOrigin, HlsTerminalTailBuildInput, + HlsTerminalTailCompatibility, HlsTerminalTailGeneration, HlsTerminalTailPlan, HlsTerminalTailPreparationInput, + HlsTransitionMarginMs, +}; +use crate::ProxySessionId; +use std::sync::Arc; +use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer; + +pub(in crate::lease::tests) const TERMINAL_ASSET_BYTES: &[u8] = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); + +pub(in crate::lease::tests) fn cutover_reserve() -> HlsLeaseReserveSnapshot { + HlsLeaseReserveSnapshot { + availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, + guaranteed_media_horizon_ms: 12_000, + conservative_playback_position_ms: 12_000, + guaranteed_reserve_ms: 0, + initial_hidden_ready_duration_ms: 0, + transition_margin: HlsTransitionMarginMs::from_millis(12_000), + key_readiness_valid_until_ms: None, + recovery_required: true, + cutover_required: true, + } +} + +pub(in crate::lease::tests) fn cutover_timing() -> HlsLeaseCutoverTiming { + HlsLeaseCutoverTiming::from_reserve(2_000, 0, HlsTransitionMarginMs::from_millis(12_000), None) +} + +pub(in crate::lease::tests) fn manifest_snapshot_for_route( + source_rendered_at_ms: u64, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, +) -> HlsLeaseManifestSnapshot { + let mut snapshot = manifest_snapshot(source_rendered_at_ms); + for segment in Arc::make_mut(&mut snapshot.visible_segments) { + segment.uri = format!("/hls/shared/live/{}/{}/{}.ts", proxy_session_id.0, lease_id.0, segment.proxy_seq).into(); + } + snapshot +} + +#[test] +fn repair_prewarm_guard_rejects_newer_publication_and_terminal_lease() { + let proxy_session_id = ProxySessionId("proxy-a".to_string()); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let mut store = HlsAccessLeaseStore::default(); + assert!(store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000))); + let first_generation = + publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000) + .snapshot_generation() + .expect("first publication"); + assert!(store.repair_prewarm_is_current(&lease_id, &proxy_session_id, 1_000, first_generation,)); + + let second_generation = + publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(2), 2_100) + .snapshot_generation() + .expect("second publication"); + assert!(!store.repair_prewarm_is_current(&lease_id, &proxy_session_id, 1_000, first_generation,)); + assert!(store.repair_prewarm_is_current(&lease_id, &proxy_session_id, 1_000, second_generation,)); + + store.by_lease_id.get_mut(&lease_id).expect("lease").playback_mode = HlsLeasePlaybackMode::Ended; + assert!(!store.repair_prewarm_is_current(&lease_id, &proxy_session_id, 1_000, second_generation,)); +} + +pub(in crate::lease::tests) fn terminal_plan( + generation: u64, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, +) -> Arc { + terminal_plan_at(generation, proxy_session_id, lease_id, 3_000) +} + +pub(in crate::lease::tests) fn terminal_plan_at( + generation: u64, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + created_at_ms: u64, +) -> Arc { + let mut base_manifest = manifest_snapshot_for_route(1, proxy_session_id, lease_id); + base_manifest.snapshot_generation = 1; + let transport_stream = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); + let asset = snapshot_terminal_media_asset(&transport_stream).expect("terminal asset"); + let expected_asset = + HlsRuntimeCustomTailAssetIdentity::channel_unavailable(HlsTerminalAssetIdentity::from_asset(&asset)); + let availability = Arc::from( + base_manifest + .visible_proxy_seqs() + .map(|proxy_seq| HlsTerminalBaseSegmentAvailability { + proxy_seq, + media_state: HlsTerminalBaseMediaState::Ready, + required_map_ready: true, + required_key_ready: true, + protection: HlsTerminalBaseProtection::Protectable, + }) + .collect::>(), + ); + let base_track_signature = Some(asset.track_signature().clone()); + let anchored_bundle = HlsTerminalTailBuildInput::anchored_bundle_for_test(&asset, base_manifest.target_duration_ms); + let base_timing = Some(HlsTerminalTailBuildInput::base_timing_for_test(&asset, &base_manifest)); + let base_splice_evidence = Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); + let terminal_splice_evidence = base_splice_evidence.clone(); + Arc::new( + build_terminal_tail_plan(HlsTerminalTailBuildInput { + generation: HlsTerminalTailGeneration(generation), + created_at_ms, + base_manifest, + base_availability: availability, + base_track_signature, + base_splice_evidence, + terminal_splice_evidence, + base_timing, + base_key_bindings: Arc::from([]), + expected_asset, + asset, + anchored_bundle, + }) + .expect("compatible terminal plan"), + ) +} + +pub(in crate::lease::tests) fn prepare_terminal( + store: &mut HlsAccessLeaseStore, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + origin_progress_generation: u64, + last_media_progress_at_ms: Option, +) -> super::super::HlsTerminalTailPreparation { + let lease = store.response_snapshot(lease_id, proxy_session_id, 2_000).expect("test lease snapshot"); + let snapshot_generation = + lease.last_manifest_snapshot.as_ref().expect("test manifest snapshot").snapshot_generation; + store + .prepare_terminal_tail( + lease_id, + proxy_session_id, + &HlsTerminalTailPreparationInput { + trigger: HlsFiniteTailTrigger::AvailabilityReserve, + expected_manifest_snapshot_generation: snapshot_generation, + expected_cursor_generation: lease.playback_cursor.cursor_generation, + origin_progress_generation, + media_readiness_generation: 0, + origin_epoch: 0, + last_media_progress_at_ms, + expected_acceptance_generation: HlsManifestAcceptanceGeneration(origin_progress_generation), + terminal_media_requirement_origin: HlsTerminalMediaRequirementOrigin::AcceptanceEpisode { + generation: HlsManifestAcceptanceGeneration(origin_progress_generation), + }, + cutover_timing: cutover_timing(), + commit_window: HlsTerminalCommitWindow::CutoverDue, + required_terminal_media_key: None, + terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, + reserve: cutover_reserve(), + }, + ) + .expect("terminal preparation") +} + +pub(in crate::lease::tests) fn prepare_runtime_policy_terminal( + store: &HlsAccessLeaseStore, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + reason: HlsRuntimeCustomTailReason, +) -> super::super::HlsTerminalTailPreparation { + let lease = store.by_lease_id.get(lease_id).expect("runtime policy lease"); + let manifest = lease.last_manifest_snapshot.as_ref().expect("published runtime policy manifest"); + store + .prepare_terminal_tail( + lease_id, + proxy_session_id, + &HlsTerminalTailPreparationInput { + trigger: HlsFiniteTailTrigger::RuntimePolicy(reason), + expected_manifest_snapshot_generation: manifest.snapshot_generation, + expected_cursor_generation: lease.playback_cursor.cursor_generation, + origin_progress_generation: 4, + media_readiness_generation: 5, + origin_epoch: 6, + last_media_progress_at_ms: Some(2_000), + expected_acceptance_generation: HlsManifestAcceptanceGeneration(4), + terminal_media_requirement_origin: HlsTerminalMediaRequirementOrigin::CutoverSnapshot, + cutover_timing: cutover_timing(), + commit_window: HlsTerminalCommitWindow::CutoverDue, + required_terminal_media_key: None, + terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, + reserve: cutover_reserve(), + }, + ) + .expect("runtime policy terminal preparation") +} + +pub(in crate::lease::tests) fn commit_prepared_terminal( + store: &mut HlsAccessLeaseStore, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + preparation: &super::super::HlsTerminalTailPreparation, +) -> HlsTerminalCommitOutcome { + store.commit_terminal_tail_if_generation_matches( + lease_id, + proxy_session_id, + preparation, + 3_000, + terminal_plan(preparation.decision_generation, proxy_session_id, lease_id), + ) +} + +#[test] +fn policy_revocation_token_is_reason_and_generation_bound() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("policy-cas".to_string()); + let proxy_session_id = ProxySessionId("proxy-policy-cas".to_string()); + assert!(store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000))); + assert!(publish_manifest_snapshot( + &mut store, + &lease_id, + &proxy_session_id, + manifest_snapshot_for_route(1, &proxy_session_id, &lease_id), + 2_000, + ) + .is_committed()); + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(10_000, 15_000)).is_activated()); + let identity = store + .response_snapshot(&lease_id, &proxy_session_id, 2_000) + .and_then(|lease| lease.media_identity()) + .expect("live identity"); + assert!(store + .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 40, 2_100,) + .is_some()); + + let started = store.begin_runtime_policy_revocation( + &lease_id, + &proxy_session_id, + HlsRuntimeCustomTailReason::UserConnectionsExhausted, + 2_200, + ); + let HlsRuntimePolicyRevocationOutcome::Started { token } = started else { + panic!("expected a new policy revocation, got {started:?}"); + }; + assert_eq!( + store.begin_runtime_policy_revocation( + &lease_id, + &proxy_session_id, + HlsRuntimeCustomTailReason::UserConnectionsExhausted, + 2_201, + ), + HlsRuntimePolicyRevocationOutcome::AlreadyPending { token: token.clone() } + ); + assert_eq!( + store.begin_runtime_policy_revocation( + &lease_id, + &proxy_session_id, + HlsRuntimeCustomTailReason::UserAccountExpired, + 2_202, + ), + HlsRuntimePolicyRevocationOutcome::NoLongerEligible + ); + let preparation = prepare_runtime_policy_terminal( + &store, + &lease_id, + &proxy_session_id, + HlsRuntimeCustomTailReason::UserConnectionsExhausted, + ); + assert_eq!(preparation.runtime_policy_revocation, Some(token)); + + store.by_lease_id.get_mut(&lease_id).expect("policy lease").admission_generation = + preparation.expected_admission_generation.saturating_add(1); + assert_eq!( + store.commit_terminal_unavailable_if_generation_matches( + &lease_id, + &proxy_session_id, + &preparation, + 2_300, + HlsTerminalTailCompatibility::MissingAsset, + ), + HlsTerminalCommitOutcome::SupersededGeneration + ); + let lease = store.response_snapshot(&lease_id, &proxy_session_id, 2_300).expect("revoking lease"); + assert_eq!(lease.state, HlsAccessLeaseState::PolicyRevoking); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); +} + +#[test] +fn unpublished_runtime_policy_denial_retains_first_reason_without_live_access() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("cold-policy-denial".to_string()); + let proxy_session_id = ProxySessionId("proxy-cold-policy-denial".to_string()); + assert!(store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000))); + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(10_000, 15_000)).is_activated()); + assert_eq!( + store.begin_runtime_policy_revocation( + &lease_id, + &proxy_session_id, + HlsRuntimeCustomTailReason::UserConnectionsExhausted, + 2_100, + ), + HlsRuntimePolicyRevocationOutcome::NoPublishedManifest + ); + assert_eq!( + store.deny_access_lease( + &lease_id, + HlsAccessLeaseDenialMode::ImmediateRuntimePolicyEnd { + reason: HlsRuntimeCustomTailReason::UserConnectionsExhausted, + }, + ), + HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } + ); + let denied = store.response_snapshot(&lease_id, &proxy_session_id, 2_100).expect("denied lease"); + assert_eq!(denied.state, HlsAccessLeaseState::Denied); + assert_eq!(denied.playback_mode, HlsLeasePlaybackMode::Ended); + assert_eq!(denied.runtime_policy_denial_reason(), Some(HlsRuntimeCustomTailReason::UserConnectionsExhausted)); + + assert_eq!( + store.deny_access_lease( + &lease_id, + HlsAccessLeaseDenialMode::ImmediateRuntimePolicyEnd { + reason: HlsRuntimeCustomTailReason::UserAccountExpired, + }, + ), + HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } + ); + let repeated = store.response_snapshot(&lease_id, &proxy_session_id, 2_100).expect("retained denied lease"); + assert_eq!(repeated.runtime_policy_denial_reason(), Some(HlsRuntimeCustomTailReason::UserConnectionsExhausted)); +} + +#[test] +fn user_revocation_tail_does_not_authorize_unfetched_live_suffix() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("policy-prefix".to_string()); + let proxy_session_id = ProxySessionId("proxy-policy-prefix".to_string()); + let mut snapshot = manifest_snapshot_for_route(1, &proxy_session_id, &lease_id); + let mut segments = snapshot.visible_segments.to_vec(); + for proxy_seq in [42_u64, 43] { + segments.push(HlsLeaseManifestSegment { + proxy_seq, + duration_ms: 6_000, + uri: format!("/hls/shared/live/{}/{}/{}.ts", proxy_session_id.0, lease_id.0, proxy_seq).into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }); + } + snapshot.visible_segments = Arc::from(segments); + snapshot.last_proxy_seq = 43; + snapshot.playlist_duration_ms = 24_000; + snapshot.last_visible_media_end_ms = 24_000; + assert!(store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000))); + assert!(publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, snapshot, 2_000).is_committed()); + assert!(store.activate_access_lease(&lease_id, &proxy_session_id, 2_000, timing(10_000, 15_000)).is_activated()); + let identity = store + .response_snapshot(&lease_id, &proxy_session_id, 2_000) + .and_then(|lease| lease.media_identity()) + .expect("live identity"); + let completed = store + .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 40, 2_100) + .expect("completed request token"); + assert_eq!( + store.record_segment_request_completed_if_identity_matches( + &lease_id, + &proxy_session_id, + identity, + completed, + 2_150, + ), + Some(HlsPlaybackCompletionOutcome::Advanced) + ); + assert!(store + .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 41, 2_175,) + .is_some()); + assert!(matches!( + store.begin_runtime_policy_revocation( + &lease_id, + &proxy_session_id, + HlsRuntimeCustomTailReason::UserConnectionsExhausted, + 2_200, + ), + HlsRuntimePolicyRevocationOutcome::Started { .. } + )); + assert!(store + .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 42, 2_201,) + .is_none()); + + let preparation = prepare_runtime_policy_terminal( + &store, + &lease_id, + &proxy_session_id, + HlsRuntimeCustomTailReason::UserConnectionsExhausted, + ); + assert_eq!(preparation.manifest_snapshot.first_proxy_seq, 40); + assert_eq!(preparation.manifest_snapshot.last_proxy_seq, 41); + assert_eq!(preparation.manifest_snapshot.visible_proxy_seqs().collect::>(), vec![40, 41]); +} + +#[test] +fn active_playback_snapshot_excludes_pending_idle_and_terminal_leases() { + let mut store = HlsAccessLeaseStore::default(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + let active_id = HlsAccessLeaseId("active".to_string()); + let pending_id = HlsAccessLeaseId("pending".to_string()); + let terminal_id = HlsAccessLeaseId("terminal".to_string()); + for lease_id in [&active_id, &pending_id, &terminal_id] { + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!(publish_manifest_snapshot(&mut store, lease_id, &proxy_session_id, manifest_snapshot(1), 2_000) + .is_committed()); + } + assert!(store.activate_access_lease(&active_id, &proxy_session_id, 2_000, timing(5_000, 30_000)).is_activated()); + assert!(store.activate_access_lease(&terminal_id, &proxy_session_id, 2_000, timing(5_000, 30_000)).is_activated()); + let preparation = prepare_terminal(&mut store, &terminal_id, &proxy_session_id, 1, Some(1_000)); + assert_eq!( + commit_prepared_terminal(&mut store, &terminal_id, &proxy_session_id, &preparation), + HlsTerminalCommitOutcome::Committed + ); + + let snapshots = store.active_live_playback_snapshots_for_session(&proxy_session_id, 2_100); + + assert_eq!(snapshots.len(), 1); + assert_eq!(snapshots[0].lease_id, active_id); +} + +#[test] +fn terminal_preparation_is_read_only() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!( + publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(7), 2_000).is_committed() + ); + let before = store.response_snapshot(&lease_id, &proxy_session_id, 2_000).expect("live lease"); + let manifest_generation = before.last_manifest_snapshot.as_ref().expect("manifest snapshot").snapshot_generation; + + let preparation = store + .prepare_terminal_tail( + &lease_id, + &proxy_session_id, + &HlsTerminalTailPreparationInput { + trigger: HlsFiniteTailTrigger::AvailabilityReserve, + expected_manifest_snapshot_generation: manifest_generation, + expected_cursor_generation: before.playback_cursor.cursor_generation, + origin_progress_generation: 4, + media_readiness_generation: 9, + origin_epoch: 0, + last_media_progress_at_ms: Some(1_900), + expected_acceptance_generation: HlsManifestAcceptanceGeneration(4), + terminal_media_requirement_origin: HlsTerminalMediaRequirementOrigin::AcceptanceEpisode { + generation: HlsManifestAcceptanceGeneration(4), + }, + cutover_timing: cutover_timing(), + commit_window: HlsTerminalCommitWindow::CutoverDue, + required_terminal_media_key: None, + terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, + reserve: cutover_reserve(), + }, + ) + .expect("terminal preparation"); + let after = store.response_snapshot(&lease_id, &proxy_session_id, 2_000).expect("live lease"); + + assert_eq!(before, after); + assert_eq!(preparation.expected_admission_generation, before.admission_generation); + assert_eq!(preparation.media_readiness_generation, 9); + assert_eq!(preparation.manifest_snapshot_generation, manifest_generation); + assert_eq!(preparation.cutover_timing, cutover_timing()); + assert_eq!(preparation.terminal_media_preparation, HlsTerminalMediaPreparationState::Failed { key: None }); +} + +#[test] +fn hls_terminal_commit_stale_snapshot_cannot_replace_live_generation() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!( + publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).is_committed() + ); + + let stale = prepare_terminal(&mut store, &lease_id, &proxy_session_id, 4, Some(2_000)); + assert!( + publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).is_committed() + ); + + assert_eq!( + store.commit_terminal_tail_if_generation_matches( + &lease_id, + &proxy_session_id, + &stale, + 3_000, + terminal_plan(stale.decision_generation, &proxy_session_id, &lease_id), + ), + HlsTerminalCommitOutcome::SupersededGeneration + ); + let lease = store.response_snapshot(&lease_id, &proxy_session_id, 3_000).expect("lease remains stored"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + assert_eq!(lease.last_manifest_snapshot.as_ref().map(|snapshot| snapshot.snapshot_generation), Some(2)); +} + +#[test] +fn hls_terminal_commit_plan_bound_to_another_route_is_superseded() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!( + publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).is_committed() + ); + let preparation = prepare_terminal(&mut store, &lease_id, &proxy_session_id, 4, Some(2_000)); + let other_lease_id = HlsAccessLeaseId("other-lease".to_string()); + + assert_eq!( + store.commit_terminal_tail_if_generation_matches( + &lease_id, + &proxy_session_id, + &preparation, + 3_000, + terminal_plan(preparation.decision_generation, &proxy_session_id, &other_lease_id), + ), + HlsTerminalCommitOutcome::SupersededGeneration + ); + assert_eq!( + store.response_snapshot(&lease_id, &proxy_session_id, 3_000).expect("live lease").playback_mode, + HlsLeasePlaybackMode::Live + ); +} + +#[test] +fn hls_terminal_commit_lease_expiry_wins_the_race() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!( + publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).is_committed() + ); + let preparation = prepare_terminal(&mut store, &lease_id, &proxy_session_id, 4, Some(2_000)); + + assert_eq!( + store.commit_terminal_tail_if_generation_matches( + &lease_id, + &proxy_session_id, + &preparation, + 16_000, + terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), + ), + HlsTerminalCommitOutcome::LeaseNoLongerEligible + ); + let lease = store.response_snapshot(&lease_id, &proxy_session_id, 16_000).expect("expired lease remains stored"); + assert_eq!(lease.state, HlsAccessLeaseState::Expired); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Ended); + assert_eq!(lease.admission_generation, preparation.expected_admission_generation); +} + +#[test] +fn hls_terminal_commit_exact_replay_is_idempotent_and_sticky() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!( + publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).is_committed() + ); + let preparation = prepare_terminal(&mut store, &lease_id, &proxy_session_id, 4, Some(2_000)); + let plan = terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id); + let concurrent_plan = terminal_plan_at(preparation.decision_generation, &proxy_session_id, &lease_id, 3_100); + + assert_eq!( + store.commit_terminal_tail_if_generation_matches( + &lease_id, + &proxy_session_id, + &preparation, + 3_000, + Arc::clone(&plan), + ), + HlsTerminalCommitOutcome::Committed + ); + assert_eq!( + store.commit_terminal_tail_if_generation_matches( + &lease_id, + &proxy_session_id, + &preparation, + 3_100, + concurrent_plan, + ), + HlsTerminalCommitOutcome::AlreadyCommitted + ); + assert!(store.prepare_manifest_publication(&lease_id, &proxy_session_id, 3_100).is_none()); +} + +#[test] +fn hls_terminal_commit_unavailable_replay_is_idempotent_for_the_same_decision_generation() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!( + publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).is_committed() + ); + let preparation = prepare_terminal(&mut store, &lease_id, &proxy_session_id, 4, Some(2_000)); + + assert_eq!( + store.commit_terminal_unavailable_if_generation_matches( + &lease_id, + &proxy_session_id, + &preparation, + 3_000, + HlsTerminalTailCompatibility::MissingAsset, + ), + HlsTerminalCommitOutcome::Committed + ); + assert_eq!( + store.commit_terminal_unavailable_if_generation_matches( + &lease_id, + &proxy_session_id, + &preparation, + 3_100, + HlsTerminalTailCompatibility::InvalidAsset, + ), + HlsTerminalCommitOutcome::AlreadyCommitted + ); +} + +#[test] +fn hls_terminal_commit_terminal_lease_rejects_recovery_while_other_lease_remains_live() { + let mut store = HlsAccessLeaseStore::default(); + let terminal_lease_id = HlsAccessLeaseId("terminal".to_string()); + let live_lease_id = HlsAccessLeaseId("live".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(terminal_lease_id.clone(), &proxy_session_id.0, 1_000)); + store.prepare_access_lease(lease(live_lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!(publish_manifest_snapshot(&mut store, &terminal_lease_id, &proxy_session_id, manifest_snapshot(1), 2_000,) + .is_committed()); + assert!(publish_manifest_snapshot(&mut store, &live_lease_id, &proxy_session_id, manifest_snapshot(1), 2_000,) + .is_committed()); + let stale_terminal_publication = store + .prepare_manifest_publication(&terminal_lease_id, &proxy_session_id, 2_000) + .expect("pre-terminal publication guard"); + let preparation = prepare_terminal(&mut store, &terminal_lease_id, &proxy_session_id, 7, Some(2_000)); + assert_eq!( + commit_prepared_terminal(&mut store, &terminal_lease_id, &proxy_session_id, &preparation), + HlsTerminalCommitOutcome::Committed + ); + + assert_eq!( + store.commit_manifest_publication( + &terminal_lease_id, + &proxy_session_id, + stale_terminal_publication, + manifest_snapshot(2), + 3_000, + ), + HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::AdmissionGenerationChanged + ) + ); + assert!(store.prepare_manifest_publication(&terminal_lease_id, &proxy_session_id, 3_000).is_none()); + assert!(publish_manifest_snapshot(&mut store, &live_lease_id, &proxy_session_id, manifest_snapshot(2), 3_000,) + .is_committed()); + assert!(matches!( + store.response_snapshot(&terminal_lease_id, &proxy_session_id, 3_000).expect("terminal lease").playback_mode, + HlsLeasePlaybackMode::TerminalTail(_) + )); + assert_eq!( + store.response_snapshot(&live_lease_id, &proxy_session_id, 3_000).expect("live lease").playback_mode, + HlsLeasePlaybackMode::Live + ); +} + +#[test] +fn late_segment_completion_after_terminal_transition_does_not_advance_cursor() { + let mut store = HlsAccessLeaseStore::default(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let proxy_session_id = ProxySessionId("proxy".to_string()); + store.prepare_access_lease(lease(lease_id.clone(), &proxy_session_id.0, 1_000)); + assert!( + publish_manifest_snapshot(&mut store, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).is_committed() + ); + let identity = store + .response_snapshot(&lease_id, &proxy_session_id, 2_100) + .and_then(|lease| lease.media_identity()) + .expect("live identity"); + let token = store + .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 40, 2_100) + .expect("live request token"); + let preparation = prepare_terminal(&mut store, &lease_id, &proxy_session_id, 8, Some(2_000)); + assert_eq!( + commit_prepared_terminal(&mut store, &lease_id, &proxy_session_id, &preparation), + HlsTerminalCommitOutcome::Committed + ); + + assert_eq!( + store.record_segment_request_completed_if_identity_matches( + &lease_id, + &proxy_session_id, + identity, + token, + 2_200, + ), + None + ); + let lease = store.response_snapshot(&lease_id, &proxy_session_id, 2_300).expect("terminal lease"); + assert_eq!(lease.playback_cursor.highest_contiguous_completed_proxy_seq, None); + assert_eq!(lease.playback_cursor.first_segment_completed_at_ms, None); +} diff --git a/backend/hls/src/lease/timing.rs b/backend/hls/src/lease/timing.rs new file mode 100644 index 000000000..217552dc5 --- /dev/null +++ b/backend/hls/src/lease/timing.rs @@ -0,0 +1,140 @@ +use super::{ + HlsAccessLease, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseState, HlsAccessLeaseStore, + HlsAvailabilityEvidenceAdvanceError, HlsLeasePlaybackCursor, HlsLeasePlaybackMode, HlsLeaseStartupAdmissionState, + HlsPlaybackFamilyKey, HlsPublishedTransientResourceIds, ProxySessionId, +}; +use std::collections::BTreeSet; +use tuliprox_session::ConnectionKind; + +impl HlsAccessLeasePendingDeadline { + pub const fn deadline_ms(self) -> u64 { + match self { + Self::Bootstrap { deadline_ms } | Self::FollowUp { deadline_ms } => deadline_ms, + } + } +} + +impl HlsAccessLease { + #[allow(clippy::too_many_arguments)] + pub fn pending( + lease_id: HlsAccessLeaseId, + family_key: HlsPlaybackFamilyKey, + proxy_session_id: ProxySessionId, + username: String, + user_session_token: String, + input_id: u16, + stream_ref: String, + virtual_id: u32, + now_ms: u64, + valid_window_ms: u64, + ) -> Self { + Self { + lease_id, + family_key, + proxy_session_id, + username, + user_session_token, + input_id, + stream_ref, + virtual_id, + known_bitrate_bps: None, + origin_connection_kind: ConnectionKind::Normal, + origin_priority: 0, + state: HlsAccessLeaseState::Pending, + issued_at_ms: now_ms, + last_seen_at_ms: now_ms, + active_until_ms: None, + pending_deadline: Some(HlsAccessLeasePendingDeadline::Bootstrap { + deadline_ms: now_ms.saturating_add(valid_window_ms), + }), + valid_until_ms: now_ms.saturating_add(valid_window_ms), + epg_reference_ts: None, + archive_origin_url: None, + playback_mode: HlsLeasePlaybackMode::Live, + startup_admission: HlsLeaseStartupAdmissionState::Pending, + playback_cursor: HlsLeasePlaybackCursor::default(), + revision_bindings: None, + progressive_startup_claimed: false, + last_manifest_snapshot: None, + published_transient_resource_ids: HlsPublishedTransientResourceIds::default(), + published_finalized_manifest_generations: BTreeSet::new(), + manifest_snapshot_generation: 0, + admission_generation: 0, + runtime_policy_revocation: None, + runtime_policy_denial_reason: None, + pending_terminal_protection_release: None, + } + } + + pub fn age_ms(&self, now_ms: u64) -> u64 { now_ms.saturating_sub(self.issued_at_ms) } + + pub fn pending_deadline_ms(&self) -> Option { + self.pending_deadline.map(HlsAccessLeasePendingDeadline::deadline_ms) + } + + pub(super) fn validity_due_at_ms(&self) -> u64 { + if self.state == HlsAccessLeaseState::Pending { + self.pending_deadline_ms().unwrap_or(self.valid_until_ms) + } else { + self.valid_until_ms + } + } + + pub(super) fn apply_pending_deadline(&mut self, deadline: HlsAccessLeasePendingDeadline) -> bool { + let previous = self.pending_deadline; + let deadline = self.pending_deadline.map_or(deadline, |current| current.tightened_with(deadline)); + self.pending_deadline = Some(deadline); + self.valid_until_ms = deadline.deadline_ms(); + previous != self.pending_deadline + } + + fn refresh_validity(&mut self, now_ms: u64) -> bool { + if self.state != HlsAccessLeaseState::Expired && self.validity_due_at_ms() <= now_ms { + self.state = HlsAccessLeaseState::Expired; + if self.playback_mode == HlsLeasePlaybackMode::Live { + self.end_playback(); + } + return true; + } + false + } +} + +impl HlsAccessLeaseStore { + pub(super) fn refresh_access_lease_validity( + &mut self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + ) -> Result { + let Some(lease) = self.by_lease_id.get(lease_id) else { + return Ok(false); + }; + if lease.state == HlsAccessLeaseState::Expired || lease.validity_due_at_ms() > now_ms { + return Ok(false); + } + let proxy_session_id = lease.proxy_session_id.clone(); + self.availability_evidence_can_advance()?; + let changed = self.by_lease_id.get_mut(lease_id).is_some_and(|lease| lease.refresh_validity(now_ms)); + if changed { + self.advance_availability_evidence(&proxy_session_id)?; + } + Ok(changed) + } + + pub(super) fn refresh_access_lease_validities_for_session( + &mut self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Result<(), HlsAvailabilityEvidenceAdvanceError> { + let lease_ids = self + .by_lease_id + .values() + .filter(|lease| lease.proxy_session_id == *proxy_session_id) + .map(|lease| lease.lease_id.clone()) + .collect::>(); + for lease_id in lease_ids { + self.refresh_access_lease_validity(&lease_id, now_ms)?; + } + Ok(()) + } +} diff --git a/backend/hls/src/lib.rs b/backend/hls/src/lib.rs index cca745e7f..fe45135b1 100644 --- a/backend/hls/src/lib.rs +++ b/backend/hls/src/lib.rs @@ -68,6 +68,8 @@ mod paths; mod post_refresh_availability; mod prefetch; mod prepared_terminal_bundle; +mod progressive_budget; +mod progressive_startup; mod qos; mod recovery_timing; mod refresh; @@ -78,10 +80,13 @@ mod response; mod runtime_custom_tail; mod segment_fetcher; mod segment_repair; +mod segment_revision; mod segment_watchdog; mod session; mod session_store; mod startup_observability; +mod startup_policy; +mod sync_ext; mod terminal_commit; mod terminal_pending; mod terminal_tail; @@ -324,7 +329,14 @@ pub use self::{ HlsTerminalTailCompatibility, HLS_TERMINAL_TAIL_SEGMENT_COUNT, }, }; +pub use cache::{HlsRevisionDiskReservation, HlsRevisionFilePin}; pub use hls_ctx::{HlsCtx, WeakHlsCtx}; +pub use progressive_budget::ProgressiveBudgetManager; +pub use segment_revision::{ + SegmentRevision, SegmentRevisionGuard, SegmentRevisionId, SegmentRevisionKey, SegmentRevisionKind, + SegmentRevisionState, SegmentRevisionStore, +}; +pub use startup_policy::{HlsManifestRevisions, HlsSessionStartup}; pub use tuliprox_mpegts::ts_inspector::{ evaluate_mpeg_ts_splice_boundary, hls_aes128_cbc_iv, inspect_mpeg_ts_async, inspect_mpeg_ts_media_evidence_async, HlsTrackEvidenceResolution, HlsTsMediaEvidence, HlsTsProbeBudget, HlsTsProbeProtection, HlsTsProtectionReason, diff --git a/backend/hls/src/manager.rs b/backend/hls/src/manager.rs index 798f04094..b57067902 100644 --- a/backend/hls/src/manager.rs +++ b/backend/hls/src/manager.rs @@ -60,13 +60,11 @@ use super::{ TransientResourceStore, }; use arc_swap::ArcSwap; -use log::{debug, error, info}; -use shared::utils::sanitize_sensitive_info; -use std::{collections::HashMap, io, path::PathBuf, sync::Arc}; -use tokio::sync::{RwLock, Semaphore}; -use tokio_util::sync::CancellationToken; -use tuliprox_core::model::{AppConfig, HlsCacheConfig, HlsManifestRecoveryBurstConfig, StripConfig}; -use tuliprox_session::{ActiveProviderManager, ActiveUserManager}; +use std::{ + collections::HashMap, + sync::{atomic::AtomicBool, Arc}, +}; +use tokio::sync::RwLock; /// Root runtime object for the future HLS cache proxy. pub struct HlsProxyManager { @@ -90,37 +88,9 @@ pub struct HlsProxyManager { availability_reevaluations: Arc, terminal_commit_clock: Arc, startup_observability: Arc, -} - -#[derive(Debug, Clone)] -struct HlsProxyRuntimeConfig { - enabled: bool, - segment_fetch_policy: SegmentFetchPolicy, - cache_duration_seconds: u64, - strip: StripConfig, - origin_manifest_timeout_ms: u64, - initial_manifest_wait_timeout_secs: u64, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig, - transient_resource_ttl_ms: u64, - gc_policy: GarbageCollectionPolicy, - rewrite_secret_fingerprint: String, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct HlsAccountOverlapCooldownKey { - input_name: Arc, - account_name: Arc, -} - -#[derive(Debug, Clone, Copy)] -struct HlsAccountOverlapCooldown { - until_ms: u64, -} - -#[derive(Debug, Clone, Copy)] -enum HlsAccountOverlapCooldownReason { - ReclaimedByOriginalOwner, - SpeculativePromoted, + revision_store: Arc, + revision_reconciliation_pending: AtomicBool, + progressive_budget: Arc, } pub struct HlsTerminalTailPreparationRequest<'a> { @@ -137,12 +107,6 @@ pub struct HlsTerminalTailPreparationRequest<'a> { pub last_media_progress_at_ms: Option, } -#[derive(Clone, Copy)] -enum HlsTerminalPreparationPurpose { - Cutover, - UnavailableAfterOwnerFailure, -} - /// Generation-bound terminal publication requested against one immutable preparation. pub struct HlsTerminalCommitRequest<'a> { pub session: &'a HlsSessionHandle, @@ -161,18 +125,6 @@ pub enum HlsTerminalCommitPayload { UnavailableAfterOwnerFailure(HlsTerminalTailCompatibility), } -impl HlsTerminalCommitPayload { - fn into_parts(self) -> (HlsTerminalLeaseDecision, Option) { - match self { - Self::Tail { plan, media_guard } => (HlsTerminalLeaseDecision::Tail(plan), Some(media_guard)), - Self::Unavailable(reason) => (HlsTerminalLeaseDecision::Unavailable(reason), None), - Self::UnavailableAfterOwnerFailure(reason) => { - (HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(reason), None) - } - } - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[must_use] pub enum HlsMediaActivityCommitOutcome { @@ -181,20 +133,8 @@ pub enum HlsMediaActivityCommitOutcome { DeferredLockContention, } -#[derive(Debug, Clone, Copy)] -enum HlsMediaActivityCommitKind { - Access, - Delivered, - LiveSegmentCompletion(HlsPlaybackRequestToken), -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum HlsMediaActivityCommitAttempt { - LockBusy, - Completed { outcome: HlsMediaActivityCommitOutcome, evidence_changed: bool }, -} - const HLS_STATE_CAS_LOCK_RETRIES: usize = 8; + const HLS_MEDIA_ACTIVITY_FALLBACK_LOCK_RETRIES: usize = 8; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -205,5295 +145,27 @@ pub enum HlsCriticalHandoffStateAccess { const HLS_SESSION_IDLE_PROTECTION_RETRY_MS: u64 = 1_000; -fn hls_session_idle_protection_retry_at(due_at_ms: u64, now_ms: u64) -> u64 { - due_at_ms.max(now_ms.saturating_add(HLS_SESSION_IDLE_PROTECTION_RETRY_MS)) -} - -fn hls_key_readiness_evidence_is_current(valid_until_ms: Option, now_ms: u64) -> bool { - valid_until_ms.is_none_or(|valid_until_ms| valid_until_ms >= now_ms) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum HlsRecoveryExecutionState { - Idle, - InFlight { estimated_completion_at: HlsEstimatedRecoveryCompletionAtMs }, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct HlsAcceptanceRecoverySnapshot { - expected_generation: HlsManifestAcceptanceGeneration, - status: HlsManifestAcceptanceEpisodeStatus, - recovery: HlsRecoveryExecutionState, - required_terminal_media_key: Option, - terminal_media_preparation: HlsTerminalMediaPreparationState, -} - -fn hls_estimated_recovery_completion_at( - recovery: HlsRecoveryExecutionState, -) -> Option { - match recovery { - HlsRecoveryExecutionState::Idle => None, - HlsRecoveryExecutionState::InFlight { estimated_completion_at } => Some(estimated_completion_at), - } -} - -fn hls_acceptance_recovery_snapshot(session: &super::HlsSession, now_ms: u64) -> HlsAcceptanceRecoverySnapshot { - let expected_generation = session.origin_control.acceptance_generation; - let status = manifest_acceptance_episode_status( - session.origin_control.acceptance_episode.as_ref(), - expected_generation, - now_ms, - ); - let matching_episode = - session.origin_control.acceptance_episode.as_ref().filter(|episode| episode.generation == expected_generation); - let recovery = match (status, matching_episode) { - (HlsManifestAcceptanceEpisodeStatus::InFlight { .. }, Some(episode)) => episode - .estimated_recovery_completion_at(expected_generation, now_ms) - .map_or(HlsRecoveryExecutionState::Idle, |estimated_completion_at| HlsRecoveryExecutionState::InFlight { - estimated_completion_at, - }), - ( - HlsManifestAcceptanceEpisodeStatus::Missing - | HlsManifestAcceptanceEpisodeStatus::Expired { .. } - | HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { .. } - | HlsManifestAcceptanceEpisodeStatus::Committed { .. } - | HlsManifestAcceptanceEpisodeStatus::Superseded { .. }, - _, - ) - | (HlsManifestAcceptanceEpisodeStatus::InFlight { .. }, None) => HlsRecoveryExecutionState::Idle, - }; - let (required_terminal_media_key, terminal_media_preparation) = - matching_episode.map_or((None, HlsTerminalMediaPreparationState::Failed { key: None }), |episode| { - let timing = episode.timing(); - (timing.required_terminal_media_key, timing.terminal_media_preparation) - }); - HlsAcceptanceRecoverySnapshot { - expected_generation, - status, - recovery, - required_terminal_media_key, - terminal_media_preparation, - } -} - -fn terminal_media_requirement_origin( - recovery_snapshot: &HlsAcceptanceRecoverySnapshot, -) -> HlsTerminalMediaRequirementOrigin { - match recovery_snapshot.status { - HlsManifestAcceptanceEpisodeStatus::Missing => HlsTerminalMediaRequirementOrigin::CutoverSnapshot, - HlsManifestAcceptanceEpisodeStatus::Committed { .. } - | HlsManifestAcceptanceEpisodeStatus::InFlight { .. } - | HlsManifestAcceptanceEpisodeStatus::Expired { .. } - | HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { .. } - | HlsManifestAcceptanceEpisodeStatus::Superseded { .. } => { - HlsTerminalMediaRequirementOrigin::AcceptanceEpisode { generation: recovery_snapshot.expected_generation } - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum HlsTerminalCommitAuthorization { - Authorized { protection_capacity_exceeded: bool }, - Rejected(HlsTerminalCommitOutcome), -} - -fn terminal_media_requirement_is_bound_to_preparation(preparation: &HlsTerminalTailPreparation) -> bool { - match preparation.terminal_media_requirement_source { - HlsTerminalMediaRequirementSource::AcceptanceEpisode { generation } => { - generation == preparation.expected_acceptance_generation - } - HlsTerminalMediaRequirementSource::CutoverSnapshotPending { decision_generation } - | HlsTerminalMediaRequirementSource::CutoverSnapshot { decision_generation, .. } => { - decision_generation == preparation.decision_generation - } - } -} - -fn evaluate_terminal_commit_authorization( - session: &super::HlsSession, - lease_id: &HlsAccessLeaseId, - preparation: &HlsTerminalTailPreparation, - decision: &HlsTerminalLeaseDecision, - now_ms: u64, -) -> HlsTerminalCommitAuthorization { - let recovery_snapshot = hls_acceptance_recovery_snapshot(session, now_ms); - // Acceptance lifecycle generations describe recovery evidence, not media - // continuity. A later episode must not invalidate a cutover snapshot while - // the media progress, readiness, epoch, and lease generations guarded below - // remain unchanged. - if !terminal_media_requirement_is_bound_to_preparation(preparation) { - return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::SupersededGeneration); - } - let protection_capacity_exceeded = matches!(decision, HlsTerminalLeaseDecision::Tail(_)) - && !session.can_install_terminal_tail_protection(lease_id); - let (terminal, terminal_preparation) = match (decision, protection_capacity_exceeded) { - (HlsTerminalLeaseDecision::Tail(plan), false) => { - let prepared_key = plan.media_preparation_key(); - if preparation.required_terminal_media_key != Some(prepared_key) - || !preparation - .terminal_media_requirement_source - .authorizes_tail(preparation.decision_generation, prepared_key) - { - return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleIncompatible); - } - ( - HlsTerminalCutoverCapability::TailCompatible { prepared_key }, - HlsTerminalMediaPreparationState::Ready { key: prepared_key }, - ) - } - (HlsTerminalLeaseDecision::Tail(_), true) => ( - HlsTerminalCutoverCapability::TailUnavailable(HlsTerminalTailCompatibility::ProtectionCapacityExceeded), - preparation.terminal_media_preparation, - ), - ( - HlsTerminalLeaseDecision::Unavailable(reason) - | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(reason), - _, - ) => (HlsTerminalCutoverCapability::TailUnavailable(*reason), preparation.terminal_media_preparation), - }; - if matches!(decision, HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_)) { - if !session.origin_control.path_condition.is_degraded() { - return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::CutoverNoLongerRequired); - } - // This fallback publishes no media bytes. Submission arbitration and - // the terminal precondition retain the preparation's original - // exclusive deadline, so this authorization is reachable only while - // the autonomous owner still has time to commit safely. - return HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: false }; - } - if preparation.trigger.is_runtime_policy() { - return HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded }; - } - let cutover = evaluate_terminal_cutover(&HlsTerminalCutoverInput { - reserve: preparation.reserve, - commit_window: preparation.commit_window, - acceptance: recovery_snapshot.status, - required_terminal_media_key: preparation.required_terminal_media_key, - terminal_preparation, - terminal, - }); - match (decision, cutover) { - (HlsTerminalLeaseDecision::Tail(_), HlsTerminalCutoverDecision::CommitTerminalTail) - if !protection_capacity_exceeded => - { - HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: false } - } - ( - HlsTerminalLeaseDecision::Tail(_), - HlsTerminalCutoverDecision::CommitTerminalUnavailable( - HlsTerminalTailCompatibility::ProtectionCapacityExceeded, - ), - ) if protection_capacity_exceeded => { - HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: true } - } - ( - HlsTerminalLeaseDecision::Unavailable(expected) - | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(expected), - HlsTerminalCutoverDecision::CommitTerminalUnavailable(actual), - ) if *expected == actual => HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: false }, - (_, HlsTerminalCutoverDecision::NotRequired) => { - HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::CutoverNoLongerRequired) - } - (_, HlsTerminalCutoverDecision::RetrySupersededSnapshot) => { - HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::SupersededGeneration) - } - (_, HlsTerminalCutoverDecision::EvaluateTerminalCapability { .. }) => { - HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleNotReady) - } - ( - _, - HlsTerminalCutoverDecision::CommitTerminalTail | HlsTerminalCutoverDecision::CommitTerminalUnavailable(_), - ) => HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleIncompatible), - } -} - -impl HlsAccountOverlapCooldownReason { - fn as_log_reason(self) -> &'static str { - match self { - Self::ReclaimedByOriginalOwner => "reclaimed-by-original-owner", - Self::SpeculativePromoted => "speculative-promoted", - } - } -} - -fn hls_pending_manifest_follow_up_window_ms(target_duration: Option) -> u64 { - let target_duration_secs = u64::from(target_duration.unwrap_or(15)).max(1); - target_duration_secs.saturating_mul(2_000).max(10_000) -} - -fn hls_pending_manifest_follow_up_deadline(now_ms: u64, target_duration: Option) -> HlsAccessLeasePendingDeadline { - HlsAccessLeasePendingDeadline::FollowUp { - deadline_ms: now_ms.saturating_add(hls_pending_manifest_follow_up_window_ms(target_duration)), - } -} - -impl HlsProxyRuntimeConfig { - fn from_config(config: &HlsCacheConfig, rewrite_secret: &[u8]) -> Self { - Self::from_config_with_enabled(config, rewrite_secret, true) - } - - fn from_config_with_enabled(config: &HlsCacheConfig, rewrite_secret: &[u8], enabled: bool) -> Self { - Self { - enabled, - segment_fetch_policy: SegmentFetchPolicy::from_config(config), - cache_duration_seconds: config.cache_duration.get(), - strip: config.strip.clone(), - origin_manifest_timeout_ms: config.origin_manifest_timeout_ms.get(), - initial_manifest_wait_timeout_secs: config.initial_manifest_wait_timeout_secs.get(), - manifest_recovery_burst: config.manifest_recovery_burst.clone(), - transient_resource_ttl_ms: config.cache_duration.as_millis().get(), - gc_policy: GarbageCollectionPolicy::from_config(config), - rewrite_secret_fingerprint: build_rewrite_secret_fingerprint(rewrite_secret), - } - } -} - -#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] -struct HlsProxySessionCleanupStats { - access_leases: usize, - repair_windows: usize, - repair_generations: usize, - repair_candidates: usize, - repair_object_metadata: usize, - repair_watchdog_metadata: usize, - repair_watchdog_locks: usize, - qos_access_leases: usize, -} - -impl HlsProxySessionCleanupStats { - fn did_cleanup(self) -> bool { - self.access_leases > 0 - || self.repair_windows > 0 - || self.repair_generations > 0 - || self.repair_candidates > 0 - || self.repair_object_metadata > 0 - || self.repair_watchdog_metadata > 0 - || self.repair_watchdog_locks > 0 - || self.qos_access_leases > 0 - } -} - -impl HlsProxyManager { - pub fn new() -> Self { - let default_dto = shared::model::HlsCacheConfigDto::default(); - let default_config = HlsCacheConfig::from(&default_dto); - Self::with_hls_cache_config(&default_config) - } - - pub fn from_hls_cache_config(config: Option<&HlsCacheConfig>) -> Self { - Self::from_hls_cache_config_and_secret(config, &[]) - } - - pub fn from_hls_cache_config_and_secret(config: Option<&HlsCacheConfig>, rewrite_secret: &[u8]) -> Self { - let default_config; - let (config, enabled) = if let Some(config) = config { - (config, true) - } else { - default_config = HlsCacheConfig::from(&shared::model::HlsCacheConfigDto::default()); - (&default_config, false) - }; - Self::with_hls_cache_config_and_secret_enabled(config, rewrite_secret, enabled) - } - - pub fn with_cache_settings(cache_path: impl Into, cache_duration_seconds: u64) -> Self { - let default_dto = shared::model::HlsCacheConfigDto { - cache_duration: shared::model::Secs::new(cache_duration_seconds), - cache_path: Some(cache_path.into().to_string_lossy().to_string()), - ..Default::default() - }; - let default_config = HlsCacheConfig::from(&default_dto); - let segment_fetch_policy = SegmentFetchPolicy::from_config(&default_config); - let global_fetch_semaphore = Arc::new(Semaphore::new(segment_fetch_policy.max_global_segment_fetches)); - let sessions = Arc::new(HlsSessionStore::new()); - let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(PathBuf::from(&default_config.cache_path))); - segment_cache - .update_cache_limits(default_config.cache_bytes.get(), default_config.cache_bytes_per_session.get()); - let segment_repair = Arc::new(HlsSegmentRepairManager::new(default_config.segment_repair.clone())); - let metrics = Arc::new(HlsCacheMetrics::default()); - let qos = Arc::new(HlsQosRegistry::default()); - let access_leases = Arc::new(RwLock::new(HlsAccessLeaseStore::default())); - let lifecycle = Arc::new(HlsLifecycleManager::new()); - let account_overlap_cooldowns = Arc::new(RwLock::new(HashMap::new())); - let gc_policy = GarbageCollectionPolicy::from_config(&default_config); - let runtime_config = HlsProxyRuntimeConfig::from_config(&default_config, &[]); - let gc = Arc::new(HlsGarbageCollector::new_with_metrics( - Arc::clone(&sessions), - Arc::clone(&segment_cache), - gc_policy.clone(), - runtime_config.rewrite_secret_fingerprint.clone(), - Arc::clone(&metrics), - )); - segment_cache.install_capacity_reclaimer(&gc); - gc.install_access_leases(&access_leases); - let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default()); - Self { - sessions, - segment_cache, - segment_repair, - segment_worker_pool: Arc::new(HlsSegmentWorkerPool::with_global_semaphore_metrics_and_availability( - segment_fetch_policy.clone(), - Arc::clone(&global_fetch_semaphore), - Arc::clone(&access_leases), - Arc::clone(&metrics), - Some(Arc::clone(&availability_reevaluations)), - )), - map_worker_pool: Arc::new(HlsMapWorkerPool::with_global_semaphore_access_leases_and_availability( - segment_fetch_policy.clone(), - global_fetch_semaphore, - Arc::clone(&access_leases), - Some(Arc::clone(&availability_reevaluations)), - )), - runtime_config: ArcSwap::from_pointee(runtime_config), - transient_resources: Arc::new(TransientResourceStore::new()), - access_leases, - lifecycle, - account_overlap_cooldowns, - metrics, - qos, - gc, - prepared_terminal_bundles: Arc::new(HlsPreparedTerminalBundleCache::new()), - standalone_custom_access: Arc::new(HlsStandaloneCustomAccessStore::default()), - terminal_commit_retries: Arc::new(HlsTerminalCommitRetryCoordinator::default()), - terminal_pending: Arc::new(HlsTerminalPendingCoordinator::default()), - availability_reevaluations, - terminal_commit_clock: Arc::new(HlsTerminalCommitClock::default()), - startup_observability: Arc::new(HlsStartupObservability::default()), - } - } - - pub fn with_hls_cache_config(config: &HlsCacheConfig) -> Self { - Self::with_hls_cache_config_and_secret(config, &[]) - } - - pub fn with_hls_cache_config_and_secret(config: &HlsCacheConfig, rewrite_secret: &[u8]) -> Self { - Self::with_hls_cache_config_and_secret_enabled(config, rewrite_secret, true) - } - - fn with_hls_cache_config_and_secret_enabled(config: &HlsCacheConfig, rewrite_secret: &[u8], enabled: bool) -> Self { - let segment_fetch_policy = SegmentFetchPolicy::from_config(config); - let global_fetch_semaphore = Arc::new(Semaphore::new(segment_fetch_policy.max_global_segment_fetches)); - let sessions = Arc::new(HlsSessionStore::new()); - let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(PathBuf::from(&config.cache_path))); - segment_cache.update_cache_limits(config.cache_bytes.get(), config.cache_bytes_per_session.get()); - let segment_repair = Arc::new(HlsSegmentRepairManager::new(config.segment_repair.clone())); - let metrics = Arc::new(HlsCacheMetrics::default()); - let qos = Arc::new(HlsQosRegistry::default()); - let access_leases = Arc::new(RwLock::new(HlsAccessLeaseStore::default())); - let lifecycle = Arc::new(HlsLifecycleManager::new()); - let account_overlap_cooldowns = Arc::new(RwLock::new(HashMap::new())); - let gc_policy = GarbageCollectionPolicy::from_config(config); - let runtime_config = HlsProxyRuntimeConfig::from_config_with_enabled(config, rewrite_secret, enabled); - let gc = Arc::new(HlsGarbageCollector::new_with_metrics( - Arc::clone(&sessions), - Arc::clone(&segment_cache), - gc_policy.clone(), - runtime_config.rewrite_secret_fingerprint.clone(), - Arc::clone(&metrics), - )); - segment_cache.install_capacity_reclaimer(&gc); - gc.install_access_leases(&access_leases); - let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default()); - Self { - sessions, - segment_cache, - segment_repair, - segment_worker_pool: Arc::new(HlsSegmentWorkerPool::with_global_semaphore_metrics_and_availability( - segment_fetch_policy.clone(), - Arc::clone(&global_fetch_semaphore), - Arc::clone(&access_leases), - Arc::clone(&metrics), - Some(Arc::clone(&availability_reevaluations)), - )), - map_worker_pool: Arc::new(HlsMapWorkerPool::with_global_semaphore_access_leases_and_availability( - segment_fetch_policy.clone(), - global_fetch_semaphore, - Arc::clone(&access_leases), - Some(Arc::clone(&availability_reevaluations)), - )), - runtime_config: ArcSwap::from_pointee(runtime_config), - transient_resources: Arc::new(TransientResourceStore::new()), - access_leases, - lifecycle, - account_overlap_cooldowns, - metrics, - qos, - gc, - prepared_terminal_bundles: Arc::new(HlsPreparedTerminalBundleCache::new()), - standalone_custom_access: Arc::new(HlsStandaloneCustomAccessStore::default()), - terminal_commit_retries: Arc::new(HlsTerminalCommitRetryCoordinator::default()), - terminal_pending: Arc::new(HlsTerminalPendingCoordinator::default()), - availability_reevaluations, - terminal_commit_clock: Arc::new(HlsTerminalCommitClock::default()), - startup_observability: Arc::new(HlsStartupObservability::default()), - } - } - - pub fn sessions(&self) -> &Arc { &self.sessions } - - pub fn segment_cache(&self) -> &Arc { &self.segment_cache } - - pub fn segment_repair(&self) -> &Arc { &self.segment_repair } - - pub fn startup_observability(&self) -> &Arc { &self.startup_observability } - - pub fn spawn_access_lease_repair_prewarm( - self: &Arc, - session: HlsSessionHandle, - lease_id: HlsAccessLeaseId, - snapshot: HlsLeaseManifestSnapshot, - snapshot_generation: u64, - ) { - if snapshot.delivery_mode != HlsManifestDeliveryMode::NormalCacheTimeline - || snapshot.container != super::terminal_tail::HlsMediaContainer::MpegTs - { - return; - } - let candidate_limit = self.segment_repair.prewarm_candidate_limit(); - if candidate_limit == 0 { - return; - } - let manager = Arc::clone(self); - tokio::spawn(async move { - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let Some(lease) = - manager.access_lease_response_snapshot(&lease_id, &proxy_session_id, current_time_millis()).await - else { - return; - }; - let guard = HlsRepairPrewarmGuard::new( - Arc::clone(&manager.access_leases), - lease_id.clone(), - proxy_session_id, - lease.issued_at_ms, - snapshot_generation, - ); - manager.segment_repair.ensure_access_lease_window(lease_id.clone()).await; - let candidates = { - let session = session.read().await; - ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, candidate_limit) - }; - manager - .segment_repair - .spawn_ready_cache_prewarm(Arc::clone(&manager.segment_cache), candidates, guard) - .await; - }); - } - - pub fn segment_worker_pool(&self) -> &Arc { &self.segment_worker_pool } - - pub fn map_worker_pool(&self) -> &Arc { &self.map_worker_pool } - - pub fn start_prepared_terminal_bundle( - &self, - asset: Arc, - target_duration_ms: u64, - segment_count: u16, - ) -> HlsPreparedTerminalBundleState { - self.prepared_terminal_bundles.start_preparation(asset, target_duration_ms, segment_count) - } - - pub fn prepared_terminal_bundle_state( - &self, - key: HlsPreparedTerminalBundleKey, - ) -> Option { - self.prepared_terminal_bundles.state(key) - } - - #[cfg(any(test, feature = "test-support"))] - pub fn install_controlled_terminal_bundle_flight_for_test( - &self, - key: HlsPreparedTerminalBundleKey, - ) -> Option { - self.prepared_terminal_bundles.install_controlled_flight_for_test(key) - } - - pub fn observe_prepared_terminal_bundle( - &self, - key: HlsPreparedTerminalBundleKey, - ) -> HlsPreparedTerminalBundleObservation { - self.prepared_terminal_bundles.observe_exact(key) - } - - pub fn register_standalone_custom_access(&self, entry: HlsStandaloneCustomAccessEntry, now_ms: u64) { - self.standalone_custom_access.register(entry, now_ms); - } - - pub fn resolve_standalone_custom_segment( - &self, - lease_id: &HlsAccessLeaseId, - asset_fingerprint: &str, - index: u16, - now_ms: u64, - ) -> Result { - self.standalone_custom_access.resolve(lease_id, asset_fingerprint, index, now_ms) - } - - pub fn terminal_pending(&self) -> Arc { Arc::clone(&self.terminal_pending) } - - /// Cancels terminal work frozen before newly committed shared media - /// progress. Callers must not hold the session or lease-store lock; late - /// registrations remain protected by the final progress-generation CAS. - pub fn cancel_superseded_terminal_work_for_session(&self, proxy_session_id: &ProxySessionId) { - self.terminal_pending.cancel_session(proxy_session_id); - self.terminal_commit_retries.cancel_session(proxy_session_id); - } - - pub fn terminal_commit_now_ms(&self) -> u64 { self.terminal_commit_clock.now_ms() } - - /// The clock every time-dependent HLS scheduler reads. - /// - /// One clock, so a test that controls time controls all of it rather than - /// some schedulers following the wall clock and others the test clock. - pub(crate) fn now_ms(&self) -> u64 { self.terminal_commit_clock.now_ms() } - - /// Anchor the scheduling clock to tokio's, for `start_paused` tests. - #[cfg(test)] - pub(crate) fn follow_tokio_clock_for_test(&self, base_ms: u64) { - self.terminal_commit_clock.follow_tokio_clock(base_ms); - } - - #[cfg(any(test, feature = "test-support"))] - pub async fn wait_for_prepared_terminal_bundle( - &self, - key: HlsPreparedTerminalBundleKey, - ) -> Option { - self.prepared_terminal_bundles.wait_for_completion(key).await - } - - pub fn reserve_switch_segment_cleanup( - &self, - key: super::SegmentCacheKey, - ) -> Option { - self.gc.reserve_switch_segment_cleanup(key) - } - - pub fn reserve_switch_map_cleanup( - &self, - key: super::MapCacheKey, - ) -> Option { - self.gc.reserve_switch_map_cleanup(key) - } - - pub fn has_pending_switch_cleanup( - &self, - segment_key: &super::SegmentCacheKey, - map_key: Option<&super::MapCacheKey>, - ) -> bool { - self.gc.has_pending_switch_cleanup(segment_key, map_key) - } - - #[cfg(any(test, feature = "test-support"))] - pub fn cache_deletion_queue_usage(&self) -> (usize, usize) { self.gc.cache_deletion_queue_usage() } - - pub fn segment_fetch_policy(&self) -> SegmentFetchPolicy { self.runtime_config.load().segment_fetch_policy.clone() } - - pub fn is_enabled(&self) -> bool { self.runtime_config.load().enabled } - - pub fn cache_duration_seconds(&self) -> u64 { self.runtime_config.load().cache_duration_seconds } - - pub fn session_idle_timeout_ms(&self) -> u64 { self.runtime_config.load().gc_policy.session_idle_timeout_ms } - - pub fn strip(&self) -> StripConfig { self.runtime_config.load().strip.clone() } - - pub fn origin_manifest_timeout_ms(&self) -> u64 { self.runtime_config.load().origin_manifest_timeout_ms } - - pub fn initial_manifest_wait_timeout_secs(&self) -> u64 { - self.runtime_config.load().initial_manifest_wait_timeout_secs - } - - pub fn manifest_recovery_burst(&self) -> HlsManifestRecoveryBurstConfig { - self.runtime_config.load().manifest_recovery_burst.clone() - } - - pub fn transient_resource_ttl_ms(&self) -> u64 { self.runtime_config.load().transient_resource_ttl_ms } - - pub fn transient_resources(&self) -> &Arc { &self.transient_resources } - - pub fn access_leases(&self) -> &Arc> { &self.access_leases } - - pub fn lifecycle(&self) -> &Arc { &self.lifecycle } - - pub fn metrics(&self) -> &Arc { &self.metrics } - - pub fn qos(&self) -> &Arc { &self.qos } - - pub fn garbage_collector(&self) -> &Arc { &self.gc } - - pub fn gc_policy(&self) -> GarbageCollectionPolicy { self.runtime_config.load().gc_policy.clone() } - - pub fn rewrite_secret_fingerprint(&self) -> String { self.runtime_config.load().rewrite_secret_fingerprint.clone() } - - pub async fn is_account_overlap_cooling_down( - &self, - input_name: &Arc, - account_name: &Arc, - now_ms: u64, - ) -> bool { - let key = - HlsAccountOverlapCooldownKey { input_name: Arc::clone(input_name), account_name: Arc::clone(account_name) }; - let mut cooldowns = self.account_overlap_cooldowns.write().await; - let Some(cooldown) = cooldowns.get(&key).copied() else { - return false; - }; - if now_ms >= cooldown.until_ms { - cooldowns.remove(&key); - return false; - } - true - } - - pub async fn mark_account_overlap_reclaimed_cooldown( - &self, - input_name: Arc, - account_name: Arc, - now_ms: u64, - hard_active_window_ms: u64, - ) { - self.mark_account_overlap_cooldown( - input_name, - account_name, - now_ms, - hard_active_window_ms, - HlsAccountOverlapCooldownReason::ReclaimedByOriginalOwner, - ) - .await; - } - - pub async fn mark_account_overlap_promoted_cooldown( - &self, - input_name: Arc, - account_name: Arc, - now_ms: u64, - hard_active_window_ms: u64, - ) { - self.mark_account_overlap_cooldown( - input_name, - account_name, - now_ms, - hard_active_window_ms, - HlsAccountOverlapCooldownReason::SpeculativePromoted, - ) - .await; - } - - async fn mark_account_overlap_cooldown( - &self, - input_name: Arc, - account_name: Arc, - now_ms: u64, - hard_active_window_ms: u64, - reason: HlsAccountOverlapCooldownReason, - ) { - let until_ms = now_ms.saturating_add(hard_active_window_ms); - if until_ms <= now_ms { - return; - } - let key = HlsAccountOverlapCooldownKey { input_name, account_name }; - self.account_overlap_cooldowns.write().await.insert(key.clone(), HlsAccountOverlapCooldown { until_ms }); - debug!( - "HLS account overlap cooldown set for input {} account {} until {} ms after {}", - sanitize_sensitive_info(key.input_name.as_ref()), - sanitize_sensitive_info(key.account_name.as_ref()), - until_ms, - reason.as_log_reason() - ); - } - - pub async fn update_config(&self, app_config: &AppConfig) { - let (hls_config, rewrite_secret, enabled) = { - let config = app_config.config.load(); - let rewrite_secret = config - .reverse_proxy - .as_ref() - .map_or(app_config.encrypt_secret, |reverse_proxy| reverse_proxy.rewrite_secret); - let hls_config = - config.reverse_proxy.as_ref().and_then(|reverse_proxy| reverse_proxy.hls_cache.as_ref()).cloned(); - let enabled = hls_config.is_some(); - let hls_config = - hls_config.unwrap_or_else(|| HlsCacheConfig::from(&shared::model::HlsCacheConfigDto::default())); - (hls_config, rewrite_secret, enabled) - }; - let runtime_config = HlsProxyRuntimeConfig::from_config_with_enabled(&hls_config, &rewrite_secret, enabled); - let cache_path_changed = self.gc.update_cache_path(PathBuf::from(&hls_config.cache_path)).await; - self.segment_cache.update_cache_limits(hls_config.cache_bytes.get(), hls_config.cache_bytes_per_session.get()); - if cache_path_changed { - self.clear_runtime_cache_state_for_cache_path_change().await; - } - for session in self.sessions.list_sessions().await { - session - .write() - .await - .configure_segment_prefetch_queue(runtime_config.segment_fetch_policy.max_prefetch_queue_depth); - } - self.segment_repair.update_config(hls_config.segment_repair.clone()); - let global_fetch_semaphore = - Arc::new(Semaphore::new(runtime_config.segment_fetch_policy.max_global_segment_fetches)); - self.segment_worker_pool - .update_config(runtime_config.segment_fetch_policy.clone(), Arc::clone(&global_fetch_semaphore)); - self.map_worker_pool.update_config(runtime_config.segment_fetch_policy.clone(), global_fetch_semaphore); - self.gc.update_config(runtime_config.gc_policy.clone(), runtime_config.rewrite_secret_fingerprint.clone()); - self.runtime_config.store(Arc::new(runtime_config)); - } - - async fn clear_runtime_cache_state_for_cache_path_change(&self) { - self.availability_reevaluations.clear(); - self.terminal_pending.clear(); - self.terminal_commit_retries.clear(); - self.sessions.clear().await; - let removed_leases = self.access_leases.write().await.clear(); - self.standalone_custom_access.clear(); - self.terminal_commit_retries.clear(); - let removed_qos = self.qos.clear().await; - self.segment_repair.clear_runtime_state().await; - self.startup_observability.clear(); - debug!( - "HLS cache runtime state cleared after cache path change: access_leases_removed={removed_leases} qos_access_leases_removed={removed_qos}" - ); - } - - pub async fn prepare_access_lease(&self, lease: HlsAccessLease) { - if !self.access_leases.write().await.prepare_access_lease(lease.clone()) { - error!( - "HLS access lease preparation rejected: lease={} proxy_session={} reason=availability_evidence_exhausted", - super::safe_hls_access_lease_id(&lease.lease_id), - safe_proxy_session_id(&lease.proxy_session_id) - ); - return; - } - self.schedule_access_lease_validity(&lease).await; - } - - pub async fn access_lease( - &self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - now_ms: u64, - ) -> Option { - let (lease, still_stored) = { - let mut access_leases = self.access_leases.write().await; - let lease = access_leases.access_lease(lease_id, proxy_session_id, now_ms); - let still_stored = access_leases.lease_state(lease_id, now_ms).is_some(); - (lease, still_stored) - }; - if lease.is_none() && !still_stored { - self.segment_repair.remove_access_lease_window(lease_id).await; - self.startup_observability.remove_access_lease(lease_id); - self.qos.remove_access_lease(lease_id).await; - } - lease - } - - pub async fn access_lease_response_snapshot( - &self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - now_ms: u64, - ) -> Option { - self.access_leases.write().await.response_snapshot(lease_id, proxy_session_id, now_ms) - } - - pub async fn active_live_playback_snapshots_for_session( - &self, - proxy_session_id: &ProxySessionId, - now_ms: u64, - ) -> Vec { - self.access_leases.write().await.active_live_playback_snapshots_for_session(proxy_session_id, now_ms) - } - - /// Runs the final Critical-Handoff revalidation and timeline commit under the - /// established lease-store -> session lock order. No lock is held across an await. - /// `LockBusy` reports contention only and is not a generation or lease invalidation. - pub async fn with_critical_handoff_state( - &self, - session_handle: &HlsSessionHandle, - operation: impl FnOnce(&mut HlsAccessLeaseStore, &mut super::HlsSession) -> T, - ) -> HlsCriticalHandoffStateAccess { - let mut operation = Some(operation); - for attempt in 0..HLS_STATE_CAS_LOCK_RETRIES { - let Ok(mut leases) = self.access_leases.try_write() else { - if attempt.saturating_add(1) < HLS_STATE_CAS_LOCK_RETRIES { - tokio::task::yield_now().await; - } - continue; - }; - let Ok(mut session) = session_handle.try_write() else { - drop(leases); - if attempt.saturating_add(1) < HLS_STATE_CAS_LOCK_RETRIES { - tokio::task::yield_now().await; - } - continue; - }; - if let Some(operation) = operation.take() { - return HlsCriticalHandoffStateAccess::Acquired(operation(&mut leases, &mut session)); - } - } - HlsCriticalHandoffStateAccess::LockBusy - } - - /// Captures the immutable identity used to deduplicate a later autonomous - /// availability evaluation. The session read is completed before the - /// lease-store -> session transaction starts, so no lock order is inverted. - pub async fn availability_reevaluation_owner_key( - &self, - session_handle: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - ) -> Option { - let session_incarnation = self.sessions.session_incarnation(session_handle)?; - let availability_evidence_generation = - self.access_leases.read().await.availability_evidence_generation(proxy_session_id); - let session = session_handle.read().await; - if session.proxy_session_id != *proxy_session_id || session.is_gc_marked_for_removal() { - return None; - } - Some(HlsAvailabilityReevaluationOwnerKey { - session_incarnation, - proxy_session_id: proxy_session_id.clone(), - origin_progress_generation: session.origin_control.progress_generation, - media_readiness_generation: session.activity.media_readiness_generation, - availability_evidence_generation, - }) - } - - pub async fn availability_reevaluation_session_is_current( - &self, - session_handle: &HlsSessionHandle, - owner_key: &HlsAvailabilityReevaluationOwnerKey, - ) -> bool { - if self.sessions.session_incarnation(session_handle) != Some(owner_key.session_incarnation) { - return false; - } - self.sessions - .get_by_proxy_session_id(&owner_key.proxy_session_id) - .await - .is_some_and(|current| Arc::ptr_eq(¤t, session_handle)) - } - - /// Runs the refresh-start mutation only while the lease evidence, session - /// index, concrete handle, incarnation, and recovery-pressure generations - /// are all current. Cross-store lock order is Session Index -> optional - /// Retry Owner -> Lease Store -> Session. Every acquisition here is - /// non-blocking and the closure must not await or perform I/O. - pub fn with_current_recovery_pressure_session( - &self, - session_handle: &HlsSessionHandle, - guard: &HlsRecoveryPressureGuard, - operation: impl FnOnce(&mut super::HlsSession) -> R, - ) -> HlsRecoveryPressureGuardAccess { - if self.sessions.session_incarnation(session_handle) != Some(guard.session_incarnation) { - return HlsRecoveryPressureGuardAccess::Superseded; - } - let access = self.sessions.try_with_current_proxy_session(&guard.proxy_session_id, session_handle, || { - let Ok(leases) = self.access_leases.try_read() else { - return HlsRecoveryPressureGuardAccess::LockBusy; - }; - if leases.availability_evidence_generation(&guard.proxy_session_id) - != guard.availability_evidence_generation - { - return HlsRecoveryPressureGuardAccess::Superseded; - } - let Ok(mut session) = session_handle.try_write() else { - return HlsRecoveryPressureGuardAccess::LockBusy; - }; - if session.proxy_session_id != guard.proxy_session_id - || session.origin_control.progress_generation != guard.origin_progress_generation - || session.activity.media_readiness_generation != guard.media_readiness_generation - || leases.availability_evidence_generation(&guard.proxy_session_id) - != guard.availability_evidence_generation - || session.is_gc_marked_for_removal() - { - return HlsRecoveryPressureGuardAccess::Superseded; - } - HlsRecoveryPressureGuardAccess::Acquired(operation(&mut session)) - }); - match access { - HlsCurrentProxySessionAccess::Acquired(access) => access, - HlsCurrentProxySessionAccess::Superseded => HlsRecoveryPressureGuardAccess::Superseded, - HlsCurrentProxySessionAccess::LockBusy => HlsRecoveryPressureGuardAccess::LockBusy, - } - } - - pub fn availability_reevaluations(&self) -> Arc { - Arc::clone(&self.availability_reevaluations) - } - - pub fn notify_session_evidence_changed(&self, proxy_session_id: &ProxySessionId) -> bool { - self.availability_reevaluations.notify_session_evidence_changed(proxy_session_id) - } - - #[cfg(any(test, feature = "test-support"))] - pub fn set_terminal_commit_retry_capacity_for_test(&self, capacity: usize) { - self.terminal_commit_retries.set_capacity_for_test(capacity); - } - - #[cfg(any(test, feature = "test-support"))] - pub async fn hold_access_lease_store_for_test(&self) -> tokio::sync::OwnedRwLockWriteGuard { - Arc::clone(&self.access_leases).write_owned().await - } - - pub async fn prepare_access_lease_manifest_publication( - &self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - now_ms: u64, - ) -> Option { - self.access_leases.write().await.prepare_manifest_publication(lease_id, proxy_session_id, now_ms) - } - - pub async fn commit_access_lease_manifest_publication_with_resources( - &self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - expected: HlsLeaseManifestPublicationGuard, - snapshot: HlsLeaseManifestSnapshot, - published_transient_resource_ids: HlsPublishedTransientResourceIds, - now_ms: u64, - ) -> HlsLeaseManifestPublicationOutcome { - let finalized_manifest_generation = snapshot.finalized_transient_manifest_generation; - let lease_issued_at_ms = expected.lease_issued_at_ms(); - let Some(finalized_manifest_generation) = finalized_manifest_generation else { - let session = self.sessions.get_by_proxy_session_id(proxy_session_id).await; - // The lease-store write lock keeps the previous lease membership stable while the - // session read produces an immutable merged membership snapshot. The snapshot can then - // be committed without reopening the lease-store transaction. - let mut leases = self.access_leases.write().await; - let published_transient_resource_ids = if let Some(session) = session.as_ref() { - let session = session.read().await; - if session.proxy_session_id == *proxy_session_id { - leases.published_transient_resource_ids(lease_id).map_or( - published_transient_resource_ids.clone(), - |previous| { - session.transient.merge_current_published_resource_ids( - previous, - published_transient_resource_ids, - now_ms, - ) - }, - ) - } else { - published_transient_resource_ids - } - } else { - published_transient_resource_ids - }; - return leases.commit_manifest_publication_with_resources( - lease_id, - proxy_session_id, - expected, - snapshot, - published_transient_resource_ids, - now_ms, - ); - }; - let Some(current_session) = self.sessions.hold_current_proxy_session(proxy_session_id).await else { - return HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::ManifestGenerationUnavailable, - ); - }; - // The index guard prevents replacement while the established index -> lease store -> session - // lock order publishes the exact snapshot and binds its finalized source generation. - let session = Arc::clone(current_session.session()); - let mut leases = self.access_leases.write().await; - let mut session = session.write().await; - if session.proxy_session_id != *proxy_session_id || session.is_gc_marked_for_removal() { - return HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::ManifestGenerationUnavailable, - ); - } - if !session.transient.has_finalized_manifest_generation(finalized_manifest_generation) { - return HlsLeaseManifestPublicationOutcome::Rejected( - HlsLeaseManifestPublicationRejectReason::ManifestGenerationUnavailable, - ); - } - let outcome = leases.commit_manifest_publication_with_resources( - lease_id, - proxy_session_id, - expected, - snapshot, - published_transient_resource_ids, - now_ms, - ); - if outcome.is_committed() { - let bound = session.transient.bind_finalized_manifest_generation(TransientManifestLeaseBinding::new( - lease_id.clone(), - lease_issued_at_ms, - finalized_manifest_generation, - )); - debug_assert!(bound, "validated finalized manifest generation must remain bindable"); - } - drop(session); - drop(leases); - drop(current_session); - outcome - } - - #[cfg(any(test, feature = "test-support"))] - pub async fn commit_access_lease_manifest_publication( - &self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - expected: HlsLeaseManifestPublicationGuard, - snapshot: HlsLeaseManifestSnapshot, - now_ms: u64, - ) -> HlsLeaseManifestPublicationOutcome { - self.commit_access_lease_manifest_publication_with_resources( - lease_id, - proxy_session_id, - expected, - snapshot, - HlsPublishedTransientResourceIds::default(), - now_ms, - ) - .await - } - - pub async fn record_access_lease_segment_request_started_if_identity_matches( - &self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - lease_identity: HlsMediaLeaseIdentity, - proxy_seq: u64, - requested_at_ms: u64, - ) -> Option { - self.access_leases.write().await.record_segment_request_started_if_identity_matches( - lease_id, - proxy_session_id, - lease_identity, - proxy_seq, - requested_at_ms, - ) - } - - pub async fn record_access_lease_segment_request_completed_and_mark_media_if_identity_matches( - &self, - session: &HlsSessionHandle, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - lease_identity: HlsMediaLeaseIdentity, - token: HlsPlaybackRequestToken, - completed_at_ms: u64, - ) -> HlsMediaActivityCommitOutcome { - self.commit_media_activity_if_identity_matches( - session, - lease_id, - proxy_session_id, - lease_identity, - completed_at_ms, - HlsMediaActivityCommitKind::LiveSegmentCompletion(token), - ) - .await - } - - pub async fn prepare_access_lease_terminal_tail( - &self, - request: HlsTerminalTailPreparationRequest<'_>, - ) -> Option { - self.prepare_access_lease_terminal_decision(request, HlsTerminalPreparationPurpose::Cutover).await - } - - pub async fn prepare_access_lease_terminal_unavailable_after_owner_failure( - &self, - request: HlsTerminalTailPreparationRequest<'_>, - ) -> Option { - self.prepare_access_lease_terminal_decision( - request, - HlsTerminalPreparationPurpose::UnavailableAfterOwnerFailure, - ) - .await - } - - pub async fn prepare_access_lease_runtime_custom_tail( - &self, - session: &HlsSessionHandle, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - reason: HlsRuntimeCustomTailReason, - now_ms: u64, - ) -> Option { - if reason.trigger_class() - != super::runtime_custom_tail::HlsRuntimeCustomTailTriggerClass::ImmediatePolicyCutover - { - return None; - } - let current_session = self.sessions.get_by_proxy_session_id(proxy_session_id).await?; - if !Arc::ptr_eq(¤t_session, session) { - return None; - } - let lease = self.access_lease_response_snapshot(lease_id, proxy_session_id, now_ms).await?; - if lease.playback_mode != super::terminal_tail::HlsLeasePlaybackMode::Live { - return None; - } - let manifest = lease.last_manifest_snapshot.as_ref()?; - let ( - origin_progress_generation, - media_readiness_generation, - origin_epoch, - last_media_progress_at_ms, - reserve, - recovery_snapshot, - ) = { - let session = session.read().await; - if session.proxy_session_id != *proxy_session_id || session.is_gc_marked_for_removal() { - return None; - } - let ready_timeline = session.ready_timeline_snapshot( - lease.playback_cursor.ready_timeline_start_proxy_seq(manifest.first_proxy_seq), - now_ms, - ); - let reserve = evaluate_lease_reserve(HlsLeaseReserveInput { - manifest, - cursor: &lease.playback_cursor, - ready_timeline: &ready_timeline, - now_ms, - playback_rate_guard_milli: super::HLS_PLAYBACK_RATE_GUARD_MILLI, - recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(0), - origin_path_degraded: true, - recovery_committed: false, - }); - ( - session.origin_control.progress_generation, - session.activity.media_readiness_generation, - session.origin_control.origin_epoch, - session.origin_control.last_media_progress_at_ms, - reserve, - hls_acceptance_recovery_snapshot(&session, now_ms), - ) - }; - let technical_commit_budget_ms = self - .origin_manifest_timeout_ms() - .max(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()); - let cutover_timing = HlsLeaseCutoverTiming::from_reserve( - now_ms, - technical_commit_budget_ms, - HlsTransitionMarginMs::from_millis(0), - None, - ); - self.access_leases.read().await.prepare_terminal_tail( - lease_id, - proxy_session_id, - &HlsTerminalTailPreparationInput { - trigger: HlsFiniteTailTrigger::RuntimePolicy(reason), - expected_manifest_snapshot_generation: manifest.snapshot_generation, - expected_cursor_generation: lease.playback_cursor.cursor_generation, - origin_progress_generation, - media_readiness_generation, - origin_epoch, - last_media_progress_at_ms, - expected_acceptance_generation: recovery_snapshot.expected_generation, - terminal_media_requirement_origin: HlsTerminalMediaRequirementOrigin::CutoverSnapshot, - cutover_timing, - commit_window: HlsTerminalCommitWindow::CutoverDue, - required_terminal_media_key: None, - terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, - reserve, - }, - ) - } - - pub async fn begin_runtime_policy_revocation( - &self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - reason: HlsRuntimeCustomTailReason, - now_ms: u64, - ) -> HlsRuntimePolicyRevocationOutcome { - self.access_leases.write().await.begin_runtime_policy_revocation(lease_id, proxy_session_id, reason, now_ms) - } - - pub async fn fail_runtime_policy_revocation( - &self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - token: &HlsRuntimePolicyRevocation, - ) -> HlsAccessLeaseDenialOutcome { - let outcome = - self.access_leases.write().await.fail_runtime_policy_revocation(lease_id, proxy_session_id, token); - if matches!(outcome, HlsAccessLeaseDenialOutcome::Ended { .. }) { - self.terminal_pending.cancel_lease(lease_id); - self.terminal_commit_retries.cancel_lease(lease_id); - } - outcome - } - - async fn prepare_access_lease_terminal_decision( - &self, - request: HlsTerminalTailPreparationRequest<'_>, - purpose: HlsTerminalPreparationPurpose, - ) -> Option { - let HlsTerminalTailPreparationRequest { - lease_id, - proxy_session_id, - manifest_snapshot_generation, - cursor_generation, - reserve, - cutover_timing, - commit_window, - now_ms, - origin_progress_generation: expected_origin_progress_generation, - media_readiness_generation: expected_media_readiness_generation, - last_media_progress_at_ms: expected_last_media_progress_at_ms, - } = request; - let session = self.sessions.get_by_proxy_session_id(proxy_session_id).await?; - let ( - origin_progress_generation, - media_readiness_generation, - origin_epoch, - last_media_progress_at_ms, - origin_path_degraded, - recovery_snapshot, - ) = { - let session = session.read().await; - let recovery_snapshot = hls_acceptance_recovery_snapshot(&session, now_ms); - ( - session.origin_control.progress_generation, - session.activity.media_readiness_generation, - session.origin_control.origin_epoch, - session.origin_control.last_media_progress_at_ms, - session.origin_control.path_condition.is_degraded(), - recovery_snapshot, - ) - }; - let expected_cutover_timing = - HlsLeaseCutoverTiming::from_reserve(now_ms, reserve.guaranteed_reserve_ms, reserve.transition_margin, None); - if origin_progress_generation != expected_origin_progress_generation - || media_readiness_generation != expected_media_readiness_generation - || last_media_progress_at_ms != expected_last_media_progress_at_ms - || cutover_timing != expected_cutover_timing - || !hls_key_readiness_evidence_is_current(reserve.key_readiness_valid_until_ms, now_ms) - { - return None; - } - if matches!(purpose, HlsTerminalPreparationPurpose::UnavailableAfterOwnerFailure) && !origin_path_degraded { - return None; - } - let cutover_timing = cutover_timing - .with_estimated_recovery_completion_at(hls_estimated_recovery_completion_at(recovery_snapshot.recovery)); - if matches!(purpose, HlsTerminalPreparationPurpose::Cutover) - && !matches!( - evaluate_terminal_cutover(&HlsTerminalCutoverInput { - reserve, - commit_window, - acceptance: recovery_snapshot.status, - required_terminal_media_key: recovery_snapshot.required_terminal_media_key, - terminal_preparation: recovery_snapshot.terminal_media_preparation, - terminal: HlsTerminalCutoverCapability::NotEvaluated, - }), - HlsTerminalCutoverDecision::EvaluateTerminalCapability { .. } - ) - { - return None; - } - let terminal_media_requirement_origin = terminal_media_requirement_origin(&recovery_snapshot); - self.access_leases.read().await.prepare_terminal_tail( - lease_id, - proxy_session_id, - &HlsTerminalTailPreparationInput { - trigger: HlsFiniteTailTrigger::AvailabilityReserve, - expected_manifest_snapshot_generation: manifest_snapshot_generation, - expected_cursor_generation: cursor_generation, - origin_progress_generation, - media_readiness_generation, - origin_epoch, - last_media_progress_at_ms, - expected_acceptance_generation: recovery_snapshot.expected_generation, - terminal_media_requirement_origin, - cutover_timing, - commit_window, - required_terminal_media_key: recovery_snapshot.required_terminal_media_key, - terminal_media_preparation: recovery_snapshot.terminal_media_preparation, - reserve, - }, - ) - } - - pub fn commit_access_lease_terminal_if_generation_matches( - &self, - request: HlsTerminalCommitRequest<'_>, - ) -> HlsTerminalCommitOutcome { - let HlsTerminalCommitRequest { - session, - lease_id, - proxy_session_id, - preparation, - now_ms, - payload, - asset_revision_guard, - } = request; - let (decision, media_guard) = payload.into_parts(); - let key = HlsTerminalCommitOwnerKey::from_preparation(proxy_session_id, lease_id, preparation); - let Some(session_incarnation) = self.sessions.session_incarnation(session) else { - return HlsTerminalCommitOutcome::SupersededGeneration; - }; - let Some((cancellation_epoch, submission_token)) = self.terminal_commit_retries.reserve_submission() else { - return HlsTerminalCommitOutcome::RetryCapacityExceeded; - }; - let command = HlsTerminalCommitCommand { - key, - session: Arc::clone(session), - session_incarnation, - preparation: preparation.clone(), - decision, - media_guard, - asset_revision_guard, - cancellation_epoch, - submission_token, - }; - let attempt_now_ms = self.terminal_commit_clock.initial_attempt_now_ms(now_ms); - let (authorized_command, owner_token) = match self.terminal_commit_retries.submit(command, attempt_now_ms) { - HlsTerminalCommitSubmissionDecision::Attempt { command, owner_token } => (command, owner_token), - HlsTerminalCommitSubmissionDecision::PendingExisting { retry_before_ms } => { - return HlsTerminalCommitOutcome::LockBusy { retry_before_ms }; - } - HlsTerminalCommitSubmissionDecision::Failed(outcome) => return outcome, - HlsTerminalCommitSubmissionDecision::Cancelled => { - return HlsTerminalCommitOutcome::SupersededGeneration; - } - HlsTerminalCommitSubmissionDecision::CapacityExceeded => { - return HlsTerminalCommitOutcome::RetryCapacityExceeded; - } - }; - let authorized_key = authorized_command.key.clone(); - let attempt = match self.sessions.try_with_current_proxy_session( - &authorized_key.proxy_session_id, - &authorized_command.session, - || { - self.terminal_commit_retries.with_current_owner( - &authorized_key, - owner_token, - |command, latest_safe_terminal_commit_at_ms| { - let attempt_now_ms = self.terminal_commit_clock.initial_attempt_now_ms(now_ms); - let attempt = Self::try_commit_access_lease_terminal_decision( - &self.access_leases, - command, - attempt_now_ms, - latest_safe_terminal_commit_at_ms, - ); - (attempt_now_ms, latest_safe_terminal_commit_at_ms, attempt) - }, - ) - }, - ) { - HlsCurrentProxySessionAccess::Acquired(attempt) => attempt, - HlsCurrentProxySessionAccess::Superseded => { - self.terminal_commit_retries.discard_owner(&authorized_key, owner_token); - None - } - HlsCurrentProxySessionAccess::LockBusy => self.terminal_commit_retries.with_current_owner( - &authorized_key, - owner_token, - |_command, latest_safe_terminal_commit_at_ms| { - ( - self.terminal_commit_clock.initial_attempt_now_ms(now_ms), - latest_safe_terminal_commit_at_ms, - HlsTerminalCommitAttempt::LockBusy, - ) - }, - ), - }; - let Some((attempts_completed, (attempt_now_ms, deadline_ms, attempt))) = attempt else { - return HlsTerminalCommitOutcome::SupersededGeneration; - }; - match attempt { - HlsTerminalCommitAttempt::Completed(outcome) => { - self.terminal_commit_retries.complete_owner(&authorized_key, owner_token); - outcome - } - HlsTerminalCommitAttempt::LockBusy => self.schedule_terminal_commit_retry( - &authorized_key, - owner_token, - attempts_completed, - attempt_now_ms, - deadline_ms, - ), - } - } - - pub(super) fn try_commit_access_lease_terminal_decision( - access_leases: &Arc>, - command: &HlsTerminalCommitCommand, - now_ms: u64, - latest_safe_commit_at_ms: u64, - ) -> HlsTerminalCommitAttempt { - // Submission arbitration releases the owner mutex before this path. - // This follows the cross-store order documented by - // with_current_recovery_pressure_session: session index -> retry owner - // -> lease store -> session. The outer two guards prevent replacement - // or cancellation TOCTOU; all inner locks are non-blocking and no - // guard crosses an await or I/O boundary. - let Ok(mut leases) = access_leases.try_write() else { - return HlsTerminalCommitAttempt::LockBusy; - }; - let Ok(mut session) = command.session.try_write() else { - return HlsTerminalCommitAttempt::LockBusy; - }; - if let Some(outcome) = - Self::terminal_commit_precondition_outcome(&mut leases, &session, command, now_ms, latest_safe_commit_at_ms) - { - return HlsTerminalCommitAttempt::Completed(outcome); - } - let lease_id = &command.key.lease_id; - let preparation = &command.preparation; - let decision = &command.decision; - // Asset identity gates only a new tail mutation. An exact terminal - // replay was handled above and remains immutable. If a prepared tail's - // asset changed while a LockBusy owner was queued, fail closed in this - // same generation-bound CAS instead of abandoning the autonomous - // intent and leaving the lease live. - match command.asset_revision_guard.validate_current() { - HlsTerminalAssetRevisionValidation::Current => {} - HlsTerminalAssetRevisionValidation::Changed { current } => { - if command.preparation.trigger.is_runtime_policy() { - return HlsTerminalCommitAttempt::Completed(HlsTerminalCommitOutcome::SupersededGeneration); - } - let reason = current.asset.map_or(HlsTerminalTailCompatibility::MissingAsset, |_| { - HlsTerminalTailCompatibility::AssetRevisionMismatch - }); - return HlsTerminalCommitAttempt::Completed(Self::commit_terminal_unavailable_fallback( - &mut leases, - &mut session, - command, - now_ms, - reason, - )); - } - } - let protection_capacity_exceeded = - match evaluate_terminal_commit_authorization(&session, lease_id, preparation, decision, now_ms) { - HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded } => { - protection_capacity_exceeded - } - HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleIncompatible) - if matches!(decision, HlsTerminalLeaseDecision::Tail(_)) => - { - return HlsTerminalCommitAttempt::Completed(Self::commit_terminal_unavailable_fallback( - &mut leases, - &mut session, - command, - now_ms, - HlsTerminalTailCompatibility::TerminalMediaNotReady, - )); - } - HlsTerminalCommitAuthorization::Rejected(outcome) => { - return HlsTerminalCommitAttempt::Completed(outcome); - } - }; - HlsTerminalCommitAttempt::Completed(Self::publish_terminal_commit( - &mut leases, - &mut session, - command, - now_ms, - protection_capacity_exceeded, - )) - } - - fn terminal_commit_precondition_outcome( - leases: &mut HlsAccessLeaseStore, - session: &super::HlsSession, - command: &HlsTerminalCommitCommand, - now_ms: u64, - latest_safe_commit_at_ms: u64, - ) -> Option { - let lease_id = &command.key.lease_id; - let proxy_session_id = &command.key.proxy_session_id; - let preparation = &command.preparation; - let replay = match &command.decision { - HlsTerminalLeaseDecision::Tail(_) => { - leases.terminal_tail_replay_outcome(lease_id, proxy_session_id, preparation, now_ms) - } - HlsTerminalLeaseDecision::Unavailable(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_) => { - leases.terminal_unavailable_replay_outcome(lease_id, proxy_session_id, preparation, now_ms) - } - }; - if replay.is_some() { - return replay; - } - if now_ms >= latest_safe_commit_at_ms { - return Some(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed); - } - if preparation.trigger.is_runtime_policy() { - return None; - } - if session.origin_control.progress_generation == preparation.origin_progress_generation - && session.activity.media_readiness_generation == preparation.media_readiness_generation - && session.origin_control.origin_epoch == preparation.origin_epoch - && session.origin_control.last_media_progress_at_ms == preparation.last_media_progress_at_ms - && hls_key_readiness_evidence_is_current(preparation.reserve.key_readiness_valid_until_ms, now_ms) - { - return None; - } - Some(if session.origin_control.last_media_progress_at_ms == preparation.last_media_progress_at_ms { - HlsTerminalCommitOutcome::SupersededGeneration - } else { - HlsTerminalCommitOutcome::RecoveryCommitted - }) - } - - fn publish_terminal_commit( - leases: &mut HlsAccessLeaseStore, - session: &mut super::HlsSession, - command: &HlsTerminalCommitCommand, - now_ms: u64, - protection_capacity_exceeded: bool, - ) -> HlsTerminalCommitOutcome { - let lease_id = &command.key.lease_id; - let proxy_session_id = &command.key.proxy_session_id; - let preparation = &command.preparation; - let decision = &command.decision; - let protection = match (decision, protection_capacity_exceeded) { - (HlsTerminalLeaseDecision::Tail(plan), false) => Some(HlsTerminalTailProtection { - generation: plan.generation, - base_proxy_seqs: Arc::clone(&plan.protected_base_proxy_seqs), - key_bindings: plan.key_bindings(), - }), - (HlsTerminalLeaseDecision::Tail(_), true) - | ( - HlsTerminalLeaseDecision::Unavailable(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_), - _, - ) => None, - }; - let previous_protection = session.remove_terminal_tail_protection(lease_id); - if let Some(protection) = protection { - if session.install_terminal_tail_protection(lease_id.clone(), protection) - != HlsTerminalTailProtectionInstall::Installed - { - session.rollback_terminal_tail_protection(lease_id.clone(), previous_protection); - return Self::commit_terminal_unavailable_fallback( - leases, - session, - command, - now_ms, - HlsTerminalTailCompatibility::ProtectionCapacityExceeded, - ); - } - } - let outcome = match (decision, protection_capacity_exceeded) { - (HlsTerminalLeaseDecision::Tail(plan), false) => leases.commit_terminal_tail_if_generation_matches( - lease_id, - proxy_session_id, - preparation, - now_ms, - Arc::clone(plan), - ), - (HlsTerminalLeaseDecision::Tail(_), true) => leases.commit_terminal_unavailable_if_generation_matches( - lease_id, - proxy_session_id, - preparation, - now_ms, - HlsTerminalTailCompatibility::ProtectionCapacityExceeded, - ), - ( - HlsTerminalLeaseDecision::Unavailable(reason) - | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(reason), - _, - ) => leases.commit_terminal_unavailable_if_generation_matches( - lease_id, - proxy_session_id, - preparation, - now_ms, - *reason, - ), - }; - if outcome != HlsTerminalCommitOutcome::Committed { - session.rollback_terminal_tail_protection(lease_id.clone(), previous_protection); - if outcome == HlsTerminalCommitOutcome::BundleIncompatible - && matches!(decision, HlsTerminalLeaseDecision::Tail(_)) - { - return Self::commit_terminal_unavailable_fallback( - leases, - session, - command, - now_ms, - HlsTerminalTailCompatibility::TerminalMediaNotReady, - ); - } - return outcome; - } - Self::finish_terminal_commit(leases, session, proxy_session_id); - HlsTerminalCommitOutcome::Committed - } - - fn commit_terminal_unavailable_fallback( - leases: &mut HlsAccessLeaseStore, - session: &mut super::HlsSession, - command: &HlsTerminalCommitCommand, - now_ms: u64, - reason: HlsTerminalTailCompatibility, - ) -> HlsTerminalCommitOutcome { - let lease_id = &command.key.lease_id; - let proxy_session_id = &command.key.proxy_session_id; - let decision = if matches!(&command.decision, HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_)) { - HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(reason) - } else { - HlsTerminalLeaseDecision::Unavailable(reason) - }; - match evaluate_terminal_commit_authorization(session, lease_id, &command.preparation, &decision, now_ms) { - HlsTerminalCommitAuthorization::Authorized { .. } => {} - HlsTerminalCommitAuthorization::Rejected(outcome) => return outcome, - } - let previous_protection = session.remove_terminal_tail_protection(lease_id); - let outcome = leases.commit_terminal_unavailable_if_generation_matches( - lease_id, - proxy_session_id, - &command.preparation, - now_ms, - reason, - ); - if outcome != HlsTerminalCommitOutcome::Committed { - session.rollback_terminal_tail_protection(lease_id.clone(), previous_protection); - return outcome; - } - Self::finish_terminal_commit(leases, session, proxy_session_id); - HlsTerminalCommitOutcome::Committed - } - - fn finish_terminal_commit( - leases: &mut HlsAccessLeaseStore, - session: &mut super::HlsSession, - proxy_session_id: &ProxySessionId, - ) { - let all_terminal = leases.all_live_leases_terminal_for_session(proxy_session_id); - session.origin_control.progress_phase = if all_terminal { - super::origin_progress::HlsOriginProgressPhase::Terminal - } else { - super::origin_progress::HlsOriginProgressPhase::TerminalPartial - }; - } - - fn schedule_terminal_commit_retry( - &self, - key: &HlsTerminalCommitOwnerKey, - owner_token: HlsTerminalCommitOwnerToken, - attempts_completed: u8, - last_attempt_at_ms: u64, - latest_safe_terminal_commit_at_ms: u64, - ) -> HlsTerminalCommitOutcome { - let (retry_at_ms, attempts_completed) = - match next_terminal_commit_retry(attempts_completed, last_attempt_at_ms, latest_safe_terminal_commit_at_ms) - { - HlsTerminalCommitRetryDecision::Schedule { retry_at_ms, attempts_completed } => { - (retry_at_ms, attempts_completed) - } - HlsTerminalCommitRetryDecision::AttemptsExhausted => { - self.terminal_commit_retries.fail_owner( - key, - owner_token, - HlsTerminalCommitOutcome::RetryAttemptsExhausted, - ); - return HlsTerminalCommitOutcome::RetryAttemptsExhausted; - } - HlsTerminalCommitRetryDecision::SafeDeadlineElapsed => { - self.terminal_commit_retries.fail_owner( - key, - owner_token, - HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed, - ); - return HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed; - } - }; - let registration_now_ms = self.terminal_commit_clock.now_ms().max(last_attempt_at_ms); - match self.terminal_commit_retries.schedule_current( - key, - owner_token, - attempts_completed, - retry_at_ms, - registration_now_ms, - ) { - HlsTerminalCommitRetryScheduleDecision::Scheduled { worker_token } => { - if let Some(worker_token) = worker_token { - spawn_terminal_commit_retry_worker( - Arc::clone(&self.sessions), - Arc::clone(&self.access_leases), - Arc::clone(&self.terminal_commit_retries), - Arc::clone(&self.terminal_commit_clock), - worker_token, - Self::try_commit_access_lease_terminal_decision, - ); - } - HlsTerminalCommitOutcome::LockBusy { retry_before_ms: retry_at_ms } - } - HlsTerminalCommitRetryScheduleDecision::Failed(outcome) => outcome, - HlsTerminalCommitRetryScheduleDecision::Cancelled => HlsTerminalCommitOutcome::SupersededGeneration, - HlsTerminalCommitRetryScheduleDecision::WorkerUnavailable => { - HlsTerminalCommitOutcome::RetryWorkerUnavailable - } - } - } - - pub async fn update_access_lease_origin_acquire_policy( - &self, - lease_id: &HlsAccessLeaseId, - connection_kind: tuliprox_session::ConnectionKind, - priority: i8, - ) -> Option { - self.access_leases.write().await.update_origin_acquire_policy(lease_id, connection_kind, priority) - } - - pub async fn activate_access_lease( - &self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - now_ms: u64, - timing: HlsAccessLeaseTiming, - ) -> HlsAccessLeaseActivation { - let activation = - self.access_leases.write().await.activate_access_lease(lease_id, proxy_session_id, now_ms, timing); - if let HlsAccessLeaseActivation::Activated { lease, previous_state } = &activation { - if *previous_state == HlsAccessLeaseState::Pending { - self.segment_repair.ensure_access_lease_window(lease.lease_id.clone()).await; - } else if *previous_state == HlsAccessLeaseState::Idle { - self.segment_repair.start_access_lease_window(lease.lease_id.clone()).await; - } - self.schedule_access_lease_activity(lease).await; - self.schedule_access_lease_validity(lease).await; - } - activation - } - - pub async fn touch_manifest_access_lease( - &self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - now_ms: u64, - active_timing: Option, - pending_deadline: Option, - ttl_ms: u64, - ) -> HlsAccessLeaseTouch { - let touch = self.access_leases.write().await.touch_manifest_access_lease( - lease_id, - proxy_session_id, - now_ms, - active_timing, - pending_deadline, - ttl_ms, - ); - if let HlsAccessLeaseTouch::Touched { lease } = &touch { - if lease.state == HlsAccessLeaseState::Activated { - self.schedule_access_lease_activity(lease).await; - } - self.schedule_access_lease_validity(lease).await; - } - touch - } - - pub async fn mark_pending_manifest_follow_up_for_lease( - &self, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - now_ms: u64, - target_duration: Option, - ) -> bool { - let deadline = hls_pending_manifest_follow_up_deadline(now_ms, target_duration); - let lease = self.access_leases.write().await.mark_pending_manifest_follow_up_for_lease( - lease_id, - proxy_session_id, - now_ms, - deadline, - ); - if let Some(lease) = lease { - self.schedule_access_lease_validity(&lease).await; - debug!( - "HLS pending manifest lease shortened after manifest response: lease={} proxy_session={}", - super::safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(proxy_session_id) - ); - true - } else { - false - } - } - - pub async fn mark_pending_manifest_follow_up_for_session( - &self, - proxy_session_id: &ProxySessionId, - now_ms: u64, - target_duration: Option, - ) -> usize { - let deadline = hls_pending_manifest_follow_up_deadline(now_ms, target_duration); - let leases = self.access_leases.write().await.mark_pending_manifest_follow_up_for_session( - proxy_session_id, - now_ms, - deadline, - ); - for lease in &leases { - self.schedule_access_lease_validity(lease).await; - } - leases.len() - } - - pub async fn touch_access_lease( - &self, - lease_id: &HlsAccessLeaseId, - now_ms: u64, - timing: HlsAccessLeaseTiming, - ) -> bool { - let lease = self.access_leases.write().await.touch_access_lease_snapshot(lease_id, now_ms, timing); - if let Some(lease) = lease { - self.schedule_access_lease_activity(&lease).await; - self.schedule_access_lease_validity(&lease).await; - true - } else { - false - } - } - - pub async fn active_access_lease_count_for_session(&self, proxy_session_id: &ProxySessionId, now_ms: u64) -> usize { - self.access_leases.write().await.active_access_lease_count_for_session(proxy_session_id, now_ms) - } - - pub async fn has_usable_access_lease_for_session(&self, proxy_session_id: &ProxySessionId, now_ms: u64) -> bool { - self.access_leases.write().await.has_usable_access_lease_for_session(proxy_session_id, now_ms) - } - - pub async fn access_lease_session_snapshot( - &self, - proxy_session_id: &ProxySessionId, - now_ms: u64, - ) -> HlsAccessLeaseSessionSnapshot { - self.access_leases.write().await.session_snapshot(proxy_session_id, now_ms) - } - - async fn access_lease_lifecycle_snapshot( - &self, - lease_id: &HlsAccessLeaseId, - now_ms: u64, - ) -> Option { - self.access_leases.write().await.lifecycle_snapshot(lease_id, now_ms) - } - - async fn remove_access_lease(&self, lease_id: &HlsAccessLeaseId) { - self.terminal_pending.cancel_lease(lease_id); - self.terminal_commit_retries.cancel_lease(lease_id); - let Some(preparation) = self.access_leases.read().await.prepare_access_lease_removal(lease_id) else { - return; - }; - if !self.remove_prepared_access_lease(lease_id, &preparation).await { - debug!( - "HLS access lease removal skipped: lease={} proxy_session={} reason=lease_instance_race", - super::safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(&preparation.proxy_session_id) - ); - } - } - - /// Removes one exact lease incarnation before releasing any media - /// protection owned by it. A stale preparation is intentionally - /// non-destructive because the replacement lease may reuse the same ID. - async fn remove_prepared_access_lease( - &self, - lease_id: &HlsAccessLeaseId, - preparation: &HlsAccessLeaseRemovalPreparation, - ) -> bool { - let removed = - self.access_leases.write().await.remove_access_lease_if_preparation_matches(lease_id, preparation); - if removed.is_none() { - return false; - } - if let Some(session) = self.sessions.get_by_proxy_session_id(&preparation.proxy_session_id).await { - let mut session = session.write().await; - let finalized_manifest_released = - session.transient.release_finalized_manifest_generations(lease_id, preparation.issued_at_ms); - if finalized_manifest_released { - debug!( - "HLS finalized transient manifest generation released: lease={} proxy_session={} reason=lease_removed", - super::safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(&preparation.proxy_session_id) - ); - } - if let Some(generation) = preparation.terminal_protection_generation { - let removal = session.remove_terminal_tail_protection_after_lease_end(lease_id, generation); - match removal { - HlsTerminalTailProtectionRemoval::Removed => { - debug!( - "HLS terminal-tail protection released: lease={} proxy_session={} generation={} reason=lease_removed", - super::safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(&preparation.proxy_session_id), - generation.0 - ); - } - HlsTerminalTailProtectionRemoval::Missing => {} - HlsTerminalTailProtectionRemoval::RemovedStaleGeneration { actual } => { - debug!( - "HLS stale terminal-tail protection released: lease={} proxy_session={} expected_generation={} actual_generation={} reason=lease_removed", - super::safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(&preparation.proxy_session_id), - generation.0, - actual.0 - ); - } - } - } else if session.remove_terminal_tail_protection(lease_id).is_some() { - debug!( - "HLS untracked terminal-tail protection released: lease={} proxy_session={} reason=lease_removed", - super::safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(&preparation.proxy_session_id) - ); - } - } - self.standalone_custom_access.remove(lease_id); - self.segment_repair.remove_access_lease_window(lease_id).await; - self.startup_observability.remove_access_lease(lease_id); - self.qos.remove_access_lease(lease_id).await; - true - } - - async fn cleanup_proxy_session_state( - &self, - proxy_session_id: &ProxySessionId, - reason: &'static str, - ) -> HlsProxySessionCleanupStats { - self.availability_reevaluations.cancel_session(proxy_session_id); - self.terminal_pending.cancel_session(proxy_session_id); - self.terminal_commit_retries.cancel_session(proxy_session_id); - let before = self.segment_repair.stats().await; - let removed_leases = self.access_leases.write().await.remove_access_leases_for_session(proxy_session_id); - let username = removed_leases.first().map(|lease| lease.username.clone()); - self.sessions.update_expired_session_marker_username(proxy_session_id, username).await; - let removed_lease_ids = removed_leases.iter().map(|lease| lease.lease_id.clone()).collect::>(); - for lease_id in &removed_lease_ids { - self.standalone_custom_access.remove(lease_id); - } - self.segment_repair.remove_proxy_session_state(proxy_session_id, &removed_lease_ids).await; - for lease_id in &removed_lease_ids { - self.startup_observability.remove_access_lease(lease_id); - } - let removed_qos = self.qos.remove_access_leases(&removed_lease_ids).await; - let removed_qos = removed_qos.saturating_add(self.qos.remove_proxy_session_state(proxy_session_id).await); - let after = self.segment_repair.stats().await; - let stats = HlsProxySessionCleanupStats { - access_leases: removed_lease_ids.len(), - repair_windows: before.windows.saturating_sub(after.windows), - repair_generations: before.generations.saturating_sub(after.generations), - repair_candidates: before.checked_candidates.saturating_sub(after.checked_candidates), - repair_object_metadata: before.object_metadata.saturating_sub(after.object_metadata), - repair_watchdog_metadata: before.watchdog_metadata.saturating_sub(after.watchdog_metadata), - repair_watchdog_locks: before.watchdog_locks.saturating_sub(after.watchdog_locks), - qos_access_leases: removed_qos, - }; - if stats.did_cleanup() { - debug!( - "HLS proxy session state cleaned: proxy_session={} reason={} access_leases={} repair_windows={} repair_generations={} repair_candidates={} repair_object_metadata={} repair_watchdog_metadata={} repair_watchdog_locks={} qos_access_leases={}", - safe_proxy_session_id(proxy_session_id), - reason, - stats.access_leases, - stats.repair_windows, - stats.repair_generations, - stats.repair_candidates, - stats.repair_object_metadata, - stats.repair_watchdog_metadata, - stats.repair_watchdog_locks, - stats.qos_access_leases - ); - } - stats - } - - pub async fn expired_session_marker( - &self, - proxy_session_id: &ProxySessionId, - now_ms: u64, - ) -> Option { - self.sessions - .expired_session_marker(proxy_session_id, now_ms, self.session_idle_timeout_ms().saturating_mul(2).max(1)) - .await - } - - async fn cleanup_all_runtime_state(&self, reason: &'static str) { - self.availability_reevaluations.clear(); - self.terminal_pending.clear(); - self.terminal_commit_retries.clear(); - let removed_access_leases = self.access_leases.write().await.clear(); - self.standalone_custom_access.clear(); - self.terminal_commit_retries.clear(); - for session in self.sessions.list_sessions().await { - session.write().await.clear_terminal_tail_protections(); - } - self.account_overlap_cooldowns.write().await.clear(); - let removed_qos = self.qos.clear().await; - let before = self.segment_repair.stats().await; - self.segment_repair.clear_runtime_state().await; - self.startup_observability.clear(); - if removed_access_leases > 0 - || before.windows > 0 - || before.generations > 0 - || before.checked_candidates > 0 - || before.metadata > 0 - || before.object_metadata > 0 - || before.locks > 0 - || before.watchdog_metadata > 0 - || before.watchdog_locks > 0 - || removed_qos > 0 - { - debug!( - "HLS runtime state cleaned: reason={} access_leases={} repair_windows={} repair_generations={} repair_candidates={} repair_metadata={} repair_object_metadata={} repair_locks={} repair_watchdog_metadata={} repair_watchdog_locks={} qos_access_leases={}", - reason, - removed_access_leases, - before.windows, - before.generations, - before.checked_candidates, - before.metadata, - before.object_metadata, - before.locks, - before.watchdog_metadata, - before.watchdog_locks, - removed_qos - ); - } - } - - async fn cleanup_after_garbage_collection(&self, report: &super::GarbageCollectionReport) { - if report.secret_cache_invalidated { - self.cleanup_all_runtime_state("secret-cache-invalidated").await; - return; - } - for proxy_session_id in &report.removed_session_ids { - self.cleanup_proxy_session_state(proxy_session_id, "gc-session-removed").await; - } - } - - async fn schedule_access_lease_activity(&self, lease: &HlsAccessLease) { - if let Some(active_until_ms) = lease.active_until_ms { - self.lifecycle - .schedule( - HlsLifecycleEventKey::AccessLeaseActive { - lease_id: lease.lease_id.clone(), - proxy_session_id: lease.proxy_session_id.clone(), - }, - active_until_ms, - ) - .await; - } - } - - async fn schedule_access_lease_validity(&self, lease: &HlsAccessLease) { - let due_at_ms = if lease.state == HlsAccessLeaseState::Pending { - lease.pending_deadline_ms().unwrap_or(lease.valid_until_ms) - } else { - lease.valid_until_ms - }; - self.lifecycle - .schedule( - HlsLifecycleEventKey::AccessLeaseValidity { - lease_id: lease.lease_id.clone(), - proxy_session_id: lease.proxy_session_id.clone(), - }, - due_at_ms, - ) - .await; - } - - async fn schedule_access_lease_lifecycle_snapshot(&self, snapshot: &HlsAccessLeaseLifecycleSnapshot) { - if snapshot.state == HlsAccessLeaseState::Activated { - if let Some(active_until_ms) = snapshot.active_until_ms { - self.lifecycle - .schedule( - HlsLifecycleEventKey::AccessLeaseActive { - lease_id: snapshot.lease_id.clone(), - proxy_session_id: snapshot.proxy_session_id.clone(), - }, - active_until_ms, - ) - .await; - } - } - if snapshot.state != HlsAccessLeaseState::Expired && snapshot.state != HlsAccessLeaseState::Denied { - let due_at_ms = if snapshot.state == HlsAccessLeaseState::Pending { - snapshot.pending_deadline.map_or(snapshot.valid_until_ms, HlsAccessLeasePendingDeadline::deadline_ms) - } else { - snapshot.valid_until_ms - }; - self.lifecycle - .schedule( - HlsLifecycleEventKey::AccessLeaseValidity { - lease_id: snapshot.lease_id.clone(), - proxy_session_id: snapshot.proxy_session_id.clone(), - }, - due_at_ms, - ) - .await; - } - } - - pub async fn schedule_session_idle_for_handle(&self, session: &HlsSessionHandle) { - let session_idle_timeout_ms = self.session_idle_timeout_ms(); - let (proxy_session_id, due_at_ms) = { - let session = session.read().await; - (session.proxy_session_id.clone(), session.idle_expiry_due_at_ms(session_idle_timeout_ms)) - }; - self.lifecycle.schedule(HlsLifecycleEventKey::SessionIdle { proxy_session_id }, due_at_ms).await; - } - - pub async fn mark_authorized_media_access_for_lease_if_identity_matches( - &self, - session: &HlsSessionHandle, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - lease_identity: HlsMediaLeaseIdentity, - now_ms: u64, - ) -> HlsMediaActivityCommitOutcome { - self.commit_media_activity_if_identity_matches( - session, - lease_id, - proxy_session_id, - lease_identity, - now_ms, - HlsMediaActivityCommitKind::Access, - ) - .await - } - - pub async fn mark_delivered_media_for_lease( - &self, - session: &HlsSessionHandle, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - lease_identity: HlsMediaLeaseIdentity, - now_ms: u64, - ) -> HlsMediaActivityCommitOutcome { - self.commit_media_activity_if_identity_matches( - session, - lease_id, - proxy_session_id, - lease_identity, - now_ms, - HlsMediaActivityCommitKind::Delivered, - ) - .await - } - - async fn commit_media_activity_if_identity_matches( - &self, - session: &HlsSessionHandle, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - lease_identity: HlsMediaLeaseIdentity, - now_ms: u64, - kind: HlsMediaActivityCommitKind, - ) -> HlsMediaActivityCommitOutcome { - let Some(current_session) = self.sessions.get_by_proxy_session_id(proxy_session_id).await else { - return HlsMediaActivityCommitOutcome::StaleLeaseIdentity; - }; - if !Arc::ptr_eq(¤t_session, session) { - return HlsMediaActivityCommitOutcome::StaleLeaseIdentity; - } - for attempt in 0..HLS_STATE_CAS_LOCK_RETRIES { - match self.try_commit_media_activity(session, lease_id, proxy_session_id, lease_identity, now_ms, kind) { - HlsMediaActivityCommitAttempt::Completed { outcome, evidence_changed } => { - return self - .finish_media_activity_commit(session, proxy_session_id, outcome, evidence_changed) - .await; - } - HlsMediaActivityCommitAttempt::LockBusy => { - if attempt.saturating_add(1) < HLS_STATE_CAS_LOCK_RETRIES { - tokio::task::yield_now().await; - } - } - } - } - - // Media activity must not be dropped under sustained lock contention. - // Wait for one contended lock at a time, then retry the established - // lease-store -> session acquisition without awaiting under either - // write guard. - let mut committed = None; - for attempt in 0..HLS_MEDIA_ACTIVITY_FALLBACK_LOCK_RETRIES { - let mut leases = self.access_leases.write().await; - let Ok(mut session_guard) = session.try_write() else { - drop(leases); - let session_wait = session.write().await; - drop(session_wait); - if attempt.saturating_add(1) < HLS_MEDIA_ACTIVITY_FALLBACK_LOCK_RETRIES { - tokio::task::yield_now().await; - } - continue; - }; - let HlsMediaActivityCommitAttempt::Completed { outcome, evidence_changed } = - Self::commit_media_activity_locked( - &mut leases, - &mut session_guard, - lease_id, - proxy_session_id, - lease_identity, - now_ms, - kind, - ) - else { - drop(session_guard); - drop(leases); - continue; - }; - drop(session_guard); - drop(leases); - committed = Some((outcome, evidence_changed)); - break; - } - let Some((outcome, evidence_changed)) = committed else { - return HlsMediaActivityCommitOutcome::DeferredLockContention; - }; - self.finish_media_activity_commit(session, proxy_session_id, outcome, evidence_changed).await - } - - async fn finish_media_activity_commit( - &self, - session: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - outcome: HlsMediaActivityCommitOutcome, - evidence_changed: bool, - ) -> HlsMediaActivityCommitOutcome { - match outcome { - HlsMediaActivityCommitOutcome::Committed => { - if evidence_changed { - self.segment_cache.notify_capacity_protection_changed(); - self.notify_session_evidence_changed(proxy_session_id); - } - self.schedule_session_idle_for_handle(session).await; - } - HlsMediaActivityCommitOutcome::StaleLeaseIdentity - | HlsMediaActivityCommitOutcome::DeferredLockContention => {} - } - outcome - } - - fn try_commit_media_activity( - &self, - session: &HlsSessionHandle, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - lease_identity: HlsMediaLeaseIdentity, - now_ms: u64, - kind: HlsMediaActivityCommitKind, - ) -> HlsMediaActivityCommitAttempt { - // Media-completion lock order matches terminal publication: lease - // store -> session. Both locks are non-blocking and no async work runs - // while either guard is held. - let Ok(mut leases) = self.access_leases.try_write() else { - return HlsMediaActivityCommitAttempt::LockBusy; - }; - let Ok(mut session) = session.try_write() else { - return HlsMediaActivityCommitAttempt::LockBusy; - }; - let result = Self::commit_media_activity_locked( - &mut leases, - &mut session, - lease_id, - proxy_session_id, - lease_identity, - now_ms, - kind, - ); - drop(session); - drop(leases); - result - } - - fn commit_media_activity_locked( - leases: &mut HlsAccessLeaseStore, - session: &mut super::HlsSession, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - lease_identity: HlsMediaLeaseIdentity, - now_ms: u64, - kind: HlsMediaActivityCommitKind, - ) -> HlsMediaActivityCommitAttempt { - if session.proxy_session_id != *proxy_session_id { - return HlsMediaActivityCommitAttempt::Completed { - outcome: HlsMediaActivityCommitOutcome::StaleLeaseIdentity, - evidence_changed: false, - }; - } - let (current, capacity_protection_released) = match kind { - HlsMediaActivityCommitKind::Access | HlsMediaActivityCommitKind::Delivered => { - (leases.media_identity_is_current(lease_id, proxy_session_id, lease_identity, now_ms), false) - } - HlsMediaActivityCommitKind::LiveSegmentCompletion(token) => { - let completion = leases.record_segment_request_completed_if_identity_matches( - lease_id, - proxy_session_id, - lease_identity, - token, - now_ms, - ); - (completion.is_some(), matches!(completion, Some(super::HlsPlaybackCompletionOutcome::Advanced))) - } - }; - if !current { - return HlsMediaActivityCommitAttempt::Completed { - outcome: HlsMediaActivityCommitOutcome::StaleLeaseIdentity, - evidence_changed: false, - }; - } - session.mark_authorized_media_access(now_ms); - if lease_identity.is_live() && matches!(kind, HlsMediaActivityCommitKind::Delivered) { - session.activity.last_delivered_media_at_ms = Some(now_ms); - } - HlsMediaActivityCommitAttempt::Completed { - outcome: HlsMediaActivityCommitOutcome::Committed, - evidence_changed: capacity_protection_released, - } - } - - pub async fn handle_lifecycle_event( - &self, - active_users: &Arc, - active_provider: &Arc, - event: HlsLifecycleEvent, - now_ms: u64, - ) { - if !self.is_enabled() { - return; - } - match event.key { - HlsLifecycleEventKey::AccessLeaseActive { lease_id, proxy_session_id } - | HlsLifecycleEventKey::AccessLeaseValidity { lease_id, proxy_session_id } => { - let mut should_sync_session = false; - if let Some(snapshot) = self.access_lease_lifecycle_snapshot(&lease_id, now_ms).await { - should_sync_session = true; - if let Some(release) = &snapshot.idle_release { - active_users - .release_session_streams_and_counted_reservation( - &release.username, - &release.user_session_token, - ) - .await; - debug!( - "HLS access lease idled: lease={} proxy_session={} user_session={}", - super::safe_hls_access_lease_id(&release.lease_id), - safe_proxy_session_id(&snapshot.proxy_session_id), - super::safe_user_session_token(&release.user_session_token) - ); - } - if matches!(snapshot.state, HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied) { - self.remove_access_lease(&snapshot.lease_id).await; - debug!( - "HLS access lease removed: lease={} proxy_session={} state={}", - super::safe_hls_access_lease_id(&snapshot.lease_id), - safe_proxy_session_id(&snapshot.proxy_session_id), - snapshot.state.as_log_value() - ); - debug!( - "HLS lifecycle state snapshot: trigger=access-lease-removed {}", - self.debug_state_summary().await - ); - } else { - self.schedule_access_lease_lifecycle_snapshot(&snapshot).await; - } - } - if should_sync_session { - if let Some(session) = self.sessions.get_by_proxy_session_id(&proxy_session_id).await { - self.sync_session_access_lease_count_and_detach_if_needed( - active_users, - active_provider, - &session, - &proxy_session_id, - now_ms, - ) - .await; - } - } - } - HlsLifecycleEventKey::SessionIdle { proxy_session_id } => { - self.handle_session_idle_lifecycle_event(&proxy_session_id, now_ms).await; - } - } - } - - async fn handle_session_idle_lifecycle_event(&self, proxy_session_id: &ProxySessionId, now_ms: u64) { - let Some(session) = self.sessions.get_by_proxy_session_id(proxy_session_id).await else { - return; - }; - let session_idle_timeout_ms = self.session_idle_timeout_ms(); - let (key, due_at_ms, can_remove) = { - let session = session.read().await; - ( - session.key.clone(), - session.idle_expiry_due_at_ms(session_idle_timeout_ms), - session.can_expire_idle_session(now_ms, session_idle_timeout_ms), - ) - }; - if !can_remove { - self.lifecycle - .schedule( - HlsLifecycleEventKey::SessionIdle { proxy_session_id: proxy_session_id.clone() }, - hls_session_idle_protection_retry_at(due_at_ms, now_ms), - ) - .await; - return; - } - if self.segment_cache.has_active_temp_files_for_session(proxy_session_id) { - self.lifecycle - .schedule( - HlsLifecycleEventKey::SessionIdle { proxy_session_id: proxy_session_id.clone() }, - now_ms.saturating_add(1_000), - ) - .await; - return; - } - let username = self.access_leases.read().await.first_username_for_session(proxy_session_id); - if self - .sessions - .remove_session_marking_expired( - &key, - proxy_session_id, - now_ms, - HlsExpiredSessionReason::SessionIdleTimeout, - username, - ) - .await - .is_some() - { - self.cleanup_proxy_session_state(proxy_session_id, "lifecycle-session-expired").await; - if let Err(err) = self.segment_cache.delete_session_dir(proxy_session_id).await { - error!( - "HLS session lifecycle cleanup failed: proxy_session={} error={err}", - safe_proxy_session_id(proxy_session_id) - ); - } else { - debug!("HLS session lifecycle expired: proxy_session={}", safe_proxy_session_id(proxy_session_id)); - debug!("HLS lifecycle state snapshot: trigger=session-expired {}", self.debug_state_summary().await); - } - } - } - - pub async fn debug_state_summary(&self) -> String { - let sessions = self.sessions.list_sessions().await; - let access_leases = self.access_leases.read().await.len(); - let qos_access_leases = self.qos.len().await; - let repair = self.segment_repair.stats().await; - let mut segments = 0_usize; - let mut maps = 0_usize; - let mut transient_resources = 0_usize; - let mut transient_objects = 0_usize; - let mut active_origin_work = 0_usize; - let mut active_segment_fetches = 0_usize; - let mut active_map_fetches = 0_usize; - for session in &sessions { - let session = session.read().await; - segments = segments.saturating_add(session.segments.len()); - maps = maps.saturating_add(session.maps.len()); - transient_resources = transient_resources.saturating_add(session.transient.resources.len()); - transient_objects = transient_objects.saturating_add(session.transient.object_cache.len()); - active_origin_work = active_origin_work - .saturating_add(session.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire)); - active_segment_fetches = active_segment_fetches.saturating_add(session.active_segment_fetches); - active_map_fetches = active_map_fetches.saturating_add(session.active_map_fetches); - } - format!( - "sessions={} access_leases={} qos_access_leases={} repair_windows={} repair_generations={} repair_candidates={} repair_metadata={} repair_object_metadata={} repair_locks={} repair_watchdog_metadata={} repair_watchdog_locks={} segments={} maps={} transient_resources={} transient_objects={} active_origin_work={} active_segment_fetches={} active_map_fetches={}", - sessions.len(), - access_leases, - qos_access_leases, - repair.windows, - repair.generations, - repair.checked_candidates, - repair.metadata, - repair.object_metadata, - repair.locks, - repair.watchdog_metadata, - repair.watchdog_locks, - segments, - maps, - transient_resources, - transient_objects, - active_origin_work, - active_segment_fetches, - active_map_fetches - ) - } - - pub async fn run_garbage_collection_once(&self, now_ms: u64) -> io::Result { - if !self.is_enabled() { - return Ok(super::GarbageCollectionReport::default()); - } - let report = self.gc.run_once(now_ms).await?; - self.cleanup_after_garbage_collection(&report).await; - Ok(report) - } - - pub async fn sync_session_access_lease_count_and_detach_if_needed( - &self, - active_users: &Arc, - _active_provider: &Arc, - session: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - now_ms: u64, - ) { - let snapshot = self.reconcile_session_access_lease_snapshot(session, proxy_session_id, now_ms).await; - for release in &snapshot.idle_releases { - active_users - .release_session_streams_and_counted_reservation(&release.username, &release.user_session_token) - .await; - debug!( - "HLS access lease idled: lease={} proxy_session={} user_session={}", - super::safe_hls_access_lease_id(&release.lease_id), - safe_proxy_session_id(proxy_session_id), - super::safe_user_session_token(&release.user_session_token) - ); - } - } - - async fn reconcile_session_access_lease_snapshot( - &self, - session: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - now_ms: u64, - ) -> HlsAccessLeaseSessionSnapshot { - // Keep snapshot creation and application in the established lease-store -> session - // lock transaction. Otherwise a publication/removal can be overwritten by a stale - // wholesale binding reconciliation. - let mut leases = self.access_leases.write().await; - let snapshot = leases.session_snapshot(proxy_session_id, now_ms); - let mut session = session.write().await; - session.activity.active_access_lease_count = snapshot.active_count; - session.reconcile_effective_origin_acquire_policy(snapshot.effective_origin_policy, now_ms); - session.transient.reconcile_finalized_manifest_lease_bindings(&snapshot.finalized_transient_manifest_bindings); - snapshot - } - - pub async fn sync_all_session_access_leases_and_detach_if_needed( - &self, - active_users: &Arc, - active_provider: &Arc, - now_ms: u64, - ) { - if !self.is_enabled() { - return; - } - for session in self.sessions.list_sessions().await { - let proxy_session_id = session.read().await.proxy_session_id.clone(); - self.sync_session_access_lease_count_and_detach_if_needed( - active_users, - active_provider, - &session, - &proxy_session_id, - now_ms, - ) - .await; - } - } - - pub async fn deny_access_lease( - &self, - lease_id: &HlsAccessLeaseId, - mode: HlsAccessLeaseDenialMode, - ) -> HlsAccessLeaseDenialOutcome { - let outcome = { - let mut access_leases = self.access_leases.write().await; - access_leases.deny_access_lease(lease_id, mode) - }; - let HlsAccessLeaseDenialOutcome::Ended { terminal_release } = &outcome else { - return outcome; - }; - self.terminal_pending.cancel_lease(lease_id); - self.terminal_commit_retries.cancel_lease(lease_id); - let Some(terminal_release) = terminal_release else { - return outcome; - }; - let removal = if let Some(session) = - self.sessions.get_by_proxy_session_id(&terminal_release.proxy_session_id).await - { - session.write().await.remove_terminal_tail_protection_after_lease_end(lease_id, terminal_release.generation) - } else { - HlsTerminalTailProtectionRemoval::Missing - }; - match removal { - HlsTerminalTailProtectionRemoval::Removed => { - debug!( - "HLS terminal-tail protection released: lease={} proxy_session={} generation={} reason=lease_denied", - super::safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(&terminal_release.proxy_session_id), - terminal_release.generation.0 - ); - } - HlsTerminalTailProtectionRemoval::Missing => {} - HlsTerminalTailProtectionRemoval::RemovedStaleGeneration { actual } => { - debug!( - "HLS stale terminal-tail protection released: lease={} proxy_session={} expected_generation={} actual_generation={} reason=lease_denied", - super::safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(&terminal_release.proxy_session_id), - terminal_release.generation.0, - actual.0 - ); - } - } - let acknowledged = self.access_leases.write().await.acknowledge_terminal_protection_release( - lease_id, - &terminal_release.proxy_session_id, - terminal_release.generation, - ); - if !acknowledged { - debug!( - "HLS terminal-tail protection release acknowledgement skipped: lease={} proxy_session={} generation={} reason=lease_generation_race", - super::safe_hls_access_lease_id(lease_id), - safe_proxy_session_id(&terminal_release.proxy_session_id), - terminal_release.generation.0 - ); - } - outcome - } - - pub async fn get_or_create_session( - &self, - key: HlsSessionKey, - reverse_proxy_rewrite_secret: &[u8], - now_ms: u64, - ) -> HlsSessionHandle { - self.get_or_create_session_with_outcome(key, reverse_proxy_rewrite_secret, now_ms).await.0 - } - - pub async fn get_or_create_session_with_outcome( - &self, - key: HlsSessionKey, - reverse_proxy_rewrite_secret: &[u8], - now_ms: u64, - ) -> (HlsSessionHandle, HlsSessionStoreOutcome) { - let origin_source = HlsOriginSource::from_session_key(&key); - self.get_or_create_session_with_source_and_outcome(key, origin_source, reverse_proxy_rewrite_secret, now_ms) - .await - } - - pub async fn get_or_create_session_with_source_and_outcome( - &self, - key: HlsSessionKey, - origin_source: HlsOriginSource, - reverse_proxy_rewrite_secret: &[u8], - now_ms: u64, - ) -> (HlsSessionHandle, HlsSessionStoreOutcome) { - let (session, outcome) = self - .sessions - .get_or_create_session_with_source_and_outcome(key, origin_source, reverse_proxy_rewrite_secret, now_ms) - .await; - let (proxy_session_id, session_key) = { - let session_guard = session.read().await; - (safe_proxy_session_id(&session_guard.proxy_session_id), safe_session_key(&session_guard.key)) - }; - match outcome { - HlsSessionStoreOutcome::Created => { - self.metrics.record_session_created(); - info!("HLS session created: session={session_key} proxy_session={proxy_session_id}"); - } - HlsSessionStoreOutcome::Reused => { - self.metrics.record_session_reused(); - debug!("HLS session reused: session={session_key} proxy_session={proxy_session_id}"); - } - } - self.schedule_session_idle_for_handle(&session).await; - session.write().await.configure_segment_prefetch_queue(self.segment_fetch_policy().max_prefetch_queue_depth); - (session, outcome) - } -} - -impl Default for HlsProxyManager { - fn default() -> Self { Self::new() } -} - -pub fn exec_hls_lifecycle(ctx: &HlsCtx, cancel_token: &CancellationToken) { - let hls_proxy = Arc::clone(&ctx.hls_proxy); - let active_users = Arc::clone(&ctx.active_users); - let active_provider = Arc::clone(&ctx.active_provider); - let cancel_token = cancel_token.clone(); - tokio::spawn(async move { - while let Some(event) = hls_proxy.lifecycle().next_event(&cancel_token).await { - let now_ms = current_time_millis(); - hls_proxy.handle_lifecycle_event(&active_users, &active_provider, event, now_ms).await; - } - }); -} - -use tuliprox_core::utils::current_time_millis; - #[cfg(test)] -mod tests { - use super::{ - super::{ - lease::{ - HlsAccessLeaseDenialMode, HlsAccessLeaseDenialOutcome, HlsLeaseManifestPublicationOutcome, - HlsLeaseManifestPublicationRejectReason, HlsTerminalMediaRequirementSource, - }, - manifest_acceptance::HlsManifestAcceptanceGeneration, - prepared_terminal_bundle::prepared_terminal_bundle_key, - recovery_timing::{ - HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, HlsLeaseCutoverTiming, - HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsRecoveryEtaMs, HlsRecoveryTimingPolicy, - HlsRecoveryWorkload, HlsTerminalCommitAcquisitionBudgetMs, HlsTerminalCommitWindow, - HlsTerminalMediaPreparationState, HlsTransitionMarginMs, - }, - runtime_custom_tail::HlsRuntimeCustomTailAssetIdentity, - session::{HlsTerminalTailProtection, HLS_TERMINAL_TAIL_PROTECTION_CAPACITY}, - terminal_commit::{ - HlsTerminalAssetRevisionGuard, HlsTerminalCommitCommand, HlsTerminalCommitOutcome, - HlsTerminalCommitOwnerKey, HlsTerminalCommitRetryDecision, HlsTerminalCommitRetryScheduleDecision, - HlsTerminalCommitSubmissionDecision, HlsTerminalLeaseDecision, - }, - terminal_tail::{snapshot_terminal_media_asset, HlsTerminalCommitMediaGuard, HlsTerminalTailPlan}, - }, - hls_acceptance_recovery_snapshot, hls_key_readiness_evidence_is_current, hls_session_idle_protection_retry_at, - next_terminal_commit_retry, spawn_terminal_commit_retry_worker, HlsCriticalHandoffStateAccess, - HlsMediaActivityCommitOutcome, HlsProxyManager, HlsRecoveryExecutionState, HlsTerminalCommitPayload, - HlsTerminalCommitRequest, HlsTerminalTailPreparationRequest, HLS_SESSION_IDLE_PROTECTION_RETRY_MS, - }; - use crate::{ - build_terminal_tail_plan, - media_reserve::{HlsLeaseReserveAvailabilityBasis, HlsLeaseReserveSnapshot, HlsManifestCommitIdentity}, - terminal_tail::HlsLeasePlaybackMode, - HlsAccessLease, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseState, HlsLeaseManifestSegment, - HlsLeaseManifestSnapshot, HlsManifestAcceptanceExhaustionReason, HlsManifestAcceptanceTrigger, - HlsManifestDeliveryMode, HlsMediaContainer, HlsMediaLeaseIdentity, HlsPlaybackFamilyKey, - HlsPublishedTransientResourceIds, HlsSegmentFailureObject, HlsSessionHandle, HlsSessionKey, - HlsSessionStoreOutcome, HlsTerminalAssetIdentity, HlsTerminalBaseMediaState, HlsTerminalBaseProtection, - HlsTerminalBaseSegmentAvailability, HlsTerminalTailBuildInput, HlsTerminalTailCompatibility, - HlsTerminalTailGeneration, ProxySessionId, TransientManifestGeneration, TransientResourceKind, - TransientResourceRef, HLS_TERMINAL_TAIL_SEGMENT_COUNT, - }; - use arc_swap::{ArcSwap, ArcSwapOption}; - use shared::model::{ - ConfigPaths, HlsCacheConfigDto, HlsManifestRecoveryBurstLevel, HlsStripConfigDto, HlsStripMode, - ReverseProxyConfigDto, - }; - use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }; - use tuliprox_core::{ - model::{AppConfig, Config, HlsCacheConfig, MediaToolCapabilities, ReverseProxyConfig, SourcesConfig}, - utils::FileLockManager, - }; - use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer; - - const TERMINAL_ASSET_BYTES: &[u8] = - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); - - #[test] - fn protected_idle_session_retry_cannot_form_a_millisecond_busy_loop() { - let now_ms = 50_000_u64; - assert_eq!( - hls_session_idle_protection_retry_at(now_ms.saturating_sub(1), now_ms), - now_ms.saturating_add(HLS_SESSION_IDLE_PROTECTION_RETRY_MS) - ); - assert_eq!(hls_session_idle_protection_retry_at(60_000, now_ms), 60_000); - } - - #[test] - fn key_readiness_evidence_is_valid_at_expiry_and_stale_one_millisecond_later() { - assert!(hls_key_readiness_evidence_is_current(Some(50_000), 50_000)); - assert!(!hls_key_readiness_evidence_is_current(Some(50_000), 50_001)); - assert!(hls_key_readiness_evidence_is_current(None, u64::MAX)); - } - - fn cutover_reserve() -> HlsLeaseReserveSnapshot { - let transition_margin = HlsTransitionMarginMs::from_millis(12_000); - let guaranteed_reserve_ms = transition_margin - .as_millis() - .saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()); - HlsLeaseReserveSnapshot { - availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, - guaranteed_media_horizon_ms: 12_000_u64.saturating_add(guaranteed_reserve_ms), - conservative_playback_position_ms: 12_000, - guaranteed_reserve_ms, - initial_hidden_ready_duration_ms: 0, - transition_margin, - key_readiness_valid_until_ms: None, - recovery_required: true, - cutover_required: false, - } - } - - fn empty_paths() -> ConfigPaths { - ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - } - } - - fn test_app_config(config: Config) -> AppConfig { - AppConfig { - config: Arc::new(ArcSwap::from_pointee(config)), - sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), - hdhomerun: Arc::new(ArcSwapOption::default()), - api_proxy: Arc::new(ArcSwapOption::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(empty_paths())), - custom_stream_response: Arc::new(ArcSwapOption::default()), - access_token_secret: [0; 32], - encrypt_secret: [7; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - } - } - - fn config_with_hls_cache(hls_cache: HlsCacheConfigDto) -> Config { - Config { - reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { - hls_cache: Some(hls_cache), - ..ReverseProxyConfigDto::default() - })), - ..Config::default() - } - } - - fn access_lease(lease_id: &str, proxy_session_id: &ProxySessionId) -> HlsAccessLease { - HlsAccessLease::pending( - HlsAccessLeaseId(lease_id.to_string()), - HlsPlaybackFamilyKey::new("alice", "client-a"), - proxy_session_id.clone(), - "alice".to_string(), - "session-a".to_string(), - 1, - "stream-a".to_string(), - 12345, - 1_000, - 60_000, - ) - } - - fn manifest_snapshot(source_rendered_at_ms: u64) -> HlsLeaseManifestSnapshot { - HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(source_rendered_at_ms), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 0, - delivered_at_ms: 2_000, - first_proxy_seq: 40, - last_proxy_seq: 41, - visible_segments: Arc::from([ - HlsLeaseManifestSegment { - proxy_seq: 40, - duration_ms: 6_000, - uri: "/live/40.ts".to_string().into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }, - HlsLeaseManifestSegment { - proxy_seq: 41, - duration_ms: 6_000, - uri: "/live/41.ts".to_string().into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }, - ]), - discontinuity_sequence: 3, - target_duration_ms: 12_000, - playlist_duration_ms: 12_000, - last_visible_media_end_ms: 12_000, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - } - } - - fn finalized_transient_manifest_snapshot( - source_rendered_at_ms: u64, - manifest_generation: TransientManifestGeneration, - resource_uri: &str, - ) -> HlsLeaseManifestSnapshot { - let mut snapshot = manifest_snapshot(source_rendered_at_ms); - snapshot.delivery_mode = HlsManifestDeliveryMode::TransientPassthrough; - snapshot.finalized_transient_manifest_generation = Some(manifest_generation); - for segment in Arc::make_mut(&mut snapshot.visible_segments) { - segment.uri = Arc::from(resource_uri); - } - snapshot - } - - async fn publish_manifest_snapshot( - manager: &HlsProxyManager, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - snapshot: HlsLeaseManifestSnapshot, - now_ms: u64, - ) -> bool { - let Some(guard) = manager.prepare_access_lease_manifest_publication(lease_id, proxy_session_id, now_ms).await - else { - return false; - }; - manager - .commit_access_lease_manifest_publication(lease_id, proxy_session_id, guard, snapshot, now_ms) - .await - .is_committed() - } - - async fn wait_until_access_lease_store_is_write_locked(manager: &HlsProxyManager) { - for _ in 0..1_000 { - if manager.access_leases.try_write().is_err() { - return; - } - tokio::task::yield_now().await; - } - panic!("access lease store was not locked by the reconciliation transaction"); - } - - fn terminal_plan( - generation: u64, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, - ) -> Arc { - let mut base_manifest = manifest_snapshot(1); - base_manifest.snapshot_generation = 1; - for segment in Arc::make_mut(&mut base_manifest.visible_segments) { - segment.uri = - format!("/hls/shared/live/{}/{}/{}.ts", proxy_session_id.0, lease_id.0, segment.proxy_seq).into(); - } - let transport_stream = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); - let asset = - super::super::terminal_tail::snapshot_terminal_media_asset(&transport_stream).expect("terminal asset"); - let expected_asset = - HlsRuntimeCustomTailAssetIdentity::channel_unavailable(HlsTerminalAssetIdentity::from_asset(&asset)); - let availability = Arc::from( - base_manifest - .visible_proxy_seqs() - .map(|proxy_seq| HlsTerminalBaseSegmentAvailability { - proxy_seq, - media_state: HlsTerminalBaseMediaState::Ready, - required_map_ready: true, - required_key_ready: true, - protection: HlsTerminalBaseProtection::Protectable, - }) - .collect::>(), - ); - let base_track_signature = Some(asset.track_signature().clone()); - let anchored_bundle = - HlsTerminalTailBuildInput::anchored_bundle_for_test(&asset, base_manifest.target_duration_ms); - let base_timing = Some(HlsTerminalTailBuildInput::base_timing_for_test(&asset, &base_manifest)); - let base_splice_evidence = Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); - let terminal_splice_evidence = base_splice_evidence.clone(); - Arc::new( - build_terminal_tail_plan(HlsTerminalTailBuildInput { - generation: HlsTerminalTailGeneration(generation), - created_at_ms: 3_000, - base_manifest, - base_availability: availability, - base_track_signature, - base_splice_evidence, - terminal_splice_evidence, - base_timing, - base_key_bindings: Arc::from([]), - expected_asset, - asset, - anchored_bundle, - }) - .expect("compatible terminal plan"), - ) - } - - fn commit_terminal_plan( - manager: &HlsProxyManager, - session: &HlsSessionHandle, - lease_id: &HlsAccessLeaseId, - proxy_session_id: &ProxySessionId, - preparation: &super::super::lease::HlsTerminalTailPreparation, - now_ms: u64, - plan: Arc, - ) -> HlsTerminalCommitOutcome { - let asset_revision_guard = HlsTerminalAssetRevisionGuard::matching_runtime_for_test(plan.asset_identity); - manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { - session, - lease_id, - proxy_session_id, - preparation, - now_ms, - payload: HlsTerminalCommitPayload::Tail { - plan, - media_guard: HlsTerminalCommitMediaGuard::empty_for_test(), - }, - asset_revision_guard, - }) - } - - async fn live_media_fixture( - lease_name: &str, - ) -> (HlsProxyManager, HlsSessionHandle, ProxySessionId, HlsAccessLeaseId, HlsMediaLeaseIdentity) { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId(lease_name.to_string()); - let lease = access_lease(lease_name, &proxy_session_id); - let lease_identity = lease.media_identity().expect("live media identity"); - manager.access_leases.write().await.prepare_access_lease(lease); - (manager, session, proxy_session_id, lease_id, lease_identity) - } - - #[tokio::test] - async fn critical_handoff_distinguishes_lock_busy_from_acquired_state() { - let (manager, session, _, _, _) = live_media_fixture("critical-lock-busy").await; - let lease_guard = manager.access_leases.write().await; - - let busy = manager.with_critical_handoff_state(&session, |_, _| 7_u8).await; - - assert_eq!(busy, HlsCriticalHandoffStateAccess::LockBusy); - drop(lease_guard); - - let acquired = manager.with_critical_handoff_state(&session, |_, _| 7_u8).await; - - assert_eq!(acquired, HlsCriticalHandoffStateAccess::Acquired(7)); - } - - #[tokio::test] - async fn late_live_completion_after_terminal_transition_updates_neither_cursor_nor_session_activity() { - let (manager, session, proxy_session_id, lease_id, live_identity) = live_media_fixture("late-terminal").await; - let token = manager - .record_access_lease_segment_request_started_if_identity_matches( - &lease_id, - &proxy_session_id, - live_identity, - 40, - 2_000, - ) - .await - .expect("current live request token"); - let plan = terminal_plan(7, &proxy_session_id, &lease_id); - { - let mut leases = manager.access_leases.write().await; - let mut lease = leases.remove_access_lease(&lease_id).expect("live lease"); - lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(plan); - leases.prepare_access_lease(lease); - } - - let outcome = manager - .record_access_lease_segment_request_completed_and_mark_media_if_identity_matches( - &session, - &lease_id, - &proxy_session_id, - live_identity, - token, - 2_100, - ) - .await; - - assert_eq!(outcome, HlsMediaActivityCommitOutcome::StaleLeaseIdentity); - assert_eq!(session.read().await.activity.last_authorized_media_at_ms, None); - let lease = - manager.access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_100).await.expect("terminal lease"); - assert_eq!(lease.playback_cursor.highest_contiguous_completed_proxy_seq, None); - assert_eq!(lease.playback_cursor.first_segment_completed_at_ms, None); - } - - #[tokio::test] - async fn contiguous_live_completion_wakes_capacity_protection_waiters() { - let (manager, session, proxy_session_id, lease_id, live_identity) = - live_media_fixture("capacity-release").await; - let revision = manager.segment_cache.capacity_revision(); - let mut capacity_wait = Box::pin(manager.segment_cache.wait_for_capacity_change(&revision)); - assert!(matches!(futures::poll!(capacity_wait.as_mut()), std::task::Poll::Pending)); - let token = manager - .record_access_lease_segment_request_started_if_identity_matches( - &lease_id, - &proxy_session_id, - live_identity, - 40, - 2_000, - ) - .await - .expect("current live request token"); - - let outcome = manager - .record_access_lease_segment_request_completed_and_mark_media_if_identity_matches( - &session, - &lease_id, - &proxy_session_id, - live_identity, - token, - 2_100, - ) - .await; - - assert_eq!(outcome, HlsMediaActivityCommitOutcome::Committed); - assert!(matches!(futures::poll!(capacity_wait.as_mut()), std::task::Poll::Ready(()))); - } - - #[tokio::test] - async fn expired_and_denied_lease_identities_cannot_extend_shared_session_activity() { - let (expired_manager, expired_session, expired_proxy, expired_lease_id, expired_identity) = - live_media_fixture("expired-media").await; - { - let mut leases = expired_manager.access_leases.write().await; - let mut lease = leases.remove_access_lease(&expired_lease_id).expect("expiring lease"); - lease.pending_deadline = Some(HlsAccessLeasePendingDeadline::Bootstrap { deadline_ms: 2_000 }); - lease.valid_until_ms = 2_000; - leases.prepare_access_lease(lease); - } - assert_eq!( - expired_manager - .mark_authorized_media_access_for_lease_if_identity_matches( - &expired_session, - &expired_lease_id, - &expired_proxy, - expired_identity, - 2_000, - ) - .await, - HlsMediaActivityCommitOutcome::StaleLeaseIdentity - ); - assert_eq!(expired_session.read().await.activity.last_authorized_media_at_ms, None); - - let (denied_manager, denied_session, denied_proxy, denied_lease_id, denied_identity) = - live_media_fixture("denied-media").await; - assert!(matches!( - denied_manager - .access_leases - .write() - .await - .deny_access_lease(&denied_lease_id, HlsAccessLeaseDenialMode::ImmediateEnd), - HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } - )); - assert_eq!( - denied_manager - .mark_authorized_media_access_for_lease_if_identity_matches( - &denied_session, - &denied_lease_id, - &denied_proxy, - denied_identity, - 2_000, - ) - .await, - HlsMediaActivityCommitOutcome::StaleLeaseIdentity - ); - assert_eq!(denied_session.read().await.activity.last_authorized_media_at_ms, None); - } - - #[tokio::test] - async fn current_terminal_identity_waits_for_lock_contention_and_marks_activity() { - let (manager, session, proxy_session_id, lease_id, _) = live_media_fixture("terminal-media").await; - let manager = Arc::new(manager); - let terminal_identity = { - let mut leases = manager.access_leases.write().await; - let mut lease = leases.remove_access_lease(&lease_id).expect("live lease"); - lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(terminal_plan(7, &proxy_session_id, &lease_id)); - let identity = lease.media_identity().expect("terminal identity"); - leases.prepare_access_lease(lease); - identity - }; - assert_eq!( - manager - .mark_authorized_media_access_for_lease_if_identity_matches( - &session, - &lease_id, - &proxy_session_id, - terminal_identity, - 2_000, - ) - .await, - HlsMediaActivityCommitOutcome::Committed - ); - assert_eq!(session.read().await.activity.last_authorized_media_at_ms, Some(2_000)); - - session.write().await.activity.last_authorized_media_at_ms = None; - let lease_guard = manager.access_leases.write().await; - let task_manager = Arc::clone(&manager); - let task_session = Arc::clone(&session); - let task_lease_id = lease_id.clone(); - let task_proxy_session_id = proxy_session_id.clone(); - let commit_task = tokio::spawn(async move { - task_manager - .mark_authorized_media_access_for_lease_if_identity_matches( - &task_session, - &task_lease_id, - &task_proxy_session_id, - terminal_identity, - 2_100, - ) - .await - }); - tokio::task::yield_now().await; - assert!(!commit_task.is_finished(), "current media activity must wait rather than be dropped"); - drop(lease_guard); - assert_eq!(commit_task.await.expect("controlled commit task"), HlsMediaActivityCommitOutcome::Committed); - assert_eq!(session.read().await.activity.last_authorized_media_at_ms, Some(2_100)); - } - - #[tokio::test] - async fn stale_session_handle_cannot_mark_recovered_session_with_same_public_id() { - let (manager, stale_session, proxy_session_id, lease_id, lease_identity) = - live_media_fixture("session-recovery").await; - let session_key = stale_session.read().await.key.clone(); - manager.sessions.remove_session(&session_key, &proxy_session_id).await.expect("remove original session"); - let (recovered_session, outcome) = - manager.get_or_create_session_with_outcome(session_key, b"secret", 2_000).await; - assert_eq!(outcome, HlsSessionStoreOutcome::Created); - assert!(!Arc::ptr_eq(&stale_session, &recovered_session)); - - assert_eq!( - manager - .mark_authorized_media_access_for_lease_if_identity_matches( - &stale_session, - &lease_id, - &proxy_session_id, - lease_identity, - 2_100, - ) - .await, - HlsMediaActivityCommitOutcome::StaleLeaseIdentity - ); - assert_eq!(stale_session.read().await.activity.last_authorized_media_at_ms, None); - assert_eq!(recovered_session.read().await.activity.last_authorized_media_at_ms, None); - } - - fn begin_test_acceptance_episode( - session: &mut super::super::HlsSession, - now_ms: u64, - ) -> HlsManifestAcceptanceGeneration { - let burst_plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let terminal_buffer = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); - let terminal_asset = snapshot_terminal_media_asset(&terminal_buffer).expect("terminal asset"); - let terminal_key = prepared_terminal_bundle_key(&terminal_asset, 12_000, HLS_TERMINAL_TAIL_SEGMENT_COUNT); - let required_terminal_media_key = Some(terminal_key); - let terminal_media_preparation = HlsTerminalMediaPreparationState::Preparing { key: terminal_key }; - let timing = HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput { - started_at_ms: now_ms, - burst_plan, - target_duration_ms: 12_000, - transition_margin: HlsTransitionMarginMs::from_millis(12_000), - workload: HlsRecoveryWorkload::clear_fetch(), - observed_latency: HlsObservedRecoveryLatency::default(), - required_terminal_media_key, - terminal_media_preparation, - policy: HlsRecoveryTimingPolicy::new( - HlsOperationTimeoutMs::from_millis(1_000), - HlsOperationTimeoutMs::from_millis(2_000), - HlsRecoveryEtaMs::from_millis(300), - HlsRecoveryEtaMs::from_millis(400), - ), - }); - session.origin_control.begin_acceptance_episode( - now_ms, - burst_plan, - HlsManifestAcceptanceTrigger::RecoveryRequired, - &timing, - ) - } - - fn complete_failed_acceptance_episode(session: &mut super::super::HlsSession, now_ms: u64) -> u64 { - let generation = begin_test_acceptance_episode(session, now_ms); - let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); - assert_eq!(episode.generation, generation); - episode.record_full_burst(); - episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::AllFailed); - episode.hold_after_uncommitted_burst(None, Some(now_ms.saturating_add(1_000))); - session.origin_control.progress_generation - } - - async fn prepared_terminal_commit_fixture( - lease_name: &str, - ) -> ( - HlsProxyManager, - HlsSessionHandle, - ProxySessionId, - HlsAccessLeaseId, - super::super::lease::HlsTerminalTailPreparation, - Arc, - ) { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - manager.terminal_commit_clock.set_fixed_now_ms(2_000); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, lease_name), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId(lease_name.to_string()); - manager.access_leases.write().await.prepare_access_lease(access_lease(lease_name, &proxy_session_id)); - assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); - let progress_generation = { - let mut session = session.write().await; - complete_failed_acceptance_episode(&mut session, 2_000) - }; - let preparation = manager - .prepare_access_lease_terminal_tail(terminal_preparation_request( - &lease_id, - &proxy_session_id, - progress_generation, - )) - .await - .expect("terminal preparation"); - let plan = terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id); - (manager, session, proxy_session_id, lease_id, preparation, plan) - } - - #[tokio::test] - async fn hls_terminal_commit_missing_acceptance_exact_ready_bundle_commits_tail() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - manager.terminal_commit_clock.set_fixed_now_ms(2_000); - let (session, _) = manager - .get_or_create_session_with_outcome(HlsSessionKey::new(1, "missing-acceptance"), b"secret", 1_000) - .await; - let (proxy_session_id, progress_generation) = { - let session = session.read().await; - assert!(session.origin_control.acceptance_episode.is_none()); - (session.proxy_session_id.clone(), session.origin_control.progress_generation) - }; - let lease_id = HlsAccessLeaseId("missing-acceptance".to_string()); - manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); - assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); - let mut preparation = manager - .prepare_access_lease_terminal_tail(terminal_preparation_request( - &lease_id, - &proxy_session_id, - progress_generation, - )) - .await - .expect("missing acceptance still permits cutover-local terminal preparation"); - assert_eq!( - preparation.terminal_media_requirement_source, - HlsTerminalMediaRequirementSource::CutoverSnapshotPending { - decision_generation: preparation.decision_generation, - } - ); - let plan = terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id); - let prepared_key = plan.media_preparation_key(); - preparation - .bind_ready_terminal_media_requirement(prepared_key) - .expect("the exact ready bundle binds to the cutover decision"); - assert_eq!( - preparation.terminal_media_requirement_source, - HlsTerminalMediaRequirementSource::CutoverSnapshot { - decision_generation: preparation.decision_generation, - asset: prepared_key.asset, - } - ); - - let outcome = commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan); - - assert_eq!(outcome, HlsTerminalCommitOutcome::Committed); - let lease = manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("committed terminal lease"); - assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalTail(_))); - } - - async fn wait_for_terminal_commit_owner(manager: &HlsProxyManager) { - for _ in 0..256 { - if manager.terminal_commit_retries.owner_count() == 0 { - return; - } - tokio::task::yield_now().await; - } - assert_eq!(manager.terminal_commit_retries.owner_count(), 0, "terminal retry owner did not finish"); - } - - #[test] - fn hls_cutover_policy_manager_uses_only_matching_acceptance_work_as_recovery_evidence() { - let mut session = super::super::HlsSession::new(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000); - let generation = begin_test_acceptance_episode(&mut session, 2_000); - session.origin_refresh.in_flight = false; - - assert!(matches!( - hls_acceptance_recovery_snapshot(&session, 2_100).recovery, - HlsRecoveryExecutionState::InFlight { .. } - )); - - let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); - episode.record_full_burst(); - episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::AllFailed); - session.origin_refresh.in_flight = true; - - let snapshot = hls_acceptance_recovery_snapshot(&session, 2_100); - assert_eq!(snapshot.expected_generation, generation); - assert_eq!(snapshot.recovery, HlsRecoveryExecutionState::Idle); - } - - fn terminal_preparation_request<'a>( - lease_id: &'a HlsAccessLeaseId, - proxy_session_id: &'a ProxySessionId, - origin_progress_generation: u64, - ) -> HlsTerminalTailPreparationRequest<'a> { - let reserve = cutover_reserve(); - let cutover_timing = - HlsLeaseCutoverTiming::from_reserve(2_000, reserve.guaranteed_reserve_ms, reserve.transition_margin, None); - HlsTerminalTailPreparationRequest { - lease_id, - proxy_session_id, - manifest_snapshot_generation: 1, - cursor_generation: 0, - reserve, - cutover_timing, - commit_window: HlsTerminalCommitWindow::AcquisitionOpen, - now_ms: 2_000, - origin_progress_generation, - media_readiness_generation: 0, - last_media_progress_at_ms: None, - } - } - - #[tokio::test] - async fn manifest_publication_manager_rejects_delayed_source_and_expiry_races() { - let manager = HlsProxyManager::new(); - let proxy_session_id = ProxySessionId("proxy".to_string()); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); - let older_request = manager - .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, 2_000) - .await - .expect("older request guard"); - let newer_request = manager - .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, 2_000) - .await - .expect("newer request guard"); - - assert_eq!( - manager - .commit_access_lease_manifest_publication( - &lease_id, - &proxy_session_id, - newer_request, - manifest_snapshot(20), - 2_100, - ) - .await, - HlsLeaseManifestPublicationOutcome::Committed { snapshot_generation: 1 } - ); - assert_eq!( - manager - .commit_access_lease_manifest_publication( - &lease_id, - &proxy_session_id, - older_request, - manifest_snapshot(10), - 2_200, - ) - .await, - HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::SourceRegressive) - ); - - let expiry_lease_id = HlsAccessLeaseId("expiry".to_string()); - manager.access_leases.write().await.prepare_access_lease(access_lease(&expiry_lease_id.0, &proxy_session_id)); - let expiry_request = manager - .prepare_access_lease_manifest_publication(&expiry_lease_id, &proxy_session_id, 2_000) - .await - .expect("pre-expiry request guard"); - assert_eq!( - manager - .commit_access_lease_manifest_publication( - &expiry_lease_id, - &proxy_session_id, - expiry_request, - manifest_snapshot(30), - 61_000, - ) - .await, - HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::LeaseExpired) - ); - } - - #[tokio::test] - async fn rolling_manifest_publication_uses_lease_store_when_session_is_gc_marked() { - let manager = HlsProxyManager::new(); - let session = manager.get_or_create_session(HlsSessionKey::new(1, "rolling"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); - let publication = manager - .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, 2_000) - .await - .expect("rolling publication guard"); - let published_resource_ids = HlsPublishedTransientResourceIds::from_manifest_body( - "#EXTM3U\n#EXTINF:6,\n/hls/shared/live/proxy/lease-a/r/resource.ts\n", - ); - session.write().await.mark_for_gc_removal(); - - assert_eq!( - manager - .commit_access_lease_manifest_publication_with_resources( - &lease_id, - &proxy_session_id, - publication, - manifest_snapshot(20), - published_resource_ids.clone(), - 2_100, - ) - .await, - HlsLeaseManifestPublicationOutcome::Committed { snapshot_generation: 1 } - ); - let lease = manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_100) - .await - .expect("published rolling lease"); - assert_eq!(lease.published_transient_resource_ids(), &published_resource_ids); - } - - #[tokio::test] - async fn finalized_manifest_publication_blocks_session_replacement_until_binding_is_committed() { - let manager = Arc::new(HlsProxyManager::new()); - let session_key = HlsSessionKey::new(1, "archive-publication-race"); - let session = manager.get_or_create_session(session_key.clone(), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "https://cdn.example/archive/segment.ts", - b"secret", - 1_000, - 1, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - let resource_uri = format!("/hls/shared/live/{}/lease-a/r/{}.ts", proxy_session_id.0, resource_id.0); - let manifest_body = - format!("#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXTINF:6,\n{resource_uri}\n#EXT-X-ENDLIST\n"); - let generation = { - let mut session = session.write().await; - session.transient.upsert_resources([resource]); - session.transient.replace_manifest_with_semantics(manifest_body.clone(), 1_000, Some(6_000)); - session.transient.current_finalized_manifest_generation().expect("finalized generation") - }; - manager.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)).await; - let publication = manager - .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, 2_000) - .await - .expect("publication guard"); - let published_resource_ids = HlsPublishedTransientResourceIds::from_manifest_body(&manifest_body); - let lease_store_guard = manager.hold_access_lease_store_for_test().await; - let publication_task = { - let manager = Arc::clone(&manager); - let lease_id = lease_id.clone(); - let proxy_session_id = proxy_session_id.clone(); - let published_resource_ids = published_resource_ids.clone(); - tokio::spawn(async move { - manager - .commit_access_lease_manifest_publication_with_resources( - &lease_id, - &proxy_session_id, - publication, - finalized_transient_manifest_snapshot(1_000, generation, &resource_uri), - published_resource_ids, - 2_000, - ) - .await - }) - }; - - let mut index_guard_observed = false; - for _ in 0..1_000 { - if manager.sessions.index_write_is_blocked_for_test() { - index_guard_observed = true; - break; - } - tokio::task::yield_now().await; - } - assert!(index_guard_observed, "publication did not acquire the session-index guard"); - assert!( - tokio::time::timeout( - std::time::Duration::from_millis(25), - manager.sessions.remove_session(&session_key, &proxy_session_id), - ) - .await - .is_err(), - "session replacement completed before finalized publication" - ); - - drop(lease_store_guard); - assert_eq!( - publication_task.await.expect("publication task"), - HlsLeaseManifestPublicationOutcome::Committed { snapshot_generation: 1 } - ); - let current_session = - manager.sessions.get_by_proxy_session_id(&proxy_session_id).await.expect("current session remains indexed"); - assert!(Arc::ptr_eq(¤t_session, &session)); - assert!(current_session - .read() - .await - .transient - .resolve_resource_for_lease(&resource_id, &lease_id, 1_000, &published_resource_ids, 2_000) - .is_some()); - } - - #[tokio::test] - #[allow(clippy::similar_names, clippy::too_many_lines)] - async fn finalized_signed_url_generations_follow_published_lease_lifetime() { - let manager = HlsProxyManager::new(); - let published_resource_ids = HlsPublishedTransientResourceIds::default(); - let session = manager.get_or_create_session(HlsSessionKey::new(1, "archive"), b"secret", 0).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_a_id = HlsAccessLeaseId("lease-a".to_string()); - let lease_b_id = HlsAccessLeaseId("lease-b".to_string()); - let resource_a = TransientResourceRef::new( - TransientResourceKind::Segment, - "https://cdn.example/archive/segment.ts?token=A", - b"secret", - 0, - 20, - Some("ts".to_string()), - ); - let resource_a_id = resource_a.id.clone(); - let resource_a_uri = format!("/hls/shared/live/{}/lease-a/r/{}.ts", proxy_session_id.0, resource_a_id.0); - let generation_a = { - let mut session = session.write().await; - session.transient.upsert_resources([resource_a]); - session.transient.replace_manifest_with_semantics( - format!( - "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{resource_a_uri}\n#EXT-X-ENDLIST\n" - ), - 0, - Some(6_000), - ); - session.transient.current_finalized_manifest_generation().expect("G1 finalized generation") - }; - - let lease_a = HlsAccessLease::pending( - lease_a_id.clone(), - HlsPlaybackFamilyKey::new("alice", "client-a"), - proxy_session_id.clone(), - "alice".to_string(), - "session-a".to_string(), - 1, - "archive".to_string(), - 1, - 0, - 40, - ); - manager.prepare_access_lease(lease_a).await; - assert!( - publish_manifest_snapshot( - &manager, - &lease_a_id, - &proxy_session_id, - finalized_transient_manifest_snapshot(0, generation_a, &resource_a_uri), - 1, - ) - .await - ); - - let resource_b = TransientResourceRef::new( - TransientResourceKind::Segment, - "https://cdn.example/archive/segment.ts?token=B", - b"secret", - 21, - 20, - Some("ts".to_string()), - ); - let resource_b_id = resource_b.id.clone(); - assert_ne!(resource_a_id, resource_b_id); - let resource_b_uri = format!("/hls/shared/live/{}/lease-b/r/{}.ts", proxy_session_id.0, resource_b_id.0); - let generation_b = { - let mut session = session.write().await; - session.transient.upsert_resources([resource_b]); - session.transient.replace_manifest_with_semantics( - format!( - "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{resource_b_uri}\n#EXT-X-ENDLIST\n" - ), - 21, - Some(6_000), - ); - session.transient.current_finalized_manifest_generation().expect("G2 finalized generation") - }; - - assert!(session - .read() - .await - .transient - .resolve_resource_for_lease(&resource_a_id, &lease_a_id, 0, &published_resource_ids, 21) - .is_some()); - assert_eq!(session.read().await.transient.finalized_manifest_generation_count(), 2); - - let lease_b = HlsAccessLease::pending( - lease_b_id.clone(), - HlsPlaybackFamilyKey::new("bob", "client-b"), - proxy_session_id.clone(), - "bob".to_string(), - "session-b".to_string(), - 1, - "archive".to_string(), - 1, - 21, - 100, - ); - manager.prepare_access_lease(lease_b).await; - assert!( - publish_manifest_snapshot( - &manager, - &lease_b_id, - &proxy_session_id, - finalized_transient_manifest_snapshot(21, generation_b, &resource_b_uri), - 22, - ) - .await - ); - assert!(session - .read() - .await - .transient - .resolve_resource_for_lease(&resource_b_id, &lease_b_id, 21, &published_resource_ids, 42) - .is_some()); - assert!(session - .read() - .await - .transient - .resolve_resource_for_lease(&resource_a_id, &lease_a_id, 0, &published_resource_ids, 39) - .is_some()); - assert_eq!(session.read().await.transient.finalized_manifest_lease_binding_count(), 2); - { - let session = session.read().await; - assert_eq!(session.transient.current_manifest_resource_ids().len(), 1); - assert!(session.transient.resolve_current_resource(&resource_b_id, 42).is_some()); - } - - let expired_a = manager - .access_lease_response_snapshot(&lease_a_id, &proxy_session_id, 41) - .await - .expect("expired lease remains available for lifecycle cleanup"); - assert_eq!(expired_a.state, HlsAccessLeaseState::Expired); - manager.remove_access_lease(&lease_a_id).await; - let mut session = session.write().await; - assert_eq!(session.transient.finalized_manifest_generation_count(), 1); - assert_eq!(session.transient.finalized_manifest_lease_binding_count(), 1); - assert!(session.transient.resolve_current_resource(&resource_a_id, 41).is_none()); - session.transient.prune_expired(41); - assert!(!session.transient.resources.contains_key(&resource_a_id)); - assert!(session.transient.resolve_current_resource(&resource_b_id, 42).is_some()); - } - - #[tokio::test] - #[allow(clippy::similar_names, clippy::too_many_lines)] - async fn same_lease_retains_every_published_finalized_generation_until_removal() { - let manager = HlsProxyManager::new(); - let published_resource_ids = HlsPublishedTransientResourceIds::default(); - let session = manager.get_or_create_session(HlsSessionKey::new(1, "archive"), b"secret", 0).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - manager - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("alice", "client-a"), - proxy_session_id.clone(), - "alice".to_string(), - "session-a".to_string(), - 1, - "archive".to_string(), - 1, - 0, - 100, - )) - .await; - - let (resource_a_id, generation_a, resource_a_uri) = { - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "https://cdn.example/archive/segment.ts?token=A", - b"secret", - 0, - 20, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - let resource_uri = format!("/hls/shared/live/{}/lease-a/r/{}.ts", proxy_session_id.0, resource_id.0); - let mut session = session.write().await; - session.transient.upsert_resources([resource]); - session.transient.replace_manifest_with_semantics( - format!( - "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{resource_uri}\n#EXT-X-ENDLIST\n" - ), - 0, - Some(6_000), - ); - let generation = - session.transient.current_finalized_manifest_generation().expect("G1 finalized generation"); - (resource_id, generation, resource_uri) - }; - assert!( - publish_manifest_snapshot( - &manager, - &lease_id, - &proxy_session_id, - finalized_transient_manifest_snapshot(0, generation_a, &resource_a_uri), - 1, - ) - .await - ); - - let (resource_b_id, generation_b, resource_b_uri) = { - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "https://cdn.example/archive/segment.ts?token=B", - b"secret", - 21, - 20, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - let resource_uri = format!("/hls/shared/live/{}/lease-a/r/{}.ts", proxy_session_id.0, resource_id.0); - let mut session = session.write().await; - session.transient.upsert_resources([resource]); - session.transient.replace_manifest_with_semantics( - format!( - "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{resource_uri}\n#EXT-X-ENDLIST\n" - ), - 21, - Some(6_000), - ); - let generation = - session.transient.current_finalized_manifest_generation().expect("G2 finalized generation"); - (resource_id, generation, resource_uri) - }; - assert_ne!(resource_a_id, resource_b_id); - assert!( - publish_manifest_snapshot( - &manager, - &lease_id, - &proxy_session_id, - finalized_transient_manifest_snapshot(21, generation_b, &resource_b_uri), - 22, - ) - .await - ); - - { - let session = session.read().await; - assert_eq!(session.transient.finalized_manifest_lease_binding_count(), 2); - assert!(session - .transient - .resolve_resource_for_lease(&resource_a_id, &lease_id, 0, &published_resource_ids, 42) - .is_some()); - assert!(session - .transient - .resolve_resource_for_lease(&resource_b_id, &lease_id, 0, &published_resource_ids, 42) - .is_some()); - } - - manager.remove_access_lease(&lease_id).await; - let session = session.read().await; - assert_eq!(session.transient.finalized_manifest_lease_binding_count(), 0); - assert_eq!(session.transient.finalized_manifest_generation_count(), 1); - assert!(session.transient.resolve_current_resource(&resource_a_id, 42).is_none()); - assert!(session.transient.resolve_current_resource(&resource_b_id, 42).is_some()); - } - - #[tokio::test] - async fn lease_reconciliation_cannot_overwrite_a_concurrent_manifest_publication() { - let manager = Arc::new(HlsProxyManager::new()); - let session = manager.get_or_create_session(HlsSessionKey::new(1, "archive"), b"secret", 0).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "https://cdn.example/archive/segment.ts?token=A", - b"secret", - 0, - 20, - Some("ts".to_string()), - ); - let resource_uri = format!("/hls/shared/live/{}/lease-a/r/{}.ts", proxy_session_id.0, resource.id.0); - let generation = { - let mut session = session.write().await; - session.transient.upsert_resources([resource]); - session.transient.replace_manifest_with_semantics( - format!( - "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{resource_uri}\n#EXT-X-ENDLIST\n" - ), - 0, - Some(6_000), - ); - session.transient.current_finalized_manifest_generation().expect("finalized generation") - }; - manager.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)).await; - let publication_guard = manager - .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, 1) - .await - .expect("publication guard"); - - let session_guard = session.write().await; - let reconcile_task = { - let manager = Arc::clone(&manager); - let session = Arc::clone(&session); - let proxy_session_id = proxy_session_id.clone(); - tokio::spawn(async move { - manager.reconcile_session_access_lease_snapshot(&session, &proxy_session_id, 1).await - }) - }; - wait_until_access_lease_store_is_write_locked(&manager).await; - let publication_task = { - let manager = Arc::clone(&manager); - let proxy_session_id = proxy_session_id.clone(); - let lease_id = lease_id.clone(); - tokio::spawn(async move { - manager - .commit_access_lease_manifest_publication( - &lease_id, - &proxy_session_id, - publication_guard, - finalized_transient_manifest_snapshot(0, generation, &resource_uri), - 2, - ) - .await - }) - }; - drop(session_guard); - - reconcile_task.await.expect("reconciliation completes"); - assert!(publication_task.await.expect("publication completes").is_committed()); - assert_eq!(session.read().await.transient.finalized_manifest_lease_binding_count(), 1); - } - - #[tokio::test] - async fn lease_reconciliation_cannot_resurrect_a_concurrently_removed_binding() { - let manager = Arc::new(HlsProxyManager::new()); - let session = manager.get_or_create_session(HlsSessionKey::new(1, "archive"), b"secret", 0).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "https://cdn.example/archive/segment.ts?token=A", - b"secret", - 0, - 20, - Some("ts".to_string()), - ); - let resource_uri = format!("/hls/shared/live/{}/lease-a/r/{}.ts", proxy_session_id.0, resource.id.0); - let generation = { - let mut session = session.write().await; - session.transient.upsert_resources([resource]); - session.transient.replace_manifest_with_semantics( - format!( - "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{resource_uri}\n#EXT-X-ENDLIST\n" - ), - 0, - Some(6_000), - ); - session.transient.current_finalized_manifest_generation().expect("finalized generation") - }; - manager.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)).await; - assert!( - publish_manifest_snapshot( - &manager, - &lease_id, - &proxy_session_id, - finalized_transient_manifest_snapshot(0, generation, &resource_uri), - 1, - ) - .await - ); - let removal = - manager.access_leases.read().await.prepare_access_lease_removal(&lease_id).expect("removal preparation"); - - let session_guard = session.write().await; - let reconcile_task = { - let manager = Arc::clone(&manager); - let session = Arc::clone(&session); - let proxy_session_id = proxy_session_id.clone(); - tokio::spawn(async move { - manager.reconcile_session_access_lease_snapshot(&session, &proxy_session_id, 1).await - }) - }; - wait_until_access_lease_store_is_write_locked(&manager).await; - let removal_task = { - let manager = Arc::clone(&manager); - let lease_id = lease_id.clone(); - tokio::spawn(async move { manager.remove_prepared_access_lease(&lease_id, &removal).await }) - }; - drop(session_guard); - - reconcile_task.await.expect("reconciliation completes"); - assert!(removal_task.await.expect("removal completes")); - assert_eq!(session.read().await.transient.finalized_manifest_lease_binding_count(), 0); - } - - #[tokio::test] - async fn update_config_applies_hls_runtime_settings_to_existing_manager() { - let initial_dto = HlsCacheConfigDto { - cache_path: Some("/tmp/tuliprox/hls-a".to_string()), - max_segments_prefetch: 1, - ..Default::default() - }; - let initial_config = HlsCacheConfig::from(&initial_dto); - let manager = HlsProxyManager::with_hls_cache_config(&initial_config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 100).await; - - let mut updated_dto = initial_dto.clone(); - updated_dto.max_segments_prefetch = 4; - updated_dto.max_concurrent_segment_fetches_per_session = 5; - updated_dto.max_concurrent_segment_fetches_global = 6; - updated_dto.origin_manifest_timeout_ms = shared::model::Millis::new(1_234); - updated_dto.origin_segment_timeout_ms = shared::model::Millis::new(5_678); - updated_dto.cache_duration = shared::model::Secs::new(99); - updated_dto.session_idle_timeout = shared::model::Secs::new(55); - updated_dto.manifest_recovery_burst.level = shared::model::HlsManifestRecoveryBurstLevel::Balanced; - updated_dto.strip = HlsStripConfigDto { mode: HlsStripMode::Seconds, value: 7 }; - let app_config = test_app_config(config_with_hls_cache(updated_dto)); - - manager.update_config(&app_config).await; - - assert_eq!(manager.segment_fetch_policy().max_prefetch_queue_depth, 4); - assert_eq!(manager.segment_fetch_policy().max_session_segment_fetches, 5); - assert_eq!(manager.segment_fetch_policy().max_global_segment_fetches, 6); - assert_eq!(manager.segment_fetch_policy().origin_segment_timeout_ms, 5_678); - assert_eq!(manager.origin_manifest_timeout_ms(), 1_234); - assert_eq!(manager.cache_duration_seconds(), 99); - assert_eq!(manager.transient_resource_ttl_ms(), 99_000); - assert_eq!(manager.session_idle_timeout_ms(), 55_000); - assert_eq!(manager.manifest_recovery_burst().level, HlsManifestRecoveryBurstLevel::Balanced); - assert_eq!(manager.strip().mode, HlsStripMode::Seconds); - assert_eq!(manager.strip().value, 7); - assert_eq!(session.read().await.segment_prefetch_queue.max_prefetch_depth(), 4); - } - - #[tokio::test] - async fn optional_hls_config_controls_runtime_enabled_state() { - let manager = HlsProxyManager::from_hls_cache_config(None); - - assert!(!manager.is_enabled()); - assert_eq!( - manager.run_garbage_collection_once(1_000).await.expect("disabled gc should no-op"), - super::super::GarbageCollectionReport::default() - ); - - let app_config = test_app_config(config_with_hls_cache(HlsCacheConfigDto::default())); - manager.update_config(&app_config).await; - - assert!(manager.is_enabled()); - } - - #[tokio::test] - async fn update_config_cache_path_change_clears_runtime_cache_state() { - let temp_dir = tempfile::tempdir().expect("temp dir"); - let old_cache = temp_dir.path().join("old"); - let new_cache = temp_dir.path().join("new"); - let initial_dto = - HlsCacheConfigDto { cache_path: Some(old_cache.to_string_lossy().to_string()), ..Default::default() }; - let initial_config = HlsCacheConfig::from(&initial_dto); - let manager = HlsProxyManager::with_hls_cache_config(&initial_config); - let _ = manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 100).await; - assert_eq!(manager.sessions().len().await, 1); - - let mut updated_dto = initial_dto; - updated_dto.cache_path = Some(new_cache.to_string_lossy().to_string()); - let app_config = test_app_config(config_with_hls_cache(updated_dto)); - - manager.update_config(&app_config).await; - - assert!(manager.sessions().is_empty().await); - assert_eq!(manager.segment_cache().cache_path(), new_cache); - } - - #[tokio::test] - async fn repeated_segment_failure_counter_does_not_terminalize_access_lease() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); - - { - let mut session = session.write().await; - for failure in 0_u64..100 { - let _ = session.record_temporary_segment_fetch_failure( - 2_000_u64.saturating_add(failure), - HlsSegmentFailureObject::Normal { proxy_seq: 40, origin_seq: 400 }, - 1, - ); - } - assert_eq!(session.segment_failure_tracker.consecutive_temporary_failures, 100); - } - - let lease = manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_100) - .await - .expect("failure tracking must not remove the lease"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - assert!(!session.read().await.has_terminal_tail_protections()); - } - - #[tokio::test] - async fn hls_cutover_policy_media_progress_supersedes_prepared_plan() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); - assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); - let progress_generation = { - let mut session = session.write().await; - complete_failed_acceptance_episode(&mut session, 2_000) - }; - let preparation = manager - .prepare_access_lease_terminal_tail(terminal_preparation_request( - &lease_id, - &proxy_session_id, - progress_generation, - )) - .await - .expect("failed acceptance permits terminal preparation"); - - { - let mut session = session.write().await; - session.origin_control.record_media_progress(2_100, 12_000); - assert_eq!(session.origin_control.progress_generation, progress_generation.saturating_add(1)); - } - assert_eq!( - commit_terminal_plan( - &manager, - &session, - &lease_id, - &proxy_session_id, - &preparation, - 2_100, - terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), - ), - HlsTerminalCommitOutcome::RecoveryCommitted - ); - - let lease = manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_200) - .await - .expect("lease remains available"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - assert!(!session.read().await.has_terminal_tail_protections()); - } - - #[tokio::test] - async fn hls_cutover_policy_matching_acceptance_commit_without_media_progress_terminalizes_lease() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); - assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); - let progress_generation = { - let mut session = session.write().await; - complete_failed_acceptance_episode(&mut session, 2_000) - }; - let preparation = manager - .prepare_access_lease_terminal_tail(terminal_preparation_request( - &lease_id, - &proxy_session_id, - progress_generation, - )) - .await - .expect("terminal preparation"); - - { - let mut session = session.write().await; - session.origin_control.acceptance_episode.as_mut().expect("acceptance episode").complete(); - assert_eq!(session.origin_control.progress_generation, preparation.origin_progress_generation); - } - - assert_eq!( - commit_terminal_plan( - &manager, - &session, - &lease_id, - &proxy_session_id, - &preparation, - 2_100, - terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), - ), - HlsTerminalCommitOutcome::Committed - ); - assert!(matches!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_100) - .await - .expect("terminal lease") - .playback_mode, - HlsLeasePlaybackMode::TerminalTail(_) - )); - } - - #[tokio::test] - async fn hls_cutover_policy_new_acceptance_episode_without_media_progress_does_not_supersede_cutover() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); - assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); - let progress_generation = { - let mut session = session.write().await; - complete_failed_acceptance_episode(&mut session, 2_000) - }; - let preparation = manager - .prepare_access_lease_terminal_tail(terminal_preparation_request( - &lease_id, - &proxy_session_id, - progress_generation, - )) - .await - .expect("terminal preparation"); - - { - let mut session = session.write().await; - let previous_acceptance_generation = session.origin_control.acceptance_generation; - let current_acceptance_generation = begin_test_acceptance_episode(&mut session, 2_050); - assert_eq!( - session.origin_control.acceptance_generation.0, - previous_acceptance_generation.0.saturating_add(1) - ); - assert_eq!(current_acceptance_generation, session.origin_control.acceptance_generation); - assert_eq!(session.origin_control.progress_generation, preparation.origin_progress_generation); - } - - assert_eq!( - commit_terminal_plan( - &manager, - &session, - &lease_id, - &proxy_session_id, - &preparation, - 2_100, - terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), - ), - HlsTerminalCommitOutcome::Committed - ); - } - - #[tokio::test] - async fn hls_terminal_commit_changed_asset_revision_fails_closed_before_tail_mutation() { - let (manager, session, proxy_session_id, lease_id, preparation, plan) = - prepared_terminal_commit_fixture("asset-revision").await; - let expected_asset = plan.asset_identity; - - let outcome = manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { - session: &session, - lease_id: &lease_id, - proxy_session_id: &proxy_session_id, - preparation: &preparation, - now_ms: 2_000, - payload: HlsTerminalCommitPayload::Tail { - plan, - media_guard: HlsTerminalCommitMediaGuard::empty_for_test(), - }, - asset_revision_guard: HlsTerminalAssetRevisionGuard::for_runtime_tail(expected_asset, || None), - }); - - assert_eq!(outcome, HlsTerminalCommitOutcome::Committed); - assert!(matches!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("asset mismatch terminalizes the lease") - .playback_mode, - HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::MissingAsset, .. } - )); - assert!(!session.read().await.has_terminal_tail_protections()); - } - - #[tokio::test] - async fn hls_terminal_commit_retry_asset_change_autonomously_fails_closed() { - let (manager, session, proxy_session_id, lease_id, preparation, plan) = - prepared_terminal_commit_fixture("retry-asset-change").await; - let expected_asset = plan.asset_identity; - let asset_is_current = Arc::new(AtomicBool::new(true)); - let revision_guard = { - let asset_is_current = Arc::clone(&asset_is_current); - HlsTerminalAssetRevisionGuard::for_runtime_tail(expected_asset, move || { - asset_is_current.load(Ordering::Acquire).then_some(expected_asset) - }) - }; - let session_guard = session.write().await; - - assert!(matches!( - manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { - session: &session, - lease_id: &lease_id, - proxy_session_id: &proxy_session_id, - preparation: &preparation, - now_ms: 2_000, - payload: HlsTerminalCommitPayload::Tail { - plan, - media_guard: HlsTerminalCommitMediaGuard::empty_for_test(), - }, - asset_revision_guard: revision_guard, - }), - HlsTerminalCommitOutcome::LockBusy { .. } - )); - asset_is_current.store(false, Ordering::Release); - drop(session_guard); - - wait_for_terminal_commit_owner(&manager).await; - assert!(matches!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("asset change terminalizes without another request") - .playback_mode, - HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::MissingAsset, .. } - )); - } - - #[tokio::test] - async fn hls_terminal_commit_exact_replay_stays_idempotent_after_asset_revision_change() { - let (manager, session, proxy_session_id, lease_id, preparation, plan) = - prepared_terminal_commit_fixture("asset-replay").await; - let expected_asset = plan.asset_identity; - - assert_eq!( - commit_terminal_plan( - &manager, - &session, - &lease_id, - &proxy_session_id, - &preparation, - 2_000, - Arc::clone(&plan), - ), - HlsTerminalCommitOutcome::Committed - ); - let replay = manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { - session: &session, - lease_id: &lease_id, - proxy_session_id: &proxy_session_id, - preparation: &preparation, - now_ms: 2_000, - payload: HlsTerminalCommitPayload::Tail { - plan, - media_guard: HlsTerminalCommitMediaGuard::empty_for_test(), - }, - asset_revision_guard: HlsTerminalAssetRevisionGuard::for_runtime_tail(expected_asset, || None), - }); - - assert_eq!(replay, HlsTerminalCommitOutcome::AlreadyCommitted); - assert!(matches!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("terminal replay stays available") - .playback_mode, - HlsLeasePlaybackMode::TerminalTail(_) - )); - } - - #[tokio::test] - async fn hls_terminal_commit_retry_lock_busy_without_a_second_client_request() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - manager.terminal_commit_clock.set_fixed_now_ms(2_000); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); - assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); - let progress_generation = { - let mut session = session.write().await; - complete_failed_acceptance_episode(&mut session, 2_000) - }; - let preparation = manager - .prepare_access_lease_terminal_tail(terminal_preparation_request( - &lease_id, - &proxy_session_id, - progress_generation, - )) - .await - .expect("terminal preparation"); - let plan = terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id); - - let session_guard = session.write().await; - assert_eq!( - commit_terminal_plan( - &manager, - &session, - &lease_id, - &proxy_session_id, - &preparation, - 2_000, - Arc::clone(&plan), - ), - HlsTerminalCommitOutcome::LockBusy { retry_before_ms: 2_001 } - ); - assert_eq!( - commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), - HlsTerminalCommitOutcome::LockBusy { retry_before_ms: 2_001 } - ); - assert_eq!(manager.terminal_commit_retries.owner_count(), 1); - drop(session_guard); - - wait_for_terminal_commit_owner(&manager).await; - assert!(matches!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("autonomous retry keeps the lease available") - .playback_mode, - HlsLeasePlaybackMode::TerminalTail(_) - )); - } - - #[tokio::test] - async fn hls_terminal_commit_submission_delayed_stale_incoming_preserves_authoritative_tail() { - let (manager, session, proxy_session_id, lease_id, preparation, plan) = - prepared_terminal_commit_fixture("submission-stale-incoming").await; - let current_asset = plan.asset_identity; - let stale_asset = HlsTerminalAssetIdentity { revision: 999, fingerprint: [9; 32] }; - let Some(session_incarnation) = manager.sessions.session_incarnation(&session) else { - panic!("fixture session has an incarnation"); - }; - let Some((cancellation_epoch, submission_token)) = manager.terminal_commit_retries.reserve_submission() else { - panic!("fixture can reserve terminal submission"); - }; - let owner_key = HlsTerminalCommitOwnerKey::from_preparation(&proxy_session_id, &lease_id, &preparation); - let command = HlsTerminalCommitCommand { - key: owner_key, - session: Arc::clone(&session), - session_incarnation, - preparation: preparation.clone(), - decision: HlsTerminalLeaseDecision::Tail(Arc::clone(&plan)), - media_guard: Some(HlsTerminalCommitMediaGuard::empty_for_test()), - asset_revision_guard: HlsTerminalAssetRevisionGuard::matching_runtime_for_test(current_asset), - cancellation_epoch, - submission_token, - }; - let HlsTerminalCommitSubmissionDecision::Attempt { command, owner_token } = - manager.terminal_commit_retries.submit(command, 2_000) - else { - panic!("authoritative tail owns the initial submission"); - }; - let HlsTerminalCommitRetryDecision::Schedule { retry_at_ms, attempts_completed } = next_terminal_commit_retry( - 1, - 2_000, - preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), - ) else { - panic!("fixture has retry budget"); - }; - let HlsTerminalCommitRetryScheduleDecision::Scheduled { worker_token: Some(worker_token) } = manager - .terminal_commit_retries - .schedule_current(&command.key, owner_token, attempts_completed, retry_at_ms, 2_000) - else { - panic!("authoritative tail schedules one bounded worker"); - }; - - let mut stale_preparation = preparation.clone(); - stale_preparation.cutover_timing = HlsLeaseCutoverTiming::from_reserve( - 2_000, - preparation.reserve.transition_margin.as_millis().saturating_add(2), - preparation.reserve.transition_margin, - None, - ); - let outcome = manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { - session: &session, - lease_id: &lease_id, - proxy_session_id: &proxy_session_id, - preparation: &stale_preparation, - now_ms: 2_000, - payload: HlsTerminalCommitPayload::Unavailable(HlsTerminalTailCompatibility::AssetRevisionMismatch), - asset_revision_guard: HlsTerminalAssetRevisionGuard::for_runtime_tail( - HlsRuntimeCustomTailAssetIdentity::new(current_asset.reason, stale_asset), - move || Some(current_asset), - ), - }); - - assert_eq!(outcome, HlsTerminalCommitOutcome::LockBusy { retry_before_ms: retry_at_ms }); - assert_eq!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("unauthorized incoming keeps lease available") - .playback_mode, - HlsLeasePlaybackMode::Live - ); - spawn_terminal_commit_retry_worker( - Arc::clone(&manager.sessions), - Arc::clone(&manager.access_leases), - Arc::clone(&manager.terminal_commit_retries), - Arc::clone(&manager.terminal_commit_clock), - worker_token, - HlsProxyManager::try_commit_access_lease_terminal_decision, - ); - wait_for_terminal_commit_owner(&manager).await; - assert!(matches!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_001) - .await - .expect("authorized owner terminalizes the lease") - .playback_mode, - HlsLeasePlaybackMode::TerminalTail(_) - )); - } - - #[tokio::test] - async fn hls_terminal_commit_submission_newer_command_replaces_lock_busy_owner() { - let (manager, session, proxy_session_id, lease_id, preparation, plan) = - prepared_terminal_commit_fixture("retry-owner-replacement").await; - let stale_asset_identity = plan.asset_identity; - let session_guard = session.write().await; - - assert!(matches!( - manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { - session: &session, - lease_id: &lease_id, - proxy_session_id: &proxy_session_id, - preparation: &preparation, - now_ms: 2_000, - payload: HlsTerminalCommitPayload::Tail { - plan, - media_guard: HlsTerminalCommitMediaGuard::empty_for_test(), - }, - asset_revision_guard: HlsTerminalAssetRevisionGuard::for_runtime_tail(stale_asset_identity, || None,), - }), - HlsTerminalCommitOutcome::LockBusy { .. } - )); - assert!(matches!( - manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { - session: &session, - lease_id: &lease_id, - proxy_session_id: &proxy_session_id, - preparation: &preparation, - now_ms: 2_000, - payload: HlsTerminalCommitPayload::Unavailable(HlsTerminalTailCompatibility::AssetRevisionMismatch,), - asset_revision_guard: HlsTerminalAssetRevisionGuard::matching_for_test(None), - }), - HlsTerminalCommitOutcome::LockBusy { .. } - )); - assert_eq!(manager.terminal_commit_retries.owner_count(), 1); - drop(session_guard); - - wait_for_terminal_commit_owner(&manager).await; - assert!(matches!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("replacement command terminalizes the lease") - .playback_mode, - HlsLeasePlaybackMode::TerminalUnavailable { - reason: HlsTerminalTailCompatibility::AssetRevisionMismatch, - .. - } - )); - } - - #[tokio::test] - async fn hls_terminal_commit_retry_never_commits_after_the_safe_deadline() { - let (manager, session, proxy_session_id, lease_id, preparation, plan) = - prepared_terminal_commit_fixture("retry-deadline").await; - let after_deadline_ms = - preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch().saturating_add(1); - manager.terminal_commit_clock.set_fixed_now_ms(after_deadline_ms); - let session_guard = session.write().await; - - assert_eq!( - commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), - HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed - ); - assert_eq!(manager.terminal_commit_retries.owner_count(), 0); - drop(session_guard); - - assert_eq!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, after_deadline_ms) - .await - .expect("deadline-expired lease remains live") - .playback_mode, - HlsLeasePlaybackMode::Live - ); - assert!(!session.read().await.has_terminal_tail_protections()); - } - - #[tokio::test] - async fn hls_terminal_commit_initial_and_retry_attempts_reject_exclusive_safe_deadline() { - let (manager, session, proxy_session_id, lease_id, preparation, plan) = - prepared_terminal_commit_fixture("initial-exclusive-deadline").await; - let safe_deadline_ms = preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); - manager.terminal_commit_clock.set_fixed_now_ms(safe_deadline_ms); - - assert_eq!( - commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), - HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed - ); - assert!(matches!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, safe_deadline_ms) - .await - .expect("exclusive deadline leaves the initial-attempt lease live") - .playback_mode, - HlsLeasePlaybackMode::Live - )); - assert!(!session.read().await.has_terminal_tail_protections()); - - let (manager, session, proxy_session_id, lease_id, preparation, plan) = - prepared_terminal_commit_fixture("retry-exclusive-deadline").await; - let safe_deadline_ms = preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); - let session_guard = session.write().await; - assert!(matches!( - commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), - HlsTerminalCommitOutcome::LockBusy { .. } - )); - assert_eq!(manager.terminal_commit_retries.owner_count(), 1); - manager.terminal_commit_clock.set_fixed_now_ms(safe_deadline_ms); - drop(session_guard); - - wait_for_terminal_commit_owner(&manager).await; - assert!(matches!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, safe_deadline_ms) - .await - .expect("exclusive deadline leaves the retry lease live") - .playback_mode, - HlsLeasePlaybackMode::Live - )); - assert!(!session.read().await.has_terminal_tail_protections()); - } - - #[tokio::test] - async fn hls_terminal_commit_session_replacement_cancels_the_detached_retry() { - let (manager, session, proxy_session_id, lease_id, preparation, plan) = - prepared_terminal_commit_fixture("retry-session-replacement").await; - let session_key = session.read().await.key.clone(); - let lease_guard = manager.access_leases.write().await; - - assert!(matches!( - commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), - HlsTerminalCommitOutcome::LockBusy { .. } - )); - manager.sessions.remove_session(&session_key, &proxy_session_id).await.expect("remove prepared session"); - let (replacement, outcome) = manager.get_or_create_session_with_outcome(session_key, b"secret", 2_000).await; - assert_eq!(outcome, HlsSessionStoreOutcome::Created); - assert!(!Arc::ptr_eq(&session, &replacement)); - drop(lease_guard); - - wait_for_terminal_commit_owner(&manager).await; - assert_eq!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("replacement race keeps lease live") - .playback_mode, - HlsLeasePlaybackMode::Live - ); - assert!(!session.read().await.has_terminal_tail_protections()); - assert!(!replacement.read().await.has_terminal_tail_protections()); - } - - #[tokio::test] - async fn hls_terminal_commit_submission_current_session_replaces_stale_incarnation_owner() { - let (manager, stale_session, proxy_session_id, lease_id, preparation, plan) = - prepared_terminal_commit_fixture("retry-current-session-replacement").await; - let session_key = stale_session.read().await.key.clone(); - let lease_guard = manager.access_leases.write().await; - - assert!(matches!( - commit_terminal_plan( - &manager, - &stale_session, - &lease_id, - &proxy_session_id, - &preparation, - 2_000, - Arc::clone(&plan), - ), - HlsTerminalCommitOutcome::LockBusy { .. } - )); - manager - .sessions - .remove_session(&session_key, &proxy_session_id) - .await - .expect("remove stale session incarnation"); - let (current_session, outcome) = - manager.get_or_create_session_with_outcome(session_key, b"secret", 2_000).await; - assert_eq!(outcome, HlsSessionStoreOutcome::Created); - assert!(!Arc::ptr_eq(&stale_session, ¤t_session)); - { - let mut current = current_session.write().await; - assert_eq!(complete_failed_acceptance_episode(&mut current, 2_000), preparation.origin_progress_generation); - } - - assert!(matches!( - commit_terminal_plan(&manager, ¤t_session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), - HlsTerminalCommitOutcome::LockBusy { .. } - )); - assert_eq!(manager.terminal_commit_retries.owner_count(), 1); - drop(lease_guard); - - wait_for_terminal_commit_owner(&manager).await; - assert!(matches!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("current session command terminalizes the lease") - .playback_mode, - HlsLeasePlaybackMode::TerminalTail(_) - )); - assert!(!stale_session.read().await.has_terminal_tail_protections()); - assert!(current_session.read().await.has_terminal_tail_protections()); - } - - #[tokio::test] - async fn hls_terminal_commit_submission_stale_session_cannot_replace_current_owner() { - let (manager, stale_session, proxy_session_id, lease_id, preparation, plan) = - prepared_terminal_commit_fixture("retry-stale-session-later").await; - let session_key = stale_session.read().await.key.clone(); - manager - .sessions - .remove_session(&session_key, &proxy_session_id) - .await - .expect("remove stale session incarnation"); - let (current_session, outcome) = - manager.get_or_create_session_with_outcome(session_key, b"secret", 2_000).await; - assert_eq!(outcome, HlsSessionStoreOutcome::Created); - { - let mut current = current_session.write().await; - assert_eq!(complete_failed_acceptance_episode(&mut current, 2_000), preparation.origin_progress_generation); - } - let lease_guard = manager.access_leases.write().await; - let index_guard = manager.sessions.hold_index_write_for_test().await; - - assert!(matches!( - commit_terminal_plan( - &manager, - ¤t_session, - &lease_id, - &proxy_session_id, - &preparation, - 2_000, - Arc::clone(&plan), - ), - HlsTerminalCommitOutcome::LockBusy { .. } - )); - assert!(matches!( - commit_terminal_plan(&manager, &stale_session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), - HlsTerminalCommitOutcome::LockBusy { .. } - )); - assert_eq!(manager.terminal_commit_retries.owner_count(), 1); - drop(index_guard); - drop(lease_guard); - - wait_for_terminal_commit_owner(&manager).await; - assert!(matches!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("current session owner survives stale later submission") - .playback_mode, - HlsLeasePlaybackMode::TerminalTail(_) - )); - assert!(!stale_session.read().await.has_terminal_tail_protections()); - assert!(current_session.read().await.has_terminal_tail_protections()); - } - - #[tokio::test] - async fn hls_terminal_commit_media_progress_before_retry_cancels_owner_and_keeps_lease_live() { - let (manager, session, proxy_session_id, lease_id, preparation, plan) = - prepared_terminal_commit_fixture("retry-recovery").await; - let lease_guard = manager.access_leases.write().await; - - assert!(matches!( - commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), - HlsTerminalCommitOutcome::LockBusy { .. } - )); - { - let mut session = session.write().await; - assert_eq!(session.origin_control.progress_generation, preparation.origin_progress_generation); - session.origin_control.record_media_progress(2_100, 12_000); - assert_eq!( - session.origin_control.progress_generation, - preparation.origin_progress_generation.saturating_add(1) - ); - } - drop(lease_guard); - - wait_for_terminal_commit_owner(&manager).await; - assert_eq!( - manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("recovered lease") - .playback_mode, - HlsLeasePlaybackMode::Live - ); - assert!(!session.read().await.has_terminal_tail_protections()); - } - - #[tokio::test] - async fn hls_terminal_commit_lease_end_before_retry_cancels_owner_without_mutation() { - let (manager, session, proxy_session_id, lease_id, preparation, plan) = - prepared_terminal_commit_fixture("retry-lease-end").await; - let mut lease_guard = manager.access_leases.write().await; - - assert!(matches!( - commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), - HlsTerminalCommitOutcome::LockBusy { .. } - )); - let _release = lease_guard.deny_access_lease(&lease_id, HlsAccessLeaseDenialMode::ImmediateEnd); - drop(lease_guard); - - wait_for_terminal_commit_owner(&manager).await; - let lease = manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("ended lease remains stored"); - assert_eq!(lease.state, HlsAccessLeaseState::Denied); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Ended); - assert!(!session.read().await.has_terminal_tail_protections()); - } - - #[tokio::test] - async fn hls_terminal_commit_new_media_generation_supersedes_prepared_plan() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); - assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); - let progress_generation = { - let mut session = session.write().await; - complete_failed_acceptance_episode(&mut session, 2_000) - }; - let preparation = manager - .prepare_access_lease_terminal_tail(terminal_preparation_request( - &lease_id, - &proxy_session_id, - progress_generation, - )) - .await - .expect("failed acceptance permits terminal preparation"); - - session.write().await.advance_media_readiness_generation(); - assert_eq!( - commit_terminal_plan( - &manager, - &session, - &lease_id, - &proxy_session_id, - &preparation, - 2_100, - terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), - ), - HlsTerminalCommitOutcome::SupersededGeneration - ); - - let lease = manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_200) - .await - .expect("lease remains available"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - assert!(!session.read().await.has_terminal_tail_protections()); - } - - #[tokio::test] - async fn hls_terminal_commit_lease_end_prevents_commit_and_gc_protection() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); - assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); - let progress_generation = { - let mut session = session.write().await; - complete_failed_acceptance_episode(&mut session, 2_000) - }; - let mut preparation_request = terminal_preparation_request(&lease_id, &proxy_session_id, progress_generation); - preparation_request.reserve.guaranteed_reserve_ms = 100_000; - preparation_request.reserve.guaranteed_media_horizon_ms = preparation_request - .reserve - .conservative_playback_position_ms - .saturating_add(preparation_request.reserve.guaranteed_reserve_ms); - preparation_request.cutover_timing = HlsLeaseCutoverTiming::from_reserve( - preparation_request.now_ms, - preparation_request.reserve.guaranteed_reserve_ms, - preparation_request.reserve.transition_margin, - None, - ); - let preparation = manager - .prepare_access_lease_terminal_tail(preparation_request) - .await - .expect("failed acceptance permits terminal preparation"); - - assert_eq!( - commit_terminal_plan( - &manager, - &session, - &lease_id, - &proxy_session_id, - &preparation, - 61_000, - terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), - ), - HlsTerminalCommitOutcome::LeaseNoLongerEligible - ); - - let lease = manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 61_000) - .await - .expect("expired lease remains stored until lifecycle cleanup"); - assert_eq!(lease.state, HlsAccessLeaseState::Expired); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Ended); - assert!(!session.read().await.has_terminal_tail_protections()); - } - - async fn assert_terminal_and_live_lease_modes( - manager: &HlsProxyManager, - proxy_session_id: &ProxySessionId, - terminal_lease_id: &HlsAccessLeaseId, - live_lease_id: &HlsAccessLeaseId, - ) { - assert!(matches!( - manager - .access_lease_response_snapshot(terminal_lease_id, proxy_session_id, 2_100) - .await - .expect("terminal lease") - .playback_mode, - HlsLeasePlaybackMode::TerminalTail(_) - )); - assert_eq!( - manager - .access_lease_response_snapshot(live_lease_id, proxy_session_id, 2_100) - .await - .expect("live lease") - .playback_mode, - HlsLeasePlaybackMode::Live - ); - } - - #[tokio::test] - async fn hls_cutover_policy_terminal_commit_uses_lease_deadline_and_keeps_farther_lease_live() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let terminal_lease_id = HlsAccessLeaseId("terminal".to_string()); - let live_lease_id = HlsAccessLeaseId("live".to_string()); - { - let mut leases = manager.access_leases.write().await; - leases.prepare_access_lease(access_lease(&terminal_lease_id.0, &proxy_session_id)); - leases.prepare_access_lease(access_lease(&live_lease_id.0, &proxy_session_id)); - } - assert!( - publish_manifest_snapshot(&manager, &terminal_lease_id, &proxy_session_id, manifest_snapshot(1), 2_000,) - .await - ); - assert!( - publish_manifest_snapshot(&manager, &live_lease_id, &proxy_session_id, manifest_snapshot(1), 2_000,).await - ); - let progress_generation = { - let mut session = session.write().await; - complete_failed_acceptance_episode(&mut session, 2_000) - }; - let far_reserve = HlsLeaseReserveSnapshot { - guaranteed_media_horizon_ms: 36_000, - conservative_playback_position_ms: 12_000, - guaranteed_reserve_ms: 24_000, - cutover_required: false, - ..cutover_reserve() - }; - let far_timing = HlsLeaseCutoverTiming::from_reserve( - 2_000, - far_reserve.guaranteed_reserve_ms, - far_reserve.transition_margin, - None, - ); - assert!(manager - .prepare_access_lease_terminal_tail(HlsTerminalTailPreparationRequest { - lease_id: &live_lease_id, - proxy_session_id: &proxy_session_id, - manifest_snapshot_generation: 1, - cursor_generation: 0, - reserve: far_reserve, - cutover_timing: far_timing, - commit_window: HlsTerminalCommitWindow::NotDue, - now_ms: 2_000, - origin_progress_generation: progress_generation, - media_readiness_generation: 0, - last_media_progress_at_ms: None, - }) - .await - .is_none()); - let preparation = manager - .prepare_access_lease_terminal_tail(terminal_preparation_request( - &terminal_lease_id, - &proxy_session_id, - progress_generation, - )) - .await - .expect("terminal preparation"); - assert_eq!( - preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), - 2_000_u64.saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()) - ); - assert_eq!(far_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), 14_000); - let plan = terminal_plan(preparation.decision_generation, &proxy_session_id, &terminal_lease_id); - let expected_protection = Arc::clone(&plan.protected_base_proxy_seqs); - - assert_eq!( - commit_terminal_plan(&manager, &session, &terminal_lease_id, &proxy_session_id, &preparation, 2_000, plan,), - HlsTerminalCommitOutcome::Committed - ); - - let session = session.read().await; - assert_eq!( - session.terminal_tail_protection(&terminal_lease_id).map(|protection| &protection.base_proxy_seqs), - Some(&expected_protection) - ); - assert_eq!( - session.origin_control.progress_phase, - super::super::origin_progress::HlsOriginProgressPhase::TerminalPartial - ); - drop(session); - assert_terminal_and_live_lease_modes(&manager, &proxy_session_id, &terminal_lease_id, &live_lease_id).await; - } - - #[tokio::test] - async fn resource_denial_does_not_destroy_existing_terminal_tail() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("terminal".to_string()); - let plan = terminal_plan(7, &proxy_session_id, &lease_id); - let mut lease = access_lease(&lease_id.0, &proxy_session_id); - lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(Arc::clone(&plan)); - manager.access_leases.write().await.prepare_access_lease(lease); - { - let mut session = session.write().await; - assert_eq!( - session.install_terminal_tail_protection( - lease_id.clone(), - HlsTerminalTailProtection { - generation: plan.generation, - base_proxy_seqs: Arc::clone(&plan.protected_base_proxy_seqs), - key_bindings: plan.key_bindings(), - }, - ), - super::super::session::HlsTerminalTailProtectionInstall::Installed - ); - } - - assert_eq!( - manager.deny_access_lease(&lease_id, HlsAccessLeaseDenialMode::PreserveCommittedFiniteTail,).await, - HlsAccessLeaseDenialOutcome::FiniteDecisionPreserved - ); - - let denied = manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("denied lease remains stored until lifecycle cleanup"); - assert_eq!(denied.state, HlsAccessLeaseState::Denied); - assert_eq!(denied.playback_mode, HlsLeasePlaybackMode::TerminalTail(Arc::clone(&plan))); - assert!(session.read().await.terminal_tail_protection(&lease_id).is_some()); - - assert_eq!( - manager.deny_access_lease(&lease_id, HlsAccessLeaseDenialMode::ImmediateEnd).await, - HlsAccessLeaseDenialOutcome::FiniteDecisionPreserved - ); - assert!(session.read().await.terminal_tail_protection(&lease_id).is_some()); - - manager.remove_access_lease(&lease_id).await; - assert!(session.read().await.terminal_tail_protection(&lease_id).is_none()); - assert_eq!( - manager.access_leases.write().await.deny_access_lease(&lease_id, HlsAccessLeaseDenialMode::ImmediateEnd), - HlsAccessLeaseDenialOutcome::UnknownLease - ); - } - - #[tokio::test] - async fn terminal_tail_protection_is_retained_until_bounded_lease_removal() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("terminal-stale-protection".to_string()); - let plan = terminal_plan(7, &proxy_session_id, &lease_id); - let stale_generation = HlsTerminalTailGeneration(plan.generation.0.saturating_add(1)); - let mut lease = access_lease(&lease_id.0, &proxy_session_id); - lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(Arc::clone(&plan)); - manager.access_leases.write().await.prepare_access_lease(lease); - assert_eq!( - session.write().await.install_terminal_tail_protection( - lease_id.clone(), - HlsTerminalTailProtection { - generation: stale_generation, - base_proxy_seqs: Arc::from([9_999]), - key_bindings: Arc::from([]), - }, - ), - super::super::session::HlsTerminalTailProtectionInstall::Installed - ); - - assert_eq!( - manager.deny_access_lease(&lease_id, HlsAccessLeaseDenialMode::PreserveCommittedFiniteTail,).await, - HlsAccessLeaseDenialOutcome::FiniteDecisionPreserved - ); - - assert!(session.read().await.terminal_tail_protection(&lease_id).is_some()); - manager.remove_access_lease(&lease_id).await; - assert!(session.read().await.terminal_tail_protection(&lease_id).is_none()); - } - - #[tokio::test] - async fn stale_lease_removal_preparation_cannot_release_replacement_protection() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("replacement-protection".to_string()); - let old_plan = terminal_plan(7, &proxy_session_id, &lease_id); - let mut old_lease = access_lease(&lease_id.0, &proxy_session_id); - old_lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(Arc::clone(&old_plan)); - manager.access_leases.write().await.prepare_access_lease(old_lease); - assert_eq!( - session.write().await.install_terminal_tail_protection( - lease_id.clone(), - HlsTerminalTailProtection { - generation: old_plan.generation, - base_proxy_seqs: Arc::clone(&old_plan.protected_base_proxy_seqs), - key_bindings: old_plan.key_bindings(), - }, - ), - super::super::session::HlsTerminalTailProtectionInstall::Installed - ); - let preparation = manager - .access_leases - .read() - .await - .prepare_access_lease_removal(&lease_id) - .expect("old lease removal preparation"); - - let replacement_plan = terminal_plan(8, &proxy_session_id, &lease_id); - let mut replacement = access_lease(&lease_id.0, &proxy_session_id); - replacement.issued_at_ms = 2_000; - replacement.playback_mode = HlsLeasePlaybackMode::TerminalTail(Arc::clone(&replacement_plan)); - assert!(manager.access_leases.write().await.prepare_access_lease(replacement)); - assert_eq!( - session.write().await.install_terminal_tail_protection( - lease_id.clone(), - HlsTerminalTailProtection { - generation: replacement_plan.generation, - base_proxy_seqs: Arc::clone(&replacement_plan.protected_base_proxy_seqs), - key_bindings: replacement_plan.key_bindings(), - }, - ), - super::super::session::HlsTerminalTailProtectionInstall::Installed - ); - - assert!(!manager.remove_prepared_access_lease(&lease_id, &preparation).await); - assert_eq!( - manager - .access_leases - .write() - .await - .response_snapshot(&lease_id, &proxy_session_id, 2_000) - .map(|lease| lease.issued_at_ms), - Some(2_000) - ); - assert_eq!( - session.read().await.terminal_tail_protection(&lease_id).map(|protection| protection.generation), - Some(replacement_plan.generation) - ); - } - - #[tokio::test] - async fn terminal_unavailable_protection_is_retained_until_bounded_lease_removal() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("terminal-unavailable".to_string()); - let decision_generation = 11; - let generation = HlsTerminalTailGeneration(decision_generation); - let mut lease = access_lease(&lease_id.0, &proxy_session_id); - lease.playback_mode = HlsLeasePlaybackMode::TerminalUnavailable { - decision_generation, - reason: HlsTerminalTailCompatibility::ProtectionCapacityExceeded, - }; - manager.access_leases.write().await.prepare_access_lease(lease); - assert_eq!( - session.write().await.install_terminal_tail_protection( - lease_id.clone(), - HlsTerminalTailProtection { - generation, - base_proxy_seqs: Arc::from([41_u64]), - key_bindings: Arc::from([]), - }, - ), - super::super::session::HlsTerminalTailProtectionInstall::Installed - ); - - assert_eq!( - manager.deny_access_lease(&lease_id, HlsAccessLeaseDenialMode::PreserveCommittedFiniteTail,).await, - HlsAccessLeaseDenialOutcome::FiniteDecisionPreserved - ); - - let denied = manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) - .await - .expect("denied lease remains stored until lifecycle cleanup"); - assert_eq!(denied.state, HlsAccessLeaseState::Denied); - assert!(matches!(denied.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); - assert!(session.read().await.terminal_tail_protection(&lease_id).is_some()); - manager.remove_access_lease(&lease_id).await; - assert!(session.read().await.terminal_tail_protection(&lease_id).is_none()); - } - - #[tokio::test] - async fn hls_terminal_commit_protection_capacity_has_typed_unavailable_outcome() { - let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); - let manager = HlsProxyManager::with_hls_cache_config(&config); - let (session, _) = - manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("overflow".to_string()); - manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); - assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); - let progress_generation = { - let mut session = session.write().await; - for index in 0..HLS_TERMINAL_TAIL_PROTECTION_CAPACITY { - session.install_terminal_tail_protection( - HlsAccessLeaseId(format!("occupied-{index}")), - HlsTerminalTailProtection { - generation: HlsTerminalTailGeneration(1), - base_proxy_seqs: Arc::from([u64::try_from(index).unwrap_or(u64::MAX)]), - key_bindings: Arc::from([]), - }, - ); - } - complete_failed_acceptance_episode(&mut session, 2_000) - }; - let preparation = manager - .prepare_access_lease_terminal_tail(terminal_preparation_request( - &lease_id, - &proxy_session_id, - progress_generation, - )) - .await - .expect("terminal preparation"); - - assert_eq!( - commit_terminal_plan( - &manager, - &session, - &lease_id, - &proxy_session_id, - &preparation, - 2_000, - terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), - ), - HlsTerminalCommitOutcome::Committed - ); - - let lease = manager - .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_100) - .await - .expect("terminal unavailable lease remains stored"); - assert!(matches!( - lease.playback_mode, - HlsLeasePlaybackMode::TerminalUnavailable { - reason: HlsTerminalTailCompatibility::ProtectionCapacityExceeded, - .. - } - )); - let session = session.read().await; - assert_eq!(session.terminal_tail_protection_count(), HLS_TERMINAL_TAIL_PROTECTION_CAPACITY); - assert!(session.terminal_tail_protection(&lease_id).is_none()); - } -} +mod tests; + +mod account_overlap; +mod activity; +mod cleanup; +mod config; +mod lifecycle; +mod publication; +mod terminal; + +pub use self::lifecycle::exec_hls_lifecycle; +#[cfg(test)] +use self::lifecycle::hls_session_idle_protection_retry_at; +#[cfg(test)] +use self::publication::HlsRecoveryExecutionState; +use self::{ + account_overlap::{HlsAccountOverlapCooldown, HlsAccountOverlapCooldownKey}, + activity::hls_key_readiness_evidence_is_current, + config::HlsProxyRuntimeConfig, + publication::{ + hls_acceptance_recovery_snapshot, hls_estimated_recovery_completion_at, HlsAcceptanceRecoverySnapshot, + }, +}; diff --git a/backend/hls/src/manager/account_overlap.rs b/backend/hls/src/manager/account_overlap.rs new file mode 100644 index 000000000..17660374c --- /dev/null +++ b/backend/hls/src/manager/account_overlap.rs @@ -0,0 +1,108 @@ +use super::HlsProxyManager; +use log::debug; +use shared::utils::sanitize_sensitive_info; +use std::sync::Arc; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub(super) struct HlsAccountOverlapCooldownKey { + pub(super) input_name: Arc, + pub(super) account_name: Arc, +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct HlsAccountOverlapCooldown { + pub(super) until_ms: u64, +} + +#[derive(Debug, Clone, Copy)] +enum HlsAccountOverlapCooldownReason { + ReclaimedByOriginalOwner, + SpeculativePromoted, +} + +impl HlsAccountOverlapCooldownReason { + fn as_log_reason(self) -> &'static str { + match self { + Self::ReclaimedByOriginalOwner => "reclaimed-by-original-owner", + Self::SpeculativePromoted => "speculative-promoted", + } + } +} + +impl HlsProxyManager { + pub async fn is_account_overlap_cooling_down( + &self, + input_name: &Arc, + account_name: &Arc, + now_ms: u64, + ) -> bool { + let key = + HlsAccountOverlapCooldownKey { input_name: Arc::clone(input_name), account_name: Arc::clone(account_name) }; + let mut cooldowns = self.account_overlap_cooldowns.write().await; + let Some(cooldown) = cooldowns.get(&key).copied() else { + return false; + }; + if now_ms >= cooldown.until_ms { + cooldowns.remove(&key); + return false; + } + true + } + + pub async fn mark_account_overlap_reclaimed_cooldown( + &self, + input_name: Arc, + account_name: Arc, + now_ms: u64, + hard_active_window_ms: u64, + ) { + self.mark_account_overlap_cooldown( + input_name, + account_name, + now_ms, + hard_active_window_ms, + HlsAccountOverlapCooldownReason::ReclaimedByOriginalOwner, + ) + .await; + } + + pub async fn mark_account_overlap_promoted_cooldown( + &self, + input_name: Arc, + account_name: Arc, + now_ms: u64, + hard_active_window_ms: u64, + ) { + self.mark_account_overlap_cooldown( + input_name, + account_name, + now_ms, + hard_active_window_ms, + HlsAccountOverlapCooldownReason::SpeculativePromoted, + ) + .await; + } + + async fn mark_account_overlap_cooldown( + &self, + input_name: Arc, + account_name: Arc, + now_ms: u64, + hard_active_window_ms: u64, + reason: HlsAccountOverlapCooldownReason, + ) { + let until_ms = now_ms.saturating_add(hard_active_window_ms); + if until_ms <= now_ms { + return; + } + let key = HlsAccountOverlapCooldownKey { input_name, account_name }; + self.account_overlap_cooldowns.write().await.insert(key.clone(), HlsAccountOverlapCooldown { until_ms }); + debug!( + "HLS account overlap cooldown set for input {} account {} until {} ms after {}", + sanitize_sensitive_info(key.input_name.as_ref()), + sanitize_sensitive_info(key.account_name.as_ref()), + until_ms, + reason.as_log_reason() + ); + } +} diff --git a/backend/hls/src/manager/activity.rs b/backend/hls/src/manager/activity.rs new file mode 100644 index 000000000..b066156d7 --- /dev/null +++ b/backend/hls/src/manager/activity.rs @@ -0,0 +1,271 @@ +use super::{ + HlsAccessLeaseId, HlsAccessLeaseStore, HlsMediaActivityCommitOutcome, HlsMediaLeaseIdentity, + HlsPlaybackRequestToken, HlsProxyManager, HlsSessionHandle, ProxySessionId, + HLS_MEDIA_ACTIVITY_FALLBACK_LOCK_RETRIES, HLS_STATE_CAS_LOCK_RETRIES, +}; +use std::sync::Arc; + +#[derive(Debug, Clone, Copy)] +enum HlsMediaActivityCommitKind { + Access, + Delivered, + LiveSegmentCompletion(HlsPlaybackRequestToken), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum HlsMediaActivityCommitAttempt { + LockBusy, + Completed { outcome: HlsMediaActivityCommitOutcome, evidence_changed: bool }, +} + +pub(super) fn hls_key_readiness_evidence_is_current(valid_until_ms: Option, now_ms: u64) -> bool { + valid_until_ms.is_none_or(|valid_until_ms| valid_until_ms >= now_ms) +} + +impl HlsProxyManager { + pub async fn record_access_lease_segment_request_started_if_identity_matches( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + lease_identity: HlsMediaLeaseIdentity, + proxy_seq: u64, + requested_at_ms: u64, + ) -> Option { + self.access_leases.write().await.record_segment_request_started_if_identity_matches( + lease_id, + proxy_session_id, + lease_identity, + proxy_seq, + requested_at_ms, + ) + } + + pub async fn record_access_lease_segment_request_completed_and_mark_media_if_identity_matches( + &self, + session: &HlsSessionHandle, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + lease_identity: HlsMediaLeaseIdentity, + token: HlsPlaybackRequestToken, + completed_at_ms: u64, + ) -> HlsMediaActivityCommitOutcome { + self.commit_media_activity_if_identity_matches( + session, + lease_id, + proxy_session_id, + lease_identity, + completed_at_ms, + HlsMediaActivityCommitKind::LiveSegmentCompletion(token), + ) + .await + } + + pub async fn mark_authorized_media_access_for_lease_if_identity_matches( + &self, + session: &HlsSessionHandle, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + lease_identity: HlsMediaLeaseIdentity, + now_ms: u64, + ) -> HlsMediaActivityCommitOutcome { + self.commit_media_activity_if_identity_matches( + session, + lease_id, + proxy_session_id, + lease_identity, + now_ms, + HlsMediaActivityCommitKind::Access, + ) + .await + } + + pub async fn mark_delivered_media_for_lease( + &self, + session: &HlsSessionHandle, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + lease_identity: HlsMediaLeaseIdentity, + now_ms: u64, + ) -> HlsMediaActivityCommitOutcome { + self.commit_media_activity_if_identity_matches( + session, + lease_id, + proxy_session_id, + lease_identity, + now_ms, + HlsMediaActivityCommitKind::Delivered, + ) + .await + } + + async fn commit_media_activity_if_identity_matches( + &self, + session: &HlsSessionHandle, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + lease_identity: HlsMediaLeaseIdentity, + now_ms: u64, + kind: HlsMediaActivityCommitKind, + ) -> HlsMediaActivityCommitOutcome { + let Some(current_session) = self.sessions.get_by_proxy_session_id(proxy_session_id).await else { + return HlsMediaActivityCommitOutcome::StaleLeaseIdentity; + }; + if !Arc::ptr_eq(¤t_session, session) { + return HlsMediaActivityCommitOutcome::StaleLeaseIdentity; + } + for attempt in 0..HLS_STATE_CAS_LOCK_RETRIES { + match self.try_commit_media_activity(session, lease_id, proxy_session_id, lease_identity, now_ms, kind) { + HlsMediaActivityCommitAttempt::Completed { outcome, evidence_changed } => { + return self + .finish_media_activity_commit(session, proxy_session_id, outcome, evidence_changed) + .await; + } + HlsMediaActivityCommitAttempt::LockBusy => { + if attempt.saturating_add(1) < HLS_STATE_CAS_LOCK_RETRIES { + tokio::task::yield_now().await; + } + } + } + } + + // Media activity must not be dropped under sustained lock contention. + // Wait for one contended lock at a time, then retry the established + // lease-store -> session acquisition without awaiting under either + // write guard. + let mut committed = None; + for attempt in 0..HLS_MEDIA_ACTIVITY_FALLBACK_LOCK_RETRIES { + let mut leases = self.access_leases.write().await; + let Ok(mut session_guard) = session.try_write() else { + drop(leases); + let session_wait = session.write().await; + drop(session_wait); + if attempt.saturating_add(1) < HLS_MEDIA_ACTIVITY_FALLBACK_LOCK_RETRIES { + tokio::task::yield_now().await; + } + continue; + }; + let HlsMediaActivityCommitAttempt::Completed { outcome, evidence_changed } = + Self::commit_media_activity_locked( + &mut leases, + &mut session_guard, + lease_id, + proxy_session_id, + lease_identity, + now_ms, + kind, + ) + else { + drop(session_guard); + drop(leases); + continue; + }; + drop(session_guard); + drop(leases); + committed = Some((outcome, evidence_changed)); + break; + } + let Some((outcome, evidence_changed)) = committed else { + return HlsMediaActivityCommitOutcome::DeferredLockContention; + }; + self.finish_media_activity_commit(session, proxy_session_id, outcome, evidence_changed).await + } + + async fn finish_media_activity_commit( + &self, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + outcome: HlsMediaActivityCommitOutcome, + evidence_changed: bool, + ) -> HlsMediaActivityCommitOutcome { + match outcome { + HlsMediaActivityCommitOutcome::Committed => { + if evidence_changed { + self.segment_cache.notify_capacity_protection_changed(); + self.notify_session_evidence_changed(proxy_session_id); + } + self.schedule_session_idle_for_handle(session).await; + } + HlsMediaActivityCommitOutcome::StaleLeaseIdentity + | HlsMediaActivityCommitOutcome::DeferredLockContention => {} + } + outcome + } + + fn try_commit_media_activity( + &self, + session: &HlsSessionHandle, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + lease_identity: HlsMediaLeaseIdentity, + now_ms: u64, + kind: HlsMediaActivityCommitKind, + ) -> HlsMediaActivityCommitAttempt { + // Media-completion lock order matches terminal publication: lease + // store -> session. Both locks are non-blocking and no async work runs + // while either guard is held. + let Ok(mut leases) = self.access_leases.try_write() else { + return HlsMediaActivityCommitAttempt::LockBusy; + }; + let Ok(mut session) = session.try_write() else { + return HlsMediaActivityCommitAttempt::LockBusy; + }; + let result = Self::commit_media_activity_locked( + &mut leases, + &mut session, + lease_id, + proxy_session_id, + lease_identity, + now_ms, + kind, + ); + drop(session); + drop(leases); + result + } + + fn commit_media_activity_locked( + leases: &mut HlsAccessLeaseStore, + session: &mut super::super::HlsSession, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + lease_identity: HlsMediaLeaseIdentity, + now_ms: u64, + kind: HlsMediaActivityCommitKind, + ) -> HlsMediaActivityCommitAttempt { + if session.proxy_session_id != *proxy_session_id { + return HlsMediaActivityCommitAttempt::Completed { + outcome: HlsMediaActivityCommitOutcome::StaleLeaseIdentity, + evidence_changed: false, + }; + } + let (current, capacity_protection_released) = match kind { + HlsMediaActivityCommitKind::Access | HlsMediaActivityCommitKind::Delivered => { + (leases.media_identity_is_current(lease_id, proxy_session_id, lease_identity, now_ms), false) + } + HlsMediaActivityCommitKind::LiveSegmentCompletion(token) => { + let completion = leases.record_segment_request_completed_if_identity_matches( + lease_id, + proxy_session_id, + lease_identity, + token, + now_ms, + ); + (completion.is_some(), matches!(completion, Some(super::super::HlsPlaybackCompletionOutcome::Advanced))) + } + }; + if !current { + return HlsMediaActivityCommitAttempt::Completed { + outcome: HlsMediaActivityCommitOutcome::StaleLeaseIdentity, + evidence_changed: false, + }; + } + session.mark_authorized_media_access(now_ms); + if lease_identity.is_live() && matches!(kind, HlsMediaActivityCommitKind::Delivered) { + session.activity.last_delivered_media_at_ms = Some(now_ms); + } + HlsMediaActivityCommitAttempt::Completed { + outcome: HlsMediaActivityCommitOutcome::Committed, + evidence_changed: capacity_protection_released, + } + } +} diff --git a/backend/hls/src/manager/cleanup.rs b/backend/hls/src/manager/cleanup.rs new file mode 100644 index 000000000..2a9817f10 --- /dev/null +++ b/backend/hls/src/manager/cleanup.rs @@ -0,0 +1,285 @@ +use super::{ + safe_proxy_session_id, HlsAccessLeaseSessionSnapshot, HlsExpiredSessionMarker, HlsProxyManager, HlsSessionHandle, + ProxySessionId, +}; +use crate::sync_ext::MutexExt; +use log::debug; +use std::{ + io, + sync::{atomic::Ordering, Arc}, +}; +use tuliprox_session::{ActiveProviderManager, ActiveUserManager}; + +#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] +pub(super) struct HlsProxySessionCleanupStats { + access_leases: usize, + repair_windows: usize, + repair_generations: usize, + repair_candidates: usize, + repair_object_metadata: usize, + repair_watchdog_metadata: usize, + repair_watchdog_locks: usize, + qos_access_leases: usize, +} + +impl HlsProxySessionCleanupStats { + fn did_cleanup(self) -> bool { + self.access_leases > 0 + || self.repair_windows > 0 + || self.repair_generations > 0 + || self.repair_candidates > 0 + || self.repair_object_metadata > 0 + || self.repair_watchdog_metadata > 0 + || self.repair_watchdog_locks > 0 + || self.qos_access_leases > 0 + } +} + +impl HlsProxyManager { + pub fn reserve_switch_segment_cleanup( + &self, + key: super::super::SegmentCacheKey, + ) -> Option { + self.gc.reserve_switch_segment_cleanup(key) + } + + pub fn reserve_switch_map_cleanup( + &self, + key: super::super::MapCacheKey, + ) -> Option { + self.gc.reserve_switch_map_cleanup(key) + } + + pub fn has_pending_switch_cleanup( + &self, + segment_key: &super::super::SegmentCacheKey, + map_key: Option<&super::super::MapCacheKey>, + ) -> bool { + self.gc.has_pending_switch_cleanup(segment_key, map_key) + } + + #[cfg(any(test, feature = "test-support"))] + pub fn cache_deletion_queue_usage(&self) -> (usize, usize) { self.gc.cache_deletion_queue_usage() } + + pub(super) async fn cleanup_proxy_session_state( + &self, + proxy_session_id: &ProxySessionId, + reason: &'static str, + ) -> HlsProxySessionCleanupStats { + self.availability_reevaluations.cancel_session(proxy_session_id); + self.terminal_pending.cancel_session(proxy_session_id); + self.terminal_commit_retries.cancel_session(proxy_session_id); + let before = self.segment_repair.stats().await; + let removed_leases = self.access_leases.write().await.remove_access_leases_for_session(proxy_session_id); + let username = removed_leases.first().map(|lease| lease.username.clone()); + self.sessions.update_expired_session_marker_username(proxy_session_id, username).await; + let removed_lease_ids = removed_leases.iter().map(|lease| lease.lease_id.clone()).collect::>(); + for lease_id in &removed_lease_ids { + self.standalone_custom_access.remove(lease_id); + } + self.segment_repair.remove_proxy_session_state(proxy_session_id, &removed_lease_ids).await; + for lease_id in &removed_lease_ids { + self.startup_observability.remove_access_lease(lease_id); + } + let removed_qos = self.qos.remove_access_leases(&removed_lease_ids).await; + let removed_qos = removed_qos.saturating_add(self.qos.remove_proxy_session_state(proxy_session_id).await); + let after = self.segment_repair.stats().await; + let stats = HlsProxySessionCleanupStats { + access_leases: removed_lease_ids.len(), + repair_windows: before.windows.saturating_sub(after.windows), + repair_generations: before.generations.saturating_sub(after.generations), + repair_candidates: before.checked_candidates.saturating_sub(after.checked_candidates), + repair_object_metadata: before.object_metadata.saturating_sub(after.object_metadata), + repair_watchdog_metadata: before.watchdog_metadata.saturating_sub(after.watchdog_metadata), + repair_watchdog_locks: before.watchdog_locks.saturating_sub(after.watchdog_locks), + qos_access_leases: removed_qos, + }; + if stats.did_cleanup() { + debug!( + "HLS proxy session state cleaned: proxy_session={} reason={} access_leases={} repair_windows={} repair_generations={} repair_candidates={} repair_object_metadata={} repair_watchdog_metadata={} repair_watchdog_locks={} qos_access_leases={}", + safe_proxy_session_id(proxy_session_id), + reason, + stats.access_leases, + stats.repair_windows, + stats.repair_generations, + stats.repair_candidates, + stats.repair_object_metadata, + stats.repair_watchdog_metadata, + stats.repair_watchdog_locks, + stats.qos_access_leases + ); + } + stats + } + + pub async fn expired_session_marker( + &self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Option { + self.sessions + .expired_session_marker(proxy_session_id, now_ms, self.session_idle_timeout_ms().saturating_mul(2).max(1)) + .await + } + + async fn cleanup_all_runtime_state(&self, reason: &'static str) { + self.availability_reevaluations.clear(); + self.terminal_pending.clear(); + self.terminal_commit_retries.clear(); + let removed_access_leases = self.access_leases.write().await.clear(); + self.standalone_custom_access.clear(); + self.terminal_commit_retries.clear(); + for session in self.sessions.list_sessions().await { + session.write().await.clear_terminal_tail_protections(); + } + self.account_overlap_cooldowns.write().await.clear(); + let removed_qos = self.qos.clear().await; + let before = self.segment_repair.stats().await; + self.segment_repair.clear_runtime_state().await; + self.startup_observability.clear(); + if removed_access_leases > 0 + || before.windows > 0 + || before.generations > 0 + || before.checked_candidates > 0 + || before.metadata > 0 + || before.object_metadata > 0 + || before.locks > 0 + || before.watchdog_metadata > 0 + || before.watchdog_locks > 0 + || removed_qos > 0 + { + debug!( + "HLS runtime state cleaned: reason={} access_leases={} repair_windows={} repair_generations={} repair_candidates={} repair_metadata={} repair_object_metadata={} repair_locks={} repair_watchdog_metadata={} repair_watchdog_locks={} qos_access_leases={}", + reason, + removed_access_leases, + before.windows, + before.generations, + before.checked_candidates, + before.metadata, + before.object_metadata, + before.locks, + before.watchdog_metadata, + before.watchdog_locks, + removed_qos + ); + } + } + + async fn cleanup_after_garbage_collection(&self, report: &super::super::GarbageCollectionReport) { + if report.secret_cache_invalidated { + self.cleanup_all_runtime_state("secret-cache-invalidated").await; + return; + } + for proxy_session_id in &report.removed_session_ids { + self.cleanup_proxy_session_state(proxy_session_id, "gc-session-removed").await; + } + } + + pub async fn run_garbage_collection_once(&self, now_ms: u64) -> io::Result { + if !self.is_enabled() { + return Ok(super::super::GarbageCollectionReport::default()); + } + let report = self.gc.run_once(now_ms).await?; + self.cleanup_after_garbage_collection(&report).await; + if self.revision_store.has_created_revisions() { + for handle in self.sessions.list_sessions().await { + let snapshot = { + let mut session = handle.write().await; + let super::super::HlsSession { segments, startup, proxy_session_id, .. } = &mut *session; + startup.as_mut().map(|startup| { + startup.revisions.retain(|seq, _| segments.contains_key(seq)); + (proxy_session_id.clone(), segments.first_key_value().map_or(u64::MAX, |(seq, _)| *seq)) + }) + }; + if let Some((id, floor)) = snapshot { + self.access_leases.write().await.prune_revision_bindings(&id, floor); + } + } + } + if self.revision_reconciliation_pending.load(Ordering::Acquire) { + let deleted = self.segment_cache.delete_orphan_revision_files(std::time::SystemTime::now()).await?; + if deleted < super::super::cache::MAX_TEMP_FILE_CLEANUP_CANDIDATES_PER_RUN { + self.revision_reconciliation_pending.store(false, Ordering::Release); + } + } + for revision in self.revision_store.retire_unpinned()? { + revision.file_pin.lock_unpoisoned().take(); + if self.segment_cache.delete_if_inactive(&revision.key).await.is_err() { + self.revision_store.requeue_retired(revision)?; + } + } + if self.revision_store.has_created_revisions() { + let bytes = self.segment_cache.revision_disk_bytes(); + self.metrics.record_revision_disk_bytes(bytes); + let (fills, replay_bytes, repairs) = self.progressive_budget.usage_snapshot(); + self.metrics.record_progressive_usage(fills, replay_bytes, repairs); + debug!( + "HLS progressive usage: active_fills={fills} replay_bytes={replay_bytes} deferred_repairs={repairs}" + ); + debug!("HLS revision disk usage: hls_revision_disk_bytes={bytes}"); + } + Ok(report) + } + + pub async fn sync_session_access_lease_count_and_detach_if_needed( + &self, + active_users: &Arc, + _active_provider: &Arc, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) { + let snapshot = self.reconcile_session_access_lease_snapshot(session, proxy_session_id, now_ms).await; + for release in &snapshot.idle_releases { + active_users + .release_session_streams_and_counted_reservation(&release.username, &release.user_session_token) + .await; + debug!( + "HLS access lease idled: lease={} proxy_session={} user_session={}", + super::super::safe_hls_access_lease_id(&release.lease_id), + safe_proxy_session_id(proxy_session_id), + super::super::safe_user_session_token(&release.user_session_token) + ); + } + } + + pub(super) async fn reconcile_session_access_lease_snapshot( + &self, + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> HlsAccessLeaseSessionSnapshot { + // Keep snapshot creation and application in the established lease-store -> session + // lock transaction. Otherwise a publication/removal can be overwritten by a stale + // wholesale binding reconciliation. + let mut leases = self.access_leases.write().await; + let snapshot = leases.session_snapshot(proxy_session_id, now_ms); + let mut session = session.write().await; + session.activity.active_access_lease_count = snapshot.active_count; + session.reconcile_effective_origin_acquire_policy(snapshot.effective_origin_policy, now_ms); + session.transient.reconcile_finalized_manifest_lease_bindings(&snapshot.finalized_transient_manifest_bindings); + snapshot + } + + pub async fn sync_all_session_access_leases_and_detach_if_needed( + &self, + active_users: &Arc, + active_provider: &Arc, + now_ms: u64, + ) { + if !self.is_enabled() { + return; + } + for session in self.sessions.list_sessions().await { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + self.sync_session_access_lease_count_and_detach_if_needed( + active_users, + active_provider, + &session, + &proxy_session_id, + now_ms, + ) + .await; + } + } +} diff --git a/backend/hls/src/manager/config.rs b/backend/hls/src/manager/config.rs new file mode 100644 index 000000000..b415011bb --- /dev/null +++ b/backend/hls/src/manager/config.rs @@ -0,0 +1,212 @@ +use super::{ + build_rewrite_secret_fingerprint, GarbageCollectionPolicy, HlsAccessLeaseStore, + HlsAvailabilityReevaluationCoordinator, HlsCacheMetrics, HlsGarbageCollector, HlsLifecycleManager, + HlsMapWorkerPool, HlsPreparedTerminalBundleCache, HlsProxyManager, HlsQosRegistry, HlsSegmentCache, + HlsSegmentRepairManager, HlsSegmentWorkerPool, HlsSessionStore, HlsStandaloneCustomAccessStore, + HlsStartupObservability, HlsTerminalCommitClock, HlsTerminalCommitRetryCoordinator, HlsTerminalPendingCoordinator, + SegmentFetchPolicy, TransientResourceStore, +}; +use arc_swap::ArcSwap; +use std::{ + collections::HashMap, + path::PathBuf, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; +use tokio::sync::{RwLock, Semaphore}; +use tuliprox_core::model::{AppConfig, HlsCacheConfig, HlsManifestRecoveryBurstConfig, StripConfig}; + +#[derive(Debug, Clone)] +pub(super) struct HlsProxyRuntimeConfig { + pub(super) enabled: bool, + pub(super) segment_fetch_policy: SegmentFetchPolicy, + pub(super) cache_duration_seconds: u64, + pub(super) strip: StripConfig, + pub(super) origin_manifest_timeout_ms: u64, + pub(super) initial_manifest_wait_timeout_secs: u64, + pub(super) manifest_recovery_burst: HlsManifestRecoveryBurstConfig, + pub(super) transient_resource_ttl_ms: u64, + pub(super) gc_policy: GarbageCollectionPolicy, + pub(super) rewrite_secret_fingerprint: String, + pub(super) startup: tuliprox_core::model::HlsStartupConfig, +} + +impl HlsProxyRuntimeConfig { + pub(super) fn from_config(config: &HlsCacheConfig, rewrite_secret: &[u8]) -> Self { + Self::from_config_with_enabled(config, rewrite_secret, true) + } + + pub(super) fn from_config_with_enabled(config: &HlsCacheConfig, rewrite_secret: &[u8], enabled: bool) -> Self { + Self { + enabled, + segment_fetch_policy: SegmentFetchPolicy::from_config(config), + cache_duration_seconds: config.cache_duration.get(), + strip: config.strip.clone(), + origin_manifest_timeout_ms: config.origin_manifest_timeout_ms.get(), + initial_manifest_wait_timeout_secs: config.initial_manifest_wait_timeout_secs.get(), + manifest_recovery_burst: config.manifest_recovery_burst.clone(), + transient_resource_ttl_ms: config.cache_duration.as_millis().get(), + gc_policy: GarbageCollectionPolicy::from_config(config), + rewrite_secret_fingerprint: build_rewrite_secret_fingerprint(rewrite_secret), + startup: config.startup.clone(), + } + } +} + +impl HlsProxyManager { + pub fn new() -> Self { + let default_dto = shared::model::HlsCacheConfigDto::default(); + let default_config = HlsCacheConfig::from(&default_dto); + Self::with_hls_cache_config(&default_config) + } + + pub fn from_hls_cache_config(config: Option<&HlsCacheConfig>) -> Self { + Self::from_hls_cache_config_and_secret(config, &[]) + } + + pub fn from_hls_cache_config_and_secret(config: Option<&HlsCacheConfig>, rewrite_secret: &[u8]) -> Self { + let default_config; + let (config, enabled) = if let Some(config) = config { + (config, true) + } else { + default_config = HlsCacheConfig::from(&shared::model::HlsCacheConfigDto::default()); + (&default_config, false) + }; + Self::with_hls_cache_config_and_secret_enabled(config, rewrite_secret, enabled) + } + + pub fn with_cache_settings(cache_path: impl Into, cache_duration_seconds: u64) -> Self { + let default_dto = shared::model::HlsCacheConfigDto { + cache_duration: shared::model::Secs::new(cache_duration_seconds), + cache_path: Some(cache_path.into().to_string_lossy().to_string()), + ..Default::default() + }; + let default_config = HlsCacheConfig::from(&default_dto); + let segment_fetch_policy = SegmentFetchPolicy::from_config(&default_config); + let global_fetch_semaphore = Arc::new(Semaphore::new(segment_fetch_policy.max_global_segment_fetches)); + let sessions = Arc::new(HlsSessionStore::new()); + let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(PathBuf::from(&default_config.cache_path))); + segment_cache + .update_cache_limits(default_config.cache_bytes.get(), default_config.cache_bytes_per_session.get()); + let segment_repair = Arc::new(HlsSegmentRepairManager::new(default_config.segment_repair.clone())); + let metrics = Arc::new(HlsCacheMetrics::default()); + let qos = Arc::new(HlsQosRegistry::default()); + let access_leases = Arc::new(RwLock::new(HlsAccessLeaseStore::default())); + let lifecycle = Arc::new(HlsLifecycleManager::new()); + let account_overlap_cooldowns = Arc::new(RwLock::new(HashMap::new())); + let gc_policy = GarbageCollectionPolicy::from_config(&default_config); + let runtime_config = HlsProxyRuntimeConfig::from_config(&default_config, &[]); + let gc = Arc::new(HlsGarbageCollector::new_with_metrics( + Arc::clone(&sessions), + Arc::clone(&segment_cache), + gc_policy.clone(), + runtime_config.rewrite_secret_fingerprint.clone(), + Arc::clone(&metrics), + )); + segment_cache.install_capacity_reclaimer(&gc); + gc.install_access_leases(&access_leases); + let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default()); + Self { + sessions, + segment_cache, + segment_repair, + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::with_global_semaphore_metrics_and_availability( + segment_fetch_policy.clone(), + Arc::clone(&global_fetch_semaphore), + Arc::clone(&access_leases), + Arc::clone(&metrics), + Some(Arc::clone(&availability_reevaluations)), + )), + map_worker_pool: Arc::new(HlsMapWorkerPool::with_global_semaphore_access_leases_and_availability( + segment_fetch_policy.clone(), + global_fetch_semaphore, + Arc::clone(&access_leases), + Some(Arc::clone(&availability_reevaluations)), + )), + progressive_budget: super::super::ProgressiveBudgetManager::new(runtime_config.startup.clone()), + revision_store: Arc::new(super::super::SegmentRevisionStore::default()), + revision_reconciliation_pending: AtomicBool::new(true), + runtime_config: ArcSwap::from_pointee(runtime_config), + transient_resources: Arc::new(TransientResourceStore::new()), + access_leases, + lifecycle, + account_overlap_cooldowns, + metrics, + qos, + gc, + prepared_terminal_bundles: Arc::new(HlsPreparedTerminalBundleCache::new()), + standalone_custom_access: Arc::new(HlsStandaloneCustomAccessStore::default()), + terminal_commit_retries: Arc::new(HlsTerminalCommitRetryCoordinator::default()), + terminal_pending: Arc::new(HlsTerminalPendingCoordinator::default()), + availability_reevaluations, + terminal_commit_clock: Arc::new(HlsTerminalCommitClock::default()), + startup_observability: Arc::new(HlsStartupObservability::default()), + } + } + + pub fn segment_fetch_policy(&self) -> SegmentFetchPolicy { self.runtime_config.load().segment_fetch_policy.clone() } + + pub fn is_enabled(&self) -> bool { self.runtime_config.load().enabled } + + pub fn cache_duration_seconds(&self) -> u64 { self.runtime_config.load().cache_duration_seconds } + + pub fn session_idle_timeout_ms(&self) -> u64 { self.runtime_config.load().gc_policy.session_idle_timeout_ms } + + pub fn strip(&self) -> StripConfig { self.runtime_config.load().strip.clone() } + + pub fn origin_manifest_timeout_ms(&self) -> u64 { self.runtime_config.load().origin_manifest_timeout_ms } + + pub fn initial_manifest_wait_timeout_secs(&self) -> u64 { + self.runtime_config.load().initial_manifest_wait_timeout_secs + } + + pub fn manifest_recovery_burst(&self) -> HlsManifestRecoveryBurstConfig { + self.runtime_config.load().manifest_recovery_burst.clone() + } + + pub fn transient_resource_ttl_ms(&self) -> u64 { self.runtime_config.load().transient_resource_ttl_ms } + + pub fn gc_policy(&self) -> GarbageCollectionPolicy { self.runtime_config.load().gc_policy.clone() } + + pub fn rewrite_secret_fingerprint(&self) -> String { self.runtime_config.load().rewrite_secret_fingerprint.clone() } + + pub async fn update_config(&self, app_config: &AppConfig) { + let (hls_config, rewrite_secret, enabled) = { + let config = app_config.config.load(); + let rewrite_secret = config + .reverse_proxy + .as_ref() + .map_or(app_config.encrypt_secret, |reverse_proxy| reverse_proxy.rewrite_secret); + let hls_config = + config.reverse_proxy.as_ref().and_then(|reverse_proxy| reverse_proxy.hls_cache.as_ref()).cloned(); + let enabled = hls_config.is_some(); + let hls_config = + hls_config.unwrap_or_else(|| HlsCacheConfig::from(&shared::model::HlsCacheConfigDto::default())); + (hls_config, rewrite_secret, enabled) + }; + let runtime_config = HlsProxyRuntimeConfig::from_config_with_enabled(&hls_config, &rewrite_secret, enabled); + let cache_path_changed = self.gc.update_cache_path(PathBuf::from(&hls_config.cache_path)).await; + self.progressive_budget.update_limits(hls_config.startup.clone()); + self.segment_cache.update_cache_limits(hls_config.cache_bytes.get(), hls_config.cache_bytes_per_session.get()); + if cache_path_changed { + self.revision_reconciliation_pending.store(true, Ordering::Release); + self.clear_runtime_cache_state_for_cache_path_change().await; + } + for session in self.sessions.list_sessions().await { + session + .write() + .await + .configure_segment_prefetch_queue(runtime_config.segment_fetch_policy.max_prefetch_queue_depth); + } + self.segment_repair.update_config(hls_config.segment_repair.clone()); + let global_fetch_semaphore = + Arc::new(Semaphore::new(runtime_config.segment_fetch_policy.max_global_segment_fetches)); + self.segment_worker_pool + .update_config(runtime_config.segment_fetch_policy.clone(), Arc::clone(&global_fetch_semaphore)); + self.map_worker_pool.update_config(runtime_config.segment_fetch_policy.clone(), global_fetch_semaphore); + self.gc.update_config(runtime_config.gc_policy.clone(), runtime_config.rewrite_secret_fingerprint.clone()); + self.runtime_config.store(Arc::new(runtime_config)); + } +} diff --git a/backend/hls/src/manager/lifecycle.rs b/backend/hls/src/manager/lifecycle.rs new file mode 100644 index 000000000..cb9b074cc --- /dev/null +++ b/backend/hls/src/manager/lifecycle.rs @@ -0,0 +1,395 @@ +use super::{ + safe_proxy_session_id, HlsAccessLease, HlsAccessLeaseDenialMode, HlsAccessLeaseDenialOutcome, HlsAccessLeaseId, + HlsAccessLeaseLifecycleSnapshot, HlsAccessLeasePendingDeadline, HlsAccessLeaseRemovalPreparation, + HlsAccessLeaseState, HlsAccessLeaseTiming, HlsCtx, HlsExpiredSessionReason, HlsLifecycleEvent, + HlsLifecycleEventKey, HlsLifecycleManager, HlsProxyManager, HlsSessionHandle, HlsTerminalTailProtectionRemoval, + ProxySessionId, HLS_SESSION_IDLE_PROTECTION_RETRY_MS, +}; +use log::{debug, error}; +use std::sync::Arc; +use tokio_util::sync::CancellationToken; +use tuliprox_core::utils::current_time_millis; +use tuliprox_session::{ActiveProviderManager, ActiveUserManager}; + +pub(super) fn hls_session_idle_protection_retry_at(due_at_ms: u64, now_ms: u64) -> u64 { + due_at_ms.max(now_ms.saturating_add(HLS_SESSION_IDLE_PROTECTION_RETRY_MS)) +} + +impl HlsProxyManager { + pub fn lifecycle(&self) -> &Arc { &self.lifecycle } + + pub async fn active_live_playback_snapshots_for_session( + &self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Vec { + self.access_leases.write().await.active_live_playback_snapshots_for_session(proxy_session_id, now_ms) + } + + pub async fn touch_access_lease( + &self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + timing: HlsAccessLeaseTiming, + ) -> bool { + let lease = self.access_leases.write().await.touch_access_lease_snapshot(lease_id, now_ms, timing); + if let Some(lease) = lease { + self.schedule_access_lease_activity(&lease).await; + self.schedule_access_lease_validity(&lease).await; + true + } else { + false + } + } + + async fn access_lease_lifecycle_snapshot( + &self, + lease_id: &HlsAccessLeaseId, + now_ms: u64, + ) -> Option { + self.access_leases.write().await.lifecycle_snapshot(lease_id, now_ms) + } + + pub(super) async fn remove_access_lease(&self, lease_id: &HlsAccessLeaseId) { + self.terminal_pending.cancel_lease(lease_id); + self.terminal_commit_retries.cancel_lease(lease_id); + let Some(preparation) = self.access_leases.read().await.prepare_access_lease_removal(lease_id) else { + return; + }; + if !self.remove_prepared_access_lease(lease_id, &preparation).await { + debug!( + "HLS access lease removal skipped: lease={} proxy_session={} reason=lease_instance_race", + super::super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(&preparation.proxy_session_id) + ); + } + } + + /// Removes one exact lease incarnation before releasing any media + /// protection owned by it. A stale preparation is intentionally + /// non-destructive because the replacement lease may reuse the same ID. + pub(super) async fn remove_prepared_access_lease( + &self, + lease_id: &HlsAccessLeaseId, + preparation: &HlsAccessLeaseRemovalPreparation, + ) -> bool { + let removed = + self.access_leases.write().await.remove_access_lease_if_preparation_matches(lease_id, preparation); + if removed.is_none() { + return false; + } + if let Some(session) = self.sessions.get_by_proxy_session_id(&preparation.proxy_session_id).await { + let mut session = session.write().await; + let finalized_manifest_released = + session.transient.release_finalized_manifest_generations(lease_id, preparation.issued_at_ms); + if finalized_manifest_released { + debug!( + "HLS finalized transient manifest generation released: lease={} proxy_session={} reason=lease_removed", + super::super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(&preparation.proxy_session_id) + ); + } + if let Some(generation) = preparation.terminal_protection_generation { + let removal = session.remove_terminal_tail_protection_after_lease_end(lease_id, generation); + match removal { + HlsTerminalTailProtectionRemoval::Removed => { + debug!( + "HLS terminal-tail protection released: lease={} proxy_session={} generation={} reason=lease_removed", + super::super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(&preparation.proxy_session_id), + generation.0 + ); + } + HlsTerminalTailProtectionRemoval::Missing => {} + HlsTerminalTailProtectionRemoval::RemovedStaleGeneration { actual } => { + debug!( + "HLS stale terminal-tail protection released: lease={} proxy_session={} expected_generation={} actual_generation={} reason=lease_removed", + super::super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(&preparation.proxy_session_id), + generation.0, + actual.0 + ); + } + } + } else if session.remove_terminal_tail_protection(lease_id).is_some() { + debug!( + "HLS untracked terminal-tail protection released: lease={} proxy_session={} reason=lease_removed", + super::super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(&preparation.proxy_session_id) + ); + } + } + self.standalone_custom_access.remove(lease_id); + self.segment_repair.remove_access_lease_window(lease_id).await; + self.startup_observability.remove_access_lease(lease_id); + self.qos.remove_access_lease(lease_id).await; + true + } + + pub(super) async fn schedule_access_lease_activity(&self, lease: &HlsAccessLease) { + if let Some(active_until_ms) = lease.active_until_ms { + self.lifecycle + .schedule( + HlsLifecycleEventKey::AccessLeaseActive { + lease_id: lease.lease_id.clone(), + proxy_session_id: lease.proxy_session_id.clone(), + }, + active_until_ms, + ) + .await; + } + } + + pub(super) async fn schedule_access_lease_validity(&self, lease: &HlsAccessLease) { + let due_at_ms = if lease.state == HlsAccessLeaseState::Pending { + lease.pending_deadline_ms().unwrap_or(lease.valid_until_ms) + } else { + lease.valid_until_ms + }; + self.lifecycle + .schedule( + HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: lease.lease_id.clone(), + proxy_session_id: lease.proxy_session_id.clone(), + }, + due_at_ms, + ) + .await; + } + + async fn schedule_access_lease_lifecycle_snapshot(&self, snapshot: &HlsAccessLeaseLifecycleSnapshot) { + if snapshot.state == HlsAccessLeaseState::Activated { + if let Some(active_until_ms) = snapshot.active_until_ms { + self.lifecycle + .schedule( + HlsLifecycleEventKey::AccessLeaseActive { + lease_id: snapshot.lease_id.clone(), + proxy_session_id: snapshot.proxy_session_id.clone(), + }, + active_until_ms, + ) + .await; + } + } + if snapshot.state != HlsAccessLeaseState::Expired && snapshot.state != HlsAccessLeaseState::Denied { + let due_at_ms = if snapshot.state == HlsAccessLeaseState::Pending { + snapshot.pending_deadline.map_or(snapshot.valid_until_ms, HlsAccessLeasePendingDeadline::deadline_ms) + } else { + snapshot.valid_until_ms + }; + self.lifecycle + .schedule( + HlsLifecycleEventKey::AccessLeaseValidity { + lease_id: snapshot.lease_id.clone(), + proxy_session_id: snapshot.proxy_session_id.clone(), + }, + due_at_ms, + ) + .await; + } + } + + pub async fn schedule_session_idle_for_handle(&self, session: &HlsSessionHandle) { + let session_idle_timeout_ms = self.session_idle_timeout_ms(); + let (proxy_session_id, due_at_ms) = { + let session = session.read().await; + (session.proxy_session_id.clone(), session.idle_expiry_due_at_ms(session_idle_timeout_ms)) + }; + self.lifecycle.schedule(HlsLifecycleEventKey::SessionIdle { proxy_session_id }, due_at_ms).await; + } + + pub async fn handle_lifecycle_event( + &self, + active_users: &Arc, + active_provider: &Arc, + event: HlsLifecycleEvent, + now_ms: u64, + ) { + if !self.is_enabled() { + return; + } + match event.key { + HlsLifecycleEventKey::AccessLeaseActive { lease_id, proxy_session_id } + | HlsLifecycleEventKey::AccessLeaseValidity { lease_id, proxy_session_id } => { + let mut should_sync_session = false; + if let Some(snapshot) = self.access_lease_lifecycle_snapshot(&lease_id, now_ms).await { + should_sync_session = true; + if let Some(release) = &snapshot.idle_release { + active_users + .release_session_streams_and_counted_reservation( + &release.username, + &release.user_session_token, + ) + .await; + debug!( + "HLS access lease idled: lease={} proxy_session={} user_session={}", + super::super::safe_hls_access_lease_id(&release.lease_id), + safe_proxy_session_id(&snapshot.proxy_session_id), + super::super::safe_user_session_token(&release.user_session_token) + ); + } + if matches!(snapshot.state, HlsAccessLeaseState::Expired | HlsAccessLeaseState::Denied) { + self.remove_access_lease(&snapshot.lease_id).await; + debug!( + "HLS access lease removed: lease={} proxy_session={} state={}", + super::super::safe_hls_access_lease_id(&snapshot.lease_id), + safe_proxy_session_id(&snapshot.proxy_session_id), + snapshot.state.as_log_value() + ); + debug!( + "HLS lifecycle state snapshot: trigger=access-lease-removed {}", + self.debug_state_summary().await + ); + } else { + self.schedule_access_lease_lifecycle_snapshot(&snapshot).await; + } + } + if should_sync_session { + if let Some(session) = self.sessions.get_by_proxy_session_id(&proxy_session_id).await { + self.sync_session_access_lease_count_and_detach_if_needed( + active_users, + active_provider, + &session, + &proxy_session_id, + now_ms, + ) + .await; + } + } + } + HlsLifecycleEventKey::SessionIdle { proxy_session_id } => { + self.handle_session_idle_lifecycle_event(&proxy_session_id, now_ms).await; + } + } + } + + async fn handle_session_idle_lifecycle_event(&self, proxy_session_id: &ProxySessionId, now_ms: u64) { + let Some(session) = self.sessions.get_by_proxy_session_id(proxy_session_id).await else { + return; + }; + let session_idle_timeout_ms = self.session_idle_timeout_ms(); + let (key, due_at_ms, can_remove) = { + let session = session.read().await; + ( + session.key.clone(), + session.idle_expiry_due_at_ms(session_idle_timeout_ms), + session.can_expire_idle_session(now_ms, session_idle_timeout_ms), + ) + }; + if !can_remove { + self.lifecycle + .schedule( + HlsLifecycleEventKey::SessionIdle { proxy_session_id: proxy_session_id.clone() }, + hls_session_idle_protection_retry_at(due_at_ms, now_ms), + ) + .await; + return; + } + if self.segment_cache.has_active_temp_files_for_session(proxy_session_id) { + self.lifecycle + .schedule( + HlsLifecycleEventKey::SessionIdle { proxy_session_id: proxy_session_id.clone() }, + now_ms.saturating_add(1_000), + ) + .await; + return; + } + let username = self.access_leases.read().await.first_username_for_session(proxy_session_id); + if self + .sessions + .remove_session_marking_expired( + &key, + proxy_session_id, + now_ms, + HlsExpiredSessionReason::SessionIdleTimeout, + username, + ) + .await + .is_some() + { + self.cleanup_proxy_session_state(proxy_session_id, "lifecycle-session-expired").await; + if let Err(err) = self.segment_cache.delete_session_dir(proxy_session_id).await { + error!( + "HLS session lifecycle cleanup failed: proxy_session={} error={err}", + safe_proxy_session_id(proxy_session_id) + ); + } else { + debug!("HLS session lifecycle expired: proxy_session={}", safe_proxy_session_id(proxy_session_id)); + debug!("HLS lifecycle state snapshot: trigger=session-expired {}", self.debug_state_summary().await); + } + } + } + + pub async fn deny_access_lease( + &self, + lease_id: &HlsAccessLeaseId, + mode: HlsAccessLeaseDenialMode, + ) -> HlsAccessLeaseDenialOutcome { + let outcome = { + let mut access_leases = self.access_leases.write().await; + access_leases.deny_access_lease(lease_id, mode) + }; + let HlsAccessLeaseDenialOutcome::Ended { terminal_release } = &outcome else { + return outcome; + }; + self.terminal_pending.cancel_lease(lease_id); + self.terminal_commit_retries.cancel_lease(lease_id); + let Some(terminal_release) = terminal_release else { + return outcome; + }; + let removal = if let Some(session) = + self.sessions.get_by_proxy_session_id(&terminal_release.proxy_session_id).await + { + session.write().await.remove_terminal_tail_protection_after_lease_end(lease_id, terminal_release.generation) + } else { + HlsTerminalTailProtectionRemoval::Missing + }; + match removal { + HlsTerminalTailProtectionRemoval::Removed => { + debug!( + "HLS terminal-tail protection released: lease={} proxy_session={} generation={} reason=lease_denied", + super::super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(&terminal_release.proxy_session_id), + terminal_release.generation.0 + ); + } + HlsTerminalTailProtectionRemoval::Missing => {} + HlsTerminalTailProtectionRemoval::RemovedStaleGeneration { actual } => { + debug!( + "HLS stale terminal-tail protection released: lease={} proxy_session={} expected_generation={} actual_generation={} reason=lease_denied", + super::super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(&terminal_release.proxy_session_id), + terminal_release.generation.0, + actual.0 + ); + } + } + let acknowledged = self.access_leases.write().await.acknowledge_terminal_protection_release( + lease_id, + &terminal_release.proxy_session_id, + terminal_release.generation, + ); + if !acknowledged { + debug!( + "HLS terminal-tail protection release acknowledgement skipped: lease={} proxy_session={} generation={} reason=lease_generation_race", + super::super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(&terminal_release.proxy_session_id), + terminal_release.generation.0 + ); + } + outcome + } +} + +pub fn exec_hls_lifecycle(ctx: &HlsCtx, cancel_token: &CancellationToken) { + let hls_proxy = Arc::clone(&ctx.hls_proxy); + let active_users = Arc::clone(&ctx.active_users); + let active_provider = Arc::clone(&ctx.active_provider); + let cancel_token = cancel_token.clone(); + tokio::spawn(async move { + while let Some(event) = hls_proxy.lifecycle().next_event(&cancel_token).await { + let now_ms = current_time_millis(); + hls_proxy.handle_lifecycle_event(&active_users, &active_provider, event, now_ms).await; + } + }); +} diff --git a/backend/hls/src/manager/publication.rs b/backend/hls/src/manager/publication.rs new file mode 100644 index 000000000..cb0253f1b --- /dev/null +++ b/backend/hls/src/manager/publication.rs @@ -0,0 +1,915 @@ +use super::{ + evaluate_lease_reserve, manifest_acceptance_episode_status, ready_segment_repair_prewarm_candidates, + safe_proxy_session_id, safe_session_key, GarbageCollectionPolicy, HlsAccessLease, HlsAccessLeaseActivation, + HlsAccessLeaseDenialOutcome, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsAccessLeaseSessionSnapshot, + HlsAccessLeaseState, HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsAccessLeaseTouch, + HlsAvailabilityReevaluationCoordinator, HlsAvailabilityReevaluationOwnerKey, HlsCacheMetrics, + HlsCriticalHandoffStateAccess, HlsCurrentProxySessionAccess, HlsEstimatedRecoveryCompletionAtMs, + HlsFiniteTailTrigger, HlsGarbageCollector, HlsLeaseCutoverTiming, HlsLeaseManifestPublicationGuard, + HlsLeaseManifestPublicationOutcome, HlsLeaseManifestPublicationRejectReason, HlsLeaseManifestSnapshot, + HlsLeaseReserveInput, HlsLifecycleManager, HlsManifestAcceptanceEpisodeStatus, HlsManifestAcceptanceGeneration, + HlsManifestDeliveryMode, HlsMapWorkerPool, HlsOriginSource, HlsPreparedTerminalBundleCache, HlsProxyManager, + HlsProxyRuntimeConfig, HlsPublishedTransientResourceIds, HlsQosRegistry, HlsRecoveryPressureGuard, + HlsRecoveryPressureGuardAccess, HlsRecoveryTriggerBudgetMs, HlsRepairPrewarmGuard, HlsRuntimeCustomTailReason, + HlsRuntimePolicyRevocation, HlsRuntimePolicyRevocationOutcome, HlsSegmentCache, HlsSegmentRepairManager, + HlsSegmentWorkerPool, HlsSessionHandle, HlsSessionKey, HlsSessionStore, HlsSessionStoreOutcome, + HlsStandaloneCustomAccessStore, HlsStartupObservability, HlsTerminalCommitAcquisitionBudgetMs, + HlsTerminalCommitClock, HlsTerminalCommitMediaGuard, HlsTerminalCommitPayload, HlsTerminalCommitRetryCoordinator, + HlsTerminalCommitWindow, HlsTerminalLeaseDecision, HlsTerminalMediaPreparationKey, + HlsTerminalMediaPreparationState, HlsTerminalMediaRequirementOrigin, HlsTerminalPendingCoordinator, + HlsTerminalTailPreparation, HlsTerminalTailPreparationInput, HlsTransitionMarginMs, ProxySessionId, + SegmentFetchPolicy, TransientManifestLeaseBinding, TransientResourceStore, HLS_STATE_CAS_LOCK_RETRIES, +}; +use arc_swap::ArcSwap; +use log::{debug, error, info}; +use std::{ + collections::HashMap, + path::PathBuf, + sync::{atomic::AtomicBool, Arc}, +}; +use tokio::sync::{RwLock, Semaphore}; +use tuliprox_core::{model::HlsCacheConfig, utils::current_time_millis}; + +impl HlsTerminalCommitPayload { + pub(super) fn into_parts(self) -> (HlsTerminalLeaseDecision, Option) { + match self { + Self::Tail { plan, media_guard } => (HlsTerminalLeaseDecision::Tail(plan), Some(media_guard)), + Self::Unavailable(reason) => (HlsTerminalLeaseDecision::Unavailable(reason), None), + Self::UnavailableAfterOwnerFailure(reason) => { + (HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(reason), None) + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum HlsRecoveryExecutionState { + Idle, + InFlight { estimated_completion_at: HlsEstimatedRecoveryCompletionAtMs }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct HlsAcceptanceRecoverySnapshot { + pub(super) expected_generation: HlsManifestAcceptanceGeneration, + pub(super) status: HlsManifestAcceptanceEpisodeStatus, + pub(super) recovery: HlsRecoveryExecutionState, + pub(super) required_terminal_media_key: Option, + pub(super) terminal_media_preparation: HlsTerminalMediaPreparationState, +} + +pub(super) fn hls_estimated_recovery_completion_at( + recovery: HlsRecoveryExecutionState, +) -> Option { + match recovery { + HlsRecoveryExecutionState::Idle => None, + HlsRecoveryExecutionState::InFlight { estimated_completion_at } => Some(estimated_completion_at), + } +} + +pub(super) fn hls_acceptance_recovery_snapshot( + session: &super::super::HlsSession, + now_ms: u64, +) -> HlsAcceptanceRecoverySnapshot { + let expected_generation = session.origin_control.acceptance_generation; + let status = manifest_acceptance_episode_status( + session.origin_control.acceptance_episode.as_ref(), + expected_generation, + now_ms, + ); + let matching_episode = + session.origin_control.acceptance_episode.as_ref().filter(|episode| episode.generation == expected_generation); + let recovery = match (status, matching_episode) { + (HlsManifestAcceptanceEpisodeStatus::InFlight { .. }, Some(episode)) => episode + .estimated_recovery_completion_at(expected_generation, now_ms) + .map_or(HlsRecoveryExecutionState::Idle, |estimated_completion_at| HlsRecoveryExecutionState::InFlight { + estimated_completion_at, + }), + ( + HlsManifestAcceptanceEpisodeStatus::Missing + | HlsManifestAcceptanceEpisodeStatus::Expired { .. } + | HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { .. } + | HlsManifestAcceptanceEpisodeStatus::Committed { .. } + | HlsManifestAcceptanceEpisodeStatus::Superseded { .. }, + _, + ) + | (HlsManifestAcceptanceEpisodeStatus::InFlight { .. }, None) => HlsRecoveryExecutionState::Idle, + }; + let (required_terminal_media_key, terminal_media_preparation) = + matching_episode.map_or((None, HlsTerminalMediaPreparationState::Failed { key: None }), |episode| { + let timing = episode.timing(); + (timing.required_terminal_media_key, timing.terminal_media_preparation) + }); + HlsAcceptanceRecoverySnapshot { + expected_generation, + status, + recovery, + required_terminal_media_key, + terminal_media_preparation, + } +} + +fn hls_pending_manifest_follow_up_window_ms(target_duration: Option) -> u64 { + let target_duration_secs = u64::from(target_duration.unwrap_or(15)).max(1); + target_duration_secs.saturating_mul(2_000).max(10_000) +} + +fn hls_pending_manifest_follow_up_deadline(now_ms: u64, target_duration: Option) -> HlsAccessLeasePendingDeadline { + HlsAccessLeasePendingDeadline::FollowUp { + deadline_ms: now_ms.saturating_add(hls_pending_manifest_follow_up_window_ms(target_duration)), + } +} + +impl HlsProxyManager { + pub fn with_hls_cache_config(config: &HlsCacheConfig) -> Self { + Self::with_hls_cache_config_and_secret(config, &[]) + } + + pub fn with_hls_cache_config_and_secret(config: &HlsCacheConfig, rewrite_secret: &[u8]) -> Self { + Self::with_hls_cache_config_and_secret_enabled(config, rewrite_secret, true) + } + + pub(super) fn with_hls_cache_config_and_secret_enabled( + config: &HlsCacheConfig, + rewrite_secret: &[u8], + enabled: bool, + ) -> Self { + let segment_fetch_policy = SegmentFetchPolicy::from_config(config); + let global_fetch_semaphore = Arc::new(Semaphore::new(segment_fetch_policy.max_global_segment_fetches)); + let sessions = Arc::new(HlsSessionStore::new()); + let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(PathBuf::from(&config.cache_path))); + segment_cache.update_cache_limits(config.cache_bytes.get(), config.cache_bytes_per_session.get()); + let segment_repair = Arc::new(HlsSegmentRepairManager::new(config.segment_repair.clone())); + let metrics = Arc::new(HlsCacheMetrics::default()); + let qos = Arc::new(HlsQosRegistry::default()); + let access_leases = Arc::new(RwLock::new(HlsAccessLeaseStore::default())); + let lifecycle = Arc::new(HlsLifecycleManager::new()); + let account_overlap_cooldowns = Arc::new(RwLock::new(HashMap::new())); + let gc_policy = GarbageCollectionPolicy::from_config(config); + let runtime_config = HlsProxyRuntimeConfig::from_config_with_enabled(config, rewrite_secret, enabled); + let gc = Arc::new(HlsGarbageCollector::new_with_metrics( + Arc::clone(&sessions), + Arc::clone(&segment_cache), + gc_policy.clone(), + runtime_config.rewrite_secret_fingerprint.clone(), + Arc::clone(&metrics), + )); + segment_cache.install_capacity_reclaimer(&gc); + gc.install_access_leases(&access_leases); + let availability_reevaluations = Arc::new(HlsAvailabilityReevaluationCoordinator::default()); + Self { + sessions, + segment_cache, + segment_repair, + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::with_global_semaphore_metrics_and_availability( + segment_fetch_policy.clone(), + Arc::clone(&global_fetch_semaphore), + Arc::clone(&access_leases), + Arc::clone(&metrics), + Some(Arc::clone(&availability_reevaluations)), + )), + map_worker_pool: Arc::new(HlsMapWorkerPool::with_global_semaphore_access_leases_and_availability( + segment_fetch_policy.clone(), + global_fetch_semaphore, + Arc::clone(&access_leases), + Some(Arc::clone(&availability_reevaluations)), + )), + progressive_budget: super::super::ProgressiveBudgetManager::new(runtime_config.startup.clone()), + revision_store: Arc::new(super::super::SegmentRevisionStore::default()), + revision_reconciliation_pending: AtomicBool::new(true), + runtime_config: ArcSwap::from_pointee(runtime_config), + transient_resources: Arc::new(TransientResourceStore::new()), + access_leases, + lifecycle, + account_overlap_cooldowns, + metrics, + qos, + gc, + prepared_terminal_bundles: Arc::new(HlsPreparedTerminalBundleCache::new()), + standalone_custom_access: Arc::new(HlsStandaloneCustomAccessStore::default()), + terminal_commit_retries: Arc::new(HlsTerminalCommitRetryCoordinator::default()), + terminal_pending: Arc::new(HlsTerminalPendingCoordinator::default()), + availability_reevaluations, + terminal_commit_clock: Arc::new(HlsTerminalCommitClock::default()), + startup_observability: Arc::new(HlsStartupObservability::default()), + } + } + + pub fn sessions(&self) -> &Arc { &self.sessions } + + pub fn segment_cache(&self) -> &Arc { &self.segment_cache } + + pub fn segment_repair(&self) -> &Arc { &self.segment_repair } + + pub fn startup_observability(&self) -> &Arc { &self.startup_observability } + + pub fn spawn_access_lease_repair_prewarm( + self: &Arc, + session: HlsSessionHandle, + lease_id: HlsAccessLeaseId, + snapshot: HlsLeaseManifestSnapshot, + snapshot_generation: u64, + ) { + if snapshot.delivery_mode != HlsManifestDeliveryMode::NormalCacheTimeline + || snapshot.container != super::super::terminal_tail::HlsMediaContainer::MpegTs + { + return; + } + let candidate_limit = self.segment_repair.prewarm_candidate_limit(); + if candidate_limit == 0 { + return; + } + let manager = Arc::clone(self); + tokio::spawn(async move { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let Some(lease) = + manager.access_lease_response_snapshot(&lease_id, &proxy_session_id, current_time_millis()).await + else { + return; + }; + let guard = HlsRepairPrewarmGuard::new( + Arc::clone(&manager.access_leases), + lease_id.clone(), + proxy_session_id, + lease.issued_at_ms, + snapshot_generation, + ); + manager.segment_repair.ensure_access_lease_window(lease_id.clone()).await; + let candidates = { + let session = session.read().await; + ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, candidate_limit) + }; + manager + .segment_repair + .spawn_ready_cache_prewarm(Arc::clone(&manager.segment_cache), candidates, guard) + .await; + }); + } + + pub fn segment_worker_pool(&self) -> &Arc { &self.segment_worker_pool } + + pub fn map_worker_pool(&self) -> &Arc { &self.map_worker_pool } + + /// The clock every time-dependent HLS scheduler reads. + /// + /// One clock, so a test that controls time controls all of it rather than + /// some schedulers following the wall clock and others the test clock. + pub(crate) fn now_ms(&self) -> u64 { self.terminal_commit_clock.now_ms() } + + /// Anchor the scheduling clock to tokio's, for `start_paused` tests. + #[cfg(test)] + pub(crate) fn follow_tokio_clock_for_test(&self, base_ms: u64) { + self.terminal_commit_clock.follow_tokio_clock(base_ms); + } + + pub fn transient_resources(&self) -> &Arc { &self.transient_resources } + + pub fn access_leases(&self) -> &Arc> { &self.access_leases } + + pub fn metrics(&self) -> &Arc { &self.metrics } + + pub fn qos(&self) -> &Arc { &self.qos } + + pub fn garbage_collector(&self) -> &Arc { &self.gc } + + pub(super) async fn clear_runtime_cache_state_for_cache_path_change(&self) { + self.availability_reevaluations.clear(); + self.terminal_pending.clear(); + self.terminal_commit_retries.clear(); + self.sessions.clear().await; + let removed_leases = self.access_leases.write().await.clear(); + self.standalone_custom_access.clear(); + self.terminal_commit_retries.clear(); + let removed_qos = self.qos.clear().await; + self.segment_repair.clear_runtime_state().await; + self.startup_observability.clear(); + debug!( + "HLS cache runtime state cleared after cache path change: access_leases_removed={removed_leases} qos_access_leases_removed={removed_qos}" + ); + } + + pub async fn prepare_access_lease(&self, lease: HlsAccessLease) { + if !self.access_leases.write().await.prepare_access_lease(lease.clone()) { + error!( + "HLS access lease preparation rejected: lease={} proxy_session={} reason=availability_evidence_exhausted", + super::super::safe_hls_access_lease_id(&lease.lease_id), + safe_proxy_session_id(&lease.proxy_session_id) + ); + return; + } + self.schedule_access_lease_validity(&lease).await; + } + + pub async fn access_lease( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Option { + let (lease, still_stored) = { + let mut access_leases = self.access_leases.write().await; + let lease = access_leases.access_lease(lease_id, proxy_session_id, now_ms); + let still_stored = access_leases.lease_state(lease_id, now_ms).is_some(); + (lease, still_stored) + }; + if lease.is_none() && !still_stored { + self.segment_repair.remove_access_lease_window(lease_id).await; + self.startup_observability.remove_access_lease(lease_id); + self.qos.remove_access_lease(lease_id).await; + } + lease + } + + pub async fn access_lease_response_snapshot( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Option { + self.access_leases.write().await.response_snapshot(lease_id, proxy_session_id, now_ms) + } + + /// Runs the final Critical-Handoff revalidation and timeline commit under the + /// established lease-store -> session lock order. No lock is held across an await. + /// `LockBusy` reports contention only and is not a generation or lease invalidation. + pub async fn with_critical_handoff_state( + &self, + session_handle: &HlsSessionHandle, + operation: impl FnOnce(&mut HlsAccessLeaseStore, &mut super::super::HlsSession) -> T, + ) -> HlsCriticalHandoffStateAccess { + let mut operation = Some(operation); + for attempt in 0..HLS_STATE_CAS_LOCK_RETRIES { + let Ok(mut leases) = self.access_leases.try_write() else { + if attempt.saturating_add(1) < HLS_STATE_CAS_LOCK_RETRIES { + tokio::task::yield_now().await; + } + continue; + }; + let Ok(mut session) = session_handle.try_write() else { + drop(leases); + if attempt.saturating_add(1) < HLS_STATE_CAS_LOCK_RETRIES { + tokio::task::yield_now().await; + } + continue; + }; + if let Some(operation) = operation.take() { + return HlsCriticalHandoffStateAccess::Acquired(operation(&mut leases, &mut session)); + } + } + HlsCriticalHandoffStateAccess::LockBusy + } + + /// Captures the immutable identity used to deduplicate a later autonomous + /// availability evaluation. The session read is completed before the + /// lease-store -> session transaction starts, so no lock order is inverted. + pub async fn availability_reevaluation_owner_key( + &self, + session_handle: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + ) -> Option { + let session_incarnation = self.sessions.session_incarnation(session_handle)?; + let availability_evidence_generation = + self.access_leases.read().await.availability_evidence_generation(proxy_session_id); + let session = session_handle.read().await; + if session.proxy_session_id != *proxy_session_id || session.is_gc_marked_for_removal() { + return None; + } + Some(HlsAvailabilityReevaluationOwnerKey { + session_incarnation, + proxy_session_id: proxy_session_id.clone(), + origin_progress_generation: session.origin_control.progress_generation, + media_readiness_generation: session.activity.media_readiness_generation, + availability_evidence_generation, + }) + } + + pub async fn availability_reevaluation_session_is_current( + &self, + session_handle: &HlsSessionHandle, + owner_key: &HlsAvailabilityReevaluationOwnerKey, + ) -> bool { + if self.sessions.session_incarnation(session_handle) != Some(owner_key.session_incarnation) { + return false; + } + self.sessions + .get_by_proxy_session_id(&owner_key.proxy_session_id) + .await + .is_some_and(|current| Arc::ptr_eq(¤t, session_handle)) + } + + /// Runs the refresh-start mutation only while the lease evidence, session + /// index, concrete handle, incarnation, and recovery-pressure generations + /// are all current. Cross-store lock order is Session Index -> optional + /// Retry Owner -> Lease Store -> Session. Every acquisition here is + /// non-blocking and the closure must not await or perform I/O. + pub fn with_current_recovery_pressure_session( + &self, + session_handle: &HlsSessionHandle, + guard: &HlsRecoveryPressureGuard, + operation: impl FnOnce(&mut super::super::HlsSession) -> R, + ) -> HlsRecoveryPressureGuardAccess { + if self.sessions.session_incarnation(session_handle) != Some(guard.session_incarnation) { + return HlsRecoveryPressureGuardAccess::Superseded; + } + let access = self.sessions.try_with_current_proxy_session(&guard.proxy_session_id, session_handle, || { + let Ok(leases) = self.access_leases.try_read() else { + return HlsRecoveryPressureGuardAccess::LockBusy; + }; + if leases.availability_evidence_generation(&guard.proxy_session_id) + != guard.availability_evidence_generation + { + return HlsRecoveryPressureGuardAccess::Superseded; + } + let Ok(mut session) = session_handle.try_write() else { + return HlsRecoveryPressureGuardAccess::LockBusy; + }; + if session.proxy_session_id != guard.proxy_session_id + || session.origin_control.progress_generation != guard.origin_progress_generation + || session.activity.media_readiness_generation != guard.media_readiness_generation + || leases.availability_evidence_generation(&guard.proxy_session_id) + != guard.availability_evidence_generation + || session.is_gc_marked_for_removal() + { + return HlsRecoveryPressureGuardAccess::Superseded; + } + HlsRecoveryPressureGuardAccess::Acquired(operation(&mut session)) + }); + match access { + HlsCurrentProxySessionAccess::Acquired(access) => access, + HlsCurrentProxySessionAccess::Superseded => HlsRecoveryPressureGuardAccess::Superseded, + HlsCurrentProxySessionAccess::LockBusy => HlsRecoveryPressureGuardAccess::LockBusy, + } + } + + pub fn availability_reevaluations(&self) -> Arc { + Arc::clone(&self.availability_reevaluations) + } + + pub fn notify_session_evidence_changed(&self, proxy_session_id: &ProxySessionId) -> bool { + self.availability_reevaluations.notify_session_evidence_changed(proxy_session_id) + } + + #[cfg(any(test, feature = "test-support"))] + pub async fn hold_access_lease_store_for_test(&self) -> tokio::sync::OwnedRwLockWriteGuard { + Arc::clone(&self.access_leases).write_owned().await + } + + pub async fn prepare_access_lease_manifest_publication( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> Option { + self.access_leases.write().await.prepare_manifest_publication(lease_id, proxy_session_id, now_ms) + } + + pub async fn commit_access_lease_manifest_publication_with_resources( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + expected: HlsLeaseManifestPublicationGuard, + snapshot: HlsLeaseManifestSnapshot, + published_transient_resource_ids: HlsPublishedTransientResourceIds, + now_ms: u64, + ) -> HlsLeaseManifestPublicationOutcome { + let finalized_manifest_generation = snapshot.finalized_transient_manifest_generation; + let lease_issued_at_ms = expected.lease_issued_at_ms(); + let Some(finalized_manifest_generation) = finalized_manifest_generation else { + let session = self.sessions.get_by_proxy_session_id(proxy_session_id).await; + // The lease-store write lock keeps the previous lease membership stable while the + // session read produces an immutable merged membership snapshot. The snapshot can then + // be committed without reopening the lease-store transaction. + let mut leases = self.access_leases.write().await; + let published_transient_resource_ids = if let Some(session) = session.as_ref() { + let session = session.read().await; + if session.proxy_session_id == *proxy_session_id { + leases.published_transient_resource_ids(lease_id).map_or( + published_transient_resource_ids.clone(), + |previous| { + session.transient.merge_current_published_resource_ids( + previous, + published_transient_resource_ids, + now_ms, + ) + }, + ) + } else { + published_transient_resource_ids + } + } else { + published_transient_resource_ids + }; + return leases.commit_manifest_publication_with_resources( + lease_id, + proxy_session_id, + expected, + snapshot, + published_transient_resource_ids, + now_ms, + ); + }; + let Some(current_session) = self.sessions.hold_current_proxy_session(proxy_session_id).await else { + return HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::ManifestGenerationUnavailable, + ); + }; + // The index guard prevents replacement while the established index -> lease store -> session + // lock order publishes the exact snapshot and binds its finalized source generation. + let session = Arc::clone(current_session.session()); + let mut leases = self.access_leases.write().await; + let mut session = session.write().await; + if session.proxy_session_id != *proxy_session_id || session.is_gc_marked_for_removal() { + return HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::ManifestGenerationUnavailable, + ); + } + if !session.transient.has_finalized_manifest_generation(finalized_manifest_generation) { + return HlsLeaseManifestPublicationOutcome::Rejected( + HlsLeaseManifestPublicationRejectReason::ManifestGenerationUnavailable, + ); + } + let outcome = leases.commit_manifest_publication_with_resources( + lease_id, + proxy_session_id, + expected, + snapshot, + published_transient_resource_ids, + now_ms, + ); + if outcome.is_committed() { + let bound = session.transient.bind_finalized_manifest_generation(TransientManifestLeaseBinding::new( + lease_id.clone(), + lease_issued_at_ms, + finalized_manifest_generation, + )); + debug_assert!(bound, "validated finalized manifest generation must remain bindable"); + } + drop(session); + drop(leases); + drop(current_session); + outcome + } + + #[cfg(any(test, feature = "test-support"))] + pub async fn commit_access_lease_manifest_publication( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + expected: HlsLeaseManifestPublicationGuard, + snapshot: HlsLeaseManifestSnapshot, + now_ms: u64, + ) -> HlsLeaseManifestPublicationOutcome { + self.commit_access_lease_manifest_publication_with_resources( + lease_id, + proxy_session_id, + expected, + snapshot, + HlsPublishedTransientResourceIds::default(), + now_ms, + ) + .await + } + + pub async fn prepare_access_lease_runtime_custom_tail( + &self, + session: &HlsSessionHandle, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + reason: HlsRuntimeCustomTailReason, + now_ms: u64, + ) -> Option { + if reason.trigger_class() + != super::super::runtime_custom_tail::HlsRuntimeCustomTailTriggerClass::ImmediatePolicyCutover + { + return None; + } + let current_session = self.sessions.get_by_proxy_session_id(proxy_session_id).await?; + if !Arc::ptr_eq(¤t_session, session) { + return None; + } + let lease = self.access_lease_response_snapshot(lease_id, proxy_session_id, now_ms).await?; + if lease.playback_mode != super::super::terminal_tail::HlsLeasePlaybackMode::Live { + return None; + } + let manifest = lease.last_manifest_snapshot.as_ref()?; + let ( + origin_progress_generation, + media_readiness_generation, + origin_epoch, + last_media_progress_at_ms, + reserve, + recovery_snapshot, + ) = { + let session = session.read().await; + if session.proxy_session_id != *proxy_session_id || session.is_gc_marked_for_removal() { + return None; + } + let ready_timeline = session.ready_timeline_snapshot( + lease.playback_cursor.ready_timeline_start_proxy_seq(manifest.first_proxy_seq), + now_ms, + ); + let reserve = evaluate_lease_reserve(HlsLeaseReserveInput { + manifest, + cursor: &lease.playback_cursor, + ready_timeline: &ready_timeline, + now_ms, + playback_rate_guard_milli: super::super::HLS_PLAYBACK_RATE_GUARD_MILLI, + recovery_trigger_budget: HlsRecoveryTriggerBudgetMs::from_millis(0), + origin_path_degraded: true, + recovery_committed: false, + }); + ( + session.origin_control.progress_generation, + session.activity.media_readiness_generation, + session.origin_control.origin_epoch, + session.origin_control.last_media_progress_at_ms, + reserve, + hls_acceptance_recovery_snapshot(&session, now_ms), + ) + }; + let technical_commit_budget_ms = self + .origin_manifest_timeout_ms() + .max(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()); + let cutover_timing = HlsLeaseCutoverTiming::from_reserve( + now_ms, + technical_commit_budget_ms, + HlsTransitionMarginMs::from_millis(0), + None, + ); + self.access_leases.read().await.prepare_terminal_tail( + lease_id, + proxy_session_id, + &HlsTerminalTailPreparationInput { + trigger: HlsFiniteTailTrigger::RuntimePolicy(reason), + expected_manifest_snapshot_generation: manifest.snapshot_generation, + expected_cursor_generation: lease.playback_cursor.cursor_generation, + origin_progress_generation, + media_readiness_generation, + origin_epoch, + last_media_progress_at_ms, + expected_acceptance_generation: recovery_snapshot.expected_generation, + terminal_media_requirement_origin: HlsTerminalMediaRequirementOrigin::CutoverSnapshot, + cutover_timing, + commit_window: HlsTerminalCommitWindow::CutoverDue, + required_terminal_media_key: None, + terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, + reserve, + }, + ) + } + + pub async fn begin_runtime_policy_revocation( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + reason: HlsRuntimeCustomTailReason, + now_ms: u64, + ) -> HlsRuntimePolicyRevocationOutcome { + self.access_leases.write().await.begin_runtime_policy_revocation(lease_id, proxy_session_id, reason, now_ms) + } + + pub async fn fail_runtime_policy_revocation( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + token: &HlsRuntimePolicyRevocation, + ) -> HlsAccessLeaseDenialOutcome { + let outcome = + self.access_leases.write().await.fail_runtime_policy_revocation(lease_id, proxy_session_id, token); + if matches!(outcome, HlsAccessLeaseDenialOutcome::Ended { .. }) { + self.terminal_pending.cancel_lease(lease_id); + self.terminal_commit_retries.cancel_lease(lease_id); + } + outcome + } + + pub async fn update_access_lease_origin_acquire_policy( + &self, + lease_id: &HlsAccessLeaseId, + connection_kind: tuliprox_session::ConnectionKind, + priority: i8, + ) -> Option { + self.access_leases.write().await.update_origin_acquire_policy(lease_id, connection_kind, priority) + } + + pub async fn activate_access_lease( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + now_ms: u64, + timing: HlsAccessLeaseTiming, + ) -> HlsAccessLeaseActivation { + let activation = + self.access_leases.write().await.activate_access_lease(lease_id, proxy_session_id, now_ms, timing); + if let HlsAccessLeaseActivation::Activated { lease, previous_state } = &activation { + if *previous_state == HlsAccessLeaseState::Pending { + self.segment_repair.ensure_access_lease_window(lease.lease_id.clone()).await; + } else if *previous_state == HlsAccessLeaseState::Idle { + self.segment_repair.start_access_lease_window(lease.lease_id.clone()).await; + } + self.schedule_access_lease_activity(lease).await; + self.schedule_access_lease_validity(lease).await; + } + activation + } + + pub async fn touch_manifest_access_lease( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + now_ms: u64, + active_timing: Option, + pending_deadline: Option, + ttl_ms: u64, + ) -> HlsAccessLeaseTouch { + let touch = self.access_leases.write().await.touch_manifest_access_lease( + lease_id, + proxy_session_id, + now_ms, + active_timing, + pending_deadline, + ttl_ms, + ); + if let HlsAccessLeaseTouch::Touched { lease } = &touch { + if lease.state == HlsAccessLeaseState::Activated { + self.schedule_access_lease_activity(lease).await; + } + self.schedule_access_lease_validity(lease).await; + } + touch + } + + pub async fn mark_pending_manifest_follow_up_for_lease( + &self, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + now_ms: u64, + target_duration: Option, + ) -> bool { + let deadline = hls_pending_manifest_follow_up_deadline(now_ms, target_duration); + let lease = self.access_leases.write().await.mark_pending_manifest_follow_up_for_lease( + lease_id, + proxy_session_id, + now_ms, + deadline, + ); + if let Some(lease) = lease { + self.schedule_access_lease_validity(&lease).await; + debug!( + "HLS pending manifest lease shortened after manifest response: lease={} proxy_session={}", + super::super::safe_hls_access_lease_id(lease_id), + safe_proxy_session_id(proxy_session_id) + ); + true + } else { + false + } + } + + pub async fn mark_pending_manifest_follow_up_for_session( + &self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + target_duration: Option, + ) -> usize { + let deadline = hls_pending_manifest_follow_up_deadline(now_ms, target_duration); + let leases = self.access_leases.write().await.mark_pending_manifest_follow_up_for_session( + proxy_session_id, + now_ms, + deadline, + ); + for lease in &leases { + self.schedule_access_lease_validity(lease).await; + } + leases.len() + } + + pub async fn active_access_lease_count_for_session(&self, proxy_session_id: &ProxySessionId, now_ms: u64) -> usize { + self.access_leases.write().await.active_access_lease_count_for_session(proxy_session_id, now_ms) + } + + pub async fn has_usable_access_lease_for_session(&self, proxy_session_id: &ProxySessionId, now_ms: u64) -> bool { + self.access_leases.write().await.has_usable_access_lease_for_session(proxy_session_id, now_ms) + } + + pub async fn access_lease_session_snapshot( + &self, + proxy_session_id: &ProxySessionId, + now_ms: u64, + ) -> HlsAccessLeaseSessionSnapshot { + self.access_leases.write().await.session_snapshot(proxy_session_id, now_ms) + } + + pub async fn debug_state_summary(&self) -> String { + let sessions = self.sessions.list_sessions().await; + let access_leases = self.access_leases.read().await.len(); + let qos_access_leases = self.qos.len().await; + let repair = self.segment_repair.stats().await; + let mut segments = 0_usize; + let mut maps = 0_usize; + let mut transient_resources = 0_usize; + let mut transient_objects = 0_usize; + let mut active_origin_work = 0_usize; + let mut active_segment_fetches = 0_usize; + let mut active_map_fetches = 0_usize; + for session in &sessions { + let session = session.read().await; + segments = segments.saturating_add(session.segments.len()); + maps = maps.saturating_add(session.maps.len()); + transient_resources = transient_resources.saturating_add(session.transient.resources.len()); + transient_objects = transient_objects.saturating_add(session.transient.object_cache.len()); + active_origin_work = active_origin_work + .saturating_add(session.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire)); + active_segment_fetches = active_segment_fetches.saturating_add(session.active_segment_fetches); + active_map_fetches = active_map_fetches.saturating_add(session.active_map_fetches); + } + format!( + "sessions={} access_leases={} qos_access_leases={} repair_windows={} repair_generations={} repair_candidates={} repair_metadata={} repair_object_metadata={} repair_locks={} repair_watchdog_metadata={} repair_watchdog_locks={} segments={} maps={} transient_resources={} transient_objects={} active_origin_work={} active_segment_fetches={} active_map_fetches={}", + sessions.len(), + access_leases, + qos_access_leases, + repair.windows, + repair.generations, + repair.checked_candidates, + repair.metadata, + repair.object_metadata, + repair.locks, + repair.watchdog_metadata, + repair.watchdog_locks, + segments, + maps, + transient_resources, + transient_objects, + active_origin_work, + active_segment_fetches, + active_map_fetches + ) + } + + pub async fn get_or_create_session( + &self, + key: HlsSessionKey, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ) -> HlsSessionHandle { + self.get_or_create_session_with_outcome(key, reverse_proxy_rewrite_secret, now_ms).await.0 + } + + pub async fn get_or_create_session_with_outcome( + &self, + key: HlsSessionKey, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ) -> (HlsSessionHandle, HlsSessionStoreOutcome) { + let origin_source = HlsOriginSource::from_session_key(&key); + self.get_or_create_session_with_source_and_outcome(key, origin_source, reverse_proxy_rewrite_secret, now_ms) + .await + } + + pub async fn get_or_create_session_with_source_and_outcome( + &self, + key: HlsSessionKey, + origin_source: HlsOriginSource, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ) -> (HlsSessionHandle, HlsSessionStoreOutcome) { + let runtime = self.runtime_config.load(); + let startup = (runtime.startup.mode != shared::model::HlsStartupMode::Conservative + && origin_source.archive_reference.is_none()) + .then(|| super::super::HlsSessionStartup { + first_data_timeout_ms: runtime.segment_fetch_policy.origin_segment_timeout_ms, + worker: Arc::downgrade(&self.segment_worker_pool), + access_leases: Arc::clone(&self.access_leases), + config: runtime.startup.clone(), + revisions: std::collections::BTreeMap::new(), + policy_fixed: false, + store: Arc::clone(&self.revision_store), + budget: Arc::clone(&self.progressive_budget), + }); + drop(runtime); + let (session, outcome) = self + .sessions + .get_or_create_session_with_startup(key, origin_source, reverse_proxy_rewrite_secret, now_ms, startup) + .await; + let (proxy_session_id, session_key) = { + let session_guard = session.read().await; + (safe_proxy_session_id(&session_guard.proxy_session_id), safe_session_key(&session_guard.key)) + }; + match outcome { + HlsSessionStoreOutcome::Created => { + self.metrics.record_session_created(); + info!("HLS session created: session={session_key} proxy_session={proxy_session_id}"); + } + HlsSessionStoreOutcome::Reused => { + self.metrics.record_session_reused(); + debug!("HLS session reused: session={session_key} proxy_session={proxy_session_id}"); + } + } + self.schedule_session_idle_for_handle(&session).await; + session.write().await.configure_segment_prefetch_queue(self.segment_fetch_policy().max_prefetch_queue_depth); + (session, outcome) + } +} + +impl Default for HlsProxyManager { + fn default() -> Self { Self::new() } +} diff --git a/backend/hls/src/manager/terminal.rs b/backend/hls/src/manager/terminal.rs new file mode 100644 index 000000000..fea5d57db --- /dev/null +++ b/backend/hls/src/manager/terminal.rs @@ -0,0 +1,749 @@ +use super::{ + evaluate_terminal_cutover, hls_acceptance_recovery_snapshot, hls_estimated_recovery_completion_at, + hls_key_readiness_evidence_is_current, next_terminal_commit_retry, spawn_terminal_commit_retry_worker, + HlsAcceptanceRecoverySnapshot, HlsAccessLeaseId, HlsAccessLeaseStore, HlsCurrentProxySessionAccess, + HlsFiniteTailTrigger, HlsLeaseCutoverTiming, HlsManifestAcceptanceEpisodeStatus, HlsPreparedTerminalBundleKey, + HlsPreparedTerminalBundleObservation, HlsPreparedTerminalBundleState, HlsProxyManager, + HlsStandaloneCustomAccessEntry, HlsStandaloneCustomSegmentAccess, HlsStandaloneCustomSegmentError, + HlsTerminalAssetRevisionValidation, HlsTerminalCommitAttempt, HlsTerminalCommitCommand, HlsTerminalCommitOutcome, + HlsTerminalCommitOwnerKey, HlsTerminalCommitOwnerToken, HlsTerminalCommitRequest, HlsTerminalCommitRetryDecision, + HlsTerminalCommitRetryScheduleDecision, HlsTerminalCommitSubmissionDecision, HlsTerminalCutoverCapability, + HlsTerminalCutoverDecision, HlsTerminalCutoverInput, HlsTerminalLeaseDecision, HlsTerminalMediaAsset, + HlsTerminalMediaPreparationState, HlsTerminalMediaRequirementOrigin, HlsTerminalMediaRequirementSource, + HlsTerminalPendingCoordinator, HlsTerminalTailCompatibility, HlsTerminalTailPreparation, + HlsTerminalTailPreparationInput, HlsTerminalTailPreparationRequest, HlsTerminalTailProtection, + HlsTerminalTailProtectionInstall, ProxySessionId, +}; +use std::sync::Arc; +use tokio::sync::RwLock; + +#[derive(Clone, Copy)] +enum HlsTerminalPreparationPurpose { + Cutover, + UnavailableAfterOwnerFailure, +} + +pub(super) fn terminal_media_requirement_origin( + recovery_snapshot: &HlsAcceptanceRecoverySnapshot, +) -> HlsTerminalMediaRequirementOrigin { + match recovery_snapshot.status { + HlsManifestAcceptanceEpisodeStatus::Missing => HlsTerminalMediaRequirementOrigin::CutoverSnapshot, + HlsManifestAcceptanceEpisodeStatus::Committed { .. } + | HlsManifestAcceptanceEpisodeStatus::InFlight { .. } + | HlsManifestAcceptanceEpisodeStatus::Expired { .. } + | HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { .. } + | HlsManifestAcceptanceEpisodeStatus::Superseded { .. } => { + HlsTerminalMediaRequirementOrigin::AcceptanceEpisode { generation: recovery_snapshot.expected_generation } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum HlsTerminalCommitAuthorization { + Authorized { protection_capacity_exceeded: bool }, + Rejected(HlsTerminalCommitOutcome), +} + +fn terminal_media_requirement_is_bound_to_preparation(preparation: &HlsTerminalTailPreparation) -> bool { + match preparation.terminal_media_requirement_source { + HlsTerminalMediaRequirementSource::AcceptanceEpisode { generation } => { + generation == preparation.expected_acceptance_generation + } + HlsTerminalMediaRequirementSource::CutoverSnapshotPending { decision_generation } + | HlsTerminalMediaRequirementSource::CutoverSnapshot { decision_generation, .. } => { + decision_generation == preparation.decision_generation + } + } +} + +fn evaluate_terminal_commit_authorization( + session: &super::super::HlsSession, + lease_id: &HlsAccessLeaseId, + preparation: &HlsTerminalTailPreparation, + decision: &HlsTerminalLeaseDecision, + now_ms: u64, +) -> HlsTerminalCommitAuthorization { + let recovery_snapshot = hls_acceptance_recovery_snapshot(session, now_ms); + // Acceptance lifecycle generations describe recovery evidence, not media + // continuity. A later episode must not invalidate a cutover snapshot while + // the media progress, readiness, epoch, and lease generations guarded below + // remain unchanged. + if !terminal_media_requirement_is_bound_to_preparation(preparation) { + return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::SupersededGeneration); + } + let protection_capacity_exceeded = matches!(decision, HlsTerminalLeaseDecision::Tail(_)) + && !session.can_install_terminal_tail_protection(lease_id); + let (terminal, terminal_preparation) = match (decision, protection_capacity_exceeded) { + (HlsTerminalLeaseDecision::Tail(plan), false) => { + let prepared_key = plan.media_preparation_key(); + if preparation.required_terminal_media_key != Some(prepared_key) + || !preparation + .terminal_media_requirement_source + .authorizes_tail(preparation.decision_generation, prepared_key) + { + return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleIncompatible); + } + ( + HlsTerminalCutoverCapability::TailCompatible { prepared_key }, + HlsTerminalMediaPreparationState::Ready { key: prepared_key }, + ) + } + (HlsTerminalLeaseDecision::Tail(_), true) => ( + HlsTerminalCutoverCapability::TailUnavailable(HlsTerminalTailCompatibility::ProtectionCapacityExceeded), + preparation.terminal_media_preparation, + ), + ( + HlsTerminalLeaseDecision::Unavailable(reason) + | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(reason), + _, + ) => (HlsTerminalCutoverCapability::TailUnavailable(*reason), preparation.terminal_media_preparation), + }; + if matches!(decision, HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_)) { + if !session.origin_control.path_condition.is_degraded() { + return HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::CutoverNoLongerRequired); + } + // This fallback publishes no media bytes. Submission arbitration and + // the terminal precondition retain the preparation's original + // exclusive deadline, so this authorization is reachable only while + // the autonomous owner still has time to commit safely. + return HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: false }; + } + if preparation.trigger.is_runtime_policy() { + return HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded }; + } + let cutover = evaluate_terminal_cutover(&HlsTerminalCutoverInput { + reserve: preparation.reserve, + commit_window: preparation.commit_window, + acceptance: recovery_snapshot.status, + required_terminal_media_key: preparation.required_terminal_media_key, + terminal_preparation, + terminal, + }); + match (decision, cutover) { + (HlsTerminalLeaseDecision::Tail(_), HlsTerminalCutoverDecision::CommitTerminalTail) + if !protection_capacity_exceeded => + { + HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: false } + } + ( + HlsTerminalLeaseDecision::Tail(_), + HlsTerminalCutoverDecision::CommitTerminalUnavailable( + HlsTerminalTailCompatibility::ProtectionCapacityExceeded, + ), + ) if protection_capacity_exceeded => { + HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: true } + } + ( + HlsTerminalLeaseDecision::Unavailable(expected) + | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(expected), + HlsTerminalCutoverDecision::CommitTerminalUnavailable(actual), + ) if *expected == actual => HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded: false }, + (_, HlsTerminalCutoverDecision::NotRequired) => { + HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::CutoverNoLongerRequired) + } + (_, HlsTerminalCutoverDecision::RetrySupersededSnapshot) => { + HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::SupersededGeneration) + } + (_, HlsTerminalCutoverDecision::EvaluateTerminalCapability { .. }) => { + HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleNotReady) + } + ( + _, + HlsTerminalCutoverDecision::CommitTerminalTail | HlsTerminalCutoverDecision::CommitTerminalUnavailable(_), + ) => HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleIncompatible), + } +} + +impl HlsProxyManager { + pub fn start_prepared_terminal_bundle( + &self, + asset: Arc, + target_duration_ms: u64, + segment_count: u16, + ) -> HlsPreparedTerminalBundleState { + self.prepared_terminal_bundles.start_preparation(asset, target_duration_ms, segment_count) + } + + pub fn prepared_terminal_bundle_state( + &self, + key: HlsPreparedTerminalBundleKey, + ) -> Option { + self.prepared_terminal_bundles.state(key) + } + + #[cfg(any(test, feature = "test-support"))] + pub fn install_controlled_terminal_bundle_flight_for_test( + &self, + key: HlsPreparedTerminalBundleKey, + ) -> Option { + self.prepared_terminal_bundles.install_controlled_flight_for_test(key) + } + + pub fn observe_prepared_terminal_bundle( + &self, + key: HlsPreparedTerminalBundleKey, + ) -> HlsPreparedTerminalBundleObservation { + self.prepared_terminal_bundles.observe_exact(key) + } + + pub fn register_standalone_custom_access(&self, entry: HlsStandaloneCustomAccessEntry, now_ms: u64) { + self.standalone_custom_access.register(entry, now_ms); + } + + pub fn resolve_standalone_custom_segment( + &self, + lease_id: &HlsAccessLeaseId, + asset_fingerprint: &str, + index: u16, + now_ms: u64, + ) -> Result { + self.standalone_custom_access.resolve(lease_id, asset_fingerprint, index, now_ms) + } + + pub fn terminal_pending(&self) -> Arc { Arc::clone(&self.terminal_pending) } + + /// Cancels terminal work frozen before newly committed shared media + /// progress. Callers must not hold the session or lease-store lock; late + /// registrations remain protected by the final progress-generation CAS. + pub fn cancel_superseded_terminal_work_for_session(&self, proxy_session_id: &ProxySessionId) { + self.terminal_pending.cancel_session(proxy_session_id); + self.terminal_commit_retries.cancel_session(proxy_session_id); + } + + pub fn terminal_commit_now_ms(&self) -> u64 { self.terminal_commit_clock.now_ms() } + + #[cfg(any(test, feature = "test-support"))] + pub async fn wait_for_prepared_terminal_bundle( + &self, + key: HlsPreparedTerminalBundleKey, + ) -> Option { + self.prepared_terminal_bundles.wait_for_completion(key).await + } + + #[cfg(any(test, feature = "test-support"))] + pub fn set_terminal_commit_retry_capacity_for_test(&self, capacity: usize) { + self.terminal_commit_retries.set_capacity_for_test(capacity); + } + + pub async fn prepare_access_lease_terminal_tail( + &self, + request: HlsTerminalTailPreparationRequest<'_>, + ) -> Option { + self.prepare_access_lease_terminal_decision(request, HlsTerminalPreparationPurpose::Cutover).await + } + + pub async fn prepare_access_lease_terminal_unavailable_after_owner_failure( + &self, + request: HlsTerminalTailPreparationRequest<'_>, + ) -> Option { + self.prepare_access_lease_terminal_decision( + request, + HlsTerminalPreparationPurpose::UnavailableAfterOwnerFailure, + ) + .await + } + + async fn prepare_access_lease_terminal_decision( + &self, + request: HlsTerminalTailPreparationRequest<'_>, + purpose: HlsTerminalPreparationPurpose, + ) -> Option { + let HlsTerminalTailPreparationRequest { + lease_id, + proxy_session_id, + manifest_snapshot_generation, + cursor_generation, + reserve, + cutover_timing, + commit_window, + now_ms, + origin_progress_generation: expected_origin_progress_generation, + media_readiness_generation: expected_media_readiness_generation, + last_media_progress_at_ms: expected_last_media_progress_at_ms, + } = request; + let session = self.sessions.get_by_proxy_session_id(proxy_session_id).await?; + let ( + origin_progress_generation, + media_readiness_generation, + origin_epoch, + last_media_progress_at_ms, + origin_path_degraded, + recovery_snapshot, + ) = { + let session = session.read().await; + let recovery_snapshot = hls_acceptance_recovery_snapshot(&session, now_ms); + ( + session.origin_control.progress_generation, + session.activity.media_readiness_generation, + session.origin_control.origin_epoch, + session.origin_control.last_media_progress_at_ms, + session.origin_control.path_condition.is_degraded(), + recovery_snapshot, + ) + }; + let expected_cutover_timing = + HlsLeaseCutoverTiming::from_reserve(now_ms, reserve.guaranteed_reserve_ms, reserve.transition_margin, None); + if origin_progress_generation != expected_origin_progress_generation + || media_readiness_generation != expected_media_readiness_generation + || last_media_progress_at_ms != expected_last_media_progress_at_ms + || cutover_timing != expected_cutover_timing + || !hls_key_readiness_evidence_is_current(reserve.key_readiness_valid_until_ms, now_ms) + { + return None; + } + if matches!(purpose, HlsTerminalPreparationPurpose::UnavailableAfterOwnerFailure) && !origin_path_degraded { + return None; + } + let cutover_timing = cutover_timing + .with_estimated_recovery_completion_at(hls_estimated_recovery_completion_at(recovery_snapshot.recovery)); + if matches!(purpose, HlsTerminalPreparationPurpose::Cutover) + && !matches!( + evaluate_terminal_cutover(&HlsTerminalCutoverInput { + reserve, + commit_window, + acceptance: recovery_snapshot.status, + required_terminal_media_key: recovery_snapshot.required_terminal_media_key, + terminal_preparation: recovery_snapshot.terminal_media_preparation, + terminal: HlsTerminalCutoverCapability::NotEvaluated, + }), + HlsTerminalCutoverDecision::EvaluateTerminalCapability { .. } + ) + { + return None; + } + let terminal_media_requirement_origin = terminal_media_requirement_origin(&recovery_snapshot); + self.access_leases.read().await.prepare_terminal_tail( + lease_id, + proxy_session_id, + &HlsTerminalTailPreparationInput { + trigger: HlsFiniteTailTrigger::AvailabilityReserve, + expected_manifest_snapshot_generation: manifest_snapshot_generation, + expected_cursor_generation: cursor_generation, + origin_progress_generation, + media_readiness_generation, + origin_epoch, + last_media_progress_at_ms, + expected_acceptance_generation: recovery_snapshot.expected_generation, + terminal_media_requirement_origin, + cutover_timing, + commit_window, + required_terminal_media_key: recovery_snapshot.required_terminal_media_key, + terminal_media_preparation: recovery_snapshot.terminal_media_preparation, + reserve, + }, + ) + } + + pub fn commit_access_lease_terminal_if_generation_matches( + &self, + request: HlsTerminalCommitRequest<'_>, + ) -> HlsTerminalCommitOutcome { + let HlsTerminalCommitRequest { + session, + lease_id, + proxy_session_id, + preparation, + now_ms, + payload, + asset_revision_guard, + } = request; + let (decision, media_guard) = payload.into_parts(); + let key = HlsTerminalCommitOwnerKey::from_preparation(proxy_session_id, lease_id, preparation); + let Some(session_incarnation) = self.sessions.session_incarnation(session) else { + return HlsTerminalCommitOutcome::SupersededGeneration; + }; + let Some((cancellation_epoch, submission_token)) = self.terminal_commit_retries.reserve_submission() else { + return HlsTerminalCommitOutcome::RetryCapacityExceeded; + }; + let command = HlsTerminalCommitCommand { + key, + session: Arc::clone(session), + session_incarnation, + preparation: preparation.clone(), + decision, + media_guard, + asset_revision_guard, + cancellation_epoch, + submission_token, + }; + let attempt_now_ms = self.terminal_commit_clock.initial_attempt_now_ms(now_ms); + let (authorized_command, owner_token) = match self.terminal_commit_retries.submit(command, attempt_now_ms) { + HlsTerminalCommitSubmissionDecision::Attempt { command, owner_token } => (command, owner_token), + HlsTerminalCommitSubmissionDecision::PendingExisting { retry_before_ms } => { + return HlsTerminalCommitOutcome::LockBusy { retry_before_ms }; + } + HlsTerminalCommitSubmissionDecision::Failed(outcome) => return outcome, + HlsTerminalCommitSubmissionDecision::Cancelled => { + return HlsTerminalCommitOutcome::SupersededGeneration; + } + HlsTerminalCommitSubmissionDecision::CapacityExceeded => { + return HlsTerminalCommitOutcome::RetryCapacityExceeded; + } + }; + let authorized_key = authorized_command.key.clone(); + let attempt = match self.sessions.try_with_current_proxy_session( + &authorized_key.proxy_session_id, + &authorized_command.session, + || { + self.terminal_commit_retries.with_current_owner( + &authorized_key, + owner_token, + |command, latest_safe_terminal_commit_at_ms| { + let attempt_now_ms = self.terminal_commit_clock.initial_attempt_now_ms(now_ms); + let attempt = Self::try_commit_access_lease_terminal_decision( + &self.access_leases, + command, + attempt_now_ms, + latest_safe_terminal_commit_at_ms, + ); + (attempt_now_ms, latest_safe_terminal_commit_at_ms, attempt) + }, + ) + }, + ) { + HlsCurrentProxySessionAccess::Acquired(attempt) => attempt, + HlsCurrentProxySessionAccess::Superseded => { + self.terminal_commit_retries.discard_owner(&authorized_key, owner_token); + None + } + HlsCurrentProxySessionAccess::LockBusy => self.terminal_commit_retries.with_current_owner( + &authorized_key, + owner_token, + |_command, latest_safe_terminal_commit_at_ms| { + ( + self.terminal_commit_clock.initial_attempt_now_ms(now_ms), + latest_safe_terminal_commit_at_ms, + HlsTerminalCommitAttempt::LockBusy, + ) + }, + ), + }; + let Some((attempts_completed, (attempt_now_ms, deadline_ms, attempt))) = attempt else { + return HlsTerminalCommitOutcome::SupersededGeneration; + }; + match attempt { + HlsTerminalCommitAttempt::Completed(outcome) => { + self.terminal_commit_retries.complete_owner(&authorized_key, owner_token); + outcome + } + HlsTerminalCommitAttempt::LockBusy => self.schedule_terminal_commit_retry( + &authorized_key, + owner_token, + attempts_completed, + attempt_now_ms, + deadline_ms, + ), + } + } + + pub(crate) fn try_commit_access_lease_terminal_decision( + access_leases: &Arc>, + command: &HlsTerminalCommitCommand, + now_ms: u64, + latest_safe_commit_at_ms: u64, + ) -> HlsTerminalCommitAttempt { + // Submission arbitration releases the owner mutex before this path. + // This follows the cross-store order documented by + // with_current_recovery_pressure_session: session index -> retry owner + // -> lease store -> session. The outer two guards prevent replacement + // or cancellation TOCTOU; all inner locks are non-blocking and no + // guard crosses an await or I/O boundary. + let Ok(mut leases) = access_leases.try_write() else { + return HlsTerminalCommitAttempt::LockBusy; + }; + let Ok(mut session) = command.session.try_write() else { + return HlsTerminalCommitAttempt::LockBusy; + }; + if let Some(outcome) = + Self::terminal_commit_precondition_outcome(&mut leases, &session, command, now_ms, latest_safe_commit_at_ms) + { + return HlsTerminalCommitAttempt::Completed(outcome); + } + let lease_id = &command.key.lease_id; + let preparation = &command.preparation; + let decision = &command.decision; + // Asset identity gates only a new tail mutation. An exact terminal + // replay was handled above and remains immutable. If a prepared tail's + // asset changed while a LockBusy owner was queued, fail closed in this + // same generation-bound CAS instead of abandoning the autonomous + // intent and leaving the lease live. + match command.asset_revision_guard.validate_current() { + HlsTerminalAssetRevisionValidation::Current => {} + HlsTerminalAssetRevisionValidation::Changed { current } => { + if command.preparation.trigger.is_runtime_policy() { + return HlsTerminalCommitAttempt::Completed(HlsTerminalCommitOutcome::SupersededGeneration); + } + let reason = current.asset.map_or(HlsTerminalTailCompatibility::MissingAsset, |_| { + HlsTerminalTailCompatibility::AssetRevisionMismatch + }); + return HlsTerminalCommitAttempt::Completed(Self::commit_terminal_unavailable_fallback( + &mut leases, + &mut session, + command, + now_ms, + reason, + )); + } + } + let protection_capacity_exceeded = + match evaluate_terminal_commit_authorization(&session, lease_id, preparation, decision, now_ms) { + HlsTerminalCommitAuthorization::Authorized { protection_capacity_exceeded } => { + protection_capacity_exceeded + } + HlsTerminalCommitAuthorization::Rejected(HlsTerminalCommitOutcome::BundleIncompatible) + if matches!(decision, HlsTerminalLeaseDecision::Tail(_)) => + { + return HlsTerminalCommitAttempt::Completed(Self::commit_terminal_unavailable_fallback( + &mut leases, + &mut session, + command, + now_ms, + HlsTerminalTailCompatibility::TerminalMediaNotReady, + )); + } + HlsTerminalCommitAuthorization::Rejected(outcome) => { + return HlsTerminalCommitAttempt::Completed(outcome); + } + }; + HlsTerminalCommitAttempt::Completed(Self::publish_terminal_commit( + &mut leases, + &mut session, + command, + now_ms, + protection_capacity_exceeded, + )) + } + + fn terminal_commit_precondition_outcome( + leases: &mut HlsAccessLeaseStore, + session: &super::super::HlsSession, + command: &HlsTerminalCommitCommand, + now_ms: u64, + latest_safe_commit_at_ms: u64, + ) -> Option { + let lease_id = &command.key.lease_id; + let proxy_session_id = &command.key.proxy_session_id; + let preparation = &command.preparation; + let replay = match &command.decision { + HlsTerminalLeaseDecision::Tail(_) => { + leases.terminal_tail_replay_outcome(lease_id, proxy_session_id, preparation, now_ms) + } + HlsTerminalLeaseDecision::Unavailable(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_) => { + leases.terminal_unavailable_replay_outcome(lease_id, proxy_session_id, preparation, now_ms) + } + }; + if replay.is_some() { + return replay; + } + if now_ms >= latest_safe_commit_at_ms { + return Some(HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed); + } + if preparation.trigger.is_runtime_policy() { + return None; + } + if session.origin_control.progress_generation == preparation.origin_progress_generation + && session.activity.media_readiness_generation == preparation.media_readiness_generation + && session.origin_control.origin_epoch == preparation.origin_epoch + && session.origin_control.last_media_progress_at_ms == preparation.last_media_progress_at_ms + && hls_key_readiness_evidence_is_current(preparation.reserve.key_readiness_valid_until_ms, now_ms) + { + return None; + } + Some(if session.origin_control.last_media_progress_at_ms == preparation.last_media_progress_at_ms { + HlsTerminalCommitOutcome::SupersededGeneration + } else { + HlsTerminalCommitOutcome::RecoveryCommitted + }) + } + + fn publish_terminal_commit( + leases: &mut HlsAccessLeaseStore, + session: &mut super::super::HlsSession, + command: &HlsTerminalCommitCommand, + now_ms: u64, + protection_capacity_exceeded: bool, + ) -> HlsTerminalCommitOutcome { + let lease_id = &command.key.lease_id; + let proxy_session_id = &command.key.proxy_session_id; + let preparation = &command.preparation; + let decision = &command.decision; + let protection = match (decision, protection_capacity_exceeded) { + (HlsTerminalLeaseDecision::Tail(plan), false) => Some(HlsTerminalTailProtection { + generation: plan.generation, + base_proxy_seqs: Arc::clone(&plan.protected_base_proxy_seqs), + key_bindings: plan.key_bindings(), + }), + (HlsTerminalLeaseDecision::Tail(_), true) + | ( + HlsTerminalLeaseDecision::Unavailable(_) | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_), + _, + ) => None, + }; + let previous_protection = session.remove_terminal_tail_protection(lease_id); + if let Some(protection) = protection { + if session.install_terminal_tail_protection(lease_id.clone(), protection) + != HlsTerminalTailProtectionInstall::Installed + { + session.rollback_terminal_tail_protection(lease_id.clone(), previous_protection); + return Self::commit_terminal_unavailable_fallback( + leases, + session, + command, + now_ms, + HlsTerminalTailCompatibility::ProtectionCapacityExceeded, + ); + } + } + let outcome = match (decision, protection_capacity_exceeded) { + (HlsTerminalLeaseDecision::Tail(plan), false) => leases.commit_terminal_tail_if_generation_matches( + lease_id, + proxy_session_id, + preparation, + now_ms, + Arc::clone(plan), + ), + (HlsTerminalLeaseDecision::Tail(_), true) => leases.commit_terminal_unavailable_if_generation_matches( + lease_id, + proxy_session_id, + preparation, + now_ms, + HlsTerminalTailCompatibility::ProtectionCapacityExceeded, + ), + ( + HlsTerminalLeaseDecision::Unavailable(reason) + | HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(reason), + _, + ) => leases.commit_terminal_unavailable_if_generation_matches( + lease_id, + proxy_session_id, + preparation, + now_ms, + *reason, + ), + }; + if outcome != HlsTerminalCommitOutcome::Committed { + session.rollback_terminal_tail_protection(lease_id.clone(), previous_protection); + if outcome == HlsTerminalCommitOutcome::BundleIncompatible + && matches!(decision, HlsTerminalLeaseDecision::Tail(_)) + { + return Self::commit_terminal_unavailable_fallback( + leases, + session, + command, + now_ms, + HlsTerminalTailCompatibility::TerminalMediaNotReady, + ); + } + return outcome; + } + Self::finish_terminal_commit(leases, session, proxy_session_id); + HlsTerminalCommitOutcome::Committed + } + + fn commit_terminal_unavailable_fallback( + leases: &mut HlsAccessLeaseStore, + session: &mut super::super::HlsSession, + command: &HlsTerminalCommitCommand, + now_ms: u64, + reason: HlsTerminalTailCompatibility, + ) -> HlsTerminalCommitOutcome { + let lease_id = &command.key.lease_id; + let proxy_session_id = &command.key.proxy_session_id; + let decision = if matches!(&command.decision, HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(_)) { + HlsTerminalLeaseDecision::UnavailableAfterOwnerFailure(reason) + } else { + HlsTerminalLeaseDecision::Unavailable(reason) + }; + match evaluate_terminal_commit_authorization(session, lease_id, &command.preparation, &decision, now_ms) { + HlsTerminalCommitAuthorization::Authorized { .. } => {} + HlsTerminalCommitAuthorization::Rejected(outcome) => return outcome, + } + let previous_protection = session.remove_terminal_tail_protection(lease_id); + let outcome = leases.commit_terminal_unavailable_if_generation_matches( + lease_id, + proxy_session_id, + &command.preparation, + now_ms, + reason, + ); + if outcome != HlsTerminalCommitOutcome::Committed { + session.rollback_terminal_tail_protection(lease_id.clone(), previous_protection); + return outcome; + } + Self::finish_terminal_commit(leases, session, proxy_session_id); + HlsTerminalCommitOutcome::Committed + } + + fn finish_terminal_commit( + leases: &mut HlsAccessLeaseStore, + session: &mut super::super::HlsSession, + proxy_session_id: &ProxySessionId, + ) { + let all_terminal = leases.all_live_leases_terminal_for_session(proxy_session_id); + session.origin_control.progress_phase = if all_terminal { + super::super::origin_progress::HlsOriginProgressPhase::Terminal + } else { + super::super::origin_progress::HlsOriginProgressPhase::TerminalPartial + }; + } + + fn schedule_terminal_commit_retry( + &self, + key: &HlsTerminalCommitOwnerKey, + owner_token: HlsTerminalCommitOwnerToken, + attempts_completed: u8, + last_attempt_at_ms: u64, + latest_safe_terminal_commit_at_ms: u64, + ) -> HlsTerminalCommitOutcome { + let (retry_at_ms, attempts_completed) = + match next_terminal_commit_retry(attempts_completed, last_attempt_at_ms, latest_safe_terminal_commit_at_ms) + { + HlsTerminalCommitRetryDecision::Schedule { retry_at_ms, attempts_completed } => { + (retry_at_ms, attempts_completed) + } + HlsTerminalCommitRetryDecision::AttemptsExhausted => { + self.terminal_commit_retries.fail_owner( + key, + owner_token, + HlsTerminalCommitOutcome::RetryAttemptsExhausted, + ); + return HlsTerminalCommitOutcome::RetryAttemptsExhausted; + } + HlsTerminalCommitRetryDecision::SafeDeadlineElapsed => { + self.terminal_commit_retries.fail_owner( + key, + owner_token, + HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed, + ); + return HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed; + } + }; + let registration_now_ms = self.terminal_commit_clock.now_ms().max(last_attempt_at_ms); + match self.terminal_commit_retries.schedule_current( + key, + owner_token, + attempts_completed, + retry_at_ms, + registration_now_ms, + ) { + HlsTerminalCommitRetryScheduleDecision::Scheduled { worker_token } => { + if let Some(worker_token) = worker_token { + spawn_terminal_commit_retry_worker( + Arc::clone(&self.sessions), + Arc::clone(&self.access_leases), + Arc::clone(&self.terminal_commit_retries), + Arc::clone(&self.terminal_commit_clock), + worker_token, + Self::try_commit_access_lease_terminal_decision, + ); + } + HlsTerminalCommitOutcome::LockBusy { retry_before_ms: retry_at_ms } + } + HlsTerminalCommitRetryScheduleDecision::Failed(outcome) => outcome, + HlsTerminalCommitRetryScheduleDecision::Cancelled => HlsTerminalCommitOutcome::SupersededGeneration, + HlsTerminalCommitRetryScheduleDecision::WorkerUnavailable => { + HlsTerminalCommitOutcome::RetryWorkerUnavailable + } + } + } +} diff --git a/backend/hls/src/manager/tests.rs b/backend/hls/src/manager/tests.rs new file mode 100644 index 000000000..58cad7626 --- /dev/null +++ b/backend/hls/src/manager/tests.rs @@ -0,0 +1,45 @@ +use super::{ + super::{ + lease::{ + HlsAccessLeaseDenialMode, HlsAccessLeaseDenialOutcome, HlsLeaseManifestPublicationOutcome, + HlsLeaseManifestPublicationRejectReason, HlsTerminalMediaRequirementSource, + }, + manifest_acceptance::HlsManifestAcceptanceGeneration, + prepared_terminal_bundle::prepared_terminal_bundle_key, + recovery_timing::{ + HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, HlsLeaseCutoverTiming, + HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsRecoveryEtaMs, HlsRecoveryTimingPolicy, + HlsRecoveryWorkload, HlsTerminalCommitAcquisitionBudgetMs, HlsTerminalCommitWindow, + HlsTerminalMediaPreparationState, HlsTransitionMarginMs, + }, + runtime_custom_tail::HlsRuntimeCustomTailAssetIdentity, + session::{HlsTerminalTailProtection, HLS_TERMINAL_TAIL_PROTECTION_CAPACITY}, + terminal_commit::{ + HlsTerminalAssetRevisionGuard, HlsTerminalCommitCommand, HlsTerminalCommitOutcome, + HlsTerminalCommitOwnerKey, HlsTerminalCommitRetryDecision, HlsTerminalCommitRetryScheduleDecision, + HlsTerminalCommitSubmissionDecision, HlsTerminalLeaseDecision, + }, + terminal_tail::{snapshot_terminal_media_asset, HlsTerminalCommitMediaGuard, HlsTerminalTailPlan}, + }, + hls_acceptance_recovery_snapshot, hls_key_readiness_evidence_is_current, hls_session_idle_protection_retry_at, + next_terminal_commit_retry, spawn_terminal_commit_retry_worker, HlsCriticalHandoffStateAccess, + HlsMediaActivityCommitOutcome, HlsProxyManager, HlsRecoveryExecutionState, HlsTerminalCommitPayload, + HlsTerminalCommitRequest, HlsTerminalTailPreparationRequest, HLS_SESSION_IDLE_PROTECTION_RETRY_MS, +}; + +mod admission; +mod behavior; +mod configuration; +mod lifecycle; +mod persistence; +mod retry; +mod startup; +mod support; +mod terminal; + +use self::support::{ + access_lease, begin_test_acceptance_episode, commit_terminal_plan, complete_failed_acceptance_episode, + config_with_hls_cache, cutover_reserve, finalized_transient_manifest_snapshot, live_media_fixture, + manifest_snapshot, prepared_terminal_commit_fixture, publish_manifest_snapshot, terminal_plan, + terminal_preparation_request, test_app_config, +}; diff --git a/backend/hls/src/manager/tests/admission.rs b/backend/hls/src/manager/tests/admission.rs new file mode 100644 index 000000000..9f2b88646 --- /dev/null +++ b/backend/hls/src/manager/tests/admission.rs @@ -0,0 +1,33 @@ +use super::{live_media_fixture, HlsMediaActivityCommitOutcome}; + +#[tokio::test] +async fn contiguous_live_completion_wakes_capacity_protection_waiters() { + let (manager, session, proxy_session_id, lease_id, live_identity) = live_media_fixture("capacity-release").await; + let revision = manager.segment_cache.capacity_revision(); + let mut capacity_wait = Box::pin(manager.segment_cache.wait_for_capacity_change(&revision)); + assert!(matches!(futures::poll!(capacity_wait.as_mut()), std::task::Poll::Pending)); + let token = manager + .record_access_lease_segment_request_started_if_identity_matches( + &lease_id, + &proxy_session_id, + live_identity, + 40, + 2_000, + ) + .await + .expect("current live request token"); + + let outcome = manager + .record_access_lease_segment_request_completed_and_mark_media_if_identity_matches( + &session, + &lease_id, + &proxy_session_id, + live_identity, + token, + 2_100, + ) + .await; + + assert_eq!(outcome, HlsMediaActivityCommitOutcome::Committed); + assert!(matches!(futures::poll!(capacity_wait.as_mut()), std::task::Poll::Ready(()))); +} diff --git a/backend/hls/src/manager/tests/behavior.rs b/backend/hls/src/manager/tests/behavior.rs new file mode 100644 index 000000000..7661e2514 --- /dev/null +++ b/backend/hls/src/manager/tests/behavior.rs @@ -0,0 +1,447 @@ +use super::{ + access_lease, finalized_transient_manifest_snapshot, live_media_fixture, publish_manifest_snapshot, + HlsCriticalHandoffStateAccess, HlsMediaActivityCommitOutcome, HlsProxyManager, +}; +use crate::{ + HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseState, HlsPlaybackFamilyKey, HlsPublishedTransientResourceIds, + HlsSessionKey, HlsSessionStoreOutcome, TransientResourceKind, TransientResourceRef, +}; +use std::sync::Arc; + +pub(in crate::manager::tests) async fn wait_until_access_lease_store_is_write_locked(manager: &HlsProxyManager) { + for _ in 0..1_000 { + if manager.access_leases.try_write().is_err() { + return; + } + tokio::task::yield_now().await; + } + panic!("access lease store was not locked by the reconciliation transaction"); +} + +#[tokio::test] +async fn critical_handoff_distinguishes_lock_busy_from_acquired_state() { + let (manager, session, _, _, _) = live_media_fixture("critical-lock-busy").await; + let lease_guard = manager.access_leases.write().await; + + let busy = manager.with_critical_handoff_state(&session, |_, _| 7_u8).await; + + assert_eq!(busy, HlsCriticalHandoffStateAccess::LockBusy); + drop(lease_guard); + + let acquired = manager.with_critical_handoff_state(&session, |_, _| 7_u8).await; + + assert_eq!(acquired, HlsCriticalHandoffStateAccess::Acquired(7)); +} + +#[tokio::test] +async fn stale_session_handle_cannot_mark_recovered_session_with_same_public_id() { + let (manager, stale_session, proxy_session_id, lease_id, lease_identity) = + live_media_fixture("session-recovery").await; + let session_key = stale_session.read().await.key.clone(); + manager.sessions.remove_session(&session_key, &proxy_session_id).await.expect("remove original session"); + let (recovered_session, outcome) = manager.get_or_create_session_with_outcome(session_key, b"secret", 2_000).await; + assert_eq!(outcome, HlsSessionStoreOutcome::Created); + assert!(!Arc::ptr_eq(&stale_session, &recovered_session)); + + assert_eq!( + manager + .mark_authorized_media_access_for_lease_if_identity_matches( + &stale_session, + &lease_id, + &proxy_session_id, + lease_identity, + 2_100, + ) + .await, + HlsMediaActivityCommitOutcome::StaleLeaseIdentity + ); + assert_eq!(stale_session.read().await.activity.last_authorized_media_at_ms, None); + assert_eq!(recovered_session.read().await.activity.last_authorized_media_at_ms, None); +} + +#[tokio::test] +#[allow(clippy::similar_names, clippy::too_many_lines)] +async fn finalized_signed_url_generations_follow_published_lease_lifetime() { + let manager = HlsProxyManager::new(); + let published_resource_ids = HlsPublishedTransientResourceIds::default(); + let session = manager.get_or_create_session(HlsSessionKey::new(1, "archive"), b"secret", 0).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_a_id = HlsAccessLeaseId("lease-a".to_string()); + let lease_b_id = HlsAccessLeaseId("lease-b".to_string()); + let resource_a = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://cdn.example/archive/segment.ts?token=A", + b"secret", + 0, + 20, + Some("ts".to_string()), + ); + let resource_a_id = resource_a.id.clone(); + let resource_a_uri = format!("/hls/shared/live/{}/lease-a/r/{}.ts", proxy_session_id.0, resource_a_id.0); + let generation_a = { + let mut session = session.write().await; + session.transient.upsert_resources([resource_a]); + session.transient.replace_manifest_with_semantics( + format!( + "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{resource_a_uri}\n#EXT-X-ENDLIST\n" + ), + 0, + Some(6_000), + ); + session.transient.current_finalized_manifest_generation().expect("G1 finalized generation") + }; + + let lease_a = HlsAccessLease::pending( + lease_a_id.clone(), + HlsPlaybackFamilyKey::new("alice", "client-a"), + proxy_session_id.clone(), + "alice".to_string(), + "session-a".to_string(), + 1, + "archive".to_string(), + 1, + 0, + 40, + ); + manager.prepare_access_lease(lease_a).await; + assert!( + publish_manifest_snapshot( + &manager, + &lease_a_id, + &proxy_session_id, + finalized_transient_manifest_snapshot(0, generation_a, &resource_a_uri), + 1, + ) + .await + ); + + let resource_b = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://cdn.example/archive/segment.ts?token=B", + b"secret", + 21, + 20, + Some("ts".to_string()), + ); + let resource_b_id = resource_b.id.clone(); + assert_ne!(resource_a_id, resource_b_id); + let resource_b_uri = format!("/hls/shared/live/{}/lease-b/r/{}.ts", proxy_session_id.0, resource_b_id.0); + let generation_b = { + let mut session = session.write().await; + session.transient.upsert_resources([resource_b]); + session.transient.replace_manifest_with_semantics( + format!( + "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{resource_b_uri}\n#EXT-X-ENDLIST\n" + ), + 21, + Some(6_000), + ); + session.transient.current_finalized_manifest_generation().expect("G2 finalized generation") + }; + + assert!(session + .read() + .await + .transient + .resolve_resource_for_lease(&resource_a_id, &lease_a_id, 0, &published_resource_ids, 21) + .is_some()); + assert_eq!(session.read().await.transient.finalized_manifest_generation_count(), 2); + + let lease_b = HlsAccessLease::pending( + lease_b_id.clone(), + HlsPlaybackFamilyKey::new("bob", "client-b"), + proxy_session_id.clone(), + "bob".to_string(), + "session-b".to_string(), + 1, + "archive".to_string(), + 1, + 21, + 100, + ); + manager.prepare_access_lease(lease_b).await; + assert!( + publish_manifest_snapshot( + &manager, + &lease_b_id, + &proxy_session_id, + finalized_transient_manifest_snapshot(21, generation_b, &resource_b_uri), + 22, + ) + .await + ); + assert!(session + .read() + .await + .transient + .resolve_resource_for_lease(&resource_b_id, &lease_b_id, 21, &published_resource_ids, 42) + .is_some()); + assert!(session + .read() + .await + .transient + .resolve_resource_for_lease(&resource_a_id, &lease_a_id, 0, &published_resource_ids, 39) + .is_some()); + assert_eq!(session.read().await.transient.finalized_manifest_lease_binding_count(), 2); + { + let session = session.read().await; + assert_eq!(session.transient.current_manifest_resource_ids().len(), 1); + assert!(session.transient.resolve_current_resource(&resource_b_id, 42).is_some()); + } + + let expired_a = manager + .access_lease_response_snapshot(&lease_a_id, &proxy_session_id, 41) + .await + .expect("expired lease remains available for lifecycle cleanup"); + assert_eq!(expired_a.state, HlsAccessLeaseState::Expired); + manager.remove_access_lease(&lease_a_id).await; + let mut session = session.write().await; + assert_eq!(session.transient.finalized_manifest_generation_count(), 1); + assert_eq!(session.transient.finalized_manifest_lease_binding_count(), 1); + assert!(session.transient.resolve_current_resource(&resource_a_id, 41).is_none()); + session.transient.prune_expired(41); + assert!(!session.transient.resources.contains_key(&resource_a_id)); + assert!(session.transient.resolve_current_resource(&resource_b_id, 42).is_some()); +} + +#[tokio::test] +#[allow(clippy::similar_names, clippy::too_many_lines)] +async fn same_lease_retains_every_published_finalized_generation_until_removal() { + let manager = HlsProxyManager::new(); + let published_resource_ids = HlsPublishedTransientResourceIds::default(); + let session = manager.get_or_create_session(HlsSessionKey::new(1, "archive"), b"secret", 0).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("alice", "client-a"), + proxy_session_id.clone(), + "alice".to_string(), + "session-a".to_string(), + 1, + "archive".to_string(), + 1, + 0, + 100, + )) + .await; + + let (resource_a_id, generation_a, resource_a_uri) = { + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://cdn.example/archive/segment.ts?token=A", + b"secret", + 0, + 20, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + let resource_uri = format!("/hls/shared/live/{}/lease-a/r/{}.ts", proxy_session_id.0, resource_id.0); + let mut session = session.write().await; + session.transient.upsert_resources([resource]); + session.transient.replace_manifest_with_semantics( + format!( + "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{resource_uri}\n#EXT-X-ENDLIST\n" + ), + 0, + Some(6_000), + ); + let generation = session.transient.current_finalized_manifest_generation().expect("G1 finalized generation"); + (resource_id, generation, resource_uri) + }; + assert!( + publish_manifest_snapshot( + &manager, + &lease_id, + &proxy_session_id, + finalized_transient_manifest_snapshot(0, generation_a, &resource_a_uri), + 1, + ) + .await + ); + + let (resource_b_id, generation_b, resource_b_uri) = { + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://cdn.example/archive/segment.ts?token=B", + b"secret", + 21, + 20, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + let resource_uri = format!("/hls/shared/live/{}/lease-a/r/{}.ts", proxy_session_id.0, resource_id.0); + let mut session = session.write().await; + session.transient.upsert_resources([resource]); + session.transient.replace_manifest_with_semantics( + format!( + "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{resource_uri}\n#EXT-X-ENDLIST\n" + ), + 21, + Some(6_000), + ); + let generation = session.transient.current_finalized_manifest_generation().expect("G2 finalized generation"); + (resource_id, generation, resource_uri) + }; + assert_ne!(resource_a_id, resource_b_id); + assert!( + publish_manifest_snapshot( + &manager, + &lease_id, + &proxy_session_id, + finalized_transient_manifest_snapshot(21, generation_b, &resource_b_uri), + 22, + ) + .await + ); + + { + let session = session.read().await; + assert_eq!(session.transient.finalized_manifest_lease_binding_count(), 2); + assert!(session + .transient + .resolve_resource_for_lease(&resource_a_id, &lease_id, 0, &published_resource_ids, 42) + .is_some()); + assert!(session + .transient + .resolve_resource_for_lease(&resource_b_id, &lease_id, 0, &published_resource_ids, 42) + .is_some()); + } + + manager.remove_access_lease(&lease_id).await; + let session = session.read().await; + assert_eq!(session.transient.finalized_manifest_lease_binding_count(), 0); + assert_eq!(session.transient.finalized_manifest_generation_count(), 1); + assert!(session.transient.resolve_current_resource(&resource_a_id, 42).is_none()); + assert!(session.transient.resolve_current_resource(&resource_b_id, 42).is_some()); +} + +#[tokio::test] +async fn lease_reconciliation_cannot_overwrite_a_concurrent_manifest_publication() { + let manager = Arc::new(HlsProxyManager::new()); + let session = manager.get_or_create_session(HlsSessionKey::new(1, "archive"), b"secret", 0).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://cdn.example/archive/segment.ts?token=A", + b"secret", + 0, + 20, + Some("ts".to_string()), + ); + let resource_uri = format!("/hls/shared/live/{}/lease-a/r/{}.ts", proxy_session_id.0, resource.id.0); + let generation = { + let mut session = session.write().await; + session.transient.upsert_resources([resource]); + session.transient.replace_manifest_with_semantics( + format!( + "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{resource_uri}\n#EXT-X-ENDLIST\n" + ), + 0, + Some(6_000), + ); + session.transient.current_finalized_manifest_generation().expect("finalized generation") + }; + manager.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)).await; + let publication_guard = manager + .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, 1) + .await + .expect("publication guard"); + + let session_guard = session.write().await; + let reconcile_task = { + let manager = Arc::clone(&manager); + let session = Arc::clone(&session); + let proxy_session_id = proxy_session_id.clone(); + tokio::spawn( + async move { manager.reconcile_session_access_lease_snapshot(&session, &proxy_session_id, 1).await }, + ) + }; + wait_until_access_lease_store_is_write_locked(&manager).await; + let publication_task = { + let manager = Arc::clone(&manager); + let proxy_session_id = proxy_session_id.clone(); + let lease_id = lease_id.clone(); + tokio::spawn(async move { + manager + .commit_access_lease_manifest_publication( + &lease_id, + &proxy_session_id, + publication_guard, + finalized_transient_manifest_snapshot(0, generation, &resource_uri), + 2, + ) + .await + }) + }; + drop(session_guard); + + reconcile_task.await.expect("reconciliation completes"); + assert!(publication_task.await.expect("publication completes").is_committed()); + assert_eq!(session.read().await.transient.finalized_manifest_lease_binding_count(), 1); +} + +#[tokio::test] +async fn lease_reconciliation_cannot_resurrect_a_concurrently_removed_binding() { + let manager = Arc::new(HlsProxyManager::new()); + let session = manager.get_or_create_session(HlsSessionKey::new(1, "archive"), b"secret", 0).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://cdn.example/archive/segment.ts?token=A", + b"secret", + 0, + 20, + Some("ts".to_string()), + ); + let resource_uri = format!("/hls/shared/live/{}/lease-a/r/{}.ts", proxy_session_id.0, resource.id.0); + let generation = { + let mut session = session.write().await; + session.transient.upsert_resources([resource]); + session.transient.replace_manifest_with_semantics( + format!( + "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n{resource_uri}\n#EXT-X-ENDLIST\n" + ), + 0, + Some(6_000), + ); + session.transient.current_finalized_manifest_generation().expect("finalized generation") + }; + manager.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)).await; + assert!( + publish_manifest_snapshot( + &manager, + &lease_id, + &proxy_session_id, + finalized_transient_manifest_snapshot(0, generation, &resource_uri), + 1, + ) + .await + ); + let removal = + manager.access_leases.read().await.prepare_access_lease_removal(&lease_id).expect("removal preparation"); + + let session_guard = session.write().await; + let reconcile_task = { + let manager = Arc::clone(&manager); + let session = Arc::clone(&session); + let proxy_session_id = proxy_session_id.clone(); + tokio::spawn( + async move { manager.reconcile_session_access_lease_snapshot(&session, &proxy_session_id, 1).await }, + ) + }; + wait_until_access_lease_store_is_write_locked(&manager).await; + let removal_task = { + let manager = Arc::clone(&manager); + let lease_id = lease_id.clone(); + tokio::spawn(async move { manager.remove_prepared_access_lease(&lease_id, &removal).await }) + }; + drop(session_guard); + + reconcile_task.await.expect("reconciliation completes"); + assert!(removal_task.await.expect("removal completes")); + assert_eq!(session.read().await.transient.finalized_manifest_lease_binding_count(), 0); +} diff --git a/backend/hls/src/manager/tests/configuration.rs b/backend/hls/src/manager/tests/configuration.rs new file mode 100644 index 000000000..2588ed510 --- /dev/null +++ b/backend/hls/src/manager/tests/configuration.rs @@ -0,0 +1,82 @@ +use super::{config_with_hls_cache, test_app_config, HlsProxyManager}; +use crate::HlsSessionKey; +use shared::model::{HlsCacheConfigDto, HlsManifestRecoveryBurstLevel, HlsStripConfigDto, HlsStripMode}; +use tuliprox_core::model::HlsCacheConfig; + +#[tokio::test] +async fn update_config_applies_hls_runtime_settings_to_existing_manager() { + let initial_dto = HlsCacheConfigDto { + cache_path: Some("/tmp/tuliprox/hls-a".to_string()), + max_segments_prefetch: 1, + ..Default::default() + }; + let initial_config = HlsCacheConfig::from(&initial_dto); + let manager = HlsProxyManager::with_hls_cache_config(&initial_config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 100).await; + + let mut updated_dto = initial_dto.clone(); + updated_dto.max_segments_prefetch = 4; + updated_dto.max_concurrent_segment_fetches_per_session = 5; + updated_dto.max_concurrent_segment_fetches_global = 6; + updated_dto.origin_manifest_timeout_ms = shared::model::Millis::new(1_234); + updated_dto.origin_segment_timeout_ms = shared::model::Millis::new(5_678); + updated_dto.cache_duration = shared::model::Secs::new(99); + updated_dto.session_idle_timeout = shared::model::Secs::new(55); + updated_dto.manifest_recovery_burst.level = shared::model::HlsManifestRecoveryBurstLevel::Balanced; + updated_dto.strip = HlsStripConfigDto { mode: HlsStripMode::Seconds, value: 7 }; + let app_config = test_app_config(config_with_hls_cache(updated_dto)); + + manager.update_config(&app_config).await; + + assert_eq!(manager.segment_fetch_policy().max_prefetch_queue_depth, 4); + assert_eq!(manager.segment_fetch_policy().max_session_segment_fetches, 5); + assert_eq!(manager.segment_fetch_policy().max_global_segment_fetches, 6); + assert_eq!(manager.segment_fetch_policy().origin_segment_timeout_ms, 5_678); + assert_eq!(manager.origin_manifest_timeout_ms(), 1_234); + assert_eq!(manager.cache_duration_seconds(), 99); + assert_eq!(manager.transient_resource_ttl_ms(), 99_000); + assert_eq!(manager.session_idle_timeout_ms(), 55_000); + assert_eq!(manager.manifest_recovery_burst().level, HlsManifestRecoveryBurstLevel::Balanced); + assert_eq!(manager.strip().mode, HlsStripMode::Seconds); + assert_eq!(manager.strip().value, 7); + assert_eq!(session.read().await.segment_prefetch_queue.max_prefetch_depth(), 4); +} + +#[tokio::test] +async fn optional_hls_config_controls_runtime_enabled_state() { + let manager = HlsProxyManager::from_hls_cache_config(None); + + assert!(!manager.is_enabled()); + assert_eq!( + manager.run_garbage_collection_once(1_000).await.expect("disabled gc should no-op"), + super::super::super::GarbageCollectionReport::default() + ); + + let app_config = test_app_config(config_with_hls_cache(HlsCacheConfigDto::default())); + manager.update_config(&app_config).await; + + assert!(manager.is_enabled()); +} + +#[tokio::test] +async fn update_config_cache_path_change_clears_runtime_cache_state() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let old_cache = temp_dir.path().join("old"); + let new_cache = temp_dir.path().join("new"); + let initial_dto = + HlsCacheConfigDto { cache_path: Some(old_cache.to_string_lossy().to_string()), ..Default::default() }; + let initial_config = HlsCacheConfig::from(&initial_dto); + let manager = HlsProxyManager::with_hls_cache_config(&initial_config); + let _ = manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 100).await; + assert_eq!(manager.sessions().len().await, 1); + + let mut updated_dto = initial_dto; + updated_dto.cache_path = Some(new_cache.to_string_lossy().to_string()); + let app_config = test_app_config(config_with_hls_cache(updated_dto)); + + manager.update_config(&app_config).await; + + assert!(manager.sessions().is_empty().await); + assert_eq!(manager.segment_cache().cache_path(), new_cache); +} diff --git a/backend/hls/src/manager/tests/lifecycle.rs b/backend/hls/src/manager/tests/lifecycle.rs new file mode 100644 index 000000000..045337f4f --- /dev/null +++ b/backend/hls/src/manager/tests/lifecycle.rs @@ -0,0 +1,228 @@ +use super::{ + access_lease, hls_key_readiness_evidence_is_current, live_media_fixture, manifest_snapshot, terminal_plan, + HlsAccessLeaseDenialMode, HlsAccessLeaseDenialOutcome, HlsLeaseManifestPublicationOutcome, + HlsLeaseManifestPublicationRejectReason, HlsMediaActivityCommitOutcome, HlsProxyManager, HlsTerminalTailProtection, +}; +use crate::{ + terminal_tail::HlsLeasePlaybackMode, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, + HlsPublishedTransientResourceIds, HlsSessionKey, ProxySessionId, +}; +use shared::model::HlsCacheConfigDto; +use std::sync::Arc; +use tuliprox_core::model::HlsCacheConfig; + +#[test] +fn key_readiness_evidence_is_valid_at_expiry_and_stale_one_millisecond_later() { + assert!(hls_key_readiness_evidence_is_current(Some(50_000), 50_000)); + assert!(!hls_key_readiness_evidence_is_current(Some(50_000), 50_001)); + assert!(hls_key_readiness_evidence_is_current(None, u64::MAX)); +} + +#[tokio::test] +async fn expired_and_denied_lease_identities_cannot_extend_shared_session_activity() { + let (expired_manager, expired_session, expired_proxy, expired_lease_id, expired_identity) = + live_media_fixture("expired-media").await; + { + let mut leases = expired_manager.access_leases.write().await; + let mut lease = leases.remove_access_lease(&expired_lease_id).expect("expiring lease"); + lease.pending_deadline = Some(HlsAccessLeasePendingDeadline::Bootstrap { deadline_ms: 2_000 }); + lease.valid_until_ms = 2_000; + leases.prepare_access_lease(lease); + } + assert_eq!( + expired_manager + .mark_authorized_media_access_for_lease_if_identity_matches( + &expired_session, + &expired_lease_id, + &expired_proxy, + expired_identity, + 2_000, + ) + .await, + HlsMediaActivityCommitOutcome::StaleLeaseIdentity + ); + assert_eq!(expired_session.read().await.activity.last_authorized_media_at_ms, None); + + let (denied_manager, denied_session, denied_proxy, denied_lease_id, denied_identity) = + live_media_fixture("denied-media").await; + assert!(matches!( + denied_manager + .access_leases + .write() + .await + .deny_access_lease(&denied_lease_id, HlsAccessLeaseDenialMode::ImmediateEnd), + HlsAccessLeaseDenialOutcome::Ended { terminal_release: None } + )); + assert_eq!( + denied_manager + .mark_authorized_media_access_for_lease_if_identity_matches( + &denied_session, + &denied_lease_id, + &denied_proxy, + denied_identity, + 2_000, + ) + .await, + HlsMediaActivityCommitOutcome::StaleLeaseIdentity + ); + assert_eq!(denied_session.read().await.activity.last_authorized_media_at_ms, None); +} + +#[tokio::test] +async fn manifest_publication_manager_rejects_delayed_source_and_expiry_races() { + let manager = HlsProxyManager::new(); + let proxy_session_id = ProxySessionId("proxy".to_string()); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); + let older_request = manager + .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, 2_000) + .await + .expect("older request guard"); + let newer_request = manager + .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, 2_000) + .await + .expect("newer request guard"); + + assert_eq!( + manager + .commit_access_lease_manifest_publication( + &lease_id, + &proxy_session_id, + newer_request, + manifest_snapshot(20), + 2_100, + ) + .await, + HlsLeaseManifestPublicationOutcome::Committed { snapshot_generation: 1 } + ); + assert_eq!( + manager + .commit_access_lease_manifest_publication( + &lease_id, + &proxy_session_id, + older_request, + manifest_snapshot(10), + 2_200, + ) + .await, + HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::SourceRegressive) + ); + + let expiry_lease_id = HlsAccessLeaseId("expiry".to_string()); + manager.access_leases.write().await.prepare_access_lease(access_lease(&expiry_lease_id.0, &proxy_session_id)); + let expiry_request = manager + .prepare_access_lease_manifest_publication(&expiry_lease_id, &proxy_session_id, 2_000) + .await + .expect("pre-expiry request guard"); + assert_eq!( + manager + .commit_access_lease_manifest_publication( + &expiry_lease_id, + &proxy_session_id, + expiry_request, + manifest_snapshot(30), + 61_000, + ) + .await, + HlsLeaseManifestPublicationOutcome::Rejected(HlsLeaseManifestPublicationRejectReason::LeaseExpired) + ); +} + +#[tokio::test] +async fn rolling_manifest_publication_uses_lease_store_when_session_is_gc_marked() { + let manager = HlsProxyManager::new(); + let session = manager.get_or_create_session(HlsSessionKey::new(1, "rolling"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); + let publication = manager + .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, 2_000) + .await + .expect("rolling publication guard"); + let published_resource_ids = HlsPublishedTransientResourceIds::from_manifest_body( + "#EXTM3U\n#EXTINF:6,\n/hls/shared/live/proxy/lease-a/r/resource.ts\n", + ); + session.write().await.mark_for_gc_removal(); + + assert_eq!( + manager + .commit_access_lease_manifest_publication_with_resources( + &lease_id, + &proxy_session_id, + publication, + manifest_snapshot(20), + published_resource_ids.clone(), + 2_100, + ) + .await, + HlsLeaseManifestPublicationOutcome::Committed { snapshot_generation: 1 } + ); + let lease = manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_100) + .await + .expect("published rolling lease"); + assert_eq!(lease.published_transient_resource_ids(), &published_resource_ids); +} + +#[tokio::test] +async fn stale_lease_removal_preparation_cannot_release_replacement_protection() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("replacement-protection".to_string()); + let old_plan = terminal_plan(7, &proxy_session_id, &lease_id); + let mut old_lease = access_lease(&lease_id.0, &proxy_session_id); + old_lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(Arc::clone(&old_plan)); + manager.access_leases.write().await.prepare_access_lease(old_lease); + assert_eq!( + session.write().await.install_terminal_tail_protection( + lease_id.clone(), + HlsTerminalTailProtection { + generation: old_plan.generation, + base_proxy_seqs: Arc::clone(&old_plan.protected_base_proxy_seqs), + key_bindings: old_plan.key_bindings(), + }, + ), + super::super::super::session::HlsTerminalTailProtectionInstall::Installed + ); + let preparation = manager + .access_leases + .read() + .await + .prepare_access_lease_removal(&lease_id) + .expect("old lease removal preparation"); + + let replacement_plan = terminal_plan(8, &proxy_session_id, &lease_id); + let mut replacement = access_lease(&lease_id.0, &proxy_session_id); + replacement.issued_at_ms = 2_000; + replacement.playback_mode = HlsLeasePlaybackMode::TerminalTail(Arc::clone(&replacement_plan)); + assert!(manager.access_leases.write().await.prepare_access_lease(replacement)); + assert_eq!( + session.write().await.install_terminal_tail_protection( + lease_id.clone(), + HlsTerminalTailProtection { + generation: replacement_plan.generation, + base_proxy_seqs: Arc::clone(&replacement_plan.protected_base_proxy_seqs), + key_bindings: replacement_plan.key_bindings(), + }, + ), + super::super::super::session::HlsTerminalTailProtectionInstall::Installed + ); + + assert!(!manager.remove_prepared_access_lease(&lease_id, &preparation).await); + assert_eq!( + manager + .access_leases + .write() + .await + .response_snapshot(&lease_id, &proxy_session_id, 2_000) + .map(|lease| lease.issued_at_ms), + Some(2_000) + ); + assert_eq!( + session.read().await.terminal_tail_protection(&lease_id).map(|protection| protection.generation), + Some(replacement_plan.generation) + ); +} diff --git a/backend/hls/src/manager/tests/persistence.rs b/backend/hls/src/manager/tests/persistence.rs new file mode 100644 index 000000000..a4ada0df6 --- /dev/null +++ b/backend/hls/src/manager/tests/persistence.rs @@ -0,0 +1,90 @@ +use super::{access_lease, finalized_transient_manifest_snapshot, HlsLeaseManifestPublicationOutcome, HlsProxyManager}; +use crate::{ + HlsAccessLeaseId, HlsPublishedTransientResourceIds, HlsSessionKey, TransientResourceKind, TransientResourceRef, +}; +use std::sync::Arc; + +#[tokio::test] +async fn finalized_manifest_publication_blocks_session_replacement_until_binding_is_committed() { + let manager = Arc::new(HlsProxyManager::new()); + let session_key = HlsSessionKey::new(1, "archive-publication-race"); + let session = manager.get_or_create_session(session_key.clone(), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://cdn.example/archive/segment.ts", + b"secret", + 1_000, + 1, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + let resource_uri = format!("/hls/shared/live/{}/lease-a/r/{}.ts", proxy_session_id.0, resource_id.0); + let manifest_body = format!("#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXTINF:6,\n{resource_uri}\n#EXT-X-ENDLIST\n"); + let generation = { + let mut session = session.write().await; + session.transient.upsert_resources([resource]); + session.transient.replace_manifest_with_semantics(manifest_body.clone(), 1_000, Some(6_000)); + session.transient.current_finalized_manifest_generation().expect("finalized generation") + }; + manager.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)).await; + let publication = manager + .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, 2_000) + .await + .expect("publication guard"); + let published_resource_ids = HlsPublishedTransientResourceIds::from_manifest_body(&manifest_body); + let lease_store_guard = manager.hold_access_lease_store_for_test().await; + let publication_task = { + let manager = Arc::clone(&manager); + let lease_id = lease_id.clone(); + let proxy_session_id = proxy_session_id.clone(); + let published_resource_ids = published_resource_ids.clone(); + tokio::spawn(async move { + manager + .commit_access_lease_manifest_publication_with_resources( + &lease_id, + &proxy_session_id, + publication, + finalized_transient_manifest_snapshot(1_000, generation, &resource_uri), + published_resource_ids, + 2_000, + ) + .await + }) + }; + + let mut index_guard_observed = false; + for _ in 0..1_000 { + if manager.sessions.index_write_is_blocked_for_test() { + index_guard_observed = true; + break; + } + tokio::task::yield_now().await; + } + assert!(index_guard_observed, "publication did not acquire the session-index guard"); + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(25), + manager.sessions.remove_session(&session_key, &proxy_session_id), + ) + .await + .is_err(), + "session replacement completed before finalized publication" + ); + + drop(lease_store_guard); + assert_eq!( + publication_task.await.expect("publication task"), + HlsLeaseManifestPublicationOutcome::Committed { snapshot_generation: 1 } + ); + let current_session = + manager.sessions.get_by_proxy_session_id(&proxy_session_id).await.expect("current session remains indexed"); + assert!(Arc::ptr_eq(¤t_session, &session)); + assert!(current_session + .read() + .await + .transient + .resolve_resource_for_lease(&resource_id, &lease_id, 1_000, &published_resource_ids, 2_000) + .is_some()); +} diff --git a/backend/hls/src/manager/tests/retry.rs b/backend/hls/src/manager/tests/retry.rs new file mode 100644 index 000000000..352c97584 --- /dev/null +++ b/backend/hls/src/manager/tests/retry.rs @@ -0,0 +1,11 @@ +use super::{hls_session_idle_protection_retry_at, HLS_SESSION_IDLE_PROTECTION_RETRY_MS}; + +#[test] +fn protected_idle_session_retry_cannot_form_a_millisecond_busy_loop() { + let now_ms = 50_000_u64; + assert_eq!( + hls_session_idle_protection_retry_at(now_ms.saturating_sub(1), now_ms), + now_ms.saturating_add(HLS_SESSION_IDLE_PROTECTION_RETRY_MS) + ); + assert_eq!(hls_session_idle_protection_retry_at(60_000, now_ms), 60_000); +} diff --git a/backend/hls/src/manager/tests/startup.rs b/backend/hls/src/manager/tests/startup.rs new file mode 100644 index 000000000..1f8d14072 --- /dev/null +++ b/backend/hls/src/manager/tests/startup.rs @@ -0,0 +1,97 @@ +use super::{ + commit_terminal_plan, config_with_hls_cache, prepared_terminal_commit_fixture, test_app_config, HlsProxyManager, + HlsTerminalAssetRevisionGuard, HlsTerminalCommitMediaGuard, HlsTerminalCommitOutcome, HlsTerminalCommitPayload, + HlsTerminalCommitRequest, +}; +use crate::{terminal_tail::HlsLeasePlaybackMode, HlsSessionKey, HlsTerminalTailCompatibility}; +use shared::model::HlsCacheConfigDto; +use std::sync::Arc; +use tuliprox_core::model::HlsCacheConfig; + +#[tokio::test] +async fn startup_reload_keeps_existing_session_policy_and_global_budget_ownership() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let mut initial_dto = + HlsCacheConfigDto { cache_path: Some(directory.path().to_string_lossy().into_owned()), ..Default::default() }; + initial_dto.startup.mode = shared::model::HlsStartupMode::Progressive; + let manager = HlsProxyManager::with_hls_cache_config(&HlsCacheConfig::from(&initial_dto)); + let (original, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "original"), b"secret", 100).await; + let budget = Arc::clone(&manager.progressive_budget); + let held = budget.try_claim().ok_or_else(|| std::io::Error::other("progressive slot"))?; + let mut updated = initial_dto; + updated.startup.mode = shared::model::HlsStartupMode::FirstReady; + updated.startup.max_progressive_segments = 1; + manager.update_config(&test_app_config(config_with_hls_cache(updated))).await; + assert!(Arc::ptr_eq(&budget, &manager.progressive_budget)); + assert!(budget.try_claim().is_none()); + let (existing, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "original"), b"secret", 200).await; + assert!(Arc::ptr_eq(&existing, &original)); + assert_eq!(existing.read().await.startup_mode(), shared::model::HlsStartupMode::Progressive); + let (new_session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "new"), b"secret", 200).await; + assert_eq!(new_session.read().await.startup_mode(), shared::model::HlsStartupMode::FirstReady); + drop(held); + assert_eq!(budget.usage(), (0, 0)); + Ok(()) +} + +#[tokio::test] +async fn hls_terminal_commit_changed_asset_revision_fails_closed_before_tail_mutation() { + let (manager, session, proxy_session_id, lease_id, preparation, plan) = + prepared_terminal_commit_fixture("asset-revision").await; + let expected_asset = plan.asset_identity; + + let outcome = manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session: &session, + lease_id: &lease_id, + proxy_session_id: &proxy_session_id, + preparation: &preparation, + now_ms: 2_000, + payload: HlsTerminalCommitPayload::Tail { plan, media_guard: HlsTerminalCommitMediaGuard::empty_for_test() }, + asset_revision_guard: HlsTerminalAssetRevisionGuard::for_runtime_tail(expected_asset, || None), + }); + + assert_eq!(outcome, HlsTerminalCommitOutcome::Committed); + assert!(matches!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("asset mismatch terminalizes the lease") + .playback_mode, + HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::MissingAsset, .. } + )); + assert!(!session.read().await.has_terminal_tail_protections()); +} + +#[tokio::test] +async fn hls_terminal_commit_exact_replay_stays_idempotent_after_asset_revision_change() { + let (manager, session, proxy_session_id, lease_id, preparation, plan) = + prepared_terminal_commit_fixture("asset-replay").await; + let expected_asset = plan.asset_identity; + + assert_eq!( + commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, Arc::clone(&plan),), + HlsTerminalCommitOutcome::Committed + ); + let replay = manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session: &session, + lease_id: &lease_id, + proxy_session_id: &proxy_session_id, + preparation: &preparation, + now_ms: 2_000, + payload: HlsTerminalCommitPayload::Tail { plan, media_guard: HlsTerminalCommitMediaGuard::empty_for_test() }, + asset_revision_guard: HlsTerminalAssetRevisionGuard::for_runtime_tail(expected_asset, || None), + }); + + assert_eq!(replay, HlsTerminalCommitOutcome::AlreadyCommitted); + assert!(matches!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("terminal replay stays available") + .playback_mode, + HlsLeasePlaybackMode::TerminalTail(_) + )); +} diff --git a/backend/hls/src/manager/tests/support.rs b/backend/hls/src/manager/tests/support.rs new file mode 100644 index 000000000..2b082da8a --- /dev/null +++ b/backend/hls/src/manager/tests/support.rs @@ -0,0 +1,366 @@ +use super::{ + prepared_terminal_bundle_key, snapshot_terminal_media_asset, HlsAcceptanceEpisodeTiming, + HlsAcceptanceEpisodeTimingInput, HlsLeaseCutoverTiming, HlsManifestAcceptanceGeneration, + HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsProxyManager, HlsRecoveryEtaMs, HlsRecoveryTimingPolicy, + HlsRecoveryWorkload, HlsRuntimeCustomTailAssetIdentity, HlsTerminalAssetRevisionGuard, + HlsTerminalCommitAcquisitionBudgetMs, HlsTerminalCommitMediaGuard, HlsTerminalCommitOutcome, + HlsTerminalCommitPayload, HlsTerminalCommitRequest, HlsTerminalCommitWindow, HlsTerminalMediaPreparationState, + HlsTerminalTailPlan, HlsTerminalTailPreparationRequest, HlsTransitionMarginMs, +}; +use crate::{ + build_terminal_tail_plan, + media_reserve::{HlsLeaseReserveAvailabilityBasis, HlsLeaseReserveSnapshot, HlsManifestCommitIdentity}, + HlsAccessLease, HlsAccessLeaseId, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, + HlsManifestAcceptanceExhaustionReason, HlsManifestAcceptanceTrigger, HlsManifestDeliveryMode, HlsMediaContainer, + HlsMediaLeaseIdentity, HlsPlaybackFamilyKey, HlsSessionHandle, HlsSessionKey, HlsTerminalAssetIdentity, + HlsTerminalBaseMediaState, HlsTerminalBaseProtection, HlsTerminalBaseSegmentAvailability, + HlsTerminalTailBuildInput, HlsTerminalTailGeneration, ProxySessionId, TransientManifestGeneration, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, +}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use shared::model::{ConfigPaths, HlsCacheConfigDto, HlsManifestRecoveryBurstLevel, ReverseProxyConfigDto}; +use std::sync::Arc; +use tuliprox_core::{ + model::{AppConfig, Config, HlsCacheConfig, MediaToolCapabilities, ReverseProxyConfig, SourcesConfig}, + utils::FileLockManager, +}; +use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer; + +pub(in crate::manager::tests) const TERMINAL_ASSET_BYTES: &[u8] = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); + +pub(in crate::manager::tests) fn cutover_reserve() -> HlsLeaseReserveSnapshot { + let transition_margin = HlsTransitionMarginMs::from_millis(12_000); + let guaranteed_reserve_ms = transition_margin + .as_millis() + .saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()); + HlsLeaseReserveSnapshot { + availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, + guaranteed_media_horizon_ms: 12_000_u64.saturating_add(guaranteed_reserve_ms), + conservative_playback_position_ms: 12_000, + guaranteed_reserve_ms, + initial_hidden_ready_duration_ms: 0, + transition_margin, + key_readiness_valid_until_ms: None, + recovery_required: true, + cutover_required: false, + } +} + +pub(in crate::manager::tests) fn empty_paths() -> ConfigPaths { + ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + } +} + +pub(in crate::manager::tests) fn test_app_config(config: Config) -> AppConfig { + AppConfig { + config: Arc::new(ArcSwap::from_pointee(config)), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(empty_paths())), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [7; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + } +} + +pub(in crate::manager::tests) fn config_with_hls_cache(hls_cache: HlsCacheConfigDto) -> Config { + Config { + reverse_proxy: Some(ReverseProxyConfig::from(&ReverseProxyConfigDto { + hls_cache: Some(hls_cache), + ..ReverseProxyConfigDto::default() + })), + ..Config::default() + } +} + +pub(in crate::manager::tests) fn access_lease(lease_id: &str, proxy_session_id: &ProxySessionId) -> HlsAccessLease { + HlsAccessLease::pending( + HlsAccessLeaseId(lease_id.to_string()), + HlsPlaybackFamilyKey::new("alice", "client-a"), + proxy_session_id.clone(), + "alice".to_string(), + "session-a".to_string(), + 1, + "stream-a".to_string(), + 12345, + 1_000, + 60_000, + ) +} + +pub(in crate::manager::tests) fn manifest_snapshot(source_rendered_at_ms: u64) -> HlsLeaseManifestSnapshot { + HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(source_rendered_at_ms), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 0, + delivered_at_ms: 2_000, + first_proxy_seq: 40, + last_proxy_seq: 41, + visible_segments: Arc::from([ + HlsLeaseManifestSegment { + proxy_seq: 40, + duration_ms: 6_000, + uri: "/live/40.ts".to_string().into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }, + HlsLeaseManifestSegment { + proxy_seq: 41, + duration_ms: 6_000, + uri: "/live/41.ts".to_string().into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }, + ]), + discontinuity_sequence: 3, + target_duration_ms: 12_000, + playlist_duration_ms: 12_000, + last_visible_media_end_ms: 12_000, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + } +} + +pub(in crate::manager::tests) fn finalized_transient_manifest_snapshot( + source_rendered_at_ms: u64, + manifest_generation: TransientManifestGeneration, + resource_uri: &str, +) -> HlsLeaseManifestSnapshot { + let mut snapshot = manifest_snapshot(source_rendered_at_ms); + snapshot.delivery_mode = HlsManifestDeliveryMode::TransientPassthrough; + snapshot.finalized_transient_manifest_generation = Some(manifest_generation); + for segment in Arc::make_mut(&mut snapshot.visible_segments) { + segment.uri = Arc::from(resource_uri); + } + snapshot +} + +pub(in crate::manager::tests) async fn publish_manifest_snapshot( + manager: &HlsProxyManager, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + snapshot: HlsLeaseManifestSnapshot, + now_ms: u64, +) -> bool { + let Some(guard) = manager.prepare_access_lease_manifest_publication(lease_id, proxy_session_id, now_ms).await + else { + return false; + }; + manager + .commit_access_lease_manifest_publication(lease_id, proxy_session_id, guard, snapshot, now_ms) + .await + .is_committed() +} + +pub(in crate::manager::tests) fn terminal_plan( + generation: u64, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, +) -> Arc { + let mut base_manifest = manifest_snapshot(1); + base_manifest.snapshot_generation = 1; + for segment in Arc::make_mut(&mut base_manifest.visible_segments) { + segment.uri = format!("/hls/shared/live/{}/{}/{}.ts", proxy_session_id.0, lease_id.0, segment.proxy_seq).into(); + } + let transport_stream = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); + let asset = + super::super::super::terminal_tail::snapshot_terminal_media_asset(&transport_stream).expect("terminal asset"); + let expected_asset = + HlsRuntimeCustomTailAssetIdentity::channel_unavailable(HlsTerminalAssetIdentity::from_asset(&asset)); + let availability = Arc::from( + base_manifest + .visible_proxy_seqs() + .map(|proxy_seq| HlsTerminalBaseSegmentAvailability { + proxy_seq, + media_state: HlsTerminalBaseMediaState::Ready, + required_map_ready: true, + required_key_ready: true, + protection: HlsTerminalBaseProtection::Protectable, + }) + .collect::>(), + ); + let base_track_signature = Some(asset.track_signature().clone()); + let anchored_bundle = HlsTerminalTailBuildInput::anchored_bundle_for_test(&asset, base_manifest.target_duration_ms); + let base_timing = Some(HlsTerminalTailBuildInput::base_timing_for_test(&asset, &base_manifest)); + let base_splice_evidence = Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); + let terminal_splice_evidence = base_splice_evidence.clone(); + Arc::new( + build_terminal_tail_plan(HlsTerminalTailBuildInput { + generation: HlsTerminalTailGeneration(generation), + created_at_ms: 3_000, + base_manifest, + base_availability: availability, + base_track_signature, + base_splice_evidence, + terminal_splice_evidence, + base_timing, + base_key_bindings: Arc::from([]), + expected_asset, + asset, + anchored_bundle, + }) + .expect("compatible terminal plan"), + ) +} + +pub(in crate::manager::tests) fn commit_terminal_plan( + manager: &HlsProxyManager, + session: &HlsSessionHandle, + lease_id: &HlsAccessLeaseId, + proxy_session_id: &ProxySessionId, + preparation: &super::super::super::lease::HlsTerminalTailPreparation, + now_ms: u64, + plan: Arc, +) -> HlsTerminalCommitOutcome { + let asset_revision_guard = HlsTerminalAssetRevisionGuard::matching_runtime_for_test(plan.asset_identity); + manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session, + lease_id, + proxy_session_id, + preparation, + now_ms, + payload: HlsTerminalCommitPayload::Tail { plan, media_guard: HlsTerminalCommitMediaGuard::empty_for_test() }, + asset_revision_guard, + }) +} + +pub(in crate::manager::tests) async fn live_media_fixture( + lease_name: &str, +) -> (HlsProxyManager, HlsSessionHandle, ProxySessionId, HlsAccessLeaseId, HlsMediaLeaseIdentity) { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId(lease_name.to_string()); + let lease = access_lease(lease_name, &proxy_session_id); + let lease_identity = lease.media_identity().expect("live media identity"); + manager.access_leases.write().await.prepare_access_lease(lease); + (manager, session, proxy_session_id, lease_id, lease_identity) +} + +pub(in crate::manager::tests) fn begin_test_acceptance_episode( + session: &mut super::super::super::HlsSession, + now_ms: u64, +) -> HlsManifestAcceptanceGeneration { + let burst_plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let terminal_buffer = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); + let terminal_asset = snapshot_terminal_media_asset(&terminal_buffer).expect("terminal asset"); + let terminal_key = prepared_terminal_bundle_key(&terminal_asset, 12_000, HLS_TERMINAL_TAIL_SEGMENT_COUNT); + let required_terminal_media_key = Some(terminal_key); + let terminal_media_preparation = HlsTerminalMediaPreparationState::Preparing { key: terminal_key }; + let timing = HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput { + started_at_ms: now_ms, + burst_plan, + target_duration_ms: 12_000, + transition_margin: HlsTransitionMarginMs::from_millis(12_000), + workload: HlsRecoveryWorkload::clear_fetch(), + observed_latency: HlsObservedRecoveryLatency::default(), + required_terminal_media_key, + terminal_media_preparation, + policy: HlsRecoveryTimingPolicy::new( + HlsOperationTimeoutMs::from_millis(1_000), + HlsOperationTimeoutMs::from_millis(2_000), + HlsRecoveryEtaMs::from_millis(300), + HlsRecoveryEtaMs::from_millis(400), + ), + }); + session.origin_control.begin_acceptance_episode( + now_ms, + burst_plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &timing, + ) +} + +pub(in crate::manager::tests) fn complete_failed_acceptance_episode( + session: &mut super::super::super::HlsSession, + now_ms: u64, +) -> u64 { + let generation = begin_test_acceptance_episode(session, now_ms); + let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); + assert_eq!(episode.generation, generation); + episode.record_full_burst(); + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::AllFailed); + episode.hold_after_uncommitted_burst(None, Some(now_ms.saturating_add(1_000))); + session.origin_control.progress_generation +} + +pub(in crate::manager::tests) async fn prepared_terminal_commit_fixture( + lease_name: &str, +) -> ( + HlsProxyManager, + HlsSessionHandle, + ProxySessionId, + HlsAccessLeaseId, + super::super::super::lease::HlsTerminalTailPreparation, + Arc, +) { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + manager.terminal_commit_clock.set_fixed_now_ms(2_000); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, lease_name), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId(lease_name.to_string()); + manager.access_leases.write().await.prepare_access_lease(access_lease(lease_name, &proxy_session_id)); + assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); + let progress_generation = { + let mut session = session.write().await; + complete_failed_acceptance_episode(&mut session, 2_000) + }; + let preparation = manager + .prepare_access_lease_terminal_tail(terminal_preparation_request( + &lease_id, + &proxy_session_id, + progress_generation, + )) + .await + .expect("terminal preparation"); + let plan = terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id); + (manager, session, proxy_session_id, lease_id, preparation, plan) +} + +pub(in crate::manager::tests) fn terminal_preparation_request<'a>( + lease_id: &'a HlsAccessLeaseId, + proxy_session_id: &'a ProxySessionId, + origin_progress_generation: u64, +) -> HlsTerminalTailPreparationRequest<'a> { + let reserve = cutover_reserve(); + let cutover_timing = + HlsLeaseCutoverTiming::from_reserve(2_000, reserve.guaranteed_reserve_ms, reserve.transition_margin, None); + HlsTerminalTailPreparationRequest { + lease_id, + proxy_session_id, + manifest_snapshot_generation: 1, + cursor_generation: 0, + reserve, + cutover_timing, + commit_window: HlsTerminalCommitWindow::AcquisitionOpen, + now_ms: 2_000, + origin_progress_generation, + media_readiness_generation: 0, + last_media_progress_at_ms: None, + } +} diff --git a/backend/hls/src/manager/tests/terminal.rs b/backend/hls/src/manager/tests/terminal.rs new file mode 100644 index 000000000..bb3f74d56 --- /dev/null +++ b/backend/hls/src/manager/tests/terminal.rs @@ -0,0 +1,1255 @@ +use super::{ + access_lease, begin_test_acceptance_episode, commit_terminal_plan, complete_failed_acceptance_episode, + cutover_reserve, hls_acceptance_recovery_snapshot, live_media_fixture, manifest_snapshot, + next_terminal_commit_retry, prepared_terminal_commit_fixture, publish_manifest_snapshot, + spawn_terminal_commit_retry_worker, terminal_plan, terminal_preparation_request, HlsAccessLeaseDenialMode, + HlsAccessLeaseDenialOutcome, HlsLeaseCutoverTiming, HlsMediaActivityCommitOutcome, HlsProxyManager, + HlsRecoveryExecutionState, HlsRuntimeCustomTailAssetIdentity, HlsTerminalAssetRevisionGuard, + HlsTerminalCommitAcquisitionBudgetMs, HlsTerminalCommitCommand, HlsTerminalCommitMediaGuard, + HlsTerminalCommitOutcome, HlsTerminalCommitOwnerKey, HlsTerminalCommitPayload, HlsTerminalCommitRequest, + HlsTerminalCommitRetryDecision, HlsTerminalCommitRetryScheduleDecision, HlsTerminalCommitSubmissionDecision, + HlsTerminalCommitWindow, HlsTerminalLeaseDecision, HlsTerminalMediaRequirementSource, + HlsTerminalTailPreparationRequest, HlsTerminalTailProtection, HLS_TERMINAL_TAIL_PROTECTION_CAPACITY, +}; +use crate::{ + media_reserve::HlsLeaseReserveSnapshot, terminal_tail::HlsLeasePlaybackMode, HlsAccessLeaseId, HlsAccessLeaseState, + HlsManifestAcceptanceExhaustionReason, HlsSegmentFailureObject, HlsSessionKey, HlsSessionStoreOutcome, + HlsTerminalAssetIdentity, HlsTerminalTailCompatibility, HlsTerminalTailGeneration, ProxySessionId, +}; +use shared::model::HlsCacheConfigDto; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use tuliprox_core::model::HlsCacheConfig; + +#[tokio::test] +async fn late_live_completion_after_terminal_transition_updates_neither_cursor_nor_session_activity() { + let (manager, session, proxy_session_id, lease_id, live_identity) = live_media_fixture("late-terminal").await; + let token = manager + .record_access_lease_segment_request_started_if_identity_matches( + &lease_id, + &proxy_session_id, + live_identity, + 40, + 2_000, + ) + .await + .expect("current live request token"); + let plan = terminal_plan(7, &proxy_session_id, &lease_id); + { + let mut leases = manager.access_leases.write().await; + let mut lease = leases.remove_access_lease(&lease_id).expect("live lease"); + lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(plan); + leases.prepare_access_lease(lease); + } + + let outcome = manager + .record_access_lease_segment_request_completed_and_mark_media_if_identity_matches( + &session, + &lease_id, + &proxy_session_id, + live_identity, + token, + 2_100, + ) + .await; + + assert_eq!(outcome, HlsMediaActivityCommitOutcome::StaleLeaseIdentity); + assert_eq!(session.read().await.activity.last_authorized_media_at_ms, None); + let lease = + manager.access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_100).await.expect("terminal lease"); + assert_eq!(lease.playback_cursor.highest_contiguous_completed_proxy_seq, None); + assert_eq!(lease.playback_cursor.first_segment_completed_at_ms, None); +} + +#[tokio::test] +async fn current_terminal_identity_waits_for_lock_contention_and_marks_activity() { + let (manager, session, proxy_session_id, lease_id, _) = live_media_fixture("terminal-media").await; + let manager = Arc::new(manager); + let terminal_identity = { + let mut leases = manager.access_leases.write().await; + let mut lease = leases.remove_access_lease(&lease_id).expect("live lease"); + lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(terminal_plan(7, &proxy_session_id, &lease_id)); + let identity = lease.media_identity().expect("terminal identity"); + leases.prepare_access_lease(lease); + identity + }; + assert_eq!( + manager + .mark_authorized_media_access_for_lease_if_identity_matches( + &session, + &lease_id, + &proxy_session_id, + terminal_identity, + 2_000, + ) + .await, + HlsMediaActivityCommitOutcome::Committed + ); + assert_eq!(session.read().await.activity.last_authorized_media_at_ms, Some(2_000)); + + session.write().await.activity.last_authorized_media_at_ms = None; + let lease_guard = manager.access_leases.write().await; + let task_manager = Arc::clone(&manager); + let task_session = Arc::clone(&session); + let task_lease_id = lease_id.clone(); + let task_proxy_session_id = proxy_session_id.clone(); + let commit_task = tokio::spawn(async move { + task_manager + .mark_authorized_media_access_for_lease_if_identity_matches( + &task_session, + &task_lease_id, + &task_proxy_session_id, + terminal_identity, + 2_100, + ) + .await + }); + tokio::task::yield_now().await; + assert!(!commit_task.is_finished(), "current media activity must wait rather than be dropped"); + drop(lease_guard); + assert_eq!(commit_task.await.expect("controlled commit task"), HlsMediaActivityCommitOutcome::Committed); + assert_eq!(session.read().await.activity.last_authorized_media_at_ms, Some(2_100)); +} + +#[tokio::test] +async fn hls_terminal_commit_missing_acceptance_exact_ready_bundle_commits_tail() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + manager.terminal_commit_clock.set_fixed_now_ms(2_000); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "missing-acceptance"), b"secret", 1_000).await; + let (proxy_session_id, progress_generation) = { + let session = session.read().await; + assert!(session.origin_control.acceptance_episode.is_none()); + (session.proxy_session_id.clone(), session.origin_control.progress_generation) + }; + let lease_id = HlsAccessLeaseId("missing-acceptance".to_string()); + manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); + assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); + let mut preparation = manager + .prepare_access_lease_terminal_tail(terminal_preparation_request( + &lease_id, + &proxy_session_id, + progress_generation, + )) + .await + .expect("missing acceptance still permits cutover-local terminal preparation"); + assert_eq!( + preparation.terminal_media_requirement_source, + HlsTerminalMediaRequirementSource::CutoverSnapshotPending { + decision_generation: preparation.decision_generation, + } + ); + let plan = terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id); + let prepared_key = plan.media_preparation_key(); + preparation + .bind_ready_terminal_media_requirement(prepared_key) + .expect("the exact ready bundle binds to the cutover decision"); + assert_eq!( + preparation.terminal_media_requirement_source, + HlsTerminalMediaRequirementSource::CutoverSnapshot { + decision_generation: preparation.decision_generation, + asset: prepared_key.asset, + } + ); + + let outcome = commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan); + + assert_eq!(outcome, HlsTerminalCommitOutcome::Committed); + let lease = manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("committed terminal lease"); + assert!(matches!(lease.playback_mode, HlsLeasePlaybackMode::TerminalTail(_))); +} + +pub(in crate::manager::tests) async fn wait_for_terminal_commit_owner(manager: &HlsProxyManager) { + for _ in 0..256 { + if manager.terminal_commit_retries.owner_count() == 0 { + return; + } + tokio::task::yield_now().await; + } + assert_eq!(manager.terminal_commit_retries.owner_count(), 0, "terminal retry owner did not finish"); +} + +#[test] +fn hls_cutover_policy_manager_uses_only_matching_acceptance_work_as_recovery_evidence() { + let mut session = super::super::super::HlsSession::new(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000); + let generation = begin_test_acceptance_episode(&mut session, 2_000); + session.origin_refresh.in_flight = false; + + assert!(matches!( + hls_acceptance_recovery_snapshot(&session, 2_100).recovery, + HlsRecoveryExecutionState::InFlight { .. } + )); + + let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); + episode.record_full_burst(); + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::AllFailed); + session.origin_refresh.in_flight = true; + + let snapshot = hls_acceptance_recovery_snapshot(&session, 2_100); + assert_eq!(snapshot.expected_generation, generation); + assert_eq!(snapshot.recovery, HlsRecoveryExecutionState::Idle); +} + +#[tokio::test] +async fn repeated_segment_failure_counter_does_not_terminalize_access_lease() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); + + { + let mut session = session.write().await; + for failure in 0_u64..100 { + let _ = session.record_temporary_segment_fetch_failure( + 2_000_u64.saturating_add(failure), + HlsSegmentFailureObject::Normal { proxy_seq: 40, origin_seq: 400 }, + 1, + ); + } + assert_eq!(session.segment_failure_tracker.consecutive_temporary_failures, 100); + } + + let lease = manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_100) + .await + .expect("failure tracking must not remove the lease"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + assert!(!session.read().await.has_terminal_tail_protections()); +} + +#[tokio::test] +async fn hls_cutover_policy_media_progress_supersedes_prepared_plan() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); + assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); + let progress_generation = { + let mut session = session.write().await; + complete_failed_acceptance_episode(&mut session, 2_000) + }; + let preparation = manager + .prepare_access_lease_terminal_tail(terminal_preparation_request( + &lease_id, + &proxy_session_id, + progress_generation, + )) + .await + .expect("failed acceptance permits terminal preparation"); + + { + let mut session = session.write().await; + session.origin_control.record_media_progress(2_100, 12_000); + assert_eq!(session.origin_control.progress_generation, progress_generation.saturating_add(1)); + } + assert_eq!( + commit_terminal_plan( + &manager, + &session, + &lease_id, + &proxy_session_id, + &preparation, + 2_100, + terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), + ), + HlsTerminalCommitOutcome::RecoveryCommitted + ); + + let lease = manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_200) + .await + .expect("lease remains available"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + assert!(!session.read().await.has_terminal_tail_protections()); +} + +#[tokio::test] +async fn hls_cutover_policy_matching_acceptance_commit_without_media_progress_terminalizes_lease() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); + assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); + let progress_generation = { + let mut session = session.write().await; + complete_failed_acceptance_episode(&mut session, 2_000) + }; + let preparation = manager + .prepare_access_lease_terminal_tail(terminal_preparation_request( + &lease_id, + &proxy_session_id, + progress_generation, + )) + .await + .expect("terminal preparation"); + + { + let mut session = session.write().await; + session.origin_control.acceptance_episode.as_mut().expect("acceptance episode").complete(); + assert_eq!(session.origin_control.progress_generation, preparation.origin_progress_generation); + } + + assert_eq!( + commit_terminal_plan( + &manager, + &session, + &lease_id, + &proxy_session_id, + &preparation, + 2_100, + terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), + ), + HlsTerminalCommitOutcome::Committed + ); + assert!(matches!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_100) + .await + .expect("terminal lease") + .playback_mode, + HlsLeasePlaybackMode::TerminalTail(_) + )); +} + +#[tokio::test] +async fn hls_cutover_policy_new_acceptance_episode_without_media_progress_does_not_supersede_cutover() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); + assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); + let progress_generation = { + let mut session = session.write().await; + complete_failed_acceptance_episode(&mut session, 2_000) + }; + let preparation = manager + .prepare_access_lease_terminal_tail(terminal_preparation_request( + &lease_id, + &proxy_session_id, + progress_generation, + )) + .await + .expect("terminal preparation"); + + { + let mut session = session.write().await; + let previous_acceptance_generation = session.origin_control.acceptance_generation; + let current_acceptance_generation = begin_test_acceptance_episode(&mut session, 2_050); + assert_eq!(session.origin_control.acceptance_generation.0, previous_acceptance_generation.0.saturating_add(1)); + assert_eq!(current_acceptance_generation, session.origin_control.acceptance_generation); + assert_eq!(session.origin_control.progress_generation, preparation.origin_progress_generation); + } + + assert_eq!( + commit_terminal_plan( + &manager, + &session, + &lease_id, + &proxy_session_id, + &preparation, + 2_100, + terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), + ), + HlsTerminalCommitOutcome::Committed + ); +} + +#[tokio::test] +async fn hls_terminal_commit_retry_asset_change_autonomously_fails_closed() { + let (manager, session, proxy_session_id, lease_id, preparation, plan) = + prepared_terminal_commit_fixture("retry-asset-change").await; + let expected_asset = plan.asset_identity; + let asset_is_current = Arc::new(AtomicBool::new(true)); + let revision_guard = { + let asset_is_current = Arc::clone(&asset_is_current); + HlsTerminalAssetRevisionGuard::for_runtime_tail(expected_asset, move || { + asset_is_current.load(Ordering::Acquire).then_some(expected_asset) + }) + }; + let session_guard = session.write().await; + + assert!(matches!( + manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session: &session, + lease_id: &lease_id, + proxy_session_id: &proxy_session_id, + preparation: &preparation, + now_ms: 2_000, + payload: HlsTerminalCommitPayload::Tail { + plan, + media_guard: HlsTerminalCommitMediaGuard::empty_for_test(), + }, + asset_revision_guard: revision_guard, + }), + HlsTerminalCommitOutcome::LockBusy { .. } + )); + asset_is_current.store(false, Ordering::Release); + drop(session_guard); + + wait_for_terminal_commit_owner(&manager).await; + assert!(matches!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("asset change terminalizes without another request") + .playback_mode, + HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::MissingAsset, .. } + )); +} + +#[tokio::test] +async fn hls_terminal_commit_retry_lock_busy_without_a_second_client_request() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + manager.terminal_commit_clock.set_fixed_now_ms(2_000); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); + assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); + let progress_generation = { + let mut session = session.write().await; + complete_failed_acceptance_episode(&mut session, 2_000) + }; + let preparation = manager + .prepare_access_lease_terminal_tail(terminal_preparation_request( + &lease_id, + &proxy_session_id, + progress_generation, + )) + .await + .expect("terminal preparation"); + let plan = terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id); + + let session_guard = session.write().await; + assert_eq!( + commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, Arc::clone(&plan),), + HlsTerminalCommitOutcome::LockBusy { retry_before_ms: 2_001 } + ); + assert_eq!( + commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), + HlsTerminalCommitOutcome::LockBusy { retry_before_ms: 2_001 } + ); + assert_eq!(manager.terminal_commit_retries.owner_count(), 1); + drop(session_guard); + + wait_for_terminal_commit_owner(&manager).await; + assert!(matches!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("autonomous retry keeps the lease available") + .playback_mode, + HlsLeasePlaybackMode::TerminalTail(_) + )); +} + +#[tokio::test] +async fn hls_terminal_commit_submission_delayed_stale_incoming_preserves_authoritative_tail() { + let (manager, session, proxy_session_id, lease_id, preparation, plan) = + prepared_terminal_commit_fixture("submission-stale-incoming").await; + let current_asset = plan.asset_identity; + let stale_asset = HlsTerminalAssetIdentity { revision: 999, fingerprint: [9; 32] }; + let Some(session_incarnation) = manager.sessions.session_incarnation(&session) else { + panic!("fixture session has an incarnation"); + }; + let Some((cancellation_epoch, submission_token)) = manager.terminal_commit_retries.reserve_submission() else { + panic!("fixture can reserve terminal submission"); + }; + let owner_key = HlsTerminalCommitOwnerKey::from_preparation(&proxy_session_id, &lease_id, &preparation); + let command = HlsTerminalCommitCommand { + key: owner_key, + session: Arc::clone(&session), + session_incarnation, + preparation: preparation.clone(), + decision: HlsTerminalLeaseDecision::Tail(Arc::clone(&plan)), + media_guard: Some(HlsTerminalCommitMediaGuard::empty_for_test()), + asset_revision_guard: HlsTerminalAssetRevisionGuard::matching_runtime_for_test(current_asset), + cancellation_epoch, + submission_token, + }; + let HlsTerminalCommitSubmissionDecision::Attempt { command, owner_token } = + manager.terminal_commit_retries.submit(command, 2_000) + else { + panic!("authoritative tail owns the initial submission"); + }; + let HlsTerminalCommitRetryDecision::Schedule { retry_at_ms, attempts_completed } = next_terminal_commit_retry( + 1, + 2_000, + preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), + ) else { + panic!("fixture has retry budget"); + }; + let HlsTerminalCommitRetryScheduleDecision::Scheduled { worker_token: Some(worker_token) } = manager + .terminal_commit_retries + .schedule_current(&command.key, owner_token, attempts_completed, retry_at_ms, 2_000) + else { + panic!("authoritative tail schedules one bounded worker"); + }; + + let mut stale_preparation = preparation.clone(); + stale_preparation.cutover_timing = HlsLeaseCutoverTiming::from_reserve( + 2_000, + preparation.reserve.transition_margin.as_millis().saturating_add(2), + preparation.reserve.transition_margin, + None, + ); + let outcome = manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session: &session, + lease_id: &lease_id, + proxy_session_id: &proxy_session_id, + preparation: &stale_preparation, + now_ms: 2_000, + payload: HlsTerminalCommitPayload::Unavailable(HlsTerminalTailCompatibility::AssetRevisionMismatch), + asset_revision_guard: HlsTerminalAssetRevisionGuard::for_runtime_tail( + HlsRuntimeCustomTailAssetIdentity::new(current_asset.reason, stale_asset), + move || Some(current_asset), + ), + }); + + assert_eq!(outcome, HlsTerminalCommitOutcome::LockBusy { retry_before_ms: retry_at_ms }); + assert_eq!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("unauthorized incoming keeps lease available") + .playback_mode, + HlsLeasePlaybackMode::Live + ); + spawn_terminal_commit_retry_worker( + Arc::clone(&manager.sessions), + Arc::clone(&manager.access_leases), + Arc::clone(&manager.terminal_commit_retries), + Arc::clone(&manager.terminal_commit_clock), + worker_token, + HlsProxyManager::try_commit_access_lease_terminal_decision, + ); + wait_for_terminal_commit_owner(&manager).await; + assert!(matches!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_001) + .await + .expect("authorized owner terminalizes the lease") + .playback_mode, + HlsLeasePlaybackMode::TerminalTail(_) + )); +} + +#[tokio::test] +async fn hls_terminal_commit_submission_newer_command_replaces_lock_busy_owner() { + let (manager, session, proxy_session_id, lease_id, preparation, plan) = + prepared_terminal_commit_fixture("retry-owner-replacement").await; + let stale_asset_identity = plan.asset_identity; + let session_guard = session.write().await; + + assert!(matches!( + manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session: &session, + lease_id: &lease_id, + proxy_session_id: &proxy_session_id, + preparation: &preparation, + now_ms: 2_000, + payload: HlsTerminalCommitPayload::Tail { + plan, + media_guard: HlsTerminalCommitMediaGuard::empty_for_test(), + }, + asset_revision_guard: HlsTerminalAssetRevisionGuard::for_runtime_tail(stale_asset_identity, || None,), + }), + HlsTerminalCommitOutcome::LockBusy { .. } + )); + assert!(matches!( + manager.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session: &session, + lease_id: &lease_id, + proxy_session_id: &proxy_session_id, + preparation: &preparation, + now_ms: 2_000, + payload: HlsTerminalCommitPayload::Unavailable(HlsTerminalTailCompatibility::AssetRevisionMismatch,), + asset_revision_guard: HlsTerminalAssetRevisionGuard::matching_for_test(None), + }), + HlsTerminalCommitOutcome::LockBusy { .. } + )); + assert_eq!(manager.terminal_commit_retries.owner_count(), 1); + drop(session_guard); + + wait_for_terminal_commit_owner(&manager).await; + assert!(matches!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("replacement command terminalizes the lease") + .playback_mode, + HlsLeasePlaybackMode::TerminalUnavailable { reason: HlsTerminalTailCompatibility::AssetRevisionMismatch, .. } + )); +} + +#[tokio::test] +async fn hls_terminal_commit_retry_never_commits_after_the_safe_deadline() { + let (manager, session, proxy_session_id, lease_id, preparation, plan) = + prepared_terminal_commit_fixture("retry-deadline").await; + let after_deadline_ms = + preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch().saturating_add(1); + manager.terminal_commit_clock.set_fixed_now_ms(after_deadline_ms); + let session_guard = session.write().await; + + assert_eq!( + commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), + HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed + ); + assert_eq!(manager.terminal_commit_retries.owner_count(), 0); + drop(session_guard); + + assert_eq!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, after_deadline_ms) + .await + .expect("deadline-expired lease remains live") + .playback_mode, + HlsLeasePlaybackMode::Live + ); + assert!(!session.read().await.has_terminal_tail_protections()); +} + +#[tokio::test] +async fn hls_terminal_commit_initial_and_retry_attempts_reject_exclusive_safe_deadline() { + let (manager, session, proxy_session_id, lease_id, preparation, plan) = + prepared_terminal_commit_fixture("initial-exclusive-deadline").await; + let safe_deadline_ms = preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + manager.terminal_commit_clock.set_fixed_now_ms(safe_deadline_ms); + + assert_eq!( + commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), + HlsTerminalCommitOutcome::SafeCommitDeadlineElapsed + ); + assert!(matches!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, safe_deadline_ms) + .await + .expect("exclusive deadline leaves the initial-attempt lease live") + .playback_mode, + HlsLeasePlaybackMode::Live + )); + assert!(!session.read().await.has_terminal_tail_protections()); + + let (manager, session, proxy_session_id, lease_id, preparation, plan) = + prepared_terminal_commit_fixture("retry-exclusive-deadline").await; + let safe_deadline_ms = preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(); + let session_guard = session.write().await; + assert!(matches!( + commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), + HlsTerminalCommitOutcome::LockBusy { .. } + )); + assert_eq!(manager.terminal_commit_retries.owner_count(), 1); + manager.terminal_commit_clock.set_fixed_now_ms(safe_deadline_ms); + drop(session_guard); + + wait_for_terminal_commit_owner(&manager).await; + assert!(matches!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, safe_deadline_ms) + .await + .expect("exclusive deadline leaves the retry lease live") + .playback_mode, + HlsLeasePlaybackMode::Live + )); + assert!(!session.read().await.has_terminal_tail_protections()); +} + +#[tokio::test] +async fn hls_terminal_commit_session_replacement_cancels_the_detached_retry() { + let (manager, session, proxy_session_id, lease_id, preparation, plan) = + prepared_terminal_commit_fixture("retry-session-replacement").await; + let session_key = session.read().await.key.clone(); + let lease_guard = manager.access_leases.write().await; + + assert!(matches!( + commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), + HlsTerminalCommitOutcome::LockBusy { .. } + )); + manager.sessions.remove_session(&session_key, &proxy_session_id).await.expect("remove prepared session"); + let (replacement, outcome) = manager.get_or_create_session_with_outcome(session_key, b"secret", 2_000).await; + assert_eq!(outcome, HlsSessionStoreOutcome::Created); + assert!(!Arc::ptr_eq(&session, &replacement)); + drop(lease_guard); + + wait_for_terminal_commit_owner(&manager).await; + assert_eq!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("replacement race keeps lease live") + .playback_mode, + HlsLeasePlaybackMode::Live + ); + assert!(!session.read().await.has_terminal_tail_protections()); + assert!(!replacement.read().await.has_terminal_tail_protections()); +} + +#[tokio::test] +async fn hls_terminal_commit_submission_current_session_replaces_stale_incarnation_owner() { + let (manager, stale_session, proxy_session_id, lease_id, preparation, plan) = + prepared_terminal_commit_fixture("retry-current-session-replacement").await; + let session_key = stale_session.read().await.key.clone(); + let lease_guard = manager.access_leases.write().await; + + assert!(matches!( + commit_terminal_plan( + &manager, + &stale_session, + &lease_id, + &proxy_session_id, + &preparation, + 2_000, + Arc::clone(&plan), + ), + HlsTerminalCommitOutcome::LockBusy { .. } + )); + manager.sessions.remove_session(&session_key, &proxy_session_id).await.expect("remove stale session incarnation"); + let (current_session, outcome) = manager.get_or_create_session_with_outcome(session_key, b"secret", 2_000).await; + assert_eq!(outcome, HlsSessionStoreOutcome::Created); + assert!(!Arc::ptr_eq(&stale_session, ¤t_session)); + { + let mut current = current_session.write().await; + assert_eq!(complete_failed_acceptance_episode(&mut current, 2_000), preparation.origin_progress_generation); + } + + assert!(matches!( + commit_terminal_plan(&manager, ¤t_session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), + HlsTerminalCommitOutcome::LockBusy { .. } + )); + assert_eq!(manager.terminal_commit_retries.owner_count(), 1); + drop(lease_guard); + + wait_for_terminal_commit_owner(&manager).await; + assert!(matches!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("current session command terminalizes the lease") + .playback_mode, + HlsLeasePlaybackMode::TerminalTail(_) + )); + assert!(!stale_session.read().await.has_terminal_tail_protections()); + assert!(current_session.read().await.has_terminal_tail_protections()); +} + +#[tokio::test] +async fn hls_terminal_commit_submission_stale_session_cannot_replace_current_owner() { + let (manager, stale_session, proxy_session_id, lease_id, preparation, plan) = + prepared_terminal_commit_fixture("retry-stale-session-later").await; + let session_key = stale_session.read().await.key.clone(); + manager.sessions.remove_session(&session_key, &proxy_session_id).await.expect("remove stale session incarnation"); + let (current_session, outcome) = manager.get_or_create_session_with_outcome(session_key, b"secret", 2_000).await; + assert_eq!(outcome, HlsSessionStoreOutcome::Created); + { + let mut current = current_session.write().await; + assert_eq!(complete_failed_acceptance_episode(&mut current, 2_000), preparation.origin_progress_generation); + } + let lease_guard = manager.access_leases.write().await; + let index_guard = manager.sessions.hold_index_write_for_test().await; + + assert!(matches!( + commit_terminal_plan( + &manager, + ¤t_session, + &lease_id, + &proxy_session_id, + &preparation, + 2_000, + Arc::clone(&plan), + ), + HlsTerminalCommitOutcome::LockBusy { .. } + )); + assert!(matches!( + commit_terminal_plan(&manager, &stale_session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), + HlsTerminalCommitOutcome::LockBusy { .. } + )); + assert_eq!(manager.terminal_commit_retries.owner_count(), 1); + drop(index_guard); + drop(lease_guard); + + wait_for_terminal_commit_owner(&manager).await; + assert!(matches!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("current session owner survives stale later submission") + .playback_mode, + HlsLeasePlaybackMode::TerminalTail(_) + )); + assert!(!stale_session.read().await.has_terminal_tail_protections()); + assert!(current_session.read().await.has_terminal_tail_protections()); +} + +#[tokio::test] +async fn hls_terminal_commit_media_progress_before_retry_cancels_owner_and_keeps_lease_live() { + let (manager, session, proxy_session_id, lease_id, preparation, plan) = + prepared_terminal_commit_fixture("retry-recovery").await; + let lease_guard = manager.access_leases.write().await; + + assert!(matches!( + commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), + HlsTerminalCommitOutcome::LockBusy { .. } + )); + { + let mut session = session.write().await; + assert_eq!(session.origin_control.progress_generation, preparation.origin_progress_generation); + session.origin_control.record_media_progress(2_100, 12_000); + assert_eq!( + session.origin_control.progress_generation, + preparation.origin_progress_generation.saturating_add(1) + ); + } + drop(lease_guard); + + wait_for_terminal_commit_owner(&manager).await; + assert_eq!( + manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("recovered lease") + .playback_mode, + HlsLeasePlaybackMode::Live + ); + assert!(!session.read().await.has_terminal_tail_protections()); +} + +#[tokio::test] +async fn hls_terminal_commit_lease_end_before_retry_cancels_owner_without_mutation() { + let (manager, session, proxy_session_id, lease_id, preparation, plan) = + prepared_terminal_commit_fixture("retry-lease-end").await; + let mut lease_guard = manager.access_leases.write().await; + + assert!(matches!( + commit_terminal_plan(&manager, &session, &lease_id, &proxy_session_id, &preparation, 2_000, plan,), + HlsTerminalCommitOutcome::LockBusy { .. } + )); + let _release = lease_guard.deny_access_lease(&lease_id, HlsAccessLeaseDenialMode::ImmediateEnd); + drop(lease_guard); + + wait_for_terminal_commit_owner(&manager).await; + let lease = manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("ended lease remains stored"); + assert_eq!(lease.state, HlsAccessLeaseState::Denied); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Ended); + assert!(!session.read().await.has_terminal_tail_protections()); +} + +#[tokio::test] +async fn hls_terminal_commit_new_media_generation_supersedes_prepared_plan() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); + assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); + let progress_generation = { + let mut session = session.write().await; + complete_failed_acceptance_episode(&mut session, 2_000) + }; + let preparation = manager + .prepare_access_lease_terminal_tail(terminal_preparation_request( + &lease_id, + &proxy_session_id, + progress_generation, + )) + .await + .expect("failed acceptance permits terminal preparation"); + + session.write().await.advance_media_readiness_generation(); + assert_eq!( + commit_terminal_plan( + &manager, + &session, + &lease_id, + &proxy_session_id, + &preparation, + 2_100, + terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), + ), + HlsTerminalCommitOutcome::SupersededGeneration + ); + + let lease = manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_200) + .await + .expect("lease remains available"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + assert!(!session.read().await.has_terminal_tail_protections()); +} + +#[tokio::test] +async fn hls_terminal_commit_lease_end_prevents_commit_and_gc_protection() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); + assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); + let progress_generation = { + let mut session = session.write().await; + complete_failed_acceptance_episode(&mut session, 2_000) + }; + let mut preparation_request = terminal_preparation_request(&lease_id, &proxy_session_id, progress_generation); + preparation_request.reserve.guaranteed_reserve_ms = 100_000; + preparation_request.reserve.guaranteed_media_horizon_ms = preparation_request + .reserve + .conservative_playback_position_ms + .saturating_add(preparation_request.reserve.guaranteed_reserve_ms); + preparation_request.cutover_timing = HlsLeaseCutoverTiming::from_reserve( + preparation_request.now_ms, + preparation_request.reserve.guaranteed_reserve_ms, + preparation_request.reserve.transition_margin, + None, + ); + let preparation = manager + .prepare_access_lease_terminal_tail(preparation_request) + .await + .expect("failed acceptance permits terminal preparation"); + + assert_eq!( + commit_terminal_plan( + &manager, + &session, + &lease_id, + &proxy_session_id, + &preparation, + 61_000, + terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), + ), + HlsTerminalCommitOutcome::LeaseNoLongerEligible + ); + + let lease = manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 61_000) + .await + .expect("expired lease remains stored until lifecycle cleanup"); + assert_eq!(lease.state, HlsAccessLeaseState::Expired); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Ended); + assert!(!session.read().await.has_terminal_tail_protections()); +} + +pub(in crate::manager::tests) async fn assert_terminal_and_live_lease_modes( + manager: &HlsProxyManager, + proxy_session_id: &ProxySessionId, + terminal_lease_id: &HlsAccessLeaseId, + live_lease_id: &HlsAccessLeaseId, +) { + assert!(matches!( + manager + .access_lease_response_snapshot(terminal_lease_id, proxy_session_id, 2_100) + .await + .expect("terminal lease") + .playback_mode, + HlsLeasePlaybackMode::TerminalTail(_) + )); + assert_eq!( + manager + .access_lease_response_snapshot(live_lease_id, proxy_session_id, 2_100) + .await + .expect("live lease") + .playback_mode, + HlsLeasePlaybackMode::Live + ); +} + +#[tokio::test] +async fn hls_cutover_policy_terminal_commit_uses_lease_deadline_and_keeps_farther_lease_live() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let terminal_lease_id = HlsAccessLeaseId("terminal".to_string()); + let live_lease_id = HlsAccessLeaseId("live".to_string()); + { + let mut leases = manager.access_leases.write().await; + leases.prepare_access_lease(access_lease(&terminal_lease_id.0, &proxy_session_id)); + leases.prepare_access_lease(access_lease(&live_lease_id.0, &proxy_session_id)); + } + assert!( + publish_manifest_snapshot(&manager, &terminal_lease_id, &proxy_session_id, manifest_snapshot(1), 2_000,).await + ); + assert!(publish_manifest_snapshot(&manager, &live_lease_id, &proxy_session_id, manifest_snapshot(1), 2_000,).await); + let progress_generation = { + let mut session = session.write().await; + complete_failed_acceptance_episode(&mut session, 2_000) + }; + let far_reserve = HlsLeaseReserveSnapshot { + guaranteed_media_horizon_ms: 36_000, + conservative_playback_position_ms: 12_000, + guaranteed_reserve_ms: 24_000, + cutover_required: false, + ..cutover_reserve() + }; + let far_timing = HlsLeaseCutoverTiming::from_reserve( + 2_000, + far_reserve.guaranteed_reserve_ms, + far_reserve.transition_margin, + None, + ); + assert!(manager + .prepare_access_lease_terminal_tail(HlsTerminalTailPreparationRequest { + lease_id: &live_lease_id, + proxy_session_id: &proxy_session_id, + manifest_snapshot_generation: 1, + cursor_generation: 0, + reserve: far_reserve, + cutover_timing: far_timing, + commit_window: HlsTerminalCommitWindow::NotDue, + now_ms: 2_000, + origin_progress_generation: progress_generation, + media_readiness_generation: 0, + last_media_progress_at_ms: None, + }) + .await + .is_none()); + let preparation = manager + .prepare_access_lease_terminal_tail(terminal_preparation_request( + &terminal_lease_id, + &proxy_session_id, + progress_generation, + )) + .await + .expect("terminal preparation"); + assert_eq!( + preparation.cutover_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), + 2_000_u64.saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()) + ); + assert_eq!(far_timing.latest_safe_terminal_commit_at.as_millis_since_epoch(), 14_000); + let plan = terminal_plan(preparation.decision_generation, &proxy_session_id, &terminal_lease_id); + let expected_protection = Arc::clone(&plan.protected_base_proxy_seqs); + + assert_eq!( + commit_terminal_plan(&manager, &session, &terminal_lease_id, &proxy_session_id, &preparation, 2_000, plan,), + HlsTerminalCommitOutcome::Committed + ); + + let session = session.read().await; + assert_eq!( + session.terminal_tail_protection(&terminal_lease_id).map(|protection| &protection.base_proxy_seqs), + Some(&expected_protection) + ); + assert_eq!( + session.origin_control.progress_phase, + super::super::super::origin_progress::HlsOriginProgressPhase::TerminalPartial + ); + drop(session); + assert_terminal_and_live_lease_modes(&manager, &proxy_session_id, &terminal_lease_id, &live_lease_id).await; +} + +#[tokio::test] +async fn resource_denial_does_not_destroy_existing_terminal_tail() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("terminal".to_string()); + let plan = terminal_plan(7, &proxy_session_id, &lease_id); + let mut lease = access_lease(&lease_id.0, &proxy_session_id); + lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(Arc::clone(&plan)); + manager.access_leases.write().await.prepare_access_lease(lease); + { + let mut session = session.write().await; + assert_eq!( + session.install_terminal_tail_protection( + lease_id.clone(), + HlsTerminalTailProtection { + generation: plan.generation, + base_proxy_seqs: Arc::clone(&plan.protected_base_proxy_seqs), + key_bindings: plan.key_bindings(), + }, + ), + super::super::super::session::HlsTerminalTailProtectionInstall::Installed + ); + } + + assert_eq!( + manager.deny_access_lease(&lease_id, HlsAccessLeaseDenialMode::PreserveCommittedFiniteTail,).await, + HlsAccessLeaseDenialOutcome::FiniteDecisionPreserved + ); + + let denied = manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("denied lease remains stored until lifecycle cleanup"); + assert_eq!(denied.state, HlsAccessLeaseState::Denied); + assert_eq!(denied.playback_mode, HlsLeasePlaybackMode::TerminalTail(Arc::clone(&plan))); + assert!(session.read().await.terminal_tail_protection(&lease_id).is_some()); + + assert_eq!( + manager.deny_access_lease(&lease_id, HlsAccessLeaseDenialMode::ImmediateEnd).await, + HlsAccessLeaseDenialOutcome::FiniteDecisionPreserved + ); + assert!(session.read().await.terminal_tail_protection(&lease_id).is_some()); + + manager.remove_access_lease(&lease_id).await; + assert!(session.read().await.terminal_tail_protection(&lease_id).is_none()); + assert_eq!( + manager.access_leases.write().await.deny_access_lease(&lease_id, HlsAccessLeaseDenialMode::ImmediateEnd), + HlsAccessLeaseDenialOutcome::UnknownLease + ); +} + +#[tokio::test] +async fn terminal_tail_protection_is_retained_until_bounded_lease_removal() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("terminal-stale-protection".to_string()); + let plan = terminal_plan(7, &proxy_session_id, &lease_id); + let stale_generation = HlsTerminalTailGeneration(plan.generation.0.saturating_add(1)); + let mut lease = access_lease(&lease_id.0, &proxy_session_id); + lease.playback_mode = HlsLeasePlaybackMode::TerminalTail(Arc::clone(&plan)); + manager.access_leases.write().await.prepare_access_lease(lease); + assert_eq!( + session.write().await.install_terminal_tail_protection( + lease_id.clone(), + HlsTerminalTailProtection { + generation: stale_generation, + base_proxy_seqs: Arc::from([9_999]), + key_bindings: Arc::from([]), + }, + ), + super::super::super::session::HlsTerminalTailProtectionInstall::Installed + ); + + assert_eq!( + manager.deny_access_lease(&lease_id, HlsAccessLeaseDenialMode::PreserveCommittedFiniteTail,).await, + HlsAccessLeaseDenialOutcome::FiniteDecisionPreserved + ); + + assert!(session.read().await.terminal_tail_protection(&lease_id).is_some()); + manager.remove_access_lease(&lease_id).await; + assert!(session.read().await.terminal_tail_protection(&lease_id).is_none()); +} + +#[tokio::test] +async fn terminal_unavailable_protection_is_retained_until_bounded_lease_removal() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("terminal-unavailable".to_string()); + let decision_generation = 11; + let generation = HlsTerminalTailGeneration(decision_generation); + let mut lease = access_lease(&lease_id.0, &proxy_session_id); + lease.playback_mode = HlsLeasePlaybackMode::TerminalUnavailable { + decision_generation, + reason: HlsTerminalTailCompatibility::ProtectionCapacityExceeded, + }; + manager.access_leases.write().await.prepare_access_lease(lease); + assert_eq!( + session.write().await.install_terminal_tail_protection( + lease_id.clone(), + HlsTerminalTailProtection { generation, base_proxy_seqs: Arc::from([41_u64]), key_bindings: Arc::from([]) }, + ), + super::super::super::session::HlsTerminalTailProtectionInstall::Installed + ); + + assert_eq!( + manager.deny_access_lease(&lease_id, HlsAccessLeaseDenialMode::PreserveCommittedFiniteTail,).await, + HlsAccessLeaseDenialOutcome::FiniteDecisionPreserved + ); + + let denied = manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_000) + .await + .expect("denied lease remains stored until lifecycle cleanup"); + assert_eq!(denied.state, HlsAccessLeaseState::Denied); + assert!(matches!(denied.playback_mode, HlsLeasePlaybackMode::TerminalUnavailable { .. })); + assert!(session.read().await.terminal_tail_protection(&lease_id).is_some()); + manager.remove_access_lease(&lease_id).await; + assert!(session.read().await.terminal_tail_protection(&lease_id).is_none()); +} + +#[tokio::test] +async fn hls_terminal_commit_protection_capacity_has_typed_unavailable_outcome() { + let config = HlsCacheConfig::from(&HlsCacheConfigDto::default()); + let manager = HlsProxyManager::with_hls_cache_config(&config); + let (session, _) = + manager.get_or_create_session_with_outcome(HlsSessionKey::new(1, "stream-a"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("overflow".to_string()); + manager.access_leases.write().await.prepare_access_lease(access_lease(&lease_id.0, &proxy_session_id)); + assert!(publish_manifest_snapshot(&manager, &lease_id, &proxy_session_id, manifest_snapshot(1), 2_000).await); + let progress_generation = { + let mut session = session.write().await; + for index in 0..HLS_TERMINAL_TAIL_PROTECTION_CAPACITY { + session.install_terminal_tail_protection( + HlsAccessLeaseId(format!("occupied-{index}")), + HlsTerminalTailProtection { + generation: HlsTerminalTailGeneration(1), + base_proxy_seqs: Arc::from([u64::try_from(index).unwrap_or(u64::MAX)]), + key_bindings: Arc::from([]), + }, + ); + } + complete_failed_acceptance_episode(&mut session, 2_000) + }; + let preparation = manager + .prepare_access_lease_terminal_tail(terminal_preparation_request( + &lease_id, + &proxy_session_id, + progress_generation, + )) + .await + .expect("terminal preparation"); + + assert_eq!( + commit_terminal_plan( + &manager, + &session, + &lease_id, + &proxy_session_id, + &preparation, + 2_000, + terminal_plan(preparation.decision_generation, &proxy_session_id, &lease_id), + ), + HlsTerminalCommitOutcome::Committed + ); + + let lease = manager + .access_lease_response_snapshot(&lease_id, &proxy_session_id, 2_100) + .await + .expect("terminal unavailable lease remains stored"); + assert!(matches!( + lease.playback_mode, + HlsLeasePlaybackMode::TerminalUnavailable { + reason: HlsTerminalTailCompatibility::ProtectionCapacityExceeded, + .. + } + )); + let session = session.read().await; + assert_eq!(session.terminal_tail_protection_count(), HLS_TERMINAL_TAIL_PROTECTION_CAPACITY); + assert!(session.terminal_tail_protection(&lease_id).is_none()); +} diff --git a/backend/hls/src/manifest_acceptance.rs b/backend/hls/src/manifest_acceptance.rs index 695f496f7..6659efad1 100644 --- a/backend/hls/src/manifest_acceptance.rs +++ b/backend/hls/src/manifest_acceptance.rs @@ -8,12 +8,14 @@ use super::{ }, resource_identity::HlsMediaResourceIdentity, }; -use sha2::{Digest, Sha256}; use shared::model::HlsManifestRecoveryBurstPlan; pub const HLS_MANIFEST_RECOVERY_BURST_SLOT_DELAY_MS: u64 = 100; + pub const HLS_MANIFEST_FINGERPRINT_SEGMENT_LIMIT: usize = 64; + const HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT: usize = 32; + pub const HLS_MANIFEST_ACCEPTANCE_MAX_REQUALIFICATIONS_PER_REFRESH: u8 = 1; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -27,115 +29,12 @@ pub struct HlsManifestRecoveryCandidateIdentity { manifest_fingerprint: [u8; 32], } -impl PartialEq for HlsManifestRecoveryCandidateIdentity { - fn eq(&self, other: &Self) -> bool { - self.candidate_index == other.candidate_index - && self.effective_host_fingerprint == other.effective_host_fingerprint - && self.manifest_fingerprint == other.manifest_fingerprint - } -} - -impl Eq for HlsManifestRecoveryCandidateIdentity {} - -impl HlsManifestRecoveryCandidateIdentity { - pub fn from_candidate(candidate_index: usize, effective_host: Option<&str>, manifest_body: &str) -> Self { - let mut host_hasher = Sha256::new(); - host_hasher.update(b"tuliprox-hls-candidate-host-v1\0"); - match effective_host { - Some(host) => { - host_hasher.update([1]); - host_hasher.update(host.as_bytes()); - } - None => host_hasher.update([0]), - } - let mut manifest_hasher = Sha256::new(); - manifest_hasher.update(b"tuliprox-hls-candidate-manifest-v1\0"); - manifest_hasher.update(manifest_body.as_bytes()); - Self { - candidate_index, - effective_host_fingerprint: host_hasher.finalize().into(), - manifest_fingerprint: manifest_hasher.finalize().into(), - } - } - - pub fn matches_candidate(self, effective_host: Option<&str>, manifest_body: &str) -> bool { - self == Self::from_candidate(self.candidate_index, effective_host, manifest_body) - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct HlsSelectedRecoveryCandidate { acceptance_generation: HlsManifestAcceptanceGeneration, candidate_identity: HlsManifestRecoveryCandidateIdentity, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum HlsRecoveryWorkloadBinding { - CandidateUnknown { - envelope: HlsRecoveryWorkloadEnvelope, - selected_candidate: Option, - }, - CandidateBound { - envelope: HlsRecoveryWorkloadEnvelope, - acceptance_generation: HlsManifestAcceptanceGeneration, - candidate_identity: HlsManifestRecoveryCandidateIdentity, - workload: HlsRecoveryWorkload, - }, -} - -impl HlsRecoveryWorkloadBinding { - const fn envelope(self) -> HlsRecoveryWorkloadEnvelope { - match self { - Self::CandidateUnknown { envelope, .. } | Self::CandidateBound { envelope, .. } => envelope, - } - } - - const fn workload(self) -> HlsRecoveryWorkload { - match self { - Self::CandidateUnknown { envelope, .. } => envelope.ceiling(), - Self::CandidateBound { workload, .. } => workload, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsRecoveryWorkloadBindingUpdate { - Applied, - StaleGeneration, - EpisodeInactive, - CandidateMismatch, - OutsideEnvelope, -} - -/// Immutable pressure snapshot that authorizes one manifest-acceptance episode. -/// -/// Entering `Recovering` is an execution-state transition, not evidence that -/// reserve pressure exists. Consumers must therefore derive acceptance policy -/// from this trigger captured before the episode starts. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub enum HlsManifestAcceptanceTrigger { - #[default] - None, - Observe, - RecoveryRequired, - Critical, -} - -impl HlsManifestAcceptanceTrigger { - pub const fn starts_episode(self) -> bool { !matches!(self, Self::None) } - - pub const fn recovery_required(self) -> bool { matches!(self, Self::RecoveryRequired | Self::Critical) } - - pub const fn as_log_value(self) -> &'static str { - match self { - Self::None => "none", - Self::Observe => "observe", - Self::RecoveryRequired => "recovery_required", - Self::Critical => "critical", - } - } -} - #[derive(Debug, Clone, PartialEq, Eq)] pub struct HlsManifestAcceptanceEpisode { pub generation: HlsManifestAcceptanceGeneration, @@ -155,2055 +54,33 @@ pub struct HlsManifestAcceptanceEpisode { deterministic_conflict_receipt: Option, } -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct HlsDeterministicConflictReceipt { - pub conflict: HlsDeterministicTimelineConflict, - pub origin_progress_generation: u64, - pub published_resource_history_generation: u64, - pub pinned_host_generation: u64, -} - -impl HlsManifestAcceptanceEpisode { - pub fn new( - generation: HlsManifestAcceptanceGeneration, - started_at_ms: u64, - burst_plan: HlsManifestRecoveryBurstPlan, - trigger: HlsManifestAcceptanceTrigger, - timing: &HlsAcceptanceEpisodeTiming, - ) -> Self { - let envelope = HlsRecoveryWorkloadEnvelope::from_timing_ceiling(timing.initial_workload); - Self { - generation, - started_at_ms, - burst_plan, - trigger, - timing: *timing, - workload_binding: HlsRecoveryWorkloadBinding::CandidateUnknown { envelope, selected_candidate: None }, - full_burst_completed: false, - full_bursts_completed: 0, - completed_burst_candidates: 0, - state: HlsManifestAcceptanceState::FullBurstPending, - outcome: HlsManifestAcceptanceEpisodeOutcome::Pending, - held_alternative: None, - observed_landscape: None, - next_retry_at_ms: None, - deterministic_conflict_receipt: None, - } - } - - pub fn required_candidates(&self) -> usize { self.burst_plan.total_candidates() } - - pub const fn trigger(&self) -> HlsManifestAcceptanceTrigger { self.trigger } - - pub const fn timing(&self) -> HlsAcceptanceEpisodeTiming { self.timing } - - pub fn deterministic_conflict_receipt(&self) -> Option<&HlsDeterministicConflictReceipt> { - self.deterministic_conflict_receipt.as_ref() - } - - pub fn record_deterministic_conflict(&mut self, receipt: HlsDeterministicConflictReceipt) { - if self.state != HlsManifestAcceptanceState::Completed { - self.deterministic_conflict_receipt = Some(receipt); - if self.full_burst_completed { - self.held_alternative = None; - self.next_retry_at_ms = None; - self.state = HlsManifestAcceptanceState::Holding; - self.outcome = HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted( - HlsManifestAcceptanceExhaustionReason::DeterministicTimelineConflict, - ); - } - } - } - - pub fn selected_candidate_identity(&self) -> Option { - match self.workload_binding { - HlsRecoveryWorkloadBinding::CandidateBound { candidate_identity, .. } => Some(candidate_identity), - HlsRecoveryWorkloadBinding::CandidateUnknown { selected_candidate, .. } => match selected_candidate { - Some(selected) if selected.acceptance_generation == self.generation => { - Some(selected.candidate_identity) - } - Some(_) | None => None, - }, - } - } - - /// Records the selected candidate identity without claiming any media-workload evidence. - /// - /// Same-host commits deliberately remain in `CandidateUnknown`; an alternative candidate is - /// bound only after its generation-local handoff preview identifies the exact recovery medium. - pub fn select_candidate( - &mut self, - expected_generation: HlsManifestAcceptanceGeneration, - candidate_identity: HlsManifestRecoveryCandidateIdentity, - ) -> HlsRecoveryWorkloadBindingUpdate { - if self.generation != expected_generation { - return HlsRecoveryWorkloadBindingUpdate::StaleGeneration; - } - if self.outcome != HlsManifestAcceptanceEpisodeOutcome::Pending - || !matches!( - self.state, - HlsManifestAcceptanceState::StagingSwitchSegment | HlsManifestAcceptanceState::Committing - ) - { - return HlsRecoveryWorkloadBindingUpdate::EpisodeInactive; - } - let HlsRecoveryWorkloadBinding::CandidateUnknown { envelope, selected_candidate } = self.workload_binding - else { - return HlsRecoveryWorkloadBindingUpdate::CandidateMismatch; - }; - let selected = HlsSelectedRecoveryCandidate { acceptance_generation: expected_generation, candidate_identity }; - if selected_candidate.is_some_and(|current| current != selected) { - return HlsRecoveryWorkloadBindingUpdate::CandidateMismatch; - } - self.workload_binding = - HlsRecoveryWorkloadBinding::CandidateUnknown { envelope, selected_candidate: Some(selected) }; - HlsRecoveryWorkloadBindingUpdate::Applied - } - - pub fn bind_selected_candidate( - &mut self, - expected_generation: HlsManifestAcceptanceGeneration, - candidate_identity: HlsManifestRecoveryCandidateIdentity, - workload: HlsRecoveryWorkload, - ) -> HlsRecoveryWorkloadBindingUpdate { - if self.generation != expected_generation { - return HlsRecoveryWorkloadBindingUpdate::StaleGeneration; - } - if self.outcome != HlsManifestAcceptanceEpisodeOutcome::Pending - || !matches!( - self.state, - HlsManifestAcceptanceState::StagingSwitchSegment | HlsManifestAcceptanceState::Committing - ) - { - return HlsRecoveryWorkloadBindingUpdate::EpisodeInactive; - } - let HlsRecoveryWorkloadBinding::CandidateUnknown { envelope, selected_candidate } = self.workload_binding - else { - return HlsRecoveryWorkloadBindingUpdate::CandidateMismatch; - }; - if selected_candidate - != Some(HlsSelectedRecoveryCandidate { acceptance_generation: expected_generation, candidate_identity }) - { - return HlsRecoveryWorkloadBindingUpdate::CandidateMismatch; - } - if !envelope.contains(workload) { - return HlsRecoveryWorkloadBindingUpdate::OutsideEnvelope; - } - self.workload_binding = HlsRecoveryWorkloadBinding::CandidateBound { - envelope, - acceptance_generation: expected_generation, - candidate_identity, - workload, - }; - HlsRecoveryWorkloadBindingUpdate::Applied - } - - pub fn advance_bound_candidate( - &mut self, - expected_generation: HlsManifestAcceptanceGeneration, - expected_identity: HlsManifestRecoveryCandidateIdentity, - workload: HlsRecoveryWorkload, - ) -> HlsRecoveryWorkloadBindingUpdate { - if self.generation != expected_generation { - return HlsRecoveryWorkloadBindingUpdate::StaleGeneration; - } - if self.outcome != HlsManifestAcceptanceEpisodeOutcome::Pending - || self.state != HlsManifestAcceptanceState::StagingSwitchSegment - { - return HlsRecoveryWorkloadBindingUpdate::EpisodeInactive; - } - let HlsRecoveryWorkloadBinding::CandidateBound { - envelope, - acceptance_generation, - candidate_identity, - workload: previous_workload, - } = self.workload_binding - else { - return HlsRecoveryWorkloadBindingUpdate::CandidateMismatch; - }; - if acceptance_generation != expected_generation || candidate_identity != expected_identity { - return HlsRecoveryWorkloadBindingUpdate::CandidateMismatch; - } - if !envelope.contains(workload) || !workload.is_no_greater_than(previous_workload) { - return HlsRecoveryWorkloadBindingUpdate::OutsideEnvelope; - } - self.workload_binding = HlsRecoveryWorkloadBinding::CandidateBound { - envelope, - acceptance_generation, - candidate_identity, - workload, - }; - HlsRecoveryWorkloadBindingUpdate::Applied - } - - /// Returns the work still attributable to this exact episode generation. - /// - /// The timing snapshot itself remains unchanged. Only completed burst work - /// and exact candidate-bound evidence may reduce the current estimate. - pub fn remaining_recovery_workload( - &self, - expected_generation: HlsManifestAcceptanceGeneration, - ) -> Option { - if self.generation != expected_generation || self.outcome != HlsManifestAcceptanceEpisodeOutcome::Pending { - return None; - } - let workload = self.workload_binding.workload(); - Some(if self.full_burst_completed { workload.after_full_burst() } else { workload }) - } - - #[cfg(any(test, feature = "test-support"))] - pub fn remaining_recovery_eta( - &self, - expected_generation: HlsManifestAcceptanceGeneration, - ) -> Option { - self.remaining_recovery_workload(expected_generation).map(|workload| self.timing.remaining_eta(workload)) - } - - pub fn estimated_recovery_completion_at( - &self, - expected_generation: HlsManifestAcceptanceGeneration, - now_ms: u64, - ) -> Option { - self.remaining_recovery_workload(expected_generation) - .map(|workload| self.timing.estimated_completion_at(now_ms, workload)) - } - - pub const fn exhaustion_reason(&self) -> Option { - match self.outcome { - HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted(reason) => Some(reason), - HlsManifestAcceptanceEpisodeOutcome::Pending | HlsManifestAcceptanceEpisodeOutcome::Committed => None, - } - } - - pub fn burst_max_stagger_ms(&self) -> u64 { - u64::try_from(self.burst_plan.slots.saturating_sub(1)) - .unwrap_or(u64::MAX) - .saturating_mul(HLS_MANIFEST_RECOVERY_BURST_SLOT_DELAY_MS) - } - - #[cfg(any(test, feature = "test-support"))] - pub fn record_full_burst(&mut self) { self.record_full_burst_candidates(self.required_candidates()); } - - pub fn record_full_burst_candidates(&mut self, completed_candidates: usize) { - self.completed_burst_candidates = completed_candidates.min(self.required_candidates()); - if self.completed_burst_candidates != self.required_candidates() { - return; - } - self.full_burst_completed = true; - self.full_bursts_completed = self.full_bursts_completed.saturating_add(1); - self.state = HlsManifestAcceptanceState::Evaluating; - } - - pub fn complete(&mut self) { - self.held_alternative = None; - self.next_retry_at_ms = None; - self.deterministic_conflict_receipt = None; - self.state = HlsManifestAcceptanceState::Completed; - self.outcome = HlsManifestAcceptanceEpisodeOutcome::Committed; - } - - pub fn record_exhaustion(&mut self, reason: HlsManifestAcceptanceExhaustionReason) { - if self.full_burst_completed && self.state != HlsManifestAcceptanceState::Completed { - self.outcome = HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted(reason); - } - } - - pub fn hold_after_uncommitted_burst( - &mut self, - mut cohort: Option, - next_retry_at_ms: Option, - ) { - if self.full_burst_completed && self.state != HlsManifestAcceptanceState::Completed { - if let Some(cohort) = cohort.as_mut() { - let limit = u16::try_from(HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT).unwrap_or(u16::MAX); - cohort.successful_samples = cohort.successful_samples.min(limit); - cohort.total_samples = cohort.total_samples.min(limit); - } - self.held_alternative = cohort; - self.next_retry_at_ms = next_retry_at_ms; - self.state = HlsManifestAcceptanceState::Holding; - self.workload_binding = HlsRecoveryWorkloadBinding::CandidateUnknown { - envelope: self.workload_binding.envelope(), - selected_candidate: None, - }; - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsManifestAcceptanceState { - FullBurstPending, - Collecting, - Evaluating, - Holding, - StagingSwitchSegment, - Committing, - Completed, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsManifestAcceptanceExhaustionReason { - AllFailed, - NoProgress, - NoCommittableCandidate, - DeterministicTimelineConflict, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum HlsManifestAcceptanceEpisodeOutcome { - Pending, - FullBurstExhausted(HlsManifestAcceptanceExhaustionReason), - Committed, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsManifestAcceptanceEpisodeStatus { - Missing, - Expired { generation: HlsManifestAcceptanceGeneration }, - InFlight { generation: HlsManifestAcceptanceGeneration }, - FullBurstExhausted { generation: HlsManifestAcceptanceGeneration, reason: HlsManifestAcceptanceExhaustionReason }, - Committed { generation: HlsManifestAcceptanceGeneration }, - Superseded { generation: HlsManifestAcceptanceGeneration, current_generation: HlsManifestAcceptanceGeneration }, -} - -/// Normalizes private episode internals into one generation-safe policy input. -/// Callers must not infer cutover state from `state` plus counters themselves. -pub fn manifest_acceptance_episode_status( - episode: Option<&HlsManifestAcceptanceEpisode>, - current_generation: HlsManifestAcceptanceGeneration, - now_ms: u64, -) -> HlsManifestAcceptanceEpisodeStatus { - let Some(episode) = episode else { - return HlsManifestAcceptanceEpisodeStatus::Missing; - }; - if episode.generation != current_generation { - return HlsManifestAcceptanceEpisodeStatus::Superseded { generation: episode.generation, current_generation }; - } - match episode.outcome { - HlsManifestAcceptanceEpisodeOutcome::Pending - if now_ms >= episode.timing.acceptance_deadline.as_millis_since_epoch() => - { - HlsManifestAcceptanceEpisodeStatus::Expired { generation: episode.generation } - } - HlsManifestAcceptanceEpisodeOutcome::Pending => { - HlsManifestAcceptanceEpisodeStatus::InFlight { generation: episode.generation } - } - HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted(reason) => { - HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { generation: episode.generation, reason } - } - HlsManifestAcceptanceEpisodeOutcome::Committed => { - HlsManifestAcceptanceEpisodeStatus::Committed { generation: episode.generation } - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsCandidateHostRelation { - PinnedHost, - OtherHost, - InitialBaseline, - Unknown, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsHostLocalSequenceRelation { - NoBaseline, - Same, - Next, - PlausibleForward, - Backward, - RolloverCandidate, - Rebase, -} - -/// Bounded, URL-normalized metadata for one candidate segment. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct HlsManifestSegmentFingerprint { - pub duration_ms: u64, - pub discontinuity_before: bool, - pub program_date_time_ms: Option, - pub normalized_resource_identity: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct HlsManifestTimelineFingerprint { - pub segment_count: u32, - pub first_program_date_time_ms: Option, - pub last_program_date_time_ms: Option, - pub duration_pattern_hash: [u8; 32], - pub discontinuity_pattern_hash: [u8; 32], - pub normalized_resource_pattern_hash: Option<[u8; 32]>, - pub map_and_encryption_hash: [u8; 32], - pub container_signature_hash: [u8; 32], - pub segment_samples: Vec, -} - -/// Stable, window-independent technical identity of an alternative timeline. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct HlsManifestTechnicalSignature { - pub map_and_encryption_hash: [u8; 32], - pub container_signature_hash: [u8; 32], -} - -impl HlsManifestTechnicalSignature { - fn from_fingerprint(fingerprint: &HlsManifestTimelineFingerprint) -> Self { - Self { - map_and_encryption_hash: fingerprint.map_and_encryption_hash, - container_signature_hash: fingerprint.container_signature_hash, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct HlsAlternativeOriginCohortIdentity { - pub effective_host: String, - pub technical_signature: HlsManifestTechnicalSignature, -} - -/// One concrete sliding playlist window observed for a stable cohort. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HlsAlternativeOriginWindow { - pub host_local_media_sequence: u64, - pub host_local_highwater: Option, - pub fingerprint: HlsManifestTimelineFingerprint, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsCrossHostAcceptanceEvidence { - Insufficient, - StrongTimelineAnchor { overlapping_segments: u16 }, - BurstConsensusNewEpoch { successful_samples: u16 }, -} - -/// State of the segment that would become the first segment after a host switch. -/// -/// Merely parsing a URI is not READY evidence. Alternative plans carrying -/// `RequiresStaging` must fetch and atomically commit that segment before the -/// commit callback may publish the selected manifest. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsSwitchSegmentReadiness { - Unavailable, - RequiresStaging, -} - -impl HlsSwitchSegmentReadiness { - const fn can_be_staged(self) -> bool { matches!(self, Self::RequiresStaging) } -} - -/// Manifest-level eligibility for the Critical single-candidate path. Track -/// compatibility is deliberately deferred until the candidate bytes are -/// staged and READY; it cannot be inferred from a URI or extension alone. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsEmergencyLiveHandoffCompatibility { - Incompatible, - RequiresStagedTrackVerification, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsTerminalAlternativeCompatibility { - /// Staged media and the configured terminal asset must still be compared. - RequiresStagedComparison, - LiveHandoffSafer, - /// Available terminal media is already known to be at least as safe. - TerminalTailPreferred, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct HlsEmergencyAcceptanceEvidence { - pub live_handoff: HlsEmergencyLiveHandoffCompatibility, - pub terminal_alternative: HlsTerminalAlternativeCompatibility, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsCommittedContentAnchorEvidence { - Unavailable, - RequiresStagedByteVerification, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsResourceTimelineEvidence { - Eligible, - ReplayOnly, - ContradictoryOrder, -} - -impl HlsResourceTimelineEvidence { - const fn permits_acceptance(self) -> bool { matches!(self, Self::Eligible) } -} - -impl HlsEmergencyAcceptanceEvidence { - pub const INCOMPATIBLE: Self = Self { - live_handoff: HlsEmergencyLiveHandoffCompatibility::Incompatible, - terminal_alternative: HlsTerminalAlternativeCompatibility::TerminalTailPreferred, - }; - - const fn requires_staged_verification(self) -> bool { - matches!( - self, - Self { - live_handoff: HlsEmergencyLiveHandoffCompatibility::RequiresStagedTrackVerification, - terminal_alternative: HlsTerminalAlternativeCompatibility::RequiresStagedComparison, - } - ) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HlsManifestCandidateObservation { - pub candidate_index: usize, - pub candidate_slot: usize, - pub effective_host: Option, - pub host_relation: HlsCandidateHostRelation, - pub host_local_media_sequence: u64, - pub host_local_highwater: Option, - pub local_sequence_relation: Option, - pub resource_timeline_evidence: HlsResourceTimelineEvidence, - pub timeline_fingerprint: HlsManifestTimelineFingerprint, - pub manifest_fetch_elapsed_ms: u64, - pub switch_segment_readiness: HlsSwitchSegmentReadiness, - pub committed_content_anchor: HlsCommittedContentAnchorEvidence, - pub emergency_evidence: HlsEmergencyAcceptanceEvidence, - pub evidence: HlsCrossHostAcceptanceEvidence, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HlsAlternativeOriginCohort { - pub identity: HlsAlternativeOriginCohortIdentity, - pub window: HlsAlternativeOriginWindow, - pub successful_samples: u16, - pub total_samples: u16, - pub consecutive_confirmed_full_bursts: u16, - pub evidence: HlsCrossHostAcceptanceEvidence, - pub best_candidate_index: usize, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsManifestCommitKind { - Pinned, - AnchoredAlternative, - ContentVerifiedAlternative, - AlternativeAsNewEpoch, - EmergencyAlternativeAsNewEpoch, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsPinnedOriginObservationState { - Missing, - Unchanged, - Progressed, - Rejected, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HlsManifestAcceptanceLandscape { - pub pinned_state: HlsPinnedOriginObservationState, - pub alternatives: Vec<(HlsAlternativeOriginCohortIdentity, HlsAlternativeOriginWindow)>, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsReducedRetryLandscapeChange { - Unchanged, - NewCohort, - TimelineConflict, - PinnedStateChanged, -} - -impl HlsReducedRetryLandscapeChange { - pub const fn requires_full_requalification(self) -> bool { !matches!(self, Self::Unchanged) } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsManifestCommitPlan { - Commit { candidate_index: usize, kind: HlsManifestCommitKind }, - StageAlternative { candidate_index: usize, kind: HlsManifestCommitKind }, - HoldAlternative, - RejectAll, -} - -#[derive(Clone, Copy)] -pub struct HlsManifestAcceptanceInput<'a> { - pub full_burst_completed: bool, - pub current_burst_is_full_plan: bool, - pub trigger: HlsManifestAcceptanceTrigger, - pub previous_alternative: Option<&'a HlsAlternativeOriginCohort>, - pub observations: &'a [HlsManifestCandidateObservation], -} - -pub fn evaluate_manifest_acceptance(input: HlsManifestAcceptanceInput<'_>) -> HlsManifestCommitPlan { - let (progressed_pinned, unchanged_pinned) = best_pinned_candidates(input.observations); - if let Some(pinned) = progressed_pinned { - return pinned_commit(pinned); - } - - // An unchanged pinned response is useful while reserve remains. Once the - // recovery deadline is reached it must not hide a qualified alternative. - if !input.trigger.recovery_required() { - if let Some(pinned) = unchanged_pinned { - return pinned_commit(pinned); - } - } - - if !input.full_burst_completed { - return if has_stageable_alternative(input.observations) { - HlsManifestCommitPlan::HoldAlternative - } else { - HlsManifestCommitPlan::RejectAll - }; - } - - if let Some(initial) = input - .observations - .iter() - .filter(|candidate| { - candidate.host_relation == HlsCandidateHostRelation::InitialBaseline - && candidate.resource_timeline_evidence.permits_acceptance() - }) - .min_by_key(|candidate| (candidate.manifest_fetch_elapsed_ms, candidate.candidate_index)) - { - return pinned_commit(initial); - } - - let cohorts = alternative_cohorts_with_history( - input.observations, - input.previous_alternative, - input.current_burst_is_full_plan, - ); - if let Some(cohort) = cohorts - .iter() - .find(|cohort| matches!(cohort.evidence, HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { .. })) - { - return alternative_plan( - input.observations, - cohort.best_candidate_index, - HlsManifestCommitKind::AnchoredAlternative, - ); - } - - if let Some(candidate) = input - .current_burst_is_full_plan - .then(|| { - input - .observations - .iter() - .filter(|candidate| candidate.host_relation == HlsCandidateHostRelation::OtherHost) - .filter(|candidate| candidate.resource_timeline_evidence.permits_acceptance()) - .filter(|candidate| { - candidate.committed_content_anchor - == HlsCommittedContentAnchorEvidence::RequiresStagedByteVerification - }) - .min_by_key(|candidate| (candidate.manifest_fetch_elapsed_ms, candidate.candidate_index)) - }) - .flatten() - { - return alternative_plan( - input.observations, - candidate.candidate_index, - HlsManifestCommitKind::ContentVerifiedAlternative, - ); - } - - if input.trigger.recovery_required() { - if let Some(cohort) = cohorts.iter().find(|cohort| { - matches!(cohort.evidence, HlsCrossHostAcceptanceEvidence::BurstConsensusNewEpoch { .. }) - || cohort.consecutive_confirmed_full_bursts >= 2 - }) { - return alternative_plan( - input.observations, - cohort.best_candidate_index, - HlsManifestCommitKind::AlternativeAsNewEpoch, - ); - } - } - - if input.trigger == HlsManifestAcceptanceTrigger::Critical { - if let Some(cohort) = critical_single_candidate_cohort(&cohorts, input.observations) { - return alternative_plan( - input.observations, - cohort.best_candidate_index, - HlsManifestCommitKind::EmergencyAlternativeAsNewEpoch, - ); - } - } - - if cohorts.is_empty() { - HlsManifestCommitPlan::RejectAll - } else { - HlsManifestCommitPlan::HoldAlternative - } -} - -fn critical_single_candidate_cohort<'a>( - cohorts: &'a [HlsAlternativeOriginCohort], - observations: &[HlsManifestCandidateObservation], -) -> Option<&'a HlsAlternativeOriginCohort> { - let [cohort] = cohorts else { - return None; - }; - if cohort.successful_samples != 1 || cohort.evidence != HlsCrossHostAcceptanceEvidence::Insufficient { - return None; - } - observations.iter().find(|candidate| candidate.candidate_index == cohort.best_candidate_index).filter( - |candidate| { - candidate.switch_segment_readiness.can_be_staged() - && candidate.emergency_evidence.requires_staged_verification() - }, - )?; - Some(cohort) -} - -fn pinned_commit(candidate: &HlsManifestCandidateObservation) -> HlsManifestCommitPlan { - HlsManifestCommitPlan::Commit { candidate_index: candidate.candidate_index, kind: HlsManifestCommitKind::Pinned } -} - -fn alternative_plan( - observations: &[HlsManifestCandidateObservation], - candidate_index: usize, - kind: HlsManifestCommitKind, -) -> HlsManifestCommitPlan { - match observations - .iter() - .find(|candidate| candidate.candidate_index == candidate_index) - .map(|candidate| candidate.switch_segment_readiness) - { - Some(HlsSwitchSegmentReadiness::RequiresStaging) => { - HlsManifestCommitPlan::StageAlternative { candidate_index, kind } - } - Some(HlsSwitchSegmentReadiness::Unavailable) | None => HlsManifestCommitPlan::RejectAll, - } -} - -fn best_pinned_candidates( - observations: &[HlsManifestCandidateObservation], -) -> (Option<&HlsManifestCandidateObservation>, Option<&HlsManifestCandidateObservation>) { - let mut progressed = None; - let mut unchanged = None; - for candidate in observations.iter().filter(|candidate| { - candidate.host_relation == HlsCandidateHostRelation::PinnedHost - && candidate.resource_timeline_evidence.permits_acceptance() - }) { - match candidate.local_sequence_relation { - Some( - HlsHostLocalSequenceRelation::Next - | HlsHostLocalSequenceRelation::PlausibleForward - | HlsHostLocalSequenceRelation::RolloverCandidate - | HlsHostLocalSequenceRelation::Rebase, - ) => { - if progressed.is_none_or(|current| pinned_order(candidate, current).is_lt()) { - progressed = Some(candidate); - } - } - Some(HlsHostLocalSequenceRelation::Same | HlsHostLocalSequenceRelation::NoBaseline) => { - if unchanged.is_none_or(|current| pinned_order(candidate, current).is_lt()) { - unchanged = Some(candidate); - } - } - Some(HlsHostLocalSequenceRelation::Backward) | None => {} - } - } - (progressed, unchanged) -} - -fn pinned_order(left: &HlsManifestCandidateObservation, right: &HlsManifestCandidateObservation) -> std::cmp::Ordering { - pinned_priority(left.local_sequence_relation) - .cmp(&pinned_priority(right.local_sequence_relation)) - .then_with(|| left.manifest_fetch_elapsed_ms.cmp(&right.manifest_fetch_elapsed_ms)) - .then_with(|| left.candidate_index.cmp(&right.candidate_index)) -} - -fn pinned_priority(relation: Option) -> u8 { - match relation { - Some(HlsHostLocalSequenceRelation::Next) => 0, - Some(HlsHostLocalSequenceRelation::PlausibleForward | HlsHostLocalSequenceRelation::Rebase) => 1, - Some(HlsHostLocalSequenceRelation::RolloverCandidate) => 2, - Some(HlsHostLocalSequenceRelation::Same | HlsHostLocalSequenceRelation::NoBaseline) => 3, - Some(HlsHostLocalSequenceRelation::Backward) | None => u8::MAX, - } -} - -fn has_stageable_alternative(observations: &[HlsManifestCandidateObservation]) -> bool { - observations.iter().any(|candidate| { - candidate.host_relation == HlsCandidateHostRelation::OtherHost - && candidate.resource_timeline_evidence.permits_acceptance() - && candidate.switch_segment_readiness.can_be_staged() - }) -} - -pub fn classify_host_local_sequence( - previous_highwater: Option, - candidate_highwater: Option, - forward_window: u64, - rebase_allowed: bool, -) -> Option { - let candidate = candidate_highwater?; - if rebase_allowed { - return Some(HlsHostLocalSequenceRelation::Rebase); - } - let Some(previous) = previous_highwater else { - return Some(HlsHostLocalSequenceRelation::NoBaseline); - }; - if candidate == previous { - return Some(HlsHostLocalSequenceRelation::Same); - } - if previous.checked_add(1) == Some(candidate) { - return Some(HlsHostLocalSequenceRelation::Next); - } - if candidate > previous { - return Some(if candidate.saturating_sub(previous) <= forward_window.max(1) { - HlsHostLocalSequenceRelation::PlausibleForward - } else { - HlsHostLocalSequenceRelation::Backward - }); - } - Some(if candidate <= forward_window.max(1) { - HlsHostLocalSequenceRelation::RolloverCandidate - } else { - HlsHostLocalSequenceRelation::Backward - }) -} - -pub fn alternative_cohorts(observations: &[HlsManifestCandidateObservation]) -> Vec { - let pinned = observations - .iter() - .filter(|candidate| { - candidate.host_relation == HlsCandidateHostRelation::PinnedHost - && candidate.resource_timeline_evidence.permits_acceptance() - }) - .collect::>(); - let mut alternatives = observations - .iter() - .filter(|candidate| { - candidate.host_relation == HlsCandidateHostRelation::OtherHost - && candidate.resource_timeline_evidence.permits_acceptance() - && candidate.switch_segment_readiness.can_be_staged() - && candidate.effective_host.is_some() - }) - .collect::>(); - alternatives.sort_by(|left, right| { - left.effective_host.cmp(&right.effective_host).then_with(|| left.candidate_index.cmp(&right.candidate_index)) - }); - let mut groups = Vec::>::new(); - for observation in alternatives { - if let Some(group) = groups.iter_mut().find(|group| { - group.iter().all(|sample| { - sample.effective_host == observation.effective_host - && same_host_timeline_compatible(sample, observation) - }) - }) { - if group.len() < HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT { - group.push(observation); - } - } else if groups.len() < HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT { - groups.push(vec![observation]); - } - } - - let mut cohorts = - groups.into_iter().filter_map(|samples| build_alternative_cohort(&samples, &pinned)).collect::>(); - cohorts.sort_by_key(|cohort| { - (evidence_priority(cohort.evidence), std::cmp::Reverse(cohort.successful_samples), cohort.best_candidate_index) - }); - cohorts -} - -pub fn manifest_acceptance_landscape( - observations: &[HlsManifestCandidateObservation], -) -> HlsManifestAcceptanceLandscape { - let pinned_state = pinned_observation_state(observations); - let alternatives = alternative_cohorts(observations) - .into_iter() - .take(HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT) - .map(|cohort| (cohort.identity, cohort.window)) - .collect(); - HlsManifestAcceptanceLandscape { pinned_state, alternatives } -} - -pub fn classify_reduced_retry_landscape( - previous: &HlsManifestAcceptanceLandscape, - observations: &[HlsManifestCandidateObservation], -) -> HlsReducedRetryLandscapeChange { - if observations.is_empty() { - return HlsReducedRetryLandscapeChange::Unchanged; - } - let current_pinned = pinned_observation_state(observations); - if current_pinned != HlsPinnedOriginObservationState::Missing && current_pinned != previous.pinned_state { - return HlsReducedRetryLandscapeChange::PinnedStateChanged; - } - if observations.iter().any(|candidate| candidate.host_relation == HlsCandidateHostRelation::Unknown) { - return HlsReducedRetryLandscapeChange::TimelineConflict; - } - for cohort in alternative_cohorts(observations) { - let Some((_, previous_window)) = - previous.alternatives.iter().find(|(identity, _)| identity == &cohort.identity) - else { - return HlsReducedRetryLandscapeChange::NewCohort; - }; - if !host_local_windows_compatible(previous_window, &cohort.window) { - return HlsReducedRetryLandscapeChange::TimelineConflict; - } - } - HlsReducedRetryLandscapeChange::Unchanged -} - -fn pinned_observation_state(observations: &[HlsManifestCandidateObservation]) -> HlsPinnedOriginObservationState { - if observations.iter().any(|candidate| { - candidate.host_relation == HlsCandidateHostRelation::PinnedHost - && !candidate.resource_timeline_evidence.permits_acceptance() - }) { - return HlsPinnedOriginObservationState::Rejected; - } - let relations = observations - .iter() - .filter(|candidate| candidate.host_relation == HlsCandidateHostRelation::PinnedHost) - .filter_map(|candidate| candidate.local_sequence_relation); - let mut state = HlsPinnedOriginObservationState::Missing; - for relation in relations { - match relation { - HlsHostLocalSequenceRelation::Next - | HlsHostLocalSequenceRelation::PlausibleForward - | HlsHostLocalSequenceRelation::RolloverCandidate - | HlsHostLocalSequenceRelation::Rebase => return HlsPinnedOriginObservationState::Progressed, - HlsHostLocalSequenceRelation::Same | HlsHostLocalSequenceRelation::NoBaseline => { - state = HlsPinnedOriginObservationState::Unchanged; - } - HlsHostLocalSequenceRelation::Backward => { - if state == HlsPinnedOriginObservationState::Missing { - state = HlsPinnedOriginObservationState::Rejected; - } - } - } - } - state -} - -pub fn alternative_cohorts_with_history( - observations: &[HlsManifestCandidateObservation], - previous: Option<&HlsAlternativeOriginCohort>, - current_burst_is_full_plan: bool, -) -> Vec { - let mut cohorts = alternative_cohorts(observations); - if current_burst_is_full_plan { - for cohort in &mut cohorts { - merge_full_burst_cohort_history(cohort, previous); - } - return cohorts; - } - // A reduced follow-up may recover the pinned host, but it cannot spend - // historical full-burst evidence on a new cross-host staging attempt. - Vec::new() -} - -pub fn held_alternative_after_burst( - observations: &[HlsManifestCandidateObservation], - previous: Option<&HlsAlternativeOriginCohort>, - current_burst_is_full_plan: bool, -) -> Option { - if current_burst_is_full_plan { - return alternative_cohorts_with_history(observations, previous, true).into_iter().next(); - } - // Cheap failures and contradictions do not erase the last completed - // configured burst. A later episode must run another full plan before the - // history can become consecutive acceptance evidence. - let previous = previous?; - let mut matching = - alternative_cohorts(observations).into_iter().find(|cohort| same_alternative_cohort(cohort, previous)); - if let Some(cohort) = matching.as_mut() { - cohort.successful_samples = cohort.successful_samples.max(previous.successful_samples); - cohort.total_samples = cohort - .total_samples - .saturating_add(previous.total_samples) - .min(u16::try_from(HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT).unwrap_or(u16::MAX)); - cohort.consecutive_confirmed_full_bursts = previous.consecutive_confirmed_full_bursts; - if evidence_priority(previous.evidence) < evidence_priority(cohort.evidence) { - cohort.evidence = previous.evidence; - } - } - matching.or_else(|| Some(previous.clone())) -} - -fn merge_full_burst_cohort_history( - current: &mut HlsAlternativeOriginCohort, - previous: Option<&HlsAlternativeOriginCohort>, -) { - current.consecutive_confirmed_full_bursts = 1; - let Some(previous) = previous.filter(|previous| same_alternative_cohort(current, previous)) else { - return; - }; - current.total_samples = current - .total_samples - .saturating_add(previous.total_samples) - .min(u16::try_from(HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT).unwrap_or(u16::MAX)); - current.consecutive_confirmed_full_bursts = previous.consecutive_confirmed_full_bursts.max(1).saturating_add(1); - if evidence_priority(previous.evidence) < evidence_priority(current.evidence) { - current.evidence = previous.evidence; - } -} - -fn same_alternative_cohort(left: &HlsAlternativeOriginCohort, right: &HlsAlternativeOriginCohort) -> bool { - left.identity == right.identity && host_local_windows_compatible(&left.window, &right.window) -} - -fn build_alternative_cohort( - samples: &[&HlsManifestCandidateObservation], - pinned: &[&HlsManifestCandidateObservation], -) -> Option { - let first = samples.first()?; - let successful_samples = u16::try_from(samples.len()).unwrap_or(u16::MAX); - let anchored_overlap = samples - .iter() - .flat_map(|sample| pinned.iter().map(move |baseline| strong_anchor_overlap(sample, baseline))) - .max() - .unwrap_or_default(); - let externally_anchored_overlap = samples - .iter() - .filter_map(|sample| match sample.evidence { - HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { overlapping_segments } => Some(overlapping_segments), - HlsCrossHostAcceptanceEvidence::Insufficient - | HlsCrossHostAcceptanceEvidence::BurstConsensusNewEpoch { .. } => None, - }) - .max() - .unwrap_or_default(); - let overlapping_segments = anchored_overlap.max(externally_anchored_overlap); - let evidence = if overlapping_segments > 0 { - HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { overlapping_segments } - } else if successful_samples >= 2 { - HlsCrossHostAcceptanceEvidence::BurstConsensusNewEpoch { successful_samples } - } else { - HlsCrossHostAcceptanceEvidence::Insufficient - }; - let best = samples.iter().min_by_key(|sample| (sample.manifest_fetch_elapsed_ms, sample.candidate_index))?; - let best_candidate_index = best.candidate_index; - let effective_host = first.effective_host.clone()?; - Some(HlsAlternativeOriginCohort { - identity: HlsAlternativeOriginCohortIdentity { - effective_host, - technical_signature: HlsManifestTechnicalSignature::from_fingerprint(&first.timeline_fingerprint), - }, - window: HlsAlternativeOriginWindow { - host_local_media_sequence: best.host_local_media_sequence, - host_local_highwater: best.host_local_highwater, - fingerprint: best.timeline_fingerprint.clone(), - }, - successful_samples, - total_samples: successful_samples, - consecutive_confirmed_full_bursts: 0, - evidence, - best_candidate_index, - }) -} - -fn evidence_priority(evidence: HlsCrossHostAcceptanceEvidence) -> u8 { - match evidence { - HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { .. } => 0, - HlsCrossHostAcceptanceEvidence::BurstConsensusNewEpoch { .. } => 1, - HlsCrossHostAcceptanceEvidence::Insufficient => 2, - } -} - -fn same_host_timeline_compatible( - left: &HlsManifestCandidateObservation, - right: &HlsManifestCandidateObservation, -) -> bool { - let left_window = HlsAlternativeOriginWindow { - host_local_media_sequence: left.host_local_media_sequence, - host_local_highwater: left.host_local_highwater, - fingerprint: left.timeline_fingerprint.clone(), - }; - let right_window = HlsAlternativeOriginWindow { - host_local_media_sequence: right.host_local_media_sequence, - host_local_highwater: right.host_local_highwater, - fingerprint: right.timeline_fingerprint.clone(), - }; - host_local_windows_compatible(&left_window, &right_window) -} - -fn host_local_windows_compatible(left: &HlsAlternativeOriginWindow, right: &HlsAlternativeOriginWindow) -> bool { - let left_fingerprint = &left.fingerprint; - let right_fingerprint = &right.fingerprint; - if HlsManifestTechnicalSignature::from_fingerprint(left_fingerprint) - != HlsManifestTechnicalSignature::from_fingerprint(right_fingerprint) - || left_fingerprint.segment_samples.is_empty() - || right_fingerprint.segment_samples.is_empty() - { - return false; - } - - let Some(left_highwater) = left.host_local_highwater else { - return false; - }; - let Some(right_highwater) = right.host_local_highwater else { - return false; - }; - let overlap_start = left.host_local_media_sequence.max(right.host_local_media_sequence); - let overlap_end = left_highwater.min(right_highwater); - if overlap_start <= overlap_end { - return (overlap_start..=overlap_end).all(|sequence| { - segment_in_window(left, sequence) - .zip(segment_in_window(right, sequence)) - .is_some_and(|(left, right)| segment_shape_matches(left, right)) - }); - } - - // Adjacent local windows are monotonic only when their non-resource shape - // and technical signatures agree. Origin sequence is used solely inside - // this already host-local cohort. - let gap = if left_highwater < right.host_local_media_sequence { - right.host_local_media_sequence.saturating_sub(left_highwater) - } else { - left.host_local_media_sequence.saturating_sub(right_highwater) - }; - gap <= 1 - && left_fingerprint.segment_count == right_fingerprint.segment_count - && left_fingerprint.duration_pattern_hash == right_fingerprint.duration_pattern_hash - && left_fingerprint.discontinuity_pattern_hash == right_fingerprint.discontinuity_pattern_hash -} - -fn segment_in_window(window: &HlsAlternativeOriginWindow, sequence: u64) -> Option<&HlsManifestSegmentFingerprint> { - let offset = sequence.checked_sub(window.host_local_media_sequence)?; - window.fingerprint.segment_samples.get(usize::try_from(offset).ok()?) -} - -fn strong_anchor_overlap( - alternative: &HlsManifestCandidateObservation, - pinned: &HlsManifestCandidateObservation, -) -> u16 { - if alternative.timeline_fingerprint.map_and_encryption_hash != pinned.timeline_fingerprint.map_and_encryption_hash - || alternative.timeline_fingerprint.container_signature_hash - != pinned.timeline_fingerprint.container_signature_hash - { - return 0; - } - let mut longest_pdt_run = 0_usize; - let alternative_segments = &alternative.timeline_fingerprint.segment_samples; - let pinned_segments = &pinned.timeline_fingerprint.segment_samples; - for (alternative_index, alternative_segment) in alternative_segments.iter().enumerate() { - for (pinned_index, pinned_segment) in pinned_segments.iter().enumerate() { - if !pdt_intervals_overlap(alternative_segment, pinned_segment) { - continue; - } - let mut run = 0_usize; - while let (Some(alternative_sample), Some(pinned_sample)) = ( - alternative_segments.get(alternative_index.saturating_add(run)), - pinned_segments.get(pinned_index.saturating_add(run)), - ) { - if alternative_sample.duration_ms != pinned_sample.duration_ms - || alternative_sample.discontinuity_before != pinned_sample.discontinuity_before - || !pdt_intervals_overlap(alternative_sample, pinned_sample) - { - break; - } - run = run.saturating_add(1); - } - longest_pdt_run = longest_pdt_run.max(run); - } - } - if longest_pdt_run >= 2 { - u16::try_from(longest_pdt_run).unwrap_or(u16::MAX) - } else { - 0 - } -} - -fn pdt_intervals_overlap(left: &HlsManifestSegmentFingerprint, right: &HlsManifestSegmentFingerprint) -> bool { - if left.duration_ms != right.duration_ms { - return false; - } - let (Some(left_start), Some(right_start)) = (left.program_date_time_ms, right.program_date_time_ms) else { - return false; - }; - let left_duration = i64::try_from(left.duration_ms).unwrap_or(i64::MAX); - let right_duration = i64::try_from(right.duration_ms).unwrap_or(i64::MAX); - let left_end = left_start.saturating_add(left_duration); - let right_end = right_start.saturating_add(right_duration); - left_start < right_end && right_start < left_end -} - -fn segment_shape_matches(left: &HlsManifestSegmentFingerprint, right: &HlsManifestSegmentFingerprint) -> bool { - left.duration_ms == right.duration_ms - && left.discontinuity_before == right.discontinuity_before - && match (left.normalized_resource_identity, right.normalized_resource_identity) { - (Some(left), Some(right)) => left.matches(right), - (None, None) => true, - (Some(_), None) | (None, Some(_)) => false, - } -} - #[cfg(test)] -mod tests { - use super::*; - use crate::recovery_timing::{ - HlsAcceptanceEpisodeTimingInput, HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsRecoveryBurstWorkload, - HlsRecoveryMapWorkload, HlsRecoverySegmentWorkload, HlsRecoveryTimingPolicy, HlsTerminalMediaPreparationState, - HlsTransitionMarginMs, - }; - - fn episode_timing(started_at_ms: u64, burst_plan: HlsManifestRecoveryBurstPlan) -> HlsAcceptanceEpisodeTiming { - HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput { - started_at_ms, - burst_plan, - target_duration_ms: 4_000, - transition_margin: HlsTransitionMarginMs::from_millis(1_000), - workload: HlsRecoveryWorkloadEnvelope::acceptance_policy().ceiling(), - observed_latency: HlsObservedRecoveryLatency::default(), - required_terminal_media_key: None, - terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, - policy: HlsRecoveryTimingPolicy::new( - HlsOperationTimeoutMs::from_millis(1_000), - HlsOperationTimeoutMs::from_millis(2_000), - HlsRecoveryEtaMs::from_millis(300), - HlsRecoveryEtaMs::from_millis(400), - ), - }) - } - - fn segment(marker: u8, pdt_ms: Option) -> HlsManifestSegmentFingerprint { - HlsManifestSegmentFingerprint { - duration_ms: 4_000, - discontinuity_before: false, - program_date_time_ms: pdt_ms, - normalized_resource_identity: Some(HlsMediaResourceIdentity::for_test(marker)), - } - } - - fn fingerprint(marker: u8) -> HlsManifestTimelineFingerprint { - HlsManifestTimelineFingerprint { - segment_count: 3, - first_program_date_time_ms: None, - last_program_date_time_ms: None, - duration_pattern_hash: [marker; 32], - discontinuity_pattern_hash: [0; 32], - normalized_resource_pattern_hash: Some([marker; 32]), - map_and_encryption_hash: [0; 32], - container_signature_hash: [1; 32], - segment_samples: vec![ - segment(marker, None), - segment(marker.saturating_add(1), None), - segment(marker.saturating_add(2), None), - ], - } - } - - fn observation(index: usize, host: &str, relation: HlsCandidateHostRelation) -> HlsManifestCandidateObservation { - HlsManifestCandidateObservation { - candidate_index: index, - candidate_slot: index / 2, - effective_host: Some(host.to_string()), - host_relation: relation, - host_local_media_sequence: 5, - host_local_highwater: Some(7), - local_sequence_relation: (relation == HlsCandidateHostRelation::PinnedHost) - .then_some(HlsHostLocalSequenceRelation::Next), - resource_timeline_evidence: HlsResourceTimelineEvidence::Eligible, - timeline_fingerprint: fingerprint(1), - manifest_fetch_elapsed_ms: 10, - switch_segment_readiness: HlsSwitchSegmentReadiness::RequiresStaging, - committed_content_anchor: HlsCommittedContentAnchorEvidence::Unavailable, - emergency_evidence: HlsEmergencyAcceptanceEvidence::INCOMPATIBLE, - evidence: HlsCrossHostAcceptanceEvidence::Insufficient, - } - } - - fn evaluate( - observations: &[HlsManifestCandidateObservation], - trigger: HlsManifestAcceptanceTrigger, - ) -> HlsManifestCommitPlan { - evaluate_manifest_acceptance(HlsManifestAcceptanceInput { - full_burst_completed: true, - current_burst_is_full_plan: true, - trigger, - previous_alternative: None, - observations, - }) - } - - #[test] - fn forward_media_sequence_does_not_override_published_resource_replay() { - let mut replay = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); - replay.local_sequence_relation = Some(HlsHostLocalSequenceRelation::PlausibleForward); - replay.resource_timeline_evidence = HlsResourceTimelineEvidence::ReplayOnly; - - assert_eq!( - evaluate(&[replay], HlsManifestAcceptanceTrigger::RecoveryRequired), - HlsManifestCommitPlan::RejectAll - ); - } - - fn mark_emergency_verification_eligible(candidate: &mut HlsManifestCandidateObservation) { - candidate.emergency_evidence = HlsEmergencyAcceptanceEvidence { - live_handoff: HlsEmergencyLiveHandoffCompatibility::RequiresStagedTrackVerification, - terminal_alternative: HlsTerminalAlternativeCompatibility::RequiresStagedComparison, - }; - } - - fn sliding_observation( - index: usize, - host: &str, - media_sequence: u64, - markers: [u8; 3], - ) -> HlsManifestCandidateObservation { - let mut candidate = observation(index, host, HlsCandidateHostRelation::OtherHost); - candidate.host_local_media_sequence = media_sequence; - candidate.host_local_highwater = Some(media_sequence.saturating_add(2)); - candidate.timeline_fingerprint.segment_samples = markers.map(|marker| segment(marker, None)).to_vec(); - candidate - } - - #[test] - fn first_episode_uses_entire_configured_beast_plan() { - let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); - let episode = HlsManifestAcceptanceEpisode::new( - HlsManifestAcceptanceGeneration(1), - 0, - plan, - HlsManifestAcceptanceTrigger::Observe, - &episode_timing(0, plan), - ); - - assert_eq!(episode.required_candidates(), plan.total_candidates()); - assert_eq!(episode.burst_max_stagger_ms(), 500); - assert!(!episode.full_burst_completed); - } - - #[test] - fn acceptance_trigger_semantics_are_explicit() { - assert!(!HlsManifestAcceptanceTrigger::None.starts_episode()); - assert!(HlsManifestAcceptanceTrigger::Observe.starts_episode()); - assert!(!HlsManifestAcceptanceTrigger::Observe.recovery_required()); - assert!(HlsManifestAcceptanceTrigger::RecoveryRequired.recovery_required()); - assert!(HlsManifestAcceptanceTrigger::Critical.recovery_required()); - } - - #[test] - fn held_episode_state_is_bounded_and_cleared_on_completion() { - let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); - let mut episode = HlsManifestAcceptanceEpisode::new( - HlsManifestAcceptanceGeneration(1), - 0, - plan, - HlsManifestAcceptanceTrigger::Observe, - &episode_timing(0, plan), - ); - let mut cohort = alternative_cohorts(&[ - observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), - observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), - ]) - .remove(0); - cohort.successful_samples = u16::MAX; - cohort.total_samples = u16::MAX; - - episode.record_full_burst(); - episode.hold_after_uncommitted_burst(Some(cohort), Some(123)); - assert_eq!(episode.held_alternative.as_ref().map(|held| held.successful_samples), Some(32)); - assert_eq!(episode.next_retry_at_ms, Some(123)); - episode.complete(); - assert_eq!(episode.held_alternative, None); - assert_eq!(episode.next_retry_at_ms, None); - } - - #[test] - fn completing_episode_clears_deterministic_conflict_receipt() { - let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); - let mut episode = HlsManifestAcceptanceEpisode::new( - HlsManifestAcceptanceGeneration(1), - 0, - plan, - HlsManifestAcceptanceTrigger::Observe, - &episode_timing(0, plan), - ); - let resource_key = HlsMediaResourceIdentity::for_test(7).semantic_key(); - episode.record_deterministic_conflict(HlsDeterministicConflictReceipt { - conflict: HlsDeterministicTimelineConflict { - previous_proxy_tail: Some(2), - existing_proxy_seq: 0, - candidate_position: 1, - candidate_origin_seq: 491, - resource_key, - decision: super::super::timeline::HlsResourceReplayDecision::RejectContradictoryOrder, - candidate_fingerprint: super::super::deterministic_conflict::HlsDeterministicConflictFingerprint { - segment_count: 1, - first_program_date_time_ms: None, - last_program_date_time_ms: None, - duration_pattern_hash: [1; 32], - discontinuity_pattern_hash: [2; 32], - semantic_resource_pattern_hash: Some(resource_key.bytes()), - map_and_encryption_hash: [3; 32], - container_signature_hash: [4; 32], - segment_samples: Vec::new(), - }, - }, - origin_progress_generation: 1, - published_resource_history_generation: 1, - pinned_host_generation: 1, - }); - - assert!(episode.deterministic_conflict_receipt().is_some()); - episode.complete(); - assert!(episode.deterministic_conflict_receipt().is_none()); - } - - #[test] - fn uncommitted_completed_burst_returns_episode_to_holding() { - let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); - let mut episode = HlsManifestAcceptanceEpisode::new( - HlsManifestAcceptanceGeneration(1), - 0, - plan, - HlsManifestAcceptanceTrigger::Observe, - &episode_timing(0, plan), - ); - - episode.record_full_burst(); - episode.state = HlsManifestAcceptanceState::StagingSwitchSegment; - episode.hold_after_uncommitted_burst(None, Some(123)); - - assert_eq!(episode.state, HlsManifestAcceptanceState::Holding); - assert!(episode.full_burst_completed); - assert_eq!(episode.full_bursts_completed, 1); - assert_eq!(episode.next_retry_at_ms, Some(123)); - } - - #[test] - fn hls_recovery_timing_status_uses_frozen_episode_deadline_and_normalizes_outcomes() { - let plan = shared::model::HlsManifestRecoveryBurstLevel::Friendly.plan(); - let generation = HlsManifestAcceptanceGeneration(9); - let timing = episode_timing(100, plan); - let deadline_ms = timing.acceptance_deadline.as_millis_since_epoch(); - let mut episode = HlsManifestAcceptanceEpisode::new( - generation, - 100, - plan, - HlsManifestAcceptanceTrigger::RecoveryRequired, - &timing, - ); - assert_eq!( - manifest_acceptance_episode_status(Some(&episode), generation, deadline_ms.saturating_sub(1)), - HlsManifestAcceptanceEpisodeStatus::InFlight { generation } - ); - episode.record_full_burst(); - episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate); - assert_eq!( - manifest_acceptance_episode_status(Some(&episode), generation, deadline_ms.saturating_sub(1)), - HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { - generation, - reason: HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate, - } - ); - assert_eq!( - manifest_acceptance_episode_status( - Some(&episode), - HlsManifestAcceptanceGeneration(10), - deadline_ms.saturating_sub(1), - ), - HlsManifestAcceptanceEpisodeStatus::Superseded { - generation, - current_generation: HlsManifestAcceptanceGeneration(10), - } - ); - assert_eq!( - manifest_acceptance_episode_status(Some(&episode), generation, deadline_ms), - HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { - generation, - reason: HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate, - } - ); - - let pending_episode = HlsManifestAcceptanceEpisode::new( - generation, - 100, - plan, - HlsManifestAcceptanceTrigger::RecoveryRequired, - &timing, - ); - assert_eq!( - manifest_acceptance_episode_status(Some(&pending_episode), generation, deadline_ms), - HlsManifestAcceptanceEpisodeStatus::Expired { generation } - ); - } - - #[test] - fn hls_recovery_timing_episode_keeps_construction_snapshot_immutable() { - let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); - let frozen_timing = episode_timing(100, plan); - let episode = HlsManifestAcceptanceEpisode::new( - HlsManifestAcceptanceGeneration(3), - 100, - plan, - HlsManifestAcceptanceTrigger::Critical, - &frozen_timing, - ); - let later_timing = episode_timing(10_000, plan); - - assert_ne!(later_timing.acceptance_deadline, frozen_timing.acceptance_deadline); - assert_eq!(episode.timing(), frozen_timing); - assert_eq!(episode.generation, HlsManifestAcceptanceGeneration(3)); - assert_eq!(episode.started_at_ms, 100); - assert_eq!(episode.burst_plan, plan); - assert_eq!(episode.trigger(), HlsManifestAcceptanceTrigger::Critical); - } - - #[test] - fn hls_recovery_timing_candidate_binding_and_staging_reduce_eta_without_moving_deadline() { - let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); - let generation = HlsManifestAcceptanceGeneration(4); - let mut episode = HlsManifestAcceptanceEpisode::new( - generation, - 100, - plan, - HlsManifestAcceptanceTrigger::RecoveryRequired, - &episode_timing(100, plan), - ); - let frozen_deadline = episode.timing().acceptance_deadline; - let initial_eta = episode.remaining_recovery_eta(generation).map(HlsRecoveryEtaMs::as_millis); - - episode.record_full_burst(); - let completed_burst_eta = episode.remaining_recovery_eta(generation).map(HlsRecoveryEtaMs::as_millis); - episode.state = HlsManifestAcceptanceState::StagingSwitchSegment; - let identity = HlsManifestRecoveryCandidateIdentity::from_candidate( - 2, - Some("candidate.example.com"), - "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n", - ); - let bound_workload = HlsRecoveryWorkload { - burst: HlsRecoveryBurstWorkload::FullBurstPending, - segment: HlsRecoverySegmentWorkload::ClearSegmentFetch, - map: HlsRecoveryMapWorkload::Fetch, - }; - assert_eq!(episode.select_candidate(generation, identity), HlsRecoveryWorkloadBindingUpdate::Applied); - assert_eq!( - episode.bind_selected_candidate(generation, identity, bound_workload), - HlsRecoveryWorkloadBindingUpdate::Applied - ); - assert_eq!( - episode.remaining_recovery_workload(generation).map(|workload| workload.map), - Some(HlsRecoveryMapWorkload::Fetch) - ); - let bound_eta = episode.remaining_recovery_eta(generation).map(HlsRecoveryEtaMs::as_millis); - episode.state = HlsManifestAcceptanceState::Committing; - let committing_eta = episode.remaining_recovery_eta(generation).map(HlsRecoveryEtaMs::as_millis); - episode.state = HlsManifestAcceptanceState::StagingSwitchSegment; - assert_eq!( - episode.advance_bound_candidate( - generation, - identity, - HlsRecoveryWorkload { - burst: HlsRecoveryBurstWorkload::FullBurstCompleted, - segment: HlsRecoverySegmentWorkload::SegmentStagedWithDependenciesReady, - map: HlsRecoveryMapWorkload::Ready, - }, - ), - HlsRecoveryWorkloadBindingUpdate::Applied - ); - let staged_eta = episode.remaining_recovery_eta(generation).map(HlsRecoveryEtaMs::as_millis); - - assert!(initial_eta > completed_burst_eta); - assert!(completed_burst_eta > bound_eta); - assert_eq!(committing_eta, bound_eta); - assert!(bound_eta > staged_eta); - assert_eq!(episode.timing().acceptance_deadline, frozen_deadline); - assert_eq!( - episode - .estimated_recovery_completion_at(generation, 5_000) - .map(HlsEstimatedRecoveryCompletionAtMs::as_millis_since_epoch), - staged_eta.map(|eta| 5_000_u64.saturating_add(eta)) - ); - assert_eq!(episode.remaining_recovery_workload(HlsManifestAcceptanceGeneration(5)), None); - - episode.complete(); - assert_eq!(episode.remaining_recovery_workload(generation), None); - assert_eq!(episode.estimated_recovery_completion_at(generation, 5_000), None); - } - - #[test] - fn hls_recovery_timing_candidate_stale_generation_and_wrong_identity_cannot_update_binding() { - let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); - let generation = HlsManifestAcceptanceGeneration(9); - let mut episode = HlsManifestAcceptanceEpisode::new( - generation, - 100, - plan, - HlsManifestAcceptanceTrigger::RecoveryRequired, - &episode_timing(100, plan), - ); - episode.record_full_burst(); - episode.state = HlsManifestAcceptanceState::StagingSwitchSegment; - let selected = HlsManifestRecoveryCandidateIdentity::from_candidate(1, Some("a.example"), "body"); - let other = HlsManifestRecoveryCandidateIdentity::from_candidate(2, Some("a.example"), "body"); - let candidate_workload = HlsRecoveryWorkload { - burst: HlsRecoveryBurstWorkload::FullBurstPending, - segment: HlsRecoverySegmentWorkload::ClearSegmentFetch, - map: HlsRecoveryMapWorkload::Fetch, - }; - - assert_eq!( - episode.select_candidate(HlsManifestAcceptanceGeneration(8), selected), - HlsRecoveryWorkloadBindingUpdate::StaleGeneration - ); - assert_eq!(episode.select_candidate(generation, selected), HlsRecoveryWorkloadBindingUpdate::Applied); - assert_eq!( - episode.bind_selected_candidate(generation, other, candidate_workload), - HlsRecoveryWorkloadBindingUpdate::CandidateMismatch - ); - assert_eq!( - episode.bind_selected_candidate(generation, selected, candidate_workload), - HlsRecoveryWorkloadBindingUpdate::Applied - ); - let before = episode.remaining_recovery_workload(generation); - assert_eq!( - episode.advance_bound_candidate( - generation, - other, - HlsRecoveryWorkload { - burst: HlsRecoveryBurstWorkload::FullBurstCompleted, - segment: HlsRecoverySegmentWorkload::SegmentStagedWithDependenciesReady, - map: HlsRecoveryMapWorkload::Ready, - }, - ), - HlsRecoveryWorkloadBindingUpdate::CandidateMismatch - ); - assert_eq!( - episode.advance_bound_candidate( - HlsManifestAcceptanceGeneration(8), - selected, - HlsRecoveryWorkload { - burst: HlsRecoveryBurstWorkload::FullBurstCompleted, - segment: HlsRecoverySegmentWorkload::SegmentStagedWithDependenciesReady, - map: HlsRecoveryMapWorkload::Ready, - }, - ), - HlsRecoveryWorkloadBindingUpdate::StaleGeneration - ); - assert_eq!(episode.remaining_recovery_workload(generation), before); - } - - #[test] - fn hls_recovery_timing_candidate_same_host_selection_keeps_conservative_unknown_workload() { - let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); - let generation = HlsManifestAcceptanceGeneration(10); - let mut episode = HlsManifestAcceptanceEpisode::new( - generation, - 100, - plan, - HlsManifestAcceptanceTrigger::RecoveryRequired, - &episode_timing(100, plan), - ); - episode.record_full_burst(); - episode.state = HlsManifestAcceptanceState::Committing; - let selected = HlsManifestRecoveryCandidateIdentity::from_candidate( - 1, - Some("pinned.example"), - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:8\n", - ); - let conservative = episode.remaining_recovery_workload(generation); - - assert_eq!(episode.select_candidate(generation, selected), HlsRecoveryWorkloadBindingUpdate::Applied); - - assert_eq!(episode.selected_candidate_identity(), Some(selected)); - assert_eq!(episode.remaining_recovery_workload(generation), conservative); - } - - #[test] - fn hls_recovery_timing_candidate_identity_is_host_local_even_for_identical_manifest_bytes() { - let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:7\n"; - - let origin_a = HlsManifestRecoveryCandidateIdentity::from_candidate(0, Some("a.example"), body); - let origin_b = HlsManifestRecoveryCandidateIdentity::from_candidate(0, Some("b.example"), body); - - assert_ne!(origin_a, origin_b); - } - - #[test] - fn identical_media_sequence_on_other_host_has_no_local_continuity_relation() { - let candidate = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); - - assert_eq!(candidate.host_local_media_sequence, 5); - assert_eq!(candidate.local_sequence_relation, None); - } - - #[test] - fn progressed_pinned_candidate_wins_over_alternative() { - let observations = [ - observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), - observation(1, "origin-a", HlsCandidateHostRelation::PinnedHost), - ]; - - assert_eq!( - evaluate(&observations, HlsManifestAcceptanceTrigger::Critical), - HlsManifestCommitPlan::Commit { candidate_index: 1, kind: HlsManifestCommitKind::Pinned } - ); - } - - #[test] - fn unchanged_pinned_does_not_hide_consensus_under_recovery_pressure() { - let alternatives = [ - observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), - observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), - ]; - let mut pinned = observation(2, "origin-a", HlsCandidateHostRelation::PinnedHost); - pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Same); - pinned.timeline_fingerprint = fingerprint(40); - let observations = [alternatives[0].clone(), alternatives[1].clone(), pinned]; - - assert_eq!( - evaluate(&observations, HlsManifestAcceptanceTrigger::RecoveryRequired), - HlsManifestCommitPlan::StageAlternative { - candidate_index: 0, - kind: HlsManifestCommitKind::AlternativeAsNewEpoch, - } - ); - } - - #[test] - fn unchanged_pinned_wins_while_recovery_is_not_required() { - let mut pinned = observation(1, "origin-a", HlsCandidateHostRelation::PinnedHost); - pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Same); - let observations = [observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), pinned]; - - assert_eq!( - evaluate(&observations, HlsManifestAcceptanceTrigger::Observe), - HlsManifestCommitPlan::Commit { candidate_index: 1, kind: HlsManifestCommitKind::Pinned } - ); - } - - #[test] - fn alternative_cannot_commit_before_full_burst() { - let observations = [observation(0, "origin-b", HlsCandidateHostRelation::OtherHost)]; - - assert_eq!( - evaluate_manifest_acceptance(HlsManifestAcceptanceInput { - full_burst_completed: false, - current_burst_is_full_plan: false, - trigger: HlsManifestAcceptanceTrigger::Critical, - previous_alternative: None, - observations: &observations, - }), - HlsManifestCommitPlan::HoldAlternative - ); - } - - #[test] - fn configured_off_single_initial_candidate_uses_acceptance_pipeline() { - let mut initial = observation(0, "origin-a", HlsCandidateHostRelation::InitialBaseline); - initial.local_sequence_relation = Some(HlsHostLocalSequenceRelation::NoBaseline); - let observations = [initial]; - - assert_eq!( - evaluate_manifest_acceptance(HlsManifestAcceptanceInput { - full_burst_completed: false, - current_burst_is_full_plan: false, - trigger: HlsManifestAcceptanceTrigger::RecoveryRequired, - previous_alternative: None, - observations: &observations, - }), - HlsManifestCommitPlan::RejectAll - ); - assert_eq!( - evaluate(&observations, HlsManifestAcceptanceTrigger::RecoveryRequired), - HlsManifestCommitPlan::Commit { candidate_index: 0, kind: HlsManifestCommitKind::Pinned } - ); - } - - #[test] - fn observe_consensus_is_held_without_reserve_pressure() { - let observations = [ - observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), - observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), - ]; - - assert_eq!( - evaluate(&observations, HlsManifestAcceptanceTrigger::Observe), - HlsManifestCommitPlan::HoldAlternative - ); - } - - #[test] - fn recovery_required_consensus_may_stage_new_epoch() { - let observations = [ - observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), - observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), - ]; - - assert_eq!( - evaluate(&observations, HlsManifestAcceptanceTrigger::RecoveryRequired), - HlsManifestCommitPlan::StageAlternative { - candidate_index: 0, - kind: HlsManifestCommitKind::AlternativeAsNewEpoch, - } - ); - } - - #[test] - fn episode_trigger_remains_authoritative_after_execution_enters_recovery() { - let observations = [ - observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), - observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), - ]; - let plan = shared::model::HlsManifestRecoveryBurstLevel::Friendly.plan(); - let mut episode = HlsManifestAcceptanceEpisode::new( - HlsManifestAcceptanceGeneration(7), - 10, - plan, - HlsManifestAcceptanceTrigger::Observe, - &episode_timing(10, plan), - ); - episode.state = HlsManifestAcceptanceState::Collecting; - episode.record_full_burst(); - - assert_eq!( - evaluate(&observations, episode.trigger()), - HlsManifestCommitPlan::HoldAlternative, - "execution state must not upgrade Observe to reserve pressure" - ); - } - - #[test] - fn alternative_requires_ready_staging_before_commit() { - let mut first = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); - let mut second = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); - first.switch_segment_readiness = HlsSwitchSegmentReadiness::RequiresStaging; - second.switch_segment_readiness = HlsSwitchSegmentReadiness::RequiresStaging; - let observations = [first, second]; - - assert_eq!( - evaluate(&observations, HlsManifestAcceptanceTrigger::RecoveryRequired), - HlsManifestCommitPlan::StageAlternative { - candidate_index: 0, - kind: HlsManifestCommitKind::AlternativeAsNewEpoch, - } - ); - } - - #[test] - fn strong_pdt_anchor_qualifies_alternative_without_sequence_comparison() { - let pdt = 1_700_000_000_000; - let mut pinned = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); - pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Backward); - pinned.timeline_fingerprint.segment_samples[0].program_date_time_ms = Some(pdt); - pinned.timeline_fingerprint.segment_samples[1].program_date_time_ms = Some(pdt + 4_000); - let mut alternative = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); - alternative.host_local_media_sequence = 900; - alternative.host_local_highwater = Some(902); - alternative.timeline_fingerprint.segment_samples[0].program_date_time_ms = Some(pdt + 500); - alternative.timeline_fingerprint.segment_samples[1].program_date_time_ms = Some(pdt + 4_500); - for (index, segment) in alternative.timeline_fingerprint.segment_samples.iter_mut().enumerate() { - segment.normalized_resource_identity = - Some(HlsMediaResourceIdentity::for_test(u8::try_from(index).unwrap_or(u8::MAX).saturating_add(90))); - } - - assert_eq!( - evaluate(&[pinned, alternative], HlsManifestAcceptanceTrigger::RecoveryRequired), - HlsManifestCommitPlan::StageAlternative { - candidate_index: 1, - kind: HlsManifestCommitKind::AnchoredAlternative, - } - ); - } - - #[test] - fn one_accidental_pdt_overlap_is_not_a_strong_cross_host_anchor() { - let pdt = 1_700_000_000_000; - let mut pinned = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); - pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Backward); - pinned.timeline_fingerprint.segment_samples[0].program_date_time_ms = Some(pdt); - let mut alternative = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); - alternative.timeline_fingerprint.segment_samples[0].program_date_time_ms = Some(pdt + 500); - for (index, segment) in alternative.timeline_fingerprint.segment_samples.iter_mut().enumerate() { - segment.normalized_resource_identity = - Some(HlsMediaResourceIdentity::for_test(u8::try_from(index).unwrap_or(u8::MAX).saturating_add(90))); - } - - assert_eq!( - evaluate(&[pinned, alternative], HlsManifestAcceptanceTrigger::RecoveryRequired), - HlsManifestCommitPlan::HoldAlternative - ); - } - - #[test] - fn same_sequence_on_different_hosts_does_not_create_strong_anchor() { - let mut pinned = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); - pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Backward); - pinned.timeline_fingerprint = fingerprint(1); - let mut alternative = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); - alternative.timeline_fingerprint = fingerprint(20); - - assert_eq!( - evaluate(&[pinned, alternative], HlsManifestAcceptanceTrigger::RecoveryRequired), - HlsManifestCommitPlan::HoldAlternative - ); - } - - #[test] - fn same_normalized_path_from_different_queries_with_unknown_global_identity_is_not_a_strong_anchor() { - let mut pinned = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); - pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Backward); - let mut alternative = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); - // Equal normalized identities model equal paths after host and query-token removal. - // Without PDT or staged byte equality the global content identity remains unknown. - alternative.timeline_fingerprint.normalized_resource_pattern_hash = - pinned.timeline_fingerprint.normalized_resource_pattern_hash; - - assert_eq!( - evaluate(&[pinned, alternative], HlsManifestAcceptanceTrigger::RecoveryRequired), - HlsManifestCommitPlan::HoldAlternative - ); - } - - #[test] - fn normalized_path_only_requests_staged_byte_verification_and_never_direct_anchor_commit() { - let mut candidate = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); - candidate.committed_content_anchor = HlsCommittedContentAnchorEvidence::RequiresStagedByteVerification; - - assert_eq!( - evaluate(&[candidate], HlsManifestAcceptanceTrigger::Observe), - HlsManifestCommitPlan::StageAlternative { - candidate_index: 0, - kind: HlsManifestCommitKind::ContentVerifiedAlternative, - } - ); - } - - #[test] - fn different_local_shapes_do_not_form_a_consensus_cohort() { - let first = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); - let mut second = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); - second.timeline_fingerprint.segment_samples[1].duration_ms = 9_000; - - let cohorts = alternative_cohorts(&[first, second]); - - assert_eq!(cohorts.len(), 2); - assert!(cohorts.iter().all(|cohort| cohort.successful_samples == 1)); - } - - #[test] - fn reduced_follow_up_preserves_matching_full_burst_evidence_without_incrementing_it() { - let observations = [ - observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), - observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), - ]; - let full = alternative_cohorts_with_history(&observations, None, true).remove(0); - let follow_up = - held_alternative_after_burst(&observations[..1], Some(&full), false).expect("matching follow-up cohort"); - - assert_eq!(full.consecutive_confirmed_full_bursts, 1); - assert_eq!(follow_up.consecutive_confirmed_full_bursts, 1); - assert_eq!(follow_up.successful_samples, full.successful_samples); - assert!(follow_up.total_samples > full.total_samples); - } - - #[test] - fn matching_later_full_burst_advances_consecutive_evidence() { - let first_window = [sliding_observation(0, "origin-b", 5, [1, 2, 3])]; - let second_window = [sliding_observation(0, "origin-b", 6, [2, 3, 4])]; - let first = alternative_cohorts_with_history(&first_window, None, true).remove(0); - let second = alternative_cohorts_with_history(&second_window, Some(&first), true).remove(0); - - assert_eq!(second.consecutive_confirmed_full_bursts, 2); - assert_ne!(first.window.fingerprint, second.window.fingerprint); - } - - #[test] - fn reduced_retry_classifies_sliding_same_cohort_separately_from_new_or_conflicting_cohorts() { - let first = [sliding_observation(0, "origin-b", 5, [1, 2, 3])]; - let landscape = manifest_acceptance_landscape(&first); - let sliding = [sliding_observation(0, "origin-b", 6, [2, 3, 4])]; - let new_host = [sliding_observation(0, "origin-c", 6, [2, 3, 4])]; - let conflict = [sliding_observation(0, "origin-b", 6, [90, 91, 92])]; - - assert_eq!(classify_reduced_retry_landscape(&landscape, &sliding), HlsReducedRetryLandscapeChange::Unchanged); - assert_eq!(classify_reduced_retry_landscape(&landscape, &new_host), HlsReducedRetryLandscapeChange::NewCohort); - assert_eq!( - classify_reduced_retry_landscape(&landscape, &conflict), - HlsReducedRetryLandscapeChange::TimelineConflict - ); - } - - #[test] - fn two_matching_single_sample_full_bursts_may_stage_under_recovery_pressure() { - let first_window = [sliding_observation(0, "origin-b", 5, [1, 2, 3])]; - let second_window = [sliding_observation(0, "origin-b", 6, [2, 3, 4])]; - let first = alternative_cohorts_with_history(&first_window, None, true).remove(0); - - assert_eq!( - evaluate_manifest_acceptance(HlsManifestAcceptanceInput { - full_burst_completed: true, - current_burst_is_full_plan: true, - trigger: HlsManifestAcceptanceTrigger::RecoveryRequired, - previous_alternative: Some(&first), - observations: &second_window, - }), - HlsManifestCommitPlan::StageAlternative { - candidate_index: 0, - kind: HlsManifestCommitKind::AlternativeAsNewEpoch, - } - ); - } - - #[test] - fn conflicting_reduced_follow_up_cannot_reuse_previous_full_burst() { - let first_observations = [ - observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), - observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), - ]; - let first = alternative_cohorts_with_history(&first_observations, None, true).remove(0); - let mut conflict = observation(0, "origin-c", HlsCandidateHostRelation::OtherHost); - conflict.timeline_fingerprint = fingerprint(20); - - assert!(alternative_cohorts_with_history(&[conflict], Some(&first), false).is_empty()); - } - - #[test] - fn failed_reduced_follow_up_does_not_erase_previous_full_burst_evidence() { - let observations = [ - observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), - observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), - ]; - let full = held_alternative_after_burst(&observations, None, true).expect("full burst cohort"); - - assert_eq!(held_alternative_after_burst(&[], Some(&full), false), Some(full)); - } - - #[test] - fn critical_single_candidate_requires_typed_staged_verification() { - let mut candidate = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); - candidate.switch_segment_readiness = HlsSwitchSegmentReadiness::RequiresStaging; - mark_emergency_verification_eligible(&mut candidate); - let observations = [candidate.clone()]; - - assert_eq!( - evaluate(&observations, HlsManifestAcceptanceTrigger::Critical), - HlsManifestCommitPlan::StageAlternative { - candidate_index: 0, - kind: HlsManifestCommitKind::EmergencyAlternativeAsNewEpoch, - } - ); - assert_eq!( - evaluate(&observations, HlsManifestAcceptanceTrigger::RecoveryRequired), - HlsManifestCommitPlan::HoldAlternative - ); - - candidate.emergency_evidence.terminal_alternative = HlsTerminalAlternativeCompatibility::TerminalTailPreferred; - assert_eq!( - evaluate(&[candidate], HlsManifestAcceptanceTrigger::Critical), - HlsManifestCommitPlan::HoldAlternative - ); - } - - #[test] - fn critical_single_candidate_without_stageable_first_segment_is_rejected() { - let mut candidate = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); - mark_emergency_verification_eligible(&mut candidate); - candidate.switch_segment_readiness = HlsSwitchSegmentReadiness::Unavailable; - - assert_eq!(evaluate(&[candidate], HlsManifestAcceptanceTrigger::Critical), HlsManifestCommitPlan::RejectAll); - } - - #[test] - fn critical_mode_rejects_ambiguous_single_sample_cohorts() { - let first = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); - let mut second = observation(1, "origin-c", HlsCandidateHostRelation::OtherHost); - second.timeline_fingerprint = fingerprint(20); - - assert_eq!( - evaluate(&[first, second], HlsManifestAcceptanceTrigger::Critical), - HlsManifestCommitPlan::HoldAlternative - ); - } -} +mod tests; + +mod cohorts; +mod episode; +mod evaluation; +mod evidence; +mod timeline; +#[allow(unused_imports, reason = "Preserves the existing module interface.")] +pub use cohorts::{ + alternative_cohorts, alternative_cohorts_with_history, classify_reduced_retry_landscape, + held_alternative_after_burst, manifest_acceptance_landscape, +}; +pub use episode::{ + manifest_acceptance_episode_status, HlsDeterministicConflictReceipt, HlsManifestAcceptanceEpisodeStatus, + HlsManifestAcceptanceExhaustionReason, HlsManifestAcceptanceState, HlsManifestAcceptanceTrigger, + HlsRecoveryWorkloadBindingUpdate, +}; +use episode::{HlsManifestAcceptanceEpisodeOutcome, HlsRecoveryWorkloadBinding}; +pub use evaluation::evaluate_manifest_acceptance; +pub use evidence::{ + HlsAlternativeOriginCohort, HlsAlternativeOriginCohortIdentity, HlsAlternativeOriginWindow, + HlsCandidateHostRelation, HlsCommittedContentAnchorEvidence, HlsCrossHostAcceptanceEvidence, + HlsEmergencyAcceptanceEvidence, HlsEmergencyLiveHandoffCompatibility, HlsHostLocalSequenceRelation, + HlsManifestAcceptanceInput, HlsManifestAcceptanceLandscape, HlsManifestCandidateObservation, HlsManifestCommitKind, + HlsManifestCommitPlan, HlsManifestSegmentFingerprint, HlsManifestTechnicalSignature, + HlsManifestTimelineFingerprint, HlsPinnedOriginObservationState, HlsReducedRetryLandscapeChange, + HlsResourceTimelineEvidence, HlsSwitchSegmentReadiness, HlsTerminalAlternativeCompatibility, +}; +pub use timeline::classify_host_local_sequence; diff --git a/backend/hls/src/manifest_acceptance/cohorts.rs b/backend/hls/src/manifest_acceptance/cohorts.rs new file mode 100644 index 000000000..92677cf33 --- /dev/null +++ b/backend/hls/src/manifest_acceptance/cohorts.rs @@ -0,0 +1,245 @@ +use super::{ + timeline::{host_local_windows_compatible, same_host_timeline_compatible, strong_anchor_overlap}, + HlsAlternativeOriginCohort, HlsAlternativeOriginCohortIdentity, HlsAlternativeOriginWindow, + HlsCandidateHostRelation, HlsCrossHostAcceptanceEvidence, HlsHostLocalSequenceRelation, + HlsManifestAcceptanceLandscape, HlsManifestCandidateObservation, HlsManifestTechnicalSignature, + HlsPinnedOriginObservationState, HlsReducedRetryLandscapeChange, HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT, +}; + +pub fn alternative_cohorts(observations: &[HlsManifestCandidateObservation]) -> Vec { + let pinned = observations + .iter() + .filter(|candidate| { + candidate.host_relation == HlsCandidateHostRelation::PinnedHost + && candidate.resource_timeline_evidence.permits_acceptance() + }) + .collect::>(); + let mut alternatives = observations + .iter() + .filter(|candidate| { + candidate.host_relation == HlsCandidateHostRelation::OtherHost + && candidate.resource_timeline_evidence.permits_acceptance() + && candidate.switch_segment_readiness.can_be_staged() + && candidate.effective_host.is_some() + }) + .collect::>(); + alternatives.sort_by(|left, right| { + left.effective_host.cmp(&right.effective_host).then_with(|| left.candidate_index.cmp(&right.candidate_index)) + }); + let mut groups = Vec::>::new(); + for observation in alternatives { + if let Some(group) = groups.iter_mut().find(|group| { + group.iter().all(|sample| { + sample.effective_host == observation.effective_host + && same_host_timeline_compatible(sample, observation) + }) + }) { + if group.len() < HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT { + group.push(observation); + } + } else if groups.len() < HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT { + groups.push(vec![observation]); + } + } + + let mut cohorts = + groups.into_iter().filter_map(|samples| build_alternative_cohort(&samples, &pinned)).collect::>(); + cohorts.sort_by_key(|cohort| { + (evidence_priority(cohort.evidence), std::cmp::Reverse(cohort.successful_samples), cohort.best_candidate_index) + }); + cohorts +} + +pub fn manifest_acceptance_landscape( + observations: &[HlsManifestCandidateObservation], +) -> HlsManifestAcceptanceLandscape { + let pinned_state = pinned_observation_state(observations); + let alternatives = alternative_cohorts(observations) + .into_iter() + .take(HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT) + .map(|cohort| (cohort.identity, cohort.window)) + .collect(); + HlsManifestAcceptanceLandscape { pinned_state, alternatives } +} + +pub fn classify_reduced_retry_landscape( + previous: &HlsManifestAcceptanceLandscape, + observations: &[HlsManifestCandidateObservation], +) -> HlsReducedRetryLandscapeChange { + if observations.is_empty() { + return HlsReducedRetryLandscapeChange::Unchanged; + } + let current_pinned = pinned_observation_state(observations); + if current_pinned != HlsPinnedOriginObservationState::Missing && current_pinned != previous.pinned_state { + return HlsReducedRetryLandscapeChange::PinnedStateChanged; + } + if observations.iter().any(|candidate| candidate.host_relation == HlsCandidateHostRelation::Unknown) { + return HlsReducedRetryLandscapeChange::TimelineConflict; + } + for cohort in alternative_cohorts(observations) { + let Some((_, previous_window)) = + previous.alternatives.iter().find(|(identity, _)| identity == &cohort.identity) + else { + return HlsReducedRetryLandscapeChange::NewCohort; + }; + if !host_local_windows_compatible(previous_window, &cohort.window) { + return HlsReducedRetryLandscapeChange::TimelineConflict; + } + } + HlsReducedRetryLandscapeChange::Unchanged +} + +fn pinned_observation_state(observations: &[HlsManifestCandidateObservation]) -> HlsPinnedOriginObservationState { + if observations.iter().any(|candidate| { + candidate.host_relation == HlsCandidateHostRelation::PinnedHost + && !candidate.resource_timeline_evidence.permits_acceptance() + }) { + return HlsPinnedOriginObservationState::Rejected; + } + let relations = observations + .iter() + .filter(|candidate| candidate.host_relation == HlsCandidateHostRelation::PinnedHost) + .filter_map(|candidate| candidate.local_sequence_relation); + let mut state = HlsPinnedOriginObservationState::Missing; + for relation in relations { + match relation { + HlsHostLocalSequenceRelation::Next + | HlsHostLocalSequenceRelation::PlausibleForward + | HlsHostLocalSequenceRelation::RolloverCandidate + | HlsHostLocalSequenceRelation::Rebase => return HlsPinnedOriginObservationState::Progressed, + HlsHostLocalSequenceRelation::Same | HlsHostLocalSequenceRelation::NoBaseline => { + state = HlsPinnedOriginObservationState::Unchanged; + } + HlsHostLocalSequenceRelation::Backward => { + if state == HlsPinnedOriginObservationState::Missing { + state = HlsPinnedOriginObservationState::Rejected; + } + } + } + } + state +} + +pub fn alternative_cohorts_with_history( + observations: &[HlsManifestCandidateObservation], + previous: Option<&HlsAlternativeOriginCohort>, + current_burst_is_full_plan: bool, +) -> Vec { + let mut cohorts = alternative_cohorts(observations); + if current_burst_is_full_plan { + for cohort in &mut cohorts { + merge_full_burst_cohort_history(cohort, previous); + } + return cohorts; + } + // A reduced follow-up may recover the pinned host, but it cannot spend + // historical full-burst evidence on a new cross-host staging attempt. + Vec::new() +} + +pub fn held_alternative_after_burst( + observations: &[HlsManifestCandidateObservation], + previous: Option<&HlsAlternativeOriginCohort>, + current_burst_is_full_plan: bool, +) -> Option { + if current_burst_is_full_plan { + return alternative_cohorts_with_history(observations, previous, true).into_iter().next(); + } + // Cheap failures and contradictions do not erase the last completed + // configured burst. A later episode must run another full plan before the + // history can become consecutive acceptance evidence. + let previous = previous?; + let mut matching = + alternative_cohorts(observations).into_iter().find(|cohort| same_alternative_cohort(cohort, previous)); + if let Some(cohort) = matching.as_mut() { + cohort.successful_samples = cohort.successful_samples.max(previous.successful_samples); + cohort.total_samples = cohort + .total_samples + .saturating_add(previous.total_samples) + .min(u16::try_from(HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT).unwrap_or(u16::MAX)); + cohort.consecutive_confirmed_full_bursts = previous.consecutive_confirmed_full_bursts; + if evidence_priority(previous.evidence) < evidence_priority(cohort.evidence) { + cohort.evidence = previous.evidence; + } + } + matching.or_else(|| Some(previous.clone())) +} + +fn merge_full_burst_cohort_history( + current: &mut HlsAlternativeOriginCohort, + previous: Option<&HlsAlternativeOriginCohort>, +) { + current.consecutive_confirmed_full_bursts = 1; + let Some(previous) = previous.filter(|previous| same_alternative_cohort(current, previous)) else { + return; + }; + current.total_samples = current + .total_samples + .saturating_add(previous.total_samples) + .min(u16::try_from(HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT).unwrap_or(u16::MAX)); + current.consecutive_confirmed_full_bursts = previous.consecutive_confirmed_full_bursts.max(1).saturating_add(1); + if evidence_priority(previous.evidence) < evidence_priority(current.evidence) { + current.evidence = previous.evidence; + } +} + +fn same_alternative_cohort(left: &HlsAlternativeOriginCohort, right: &HlsAlternativeOriginCohort) -> bool { + left.identity == right.identity && host_local_windows_compatible(&left.window, &right.window) +} + +fn build_alternative_cohort( + samples: &[&HlsManifestCandidateObservation], + pinned: &[&HlsManifestCandidateObservation], +) -> Option { + let first = samples.first()?; + let successful_samples = u16::try_from(samples.len()).unwrap_or(u16::MAX); + let anchored_overlap = samples + .iter() + .flat_map(|sample| pinned.iter().map(move |baseline| strong_anchor_overlap(sample, baseline))) + .max() + .unwrap_or_default(); + let externally_anchored_overlap = samples + .iter() + .filter_map(|sample| match sample.evidence { + HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { overlapping_segments } => Some(overlapping_segments), + HlsCrossHostAcceptanceEvidence::Insufficient + | HlsCrossHostAcceptanceEvidence::BurstConsensusNewEpoch { .. } => None, + }) + .max() + .unwrap_or_default(); + let overlapping_segments = anchored_overlap.max(externally_anchored_overlap); + let evidence = if overlapping_segments > 0 { + HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { overlapping_segments } + } else if successful_samples >= 2 { + HlsCrossHostAcceptanceEvidence::BurstConsensusNewEpoch { successful_samples } + } else { + HlsCrossHostAcceptanceEvidence::Insufficient + }; + let best = samples.iter().min_by_key(|sample| (sample.manifest_fetch_elapsed_ms, sample.candidate_index))?; + let best_candidate_index = best.candidate_index; + let effective_host = first.effective_host.clone()?; + Some(HlsAlternativeOriginCohort { + identity: HlsAlternativeOriginCohortIdentity { + effective_host, + technical_signature: HlsManifestTechnicalSignature::from_fingerprint(&first.timeline_fingerprint), + }, + window: HlsAlternativeOriginWindow { + host_local_media_sequence: best.host_local_media_sequence, + host_local_highwater: best.host_local_highwater, + fingerprint: best.timeline_fingerprint.clone(), + }, + successful_samples, + total_samples: successful_samples, + consecutive_confirmed_full_bursts: 0, + evidence, + best_candidate_index, + }) +} + +fn evidence_priority(evidence: HlsCrossHostAcceptanceEvidence) -> u8 { + match evidence { + HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { .. } => 0, + HlsCrossHostAcceptanceEvidence::BurstConsensusNewEpoch { .. } => 1, + HlsCrossHostAcceptanceEvidence::Insufficient => 2, + } +} diff --git a/backend/hls/src/manifest_acceptance/episode.rs b/backend/hls/src/manifest_acceptance/episode.rs new file mode 100644 index 000000000..8e45c0500 --- /dev/null +++ b/backend/hls/src/manifest_acceptance/episode.rs @@ -0,0 +1,456 @@ +#[cfg(any(test, feature = "test-support"))] +use super::HlsRecoveryEtaMs; +use super::{ + HlsAcceptanceEpisodeTiming, HlsAlternativeOriginCohort, HlsDeterministicTimelineConflict, + HlsEstimatedRecoveryCompletionAtMs, HlsManifestAcceptanceEpisode, HlsManifestAcceptanceGeneration, + HlsManifestRecoveryCandidateIdentity, HlsRecoveryWorkload, HlsRecoveryWorkloadEnvelope, + HlsSelectedRecoveryCandidate, HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT, + HLS_MANIFEST_RECOVERY_BURST_SLOT_DELAY_MS, +}; +use sha2::{Digest, Sha256}; +use shared::model::HlsManifestRecoveryBurstPlan; + +impl PartialEq for HlsManifestRecoveryCandidateIdentity { + fn eq(&self, other: &Self) -> bool { + self.candidate_index == other.candidate_index + && self.effective_host_fingerprint == other.effective_host_fingerprint + && self.manifest_fingerprint == other.manifest_fingerprint + } +} + +impl Eq for HlsManifestRecoveryCandidateIdentity {} + +impl HlsManifestRecoveryCandidateIdentity { + pub fn from_candidate(candidate_index: usize, effective_host: Option<&str>, manifest_body: &str) -> Self { + let mut host_hasher = Sha256::new(); + host_hasher.update(b"tuliprox-hls-candidate-host-v1\0"); + match effective_host { + Some(host) => { + host_hasher.update([1]); + host_hasher.update(host.as_bytes()); + } + None => host_hasher.update([0]), + } + let mut manifest_hasher = Sha256::new(); + manifest_hasher.update(b"tuliprox-hls-candidate-manifest-v1\0"); + manifest_hasher.update(manifest_body.as_bytes()); + Self { + candidate_index, + effective_host_fingerprint: host_hasher.finalize().into(), + manifest_fingerprint: manifest_hasher.finalize().into(), + } + } + + pub fn matches_candidate(self, effective_host: Option<&str>, manifest_body: &str) -> bool { + self == Self::from_candidate(self.candidate_index, effective_host, manifest_body) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum HlsRecoveryWorkloadBinding { + CandidateUnknown { + envelope: HlsRecoveryWorkloadEnvelope, + selected_candidate: Option, + }, + CandidateBound { + envelope: HlsRecoveryWorkloadEnvelope, + acceptance_generation: HlsManifestAcceptanceGeneration, + candidate_identity: HlsManifestRecoveryCandidateIdentity, + workload: HlsRecoveryWorkload, + }, +} + +impl HlsRecoveryWorkloadBinding { + pub(super) const fn envelope(self) -> HlsRecoveryWorkloadEnvelope { + match self { + Self::CandidateUnknown { envelope, .. } | Self::CandidateBound { envelope, .. } => envelope, + } + } + + pub(super) const fn workload(self) -> HlsRecoveryWorkload { + match self { + Self::CandidateUnknown { envelope, .. } => envelope.ceiling(), + Self::CandidateBound { workload, .. } => workload, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsRecoveryWorkloadBindingUpdate { + Applied, + StaleGeneration, + EpisodeInactive, + CandidateMismatch, + OutsideEnvelope, +} + +/// Immutable pressure snapshot that authorizes one manifest-acceptance episode. +/// +/// Entering `Recovering` is an execution-state transition, not evidence that +/// reserve pressure exists. Consumers must therefore derive acceptance policy +/// from this trigger captured before the episode starts. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum HlsManifestAcceptanceTrigger { + #[default] + None, + Observe, + RecoveryRequired, + Critical, +} + +impl HlsManifestAcceptanceTrigger { + pub const fn starts_episode(self) -> bool { !matches!(self, Self::None) } + + pub const fn recovery_required(self) -> bool { matches!(self, Self::RecoveryRequired | Self::Critical) } + + pub const fn as_log_value(self) -> &'static str { + match self { + Self::None => "none", + Self::Observe => "observe", + Self::RecoveryRequired => "recovery_required", + Self::Critical => "critical", + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct HlsDeterministicConflictReceipt { + pub conflict: HlsDeterministicTimelineConflict, + pub origin_progress_generation: u64, + pub published_resource_history_generation: u64, + pub pinned_host_generation: u64, +} + +impl HlsManifestAcceptanceEpisode { + pub fn new( + generation: HlsManifestAcceptanceGeneration, + started_at_ms: u64, + burst_plan: HlsManifestRecoveryBurstPlan, + trigger: HlsManifestAcceptanceTrigger, + timing: &HlsAcceptanceEpisodeTiming, + ) -> Self { + let envelope = HlsRecoveryWorkloadEnvelope::from_timing_ceiling(timing.initial_workload); + Self { + generation, + started_at_ms, + burst_plan, + trigger, + timing: *timing, + workload_binding: HlsRecoveryWorkloadBinding::CandidateUnknown { envelope, selected_candidate: None }, + full_burst_completed: false, + full_bursts_completed: 0, + completed_burst_candidates: 0, + state: HlsManifestAcceptanceState::FullBurstPending, + outcome: HlsManifestAcceptanceEpisodeOutcome::Pending, + held_alternative: None, + observed_landscape: None, + next_retry_at_ms: None, + deterministic_conflict_receipt: None, + } + } + + pub fn required_candidates(&self) -> usize { self.burst_plan.total_candidates() } + + pub const fn trigger(&self) -> HlsManifestAcceptanceTrigger { self.trigger } + + pub const fn timing(&self) -> HlsAcceptanceEpisodeTiming { self.timing } + + pub fn deterministic_conflict_receipt(&self) -> Option<&HlsDeterministicConflictReceipt> { + self.deterministic_conflict_receipt.as_ref() + } + + pub fn record_deterministic_conflict(&mut self, receipt: HlsDeterministicConflictReceipt) { + if self.state != HlsManifestAcceptanceState::Completed { + self.deterministic_conflict_receipt = Some(receipt); + if self.full_burst_completed { + self.held_alternative = None; + self.next_retry_at_ms = None; + self.state = HlsManifestAcceptanceState::Holding; + self.outcome = HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted( + HlsManifestAcceptanceExhaustionReason::DeterministicTimelineConflict, + ); + } + } + } + + pub fn selected_candidate_identity(&self) -> Option { + match self.workload_binding { + HlsRecoveryWorkloadBinding::CandidateBound { candidate_identity, .. } => Some(candidate_identity), + HlsRecoveryWorkloadBinding::CandidateUnknown { selected_candidate, .. } => match selected_candidate { + Some(selected) if selected.acceptance_generation == self.generation => { + Some(selected.candidate_identity) + } + Some(_) | None => None, + }, + } + } + + /// Records the selected candidate identity without claiming any media-workload evidence. + /// + /// Same-host commits deliberately remain in `CandidateUnknown`; an alternative candidate is + /// bound only after its generation-local handoff preview identifies the exact recovery medium. + pub fn select_candidate( + &mut self, + expected_generation: HlsManifestAcceptanceGeneration, + candidate_identity: HlsManifestRecoveryCandidateIdentity, + ) -> HlsRecoveryWorkloadBindingUpdate { + if self.generation != expected_generation { + return HlsRecoveryWorkloadBindingUpdate::StaleGeneration; + } + if self.outcome != HlsManifestAcceptanceEpisodeOutcome::Pending + || !matches!( + self.state, + HlsManifestAcceptanceState::StagingSwitchSegment | HlsManifestAcceptanceState::Committing + ) + { + return HlsRecoveryWorkloadBindingUpdate::EpisodeInactive; + } + let HlsRecoveryWorkloadBinding::CandidateUnknown { envelope, selected_candidate } = self.workload_binding + else { + return HlsRecoveryWorkloadBindingUpdate::CandidateMismatch; + }; + let selected = HlsSelectedRecoveryCandidate { acceptance_generation: expected_generation, candidate_identity }; + if selected_candidate.is_some_and(|current| current != selected) { + return HlsRecoveryWorkloadBindingUpdate::CandidateMismatch; + } + self.workload_binding = + HlsRecoveryWorkloadBinding::CandidateUnknown { envelope, selected_candidate: Some(selected) }; + HlsRecoveryWorkloadBindingUpdate::Applied + } + + pub fn bind_selected_candidate( + &mut self, + expected_generation: HlsManifestAcceptanceGeneration, + candidate_identity: HlsManifestRecoveryCandidateIdentity, + workload: HlsRecoveryWorkload, + ) -> HlsRecoveryWorkloadBindingUpdate { + if self.generation != expected_generation { + return HlsRecoveryWorkloadBindingUpdate::StaleGeneration; + } + if self.outcome != HlsManifestAcceptanceEpisodeOutcome::Pending + || !matches!( + self.state, + HlsManifestAcceptanceState::StagingSwitchSegment | HlsManifestAcceptanceState::Committing + ) + { + return HlsRecoveryWorkloadBindingUpdate::EpisodeInactive; + } + let HlsRecoveryWorkloadBinding::CandidateUnknown { envelope, selected_candidate } = self.workload_binding + else { + return HlsRecoveryWorkloadBindingUpdate::CandidateMismatch; + }; + if selected_candidate + != Some(HlsSelectedRecoveryCandidate { acceptance_generation: expected_generation, candidate_identity }) + { + return HlsRecoveryWorkloadBindingUpdate::CandidateMismatch; + } + if !envelope.contains(workload) { + return HlsRecoveryWorkloadBindingUpdate::OutsideEnvelope; + } + self.workload_binding = HlsRecoveryWorkloadBinding::CandidateBound { + envelope, + acceptance_generation: expected_generation, + candidate_identity, + workload, + }; + HlsRecoveryWorkloadBindingUpdate::Applied + } + + pub fn advance_bound_candidate( + &mut self, + expected_generation: HlsManifestAcceptanceGeneration, + expected_identity: HlsManifestRecoveryCandidateIdentity, + workload: HlsRecoveryWorkload, + ) -> HlsRecoveryWorkloadBindingUpdate { + if self.generation != expected_generation { + return HlsRecoveryWorkloadBindingUpdate::StaleGeneration; + } + if self.outcome != HlsManifestAcceptanceEpisodeOutcome::Pending + || self.state != HlsManifestAcceptanceState::StagingSwitchSegment + { + return HlsRecoveryWorkloadBindingUpdate::EpisodeInactive; + } + let HlsRecoveryWorkloadBinding::CandidateBound { + envelope, + acceptance_generation, + candidate_identity, + workload: previous_workload, + } = self.workload_binding + else { + return HlsRecoveryWorkloadBindingUpdate::CandidateMismatch; + }; + if acceptance_generation != expected_generation || candidate_identity != expected_identity { + return HlsRecoveryWorkloadBindingUpdate::CandidateMismatch; + } + if !envelope.contains(workload) || !workload.is_no_greater_than(previous_workload) { + return HlsRecoveryWorkloadBindingUpdate::OutsideEnvelope; + } + self.workload_binding = HlsRecoveryWorkloadBinding::CandidateBound { + envelope, + acceptance_generation, + candidate_identity, + workload, + }; + HlsRecoveryWorkloadBindingUpdate::Applied + } + + /// Returns the work still attributable to this exact episode generation. + /// + /// The timing snapshot itself remains unchanged. Only completed burst work + /// and exact candidate-bound evidence may reduce the current estimate. + pub fn remaining_recovery_workload( + &self, + expected_generation: HlsManifestAcceptanceGeneration, + ) -> Option { + if self.generation != expected_generation || self.outcome != HlsManifestAcceptanceEpisodeOutcome::Pending { + return None; + } + let workload = self.workload_binding.workload(); + Some(if self.full_burst_completed { workload.after_full_burst() } else { workload }) + } + + #[cfg(any(test, feature = "test-support"))] + pub fn remaining_recovery_eta( + &self, + expected_generation: HlsManifestAcceptanceGeneration, + ) -> Option { + self.remaining_recovery_workload(expected_generation).map(|workload| self.timing.remaining_eta(workload)) + } + + pub fn estimated_recovery_completion_at( + &self, + expected_generation: HlsManifestAcceptanceGeneration, + now_ms: u64, + ) -> Option { + self.remaining_recovery_workload(expected_generation) + .map(|workload| self.timing.estimated_completion_at(now_ms, workload)) + } + + pub const fn exhaustion_reason(&self) -> Option { + match self.outcome { + HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted(reason) => Some(reason), + HlsManifestAcceptanceEpisodeOutcome::Pending | HlsManifestAcceptanceEpisodeOutcome::Committed => None, + } + } + + pub fn burst_max_stagger_ms(&self) -> u64 { + u64::try_from(self.burst_plan.slots.saturating_sub(1)) + .unwrap_or(u64::MAX) + .saturating_mul(HLS_MANIFEST_RECOVERY_BURST_SLOT_DELAY_MS) + } + + #[cfg(any(test, feature = "test-support"))] + pub fn record_full_burst(&mut self) { self.record_full_burst_candidates(self.required_candidates()); } + + pub fn record_full_burst_candidates(&mut self, completed_candidates: usize) { + self.completed_burst_candidates = completed_candidates.min(self.required_candidates()); + if self.completed_burst_candidates != self.required_candidates() { + return; + } + self.full_burst_completed = true; + self.full_bursts_completed = self.full_bursts_completed.saturating_add(1); + self.state = HlsManifestAcceptanceState::Evaluating; + } + + pub fn complete(&mut self) { + self.held_alternative = None; + self.next_retry_at_ms = None; + self.deterministic_conflict_receipt = None; + self.state = HlsManifestAcceptanceState::Completed; + self.outcome = HlsManifestAcceptanceEpisodeOutcome::Committed; + } + + pub fn record_exhaustion(&mut self, reason: HlsManifestAcceptanceExhaustionReason) { + if self.full_burst_completed && self.state != HlsManifestAcceptanceState::Completed { + self.outcome = HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted(reason); + } + } + + pub fn hold_after_uncommitted_burst( + &mut self, + mut cohort: Option, + next_retry_at_ms: Option, + ) { + if self.full_burst_completed && self.state != HlsManifestAcceptanceState::Completed { + if let Some(cohort) = cohort.as_mut() { + let limit = u16::try_from(HLS_MANIFEST_ACCEPTANCE_COHORT_SAMPLE_LIMIT).unwrap_or(u16::MAX); + cohort.successful_samples = cohort.successful_samples.min(limit); + cohort.total_samples = cohort.total_samples.min(limit); + } + self.held_alternative = cohort; + self.next_retry_at_ms = next_retry_at_ms; + self.state = HlsManifestAcceptanceState::Holding; + self.workload_binding = HlsRecoveryWorkloadBinding::CandidateUnknown { + envelope: self.workload_binding.envelope(), + selected_candidate: None, + }; + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsManifestAcceptanceState { + FullBurstPending, + Collecting, + Evaluating, + Holding, + StagingSwitchSegment, + Committing, + Completed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsManifestAcceptanceExhaustionReason { + AllFailed, + NoProgress, + NoCommittableCandidate, + DeterministicTimelineConflict, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum HlsManifestAcceptanceEpisodeOutcome { + Pending, + FullBurstExhausted(HlsManifestAcceptanceExhaustionReason), + Committed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsManifestAcceptanceEpisodeStatus { + Missing, + Expired { generation: HlsManifestAcceptanceGeneration }, + InFlight { generation: HlsManifestAcceptanceGeneration }, + FullBurstExhausted { generation: HlsManifestAcceptanceGeneration, reason: HlsManifestAcceptanceExhaustionReason }, + Committed { generation: HlsManifestAcceptanceGeneration }, + Superseded { generation: HlsManifestAcceptanceGeneration, current_generation: HlsManifestAcceptanceGeneration }, +} + +/// Normalizes private episode internals into one generation-safe policy input. +/// Callers must not infer cutover state from `state` plus counters themselves. +pub fn manifest_acceptance_episode_status( + episode: Option<&HlsManifestAcceptanceEpisode>, + current_generation: HlsManifestAcceptanceGeneration, + now_ms: u64, +) -> HlsManifestAcceptanceEpisodeStatus { + let Some(episode) = episode else { + return HlsManifestAcceptanceEpisodeStatus::Missing; + }; + if episode.generation != current_generation { + return HlsManifestAcceptanceEpisodeStatus::Superseded { generation: episode.generation, current_generation }; + } + match episode.outcome { + HlsManifestAcceptanceEpisodeOutcome::Pending + if now_ms >= episode.timing.acceptance_deadline.as_millis_since_epoch() => + { + HlsManifestAcceptanceEpisodeStatus::Expired { generation: episode.generation } + } + HlsManifestAcceptanceEpisodeOutcome::Pending => { + HlsManifestAcceptanceEpisodeStatus::InFlight { generation: episode.generation } + } + HlsManifestAcceptanceEpisodeOutcome::FullBurstExhausted(reason) => { + HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { generation: episode.generation, reason } + } + HlsManifestAcceptanceEpisodeOutcome::Committed => { + HlsManifestAcceptanceEpisodeStatus::Committed { generation: episode.generation } + } + } +} diff --git a/backend/hls/src/manifest_acceptance/evaluation.rs b/backend/hls/src/manifest_acceptance/evaluation.rs new file mode 100644 index 000000000..2ff6244bf --- /dev/null +++ b/backend/hls/src/manifest_acceptance/evaluation.rs @@ -0,0 +1,205 @@ +use super::{ + alternative_cohorts_with_history, HlsAlternativeOriginCohort, HlsCandidateHostRelation, + HlsCommittedContentAnchorEvidence, HlsCrossHostAcceptanceEvidence, HlsHostLocalSequenceRelation, + HlsManifestAcceptanceInput, HlsManifestAcceptanceTrigger, HlsManifestCandidateObservation, HlsManifestCommitKind, + HlsManifestCommitPlan, HlsSwitchSegmentReadiness, +}; + +pub fn evaluate_manifest_acceptance(input: HlsManifestAcceptanceInput<'_>) -> HlsManifestCommitPlan { + let (progressed_pinned, unchanged_pinned) = best_pinned_candidates(input.observations); + if let Some(pinned) = progressed_pinned { + return pinned_commit(pinned); + } + + // An unchanged pinned response is useful while reserve remains. Once the + // recovery deadline is reached it must not hide a qualified alternative. + if !input.trigger.recovery_required() { + if let Some(pinned) = unchanged_pinned { + return pinned_commit(pinned); + } + } + + if !input.full_burst_completed { + return if has_stageable_alternative(input.observations) { + HlsManifestCommitPlan::HoldAlternative + } else { + HlsManifestCommitPlan::RejectAll + }; + } + + if let Some(initial) = input + .observations + .iter() + .filter(|candidate| { + candidate.host_relation == HlsCandidateHostRelation::InitialBaseline + && candidate.resource_timeline_evidence.permits_acceptance() + }) + .min_by_key(|candidate| (candidate.manifest_fetch_elapsed_ms, candidate.candidate_index)) + { + return pinned_commit(initial); + } + + let cohorts = alternative_cohorts_with_history( + input.observations, + input.previous_alternative, + input.current_burst_is_full_plan, + ); + if let Some(cohort) = cohorts + .iter() + .find(|cohort| matches!(cohort.evidence, HlsCrossHostAcceptanceEvidence::StrongTimelineAnchor { .. })) + { + return alternative_plan( + input.observations, + cohort.best_candidate_index, + HlsManifestCommitKind::AnchoredAlternative, + ); + } + + if let Some(candidate) = input + .current_burst_is_full_plan + .then(|| { + input + .observations + .iter() + .filter(|candidate| candidate.host_relation == HlsCandidateHostRelation::OtherHost) + .filter(|candidate| candidate.resource_timeline_evidence.permits_acceptance()) + .filter(|candidate| { + candidate.committed_content_anchor + == HlsCommittedContentAnchorEvidence::RequiresStagedByteVerification + }) + .min_by_key(|candidate| (candidate.manifest_fetch_elapsed_ms, candidate.candidate_index)) + }) + .flatten() + { + return alternative_plan( + input.observations, + candidate.candidate_index, + HlsManifestCommitKind::ContentVerifiedAlternative, + ); + } + + if input.trigger.recovery_required() { + if let Some(cohort) = cohorts.iter().find(|cohort| { + matches!(cohort.evidence, HlsCrossHostAcceptanceEvidence::BurstConsensusNewEpoch { .. }) + || cohort.consecutive_confirmed_full_bursts >= 2 + }) { + return alternative_plan( + input.observations, + cohort.best_candidate_index, + HlsManifestCommitKind::AlternativeAsNewEpoch, + ); + } + } + + if input.trigger == HlsManifestAcceptanceTrigger::Critical { + if let Some(cohort) = critical_single_candidate_cohort(&cohorts, input.observations) { + return alternative_plan( + input.observations, + cohort.best_candidate_index, + HlsManifestCommitKind::EmergencyAlternativeAsNewEpoch, + ); + } + } + + if cohorts.is_empty() { + HlsManifestCommitPlan::RejectAll + } else { + HlsManifestCommitPlan::HoldAlternative + } +} + +fn critical_single_candidate_cohort<'a>( + cohorts: &'a [HlsAlternativeOriginCohort], + observations: &[HlsManifestCandidateObservation], +) -> Option<&'a HlsAlternativeOriginCohort> { + let [cohort] = cohorts else { + return None; + }; + if cohort.successful_samples != 1 || cohort.evidence != HlsCrossHostAcceptanceEvidence::Insufficient { + return None; + } + observations.iter().find(|candidate| candidate.candidate_index == cohort.best_candidate_index).filter( + |candidate| { + candidate.switch_segment_readiness.can_be_staged() + && candidate.emergency_evidence.requires_staged_verification() + }, + )?; + Some(cohort) +} + +fn pinned_commit(candidate: &HlsManifestCandidateObservation) -> HlsManifestCommitPlan { + HlsManifestCommitPlan::Commit { candidate_index: candidate.candidate_index, kind: HlsManifestCommitKind::Pinned } +} + +fn alternative_plan( + observations: &[HlsManifestCandidateObservation], + candidate_index: usize, + kind: HlsManifestCommitKind, +) -> HlsManifestCommitPlan { + match observations + .iter() + .find(|candidate| candidate.candidate_index == candidate_index) + .map(|candidate| candidate.switch_segment_readiness) + { + Some(HlsSwitchSegmentReadiness::RequiresStaging) => { + HlsManifestCommitPlan::StageAlternative { candidate_index, kind } + } + Some(HlsSwitchSegmentReadiness::Unavailable) | None => HlsManifestCommitPlan::RejectAll, + } +} + +fn best_pinned_candidates( + observations: &[HlsManifestCandidateObservation], +) -> (Option<&HlsManifestCandidateObservation>, Option<&HlsManifestCandidateObservation>) { + let mut progressed = None; + let mut unchanged = None; + for candidate in observations.iter().filter(|candidate| { + candidate.host_relation == HlsCandidateHostRelation::PinnedHost + && candidate.resource_timeline_evidence.permits_acceptance() + }) { + match candidate.local_sequence_relation { + Some( + HlsHostLocalSequenceRelation::Next + | HlsHostLocalSequenceRelation::PlausibleForward + | HlsHostLocalSequenceRelation::RolloverCandidate + | HlsHostLocalSequenceRelation::Rebase, + ) => { + if progressed.is_none_or(|current| pinned_order(candidate, current).is_lt()) { + progressed = Some(candidate); + } + } + Some(HlsHostLocalSequenceRelation::Same | HlsHostLocalSequenceRelation::NoBaseline) => { + if unchanged.is_none_or(|current| pinned_order(candidate, current).is_lt()) { + unchanged = Some(candidate); + } + } + Some(HlsHostLocalSequenceRelation::Backward) | None => {} + } + } + (progressed, unchanged) +} + +fn pinned_order(left: &HlsManifestCandidateObservation, right: &HlsManifestCandidateObservation) -> std::cmp::Ordering { + pinned_priority(left.local_sequence_relation) + .cmp(&pinned_priority(right.local_sequence_relation)) + .then_with(|| left.manifest_fetch_elapsed_ms.cmp(&right.manifest_fetch_elapsed_ms)) + .then_with(|| left.candidate_index.cmp(&right.candidate_index)) +} + +fn pinned_priority(relation: Option) -> u8 { + match relation { + Some(HlsHostLocalSequenceRelation::Next) => 0, + Some(HlsHostLocalSequenceRelation::PlausibleForward | HlsHostLocalSequenceRelation::Rebase) => 1, + Some(HlsHostLocalSequenceRelation::RolloverCandidate) => 2, + Some(HlsHostLocalSequenceRelation::Same | HlsHostLocalSequenceRelation::NoBaseline) => 3, + Some(HlsHostLocalSequenceRelation::Backward) | None => u8::MAX, + } +} + +fn has_stageable_alternative(observations: &[HlsManifestCandidateObservation]) -> bool { + observations.iter().any(|candidate| { + candidate.host_relation == HlsCandidateHostRelation::OtherHost + && candidate.resource_timeline_evidence.permits_acceptance() + && candidate.switch_segment_readiness.can_be_staged() + }) +} diff --git a/backend/hls/src/manifest_acceptance/evidence.rs b/backend/hls/src/manifest_acceptance/evidence.rs new file mode 100644 index 000000000..7ff92624b --- /dev/null +++ b/backend/hls/src/manifest_acceptance/evidence.rs @@ -0,0 +1,233 @@ +use super::{HlsManifestAcceptanceTrigger, HlsMediaResourceIdentity}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsCandidateHostRelation { + PinnedHost, + OtherHost, + InitialBaseline, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsHostLocalSequenceRelation { + NoBaseline, + Same, + Next, + PlausibleForward, + Backward, + RolloverCandidate, + Rebase, +} + +/// Bounded, URL-normalized metadata for one candidate segment. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct HlsManifestSegmentFingerprint { + pub duration_ms: u64, + pub discontinuity_before: bool, + pub program_date_time_ms: Option, + pub normalized_resource_identity: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct HlsManifestTimelineFingerprint { + pub segment_count: u32, + pub first_program_date_time_ms: Option, + pub last_program_date_time_ms: Option, + pub duration_pattern_hash: [u8; 32], + pub discontinuity_pattern_hash: [u8; 32], + pub normalized_resource_pattern_hash: Option<[u8; 32]>, + pub map_and_encryption_hash: [u8; 32], + pub container_signature_hash: [u8; 32], + pub segment_samples: Vec, +} + +/// Stable, window-independent technical identity of an alternative timeline. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct HlsManifestTechnicalSignature { + pub map_and_encryption_hash: [u8; 32], + pub container_signature_hash: [u8; 32], +} + +impl HlsManifestTechnicalSignature { + pub(super) fn from_fingerprint(fingerprint: &HlsManifestTimelineFingerprint) -> Self { + Self { + map_and_encryption_hash: fingerprint.map_and_encryption_hash, + container_signature_hash: fingerprint.container_signature_hash, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct HlsAlternativeOriginCohortIdentity { + pub effective_host: String, + pub technical_signature: HlsManifestTechnicalSignature, +} + +/// One concrete sliding playlist window observed for a stable cohort. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsAlternativeOriginWindow { + pub host_local_media_sequence: u64, + pub host_local_highwater: Option, + pub fingerprint: HlsManifestTimelineFingerprint, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsCrossHostAcceptanceEvidence { + Insufficient, + StrongTimelineAnchor { overlapping_segments: u16 }, + BurstConsensusNewEpoch { successful_samples: u16 }, +} + +/// State of the segment that would become the first segment after a host switch. +/// +/// Merely parsing a URI is not READY evidence. Alternative plans carrying +/// `RequiresStaging` must fetch and atomically commit that segment before the +/// commit callback may publish the selected manifest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsSwitchSegmentReadiness { + Unavailable, + RequiresStaging, +} + +impl HlsSwitchSegmentReadiness { + pub(super) const fn can_be_staged(self) -> bool { matches!(self, Self::RequiresStaging) } +} + +/// Manifest-level eligibility for the Critical single-candidate path. Track +/// compatibility is deliberately deferred until the candidate bytes are +/// staged and READY; it cannot be inferred from a URI or extension alone. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsEmergencyLiveHandoffCompatibility { + Incompatible, + RequiresStagedTrackVerification, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsTerminalAlternativeCompatibility { + /// Staged media and the configured terminal asset must still be compared. + RequiresStagedComparison, + LiveHandoffSafer, + /// Available terminal media is already known to be at least as safe. + TerminalTailPreferred, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HlsEmergencyAcceptanceEvidence { + pub live_handoff: HlsEmergencyLiveHandoffCompatibility, + pub terminal_alternative: HlsTerminalAlternativeCompatibility, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsCommittedContentAnchorEvidence { + Unavailable, + RequiresStagedByteVerification, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsResourceTimelineEvidence { + Eligible, + ReplayOnly, + ContradictoryOrder, +} + +impl HlsResourceTimelineEvidence { + pub(super) const fn permits_acceptance(self) -> bool { matches!(self, Self::Eligible) } +} + +impl HlsEmergencyAcceptanceEvidence { + pub const INCOMPATIBLE: Self = Self { + live_handoff: HlsEmergencyLiveHandoffCompatibility::Incompatible, + terminal_alternative: HlsTerminalAlternativeCompatibility::TerminalTailPreferred, + }; + + pub(super) const fn requires_staged_verification(self) -> bool { + matches!( + self, + Self { + live_handoff: HlsEmergencyLiveHandoffCompatibility::RequiresStagedTrackVerification, + terminal_alternative: HlsTerminalAlternativeCompatibility::RequiresStagedComparison, + } + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsManifestCandidateObservation { + pub candidate_index: usize, + pub candidate_slot: usize, + pub effective_host: Option, + pub host_relation: HlsCandidateHostRelation, + pub host_local_media_sequence: u64, + pub host_local_highwater: Option, + pub local_sequence_relation: Option, + pub resource_timeline_evidence: HlsResourceTimelineEvidence, + pub timeline_fingerprint: HlsManifestTimelineFingerprint, + pub manifest_fetch_elapsed_ms: u64, + pub switch_segment_readiness: HlsSwitchSegmentReadiness, + pub committed_content_anchor: HlsCommittedContentAnchorEvidence, + pub emergency_evidence: HlsEmergencyAcceptanceEvidence, + pub evidence: HlsCrossHostAcceptanceEvidence, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsAlternativeOriginCohort { + pub identity: HlsAlternativeOriginCohortIdentity, + pub window: HlsAlternativeOriginWindow, + pub successful_samples: u16, + pub total_samples: u16, + pub consecutive_confirmed_full_bursts: u16, + pub evidence: HlsCrossHostAcceptanceEvidence, + pub best_candidate_index: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsManifestCommitKind { + Pinned, + AnchoredAlternative, + ContentVerifiedAlternative, + AlternativeAsNewEpoch, + EmergencyAlternativeAsNewEpoch, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsPinnedOriginObservationState { + Missing, + Unchanged, + Progressed, + Rejected, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsManifestAcceptanceLandscape { + pub pinned_state: HlsPinnedOriginObservationState, + pub alternatives: Vec<(HlsAlternativeOriginCohortIdentity, HlsAlternativeOriginWindow)>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsReducedRetryLandscapeChange { + Unchanged, + NewCohort, + TimelineConflict, + PinnedStateChanged, +} + +impl HlsReducedRetryLandscapeChange { + pub const fn requires_full_requalification(self) -> bool { !matches!(self, Self::Unchanged) } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsManifestCommitPlan { + Commit { candidate_index: usize, kind: HlsManifestCommitKind }, + StageAlternative { candidate_index: usize, kind: HlsManifestCommitKind }, + HoldAlternative, + RejectAll, +} + +#[derive(Clone, Copy)] +pub struct HlsManifestAcceptanceInput<'a> { + pub full_burst_completed: bool, + pub current_burst_is_full_plan: bool, + pub trigger: HlsManifestAcceptanceTrigger, + pub previous_alternative: Option<&'a HlsAlternativeOriginCohort>, + pub observations: &'a [HlsManifestCandidateObservation], +} diff --git a/backend/hls/src/manifest_acceptance/tests.rs b/backend/hls/src/manifest_acceptance/tests.rs new file mode 100644 index 000000000..582606885 --- /dev/null +++ b/backend/hls/src/manifest_acceptance/tests.rs @@ -0,0 +1,108 @@ +use super::*; +use crate::recovery_timing::{ + HlsAcceptanceEpisodeTimingInput, HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsRecoveryBurstWorkload, + HlsRecoveryMapWorkload, HlsRecoverySegmentWorkload, HlsRecoveryTimingPolicy, HlsTerminalMediaPreparationState, + HlsTransitionMarginMs, +}; + +fn episode_timing(started_at_ms: u64, burst_plan: HlsManifestRecoveryBurstPlan) -> HlsAcceptanceEpisodeTiming { + HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput { + started_at_ms, + burst_plan, + target_duration_ms: 4_000, + transition_margin: HlsTransitionMarginMs::from_millis(1_000), + workload: HlsRecoveryWorkloadEnvelope::acceptance_policy().ceiling(), + observed_latency: HlsObservedRecoveryLatency::default(), + required_terminal_media_key: None, + terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, + policy: HlsRecoveryTimingPolicy::new( + HlsOperationTimeoutMs::from_millis(1_000), + HlsOperationTimeoutMs::from_millis(2_000), + HlsRecoveryEtaMs::from_millis(300), + HlsRecoveryEtaMs::from_millis(400), + ), + }) +} + +fn segment(marker: u8, pdt_ms: Option) -> HlsManifestSegmentFingerprint { + HlsManifestSegmentFingerprint { + duration_ms: 4_000, + discontinuity_before: false, + program_date_time_ms: pdt_ms, + normalized_resource_identity: Some(HlsMediaResourceIdentity::for_test(marker)), + } +} + +fn fingerprint(marker: u8) -> HlsManifestTimelineFingerprint { + HlsManifestTimelineFingerprint { + segment_count: 3, + first_program_date_time_ms: None, + last_program_date_time_ms: None, + duration_pattern_hash: [marker; 32], + discontinuity_pattern_hash: [0; 32], + normalized_resource_pattern_hash: Some([marker; 32]), + map_and_encryption_hash: [0; 32], + container_signature_hash: [1; 32], + segment_samples: vec![ + segment(marker, None), + segment(marker.saturating_add(1), None), + segment(marker.saturating_add(2), None), + ], + } +} + +fn observation(index: usize, host: &str, relation: HlsCandidateHostRelation) -> HlsManifestCandidateObservation { + HlsManifestCandidateObservation { + candidate_index: index, + candidate_slot: index / 2, + effective_host: Some(host.to_string()), + host_relation: relation, + host_local_media_sequence: 5, + host_local_highwater: Some(7), + local_sequence_relation: (relation == HlsCandidateHostRelation::PinnedHost) + .then_some(HlsHostLocalSequenceRelation::Next), + resource_timeline_evidence: HlsResourceTimelineEvidence::Eligible, + timeline_fingerprint: fingerprint(1), + manifest_fetch_elapsed_ms: 10, + switch_segment_readiness: HlsSwitchSegmentReadiness::RequiresStaging, + committed_content_anchor: HlsCommittedContentAnchorEvidence::Unavailable, + emergency_evidence: HlsEmergencyAcceptanceEvidence::INCOMPATIBLE, + evidence: HlsCrossHostAcceptanceEvidence::Insufficient, + } +} + +fn evaluate( + observations: &[HlsManifestCandidateObservation], + trigger: HlsManifestAcceptanceTrigger, +) -> HlsManifestCommitPlan { + evaluate_manifest_acceptance(HlsManifestAcceptanceInput { + full_burst_completed: true, + current_burst_is_full_plan: true, + trigger, + previous_alternative: None, + observations, + }) +} + +fn mark_emergency_verification_eligible(candidate: &mut HlsManifestCandidateObservation) { + candidate.emergency_evidence = HlsEmergencyAcceptanceEvidence { + live_handoff: HlsEmergencyLiveHandoffCompatibility::RequiresStagedTrackVerification, + terminal_alternative: HlsTerminalAlternativeCompatibility::RequiresStagedComparison, + }; +} + +fn sliding_observation( + index: usize, + host: &str, + media_sequence: u64, + markers: [u8; 3], +) -> HlsManifestCandidateObservation { + let mut candidate = observation(index, host, HlsCandidateHostRelation::OtherHost); + candidate.host_local_media_sequence = media_sequence; + candidate.host_local_highwater = Some(media_sequence.saturating_add(2)); + candidate.timeline_fingerprint.segment_samples = markers.map(|marker| segment(marker, None)).to_vec(); + candidate +} + +mod episode; +mod evaluation; diff --git a/backend/hls/src/manifest_acceptance/tests/episode.rs b/backend/hls/src/manifest_acceptance/tests/episode.rs new file mode 100644 index 000000000..0a1b97c0c --- /dev/null +++ b/backend/hls/src/manifest_acceptance/tests/episode.rs @@ -0,0 +1,364 @@ +use super::*; + +#[test] +fn first_episode_uses_entire_configured_beast_plan() { + let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); + let episode = HlsManifestAcceptanceEpisode::new( + HlsManifestAcceptanceGeneration(1), + 0, + plan, + HlsManifestAcceptanceTrigger::Observe, + &episode_timing(0, plan), + ); + + assert_eq!(episode.required_candidates(), plan.total_candidates()); + assert_eq!(episode.burst_max_stagger_ms(), 500); + assert!(!episode.full_burst_completed); +} + +#[test] +fn acceptance_trigger_semantics_are_explicit() { + assert!(!HlsManifestAcceptanceTrigger::None.starts_episode()); + assert!(HlsManifestAcceptanceTrigger::Observe.starts_episode()); + assert!(!HlsManifestAcceptanceTrigger::Observe.recovery_required()); + assert!(HlsManifestAcceptanceTrigger::RecoveryRequired.recovery_required()); + assert!(HlsManifestAcceptanceTrigger::Critical.recovery_required()); +} + +#[test] +fn held_episode_state_is_bounded_and_cleared_on_completion() { + let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); + let mut episode = HlsManifestAcceptanceEpisode::new( + HlsManifestAcceptanceGeneration(1), + 0, + plan, + HlsManifestAcceptanceTrigger::Observe, + &episode_timing(0, plan), + ); + let mut cohort = alternative_cohorts(&[ + observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), + observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), + ]) + .remove(0); + cohort.successful_samples = u16::MAX; + cohort.total_samples = u16::MAX; + + episode.record_full_burst(); + episode.hold_after_uncommitted_burst(Some(cohort), Some(123)); + assert_eq!(episode.held_alternative.as_ref().map(|held| held.successful_samples), Some(32)); + assert_eq!(episode.next_retry_at_ms, Some(123)); + episode.complete(); + assert_eq!(episode.held_alternative, None); + assert_eq!(episode.next_retry_at_ms, None); +} + +#[test] +fn completing_episode_clears_deterministic_conflict_receipt() { + let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); + let mut episode = HlsManifestAcceptanceEpisode::new( + HlsManifestAcceptanceGeneration(1), + 0, + plan, + HlsManifestAcceptanceTrigger::Observe, + &episode_timing(0, plan), + ); + let resource_key = HlsMediaResourceIdentity::for_test(7).semantic_key(); + episode.record_deterministic_conflict(HlsDeterministicConflictReceipt { + conflict: HlsDeterministicTimelineConflict { + previous_proxy_tail: Some(2), + existing_proxy_seq: 0, + candidate_position: 1, + candidate_origin_seq: 491, + resource_key, + decision: super::super::super::timeline::HlsResourceReplayDecision::RejectContradictoryOrder, + candidate_fingerprint: super::super::super::deterministic_conflict::HlsDeterministicConflictFingerprint { + segment_count: 1, + first_program_date_time_ms: None, + last_program_date_time_ms: None, + duration_pattern_hash: [1; 32], + discontinuity_pattern_hash: [2; 32], + semantic_resource_pattern_hash: Some(resource_key.bytes()), + map_and_encryption_hash: [3; 32], + container_signature_hash: [4; 32], + segment_samples: Vec::new(), + }, + }, + origin_progress_generation: 1, + published_resource_history_generation: 1, + pinned_host_generation: 1, + }); + + assert!(episode.deterministic_conflict_receipt().is_some()); + episode.complete(); + assert!(episode.deterministic_conflict_receipt().is_none()); +} + +#[test] +fn uncommitted_completed_burst_returns_episode_to_holding() { + let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); + let mut episode = HlsManifestAcceptanceEpisode::new( + HlsManifestAcceptanceGeneration(1), + 0, + plan, + HlsManifestAcceptanceTrigger::Observe, + &episode_timing(0, plan), + ); + + episode.record_full_burst(); + episode.state = HlsManifestAcceptanceState::StagingSwitchSegment; + episode.hold_after_uncommitted_burst(None, Some(123)); + + assert_eq!(episode.state, HlsManifestAcceptanceState::Holding); + assert!(episode.full_burst_completed); + assert_eq!(episode.full_bursts_completed, 1); + assert_eq!(episode.next_retry_at_ms, Some(123)); +} + +#[test] +fn hls_recovery_timing_status_uses_frozen_episode_deadline_and_normalizes_outcomes() { + let plan = shared::model::HlsManifestRecoveryBurstLevel::Friendly.plan(); + let generation = HlsManifestAcceptanceGeneration(9); + let timing = episode_timing(100, plan); + let deadline_ms = timing.acceptance_deadline.as_millis_since_epoch(); + let mut episode = HlsManifestAcceptanceEpisode::new( + generation, + 100, + plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &timing, + ); + assert_eq!( + manifest_acceptance_episode_status(Some(&episode), generation, deadline_ms.saturating_sub(1)), + HlsManifestAcceptanceEpisodeStatus::InFlight { generation } + ); + episode.record_full_burst(); + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate); + assert_eq!( + manifest_acceptance_episode_status(Some(&episode), generation, deadline_ms.saturating_sub(1)), + HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { + generation, + reason: HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate, + } + ); + assert_eq!( + manifest_acceptance_episode_status( + Some(&episode), + HlsManifestAcceptanceGeneration(10), + deadline_ms.saturating_sub(1), + ), + HlsManifestAcceptanceEpisodeStatus::Superseded { + generation, + current_generation: HlsManifestAcceptanceGeneration(10), + } + ); + assert_eq!( + manifest_acceptance_episode_status(Some(&episode), generation, deadline_ms), + HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { + generation, + reason: HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate, + } + ); + + let pending_episode = HlsManifestAcceptanceEpisode::new( + generation, + 100, + plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &timing, + ); + assert_eq!( + manifest_acceptance_episode_status(Some(&pending_episode), generation, deadline_ms), + HlsManifestAcceptanceEpisodeStatus::Expired { generation } + ); +} + +#[test] +fn hls_recovery_timing_episode_keeps_construction_snapshot_immutable() { + let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); + let frozen_timing = episode_timing(100, plan); + let episode = HlsManifestAcceptanceEpisode::new( + HlsManifestAcceptanceGeneration(3), + 100, + plan, + HlsManifestAcceptanceTrigger::Critical, + &frozen_timing, + ); + let later_timing = episode_timing(10_000, plan); + + assert_ne!(later_timing.acceptance_deadline, frozen_timing.acceptance_deadline); + assert_eq!(episode.timing(), frozen_timing); + assert_eq!(episode.generation, HlsManifestAcceptanceGeneration(3)); + assert_eq!(episode.started_at_ms, 100); + assert_eq!(episode.burst_plan, plan); + assert_eq!(episode.trigger(), HlsManifestAcceptanceTrigger::Critical); +} + +#[test] +fn hls_recovery_timing_candidate_binding_and_staging_reduce_eta_without_moving_deadline() { + let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); + let generation = HlsManifestAcceptanceGeneration(4); + let mut episode = HlsManifestAcceptanceEpisode::new( + generation, + 100, + plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &episode_timing(100, plan), + ); + let frozen_deadline = episode.timing().acceptance_deadline; + let initial_eta = episode.remaining_recovery_eta(generation).map(HlsRecoveryEtaMs::as_millis); + + episode.record_full_burst(); + let completed_burst_eta = episode.remaining_recovery_eta(generation).map(HlsRecoveryEtaMs::as_millis); + episode.state = HlsManifestAcceptanceState::StagingSwitchSegment; + let identity = HlsManifestRecoveryCandidateIdentity::from_candidate( + 2, + Some("candidate.example.com"), + "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n", + ); + let bound_workload = HlsRecoveryWorkload { + burst: HlsRecoveryBurstWorkload::FullBurstPending, + segment: HlsRecoverySegmentWorkload::ClearSegmentFetch, + map: HlsRecoveryMapWorkload::Fetch, + }; + assert_eq!(episode.select_candidate(generation, identity), HlsRecoveryWorkloadBindingUpdate::Applied); + assert_eq!( + episode.bind_selected_candidate(generation, identity, bound_workload), + HlsRecoveryWorkloadBindingUpdate::Applied + ); + assert_eq!( + episode.remaining_recovery_workload(generation).map(|workload| workload.map), + Some(HlsRecoveryMapWorkload::Fetch) + ); + let bound_eta = episode.remaining_recovery_eta(generation).map(HlsRecoveryEtaMs::as_millis); + episode.state = HlsManifestAcceptanceState::Committing; + let committing_eta = episode.remaining_recovery_eta(generation).map(HlsRecoveryEtaMs::as_millis); + episode.state = HlsManifestAcceptanceState::StagingSwitchSegment; + assert_eq!( + episode.advance_bound_candidate( + generation, + identity, + HlsRecoveryWorkload { + burst: HlsRecoveryBurstWorkload::FullBurstCompleted, + segment: HlsRecoverySegmentWorkload::SegmentStagedWithDependenciesReady, + map: HlsRecoveryMapWorkload::Ready, + }, + ), + HlsRecoveryWorkloadBindingUpdate::Applied + ); + let staged_eta = episode.remaining_recovery_eta(generation).map(HlsRecoveryEtaMs::as_millis); + + assert!(initial_eta > completed_burst_eta); + assert!(completed_burst_eta > bound_eta); + assert_eq!(committing_eta, bound_eta); + assert!(bound_eta > staged_eta); + assert_eq!(episode.timing().acceptance_deadline, frozen_deadline); + assert_eq!( + episode + .estimated_recovery_completion_at(generation, 5_000) + .map(HlsEstimatedRecoveryCompletionAtMs::as_millis_since_epoch), + staged_eta.map(|eta| 5_000_u64.saturating_add(eta)) + ); + assert_eq!(episode.remaining_recovery_workload(HlsManifestAcceptanceGeneration(5)), None); + + episode.complete(); + assert_eq!(episode.remaining_recovery_workload(generation), None); + assert_eq!(episode.estimated_recovery_completion_at(generation, 5_000), None); +} + +#[test] +fn hls_recovery_timing_candidate_stale_generation_and_wrong_identity_cannot_update_binding() { + let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); + let generation = HlsManifestAcceptanceGeneration(9); + let mut episode = HlsManifestAcceptanceEpisode::new( + generation, + 100, + plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &episode_timing(100, plan), + ); + episode.record_full_burst(); + episode.state = HlsManifestAcceptanceState::StagingSwitchSegment; + let selected = HlsManifestRecoveryCandidateIdentity::from_candidate(1, Some("a.example"), "body"); + let other = HlsManifestRecoveryCandidateIdentity::from_candidate(2, Some("a.example"), "body"); + let candidate_workload = HlsRecoveryWorkload { + burst: HlsRecoveryBurstWorkload::FullBurstPending, + segment: HlsRecoverySegmentWorkload::ClearSegmentFetch, + map: HlsRecoveryMapWorkload::Fetch, + }; + + assert_eq!( + episode.select_candidate(HlsManifestAcceptanceGeneration(8), selected), + HlsRecoveryWorkloadBindingUpdate::StaleGeneration + ); + assert_eq!(episode.select_candidate(generation, selected), HlsRecoveryWorkloadBindingUpdate::Applied); + assert_eq!( + episode.bind_selected_candidate(generation, other, candidate_workload), + HlsRecoveryWorkloadBindingUpdate::CandidateMismatch + ); + assert_eq!( + episode.bind_selected_candidate(generation, selected, candidate_workload), + HlsRecoveryWorkloadBindingUpdate::Applied + ); + let before = episode.remaining_recovery_workload(generation); + assert_eq!( + episode.advance_bound_candidate( + generation, + other, + HlsRecoveryWorkload { + burst: HlsRecoveryBurstWorkload::FullBurstCompleted, + segment: HlsRecoverySegmentWorkload::SegmentStagedWithDependenciesReady, + map: HlsRecoveryMapWorkload::Ready, + }, + ), + HlsRecoveryWorkloadBindingUpdate::CandidateMismatch + ); + assert_eq!( + episode.advance_bound_candidate( + HlsManifestAcceptanceGeneration(8), + selected, + HlsRecoveryWorkload { + burst: HlsRecoveryBurstWorkload::FullBurstCompleted, + segment: HlsRecoverySegmentWorkload::SegmentStagedWithDependenciesReady, + map: HlsRecoveryMapWorkload::Ready, + }, + ), + HlsRecoveryWorkloadBindingUpdate::StaleGeneration + ); + assert_eq!(episode.remaining_recovery_workload(generation), before); +} + +#[test] +fn hls_recovery_timing_candidate_same_host_selection_keeps_conservative_unknown_workload() { + let plan = shared::model::HlsManifestRecoveryBurstLevel::Beast.plan(); + let generation = HlsManifestAcceptanceGeneration(10); + let mut episode = HlsManifestAcceptanceEpisode::new( + generation, + 100, + plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &episode_timing(100, plan), + ); + episode.record_full_burst(); + episode.state = HlsManifestAcceptanceState::Committing; + let selected = HlsManifestRecoveryCandidateIdentity::from_candidate( + 1, + Some("pinned.example"), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:8\n", + ); + let conservative = episode.remaining_recovery_workload(generation); + + assert_eq!(episode.select_candidate(generation, selected), HlsRecoveryWorkloadBindingUpdate::Applied); + + assert_eq!(episode.selected_candidate_identity(), Some(selected)); + assert_eq!(episode.remaining_recovery_workload(generation), conservative); +} + +#[test] +fn hls_recovery_timing_candidate_identity_is_host_local_even_for_identical_manifest_bytes() { + let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:7\n"; + + let origin_a = HlsManifestRecoveryCandidateIdentity::from_candidate(0, Some("a.example"), body); + let origin_b = HlsManifestRecoveryCandidateIdentity::from_candidate(0, Some("b.example"), body); + + assert_ne!(origin_a, origin_b); +} diff --git a/backend/hls/src/manifest_acceptance/tests/evaluation.rs b/backend/hls/src/manifest_acceptance/tests/evaluation.rs new file mode 100644 index 000000000..05be5fb91 --- /dev/null +++ b/backend/hls/src/manifest_acceptance/tests/evaluation.rs @@ -0,0 +1,401 @@ +use super::*; + +#[test] +fn forward_media_sequence_does_not_override_published_resource_replay() { + let mut replay = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); + replay.local_sequence_relation = Some(HlsHostLocalSequenceRelation::PlausibleForward); + replay.resource_timeline_evidence = HlsResourceTimelineEvidence::ReplayOnly; + + assert_eq!(evaluate(&[replay], HlsManifestAcceptanceTrigger::RecoveryRequired), HlsManifestCommitPlan::RejectAll); +} + +#[test] +fn identical_media_sequence_on_other_host_has_no_local_continuity_relation() { + let candidate = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); + + assert_eq!(candidate.host_local_media_sequence, 5); + assert_eq!(candidate.local_sequence_relation, None); +} + +#[test] +fn progressed_pinned_candidate_wins_over_alternative() { + let observations = [ + observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), + observation(1, "origin-a", HlsCandidateHostRelation::PinnedHost), + ]; + + assert_eq!( + evaluate(&observations, HlsManifestAcceptanceTrigger::Critical), + HlsManifestCommitPlan::Commit { candidate_index: 1, kind: HlsManifestCommitKind::Pinned } + ); +} + +#[test] +fn unchanged_pinned_does_not_hide_consensus_under_recovery_pressure() { + let alternatives = [ + observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), + observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), + ]; + let mut pinned = observation(2, "origin-a", HlsCandidateHostRelation::PinnedHost); + pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Same); + pinned.timeline_fingerprint = fingerprint(40); + let observations = [alternatives[0].clone(), alternatives[1].clone(), pinned]; + + assert_eq!( + evaluate(&observations, HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 0, + kind: HlsManifestCommitKind::AlternativeAsNewEpoch, + } + ); +} + +#[test] +fn unchanged_pinned_wins_while_recovery_is_not_required() { + let mut pinned = observation(1, "origin-a", HlsCandidateHostRelation::PinnedHost); + pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Same); + let observations = [observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), pinned]; + + assert_eq!( + evaluate(&observations, HlsManifestAcceptanceTrigger::Observe), + HlsManifestCommitPlan::Commit { candidate_index: 1, kind: HlsManifestCommitKind::Pinned } + ); +} + +#[test] +fn alternative_cannot_commit_before_full_burst() { + let observations = [observation(0, "origin-b", HlsCandidateHostRelation::OtherHost)]; + + assert_eq!( + evaluate_manifest_acceptance(HlsManifestAcceptanceInput { + full_burst_completed: false, + current_burst_is_full_plan: false, + trigger: HlsManifestAcceptanceTrigger::Critical, + previous_alternative: None, + observations: &observations, + }), + HlsManifestCommitPlan::HoldAlternative + ); +} + +#[test] +fn configured_off_single_initial_candidate_uses_acceptance_pipeline() { + let mut initial = observation(0, "origin-a", HlsCandidateHostRelation::InitialBaseline); + initial.local_sequence_relation = Some(HlsHostLocalSequenceRelation::NoBaseline); + let observations = [initial]; + + assert_eq!( + evaluate_manifest_acceptance(HlsManifestAcceptanceInput { + full_burst_completed: false, + current_burst_is_full_plan: false, + trigger: HlsManifestAcceptanceTrigger::RecoveryRequired, + previous_alternative: None, + observations: &observations, + }), + HlsManifestCommitPlan::RejectAll + ); + assert_eq!( + evaluate(&observations, HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::Commit { candidate_index: 0, kind: HlsManifestCommitKind::Pinned } + ); +} + +#[test] +fn observe_consensus_is_held_without_reserve_pressure() { + let observations = [ + observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), + observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), + ]; + + assert_eq!(evaluate(&observations, HlsManifestAcceptanceTrigger::Observe), HlsManifestCommitPlan::HoldAlternative); +} + +#[test] +fn recovery_required_consensus_may_stage_new_epoch() { + let observations = [ + observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), + observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), + ]; + + assert_eq!( + evaluate(&observations, HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 0, + kind: HlsManifestCommitKind::AlternativeAsNewEpoch, + } + ); +} + +#[test] +fn episode_trigger_remains_authoritative_after_execution_enters_recovery() { + let observations = [ + observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), + observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), + ]; + let plan = shared::model::HlsManifestRecoveryBurstLevel::Friendly.plan(); + let mut episode = HlsManifestAcceptanceEpisode::new( + HlsManifestAcceptanceGeneration(7), + 10, + plan, + HlsManifestAcceptanceTrigger::Observe, + &episode_timing(10, plan), + ); + episode.state = HlsManifestAcceptanceState::Collecting; + episode.record_full_burst(); + + assert_eq!( + evaluate(&observations, episode.trigger()), + HlsManifestCommitPlan::HoldAlternative, + "execution state must not upgrade Observe to reserve pressure" + ); +} + +#[test] +fn alternative_requires_ready_staging_before_commit() { + let mut first = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); + let mut second = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); + first.switch_segment_readiness = HlsSwitchSegmentReadiness::RequiresStaging; + second.switch_segment_readiness = HlsSwitchSegmentReadiness::RequiresStaging; + let observations = [first, second]; + + assert_eq!( + evaluate(&observations, HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 0, + kind: HlsManifestCommitKind::AlternativeAsNewEpoch, + } + ); +} + +#[test] +fn strong_pdt_anchor_qualifies_alternative_without_sequence_comparison() { + let pdt = 1_700_000_000_000; + let mut pinned = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); + pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Backward); + pinned.timeline_fingerprint.segment_samples[0].program_date_time_ms = Some(pdt); + pinned.timeline_fingerprint.segment_samples[1].program_date_time_ms = Some(pdt + 4_000); + let mut alternative = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); + alternative.host_local_media_sequence = 900; + alternative.host_local_highwater = Some(902); + alternative.timeline_fingerprint.segment_samples[0].program_date_time_ms = Some(pdt + 500); + alternative.timeline_fingerprint.segment_samples[1].program_date_time_ms = Some(pdt + 4_500); + for (index, segment) in alternative.timeline_fingerprint.segment_samples.iter_mut().enumerate() { + segment.normalized_resource_identity = + Some(HlsMediaResourceIdentity::for_test(u8::try_from(index).unwrap_or(u8::MAX).saturating_add(90))); + } + + assert_eq!( + evaluate(&[pinned, alternative], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 1, + kind: HlsManifestCommitKind::AnchoredAlternative, + } + ); +} + +#[test] +fn one_accidental_pdt_overlap_is_not_a_strong_cross_host_anchor() { + let pdt = 1_700_000_000_000; + let mut pinned = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); + pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Backward); + pinned.timeline_fingerprint.segment_samples[0].program_date_time_ms = Some(pdt); + let mut alternative = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); + alternative.timeline_fingerprint.segment_samples[0].program_date_time_ms = Some(pdt + 500); + for (index, segment) in alternative.timeline_fingerprint.segment_samples.iter_mut().enumerate() { + segment.normalized_resource_identity = + Some(HlsMediaResourceIdentity::for_test(u8::try_from(index).unwrap_or(u8::MAX).saturating_add(90))); + } + + assert_eq!( + evaluate(&[pinned, alternative], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::HoldAlternative + ); +} + +#[test] +fn same_sequence_on_different_hosts_does_not_create_strong_anchor() { + let mut pinned = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); + pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Backward); + pinned.timeline_fingerprint = fingerprint(1); + let mut alternative = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); + alternative.timeline_fingerprint = fingerprint(20); + + assert_eq!( + evaluate(&[pinned, alternative], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::HoldAlternative + ); +} + +#[test] +fn same_normalized_path_from_different_queries_with_unknown_global_identity_is_not_a_strong_anchor() { + let mut pinned = observation(0, "origin-a", HlsCandidateHostRelation::PinnedHost); + pinned.local_sequence_relation = Some(HlsHostLocalSequenceRelation::Backward); + let mut alternative = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); + // Equal normalized identities model equal paths after host and query-token removal. + // Without PDT or staged byte equality the global content identity remains unknown. + alternative.timeline_fingerprint.normalized_resource_pattern_hash = + pinned.timeline_fingerprint.normalized_resource_pattern_hash; + + assert_eq!( + evaluate(&[pinned, alternative], HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::HoldAlternative + ); +} + +#[test] +fn normalized_path_only_requests_staged_byte_verification_and_never_direct_anchor_commit() { + let mut candidate = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); + candidate.committed_content_anchor = HlsCommittedContentAnchorEvidence::RequiresStagedByteVerification; + + assert_eq!( + evaluate(&[candidate], HlsManifestAcceptanceTrigger::Observe), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 0, + kind: HlsManifestCommitKind::ContentVerifiedAlternative, + } + ); +} + +#[test] +fn different_local_shapes_do_not_form_a_consensus_cohort() { + let first = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); + let mut second = observation(1, "origin-b", HlsCandidateHostRelation::OtherHost); + second.timeline_fingerprint.segment_samples[1].duration_ms = 9_000; + + let cohorts = alternative_cohorts(&[first, second]); + + assert_eq!(cohorts.len(), 2); + assert!(cohorts.iter().all(|cohort| cohort.successful_samples == 1)); +} + +#[test] +fn reduced_follow_up_preserves_matching_full_burst_evidence_without_incrementing_it() { + let observations = [ + observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), + observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), + ]; + let full = alternative_cohorts_with_history(&observations, None, true).remove(0); + let follow_up = + held_alternative_after_burst(&observations[..1], Some(&full), false).expect("matching follow-up cohort"); + + assert_eq!(full.consecutive_confirmed_full_bursts, 1); + assert_eq!(follow_up.consecutive_confirmed_full_bursts, 1); + assert_eq!(follow_up.successful_samples, full.successful_samples); + assert!(follow_up.total_samples > full.total_samples); +} + +#[test] +fn matching_later_full_burst_advances_consecutive_evidence() { + let first_window = [sliding_observation(0, "origin-b", 5, [1, 2, 3])]; + let second_window = [sliding_observation(0, "origin-b", 6, [2, 3, 4])]; + let first = alternative_cohorts_with_history(&first_window, None, true).remove(0); + let second = alternative_cohorts_with_history(&second_window, Some(&first), true).remove(0); + + assert_eq!(second.consecutive_confirmed_full_bursts, 2); + assert_ne!(first.window.fingerprint, second.window.fingerprint); +} + +#[test] +fn reduced_retry_classifies_sliding_same_cohort_separately_from_new_or_conflicting_cohorts() { + let first = [sliding_observation(0, "origin-b", 5, [1, 2, 3])]; + let landscape = manifest_acceptance_landscape(&first); + let sliding = [sliding_observation(0, "origin-b", 6, [2, 3, 4])]; + let new_host = [sliding_observation(0, "origin-c", 6, [2, 3, 4])]; + let conflict = [sliding_observation(0, "origin-b", 6, [90, 91, 92])]; + + assert_eq!(classify_reduced_retry_landscape(&landscape, &sliding), HlsReducedRetryLandscapeChange::Unchanged); + assert_eq!(classify_reduced_retry_landscape(&landscape, &new_host), HlsReducedRetryLandscapeChange::NewCohort); + assert_eq!( + classify_reduced_retry_landscape(&landscape, &conflict), + HlsReducedRetryLandscapeChange::TimelineConflict + ); +} + +#[test] +fn two_matching_single_sample_full_bursts_may_stage_under_recovery_pressure() { + let first_window = [sliding_observation(0, "origin-b", 5, [1, 2, 3])]; + let second_window = [sliding_observation(0, "origin-b", 6, [2, 3, 4])]; + let first = alternative_cohorts_with_history(&first_window, None, true).remove(0); + + assert_eq!( + evaluate_manifest_acceptance(HlsManifestAcceptanceInput { + full_burst_completed: true, + current_burst_is_full_plan: true, + trigger: HlsManifestAcceptanceTrigger::RecoveryRequired, + previous_alternative: Some(&first), + observations: &second_window, + }), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 0, + kind: HlsManifestCommitKind::AlternativeAsNewEpoch, + } + ); +} + +#[test] +fn conflicting_reduced_follow_up_cannot_reuse_previous_full_burst() { + let first_observations = [ + observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), + observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), + ]; + let first = alternative_cohorts_with_history(&first_observations, None, true).remove(0); + let mut conflict = observation(0, "origin-c", HlsCandidateHostRelation::OtherHost); + conflict.timeline_fingerprint = fingerprint(20); + + assert!(alternative_cohorts_with_history(&[conflict], Some(&first), false).is_empty()); +} + +#[test] +fn failed_reduced_follow_up_does_not_erase_previous_full_burst_evidence() { + let observations = [ + observation(0, "origin-b", HlsCandidateHostRelation::OtherHost), + observation(1, "origin-b", HlsCandidateHostRelation::OtherHost), + ]; + let full = held_alternative_after_burst(&observations, None, true).expect("full burst cohort"); + + assert_eq!(held_alternative_after_burst(&[], Some(&full), false), Some(full)); +} + +#[test] +fn critical_single_candidate_requires_typed_staged_verification() { + let mut candidate = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); + candidate.switch_segment_readiness = HlsSwitchSegmentReadiness::RequiresStaging; + mark_emergency_verification_eligible(&mut candidate); + let observations = [candidate.clone()]; + + assert_eq!( + evaluate(&observations, HlsManifestAcceptanceTrigger::Critical), + HlsManifestCommitPlan::StageAlternative { + candidate_index: 0, + kind: HlsManifestCommitKind::EmergencyAlternativeAsNewEpoch, + } + ); + assert_eq!( + evaluate(&observations, HlsManifestAcceptanceTrigger::RecoveryRequired), + HlsManifestCommitPlan::HoldAlternative + ); + + candidate.emergency_evidence.terminal_alternative = HlsTerminalAlternativeCompatibility::TerminalTailPreferred; + assert_eq!(evaluate(&[candidate], HlsManifestAcceptanceTrigger::Critical), HlsManifestCommitPlan::HoldAlternative); +} + +#[test] +fn critical_single_candidate_without_stageable_first_segment_is_rejected() { + let mut candidate = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); + mark_emergency_verification_eligible(&mut candidate); + candidate.switch_segment_readiness = HlsSwitchSegmentReadiness::Unavailable; + + assert_eq!(evaluate(&[candidate], HlsManifestAcceptanceTrigger::Critical), HlsManifestCommitPlan::RejectAll); +} + +#[test] +fn critical_mode_rejects_ambiguous_single_sample_cohorts() { + let first = observation(0, "origin-b", HlsCandidateHostRelation::OtherHost); + let mut second = observation(1, "origin-c", HlsCandidateHostRelation::OtherHost); + second.timeline_fingerprint = fingerprint(20); + + assert_eq!( + evaluate(&[first, second], HlsManifestAcceptanceTrigger::Critical), + HlsManifestCommitPlan::HoldAlternative + ); +} diff --git a/backend/hls/src/manifest_acceptance/timeline.rs b/backend/hls/src/manifest_acceptance/timeline.rs new file mode 100644 index 000000000..2d2113536 --- /dev/null +++ b/backend/hls/src/manifest_acceptance/timeline.rs @@ -0,0 +1,168 @@ +use super::{ + HlsAlternativeOriginWindow, HlsHostLocalSequenceRelation, HlsManifestCandidateObservation, + HlsManifestSegmentFingerprint, HlsManifestTechnicalSignature, +}; + +pub fn classify_host_local_sequence( + previous_highwater: Option, + candidate_highwater: Option, + forward_window: u64, + rebase_allowed: bool, +) -> Option { + let candidate = candidate_highwater?; + if rebase_allowed { + return Some(HlsHostLocalSequenceRelation::Rebase); + } + let Some(previous) = previous_highwater else { + return Some(HlsHostLocalSequenceRelation::NoBaseline); + }; + if candidate == previous { + return Some(HlsHostLocalSequenceRelation::Same); + } + if previous.checked_add(1) == Some(candidate) { + return Some(HlsHostLocalSequenceRelation::Next); + } + if candidate > previous { + return Some(if candidate.saturating_sub(previous) <= forward_window.max(1) { + HlsHostLocalSequenceRelation::PlausibleForward + } else { + HlsHostLocalSequenceRelation::Backward + }); + } + Some(if candidate <= forward_window.max(1) { + HlsHostLocalSequenceRelation::RolloverCandidate + } else { + HlsHostLocalSequenceRelation::Backward + }) +} + +pub(super) fn same_host_timeline_compatible( + left: &HlsManifestCandidateObservation, + right: &HlsManifestCandidateObservation, +) -> bool { + let left_window = HlsAlternativeOriginWindow { + host_local_media_sequence: left.host_local_media_sequence, + host_local_highwater: left.host_local_highwater, + fingerprint: left.timeline_fingerprint.clone(), + }; + let right_window = HlsAlternativeOriginWindow { + host_local_media_sequence: right.host_local_media_sequence, + host_local_highwater: right.host_local_highwater, + fingerprint: right.timeline_fingerprint.clone(), + }; + host_local_windows_compatible(&left_window, &right_window) +} + +pub(super) fn host_local_windows_compatible( + left: &HlsAlternativeOriginWindow, + right: &HlsAlternativeOriginWindow, +) -> bool { + let left_fingerprint = &left.fingerprint; + let right_fingerprint = &right.fingerprint; + if HlsManifestTechnicalSignature::from_fingerprint(left_fingerprint) + != HlsManifestTechnicalSignature::from_fingerprint(right_fingerprint) + || left_fingerprint.segment_samples.is_empty() + || right_fingerprint.segment_samples.is_empty() + { + return false; + } + + let Some(left_highwater) = left.host_local_highwater else { + return false; + }; + let Some(right_highwater) = right.host_local_highwater else { + return false; + }; + let overlap_start = left.host_local_media_sequence.max(right.host_local_media_sequence); + let overlap_end = left_highwater.min(right_highwater); + if overlap_start <= overlap_end { + return (overlap_start..=overlap_end).all(|sequence| { + segment_in_window(left, sequence) + .zip(segment_in_window(right, sequence)) + .is_some_and(|(left, right)| segment_shape_matches(left, right)) + }); + } + + // Adjacent local windows are monotonic only when their non-resource shape + // and technical signatures agree. Origin sequence is used solely inside + // this already host-local cohort. + let gap = if left_highwater < right.host_local_media_sequence { + right.host_local_media_sequence.saturating_sub(left_highwater) + } else { + left.host_local_media_sequence.saturating_sub(right_highwater) + }; + gap <= 1 + && left_fingerprint.segment_count == right_fingerprint.segment_count + && left_fingerprint.duration_pattern_hash == right_fingerprint.duration_pattern_hash + && left_fingerprint.discontinuity_pattern_hash == right_fingerprint.discontinuity_pattern_hash +} + +fn segment_in_window(window: &HlsAlternativeOriginWindow, sequence: u64) -> Option<&HlsManifestSegmentFingerprint> { + let offset = sequence.checked_sub(window.host_local_media_sequence)?; + window.fingerprint.segment_samples.get(usize::try_from(offset).ok()?) +} + +pub(super) fn strong_anchor_overlap( + alternative: &HlsManifestCandidateObservation, + pinned: &HlsManifestCandidateObservation, +) -> u16 { + if alternative.timeline_fingerprint.map_and_encryption_hash != pinned.timeline_fingerprint.map_and_encryption_hash + || alternative.timeline_fingerprint.container_signature_hash + != pinned.timeline_fingerprint.container_signature_hash + { + return 0; + } + let mut longest_pdt_run = 0_usize; + let alternative_segments = &alternative.timeline_fingerprint.segment_samples; + let pinned_segments = &pinned.timeline_fingerprint.segment_samples; + for (alternative_index, alternative_segment) in alternative_segments.iter().enumerate() { + for (pinned_index, pinned_segment) in pinned_segments.iter().enumerate() { + if !pdt_intervals_overlap(alternative_segment, pinned_segment) { + continue; + } + let mut run = 0_usize; + while let (Some(alternative_sample), Some(pinned_sample)) = ( + alternative_segments.get(alternative_index.saturating_add(run)), + pinned_segments.get(pinned_index.saturating_add(run)), + ) { + if alternative_sample.duration_ms != pinned_sample.duration_ms + || alternative_sample.discontinuity_before != pinned_sample.discontinuity_before + || !pdt_intervals_overlap(alternative_sample, pinned_sample) + { + break; + } + run = run.saturating_add(1); + } + longest_pdt_run = longest_pdt_run.max(run); + } + } + if longest_pdt_run >= 2 { + u16::try_from(longest_pdt_run).unwrap_or(u16::MAX) + } else { + 0 + } +} + +fn pdt_intervals_overlap(left: &HlsManifestSegmentFingerprint, right: &HlsManifestSegmentFingerprint) -> bool { + if left.duration_ms != right.duration_ms { + return false; + } + let (Some(left_start), Some(right_start)) = (left.program_date_time_ms, right.program_date_time_ms) else { + return false; + }; + let left_duration = i64::try_from(left.duration_ms).unwrap_or(i64::MAX); + let right_duration = i64::try_from(right.duration_ms).unwrap_or(i64::MAX); + let left_end = left_start.saturating_add(left_duration); + let right_end = right_start.saturating_add(right_duration); + left_start < right_end && right_start < left_end +} + +fn segment_shape_matches(left: &HlsManifestSegmentFingerprint, right: &HlsManifestSegmentFingerprint) -> bool { + left.duration_ms == right.duration_ms + && left.discontinuity_before == right.discontinuity_before + && match (left.normalized_resource_identity, right.normalized_resource_identity) { + (Some(left), Some(right)) => left.matches(right), + (None, None) => true, + (Some(_), None) | (None, Some(_)) => false, + } +} diff --git a/backend/hls/src/manifest_fetch/error.rs b/backend/hls/src/manifest_fetch/error.rs index f3908f3a4..57608710e 100644 --- a/backend/hls/src/manifest_fetch/error.rs +++ b/backend/hls/src/manifest_fetch/error.rs @@ -221,6 +221,7 @@ pub enum HlsManifestRejectLogReason { CriticalHandoffLockContentionExhausted, StagedSwitchInvalidated, MalformedTransientTimeline, + StartupRepresentationChange, } impl HlsManifestRejectLogReason { @@ -271,6 +272,7 @@ impl HlsManifestRejectLogReason { Self::CriticalHandoffLockContentionExhausted => "critical-handoff-lock-contention-exhausted".to_string(), Self::StagedSwitchInvalidated => "staged-switch-invalidated".to_string(), Self::MalformedTransientTimeline => "malformed-transient-timeline".to_string(), + Self::StartupRepresentationChange => "startup-representation-change".to_string(), } } } diff --git a/backend/hls/src/manifest_snapshot.rs b/backend/hls/src/manifest_snapshot.rs index 408b75ea4..de7015f94 100644 --- a/backend/hls/src/manifest_snapshot.rs +++ b/backend/hls/src/manifest_snapshot.rs @@ -192,7 +192,7 @@ fn derive_normal_snapshot( { return None; } - snapshot_from_segments(HlsLeaseManifestSnapshotParts { + let mut snapshot = snapshot_from_segments(HlsLeaseManifestSnapshotParts { delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, source_commit_identity, uri_materialization: None, @@ -203,7 +203,19 @@ fn derive_normal_snapshot( segments, active_map: parsed.units.last().and_then(|unit| unit.active_map.clone()), active_encryption: parsed.units.last().and_then(|unit| unit.active_encryption.clone()), - }) + })?; + if let Some(startup) = &session.startup { + let revisions = visible_proxy_seqs + .iter() + .map(|seq| Some((*seq, startup.revisions.get(seq)?.clone()))) + .collect::>>()?; + snapshot.startup_revisions = Some(Arc::new(super::HlsManifestRevisions { + mode: startup.config.mode, + retained_start_seq: session.segments.first_key_value().map_or(first_proxy_seq, |(seq, _)| *seq), + revisions, + })); + } + Some(snapshot) })()) } @@ -217,6 +229,7 @@ fn derive_transient_snapshot( let first_proxy_seq = template.segments.first()?.proxy_seq; let last_proxy_seq = template.segments.last()?.proxy_seq; Some(HlsLeaseManifestSnapshot { + startup_revisions: None, delivery_mode: HlsManifestDeliveryMode::TransientPassthrough, source_commit_identity, uri_materialization, @@ -268,6 +281,7 @@ fn snapshot_from_segments(parts: HlsLeaseManifestSnapshotParts) -> Option>, pub delivery_mode: HlsManifestDeliveryMode, pub source_commit_identity: HlsManifestCommitIdentity, pub uri_materialization: Option, @@ -547,7 +548,25 @@ pub fn evaluate_startup_admission(input: HlsStartupAdmissionInput<'_>) -> HlsSta guaranteed_transition_margin(input.ready_timeline, guaranteed_ready_span, visible_tail_ms) .unwrap_or(input.manifest.target_duration_ms), ); - let decision = if visible_contiguous_ready_duration_ms < minimum_startup_duration_ms { + let accelerated = input.manifest.startup_revisions.as_ref().is_some_and(|publication| { + input.origin_state != HlsStartupAdmissionOriginState::Degraded + && input + .manifest + .visible_segments + .iter() + .all(|segment| segment.encryption.is_none() && segment.map_ref_ready) + && input.manifest.visible_segments.iter().all(|segment| { + publication + .revisions + .get(&segment.proxy_seq) + .is_some_and(|revision| revision.revision().is_publishable()) + }) + && publication + .revisions + .get(&input.manifest.first_proxy_seq) + .is_some_and(|head| head.revision().is_startup_head_ready(publication.mode)) + }); + let decision = if !accelerated && visible_contiguous_ready_duration_ms < minimum_startup_duration_ms { HlsStartupAdmissionDecision::Reject(HlsStartupAdmissionRejection::InsufficientVisibleReadyMedia) } else if input.origin_state == HlsStartupAdmissionOriginState::Degraded && prospective_lease_reserve_ms @@ -572,6 +591,7 @@ mod tests { fn manifest() -> HlsLeaseManifestSnapshot { HlsLeaseManifestSnapshot { + startup_revisions: None, delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, source_commit_identity: HlsManifestCommitIdentity::new(1), uri_materialization: None, @@ -910,6 +930,88 @@ mod tests { assert!(!reserve.cutover_required); } + #[test] + fn accelerated_admission_requires_every_published_prefix_and_a_complete_processed_head() -> std::io::Result<()> { + let store = crate::SegmentRevisionStore::default(); + let mut manifest = manifest(); + let mut owners = std::collections::BTreeMap::new(); + for seq in 0..3 { + owners.insert( + seq, + store.create(crate::ProxySessionId("admission".into()), seq, crate::SegmentRevisionKind::Processed)?, + ); + } + manifest.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + mode: shared::model::HlsStartupMode::FirstReady, + retained_start_seq: 0, + revisions: owners, + })); + let publication = manifest.startup_revisions.as_ref().ok_or_else(|| std::io::Error::other("publication"))?; + for owner in publication.revisions.values() { + owner.revision().prefix_available.store(1, std::sync::atomic::Ordering::Release); + } + let timeline = HlsReadyTimelineSnapshot { units: Arc::from([]) }; + assert!(matches!( + admission(&manifest, &timeline, HlsStartupAdmissionOriginState::Healthy, 0).decision, + HlsStartupAdmissionDecision::Reject(_) + )); + let head = publication.revisions.get(&0).ok_or_else(|| std::io::Error::other("head"))?; + head.revision().complete(crate::CachedSegmentMetadata { path: "head.ts".into(), size: 1 }); + assert_eq!( + admission(&manifest, &timeline, HlsStartupAdmissionOriginState::Healthy, 0).decision, + HlsStartupAdmissionDecision::Admit + ); + assert!(matches!( + admission(&manifest, &timeline, HlsStartupAdmissionOriginState::Degraded, 0).decision, + HlsStartupAdmissionDecision::Reject(_) + )); + Arc::make_mut(manifest.startup_revisions.as_mut().ok_or_else(|| std::io::Error::other("publication"))?) + .revisions + .remove(&2); + assert!(matches!( + admission(&manifest, &timeline, HlsStartupAdmissionOriginState::Healthy, 0).decision, + HlsStartupAdmissionDecision::Reject(_) + )); + Ok(()) + } + + #[test] + fn progressive_admission_rejects_discovered_header_only_and_failed_owners() -> std::io::Result<()> { + let store = crate::SegmentRevisionStore::default(); + let mut manifest = manifest(); + let mut owners = std::collections::BTreeMap::new(); + for seq in 0..3 { + owners.insert( + seq, + store.create(crate::ProxySessionId("admission".into()), seq, crate::SegmentRevisionKind::Raw)?, + ); + } + manifest.startup_revisions = Some(Arc::new(crate::HlsManifestRevisions { + mode: shared::model::HlsStartupMode::Progressive, + retained_start_seq: 0, + revisions: owners, + })); + let timeline = HlsReadyTimelineSnapshot { units: Arc::from([]) }; + assert!(matches!( + admission(&manifest, &timeline, HlsStartupAdmissionOriginState::Healthy, 0).decision, + HlsStartupAdmissionDecision::Reject(_) + )); + let publication = manifest.startup_revisions.as_ref().ok_or_else(|| std::io::Error::other("publication"))?; + for owner in publication.revisions.values() { + owner.revision().prefix_available.store(1, std::sync::atomic::Ordering::Release); + } + assert_eq!( + admission(&manifest, &timeline, HlsStartupAdmissionOriginState::Healthy, 0).decision, + HlsStartupAdmissionDecision::Admit + ); + publication.revisions.get(&1).ok_or_else(|| std::io::Error::other("successor"))?.revision().fail(); + assert!(matches!( + admission(&manifest, &timeline, HlsStartupAdmissionOriginState::Healthy, 0).decision, + HlsStartupAdmissionDecision::Reject(_) + )); + Ok(()) + } + #[test] fn healthy_startup_admission_uses_contiguous_visible_ready_window() { let admission = admission(&manifest(), &ready_timeline(), HlsStartupAdmissionOriginState::Healthy, u64::MAX); diff --git a/backend/hls/src/observability.rs b/backend/hls/src/observability.rs index b731a7536..7d9ae134f 100644 --- a/backend/hls/src/observability.rs +++ b/backend/hls/src/observability.rs @@ -20,6 +20,10 @@ pub use tuliprox_core::utils::content_coding::{ /// In-memory counters for the live HLS cache path. #[derive(Debug, Default)] pub struct HlsCacheMetrics { + revision_disk_bytes: AtomicU64, + progressive_replay_bytes: AtomicU64, + progressive_active_fills: AtomicU64, + deferred_repairs: AtomicU64, sessions_created: AtomicU64, sessions_reused: AtomicU64, lease_granted: AtomicU64, @@ -56,6 +60,10 @@ pub struct HlsCacheMetrics { #[derive(Debug, Clone, Default, Eq, PartialEq)] pub struct HlsCacheMetricsSnapshot { + pub revision_disk_bytes: u64, + pub progressive_replay_bytes: u64, + pub progressive_active_fills: u64, + pub deferred_repairs: u64, pub sessions_created: u64, pub sessions_reused: u64, pub lease_granted: u64, @@ -91,6 +99,12 @@ pub struct HlsCacheMetricsSnapshot { } impl HlsCacheMetrics { + pub fn record_progressive_usage(&self, fills: usize, bytes: usize, repairs: usize) { + self.progressive_active_fills.store(u64::try_from(fills).unwrap_or(u64::MAX), Ordering::Relaxed); + self.progressive_replay_bytes.store(u64::try_from(bytes).unwrap_or(u64::MAX), Ordering::Relaxed); + self.deferred_repairs.store(u64::try_from(repairs).unwrap_or(u64::MAX), Ordering::Relaxed); + } + pub fn record_revision_disk_bytes(&self, bytes: u64) { self.revision_disk_bytes.store(bytes, Ordering::Relaxed); } pub fn record_session_created(&self) { increment(&self.sessions_created, 1); } pub fn record_session_reused(&self) { increment(&self.sessions_reused, 1); } pub fn record_lease_granted(&self) { increment(&self.lease_granted, 1); } @@ -142,6 +156,10 @@ impl HlsCacheMetrics { pub fn snapshot(&self) -> HlsCacheMetricsSnapshot { HlsCacheMetricsSnapshot { + revision_disk_bytes: load(&self.revision_disk_bytes), + progressive_replay_bytes: load(&self.progressive_replay_bytes), + progressive_active_fills: load(&self.progressive_active_fills), + deferred_repairs: load(&self.deferred_repairs), sessions_created: load(&self.sessions_created), sessions_reused: load(&self.sessions_reused), lease_granted: load(&self.lease_granted), diff --git a/backend/hls/src/origin.rs b/backend/hls/src/origin.rs index 2d8b270c9..1c533730f 100644 --- a/backend/hls/src/origin.rs +++ b/backend/hls/src/origin.rs @@ -531,9 +531,17 @@ impl HlsBoundAccountAcquireErrorKind { pub fn allows_rebind(self) -> bool { matches!(self, Self::Missing | Self::Expired) } pub fn is_retryable_resource_failure(self) -> bool { + // A competing session or an in-flight binding handoff can temporarily + // deny origin I/O. Retry acquisition within the existing resource + // budget; never bypass ownership checks or rebind on this error. matches!( self, - Self::Exhausted | Self::WaitTimedOut | Self::AcquireTimedOut | Self::StoreRace | Self::Unavailable + Self::Exhausted + | Self::ReservedForOther + | Self::WaitTimedOut + | Self::AcquireTimedOut + | Self::StoreRace + | Self::Unavailable ) } @@ -1359,7 +1367,7 @@ mod tests { fn bound_account_acquire_error_resource_retry_policy_matches_handoff_semantics() { assert!(!HlsBoundAccountAcquireErrorKind::Missing.is_retryable_resource_failure()); assert!(!HlsBoundAccountAcquireErrorKind::Expired.is_retryable_resource_failure()); - assert!(!HlsBoundAccountAcquireErrorKind::ReservedForOther.is_retryable_resource_failure()); + assert!(HlsBoundAccountAcquireErrorKind::ReservedForOther.is_retryable_resource_failure()); assert!(!HlsBoundAccountAcquireErrorKind::Detached.is_retryable_resource_failure()); assert!(HlsBoundAccountAcquireErrorKind::Exhausted.is_retryable_resource_failure()); assert!(HlsBoundAccountAcquireErrorKind::WaitTimedOut.is_retryable_resource_failure()); diff --git a/backend/hls/src/progressive_budget.rs b/backend/hls/src/progressive_budget.rs new file mode 100644 index 000000000..503008842 --- /dev/null +++ b/backend/hls/src/progressive_budget.rs @@ -0,0 +1,263 @@ +use crate::sync_ext::MutexExt; +use bytes::Bytes; +use shared::model::Bytes as ConfigBytes; +use std::{ + io, + sync::{Arc, Mutex}, +}; +use tuliprox_core::model::HlsStartupConfig; + +/// Fixed budget charge for one claimed progressive slot. +pub(crate) const PROGRESSIVE_SLOT_FIXED_BYTES: usize = 512; +/// Allocator bookkeeping charged on top of every replay chunk. +const PROGRESSIVE_CHUNK_OVERHEAD_BYTES: usize = std::mem::size_of::() + 64; + +fn exceeds(bytes: usize, limit: ConfigBytes) -> bool { !u64::try_from(bytes).is_ok_and(|bytes| bytes <= limit.get()) } + +#[derive(Debug)] +struct Usage { + slots: usize, + bytes: usize, + repairs: usize, + limits: HlsStartupConfig, +} + +#[derive(Debug)] +pub struct ProgressiveBudgetManager { + usage: Mutex, + changed: tokio::sync::Notify, +} + +impl ProgressiveBudgetManager { + pub fn new(limits: HlsStartupConfig) -> Arc { + Arc::new(Self { + usage: Mutex::new(Usage { slots: 0, bytes: 0, repairs: 0, limits }), + changed: tokio::sync::Notify::new(), + }) + } + + pub fn update_limits(&self, limits: HlsStartupConfig) { + self.usage.lock_unpoisoned().limits = limits; + self.changed.notify_waiters(); + } + + pub fn try_claim(self: &Arc) -> Option { + let mut usage = self.usage.lock_unpoisoned(); + if usage.slots >= usage.limits.max_progressive_segments { + return None; + } + let fixed = PROGRESSIVE_SLOT_FIXED_BYTES; + if exceeds(usage.bytes.saturating_add(fixed), usage.limits.max_progressive_bytes_total) { + return None; + } + usage.slots += 1; + usage.bytes += fixed; + Some(ProgressiveSlot { manager: Arc::clone(self), bytes: fixed, fixed, metadata_bytes: 0 }) + } + + pub async fn wait_for_repair(self: &Arc, deadline: tokio::time::Instant) -> io::Result { + loop { + let changed = self.changed.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + let available = { + let mut usage = self.usage.lock_unpoisoned(); + if usage.repairs < usage.limits.max_deferred_repairs { + usage.repairs += 1; + true + } else { + false + } + }; + if available { + return Ok(DeferredRepairPermit { manager: Arc::clone(self) }); + } + tokio::time::timeout_at(deadline, changed) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "deferred HLS repair capacity timed out"))?; + } + } + + pub fn usage_snapshot(&self) -> (usize, usize, usize) { + let usage = self.usage.lock_unpoisoned(); + (usage.slots, usage.bytes, usage.repairs) + } + + pub fn usage(&self) -> (usize, usize) { + let usage = self.usage.lock_unpoisoned(); + (usage.slots, usage.bytes) + } +} + +pub struct ProgressiveSlot { + fixed: usize, + manager: Arc, + bytes: usize, + metadata_bytes: usize, +} + +impl ProgressiveSlot { + pub fn copy_chunk(&mut self, data: &[u8]) -> io::Result { + let charge = data + .len() + .checked_add(PROGRESSIVE_CHUNK_OVERHEAD_BYTES) + .ok_or_else(|| io::Error::other("progressive chunk size overflow"))?; + self.reserve_bytes(charge)?; + let mut data_copy = Vec::new(); + if let Err(error) = data_copy.try_reserve_exact(data.len()) { + self.release_failed_reservation(charge); + return Err(io::Error::other(error)); + } + let excess = data_copy.capacity().saturating_sub(data.len()); + if excess > 0 { + if let Err(error) = self.reserve_bytes(excess) { + self.release_failed_reservation(charge); + return Err(error); + } + } + data_copy.extend_from_slice(data); + Ok(Bytes::from_owner(BudgetedChunk { + data: data_copy, + charge: charge + excess, + manager: Arc::clone(&self.manager), + })) + } + + fn reserve_bytes(&mut self, charge: usize) -> io::Result<()> { + let mut usage = self.manager.usage.lock_unpoisoned(); + let next_segment = + self.bytes.checked_add(charge).ok_or_else(|| io::Error::other("progressive segment budget overflow"))?; + let next_total = + usage.bytes.checked_add(charge).ok_or_else(|| io::Error::other("progressive byte budget overflow"))?; + if exceeds(next_segment, usage.limits.max_progressive_bytes_per_segment) + || exceeds(next_total, usage.limits.max_progressive_bytes_total) + { + return Err(io::Error::new(io::ErrorKind::OutOfMemory, "progressive byte budget exhausted")); + } + usage.bytes = next_total; + self.bytes = next_segment; + Ok(()) + } + + pub fn reserve_metadata(&mut self, bytes: usize) -> io::Result<()> { + self.reserve_bytes(bytes)?; + self.metadata_bytes += bytes; + Ok(()) + } + + pub fn release_metadata(&mut self, bytes: usize) { + self.release_failed_reservation(bytes); + self.metadata_bytes = self.metadata_bytes.saturating_sub(bytes); + } + + fn release_failed_reservation(&mut self, charge: usize) { + let mut usage = self.manager.usage.lock_unpoisoned(); + usage.bytes = usage.bytes.saturating_sub(charge); + self.bytes = self.bytes.saturating_sub(charge); + } +} + +impl Drop for ProgressiveSlot { + fn drop(&mut self) { + let mut usage = self.manager.usage.lock_unpoisoned(); + usage.slots = usage.slots.saturating_sub(1); + usage.bytes = usage.bytes.saturating_sub(self.fixed.saturating_add(self.metadata_bytes)); + } +} + +struct BudgetedChunk { + data: Vec, + charge: usize, + manager: Arc, +} +impl AsRef<[u8]> for BudgetedChunk { + fn as_ref(&self) -> &[u8] { &self.data } +} +impl Drop for BudgetedChunk { + fn drop(&mut self) { + drop(std::mem::take(&mut self.data)); + let mut usage = self.manager.usage.lock_unpoisoned(); + usage.bytes = usage.bytes.saturating_sub(self.charge); + } +} + +pub struct DeferredRepairPermit { + manager: Arc, +} +impl Drop for DeferredRepairPermit { + fn drop(&mut self) { + let mut usage = self.manager.usage.lock_unpoisoned(); + usage.repairs = usage.repairs.saturating_sub(1); + drop(usage); + self.manager.changed.notify_waiters(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clones_and_slices_keep_the_allocation_charged_after_slot_release() -> io::Result<()> { + let manager = ProgressiveBudgetManager::new(HlsStartupConfig::default()); + let mut slot = manager.try_claim().ok_or_else(|| io::Error::other("slot"))?; + let chunk = slot.copy_chunk(&[1; 188])?; + let slice = chunk.slice(1..3); + let charged = manager.usage().1; + drop(chunk); + drop(slot); + assert_eq!(manager.usage(), (0, charged.saturating_sub(PROGRESSIVE_SLOT_FIXED_BYTES))); + drop(slice); + assert_eq!(manager.usage(), (0, 0)); + Ok(()) + } + + #[tokio::test] + async fn deferred_repair_waiters_wake_on_release_and_reload() -> io::Result<()> { + let mut limits = HlsStartupConfig { max_deferred_repairs: 1, ..Default::default() }; + let manager = ProgressiveBudgetManager::new(limits.clone()); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1); + let first = manager.wait_for_repair(deadline).await?; + let waiting_manager = Arc::clone(&manager); + let waiting = tokio::spawn(async move { waiting_manager.wait_for_repair(deadline).await }); + tokio::task::yield_now().await; + assert!(!waiting.is_finished()); + drop(first); + let second = waiting.await.map_err(io::Error::other)??; + let waiting_manager = Arc::clone(&manager); + let waiting = tokio::spawn(async move { waiting_manager.wait_for_repair(deadline).await }); + tokio::task::yield_now().await; + assert!(!waiting.is_finished()); + limits.max_deferred_repairs = 2; + manager.update_limits(limits); + let third = waiting.await.map_err(io::Error::other)??; + drop(second); + drop(third); + assert_eq!(manager.usage.lock().map_err(|_| io::Error::other("budget lock"))?.repairs, 0); + Ok(()) + } + + #[test] + fn total_limit_and_reload_apply_to_all_slots() -> io::Result<()> { + let mut limits = HlsStartupConfig { + max_progressive_bytes_total: ConfigBytes::new(4096), + max_progressive_bytes_per_segment: ConfigBytes::new(4096), + ..Default::default() + }; + let manager = ProgressiveBudgetManager::new(limits.clone()); + let mut first = manager.try_claim().ok_or_else(|| io::Error::other("slot"))?; + let mut second = manager.try_claim().ok_or_else(|| io::Error::other("slot"))?; + let retained = first.copy_chunk(&[0; 2000])?; + assert!(second.copy_chunk(&[0; 2000]).is_err()); + limits.max_progressive_bytes_total = ConfigBytes::new(1); + limits.max_progressive_segments = 1; + manager.update_limits(limits); + assert!(manager.try_claim().is_none()); + assert!(first.copy_chunk(&[0]).is_err()); + drop(retained); + drop(first); + drop(second); + assert_eq!(manager.usage(), (0, 0)); + Ok(()) + } +} diff --git a/backend/hls/src/progressive_startup.rs b/backend/hls/src/progressive_startup.rs new file mode 100644 index 000000000..e9af89fbd --- /dev/null +++ b/backend/hls/src/progressive_startup.rs @@ -0,0 +1,344 @@ +use super::{ProgressiveBudgetManager, SegmentRevisionGuard, SegmentRevisionState}; +use crate::sync_ext::MutexExt; +use bytes::Bytes; +use futures::{Stream, StreamExt}; +use std::{ + io::{self, Read, Seek, SeekFrom}, + path::PathBuf, + pin::Pin, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + sync::{mpsc, watch}, + time::{timeout_at, Instant}, +}; + +/// Largest chunk a reader takes from a completed revision file per read. +const REVISION_FILE_CHUNK_BYTES: u64 = 16 * 1024; + +pub struct ProgressiveReplay { + pub slot: super::progressive_budget::ProgressiveSlot, + chunks: Vec<(u64, Bytes)>, +} + +impl std::fmt::Debug for ProgressiveReplay { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProgressiveReplay").field("chunks", &self.chunks.len()).finish_non_exhaustive() + } +} + +impl ProgressiveReplay { + pub fn new(budget: &Arc) -> Option { + budget.try_claim().map(|slot| Self { slot, chunks: Vec::new() }) + } + pub fn push(&mut self, chunk: &[u8]) -> io::Result<()> { + let start = self + .chunks + .last() + .map_or(Some(0), |(start, bytes)| start.checked_add(u64::try_from(bytes.len()).ok()?)) + .ok_or_else(|| io::Error::other("progressive replay offset overflow"))?; + if self.chunks.len() == self.chunks.capacity() { + self.grow_chunk_index()?; + } + let chunk = self.slot.copy_chunk(chunk)?; + self.chunks.push((start, chunk)); + Ok(()) + } + fn grow_chunk_index(&mut self) -> io::Result<()> { + let item_size = std::mem::size_of::<(u64, Bytes)>(); + let old_bytes = self + .chunks + .capacity() + .checked_mul(item_size) + .ok_or_else(|| io::Error::other("replay index capacity overflow"))?; + let capacity = self + .chunks + .capacity() + .max(2) + .checked_mul(2) + .ok_or_else(|| io::Error::other("replay index growth overflow"))?; + let reserved = + capacity.checked_mul(item_size).ok_or_else(|| io::Error::other("replay index budget overflow"))?; + self.slot.reserve_metadata(reserved)?; + let mut next = Vec::new(); + if let Err(error) = next.try_reserve_exact(capacity) { + self.slot.release_metadata(reserved); + return Err(io::Error::other(error)); + } + let actual = next + .capacity() + .checked_mul(item_size) + .ok_or_else(|| io::Error::other("replay allocation capacity overflow"))?; + if let Err(error) = self.slot.reserve_metadata(actual.saturating_sub(reserved)) { + drop(next); + self.slot.release_metadata(reserved); + return Err(error); + } + next.append(&mut self.chunks); + drop(std::mem::replace(&mut self.chunks, next)); + self.slot.release_metadata(old_bytes); + Ok(()) + } + + fn at_offset(&self, offset: u64) -> Option { + let index = self.chunks.partition_point(|(start, _)| *start <= offset).checked_sub(1)?; + let (start, chunk) = self.chunks.get(index)?; + let local_offset = usize::try_from(offset.checked_sub(*start)?).ok()?; + (local_offset < chunk.len()).then(|| chunk.slice(local_offset..)) + } +} + +impl Drop for ProgressiveReplay { + fn drop(&mut self) { + // The index allocation is released before its slot-owned metadata budget. + drop(std::mem::take(&mut self.chunks)); + } +} + +/// An open revision file that owns its own pin, so GC cannot retire the revision +/// while any handle to it is open, including one inside an in-flight blocking read. +struct PinnedRevisionFile { + file: std::fs::File, + // Fields drop in declaration order: the handle closes before the pin is released. + _pin: SegmentRevisionGuard, +} + +/// Reads one response body on its own task, so the deadline can cancel it (and close +/// its file) while a stalled client keeps the response stream alive unpolled. +struct RevisionReader { + guard: SegmentRevisionGuard, + changes: watch::Receiver, + file: Option>, + offset: u64, + end: Option, +} + +impl RevisionReader { + async fn run(mut self, sender: mpsc::Sender>, finished: Arc) { + let failure = loop { + let next = tokio::select! { + biased; + () = sender.closed() => return, + next = self.next() => next, + }; + match next { + Ok(Some(chunk)) => { + if sender.send(Ok(chunk)).await.is_err() { + return; + } + } + Ok(None) => break None, + Err(error) => break Some(error), + } + }; + // The file and pin are released before a stalled client receives the last item. + drop(self); + finished.store(true, Ordering::Release); + if let Some(error) = failure { + let _ = sender.send(Err(error)).await; + } + } + + async fn next(&mut self) -> io::Result> { + loop { + let state = self.changes.borrow_and_update().clone(); + match state { + SegmentRevisionState::Failed => { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "HLS published origin fill failed")) + } + SegmentRevisionState::Complete(metadata) => { + let end = self.end.unwrap_or(metadata.size).min(metadata.size); + if self.offset >= end { + return Ok(None); + } + let limit = usize::try_from(end.saturating_sub(self.offset).min(REVISION_FILE_CHUNK_BYTES)) + .map_err(io::Error::other)?; + let file = self.file_at_offset(metadata.path).await?; + let bytes = run_blocking(move || { + let mut bytes = vec![0u8; limit]; + let count = (&file.file).read(&mut bytes)?; + bytes.truncate(count); + Ok(bytes) + }) + .await?; + if bytes.is_empty() { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "immutable HLS revision was truncated", + )); + } + self.offset = self.offset.saturating_add(u64::try_from(bytes.len()).map_err(io::Error::other)?); + return Ok(Some(Bytes::from(bytes))); + } + SegmentRevisionState::Pending => { + let chunk = self + .guard + .revision() + .replay + .lock_unpoisoned() + .as_ref() + .and_then(|replay| replay.at_offset(self.offset)); + if let Some(chunk) = chunk { + self.offset = self.offset.saturating_add(u64::try_from(chunk.len()).map_err(io::Error::other)?); + return Ok(Some(chunk)); + } + self.changes + .changed() + .await + .map_err(|_| io::Error::new(io::ErrorKind::UnexpectedEof, "HLS revision producer ended"))?; + } + } + } + } + + /// A reused handle is already positioned at `offset`. + async fn file_at_offset(&mut self, path: PathBuf) -> io::Result> { + if let Some(file) = &self.file { + return Ok(Arc::clone(file)); + } + let pin = self.guard.clone(); + let offset = self.offset; + let file = run_blocking(move || { + let mut file = std::fs::File::open(path)?; + file.seek(SeekFrom::Start(offset))?; + Ok(Arc::new(PinnedRevisionFile { file, _pin: pin })) + }) + .await?; + Ok(Arc::clone(self.file.insert(file))) + } +} + +async fn run_blocking(work: impl FnOnce() -> io::Result + Send + 'static) -> io::Result { + tokio::task::spawn_blocking(work).await.map_err(io::Error::other)? +} + +struct RevisionBodyState { + items: mpsc::Receiver>, + finished: Arc, + deadline: Instant, + ended: bool, +} + +type RevisionBody = Pin> + Send>>; + +pub fn revision_body(guard: SegmentRevisionGuard, lifetime: Duration, start: u64, end: Option) -> RevisionBody { + let deadline = Instant::now() + lifetime; + let (sender, items) = mpsc::channel(1); + let finished = Arc::new(AtomicBool::new(false)); + let changes = guard.revision().subscribe(); + let reader = RevisionReader { guard, changes, file: None, offset: start, end }; + let mut reader = tokio::spawn(reader.run(sender, Arc::clone(&finished))); + tokio::spawn(async move { + if timeout_at(deadline, &mut reader).await.is_err() { + reader.abort(); + // Awaiting the cancelled task drops its guard and file handle before this task ends. + let _ = reader.await; + } + }); + futures::stream::unfold(RevisionBodyState { items, finished, deadline, ended: false }, |mut state| async move { + if state.ended { + return None; + } + // The lifetime is absolute: data still buffered at the deadline is not served. + let next = if Instant::now() < state.deadline { + timeout_at(state.deadline, state.items.recv()).await.ok() + } else { + None + }; + match next { + Some(Some(item)) => Some((item, state)), + Some(None) if state.finished.load(Ordering::Acquire) => None, + Some(None) => { + state.ended = true; + Some(( + Err(io::Error::new(io::ErrorKind::UnexpectedEof, "HLS revision reader ended unexpectedly")), + state, + )) + } + None => { + state.ended = true; + Some((Err(io::Error::new(io::ErrorKind::TimedOut, "HLS revision reader lifetime exceeded")), state)) + } + } + }) + .boxed() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn replay_index_growth_and_retained_chunk_have_separate_budget_lifetimes() -> io::Result<()> { + let budget = ProgressiveBudgetManager::new(tuliprox_core::model::HlsStartupConfig::default()); + let mut replay = ProgressiveReplay::new(&budget).ok_or_else(|| io::Error::other("slot"))?; + for _ in 0..100 { + replay.push(b"x")?; + } + let retained = replay.at_offset(99).ok_or_else(|| io::Error::other("retained chunk"))?; + let total = budget.usage().1; + let index_bytes = replay.chunks.capacity() * std::mem::size_of::<(u64, Bytes)>(); + assert!(total > index_bytes); + drop(replay); + assert_eq!(budget.usage().0, 0); + assert!(budget.usage().1 > 0 && budget.usage().1 < index_bytes); + assert_eq!(retained.as_ref(), b"x"); + drop(retained); + assert_eq!(budget.usage(), (0, 0)); + Ok(()) + } + + #[tokio::test] + async fn unpolled_reader_detaches_at_absolute_deadline() -> io::Result<()> { + let store = super::super::SegmentRevisionStore::default(); + let revision = + store.create(super::super::ProxySessionId("deadline".into()), 0, super::super::SegmentRevisionKind::Raw)?; + let body = revision_body(revision.clone(), Duration::from_millis(10), 0, None); + drop(revision); + assert!(store.retire_unpinned()?.is_empty()); + tokio::time::sleep(Duration::from_millis(30)).await; + assert_eq!(store.retire_unpinned()?.len(), 1); + drop(body); + Ok(()) + } + #[tokio::test] + async fn stalled_reader_closes_its_revision_file_at_the_deadline() -> io::Result<()> { + use futures::FutureExt; + + let directory = tempfile::tempdir()?; + let path = directory.path().join("revision.ts"); + let size = REVISION_FILE_CHUNK_BYTES * 8; + tokio::fs::write(&path, vec![0x47; usize::try_from(size).map_err(io::Error::other)?]).await?; + let store = super::super::SegmentRevisionStore::default(); + let revision = + store.create(super::super::ProxySessionId("stalled".into()), 0, super::super::SegmentRevisionKind::Raw)?; + assert!(revision.revision().complete(super::super::CachedSegmentMetadata { path, size })); + let mut body = revision_body(revision.clone(), Duration::from_millis(100), 0, None); + + let first = body.next().await.ok_or_else(|| io::Error::other("first chunk"))??; + assert_eq!(u64::try_from(first.len()).map_err(io::Error::other)?, REVISION_FILE_CHUNK_BYTES); + // The client polls the next read once and then stops reading mid-response. + let _ = body.next().now_or_never(); + // The open file owns its own pin, so the pin count observes the handle on every platform: + // this test, the reader and the open revision file. + assert_eq!(revision.revision().pin_count(), 3, "an unfinished reader keeps its revision file open"); + + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!( + revision.revision().pin_count(), + 1, + "the deadline closes the file while the response stream is alive" + ); + drop(revision); + assert_eq!(store.retire_unpinned()?.len(), 1, "the deadline releases the revision pin"); + let tail: Vec<_> = body.collect().await; + assert!(tail + .last() + .is_some_and(|item| item.as_ref().is_err_and(|error| error.kind() == io::ErrorKind::TimedOut))); + Ok(()) + } +} diff --git a/backend/hls/src/refresh/commit.rs b/backend/hls/src/refresh/commit.rs index 6a35bba20..2c3078553 100644 --- a/backend/hls/src/refresh/commit.rs +++ b/backend/hls/src/refresh/commit.rs @@ -176,6 +176,9 @@ pub(super) fn commit_fetched_manifest_with_acceptance_mode( ), }; if result.is_ok() { + if matches!(session.mode, HlsSessionMode::TransientPassthrough { .. }) { + session.startup = None; + } clear_satisfied_fresh_manifest_requirement(session, request); } let configured_target_duration = session.target_duration; @@ -274,6 +277,11 @@ fn commit_transient_fetched_manifest( fetch_finished_at_ms: u64, options: HlsTransientFetchedCommitOptions, ) -> Result<(HlsManifestCommitProgressEvidence, bool, bool), HlsManifestCommitError> { + if session.startup.is_some() && session.published_live_origin_baseline.is_some() { + return Err(HlsManifestCommitError::TimelineRejected { + reason: HlsManifestRejectLogReason::StartupRepresentationChange, + }); + } let HlsTransientFetchedCommitOptions { acceptance_mode, reason, manifest_semantics, switch_metric } = options; let timeline = parse_transient_manifest_timeline_for_commit(session, &fetched.body)?; let quality = evaluate_manifest_acceptance_for_commit( diff --git a/backend/hls/src/refresh/tests.rs b/backend/hls/src/refresh/tests.rs index eb6a883bb..a72e98143 100644 --- a/backend/hls/src/refresh/tests.rs +++ b/backend/hls/src/refresh/tests.rs @@ -43,7 +43,6 @@ use super::{ commit_fetched_manifest, key_resource_extension, record_committed_manifest_media_progress, refresh_origin_work_generation_matches, transient_reason_log_fields, HlsManifestCommitProgressEvidence, }, - current_time_millis, failure::{ apply_manifest_fetch_failure_signal, classify_manifest_fetch_failure, manifest_hard_fetch_error, request_error_indicates_timeout, HlsManifestFetchFailureKind, HlsManifestFetchFailureSignal, @@ -60,6860 +59,33 @@ use super::{ HlsOriginRefreshTriggerOutcome, HlsPostRefreshAvailabilityAction, HlsPostRefreshAvailabilityReason, HlsPostRefreshRuntime, OriginRefreshRequest, OriginRefreshState, }; -use crate::{ - build_rewrite_secret_fingerprint, is_hls_provisioning_gap_segment, is_hls_provisioning_segment, - manifest_fetch::fetched_effective_manifest_host, - media_reserve::{HlsLeaseReserveAvailabilityBasis, HlsLeaseReserveSnapshot}, - refresh::maybe_trigger_origin_refresh, - terminal_tail::HlsLeasePlaybackMode, - timeline::HLS_PROVISIONING_ORIGIN_EPOCH, - CacheAccessState, GarbageCollectionPolicy, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, - HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsBoundAccountAcquireErrorKind, HlsFreshManifestRequiredReason, - HlsGarbageCollector, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsManifestAcceptanceDirective, - HlsManifestAcceptanceExhaustionReason, HlsManifestCommitIdentity, HlsManifestDeliveryMode, HlsMapWorkerPool, - HlsMediaContainer, HlsOriginAccountBinding, HlsOriginIoContext, HlsPlaybackFamilyKey, - HlsPreparedTerminalBundleState, HlsProxyManager, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentWorkerPool, - HlsSession, HlsSessionKey, HlsSessionMode, HlsSessionStore, HlsTerminalTailCompatibility, MapCacheStatus, - OriginSegmentKey, ProxySessionId, RenderedManifest, RenderedManifestStoreOutcome, SegmentCacheKey, - SegmentCacheStatus, SegmentEntry, SegmentFetchPolicy, SegmentFetchPriority, TimelineMapError, - TransientObjectFetchDecision, TransientPassthroughReason, TransientResourceFile, TransientResourceId, - TransientResourceKind, + +mod acceptance; +mod archive_master; +mod critical_handoff; +mod headers_redirects; +mod origin_binding; +mod ownership; +mod retry_policy; +mod startup_representation; +mod support; +mod switch_staging; +mod timing; +mod transient; + +use self::support::{ + assert_incompatible_switch_is_rejected_before_timeline_commit, assert_stale_switch_generation_rejects_commit, + await_controlled_switch_segment_prefix, bind_refresh_request_to_app_state, candidate_handoff_preview, + commit_ready_baseline_snapshot, critical_handoff_app_config, fetched_manifest, host_from_base_url, + install_published_recovery_binding, manifest_body, mark_full_burst_ready_for_switch_staging, no_delay_policy, + path_has_extension, post_refresh_live_manifest_snapshot, prepare_active_critical_handoff_lease, + prepare_cross_host_baseline, publish_ready_test_manifest, refresh_session_with_origin_body, + repeated_transient_manifest, request_header_value, retry_test_manifest_recovery_chain, + spawn_controlled_switch_origin, spawn_critical_emergency_origin, spawn_critical_emergency_origin_with_control, + spawn_test_origin, switch_fetched_manifest, switch_manifest_body, switch_test_request, + test_acceptance_episode_timing, test_app_config, test_deterministic_timeline_conflict, + test_manifest_origin_binding, test_origin_refresh_request, test_recovery_timing_policy, + test_segment_repair_manager, test_session, three_segment_manifest_body, CriticalEmergencyOriginServer, + StaleSwitchGeneration, TestOriginServer, CRITICAL_HANDOFF_MANIFEST_BODY, CRITICAL_HANDOFF_TS_BODY, SWITCH_MAP_BODY, + SWITCH_SEGMENT_BODY, }; -use arc_swap::{ArcSwap, ArcSwapOption}; -use axum::http::{header, HeaderMap, HeaderName, HeaderValue, StatusCode}; -use shared::model::{ - ConfigPaths, ConfigProviderDto, HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripMode, - ProviderUrlSelectionPolicy, -}; -use std::{ - fmt::Write, - io, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }, - time::Duration, -}; -use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpListener, - sync::{oneshot, Mutex, Notify, RwLock}, -}; -use tuliprox_core::{ - model::{ - AppConfig, Config, ConfigProvider, CustomStreamResponse, HlsManifestRecoveryBurstConfig, - HlsSegmentRepairConfig, ReverseProxyDisabledHeaderConfig, SourcesConfig, StripConfig, - }, - utils::content_coding::{ContentCoding, ContentCodingError}, -}; -use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer; -use tuliprox_session::ConnectionKind; -use url::Url; - -async fn retry_test_manifest_recovery_chain( - request: &OriginRefreshRequest, - target_url: Url, - reject_reason: HlsManifestRejectLogReason, -) -> Result { - let fetch_context = manifest_fetch_context(request); - retry_hls_origin_manifest_recovery_chain( - &fetch_context, - test_manifest_origin_binding(target_url), - Some(reject_reason), - None, - HlsManifestAcceptanceTrigger::RecoveryRequired, - HlsManifestCommitAcceptanceMode::StrictPinnedHost, - |fetched, acceptance_mode| super::commit_manifest_recovery_candidate(request, fetched, acceptance_mode), - ) - .await -} - -fn path_has_extension(path: &str, extension: &str) -> bool { - std::path::Path::new(path).extension().is_some_and(|actual| actual.eq_ignore_ascii_case(extension)) -} - -fn test_manifest_origin_binding(target_url: Url) -> HlsManifestOriginBinding { - HlsManifestOriginBinding::new(target_url, None).expect("concrete HTTP test binding") -} - -fn test_session() -> Arc> { - Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))) -} - -fn repeated_transient_manifest(segment_count: usize) -> String { - let mut body = String::from( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n", - ); - for _ in 0..segment_count { - body.push_str("#EXTINF:4,\nsame.ts\n"); - } - body -} - -fn test_segment_repair_manager() -> Arc { - Arc::new(HlsSegmentRepairManager::new(HlsSegmentRepairConfig { - max_level: HlsSegmentRepairMode::Off, - apply_to_first_segments: 1, - max_parallel_repairs: 1, - ..Default::default() - })) -} - -fn test_app_config() -> Arc { - Arc::new(AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config::default())), - sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), - hdhomerun: Arc::new(ArcSwapOption::empty()), - api_proxy: Arc::new(ArcSwapOption::empty()), - file_locks: Arc::new(tuliprox_core::utils::FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::empty()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(tuliprox_core::model::MediaToolCapabilities::default()), - }) -} - -#[test] -fn origin_refresh_state_starts_only_when_due_and_not_in_flight() { - let mut state = OriginRefreshState { next_fetch_allowed_at_ms: 100, ..OriginRefreshState::default() }; - assert!(!state.is_due(99)); - assert!(state.is_due(100)); - state.mark_started(100); - assert!(!state.is_due(101)); -} - -#[test] -fn normal_key_extension_is_always_compatible_with_the_transient_route() { - let extensions = [ - key_resource_extension("https://origin.example/live/key.php?token=secret"), - key_resource_extension("https://origin.example/live/opaque"), - key_resource_extension("https://origin.example/live/key.BIN"), - key_resource_extension("https://origin.example/live/key.key"), - ]; - - assert_eq!(extensions, ["key", "key", "bin", "key"]); - for extension in extensions { - let file_name = format!("{}.{}", "abcdefghijklmnop", extension); - let parsed = TransientResourceFile::parse(&file_name).expect("normalized key route parses"); - assert_eq!(parsed.resource_id, TransientResourceId("abcdefghijklmnop".to_string())); - assert_eq!(parsed.extension, extension); - } -} - -fn test_deterministic_timeline_conflict() -> HlsDeterministicTimelineConflict { - let fetched = fetched_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:490\n\ - #EXTINF:4,\n490.ts\n#EXTINF:4,\n480.ts\n#EXTINF:4,\n491.ts\n", - ); - deterministic_timeline_conflict_from_rejection( - &fetched, - &HlsManifestRejectLogReason::PublishedResourceReplay { - previous_proxy_tail: Some(2), - existing_proxy_seq: 0, - candidate_position: 1, - candidate_origin_seq: 491, - resource_key: super::super::resource_identity::HlsMediaResourceIdentity::from_url( - "http://origin.example.com/live/final/480.ts", - None, - ) - .semantic_key(), - decision: super::super::timeline::HlsResourceReplayDecision::RejectContradictoryOrder, - }, - ) - .expect("published replay rejection produces deterministic evidence") -} - -#[test] -fn origin_refresh_failure_backoff_ramps_and_success_resets_counter() { - let mut state = OriginRefreshState::default(); - - state.mark_started(1_000); - state.mark_failure(1_100); - assert_eq!(state.consecutive_failures, 1); - assert_eq!(state.last_error_at_ms, Some(1_100)); - assert_eq!(state.next_fetch_allowed_at_ms, 1_100); - assert!(state.is_due(1_100)); - - state.mark_started(1_200); - state.mark_failure(1_300); - assert_eq!(state.consecutive_failures, 2); - assert_eq!(state.next_fetch_allowed_at_ms, 1_800); - assert!(!state.is_due(1_799)); - assert!(state.is_due(1_800)); - - state.mark_started(1_900); - state.mark_failure(2_000); - assert_eq!(state.consecutive_failures, 3); - assert_eq!(state.next_fetch_allowed_at_ms, 3_000); - - state.mark_started(3_100); - let success_timing = build_manifest_refresh_timing(Some(20_000), None, HlsManifestProgress::Advanced); - assert_eq!(state.mark_success_with_timing(3_100, 3_200, success_timing), 10_000); - assert_eq!(state.consecutive_failures, 0); - assert_eq!(state.last_error_at_ms, None); - assert_eq!(state.next_fetch_allowed_at_ms, 13_100); - - state.mark_started(13_100); - state.mark_failure(13_200); - assert_eq!(state.consecutive_failures, 1); - assert_eq!(state.next_fetch_allowed_at_ms, 13_200); -} - -#[test] -fn status_classification_matches_hls_retry_policy() { - for status in [ - StatusCode::PROXY_AUTHENTICATION_REQUIRED, - StatusCode::REQUEST_TIMEOUT, - StatusCode::TOO_EARLY, - StatusCode::TOO_MANY_REQUESTS, - StatusCode::INTERNAL_SERVER_ERROR, - StatusCode::BAD_GATEWAY, - ] { - assert_eq!(classify_origin_manifest_status(status), OriginManifestStatusClass::Retryable); - } - for status in [ - StatusCode::BAD_REQUEST, - StatusCode::UNAUTHORIZED, - StatusCode::FORBIDDEN, - StatusCode::NOT_FOUND, - StatusCode::GONE, - ] { - assert_eq!(classify_origin_manifest_status(status), OriginManifestStatusClass::PermanentFailure); - } -} - -type ManifestFetchFailureCase = (&'static str, OriginManifestFetchError, HlsManifestFetchFailureSignal); - -fn manifest_response_failure_cases() -> Vec { - use HlsManifestFetchFailureKind as Kind; - use HlsManifestHttpResponseEvidence::{None as NoResponse, ValidResponse}; - - vec![ - ( - "permanent status", - OriginManifestFetchError::PermanentStatus(StatusCode::NOT_FOUND), - HlsManifestFetchFailureSignal::hard(Kind::HttpStatus { status: StatusCode::NOT_FOUND }, ValidResponse), - ), - ( - "retryable status", - OriginManifestFetchError::RetryableStatus(StatusCode::PROXY_AUTHENTICATION_REQUIRED, None), - HlsManifestFetchFailureSignal::retryable( - Kind::HttpStatus { status: StatusCode::PROXY_AUTHENTICATION_REQUIRED }, - ValidResponse, - ), - ), - ( - "retry exhausted", - OriginManifestFetchError::RetryExhausted, - HlsManifestFetchFailureSignal::retryable(Kind::RetryExhausted, NoResponse), - ), - ( - "commit generation exhausted", - OriginManifestFetchError::CommitGenerationExhausted, - HlsManifestFetchFailureSignal::discarded(Kind::CommitGenerationExhausted), - ), - ( - "recovery unavailable after response", - OriginManifestFetchError::RecoveryUnavailable { - reason: HlsManifestRecoveryUnavailableReason::NoEstablishedBindingAfterResponse, - }, - HlsManifestFetchFailureSignal::retryable(Kind::AcceptanceConflict, ValidResponse), - ), - ( - "recovery binding superseded", - OriginManifestFetchError::RecoveryUnavailable { - reason: HlsManifestRecoveryUnavailableReason::BindingSuperseded, - }, - HlsManifestFetchFailureSignal::discarded(Kind::Superseded), - ), - ( - "deterministic acceptance conflict", - OriginManifestFetchError::DeterministicTimelineConflict(Box::new(test_deterministic_timeline_conflict())), - HlsManifestFetchFailureSignal::retryable(Kind::AcceptanceConflict, ValidResponse), - ), - ( - "non-retryable status", - OriginManifestFetchError::NonRetryableStatus(StatusCode::IM_A_TEAPOT), - HlsManifestFetchFailureSignal::hard(Kind::HttpStatus { status: StatusCode::IM_A_TEAPOT }, ValidResponse), - ), - ( - "request timeout wording", - OriginManifestFetchError::Request("Request timed out and no retries left".to_string()), - HlsManifestFetchFailureSignal::retryable(Kind::Timeout, NoResponse), - ), - ( - "transport, connect, or DNS failure", - OriginManifestFetchError::Request("dns lookup failed".to_string()), - HlsManifestFetchFailureSignal::retryable(Kind::Transport, NoResponse), - ), - ( - "redirect failure", - OriginManifestFetchError::Redirect("redirect location invalid".to_string()), - HlsManifestFetchFailureSignal::retryable(Kind::Redirect, ValidResponse), - ), - ( - "timeout", - OriginManifestFetchError::Timeout, - HlsManifestFetchFailureSignal::retryable(Kind::Timeout, NoResponse), - ), - ] -} - -fn manifest_content_failure_cases() -> Vec { - use HlsManifestFetchFailureKind as Kind; - use HlsManifestHttpResponseEvidence::{None as NoResponse, ValidResponse}; - - vec![ - ( - "retryable provider acquire", - OriginManifestFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::WaitTimedOut), - HlsManifestFetchFailureSignal::retryable( - Kind::ProviderAcquire { kind: HlsBoundAccountAcquireErrorKind::WaitTimedOut }, - NoResponse, - ), - ), - ( - "hard provider acquire", - OriginManifestFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Expired), - HlsManifestFetchFailureSignal::hard( - Kind::ProviderAcquire { kind: HlsBoundAccountAcquireErrorKind::Expired }, - NoResponse, - ), - ), - ( - "invalid content-coding header", - OriginManifestFetchError::ContentCoding(ContentCodingError::InvalidHeader), - HlsManifestFetchFailureSignal::hard(Kind::InvalidContentCodingHeader, ValidResponse), - ), - ( - "unsupported content coding", - OriginManifestFetchError::ContentCoding(ContentCodingError::Unsupported("unknown".to_string())), - HlsManifestFetchFailureSignal::hard(Kind::UnsupportedContentCoding, ValidResponse), - ), - ( - "encoded partial content", - OriginManifestFetchError::ContentCoding(ContentCodingError::EncodedPartialContent), - HlsManifestFetchFailureSignal::hard(Kind::EncodedPartialContent, ValidResponse), - ), - ( - "content prefix read", - OriginManifestFetchError::ContentCoding(ContentCodingError::PrefixRead(io::Error::other( - "prefix read failed", - ))), - HlsManifestFetchFailureSignal::retryable(Kind::ContentPrefixRead, ValidResponse), - ), - ( - "content decoding", - OriginManifestFetchError::ContentDecoding { coding: ContentCoding::Gzip }, - HlsManifestFetchFailureSignal::retryable( - Kind::ContentDecoding { coding: ContentCoding::Gzip }, - ValidResponse, - ), - ), - ( - "decoded body limit", - OriginManifestFetchError::DecodedBodyLimitExceeded { limit: 1_024 }, - HlsManifestFetchFailureSignal::hard(Kind::DecodedBodyLimit, ValidResponse), - ), - ( - "invalid UTF-8", - OriginManifestFetchError::InvalidUtf8 { valid_up_to: 7, error_len: Some(1) }, - HlsManifestFetchFailureSignal::hard(Kind::InvalidUtf8, ValidResponse), - ), - ] -} - -#[test] -fn manifest_fetch_failures_have_exhaustive_typed_signals() { - for (label, error, expected) in - manifest_response_failure_cases().into_iter().chain(manifest_content_failure_cases()) - { - assert_eq!(classify_manifest_fetch_failure(&error), expected, "unexpected signal: {label}"); - assert_eq!(manifest_hard_fetch_error(&error), expected.is_hard(), "unexpected disposition: {label}"); - } -} - -#[test] -fn manifest_hard_fetch_error_matches_permanent_and_non_retryable_status_only() { - assert!(manifest_hard_fetch_error(&OriginManifestFetchError::PermanentStatus(StatusCode::NOT_FOUND))); - assert!(manifest_hard_fetch_error(&OriginManifestFetchError::NonRetryableStatus(StatusCode::IM_A_TEAPOT))); - assert!(!manifest_hard_fetch_error(&OriginManifestFetchError::RetryableStatus( - StatusCode::TOO_MANY_REQUESTS, - None, - ))); - assert!(!manifest_hard_fetch_error(&OriginManifestFetchError::Timeout)); - assert!(!manifest_hard_fetch_error(&OriginManifestFetchError::ProviderUnavailable( - HlsBoundAccountAcquireErrorKind::WaitTimedOut, - ))); - assert!(manifest_hard_fetch_error(&OriginManifestFetchError::ProviderUnavailable( - HlsBoundAccountAcquireErrorKind::Expired, - ))); -} - -#[test] -fn deterministic_conflict_returns_generation_bound_post_refresh_action() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.origin_control.progress_generation = 7; - session.activity.media_readiness_generation = 11; - - let action = apply_manifest_fetch_failure_signal( - &mut session, - &OriginManifestFetchError::DeterministicTimelineConflict(Box::new(test_deterministic_timeline_conflict())), - 100, - ); - - assert_eq!( - action, - HlsPostRefreshAvailabilityAction::Reevaluate { - reason: HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, - origin_progress_generation: 7, - media_readiness_generation: 11, - } - ); - assert_eq!(session.origin_control.last_origin_response_at_ms, Some(100)); - assert_eq!( - session.origin_control.path_condition, - super::super::origin_progress::HlsOriginPathCondition::AcceptanceConflict - ); - assert!(session.fresh_manifest_commit_required.is_none()); -} - -#[test] -fn failure_signal_advances_response_clock_only_with_http_evidence() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - - for error in [ - OriginManifestFetchError::Timeout, - OriginManifestFetchError::Request("connection refused".to_string()), - OriginManifestFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::WaitTimedOut), - ] { - apply_manifest_fetch_failure_signal(&mut session, &error, 100); - assert_eq!(session.origin_control.last_origin_response_at_ms, None); - assert_eq!( - session.origin_control.path_condition, - super::super::origin_progress::HlsOriginPathCondition::RetryableFetchFailure - ); - } - - apply_manifest_fetch_failure_signal( - &mut session, - &OriginManifestFetchError::RetryableStatus(StatusCode::PROXY_AUTHENTICATION_REQUIRED, None), - 200, - ); - assert_eq!(session.origin_control.last_origin_response_at_ms, Some(200)); - - apply_manifest_fetch_failure_signal( - &mut session, - &OriginManifestFetchError::Redirect("redirect location invalid".to_string()), - 250, - ); - assert_eq!(session.origin_control.last_origin_response_at_ms, Some(250)); - - apply_manifest_fetch_failure_signal( - &mut session, - &OriginManifestFetchError::ContentDecoding { coding: ContentCoding::Brotli }, - 300, - ); - assert_eq!(session.origin_control.last_origin_response_at_ms, Some(300)); - - let hard_action = apply_manifest_fetch_failure_signal( - &mut session, - &OriginManifestFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Expired), - 400, - ); - assert_eq!(session.origin_control.last_origin_response_at_ms, Some(300)); - assert_eq!( - session.origin_control.path_condition, - super::super::origin_progress::HlsOriginPathCondition::HardFetchFailure - ); - assert_eq!( - session.fresh_manifest_commit_required, - Some(HlsFreshManifestRequiredReason::PreviousHardManifestFailure) - ); - assert_eq!( - hard_action, - HlsPostRefreshAvailabilityAction::Reevaluate { - reason: HlsPostRefreshAvailabilityReason::HardManifestFailure, - origin_progress_generation: session.origin_control.progress_generation, - media_readiness_generation: session.activity.media_readiness_generation, - } - ); -} - -#[test] -fn retry_exhaustion_preserves_qualified_acceptance_conflict_but_not_all_failed_transport_state() { - for (reason, expected) in [ - ( - HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate, - super::super::origin_progress::HlsOriginPathCondition::AcceptanceConflict, - ), - ( - HlsManifestAcceptanceExhaustionReason::AllFailed, - super::super::origin_progress::HlsOriginPathCondition::RetryableFetchFailure, - ), - ] { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - let burst_plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); - session.origin_control.begin_acceptance_episode( - 100, - burst_plan, - HlsManifestAcceptanceTrigger::Observe, - &test_acceptance_episode_timing(100, burst_plan), - ); - let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); - episode.record_full_burst(); - episode.record_exhaustion(reason); - episode.hold_after_uncommitted_burst(None, None); - session.origin_control.path_condition = - super::super::origin_progress::HlsOriginPathCondition::AcceptanceConflict; - - apply_manifest_fetch_failure_signal(&mut session, &OriginManifestFetchError::RetryExhausted, 200); - - assert_eq!(session.origin_control.path_condition, expected); - } -} - -#[test] -fn invalidated_origin_work_generation_rejects_late_completion_without_counting_failure() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - let started_generation = session.start_origin_work(); - let mut refresh_state = OriginRefreshState::default(); - refresh_state.mark_started(100); - - assert!(refresh_origin_work_generation_matches(&session, Some(started_generation))); - session.invalidate_queued_origin_work(); - assert!(!refresh_origin_work_generation_matches(&session, Some(started_generation))); - assert!(refresh_origin_work_generation_matches(&session, None)); - - refresh_state.mark_invalidated(200); - assert!(!refresh_state.in_flight); - assert_eq!(refresh_state.last_fetch_finished_at_ms, Some(200)); - assert_eq!(refresh_state.consecutive_failures, 0); - assert_eq!(refresh_state.last_error_at_ms, None); -} - -#[test] -fn request_error_timeout_detection_matches_global_helper_wording() { - assert!(request_error_indicates_timeout("Request timed out and no retries left")); - assert!(request_error_indicates_timeout("idle timeout while trying provider://demo")); - assert!(!request_error_indicates_timeout("Request error: error sending request")); -} - -#[test] -fn manifest_reject_log_reason_preserves_timeline_mapping_error() { - assert_eq!( - HlsManifestRejectLogReason::from(TimelineMapError::UnsupportedSegmentExtension).status_label(), - "unsupported-segment-extension" - ); - assert_eq!( - HlsManifestRejectLogReason::from(TimelineMapError::ProxyMapIdOverflow).status_label(), - "proxy-map-id-overflow" - ); -} - -#[test] -fn manifest_highwater_policy_limit_uses_target_duration_fallback() { - assert_eq!(origin_highwater_policy_limit(60, None), Some(4)); - assert_eq!(origin_highwater_policy_limit(61, None), Some(5)); - assert_eq!(origin_highwater_policy_limit(60, Some(12)), Some(5)); -} - -#[test] -fn manifest_recovery_burst_levels_map_to_candidate_counts() { - let cases = [ - (HlsManifestRecoveryBurstLevel::Off, 1, 1), - (HlsManifestRecoveryBurstLevel::Friendly, 2, 1), - (HlsManifestRecoveryBurstLevel::Cautious, 3, 1), - (HlsManifestRecoveryBurstLevel::Balanced, 4, 1), - (HlsManifestRecoveryBurstLevel::Intense, 5, 1), - (HlsManifestRecoveryBurstLevel::Aggressive, 6, 1), - (HlsManifestRecoveryBurstLevel::Beast, 6, 2), - ]; - for (level, expected_slots, expected_lanes) in cases { - let plan = level.plan(); - let expected_candidates = expected_slots * expected_lanes; - assert_eq!(plan.slots, expected_slots); - assert_eq!(plan.lanes_per_slot, expected_lanes); - assert_eq!(plan.total_candidates(), expected_candidates); - assert_eq!(level.total_candidates(), expected_candidates); - } -} - -#[test] -fn retry_after_header_is_parsed_as_milliseconds() { - let mut headers = HeaderMap::new(); - headers.insert(header::RETRY_AFTER, HeaderValue::from_static("3")); - assert_eq!(retry_after_delay_ms(&headers), Some(3_000)); -} - -#[test] -fn resolved_hls_manifest_request_url_uses_provider_index_locally() { - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "demo".into(), - urls: vec!["http://provider-a.example".into(), "http://provider-b.example".into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), - dns: None, - })); - let provider_entry = - LiveHlsOriginEntry::parse_with_url_failover_provider("provider://demo/live/u/p/1.m3u8", Some(provider)) - .unwrap(); - - let resolved = - resolved_hls_manifest_request_url_from_input(&provider_entry.to_input_source(), Some(1), provider_entry.url()); - assert_eq!(resolved.as_str(), "http://provider-b.example/live/u/p/1.m3u8"); - assert!(!resolved.as_str().contains("provider://")); - - let direct_entry = LiveHlsOriginEntry::parse("http://origin.example/live/u/p/1.m3u8").unwrap(); - assert_eq!( - resolved_hls_manifest_request_url_from_input(&direct_entry.to_input_source(), Some(1), direct_entry.url()) - .as_str(), - "http://origin.example/live/u/p/1.m3u8" - ); -} - -#[test] -fn manifest_timing_log_values_are_seconds_or_none() { - assert_eq!(format_optional_millis_as_seconds(Some(4_500)), "4.500"); - assert_eq!(format_optional_millis_as_seconds(None), "none"); - assert_eq!(format_millis_as_seconds(2_000), "2.000"); -} - -#[test] -fn refresh_interval_uses_half_reference_duration_without_upper_clamp() { - assert_eq!(compute_origin_refresh_interval_ms(Some(8_000), None), 4_000); - assert_eq!(compute_origin_refresh_interval_ms(Some(500), None), 1_000); - assert_eq!(compute_origin_refresh_interval_ms(Some(20_000), None), 10_000); - assert_eq!(compute_origin_refresh_interval_ms(None, None), 2_000); -} - -#[test] -fn empty_refresh_rampdown_halves_until_one_second() { - let mut state = OriginRefreshState::default(); - let timing = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Unchanged); - - state.mark_started(0); - assert_eq!(state.mark_success_with_timing(0, 100, timing), 3_000); - assert_eq!(state.consecutive_empty_refreshes, 1); - assert_eq!(state.next_fetch_allowed_at_ms, 3_000); - - state.mark_started(3_000); - assert_eq!(state.mark_success_with_timing(3_000, 3_100, timing), 1_500); - assert_eq!(state.consecutive_empty_refreshes, 2); - assert_eq!(state.next_fetch_allowed_at_ms, 4_500); - - state.mark_started(4_500); - assert_eq!(state.mark_success_with_timing(4_500, 4_600, timing), 1_000); - assert_eq!(state.consecutive_empty_refreshes, 3); - assert_eq!(state.next_fetch_allowed_at_ms, 5_500); - - state.mark_started(5_500); - assert_eq!(state.mark_success_with_timing(5_500, 5_600, timing), 1_000); - assert_eq!(state.consecutive_empty_refreshes, 4); - assert_eq!(state.next_fetch_allowed_at_ms, 6_500); -} - -#[test] -fn advanced_or_rollover_refresh_resets_empty_refresh_counter() { - let mut state = OriginRefreshState::default(); - let unchanged = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Unchanged); - let advanced = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Advanced); - let rollover = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Rollover); - - state.mark_started(0); - assert_eq!(state.mark_success_with_timing(0, 100, unchanged), 3_000); - state.mark_started(3_000); - assert_eq!(state.mark_success_with_timing(3_000, 3_100, unchanged), 1_500); - assert_eq!(state.consecutive_empty_refreshes, 2); - - state.mark_started(4_500); - assert_eq!(state.mark_success_with_timing(4_500, 4_600, advanced), 6_000); - assert_eq!(state.consecutive_empty_refreshes, 0); - assert_eq!(state.next_fetch_allowed_at_ms, 10_500); - - state.mark_started(10_500); - assert_eq!(state.mark_success_with_timing(10_500, 10_600, unchanged), 3_000); - assert_eq!(state.consecutive_empty_refreshes, 1); - - state.mark_started(13_500); - assert_eq!(state.mark_success_with_timing(13_500, 13_600, rollover), 6_000); - assert_eq!(state.consecutive_empty_refreshes, 0); - assert_eq!(state.next_fetch_allowed_at_ms, 19_500); -} - -#[test] -fn failure_backoff_does_not_increment_empty_refresh_counter() { - let mut state = OriginRefreshState::default(); - let unchanged = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Unchanged); - - state.mark_started(0); - assert_eq!(state.mark_success_with_timing(0, 100, unchanged), 3_000); - state.mark_started(3_000); - state.mark_failure(3_100); - - assert_eq!(state.consecutive_failures, 1); - assert_eq!(state.consecutive_empty_refreshes, 1); -} - -#[test] -fn manifest_progress_tracks_highwater_advancement() { - assert_eq!( - manifest_progress_from_highwater(None, Some(10), HlsManifestSequenceRelation::NoPreviousHighwater), - HlsManifestProgress::Advanced - ); - assert_eq!( - manifest_progress_from_highwater(Some(10), Some(11), HlsManifestSequenceRelation::Next), - HlsManifestProgress::Advanced - ); - assert_eq!( - manifest_progress_from_highwater(Some(10), Some(10), HlsManifestSequenceRelation::Same), - HlsManifestProgress::Unchanged - ); - assert_eq!( - manifest_progress_from_highwater(Some(10), Some(9), HlsManifestSequenceRelation::Backward), - HlsManifestProgress::Unchanged - ); - assert_eq!( - manifest_progress_from_highwater(Some(10), Some(1), HlsManifestSequenceRelation::RolloverCandidate), - HlsManifestProgress::Rollover - ); - assert_eq!( - manifest_progress_from_highwater(None, None, HlsManifestSequenceRelation::NoOriginHighwater), - HlsManifestProgress::Unchanged - ); -} - -#[test] -fn hls_cutover_policy_only_advanced_and_rollover_commits_advance_progress_generation() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - let initial_generation = session.origin_control.progress_generation; - - record_committed_manifest_media_progress( - &mut session.origin_control, - HlsManifestCommitProgressEvidence::CacheTimeline(build_manifest_refresh_timing( - None, - Some(4_000), - HlsManifestProgress::Unchanged, - )), - Some(4), - 1_000, - ); - assert_eq!(session.origin_control.progress_generation, initial_generation); - assert_eq!(session.origin_control.last_media_progress_at_ms, None); - - record_committed_manifest_media_progress( - &mut session.origin_control, - HlsManifestCommitProgressEvidence::Transient(build_manifest_refresh_timing( - None, - Some(4_000), - HlsManifestProgress::Advanced, - )), - Some(4), - 1_500, - ); - assert_eq!(session.origin_control.progress_generation, initial_generation); - assert_eq!(session.origin_control.last_media_progress_at_ms, None); - - record_committed_manifest_media_progress( - &mut session.origin_control, - HlsManifestCommitProgressEvidence::CacheTimeline(build_manifest_refresh_timing( - None, - Some(4_000), - HlsManifestProgress::Advanced, - )), - Some(4), - 2_000, - ); - assert_eq!(session.origin_control.progress_generation, initial_generation.saturating_add(1)); - assert_eq!(session.origin_control.last_media_progress_at_ms, Some(2_000)); - - record_committed_manifest_media_progress( - &mut session.origin_control, - HlsManifestCommitProgressEvidence::CacheTimeline(build_manifest_refresh_timing( - None, - Some(4_000), - HlsManifestProgress::Rollover, - )), - Some(4), - 3_000, - ); - assert_eq!(session.origin_control.progress_generation, initial_generation.saturating_add(2)); - assert_eq!(session.origin_control.last_media_progress_at_ms, Some(3_000)); -} - -#[tokio::test] -async fn hls_terminal_commit_media_progress_cancels_pending_owner_after_session_lock_release() { - let session = test_session(); - let request = test_origin_refresh_request(Arc::clone(&session)); - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let coordinator = request.hls_proxy.terminal_pending(); - let (started_tx, started_rx) = oneshot::channel(); - let (cancelled_tx, cancelled_rx) = oneshot::channel(); - let owner_key = HlsTerminalPendingOwnerKey { - session_incarnation: HlsSessionIncarnation::for_test(1), - proxy_session_id, - lease_id: HlsAccessLeaseId("progress-cancel".to_string()), - lease_issued_at_ms: 10, - expected_admission_generation: 20, - manifest_snapshot_generation: 30, - cursor_generation: 40, - decision_generation: 50, - reason: HlsRuntimeCustomTailReason::ChannelUnavailable, - bundle_key: HlsPreparedTerminalBundleKey { - asset: HlsTerminalAssetIdentity { revision: 60, fingerprint: [6; 32] }, - target_duration_ms: 4_000, - segment_count: super::HLS_TERMINAL_TAIL_SEGMENT_COUNT, - }, - latest_safe_commit_at_ms: 10_000, - }; - let asset_guard = HlsTerminalAssetRevisionGuard::matching_for_test(Some(owner_key.bundle_key.asset)); - assert_eq!( - coordinator.register(owner_key, &asset_guard, move |ownership| async move { - assert!(started_tx.send(()).is_ok()); - ownership.cancelled().await; - assert!(cancelled_tx.send(()).is_ok()); - }), - HlsTerminalPendingRegistration::Scheduled - ); - assert!(started_rx.await.is_ok()); - - let commit_result = Ok(( - HlsManifestCommitProgressEvidence::CacheTimeline(build_manifest_refresh_timing( - None, - Some(4_000), - HlsManifestProgress::Advanced, - )), - false, - false, - )); - cancel_superseded_terminal_work_after_media_progress(&request, &commit_result).await; - - assert!(cancelled_rx.await.is_ok()); - assert_eq!(coordinator.owner_count(), 0); -} - -#[test] -fn hls_cutover_policy_transient_success_preserves_recovery_episode_and_empty_refresh_evidence() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - let burst_plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); - session.origin_control.begin_acceptance_episode( - 1_000, - burst_plan, - HlsManifestAcceptanceTrigger::RecoveryRequired, - &test_acceptance_episode_timing(1_000, burst_plan), - ); - session.origin_control.acceptance_episode.as_mut().expect("acceptance episode").complete(); - session.origin_refresh.consecutive_empty_refreshes = 2; - session.origin_refresh.mark_started(10_000); - - let (bookkeeping_timing, applied_interval_ms) = record_committed_manifest_success( - &mut session, - HlsManifestCommitProgressEvidence::Transient(build_manifest_refresh_timing( - None, - Some(12_000), - HlsManifestProgress::Advanced, - )), - 10_000, - 10_100, - ); - - assert_eq!(bookkeeping_timing.progress, HlsManifestProgress::Unchanged); - assert_eq!(applied_interval_ms, 1_000); - assert_eq!(session.origin_refresh.consecutive_empty_refreshes, 3); - assert_eq!(session.origin_refresh.next_fetch_allowed_at_ms, 11_000); - assert!(session.origin_control.acceptance_episode.is_some()); - assert_eq!(session.origin_control.recovery_samples.p95_ms(), None); - assert_eq!(session.origin_control.last_origin_response_at_ms, Some(10_100)); -} - -#[test] -fn hls_cutover_policy_cache_timeline_progress_keeps_recovery_success_bookkeeping() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - let burst_plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); - session.origin_control.begin_acceptance_episode( - 1_000, - burst_plan, - HlsManifestAcceptanceTrigger::RecoveryRequired, - &test_acceptance_episode_timing(1_000, burst_plan), - ); - session.origin_control.acceptance_episode.as_mut().expect("acceptance episode").complete(); - session.origin_refresh.consecutive_empty_refreshes = 2; - session.origin_refresh.mark_started(10_000); - - let (bookkeeping_timing, applied_interval_ms) = record_committed_manifest_success( - &mut session, - HlsManifestCommitProgressEvidence::CacheTimeline(build_manifest_refresh_timing( - None, - Some(12_000), - HlsManifestProgress::Advanced, - )), - 10_000, - 10_100, - ); - - assert_eq!(bookkeeping_timing.progress, HlsManifestProgress::Advanced); - assert_eq!(applied_interval_ms, 6_000); - assert_eq!(session.origin_refresh.consecutive_empty_refreshes, 0); - assert!(session.origin_control.acceptance_episode.is_none()); - assert_eq!(session.origin_control.recovery_samples.p95_ms(), Some(9_100)); -} - -#[test] -fn recovery_selection_log_phase_distinguishes_single_candidate_from_burst() { - assert_eq!(ManifestRecoverySelectionLogPhase::from_candidate_count(1), ManifestRecoverySelectionLogPhase::Recovery); - assert_eq!(ManifestRecoverySelectionLogPhase::from_candidate_count(2), ManifestRecoverySelectionLogPhase::Burst); - assert_eq!(ManifestRecoverySelectionLogPhase::Recovery.as_log_label(), "recovery"); - assert_eq!(ManifestRecoverySelectionLogPhase::Burst.as_log_label(), "burst"); -} - -#[test] -fn transient_reason_log_fields_include_unsupported_tag() { - let reason = TransientPassthroughReason::UnsupportedTag { tag: "#EXT-X-PART".to_string() }; - - assert_eq!(transient_reason_log_fields(&reason), "reason=unsupported_tag tag=#EXT-X-PART"); -} - -#[test] -fn different_host_candidate_is_not_committed_immediately() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.origin_seq_highwater = Some(758); - session.last_effective_manifest_host = Some("previous.example.com".to_string()); - let binding = HlsManifestOriginBinding::new( - Url::parse("https://previous.example.com/live/index.m3u8?token=baseline").expect("baseline URL"), - Some(0), - ) - .expect("baseline binding"); - session.origin_control.manifest_origin_binding = Some(binding.clone()); - let request = test_origin_refresh_request(test_session()); - let fetched = fetched_manifest( - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:758\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg758.ts\n#EXTINF:4.0,\nseg759.ts\n", - ); - - let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); - - assert!(matches!(result, Err(HlsManifestCommitError::RetryCurrentTarget))); - assert!(session.transient.last_manifest_body.is_none()); - assert_eq!(session.origin_control.manifest_origin_binding.as_ref(), Some(&binding)); -} - -#[test] -fn fresh_revalidation_rebases_normal_manifest_on_the_pinned_host() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.origin_seq_highwater = Some(1_000); - session.last_effective_manifest_host = Some("origin.example.com".to_string()); - let mut request = test_origin_refresh_request(test_session()); - request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { - reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, - }; - let fetched = - fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:4.0,\nseg10.ts\n"); - - let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); - - assert!(result.is_ok()); - assert_eq!(session.origin_seq_highwater, Some(10)); - assert_eq!(session.origin_epoch, 1); - assert_eq!(session.last_effective_manifest_host.as_deref(), Some("origin.example.com")); -} - -#[test] -fn refresh_commit_trims_forward_manifest_with_published_stale_resource_prefix() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - let request = test_origin_refresh_request(test_session()); - let baseline = fetched_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:471\n\ - #EXTINF:4,\n471.ts\n#EXTINF:4,\n472.ts\n#EXTINF:4,\n473.ts\n\ - #EXTINF:4,\n474.ts\n#EXTINF:4,\n475.ts\n#EXTINF:4,\n476.ts\n", - ); - let replay_then_new = fetched_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:476\n\ - #EXTINF:4,\n474.ts\n#EXTINF:4,\n480.ts\n#EXTINF:4,\n481.ts\n", - ); - - commit_fetched_manifest(&mut session, &baseline, &request, 100).expect("baseline refresh commits"); - for segment in session.segments.values_mut() { - segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 101 }; - } - session.render_and_store_manifest(101).expect("baseline publishes"); - commit_fetched_manifest(&mut session, &replay_then_new, &request, 200) - .expect("stale prefix is safely trimmed by refresh commit"); - - assert_eq!(session.proxy_next_seq, Some(8)); - assert_eq!(session.publishable_origin_head_proxy_seq, Some(0)); - assert!(session.segments.get(&6).expect("first genuine media").discontinuity_before); - for proxy_seq in [6_u64, 7] { - session.segments.get_mut(&proxy_seq).expect("new segment").status = - SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 201 }; - } - let rendered = session.render_and_store_manifest(201).expect("forward refresh renders without replay"); - assert_eq!(rendered.first_proxy_seq, 2); - assert!(rendered.body.contains("#EXT-X-DISCONTINUITY\n#EXTINF:4.000,")); - let identities = rendered - .segment_proxy_seqs - .iter() - .filter_map(|proxy_seq| session.segments.get(proxy_seq)?.media_resource_identity()) - .collect::>(); - assert!(identities - .iter() - .enumerate() - .all(|(index, identity)| { identities[..index].iter().all(|previous| !previous.matches(*identity)) })); -} - -async fn spawn_rotating_parent_origin() -> TestOriginServer { - let rotating_parent = Arc::new(AtomicUsize::new(0)); - let handler_counter = Arc::clone(&rotating_parent); - spawn_test_origin(Arc::new(move |_path| { - let parent = handler_counter.fetch_add(1, Ordering::AcqRel).saturating_add(1); - ( - 200, - Vec::new(), - format!( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:490\n\ - #EXTINF:4,\n/stream/{parent:016x}/1745190_490.ts\n\ - #EXTINF:4,\n/stream/{parent:016x}/1745180_480.ts\n\ - #EXTINF:4,\n/stream/{parent:016x}/1745191_491.ts\n" - ), - ) - })) - .await -} - -#[tokio::test] -async fn rotating_volatile_parent_proves_one_deterministic_conflict() { - let baseline_body = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:480\n\ - #EXTINF:4,\n/stream/aaaaaaaaaaaaaaaa/1745180_480.ts\n\ - #EXTINF:4,\n/stream/aaaaaaaaaaaaaaaa/1745181_481.ts\n\ - #EXTINF:4,\n/stream/aaaaaaaaaaaaaaaa/1745182_482.ts\n"; - let server = spawn_rotating_parent_origin().await; - let session = test_session(); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - let manifest_url = format!("{}/live/user/pass/12345.m3u8", server.base_url); - request.origin_entry = LiveHlsOriginEntry::parse(&manifest_url).expect("local replay origin entry"); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - let plan = request.manifest_recovery_burst.level.plan(); - let mut baseline = fetched_manifest(baseline_body); - baseline.final_manifest_url = manifest_url.clone(); - baseline.resolved_request_url = manifest_url; - baseline.redirect_host = Some("127.0.0.1".to_string()); - { - let mut session = session.write().await; - commit_fetched_manifest(&mut session, &baseline, &request, 100).expect("baseline commits"); - for segment in session.segments.values_mut() { - segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 101 }; - } - session.render_and_store_manifest(101).expect("baseline publishes"); - assert_eq!(session.published_resource_history.generation(), 3); - } - - let Err(first) = fetch_and_commit_manifest_with_policy(&mut request).await else { - panic!("one complete candidate landscape must prove the replay conflict"); - }; - let OriginManifestFetchError::DeterministicTimelineConflict(first_conflict) = first else { - panic!("deterministic replay conflict must not collapse to retry exhaustion"); - }; - assert_eq!(first_conflict.previous_proxy_tail, Some(2)); - assert_eq!(first_conflict.existing_proxy_seq, 0); - assert_eq!(first_conflict.candidate_position, 1); - assert_eq!(first_conflict.candidate_origin_seq, 491); - assert_eq!(server.requests.lock().await.len(), plan.total_candidates().saturating_add(1)); - { - let session = session.read().await; - let episode = session.origin_control.acceptance_episode.as_ref().expect("acceptance episode retained"); - assert_eq!(episode.full_bursts_completed, 1); - assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); - assert!(episode.deterministic_conflict_receipt().is_some()); - assert!(matches!( - super::super::manifest_acceptance::manifest_acceptance_episode_status( - Some(episode), - episode.generation, - request.now_ms, - ), - super::super::manifest_acceptance::HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { - reason: HlsManifestAcceptanceExhaustionReason::DeterministicTimelineConflict, - .. - } - )); - assert_eq!(session.proxy_next_seq, Some(3), "replayed content receives no proxy sequence"); - } - - let request_count_before_receipt_sample = server.requests.lock().await.len(); - request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::Observe; - let Err(repeated) = fetch_and_commit_manifest_with_policy(&mut request).await else { - panic!("an unchanged ordinary sample must remain rejected"); - }; - assert!(matches!( - repeated, - OriginManifestFetchError::DeterministicTimelineConflict(ref conflict) - if conflict.as_ref() == first_conflict.as_ref() - )); - assert_eq!( - server.requests.lock().await.len(), - request_count_before_receipt_sample.saturating_add(1), - "an unchanged receipt permits one ordinary request but no second burst" - ); - - { - let mut session = session.write().await; - session.published_resource_history.record( - super::super::resource_identity::HlsMediaResourceIdentity::from_url( - "http://127.0.0.1/live/user/pass/unrelated-published.ts", - None, - ), - 99, - ); - } - let request_count_before_changed_landscape = server.requests.lock().await.len(); - let Err(changed) = fetch_and_commit_manifest_with_policy(&mut request).await else { - panic!("changed published evidence must still reject the replay"); - }; - assert!(matches!(changed, OriginManifestFetchError::DeterministicTimelineConflict(_))); - assert_eq!( - server.requests.lock().await.len(), - request_count_before_changed_landscape.saturating_add(plan.total_candidates()), - "changed receipt evidence authorizes exactly one newly configured full burst" - ); - let session = session.read().await; - assert_eq!(session.proxy_next_seq, Some(3), "the replay stays blocked after reevaluation"); - assert!(!session.segments.contains_key(&3)); -} - -#[tokio::test] -async fn background_replay_failure_registers_post_refresh_reevaluation_without_client_demand() { - let hls_ctx = crate::HlsCtx::for_test(Config::default()); - let ctx = &hls_ctx; - let now_ms = current_time_millis(); - let (session, _) = ctx - .hls_proxy - .get_or_create_session_with_outcome(HlsSessionKey::new(1, "post-refresh-conflict"), b"secret", now_ms) - .await; - let conflicting_body = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:490\n\ - #EXTINF:4,\n490.ts\n#EXTINF:4,\n480.ts\n#EXTINF:4,\n491.ts\n"; - let server = spawn_test_origin(Arc::new(move |_path| (200, Vec::new(), conflicting_body.to_string()))).await; - let manifest_url = format!("{}/live/user/pass/12345.m3u8", server.base_url); - let mut request = bind_refresh_request_to_app_state(test_origin_refresh_request(Arc::clone(&session)), ctx); - request.origin_entry = LiveHlsOriginEntry::parse(&manifest_url).expect("local replay origin entry"); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - request.now_ms = now_ms; - let mut baseline = fetched_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:480\n\ - #EXTINF:4,\n480.ts\n#EXTINF:4,\n481.ts\n#EXTINF:4,\n482.ts\n", - ); - baseline.final_manifest_url = manifest_url.clone(); - baseline.resolved_request_url = manifest_url; - baseline.redirect_host = Some("127.0.0.1".to_string()); - { - let mut session = session.write().await; - commit_fetched_manifest(&mut session, &baseline, &request, now_ms).expect("baseline commits"); - for segment in session.segments.values_mut() { - segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: now_ms }; - } - session.render_and_store_manifest(now_ms).expect("baseline publishes"); - session.segments.get_mut(&1).expect("deferred boundary segment").status = - SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Prefetch, deferred_at_ms: now_ms }; - } - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("post-refresh-conflict-lease".to_string()); - let mut lease = HlsAccessLease::pending( - lease_id, - HlsPlaybackFamilyKey::new("post-refresh-user", "post-refresh-client"), - proxy_session_id, - "post-refresh-user".to_string(), - "post-refresh-token".to_string(), - 1, - "post-refresh-stream".to_string(), - 1, - now_ms, - 60_000, - ); - lease.state = crate::HlsAccessLeaseState::Activated; - lease.active_until_ms = Some(now_ms.saturating_add(60_000)); - lease.pending_deadline = None; - lease.last_manifest_snapshot = Some(post_refresh_live_manifest_snapshot()); - ctx.hls_proxy.prepare_access_lease(lease).await; - - assert!(trigger_origin_refresh_sync(request).await); - - assert_eq!( - session.read().await.origin_control.path_condition, - super::super::origin_progress::HlsOriginPathCondition::AcceptanceConflict - ); - assert_eq!(ctx.hls_proxy.availability_reevaluations().owner_count(), 1); -} - -#[tokio::test] -async fn background_success_does_not_schedule_failure_reevaluation() { - let hls_ctx = crate::HlsCtx::for_test(Config::default()); - let ctx = &hls_ctx; - let now_ms = current_time_millis(); - let (session, _) = ctx - .hls_proxy - .get_or_create_session_with_outcome(HlsSessionKey::new(1, "post-refresh-success"), b"secret", now_ms) - .await; - let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; - let mut request = bind_refresh_request_to_app_state(test_origin_refresh_request(Arc::clone(&session)), ctx); - request.origin_entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) - .expect("local successful origin entry"); - request.now_ms = now_ms; - - assert!(trigger_origin_refresh_sync(request).await); - - assert!(session.read().await.origin_seq_highwater.is_some()); - assert_eq!(ctx.hls_proxy.availability_reevaluations().owner_count(), 0); -} - -#[tokio::test] -async fn successful_refresh_wakes_sleeping_post_refresh_owner() { - let hls_ctx = crate::HlsCtx::for_test(Config::default()); - let ctx = &hls_ctx; - let now_ms = current_time_millis(); - let progressed_body = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:481\n\ - #EXTINF:4,\n481.ts\n#EXTINF:4,\n482.ts\n#EXTINF:4,\n483.ts\n"; - let server = spawn_test_origin(Arc::new(move |_path| (200, Vec::new(), progressed_body.to_string()))).await; - let (session, _) = ctx - .hls_proxy - .get_or_create_session_with_outcome(HlsSessionKey::new(1, "post-refresh-wakeup"), b"secret", now_ms) - .await; - let manifest_url = format!("{}/live/user/pass/12345.m3u8", server.base_url); - let mut request = bind_refresh_request_to_app_state(test_origin_refresh_request(Arc::clone(&session)), ctx); - request.origin_entry = LiveHlsOriginEntry::parse(&manifest_url).expect("local successful origin entry"); - request.now_ms = now_ms; - let mut baseline = fetched_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:480\n\ - #EXTINF:4,\n480.ts\n#EXTINF:4,\n481.ts\n#EXTINF:4,\n482.ts\n", - ); - baseline.final_manifest_url = manifest_url.clone(); - baseline.resolved_request_url = manifest_url; - baseline.redirect_host = Some("127.0.0.1".to_string()); - { - let mut session = session.write().await; - commit_fetched_manifest(&mut session, &baseline, &request, now_ms).expect("baseline commits"); - for segment in session.segments.values_mut() { - segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: now_ms }; - } - session.render_and_store_manifest(now_ms).expect("baseline publishes"); - session.segments.get_mut(&1).expect("deferred boundary segment").status = - SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Prefetch, deferred_at_ms: now_ms }; - session.origin_control.path_condition = - super::super::origin_progress::HlsOriginPathCondition::AcceptanceConflict; - } - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("post-refresh-wakeup-lease".to_string()); - let mut lease = HlsAccessLease::pending( - lease_id, - HlsPlaybackFamilyKey::new("post-refresh-user", "post-refresh-client"), - proxy_session_id, - "post-refresh-user".to_string(), - "post-refresh-token".to_string(), - 1, - "post-refresh-stream".to_string(), - 1, - now_ms, - 60_000, - ); - lease.state = crate::HlsAccessLeaseState::Activated; - lease.active_until_ms = Some(now_ms.saturating_add(60_000)); - lease.pending_deadline = None; - lease.last_manifest_snapshot = Some(post_refresh_live_manifest_snapshot()); - ctx.hls_proxy.prepare_access_lease(lease).await; - - assert!(mark_origin_refresh_started(&mut request, now_ms).await); - let coordinator = ctx.hls_proxy.availability_reevaluations(); - let initial_permits = coordinator.available_task_permits_for_test(); - let (origin_progress_generation, media_readiness_generation) = { - let session = session.read().await; - (session.origin_control.progress_generation, session.activity.media_readiness_generation) - }; - assert_eq!( - super::super::availability::register_post_refresh_availability_reevaluation( - ctx.clone(), - Arc::clone(&session), - request.clone(), - HlsPostRefreshAvailabilityAction::Reevaluate { - reason: HlsPostRefreshAvailabilityReason::HardManifestFailure, - origin_progress_generation, - media_readiness_generation, - }, - ) - .await, - HlsAvailabilityReevaluationRegistration::Scheduled - ); - assert_eq!(coordinator.owner_count(), 1); - assert_eq!(coordinator.available_task_permits_for_test(), initial_permits.saturating_sub(1)); - let progress_generation_before = session.read().await.origin_control.progress_generation; - - refresh_and_commit(request, now_ms).await; - for _ in 0..256 { - if coordinator.owner_count() == 0 && coordinator.available_task_permits_for_test() == initial_permits { - break; - } - tokio::task::yield_now().await; - } - - assert!(session.read().await.origin_control.progress_generation > progress_generation_before); - assert_eq!(coordinator.owner_count(), 0); - assert_eq!(coordinator.available_task_permits_for_test(), initial_permits); -} - -async fn assert_requirement_set_after_refresh_start_survives_fresh_commit( - replacement_reason: HlsFreshManifestRequiredReason, -) { - let session = test_session(); - { - let mut session = session.write().await; - session.origin_seq_highwater = Some(1_000); - session.last_effective_manifest_host = Some("origin.example.com".to_string()); - session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::PreviousHardManifestFailure); - } - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { - reason: HlsFreshManifestRequiredReason::PreviousHardManifestFailure, - }; - assert!(mark_origin_refresh_started(&mut request, 100).await); - - let fetched = - fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:4.0,\nseg10.ts\n"); - let result = { - let mut session = session.write().await; - session.require_fresh_manifest_commit(replacement_reason); - commit_fetched_manifest(&mut session, &fetched, &request, 200) - }; - - assert!(result.is_ok()); - assert_eq!(session.read().await.fresh_manifest_commit_required, Some(replacement_reason)); -} - -#[tokio::test] -async fn fresh_commit_preserves_new_or_same_reason_requirement_set_after_refresh_start() { - assert_requirement_set_after_refresh_start_survives_fresh_commit( - HlsFreshManifestRequiredReason::ExpiredRevalidation, - ) - .await; - assert_requirement_set_after_refresh_start_survives_fresh_commit( - HlsFreshManifestRequiredReason::PreviousHardManifestFailure, - ) - .await; -} - -#[tokio::test] -async fn concurrent_maybe_trigger_origin_refresh_starts_singleflight_once() { - let session = test_session(); - let entry = LiveHlsOriginEntry::parse("http://127.0.0.1:9/live/user/pass/12345.m3u8").expect("valid origin entry"); - let client = reqwest::Client::new(); - let no_redirect_client = - reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()).build().expect("client builds"); - let request = OriginRefreshRequest { - app_config: test_app_config(), - session: Arc::clone(&session), - origin_entry: entry.clone(), - headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - client, - no_redirect_client, - use_manual_redirects: false, - segment_cache: Arc::new(HlsSegmentCache::new()), - hls_proxy: Arc::new(HlsProxyManager::new()), - segment_repair: test_segment_repair_manager(), - segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), - map_worker_pool: Arc::new(HlsMapWorkerPool::default()), - origin_manifest_timeout_ms: 1, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - strip: StripConfig { mode: HlsStripMode::Segments, value: 3 }, - retry_policy: RetryPolicy { delays_ms: [0, 0, 0, 0, 0], jitter_max_ms: 0 }, - reverse_proxy_rewrite_secret: b"secret".to_vec(), - transient_resource_ttl_ms: 300_000, - manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, - fresh_manifest_requirement_generation: None, - acceptance_directive: HlsManifestAcceptanceDirective::none(), - access_lease_id: None, - disabled_headers: None, - now_ms: 100, - origin_io: None, - post_refresh_runtime: None, - }; - - let mut handles = Vec::new(); - for _ in 0..8 { - let request = request.clone(); - handles.push(tokio::spawn(async move { maybe_trigger_origin_refresh(request).await })); - } - - let started = futures::future::join_all(handles) - .await - .into_iter() - .filter(|result| result.as_ref().is_ok_and(|started| *started)) - .count(); - assert_eq!(started, 1); -} - -#[tokio::test] -async fn fresh_manifest_commit_bypasses_refresh_debounce() { - let session = test_session(); - session.write().await.origin_refresh.next_fetch_allowed_at_ms = 10_000; - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.manifest_commit_requirement = - HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; - - assert!(mark_origin_refresh_started(&mut request, 1_000).await); - assert!(session.read().await.origin_refresh.in_flight); -} - -#[tokio::test] -async fn committed_manifest_refresh_still_obeys_debounce() { - let session = test_session(); - session.write().await.origin_refresh.next_fetch_allowed_at_ms = 10_000; - let mut request = test_origin_refresh_request(Arc::clone(&session)); - - assert_eq!( - mark_origin_refresh_started_with_outcome(&mut request, 1_000).await, - HlsOriginRefreshTriggerOutcome::DebouncedUntil { retry_at_ms: 10_000 } - ); - assert!(!session.read().await.origin_refresh.in_flight); -} - -#[tokio::test] -async fn concurrent_refresh_suppression_reports_in_flight_state() { - let session = test_session(); - session.write().await.origin_refresh.mark_started(900); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - - assert_eq!( - mark_origin_refresh_started_with_outcome(&mut request, 1_000).await, - HlsOriginRefreshTriggerOutcome::InFlight - ); - assert_eq!(session.read().await.origin_refresh.last_fetch_started_at_ms, Some(900)); -} - -#[tokio::test] -async fn hls_manifest_acceptance_directive_stale_guard_prevents_refresh_start() { - let hls_proxy = Arc::new(HlsProxyManager::new()); - let (session, _) = hls_proxy - .get_or_create_session_with_outcome(HlsSessionKey::new(1, "stale-pressure-guard"), b"secret", 100) - .await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let owner_key = hls_proxy - .availability_reevaluation_owner_key(&session, &proxy_session_id) - .await - .expect("registered session has recovery-pressure evidence"); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.hls_proxy = Arc::clone(&hls_proxy); - request.acceptance_directive.recovery_pressure_guard = - Some(super::super::availability_reevaluation::HlsRecoveryPressureGuard::from_owner_key(&owner_key)); - session.write().await.origin_control.progress_generation = owner_key.origin_progress_generation.saturating_add(1); - - assert_eq!( - mark_origin_refresh_started_with_outcome(&mut request, 100).await, - HlsOriginRefreshTriggerOutcome::RecoveryPressureSuperseded - ); - let session = session.read().await; - assert!(!session.origin_refresh.in_flight); - assert!(session.origin_control.acceptance_episode.is_none()); -} - -#[tokio::test] -async fn hls_availability_reevaluation_cursor_evidence_supersedes_refresh_guard() { - let hls_proxy = Arc::new(HlsProxyManager::new()); - let (session, _) = hls_proxy - .get_or_create_session_with_outcome(HlsSessionKey::new(1, "cursor-pressure-guard"), b"secret", 100) - .await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("cursor-pressure-guard".to_string()); - { - let mut leases = hls_proxy.access_leases().write().await; - assert!(leases.prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("user", "client"), - proxy_session_id.clone(), - "user".to_string(), - "session-token".to_string(), - 1, - "cursor-pressure-guard".to_string(), - 1, - 100, - 60_000, - ))); - assert!(leases - .activate_access_lease( - &lease_id, - &proxy_session_id, - 100, - HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 }, - ) - .is_activated()); - } - let owner_key = hls_proxy - .availability_reevaluation_owner_key(&session, &proxy_session_id) - .await - .expect("live lease has availability evidence"); - { - let mut leases = hls_proxy.access_leases().write().await; - let identity = leases - .response_snapshot(&lease_id, &proxy_session_id, 100) - .and_then(|lease| lease.media_identity()) - .expect("live lease identity"); - assert!(leases - .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 20, 101,) - .is_some()); - } - let current_owner_key = hls_proxy - .availability_reevaluation_owner_key(&session, &proxy_session_id) - .await - .expect("updated availability evidence"); - assert!(current_owner_key.availability_evidence_generation > owner_key.availability_evidence_generation); - assert_eq!(current_owner_key.origin_progress_generation, owner_key.origin_progress_generation); - assert_eq!(current_owner_key.media_readiness_generation, owner_key.media_readiness_generation); - - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.hls_proxy = Arc::clone(&hls_proxy); - request.acceptance_directive.recovery_pressure_guard = - Some(super::super::availability_reevaluation::HlsRecoveryPressureGuard::from_owner_key(&owner_key)); - assert_eq!( - mark_origin_refresh_started_with_outcome(&mut request, 102).await, - HlsOriginRefreshTriggerOutcome::RecoveryPressureSuperseded - ); - assert!(!session.read().await.origin_refresh.in_flight); -} - -#[tokio::test] -async fn hls_availability_reevaluation_guard_contention_is_typed() { - let hls_proxy = Arc::new(HlsProxyManager::new()); - let (session, _) = hls_proxy - .get_or_create_session_with_outcome(HlsSessionKey::new(1, "contended-pressure-guard"), b"secret", 100) - .await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let owner_key = - hls_proxy.availability_reevaluation_owner_key(&session, &proxy_session_id).await.expect("session evidence"); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.hls_proxy = Arc::clone(&hls_proxy); - request.acceptance_directive.recovery_pressure_guard = - Some(super::super::availability_reevaluation::HlsRecoveryPressureGuard::from_owner_key(&owner_key)); - let lease_guard = hls_proxy.access_leases().write().await; - - assert_eq!( - mark_origin_refresh_started_with_outcome(&mut request, 100).await, - HlsOriginRefreshTriggerOutcome::RecoveryPressureStateContention - ); - assert!(!session.read().await.origin_refresh.in_flight); - drop(lease_guard); -} - -#[tokio::test] -async fn hls_availability_reevaluation_other_session_evidence_keeps_guard_current() { - let hls_proxy = Arc::new(HlsProxyManager::new()); - let (session, _) = - hls_proxy.get_or_create_session_with_outcome(HlsSessionKey::new(1, "guard-session-a"), b"secret", 100).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let owner_key = - hls_proxy.availability_reevaluation_owner_key(&session, &proxy_session_id).await.expect("session evidence"); - let other_proxy_session_id = ProxySessionId("guard-session-b".to_string()); - assert!(hls_proxy.access_leases().write().await.prepare_access_lease(HlsAccessLease::pending( - HlsAccessLeaseId("guard-session-b".to_string()), - HlsPlaybackFamilyKey::new("other", "client"), - other_proxy_session_id, - "other".to_string(), - "other-session".to_string(), - 1, - "guard-session-b".to_string(), - 2, - 100, - 60_000, - ))); - let guard = super::super::availability_reevaluation::HlsRecoveryPressureGuard::from_owner_key(&owner_key); - - assert!(matches!( - hls_proxy.with_current_recovery_pressure_session(&session, &guard, |_| 7_u8), - super::super::availability_reevaluation::HlsRecoveryPressureGuardAccess::Acquired(7) - )); -} - -#[tokio::test] -async fn hls_availability_reevaluation_removed_lease_supersedes_old_guard() { - let hls_proxy = Arc::new(HlsProxyManager::new()); - let (session, _) = hls_proxy - .get_or_create_session_with_outcome(HlsSessionKey::new(1, "removed-pressure-guard"), b"secret", 100) - .await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("removed-pressure-guard".to_string()); - assert!(hls_proxy.access_leases().write().await.prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("user", "client"), - proxy_session_id.clone(), - "user".to_string(), - "session-token".to_string(), - 1, - "removed-pressure-guard".to_string(), - 1, - 100, - 60_000, - ))); - let owner_key = - hls_proxy.availability_reevaluation_owner_key(&session, &proxy_session_id).await.expect("lease evidence"); - assert!(hls_proxy.access_leases().write().await.remove_access_lease(&lease_id).is_some()); - let guard = super::super::availability_reevaluation::HlsRecoveryPressureGuard::from_owner_key(&owner_key); - - assert!(matches!( - hls_proxy.with_current_recovery_pressure_session(&session, &guard, |_| ()), - super::super::availability_reevaluation::HlsRecoveryPressureGuardAccess::Superseded - )); - assert!(!session.read().await.origin_refresh.in_flight); -} - -fn cutover_live_manifest_snapshot() -> HlsLeaseManifestSnapshot { - HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(1_000), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 0, - delivered_at_ms: 1_000, - first_proxy_seq: 40, - last_proxy_seq: 41, - visible_segments: Arc::from([ - HlsLeaseManifestSegment { - proxy_seq: 40, - duration_ms: 4_000, - uri: "/live/40.ts".into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }, - HlsLeaseManifestSegment { - proxy_seq: 41, - duration_ms: 4_000, - uri: "/live/41.ts".into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }, - ]), - discontinuity_sequence: 0, - target_duration_ms: 4_000, - playlist_duration_ms: 8_000, - last_visible_media_end_ms: 8_000, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - } -} - -async fn assert_recovery_supersedes_terminal_preparation( - hls_proxy: &HlsProxyManager, - session: &Arc>, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, - preparation: &super::super::lease::HlsTerminalTailPreparation, -) { - assert_eq!( - hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { - session, - lease_id, - proxy_session_id, - preparation, - now_ms: 2_100, - payload: HlsTerminalCommitPayload::Unavailable(HlsTerminalTailCompatibility::MissingAsset), - asset_revision_guard: super::super::terminal_commit::HlsTerminalAssetRevisionGuard::matching_for_test(None), - }), - super::super::terminal_commit::HlsTerminalCommitOutcome::RecoveryCommitted - ); - let lease = hls_proxy - .access_lease_response_snapshot(lease_id, proxy_session_id, 2_100) - .await - .expect("recovered lease remains live"); - assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); - assert!(!session.read().await.has_terminal_tail_protections()); -} - -async fn prepare_failed_acceptance_episode(session: &Arc>) -> u64 { - let mut session = session.write().await; - let burst_plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - session.origin_control.begin_acceptance_episode( - 1_000, - burst_plan, - HlsManifestAcceptanceTrigger::RecoveryRequired, - &test_acceptance_episode_timing(1_000, burst_plan), - ); - let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); - episode.record_full_burst(); - episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::AllFailed); - episode.hold_after_uncommitted_burst(None, Some(2_000)); - session.origin_control.progress_generation -} - -#[tokio::test] -async fn hls_cutover_policy_advanced_commit_supersedes_preparation_and_keeps_lease_live() { - let hls_proxy = Arc::new(HlsProxyManager::new()); - let (session, _) = - hls_proxy.get_or_create_session_with_outcome(HlsSessionKey::new(1, "12345"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - hls_proxy.access_leases().write().await.prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("user", "client"), - proxy_session_id.clone(), - "user".to_string(), - "session-token".to_string(), - 1, - "12345".to_string(), - 12345, - 1_000, - 60_000, - )); - let publication_guard = hls_proxy - .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, 1_000) - .await - .expect("live manifest publication"); - assert!(hls_proxy - .commit_access_lease_manifest_publication( - &lease_id, - &proxy_session_id, - publication_guard, - cutover_live_manifest_snapshot(), - 1_000, - ) - .await - .is_committed()); - let prepared_progress_generation = prepare_failed_acceptance_episode(&session).await; - let transition_margin = HlsTransitionMarginMs::from_millis(4_000); - let guaranteed_reserve_ms = transition_margin - .as_millis() - .saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()); - let reserve = HlsLeaseReserveSnapshot { - availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, - guaranteed_media_horizon_ms: 8_000_u64.saturating_add(guaranteed_reserve_ms), - conservative_playback_position_ms: 8_000, - guaranteed_reserve_ms, - initial_hidden_ready_duration_ms: 0, - transition_margin, - key_readiness_valid_until_ms: None, - recovery_required: true, - cutover_required: false, - }; - let cutover_timing = - HlsLeaseCutoverTiming::from_reserve(2_000, reserve.guaranteed_reserve_ms, reserve.transition_margin, None); - let preparation = hls_proxy - .prepare_access_lease_terminal_tail(HlsTerminalTailPreparationRequest { - lease_id: &lease_id, - proxy_session_id: &proxy_session_id, - manifest_snapshot_generation: 1, - cursor_generation: 0, - reserve, - cutover_timing, - commit_window: HlsTerminalCommitWindow::AcquisitionOpen, - now_ms: 2_000, - origin_progress_generation: prepared_progress_generation, - media_readiness_generation: 0, - last_media_progress_at_ms: None, - }) - .await - .expect("exhausted acceptance permits terminal preparation"); - - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.hls_proxy = Arc::clone(&hls_proxy); - request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; - let fetched = fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:42\n#EXTINF:4.0,\n42.ts\n"); - { - let mut session = session.write().await; - let (progress_evidence, _, _) = - commit_fetched_manifest(&mut session, &fetched, &request, 2_100).expect("recovery manifest commits"); - assert_eq!(progress_evidence.refresh_timing().progress, HlsManifestProgress::Advanced); - assert_eq!(session.origin_control.progress_generation, prepared_progress_generation.saturating_add(1)); - assert_eq!(session.origin_control.last_media_progress_at_ms, Some(2_100)); - } - - assert_recovery_supersedes_terminal_preparation(&hls_proxy, &session, &proxy_session_id, &lease_id, &preparation) - .await; -} - -#[tokio::test] -async fn successful_manifest_commit_shortens_pending_leases_without_response_path() { - let session = test_session(); - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let hls_proxy = Arc::new(HlsProxyManager::new()); - let now_ms = super::current_time_millis(); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - hls_proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("user", "client"), - proxy_session_id.clone(), - "user".to_string(), - "session-token".to_string(), - 1, - "12345".to_string(), - 12345, - now_ms, - 90_000, - )) - .await; - let server = spawn_test_origin(Arc::new(|_path| { - (200, Vec::new(), "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nseg.ts\n".to_string()) - })) - .await; - let entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) - .expect("valid origin entry"); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.hls_proxy = Arc::clone(&hls_proxy); - request.origin_entry = entry; - request.now_ms = now_ms; - - assert!(Box::pin(trigger_origin_refresh_sync(request)).await); - - let lease = hls_proxy - .access_leases() - .write() - .await - .response_snapshot(&lease_id, &proxy_session_id, super::current_time_millis()) - .expect("pending lease should remain available"); - let Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms }) = lease.pending_deadline else { - panic!("pending lease should be shortened to follow-up"); - }; - assert!(deadline_ms < now_ms.saturating_add(90_000)); - assert!(deadline_ms <= super::current_time_millis().saturating_add(10_000)); -} - -async fn refresh_session_with_origin_body(body: &'static str) -> Arc> { - let session = test_session(); - let server = spawn_test_origin(Arc::new(move |_path| (200, Vec::new(), body.to_string()))).await; - let entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) - .expect("valid origin entry"); - let request = OriginRefreshRequest { - app_config: test_app_config(), - session: Arc::clone(&session), - origin_entry: entry.clone(), - headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("client builds"), - use_manual_redirects: false, - segment_cache: Arc::new(HlsSegmentCache::new()), - hls_proxy: Arc::new(HlsProxyManager::new()), - segment_repair: test_segment_repair_manager(), - segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), - map_worker_pool: Arc::new(HlsMapWorkerPool::default()), - origin_manifest_timeout_ms: 2_000, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - strip: StripConfig { mode: HlsStripMode::Segments, value: 3 }, - retry_policy: no_delay_policy(), - reverse_proxy_rewrite_secret: b"secret".to_vec(), - transient_resource_ttl_ms: 300_000, - manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, - fresh_manifest_requirement_generation: None, - acceptance_directive: HlsManifestAcceptanceDirective::none(), - access_lease_id: None, - disabled_headers: None, - now_ms: 100, - origin_io: None, - post_refresh_runtime: None, - }; - - assert!(trigger_origin_refresh_sync(request).await); - { - let session = session.read().await; - assert!( - !session.segments.is_empty() || session.transient.last_manifest_body.is_some(), - "synchronous controlled origin work must commit origin state: mode={:?} path_condition={:?} failures={}", - session.mode, - session.origin_control.path_condition, - session.origin_refresh.consecutive_failures - ); - } - session -} - -#[tokio::test] -async fn refresh_stores_headers_after_hls_origin_policy() { - let session = test_session(); - let server = spawn_test_origin(Arc::new(|_path| { - (200, Vec::new(), "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nseg.ts\n".to_string()) - })) - .await; - let entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) - .expect("valid origin entry"); - let mut headers = HeaderMap::new(); - headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); - headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); - headers.insert(HeaderName::from_static("proxy-authorization"), HeaderValue::from_static("Basic secret")); - headers.insert(header::HOST, HeaderValue::from_static("proxy.example.com")); - headers.insert(HeaderName::from_static("x-blocked"), HeaderValue::from_static("blocked")); - headers.insert(HeaderName::from_static("cf-ray"), HeaderValue::from_static("cf")); - headers.insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("de")); - - let request = OriginRefreshRequest { - app_config: test_app_config(), - session: Arc::clone(&session), - origin_entry: entry.clone(), - headers, - origin_provider_session_headers: HeaderMap::new(), - disabled_headers: Some(ReverseProxyDisabledHeaderConfig { - referer_header: false, - x_header: true, - cloudflare_header: true, - custom_header: Vec::new(), - }), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("client builds"), - use_manual_redirects: false, - segment_cache: Arc::new(HlsSegmentCache::new()), - hls_proxy: Arc::new(HlsProxyManager::new()), - segment_repair: test_segment_repair_manager(), - segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), - map_worker_pool: Arc::new(HlsMapWorkerPool::default()), - origin_manifest_timeout_ms: 2_000, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, - retry_policy: no_delay_policy(), - reverse_proxy_rewrite_secret: b"secret".to_vec(), - transient_resource_ttl_ms: 300_000, - manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, - fresh_manifest_requirement_generation: None, - acceptance_directive: HlsManifestAcceptanceDirective::none(), - access_lease_id: None, - now_ms: 100, - origin_io: None, - post_refresh_runtime: None, - }; - - assert!(maybe_trigger_origin_refresh(request).await); - for _ in 0..50 { - if session.read().await.last_rendered_manifest.is_some() { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - - let session = session.read().await; - assert!(!session.origin_request_headers.contains_key(header::AUTHORIZATION)); - assert!(!session.origin_request_headers.contains_key(header::COOKIE)); - assert!(!session.origin_request_headers.contains_key("proxy-authorization")); - assert!(!session.origin_request_headers.contains_key(header::HOST)); - assert!(!session.origin_request_headers.contains_key("x-blocked")); - assert!(!session.origin_request_headers.contains_key("cf-ray")); - assert_eq!(session.origin_request_headers.get(header::ACCEPT_LANGUAGE).expect("language"), "de"); -} - -#[tokio::test] -async fn accepted_manifest_commit_stores_provider_session_cookie_separately() { - let session = test_session(); - let server = spawn_test_origin(Arc::new(|_path| { - ( - 200, - vec![ - ("Set-Cookie", "sid=abc; Path=/; HttpOnly".to_string()), - ("Set-Cookie", "pref=1; SameSite=Lax".to_string()), - ], - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nseg.ts\n".to_string(), - ) - })) - .await; - let entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) - .expect("valid origin entry"); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = entry; - - assert!(Box::pin(trigger_origin_refresh_sync(request)).await); - - let session = session.read().await; - assert!(!session.origin_request_headers.contains_key(header::COOKIE)); - assert_eq!( - session.origin_provider_session_headers.get(header::COOKIE).expect("provider cookie"), - "sid=abc; pref=1" - ); -} - -#[test] -fn origin_account_binding_change_clears_provider_session_headers() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "1"), b"secret", 100); - session.origin_provider_session_headers.insert(header::COOKIE, HeaderValue::from_static("sid=abc")); - session.replace_origin_account_binding(Some(HlsOriginAccountBinding::new( - Arc::::from("input"), - Arc::::from("account-a"), - &session.proxy_session_id.clone(), - 100, - ))); - assert!(session.origin_provider_session_headers.is_empty()); - - session.origin_provider_session_headers.insert(header::COOKIE, HeaderValue::from_static("sid=next")); - session.replace_origin_account_binding(Some(HlsOriginAccountBinding::new( - Arc::::from("input"), - Arc::::from("account-a"), - &session.proxy_session_id.clone(), - 200, - ))); - assert!(!session.origin_provider_session_headers.is_empty()); - - session.replace_origin_account_binding(Some(HlsOriginAccountBinding::new( - Arc::::from("input"), - Arc::::from("account-b"), - &session.proxy_session_id.clone(), - 300, - ))); - assert!(session.origin_provider_session_headers.is_empty()); -} - -#[tokio::test] -async fn ext_x_key_manifest_commits_transient_rewrite() { - let session = refresh_session_with_origin_body( - "#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg.ts\n", - ) - .await; - let session = session.read().await; - let body = session.transient.last_manifest_body.as_ref().expect("transient body"); - - assert!(matches!( - session.mode, - HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey } - )); - assert!(body.contains("/hls/shared/live/")); - assert!(body.contains("/r/")); - assert!(!body.contains("/hls/user/")); - assert_eq!(session.transient.resources.len(), 2); - assert_eq!(session.target_duration, Some(12)); - assert_eq!(session.account_overlap_timing().target_duration_ms, 12_000); -} - -#[tokio::test] -async fn finalized_event_manifest_commits_full_transient_lifecycle() { - let mut origin_body = - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:1\n".to_string(); - for sequence in 1..=8 { - let _ = write!(origin_body, "#EXTINF:4.0,\n{sequence}.ts\n"); - } - origin_body.push_str("#EXT-X-ENDLIST\n"); - let origin_body: &'static str = Box::leak(origin_body.into_boxed_str()); - let session = refresh_session_with_origin_body(origin_body).await; - let session = session.read().await; - let body = session.transient.last_manifest_body.as_ref().expect("finalized transient body"); - - assert!(matches!( - session.mode, - HlsSessionMode::TransientPassthrough { - reason: TransientPassthroughReason::UnsupportedTag { ref tag } - } if tag == "#EXT-X-PLAYLIST-TYPE" - )); - let stored_body_lifecycle = tuliprox_parser::hls::origin_manifest::parse_manifest_semantics(body).lifecycle(); - assert_eq!(session.transient.last_manifest_finalized(), stored_body_lifecycle.is_finalized()); - assert_eq!( - session.transient.last_manifest_window_policy(), - tuliprox_parser::hls::origin_manifest::HlsManifestWindowPolicy::PreserveFullManifest - ); - assert_eq!(session.transient.current_manifest_resource_ids().len(), 8); - assert_eq!(session.transient.last_manifest_playlist_duration_ms, Some(32_000)); - assert_eq!(session.transient.last_manifest_valid_until_ms(), None); - assert!(body.contains("#EXT-X-PLAYLIST-TYPE:EVENT")); - assert!(body.contains("#EXT-X-ENDLIST")); - assert_eq!(body.matches("/r/").count(), 8); -} - -#[tokio::test] -async fn endlist_only_manifest_commits_complete_body_with_consistent_finalized_state() { - let mut origin_body = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n".to_string(); - for sequence in 1..=8 { - let _ = write!(origin_body, "#EXTINF:4.0,\n{sequence}.ts\n"); - } - origin_body.push_str("#EXT-X-ENDLIST\n"); - let origin_body: &'static str = Box::leak(origin_body.into_boxed_str()); - let session = refresh_session_with_origin_body(origin_body).await; - let session = session.read().await; - let body = session.transient.last_manifest_body.as_ref().expect("finalized transient body"); - - assert!(matches!( - session.mode, - HlsSessionMode::TransientPassthrough { - reason: TransientPassthroughReason::UnsupportedTag { ref tag } - } if tag == "#EXT-X-ENDLIST" - )); - let stored_body_lifecycle = tuliprox_parser::hls::origin_manifest::parse_manifest_semantics(body).lifecycle(); - assert_eq!(session.transient.last_manifest_finalized(), stored_body_lifecycle.is_finalized()); - assert_eq!( - session.transient.last_manifest_window_policy(), - tuliprox_parser::hls::origin_manifest::HlsManifestWindowPolicy::PreserveFullManifest - ); - assert_eq!(session.transient.current_manifest_resource_ids().len(), 8); - assert_eq!(body.matches("/r/").count(), 8); - assert!(body.contains("#EXT-X-ENDLIST")); - assert!(stored_body_lifecycle.is_finalized()); -} - -#[test] -fn duplicate_media_units_are_rejected_locally_before_transient_commit() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; - let request = test_origin_refresh_request(test_session()); - let body = repeated_transient_manifest(crate::manifest_limits::MAX_HLS_LEASE_SNAPSHOT_SEGMENTS + 1); - - let result = commit_fetched_manifest(&mut session, &fetched_manifest(&body), &request, 100); - - assert!(matches!( - result, - Err(HlsManifestCommitError::LocalRepresentationLimit(violation)) - if violation.kind == crate::manifest_limits::HlsManifestLimitKind::LeaseSnapshotSegments - )); - assert!(session.transient.last_manifest_body.is_none()); - let signal = classify_manifest_fetch_failure(&OriginManifestFetchError::LocalRepresentationLimit( - crate::manifest_limits::HlsManifestLimitViolation::new( - crate::manifest_limits::HlsManifestLimitKind::LeaseSnapshotSegments, - crate::manifest_limits::MAX_HLS_LEASE_SNAPSHOT_SEGMENTS + 1, - crate::manifest_limits::MAX_HLS_LEASE_SNAPSHOT_SEGMENTS, - ), - )); - assert_eq!(signal, HlsManifestFetchFailureSignal::discarded(HlsManifestFetchFailureKind::LocalRepresentationLimit)); -} - -#[test] -fn transient_commit_generation_exhaustion_is_not_reported_as_manifest_limit() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; - session.set_manifest_commit_generation_for_test(u64::MAX); - let request = test_origin_refresh_request(test_session()); - let manifest_limit_rejections_before = request.segment_worker_pool.metrics().snapshot().manifest_limit_rejections; - let fetched = fetched_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4,\nseg.ts\n", - ); - - let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); - - assert!(matches!(result, Err(HlsManifestCommitError::CommitGenerationExhausted))); - assert_eq!( - request.segment_worker_pool.metrics().snapshot().manifest_limit_rejections, - manifest_limit_rejections_before - ); - assert!(session.transient.last_manifest_body.is_none()); -} - -#[tokio::test] -async fn initial_local_representation_limit_does_not_enter_origin_recovery() { - let session = test_session(); - session.write().await.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; - let request = test_origin_refresh_request(Arc::clone(&session)); - let fetch_context = manifest_fetch_context(&request); - let fetched = - fetched_manifest(&repeated_transient_manifest(crate::manifest_limits::MAX_HLS_LEASE_SNAPSHOT_SEGMENTS + 1)); - let recovery_binding = - test_manifest_origin_binding(Url::parse("http://127.0.0.1:9/recovery.m3u8").expect("test recovery URL")); - - let result = - super::commit_initial_fetched_manifest(&request, &fetch_context, fetched, Some(recovery_binding), false).await; - - assert!(matches!( - result, - Err(OriginManifestFetchError::LocalRepresentationLimit(violation)) - if violation.kind == crate::manifest_limits::HlsManifestLimitKind::LeaseSnapshotSegments - )); -} - -#[tokio::test] -async fn initial_transient_manifest_without_snapshot_template_is_rejected_locally() { - let session = test_session(); - session.write().await.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; - let request = test_origin_refresh_request(Arc::clone(&session)); - let fetch_context = manifest_fetch_context(&request); - let fetched = fetched_manifest( - "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:1\n\ - #EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4,\nsegment.ts\n#EXT-X-ENDLIST\n", - ); - let recovery_binding = - test_manifest_origin_binding(Url::parse("http://127.0.0.1:9/recovery.m3u8").expect("test recovery URL")); - - let result = - super::commit_initial_fetched_manifest(&request, &fetch_context, fetched, Some(recovery_binding), false).await; - - assert!(matches!(result, Err(OriginManifestFetchError::MalformedTransientRepresentation))); - assert_eq!( - classify_manifest_fetch_failure(&OriginManifestFetchError::MalformedTransientRepresentation), - HlsManifestFetchFailureSignal::discarded(HlsManifestFetchFailureKind::MalformedTransientRepresentation) - ); - let session = session.read().await; - assert!(session.transient.last_manifest_body.is_none()); - assert!(session.transient.last_manifest_template().is_none()); - assert!(session.transient.last_manifest_commit_identity().is_none()); - assert_eq!(session.transient.manifest_generation(), 0); - assert!(session.transient.resources.is_empty()); - assert!(crate::manifest_commit::hls_committed_manifest_body_for_request( - &session, - crate::manifest_commit::HlsCachedManifestOptions::initial(Duration::ZERO), - 0, - 100, - ) - .is_none()); -} - -#[test] -fn rejected_transient_candidate_preserves_committed_state_and_pending_handoff() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; - let mut request = test_origin_refresh_request(test_session()); - request.transient_resource_ttl_ms = 1; - let baseline = fetched_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ - #EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4,\nsegment.ts\n", - ); - assert!(commit_fetched_manifest(&mut session, &baseline, &request, 100).is_ok()); - session.mark_pending_handoff_discontinuity(7); - let baseline_body = session.transient.last_manifest_body.clone().expect("baseline manifest"); - let baseline_generation = session.transient.manifest_generation(); - let baseline_resources = session.transient.resources.keys().cloned().collect::>(); - let baseline_highwater = session.origin_seq_highwater; - let key_format = "x".repeat(crate::manifest_limits::MAX_HLS_ENCRYPTION_DIRECTIVE_BYTES); - let candidate = fetched_manifest(&format!( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ - #EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\",KEYFORMAT=\"{key_format}\"\n\ - #EXTINF:4,\nsegment.ts\n" - )); - - let result = commit_fetched_manifest(&mut session, &candidate, &request, 102); - - assert!(matches!( - result, - Err(HlsManifestCommitError::LocalRepresentationLimit(violation)) - if violation.kind - == crate::manifest_limits::HlsManifestLimitKind::LeaseSnapshotEncryptionDirectiveBytes - )); - assert!(Arc::ptr_eq( - session.transient.last_manifest_body.as_ref().expect("baseline remains current"), - &baseline_body - )); - assert_eq!(session.transient.manifest_generation(), baseline_generation); - assert_eq!( - session.transient.resources.keys().cloned().collect::>(), - baseline_resources - ); - assert_eq!(session.pending_handoff_discontinuity_sequence, Some(7)); - assert_eq!(session.origin_seq_highwater, baseline_highwater); -} - -#[test] -fn transient_manifest_without_snapshot_template_preserves_deliverable_commit() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; - let request = test_origin_refresh_request(test_session()); - let baseline = fetched_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:1\n\ - #EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4,\nsegment.ts\n#EXT-X-ENDLIST\n", - ); - assert!(commit_fetched_manifest(&mut session, &baseline, &request, 100).is_ok()); - session.mark_pending_handoff_discontinuity(7); - let baseline_body = session.transient.last_manifest_body.clone().expect("baseline manifest"); - let baseline_template = session.transient.last_manifest_template().expect("baseline template"); - let baseline_generation = session.transient.manifest_generation(); - let baseline_finalized_generation = - session.transient.current_finalized_manifest_generation().expect("baseline finalized generation"); - let baseline_finalized_generation_count = session.transient.finalized_manifest_generation_count(); - let baseline_commit_identity = session.transient.last_manifest_commit_identity().expect("baseline identity"); - let baseline_resources = session.transient.resources.keys().cloned().collect::>(); - let baseline_manifest_resource_ids = session.transient.current_manifest_resource_ids().clone(); - let baseline_highwater = session.origin_seq_highwater; - let candidate = fetched_manifest( - "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:2\n\ - #EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"rotated-key.bin\"\n#EXTINF:4,\nreplacement.ts\n#EXT-X-ENDLIST\n", - ); - - let result = commit_fetched_manifest(&mut session, &candidate, &request, 102); - - assert!(matches!(result, Err(HlsManifestCommitError::MalformedTransientRepresentation))); - assert!(Arc::ptr_eq( - session.transient.last_manifest_body.as_ref().expect("baseline remains current"), - &baseline_body - )); - assert!(Arc::ptr_eq( - &session.transient.last_manifest_template().expect("baseline template remains current"), - &baseline_template - )); - assert_eq!(session.transient.manifest_generation(), baseline_generation); - assert_eq!(session.transient.current_finalized_manifest_generation(), Some(baseline_finalized_generation)); - assert_eq!(session.transient.finalized_manifest_generation_count(), baseline_finalized_generation_count); - assert_eq!(session.transient.last_manifest_commit_identity(), Some(baseline_commit_identity)); - assert_eq!( - session.transient.resources.keys().cloned().collect::>(), - baseline_resources - ); - assert_eq!(session.transient.current_manifest_resource_ids(), &baseline_manifest_resource_ids); - assert_eq!(session.pending_handoff_discontinuity_sequence, Some(7)); - assert_eq!(session.origin_seq_highwater, baseline_highwater); - assert!(session - .transient - .last_manifest_template() - .zip(session.transient.last_manifest_commit_identity()) - .is_some()); -} - -#[tokio::test] -async fn compatible_aes_128_manifest_commits_normal_timeline_with_opaque_key_resource() { - let session = refresh_session_with_origin_body( - "#EXTM3U\n#EXT-X-VERSION:5\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ - #EXT-X-KEY:METHOD=AES-128,URI=\"origin-key.bin\",IV=0x00000000000000000000000000000001,KEYFORMAT=\"identity\",KEYFORMATVERSIONS=\"1\"\n\ - #EXTINF:4,\n1.ts\n#EXTINF:4,\n2.ts\n#EXTINF:4,\n3.ts\n#EXTINF:4,\n4.ts\n#EXTINF:4,\n5.ts\n#EXTINF:4,\n6.ts\n", - ) - .await; - let session = session.read().await; - - assert_eq!(session.mode, HlsSessionMode::NormalCacheTimeline); - assert_eq!(session.transient.resources.len(), 1); - assert!(session.transient.resources.values().all(|resource| resource.kind == TransientResourceKind::Key)); - assert!(session.segments.values().all(|segment| { - segment.encryption.as_ref().is_some_and(|encryption| { - encryption.resource_extension == "bin" - && session.transient.resources.contains_key(&encryption.resource_id) - && !encryption.resource_id.0.contains("origin-key.bin") - }) - })); - assert!( - session.last_rendered_manifest.is_none(), - "a refresh fixture without a usable lease must not publish unavailable media" - ); -} - -#[test] -fn transient_commit_accepts_plausible_same_redirect_host_rollover_and_resets_highwater() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; - session.origin_seq_highwater = Some(758); - session.last_effective_manifest_host = Some("origin.example.com".to_string()); - session.mark_authorized_media_access(100); - let previous_manifest = - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:757\n#EXTINF:4.0,\n/hls/shared/live/session/lease/r/old.ts\n".to_string(); - session.transient.replace_manifest_with_semantics(previous_manifest.clone(), 10, None); - let request = test_origin_refresh_request(test_session()); - let fetched = fetched_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg.ts\n", - ); - - let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); - - assert!(result.is_ok()); - assert_ne!(session.transient.last_manifest_body.as_deref(), Some(previous_manifest.as_str())); - assert_eq!(session.origin_seq_highwater, Some(0)); - assert!(session.transient.last_manifest_body.as_ref().is_some_and(|body| body.contains("/r/"))); -} - -#[test] -fn transient_commit_rejects_same_host_backward_manifest_outside_rollover_window() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; - session.origin_seq_highwater = Some(758); - session.last_effective_manifest_host = Some("origin.example.com".to_string()); - session.mark_authorized_media_access(100); - let previous_manifest = - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:757\n#EXTINF:4.0,\n/hls/shared/live/session/lease/r/old.ts\n".to_string(); - session.transient.replace_manifest_with_semantics(previous_manifest.clone(), 10, None); - let request = test_origin_refresh_request(test_session()); - let fetched = fetched_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:226\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg.ts\n", - ); - - let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); - - assert!(matches!(result, Err(HlsManifestCommitError::TimelineRejected { .. }))); - assert_eq!(session.transient.last_manifest_body.as_deref(), Some(previous_manifest.as_str())); - assert_eq!(session.origin_seq_highwater, Some(758)); -} - -#[test] -fn transient_commit_rebases_expired_session_highwater() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; - session.target_duration = Some(12); - session.origin_seq_highwater = Some(758); - session.last_effective_manifest_host = Some("origin.example.com".to_string()); - session.mark_authorized_media_access(1_000); - let previous_manifest = - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:757\n#EXTINF:4.0,\n/hls/shared/live/session/lease/r/old.ts\n".to_string(); - session.transient.replace_manifest_with_semantics(previous_manifest.clone(), 10, None); - let request = test_origin_refresh_request(test_session()); - let fetched = fetched_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg.ts\n", - ); - - let result = commit_fetched_manifest(&mut session, &fetched, &request, 40_000); - - assert!(result.is_ok()); - assert_ne!(session.transient.last_manifest_body.as_deref(), Some(previous_manifest.as_str())); - assert_eq!(session.origin_seq_highwater, Some(900)); -} - -#[test] -fn transient_commit_accepts_monotonic_media_sequence_and_updates_highwater() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; - session.origin_seq_highwater = Some(758); - session.last_effective_manifest_host = Some("origin.example.com".to_string()); - session.transient.replace_manifest_with_semantics( - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:757\n#EXTINF:4.0,\n/hls/shared/live/session/lease/r/old.ts\n".to_string(), - 10, - None, - ); - let request = test_origin_refresh_request(test_session()); - let fetched = fetched_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:759\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg759.ts\n#EXTINF:4.0,\nseg760.ts\n", - ); - - let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); - - assert!(result.is_ok()); - assert_eq!(session.origin_seq_highwater, Some(760)); - assert!(session.transient.last_manifest_body.as_ref().is_some_and(|body| body.contains("/r/"))); -} - -#[test] -fn transient_commit_with_different_redirect_host_is_held_as_candidate() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; - session.origin_seq_highwater = Some(758); - session.last_effective_manifest_host = Some("previous.example.com".to_string()); - let request = test_origin_refresh_request(test_session()); - let fetched = fetched_manifest( - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:758\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg758.ts\n#EXTINF:4.0,\nseg759.ts\n", - ); - - let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); - - assert!(matches!(result, Err(HlsManifestCommitError::RetryCurrentTarget))); - assert_eq!(session.origin_seq_highwater, Some(758)); - assert!(session.transient.last_manifest_body.is_none()); -} - -#[test] -fn fresh_revalidation_rebases_transient_manifest_on_the_pinned_host() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; - session.origin_seq_highwater = Some(1_000); - session.last_effective_manifest_host = Some("origin.example.com".to_string()); - let mut request = test_origin_refresh_request(test_session()); - request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { - reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, - }; - let fetched = fetched_manifest( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:10\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg10.ts\n", - ); - - let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); - - assert!(result.is_ok()); - assert_eq!(session.origin_seq_highwater, Some(10)); - assert_eq!(session.last_effective_manifest_host.as_deref(), Some("origin.example.com")); - assert!(session.transient.last_manifest_body.as_ref().is_some_and(|body| body.contains("/r/"))); -} - -#[tokio::test] -async fn unsupported_tag_manifest_commits_transient_rewrite() { - let session = refresh_session_with_origin_body( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-PART:DURATION=1.0,URI=\"part.m4s\"\n\ - #EXTINF:4.0,\nseg.ts\n", - ) - .await; - let session = session.read().await; - - assert!(matches!( - session.mode, - HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::UnsupportedTag { .. } } - )); - assert!(session.transient.last_manifest_body.is_some()); -} - -#[tokio::test] -async fn parser_unsupported_feature_manifest_commits_transient_rewrite() { - let session = refresh_session_with_origin_body( - "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-BYTERANGE:10\n#EXTINF:4.0,\nseg.ts\n", - ) - .await; - let session = session.read().await; - - assert!(matches!( - session.mode, - HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ParserUnsupportedFeature { .. } } - )); - assert!(session.transient.last_manifest_body.is_some()); -} - -struct TestOriginServer { - base_url: String, - requests: Arc>>, - raw_requests: Arc>>, - task: tokio::task::JoinHandle<()>, -} - -type TestOriginHandler = Arc (u16, Vec<(&'static str, String)>, String) + Send + Sync>; - -impl Drop for TestOriginServer { - fn drop(&mut self) { self.task.abort(); } -} - -async fn spawn_test_origin(handler: TestOriginHandler) -> TestOriginServer { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let addr = listener.local_addr().expect("local addr"); - let requests = Arc::new(Mutex::new(Vec::new())); - let requests_for_task = Arc::clone(&requests); - let raw_requests = Arc::new(Mutex::new(Vec::new())); - let raw_requests_for_task = Arc::clone(&raw_requests); - let task = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let requests = Arc::clone(&requests_for_task); - let raw_requests = Arc::clone(&raw_requests_for_task); - let handler = Arc::clone(&handler); - tokio::spawn(async move { - let mut buf = vec![0_u8; 4096]; - let mut used = 0_usize; - loop { - let Ok(read) = socket.read(&mut buf[used..]).await else { - return; - }; - if read == 0 { - return; - } - used += read; - if used >= 4 && buf[..used].windows(4).any(|window| window == b"\r\n\r\n") { - break; - } - if used == buf.len() { - return; - } - } - let request = String::from_utf8_lossy(&buf[..used]).into_owned(); - let path = - request.lines().next().and_then(|line| line.split_whitespace().nth(1)).unwrap_or("/").to_string(); - requests.lock().await.push(path.clone()); - raw_requests.lock().await.push(request); - let (status, headers, body) = handler(path); - let reason = match status { - 200 => "OK", - 302 => "Found", - 404 => "Not Found", - 407 => "Proxy Authentication Required", - 500 => "Internal Server Error", - _ => "Status", - }; - let mut response = - format!("HTTP/1.1 {status} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n", body.len()); - for (name, value) in headers { - response.push_str(name); - response.push_str(": "); - response.push_str(&value); - response.push_str("\r\n"); - } - response.push_str("\r\n"); - response.push_str(&body); - let _ = socket.write_all(response.as_bytes()).await; - }); - } - }); - TestOriginServer { base_url: format!("http://{addr}"), requests, raw_requests, task } -} - -fn request_header_value<'a>(request: &'a str, expected_name: &str) -> Option<&'a str> { - request.lines().find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case(expected_name).then_some(value.trim()) - }) -} - -fn no_delay_policy() -> RetryPolicy { RetryPolicy { delays_ms: [0, 0, 0, 0, 0], jitter_max_ms: 0 } } - -fn test_recovery_timing_policy(expected_eta_ms: u64) -> HlsRecoveryTimingPolicy { - HlsRecoveryTimingPolicy::new( - HlsOperationTimeoutMs::from_millis(2_000), - HlsOperationTimeoutMs::from_millis(2_000), - HlsRecoveryEtaMs::from_millis(expected_eta_ms), - HlsRecoveryEtaMs::from_millis(expected_eta_ms), - ) -} - -fn test_acceptance_episode_timing( - started_at_ms: u64, - burst_plan: shared::model::HlsManifestRecoveryBurstPlan, -) -> HlsAcceptanceEpisodeTiming { - HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput { - started_at_ms, - burst_plan, - target_duration_ms: 4_000, - transition_margin: HlsTransitionMarginMs::from_millis(4_000), - workload: HlsRecoveryWorkload::clear_fetch(), - observed_latency: HlsObservedRecoveryLatency::default(), - required_terminal_media_key: None, - terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, - policy: test_recovery_timing_policy(2_000), - }) -} - -fn test_switch_staging_acceptance_episode_timing( - started_at_ms: u64, - burst_plan: shared::model::HlsManifestRecoveryBurstPlan, -) -> HlsAcceptanceEpisodeTiming { - HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput { - started_at_ms, - burst_plan, - target_duration_ms: 4_000, - transition_margin: HlsTransitionMarginMs::from_millis(4_000), - workload: HlsRecoveryWorkloadEnvelope::acceptance_policy().ceiling(), - observed_latency: HlsObservedRecoveryLatency::default(), - required_terminal_media_key: None, - terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, - policy: test_recovery_timing_policy(2_000), - }) -} - -fn test_origin_refresh_request(session: Arc>) -> OriginRefreshRequest { - let entry = - LiveHlsOriginEntry::parse("http://origin.example.com/live/user/pass/12345.m3u8").expect("valid origin entry"); - OriginRefreshRequest { - app_config: test_app_config(), - session, - origin_entry: entry.clone(), - headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("client builds"), - use_manual_redirects: false, - segment_cache: Arc::new(HlsSegmentCache::new()), - hls_proxy: Arc::new(HlsProxyManager::new()), - segment_repair: test_segment_repair_manager(), - segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), - map_worker_pool: Arc::new(HlsMapWorkerPool::default()), - origin_manifest_timeout_ms: 2_000, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, - retry_policy: no_delay_policy(), - reverse_proxy_rewrite_secret: b"secret".to_vec(), - transient_resource_ttl_ms: 300_000, - manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, - fresh_manifest_requirement_generation: None, - acceptance_directive: HlsManifestAcceptanceDirective::none(), - access_lease_id: None, - disabled_headers: None, - now_ms: 100, - origin_io: None, - post_refresh_runtime: None, - } -} - -fn bind_refresh_request_to_app_state(mut request: OriginRefreshRequest, ctx: &crate::HlsCtx) -> OriginRefreshRequest { - request.segment_cache = Arc::clone(ctx.hls_proxy.segment_cache()); - request.hls_proxy = Arc::clone(&ctx.hls_proxy); - request.segment_repair = Arc::clone(ctx.hls_proxy.segment_repair()); - request.segment_worker_pool = Arc::clone(ctx.hls_proxy.segment_worker_pool()); - request.map_worker_pool = Arc::clone(ctx.hls_proxy.map_worker_pool()); - request.post_refresh_runtime = Some(HlsPostRefreshRuntime { ctx: ctx.downgrade() }); - request -} - -fn post_refresh_live_manifest_snapshot() -> HlsLeaseManifestSnapshot { - HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(1), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 1, - delivered_at_ms: 1, - first_proxy_seq: 0, - last_proxy_seq: 2, - visible_segments: Arc::from([ - HlsLeaseManifestSegment { - proxy_seq: 0, - duration_ms: 4_000, - uri: "0.ts".into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }, - HlsLeaseManifestSegment { - proxy_seq: 1, - duration_ms: 4_000, - uri: "1.ts".into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }, - HlsLeaseManifestSegment { - proxy_seq: 2, - duration_ms: 4_000, - uri: "2.ts".into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }, - ]), - discontinuity_sequence: 0, - target_duration_ms: 4_000, - playlist_duration_ms: 12_000, - last_visible_media_end_ms: 12_000, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - } -} - -fn fetched_manifest(body: &str) -> FetchedOriginManifest { - FetchedOriginManifest { - body: body.to_string(), - final_manifest_url: "http://origin.example.com/live/final/index.m3u8".to_string(), - resolved_request_url: "http://origin.example.com/live/user/pass/12345.m3u8".to_string(), - redirect_host: Some("origin.example.com".to_string()), - provider_url_index: None, - provider_session_headers: HeaderMap::new(), - status: StatusCode::OK, - attempts: 1, - candidate_requests: 1, - selection: HlsManifestFetchSelection::Initial, - } -} - -#[test] -fn refresh_completion_diagnostics_separate_logical_recovery_from_beast_candidate_requests() { - let initial = fetched_manifest("#EXTM3U\n#EXTINF:4.0,\nseg.ts\n"); - let initial_diagnostic = HlsManifestRefreshCompletionDiagnostic::from_fetched(&initial); - assert_eq!(initial_diagnostic.recovery_attempts, 0); - assert_eq!(initial_diagnostic.candidate_requests, 1); - assert_eq!(initial_diagnostic.selection, HlsManifestFetchSelection::Initial); - - let mut burst = initial; - burst.attempts = 1; - burst.candidate_requests = HlsManifestRecoveryBurstLevel::Beast.plan().total_candidates(); - burst.selection = HlsManifestFetchSelection::Burst; - let burst_diagnostic = HlsManifestRefreshCompletionDiagnostic::from_fetched(&burst); - assert_eq!(burst_diagnostic.recovery_attempts, 1); - assert_eq!(burst_diagnostic.candidate_requests, HlsManifestRecoveryBurstLevel::Beast.plan().total_candidates()); - assert_eq!(burst_diagnostic.selection, HlsManifestFetchSelection::Burst); -} - -#[test] -fn provider_failover_mirror_without_redirect_uses_resolved_host_signal() { - let mut fetched = fetched_manifest("#EXTM3U\n#EXTINF:4.0,\nseg.ts\n"); - fetched.redirect_host = None; - fetched.resolved_request_url = "http://mirror.example.com/live/user/pass/12345.m3u8".to_string(); - fetched.provider_url_index = Some(1); - - assert_eq!(fetched_effective_manifest_host(&fetched).as_deref(), Some("mirror.example.com")); -} - -#[test] -fn provider_failover_with_redirect_uses_redirect_host_as_manifest_host_signal() { - let mut fetched = fetched_manifest("#EXTM3U\n#EXTINF:4.0,\nseg.ts\n"); - fetched.redirect_host = Some("redirect.example.com".to_string()); - fetched.resolved_request_url = "http://mirror.example.com/live/user/pass/12345.m3u8".to_string(); - fetched.provider_url_index = Some(1); - - assert_eq!(fetched_effective_manifest_host(&fetched).as_deref(), Some("redirect.example.com")); -} - -#[test] -fn manifest_redirect_host_is_only_set_for_actual_redirect_host_switch() { - let resolved = Url::parse("http://mirror.example.com/live/user/pass/12345.m3u8").expect("resolved url"); - let same_target = Url::parse("http://mirror.example.com/live/user/pass/12345.m3u8").expect("same url"); - let redirected = Url::parse("http://cdn.example.net/live/play/12345.m3u8").expect("redirect url"); - - assert_eq!(hls_manifest_redirect_host(&resolved, &same_target), None); - assert_eq!(hls_manifest_redirect_host(&resolved, &redirected).as_deref(), Some("cdn.example.net")); -} - -fn host_from_base_url(base_url: &str) -> String { - url::Url::parse(base_url).expect("base url").host_str().expect("host").to_string() -} - -fn manifest_body() -> String { "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nseg.ts\n".to_string() } - -fn three_segment_manifest_body(media_sequence: u64) -> String { - format!( - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:{media_sequence}\n#EXT-X-TARGETDURATION:4\n\ - #EXTINF:4.0,\n{media_sequence}.ts\n\ - #EXTINF:4.0,\n{}.ts\n\ - #EXTINF:4.0,\n{}.ts\n", - media_sequence.saturating_add(1), - media_sequence.saturating_add(2) - ) -} - -async fn publish_ready_test_manifest(session: &Arc>, rendered_at_ms: u64) { - let mut session = session.write().await; - for segment in session.segments.values_mut() { - segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }; - } - session.render_and_store_manifest(rendered_at_ms).expect("test live manifest publishes"); -} - -fn install_stored_provisioning_manifest(session: &mut HlsSession, rendered_at_ms: u64) { - let segment_proxy_seqs = (0_u64..3).collect::>(); - for proxy_seq in &segment_proxy_seqs { - session.segments.insert( - *proxy_seq, - SegmentEntry { - origin_key: OriginSegmentKey { - origin_epoch: HLS_PROVISIONING_ORIGIN_EPOCH, - effective_host_id: 0, - host_local_sequence: *proxy_seq, - host_local_index: u32::try_from(*proxy_seq).unwrap_or(u32::MAX), - }, - proxy_seq: *proxy_seq, - duration_ms: 2_000, - proxy_file_ext: "ts".to_string(), - content_type: "video/mp2t".to_string(), - cache_key: SegmentCacheKey::new(session.proxy_session_id.clone(), *proxy_seq, "ts"), - discontinuity_before: false, - program_date_time: None, - daterange_tags_before: Vec::new(), - origin_byte_range: None, - map_ref: None, - encryption: None, - origin_fetch_ref: None, - status: SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }, - last_rendered_at_ms: None, - access: Arc::new(CacheAccessState::new()), - }, - ); - } - session.publishable_origin_head_proxy_seq = segment_proxy_seqs.first().copied(); - session.publishable_origin_tail_proxy_seq = segment_proxy_seqs.last().copied(); - session.proxy_next_seq = Some(3); - session.target_duration = Some(2); - assert_eq!( - session.store_rendered_manifest(RenderedManifest { - body: "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n".to_string(), - first_proxy_seq: 0, - last_proxy_seq: 2, - discontinuity_sequence: 0, - target_duration_ms: 2_000, - playlist_duration_ms: 6_000, - valid_until_ms: rendered_at_ms.saturating_add(6_000), - render_gap_segments: 0, - rendered_at_ms, - segment_proxy_seqs, - }), - RenderedManifestStoreOutcome::Stored - ); - assert!(session.published_live_origin_baseline.is_none()); -} - -#[tokio::test] -async fn cold_recovery_directive_is_suppressed_until_baseline() { - let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; - let session = test_session(); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; - - assert!(trigger_origin_refresh_sync(request).await); - - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let manifest_requests = - server.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); - assert_eq!(manifest_requests, 1); - assert_ne!(manifest_requests, plan.total_candidates()); - let session = session.read().await; - assert_eq!(session.origin_seq_highwater, Some(0)); - assert!(session.origin_control.manifest_origin_binding.is_some()); - assert!(session.origin_control.acceptance_episode.is_none()); -} - -#[tokio::test] -async fn warm_retryable_initial_failure_does_not_open_acceptance_episode() { - let server = spawn_test_origin(Arc::new(|_path| (500, Vec::new(), "retry".to_string()))).await; - let session = test_session(); - { - let mut session = session.write().await; - session.origin_seq_highwater = Some(10); - session.origin_control.record_media_progress(50, 4_000); - } - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - - assert!(trigger_origin_refresh_sync(request.clone()).await); - - let manifest_requests = - server.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); - assert_eq!(manifest_requests, request.retry_policy.attempt_count()); - assert!(session.read().await.origin_control.acceptance_episode.is_none()); -} - -#[tokio::test] -async fn shared_initial_manifest_decoder_failure_retries_until_success() { - let hits = Arc::new(AtomicUsize::new(0)); - let hits_for_handler = Arc::clone(&hits); - let server = spawn_test_origin(Arc::new(move |_path| { - if hits_for_handler.fetch_add(1, Ordering::SeqCst) < 2 { - return (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()); - } - (200, Vec::new(), manifest_body()) - })) - .await; - let origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - let context = HlsOriginManifestFetchContext { - app_config: test_app_config(), - session: test_session(), - origin_entry, - headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("no-redirect client"), - use_manual_redirects: false, - origin_manifest_timeout_ms: 2_000, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - retry_policy: no_delay_policy(), - recovery_timing_policy: test_recovery_timing_policy(2_000), - acceptance_timing_seed: None, - }; - - let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) - .await - .expect("shared initial manifest should retry decoder failures"); - - assert_eq!(fetched.body, manifest_body()); - assert_eq!(fetched.attempts, 3); - assert_eq!(server.requests.lock().await.len(), 3); - assert!(server - .raw_requests - .lock() - .await - .iter() - .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); -} - -#[tokio::test] -async fn shared_initial_manifest_waits_for_next_attempt_base_delay() { - let hits = Arc::new(AtomicUsize::new(0)); - let hits_for_handler = Arc::clone(&hits); - let server = spawn_test_origin(Arc::new(move |_path| { - if hits_for_handler.fetch_add(1, Ordering::SeqCst) == 0 { - return (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()); - } - (200, Vec::new(), manifest_body()) - })) - .await; - let origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - let context = HlsOriginManifestFetchContext { - app_config: test_app_config(), - session: test_session(), - origin_entry, - headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("no-redirect client"), - use_manual_redirects: false, - origin_manifest_timeout_ms: 2_000, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - retry_policy: RetryPolicy { delays_ms: [0, 100, 250, 500, 750], jitter_max_ms: 0 }, - recovery_timing_policy: test_recovery_timing_policy(2_000), - acceptance_timing_seed: None, - }; - let started_at = std::time::Instant::now(); - - let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) - .await - .expect("second logical attempt should succeed after its base delay"); - - assert_eq!(fetched.attempts, 2); - assert_eq!(server.requests.lock().await.len(), 2); - assert!(started_at.elapsed() >= std::time::Duration::from_millis(100)); -} - -#[tokio::test] -async fn archive_master_resolution_binds_recovery_to_entry_and_scrubs_redirected_variant_credentials() { - let archive = spawn_test_origin(Arc::new(|path| { - if path.starts_with("/channel/tracks-v1/") { - (200, vec![("Set-Cookie", "route=b; Path=/".to_string())], manifest_body()) - } else { - let master = "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=1000\ntracks-v1/archive-1-2.ts.m3u8\n"; - (200, vec![("Set-Cookie", "session=a; Path=/".to_string())], master.to_string()) - } - })) - .await; - let master_url = format!("{}/channel/archive-1-2.m3u8", archive.base_url); - let variant_url = format!("{}/channel/tracks-v1/archive-1-2.ts.m3u8", archive.base_url); - let entry = - spawn_test_origin(Arc::new(move |_path| (302, vec![("Location", master_url.clone())], String::new()))).await; - let entry_url = format!("{}/live/user/pass/archive-1-2.m3u8", entry.base_url); - let mut headers = HeaderMap::new(); - headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer entry-secret")); - headers.insert(header::COOKIE, HeaderValue::from_static("sid=entry-secret")); - let context = HlsOriginManifestFetchContext { - app_config: test_app_config(), - session: Arc::new(RwLock::new(HlsSession::new( - HlsSessionKey::new(1, "12345").with_archive_reference(1_784_898_000), - b"secret", - 0, - ))), - origin_entry: LiveHlsOriginEntry::parse(&entry_url).expect("entry url"), - headers, - client: reqwest::Client::builder().no_proxy().build().expect("client"), - no_redirect_client: reqwest::Client::builder() - .no_proxy() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("no-redirect client"), - use_manual_redirects: false, - origin_manifest_timeout_ms: 2_000, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - retry_policy: no_delay_policy(), - recovery_timing_policy: test_recovery_timing_policy(2_000), - acceptance_timing_seed: None, - }; - - let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) - .await - .expect("archive master resolves to its variant"); - assert_eq!(fetched.body, manifest_body()); - assert_eq!(fetched.final_manifest_url, variant_url); - assert_eq!(fetched.resolved_request_url, entry_url); - assert_eq!(fetched.provider_session_headers[header::COOKIE], "session=a; route=b"); - { - let raw_requests = archive.raw_requests.lock().await; - let variant_request = raw_requests - .iter() - .find(|request| request.starts_with("GET /channel/tracks-v1/")) - .expect("variant request"); - assert_eq!(request_header_value(variant_request, "authorization"), None); - assert_eq!(request_header_value(variant_request, "cookie"), Some("session=a")); - } - - let binding = HlsManifestOriginBinding::new( - Url::parse(&fetched.resolved_request_url).expect("binding url"), - fetched.provider_url_index, - ) - .expect("binding"); - let recovered = crate::manifest_fetch::fetch_hls_origin_manifest_recovery_once(&context, &binding) - .await - .expect("recovery re-resolves the archive master"); - assert_eq!(recovered.body, manifest_body()); - assert_eq!(recovered.final_manifest_url, variant_url); - assert_eq!(recovered.resolved_request_url, entry_url); - assert_eq!(entry.requests.lock().await.len(), 2); - let raw_requests = archive.raw_requests.lock().await; - assert_eq!(raw_requests.len(), 4); - assert!(raw_requests.iter().all(|request| request_header_value(request, "authorization").is_none())); - assert!(raw_requests.iter().all(|request| request_header_value(request, "cookie") != Some("sid=entry-secret"))); -} - -#[tokio::test] -async fn shared_initial_manifest_automatic_cross_origin_redirect_keeps_identity_and_scrubs_credentials() { - let target = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; - let target_url = format!("{}/final/manifest.m3u8", target.base_url); - let expected_target_url = target_url.clone(); - let redirect = - spawn_test_origin(Arc::new(move |_path| (302, vec![("Location", target_url.clone())], String::new()))).await; - let origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", redirect.base_url)).expect("entry url"); - let mut headers = HeaderMap::new(); - headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); - headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer origin-secret")); - headers.insert(header::COOKIE, HeaderValue::from_static("sid=origin-secret")); - let context = HlsOriginManifestFetchContext { - app_config: test_app_config(), - session: test_session(), - origin_entry, - headers, - client: reqwest::Client::builder().no_proxy().build().expect("client"), - no_redirect_client: reqwest::Client::builder() - .no_proxy() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("no-redirect client"), - use_manual_redirects: false, - origin_manifest_timeout_ms: 2_000, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - retry_policy: no_delay_policy(), - recovery_timing_policy: test_recovery_timing_policy(2_000), - acceptance_timing_seed: None, - }; - - let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) - .await - .expect("automatic redirect should return the decoded manifest"); - - assert_eq!(fetched.body, manifest_body()); - assert_eq!(fetched.attempts, 1); - assert_eq!(fetched.final_manifest_url, expected_target_url); - let redirect_requests = redirect.raw_requests.lock().await; - let target_requests = target.raw_requests.lock().await; - assert_eq!(redirect_requests.len(), 1); - assert_eq!(target_requests.len(), 1); - assert_eq!(request_header_value(&redirect_requests[0], "accept-encoding"), Some("identity")); - assert_eq!(request_header_value(&target_requests[0], "accept-encoding"), Some("identity")); - assert_eq!(request_header_value(&redirect_requests[0], "authorization"), Some("Bearer origin-secret")); - assert_eq!(request_header_value(&redirect_requests[0], "cookie"), Some("sid=origin-secret")); - assert!(request_header_value(&target_requests[0], "authorization").is_none()); - assert!(request_header_value(&target_requests[0], "cookie").is_none()); -} - -#[tokio::test] -async fn shared_initial_manifest_budget_covers_status_decoder_redirect_and_success() { - let target_hits = Arc::new(AtomicUsize::new(0)); - let target_hits_for_handler = Arc::clone(&target_hits); - let server = spawn_test_origin(Arc::new(move |path| { - if path == "/live/user/pass/12345.m3u8" { - return (302, vec![("Location", "/live/play/once/12345".to_string())], String::new()); - } - if path == "/live/play/once/12345" { - return match target_hits_for_handler.fetch_add(1, Ordering::SeqCst) { - 0 => (500, Vec::new(), "temporary".to_string()), - 1 => (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()), - _ => (200, Vec::new(), manifest_body()), - }; - } - (404, Vec::new(), String::new()) - })) - .await; - let origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - let context = HlsOriginManifestFetchContext { - app_config: test_app_config(), - session: test_session(), - origin_entry, - headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("no-redirect client"), - use_manual_redirects: true, - origin_manifest_timeout_ms: 2_000, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - retry_policy: no_delay_policy(), - recovery_timing_policy: test_recovery_timing_policy(2_000), - acceptance_timing_seed: None, - }; - - let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) - .await - .expect("third logical attempt should succeed"); - - assert_eq!(fetched.body, manifest_body()); - assert_eq!(fetched.attempts, 3); - assert_eq!( - *server.requests.lock().await, - vec![ - "/live/user/pass/12345.m3u8", - "/live/play/once/12345", - "/live/user/pass/12345.m3u8", - "/live/play/once/12345", - "/live/user/pass/12345.m3u8", - "/live/play/once/12345", - ] - ); - assert!(server - .raw_requests - .lock() - .await - .iter() - .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); -} - -#[tokio::test] -async fn shared_initial_manifest_decoder_failures_stop_at_attempt_budget() { - let server = spawn_test_origin(Arc::new(|_path| { - (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()) - })) - .await; - let origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - let retry_policy = no_delay_policy(); - let expected_attempts = retry_policy.attempt_count(); - let context = HlsOriginManifestFetchContext { - app_config: test_app_config(), - session: test_session(), - origin_entry, - headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("no-redirect client"), - use_manual_redirects: false, - origin_manifest_timeout_ms: 2_000, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - retry_policy, - recovery_timing_policy: test_recovery_timing_policy(2_000), - acceptance_timing_seed: None, - }; - - let error = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) - .await - .expect_err("decoder failures must exhaust the logical attempt budget"); - - assert!(matches!(error, OriginManifestFetchError::ContentDecoding { .. })); - assert_eq!(server.requests.lock().await.len(), expected_attempts); - assert!(server - .raw_requests - .lock() - .await - .iter() - .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); -} - -#[tokio::test] -async fn shared_refresh_metrics_use_successful_manifest_attempt_count() { - let manifest_hits = Arc::new(AtomicUsize::new(0)); - let manifest_hits_for_handler = Arc::clone(&manifest_hits); - let server = spawn_test_origin(Arc::new(move |path| { - if path != "/live/user/pass/12345.m3u8" { - return (404, Vec::new(), String::new()); - } - if manifest_hits_for_handler.fetch_add(1, Ordering::SeqCst) < 2 { - return (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()); - } - (200, Vec::new(), manifest_body()) - })) - .await; - let session = test_session(); - let mut request = test_origin_refresh_request(session); - request.origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - let metrics = Arc::clone(request.segment_worker_pool.metrics()); - - assert!(Box::pin(trigger_origin_refresh_sync(request)).await); - - let snapshot = metrics.snapshot(); - assert_eq!(snapshot.refresh_started, 1); - assert_eq!(snapshot.refresh_completed, 1); - assert_eq!(snapshot.refresh_retried, 2); - assert_eq!(snapshot.refresh_failed, 0); - let manifest_requests = server - .raw_requests - .lock() - .await - .iter() - .filter(|request| request.lines().next().is_some_and(|line| line.contains(" /live/user/pass/12345.m3u8 "))) - .cloned() - .collect::>(); - assert_eq!(manifest_requests.len(), 3); - assert!(manifest_requests - .iter() - .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); -} - -#[tokio::test] -async fn manifest_retry_starts_at_entrypoint_after_redirect_failure() { - let redirect_hits = Arc::new(AtomicUsize::new(0)); - let redirect_hits_for_handler = Arc::clone(&redirect_hits); - let server = spawn_test_origin(Arc::new(move |path| { - if path == "/live/user/pass/12345.m3u8" { - return (302, vec![("Location", "/live/play/once/12345".to_string())], String::new()); - } - if path == "/live/play/once/12345" { - let hit = redirect_hits_for_handler.fetch_add(1, Ordering::SeqCst); - if hit < 2 { - return (500, Vec::new(), "fail".to_string()); - } - return (200, Vec::new(), manifest_body()); - } - (404, Vec::new(), String::new()) - })) - .await; - let entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - let no_redirect_client = - reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()).build().expect("client builds"); - - let fetched = refresh_from_live_hls_entrypoint_with_retries( - &entry, - &HeaderMap::new(), - &reqwest::Client::new(), - &no_redirect_client, - true, - 2_000, - &no_delay_policy(), - ) - .await - .expect("refresh eventually succeeds"); - - assert_eq!(fetched.attempts, 3); - assert_eq!( - *server.requests.lock().await, - vec![ - "/live/user/pass/12345.m3u8", - "/live/play/once/12345", - "/live/user/pass/12345.m3u8", - "/live/play/once/12345", - "/live/user/pass/12345.m3u8", - "/live/play/once/12345" - ] - ); -} - -#[tokio::test] -async fn retryable_407_retries_until_success() { - let hits = Arc::new(AtomicUsize::new(0)); - let hits_for_handler = Arc::clone(&hits); - let server = spawn_test_origin(Arc::new(move |_path| { - let hit = hits_for_handler.fetch_add(1, Ordering::SeqCst); - if hit < 2 { - return (407, Vec::new(), "retry".to_string()); - } - (200, Vec::new(), manifest_body()) - })) - .await; - let entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - - let fetched = refresh_from_live_hls_entrypoint_with_retries( - &entry, - &HeaderMap::new(), - &reqwest::Client::new(), - &reqwest::Client::new(), - false, - 2_000, - &no_delay_policy(), - ) - .await - .expect("refresh eventually succeeds"); - - assert_eq!(fetched.attempts, 3); - assert_eq!(server.requests.lock().await.len(), 3); -} - -#[tokio::test] -async fn permanent_404_does_not_retry() { - let server = spawn_test_origin(Arc::new(|_path| (404, Vec::new(), "missing".to_string()))).await; - let entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - - let err = refresh_from_live_hls_entrypoint_with_retries( - &entry, - &HeaderMap::new(), - &reqwest::Client::new(), - &reqwest::Client::new(), - false, - 2_000, - &no_delay_policy(), - ) - .await - .expect_err("404 is permanent"); - - assert!(matches!(err, OriginManifestFetchError::PermanentStatus(StatusCode::NOT_FOUND))); - assert_eq!(server.requests.lock().await.len(), 1); -} - -#[tokio::test] -async fn provider_failover_status_does_not_count_as_hls_retry() { - let first = spawn_test_origin(Arc::new(|_path| (407, Vec::new(), "rotate".to_string()))).await; - let second = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "demo".into(), - urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, - dns: None, - })); - let session = test_session(); - let initial_session_key = session.read().await.key.stable_value(); - let initial_proxy_session_id = session.read().await.proxy_session_id.clone(); - let entry = LiveHlsOriginEntry::parse_with_url_failover_provider( - "provider://demo/live/user/pass/12345.m3u8", - Some(Arc::clone(&provider)), - ) - .expect("provider entry url"); - let segment_worker_pool = Arc::new(HlsSegmentWorkerPool::default()); - let request = OriginRefreshRequest { - app_config: test_app_config(), - session: Arc::clone(&session), - origin_entry: entry.clone(), - headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("client builds"), - use_manual_redirects: false, - segment_cache: Arc::new(HlsSegmentCache::new()), - hls_proxy: Arc::new(HlsProxyManager::new()), - segment_repair: test_segment_repair_manager(), - segment_worker_pool: Arc::clone(&segment_worker_pool), - map_worker_pool: Arc::new(HlsMapWorkerPool::default()), - origin_manifest_timeout_ms: 2_000, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, - retry_policy: no_delay_policy(), - reverse_proxy_rewrite_secret: b"secret".to_vec(), - transient_resource_ttl_ms: 300_000, - manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, - fresh_manifest_requirement_generation: None, - acceptance_directive: HlsManifestAcceptanceDirective::none(), - access_lease_id: None, - disabled_headers: None, - now_ms: 100, - origin_io: None, - post_refresh_runtime: None, - }; - - assert!(maybe_trigger_origin_refresh(request).await); - for _ in 0..50 { - if session.read().await.origin_seq_highwater == Some(102) { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - - let first_manifest_requests = - first.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); - let second_manifest_requests = - second.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); - assert_eq!(first_manifest_requests, 1); - assert_eq!(second_manifest_requests, 1); - for server in [&first, &second] { - let manifest_requests = server - .raw_requests - .lock() - .await - .iter() - .filter(|request| request.lines().next().is_some_and(|line| line.contains(" /live/user/pass/12345.m3u8 "))) - .cloned() - .collect::>(); - assert_eq!(manifest_requests.len(), 1); - assert_eq!(request_header_value(&manifest_requests[0], "accept-encoding"), Some("identity")); - } - let session = session.read().await; - assert_eq!(session.key.stable_value(), initial_session_key); - assert_eq!(session.proxy_session_id, initial_proxy_session_id); - assert!(!session.key.stable_value().contains("provider://")); - assert!(!session.key.stable_value().contains(first.base_url.as_str())); - assert!(!session.key.stable_value().contains(second.base_url.as_str())); - assert_eq!(session.origin_seq_highwater, Some(0)); - assert_eq!(session.last_effective_manifest_host.as_deref(), Some("127.0.0.1")); - let metrics = segment_worker_pool.metrics().snapshot(); - assert_eq!(metrics.refresh_started, 1); - assert_eq!(metrics.refresh_completed, 1); - assert_eq!(metrics.refresh_retried, 0); - assert_eq!(metrics.refresh_failed, 0); -} - -#[tokio::test] -async fn established_recovery_burst_uses_one_fixed_provider_url_for_every_candidate() { - let phase = Arc::new(AtomicUsize::new(0)); - let phase_for_first = Arc::clone(&phase); - let first = spawn_test_origin(Arc::new(move |_path| { - let media_sequence = if phase_for_first.load(Ordering::SeqCst) == 0 { 100 } else { 103 }; - (200, Vec::new(), three_segment_manifest_body(media_sequence)) - })) - .await; - let second = spawn_test_origin(Arc::new(|_path| (500, Vec::new(), "unexpected".to_string()))).await; - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "demo".into(), - urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, - dns: None, - })); - let session = test_session(); - let entry = LiveHlsOriginEntry::parse_with_url_failover_provider( - "provider://demo/live/user/pass/12345.m3u8?token=fixed", - Some(Arc::clone(&provider)), - ) - .expect("provider entry url"); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = entry; - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - - let baseline = fetch_and_commit_manifest_with_policy(&mut request).await.expect("initial baseline commits"); - assert_eq!(baseline.fetched.provider_url_index, Some(0)); - let concrete_request_url = format!("{}/live/user/pass/12345.m3u8?token=fixed", first.base_url); - { - let session = session.read().await; - let binding = - session.origin_control.manifest_origin_binding.as_ref().expect("successful commit stores binding"); - assert_eq!(binding.request_url().as_str(), concrete_request_url); - assert_eq!(binding.provider_url_index(), Some(0)); - } - publish_ready_test_manifest(&session, 200).await; - assert!(session.read().await.established_manifest_recovery_binding().is_some()); - - phase.store(1, Ordering::SeqCst); - let provider_index_before_burst = provider.get_current_index(); - request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; - let selected = - fetch_and_commit_manifest_with_policy(&mut request).await.expect("established fixed-binding burst commits"); - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let first_requests = first.requests.lock().await; - assert_eq!(first_requests.len(), 1_usize.saturating_add(plan.total_candidates())); - assert!(first_requests.iter().all(|path| path == "/live/user/pass/12345.m3u8?token=fixed")); - assert!(second.requests.lock().await.is_empty()); - assert_eq!(provider.get_current_index(), provider_index_before_burst); - assert_eq!(selected.fetched.provider_url_index, Some(0)); - assert_eq!(selected.fetched.candidate_requests, plan.total_candidates()); - assert_eq!(selected.fetched.selection, HlsManifestFetchSelection::Burst); - assert_eq!(session.read().await.origin_seq_highwater, Some(105)); -} - -#[tokio::test] -async fn successful_redirected_baseline_binds_the_original_request_entry() { - let target = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(100)))).await; - let target_url = format!("{}/redirected/final.m3u8", target.base_url); - let target_url_for_handler = target_url.clone(); - let entry = spawn_test_origin(Arc::new(move |_path| { - (302, vec![("Location", target_url_for_handler.clone())], String::new()) - })) - .await; - let request_url = format!("{}/live/user/pass/12345.m3u8?token=fixed", entry.base_url); - let session = test_session(); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = LiveHlsOriginEntry::parse(&request_url).expect("redirecting entry URL"); - - let committed = - fetch_and_commit_manifest_with_policy(&mut request).await.expect("redirected initial manifest commits"); - - assert_eq!(committed.fetched.final_manifest_url, target_url); - assert_eq!(committed.fetched.resolved_request_url, request_url); - let binding_url = { - let session = session.read().await; - session - .origin_control - .manifest_origin_binding - .as_ref() - .expect("redirected commit stores binding") - .request_url() - .to_string() - }; - assert_eq!(binding_url, request_url); - assert_eq!(entry.requests.lock().await.as_slice(), ["/live/user/pass/12345.m3u8?token=fixed"]); - assert_eq!(target.requests.lock().await.as_slice(), ["/redirected/final.m3u8"]); -} - -fn assert_binding_superseded_error(error: &OriginManifestFetchError) { - assert!(matches!( - error, - OriginManifestFetchError::RecoveryUnavailable { - reason: HlsManifestRecoveryUnavailableReason::BindingSuperseded, - } - )); - assert_eq!(error.log_label(), "recovery_binding_superseded"); - assert_eq!(error.to_string(), "origin manifest recovery unavailable: manifest origin binding superseded"); - assert_eq!( - classify_manifest_fetch_failure(error), - HlsManifestFetchFailureSignal::discarded(HlsManifestFetchFailureKind::Superseded) - ); -} - -#[tokio::test] -async fn recovery_binding_supersession_is_discarded_without_failure_signal() { - let origin = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(100)))).await; - let hls_ctx = crate::HlsCtx::for_test(Config::default()); - let ctx = &hls_ctx; - let session = Arc::new(RwLock::with_max_readers( - HlsSession::new(HlsSessionKey::new(1, "binding-supersession"), b"secret", 0), - 1, - )); - let mut request = bind_refresh_request_to_app_state(test_origin_refresh_request(Arc::clone(&session)), ctx); - let baseline_binding_url = format!("{}/live/a/index.m3u8", origin.base_url); - request.origin_entry = LiveHlsOriginEntry::parse(&baseline_binding_url).expect("binding A entry URL"); - request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; - - fetch_and_commit_manifest_with_policy(&mut request).await.expect("binding A baseline commits"); - publish_ready_test_manifest(&session, 200).await; - let baseline_binding = - session.read().await.established_manifest_recovery_binding().expect("published baseline has binding A"); - let requests_before_supersession = origin.requests.lock().await.len(); - - let replacement_binding_url = format!("{}/live/b/index.m3u8", origin.base_url); - let mut binding_b_manifest = fetched_manifest(&three_segment_manifest_body(103)); - binding_b_manifest.resolved_request_url.clone_from(&replacement_binding_url); - binding_b_manifest.final_manifest_url.clone_from(&replacement_binding_url); - binding_b_manifest.redirect_host = Some(host_from_base_url(&origin.base_url)); - let metrics = Arc::clone(request.segment_worker_pool.metrics()); - assert!(mark_origin_refresh_started(&mut request, 300).await); - let verification_request = request.clone(); - let replacement_request = request.clone(); - let read_blocker = session.read().await; - let refresh = tokio::spawn(refresh_and_commit(request, 300)); - tokio::task::yield_now().await; - let replacement_session = Arc::clone(&session); - let replacement = tokio::spawn(async move { - let mut session = replacement_session.write().await; - commit_fetched_manifest(&mut session, &binding_b_manifest, &replacement_request, 325) - .expect("newer binding B manifest commits"); - session.origin_control.record_origin_response(275); - session.origin_control.path_condition = super::super::origin_progress::HlsOriginPathCondition::PublicationLate; - session.origin_refresh.consecutive_failures = 3; - session.origin_refresh.last_error_at_ms = Some(250); - }); - tokio::task::yield_now().await; - drop(read_blocker); - replacement.await.expect("replacement binding task joins"); - refresh.await.expect("superseded refresh task joins"); - - let metrics_after = metrics.snapshot(); - let replacement_binding = { - let session = session.read().await; - assert_eq!(session.origin_control.last_origin_response_at_ms, Some(275)); - assert_eq!( - session.origin_control.path_condition, - super::super::origin_progress::HlsOriginPathCondition::PublicationLate - ); - assert_eq!(session.origin_refresh.consecutive_failures, 3); - assert_eq!(session.origin_refresh.last_error_at_ms, Some(250)); - assert!(!session.origin_refresh.in_flight); - assert!(session.origin_control.acceptance_episode.is_none()); - session.origin_control.manifest_origin_binding.clone().expect("newer binding B remains installed") - }; - assert_ne!(baseline_binding, replacement_binding); - assert_eq!(metrics_after.refresh_started, 1); - assert_eq!(metrics_after.refresh_completed, 0); - assert_eq!(metrics_after.refresh_retried, 0); - assert_eq!(metrics_after.refresh_failed, 0); - assert_eq!(metrics_after.refresh_skipped, 1); - assert_eq!(ctx.hls_proxy.availability_reevaluations().owner_count(), 0); - assert_eq!(origin.requests.lock().await.len(), requests_before_supersession); - - let Err(error) = super::recover_manifest_for_request( - &manifest_fetch_context(&verification_request), - &verification_request, - super::HlsManifestRecoveryPath { - binding: baseline_binding, - reject_reason: None, - deterministic_conflict: None, - trigger: HlsManifestAcceptanceTrigger::RecoveryRequired, - diagnostic: super::HlsRecoveryTriggerDiagnostic::new(super::HlsRecoveryTriggerSource::Other), - }, - ) - .await - else { - panic!("superseded recovery binding must be discarded"); - }; - assert_binding_superseded_error(&error); - assert_eq!(origin.requests.lock().await.len(), requests_before_supersession); -} - -#[tokio::test] -async fn recovery_unavailable_after_valid_response_preserves_response_evidence() { - let origin = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(758)))).await; - let session = test_session(); - prepare_cross_host_baseline(&session).await; - { - let mut session = session.write().await; - session.origin_control.record_origin_response(50); - } - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/new/index.m3u8", origin.base_url)).expect("candidate entry URL"); - - let Err(error) = fetch_and_commit_manifest_with_policy(&mut request).await else { - panic!("cross-host response without recovery binding must remain uncommitted"); - }; - - assert!(matches!( - error, - OriginManifestFetchError::RecoveryUnavailable { - reason: HlsManifestRecoveryUnavailableReason::NoEstablishedBindingAfterResponse, - } - )); - assert_eq!(error.log_label(), "recovery_unavailable_after_response"); - assert_eq!(error.to_string(), "origin manifest recovery unavailable: no established binding after origin response"); - assert_eq!( - classify_manifest_fetch_failure(&error), - HlsManifestFetchFailureSignal::retryable( - HlsManifestFetchFailureKind::AcceptanceConflict, - HlsManifestHttpResponseEvidence::ValidResponse, - ) - ); - assert_eq!(origin.requests.lock().await.len(), 1); - { - let session = session.read().await; - assert_eq!(session.origin_control.last_origin_response_at_ms, Some(50)); - assert!(session.origin_control.manifest_origin_binding.is_none()); - assert!(session.origin_control.acceptance_episode.is_none()); - } - - let evidence_origin = - spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(758)))).await; - let evidence_session = test_session(); - prepare_cross_host_baseline(&evidence_session).await; - let progress_generation = { - let mut session = evidence_session.write().await; - session.origin_control.record_origin_response(50); - session.origin_control.progress_generation - }; - let mut evidence_request = test_origin_refresh_request(Arc::clone(&evidence_session)); - evidence_request.origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/new/index.m3u8", evidence_origin.base_url)) - .expect("evidence candidate entry URL"); - let metrics = Arc::clone(evidence_request.segment_worker_pool.metrics()); - - assert!(trigger_origin_refresh_sync(evidence_request).await); - assert_eq!(evidence_origin.requests.lock().await.len(), 1); - { - let session = evidence_session.read().await; - assert!(session.origin_control.last_origin_response_at_ms.is_some_and(|recorded_at_ms| recorded_at_ms > 50)); - assert_eq!(session.origin_control.progress_generation, progress_generation); - assert!(session.origin_control.manifest_origin_binding.is_none()); - assert!(session.origin_control.acceptance_episode.is_none()); - } - let metrics = metrics.snapshot(); - assert_eq!(metrics.refresh_started, 1); - assert_eq!(metrics.refresh_failed, 1); - assert_eq!(metrics.refresh_completed, 0); - assert_eq!(metrics.refresh_retried, 0); - assert_eq!(metrics.refresh_skipped, 0); -} - -#[tokio::test] -async fn initial_provider_failover_does_not_leak_into_established_burst() { - let phase = Arc::new(AtomicUsize::new(0)); - let phase_for_first = Arc::clone(&phase); - let first = spawn_test_origin(Arc::new(move |_path| { - if phase_for_first.load(Ordering::SeqCst) == 0 { - (200, Vec::new(), three_segment_manifest_body(100)) - } else { - (407, Vec::new(), "rotate".to_string()) - } - })) - .await; - let phase_for_second = Arc::clone(&phase); - let second = spawn_test_origin(Arc::new(move |_path| { - if phase_for_second.load(Ordering::SeqCst) == 0 { - (500, Vec::new(), "unexpected".to_string()) - } else { - (461, Vec::new(), "final-hard-failure".to_string()) - } - })) - .await; - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "demo".into(), - urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, - dns: None, - })); - let session = test_session(); - let entry = LiveHlsOriginEntry::parse_with_url_failover_provider( - "provider://demo/live/user/pass/12345.m3u8?token=fixed", - Some(Arc::clone(&provider)), - ) - .expect("provider entry url"); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = entry; - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Friendly }; - - fetch_and_commit_manifest_with_policy(&mut request).await.expect("initial provider URL commits baseline"); - publish_ready_test_manifest(&session, 200).await; - let binding = - session.read().await.established_manifest_recovery_binding().expect("published baseline has fixed binding"); - assert_eq!(binding.provider_url_index(), Some(0)); - let provider_index_before_failure = provider.get_current_index(); - - phase.store(1, Ordering::SeqCst); - let Err(error) = fetch_and_commit_manifest_with_policy(&mut request).await else { - panic!("hard initial provider cycle and fixed-target recovery must remain failed"); - }; - assert!(matches!(error, OriginManifestFetchError::RetryableStatus(StatusCode::PROXY_AUTHENTICATION_REQUIRED, _))); - - let plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); - let first_requests = first.requests.lock().await; - let second_requests = second.requests.lock().await; - assert!( - first_requests.len() >= 2_usize.saturating_add(plan.total_candidates()), - "baseline, ordinary provider attempt, and at least one full fixed-target burst are required" - ); - assert_eq!(second_requests.len(), 1, "only the ordinary provider cycle may reach URL index 1"); - assert!(first_requests.iter().all(|path| path == "/live/user/pass/12345.m3u8?token=fixed")); - assert_eq!(second_requests[0], "/live/user/pass/12345.m3u8?token=fixed"); - assert_eq!( - provider.get_current_index(), - provider_index_before_failure, - "fixed-target recovery must not rotate provider state" - ); - assert_eq!( - session - .read() - .await - .origin_control - .manifest_origin_binding - .as_ref() - .map(HlsManifestOriginBinding::provider_url_index), - Some(Some(0)) - ); -} - -#[tokio::test] -async fn expired_acceptance_deadline_still_completes_first_beast_burst_without_follow_up() { - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let origin = spawn_test_origin(Arc::new(|_path| (407, Vec::new(), "retry".to_string()))).await; - let session = test_session(); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)) - .expect("deadline test origin entry"); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - let target_url = request.origin_entry.url().clone(); - let mut context = manifest_fetch_context(&request); - let mut expired_after_full_burst = test_recovery_timing_policy(0); - expired_after_full_burst.evaluation_eta = HlsRecoveryEtaMs::default(); - expired_after_full_burst.commit_eta = HlsRecoveryEtaMs::default(); - expired_after_full_burst.scheduling_eta = HlsRecoveryEtaMs::default(); - context.recovery_timing_policy = expired_after_full_burst; - - let result = retry_hls_origin_manifest_recovery_chain( - &context, - test_manifest_origin_binding(target_url), - Some(HlsManifestRejectLogReason::PinnedHostRecoveryRejected), - None, - HlsManifestAcceptanceTrigger::RecoveryRequired, - HlsManifestCommitAcceptanceMode::StrictPinnedHost, - |fetched, acceptance_mode| super::commit_manifest_recovery_candidate(&request, fetched, acceptance_mode), - ) - .await; - - assert!(result.is_err()); - let manifest_requests = - origin.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); - assert_eq!(manifest_requests, plan.total_candidates()); - let session = session.read().await; - let episode = session.origin_control.acceptance_episode.as_ref().expect("completed first acceptance burst"); - assert!(episode.full_burst_completed); - assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); - assert_eq!(episode.full_bursts_completed, 1); -} - -#[tokio::test] -async fn cold_start_uses_one_initial_fetch_and_commits_a_fresh_baseline_before_recovery() { - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; - let session = test_session(); - { - let mut session = session.write().await; - session.origin_seq_highwater = Some(1_000); - session.last_effective_manifest_host = Some("stale.example.com".to_string()); - } - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - request.manifest_commit_requirement = - HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; - - assert!(trigger_origin_refresh_sync(request).await); - - let manifest_requests = - server.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); - assert_eq!(manifest_requests, 1); - assert_ne!(manifest_requests, plan.total_candidates()); - let session = session.read().await; - assert_eq!(session.origin_seq_highwater, Some(0)); - assert_eq!(session.last_effective_manifest_host.as_deref(), Some("127.0.0.1")); - assert!(session.origin_control.acceptance_episode.is_none()); -} - -#[tokio::test] -async fn cold_start_hard_initial_failure_does_not_start_acceptance_burst() { - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let server = spawn_test_origin(Arc::new(|_path| (404, Vec::new(), "missing".to_string()))).await; - let session = test_session(); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - request.manifest_commit_requirement = - HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; - let metrics = Arc::clone(request.segment_worker_pool.metrics()); - - assert!(trigger_origin_refresh_sync(request).await); - - let manifest_requests = - server.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); - assert_eq!(manifest_requests, 1); - assert_ne!(manifest_requests, 1_usize.saturating_add(plan.total_candidates())); - let session = session.read().await; - assert!(session.origin_control.acceptance_episode.is_none()); - assert_eq!(session.origin_control.progress_phase, super::super::origin_progress::HlsOriginProgressPhase::Cold); - assert_eq!( - session.origin_control.path_condition, - super::super::origin_progress::HlsOriginPathCondition::HardFetchFailure - ); - let metrics = metrics.snapshot(); - assert_eq!(metrics.refresh_failed, 1); - assert_eq!(metrics.refresh_completed, 0); -} - -#[tokio::test] -async fn cold_start_hard_error_is_preserved_by_fetch_policy() { - let server = spawn_test_origin(Arc::new(|_path| (404, Vec::new(), "missing".to_string()))).await; - let session = test_session(); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; - - let Err(error) = fetch_and_commit_manifest_with_policy(&mut request).await else { - panic!("cold hard status must be returned without synthetic recovery"); - }; - - assert!(matches!(error, OriginManifestFetchError::PermanentStatus(StatusCode::NOT_FOUND))); - assert_eq!(server.requests.lock().await.len(), 1); - let session = session.read().await; - assert!(session.origin_control.acceptance_episode.is_none()); - assert_eq!(session.origin_control.progress_phase, super::super::origin_progress::HlsOriginProgressPhase::Cold); -} - -#[tokio::test] -async fn cold_provider_failover_cycle_does_not_start_acceptance_burst() { - let first = spawn_test_origin(Arc::new(|_path| (500, Vec::new(), "rotate".to_string()))).await; - let second = spawn_test_origin(Arc::new(|_path| (461, Vec::new(), "final-hard-failure".to_string()))).await; - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "demo".into(), - urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, - dns: None, - })); - let session = test_session(); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = LiveHlsOriginEntry::parse_with_url_failover_provider( - "provider://demo/live/user/pass/12345.m3u8", - Some(provider), - ) - .expect("provider entry URL"); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - request.manifest_commit_requirement = - HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; - - let Err(error) = fetch_and_commit_manifest_with_policy(&mut request).await else { - panic!("cold provider cycle must preserve its final hard response"); - }; - - assert!(matches!( - error, - OriginManifestFetchError::NonRetryableStatus(status) if status.as_u16() == 461 - )); - assert_eq!(first.requests.lock().await.len(), 1); - assert_eq!(second.requests.lock().await.len(), 1); - let session = session.read().await; - assert!(session.origin_control.acceptance_episode.is_none()); - assert!(session.origin_control.manifest_origin_binding.is_none()); - assert_eq!(session.origin_control.progress_phase, super::super::origin_progress::HlsOriginProgressPhase::Cold); -} - -#[tokio::test] -async fn provisioning_handoff_without_established_origin_baseline_uses_one_initial_fetch() { - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; - let session = test_session(); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { - reason: HlsFreshManifestRequiredReason::ProvisioningHandoff, - }; - assert_eq!(manifest_recovery_trigger(&request), HlsManifestAcceptanceTrigger::Critical); - - assert!(trigger_origin_refresh_sync(request).await); - - let manifest_requests = - server.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); - assert_eq!(manifest_requests, 1); - assert_ne!(manifest_requests, plan.total_candidates()); - let session = session.read().await; - assert_eq!(session.origin_seq_highwater, Some(0)); - assert!(session.origin_control.manifest_origin_binding.is_some()); - assert!(session.origin_control.acceptance_episode.is_none()); -} - -#[tokio::test] -async fn successful_origin_commit_without_renderable_origin_window_does_not_enable_burst() { - let manifest_hits = Arc::new(AtomicUsize::new(0)); - let manifest_hits_for_handler = Arc::clone(&manifest_hits); - let server = spawn_test_origin(Arc::new(move |_path| { - let media_sequence = if manifest_hits_for_handler.fetch_add(1, Ordering::SeqCst) == 0 { 100 } else { 103 }; - (200, Vec::new(), three_segment_manifest_body(media_sequence)) - })) - .await; - let session = test_session(); - { - let mut session = session.write().await; - install_stored_provisioning_manifest(&mut session, 10); - } - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry URL"); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - - let baseline = fetch_and_commit_manifest_with_policy(&mut request).await.expect("ordinary origin baseline commits"); - assert_eq!(baseline.fetched.selection, HlsManifestFetchSelection::Initial); - assert_eq!(baseline.fetched.candidate_requests, 1); - { - let session = session.read().await; - assert!(session.origin_control.manifest_origin_binding.is_some()); - assert!(session.origin_control.pinned_host.is_some()); - assert_eq!(session.origin_seq_highwater, Some(102)); - assert_eq!(session.origin_control.progress_phase, super::super::origin_progress::HlsOriginProgressPhase::Fresh); - assert_eq!( - session.last_rendered_manifest.as_ref().expect("provisioning manifest remains stored").segment_proxy_seqs, - vec![0, 1, 2] - ); - assert!(session - .segments - .values() - .filter(|segment| segment.origin_key.origin_epoch != HLS_PROVISIONING_ORIGIN_EPOCH) - .all(|segment| !matches!(segment.status, SegmentCacheStatus::Ready { .. }))); - assert!(session.published_live_origin_baseline.is_none()); - assert!(session.established_manifest_recovery_binding().is_none()); - assert!(session.origin_control.acceptance_episode.is_none()); - } - - request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; - let follow_up = - fetch_and_commit_manifest_with_policy(&mut request).await.expect("suppressed recovery uses ordinary fetch"); - - assert_eq!(follow_up.fetched.selection, HlsManifestFetchSelection::Initial); - assert_eq!(follow_up.fetched.candidate_requests, 1); - assert_eq!(manifest_hits.load(Ordering::SeqCst), 2); - assert_ne!(manifest_hits.load(Ordering::SeqCst), HlsManifestRecoveryBurstLevel::Beast.plan().total_candidates()); - let session = session.read().await; - assert!(session.published_live_origin_baseline.is_none()); - assert!(session.established_manifest_recovery_binding().is_none()); - assert!(session.origin_control.acceptance_episode.is_none()); -} - -#[tokio::test] -async fn different_host_single_candidate_retries_do_not_prove_acceptance() { - let candidate_hits = Arc::new(AtomicUsize::new(0)); - let candidate_hits_for_handler = Arc::clone(&candidate_hits); - let candidate = spawn_test_origin(Arc::new(move |_path| { - candidate_hits_for_handler.fetch_add(1, Ordering::SeqCst); - ( - 200, - Vec::new(), - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n#EXTINF:4.0,\n102.ts\n" - .to_string(), - ) - })) - .await; - let session = test_session(); - { - let mut session = session.write().await; - session.origin_seq_highwater = Some(100); - session.last_effective_manifest_host = Some("previous.example.com".to_string()); - session.origin_control.pinned_host = Some("previous.example.com".to_string()); - } - let candidate_entry_url = - format!("{}/live/user/pass/12345.m3u8", candidate.base_url).replacen("127.0.0.1", "localhost", 1); - install_published_recovery_binding(&session, &candidate_entry_url, None).await; - let entry = LiveHlsOriginEntry::parse(&candidate_entry_url).expect("entry url"); - let request = OriginRefreshRequest { - app_config: test_app_config(), - session: Arc::clone(&session), - origin_entry: entry.clone(), - headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("client builds"), - use_manual_redirects: false, - segment_cache: Arc::new(HlsSegmentCache::new()), - hls_proxy: Arc::new(HlsProxyManager::new()), - segment_repair: test_segment_repair_manager(), - segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), - map_worker_pool: Arc::new(HlsMapWorkerPool::default()), - origin_manifest_timeout_ms: 2_000, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, - retry_policy: no_delay_policy(), - reverse_proxy_rewrite_secret: b"secret".to_vec(), - transient_resource_ttl_ms: 300_000, - manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, - fresh_manifest_requirement_generation: None, - acceptance_directive: HlsManifestAcceptanceDirective::none(), - access_lease_id: None, - disabled_headers: None, - now_ms: 100, - origin_io: None, - post_refresh_runtime: None, - }; - - assert!(trigger_origin_refresh_sync(request).await); - - let candidate_requests = - candidate.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); - assert_eq!(candidate_requests, 6); - let session = session.read().await; - assert_eq!(session.origin_seq_highwater, Some(100)); - assert_eq!(session.origin_epoch, 0); - let episode = session.origin_control.acceptance_episode.as_ref().expect("acceptance episode remains held"); - assert!(episode.full_burst_completed); - assert_eq!(episode.full_bursts_completed, 1); - assert_eq!(episode.trigger(), HlsManifestAcceptanceTrigger::Observe); - assert_eq!(episode.state, super::super::manifest_acceptance::HlsManifestAcceptanceState::Holding); - assert_eq!( - session.origin_control.path_condition, - super::super::origin_progress::HlsOriginPathCondition::AcceptanceConflict - ); -} - -#[tokio::test] -async fn manifest_recovery_burst_skips_rejected_candidate() { - let hits = Arc::new(AtomicUsize::new(0)); - let hits_for_handler = Arc::clone(&hits); - let origin = spawn_test_origin(Arc::new(move |_path| { - let hit = hits_for_handler.fetch_add(1, Ordering::SeqCst); - if hit == 0 { - return ( - 200, - Vec::new(), - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:50\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n50.bin\n".to_string(), - ); - } - ( - 200, - Vec::new(), - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n".to_string(), - ) - })) - .await; - let session = test_session(); - { - let mut session = session.write().await; - session.origin_seq_highwater = Some(100); - session.last_effective_manifest_host = Some(host_from_base_url(&origin.base_url)); - session.mark_authorized_media_access(super::current_time_millis()); - } - let entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("entry url"); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = entry; - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Friendly }; - - let target_url = Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("target url"); - let committed = retry_test_manifest_recovery_chain( - &request, - target_url, - HlsManifestRejectLogReason::PinnedHostRecoveryRejected, - ) - .await - .expect("burst should commit accepted candidate"); - - assert_eq!(committed.fetched.attempts, 1); - assert_eq!(hits.load(Ordering::SeqCst), 2); - assert_eq!(session.read().await.origin_seq_highwater, Some(101)); -} - -#[test] -fn manifest_recovery_candidate_score_prefers_same_host_next_sequence() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - session.origin_seq_highwater = Some(100); - session.last_effective_manifest_host = Some("origin.example.com".to_string()); - session.mark_authorized_media_access(super::current_time_millis()); - let request = test_origin_refresh_request(test_session()); - let fetch_context = manifest_fetch_context(&request); - let same_host_unchanged = - fetched_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n100.ts\n"); - let same_host_next = - fetched_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n"); - let mut other_host_next = same_host_next.clone(); - other_host_next.redirect_host = Some("other.example.com".to_string()); - - assert_eq!( - score_manifest_recovery_candidate(&session, &same_host_unchanged, &fetch_context).expect("score").quality.score, - HlsManifestOriginQualityScore::SameHostUnchanged - ); - assert_eq!( - score_manifest_recovery_candidate(&session, &same_host_next, &fetch_context).expect("score").quality.score, - HlsManifestOriginQualityScore::SameHostNextSequence - ); - let other_host_score = - score_manifest_recovery_candidate(&session, &other_host_next, &fetch_context).expect("score").quality; - assert_eq!(other_host_score.score, HlsManifestOriginQualityScore::OtherHostCandidate); - assert!(other_host_score.requires_handoff_discontinuity); -} - -#[tokio::test] -async fn manifest_recovery_burst_commits_best_same_host_candidate() { - let hits = Arc::new(AtomicUsize::new(0)); - let hits_for_handler = Arc::clone(&hits); - let origin = spawn_test_origin(Arc::new(move |_path| { - let hit = hits_for_handler.fetch_add(1, Ordering::SeqCst); - if hit == 0 { - return ( - 200, - Vec::new(), - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n100.ts\n".to_string(), - ); - } - ( - 200, - Vec::new(), - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n".to_string(), - ) - })) - .await; - let session = test_session(); - { - let mut session = session.write().await; - session.origin_seq_highwater = Some(100); - session.last_effective_manifest_host = Some(host_from_base_url(&origin.base_url)); - session.mark_authorized_media_access(super::current_time_millis()); - } - let entry = - LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("entry url"); - let mut request = test_origin_refresh_request(Arc::clone(&session)); - request.origin_entry = entry; - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Friendly }; - - let target_url = Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("target url"); - let committed = retry_test_manifest_recovery_chain( - &request, - target_url, - HlsManifestRejectLogReason::PinnedHostRecoveryRejected, - ) - .await - .expect("burst should commit best same-host candidate"); - - assert_eq!(committed.fetched.attempts, 1); - assert_eq!(hits.load(Ordering::SeqCst), 2); - assert_eq!(session.read().await.origin_seq_highwater, Some(101)); -} - -#[tokio::test] -async fn provider_failover_initial_success_commits_without_hls_host_retry_when_unpinned() { - let first = spawn_test_origin(Arc::new(|_path| (407, Vec::new(), "rotate".to_string()))).await; - let second_hits = Arc::new(AtomicUsize::new(0)); - let second_hits_for_handler = Arc::clone(&second_hits); - let second = spawn_test_origin(Arc::new(move |_path| { - let hit = second_hits_for_handler.fetch_add(1, Ordering::SeqCst); - if hit == 0 { - return ( - 200, - Vec::new(), - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:102\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n102.ts\n".to_string(), - ); - } - ( - 200, - Vec::new(), - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n#EXTINF:4.0,\n102.ts\n" - .to_string(), - ) - })) - .await; - let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { - name: "demo".into(), - urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, - dns: None, - })); - let session = test_session(); - { - session.write().await.origin_seq_highwater = Some(100); - } - let entry = LiveHlsOriginEntry::parse_with_url_failover_provider( - "provider://demo/live/user/pass/12345.m3u8", - Some(Arc::clone(&provider)), - ) - .expect("provider entry url"); - let request = OriginRefreshRequest { - app_config: test_app_config(), - session: Arc::clone(&session), - origin_entry: entry.clone(), - headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("client builds"), - use_manual_redirects: false, - segment_cache: Arc::new(HlsSegmentCache::new()), - hls_proxy: Arc::new(HlsProxyManager::new()), - segment_repair: test_segment_repair_manager(), - segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), - map_worker_pool: Arc::new(HlsMapWorkerPool::default()), - origin_manifest_timeout_ms: 2_000, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, - retry_policy: no_delay_policy(), - reverse_proxy_rewrite_secret: b"secret".to_vec(), - transient_resource_ttl_ms: 300_000, - manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, - fresh_manifest_requirement_generation: None, - acceptance_directive: HlsManifestAcceptanceDirective::none(), - access_lease_id: None, - disabled_headers: None, - now_ms: 100, - origin_io: None, - post_refresh_runtime: None, - }; - - assert!(maybe_trigger_origin_refresh(request).await); - for _ in 0..50 { - if session.read().await.origin_seq_highwater == Some(102) { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - - let first_manifest_requests = - first.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); - let second_manifest_requests = - second.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); - assert_eq!(first_manifest_requests, 1); - assert_eq!(second_manifest_requests, 1); - assert_eq!(session.read().await.origin_seq_highwater, Some(102)); -} - -#[tokio::test] -async fn host_switch_conflict_runs_one_full_plan_then_only_bounded_follow_ups() { - let candidate_hits = Arc::new(AtomicUsize::new(0)); - let candidate_hits_for_handler = Arc::clone(&candidate_hits); - let candidate = spawn_test_origin(Arc::new(move |_path| { - candidate_hits_for_handler.fetch_add(1, Ordering::SeqCst); - ( - 200, - Vec::new(), - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n900.ts\n".to_string(), - ) - })) - .await; - let session = test_session(); - { - let mut session = session.write().await; - session.origin_seq_highwater = Some(100); - session.last_effective_manifest_host = Some("previous.example.com".to_string()); - session.origin_control.pinned_host = Some("previous.example.com".to_string()); - } - let candidate_entry_url = - format!("{}/live/user/pass/12345.m3u8", candidate.base_url).replacen("127.0.0.1", "localhost", 1); - install_published_recovery_binding(&session, &candidate_entry_url, None).await; - let entry = LiveHlsOriginEntry::parse(&candidate_entry_url).expect("entry url"); - let request = OriginRefreshRequest { - app_config: test_app_config(), - session: Arc::clone(&session), - origin_entry: entry.clone(), - headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("client builds"), - use_manual_redirects: false, - segment_cache: Arc::new(HlsSegmentCache::new()), - hls_proxy: Arc::new(HlsProxyManager::new()), - segment_repair: test_segment_repair_manager(), - segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), - map_worker_pool: Arc::new(HlsMapWorkerPool::default()), - origin_manifest_timeout_ms: 2_000, - manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), - strip: StripConfig { mode: HlsStripMode::Segments, value: 3 }, - retry_policy: no_delay_policy(), - reverse_proxy_rewrite_secret: b"secret".to_vec(), - transient_resource_ttl_ms: 300_000, - manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, - fresh_manifest_requirement_generation: None, - acceptance_directive: HlsManifestAcceptanceDirective::none(), - access_lease_id: None, - disabled_headers: None, - now_ms: 100, - origin_io: None, - post_refresh_runtime: None, - }; - - assert!(trigger_origin_refresh_sync(request).await); - - assert_eq!(candidate_hits.load(Ordering::SeqCst), 6); - let session = session.read().await; - assert_eq!(session.origin_seq_highwater, Some(100)); - let episode = session.origin_control.acceptance_episode.as_ref().expect("held acceptance episode"); - assert_eq!(episode.full_bursts_completed, 1); -} - -#[tokio::test] -async fn material_reduced_retry_timeline_change_starts_one_new_complete_configured_burst() { - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let hits = Arc::new(AtomicUsize::new(0)); - let hits_for_handler = Arc::clone(&hits); - let origin = spawn_test_origin(Arc::new(move |_path| { - let hit = hits_for_handler.fetch_add(1, Ordering::SeqCst); - let resource = if hit < plan.total_candidates() { "old900.ts" } else { "new900.ts" }; - ( - 200, - Vec::new(), - format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n{resource}\n"), - ) - })) - .await; - let session = test_session(); - prepare_cross_host_baseline(&session).await; - let cache_dir = tempfile::tempdir().expect("requalification cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(cache_dir.path())); - let mut request = switch_test_request(Arc::clone(&session), cache, &origin.base_url); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - let target_url = - Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("requalification target"); - let attempts = request.retry_policy.attempt_count(); - let context = manifest_fetch_context(&request); - - let result = retry_hls_origin_manifest_recovery_chain( - &context, - test_manifest_origin_binding(target_url), - Some(HlsManifestRejectLogReason::PinnedHostRecoveryRejected), - None, - HlsManifestAcceptanceTrigger::Observe, - HlsManifestCommitAcceptanceMode::StrictPinnedHost, - |fetched, acceptance_mode| super::commit_manifest_recovery_candidate(&request, fetched, acceptance_mode), - ) - .await; - - assert!(result.is_err()); - let reduced_attempts = attempts.saturating_sub(2); - assert_eq!(hits.load(Ordering::SeqCst), plan.total_candidates().saturating_mul(2).saturating_add(reduced_attempts)); - let session = session.read().await; - let episode = session.origin_control.acceptance_episode.as_ref().expect("requalified episode"); - assert!(episode.generation.0 >= 2); - assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); - assert!(episode.full_burst_completed); - assert_eq!(episode.full_bursts_completed, 1); -} - -#[tokio::test] -async fn material_change_in_last_reduced_slot_still_runs_new_episode_full_burst() { - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let retry_policy = no_delay_policy(); - let attempts = retry_policy.attempt_count(); - let unchanged_candidate_count = plan.total_candidates().saturating_add(attempts.saturating_sub(2)); - let hits = Arc::new(AtomicUsize::new(0)); - let hits_for_handler = Arc::clone(&hits); - let origin = spawn_test_origin(Arc::new(move |_path| { - let hit = hits_for_handler.fetch_add(1, Ordering::SeqCst); - let resource = if hit < unchanged_candidate_count { "old900.ts" } else { "new900.ts" }; - ( - 200, - Vec::new(), - format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n{resource}\n"), - ) - })) - .await; - let session = test_session(); - prepare_cross_host_baseline(&session).await; - let cache_dir = tempfile::tempdir().expect("last-slot requalification cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(cache_dir.path())); - let mut request = switch_test_request(Arc::clone(&session), cache, &origin.base_url); - request.retry_policy = retry_policy; - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - let target_url = Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)) - .expect("last-slot requalification target"); - let context = manifest_fetch_context(&request); - - let result = retry_hls_origin_manifest_recovery_chain( - &context, - test_manifest_origin_binding(target_url), - Some(HlsManifestRejectLogReason::PinnedHostRecoveryRejected), - None, - HlsManifestAcceptanceTrigger::Observe, - HlsManifestCommitAcceptanceMode::StrictPinnedHost, - |fetched, acceptance_mode| super::commit_manifest_recovery_candidate(&request, fetched, acceptance_mode), - ) - .await; - - assert!(result.is_err()); - assert_eq!( - hits.load(Ordering::SeqCst), - plan.total_candidates().saturating_mul(2).saturating_add(attempts.saturating_sub(1)) - ); - let session = session.read().await; - let episode = session.origin_control.acceptance_episode.as_ref().expect("last-slot requalified episode"); - assert!(episode.generation.0 >= 2); - assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); - assert!(episode.full_burst_completed); - assert_eq!(episode.full_bursts_completed, 1); -} - -const SWITCH_MAP_BODY: &[u8] = b"complete-switch-map"; -const SWITCH_SEGMENT_BODY: &[u8] = b"complete-switch-segment-body"; -const CRITICAL_HANDOFF_TS_BODY: &[u8] = - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); -const CRITICAL_HANDOFF_MANIFEST_BODY: &[u8] = b"#EXTM3U\n\ - #EXT-X-MEDIA-SEQUENCE:900\n\ - #EXT-X-TARGETDURATION:4\n\ - #EXTINF:4.0,\nfirst.ts\n\ - #EXTINF:4.0,\nsecond.ts\n"; - -struct ControlledSwitchOriginServer { - base_url: String, - requests: Arc>>, - segment_prefix_written: Arc, - release_segment_body: Arc, - task: tokio::task::JoinHandle<()>, -} - -impl Drop for ControlledSwitchOriginServer { - fn drop(&mut self) { self.task.abort(); } -} - -async fn await_controlled_switch_segment_prefix( - origin: &ControlledSwitchOriginServer, - task: &mut tokio::task::JoinHandle, - task_name: &str, -) { - tokio::select! { - () = origin.segment_prefix_written.notified() => {} - result = &mut *task => { - panic!("{task_name} completed before controlled segment staging reached the network boundary: {result:?}"); - } - () = tokio::time::sleep(Duration::from_secs(10)) => { - panic!("{task_name} did not reach the controlled segment staging boundary before the test deadline"); - } - } -} - -struct CriticalEmergencyOriginServer { - base_url: String, - manifest_requests: Arc, - segment_requests: Arc, - segment_prefix_written: Arc, - release_segment_body: Arc, - task: tokio::task::JoinHandle<()>, -} - -impl Drop for CriticalEmergencyOriginServer { - fn drop(&mut self) { self.task.abort(); } -} - -async fn read_test_request_path(socket: &mut tokio::net::TcpStream) -> Option { - let mut buf = vec![0_u8; 4_096]; - let mut used = 0_usize; - loop { - let read = socket.read(&mut buf[used..]).await.ok()?; - if read == 0 { - return None; - } - used = used.saturating_add(read); - if used >= 4 && buf[..used].windows(4).any(|window| window == b"\r\n\r\n") { - break; - } - if used == buf.len() { - return None; - } - } - String::from_utf8_lossy(&buf[..used]) - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .map(str::to_string) -} - -async fn write_test_response(socket: &mut tokio::net::TcpStream, status: u16, body: &[u8]) { - let reason = if status == 200 { "OK" } else { "Not Found" }; - let head = format!("HTTP/1.1 {status} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); - let _ = socket.write_all(head.as_bytes()).await; - let _ = socket.write_all(body).await; -} - -async fn spawn_controlled_switch_origin() -> ControlledSwitchOriginServer { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("controlled switch origin binds"); - let addr = listener.local_addr().expect("controlled switch origin address"); - let requests = Arc::new(Mutex::new(Vec::new())); - let requests_for_task = Arc::clone(&requests); - let segment_prefix_written = Arc::new(Notify::new()); - let segment_prefix_written_for_task = Arc::clone(&segment_prefix_written); - let release_segment_body = Arc::new(Notify::new()); - let release_segment_body_for_task = Arc::clone(&release_segment_body); - let task = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let requests = Arc::clone(&requests_for_task); - let segment_prefix_written = Arc::clone(&segment_prefix_written_for_task); - let release_segment_body = Arc::clone(&release_segment_body_for_task); - tokio::spawn(async move { - let Some(path) = read_test_request_path(&mut socket).await else { - return; - }; - requests.lock().await.push(path.clone()); - match path.as_str() { - "/live/final/init.mp4" => write_test_response(&mut socket, 200, SWITCH_MAP_BODY).await, - "/live/final/first.ts" => { - let head = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - SWITCH_SEGMENT_BODY.len() - ); - if socket.write_all(head.as_bytes()).await.is_err() { - return; - } - let split = SWITCH_SEGMENT_BODY.len() / 2; - if socket.write_all(&SWITCH_SEGMENT_BODY[..split]).await.is_err() { - return; - } - segment_prefix_written.notify_one(); - release_segment_body.notified().await; - let _ = socket.write_all(&SWITCH_SEGMENT_BODY[split..]).await; - } - _ => write_test_response(&mut socket, 404, &[]).await, - } - }); - } - }); - ControlledSwitchOriginServer { - base_url: format!("http://{addr}"), - requests, - segment_prefix_written, - release_segment_body, - task, - } -} - -async fn spawn_critical_emergency_origin() -> CriticalEmergencyOriginServer { - spawn_critical_emergency_origin_with_control(false).await -} - -async fn spawn_controlled_critical_emergency_origin() -> CriticalEmergencyOriginServer { - spawn_critical_emergency_origin_with_control(true).await -} - -async fn spawn_critical_emergency_origin_with_control(pause_segment: bool) -> CriticalEmergencyOriginServer { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("critical emergency origin binds"); - let addr = listener.local_addr().expect("critical emergency origin address"); - let manifest_requests = Arc::new(AtomicUsize::new(0)); - let manifest_requests_for_task = Arc::clone(&manifest_requests); - let segment_requests = Arc::new(AtomicUsize::new(0)); - let segment_requests_for_task = Arc::clone(&segment_requests); - let segment_prefix_written = Arc::new(Notify::new()); - let segment_prefix_written_for_task = Arc::clone(&segment_prefix_written); - let release_segment_body = Arc::new(Notify::new()); - let release_segment_body_for_task = Arc::clone(&release_segment_body); - let task = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let manifest_requests = Arc::clone(&manifest_requests_for_task); - let segment_requests = Arc::clone(&segment_requests_for_task); - let segment_prefix_written = Arc::clone(&segment_prefix_written_for_task); - let release_segment_body = Arc::clone(&release_segment_body_for_task); - tokio::spawn(async move { - let Some(path) = read_test_request_path(&mut socket).await else { - return; - }; - if path_has_extension(&path, "m3u8") { - let request_index = manifest_requests.fetch_add(1, Ordering::SeqCst); - if request_index == 0 { - write_test_response(&mut socket, 200, CRITICAL_HANDOFF_MANIFEST_BODY).await; - } else { - write_test_response(&mut socket, 407, b"retryable manifest failure").await; - } - } else if path.ends_with("/first.ts") { - segment_requests.fetch_add(1, Ordering::SeqCst); - let head = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - CRITICAL_HANDOFF_TS_BODY.len() - ); - if socket.write_all(head.as_bytes()).await.is_err() { - return; - } - let split = CRITICAL_HANDOFF_TS_BODY.len() / 2; - if socket.write_all(&CRITICAL_HANDOFF_TS_BODY[..split]).await.is_err() { - return; - } - segment_prefix_written.notify_one(); - if pause_segment { - release_segment_body.notified().await; - } - let _ = socket.write_all(&CRITICAL_HANDOFF_TS_BODY[split..]).await; - } else { - write_test_response(&mut socket, 404, &[]).await; - } - }); - } - }); - CriticalEmergencyOriginServer { - base_url: format!("http://{addr}"), - manifest_requests, - segment_requests, - segment_prefix_written, - release_segment_body, - task, - } -} - -fn switch_manifest_body(include_map: bool) -> String { - let map = if include_map { "#EXT-X-MAP:URI=\"init.mp4\"\n" } else { "" }; - format!( - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:500\n#EXT-X-TARGETDURATION:4\n{map}#EXTINF:4.0,\nfirst.ts\n#EXTINF:4.0,\nsecond.ts\n" - ) -} - -fn switch_fetched_manifest(base_url: &str, include_map: bool) -> FetchedOriginManifest { - FetchedOriginManifest { - body: switch_manifest_body(include_map), - final_manifest_url: format!("{base_url}/live/final/index.m3u8"), - resolved_request_url: format!("{base_url}/live/user/pass/12345.m3u8"), - redirect_host: None, - provider_url_index: None, - provider_session_headers: HeaderMap::new(), - status: StatusCode::OK, - attempts: 1, - candidate_requests: 1, - selection: HlsManifestFetchSelection::Initial, - } -} - -fn switch_test_request( - session: Arc>, - segment_cache: Arc, - base_url: &str, -) -> OriginRefreshRequest { - let mut request = test_origin_refresh_request(session); - request.origin_entry = - LiveHlsOriginEntry::parse(&format!("{base_url}/live/user/pass/12345.m3u8")).expect("switch origin entry"); - request.client = reqwest::Client::builder().no_proxy().build().expect("switch client"); - request.no_redirect_client = reqwest::Client::builder() - .no_proxy() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("switch no-redirect client"); - request.hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(segment_cache.cache_path(), 300)); - request.segment_cache = segment_cache; - request.segment_worker_pool = Arc::new(HlsSegmentWorkerPool::new(SegmentFetchPolicy { - origin_segment_timeout_ms: 2_000, - retry_delays_ms: [0; 5], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - })); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Friendly }; - request -} - -async fn prepare_cross_host_baseline(session: &Arc>) { - let body = - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:500\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nold500.ts\n#EXTINF:4.0,\nold501.ts\n"; - let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = - tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( - body, - "http://previous.example.com/live/user/pass/12345.m3u8", - ) - else { - panic!("baseline manifest parses as normal timeline"); - }; - let mut session = session.write().await; - session - .apply_origin_manifest_for_host(&manifest, crate::timeline::effective_origin_host_id("previous.example.com")) - .expect("baseline timeline commits"); - session.last_effective_manifest_host = Some("previous.example.com".to_string()); - session.origin_control.pinned_host = Some("previous.example.com".to_string()); - session.origin_control.origin_epoch = session.origin_epoch; -} - -async fn install_published_recovery_binding( - session: &Arc>, - request_url: &str, - provider_url_index: Option, -) { - let binding = HlsManifestOriginBinding::new( - Url::parse(request_url).expect("published recovery request URL"), - provider_url_index, - ) - .expect("published recovery binding"); - let mut session = session.write().await; - session.origin_control.manifest_origin_binding = Some(binding); - session.origin_control.record_media_progress(1, 4_000); - let evidence_proxy_seq = session - .segments - .iter() - .find_map(|(proxy_seq, segment)| { - (!is_hls_provisioning_segment(segment) && !is_hls_provisioning_gap_segment(segment)).then_some(*proxy_seq) - }) - .unwrap_or_else(|| { - let proxy_seq = session.proxy_next_seq.unwrap_or(0); - let origin_sequence = session.origin_seq_highwater.unwrap_or(proxy_seq); - let origin_epoch = session.origin_epoch; - let cache_key = SegmentCacheKey::new(session.proxy_session_id.clone(), proxy_seq, "ts"); - session.segments.insert( - proxy_seq, - SegmentEntry { - origin_key: OriginSegmentKey { - origin_epoch, - effective_host_id: 0, - host_local_sequence: origin_sequence, - host_local_index: 0, - }, - proxy_seq, - duration_ms: 4_000, - proxy_file_ext: "ts".to_string(), - content_type: "video/mp2t".to_string(), - cache_key, - discontinuity_before: false, - program_date_time: None, - daterange_tags_before: Vec::new(), - origin_byte_range: None, - map_ref: None, - encryption: None, - origin_fetch_ref: None, - status: SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1 }, - last_rendered_at_ms: None, - access: Arc::new(CacheAccessState::new()), - }, - ); - session.publishable_origin_head_proxy_seq.get_or_insert(proxy_seq); - session.publishable_origin_tail_proxy_seq = Some(proxy_seq); - session.proxy_next_seq = Some(proxy_seq.saturating_add(1)); - proxy_seq - }); - let discontinuity_sequence = session.discontinuity_sequence; - let outcome = session.store_rendered_manifest(RenderedManifest { - body: "#EXTM3U\n".to_string(), - first_proxy_seq: evidence_proxy_seq, - last_proxy_seq: evidence_proxy_seq, - discontinuity_sequence, - target_duration_ms: 4_000, - playlist_duration_ms: 4_000, - valid_until_ms: 10_000, - render_gap_segments: 0, - rendered_at_ms: 1, - segment_proxy_seqs: vec![evidence_proxy_seq], - }); - assert_eq!(outcome, RenderedManifestStoreOutcome::Stored); - assert!(session.published_live_origin_baseline.is_some()); -} - -async fn commit_ready_baseline_snapshot( - session: &Arc>, - cache: &HlsSegmentCache, - now_ms: u64, -) -> HlsLeaseManifestSnapshot { - let cache_keys = { - let session = session.read().await; - session.segments.values().map(|segment| segment.cache_key.clone()).collect::>() - }; - for cache_key in &cache_keys { - cache.write_bytes_and_commit(cache_key, CRITICAL_HANDOFF_TS_BODY).await.expect("baseline TS object commits"); - } - - let (visible_segments, discontinuity_sequence, target_duration_ms) = { - let mut session = session.write().await; - for segment in session.segments.values_mut() { - assert!(segment.map_ref.is_none()); - assert!(segment.encryption.is_none()); - segment.status = SegmentCacheStatus::Ready { - content_length: u64::try_from(CRITICAL_HANDOFF_TS_BODY.len()).unwrap_or(u64::MAX), - ready_at_ms: now_ms, - }; - } - let visible_segments = session - .segments - .values() - .map(|segment| HlsLeaseManifestSegment { - proxy_seq: segment.proxy_seq, - duration_ms: segment.duration_ms, - uri: format!("/hls/test/{:06}.ts", segment.proxy_seq).into(), - discontinuity_before: segment.discontinuity_before, - map_ref_ready: true, - encryption: None, - }) - .collect::>(); - ( - visible_segments, - session.discontinuity_sequence, - session.target_duration.map_or(4_000, |seconds| u64::from(seconds).saturating_mul(1_000)), - ) - }; - let first_proxy_seq = visible_segments.first().expect("baseline head").proxy_seq; - let last_proxy_seq = visible_segments.last().expect("baseline tail").proxy_seq; - let playlist_duration_ms = - visible_segments.iter().fold(0_u64, |duration_ms, segment| duration_ms.saturating_add(segment.duration_ms)); - HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(now_ms), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 0, - delivered_at_ms: now_ms, - first_proxy_seq, - last_proxy_seq, - visible_segments: Arc::from(visible_segments), - discontinuity_sequence, - target_duration_ms, - playlist_duration_ms, - last_visible_media_end_ms: playlist_duration_ms, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - } -} - -fn active_critical_test_lease( - lease_id: &str, - proxy_session_id: &super::super::ProxySessionId, - issued_at_ms: u64, - now_ms: u64, - manifest: HlsLeaseManifestSnapshot, -) -> HlsAccessLease { - let mut lease = HlsAccessLease::pending( - HlsAccessLeaseId(lease_id.to_string()), - HlsPlaybackFamilyKey::new(lease_id, lease_id), - proxy_session_id.clone(), - lease_id.to_string(), - format!("{lease_id}-session"), - 1, - "12345".to_string(), - 12345, - issued_at_ms, - 120_000, - ); - lease.state = super::super::HlsAccessLeaseState::Activated; - lease.active_until_ms = Some(now_ms.saturating_add(120_000)); - lease.valid_until_ms = now_ms.saturating_add(120_000); - lease.playback_mode = HlsLeasePlaybackMode::Live; - lease.admission_generation = 1; - lease.last_manifest_snapshot = Some(manifest); - lease -} - -async fn prepare_three_segment_critical_timeline( - session: &Arc>, - cache: &HlsSegmentCache, - now_ms: u64, -) -> HlsLeaseManifestSnapshot { - let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:500\n#EXT-X-TARGETDURATION:4\n\ - #EXTINF:4.0,\nold500.ts\n#EXTINF:4.0,\nold501.ts\n#EXTINF:4.0,\nold502.ts\n"; - let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = - tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( - body, - "http://previous.example.com/live/user/pass/12345.m3u8", - ) - else { - panic!("three-segment baseline parses"); - }; - { - let mut session = session.write().await; - session - .apply_origin_manifest_for_host( - &manifest, - crate::timeline::effective_origin_host_id("previous.example.com"), - ) - .expect("three-segment baseline commits"); - session.last_effective_manifest_host = Some("previous.example.com".to_string()); - session.origin_control.pinned_host = Some("previous.example.com".to_string()); - session.origin_control.origin_epoch = session.origin_epoch; - } - commit_ready_baseline_snapshot(session, cache, now_ms).await -} - -fn critical_staging_generation( - session: &mut HlsSession, - now_ms: u64, -) -> super::switch_staging::HlsSwitchStagingGeneration { - let burst_plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - session.origin_control.begin_acceptance_episode( - now_ms, - burst_plan, - HlsManifestAcceptanceTrigger::Critical, - &test_acceptance_episode_timing(now_ms, burst_plan), - ); - let episode = session.origin_control.acceptance_episode.as_mut().expect("critical acceptance episode"); - episode.record_full_burst(); - episode.state = super::super::manifest_acceptance::HlsManifestAcceptanceState::StagingSwitchSegment; - let identity = super::super::manifest_acceptance::HlsManifestRecoveryCandidateIdentity::from_candidate( - 0, - Some("candidate.example.com"), - "test-candidate", - ); - assert_eq!( - episode.select_candidate(episode.generation, identity), - super::super::manifest_acceptance::HlsRecoveryWorkloadBindingUpdate::Applied - ); - assert_eq!( - episode.bind_selected_candidate( - episode.generation, - identity, - super::super::recovery_timing::HlsRecoveryWorkload::clear_fetch(), - ), - super::super::manifest_acceptance::HlsRecoveryWorkloadBindingUpdate::Applied - ); - super::switch_staging::switch_staging_generation(session).expect("critical staging generation") -} - -#[tokio::test] -async fn critical_handoff_lock_contention_exhaustion_is_typed_and_bounded() { - let attempts = Arc::new(AtomicUsize::new(0)); - let attempts_for_access = Arc::clone(&attempts); - - let result: Result<(), HlsManifestCommitError> = super::retry_critical_handoff_state_access( - crate::manifest_acceptance::HlsManifestAcceptanceGeneration(8), - || { - attempts_for_access.fetch_add(1, Ordering::SeqCst); - async { crate::HlsCriticalHandoffStateAccess::LockBusy } - }, - ) - .await; - - assert_eq!(attempts.load(Ordering::SeqCst), crate::critical_handoff::HLS_CRITICAL_HANDOFF_COMMIT_RETRIES); - let Err(HlsManifestCommitError::TimelineRejected { reason }) = result else { - panic!("exhausted lock contention must return a typed timeline rejection"); - }; - assert_eq!(reason, HlsManifestRejectLogReason::CriticalHandoffLockContentionExhausted); - assert_eq!(reason.status_label(), "critical-handoff-lock-contention-exhausted"); - assert_ne!(reason, HlsManifestRejectLogReason::SwitchResourceUnavailable); - assert_ne!(reason, HlsManifestRejectLogReason::StagedSwitchInvalidated); - assert_ne!(reason, HlsManifestRejectLogReason::PinnedHostRecoveryRejected); -} - -#[tokio::test] -async fn critical_handoff_lock_busy_retry_retains_staged_state_until_acquired() { - struct StagedRetentionProbe { - identity: u64, - drops: Arc, - } - - impl Drop for StagedRetentionProbe { - fn drop(&mut self) { self.drops.fetch_add(1, Ordering::SeqCst); } - } - - let drops = Arc::new(AtomicUsize::new(0)); - let attempts = Arc::new(AtomicUsize::new(0)); - let staged = StagedRetentionProbe { identity: 41, drops: Arc::clone(&drops) }; - let staged_address = std::ptr::from_ref(&staged); - let attempts_for_access = Arc::clone(&attempts); - - let committed = super::retry_critical_handoff_state_access( - crate::manifest_acceptance::HlsManifestAcceptanceGeneration(9), - || { - let attempt = attempts_for_access.fetch_add(1, Ordering::SeqCst); - let staged = &staged; - let drops_for_attempt = Arc::clone(&drops); - async move { - assert_eq!(std::ptr::from_ref(staged), staged_address); - assert_eq!(drops_for_attempt.load(Ordering::SeqCst), 0); - if attempt == 0 { - crate::HlsCriticalHandoffStateAccess::LockBusy - } else { - crate::HlsCriticalHandoffStateAccess::Acquired(Ok(staged.identity)) - } - } - }, - ) - .await - .expect("the retained staged state commits after transient contention"); - - assert_eq!(committed, 41); - assert_eq!(attempts.load(Ordering::SeqCst), 2); - assert_eq!(drops.load(Ordering::SeqCst), 0); - drop(staged); - assert_eq!(drops.load(Ordering::SeqCst), 1); -} - -#[tokio::test] -async fn critical_handoff_selects_endangered_lease_base_instead_of_session_tail() { - let temp_dir = tempfile::tempdir().expect("critical selection cache tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let session = test_session(); - let now_ms = super::current_time_millis(); - let complete = prepare_three_segment_critical_timeline(&session, &cache, now_ms).await; - let mut first_window = complete.clone(); - first_window.snapshot_generation = 1; - first_window.visible_segments = Arc::from([complete.visible_segments[0].clone()]); - first_window.last_proxy_seq = first_window.first_proxy_seq; - first_window.playlist_duration_ms = 4_000; - first_window.last_visible_media_end_ms = 4_000; - let mut endangered_window = complete.clone(); - endangered_window.snapshot_generation = 2; - endangered_window.visible_segments = - Arc::from([complete.visible_segments[0].clone(), complete.visible_segments[1].clone()]); - endangered_window.last_proxy_seq = complete.visible_segments[1].proxy_seq; - endangered_window.playlist_duration_ms = 8_000; - endangered_window.last_visible_media_end_ms = 8_000; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let leases = vec![ - active_critical_test_lease("more-reserve", &proxy_session_id, now_ms, now_ms, first_window), - active_critical_test_lease( - "endangered", - &proxy_session_id, - now_ms.saturating_add(1), - now_ms, - endangered_window, - ), - ]; - let mut session = session.write().await; - let generation = critical_staging_generation(&mut session, now_ms); - let (selected, snapshot) = - super::switch_staging::select_critical_handoff_lease(&session, &leases, &generation, now_ms) - .expect("one lease is cutover critical"); - - assert_eq!(selected.lease_id, HlsAccessLeaseId("endangered".to_string())); - assert_eq!(snapshot.base.proxy_seq, complete.visible_segments[1].proxy_seq); - assert_ne!(snapshot.base.proxy_seq, complete.visible_segments[2].proxy_seq); -} - -#[tokio::test] -async fn critical_handoff_prioritizes_earliest_safe_cutover_deadline() { - let temp_dir = tempfile::tempdir().expect("critical deadline cache tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let session = test_session(); - let now_ms = super::current_time_millis(); - let complete = prepare_three_segment_critical_timeline(&session, &cache, now_ms).await; - let mut wider_margin = complete.clone(); - wider_margin.snapshot_generation = 3; - wider_margin.target_duration_ms = 4_000; - let mut exhausted_short_margin = complete.clone(); - exhausted_short_margin.snapshot_generation = 4; - exhausted_short_margin.target_duration_ms = 1_000; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let mut wider_margin_lease = - active_critical_test_lease("wider-margin", &proxy_session_id, now_ms, now_ms, wider_margin); - let playback_at_ms = now_ms.saturating_sub(3_000); - let token = wider_margin_lease.playback_cursor.record_request_started(complete.last_proxy_seq, playback_at_ms); - let _ = wider_margin_lease.playback_cursor.record_request_completed(token, playback_at_ms); - let leases = vec![ - wider_margin_lease, - active_critical_test_lease( - "exhausted-short-margin", - &proxy_session_id, - now_ms.saturating_add(1), - now_ms, - exhausted_short_margin, - ), - ]; - let mut session = session.write().await; - let generation = critical_staging_generation(&mut session, now_ms); - let (selected, snapshot) = - super::switch_staging::select_critical_handoff_lease(&session, &leases, &generation, now_ms) - .expect("critical leases have a deterministic deadline order"); - - assert_eq!(selected.lease_id, HlsAccessLeaseId("wider-margin".to_string())); - assert_eq!(snapshot.base.proxy_seq, complete.last_proxy_seq); -} - -#[tokio::test] -async fn critical_handoff_manifest_supersession_invalidates_frozen_snapshot() { - let temp_dir = tempfile::tempdir().expect("critical supersession cache tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let session = test_session(); - let now_ms = super::current_time_millis(); - let complete = prepare_three_segment_critical_timeline(&session, &cache, now_ms).await; - let mut endangered_window = complete.clone(); - endangered_window.snapshot_generation = 7; - endangered_window.visible_segments = - Arc::from([complete.visible_segments[0].clone(), complete.visible_segments[1].clone()]); - endangered_window.last_proxy_seq = complete.visible_segments[1].proxy_seq; - endangered_window.playlist_duration_ms = 8_000; - endangered_window.last_visible_media_end_ms = 8_000; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let mut lease = active_critical_test_lease("endangered", &proxy_session_id, now_ms, now_ms, endangered_window); - let mut session = session.write().await; - let generation = critical_staging_generation(&mut session, now_ms); - let (_, frozen) = - super::switch_staging::select_critical_handoff_lease(&session, &[lease.clone()], &generation, now_ms) - .expect("initial critical snapshot"); - let mut leases = HlsAccessLeaseStore::default(); - leases.prepare_access_lease(lease.clone()); - assert!(super::switch_staging::critical_handoff_snapshot_is_current( - &mut leases, - &session, - &generation, - &frozen, - now_ms, - )); - if let Some(manifest) = lease.last_manifest_snapshot.as_mut() { - manifest.snapshot_generation = manifest.snapshot_generation.saturating_add(1); - } - leases.prepare_access_lease(lease); - - assert!(!super::switch_staging::critical_handoff_snapshot_is_current( - &mut leases, - &session, - &generation, - &frozen, - now_ms, - )); -} - -async fn prepare_active_critical_handoff_lease( - session: &Arc>, - cache: &HlsSegmentCache, - now_ms: u64, -) -> (Arc, HlsAccessLeaseId, HlsAccessLease) { - let manifest_snapshot = commit_ready_baseline_snapshot(session, cache, now_ms).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease_id = HlsAccessLeaseId("critical-emergency-handoff".to_string()); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(cache.cache_path(), 300)); - hls_proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("critical-user", "critical-client"), - proxy_session_id.clone(), - "critical-user".to_string(), - "critical-session".to_string(), - 1, - "12345".to_string(), - 12345, - now_ms, - 120_000, - )) - .await; - assert!(hls_proxy - .activate_access_lease( - &lease_id, - &proxy_session_id, - now_ms, - HlsAccessLeaseTiming { active_window_ms: 120_000, valid_window_ms: 120_000 }, - ) - .await - .is_activated()); - let publication_guard = hls_proxy - .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, now_ms) - .await - .expect("active lease accepts authoritative manifest publication"); - assert!(hls_proxy - .commit_access_lease_manifest_publication( - &lease_id, - &proxy_session_id, - publication_guard, - manifest_snapshot, - now_ms, - ) - .await - .is_committed()); - let lease = hls_proxy - .access_lease_response_snapshot(&lease_id, &proxy_session_id, now_ms) - .await - .expect("critical live lease snapshot"); - (hls_proxy, lease_id, lease) -} - -async fn install_active_critical_test_lease( - hls_proxy: &HlsProxyManager, - lease_id: &str, - proxy_session_id: &super::super::ProxySessionId, - manifest: HlsLeaseManifestSnapshot, - now_ms: u64, -) -> HlsAccessLeaseId { - let lease_id = HlsAccessLeaseId(lease_id.to_string()); - hls_proxy - .prepare_access_lease(HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new(lease_id.0.as_str(), lease_id.0.as_str()), - proxy_session_id.clone(), - lease_id.0.clone(), - format!("{}-session", lease_id.0), - 1, - "12345".to_string(), - 12345, - now_ms, - 120_000, - )) - .await; - assert!(hls_proxy - .activate_access_lease( - &lease_id, - proxy_session_id, - now_ms, - HlsAccessLeaseTiming { active_window_ms: 120_000, valid_window_ms: 120_000 }, - ) - .await - .is_activated()); - let publication_guard = hls_proxy - .prepare_access_lease_manifest_publication(&lease_id, proxy_session_id, now_ms) - .await - .expect("critical test lease publication guard"); - assert!(hls_proxy - .commit_access_lease_manifest_publication(&lease_id, proxy_session_id, publication_guard, manifest, now_ms,) - .await - .is_committed()); - lease_id -} - -fn critical_handoff_app_config() -> Arc { - let app_config = test_app_config(); - app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { - channel_unavailable: Some(TransportStreamBuffer::new(CRITICAL_HANDOFF_TS_BODY.to_vec())), - user_connections_exhausted: None, - provider_connections_exhausted: None, - low_priority_preempted: None, - user_account_expired: None, - panel_api_provisioning: None, - hls_session_or_lease_expired: None, - panel_api_provisioning_hls_segments: Vec::new(), - }))); - app_config -} - -#[tokio::test] -async fn hls_prepared_terminal_bundle_early_refresh_hook_starts_singleflight_for_known_target_duration() { - let mut request = test_origin_refresh_request(test_session()); - request.app_config = critical_handoff_app_config(); - let terminal_response = request.app_config.custom_stream_response.load_full(); - let asset = terminal_response - .as_ref() - .and_then(|response| response.channel_unavailable.as_ref()) - .and_then(|buffer| super::snapshot_terminal_media_asset(buffer).ok()) - .expect("terminal refresh asset"); - let target_duration_ms = asset.duration_ms().saturating_add(1_000); - let key = super::super::prepared_terminal_bundle::prepared_terminal_bundle_key( - &asset, - target_duration_ms, - super::HLS_TERMINAL_TAIL_SEGMENT_COUNT, - ); - - super::start_refresh_terminal_bundle_preparation(&request, target_duration_ms); - let state = - request.hls_proxy.wait_for_prepared_terminal_bundle(key).await.expect("early terminal preparation completes"); - - assert!(matches!( - state, - HlsPreparedTerminalBundleState::Ready { bundle } if bundle.key == key - )); -} - -#[test] -fn critical_handoff_terminal_response_revision_must_remain_current() { - let app_config = critical_handoff_app_config(); - let frozen = app_config.custom_stream_response.load_full(); - assert!(super::critical_handoff_terminal_response_is_current(&app_config, frozen.as_ref())); - let replacement = frozen.as_ref().map(|response| Arc::new(response.as_ref().clone())); - app_config.custom_stream_response.store(replacement); - - assert!(!super::critical_handoff_terminal_response_is_current(&app_config, frozen.as_ref())); -} - -#[test] -fn critical_handoff_and_terminal_tail_share_ts_inspector_signature() { - let terminal_buffer = TransportStreamBuffer::new(CRITICAL_HANDOFF_TS_BODY.to_vec()); - let terminal_asset = super::snapshot_terminal_media_asset(&terminal_buffer).expect("terminal asset"); - let critical_signature = match tuliprox_mpegts::ts_inspector::inspect_mpeg_ts( - std::io::Cursor::new(CRITICAL_HANDOFF_TS_BODY), - tuliprox_mpegts::ts_inspector::HlsTsProbeProtection::Clear, - tuliprox_mpegts::ts_inspector::HlsTsProbeBudget::default(), - ) - .expect("critical handoff fixture inspection succeeds") - { - tuliprox_mpegts::ts_inspector::HlsTsProbeOutcome::Found(signature) => signature, - outcome => panic!("critical handoff fixture has no MPEG-TS tracks: {outcome:?}"), - }; - - assert_eq!(critical_signature, terminal_asset.track_signature().clone()); -} - -async fn assert_critical_handoff_evidence(request: &OriginRefreshRequest, live_lease: &HlsAccessLease, now_ms: u64) { - let manifest = live_lease.last_manifest_snapshot.as_ref().expect("authoritative manifest snapshot"); - assert_eq!(manifest.snapshot_generation, 1); - let ready_timeline = { - let session = request.session.read().await; - session.ready_timeline_snapshot( - live_lease.playback_cursor.ready_timeline_start_proxy_seq(manifest.first_proxy_seq), - now_ms, - ) - }; - let reserve = super::super::evaluate_lease_reserve(super::super::HlsLeaseReserveInput { - manifest, - cursor: &live_lease.playback_cursor, - ready_timeline: &ready_timeline, - now_ms, - playback_rate_guard_milli: super::super::HLS_PLAYBACK_RATE_GUARD_MILLI, - recovery_trigger_budget: super::super::recovery_timing::HlsRecoveryTriggerBudgetMs::from_millis(0), - origin_path_degraded: true, - recovery_committed: false, - }); - assert_eq!(reserve.availability_basis, HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline); - assert_eq!(reserve.guaranteed_media_horizon_ms, manifest.last_visible_media_end_ms); - assert_eq!(reserve.guaranteed_reserve_ms, 0); - assert!(reserve.cutover_required); - let candidate_tracks = match tuliprox_mpegts::ts_inspector::inspect_mpeg_ts( - std::io::Cursor::new(CRITICAL_HANDOFF_TS_BODY), - tuliprox_mpegts::ts_inspector::HlsTsProbeProtection::Clear, - tuliprox_mpegts::ts_inspector::HlsTsProbeBudget::default(), - ) - .expect("critical handoff fixture inspection succeeds") - { - tuliprox_mpegts::ts_inspector::HlsTsProbeOutcome::Found(signature) => signature, - outcome => panic!("critical handoff fixture has no MPEG-TS tracks: {outcome:?}"), - }; - let terminal_response = - request.app_config.custom_stream_response.load_full().expect("critical handoff terminal response"); - let terminal_asset = super::snapshot_terminal_media_asset( - terminal_response.channel_unavailable.as_ref().expect("critical handoff terminal buffer"), - ) - .expect("critical handoff terminal asset"); - assert_eq!( - crate::critical_handoff::terminal_alternative_compatibility_for_critical_lease( - Some(terminal_asset.as_ref()), - live_lease, - &candidate_tracks, - ), - crate::manifest_acceptance::HlsTerminalAlternativeCompatibility::LiveHandoffSafer - ); -} - -async fn assert_critical_handoff_timeline_commit( - session: &Arc>, - cache: &HlsSegmentCache, - origin: &CriticalEmergencyOriginServer, - baseline_epoch: u64, - expected_candidate_count: usize, -) { - let (staged_cache_key, staged_content_length) = { - let session = session.read().await; - assert_eq!(session.origin_epoch, baseline_epoch.saturating_add(1)); - let effective_host = host_from_base_url(&origin.base_url); - assert_eq!(session.last_effective_manifest_host.as_deref(), Some(effective_host.as_str())); - assert_eq!( - session.origin_epoch_effective_host_id, - Some(crate::timeline::effective_origin_host_id(&effective_host)) - ); - let episode = session.origin_control.acceptance_episode.as_ref().expect("completed acceptance episode"); - assert!(episode.full_burst_completed); - assert_eq!(episode.completed_burst_candidates, expected_candidate_count); - assert_eq!(episode.state, super::super::manifest_acceptance::HlsManifestAcceptanceState::Completed); - let switched = session - .segments - .values() - .filter(|segment| segment.origin_key.origin_epoch == session.origin_epoch) - .collect::>(); - assert_eq!(switched.len(), 2); - let staged = switched.first().expect("staged handoff segment"); - assert_eq!(staged.origin_key.host_local_sequence, 900); - assert!(staged.discontinuity_before); - let content_length = match &staged.status { - SegmentCacheStatus::Ready { content_length, .. } => *content_length, - SegmentCacheStatus::Discovered - | SegmentCacheStatus::Queued { .. } - | SegmentCacheStatus::Fetching { .. } - | SegmentCacheStatus::CapacityDeferred { .. } - | SegmentCacheStatus::FailedRetryable { .. } - | SegmentCacheStatus::FailedPermanent { .. } - | SegmentCacheStatus::Expired => panic!("staged handoff segment must be READY"), - }; - assert_eq!(content_length, u64::try_from(CRITICAL_HANDOFF_TS_BODY.len()).unwrap_or(u64::MAX)); - assert!(!matches!(switched[1].status, SegmentCacheStatus::Ready { .. })); - (staged.cache_key.clone(), content_length) - }; - let staged_metadata = cache - .metadata(&staged_cache_key) - .await - .expect("staged cache metadata reads") - .expect("staged cache object exists"); - assert_eq!(staged_metadata.size, staged_content_length); -} - -#[tokio::test] -async fn critical_single_alternative_full_acceptance_commits_verified_new_origin_epoch() { - let origin = spawn_critical_emergency_origin().await; - let temp_dir = tempfile::tempdir().expect("critical handoff cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let session = test_session(); - prepare_cross_host_baseline(&session).await; - let baseline_binding = HlsManifestOriginBinding::new( - Url::parse("https://previous.example.com/live/index.m3u8?token=baseline").expect("baseline URL"), - Some(0), - ) - .expect("baseline binding"); - session.write().await.origin_control.manifest_origin_binding = Some(baseline_binding); - let now_ms = super::current_time_millis(); - let (baseline_epoch, baseline_readiness_generation) = { - let session = session.read().await; - (session.origin_epoch, session.activity.media_readiness_generation) - }; - let (hls_proxy, lease_id, live_lease) = prepare_active_critical_handoff_lease(&session, &cache, now_ms).await; - let mut request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); - request.app_config = critical_handoff_app_config(); - request.hls_proxy = Arc::clone(&hls_proxy); - request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::Critical; - request.access_lease_id = Some(lease_id); - request.now_ms = now_ms; - assert_critical_handoff_evidence(&request, &live_lease, now_ms).await; - - let plan = request.manifest_recovery_burst.level.plan(); - let target_url = request.origin_entry.url().clone(); - let fetch_context = manifest_fetch_context(&request); - let committed = retry_hls_origin_manifest_recovery_chain( - &fetch_context, - test_manifest_origin_binding(target_url.clone()), - Some(HlsManifestRejectLogReason::PinnedHostRecoveryRejected), - None, - HlsManifestAcceptanceTrigger::Critical, - HlsManifestCommitAcceptanceMode::StrictPinnedHost, - |fetched, acceptance_mode| super::commit_manifest_recovery_candidate(&request, fetched, acceptance_mode), - ) - .await - .expect("critical emergency handoff commits"); - - assert_eq!(committed.fetched.body.as_bytes(), CRITICAL_HANDOFF_MANIFEST_BODY); - assert_eq!(origin.manifest_requests.load(Ordering::SeqCst), plan.total_candidates()); - assert_eq!(origin.segment_requests.load(Ordering::SeqCst), 1); - assert_critical_handoff_timeline_commit(&session, &cache, &origin, baseline_epoch, plan.total_candidates()).await; - assert_eq!( - session.read().await.activity.media_readiness_generation, - baseline_readiness_generation.saturating_add(1), - "one staged READY transaction advances media readiness exactly once" - ); - assert_eq!( - session - .read() - .await - .origin_control - .manifest_origin_binding - .as_ref() - .map(super::super::manifest_origin_binding::HlsManifestOriginBinding::request_url), - Some(&target_url) - ); -} - -#[tokio::test] -async fn critical_handoff_uses_endangered_lease_base_when_session_tail_is_incompatible() { - let origin = spawn_critical_emergency_origin().await; - let temp_dir = tempfile::tempdir().expect("lease-specific critical handoff cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let session = test_session(); - let now_ms = super::current_time_millis(); - let complete = prepare_three_segment_critical_timeline(&session, &cache, now_ms).await; - let mut noncritical_window = complete.clone(); - noncritical_window.visible_segments = Arc::from([complete.visible_segments[0].clone()]); - noncritical_window.last_proxy_seq = noncritical_window.first_proxy_seq; - noncritical_window.playlist_duration_ms = 4_000; - noncritical_window.last_visible_media_end_ms = 4_000; - let mut endangered_window = complete.clone(); - endangered_window.visible_segments = - Arc::from([complete.visible_segments[0].clone(), complete.visible_segments[1].clone()]); - endangered_window.last_proxy_seq = complete.visible_segments[1].proxy_seq; - endangered_window.playlist_duration_ms = 8_000; - endangered_window.last_visible_media_end_ms = 8_000; - let (tail_key, lease_base_key, baseline_epoch) = { - let session = session.read().await; - ( - session.segments.get(&complete.last_proxy_seq).expect("session-wide tail").cache_key.clone(), - session.segments.get(&endangered_window.last_proxy_seq).expect("endangered lease base").cache_key.clone(), - session.origin_epoch, - ) - }; - cache.delete(&tail_key).await.expect("replace session-wide tail fixture"); - let invalid_tail = vec![0_u8; 188 * 2]; - cache.write_bytes_and_commit(&tail_key, &invalid_tail).await.expect("incompatible session-wide tail commits"); - { - let mut session = session.write().await; - let tail = session.segments.get_mut(&complete.last_proxy_seq).expect("session-wide tail entry"); - tail.status = SegmentCacheStatus::Ready { - content_length: u64::try_from(invalid_tail.len()).unwrap_or(u64::MAX), - ready_at_ms: now_ms, - }; - } - assert!(super::switch_staging::inspect_cache_object_tracks(&cache, &tail_key).await.signature().is_none()); - assert!(super::switch_staging::inspect_cache_object_tracks(&cache, &lease_base_key).await.signature().is_some()); - - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(cache.cache_path(), 300)); - let _noncritical_lease = install_active_critical_test_lease( - &hls_proxy, - "noncritical-window", - &proxy_session_id, - noncritical_window, - now_ms, - ) - .await; - let endangered_lease = install_active_critical_test_lease( - &hls_proxy, - "endangered-window", - &proxy_session_id, - endangered_window, - now_ms, - ) - .await; - let mut request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); - request.hls_proxy = hls_proxy; - request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::Critical; - request.access_lease_id = Some(endangered_lease); - request.now_ms = now_ms; - let plan = request.manifest_recovery_burst.level.plan(); - let target_url = request.origin_entry.url().clone(); - let fetch_context = manifest_fetch_context(&request); - - let committed = retry_hls_origin_manifest_recovery_chain( - &fetch_context, - test_manifest_origin_binding(target_url), - Some(HlsManifestRejectLogReason::PinnedHostRecoveryRejected), - None, - HlsManifestAcceptanceTrigger::Critical, - HlsManifestCommitAcceptanceMode::StrictPinnedHost, - |fetched, acceptance_mode| super::commit_manifest_recovery_candidate(&request, fetched, acceptance_mode), - ) - .await - .expect("lease-specific compatible base permits critical handoff"); - - assert_eq!(committed.fetched.body.as_bytes(), CRITICAL_HANDOFF_MANIFEST_BODY); - assert_eq!(origin.manifest_requests.load(Ordering::SeqCst), plan.total_candidates()); - assert_eq!(origin.segment_requests.load(Ordering::SeqCst), 1); - let session = session.read().await; - assert_eq!(session.origin_epoch, baseline_epoch.saturating_add(1)); - assert!(session - .segments - .values() - .find(|segment| segment.origin_key.origin_epoch == session.origin_epoch) - .is_some_and(|segment| segment.discontinuity_before)); -} - -#[tokio::test] -async fn critical_handoff_manifest_supersession_during_candidate_io_rolls_back_staging() { - let origin = spawn_controlled_critical_emergency_origin().await; - let temp_dir = tempfile::tempdir().expect("critical supersession cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let session = test_session(); - prepare_cross_host_baseline(&session).await; - let now_ms = super::current_time_millis(); - let (hls_proxy, lease_id, live_lease) = prepare_active_critical_handoff_lease(&session, &cache, now_ms).await; - let proxy_session_id = live_lease.proxy_session_id.clone(); - let base_proxy_seq = live_lease.last_manifest_snapshot.as_ref().expect("critical manifest snapshot").last_proxy_seq; - let (baseline_epoch, baseline_proxy_next_seq, baseline_readiness_generation, baseline_segments, base_access) = { - let session = session.read().await; - ( - session.origin_epoch, - session.proxy_next_seq, - session.activity.media_readiness_generation, - session - .segments - .iter() - .map(|(proxy_seq, segment)| (*proxy_seq, segment.cache_key.clone())) - .collect::>(), - Arc::clone(&session.segments.get(&base_proxy_seq).expect("critical lease base segment").access), - ) - }; - let candidate_cache_key = super::super::SegmentCacheKey::new( - proxy_session_id.clone(), - baseline_proxy_next_seq.expect("cross-host preview sequence"), - "ts", - ); - let mut request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); - request.app_config = critical_handoff_app_config(); - request.hls_proxy = Arc::clone(&hls_proxy); - request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::Critical; - request.access_lease_id = Some(lease_id.clone()); - request.now_ms = now_ms; - let request = Arc::new(request); - let task_request = Arc::clone(&request); - let target_url = task_request.origin_entry.url().clone(); - let commit_task = tokio::spawn(async move { - let fetch_context = manifest_fetch_context(&task_request); - retry_hls_origin_manifest_recovery_chain( - &fetch_context, - test_manifest_origin_binding(target_url), - Some(HlsManifestRejectLogReason::PinnedHostRecoveryRejected), - None, - HlsManifestAcceptanceTrigger::Critical, - HlsManifestCommitAcceptanceMode::StrictPinnedHost, - |fetched, acceptance_mode| { - super::commit_manifest_recovery_candidate(&task_request, fetched, acceptance_mode) - }, - ) - .await - }); - - origin.segment_prefix_written.notified().await; - assert!(base_access.active_readers() > 0, "lease base is pinned before candidate media completes"); - let publication_now_ms = super::current_time_millis(); - let publication_guard = hls_proxy - .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, publication_now_ms) - .await - .expect("superseding manifest publication guard"); - let superseding_snapshot = live_lease.last_manifest_snapshot.clone().expect("superseding snapshot"); - assert!(hls_proxy - .commit_access_lease_manifest_publication( - &lease_id, - &proxy_session_id, - publication_guard, - superseding_snapshot, - publication_now_ms, - ) - .await - .is_committed()); - origin.release_segment_body.notify_one(); - - let result = commit_task.await.expect("critical handoff task joins"); - assert!(result.is_err(), "superseded lease snapshot cannot commit"); - assert_eq!(base_access.active_readers(), 0, "rollback releases the frozen base pin"); - { - let session = session.read().await; - assert_eq!(session.origin_epoch, baseline_epoch); - assert_eq!(session.proxy_next_seq, baseline_proxy_next_seq); - assert_eq!(session.activity.media_readiness_generation, baseline_readiness_generation); - assert_eq!( - session - .segments - .iter() - .map(|(proxy_seq, segment)| (*proxy_seq, segment.cache_key.clone())) - .collect::>(), - baseline_segments - ); - } - assert!( - cache.metadata(&candidate_cache_key).await.expect("candidate cache metadata reads").is_none(), - "final CAS rejection removes the staged candidate object" - ); -} - -async fn assert_fresh_revalidation_cross_host_switch_runs_complete_beast_plan(reason: HlsFreshManifestRequiredReason) { - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let manifest = - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nfirst.ts\n#EXTINF:4.0,\nsecond.ts\n" - .to_string(); - let origin = spawn_test_origin(Arc::new(move |path| { - if path_has_extension(&path, "m3u8") { - (200, Vec::new(), manifest.clone()) - } else if path.ends_with("/first.ts") { - (200, Vec::new(), String::from_utf8_lossy(SWITCH_SEGMENT_BODY).into_owned()) - } else { - (404, Vec::new(), String::new()) - } - })) - .await; - let cache_dir = tempfile::tempdir().expect("fresh revalidation cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(cache_dir.path())); - let session = test_session(); - prepare_cross_host_baseline(&session).await; - let recovery_request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); - install_published_recovery_binding(&session, &recovery_request_url, None).await; - let baseline_epoch = session.read().await.origin_epoch; - let mut request = switch_test_request(Arc::clone(&session), cache, &origin.base_url); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { reason }; - // A weaker observation signal must not reduce the strict revalidation - // policy selected by the commit requirement. - request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::Observe; - - assert!(trigger_origin_refresh_sync(request).await); - - let (manifest_requests, staged_segment_requests) = { - let requests = origin.requests.lock().await; - ( - requests.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(), - requests.iter().filter(|path| path.ends_with("/first.ts")).count(), - ) - }; - assert_eq!(manifest_requests, plan.total_candidates()); - assert_eq!(staged_segment_requests, 1); - - let effective_host = host_from_base_url(&origin.base_url); - let session = session.read().await; - assert_eq!(session.origin_epoch, baseline_epoch.saturating_add(1)); - assert_eq!(session.last_effective_manifest_host.as_deref(), Some(effective_host.as_str())); - let switched_head = session - .segments - .values() - .find(|segment| segment.origin_key.origin_epoch == session.origin_epoch) - .expect("fresh revalidation switched timeline head"); - assert!(switched_head.discontinuity_before); - assert!(matches!(switched_head.status, SegmentCacheStatus::Ready { .. })); -} - -#[tokio::test] -async fn expired_revalidation_cross_host_switch_runs_complete_beast_acceptance() { - assert_fresh_revalidation_cross_host_switch_runs_complete_beast_plan( - HlsFreshManifestRequiredReason::ExpiredRevalidation, - ) - .await; -} - -#[tokio::test] -async fn hard_failure_revalidation_cross_host_switch_runs_complete_beast_acceptance() { - assert_fresh_revalidation_cross_host_switch_runs_complete_beast_plan( - HlsFreshManifestRequiredReason::PreviousHardManifestFailure, - ) - .await; -} - -#[tokio::test] -async fn fresh_pinned_revalidation_same_host_rebase_runs_complete_beast_plan() { - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let origin = spawn_test_origin(Arc::new(|path| { - if path_has_extension(&path, "m3u8") { - ( - 200, - Vec::new(), - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:10\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n10.ts\n#EXTINF:4.0,\n11.ts\n" - .to_string(), - ) - } else { - (404, Vec::new(), String::new()) - } - })) - .await; - let effective_host = host_from_base_url(&origin.base_url); - let baseline_body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1000\n#EXT-X-TARGETDURATION:4\n\ - #EXTINF:4.0,\n1000.ts\n#EXTINF:4.0,\n1001.ts\n"; - let baseline_manifest_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); - let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(baseline) = - tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest(baseline_body, &baseline_manifest_url) - else { - panic!("same-host stale baseline parses"); - }; - let session = test_session(); - let baseline_epoch = { - let mut session = session.write().await; - session - .apply_origin_manifest_for_host(&baseline, crate::timeline::effective_origin_host_id(&effective_host)) - .expect("same-host stale baseline commits"); - session.last_effective_manifest_host = Some(effective_host.clone()); - session.origin_control.pinned_host = Some(effective_host.clone()); - session.origin_control.origin_epoch = session.origin_epoch; - session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::PreviousHardManifestFailure); - session.origin_epoch - }; - install_published_recovery_binding(&session, &baseline_manifest_url, None).await; - let cache_dir = tempfile::tempdir().expect("fresh pinned revalidation cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(cache_dir.path())); - let mut request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { - reason: HlsFreshManifestRequiredReason::PreviousHardManifestFailure, - }; - request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::Observe; - - assert!(trigger_origin_refresh_sync(request).await); - - let manifest_requests = - origin.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); - assert_eq!(manifest_requests, plan.total_candidates()); - let next_fetch_allowed_at_ms = { - let session = session.read().await; - assert_eq!(session.origin_epoch, baseline_epoch.saturating_add(1)); - assert_eq!(session.origin_seq_highwater, Some(11)); - assert_eq!(session.last_effective_manifest_host.as_deref(), Some(effective_host.as_str())); - assert_eq!(session.fresh_manifest_commit_required, None); - let rebased_head = session - .segments - .values() - .find(|segment| segment.origin_key.origin_epoch == session.origin_epoch) - .expect("same-host rebased timeline head"); - assert!(rebased_head.discontinuity_before); - session.origin_refresh.next_fetch_allowed_at_ms - }; - - let mut follow_up = switch_test_request(Arc::clone(&session), cache, &origin.base_url); - follow_up.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - follow_up.manifest_commit_requirement = session - .read() - .await - .fresh_manifest_commit_required - .map_or(HlsManifestCommitRequirement::CommittedManifestAllowed, |reason| { - HlsManifestCommitRequirement::FreshCommitRequired { reason } - }); - follow_up.now_ms = next_fetch_allowed_at_ms; - - assert!(trigger_origin_refresh_sync(follow_up).await); - let manifest_requests_after_follow_up = - origin.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); - assert_eq!(manifest_requests_after_follow_up, plan.total_candidates().saturating_add(1)); -} - -async fn prepare_ready_content_anchor(session: &Arc>, cache: &HlsSegmentCache, bytes: &[u8]) { - let key = { - let mut session = session.write().await; - let segment = session.segments.values_mut().next().expect("baseline segment"); - segment.status = SegmentCacheStatus::Ready { - content_length: u64::try_from(bytes.len()).unwrap_or(u64::MAX), - ready_at_ms: 1, - }; - segment.cache_key.clone() - }; - cache.write_bytes_and_commit(&key, bytes).await.expect("committed anchor bytes"); -} - -async fn assert_cross_host_replay_does_not_commit(candidate_bytes: &'static str) { - const COMMITTED_BYTES: &[u8] = b"committed-media-bytes"; - let manifest = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nold500.ts\n".to_string(); - let origin = spawn_test_origin(Arc::new(move |path| { - if path_has_extension(&path, "m3u8") { - (200, Vec::new(), manifest.clone()) - } else if path.ends_with("/old500.ts") { - (200, Vec::new(), candidate_bytes.to_string()) - } else { - (404, Vec::new(), String::new()) - } - })) - .await; - let cache_dir = tempfile::tempdir().expect("content anchor cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(cache_dir.path())); - let session = test_session(); - prepare_cross_host_baseline(&session).await; - prepare_ready_content_anchor(&session, &cache, COMMITTED_BYTES).await; - let baseline_epoch = session.read().await.origin_epoch; - let mut request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Friendly }; - let target_url = - Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("content anchor target"); - - let result = retry_test_manifest_recovery_chain( - &request, - target_url, - HlsManifestRejectLogReason::PinnedHostRecoveryRejected, - ) - .await; - - let requests = origin.requests.lock().await.clone(); - let session = session.read().await; - assert!(result.is_err(), "replay-only cross-host candidate committed: requests={requests:?}"); - assert_eq!(session.origin_epoch, baseline_epoch); -} - -#[tokio::test] -async fn same_cross_host_sequence_path_and_equal_bytes_without_forward_progress_never_commits() { - assert_cross_host_replay_does_not_commit("committed-media-bytes").await; -} - -#[tokio::test] -async fn same_cross_host_sequence_and_path_with_different_bytes_never_commits_as_anchor() { - assert_cross_host_replay_does_not_commit("different-media-bytes").await; -} - -async fn assert_committed_content_anchor_is_gc_pinned_until_read_pin_drop() { - const COMMITTED_BYTES: &[u8] = b"committed-media-bytes"; - - let temp_dir = tempfile::tempdir().expect("acceptance pin cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let sessions = Arc::new(HlsSessionStore::new()); - let session = sessions.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; - let gc = HlsGarbageCollector::new( - Arc::clone(&sessions), - Arc::clone(&cache), - GarbageCollectionPolicy { - cache_duration_ms: 0, - cache_bytes_global: 10_000, - cache_bytes_per_session: 10_000, - session_idle_timeout_ms: u64::MAX, - temp_file_retention_ms: 30_000, - failed_segment_retention_ms: 10, - }, - build_rewrite_secret_fingerprint(b"secret"), - ); - let manifest_body = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ - #EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n\ - #EXTINF:4.0,\n4.ts\n#EXTINF:4.0,\n5.ts\n#EXTINF:4.0,\n6.ts\n"; - let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = - tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( - manifest_body, - "http://origin.example/live/index.m3u8", - ) - else { - panic!("acceptance pin manifest must parse as a normal timeline"); - }; - let cache_key = { - let mut session = session.write().await; - session.proxy_next_seq = Some(1); - session.apply_origin_manifest(&manifest).expect("acceptance pin timeline commits"); - session.segments.get(&1).expect("first committed segment").cache_key.clone() - }; - cache.write_bytes_and_commit(&cache_key, COMMITTED_BYTES).await.expect("committed acceptance object writes"); - let access = { - let mut session = session.write().await; - let segment = session.segments.get_mut(&1).expect("first committed segment"); - segment.status = SegmentCacheStatus::Ready { - content_length: u64::try_from(COMMITTED_BYTES.len()).unwrap_or(u64::MAX), - ready_at_ms: 0, - }; - let expected_identity = - crate::resource_identity::HlsMediaResourceIdentity::from_url("http://origin.example/live/1.ts", None); - session - .segments - .values() - .rev() - .find_map(|entry| { - let fetch_ref = entry.origin_fetch_ref.as_ref()?; - crate::resource_identity::HlsMediaResourceIdentity::from_url( - &fetch_ref.resolved_origin_url, - fetch_ref.byte_range, - ) - .matches(expected_identity) - .then(|| Arc::clone(&entry.access)) - }) - .expect("content anchor object selection") - }; - let read_pin = super::commit::HlsCommittedAcceptanceReadPin::acquire(Arc::clone(&access), 5); - assert_eq!(access.active_readers(), 1); - - let pinned_report = gc.run_once(10_000).await.expect("GC runs while acceptance object is pinned"); - assert_eq!(pinned_report.segments_deleted_duration, 0); - assert!(session.read().await.segments.contains_key(&1)); - assert!(cache.metadata(&cache_key).await.expect("pinned object metadata reads").is_some()); - - drop(read_pin); - assert_eq!(access.active_readers(), 0); - let released_report = gc.run_once(10_001).await.expect("GC runs after acceptance pin release"); - assert_eq!(released_report.segments_deleted_duration, 1); - assert!(!session.read().await.segments.contains_key(&1)); - assert!(cache.metadata(&cache_key).await.expect("released object metadata reads").is_none()); -} - -#[tokio::test] -async fn committed_content_anchor_object_survives_gc_until_acceptance_read_pin_drop() { - assert_committed_content_anchor_is_gc_pinned_until_read_pin_drop().await; -} - -#[tokio::test] -async fn critical_handoff_base_evidence_survives_gc_until_preparation_drop() { - let temp_dir = tempfile::tempdir().expect("critical evidence GC tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let sessions = Arc::new(HlsSessionStore::new()); - let session = sessions.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; - prepare_cross_host_baseline(&session).await; - let manifest = commit_ready_baseline_snapshot(&session, &cache, 0).await; - let base_proxy_seq = manifest.last_proxy_seq; - let (base_key, base_access) = { - let session = session.read().await; - let base = session.segments.get(&base_proxy_seq).expect("lease-specific terminal base"); - (base.cache_key.clone(), Arc::clone(&base.access)) - }; - let gc = HlsGarbageCollector::new( - Arc::clone(&sessions), - Arc::clone(&cache), - GarbageCollectionPolicy { - cache_duration_ms: 0, - cache_bytes_global: 10_000_000, - cache_bytes_per_session: 10_000_000, - session_idle_timeout_ms: u64::MAX, - temp_file_retention_ms: 30_000, - failed_segment_retention_ms: 10, - }, - build_rewrite_secret_fingerprint(b"secret"), - ); - - let evidence = super::super::prepare_terminal_base_evidence(&session, &cache, &manifest, 5).await; - assert_eq!(evidence.track_base().map(|base| base.proxy_seq), Some(base_proxy_seq)); - assert!(evidence.track_signature().is_some()); - assert!(base_access.active_readers() > 0); - let pinned_report = gc.run_once(10_000).await.expect("GC runs while critical base evidence is pinned"); - assert_eq!(pinned_report.segments_deleted_duration, 0); - assert!(cache.metadata(&base_key).await.expect("pinned base metadata reads").is_some()); - - evidence.release(); - assert_eq!(base_access.active_readers(), 0); - let released_report = gc.run_once(10_001).await.expect("GC runs after critical base evidence release"); - assert!(released_report.segments_deleted_duration > 0); - assert!(cache.metadata(&base_key).await.expect("released base metadata reads").is_none()); -} - -fn candidate_handoff_preview( - session: &HlsSession, - body: &str, - now_ms: u64, -) -> ( - tuliprox_parser::hls::origin_manifest::ParsedOriginManifest, - Vec, - crate::timeline::HlsOriginHandoffPreview, -) { - let final_url = "https://candidate.example/live/index.m3u8"; - let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(mut manifest) = - tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest(body, final_url) - else { - panic!("candidate fixture must parse as a normal manifest"); - }; - let key_resources = super::commit::materialize_normal_key_resources(&mut manifest, b"secret", now_ms, 60_000); - let preview = session - .preview_origin_handoff_manifest(&manifest, crate::timeline::effective_origin_host_id("candidate.example"), 0) - .expect("candidate handoff preview"); - (manifest, key_resources, preview) -} - -#[test] -fn hls_recovery_timing_candidate_preview_uses_actual_map_medium_not_playlist_head() { - let session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:40\n#EXT-X-TARGETDURATION:4\n\ - #EXTINF:4.0,\nhead40.ts\n\ - #EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\nrecovery41.m4s\n"; - let (manifest, key_resources, preview) = candidate_handoff_preview(&session, body, 100); - assert!(manifest.segments.first().is_some_and(|segment| segment.map_ref.is_none())); - let actual_recovery_segment = preview.segments.get(1).expect("actual recovery medium after clear head"); - let required_map = - actual_recovery_segment.map_ref.and_then(|map_id| preview.maps.iter().find(|map| map.proxy_map_id == map_id)); - - let workload = super::switch_staging::handoff_preview_recovery_workload( - &session, - actual_recovery_segment, - required_map, - &key_resources, - 100, - ); - - assert_eq!(workload.segment, HlsRecoverySegmentWorkload::ClearSegmentFetch); - assert_eq!(workload.map, HlsRecoveryMapWorkload::Fetch); -} - -#[test] -fn hls_recovery_timing_candidate_preview_detects_generation_local_ready_aes128_key() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:40\n#EXT-X-TARGETDURATION:4\n\ - #EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nrecovery40.ts\n"; - let (_, key_resources, preview) = candidate_handoff_preview(&session, body, 100); - let key_resource = key_resources.first().cloned().expect("candidate key resource"); - session.transient.upsert_resources([key_resource.clone()]); - let proxy_session_id = session.proxy_session_id.clone(); - let fetch = session.transient.begin_object_fetch(&proxy_session_id, &key_resource, "bin", 100, 60_000); - let TransientObjectFetchDecision::Fetch(token) = fetch else { - panic!("new candidate key starts one cache fill"); - }; - assert!(session.transient.mark_object_ready_if_current( - &token, - "application/octet-stream".to_string(), - 16, - 100, - 60_100, - )); - let actual_recovery_segment = preview.segments.first().expect("AES recovery medium"); - - let workload = super::switch_staging::handoff_preview_recovery_workload( - &session, - actual_recovery_segment, - None, - &key_resources, - 101, - ); - - assert_eq!(workload.segment, HlsRecoverySegmentWorkload::Aes128SegmentFetchWithReadyKey); - assert_eq!( - super::switch_staging::staged_switch_media_compatibility( - &session, - actual_recovery_segment, - &key_resources, - 101 - ), - super::switch_staging::HlsStagedSwitchMediaCompatibility::Compatible - ); - assert_eq!( - super::switch_staging::staged_switch_media_compatibility( - &session, - actual_recovery_segment, - &key_resources, - 60_101 - ), - super::switch_staging::HlsStagedSwitchMediaCompatibility::RequiresUnstagedEncryptionKey, - "the final commit revalidation must reject an expired READY key" - ); -} - -#[test] -fn hls_recovery_timing_candidate_preview_requires_aes128_key_fetch_without_ready_evidence() { - let session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:40\n#EXT-X-TARGETDURATION:4\n\ - #EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nrecovery40.ts\n"; - let (_, key_resources, preview) = candidate_handoff_preview(&session, body, 100); - let actual_recovery_segment = preview.segments.first().expect("AES recovery medium"); - - let workload = super::switch_staging::handoff_preview_recovery_workload( - &session, - actual_recovery_segment, - None, - &key_resources, - 101, - ); - - assert_eq!(workload.segment, HlsRecoverySegmentWorkload::Aes128SegmentFetchWithKeyFetch); - assert_eq!( - super::switch_staging::staged_switch_media_compatibility( - &session, - actual_recovery_segment, - &key_resources, - 101 - ), - super::switch_staging::HlsStagedSwitchMediaCompatibility::RequiresUnstagedEncryptionKey - ); -} - -async fn mark_full_burst_ready_for_switch_staging(session: &Arc>, fetched: &FetchedOriginManifest) { - let candidate_workload = { - let session = session.read().await; - let (_, key_resources, preview) = candidate_handoff_preview(&session, &fetched.body, 100); - let first_segment = preview.segments.first().expect("switch candidate recovery segment"); - let required_map = - first_segment.map_ref.and_then(|map_id| preview.maps.iter().find(|map| map.proxy_map_id == map_id)); - super::switch_staging::handoff_preview_recovery_workload( - &session, - first_segment, - required_map, - &key_resources, - 100, - ) - }; - let mut session = session.write().await; - let burst_plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); - session.origin_control.begin_acceptance_episode( - 100, - burst_plan, - HlsManifestAcceptanceTrigger::RecoveryRequired, - &test_switch_staging_acceptance_episode_timing(100, burst_plan), - ); - let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); - episode.record_full_burst(); - episode.state = super::super::manifest_acceptance::HlsManifestAcceptanceState::StagingSwitchSegment; - let effective_host = fetched_effective_manifest_host(fetched); - let identity = super::super::manifest_acceptance::HlsManifestRecoveryCandidateIdentity::from_candidate( - 0, - effective_host.as_deref(), - &fetched.body, - ); - assert_eq!( - episode.select_candidate(episode.generation, identity), - super::super::manifest_acceptance::HlsRecoveryWorkloadBindingUpdate::Applied - ); - let mut binding_probe = episode.clone(); - assert_eq!( - binding_probe.bind_selected_candidate(episode.generation, identity, candidate_workload), - super::super::manifest_acceptance::HlsRecoveryWorkloadBindingUpdate::Applied, - "switch-staging fixture must admit the selected candidate before network staging" - ); -} - -#[test] -fn switch_staging_rejects_map_fetch_outside_fixture_envelope_before_network() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); - let burst_plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); - session.origin_control.begin_acceptance_episode( - 100, - burst_plan, - HlsManifestAcceptanceTrigger::RecoveryRequired, - &test_acceptance_episode_timing(100, burst_plan), - ); - let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); - episode.record_full_burst(); - episode.state = super::super::manifest_acceptance::HlsManifestAcceptanceState::StagingSwitchSegment; - let identity = super::super::manifest_acceptance::HlsManifestRecoveryCandidateIdentity::from_candidate( - 0, - Some("candidate.example"), - &switch_manifest_body(true), - ); - assert_eq!( - episode.select_candidate(episode.generation, identity), - super::super::manifest_acceptance::HlsRecoveryWorkloadBindingUpdate::Applied - ); - let map_fetch = HlsRecoveryWorkload::from_recovery_medium(HlsRecoveryMediumReadiness { - segment: HlsRecoveryObjectReadiness::Fetch, - map: Some(HlsRecoveryObjectReadiness::Fetch), - encryption: HlsRecoveryEncryptionReadiness::Clear, - }); - - assert_eq!( - episode.bind_selected_candidate(episode.generation, identity, map_fetch), - super::super::manifest_acceptance::HlsRecoveryWorkloadBindingUpdate::OutsideEnvelope - ); -} - -async fn assert_incompatible_switch_is_rejected_before_timeline_commit( - baseline_body: &str, - candidate_body: &str, - expected_reason: HlsManifestRejectLogReason, -) { - let session = test_session(); - let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(baseline) = - tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( - baseline_body, - "http://previous.example.com/live/index.m3u8", - ) - else { - panic!("baseline parses as normal timeline"); - }; - { - let mut session = session.write().await; - session - .apply_origin_manifest_for_host( - &baseline, - crate::timeline::effective_origin_host_id("previous.example.com"), - ) - .expect("baseline timeline commits"); - session.last_effective_manifest_host = Some("previous.example.com".to_string()); - session.origin_control.pinned_host = Some("previous.example.com".to_string()); - session.origin_control.origin_epoch = session.origin_epoch; - } - let temp_dir = tempfile::tempdir().expect("switch cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let base_url = "http://127.0.0.1:9"; - let request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), base_url); - let fetched = FetchedOriginManifest { - body: candidate_body.to_string(), - final_manifest_url: format!("{base_url}/live/final/index.m3u8"), - resolved_request_url: format!("{base_url}/live/user/pass/12345.m3u8"), - redirect_host: None, - provider_url_index: None, - provider_session_headers: HeaderMap::new(), - status: StatusCode::OK, - attempts: 1, - candidate_requests: 1, - selection: HlsManifestFetchSelection::Initial, - }; - mark_full_burst_ready_for_switch_staging(&session, &fetched).await; - let before = { - let session = session.read().await; - (session.origin_epoch, session.proxy_next_seq, session.segments.len(), session.maps.len()) - }; - - let result = super::commit_manifest_recovery_candidate( - &request, - fetched, - HlsManifestCommitAcceptanceMode::AllowHeldHostSwitchCandidate, - ) - .await; - - assert!(matches!( - result, - Err(HlsManifestCommitError::TimelineRejected { reason }) if reason == expected_reason - )); - let session = session.read().await; - assert_eq!((session.origin_epoch, session.proxy_next_seq, session.segments.len(), session.maps.len()), before); - drop(session); - assert!(!cache.has_active_temp_files()); -} - -#[tokio::test] -async fn staged_cross_host_aes_candidate_without_ready_key_evidence_never_commits() { - let baseline = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-TARGETDURATION:4\n\ - #EXTINF:4.0,\nold1.ts\n#EXTINF:4.0,\nold2.ts\n#EXTINF:4.0,\nold3.ts\n"; - let candidate = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:40\n#EXT-X-TARGETDURATION:4\n\ - #EXT-X-KEY:METHOD=AES-128,URI=\"key.php\"\n\ - #EXTINF:4.0,\nnew40.ts\n#EXTINF:4.0,\nnew41.ts\n#EXTINF:4.0,\nnew42.ts\n"; - - assert_incompatible_switch_is_rejected_before_timeline_commit( - baseline, - candidate, - HlsManifestRejectLogReason::SwitchEncryptionKeyNotReady, - ) - .await; -} - -#[tokio::test] -async fn staged_cross_host_aes_candidate_with_generation_local_ready_key_commits() { - let temp_dir = tempfile::tempdir().expect("switch cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let origin = spawn_controlled_switch_origin().await; - let session = test_session(); - prepare_cross_host_baseline(&session).await; - let now_ms = current_time_millis(); - let candidate = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:500\n#EXT-X-TARGETDURATION:4\n\ - #EXT-X-KEY:METHOD=AES-128,URI=\"key.php\"\n\ - #EXTINF:4.0,\nfirst.ts\n#EXTINF:4.0,\nsecond.ts\n"; - let mut fetched = switch_fetched_manifest(&origin.base_url, false); - fetched.body = candidate.to_string(); - let key_resource = { - let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(mut manifest) = - tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest(candidate, &fetched.final_manifest_url) - else { - panic!("AES switch manifest parses as a normal timeline"); - }; - super::commit::materialize_normal_key_resources(&mut manifest, b"secret", now_ms, 300_000) - .into_iter() - .next() - .expect("AES switch candidate has one key resource") - }; - { - let mut session = session.write().await; - session.transient.upsert_resources([key_resource.clone()]); - let proxy_session_id = session.proxy_session_id.clone(); - let extension = key_resource.file_ext_hint.as_deref().expect("key extension"); - let fetch = session.transient.begin_object_fetch(&proxy_session_id, &key_resource, extension, now_ms, 300_000); - let TransientObjectFetchDecision::Fetch(token) = fetch else { - panic!("candidate key starts one cache fill"); - }; - assert!(session.transient.mark_object_ready_if_current( - &token, - "application/octet-stream".to_string(), - 16, - now_ms, - now_ms.saturating_add(300_000), - )); - } - mark_full_burst_ready_for_switch_staging(&session, &fetched).await; - let mut request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); - request.now_ms = now_ms; - let mut commit_task = tokio::spawn(async move { - super::commit_manifest_recovery_candidate( - &request, - fetched, - HlsManifestCommitAcceptanceMode::AllowHeldHostSwitchCandidate, - ) - .await - .map(|_| ()) - }); - - await_controlled_switch_segment_prefix(&origin, &mut commit_task, "encrypted cross-host switch commit").await; - origin.release_segment_body.notify_one(); - commit_task.await.expect("encrypted switch commit task").expect("encrypted switch commit succeeds"); - - let session = session.read().await; - let switched = session - .segments - .values() - .find(|segment| segment.origin_key.origin_epoch == session.origin_epoch) - .expect("switched segment committed"); - assert!(switched.encryption.is_some()); - assert!(session - .transient - .ready_key_object_valid_until_ms( - &session.proxy_session_id, - &key_resource.id, - key_resource.file_ext_hint.as_deref().expect("key extension"), - current_time_millis(), - ) - .is_some()); -} - -#[tokio::test] -async fn mapped_fmp4_tail_cannot_handoff_to_mapless_ts_timeline() { - let baseline = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-TARGETDURATION:4\n\ - #EXT-X-MAP:URI=\"init.mp4\"\n\ - #EXTINF:4.0,\nold1.m4s\n#EXTINF:4.0,\nold2.m4s\n#EXTINF:4.0,\nold3.m4s\n"; - let candidate = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:40\n#EXT-X-TARGETDURATION:4\n\ - #EXTINF:4.0,\nnew40.ts\n#EXTINF:4.0,\nnew41.ts\n#EXTINF:4.0,\nnew42.ts\n"; - - assert_incompatible_switch_is_rejected_before_timeline_commit( - baseline, - candidate, - HlsManifestRejectLogReason::SwitchMapResetUnsupported, - ) - .await; -} - -#[tokio::test] -async fn cross_host_switch_commits_only_after_complete_map_and_segment_staging() { - let temp_dir = tempfile::tempdir().expect("switch cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let origin = spawn_controlled_switch_origin().await; - let session = test_session(); - prepare_cross_host_baseline(&session).await; - let fetched = switch_fetched_manifest(&origin.base_url, true); - mark_full_burst_ready_for_switch_staging(&session, &fetched).await; - let effective_host = host_from_base_url(&origin.base_url); - let effective_host_id = crate::timeline::effective_origin_host_id(&effective_host); - let (baseline_epoch, baseline_proxy_next, staged_segment_key, staged_map_key) = { - let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = - tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( - &fetched.body, - &fetched.final_manifest_url, - ) - else { - panic!("switch manifest parses as normal timeline"); - }; - let session = session.read().await; - let preview = session.preview_origin_handoff_manifest(&manifest, effective_host_id, 0).expect("switch preview"); - ( - session.origin_epoch, - session.proxy_next_seq, - preview.segments.first().expect("first staged segment").cache_key.clone(), - preview.maps.first().expect("required staged map").cache_key.clone(), - ) - }; - let request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); - let cleanup_manager = Arc::clone(&request.hls_proxy); - let mut commit_task = tokio::spawn(async move { - super::commit_manifest_recovery_candidate( - &request, - fetched, - HlsManifestCommitAcceptanceMode::AllowHeldHostSwitchCandidate, - ) - .await - .map(|_| ()) - }); - - await_controlled_switch_segment_prefix(&origin, &mut commit_task, "cross-host switch commit").await; - - assert!(!commit_task.is_finished()); - { - let session = session.read().await; - assert_eq!(session.origin_epoch, baseline_epoch); - assert_eq!(session.proxy_next_seq, baseline_proxy_next); - assert!(session.segments.values().all(|segment| segment.origin_key.origin_epoch == baseline_epoch)); - } - assert_eq!(cache.metadata(&staged_segment_key).await.expect("segment metadata"), None); - assert_eq!(cache.metadata(&staged_map_key).await.expect("map metadata"), None); - assert_eq!( - *origin.requests.lock().await, - vec!["/live/final/init.mp4".to_string(), "/live/final/first.ts".to_string()] - ); - - origin.release_segment_body.notify_one(); - commit_task.await.expect("switch commit task").expect("switch commit succeeds"); - assert_eq!(cleanup_manager.cache_deletion_queue_usage(), (0, 0)); - - let (first_segment_key, first_map_key) = { - let session = session.read().await; - assert_eq!(session.origin_epoch, baseline_epoch.saturating_add(1)); - assert_eq!(session.origin_epoch_effective_host_id, Some(effective_host_id)); - let switched = session - .segments - .values() - .filter(|segment| segment.origin_key.origin_epoch == session.origin_epoch) - .collect::>(); - assert_eq!(switched.len(), 2); - assert_eq!(switched[0].origin_key.host_local_sequence, 500); - assert!(switched[0].discontinuity_before); - assert!(matches!( - switched[0].status, - SegmentCacheStatus::Ready { content_length, .. } - if content_length == u64::try_from(SWITCH_SEGMENT_BODY.len()).unwrap_or(u64::MAX) - )); - assert!(!matches!(switched[1].status, SegmentCacheStatus::Ready { .. })); - let map_id = switched[0].map_ref.expect("first switched segment requires map"); - assert_eq!(switched[1].map_ref, Some(map_id)); - let map = session.maps.get(&map_id).expect("staged map committed to timeline"); - assert!(matches!( - map.status, - MapCacheStatus::Ready { content_length, .. } - if content_length == u64::try_from(SWITCH_MAP_BODY.len()).unwrap_or(u64::MAX) - )); - (switched[0].cache_key.clone(), map.cache_key.clone()) - }; - let mut segment_file = cache.open_range(&first_segment_key, 0).await.expect("open staged segment"); - let mut segment_bytes = Vec::new(); - segment_file.read_to_end(&mut segment_bytes).await.expect("read staged segment"); - assert_eq!(segment_bytes, SWITCH_SEGMENT_BODY); - let mut map_file = cache.open_range(&first_map_key, 0).await.expect("open staged map"); - let mut map_bytes = Vec::new(); - map_file.read_to_end(&mut map_bytes).await.expect("read staged map"); - assert_eq!(map_bytes, SWITCH_MAP_BODY); -} - -#[tokio::test] -async fn caller_cancellation_after_owned_switch_staging_queues_and_collects_rollback() { - let temp_dir = tempfile::tempdir().expect("switch cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let origin = spawn_controlled_switch_origin().await; - let session = test_session(); - prepare_cross_host_baseline(&session).await; - let fetched = switch_fetched_manifest(&origin.base_url, true); - mark_full_burst_ready_for_switch_staging(&session, &fetched).await; - let effective_host_id = crate::timeline::effective_origin_host_id(&host_from_base_url(&origin.base_url)); - let (baseline_epoch, staged_segment_key, staged_map_key) = { - let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = - tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( - &fetched.body, - &fetched.final_manifest_url, - ) - else { - panic!("switch manifest parses as normal timeline"); - }; - let session = session.read().await; - let preview = session.preview_origin_handoff_manifest(&manifest, effective_host_id, 0).expect("switch preview"); - ( - session.origin_epoch, - preview.segments.first().expect("first staged segment").cache_key.clone(), - preview.maps.first().expect("required staged map").cache_key.clone(), - ) - }; - let request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); - let cleanup_manager = Arc::clone(&request.hls_proxy); - let (staging_complete_tx, staging_complete_rx) = tokio::sync::oneshot::channel(); - let (hold_tx, hold_rx) = tokio::sync::oneshot::channel::<()>(); - let mut staging_task = tokio::spawn(async move { - let staged = super::switch_staging::stage_alternative_manifest_switch(&request, &fetched) - .await - .expect("switch staging succeeds"); - staging_complete_tx.send(()).expect("staging completion receiver remains"); - let _ = hold_rx.await; - drop(staged); - }); - - await_controlled_switch_segment_prefix(&origin, &mut staging_task, "owned switch staging").await; - origin.release_segment_body.notify_one(); - staging_complete_rx.await.expect("owned cache commits complete"); - assert_eq!(cleanup_manager.cache_deletion_queue_usage(), (0, 2)); - assert!(cache.metadata(&staged_segment_key).await.expect("staged segment metadata").is_some()); - assert!(cache.metadata(&staged_map_key).await.expect("staged map metadata").is_some()); - - staging_task.abort(); - assert!(staging_task.await.expect_err("staging task is cancelled").is_cancelled()); - drop(hold_tx); - assert_eq!(cleanup_manager.cache_deletion_queue_usage(), (2, 0)); - assert_eq!(session.read().await.origin_epoch, baseline_epoch); - - let report = cleanup_manager.run_garbage_collection_once(200).await.expect("rollback GC succeeds"); - assert_eq!(report.cache_object_deletions_succeeded, 2); - assert_eq!(cleanup_manager.cache_deletion_queue_usage(), (0, 0)); - assert_eq!(cache.metadata(&staged_segment_key).await.expect("rolled-back segment metadata"), None); - assert_eq!(cache.metadata(&staged_map_key).await.expect("rolled-back map metadata"), None); -} - -#[derive(Clone, Copy)] -enum StaleSwitchGeneration { - Acceptance, - Progress, - PinnedHostRecovered, -} - -async fn assert_stale_switch_generation_rejects_commit(stale_generation: StaleSwitchGeneration) { - let temp_dir = tempfile::tempdir().expect("switch cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let origin = spawn_controlled_switch_origin().await; - let session = test_session(); - prepare_cross_host_baseline(&session).await; - let fetched = switch_fetched_manifest(&origin.base_url, true); - mark_full_burst_ready_for_switch_staging(&session, &fetched).await; - let effective_host_id = crate::timeline::effective_origin_host_id(&host_from_base_url(&origin.base_url)); - let (baseline_epoch, baseline_proxy_next, staged_segment_key, staged_map_key) = { - let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = - tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( - &fetched.body, - &fetched.final_manifest_url, - ) - else { - panic!("switch manifest parses as normal timeline"); - }; - let session = session.read().await; - let preview = session.preview_origin_handoff_manifest(&manifest, effective_host_id, 0).expect("switch preview"); - ( - session.origin_epoch, - session.proxy_next_seq, - preview.segments.first().expect("first staged segment").cache_key.clone(), - preview.maps.first().expect("staged map").cache_key.clone(), - ) - }; - let request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); - let cleanup_manager = Arc::clone(&request.hls_proxy); - let mut commit_task = tokio::spawn(async move { - super::commit_manifest_recovery_candidate( - &request, - fetched, - HlsManifestCommitAcceptanceMode::AllowHeldHostSwitchCandidate, - ) - .await - .map(|_| ()) - }); - - await_controlled_switch_segment_prefix(&origin, &mut commit_task, "stale-generation switch commit").await; - { - let mut session = session.write().await; - match stale_generation { - StaleSwitchGeneration::Acceptance => { - let episode = session.origin_control.acceptance_episode.as_mut().expect("active acceptance episode"); - episode.generation = super::super::manifest_acceptance::HlsManifestAcceptanceGeneration( - episode.generation.0.saturating_add(1), - ); - } - StaleSwitchGeneration::Progress => { - session.origin_control.progress_generation = - session.origin_control.progress_generation.saturating_add(1); - } - StaleSwitchGeneration::PinnedHostRecovered => { - session.origin_control.pinned_host = Some("recovered.example.com".to_string()); - session.last_effective_manifest_host = Some("recovered.example.com".to_string()); - } - } - } - origin.release_segment_body.notify_one(); - - let result = commit_task.await.expect("stale switch task"); - assert!(matches!( - result, - Err(HlsManifestCommitError::TimelineRejected { reason: HlsManifestRejectLogReason::StagedSwitchInvalidated }) - )); - { - let session = session.read().await; - assert_eq!(session.origin_epoch, baseline_epoch); - assert_eq!(session.proxy_next_seq, baseline_proxy_next); - assert!(session.segments.values().all(|segment| segment.origin_key.origin_epoch == baseline_epoch)); - } - assert_eq!(cache.metadata(&staged_segment_key).await.expect("stale segment metadata"), None); - assert_eq!(cache.metadata(&staged_map_key).await.expect("stale map metadata"), None); - assert_eq!(cleanup_manager.cache_deletion_queue_usage(), (0, 0)); - assert!(!cache.has_active_temp_files()); -} - -#[tokio::test] -async fn stale_acceptance_generation_rejects_and_removes_completed_switch_staging() { - assert_stale_switch_generation_rejects_commit(StaleSwitchGeneration::Acceptance).await; -} - -#[tokio::test] -async fn stale_progress_generation_rejects_and_removes_completed_switch_staging() { - assert_stale_switch_generation_rejects_commit(StaleSwitchGeneration::Progress).await; -} - -#[tokio::test] -async fn pinned_origin_recovery_during_staging_rejects_alternative_commit() { - assert_stale_switch_generation_rejects_commit(StaleSwitchGeneration::PinnedHostRecovered).await; -} - -#[tokio::test] -async fn switch_segment_staging_failure_holds_episode_without_timeline_mutation() { - let manifest = switch_manifest_body(false); - let origin = spawn_test_origin(Arc::new(move |path| { - if path_has_extension(&path, "m3u8") { - (200, Vec::new(), manifest.clone()) - } else { - (404, Vec::new(), String::new()) - } - })) - .await; - let temp_dir = tempfile::tempdir().expect("switch cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let session = test_session(); - prepare_cross_host_baseline(&session).await; - let (baseline_epoch, baseline_proxy_next, baseline_sequences) = { - let session = session.read().await; - (session.origin_epoch, session.proxy_next_seq, session.segments.keys().copied().collect::>()) - }; - let request = switch_test_request(Arc::clone(&session), cache, &origin.base_url); - let target_url = Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("target url"); - - let result = retry_test_manifest_recovery_chain( - &request, - target_url, - HlsManifestRejectLogReason::PinnedHostRecoveryRejected, - ) - .await; - - assert!(matches!(result, Err(OriginManifestFetchError::RetryExhausted))); - let session = session.read().await; - assert_eq!(session.origin_epoch, baseline_epoch); - assert_eq!(session.proxy_next_seq, baseline_proxy_next); - assert_eq!(session.segments.keys().copied().collect::>(), baseline_sequences); - let episode = session.origin_control.acceptance_episode.as_ref().expect("failed switch episode retained"); - assert!(episode.full_burst_completed); - assert_eq!(episode.full_bursts_completed, 1); - assert_eq!(episode.state, super::super::manifest_acceptance::HlsManifestAcceptanceState::Holding); - drop(session); - let requests = origin.requests.lock().await; - assert_eq!(requests.iter().filter(|path| path_has_extension(path, "ts")).count(), 1); -} - -#[tokio::test] -async fn recovery_required_orchestrator_requalifies_changed_pinned_landscape_with_full_plan() { - let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); - let entry_hits = Arc::new(AtomicUsize::new(0)); - let entry_hits_for_handler = Arc::clone(&entry_hits); - let pinned_hits = Arc::new(AtomicUsize::new(0)); - let pinned_hits_for_handler = Arc::clone(&pinned_hits); - let origin_port = Arc::new(AtomicUsize::new(0)); - let origin_port_for_handler = Arc::clone(&origin_port); - let origin = spawn_test_origin(Arc::new(move |path| { - if path == "/live/user/pass/12345.m3u8" { - let hit = entry_hits_for_handler.fetch_add(1, Ordering::SeqCst); - if hit < plan.total_candidates() { - return (200, Vec::new(), switch_manifest_body(false)); - } - return ( - 302, - vec![( - "Location", - format!("http://127.0.0.1:{}/pinned/index.m3u8", origin_port_for_handler.load(Ordering::SeqCst)), - )], - String::new(), - ); - } - if path == "/pinned/index.m3u8" { - pinned_hits_for_handler.fetch_add(1, Ordering::SeqCst); - return ( - 200, - Vec::new(), - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\npinned101.ts\n".to_string(), - ); - } - (404, Vec::new(), String::new()) - })) - .await; - let parsed_origin_url = Url::parse(&origin.base_url).expect("origin base URL"); - origin_port.store(usize::from(parsed_origin_url.port().expect("test origin port")), Ordering::SeqCst); - - let session = test_session(); - let baseline = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\npinned100.ts\n"; - let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(baseline) = - tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( - baseline, - "http://127.0.0.1/pinned/index.m3u8", - ) - else { - panic!("pinned baseline parses"); - }; - { - let mut session = session.write().await; - session - .apply_origin_manifest_for_host(&baseline, crate::timeline::effective_origin_host_id("127.0.0.1")) - .expect("pinned baseline commits"); - session.last_effective_manifest_host = Some("127.0.0.1".to_string()); - session.origin_control.pinned_host = Some("127.0.0.1".to_string()); - session.origin_control.origin_epoch = session.origin_epoch; - } - - let temp_dir = tempfile::tempdir().expect("orchestrator cache tempdir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let mut request = switch_test_request(Arc::clone(&session), cache, &origin.base_url); - let alternative_entry = - format!("{}/live/user/pass/12345.m3u8", origin.base_url).replacen("127.0.0.1", "localhost", 1); - request.origin_entry = LiveHlsOriginEntry::parse(&alternative_entry).expect("alternative entry URL"); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - let target_url = Url::parse(&alternative_entry).expect("alternative target URL"); - - let committed = retry_test_manifest_recovery_chain( - &request, - target_url, - HlsManifestRejectLogReason::PinnedHostRecoveryRejected, - ) - .await - .expect("pinned follow-up commits"); - - assert_eq!(committed.fetched.redirect_host.as_deref(), Some("127.0.0.1")); - assert_eq!(entry_hits.load(Ordering::SeqCst), plan.total_candidates().saturating_mul(2).saturating_add(1)); - assert_eq!(pinned_hits.load(Ordering::SeqCst), plan.total_candidates().saturating_add(1)); - let session = session.read().await; - assert_eq!(session.origin_seq_highwater, Some(101)); - assert_eq!(session.last_effective_manifest_host.as_deref(), Some("127.0.0.1")); - let episode = session.origin_control.acceptance_episode.as_ref().expect("completed acceptance episode"); - assert_eq!(episode.full_bursts_completed, 1); - assert_eq!(episode.state, super::super::manifest_acceptance::HlsManifestAcceptanceState::Completed); -} - -#[tokio::test] -async fn provider_preflight_failure_real_refresh_path_has_zero_candidates() { - let origin = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; - let hls_ctx = crate::HlsCtx::for_test(Config::default()); - let ctx = &hls_ctx; - let session = test_session(); - let proxy_session_id = session.read().await.proxy_session_id.clone(); - session.write().await.origin_account_binding = Some(HlsOriginAccountBinding::new( - Arc::from("missing-input"), - Arc::from("missing-account"), - &proxy_session_id, - 0, - )); - let mut request = bind_refresh_request_to_app_state(test_origin_refresh_request(Arc::clone(&session)), ctx); - request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; - request.origin_entry = - LiveHlsOriginEntry::parse(&format!("{}/live/index.m3u8", origin.base_url)).expect("unused origin URL"); - request.origin_io = Some(HlsOriginIoContext { - ctx: ctx.clone(), - client_addr: "127.0.0.1:12345".parse().expect("test client address"), - allow_grace: false, - priority: 0, - connection_kind: ConnectionKind::Normal, - reservation_ttl_secs: 60, - preacquired_provider_handle: None, - started_generation: None, - }); - let metrics = Arc::clone(request.segment_worker_pool.metrics()); - let error = super::provider_preflight_manifest_error(HlsBoundAccountAcquireErrorKind::Missing); - - assert!(matches!(&error, OriginManifestFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Missing))); - assert!(trigger_origin_refresh_sync(request).await); - assert!(origin.requests.lock().await.is_empty()); - - let session = session.read().await; - assert!(session.origin_control.acceptance_episode.is_none()); - assert_eq!(session.origin_control.progress_phase, super::super::origin_progress::HlsOriginProgressPhase::Cold); - assert_eq!( - session.origin_control.path_condition, - super::super::origin_progress::HlsOriginPathCondition::HardFetchFailure - ); - assert!(session.origin_control.last_origin_response_at_ms.is_none()); - assert!(session.origin_control.manifest_origin_binding.is_none()); - assert!(session.last_rendered_manifest.is_none()); - assert!(session.published_live_origin_baseline.is_none()); - assert!(session.established_manifest_recovery_binding().is_none()); - drop(session); - let metrics = metrics.snapshot(); - assert_eq!(metrics.refresh_started, 1); - assert_eq!(metrics.refresh_completed, 0); - assert_eq!(metrics.refresh_retried, 0); - assert_eq!(metrics.refresh_skipped, 0); - assert_eq!(metrics.refresh_failed, 1); -} diff --git a/backend/hls/src/refresh/tests/acceptance.rs b/backend/hls/src/refresh/tests/acceptance.rs new file mode 100644 index 000000000..8751977c8 --- /dev/null +++ b/backend/hls/src/refresh/tests/acceptance.rs @@ -0,0 +1,909 @@ +use super::{ + apply_manifest_fetch_failure_signal, commit_fetched_manifest, critical_handoff_app_config, + fetch_and_commit_manifest_with_policy, fetched_manifest, host_from_base_url, manifest_fetch_context, + mark_origin_refresh_started_with_outcome, no_delay_policy, prepare_active_critical_handoff_lease, + prepare_cross_host_baseline, publish_ready_test_manifest, retry_hls_origin_manifest_recovery_chain, + spawn_critical_emergency_origin, spawn_test_origin, switch_test_request, test_acceptance_episode_timing, + test_deterministic_timeline_conflict, test_manifest_origin_binding, test_origin_refresh_request, + test_recovery_timing_policy, test_session, three_segment_manifest_body, trigger_origin_refresh_sync, + CriticalEmergencyOriginServer, HlsManifestAcceptanceTrigger, HlsManifestCommitAcceptanceMode, + HlsManifestCommitRequirement, HlsManifestFetchSelection, HlsManifestOriginBinding, HlsManifestRejectLogReason, + HlsOriginRefreshTriggerOutcome, HlsPostRefreshAvailabilityAction, HlsPostRefreshAvailabilityReason, + HlsRecoveryEtaMs, LiveHlsOriginEntry, OriginManifestFetchError, OriginRefreshRequest, TestOriginServer, + CRITICAL_HANDOFF_MANIFEST_BODY, CRITICAL_HANDOFF_TS_BODY, +}; +use crate::{ + build_rewrite_secret_fingerprint, media_reserve::HlsLeaseReserveAvailabilityBasis, + timeline::HLS_PROVISIONING_ORIGIN_EPOCH, CacheAccessState, GarbageCollectionPolicy, HlsAccessLease, + HlsFreshManifestRequiredReason, HlsGarbageCollector, HlsManifestAcceptanceExhaustionReason, HlsProxyManager, + HlsSegmentCache, HlsSession, HlsSessionKey, HlsSessionStore, OriginSegmentKey, RenderedManifest, + RenderedManifestStoreOutcome, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, +}; +use axum::http::StatusCode; +use shared::model::{ConfigProviderDto, HlsManifestRecoveryBurstLevel, ProviderUrlSelectionPolicy}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; +use tokio::sync::RwLock; +use tuliprox_core::model::{ConfigProvider, HlsManifestRecoveryBurstConfig}; +use url::Url; + +#[test] +fn deterministic_conflict_returns_generation_bound_post_refresh_action() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.origin_control.progress_generation = 7; + session.activity.media_readiness_generation = 11; + + let action = apply_manifest_fetch_failure_signal( + &mut session, + &OriginManifestFetchError::DeterministicTimelineConflict(Box::new(test_deterministic_timeline_conflict())), + 100, + ); + + assert_eq!( + action, + HlsPostRefreshAvailabilityAction::Reevaluate { + reason: HlsPostRefreshAvailabilityReason::DeterministicTimelineConflict, + origin_progress_generation: 7, + media_readiness_generation: 11, + } + ); + assert_eq!(session.origin_control.last_origin_response_at_ms, Some(100)); + assert_eq!( + session.origin_control.path_condition, + super::super::super::origin_progress::HlsOriginPathCondition::AcceptanceConflict + ); + assert!(session.fresh_manifest_commit_required.is_none()); +} + +#[test] +fn retry_exhaustion_preserves_qualified_acceptance_conflict_but_not_all_failed_transport_state() { + for (reason, expected) in [ + ( + HlsManifestAcceptanceExhaustionReason::NoCommittableCandidate, + super::super::super::origin_progress::HlsOriginPathCondition::AcceptanceConflict, + ), + ( + HlsManifestAcceptanceExhaustionReason::AllFailed, + super::super::super::origin_progress::HlsOriginPathCondition::RetryableFetchFailure, + ), + ] { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let burst_plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); + session.origin_control.begin_acceptance_episode( + 100, + burst_plan, + HlsManifestAcceptanceTrigger::Observe, + &test_acceptance_episode_timing(100, burst_plan), + ); + let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); + episode.record_full_burst(); + episode.record_exhaustion(reason); + episode.hold_after_uncommitted_burst(None, None); + session.origin_control.path_condition = + super::super::super::origin_progress::HlsOriginPathCondition::AcceptanceConflict; + + apply_manifest_fetch_failure_signal(&mut session, &OriginManifestFetchError::RetryExhausted, 200); + + assert_eq!(session.origin_control.path_condition, expected); + } +} + +pub(in crate::refresh::tests) async fn spawn_rotating_parent_origin() -> TestOriginServer { + let rotating_parent = Arc::new(AtomicUsize::new(0)); + let handler_counter = Arc::clone(&rotating_parent); + spawn_test_origin(Arc::new(move |_path| { + let parent = handler_counter.fetch_add(1, Ordering::AcqRel).saturating_add(1); + ( + 200, + Vec::new(), + format!( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:490\n\ + #EXTINF:4,\n/stream/{parent:016x}/1745190_490.ts\n\ + #EXTINF:4,\n/stream/{parent:016x}/1745180_480.ts\n\ + #EXTINF:4,\n/stream/{parent:016x}/1745191_491.ts\n" + ), + ) + })) + .await +} + +#[tokio::test] +async fn rotating_volatile_parent_proves_one_deterministic_conflict() { + let baseline_body = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:480\n\ + #EXTINF:4,\n/stream/aaaaaaaaaaaaaaaa/1745180_480.ts\n\ + #EXTINF:4,\n/stream/aaaaaaaaaaaaaaaa/1745181_481.ts\n\ + #EXTINF:4,\n/stream/aaaaaaaaaaaaaaaa/1745182_482.ts\n"; + let server = spawn_rotating_parent_origin().await; + let session = test_session(); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + let manifest_url = format!("{}/live/user/pass/12345.m3u8", server.base_url); + request.origin_entry = LiveHlsOriginEntry::parse(&manifest_url).expect("local replay origin entry"); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + let plan = request.manifest_recovery_burst.level.plan(); + let mut baseline = fetched_manifest(baseline_body); + baseline.final_manifest_url = manifest_url.clone(); + baseline.resolved_request_url = manifest_url; + baseline.redirect_host = Some("127.0.0.1".to_string()); + { + let mut session = session.write().await; + commit_fetched_manifest(&mut session, &baseline, &request, 100).expect("baseline commits"); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 101 }; + } + session.render_and_store_manifest(101).expect("baseline publishes"); + assert_eq!(session.published_resource_history.generation(), 3); + } + + let Err(first) = fetch_and_commit_manifest_with_policy(&mut request).await else { + panic!("one complete candidate landscape must prove the replay conflict"); + }; + let OriginManifestFetchError::DeterministicTimelineConflict(first_conflict) = first else { + panic!("deterministic replay conflict must not collapse to retry exhaustion"); + }; + assert_eq!(first_conflict.previous_proxy_tail, Some(2)); + assert_eq!(first_conflict.existing_proxy_seq, 0); + assert_eq!(first_conflict.candidate_position, 1); + assert_eq!(first_conflict.candidate_origin_seq, 491); + assert_eq!(server.requests.lock().await.len(), plan.total_candidates().saturating_add(1)); + { + let session = session.read().await; + let episode = session.origin_control.acceptance_episode.as_ref().expect("acceptance episode retained"); + assert_eq!(episode.full_bursts_completed, 1); + assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); + assert!(episode.deterministic_conflict_receipt().is_some()); + assert!(matches!( + super::super::super::manifest_acceptance::manifest_acceptance_episode_status( + Some(episode), + episode.generation, + request.now_ms, + ), + super::super::super::manifest_acceptance::HlsManifestAcceptanceEpisodeStatus::FullBurstExhausted { + reason: HlsManifestAcceptanceExhaustionReason::DeterministicTimelineConflict, + .. + } + )); + assert_eq!(session.proxy_next_seq, Some(3), "replayed content receives no proxy sequence"); + } + + let request_count_before_receipt_sample = server.requests.lock().await.len(); + request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::Observe; + let Err(repeated) = fetch_and_commit_manifest_with_policy(&mut request).await else { + panic!("an unchanged ordinary sample must remain rejected"); + }; + assert!(matches!( + repeated, + OriginManifestFetchError::DeterministicTimelineConflict(ref conflict) + if conflict.as_ref() == first_conflict.as_ref() + )); + assert_eq!( + server.requests.lock().await.len(), + request_count_before_receipt_sample.saturating_add(1), + "an unchanged receipt permits one ordinary request but no second burst" + ); + + { + let mut session = session.write().await; + session.published_resource_history.record( + super::super::super::resource_identity::HlsMediaResourceIdentity::from_url( + "http://127.0.0.1/live/user/pass/unrelated-published.ts", + None, + ), + 99, + ); + } + let request_count_before_changed_landscape = server.requests.lock().await.len(); + let Err(changed) = fetch_and_commit_manifest_with_policy(&mut request).await else { + panic!("changed published evidence must still reject the replay"); + }; + assert!(matches!(changed, OriginManifestFetchError::DeterministicTimelineConflict(_))); + assert_eq!( + server.requests.lock().await.len(), + request_count_before_changed_landscape.saturating_add(plan.total_candidates()), + "changed receipt evidence authorizes exactly one newly configured full burst" + ); + let session = session.read().await; + assert_eq!(session.proxy_next_seq, Some(3), "the replay stays blocked after reevaluation"); + assert!(!session.segments.contains_key(&3)); +} + +#[tokio::test] +async fn hls_manifest_acceptance_directive_stale_guard_prevents_refresh_start() { + let hls_proxy = Arc::new(HlsProxyManager::new()); + let (session, _) = hls_proxy + .get_or_create_session_with_outcome(HlsSessionKey::new(1, "stale-pressure-guard"), b"secret", 100) + .await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let owner_key = hls_proxy + .availability_reevaluation_owner_key(&session, &proxy_session_id) + .await + .expect("registered session has recovery-pressure evidence"); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.hls_proxy = Arc::clone(&hls_proxy); + request.acceptance_directive.recovery_pressure_guard = + Some(super::super::super::availability_reevaluation::HlsRecoveryPressureGuard::from_owner_key(&owner_key)); + session.write().await.origin_control.progress_generation = owner_key.origin_progress_generation.saturating_add(1); + + assert_eq!( + mark_origin_refresh_started_with_outcome(&mut request, 100).await, + HlsOriginRefreshTriggerOutcome::RecoveryPressureSuperseded + ); + let session = session.read().await; + assert!(!session.origin_refresh.in_flight); + assert!(session.origin_control.acceptance_episode.is_none()); +} + +pub(in crate::refresh::tests) fn install_stored_provisioning_manifest(session: &mut HlsSession, rendered_at_ms: u64) { + let segment_proxy_seqs = (0_u64..3).collect::>(); + for proxy_seq in &segment_proxy_seqs { + session.segments.insert( + *proxy_seq, + SegmentEntry { + origin_key: OriginSegmentKey { + origin_epoch: HLS_PROVISIONING_ORIGIN_EPOCH, + effective_host_id: 0, + host_local_sequence: *proxy_seq, + host_local_index: u32::try_from(*proxy_seq).unwrap_or(u32::MAX), + }, + proxy_seq: *proxy_seq, + duration_ms: 2_000, + proxy_file_ext: "ts".to_string(), + content_type: "video/mp2t".to_string(), + cache_key: SegmentCacheKey::new(session.proxy_session_id.clone(), *proxy_seq, "ts"), + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + encryption: None, + origin_fetch_ref: None, + status: SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }, + last_rendered_at_ms: None, + access: Arc::new(CacheAccessState::new()), + }, + ); + } + session.publishable_origin_head_proxy_seq = segment_proxy_seqs.first().copied(); + session.publishable_origin_tail_proxy_seq = segment_proxy_seqs.last().copied(); + session.proxy_next_seq = Some(3); + session.target_duration = Some(2); + assert_eq!( + session.store_rendered_manifest(RenderedManifest { + body: "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n".to_string(), + first_proxy_seq: 0, + last_proxy_seq: 2, + discontinuity_sequence: 0, + target_duration_ms: 2_000, + playlist_duration_ms: 6_000, + valid_until_ms: rendered_at_ms.saturating_add(6_000), + render_gap_segments: 0, + rendered_at_ms, + segment_proxy_seqs, + }), + RenderedManifestStoreOutcome::Stored + ); + assert!(session.published_live_origin_baseline.is_none()); +} + +#[tokio::test] +async fn warm_retryable_initial_failure_does_not_open_acceptance_episode() { + let server = spawn_test_origin(Arc::new(|_path| (500, Vec::new(), "retry".to_string()))).await; + let session = test_session(); + { + let mut session = session.write().await; + session.origin_seq_highwater = Some(10); + session.origin_control.record_media_progress(50, 4_000); + } + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + + assert!(trigger_origin_refresh_sync(request.clone()).await); + + let manifest_requests = + server.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(manifest_requests, request.retry_policy.attempt_count()); + assert!(session.read().await.origin_control.acceptance_episode.is_none()); +} + +#[tokio::test] +async fn initial_provider_failover_does_not_leak_into_established_burst() { + let phase = Arc::new(AtomicUsize::new(0)); + let phase_for_first = Arc::clone(&phase); + let first = spawn_test_origin(Arc::new(move |_path| { + if phase_for_first.load(Ordering::SeqCst) == 0 { + (200, Vec::new(), three_segment_manifest_body(100)) + } else { + (407, Vec::new(), "rotate".to_string()) + } + })) + .await; + let phase_for_second = Arc::clone(&phase); + let second = spawn_test_origin(Arc::new(move |_path| { + if phase_for_second.load(Ordering::SeqCst) == 0 { + (500, Vec::new(), "unexpected".to_string()) + } else { + (461, Vec::new(), "final-hard-failure".to_string()) + } + })) + .await; + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + })); + let session = test_session(); + let entry = LiveHlsOriginEntry::parse_with_url_failover_provider( + "provider://demo/live/user/pass/12345.m3u8?token=fixed", + Some(Arc::clone(&provider)), + ) + .expect("provider entry url"); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = entry; + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Friendly }; + + fetch_and_commit_manifest_with_policy(&mut request).await.expect("initial provider URL commits baseline"); + publish_ready_test_manifest(&session, 200).await; + let binding = + session.read().await.established_manifest_recovery_binding().expect("published baseline has fixed binding"); + assert_eq!(binding.provider_url_index(), Some(0)); + let provider_index_before_failure = provider.get_current_index(); + + phase.store(1, Ordering::SeqCst); + let Err(error) = fetch_and_commit_manifest_with_policy(&mut request).await else { + panic!("hard initial provider cycle and fixed-target recovery must remain failed"); + }; + assert!(matches!(error, OriginManifestFetchError::RetryableStatus(StatusCode::PROXY_AUTHENTICATION_REQUIRED, _))); + + let plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); + let first_requests = first.requests.lock().await; + let second_requests = second.requests.lock().await; + assert!( + first_requests.len() >= 2_usize.saturating_add(plan.total_candidates()), + "baseline, ordinary provider attempt, and at least one full fixed-target burst are required" + ); + assert_eq!(second_requests.len(), 1, "only the ordinary provider cycle may reach URL index 1"); + assert!(first_requests.iter().all(|path| path == "/live/user/pass/12345.m3u8?token=fixed")); + assert_eq!(second_requests[0], "/live/user/pass/12345.m3u8?token=fixed"); + assert_eq!( + provider.get_current_index(), + provider_index_before_failure, + "fixed-target recovery must not rotate provider state" + ); + assert_eq!( + session + .read() + .await + .origin_control + .manifest_origin_binding + .as_ref() + .map(HlsManifestOriginBinding::provider_url_index), + Some(Some(0)) + ); +} + +#[tokio::test] +async fn expired_acceptance_deadline_still_completes_first_beast_burst_without_follow_up() { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let origin = spawn_test_origin(Arc::new(|_path| (407, Vec::new(), "retry".to_string()))).await; + let session = test_session(); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)) + .expect("deadline test origin entry"); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + let target_url = request.origin_entry.url().clone(); + let mut context = manifest_fetch_context(&request); + let mut expired_after_full_burst = test_recovery_timing_policy(0); + expired_after_full_burst.evaluation_eta = HlsRecoveryEtaMs::default(); + expired_after_full_burst.commit_eta = HlsRecoveryEtaMs::default(); + expired_after_full_burst.scheduling_eta = HlsRecoveryEtaMs::default(); + context.recovery_timing_policy = expired_after_full_burst; + + let result = retry_hls_origin_manifest_recovery_chain( + &context, + test_manifest_origin_binding(target_url), + Some(HlsManifestRejectLogReason::PinnedHostRecoveryRejected), + None, + HlsManifestAcceptanceTrigger::RecoveryRequired, + HlsManifestCommitAcceptanceMode::StrictPinnedHost, + |fetched, acceptance_mode| super::super::commit_manifest_recovery_candidate(&request, fetched, acceptance_mode), + ) + .await; + + assert!(result.is_err()); + let manifest_requests = + origin.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(manifest_requests, plan.total_candidates()); + let session = session.read().await; + let episode = session.origin_control.acceptance_episode.as_ref().expect("completed first acceptance burst"); + assert!(episode.full_burst_completed); + assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); + assert_eq!(episode.full_bursts_completed, 1); +} + +#[tokio::test] +async fn cold_start_hard_initial_failure_does_not_start_acceptance_burst() { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let server = spawn_test_origin(Arc::new(|_path| (404, Vec::new(), "missing".to_string()))).await; + let session = test_session(); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + request.manifest_commit_requirement = + HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; + let metrics = Arc::clone(request.segment_worker_pool.metrics()); + + assert!(trigger_origin_refresh_sync(request).await); + + let manifest_requests = + server.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(manifest_requests, 1); + assert_ne!(manifest_requests, 1_usize.saturating_add(plan.total_candidates())); + let session = session.read().await; + assert!(session.origin_control.acceptance_episode.is_none()); + assert_eq!( + session.origin_control.progress_phase, + super::super::super::origin_progress::HlsOriginProgressPhase::Cold + ); + assert_eq!( + session.origin_control.path_condition, + super::super::super::origin_progress::HlsOriginPathCondition::HardFetchFailure + ); + let metrics = metrics.snapshot(); + assert_eq!(metrics.refresh_failed, 1); + assert_eq!(metrics.refresh_completed, 0); +} + +#[tokio::test] +async fn cold_provider_failover_cycle_does_not_start_acceptance_burst() { + let first = spawn_test_origin(Arc::new(|_path| (500, Vec::new(), "rotate".to_string()))).await; + let second = spawn_test_origin(Arc::new(|_path| (461, Vec::new(), "final-hard-failure".to_string()))).await; + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + })); + let session = test_session(); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = LiveHlsOriginEntry::parse_with_url_failover_provider( + "provider://demo/live/user/pass/12345.m3u8", + Some(provider), + ) + .expect("provider entry URL"); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + request.manifest_commit_requirement = + HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; + + let Err(error) = fetch_and_commit_manifest_with_policy(&mut request).await else { + panic!("cold provider cycle must preserve its final hard response"); + }; + + assert!(matches!( + error, + OriginManifestFetchError::NonRetryableStatus(status) if status.as_u16() == 461 + )); + assert_eq!(first.requests.lock().await.len(), 1); + assert_eq!(second.requests.lock().await.len(), 1); + let session = session.read().await; + assert!(session.origin_control.acceptance_episode.is_none()); + assert!(session.origin_control.manifest_origin_binding.is_none()); + assert_eq!( + session.origin_control.progress_phase, + super::super::super::origin_progress::HlsOriginProgressPhase::Cold + ); +} + +#[tokio::test] +async fn successful_origin_commit_without_renderable_origin_window_does_not_enable_burst() { + let manifest_hits = Arc::new(AtomicUsize::new(0)); + let manifest_hits_for_handler = Arc::clone(&manifest_hits); + let server = spawn_test_origin(Arc::new(move |_path| { + let media_sequence = if manifest_hits_for_handler.fetch_add(1, Ordering::SeqCst) == 0 { 100 } else { 103 }; + (200, Vec::new(), three_segment_manifest_body(media_sequence)) + })) + .await; + let session = test_session(); + { + let mut session = session.write().await; + install_stored_provisioning_manifest(&mut session, 10); + } + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry URL"); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + + let baseline = fetch_and_commit_manifest_with_policy(&mut request).await.expect("ordinary origin baseline commits"); + assert_eq!(baseline.fetched.selection, HlsManifestFetchSelection::Initial); + assert_eq!(baseline.fetched.candidate_requests, 1); + { + let session = session.read().await; + assert!(session.origin_control.manifest_origin_binding.is_some()); + assert!(session.origin_control.pinned_host.is_some()); + assert_eq!(session.origin_seq_highwater, Some(102)); + assert_eq!( + session.origin_control.progress_phase, + super::super::super::origin_progress::HlsOriginProgressPhase::Fresh + ); + assert_eq!( + session.last_rendered_manifest.as_ref().expect("provisioning manifest remains stored").segment_proxy_seqs, + vec![0, 1, 2] + ); + assert!(session + .segments + .values() + .filter(|segment| segment.origin_key.origin_epoch != HLS_PROVISIONING_ORIGIN_EPOCH) + .all(|segment| !matches!(segment.status, SegmentCacheStatus::Ready { .. }))); + assert!(session.published_live_origin_baseline.is_none()); + assert!(session.established_manifest_recovery_binding().is_none()); + assert!(session.origin_control.acceptance_episode.is_none()); + } + + request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; + let follow_up = + fetch_and_commit_manifest_with_policy(&mut request).await.expect("suppressed recovery uses ordinary fetch"); + + assert_eq!(follow_up.fetched.selection, HlsManifestFetchSelection::Initial); + assert_eq!(follow_up.fetched.candidate_requests, 1); + assert_eq!(manifest_hits.load(Ordering::SeqCst), 2); + assert_ne!(manifest_hits.load(Ordering::SeqCst), HlsManifestRecoveryBurstLevel::Beast.plan().total_candidates()); + let session = session.read().await; + assert!(session.published_live_origin_baseline.is_none()); + assert!(session.established_manifest_recovery_binding().is_none()); + assert!(session.origin_control.acceptance_episode.is_none()); +} + +#[tokio::test] +async fn material_reduced_retry_timeline_change_starts_one_new_complete_configured_burst() { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let origin = spawn_test_origin(Arc::new(move |_path| { + let hit = hits_for_handler.fetch_add(1, Ordering::SeqCst); + let resource = if hit < plan.total_candidates() { "old900.ts" } else { "new900.ts" }; + ( + 200, + Vec::new(), + format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n{resource}\n"), + ) + })) + .await; + let session = test_session(); + prepare_cross_host_baseline(&session).await; + let cache_dir = tempfile::tempdir().expect("requalification cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(cache_dir.path())); + let mut request = switch_test_request(Arc::clone(&session), cache, &origin.base_url); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + let target_url = + Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("requalification target"); + let attempts = request.retry_policy.attempt_count(); + let context = manifest_fetch_context(&request); + + let result = retry_hls_origin_manifest_recovery_chain( + &context, + test_manifest_origin_binding(target_url), + Some(HlsManifestRejectLogReason::PinnedHostRecoveryRejected), + None, + HlsManifestAcceptanceTrigger::Observe, + HlsManifestCommitAcceptanceMode::StrictPinnedHost, + |fetched, acceptance_mode| super::super::commit_manifest_recovery_candidate(&request, fetched, acceptance_mode), + ) + .await; + + assert!(result.is_err()); + let reduced_attempts = attempts.saturating_sub(2); + assert_eq!(hits.load(Ordering::SeqCst), plan.total_candidates().saturating_mul(2).saturating_add(reduced_attempts)); + let session = session.read().await; + let episode = session.origin_control.acceptance_episode.as_ref().expect("requalified episode"); + assert!(episode.generation.0 >= 2); + assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); + assert!(episode.full_burst_completed); + assert_eq!(episode.full_bursts_completed, 1); +} + +#[tokio::test] +async fn material_change_in_last_reduced_slot_still_runs_new_episode_full_burst() { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let retry_policy = no_delay_policy(); + let attempts = retry_policy.attempt_count(); + let unchanged_candidate_count = plan.total_candidates().saturating_add(attempts.saturating_sub(2)); + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let origin = spawn_test_origin(Arc::new(move |_path| { + let hit = hits_for_handler.fetch_add(1, Ordering::SeqCst); + let resource = if hit < unchanged_candidate_count { "old900.ts" } else { "new900.ts" }; + ( + 200, + Vec::new(), + format!("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n{resource}\n"), + ) + })) + .await; + let session = test_session(); + prepare_cross_host_baseline(&session).await; + let cache_dir = tempfile::tempdir().expect("last-slot requalification cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(cache_dir.path())); + let mut request = switch_test_request(Arc::clone(&session), cache, &origin.base_url); + request.retry_policy = retry_policy; + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + let target_url = Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)) + .expect("last-slot requalification target"); + let context = manifest_fetch_context(&request); + + let result = retry_hls_origin_manifest_recovery_chain( + &context, + test_manifest_origin_binding(target_url), + Some(HlsManifestRejectLogReason::PinnedHostRecoveryRejected), + None, + HlsManifestAcceptanceTrigger::Observe, + HlsManifestCommitAcceptanceMode::StrictPinnedHost, + |fetched, acceptance_mode| super::super::commit_manifest_recovery_candidate(&request, fetched, acceptance_mode), + ) + .await; + + assert!(result.is_err()); + assert_eq!( + hits.load(Ordering::SeqCst), + plan.total_candidates().saturating_mul(2).saturating_add(attempts.saturating_sub(1)) + ); + let session = session.read().await; + let episode = session.origin_control.acceptance_episode.as_ref().expect("last-slot requalified episode"); + assert!(episode.generation.0 >= 2); + assert_eq!(episode.completed_burst_candidates, plan.total_candidates()); + assert!(episode.full_burst_completed); + assert_eq!(episode.full_bursts_completed, 1); +} + +pub(in crate::refresh::tests) async fn assert_critical_handoff_evidence( + request: &OriginRefreshRequest, + live_lease: &HlsAccessLease, + now_ms: u64, +) { + let manifest = live_lease.last_manifest_snapshot.as_ref().expect("authoritative manifest snapshot"); + assert_eq!(manifest.snapshot_generation, 1); + let ready_timeline = { + let session = request.session.read().await; + session.ready_timeline_snapshot( + live_lease.playback_cursor.ready_timeline_start_proxy_seq(manifest.first_proxy_seq), + now_ms, + ) + }; + let reserve = super::super::super::evaluate_lease_reserve(super::super::super::HlsLeaseReserveInput { + manifest, + cursor: &live_lease.playback_cursor, + ready_timeline: &ready_timeline, + now_ms, + playback_rate_guard_milli: super::super::super::HLS_PLAYBACK_RATE_GUARD_MILLI, + recovery_trigger_budget: super::super::super::recovery_timing::HlsRecoveryTriggerBudgetMs::from_millis(0), + origin_path_degraded: true, + recovery_committed: false, + }); + assert_eq!(reserve.availability_basis, HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline); + assert_eq!(reserve.guaranteed_media_horizon_ms, manifest.last_visible_media_end_ms); + assert_eq!(reserve.guaranteed_reserve_ms, 0); + assert!(reserve.cutover_required); + let candidate_tracks = match tuliprox_mpegts::ts_inspector::inspect_mpeg_ts( + std::io::Cursor::new(CRITICAL_HANDOFF_TS_BODY), + tuliprox_mpegts::ts_inspector::HlsTsProbeProtection::Clear, + tuliprox_mpegts::ts_inspector::HlsTsProbeBudget::default(), + ) + .expect("critical handoff fixture inspection succeeds") + { + tuliprox_mpegts::ts_inspector::HlsTsProbeOutcome::Found(signature) => signature, + outcome => panic!("critical handoff fixture has no MPEG-TS tracks: {outcome:?}"), + }; + let terminal_response = + request.app_config.custom_stream_response.load_full().expect("critical handoff terminal response"); + let terminal_asset = super::super::snapshot_terminal_media_asset( + terminal_response.channel_unavailable.as_ref().expect("critical handoff terminal buffer"), + ) + .expect("critical handoff terminal asset"); + assert_eq!( + crate::critical_handoff::terminal_alternative_compatibility_for_critical_lease( + Some(terminal_asset.as_ref()), + live_lease, + &candidate_tracks, + ), + crate::manifest_acceptance::HlsTerminalAlternativeCompatibility::LiveHandoffSafer + ); +} + +pub(in crate::refresh::tests) async fn assert_critical_handoff_timeline_commit( + session: &Arc>, + cache: &HlsSegmentCache, + origin: &CriticalEmergencyOriginServer, + baseline_epoch: u64, + expected_candidate_count: usize, +) { + let (staged_cache_key, staged_content_length) = { + let session = session.read().await; + assert_eq!(session.origin_epoch, baseline_epoch.saturating_add(1)); + let effective_host = host_from_base_url(&origin.base_url); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some(effective_host.as_str())); + assert_eq!( + session.origin_epoch_effective_host_id, + Some(crate::timeline::effective_origin_host_id(&effective_host)) + ); + let episode = session.origin_control.acceptance_episode.as_ref().expect("completed acceptance episode"); + assert!(episode.full_burst_completed); + assert_eq!(episode.completed_burst_candidates, expected_candidate_count); + assert_eq!(episode.state, super::super::super::manifest_acceptance::HlsManifestAcceptanceState::Completed); + let switched = session + .segments + .values() + .filter(|segment| segment.origin_key.origin_epoch == session.origin_epoch) + .collect::>(); + assert_eq!(switched.len(), 2); + let staged = switched.first().expect("staged handoff segment"); + assert_eq!(staged.origin_key.host_local_sequence, 900); + assert!(staged.discontinuity_before); + let content_length = match &staged.status { + SegmentCacheStatus::Ready { content_length, .. } => *content_length, + SegmentCacheStatus::Discovered + | SegmentCacheStatus::Queued { .. } + | SegmentCacheStatus::Fetching { .. } + | SegmentCacheStatus::CapacityDeferred { .. } + | SegmentCacheStatus::FailedRetryable { .. } + | SegmentCacheStatus::FailedPermanent { .. } + | SegmentCacheStatus::Expired => panic!("staged handoff segment must be READY"), + }; + assert_eq!(content_length, u64::try_from(CRITICAL_HANDOFF_TS_BODY.len()).unwrap_or(u64::MAX)); + assert!(!matches!(switched[1].status, SegmentCacheStatus::Ready { .. })); + (staged.cache_key.clone(), content_length) + }; + let staged_metadata = cache + .metadata(&staged_cache_key) + .await + .expect("staged cache metadata reads") + .expect("staged cache object exists"); + assert_eq!(staged_metadata.size, staged_content_length); +} + +#[tokio::test] +async fn critical_single_alternative_full_acceptance_commits_verified_new_origin_epoch() { + let origin = spawn_critical_emergency_origin().await; + let temp_dir = tempfile::tempdir().expect("critical handoff cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let session = test_session(); + prepare_cross_host_baseline(&session).await; + let baseline_binding = HlsManifestOriginBinding::new( + Url::parse("https://previous.example.com/live/index.m3u8?token=baseline").expect("baseline URL"), + Some(0), + ) + .expect("baseline binding"); + session.write().await.origin_control.manifest_origin_binding = Some(baseline_binding); + let now_ms = super::super::current_time_millis(); + let (baseline_epoch, baseline_readiness_generation) = { + let session = session.read().await; + (session.origin_epoch, session.activity.media_readiness_generation) + }; + let (hls_proxy, lease_id, live_lease) = prepare_active_critical_handoff_lease(&session, &cache, now_ms).await; + let mut request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); + request.app_config = critical_handoff_app_config(); + request.hls_proxy = Arc::clone(&hls_proxy); + request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::Critical; + request.access_lease_id = Some(lease_id); + request.now_ms = now_ms; + assert_critical_handoff_evidence(&request, &live_lease, now_ms).await; + + let plan = request.manifest_recovery_burst.level.plan(); + let target_url = request.origin_entry.url().clone(); + let fetch_context = manifest_fetch_context(&request); + let committed = retry_hls_origin_manifest_recovery_chain( + &fetch_context, + test_manifest_origin_binding(target_url.clone()), + Some(HlsManifestRejectLogReason::PinnedHostRecoveryRejected), + None, + HlsManifestAcceptanceTrigger::Critical, + HlsManifestCommitAcceptanceMode::StrictPinnedHost, + |fetched, acceptance_mode| super::super::commit_manifest_recovery_candidate(&request, fetched, acceptance_mode), + ) + .await + .expect("critical emergency handoff commits"); + + assert_eq!(committed.fetched.body.as_bytes(), CRITICAL_HANDOFF_MANIFEST_BODY); + assert_eq!(origin.manifest_requests.load(Ordering::SeqCst), plan.total_candidates()); + assert_eq!(origin.segment_requests.load(Ordering::SeqCst), 1); + assert_critical_handoff_timeline_commit(&session, &cache, &origin, baseline_epoch, plan.total_candidates()).await; + assert_eq!( + session.read().await.activity.media_readiness_generation, + baseline_readiness_generation.saturating_add(1), + "one staged READY transaction advances media readiness exactly once" + ); + assert_eq!( + session + .read() + .await + .origin_control + .manifest_origin_binding + .as_ref() + .map(super::super::super::manifest_origin_binding::HlsManifestOriginBinding::request_url), + Some(&target_url) + ); +} + +pub(in crate::refresh::tests) async fn assert_committed_content_anchor_is_gc_pinned_until_read_pin_drop() { + const COMMITTED_BYTES: &[u8] = b"committed-media-bytes"; + + let temp_dir = tempfile::tempdir().expect("acceptance pin cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let sessions = Arc::new(HlsSessionStore::new()); + let session = sessions.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; + let gc = HlsGarbageCollector::new( + Arc::clone(&sessions), + Arc::clone(&cache), + GarbageCollectionPolicy { + cache_duration_ms: 0, + cache_bytes_global: 10_000, + cache_bytes_per_session: 10_000, + session_idle_timeout_ms: u64::MAX, + temp_file_retention_ms: 30_000, + failed_segment_retention_ms: 10, + }, + build_rewrite_secret_fingerprint(b"secret"), + ); + let manifest_body = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ + #EXTINF:4.0,\n1.ts\n#EXTINF:4.0,\n2.ts\n#EXTINF:4.0,\n3.ts\n\ + #EXTINF:4.0,\n4.ts\n#EXTINF:4.0,\n5.ts\n#EXTINF:4.0,\n6.ts\n"; + let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = + tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( + manifest_body, + "http://origin.example/live/index.m3u8", + ) + else { + panic!("acceptance pin manifest must parse as a normal timeline"); + }; + let cache_key = { + let mut session = session.write().await; + session.proxy_next_seq = Some(1); + session.apply_origin_manifest(&manifest).expect("acceptance pin timeline commits"); + session.segments.get(&1).expect("first committed segment").cache_key.clone() + }; + cache.write_bytes_and_commit(&cache_key, COMMITTED_BYTES).await.expect("committed acceptance object writes"); + let access = { + let mut session = session.write().await; + let segment = session.segments.get_mut(&1).expect("first committed segment"); + segment.status = SegmentCacheStatus::Ready { + content_length: u64::try_from(COMMITTED_BYTES.len()).unwrap_or(u64::MAX), + ready_at_ms: 0, + }; + let expected_identity = + crate::resource_identity::HlsMediaResourceIdentity::from_url("http://origin.example/live/1.ts", None); + session + .segments + .values() + .rev() + .find_map(|entry| { + let fetch_ref = entry.origin_fetch_ref.as_ref()?; + crate::resource_identity::HlsMediaResourceIdentity::from_url( + &fetch_ref.resolved_origin_url, + fetch_ref.byte_range, + ) + .matches(expected_identity) + .then(|| Arc::clone(&entry.access)) + }) + .expect("content anchor object selection") + }; + let read_pin = super::super::commit::HlsCommittedAcceptanceReadPin::acquire(Arc::clone(&access), 5); + assert_eq!(access.active_readers(), 1); + + let pinned_report = gc.run_once(10_000).await.expect("GC runs while acceptance object is pinned"); + assert_eq!(pinned_report.segments_deleted_duration, 0); + assert!(session.read().await.segments.contains_key(&1)); + assert!(cache.metadata(&cache_key).await.expect("pinned object metadata reads").is_some()); + + drop(read_pin); + assert_eq!(access.active_readers(), 0); + let released_report = gc.run_once(10_001).await.expect("GC runs after acceptance pin release"); + assert_eq!(released_report.segments_deleted_duration, 1); + assert!(!session.read().await.segments.contains_key(&1)); + assert!(cache.metadata(&cache_key).await.expect("released object metadata reads").is_none()); +} + +#[tokio::test] +async fn committed_content_anchor_object_survives_gc_until_acceptance_read_pin_drop() { + assert_committed_content_anchor_is_gc_pinned_until_read_pin_drop().await; +} diff --git a/backend/hls/src/refresh/tests/archive_master.rs b/backend/hls/src/refresh/tests/archive_master.rs new file mode 100644 index 000000000..f3bc2f2af --- /dev/null +++ b/backend/hls/src/refresh/tests/archive_master.rs @@ -0,0 +1,88 @@ +use super::{ + fetch_hls_origin_manifest_request, manifest_body, no_delay_policy, request_header_value, spawn_test_origin, + test_app_config, test_recovery_timing_policy, HlsManifestOriginBinding, HlsOriginManifestFetchContext, + HlsOriginManifestFetchRequest, LiveHlsOriginEntry, +}; +use crate::{HlsSession, HlsSessionKey}; +use axum::http::{header, HeaderMap, HeaderValue}; +use std::sync::Arc; +use tokio::sync::RwLock; +use tuliprox_core::model::HlsManifestRecoveryBurstConfig; +use url::Url; + +#[tokio::test] +async fn archive_master_resolution_binds_recovery_to_entry_and_scrubs_redirected_variant_credentials() { + let archive = spawn_test_origin(Arc::new(|path| { + if path.starts_with("/channel/tracks-v1/") { + (200, vec![("Set-Cookie", "route=b; Path=/".to_string())], manifest_body()) + } else { + let master = "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=1000\ntracks-v1/archive-1-2.ts.m3u8\n"; + (200, vec![("Set-Cookie", "session=a; Path=/".to_string())], master.to_string()) + } + })) + .await; + let master_url = format!("{}/channel/archive-1-2.m3u8", archive.base_url); + let variant_url = format!("{}/channel/tracks-v1/archive-1-2.ts.m3u8", archive.base_url); + let entry = + spawn_test_origin(Arc::new(move |_path| (302, vec![("Location", master_url.clone())], String::new()))).await; + let entry_url = format!("{}/live/user/pass/archive-1-2.m3u8", entry.base_url); + let mut headers = HeaderMap::new(); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer entry-secret")); + headers.insert(header::COOKIE, HeaderValue::from_static("sid=entry-secret")); + let context = HlsOriginManifestFetchContext { + app_config: test_app_config(), + session: Arc::new(RwLock::new(HlsSession::new( + HlsSessionKey::new(1, "12345").with_archive_reference(1_784_898_000), + b"secret", + 0, + ))), + origin_entry: LiveHlsOriginEntry::parse(&entry_url).expect("entry url"), + headers, + client: reqwest::Client::builder().no_proxy().build().expect("client"), + no_redirect_client: reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client"), + use_manual_redirects: false, + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + retry_policy: no_delay_policy(), + recovery_timing_policy: test_recovery_timing_policy(2_000), + acceptance_timing_seed: None, + }; + + let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) + .await + .expect("archive master resolves to its variant"); + assert_eq!(fetched.body, manifest_body()); + assert_eq!(fetched.final_manifest_url, variant_url); + assert_eq!(fetched.resolved_request_url, entry_url); + assert_eq!(fetched.provider_session_headers[header::COOKIE], "session=a; route=b"); + { + let raw_requests = archive.raw_requests.lock().await; + let variant_request = raw_requests + .iter() + .find(|request| request.starts_with("GET /channel/tracks-v1/")) + .expect("variant request"); + assert_eq!(request_header_value(variant_request, "authorization"), None); + assert_eq!(request_header_value(variant_request, "cookie"), Some("session=a")); + } + + let binding = HlsManifestOriginBinding::new( + Url::parse(&fetched.resolved_request_url).expect("binding url"), + fetched.provider_url_index, + ) + .expect("binding"); + let recovered = crate::manifest_fetch::fetch_hls_origin_manifest_recovery_once(&context, &binding) + .await + .expect("recovery re-resolves the archive master"); + assert_eq!(recovered.body, manifest_body()); + assert_eq!(recovered.final_manifest_url, variant_url); + assert_eq!(recovered.resolved_request_url, entry_url); + assert_eq!(entry.requests.lock().await.len(), 2); + let raw_requests = archive.raw_requests.lock().await; + assert_eq!(raw_requests.len(), 4); + assert!(raw_requests.iter().all(|request| request_header_value(request, "authorization").is_none())); + assert!(raw_requests.iter().all(|request| request_header_value(request, "cookie") != Some("sid=entry-secret"))); +} diff --git a/backend/hls/src/refresh/tests/critical_handoff.rs b/backend/hls/src/refresh/tests/critical_handoff.rs new file mode 100644 index 000000000..ecb372063 --- /dev/null +++ b/backend/hls/src/refresh/tests/critical_handoff.rs @@ -0,0 +1,635 @@ +use super::{ + assert_incompatible_switch_is_rejected_before_timeline_commit, commit_ready_baseline_snapshot, + critical_handoff_app_config, manifest_fetch_context, prepare_active_critical_handoff_lease, + prepare_cross_host_baseline, retry_hls_origin_manifest_recovery_chain, spawn_critical_emergency_origin, + spawn_critical_emergency_origin_with_control, switch_test_request, test_acceptance_episode_timing, + test_manifest_origin_binding, test_session, CriticalEmergencyOriginServer, HlsManifestAcceptanceTrigger, + HlsManifestCommitAcceptanceMode, HlsManifestCommitError, HlsManifestRejectLogReason, + CRITICAL_HANDOFF_MANIFEST_BODY, CRITICAL_HANDOFF_TS_BODY, +}; +use crate::{ + build_rewrite_secret_fingerprint, terminal_tail::HlsLeasePlaybackMode, GarbageCollectionPolicy, HlsAccessLease, + HlsAccessLeaseId, HlsAccessLeaseStore, HlsAccessLeaseTiming, HlsGarbageCollector, HlsLeaseManifestSnapshot, + HlsPlaybackFamilyKey, HlsProxyManager, HlsSegmentCache, HlsSession, HlsSessionKey, HlsSessionStore, + SegmentCacheStatus, +}; +use shared::model::HlsManifestRecoveryBurstLevel; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; +use tokio::sync::RwLock; +use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer; + +pub(in crate::refresh::tests) async fn spawn_controlled_critical_emergency_origin() -> CriticalEmergencyOriginServer { + spawn_critical_emergency_origin_with_control(true).await +} + +pub(in crate::refresh::tests) fn active_critical_test_lease( + lease_id: &str, + proxy_session_id: &super::super::super::ProxySessionId, + issued_at_ms: u64, + now_ms: u64, + manifest: HlsLeaseManifestSnapshot, +) -> HlsAccessLease { + let mut lease = HlsAccessLease::pending( + HlsAccessLeaseId(lease_id.to_string()), + HlsPlaybackFamilyKey::new(lease_id, lease_id), + proxy_session_id.clone(), + lease_id.to_string(), + format!("{lease_id}-session"), + 1, + "12345".to_string(), + 12345, + issued_at_ms, + 120_000, + ); + lease.state = super::super::super::HlsAccessLeaseState::Activated; + lease.active_until_ms = Some(now_ms.saturating_add(120_000)); + lease.valid_until_ms = now_ms.saturating_add(120_000); + lease.playback_mode = HlsLeasePlaybackMode::Live; + lease.admission_generation = 1; + lease.last_manifest_snapshot = Some(manifest); + lease +} + +pub(in crate::refresh::tests) async fn prepare_three_segment_critical_timeline( + session: &Arc>, + cache: &HlsSegmentCache, + now_ms: u64, +) -> HlsLeaseManifestSnapshot { + let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:500\n#EXT-X-TARGETDURATION:4\n\ + #EXTINF:4.0,\nold500.ts\n#EXTINF:4.0,\nold501.ts\n#EXTINF:4.0,\nold502.ts\n"; + let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = + tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( + body, + "http://previous.example.com/live/user/pass/12345.m3u8", + ) + else { + panic!("three-segment baseline parses"); + }; + { + let mut session = session.write().await; + session + .apply_origin_manifest_for_host( + &manifest, + crate::timeline::effective_origin_host_id("previous.example.com"), + ) + .expect("three-segment baseline commits"); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + session.origin_control.pinned_host = Some("previous.example.com".to_string()); + session.origin_control.origin_epoch = session.origin_epoch; + } + commit_ready_baseline_snapshot(session, cache, now_ms).await +} + +pub(in crate::refresh::tests) fn critical_staging_generation( + session: &mut HlsSession, + now_ms: u64, +) -> super::super::switch_staging::HlsSwitchStagingGeneration { + let burst_plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + session.origin_control.begin_acceptance_episode( + now_ms, + burst_plan, + HlsManifestAcceptanceTrigger::Critical, + &test_acceptance_episode_timing(now_ms, burst_plan), + ); + let episode = session.origin_control.acceptance_episode.as_mut().expect("critical acceptance episode"); + episode.record_full_burst(); + episode.state = super::super::super::manifest_acceptance::HlsManifestAcceptanceState::StagingSwitchSegment; + let identity = super::super::super::manifest_acceptance::HlsManifestRecoveryCandidateIdentity::from_candidate( + 0, + Some("candidate.example.com"), + "test-candidate", + ); + assert_eq!( + episode.select_candidate(episode.generation, identity), + super::super::super::manifest_acceptance::HlsRecoveryWorkloadBindingUpdate::Applied + ); + assert_eq!( + episode.bind_selected_candidate( + episode.generation, + identity, + super::super::super::recovery_timing::HlsRecoveryWorkload::clear_fetch(), + ), + super::super::super::manifest_acceptance::HlsRecoveryWorkloadBindingUpdate::Applied + ); + super::super::switch_staging::switch_staging_generation(session).expect("critical staging generation") +} + +#[tokio::test] +async fn critical_handoff_lock_contention_exhaustion_is_typed_and_bounded() { + let attempts = Arc::new(AtomicUsize::new(0)); + let attempts_for_access = Arc::clone(&attempts); + + let result: Result<(), HlsManifestCommitError> = super::super::retry_critical_handoff_state_access( + crate::manifest_acceptance::HlsManifestAcceptanceGeneration(8), + || { + attempts_for_access.fetch_add(1, Ordering::SeqCst); + async { crate::HlsCriticalHandoffStateAccess::LockBusy } + }, + ) + .await; + + assert_eq!(attempts.load(Ordering::SeqCst), crate::critical_handoff::HLS_CRITICAL_HANDOFF_COMMIT_RETRIES); + let Err(HlsManifestCommitError::TimelineRejected { reason }) = result else { + panic!("exhausted lock contention must return a typed timeline rejection"); + }; + assert_eq!(reason, HlsManifestRejectLogReason::CriticalHandoffLockContentionExhausted); + assert_eq!(reason.status_label(), "critical-handoff-lock-contention-exhausted"); + assert_ne!(reason, HlsManifestRejectLogReason::SwitchResourceUnavailable); + assert_ne!(reason, HlsManifestRejectLogReason::StagedSwitchInvalidated); + assert_ne!(reason, HlsManifestRejectLogReason::PinnedHostRecoveryRejected); +} + +#[tokio::test] +async fn critical_handoff_lock_busy_retry_retains_staged_state_until_acquired() { + struct StagedRetentionProbe { + identity: u64, + drops: Arc, + } + + impl Drop for StagedRetentionProbe { + fn drop(&mut self) { self.drops.fetch_add(1, Ordering::SeqCst); } + } + + let drops = Arc::new(AtomicUsize::new(0)); + let attempts = Arc::new(AtomicUsize::new(0)); + let staged = StagedRetentionProbe { identity: 41, drops: Arc::clone(&drops) }; + let staged_address = std::ptr::from_ref(&staged); + let attempts_for_access = Arc::clone(&attempts); + + let committed = super::super::retry_critical_handoff_state_access( + crate::manifest_acceptance::HlsManifestAcceptanceGeneration(9), + || { + let attempt = attempts_for_access.fetch_add(1, Ordering::SeqCst); + let staged = &staged; + let drops_for_attempt = Arc::clone(&drops); + async move { + assert_eq!(std::ptr::from_ref(staged), staged_address); + assert_eq!(drops_for_attempt.load(Ordering::SeqCst), 0); + if attempt == 0 { + crate::HlsCriticalHandoffStateAccess::LockBusy + } else { + crate::HlsCriticalHandoffStateAccess::Acquired(Ok(staged.identity)) + } + } + }, + ) + .await + .expect("the retained staged state commits after transient contention"); + + assert_eq!(committed, 41); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + assert_eq!(drops.load(Ordering::SeqCst), 0); + drop(staged); + assert_eq!(drops.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn critical_handoff_selects_endangered_lease_base_instead_of_session_tail() { + let temp_dir = tempfile::tempdir().expect("critical selection cache tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let session = test_session(); + let now_ms = super::super::current_time_millis(); + let complete = prepare_three_segment_critical_timeline(&session, &cache, now_ms).await; + let mut first_window = complete.clone(); + first_window.snapshot_generation = 1; + first_window.visible_segments = Arc::from([complete.visible_segments[0].clone()]); + first_window.last_proxy_seq = first_window.first_proxy_seq; + first_window.playlist_duration_ms = 4_000; + first_window.last_visible_media_end_ms = 4_000; + let mut endangered_window = complete.clone(); + endangered_window.snapshot_generation = 2; + endangered_window.visible_segments = + Arc::from([complete.visible_segments[0].clone(), complete.visible_segments[1].clone()]); + endangered_window.last_proxy_seq = complete.visible_segments[1].proxy_seq; + endangered_window.playlist_duration_ms = 8_000; + endangered_window.last_visible_media_end_ms = 8_000; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let leases = vec![ + active_critical_test_lease("more-reserve", &proxy_session_id, now_ms, now_ms, first_window), + active_critical_test_lease( + "endangered", + &proxy_session_id, + now_ms.saturating_add(1), + now_ms, + endangered_window, + ), + ]; + let mut session = session.write().await; + let generation = critical_staging_generation(&mut session, now_ms); + let (selected, snapshot) = + super::super::switch_staging::select_critical_handoff_lease(&session, &leases, &generation, now_ms) + .expect("one lease is cutover critical"); + + assert_eq!(selected.lease_id, HlsAccessLeaseId("endangered".to_string())); + assert_eq!(snapshot.base.proxy_seq, complete.visible_segments[1].proxy_seq); + assert_ne!(snapshot.base.proxy_seq, complete.visible_segments[2].proxy_seq); +} + +#[tokio::test] +async fn critical_handoff_prioritizes_earliest_safe_cutover_deadline() { + let temp_dir = tempfile::tempdir().expect("critical deadline cache tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let session = test_session(); + let now_ms = super::super::current_time_millis(); + let complete = prepare_three_segment_critical_timeline(&session, &cache, now_ms).await; + let mut wider_margin = complete.clone(); + wider_margin.snapshot_generation = 3; + wider_margin.target_duration_ms = 4_000; + let mut exhausted_short_margin = complete.clone(); + exhausted_short_margin.snapshot_generation = 4; + exhausted_short_margin.target_duration_ms = 1_000; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let mut wider_margin_lease = + active_critical_test_lease("wider-margin", &proxy_session_id, now_ms, now_ms, wider_margin); + let playback_at_ms = now_ms.saturating_sub(3_000); + let token = wider_margin_lease.playback_cursor.record_request_started(complete.last_proxy_seq, playback_at_ms); + let _ = wider_margin_lease.playback_cursor.record_request_completed(token, playback_at_ms); + let leases = vec![ + wider_margin_lease, + active_critical_test_lease( + "exhausted-short-margin", + &proxy_session_id, + now_ms.saturating_add(1), + now_ms, + exhausted_short_margin, + ), + ]; + let mut session = session.write().await; + let generation = critical_staging_generation(&mut session, now_ms); + let (selected, snapshot) = + super::super::switch_staging::select_critical_handoff_lease(&session, &leases, &generation, now_ms) + .expect("critical leases have a deterministic deadline order"); + + assert_eq!(selected.lease_id, HlsAccessLeaseId("wider-margin".to_string())); + assert_eq!(snapshot.base.proxy_seq, complete.last_proxy_seq); +} + +#[tokio::test] +async fn critical_handoff_manifest_supersession_invalidates_frozen_snapshot() { + let temp_dir = tempfile::tempdir().expect("critical supersession cache tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let session = test_session(); + let now_ms = super::super::current_time_millis(); + let complete = prepare_three_segment_critical_timeline(&session, &cache, now_ms).await; + let mut endangered_window = complete.clone(); + endangered_window.snapshot_generation = 7; + endangered_window.visible_segments = + Arc::from([complete.visible_segments[0].clone(), complete.visible_segments[1].clone()]); + endangered_window.last_proxy_seq = complete.visible_segments[1].proxy_seq; + endangered_window.playlist_duration_ms = 8_000; + endangered_window.last_visible_media_end_ms = 8_000; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let mut lease = active_critical_test_lease("endangered", &proxy_session_id, now_ms, now_ms, endangered_window); + let mut session = session.write().await; + let generation = critical_staging_generation(&mut session, now_ms); + let (_, frozen) = + super::super::switch_staging::select_critical_handoff_lease(&session, &[lease.clone()], &generation, now_ms) + .expect("initial critical snapshot"); + let mut leases = HlsAccessLeaseStore::default(); + leases.prepare_access_lease(lease.clone()); + assert!(super::super::switch_staging::critical_handoff_snapshot_is_current( + &mut leases, + &session, + &generation, + &frozen, + now_ms, + )); + if let Some(manifest) = lease.last_manifest_snapshot.as_mut() { + manifest.snapshot_generation = manifest.snapshot_generation.saturating_add(1); + } + leases.prepare_access_lease(lease); + + assert!(!super::super::switch_staging::critical_handoff_snapshot_is_current( + &mut leases, + &session, + &generation, + &frozen, + now_ms, + )); +} + +pub(in crate::refresh::tests) async fn install_active_critical_test_lease( + hls_proxy: &HlsProxyManager, + lease_id: &str, + proxy_session_id: &super::super::super::ProxySessionId, + manifest: HlsLeaseManifestSnapshot, + now_ms: u64, +) -> HlsAccessLeaseId { + let lease_id = HlsAccessLeaseId(lease_id.to_string()); + hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new(lease_id.0.as_str(), lease_id.0.as_str()), + proxy_session_id.clone(), + lease_id.0.clone(), + format!("{}-session", lease_id.0), + 1, + "12345".to_string(), + 12345, + now_ms, + 120_000, + )) + .await; + assert!(hls_proxy + .activate_access_lease( + &lease_id, + proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 120_000, valid_window_ms: 120_000 }, + ) + .await + .is_activated()); + let publication_guard = hls_proxy + .prepare_access_lease_manifest_publication(&lease_id, proxy_session_id, now_ms) + .await + .expect("critical test lease publication guard"); + assert!(hls_proxy + .commit_access_lease_manifest_publication(&lease_id, proxy_session_id, publication_guard, manifest, now_ms,) + .await + .is_committed()); + lease_id +} + +#[test] +fn critical_handoff_terminal_response_revision_must_remain_current() { + let app_config = critical_handoff_app_config(); + let frozen = app_config.custom_stream_response.load_full(); + assert!(super::super::critical_handoff_terminal_response_is_current(&app_config, frozen.as_ref())); + let replacement = frozen.as_ref().map(|response| Arc::new(response.as_ref().clone())); + app_config.custom_stream_response.store(replacement); + + assert!(!super::super::critical_handoff_terminal_response_is_current(&app_config, frozen.as_ref())); +} + +#[test] +fn critical_handoff_and_terminal_tail_share_ts_inspector_signature() { + let terminal_buffer = TransportStreamBuffer::new(CRITICAL_HANDOFF_TS_BODY.to_vec()); + let terminal_asset = super::super::snapshot_terminal_media_asset(&terminal_buffer).expect("terminal asset"); + let critical_signature = match tuliprox_mpegts::ts_inspector::inspect_mpeg_ts( + std::io::Cursor::new(CRITICAL_HANDOFF_TS_BODY), + tuliprox_mpegts::ts_inspector::HlsTsProbeProtection::Clear, + tuliprox_mpegts::ts_inspector::HlsTsProbeBudget::default(), + ) + .expect("critical handoff fixture inspection succeeds") + { + tuliprox_mpegts::ts_inspector::HlsTsProbeOutcome::Found(signature) => signature, + outcome => panic!("critical handoff fixture has no MPEG-TS tracks: {outcome:?}"), + }; + + assert_eq!(critical_signature, terminal_asset.track_signature().clone()); +} + +#[tokio::test] +async fn critical_handoff_uses_endangered_lease_base_when_session_tail_is_incompatible() { + let origin = spawn_critical_emergency_origin().await; + let temp_dir = tempfile::tempdir().expect("lease-specific critical handoff cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let session = test_session(); + let now_ms = super::super::current_time_millis(); + let complete = prepare_three_segment_critical_timeline(&session, &cache, now_ms).await; + let mut noncritical_window = complete.clone(); + noncritical_window.visible_segments = Arc::from([complete.visible_segments[0].clone()]); + noncritical_window.last_proxy_seq = noncritical_window.first_proxy_seq; + noncritical_window.playlist_duration_ms = 4_000; + noncritical_window.last_visible_media_end_ms = 4_000; + let mut endangered_window = complete.clone(); + endangered_window.visible_segments = + Arc::from([complete.visible_segments[0].clone(), complete.visible_segments[1].clone()]); + endangered_window.last_proxy_seq = complete.visible_segments[1].proxy_seq; + endangered_window.playlist_duration_ms = 8_000; + endangered_window.last_visible_media_end_ms = 8_000; + let (tail_key, lease_base_key, baseline_epoch) = { + let session = session.read().await; + ( + session.segments.get(&complete.last_proxy_seq).expect("session-wide tail").cache_key.clone(), + session.segments.get(&endangered_window.last_proxy_seq).expect("endangered lease base").cache_key.clone(), + session.origin_epoch, + ) + }; + cache.delete(&tail_key).await.expect("replace session-wide tail fixture"); + let invalid_tail = vec![0_u8; 188 * 2]; + cache.write_bytes_and_commit(&tail_key, &invalid_tail).await.expect("incompatible session-wide tail commits"); + { + let mut session = session.write().await; + let tail = session.segments.get_mut(&complete.last_proxy_seq).expect("session-wide tail entry"); + tail.status = SegmentCacheStatus::Ready { + content_length: u64::try_from(invalid_tail.len()).unwrap_or(u64::MAX), + ready_at_ms: now_ms, + }; + } + assert!(super::super::switch_staging::inspect_cache_object_tracks(&cache, &tail_key).await.signature().is_none()); + assert!(super::super::switch_staging::inspect_cache_object_tracks(&cache, &lease_base_key) + .await + .signature() + .is_some()); + + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(cache.cache_path(), 300)); + let _noncritical_lease = install_active_critical_test_lease( + &hls_proxy, + "noncritical-window", + &proxy_session_id, + noncritical_window, + now_ms, + ) + .await; + let endangered_lease = install_active_critical_test_lease( + &hls_proxy, + "endangered-window", + &proxy_session_id, + endangered_window, + now_ms, + ) + .await; + let mut request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); + request.hls_proxy = hls_proxy; + request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::Critical; + request.access_lease_id = Some(endangered_lease); + request.now_ms = now_ms; + let plan = request.manifest_recovery_burst.level.plan(); + let target_url = request.origin_entry.url().clone(); + let fetch_context = manifest_fetch_context(&request); + + let committed = retry_hls_origin_manifest_recovery_chain( + &fetch_context, + test_manifest_origin_binding(target_url), + Some(HlsManifestRejectLogReason::PinnedHostRecoveryRejected), + None, + HlsManifestAcceptanceTrigger::Critical, + HlsManifestCommitAcceptanceMode::StrictPinnedHost, + |fetched, acceptance_mode| super::super::commit_manifest_recovery_candidate(&request, fetched, acceptance_mode), + ) + .await + .expect("lease-specific compatible base permits critical handoff"); + + assert_eq!(committed.fetched.body.as_bytes(), CRITICAL_HANDOFF_MANIFEST_BODY); + assert_eq!(origin.manifest_requests.load(Ordering::SeqCst), plan.total_candidates()); + assert_eq!(origin.segment_requests.load(Ordering::SeqCst), 1); + let session = session.read().await; + assert_eq!(session.origin_epoch, baseline_epoch.saturating_add(1)); + assert!(session + .segments + .values() + .find(|segment| segment.origin_key.origin_epoch == session.origin_epoch) + .is_some_and(|segment| segment.discontinuity_before)); +} + +#[tokio::test] +async fn critical_handoff_manifest_supersession_during_candidate_io_rolls_back_staging() { + let origin = spawn_controlled_critical_emergency_origin().await; + let temp_dir = tempfile::tempdir().expect("critical supersession cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let session = test_session(); + prepare_cross_host_baseline(&session).await; + let now_ms = super::super::current_time_millis(); + let (hls_proxy, lease_id, live_lease) = prepare_active_critical_handoff_lease(&session, &cache, now_ms).await; + let proxy_session_id = live_lease.proxy_session_id.clone(); + let base_proxy_seq = live_lease.last_manifest_snapshot.as_ref().expect("critical manifest snapshot").last_proxy_seq; + let (baseline_epoch, baseline_proxy_next_seq, baseline_readiness_generation, baseline_segments, base_access) = { + let session = session.read().await; + ( + session.origin_epoch, + session.proxy_next_seq, + session.activity.media_readiness_generation, + session + .segments + .iter() + .map(|(proxy_seq, segment)| (*proxy_seq, segment.cache_key.clone())) + .collect::>(), + Arc::clone(&session.segments.get(&base_proxy_seq).expect("critical lease base segment").access), + ) + }; + let candidate_cache_key = super::super::super::SegmentCacheKey::new( + proxy_session_id.clone(), + baseline_proxy_next_seq.expect("cross-host preview sequence"), + "ts", + ); + let mut request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); + request.app_config = critical_handoff_app_config(); + request.hls_proxy = Arc::clone(&hls_proxy); + request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::Critical; + request.access_lease_id = Some(lease_id.clone()); + request.now_ms = now_ms; + let request = Arc::new(request); + let task_request = Arc::clone(&request); + let target_url = task_request.origin_entry.url().clone(); + let commit_task = tokio::spawn(async move { + let fetch_context = manifest_fetch_context(&task_request); + retry_hls_origin_manifest_recovery_chain( + &fetch_context, + test_manifest_origin_binding(target_url), + Some(HlsManifestRejectLogReason::PinnedHostRecoveryRejected), + None, + HlsManifestAcceptanceTrigger::Critical, + HlsManifestCommitAcceptanceMode::StrictPinnedHost, + |fetched, acceptance_mode| { + super::super::commit_manifest_recovery_candidate(&task_request, fetched, acceptance_mode) + }, + ) + .await + }); + + origin.segment_prefix_written.notified().await; + assert!(base_access.active_readers() > 0, "lease base is pinned before candidate media completes"); + let publication_now_ms = super::super::current_time_millis(); + let publication_guard = hls_proxy + .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, publication_now_ms) + .await + .expect("superseding manifest publication guard"); + let superseding_snapshot = live_lease.last_manifest_snapshot.clone().expect("superseding snapshot"); + assert!(hls_proxy + .commit_access_lease_manifest_publication( + &lease_id, + &proxy_session_id, + publication_guard, + superseding_snapshot, + publication_now_ms, + ) + .await + .is_committed()); + origin.release_segment_body.notify_one(); + + let result = commit_task.await.expect("critical handoff task joins"); + assert!(result.is_err(), "superseded lease snapshot cannot commit"); + assert_eq!(base_access.active_readers(), 0, "rollback releases the frozen base pin"); + { + let session = session.read().await; + assert_eq!(session.origin_epoch, baseline_epoch); + assert_eq!(session.proxy_next_seq, baseline_proxy_next_seq); + assert_eq!(session.activity.media_readiness_generation, baseline_readiness_generation); + assert_eq!( + session + .segments + .iter() + .map(|(proxy_seq, segment)| (*proxy_seq, segment.cache_key.clone())) + .collect::>(), + baseline_segments + ); + } + assert!( + cache.metadata(&candidate_cache_key).await.expect("candidate cache metadata reads").is_none(), + "final CAS rejection removes the staged candidate object" + ); +} + +#[tokio::test] +async fn critical_handoff_base_evidence_survives_gc_until_preparation_drop() { + let temp_dir = tempfile::tempdir().expect("critical evidence GC tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let sessions = Arc::new(HlsSessionStore::new()); + let session = sessions.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; + prepare_cross_host_baseline(&session).await; + let manifest = commit_ready_baseline_snapshot(&session, &cache, 0).await; + let base_proxy_seq = manifest.last_proxy_seq; + let (base_key, base_access) = { + let session = session.read().await; + let base = session.segments.get(&base_proxy_seq).expect("lease-specific terminal base"); + (base.cache_key.clone(), Arc::clone(&base.access)) + }; + let gc = HlsGarbageCollector::new( + Arc::clone(&sessions), + Arc::clone(&cache), + GarbageCollectionPolicy { + cache_duration_ms: 0, + cache_bytes_global: 10_000_000, + cache_bytes_per_session: 10_000_000, + session_idle_timeout_ms: u64::MAX, + temp_file_retention_ms: 30_000, + failed_segment_retention_ms: 10, + }, + build_rewrite_secret_fingerprint(b"secret"), + ); + + let evidence = super::super::super::prepare_terminal_base_evidence(&session, &cache, &manifest, 5).await; + assert_eq!(evidence.track_base().map(|base| base.proxy_seq), Some(base_proxy_seq)); + assert!(evidence.track_signature().is_some()); + assert!(base_access.active_readers() > 0); + let pinned_report = gc.run_once(10_000).await.expect("GC runs while critical base evidence is pinned"); + assert_eq!(pinned_report.segments_deleted_duration, 0); + assert!(cache.metadata(&base_key).await.expect("pinned base metadata reads").is_some()); + + evidence.release(); + assert_eq!(base_access.active_readers(), 0); + let released_report = gc.run_once(10_001).await.expect("GC runs after critical base evidence release"); + assert!(released_report.segments_deleted_duration > 0); + assert!(cache.metadata(&base_key).await.expect("released base metadata reads").is_none()); +} + +#[tokio::test] +async fn mapped_fmp4_tail_cannot_handoff_to_mapless_ts_timeline() { + let baseline = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-TARGETDURATION:4\n\ + #EXT-X-MAP:URI=\"init.mp4\"\n\ + #EXTINF:4.0,\nold1.m4s\n#EXTINF:4.0,\nold2.m4s\n#EXTINF:4.0,\nold3.m4s\n"; + let candidate = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:40\n#EXT-X-TARGETDURATION:4\n\ + #EXTINF:4.0,\nnew40.ts\n#EXTINF:4.0,\nnew41.ts\n#EXTINF:4.0,\nnew42.ts\n"; + + assert_incompatible_switch_is_rejected_before_timeline_commit( + baseline, + candidate, + HlsManifestRejectLogReason::SwitchMapResetUnsupported, + ) + .await; +} diff --git a/backend/hls/src/refresh/tests/headers_redirects.rs b/backend/hls/src/refresh/tests/headers_redirects.rs new file mode 100644 index 000000000..c1aa7e8c6 --- /dev/null +++ b/backend/hls/src/refresh/tests/headers_redirects.rs @@ -0,0 +1,400 @@ +use super::{ + commit_fetched_manifest, fetch_and_commit_manifest_with_policy, fetch_hls_origin_manifest_request, + fetched_manifest, hls_manifest_redirect_host, manifest_body, no_delay_policy, + refresh_from_live_hls_entrypoint_with_retries, request_header_value, retry_after_delay_ms, spawn_test_origin, + test_app_config, test_origin_refresh_request, test_recovery_timing_policy, test_segment_repair_manager, + test_session, three_segment_manifest_body, HlsManifestCommitError, HlsManifestCommitRequirement, + HlsOriginManifestFetchContext, HlsOriginManifestFetchRequest, LiveHlsOriginEntry, OriginRefreshRequest, +}; +use crate::{ + manifest_fetch::fetched_effective_manifest_host, refresh::maybe_trigger_origin_refresh, + HlsManifestAcceptanceDirective, HlsMapWorkerPool, HlsOriginAccountBinding, HlsProxyManager, HlsSegmentCache, + HlsSegmentWorkerPool, HlsSession, HlsSessionKey, HlsSessionMode, TransientPassthroughReason, +}; +use axum::http::{header, HeaderMap, HeaderName, HeaderValue}; +use shared::model::HlsStripMode; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; +use tuliprox_core::model::{HlsManifestRecoveryBurstConfig, ReverseProxyDisabledHeaderConfig, StripConfig}; +use url::Url; + +#[test] +fn retry_after_header_is_parsed_as_milliseconds() { + let mut headers = HeaderMap::new(); + headers.insert(header::RETRY_AFTER, HeaderValue::from_static("3")); + assert_eq!(retry_after_delay_ms(&headers), Some(3_000)); +} + +#[tokio::test] +async fn refresh_stores_headers_after_hls_origin_policy() { + let session = test_session(); + let server = spawn_test_origin(Arc::new(|_path| { + (200, Vec::new(), "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nseg.ts\n".to_string()) + })) + .await; + let entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) + .expect("valid origin entry"); + let mut headers = HeaderMap::new(); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); + headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); + headers.insert(HeaderName::from_static("proxy-authorization"), HeaderValue::from_static("Basic secret")); + headers.insert(header::HOST, HeaderValue::from_static("proxy.example.com")); + headers.insert(HeaderName::from_static("x-blocked"), HeaderValue::from_static("blocked")); + headers.insert(HeaderName::from_static("cf-ray"), HeaderValue::from_static("cf")); + headers.insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("de")); + + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers, + origin_provider_session_headers: HeaderMap::new(), + disabled_headers: Some(ReverseProxyDisabledHeaderConfig { + referer_header: false, + x_header: true, + cloudflare_header: true, + custom_header: Vec::new(), + }), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + fresh_manifest_requirement_generation: None, + acceptance_directive: HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + now_ms: 100, + origin_io: None, + post_refresh_runtime: None, + }; + + assert!(maybe_trigger_origin_refresh(request).await); + for _ in 0..50 { + if session.read().await.last_rendered_manifest.is_some() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + let session = session.read().await; + assert!(!session.origin_request_headers.contains_key(header::AUTHORIZATION)); + assert!(!session.origin_request_headers.contains_key(header::COOKIE)); + assert!(!session.origin_request_headers.contains_key("proxy-authorization")); + assert!(!session.origin_request_headers.contains_key(header::HOST)); + assert!(!session.origin_request_headers.contains_key("x-blocked")); + assert!(!session.origin_request_headers.contains_key("cf-ray")); + assert_eq!(session.origin_request_headers.get(header::ACCEPT_LANGUAGE).expect("language"), "de"); +} + +#[test] +fn origin_account_binding_change_clears_provider_session_headers() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "1"), b"secret", 100); + session.origin_provider_session_headers.insert(header::COOKIE, HeaderValue::from_static("sid=abc")); + session.replace_origin_account_binding(Some(HlsOriginAccountBinding::new( + Arc::::from("input"), + Arc::::from("account-a"), + &session.proxy_session_id.clone(), + 100, + ))); + assert!(session.origin_provider_session_headers.is_empty()); + + session.origin_provider_session_headers.insert(header::COOKIE, HeaderValue::from_static("sid=next")); + session.replace_origin_account_binding(Some(HlsOriginAccountBinding::new( + Arc::::from("input"), + Arc::::from("account-a"), + &session.proxy_session_id.clone(), + 200, + ))); + assert!(!session.origin_provider_session_headers.is_empty()); + + session.replace_origin_account_binding(Some(HlsOriginAccountBinding::new( + Arc::::from("input"), + Arc::::from("account-b"), + &session.proxy_session_id.clone(), + 300, + ))); + assert!(session.origin_provider_session_headers.is_empty()); +} + +#[test] +fn transient_commit_accepts_plausible_same_redirect_host_rollover_and_resets_highwater() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.origin_seq_highwater = Some(758); + session.last_effective_manifest_host = Some("origin.example.com".to_string()); + session.mark_authorized_media_access(100); + let previous_manifest = + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:757\n#EXTINF:4.0,\n/hls/shared/live/session/lease/r/old.ts\n".to_string(); + session.transient.replace_manifest_with_semantics(previous_manifest.clone(), 10, None); + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(result.is_ok()); + assert_ne!(session.transient.last_manifest_body.as_deref(), Some(previous_manifest.as_str())); + assert_eq!(session.origin_seq_highwater, Some(0)); + assert!(session.transient.last_manifest_body.as_ref().is_some_and(|body| body.contains("/r/"))); +} + +#[test] +fn transient_commit_with_different_redirect_host_is_held_as_candidate() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.origin_seq_highwater = Some(758); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:758\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg758.ts\n#EXTINF:4.0,\nseg759.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(matches!(result, Err(HlsManifestCommitError::RetryCurrentTarget))); + assert_eq!(session.origin_seq_highwater, Some(758)); + assert!(session.transient.last_manifest_body.is_none()); +} + +#[test] +fn provider_failover_mirror_without_redirect_uses_resolved_host_signal() { + let mut fetched = fetched_manifest("#EXTM3U\n#EXTINF:4.0,\nseg.ts\n"); + fetched.redirect_host = None; + fetched.resolved_request_url = "http://mirror.example.com/live/user/pass/12345.m3u8".to_string(); + fetched.provider_url_index = Some(1); + + assert_eq!(fetched_effective_manifest_host(&fetched).as_deref(), Some("mirror.example.com")); +} + +#[test] +fn provider_failover_with_redirect_uses_redirect_host_as_manifest_host_signal() { + let mut fetched = fetched_manifest("#EXTM3U\n#EXTINF:4.0,\nseg.ts\n"); + fetched.redirect_host = Some("redirect.example.com".to_string()); + fetched.resolved_request_url = "http://mirror.example.com/live/user/pass/12345.m3u8".to_string(); + fetched.provider_url_index = Some(1); + + assert_eq!(fetched_effective_manifest_host(&fetched).as_deref(), Some("redirect.example.com")); +} + +#[test] +fn manifest_redirect_host_is_only_set_for_actual_redirect_host_switch() { + let resolved = Url::parse("http://mirror.example.com/live/user/pass/12345.m3u8").expect("resolved url"); + let same_target = Url::parse("http://mirror.example.com/live/user/pass/12345.m3u8").expect("same url"); + let redirected = Url::parse("http://cdn.example.net/live/play/12345.m3u8").expect("redirect url"); + + assert_eq!(hls_manifest_redirect_host(&resolved, &same_target), None); + assert_eq!(hls_manifest_redirect_host(&resolved, &redirected).as_deref(), Some("cdn.example.net")); +} + +#[tokio::test] +async fn shared_initial_manifest_automatic_cross_origin_redirect_keeps_identity_and_scrubs_credentials() { + let target = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let target_url = format!("{}/final/manifest.m3u8", target.base_url); + let expected_target_url = target_url.clone(); + let redirect = + spawn_test_origin(Arc::new(move |_path| (302, vec![("Location", target_url.clone())], String::new()))).await; + let origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", redirect.base_url)).expect("entry url"); + let mut headers = HeaderMap::new(); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("br")); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer origin-secret")); + headers.insert(header::COOKIE, HeaderValue::from_static("sid=origin-secret")); + let context = HlsOriginManifestFetchContext { + app_config: test_app_config(), + session: test_session(), + origin_entry, + headers, + client: reqwest::Client::builder().no_proxy().build().expect("client"), + no_redirect_client: reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client"), + use_manual_redirects: false, + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + retry_policy: no_delay_policy(), + recovery_timing_policy: test_recovery_timing_policy(2_000), + acceptance_timing_seed: None, + }; + + let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) + .await + .expect("automatic redirect should return the decoded manifest"); + + assert_eq!(fetched.body, manifest_body()); + assert_eq!(fetched.attempts, 1); + assert_eq!(fetched.final_manifest_url, expected_target_url); + let redirect_requests = redirect.raw_requests.lock().await; + let target_requests = target.raw_requests.lock().await; + assert_eq!(redirect_requests.len(), 1); + assert_eq!(target_requests.len(), 1); + assert_eq!(request_header_value(&redirect_requests[0], "accept-encoding"), Some("identity")); + assert_eq!(request_header_value(&target_requests[0], "accept-encoding"), Some("identity")); + assert_eq!(request_header_value(&redirect_requests[0], "authorization"), Some("Bearer origin-secret")); + assert_eq!(request_header_value(&redirect_requests[0], "cookie"), Some("sid=origin-secret")); + assert!(request_header_value(&target_requests[0], "authorization").is_none()); + assert!(request_header_value(&target_requests[0], "cookie").is_none()); +} + +#[tokio::test] +async fn shared_initial_manifest_budget_covers_status_decoder_redirect_and_success() { + let target_hits = Arc::new(AtomicUsize::new(0)); + let target_hits_for_handler = Arc::clone(&target_hits); + let server = spawn_test_origin(Arc::new(move |path| { + if path == "/live/user/pass/12345.m3u8" { + return (302, vec![("Location", "/live/play/once/12345".to_string())], String::new()); + } + if path == "/live/play/once/12345" { + return match target_hits_for_handler.fetch_add(1, Ordering::SeqCst) { + 0 => (500, Vec::new(), "temporary".to_string()), + 1 => (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()), + _ => (200, Vec::new(), manifest_body()), + }; + } + (404, Vec::new(), String::new()) + })) + .await; + let origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + let context = HlsOriginManifestFetchContext { + app_config: test_app_config(), + session: test_session(), + origin_entry, + headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client"), + use_manual_redirects: true, + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + retry_policy: no_delay_policy(), + recovery_timing_policy: test_recovery_timing_policy(2_000), + acceptance_timing_seed: None, + }; + + let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) + .await + .expect("third logical attempt should succeed"); + + assert_eq!(fetched.body, manifest_body()); + assert_eq!(fetched.attempts, 3); + assert_eq!( + *server.requests.lock().await, + vec![ + "/live/user/pass/12345.m3u8", + "/live/play/once/12345", + "/live/user/pass/12345.m3u8", + "/live/play/once/12345", + "/live/user/pass/12345.m3u8", + "/live/play/once/12345", + ] + ); + assert!(server + .raw_requests + .lock() + .await + .iter() + .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); +} + +#[tokio::test] +async fn manifest_retry_starts_at_entrypoint_after_redirect_failure() { + let redirect_hits = Arc::new(AtomicUsize::new(0)); + let redirect_hits_for_handler = Arc::clone(&redirect_hits); + let server = spawn_test_origin(Arc::new(move |path| { + if path == "/live/user/pass/12345.m3u8" { + return (302, vec![("Location", "/live/play/once/12345".to_string())], String::new()); + } + if path == "/live/play/once/12345" { + let hit = redirect_hits_for_handler.fetch_add(1, Ordering::SeqCst); + if hit < 2 { + return (500, Vec::new(), "fail".to_string()); + } + return (200, Vec::new(), manifest_body()); + } + (404, Vec::new(), String::new()) + })) + .await; + let entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + let no_redirect_client = + reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()).build().expect("client builds"); + + let fetched = refresh_from_live_hls_entrypoint_with_retries( + &entry, + &HeaderMap::new(), + &reqwest::Client::new(), + &no_redirect_client, + true, + 2_000, + &no_delay_policy(), + ) + .await + .expect("refresh eventually succeeds"); + + assert_eq!(fetched.attempts, 3); + assert_eq!( + *server.requests.lock().await, + vec![ + "/live/user/pass/12345.m3u8", + "/live/play/once/12345", + "/live/user/pass/12345.m3u8", + "/live/play/once/12345", + "/live/user/pass/12345.m3u8", + "/live/play/once/12345" + ] + ); +} + +#[tokio::test] +async fn successful_redirected_baseline_binds_the_original_request_entry() { + let target = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(100)))).await; + let target_url = format!("{}/redirected/final.m3u8", target.base_url); + let target_url_for_handler = target_url.clone(); + let entry = spawn_test_origin(Arc::new(move |_path| { + (302, vec![("Location", target_url_for_handler.clone())], String::new()) + })) + .await; + let request_url = format!("{}/live/user/pass/12345.m3u8?token=fixed", entry.base_url); + let session = test_session(); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = LiveHlsOriginEntry::parse(&request_url).expect("redirecting entry URL"); + + let committed = + fetch_and_commit_manifest_with_policy(&mut request).await.expect("redirected initial manifest commits"); + + assert_eq!(committed.fetched.final_manifest_url, target_url); + assert_eq!(committed.fetched.resolved_request_url, request_url); + let binding_url = { + let session = session.read().await; + session + .origin_control + .manifest_origin_binding + .as_ref() + .expect("redirected commit stores binding") + .request_url() + .to_string() + }; + assert_eq!(binding_url, request_url); + assert_eq!(entry.requests.lock().await.as_slice(), ["/live/user/pass/12345.m3u8?token=fixed"]); + assert_eq!(target.requests.lock().await.as_slice(), ["/redirected/final.m3u8"]); +} diff --git a/backend/hls/src/refresh/tests/origin_binding.rs b/backend/hls/src/refresh/tests/origin_binding.rs new file mode 100644 index 000000000..80534fc1b --- /dev/null +++ b/backend/hls/src/refresh/tests/origin_binding.rs @@ -0,0 +1,624 @@ +use super::{ + assert_stale_switch_generation_rejects_commit, bind_refresh_request_to_app_state, classify_manifest_fetch_failure, + commit_fetched_manifest, fetch_and_commit_manifest_with_policy, fetched_manifest, host_from_base_url, + install_published_recovery_binding, manifest_body, manifest_fetch_context, mark_origin_refresh_started, + no_delay_policy, path_has_extension, prepare_cross_host_baseline, publish_ready_test_manifest, refresh_and_commit, + request_header_value, resolved_hls_manifest_request_url_from_input, retry_test_manifest_recovery_chain, + spawn_test_origin, switch_manifest_body, switch_test_request, test_app_config, test_origin_refresh_request, + test_segment_repair_manager, test_session, three_segment_manifest_body, trigger_origin_refresh_sync, + HlsManifestAcceptanceTrigger, HlsManifestCommitRequirement, HlsManifestFetchFailureKind, + HlsManifestFetchFailureSignal, HlsManifestRecoveryUnavailableReason, HlsManifestRejectLogReason, + LiveHlsOriginEntry, OriginManifestFetchError, OriginRefreshRequest, StaleSwitchGeneration, +}; +use crate::{ + refresh::maybe_trigger_origin_refresh, HlsFreshManifestRequiredReason, HlsManifestAcceptanceDirective, + HlsMapWorkerPool, HlsProxyManager, HlsSegmentCache, HlsSegmentWorkerPool, HlsSession, HlsSessionKey, + SegmentCacheStatus, +}; +use axum::http::{header, HeaderMap}; +use shared::model::{ConfigProviderDto, HlsManifestRecoveryBurstLevel, HlsStripMode, ProviderUrlSelectionPolicy}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; +use tokio::sync::RwLock; +use tuliprox_core::model::{Config, ConfigProvider, HlsManifestRecoveryBurstConfig, StripConfig}; +use url::Url; + +#[test] +fn resolved_hls_manifest_request_url_uses_provider_index_locally() { + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec!["http://provider-a.example".into(), "http://provider-b.example".into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), + dns: None, + })); + let provider_entry = + LiveHlsOriginEntry::parse_with_url_failover_provider("provider://demo/live/u/p/1.m3u8", Some(provider)) + .unwrap(); + + let resolved = + resolved_hls_manifest_request_url_from_input(&provider_entry.to_input_source(), Some(1), provider_entry.url()); + assert_eq!(resolved.as_str(), "http://provider-b.example/live/u/p/1.m3u8"); + assert!(!resolved.as_str().contains("provider://")); + + let direct_entry = LiveHlsOriginEntry::parse("http://origin.example/live/u/p/1.m3u8").unwrap(); + assert_eq!( + resolved_hls_manifest_request_url_from_input(&direct_entry.to_input_source(), Some(1), direct_entry.url()) + .as_str(), + "http://origin.example/live/u/p/1.m3u8" + ); +} + +#[test] +fn fresh_revalidation_rebases_normal_manifest_on_the_pinned_host() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.origin_seq_highwater = Some(1_000); + session.last_effective_manifest_host = Some("origin.example.com".to_string()); + let mut request = test_origin_refresh_request(test_session()); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + let fetched = + fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:4.0,\nseg10.ts\n"); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(result.is_ok()); + assert_eq!(session.origin_seq_highwater, Some(10)); + assert_eq!(session.origin_epoch, 1); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some("origin.example.com")); +} + +#[tokio::test] +async fn accepted_manifest_commit_stores_provider_session_cookie_separately() { + let session = test_session(); + let server = spawn_test_origin(Arc::new(|_path| { + ( + 200, + vec![ + ("Set-Cookie", "sid=abc; Path=/; HttpOnly".to_string()), + ("Set-Cookie", "pref=1; SameSite=Lax".to_string()), + ], + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nseg.ts\n".to_string(), + ) + })) + .await; + let entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) + .expect("valid origin entry"); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = entry; + + assert!(Box::pin(trigger_origin_refresh_sync(request)).await); + + let session = session.read().await; + assert!(!session.origin_request_headers.contains_key(header::COOKIE)); + assert_eq!( + session.origin_provider_session_headers.get(header::COOKIE).expect("provider cookie"), + "sid=abc; pref=1" + ); +} + +#[tokio::test] +async fn provider_failover_status_does_not_count_as_hls_retry() { + let first = spawn_test_origin(Arc::new(|_path| (407, Vec::new(), "rotate".to_string()))).await; + let second = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + })); + let session = test_session(); + let initial_session_key = session.read().await.key.stable_value(); + let initial_proxy_session_id = session.read().await.proxy_session_id.clone(); + let entry = LiveHlsOriginEntry::parse_with_url_failover_provider( + "provider://demo/live/user/pass/12345.m3u8", + Some(Arc::clone(&provider)), + ) + .expect("provider entry url"); + let segment_worker_pool = Arc::new(HlsSegmentWorkerPool::default()); + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::clone(&segment_worker_pool), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + fresh_manifest_requirement_generation: None, + acceptance_directive: HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + post_refresh_runtime: None, + }; + + assert!(maybe_trigger_origin_refresh(request).await); + for _ in 0..50 { + if session.read().await.origin_seq_highwater == Some(102) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + let first_manifest_requests = + first.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + let second_manifest_requests = + second.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(first_manifest_requests, 1); + assert_eq!(second_manifest_requests, 1); + for server in [&first, &second] { + let manifest_requests = server + .raw_requests + .lock() + .await + .iter() + .filter(|request| request.lines().next().is_some_and(|line| line.contains(" /live/user/pass/12345.m3u8 "))) + .cloned() + .collect::>(); + assert_eq!(manifest_requests.len(), 1); + assert_eq!(request_header_value(&manifest_requests[0], "accept-encoding"), Some("identity")); + } + let session = session.read().await; + assert_eq!(session.key.stable_value(), initial_session_key); + assert_eq!(session.proxy_session_id, initial_proxy_session_id); + assert!(!session.key.stable_value().contains("provider://")); + assert!(!session.key.stable_value().contains(first.base_url.as_str())); + assert!(!session.key.stable_value().contains(second.base_url.as_str())); + assert_eq!(session.origin_seq_highwater, Some(0)); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some("127.0.0.1")); + let metrics = segment_worker_pool.metrics().snapshot(); + assert_eq!(metrics.refresh_started, 1); + assert_eq!(metrics.refresh_completed, 1); + assert_eq!(metrics.refresh_retried, 0); + assert_eq!(metrics.refresh_failed, 0); +} + +pub(in crate::refresh::tests) fn assert_binding_superseded_error(error: &OriginManifestFetchError) { + assert!(matches!( + error, + OriginManifestFetchError::RecoveryUnavailable { + reason: HlsManifestRecoveryUnavailableReason::BindingSuperseded, + } + )); + assert_eq!(error.log_label(), "recovery_binding_superseded"); + assert_eq!(error.to_string(), "origin manifest recovery unavailable: manifest origin binding superseded"); + assert_eq!( + classify_manifest_fetch_failure(error), + HlsManifestFetchFailureSignal::discarded(HlsManifestFetchFailureKind::Superseded) + ); +} + +#[tokio::test] +async fn recovery_binding_supersession_is_discarded_without_failure_signal() { + let origin = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(100)))).await; + let hls_ctx = crate::HlsCtx::for_test(Config::default()); + let ctx = &hls_ctx; + let session = Arc::new(RwLock::with_max_readers( + HlsSession::new(HlsSessionKey::new(1, "binding-supersession"), b"secret", 0), + 1, + )); + let mut request = bind_refresh_request_to_app_state(test_origin_refresh_request(Arc::clone(&session)), ctx); + let baseline_binding_url = format!("{}/live/a/index.m3u8", origin.base_url); + request.origin_entry = LiveHlsOriginEntry::parse(&baseline_binding_url).expect("binding A entry URL"); + request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; + + fetch_and_commit_manifest_with_policy(&mut request).await.expect("binding A baseline commits"); + publish_ready_test_manifest(&session, 200).await; + let baseline_binding = + session.read().await.established_manifest_recovery_binding().expect("published baseline has binding A"); + let requests_before_supersession = origin.requests.lock().await.len(); + + let replacement_binding_url = format!("{}/live/b/index.m3u8", origin.base_url); + let mut binding_b_manifest = fetched_manifest(&three_segment_manifest_body(103)); + binding_b_manifest.resolved_request_url.clone_from(&replacement_binding_url); + binding_b_manifest.final_manifest_url.clone_from(&replacement_binding_url); + binding_b_manifest.redirect_host = Some(host_from_base_url(&origin.base_url)); + let metrics = Arc::clone(request.segment_worker_pool.metrics()); + assert!(mark_origin_refresh_started(&mut request, 300).await); + let verification_request = request.clone(); + let replacement_request = request.clone(); + let read_blocker = session.read().await; + let refresh = tokio::spawn(refresh_and_commit(request, 300)); + tokio::task::yield_now().await; + let replacement_session = Arc::clone(&session); + let replacement = tokio::spawn(async move { + let mut session = replacement_session.write().await; + commit_fetched_manifest(&mut session, &binding_b_manifest, &replacement_request, 325) + .expect("newer binding B manifest commits"); + session.origin_control.record_origin_response(275); + session.origin_control.path_condition = + super::super::super::origin_progress::HlsOriginPathCondition::PublicationLate; + session.origin_refresh.consecutive_failures = 3; + session.origin_refresh.last_error_at_ms = Some(250); + }); + tokio::task::yield_now().await; + drop(read_blocker); + replacement.await.expect("replacement binding task joins"); + refresh.await.expect("superseded refresh task joins"); + + let metrics_after = metrics.snapshot(); + let replacement_binding = { + let session = session.read().await; + assert_eq!(session.origin_control.last_origin_response_at_ms, Some(275)); + assert_eq!( + session.origin_control.path_condition, + super::super::super::origin_progress::HlsOriginPathCondition::PublicationLate + ); + assert_eq!(session.origin_refresh.consecutive_failures, 3); + assert_eq!(session.origin_refresh.last_error_at_ms, Some(250)); + assert!(!session.origin_refresh.in_flight); + assert!(session.origin_control.acceptance_episode.is_none()); + session.origin_control.manifest_origin_binding.clone().expect("newer binding B remains installed") + }; + assert_ne!(baseline_binding, replacement_binding); + assert_eq!(metrics_after.refresh_started, 1); + assert_eq!(metrics_after.refresh_completed, 0); + assert_eq!(metrics_after.refresh_retried, 0); + assert_eq!(metrics_after.refresh_failed, 0); + assert_eq!(metrics_after.refresh_skipped, 1); + assert_eq!(ctx.hls_proxy.availability_reevaluations().owner_count(), 0); + assert_eq!(origin.requests.lock().await.len(), requests_before_supersession); + + let Err(error) = super::super::recover_manifest_for_request( + &manifest_fetch_context(&verification_request), + &verification_request, + super::super::HlsManifestRecoveryPath { + binding: baseline_binding, + reject_reason: None, + deterministic_conflict: None, + trigger: HlsManifestAcceptanceTrigger::RecoveryRequired, + diagnostic: super::super::HlsRecoveryTriggerDiagnostic::new(super::super::HlsRecoveryTriggerSource::Other), + }, + ) + .await + else { + panic!("superseded recovery binding must be discarded"); + }; + assert_binding_superseded_error(&error); + assert_eq!(origin.requests.lock().await.len(), requests_before_supersession); +} + +#[tokio::test] +async fn provider_failover_initial_success_commits_without_hls_host_retry_when_unpinned() { + let first = spawn_test_origin(Arc::new(|_path| (407, Vec::new(), "rotate".to_string()))).await; + let second_hits = Arc::new(AtomicUsize::new(0)); + let second_hits_for_handler = Arc::clone(&second_hits); + let second = spawn_test_origin(Arc::new(move |_path| { + let hit = second_hits_for_handler.fetch_add(1, Ordering::SeqCst); + if hit == 0 { + return ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:102\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n102.ts\n".to_string(), + ); + } + ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n#EXTINF:4.0,\n102.ts\n" + .to_string(), + ) + })) + .await; + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + })); + let session = test_session(); + { + session.write().await.origin_seq_highwater = Some(100); + } + let entry = LiveHlsOriginEntry::parse_with_url_failover_provider( + "provider://demo/live/user/pass/12345.m3u8", + Some(Arc::clone(&provider)), + ) + .expect("provider entry url"); + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + fresh_manifest_requirement_generation: None, + acceptance_directive: HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + post_refresh_runtime: None, + }; + + assert!(maybe_trigger_origin_refresh(request).await); + for _ in 0..50 { + if session.read().await.origin_seq_highwater == Some(102) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + let first_manifest_requests = + first.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + let second_manifest_requests = + second.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(first_manifest_requests, 1); + assert_eq!(second_manifest_requests, 1); + assert_eq!(session.read().await.origin_seq_highwater, Some(102)); +} + +#[tokio::test] +async fn fresh_pinned_revalidation_same_host_rebase_runs_complete_beast_plan() { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let origin = spawn_test_origin(Arc::new(|path| { + if path_has_extension(&path, "m3u8") { + ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:10\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n10.ts\n#EXTINF:4.0,\n11.ts\n" + .to_string(), + ) + } else { + (404, Vec::new(), String::new()) + } + })) + .await; + let effective_host = host_from_base_url(&origin.base_url); + let baseline_body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1000\n#EXT-X-TARGETDURATION:4\n\ + #EXTINF:4.0,\n1000.ts\n#EXTINF:4.0,\n1001.ts\n"; + let baseline_manifest_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); + let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(baseline) = + tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest(baseline_body, &baseline_manifest_url) + else { + panic!("same-host stale baseline parses"); + }; + let session = test_session(); + let baseline_epoch = { + let mut session = session.write().await; + session + .apply_origin_manifest_for_host(&baseline, crate::timeline::effective_origin_host_id(&effective_host)) + .expect("same-host stale baseline commits"); + session.last_effective_manifest_host = Some(effective_host.clone()); + session.origin_control.pinned_host = Some(effective_host.clone()); + session.origin_control.origin_epoch = session.origin_epoch; + session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::PreviousHardManifestFailure); + session.origin_epoch + }; + install_published_recovery_binding(&session, &baseline_manifest_url, None).await; + let cache_dir = tempfile::tempdir().expect("fresh pinned revalidation cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(cache_dir.path())); + let mut request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::PreviousHardManifestFailure, + }; + request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::Observe; + + assert!(trigger_origin_refresh_sync(request).await); + + let manifest_requests = + origin.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(manifest_requests, plan.total_candidates()); + let next_fetch_allowed_at_ms = { + let session = session.read().await; + assert_eq!(session.origin_epoch, baseline_epoch.saturating_add(1)); + assert_eq!(session.origin_seq_highwater, Some(11)); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some(effective_host.as_str())); + assert_eq!(session.fresh_manifest_commit_required, None); + let rebased_head = session + .segments + .values() + .find(|segment| segment.origin_key.origin_epoch == session.origin_epoch) + .expect("same-host rebased timeline head"); + assert!(rebased_head.discontinuity_before); + session.origin_refresh.next_fetch_allowed_at_ms + }; + + let mut follow_up = switch_test_request(Arc::clone(&session), cache, &origin.base_url); + follow_up.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + follow_up.manifest_commit_requirement = session + .read() + .await + .fresh_manifest_commit_required + .map_or(HlsManifestCommitRequirement::CommittedManifestAllowed, |reason| { + HlsManifestCommitRequirement::FreshCommitRequired { reason } + }); + follow_up.now_ms = next_fetch_allowed_at_ms; + + assert!(trigger_origin_refresh_sync(follow_up).await); + let manifest_requests_after_follow_up = + origin.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(manifest_requests_after_follow_up, plan.total_candidates().saturating_add(1)); +} + +pub(in crate::refresh::tests) async fn prepare_ready_content_anchor( + session: &Arc>, + cache: &HlsSegmentCache, + bytes: &[u8], +) { + let key = { + let mut session = session.write().await; + let segment = session.segments.values_mut().next().expect("baseline segment"); + segment.status = SegmentCacheStatus::Ready { + content_length: u64::try_from(bytes.len()).unwrap_or(u64::MAX), + ready_at_ms: 1, + }; + segment.cache_key.clone() + }; + cache.write_bytes_and_commit(&key, bytes).await.expect("committed anchor bytes"); +} + +pub(in crate::refresh::tests) async fn assert_cross_host_replay_does_not_commit(candidate_bytes: &'static str) { + const COMMITTED_BYTES: &[u8] = b"committed-media-bytes"; + let manifest = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nold500.ts\n".to_string(); + let origin = spawn_test_origin(Arc::new(move |path| { + if path_has_extension(&path, "m3u8") { + (200, Vec::new(), manifest.clone()) + } else if path.ends_with("/old500.ts") { + (200, Vec::new(), candidate_bytes.to_string()) + } else { + (404, Vec::new(), String::new()) + } + })) + .await; + let cache_dir = tempfile::tempdir().expect("content anchor cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(cache_dir.path())); + let session = test_session(); + prepare_cross_host_baseline(&session).await; + prepare_ready_content_anchor(&session, &cache, COMMITTED_BYTES).await; + let baseline_epoch = session.read().await.origin_epoch; + let mut request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Friendly }; + let target_url = + Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("content anchor target"); + + let result = retry_test_manifest_recovery_chain( + &request, + target_url, + HlsManifestRejectLogReason::PinnedHostRecoveryRejected, + ) + .await; + + let requests = origin.requests.lock().await.clone(); + let session = session.read().await; + assert!(result.is_err(), "replay-only cross-host candidate committed: requests={requests:?}"); + assert_eq!(session.origin_epoch, baseline_epoch); +} + +#[tokio::test] +async fn same_cross_host_sequence_path_and_equal_bytes_without_forward_progress_never_commits() { + assert_cross_host_replay_does_not_commit("committed-media-bytes").await; +} + +#[tokio::test] +async fn same_cross_host_sequence_and_path_with_different_bytes_never_commits_as_anchor() { + assert_cross_host_replay_does_not_commit("different-media-bytes").await; +} + +#[tokio::test] +async fn pinned_origin_recovery_during_staging_rejects_alternative_commit() { + assert_stale_switch_generation_rejects_commit(StaleSwitchGeneration::PinnedHostRecovered).await; +} + +#[tokio::test] +async fn recovery_required_orchestrator_requalifies_changed_pinned_landscape_with_full_plan() { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let entry_hits = Arc::new(AtomicUsize::new(0)); + let entry_hits_for_handler = Arc::clone(&entry_hits); + let pinned_hits = Arc::new(AtomicUsize::new(0)); + let pinned_hits_for_handler = Arc::clone(&pinned_hits); + let origin_port = Arc::new(AtomicUsize::new(0)); + let origin_port_for_handler = Arc::clone(&origin_port); + let origin = spawn_test_origin(Arc::new(move |path| { + if path == "/live/user/pass/12345.m3u8" { + let hit = entry_hits_for_handler.fetch_add(1, Ordering::SeqCst); + if hit < plan.total_candidates() { + return (200, Vec::new(), switch_manifest_body(false)); + } + return ( + 302, + vec![( + "Location", + format!("http://127.0.0.1:{}/pinned/index.m3u8", origin_port_for_handler.load(Ordering::SeqCst)), + )], + String::new(), + ); + } + if path == "/pinned/index.m3u8" { + pinned_hits_for_handler.fetch_add(1, Ordering::SeqCst); + return ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\npinned101.ts\n".to_string(), + ); + } + (404, Vec::new(), String::new()) + })) + .await; + let parsed_origin_url = Url::parse(&origin.base_url).expect("origin base URL"); + origin_port.store(usize::from(parsed_origin_url.port().expect("test origin port")), Ordering::SeqCst); + + let session = test_session(); + let baseline = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\npinned100.ts\n"; + let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(baseline) = + tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( + baseline, + "http://127.0.0.1/pinned/index.m3u8", + ) + else { + panic!("pinned baseline parses"); + }; + { + let mut session = session.write().await; + session + .apply_origin_manifest_for_host(&baseline, crate::timeline::effective_origin_host_id("127.0.0.1")) + .expect("pinned baseline commits"); + session.last_effective_manifest_host = Some("127.0.0.1".to_string()); + session.origin_control.pinned_host = Some("127.0.0.1".to_string()); + session.origin_control.origin_epoch = session.origin_epoch; + } + + let temp_dir = tempfile::tempdir().expect("orchestrator cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let mut request = switch_test_request(Arc::clone(&session), cache, &origin.base_url); + let alternative_entry = + format!("{}/live/user/pass/12345.m3u8", origin.base_url).replacen("127.0.0.1", "localhost", 1); + request.origin_entry = LiveHlsOriginEntry::parse(&alternative_entry).expect("alternative entry URL"); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + let target_url = Url::parse(&alternative_entry).expect("alternative target URL"); + + let committed = retry_test_manifest_recovery_chain( + &request, + target_url, + HlsManifestRejectLogReason::PinnedHostRecoveryRejected, + ) + .await + .expect("pinned follow-up commits"); + + assert_eq!(committed.fetched.redirect_host.as_deref(), Some("127.0.0.1")); + assert_eq!(entry_hits.load(Ordering::SeqCst), plan.total_candidates().saturating_mul(2).saturating_add(1)); + assert_eq!(pinned_hits.load(Ordering::SeqCst), plan.total_candidates().saturating_add(1)); + let session = session.read().await; + assert_eq!(session.origin_seq_highwater, Some(101)); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some("127.0.0.1")); + let episode = session.origin_control.acceptance_episode.as_ref().expect("completed acceptance episode"); + assert_eq!(episode.full_bursts_completed, 1); + assert_eq!(episode.state, super::super::super::manifest_acceptance::HlsManifestAcceptanceState::Completed); +} diff --git a/backend/hls/src/refresh/tests/ownership.rs b/backend/hls/src/refresh/tests/ownership.rs new file mode 100644 index 000000000..fc9501bd7 --- /dev/null +++ b/backend/hls/src/refresh/tests/ownership.rs @@ -0,0 +1,332 @@ +use super::{ + bind_refresh_request_to_app_state, build_manifest_refresh_timing, + cancel_superseded_terminal_work_after_media_progress, commit_fetched_manifest, fetched_manifest, + mark_origin_refresh_started, mark_origin_refresh_started_with_outcome, post_refresh_live_manifest_snapshot, + refresh_and_commit, spawn_test_origin, test_origin_refresh_request, test_session, + HlsAvailabilityReevaluationRegistration, HlsManifestCommitProgressEvidence, HlsManifestProgress, + HlsOriginRefreshTriggerOutcome, HlsPostRefreshAvailabilityAction, HlsPostRefreshAvailabilityReason, + HlsPreparedTerminalBundleKey, HlsRuntimeCustomTailReason, HlsSessionIncarnation, HlsTerminalAssetIdentity, + HlsTerminalAssetRevisionGuard, HlsTerminalPendingOwnerKey, HlsTerminalPendingRegistration, LiveHlsOriginEntry, + OriginRefreshState, +}; +use crate::{ + HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseTiming, HlsPlaybackFamilyKey, HlsProxyManager, HlsSessionKey, + ProxySessionId, SegmentCacheStatus, SegmentFetchPriority, +}; +use std::sync::Arc; +use tokio::sync::oneshot; +use tuliprox_core::{model::Config, utils::current_time_millis}; + +#[test] +fn origin_refresh_state_starts_only_when_due_and_not_in_flight() { + let mut state = OriginRefreshState { next_fetch_allowed_at_ms: 100, ..OriginRefreshState::default() }; + assert!(!state.is_due(99)); + assert!(state.is_due(100)); + state.mark_started(100); + assert!(!state.is_due(101)); +} + +#[tokio::test] +async fn hls_terminal_commit_media_progress_cancels_pending_owner_after_session_lock_release() { + let session = test_session(); + let request = test_origin_refresh_request(Arc::clone(&session)); + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let coordinator = request.hls_proxy.terminal_pending(); + let (started_tx, started_rx) = oneshot::channel(); + let (cancelled_tx, cancelled_rx) = oneshot::channel(); + let owner_key = HlsTerminalPendingOwnerKey { + session_incarnation: HlsSessionIncarnation::for_test(1), + proxy_session_id, + lease_id: HlsAccessLeaseId("progress-cancel".to_string()), + lease_issued_at_ms: 10, + expected_admission_generation: 20, + manifest_snapshot_generation: 30, + cursor_generation: 40, + decision_generation: 50, + reason: HlsRuntimeCustomTailReason::ChannelUnavailable, + bundle_key: HlsPreparedTerminalBundleKey { + asset: HlsTerminalAssetIdentity { revision: 60, fingerprint: [6; 32] }, + target_duration_ms: 4_000, + segment_count: super::super::HLS_TERMINAL_TAIL_SEGMENT_COUNT, + }, + latest_safe_commit_at_ms: 10_000, + }; + let asset_guard = HlsTerminalAssetRevisionGuard::matching_for_test(Some(owner_key.bundle_key.asset)); + assert_eq!( + coordinator.register(owner_key, &asset_guard, move |ownership| async move { + assert!(started_tx.send(()).is_ok()); + ownership.cancelled().await; + assert!(cancelled_tx.send(()).is_ok()); + }), + HlsTerminalPendingRegistration::Scheduled + ); + assert!(started_rx.await.is_ok()); + + let commit_result = Ok(( + HlsManifestCommitProgressEvidence::CacheTimeline(build_manifest_refresh_timing( + None, + Some(4_000), + HlsManifestProgress::Advanced, + )), + false, + false, + )); + cancel_superseded_terminal_work_after_media_progress(&request, &commit_result).await; + + assert!(cancelled_rx.await.is_ok()); + assert_eq!(coordinator.owner_count(), 0); +} + +#[tokio::test] +async fn successful_refresh_wakes_sleeping_post_refresh_owner() { + let hls_ctx = crate::HlsCtx::for_test(Config::default()); + let ctx = &hls_ctx; + let now_ms = current_time_millis(); + let progressed_body = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:481\n\ + #EXTINF:4,\n481.ts\n#EXTINF:4,\n482.ts\n#EXTINF:4,\n483.ts\n"; + let server = spawn_test_origin(Arc::new(move |_path| (200, Vec::new(), progressed_body.to_string()))).await; + let (session, _) = ctx + .hls_proxy + .get_or_create_session_with_outcome(HlsSessionKey::new(1, "post-refresh-wakeup"), b"secret", now_ms) + .await; + let manifest_url = format!("{}/live/user/pass/12345.m3u8", server.base_url); + let mut request = bind_refresh_request_to_app_state(test_origin_refresh_request(Arc::clone(&session)), ctx); + request.origin_entry = LiveHlsOriginEntry::parse(&manifest_url).expect("local successful origin entry"); + request.now_ms = now_ms; + let mut baseline = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:480\n\ + #EXTINF:4,\n480.ts\n#EXTINF:4,\n481.ts\n#EXTINF:4,\n482.ts\n", + ); + baseline.final_manifest_url = manifest_url.clone(); + baseline.resolved_request_url = manifest_url; + baseline.redirect_host = Some("127.0.0.1".to_string()); + { + let mut session = session.write().await; + commit_fetched_manifest(&mut session, &baseline, &request, now_ms).expect("baseline commits"); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: now_ms }; + } + session.render_and_store_manifest(now_ms).expect("baseline publishes"); + session.segments.get_mut(&1).expect("deferred boundary segment").status = + SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Prefetch, deferred_at_ms: now_ms }; + session.origin_control.path_condition = + super::super::super::origin_progress::HlsOriginPathCondition::AcceptanceConflict; + } + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("post-refresh-wakeup-lease".to_string()); + let mut lease = HlsAccessLease::pending( + lease_id, + HlsPlaybackFamilyKey::new("post-refresh-user", "post-refresh-client"), + proxy_session_id, + "post-refresh-user".to_string(), + "post-refresh-token".to_string(), + 1, + "post-refresh-stream".to_string(), + 1, + now_ms, + 60_000, + ); + lease.state = crate::HlsAccessLeaseState::Activated; + lease.active_until_ms = Some(now_ms.saturating_add(60_000)); + lease.pending_deadline = None; + lease.last_manifest_snapshot = Some(post_refresh_live_manifest_snapshot()); + ctx.hls_proxy.prepare_access_lease(lease).await; + + assert!(mark_origin_refresh_started(&mut request, now_ms).await); + let coordinator = ctx.hls_proxy.availability_reevaluations(); + let initial_permits = coordinator.available_task_permits_for_test(); + let (origin_progress_generation, media_readiness_generation) = { + let session = session.read().await; + (session.origin_control.progress_generation, session.activity.media_readiness_generation) + }; + assert_eq!( + super::super::super::availability::register_post_refresh_availability_reevaluation( + ctx.clone(), + Arc::clone(&session), + request.clone(), + HlsPostRefreshAvailabilityAction::Reevaluate { + reason: HlsPostRefreshAvailabilityReason::HardManifestFailure, + origin_progress_generation, + media_readiness_generation, + }, + ) + .await, + HlsAvailabilityReevaluationRegistration::Scheduled + ); + assert_eq!(coordinator.owner_count(), 1); + assert_eq!(coordinator.available_task_permits_for_test(), initial_permits.saturating_sub(1)); + let progress_generation_before = session.read().await.origin_control.progress_generation; + + refresh_and_commit(request, now_ms).await; + for _ in 0..256 { + if coordinator.owner_count() == 0 && coordinator.available_task_permits_for_test() == initial_permits { + break; + } + tokio::task::yield_now().await; + } + + assert!(session.read().await.origin_control.progress_generation > progress_generation_before); + assert_eq!(coordinator.owner_count(), 0); + assert_eq!(coordinator.available_task_permits_for_test(), initial_permits); +} + +#[tokio::test] +async fn concurrent_refresh_suppression_reports_in_flight_state() { + let session = test_session(); + session.write().await.origin_refresh.mark_started(900); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + + assert_eq!( + mark_origin_refresh_started_with_outcome(&mut request, 1_000).await, + HlsOriginRefreshTriggerOutcome::InFlight + ); + assert_eq!(session.read().await.origin_refresh.last_fetch_started_at_ms, Some(900)); +} + +#[tokio::test] +async fn hls_availability_reevaluation_cursor_evidence_supersedes_refresh_guard() { + let hls_proxy = Arc::new(HlsProxyManager::new()); + let (session, _) = hls_proxy + .get_or_create_session_with_outcome(HlsSessionKey::new(1, "cursor-pressure-guard"), b"secret", 100) + .await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("cursor-pressure-guard".to_string()); + { + let mut leases = hls_proxy.access_leases().write().await; + assert!(leases.prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("user", "client"), + proxy_session_id.clone(), + "user".to_string(), + "session-token".to_string(), + 1, + "cursor-pressure-guard".to_string(), + 1, + 100, + 60_000, + ))); + assert!(leases + .activate_access_lease( + &lease_id, + &proxy_session_id, + 100, + HlsAccessLeaseTiming { active_window_ms: 30_000, valid_window_ms: 60_000 }, + ) + .is_activated()); + } + let owner_key = hls_proxy + .availability_reevaluation_owner_key(&session, &proxy_session_id) + .await + .expect("live lease has availability evidence"); + { + let mut leases = hls_proxy.access_leases().write().await; + let identity = leases + .response_snapshot(&lease_id, &proxy_session_id, 100) + .and_then(|lease| lease.media_identity()) + .expect("live lease identity"); + assert!(leases + .record_segment_request_started_if_identity_matches(&lease_id, &proxy_session_id, identity, 20, 101,) + .is_some()); + } + let current_owner_key = hls_proxy + .availability_reevaluation_owner_key(&session, &proxy_session_id) + .await + .expect("updated availability evidence"); + assert!(current_owner_key.availability_evidence_generation > owner_key.availability_evidence_generation); + assert_eq!(current_owner_key.origin_progress_generation, owner_key.origin_progress_generation); + assert_eq!(current_owner_key.media_readiness_generation, owner_key.media_readiness_generation); + + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.hls_proxy = Arc::clone(&hls_proxy); + request.acceptance_directive.recovery_pressure_guard = + Some(super::super::super::availability_reevaluation::HlsRecoveryPressureGuard::from_owner_key(&owner_key)); + assert_eq!( + mark_origin_refresh_started_with_outcome(&mut request, 102).await, + HlsOriginRefreshTriggerOutcome::RecoveryPressureSuperseded + ); + assert!(!session.read().await.origin_refresh.in_flight); +} + +#[tokio::test] +async fn hls_availability_reevaluation_guard_contention_is_typed() { + let hls_proxy = Arc::new(HlsProxyManager::new()); + let (session, _) = hls_proxy + .get_or_create_session_with_outcome(HlsSessionKey::new(1, "contended-pressure-guard"), b"secret", 100) + .await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let owner_key = + hls_proxy.availability_reevaluation_owner_key(&session, &proxy_session_id).await.expect("session evidence"); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.hls_proxy = Arc::clone(&hls_proxy); + request.acceptance_directive.recovery_pressure_guard = + Some(super::super::super::availability_reevaluation::HlsRecoveryPressureGuard::from_owner_key(&owner_key)); + let lease_guard = hls_proxy.access_leases().write().await; + + assert_eq!( + mark_origin_refresh_started_with_outcome(&mut request, 100).await, + HlsOriginRefreshTriggerOutcome::RecoveryPressureStateContention + ); + assert!(!session.read().await.origin_refresh.in_flight); + drop(lease_guard); +} + +#[tokio::test] +async fn hls_availability_reevaluation_other_session_evidence_keeps_guard_current() { + let hls_proxy = Arc::new(HlsProxyManager::new()); + let (session, _) = + hls_proxy.get_or_create_session_with_outcome(HlsSessionKey::new(1, "guard-session-a"), b"secret", 100).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let owner_key = + hls_proxy.availability_reevaluation_owner_key(&session, &proxy_session_id).await.expect("session evidence"); + let other_proxy_session_id = ProxySessionId("guard-session-b".to_string()); + assert!(hls_proxy.access_leases().write().await.prepare_access_lease(HlsAccessLease::pending( + HlsAccessLeaseId("guard-session-b".to_string()), + HlsPlaybackFamilyKey::new("other", "client"), + other_proxy_session_id, + "other".to_string(), + "other-session".to_string(), + 1, + "guard-session-b".to_string(), + 2, + 100, + 60_000, + ))); + let guard = super::super::super::availability_reevaluation::HlsRecoveryPressureGuard::from_owner_key(&owner_key); + + assert!(matches!( + hls_proxy.with_current_recovery_pressure_session(&session, &guard, |_| 7_u8), + super::super::super::availability_reevaluation::HlsRecoveryPressureGuardAccess::Acquired(7) + )); +} + +#[tokio::test] +async fn hls_availability_reevaluation_removed_lease_supersedes_old_guard() { + let hls_proxy = Arc::new(HlsProxyManager::new()); + let (session, _) = hls_proxy + .get_or_create_session_with_outcome(HlsSessionKey::new(1, "removed-pressure-guard"), b"secret", 100) + .await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("removed-pressure-guard".to_string()); + assert!(hls_proxy.access_leases().write().await.prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("user", "client"), + proxy_session_id.clone(), + "user".to_string(), + "session-token".to_string(), + 1, + "removed-pressure-guard".to_string(), + 1, + 100, + 60_000, + ))); + let owner_key = + hls_proxy.availability_reevaluation_owner_key(&session, &proxy_session_id).await.expect("lease evidence"); + assert!(hls_proxy.access_leases().write().await.remove_access_lease(&lease_id).is_some()); + let guard = super::super::super::availability_reevaluation::HlsRecoveryPressureGuard::from_owner_key(&owner_key); + + assert!(matches!( + hls_proxy.with_current_recovery_pressure_session(&session, &guard, |_| ()), + super::super::super::availability_reevaluation::HlsRecoveryPressureGuardAccess::Superseded + )); + assert!(!session.read().await.origin_refresh.in_flight); +} diff --git a/backend/hls/src/refresh/tests/retry_policy.rs b/backend/hls/src/refresh/tests/retry_policy.rs new file mode 100644 index 000000000..1c7dba1ab --- /dev/null +++ b/backend/hls/src/refresh/tests/retry_policy.rs @@ -0,0 +1,1449 @@ +use super::{ + apply_manifest_fetch_failure_signal, bind_refresh_request_to_app_state, build_manifest_refresh_timing, + classify_manifest_fetch_failure, classify_origin_manifest_status, commit_fetched_manifest, + fetch_and_commit_manifest_with_policy, fetch_hls_origin_manifest_request, fetched_manifest, manifest_body, + manifest_hard_fetch_error, manifest_progress_from_highwater, mark_origin_refresh_started, + mark_origin_refresh_started_with_outcome, no_delay_policy, post_refresh_live_manifest_snapshot, + prepare_cross_host_baseline, record_committed_manifest_media_progress, record_committed_manifest_success, + refresh_from_live_hls_entrypoint_with_retries, refresh_origin_work_generation_matches, + refresh_session_with_origin_body, request_error_indicates_timeout, request_header_value, spawn_test_origin, + test_acceptance_episode_timing, test_app_config, test_deterministic_timeline_conflict, test_origin_refresh_request, + test_recovery_timing_policy, test_segment_repair_manager, test_session, three_segment_manifest_body, + trigger_origin_refresh_sync, HlsLeaseCutoverTiming, HlsManifestAcceptanceTrigger, + HlsManifestCommitProgressEvidence, HlsManifestCommitRequirement, HlsManifestFetchFailureKind, + HlsManifestFetchFailureSignal, HlsManifestHttpResponseEvidence, HlsManifestProgress, + HlsManifestRecoveryUnavailableReason, HlsManifestRejectLogReason, HlsManifestSequenceRelation, + HlsOriginManifestFetchContext, HlsOriginManifestFetchRequest, HlsOriginRefreshTriggerOutcome, + HlsPostRefreshAvailabilityAction, HlsPostRefreshAvailabilityReason, HlsTerminalCommitAcquisitionBudgetMs, + HlsTerminalCommitPayload, HlsTerminalCommitRequest, HlsTerminalCommitWindow, HlsTerminalTailPreparationRequest, + HlsTransitionMarginMs, LiveHlsOriginEntry, OriginManifestFetchError, OriginManifestStatusClass, + OriginRefreshRequest, OriginRefreshState, RetryPolicy, +}; +use crate::{ + media_reserve::{HlsLeaseReserveAvailabilityBasis, HlsLeaseReserveSnapshot}, + refresh::maybe_trigger_origin_refresh, + terminal_tail::HlsLeasePlaybackMode, + HlsAccessLease, HlsAccessLeaseId, HlsAccessLeasePendingDeadline, HlsBoundAccountAcquireErrorKind, + HlsFreshManifestRequiredReason, HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsManifestAcceptanceDirective, + HlsManifestAcceptanceExhaustionReason, HlsManifestCommitIdentity, HlsManifestDeliveryMode, HlsMapWorkerPool, + HlsMediaContainer, HlsPlaybackFamilyKey, HlsProxyManager, HlsSegmentCache, HlsSegmentWorkerPool, HlsSession, + HlsSessionKey, HlsSessionMode, HlsTerminalTailCompatibility, ProxySessionId, SegmentCacheStatus, + SegmentFetchPriority, TimelineMapError, TransientPassthroughReason, TransientResourceKind, +}; +use axum::http::{HeaderMap, StatusCode}; +use shared::model::{HlsManifestRecoveryBurstLevel, HlsStripMode}; +use std::{ + fmt::Write, + io, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, +}; +use tokio::sync::RwLock; +use tuliprox_core::{ + model::{Config, HlsManifestRecoveryBurstConfig, StripConfig}, + utils::{ + content_coding::{ContentCoding, ContentCodingError}, + current_time_millis, + }, +}; + +#[test] +fn origin_refresh_failure_backoff_ramps_and_success_resets_counter() { + let mut state = OriginRefreshState::default(); + + state.mark_started(1_000); + state.mark_failure(1_100); + assert_eq!(state.consecutive_failures, 1); + assert_eq!(state.last_error_at_ms, Some(1_100)); + assert_eq!(state.next_fetch_allowed_at_ms, 1_100); + assert!(state.is_due(1_100)); + + state.mark_started(1_200); + state.mark_failure(1_300); + assert_eq!(state.consecutive_failures, 2); + assert_eq!(state.next_fetch_allowed_at_ms, 1_800); + assert!(!state.is_due(1_799)); + assert!(state.is_due(1_800)); + + state.mark_started(1_900); + state.mark_failure(2_000); + assert_eq!(state.consecutive_failures, 3); + assert_eq!(state.next_fetch_allowed_at_ms, 3_000); + + state.mark_started(3_100); + let success_timing = build_manifest_refresh_timing(Some(20_000), None, HlsManifestProgress::Advanced); + assert_eq!(state.mark_success_with_timing(3_100, 3_200, success_timing), 10_000); + assert_eq!(state.consecutive_failures, 0); + assert_eq!(state.last_error_at_ms, None); + assert_eq!(state.next_fetch_allowed_at_ms, 13_100); + + state.mark_started(13_100); + state.mark_failure(13_200); + assert_eq!(state.consecutive_failures, 1); + assert_eq!(state.next_fetch_allowed_at_ms, 13_200); +} + +#[test] +fn status_classification_matches_hls_retry_policy() { + for status in [ + StatusCode::PROXY_AUTHENTICATION_REQUIRED, + StatusCode::REQUEST_TIMEOUT, + StatusCode::TOO_EARLY, + StatusCode::TOO_MANY_REQUESTS, + StatusCode::INTERNAL_SERVER_ERROR, + StatusCode::BAD_GATEWAY, + ] { + assert_eq!(classify_origin_manifest_status(status), OriginManifestStatusClass::Retryable); + } + for status in [ + StatusCode::BAD_REQUEST, + StatusCode::UNAUTHORIZED, + StatusCode::FORBIDDEN, + StatusCode::NOT_FOUND, + StatusCode::GONE, + ] { + assert_eq!(classify_origin_manifest_status(status), OriginManifestStatusClass::PermanentFailure); + } +} + +pub(in crate::refresh::tests) type ManifestFetchFailureCase = + (&'static str, OriginManifestFetchError, HlsManifestFetchFailureSignal); + +pub(in crate::refresh::tests) fn manifest_response_failure_cases() -> Vec { + use HlsManifestFetchFailureKind as Kind; + use HlsManifestHttpResponseEvidence::{None as NoResponse, ValidResponse}; + + vec![ + ( + "permanent status", + OriginManifestFetchError::PermanentStatus(StatusCode::NOT_FOUND), + HlsManifestFetchFailureSignal::hard(Kind::HttpStatus { status: StatusCode::NOT_FOUND }, ValidResponse), + ), + ( + "retryable status", + OriginManifestFetchError::RetryableStatus(StatusCode::PROXY_AUTHENTICATION_REQUIRED, None), + HlsManifestFetchFailureSignal::retryable( + Kind::HttpStatus { status: StatusCode::PROXY_AUTHENTICATION_REQUIRED }, + ValidResponse, + ), + ), + ( + "retry exhausted", + OriginManifestFetchError::RetryExhausted, + HlsManifestFetchFailureSignal::retryable(Kind::RetryExhausted, NoResponse), + ), + ( + "commit generation exhausted", + OriginManifestFetchError::CommitGenerationExhausted, + HlsManifestFetchFailureSignal::discarded(Kind::CommitGenerationExhausted), + ), + ( + "recovery unavailable after response", + OriginManifestFetchError::RecoveryUnavailable { + reason: HlsManifestRecoveryUnavailableReason::NoEstablishedBindingAfterResponse, + }, + HlsManifestFetchFailureSignal::retryable(Kind::AcceptanceConflict, ValidResponse), + ), + ( + "recovery binding superseded", + OriginManifestFetchError::RecoveryUnavailable { + reason: HlsManifestRecoveryUnavailableReason::BindingSuperseded, + }, + HlsManifestFetchFailureSignal::discarded(Kind::Superseded), + ), + ( + "deterministic acceptance conflict", + OriginManifestFetchError::DeterministicTimelineConflict(Box::new(test_deterministic_timeline_conflict())), + HlsManifestFetchFailureSignal::retryable(Kind::AcceptanceConflict, ValidResponse), + ), + ( + "non-retryable status", + OriginManifestFetchError::NonRetryableStatus(StatusCode::IM_A_TEAPOT), + HlsManifestFetchFailureSignal::hard(Kind::HttpStatus { status: StatusCode::IM_A_TEAPOT }, ValidResponse), + ), + ( + "request timeout wording", + OriginManifestFetchError::Request("Request timed out and no retries left".to_string()), + HlsManifestFetchFailureSignal::retryable(Kind::Timeout, NoResponse), + ), + ( + "transport, connect, or DNS failure", + OriginManifestFetchError::Request("dns lookup failed".to_string()), + HlsManifestFetchFailureSignal::retryable(Kind::Transport, NoResponse), + ), + ( + "redirect failure", + OriginManifestFetchError::Redirect("redirect location invalid".to_string()), + HlsManifestFetchFailureSignal::retryable(Kind::Redirect, ValidResponse), + ), + ( + "timeout", + OriginManifestFetchError::Timeout, + HlsManifestFetchFailureSignal::retryable(Kind::Timeout, NoResponse), + ), + ] +} + +pub(in crate::refresh::tests) fn manifest_content_failure_cases() -> Vec { + use HlsManifestFetchFailureKind as Kind; + use HlsManifestHttpResponseEvidence::{None as NoResponse, ValidResponse}; + + vec![ + ( + "retryable provider acquire", + OriginManifestFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::WaitTimedOut), + HlsManifestFetchFailureSignal::retryable( + Kind::ProviderAcquire { kind: HlsBoundAccountAcquireErrorKind::WaitTimedOut }, + NoResponse, + ), + ), + ( + "hard provider acquire", + OriginManifestFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Expired), + HlsManifestFetchFailureSignal::hard( + Kind::ProviderAcquire { kind: HlsBoundAccountAcquireErrorKind::Expired }, + NoResponse, + ), + ), + ( + "invalid content-coding header", + OriginManifestFetchError::ContentCoding(ContentCodingError::InvalidHeader), + HlsManifestFetchFailureSignal::hard(Kind::InvalidContentCodingHeader, ValidResponse), + ), + ( + "unsupported content coding", + OriginManifestFetchError::ContentCoding(ContentCodingError::Unsupported("unknown".to_string())), + HlsManifestFetchFailureSignal::hard(Kind::UnsupportedContentCoding, ValidResponse), + ), + ( + "encoded partial content", + OriginManifestFetchError::ContentCoding(ContentCodingError::EncodedPartialContent), + HlsManifestFetchFailureSignal::hard(Kind::EncodedPartialContent, ValidResponse), + ), + ( + "content prefix read", + OriginManifestFetchError::ContentCoding(ContentCodingError::PrefixRead(io::Error::other( + "prefix read failed", + ))), + HlsManifestFetchFailureSignal::retryable(Kind::ContentPrefixRead, ValidResponse), + ), + ( + "content decoding", + OriginManifestFetchError::ContentDecoding { coding: ContentCoding::Gzip }, + HlsManifestFetchFailureSignal::retryable( + Kind::ContentDecoding { coding: ContentCoding::Gzip }, + ValidResponse, + ), + ), + ( + "decoded body limit", + OriginManifestFetchError::DecodedBodyLimitExceeded { limit: 1_024 }, + HlsManifestFetchFailureSignal::hard(Kind::DecodedBodyLimit, ValidResponse), + ), + ( + "invalid UTF-8", + OriginManifestFetchError::InvalidUtf8 { valid_up_to: 7, error_len: Some(1) }, + HlsManifestFetchFailureSignal::hard(Kind::InvalidUtf8, ValidResponse), + ), + ] +} + +#[test] +fn manifest_fetch_failures_have_exhaustive_typed_signals() { + for (label, error, expected) in + manifest_response_failure_cases().into_iter().chain(manifest_content_failure_cases()) + { + assert_eq!(classify_manifest_fetch_failure(&error), expected, "unexpected signal: {label}"); + assert_eq!(manifest_hard_fetch_error(&error), expected.is_hard(), "unexpected disposition: {label}"); + } +} + +#[test] +fn manifest_hard_fetch_error_matches_permanent_and_non_retryable_status_only() { + assert!(manifest_hard_fetch_error(&OriginManifestFetchError::PermanentStatus(StatusCode::NOT_FOUND))); + assert!(manifest_hard_fetch_error(&OriginManifestFetchError::NonRetryableStatus(StatusCode::IM_A_TEAPOT))); + assert!(!manifest_hard_fetch_error(&OriginManifestFetchError::RetryableStatus( + StatusCode::TOO_MANY_REQUESTS, + None, + ))); + assert!(!manifest_hard_fetch_error(&OriginManifestFetchError::Timeout)); + assert!(!manifest_hard_fetch_error(&OriginManifestFetchError::ProviderUnavailable( + HlsBoundAccountAcquireErrorKind::WaitTimedOut, + ))); + assert!(manifest_hard_fetch_error(&OriginManifestFetchError::ProviderUnavailable( + HlsBoundAccountAcquireErrorKind::Expired, + ))); +} + +#[test] +fn failure_signal_advances_response_clock_only_with_http_evidence() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + + for error in [ + OriginManifestFetchError::Timeout, + OriginManifestFetchError::Request("connection refused".to_string()), + OriginManifestFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::WaitTimedOut), + ] { + apply_manifest_fetch_failure_signal(&mut session, &error, 100); + assert_eq!(session.origin_control.last_origin_response_at_ms, None); + assert_eq!( + session.origin_control.path_condition, + super::super::super::origin_progress::HlsOriginPathCondition::RetryableFetchFailure + ); + } + + apply_manifest_fetch_failure_signal( + &mut session, + &OriginManifestFetchError::RetryableStatus(StatusCode::PROXY_AUTHENTICATION_REQUIRED, None), + 200, + ); + assert_eq!(session.origin_control.last_origin_response_at_ms, Some(200)); + + apply_manifest_fetch_failure_signal( + &mut session, + &OriginManifestFetchError::Redirect("redirect location invalid".to_string()), + 250, + ); + assert_eq!(session.origin_control.last_origin_response_at_ms, Some(250)); + + apply_manifest_fetch_failure_signal( + &mut session, + &OriginManifestFetchError::ContentDecoding { coding: ContentCoding::Brotli }, + 300, + ); + assert_eq!(session.origin_control.last_origin_response_at_ms, Some(300)); + + let hard_action = apply_manifest_fetch_failure_signal( + &mut session, + &OriginManifestFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Expired), + 400, + ); + assert_eq!(session.origin_control.last_origin_response_at_ms, Some(300)); + assert_eq!( + session.origin_control.path_condition, + super::super::super::origin_progress::HlsOriginPathCondition::HardFetchFailure + ); + assert_eq!( + session.fresh_manifest_commit_required, + Some(HlsFreshManifestRequiredReason::PreviousHardManifestFailure) + ); + assert_eq!( + hard_action, + HlsPostRefreshAvailabilityAction::Reevaluate { + reason: HlsPostRefreshAvailabilityReason::HardManifestFailure, + origin_progress_generation: session.origin_control.progress_generation, + media_readiness_generation: session.activity.media_readiness_generation, + } + ); +} + +#[test] +fn invalidated_origin_work_generation_rejects_late_completion_without_counting_failure() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let started_generation = session.start_origin_work(); + let mut refresh_state = OriginRefreshState::default(); + refresh_state.mark_started(100); + + assert!(refresh_origin_work_generation_matches(&session, Some(started_generation))); + session.invalidate_queued_origin_work(); + assert!(!refresh_origin_work_generation_matches(&session, Some(started_generation))); + assert!(refresh_origin_work_generation_matches(&session, None)); + + refresh_state.mark_invalidated(200); + assert!(!refresh_state.in_flight); + assert_eq!(refresh_state.last_fetch_finished_at_ms, Some(200)); + assert_eq!(refresh_state.consecutive_failures, 0); + assert_eq!(refresh_state.last_error_at_ms, None); +} + +#[test] +fn request_error_timeout_detection_matches_global_helper_wording() { + assert!(request_error_indicates_timeout("Request timed out and no retries left")); + assert!(request_error_indicates_timeout("idle timeout while trying provider://demo")); + assert!(!request_error_indicates_timeout("Request error: error sending request")); +} + +#[test] +fn manifest_reject_log_reason_preserves_timeline_mapping_error() { + assert_eq!( + HlsManifestRejectLogReason::from(TimelineMapError::UnsupportedSegmentExtension).status_label(), + "unsupported-segment-extension" + ); + assert_eq!( + HlsManifestRejectLogReason::from(TimelineMapError::ProxyMapIdOverflow).status_label(), + "proxy-map-id-overflow" + ); +} + +#[test] +fn empty_refresh_rampdown_halves_until_one_second() { + let mut state = OriginRefreshState::default(); + let timing = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Unchanged); + + state.mark_started(0); + assert_eq!(state.mark_success_with_timing(0, 100, timing), 3_000); + assert_eq!(state.consecutive_empty_refreshes, 1); + assert_eq!(state.next_fetch_allowed_at_ms, 3_000); + + state.mark_started(3_000); + assert_eq!(state.mark_success_with_timing(3_000, 3_100, timing), 1_500); + assert_eq!(state.consecutive_empty_refreshes, 2); + assert_eq!(state.next_fetch_allowed_at_ms, 4_500); + + state.mark_started(4_500); + assert_eq!(state.mark_success_with_timing(4_500, 4_600, timing), 1_000); + assert_eq!(state.consecutive_empty_refreshes, 3); + assert_eq!(state.next_fetch_allowed_at_ms, 5_500); + + state.mark_started(5_500); + assert_eq!(state.mark_success_with_timing(5_500, 5_600, timing), 1_000); + assert_eq!(state.consecutive_empty_refreshes, 4); + assert_eq!(state.next_fetch_allowed_at_ms, 6_500); +} + +#[test] +fn advanced_or_rollover_refresh_resets_empty_refresh_counter() { + let mut state = OriginRefreshState::default(); + let unchanged = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Unchanged); + let advanced = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Advanced); + let rollover = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Rollover); + + state.mark_started(0); + assert_eq!(state.mark_success_with_timing(0, 100, unchanged), 3_000); + state.mark_started(3_000); + assert_eq!(state.mark_success_with_timing(3_000, 3_100, unchanged), 1_500); + assert_eq!(state.consecutive_empty_refreshes, 2); + + state.mark_started(4_500); + assert_eq!(state.mark_success_with_timing(4_500, 4_600, advanced), 6_000); + assert_eq!(state.consecutive_empty_refreshes, 0); + assert_eq!(state.next_fetch_allowed_at_ms, 10_500); + + state.mark_started(10_500); + assert_eq!(state.mark_success_with_timing(10_500, 10_600, unchanged), 3_000); + assert_eq!(state.consecutive_empty_refreshes, 1); + + state.mark_started(13_500); + assert_eq!(state.mark_success_with_timing(13_500, 13_600, rollover), 6_000); + assert_eq!(state.consecutive_empty_refreshes, 0); + assert_eq!(state.next_fetch_allowed_at_ms, 19_500); +} + +#[test] +fn failure_backoff_does_not_increment_empty_refresh_counter() { + let mut state = OriginRefreshState::default(); + let unchanged = build_manifest_refresh_timing(None, Some(12_000), HlsManifestProgress::Unchanged); + + state.mark_started(0); + assert_eq!(state.mark_success_with_timing(0, 100, unchanged), 3_000); + state.mark_started(3_000); + state.mark_failure(3_100); + + assert_eq!(state.consecutive_failures, 1); + assert_eq!(state.consecutive_empty_refreshes, 1); +} + +#[test] +fn manifest_progress_tracks_highwater_advancement() { + assert_eq!( + manifest_progress_from_highwater(None, Some(10), HlsManifestSequenceRelation::NoPreviousHighwater), + HlsManifestProgress::Advanced + ); + assert_eq!( + manifest_progress_from_highwater(Some(10), Some(11), HlsManifestSequenceRelation::Next), + HlsManifestProgress::Advanced + ); + assert_eq!( + manifest_progress_from_highwater(Some(10), Some(10), HlsManifestSequenceRelation::Same), + HlsManifestProgress::Unchanged + ); + assert_eq!( + manifest_progress_from_highwater(Some(10), Some(9), HlsManifestSequenceRelation::Backward), + HlsManifestProgress::Unchanged + ); + assert_eq!( + manifest_progress_from_highwater(Some(10), Some(1), HlsManifestSequenceRelation::RolloverCandidate), + HlsManifestProgress::Rollover + ); + assert_eq!( + manifest_progress_from_highwater(None, None, HlsManifestSequenceRelation::NoOriginHighwater), + HlsManifestProgress::Unchanged + ); +} + +#[test] +fn hls_cutover_policy_only_advanced_and_rollover_commits_advance_progress_generation() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let initial_generation = session.origin_control.progress_generation; + + record_committed_manifest_media_progress( + &mut session.origin_control, + HlsManifestCommitProgressEvidence::CacheTimeline(build_manifest_refresh_timing( + None, + Some(4_000), + HlsManifestProgress::Unchanged, + )), + Some(4), + 1_000, + ); + assert_eq!(session.origin_control.progress_generation, initial_generation); + assert_eq!(session.origin_control.last_media_progress_at_ms, None); + + record_committed_manifest_media_progress( + &mut session.origin_control, + HlsManifestCommitProgressEvidence::Transient(build_manifest_refresh_timing( + None, + Some(4_000), + HlsManifestProgress::Advanced, + )), + Some(4), + 1_500, + ); + assert_eq!(session.origin_control.progress_generation, initial_generation); + assert_eq!(session.origin_control.last_media_progress_at_ms, None); + + record_committed_manifest_media_progress( + &mut session.origin_control, + HlsManifestCommitProgressEvidence::CacheTimeline(build_manifest_refresh_timing( + None, + Some(4_000), + HlsManifestProgress::Advanced, + )), + Some(4), + 2_000, + ); + assert_eq!(session.origin_control.progress_generation, initial_generation.saturating_add(1)); + assert_eq!(session.origin_control.last_media_progress_at_ms, Some(2_000)); + + record_committed_manifest_media_progress( + &mut session.origin_control, + HlsManifestCommitProgressEvidence::CacheTimeline(build_manifest_refresh_timing( + None, + Some(4_000), + HlsManifestProgress::Rollover, + )), + Some(4), + 3_000, + ); + assert_eq!(session.origin_control.progress_generation, initial_generation.saturating_add(2)); + assert_eq!(session.origin_control.last_media_progress_at_ms, Some(3_000)); +} + +#[test] +fn hls_cutover_policy_cache_timeline_progress_keeps_recovery_success_bookkeeping() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let burst_plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); + session.origin_control.begin_acceptance_episode( + 1_000, + burst_plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &test_acceptance_episode_timing(1_000, burst_plan), + ); + session.origin_control.acceptance_episode.as_mut().expect("acceptance episode").complete(); + session.origin_refresh.consecutive_empty_refreshes = 2; + session.origin_refresh.mark_started(10_000); + + let (bookkeeping_timing, applied_interval_ms) = record_committed_manifest_success( + &mut session, + HlsManifestCommitProgressEvidence::CacheTimeline(build_manifest_refresh_timing( + None, + Some(12_000), + HlsManifestProgress::Advanced, + )), + 10_000, + 10_100, + ); + + assert_eq!(bookkeeping_timing.progress, HlsManifestProgress::Advanced); + assert_eq!(applied_interval_ms, 6_000); + assert_eq!(session.origin_refresh.consecutive_empty_refreshes, 0); + assert!(session.origin_control.acceptance_episode.is_none()); + assert_eq!(session.origin_control.recovery_samples.p95_ms(), Some(9_100)); +} + +#[test] +fn refresh_commit_trims_forward_manifest_with_published_stale_resource_prefix() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let request = test_origin_refresh_request(test_session()); + let baseline = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:471\n\ + #EXTINF:4,\n471.ts\n#EXTINF:4,\n472.ts\n#EXTINF:4,\n473.ts\n\ + #EXTINF:4,\n474.ts\n#EXTINF:4,\n475.ts\n#EXTINF:4,\n476.ts\n", + ); + let replay_then_new = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:476\n\ + #EXTINF:4,\n474.ts\n#EXTINF:4,\n480.ts\n#EXTINF:4,\n481.ts\n", + ); + + commit_fetched_manifest(&mut session, &baseline, &request, 100).expect("baseline refresh commits"); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 101 }; + } + session.render_and_store_manifest(101).expect("baseline publishes"); + commit_fetched_manifest(&mut session, &replay_then_new, &request, 200) + .expect("stale prefix is safely trimmed by refresh commit"); + + assert_eq!(session.proxy_next_seq, Some(8)); + assert_eq!(session.publishable_origin_head_proxy_seq, Some(0)); + assert!(session.segments.get(&6).expect("first genuine media").discontinuity_before); + for proxy_seq in [6_u64, 7] { + session.segments.get_mut(&proxy_seq).expect("new segment").status = + SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 201 }; + } + let rendered = session.render_and_store_manifest(201).expect("forward refresh renders without replay"); + assert_eq!(rendered.first_proxy_seq, 2); + assert!(rendered.body.contains("#EXT-X-DISCONTINUITY\n#EXTINF:4.000,")); + let identities = rendered + .segment_proxy_seqs + .iter() + .filter_map(|proxy_seq| session.segments.get(proxy_seq)?.media_resource_identity()) + .collect::>(); + assert!(identities + .iter() + .enumerate() + .all(|(index, identity)| { identities[..index].iter().all(|previous| !previous.matches(*identity)) })); +} + +#[tokio::test] +async fn background_replay_failure_registers_post_refresh_reevaluation_without_client_demand() { + let hls_ctx = crate::HlsCtx::for_test(Config::default()); + let ctx = &hls_ctx; + let now_ms = current_time_millis(); + let (session, _) = ctx + .hls_proxy + .get_or_create_session_with_outcome(HlsSessionKey::new(1, "post-refresh-conflict"), b"secret", now_ms) + .await; + let conflicting_body = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:490\n\ + #EXTINF:4,\n490.ts\n#EXTINF:4,\n480.ts\n#EXTINF:4,\n491.ts\n"; + let server = spawn_test_origin(Arc::new(move |_path| (200, Vec::new(), conflicting_body.to_string()))).await; + let manifest_url = format!("{}/live/user/pass/12345.m3u8", server.base_url); + let mut request = bind_refresh_request_to_app_state(test_origin_refresh_request(Arc::clone(&session)), ctx); + request.origin_entry = LiveHlsOriginEntry::parse(&manifest_url).expect("local replay origin entry"); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + request.now_ms = now_ms; + let mut baseline = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:480\n\ + #EXTINF:4,\n480.ts\n#EXTINF:4,\n481.ts\n#EXTINF:4,\n482.ts\n", + ); + baseline.final_manifest_url = manifest_url.clone(); + baseline.resolved_request_url = manifest_url; + baseline.redirect_host = Some("127.0.0.1".to_string()); + { + let mut session = session.write().await; + commit_fetched_manifest(&mut session, &baseline, &request, now_ms).expect("baseline commits"); + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: now_ms }; + } + session.render_and_store_manifest(now_ms).expect("baseline publishes"); + session.segments.get_mut(&1).expect("deferred boundary segment").status = + SegmentCacheStatus::CapacityDeferred { priority: SegmentFetchPriority::Prefetch, deferred_at_ms: now_ms }; + } + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("post-refresh-conflict-lease".to_string()); + let mut lease = HlsAccessLease::pending( + lease_id, + HlsPlaybackFamilyKey::new("post-refresh-user", "post-refresh-client"), + proxy_session_id, + "post-refresh-user".to_string(), + "post-refresh-token".to_string(), + 1, + "post-refresh-stream".to_string(), + 1, + now_ms, + 60_000, + ); + lease.state = crate::HlsAccessLeaseState::Activated; + lease.active_until_ms = Some(now_ms.saturating_add(60_000)); + lease.pending_deadline = None; + lease.last_manifest_snapshot = Some(post_refresh_live_manifest_snapshot()); + ctx.hls_proxy.prepare_access_lease(lease).await; + + assert!(trigger_origin_refresh_sync(request).await); + + assert_eq!( + session.read().await.origin_control.path_condition, + super::super::super::origin_progress::HlsOriginPathCondition::AcceptanceConflict + ); + assert_eq!(ctx.hls_proxy.availability_reevaluations().owner_count(), 1); +} + +#[tokio::test] +async fn background_success_does_not_schedule_failure_reevaluation() { + let hls_ctx = crate::HlsCtx::for_test(Config::default()); + let ctx = &hls_ctx; + let now_ms = current_time_millis(); + let (session, _) = ctx + .hls_proxy + .get_or_create_session_with_outcome(HlsSessionKey::new(1, "post-refresh-success"), b"secret", now_ms) + .await; + let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let mut request = bind_refresh_request_to_app_state(test_origin_refresh_request(Arc::clone(&session)), ctx); + request.origin_entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) + .expect("local successful origin entry"); + request.now_ms = now_ms; + + assert!(trigger_origin_refresh_sync(request).await); + + assert!(session.read().await.origin_seq_highwater.is_some()); + assert_eq!(ctx.hls_proxy.availability_reevaluations().owner_count(), 0); +} + +pub(in crate::refresh::tests) async fn assert_requirement_set_after_refresh_start_survives_fresh_commit( + replacement_reason: HlsFreshManifestRequiredReason, +) { + let session = test_session(); + { + let mut session = session.write().await; + session.origin_seq_highwater = Some(1_000); + session.last_effective_manifest_host = Some("origin.example.com".to_string()); + session.require_fresh_manifest_commit(HlsFreshManifestRequiredReason::PreviousHardManifestFailure); + } + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::PreviousHardManifestFailure, + }; + assert!(mark_origin_refresh_started(&mut request, 100).await); + + let fetched = + fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:4.0,\nseg10.ts\n"); + let result = { + let mut session = session.write().await; + session.require_fresh_manifest_commit(replacement_reason); + commit_fetched_manifest(&mut session, &fetched, &request, 200) + }; + + assert!(result.is_ok()); + assert_eq!(session.read().await.fresh_manifest_commit_required, Some(replacement_reason)); +} + +#[tokio::test] +async fn fresh_commit_preserves_new_or_same_reason_requirement_set_after_refresh_start() { + assert_requirement_set_after_refresh_start_survives_fresh_commit( + HlsFreshManifestRequiredReason::ExpiredRevalidation, + ) + .await; + assert_requirement_set_after_refresh_start_survives_fresh_commit( + HlsFreshManifestRequiredReason::PreviousHardManifestFailure, + ) + .await; +} + +#[tokio::test] +async fn concurrent_maybe_trigger_origin_refresh_starts_singleflight_once() { + let session = test_session(); + let entry = LiveHlsOriginEntry::parse("http://127.0.0.1:9/live/user/pass/12345.m3u8").expect("valid origin entry"); + let client = reqwest::Client::new(); + let no_redirect_client = + reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()).build().expect("client builds"); + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client, + no_redirect_client, + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 1, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 3 }, + retry_policy: RetryPolicy { delays_ms: [0, 0, 0, 0, 0], jitter_max_ms: 0 }, + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + fresh_manifest_requirement_generation: None, + acceptance_directive: HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + post_refresh_runtime: None, + }; + + let mut handles = Vec::new(); + for _ in 0..8 { + let request = request.clone(); + handles.push(tokio::spawn(async move { maybe_trigger_origin_refresh(request).await })); + } + + let started = futures::future::join_all(handles) + .await + .into_iter() + .filter(|result| result.as_ref().is_ok_and(|started| *started)) + .count(); + assert_eq!(started, 1); +} + +#[tokio::test] +async fn fresh_manifest_commit_bypasses_refresh_debounce() { + let session = test_session(); + session.write().await.origin_refresh.next_fetch_allowed_at_ms = 10_000; + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.manifest_commit_requirement = + HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; + + assert!(mark_origin_refresh_started(&mut request, 1_000).await); + assert!(session.read().await.origin_refresh.in_flight); +} + +#[tokio::test] +async fn committed_manifest_refresh_still_obeys_debounce() { + let session = test_session(); + session.write().await.origin_refresh.next_fetch_allowed_at_ms = 10_000; + let mut request = test_origin_refresh_request(Arc::clone(&session)); + + assert_eq!( + mark_origin_refresh_started_with_outcome(&mut request, 1_000).await, + HlsOriginRefreshTriggerOutcome::DebouncedUntil { retry_at_ms: 10_000 } + ); + assert!(!session.read().await.origin_refresh.in_flight); +} + +pub(in crate::refresh::tests) fn cutover_live_manifest_snapshot() -> HlsLeaseManifestSnapshot { + HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(1_000), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 0, + delivered_at_ms: 1_000, + first_proxy_seq: 40, + last_proxy_seq: 41, + visible_segments: Arc::from([ + HlsLeaseManifestSegment { + proxy_seq: 40, + duration_ms: 4_000, + uri: "/live/40.ts".into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }, + HlsLeaseManifestSegment { + proxy_seq: 41, + duration_ms: 4_000, + uri: "/live/41.ts".into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }, + ]), + discontinuity_sequence: 0, + target_duration_ms: 4_000, + playlist_duration_ms: 8_000, + last_visible_media_end_ms: 8_000, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + } +} + +pub(in crate::refresh::tests) async fn assert_recovery_supersedes_terminal_preparation( + hls_proxy: &HlsProxyManager, + session: &Arc>, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + preparation: &super::super::super::lease::HlsTerminalTailPreparation, +) { + assert_eq!( + hls_proxy.commit_access_lease_terminal_if_generation_matches(HlsTerminalCommitRequest { + session, + lease_id, + proxy_session_id, + preparation, + now_ms: 2_100, + payload: HlsTerminalCommitPayload::Unavailable(HlsTerminalTailCompatibility::MissingAsset), + asset_revision_guard: + super::super::super::terminal_commit::HlsTerminalAssetRevisionGuard::matching_for_test(None), + }), + super::super::super::terminal_commit::HlsTerminalCommitOutcome::RecoveryCommitted + ); + let lease = hls_proxy + .access_lease_response_snapshot(lease_id, proxy_session_id, 2_100) + .await + .expect("recovered lease remains live"); + assert_eq!(lease.playback_mode, HlsLeasePlaybackMode::Live); + assert!(!session.read().await.has_terminal_tail_protections()); +} + +pub(in crate::refresh::tests) async fn prepare_failed_acceptance_episode(session: &Arc>) -> u64 { + let mut session = session.write().await; + let burst_plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + session.origin_control.begin_acceptance_episode( + 1_000, + burst_plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &test_acceptance_episode_timing(1_000, burst_plan), + ); + let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); + episode.record_full_burst(); + episode.record_exhaustion(HlsManifestAcceptanceExhaustionReason::AllFailed); + episode.hold_after_uncommitted_burst(None, Some(2_000)); + session.origin_control.progress_generation +} + +#[tokio::test] +async fn hls_cutover_policy_advanced_commit_supersedes_preparation_and_keeps_lease_live() { + let hls_proxy = Arc::new(HlsProxyManager::new()); + let (session, _) = + hls_proxy.get_or_create_session_with_outcome(HlsSessionKey::new(1, "12345"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + hls_proxy.access_leases().write().await.prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("user", "client"), + proxy_session_id.clone(), + "user".to_string(), + "session-token".to_string(), + 1, + "12345".to_string(), + 12345, + 1_000, + 60_000, + )); + let publication_guard = hls_proxy + .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, 1_000) + .await + .expect("live manifest publication"); + assert!(hls_proxy + .commit_access_lease_manifest_publication( + &lease_id, + &proxy_session_id, + publication_guard, + cutover_live_manifest_snapshot(), + 1_000, + ) + .await + .is_committed()); + let prepared_progress_generation = prepare_failed_acceptance_episode(&session).await; + let transition_margin = HlsTransitionMarginMs::from_millis(4_000); + let guaranteed_reserve_ms = transition_margin + .as_millis() + .saturating_add(HlsTerminalCommitAcquisitionBudgetMs::from_retry_policy().as_millis()); + let reserve = HlsLeaseReserveSnapshot { + availability_basis: HlsLeaseReserveAvailabilityBasis::ReadyCacheTimeline, + guaranteed_media_horizon_ms: 8_000_u64.saturating_add(guaranteed_reserve_ms), + conservative_playback_position_ms: 8_000, + guaranteed_reserve_ms, + initial_hidden_ready_duration_ms: 0, + transition_margin, + key_readiness_valid_until_ms: None, + recovery_required: true, + cutover_required: false, + }; + let cutover_timing = + HlsLeaseCutoverTiming::from_reserve(2_000, reserve.guaranteed_reserve_ms, reserve.transition_margin, None); + let preparation = hls_proxy + .prepare_access_lease_terminal_tail(HlsTerminalTailPreparationRequest { + lease_id: &lease_id, + proxy_session_id: &proxy_session_id, + manifest_snapshot_generation: 1, + cursor_generation: 0, + reserve, + cutover_timing, + commit_window: HlsTerminalCommitWindow::AcquisitionOpen, + now_ms: 2_000, + origin_progress_generation: prepared_progress_generation, + media_readiness_generation: 0, + last_media_progress_at_ms: None, + }) + .await + .expect("exhausted acceptance permits terminal preparation"); + + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.hls_proxy = Arc::clone(&hls_proxy); + request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; + let fetched = fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:42\n#EXTINF:4.0,\n42.ts\n"); + { + let mut session = session.write().await; + let (progress_evidence, _, _) = + commit_fetched_manifest(&mut session, &fetched, &request, 2_100).expect("recovery manifest commits"); + assert_eq!(progress_evidence.refresh_timing().progress, HlsManifestProgress::Advanced); + assert_eq!(session.origin_control.progress_generation, prepared_progress_generation.saturating_add(1)); + assert_eq!(session.origin_control.last_media_progress_at_ms, Some(2_100)); + } + + assert_recovery_supersedes_terminal_preparation(&hls_proxy, &session, &proxy_session_id, &lease_id, &preparation) + .await; +} + +#[tokio::test] +async fn successful_manifest_commit_shortens_pending_leases_without_response_path() { + let session = test_session(); + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let hls_proxy = Arc::new(HlsProxyManager::new()); + let now_ms = super::super::current_time_millis(); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("user", "client"), + proxy_session_id.clone(), + "user".to_string(), + "session-token".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + 90_000, + )) + .await; + let server = spawn_test_origin(Arc::new(|_path| { + (200, Vec::new(), "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nseg.ts\n".to_string()) + })) + .await; + let entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) + .expect("valid origin entry"); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.hls_proxy = Arc::clone(&hls_proxy); + request.origin_entry = entry; + request.now_ms = now_ms; + + assert!(Box::pin(trigger_origin_refresh_sync(request)).await); + + let lease = hls_proxy + .access_leases() + .write() + .await + .response_snapshot(&lease_id, &proxy_session_id, super::super::current_time_millis()) + .expect("pending lease should remain available"); + let Some(HlsAccessLeasePendingDeadline::FollowUp { deadline_ms }) = lease.pending_deadline else { + panic!("pending lease should be shortened to follow-up"); + }; + assert!(deadline_ms < now_ms.saturating_add(90_000)); + assert!(deadline_ms <= super::super::current_time_millis().saturating_add(10_000)); +} + +#[tokio::test] +async fn endlist_only_manifest_commits_complete_body_with_consistent_finalized_state() { + let mut origin_body = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n".to_string(); + for sequence in 1..=8 { + let _ = write!(origin_body, "#EXTINF:4.0,\n{sequence}.ts\n"); + } + origin_body.push_str("#EXT-X-ENDLIST\n"); + let origin_body: &'static str = Box::leak(origin_body.into_boxed_str()); + let session = refresh_session_with_origin_body(origin_body).await; + let session = session.read().await; + let body = session.transient.last_manifest_body.as_ref().expect("finalized transient body"); + + assert!(matches!( + session.mode, + HlsSessionMode::TransientPassthrough { + reason: TransientPassthroughReason::UnsupportedTag { ref tag } + } if tag == "#EXT-X-ENDLIST" + )); + let stored_body_lifecycle = tuliprox_parser::hls::origin_manifest::parse_manifest_semantics(body).lifecycle(); + assert_eq!(session.transient.last_manifest_finalized(), stored_body_lifecycle.is_finalized()); + assert_eq!( + session.transient.last_manifest_window_policy(), + tuliprox_parser::hls::origin_manifest::HlsManifestWindowPolicy::PreserveFullManifest + ); + assert_eq!(session.transient.current_manifest_resource_ids().len(), 8); + assert_eq!(body.matches("/r/").count(), 8); + assert!(body.contains("#EXT-X-ENDLIST")); + assert!(stored_body_lifecycle.is_finalized()); +} + +#[tokio::test] +async fn compatible_aes_128_manifest_commits_normal_timeline_with_opaque_key_resource() { + let session = refresh_session_with_origin_body( + "#EXTM3U\n#EXT-X-VERSION:5\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ + #EXT-X-KEY:METHOD=AES-128,URI=\"origin-key.bin\",IV=0x00000000000000000000000000000001,KEYFORMAT=\"identity\",KEYFORMATVERSIONS=\"1\"\n\ + #EXTINF:4,\n1.ts\n#EXTINF:4,\n2.ts\n#EXTINF:4,\n3.ts\n#EXTINF:4,\n4.ts\n#EXTINF:4,\n5.ts\n#EXTINF:4,\n6.ts\n", + ) + .await; + let session = session.read().await; + + assert_eq!(session.mode, HlsSessionMode::NormalCacheTimeline); + assert_eq!(session.transient.resources.len(), 1); + assert!(session.transient.resources.values().all(|resource| resource.kind == TransientResourceKind::Key)); + assert!(session.segments.values().all(|segment| { + segment.encryption.as_ref().is_some_and(|encryption| { + encryption.resource_extension == "bin" + && session.transient.resources.contains_key(&encryption.resource_id) + && !encryption.resource_id.0.contains("origin-key.bin") + }) + })); + assert!( + session.last_rendered_manifest.is_none(), + "a refresh fixture without a usable lease must not publish unavailable media" + ); +} + +#[tokio::test] +async fn cold_recovery_directive_is_suppressed_until_baseline() { + let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let session = test_session(); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; + + assert!(trigger_origin_refresh_sync(request).await); + + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let manifest_requests = + server.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(manifest_requests, 1); + assert_ne!(manifest_requests, plan.total_candidates()); + let session = session.read().await; + assert_eq!(session.origin_seq_highwater, Some(0)); + assert!(session.origin_control.manifest_origin_binding.is_some()); + assert!(session.origin_control.acceptance_episode.is_none()); +} + +#[tokio::test] +async fn shared_initial_manifest_decoder_failure_retries_until_success() { + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let server = spawn_test_origin(Arc::new(move |_path| { + if hits_for_handler.fetch_add(1, Ordering::SeqCst) < 2 { + return (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()); + } + (200, Vec::new(), manifest_body()) + })) + .await; + let origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + let context = HlsOriginManifestFetchContext { + app_config: test_app_config(), + session: test_session(), + origin_entry, + headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client"), + use_manual_redirects: false, + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + retry_policy: no_delay_policy(), + recovery_timing_policy: test_recovery_timing_policy(2_000), + acceptance_timing_seed: None, + }; + + let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) + .await + .expect("shared initial manifest should retry decoder failures"); + + assert_eq!(fetched.body, manifest_body()); + assert_eq!(fetched.attempts, 3); + assert_eq!(server.requests.lock().await.len(), 3); + assert!(server + .raw_requests + .lock() + .await + .iter() + .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); +} + +#[tokio::test] +async fn shared_initial_manifest_waits_for_next_attempt_base_delay() { + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let server = spawn_test_origin(Arc::new(move |_path| { + if hits_for_handler.fetch_add(1, Ordering::SeqCst) == 0 { + return (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()); + } + (200, Vec::new(), manifest_body()) + })) + .await; + let origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + let context = HlsOriginManifestFetchContext { + app_config: test_app_config(), + session: test_session(), + origin_entry, + headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client"), + use_manual_redirects: false, + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + retry_policy: RetryPolicy { delays_ms: [0, 100, 250, 500, 750], jitter_max_ms: 0 }, + recovery_timing_policy: test_recovery_timing_policy(2_000), + acceptance_timing_seed: None, + }; + let started_at = std::time::Instant::now(); + + let fetched = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) + .await + .expect("second logical attempt should succeed after its base delay"); + + assert_eq!(fetched.attempts, 2); + assert_eq!(server.requests.lock().await.len(), 2); + assert!(started_at.elapsed() >= std::time::Duration::from_millis(100)); +} + +#[tokio::test] +async fn shared_initial_manifest_decoder_failures_stop_at_attempt_budget() { + let server = spawn_test_origin(Arc::new(|_path| { + (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()) + })) + .await; + let origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + let retry_policy = no_delay_policy(); + let expected_attempts = retry_policy.attempt_count(); + let context = HlsOriginManifestFetchContext { + app_config: test_app_config(), + session: test_session(), + origin_entry, + headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client"), + use_manual_redirects: false, + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + retry_policy, + recovery_timing_policy: test_recovery_timing_policy(2_000), + acceptance_timing_seed: None, + }; + + let error = fetch_hls_origin_manifest_request(HlsOriginManifestFetchRequest::initial_global_policy(&context)) + .await + .expect_err("decoder failures must exhaust the logical attempt budget"); + + assert!(matches!(error, OriginManifestFetchError::ContentDecoding { .. })); + assert_eq!(server.requests.lock().await.len(), expected_attempts); + assert!(server + .raw_requests + .lock() + .await + .iter() + .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); +} + +#[tokio::test] +async fn shared_refresh_metrics_use_successful_manifest_attempt_count() { + let manifest_hits = Arc::new(AtomicUsize::new(0)); + let manifest_hits_for_handler = Arc::clone(&manifest_hits); + let server = spawn_test_origin(Arc::new(move |path| { + if path != "/live/user/pass/12345.m3u8" { + return (404, Vec::new(), String::new()); + } + if manifest_hits_for_handler.fetch_add(1, Ordering::SeqCst) < 2 { + return (200, vec![("Content-Encoding", "gzip".to_string())], "corrupt-gzip".to_string()); + } + (200, Vec::new(), manifest_body()) + })) + .await; + let session = test_session(); + let mut request = test_origin_refresh_request(session); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + let metrics = Arc::clone(request.segment_worker_pool.metrics()); + + assert!(Box::pin(trigger_origin_refresh_sync(request)).await); + + let snapshot = metrics.snapshot(); + assert_eq!(snapshot.refresh_started, 1); + assert_eq!(snapshot.refresh_completed, 1); + assert_eq!(snapshot.refresh_retried, 2); + assert_eq!(snapshot.refresh_failed, 0); + let manifest_requests = server + .raw_requests + .lock() + .await + .iter() + .filter(|request| request.lines().next().is_some_and(|line| line.contains(" /live/user/pass/12345.m3u8 "))) + .cloned() + .collect::>(); + assert_eq!(manifest_requests.len(), 3); + assert!(manifest_requests + .iter() + .all(|request| request_header_value(request, "accept-encoding") == Some("identity"))); +} + +#[tokio::test] +async fn retryable_407_retries_until_success() { + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let server = spawn_test_origin(Arc::new(move |_path| { + let hit = hits_for_handler.fetch_add(1, Ordering::SeqCst); + if hit < 2 { + return (407, Vec::new(), "retry".to_string()); + } + (200, Vec::new(), manifest_body()) + })) + .await; + let entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + + let fetched = refresh_from_live_hls_entrypoint_with_retries( + &entry, + &HeaderMap::new(), + &reqwest::Client::new(), + &reqwest::Client::new(), + false, + 2_000, + &no_delay_policy(), + ) + .await + .expect("refresh eventually succeeds"); + + assert_eq!(fetched.attempts, 3); + assert_eq!(server.requests.lock().await.len(), 3); +} + +#[tokio::test] +async fn permanent_404_does_not_retry() { + let server = spawn_test_origin(Arc::new(|_path| (404, Vec::new(), "missing".to_string()))).await; + let entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + + let err = refresh_from_live_hls_entrypoint_with_retries( + &entry, + &HeaderMap::new(), + &reqwest::Client::new(), + &reqwest::Client::new(), + false, + 2_000, + &no_delay_policy(), + ) + .await + .expect_err("404 is permanent"); + + assert!(matches!(err, OriginManifestFetchError::PermanentStatus(StatusCode::NOT_FOUND))); + assert_eq!(server.requests.lock().await.len(), 1); +} + +#[tokio::test] +async fn recovery_unavailable_after_valid_response_preserves_response_evidence() { + let origin = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(758)))).await; + let session = test_session(); + prepare_cross_host_baseline(&session).await; + { + let mut session = session.write().await; + session.origin_control.record_origin_response(50); + } + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/new/index.m3u8", origin.base_url)).expect("candidate entry URL"); + + let Err(error) = fetch_and_commit_manifest_with_policy(&mut request).await else { + panic!("cross-host response without recovery binding must remain uncommitted"); + }; + + assert!(matches!( + error, + OriginManifestFetchError::RecoveryUnavailable { + reason: HlsManifestRecoveryUnavailableReason::NoEstablishedBindingAfterResponse, + } + )); + assert_eq!(error.log_label(), "recovery_unavailable_after_response"); + assert_eq!(error.to_string(), "origin manifest recovery unavailable: no established binding after origin response"); + assert_eq!( + classify_manifest_fetch_failure(&error), + HlsManifestFetchFailureSignal::retryable( + HlsManifestFetchFailureKind::AcceptanceConflict, + HlsManifestHttpResponseEvidence::ValidResponse, + ) + ); + assert_eq!(origin.requests.lock().await.len(), 1); + { + let session = session.read().await; + assert_eq!(session.origin_control.last_origin_response_at_ms, Some(50)); + assert!(session.origin_control.manifest_origin_binding.is_none()); + assert!(session.origin_control.acceptance_episode.is_none()); + } + + let evidence_origin = + spawn_test_origin(Arc::new(|_path| (200, Vec::new(), three_segment_manifest_body(758)))).await; + let evidence_session = test_session(); + prepare_cross_host_baseline(&evidence_session).await; + let progress_generation = { + let mut session = evidence_session.write().await; + session.origin_control.record_origin_response(50); + session.origin_control.progress_generation + }; + let mut evidence_request = test_origin_refresh_request(Arc::clone(&evidence_session)); + evidence_request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/new/index.m3u8", evidence_origin.base_url)) + .expect("evidence candidate entry URL"); + let metrics = Arc::clone(evidence_request.segment_worker_pool.metrics()); + + assert!(trigger_origin_refresh_sync(evidence_request).await); + assert_eq!(evidence_origin.requests.lock().await.len(), 1); + { + let session = evidence_session.read().await; + assert!(session.origin_control.last_origin_response_at_ms.is_some_and(|recorded_at_ms| recorded_at_ms > 50)); + assert_eq!(session.origin_control.progress_generation, progress_generation); + assert!(session.origin_control.manifest_origin_binding.is_none()); + assert!(session.origin_control.acceptance_episode.is_none()); + } + let metrics = metrics.snapshot(); + assert_eq!(metrics.refresh_started, 1); + assert_eq!(metrics.refresh_failed, 1); + assert_eq!(metrics.refresh_completed, 0); + assert_eq!(metrics.refresh_retried, 0); + assert_eq!(metrics.refresh_skipped, 0); +} + +#[tokio::test] +async fn cold_start_uses_one_initial_fetch_and_commits_a_fresh_baseline_before_recovery() { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let session = test_session(); + { + let mut session = session.write().await; + session.origin_seq_highwater = Some(1_000); + session.last_effective_manifest_host = Some("stale.example.com".to_string()); + } + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + request.manifest_commit_requirement = + HlsManifestCommitRequirement::FreshCommitRequired { reason: HlsFreshManifestRequiredReason::ColdStart }; + + assert!(trigger_origin_refresh_sync(request).await); + + let manifest_requests = + server.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(manifest_requests, 1); + assert_ne!(manifest_requests, plan.total_candidates()); + let session = session.read().await; + assert_eq!(session.origin_seq_highwater, Some(0)); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some("127.0.0.1")); + assert!(session.origin_control.acceptance_episode.is_none()); +} + +#[tokio::test] +async fn cold_start_hard_error_is_preserved_by_fetch_policy() { + let server = spawn_test_origin(Arc::new(|_path| (404, Vec::new(), "missing".to_string()))).await; + let session = test_session(); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; + + let Err(error) = fetch_and_commit_manifest_with_policy(&mut request).await else { + panic!("cold hard status must be returned without synthetic recovery"); + }; + + assert!(matches!(error, OriginManifestFetchError::PermanentStatus(StatusCode::NOT_FOUND))); + assert_eq!(server.requests.lock().await.len(), 1); + let session = session.read().await; + assert!(session.origin_control.acceptance_episode.is_none()); + assert_eq!( + session.origin_control.progress_phase, + super::super::super::origin_progress::HlsOriginProgressPhase::Cold + ); +} diff --git a/backend/hls/src/refresh/tests/startup_representation.rs b/backend/hls/src/refresh/tests/startup_representation.rs new file mode 100644 index 000000000..3d953174c --- /dev/null +++ b/backend/hls/src/refresh/tests/startup_representation.rs @@ -0,0 +1,29 @@ +use super::{ + fetched_manifest, manifest_fetch_context, repeated_transient_manifest, test_manifest_origin_binding, + test_origin_refresh_request, test_session, OriginManifestFetchError, +}; +use crate::{HlsSessionMode, TransientPassthroughReason}; +use std::sync::Arc; +use url::Url; + +#[tokio::test] +async fn initial_local_representation_limit_does_not_enter_origin_recovery() { + let session = test_session(); + session.write().await.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + let request = test_origin_refresh_request(Arc::clone(&session)); + let fetch_context = manifest_fetch_context(&request); + let fetched = + fetched_manifest(&repeated_transient_manifest(crate::manifest_limits::MAX_HLS_LEASE_SNAPSHOT_SEGMENTS + 1)); + let recovery_binding = + test_manifest_origin_binding(Url::parse("http://127.0.0.1:9/recovery.m3u8").expect("test recovery URL")); + + let result = + super::super::commit_initial_fetched_manifest(&request, &fetch_context, fetched, Some(recovery_binding), false) + .await; + + assert!(matches!( + result, + Err(OriginManifestFetchError::LocalRepresentationLimit(violation)) + if violation.kind == crate::manifest_limits::HlsManifestLimitKind::LeaseSnapshotSegments + )); +} diff --git a/backend/hls/src/refresh/tests/support.rs b/backend/hls/src/refresh/tests/support.rs new file mode 100644 index 000000000..07ce920f9 --- /dev/null +++ b/backend/hls/src/refresh/tests/support.rs @@ -0,0 +1,1173 @@ +use super::{ + deterministic_timeline_conflict_from_rejection, manifest_fetch_context, retry_hls_origin_manifest_recovery_chain, + trigger_origin_refresh_sync, FetchedOriginManifest, HlsAcceptanceEpisodeTiming, HlsAcceptanceEpisodeTimingInput, + HlsDeterministicTimelineConflict, HlsManifestAcceptanceTrigger, HlsManifestCommitAcceptanceMode, + HlsManifestCommitError, HlsManifestCommitRequirement, HlsManifestFetchSelection, HlsManifestOriginBinding, + HlsManifestRejectLogReason, HlsObservedRecoveryLatency, HlsOperationTimeoutMs, HlsPostRefreshRuntime, + HlsRecoveryEtaMs, HlsRecoveryTimingPolicy, HlsRecoveryWorkload, HlsRecoveryWorkloadEnvelope, + HlsTerminalMediaPreparationState, HlsTransitionMarginMs, LiveHlsOriginEntry, OriginManifestFetchError, + OriginRefreshRequest, RetryPolicy, +}; +use crate::{ + is_hls_provisioning_gap_segment, is_hls_provisioning_segment, manifest_fetch::fetched_effective_manifest_host, + CacheAccessState, HlsAccessLease, HlsAccessLeaseId, HlsAccessLeaseTiming, HlsLeaseManifestSegment, + HlsLeaseManifestSnapshot, HlsManifestAcceptanceDirective, HlsManifestCommitIdentity, HlsManifestDeliveryMode, + HlsMapWorkerPool, HlsMediaContainer, HlsPlaybackFamilyKey, HlsProxyManager, HlsSegmentCache, + HlsSegmentRepairManager, HlsSegmentWorkerPool, HlsSession, HlsSessionKey, OriginSegmentKey, RenderedManifest, + RenderedManifestStoreOutcome, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, SegmentFetchPolicy, +}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use axum::http::{HeaderMap, StatusCode}; +use shared::model::{ConfigPaths, HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripMode}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::{Mutex, Notify, RwLock}, +}; +use tuliprox_core::model::{ + AppConfig, Config, CustomStreamResponse, HlsManifestRecoveryBurstConfig, HlsSegmentRepairConfig, SourcesConfig, + StripConfig, +}; +use tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer; +use url::Url; + +pub(in crate::refresh::tests) async fn retry_test_manifest_recovery_chain( + request: &OriginRefreshRequest, + target_url: Url, + reject_reason: HlsManifestRejectLogReason, +) -> Result { + let fetch_context = manifest_fetch_context(request); + retry_hls_origin_manifest_recovery_chain( + &fetch_context, + test_manifest_origin_binding(target_url), + Some(reject_reason), + None, + HlsManifestAcceptanceTrigger::RecoveryRequired, + HlsManifestCommitAcceptanceMode::StrictPinnedHost, + |fetched, acceptance_mode| super::super::commit_manifest_recovery_candidate(request, fetched, acceptance_mode), + ) + .await +} + +pub(in crate::refresh::tests) fn path_has_extension(path: &str, extension: &str) -> bool { + std::path::Path::new(path).extension().is_some_and(|actual| actual.eq_ignore_ascii_case(extension)) +} + +pub(in crate::refresh::tests) fn test_manifest_origin_binding(target_url: Url) -> HlsManifestOriginBinding { + HlsManifestOriginBinding::new(target_url, None).expect("concrete HTTP test binding") +} + +pub(in crate::refresh::tests) fn test_session() -> Arc> { + Arc::new(RwLock::new(HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0))) +} + +pub(in crate::refresh::tests) fn repeated_transient_manifest(segment_count: usize) -> String { + let mut body = String::from( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n", + ); + for _ in 0..segment_count { + body.push_str("#EXTINF:4,\nsame.ts\n"); + } + body +} + +pub(in crate::refresh::tests) fn test_segment_repair_manager() -> Arc { + Arc::new(HlsSegmentRepairManager::new(HlsSegmentRepairConfig { + max_level: HlsSegmentRepairMode::Off, + apply_to_first_segments: 1, + max_parallel_repairs: 1, + ..Default::default() + })) +} + +pub(in crate::refresh::tests) fn test_app_config() -> Arc { + Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::empty()), + api_proxy: Arc::new(ArcSwapOption::empty()), + file_locks: Arc::new(tuliprox_core::utils::FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::empty()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(tuliprox_core::model::MediaToolCapabilities::default()), + }) +} + +pub(in crate::refresh::tests) fn test_deterministic_timeline_conflict() -> HlsDeterministicTimelineConflict { + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:490\n\ + #EXTINF:4,\n490.ts\n#EXTINF:4,\n480.ts\n#EXTINF:4,\n491.ts\n", + ); + deterministic_timeline_conflict_from_rejection( + &fetched, + &HlsManifestRejectLogReason::PublishedResourceReplay { + previous_proxy_tail: Some(2), + existing_proxy_seq: 0, + candidate_position: 1, + candidate_origin_seq: 491, + resource_key: super::super::super::resource_identity::HlsMediaResourceIdentity::from_url( + "http://origin.example.com/live/final/480.ts", + None, + ) + .semantic_key(), + decision: super::super::super::timeline::HlsResourceReplayDecision::RejectContradictoryOrder, + }, + ) + .expect("published replay rejection produces deterministic evidence") +} + +pub(in crate::refresh::tests) async fn refresh_session_with_origin_body(body: &'static str) -> Arc> { + let session = test_session(); + let server = spawn_test_origin(Arc::new(move |_path| (200, Vec::new(), body.to_string()))).await; + let entry = LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)) + .expect("valid origin entry"); + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 3 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + fresh_manifest_requirement_generation: None, + acceptance_directive: HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + post_refresh_runtime: None, + }; + + assert!(trigger_origin_refresh_sync(request).await); + { + let session = session.read().await; + assert!( + !session.segments.is_empty() || session.transient.last_manifest_body.is_some(), + "synchronous controlled origin work must commit origin state: mode={:?} path_condition={:?} failures={}", + session.mode, + session.origin_control.path_condition, + session.origin_refresh.consecutive_failures + ); + } + session +} + +pub(in crate::refresh::tests) struct TestOriginServer { + pub(in crate::refresh::tests) base_url: String, + pub(in crate::refresh::tests) requests: Arc>>, + pub(in crate::refresh::tests) raw_requests: Arc>>, + pub(in crate::refresh::tests) task: tokio::task::JoinHandle<()>, +} + +pub(in crate::refresh::tests) type TestOriginHandler = + Arc (u16, Vec<(&'static str, String)>, String) + Send + Sync>; + +impl Drop for TestOriginServer { + fn drop(&mut self) { self.task.abort(); } +} + +pub(in crate::refresh::tests) async fn spawn_test_origin(handler: TestOriginHandler) -> TestOriginServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let requests = Arc::new(Mutex::new(Vec::new())); + let requests_for_task = Arc::clone(&requests); + let raw_requests = Arc::new(Mutex::new(Vec::new())); + let raw_requests_for_task = Arc::clone(&raw_requests); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&requests_for_task); + let raw_requests = Arc::clone(&raw_requests_for_task); + let handler = Arc::clone(&handler); + tokio::spawn(async move { + let mut buf = vec![0_u8; 4096]; + let mut used = 0_usize; + loop { + let Ok(read) = socket.read(&mut buf[used..]).await else { + return; + }; + if read == 0 { + return; + } + used += read; + if used >= 4 && buf[..used].windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + if used == buf.len() { + return; + } + } + let request = String::from_utf8_lossy(&buf[..used]).into_owned(); + let path = + request.lines().next().and_then(|line| line.split_whitespace().nth(1)).unwrap_or("/").to_string(); + requests.lock().await.push(path.clone()); + raw_requests.lock().await.push(request); + let (status, headers, body) = handler(path); + let reason = match status { + 200 => "OK", + 302 => "Found", + 404 => "Not Found", + 407 => "Proxy Authentication Required", + 500 => "Internal Server Error", + _ => "Status", + }; + let mut response = + format!("HTTP/1.1 {status} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n", body.len()); + for (name, value) in headers { + response.push_str(name); + response.push_str(": "); + response.push_str(&value); + response.push_str("\r\n"); + } + response.push_str("\r\n"); + response.push_str(&body); + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + TestOriginServer { base_url: format!("http://{addr}"), requests, raw_requests, task } +} + +pub(in crate::refresh::tests) fn request_header_value<'a>(request: &'a str, expected_name: &str) -> Option<&'a str> { + request.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case(expected_name).then_some(value.trim()) + }) +} + +pub(in crate::refresh::tests) fn no_delay_policy() -> RetryPolicy { + RetryPolicy { delays_ms: [0, 0, 0, 0, 0], jitter_max_ms: 0 } +} + +pub(in crate::refresh::tests) fn test_recovery_timing_policy(expected_eta_ms: u64) -> HlsRecoveryTimingPolicy { + HlsRecoveryTimingPolicy::new( + HlsOperationTimeoutMs::from_millis(2_000), + HlsOperationTimeoutMs::from_millis(2_000), + HlsRecoveryEtaMs::from_millis(expected_eta_ms), + HlsRecoveryEtaMs::from_millis(expected_eta_ms), + ) +} + +pub(in crate::refresh::tests) fn test_acceptance_episode_timing( + started_at_ms: u64, + burst_plan: shared::model::HlsManifestRecoveryBurstPlan, +) -> HlsAcceptanceEpisodeTiming { + HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput { + started_at_ms, + burst_plan, + target_duration_ms: 4_000, + transition_margin: HlsTransitionMarginMs::from_millis(4_000), + workload: HlsRecoveryWorkload::clear_fetch(), + observed_latency: HlsObservedRecoveryLatency::default(), + required_terminal_media_key: None, + terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, + policy: test_recovery_timing_policy(2_000), + }) +} + +pub(in crate::refresh::tests) fn test_switch_staging_acceptance_episode_timing( + started_at_ms: u64, + burst_plan: shared::model::HlsManifestRecoveryBurstPlan, +) -> HlsAcceptanceEpisodeTiming { + HlsAcceptanceEpisodeTiming::from_input(&HlsAcceptanceEpisodeTimingInput { + started_at_ms, + burst_plan, + target_duration_ms: 4_000, + transition_margin: HlsTransitionMarginMs::from_millis(4_000), + workload: HlsRecoveryWorkloadEnvelope::acceptance_policy().ceiling(), + observed_latency: HlsObservedRecoveryLatency::default(), + required_terminal_media_key: None, + terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, + policy: test_recovery_timing_policy(2_000), + }) +} + +pub(in crate::refresh::tests) fn test_origin_refresh_request(session: Arc>) -> OriginRefreshRequest { + let entry = + LiveHlsOriginEntry::parse("http://origin.example.com/live/user/pass/12345.m3u8").expect("valid origin entry"); + OriginRefreshRequest { + app_config: test_app_config(), + session, + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + fresh_manifest_requirement_generation: None, + acceptance_directive: HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + post_refresh_runtime: None, + } +} + +pub(in crate::refresh::tests) fn bind_refresh_request_to_app_state( + mut request: OriginRefreshRequest, + ctx: &crate::HlsCtx, +) -> OriginRefreshRequest { + request.segment_cache = Arc::clone(ctx.hls_proxy.segment_cache()); + request.hls_proxy = Arc::clone(&ctx.hls_proxy); + request.segment_repair = Arc::clone(ctx.hls_proxy.segment_repair()); + request.segment_worker_pool = Arc::clone(ctx.hls_proxy.segment_worker_pool()); + request.map_worker_pool = Arc::clone(ctx.hls_proxy.map_worker_pool()); + request.post_refresh_runtime = Some(HlsPostRefreshRuntime { ctx: ctx.downgrade() }); + request +} + +pub(in crate::refresh::tests) fn post_refresh_live_manifest_snapshot() -> HlsLeaseManifestSnapshot { + HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(1), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 1, + delivered_at_ms: 1, + first_proxy_seq: 0, + last_proxy_seq: 2, + visible_segments: Arc::from([ + HlsLeaseManifestSegment { + proxy_seq: 0, + duration_ms: 4_000, + uri: "0.ts".into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }, + HlsLeaseManifestSegment { + proxy_seq: 1, + duration_ms: 4_000, + uri: "1.ts".into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }, + HlsLeaseManifestSegment { + proxy_seq: 2, + duration_ms: 4_000, + uri: "2.ts".into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }, + ]), + discontinuity_sequence: 0, + target_duration_ms: 4_000, + playlist_duration_ms: 12_000, + last_visible_media_end_ms: 12_000, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + } +} + +pub(in crate::refresh::tests) fn fetched_manifest(body: &str) -> FetchedOriginManifest { + FetchedOriginManifest { + body: body.to_string(), + final_manifest_url: "http://origin.example.com/live/final/index.m3u8".to_string(), + resolved_request_url: "http://origin.example.com/live/user/pass/12345.m3u8".to_string(), + redirect_host: Some("origin.example.com".to_string()), + provider_url_index: None, + provider_session_headers: HeaderMap::new(), + status: StatusCode::OK, + attempts: 1, + candidate_requests: 1, + selection: HlsManifestFetchSelection::Initial, + } +} + +pub(in crate::refresh::tests) fn host_from_base_url(base_url: &str) -> String { + url::Url::parse(base_url).expect("base url").host_str().expect("host").to_string() +} + +pub(in crate::refresh::tests) fn manifest_body() -> String { + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nseg.ts\n".to_string() +} + +pub(in crate::refresh::tests) fn three_segment_manifest_body(media_sequence: u64) -> String { + format!( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:{media_sequence}\n#EXT-X-TARGETDURATION:4\n\ + #EXTINF:4.0,\n{media_sequence}.ts\n\ + #EXTINF:4.0,\n{}.ts\n\ + #EXTINF:4.0,\n{}.ts\n", + media_sequence.saturating_add(1), + media_sequence.saturating_add(2) + ) +} + +pub(in crate::refresh::tests) async fn publish_ready_test_manifest( + session: &Arc>, + rendered_at_ms: u64, +) { + let mut session = session.write().await; + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: rendered_at_ms }; + } + session.render_and_store_manifest(rendered_at_ms).expect("test live manifest publishes"); +} + +pub(in crate::refresh::tests) const SWITCH_MAP_BODY: &[u8] = b"complete-switch-map"; + +pub(in crate::refresh::tests) const SWITCH_SEGMENT_BODY: &[u8] = b"complete-switch-segment-body"; + +pub(in crate::refresh::tests) const CRITICAL_HANDOFF_TS_BODY: &[u8] = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); + +pub(in crate::refresh::tests) const CRITICAL_HANDOFF_MANIFEST_BODY: &[u8] = b"#EXTM3U\n\ + #EXT-X-MEDIA-SEQUENCE:900\n\ + #EXT-X-TARGETDURATION:4\n\ + #EXTINF:4.0,\nfirst.ts\n\ + #EXTINF:4.0,\nsecond.ts\n"; + +pub(in crate::refresh::tests) struct ControlledSwitchOriginServer { + pub(in crate::refresh::tests) base_url: String, + pub(in crate::refresh::tests) requests: Arc>>, + pub(in crate::refresh::tests) segment_prefix_written: Arc, + pub(in crate::refresh::tests) release_segment_body: Arc, + pub(in crate::refresh::tests) task: tokio::task::JoinHandle<()>, +} + +impl Drop for ControlledSwitchOriginServer { + fn drop(&mut self) { self.task.abort(); } +} + +pub(in crate::refresh::tests) async fn await_controlled_switch_segment_prefix( + origin: &ControlledSwitchOriginServer, + task: &mut tokio::task::JoinHandle, + task_name: &str, +) { + tokio::select! { + () = origin.segment_prefix_written.notified() => {} + result = &mut *task => { + panic!("{task_name} completed before controlled segment staging reached the network boundary: {result:?}"); + } + () = tokio::time::sleep(Duration::from_secs(10)) => { + panic!("{task_name} did not reach the controlled segment staging boundary before the test deadline"); + } + } +} + +pub(in crate::refresh::tests) struct CriticalEmergencyOriginServer { + pub(in crate::refresh::tests) base_url: String, + pub(in crate::refresh::tests) manifest_requests: Arc, + pub(in crate::refresh::tests) segment_requests: Arc, + pub(in crate::refresh::tests) segment_prefix_written: Arc, + pub(in crate::refresh::tests) release_segment_body: Arc, + pub(in crate::refresh::tests) task: tokio::task::JoinHandle<()>, +} + +impl Drop for CriticalEmergencyOriginServer { + fn drop(&mut self) { self.task.abort(); } +} + +pub(in crate::refresh::tests) async fn read_test_request_path(socket: &mut tokio::net::TcpStream) -> Option { + let mut buf = vec![0_u8; 4_096]; + let mut used = 0_usize; + loop { + let read = socket.read(&mut buf[used..]).await.ok()?; + if read == 0 { + return None; + } + used = used.saturating_add(read); + if used >= 4 && buf[..used].windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + if used == buf.len() { + return None; + } + } + String::from_utf8_lossy(&buf[..used]) + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .map(str::to_string) +} + +pub(in crate::refresh::tests) async fn write_test_response( + socket: &mut tokio::net::TcpStream, + status: u16, + body: &[u8], +) { + let reason = if status == 200 { "OK" } else { "Not Found" }; + let head = format!("HTTP/1.1 {status} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(body).await; +} + +pub(in crate::refresh::tests) async fn spawn_controlled_switch_origin() -> ControlledSwitchOriginServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("controlled switch origin binds"); + let addr = listener.local_addr().expect("controlled switch origin address"); + let requests = Arc::new(Mutex::new(Vec::new())); + let requests_for_task = Arc::clone(&requests); + let segment_prefix_written = Arc::new(Notify::new()); + let segment_prefix_written_for_task = Arc::clone(&segment_prefix_written); + let release_segment_body = Arc::new(Notify::new()); + let release_segment_body_for_task = Arc::clone(&release_segment_body); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&requests_for_task); + let segment_prefix_written = Arc::clone(&segment_prefix_written_for_task); + let release_segment_body = Arc::clone(&release_segment_body_for_task); + tokio::spawn(async move { + let Some(path) = read_test_request_path(&mut socket).await else { + return; + }; + requests.lock().await.push(path.clone()); + match path.as_str() { + "/live/final/init.mp4" => write_test_response(&mut socket, 200, SWITCH_MAP_BODY).await, + "/live/final/first.ts" => { + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + SWITCH_SEGMENT_BODY.len() + ); + if socket.write_all(head.as_bytes()).await.is_err() { + return; + } + let split = SWITCH_SEGMENT_BODY.len() / 2; + if socket.write_all(&SWITCH_SEGMENT_BODY[..split]).await.is_err() { + return; + } + segment_prefix_written.notify_one(); + release_segment_body.notified().await; + let _ = socket.write_all(&SWITCH_SEGMENT_BODY[split..]).await; + } + _ => write_test_response(&mut socket, 404, &[]).await, + } + }); + } + }); + ControlledSwitchOriginServer { + base_url: format!("http://{addr}"), + requests, + segment_prefix_written, + release_segment_body, + task, + } +} + +pub(in crate::refresh::tests) async fn spawn_critical_emergency_origin() -> CriticalEmergencyOriginServer { + spawn_critical_emergency_origin_with_control(false).await +} + +pub(in crate::refresh::tests) async fn spawn_critical_emergency_origin_with_control( + pause_segment: bool, +) -> CriticalEmergencyOriginServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("critical emergency origin binds"); + let addr = listener.local_addr().expect("critical emergency origin address"); + let manifest_requests = Arc::new(AtomicUsize::new(0)); + let manifest_requests_for_task = Arc::clone(&manifest_requests); + let segment_requests = Arc::new(AtomicUsize::new(0)); + let segment_requests_for_task = Arc::clone(&segment_requests); + let segment_prefix_written = Arc::new(Notify::new()); + let segment_prefix_written_for_task = Arc::clone(&segment_prefix_written); + let release_segment_body = Arc::new(Notify::new()); + let release_segment_body_for_task = Arc::clone(&release_segment_body); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let manifest_requests = Arc::clone(&manifest_requests_for_task); + let segment_requests = Arc::clone(&segment_requests_for_task); + let segment_prefix_written = Arc::clone(&segment_prefix_written_for_task); + let release_segment_body = Arc::clone(&release_segment_body_for_task); + tokio::spawn(async move { + let Some(path) = read_test_request_path(&mut socket).await else { + return; + }; + if path_has_extension(&path, "m3u8") { + let request_index = manifest_requests.fetch_add(1, Ordering::SeqCst); + if request_index == 0 { + write_test_response(&mut socket, 200, CRITICAL_HANDOFF_MANIFEST_BODY).await; + } else { + write_test_response(&mut socket, 407, b"retryable manifest failure").await; + } + } else if path.ends_with("/first.ts") { + segment_requests.fetch_add(1, Ordering::SeqCst); + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + CRITICAL_HANDOFF_TS_BODY.len() + ); + if socket.write_all(head.as_bytes()).await.is_err() { + return; + } + let split = CRITICAL_HANDOFF_TS_BODY.len() / 2; + if socket.write_all(&CRITICAL_HANDOFF_TS_BODY[..split]).await.is_err() { + return; + } + segment_prefix_written.notify_one(); + if pause_segment { + release_segment_body.notified().await; + } + let _ = socket.write_all(&CRITICAL_HANDOFF_TS_BODY[split..]).await; + } else { + write_test_response(&mut socket, 404, &[]).await; + } + }); + } + }); + CriticalEmergencyOriginServer { + base_url: format!("http://{addr}"), + manifest_requests, + segment_requests, + segment_prefix_written, + release_segment_body, + task, + } +} + +pub(in crate::refresh::tests) fn switch_manifest_body(include_map: bool) -> String { + let map = if include_map { "#EXT-X-MAP:URI=\"init.mp4\"\n" } else { "" }; + format!( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:500\n#EXT-X-TARGETDURATION:4\n{map}#EXTINF:4.0,\nfirst.ts\n#EXTINF:4.0,\nsecond.ts\n" + ) +} + +pub(in crate::refresh::tests) fn switch_fetched_manifest(base_url: &str, include_map: bool) -> FetchedOriginManifest { + FetchedOriginManifest { + body: switch_manifest_body(include_map), + final_manifest_url: format!("{base_url}/live/final/index.m3u8"), + resolved_request_url: format!("{base_url}/live/user/pass/12345.m3u8"), + redirect_host: None, + provider_url_index: None, + provider_session_headers: HeaderMap::new(), + status: StatusCode::OK, + attempts: 1, + candidate_requests: 1, + selection: HlsManifestFetchSelection::Initial, + } +} + +pub(in crate::refresh::tests) fn switch_test_request( + session: Arc>, + segment_cache: Arc, + base_url: &str, +) -> OriginRefreshRequest { + let mut request = test_origin_refresh_request(session); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{base_url}/live/user/pass/12345.m3u8")).expect("switch origin entry"); + request.client = reqwest::Client::builder().no_proxy().build().expect("switch client"); + request.no_redirect_client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("switch no-redirect client"); + request.hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(segment_cache.cache_path(), 300)); + request.segment_cache = segment_cache; + request.segment_worker_pool = Arc::new(HlsSegmentWorkerPool::new(SegmentFetchPolicy { + origin_segment_timeout_ms: 2_000, + retry_delays_ms: [0; 5], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + })); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Friendly }; + request +} + +pub(in crate::refresh::tests) async fn prepare_cross_host_baseline(session: &Arc>) { + let body = + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:500\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nold500.ts\n#EXTINF:4.0,\nold501.ts\n"; + let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = + tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( + body, + "http://previous.example.com/live/user/pass/12345.m3u8", + ) + else { + panic!("baseline manifest parses as normal timeline"); + }; + let mut session = session.write().await; + session + .apply_origin_manifest_for_host(&manifest, crate::timeline::effective_origin_host_id("previous.example.com")) + .expect("baseline timeline commits"); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + session.origin_control.pinned_host = Some("previous.example.com".to_string()); + session.origin_control.origin_epoch = session.origin_epoch; +} + +pub(in crate::refresh::tests) async fn install_published_recovery_binding( + session: &Arc>, + request_url: &str, + provider_url_index: Option, +) { + let binding = HlsManifestOriginBinding::new( + Url::parse(request_url).expect("published recovery request URL"), + provider_url_index, + ) + .expect("published recovery binding"); + let mut session = session.write().await; + session.origin_control.manifest_origin_binding = Some(binding); + session.origin_control.record_media_progress(1, 4_000); + let evidence_proxy_seq = session + .segments + .iter() + .find_map(|(proxy_seq, segment)| { + (!is_hls_provisioning_segment(segment) && !is_hls_provisioning_gap_segment(segment)).then_some(*proxy_seq) + }) + .unwrap_or_else(|| { + let proxy_seq = session.proxy_next_seq.unwrap_or(0); + let origin_sequence = session.origin_seq_highwater.unwrap_or(proxy_seq); + let origin_epoch = session.origin_epoch; + let cache_key = SegmentCacheKey::new(session.proxy_session_id.clone(), proxy_seq, "ts"); + session.segments.insert( + proxy_seq, + SegmentEntry { + origin_key: OriginSegmentKey { + origin_epoch, + effective_host_id: 0, + host_local_sequence: origin_sequence, + host_local_index: 0, + }, + proxy_seq, + duration_ms: 4_000, + proxy_file_ext: "ts".to_string(), + content_type: "video/mp2t".to_string(), + cache_key, + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + encryption: None, + origin_fetch_ref: None, + status: SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 1 }, + last_rendered_at_ms: None, + access: Arc::new(CacheAccessState::new()), + }, + ); + session.publishable_origin_head_proxy_seq.get_or_insert(proxy_seq); + session.publishable_origin_tail_proxy_seq = Some(proxy_seq); + session.proxy_next_seq = Some(proxy_seq.saturating_add(1)); + proxy_seq + }); + let discontinuity_sequence = session.discontinuity_sequence; + let outcome = session.store_rendered_manifest(RenderedManifest { + body: "#EXTM3U\n".to_string(), + first_proxy_seq: evidence_proxy_seq, + last_proxy_seq: evidence_proxy_seq, + discontinuity_sequence, + target_duration_ms: 4_000, + playlist_duration_ms: 4_000, + valid_until_ms: 10_000, + render_gap_segments: 0, + rendered_at_ms: 1, + segment_proxy_seqs: vec![evidence_proxy_seq], + }); + assert_eq!(outcome, RenderedManifestStoreOutcome::Stored); + assert!(session.published_live_origin_baseline.is_some()); +} + +pub(in crate::refresh::tests) async fn commit_ready_baseline_snapshot( + session: &Arc>, + cache: &HlsSegmentCache, + now_ms: u64, +) -> HlsLeaseManifestSnapshot { + let cache_keys = { + let session = session.read().await; + session.segments.values().map(|segment| segment.cache_key.clone()).collect::>() + }; + for cache_key in &cache_keys { + cache.write_bytes_and_commit(cache_key, CRITICAL_HANDOFF_TS_BODY).await.expect("baseline TS object commits"); + } + + let (visible_segments, discontinuity_sequence, target_duration_ms) = { + let mut session = session.write().await; + for segment in session.segments.values_mut() { + assert!(segment.map_ref.is_none()); + assert!(segment.encryption.is_none()); + segment.status = SegmentCacheStatus::Ready { + content_length: u64::try_from(CRITICAL_HANDOFF_TS_BODY.len()).unwrap_or(u64::MAX), + ready_at_ms: now_ms, + }; + } + let visible_segments = session + .segments + .values() + .map(|segment| HlsLeaseManifestSegment { + proxy_seq: segment.proxy_seq, + duration_ms: segment.duration_ms, + uri: format!("/hls/test/{:06}.ts", segment.proxy_seq).into(), + discontinuity_before: segment.discontinuity_before, + map_ref_ready: true, + encryption: None, + }) + .collect::>(); + ( + visible_segments, + session.discontinuity_sequence, + session.target_duration.map_or(4_000, |seconds| u64::from(seconds).saturating_mul(1_000)), + ) + }; + let first_proxy_seq = visible_segments.first().expect("baseline head").proxy_seq; + let last_proxy_seq = visible_segments.last().expect("baseline tail").proxy_seq; + let playlist_duration_ms = + visible_segments.iter().fold(0_u64, |duration_ms, segment| duration_ms.saturating_add(segment.duration_ms)); + HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(now_ms), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 0, + delivered_at_ms: now_ms, + first_proxy_seq, + last_proxy_seq, + visible_segments: Arc::from(visible_segments), + discontinuity_sequence, + target_duration_ms, + playlist_duration_ms, + last_visible_media_end_ms: playlist_duration_ms, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + } +} + +pub(in crate::refresh::tests) async fn prepare_active_critical_handoff_lease( + session: &Arc>, + cache: &HlsSegmentCache, + now_ms: u64, +) -> (Arc, HlsAccessLeaseId, HlsAccessLease) { + let manifest_snapshot = commit_ready_baseline_snapshot(session, cache, now_ms).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease_id = HlsAccessLeaseId("critical-emergency-handoff".to_string()); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(cache.cache_path(), 300)); + hls_proxy + .prepare_access_lease(HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("critical-user", "critical-client"), + proxy_session_id.clone(), + "critical-user".to_string(), + "critical-session".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + 120_000, + )) + .await; + assert!(hls_proxy + .activate_access_lease( + &lease_id, + &proxy_session_id, + now_ms, + HlsAccessLeaseTiming { active_window_ms: 120_000, valid_window_ms: 120_000 }, + ) + .await + .is_activated()); + let publication_guard = hls_proxy + .prepare_access_lease_manifest_publication(&lease_id, &proxy_session_id, now_ms) + .await + .expect("active lease accepts authoritative manifest publication"); + assert!(hls_proxy + .commit_access_lease_manifest_publication( + &lease_id, + &proxy_session_id, + publication_guard, + manifest_snapshot, + now_ms, + ) + .await + .is_committed()); + let lease = hls_proxy + .access_lease_response_snapshot(&lease_id, &proxy_session_id, now_ms) + .await + .expect("critical live lease snapshot"); + (hls_proxy, lease_id, lease) +} + +pub(in crate::refresh::tests) fn critical_handoff_app_config() -> Arc { + let app_config = test_app_config(); + app_config.custom_stream_response.store(Some(Arc::new(CustomStreamResponse { + channel_unavailable: Some(TransportStreamBuffer::new(CRITICAL_HANDOFF_TS_BODY.to_vec())), + user_connections_exhausted: None, + provider_connections_exhausted: None, + low_priority_preempted: None, + user_account_expired: None, + panel_api_provisioning: None, + hls_session_or_lease_expired: None, + panel_api_provisioning_hls_segments: Vec::new(), + }))); + app_config +} + +pub(in crate::refresh::tests) fn candidate_handoff_preview( + session: &HlsSession, + body: &str, + now_ms: u64, +) -> ( + tuliprox_parser::hls::origin_manifest::ParsedOriginManifest, + Vec, + crate::timeline::HlsOriginHandoffPreview, +) { + let final_url = "https://candidate.example/live/index.m3u8"; + let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(mut manifest) = + tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest(body, final_url) + else { + panic!("candidate fixture must parse as a normal manifest"); + }; + let key_resources = + super::super::commit::materialize_normal_key_resources(&mut manifest, b"secret", now_ms, 60_000); + let preview = session + .preview_origin_handoff_manifest(&manifest, crate::timeline::effective_origin_host_id("candidate.example"), 0) + .expect("candidate handoff preview"); + (manifest, key_resources, preview) +} + +pub(in crate::refresh::tests) async fn mark_full_burst_ready_for_switch_staging( + session: &Arc>, + fetched: &FetchedOriginManifest, +) { + let candidate_workload = { + let session = session.read().await; + let (_, key_resources, preview) = candidate_handoff_preview(&session, &fetched.body, 100); + let first_segment = preview.segments.first().expect("switch candidate recovery segment"); + let required_map = + first_segment.map_ref.and_then(|map_id| preview.maps.iter().find(|map| map.proxy_map_id == map_id)); + super::super::switch_staging::handoff_preview_recovery_workload( + &session, + first_segment, + required_map, + &key_resources, + 100, + ) + }; + let mut session = session.write().await; + let burst_plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); + session.origin_control.begin_acceptance_episode( + 100, + burst_plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &test_switch_staging_acceptance_episode_timing(100, burst_plan), + ); + let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); + episode.record_full_burst(); + episode.state = super::super::super::manifest_acceptance::HlsManifestAcceptanceState::StagingSwitchSegment; + let effective_host = fetched_effective_manifest_host(fetched); + let identity = super::super::super::manifest_acceptance::HlsManifestRecoveryCandidateIdentity::from_candidate( + 0, + effective_host.as_deref(), + &fetched.body, + ); + assert_eq!( + episode.select_candidate(episode.generation, identity), + super::super::super::manifest_acceptance::HlsRecoveryWorkloadBindingUpdate::Applied + ); + let mut binding_probe = episode.clone(); + assert_eq!( + binding_probe.bind_selected_candidate(episode.generation, identity, candidate_workload), + super::super::super::manifest_acceptance::HlsRecoveryWorkloadBindingUpdate::Applied, + "switch-staging fixture must admit the selected candidate before network staging" + ); +} + +pub(in crate::refresh::tests) async fn assert_incompatible_switch_is_rejected_before_timeline_commit( + baseline_body: &str, + candidate_body: &str, + expected_reason: HlsManifestRejectLogReason, +) { + let session = test_session(); + let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(baseline) = + tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( + baseline_body, + "http://previous.example.com/live/index.m3u8", + ) + else { + panic!("baseline parses as normal timeline"); + }; + { + let mut session = session.write().await; + session + .apply_origin_manifest_for_host( + &baseline, + crate::timeline::effective_origin_host_id("previous.example.com"), + ) + .expect("baseline timeline commits"); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + session.origin_control.pinned_host = Some("previous.example.com".to_string()); + session.origin_control.origin_epoch = session.origin_epoch; + } + let temp_dir = tempfile::tempdir().expect("switch cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let base_url = "http://127.0.0.1:9"; + let request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), base_url); + let fetched = FetchedOriginManifest { + body: candidate_body.to_string(), + final_manifest_url: format!("{base_url}/live/final/index.m3u8"), + resolved_request_url: format!("{base_url}/live/user/pass/12345.m3u8"), + redirect_host: None, + provider_url_index: None, + provider_session_headers: HeaderMap::new(), + status: StatusCode::OK, + attempts: 1, + candidate_requests: 1, + selection: HlsManifestFetchSelection::Initial, + }; + mark_full_burst_ready_for_switch_staging(&session, &fetched).await; + let before = { + let session = session.read().await; + (session.origin_epoch, session.proxy_next_seq, session.segments.len(), session.maps.len()) + }; + + let result = super::super::commit_manifest_recovery_candidate( + &request, + fetched, + HlsManifestCommitAcceptanceMode::AllowHeldHostSwitchCandidate, + ) + .await; + + assert!(matches!( + result, + Err(HlsManifestCommitError::TimelineRejected { reason }) if reason == expected_reason + )); + let session = session.read().await; + assert_eq!((session.origin_epoch, session.proxy_next_seq, session.segments.len(), session.maps.len()), before); + drop(session); + assert!(!cache.has_active_temp_files()); +} + +#[derive(Clone, Copy)] +pub(in crate::refresh::tests) enum StaleSwitchGeneration { + Acceptance, + Progress, + PinnedHostRecovered, +} + +pub(in crate::refresh::tests) async fn assert_stale_switch_generation_rejects_commit( + stale_generation: StaleSwitchGeneration, +) { + let temp_dir = tempfile::tempdir().expect("switch cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let origin = spawn_controlled_switch_origin().await; + let session = test_session(); + prepare_cross_host_baseline(&session).await; + let fetched = switch_fetched_manifest(&origin.base_url, true); + mark_full_burst_ready_for_switch_staging(&session, &fetched).await; + let effective_host_id = crate::timeline::effective_origin_host_id(&host_from_base_url(&origin.base_url)); + let (baseline_epoch, baseline_proxy_next, staged_segment_key, staged_map_key) = { + let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = + tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( + &fetched.body, + &fetched.final_manifest_url, + ) + else { + panic!("switch manifest parses as normal timeline"); + }; + let session = session.read().await; + let preview = session.preview_origin_handoff_manifest(&manifest, effective_host_id, 0).expect("switch preview"); + ( + session.origin_epoch, + session.proxy_next_seq, + preview.segments.first().expect("first staged segment").cache_key.clone(), + preview.maps.first().expect("staged map").cache_key.clone(), + ) + }; + let request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); + let cleanup_manager = Arc::clone(&request.hls_proxy); + let mut commit_task = tokio::spawn(async move { + super::super::commit_manifest_recovery_candidate( + &request, + fetched, + HlsManifestCommitAcceptanceMode::AllowHeldHostSwitchCandidate, + ) + .await + .map(|_| ()) + }); + + await_controlled_switch_segment_prefix(&origin, &mut commit_task, "stale-generation switch commit").await; + { + let mut session = session.write().await; + match stale_generation { + StaleSwitchGeneration::Acceptance => { + let episode = session.origin_control.acceptance_episode.as_mut().expect("active acceptance episode"); + episode.generation = super::super::super::manifest_acceptance::HlsManifestAcceptanceGeneration( + episode.generation.0.saturating_add(1), + ); + } + StaleSwitchGeneration::Progress => { + session.origin_control.progress_generation = + session.origin_control.progress_generation.saturating_add(1); + } + StaleSwitchGeneration::PinnedHostRecovered => { + session.origin_control.pinned_host = Some("recovered.example.com".to_string()); + session.last_effective_manifest_host = Some("recovered.example.com".to_string()); + } + } + } + origin.release_segment_body.notify_one(); + + let result = commit_task.await.expect("stale switch task"); + assert!(matches!( + result, + Err(HlsManifestCommitError::TimelineRejected { reason: HlsManifestRejectLogReason::StagedSwitchInvalidated }) + )); + { + let session = session.read().await; + assert_eq!(session.origin_epoch, baseline_epoch); + assert_eq!(session.proxy_next_seq, baseline_proxy_next); + assert!(session.segments.values().all(|segment| segment.origin_key.origin_epoch == baseline_epoch)); + } + assert_eq!(cache.metadata(&staged_segment_key).await.expect("stale segment metadata"), None); + assert_eq!(cache.metadata(&staged_map_key).await.expect("stale map metadata"), None); + assert_eq!(cleanup_manager.cache_deletion_queue_usage(), (0, 0)); + assert!(!cache.has_active_temp_files()); +} diff --git a/backend/hls/src/refresh/tests/switch_staging.rs b/backend/hls/src/refresh/tests/switch_staging.rs new file mode 100644 index 000000000..29875c1b0 --- /dev/null +++ b/backend/hls/src/refresh/tests/switch_staging.rs @@ -0,0 +1,1070 @@ +use super::{ + assert_incompatible_switch_is_rejected_before_timeline_commit, assert_stale_switch_generation_rejects_commit, + await_controlled_switch_segment_prefix, bind_refresh_request_to_app_state, candidate_handoff_preview, + commit_fetched_manifest, fetch_and_commit_manifest_with_policy, fetched_manifest, host_from_base_url, + install_published_recovery_binding, manifest_body, manifest_fetch_context, manifest_recovery_trigger, + mark_full_burst_ready_for_switch_staging, no_delay_policy, path_has_extension, prepare_cross_host_baseline, + publish_ready_test_manifest, retry_test_manifest_recovery_chain, score_manifest_recovery_candidate, + spawn_controlled_switch_origin, spawn_test_origin, switch_fetched_manifest, switch_manifest_body, + switch_test_request, test_acceptance_episode_timing, test_app_config, test_origin_refresh_request, + test_segment_repair_manager, test_session, three_segment_manifest_body, trigger_origin_refresh_sync, + HlsManifestAcceptanceTrigger, HlsManifestCommitAcceptanceMode, HlsManifestCommitError, + HlsManifestCommitRequirement, HlsManifestFetchSelection, HlsManifestOriginBinding, HlsManifestOriginQualityScore, + HlsManifestRefreshCompletionDiagnostic, HlsManifestRejectLogReason, HlsRecoveryEncryptionReadiness, + HlsRecoveryMapWorkload, HlsRecoveryMediumReadiness, HlsRecoveryObjectReadiness, HlsRecoverySegmentWorkload, + HlsRecoveryWorkload, LiveHlsOriginEntry, ManifestRecoverySelectionLogPhase, OriginManifestFetchError, + OriginRefreshRequest, StaleSwitchGeneration, SWITCH_MAP_BODY, SWITCH_SEGMENT_BODY, +}; +use crate::{ + HlsBoundAccountAcquireErrorKind, HlsFreshManifestRequiredReason, HlsManifestAcceptanceDirective, HlsMapWorkerPool, + HlsOriginAccountBinding, HlsOriginIoContext, HlsProxyManager, HlsSegmentCache, HlsSegmentWorkerPool, HlsSession, + HlsSessionKey, HlsSessionMode, MapCacheStatus, SegmentCacheStatus, TransientObjectFetchDecision, + TransientPassthroughReason, +}; +use axum::http::HeaderMap; +use shared::model::{ConfigProviderDto, HlsManifestRecoveryBurstLevel, HlsStripMode, ProviderUrlSelectionPolicy}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; +use tokio::io::AsyncReadExt; +use tuliprox_core::{ + model::{Config, ConfigProvider, HlsManifestRecoveryBurstConfig, StripConfig}, + utils::current_time_millis, +}; +use tuliprox_session::ConnectionKind; +use url::Url; + +#[test] +fn manifest_recovery_burst_levels_map_to_candidate_counts() { + let cases = [ + (HlsManifestRecoveryBurstLevel::Off, 1, 1), + (HlsManifestRecoveryBurstLevel::Friendly, 2, 1), + (HlsManifestRecoveryBurstLevel::Cautious, 3, 1), + (HlsManifestRecoveryBurstLevel::Balanced, 4, 1), + (HlsManifestRecoveryBurstLevel::Intense, 5, 1), + (HlsManifestRecoveryBurstLevel::Aggressive, 6, 1), + (HlsManifestRecoveryBurstLevel::Beast, 6, 2), + ]; + for (level, expected_slots, expected_lanes) in cases { + let plan = level.plan(); + let expected_candidates = expected_slots * expected_lanes; + assert_eq!(plan.slots, expected_slots); + assert_eq!(plan.lanes_per_slot, expected_lanes); + assert_eq!(plan.total_candidates(), expected_candidates); + assert_eq!(level.total_candidates(), expected_candidates); + } +} + +#[test] +fn recovery_selection_log_phase_distinguishes_single_candidate_from_burst() { + assert_eq!(ManifestRecoverySelectionLogPhase::from_candidate_count(1), ManifestRecoverySelectionLogPhase::Recovery); + assert_eq!(ManifestRecoverySelectionLogPhase::from_candidate_count(2), ManifestRecoverySelectionLogPhase::Burst); + assert_eq!(ManifestRecoverySelectionLogPhase::Recovery.as_log_label(), "recovery"); + assert_eq!(ManifestRecoverySelectionLogPhase::Burst.as_log_label(), "burst"); +} + +#[test] +fn different_host_candidate_is_not_committed_immediately() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.origin_seq_highwater = Some(758); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + let binding = HlsManifestOriginBinding::new( + Url::parse("https://previous.example.com/live/index.m3u8?token=baseline").expect("baseline URL"), + Some(0), + ) + .expect("baseline binding"); + session.origin_control.manifest_origin_binding = Some(binding.clone()); + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:758\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg758.ts\n#EXTINF:4.0,\nseg759.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(matches!(result, Err(HlsManifestCommitError::RetryCurrentTarget))); + assert!(session.transient.last_manifest_body.is_none()); + assert_eq!(session.origin_control.manifest_origin_binding.as_ref(), Some(&binding)); +} + +#[test] +fn rejected_transient_candidate_preserves_committed_state_and_pending_handoff() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + let mut request = test_origin_refresh_request(test_session()); + request.transient_resource_ttl_ms = 1; + let baseline = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ + #EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4,\nsegment.ts\n", + ); + assert!(commit_fetched_manifest(&mut session, &baseline, &request, 100).is_ok()); + session.mark_pending_handoff_discontinuity(7); + let baseline_body = session.transient.last_manifest_body.clone().expect("baseline manifest"); + let baseline_generation = session.transient.manifest_generation(); + let baseline_resources = session.transient.resources.keys().cloned().collect::>(); + let baseline_highwater = session.origin_seq_highwater; + let key_format = "x".repeat(crate::manifest_limits::MAX_HLS_ENCRYPTION_DIRECTIVE_BYTES); + let candidate = fetched_manifest(&format!( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n\ + #EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\",KEYFORMAT=\"{key_format}\"\n\ + #EXTINF:4,\nsegment.ts\n" + )); + + let result = commit_fetched_manifest(&mut session, &candidate, &request, 102); + + assert!(matches!( + result, + Err(HlsManifestCommitError::LocalRepresentationLimit(violation)) + if violation.kind + == crate::manifest_limits::HlsManifestLimitKind::LeaseSnapshotEncryptionDirectiveBytes + )); + assert!(Arc::ptr_eq( + session.transient.last_manifest_body.as_ref().expect("baseline remains current"), + &baseline_body + )); + assert_eq!(session.transient.manifest_generation(), baseline_generation); + assert_eq!( + session.transient.resources.keys().cloned().collect::>(), + baseline_resources + ); + assert_eq!(session.pending_handoff_discontinuity_sequence, Some(7)); + assert_eq!(session.origin_seq_highwater, baseline_highwater); +} + +#[test] +fn refresh_completion_diagnostics_separate_logical_recovery_from_beast_candidate_requests() { + let initial = fetched_manifest("#EXTM3U\n#EXTINF:4.0,\nseg.ts\n"); + let initial_diagnostic = HlsManifestRefreshCompletionDiagnostic::from_fetched(&initial); + assert_eq!(initial_diagnostic.recovery_attempts, 0); + assert_eq!(initial_diagnostic.candidate_requests, 1); + assert_eq!(initial_diagnostic.selection, HlsManifestFetchSelection::Initial); + + let mut burst = initial; + burst.attempts = 1; + burst.candidate_requests = HlsManifestRecoveryBurstLevel::Beast.plan().total_candidates(); + burst.selection = HlsManifestFetchSelection::Burst; + let burst_diagnostic = HlsManifestRefreshCompletionDiagnostic::from_fetched(&burst); + assert_eq!(burst_diagnostic.recovery_attempts, 1); + assert_eq!(burst_diagnostic.candidate_requests, HlsManifestRecoveryBurstLevel::Beast.plan().total_candidates()); + assert_eq!(burst_diagnostic.selection, HlsManifestFetchSelection::Burst); +} + +#[tokio::test] +async fn established_recovery_burst_uses_one_fixed_provider_url_for_every_candidate() { + let phase = Arc::new(AtomicUsize::new(0)); + let phase_for_first = Arc::clone(&phase); + let first = spawn_test_origin(Arc::new(move |_path| { + let media_sequence = if phase_for_first.load(Ordering::SeqCst) == 0 { 100 } else { 103 }; + (200, Vec::new(), three_segment_manifest_body(media_sequence)) + })) + .await; + let second = spawn_test_origin(Arc::new(|_path| (500, Vec::new(), "unexpected".to_string()))).await; + let provider = Arc::new(ConfigProvider::from(&ConfigProviderDto { + name: "demo".into(), + urls: vec![first.base_url.as_str().into(), second.base_url.as_str().into()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + })); + let session = test_session(); + let entry = LiveHlsOriginEntry::parse_with_url_failover_provider( + "provider://demo/live/user/pass/12345.m3u8?token=fixed", + Some(Arc::clone(&provider)), + ) + .expect("provider entry url"); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = entry; + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + + let baseline = fetch_and_commit_manifest_with_policy(&mut request).await.expect("initial baseline commits"); + assert_eq!(baseline.fetched.provider_url_index, Some(0)); + let concrete_request_url = format!("{}/live/user/pass/12345.m3u8?token=fixed", first.base_url); + { + let session = session.read().await; + let binding = + session.origin_control.manifest_origin_binding.as_ref().expect("successful commit stores binding"); + assert_eq!(binding.request_url().as_str(), concrete_request_url); + assert_eq!(binding.provider_url_index(), Some(0)); + } + publish_ready_test_manifest(&session, 200).await; + assert!(session.read().await.established_manifest_recovery_binding().is_some()); + + phase.store(1, Ordering::SeqCst); + let provider_index_before_burst = provider.get_current_index(); + request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::RecoveryRequired; + let selected = + fetch_and_commit_manifest_with_policy(&mut request).await.expect("established fixed-binding burst commits"); + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let first_requests = first.requests.lock().await; + assert_eq!(first_requests.len(), 1_usize.saturating_add(plan.total_candidates())); + assert!(first_requests.iter().all(|path| path == "/live/user/pass/12345.m3u8?token=fixed")); + assert!(second.requests.lock().await.is_empty()); + assert_eq!(provider.get_current_index(), provider_index_before_burst); + assert_eq!(selected.fetched.provider_url_index, Some(0)); + assert_eq!(selected.fetched.candidate_requests, plan.total_candidates()); + assert_eq!(selected.fetched.selection, HlsManifestFetchSelection::Burst); + assert_eq!(session.read().await.origin_seq_highwater, Some(105)); +} + +#[tokio::test] +async fn provisioning_handoff_without_established_origin_baseline_uses_one_initial_fetch() { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let server = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let session = test_session(); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", server.base_url)).expect("entry url"); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ProvisioningHandoff, + }; + assert_eq!(manifest_recovery_trigger(&request), HlsManifestAcceptanceTrigger::Critical); + + assert!(trigger_origin_refresh_sync(request).await); + + let manifest_requests = + server.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(manifest_requests, 1); + assert_ne!(manifest_requests, plan.total_candidates()); + let session = session.read().await; + assert_eq!(session.origin_seq_highwater, Some(0)); + assert!(session.origin_control.manifest_origin_binding.is_some()); + assert!(session.origin_control.acceptance_episode.is_none()); +} + +#[tokio::test] +async fn different_host_single_candidate_retries_do_not_prove_acceptance() { + let candidate_hits = Arc::new(AtomicUsize::new(0)); + let candidate_hits_for_handler = Arc::clone(&candidate_hits); + let candidate = spawn_test_origin(Arc::new(move |_path| { + candidate_hits_for_handler.fetch_add(1, Ordering::SeqCst); + ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n#EXTINF:4.0,\n102.ts\n" + .to_string(), + ) + })) + .await; + let session = test_session(); + { + let mut session = session.write().await; + session.origin_seq_highwater = Some(100); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + session.origin_control.pinned_host = Some("previous.example.com".to_string()); + } + let candidate_entry_url = + format!("{}/live/user/pass/12345.m3u8", candidate.base_url).replacen("127.0.0.1", "localhost", 1); + install_published_recovery_binding(&session, &candidate_entry_url, None).await; + let entry = LiveHlsOriginEntry::parse(&candidate_entry_url).expect("entry url"); + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 0 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + fresh_manifest_requirement_generation: None, + acceptance_directive: HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + post_refresh_runtime: None, + }; + + assert!(trigger_origin_refresh_sync(request).await); + + let candidate_requests = + candidate.requests.lock().await.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(); + assert_eq!(candidate_requests, 6); + let session = session.read().await; + assert_eq!(session.origin_seq_highwater, Some(100)); + assert_eq!(session.origin_epoch, 0); + let episode = session.origin_control.acceptance_episode.as_ref().expect("acceptance episode remains held"); + assert!(episode.full_burst_completed); + assert_eq!(episode.full_bursts_completed, 1); + assert_eq!(episode.trigger(), HlsManifestAcceptanceTrigger::Observe); + assert_eq!(episode.state, super::super::super::manifest_acceptance::HlsManifestAcceptanceState::Holding); + assert_eq!( + session.origin_control.path_condition, + super::super::super::origin_progress::HlsOriginPathCondition::AcceptanceConflict + ); +} + +#[tokio::test] +async fn manifest_recovery_burst_skips_rejected_candidate() { + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let origin = spawn_test_origin(Arc::new(move |_path| { + let hit = hits_for_handler.fetch_add(1, Ordering::SeqCst); + if hit == 0 { + return ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:50\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n50.bin\n".to_string(), + ); + } + ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n".to_string(), + ) + })) + .await; + let session = test_session(); + { + let mut session = session.write().await; + session.origin_seq_highwater = Some(100); + session.last_effective_manifest_host = Some(host_from_base_url(&origin.base_url)); + session.mark_authorized_media_access(super::super::current_time_millis()); + } + let entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("entry url"); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = entry; + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Friendly }; + + let target_url = Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("target url"); + let committed = retry_test_manifest_recovery_chain( + &request, + target_url, + HlsManifestRejectLogReason::PinnedHostRecoveryRejected, + ) + .await + .expect("burst should commit accepted candidate"); + + assert_eq!(committed.fetched.attempts, 1); + assert_eq!(hits.load(Ordering::SeqCst), 2); + assert_eq!(session.read().await.origin_seq_highwater, Some(101)); +} + +#[test] +fn manifest_recovery_candidate_score_prefers_same_host_next_sequence() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.origin_seq_highwater = Some(100); + session.last_effective_manifest_host = Some("origin.example.com".to_string()); + session.mark_authorized_media_access(super::super::current_time_millis()); + let request = test_origin_refresh_request(test_session()); + let fetch_context = manifest_fetch_context(&request); + let same_host_unchanged = + fetched_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n100.ts\n"); + let same_host_next = + fetched_manifest("#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n"); + let mut other_host_next = same_host_next.clone(); + other_host_next.redirect_host = Some("other.example.com".to_string()); + + assert_eq!( + score_manifest_recovery_candidate(&session, &same_host_unchanged, &fetch_context).expect("score").quality.score, + HlsManifestOriginQualityScore::SameHostUnchanged + ); + assert_eq!( + score_manifest_recovery_candidate(&session, &same_host_next, &fetch_context).expect("score").quality.score, + HlsManifestOriginQualityScore::SameHostNextSequence + ); + let other_host_score = + score_manifest_recovery_candidate(&session, &other_host_next, &fetch_context).expect("score").quality; + assert_eq!(other_host_score.score, HlsManifestOriginQualityScore::OtherHostCandidate); + assert!(other_host_score.requires_handoff_discontinuity); +} + +#[tokio::test] +async fn manifest_recovery_burst_commits_best_same_host_candidate() { + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_handler = Arc::clone(&hits); + let origin = spawn_test_origin(Arc::new(move |_path| { + let hit = hits_for_handler.fetch_add(1, Ordering::SeqCst); + if hit == 0 { + return ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:100\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n100.ts\n".to_string(), + ); + } + ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:101\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n101.ts\n".to_string(), + ) + })) + .await; + let session = test_session(); + { + let mut session = session.write().await; + session.origin_seq_highwater = Some(100); + session.last_effective_manifest_host = Some(host_from_base_url(&origin.base_url)); + session.mark_authorized_media_access(super::super::current_time_millis()); + } + let entry = + LiveHlsOriginEntry::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("entry url"); + let mut request = test_origin_refresh_request(Arc::clone(&session)); + request.origin_entry = entry; + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Friendly }; + + let target_url = Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("target url"); + let committed = retry_test_manifest_recovery_chain( + &request, + target_url, + HlsManifestRejectLogReason::PinnedHostRecoveryRejected, + ) + .await + .expect("burst should commit best same-host candidate"); + + assert_eq!(committed.fetched.attempts, 1); + assert_eq!(hits.load(Ordering::SeqCst), 2); + assert_eq!(session.read().await.origin_seq_highwater, Some(101)); +} + +#[tokio::test] +async fn host_switch_conflict_runs_one_full_plan_then_only_bounded_follow_ups() { + let candidate_hits = Arc::new(AtomicUsize::new(0)); + let candidate_hits_for_handler = Arc::clone(&candidate_hits); + let candidate = spawn_test_origin(Arc::new(move |_path| { + candidate_hits_for_handler.fetch_add(1, Ordering::SeqCst); + ( + 200, + Vec::new(), + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\n900.ts\n".to_string(), + ) + })) + .await; + let session = test_session(); + { + let mut session = session.write().await; + session.origin_seq_highwater = Some(100); + session.last_effective_manifest_host = Some("previous.example.com".to_string()); + session.origin_control.pinned_host = Some("previous.example.com".to_string()); + } + let candidate_entry_url = + format!("{}/live/user/pass/12345.m3u8", candidate.base_url).replacen("127.0.0.1", "localhost", 1); + install_published_recovery_binding(&session, &candidate_entry_url, None).await; + let entry = LiveHlsOriginEntry::parse(&candidate_entry_url).expect("entry url"); + let request = OriginRefreshRequest { + app_config: test_app_config(), + session: Arc::clone(&session), + origin_entry: entry.clone(), + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: false, + segment_cache: Arc::new(HlsSegmentCache::new()), + hls_proxy: Arc::new(HlsProxyManager::new()), + segment_repair: test_segment_repair_manager(), + segment_worker_pool: Arc::new(HlsSegmentWorkerPool::default()), + map_worker_pool: Arc::new(HlsMapWorkerPool::default()), + origin_manifest_timeout_ms: 2_000, + manifest_recovery_burst: HlsManifestRecoveryBurstConfig::default(), + strip: StripConfig { mode: HlsStripMode::Segments, value: 3 }, + retry_policy: no_delay_policy(), + reverse_proxy_rewrite_secret: b"secret".to_vec(), + transient_resource_ttl_ms: 300_000, + manifest_commit_requirement: HlsManifestCommitRequirement::CommittedManifestAllowed, + fresh_manifest_requirement_generation: None, + acceptance_directive: HlsManifestAcceptanceDirective::none(), + access_lease_id: None, + disabled_headers: None, + now_ms: 100, + origin_io: None, + post_refresh_runtime: None, + }; + + assert!(trigger_origin_refresh_sync(request).await); + + assert_eq!(candidate_hits.load(Ordering::SeqCst), 6); + let session = session.read().await; + assert_eq!(session.origin_seq_highwater, Some(100)); + let episode = session.origin_control.acceptance_episode.as_ref().expect("held acceptance episode"); + assert_eq!(episode.full_bursts_completed, 1); +} + +pub(in crate::refresh::tests) async fn assert_fresh_revalidation_cross_host_switch_runs_complete_beast_plan( + reason: HlsFreshManifestRequiredReason, +) { + let plan = HlsManifestRecoveryBurstLevel::Beast.plan(); + let manifest = + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-TARGETDURATION:4\n#EXTINF:4.0,\nfirst.ts\n#EXTINF:4.0,\nsecond.ts\n" + .to_string(); + let origin = spawn_test_origin(Arc::new(move |path| { + if path_has_extension(&path, "m3u8") { + (200, Vec::new(), manifest.clone()) + } else if path.ends_with("/first.ts") { + (200, Vec::new(), String::from_utf8_lossy(SWITCH_SEGMENT_BODY).into_owned()) + } else { + (404, Vec::new(), String::new()) + } + })) + .await; + let cache_dir = tempfile::tempdir().expect("fresh revalidation cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(cache_dir.path())); + let session = test_session(); + prepare_cross_host_baseline(&session).await; + let recovery_request_url = format!("{}/live/user/pass/12345.m3u8", origin.base_url); + install_published_recovery_binding(&session, &recovery_request_url, None).await; + let baseline_epoch = session.read().await.origin_epoch; + let mut request = switch_test_request(Arc::clone(&session), cache, &origin.base_url); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { reason }; + // A weaker observation signal must not reduce the strict revalidation + // policy selected by the commit requirement. + request.acceptance_directive.trigger = HlsManifestAcceptanceTrigger::Observe; + + assert!(trigger_origin_refresh_sync(request).await); + + let (manifest_requests, staged_segment_requests) = { + let requests = origin.requests.lock().await; + ( + requests.iter().filter(|path| path.as_str() == "/live/user/pass/12345.m3u8").count(), + requests.iter().filter(|path| path.ends_with("/first.ts")).count(), + ) + }; + assert_eq!(manifest_requests, plan.total_candidates()); + assert_eq!(staged_segment_requests, 1); + + let effective_host = host_from_base_url(&origin.base_url); + let session = session.read().await; + assert_eq!(session.origin_epoch, baseline_epoch.saturating_add(1)); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some(effective_host.as_str())); + let switched_head = session + .segments + .values() + .find(|segment| segment.origin_key.origin_epoch == session.origin_epoch) + .expect("fresh revalidation switched timeline head"); + assert!(switched_head.discontinuity_before); + assert!(matches!(switched_head.status, SegmentCacheStatus::Ready { .. })); +} + +#[tokio::test] +async fn expired_revalidation_cross_host_switch_runs_complete_beast_acceptance() { + assert_fresh_revalidation_cross_host_switch_runs_complete_beast_plan( + HlsFreshManifestRequiredReason::ExpiredRevalidation, + ) + .await; +} + +#[tokio::test] +async fn hard_failure_revalidation_cross_host_switch_runs_complete_beast_acceptance() { + assert_fresh_revalidation_cross_host_switch_runs_complete_beast_plan( + HlsFreshManifestRequiredReason::PreviousHardManifestFailure, + ) + .await; +} + +#[test] +fn hls_recovery_timing_candidate_preview_uses_actual_map_medium_not_playlist_head() { + let session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:40\n#EXT-X-TARGETDURATION:4\n\ + #EXTINF:4.0,\nhead40.ts\n\ + #EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:4.0,\nrecovery41.m4s\n"; + let (manifest, key_resources, preview) = candidate_handoff_preview(&session, body, 100); + assert!(manifest.segments.first().is_some_and(|segment| segment.map_ref.is_none())); + let actual_recovery_segment = preview.segments.get(1).expect("actual recovery medium after clear head"); + let required_map = + actual_recovery_segment.map_ref.and_then(|map_id| preview.maps.iter().find(|map| map.proxy_map_id == map_id)); + + let workload = super::super::switch_staging::handoff_preview_recovery_workload( + &session, + actual_recovery_segment, + required_map, + &key_resources, + 100, + ); + + assert_eq!(workload.segment, HlsRecoverySegmentWorkload::ClearSegmentFetch); + assert_eq!(workload.map, HlsRecoveryMapWorkload::Fetch); +} + +#[test] +fn hls_recovery_timing_candidate_preview_detects_generation_local_ready_aes128_key() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:40\n#EXT-X-TARGETDURATION:4\n\ + #EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nrecovery40.ts\n"; + let (_, key_resources, preview) = candidate_handoff_preview(&session, body, 100); + let key_resource = key_resources.first().cloned().expect("candidate key resource"); + session.transient.upsert_resources([key_resource.clone()]); + let proxy_session_id = session.proxy_session_id.clone(); + let fetch = session.transient.begin_object_fetch(&proxy_session_id, &key_resource, "bin", 100, 60_000); + let TransientObjectFetchDecision::Fetch(token) = fetch else { + panic!("new candidate key starts one cache fill"); + }; + assert!(session.transient.mark_object_ready_if_current( + &token, + "application/octet-stream".to_string(), + 16, + 100, + 60_100, + )); + let actual_recovery_segment = preview.segments.first().expect("AES recovery medium"); + + let workload = super::super::switch_staging::handoff_preview_recovery_workload( + &session, + actual_recovery_segment, + None, + &key_resources, + 101, + ); + + assert_eq!(workload.segment, HlsRecoverySegmentWorkload::Aes128SegmentFetchWithReadyKey); + assert_eq!( + super::super::switch_staging::staged_switch_media_compatibility( + &session, + actual_recovery_segment, + &key_resources, + 101 + ), + super::super::switch_staging::HlsStagedSwitchMediaCompatibility::Compatible + ); + assert_eq!( + super::super::switch_staging::staged_switch_media_compatibility( + &session, + actual_recovery_segment, + &key_resources, + 60_101 + ), + super::super::switch_staging::HlsStagedSwitchMediaCompatibility::RequiresUnstagedEncryptionKey, + "the final commit revalidation must reject an expired READY key" + ); +} + +#[test] +fn hls_recovery_timing_candidate_preview_requires_aes128_key_fetch_without_ready_evidence() { + let session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let body = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:40\n#EXT-X-TARGETDURATION:4\n\ + #EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\"\n#EXTINF:4.0,\nrecovery40.ts\n"; + let (_, key_resources, preview) = candidate_handoff_preview(&session, body, 100); + let actual_recovery_segment = preview.segments.first().expect("AES recovery medium"); + + let workload = super::super::switch_staging::handoff_preview_recovery_workload( + &session, + actual_recovery_segment, + None, + &key_resources, + 101, + ); + + assert_eq!(workload.segment, HlsRecoverySegmentWorkload::Aes128SegmentFetchWithKeyFetch); + assert_eq!( + super::super::switch_staging::staged_switch_media_compatibility( + &session, + actual_recovery_segment, + &key_resources, + 101 + ), + super::super::switch_staging::HlsStagedSwitchMediaCompatibility::RequiresUnstagedEncryptionKey + ); +} + +#[test] +fn switch_staging_rejects_map_fetch_outside_fixture_envelope_before_network() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let burst_plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); + session.origin_control.begin_acceptance_episode( + 100, + burst_plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &test_acceptance_episode_timing(100, burst_plan), + ); + let episode = session.origin_control.acceptance_episode.as_mut().expect("acceptance episode"); + episode.record_full_burst(); + episode.state = super::super::super::manifest_acceptance::HlsManifestAcceptanceState::StagingSwitchSegment; + let identity = super::super::super::manifest_acceptance::HlsManifestRecoveryCandidateIdentity::from_candidate( + 0, + Some("candidate.example"), + &switch_manifest_body(true), + ); + assert_eq!( + episode.select_candidate(episode.generation, identity), + super::super::super::manifest_acceptance::HlsRecoveryWorkloadBindingUpdate::Applied + ); + let map_fetch = HlsRecoveryWorkload::from_recovery_medium(HlsRecoveryMediumReadiness { + segment: HlsRecoveryObjectReadiness::Fetch, + map: Some(HlsRecoveryObjectReadiness::Fetch), + encryption: HlsRecoveryEncryptionReadiness::Clear, + }); + + assert_eq!( + episode.bind_selected_candidate(episode.generation, identity, map_fetch), + super::super::super::manifest_acceptance::HlsRecoveryWorkloadBindingUpdate::OutsideEnvelope + ); +} + +#[tokio::test] +async fn staged_cross_host_aes_candidate_without_ready_key_evidence_never_commits() { + let baseline = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-TARGETDURATION:4\n\ + #EXTINF:4.0,\nold1.ts\n#EXTINF:4.0,\nold2.ts\n#EXTINF:4.0,\nold3.ts\n"; + let candidate = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:40\n#EXT-X-TARGETDURATION:4\n\ + #EXT-X-KEY:METHOD=AES-128,URI=\"key.php\"\n\ + #EXTINF:4.0,\nnew40.ts\n#EXTINF:4.0,\nnew41.ts\n#EXTINF:4.0,\nnew42.ts\n"; + + assert_incompatible_switch_is_rejected_before_timeline_commit( + baseline, + candidate, + HlsManifestRejectLogReason::SwitchEncryptionKeyNotReady, + ) + .await; +} + +#[tokio::test] +async fn staged_cross_host_aes_candidate_with_generation_local_ready_key_commits() { + let temp_dir = tempfile::tempdir().expect("switch cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let origin = spawn_controlled_switch_origin().await; + let session = test_session(); + prepare_cross_host_baseline(&session).await; + let now_ms = current_time_millis(); + let candidate = "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:500\n#EXT-X-TARGETDURATION:4\n\ + #EXT-X-KEY:METHOD=AES-128,URI=\"key.php\"\n\ + #EXTINF:4.0,\nfirst.ts\n#EXTINF:4.0,\nsecond.ts\n"; + let mut fetched = switch_fetched_manifest(&origin.base_url, false); + fetched.body = candidate.to_string(); + let key_resource = { + let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(mut manifest) = + tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest(candidate, &fetched.final_manifest_url) + else { + panic!("AES switch manifest parses as a normal timeline"); + }; + super::super::commit::materialize_normal_key_resources(&mut manifest, b"secret", now_ms, 300_000) + .into_iter() + .next() + .expect("AES switch candidate has one key resource") + }; + { + let mut session = session.write().await; + session.transient.upsert_resources([key_resource.clone()]); + let proxy_session_id = session.proxy_session_id.clone(); + let extension = key_resource.file_ext_hint.as_deref().expect("key extension"); + let fetch = session.transient.begin_object_fetch(&proxy_session_id, &key_resource, extension, now_ms, 300_000); + let TransientObjectFetchDecision::Fetch(token) = fetch else { + panic!("candidate key starts one cache fill"); + }; + assert!(session.transient.mark_object_ready_if_current( + &token, + "application/octet-stream".to_string(), + 16, + now_ms, + now_ms.saturating_add(300_000), + )); + } + mark_full_burst_ready_for_switch_staging(&session, &fetched).await; + let mut request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); + request.now_ms = now_ms; + let mut commit_task = tokio::spawn(async move { + super::super::commit_manifest_recovery_candidate( + &request, + fetched, + HlsManifestCommitAcceptanceMode::AllowHeldHostSwitchCandidate, + ) + .await + .map(|_| ()) + }); + + await_controlled_switch_segment_prefix(&origin, &mut commit_task, "encrypted cross-host switch commit").await; + origin.release_segment_body.notify_one(); + commit_task.await.expect("encrypted switch commit task").expect("encrypted switch commit succeeds"); + + let session = session.read().await; + let switched = session + .segments + .values() + .find(|segment| segment.origin_key.origin_epoch == session.origin_epoch) + .expect("switched segment committed"); + assert!(switched.encryption.is_some()); + assert!(session + .transient + .ready_key_object_valid_until_ms( + &session.proxy_session_id, + &key_resource.id, + key_resource.file_ext_hint.as_deref().expect("key extension"), + current_time_millis(), + ) + .is_some()); +} + +#[tokio::test] +async fn cross_host_switch_commits_only_after_complete_map_and_segment_staging() { + let temp_dir = tempfile::tempdir().expect("switch cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let origin = spawn_controlled_switch_origin().await; + let session = test_session(); + prepare_cross_host_baseline(&session).await; + let fetched = switch_fetched_manifest(&origin.base_url, true); + mark_full_burst_ready_for_switch_staging(&session, &fetched).await; + let effective_host = host_from_base_url(&origin.base_url); + let effective_host_id = crate::timeline::effective_origin_host_id(&effective_host); + let (baseline_epoch, baseline_proxy_next, staged_segment_key, staged_map_key) = { + let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = + tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( + &fetched.body, + &fetched.final_manifest_url, + ) + else { + panic!("switch manifest parses as normal timeline"); + }; + let session = session.read().await; + let preview = session.preview_origin_handoff_manifest(&manifest, effective_host_id, 0).expect("switch preview"); + ( + session.origin_epoch, + session.proxy_next_seq, + preview.segments.first().expect("first staged segment").cache_key.clone(), + preview.maps.first().expect("required staged map").cache_key.clone(), + ) + }; + let request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); + let cleanup_manager = Arc::clone(&request.hls_proxy); + let mut commit_task = tokio::spawn(async move { + super::super::commit_manifest_recovery_candidate( + &request, + fetched, + HlsManifestCommitAcceptanceMode::AllowHeldHostSwitchCandidate, + ) + .await + .map(|_| ()) + }); + + await_controlled_switch_segment_prefix(&origin, &mut commit_task, "cross-host switch commit").await; + + assert!(!commit_task.is_finished()); + { + let session = session.read().await; + assert_eq!(session.origin_epoch, baseline_epoch); + assert_eq!(session.proxy_next_seq, baseline_proxy_next); + assert!(session.segments.values().all(|segment| segment.origin_key.origin_epoch == baseline_epoch)); + } + assert_eq!(cache.metadata(&staged_segment_key).await.expect("segment metadata"), None); + assert_eq!(cache.metadata(&staged_map_key).await.expect("map metadata"), None); + assert_eq!( + *origin.requests.lock().await, + vec!["/live/final/init.mp4".to_string(), "/live/final/first.ts".to_string()] + ); + + origin.release_segment_body.notify_one(); + commit_task.await.expect("switch commit task").expect("switch commit succeeds"); + assert_eq!(cleanup_manager.cache_deletion_queue_usage(), (0, 0)); + + let (first_segment_key, first_map_key) = { + let session = session.read().await; + assert_eq!(session.origin_epoch, baseline_epoch.saturating_add(1)); + assert_eq!(session.origin_epoch_effective_host_id, Some(effective_host_id)); + let switched = session + .segments + .values() + .filter(|segment| segment.origin_key.origin_epoch == session.origin_epoch) + .collect::>(); + assert_eq!(switched.len(), 2); + assert_eq!(switched[0].origin_key.host_local_sequence, 500); + assert!(switched[0].discontinuity_before); + assert!(matches!( + switched[0].status, + SegmentCacheStatus::Ready { content_length, .. } + if content_length == u64::try_from(SWITCH_SEGMENT_BODY.len()).unwrap_or(u64::MAX) + )); + assert!(!matches!(switched[1].status, SegmentCacheStatus::Ready { .. })); + let map_id = switched[0].map_ref.expect("first switched segment requires map"); + assert_eq!(switched[1].map_ref, Some(map_id)); + let map = session.maps.get(&map_id).expect("staged map committed to timeline"); + assert!(matches!( + map.status, + MapCacheStatus::Ready { content_length, .. } + if content_length == u64::try_from(SWITCH_MAP_BODY.len()).unwrap_or(u64::MAX) + )); + (switched[0].cache_key.clone(), map.cache_key.clone()) + }; + let mut segment_file = cache.open_range(&first_segment_key, 0).await.expect("open staged segment"); + let mut segment_bytes = Vec::new(); + segment_file.read_to_end(&mut segment_bytes).await.expect("read staged segment"); + assert_eq!(segment_bytes, SWITCH_SEGMENT_BODY); + let mut map_file = cache.open_range(&first_map_key, 0).await.expect("open staged map"); + let mut map_bytes = Vec::new(); + map_file.read_to_end(&mut map_bytes).await.expect("read staged map"); + assert_eq!(map_bytes, SWITCH_MAP_BODY); +} + +#[tokio::test] +async fn caller_cancellation_after_owned_switch_staging_queues_and_collects_rollback() { + let temp_dir = tempfile::tempdir().expect("switch cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let origin = spawn_controlled_switch_origin().await; + let session = test_session(); + prepare_cross_host_baseline(&session).await; + let fetched = switch_fetched_manifest(&origin.base_url, true); + mark_full_burst_ready_for_switch_staging(&session, &fetched).await; + let effective_host_id = crate::timeline::effective_origin_host_id(&host_from_base_url(&origin.base_url)); + let (baseline_epoch, staged_segment_key, staged_map_key) = { + let tuliprox_parser::hls::origin_manifest::OriginManifestParseOutcome::Normal(manifest) = + tuliprox_parser::hls::origin_manifest::parse_origin_media_manifest( + &fetched.body, + &fetched.final_manifest_url, + ) + else { + panic!("switch manifest parses as normal timeline"); + }; + let session = session.read().await; + let preview = session.preview_origin_handoff_manifest(&manifest, effective_host_id, 0).expect("switch preview"); + ( + session.origin_epoch, + preview.segments.first().expect("first staged segment").cache_key.clone(), + preview.maps.first().expect("required staged map").cache_key.clone(), + ) + }; + let request = switch_test_request(Arc::clone(&session), Arc::clone(&cache), &origin.base_url); + let cleanup_manager = Arc::clone(&request.hls_proxy); + let (staging_complete_tx, staging_complete_rx) = tokio::sync::oneshot::channel(); + let (hold_tx, hold_rx) = tokio::sync::oneshot::channel::<()>(); + let mut staging_task = tokio::spawn(async move { + let staged = super::super::switch_staging::stage_alternative_manifest_switch(&request, &fetched) + .await + .expect("switch staging succeeds"); + staging_complete_tx.send(()).expect("staging completion receiver remains"); + let _ = hold_rx.await; + drop(staged); + }); + + await_controlled_switch_segment_prefix(&origin, &mut staging_task, "owned switch staging").await; + origin.release_segment_body.notify_one(); + staging_complete_rx.await.expect("owned cache commits complete"); + assert_eq!(cleanup_manager.cache_deletion_queue_usage(), (0, 2)); + assert!(cache.metadata(&staged_segment_key).await.expect("staged segment metadata").is_some()); + assert!(cache.metadata(&staged_map_key).await.expect("staged map metadata").is_some()); + + staging_task.abort(); + assert!(staging_task.await.expect_err("staging task is cancelled").is_cancelled()); + drop(hold_tx); + assert_eq!(cleanup_manager.cache_deletion_queue_usage(), (2, 0)); + assert_eq!(session.read().await.origin_epoch, baseline_epoch); + + let report = cleanup_manager.run_garbage_collection_once(200).await.expect("rollback GC succeeds"); + assert_eq!(report.cache_object_deletions_succeeded, 2); + assert_eq!(cleanup_manager.cache_deletion_queue_usage(), (0, 0)); + assert_eq!(cache.metadata(&staged_segment_key).await.expect("rolled-back segment metadata"), None); + assert_eq!(cache.metadata(&staged_map_key).await.expect("rolled-back map metadata"), None); +} + +#[tokio::test] +async fn stale_acceptance_generation_rejects_and_removes_completed_switch_staging() { + assert_stale_switch_generation_rejects_commit(StaleSwitchGeneration::Acceptance).await; +} + +#[tokio::test] +async fn stale_progress_generation_rejects_and_removes_completed_switch_staging() { + assert_stale_switch_generation_rejects_commit(StaleSwitchGeneration::Progress).await; +} + +#[tokio::test] +async fn switch_segment_staging_failure_holds_episode_without_timeline_mutation() { + let manifest = switch_manifest_body(false); + let origin = spawn_test_origin(Arc::new(move |path| { + if path_has_extension(&path, "m3u8") { + (200, Vec::new(), manifest.clone()) + } else { + (404, Vec::new(), String::new()) + } + })) + .await; + let temp_dir = tempfile::tempdir().expect("switch cache tempdir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let session = test_session(); + prepare_cross_host_baseline(&session).await; + let (baseline_epoch, baseline_proxy_next, baseline_sequences) = { + let session = session.read().await; + (session.origin_epoch, session.proxy_next_seq, session.segments.keys().copied().collect::>()) + }; + let request = switch_test_request(Arc::clone(&session), cache, &origin.base_url); + let target_url = Url::parse(&format!("{}/live/user/pass/12345.m3u8", origin.base_url)).expect("target url"); + + let result = retry_test_manifest_recovery_chain( + &request, + target_url, + HlsManifestRejectLogReason::PinnedHostRecoveryRejected, + ) + .await; + + assert!(matches!(result, Err(OriginManifestFetchError::RetryExhausted))); + let session = session.read().await; + assert_eq!(session.origin_epoch, baseline_epoch); + assert_eq!(session.proxy_next_seq, baseline_proxy_next); + assert_eq!(session.segments.keys().copied().collect::>(), baseline_sequences); + let episode = session.origin_control.acceptance_episode.as_ref().expect("failed switch episode retained"); + assert!(episode.full_burst_completed); + assert_eq!(episode.full_bursts_completed, 1); + assert_eq!(episode.state, super::super::super::manifest_acceptance::HlsManifestAcceptanceState::Holding); + drop(session); + let requests = origin.requests.lock().await; + assert_eq!(requests.iter().filter(|path| path_has_extension(path, "ts")).count(), 1); +} + +#[tokio::test] +async fn provider_preflight_failure_real_refresh_path_has_zero_candidates() { + let origin = spawn_test_origin(Arc::new(|_path| (200, Vec::new(), manifest_body()))).await; + let hls_ctx = crate::HlsCtx::for_test(Config::default()); + let ctx = &hls_ctx; + let session = test_session(); + let proxy_session_id = session.read().await.proxy_session_id.clone(); + session.write().await.origin_account_binding = Some(HlsOriginAccountBinding::new( + Arc::from("missing-input"), + Arc::from("missing-account"), + &proxy_session_id, + 0, + )); + let mut request = bind_refresh_request_to_app_state(test_origin_refresh_request(Arc::clone(&session)), ctx); + request.manifest_recovery_burst = HlsManifestRecoveryBurstConfig { level: HlsManifestRecoveryBurstLevel::Beast }; + request.origin_entry = + LiveHlsOriginEntry::parse(&format!("{}/live/index.m3u8", origin.base_url)).expect("unused origin URL"); + request.origin_io = Some(HlsOriginIoContext { + ctx: ctx.clone(), + client_addr: "127.0.0.1:12345".parse().expect("test client address"), + allow_grace: false, + priority: 0, + connection_kind: ConnectionKind::Normal, + reservation_ttl_secs: 60, + preacquired_provider_handle: None, + started_generation: None, + }); + let metrics = Arc::clone(request.segment_worker_pool.metrics()); + let error = super::super::provider_preflight_manifest_error(HlsBoundAccountAcquireErrorKind::Missing); + + assert!(matches!(&error, OriginManifestFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Missing))); + assert!(trigger_origin_refresh_sync(request).await); + assert!(origin.requests.lock().await.is_empty()); + + let session = session.read().await; + assert!(session.origin_control.acceptance_episode.is_none()); + assert_eq!( + session.origin_control.progress_phase, + super::super::super::origin_progress::HlsOriginProgressPhase::Cold + ); + assert_eq!( + session.origin_control.path_condition, + super::super::super::origin_progress::HlsOriginPathCondition::HardFetchFailure + ); + assert!(session.origin_control.last_origin_response_at_ms.is_none()); + assert!(session.origin_control.manifest_origin_binding.is_none()); + assert!(session.last_rendered_manifest.is_none()); + assert!(session.published_live_origin_baseline.is_none()); + assert!(session.established_manifest_recovery_binding().is_none()); + drop(session); + let metrics = metrics.snapshot(); + assert_eq!(metrics.refresh_started, 1); + assert_eq!(metrics.refresh_completed, 0); + assert_eq!(metrics.refresh_retried, 0); + assert_eq!(metrics.refresh_skipped, 0); + assert_eq!(metrics.refresh_failed, 1); +} diff --git a/backend/hls/src/refresh/tests/timing.rs b/backend/hls/src/refresh/tests/timing.rs new file mode 100644 index 000000000..9b0a2aabf --- /dev/null +++ b/backend/hls/src/refresh/tests/timing.rs @@ -0,0 +1,54 @@ +use super::{ + compute_origin_refresh_interval_ms, critical_handoff_app_config, format_millis_as_seconds, + format_optional_millis_as_seconds, origin_highwater_policy_limit, test_origin_refresh_request, test_session, +}; +use crate::HlsPreparedTerminalBundleState; + +#[test] +fn manifest_highwater_policy_limit_uses_target_duration_fallback() { + assert_eq!(origin_highwater_policy_limit(60, None), Some(4)); + assert_eq!(origin_highwater_policy_limit(61, None), Some(5)); + assert_eq!(origin_highwater_policy_limit(60, Some(12)), Some(5)); +} + +#[test] +fn manifest_timing_log_values_are_seconds_or_none() { + assert_eq!(format_optional_millis_as_seconds(Some(4_500)), "4.500"); + assert_eq!(format_optional_millis_as_seconds(None), "none"); + assert_eq!(format_millis_as_seconds(2_000), "2.000"); +} + +#[test] +fn refresh_interval_uses_half_reference_duration_without_upper_clamp() { + assert_eq!(compute_origin_refresh_interval_ms(Some(8_000), None), 4_000); + assert_eq!(compute_origin_refresh_interval_ms(Some(500), None), 1_000); + assert_eq!(compute_origin_refresh_interval_ms(Some(20_000), None), 10_000); + assert_eq!(compute_origin_refresh_interval_ms(None, None), 2_000); +} + +#[tokio::test] +async fn hls_prepared_terminal_bundle_early_refresh_hook_starts_singleflight_for_known_target_duration() { + let mut request = test_origin_refresh_request(test_session()); + request.app_config = critical_handoff_app_config(); + let terminal_response = request.app_config.custom_stream_response.load_full(); + let asset = terminal_response + .as_ref() + .and_then(|response| response.channel_unavailable.as_ref()) + .and_then(|buffer| super::super::snapshot_terminal_media_asset(buffer).ok()) + .expect("terminal refresh asset"); + let target_duration_ms = asset.duration_ms().saturating_add(1_000); + let key = super::super::super::prepared_terminal_bundle::prepared_terminal_bundle_key( + &asset, + target_duration_ms, + super::super::HLS_TERMINAL_TAIL_SEGMENT_COUNT, + ); + + super::super::start_refresh_terminal_bundle_preparation(&request, target_duration_ms); + let state = + request.hls_proxy.wait_for_prepared_terminal_bundle(key).await.expect("early terminal preparation completes"); + + assert!(matches!( + state, + HlsPreparedTerminalBundleState::Ready { bundle } if bundle.key == key + )); +} diff --git a/backend/hls/src/refresh/tests/transient.rs b/backend/hls/src/refresh/tests/transient.rs new file mode 100644 index 000000000..12e9b1c28 --- /dev/null +++ b/backend/hls/src/refresh/tests/transient.rs @@ -0,0 +1,439 @@ +use super::{ + build_manifest_refresh_timing, classify_manifest_fetch_failure, commit_fetched_manifest, fetched_manifest, + key_resource_extension, manifest_fetch_context, record_committed_manifest_success, + refresh_session_with_origin_body, repeated_transient_manifest, test_acceptance_episode_timing, + test_manifest_origin_binding, test_origin_refresh_request, test_session, transient_reason_log_fields, + HlsManifestAcceptanceTrigger, HlsManifestCommitError, HlsManifestCommitProgressEvidence, + HlsManifestCommitRequirement, HlsManifestFetchFailureKind, HlsManifestFetchFailureSignal, HlsManifestProgress, + HlsManifestRejectLogReason, OriginManifestFetchError, +}; +use crate::{ + HlsAccessLeaseStore, HlsFreshManifestRequiredReason, HlsSegmentWorkerPool, HlsSession, HlsSessionKey, + HlsSessionMode, TransientPassthroughReason, TransientResourceFile, TransientResourceId, +}; +use shared::model::HlsManifestRecoveryBurstLevel; +use std::{fmt::Write, sync::Arc, time::Duration}; +use tokio::sync::RwLock; +use url::Url; + +#[test] +fn normal_key_extension_is_always_compatible_with_the_transient_route() { + let extensions = [ + key_resource_extension("https://origin.example/live/key.php?token=secret"), + key_resource_extension("https://origin.example/live/opaque"), + key_resource_extension("https://origin.example/live/key.BIN"), + key_resource_extension("https://origin.example/live/key.key"), + ]; + + assert_eq!(extensions, ["key", "key", "bin", "key"]); + for extension in extensions { + let file_name = format!("{}.{}", "abcdefghijklmnop", extension); + let parsed = TransientResourceFile::parse(&file_name).expect("normalized key route parses"); + assert_eq!(parsed.resource_id, TransientResourceId("abcdefghijklmnop".to_string())); + assert_eq!(parsed.extension, extension); + } +} + +#[test] +fn hls_cutover_policy_transient_success_preserves_recovery_episode_and_empty_refresh_evidence() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + let burst_plan = HlsManifestRecoveryBurstLevel::Friendly.plan(); + session.origin_control.begin_acceptance_episode( + 1_000, + burst_plan, + HlsManifestAcceptanceTrigger::RecoveryRequired, + &test_acceptance_episode_timing(1_000, burst_plan), + ); + session.origin_control.acceptance_episode.as_mut().expect("acceptance episode").complete(); + session.origin_refresh.consecutive_empty_refreshes = 2; + session.origin_refresh.mark_started(10_000); + + let (bookkeeping_timing, applied_interval_ms) = record_committed_manifest_success( + &mut session, + HlsManifestCommitProgressEvidence::Transient(build_manifest_refresh_timing( + None, + Some(12_000), + HlsManifestProgress::Advanced, + )), + 10_000, + 10_100, + ); + + assert_eq!(bookkeeping_timing.progress, HlsManifestProgress::Unchanged); + assert_eq!(applied_interval_ms, 1_000); + assert_eq!(session.origin_refresh.consecutive_empty_refreshes, 3); + assert_eq!(session.origin_refresh.next_fetch_allowed_at_ms, 11_000); + assert!(session.origin_control.acceptance_episode.is_some()); + assert_eq!(session.origin_control.recovery_samples.p95_ms(), None); + assert_eq!(session.origin_control.last_origin_response_at_ms, Some(10_100)); +} + +#[test] +fn transient_reason_log_fields_include_unsupported_tag() { + let reason = TransientPassthroughReason::UnsupportedTag { tag: "#EXT-X-PART".to_string() }; + + assert_eq!(transient_reason_log_fields(&reason), "reason=unsupported_tag tag=#EXT-X-PART"); +} + +#[tokio::test] +async fn ext_x_key_manifest_commits_transient_rewrite() { + let session = refresh_session_with_origin_body( + "#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg.ts\n", + ) + .await; + let session = session.read().await; + let body = session.transient.last_manifest_body.as_ref().expect("transient body"); + + assert!(matches!( + session.mode, + HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey } + )); + assert!(body.contains("/hls/shared/live/")); + assert!(body.contains("/r/")); + assert!(!body.contains("/hls/user/")); + assert_eq!(session.transient.resources.len(), 2); + assert_eq!(session.target_duration, Some(12)); + assert_eq!(session.account_overlap_timing().target_duration_ms, 12_000); +} + +#[tokio::test] +async fn finalized_event_manifest_commits_full_transient_lifecycle() { + let mut origin_body = + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:1\n".to_string(); + for sequence in 1..=8 { + let _ = write!(origin_body, "#EXTINF:4.0,\n{sequence}.ts\n"); + } + origin_body.push_str("#EXT-X-ENDLIST\n"); + let origin_body: &'static str = Box::leak(origin_body.into_boxed_str()); + let session = refresh_session_with_origin_body(origin_body).await; + let session = session.read().await; + let body = session.transient.last_manifest_body.as_ref().expect("finalized transient body"); + + assert!(matches!( + session.mode, + HlsSessionMode::TransientPassthrough { + reason: TransientPassthroughReason::UnsupportedTag { ref tag } + } if tag == "#EXT-X-PLAYLIST-TYPE" + )); + let stored_body_lifecycle = tuliprox_parser::hls::origin_manifest::parse_manifest_semantics(body).lifecycle(); + assert_eq!(session.transient.last_manifest_finalized(), stored_body_lifecycle.is_finalized()); + assert_eq!( + session.transient.last_manifest_window_policy(), + tuliprox_parser::hls::origin_manifest::HlsManifestWindowPolicy::PreserveFullManifest + ); + assert_eq!(session.transient.current_manifest_resource_ids().len(), 8); + assert_eq!(session.transient.last_manifest_playlist_duration_ms, Some(32_000)); + assert_eq!(session.transient.last_manifest_valid_until_ms(), None); + assert!(body.contains("#EXT-X-PLAYLIST-TYPE:EVENT")); + assert!(body.contains("#EXT-X-ENDLIST")); + assert_eq!(body.matches("/r/").count(), 8); +} + +#[test] +fn duplicate_media_units_are_rejected_locally_before_transient_commit() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + let request = test_origin_refresh_request(test_session()); + let body = repeated_transient_manifest(crate::manifest_limits::MAX_HLS_LEASE_SNAPSHOT_SEGMENTS + 1); + + let result = commit_fetched_manifest(&mut session, &fetched_manifest(&body), &request, 100); + + assert!(matches!( + result, + Err(HlsManifestCommitError::LocalRepresentationLimit(violation)) + if violation.kind == crate::manifest_limits::HlsManifestLimitKind::LeaseSnapshotSegments + )); + assert!(session.transient.last_manifest_body.is_none()); + let signal = classify_manifest_fetch_failure(&OriginManifestFetchError::LocalRepresentationLimit( + crate::manifest_limits::HlsManifestLimitViolation::new( + crate::manifest_limits::HlsManifestLimitKind::LeaseSnapshotSegments, + crate::manifest_limits::MAX_HLS_LEASE_SNAPSHOT_SEGMENTS + 1, + crate::manifest_limits::MAX_HLS_LEASE_SNAPSHOT_SEGMENTS, + ), + )); + assert_eq!(signal, HlsManifestFetchFailureSignal::discarded(HlsManifestFetchFailureKind::LocalRepresentationLimit)); +} + +#[test] +fn transient_commit_generation_exhaustion_is_not_reported_as_manifest_limit() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.set_manifest_commit_generation_for_test(u64::MAX); + let request = test_origin_refresh_request(test_session()); + let manifest_limit_rejections_before = request.segment_worker_pool.metrics().snapshot().manifest_limit_rejections; + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:0\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4,\nseg.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(matches!(result, Err(HlsManifestCommitError::CommitGenerationExhausted))); + assert_eq!( + request.segment_worker_pool.metrics().snapshot().manifest_limit_rejections, + manifest_limit_rejections_before + ); + assert!(session.transient.last_manifest_body.is_none()); +} + +#[tokio::test] +async fn initial_transient_manifest_without_snapshot_template_is_rejected_locally() { + let session = test_session(); + session.write().await.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + let request = test_origin_refresh_request(Arc::clone(&session)); + let fetch_context = manifest_fetch_context(&request); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:1\n\ + #EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4,\nsegment.ts\n#EXT-X-ENDLIST\n", + ); + let recovery_binding = + test_manifest_origin_binding(Url::parse("http://127.0.0.1:9/recovery.m3u8").expect("test recovery URL")); + + let result = + super::super::commit_initial_fetched_manifest(&request, &fetch_context, fetched, Some(recovery_binding), false) + .await; + + assert!(matches!(result, Err(OriginManifestFetchError::MalformedTransientRepresentation))); + assert_eq!( + classify_manifest_fetch_failure(&OriginManifestFetchError::MalformedTransientRepresentation), + HlsManifestFetchFailureSignal::discarded(HlsManifestFetchFailureKind::MalformedTransientRepresentation) + ); + let session = session.read().await; + assert!(session.transient.last_manifest_body.is_none()); + assert!(session.transient.last_manifest_template().is_none()); + assert!(session.transient.last_manifest_commit_identity().is_none()); + assert_eq!(session.transient.manifest_generation(), 0); + assert!(session.transient.resources.is_empty()); + assert!(crate::manifest_commit::hls_committed_manifest_body_for_request( + &session, + crate::manifest_commit::HlsCachedManifestOptions::initial(Duration::ZERO), + 0, + 100, + ) + .is_none()); +} + +#[test] +fn transient_manifest_without_snapshot_template_preserves_deliverable_commit() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + let request = test_origin_refresh_request(test_session()); + let baseline = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:1\n\ + #EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4,\nsegment.ts\n#EXT-X-ENDLIST\n", + ); + assert!(commit_fetched_manifest(&mut session, &baseline, &request, 100).is_ok()); + session.mark_pending_handoff_discontinuity(7); + let baseline_body = session.transient.last_manifest_body.clone().expect("baseline manifest"); + let baseline_template = session.transient.last_manifest_template().expect("baseline template"); + let baseline_generation = session.transient.manifest_generation(); + let baseline_finalized_generation = + session.transient.current_finalized_manifest_generation().expect("baseline finalized generation"); + let baseline_finalized_generation_count = session.transient.finalized_manifest_generation_count(); + let baseline_commit_identity = session.transient.last_manifest_commit_identity().expect("baseline identity"); + let baseline_resources = session.transient.resources.keys().cloned().collect::>(); + let baseline_manifest_resource_ids = session.transient.current_manifest_resource_ids().clone(); + let baseline_highwater = session.origin_seq_highwater; + let candidate = fetched_manifest( + "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:2\n\ + #EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"rotated-key.bin\"\n#EXTINF:4,\nreplacement.ts\n#EXT-X-ENDLIST\n", + ); + + let result = commit_fetched_manifest(&mut session, &candidate, &request, 102); + + assert!(matches!(result, Err(HlsManifestCommitError::MalformedTransientRepresentation))); + assert!(Arc::ptr_eq( + session.transient.last_manifest_body.as_ref().expect("baseline remains current"), + &baseline_body + )); + assert!(Arc::ptr_eq( + &session.transient.last_manifest_template().expect("baseline template remains current"), + &baseline_template + )); + assert_eq!(session.transient.manifest_generation(), baseline_generation); + assert_eq!(session.transient.current_finalized_manifest_generation(), Some(baseline_finalized_generation)); + assert_eq!(session.transient.finalized_manifest_generation_count(), baseline_finalized_generation_count); + assert_eq!(session.transient.last_manifest_commit_identity(), Some(baseline_commit_identity)); + assert_eq!( + session.transient.resources.keys().cloned().collect::>(), + baseline_resources + ); + assert_eq!(session.transient.current_manifest_resource_ids(), &baseline_manifest_resource_ids); + assert_eq!(session.pending_handoff_discontinuity_sequence, Some(7)); + assert_eq!(session.origin_seq_highwater, baseline_highwater); + assert!(session + .transient + .last_manifest_template() + .zip(session.transient.last_manifest_commit_identity()) + .is_some()); +} + +#[test] +fn transient_commit_rejects_same_host_backward_manifest_outside_rollover_window() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.origin_seq_highwater = Some(758); + session.last_effective_manifest_host = Some("origin.example.com".to_string()); + session.mark_authorized_media_access(100); + let previous_manifest = + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:757\n#EXTINF:4.0,\n/hls/shared/live/session/lease/r/old.ts\n".to_string(); + session.transient.replace_manifest_with_semantics(previous_manifest.clone(), 10, None); + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:226\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(matches!(result, Err(HlsManifestCommitError::TimelineRejected { .. }))); + assert_eq!(session.transient.last_manifest_body.as_deref(), Some(previous_manifest.as_str())); + assert_eq!(session.origin_seq_highwater, Some(758)); +} + +#[test] +fn transient_commit_rebases_expired_session_highwater() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.target_duration = Some(12); + session.origin_seq_highwater = Some(758); + session.last_effective_manifest_host = Some("origin.example.com".to_string()); + session.mark_authorized_media_access(1_000); + let previous_manifest = + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:757\n#EXTINF:4.0,\n/hls/shared/live/session/lease/r/old.ts\n".to_string(); + session.transient.replace_manifest_with_semantics(previous_manifest.clone(), 10, None); + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:12\n#EXT-X-MEDIA-SEQUENCE:900\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 40_000); + + assert!(result.is_ok()); + assert_ne!(session.transient.last_manifest_body.as_deref(), Some(previous_manifest.as_str())); + assert_eq!(session.origin_seq_highwater, Some(900)); +} + +#[test] +fn transient_commit_accepts_monotonic_media_sequence_and_updates_highwater() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.origin_seq_highwater = Some(758); + session.last_effective_manifest_host = Some("origin.example.com".to_string()); + session.transient.replace_manifest_with_semantics( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:757\n#EXTINF:4.0,\n/hls/shared/live/session/lease/r/old.ts\n".to_string(), + 10, + None, + ); + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:759\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg759.ts\n#EXTINF:4.0,\nseg760.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(result.is_ok()); + assert_eq!(session.origin_seq_highwater, Some(760)); + assert!(session.transient.last_manifest_body.as_ref().is_some_and(|body| body.contains("/r/"))); +} + +#[test] +fn fresh_revalidation_rebases_transient_manifest_on_the_pinned_host() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "12345"), b"secret", 0); + session.mode = HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ExtXKey }; + session.origin_seq_highwater = Some(1_000); + session.last_effective_manifest_host = Some("origin.example.com".to_string()); + let mut request = test_origin_refresh_request(test_session()); + request.manifest_commit_requirement = HlsManifestCommitRequirement::FreshCommitRequired { + reason: HlsFreshManifestRequiredReason::ExpiredRevalidation, + }; + let fetched = fetched_manifest( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:10\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4.0,\nseg10.ts\n", + ); + + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + + assert!(result.is_ok()); + assert_eq!(session.origin_seq_highwater, Some(10)); + assert_eq!(session.last_effective_manifest_host.as_deref(), Some("origin.example.com")); + assert!(session.transient.last_manifest_body.as_ref().is_some_and(|body| body.contains("/r/"))); +} + +#[tokio::test] +async fn unsupported_tag_manifest_commits_transient_rewrite() { + let session = refresh_session_with_origin_body( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-PART:DURATION=1.0,URI=\"part.m4s\"\n\ + #EXTINF:4.0,\nseg.ts\n", + ) + .await; + let session = session.read().await; + + assert!(matches!( + session.mode, + HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::UnsupportedTag { .. } } + )); + assert!(session.transient.last_manifest_body.is_some()); +} + +#[tokio::test] +async fn parser_unsupported_feature_manifest_commits_transient_rewrite() { + let session = refresh_session_with_origin_body( + "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-BYTERANGE:10\n#EXTINF:4.0,\nseg.ts\n", + ) + .await; + let session = session.read().await; + + assert!(matches!( + session.mode, + HlsSessionMode::TransientPassthrough { reason: TransientPassthroughReason::ParserUnsupportedFeature { .. } } + )); + assert!(session.transient.last_manifest_body.is_some()); +} + +#[test] +fn fast_start_transient_transition_is_allowed_only_before_publication() { + for mode in [shared::model::HlsStartupMode::FirstReady, shared::model::HlsStartupMode::Progressive] { + for published in [false, true] { + let mut session = HlsSession::new(HlsSessionKey::new(1, "transition"), b"secret", 0); + let config = tuliprox_core::model::HlsStartupConfig { mode, ..Default::default() }; + let worker = Arc::new(HlsSegmentWorkerPool::default()); + session.startup = Some(crate::HlsSessionStartup { + config: config.clone(), + first_data_timeout_ms: 10000, + worker: Arc::downgrade(&worker), + access_leases: Arc::new(RwLock::new(HlsAccessLeaseStore::default())), + revisions: std::collections::BTreeMap::new(), + policy_fixed: true, + store: Arc::new(crate::SegmentRevisionStore::default()), + budget: crate::ProgressiveBudgetManager::new(config), + }); + if published { + session.published_live_origin_baseline = Some(crate::session::HlsPublishedLiveOriginBaseline { + evidence_proxy_seq: 0, + origin_epoch: 0, + rendered_at_ms: 10, + }); + } + let before = session.published_live_origin_baseline; + let request = test_origin_refresh_request(test_session()); + let fetched = fetched_manifest("#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-KEY:METHOD=SAMPLE-AES,URI=\"key.bin\"\n#EXTINF:4,\n1.ts\n"); + let result = commit_fetched_manifest(&mut session, &fetched, &request, 100); + if published { + assert!(matches!( + result, + Err(HlsManifestCommitError::TimelineRejected { + reason: HlsManifestRejectLogReason::StartupRepresentationChange, + }) + )); + assert!(matches!(session.mode, HlsSessionMode::NormalCacheTimeline)); + assert!(session.startup.is_some()); + assert_eq!(session.published_live_origin_baseline, before); + assert!(session.transient.last_manifest_body.is_none()); + assert!(session.transient.resources.is_empty()); + } else { + assert!(result.is_ok()); + assert!(matches!(session.mode, HlsSessionMode::TransientPassthrough { .. })); + assert!(session.startup.is_none()); + assert!(session.transient.last_manifest_body.is_some()); + } + } + } +} diff --git a/backend/hls/src/renderer.rs b/backend/hls/src/renderer.rs index b2ead77a9..148475bdc 100644 --- a/backend/hls/src/renderer.rs +++ b/backend/hls/src/renderer.rs @@ -107,6 +107,8 @@ impl HlsManifestRenderer { impl HlsSession { pub fn render_and_store_manifest(&mut self, rendered_at_ms: u64) -> Result { + self.reconcile_startup_eligibility(); + super::segment_fetcher::recompute_unpublished_live_head(self); let rendered = HlsManifestRenderer::render(self, rendered_at_ms)?; match self.store_rendered_manifest(rendered.clone()) { RenderedManifestStoreOutcome::Stored => Ok(rendered), @@ -194,6 +196,24 @@ impl HlsSession { } pub fn renderer_candidate_window_proxy_seqs(session: &HlsSession) -> Vec { + if session.startup.is_some() { + let Some((head, tail)) = + session.publishable_origin_head_proxy_seq.zip(session.publishable_origin_tail_proxy_seq) + else { + return Vec::new(); + }; + return (head..=tail.min(head.saturating_add(super::startup_policy::STARTUP_WINDOW_SUCCESSORS))) + .take_while(|seq| { + session.segments.get(seq).is_some_and(|entry| { + entry.origin_fetch_ref.is_some() + && !matches!( + entry.status, + SegmentCacheStatus::FailedPermanent { .. } | SegmentCacheStatus::Expired + ) + }) + }) + .collect(); + } for render_gap_segments in 0..=session.render_policy.initial_render_gap_segments { if let Some(window) = select_window(session, render_gap_segments, RenderTailPolicy::RequirePlannedTail) { return window; @@ -203,6 +223,10 @@ pub fn renderer_candidate_window_proxy_seqs(session: &HlsSession) -> Vec { } fn is_renderable(entry: &SegmentEntry, session: &HlsSession) -> bool { + if let Some(startup) = &session.startup { + return entry.is_fast_start_eligible() + && startup.revisions.get(&entry.proxy_seq).is_some_and(|revision| revision.revision().is_publishable()); + } match entry.status { SegmentCacheStatus::Ready { .. } => {} SegmentCacheStatus::Queued { .. } | SegmentCacheStatus::Fetching { .. } => { @@ -226,6 +250,23 @@ fn is_renderable(entry: &SegmentEntry, session: &HlsSession) -> bool { fn select_window(session: &HlsSession, render_gap_segments: usize, tail_policy: RenderTailPolicy) -> Option> { let head_seq = session.publishable_origin_head_proxy_seq?; + if session.startup.is_some() { + let planned_tail = + session.publishable_origin_tail_proxy_seq?.checked_sub(u64::try_from(render_gap_segments).ok()?)?; + let wanted = if session.published_live_origin_baseline.is_none() { + MIN_VISIBLE_SEGMENTS + } else { + TARGET_VISIBLE_SEGMENTS + }; + let mut window = Vec::new(); + for seq in head_seq..=planned_tail.min(head_seq.saturating_add(u64::try_from(wanted).ok()?.saturating_sub(1))) { + if !session.segments.get(&seq).is_some_and(|entry| is_renderable(entry, session)) { + break; + } + window.push(seq); + } + return (window.len() >= MIN_VISIBLE_SEGMENTS).then_some(window); + } let requested_tail_seq = session.publishable_origin_tail_proxy_seq?.checked_sub(u64::try_from(render_gap_segments).ok()?)?; if requested_tail_seq < head_seq { diff --git a/backend/hls/src/resource_fetch.rs b/backend/hls/src/resource_fetch.rs index 2ffa64489..3234f73cb 100644 --- a/backend/hls/src/resource_fetch.rs +++ b/backend/hls/src/resource_fetch.rs @@ -155,6 +155,7 @@ pub enum HlsOriginResourceFetchError { InvalidOriginUrl, InvalidByteRange, UnexpectedByteRangeStatus, + UnsupportedStartupMedia, ContentCoding(HlsContentCodingFailure), ContentDecoding(HlsContentDecodingFailure), CacheObjectLimit { @@ -327,6 +328,7 @@ impl HlsOriginResourceFetchError { | Self::InvalidOriginUrl | Self::InvalidByteRange | Self::UnexpectedByteRangeStatus + | Self::UnsupportedStartupMedia | Self::ContentCoding(_) | Self::CacheObjectLimit { .. } ) @@ -340,6 +342,7 @@ impl HlsOriginResourceFetchError { Self::Transport(_) | Self::InvalidOriginUrl | Self::InvalidByteRange | Self::UnexpectedByteRangeStatus => { HlsResourceFetchLogStatus::TransportError } + Self::UnsupportedStartupMedia => HlsResourceFetchLogStatus::UnsupportedStartupMedia, Self::Redirect => HlsResourceFetchLogStatus::RedirectError, Self::Timeout => HlsResourceFetchLogStatus::Timeout, Self::Superseded => HlsResourceFetchLogStatus::Superseded, @@ -364,6 +367,7 @@ impl HlsOriginResourceFetchError { | Self::InvalidOriginUrl | Self::InvalidByteRange | Self::UnexpectedByteRangeStatus + | Self::UnsupportedStartupMedia | Self::ContentCoding(_) | Self::CacheObjectLimit { .. } | Self::LocalCacheCapacity { .. } @@ -744,6 +748,7 @@ enum HlsResourceFetchLogStatus { Superseded, TransportError, RedirectError, + UnsupportedStartupMedia, ContentCodingError(HlsContentCodingFailure), ContentDecodingError(HlsContentDecodingFailure), CacheObjectLimit { limit: u64 }, @@ -763,6 +768,7 @@ impl HlsResourceFetchLogStatus { Self::Superseded => "Superseded".to_string(), Self::TransportError => "TransportError".to_string(), Self::RedirectError => "RedirectError".to_string(), + Self::UnsupportedStartupMedia => "UnsupportedStartupMedia".to_string(), Self::ContentCodingError(failure) => failure.label(), Self::ContentDecodingError(failure) => failure.label(), Self::CacheObjectLimit { limit } => format!("CacheObjectLimit(limit={limit})"), @@ -1039,6 +1045,16 @@ mod tests { assert_eq!(error.log_status().label(), "ContentCodingError(Unsupported)"); } + #[test] + fn incompatible_startup_media_is_permanent_without_a_fabricated_origin_status() { + let error = HlsOriginResourceFetchError::UnsupportedStartupMedia; + assert!(!error.retryable_failure()); + assert!(error.object_failure_is_permanent()); + assert!(error.aborts_without_retry()); + assert_eq!(error.permanent_status(), None); + assert_eq!(error.log_status().label(), "UnsupportedStartupMedia"); + } + #[test] fn streaming_decoder_failure_is_retryable_and_not_a_cache_commit_failure() { let error = io::Error::new(io::ErrorKind::InvalidData, ContentDecodingIoError { coding: ContentCoding::Gzip }); @@ -1445,6 +1461,90 @@ mod tests { assert!(!err.aborts_without_retry()); } + // Exercise the real retry runner, including attempt preparation and HTTP + // delivery. A denied reservation must never issue an upstream request. + async fn run_reserved_provider_scenario( + denied_attempts: usize, + ) -> std::io::Result<(Result, usize, usize, usize)> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let address = listener.local_addr()?; + let request_count = Arc::new(AtomicUsize::new(0)); + let server_request_count = Arc::clone(&request_count); + let server = tokio::spawn(async move { + while let Ok((mut stream, _)) = listener.accept().await { + read_request(&mut stream).await; + server_request_count.fetch_add(1, Ordering::Relaxed); + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await?; + } + Ok::<(), std::io::Error>(()) + }); + let prepare_count = Arc::new(AtomicUsize::new(0)); + let attempt_prepare_count = Arc::clone(&prepare_count); + let commit_count = Arc::new(AtomicUsize::new(0)); + let response_commit_count = Arc::clone(&commit_count); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let identity = test_log_identity(); + let result = run_hls_origin_resource_retry_loop_with_attempt_prepare( + fetch_target(format!("http://{address}/segment.ts")), + fetch_clients(false), + &policy, + &identity, + move |attempt| { + attempt_prepare_count.fetch_add(1, Ordering::Relaxed); + async move { + if attempt.attempt_index < denied_attempts { + Err(HlsOriginResourceFetchError::ProviderUnavailable( + HlsBoundAccountAcquireErrorKind::ReservedForOther, + )) + } else { + Ok(attempt.attempt_index) + } + } + .boxed() + }, + |_guard| async {}.boxed(), + move |_response, _attempt, _deadline, guard| { + response_commit_count.fetch_add(1, Ordering::Relaxed); + async move { Ok(guard) }.boxed() + }, + ) + .await; + server.abort(); + Ok(( + result, + prepare_count.load(Ordering::Relaxed), + request_count.load(Ordering::Relaxed), + commit_count.load(Ordering::Relaxed), + )) + } + + #[tokio::test] + async fn reserved_provider_retries_until_reservation_clears() -> std::io::Result<()> { + let (result, preparations, requests, commits) = run_reserved_provider_scenario(1).await?; + assert!(matches!(result, Ok(1))); + assert_eq!(preparations, 2); + assert_eq!(requests, 1); + assert_eq!(commits, 1); + Ok(()) + } + + #[tokio::test] + async fn reserved_provider_stops_at_existing_retry_budget_without_origin_io() -> std::io::Result<()> { + let (result, preparations, requests, commits) = run_reserved_provider_scenario(usize::MAX).await?; + assert!(matches!( + result, + Err(HlsOriginResourceFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::ReservedForOther)) + )); + assert_eq!(preparations, SegmentFetchPolicy::default().retry_delays_ms.len()); + assert_eq!(requests, 0); + assert_eq!(commits, 0); + Ok(()) + } + #[test] fn missing_provider_account_aborts_resource_retry() { let err = HlsOriginResourceFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Missing); diff --git a/backend/hls/src/response.rs b/backend/hls/src/response.rs index a6c8af931..28aa09365 100644 --- a/backend/hls/src/response.rs +++ b/backend/hls/src/response.rs @@ -1,4 +1,4 @@ -#![allow(clippy::large_futures, clippy::large_enum_variant, clippy::too_many_lines)] +#![allow(clippy::large_futures, clippy::large_enum_variant)] use super::{ hls_client_body_send_deadline, refresh_hls_client_body_send_deadline, safe_hls_access_lease_id, @@ -10,43 +10,33 @@ use super::{ TransientResourceKind, }; use arc_swap::ArcSwapOption; -use axum::{ - body::Body, - http::{header, HeaderValue, Response, StatusCode}, - response::IntoResponse, -}; +use axum::http::StatusCode; use bytes::Bytes; -use futures::{Stream, StreamExt}; -use log::debug; +use futures::Stream; use std::{ - future::Future, io, pin::Pin, sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU64}, Arc, LazyLock, }, - task::{Context, Poll}, - time::{Duration, Instant}, -}; -use tokio::{ - io::AsyncReadExt, - sync::Semaphore, - time::{sleep, Sleep}, -}; -use tokio_util::io::ReaderStream; -use tuliprox_core::utils::{ - byte_range::{resolve_single_byte_range, SingleByteRange}, - response_compression::mark_response_as_uncompressed, + time::Instant, }; +use tokio::{sync::Semaphore, time::Sleep}; use tuliprox_session::StreamMeterHandle; const ACCEPT_RANGES_VALUE: &str = "bytes"; + const NOT_READY_RETRY_AFTER_MS: u64 = 1_000; + const BODY_READER_WAIT_LOG_THRESHOLD_MS: u128 = 10; + const PREPARED_MEDIA_CHUNK_SIZE: usize = 4 * 1_024; + static NEXT_HLS_BODY_LOG_ID: AtomicU64 = AtomicU64::new(1); + const HLS_MEDIA_ACTIVITY_TASK_CAPACITY: usize = 256; + static HLS_MEDIA_ACTIVITY_TASK_PERMITS: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(HLS_MEDIA_ACTIVITY_TASK_CAPACITY))); @@ -57,172 +47,6 @@ struct FiniteBytesSelection { content_range: Option, } -fn select_finite_bytes(bytes: Bytes, range_header: Option<&HeaderValue>) -> Result { - let full_size = u64::try_from(bytes.len()).unwrap_or(u64::MAX); - match resolve_single_byte_range(range_header, full_size) { - SingleByteRange::Full => Ok(FiniteBytesSelection { - status: StatusCode::OK, - content_length: full_size, - body: bytes, - content_range: None, - }), - SingleByteRange::Partial { start, end, length } => { - let start_usize = usize::try_from(start).unwrap_or(bytes.len()); - let end_exclusive = usize::try_from(end.saturating_add(1)).unwrap_or(bytes.len()).min(bytes.len()); - Ok(FiniteBytesSelection { - status: StatusCode::PARTIAL_CONTENT, - body: bytes.slice(start_usize.min(end_exclusive)..end_exclusive), - content_length: length, - content_range: Some(format!("bytes {start}-{end}/{full_size}")), - }) - } - SingleByteRange::Unsatisfiable => Err(full_size), - } -} - -fn build_finite_bytes_response( - selection: &FiniteBytesSelection, - body: Body, - content_type: &str, - cache_control: &'static str, -) -> Response { - let mut response = Response::new(body); - *response.status_mut() = selection.status; - let headers = response.headers_mut(); - insert_header_value(headers, header::CONTENT_TYPE, content_type); - insert_header_value(headers, header::ACCEPT_RANGES, ACCEPT_RANGES_VALUE); - insert_header_value(headers, header::CACHE_CONTROL, cache_control); - insert_u64_header(headers, header::CONTENT_LENGTH, selection.content_length); - if let Some(content_range) = selection.content_range.as_deref() { - insert_header_value(headers, header::CONTENT_RANGE, content_range); - } - mark_response_as_uncompressed(&mut response); - response -} - -/// Serves immutable prepared bytes without touching the live-lease state. -/// -/// The route is the authorization boundary; callers do not get any -/// upstream origin work on top of the response. -pub fn finite_hls_immutable_media_response( - bytes: Bytes, - range_header: Option<&HeaderValue>, - content_type: &'static str, - cache_control: &'static str, - head_only: bool, -) -> Response { - let selection = match select_finite_bytes(bytes, range_header) { - Ok(selection) => selection, - Err(full_size) => return range_not_satisfiable_response(full_size), - }; - let body = if head_only { Body::empty() } else { Body::from(selection.body.clone()) }; - build_finite_bytes_response(&selection, body, content_type, cache_control) -} - -/// Serves immutable in-memory media through the same `QoS`, completion, drop and -/// client-send-deadline stream used by disk-backed Shared-HLS media. -pub fn finite_hls_media_response( - bytes: Bytes, - range_header: Option<&HeaderValue>, - content_type: &'static str, - cache_control: &'static str, - context: &HlsCacheResponseContext, - _proxy_session_id: &ProxySessionId, - resource_id: String, -) -> Response { - let selection = match select_finite_bytes(bytes, range_header) { - Ok(selection) => selection, - Err(full_size) => return range_not_satisfiable_response(full_size), - }; - context.metrics.record_cache_hit(); - if selection.status == StatusCode::PARTIAL_CONTENT { - context.metrics.record_cache_range_hit(); - } - let body_context = CacheBodyLogContext { - body_id: next_hls_body_log_id(), - identity: context.log_identity.clone(), - resource_id, - object_kind: "TerminalSegment", - source: "prepared", - content_length: selection.content_length, - }; - let stream = PreparedBytesStream::new(selection.body.clone()); - let stream = ActiveReaderStream::new( - Box::pin(stream), - None, - body_context, - Arc::clone(&context.qos_meter), - context.media_activity_marker.as_ref().and_then(HlsMediaActivityMarker::completed_segment_marker), - None, - ); - build_finite_bytes_response(&selection, Body::from_stream(stream), content_type, cache_control) -} - -/// Serves one frozen lease-bound AES key revision without consulting mutable -/// transient-resource state or performing origin I/O. -pub fn finite_hls_terminal_key_response( - bytes: Bytes, - range_header: Option<&HeaderValue>, - content_type: &str, - cache_control: &'static str, - context: &HlsCacheResponseContext, - _proxy_session_id: &ProxySessionId, - resource_id: String, -) -> Response { - let selection = match select_finite_bytes(bytes, range_header) { - Ok(selection) => selection, - Err(full_size) => return range_not_satisfiable_response(full_size), - }; - context.metrics.record_cache_hit(); - if selection.status == StatusCode::PARTIAL_CONTENT { - context.metrics.record_cache_range_hit(); - } - let body_context = CacheBodyLogContext { - body_id: next_hls_body_log_id(), - identity: context.log_identity.clone(), - resource_id, - object_kind: "TerminalKey", - source: "prepared", - content_length: selection.content_length, - }; - let stream = PreparedBytesStream::new(selection.body.clone()); - let stream = ActiveReaderStream::new( - Box::pin(stream), - None, - body_context, - Arc::clone(&context.qos_meter), - context.media_activity_marker.as_ref().and_then(HlsMediaActivityMarker::completed_segment_marker), - None, - ); - build_finite_bytes_response(&selection, Body::from_stream(stream), content_type, cache_control) -} - -/// Builds terminal-media HEAD metadata without rendering bytes or creating a -/// reader, `QoS` stream, completion callback, or media-activity side effect. -pub fn finite_hls_media_head_response( - full_size: u64, - range_header: Option<&HeaderValue>, - content_type: &'static str, - cache_control: &'static str, -) -> Response { - let selection = match resolve_single_byte_range(range_header, full_size) { - SingleByteRange::Full => FiniteBytesSelection { - status: StatusCode::OK, - body: Bytes::new(), - content_length: full_size, - content_range: None, - }, - SingleByteRange::Partial { start, end, length } => FiniteBytesSelection { - status: StatusCode::PARTIAL_CONTENT, - body: Bytes::new(), - content_length: length, - content_range: Some(format!("bytes {start}-{end}/{full_size}")), - }, - SingleByteRange::Unsatisfiable => return range_not_satisfiable_response(full_size), - }; - build_finite_bytes_response(&selection, Body::empty(), content_type, cache_control) -} - #[derive(Clone)] struct CacheObject { is_media: bool, @@ -252,26 +76,6 @@ struct CacheBodyLogContext { content_length: u64, } -enum CacheObjectLookup { - Ready(CacheObject), - Failure(HlsResourceServeFailure), -} - -/// Result of a Shared-HLS cache object serve decision. -pub enum HlsResourceServeOutcome { - Ready(Response), - Failure(HlsResourceServeFailure), -} - -/// Typed Shared-HLS resource failure before endpoint-level custom-response mapping. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsResourceServeFailure { - TemporaryUnavailable { retry_after_ms: u64 }, - Missing, - Expired, - PermanentFailed { status: Option }, -} - pub struct HlsCacheResponseContext { pub hls_access_lease_id: HlsAccessLeaseId, log_identity: HlsLogIdentity, @@ -283,39 +87,6 @@ pub struct HlsCacheResponseContext { pub now_ms: u64, } -impl HlsCacheResponseContext { - #[allow(clippy::too_many_arguments)] - pub fn new( - hls_access_lease_id: HlsAccessLeaseId, - log_identity: HlsLogIdentity, - cache_duration_seconds: u64, - metrics: Arc, - segment_repair: Arc, - qos_meter: Option>, - media_activity_marker: Option, - now_ms: u64, - ) -> Self { - Self { - hls_access_lease_id, - log_identity, - cache_duration_seconds, - metrics, - segment_repair, - qos_meter: Arc::new(ArcSwapOption::from(qos_meter)), - media_activity_marker, - now_ms, - } - } - - pub fn set_qos_meter(&self, qos_meter: Option>) { self.qos_meter.store(qos_meter); } - - pub async fn mark_media_activity(&self) { - if let Some(marker) = &self.media_activity_marker { - marker.mark_at(self.now_ms).await; - } - } -} - #[derive(Clone)] pub struct HlsMediaActivityMarker { manager: Arc, @@ -330,198 +101,6 @@ pub struct HlsMediaActivityMarker { playback_request_id: Option, } -impl HlsMediaActivityMarker { - /// Confirms only a polled, non-empty media body. Bytes retain their shared - /// backing storage; no payload copy or detached per-chunk task is needed. - pub fn confirm_media_response(&self, response: Response) -> Response { - let (parts, body) = response.into_parts(); - let mut marker = Some(self.clone()); - let stream = body.into_data_stream().then(move |chunk| { - let confirmation = if chunk.as_ref().is_ok_and(|bytes| !bytes.is_empty()) { marker.take() } else { None }; - async move { - if let Some(marker) = confirmation { - let outcome = marker - .manager - .mark_delivered_media_for_lease( - &marker.session, - &marker.lease_id, - &marker.proxy_session_id, - marker.lease_identity, - current_time_millis(), - ) - .await; - if matches!(outcome, HlsMediaActivityCommitOutcome::Committed) { - if let (Some(active_provider), Some(owner), Some(request_id)) = ( - marker.active_provider.as_ref(), - marker.session_owner.as_deref(), - marker.playback_request_id, - ) { - active_provider.confirm_identified_playback_activity(owner, request_id); - } - } - marker.log_uncommitted_activity(outcome, "media-delivery"); - } - chunk - } - }); - Response::from_parts(parts, Body::from_stream(stream)) - } - pub fn new( - manager: Arc, - session: HlsSessionHandle, - proxy_session_id: ProxySessionId, - lease_id: HlsAccessLeaseId, - lease_identity: HlsMediaLeaseIdentity, - ) -> Self { - Self { - manager, - session, - proxy_session_id, - lease_id, - lease_identity, - completed_segment: None, - completion_scheduled: None, - active_provider: None, - session_owner: None, - playback_request_id: None, - } - } - - pub fn with_active_provider( - mut self, - active_provider: Arc, - session_owner: Option, - playback_request_id: Option, - ) -> Self { - self.active_provider = Some(active_provider); - self.session_owner = session_owner; - self.playback_request_id = playback_request_id; - self - } - - async fn for_segment_request(mut self, proxy_seq: u64, requested_at_ms: u64) -> Option { - // A terminal lease may still read the immutable READY live-tail segment - // protected by its plan. That read marks terminal activity, but it must - // never mutate the live playback cursor. - if !self.lease_identity.is_live() { - return Some(self); - } - self.completed_segment = self - .manager - .record_access_lease_segment_request_started_if_identity_matches( - &self.lease_id, - &self.proxy_session_id, - self.lease_identity, - proxy_seq, - requested_at_ms, - ) - .await; - let _ = self.completed_segment.as_ref()?; - self.completion_scheduled = Some(Arc::new(AtomicBool::new(false))); - Some(self) - } - - pub async fn mark_at(&self, now_ms: u64) { - let outcome = self - .manager - .mark_authorized_media_access_for_lease_if_identity_matches( - &self.session, - &self.lease_id, - &self.proxy_session_id, - self.lease_identity, - now_ms, - ) - .await; - self.log_uncommitted_activity(outcome, "access"); - } - - async fn mark_completion_at(&self, now_ms: u64) { - let Some(token) = self.completed_segment else { - self.mark_at(now_ms).await; - return; - }; - let outcome = self - .manager - .record_access_lease_segment_request_completed_and_mark_media_if_identity_matches( - &self.session, - &self.lease_id, - &self.proxy_session_id, - self.lease_identity, - token, - now_ms, - ) - .await; - self.log_uncommitted_activity(outcome, "live-segment-completion"); - } - - fn spawn_mark_completion_now(&self) { - let Some(scheduled) = self.completion_scheduled.as_ref() else { - debug!( - "HLS media completion ignored: lease={} proxy_session={} reason=missing-completion-token", - safe_hls_access_lease_id(&self.lease_id), - safe_proxy_session_id(&self.proxy_session_id) - ); - return; - }; - if scheduled.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire).is_err() { - debug!( - "HLS media completion ignored: lease={} proxy_session={} reason=completion-already-scheduled", - safe_hls_access_lease_id(&self.lease_id), - safe_proxy_session_id(&self.proxy_session_id) - ); - return; - } - let marker = self.clone(); - spawn_bounded_media_completion(Arc::clone(&HLS_MEDIA_ACTIVITY_TASK_PERMITS), async move { - marker.mark_completion_at(current_time_millis()).await; - }); - } - - fn completed_segment_marker(&self) -> Option { self.completed_segment.map(|_| self.clone()) } - - fn record_startup_segment_request(&self, proxy_seq: u64, now_ms: u64) { - self.manager.startup_observability().record_first_visible_segment_request(&self.lease_id, proxy_seq, now_ms); - } - - fn record_startup_repair_decision(&self, proxy_seq: u64, now_ms: u64) { - self.manager.startup_observability().record_repair_decision(&self.lease_id, proxy_seq, now_ms); - } - - fn begin_startup_cache_response( - &self, - proxy_seq: u64, - body_id: &str, - now_ms: u64, - ) -> Option { - self.manager.startup_observability().begin_cache_response(&self.lease_id, proxy_seq, body_id, now_ms) - } - - fn log_uncommitted_activity(&self, outcome: HlsMediaActivityCommitOutcome, phase: &'static str) { - let reason = match outcome { - HlsMediaActivityCommitOutcome::Committed => return, - HlsMediaActivityCommitOutcome::StaleLeaseIdentity => "expired-or-playback-generation-race", - HlsMediaActivityCommitOutcome::DeferredLockContention => "lock-contention", - }; - debug!( - "HLS media activity ignored: phase={phase} lease={} proxy_session={} reason={reason}", - safe_hls_access_lease_id(&self.lease_id), - safe_proxy_session_id(&self.proxy_session_id) - ); - } -} - -fn spawn_bounded_media_completion( - completion_permits: Arc, - completion: impl Future + Send + 'static, -) { - tokio::spawn(async move { - let Ok(_permit) = completion_permits.acquire_owned().await else { - return; - }; - completion.await; - }); -} - struct CacheObjectServeContext { cache_duration_seconds: u64, metrics: Option>, @@ -532,547 +111,6 @@ struct CacheObjectServeContext { now_ms: u64, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum HlsPlaybackCursorTracking { - Disabled, - Segment { proxy_seq: u64 }, -} - -impl HlsPlaybackCursorTracking { - fn full_object_proxy_seq(self, range: SingleByteRange, full_size: u64) -> Option { - match self { - Self::Segment { proxy_seq } if resolved_range_covers_full_object(range, full_size) => Some(proxy_seq), - Self::Disabled | Self::Segment { .. } => None, - } - } -} - -fn resolved_range_covers_full_object(range: SingleByteRange, full_size: u64) -> bool { - match range { - SingleByteRange::Full => true, - SingleByteRange::Partial { start, end, length } => { - start == 0 && full_size.checked_sub(1) == Some(end) && length == full_size - } - SingleByteRange::Unsatisfiable => false, - } -} - -impl CacheObjectServeContext { - fn from_response_context(context: &HlsCacheResponseContext) -> Self { - Self { - cache_duration_seconds: context.cache_duration_seconds, - metrics: Some(Arc::clone(&context.metrics)), - segment_repair: Arc::clone(&context.segment_repair), - qos_meter: Arc::clone(&context.qos_meter), - media_activity_marker: context.media_activity_marker.clone(), - playback_cursor_tracking: HlsPlaybackCursorTracking::Disabled, - now_ms: context.now_ms, - } - } -} - -/// Serves a committed Ready segment or returns a typed failure for endpoint-level mapping. -pub async fn serve_hls_segment_cache_outcome( - segment_cache: Arc, - session: HlsSessionHandle, - segment_file: HlsSegmentFile, - range_header: Option, - context: &HlsCacheResponseContext, -) -> HlsResourceServeOutcome { - match lookup_segment_cache_object(&session, &segment_file, &context.hls_access_lease_id).await { - CacheObjectLookup::Ready(object) => cache_object_serve_outcome( - serve_cache_object( - segment_cache, - object, - range_header, - CacheObjectServeContext { - playback_cursor_tracking: HlsPlaybackCursorTracking::Segment { proxy_seq: segment_file.proxy_seq }, - ..CacheObjectServeContext::from_response_context(context) - }, - ) - .await, - ), - CacheObjectLookup::Failure(failure) => HlsResourceServeOutcome::Failure(failure), - } -} - -/// Serves a committed Ready EXT-X-MAP object or returns a typed failure for endpoint-level mapping. -pub async fn serve_hls_map_cache_outcome( - segment_cache: Arc, - session: HlsSessionHandle, - map_file: HlsMapFile, - range_header: Option, - context: &HlsCacheResponseContext, -) -> HlsResourceServeOutcome { - match lookup_map_cache_object(&session, &map_file, &context.hls_access_lease_id).await { - CacheObjectLookup::Ready(object) => cache_object_serve_outcome( - serve_cache_object( - segment_cache, - object, - range_header, - CacheObjectServeContext::from_response_context(context), - ) - .await, - ), - CacheObjectLookup::Failure(failure) => HlsResourceServeOutcome::Failure(failure), - } -} - -/// Serves a committed Ready transient passthrough full object from the HLS cache. -pub async fn serve_hls_transient_object_cache_response( - segment_cache: Arc, - session: HlsSessionHandle, - resource_file: TransientResourceFile, - range_header: Option, - context: &HlsCacheResponseContext, -) -> Response { - match serve_hls_transient_object_cache_outcome(segment_cache, session, resource_file, range_header, context).await { - HlsResourceServeOutcome::Ready(response) => response, - HlsResourceServeOutcome::Failure(failure) => hls_resource_failure_default_response(failure), - } -} - -/// Serves a committed Ready transient passthrough object or returns a typed failure for endpoint-level mapping. -pub async fn serve_hls_transient_object_cache_outcome( - segment_cache: Arc, - session: HlsSessionHandle, - resource_file: TransientResourceFile, - range_header: Option, - context: &HlsCacheResponseContext, -) -> HlsResourceServeOutcome { - match lookup_transient_object_cache_object(&session, &resource_file, &context.hls_access_lease_id, context.now_ms) - .await - { - CacheObjectLookup::Ready(object) => cache_object_serve_outcome( - serve_cache_object( - segment_cache, - object, - range_header, - CacheObjectServeContext::from_response_context(context), - ) - .await, - ), - CacheObjectLookup::Failure(failure) => HlsResourceServeOutcome::Failure(failure), - } -} - -fn hls_resource_failure_default_response(failure: HlsResourceServeFailure) -> Response { - match failure { - HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms } => { - service_unavailable_not_ready_response(retry_after_ms) - } - HlsResourceServeFailure::Missing - | HlsResourceServeFailure::Expired - | HlsResourceServeFailure::PermanentFailed { .. } => StatusCode::NOT_FOUND.into_response(), - } -} - -fn cache_object_serve_outcome(result: Result, HlsResourceServeFailure>) -> HlsResourceServeOutcome { - match result { - Ok(response) => HlsResourceServeOutcome::Ready(response), - Err(failure) => HlsResourceServeOutcome::Failure(failure), - } -} - -async fn serve_cache_object( - segment_cache: Arc, - object: CacheObject, - range_header: Option, - context: CacheObjectServeContext, -) -> Result, HlsResourceServeFailure> -where - K: super::HlsCacheObjectKey + Send + Sync + 'static, -{ - let guard = CacheReadGuard::new(Arc::clone(&object.access), context.now_ms); - let requested_proxy_seq = match context.playback_cursor_tracking { - HlsPlaybackCursorTracking::Segment { proxy_seq } => Some(proxy_seq), - HlsPlaybackCursorTracking::Disabled => None, - }; - if let (Some(proxy_seq), Some(marker)) = (requested_proxy_seq, context.media_activity_marker.as_ref()) { - marker.record_startup_segment_request(proxy_seq, context.now_ms); - } - if let Some(repair_context) = object.repair_context.clone() { - if let Err(err) = - context.segment_repair.repair_ready_cache_hit(&segment_cache, &object.key, repair_context).await - { - debug!( - "HLS segment repair skipped for ready cache hit: session={} proxy_session={} resource={} error={err}", - object.log_context.identity.session(), - object.log_context.identity.proxy_session(), - object.log_context.resource_id - ); - } - } - if let (Some(proxy_seq), Some(marker)) = (requested_proxy_seq, context.media_activity_marker.as_ref()) { - marker.record_startup_repair_decision(proxy_seq, current_time_millis()); - } - let metadata_started_at = Instant::now(); - let metadata = match segment_cache.metadata(&object.key).await { - Ok(Some(metadata)) => metadata, - Ok(None) => return Ok(StatusCode::NOT_FOUND.into_response()), - Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(StatusCode::NOT_FOUND.into_response()), - Err(_) => return Ok(StatusCode::INTERNAL_SERVER_ERROR.into_response()), - }; - log_body_reader_wait_if_slow(&object.log_context, "metadata", metadata_started_at.elapsed().as_millis()); - - let body_id = next_hls_body_log_id(); - let range = resolve_single_byte_range(range_header.as_ref(), metadata.size); - let cursor_proxy_seq = context.playback_cursor_tracking.full_object_proxy_seq(range, metadata.size); - let (status, start, end, content_length) = match range { - SingleByteRange::Full => { - if metadata.size == 0 { - if let Some(metrics) = &context.metrics { - metrics.record_cache_hit(); - } - if let Some(marker) = &context.media_activity_marker { - marker.mark_at(context.now_ms).await; - } - return Ok(empty_ok_response(&object.content_type, context.cache_duration_seconds)); - } - if let Some(metrics) = &context.metrics { - metrics.record_cache_hit(); - } - debug!( - "HLS cache response prepared: body_id={} lease={} session={} proxy_session={} resource={} source=cache range=full content_length={} content_type={}", - body_id, - object.log_context.lease, - object.log_context.identity.session(), - object.log_context.identity.proxy_session(), - object.log_context.resource_id, - metadata.size, - object.content_type - ); - (StatusCode::OK, 0, metadata.size - 1, metadata.size) - } - SingleByteRange::Partial { start, end, length } => { - if let Some(metrics) = &context.metrics { - metrics.record_cache_hit(); - metrics.record_cache_range_hit(); - } - debug!( - "HLS cache response prepared: body_id={} lease={} session={} proxy_session={} resource={} source=cache range={start}-{end} content_length={length} content_type={}", - body_id, - object.log_context.lease, - object.log_context.identity.session(), - object.log_context.identity.proxy_session(), - object.log_context.resource_id, - object.content_type - ); - (StatusCode::PARTIAL_CONTENT, start, end, length) - } - SingleByteRange::Unsatisfiable => return Ok(range_not_satisfiable_response(metadata.size)), - }; - let file_started_at = Instant::now(); - let file = match segment_cache.open_range(&object.key, start).await { - Ok(file) => file, - Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(StatusCode::NOT_FOUND.into_response()), - Err(_) => return Ok(StatusCode::INTERNAL_SERVER_ERROR.into_response()), - }; - log_body_reader_wait_if_slow(&object.log_context, "file", file_started_at.elapsed().as_millis()); - let cursor_marker = - if let (Some(proxy_seq), Some(marker)) = (cursor_proxy_seq, context.media_activity_marker.clone()) { - let Some(marker) = marker.for_segment_request(proxy_seq, context.now_ms).await else { - return Err(HlsResourceServeFailure::Expired); - }; - Some(marker) - } else { - None - }; - let startup_body_observation = requested_proxy_seq.and_then(|proxy_seq| { - context - .media_activity_marker - .as_ref() - .and_then(|marker| marker.begin_startup_cache_response(proxy_seq, &body_id, current_time_millis())) - }); - if let Some(marker) = &context.media_activity_marker { - marker.mark_at(context.now_ms).await; - } - let completion_marker = cursor_marker.as_ref().and_then(HlsMediaActivityMarker::completed_segment_marker); - - let stream = ReaderStream::new(file.take(content_length)); - let body_context = CacheBodyLogContext { - body_id, - identity: object.log_context.identity.clone(), - resource_id: object.log_context.resource_id.clone(), - object_kind: object.log_context.object_kind, - source: object.log_context.body_source, - content_length, - }; - let stream = ActiveReaderStream::new( - Box::pin(stream), - Some(guard), - body_context, - context.qos_meter, - completion_marker, - startup_body_observation, - ); - let mut response = Response::new(Body::from_stream(stream)); - *response.status_mut() = status; - - let headers = response.headers_mut(); - insert_header_value(headers, header::CONTENT_TYPE, &object.content_type); - headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static(ACCEPT_RANGES_VALUE)); - insert_u64_header(headers, header::CONTENT_LENGTH, content_length); - insert_cache_control(headers, context.cache_duration_seconds); - if status == StatusCode::PARTIAL_CONTENT { - insert_header_value(headers, header::CONTENT_RANGE, &format!("bytes {start}-{end}/{}", metadata.size)); - } - mark_response_as_uncompressed(&mut response); - if object.is_media { - if let Some(marker) = &context.media_activity_marker { - response = marker.confirm_media_response(response); - } - } - Ok(response) -} - -async fn lookup_segment_cache_object( - session: &HlsSessionHandle, - segment_file: &HlsSegmentFile, - hls_access_lease_id: &HlsAccessLeaseId, -) -> CacheObjectLookup { - let session = session.read().await; - if session.is_gc_marked_for_removal() { - return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); - } - let Some(entry) = session.segments.get(&segment_file.proxy_seq) else { - return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); - }; - if entry.proxy_file_ext != segment_file.extension { - return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); - } - match entry.status { - SegmentCacheStatus::Ready { .. } => {} - SegmentCacheStatus::Fetching { .. } - | SegmentCacheStatus::Queued { .. } - | SegmentCacheStatus::Discovered - | SegmentCacheStatus::CapacityDeferred { .. } => { - return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { - retry_after_ms: NOT_READY_RETRY_AFTER_MS, - }); - } - SegmentCacheStatus::FailedRetryable { retry_after_ms, .. } => { - return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms }); - } - SegmentCacheStatus::FailedPermanent { status, .. } => { - return CacheObjectLookup::Failure(HlsResourceServeFailure::PermanentFailed { status }); - } - SegmentCacheStatus::Expired => { - return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); - } - } - CacheObjectLookup::Ready(CacheObject { - is_media: true, - key: entry.cache_key.clone(), - access: Arc::clone(&entry.access), - content_type: entry.content_type.clone(), - log_context: CacheObjectLogContext { - lease: safe_hls_access_lease_id(hls_access_lease_id), - identity: HlsLogIdentity::from_session(&session), - resource_id: format!("{:06}", segment_file.proxy_seq), - object_kind: "Segment", - body_source: "normal", - }, - repair_context: Some(HlsSegmentRepairObjectContext { - source: HlsSegmentRepairSource::Normal, - log_identity: HlsLogIdentity::from_session(&session), - proxy_session_id: session.proxy_session_id.clone(), - hls_access_lease_id: Some(hls_access_lease_id.clone()), - rendered_object_id: HlsRepairRenderedObjectId::Normal { proxy_seq: segment_file.proxy_seq }, - resource_id: format!("{:06}", segment_file.proxy_seq), - file_ext: entry.proxy_file_ext.clone(), - // Cache-hit repair validation may carry the concrete fetch URL as diagnostic metadata only. - origin_fetch_uri_for_diagnostics: entry - .origin_fetch_ref - .as_ref() - .map(|fetch_ref| fetch_ref.resolved_origin_url.clone()) - .unwrap_or_default(), - media_sequence: Some(entry.origin_key.host_local_sequence), - discontinuity_sequence: Some(session.discontinuity_sequence), - complete_object: entry.origin_byte_range.is_none(), - encrypted: entry.encryption.is_some(), - custom_response: false, - }), - }) -} - -async fn lookup_map_cache_object( - session: &HlsSessionHandle, - map_file: &HlsMapFile, - hls_access_lease_id: &HlsAccessLeaseId, -) -> CacheObjectLookup { - let session = session.read().await; - if session.is_gc_marked_for_removal() { - return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); - } - let Some(entry) = session.maps.get(&ProxyMapId(map_file.proxy_map_id)) else { - return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); - }; - if entry.proxy_file_ext != map_file.extension { - return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); - } - match entry.status { - MapCacheStatus::Ready { .. } => {} - MapCacheStatus::Fetching { .. } | MapCacheStatus::Queued { .. } | MapCacheStatus::Discovered => { - return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { - retry_after_ms: NOT_READY_RETRY_AFTER_MS, - }); - } - MapCacheStatus::FailedRetryable { retry_after_ms, .. } => { - return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms }); - } - MapCacheStatus::FailedPermanent { status, .. } => { - return CacheObjectLookup::Failure(HlsResourceServeFailure::PermanentFailed { status }); - } - MapCacheStatus::Expired => { - return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); - } - } - CacheObjectLookup::Ready(CacheObject { - is_media: false, - key: entry.cache_key.clone(), - access: Arc::clone(&entry.access), - content_type: entry.content_type.clone(), - log_context: CacheObjectLogContext { - lease: safe_hls_access_lease_id(hls_access_lease_id), - identity: HlsLogIdentity::from_session(&session), - resource_id: format!("map:{:06}", map_file.proxy_map_id), - object_kind: "Map", - body_source: "normal", - }, - repair_context: None, - }) -} - -async fn lookup_transient_object_cache_object( - session: &HlsSessionHandle, - resource_file: &TransientResourceFile, - hls_access_lease_id: &HlsAccessLeaseId, - now_ms: u64, -) -> CacheObjectLookup { - let mut session = session.write().await; - if session.is_gc_marked_for_removal() { - return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); - } - let proxy_session_id = session.proxy_session_id.clone(); - let key = super::TransientPassthroughState::transient_object_key( - &proxy_session_id, - &resource_file.resource_id, - resource_file.extension.clone(), - ); - let Some(resource) = session.transient.resolve_current_resource(&resource_file.resource_id, now_ms) else { - return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); - }; - if resource.file_ext_hint.as_deref() != Some(resource_file.extension.as_str()) { - return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); - } - let resource_state = (resource.kind, resource.encrypted_media); - let resource_kind = Some(resource_state.0); - let Some(entry) = session.transient.ready_object(&key, resource_state.0, now_ms) else { - return match session.transient.object_cache.get(&key).map(|entry| &entry.status) { - Some(super::TransientObjectCacheStatus::Fetching { .. }) => { - CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { - retry_after_ms: NOT_READY_RETRY_AFTER_MS, - }) - } - Some(super::TransientObjectCacheStatus::FailedRetryable { retry_after_ms, .. }) => { - CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { - retry_after_ms: *retry_after_ms, - }) - } - Some(super::TransientObjectCacheStatus::FailedPermanent { status, .. }) => { - CacheObjectLookup::Failure(HlsResourceServeFailure::PermanentFailed { status: *status }) - } - Some(super::TransientObjectCacheStatus::Ready { .. }) => { - CacheObjectLookup::Failure(HlsResourceServeFailure::Expired) - } - None => CacheObjectLookup::Failure(HlsResourceServeFailure::Missing), - }; - }; - CacheObjectLookup::Ready(CacheObject { - is_media: matches!(resource_state.0, TransientResourceKind::Segment | TransientResourceKind::Part), - key: entry.key, - access: Arc::clone(&entry.access), - content_type: entry.content_type, - log_context: CacheObjectLogContext { - lease: safe_hls_access_lease_id(hls_access_lease_id), - identity: HlsLogIdentity::from_session(&session), - resource_id: resource_file.resource_id.0.clone(), - object_kind: transient_body_object_kind(resource_kind, &resource_file.extension), - body_source: "transient", - }, - repair_context: Some(HlsSegmentRepairObjectContext { - source: HlsSegmentRepairSource::Transient, - log_identity: HlsLogIdentity::from_session(&session), - proxy_session_id, - hls_access_lease_id: Some(hls_access_lease_id.clone()), - rendered_object_id: HlsRepairRenderedObjectId::Transient { - resource_id: resource_file.resource_id.0.clone(), - }, - resource_id: resource_file.resource_id.0.clone(), - file_ext: resource_file.extension.clone(), - origin_fetch_uri_for_diagnostics: resource_file.resource_id.0.clone(), - media_sequence: None, - discontinuity_sequence: None, - complete_object: true, - encrypted: resource_state.1 || resource_state.0 == TransientResourceKind::Key, - custom_response: false, - }), - }) -} - -fn empty_ok_response(content_type: &str, cache_duration_seconds: u64) -> Response { - let mut response = Response::new(Body::empty()); - *response.status_mut() = StatusCode::OK; - let headers = response.headers_mut(); - insert_header_value(headers, header::CONTENT_TYPE, content_type); - headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static(ACCEPT_RANGES_VALUE)); - headers.insert(header::CONTENT_LENGTH, HeaderValue::from_static("0")); - insert_cache_control(headers, cache_duration_seconds); - mark_response_as_uncompressed(&mut response); - response -} - -fn range_not_satisfiable_response(full_size: u64) -> Response { - let mut response = Response::new(Body::empty()); - *response.status_mut() = StatusCode::RANGE_NOT_SATISFIABLE; - let headers = response.headers_mut(); - headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static(ACCEPT_RANGES_VALUE)); - headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); - headers.insert(header::CONTENT_LENGTH, HeaderValue::from_static("0")); - insert_header_value(headers, header::CONTENT_RANGE, &format!("bytes */{full_size}")); - mark_response_as_uncompressed(&mut response); - response -} - -fn service_unavailable_not_ready_response(retry_after_ms: u64) -> Response { - let mut response = Response::new(Body::empty()); - *response.status_mut() = StatusCode::SERVICE_UNAVAILABLE; - let headers = response.headers_mut(); - insert_header_value(headers, header::RETRY_AFTER, &super::retry_after_secs_from_ms(retry_after_ms).to_string()); - headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); - mark_response_as_uncompressed(&mut response); - response -} - -fn insert_cache_control(headers: &mut axum::http::HeaderMap, cache_duration_seconds: u64) { - insert_header_value( - headers, - header::CACHE_CONTROL, - &format!("public, max-age={cache_duration_seconds}, immutable"), - ); -} - -fn insert_u64_header(headers: &mut axum::http::HeaderMap, name: header::HeaderName, value: u64) { - insert_header_value(headers, name, &value.to_string()); -} - -fn insert_header_value(headers: &mut axum::http::HeaderMap, name: header::HeaderName, value: &str) { - if let Ok(value) = HeaderValue::from_str(value) { - headers.insert(name, value); - } -} - struct ActiveReaderStream { inner: Pin> + Send>>, _guard: Option, @@ -1093,868 +131,42 @@ struct PreparedBytesStream { remaining: Bytes, } -impl PreparedBytesStream { - fn new(bytes: Bytes) -> Self { Self { remaining: bytes } } -} - -impl Stream for PreparedBytesStream { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - if self.remaining.is_empty() { - return Poll::Ready(None); - } - let chunk_len = self.remaining.len().min(PREPARED_MEDIA_CHUNK_SIZE); - Poll::Ready(Some(Ok(self.remaining.split_to(chunk_len)))) - } -} - -impl ActiveReaderStream { - fn new( - inner: Pin> + Send>>, - guard: Option, - context: CacheBodyLogContext, - meter: Arc>, - media_activity_marker: Option, - startup_body_observation: Option, - ) -> Self { - Self { - inner, - _guard: guard, - context, - started_at: Instant::now(), - last_yield_at: Instant::now(), - send_deadline: Box::pin(sleep(hls_client_body_send_deadline())), - max_idle_ms: 0, - completed_logged: false, - finished: false, - bytes_yielded: 0, - meter, - media_activity_marker, - startup_body_observation, - } - } - - fn log_completed(&mut self, outcome: &'static str) { - if self.completed_logged { - return; - } - self.completed_logged = true; - debug!( - "{} '{}' body completed: body_id={} session={} proxy_session={} source={} elapsed_s={:.3} idle_max_s={:.3} bytes={}/{} outcome={}", - self.context.object_kind, - self.context.resource_id, - self.context.body_id, - self.context.identity.session(), - self.context.identity.proxy_session(), - self.context.source, - duration_secs(self.started_at.elapsed().as_millis()), - duration_secs(self.max_idle_ms), - self.bytes_yielded, - self.context.content_length, - outcome - ); - if let Some(observation) = &self.startup_body_observation { - observation.finish(current_time_millis(), outcome); - } - if self.bytes_yielded >= self.context.content_length { - if let Some(marker) = &self.media_activity_marker { - marker.spawn_mark_completion_now(); - } - } - } -} - -impl Stream for ActiveReaderStream { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if self.finished { - return Poll::Ready(None); - } - if self.send_deadline.as_mut().poll(cx).is_ready() { - self.finished = true; - self.log_completed("timeout"); - return Poll::Ready(Some(Err(io::Error::new(io::ErrorKind::TimedOut, "hls client body send timed out")))); - } - match self.inner.as_mut().poll_next(cx) { - Poll::Ready(Some(Ok(chunk))) => { - let first_chunk = self.bytes_yielded == 0; - refresh_hls_client_body_send_deadline(self.send_deadline.as_mut()); - let idle_ms = self.last_yield_at.elapsed().as_millis(); - self.max_idle_ms = self.max_idle_ms.max(idle_ms); - self.last_yield_at = Instant::now(); - self.bytes_yielded = self.bytes_yielded.saturating_add(chunk.len() as u64); - if let Some(meter) = self.meter.load_full() { - meter.record_bytes(chunk.len() as u64); - } - if first_chunk - && self - .startup_body_observation - .as_ref() - .is_some_and(|observation| observation.record_first_chunk(current_time_millis())) - { - debug!( - "HLS cache body first chunk: body_id={} session={} proxy_session={} resource={} bytes={}", - self.context.body_id, - self.context.identity.session(), - self.context.identity.proxy_session(), - self.context.resource_id, - chunk.len() - ); - } - Poll::Ready(Some(Ok(chunk))) - } - Poll::Ready(Some(Err(err))) => { - self.finished = true; - self.log_completed("error"); - Poll::Ready(Some(Err(err))) - } - Poll::Ready(None) => { - self.finished = true; - self.log_completed("ok"); - Poll::Ready(None) - } - Poll::Pending => Poll::Pending, - } - } -} - -impl Drop for ActiveReaderStream { - fn drop(&mut self) { - let outcome = if self.bytes_yielded >= self.context.content_length { "ok" } else { "drop" }; - self.log_completed(outcome); - } -} - struct CacheReadGuard { access: Arc, } -impl CacheReadGuard { - fn new(access: Arc, now_ms: u64) -> Self { - access.reader_started(now_ms); - Self { access } - } +struct RevisionResponseStream { + inner: Pin> + Send>>, + qos_meter: Arc>, + /// Marks the segment completed once the body ends without an error. + completion: Option, + observation: Option, + first_chunk: bool, + finished: bool, + idle_deadline: Pin>, } -impl Drop for CacheReadGuard { - fn drop(&mut self) { self.access.reader_finished(); } -} - -fn log_body_reader_wait_if_slow(context: &CacheObjectLogContext, wait_for: &'static str, elapsed_ms: u128) { - if elapsed_ms < BODY_READER_WAIT_LOG_THRESHOLD_MS { - return; - } - debug!( - "HLS cache reader wait: lease={} session={} proxy_session={} resource={} wait_for={} elapsed_ms={}", - context.lease, - context.identity.session(), - context.identity.proxy_session(), - context.resource_id, - wait_for, - elapsed_ms - ); -} - -fn duration_secs(elapsed_ms: u128) -> f64 { - Duration::from_millis(u64::try_from(elapsed_ms).unwrap_or(u64::MAX)).as_secs_f64() -} - -fn transient_body_object_kind(resource_kind: Option, extension: &str) -> &'static str { - match resource_kind { - Some(TransientResourceKind::Key) => "Key", - Some(TransientResourceKind::Map) => "Map", - Some(TransientResourceKind::Segment | TransientResourceKind::Part | TransientResourceKind::Other) => "Segment", - None => { - if extension.eq_ignore_ascii_case("key") { - "Key" - } else { - "Segment" - } - } - } -} - -fn next_hls_body_log_id() -> String { - let value = NEXT_HLS_BODY_LOG_ID.fetch_add(1, Ordering::Relaxed); - format!("{value:08x}") -} - -use tuliprox_core::utils::current_time_millis; - #[cfg(test)] -mod tests { - use super::{ - current_time_millis, finite_hls_media_response, serve_cache_object, serve_hls_segment_cache_outcome, - spawn_bounded_media_completion, transient_body_object_kind, ActiveReaderStream, CacheBodyLogContext, - CacheObject, CacheObjectLogContext, CacheObjectServeContext, HlsCacheResponseContext, HlsMediaActivityMarker, - HlsPlaybackCursorTracking, HlsResourceServeFailure, HlsResourceServeOutcome, PREPARED_MEDIA_CHUNK_SIZE, - }; - use crate::{ - media_reserve::HlsLeasePlaybackCursor, CacheAccessState, HlsAccessLease, HlsAccessLeaseId, HlsLogIdentity, - HlsPlaybackFamilyKey, HlsProxyManager, HlsSegmentCache, HlsSegmentFile, HlsSegmentRepairManager, - HlsSessionHandle, HlsSessionKey, OriginSegmentKey, ProxySessionId, SegmentCacheKey, SegmentCacheStatus, - SegmentEntry, TransientResourceKind, - }; - use tuliprox_core::model::HlsSegmentRepairConfig; - use tuliprox_session::StreamMeterHandle; +mod tests; - fn test_log_identity() -> HlsLogIdentity { HlsLogIdentity::for_test("content-session", "proxy-session") } - use arc_swap::ArcSwapOption; - use axum::http::{header, HeaderValue, StatusCode}; - use bytes::Bytes; - use futures::StreamExt; - use http_body_util::BodyExt; - use shared::model::HlsSegmentRepairMode; - use std::{ - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, Weak, - }, - time::Duration, - }; - use tokio::{sync::Semaphore, time::advance}; +mod activity; +mod cache; +mod finite; +mod reader; +mod revision; - fn header(value: &str) -> HeaderValue { HeaderValue::from_str(value).expect("valid header") } - - fn test_segment_repair_manager() -> Arc { - Arc::new(HlsSegmentRepairManager::new(HlsSegmentRepairConfig { - max_level: HlsSegmentRepairMode::Off, - apply_to_first_segments: 1, - max_parallel_repairs: 1, - ..Default::default() - })) - } - - #[tokio::test] - async fn media_completion_scheduling_does_not_block_when_completion_capacity_is_busy() { - let permits = Arc::new(Semaphore::new(1)); - let held = Arc::clone(&permits).acquire_owned().await.expect("initial permit"); - let completed = Arc::new(AtomicBool::new(false)); - let completed_for_task = Arc::clone(&completed); - spawn_bounded_media_completion(Arc::clone(&permits), async move { - completed_for_task.store(true, Ordering::Release); - }); - - tokio::task::yield_now().await; - assert!(!completed.load(Ordering::Acquire)); - drop(held); - - for _ in 0..16 { - if completed.load(Ordering::Acquire) { - break; - } - tokio::task::yield_now().await; - } - assert!(completed.load(Ordering::Acquire)); - assert_eq!(permits.available_permits(), 1); - } - - async fn live_media_marker( - manager: &Arc, - session: &HlsSessionHandle, - lease_id: HlsAccessLeaseId, - ) -> HlsMediaActivityMarker { - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let lease = HlsAccessLease::pending( - lease_id.clone(), - HlsPlaybackFamilyKey::new("test-user", "test-fingerprint"), - proxy_session_id.clone(), - "test-user".to_string(), - "test-session".to_string(), - 1, - "stream".to_string(), - 1, - current_time_millis(), - 60_000, - ); - let lease_identity = lease.media_identity().expect("live lease identity"); - manager.prepare_access_lease(lease).await; - HlsMediaActivityMarker::new( - Arc::clone(manager), - Arc::clone(session), - proxy_session_id, - lease_id, - lease_identity, - ) - } - - struct CachedSegmentFixture { - _temp_dir: tempfile::TempDir, - manager: Arc, - segment_cache: Arc, - session: HlsSessionHandle, - proxy_session_id: ProxySessionId, - lease_id: HlsAccessLeaseId, - segment_file: HlsSegmentFile, - access: Arc, - context: HlsCacheResponseContext, - meter: Arc, - full_size: u64, - } - - impl CachedSegmentFixture { - async fn new(bytes: Vec) -> Self { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let manager = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let segment_cache = Arc::clone(manager.segment_cache()); - let session = manager.get_or_create_session(HlsSessionKey::new(1, "range-stream"), b"secret", 1_000).await; - let proxy_session_id = session.read().await.proxy_session_id.clone(); - let proxy_seq = 12; - let cache_key = SegmentCacheKey::new(proxy_session_id.clone(), proxy_seq, "ts"); - segment_cache.write_bytes_and_commit(&cache_key, &bytes).await.expect("segment cache commit"); - let full_size = u64::try_from(bytes.len()).expect("test segment size"); - let access = Arc::new(CacheAccessState::new()); - session.write().await.segments.insert( - proxy_seq, - SegmentEntry { - origin_key: OriginSegmentKey { - origin_epoch: 1, - effective_host_id: 1, - host_local_sequence: proxy_seq, - host_local_index: u32::try_from(proxy_seq).expect("test proxy sequence"), - }, - proxy_seq, - duration_ms: 4_000, - proxy_file_ext: "ts".to_string(), - content_type: "video/mp2t".to_string(), - cache_key, - discontinuity_before: false, - program_date_time: None, - daterange_tags_before: Vec::new(), - origin_byte_range: None, - map_ref: None, - encryption: None, - origin_fetch_ref: None, - status: SegmentCacheStatus::Ready { content_length: full_size, ready_at_ms: 1_000 }, - last_rendered_at_ms: Some(1_000), - access: Arc::clone(&access), - }, - ); - let lease_id = HlsAccessLeaseId("range-lease".to_string()); - let marker = live_media_marker(&manager, &session, lease_id.clone()).await; - let meter = Arc::new(StreamMeterHandle::new(7, Weak::new())); - let log_identity = { - let session = session.read().await; - HlsLogIdentity::from_session(&session) - }; - let context = HlsCacheResponseContext::new( - lease_id.clone(), - log_identity, - 300, - Arc::clone(manager.metrics()), - Arc::clone(manager.segment_repair()), - Some(Arc::clone(&meter)), - Some(marker), - current_time_millis(), - ); - Self { - _temp_dir: temp_dir, - manager, - segment_cache, - session, - proxy_session_id, - lease_id, - segment_file: HlsSegmentFile { proxy_seq, extension: "ts".to_string() }, - access, - context, - meter, - full_size, - } - } - - async fn serve(&self, range: Option<&str>) -> HlsResourceServeOutcome { - serve_hls_segment_cache_outcome( - Arc::clone(&self.segment_cache), - Arc::clone(&self.session), - self.segment_file.clone(), - range.map(header), - &self.context, - ) - .await - } - - async fn cursor(&self) -> HlsLeasePlaybackCursor { - self.manager - .access_lease_response_snapshot(&self.lease_id, &self.proxy_session_id, current_time_millis()) - .await - .expect("range test lease") - .playback_cursor - } - - async fn wait_for_completion(&self) -> HlsLeasePlaybackCursor { - for _ in 0..64 { - let cursor = self.cursor().await; - if cursor.highest_contiguous_completed_proxy_seq == Some(self.segment_file.proxy_seq) { - return cursor; - } - tokio::task::yield_now().await; - } - panic!("segment completion task did not commit") - } - } - - fn ready_response(outcome: HlsResourceServeOutcome) -> axum::response::Response { - match outcome { - HlsResourceServeOutcome::Ready(response) => response, - HlsResourceServeOutcome::Failure(failure) => panic!("expected ready response, got {failure:?}"), - } - } - - #[tokio::test] - async fn finite_terminal_media_supports_full_range_and_unsatisfiable_responses() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let manager = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let context = HlsCacheResponseContext::new( - HlsAccessLeaseId("terminal-lease".to_string()), - test_log_identity(), - 300, - Arc::clone(manager.metrics()), - Arc::clone(manager.segment_repair()), - None, - None, - 1_000, - ); - let proxy_session_id = ProxySessionId("terminal-session".to_string()); - let full = finite_hls_media_response( - Bytes::from_static(b"0123456789"), - None, - "video/mp2t", - "private, immutable", - &context, - &proxy_session_id, - "terminal/1/0".to_string(), - ); - assert_eq!(full.status(), StatusCode::OK); - assert!(!tuliprox_core::utils::response_compression::should_compress_response(&full)); - assert_eq!(full.headers()[header::CONTENT_LENGTH], "10"); - assert_eq!(full.into_body().collect().await.expect("body").to_bytes(), Bytes::from_static(b"0123456789")); - - let range_header = header("bytes=2-5"); - let partial = finite_hls_media_response( - Bytes::from_static(b"0123456789"), - Some(&range_header), - "video/mp2t", - "private, immutable", - &context, - &proxy_session_id, - "terminal/1/0".to_string(), - ); - assert_eq!(partial.status(), StatusCode::PARTIAL_CONTENT); - assert!(!tuliprox_core::utils::response_compression::should_compress_response(&partial)); - assert_eq!(partial.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); - assert_eq!(partial.into_body().collect().await.expect("body").to_bytes(), Bytes::from_static(b"2345")); - - let invalid_header = header("bytes=20-"); - let invalid = finite_hls_media_response( - Bytes::from_static(b"0123456789"), - Some(&invalid_header), - "video/mp2t", - "private, immutable", - &context, - &proxy_session_id, - "terminal/1/0".to_string(), - ); - assert_eq!(invalid.status(), StatusCode::RANGE_NOT_SATISFIABLE); - assert!(!tuliprox_core::utils::response_compression::should_compress_response(&invalid)); - assert_eq!(invalid.headers()[header::CONTENT_RANGE], "bytes */10"); - } - - #[tokio::test] - async fn finite_terminal_media_uses_qos_completion_and_drop_semantics() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let manager = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let session = manager.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 1_000).await; - let lease_id = HlsAccessLeaseId("terminal-lease".to_string()); - let marker = live_media_marker(&manager, &session, lease_id.clone()).await; - let meter = Arc::new(StreamMeterHandle::new(7, Weak::new())); - let context = HlsCacheResponseContext::new( - lease_id, - test_log_identity(), - 300, - Arc::clone(manager.metrics()), - Arc::clone(manager.segment_repair()), - Some(Arc::clone(&meter)), - Some(marker), - 1_000, - ); - let proxy_session_id = ProxySessionId("terminal-session".to_string()); - - let dropped = finite_hls_media_response( - Bytes::from_static(b"0123456789"), - None, - "video/mp2t", - "private, immutable", - &context, - &proxy_session_id, - "terminal/1/0".to_string(), - ); - drop(dropped); - tokio::task::yield_now().await; - assert_eq!(meter.bytes_total(), 0); - assert_eq!(session.read().await.activity.last_authorized_media_at_ms, None); - - let partially_consumed = finite_hls_media_response( - Bytes::from(vec![7_u8; PREPARED_MEDIA_CHUNK_SIZE.saturating_mul(2)]), - None, - "video/mp2t", - "private, immutable", - &context, - &proxy_session_id, - "terminal/1/0".to_string(), - ); - let mut partial_body = partially_consumed.into_body(); - let partial_chunk = partial_body - .frame() - .await - .expect("first frame") - .expect("first frame body") - .into_data() - .expect("first data frame"); - drop(partial_body); - tokio::task::yield_now().await; - assert_eq!(partial_chunk.len(), PREPARED_MEDIA_CHUNK_SIZE); - assert_eq!(meter.bytes_total(), PREPARED_MEDIA_CHUNK_SIZE as u64); - assert_eq!(session.read().await.activity.last_authorized_media_at_ms, None); - - context.mark_media_activity().await; - let range = header("bytes=2-5"); - let completed = finite_hls_media_response( - Bytes::from_static(b"0123456789"), - Some(&range), - "video/mp2t", - "private, immutable", - &context, - &proxy_session_id, - "terminal/1/0".to_string(), - ); - assert_eq!(completed.status(), StatusCode::PARTIAL_CONTENT); - assert!(!tuliprox_core::utils::response_compression::should_compress_response(&completed)); - assert_eq!(completed.into_body().collect().await.expect("body").to_bytes(), Bytes::from_static(b"2345")); - assert_eq!(meter.bytes_total(), PREPARED_MEDIA_CHUNK_SIZE as u64 + 4); - assert_eq!(session.read().await.activity.last_authorized_media_at_ms, Some(1_000)); - } - - #[tokio::test(start_paused = true)] - async fn finite_terminal_media_stream_honors_shared_send_deadline() { - let stream = ActiveReaderStream::new( - Box::pin(futures::stream::pending()), - None, - CacheBodyLogContext { - body_id: "00000001".to_string(), - identity: test_log_identity(), - resource_id: "terminal/1/0".to_string(), - object_kind: "TerminalSegment", - source: "prepared", - content_length: 1, - }, - Arc::new(ArcSwapOption::from(None::>)), - None, - None, - ); - let task = tokio::spawn(async move { - let mut stream = stream; - stream.next().await - }); - tokio::task::yield_now().await; - advance(super::hls_client_body_send_deadline().saturating_add(Duration::from_millis(1))).await; - - let result = task.await.expect("deadline task").expect("deadline result"); - assert_eq!(result.expect_err("deadline error").kind(), std::io::ErrorKind::TimedOut); - } - - #[test] - fn transient_body_log_kind_uses_resource_kind() { - assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Key), "bin"), "Key"); - assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Map), "bin"), "Map"); - assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Segment), "key"), "Segment"); - assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Part), "m4s"), "Segment"); - assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Other), "bin"), "Segment"); - } - - #[test] - fn transient_body_log_kind_falls_back_to_key_extension() { - assert_eq!(transient_body_object_kind(None, "key"), "Key"); - assert_eq!(transient_body_object_kind(None, "KEY"), "Key"); - assert_eq!(transient_body_object_kind(None, "ts"), "Segment"); - } - - #[test] - fn temporary_unavailable_response_uses_concrete_retry_after() { - let response = - super::hls_resource_failure_default_response(super::HlsResourceServeFailure::TemporaryUnavailable { - retry_after_ms: 2_500, - }); - - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!(response.headers().get(header::RETRY_AFTER).expect("retry-after"), "3"); - } - - #[tokio::test] - async fn cache_hit_bodies_use_independent_readers_for_same_object() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let key = SegmentCacheKey::new(ProxySessionId("proxy-session".to_string()), 12, "ts"); - segment_cache.write_bytes_and_commit(&key, b"0123456789").await.expect("commit should succeed"); - let access = Arc::new(CacheAccessState::new()); - let object = CacheObject { - is_media: true, - key, - access: Arc::clone(&access), - content_type: "video/mp2t".to_string(), - log_context: CacheObjectLogContext { - lease: "lease-a".to_string(), - identity: test_log_identity(), - resource_id: "000012".to_string(), - object_kind: "Segment", - body_source: "normal", - }, - repair_context: None, - }; - - let first = serve_cache_object( - Arc::clone(&segment_cache), - object.clone(), - Some(header("bytes=0-")), - CacheObjectServeContext { - cache_duration_seconds: 300, - metrics: None, - segment_repair: test_segment_repair_manager(), - qos_meter: Arc::new(ArcSwapOption::from(None::>)), - media_activity_marker: None, - playback_cursor_tracking: HlsPlaybackCursorTracking::Disabled, - now_ms: 1, - }, - ) - .await - .expect("first cache object response"); - let second = serve_cache_object( - segment_cache, - CacheObject { - is_media: true, - log_context: CacheObjectLogContext { - lease: "lease-b".to_string(), - identity: test_log_identity(), - resource_id: "000012".to_string(), - object_kind: "Segment", - body_source: "normal", - }, - ..object - }, - Some(header("bytes=0-")), - CacheObjectServeContext { - cache_duration_seconds: 300, - metrics: None, - segment_repair: test_segment_repair_manager(), - qos_meter: Arc::new(ArcSwapOption::from(None::>)), - media_activity_marker: None, - playback_cursor_tracking: HlsPlaybackCursorTracking::Disabled, - now_ms: 2, - }, - ) - .await - .expect("second cache object response"); - - assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(access.active_readers(), 2); - - let (first_body, second_body) = tokio::join!(first.into_body().collect(), second.into_body().collect(),); - - assert_eq!(first_body.expect("first body").to_bytes(), Bytes::from_static(b"0123456789")); - assert_eq!(second_body.expect("second body").to_bytes(), Bytes::from_static(b"0123456789")); - assert_eq!(access.active_readers(), 0); - assert_eq!(access.last_accessed_at_ms(), 2); - } - - #[tokio::test] - async fn full_object_segment_ranges_record_request_and_completion_without_changing_http_range_status() { - const FULL_SIZE: usize = 512; - for (range, expected_status) in [ - (None, StatusCode::OK), - (Some("bytes=0-"), StatusCode::PARTIAL_CONTENT), - (Some("bytes=0-511"), StatusCode::PARTIAL_CONTENT), - (Some("bytes=-512"), StatusCode::PARTIAL_CONTENT), - ] { - let expected_body = vec![7_u8; FULL_SIZE]; - let fixture = CachedSegmentFixture::new(expected_body.clone()).await; - let response = ready_response(fixture.serve(range).await); - - assert_eq!(response.status(), expected_status, "unexpected status for {range:?}"); - assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); - assert!(!tuliprox_core::utils::response_compression::should_compress_response(&response)); - assert_eq!(response.headers()[header::CONTENT_LENGTH], FULL_SIZE.to_string()); - if range.is_some() { - assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 0-511/512"); - } else { - assert!(!response.headers().contains_key(header::CONTENT_RANGE)); - } - assert_eq!(fixture.access.active_readers(), 1); - let requested = fixture.cursor().await; - assert_eq!(requested.first_requested_proxy_seq, Some(12), "missing request for {range:?}"); - assert_eq!(requested.last_requested_proxy_seq, Some(12), "missing request for {range:?}"); - assert_eq!(requested.highest_contiguous_completed_proxy_seq, None); - assert!(fixture.session.read().await.activity.last_authorized_media_at_ms.is_some()); - - assert_eq!( - response.into_body().collect().await.expect("full range body").to_bytes(), - Bytes::from(expected_body) - ); - let completed = fixture.wait_for_completion().await; - assert_eq!(completed.highest_contiguous_completed_proxy_seq, Some(12)); - assert!(completed.first_segment_completed_at_ms.is_some()); - assert_eq!(fixture.meter.bytes_total(), fixture.full_size); - assert_eq!(fixture.access.active_readers(), 0); - } - } - - #[tokio::test] - async fn partial_segment_ranges_mark_activity_without_mutating_playback_cursor() { - const FULL_SIZE: usize = 512; - for (range, expected_start, expected_end) in [("bytes=1-", 1, 511), ("bytes=0-187", 0, 187)] { - let bytes = (0..FULL_SIZE).map(|value| u8::try_from(value % 251).expect("test byte")).collect::>(); - let fixture = CachedSegmentFixture::new(bytes.clone()).await; - let response = ready_response(fixture.serve(Some(range)).await); - let expected_body = &bytes[expected_start..=expected_end]; - - assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); - assert!(!tuliprox_core::utils::response_compression::should_compress_response(&response)); - assert_eq!( - response.headers()[header::CONTENT_RANGE], - format!("bytes {expected_start}-{expected_end}/{FULL_SIZE}") - ); - assert_eq!(response.headers()[header::CONTENT_LENGTH], expected_body.len().to_string()); - assert!(fixture.session.read().await.activity.last_authorized_media_at_ms.is_some()); - assert_eq!(fixture.cursor().await, HlsLeasePlaybackCursor::default()); - - assert_eq!( - response.into_body().collect().await.expect("partial range body").to_bytes(), - Bytes::copy_from_slice(expected_body) - ); - for _ in 0..8 { - tokio::task::yield_now().await; - } - assert_eq!(fixture.cursor().await, HlsLeasePlaybackCursor::default()); - assert_eq!(fixture.meter.bytes_total(), u64::try_from(expected_body.len()).expect("partial body size")); - assert_eq!(fixture.access.active_readers(), 0); - } - } - - #[tokio::test] - async fn dropped_full_range_segment_body_records_request_without_completion() { - const FULL_SIZE: usize = 512 * 1_024; - let fixture = CachedSegmentFixture::new(vec![9_u8; FULL_SIZE]).await; - let response = ready_response(fixture.serve(Some("bytes=0-")).await); - - assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response.headers()[header::CONTENT_RANGE], format!("bytes 0-{}/{FULL_SIZE}", FULL_SIZE - 1)); - let requested = fixture.cursor().await; - assert_eq!(requested.first_requested_proxy_seq, Some(12)); - assert_eq!(requested.last_requested_proxy_seq, Some(12)); - assert_eq!(requested.highest_contiguous_completed_proxy_seq, None); - assert!(fixture.session.read().await.activity.last_authorized_media_at_ms.is_some()); - - let mut body = response.into_body(); - let first_chunk = body - .frame() - .await - .expect("first segment frame") - .expect("first segment frame body") - .into_data() - .expect("first segment data frame"); - assert!(first_chunk.len() < FULL_SIZE); - drop(body); - for _ in 0..8 { - tokio::task::yield_now().await; - } - - let dropped = fixture.cursor().await; - assert_eq!(dropped.first_requested_proxy_seq, Some(12)); - assert_eq!(dropped.last_requested_proxy_seq, Some(12)); - assert_eq!(dropped.highest_contiguous_completed_proxy_seq, None); - assert_eq!(dropped.first_segment_completed_at_ms, None); - assert_eq!(fixture.meter.bytes_total(), u64::try_from(first_chunk.len()).expect("first chunk size")); - assert_eq!(fixture.access.active_readers(), 0); - } - - #[tokio::test] - async fn unsatisfiable_or_missing_segment_records_neither_activity_nor_cursor() { - const FULL_SIZE: usize = 512; - let fixture = CachedSegmentFixture::new(vec![5_u8; FULL_SIZE]).await; - let unsatisfiable = ready_response(fixture.serve(Some("bytes=512-")).await); - - assert_eq!(unsatisfiable.status(), StatusCode::RANGE_NOT_SATISFIABLE); - assert_eq!(unsatisfiable.headers()[header::CONTENT_RANGE], "bytes */512"); - assert_eq!(fixture.session.read().await.activity.last_authorized_media_at_ms, None); - assert_eq!(fixture.cursor().await, HlsLeasePlaybackCursor::default()); - assert_eq!(fixture.meter.bytes_total(), 0); - assert_eq!(fixture.access.active_readers(), 0); - - let missing = serve_hls_segment_cache_outcome( - Arc::clone(&fixture.segment_cache), - Arc::clone(&fixture.session), - HlsSegmentFile { proxy_seq: 99, extension: "ts".to_string() }, - Some(header("bytes=0-")), - &fixture.context, - ) - .await; - assert!(matches!(missing, HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing))); - assert_eq!(fixture.session.read().await.activity.last_authorized_media_at_ms, None); - assert_eq!(fixture.cursor().await, HlsLeasePlaybackCursor::default()); - assert_eq!(fixture.meter.bytes_total(), 0); - } - - #[tokio::test] - async fn cache_body_marks_media_activity_at_start_and_body_end() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); - let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let key = SegmentCacheKey::new(ProxySessionId("proxy-session".to_string()), 12, "ts"); - segment_cache.write_bytes_and_commit(&key, b"0123456789").await.expect("commit should succeed"); - let access = Arc::new(CacheAccessState::new()); - let session = hls_proxy.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 1_000).await; - let marker = live_media_marker(&hls_proxy, &session, HlsAccessLeaseId("lease-a".to_string())).await; - - let response = serve_cache_object( - segment_cache, - CacheObject { - is_media: true, - key, - access, - content_type: "video/mp2t".to_string(), - log_context: CacheObjectLogContext { - lease: "lease-a".to_string(), - identity: test_log_identity(), - resource_id: "000012".to_string(), - object_kind: "Segment", - body_source: "normal", - }, - repair_context: None, - }, - Some(header("bytes=0-")), - CacheObjectServeContext { - cache_duration_seconds: 300, - metrics: None, - segment_repair: test_segment_repair_manager(), - qos_meter: Arc::new(ArcSwapOption::from(None::>)), - media_activity_marker: Some(marker), - playback_cursor_tracking: HlsPlaybackCursorTracking::Disabled, - now_ms: 1_000, - }, - ) - .await - .expect("cache object response"); - - assert_eq!(session.read().await.activity.last_authorized_media_at_ms, Some(1_000)); - - assert_eq!(response.into_body().collect().await.expect("body").to_bytes(), Bytes::from_static(b"0123456789")); - tokio::time::sleep(Duration::from_millis(10)).await; - - assert!( - session.read().await.activity.last_authorized_media_at_ms.expect("media activity") >= 1_000, - "body completion should not move media activity backwards" - ); - } -} +#[cfg(test)] +use activity::spawn_bounded_media_completion; +use cache::HlsPlaybackCursorTracking; +#[cfg(test)] +use cache::{hls_resource_failure_default_response, serve_cache_object}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use cache::{ + serve_hls_map_cache_outcome, serve_hls_segment_cache_outcome, serve_hls_transient_object_cache_outcome, + serve_hls_transient_object_cache_response, HlsResourceServeFailure, HlsResourceServeOutcome, +}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use finite::{ + finite_hls_immutable_media_response, finite_hls_media_head_response, finite_hls_media_response, + finite_hls_terminal_key_response, +}; diff --git a/backend/hls/src/response/activity.rs b/backend/hls/src/response/activity.rs new file mode 100644 index 000000000..8a79a1b3d --- /dev/null +++ b/backend/hls/src/response/activity.rs @@ -0,0 +1,209 @@ +use super::{ + safe_hls_access_lease_id, safe_proxy_session_id, HlsAccessLeaseId, HlsMediaActivityCommitOutcome, + HlsMediaActivityMarker, HlsMediaLeaseIdentity, HlsProxyManager, HlsSessionHandle, HlsStartupBodyObservation, + ProxySessionId, HLS_MEDIA_ACTIVITY_TASK_PERMITS, +}; +use axum::{body::Body, http::Response}; +use futures::StreamExt; +use log::debug; +use std::{ + future::Future, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; +use tokio::sync::Semaphore; +use tuliprox_core::utils::current_time_millis; + +impl HlsMediaActivityMarker { + /// Confirms only a polled, non-empty media body. Bytes retain their shared + /// backing storage; no payload copy or detached per-chunk task is needed. + pub fn confirm_media_response(&self, response: Response) -> Response { + let (parts, body) = response.into_parts(); + let mut marker = Some(self.clone()); + let stream = body.into_data_stream().then(move |chunk| { + let confirmation = if chunk.as_ref().is_ok_and(|bytes| !bytes.is_empty()) { marker.take() } else { None }; + async move { + if let Some(marker) = confirmation { + let outcome = marker + .manager + .mark_delivered_media_for_lease( + &marker.session, + &marker.lease_id, + &marker.proxy_session_id, + marker.lease_identity, + current_time_millis(), + ) + .await; + if matches!(outcome, HlsMediaActivityCommitOutcome::Committed) { + if let (Some(active_provider), Some(owner), Some(request_id)) = ( + marker.active_provider.as_ref(), + marker.session_owner.as_deref(), + marker.playback_request_id, + ) { + active_provider.confirm_identified_playback_activity(owner, request_id); + } + } + marker.log_uncommitted_activity(outcome, "media-delivery"); + } + chunk + } + }); + Response::from_parts(parts, Body::from_stream(stream)) + } + pub fn new( + manager: Arc, + session: HlsSessionHandle, + proxy_session_id: ProxySessionId, + lease_id: HlsAccessLeaseId, + lease_identity: HlsMediaLeaseIdentity, + ) -> Self { + Self { + manager, + session, + proxy_session_id, + lease_id, + lease_identity, + completed_segment: None, + completion_scheduled: None, + active_provider: None, + session_owner: None, + playback_request_id: None, + } + } + + pub fn with_active_provider( + mut self, + active_provider: Arc, + session_owner: Option, + playback_request_id: Option, + ) -> Self { + self.active_provider = Some(active_provider); + self.session_owner = session_owner; + self.playback_request_id = playback_request_id; + self + } + + pub(super) async fn for_segment_request(mut self, proxy_seq: u64, requested_at_ms: u64) -> Option { + // A terminal lease may still read the immutable READY live-tail segment + // protected by its plan. That read marks terminal activity, but it must + // never mutate the live playback cursor. + if !self.lease_identity.is_live() { + return Some(self); + } + self.completed_segment = self + .manager + .record_access_lease_segment_request_started_if_identity_matches( + &self.lease_id, + &self.proxy_session_id, + self.lease_identity, + proxy_seq, + requested_at_ms, + ) + .await; + let _ = self.completed_segment.as_ref()?; + self.completion_scheduled = Some(Arc::new(AtomicBool::new(false))); + Some(self) + } + + pub async fn mark_at(&self, now_ms: u64) { + let outcome = self + .manager + .mark_authorized_media_access_for_lease_if_identity_matches( + &self.session, + &self.lease_id, + &self.proxy_session_id, + self.lease_identity, + now_ms, + ) + .await; + self.log_uncommitted_activity(outcome, "access"); + } + + pub(super) async fn mark_completion_at(&self, now_ms: u64) { + let Some(token) = self.completed_segment else { + self.mark_at(now_ms).await; + return; + }; + let outcome = self + .manager + .record_access_lease_segment_request_completed_and_mark_media_if_identity_matches( + &self.session, + &self.lease_id, + &self.proxy_session_id, + self.lease_identity, + token, + now_ms, + ) + .await; + self.log_uncommitted_activity(outcome, "live-segment-completion"); + } + + pub(super) fn spawn_mark_completion_now(&self) { + let Some(scheduled) = self.completion_scheduled.as_ref() else { + debug!( + "HLS media completion ignored: lease={} proxy_session={} reason=missing-completion-token", + safe_hls_access_lease_id(&self.lease_id), + safe_proxy_session_id(&self.proxy_session_id) + ); + return; + }; + if scheduled.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire).is_err() { + debug!( + "HLS media completion ignored: lease={} proxy_session={} reason=completion-already-scheduled", + safe_hls_access_lease_id(&self.lease_id), + safe_proxy_session_id(&self.proxy_session_id) + ); + return; + } + let marker = self.clone(); + spawn_bounded_media_completion(Arc::clone(&HLS_MEDIA_ACTIVITY_TASK_PERMITS), async move { + marker.mark_completion_at(current_time_millis()).await; + }); + } + + pub(super) fn completed_segment_marker(&self) -> Option { self.completed_segment.map(|_| self.clone()) } + + pub(super) fn record_startup_segment_request(&self, proxy_seq: u64, now_ms: u64) { + self.manager.startup_observability().record_first_visible_segment_request(&self.lease_id, proxy_seq, now_ms); + } + + pub(super) fn record_startup_repair_decision(&self, proxy_seq: u64, now_ms: u64) { + self.manager.startup_observability().record_repair_decision(&self.lease_id, proxy_seq, now_ms); + } + + pub(super) fn begin_startup_cache_response( + &self, + proxy_seq: u64, + body_id: &str, + now_ms: u64, + ) -> Option { + self.manager.startup_observability().begin_cache_response(&self.lease_id, proxy_seq, body_id, now_ms) + } + + pub(super) fn log_uncommitted_activity(&self, outcome: HlsMediaActivityCommitOutcome, phase: &'static str) { + let reason = match outcome { + HlsMediaActivityCommitOutcome::Committed => return, + HlsMediaActivityCommitOutcome::StaleLeaseIdentity => "expired-or-playback-generation-race", + HlsMediaActivityCommitOutcome::DeferredLockContention => "lock-contention", + }; + debug!( + "HLS media activity ignored: phase={phase} lease={} proxy_session={} reason={reason}", + safe_hls_access_lease_id(&self.lease_id), + safe_proxy_session_id(&self.proxy_session_id) + ); + } +} + +pub(super) fn spawn_bounded_media_completion( + completion_permits: Arc, + completion: impl Future + Send + 'static, +) { + tokio::spawn(async move { + let Ok(_permit) = completion_permits.acquire_owned().await else { + return; + }; + completion.await; + }); +} diff --git a/backend/hls/src/response/cache.rs b/backend/hls/src/response/cache.rs new file mode 100644 index 000000000..916a4d176 --- /dev/null +++ b/backend/hls/src/response/cache.rs @@ -0,0 +1,576 @@ +use super::{ + reader::{ + empty_ok_response, insert_cache_control, insert_header_value, insert_u64_header, log_body_reader_wait_if_slow, + next_hls_body_log_id, range_not_satisfiable_response, service_unavailable_not_ready_response, + transient_body_object_kind, + }, + revision::serve_hls_revision_outcome, + safe_hls_access_lease_id, ActiveReaderStream, CacheBodyLogContext, CacheObject, CacheObjectLogContext, + CacheObjectServeContext, CacheReadGuard, HlsAccessLeaseId, HlsCacheMetrics, HlsCacheResponseContext, + HlsLogIdentity, HlsMapFile, HlsMediaActivityMarker, HlsRepairRenderedObjectId, HlsSegmentCache, HlsSegmentFile, + HlsSegmentRepairManager, HlsSegmentRepairObjectContext, HlsSegmentRepairSource, HlsSessionHandle, MapCacheKey, + MapCacheStatus, ProxyMapId, SegmentCacheKey, SegmentCacheStatus, TransientObjectCacheKey, TransientResourceFile, + TransientResourceKind, ACCEPT_RANGES_VALUE, NOT_READY_RETRY_AFTER_MS, +}; +use arc_swap::ArcSwapOption; +use axum::{ + body::Body, + http::{header, HeaderValue, Response, StatusCode}, + response::IntoResponse, +}; +use log::debug; +use std::{io, sync::Arc, time::Instant}; +use tokio::io::AsyncReadExt; +use tokio_util::io::ReaderStream; +use tuliprox_core::utils::{ + byte_range::{resolve_single_byte_range, SingleByteRange}, + current_time_millis, + response_compression::mark_response_as_uncompressed, +}; +use tuliprox_session::StreamMeterHandle; + +pub(super) enum CacheObjectLookup { + Ready(CacheObject), + Failure(HlsResourceServeFailure), +} + +/// Result of a Shared-HLS cache object serve decision. +pub enum HlsResourceServeOutcome { + Ready(Response), + Failure(HlsResourceServeFailure), +} + +/// Typed Shared-HLS resource failure before endpoint-level custom-response mapping. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsResourceServeFailure { + TemporaryUnavailable { retry_after_ms: u64 }, + Missing, + Expired, + PermanentFailed { status: Option }, +} + +impl HlsCacheResponseContext { + #[allow(clippy::too_many_arguments)] + pub fn new( + hls_access_lease_id: HlsAccessLeaseId, + log_identity: HlsLogIdentity, + cache_duration_seconds: u64, + metrics: Arc, + segment_repair: Arc, + qos_meter: Option>, + media_activity_marker: Option, + now_ms: u64, + ) -> Self { + Self { + hls_access_lease_id, + log_identity, + cache_duration_seconds, + metrics, + segment_repair, + qos_meter: Arc::new(ArcSwapOption::from(qos_meter)), + media_activity_marker, + now_ms, + } + } + + pub fn set_qos_meter(&self, qos_meter: Option>) { self.qos_meter.store(qos_meter); } + + pub async fn mark_media_activity(&self) { + if let Some(marker) = &self.media_activity_marker { + marker.mark_at(self.now_ms).await; + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum HlsPlaybackCursorTracking { + Disabled, + Segment { proxy_seq: u64 }, +} + +impl HlsPlaybackCursorTracking { + pub(super) fn full_object_proxy_seq(self, range: SingleByteRange, full_size: u64) -> Option { + match self { + Self::Segment { proxy_seq } if resolved_range_covers_full_object(range, full_size) => Some(proxy_seq), + Self::Disabled | Self::Segment { .. } => None, + } + } +} + +fn resolved_range_covers_full_object(range: SingleByteRange, full_size: u64) -> bool { + match range { + SingleByteRange::Full => true, + SingleByteRange::Partial { start, end, length } => { + start == 0 && full_size.checked_sub(1) == Some(end) && length == full_size + } + SingleByteRange::Unsatisfiable => false, + } +} + +impl CacheObjectServeContext { + pub(super) fn from_response_context(context: &HlsCacheResponseContext) -> Self { + Self { + cache_duration_seconds: context.cache_duration_seconds, + metrics: Some(Arc::clone(&context.metrics)), + segment_repair: Arc::clone(&context.segment_repair), + qos_meter: Arc::clone(&context.qos_meter), + media_activity_marker: context.media_activity_marker.clone(), + playback_cursor_tracking: HlsPlaybackCursorTracking::Disabled, + now_ms: context.now_ms, + } + } +} + +/// Serves a committed Ready segment or returns a typed failure for endpoint-level mapping. +pub async fn serve_hls_segment_cache_outcome( + segment_cache: Arc, + session: HlsSessionHandle, + segment_file: HlsSegmentFile, + range_header: Option, + context: &HlsCacheResponseContext, +) -> HlsResourceServeOutcome { + if session.read().await.startup.is_some() { + return serve_hls_revision_outcome(&session, &segment_file, range_header.as_ref(), context).await; + } + match lookup_segment_cache_object(&session, &segment_file, &context.hls_access_lease_id).await { + CacheObjectLookup::Ready(object) => cache_object_serve_outcome( + serve_cache_object( + segment_cache, + object, + range_header, + CacheObjectServeContext { + playback_cursor_tracking: HlsPlaybackCursorTracking::Segment { proxy_seq: segment_file.proxy_seq }, + ..CacheObjectServeContext::from_response_context(context) + }, + ) + .await, + ), + CacheObjectLookup::Failure(failure) => HlsResourceServeOutcome::Failure(failure), + } +} + +/// Serves a committed Ready EXT-X-MAP object or returns a typed failure for endpoint-level mapping. +pub async fn serve_hls_map_cache_outcome( + segment_cache: Arc, + session: HlsSessionHandle, + map_file: HlsMapFile, + range_header: Option, + context: &HlsCacheResponseContext, +) -> HlsResourceServeOutcome { + match lookup_map_cache_object(&session, &map_file, &context.hls_access_lease_id).await { + CacheObjectLookup::Ready(object) => cache_object_serve_outcome( + serve_cache_object( + segment_cache, + object, + range_header, + CacheObjectServeContext::from_response_context(context), + ) + .await, + ), + CacheObjectLookup::Failure(failure) => HlsResourceServeOutcome::Failure(failure), + } +} + +/// Serves a committed Ready transient passthrough full object from the HLS cache. +pub async fn serve_hls_transient_object_cache_response( + segment_cache: Arc, + session: HlsSessionHandle, + resource_file: TransientResourceFile, + range_header: Option, + context: &HlsCacheResponseContext, +) -> Response { + match serve_hls_transient_object_cache_outcome(segment_cache, session, resource_file, range_header, context).await { + HlsResourceServeOutcome::Ready(response) => response, + HlsResourceServeOutcome::Failure(failure) => hls_resource_failure_default_response(failure), + } +} + +/// Serves a committed Ready transient passthrough object or returns a typed failure for endpoint-level mapping. +pub async fn serve_hls_transient_object_cache_outcome( + segment_cache: Arc, + session: HlsSessionHandle, + resource_file: TransientResourceFile, + range_header: Option, + context: &HlsCacheResponseContext, +) -> HlsResourceServeOutcome { + match lookup_transient_object_cache_object(&session, &resource_file, &context.hls_access_lease_id, context.now_ms) + .await + { + CacheObjectLookup::Ready(object) => cache_object_serve_outcome( + serve_cache_object( + segment_cache, + object, + range_header, + CacheObjectServeContext::from_response_context(context), + ) + .await, + ), + CacheObjectLookup::Failure(failure) => HlsResourceServeOutcome::Failure(failure), + } +} + +pub(super) fn hls_resource_failure_default_response(failure: HlsResourceServeFailure) -> Response { + match failure { + HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms } => { + service_unavailable_not_ready_response(retry_after_ms) + } + HlsResourceServeFailure::Missing + | HlsResourceServeFailure::Expired + | HlsResourceServeFailure::PermanentFailed { .. } => StatusCode::NOT_FOUND.into_response(), + } +} + +fn cache_object_serve_outcome(result: Result, HlsResourceServeFailure>) -> HlsResourceServeOutcome { + match result { + Ok(response) => HlsResourceServeOutcome::Ready(response), + Err(failure) => HlsResourceServeOutcome::Failure(failure), + } +} + +#[allow(clippy::too_many_lines)] +pub(super) async fn serve_cache_object( + segment_cache: Arc, + object: CacheObject, + range_header: Option, + context: CacheObjectServeContext, +) -> Result, HlsResourceServeFailure> +where + K: super::super::HlsCacheObjectKey + Send + Sync + 'static, +{ + let guard = CacheReadGuard::new(Arc::clone(&object.access), context.now_ms); + let requested_proxy_seq = match context.playback_cursor_tracking { + HlsPlaybackCursorTracking::Segment { proxy_seq } => Some(proxy_seq), + HlsPlaybackCursorTracking::Disabled => None, + }; + if let (Some(proxy_seq), Some(marker)) = (requested_proxy_seq, context.media_activity_marker.as_ref()) { + marker.record_startup_segment_request(proxy_seq, context.now_ms); + } + if let Some(repair_context) = object.repair_context.clone() { + if let Err(err) = + context.segment_repair.repair_ready_cache_hit(&segment_cache, &object.key, repair_context).await + { + debug!( + "HLS segment repair skipped for ready cache hit: session={} proxy_session={} resource={} error={err}", + object.log_context.identity.session(), + object.log_context.identity.proxy_session(), + object.log_context.resource_id + ); + } + } + if let (Some(proxy_seq), Some(marker)) = (requested_proxy_seq, context.media_activity_marker.as_ref()) { + marker.record_startup_repair_decision(proxy_seq, current_time_millis()); + } + let metadata_started_at = Instant::now(); + let metadata = match segment_cache.metadata(&object.key).await { + Ok(Some(metadata)) => metadata, + Ok(None) => return Ok(StatusCode::NOT_FOUND.into_response()), + Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(StatusCode::NOT_FOUND.into_response()), + Err(_) => return Ok(StatusCode::INTERNAL_SERVER_ERROR.into_response()), + }; + log_body_reader_wait_if_slow(&object.log_context, "metadata", metadata_started_at.elapsed().as_millis()); + + let body_id = next_hls_body_log_id(); + let range = resolve_single_byte_range(range_header.as_ref(), metadata.size); + let cursor_proxy_seq = context.playback_cursor_tracking.full_object_proxy_seq(range, metadata.size); + let (status, start, end, content_length) = match range { + SingleByteRange::Full => { + if metadata.size == 0 { + if let Some(metrics) = &context.metrics { + metrics.record_cache_hit(); + } + if let Some(marker) = &context.media_activity_marker { + marker.mark_at(context.now_ms).await; + } + return Ok(empty_ok_response(&object.content_type, context.cache_duration_seconds)); + } + if let Some(metrics) = &context.metrics { + metrics.record_cache_hit(); + } + debug!( + "HLS cache response prepared: body_id={} lease={} session={} proxy_session={} resource={} source=cache range=full content_length={} content_type={}", + body_id, + object.log_context.lease, + object.log_context.identity.session(), + object.log_context.identity.proxy_session(), + object.log_context.resource_id, + metadata.size, + object.content_type + ); + (StatusCode::OK, 0, metadata.size - 1, metadata.size) + } + SingleByteRange::Partial { start, end, length } => { + if let Some(metrics) = &context.metrics { + metrics.record_cache_hit(); + metrics.record_cache_range_hit(); + } + debug!( + "HLS cache response prepared: body_id={} lease={} session={} proxy_session={} resource={} source=cache range={start}-{end} content_length={length} content_type={}", + body_id, + object.log_context.lease, + object.log_context.identity.session(), + object.log_context.identity.proxy_session(), + object.log_context.resource_id, + object.content_type + ); + (StatusCode::PARTIAL_CONTENT, start, end, length) + } + SingleByteRange::Unsatisfiable => return Ok(range_not_satisfiable_response(metadata.size)), + }; + let file_started_at = Instant::now(); + let file = match segment_cache.open_range(&object.key, start).await { + Ok(file) => file, + Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(StatusCode::NOT_FOUND.into_response()), + Err(_) => return Ok(StatusCode::INTERNAL_SERVER_ERROR.into_response()), + }; + log_body_reader_wait_if_slow(&object.log_context, "file", file_started_at.elapsed().as_millis()); + let cursor_marker = + if let (Some(proxy_seq), Some(marker)) = (cursor_proxy_seq, context.media_activity_marker.clone()) { + let Some(marker) = marker.for_segment_request(proxy_seq, context.now_ms).await else { + return Err(HlsResourceServeFailure::Expired); + }; + Some(marker) + } else { + None + }; + let startup_body_observation = requested_proxy_seq.and_then(|proxy_seq| { + context + .media_activity_marker + .as_ref() + .and_then(|marker| marker.begin_startup_cache_response(proxy_seq, &body_id, current_time_millis())) + }); + if let Some(marker) = &context.media_activity_marker { + marker.mark_at(context.now_ms).await; + } + let completion_marker = cursor_marker.as_ref().and_then(HlsMediaActivityMarker::completed_segment_marker); + + let stream = ReaderStream::new(file.take(content_length)); + let body_context = CacheBodyLogContext { + body_id, + identity: object.log_context.identity.clone(), + resource_id: object.log_context.resource_id.clone(), + object_kind: object.log_context.object_kind, + source: object.log_context.body_source, + content_length, + }; + let stream = ActiveReaderStream::new( + Box::pin(stream), + Some(guard), + body_context, + context.qos_meter, + completion_marker, + startup_body_observation, + ); + let mut response = Response::new(Body::from_stream(stream)); + *response.status_mut() = status; + + let headers = response.headers_mut(); + insert_header_value(headers, header::CONTENT_TYPE, &object.content_type); + headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static(ACCEPT_RANGES_VALUE)); + insert_u64_header(headers, header::CONTENT_LENGTH, content_length); + insert_cache_control(headers, context.cache_duration_seconds); + if status == StatusCode::PARTIAL_CONTENT { + insert_header_value(headers, header::CONTENT_RANGE, &format!("bytes {start}-{end}/{}", metadata.size)); + } + mark_response_as_uncompressed(&mut response); + if object.is_media { + if let Some(marker) = &context.media_activity_marker { + response = marker.confirm_media_response(response); + } + } + Ok(response) +} + +async fn lookup_segment_cache_object( + session: &HlsSessionHandle, + segment_file: &HlsSegmentFile, + hls_access_lease_id: &HlsAccessLeaseId, +) -> CacheObjectLookup { + let session = session.read().await; + if session.is_gc_marked_for_removal() { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); + } + let Some(entry) = session.segments.get(&segment_file.proxy_seq) else { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); + }; + if entry.proxy_file_ext != segment_file.extension { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); + } + match entry.status { + SegmentCacheStatus::Ready { .. } => {} + SegmentCacheStatus::Fetching { .. } + | SegmentCacheStatus::Queued { .. } + | SegmentCacheStatus::Discovered + | SegmentCacheStatus::CapacityDeferred { .. } => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { + retry_after_ms: NOT_READY_RETRY_AFTER_MS, + }); + } + SegmentCacheStatus::FailedRetryable { retry_after_ms, .. } => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms }); + } + SegmentCacheStatus::FailedPermanent { status, .. } => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::PermanentFailed { status }); + } + SegmentCacheStatus::Expired => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); + } + } + CacheObjectLookup::Ready(CacheObject { + is_media: true, + key: entry.cache_key.clone(), + access: Arc::clone(&entry.access), + content_type: entry.content_type.clone(), + log_context: CacheObjectLogContext { + lease: safe_hls_access_lease_id(hls_access_lease_id), + identity: HlsLogIdentity::from_session(&session), + resource_id: format!("{:06}", segment_file.proxy_seq), + object_kind: "Segment", + body_source: "normal", + }, + repair_context: Some(HlsSegmentRepairObjectContext { + source: HlsSegmentRepairSource::Normal, + log_identity: HlsLogIdentity::from_session(&session), + proxy_session_id: session.proxy_session_id.clone(), + hls_access_lease_id: Some(hls_access_lease_id.clone()), + rendered_object_id: HlsRepairRenderedObjectId::Normal { proxy_seq: segment_file.proxy_seq }, + resource_id: format!("{:06}", segment_file.proxy_seq), + file_ext: entry.proxy_file_ext.clone(), + // Cache-hit repair validation may carry the concrete fetch URL as diagnostic metadata only. + origin_fetch_uri_for_diagnostics: entry + .origin_fetch_ref + .as_ref() + .map(|fetch_ref| fetch_ref.resolved_origin_url.clone()) + .unwrap_or_default(), + media_sequence: Some(entry.origin_key.host_local_sequence), + discontinuity_sequence: Some(session.discontinuity_sequence), + complete_object: entry.origin_byte_range.is_none(), + encrypted: entry.encryption.is_some(), + custom_response: false, + }), + }) +} + +async fn lookup_map_cache_object( + session: &HlsSessionHandle, + map_file: &HlsMapFile, + hls_access_lease_id: &HlsAccessLeaseId, +) -> CacheObjectLookup { + let session = session.read().await; + if session.is_gc_marked_for_removal() { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); + } + let Some(entry) = session.maps.get(&ProxyMapId(map_file.proxy_map_id)) else { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); + }; + if entry.proxy_file_ext != map_file.extension { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); + } + match entry.status { + MapCacheStatus::Ready { .. } => {} + MapCacheStatus::Fetching { .. } | MapCacheStatus::Queued { .. } | MapCacheStatus::Discovered => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { + retry_after_ms: NOT_READY_RETRY_AFTER_MS, + }); + } + MapCacheStatus::FailedRetryable { retry_after_ms, .. } => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms }); + } + MapCacheStatus::FailedPermanent { status, .. } => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::PermanentFailed { status }); + } + MapCacheStatus::Expired => { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); + } + } + CacheObjectLookup::Ready(CacheObject { + is_media: false, + key: entry.cache_key.clone(), + access: Arc::clone(&entry.access), + content_type: entry.content_type.clone(), + log_context: CacheObjectLogContext { + lease: safe_hls_access_lease_id(hls_access_lease_id), + identity: HlsLogIdentity::from_session(&session), + resource_id: format!("map:{:06}", map_file.proxy_map_id), + object_kind: "Map", + body_source: "normal", + }, + repair_context: None, + }) +} + +async fn lookup_transient_object_cache_object( + session: &HlsSessionHandle, + resource_file: &TransientResourceFile, + hls_access_lease_id: &HlsAccessLeaseId, + now_ms: u64, +) -> CacheObjectLookup { + let mut session = session.write().await; + if session.is_gc_marked_for_removal() { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Expired); + } + let proxy_session_id = session.proxy_session_id.clone(); + let key = super::super::TransientPassthroughState::transient_object_key( + &proxy_session_id, + &resource_file.resource_id, + resource_file.extension.clone(), + ); + let Some(resource) = session.transient.resolve_current_resource(&resource_file.resource_id, now_ms) else { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); + }; + if resource.file_ext_hint.as_deref() != Some(resource_file.extension.as_str()) { + return CacheObjectLookup::Failure(HlsResourceServeFailure::Missing); + } + let resource_state = (resource.kind, resource.encrypted_media); + let resource_kind = Some(resource_state.0); + let Some(entry) = session.transient.ready_object(&key, resource_state.0, now_ms) else { + return match session.transient.object_cache.get(&key).map(|entry| &entry.status) { + Some(super::super::TransientObjectCacheStatus::Fetching { .. }) => { + CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { + retry_after_ms: NOT_READY_RETRY_AFTER_MS, + }) + } + Some(super::super::TransientObjectCacheStatus::FailedRetryable { retry_after_ms, .. }) => { + CacheObjectLookup::Failure(HlsResourceServeFailure::TemporaryUnavailable { + retry_after_ms: *retry_after_ms, + }) + } + Some(super::super::TransientObjectCacheStatus::FailedPermanent { status, .. }) => { + CacheObjectLookup::Failure(HlsResourceServeFailure::PermanentFailed { status: *status }) + } + Some(super::super::TransientObjectCacheStatus::Ready { .. }) => { + CacheObjectLookup::Failure(HlsResourceServeFailure::Expired) + } + None => CacheObjectLookup::Failure(HlsResourceServeFailure::Missing), + }; + }; + CacheObjectLookup::Ready(CacheObject { + is_media: matches!(resource_state.0, TransientResourceKind::Segment | TransientResourceKind::Part), + key: entry.key, + access: Arc::clone(&entry.access), + content_type: entry.content_type, + log_context: CacheObjectLogContext { + lease: safe_hls_access_lease_id(hls_access_lease_id), + identity: HlsLogIdentity::from_session(&session), + resource_id: resource_file.resource_id.0.clone(), + object_kind: transient_body_object_kind(resource_kind, &resource_file.extension), + body_source: "transient", + }, + repair_context: Some(HlsSegmentRepairObjectContext { + source: HlsSegmentRepairSource::Transient, + log_identity: HlsLogIdentity::from_session(&session), + proxy_session_id, + hls_access_lease_id: Some(hls_access_lease_id.clone()), + rendered_object_id: HlsRepairRenderedObjectId::Transient { + resource_id: resource_file.resource_id.0.clone(), + }, + resource_id: resource_file.resource_id.0.clone(), + file_ext: resource_file.extension.clone(), + origin_fetch_uri_for_diagnostics: resource_file.resource_id.0.clone(), + media_sequence: None, + discontinuity_sequence: None, + complete_object: true, + encrypted: resource_state.1 || resource_state.0 == TransientResourceKind::Key, + custom_response: false, + }), + }) +} diff --git a/backend/hls/src/response/finite.rs b/backend/hls/src/response/finite.rs new file mode 100644 index 000000000..ea0780d37 --- /dev/null +++ b/backend/hls/src/response/finite.rs @@ -0,0 +1,181 @@ +use super::{ + reader::{insert_header_value, insert_u64_header, next_hls_body_log_id, range_not_satisfiable_response}, + ActiveReaderStream, CacheBodyLogContext, FiniteBytesSelection, HlsCacheResponseContext, HlsMediaActivityMarker, + PreparedBytesStream, ProxySessionId, ACCEPT_RANGES_VALUE, +}; +use axum::{ + body::Body, + http::{header, HeaderValue, Response, StatusCode}, +}; +use bytes::Bytes; +use std::sync::Arc; +use tuliprox_core::utils::{ + byte_range::{resolve_single_byte_range, SingleByteRange}, + response_compression::mark_response_as_uncompressed, +}; + +fn select_finite_bytes(bytes: Bytes, range_header: Option<&HeaderValue>) -> Result { + let full_size = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + match resolve_single_byte_range(range_header, full_size) { + SingleByteRange::Full => Ok(FiniteBytesSelection { + status: StatusCode::OK, + content_length: full_size, + body: bytes, + content_range: None, + }), + SingleByteRange::Partial { start, end, length } => { + let start_usize = usize::try_from(start).unwrap_or(bytes.len()); + let end_exclusive = usize::try_from(end.saturating_add(1)).unwrap_or(bytes.len()).min(bytes.len()); + Ok(FiniteBytesSelection { + status: StatusCode::PARTIAL_CONTENT, + body: bytes.slice(start_usize.min(end_exclusive)..end_exclusive), + content_length: length, + content_range: Some(format!("bytes {start}-{end}/{full_size}")), + }) + } + SingleByteRange::Unsatisfiable => Err(full_size), + } +} + +fn build_finite_bytes_response( + selection: &FiniteBytesSelection, + body: Body, + content_type: &str, + cache_control: &'static str, +) -> Response { + let mut response = Response::new(body); + *response.status_mut() = selection.status; + let headers = response.headers_mut(); + insert_header_value(headers, header::CONTENT_TYPE, content_type); + insert_header_value(headers, header::ACCEPT_RANGES, ACCEPT_RANGES_VALUE); + insert_header_value(headers, header::CACHE_CONTROL, cache_control); + insert_u64_header(headers, header::CONTENT_LENGTH, selection.content_length); + if let Some(content_range) = selection.content_range.as_deref() { + insert_header_value(headers, header::CONTENT_RANGE, content_range); + } + mark_response_as_uncompressed(&mut response); + response +} + +/// Serves immutable prepared bytes without touching the live-lease state. +/// +/// The route is the authorization boundary; callers do not get any +/// upstream origin work on top of the response. +pub fn finite_hls_immutable_media_response( + bytes: Bytes, + range_header: Option<&HeaderValue>, + content_type: &'static str, + cache_control: &'static str, + head_only: bool, +) -> Response { + let selection = match select_finite_bytes(bytes, range_header) { + Ok(selection) => selection, + Err(full_size) => return range_not_satisfiable_response(full_size), + }; + let body = if head_only { Body::empty() } else { Body::from(selection.body.clone()) }; + build_finite_bytes_response(&selection, body, content_type, cache_control) +} + +/// Serves immutable in-memory media through the same `QoS`, completion, drop and +/// client-send-deadline stream used by disk-backed Shared-HLS media. +pub fn finite_hls_media_response( + bytes: Bytes, + range_header: Option<&HeaderValue>, + content_type: &'static str, + cache_control: &'static str, + context: &HlsCacheResponseContext, + _proxy_session_id: &ProxySessionId, + resource_id: String, +) -> Response { + let selection = match select_finite_bytes(bytes, range_header) { + Ok(selection) => selection, + Err(full_size) => return range_not_satisfiable_response(full_size), + }; + context.metrics.record_cache_hit(); + if selection.status == StatusCode::PARTIAL_CONTENT { + context.metrics.record_cache_range_hit(); + } + let body_context = CacheBodyLogContext { + body_id: next_hls_body_log_id(), + identity: context.log_identity.clone(), + resource_id, + object_kind: "TerminalSegment", + source: "prepared", + content_length: selection.content_length, + }; + let stream = PreparedBytesStream::new(selection.body.clone()); + let stream = ActiveReaderStream::new( + Box::pin(stream), + None, + body_context, + Arc::clone(&context.qos_meter), + context.media_activity_marker.as_ref().and_then(HlsMediaActivityMarker::completed_segment_marker), + None, + ); + build_finite_bytes_response(&selection, Body::from_stream(stream), content_type, cache_control) +} + +/// Serves one frozen lease-bound AES key revision without consulting mutable +/// transient-resource state or performing origin I/O. +pub fn finite_hls_terminal_key_response( + bytes: Bytes, + range_header: Option<&HeaderValue>, + content_type: &str, + cache_control: &'static str, + context: &HlsCacheResponseContext, + _proxy_session_id: &ProxySessionId, + resource_id: String, +) -> Response { + let selection = match select_finite_bytes(bytes, range_header) { + Ok(selection) => selection, + Err(full_size) => return range_not_satisfiable_response(full_size), + }; + context.metrics.record_cache_hit(); + if selection.status == StatusCode::PARTIAL_CONTENT { + context.metrics.record_cache_range_hit(); + } + let body_context = CacheBodyLogContext { + body_id: next_hls_body_log_id(), + identity: context.log_identity.clone(), + resource_id, + object_kind: "TerminalKey", + source: "prepared", + content_length: selection.content_length, + }; + let stream = PreparedBytesStream::new(selection.body.clone()); + let stream = ActiveReaderStream::new( + Box::pin(stream), + None, + body_context, + Arc::clone(&context.qos_meter), + context.media_activity_marker.as_ref().and_then(HlsMediaActivityMarker::completed_segment_marker), + None, + ); + build_finite_bytes_response(&selection, Body::from_stream(stream), content_type, cache_control) +} + +/// Builds terminal-media HEAD metadata without rendering bytes or creating a +/// reader, `QoS` stream, completion callback, or media-activity side effect. +pub fn finite_hls_media_head_response( + full_size: u64, + range_header: Option<&HeaderValue>, + content_type: &'static str, + cache_control: &'static str, +) -> Response { + let selection = match resolve_single_byte_range(range_header, full_size) { + SingleByteRange::Full => FiniteBytesSelection { + status: StatusCode::OK, + body: Bytes::new(), + content_length: full_size, + content_range: None, + }, + SingleByteRange::Partial { start, end, length } => FiniteBytesSelection { + status: StatusCode::PARTIAL_CONTENT, + body: Bytes::new(), + content_length: length, + content_range: Some(format!("bytes {start}-{end}/{full_size}")), + }, + SingleByteRange::Unsatisfiable => return range_not_satisfiable_response(full_size), + }; + build_finite_bytes_response(&selection, Body::empty(), content_type, cache_control) +} diff --git a/backend/hls/src/response/reader.rs b/backend/hls/src/response/reader.rs new file mode 100644 index 000000000..a1542804e --- /dev/null +++ b/backend/hls/src/response/reader.rs @@ -0,0 +1,268 @@ +use super::{ + hls_client_body_send_deadline, refresh_hls_client_body_send_deadline, ActiveReaderStream, CacheAccessState, + CacheBodyLogContext, CacheObjectLogContext, CacheReadGuard, HlsMediaActivityMarker, HlsStartupBodyObservation, + PreparedBytesStream, TransientResourceKind, ACCEPT_RANGES_VALUE, BODY_READER_WAIT_LOG_THRESHOLD_MS, + NEXT_HLS_BODY_LOG_ID, PREPARED_MEDIA_CHUNK_SIZE, +}; +use arc_swap::ArcSwapOption; +use axum::{ + body::Body, + http::{header, HeaderValue, Response, StatusCode}, +}; +use bytes::Bytes; +use futures::Stream; +use log::debug; +use std::{ + future::Future, + io, + pin::Pin, + sync::{atomic::Ordering, Arc}, + task::{Context, Poll}, + time::{Duration, Instant}, +}; +use tokio::time::sleep; +use tuliprox_core::utils::{current_time_millis, response_compression::mark_response_as_uncompressed}; +use tuliprox_session::StreamMeterHandle; + +pub(super) fn empty_ok_response(content_type: &str, cache_duration_seconds: u64) -> Response { + let mut response = Response::new(Body::empty()); + *response.status_mut() = StatusCode::OK; + let headers = response.headers_mut(); + insert_header_value(headers, header::CONTENT_TYPE, content_type); + headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static(ACCEPT_RANGES_VALUE)); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from_static("0")); + insert_cache_control(headers, cache_duration_seconds); + mark_response_as_uncompressed(&mut response); + response +} + +pub(super) fn range_not_satisfiable_response(full_size: u64) -> Response { + let mut response = Response::new(Body::empty()); + *response.status_mut() = StatusCode::RANGE_NOT_SATISFIABLE; + let headers = response.headers_mut(); + headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static(ACCEPT_RANGES_VALUE)); + headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + headers.insert(header::CONTENT_LENGTH, HeaderValue::from_static("0")); + insert_header_value(headers, header::CONTENT_RANGE, &format!("bytes */{full_size}")); + mark_response_as_uncompressed(&mut response); + response +} + +pub(super) fn service_unavailable_not_ready_response(retry_after_ms: u64) -> Response { + let mut response = Response::new(Body::empty()); + *response.status_mut() = StatusCode::SERVICE_UNAVAILABLE; + let headers = response.headers_mut(); + insert_header_value( + headers, + header::RETRY_AFTER, + &super::super::retry_after_secs_from_ms(retry_after_ms).to_string(), + ); + headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + mark_response_as_uncompressed(&mut response); + response +} + +pub(super) fn insert_cache_control(headers: &mut axum::http::HeaderMap, cache_duration_seconds: u64) { + insert_header_value( + headers, + header::CACHE_CONTROL, + &format!("public, max-age={cache_duration_seconds}, immutable"), + ); +} + +pub(super) fn insert_u64_header(headers: &mut axum::http::HeaderMap, name: header::HeaderName, value: u64) { + insert_header_value(headers, name, &value.to_string()); +} + +pub(super) fn insert_header_value(headers: &mut axum::http::HeaderMap, name: header::HeaderName, value: &str) { + if let Ok(value) = HeaderValue::from_str(value) { + headers.insert(name, value); + } +} + +impl PreparedBytesStream { + pub(super) fn new(bytes: Bytes) -> Self { Self { remaining: bytes } } +} + +impl Stream for PreparedBytesStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + if self.remaining.is_empty() { + return Poll::Ready(None); + } + let chunk_len = self.remaining.len().min(PREPARED_MEDIA_CHUNK_SIZE); + Poll::Ready(Some(Ok(self.remaining.split_to(chunk_len)))) + } +} + +impl ActiveReaderStream { + pub(super) fn new( + inner: Pin> + Send>>, + guard: Option, + context: CacheBodyLogContext, + meter: Arc>, + media_activity_marker: Option, + startup_body_observation: Option, + ) -> Self { + Self { + inner, + _guard: guard, + context, + started_at: Instant::now(), + last_yield_at: Instant::now(), + send_deadline: Box::pin(sleep(hls_client_body_send_deadline())), + max_idle_ms: 0, + completed_logged: false, + finished: false, + bytes_yielded: 0, + meter, + media_activity_marker, + startup_body_observation, + } + } + + pub(super) fn log_completed(&mut self, outcome: &'static str) { + if self.completed_logged { + return; + } + self.completed_logged = true; + debug!( + "{} '{}' body completed: body_id={} session={} proxy_session={} source={} elapsed_s={:.3} idle_max_s={:.3} bytes={}/{} outcome={}", + self.context.object_kind, + self.context.resource_id, + self.context.body_id, + self.context.identity.session(), + self.context.identity.proxy_session(), + self.context.source, + duration_secs(self.started_at.elapsed().as_millis()), + duration_secs(self.max_idle_ms), + self.bytes_yielded, + self.context.content_length, + outcome + ); + if let Some(observation) = &self.startup_body_observation { + observation.finish(current_time_millis(), outcome); + } + if self.bytes_yielded >= self.context.content_length { + if let Some(marker) = &self.media_activity_marker { + marker.spawn_mark_completion_now(); + } + } + } +} + +impl Stream for ActiveReaderStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.finished { + return Poll::Ready(None); + } + if self.send_deadline.as_mut().poll(cx).is_ready() { + self.finished = true; + self.log_completed("timeout"); + return Poll::Ready(Some(Err(io::Error::new(io::ErrorKind::TimedOut, "hls client body send timed out")))); + } + match self.inner.as_mut().poll_next(cx) { + Poll::Ready(Some(Ok(chunk))) => { + let first_chunk = self.bytes_yielded == 0; + refresh_hls_client_body_send_deadline(self.send_deadline.as_mut()); + let idle_ms = self.last_yield_at.elapsed().as_millis(); + self.max_idle_ms = self.max_idle_ms.max(idle_ms); + self.last_yield_at = Instant::now(); + self.bytes_yielded = self.bytes_yielded.saturating_add(chunk.len() as u64); + if let Some(meter) = self.meter.load_full() { + meter.record_bytes(chunk.len() as u64); + } + if first_chunk + && self + .startup_body_observation + .as_ref() + .is_some_and(|observation| observation.record_first_chunk(current_time_millis())) + { + debug!( + "HLS cache body first chunk: body_id={} session={} proxy_session={} resource={} bytes={}", + self.context.body_id, + self.context.identity.session(), + self.context.identity.proxy_session(), + self.context.resource_id, + chunk.len() + ); + } + Poll::Ready(Some(Ok(chunk))) + } + Poll::Ready(Some(Err(err))) => { + self.finished = true; + self.log_completed("error"); + Poll::Ready(Some(Err(err))) + } + Poll::Ready(None) => { + self.finished = true; + self.log_completed("ok"); + Poll::Ready(None) + } + Poll::Pending => Poll::Pending, + } + } +} + +impl Drop for ActiveReaderStream { + fn drop(&mut self) { + let outcome = if self.bytes_yielded >= self.context.content_length { "ok" } else { "drop" }; + self.log_completed(outcome); + } +} + +impl CacheReadGuard { + pub(super) fn new(access: Arc, now_ms: u64) -> Self { + access.reader_started(now_ms); + Self { access } + } +} + +impl Drop for CacheReadGuard { + fn drop(&mut self) { self.access.reader_finished(); } +} + +pub(super) fn log_body_reader_wait_if_slow(context: &CacheObjectLogContext, wait_for: &'static str, elapsed_ms: u128) { + if elapsed_ms < BODY_READER_WAIT_LOG_THRESHOLD_MS { + return; + } + debug!( + "HLS cache reader wait: lease={} session={} proxy_session={} resource={} wait_for={} elapsed_ms={}", + context.lease, + context.identity.session(), + context.identity.proxy_session(), + context.resource_id, + wait_for, + elapsed_ms + ); +} + +fn duration_secs(elapsed_ms: u128) -> f64 { + Duration::from_millis(u64::try_from(elapsed_ms).unwrap_or(u64::MAX)).as_secs_f64() +} + +pub(super) fn transient_body_object_kind( + resource_kind: Option, + extension: &str, +) -> &'static str { + match resource_kind { + Some(TransientResourceKind::Key) => "Key", + Some(TransientResourceKind::Map) => "Map", + Some(TransientResourceKind::Segment | TransientResourceKind::Part | TransientResourceKind::Other) => "Segment", + None => { + if extension.eq_ignore_ascii_case("key") { + "Key" + } else { + "Segment" + } + } + } +} + +pub(super) fn next_hls_body_log_id() -> String { + let value = NEXT_HLS_BODY_LOG_ID.fetch_add(1, Ordering::Relaxed); + format!("{value:08x}") +} diff --git a/backend/hls/src/response/revision.rs b/backend/hls/src/response/revision.rs new file mode 100644 index 000000000..c744b697a --- /dev/null +++ b/backend/hls/src/response/revision.rs @@ -0,0 +1,297 @@ +use super::{ + hls_client_body_send_deadline, + reader::{ + insert_cache_control, insert_header_value, insert_u64_header, next_hls_body_log_id, + range_not_satisfiable_response, + }, + refresh_hls_client_body_send_deadline, HlsCacheResponseContext, HlsResourceServeFailure, HlsResourceServeOutcome, + HlsSegmentFile, HlsSessionHandle, RevisionResponseStream, ACCEPT_RANGES_VALUE, +}; +use crate::{CachedSegmentMetadata, SegmentRevision, SegmentRevisionState}; +use axum::{ + body::Body, + http::{header, HeaderValue, Response, StatusCode}, +}; +use bytes::Bytes; +use futures::Stream; +use std::{ + future::Future, + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, + time::Duration, +}; +use tokio::time::sleep; +use tuliprox_core::utils::{ + byte_range::{resolve_single_byte_range, SingleByteRange}, + current_time_millis, + response_compression::mark_response_as_uncompressed, +}; + +pub(super) async fn serve_hls_revision_outcome( + session: &HlsSessionHandle, + segment_file: &HlsSegmentFile, + range: Option<&HeaderValue>, + context: &HlsCacheResponseContext, +) -> HlsResourceServeOutcome { + let Some(marker) = &context.media_activity_marker else { + return HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Expired); + }; + let deadline = + remaining_wait_deadline(context.now_ms, marker.manager.segment_fetch_policy().origin_segment_timeout_ms); + match tokio::time::timeout_at( + deadline, + prepare_hls_revision_response(session, segment_file, range, context, deadline), + ) + .await + { + Ok(outcome) => outcome, + Err(_) => { + HlsResourceServeOutcome::Failure(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms: 1000 }) + } + } +} + +async fn prepare_hls_revision_response( + session: &HlsSessionHandle, + segment_file: &HlsSegmentFile, + range: Option<&HeaderValue>, + context: &HlsCacheResponseContext, + deadline: tokio::time::Instant, +) -> HlsResourceServeOutcome { + let Some(marker) = &context.media_activity_marker else { + return HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Expired); + }; + let (lifetime, wait_ms) = { + let session = session.read().await; + let Some(startup) = &session.startup else { + return HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Expired); + }; + (Duration::from_secs(startup.config.max_progressive_reader_lifetime_secs.get()), startup.first_data_timeout_ms) + }; + let deadline = deadline.min(remaining_wait_deadline(context.now_ms, wait_ms)); + let (revision, progressive) = { + let mut leases = marker.manager.access_leases().write().await; + let Some(revision) = leases.published_segment_revision(&context.hls_access_lease_id, segment_file.proxy_seq) + else { + return HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing); + }; + let full_request = range.is_none_or(|range| range.as_bytes() == b"bytes=0-"); + let progressive = revision.revision().key.kind == super::super::SegmentRevisionKind::Raw + && full_request + && leases.try_claim_progressive_startup( + &context.hls_access_lease_id, + segment_file.proxy_seq, + context.now_ms, + ); + (revision, progressive) + }; + let metadata = match revision_metadata(revision.revision(), progressive, deadline).await { + Ok(metadata) => metadata, + Err(failure) => return HlsResourceServeOutcome::Failure(failure), + }; + let authorized_at_ms = current_time_millis(); + if !marker.manager.access_leases().write().await.media_identity_is_current( + &marker.lease_id, + &marker.proxy_session_id, + marker.lease_identity, + authorized_at_ms, + ) { + return HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Expired); + } + let Some(body_range) = RevisionBodyRange::resolve(metadata.as_ref(), range) else { + let size = metadata.as_ref().map_or(0, |metadata| metadata.size); + return HlsResourceServeOutcome::Ready(range_not_satisfiable_response(size)); + }; + // Reject an unexposable progressive revision before the playback cursor and + // startup observations record this request. `expose()` below still decides, + // because the producer may exhaust its replay in between. + if metadata.is_none() && !revision.revision().can_expose() { + return HlsResourceServeOutcome::Failure(HlsResourceServeFailure::TemporaryUnavailable { + retry_after_ms: 1000, + }); + } + let completion = if body_range.full { + let Some(cursor_marker) = marker.clone().for_segment_request(segment_file.proxy_seq, authorized_at_ms).await + else { + return HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Expired); + }; + cursor_marker.completed_segment_marker() + } else { + None + }; + marker.mark_at(authorized_at_ms).await; + marker.record_startup_segment_request(segment_file.proxy_seq, context.now_ms); + marker.record_startup_repair_decision(segment_file.proxy_seq, current_time_millis()); + if metadata.is_none() && !revision.revision().expose() { + return HlsResourceServeOutcome::Failure(HlsResourceServeFailure::TemporaryUnavailable { + retry_after_ms: 1000, + }); + } + let observation = + marker.begin_startup_cache_response(segment_file.proxy_seq, &next_hls_body_log_id(), current_time_millis()); + let stream = RevisionResponseStream { + inner: super::super::progressive_startup::revision_body(revision, lifetime, body_range.start, body_range.end), + qos_meter: Arc::clone(&context.qos_meter), + completion, + observation, + first_chunk: true, + finished: false, + idle_deadline: Box::pin(sleep(hls_client_body_send_deadline())), + }; + let mut response = Response::new(Body::from_stream(stream)); + body_range.apply_headers(&mut response, metadata.as_ref(), context.cache_duration_seconds); + mark_response_as_uncompressed(&mut response); + context.metrics.record_cache_hit(); + HlsResourceServeOutcome::Ready(marker.confirm_media_response(response)) +} + +fn remaining_wait_deadline(requested_at_ms: u64, wait_ms: u64) -> tokio::time::Instant { + let elapsed_ms = current_time_millis().saturating_sub(requested_at_ms); + tokio::time::Instant::now() + Duration::from_millis(wait_ms.saturating_sub(elapsed_ms)) +} + +/// A progressive claim streams a publishable pending revision; every other +/// request waits for the complete revision file. +async fn revision_metadata( + revision: &SegmentRevision, + progressive: bool, + deadline: tokio::time::Instant, +) -> Result, HlsResourceServeFailure> { + if progressive { + return match revision.state() { + SegmentRevisionState::Complete(metadata) => Ok(Some(metadata)), + SegmentRevisionState::Pending if revision.is_publishable() => Ok(None), + _ => Err(HlsResourceServeFailure::PermanentFailed { status: None }), + }; + } + match revision.wait_complete(deadline).await { + Ok(metadata) => Ok(Some(metadata)), + Err(error) if error.kind() == io::ErrorKind::TimedOut => { + Err(HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms: 1000 }) + } + Err(_) => Err(HlsResourceServeFailure::PermanentFailed { status: None }), + } +} + +/// Status and byte window of one revision response. +struct RevisionBodyRange { + status: StatusCode, + start: u64, + end: Option, + length: Option, + /// The response covers the whole object, so it may complete the segment. + full: bool, +} + +impl RevisionBodyRange { + /// `None` means the range cannot be satisfied. A pending revision has no + /// known size yet and is always streamed whole. + fn resolve(metadata: Option<&CachedSegmentMetadata>, range: Option<&HeaderValue>) -> Option { + let Some(metadata) = metadata else { + return Some(Self { status: StatusCode::OK, start: 0, end: None, length: None, full: true }); + }; + match resolve_single_byte_range(range, metadata.size) { + SingleByteRange::Full => Some(Self { + status: StatusCode::OK, + start: 0, + end: Some(metadata.size), + length: Some(metadata.size), + full: true, + }), + SingleByteRange::Partial { start, end, length } => Some(Self { + status: StatusCode::PARTIAL_CONTENT, + start, + end: Some(end.saturating_add(1)), + length: Some(length), + full: start == 0 && length == metadata.size, + }), + SingleByteRange::Unsatisfiable => None, + } + } + + fn apply_headers( + &self, + response: &mut Response, + metadata: Option<&CachedSegmentMetadata>, + cache_duration_seconds: u64, + ) { + *response.status_mut() = self.status; + insert_header_value(response.headers_mut(), header::CONTENT_TYPE, "video/mp2t"); + insert_cache_control(response.headers_mut(), cache_duration_seconds); + if let Some(length) = self.length { + insert_u64_header(response.headers_mut(), header::CONTENT_LENGTH, length); + } + if let Some(metadata) = metadata { + response.headers_mut().insert(header::ACCEPT_RANGES, HeaderValue::from_static(ACCEPT_RANGES_VALUE)); + if self.status == StatusCode::PARTIAL_CONTENT { + insert_header_value( + response.headers_mut(), + header::CONTENT_RANGE, + &format!( + "bytes {}-{}/{}", + self.start, + self.end.unwrap_or(metadata.size).saturating_sub(1), + metadata.size + ), + ); + } + } + } +} + +impl RevisionResponseStream { + pub(super) fn finish(&mut self, outcome: &'static str) { + if !self.finished { + self.finished = true; + if let Some(observation) = &self.observation { + observation.finish(current_time_millis(), outcome); + } + } + } +} + +impl Stream for RevisionResponseStream { + type Item = io::Result; + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.finished { + return Poll::Ready(None); + } + if self.idle_deadline.as_mut().poll(cx).is_ready() { + self.finish("timeout"); + return Poll::Ready(Some(Err(io::Error::new(io::ErrorKind::TimedOut, "HLS revision client body stalled")))); + } + match self.inner.as_mut().poll_next(cx) { + Poll::Ready(Some(Ok(chunk))) => { + refresh_hls_client_body_send_deadline(self.idle_deadline.as_mut()); + if let Some(meter) = self.qos_meter.load_full() { + meter.record_bytes(chunk.len() as u64); + } + if self.first_chunk { + self.first_chunk = false; + if let Some(observation) = &self.observation { + observation.record_first_chunk(current_time_millis()); + } + } + Poll::Ready(Some(Ok(chunk))) + } + Poll::Ready(Some(Err(error))) => { + self.finish("error"); + Poll::Ready(Some(Err(error))) + } + Poll::Ready(None) => { + if let Some(marker) = self.completion.take() { + marker.spawn_mark_completion_now(); + } + self.finish("completed"); + Poll::Ready(None) + } + Poll::Pending => Poll::Pending, + } + } +} + +impl Drop for RevisionResponseStream { + fn drop(&mut self) { self.finish("dropped"); } +} diff --git a/backend/hls/src/response/tests.rs b/backend/hls/src/response/tests.rs new file mode 100644 index 000000000..12852e7d0 --- /dev/null +++ b/backend/hls/src/response/tests.rs @@ -0,0 +1,289 @@ +use super::{ + cache::HlsPlaybackCursorTracking, finite_hls_media_response, reader::transient_body_object_kind, + serve_cache_object, serve_hls_segment_cache_outcome, spawn_bounded_media_completion, ActiveReaderStream, + CacheBodyLogContext, CacheObject, CacheObjectLogContext, CacheObjectServeContext, HlsCacheResponseContext, + HlsMediaActivityMarker, HlsResourceServeFailure, HlsResourceServeOutcome, PREPARED_MEDIA_CHUNK_SIZE, +}; +use crate::{ + media_reserve::HlsLeasePlaybackCursor, CacheAccessState, HlsAccessLease, HlsAccessLeaseId, HlsLogIdentity, + HlsPlaybackFamilyKey, HlsProxyManager, HlsSegmentCache, HlsSegmentFile, HlsSegmentRepairManager, HlsSessionHandle, + HlsSessionKey, OriginSegmentKey, ProxySessionId, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, + TransientResourceKind, +}; +use tuliprox_core::{model::HlsSegmentRepairConfig, utils::current_time_millis}; +use tuliprox_session::StreamMeterHandle; + +fn test_log_identity() -> HlsLogIdentity { HlsLogIdentity::for_test("content-session", "proxy-session") } + +use arc_swap::ArcSwapOption; +use axum::http::{header, HeaderValue, StatusCode}; +use bytes::Bytes; +use futures::StreamExt; +use http_body_util::BodyExt; +use shared::model::HlsSegmentRepairMode; +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Weak, + }, + time::Duration, +}; +use tokio::{sync::Semaphore, time::advance}; + +fn header(value: &str) -> HeaderValue { HeaderValue::from_str(value).expect("valid header") } + +fn test_segment_repair_manager() -> Arc { + Arc::new(HlsSegmentRepairManager::new(HlsSegmentRepairConfig { + max_level: HlsSegmentRepairMode::Off, + apply_to_first_segments: 1, + max_parallel_repairs: 1, + ..Default::default() + })) +} + +async fn live_media_marker( + manager: &Arc, + session: &HlsSessionHandle, + lease_id: HlsAccessLeaseId, +) -> HlsMediaActivityMarker { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let lease = HlsAccessLease::pending( + lease_id.clone(), + HlsPlaybackFamilyKey::new("test-user", "test-fingerprint"), + proxy_session_id.clone(), + "test-user".to_string(), + "test-session".to_string(), + 1, + "stream".to_string(), + 1, + current_time_millis(), + 60_000, + ); + let lease_identity = lease.media_identity().expect("live lease identity"); + manager.prepare_access_lease(lease).await; + HlsMediaActivityMarker::new(Arc::clone(manager), Arc::clone(session), proxy_session_id, lease_id, lease_identity) +} + +struct CachedSegmentFixture { + _temp_dir: tempfile::TempDir, + manager: Arc, + segment_cache: Arc, + session: HlsSessionHandle, + proxy_session_id: ProxySessionId, + lease_id: HlsAccessLeaseId, + segment_file: HlsSegmentFile, + access: Arc, + context: HlsCacheResponseContext, + meter: Arc, + full_size: u64, +} + +impl CachedSegmentFixture { + async fn new(bytes: Vec) -> Self { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let manager = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let segment_cache = Arc::clone(manager.segment_cache()); + let session = manager.get_or_create_session(HlsSessionKey::new(1, "range-stream"), b"secret", 1_000).await; + let proxy_session_id = session.read().await.proxy_session_id.clone(); + let proxy_seq = 12; + let cache_key = SegmentCacheKey::new(proxy_session_id.clone(), proxy_seq, "ts"); + segment_cache.write_bytes_and_commit(&cache_key, &bytes).await.expect("segment cache commit"); + let full_size = u64::try_from(bytes.len()).expect("test segment size"); + let access = Arc::new(CacheAccessState::new()); + session.write().await.segments.insert( + proxy_seq, + SegmentEntry { + origin_key: OriginSegmentKey { + origin_epoch: 1, + effective_host_id: 1, + host_local_sequence: proxy_seq, + host_local_index: u32::try_from(proxy_seq).expect("test proxy sequence"), + }, + proxy_seq, + duration_ms: 4_000, + proxy_file_ext: "ts".to_string(), + content_type: "video/mp2t".to_string(), + cache_key, + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + encryption: None, + origin_fetch_ref: None, + status: SegmentCacheStatus::Ready { content_length: full_size, ready_at_ms: 1_000 }, + last_rendered_at_ms: Some(1_000), + access: Arc::clone(&access), + }, + ); + let lease_id = HlsAccessLeaseId("range-lease".to_string()); + let marker = live_media_marker(&manager, &session, lease_id.clone()).await; + let meter = Arc::new(StreamMeterHandle::new(7, Weak::new())); + let log_identity = { + let session = session.read().await; + HlsLogIdentity::from_session(&session) + }; + let context = HlsCacheResponseContext::new( + lease_id.clone(), + log_identity, + 300, + Arc::clone(manager.metrics()), + Arc::clone(manager.segment_repair()), + Some(Arc::clone(&meter)), + Some(marker), + current_time_millis(), + ); + Self { + _temp_dir: temp_dir, + manager, + segment_cache, + session, + proxy_session_id, + lease_id, + segment_file: HlsSegmentFile { proxy_seq, extension: "ts".to_string() }, + access, + context, + meter, + full_size, + } + } + + async fn serve(&self, range: Option<&str>) -> HlsResourceServeOutcome { + serve_hls_segment_cache_outcome( + Arc::clone(&self.segment_cache), + Arc::clone(&self.session), + self.segment_file.clone(), + range.map(header), + &self.context, + ) + .await + } + + async fn cursor(&self) -> HlsLeasePlaybackCursor { + self.manager + .access_lease_response_snapshot(&self.lease_id, &self.proxy_session_id, current_time_millis()) + .await + .expect("range test lease") + .playback_cursor + } + + async fn wait_for_completion(&self) -> HlsLeasePlaybackCursor { + for _ in 0..64 { + let cursor = self.cursor().await; + if cursor.highest_contiguous_completed_proxy_seq == Some(self.segment_file.proxy_seq) { + return cursor; + } + tokio::task::yield_now().await; + } + panic!("segment completion task did not commit") + } +} + +fn ready_response(outcome: HlsResourceServeOutcome) -> axum::response::Response { + match outcome { + HlsResourceServeOutcome::Ready(response) => response, + HlsResourceServeOutcome::Failure(failure) => panic!("expected ready response, got {failure:?}"), + } +} + +async fn publish_revision_fixture( + fixture: &CachedSegmentFixture, + raw: bool, +) -> std::io::Result { + let config = tuliprox_core::model::HlsStartupConfig { + mode: if raw { shared::model::HlsStartupMode::Progressive } else { shared::model::HlsStartupMode::FirstReady }, + ..Default::default() + }; + let store = Arc::new(crate::SegmentRevisionStore::default()); + let budget = crate::ProgressiveBudgetManager::new(config.clone()); + let revision = store.create( + fixture.proxy_session_id.clone(), + fixture.segment_file.proxy_seq, + if raw { crate::SegmentRevisionKind::Raw } else { crate::SegmentRevisionKind::Processed }, + )?; + if raw { + let mut replay = + crate::progressive_startup::ProgressiveReplay::new(&budget).ok_or_else(|| std::io::Error::other("slot"))?; + replay.push(b"prefix")?; + *revision.revision().replay.lock().map_err(|_| std::io::Error::other("replay"))? = Some(replay); + revision.revision().prefix_available.store(6, Ordering::Release); + } else { + let entry = fixture + .session + .read() + .await + .segments + .get(&fixture.segment_file.proxy_seq) + .cloned() + .ok_or_else(|| std::io::Error::other("segment"))?; + let pin = + fixture.segment_cache.pin_revision_file(&fixture.segment_cache.object_path(&revision.revision().key))?; + *revision.revision().file_pin.lock().map_err(|_| std::io::Error::other("pin"))? = Some(pin); + let metadata = fixture + .segment_cache + .publish_processed_revision( + &revision.revision().key, + &fixture.segment_cache.object_path(&entry.cache_key), + tokio::time::Instant::now() + Duration::from_secs(5), + ) + .await?; + revision.revision().complete(metadata); + } + let revisions = std::collections::BTreeMap::from([(fixture.segment_file.proxy_seq, revision.clone())]); + fixture.session.write().await.startup = Some(crate::HlsSessionStartup { + config: config.clone(), + first_data_timeout_ms: 1000, + worker: Weak::new(), + access_leases: Arc::clone(fixture.manager.access_leases()), + revisions: revisions.clone(), + policy_fixed: true, + store, + budget, + }); + let snapshot = crate::HlsLeaseManifestSnapshot { + startup_revisions: Some(Arc::new(crate::HlsManifestRevisions { + mode: config.mode, + retained_start_seq: fixture.segment_file.proxy_seq, + revisions, + })), + delivery_mode: crate::HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: crate::HlsManifestCommitIdentity::new(1), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 0, + delivered_at_ms: current_time_millis(), + first_proxy_seq: fixture.segment_file.proxy_seq, + last_proxy_seq: fixture.segment_file.proxy_seq, + visible_segments: Arc::from([crate::HlsLeaseManifestSegment { + proxy_seq: fixture.segment_file.proxy_seq, + duration_ms: 4000, + uri: "12.ts".into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }]), + discontinuity_sequence: 0, + target_duration_ms: 4000, + playlist_duration_ms: 4000, + last_visible_media_end_ms: 4000, + active_map: None, + active_encryption: None, + container: crate::HlsMediaContainer::MpegTs, + }; + let mut leases = fixture.manager.access_leases().write().await; + let now = current_time_millis(); + let guard = leases + .prepare_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, now) + .ok_or_else(|| std::io::Error::other("publication guard"))?; + assert!(leases + .commit_manifest_publication(&fixture.lease_id, &fixture.proxy_session_id, guard, snapshot, now) + .is_committed()); + Ok(revision) +} + +mod admission; +mod http; +mod lifecycle; +mod policy; +mod recovery; +mod terminal; diff --git a/backend/hls/src/response/tests/admission.rs b/backend/hls/src/response/tests/admission.rs new file mode 100644 index 000000000..a6c9717aa --- /dev/null +++ b/backend/hls/src/response/tests/admission.rs @@ -0,0 +1,25 @@ +use super::*; + +#[tokio::test] +async fn media_completion_scheduling_does_not_block_when_completion_capacity_is_busy() { + let permits = Arc::new(Semaphore::new(1)); + let held = Arc::clone(&permits).acquire_owned().await.expect("initial permit"); + let completed = Arc::new(AtomicBool::new(false)); + let completed_for_task = Arc::clone(&completed); + spawn_bounded_media_completion(Arc::clone(&permits), async move { + completed_for_task.store(true, Ordering::Release); + }); + + tokio::task::yield_now().await; + assert!(!completed.load(Ordering::Acquire)); + drop(held); + + for _ in 0..16 { + if completed.load(Ordering::Acquire) { + break; + } + tokio::task::yield_now().await; + } + assert!(completed.load(Ordering::Acquire)); + assert_eq!(permits.available_permits(), 1); +} diff --git a/backend/hls/src/response/tests/http.rs b/backend/hls/src/response/tests/http.rs new file mode 100644 index 000000000..a7aae694c --- /dev/null +++ b/backend/hls/src/response/tests/http.rs @@ -0,0 +1,170 @@ +use super::*; + +#[test] +fn transient_body_log_kind_uses_resource_kind() { + assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Key), "bin"), "Key"); + assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Map), "bin"), "Map"); + assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Segment), "key"), "Segment"); + assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Part), "m4s"), "Segment"); + assert_eq!(transient_body_object_kind(Some(TransientResourceKind::Other), "bin"), "Segment"); +} + +#[test] +fn transient_body_log_kind_falls_back_to_key_extension() { + assert_eq!(transient_body_object_kind(None, "key"), "Key"); + assert_eq!(transient_body_object_kind(None, "KEY"), "Key"); + assert_eq!(transient_body_object_kind(None, "ts"), "Segment"); +} + +#[tokio::test] +async fn full_object_segment_ranges_record_request_and_completion_without_changing_http_range_status() { + const FULL_SIZE: usize = 512; + for (range, expected_status) in [ + (None, StatusCode::OK), + (Some("bytes=0-"), StatusCode::PARTIAL_CONTENT), + (Some("bytes=0-511"), StatusCode::PARTIAL_CONTENT), + (Some("bytes=-512"), StatusCode::PARTIAL_CONTENT), + ] { + let expected_body = vec![7_u8; FULL_SIZE]; + let fixture = CachedSegmentFixture::new(expected_body.clone()).await; + let response = ready_response(fixture.serve(range).await); + + assert_eq!(response.status(), expected_status, "unexpected status for {range:?}"); + assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); + assert!(!tuliprox_core::utils::response_compression::should_compress_response(&response)); + assert_eq!(response.headers()[header::CONTENT_LENGTH], FULL_SIZE.to_string()); + if range.is_some() { + assert_eq!(response.headers()[header::CONTENT_RANGE], "bytes 0-511/512"); + } else { + assert!(!response.headers().contains_key(header::CONTENT_RANGE)); + } + assert_eq!(fixture.access.active_readers(), 1); + let requested = fixture.cursor().await; + assert_eq!(requested.first_requested_proxy_seq, Some(12), "missing request for {range:?}"); + assert_eq!(requested.last_requested_proxy_seq, Some(12), "missing request for {range:?}"); + assert_eq!(requested.highest_contiguous_completed_proxy_seq, None); + assert!(fixture.session.read().await.activity.last_authorized_media_at_ms.is_some()); + + assert_eq!( + response.into_body().collect().await.expect("full range body").to_bytes(), + Bytes::from(expected_body) + ); + let completed = fixture.wait_for_completion().await; + assert_eq!(completed.highest_contiguous_completed_proxy_seq, Some(12)); + assert!(completed.first_segment_completed_at_ms.is_some()); + assert_eq!(fixture.meter.bytes_total(), fixture.full_size); + assert_eq!(fixture.access.active_readers(), 0); + } +} + +#[tokio::test] +async fn partial_segment_ranges_mark_activity_without_mutating_playback_cursor() { + const FULL_SIZE: usize = 512; + for (range, expected_start, expected_end) in [("bytes=1-", 1, 511), ("bytes=0-187", 0, 187)] { + let bytes = (0..FULL_SIZE).map(|value| u8::try_from(value % 251).expect("test byte")).collect::>(); + let fixture = CachedSegmentFixture::new(bytes.clone()).await; + let response = ready_response(fixture.serve(Some(range)).await); + let expected_body = &bytes[expected_start..=expected_end]; + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::ACCEPT_RANGES], "bytes"); + assert!(!tuliprox_core::utils::response_compression::should_compress_response(&response)); + assert_eq!( + response.headers()[header::CONTENT_RANGE], + format!("bytes {expected_start}-{expected_end}/{FULL_SIZE}") + ); + assert_eq!(response.headers()[header::CONTENT_LENGTH], expected_body.len().to_string()); + assert!(fixture.session.read().await.activity.last_authorized_media_at_ms.is_some()); + assert_eq!(fixture.cursor().await, HlsLeasePlaybackCursor::default()); + + assert_eq!( + response.into_body().collect().await.expect("partial range body").to_bytes(), + Bytes::copy_from_slice(expected_body) + ); + for _ in 0..8 { + tokio::task::yield_now().await; + } + assert_eq!(fixture.cursor().await, HlsLeasePlaybackCursor::default()); + assert_eq!(fixture.meter.bytes_total(), u64::try_from(expected_body.len()).expect("partial body size")); + assert_eq!(fixture.access.active_readers(), 0); + } +} + +#[tokio::test] +async fn unsatisfiable_or_missing_segment_records_neither_activity_nor_cursor() { + const FULL_SIZE: usize = 512; + let fixture = CachedSegmentFixture::new(vec![5_u8; FULL_SIZE]).await; + let unsatisfiable = ready_response(fixture.serve(Some("bytes=512-")).await); + + assert_eq!(unsatisfiable.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert_eq!(unsatisfiable.headers()[header::CONTENT_RANGE], "bytes */512"); + assert_eq!(fixture.session.read().await.activity.last_authorized_media_at_ms, None); + assert_eq!(fixture.cursor().await, HlsLeasePlaybackCursor::default()); + assert_eq!(fixture.meter.bytes_total(), 0); + assert_eq!(fixture.access.active_readers(), 0); + + let missing = serve_hls_segment_cache_outcome( + Arc::clone(&fixture.segment_cache), + Arc::clone(&fixture.session), + HlsSegmentFile { proxy_seq: 99, extension: "ts".to_string() }, + Some(header("bytes=0-")), + &fixture.context, + ) + .await; + assert!(matches!(missing, HlsResourceServeOutcome::Failure(HlsResourceServeFailure::Missing))); + assert_eq!(fixture.session.read().await.activity.last_authorized_media_at_ms, None); + assert_eq!(fixture.cursor().await, HlsLeasePlaybackCursor::default()); + assert_eq!(fixture.meter.bytes_total(), 0); +} + +#[tokio::test] +async fn cache_body_marks_media_activity_at_start_and_body_end() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let hls_proxy = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let key = SegmentCacheKey::new(ProxySessionId("proxy-session".to_string()), 12, "ts"); + segment_cache.write_bytes_and_commit(&key, b"0123456789").await.expect("commit should succeed"); + let access = Arc::new(CacheAccessState::new()); + let session = hls_proxy.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 1_000).await; + let marker = live_media_marker(&hls_proxy, &session, HlsAccessLeaseId("lease-a".to_string())).await; + + let response = serve_cache_object( + segment_cache, + CacheObject { + is_media: true, + key, + access, + content_type: "video/mp2t".to_string(), + log_context: CacheObjectLogContext { + lease: "lease-a".to_string(), + identity: test_log_identity(), + resource_id: "000012".to_string(), + object_kind: "Segment", + body_source: "normal", + }, + repair_context: None, + }, + Some(header("bytes=0-")), + CacheObjectServeContext { + cache_duration_seconds: 300, + metrics: None, + segment_repair: test_segment_repair_manager(), + qos_meter: Arc::new(ArcSwapOption::from(None::>)), + media_activity_marker: Some(marker), + playback_cursor_tracking: HlsPlaybackCursorTracking::Disabled, + now_ms: 1_000, + }, + ) + .await + .expect("cache object response"); + + assert_eq!(session.read().await.activity.last_authorized_media_at_ms, Some(1_000)); + + assert_eq!(response.into_body().collect().await.expect("body").to_bytes(), Bytes::from_static(b"0123456789")); + tokio::time::sleep(Duration::from_millis(10)).await; + + assert!( + session.read().await.activity.last_authorized_media_at_ms.expect("media activity") >= 1_000, + "body completion should not move media activity backwards" + ); +} diff --git a/backend/hls/src/response/tests/lifecycle.rs b/backend/hls/src/response/tests/lifecycle.rs new file mode 100644 index 000000000..da261ac2d --- /dev/null +++ b/backend/hls/src/response/tests/lifecycle.rs @@ -0,0 +1,131 @@ +use super::*; + +#[tokio::test] +async fn progressive_http_drop_errors_without_content_length_or_cursor_completion() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"canonical-body".to_vec()).await; + let owner = publish_revision_fixture(&fixture, true).await?; + let response = match fixture.serve(None).await { + HlsResourceServeOutcome::Ready(response) => response, + HlsResourceServeOutcome::Failure(failure) => return Err(std::io::Error::other(format!("{failure:?}"))), + }; + assert!(!response.headers().contains_key(header::CONTENT_LENGTH)); + let mut body = response.into_body().into_data_stream(); + assert_eq!(body.next().await.transpose().map_err(std::io::Error::other)?.as_deref(), Some(b"prefix".as_slice())); + owner.revision().fail(); + assert!(body.next().await.is_some_and(|chunk| chunk.is_err())); + assert!(body.next().await.is_none()); + assert_eq!(fixture.cursor().await.highest_contiguous_completed_proxy_seq, None); + Ok(()) +} + +#[tokio::test] +async fn finite_terminal_media_uses_qos_completion_and_drop_semantics() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let manager = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let session = manager.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 1_000).await; + let lease_id = HlsAccessLeaseId("terminal-lease".to_string()); + let marker = live_media_marker(&manager, &session, lease_id.clone()).await; + let meter = Arc::new(StreamMeterHandle::new(7, Weak::new())); + let context = HlsCacheResponseContext::new( + lease_id, + test_log_identity(), + 300, + Arc::clone(manager.metrics()), + Arc::clone(manager.segment_repair()), + Some(Arc::clone(&meter)), + Some(marker), + 1_000, + ); + let proxy_session_id = ProxySessionId("terminal-session".to_string()); + + let dropped = finite_hls_media_response( + Bytes::from_static(b"0123456789"), + None, + "video/mp2t", + "private, immutable", + &context, + &proxy_session_id, + "terminal/1/0".to_string(), + ); + drop(dropped); + tokio::task::yield_now().await; + assert_eq!(meter.bytes_total(), 0); + assert_eq!(session.read().await.activity.last_authorized_media_at_ms, None); + + let partially_consumed = finite_hls_media_response( + Bytes::from(vec![7_u8; PREPARED_MEDIA_CHUNK_SIZE.saturating_mul(2)]), + None, + "video/mp2t", + "private, immutable", + &context, + &proxy_session_id, + "terminal/1/0".to_string(), + ); + let mut partial_body = partially_consumed.into_body(); + let partial_chunk = partial_body + .frame() + .await + .expect("first frame") + .expect("first frame body") + .into_data() + .expect("first data frame"); + drop(partial_body); + tokio::task::yield_now().await; + assert_eq!(partial_chunk.len(), PREPARED_MEDIA_CHUNK_SIZE); + assert_eq!(meter.bytes_total(), PREPARED_MEDIA_CHUNK_SIZE as u64); + assert_eq!(session.read().await.activity.last_authorized_media_at_ms, None); + + context.mark_media_activity().await; + let range = header("bytes=2-5"); + let completed = finite_hls_media_response( + Bytes::from_static(b"0123456789"), + Some(&range), + "video/mp2t", + "private, immutable", + &context, + &proxy_session_id, + "terminal/1/0".to_string(), + ); + assert_eq!(completed.status(), StatusCode::PARTIAL_CONTENT); + assert!(!tuliprox_core::utils::response_compression::should_compress_response(&completed)); + assert_eq!(completed.into_body().collect().await.expect("body").to_bytes(), Bytes::from_static(b"2345")); + assert_eq!(meter.bytes_total(), PREPARED_MEDIA_CHUNK_SIZE as u64 + 4); + assert_eq!(session.read().await.activity.last_authorized_media_at_ms, Some(1_000)); +} + +#[tokio::test] +async fn dropped_full_range_segment_body_records_request_without_completion() { + const FULL_SIZE: usize = 512 * 1_024; + let fixture = CachedSegmentFixture::new(vec![9_u8; FULL_SIZE]).await; + let response = ready_response(fixture.serve(Some("bytes=0-")).await); + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[header::CONTENT_RANGE], format!("bytes 0-{}/{FULL_SIZE}", FULL_SIZE - 1)); + let requested = fixture.cursor().await; + assert_eq!(requested.first_requested_proxy_seq, Some(12)); + assert_eq!(requested.last_requested_proxy_seq, Some(12)); + assert_eq!(requested.highest_contiguous_completed_proxy_seq, None); + assert!(fixture.session.read().await.activity.last_authorized_media_at_ms.is_some()); + + let mut body = response.into_body(); + let first_chunk = body + .frame() + .await + .expect("first segment frame") + .expect("first segment frame body") + .into_data() + .expect("first segment data frame"); + assert!(first_chunk.len() < FULL_SIZE); + drop(body); + for _ in 0..8 { + tokio::task::yield_now().await; + } + + let dropped = fixture.cursor().await; + assert_eq!(dropped.first_requested_proxy_seq, Some(12)); + assert_eq!(dropped.last_requested_proxy_seq, Some(12)); + assert_eq!(dropped.highest_contiguous_completed_proxy_seq, None); + assert_eq!(dropped.first_segment_completed_at_ms, None); + assert_eq!(fixture.meter.bytes_total(), u64::try_from(first_chunk.len()).expect("first chunk size")); + assert_eq!(fixture.access.active_readers(), 0); +} diff --git a/backend/hls/src/response/tests/policy.rs b/backend/hls/src/response/tests/policy.rs new file mode 100644 index 000000000..f070024e7 --- /dev/null +++ b/backend/hls/src/response/tests/policy.rs @@ -0,0 +1,78 @@ +use super::*; + +#[tokio::test] +async fn cache_hit_bodies_use_independent_readers_for_same_object() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let key = SegmentCacheKey::new(ProxySessionId("proxy-session".to_string()), 12, "ts"); + segment_cache.write_bytes_and_commit(&key, b"0123456789").await.expect("commit should succeed"); + let access = Arc::new(CacheAccessState::new()); + let object = CacheObject { + is_media: true, + key, + access: Arc::clone(&access), + content_type: "video/mp2t".to_string(), + log_context: CacheObjectLogContext { + lease: "lease-a".to_string(), + identity: test_log_identity(), + resource_id: "000012".to_string(), + object_kind: "Segment", + body_source: "normal", + }, + repair_context: None, + }; + + let first = serve_cache_object( + Arc::clone(&segment_cache), + object.clone(), + Some(header("bytes=0-")), + CacheObjectServeContext { + cache_duration_seconds: 300, + metrics: None, + segment_repair: test_segment_repair_manager(), + qos_meter: Arc::new(ArcSwapOption::from(None::>)), + media_activity_marker: None, + playback_cursor_tracking: HlsPlaybackCursorTracking::Disabled, + now_ms: 1, + }, + ) + .await + .expect("first cache object response"); + let second = serve_cache_object( + segment_cache, + CacheObject { + is_media: true, + log_context: CacheObjectLogContext { + lease: "lease-b".to_string(), + identity: test_log_identity(), + resource_id: "000012".to_string(), + object_kind: "Segment", + body_source: "normal", + }, + ..object + }, + Some(header("bytes=0-")), + CacheObjectServeContext { + cache_duration_seconds: 300, + metrics: None, + segment_repair: test_segment_repair_manager(), + qos_meter: Arc::new(ArcSwapOption::from(None::>)), + media_activity_marker: None, + playback_cursor_tracking: HlsPlaybackCursorTracking::Disabled, + now_ms: 2, + }, + ) + .await + .expect("second cache object response"); + + assert_eq!(first.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(second.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(access.active_readers(), 2); + + let (first_body, second_body) = tokio::join!(first.into_body().collect(), second.into_body().collect(),); + + assert_eq!(first_body.expect("first body").to_bytes(), Bytes::from_static(b"0123456789")); + assert_eq!(second_body.expect("second body").to_bytes(), Bytes::from_static(b"0123456789")); + assert_eq!(access.active_readers(), 0); + assert_eq!(access.last_accessed_at_ms(), 2); +} diff --git a/backend/hls/src/response/tests/recovery.rs b/backend/hls/src/response/tests/recovery.rs new file mode 100644 index 000000000..be9f6b1ba --- /dev/null +++ b/backend/hls/src/response/tests/recovery.rs @@ -0,0 +1,36 @@ +use super::*; + +#[tokio::test] +async fn processed_http_retry_and_range_use_the_published_revision() -> std::io::Result<()> { + let fixture = CachedSegmentFixture::new(b"original-segment".to_vec()).await; + let _owner = publish_revision_fixture(&fixture, false).await?; + let key = SegmentCacheKey::new(fixture.proxy_session_id.clone(), fixture.segment_file.proxy_seq, "ts"); + fixture.segment_cache.write_bytes_and_commit(&key, b"longer-repaired-replacement").await?; + for (range, status, bytes) in [ + (None, StatusCode::OK, b"original-segment".as_slice()), + (Some("bytes=3-10"), StatusCode::PARTIAL_CONTENT, b"ginal-se".as_slice()), + (None, StatusCode::OK, b"original-segment".as_slice()), + ] { + let response = match fixture.serve(range).await { + HlsResourceServeOutcome::Ready(response) => response, + HlsResourceServeOutcome::Failure(failure) => return Err(std::io::Error::other(format!("{failure:?}"))), + }; + assert_eq!(response.status(), status); + assert_eq!( + response.headers().get(header::CONTENT_LENGTH).and_then(|value| value.to_str().ok()), + Some(bytes.len().to_string().as_str()) + ); + assert_eq!(response.into_body().collect().await.map_err(std::io::Error::other)?.to_bytes().as_ref(), bytes); + } + Ok(()) +} + +#[test] +fn temporary_unavailable_response_uses_concrete_retry_after() { + let response = super::super::hls_resource_failure_default_response( + super::super::HlsResourceServeFailure::TemporaryUnavailable { retry_after_ms: 2_500 }, + ); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response.headers().get(header::RETRY_AFTER).expect("retry-after"), "3"); +} diff --git a/backend/hls/src/response/tests/terminal.rs b/backend/hls/src/response/tests/terminal.rs new file mode 100644 index 000000000..830986695 --- /dev/null +++ b/backend/hls/src/response/tests/terminal.rs @@ -0,0 +1,88 @@ +use super::*; + +#[tokio::test] +async fn finite_terminal_media_supports_full_range_and_unsatisfiable_responses() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let manager = Arc::new(HlsProxyManager::with_cache_settings(temp_dir.path(), 300)); + let context = HlsCacheResponseContext::new( + HlsAccessLeaseId("terminal-lease".to_string()), + test_log_identity(), + 300, + Arc::clone(manager.metrics()), + Arc::clone(manager.segment_repair()), + None, + None, + 1_000, + ); + let proxy_session_id = ProxySessionId("terminal-session".to_string()); + let full = finite_hls_media_response( + Bytes::from_static(b"0123456789"), + None, + "video/mp2t", + "private, immutable", + &context, + &proxy_session_id, + "terminal/1/0".to_string(), + ); + assert_eq!(full.status(), StatusCode::OK); + assert!(!tuliprox_core::utils::response_compression::should_compress_response(&full)); + assert_eq!(full.headers()[header::CONTENT_LENGTH], "10"); + assert_eq!(full.into_body().collect().await.expect("body").to_bytes(), Bytes::from_static(b"0123456789")); + + let range_header = header("bytes=2-5"); + let partial = finite_hls_media_response( + Bytes::from_static(b"0123456789"), + Some(&range_header), + "video/mp2t", + "private, immutable", + &context, + &proxy_session_id, + "terminal/1/0".to_string(), + ); + assert_eq!(partial.status(), StatusCode::PARTIAL_CONTENT); + assert!(!tuliprox_core::utils::response_compression::should_compress_response(&partial)); + assert_eq!(partial.headers()[header::CONTENT_RANGE], "bytes 2-5/10"); + assert_eq!(partial.into_body().collect().await.expect("body").to_bytes(), Bytes::from_static(b"2345")); + + let invalid_header = header("bytes=20-"); + let invalid = finite_hls_media_response( + Bytes::from_static(b"0123456789"), + Some(&invalid_header), + "video/mp2t", + "private, immutable", + &context, + &proxy_session_id, + "terminal/1/0".to_string(), + ); + assert_eq!(invalid.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert!(!tuliprox_core::utils::response_compression::should_compress_response(&invalid)); + assert_eq!(invalid.headers()[header::CONTENT_RANGE], "bytes */10"); +} + +#[tokio::test(start_paused = true)] +async fn finite_terminal_media_stream_honors_shared_send_deadline() { + let stream = ActiveReaderStream::new( + Box::pin(futures::stream::pending()), + None, + CacheBodyLogContext { + body_id: "00000001".to_string(), + identity: test_log_identity(), + resource_id: "terminal/1/0".to_string(), + object_kind: "TerminalSegment", + source: "prepared", + content_length: 1, + }, + Arc::new(ArcSwapOption::from(None::>)), + None, + None, + ); + let task = tokio::spawn(async move { + let mut stream = stream; + stream.next().await + }); + tokio::task::yield_now().await; + advance(super::super::hls_client_body_send_deadline().saturating_add(Duration::from_millis(1))).await; + + let result = task.await.expect("deadline task").expect("deadline result"); + assert_eq!(result.expect_err("deadline error").kind(), std::io::ErrorKind::TimedOut); +} diff --git a/backend/hls/src/segment_fetcher.rs b/backend/hls/src/segment_fetcher.rs index 9808e1805..c4c12a54f 100644 --- a/backend/hls/src/segment_fetcher.rs +++ b/backend/hls/src/segment_fetcher.rs @@ -18,23 +18,20 @@ use super::{ }; use arc_swap::ArcSwap; use axum::http::HeaderMap; -use futures::FutureExt; -use log::{debug, warn}; use reqwest::Client; -use shared::model::HlsSegmentRepairMode; -use std::{fmt, sync::Arc, time::Duration}; -use tokio::{ - sync::{Notify, OwnedSemaphorePermit, RwLock, Semaphore}, - time::{timeout, timeout_at, Instant}, -}; -use tuliprox_core::{model::HlsCacheConfig, utils::content_coding::DecodedHttpResponse}; -use tuliprox_parser::hls::origin_manifest::ParsedByteRange; +use std::sync::Arc; +use tokio::sync::RwLock; const DEFAULT_MAX_GLOBAL_SEGMENT_FETCHES: usize = 64; + const DEFAULT_MAX_SESSION_SEGMENT_FETCHES: usize = 2; + const DEFAULT_MAX_PREFETCH_QUEUE_DEPTH: usize = 6; + const DEFAULT_ORIGIN_SEGMENT_TIMEOUT_MS: u64 = 10_000; + const DEFAULT_REPAIR_POSTPROCESS_TIMEOUT_MS: u64 = 2_000; + const SEGMENT_FETCH_SCHEDULING_MARGIN_MS: u64 = 1_000; /// Origin-object work which must complete before one media segment is usable. @@ -44,23 +41,6 @@ pub enum HlsSegmentFetchWorkload { EncryptedWithKey, } -impl HlsSegmentFetchWorkload { - const fn serialized_origin_objects(self) -> u64 { - match self { - Self::Clear => 1, - Self::EncryptedWithKey => 2, - } - } - - const fn from_encrypted(encrypted: bool) -> Self { - if encrypted { - Self::EncryptedWithKey - } else { - Self::Clear - } - } -} - /// Runtime policy for bounded live HLS segment origin fetches. #[derive(Debug, Clone, Eq, PartialEq)] pub struct SegmentFetchPolicy { @@ -74,98 +54,6 @@ pub struct SegmentFetchPolicy { pub permanent_failure_segment_threshold: u32, } -impl SegmentFetchPolicy { - pub fn from_config(config: &HlsCacheConfig) -> Self { - let postprocess_enabled = (config.segment_repair.max_level != HlsSegmentRepairMode::Off - && config.segment_repair.apply_to_first_segments > 0) - || config.segment_repair.corrupt_segment_watchdog.mode.is_enabled(); - Self { - max_global_segment_fetches: config.max_concurrent_segment_fetches_global.max(1), - max_session_segment_fetches: config.max_concurrent_segment_fetches_per_session.max(1), - max_prefetch_queue_depth: config.max_segments_prefetch, - origin_segment_timeout_ms: config.origin_segment_timeout_ms.get().max(1), - effective_repair_postprocess_timeout_ms: if postprocess_enabled { - config.segment_repair.postprocess_timeout_ms.get().max(100) - } else { - 0 - }, - // Object retry classification is deliberately independent from initial-strip policy. - permanent_failure_segment_threshold: Self::default().permanent_failure_segment_threshold, - ..Self::default() - } - } - - fn origin_object_retry_chain_budget_ms(&self) -> u64 { - let attempts = u64::try_from(self.retry_delays_ms.len()).unwrap_or(u64::MAX); - let per_attempt_budget = - self.origin_segment_timeout_ms.saturating_add(self.effective_repair_postprocess_timeout_ms); - let retry_delay_budget = self.retry_delays_ms.iter().copied().fold(0_u64, u64::saturating_add); - let jitter_budget = attempts.saturating_mul(self.retry_jitter_max_ms); - attempts.saturating_mul(per_attempt_budget).saturating_add(retry_delay_budget).saturating_add(jitter_budget) - } - - pub fn workload_budget_ms(&self, workload: HlsSegmentFetchWorkload) -> u64 { - self.origin_object_retry_chain_budget_ms() - .saturating_mul(workload.serialized_origin_objects()) - .saturating_add(SEGMENT_FETCH_SCHEDULING_MARGIN_MS) - } - - /// Expected latency of one successful media-object operation. - /// - /// This deliberately excludes the complete retry chain. Callers may use it - /// for recovery ETA, while `workload_budget_ms` remains the hard wait bound. - /// Configured protection timeouts above the conservative fallback are not - /// predictions of successful-object latency. - pub fn recovery_object_eta_ms(&self) -> u64 { - self.origin_segment_timeout_ms - .min(DEFAULT_ORIGIN_SEGMENT_TIMEOUT_MS) - .saturating_add(self.effective_repair_postprocess_timeout_ms.min(DEFAULT_REPAIR_POSTPROCESS_TIMEOUT_MS)) - .saturating_add(SEGMENT_FETCH_SCHEDULING_MARGIN_MS) - } - - /// Conservative public wait bound for a segment whose encryption state is not known at the call site. - pub fn demand_wait_timeout(&self) -> Duration { - self.demand_wait_timeout_for(HlsSegmentFetchWorkload::EncryptedWithKey) - } - - pub fn demand_wait_timeout_for(&self, workload: HlsSegmentFetchWorkload) -> Duration { - Duration::from_millis(self.workload_budget_ms(workload)) - } - - /// Wait bound for one transient origin object such as a key or MAP. - pub fn origin_object_wait_timeout(&self) -> Duration { - self.demand_wait_timeout_for(HlsSegmentFetchWorkload::Clear) - } - - pub fn retry_delay_ms(&self, attempt_index: usize) -> u64 { - let base_delay_ms = self.retry_delays_ms[attempt_index]; - if self.retry_jitter_max_ms == 0 { - return base_delay_ms; - } - let jitter_ms = fastrand::u64(0..=self.retry_jitter_max_ms); - if fastrand::bool() { - base_delay_ms.saturating_sub(jitter_ms) - } else { - base_delay_ms.saturating_add(jitter_ms) - } - } -} - -impl Default for SegmentFetchPolicy { - fn default() -> Self { - Self { - max_global_segment_fetches: DEFAULT_MAX_GLOBAL_SEGMENT_FETCHES, - max_session_segment_fetches: DEFAULT_MAX_SESSION_SEGMENT_FETCHES, - max_prefetch_queue_depth: DEFAULT_MAX_PREFETCH_QUEUE_DEPTH, - origin_segment_timeout_ms: DEFAULT_ORIGIN_SEGMENT_TIMEOUT_MS, - effective_repair_postprocess_timeout_ms: 0, - retry_delays_ms: [0, 100, 250, 500, 750], - retry_jitter_max_ms: 100, - permanent_failure_segment_threshold: 3, - } - } -} - /// Shared context required to schedule a segment fetch without holding session locks. #[derive(Clone)] pub struct SegmentFetchContext { @@ -197,243 +85,6 @@ pub enum HlsSwitchResourceKind { Map, } -/// Fetches one candidate-bound switch resource into an uncommitted cache object. -/// -/// The caller owns the returned object and must either commit it to the previewed cache key or remove it. This helper -/// shares the normal origin-resource Range/content-coding/retry implementation and performs no session mutation. -pub async fn stage_hls_switch_resource( - context: &SegmentFetchContext, - policy: &SegmentFetchPolicy, - cache_key: K, - origin_url: String, - byte_range: Option, - kind: HlsSwitchResourceKind, -) -> Result -where - K: HlsCacheObjectKey + Clone + Send + Sync + 'static, -{ - let headers = build_segment_origin_headers(&context.headers, &context.origin_provider_session_headers, byte_range)?; - let resource_kind = match kind { - HlsSwitchResourceKind::Segment => HlsResourceFetchKind::Segment, - HlsSwitchResourceKind::Map => HlsResourceFetchKind::Map, - }; - let target = HlsOriginResourceFetchTarget { - kind: resource_kind, - source: HlsResourceFetchSource::Normal, - object_id: match kind { - HlsSwitchResourceKind::Segment => "switch-segment".to_string(), - HlsSwitchResourceKind::Map => "switch-map".to_string(), - }, - origin_url, - headers, - byte_range_expectation: if byte_range.is_some() { - HlsOriginByteRangeExpectation::PartialContent - } else { - HlsOriginByteRangeExpectation::FullObject - }, - }; - let clients = HlsOriginResourceClients { - client: context.client.clone(), - no_redirect_client: context.no_redirect_client.clone(), - use_manual_redirects: context.use_manual_redirects, - }; - let log_identity = { - let session = context.session.read().await; - super::HlsLogIdentity::from_session(&session) - }; - let cache = Arc::clone(&context.segment_cache); - run_hls_origin_resource_retry_loop_with_attempt_prepare( - target, - clients, - policy, - &log_identity, - |_attempt| async { Ok(()) }.boxed(), - |()| async {}.boxed(), - move |response, _attempt, body_deadline, ()| { - let cache = Arc::clone(&cache); - let cache_key = cache_key.clone(); - async move { - let staged = cache - .stage_temp_with_deadline(&cache_key, response.body, body_deadline.deadline()) - .await - .map_err(|err| HlsOriginResourceFetchError::cache_body(&err))?; - if staged.size == 0 { - if let Err(err) = cache.remove_staged(staged).await { - warn!("HLS empty staged switch resource cleanup failed: error={err}"); - } - return Err(HlsOriginResourceFetchError::cache_body(&std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - "empty switch resource", - ))); - } - Ok(staged) - } - .boxed() - }, - ) - .await -} - -#[derive(Clone)] -struct SegmentFetchSnapshot { - proxy_seq: u64, - proxy_seq_log: String, - origin_key: OriginSegmentKey, - cache_key: SegmentCacheKey, - fetch_ref: OriginSegmentFetchRef, - encryption: Option, - priority: SegmentFetchPriority, - started_at_ms: u64, - proxy_file_ext: String, - origin_seq: u64, - complete_object: bool, - key_dependency: Option, - origin_work_generation: u64, -} - -#[derive(Clone)] -struct SegmentKeyBindingSnapshot { - proxy_seq: u64, - cache_key: SegmentCacheKey, - origin_work_generation: u64, -} - -impl From<&SegmentFetchSnapshot> for SegmentKeyBindingSnapshot { - fn from(snapshot: &SegmentFetchSnapshot) -> Self { - Self { - proxy_seq: snapshot.proxy_seq, - cache_key: snapshot.cache_key.clone(), - origin_work_generation: snapshot.origin_work_generation, - } - } -} - -struct ReadySegmentKeyFetchSnapshot { - binding: SegmentKeyBindingSnapshot, - dependency: SegmentKeyFetchDependency, -} - -enum SegmentWorkerTask { - ReadyKey(ReadySegmentKeyFetchSnapshot), - Segment(SegmentFetchSnapshot), -} - -struct QueuedSegmentFetchCandidate { - origin_key: OriginSegmentKey, - cache_key: SegmentCacheKey, - fetch_ref: OriginSegmentFetchRef, - encryption: Option, - proxy_file_ext: String, - complete_object: bool, -} - -#[derive(Clone)] -enum SegmentKeyDependency { - Fetch(Box), - Wait { notifier: Arc, resource_id: TransientResourceId, resource_extension: String }, -} - -#[derive(Clone)] -struct SegmentKeyFetchDependency { - token: TransientObjectFetchToken, - resource: TransientResourceRef, - resource_file: TransientResourceFile, -} - -enum SegmentKeyDependencySelection { - Ready(Option), - Unavailable, -} - -struct SegmentFetchCompletion { - generation_valid: bool, - stale_commit_cleanup_reserved: bool, - notifier: Option>, - scheduled_retry: Option, - evidence_changed_for: Option, -} - -struct ScheduledSegmentRetry { - wake: SegmentRetryWake, - priority: SegmentFetchPriority, -} - -enum SegmentRetryWake { - CapacityRevision { revision: HlsCacheCapacityRevision, projected_write_bytes: u64 }, -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum CapacityRetryAdmission { - Ready, - BindingExpired, - LocalIoFailure, -} - -struct SegmentFetchCommit { - content_length: u64, - generation_valid: bool, -} - -#[derive(Clone, Copy)] -struct SegmentOriginWorkFinish { - generation_valid: bool, - refresh_reservation: bool, -} - -impl fmt::Debug for SegmentFetchSnapshot { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SegmentFetchSnapshot") - .field("proxy_seq", &self.proxy_seq) - .field("proxy_seq_log", &self.proxy_seq_log) - .field("origin_key", &self.origin_key) - .field("cache_key", &self.cache_key) - .field("fetch_ref", &self.fetch_ref) - .field("priority", &self.priority) - .field("started_at_ms", &self.started_at_ms) - .field("proxy_file_ext", &self.proxy_file_ext) - .field("origin_seq", &self.origin_seq) - .field("complete_object", &self.complete_object) - .field("encrypted", &self.encryption.is_some()) - .field( - "key_dependency", - &self.key_dependency.as_ref().map(|dependency| match dependency { - SegmentKeyDependency::Fetch { .. } => "fetch", - SegmentKeyDependency::Wait { .. } => "wait", - }), - ) - .field("origin_work_generation", &self.origin_work_generation) - .finish() - } -} - -fn segment_fetch_attempt_matches(entry: &SegmentEntry, snapshot: &SegmentFetchSnapshot) -> bool { - entry.origin_key == snapshot.origin_key - && entry.cache_key == snapshot.cache_key - && matches!( - entry.status, - SegmentCacheStatus::Fetching { priority, started_at_ms } - if priority == snapshot.priority && started_at_ms == snapshot.started_at_ms - ) -} - -fn segment_fetch_binding_matches(entry: &SegmentEntry, snapshot: &SegmentFetchSnapshot) -> bool { - segment_fetch_attempt_matches(entry, snapshot) - && entry.origin_fetch_ref.as_ref() == Some(&snapshot.fetch_ref) - && entry.encryption == snapshot.encryption -} - -type SegmentFetchError = HlsOriginResourceFetchError; - -#[derive(Clone)] -struct SegmentWorkerRuntime { - global_semaphore: Arc, - policy: SegmentFetchPolicy, -} - -impl SegmentWorkerRuntime { - fn new(policy: SegmentFetchPolicy, global_semaphore: Arc) -> Self { Self { global_semaphore, policy } } -} - /// Bounded scheduler for live HLS segment demand fetches and prefetches. pub struct HlsSegmentWorkerPool { runtime: ArcSwap, @@ -442,3223 +93,48 @@ pub struct HlsSegmentWorkerPool { availability_reevaluations: Option>, } -impl HlsSegmentWorkerPool { - pub fn new(policy: SegmentFetchPolicy) -> Self { - let global_semaphore = Arc::new(Semaphore::new(policy.max_global_segment_fetches)); - Self::with_global_semaphore(policy, global_semaphore) - } +#[cfg(test)] +mod tests; - pub fn with_global_semaphore(policy: SegmentFetchPolicy, global_semaphore: Arc) -> Self { - Self::with_global_semaphore_and_metrics( - policy, - global_semaphore, - Arc::new(RwLock::new(HlsAccessLeaseStore::default())), - Arc::new(HlsCacheMetrics::default()), - ) - } - - pub fn with_global_semaphore_and_metrics( - policy: SegmentFetchPolicy, - global_semaphore: Arc, - access_leases: Arc>, - metrics: Arc, - ) -> Self { - Self::with_global_semaphore_metrics_and_availability(policy, global_semaphore, access_leases, metrics, None) - } - - pub fn with_global_semaphore_metrics_and_availability( - policy: SegmentFetchPolicy, - global_semaphore: Arc, - access_leases: Arc>, - metrics: Arc, - availability_reevaluations: Option< - Arc, - >, - ) -> Self { - Self { - runtime: ArcSwap::from_pointee(SegmentWorkerRuntime::new(policy, global_semaphore)), - access_leases, - metrics, - availability_reevaluations, - } - } - - pub fn update_config(&self, policy: SegmentFetchPolicy, global_semaphore: Arc) { - self.runtime.store(Arc::new(SegmentWorkerRuntime::new(policy, global_semaphore))); - } - - pub fn policy(&self) -> SegmentFetchPolicy { self.runtime.load().policy.clone() } - - pub fn access_leases(&self) -> &Arc> { &self.access_leases } - - pub fn metrics(&self) -> &Arc { &self.metrics } - - pub async fn classify_backpressure(&self, session: &HlsSessionHandle) -> HlsBackpressureState { - let session = session.read().await; - self.classify_backpressure_for_session(&session) - } - - pub fn classify_backpressure_for_session(&self, session: &super::HlsSession) -> HlsBackpressureState { - let runtime = self.runtime.load(); - classify_hls_backpressure( - session, - runtime.global_semaphore.available_permits(), - runtime.policy.max_session_segment_fetches, - ) - } - - pub async fn demand_fetch_and_wait( - self: &Arc, - context: SegmentFetchContext, - segment_file: &HlsSegmentFile, - now_ms: u64, - ) -> SegmentDemandFetchOutcome { - let (notifier, workload) = { - let mut session = context.session.write().await; - if session.is_gc_marked_for_removal() { - return SegmentDemandFetchOutcome::NotFound; - } - let Some(entry) = session.segments.get(&segment_file.proxy_seq) else { - return SegmentDemandFetchOutcome::NotFound; - }; - if entry.proxy_file_ext != segment_file.extension { - return SegmentDemandFetchOutcome::NotFound; - } - if let SegmentCacheStatus::FailedRetryable { failed_at_ms, retry_after_ms } = entry.status { - if now_ms < failed_at_ms.saturating_add(retry_after_ms) { - return SegmentDemandFetchOutcome::TimedOut; - } - if let Some(entry) = session.segments.get_mut(&segment_file.proxy_seq) { - entry.status = SegmentCacheStatus::Discovered; - } - } - let Some(entry) = session.segments.get(&segment_file.proxy_seq) else { - return SegmentDemandFetchOutcome::NotFound; - }; - let workload = HlsSegmentFetchWorkload::from_encrypted(entry.encryption.is_some()); - let notifier = match entry.status { - SegmentCacheStatus::Ready { .. } => return SegmentDemandFetchOutcome::Ready, - SegmentCacheStatus::Fetching { .. } | SegmentCacheStatus::CapacityDeferred { .. } => { - session.segment_fetch_notifiers.entry(segment_file.proxy_seq).or_default().clone() - } - SegmentCacheStatus::Discovered | SegmentCacheStatus::Queued { .. } => { - if entry.origin_fetch_ref.is_none() { - return SegmentDemandFetchOutcome::Unavailable; - } - let backpressure = self.classify_backpressure_for_session(&session); - if !backpressure.allows_new_demand_fetch() { - if log::log_enabled!(log::Level::Warn) { - let identity = super::HlsLogIdentity::from_session(&session); - warn!( - "HLS segment demand fetch skipped by backpressure: session={} proxy_session={} source=normal resource={:06} state={backpressure:?}", - identity.session(), - identity.proxy_session(), - segment_file.proxy_seq - ); - } - return SegmentDemandFetchOutcome::Unavailable; - } - session.queue_segment_fetch_candidate(segment_file.proxy_seq, SegmentFetchPriority::Demand, now_ms); - self.metrics.record_demand_fetch_started(); - if log::log_enabled!(log::Level::Debug) { - let identity = super::HlsLogIdentity::from_session(&session); - debug!( - "HLS segment demand fetch started: session={} proxy_session={} source=normal resource={:06}", - identity.session(), - identity.proxy_session(), - segment_file.proxy_seq - ); - } - session.segment_fetch_notifiers.entry(segment_file.proxy_seq).or_default().clone() - } - SegmentCacheStatus::FailedPermanent { .. } | SegmentCacheStatus::Expired => { - return SegmentDemandFetchOutcome::Unavailable; - } - SegmentCacheStatus::FailedRetryable { .. } => return SegmentDemandFetchOutcome::TimedOut, - }; - (notifier, workload) - }; - - self.wake_scheduler(context.clone(), now_ms).await; - - let wait_timeout = self.runtime.load().policy.demand_wait_timeout_for(workload); - if timeout(wait_timeout, notifier.notified()).await.is_err() { - return SegmentDemandFetchOutcome::TimedOut; - } - - let session = context.session.read().await; - if session.is_gc_marked_for_removal() { - return SegmentDemandFetchOutcome::NotFound; - } - match session.segments.get(&segment_file.proxy_seq).map(|entry| &entry.status) { - Some(SegmentCacheStatus::Ready { .. }) => SegmentDemandFetchOutcome::Ready, - Some(SegmentCacheStatus::Queued { .. } | SegmentCacheStatus::Fetching { .. }) => { - SegmentDemandFetchOutcome::QueuedOrFetching - } - Some(SegmentCacheStatus::CapacityDeferred { .. } | SegmentCacheStatus::FailedRetryable { .. }) => { - SegmentDemandFetchOutcome::TimedOut - } - Some( - SegmentCacheStatus::Discovered - | SegmentCacheStatus::Expired - | SegmentCacheStatus::FailedPermanent { .. }, - ) => SegmentDemandFetchOutcome::Unavailable, - None => SegmentDemandFetchOutcome::NotFound, - } - } - - pub async fn wake_scheduler(self: &Arc, context: SegmentFetchContext, now_ms: u64) { - loop { - let runtime = self.runtime.load_full(); - let Ok(permit) = Arc::clone(&runtime.global_semaphore).try_acquire_owned() else { - return; - }; - let task = if let Some(snapshot) = - self.next_ready_segment_key_fetch_snapshot(&context, now_ms, &runtime.policy).await - { - Some(SegmentWorkerTask::ReadyKey(snapshot)) - } else { - self.next_fetch_snapshot(&context, now_ms, &runtime.policy).await.map(SegmentWorkerTask::Segment) - }; - let Some(task) = task else { - drop(permit); - return; - }; - - let worker = Arc::clone(self); - let task_context = context.clone(); - tokio::spawn(async move { - match task { - SegmentWorkerTask::ReadyKey(snapshot) => { - worker.fetch_ready_segment_key(task_context, snapshot, runtime.policy.clone(), permit).await; - } - SegmentWorkerTask::Segment(snapshot) => { - worker.fetch_one_segment(task_context, snapshot, runtime.policy.clone(), permit).await; - } - } - }); - } - } - - async fn next_ready_segment_key_fetch_snapshot( - &self, - context: &SegmentFetchContext, - now_ms: u64, - policy: &SegmentFetchPolicy, - ) -> Option { - let (proxy_session_id, gc_marked_for_removal) = { - let session = context.session.read().await; - (session.proxy_session_id.clone(), session.is_gc_marked_for_removal()) - }; - if gc_marked_for_removal - || !self.access_leases.write().await.has_usable_access_lease_for_session(&proxy_session_id, now_ms) - { - return None; - } - let mut session = context.session.write().await; - if session.is_gc_marked_for_removal() || session.active_segment_fetches >= policy.max_session_segment_fetches { - return None; - } - let rendered_proxy_seqs = session.last_rendered_manifest.as_ref()?.segment_proxy_seqs.clone(); - for proxy_seq in rendered_proxy_seqs { - let Some((cache_key, encryption)) = session.segments.get(&proxy_seq).and_then(|segment| { - matches!(segment.status, SegmentCacheStatus::Ready { .. }) - .then(|| (segment.cache_key.clone(), segment.encryption.clone())) - }) else { - continue; - }; - let Some(encryption) = encryption else { - continue; - }; - let SegmentKeyDependencySelection::Ready(Some(SegmentKeyDependency::Fetch(dependency))) = - select_key_dependency(&mut session, &proxy_session_id, &encryption, now_ms) - else { - continue; - }; - session.active_segment_fetches = session.active_segment_fetches.saturating_add(1); - return Some(ReadySegmentKeyFetchSnapshot { - binding: SegmentKeyBindingSnapshot { - proxy_seq, - cache_key, - origin_work_generation: session.activity.origin_work_generation, - }, - dependency: *dependency, - }); - } - None - } - - async fn fetch_ready_segment_key( - self: Arc, - context: SegmentFetchContext, - snapshot: ReadySegmentKeyFetchSnapshot, - policy: SegmentFetchPolicy, - permit: OwnedSemaphorePermit, - ) { - let SegmentKeyFetchDependency { token, resource, resource_file } = snapshot.dependency; - let result = fetch_segment_key_dependency_into_cache( - &context, - &snapshot.binding, - &policy, - token, - resource, - resource_file, - ) - .await; - let proxy_session_id = { - let mut session = context.session.write().await; - session.active_segment_fetches = session.active_segment_fetches.saturating_sub(1); - session.proxy_session_id.clone() - }; - if result.is_ok() { - if let Some(coordinator) = self.availability_reevaluations.as_ref() { - coordinator.notify_session_evidence_changed(&proxy_session_id); - } - } - drop(permit); - if result.is_ok() { - self.schedule_wake(context, current_time_millis()); - } - } - - async fn next_fetch_snapshot( - &self, - context: &SegmentFetchContext, - now_ms: u64, - policy: &SegmentFetchPolicy, - ) -> Option { - let (proxy_session_id, gc_marked_for_removal) = { - let session = context.session.read().await; - (session.proxy_session_id.clone(), session.is_gc_marked_for_removal()) - }; - if gc_marked_for_removal { - return None; - } - let has_usable_access_lease = - self.access_leases.write().await.has_usable_access_lease_for_session(&proxy_session_id, now_ms); - let mut session = context.session.write().await; - if session.is_gc_marked_for_removal() { - return None; - } - if session.active_segment_fetches >= policy.max_session_segment_fetches { - return None; - } - - while let Some((proxy_seq, priority)) = session.segment_prefetch_queue.pop_next() { - let origin_work_generation = session.activity.origin_work_generation; - let Some(candidate) = - take_queued_segment_fetch_candidate(&mut session, proxy_seq, priority, now_ms, has_usable_access_lease) - else { - continue; - }; - let key_dependency = match select_segment_key_dependency( - &mut session, - &proxy_session_id, - proxy_seq, - candidate.encryption.as_ref(), - now_ms, - ) { - SegmentKeyDependencySelection::Ready(dependency) => dependency, - SegmentKeyDependencySelection::Unavailable => continue, - }; - if let Some(entry) = session.segments.get_mut(&proxy_seq) { - entry.status = SegmentCacheStatus::Fetching { priority, started_at_ms: now_ms }; - } else { - continue; - } - session.active_segment_fetches = session.active_segment_fetches.saturating_add(1); - let proxy_seq_log = format!("{proxy_seq:06}"); - if log::log_enabled!(log::Level::Debug) { - let identity = super::HlsLogIdentity::from_session(&session); - debug!( - "HLS segment fetch started: session={} proxy_session={} source=normal resource={} priority={priority:?}", - identity.session(), - identity.proxy_session(), - proxy_seq_log - ); - } - return Some(SegmentFetchSnapshot { - proxy_seq, - proxy_seq_log, - origin_key: candidate.origin_key, - cache_key: candidate.cache_key, - fetch_ref: candidate.fetch_ref, - encryption: candidate.encryption, - priority, - started_at_ms: now_ms, - proxy_file_ext: candidate.proxy_file_ext, - origin_seq: candidate.origin_key.host_local_sequence, - complete_object: candidate.complete_object, - key_dependency, - origin_work_generation, - }); - } - - None - } - - async fn fetch_one_segment( - self: Arc, - context: SegmentFetchContext, - snapshot: SegmentFetchSnapshot, - policy: SegmentFetchPolicy, - permit: OwnedSemaphorePermit, - ) { - let result = fetch_segment_into_cache(&context, &snapshot, &policy).await; - let finished_at_ms = current_time_millis(); - let completion = { - let mut session = context.session.write().await; - self.apply_segment_fetch_result(&mut session, &snapshot, &policy, result, finished_at_ms) - }; - let mut notifier = completion.notifier; - if completion.stale_commit_cleanup_reserved { - if let Err(err) = context.segment_cache.delete(&snapshot.cache_key).await { - if log::log_enabled!(log::Level::Warn) { - let identity = { - let session = context.session.read().await; - super::HlsLogIdentity::from_session(&session) - }; - warn!( - "HLS stale segment cache cleanup failed: session={} proxy_session={} resource={} error={err}", - identity.session(), - identity.proxy_session(), - snapshot.proxy_seq_log - ); - } - } - let mut session = context.session.write().await; - if let Some(entry) = session - .segments - .get_mut(&snapshot.proxy_seq) - .filter(|entry| segment_fetch_attempt_matches(entry, &snapshot)) - { - entry.status = SegmentCacheStatus::Discovered; - } - notifier = session.segment_fetch_notifiers.remove(&snapshot.proxy_seq); - } - if let Some(notifier) = notifier { - notifier.notify_waiters(); - } - if let (Some(coordinator), Some(proxy_session_id)) = - (self.availability_reevaluations.as_ref(), completion.evidence_changed_for.as_ref()) - { - coordinator.notify_session_evidence_changed(proxy_session_id); - } - drop(permit); - if let Some(retry) = completion.scheduled_retry { - self.schedule_segment_retry(context, snapshot, retry); - return; - } - if completion.generation_valid { - self.schedule_wake(context, finished_at_ms); - } - } - - fn apply_segment_fetch_result( - &self, - session: &mut super::HlsSession, - snapshot: &SegmentFetchSnapshot, - policy: &SegmentFetchPolicy, - result: Result, - finished_at_ms: u64, - ) -> SegmentFetchCompletion { - session.active_segment_fetches = session.active_segment_fetches.saturating_sub(1); - let fetch_succeeded = result.is_ok(); - let attempt_matches = session - .segments - .get(&snapshot.proxy_seq) - .is_some_and(|entry| segment_fetch_attempt_matches(entry, snapshot)); - let binding_matches = session - .segments - .get(&snapshot.proxy_seq) - .is_some_and(|entry| segment_fetch_binding_matches(entry, snapshot)); - let generation_valid = binding_matches - && session.activity.origin_work_generation == snapshot.origin_work_generation - && result.as_ref().map_or(true, |commit| commit.generation_valid); - // A successful origin/repair path has already atomically installed its physical object. Keep this fetch's - // entry in `Fetching` until stale cleanup finishes so no rebound attempt can install newer bytes at the same - // stable cache path and then lose them to this worker's cleanup. - let stale_commit_cleanup_reserved = fetch_succeeded && !generation_valid && attempt_matches; - let capacity_retry = if generation_valid { - result.as_ref().err().and_then(|error| { - error.capacity_revision().cloned().zip(error.projected_write_bytes()).map( - |(revision, projected_write_bytes)| ScheduledSegmentRetry { - wake: SegmentRetryWake::CapacityRevision { revision, projected_write_bytes }, - priority: snapshot.priority, - }, - ) - }) - } else { - None - }; - if generation_valid { - match result { - Ok(commit) => self.commit_successful_segment_fetch(session, snapshot, &commit, finished_at_ms), - Err(err) => commit_failed_segment_fetch(session, snapshot, policy, &err, finished_at_ms), - } - } else if !stale_commit_cleanup_reserved { - if let Some(entry) = session - .segments - .get_mut(&snapshot.proxy_seq) - .filter(|entry| segment_fetch_attempt_matches(entry, snapshot)) - { - entry.status = SegmentCacheStatus::Discovered; - } - } - let scheduled_retry = capacity_retry; - if generation_valid { - if let Err(err) = session.render_and_store_manifest(finished_at_ms) { - if log::log_enabled!(log::Level::Debug) { - let identity = super::HlsLogIdentity::from_session(session); - debug!( - "HLS manifest render deferred after segment state change: session={} proxy_session={} resource={} error={err:?}", - identity.session(), - identity.proxy_session(), - snapshot.proxy_seq_log - ); - } - } - } - let notifier = (!stale_commit_cleanup_reserved && scheduled_retry.is_none()) - .then(|| session.segment_fetch_notifiers.remove(&snapshot.proxy_seq)) - .flatten(); - let evidence_changed_for = (generation_valid && fetch_succeeded).then(|| session.proxy_session_id.clone()); - SegmentFetchCompletion { - generation_valid, - stale_commit_cleanup_reserved, - notifier, - scheduled_retry, - evidence_changed_for, - } - } - - fn commit_successful_segment_fetch( - &self, - session: &mut super::HlsSession, - snapshot: &SegmentFetchSnapshot, - commit: &SegmentFetchCommit, - finished_at_ms: u64, - ) { - let content_length = commit.content_length; - if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { - entry.status = SegmentCacheStatus::Ready { content_length, ready_at_ms: finished_at_ms }; - session.advance_media_readiness_generation(); - } - let reset_failures = session.record_successful_segment_fetch(); - self.metrics.record_segment_cached(); - if log::log_enabled!(log::Level::Debug) { - let identity = super::HlsLogIdentity::from_session(session); - debug!( - "HLS segment cached: session={} proxy_session={} source=normal resource={} content_length={content_length}", - identity.session(), - identity.proxy_session(), - snapshot.proxy_seq_log - ); - if let Some(reset_failures) = reset_failures { - debug!( - "HLS segment temporary failure counter reset: session={} proxy_session={} previous_failures={reset_failures}", - identity.session(), - identity.proxy_session() - ); - } - } - } - - fn schedule_wake(self: &Arc, context: SegmentFetchContext, now_ms: u64) { - let worker = Arc::clone(self); - tokio::spawn(async move { - worker.wake_scheduler(context, now_ms).await; - }); - } - - fn schedule_segment_retry( - self: &Arc, - context: SegmentFetchContext, - snapshot: SegmentFetchSnapshot, - retry: ScheduledSegmentRetry, - ) { - let worker = Arc::clone(self); - tokio::spawn(async move { - let capacity_admission = match &retry.wake { - SegmentRetryWake::CapacityRevision { revision, projected_write_bytes } => { - wait_for_capacity_retry_admission(&context, &snapshot, revision.clone(), *projected_write_bytes) - .await - } - }; - let now_ms = current_time_millis(); - let (requeued, abandoned_notifier) = { - let mut session = context.session.write().await; - let retry_state_matches = session.segments.get(&snapshot.proxy_seq).is_some_and(|entry| { - matches!( - (&entry.status, &retry.wake), - (SegmentCacheStatus::CapacityDeferred { .. }, SegmentRetryWake::CapacityRevision { .. }) - ) - }); - if retry_state_matches { - let binding_current = session.segments.get(&snapshot.proxy_seq).is_some_and(|entry| { - entry.origin_key == snapshot.origin_key - && entry.cache_key == snapshot.cache_key - && entry.origin_fetch_ref.as_ref() == Some(&snapshot.fetch_ref) - && entry.encryption == snapshot.encryption - && session.activity.origin_work_generation == snapshot.origin_work_generation - }); - if !binding_current - || session.is_gc_marked_for_removal() - || capacity_admission != CapacityRetryAdmission::Ready - { - if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { - entry.status = SegmentCacheStatus::Discovered; - } - (false, session.segment_fetch_notifiers.remove(&snapshot.proxy_seq)) - } else { - if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { - entry.status = SegmentCacheStatus::Discovered; - } - (session.queue_segment_fetch_candidate(snapshot.proxy_seq, retry.priority, now_ms), None) - } - } else { - (false, None) - } - }; - if let Some(notifier) = abandoned_notifier { - notifier.notify_waiters(); - } - if requeued { - worker.wake_scheduler(context, now_ms).await; - } - }); - } -} - -async fn wait_for_capacity_retry_admission( - context: &SegmentFetchContext, - snapshot: &SegmentFetchSnapshot, - mut revision: HlsCacheCapacityRevision, - projected_write_bytes: u64, -) -> CapacityRetryAdmission { - loop { - context.segment_cache.wait_for_capacity_change(&revision).await; - let binding_current = { - let session = context.session.read().await; - !session.is_gc_marked_for_removal() - && session.activity.origin_work_generation == snapshot.origin_work_generation - && session.segments.get(&snapshot.proxy_seq).is_some_and(|entry| { - entry.origin_key == snapshot.origin_key - && entry.cache_key == snapshot.cache_key - && entry.origin_fetch_ref.as_ref() == Some(&snapshot.fetch_ref) - && entry.encryption == snapshot.encryption - && entry.status.awaits_capacity_recovery() - }) - }; - if !binding_current { - return CapacityRetryAdmission::BindingExpired; - } - match context.segment_cache.ensure_projected_write_capacity(&snapshot.cache_key, projected_write_bytes).await { - Ok(()) => return CapacityRetryAdmission::Ready, - Err(error) => { - let Some(capacity) = super::cache::hls_cache_capacity_from_io(&error) else { - return CapacityRetryAdmission::LocalIoFailure; - }; - revision = capacity.revision().clone(); - } - } - } -} - -fn take_queued_segment_fetch_candidate( - session: &mut super::HlsSession, - proxy_seq: u64, - priority: SegmentFetchPriority, - now_ms: u64, - has_usable_access_lease: bool, -) -> Option { - let entry = session.segments.get_mut(&proxy_seq)?; - if !matches!(entry.status, SegmentCacheStatus::Queued { .. }) { - return None; - } - if priority != SegmentFetchPriority::Demand && !has_usable_access_lease { - entry.status = SegmentCacheStatus::Discovered; - return None; - } - let Some(fetch_ref) = entry.origin_fetch_ref.clone() else { - entry.status = SegmentCacheStatus::Discovered; - return None; - }; - if !fetch_ref.is_valid_at(now_ms) { - entry.status = SegmentCacheStatus::Discovered; - return None; - } - Some(QueuedSegmentFetchCandidate { - origin_key: entry.origin_key, - cache_key: entry.cache_key.clone(), - fetch_ref, - encryption: entry.encryption.clone(), - proxy_file_ext: entry.proxy_file_ext.clone(), - complete_object: entry.origin_byte_range.is_none(), - }) -} - -fn select_segment_key_dependency( - session: &mut super::HlsSession, - proxy_session_id: &ProxySessionId, - proxy_seq: u64, - encryption: Option<&HlsSegmentEncryption>, - now_ms: u64, -) -> SegmentKeyDependencySelection { - let Some(encryption) = encryption else { - return SegmentKeyDependencySelection::Ready(None); - }; - let selection = select_key_dependency(session, proxy_session_id, encryption, now_ms); - if matches!(&selection, SegmentKeyDependencySelection::Unavailable) { - mark_segment_discovered(session, proxy_seq); - } - selection -} - -fn select_key_dependency( - session: &mut super::HlsSession, - proxy_session_id: &ProxySessionId, - encryption: &HlsSegmentEncryption, - now_ms: u64, -) -> SegmentKeyDependencySelection { - let Some(resource) = session.transient.resolve_current_resource(&encryption.resource_id, now_ms) else { - return SegmentKeyDependencySelection::Unavailable; - }; - if resource.kind != TransientResourceKind::Key - || resource.file_ext_hint.as_deref() != Some(encryption.resource_extension.as_str()) - { - return SegmentKeyDependencySelection::Unavailable; - } - let resource_file = TransientResourceFile { - resource_id: encryption.resource_id.clone(), - extension: encryption.resource_extension.clone(), - }; - let cache_duration_ms = session.transient.resource_ttl_ms; - let decision = session.transient.begin_object_fetch( - proxy_session_id, - &resource, - &encryption.resource_extension, - now_ms, - cache_duration_ms, - ); - let dependency = match decision { - TransientObjectFetchDecision::Ready => None, - TransientObjectFetchDecision::Fetch(token) => { - Some(SegmentKeyDependency::Fetch(Box::new(SegmentKeyFetchDependency { - token: *token, - resource, - resource_file, - }))) - } - TransientObjectFetchDecision::Wait(notifier) => Some(SegmentKeyDependency::Wait { - notifier, - resource_id: encryption.resource_id.clone(), - resource_extension: encryption.resource_extension.clone(), - }), - }; - SegmentKeyDependencySelection::Ready(dependency) -} - -fn mark_segment_discovered(session: &mut super::HlsSession, proxy_seq: u64) { - if let Some(entry) = session.segments.get_mut(&proxy_seq) { - entry.status = SegmentCacheStatus::Discovered; - } -} - -fn commit_failed_segment_fetch( - session: &mut super::HlsSession, - snapshot: &SegmentFetchSnapshot, - policy: &SegmentFetchPolicy, - error: &SegmentFetchError, - finished_at_ms: u64, -) { - if !error.is_local_cache_capacity() { - session.origin_control.path_condition = super::origin_progress::HlsOriginPathCondition::SegmentReadinessFailure; - } - let (status, invalidate_queued_origin_work) = - failed_segment_status(session, snapshot, policy, error, finished_at_ms); - if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { - entry.status = status; - } - if invalidate_queued_origin_work { - session.invalidate_queued_origin_work(); - if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { - entry.status = SegmentCacheStatus::FailedPermanent { failed_at_ms: finished_at_ms, status: None }; - } - } -} - -fn failed_segment_status( - session: &mut super::HlsSession, - snapshot: &SegmentFetchSnapshot, - policy: &SegmentFetchPolicy, - error: &SegmentFetchError, - finished_at_ms: u64, -) -> (SegmentCacheStatus, bool) { - if error.is_local_cache_capacity() { - return ( - SegmentCacheStatus::CapacityDeferred { priority: snapshot.priority, deferred_at_ms: finished_at_ms }, - false, - ); - } - if !error.retryable_failure() { - return ( - SegmentCacheStatus::FailedPermanent { failed_at_ms: finished_at_ms, status: error.permanent_status() }, - false, - ); - } - let threshold = policy.permanent_failure_segment_threshold.max(1); - let transition = session.record_temporary_segment_fetch_failure( - finished_at_ms, - HlsSegmentFailureObject::Normal { proxy_seq: snapshot.proxy_seq, origin_seq: snapshot.origin_seq }, - threshold, - ); - match transition { - HlsSegmentFailureTransition::StillRetryable { failures, threshold } => { - if log::log_enabled!(log::Level::Debug) { - let identity = super::HlsLogIdentity::from_session(session); - debug!( - "HLS segment temporary failure counted: session={} proxy_session={} object={} failures={} threshold={}", - identity.session(), - identity.proxy_session(), - snapshot.proxy_seq_log, - failures, - threshold - ); - } - (SegmentCacheStatus::FailedRetryable { failed_at_ms: finished_at_ms, retry_after_ms: 1_000 }, false) - } - HlsSegmentFailureTransition::BecamePermanentlyFailed { failures, threshold } => { - if log::log_enabled!(log::Level::Warn) { - let identity = super::HlsLogIdentity::from_session(session); - warn!( - "HLS segment temporary failure threshold reached: session={} proxy_session={} failures={} threshold={}", - identity.session(), - identity.proxy_session(), - failures, - threshold - ); - } - (SegmentCacheStatus::FailedPermanent { failed_at_ms: finished_at_ms, status: None }, true) - } - } -} - -impl Default for HlsSegmentWorkerPool { - fn default() -> Self { Self::new(SegmentFetchPolicy::default()) } -} - -async fn fetch_segment_into_cache( - context: &SegmentFetchContext, - snapshot: &SegmentFetchSnapshot, - policy: &SegmentFetchPolicy, -) -> Result { - ensure_segment_key_dependency_ready(context, snapshot, policy).await?; - fetch_segment_with_retries_into_cache(context, snapshot, policy).await -} - -async fn ensure_segment_key_dependency_ready( - context: &SegmentFetchContext, - snapshot: &SegmentFetchSnapshot, - policy: &SegmentFetchPolicy, -) -> Result<(), SegmentFetchError> { - let Some(dependency) = snapshot.key_dependency.clone() else { - return Ok(()); - }; - match dependency { - SegmentKeyDependency::Wait { notifier, resource_id, resource_extension } => { - wait_for_segment_key_dependency( - context, - notifier, - resource_id, - resource_extension, - policy.origin_object_wait_timeout(), - ) - .await - } - SegmentKeyDependency::Fetch(dependency) => { - let SegmentKeyFetchDependency { token, resource, resource_file } = *dependency; - let binding = SegmentKeyBindingSnapshot::from(snapshot); - fetch_segment_key_dependency_into_cache(context, &binding, policy, token, resource, resource_file).await - } - } -} - -async fn wait_for_segment_key_dependency( - context: &SegmentFetchContext, - notifier: Arc, - resource_id: TransientResourceId, - resource_extension: String, - wait_timeout: Duration, -) -> Result<(), SegmentFetchError> { - let deadline = Instant::now() + wait_timeout; - loop { - let wake = notifier.notified(); - tokio::pin!(wake); - wake.as_mut().enable(); - let should_wait = { - let now_ms = current_time_millis(); - let session = context.session.read().await; - if session - .transient - .ready_key_object_valid_until_ms(&session.proxy_session_id, &resource_id, &resource_extension, now_ms) - .is_some() - { - return Ok(()); - } - let key = TransientPassthroughState::transient_object_key( - &session.proxy_session_id, - &resource_id, - resource_extension.clone(), - ); - matches!( - session.transient.object_unavailable_state(&key, now_ms), - TransientObjectUnavailableState::Fetching - ) - }; - if !should_wait { - return Err(SegmentFetchError::Timeout); - } - timeout_at(deadline, wake).await.map_err(|_| SegmentFetchError::Timeout)?; - } -} - -async fn fetch_segment_key_dependency_into_cache( - context: &SegmentFetchContext, - binding: &SegmentKeyBindingSnapshot, - policy: &SegmentFetchPolicy, - token: TransientObjectFetchToken, - resource: TransientResourceRef, - resource_file: TransientResourceFile, -) -> Result<(), SegmentFetchError> { - let mut fetch_finalizer = HlsTransientObjectFetchFinalizer::new( - context.session.clone(), - Arc::clone(&context.segment_cache), - token.clone(), - 1_000, - ); - let (staged, origin_work) = - stage_segment_key_dependency(context, policy, &token, &resource, &resource_file).await?; - let generation_valid = origin_work.generation_valid && { - let session = context.session.read().await; - segment_key_dependency_generation_matches( - &session, - binding, - &token, - &resource, - &resource_file, - current_time_millis(), - ) - }; - if !generation_valid { - if let Err(err) = context.segment_cache.remove_staged(staged).await { - warn!("HLS stale staged AES key cleanup failed: error={err}"); - } - context.session.write().await.fail_transient_object_retryable_if_current(&token, current_time_millis(), 1_000); - return Err(SegmentFetchError::Timeout); - } - let metadata = match context.segment_cache.commit_staged(token.cache_key(), staged).await { - Ok(metadata) => metadata, - Err(err) => { - context.session.write().await.fail_transient_object_retryable_if_current( - &token, - current_time_millis(), - 1_000, - ); - return Err(SegmentFetchError::cache_commit(&err)); - } - }; - let ready_at_ms = current_time_millis(); - super::transient_fetcher::delete_superseded_transient_object(&context.segment_cache, &token).await; - let mut session = context.session.write().await; - if !segment_key_dependency_generation_matches(&session, binding, &token, &resource, &resource_file, ready_at_ms) { - session.fail_transient_object_retryable_if_current(&token, ready_at_ms, 1_000); - return Err(SegmentFetchError::Timeout); - } - let expires_at_ms = transient_object_expires_at(ready_at_ms, session.transient.resource_ttl_ms); - if !session.commit_transient_object_ready_if_current( - TransientResourceKind::Key, - &token, - resource.content_type_hint.clone().unwrap_or_else(|| "application/octet-stream".to_string()), - metadata.size, - ready_at_ms, - expires_at_ms, - ) { - return Err(SegmentFetchError::Timeout); - } - fetch_finalizer.complete(); - Ok(()) -} - -async fn stage_segment_key_dependency( - context: &SegmentFetchContext, - policy: &SegmentFetchPolicy, - token: &TransientObjectFetchToken, - resource: &TransientResourceRef, - resource_file: &TransientResourceFile, -) -> Result<(StagedCacheObject, SegmentOriginWorkFinish), SegmentFetchError> { - let request = HlsTransientOriginFetchRequest { - resolved_origin_uri: resource.resolved_origin_uri.clone(), - origin_headers: context.headers.clone(), - origin_provider_session_headers: context.origin_provider_session_headers.clone(), - range_header: None, - resource_file: resource_file.clone(), - resource_kind: TransientResourceKind::Key, - clients: HlsOriginResourceClients { - client: context.client.clone(), - no_redirect_client: context.no_redirect_client.clone(), - use_manual_redirects: context.use_manual_redirects, - }, - policy: policy.clone(), - log_identity: { - let session = context.session.read().await; - super::HlsLogIdentity::from_session(&session) - }, - }; - let prepare_context = context.clone(); - let prepare_policy = policy.clone(); - let response = fetch_hls_transient_origin_response_with_attempt_prepare(request, move |_attempt| { - let context = prepare_context.clone(); - let policy = prepare_policy.clone(); - async move { prepare_segment_origin_attempt(context, policy).await }.boxed() - }) - .await; - let response = match response { - Ok(response) => response, - Err(err) => { - context.session.write().await.fail_transient_object_retryable_if_current( - token, - current_time_millis(), - 1_000, - ); - return Err(err); - } - }; - let staged = context - .segment_cache - .stage_temp_with_deadline(token.cache_key(), response.decoded.body, response.body_deadline.deadline()) - .await; - let origin_work = finish_segment_origin_attempt(context.clone(), response.guard).await; - let staged = match staged { - Ok(staged) if staged.size == 16 => staged, - Ok(staged) => { - if let Err(err) = context.segment_cache.remove_staged(staged).await { - warn!("HLS invalid staged AES key cleanup failed: error={err}"); - } - context.session.write().await.fail_transient_object_permanent_if_current( - token, - current_time_millis(), - Some(axum::http::StatusCode::BAD_GATEWAY), - ); - return Err(SegmentFetchError::NonRetryableStatus(axum::http::StatusCode::BAD_GATEWAY)); - } - Err(err) => { - context.session.write().await.fail_transient_object_retryable_if_current( - token, - current_time_millis(), - 1_000, - ); - return Err(SegmentFetchError::cache_body(&err)); - } - }; - Ok((staged, origin_work)) -} - -fn segment_key_dependency_generation_matches( - session: &super::HlsSession, - binding: &SegmentKeyBindingSnapshot, - token: &TransientObjectFetchToken, - resource: &TransientResourceRef, - resource_file: &TransientResourceFile, - now_ms: u64, -) -> bool { - if session.activity.origin_work_generation != binding.origin_work_generation - || !session.transient.object_fetch_token_matches(token) - { - return false; - } - let segment_matches = session.segments.get(&binding.proxy_seq).is_some_and(|segment| { - segment.cache_key == binding.cache_key - && segment.encryption.as_ref().is_some_and(|encryption| { - encryption.resource_id == resource_file.resource_id - && encryption.resource_extension == resource_file.extension - }) - }); - let resource_matches = resource.id == resource_file.resource_id - && resource.kind == TransientResourceKind::Key - && resource.file_ext_hint.as_deref() == Some(resource_file.extension.as_str()) - && session.transient.resource_matches_current(resource, now_ms); - segment_matches && resource_matches -} - -struct SegmentOriginAttemptGuard { - started_generation: Option, - provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, -} - -async fn prepare_segment_origin_attempt( - context: SegmentFetchContext, - policy: SegmentFetchPolicy, -) -> Result { - let started_generation = start_segment_origin_work(&context).await; - let binding = - if context.origin_io.is_some() { context.session.read().await.origin_account_binding.clone() } else { None }; - let provider_lease = if let (Some(origin_io), Some(binding)) = (context.origin_io.as_ref(), binding.as_ref()) { - if binding.is_detached() { - finish_segment_origin_work(&context, started_generation).await; - touch_segment_origin_account_binding(&context, false).await; - return Err(SegmentFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Detached)); - } - let guard = match begin_hls_origin_account_io_bounded( - origin_io, - &context.session, - binding, - hls_object_body_deadline(policy.origin_segment_timeout_ms), - ) - .await - { - Ok(guard) => guard, - Err(err) => { - finish_segment_origin_work(&context, started_generation).await; - touch_segment_origin_account_binding(&context, false).await; - return Err(SegmentFetchError::ProviderUnavailable(err)); - } - }; - Some((origin_io.clone(), guard)) - } else { - None - }; - Ok(SegmentOriginAttemptGuard { started_generation, provider_lease }) -} - -async fn finish_segment_origin_attempt( - context: SegmentFetchContext, - guard: SegmentOriginAttemptGuard, -) -> SegmentOriginWorkFinish { - finish_segment_origin_io(&context, guard.started_generation, guard.provider_lease).await -} - -async fn finish_segment_origin_io( - context: &SegmentFetchContext, - started_generation: Option, - provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, -) -> SegmentOriginWorkFinish { - let origin_work = finish_segment_origin_work(context, started_generation).await; - if let Some((origin_io, guard)) = provider_lease { - finish_hls_origin_account_io( - &origin_io, - &context.session, - guard, - origin_work.generation_valid && origin_work.refresh_reservation, - ) - .await; - touch_segment_origin_account_binding(context, origin_work.generation_valid && origin_work.refresh_reservation) - .await; - } - origin_work -} - -async fn start_segment_origin_work(context: &SegmentFetchContext) -> Option { - context.origin_io.as_ref()?; - let mut session = context.session.write().await; - Some(session.start_origin_work()) -} - -async fn finish_segment_origin_work( - context: &SegmentFetchContext, - started_generation: Option, -) -> SegmentOriginWorkFinish { - let Some(started_generation) = started_generation else { - return SegmentOriginWorkFinish { generation_valid: true, refresh_reservation: false }; - }; - let mut session = context.session.write().await; - let generation_valid = session.finish_origin_work(started_generation); - let refresh_reservation = session.should_refresh_origin_reservation(current_time_millis()); - SegmentOriginWorkFinish { generation_valid, refresh_reservation } -} - -async fn touch_segment_origin_account_binding(context: &SegmentFetchContext, reservation_refreshed: bool) { - let mut session = context.session.write().await; - if let Some(binding) = session.origin_account_binding.as_mut() { - let now_ms = current_time_millis(); - binding.last_origin_io_at_ms = Some(now_ms); - if reservation_refreshed { - binding.last_reservation_refresh_at_ms = Some(now_ms); - } - } -} - -#[allow(clippy::too_many_lines)] -async fn fetch_segment_with_retries_into_cache( - context: &SegmentFetchContext, - snapshot: &SegmentFetchSnapshot, - policy: &SegmentFetchPolicy, -) -> Result { - let headers = build_segment_origin_headers( - &context.headers, - &context.origin_provider_session_headers, - snapshot.fetch_ref.byte_range, - )?; - let target = HlsOriginResourceFetchTarget { - kind: HlsResourceFetchKind::Segment, - source: HlsResourceFetchSource::Normal, - object_id: snapshot.proxy_seq_log.clone(), - origin_url: snapshot.fetch_ref.resolved_origin_url.clone(), - headers, - byte_range_expectation: if snapshot.fetch_ref.byte_range.is_some() { - HlsOriginByteRangeExpectation::PartialContent - } else { - HlsOriginByteRangeExpectation::FullObject - }, - }; - let clients = HlsOriginResourceClients { - client: context.client.clone(), - no_redirect_client: context.no_redirect_client.clone(), - use_manual_redirects: context.use_manual_redirects, - }; - let log_identity = { - let session = context.session.read().await; - super::HlsLogIdentity::from_session(&session) - }; - let context = context.clone(); - let snapshot = snapshot.clone(); - let policy_for_prepare = policy.clone(); - let prepare_context = context.clone(); - let cleanup_context = context.clone(); - run_hls_origin_resource_retry_loop_with_attempt_prepare( - target, - clients, - policy, - &log_identity, - move |_attempt| { - let context = prepare_context.clone(); - let policy = policy_for_prepare.clone(); - async move { prepare_segment_origin_attempt(context, policy).await }.boxed() - }, - move |guard| { - let context = cleanup_context.clone(); - async move { - finish_segment_origin_attempt(context, guard).await; - } - .boxed() - }, - move |response, _attempt, body_deadline, guard| { - let context = context.clone(); - let snapshot = snapshot.clone(); - async move { - let commit_result = - commit_segment_response_into_cache(&context, &snapshot, response, body_deadline).await; - let origin_work = finish_segment_origin_attempt(context, guard).await; - commit_result.map(|metadata| SegmentFetchCommit { - content_length: metadata.size, - generation_valid: origin_work.generation_valid, - }) - } - .boxed() - }, - ) - .await -} - -async fn commit_segment_response_into_cache( - context: &SegmentFetchContext, - snapshot: &SegmentFetchSnapshot, - response: DecodedHttpResponse, - body_deadline: HlsOriginResourceBodyDeadline, -) -> Result { - let (proxy_session_id, log_identity) = { - let session = context.session.read().await; - (session.proxy_session_id.clone(), super::HlsLogIdentity::from_session(&session)) - }; - let repair_context = HlsSegmentRepairObjectContext { - source: HlsSegmentRepairSource::Normal, - log_identity, - proxy_session_id, - hls_access_lease_id: context.repair_access_lease_id.clone(), - rendered_object_id: HlsRepairRenderedObjectId::Normal { proxy_seq: snapshot.proxy_seq }, - resource_id: format!("{:06}", snapshot.proxy_seq), - file_ext: snapshot.proxy_file_ext.clone(), - // Segment repair uses the concrete fetch URL for diagnostics/postprocess metadata only. - origin_fetch_uri_for_diagnostics: snapshot.fetch_ref.resolved_origin_url.clone(), - media_sequence: Some(snapshot.origin_seq), - discontinuity_sequence: None, - complete_object: snapshot.complete_object, - encrypted: snapshot.encryption.is_some(), - custom_response: false, - }; - if let Some(content_length) = reliable_decoded_content_length(&response) { - context - .segment_cache - .ensure_projected_write_capacity(&snapshot.cache_key, content_length) - .await - .map_err(|error| SegmentFetchError::cache_body(&error))?; - } - context - .segment_repair - .commit_origin_response( - &context.segment_cache, - &snapshot.cache_key, - response.body, - body_deadline.deadline(), - repair_context, - ) - .await - .map_err(|err| SegmentFetchError::cache_body(&err)) -} - -fn reliable_decoded_content_length(response: &DecodedHttpResponse) -> Option { - if response.was_content_decoded() { - return None; - } - response.headers.get(axum::http::header::CONTENT_LENGTH)?.to_str().ok()?.parse().ok() -} - -fn build_segment_origin_headers( - source_headers: &HeaderMap, - provider_session_headers: &HeaderMap, - byte_range: Option, -) -> Result { - build_hls_origin_resource_headers(source_headers, provider_session_headers, byte_range) -} - -use tuliprox_core::utils::current_time_millis; +mod commit; +mod demand; +mod error; +mod key_dependency; +mod origin; +mod policy; +mod scheduler; +mod startup_fill; +mod switch_staging; #[cfg(test)] -mod tests { - use super::{ - build_segment_origin_headers, commit_failed_segment_fetch, current_time_millis, - take_queued_segment_fetch_candidate, wait_for_segment_key_dependency, HlsSegmentFetchWorkload, - SegmentFetchContext, SegmentFetchPolicy, SegmentFetchPriority, - }; - use crate::{ - build_rewrite_secret_fingerprint, GarbageCollectionPolicy, HlsAccessLease, HlsAccessLeaseId, - HlsGarbageCollector, HlsOriginResourceFetchError, HlsPlaybackFamilyKey, HlsSegmentCache, HlsSegmentFile, - HlsSegmentRepairManager, HlsSegmentWorkerPool, HlsSessionKey, HlsSessionStore, ProxySessionId, - RenderedManifest, SegmentCacheStatus, TimelineMapError, TransientObjectFetchDecision, - TransientObjectFetchToken, TransientResourceId, TransientResourceKind, TransientResourceRef, - }; - use async_compression::tokio::bufread::{BrotliEncoder, DeflateEncoder, GzipEncoder, ZstdEncoder}; - use axum::http::{header, HeaderMap, HeaderValue}; - use futures::poll; - use shared::model::{HlsCacheConfigDto, HlsSegmentRepairMode, HlsStripMode}; - use std::{ - collections::VecDeque, - io::Cursor, - path::{Path, PathBuf}, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }, - task::Poll, - time::Duration, - }; - use tokio::{ - io::{AsyncReadExt, AsyncWriteExt, BufReader}, - net::TcpListener, - sync::{oneshot, Mutex, Notify}, - }; - use tuliprox_core::model::{HlsCacheConfig, HlsSegmentRepairConfig}; - use tuliprox_parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}; - - const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; - - #[test] - fn segment_fetch_budget_accounts_for_serialized_key_and_media_retry_chains() { - let policy = SegmentFetchPolicy { - origin_segment_timeout_ms: 10_000, - effective_repair_postprocess_timeout_ms: 2_000, - retry_delays_ms: [0, 100, 250, 500, 750], - retry_jitter_max_ms: 100, - ..SegmentFetchPolicy::default() - }; - - assert_eq!(policy.demand_wait_timeout_for(HlsSegmentFetchWorkload::Clear), Duration::from_millis(63_100)); - assert_eq!( - policy.demand_wait_timeout_for(HlsSegmentFetchWorkload::EncryptedWithKey), - Duration::from_millis(125_200) - ); - assert_eq!(policy.demand_wait_timeout(), Duration::from_millis(125_200)); - assert_eq!(policy.origin_object_wait_timeout(), Duration::from_millis(63_100)); - } - - #[test] - fn hls_recovery_timing_expected_object_eta_is_not_the_retry_chain_timeout() { - let policy = SegmentFetchPolicy { - origin_segment_timeout_ms: 10_000, - effective_repair_postprocess_timeout_ms: 2_000, - retry_delays_ms: [0, 100, 250, 500, 750], - retry_jitter_max_ms: 100, - ..SegmentFetchPolicy::default() - }; - - assert_eq!(policy.recovery_object_eta_ms(), 13_000); - assert!(policy.recovery_object_eta_ms() < policy.workload_budget_ms(HlsSegmentFetchWorkload::Clear)); - assert!(policy.recovery_object_eta_ms() < policy.workload_budget_ms(HlsSegmentFetchWorkload::EncryptedWithKey)); - - let long_hard_timeout = SegmentFetchPolicy { - origin_segment_timeout_ms: 120_000, - effective_repair_postprocess_timeout_ms: 30_000, - ..policy - }; - assert_eq!(long_hard_timeout.recovery_object_eta_ms(), 13_000); - assert!( - long_hard_timeout.recovery_object_eta_ms() - < long_hard_timeout.workload_budget_ms(HlsSegmentFetchWorkload::Clear) - ); - } - - #[tokio::test] - async fn popped_candidate_without_usable_fetch_binding_returns_to_discovered() { - let server = spawn_sequence_response_server(vec![TestOriginResponse { - status: 200, - headers: Vec::new(), - body: b"unused".to_vec(), - }]) - .await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy::default(); - let (_, context, _) = fetch_context(&server, &temp_dir, &policy).await; - let mut session = context.session.write().await; - let entry = session.segments.get_mut(&1).expect("segment"); - entry.status = SegmentCacheStatus::Queued { priority: SegmentFetchPriority::Prefetch, queued_at_ms: 1 }; - entry.origin_fetch_ref = None; - - assert!( - take_queued_segment_fetch_candidate(&mut session, 1, SegmentFetchPriority::Prefetch, 10, true,).is_none() - ); - assert!(matches!(session.segments[&1].status, SegmentCacheStatus::Discovered)); - } - - #[tokio::test] - async fn popped_candidate_with_expired_fetch_binding_returns_to_discovered() { - let server = spawn_sequence_response_server(vec![TestOriginResponse { - status: 200, - headers: Vec::new(), - body: b"unused".to_vec(), - }]) - .await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy::default(); - let (_, context, _) = fetch_context(&server, &temp_dir, &policy).await; - let mut session = context.session.write().await; - let entry = session.segments.get_mut(&1).expect("segment"); - entry.status = SegmentCacheStatus::Queued { priority: SegmentFetchPriority::Prefetch, queued_at_ms: 1 }; - entry.origin_fetch_ref.as_mut().expect("fetch binding").valid_until_ms = Some(9); - - assert!( - take_queued_segment_fetch_candidate(&mut session, 1, SegmentFetchPriority::Prefetch, 10, true,).is_none() - ); - assert!(matches!(session.segments[&1].status, SegmentCacheStatus::Discovered)); - } - - #[test] - fn object_failure_threshold_is_independent_from_initial_strip() { - let mut without_strip = HlsCacheConfigDto::default(); - without_strip.strip.mode = HlsStripMode::Segments; - without_strip.strip.value = 0; - let mut with_large_strip = without_strip.clone(); - with_large_strip.strip.mode = HlsStripMode::Seconds; - with_large_strip.strip.value = u64::MAX; - - let without_strip = SegmentFetchPolicy::from_config(&HlsCacheConfig::from(&without_strip)); - let with_large_strip = SegmentFetchPolicy::from_config(&HlsCacheConfig::from(&with_large_strip)); - - assert_eq!( - without_strip.permanent_failure_segment_threshold, - with_large_strip.permanent_failure_segment_threshold, - ); - assert_eq!( - without_strip.permanent_failure_segment_threshold, - SegmentFetchPolicy::default().permanent_failure_segment_threshold - ); - } - - #[tokio::test] - async fn local_capacity_failure_does_not_change_origin_progress_or_failure_counter() { - let server = spawn_segment_server(0).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy::default(); - let (worker, context, _) = fetch_context(&server, &temp_dir, &policy).await; - let snapshot = worker.next_fetch_snapshot(&context, 20, &policy).await.expect("fetch snapshot"); - let error = HlsOriginResourceFetchError::LocalCacheCapacity { - required_session_bytes: 10, - required_global_bytes: 0, - projected_write_bytes: 10, - revision: context.segment_cache.capacity_revision(), - }; - let mut session = context.session.write().await; - let original_path_condition = session.origin_control.path_condition; - - commit_failed_segment_fetch(&mut session, &snapshot, &policy, &error, 21); - - assert_eq!(session.origin_control.path_condition, original_path_condition); - assert_eq!(session.segment_failure_tracker.consecutive_temporary_failures, 0); - assert!(matches!( - session.segments.get(&snapshot.proxy_seq).map(|segment| &segment.status), - Some(SegmentCacheStatus::CapacityDeferred { .. }) - )); - } - - #[tokio::test] - async fn capacity_deferred_segment_requeues_only_after_capacity_revision_changes() { - let server = spawn_segment_server(0).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..Default::default() }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - clear_scheduled_prefetch(&context, &policy).await; - { - let mut session = context.session.write().await; - for proxy_seq in [2_u64, 3] { - session.segments.get_mut(&proxy_seq).expect("ready tail segment").status = - SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 19 }; - } - assert!(session.queue_segment_fetch_candidate(1, SegmentFetchPriority::Demand, 20)); - } - let snapshot = worker.next_fetch_snapshot(&context, 20, &policy).await.expect("fetch snapshot"); - let notifier = { - let mut session = context.session.write().await; - session.segment_fetch_notifiers.entry(snapshot.proxy_seq).or_default().clone() - }; - let completion = { - let revision = context.segment_cache.capacity_revision(); - let error = HlsOriginResourceFetchError::LocalCacheCapacity { - required_session_bytes: 1, - required_global_bytes: 0, - projected_write_bytes: 1, - revision, - }; - let mut session = context.session.write().await; - worker.apply_segment_fetch_result(&mut session, &snapshot, &policy, Err(error), 21) - }; - let retry = completion.scheduled_retry.expect("capacity deferral owns a revision retry"); - assert!(completion.notifier.is_none(), "deferred work retains its existing demand notifier"); - worker.schedule_segment_retry(context.clone(), snapshot, retry); - let first_demand = worker.demand_fetch_and_wait(context.clone(), &segment_file, 22); - let repeated_demand = worker.demand_fetch_and_wait(context.clone(), &segment_file, 23); - tokio::pin!(first_demand, repeated_demand); - assert!(matches!(futures::poll!(first_demand.as_mut()), std::task::Poll::Pending)); - assert!(matches!(futures::poll!(repeated_demand.as_mut()), std::task::Poll::Pending)); - assert!(server.requests.lock().await.is_empty(), "capacity wait does not poll the origin"); - - let completed = notifier.notified(); - tokio::pin!(completed); - completed.as_mut().enable(); - // The revision may change before the spawned waiter first polls. Its - // token comparison must make that race lossless. - context.segment_cache.notify_capacity_protection_changed(); - let ((), first_demand_outcome, repeated_demand_outcome) = - tokio::time::timeout(Duration::from_secs(10), async { - tokio::join!(completed.as_mut(), first_demand.as_mut(), repeated_demand.as_mut()) - }) - .await - .expect("revision wake completes one requeued fetch"); - - assert_eq!(first_demand_outcome, super::SegmentDemandFetchOutcome::Ready); - assert_eq!(repeated_demand_outcome, super::SegmentDemandFetchOutcome::Ready); - assert_eq!(server.requests.lock().await.len(), 1); - let session = context.session.read().await; - assert!(matches!( - session.segments.get(&1).map(|segment| &segment.status), - Some(SegmentCacheStatus::Ready { .. }) - )); - let rendered = session.last_rendered_manifest.as_ref().expect("recovered timeline renders a manifest"); - assert_eq!(rendered.first_proxy_seq, 1); - assert_eq!(rendered.last_proxy_seq, 3); - assert!(rendered.body.contains("000001.ts")); - } - - async fn wait_for_concurrent_capacity_deferral(context: &SegmentFetchContext, server: &TestSegmentServer) { - tokio::time::timeout(Duration::from_secs(10), async { - loop { - let both_deferred = { - let session = context.session.read().await; - [2_u64, 3].into_iter().all(|proxy_seq| { - matches!( - session.segments.get(&proxy_seq).map(|segment| &segment.status), - Some(SegmentCacheStatus::CapacityDeferred { .. }) - ) - }) - }; - if both_deferred && server.requests.lock().await.len() >= 2 { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("both capacity failures become revision-bound deferrals"); - } - - async fn assert_segment_request_counts(server: &TestSegmentServer, expected: usize, failure_context: &str) { - let requests = server.requests.lock().await; - for proxy_seq in [2_u64, 3] { - assert_eq!( - requests.iter().filter(|request| request.starts_with(&format!("GET /{proxy_seq}.ts "))).count(), - expected, - "{failure_context}: segment {proxy_seq}" - ); - } - } - - async fn assert_concurrent_capacity_recovery_ready(context: &SegmentFetchContext) { - let session = context.session.read().await; - for proxy_seq in [2_u64, 3, 4] { - assert!(matches!( - session.segments.get(&proxy_seq).map(|segment| &segment.status), - Some(SegmentCacheStatus::Ready { .. }) - )); - } - let rendered = session.last_rendered_manifest.as_ref().expect("recovered timeline renders"); - assert_eq!(rendered.first_proxy_seq, 2); - assert_eq!(rendered.last_proxy_seq, 4); - assert!(rendered.body.contains("000002.ts")); - assert!(rendered.body.contains("000003.ts")); - } - - #[tokio::test] - async fn concurrent_capacity_deferred_segments_retry_once_after_real_cache_release() { - let response = TestOriginResponse { status: 200, headers: Vec::new(), body: b"12345".to_vec() }; - let server = spawn_sequence_response_server(vec![response; 4]).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..Default::default() }; - let (worker, context, _) = fetch_context(&server, &temp_dir, &policy).await; - { - let mut session = context.session.write().await; - session - .apply_origin_manifest(&normal_manifest(&format!( - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\n{0}/1.ts\n#EXTINF:4.0,\n{0}/2.ts\n#EXTINF:4.0,\n{0}/3.ts\n#EXTINF:4.0,\n{0}/4.ts\n", - server.base_url - ))) - .expect("four-segment manifest maps"); - } - clear_scheduled_prefetch(&context, &policy).await; - let (released_key, tail_key) = { - let session = context.session.read().await; - ( - session.segments.get(&1).expect("released segment").cache_key.clone(), - session.segments.get(&4).expect("ready tail segment").cache_key.clone(), - ) - }; - context - .segment_cache - .write_bytes_and_commit(&released_key, b"12345678901") - .await - .expect("released fixture commits"); - context.segment_cache.write_bytes_and_commit(&tail_key, b"t").await.expect("tail fixture commits"); - { - let mut session = context.session.write().await; - session.segments.get_mut(&1).expect("released segment").status = - SegmentCacheStatus::Ready { content_length: 11, ready_at_ms: 10 }; - session.segments.get_mut(&4).expect("ready tail segment").status = - SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 10 }; - } - context.segment_cache.update_cache_limits(100, 12); - - let first_worker = Arc::clone(&worker); - let first_context = context.clone(); - let first = tokio::spawn(async move { - first_worker - .demand_fetch_and_wait(first_context, &HlsSegmentFile { proxy_seq: 2, extension: "ts".to_string() }, 20) - .await - }); - let second_worker = Arc::clone(&worker); - let second_context = context.clone(); - let second = tokio::spawn(async move { - second_worker - .demand_fetch_and_wait( - second_context, - &HlsSegmentFile { proxy_seq: 3, extension: "ts".to_string() }, - 20, - ) - .await - }); - - wait_for_concurrent_capacity_deferral(&context, &server).await; - for _ in 0..32 { - tokio::task::yield_now().await; - } - assert_segment_request_counts(&server, 1, "initial capacity deferral").await; - - for _ in 0..3 { - context.segment_cache.notify_capacity_protection_changed(); - for _ in 0..32 { - tokio::task::yield_now().await; - } - } - assert_segment_request_counts(&server, 1, "insufficient protection revision").await; - { - let session = context.session.read().await; - assert!([2_u64, 3].into_iter().all(|proxy_seq| { - session.segments.get(&proxy_seq).is_some_and(|segment| segment.status.awaits_capacity_recovery()) - })); - } - - { - let mut session = context.session.write().await; - session.segments.get_mut(&1).expect("completed segment").status = SegmentCacheStatus::Expired; - session.publishable_origin_head_proxy_seq = Some(2); - session.advance_media_readiness_generation(); - } - context.segment_cache.delete(&released_key).await.expect("completed cache object is released"); - - let (first, second) = tokio::time::timeout(Duration::from_secs(10), async { tokio::join!(first, second) }) - .await - .expect("both deferred fetches resume after the accounting revision"); - assert_eq!(first.expect("first demand task joins"), super::SegmentDemandFetchOutcome::Ready); - assert_eq!(second.expect("second demand task joins"), super::SegmentDemandFetchOutcome::Ready); - assert_segment_request_counts(&server, 2, "real capacity release").await; - assert_concurrent_capacity_recovery_ready(&context).await; - } - - #[tokio::test] - async fn failed_retryable_segment_has_a_bounded_demand_requeue_contract() { - let server = spawn_segment_server(0).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..Default::default() }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - clear_scheduled_prefetch(&context, &policy).await; - context.session.write().await.segments.get_mut(&segment_file.proxy_seq).expect("segment").status = - SegmentCacheStatus::FailedRetryable { failed_at_ms: 20, retry_after_ms: 1_000 }; - - let early = worker.demand_fetch_and_wait(context.clone(), &segment_file, 1_019).await; - assert_eq!(early, super::SegmentDemandFetchOutcome::TimedOut); - assert!(server.requests.lock().await.is_empty()); - - let retried = worker.demand_fetch_and_wait(context.clone(), &segment_file, 1_020).await; - assert_eq!(retried, super::SegmentDemandFetchOutcome::Ready); - assert_eq!(server.requests.lock().await.len(), 1); - } - - fn normal_manifest(body: &str) -> tuliprox_parser::hls::origin_manifest::ParsedOriginManifest { - match parse_origin_media_manifest(body, BASE_URL) { - OriginManifestParseOutcome::Normal(manifest) => manifest, - OriginManifestParseOutcome::TransientPassthrough { reason } => { - panic!("expected normal manifest: {reason:?}") - } - } - } - - fn test_segment_repair_manager() -> Arc { - Arc::new(HlsSegmentRepairManager::new(HlsSegmentRepairConfig { - max_level: HlsSegmentRepairMode::Off, - apply_to_first_segments: 1, - max_parallel_repairs: 1, - ..Default::default() - })) - } - - #[test] - fn segment_origin_headers_remove_client_range_and_force_identity() { - let mut headers = HeaderMap::new(); - headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-")); - headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip")); - headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); - headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); - headers.insert(header::HOST, HeaderValue::from_static("origin.example.com")); - let headers = build_segment_origin_headers(&headers, &HeaderMap::new(), None).expect("headers should build"); - - assert!(!headers.contains_key(header::RANGE)); - assert!(!headers.contains_key(header::AUTHORIZATION)); - assert!(!headers.contains_key(header::COOKIE)); - assert!(!headers.contains_key(header::HOST)); - assert_eq!(headers.get(header::ACCEPT_ENCODING).expect("encoding"), "identity"); - } - - #[test] - fn segment_origin_headers_apply_byterange() { - let headers = build_segment_origin_headers( - &HeaderMap::new(), - &HeaderMap::new(), - Some(tuliprox_parser::hls::origin_manifest::ParsedByteRange { offset: 10, length: 5 }), - ) - .expect("headers should build"); - - assert_eq!(headers.get(header::RANGE).expect("range"), "bytes=10-14"); - } - - #[tokio::test] - async fn segment_fetch_snapshot_uses_concrete_final_segment_fetch_url() { - let manifest = match parse_origin_media_manifest( - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:4.0,\nmedia/seg001.ts\n", - "https://cdn.example.net/live/redirected/playlist.m3u8", - ) { - OriginManifestParseOutcome::Normal(manifest) => manifest, - OriginManifestParseOutcome::TransientPassthrough { reason } => { - panic!("expected normal manifest: {reason:?}") - } - }; - let store = HlsSessionStore::new(); - let session = store.get_or_create_session(HlsSessionKey::new(1, "1"), b"secret", 0).await; - { - let mut session = session.write().await; - session.configure_segment_prefetch_queue(SegmentFetchPolicy::default().max_prefetch_queue_depth); - session.apply_origin_manifest(&manifest).expect("manifest maps"); - session.queue_manifest_prefetch_candidates(10); - } - - let worker = Arc::new(HlsSegmentWorkerPool::new(SegmentFetchPolicy::default())); - let proxy_session_id = session.read().await.proxy_session_id.clone(); - grant_usable_worker_access_lease(&worker, &proxy_session_id).await; - let temp_dir = tempfile::tempdir().expect("tempdir"); - let context = SegmentFetchContext { - session: Arc::clone(&session), - segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())), - segment_repair: test_segment_repair_manager(), - repair_access_lease_id: None, - headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::new(), - use_manual_redirects: false, - origin_io: None, - }; - - let snapshot = - worker.next_fetch_snapshot(&context, 11, &SegmentFetchPolicy::default()).await.expect("segment snapshot"); - - assert_eq!(snapshot.fetch_ref.resolved_origin_url, "https://cdn.example.net/live/redirected/media/seg001.ts"); - assert_eq!(snapshot.fetch_ref.byte_range, None); - assert_eq!(snapshot.origin_seq, 10); - } - - struct TestSegmentServer { - base_url: String, - requests: Arc>>, - task: tokio::task::JoinHandle<()>, - } - - impl Drop for TestSegmentServer { - fn drop(&mut self) { self.task.abort(); } - } - - #[derive(Clone)] - struct TestOriginResponse { - status: u16, - headers: Vec<(String, String)>, - body: Vec, - } - - impl TestOriginResponse { - fn encoded(content_encoding: &str, body: Vec) -> Self { - Self { status: 200, headers: vec![("Content-Encoding".to_string(), content_encoding.to_string())], body } - } - } - - #[derive(Debug, Clone, Copy)] - enum TestContentEncoding { - Gzip, - RawDeflate, - Brotli, - Zstd, - } - - impl TestContentEncoding { - const fn header_value(self) -> &'static str { - match self { - Self::Gzip => "gzip", - Self::RawDeflate => "deflate", - Self::Brotli => "br", - Self::Zstd => "zstd", - } - } - } - - async fn encode_test_body(body: &[u8], encoding: TestContentEncoding) -> Vec { - let reader = BufReader::new(Cursor::new(body.to_vec())); - let mut encoded = Vec::new(); - match encoding { - TestContentEncoding::Gzip => GzipEncoder::new(reader).read_to_end(&mut encoded).await, - TestContentEncoding::RawDeflate => DeflateEncoder::new(reader).read_to_end(&mut encoded).await, - TestContentEncoding::Brotli => BrotliEncoder::new(reader).read_to_end(&mut encoded).await, - TestContentEncoding::Zstd => ZstdEncoder::new(reader).read_to_end(&mut encoded).await, - } - .expect("test body encodes"); - encoded - } - - async fn spawn_sequence_response_server(responses: Vec) -> TestSegmentServer { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let addr = listener.local_addr().expect("local addr"); - let requests = Arc::new(Mutex::new(Vec::new())); - let responses = Arc::new(Mutex::new(VecDeque::from(responses))); - let task_requests = Arc::clone(&requests); - let task_responses = Arc::clone(&responses); - let task = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let requests = Arc::clone(&task_requests); - let responses = Arc::clone(&task_responses); - tokio::spawn(async move { - use std::fmt::Write as _; - - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 2048]; - let Ok(read) = socket.read(&mut chunk).await else { - return; - }; - if read == 0 { - return; - } - request.extend_from_slice(&chunk[..read]); - } - requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); - let response = responses.lock().await.pop_front().unwrap_or(TestOriginResponse { - status: 500, - headers: Vec::new(), - body: Vec::new(), - }); - let reason = if response.status == 200 { "OK" } else { "Status" }; - let mut head = - format!("HTTP/1.1 {} {reason}\r\nContent-Length: {}\r\n", response.status, response.body.len()); - for (name, value) in response.headers { - let _ = write!(head, "{name}: {value}\r\n"); - } - head.push_str("Connection: close\r\n\r\n"); - let _ = socket.write_all(head.as_bytes()).await; - let _ = socket.write_all(&response.body).await; - }); - } - }); - - TestSegmentServer { base_url: format!("http://{addr}"), requests, task } - } - - fn temp_cache_files(root: &Path) -> Vec { - fn visit(path: &Path, found: &mut Vec) { - let Ok(entries) = std::fs::read_dir(path) else { - return; - }; - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - visit(&path, found); - } else if path.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.contains(".tmp.")) { - found.push(path); - } - } - } - - let mut found = Vec::new(); - visit(root, &mut found); - found - } - - async fn spawn_segment_server(delay_ms: u64) -> TestSegmentServer { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let addr = listener.local_addr().expect("local addr"); - let requests = Arc::new(Mutex::new(Vec::new())); - let task_requests = Arc::clone(&requests); - let task = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let requests = Arc::clone(&task_requests); - tokio::spawn(async move { - let mut buf = vec![0_u8; 2048]; - let mut used = 0_usize; - loop { - let Ok(read) = socket.read(&mut buf[used..]).await else { - return; - }; - if read == 0 { - return; - } - used += read; - if used >= 4 && buf[..used].windows(4).any(|window| window == b"\r\n\r\n") { - break; - } - if used == buf.len() { - return; - } - } - let request = String::from_utf8_lossy(&buf[..used]).to_string(); - requests.lock().await.push(request.clone()); - if delay_ms > 0 { - tokio::time::sleep(Duration::from_millis(delay_ms)).await; - } - let path = request.lines().next().and_then(|line| line.split_whitespace().nth(1)).unwrap_or("/"); - let body = format!("body:{path}"); - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ); - let _ = socket.write_all(response.as_bytes()).await; - }); - } - }); - - TestSegmentServer { base_url: format!("http://{addr}"), requests, task } - } - - async fn spawn_controlled_segment_server() -> (TestSegmentServer, oneshot::Receiver<()>, oneshot::Sender<()>) { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let addr = listener.local_addr().expect("local addr"); - let requests = Arc::new(Mutex::new(Vec::new())); - let task_requests = Arc::clone(&requests); - let (request_seen_tx, request_seen_rx) = oneshot::channel(); - let (release_tx, release_rx) = oneshot::channel(); - let task = tokio::spawn(async move { - let Ok((mut socket, _)) = listener.accept().await else { - return; - }; - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 2048]; - let Ok(read) = socket.read(&mut chunk).await else { - return; - }; - if read == 0 { - return; - } - request.extend_from_slice(&chunk[..read]); - } - task_requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); - let _ = request_seen_tx.send(()); - if release_rx.await.is_err() { - return; - } - let body = b"controlled-segment"; - let head = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); - let _ = socket.write_all(head.as_bytes()).await; - let _ = socket.write_all(body).await; - }); - (TestSegmentServer { base_url: format!("http://{addr}"), requests, task }, request_seen_rx, release_tx) - } - - async fn spawn_sequence_status_server(statuses: Vec) -> TestSegmentServer { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let addr = listener.local_addr().expect("local addr"); - let requests = Arc::new(Mutex::new(Vec::new())); - let statuses = Arc::new(Mutex::new(VecDeque::from(statuses))); - let task_requests = Arc::clone(&requests); - let task_statuses = Arc::clone(&statuses); - let task = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let requests = Arc::clone(&task_requests); - let statuses = Arc::clone(&task_statuses); - tokio::spawn(async move { - let mut buf = vec![0_u8; 2048]; - let Ok(read) = socket.read(&mut buf).await else { - return; - }; - if read == 0 { - return; - } - let request = String::from_utf8_lossy(&buf[..read]).to_string(); - requests.lock().await.push(request.clone()); - let status = statuses.lock().await.pop_front().unwrap_or(200); - let reason = if status == 200 { "OK" } else { "Error" }; - let body = if status == 200 { "segment-body" } else { "" }; - let response = format!( - "HTTP/1.1 {status} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ); - let _ = socket.write_all(response.as_bytes()).await; - }); - } - }); - - TestSegmentServer { base_url: format!("http://{addr}"), requests, task } - } - - async fn spawn_redirect_retry_server() -> TestSegmentServer { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let addr = listener.local_addr().expect("local addr"); - let requests = Arc::new(Mutex::new(Vec::new())); - let task_requests = Arc::clone(&requests); - let task = tokio::spawn(async move { - let request_count = Arc::new(AtomicUsize::new(0)); - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let requests = Arc::clone(&task_requests); - let request_count = Arc::clone(&request_count); - tokio::spawn(async move { - let mut buf = vec![0_u8; 2048]; - let Ok(read) = socket.read(&mut buf).await else { - return; - }; - if read == 0 { - return; - } - let request = String::from_utf8_lossy(&buf[..read]).to_string(); - let path = request.lines().next().and_then(|line| line.split_whitespace().nth(1)).unwrap_or("/"); - requests.lock().await.push(path.to_string()); - let count = request_count.fetch_add(1, Ordering::SeqCst); - let response = if path == "/1.ts" && count == 0 { - "HTTP/1.1 302 Found\r\nLocation: /redirected.ts\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string() - } else if path == "/redirected.ts" { - "HTTP/1.1 500 Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string() - } else { - "HTTP/1.1 200 OK\r\nContent-Length: 12\r\nConnection: close\r\n\r\nsegment-body".to_string() - }; - let _ = socket.write_all(response.as_bytes()).await; - }); - } - }); - - TestSegmentServer { base_url: format!("http://{addr}"), requests, task } - } - - async fn spawn_range_segment_server() -> TestSegmentServer { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let addr = listener.local_addr().expect("local addr"); - let requests = Arc::new(Mutex::new(Vec::new())); - let task_requests = Arc::clone(&requests); - let task = tokio::spawn(async move { - let Ok((mut socket, _)) = listener.accept().await else { - return; - }; - let mut buf = vec![0_u8; 2048]; - let Ok(read) = socket.read(&mut buf).await else { - return; - }; - if read == 0 { - return; - } - let request = String::from_utf8_lossy(&buf[..read]).to_string(); - task_requests.lock().await.push(request.clone()); - let body = if request.to_ascii_lowercase().contains("range: bytes=10-14") { "seg!!" } else { "bad" }; - let status = if body == "seg!!" { "206 Partial Content" } else { "200 OK" }; - let response = format!( - "HTTP/1.1 {status}\r\nContent-Length: {}\r\nContent-Range: bytes 10-14/20\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - let _ = socket.write_all(response.as_bytes()).await; - }); - - TestSegmentServer { base_url: format!("http://{addr}"), requests, task } - } - - async fn grant_usable_worker_access_lease(worker: &HlsSegmentWorkerPool, proxy_session_id: &ProxySessionId) { - grant_usable_worker_access_lease_at(worker, proxy_session_id, 10).await; - } - - async fn grant_usable_worker_access_lease_at( - worker: &HlsSegmentWorkerPool, - proxy_session_id: &ProxySessionId, - now_ms: u64, - ) { - worker.access_leases().write().await.prepare_access_lease(HlsAccessLease::pending( - HlsAccessLeaseId("worker-lease".to_string()), - HlsPlaybackFamilyKey::new("alice", "client-a"), - proxy_session_id.clone(), - "alice".to_string(), - "session-a".to_string(), - 1, - "12345".to_string(), - 12345, - now_ms, - 15_000, - )); - } - - async fn fetch_context_with_access_lease( - server: &TestSegmentServer, - temp_dir: &tempfile::TempDir, - policy: &SegmentFetchPolicy, - grant_access_lease: bool, - ) -> (Arc, SegmentFetchContext, HlsSegmentFile) { - let store = HlsSessionStore::new(); - let session = store.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; - { - let mut session = session.write().await; - session.configure_segment_prefetch_queue(policy.max_prefetch_queue_depth); - session.proxy_next_seq = Some(1); - session - .apply_origin_manifest(&normal_manifest(&format!( - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\n{}/1.ts\n#EXTINF:4.0,\n{}/2.ts\n#EXTINF:4.0,\n{}/3.ts\n", - server.base_url, server.base_url, server.base_url - ))) - .expect("manifest maps"); - session.queue_manifest_prefetch_candidates(10); - } - let worker = Arc::new(HlsSegmentWorkerPool::new(policy.clone())); - if grant_access_lease { - let proxy_session_id = session.read().await.proxy_session_id.clone(); - grant_usable_worker_access_lease(&worker, &proxy_session_id).await; - } - let context = SegmentFetchContext { - session: Arc::clone(&session), - segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())), - segment_repair: test_segment_repair_manager(), - repair_access_lease_id: None, - headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("client builds"), - use_manual_redirects: true, - origin_io: None, - }; - (worker, context, HlsSegmentFile { proxy_seq: 1, extension: "ts".to_string() }) - } - - async fn fetch_context( - server: &TestSegmentServer, - temp_dir: &tempfile::TempDir, - policy: &SegmentFetchPolicy, - ) -> (Arc, SegmentFetchContext, HlsSegmentFile) { - fetch_context_with_access_lease(server, temp_dir, policy, true).await - } - - async fn encrypted_fetch_context( - server: &TestSegmentServer, - temp_dir: &tempfile::TempDir, - policy: &SegmentFetchPolicy, - ) -> (Arc, SegmentFetchContext, HlsSegmentFile) { - let store = HlsSessionStore::new(); - let session = store.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; - let key_uri = format!("{}/key.key", server.base_url); - let key_resource = TransientResourceRef::new( - TransientResourceKind::Key, - &key_uri, - b"secret", - 0, - u64::MAX, - Some("key".to_string()), - ); - let mut manifest = normal_manifest(&format!( - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-KEY:METHOD=AES-128,URI=\"{key_uri}\"\n\ - #EXTINF:4.0,\n{}/1.ts\n#EXTINF:4.0,\n{}/2.ts\n#EXTINF:4.0,\n{}/3.ts\n", - server.base_url, server.base_url, server.base_url - )); - for encryption in manifest.segments.iter_mut().filter_map(|segment| segment.encryption.as_mut()) { - encryption.proxy_resource_id = Some(key_resource.id.0.clone()); - encryption.proxy_resource_extension = Some("key".to_string()); - } - { - let mut session = session.write().await; - session.configure_segment_prefetch_queue(policy.max_prefetch_queue_depth); - session.proxy_next_seq = Some(1); - session.transient.upsert_resources([key_resource]); - session.apply_origin_manifest(&manifest).expect("encrypted manifest maps"); - } - let worker = Arc::new(HlsSegmentWorkerPool::new(policy.clone())); - let context = SegmentFetchContext { - session: Arc::clone(&session), - segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())), - segment_repair: test_segment_repair_manager(), - repair_access_lease_id: None, - headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::new(), - use_manual_redirects: false, - origin_io: None, - }; - (worker, context, HlsSegmentFile { proxy_seq: 1, extension: "ts".to_string() }) - } - - async fn shared_key_fetch_and_wait( - context: &SegmentFetchContext, - now_ms: u64, - ) -> (TransientObjectFetchToken, Arc, TransientResourceId, String) { - let mut session = context.session.write().await; - let resource = session - .transient - .resources - .values() - .find(|resource| resource.kind == TransientResourceKind::Key) - .cloned() - .expect("encrypted fixture key resource"); - let extension = resource.file_ext_hint.clone().expect("key resource extension"); - let proxy_session_id = session.proxy_session_id.clone(); - let cache_duration_ms = session.transient.resource_ttl_ms; - let fetch_token = match session.transient.begin_object_fetch( - &proxy_session_id, - &resource, - &extension, - now_ms, - cache_duration_ms, - ) { - TransientObjectFetchDecision::Fetch(token) => token, - TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { - panic!("first shared-key decision must own the fetch") - } - }; - let notifier = match session.transient.begin_object_fetch( - &proxy_session_id, - &resource, - &extension, - now_ms, - cache_duration_ms, - ) { - TransientObjectFetchDecision::Wait(notifier) => notifier, - TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Fetch(_) => { - panic!("second shared-key decision must wait for the existing fetch") - } - }; - (*fetch_token, notifier, resource.id, extension) - } - - async fn commit_test_key_ready(context: &SegmentFetchContext, token: &TransientObjectFetchToken, now_ms: u64) { - assert!(context.session.write().await.commit_transient_object_ready_if_current( - TransientResourceKind::Key, - token, - "application/octet-stream".to_string(), - 16, - now_ms, - u64::MAX, - )); - } - - async fn clear_scheduled_prefetch(context: &SegmentFetchContext, policy: &SegmentFetchPolicy) { - let mut session = context.session.write().await; - session.segment_prefetch_queue = crate::SegmentPrefetchQueue::new(policy.max_prefetch_queue_depth); - for segment in session.segments.values_mut() { - if !matches!(segment.status, SegmentCacheStatus::Ready { .. }) { - segment.status = SegmentCacheStatus::Discovered; - } - } - } - - async fn assert_only_one_prefetch_is_active_before_controlled_release(policy: SegmentFetchPolicy) { - let (server, request_seen, release_response) = spawn_controlled_segment_server().await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let (worker, context, _) = fetch_context(&server, &temp_dir, &policy).await; - - worker.wake_scheduler(context.clone(), 20).await; - tokio::time::timeout(Duration::from_secs(10), request_seen) - .await - .expect("origin request starts before test deadline") - .expect("controlled origin observes the request"); - - let (active_proxy_seq, completion_notifier) = { - let mut session = context.session.write().await; - let fetching_proxy_seqs = session - .segments - .iter() - .filter_map(|(proxy_seq, segment)| { - matches!(segment.status, SegmentCacheStatus::Fetching { .. }).then_some(*proxy_seq) - }) - .collect::>(); - assert_eq!(session.active_segment_fetches, 1); - assert_eq!(fetching_proxy_seqs.len(), 1); - let active_proxy_seq = fetching_proxy_seqs[0]; - let completion_notifier = session.segment_fetch_notifiers.entry(active_proxy_seq).or_default().clone(); - session.invalidate_queued_origin_work(); - (active_proxy_seq, completion_notifier) - }; - assert_eq!(server.requests.lock().await.len(), 1); - - let completion = completion_notifier.notified(); - tokio::pin!(completion); - completion.as_mut().enable(); - release_response.send(()).expect("controlled origin response releases"); - tokio::time::timeout(Duration::from_secs(10), completion.as_mut()) - .await - .expect("active prefetch completes before test deadline"); - - let session = context.session.read().await; - assert_eq!(session.active_segment_fetches, 0); - assert!(matches!( - session.segments.get(&active_proxy_seq).map(|segment| &segment.status), - Some(SegmentCacheStatus::Discovered) - )); - } - - async fn committed_segment(context: &SegmentFetchContext, proxy_seq: u64) -> (crate::SegmentCacheKey, Vec) { - let cache_key = context.session.read().await.segments.get(&proxy_seq).expect("segment").cache_key.clone(); - let metadata = context - .segment_cache - .metadata(&cache_key) - .await - .expect("cache metadata reads") - .expect("cache object exists"); - let bytes = tokio::fs::read(metadata.path).await.expect("cache object reads"); - (cache_key, bytes) - } - - #[tokio::test] - async fn demand_fetch_writes_cache_and_sets_ready_after_commit() { - let server = spawn_segment_server(0).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - - let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); - let session = context.session.read().await; - assert!(matches!(session.segments.get(&1).expect("segment").status, SegmentCacheStatus::Ready { .. })); - } - - #[tokio::test] - async fn projected_capacity_reclamation_commits_one_origin_download() { - let server = spawn_sequence_response_server(vec![TestOriginResponse { - status: 200, - headers: Vec::new(), - body: b"abcdefghij".to_vec(), - }]) - .await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let store = Arc::new(HlsSessionStore::new()); - let session = store.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; - { - let mut session = session.write().await; - session.configure_segment_prefetch_queue(policy.max_prefetch_queue_depth); - session.proxy_next_seq = Some(1); - session - .apply_origin_manifest(&normal_manifest(&format!( - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\n{0}/1.ts\n#EXTINF:4.0,\n{0}/2.ts\n#EXTINF:4.0,\n{0}/3.ts\n#EXTINF:4.0,\n{0}/4.ts\n#EXTINF:4.0,\n{0}/5.ts\n#EXTINF:4.0,\n{0}/6.ts\n", - server.base_url - ))) - .expect("manifest maps"); - } - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let gc = Arc::new(HlsGarbageCollector::new( - Arc::clone(&store), - Arc::clone(&cache), - GarbageCollectionPolicy::default(), - build_rewrite_secret_fingerprint(b"secret"), - )); - cache.install_capacity_reclaimer(&gc); - let old_keys = { - let session = session.read().await; - assert!(session.segments.contains_key(&6), "mapped proxy sequences: {:?}", session.segments.keys()); - [1_u64, 2].map(|proxy_seq| session.segments.get(&proxy_seq).expect("old segment").cache_key.clone()) - }; - for key in &old_keys { - cache.write_bytes_and_commit(key, b"0123456789").await.expect("old segment commits"); - } - { - let mut session = session.write().await; - for proxy_seq in [1_u64, 2] { - session.segments.get_mut(&proxy_seq).expect("old segment").status = - SegmentCacheStatus::Ready { content_length: 10, ready_at_ms: proxy_seq }; - } - } - cache.update_cache_limits(100, 25); - let worker = Arc::new(HlsSegmentWorkerPool::new(policy.clone())); - let proxy_session_id = session.read().await.proxy_session_id.clone(); - grant_usable_worker_access_lease(&worker, &proxy_session_id).await; - let context = SegmentFetchContext { - session: Arc::clone(&session), - segment_cache: Arc::clone(&cache), - segment_repair: test_segment_repair_manager(), - repair_access_lease_id: None, - headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("client builds"), - use_manual_redirects: true, - origin_io: None, - }; - clear_scheduled_prefetch(&context, &policy).await; - - let outcome = worker - .demand_fetch_and_wait(context.clone(), &HlsSegmentFile { proxy_seq: 6, extension: "ts".to_string() }, 20) - .await; - - let target_status = format!("{:?}", session.read().await.segments.get(&6).map(|segment| &segment.status)); - assert_eq!( - outcome, - super::SegmentDemandFetchOutcome::Ready, - "requests={} target_status={target_status}", - server.requests.lock().await.len(), - ); - assert_eq!(server.requests.lock().await.len(), 1, "staged body is not downloaded again"); - assert_eq!(committed_segment(&context, 6).await.1, b"abcdefghij"); - let session = session.read().await; - assert!(!session.segments.contains_key(&1)); - assert!(session.segments.contains_key(&2)); - drop(session); - let usage = cache.capacity_usage(&proxy_session_id).await.expect("capacity usage"); - assert_eq!(usage.session_bytes, 20); - assert!(!cache.has_active_temp_files()); - } - - #[tokio::test] - async fn encrypted_segment_fetch_commits_exact_key_before_media_becomes_ready() { - let server = spawn_sequence_response_server(vec![ - TestOriginResponse { status: 200, headers: Vec::new(), body: b"0123456789abcdef".to_vec() }, - TestOriginResponse { status: 200, headers: Vec::new(), body: b"encrypted-media".to_vec() }, - ]) - .await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = - SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; - let (worker, context, segment_file) = encrypted_fetch_context(&server, &temp_dir, &policy).await; - - let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); - let session = context.session.read().await; - let first = session.segments.get(&1).expect("encrypted segment"); - assert!(matches!(first.status, SegmentCacheStatus::Ready { .. })); - let encryption = first.encryption.as_ref().expect("segment keeps key dependency"); - assert!(session - .transient - .ready_key_object_valid_until_ms( - &session.proxy_session_id, - &encryption.resource_id, - &encryption.resource_extension, - 20, - ) - .is_some()); - assert!(session.ready_timeline_snapshot(1, 20).units[0].required_key_ready); - assert!(session.activity.media_readiness_generation >= 2); - drop(session); - let requests = server.requests.lock().await; - assert!(requests[0].starts_with("GET /key.key ")); - assert!(requests[1].starts_with("GET /1.ts ")); - } - - #[tokio::test] - async fn expired_aes_key_is_refetched_before_a_second_segment_becomes_ready() { - let server = spawn_sequence_response_server(vec![ - TestOriginResponse { status: 200, headers: Vec::new(), body: b"0123456789abcdef".to_vec() }, - TestOriginResponse { status: 200, headers: Vec::new(), body: b"encrypted-media-a".to_vec() }, - TestOriginResponse { status: 200, headers: Vec::new(), body: b"fedcba9876543210".to_vec() }, - TestOriginResponse { status: 200, headers: Vec::new(), body: b"encrypted-media-b".to_vec() }, - ]) - .await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = - SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; - let (worker, context, _) = encrypted_fetch_context(&server, &temp_dir, &policy).await; - { - let mut session = context.session.write().await; - session.transient.set_resource_ttl_ms(1); - } - - let first_segment = HlsSegmentFile { proxy_seq: 1, extension: "ts".to_string() }; - let first_outcome = worker.demand_fetch_and_wait(context.clone(), &first_segment, current_time_millis()).await; - assert_eq!(first_outcome, super::SegmentDemandFetchOutcome::Ready); - - tokio::time::sleep(Duration::from_millis(50)).await; - { - let session = context.session.read().await; - let key = session - .transient - .resources - .values() - .find(|resource| resource.kind == TransientResourceKind::Key) - .cloned() - .expect("encrypted fixture key resource"); - assert!(session - .transient - .ready_key_object_valid_until_ms( - &session.proxy_session_id, - &key.id, - key.file_ext_hint.as_deref().expect("key extension"), - current_time_millis(), - ) - .is_none()); - } - - let second_segment = HlsSegmentFile { proxy_seq: 2, extension: "ts".to_string() }; - let second_outcome = - worker.demand_fetch_and_wait(context.clone(), &second_segment, current_time_millis()).await; - assert_eq!(second_outcome, super::SegmentDemandFetchOutcome::Ready); - - { - let session = context.session.read().await; - assert!(matches!( - session.segments.get(&2).map(|segment| &segment.status), - Some(SegmentCacheStatus::Ready { .. }) - )); - } - - let requests = server.requests.lock().await; - assert_eq!(requests.len(), 4, "requests={requests:?}"); - assert!(requests[0].starts_with("GET /key.key ")); - assert!(requests[1].starts_with("GET /1.ts ")); - assert!(requests[2].starts_with("GET /key.key ")); - assert!(requests[3].starts_with("GET /2.ts ")); - } - - #[tokio::test] - async fn ready_encrypted_segments_schedule_one_key_only_fetch() { - let server = spawn_sequence_response_server(vec![TestOriginResponse { - status: 200, - headers: Vec::new(), - body: b"0123456789abcdef".to_vec(), - }]) - .await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = - SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; - let (worker, context, _) = encrypted_fetch_context(&server, &temp_dir, &policy).await; - let proxy_session_id = { - let mut session = context.session.write().await; - for segment in session.segments.values_mut() { - segment.status = SegmentCacheStatus::Ready { content_length: 32, ready_at_ms: 10 }; - } - session.last_rendered_manifest = Some(RenderedManifest { - body: "#EXTM3U\n".to_string(), - first_proxy_seq: 1, - last_proxy_seq: 3, - discontinuity_sequence: 0, - target_duration_ms: 4_000, - playlist_duration_ms: 12_000, - valid_until_ms: u64::MAX, - render_gap_segments: 0, - rendered_at_ms: 10, - segment_proxy_seqs: vec![1, 2, 3], - }); - session.proxy_session_id.clone() - }; - let now_ms = current_time_millis(); - grant_usable_worker_access_lease_at(&worker, &proxy_session_id, now_ms).await; - - tokio::join!(worker.wake_scheduler(context.clone(), now_ms), worker.wake_scheduler(context.clone(), now_ms)); - - let pending_key = { - let mut session = context.session.write().await; - let resource = session - .transient - .resources - .values() - .find(|resource| resource.kind == TransientResourceKind::Key) - .cloned() - .expect("encrypted fixture key resource"); - let extension = resource.file_ext_hint.clone().expect("key extension"); - let proxy_session_id = session.proxy_session_id.clone(); - let cache_duration_ms = session.transient.resource_ttl_ms; - match session.transient.begin_object_fetch( - &proxy_session_id, - &resource, - &extension, - now_ms, - cache_duration_ms, - ) { - TransientObjectFetchDecision::Ready => None, - TransientObjectFetchDecision::Wait(notifier) => Some((notifier, resource.id, extension)), - TransientObjectFetchDecision::Fetch(_) => { - panic!("rendered READY media must already have scheduled its key-only fetch") - } - } - }; - if let Some((notifier, resource_id, extension)) = pending_key { - tokio::time::timeout( - Duration::from_secs(10), - wait_for_segment_key_dependency(&context, notifier, resource_id, extension, Duration::from_secs(10)), - ) - .await - .expect("key-only readiness completes before test deadline") - .expect("key-only readiness succeeds"); - } - - let session = context.session.read().await; - assert_eq!(session.active_segment_fetches, 0); - assert!(session.ready_timeline_snapshot(1, now_ms).units.iter().all(|unit| unit.required_key_ready)); - assert!(session.segments.values().all(|segment| matches!(segment.status, SegmentCacheStatus::Ready { .. }))); - drop(session); - let requests = server.requests.lock().await; - assert_eq!(requests.len(), 1); - assert!(requests[0].starts_with("GET /key.key ")); - } - - #[tokio::test] - async fn completed_shared_key_before_wait_registration_is_observed_without_timeout() { - let server = spawn_segment_server(0).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy::default(); - let (_, context, _) = encrypted_fetch_context(&server, &temp_dir, &policy).await; - let now_ms = current_time_millis(); - let (fetch_token, notifier, resource_id, extension) = shared_key_fetch_and_wait(&context, now_ms).await; - commit_test_key_ready(&context, &fetch_token, now_ms.saturating_add(1)).await; - - let result = - wait_for_segment_key_dependency(&context, notifier, resource_id, extension, Duration::from_millis(1)).await; - - assert!(result.is_ok()); - } - - #[tokio::test] - async fn registered_shared_key_waiter_observes_controlled_ready_transition() { - let server = spawn_segment_server(0).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy::default(); - let (_, context, _) = encrypted_fetch_context(&server, &temp_dir, &policy).await; - let now_ms = current_time_millis(); - let (fetch_token, notifier, resource_id, extension) = shared_key_fetch_and_wait(&context, now_ms).await; - let mut wait = Box::pin(wait_for_segment_key_dependency( - &context, - notifier, - resource_id, - extension, - Duration::from_secs(1), - )); - assert!(matches!(poll!(wait.as_mut()), Poll::Pending)); - - commit_test_key_ready(&context, &fetch_token, now_ms.saturating_add(1)).await; - - assert!(matches!(poll!(wait.as_mut()), Poll::Ready(Ok(())))); - } - - #[tokio::test] - async fn invalid_aes_key_size_blocks_segment_fetch_and_ready_reserve() { - let server = spawn_sequence_response_server(vec![TestOriginResponse { - status: 200, - headers: Vec::new(), - body: vec![b'k'; 15], - }]) - .await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = - SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; - let (worker, context, segment_file) = encrypted_fetch_context(&server, &temp_dir, &policy).await; - - let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Unavailable); - let session = context.session.read().await; - let first = session.segments.get(&1).expect("encrypted segment"); - assert!(matches!(first.status, SegmentCacheStatus::FailedPermanent { .. })); - assert!(!session.ready_timeline_snapshot(1, 20).units[0].required_key_ready); - assert_eq!(session.activity.media_readiness_generation, 0); - drop(session); - let requests = server.requests.lock().await; - assert_eq!(requests.len(), 1); - assert!(requests[0].starts_with("GET /key.key ")); - } - - #[tokio::test] - async fn invalidated_generation_discards_late_segment_completion_without_sleep() { - let (server, request_seen, release_response) = spawn_controlled_segment_server().await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - clear_scheduled_prefetch(&context, &policy).await; - let fetch_context = context.clone(); - let fetch = tokio::spawn(async move { worker.demand_fetch_and_wait(fetch_context, &segment_file, 20).await }); - - request_seen.await.expect("origin request starts"); - context.session.write().await.invalidate_queued_origin_work(); - release_response.send(()).expect("origin response released"); - - assert_eq!(fetch.await.expect("fetch task joins"), super::SegmentDemandFetchOutcome::Unavailable); - let cache_key = { - let session = context.session.read().await; - assert_eq!(session.active_segment_fetches, 0); - let segment = session.segments.get(&1).expect("segment remains mapped"); - assert!(matches!(segment.status, SegmentCacheStatus::Discovered)); - segment.cache_key.clone() - }; - assert!(context.segment_cache.metadata(&cache_key).await.expect("metadata reads").is_none()); - } - - #[tokio::test] - async fn same_origin_sequence_resource_conflict_preserves_in_flight_binding() { - let (server, request_seen, release_response) = spawn_controlled_segment_server().await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - clear_scheduled_prefetch(&context, &policy).await; - let fetch_context = context.clone(); - let fetch = tokio::spawn(async move { worker.demand_fetch_and_wait(fetch_context, &segment_file, 20).await }); - - request_seen.await.expect("origin request starts"); - let key_uri = format!("{}/rebound.key", server.base_url); - let key_resource = TransientResourceRef::new( - TransientResourceKind::Key, - &key_uri, - b"secret", - 0, - u64::MAX, - Some("key".to_string()), - ); - let mut rebound_manifest = normal_manifest(&format!( - "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-KEY:METHOD=AES-128,URI=\"{key_uri}\"\n\ - #EXTINF:4.0,\n{}/rebound.ts\n#EXT-X-KEY:METHOD=NONE\n#EXTINF:4.0,\n{}/2.ts\n\ - #EXTINF:4.0,\n{}/3.ts\n", - server.base_url, server.base_url, server.base_url - )); - for encryption in rebound_manifest.segments.iter_mut().filter_map(|segment| segment.encryption.as_mut()) { - encryption.proxy_resource_id = Some(key_resource.id.0.clone()); - encryption.proxy_resource_extension = Some("key".to_string()); - } - let (origin_work_generation, cache_key) = { - let mut session = context.session.write().await; - let origin_work_generation = session.activity.origin_work_generation; - session.transient.upsert_resources([key_resource]); - assert!(matches!( - session.apply_origin_manifest(&rebound_manifest), - Err(TimelineMapError::OriginSequenceResourceConflict { candidate_origin_seq: 1, .. }) - )); - assert_eq!(session.activity.origin_work_generation, origin_work_generation); - let segment = session.segments.get(&1).expect("original proxy segment remains mapped"); - assert!(matches!(segment.status, SegmentCacheStatus::Fetching { .. })); - assert_eq!( - segment.origin_fetch_ref.as_ref().expect("original fetch ref").resolved_origin_url, - format!("{}/1.ts", server.base_url) - ); - assert!(segment.encryption.is_none()); - (origin_work_generation, segment.cache_key.clone()) - }; - - release_response.send(()).expect("origin response released"); - - assert_eq!(fetch.await.expect("fetch task joins"), super::SegmentDemandFetchOutcome::Ready); - let session = context.session.read().await; - assert_eq!(session.activity.origin_work_generation, origin_work_generation); - assert_eq!(session.active_segment_fetches, 0); - let segment = session.segments.get(&1).expect("original segment remains mapped"); - assert!(matches!(segment.status, SegmentCacheStatus::Ready { .. })); - assert_eq!( - segment.origin_fetch_ref.as_ref().expect("original fetch ref remains authoritative").resolved_origin_url, - format!("{}/1.ts", server.base_url) - ); - assert!(segment.encryption.is_none()); - drop(session); - let (committed_key, bytes) = committed_segment(&context, 1).await; - assert_eq!(committed_key, cache_key); - assert_eq!(bytes, b"controlled-segment"); - assert!(temp_cache_files(temp_dir.path()).is_empty()); - } - - #[tokio::test] - async fn segment_declared_codings_decode_to_identity_before_cache_and_preserve_opaque_bytes() { - let identity_bytes = b"\x00\xffopaque-hls-ciphertext\x1f\x8bmedia".to_vec(); - - for encoding in [ - TestContentEncoding::Gzip, - TestContentEncoding::RawDeflate, - TestContentEncoding::Brotli, - TestContentEncoding::Zstd, - ] { - let encoded = encode_test_body(&identity_bytes, encoding).await; - let server = - spawn_sequence_response_server(vec![TestOriginResponse::encoded(encoding.header_value(), encoded)]) - .await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - clear_scheduled_prefetch(&context, &policy).await; - - let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready, "failed for {encoding:?}"); - let (cache_key, cached) = committed_segment(&context, 1).await; - assert_eq!(cached, identity_bytes, "cache retained HTTP coding for {encoding:?}"); - let mut range = context.segment_cache.open_range(&cache_key, 5).await.expect("cache range opens"); - let mut ranged = Vec::new(); - range.read_to_end(&mut ranged).await.expect("cache range reads"); - assert_eq!(ranged, identity_bytes[5..], "range did not address Identity bytes for {encoding:?}"); - let requests = server.requests.lock().await; - assert_eq!(requests.len(), 1, "unexpected retries for {encoding:?}"); - assert!( - requests[0].to_ascii_lowercase().contains("accept-encoding: identity"), - "request did not enforce identity for {encoding:?}" - ); - } - } - - #[tokio::test] - async fn segment_decoder_failure_retries_then_commits_and_cleans_temp_file() { - let identity_bytes = b"identity-media-after-retry".to_vec(); - let valid = encode_test_body(&identity_bytes, TestContentEncoding::Gzip).await; - let mut corrupt = valid.clone(); - corrupt.truncate(corrupt.len() / 2); - let server = spawn_sequence_response_server(vec![ - TestOriginResponse::encoded("gzip", corrupt), - TestOriginResponse::encoded("gzip", valid), - ]) - .await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - clear_scheduled_prefetch(&context, &policy).await; - - let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); - assert_eq!(committed_segment(&context, 1).await.1, identity_bytes); - let requests = server.requests.lock().await; - assert_eq!(requests.len(), 2); - assert!(requests.iter().all(|request| request.to_ascii_lowercase().contains("accept-encoding: identity"))); - drop(requests); - assert!(!context.segment_cache.has_active_temp_files()); - assert!(temp_cache_files(temp_dir.path()).is_empty()); - } - - #[tokio::test] - async fn segment_decoder_failure_exhausts_attempt_budget_without_cache_or_temp_file() { - let valid = encode_test_body(b"never committed", TestContentEncoding::Gzip).await; - let mut corrupt = valid; - corrupt.truncate(corrupt.len() / 2); - let server = spawn_sequence_response_server( - (0..5).map(|_| TestOriginResponse::encoded("gzip", corrupt.clone())).collect(), - ) - .await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - clear_scheduled_prefetch(&context, &policy).await; - - let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::TimedOut); - assert_eq!(server.requests.lock().await.len(), 5); - let cache_key = context.session.read().await.segments.get(&1).expect("segment").cache_key.clone(); - assert!(context.segment_cache.metadata(&cache_key).await.expect("metadata reads").is_none()); - assert!(!context.segment_cache.has_active_temp_files()); - assert!(temp_cache_files(temp_dir.path()).is_empty()); - } - - #[tokio::test] - async fn segment_decoded_object_limit_is_authoritative_and_non_retryable() { - let identity_bytes = vec![b'x'; 512]; - let encoded = encode_test_body(&identity_bytes, TestContentEncoding::Gzip).await; - assert!(encoded.len() < 64, "fixture must be smaller on the wire than the decoded limit"); - let server = spawn_sequence_response_server(vec![TestOriginResponse::encoded("gzip", encoded)]).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - context.segment_cache.update_cache_limits(64, 64); - clear_scheduled_prefetch(&context, &policy).await; - - let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Unavailable); - assert_eq!(server.requests.lock().await.len(), 1); - let cache_key = context.session.read().await.segments.get(&1).expect("segment").cache_key.clone(); - assert!(context.segment_cache.metadata(&cache_key).await.expect("metadata reads").is_none()); - assert!(!context.segment_cache.has_active_temp_files()); - assert!(temp_cache_files(temp_dir.path()).is_empty()); - } - - #[tokio::test] - async fn demand_fetch_is_blocked_for_gc_marked_session() { - let server = spawn_segment_server(0).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - context.session.write().await.mark_for_gc_removal(); - - let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::NotFound); - assert!(server.requests.lock().await.is_empty()); - } - - #[tokio::test] - async fn background_segment_fetch_without_usable_access_lease_resets_queue_without_origin_request() { - let server = spawn_segment_server(0).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, _) = fetch_context_with_access_lease(&server, &temp_dir, &policy, false).await; - - worker.wake_scheduler(context.clone(), 20).await; - - assert!(server.requests.lock().await.is_empty()); - let session = context.session.read().await; - assert_eq!(session.active_segment_fetches, 0); - assert!(session.segment_prefetch_queue.is_empty()); - assert!(session.segments.values().all(|segment| matches!(segment.status, SegmentCacheStatus::Discovered))); - } - - #[tokio::test] - async fn demand_fetch_starts_without_worker_usable_access_lease() { - let server = spawn_segment_server(0).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context_with_access_lease(&server, &temp_dir, &policy, false).await; - clear_scheduled_prefetch(&context, &policy).await; - - let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); - assert_eq!(server.requests.lock().await.len(), 1); - } - - #[tokio::test] - async fn demand_fetch_returns_unavailable_when_fetch_slots_are_saturated() { - let server = spawn_segment_server(0).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - max_global_segment_fetches: 1, - max_session_segment_fetches: 1, - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - clear_scheduled_prefetch(&context, &policy).await; - context.session.write().await.active_segment_fetches = 1; - - let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Unavailable); - assert!(server.requests.lock().await.is_empty()); - } - - #[tokio::test] - async fn ready_cache_hit_is_allowed_when_fetch_slots_are_saturated() { - let server = spawn_segment_server(0).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - max_global_segment_fetches: 1, - max_session_segment_fetches: 1, - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - { - let mut session = context.session.write().await; - session.active_segment_fetches = 1; - session.segments.get_mut(&1).expect("segment").status = - SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 20 }; - } - - let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); - assert!(server.requests.lock().await.is_empty()); - } - - #[tokio::test] - async fn origin_byterange_segment_fetch_uses_http_range_and_stores_logical_segment() { - let server = spawn_range_segment_server().await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - origin_segment_timeout_ms: 1_000, - ..SegmentFetchPolicy::default() - }; - let store = HlsSessionStore::new(); - let session = store.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; - { - let mut session = session.write().await; - session - .apply_origin_manifest(&normal_manifest(&format!( - "#EXTM3U\n#EXT-X-BYTERANGE:5@10\n#EXTINF:4.0,\n{}/big.m4s\n", - server.base_url - ))) - .expect("manifest maps"); - } - let context = SegmentFetchContext { - session: Arc::clone(&session), - segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())), - segment_repair: test_segment_repair_manager(), - repair_access_lease_id: None, - headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("client builds"), - use_manual_redirects: true, - origin_io: None, - }; - let worker = Arc::new(HlsSegmentWorkerPool::new(policy)); - let segment_file = HlsSegmentFile { proxy_seq: 0, extension: "m4s".to_string() }; - - let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); - let requests = server.requests.lock().await; - assert_eq!(requests.len(), 1); - assert!(requests[0].to_ascii_lowercase().contains("range: bytes=10-14")); - let session = context.session.read().await; - let segment = session.segments.get(&0).expect("segment"); - assert!(matches!(segment.status, SegmentCacheStatus::Ready { content_length: 5, .. })); - assert!(context.segment_cache.metadata(&segment.cache_key).await.expect("metadata").is_some()); - } - - #[tokio::test] - async fn one_proxy_sequence_has_at_most_one_active_origin_fetch() { - let server = spawn_segment_server(80).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - max_global_segment_fetches: 4, - max_session_segment_fetches: 4, - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - clear_scheduled_prefetch(&context, &policy).await; - - let first = { - let worker = Arc::clone(&worker); - let context = context.clone(); - let segment_file = segment_file.clone(); - tokio::spawn(async move { worker.demand_fetch_and_wait(context, &segment_file, 20).await }) - }; - let second = { - let worker = Arc::clone(&worker); - let context = context.clone(); - let segment_file = segment_file.clone(); - tokio::spawn(async move { worker.demand_fetch_and_wait(context, &segment_file, 21).await }) - }; - - assert_eq!(first.await.expect("task"), super::SegmentDemandFetchOutcome::Ready); - assert_eq!(second.await.expect("task"), super::SegmentDemandFetchOutcome::Ready); - assert_eq!(server.requests.lock().await.len(), 1); - } - - #[tokio::test] - async fn retryable_407_retries_segment_fetch_until_success() { - let server = spawn_sequence_status_server(vec![407, 200]).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - clear_scheduled_prefetch(&context, &policy).await; - - let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); - assert_eq!(server.requests.lock().await.len(), 2); - } - - #[tokio::test] - async fn permanent_404_does_not_retry_segment_fetch() { - let server = spawn_sequence_status_server(vec![404, 200]).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - clear_scheduled_prefetch(&context, &policy).await; - - let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Unavailable); - assert_eq!(server.requests.lock().await.len(), 1); - } - - #[tokio::test] - async fn segment_retry_starts_again_at_fetch_ref_after_redirect_failure() { - let server = spawn_redirect_retry_server().await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - clear_scheduled_prefetch(&context, &policy).await; - - let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); - assert_eq!(server.requests.lock().await.as_slice(), ["/1.ts", "/redirected.ts", "/1.ts"]); - } - - #[tokio::test] - async fn session_limit_is_respected() { - let policy = SegmentFetchPolicy { - max_global_segment_fetches: 4, - max_session_segment_fetches: 1, - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - - assert_only_one_prefetch_is_active_before_controlled_release(policy).await; - } - - #[tokio::test] - async fn global_limit_is_respected() { - let policy = SegmentFetchPolicy { - max_global_segment_fetches: 1, - max_session_segment_fetches: 3, - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - - assert_only_one_prefetch_is_active_before_controlled_release(policy).await; - } - - #[tokio::test] - async fn demand_priority_runs_before_prefetch() { - let server = spawn_segment_server(0).await; - let temp_dir = tempfile::tempdir().expect("temp dir"); - let policy = SegmentFetchPolicy { - max_global_segment_fetches: 1, - max_session_segment_fetches: 1, - retry_delays_ms: [0, 0, 0, 0, 0], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }; - let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; - { - let mut session = context.session.write().await; - session.segment_prefetch_queue = crate::SegmentPrefetchQueue::new(6); - session.segments.get_mut(&1).expect("segment").status = SegmentCacheStatus::Discovered; - session.segments.get_mut(&2).expect("segment").status = SegmentCacheStatus::Discovered; - session.queue_segment_fetch_candidate(2, SegmentFetchPriority::Prefetch, 10); - } - - let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; - - assert_eq!(outcome, super::SegmentDemandFetchOutcome::Ready); - let requests = server.requests.lock().await; - let first_request = requests.first().expect("request should be made"); - assert!(first_request.starts_with("GET /1.ts ")); - } -} +use self::commit::commit_failed_segment_fetch; +pub(crate) use self::commit::recompute_unpublished_live_head; +#[cfg(test)] +use self::key_dependency::wait_for_segment_key_dependency; +#[cfg(test)] +use self::origin::fetch_segment_with_retries_into_cache; +#[cfg(test)] +use self::scheduler::take_queued_segment_fetch_candidate; +#[cfg(test)] +use self::startup_fill::prepare_startup_fill; +pub use self::switch_staging::stage_hls_switch_resource; +use self::{ + commit::{ + segment_fetch_attempt_matches, segment_fetch_binding_matches, SegmentFetchCommit, SegmentOriginWorkFinish, + }, + error::SegmentFetchError, + key_dependency::{ + ensure_segment_key_dependency_ready, fetch_segment_key_dependency_into_cache, select_key_dependency, + select_segment_key_dependency, ReadySegmentKeyFetchSnapshot, SegmentKeyBindingSnapshot, SegmentKeyDependency, + SegmentKeyDependencySelection, + }, + origin::{ + build_segment_origin_headers, fetch_segment_into_cache, finish_segment_origin_attempt, + prepare_segment_origin_attempt, + }, + scheduler::{ + mark_segment_discovered, ScheduledSegmentRetry, SegmentFetchSnapshot, SegmentKeyFetchDependency, + SegmentRetryWake, SegmentWorkerRuntime, + }, + startup_fill::{commit_startup_response, reliable_decoded_content_length}, +}; diff --git a/backend/hls/src/segment_fetcher/commit.rs b/backend/hls/src/segment_fetcher/commit.rs new file mode 100644 index 000000000..06340279c --- /dev/null +++ b/backend/hls/src/segment_fetcher/commit.rs @@ -0,0 +1,298 @@ +use super::{ + HlsSegmentFailureObject, HlsSegmentFailureTransition, HlsSegmentWorkerPool, ProxySessionId, ScheduledSegmentRetry, + SegmentCacheStatus, SegmentEntry, SegmentFetchError, SegmentFetchPolicy, SegmentFetchSnapshot, SegmentRetryWake, +}; +use log::{debug, warn}; +use std::sync::Arc; +use tokio::sync::Notify; + +pub(super) struct SegmentFetchCompletion { + pub(super) generation_valid: bool, + pub(super) stale_commit_cleanup_reserved: bool, + pub(super) notifier: Option>, + pub(super) scheduled_retry: Option, + pub(super) evidence_changed_for: Option, +} + +pub(super) struct SegmentFetchCommit { + pub(super) content_length: u64, + pub(super) generation_valid: bool, +} + +#[derive(Clone, Copy)] +pub(super) struct SegmentOriginWorkFinish { + pub(super) generation_valid: bool, + pub(super) refresh_reservation: bool, +} + +pub(super) fn segment_fetch_attempt_matches(entry: &SegmentEntry, snapshot: &SegmentFetchSnapshot) -> bool { + entry.origin_key == snapshot.origin_key + && entry.cache_key == snapshot.cache_key + && matches!( + entry.status, + SegmentCacheStatus::Fetching { priority, started_at_ms } + if priority == snapshot.priority && started_at_ms == snapshot.started_at_ms + ) +} + +pub(super) fn segment_fetch_binding_matches(entry: &SegmentEntry, snapshot: &SegmentFetchSnapshot) -> bool { + segment_fetch_attempt_matches(entry, snapshot) + && entry.origin_fetch_ref.as_ref() == Some(&snapshot.fetch_ref) + && entry.encryption == snapshot.encryption +} + +impl HlsSegmentWorkerPool { + pub(super) fn apply_segment_fetch_result( + &self, + session: &mut super::super::HlsSession, + snapshot: &SegmentFetchSnapshot, + policy: &SegmentFetchPolicy, + result: Result, + finished_at_ms: u64, + ) -> SegmentFetchCompletion { + session.active_segment_fetches = session.active_segment_fetches.saturating_sub(1); + let fetch_succeeded = result.is_ok(); + let attempt_matches = session + .segments + .get(&snapshot.proxy_seq) + .is_some_and(|entry| segment_fetch_attempt_matches(entry, snapshot)); + let binding_matches = session + .segments + .get(&snapshot.proxy_seq) + .is_some_and(|entry| segment_fetch_binding_matches(entry, snapshot)); + let generation_valid = binding_matches + && session.activity.origin_work_generation == snapshot.origin_work_generation + && result.as_ref().map_or(true, |commit| commit.generation_valid); + // A successful origin/repair path has already atomically installed its physical object. Keep this fetch's + // entry in `Fetching` until stale cleanup finishes so no rebound attempt can install newer bytes at the same + // stable cache path and then lose them to this worker's cleanup. + let stale_commit_cleanup_reserved = fetch_succeeded && !generation_valid && attempt_matches; + let capacity_retry = if generation_valid { + result.as_ref().err().and_then(|error| { + error.capacity_revision().cloned().zip(error.projected_write_bytes()).map( + |(revision, projected_write_bytes)| ScheduledSegmentRetry { + wake: SegmentRetryWake::CapacityRevision { revision, projected_write_bytes }, + priority: snapshot.priority, + }, + ) + }) + } else { + None + }; + if generation_valid { + match result { + Ok(commit) => self.commit_successful_segment_fetch(session, snapshot, &commit, finished_at_ms), + Err(err) => commit_failed_segment_fetch(session, snapshot, policy, &err, finished_at_ms), + } + } else if !stale_commit_cleanup_reserved { + if let Some(entry) = session + .segments + .get_mut(&snapshot.proxy_seq) + .filter(|entry| segment_fetch_attempt_matches(entry, snapshot)) + { + entry.status = SegmentCacheStatus::Discovered; + } + } + let scheduled_retry = capacity_retry; + if generation_valid { + if let Err(err) = session.render_and_store_manifest(finished_at_ms) { + if log::log_enabled!(log::Level::Debug) { + let identity = super::super::HlsLogIdentity::from_session(session); + debug!( + "HLS manifest render deferred after segment state change: session={} proxy_session={} resource={} error={err:?}", + identity.session(), + identity.proxy_session(), + snapshot.proxy_seq_log + ); + } + } + } + let notifier = (!stale_commit_cleanup_reserved && scheduled_retry.is_none()) + .then(|| session.segment_fetch_notifiers.remove(&snapshot.proxy_seq)) + .flatten(); + let evidence_changed_for = (generation_valid && fetch_succeeded).then(|| session.proxy_session_id.clone()); + SegmentFetchCompletion { + generation_valid, + stale_commit_cleanup_reserved, + notifier, + scheduled_retry, + evidence_changed_for, + } + } + + fn commit_successful_segment_fetch( + &self, + session: &mut super::super::HlsSession, + snapshot: &SegmentFetchSnapshot, + commit: &SegmentFetchCommit, + finished_at_ms: u64, + ) { + let content_length = commit.content_length; + if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { + entry.status = SegmentCacheStatus::Ready { content_length, ready_at_ms: finished_at_ms }; + session.advance_media_readiness_generation(); + } + recompute_unpublished_live_head(session); + let reset_failures = session.record_successful_segment_fetch(); + self.metrics.record_segment_cached(); + if log::log_enabled!(log::Level::Debug) { + let identity = super::super::HlsLogIdentity::from_session(session); + debug!( + "HLS segment cached: session={} proxy_session={} source=normal resource={} content_length={content_length}", + identity.session(), + identity.proxy_session(), + snapshot.proxy_seq_log + ); + if let Some(reset_failures) = reset_failures { + debug!( + "HLS segment temporary failure counter reset: session={} proxy_session={} previous_failures={reset_failures}", + identity.session(), + identity.proxy_session() + ); + } + } + } +} + +pub(super) fn commit_failed_segment_fetch( + session: &mut super::super::HlsSession, + snapshot: &SegmentFetchSnapshot, + policy: &SegmentFetchPolicy, + error: &SegmentFetchError, + finished_at_ms: u64, +) { + let missing_candidate = session.origin_source.archive_reference.is_none() + && session.published_live_origin_baseline.is_none() + && session.publishable_origin_head_proxy_seq.is_some_and(|head| { + snapshot.proxy_seq >= head + && snapshot.proxy_seq + <= session + .missing_head_limit + .unwrap_or_else(|| head.saturating_add(super::super::startup_policy::STARTUP_WINDOW_SUCCESSORS)) + }) + && error.permanent_status().is_some_and(is_missing_origin_status); + if !error.is_local_cache_capacity() && !missing_candidate { + session.origin_control.path_condition = + super::super::origin_progress::HlsOriginPathCondition::SegmentReadinessFailure; + } + let (status, invalidate_queued_origin_work) = + failed_segment_status(session, snapshot, policy, error, finished_at_ms); + if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { + entry.status = status; + } + if missing_candidate { + recompute_unpublished_live_head(session); + } + if invalidate_queued_origin_work { + session.invalidate_queued_origin_work(); + if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { + entry.status = SegmentCacheStatus::FailedPermanent { failed_at_ms: finished_at_ms, status: None }; + } + } +} + +fn is_missing_origin_status(status: axum::http::StatusCode) -> bool { + matches!(status, axum::http::StatusCode::NOT_FOUND | axum::http::StatusCode::GONE) +} + +fn is_missing_origin_segment(status: &SegmentCacheStatus) -> bool { + matches!(status, SegmentCacheStatus::FailedPermanent { status: Some(status), .. } if is_missing_origin_status(*status)) +} + +pub(crate) fn recompute_unpublished_live_head(session: &mut super::super::HlsSession) { + if session.origin_source.archive_reference.is_some() || session.published_live_origin_baseline.is_some() { + return; + } + let Some(head) = session.publishable_origin_head_proxy_seq else { return }; + let Some(tail) = session.publishable_origin_tail_proxy_seq else { return }; + let bound = session + .missing_head_limit + .unwrap_or_else(|| head.saturating_add(super::super::startup_policy::STARTUP_WINDOW_SUCCESSORS)) + .min(tail); + if session.missing_head_limit.is_none() + && session.segments.range(head..=bound).any(|(_, entry)| is_missing_origin_segment(&entry.status)) + { + session.missing_head_limit = Some(bound); + } + let mut next_head = head; + for seq in head..=bound { + let Some(entry) = session.segments.get(&seq) else { break }; + match entry.status { + ref status if is_missing_origin_segment(status) => { + let Some(next) = seq.checked_add(1) else { return }; + next_head = next; + } + SegmentCacheStatus::Ready { .. } => { + if session.segments.range(seq..=bound).any(|(_, entry)| is_missing_origin_segment(&entry.status)) { + session.origin_control.path_condition = + super::super::origin_progress::HlsOriginPathCondition::SegmentReadinessFailure; + } + break; + } + _ => break, + } + } + if next_head > head { + session.publishable_origin_head_proxy_seq = Some(next_head); + debug!( + "HLS missing live head recovered: proxy_session={} previous_head={head} new_head={next_head}", + super::super::safe_proxy_session_id(&session.proxy_session_id) + ); + } +} + +fn failed_segment_status( + session: &mut super::super::HlsSession, + snapshot: &SegmentFetchSnapshot, + policy: &SegmentFetchPolicy, + error: &SegmentFetchError, + finished_at_ms: u64, +) -> (SegmentCacheStatus, bool) { + if error.is_local_cache_capacity() { + return ( + SegmentCacheStatus::CapacityDeferred { priority: snapshot.priority, deferred_at_ms: finished_at_ms }, + false, + ); + } + if !error.retryable_failure() { + return ( + SegmentCacheStatus::FailedPermanent { failed_at_ms: finished_at_ms, status: error.permanent_status() }, + false, + ); + } + let threshold = policy.permanent_failure_segment_threshold.max(1); + let transition = session.record_temporary_segment_fetch_failure( + finished_at_ms, + HlsSegmentFailureObject::Normal { proxy_seq: snapshot.proxy_seq, origin_seq: snapshot.origin_seq }, + threshold, + ); + match transition { + HlsSegmentFailureTransition::StillRetryable { failures, threshold } => { + if log::log_enabled!(log::Level::Debug) { + let identity = super::super::HlsLogIdentity::from_session(session); + debug!( + "HLS segment temporary failure counted: session={} proxy_session={} object={} failures={} threshold={}", + identity.session(), + identity.proxy_session(), + snapshot.proxy_seq_log, + failures, + threshold + ); + } + (SegmentCacheStatus::FailedRetryable { failed_at_ms: finished_at_ms, retry_after_ms: 1_000 }, false) + } + HlsSegmentFailureTransition::BecamePermanentlyFailed { failures, threshold } => { + if log::log_enabled!(log::Level::Warn) { + let identity = super::super::HlsLogIdentity::from_session(session); + warn!( + "HLS segment temporary failure threshold reached: session={} proxy_session={} failures={} threshold={}", + identity.session(), + identity.proxy_session(), + failures, + threshold + ); + } + (SegmentCacheStatus::FailedPermanent { failed_at_ms: finished_at_ms, status: None }, true) + } + } +} diff --git a/backend/hls/src/segment_fetcher/demand.rs b/backend/hls/src/segment_fetcher/demand.rs new file mode 100644 index 000000000..cef185b08 --- /dev/null +++ b/backend/hls/src/segment_fetcher/demand.rs @@ -0,0 +1,193 @@ +use super::{ + fetch_segment_key_dependency_into_cache, select_key_dependency, HlsSegmentFetchWorkload, HlsSegmentFile, + HlsSegmentWorkerPool, ReadySegmentKeyFetchSnapshot, SegmentCacheStatus, SegmentDemandFetchOutcome, + SegmentFetchContext, SegmentFetchPolicy, SegmentFetchPriority, SegmentKeyBindingSnapshot, SegmentKeyDependency, + SegmentKeyDependencySelection, SegmentKeyFetchDependency, +}; +use log::{debug, warn}; +use std::sync::Arc; +use tokio::{sync::OwnedSemaphorePermit, time::timeout}; +use tuliprox_core::utils::current_time_millis; + +impl HlsSegmentWorkerPool { + pub async fn demand_fetch_and_wait( + self: &Arc, + context: SegmentFetchContext, + segment_file: &HlsSegmentFile, + now_ms: u64, + ) -> SegmentDemandFetchOutcome { + let (notifier, workload) = { + let mut session = context.session.write().await; + if session.is_gc_marked_for_removal() { + return SegmentDemandFetchOutcome::NotFound; + } + let Some(entry) = session.segments.get(&segment_file.proxy_seq) else { + return SegmentDemandFetchOutcome::NotFound; + }; + if entry.proxy_file_ext != segment_file.extension { + return SegmentDemandFetchOutcome::NotFound; + } + if let SegmentCacheStatus::FailedRetryable { failed_at_ms, retry_after_ms } = entry.status { + if now_ms < failed_at_ms.saturating_add(retry_after_ms) { + return SegmentDemandFetchOutcome::TimedOut; + } + if let Some(entry) = session.segments.get_mut(&segment_file.proxy_seq) { + entry.status = SegmentCacheStatus::Discovered; + } + } + let Some(entry) = session.segments.get(&segment_file.proxy_seq) else { + return SegmentDemandFetchOutcome::NotFound; + }; + let workload = HlsSegmentFetchWorkload::from_encrypted(entry.encryption.is_some()); + let notifier = match entry.status { + SegmentCacheStatus::Ready { .. } => return SegmentDemandFetchOutcome::Ready, + SegmentCacheStatus::Fetching { .. } | SegmentCacheStatus::CapacityDeferred { .. } => { + session.segment_fetch_notifiers.entry(segment_file.proxy_seq).or_default().clone() + } + SegmentCacheStatus::Discovered | SegmentCacheStatus::Queued { .. } => { + if entry.origin_fetch_ref.is_none() { + return SegmentDemandFetchOutcome::Unavailable; + } + let backpressure = self.classify_backpressure_for_session(&session); + if !backpressure.allows_new_demand_fetch() { + if log::log_enabled!(log::Level::Warn) { + let identity = super::super::HlsLogIdentity::from_session(&session); + warn!( + "HLS segment demand fetch skipped by backpressure: session={} proxy_session={} source=normal resource={:06} state={backpressure:?}", + identity.session(), + identity.proxy_session(), + segment_file.proxy_seq + ); + } + return SegmentDemandFetchOutcome::Unavailable; + } + session.queue_segment_fetch_candidate(segment_file.proxy_seq, SegmentFetchPriority::Demand, now_ms); + self.metrics.record_demand_fetch_started(); + if log::log_enabled!(log::Level::Debug) { + let identity = super::super::HlsLogIdentity::from_session(&session); + debug!( + "HLS segment demand fetch started: session={} proxy_session={} source=normal resource={:06}", + identity.session(), + identity.proxy_session(), + segment_file.proxy_seq + ); + } + session.segment_fetch_notifiers.entry(segment_file.proxy_seq).or_default().clone() + } + SegmentCacheStatus::FailedPermanent { .. } | SegmentCacheStatus::Expired => { + return SegmentDemandFetchOutcome::Unavailable; + } + SegmentCacheStatus::FailedRetryable { .. } => return SegmentDemandFetchOutcome::TimedOut, + }; + (notifier, workload) + }; + + self.wake_scheduler(context.clone(), now_ms).await; + + let wait_timeout = self.runtime.load().policy.demand_wait_timeout_for(workload); + if timeout(wait_timeout, notifier.notified()).await.is_err() { + return SegmentDemandFetchOutcome::TimedOut; + } + + let session = context.session.read().await; + if session.is_gc_marked_for_removal() { + return SegmentDemandFetchOutcome::NotFound; + } + match session.segments.get(&segment_file.proxy_seq).map(|entry| &entry.status) { + Some(SegmentCacheStatus::Ready { .. }) => SegmentDemandFetchOutcome::Ready, + Some(SegmentCacheStatus::Queued { .. } | SegmentCacheStatus::Fetching { .. }) => { + SegmentDemandFetchOutcome::QueuedOrFetching + } + Some(SegmentCacheStatus::CapacityDeferred { .. } | SegmentCacheStatus::FailedRetryable { .. }) => { + SegmentDemandFetchOutcome::TimedOut + } + Some( + SegmentCacheStatus::Discovered + | SegmentCacheStatus::Expired + | SegmentCacheStatus::FailedPermanent { .. }, + ) => SegmentDemandFetchOutcome::Unavailable, + None => SegmentDemandFetchOutcome::NotFound, + } + } + + pub(super) async fn next_ready_segment_key_fetch_snapshot( + &self, + context: &SegmentFetchContext, + now_ms: u64, + policy: &SegmentFetchPolicy, + ) -> Option { + let (proxy_session_id, gc_marked_for_removal) = { + let session = context.session.read().await; + (session.proxy_session_id.clone(), session.is_gc_marked_for_removal()) + }; + if gc_marked_for_removal + || !self.access_leases.write().await.has_usable_access_lease_for_session(&proxy_session_id, now_ms) + { + return None; + } + let mut session = context.session.write().await; + if session.is_gc_marked_for_removal() || session.active_segment_fetches >= policy.max_session_segment_fetches { + return None; + } + let rendered_proxy_seqs = session.last_rendered_manifest.as_ref()?.segment_proxy_seqs.clone(); + for proxy_seq in rendered_proxy_seqs { + let Some((cache_key, encryption)) = session.segments.get(&proxy_seq).and_then(|segment| { + matches!(segment.status, SegmentCacheStatus::Ready { .. }) + .then(|| (segment.cache_key.clone(), segment.encryption.clone())) + }) else { + continue; + }; + let Some(encryption) = encryption else { + continue; + }; + let SegmentKeyDependencySelection::Ready(Some(SegmentKeyDependency::Fetch(dependency))) = + select_key_dependency(&mut session, &proxy_session_id, &encryption, now_ms) + else { + continue; + }; + session.active_segment_fetches = session.active_segment_fetches.saturating_add(1); + return Some(ReadySegmentKeyFetchSnapshot { + binding: SegmentKeyBindingSnapshot { + proxy_seq, + cache_key, + origin_work_generation: session.activity.origin_work_generation, + }, + dependency: *dependency, + }); + } + None + } + + pub(super) async fn fetch_ready_segment_key( + self: Arc, + context: SegmentFetchContext, + snapshot: ReadySegmentKeyFetchSnapshot, + policy: SegmentFetchPolicy, + permit: OwnedSemaphorePermit, + ) { + let SegmentKeyFetchDependency { token, resource, resource_file } = snapshot.dependency; + let result = fetch_segment_key_dependency_into_cache( + &context, + &snapshot.binding, + &policy, + token, + resource, + resource_file, + ) + .await; + let proxy_session_id = { + let mut session = context.session.write().await; + session.active_segment_fetches = session.active_segment_fetches.saturating_sub(1); + session.proxy_session_id.clone() + }; + if result.is_ok() { + if let Some(coordinator) = self.availability_reevaluations.as_ref() { + coordinator.notify_session_evidence_changed(&proxy_session_id); + } + } + drop(permit); + if result.is_ok() { + self.schedule_wake(context, current_time_millis()); + } + } +} diff --git a/backend/hls/src/segment_fetcher/error.rs b/backend/hls/src/segment_fetcher/error.rs new file mode 100644 index 000000000..35d435963 --- /dev/null +++ b/backend/hls/src/segment_fetcher/error.rs @@ -0,0 +1,3 @@ +use super::HlsOriginResourceFetchError; + +pub(super) type SegmentFetchError = HlsOriginResourceFetchError; diff --git a/backend/hls/src/segment_fetcher/key_dependency.rs b/backend/hls/src/segment_fetcher/key_dependency.rs new file mode 100644 index 000000000..18fce0043 --- /dev/null +++ b/backend/hls/src/segment_fetcher/key_dependency.rs @@ -0,0 +1,345 @@ +use super::{ + fetch_hls_transient_origin_response_with_attempt_prepare, finish_segment_origin_attempt, mark_segment_discovered, + prepare_segment_origin_attempt, transient_object_expires_at, HlsOriginResourceClients, HlsSegmentEncryption, + HlsTransientObjectFetchFinalizer, HlsTransientOriginFetchRequest, ProxySessionId, SegmentCacheKey, + SegmentFetchContext, SegmentFetchError, SegmentFetchPolicy, SegmentFetchSnapshot, SegmentKeyFetchDependency, + SegmentOriginWorkFinish, StagedCacheObject, TransientObjectFetchDecision, TransientObjectFetchToken, + TransientObjectUnavailableState, TransientPassthroughState, TransientResourceFile, TransientResourceId, + TransientResourceKind, TransientResourceRef, +}; +use futures::FutureExt; +use log::warn; +use std::{sync::Arc, time::Duration}; +use tokio::{ + sync::Notify, + time::{timeout_at, Instant}, +}; +use tuliprox_core::utils::current_time_millis; + +#[derive(Clone)] +pub(super) struct SegmentKeyBindingSnapshot { + pub(super) proxy_seq: u64, + pub(super) cache_key: SegmentCacheKey, + pub(super) origin_work_generation: u64, +} + +impl From<&SegmentFetchSnapshot> for SegmentKeyBindingSnapshot { + fn from(snapshot: &SegmentFetchSnapshot) -> Self { + Self { + proxy_seq: snapshot.proxy_seq, + cache_key: snapshot.cache_key.clone(), + origin_work_generation: snapshot.origin_work_generation, + } + } +} + +pub(super) struct ReadySegmentKeyFetchSnapshot { + pub(super) binding: SegmentKeyBindingSnapshot, + pub(super) dependency: SegmentKeyFetchDependency, +} + +#[derive(Clone)] +pub(super) enum SegmentKeyDependency { + Fetch(Box), + Wait { notifier: Arc, resource_id: TransientResourceId, resource_extension: String }, +} + +pub(super) enum SegmentKeyDependencySelection { + Ready(Option), + Unavailable, +} + +pub(super) fn select_segment_key_dependency( + session: &mut super::super::HlsSession, + proxy_session_id: &ProxySessionId, + proxy_seq: u64, + encryption: Option<&HlsSegmentEncryption>, + now_ms: u64, +) -> SegmentKeyDependencySelection { + let Some(encryption) = encryption else { + return SegmentKeyDependencySelection::Ready(None); + }; + let selection = select_key_dependency(session, proxy_session_id, encryption, now_ms); + if matches!(&selection, SegmentKeyDependencySelection::Unavailable) { + mark_segment_discovered(session, proxy_seq); + } + selection +} + +pub(super) fn select_key_dependency( + session: &mut super::super::HlsSession, + proxy_session_id: &ProxySessionId, + encryption: &HlsSegmentEncryption, + now_ms: u64, +) -> SegmentKeyDependencySelection { + let Some(resource) = session.transient.resolve_current_resource(&encryption.resource_id, now_ms) else { + return SegmentKeyDependencySelection::Unavailable; + }; + if resource.kind != TransientResourceKind::Key + || resource.file_ext_hint.as_deref() != Some(encryption.resource_extension.as_str()) + { + return SegmentKeyDependencySelection::Unavailable; + } + let resource_file = TransientResourceFile { + resource_id: encryption.resource_id.clone(), + extension: encryption.resource_extension.clone(), + }; + let cache_duration_ms = session.transient.resource_ttl_ms; + let decision = session.transient.begin_object_fetch( + proxy_session_id, + &resource, + &encryption.resource_extension, + now_ms, + cache_duration_ms, + ); + let dependency = match decision { + TransientObjectFetchDecision::Ready => None, + TransientObjectFetchDecision::Fetch(token) => { + Some(SegmentKeyDependency::Fetch(Box::new(SegmentKeyFetchDependency { + token: *token, + resource, + resource_file, + }))) + } + TransientObjectFetchDecision::Wait(notifier) => Some(SegmentKeyDependency::Wait { + notifier, + resource_id: encryption.resource_id.clone(), + resource_extension: encryption.resource_extension.clone(), + }), + }; + SegmentKeyDependencySelection::Ready(dependency) +} + +pub(super) async fn ensure_segment_key_dependency_ready( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + policy: &SegmentFetchPolicy, +) -> Result<(), SegmentFetchError> { + let Some(dependency) = snapshot.key_dependency.clone() else { + return Ok(()); + }; + match dependency { + SegmentKeyDependency::Wait { notifier, resource_id, resource_extension } => { + wait_for_segment_key_dependency( + context, + notifier, + resource_id, + resource_extension, + policy.origin_object_wait_timeout(), + ) + .await + } + SegmentKeyDependency::Fetch(dependency) => { + let SegmentKeyFetchDependency { token, resource, resource_file } = *dependency; + let binding = SegmentKeyBindingSnapshot::from(snapshot); + fetch_segment_key_dependency_into_cache(context, &binding, policy, token, resource, resource_file).await + } + } +} + +pub(super) async fn wait_for_segment_key_dependency( + context: &SegmentFetchContext, + notifier: Arc, + resource_id: TransientResourceId, + resource_extension: String, + wait_timeout: Duration, +) -> Result<(), SegmentFetchError> { + let deadline = Instant::now() + wait_timeout; + loop { + let wake = notifier.notified(); + tokio::pin!(wake); + wake.as_mut().enable(); + let should_wait = { + let now_ms = current_time_millis(); + let session = context.session.read().await; + if session + .transient + .ready_key_object_valid_until_ms(&session.proxy_session_id, &resource_id, &resource_extension, now_ms) + .is_some() + { + return Ok(()); + } + let key = TransientPassthroughState::transient_object_key( + &session.proxy_session_id, + &resource_id, + resource_extension.clone(), + ); + matches!( + session.transient.object_unavailable_state(&key, now_ms), + TransientObjectUnavailableState::Fetching + ) + }; + if !should_wait { + return Err(SegmentFetchError::Timeout); + } + timeout_at(deadline, wake).await.map_err(|_| SegmentFetchError::Timeout)?; + } +} + +pub(super) async fn fetch_segment_key_dependency_into_cache( + context: &SegmentFetchContext, + binding: &SegmentKeyBindingSnapshot, + policy: &SegmentFetchPolicy, + token: TransientObjectFetchToken, + resource: TransientResourceRef, + resource_file: TransientResourceFile, +) -> Result<(), SegmentFetchError> { + let mut fetch_finalizer = HlsTransientObjectFetchFinalizer::new( + context.session.clone(), + Arc::clone(&context.segment_cache), + token.clone(), + 1_000, + ); + let (staged, origin_work) = + stage_segment_key_dependency(context, policy, &token, &resource, &resource_file).await?; + let generation_valid = origin_work.generation_valid && { + let session = context.session.read().await; + segment_key_dependency_generation_matches( + &session, + binding, + &token, + &resource, + &resource_file, + current_time_millis(), + ) + }; + if !generation_valid { + if let Err(err) = context.segment_cache.remove_staged(staged).await { + warn!("HLS stale staged AES key cleanup failed: error={err}"); + } + context.session.write().await.fail_transient_object_retryable_if_current(&token, current_time_millis(), 1_000); + return Err(SegmentFetchError::Timeout); + } + let metadata = match context.segment_cache.commit_staged(token.cache_key(), staged).await { + Ok(metadata) => metadata, + Err(err) => { + context.session.write().await.fail_transient_object_retryable_if_current( + &token, + current_time_millis(), + 1_000, + ); + return Err(SegmentFetchError::cache_commit(&err)); + } + }; + let ready_at_ms = current_time_millis(); + super::super::transient_fetcher::delete_superseded_transient_object(&context.segment_cache, &token).await; + let mut session = context.session.write().await; + if !segment_key_dependency_generation_matches(&session, binding, &token, &resource, &resource_file, ready_at_ms) { + session.fail_transient_object_retryable_if_current(&token, ready_at_ms, 1_000); + return Err(SegmentFetchError::Timeout); + } + let expires_at_ms = transient_object_expires_at(ready_at_ms, session.transient.resource_ttl_ms); + if !session.commit_transient_object_ready_if_current( + TransientResourceKind::Key, + &token, + resource.content_type_hint.clone().unwrap_or_else(|| "application/octet-stream".to_string()), + metadata.size, + ready_at_ms, + expires_at_ms, + ) { + return Err(SegmentFetchError::Timeout); + } + fetch_finalizer.complete(); + Ok(()) +} + +async fn stage_segment_key_dependency( + context: &SegmentFetchContext, + policy: &SegmentFetchPolicy, + token: &TransientObjectFetchToken, + resource: &TransientResourceRef, + resource_file: &TransientResourceFile, +) -> Result<(StagedCacheObject, SegmentOriginWorkFinish), SegmentFetchError> { + let request = HlsTransientOriginFetchRequest { + resolved_origin_uri: resource.resolved_origin_uri.clone(), + origin_headers: context.headers.clone(), + origin_provider_session_headers: context.origin_provider_session_headers.clone(), + range_header: None, + resource_file: resource_file.clone(), + resource_kind: TransientResourceKind::Key, + clients: HlsOriginResourceClients { + client: context.client.clone(), + no_redirect_client: context.no_redirect_client.clone(), + use_manual_redirects: context.use_manual_redirects, + }, + policy: policy.clone(), + log_identity: { + let session = context.session.read().await; + super::super::HlsLogIdentity::from_session(&session) + }, + }; + let prepare_context = context.clone(); + let prepare_policy = policy.clone(); + let response = fetch_hls_transient_origin_response_with_attempt_prepare(request, move |_attempt| { + let context = prepare_context.clone(); + let policy = prepare_policy.clone(); + async move { prepare_segment_origin_attempt(context, policy).await }.boxed() + }) + .await; + let response = match response { + Ok(response) => response, + Err(err) => { + context.session.write().await.fail_transient_object_retryable_if_current( + token, + current_time_millis(), + 1_000, + ); + return Err(err); + } + }; + let staged = context + .segment_cache + .stage_temp_with_deadline(token.cache_key(), response.decoded.body, response.body_deadline.deadline()) + .await; + let origin_work = finish_segment_origin_attempt(context.clone(), response.guard).await; + let staged = match staged { + Ok(staged) if staged.size == 16 => staged, + Ok(staged) => { + if let Err(err) = context.segment_cache.remove_staged(staged).await { + warn!("HLS invalid staged AES key cleanup failed: error={err}"); + } + context.session.write().await.fail_transient_object_permanent_if_current( + token, + current_time_millis(), + Some(axum::http::StatusCode::BAD_GATEWAY), + ); + return Err(SegmentFetchError::NonRetryableStatus(axum::http::StatusCode::BAD_GATEWAY)); + } + Err(err) => { + context.session.write().await.fail_transient_object_retryable_if_current( + token, + current_time_millis(), + 1_000, + ); + return Err(SegmentFetchError::cache_body(&err)); + } + }; + Ok((staged, origin_work)) +} + +fn segment_key_dependency_generation_matches( + session: &super::super::HlsSession, + binding: &SegmentKeyBindingSnapshot, + token: &TransientObjectFetchToken, + resource: &TransientResourceRef, + resource_file: &TransientResourceFile, + now_ms: u64, +) -> bool { + if session.activity.origin_work_generation != binding.origin_work_generation + || !session.transient.object_fetch_token_matches(token) + { + return false; + } + let segment_matches = session.segments.get(&binding.proxy_seq).is_some_and(|segment| { + segment.cache_key == binding.cache_key + && segment.encryption.as_ref().is_some_and(|encryption| { + encryption.resource_id == resource_file.resource_id + && encryption.resource_extension == resource_file.extension + }) + }); + let resource_matches = resource.id == resource_file.resource_id + && resource.kind == TransientResourceKind::Key + && resource.file_ext_hint.as_deref() == Some(resource_file.extension.as_str()) + && session.transient.resource_matches_current(resource, now_ms); + segment_matches && resource_matches +} diff --git a/backend/hls/src/segment_fetcher/origin.rs b/backend/hls/src/segment_fetcher/origin.rs new file mode 100644 index 000000000..0eb8e1754 --- /dev/null +++ b/backend/hls/src/segment_fetcher/origin.rs @@ -0,0 +1,253 @@ +use super::{ + begin_hls_origin_account_io_bounded, build_hls_origin_resource_headers, commit_startup_response, + ensure_segment_key_dependency_ready, finish_hls_origin_account_io, hls_object_body_deadline, + reliable_decoded_content_length, run_hls_origin_resource_retry_loop_with_attempt_prepare, CachedSegmentMetadata, + HlsBoundAccountAcquireErrorKind, HlsOriginAccountIoLeaseGuard, HlsOriginByteRangeExpectation, HlsOriginIoContext, + HlsOriginResourceBodyDeadline, HlsOriginResourceClients, HlsOriginResourceFetchTarget, HlsRepairRenderedObjectId, + HlsResourceFetchKind, HlsResourceFetchSource, HlsSegmentRepairObjectContext, HlsSegmentRepairSource, + SegmentFetchCommit, SegmentFetchContext, SegmentFetchError, SegmentFetchPolicy, SegmentFetchSnapshot, + SegmentOriginWorkFinish, +}; +use axum::http::HeaderMap; +use futures::FutureExt; +use tuliprox_core::utils::{content_coding::DecodedHttpResponse, current_time_millis}; +use tuliprox_parser::hls::origin_manifest::ParsedByteRange; + +pub(super) async fn fetch_segment_into_cache( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + policy: &SegmentFetchPolicy, +) -> Result { + ensure_segment_key_dependency_ready(context, snapshot, policy).await?; + fetch_segment_with_retries_into_cache(context, snapshot, policy).await +} + +pub(super) struct SegmentOriginAttemptGuard { + started_generation: Option, + provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, +} + +pub(super) async fn prepare_segment_origin_attempt( + context: SegmentFetchContext, + policy: SegmentFetchPolicy, +) -> Result { + let started_generation = start_segment_origin_work(&context).await; + let binding = + if context.origin_io.is_some() { context.session.read().await.origin_account_binding.clone() } else { None }; + let provider_lease = if let (Some(origin_io), Some(binding)) = (context.origin_io.as_ref(), binding.as_ref()) { + if binding.is_detached() { + finish_segment_origin_work(&context, started_generation).await; + touch_segment_origin_account_binding(&context, false).await; + return Err(SegmentFetchError::ProviderUnavailable(HlsBoundAccountAcquireErrorKind::Detached)); + } + let guard = match begin_hls_origin_account_io_bounded( + origin_io, + &context.session, + binding, + hls_object_body_deadline(policy.origin_segment_timeout_ms), + ) + .await + { + Ok(guard) => guard, + Err(err) => { + finish_segment_origin_work(&context, started_generation).await; + touch_segment_origin_account_binding(&context, false).await; + return Err(SegmentFetchError::ProviderUnavailable(err)); + } + }; + Some((origin_io.clone(), guard)) + } else { + None + }; + Ok(SegmentOriginAttemptGuard { started_generation, provider_lease }) +} + +pub(super) async fn finish_segment_origin_attempt( + context: SegmentFetchContext, + guard: SegmentOriginAttemptGuard, +) -> SegmentOriginWorkFinish { + finish_segment_origin_io(&context, guard.started_generation, guard.provider_lease).await +} + +async fn finish_segment_origin_io( + context: &SegmentFetchContext, + started_generation: Option, + provider_lease: Option<(HlsOriginIoContext, HlsOriginAccountIoLeaseGuard)>, +) -> SegmentOriginWorkFinish { + let origin_work = finish_segment_origin_work(context, started_generation).await; + if let Some((origin_io, guard)) = provider_lease { + finish_hls_origin_account_io( + &origin_io, + &context.session, + guard, + origin_work.generation_valid && origin_work.refresh_reservation, + ) + .await; + touch_segment_origin_account_binding(context, origin_work.generation_valid && origin_work.refresh_reservation) + .await; + } + origin_work +} + +async fn start_segment_origin_work(context: &SegmentFetchContext) -> Option { + context.origin_io.as_ref()?; + let mut session = context.session.write().await; + Some(session.start_origin_work()) +} + +async fn finish_segment_origin_work( + context: &SegmentFetchContext, + started_generation: Option, +) -> SegmentOriginWorkFinish { + let Some(started_generation) = started_generation else { + return SegmentOriginWorkFinish { generation_valid: true, refresh_reservation: false }; + }; + let mut session = context.session.write().await; + let generation_valid = session.finish_origin_work(started_generation); + let refresh_reservation = session.should_refresh_origin_reservation(current_time_millis()); + SegmentOriginWorkFinish { generation_valid, refresh_reservation } +} + +async fn touch_segment_origin_account_binding(context: &SegmentFetchContext, reservation_refreshed: bool) { + let mut session = context.session.write().await; + if let Some(binding) = session.origin_account_binding.as_mut() { + let now_ms = current_time_millis(); + binding.last_origin_io_at_ms = Some(now_ms); + if reservation_refreshed { + binding.last_reservation_refresh_at_ms = Some(now_ms); + } + } +} + +#[allow(clippy::too_many_lines)] +pub(super) async fn fetch_segment_with_retries_into_cache( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + policy: &SegmentFetchPolicy, +) -> Result { + let headers = build_segment_origin_headers( + &context.headers, + &context.origin_provider_session_headers, + snapshot.fetch_ref.byte_range, + )?; + let target = HlsOriginResourceFetchTarget { + kind: HlsResourceFetchKind::Segment, + source: HlsResourceFetchSource::Normal, + object_id: snapshot.proxy_seq_log.clone(), + origin_url: snapshot.fetch_ref.resolved_origin_url.clone(), + headers, + byte_range_expectation: if snapshot.fetch_ref.byte_range.is_some() { + HlsOriginByteRangeExpectation::PartialContent + } else { + HlsOriginByteRangeExpectation::FullObject + }, + }; + let clients = HlsOriginResourceClients { + client: context.client.clone(), + no_redirect_client: context.no_redirect_client.clone(), + use_manual_redirects: context.use_manual_redirects, + }; + let log_identity = { + let session = context.session.read().await; + super::super::HlsLogIdentity::from_session(&session) + }; + let context = context.clone(); + let snapshot = snapshot.clone(); + let policy_for_prepare = policy.clone(); + let prepare_context = context.clone(); + let cleanup_context = context.clone(); + run_hls_origin_resource_retry_loop_with_attempt_prepare( + target, + clients, + policy, + &log_identity, + move |_attempt| { + let context = prepare_context.clone(); + let policy = policy_for_prepare.clone(); + async move { prepare_segment_origin_attempt(context, policy).await }.boxed() + }, + move |guard| { + let context = cleanup_context.clone(); + async move { + finish_segment_origin_attempt(context, guard).await; + } + .boxed() + }, + move |response, _attempt, body_deadline, guard| { + let context = context.clone(); + let snapshot = snapshot.clone(); + async move { + let commit_result = + commit_segment_response_into_cache(&context, &snapshot, response, body_deadline).await; + let origin_work = finish_segment_origin_attempt(context, guard).await; + commit_result.map(|metadata| SegmentFetchCommit { + content_length: metadata.size, + generation_valid: origin_work.generation_valid, + }) + } + .boxed() + }, + ) + .await +} + +async fn commit_segment_response_into_cache( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + response: DecodedHttpResponse, + body_deadline: HlsOriginResourceBodyDeadline, +) -> Result { + let (proxy_session_id, log_identity, accelerated) = { + let session = context.session.read().await; + ( + session.proxy_session_id.clone(), + super::super::HlsLogIdentity::from_session(&session), + session.startup.is_some(), + ) + }; + let repair_context = HlsSegmentRepairObjectContext { + source: HlsSegmentRepairSource::Normal, + log_identity, + proxy_session_id, + hls_access_lease_id: context.repair_access_lease_id.clone(), + rendered_object_id: HlsRepairRenderedObjectId::Normal { proxy_seq: snapshot.proxy_seq }, + resource_id: format!("{:06}", snapshot.proxy_seq), + file_ext: snapshot.proxy_file_ext.clone(), + // Segment repair uses the concrete fetch URL for diagnostics/postprocess metadata only. + origin_fetch_uri_for_diagnostics: snapshot.fetch_ref.resolved_origin_url.clone(), + media_sequence: Some(snapshot.origin_seq), + discontinuity_sequence: None, + complete_object: snapshot.complete_object, + encrypted: snapshot.encryption.is_some(), + custom_response: false, + }; + if accelerated { + return commit_startup_response(context, snapshot, response, body_deadline.deadline(), repair_context).await; + } + if let Some(content_length) = reliable_decoded_content_length(&response) { + context + .segment_cache + .ensure_projected_write_capacity(&snapshot.cache_key, content_length) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + } + context + .segment_repair + .commit_origin_response( + &context.segment_cache, + &snapshot.cache_key, + response.body, + body_deadline.deadline(), + repair_context, + ) + .await + .map_err(|err| SegmentFetchError::cache_body(&err)) +} + +pub(super) fn build_segment_origin_headers( + source_headers: &HeaderMap, + provider_session_headers: &HeaderMap, + byte_range: Option, +) -> Result { + build_hls_origin_resource_headers(source_headers, provider_session_headers, byte_range) +} diff --git a/backend/hls/src/segment_fetcher/policy.rs b/backend/hls/src/segment_fetcher/policy.rs new file mode 100644 index 000000000..5a1e03d2e --- /dev/null +++ b/backend/hls/src/segment_fetcher/policy.rs @@ -0,0 +1,73 @@ +use super::{ + HlsSegmentFetchWorkload, SegmentFetchPolicy, DEFAULT_MAX_GLOBAL_SEGMENT_FETCHES, DEFAULT_MAX_PREFETCH_QUEUE_DEPTH, + DEFAULT_MAX_SESSION_SEGMENT_FETCHES, DEFAULT_ORIGIN_SEGMENT_TIMEOUT_MS, SEGMENT_FETCH_SCHEDULING_MARGIN_MS, +}; +use std::time::Duration; + +impl HlsSegmentFetchWorkload { + const fn serialized_origin_objects(self) -> u64 { + match self { + Self::Clear => 1, + Self::EncryptedWithKey => 2, + } + } +} + +impl SegmentFetchPolicy { + fn origin_object_retry_chain_budget_ms(&self) -> u64 { + let attempts = u64::try_from(self.retry_delays_ms.len()).unwrap_or(u64::MAX); + let per_attempt_budget = + self.origin_segment_timeout_ms.saturating_add(self.effective_repair_postprocess_timeout_ms); + let retry_delay_budget = self.retry_delays_ms.iter().copied().fold(0_u64, u64::saturating_add); + let jitter_budget = attempts.saturating_mul(self.retry_jitter_max_ms); + attempts.saturating_mul(per_attempt_budget).saturating_add(retry_delay_budget).saturating_add(jitter_budget) + } + + pub fn workload_budget_ms(&self, workload: HlsSegmentFetchWorkload) -> u64 { + self.origin_object_retry_chain_budget_ms() + .saturating_mul(workload.serialized_origin_objects()) + .saturating_add(SEGMENT_FETCH_SCHEDULING_MARGIN_MS) + } + + /// Conservative public wait bound for a segment whose encryption state is not known at the call site. + pub fn demand_wait_timeout(&self) -> Duration { + self.demand_wait_timeout_for(HlsSegmentFetchWorkload::EncryptedWithKey) + } + + pub fn demand_wait_timeout_for(&self, workload: HlsSegmentFetchWorkload) -> Duration { + Duration::from_millis(self.workload_budget_ms(workload)) + } + + /// Wait bound for one transient origin object such as a key or MAP. + pub fn origin_object_wait_timeout(&self) -> Duration { + self.demand_wait_timeout_for(HlsSegmentFetchWorkload::Clear) + } + + pub fn retry_delay_ms(&self, attempt_index: usize) -> u64 { + let base_delay_ms = self.retry_delays_ms[attempt_index]; + if self.retry_jitter_max_ms == 0 { + return base_delay_ms; + } + let jitter_ms = fastrand::u64(0..=self.retry_jitter_max_ms); + if fastrand::bool() { + base_delay_ms.saturating_sub(jitter_ms) + } else { + base_delay_ms.saturating_add(jitter_ms) + } + } +} + +impl Default for SegmentFetchPolicy { + fn default() -> Self { + Self { + max_global_segment_fetches: DEFAULT_MAX_GLOBAL_SEGMENT_FETCHES, + max_session_segment_fetches: DEFAULT_MAX_SESSION_SEGMENT_FETCHES, + max_prefetch_queue_depth: DEFAULT_MAX_PREFETCH_QUEUE_DEPTH, + origin_segment_timeout_ms: DEFAULT_ORIGIN_SEGMENT_TIMEOUT_MS, + effective_repair_postprocess_timeout_ms: 0, + retry_delays_ms: [0, 100, 250, 500, 750], + retry_jitter_max_ms: 100, + permanent_failure_segment_threshold: 3, + } + } +} diff --git a/backend/hls/src/segment_fetcher/scheduler.rs b/backend/hls/src/segment_fetcher/scheduler.rs new file mode 100644 index 000000000..d282c2e6e --- /dev/null +++ b/backend/hls/src/segment_fetcher/scheduler.rs @@ -0,0 +1,537 @@ +use super::{ + classify_hls_backpressure, fetch_segment_into_cache, segment_fetch_attempt_matches, select_segment_key_dependency, + HlsAccessLeaseStore, HlsBackpressureState, HlsCacheCapacityRevision, HlsCacheMetrics, HlsSegmentEncryption, + HlsSegmentFetchWorkload, HlsSegmentWorkerPool, HlsSessionHandle, OriginSegmentFetchRef, OriginSegmentKey, + ReadySegmentKeyFetchSnapshot, SegmentCacheKey, SegmentCacheStatus, SegmentFetchContext, SegmentFetchPolicy, + SegmentFetchPriority, SegmentKeyDependency, SegmentKeyDependencySelection, TransientObjectFetchToken, + TransientResourceFile, TransientResourceRef, DEFAULT_ORIGIN_SEGMENT_TIMEOUT_MS, + DEFAULT_REPAIR_POSTPROCESS_TIMEOUT_MS, SEGMENT_FETCH_SCHEDULING_MARGIN_MS, +}; +use arc_swap::ArcSwap; +use log::{debug, warn}; +use shared::model::HlsSegmentRepairMode; +use std::{fmt, sync::Arc}; +use tokio::sync::{OwnedSemaphorePermit, RwLock, Semaphore}; +use tuliprox_core::{model::HlsCacheConfig, utils::current_time_millis}; + +impl HlsSegmentFetchWorkload { + pub(super) const fn from_encrypted(encrypted: bool) -> Self { + if encrypted { + Self::EncryptedWithKey + } else { + Self::Clear + } + } +} + +impl SegmentFetchPolicy { + pub fn from_config(config: &HlsCacheConfig) -> Self { + let postprocess_enabled = (config.segment_repair.max_level != HlsSegmentRepairMode::Off + && config.segment_repair.apply_to_first_segments > 0) + || config.segment_repair.corrupt_segment_watchdog.mode.is_enabled(); + Self { + max_global_segment_fetches: config.max_concurrent_segment_fetches_global.max(1), + max_session_segment_fetches: config.max_concurrent_segment_fetches_per_session.max(1), + max_prefetch_queue_depth: config.max_segments_prefetch, + origin_segment_timeout_ms: config.origin_segment_timeout_ms.get().max(1), + effective_repair_postprocess_timeout_ms: if postprocess_enabled { + config.segment_repair.postprocess_timeout_ms.get().max(100) + } else { + 0 + }, + // Object retry classification is deliberately independent from initial-strip policy. + permanent_failure_segment_threshold: Self::default().permanent_failure_segment_threshold, + ..Self::default() + } + } + + /// Expected latency of one successful media-object operation. + /// + /// This deliberately excludes the complete retry chain. Callers may use it + /// for recovery ETA, while `workload_budget_ms` remains the hard wait bound. + /// Configured protection timeouts above the conservative fallback are not + /// predictions of successful-object latency. + pub fn recovery_object_eta_ms(&self) -> u64 { + self.origin_segment_timeout_ms + .min(DEFAULT_ORIGIN_SEGMENT_TIMEOUT_MS) + .saturating_add(self.effective_repair_postprocess_timeout_ms.min(DEFAULT_REPAIR_POSTPROCESS_TIMEOUT_MS)) + .saturating_add(SEGMENT_FETCH_SCHEDULING_MARGIN_MS) + } +} + +#[derive(Clone)] +pub(super) struct SegmentFetchSnapshot { + pub(super) proxy_seq: u64, + pub(super) proxy_seq_log: String, + pub(super) origin_key: OriginSegmentKey, + pub(super) cache_key: SegmentCacheKey, + pub(super) fetch_ref: OriginSegmentFetchRef, + pub(super) encryption: Option, + pub(super) priority: SegmentFetchPriority, + pub(super) started_at_ms: u64, + pub(super) proxy_file_ext: String, + pub(super) origin_seq: u64, + pub(super) complete_object: bool, + pub(super) key_dependency: Option, + pub(super) origin_work_generation: u64, +} + +enum SegmentWorkerTask { + ReadyKey(ReadySegmentKeyFetchSnapshot), + Segment(SegmentFetchSnapshot), +} + +pub(super) struct QueuedSegmentFetchCandidate { + origin_key: OriginSegmentKey, + cache_key: SegmentCacheKey, + fetch_ref: OriginSegmentFetchRef, + encryption: Option, + proxy_file_ext: String, + complete_object: bool, +} + +#[derive(Clone)] +pub(super) struct SegmentKeyFetchDependency { + pub(super) token: TransientObjectFetchToken, + pub(super) resource: TransientResourceRef, + pub(super) resource_file: TransientResourceFile, +} + +pub(super) struct ScheduledSegmentRetry { + pub(super) wake: SegmentRetryWake, + pub(super) priority: SegmentFetchPriority, +} + +pub(super) enum SegmentRetryWake { + CapacityRevision { revision: HlsCacheCapacityRevision, projected_write_bytes: u64 }, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum CapacityRetryAdmission { + Ready, + BindingExpired, + LocalIoFailure, +} + +impl fmt::Debug for SegmentFetchSnapshot { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SegmentFetchSnapshot") + .field("proxy_seq", &self.proxy_seq) + .field("proxy_seq_log", &self.proxy_seq_log) + .field("origin_key", &self.origin_key) + .field("cache_key", &self.cache_key) + .field("fetch_ref", &self.fetch_ref) + .field("priority", &self.priority) + .field("started_at_ms", &self.started_at_ms) + .field("proxy_file_ext", &self.proxy_file_ext) + .field("origin_seq", &self.origin_seq) + .field("complete_object", &self.complete_object) + .field("encrypted", &self.encryption.is_some()) + .field( + "key_dependency", + &self.key_dependency.as_ref().map(|dependency| match dependency { + SegmentKeyDependency::Fetch { .. } => "fetch", + SegmentKeyDependency::Wait { .. } => "wait", + }), + ) + .field("origin_work_generation", &self.origin_work_generation) + .finish() + } +} + +#[derive(Clone)] +pub(super) struct SegmentWorkerRuntime { + pub(super) global_semaphore: Arc, + pub(super) policy: SegmentFetchPolicy, +} + +impl SegmentWorkerRuntime { + pub(super) fn new(policy: SegmentFetchPolicy, global_semaphore: Arc) -> Self { + Self { global_semaphore, policy } + } +} + +impl HlsSegmentWorkerPool { + pub fn new(policy: SegmentFetchPolicy) -> Self { + let global_semaphore = Arc::new(Semaphore::new(policy.max_global_segment_fetches)); + Self::with_global_semaphore(policy, global_semaphore) + } + + pub fn with_global_semaphore(policy: SegmentFetchPolicy, global_semaphore: Arc) -> Self { + Self::with_global_semaphore_and_metrics( + policy, + global_semaphore, + Arc::new(RwLock::new(HlsAccessLeaseStore::default())), + Arc::new(HlsCacheMetrics::default()), + ) + } + + pub fn with_global_semaphore_and_metrics( + policy: SegmentFetchPolicy, + global_semaphore: Arc, + access_leases: Arc>, + metrics: Arc, + ) -> Self { + Self::with_global_semaphore_metrics_and_availability(policy, global_semaphore, access_leases, metrics, None) + } + + pub fn with_global_semaphore_metrics_and_availability( + policy: SegmentFetchPolicy, + global_semaphore: Arc, + access_leases: Arc>, + metrics: Arc, + availability_reevaluations: Option< + Arc, + >, + ) -> Self { + Self { + runtime: ArcSwap::from_pointee(SegmentWorkerRuntime::new(policy, global_semaphore)), + access_leases, + metrics, + availability_reevaluations, + } + } + + pub fn update_config(&self, policy: SegmentFetchPolicy, global_semaphore: Arc) { + self.runtime.store(Arc::new(SegmentWorkerRuntime::new(policy, global_semaphore))); + } + + pub fn policy(&self) -> SegmentFetchPolicy { self.runtime.load().policy.clone() } + + pub fn access_leases(&self) -> &Arc> { &self.access_leases } + + pub fn metrics(&self) -> &Arc { &self.metrics } + + pub async fn classify_backpressure(&self, session: &HlsSessionHandle) -> HlsBackpressureState { + let session = session.read().await; + self.classify_backpressure_for_session(&session) + } + + pub fn classify_backpressure_for_session(&self, session: &super::super::HlsSession) -> HlsBackpressureState { + let runtime = self.runtime.load(); + classify_hls_backpressure( + session, + runtime.global_semaphore.available_permits(), + runtime.policy.max_session_segment_fetches, + ) + } + + pub async fn wake_scheduler(self: &Arc, context: SegmentFetchContext, now_ms: u64) { + loop { + let runtime = self.runtime.load_full(); + let Ok(permit) = Arc::clone(&runtime.global_semaphore).try_acquire_owned() else { + return; + }; + let task = if let Some(snapshot) = + self.next_ready_segment_key_fetch_snapshot(&context, now_ms, &runtime.policy).await + { + Some(SegmentWorkerTask::ReadyKey(snapshot)) + } else { + self.next_fetch_snapshot(&context, now_ms, &runtime.policy).await.map(SegmentWorkerTask::Segment) + }; + let Some(task) = task else { + drop(permit); + return; + }; + + let worker = Arc::clone(self); + let task_context = context.clone(); + tokio::spawn(async move { + match task { + SegmentWorkerTask::ReadyKey(snapshot) => { + worker.fetch_ready_segment_key(task_context, snapshot, runtime.policy.clone(), permit).await; + } + SegmentWorkerTask::Segment(snapshot) => { + worker.fetch_one_segment(task_context, snapshot, runtime.policy.clone(), permit).await; + } + } + }); + } + } + + pub(super) async fn next_fetch_snapshot( + &self, + context: &SegmentFetchContext, + now_ms: u64, + policy: &SegmentFetchPolicy, + ) -> Option { + let (proxy_session_id, gc_marked_for_removal) = { + let session = context.session.read().await; + (session.proxy_session_id.clone(), session.is_gc_marked_for_removal()) + }; + if gc_marked_for_removal { + return None; + } + let has_usable_access_lease = + self.access_leases.write().await.has_usable_access_lease_for_session(&proxy_session_id, now_ms); + let mut session = context.session.write().await; + if session.is_gc_marked_for_removal() { + return None; + } + let initial_barrier = session.startup.as_ref().is_some_and(|startup| { + let Some(head) = session.publishable_origin_head_proxy_seq else { + return true; + }; + let mode = startup.config.mode; + startup.revisions.get(&head).is_none_or(|revision| { + mode != shared::model::HlsStartupMode::Conservative && !revision.revision().is_startup_head_ready(mode) + }) + }); + let fetch_limit = if initial_barrier { 1 } else { policy.max_session_segment_fetches }; + if session.active_segment_fetches >= fetch_limit { + return None; + } + + while let Some((proxy_seq, priority)) = session.segment_prefetch_queue.pop_next() { + let origin_work_generation = session.activity.origin_work_generation; + let Some(candidate) = + take_queued_segment_fetch_candidate(&mut session, proxy_seq, priority, now_ms, has_usable_access_lease) + else { + continue; + }; + let key_dependency = match select_segment_key_dependency( + &mut session, + &proxy_session_id, + proxy_seq, + candidate.encryption.as_ref(), + now_ms, + ) { + SegmentKeyDependencySelection::Ready(dependency) => dependency, + SegmentKeyDependencySelection::Unavailable => continue, + }; + if let Some(entry) = session.segments.get_mut(&proxy_seq) { + entry.status = SegmentCacheStatus::Fetching { priority, started_at_ms: now_ms }; + } else { + continue; + } + session.active_segment_fetches = session.active_segment_fetches.saturating_add(1); + let proxy_seq_log = format!("{proxy_seq:06}"); + if log::log_enabled!(log::Level::Debug) { + let identity = super::super::HlsLogIdentity::from_session(&session); + debug!( + "HLS segment fetch started: session={} proxy_session={} source=normal resource={} priority={priority:?}", + identity.session(), + identity.proxy_session(), + proxy_seq_log + ); + } + return Some(SegmentFetchSnapshot { + proxy_seq, + proxy_seq_log, + origin_key: candidate.origin_key, + cache_key: candidate.cache_key, + fetch_ref: candidate.fetch_ref, + encryption: candidate.encryption, + priority, + started_at_ms: now_ms, + proxy_file_ext: candidate.proxy_file_ext, + origin_seq: candidate.origin_key.host_local_sequence, + complete_object: candidate.complete_object, + key_dependency, + origin_work_generation, + }); + } + + None + } + + async fn fetch_one_segment( + self: Arc, + context: SegmentFetchContext, + snapshot: SegmentFetchSnapshot, + policy: SegmentFetchPolicy, + permit: OwnedSemaphorePermit, + ) { + let result = fetch_segment_into_cache(&context, &snapshot, &policy).await; + let finished_at_ms = current_time_millis(); + let completion = { + let mut session = context.session.write().await; + self.apply_segment_fetch_result(&mut session, &snapshot, &policy, result, finished_at_ms) + }; + let mut notifier = completion.notifier; + if completion.stale_commit_cleanup_reserved { + if let Err(err) = context.segment_cache.delete(&snapshot.cache_key).await { + if log::log_enabled!(log::Level::Warn) { + let identity = { + let session = context.session.read().await; + super::super::HlsLogIdentity::from_session(&session) + }; + warn!( + "HLS stale segment cache cleanup failed: session={} proxy_session={} resource={} error={err}", + identity.session(), + identity.proxy_session(), + snapshot.proxy_seq_log + ); + } + } + let mut session = context.session.write().await; + if let Some(entry) = session + .segments + .get_mut(&snapshot.proxy_seq) + .filter(|entry| segment_fetch_attempt_matches(entry, &snapshot)) + { + entry.status = SegmentCacheStatus::Discovered; + } + notifier = session.segment_fetch_notifiers.remove(&snapshot.proxy_seq); + } + if let Some(notifier) = notifier { + notifier.notify_waiters(); + } + if let (Some(coordinator), Some(proxy_session_id)) = + (self.availability_reevaluations.as_ref(), completion.evidence_changed_for.as_ref()) + { + coordinator.notify_session_evidence_changed(proxy_session_id); + } + drop(permit); + if let Some(retry) = completion.scheduled_retry { + self.schedule_segment_retry(context, snapshot, retry); + return; + } + if completion.generation_valid { + self.schedule_wake(context, finished_at_ms); + } + } + + pub(super) fn schedule_wake(self: &Arc, context: SegmentFetchContext, now_ms: u64) { + let worker = Arc::clone(self); + tokio::spawn(async move { + worker.wake_scheduler(context, now_ms).await; + }); + } + + pub(super) fn schedule_segment_retry( + self: &Arc, + context: SegmentFetchContext, + snapshot: SegmentFetchSnapshot, + retry: ScheduledSegmentRetry, + ) { + let worker = Arc::clone(self); + tokio::spawn(async move { + let capacity_admission = match &retry.wake { + SegmentRetryWake::CapacityRevision { revision, projected_write_bytes } => { + wait_for_capacity_retry_admission(&context, &snapshot, revision.clone(), *projected_write_bytes) + .await + } + }; + let now_ms = current_time_millis(); + let (requeued, abandoned_notifier) = { + let mut session = context.session.write().await; + let retry_state_matches = session.segments.get(&snapshot.proxy_seq).is_some_and(|entry| { + matches!( + (&entry.status, &retry.wake), + (SegmentCacheStatus::CapacityDeferred { .. }, SegmentRetryWake::CapacityRevision { .. }) + ) + }); + if retry_state_matches { + let binding_current = session.segments.get(&snapshot.proxy_seq).is_some_and(|entry| { + entry.origin_key == snapshot.origin_key + && entry.cache_key == snapshot.cache_key + && entry.origin_fetch_ref.as_ref() == Some(&snapshot.fetch_ref) + && entry.encryption == snapshot.encryption + && session.activity.origin_work_generation == snapshot.origin_work_generation + }); + if !binding_current + || session.is_gc_marked_for_removal() + || capacity_admission != CapacityRetryAdmission::Ready + { + if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { + entry.status = SegmentCacheStatus::Discovered; + } + (false, session.segment_fetch_notifiers.remove(&snapshot.proxy_seq)) + } else { + if let Some(entry) = session.segments.get_mut(&snapshot.proxy_seq) { + entry.status = SegmentCacheStatus::Discovered; + } + (session.queue_segment_fetch_candidate(snapshot.proxy_seq, retry.priority, now_ms), None) + } + } else { + (false, None) + } + }; + if let Some(notifier) = abandoned_notifier { + notifier.notify_waiters(); + } + if requeued { + worker.wake_scheduler(context, now_ms).await; + } + }); + } +} + +async fn wait_for_capacity_retry_admission( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + mut revision: HlsCacheCapacityRevision, + projected_write_bytes: u64, +) -> CapacityRetryAdmission { + loop { + context.segment_cache.wait_for_capacity_change(&revision).await; + let binding_current = { + let session = context.session.read().await; + !session.is_gc_marked_for_removal() + && session.activity.origin_work_generation == snapshot.origin_work_generation + && session.segments.get(&snapshot.proxy_seq).is_some_and(|entry| { + entry.origin_key == snapshot.origin_key + && entry.cache_key == snapshot.cache_key + && entry.origin_fetch_ref.as_ref() == Some(&snapshot.fetch_ref) + && entry.encryption == snapshot.encryption + && entry.status.awaits_capacity_recovery() + }) + }; + if !binding_current { + return CapacityRetryAdmission::BindingExpired; + } + match context.segment_cache.ensure_projected_write_capacity(&snapshot.cache_key, projected_write_bytes).await { + Ok(()) => return CapacityRetryAdmission::Ready, + Err(error) => { + let Some(capacity) = super::super::cache::hls_cache_capacity_from_io(&error) else { + return CapacityRetryAdmission::LocalIoFailure; + }; + revision = capacity.revision().clone(); + } + } + } +} + +pub(super) fn take_queued_segment_fetch_candidate( + session: &mut super::super::HlsSession, + proxy_seq: u64, + priority: SegmentFetchPriority, + now_ms: u64, + has_usable_access_lease: bool, +) -> Option { + let entry = session.segments.get_mut(&proxy_seq)?; + if !matches!(entry.status, SegmentCacheStatus::Queued { .. }) { + return None; + } + if priority != SegmentFetchPriority::Demand && !has_usable_access_lease { + entry.status = SegmentCacheStatus::Discovered; + return None; + } + let Some(fetch_ref) = entry.origin_fetch_ref.clone() else { + entry.status = SegmentCacheStatus::Discovered; + return None; + }; + if !fetch_ref.is_valid_at(now_ms) { + entry.status = SegmentCacheStatus::Discovered; + return None; + } + Some(QueuedSegmentFetchCandidate { + origin_key: entry.origin_key, + cache_key: entry.cache_key.clone(), + fetch_ref, + encryption: entry.encryption.clone(), + proxy_file_ext: entry.proxy_file_ext.clone(), + complete_object: entry.origin_byte_range.is_none(), + }) +} + +pub(super) fn mark_segment_discovered(session: &mut super::super::HlsSession, proxy_seq: u64) { + if let Some(entry) = session.segments.get_mut(&proxy_seq) { + entry.status = SegmentCacheStatus::Discovered; + } +} + +impl Default for HlsSegmentWorkerPool { + fn default() -> Self { Self::new(SegmentFetchPolicy::default()) } +} diff --git a/backend/hls/src/segment_fetcher/startup_fill.rs b/backend/hls/src/segment_fetcher/startup_fill.rs new file mode 100644 index 000000000..8b767405b --- /dev/null +++ b/backend/hls/src/segment_fetcher/startup_fill.rs @@ -0,0 +1,365 @@ +use super::{ + segment_fetch_binding_matches, CachedSegmentMetadata, HlsSegmentRepairObjectContext, HlsSessionHandle, + SegmentFetchContext, SegmentFetchError, SegmentFetchSnapshot, +}; +use crate::sync_ext::MutexExt; +use log::debug; +use std::sync::Arc; +use tokio::time::Instant; +use tuliprox_core::utils::{content_coding::DecodedHttpResponse, current_time_millis}; + +/// Worst-case simultaneous on-disk copies of one segment body while a raw revision is filled. +const RAW_REVISION_PEAK_FACTOR: u64 = 5; +/// Worst-case simultaneous on-disk copies of one segment body while a processed revision is filled. +const PROCESSED_REVISION_PEAK_FACTOR: u64 = 3; + +const fn revision_peak_factor(raw: bool) -> u64 { + if raw { + RAW_REVISION_PEAK_FACTOR + } else { + PROCESSED_REVISION_PEAK_FACTOR + } +} + +pub(super) fn reliable_decoded_content_length(response: &DecodedHttpResponse) -> Option { + if response.was_content_decoded() { + return None; + } + response.headers.get(axum::http::header::CONTENT_LENGTH)?.to_str().ok()?.parse().ok() +} + +pub(super) struct HlsObservedFill { + pub(super) revision: super::super::SegmentRevisionGuard, + pub(super) session: HlsSessionHandle, + pub(super) context: SegmentFetchContext, + pub(super) file: Option, + pub(super) bytes: u64, + pub(super) expected: Option, + pub(super) limit: u64, + pub(super) path: std::path::PathBuf, + pub(super) generation: u64, + pub(super) budget: Arc, + pub(super) deadline: Instant, + pub(super) repair_permit: Option, + pub(super) additional_reservations: Vec, +} + +impl Drop for HlsObservedFill { + fn drop(&mut self) { self.revision.revision().fail(); } +} + +impl HlsObservedFill { + pub(super) async fn observe(&mut self, chunk: &[u8]) -> std::io::Result<()> { + use tokio::io::AsyncWriteExt; + self.bytes = self + .bytes + .checked_add(u64::try_from(chunk.len()).map_err(std::io::Error::other)?) + .ok_or_else(|| std::io::Error::other("origin body length overflow"))?; + if self.bytes > self.limit { + self.extend_disk_reservation().await?; + } + if let Some(file) = &mut self.file { + file.write_all(chunk).await?; + } + let has_prefix = if self.file.is_some() { + let mut replay = self.revision.revision().replay.lock_unpoisoned(); + let result = replay.as_mut().map(|replay| replay.push(chunk)); + match result { + Some(Err(_)) => { + replay.take(); + drop(replay); + self.revision.revision().replay_exhausted(); + debug!("HLS startup replay exhausted: proxy_seq={}", self.revision.revision().key.proxy_seq); + false + } + Some(Ok(())) => true, + None => false, + } + } else { + true + }; + if has_prefix { + let first = self.revision.revision().prefix_available.fetch_add( + u64::try_from(chunk.len()).map_err(std::io::Error::other)?, + std::sync::atomic::Ordering::AcqRel, + ) == 0; + self.revision.revision().prefix_changed(); + if first { + let mut session = self.session.write().await; + if session.activity.origin_work_generation == self.generation { + if let Err(error) = session.render_and_store_manifest(current_time_millis()) { + debug!( + "HLS startup manifest render after first prefix failed: proxy_seq={} error={error:?}", + self.revision.revision().key.proxy_seq + ); + } + let worker = session.startup.as_ref().and_then(|startup| startup.worker.upgrade()); + drop(session); + if let Some(worker) = worker { + worker.schedule_wake(self.context.clone(), current_time_millis()); + } + } + } + } + Ok(()) + } + + async fn extend_disk_reservation(&mut self) -> std::io::Result<()> { + if self.expected.is_some() { + return Err(std::io::Error::other("origin body exceeds decoded content length")); + } + let next_limit = self.limit.saturating_mul(2).max(self.bytes); + let factor = revision_peak_factor(self.file.is_some()); + let additional_peak = next_limit + .checked_sub(self.limit) + .and_then(|bytes| bytes.checked_mul(factor)) + .ok_or_else(|| std::io::Error::other("revision disk reservation overflow"))?; + let mut reservation = + self.context.segment_cache.reserve_revision_peak(&self.revision.revision().key, additional_peak).await?; + if self.file.is_some() { + reservation.mark_unaccounted_writes(); + } + self.additional_reservations.push(reservation); + self.limit = next_limit; + Ok(()) + } + + pub(super) async fn finish(&mut self) -> std::io::Result<()> { + use tokio::io::AsyncWriteExt; + if self.bytes == 0 || self.expected.is_some_and(|length| length != self.bytes) { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "incomplete origin segment body")); + } + if let Some(file) = self.file.take() { + let mut file = file; + file.flush().await?; + drop(file); + self.revision.revision().complete(CachedSegmentMetadata { path: self.path.clone(), size: self.bytes }); + self.revision.revision().replay.lock_unpoisoned().take(); + } + self.repair_permit = Some(self.budget.wait_for_repair(self.deadline).await?); + Ok(()) + } +} + +pub(super) async fn commit_startup_response( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + response: DecodedHttpResponse, + deadline: Instant, + mut repair_context: HlsSegmentRepairObjectContext, +) -> Result { + use futures::StreamExt; + if repair_context.hls_access_lease_id.is_none() { + let leases = context.session.read().await.startup.as_ref().map(|startup| Arc::clone(&startup.access_leases)); + if let Some(leases) = leases { + repair_context.hls_access_lease_id = leases + .write() + .await + .select_startup_repair_lease(&repair_context.proxy_session_id, current_time_millis()); + } + } + if let Some(lease_id) = &repair_context.hls_access_lease_id { + context.segment_repair.ensure_access_lease_window(lease_id.clone()).await; + } + let expected = reliable_decoded_content_length(&response); + if response.status != axum::http::StatusCode::OK { + return Err(SegmentFetchError::cache_body(&std::io::Error::other( + "startup requires a complete HTTP 200 response", + ))); + } + let prepared = prepare_startup_fill(context, snapshot, expected, deadline).await?; + let PreparedStartupFill { revision, observed, _reservation, raw } = prepared; + let chunks_owner = Arc::clone(&observed); + let finish_owner = Arc::clone(&observed); + let stream = tokio_util::io::ReaderStream::new(response.body) + .then(move |chunk| { + let observed = Arc::clone(&chunks_owner); + async move { + let chunk = chunk?; + observed.lock().await.observe(&chunk).await?; + Ok::<_, std::io::Error>(chunk) + } + }) + .chain(futures::stream::once(async move { + finish_owner.lock().await.finish().await?; + Ok::<_, std::io::Error>(bytes::Bytes::new()) + })); + let reader = Box::pin(tokio_util::io::StreamReader::new(stream.boxed())); + let mut metadata = context + .segment_repair + .commit_origin_response(&context.segment_cache, &snapshot.cache_key, reader, deadline, repair_context.clone()) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + if !raw { + context + .segment_repair + .repair_ready_cache_hit(&context.segment_cache, &snapshot.cache_key, repair_context) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + let latest = context + .segment_cache + .metadata(&snapshot.cache_key) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))? + .ok_or_else(|| { + SegmentFetchError::cache_body(&std::io::Error::other("processed cache object disappeared")) + })?; + metadata = latest; + let processed = context + .segment_cache + .publish_processed_revision(&revision.revision().key, &metadata.path, deadline) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + revision.revision().complete(processed); + } + drop(observed); + Ok(metadata) +} + +pub(super) struct PreparedStartupFill { + pub(super) revision: super::super::SegmentRevisionGuard, + pub(super) observed: Arc>, + pub(super) _reservation: super::super::HlsRevisionDiskReservation, + pub(super) raw: bool, +} + +pub(super) async fn prepare_startup_fill( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + expected: Option, + deadline: Instant, +) -> Result { + // A disk-capacity fallback switches the session to `first_ready` before it + // asks for another attempt, so the retry prepares a processed revision. + loop { + if let Some(prepared) = try_prepare_startup_fill(context, snapshot, expected, deadline).await? { + return Ok(prepared); + } + } +} + +async fn try_prepare_startup_fill( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + expected: Option, + deadline: Instant, +) -> Result, SegmentFetchError> { + let (store, budget, mut mode, limit, session_id) = { + let session = context.session.read().await; + if !session.segments.get(&snapshot.proxy_seq).is_some_and(super::super::SegmentEntry::is_fast_start_eligible) { + return Err(SegmentFetchError::UnsupportedStartupMedia); + } + let startup = session + .startup + .as_ref() + .ok_or_else(|| SegmentFetchError::cache_body(&std::io::Error::other("startup policy unavailable")))?; + ( + Arc::clone(&startup.store), + Arc::clone(&startup.budget), + startup.config.mode, + startup.config.max_progressive_bytes_per_segment.get(), + session.proxy_session_id.clone(), + ) + }; + let mut replay = (mode == shared::model::HlsStartupMode::Progressive + && expected.is_none_or(|length| length <= limit)) + .then(|| super::super::progressive_startup::ProgressiveReplay::new(&budget)) + .flatten(); + if mode == shared::model::HlsStartupMode::Progressive && replay.is_none() { + let mut session = context.session.write().await; + if session.startup.as_ref().is_some_and(|startup| !startup.policy_fixed) { + if let Some(startup) = &mut session.startup { + startup.config.mode = shared::model::HlsStartupMode::FirstReady; + } + mode = shared::model::HlsStartupMode::FirstReady; + debug!("HLS startup fallback: reason=replay_admission effective_mode=first_ready"); + } + } + let raw = mode == shared::model::HlsStartupMode::Progressive; + let kind = if raw { super::super::SegmentRevisionKind::Raw } else { super::super::SegmentRevisionKind::Processed }; + let revision = + store.create(session_id, snapshot.proxy_seq, kind).map_err(|error| SegmentFetchError::cache_body(&error))?; + let body_bound = expected.unwrap_or(limit).max(1); + let peak = body_bound + .checked_mul(revision_peak_factor(raw)) + .ok_or_else(|| SegmentFetchError::cache_body(&std::io::Error::other("revision peak overflow")))?; + let mut reservation = match context.segment_cache.reserve_revision_peak(&revision.revision().key, peak).await { + Ok(reservation) => reservation, + Err(_) if raw && fallback_to_first_ready(&context.session).await => { + debug!("HLS startup fallback: reason=disk_capacity effective_mode=first_ready"); + return Ok(None); + } + Err(error) => return Err(SegmentFetchError::cache_body(&error)), + }; + let path = context.segment_cache.object_path(&revision.revision().key); + let file = if raw { + let (file, pin) = context + .segment_cache + .create_revision_spool(&revision.revision().key) + .await + .map_err(|error| SegmentFetchError::cache_body(&error))?; + *revision.revision().file_pin.lock_unpoisoned() = Some(pin); + *revision.revision().replay.lock_unpoisoned() = replay.take(); + reservation.mark_unaccounted_writes(); + Some(file) + } else { + let pin = + context.segment_cache.pin_revision_file(&path).map_err(|error| SegmentFetchError::cache_body(&error))?; + *revision.revision().file_pin.lock_unpoisoned() = Some(pin); + None + }; + bind_startup_revision(context, snapshot, &revision).await?; + debug!("HLS startup fill: mode={mode} proxy_seq={} representation={kind:?}", snapshot.proxy_seq); + let observed = Arc::new(tokio::sync::Mutex::new(HlsObservedFill { + revision: revision.clone(), + session: Arc::clone(&context.session), + context: context.clone(), + file, + bytes: 0, + expected, + limit: body_bound, + path, + generation: snapshot.origin_work_generation, + budget, + deadline, + repair_permit: None, + additional_reservations: Vec::new(), + })); + Ok(Some(PreparedStartupFill { revision, observed, _reservation: reservation, raw })) +} + +async fn bind_startup_revision( + context: &SegmentFetchContext, + snapshot: &SegmentFetchSnapshot, + revision: &super::super::SegmentRevisionGuard, +) -> Result<(), SegmentFetchError> { + let mut session = context.session.write().await; + if session.activity.origin_work_generation != snapshot.origin_work_generation + || !session + .segments + .get(&snapshot.proxy_seq) + .is_some_and(|entry| segment_fetch_binding_matches(entry, snapshot)) + { + return Err(SegmentFetchError::cache_body(&std::io::Error::new( + std::io::ErrorKind::Interrupted, + "startup fetch generation retired", + ))); + } + if let Some(startup) = &mut session.startup { + startup.policy_fixed = true; + startup.revisions.insert(snapshot.proxy_seq, revision.clone()); + } + Ok(()) +} + +async fn fallback_to_first_ready(session: &HlsSessionHandle) -> bool { + let mut session = session.write().await; + if session.published_live_origin_baseline.is_some() { + return false; + } + let Some(startup) = session.startup.as_mut().filter(|startup| !startup.policy_fixed) else { + return false; + }; + startup.config.mode = shared::model::HlsStartupMode::FirstReady; + true +} diff --git a/backend/hls/src/segment_fetcher/switch_staging.rs b/backend/hls/src/segment_fetcher/switch_staging.rs new file mode 100644 index 000000000..3bd74a65f --- /dev/null +++ b/backend/hls/src/segment_fetcher/switch_staging.rs @@ -0,0 +1,87 @@ +use super::{ + build_segment_origin_headers, run_hls_origin_resource_retry_loop_with_attempt_prepare, HlsCacheObjectKey, + HlsOriginByteRangeExpectation, HlsOriginResourceClients, HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, + HlsResourceFetchKind, HlsResourceFetchSource, HlsSwitchResourceKind, SegmentFetchContext, SegmentFetchPolicy, + StagedCacheObject, +}; +use futures::FutureExt; +use log::warn; +use std::sync::Arc; +use tuliprox_parser::hls::origin_manifest::ParsedByteRange; + +/// Fetches one candidate-bound switch resource into an uncommitted cache object. +/// +/// The caller owns the returned object and must either commit it to the previewed cache key or remove it. This helper +/// shares the normal origin-resource Range/content-coding/retry implementation and performs no session mutation. +pub async fn stage_hls_switch_resource( + context: &SegmentFetchContext, + policy: &SegmentFetchPolicy, + cache_key: K, + origin_url: String, + byte_range: Option, + kind: HlsSwitchResourceKind, +) -> Result +where + K: HlsCacheObjectKey + Clone + Send + Sync + 'static, +{ + let headers = build_segment_origin_headers(&context.headers, &context.origin_provider_session_headers, byte_range)?; + let resource_kind = match kind { + HlsSwitchResourceKind::Segment => HlsResourceFetchKind::Segment, + HlsSwitchResourceKind::Map => HlsResourceFetchKind::Map, + }; + let target = HlsOriginResourceFetchTarget { + kind: resource_kind, + source: HlsResourceFetchSource::Normal, + object_id: match kind { + HlsSwitchResourceKind::Segment => "switch-segment".to_string(), + HlsSwitchResourceKind::Map => "switch-map".to_string(), + }, + origin_url, + headers, + byte_range_expectation: if byte_range.is_some() { + HlsOriginByteRangeExpectation::PartialContent + } else { + HlsOriginByteRangeExpectation::FullObject + }, + }; + let clients = HlsOriginResourceClients { + client: context.client.clone(), + no_redirect_client: context.no_redirect_client.clone(), + use_manual_redirects: context.use_manual_redirects, + }; + let log_identity = { + let session = context.session.read().await; + super::super::HlsLogIdentity::from_session(&session) + }; + let cache = Arc::clone(&context.segment_cache); + run_hls_origin_resource_retry_loop_with_attempt_prepare( + target, + clients, + policy, + &log_identity, + |_attempt| async { Ok(()) }.boxed(), + |()| async {}.boxed(), + move |response, _attempt, body_deadline, ()| { + let cache = Arc::clone(&cache); + let cache_key = cache_key.clone(); + async move { + let staged = cache + .stage_temp_with_deadline(&cache_key, response.body, body_deadline.deadline()) + .await + .map_err(|err| HlsOriginResourceFetchError::cache_body(&err))?; + if staged.size == 0 { + if let Err(err) = cache.remove_staged(staged).await { + warn!("HLS empty staged switch resource cleanup failed: error={err}"); + } + return Err(HlsOriginResourceFetchError::cache_body(&std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "empty switch resource", + ))); + } + Ok(staged) + } + .boxed() + }, + ) + .await +} diff --git a/backend/hls/src/segment_fetcher/tests.rs b/backend/hls/src/segment_fetcher/tests.rs new file mode 100644 index 000000000..a681a1046 --- /dev/null +++ b/backend/hls/src/segment_fetcher/tests.rs @@ -0,0 +1,23 @@ +use super::{ + build_segment_origin_headers, commit_failed_segment_fetch, take_queued_segment_fetch_candidate, + wait_for_segment_key_dependency, HlsSegmentFetchWorkload, SegmentFetchContext, SegmentFetchPolicy, + SegmentFetchPriority, +}; + +mod admission; +mod behavior; +mod lifecycle; +mod persistence; +mod protocol; +mod query; +mod retry; +mod startup; +mod support; + +use self::support::{ + clear_scheduled_prefetch, cold_startup_round, commit_test_key_ready, committed_segment, encode_test_body, + encrypted_fetch_context, fetch_context, fetch_context_with_access_lease, grant_usable_worker_access_lease, + grant_usable_worker_access_lease_at, install_startup, normal_manifest, prefix_drop_rounds, + shared_key_fetch_and_wait, spawn_segment_server, spawn_sequence_response_server, temp_cache_files, + test_segment_repair_manager, TestContentEncoding, TestOriginResponse, TestSegmentServer, +}; diff --git a/backend/hls/src/segment_fetcher/tests/admission.rs b/backend/hls/src/segment_fetcher/tests/admission.rs new file mode 100644 index 000000000..eb5045e95 --- /dev/null +++ b/backend/hls/src/segment_fetcher/tests/admission.rs @@ -0,0 +1,158 @@ +use super::{ + clear_scheduled_prefetch, commit_failed_segment_fetch, committed_segment, fetch_context, + grant_usable_worker_access_lease, normal_manifest, spawn_segment_server, spawn_sequence_response_server, + test_segment_repair_manager, SegmentFetchContext, SegmentFetchPolicy, SegmentFetchPriority, TestOriginResponse, +}; +use crate::{ + build_rewrite_secret_fingerprint, GarbageCollectionPolicy, HlsGarbageCollector, HlsOriginResourceFetchError, + HlsSegmentCache, HlsSegmentFile, HlsSegmentWorkerPool, HlsSessionKey, HlsSessionStore, SegmentCacheStatus, +}; +use axum::http::HeaderMap; +use std::sync::Arc; + +#[tokio::test] +async fn local_capacity_failure_does_not_change_origin_progress_or_failure_counter() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy::default(); + let (worker, context, _) = fetch_context(&server, &temp_dir, &policy).await; + let snapshot = worker.next_fetch_snapshot(&context, 20, &policy).await.expect("fetch snapshot"); + let error = HlsOriginResourceFetchError::LocalCacheCapacity { + required_session_bytes: 10, + required_global_bytes: 0, + projected_write_bytes: 10, + revision: context.segment_cache.capacity_revision(), + }; + let mut session = context.session.write().await; + let original_path_condition = session.origin_control.path_condition; + + commit_failed_segment_fetch(&mut session, &snapshot, &policy, &error, 21); + + assert_eq!(session.origin_control.path_condition, original_path_condition); + assert_eq!(session.segment_failure_tracker.consecutive_temporary_failures, 0); + assert!(matches!( + session.segments.get(&snapshot.proxy_seq).map(|segment| &segment.status), + Some(SegmentCacheStatus::CapacityDeferred { .. }) + )); +} + +#[tokio::test] +async fn projected_capacity_reclamation_commits_one_origin_download() { + let server = spawn_sequence_response_server(vec![TestOriginResponse { + status: 200, + headers: Vec::new(), + body: b"abcdefghij".to_vec(), + }]) + .await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let store = Arc::new(HlsSessionStore::new()); + let session = store.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; + { + let mut session = session.write().await; + session.configure_segment_prefetch_queue(policy.max_prefetch_queue_depth); + session.proxy_next_seq = Some(1); + session + .apply_origin_manifest(&normal_manifest(&format!( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\n{0}/1.ts\n#EXTINF:4.0,\n{0}/2.ts\n#EXTINF:4.0,\n{0}/3.ts\n#EXTINF:4.0,\n{0}/4.ts\n#EXTINF:4.0,\n{0}/5.ts\n#EXTINF:4.0,\n{0}/6.ts\n", + server.base_url + ))) + .expect("manifest maps"); + } + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let gc = Arc::new(HlsGarbageCollector::new( + Arc::clone(&store), + Arc::clone(&cache), + GarbageCollectionPolicy::default(), + build_rewrite_secret_fingerprint(b"secret"), + )); + cache.install_capacity_reclaimer(&gc); + let old_keys = { + let session = session.read().await; + assert!(session.segments.contains_key(&6), "mapped proxy sequences: {:?}", session.segments.keys()); + [1_u64, 2].map(|proxy_seq| session.segments.get(&proxy_seq).expect("old segment").cache_key.clone()) + }; + for key in &old_keys { + cache.write_bytes_and_commit(key, b"0123456789").await.expect("old segment commits"); + } + { + let mut session = session.write().await; + for proxy_seq in [1_u64, 2] { + session.segments.get_mut(&proxy_seq).expect("old segment").status = + SegmentCacheStatus::Ready { content_length: 10, ready_at_ms: proxy_seq }; + } + } + cache.update_cache_limits(100, 25); + let worker = Arc::new(HlsSegmentWorkerPool::new(policy.clone())); + let proxy_session_id = session.read().await.proxy_session_id.clone(); + grant_usable_worker_access_lease(&worker, &proxy_session_id).await; + let context = SegmentFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::clone(&cache), + segment_repair: test_segment_repair_manager(), + repair_access_lease_id: None, + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: true, + origin_io: None, + }; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker + .demand_fetch_and_wait(context.clone(), &HlsSegmentFile { proxy_seq: 6, extension: "ts".to_string() }, 20) + .await; + + let target_status = format!("{:?}", session.read().await.segments.get(&6).map(|segment| &segment.status)); + assert_eq!( + outcome, + super::super::SegmentDemandFetchOutcome::Ready, + "requests={} target_status={target_status}", + server.requests.lock().await.len(), + ); + assert_eq!(server.requests.lock().await.len(), 1, "staged body is not downloaded again"); + assert_eq!(committed_segment(&context, 6).await.1, b"abcdefghij"); + let session = session.read().await; + assert!(!session.segments.contains_key(&1)); + assert!(session.segments.contains_key(&2)); + drop(session); + let usage = cache.capacity_usage(&proxy_session_id).await.expect("capacity usage"); + assert_eq!(usage.session_bytes, 20); + assert!(!cache.has_active_temp_files()); +} + +#[tokio::test] +async fn demand_priority_runs_before_prefetch() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + max_global_segment_fetches: 1, + max_session_segment_fetches: 1, + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + { + let mut session = context.session.write().await; + session.segment_prefetch_queue = crate::SegmentPrefetchQueue::new(6); + session.segments.get_mut(&1).expect("segment").status = SegmentCacheStatus::Discovered; + session.segments.get_mut(&2).expect("segment").status = SegmentCacheStatus::Discovered; + session.queue_segment_fetch_candidate(2, SegmentFetchPriority::Prefetch, 10); + } + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Ready); + let requests = server.requests.lock().await; + let first_request = requests.first().expect("request should be made"); + assert!(first_request.starts_with("GET /1.ts ")); +} diff --git a/backend/hls/src/segment_fetcher/tests/behavior.rs b/backend/hls/src/segment_fetcher/tests/behavior.rs new file mode 100644 index 000000000..f77768134 --- /dev/null +++ b/backend/hls/src/segment_fetcher/tests/behavior.rs @@ -0,0 +1,747 @@ +use super::{ + clear_scheduled_prefetch, cold_startup_round, commit_test_key_ready, committed_segment, encode_test_body, + encrypted_fetch_context, fetch_context, fetch_context_with_access_lease, grant_usable_worker_access_lease, + grant_usable_worker_access_lease_at, install_startup, normal_manifest, prefix_drop_rounds, + shared_key_fetch_and_wait, spawn_segment_server, spawn_sequence_response_server, + take_queued_segment_fetch_candidate, temp_cache_files, test_segment_repair_manager, + wait_for_segment_key_dependency, SegmentFetchContext, SegmentFetchPolicy, SegmentFetchPriority, + TestContentEncoding, TestOriginResponse, TestSegmentServer, +}; +use crate::{ + HlsSegmentCache, HlsSegmentWorkerPool, HlsSessionKey, HlsSessionStore, RenderedManifest, SegmentCacheStatus, + TimelineMapError, TransientObjectFetchDecision, TransientResourceKind, TransientResourceRef, +}; +use axum::http::HeaderMap; +use shared::model::{HlsCacheConfigDto, HlsStripMode}; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::{oneshot, Mutex}, +}; +use tuliprox_core::{model::HlsCacheConfig, utils::current_time_millis}; +use tuliprox_parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}; + +#[test] +fn missing_segment_behind_ready_media_remains_a_readiness_failure() -> std::io::Result<()> { + let mut session = crate::HlsSession::new(HlsSessionKey::new(1, "startup"), b"secret", 0); + session + .apply_origin_manifest(&normal_manifest("#EXTM3U\n#EXTINF:4,\n0.ts\n#EXTINF:4,\n1.ts\n#EXTINF:4,\n2.ts\n")) + .map_err(|error| std::io::Error::other(format!("{error:?}")))?; + let head = session.publishable_origin_head_proxy_seq.ok_or_else(|| std::io::Error::other("head"))?; + session.segments.get_mut(&head).ok_or_else(|| std::io::Error::other("head entry"))?.status = + SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 0 }; + session.segments.get_mut(&(head + 1)).ok_or_else(|| std::io::Error::other("missing entry"))?.status = + SegmentCacheStatus::FailedPermanent { failed_at_ms: 1, status: Some(axum::http::StatusCode::GONE) }; + super::super::recompute_unpublished_live_head(&mut session); + assert_eq!(session.publishable_origin_head_proxy_seq, Some(head)); + assert_eq!(session.origin_control.path_condition, crate::HlsOriginPathCondition::SegmentReadinessFailure); + Ok(()) +} + +#[tokio::test] +async fn popped_candidate_without_usable_fetch_binding_returns_to_discovered() { + let server = spawn_sequence_response_server(vec![TestOriginResponse { + status: 200, + headers: Vec::new(), + body: b"unused".to_vec(), + }]) + .await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy::default(); + let (_, context, _) = fetch_context(&server, &temp_dir, &policy).await; + let mut session = context.session.write().await; + let entry = session.segments.get_mut(&1).expect("segment"); + entry.status = SegmentCacheStatus::Queued { priority: SegmentFetchPriority::Prefetch, queued_at_ms: 1 }; + entry.origin_fetch_ref = None; + + assert!(take_queued_segment_fetch_candidate(&mut session, 1, SegmentFetchPriority::Prefetch, 10, true,).is_none()); + assert!(matches!(session.segments[&1].status, SegmentCacheStatus::Discovered)); +} + +#[test] +fn object_failure_threshold_is_independent_from_initial_strip() { + let mut without_strip = HlsCacheConfigDto::default(); + without_strip.strip.mode = HlsStripMode::Segments; + without_strip.strip.value = 0; + let mut with_large_strip = without_strip.clone(); + with_large_strip.strip.mode = HlsStripMode::Seconds; + with_large_strip.strip.value = u64::MAX; + + let without_strip = SegmentFetchPolicy::from_config(&HlsCacheConfig::from(&without_strip)); + let with_large_strip = SegmentFetchPolicy::from_config(&HlsCacheConfig::from(&with_large_strip)); + + assert_eq!(without_strip.permanent_failure_segment_threshold, with_large_strip.permanent_failure_segment_threshold,); + assert_eq!( + without_strip.permanent_failure_segment_threshold, + SegmentFetchPolicy::default().permanent_failure_segment_threshold + ); +} + +#[tokio::test] +async fn segment_fetch_snapshot_uses_concrete_final_segment_fetch_url() { + let manifest = match parse_origin_media_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:4.0,\nmedia/seg001.ts\n", + "https://cdn.example.net/live/redirected/playlist.m3u8", + ) { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + }; + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "1"), b"secret", 0).await; + { + let mut session = session.write().await; + session.configure_segment_prefetch_queue(SegmentFetchPolicy::default().max_prefetch_queue_depth); + session.apply_origin_manifest(&manifest).expect("manifest maps"); + session.queue_manifest_prefetch_candidates(10); + } + + let worker = Arc::new(HlsSegmentWorkerPool::new(SegmentFetchPolicy::default())); + let proxy_session_id = session.read().await.proxy_session_id.clone(); + grant_usable_worker_access_lease(&worker, &proxy_session_id).await; + let temp_dir = tempfile::tempdir().expect("tempdir"); + let context = SegmentFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())), + segment_repair: test_segment_repair_manager(), + repair_access_lease_id: None, + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::new(), + use_manual_redirects: false, + origin_io: None, + }; + + let snapshot = + worker.next_fetch_snapshot(&context, 11, &SegmentFetchPolicy::default()).await.expect("segment snapshot"); + + assert_eq!(snapshot.fetch_ref.resolved_origin_url, "https://cdn.example.net/live/redirected/media/seg001.ts"); + assert_eq!(snapshot.fetch_ref.byte_range, None); + assert_eq!(snapshot.origin_seq, 10); +} + +pub(in crate::segment_fetcher::tests) async fn spawn_controlled_segment_server( +) -> (TestSegmentServer, oneshot::Receiver<()>, oneshot::Sender<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let (request_seen_tx, request_seen_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + let task = tokio::spawn(async move { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 2048]; + let Ok(read) = socket.read(&mut chunk).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + task_requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); + let _ = request_seen_tx.send(()); + if release_rx.await.is_err() { + return; + } + let body = b"controlled-segment"; + let head = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(body).await; + }); + (TestSegmentServer { base_url: format!("http://{addr}"), requests, task }, request_seen_rx, release_tx) +} + +pub(in crate::segment_fetcher::tests) async fn assert_only_one_prefetch_is_active_before_controlled_release( + policy: SegmentFetchPolicy, +) { + let (server, request_seen, release_response) = spawn_controlled_segment_server().await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let (worker, context, _) = fetch_context(&server, &temp_dir, &policy).await; + + worker.wake_scheduler(context.clone(), 20).await; + tokio::time::timeout(Duration::from_secs(10), request_seen) + .await + .expect("origin request starts before test deadline") + .expect("controlled origin observes the request"); + + let (active_proxy_seq, completion_notifier) = { + let mut session = context.session.write().await; + let fetching_proxy_seqs = session + .segments + .iter() + .filter_map(|(proxy_seq, segment)| { + matches!(segment.status, SegmentCacheStatus::Fetching { .. }).then_some(*proxy_seq) + }) + .collect::>(); + assert_eq!(session.active_segment_fetches, 1); + assert_eq!(fetching_proxy_seqs.len(), 1); + let active_proxy_seq = fetching_proxy_seqs[0]; + let completion_notifier = session.segment_fetch_notifiers.entry(active_proxy_seq).or_default().clone(); + session.invalidate_queued_origin_work(); + (active_proxy_seq, completion_notifier) + }; + assert_eq!(server.requests.lock().await.len(), 1); + + let completion = completion_notifier.notified(); + tokio::pin!(completion); + completion.as_mut().enable(); + release_response.send(()).expect("controlled origin response releases"); + tokio::time::timeout(Duration::from_secs(10), completion.as_mut()) + .await + .expect("active prefetch completes before test deadline"); + + let session = context.session.read().await; + assert_eq!(session.active_segment_fetches, 0); + assert!(matches!( + session.segments.get(&active_proxy_seq).map(|segment| &segment.status), + Some(SegmentCacheStatus::Discovered) + )); +} + +#[tokio::test] +async fn ready_encrypted_segments_schedule_one_key_only_fetch() { + let server = spawn_sequence_response_server(vec![TestOriginResponse { + status: 200, + headers: Vec::new(), + body: b"0123456789abcdef".to_vec(), + }]) + .await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = + SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; + let (worker, context, _) = encrypted_fetch_context(&server, &temp_dir, &policy).await; + let proxy_session_id = { + let mut session = context.session.write().await; + for segment in session.segments.values_mut() { + segment.status = SegmentCacheStatus::Ready { content_length: 32, ready_at_ms: 10 }; + } + session.last_rendered_manifest = Some(RenderedManifest { + body: "#EXTM3U\n".to_string(), + first_proxy_seq: 1, + last_proxy_seq: 3, + discontinuity_sequence: 0, + target_duration_ms: 4_000, + playlist_duration_ms: 12_000, + valid_until_ms: u64::MAX, + render_gap_segments: 0, + rendered_at_ms: 10, + segment_proxy_seqs: vec![1, 2, 3], + }); + session.proxy_session_id.clone() + }; + let now_ms = current_time_millis(); + grant_usable_worker_access_lease_at(&worker, &proxy_session_id, now_ms).await; + + tokio::join!(worker.wake_scheduler(context.clone(), now_ms), worker.wake_scheduler(context.clone(), now_ms)); + + let pending_key = { + let mut session = context.session.write().await; + let resource = session + .transient + .resources + .values() + .find(|resource| resource.kind == TransientResourceKind::Key) + .cloned() + .expect("encrypted fixture key resource"); + let extension = resource.file_ext_hint.clone().expect("key extension"); + let proxy_session_id = session.proxy_session_id.clone(); + let cache_duration_ms = session.transient.resource_ttl_ms; + match session.transient.begin_object_fetch(&proxy_session_id, &resource, &extension, now_ms, cache_duration_ms) + { + TransientObjectFetchDecision::Ready => None, + TransientObjectFetchDecision::Wait(notifier) => Some((notifier, resource.id, extension)), + TransientObjectFetchDecision::Fetch(_) => { + panic!("rendered READY media must already have scheduled its key-only fetch") + } + } + }; + if let Some((notifier, resource_id, extension)) = pending_key { + tokio::time::timeout( + Duration::from_secs(10), + wait_for_segment_key_dependency(&context, notifier, resource_id, extension, Duration::from_secs(10)), + ) + .await + .expect("key-only readiness completes before test deadline") + .expect("key-only readiness succeeds"); + } + + let session = context.session.read().await; + assert_eq!(session.active_segment_fetches, 0); + assert!(session.ready_timeline_snapshot(1, now_ms).units.iter().all(|unit| unit.required_key_ready)); + assert!(session.segments.values().all(|segment| matches!(segment.status, SegmentCacheStatus::Ready { .. }))); + drop(session); + let requests = server.requests.lock().await; + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with("GET /key.key ")); +} + +#[tokio::test] +async fn completed_shared_key_before_wait_registration_is_observed_without_timeout() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy::default(); + let (_, context, _) = encrypted_fetch_context(&server, &temp_dir, &policy).await; + let now_ms = current_time_millis(); + let (fetch_token, notifier, resource_id, extension) = shared_key_fetch_and_wait(&context, now_ms).await; + commit_test_key_ready(&context, &fetch_token, now_ms.saturating_add(1)).await; + + let result = + wait_for_segment_key_dependency(&context, notifier, resource_id, extension, Duration::from_millis(1)).await; + + assert!(result.is_ok()); +} + +#[tokio::test] +async fn invalid_aes_key_size_blocks_segment_fetch_and_ready_reserve() { + let server = spawn_sequence_response_server(vec![TestOriginResponse { + status: 200, + headers: Vec::new(), + body: vec![b'k'; 15], + }]) + .await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = + SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; + let (worker, context, segment_file) = encrypted_fetch_context(&server, &temp_dir, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Unavailable); + let session = context.session.read().await; + let first = session.segments.get(&1).expect("encrypted segment"); + assert!(matches!(first.status, SegmentCacheStatus::FailedPermanent { .. })); + assert!(!session.ready_timeline_snapshot(1, 20).units[0].required_key_ready); + assert_eq!(session.activity.media_readiness_generation, 0); + drop(session); + let requests = server.requests.lock().await; + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with("GET /key.key ")); +} + +#[tokio::test] +async fn invalidated_generation_discards_late_segment_completion_without_sleep() { + let (server, request_seen, release_response) = spawn_controlled_segment_server().await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + let fetch_context = context.clone(); + let fetch = tokio::spawn(async move { worker.demand_fetch_and_wait(fetch_context, &segment_file, 20).await }); + + request_seen.await.expect("origin request starts"); + context.session.write().await.invalidate_queued_origin_work(); + release_response.send(()).expect("origin response released"); + + assert_eq!(fetch.await.expect("fetch task joins"), super::super::SegmentDemandFetchOutcome::Unavailable); + let cache_key = { + let session = context.session.read().await; + assert_eq!(session.active_segment_fetches, 0); + let segment = session.segments.get(&1).expect("segment remains mapped"); + assert!(matches!(segment.status, SegmentCacheStatus::Discovered)); + segment.cache_key.clone() + }; + assert!(context.segment_cache.metadata(&cache_key).await.expect("metadata reads").is_none()); +} + +#[tokio::test] +async fn same_origin_sequence_resource_conflict_preserves_in_flight_binding() { + let (server, request_seen, release_response) = spawn_controlled_segment_server().await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + let fetch_context = context.clone(); + let fetch = tokio::spawn(async move { worker.demand_fetch_and_wait(fetch_context, &segment_file, 20).await }); + + request_seen.await.expect("origin request starts"); + let key_uri = format!("{}/rebound.key", server.base_url); + let key_resource = TransientResourceRef::new( + TransientResourceKind::Key, + &key_uri, + b"secret", + 0, + u64::MAX, + Some("key".to_string()), + ); + let mut rebound_manifest = normal_manifest(&format!( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-KEY:METHOD=AES-128,URI=\"{key_uri}\"\n\ + #EXTINF:4.0,\n{}/rebound.ts\n#EXT-X-KEY:METHOD=NONE\n#EXTINF:4.0,\n{}/2.ts\n\ + #EXTINF:4.0,\n{}/3.ts\n", + server.base_url, server.base_url, server.base_url + )); + for encryption in rebound_manifest.segments.iter_mut().filter_map(|segment| segment.encryption.as_mut()) { + encryption.proxy_resource_id = Some(key_resource.id.0.clone()); + encryption.proxy_resource_extension = Some("key".to_string()); + } + let (origin_work_generation, cache_key) = { + let mut session = context.session.write().await; + let origin_work_generation = session.activity.origin_work_generation; + session.transient.upsert_resources([key_resource]); + assert!(matches!( + session.apply_origin_manifest(&rebound_manifest), + Err(TimelineMapError::OriginSequenceResourceConflict { candidate_origin_seq: 1, .. }) + )); + assert_eq!(session.activity.origin_work_generation, origin_work_generation); + let segment = session.segments.get(&1).expect("original proxy segment remains mapped"); + assert!(matches!(segment.status, SegmentCacheStatus::Fetching { .. })); + assert_eq!( + segment.origin_fetch_ref.as_ref().expect("original fetch ref").resolved_origin_url, + format!("{}/1.ts", server.base_url) + ); + assert!(segment.encryption.is_none()); + (origin_work_generation, segment.cache_key.clone()) + }; + + release_response.send(()).expect("origin response released"); + + assert_eq!(fetch.await.expect("fetch task joins"), super::super::SegmentDemandFetchOutcome::Ready); + let session = context.session.read().await; + assert_eq!(session.activity.origin_work_generation, origin_work_generation); + assert_eq!(session.active_segment_fetches, 0); + let segment = session.segments.get(&1).expect("original segment remains mapped"); + assert!(matches!(segment.status, SegmentCacheStatus::Ready { .. })); + assert_eq!( + segment.origin_fetch_ref.as_ref().expect("original fetch ref remains authoritative").resolved_origin_url, + format!("{}/1.ts", server.base_url) + ); + assert!(segment.encryption.is_none()); + drop(session); + let (committed_key, bytes) = committed_segment(&context, 1).await; + assert_eq!(committed_key, cache_key); + assert_eq!(bytes, b"controlled-segment"); + assert!(temp_cache_files(temp_dir.path()).is_empty()); +} + +#[tokio::test] +async fn segment_declared_codings_decode_to_identity_before_cache_and_preserve_opaque_bytes() { + let identity_bytes = b"\x00\xffopaque-hls-ciphertext\x1f\x8bmedia".to_vec(); + + for encoding in [ + TestContentEncoding::Gzip, + TestContentEncoding::RawDeflate, + TestContentEncoding::Brotli, + TestContentEncoding::Zstd, + ] { + let encoded = encode_test_body(&identity_bytes, encoding).await; + let server = + spawn_sequence_response_server(vec![TestOriginResponse::encoded(encoding.header_value(), encoded)]).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Ready, "failed for {encoding:?}"); + let (cache_key, cached) = committed_segment(&context, 1).await; + assert_eq!(cached, identity_bytes, "cache retained HTTP coding for {encoding:?}"); + let mut range = context.segment_cache.open_range(&cache_key, 5).await.expect("cache range opens"); + let mut ranged = Vec::new(); + range.read_to_end(&mut ranged).await.expect("cache range reads"); + assert_eq!(ranged, identity_bytes[5..], "range did not address Identity bytes for {encoding:?}"); + let requests = server.requests.lock().await; + assert_eq!(requests.len(), 1, "unexpected retries for {encoding:?}"); + assert!( + requests[0].to_ascii_lowercase().contains("accept-encoding: identity"), + "request did not enforce identity for {encoding:?}" + ); + } +} + +#[tokio::test] +async fn segment_decoder_failure_exhausts_attempt_budget_without_cache_or_temp_file() { + let valid = encode_test_body(b"never committed", TestContentEncoding::Gzip).await; + let mut corrupt = valid; + corrupt.truncate(corrupt.len() / 2); + let server = + spawn_sequence_response_server((0..5).map(|_| TestOriginResponse::encoded("gzip", corrupt.clone())).collect()) + .await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::TimedOut); + assert_eq!(server.requests.lock().await.len(), 5); + let cache_key = context.session.read().await.segments.get(&1).expect("segment").cache_key.clone(); + assert!(context.segment_cache.metadata(&cache_key).await.expect("metadata reads").is_none()); + assert!(!context.segment_cache.has_active_temp_files()); + assert!(temp_cache_files(temp_dir.path()).is_empty()); +} + +#[tokio::test] +async fn background_segment_fetch_without_usable_access_lease_resets_queue_without_origin_request() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, _) = fetch_context_with_access_lease(&server, &temp_dir, &policy, false).await; + + worker.wake_scheduler(context.clone(), 20).await; + + assert!(server.requests.lock().await.is_empty()); + let session = context.session.read().await; + assert_eq!(session.active_segment_fetches, 0); + assert!(session.segment_prefetch_queue.is_empty()); + assert!(session.segments.values().all(|segment| matches!(segment.status, SegmentCacheStatus::Discovered))); +} + +#[tokio::test] +async fn demand_fetch_starts_without_worker_usable_access_lease() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context_with_access_lease(&server, &temp_dir, &policy, false).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Ready); + assert_eq!(server.requests.lock().await.len(), 1); +} + +#[tokio::test] +async fn demand_fetch_returns_unavailable_when_fetch_slots_are_saturated() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + max_global_segment_fetches: 1, + max_session_segment_fetches: 1, + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + context.session.write().await.active_segment_fetches = 1; + + let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Unavailable); + assert!(server.requests.lock().await.is_empty()); +} + +#[tokio::test] +async fn ready_cache_hit_is_allowed_when_fetch_slots_are_saturated() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + max_global_segment_fetches: 1, + max_session_segment_fetches: 1, + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + { + let mut session = context.session.write().await; + session.active_segment_fetches = 1; + session.segments.get_mut(&1).expect("segment").status = + SegmentCacheStatus::Ready { content_length: 12, ready_at_ms: 20 }; + } + + let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Ready); + assert!(server.requests.lock().await.is_empty()); +} + +#[tokio::test] +async fn one_proxy_sequence_has_at_most_one_active_origin_fetch() { + let server = spawn_segment_server(80).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + max_global_segment_fetches: 4, + max_session_segment_fetches: 4, + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let first = { + let worker = Arc::clone(&worker); + let context = context.clone(); + let segment_file = segment_file.clone(); + tokio::spawn(async move { worker.demand_fetch_and_wait(context, &segment_file, 20).await }) + }; + let second = { + let worker = Arc::clone(&worker); + let context = context.clone(); + let segment_file = segment_file.clone(); + tokio::spawn(async move { worker.demand_fetch_and_wait(context, &segment_file, 21).await }) + }; + + assert_eq!(first.await.expect("task"), super::super::SegmentDemandFetchOutcome::Ready); + assert_eq!(second.await.expect("task"), super::super::SegmentDemandFetchOutcome::Ready); + assert_eq!(server.requests.lock().await.len(), 1); +} + +#[tokio::test] +async fn session_limit_is_respected() { + let policy = SegmentFetchPolicy { + max_global_segment_fetches: 4, + max_session_segment_fetches: 1, + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + + assert_only_one_prefetch_is_active_before_controlled_release(policy).await; +} + +#[tokio::test] +async fn global_limit_is_respected() { + let policy = SegmentFetchPolicy { + max_global_segment_fetches: 1, + max_session_segment_fetches: 3, + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + + assert_only_one_prefetch_is_active_before_controlled_release(policy).await; +} + +#[tokio::test] +async fn unknown_body_reserves_more_disk_before_growing_its_raw_spool() -> std::io::Result<()> { + let server = spawn_sequence_response_server(Vec::new()).await; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy::default(); + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, shared::model::HlsStartupMode::Progressive).await; + let snapshot = worker + .next_fetch_snapshot(&context, 10, &policy) + .await + .ok_or_else(|| std::io::Error::other("fetch snapshot"))?; + let prepared = super::super::prepare_startup_fill( + &context, + &snapshot, + None, + tokio::time::Instant::now() + Duration::from_secs(1), + ) + .await + .map_err(|error| std::io::Error::other(format!("{error:?}")))?; + let mut observed = prepared.observed.lock().await; + observed.limit = 8; + observed.observe(b"segment-body").await?; + assert_eq!(observed.additional_reservations.len(), 1); + observed.finish().await?; + let metadata = + prepared.revision.revision().wait_complete(tokio::time::Instant::now() + Duration::from_secs(1)).await?; + assert_eq!(tokio::fs::read(metadata.path).await?, b"segment-body"); + Ok(()) +} + +#[tokio::test] +#[ignore = "extended single-process loopback startup, origin-drop and retention soak"] +async fn extended_fast_start_soak() -> std::io::Result<()> { + let seconds = std::env::var("TULIPROX_HLS_SOAK_SECS") + .ok() + .map_or(Ok(1800), |value| value.parse::().map_err(std::io::Error::other))?; + let directory = tempfile::tempdir()?; + let store = Arc::new(crate::SegmentRevisionStore::default()); + let budget = crate::ProgressiveBudgetManager::new(tuliprox_core::model::HlsStartupConfig::default()); + let cache = Arc::new(HlsSegmentCache::with_cache_path(directory.path())); + cache.update_cache_limits(1000, 500); + let started = tokio::time::Instant::now(); + let end = started + Duration::from_secs(seconds); + let mut cycles = 0_u64; + let mut reported = 0_u64; + while tokio::time::Instant::now() < end { + let mode = if cycles.is_multiple_of(2) { + shared::model::HlsStartupMode::FirstReady + } else { + shared::model::HlsStartupMode::Progressive + }; + cold_startup_round(mode, Some((&directory, &store, &budget, &cache))).await?; + if cycles.is_multiple_of(10) { + prefix_drop_rounds().await?; + } + tokio::time::sleep(Duration::from_millis(50)).await; + for owner in store.retire_unpinned()? { + owner.file_pin.lock().map_err(|_| std::io::Error::other("file pin"))?.take(); + cache.delete_if_inactive(&owner.key).await?; + } + assert_eq!(budget.usage(), (0, 0)); + assert_eq!(cache.scan_revision_disk_bytes().await?, 0); + assert!(store.retire_unpinned()?.is_empty()); + let tasks = tokio::runtime::Handle::current().metrics().num_alive_tasks(); + assert!(tasks <= 8, "completed startup retained {tasks} tasks"); + cycles += 1; + let elapsed = started.elapsed().as_secs(); + if elapsed / 60 > reported { + reported = elapsed / 60; + let rss = std::fs::read_to_string("/proc/self/status") + .ok() + .and_then(|status| status.lines().find(|line| line.starts_with("VmRSS:")).map(str::to_owned)) + .unwrap_or_default(); + println!("startup_soak elapsed_secs={elapsed} cycles={cycles} live_tasks={tasks} replay_bytes=0 revision_disk_bytes=0 {rss}"); + } + } + println!("startup_soak completed elapsed_secs={} cycles={cycles}", started.elapsed().as_secs()); + Ok(()) +} + +#[tokio::test] +async fn a_published_raw_session_rejects_later_incompatible_segments() -> std::io::Result<()> { + let server = spawn_segment_server(0).await; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy::default(); + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, shared::model::HlsStartupMode::Progressive).await; + let mut session = context.session.write().await; + let id = session.proxy_session_id.clone(); + if let Some(startup) = &mut session.startup { + for seq in 1..=3 { + let owner = startup.store.create(id.clone(), seq, crate::SegmentRevisionKind::Raw)?; + owner.revision().prefix_available.store(8, Ordering::Release); + startup.revisions.insert(seq, owner); + } + } + session + .render_and_store_manifest(current_time_millis()) + .map_err(|error| std::io::Error::other(format!("{error:?}")))?; + let successor = session.segments.get_mut(&2).ok_or_else(|| std::io::Error::other("successor"))?; + successor.proxy_file_ext = "m4s".into(); + assert!(session.render_and_store_manifest(current_time_millis()).is_err()); + assert_eq!(session.startup_mode(), shared::model::HlsStartupMode::Progressive); + Ok(()) +} diff --git a/backend/hls/src/segment_fetcher/tests/lifecycle.rs b/backend/hls/src/segment_fetcher/tests/lifecycle.rs new file mode 100644 index 000000000..6f3ddbe60 --- /dev/null +++ b/backend/hls/src/segment_fetcher/tests/lifecycle.rs @@ -0,0 +1,108 @@ +use super::{ + encrypted_fetch_context, fetch_context, spawn_segment_server, spawn_sequence_response_server, + take_queued_segment_fetch_candidate, SegmentFetchPolicy, SegmentFetchPriority, TestOriginResponse, +}; +use crate::{HlsSegmentFile, SegmentCacheStatus, TransientResourceKind}; +use std::time::Duration; +use tuliprox_core::utils::current_time_millis; + +#[tokio::test] +async fn popped_candidate_with_expired_fetch_binding_returns_to_discovered() { + let server = spawn_sequence_response_server(vec![TestOriginResponse { + status: 200, + headers: Vec::new(), + body: b"unused".to_vec(), + }]) + .await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy::default(); + let (_, context, _) = fetch_context(&server, &temp_dir, &policy).await; + let mut session = context.session.write().await; + let entry = session.segments.get_mut(&1).expect("segment"); + entry.status = SegmentCacheStatus::Queued { priority: SegmentFetchPriority::Prefetch, queued_at_ms: 1 }; + entry.origin_fetch_ref.as_mut().expect("fetch binding").valid_until_ms = Some(9); + + assert!(take_queued_segment_fetch_candidate(&mut session, 1, SegmentFetchPriority::Prefetch, 10, true,).is_none()); + assert!(matches!(session.segments[&1].status, SegmentCacheStatus::Discovered)); +} + +#[tokio::test] +async fn expired_aes_key_is_refetched_before_a_second_segment_becomes_ready() { + let server = spawn_sequence_response_server(vec![ + TestOriginResponse { status: 200, headers: Vec::new(), body: b"0123456789abcdef".to_vec() }, + TestOriginResponse { status: 200, headers: Vec::new(), body: b"encrypted-media-a".to_vec() }, + TestOriginResponse { status: 200, headers: Vec::new(), body: b"fedcba9876543210".to_vec() }, + TestOriginResponse { status: 200, headers: Vec::new(), body: b"encrypted-media-b".to_vec() }, + ]) + .await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = + SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; + let (worker, context, _) = encrypted_fetch_context(&server, &temp_dir, &policy).await; + { + let mut session = context.session.write().await; + session.transient.set_resource_ttl_ms(1); + } + + let first_segment = HlsSegmentFile { proxy_seq: 1, extension: "ts".to_string() }; + let first_outcome = worker.demand_fetch_and_wait(context.clone(), &first_segment, current_time_millis()).await; + assert_eq!(first_outcome, super::super::SegmentDemandFetchOutcome::Ready); + + tokio::time::sleep(Duration::from_millis(50)).await; + { + let session = context.session.read().await; + let key = session + .transient + .resources + .values() + .find(|resource| resource.kind == TransientResourceKind::Key) + .cloned() + .expect("encrypted fixture key resource"); + assert!(session + .transient + .ready_key_object_valid_until_ms( + &session.proxy_session_id, + &key.id, + key.file_ext_hint.as_deref().expect("key extension"), + current_time_millis(), + ) + .is_none()); + } + + let second_segment = HlsSegmentFile { proxy_seq: 2, extension: "ts".to_string() }; + let second_outcome = worker.demand_fetch_and_wait(context.clone(), &second_segment, current_time_millis()).await; + assert_eq!(second_outcome, super::super::SegmentDemandFetchOutcome::Ready); + + { + let session = context.session.read().await; + assert!(matches!( + session.segments.get(&2).map(|segment| &segment.status), + Some(SegmentCacheStatus::Ready { .. }) + )); + } + + let requests = server.requests.lock().await; + assert_eq!(requests.len(), 4, "requests={requests:?}"); + assert!(requests[0].starts_with("GET /key.key ")); + assert!(requests[1].starts_with("GET /1.ts ")); + assert!(requests[2].starts_with("GET /key.key ")); + assert!(requests[3].starts_with("GET /2.ts ")); +} + +#[tokio::test] +async fn demand_fetch_is_blocked_for_gc_marked_session() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + context.session.write().await.mark_for_gc_removal(); + + let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::NotFound); + assert!(server.requests.lock().await.is_empty()); +} diff --git a/backend/hls/src/segment_fetcher/tests/persistence.rs b/backend/hls/src/segment_fetcher/tests/persistence.rs new file mode 100644 index 000000000..1da9dae2a --- /dev/null +++ b/backend/hls/src/segment_fetcher/tests/persistence.rs @@ -0,0 +1,92 @@ +use super::{ + clear_scheduled_prefetch, committed_segment, encode_test_body, encrypted_fetch_context, fetch_context, + spawn_segment_server, spawn_sequence_response_server, temp_cache_files, SegmentFetchPolicy, TestContentEncoding, + TestOriginResponse, +}; +use crate::SegmentCacheStatus; + +#[tokio::test] +async fn demand_fetch_writes_cache_and_sets_ready_after_commit() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Ready); + let session = context.session.read().await; + assert!(matches!(session.segments.get(&1).expect("segment").status, SegmentCacheStatus::Ready { .. })); +} + +#[tokio::test] +async fn encrypted_segment_fetch_commits_exact_key_before_media_becomes_ready() { + let server = spawn_sequence_response_server(vec![ + TestOriginResponse { status: 200, headers: Vec::new(), body: b"0123456789abcdef".to_vec() }, + TestOriginResponse { status: 200, headers: Vec::new(), body: b"encrypted-media".to_vec() }, + ]) + .await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = + SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; + let (worker, context, segment_file) = encrypted_fetch_context(&server, &temp_dir, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Ready); + let session = context.session.read().await; + let first = session.segments.get(&1).expect("encrypted segment"); + assert!(matches!(first.status, SegmentCacheStatus::Ready { .. })); + let encryption = first.encryption.as_ref().expect("segment keeps key dependency"); + assert!(session + .transient + .ready_key_object_valid_until_ms( + &session.proxy_session_id, + &encryption.resource_id, + &encryption.resource_extension, + 20, + ) + .is_some()); + assert!(session.ready_timeline_snapshot(1, 20).units[0].required_key_ready); + assert!(session.activity.media_readiness_generation >= 2); + drop(session); + let requests = server.requests.lock().await; + assert!(requests[0].starts_with("GET /key.key ")); + assert!(requests[1].starts_with("GET /1.ts ")); +} + +#[tokio::test] +async fn segment_decoder_failure_retries_then_commits_and_cleans_temp_file() { + let identity_bytes = b"identity-media-after-retry".to_vec(); + let valid = encode_test_body(&identity_bytes, TestContentEncoding::Gzip).await; + let mut corrupt = valid.clone(); + corrupt.truncate(corrupt.len() / 2); + let server = spawn_sequence_response_server(vec![ + TestOriginResponse::encoded("gzip", corrupt), + TestOriginResponse::encoded("gzip", valid), + ]) + .await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Ready); + assert_eq!(committed_segment(&context, 1).await.1, identity_bytes); + let requests = server.requests.lock().await; + assert_eq!(requests.len(), 2); + assert!(requests.iter().all(|request| request.to_ascii_lowercase().contains("accept-encoding: identity"))); + drop(requests); + assert!(!context.segment_cache.has_active_temp_files()); + assert!(temp_cache_files(temp_dir.path()).is_empty()); +} diff --git a/backend/hls/src/segment_fetcher/tests/protocol.rs b/backend/hls/src/segment_fetcher/tests/protocol.rs new file mode 100644 index 000000000..c29654047 --- /dev/null +++ b/backend/hls/src/segment_fetcher/tests/protocol.rs @@ -0,0 +1,124 @@ +use super::{ + build_segment_origin_headers, normal_manifest, test_segment_repair_manager, SegmentFetchContext, + SegmentFetchPolicy, TestSegmentServer, +}; +use crate::{ + HlsSegmentCache, HlsSegmentFile, HlsSegmentWorkerPool, HlsSessionKey, HlsSessionStore, SegmentCacheStatus, +}; +use axum::http::{header, HeaderMap, HeaderValue}; +use std::sync::Arc; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::Mutex, +}; + +#[test] +fn segment_origin_headers_remove_client_range_and_force_identity() { + let mut headers = HeaderMap::new(); + headers.insert(header::RANGE, HeaderValue::from_static("bytes=0-")); + headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip")); + headers.insert(header::AUTHORIZATION, HeaderValue::from_static("Bearer secret")); + headers.insert(header::COOKIE, HeaderValue::from_static("sid=secret")); + headers.insert(header::HOST, HeaderValue::from_static("origin.example.com")); + let headers = build_segment_origin_headers(&headers, &HeaderMap::new(), None).expect("headers should build"); + + assert!(!headers.contains_key(header::RANGE)); + assert!(!headers.contains_key(header::AUTHORIZATION)); + assert!(!headers.contains_key(header::COOKIE)); + assert!(!headers.contains_key(header::HOST)); + assert_eq!(headers.get(header::ACCEPT_ENCODING).expect("encoding"), "identity"); +} + +#[test] +fn segment_origin_headers_apply_byterange() { + let headers = build_segment_origin_headers( + &HeaderMap::new(), + &HeaderMap::new(), + Some(tuliprox_parser::hls::origin_manifest::ParsedByteRange { offset: 10, length: 5 }), + ) + .expect("headers should build"); + + assert_eq!(headers.get(header::RANGE).expect("range"), "bytes=10-14"); +} + +pub(in crate::segment_fetcher::tests) async fn spawn_range_segment_server() -> TestSegmentServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let task = tokio::spawn(async move { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + let mut buf = vec![0_u8; 2048]; + let Ok(read) = socket.read(&mut buf).await else { + return; + }; + if read == 0 { + return; + } + let request = String::from_utf8_lossy(&buf[..read]).to_string(); + task_requests.lock().await.push(request.clone()); + let body = if request.to_ascii_lowercase().contains("range: bytes=10-14") { "seg!!" } else { "bad" }; + let status = if body == "seg!!" { "206 Partial Content" } else { "200 OK" }; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nContent-Range: bytes 10-14/20\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = socket.write_all(response.as_bytes()).await; + }); + + TestSegmentServer { base_url: format!("http://{addr}"), requests, task } +} + +#[tokio::test] +async fn origin_byterange_segment_fetch_uses_http_range_and_stores_logical_segment() { + let server = spawn_range_segment_server().await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + origin_segment_timeout_ms: 1_000, + ..SegmentFetchPolicy::default() + }; + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; + { + let mut session = session.write().await; + session + .apply_origin_manifest(&normal_manifest(&format!( + "#EXTM3U\n#EXT-X-BYTERANGE:5@10\n#EXTINF:4.0,\n{}/big.m4s\n", + server.base_url + ))) + .expect("manifest maps"); + } + let context = SegmentFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())), + segment_repair: test_segment_repair_manager(), + repair_access_lease_id: None, + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: true, + origin_io: None, + }; + let worker = Arc::new(HlsSegmentWorkerPool::new(policy)); + let segment_file = HlsSegmentFile { proxy_seq: 0, extension: "m4s".to_string() }; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Ready); + let requests = server.requests.lock().await; + assert_eq!(requests.len(), 1); + assert!(requests[0].to_ascii_lowercase().contains("range: bytes=10-14")); + let session = context.session.read().await; + let segment = session.segments.get(&0).expect("segment"); + assert!(matches!(segment.status, SegmentCacheStatus::Ready { content_length: 5, .. })); + assert!(context.segment_cache.metadata(&segment.cache_key).await.expect("metadata").is_some()); +} diff --git a/backend/hls/src/segment_fetcher/tests/query.rs b/backend/hls/src/segment_fetcher/tests/query.rs new file mode 100644 index 000000000..0c0e27112 --- /dev/null +++ b/backend/hls/src/segment_fetcher/tests/query.rs @@ -0,0 +1,24 @@ +use super::{ + commit_test_key_ready, encrypted_fetch_context, shared_key_fetch_and_wait, spawn_segment_server, + wait_for_segment_key_dependency, SegmentFetchPolicy, +}; +use futures::poll; +use std::{task::Poll, time::Duration}; +use tuliprox_core::utils::current_time_millis; + +#[tokio::test] +async fn registered_shared_key_waiter_observes_controlled_ready_transition() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy::default(); + let (_, context, _) = encrypted_fetch_context(&server, &temp_dir, &policy).await; + let now_ms = current_time_millis(); + let (fetch_token, notifier, resource_id, extension) = shared_key_fetch_and_wait(&context, now_ms).await; + let mut wait = + Box::pin(wait_for_segment_key_dependency(&context, notifier, resource_id, extension, Duration::from_secs(1))); + assert!(matches!(poll!(wait.as_mut()), Poll::Pending)); + + commit_test_key_ready(&context, &fetch_token, now_ms.saturating_add(1)).await; + + assert!(matches!(poll!(wait.as_mut()), Poll::Ready(Ok(())))); +} diff --git a/backend/hls/src/segment_fetcher/tests/retry.rs b/backend/hls/src/segment_fetcher/tests/retry.rs new file mode 100644 index 000000000..ea38ba54b --- /dev/null +++ b/backend/hls/src/segment_fetcher/tests/retry.rs @@ -0,0 +1,420 @@ +use super::{ + clear_scheduled_prefetch, encode_test_body, fetch_context, normal_manifest, spawn_segment_server, + spawn_sequence_response_server, temp_cache_files, HlsSegmentFetchWorkload, SegmentFetchContext, SegmentFetchPolicy, + TestContentEncoding, TestOriginResponse, TestSegmentServer, +}; +use crate::{HlsSegmentFile, HlsSessionKey, SegmentCacheStatus}; +use std::{ + collections::VecDeque, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::Mutex, +}; + +#[test] +fn missing_head_is_order_independent_and_preserves_unrelated_degradation() -> std::io::Result<()> { + for order in [[1_u64, 0], [0, 1]] { + let mut session = crate::HlsSession::new(HlsSessionKey::new(1, "startup"), b"secret", 0); + session.apply_origin_manifest(&normal_manifest( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:4,\n0.ts\n#EXTINF:4,\n1.ts\n#EXTINF:4,\n2.ts\n#EXTINF:4,\n3.ts\n#EXTINF:4,\n4.ts\n" + )).map_err(|error| std::io::Error::other(format!("{error:?}")))?; + let head = session.publishable_origin_head_proxy_seq.ok_or_else(|| std::io::Error::other("head"))?; + for seq in head.saturating_add(2)..=head.saturating_add(4) { + let entry = session.segments.get_mut(&seq).ok_or_else(|| std::io::Error::other("suffix"))?; + entry.status = SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 0 }; + } + session.origin_control.path_condition = crate::HlsOriginPathCondition::HardFetchFailure; + for offset in order { + let entry = + session.segments.get_mut(&(head + offset)).ok_or_else(|| std::io::Error::other("head entry"))?; + entry.status = SegmentCacheStatus::FailedPermanent { + failed_at_ms: 1, + status: Some(axum::http::StatusCode::NOT_FOUND), + }; + super::super::recompute_unpublished_live_head(&mut session); + } + assert_eq!(session.publishable_origin_head_proxy_seq, Some(head + 2)); + assert_eq!(session.origin_control.path_condition, crate::HlsOriginPathCondition::HardFetchFailure); + } + Ok(()) +} + +#[test] +fn segment_fetch_budget_accounts_for_serialized_key_and_media_retry_chains() { + let policy = SegmentFetchPolicy { + origin_segment_timeout_ms: 10_000, + effective_repair_postprocess_timeout_ms: 2_000, + retry_delays_ms: [0, 100, 250, 500, 750], + retry_jitter_max_ms: 100, + ..SegmentFetchPolicy::default() + }; + + assert_eq!(policy.demand_wait_timeout_for(HlsSegmentFetchWorkload::Clear), Duration::from_millis(63_100)); + assert_eq!( + policy.demand_wait_timeout_for(HlsSegmentFetchWorkload::EncryptedWithKey), + Duration::from_millis(125_200) + ); + assert_eq!(policy.demand_wait_timeout(), Duration::from_millis(125_200)); + assert_eq!(policy.origin_object_wait_timeout(), Duration::from_millis(63_100)); +} + +#[test] +fn hls_recovery_timing_expected_object_eta_is_not_the_retry_chain_timeout() { + let policy = SegmentFetchPolicy { + origin_segment_timeout_ms: 10_000, + effective_repair_postprocess_timeout_ms: 2_000, + retry_delays_ms: [0, 100, 250, 500, 750], + retry_jitter_max_ms: 100, + ..SegmentFetchPolicy::default() + }; + + assert_eq!(policy.recovery_object_eta_ms(), 13_000); + assert!(policy.recovery_object_eta_ms() < policy.workload_budget_ms(HlsSegmentFetchWorkload::Clear)); + assert!(policy.recovery_object_eta_ms() < policy.workload_budget_ms(HlsSegmentFetchWorkload::EncryptedWithKey)); + + let long_hard_timeout = SegmentFetchPolicy { + origin_segment_timeout_ms: 120_000, + effective_repair_postprocess_timeout_ms: 30_000, + ..policy + }; + assert_eq!(long_hard_timeout.recovery_object_eta_ms(), 13_000); + assert!( + long_hard_timeout.recovery_object_eta_ms() + < long_hard_timeout.workload_budget_ms(HlsSegmentFetchWorkload::Clear) + ); +} + +pub(in crate::segment_fetcher::tests) async fn wait_for_concurrent_capacity_deferral( + context: &SegmentFetchContext, + server: &TestSegmentServer, +) { + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let both_deferred = { + let session = context.session.read().await; + [2_u64, 3].into_iter().all(|proxy_seq| { + matches!( + session.segments.get(&proxy_seq).map(|segment| &segment.status), + Some(SegmentCacheStatus::CapacityDeferred { .. }) + ) + }) + }; + if both_deferred && server.requests.lock().await.len() >= 2 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("both capacity failures become revision-bound deferrals"); +} + +pub(in crate::segment_fetcher::tests) async fn assert_segment_request_counts( + server: &TestSegmentServer, + expected: usize, + failure_context: &str, +) { + let requests = server.requests.lock().await; + for proxy_seq in [2_u64, 3] { + assert_eq!( + requests.iter().filter(|request| request.starts_with(&format!("GET /{proxy_seq}.ts "))).count(), + expected, + "{failure_context}: segment {proxy_seq}" + ); + } +} + +pub(in crate::segment_fetcher::tests) async fn assert_concurrent_capacity_recovery_ready( + context: &SegmentFetchContext, +) { + let session = context.session.read().await; + for proxy_seq in [2_u64, 3, 4] { + assert!(matches!( + session.segments.get(&proxy_seq).map(|segment| &segment.status), + Some(SegmentCacheStatus::Ready { .. }) + )); + } + let rendered = session.last_rendered_manifest.as_ref().expect("recovered timeline renders"); + assert_eq!(rendered.first_proxy_seq, 2); + assert_eq!(rendered.last_proxy_seq, 4); + assert!(rendered.body.contains("000002.ts")); + assert!(rendered.body.contains("000003.ts")); +} + +#[tokio::test] +async fn concurrent_capacity_deferred_segments_retry_once_after_real_cache_release() { + let response = TestOriginResponse { status: 200, headers: Vec::new(), body: b"12345".to_vec() }; + let server = spawn_sequence_response_server(vec![response; 4]).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..Default::default() }; + let (worker, context, _) = fetch_context(&server, &temp_dir, &policy).await; + { + let mut session = context.session.write().await; + session + .apply_origin_manifest(&normal_manifest(&format!( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\n{0}/1.ts\n#EXTINF:4.0,\n{0}/2.ts\n#EXTINF:4.0,\n{0}/3.ts\n#EXTINF:4.0,\n{0}/4.ts\n", + server.base_url + ))) + .expect("four-segment manifest maps"); + } + clear_scheduled_prefetch(&context, &policy).await; + let (released_key, tail_key) = { + let session = context.session.read().await; + ( + session.segments.get(&1).expect("released segment").cache_key.clone(), + session.segments.get(&4).expect("ready tail segment").cache_key.clone(), + ) + }; + context + .segment_cache + .write_bytes_and_commit(&released_key, b"12345678901") + .await + .expect("released fixture commits"); + context.segment_cache.write_bytes_and_commit(&tail_key, b"t").await.expect("tail fixture commits"); + { + let mut session = context.session.write().await; + session.segments.get_mut(&1).expect("released segment").status = + SegmentCacheStatus::Ready { content_length: 11, ready_at_ms: 10 }; + session.segments.get_mut(&4).expect("ready tail segment").status = + SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 10 }; + } + context.segment_cache.update_cache_limits(100, 12); + + let first_worker = Arc::clone(&worker); + let first_context = context.clone(); + let first = tokio::spawn(async move { + first_worker + .demand_fetch_and_wait(first_context, &HlsSegmentFile { proxy_seq: 2, extension: "ts".to_string() }, 20) + .await + }); + let second_worker = Arc::clone(&worker); + let second_context = context.clone(); + let second = tokio::spawn(async move { + second_worker + .demand_fetch_and_wait(second_context, &HlsSegmentFile { proxy_seq: 3, extension: "ts".to_string() }, 20) + .await + }); + + wait_for_concurrent_capacity_deferral(&context, &server).await; + for _ in 0..32 { + tokio::task::yield_now().await; + } + assert_segment_request_counts(&server, 1, "initial capacity deferral").await; + + for _ in 0..3 { + context.segment_cache.notify_capacity_protection_changed(); + for _ in 0..32 { + tokio::task::yield_now().await; + } + } + assert_segment_request_counts(&server, 1, "insufficient protection revision").await; + { + let session = context.session.read().await; + assert!([2_u64, 3].into_iter().all(|proxy_seq| { + session.segments.get(&proxy_seq).is_some_and(|segment| segment.status.awaits_capacity_recovery()) + })); + } + + { + let mut session = context.session.write().await; + session.segments.get_mut(&1).expect("completed segment").status = SegmentCacheStatus::Expired; + session.publishable_origin_head_proxy_seq = Some(2); + session.advance_media_readiness_generation(); + } + context.segment_cache.delete(&released_key).await.expect("completed cache object is released"); + + let (first, second) = tokio::time::timeout(Duration::from_secs(10), async { tokio::join!(first, second) }) + .await + .expect("both deferred fetches resume after the accounting revision"); + assert_eq!(first.expect("first demand task joins"), super::super::SegmentDemandFetchOutcome::Ready); + assert_eq!(second.expect("second demand task joins"), super::super::SegmentDemandFetchOutcome::Ready); + assert_segment_request_counts(&server, 2, "real capacity release").await; + assert_concurrent_capacity_recovery_ready(&context).await; +} + +#[tokio::test] +async fn failed_retryable_segment_has_a_bounded_demand_requeue_contract() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..Default::default() }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + context.session.write().await.segments.get_mut(&segment_file.proxy_seq).expect("segment").status = + SegmentCacheStatus::FailedRetryable { failed_at_ms: 20, retry_after_ms: 1_000 }; + + let early = worker.demand_fetch_and_wait(context.clone(), &segment_file, 1_019).await; + assert_eq!(early, super::super::SegmentDemandFetchOutcome::TimedOut); + assert!(server.requests.lock().await.is_empty()); + + let retried = worker.demand_fetch_and_wait(context.clone(), &segment_file, 1_020).await; + assert_eq!(retried, super::super::SegmentDemandFetchOutcome::Ready); + assert_eq!(server.requests.lock().await.len(), 1); +} + +pub(in crate::segment_fetcher::tests) async fn spawn_sequence_status_server(statuses: Vec) -> TestSegmentServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let requests = Arc::new(Mutex::new(Vec::new())); + let statuses = Arc::new(Mutex::new(VecDeque::from(statuses))); + let task_requests = Arc::clone(&requests); + let task_statuses = Arc::clone(&statuses); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + let statuses = Arc::clone(&task_statuses); + tokio::spawn(async move { + let mut buf = vec![0_u8; 2048]; + let Ok(read) = socket.read(&mut buf).await else { + return; + }; + if read == 0 { + return; + } + let request = String::from_utf8_lossy(&buf[..read]).to_string(); + requests.lock().await.push(request.clone()); + let status = statuses.lock().await.pop_front().unwrap_or(200); + let reason = if status == 200 { "OK" } else { "Error" }; + let body = if status == 200 { "segment-body" } else { "" }; + let response = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + + TestSegmentServer { base_url: format!("http://{addr}"), requests, task } +} + +pub(in crate::segment_fetcher::tests) async fn spawn_redirect_retry_server() -> TestSegmentServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let task = tokio::spawn(async move { + let request_count = Arc::new(AtomicUsize::new(0)); + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + let request_count = Arc::clone(&request_count); + tokio::spawn(async move { + let mut buf = vec![0_u8; 2048]; + let Ok(read) = socket.read(&mut buf).await else { + return; + }; + if read == 0 { + return; + } + let request = String::from_utf8_lossy(&buf[..read]).to_string(); + let path = request.lines().next().and_then(|line| line.split_whitespace().nth(1)).unwrap_or("/"); + requests.lock().await.push(path.to_string()); + let count = request_count.fetch_add(1, Ordering::SeqCst); + let response = if path == "/1.ts" && count == 0 { + "HTTP/1.1 302 Found\r\nLocation: /redirected.ts\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + .to_string() + } else if path == "/redirected.ts" { + "HTTP/1.1 500 Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string() + } else { + "HTTP/1.1 200 OK\r\nContent-Length: 12\r\nConnection: close\r\n\r\nsegment-body".to_string() + }; + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + + TestSegmentServer { base_url: format!("http://{addr}"), requests, task } +} + +#[tokio::test] +async fn segment_decoded_object_limit_is_authoritative_and_non_retryable() { + let identity_bytes = vec![b'x'; 512]; + let encoded = encode_test_body(&identity_bytes, TestContentEncoding::Gzip).await; + assert!(encoded.len() < 64, "fixture must be smaller on the wire than the decoded limit"); + let server = spawn_sequence_response_server(vec![TestOriginResponse::encoded("gzip", encoded)]).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + context.segment_cache.update_cache_limits(64, 64); + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Unavailable); + assert_eq!(server.requests.lock().await.len(), 1); + let cache_key = context.session.read().await.segments.get(&1).expect("segment").cache_key.clone(); + assert!(context.segment_cache.metadata(&cache_key).await.expect("metadata reads").is_none()); + assert!(!context.segment_cache.has_active_temp_files()); + assert!(temp_cache_files(temp_dir.path()).is_empty()); +} + +#[tokio::test] +async fn retryable_407_retries_segment_fetch_until_success() { + let server = spawn_sequence_status_server(vec![407, 200]).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Ready); + assert_eq!(server.requests.lock().await.len(), 2); +} + +#[tokio::test] +async fn permanent_404_does_not_retry_segment_fetch() { + let server = spawn_sequence_status_server(vec![404, 200]).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Unavailable); + assert_eq!(server.requests.lock().await.len(), 1); +} + +#[tokio::test] +async fn segment_retry_starts_again_at_fetch_ref_after_redirect_failure() { + let server = spawn_redirect_retry_server().await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { + retry_delays_ms: [0, 0, 0, 0, 0], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + + let outcome = worker.demand_fetch_and_wait(context, &segment_file, 20).await; + + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Ready); + assert_eq!(server.requests.lock().await.as_slice(), ["/1.ts", "/redirected.ts", "/1.ts"]); +} diff --git a/backend/hls/src/segment_fetcher/tests/startup.rs b/backend/hls/src/segment_fetcher/tests/startup.rs new file mode 100644 index 000000000..4b6a7c12e --- /dev/null +++ b/backend/hls/src/segment_fetcher/tests/startup.rs @@ -0,0 +1,148 @@ +use super::{ + clear_scheduled_prefetch, cold_startup_round, fetch_context, install_startup, prefix_drop_rounds, + spawn_segment_server, spawn_sequence_response_server, SegmentFetchPolicy, SegmentFetchPriority, +}; +use crate::{HlsOriginResourceFetchError, SegmentCacheStatus}; +use std::{sync::Arc, time::Duration}; + +#[tokio::test] +async fn capacity_deferred_segment_requeues_only_after_capacity_revision_changes() { + let server = spawn_segment_server(0).await; + let temp_dir = tempfile::tempdir().expect("temp dir"); + let policy = SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..Default::default() }; + let (worker, context, segment_file) = fetch_context(&server, &temp_dir, &policy).await; + clear_scheduled_prefetch(&context, &policy).await; + { + let mut session = context.session.write().await; + for proxy_seq in [2_u64, 3] { + session.segments.get_mut(&proxy_seq).expect("ready tail segment").status = + SegmentCacheStatus::Ready { content_length: 1, ready_at_ms: 19 }; + } + assert!(session.queue_segment_fetch_candidate(1, SegmentFetchPriority::Demand, 20)); + } + let snapshot = worker.next_fetch_snapshot(&context, 20, &policy).await.expect("fetch snapshot"); + let notifier = { + let mut session = context.session.write().await; + session.segment_fetch_notifiers.entry(snapshot.proxy_seq).or_default().clone() + }; + let completion = { + let revision = context.segment_cache.capacity_revision(); + let error = HlsOriginResourceFetchError::LocalCacheCapacity { + required_session_bytes: 1, + required_global_bytes: 0, + projected_write_bytes: 1, + revision, + }; + let mut session = context.session.write().await; + worker.apply_segment_fetch_result(&mut session, &snapshot, &policy, Err(error), 21) + }; + let retry = completion.scheduled_retry.expect("capacity deferral owns a revision retry"); + assert!(completion.notifier.is_none(), "deferred work retains its existing demand notifier"); + worker.schedule_segment_retry(context.clone(), snapshot, retry); + let first_demand = worker.demand_fetch_and_wait(context.clone(), &segment_file, 22); + let repeated_demand = worker.demand_fetch_and_wait(context.clone(), &segment_file, 23); + tokio::pin!(first_demand, repeated_demand); + assert!(matches!(futures::poll!(first_demand.as_mut()), std::task::Poll::Pending)); + assert!(matches!(futures::poll!(repeated_demand.as_mut()), std::task::Poll::Pending)); + assert!(server.requests.lock().await.is_empty(), "capacity wait does not poll the origin"); + + let completed = notifier.notified(); + tokio::pin!(completed); + completed.as_mut().enable(); + // The revision may change before the spawned waiter first polls. Its + // token comparison must make that race lossless. + context.segment_cache.notify_capacity_protection_changed(); + let ((), first_demand_outcome, repeated_demand_outcome) = tokio::time::timeout(Duration::from_secs(10), async { + tokio::join!(completed.as_mut(), first_demand.as_mut(), repeated_demand.as_mut()) + }) + .await + .expect("revision wake completes one requeued fetch"); + + assert_eq!(first_demand_outcome, super::super::SegmentDemandFetchOutcome::Ready); + assert_eq!(repeated_demand_outcome, super::super::SegmentDemandFetchOutcome::Ready); + assert_eq!(server.requests.lock().await.len(), 1); + let session = context.session.read().await; + assert!(matches!(session.segments.get(&1).map(|segment| &segment.status), Some(SegmentCacheStatus::Ready { .. }))); + let rendered = session.last_rendered_manifest.as_ref().expect("recovered timeline renders a manifest"); + assert_eq!(rendered.first_proxy_seq, 1); + assert_eq!(rendered.last_proxy_seq, 3); + assert!(rendered.body.contains("000001.ts")); +} + +#[tokio::test] +async fn known_oversized_startup_body_falls_back_before_replay_or_publication() -> std::io::Result<()> { + let server = spawn_sequence_response_server(Vec::new()).await; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy::default(); + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, shared::model::HlsStartupMode::Progressive).await; + let snapshot = worker + .next_fetch_snapshot(&context, 10, &policy) + .await + .ok_or_else(|| std::io::Error::other("fetch snapshot"))?; + let budget = { + let mut session = context.session.write().await; + let startup = session.startup.as_mut().ok_or_else(|| std::io::Error::other("startup policy"))?; + startup.config.max_progressive_bytes_per_segment = shared::model::Bytes::new(8); + Arc::clone(&startup.budget) + }; + let prepared = super::super::prepare_startup_fill( + &context, + &snapshot, + Some(12), + tokio::time::Instant::now() + Duration::from_secs(1), + ) + .await + .map_err(|error| std::io::Error::other(format!("{error:?}")))?; + assert!(!prepared.raw); + assert_eq!(prepared.revision.revision().key.kind, crate::SegmentRevisionKind::Processed); + assert!(!prepared.revision.revision().is_publishable()); + assert_eq!(context.session.read().await.startup_mode(), shared::model::HlsStartupMode::FirstReady); + assert_eq!(budget.usage(), (0, 0)); + assert!(server.requests.lock().await.is_empty()); + Ok(()) +} + +#[tokio::test] +async fn cold_startup_publication_and_first_data_precede_successor_eof() -> std::io::Result<()> { + for mode in [shared::model::HlsStartupMode::FirstReady, shared::model::HlsStartupMode::Progressive] { + cold_startup_round(mode, None).await?; + } + Ok(()) +} + +#[tokio::test] +async fn first_ready_and_progressive_keep_one_origin_fill_and_exact_bytes() -> std::io::Result<()> { + for mode in [shared::model::HlsStartupMode::FirstReady, shared::model::HlsStartupMode::Progressive] { + let server = spawn_segment_server(0).await; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy::default(); + let (worker, context, segment) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, mode).await; + clear_scheduled_prefetch(&context, &policy).await; + let outcome = worker.demand_fetch_and_wait(context.clone(), &segment, 10).await; + assert_eq!(outcome, super::super::SegmentDemandFetchOutcome::Ready); + let revision = { + let session = context.session.read().await; + session + .startup + .as_ref() + .and_then(|startup| startup.revisions.get(&segment.proxy_seq)) + .cloned() + .ok_or_else(|| std::io::Error::other("completed revision missing"))? + }; + let metadata = revision.revision().wait_complete(tokio::time::Instant::now()).await?; + assert_eq!(tokio::fs::read(metadata.path).await?, b"body:/1.ts"); + assert_eq!(server.requests.lock().await.iter().filter(|request| request.starts_with("GET /1.ts ")).count(), 1); + if mode == shared::model::HlsStartupMode::Progressive { + assert_eq!(revision.revision().key.kind, crate::SegmentRevisionKind::Raw); + assert_eq!(context.session.read().await.startup.as_ref().map(|s| s.budget.usage()), Some((0, 0))); + } + } + Ok(()) +} + +#[tokio::test] +async fn progressive_exposes_prefix_before_origin_eof_and_drop_is_terminal() -> std::io::Result<()> { + prefix_drop_rounds().await +} diff --git a/backend/hls/src/segment_fetcher/tests/support.rs b/backend/hls/src/segment_fetcher/tests/support.rs new file mode 100644 index 000000000..42a32a45d --- /dev/null +++ b/backend/hls/src/segment_fetcher/tests/support.rs @@ -0,0 +1,713 @@ +use super::{SegmentFetchContext, SegmentFetchPolicy}; +use crate::{ + HlsAccessLease, HlsAccessLeaseId, HlsPlaybackFamilyKey, HlsSegmentCache, HlsSegmentFile, HlsSegmentRepairManager, + HlsSegmentWorkerPool, HlsSessionKey, HlsSessionStore, ProxySessionId, SegmentCacheStatus, + TransientObjectFetchDecision, TransientObjectFetchToken, TransientResourceId, TransientResourceKind, + TransientResourceRef, +}; +use async_compression::tokio::bufread::{BrotliEncoder, DeflateEncoder, GzipEncoder, ZstdEncoder}; +use axum::http::HeaderMap; +use shared::model::HlsSegmentRepairMode; +use std::{ + collections::VecDeque, + io::Cursor, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt, BufReader}, + net::TcpListener, + sync::{Mutex, Notify}, +}; +use tuliprox_core::{model::HlsSegmentRepairConfig, utils::current_time_millis}; +use tuliprox_parser::hls::origin_manifest::{parse_origin_media_manifest, OriginManifestParseOutcome}; + +pub(in crate::segment_fetcher::tests) const BASE_URL: &str = "http://origin.example.com/live/final/index.m3u8"; + +pub(in crate::segment_fetcher::tests) fn normal_manifest( + body: &str, +) -> tuliprox_parser::hls::origin_manifest::ParsedOriginManifest { + match parse_origin_media_manifest(body, BASE_URL) { + OriginManifestParseOutcome::Normal(manifest) => manifest, + OriginManifestParseOutcome::TransientPassthrough { reason } => { + panic!("expected normal manifest: {reason:?}") + } + } +} + +pub(in crate::segment_fetcher::tests) fn test_segment_repair_manager() -> Arc { + Arc::new(HlsSegmentRepairManager::new(HlsSegmentRepairConfig { + max_level: HlsSegmentRepairMode::Off, + apply_to_first_segments: 1, + max_parallel_repairs: 1, + ..Default::default() + })) +} + +pub(in crate::segment_fetcher::tests) struct TestSegmentServer { + pub(in crate::segment_fetcher::tests) base_url: String, + pub(in crate::segment_fetcher::tests) requests: Arc>>, + pub(in crate::segment_fetcher::tests) task: tokio::task::JoinHandle<()>, +} + +impl Drop for TestSegmentServer { + fn drop(&mut self) { self.task.abort(); } +} + +#[derive(Clone)] +pub(in crate::segment_fetcher::tests) struct TestOriginResponse { + pub(in crate::segment_fetcher::tests) status: u16, + pub(in crate::segment_fetcher::tests) headers: Vec<(String, String)>, + pub(in crate::segment_fetcher::tests) body: Vec, +} + +impl TestOriginResponse { + pub(in crate::segment_fetcher::tests) fn encoded(content_encoding: &str, body: Vec) -> Self { + Self { status: 200, headers: vec![("Content-Encoding".to_string(), content_encoding.to_string())], body } + } +} + +#[derive(Debug, Clone, Copy)] +pub(in crate::segment_fetcher::tests) enum TestContentEncoding { + Gzip, + RawDeflate, + Brotli, + Zstd, +} + +impl TestContentEncoding { + pub(in crate::segment_fetcher::tests) const fn header_value(self) -> &'static str { + match self { + Self::Gzip => "gzip", + Self::RawDeflate => "deflate", + Self::Brotli => "br", + Self::Zstd => "zstd", + } + } +} + +pub(in crate::segment_fetcher::tests) async fn encode_test_body(body: &[u8], encoding: TestContentEncoding) -> Vec { + let reader = BufReader::new(Cursor::new(body.to_vec())); + let mut encoded = Vec::new(); + match encoding { + TestContentEncoding::Gzip => GzipEncoder::new(reader).read_to_end(&mut encoded).await, + TestContentEncoding::RawDeflate => DeflateEncoder::new(reader).read_to_end(&mut encoded).await, + TestContentEncoding::Brotli => BrotliEncoder::new(reader).read_to_end(&mut encoded).await, + TestContentEncoding::Zstd => ZstdEncoder::new(reader).read_to_end(&mut encoded).await, + } + .expect("test body encodes"); + encoded +} + +pub(in crate::segment_fetcher::tests) async fn spawn_sequence_response_server( + responses: Vec, +) -> TestSegmentServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let requests = Arc::new(Mutex::new(Vec::new())); + let responses = Arc::new(Mutex::new(VecDeque::from(responses))); + let task_requests = Arc::clone(&requests); + let task_responses = Arc::clone(&responses); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + let responses = Arc::clone(&task_responses); + tokio::spawn(async move { + use std::fmt::Write as _; + + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 2048]; + let Ok(read) = socket.read(&mut chunk).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); + let response = responses.lock().await.pop_front().unwrap_or(TestOriginResponse { + status: 500, + headers: Vec::new(), + body: Vec::new(), + }); + let reason = if response.status == 200 { "OK" } else { "Status" }; + let mut head = + format!("HTTP/1.1 {} {reason}\r\nContent-Length: {}\r\n", response.status, response.body.len()); + for (name, value) in response.headers { + let _ = write!(head, "{name}: {value}\r\n"); + } + head.push_str("Connection: close\r\n\r\n"); + let _ = socket.write_all(head.as_bytes()).await; + let _ = socket.write_all(&response.body).await; + }); + } + }); + + TestSegmentServer { base_url: format!("http://{addr}"), requests, task } +} + +pub(in crate::segment_fetcher::tests) fn temp_cache_files(root: &Path) -> Vec { + fn visit(path: &Path, found: &mut Vec) { + let Ok(entries) = std::fs::read_dir(path) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + visit(&path, found); + } else if path.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.contains(".tmp.")) { + found.push(path); + } + } + } + + let mut found = Vec::new(); + visit(root, &mut found); + found +} + +pub(in crate::segment_fetcher::tests) async fn spawn_segment_server(delay_ms: u64) -> TestSegmentServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("local addr"); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + tokio::spawn(async move { + let mut buf = vec![0_u8; 2048]; + let mut used = 0_usize; + loop { + let Ok(read) = socket.read(&mut buf[used..]).await else { + return; + }; + if read == 0 { + return; + } + used += read; + if used >= 4 && buf[..used].windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + if used == buf.len() { + return; + } + } + let request = String::from_utf8_lossy(&buf[..used]).to_string(); + requests.lock().await.push(request.clone()); + if delay_ms > 0 { + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + } + let path = request.lines().next().and_then(|line| line.split_whitespace().nth(1)).unwrap_or("/"); + let body = format!("body:{path}"); + let response = + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body); + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + + TestSegmentServer { base_url: format!("http://{addr}"), requests, task } +} + +pub(in crate::segment_fetcher::tests) async fn grant_usable_worker_access_lease( + worker: &HlsSegmentWorkerPool, + proxy_session_id: &ProxySessionId, +) { + grant_usable_worker_access_lease_at(worker, proxy_session_id, 10).await; +} + +pub(in crate::segment_fetcher::tests) async fn grant_usable_worker_access_lease_at( + worker: &HlsSegmentWorkerPool, + proxy_session_id: &ProxySessionId, + now_ms: u64, +) { + worker.access_leases().write().await.prepare_access_lease(HlsAccessLease::pending( + HlsAccessLeaseId("worker-lease".to_string()), + HlsPlaybackFamilyKey::new("alice", "client-a"), + proxy_session_id.clone(), + "alice".to_string(), + "session-a".to_string(), + 1, + "12345".to_string(), + 12345, + now_ms, + 15_000, + )); +} + +pub(in crate::segment_fetcher::tests) async fn fetch_context_with_access_lease( + server: &TestSegmentServer, + temp_dir: &tempfile::TempDir, + policy: &SegmentFetchPolicy, + grant_access_lease: bool, +) -> (Arc, SegmentFetchContext, HlsSegmentFile) { + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; + { + let mut session = session.write().await; + session.configure_segment_prefetch_queue(policy.max_prefetch_queue_depth); + session.proxy_next_seq = Some(1); + session + .apply_origin_manifest(&normal_manifest(&format!( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:4.0,\n{}/1.ts\n#EXTINF:4.0,\n{}/2.ts\n#EXTINF:4.0,\n{}/3.ts\n", + server.base_url, server.base_url, server.base_url + ))) + .expect("manifest maps"); + session.queue_manifest_prefetch_candidates(10); + } + let worker = Arc::new(HlsSegmentWorkerPool::new(policy.clone())); + if grant_access_lease { + let proxy_session_id = session.read().await.proxy_session_id.clone(); + grant_usable_worker_access_lease(&worker, &proxy_session_id).await; + } + let context = SegmentFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())), + segment_repair: test_segment_repair_manager(), + repair_access_lease_id: None, + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("client builds"), + use_manual_redirects: true, + origin_io: None, + }; + (worker, context, HlsSegmentFile { proxy_seq: 1, extension: "ts".to_string() }) +} + +pub(in crate::segment_fetcher::tests) async fn fetch_context( + server: &TestSegmentServer, + temp_dir: &tempfile::TempDir, + policy: &SegmentFetchPolicy, +) -> (Arc, SegmentFetchContext, HlsSegmentFile) { + fetch_context_with_access_lease(server, temp_dir, policy, true).await +} + +pub(in crate::segment_fetcher::tests) async fn encrypted_fetch_context( + server: &TestSegmentServer, + temp_dir: &tempfile::TempDir, + policy: &SegmentFetchPolicy, +) -> (Arc, SegmentFetchContext, HlsSegmentFile) { + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "12345"), b"secret", 0).await; + let key_uri = format!("{}/key.key", server.base_url); + let key_resource = TransientResourceRef::new( + TransientResourceKind::Key, + &key_uri, + b"secret", + 0, + u64::MAX, + Some("key".to_string()), + ); + let mut manifest = normal_manifest(&format!( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:1\n#EXT-X-KEY:METHOD=AES-128,URI=\"{key_uri}\"\n\ + #EXTINF:4.0,\n{}/1.ts\n#EXTINF:4.0,\n{}/2.ts\n#EXTINF:4.0,\n{}/3.ts\n", + server.base_url, server.base_url, server.base_url + )); + for encryption in manifest.segments.iter_mut().filter_map(|segment| segment.encryption.as_mut()) { + encryption.proxy_resource_id = Some(key_resource.id.0.clone()); + encryption.proxy_resource_extension = Some("key".to_string()); + } + { + let mut session = session.write().await; + session.configure_segment_prefetch_queue(policy.max_prefetch_queue_depth); + session.proxy_next_seq = Some(1); + session.transient.upsert_resources([key_resource]); + session.apply_origin_manifest(&manifest).expect("encrypted manifest maps"); + } + let worker = Arc::new(HlsSegmentWorkerPool::new(policy.clone())); + let context = SegmentFetchContext { + session: Arc::clone(&session), + segment_cache: Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())), + segment_repair: test_segment_repair_manager(), + repair_access_lease_id: None, + headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::new(), + use_manual_redirects: false, + origin_io: None, + }; + (worker, context, HlsSegmentFile { proxy_seq: 1, extension: "ts".to_string() }) +} + +pub(in crate::segment_fetcher::tests) async fn shared_key_fetch_and_wait( + context: &SegmentFetchContext, + now_ms: u64, +) -> (TransientObjectFetchToken, Arc, TransientResourceId, String) { + let mut session = context.session.write().await; + let resource = session + .transient + .resources + .values() + .find(|resource| resource.kind == TransientResourceKind::Key) + .cloned() + .expect("encrypted fixture key resource"); + let extension = resource.file_ext_hint.clone().expect("key resource extension"); + let proxy_session_id = session.proxy_session_id.clone(); + let cache_duration_ms = session.transient.resource_ttl_ms; + let fetch_token = + match session.transient.begin_object_fetch(&proxy_session_id, &resource, &extension, now_ms, cache_duration_ms) + { + TransientObjectFetchDecision::Fetch(token) => token, + TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { + panic!("first shared-key decision must own the fetch") + } + }; + let notifier = + match session.transient.begin_object_fetch(&proxy_session_id, &resource, &extension, now_ms, cache_duration_ms) + { + TransientObjectFetchDecision::Wait(notifier) => notifier, + TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Fetch(_) => { + panic!("second shared-key decision must wait for the existing fetch") + } + }; + (*fetch_token, notifier, resource.id, extension) +} + +pub(in crate::segment_fetcher::tests) async fn commit_test_key_ready( + context: &SegmentFetchContext, + token: &TransientObjectFetchToken, + now_ms: u64, +) { + assert!(context.session.write().await.commit_transient_object_ready_if_current( + TransientResourceKind::Key, + token, + "application/octet-stream".to_string(), + 16, + now_ms, + u64::MAX, + )); +} + +pub(in crate::segment_fetcher::tests) async fn clear_scheduled_prefetch( + context: &SegmentFetchContext, + policy: &SegmentFetchPolicy, +) { + let mut session = context.session.write().await; + session.segment_prefetch_queue = crate::SegmentPrefetchQueue::new(policy.max_prefetch_queue_depth); + for segment in session.segments.values_mut() { + if !matches!(segment.status, SegmentCacheStatus::Ready { .. }) { + segment.status = SegmentCacheStatus::Discovered; + } + } +} + +pub(in crate::segment_fetcher::tests) async fn committed_segment( + context: &SegmentFetchContext, + proxy_seq: u64, +) -> (crate::SegmentCacheKey, Vec) { + let cache_key = context.session.read().await.segments.get(&proxy_seq).expect("segment").cache_key.clone(); + let metadata = + context.segment_cache.metadata(&cache_key).await.expect("cache metadata reads").expect("cache object exists"); + let bytes = tokio::fs::read(metadata.path).await.expect("cache object reads"); + (cache_key, bytes) +} + +pub(in crate::segment_fetcher::tests) async fn install_startup( + context: &SegmentFetchContext, + worker: &Arc, + mode: shared::model::HlsStartupMode, +) { + let config = tuliprox_core::model::HlsStartupConfig { mode, ..Default::default() }; + let budget = crate::ProgressiveBudgetManager::new(config.clone()); + context.session.write().await.startup = Some(crate::HlsSessionStartup { + config, + first_data_timeout_ms: 10_000, + worker: Arc::downgrade(worker), + access_leases: Arc::clone(worker.access_leases()), + revisions: std::collections::BTreeMap::new(), + policy_fixed: false, + store: Arc::new(crate::SegmentRevisionStore::default()), + budget, + }); +} + +pub(in crate::segment_fetcher::tests) async fn prefix_origin( + auto_complete_head: bool, +) -> std::io::Result<(TestSegmentServer, tokio::sync::watch::Sender)> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base_url = format!("http://{}", listener.local_addr()?); + let requests = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&requests); + let (release, released) = tokio::sync::watch::channel(false); + let task = tokio::spawn(async move { + let mut children = tokio::task::JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => { + let Ok((socket, _)) = accepted else { break; }; + let recorded = Arc::clone(&recorded); + let mut released = released.clone(); + children.spawn(async move { + let mut reader = BufReader::new(socket); + let mut request = String::new(); + tokio::io::AsyncBufReadExt::read_line(&mut reader, &mut request).await?; + let head = request.starts_with("GET /1.ts "); + recorded.lock().await.push(request); + loop { + let mut line = String::new(); + tokio::io::AsyncBufReadExt::read_line(&mut reader, &mut line).await?; + if line == "\r\n" || line.is_empty() { break; } + } + let mut socket = reader.into_inner(); + socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 12\r\nConnection: close\r\n\r\nsegment-").await?; + if !(auto_complete_head && head) { + while !*released.borrow_and_update() { + released.changed().await.map_err(std::io::Error::other)?; + } + } + socket.write_all(b"body").await?; + socket.shutdown().await?; + Ok::<_, std::io::Error>(()) + }); + } + _ = children.join_next(), if !children.is_empty() => {} + } + } + }); + Ok((TestSegmentServer { base_url, requests, task }, release)) +} + +pub(in crate::segment_fetcher::tests) type StartupSoakResources<'a> = ( + &'a tempfile::TempDir, + &'a Arc, + &'a Arc, + &'a Arc, +); + +pub(in crate::segment_fetcher::tests) async fn cold_startup_round( + mode: shared::model::HlsStartupMode, + shared: Option>, +) -> std::io::Result<()> { + use futures::StreamExt; + let started = tokio::time::Instant::now(); + let wall_started_at_ms = current_time_millis(); + let (server, release) = prefix_origin(mode == shared::model::HlsStartupMode::FirstReady).await?; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy { max_session_segment_fetches: 3, ..Default::default() }; + let fixture_dir = shared.map_or(&directory, |state| state.0); + let (worker, mut context, _) = fetch_context_with_access_lease(&server, fixture_dir, &policy, false).await; + install_startup(&context, &worker, mode).await; + if let Some((_, store, budget, cache)) = shared { + context.segment_cache = Arc::clone(cache); + let mut session = context.session.write().await; + if let Some(startup) = &mut session.startup { + startup.store = Arc::clone(store); + startup.budget = Arc::clone(budget); + } + } + let session_id = context.session.read().await.proxy_session_id.clone(); + grant_usable_worker_access_lease_at(&worker, &session_id, current_time_millis()).await; + worker.wake_scheduler(context.clone(), current_time_millis()).await; + let publication = tokio::time::timeout(Duration::from_secs(3), wait_for_cold_publication(&context, mode)).await; + let publication = match publication { + Ok(publication) => publication?, + Err(_) => { + return Err( + startup_timeout_diagnostic(&context, &server, "cold publication", started, wall_started_at_ms).await + ) + } + }; + let timeline = crate::media_reserve::HlsReadyTimelineSnapshot { units: Arc::from([]) }; + let admission = crate::media_reserve::evaluate_startup_admission(crate::media_reserve::HlsStartupAdmissionInput { + manifest: &publication, + ready_timeline: &timeline, + origin_state: crate::media_reserve::HlsStartupAdmissionOriginState::Healthy, + recovery_trigger_budget: crate::recovery_timing::HlsRecoveryTriggerBudgetMs::from_millis(4000), + }); + assert_eq!(admission.decision, crate::media_reserve::HlsStartupAdmissionDecision::Admit); + let evidence = publication.startup_revisions.as_ref().ok_or_else(|| std::io::Error::other("evidence"))?; + let head = evidence.revisions.get(&1).ok_or_else(|| std::io::Error::other("head"))?; + let successor = evidence.revisions.get(&2).ok_or_else(|| std::io::Error::other("successor"))?; + assert!(matches!(*successor.revision().subscribe().borrow(), crate::SegmentRevisionState::Pending)); + let mut stalled = crate::progressive_startup::revision_body(head.clone(), Duration::from_millis(20), 0, None); + let aborted = crate::progressive_startup::revision_body(head.clone(), Duration::from_secs(3), 0, None); + drop(aborted); + let mut body = crate::progressive_startup::revision_body(head.clone(), Duration::from_secs(3), 0, None); + let first = body.next().await.transpose()?.ok_or_else(|| std::io::Error::other("first data"))?; + assert!(first.starts_with(b"segment-")); + assert_eq!(server.requests.lock().await.len(), 3); + assert_eq!( + successor + .revision() + .wait_complete(tokio::time::Instant::now() + Duration::from_millis(25)) + .await + .err() + .map(|error| error.kind()), + Some(std::io::ErrorKind::TimedOut) + ); + assert!(stalled.next().await.is_some_and(|chunk| chunk.is_err())); + release.send(true).map_err(std::io::Error::other)?; + let mut received = first.to_vec(); + while let Some(chunk) = body.next().await { + received.extend_from_slice(&chunk?); + } + assert_eq!(received, b"segment-body"); + successor.revision().wait_complete(tokio::time::Instant::now() + Duration::from_secs(3)).await?; + assert_eq!(server.requests.lock().await.len(), 3); + if tokio::time::timeout(Duration::from_secs(3), async { + while context.session.read().await.active_segment_fetches != 0 { + tokio::task::yield_now().await; + } + }) + .await + .is_err() + { + return Err( + startup_timeout_diagnostic(&context, &server, "producer completion", started, wall_started_at_ms).await + ); + } + Ok(()) +} + +pub(in crate::segment_fetcher::tests) async fn wait_for_cold_publication( + context: &SegmentFetchContext, + mode: shared::model::HlsStartupMode, +) -> std::io::Result { + loop { + let mut session = context.session.write().await; + if let Ok(rendered) = session.render_and_store_manifest(current_time_millis()) { + let head = session.startup.as_ref().and_then(|startup| startup.revisions.get(&1)).cloned(); + if mode == shared::model::HlsStartupMode::Progressive + || head.as_ref().is_some_and(|head| { + matches!(*head.revision().subscribe().borrow(), crate::SegmentRevisionState::Complete(_)) + }) + { + let snapshot = crate::manifest_snapshot::derive_hls_lease_manifest_snapshot( + &crate::manifest_snapshot::HlsLeaseManifestSnapshotInput::NormalCacheTimeline { + session: &session, + committed_body: &rendered.body, + materialized_body: &rendered.body, + stripped_tail_segments: 0, + }, + current_time_millis(), + ) + .map_err(|error| std::io::Error::other(format!("{error:?}")))? + .ok_or_else(|| std::io::Error::other("publication snapshot"))?; + return Ok::<_, std::io::Error>(snapshot); + } + } + drop(session); + tokio::task::yield_now().await; + } +} + +pub(in crate::segment_fetcher::tests) async fn startup_timeout_diagnostic( + context: &SegmentFetchContext, + server: &TestSegmentServer, + phase: &str, + started: tokio::time::Instant, + wall_started_at_ms: u64, +) -> std::io::Error { + let origin_requests = server.requests.lock().await.len(); + let session = context.session.read().await; + let statuses = session.segments.iter().map(|(seq, entry)| (*seq, entry.status.clone())).collect::>(); + let revisions = session.startup.as_ref().map(|startup| { + startup + .revisions + .iter() + .map(|(seq, owner)| (*seq, owner.revision().subscribe().borrow().clone())) + .collect::>() + }); + std::io::Error::new(std::io::ErrorKind::TimedOut, format!( + "{phase}: monotonic_ms={} wall_ms={} active={} origin_requests={} statuses={statuses:?} revisions={revisions:?}", + started.elapsed().as_millis(), current_time_millis().saturating_sub(wall_started_at_ms), + session.active_segment_fetches, origin_requests + )) +} + +pub(in crate::segment_fetcher::tests) async fn prefix_drop_rounds() -> std::io::Result<()> { + use futures::StreamExt; + for drop_body in [false, true] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let (release, released) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0u8; 2048]; + let count = socket.read(&mut chunk).await?; + if count == 0 { + return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "incomplete test request")); + } + request.extend_from_slice(&chunk[..count]); + } + socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 12\r\nConnection: close\r\n\r\nsegment-").await?; + let _ = released.await; + if !drop_body { + socket.write_all(b"body").await?; + } + Ok::<_, std::io::Error>(()) + }); + let server = TestSegmentServer { + base_url: format!("http://{addr}"), + requests: Arc::new(Mutex::new(Vec::new())), + task: tokio::spawn(async {}), + }; + let directory = tempfile::tempdir()?; + let policy = SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..Default::default() }; + let (worker, context, _) = fetch_context(&server, &directory, &policy).await; + install_startup(&context, &worker, shared::model::HlsStartupMode::Progressive).await; + let snapshot = worker + .next_fetch_snapshot(&context, 10, &policy) + .await + .ok_or_else(|| std::io::Error::other("fetch snapshot"))?; + let fill_context = context.clone(); + let fill = tokio::spawn(async move { + super::super::fetch_segment_with_retries_into_cache(&fill_context, &snapshot, &policy).await + }); + let revision = tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + let found = context + .session + .read() + .await + .startup + .as_ref() + .and_then(|startup| startup.revisions.get(&1)) + .cloned(); + if let Some(revision) = found.filter(|revision| revision.revision().is_publishable()) { + return revision; + } + tokio::task::yield_now().await; + } + }) + .await + .map_err(std::io::Error::other)?; + let mut body = + crate::progressive_startup::revision_body(revision.clone(), std::time::Duration::from_secs(2), 0, None); + let prefix = tokio::time::timeout(std::time::Duration::from_secs(1), body.next()) + .await + .map_err(std::io::Error::other)? + .ok_or_else(|| std::io::Error::other("no progressive prefix"))??; + assert_eq!(prefix, b"segment-".as_slice()); + assert!(!fill.is_finished()); + release.send(()).map_err(|()| std::io::Error::other("origin release"))?; + task.await.map_err(std::io::Error::other)??; + if drop_body { + assert!(body.next().await.is_some_and(|chunk| chunk.is_err())); + assert!(matches!(*revision.revision().subscribe().borrow(), crate::SegmentRevisionState::Failed)); + fill.abort(); + let _ = fill.await; + } else { + assert_eq!(body.next().await.transpose()?.as_deref(), Some(b"body".as_slice())); + assert!(body.next().await.is_none()); + assert!(fill.await.map_err(std::io::Error::other)?.is_ok()); + } + } + Ok(()) +} diff --git a/backend/hls/src/segment_repair.rs b/backend/hls/src/segment_repair.rs index 22d48e132..8ce7d9be3 100644 --- a/backend/hls/src/segment_repair.rs +++ b/backend/hls/src/segment_repair.rs @@ -7,121 +7,23 @@ use super::{ SegmentCacheKey, SegmentCacheStatus, StagedCacheObject, }; use arc_swap::ArcSwap; -use log::{debug, warn}; -use serde_json::Value; -use sha2::{Digest, Sha256}; -use shared::model::{HlsSegmentRepairExecutionPlan, HlsSegmentRepairMode}; +use shared::model::HlsSegmentRepairMode; use std::{ collections::{HashMap, HashSet, VecDeque}, - fmt, io, - path::{Path, PathBuf}, sync::Arc, time::{Duration, Instant}, }; -use tokio::{ - fs::{self, File}, - io::{AsyncRead, AsyncReadExt}, - process::Command, - sync::{Mutex, RwLock, Semaphore}, - time::timeout, -}; -use tuliprox_core::model::{HlsCorruptSegmentWatchdogConfig, HlsSegmentRepairConfig}; +use tokio::sync::{Mutex, RwLock, Semaphore}; +use tuliprox_core::model::HlsSegmentRepairConfig; const COMMAND_VERSION: u32 = 1; + const REPAIR_METADATA_MAX_ENTRIES: usize = 4_096; + const REPAIR_OBJECT_METADATA_MAX_ENTRIES: usize = 8_192; + const REPAIR_CANDIDATE_MAX_ENTRIES: usize = 8_192; -#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] -pub enum HlsSegmentRepairSource { - Normal, - Transient, -} - -impl HlsSegmentRepairSource { - pub(super) const fn as_log_value(self) -> &'static str { - match self { - Self::Normal => "normal", - Self::Transient => "transient", - } - } -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub enum HlsRepairRenderedObjectId { - Normal { proxy_seq: u64 }, - Transient { resource_id: String }, -} - -#[derive(Debug, Clone)] -pub struct HlsSegmentRepairObjectContext { - pub source: HlsSegmentRepairSource, - pub log_identity: HlsLogIdentity, - pub proxy_session_id: ProxySessionId, - pub hls_access_lease_id: Option, - pub rendered_object_id: HlsRepairRenderedObjectId, - pub resource_id: String, - pub file_ext: String, - /// Concrete origin fetch URI retained for diagnostics and postprocess metadata. - /// - /// This may include a provider mirror or redirect/CDN host. It must not be used as HLS session identity, account - /// binding, provider-failover state, repair object identity, or repair-window candidate identity. - pub origin_fetch_uri_for_diagnostics: String, - pub media_sequence: Option, - pub discontinuity_sequence: Option, - pub complete_object: bool, - pub encrypted: bool, - pub custom_response: bool, -} - -pub fn ready_segment_repair_prewarm_candidates( - session: &HlsSession, - lease_id: &HlsAccessLeaseId, - snapshot: &HlsLeaseManifestSnapshot, - candidate_limit: usize, -) -> Vec<(SegmentCacheKey, HlsSegmentRepairObjectContext)> { - let identity = HlsLogIdentity::from_session(session); - snapshot - .visible_segments - .iter() - .take(candidate_limit) - .filter_map(|visible| { - let entry = session.segments.get(&visible.proxy_seq)?; - if entry.proxy_seq != visible.proxy_seq - || is_hls_provisioning_segment(entry) - || entry.proxy_file_ext != "ts" - || entry.origin_byte_range.is_some() - || entry.encryption.is_some() - || !matches!(entry.status, SegmentCacheStatus::Ready { .. }) - { - return None; - } - Some(( - entry.cache_key.clone(), - HlsSegmentRepairObjectContext { - source: HlsSegmentRepairSource::Normal, - log_identity: identity.clone(), - proxy_session_id: session.proxy_session_id.clone(), - hls_access_lease_id: Some(lease_id.clone()), - rendered_object_id: HlsRepairRenderedObjectId::Normal { proxy_seq: visible.proxy_seq }, - resource_id: format!("{:06}", visible.proxy_seq), - file_ext: entry.proxy_file_ext.clone(), - origin_fetch_uri_for_diagnostics: entry - .origin_fetch_ref - .as_ref() - .map(|fetch_ref| fetch_ref.resolved_origin_url.clone()) - .unwrap_or_default(), - media_sequence: Some(entry.origin_key.host_local_sequence), - discontinuity_sequence: Some(session.discontinuity_sequence), - complete_object: true, - encrypted: false, - custom_response: false, - }, - )) - }) - .collect() -} - #[derive(Clone)] pub struct HlsRepairPrewarmGuard { access_leases: Arc>, @@ -138,61 +40,6 @@ struct HlsSelectedRepairCandidate<'a> { prewarm_guard: Option<&'a HlsRepairPrewarmGuard>, } -impl HlsRepairPrewarmGuard { - pub fn new( - access_leases: Arc>, - lease_id: HlsAccessLeaseId, - proxy_session_id: ProxySessionId, - issued_at_ms: u64, - snapshot_generation: u64, - ) -> Self { - Self { access_leases, lease_id, proxy_session_id, issued_at_ms, snapshot_generation } - } - - async fn is_current(&self) -> bool { - self.access_leases.read().await.repair_prewarm_is_current( - &self.lease_id, - &self.proxy_session_id, - self.issued_at_ms, - self.snapshot_generation, - ) - } -} - -impl HlsSegmentRepairObjectContext { - pub(super) fn is_repairable_ts(&self) -> bool { - self.file_ext.eq_ignore_ascii_case("ts") && self.complete_object && !self.encrypted && !self.custom_response - } - - fn repair_skip_reason(&self) -> Option<&'static str> { - if self.is_repairable_ts() { - return None; - } - if !self.file_ext.eq_ignore_ascii_case("ts") { - return Some("not-ts"); - } - if !self.complete_object { - return Some("partial-object"); - } - if self.encrypted { - return Some("encrypted"); - } - if self.custom_response { - return Some("custom-response"); - } - None - } - - fn log_identity_fields(&self) -> String { - format!( - "session={} proxy_session={} lease={}", - self.log_identity.session(), - self.log_identity.proxy_session(), - self.hls_access_lease_id.as_ref().map_or_else(|| "".to_string(), safe_hls_access_lease_id) - ) - } -} - #[derive(Debug, Clone, Eq, PartialEq, Hash)] struct HlsRepairWindowCandidateKey { proxy_session_id: ProxySessionId, @@ -230,77 +77,6 @@ struct HlsRepairObjectMetadataKey { ffmpeg_version: String, } -#[derive(Debug, Clone, Eq, PartialEq)] -enum RepairCandidateSelection { - Selected(HlsSegmentRepairMode, HlsRepairWindowCandidateKey), - Skipped(&'static str), - AlreadyChecked(HlsSegmentRepairMode, HlsRepairWindowCandidateKey), -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum RepairStatus { - Clean, - Fixed, - PolicyLimited, - Unsupported, - Timeout, - RemuxFailed, - ValidationFailed, -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum RepairVideoCodec { - H264, - Hevc, - Unsupported, -} - -impl RepairVideoCodec { - const fn as_log_value(self) -> &'static str { - match self { - Self::H264 => "h264", - Self::Hevc => "hevc", - Self::Unsupported => "unsupported", - } - } -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] -#[allow(dead_code)] -enum HlsSegmentRepairWarningKind { - MissingPat, - MissingPmt, - MissingPmtPid, - MissingVideoPidInPmt, - MissingAudioPidInPmt, - MultiplePrograms, - PesPacketSizeMismatch, - PacketCorrupt, - ContinuityCheckFailed, - MissingVps, - MissingSps, - MissingPps, - VpsOutOfRange, - SpsOutOfRange, - PpsOutOfRange, - NoFrame, - DecodeSliceHeaderError, - MissingPicture, - InvalidNal, - InvalidData, - CodecParametersMissing, - MmcoUnrefShortFailure, - ReorderBufferIncrease, - PpsIdOutOfRange, - NoStartCode, - NalSplitError, - NalParseError, - InvalidVclNalu, - InvalidMetadataNalu, - MultipleDolbyVisionRpus, - AvDesync, -} - #[derive(Debug, Clone, Copy, Eq, PartialEq)] struct HlsSegmentRepairDecision { codec: RepairVideoCodec, @@ -310,31 +86,6 @@ struct HlsSegmentRepairDecision { codec_medium_trigger: bool, } -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum HlsSegmentRepairTriggerSource { - Off, - CommonMpegTsLow, - H264Medium, - H264High, - HevcMedium, - HevcHigh, - UnsupportedCodec, -} - -impl HlsSegmentRepairTriggerSource { - const fn as_log_value(self) -> &'static str { - match self { - Self::Off => "off", - Self::CommonMpegTsLow => "common-mpegts-low", - Self::H264Medium => "h264-medium", - Self::H264High => "h264-high", - Self::HevcMedium => "hevc-medium", - Self::HevcHigh => "hevc-high", - Self::UnsupportedCodec => "unsupported-codec", - } - } -} - #[derive(Debug, Clone)] #[allow(dead_code)] struct SegmentRepairMetadata { @@ -377,145 +128,6 @@ struct HlsRepairWindowRegistry { checked_candidate_order: VecDeque, } -impl HlsRepairWindowRegistry { - fn start_window(&mut self, lease_id: HlsAccessLeaseId, config: &HlsSegmentRepairConfig) { - if config.max_level == HlsSegmentRepairMode::Off || config.apply_to_first_segments == 0 { - self.windows.remove(&lease_id); - return; - } - let generation = self - .generations - .entry(lease_id.clone()) - .and_modify(|generation| *generation = generation.saturating_add(1)) - .or_insert(1); - self.windows.insert( - lease_id, - HlsRepairWindow { - mode: config.max_level, - activation_generation: *generation, - remaining_segments: config.apply_to_first_segments, - seen_candidates: HashSet::new(), - }, - ); - } - - fn ensure_window(&mut self, lease_id: HlsAccessLeaseId, config: &HlsSegmentRepairConfig) -> bool { - if self.windows.contains_key(&lease_id) { - return false; - } - self.start_window(lease_id, config); - true - } - - fn try_select_candidate(&mut self, context: &HlsSegmentRepairObjectContext) -> RepairCandidateSelection { - if let Some(reason) = context.repair_skip_reason() { - return RepairCandidateSelection::Skipped(reason); - } - let Some(lease_id) = context.hls_access_lease_id.as_ref() else { - return RepairCandidateSelection::Skipped("missing-lease"); - }; - let Some((activation_generation, mode)) = - self.windows.get(lease_id).map(|window| (window.activation_generation, window.mode)) - else { - return RepairCandidateSelection::Skipped("no-window"); - }; - let candidate_key = HlsRepairWindowCandidateKey { - proxy_session_id: context.proxy_session_id.clone(), - hls_access_lease_id: lease_id.clone(), - activation_generation, - object_id: context.rendered_object_id.clone(), - file_ext: context.file_ext.clone(), - }; - let Some(window) = self.windows.get(lease_id) else { - return RepairCandidateSelection::Skipped("no-window"); - }; - if window.seen_candidates.contains(&candidate_key) { - return RepairCandidateSelection::AlreadyChecked(mode, candidate_key); - } - if window.remaining_segments == 0 { - return RepairCandidateSelection::Skipped("window-exhausted"); - } - if !self.remember_candidate(candidate_key.clone()) { - return RepairCandidateSelection::Skipped("candidate-not-admitted"); - } - let Some(window) = self.windows.get_mut(lease_id) else { - return RepairCandidateSelection::Skipped("no-window"); - }; - window.seen_candidates.insert(candidate_key.clone()); - window.remaining_segments = window.remaining_segments.saturating_sub(1); - RepairCandidateSelection::Selected(window.mode, candidate_key) - } - - fn candidate_is_current(&self, candidate: &HlsRepairWindowCandidateKey) -> bool { - self.windows.get(&candidate.hls_access_lease_id).is_some_and(|window| { - window.activation_generation == candidate.activation_generation - && window.seen_candidates.contains(candidate) - }) - } - - fn remove_access_lease(&mut self, lease_id: &HlsAccessLeaseId) { - self.windows.remove(lease_id); - self.generations.remove(lease_id); - self.checked_candidates.retain(|key| key.hls_access_lease_id != *lease_id); - self.checked_candidate_order.retain(|key| key.hls_access_lease_id != *lease_id); - } - - fn remove_proxy_session(&mut self, proxy_session_id: &ProxySessionId, lease_ids: &[HlsAccessLeaseId]) { - for lease_id in lease_ids { - self.windows.remove(lease_id); - self.generations.remove(lease_id); - } - self.checked_candidates.retain(|key| key.proxy_session_id != *proxy_session_id); - self.checked_candidate_order.retain(|key| key.proxy_session_id != *proxy_session_id); - } - - fn clear(&mut self) { - self.windows.clear(); - self.generations.clear(); - self.checked_candidates.clear(); - self.checked_candidate_order.clear(); - } - - fn stats(&self) -> HlsSegmentRepairStats { - HlsSegmentRepairStats { - windows: self.windows.len(), - generations: self.generations.len(), - checked_candidates: self.checked_candidates.len(), - ..HlsSegmentRepairStats::default() - } - } - - fn remember_candidate(&mut self, key: HlsRepairWindowCandidateKey) -> bool { - if !self.checked_candidates.insert(key.clone()) { - return false; - } - self.checked_candidate_order.push_back(key); - self.prune_checked_candidates(); - true - } - - fn prune_checked_candidates(&mut self) { - while self.checked_candidates.len() > REPAIR_CANDIDATE_MAX_ENTRIES { - let Some(oldest) = self.checked_candidate_order.pop_front() else { - return; - }; - self.checked_candidates.remove(&oldest); - } - } -} - -#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] -pub struct HlsSegmentRepairStats { - pub windows: usize, - pub generations: usize, - pub checked_candidates: usize, - pub metadata: usize, - pub object_metadata: usize, - pub locks: usize, - pub watchdog_metadata: usize, - pub watchdog_locks: usize, -} - #[derive(Debug)] pub struct HlsSegmentRepairManager { runtime: ArcSwap, @@ -535,842 +147,6 @@ struct HlsSegmentRepairRuntime { watchdog_semaphore: Arc, } -impl HlsSegmentRepairRuntime { - fn new(config: HlsSegmentRepairConfig) -> Self { - let semaphore = if config.max_parallel_repairs == 0 { - None - } else { - Some(Arc::new(Semaphore::new(config.max_parallel_repairs))) - }; - let watchdog_config = &config.corrupt_segment_watchdog; - let watchdog_semaphore = Arc::new(Semaphore::new(watchdog_config.max_parallel_jobs.max(1))); - Self { config, semaphore, watchdog_semaphore } - } - - fn repair_enabled(&self) -> bool { - self.config.max_level != HlsSegmentRepairMode::Off && self.config.apply_to_first_segments > 0 - } - - fn postprocessing_enabled(&self) -> bool { - self.repair_enabled() || self.config.corrupt_segment_watchdog.mode.is_enabled() - } - - fn postprocess_timeout_ms(&self) -> u64 { self.config.postprocess_timeout_ms.get().max(100) } -} - -fn log_segment_repair_config(config: &HlsSegmentRepairConfig) { - debug!( - "HLS segment repair configured: max_level={} segments={} max_parallel={} postprocess_timeout_ms={}", - config.max_level.as_log_value(), - config.apply_to_first_segments, - config.max_parallel_repairs, - config.postprocess_timeout_ms - ); -} - -fn log_corrupt_segment_watchdog_config(config: &HlsCorruptSegmentWatchdogConfig) { - debug!( - "HLS corrupt segment watchdog configured: mode={} max_parallel_jobs={}", - config.mode.as_log_value(), - config.max_parallel_jobs - ); -} - -impl HlsSegmentRepairManager { - pub fn new(config: HlsSegmentRepairConfig) -> Self { - let watchdog_config = config.corrupt_segment_watchdog.clone(); - log_segment_repair_config(&config); - log_corrupt_segment_watchdog_config(&watchdog_config); - Self { - runtime: ArcSwap::from_pointee(HlsSegmentRepairRuntime::new(config)), - watchdog: HlsCorruptSegmentWatchdogManager::new(), - windows: RwLock::new(HlsRepairWindowRegistry::default()), - metadata: RwLock::new(HashMap::new()), - metadata_order: Mutex::new(VecDeque::new()), - object_metadata: RwLock::new(HashMap::new()), - object_metadata_order: Mutex::new(VecDeque::new()), - locks: Mutex::new(HashMap::new()), - } - } - - pub fn update_config(&self, config: HlsSegmentRepairConfig) { - let current = self.runtime.load(); - if current.config == config { - return; - } - let watchdog_config = config.corrupt_segment_watchdog.clone(); - log_segment_repair_config(&config); - log_corrupt_segment_watchdog_config(&watchdog_config); - self.runtime.store(Arc::new(HlsSegmentRepairRuntime::new(config))); - } - - pub async fn start_access_lease_window(&self, lease_id: HlsAccessLeaseId) { - let runtime = self.runtime.load_full(); - if !runtime.repair_enabled() { - return; - } - self.windows.write().await.start_window(lease_id.clone(), &runtime.config); - debug!( - "HLS segment repair window started: lease={} max_level={} segments={}", - safe_hls_access_lease_id(&lease_id), - runtime.config.max_level.as_log_value(), - runtime.config.apply_to_first_segments - ); - } - - pub async fn ensure_access_lease_window(&self, lease_id: HlsAccessLeaseId) { - let runtime = self.runtime.load_full(); - if !runtime.repair_enabled() { - return; - } - let started = self.windows.write().await.ensure_window(lease_id.clone(), &runtime.config); - if started { - debug!( - "HLS segment repair window started: lease={} max_level={} segments={}", - safe_hls_access_lease_id(&lease_id), - runtime.config.max_level.as_log_value(), - runtime.config.apply_to_first_segments - ); - } - } - - pub fn prewarm_candidate_limit(&self) -> usize { - let runtime = self.runtime.load(); - if runtime.repair_enabled() { - usize::from(runtime.config.apply_to_first_segments) - } else { - 0 - } - } - - pub async fn spawn_ready_cache_prewarm( - self: &Arc, - segment_cache: Arc, - candidates: Vec<(K, HlsSegmentRepairObjectContext)>, - guard: HlsRepairPrewarmGuard, - ) where - K: HlsCacheObjectKey + Send + Sync + 'static, - { - self.spawn_ready_cache_prewarm_inner(segment_cache, candidates, Some(guard)).await; - } - - async fn spawn_ready_cache_prewarm_inner( - self: &Arc, - segment_cache: Arc, - candidates: Vec<(K, HlsSegmentRepairObjectContext)>, - guard: Option, - ) where - K: HlsCacheObjectKey + Send + Sync + 'static, - { - for (key, context) in candidates { - let Some((mode, runtime, candidate)) = self.try_select_candidate(&context).await else { - continue; - }; - let manager = Arc::clone(self); - let segment_cache = Arc::clone(&segment_cache); - let guard = guard.clone(); - tokio::spawn(async move { - let selection = HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard: guard.as_ref() }; - if let Err(err) = - manager.repair_ready_cache_hit_with_candidate(&segment_cache, &key, context, selection).await - { - debug!("HLS segment repair prewarm ended without a reusable decision: error_kind={:?}", err.kind()); - } - }); - } - } - - pub async fn remove_access_lease_window(&self, lease_id: &HlsAccessLeaseId) { - self.windows.write().await.remove_access_lease(lease_id); - } - - pub async fn remove_proxy_session_state(&self, proxy_session_id: &ProxySessionId, lease_ids: &[HlsAccessLeaseId]) { - self.windows.write().await.remove_proxy_session(proxy_session_id, lease_ids); - self.object_metadata.write().await.retain(|key, _| key.proxy_session_id != *proxy_session_id); - self.object_metadata_order.lock().await.retain(|key| key.proxy_session_id != *proxy_session_id); - } - - pub async fn clear_runtime_state(&self) { - self.windows.write().await.clear(); - self.metadata.write().await.clear(); - self.metadata_order.lock().await.clear(); - self.object_metadata.write().await.clear(); - self.object_metadata_order.lock().await.clear(); - self.locks.lock().await.clear(); - self.watchdog.clear_runtime_state().await; - } - - pub async fn stats(&self) -> HlsSegmentRepairStats { - let mut stats = self.windows.read().await.stats(); - stats.metadata = self.metadata.read().await.len(); - stats.object_metadata = self.object_metadata.read().await.len(); - stats.locks = self.locks.lock().await.len(); - let watchdog = self.watchdog.stats().await; - stats.watchdog_metadata = watchdog.metadata; - stats.watchdog_locks = watchdog.locks; - stats - } - - pub async fn commit_origin_response( - &self, - segment_cache: &HlsSegmentCache, - key: &K, - reader: R, - deadline: tokio::time::Instant, - context: HlsSegmentRepairObjectContext, - ) -> io::Result - where - K: HlsCacheObjectKey, - R: AsyncRead + Unpin, - { - let raw = segment_cache.stage_temp_with_deadline(key, reader, deadline).await?; - let runtime = self.runtime.load_full(); - if !runtime.postprocessing_enabled() { - return segment_cache.commit_staged(key, raw).await; - } - let postprocessing_deadline = HlsPostProcessingDeadline::new(runtime.postprocess_timeout_ms()); - self.process_staged_and_commit(segment_cache, key, raw, context, runtime, postprocessing_deadline).await - } - - #[allow(clippy::too_many_lines)] - pub async fn repair_ready_cache_hit( - &self, - segment_cache: &HlsSegmentCache, - key: &K, - context: HlsSegmentRepairObjectContext, - ) -> io::Result> - where - K: HlsCacheObjectKey, - { - let Some((mode, runtime, candidate)) = self.try_select_or_join_candidate(&context).await else { - return Ok(None); - }; - self.repair_ready_cache_hit_with_candidate( - segment_cache, - key, - context, - HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard: None }, - ) - .await - } - - #[allow(clippy::too_many_lines)] - async fn repair_ready_cache_hit_with_candidate( - &self, - segment_cache: &HlsSegmentCache, - key: &K, - context: HlsSegmentRepairObjectContext, - selection: HlsSelectedRepairCandidate<'_>, - ) -> io::Result> - where - K: HlsCacheObjectKey, - { - let HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard } = selection; - let Some(metadata) = segment_cache.metadata(key).await? else { - return Ok(None); - }; - let raw_hash = sha256_file(&metadata.path).await?; - let object_key = repair_object_metadata_key(&context, mode); - if self.object_metadata_matches(&object_key, &raw_hash).await { - return Ok(None); - } - let identity = RepairIdentity { - raw_sha256: raw_hash.clone(), - repair_mode: mode, - command_version: COMMAND_VERSION, - ffmpeg_version: ffmpeg_identity_version(), - }; - if self.repair_metadata(&identity).await.is_some() { - self.record_object_metadata_from_repair_identity(object_key, raw_hash.clone(), Some(raw_hash), &identity) - .await; - return Ok(None); - } - let lock = self.lock_for_identity(identity.clone()).await; - let result = { - let _guard = lock.lock().await; - let prewarm_is_current = match prewarm_guard { - Some(prewarm_guard) => prewarm_guard.is_current().await, - None => true, - }; - if !prewarm_is_current { - Ok(None) - } else if let Some(current_metadata) = segment_cache.metadata(key).await? { - let current_hash = sha256_file(¤t_metadata.path).await?; - if current_hash != raw_hash { - Ok(None) - } else if self.repair_metadata(&identity).await.is_some() { - self.record_object_metadata_from_repair_identity( - object_key.clone(), - current_hash.clone(), - Some(current_hash), - &identity, - ) - .await; - Ok(None) - } else { - let deadline = HlsPostProcessingDeadline::new(runtime.postprocess_timeout_ms()); - if let Some(fixed_path) = self - .repair_file( - ¤t_metadata.path, - current_metadata.size, - &identity, - &context, - runtime.clone(), - &deadline, - ) - .await? - { - let candidate_current = self.windows.read().await.candidate_is_current(&candidate) - && match prewarm_guard { - Some(prewarm_guard) => prewarm_guard.is_current().await, - None => true, - }; - let latest_hash = match segment_cache.metadata(key).await? { - Some(latest_metadata) => Some(sha256_file(&latest_metadata.path).await?), - None => None, - }; - if !candidate_current || latest_hash.as_deref() != Some(current_hash.as_str()) { - let _ = fs::remove_file(&fixed_path).await; - Ok(None) - } else { - let fixed_size = fs::metadata(&fixed_path).await?.len(); - let staged = adopt_repair_output(segment_cache, fixed_path, fixed_size).await?; - let committed = segment_cache.commit_staged(key, staged).await?; - self.record_metadata( - identity.clone(), - RepairStatus::Fixed, - current_metadata.size, - committed.size, - None, - ) - .await; - let committed_hash = sha256_file(&committed.path).await?; - self.record_object_metadata( - object_key.clone(), - HlsRepairObjectMetadata { - committed_sha256: committed_hash, - raw_sha256: Some(current_hash), - status: RepairStatus::Fixed, - raw_size: current_metadata.size, - final_size: committed.size, - validation_reason: None, - }, - ) - .await; - Ok(Some(committed)) - } - } else { - self.record_object_metadata_from_repair_identity( - object_key.clone(), - current_hash.clone(), - Some(current_hash), - &identity, - ) - .await; - Ok(None) - } - } - } else { - Ok(None) - } - }; - self.remove_lock_if_unused(&identity, &lock).await; - result - } - - async fn process_staged_and_commit( - &self, - segment_cache: &HlsSegmentCache, - key: &K, - raw: StagedCacheObject, - context: HlsSegmentRepairObjectContext, - runtime: Arc, - deadline: HlsPostProcessingDeadline, - ) -> io::Result - where - K: HlsCacheObjectKey, - { - let selected_repair = self.try_select_candidate_with_runtime(&context, &runtime).await; - if selected_repair.is_none() - && runtime.config.corrupt_segment_watchdog.mode.is_enabled() - && context.is_repairable_ts() - { - let raw_hash = sha256_file(&raw.path).await?; - return self - .watchdog - .process_staged_and_commit( - segment_cache, - key, - raw, - &context, - &runtime.config.corrupt_segment_watchdog, - &runtime.watchdog_semaphore, - raw_hash, - &deadline, - ) - .await; - } - let Some(mode) = selected_repair else { - return segment_cache.commit_staged(key, raw).await; - }; - let raw_hash = sha256_file(&raw.path).await?; - let object_key = repair_object_metadata_key(&context, mode); - if self.object_metadata_matches(&object_key, &raw_hash).await { - return segment_cache.commit_staged(key, raw).await; - } - let identity = RepairIdentity { - raw_sha256: raw_hash.clone(), - repair_mode: mode, - command_version: COMMAND_VERSION, - ffmpeg_version: ffmpeg_identity_version(), - }; - if self.repair_metadata(&identity).await.is_some() { - let committed = segment_cache.commit_staged(key, raw).await?; - self.record_object_metadata_from_repair_identity(object_key, raw_hash.clone(), Some(raw_hash), &identity) - .await; - return Ok(committed); - } - let lock = self.lock_for_identity(identity.clone()).await; - let result = { - let _guard = lock.lock().await; - if self.repair_metadata(&identity).await.is_some() { - let committed = segment_cache.commit_staged(key, raw).await?; - self.record_object_metadata_from_repair_identity( - object_key.clone(), - raw_hash.clone(), - Some(raw_hash.clone()), - &identity, - ) - .await; - Ok(committed) - } else { - let raw_size = raw.size; - if let Some(fixed_path) = - self.repair_file(&raw.path, raw.size, &identity, &context, runtime.clone(), &deadline).await? - { - let fixed_size = fs::metadata(&fixed_path).await?.len(); - if let Err(error) = segment_cache.remove_staged(raw.clone()).await { - warn!("HLS repair raw staging cleanup deferred: error_kind={:?}", error.kind()); - } - let fixed = adopt_repair_output(segment_cache, fixed_path, fixed_size).await?; - let committed = segment_cache.commit_staged(key, fixed).await?; - self.record_metadata(identity.clone(), RepairStatus::Fixed, raw.size, committed.size, None).await; - let committed_hash = sha256_file(&committed.path).await?; - self.record_object_metadata( - object_key.clone(), - HlsRepairObjectMetadata { - committed_sha256: committed_hash, - raw_sha256: Some(raw_hash.clone()), - status: RepairStatus::Fixed, - raw_size, - final_size: committed.size, - validation_reason: None, - }, - ) - .await; - Ok(committed) - } else { - let committed = segment_cache.commit_staged(key, raw).await?; - self.record_object_metadata_from_repair_identity( - object_key.clone(), - raw_hash.clone(), - Some(raw_hash.clone()), - &identity, - ) - .await; - Ok(committed) - } - } - }; - self.remove_lock_if_unused(&identity, &lock).await; - result - } - - async fn try_select_candidate( - &self, - context: &HlsSegmentRepairObjectContext, - ) -> Option<(HlsSegmentRepairMode, Arc, HlsRepairWindowCandidateKey)> { - let runtime = self.runtime.load_full(); - match self.select_candidate(context, &runtime).await { - RepairCandidateSelection::Selected(mode, candidate) => Some((mode, runtime, candidate)), - RepairCandidateSelection::Skipped(_) | RepairCandidateSelection::AlreadyChecked(_, _) => None, - } - } - - async fn try_select_or_join_candidate( - &self, - context: &HlsSegmentRepairObjectContext, - ) -> Option<(HlsSegmentRepairMode, Arc, HlsRepairWindowCandidateKey)> { - let runtime = self.runtime.load_full(); - match self.select_candidate(context, &runtime).await { - RepairCandidateSelection::Selected(mode, candidate) - | RepairCandidateSelection::AlreadyChecked(mode, candidate) => Some((mode, runtime, candidate)), - RepairCandidateSelection::Skipped(_) => None, - } - } - - async fn try_select_candidate_with_runtime( - &self, - context: &HlsSegmentRepairObjectContext, - runtime: &Arc, - ) -> Option { - match self.select_candidate(context, runtime).await { - RepairCandidateSelection::Selected(mode, _) => Some(mode), - RepairCandidateSelection::Skipped(_) | RepairCandidateSelection::AlreadyChecked(_, _) => None, - } - } - - async fn select_candidate( - &self, - context: &HlsSegmentRepairObjectContext, - runtime: &Arc, - ) -> RepairCandidateSelection { - if !runtime.repair_enabled() { - return RepairCandidateSelection::Skipped("disabled"); - } - if context.hls_access_lease_id.is_none() { - return RepairCandidateSelection::Skipped("missing-lease"); - } - let selection = self.windows.write().await.try_select_candidate(context); - match &selection { - RepairCandidateSelection::Selected(mode, _) => debug!( - "HLS segment repair candidate selected: {} source={} resource={} mode={}", - context.log_identity_fields(), - context.source.as_log_value(), - context.resource_id, - mode.as_log_value() - ), - RepairCandidateSelection::Skipped(reason) => debug!( - "HLS segment repair candidate skipped: {} source={} resource={} reason={}", - context.log_identity_fields(), - context.source.as_log_value(), - context.resource_id, - reason - ), - RepairCandidateSelection::AlreadyChecked(_, _) => {} - } - selection - } - - #[allow(clippy::too_many_lines)] - async fn repair_file( - &self, - raw_path: &Path, - raw_size: u64, - identity: &RepairIdentity, - context: &HlsSegmentRepairObjectContext, - runtime: Arc, - deadline: &HlsPostProcessingDeadline, - ) -> io::Result> { - let _permit = match &runtime.semaphore { - Some(semaphore) => { - let Some(remaining) = deadline.remaining() else { - self.record_metadata( - identity.clone(), - RepairStatus::Timeout, - raw_size, - raw_size, - Some("timeout".to_string()), - ) - .await; - return Ok(None); - }; - Some( - timeout(remaining, semaphore.acquire()) - .await - .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "repair semaphore timed out"))? - .map_err(|_| io::Error::other("repair semaphore closed"))?, - ) - } - None => None, - }; - let started = Instant::now(); - let raw_scan = match analyze_segment(raw_path, deadline).await { - Ok(scan) => scan, - Err(reason) => { - debug_repair_event(context, identity.repair_mode, "analysis skipped", Some(&reason)); - self.record_metadata(identity.clone(), RepairStatus::Unsupported, raw_size, raw_size, Some(reason)) - .await; - return Ok(None); - } - }; - let codec = detect_video_codec(&raw_scan); - let decision = decide_repair(codec, &raw_scan.warnings); - let execution_plan = identity.repair_mode.execution_plan(decision.required_level); - let executed_level = match execution_plan { - HlsSegmentRepairExecutionPlan::Repair(level) => Some(level), - HlsSegmentRepairExecutionPlan::SkipNoTrigger - | HlsSegmentRepairExecutionPlan::SkipConfiguredMaxBelowRequired => None, - }; - debug_repair_analysis(context, identity.repair_mode, decision, executed_level, &raw_scan.warnings); - let executed_level = match execution_plan { - HlsSegmentRepairExecutionPlan::Repair(level) => level, - HlsSegmentRepairExecutionPlan::SkipNoTrigger => { - self.record_metadata(identity.clone(), RepairStatus::Clean, raw_size, raw_size, None).await; - return Ok(None); - } - HlsSegmentRepairExecutionPlan::SkipConfiguredMaxBelowRequired => { - warn!( - "HLS segment repair required level exceeds configured max: {} source={} resource={} configured_max_level={} required_level={} trigger={} action=raw_commit", - context.log_identity_fields(), - context.source.as_log_value(), - context.resource_id, - identity.repair_mode.as_log_value(), - decision.required_level.as_log_value(), - decision.trigger_source.as_log_value() - ); - self.record_metadata( - identity.clone(), - RepairStatus::PolicyLimited, - raw_size, - raw_size, - Some("configured_max_level_below_required_level".to_string()), - ) - .await; - return Ok(None); - } - }; - let stream_selection = match select_repair_remux_streams(&raw_scan) { - Ok(selection) => selection, - Err(reason) => { - debug_repair_event(context, executed_level, "stream selection skipped", Some(&reason)); - self.record_metadata(identity.clone(), RepairStatus::Unsupported, raw_size, raw_size, Some(reason)) - .await; - return Ok(None); - } - }; - for dropped in &stream_selection.dropped_streams { - debug_repair_stream_dropped(context, dropped); - } - let fixed_path = repair_output_path(raw_path); - debug!( - "HLS segment repair remux started: {} source={} resource={} configured_max_level={} executed_level={}", - context.log_identity_fields(), - context.source.as_log_value(), - context.resource_id, - identity.repair_mode.as_log_value(), - executed_level.as_log_value() - ); - let remux = run_remux(raw_path, &fixed_path, executed_level, &stream_selection, deadline).await; - if let Err(reason) = remux { - debug_repair_event(context, executed_level, "remux failed", Some(&reason)); - let status = if reason == "timeout" { RepairStatus::Timeout } else { RepairStatus::RemuxFailed }; - cleanup_repair_output(&fixed_path).await; - self.record_metadata(identity.clone(), status, raw_size, raw_size, Some(reason)).await; - return Ok(None); - } - let fixed_scan = match analyze_segment(&fixed_path, deadline).await { - Ok(scan) => scan, - Err(reason) => { - debug_repair_event(context, executed_level, "validation probe failed", Some(&reason)); - cleanup_repair_output(&fixed_path).await; - self.record_metadata( - identity.clone(), - RepairStatus::ValidationFailed, - raw_size, - raw_size, - Some(reason), - ) - .await; - return Ok(None); - } - }; - let validation = - validate_repair(&runtime.config, codec, &raw_scan, &fixed_scan, executed_level, &stream_selection); - if let Err(reason) = validation { - debug_repair_event(context, executed_level, "validation failed", Some(&reason)); - cleanup_repair_output(&fixed_path).await; - self.record_metadata(identity.clone(), RepairStatus::ValidationFailed, raw_size, raw_size, Some(reason)) - .await; - return Ok(None); - } - debug!( - "HLS segment repair remux completed: {} source={} resource={} configured_max_level={} executed_level={} duration_ms={}", - context.log_identity_fields(), - context.source.as_log_value(), - context.resource_id, - identity.repair_mode.as_log_value(), - executed_level.as_log_value(), - started.elapsed().as_millis() - ); - Ok(Some(fixed_path)) - } - - async fn record_metadata( - &self, - identity: RepairIdentity, - status: RepairStatus, - raw_size: u64, - final_size: u64, - validation_reason: Option, - ) { - let inserted_new = { - let mut metadata = self.metadata.write().await; - let inserted_new = !metadata.contains_key(&identity); - metadata - .insert(identity.clone(), SegmentRepairMetadata { status, raw_size, final_size, validation_reason }); - inserted_new - }; - if inserted_new { - self.metadata_order.lock().await.push_back(identity); - } - self.prune_metadata().await; - } - - async fn repair_metadata(&self, identity: &RepairIdentity) -> Option { - self.metadata.read().await.get(identity).cloned() - } - - async fn object_metadata_matches(&self, key: &HlsRepairObjectMetadataKey, committed_sha256: &str) -> bool { - let matches = self - .object_metadata - .read() - .await - .get(key) - .is_some_and(|metadata| metadata.committed_sha256 == committed_sha256); - if matches { - let (source, resource) = match &key.rendered_object_id { - HlsRepairRenderedObjectId::Normal { proxy_seq } => ("normal", format!("{proxy_seq:06}")), - HlsRepairRenderedObjectId::Transient { resource_id } => ("transient", resource_id.clone()), - }; - debug!( - "HLS segment repair object metadata hit: proxy_session={} source={} resource={} mode={}", - safe_proxy_session_id(&key.proxy_session_id), - source, - resource, - key.repair_mode.as_log_value() - ); - } - matches - } - - async fn record_object_metadata_from_repair_identity( - &self, - key: HlsRepairObjectMetadataKey, - committed_sha256: String, - raw_sha256: Option, - identity: &RepairIdentity, - ) { - let Some(metadata) = self.repair_metadata(identity).await else { - return; - }; - self.record_object_metadata( - key, - HlsRepairObjectMetadata { - committed_sha256, - raw_sha256, - status: metadata.status, - raw_size: metadata.raw_size, - final_size: metadata.final_size, - validation_reason: metadata.validation_reason, - }, - ) - .await; - } - - async fn record_object_metadata(&self, key: HlsRepairObjectMetadataKey, metadata: HlsRepairObjectMetadata) { - let inserted_new = { - let mut object_metadata = self.object_metadata.write().await; - let inserted_new = !object_metadata.contains_key(&key); - object_metadata.insert(key.clone(), metadata); - inserted_new - }; - if inserted_new { - self.object_metadata_order.lock().await.push_back(key); - } - self.prune_object_metadata().await; - } - - async fn lock_for_identity(&self, identity: RepairIdentity) -> Arc> { - let mut locks = self.locks.lock().await; - Arc::clone(locks.entry(identity).or_insert_with(|| Arc::new(Mutex::new(())))) - } - - async fn remove_lock_if_unused(&self, identity: &RepairIdentity, lock: &Arc>) { - let mut locks = self.locks.lock().await; - if Arc::strong_count(lock) <= 2 && locks.get(identity).is_some_and(|current| Arc::ptr_eq(current, lock)) { - locks.remove(identity); - } - } - - async fn prune_metadata(&self) { - loop { - let should_prune = self.metadata.read().await.len() > REPAIR_METADATA_MAX_ENTRIES; - if !should_prune { - return; - } - let Some(oldest) = self.metadata_order.lock().await.pop_front() else { - return; - }; - self.metadata.write().await.remove(&oldest); - } - } - - async fn prune_object_metadata(&self) { - loop { - let should_prune = self.object_metadata.read().await.len() > REPAIR_OBJECT_METADATA_MAX_ENTRIES; - if !should_prune { - return; - } - let Some(oldest) = self.object_metadata_order.lock().await.pop_front() else { - return; - }; - self.object_metadata.write().await.remove(&oldest); - } - } -} - -fn repair_object_metadata_key( - context: &HlsSegmentRepairObjectContext, - repair_mode: HlsSegmentRepairMode, -) -> HlsRepairObjectMetadataKey { - HlsRepairObjectMetadataKey { - proxy_session_id: context.proxy_session_id.clone(), - rendered_object_id: context.rendered_object_id.clone(), - file_ext: context.file_ext.to_ascii_lowercase(), - repair_mode, - command_version: COMMAND_VERSION, - ffmpeg_version: ffmpeg_identity_version(), - } -} - -#[derive(Debug, Clone, Default, Eq, PartialEq)] -pub struct WarningCounters { - pub missing_pat: u32, - pub missing_pmt: u32, - pub missing_pmt_pid: u32, - pub missing_video_pid_in_pmt: u32, - pub missing_audio_pid_in_pmt: u32, - pub multiple_programs: u32, - pub pes_packet_size_mismatch: u32, - pub packet_corrupt: u32, - pub continuity_check_failed: u32, - pub missing_vps: u32, - pub missing_sps: u32, - pub missing_pps: u32, - pub vps_out_of_range: u32, - pub sps_out_of_range: u32, - pub pps_out_of_range: u32, - pub no_frame: u32, - pub decode_slice_header_error: u32, - pub missing_picture: u32, - pub invalid_nal: u32, - pub invalid_data: u32, - pub codec_parameters_missing: u32, - pub mmco_unref_short_failure: u32, - pub reorder_buffer: u32, - pub pps_id_out_of_range: u32, - pub no_start_code: u32, - pub nal_split_error: u32, - pub nal_parse_error: u32, - pub invalid_undecodable_nalu_total: u32, - pub invalid_undecodable_nalu_non_metadata: u32, - pub invalid_undecodable_nalu_keyframe: u32, - pub invalid_undecodable_nalu_metadata: u32, - pub dolby_vision_rpu: u32, - pub av_desync: u32, -} - #[derive(Debug, Clone, Default)] struct SegmentProbe { duration_ms: Option, @@ -1396,1840 +172,35 @@ struct SegmentProbeStream { channels: Option, } -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum SegmentProbeStreamType { - Video, - Audio, - Other, -} - #[derive(Debug, Clone, Eq, PartialEq)] struct RepairRemuxStreamSelection { mapped_streams: Vec, dropped_streams: Vec, } -#[cfg(any(test, feature = "test-support"))] -impl RepairRemuxStreamSelection { - fn preserve_all(probe: &SegmentProbe) -> Self { - Self { mapped_streams: probe.streams.iter().map(|stream| stream.index).collect(), dropped_streams: Vec::new() } - } -} - #[derive(Debug, Clone, Eq, PartialEq)] struct RepairRemuxDroppedStream { index: usize, reason: &'static str, } -pub fn parse_ffmpeg_warnings(stderr: &str) -> WarningCounters { - let mut counters = WarningCounters::default(); - let mut last_increment: Option = None; - for line in stderr.lines() { - let trimmed = line.trim(); - if let Some(repeated) = parse_repeated_count(trimmed) { - if let Some(increment) = last_increment { - increment(&mut counters, repeated); - } - continue; - } - if let Some(increment) = warning_increment(trimmed) { - increment(&mut counters, 1); - last_increment = Some(increment); - } - } - counters -} - -fn parse_repeated_count(line: &str) -> Option { - let rest = line.strip_prefix("Last message repeated ")?; - let count = rest.strip_suffix(" times")?.parse().ok()?; - Some(count) -} - -#[allow(clippy::too_many_lines)] -fn warning_increment(line: &str) -> Option { - let lower = line.to_ascii_lowercase(); - if lower.contains("missing pat") { - return Some(|counters, count| counters.missing_pat = counters.missing_pat.saturating_add(count)); - } - if lower.contains("missing pmt pid") { - return Some(|counters, count| counters.missing_pmt_pid = counters.missing_pmt_pid.saturating_add(count)); - } - if lower.contains("missing pmt") { - return Some(|counters, count| counters.missing_pmt = counters.missing_pmt.saturating_add(count)); - } - if lower.contains("missing video pid") && lower.contains("pmt") { - return Some(|counters, count| { - counters.missing_video_pid_in_pmt = counters.missing_video_pid_in_pmt.saturating_add(count); - }); - } - if lower.contains("missing audio pid") && lower.contains("pmt") { - return Some(|counters, count| { - counters.missing_audio_pid_in_pmt = counters.missing_audio_pid_in_pmt.saturating_add(count); - }); - } - if lower.contains("multiple mpeg-ts programs") || lower.contains("multiple programs") { - return Some(|counters, count| counters.multiple_programs = counters.multiple_programs.saturating_add(count)); - } - if lower.contains("pes packet size mismatch") { - return Some(|counters, count| { - counters.pes_packet_size_mismatch = counters.pes_packet_size_mismatch.saturating_add(count); - }); - } - if lower.contains("packet corrupt") { - return Some(|counters, count| counters.packet_corrupt = counters.packet_corrupt.saturating_add(count)); - } - if lower.contains("continuity check failed") { - return Some(|counters, count| { - counters.continuity_check_failed = counters.continuity_check_failed.saturating_add(count); - }); - } - if lower.contains("non-existing vps") || lower.contains("missing vps") { - return Some(|counters, count| counters.missing_vps = counters.missing_vps.saturating_add(count)); - } - if lower.contains("non-existing sps") || lower.contains("missing sps") { - return Some(|counters, count| counters.missing_sps = counters.missing_sps.saturating_add(count)); - } - if lower.contains("non-existing pps") || lower.contains("missing pps") { - return Some(|counters, count| counters.missing_pps = counters.missing_pps.saturating_add(count)); - } - if lower.contains("vps id out of range") { - return Some(|counters, count| counters.vps_out_of_range = counters.vps_out_of_range.saturating_add(count)); - } - if lower.contains("sps id out of range") { - return Some(|counters, count| counters.sps_out_of_range = counters.sps_out_of_range.saturating_add(count)); - } - if lower.contains("pps id out of range") { - return Some(|counters, count| { - counters.pps_id_out_of_range = counters.pps_id_out_of_range.saturating_add(count); - counters.pps_out_of_range = counters.pps_out_of_range.saturating_add(count); - }); - } - if lower.contains("vps") && lower.contains("out of range") { - return Some(|counters, count| counters.vps_out_of_range = counters.vps_out_of_range.saturating_add(count)); - } - if lower.contains("sps") && lower.contains("out of range") { - return Some(|counters, count| counters.sps_out_of_range = counters.sps_out_of_range.saturating_add(count)); - } - if lower.contains("pps") && lower.contains("out of range") { - return Some(|counters, count| counters.pps_out_of_range = counters.pps_out_of_range.saturating_add(count)); - } - if lower.contains("no frame") { - return Some(|counters, count| counters.no_frame = counters.no_frame.saturating_add(count)); - } - if lower.contains("decode_slice_header error") { - return Some(|counters, count| { - counters.decode_slice_header_error = counters.decode_slice_header_error.saturating_add(count); - }); - } - if lower.contains("missing picture") { - return Some(|counters, count| counters.missing_picture = counters.missing_picture.saturating_add(count)); - } - if lower.contains("invalid nal") { - return Some(|counters, count| counters.invalid_nal = counters.invalid_nal.saturating_add(count)); - } - if lower.contains("invalid data found") { - return Some(|counters, count| counters.invalid_data = counters.invalid_data.saturating_add(count)); - } - if lower.contains("could not find codec parameters") { - return Some(|counters, count| { - counters.codec_parameters_missing = counters.codec_parameters_missing.saturating_add(count); - }); - } - if lower.contains("mmco: unref short failure") { - return Some(|counters, count| { - counters.mmco_unref_short_failure = counters.mmco_unref_short_failure.saturating_add(count); - }); - } - if lower.contains("increasing reorder buffer") { - return Some(|counters, count| counters.reorder_buffer = counters.reorder_buffer.saturating_add(count)); - } - if lower.contains("no start code is found") { - return Some(|counters, count| counters.no_start_code = counters.no_start_code.saturating_add(count)); - } - if lower.contains("error splitting") && lower.contains("nal") { - return Some(|counters, count| counters.nal_split_error = counters.nal_split_error.saturating_add(count)); - } - if lower.contains("error parsing nal unit") { - return Some(|counters, count| counters.nal_parse_error = counters.nal_parse_error.saturating_add(count)); - } - if let Some(nalu_type) = parse_invalid_undecodable_nalu_type(line) { - return Some(match nalu_type { - 0..=20 | 22..=31 => |counters: &mut WarningCounters, count| { - counters.invalid_undecodable_nalu_total = counters.invalid_undecodable_nalu_total.saturating_add(count); - counters.invalid_undecodable_nalu_non_metadata = - counters.invalid_undecodable_nalu_non_metadata.saturating_add(count); - }, - 21 => |counters: &mut WarningCounters, count| { - counters.invalid_undecodable_nalu_total = counters.invalid_undecodable_nalu_total.saturating_add(count); - counters.invalid_undecodable_nalu_keyframe = - counters.invalid_undecodable_nalu_keyframe.saturating_add(count); - }, - 39 => |counters: &mut WarningCounters, count| { - counters.invalid_undecodable_nalu_total = counters.invalid_undecodable_nalu_total.saturating_add(count); - counters.invalid_undecodable_nalu_metadata = - counters.invalid_undecodable_nalu_metadata.saturating_add(count); - }, - _ => |counters: &mut WarningCounters, count| { - counters.invalid_undecodable_nalu_total = counters.invalid_undecodable_nalu_total.saturating_add(count); - }, - }); - } - if lower.contains("multiple dolby vision rpus found in one au") { - return Some(|counters, count| counters.dolby_vision_rpu = counters.dolby_vision_rpu.saturating_add(count)); - } - if lower.contains("audio/video desynchronisation detected") { - return Some(|counters, count| counters.av_desync = counters.av_desync.saturating_add(count)); - } - None -} - -fn parse_invalid_undecodable_nalu_type(line: &str) -> Option { - let (_, rest) = line.split_once("Skipping invalid undecodable NALU:")?; - rest.trim().split(|ch: char| !ch.is_ascii_digit()).next().filter(|value| !value.is_empty())?.parse().ok() -} - -fn warning_count(warnings: &WarningCounters, kind: HlsSegmentRepairWarningKind) -> u32 { - match kind { - HlsSegmentRepairWarningKind::MissingPat => warnings.missing_pat, - HlsSegmentRepairWarningKind::MissingPmt => warnings.missing_pmt, - HlsSegmentRepairWarningKind::MissingPmtPid => warnings.missing_pmt_pid, - HlsSegmentRepairWarningKind::MissingVideoPidInPmt => warnings.missing_video_pid_in_pmt, - HlsSegmentRepairWarningKind::MissingAudioPidInPmt => warnings.missing_audio_pid_in_pmt, - HlsSegmentRepairWarningKind::MultiplePrograms => warnings.multiple_programs, - HlsSegmentRepairWarningKind::PesPacketSizeMismatch => warnings.pes_packet_size_mismatch, - HlsSegmentRepairWarningKind::PacketCorrupt => warnings.packet_corrupt, - HlsSegmentRepairWarningKind::ContinuityCheckFailed => warnings.continuity_check_failed, - HlsSegmentRepairWarningKind::MissingVps => warnings.missing_vps, - HlsSegmentRepairWarningKind::MissingSps => warnings.missing_sps, - HlsSegmentRepairWarningKind::MissingPps => warnings.missing_pps, - HlsSegmentRepairWarningKind::VpsOutOfRange => warnings.vps_out_of_range, - HlsSegmentRepairWarningKind::SpsOutOfRange => warnings.sps_out_of_range, - HlsSegmentRepairWarningKind::PpsOutOfRange => warnings.pps_out_of_range, - HlsSegmentRepairWarningKind::NoFrame => warnings.no_frame, - HlsSegmentRepairWarningKind::DecodeSliceHeaderError => warnings.decode_slice_header_error, - HlsSegmentRepairWarningKind::MissingPicture => warnings.missing_picture, - HlsSegmentRepairWarningKind::InvalidNal => warnings.invalid_nal, - HlsSegmentRepairWarningKind::InvalidData => warnings.invalid_data, - HlsSegmentRepairWarningKind::CodecParametersMissing => warnings.codec_parameters_missing, - HlsSegmentRepairWarningKind::MmcoUnrefShortFailure => warnings.mmco_unref_short_failure, - HlsSegmentRepairWarningKind::ReorderBufferIncrease => warnings.reorder_buffer, - HlsSegmentRepairWarningKind::PpsIdOutOfRange => warnings.pps_id_out_of_range, - HlsSegmentRepairWarningKind::NoStartCode => warnings.no_start_code, - HlsSegmentRepairWarningKind::NalSplitError => warnings.nal_split_error, - HlsSegmentRepairWarningKind::NalParseError => warnings.nal_parse_error, - HlsSegmentRepairWarningKind::InvalidVclNalu => { - warnings.invalid_undecodable_nalu_non_metadata.saturating_add(warnings.invalid_undecodable_nalu_keyframe) - } - HlsSegmentRepairWarningKind::InvalidMetadataNalu => warnings.invalid_undecodable_nalu_metadata, - HlsSegmentRepairWarningKind::MultipleDolbyVisionRpus => warnings.dolby_vision_rpu, - HlsSegmentRepairWarningKind::AvDesync => warnings.av_desync, - } -} - -fn has_any_warning(warnings: &WarningCounters, kinds: &[HlsSegmentRepairWarningKind]) -> bool { - kinds.iter().any(|kind| warning_count(warnings, *kind) > 0) -} - -fn common_mpegts_low_trigger(warnings: &WarningCounters) -> bool { - has_any_warning( - warnings, - &[ - HlsSegmentRepairWarningKind::MissingPat, - HlsSegmentRepairWarningKind::MissingPmt, - HlsSegmentRepairWarningKind::MissingPmtPid, - HlsSegmentRepairWarningKind::MissingVideoPidInPmt, - HlsSegmentRepairWarningKind::MissingAudioPidInPmt, - HlsSegmentRepairWarningKind::MultiplePrograms, - HlsSegmentRepairWarningKind::PesPacketSizeMismatch, - HlsSegmentRepairWarningKind::PacketCorrupt, - HlsSegmentRepairWarningKind::ContinuityCheckFailed, - HlsSegmentRepairWarningKind::CodecParametersMissing, - ], - ) -} - -fn h264_parameter_trigger(warnings: &WarningCounters) -> bool { - has_any_warning( - warnings, - &[ - HlsSegmentRepairWarningKind::MissingSps, - HlsSegmentRepairWarningKind::MissingPps, - HlsSegmentRepairWarningKind::SpsOutOfRange, - HlsSegmentRepairWarningKind::PpsOutOfRange, - ], - ) -} - -fn h264_medium_trigger(warnings: &WarningCounters) -> bool { - h264_parameter_trigger(warnings) - || warnings.decode_slice_header_error > 0 - || warnings.no_frame > 0 - || (warnings.missing_picture > 0 - && (warnings.missing_sps > 0 || warnings.missing_pps > 0 || warnings.decode_slice_header_error > 0)) - || (warnings.invalid_nal > 0 && h264_parameter_trigger(warnings)) - || (warnings.no_start_code > 0 && (warnings.missing_sps > 0 || warnings.missing_pps > 0)) - || (warnings.nal_split_error > 0 && (warnings.missing_sps > 0 || warnings.missing_pps > 0)) -} - -fn hevc_missing_parameter_trigger(warnings: &WarningCounters) -> bool { - warnings.missing_vps > 0 || warnings.missing_sps > 0 || warnings.missing_pps > 0 -} - -fn hevc_parameter_trigger(warnings: &WarningCounters) -> bool { - hevc_missing_parameter_trigger(warnings) - || warnings.vps_out_of_range > 0 - || warnings.sps_out_of_range > 0 - || warnings.pps_out_of_range > 0 - || warnings.pps_id_out_of_range > 0 -} - -fn hevc_invalid_vcl_nalu(warnings: &WarningCounters) -> bool { - warning_count(warnings, HlsSegmentRepairWarningKind::InvalidVclNalu) > 0 -} - -fn hevc_medium_trigger(warnings: &WarningCounters) -> bool { - hevc_parameter_trigger(warnings) - || (hevc_invalid_vcl_nalu(warnings) && hevc_parameter_trigger(warnings)) - || (warnings.nal_parse_error > 0 && hevc_parameter_trigger(warnings)) - || (warnings.invalid_nal > 0 && hevc_parameter_trigger(warnings)) - || (warnings.no_start_code > 0 && hevc_missing_parameter_trigger(warnings)) - || (warnings.nal_split_error > 0 && hevc_missing_parameter_trigger(warnings)) -} - -fn decide_repair(codec: RepairVideoCodec, warnings: &WarningCounters) -> HlsSegmentRepairDecision { - let common_low_trigger = common_mpegts_low_trigger(warnings); - let codec_parameters_missing = warnings.codec_parameters_missing > 0; - let codec_medium_trigger = match codec { - RepairVideoCodec::H264 => h264_medium_trigger(warnings), - RepairVideoCodec::Hevc => hevc_medium_trigger(warnings), - RepairVideoCodec::Unsupported => false, - }; - match codec { - RepairVideoCodec::H264 if codec_medium_trigger && (common_low_trigger || codec_parameters_missing) => { - HlsSegmentRepairDecision { - codec, - required_level: HlsSegmentRepairMode::High, - trigger_source: HlsSegmentRepairTriggerSource::H264High, - common_low_trigger, - codec_medium_trigger, - } - } - RepairVideoCodec::H264 if codec_medium_trigger => HlsSegmentRepairDecision { - codec, - required_level: HlsSegmentRepairMode::Medium, - trigger_source: HlsSegmentRepairTriggerSource::H264Medium, - common_low_trigger, - codec_medium_trigger, - }, - RepairVideoCodec::Hevc - if (common_low_trigger && (codec_medium_trigger || hevc_invalid_vcl_nalu(warnings))) - || (codec_parameters_missing && codec_medium_trigger) => - { - HlsSegmentRepairDecision { - codec, - required_level: HlsSegmentRepairMode::High, - trigger_source: HlsSegmentRepairTriggerSource::HevcHigh, - common_low_trigger, - codec_medium_trigger, - } - } - RepairVideoCodec::Hevc if codec_medium_trigger => HlsSegmentRepairDecision { - codec, - required_level: HlsSegmentRepairMode::Medium, - trigger_source: HlsSegmentRepairTriggerSource::HevcMedium, - common_low_trigger, - codec_medium_trigger, - }, - RepairVideoCodec::H264 | RepairVideoCodec::Hevc if common_low_trigger => HlsSegmentRepairDecision { - codec, - required_level: HlsSegmentRepairMode::Low, - trigger_source: HlsSegmentRepairTriggerSource::CommonMpegTsLow, - common_low_trigger, - codec_medium_trigger, - }, - RepairVideoCodec::Unsupported => HlsSegmentRepairDecision { - codec, - required_level: HlsSegmentRepairMode::Off, - trigger_source: HlsSegmentRepairTriggerSource::UnsupportedCodec, - common_low_trigger, - codec_medium_trigger, - }, - _ => HlsSegmentRepairDecision { - codec, - required_level: HlsSegmentRepairMode::Off, - trigger_source: HlsSegmentRepairTriggerSource::Off, - common_low_trigger, - codec_medium_trigger, - }, - } -} - -fn debug_repair_analysis( - context: &HlsSegmentRepairObjectContext, - configured_max_level: HlsSegmentRepairMode, - decision: HlsSegmentRepairDecision, - executed_level: Option, - warnings: &WarningCounters, -) { - let executed_level = executed_level.map_or("off", HlsSegmentRepairMode::as_log_value); - match decision.codec { - RepairVideoCodec::H264 => debug!( - "HLS segment repair analysis completed: {} source={} resource={} configured_max_level={} required_level={} executed_level={} codec={} trigger={} common_low={} codec_medium={} missing_sps={} missing_pps={} sps_out_of_range={} pps_out_of_range={} no_frame={} decode_slice_header_error={} missing_picture={} invalid_nal={} no_start_code={} nal_split_error={} packet_corrupt={} continuity_check_failed={} codec_parameters_missing={}", - context.log_identity_fields(), - context.source.as_log_value(), - context.resource_id, - configured_max_level.as_log_value(), - decision.required_level.as_log_value(), - executed_level, - decision.codec.as_log_value(), - decision.trigger_source.as_log_value(), - decision.common_low_trigger, - decision.codec_medium_trigger, - warnings.missing_sps, - warnings.missing_pps, - warnings.sps_out_of_range, - warnings.pps_out_of_range, - warnings.no_frame, - warnings.decode_slice_header_error, - warnings.missing_picture, - warnings.invalid_nal, - warnings.no_start_code, - warnings.nal_split_error, - warnings.packet_corrupt, - warnings.continuity_check_failed, - warnings.codec_parameters_missing - ), - RepairVideoCodec::Hevc => debug!( - "HLS segment repair analysis completed: {} source={} resource={} configured_max_level={} required_level={} executed_level={} codec={} trigger={} common_low={} codec_medium={} missing_vps={} missing_sps={} missing_pps={} vps_out_of_range={} sps_out_of_range={} pps_out_of_range={} pps_id_out_of_range={} invalid_vcl_nalu={} nal_parse_error={} invalid_nal={} no_start_code={} nal_split_error={} packet_corrupt={} continuity_check_failed={} codec_parameters_missing={}", - context.log_identity_fields(), - context.source.as_log_value(), - context.resource_id, - configured_max_level.as_log_value(), - decision.required_level.as_log_value(), - executed_level, - decision.codec.as_log_value(), - decision.trigger_source.as_log_value(), - decision.common_low_trigger, - decision.codec_medium_trigger, - warnings.missing_vps, - warnings.missing_sps, - warnings.missing_pps, - warnings.vps_out_of_range, - warnings.sps_out_of_range, - warnings.pps_out_of_range, - warnings.pps_id_out_of_range, - warning_count(warnings, HlsSegmentRepairWarningKind::InvalidVclNalu), - warnings.nal_parse_error, - warnings.invalid_nal, - warnings.no_start_code, - warnings.nal_split_error, - warnings.packet_corrupt, - warnings.continuity_check_failed, - warnings.codec_parameters_missing - ), - RepairVideoCodec::Unsupported => debug!( - "HLS segment repair analysis completed: {} source={} resource={} configured_max_level={} required_level={} executed_level={} codec={} trigger={}", - context.log_identity_fields(), - context.source.as_log_value(), - context.resource_id, - configured_max_level.as_log_value(), - decision.required_level.as_log_value(), - executed_level, - decision.codec.as_log_value(), - decision.trigger_source.as_log_value() - ), - } -} - -fn debug_repair_event( - context: &HlsSegmentRepairObjectContext, - mode: HlsSegmentRepairMode, - event: &'static str, - reason: Option<&str>, -) { - if let Some(reason) = reason { - debug!( - "HLS segment repair {event}: {} source={} resource={} mode={} reason={}", - context.log_identity_fields(), - context.source.as_log_value(), - context.resource_id, - mode.as_log_value(), - reason - ); - } else { - debug!( - "HLS segment repair {event}: {} source={} resource={} mode={}", - context.log_identity_fields(), - context.source.as_log_value(), - context.resource_id, - mode.as_log_value() - ); - } -} - -fn debug_repair_stream_dropped(context: &HlsSegmentRepairObjectContext, dropped: &RepairRemuxDroppedStream) { - debug!( - "HLS segment repair stream dropped: {} source={} resource={} stream={} reason={}", - context.log_identity_fields(), - context.source.as_log_value(), - context.resource_id, - dropped.index, - dropped.reason - ); -} - -async fn analyze_segment(path: &Path, deadline: &HlsPostProcessingDeadline) -> Result { - let probe_output = run_command_with_deadline( - "ffprobe", - &[ - "-hide_banner", - "-v", - "warning", - "-show_entries", - "format=duration,size,bit_rate", - "-show_streams", - "-of", - "json", - path.to_str().ok_or_else(|| "invalid_path".to_string())?, - ], - deadline, - ) - .await?; - let warnings_output = run_command_with_deadline( - "ffmpeg", - &[ - "-hide_banner", - "-nostdin", - "-v", - "warning", - "-i", - path.to_str().ok_or_else(|| "invalid_path".to_string())?, - "-map", - "0", - "-c", - "copy", - "-f", - "null", - "-", - ], - deadline, - ) - .await - .unwrap_or_else(|stderr| stderr); - parse_probe(&probe_output, parse_ffmpeg_warnings(&warnings_output)) -} - -async fn run_remux( - input_path: &Path, - output_path: &Path, - mode: HlsSegmentRepairMode, - stream_selection: &RepairRemuxStreamSelection, - deadline: &HlsPostProcessingDeadline, -) -> Result<(), String> { - let input = input_path.to_str().ok_or_else(|| "invalid_input_path".to_string())?; - let output = output_path.to_str().ok_or_else(|| "invalid_output_path".to_string())?; - let mut args = ["-hide_banner", "-nostdin", "-y", "-copyts", "-i", input] - .into_iter() - .map(ToOwned::to_owned) - .collect::>(); - for stream_index in &stream_selection.mapped_streams { - args.push("-map".to_string()); - args.push(format!("0:{stream_index}")); - } - args.extend(["-c", "copy"].into_iter().map(ToOwned::to_owned)); - if matches!(mode, HlsSegmentRepairMode::Medium | HlsSegmentRepairMode::High) { - args.push("-bsf:v".to_string()); - args.push("dump_extra=freq=keyframe".to_string()); - } - args.push("-mpegts_flags".to_string()); - args.push( - if mode == HlsSegmentRepairMode::High { "+resend_headers+pat_pmt_at_frames" } else { "+resend_headers" } - .to_string(), - ); - args.extend( - ["-mpegts_copyts", "1", "-muxpreload", "0", "-muxdelay", "0", "-f", "mpegts", output] - .into_iter() - .map(ToOwned::to_owned), - ); - let args = args.iter().map(String::as_str).collect::>(); - run_command_with_deadline("ffmpeg", &args, deadline).await.map(|_| ()) -} - -pub(super) async fn run_command_with_deadline( - binary: &str, - args: &[&str], - deadline: &HlsPostProcessingDeadline, -) -> Result { - let Some(remaining) = deadline.remaining() else { - return Err("timeout".to_string()); - }; - let output = timeout(remaining, { - let mut command = Command::new(binary); - command.args(args).stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::piped()).kill_on_drop(true); - command.output() - }) - .await - .map_err(|_| "timeout".to_string())? - .map_err( - |err| { - if err.kind() == io::ErrorKind::NotFound { - "unsupported".to_string() - } else { - err.to_string() - } - }, - )?; - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - if output.status.success() { - Ok(if stdout.is_empty() { stderr } else { stdout }) - } else { - Err(if stderr.is_empty() { "command_failed".to_string() } else { stderr }) - } -} - -fn parse_probe(json: &str, warnings: WarningCounters) -> Result { - let value = serde_json::from_str::(json).map_err(|_| "invalid_probe_json".to_string())?; - let streams = value.get("streams").and_then(Value::as_array).ok_or_else(|| "missing_streams".to_string())?; - let mut probe = SegmentProbe { stream_count: streams.len(), warnings, ..SegmentProbe::default() }; - if let Some(format) = value.get("format") { - probe.duration_ms = format.get("duration").and_then(Value::as_str).and_then(parse_seconds_ms_u64); - probe.size = - format.get("size").and_then(Value::as_str).and_then(|value| value.parse().ok()).unwrap_or_default(); - } - for stream in streams { - let codec_type = stream.get("codec_type").and_then(Value::as_str); - let codec_name = stream.get("codec_name").and_then(Value::as_str).map(ToOwned::to_owned); - let start_time_ms = stream.get("start_time").and_then(Value::as_str).and_then(parse_seconds_ms_i64); - let extradata_size = stream - .get("extradata_size") - .and_then(Value::as_u64) - .or_else(|| stream.get("extradata_size").and_then(Value::as_str).and_then(|value| value.parse().ok())); - let stream_index = - stream.get("index").and_then(parse_u32_value).map_or(probe.streams.len(), |value| value as usize); - let probe_stream = SegmentProbeStream { - index: stream_index, - stream_type: match codec_type { - Some("video") => SegmentProbeStreamType::Video, - Some("audio") => SegmentProbeStreamType::Audio, - _ => SegmentProbeStreamType::Other, - }, - codec_name: codec_name.clone(), - width: stream.get("width").and_then(parse_u32_value), - height: stream.get("height").and_then(parse_u32_value), - sample_rate: stream.get("sample_rate").and_then(parse_u32_value), - channels: stream.get("channels").and_then(parse_u32_value), - }; - match codec_type { - Some("video") if probe.primary_video_codec.is_none() => { - probe.primary_video_codec = codec_name; - probe.primary_video_start_time_ms = start_time_ms; - probe.primary_video_extradata_size = extradata_size; - } - Some("audio") if probe.primary_audio_codec.is_none() => { - probe.primary_audio_codec = codec_name; - probe.primary_audio_start_time_ms = start_time_ms; - } - _ => {} - } - probe.streams.push(probe_stream); - } - Ok(probe) -} - -fn parse_u32_value(value: &Value) -> Option { - value.as_u64().and_then(|value| u32::try_from(value).ok()).or_else(|| { - value - .as_str() - .map(str::trim) - .filter(|value| !value.is_empty() && *value != "N/A") - .and_then(|value| value.parse::().ok()) - }) -} - -fn detect_video_codec(probe: &SegmentProbe) -> RepairVideoCodec { - match probe.primary_video_codec.as_deref() { - Some("h264") => RepairVideoCodec::H264, - Some("hevc" | "h265") => RepairVideoCodec::Hevc, - _ => RepairVideoCodec::Unsupported, - } -} - -fn select_repair_remux_streams(probe: &SegmentProbe) -> Result { - let mut mapped_streams = Vec::new(); - let mut dropped_streams = Vec::new(); - let mut has_video = false; - for stream in &probe.streams { - match stream.stream_type { - SegmentProbeStreamType::Video if valid_video_stream(stream) => { - has_video = true; - mapped_streams.push(stream.index); - } - SegmentProbeStreamType::Audio if valid_audio_stream(stream) => { - mapped_streams.push(stream.index); - } - SegmentProbeStreamType::Video => dropped_streams - .push(RepairRemuxDroppedStream { index: stream.index, reason: "invalid-video-parameters" }), - SegmentProbeStreamType::Audio => dropped_streams - .push(RepairRemuxDroppedStream { index: stream.index, reason: "invalid-audio-parameters" }), - SegmentProbeStreamType::Other => dropped_streams - .push(RepairRemuxDroppedStream { index: stream.index, reason: "unsupported-stream-type" }), - } - } - if !has_video { - return Err("no_valid_video_stream".to_string()); - } - Ok(RepairRemuxStreamSelection { mapped_streams, dropped_streams }) -} - -fn valid_video_stream(stream: &SegmentProbeStream) -> bool { - stream.codec_name.as_deref().is_some_and(|codec| !codec.is_empty()) - && stream.width.unwrap_or_default() > 0 - && stream.height.unwrap_or_default() > 0 -} - -fn valid_audio_stream(stream: &SegmentProbeStream) -> bool { - stream.codec_name.as_deref().is_some_and(|codec| !codec.is_empty()) - && stream.sample_rate.unwrap_or_default() > 0 - && stream.channels.unwrap_or_default() > 0 -} - -fn validate_repair( - config: &HlsSegmentRepairConfig, - codec: RepairVideoCodec, - raw: &SegmentProbe, - fixed: &SegmentProbe, - executed_level: HlsSegmentRepairMode, - stream_selection: &RepairRemuxStreamSelection, -) -> Result<(), String> { - if codec == RepairVideoCodec::Unsupported { - return Err("unsupported_codec".to_string()); - } - let expected_stream_count = if stream_selection.dropped_streams.is_empty() { - raw.stream_count - } else { - stream_selection.mapped_streams.len() - }; - if expected_stream_count != fixed.stream_count { - return Err("stream_count_changed".to_string()); - } - if raw.primary_video_codec != fixed.primary_video_codec { - return Err("video_codec_changed".to_string()); - } - if raw.primary_audio_codec != fixed.primary_audio_codec { - return Err("audio_codec_changed".to_string()); - } - if delta_u64(raw.duration_ms, fixed.duration_ms) > 250 { - return Err("duration_delta_too_large".to_string()); - } - if delta_i64(raw.primary_video_start_time_ms, fixed.primary_video_start_time_ms) > 250 { - return Err("video_start_time_delta_too_large".to_string()); - } - if delta_i64(raw.primary_audio_start_time_ms, fixed.primary_audio_start_time_ms) > 250 { - return Err("audio_start_time_delta_too_large".to_string()); - } - let raw_decision = decide_repair(codec, &raw.warnings); - let fixed_decision = decide_repair(codec, &fixed.warnings); - if raw_decision.required_level != HlsSegmentRepairMode::Off - && fixed_decision.required_level != HlsSegmentRepairMode::Off - { - return Err("repair_triggers_remaining".to_string()); - } - if raw.size > 0 { - let allowed = - raw.size.saturating_add(raw.size.saturating_mul(size_increase_percent(config, executed_level)) / 100); - if fixed.size > allowed { - return Err("size_increase_too_large".to_string()); - } - } - Ok(()) -} - -fn size_increase_percent(config: &HlsSegmentRepairConfig, level: HlsSegmentRepairMode) -> u64 { - match level { - HlsSegmentRepairMode::Off => 0, - HlsSegmentRepairMode::Low => u64::from(config.size_increase.low_percent), - HlsSegmentRepairMode::Medium => u64::from(config.size_increase.medium_percent), - HlsSegmentRepairMode::High => u64::from(config.size_increase.high_percent), - } -} - -fn parse_seconds_ms_u64(value: &str) -> Option { - let (whole, frac) = value.split_once('.').unwrap_or((value, "")); - let whole_ms = whole.parse::().ok()?.checked_mul(1_000)?; - let frac_ms = frac.chars().take(3).collect::(); - let frac_ms = format!("{frac_ms:0<3}").parse::().ok()?; - Some(whole_ms.saturating_add(frac_ms)) -} - -fn parse_seconds_ms_i64(value: &str) -> Option { - let negative = value.starts_with('-'); - let unsigned = value.strip_prefix('-').unwrap_or(value); - let parsed = i64::try_from(parse_seconds_ms_u64(unsigned)?).ok()?; - Some(if negative { -parsed } else { parsed }) -} - -fn delta_u64(lhs: Option, rhs: Option) -> u64 { - match (lhs, rhs) { - (Some(lhs), Some(rhs)) => lhs.abs_diff(rhs), - (None, None) => 0, - _ => u64::MAX, - } -} - -fn delta_i64(lhs: Option, rhs: Option) -> u64 { - match (lhs, rhs) { - (Some(lhs), Some(rhs)) => lhs.abs_diff(rhs), - (None, None) => 0, - _ => u64::MAX, - } -} - -pub(super) async fn sha256_file(path: &Path) -> io::Result { - let mut file = File::open(path).await?; - let mut hasher = Sha256::new(); - let mut buffer = [0_u8; 16 * 1024]; - loop { - let read = file.read(&mut buffer).await?; - if read == 0 { - break; - } - hasher.update(&buffer[..read]); - } - let digest: [u8; 32] = hasher.finalize().into(); - let mut hex = String::with_capacity(64); - for byte in digest { - use std::fmt::Write as _; - let _ = write!(hex, "{byte:02x}"); - } - Ok(hex) -} - -pub(super) fn ffmpeg_identity_version() -> String { "system".to_string() } - -fn repair_output_path(raw_path: &Path) -> PathBuf { - let suffix = fastrand::u64(..); - let file_name = raw_path.file_name().and_then(|file_name| file_name.to_str()).unwrap_or("segment"); - raw_path.with_file_name(format!("{file_name}.repair.tmp.{suffix:016x}")) -} - -async fn cleanup_repair_output(path: &Path) { - match fs::remove_file(path).await { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => warn!("HLS repair fixed staging cleanup deferred: error_kind={:?}", error.kind()), - } -} - -async fn adopt_repair_output( - segment_cache: &HlsSegmentCache, - path: PathBuf, - size: u64, -) -> io::Result { - match segment_cache.adopt_staged_file(path.clone(), size) { - Ok(staged) => Ok(staged), - Err(error) => { - cleanup_repair_output(&path).await; - Err(error) - } - } -} - -impl fmt::Display for RepairStatus { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Clean => f.write_str("clean"), - Self::Fixed => f.write_str("fixed"), - Self::PolicyLimited => f.write_str("policy_limited"), - Self::Unsupported => f.write_str("unsupported"), - Self::Timeout => f.write_str("timeout"), - Self::RemuxFailed => f.write_str("remux_failed"), - Self::ValidationFailed => f.write_str("validation_failed"), - } - } -} - #[cfg(test)] -mod tests { - use super::{ - super::{ - media_reserve::{ - HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsManifestCommitIdentity, HlsManifestDeliveryMode, - }, - terminal_tail::HlsMediaContainer, - }, - adopt_repair_output, detect_video_codec, parse_ffmpeg_warnings, parse_probe, - ready_segment_repair_prewarm_candidates, repair_object_metadata_key, select_repair_remux_streams, sha256_file, - validate_repair, HlsRepairObjectMetadata, HlsRepairRenderedObjectId, HlsSegmentRepairManager, - HlsSegmentRepairObjectContext, HlsSegmentRepairSource, RepairIdentity, RepairRemuxStreamSelection, - RepairStatus, RepairVideoCodec, WarningCounters, REPAIR_METADATA_MAX_ENTRIES, - }; - use crate::{ - timeline::HLS_PROVISIONING_ORIGIN_EPOCH, CacheAccessState, HlsAccessLeaseId, HlsSegmentCache, - HlsSegmentEncryption, HlsSession, HlsSessionKey, OriginSegmentKey, ProxySessionId, SegmentCacheKey, - SegmentCacheStatus, SegmentEntry, TransientObjectCacheKey, TransientResourceId, - }; - use shared::model::HlsSegmentRepairMode; - use std::{sync::Arc, time::Duration}; - use tuliprox_core::model::HlsSegmentRepairConfig; - - fn repair_config(mode: HlsSegmentRepairMode, apply_to_first_segments: u8) -> HlsSegmentRepairConfig { - HlsSegmentRepairConfig { - max_level: mode, - apply_to_first_segments, - max_parallel_repairs: 1, - ..Default::default() - } - } - - fn should_repair(codec: RepairVideoCodec, warnings: &WarningCounters) -> bool { - super::decide_repair(codec, warnings).required_level != HlsSegmentRepairMode::Off - } - - fn repair_context(lease_id: &str, resource_id: &str) -> HlsSegmentRepairObjectContext { - HlsSegmentRepairObjectContext { - source: HlsSegmentRepairSource::Normal, - log_identity: super::HlsLogIdentity::for_test("input:1|hls|stream-a", "proxy-session"), - proxy_session_id: ProxySessionId("proxy-session".to_string()), - hls_access_lease_id: Some(HlsAccessLeaseId(lease_id.to_string())), - rendered_object_id: HlsRepairRenderedObjectId::Normal { proxy_seq: resource_id.parse().unwrap_or(1) }, - resource_id: resource_id.to_string(), - file_ext: "ts".to_string(), - origin_fetch_uri_for_diagnostics: format!("http://origin.example/{resource_id}.ts"), - media_sequence: Some(1), - discontinuity_sequence: Some(0), - complete_object: true, - encrypted: false, - custom_response: false, - } - } - - #[test] - fn repair_log_identity_fields_separate_content_proxy_and_lease_identities() { - let context = repair_context("lease-a", "000001"); - let fields = context.log_identity_fields(); - - assert_eq!( - fields, - format!( - "session={} proxy_session={} lease={}", - context.log_identity.session(), - context.log_identity.proxy_session(), - super::safe_hls_access_lease_id(context.hls_access_lease_id.as_ref().expect("test lease")) - ) - ); - assert_ne!(context.log_identity.session(), context.log_identity.proxy_session()); - } - - #[tokio::test] - async fn failed_repair_output_adoption_removes_the_unowned_file() { - let cache_root = tempfile::tempdir().expect("cache root"); - let outside = tempfile::tempdir().expect("outside root"); - let output = outside.path().join("fixed.ts"); - tokio::fs::write(&output, b"fixed").await.expect("write output"); - let cache = HlsSegmentCache::with_cache_path(cache_root.path()); - - assert!(adopt_repair_output(&cache, output.clone(), 5).await.is_err()); - assert!(!output.exists()); - } - - #[test] - fn repair_prewarm_candidates_include_only_ready_clear_origin_ts_media() { - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let mut session = HlsSession::new(HlsSessionKey::new(1, "prewarm-candidates"), b"secret", 0); - let proxy_seq = 10; - let cache_key = SegmentCacheKey::new(session.proxy_session_id.clone(), proxy_seq, "ts"); - session.segments.insert( - proxy_seq, - SegmentEntry { - origin_key: OriginSegmentKey { - origin_epoch: 1, - effective_host_id: 1, - host_local_sequence: 100, - host_local_index: 0, - }, - proxy_seq, - duration_ms: 6_000, - proxy_file_ext: "ts".to_string(), - content_type: "video/mp2t".to_string(), - cache_key, - discontinuity_before: false, - program_date_time: None, - daterange_tags_before: Vec::new(), - origin_byte_range: None, - map_ref: None, - encryption: None, - origin_fetch_ref: None, - status: SegmentCacheStatus::Ready { content_length: 1_024, ready_at_ms: 1 }, - last_rendered_at_ms: Some(1), - access: Arc::new(CacheAccessState::new()), - }, - ); - let snapshot = HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(1), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 1, - delivered_at_ms: 1, - first_proxy_seq: proxy_seq, - last_proxy_seq: proxy_seq, - visible_segments: Arc::from([HlsLeaseManifestSegment { - proxy_seq, - duration_ms: 6_000, - uri: "/live/10.ts".to_string().into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }]), - discontinuity_sequence: 0, - target_duration_ms: 6_000, - playlist_duration_ms: 6_000, - last_visible_media_end_ms: 6_000, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - }; - - assert_eq!(ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, 1).len(), 1); - assert!(ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, 0).is_empty()); - - let entry = session.segments.get_mut(&proxy_seq).expect("candidate segment"); - entry.proxy_file_ext = "m4s".to_string(); - assert!(ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, 1).is_empty()); - let entry = session.segments.get_mut(&proxy_seq).expect("candidate segment"); - entry.proxy_file_ext = "ts".to_string(); - entry.encryption = Some(HlsSegmentEncryption { - resource_id: TransientResourceId("key-a".to_string()), - resource_extension: "key".to_string(), - iv: None, - key_format: None, - key_format_versions: None, - }); - assert!(ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, 1).is_empty()); - let entry = session.segments.get_mut(&proxy_seq).expect("candidate segment"); - entry.encryption = None; - entry.status = SegmentCacheStatus::Discovered; - assert!(ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, 1).is_empty()); - let entry = session.segments.get_mut(&proxy_seq).expect("candidate segment"); - entry.status = SegmentCacheStatus::Ready { content_length: 1_024, ready_at_ms: 1 }; - entry.origin_key.origin_epoch = HLS_PROVISIONING_ORIGIN_EPOCH; - assert!(ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, 1).is_empty()); - } - - async fn selected_repair_mode( - manager: &HlsSegmentRepairManager, - context: &HlsSegmentRepairObjectContext, - ) -> Option { - manager.try_select_candidate(context).await.map(|(mode, _, _)| mode) - } - - #[tokio::test] - async fn update_config_applies_to_new_access_lease_windows() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Medium, 2)); - let mut updated = repair_config(HlsSegmentRepairMode::Medium, 3); - updated.max_parallel_repairs = 2; - manager.update_config(updated); - - manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; - - for resource_id in ["1", "2", "3"] { - assert!( - manager.try_select_candidate(&repair_context("lease-a", resource_id)).await.is_some(), - "resource {resource_id} should be inside updated repair window" - ); - } - assert!( - manager.try_select_candidate(&repair_context("lease-a", "4")).await.is_none(), - "fourth resource should be outside updated repair window" - ); - } - - #[test] - fn warning_parser_expands_repeated_messages() { - let warnings = parse_ffmpeg_warnings( - "non-existing SPS 0 referenced in buffering period\nLast message repeated 2 times\nno frame!\n", - ); - - assert_eq!(warnings.missing_sps, 3); - assert_eq!(warnings.no_frame, 1); - } - - #[test] - fn mmco_warning_alone_does_not_trigger_repair() { - let warnings = WarningCounters { mmco_unref_short_failure: 20, ..WarningCounters::default() }; - - assert!(!should_repair(RepairVideoCodec::H264, &warnings)); - } - - #[test] - fn critical_warning_triggers_repair() { - let warnings = WarningCounters { missing_sps: 1, ..WarningCounters::default() }; - - assert!(should_repair(RepairVideoCodec::H264, &warnings)); - } - - #[test] - fn hevc_pps_warning_triggers_repair() { - let warnings = parse_ffmpeg_warnings("[hevc @ 0x1] PPS id out of range: 0\n"); - - assert_eq!(warnings.pps_id_out_of_range, 1); - assert!(should_repair(RepairVideoCodec::Hevc, &warnings)); - } - - #[test] - fn hevc_invalid_slice_nalus_with_parameter_issue_trigger_repair() { - let warnings = parse_ffmpeg_warnings( - "[hevc @ 0x1] missing SPS\n[hevc @ 0x1] Skipping invalid undecodable NALU: 0\n[hevc @ 0x1] Skipping invalid undecodable NALU: 1\n", - ); - - assert_eq!(warnings.invalid_undecodable_nalu_total, 2); - assert_eq!(warnings.invalid_undecodable_nalu_non_metadata, 2); - assert!(should_repair(RepairVideoCodec::Hevc, &warnings)); - } - - #[test] - fn hevc_keyframe_nalu_counts_as_vcl_but_does_not_trigger_alone() { - let warnings = parse_ffmpeg_warnings("[hevc @ 0x1] Skipping invalid undecodable NALU: 21\n"); - - assert_eq!(warnings.invalid_undecodable_nalu_total, 1); - assert_eq!(warnings.invalid_undecodable_nalu_keyframe, 1); - assert!(!should_repair(RepairVideoCodec::Hevc, &warnings)); - } - - #[test] - fn hevc_metadata_and_dolby_warnings_do_not_trigger_repair_alone() { - let warnings = parse_ffmpeg_warnings( - "[hevc @ 0x1] Skipping invalid undecodable NALU: 39\nMultiple Dolby Vision RPUs found in one AU. Skipping previous.\nAudio/Video desynchronisation detected!\n", - ); - - assert_eq!(warnings.invalid_undecodable_nalu_total, 1); - assert_eq!(warnings.invalid_undecodable_nalu_metadata, 1); - assert_eq!(warnings.dolby_vision_rpu, 1); - assert_eq!(warnings.av_desync, 1); - assert!(!should_repair(RepairVideoCodec::Hevc, &warnings)); - } - - #[test] - fn hevc_repeated_messages_expand_counters_without_triggering_alone() { - let warnings = - parse_ffmpeg_warnings("[hevc @ 0x1] Skipping invalid undecodable NALU: 0\nLast message repeated 2 times\n"); - - assert_eq!(warnings.invalid_undecodable_nalu_total, 3); - assert_eq!(warnings.invalid_undecodable_nalu_non_metadata, 3); - assert!(!should_repair(RepairVideoCodec::Hevc, &warnings)); - } - - #[test] - fn hevc_invalid_undecodable_nalu_0_to_31_counts_as_vcl_trigger_input() { - let warnings = - parse_ffmpeg_warnings("[hevc @ 0x1] missing SPS\n[hevc @ 0x1] Skipping invalid undecodable NALU: 30\n"); - - assert_eq!(warnings.invalid_undecodable_nalu_total, 1); - assert_eq!(warnings.invalid_undecodable_nalu_non_metadata, 1); - assert!(should_repair(RepairVideoCodec::Hevc, &warnings)); - } - - #[test] - fn warning_parser_matches_trigger_patterns_case_insensitively() { - let warnings = parse_ffmpeg_warnings( - "NON-EXISTING SPS 0 referenced\ninvalid nal unit 1\ncould not find codec parameters for stream 0\n", - ); - - assert_eq!(warnings.missing_sps, 1); - assert_eq!(warnings.invalid_nal, 1); - assert_eq!(warnings.codec_parameters_missing, 1); - assert!(should_repair(RepairVideoCodec::H264, &warnings)); - } - - #[test] - fn unsupported_codec_never_triggers_repair() { - let warnings = WarningCounters { missing_sps: 1, pps_id_out_of_range: 1, ..WarningCounters::default() }; - - assert!(!should_repair(RepairVideoCodec::Unsupported, &warnings)); - } - - #[test] - fn parse_probe_detects_hevc_codec_and_extradata() { - let probe = parse_probe( - r#"{ - "streams": [ - { - "index": 0, - "codec_type": "video", - "codec_name": "hevc", - "start_time": "1.250000", - "extradata_size": 96 - } - ], - "format": { "duration": "2.000000", "size": "1000" } - }"#, - WarningCounters::default(), - ) - .expect("probe should parse"); - - assert_eq!(detect_video_codec(&probe), RepairVideoCodec::Hevc); - assert_eq!(probe.primary_video_extradata_size, Some(96)); - } - - #[test] - fn repair_remux_selection_drops_invalid_audio_side_stream() { - let probe = parse_probe( - r#"{ - "streams": [ - { "index": 0, "codec_type": "video", "codec_name": "hevc", "width": 1916, "height": 1080 }, - { "index": 1, "codec_type": "audio", "codec_name": "ac3", "sample_rate": "48000", "channels": 6 }, - { "index": 2, "codec_type": "audio", "codec_name": "ac3", "channels": 0 } - ], - "format": { "duration": "2.000000", "size": "1000" } - }"#, - WarningCounters::default(), - ) - .expect("probe should parse"); - - let selection = select_repair_remux_streams(&probe).expect("valid video should allow remux"); - - assert_eq!(selection.mapped_streams, vec![0, 1]); - assert_eq!(selection.dropped_streams.len(), 1); - assert_eq!(selection.dropped_streams[0].index, 2); - assert_eq!(selection.dropped_streams[0].reason, "invalid-audio-parameters"); - } - - #[test] - fn repair_validation_allows_configured_stream_drop() { - let raw = parse_probe( - r#"{ - "streams": [ - { "index": 0, "codec_type": "video", "codec_name": "hevc", "width": 1916, "height": 1080, "start_time": "0.000000" }, - { "index": 1, "codec_type": "audio", "codec_name": "ac3", "sample_rate": "48000", "channels": 6, "start_time": "0.000000" }, - { "index": 2, "codec_type": "audio", "codec_name": "ac3", "channels": 0, "start_time": "0.000000" } - ], - "format": { "duration": "2.000000", "size": "1000" } - }"#, - WarningCounters { codec_parameters_missing: 1, ..WarningCounters::default() }, - ) - .expect("raw probe should parse"); - let fixed = parse_probe( - r#"{ - "streams": [ - { "index": 0, "codec_type": "video", "codec_name": "hevc", "width": 1916, "height": 1080, "start_time": "0.000000" }, - { "index": 1, "codec_type": "audio", "codec_name": "ac3", "sample_rate": "48000", "channels": 6, "start_time": "0.000000" } - ], - "format": { "duration": "2.000000", "size": "1000" } - }"#, - WarningCounters::default(), - ) - .expect("fixed probe should parse"); - let selection = select_repair_remux_streams(&raw).expect("raw should select valid streams"); - - assert!(validate_repair( - &repair_config(HlsSegmentRepairMode::Low, 1), - RepairVideoCodec::Hevc, - &raw, - &fixed, - HlsSegmentRepairMode::Low, - &selection - ) - .is_ok()); - } - - #[test] - fn repair_remux_selection_rejects_without_valid_video() { - let probe = parse_probe( - r#"{ - "streams": [ - { "index": 0, "codec_type": "video", "codec_name": "hevc", "height": 1080 }, - { "index": 1, "codec_type": "audio", "codec_name": "ac3", "sample_rate": "48000", "channels": 6 } - ], - "format": { "duration": "2.000000", "size": "1000" } - }"#, - WarningCounters::default(), - ) - .expect("probe should parse"); - - let err = select_repair_remux_streams(&probe).expect_err("missing width should reject video"); - - assert_eq!(err, "no_valid_video_stream"); - } - - #[test] - fn hevc_validation_accepts_when_repair_triggers_are_removed() { - let raw = parse_probe( - r#"{ - "streams": [ - { "codec_type": "video", "codec_name": "hevc", "start_time": "0.000000" }, - { "codec_type": "audio", "codec_name": "aac", "start_time": "0.000000" } - ], - "format": { "duration": "2.000000", "size": "1000" } - }"#, - WarningCounters { - pps_id_out_of_range: 1, - invalid_undecodable_nalu_non_metadata: 2, - invalid_undecodable_nalu_metadata: 1, - dolby_vision_rpu: 1, - ..WarningCounters::default() - }, - ) - .expect("raw probe should parse"); - let fixed = parse_probe( - r#"{ - "streams": [ - { "codec_type": "video", "codec_name": "hevc", "start_time": "0.000000" }, - { "codec_type": "audio", "codec_name": "aac", "start_time": "0.000000" } - ], - "format": { "duration": "2.000000", "size": "1010" } - }"#, - WarningCounters { invalid_undecodable_nalu_metadata: 1, dolby_vision_rpu: 1, ..WarningCounters::default() }, - ) - .expect("fixed probe should parse"); - - assert!(validate_repair( - &repair_config(HlsSegmentRepairMode::Medium, 1), - RepairVideoCodec::Hevc, - &raw, - &fixed, - HlsSegmentRepairMode::Medium, - &RepairRemuxStreamSelection::preserve_all(&raw) - ) - .is_ok()); - } - - #[test] - fn validation_rejects_remaining_repair_triggers_even_when_level_improves() { - let raw = parse_probe( - r#"{ - "streams": [ - { "codec_type": "video", "codec_name": "hevc", "start_time": "0.000000" }, - { "codec_type": "audio", "codec_name": "aac", "start_time": "0.000000" } - ], - "format": { "duration": "2.000000", "size": "1000" } - }"#, - WarningCounters { codec_parameters_missing: 1, missing_sps: 1, ..WarningCounters::default() }, - ) - .expect("raw probe should parse"); - let fixed = parse_probe( - r#"{ - "streams": [ - { "codec_type": "video", "codec_name": "hevc", "start_time": "0.000000" }, - { "codec_type": "audio", "codec_name": "aac", "start_time": "0.000000" } - ], - "format": { "duration": "2.000000", "size": "1000" } - }"#, - WarningCounters { missing_sps: 1, ..WarningCounters::default() }, - ) - .expect("fixed probe should parse"); - - let err = validate_repair( - &repair_config(HlsSegmentRepairMode::High, 1), - RepairVideoCodec::Hevc, - &raw, - &fixed, - HlsSegmentRepairMode::High, - &RepairRemuxStreamSelection::preserve_all(&raw), - ) - .expect_err("remaining medium trigger should fail validation"); - assert_eq!(err, "repair_triggers_remaining"); - } - - #[test] - fn configured_max_below_required_skips_repair() { - assert_eq!( - HlsSegmentRepairMode::Low.execution_plan(HlsSegmentRepairMode::High), - super::HlsSegmentRepairExecutionPlan::SkipConfiguredMaxBelowRequired - ); - assert_eq!( - HlsSegmentRepairMode::Medium.execution_plan(HlsSegmentRepairMode::Low), - super::HlsSegmentRepairExecutionPlan::Repair(HlsSegmentRepairMode::Low) - ); - assert_eq!( - HlsSegmentRepairMode::Off.execution_plan(HlsSegmentRepairMode::High), - super::HlsSegmentRepairExecutionPlan::SkipNoTrigger - ); - } - - #[tokio::test] - async fn repair_window_selects_first_unique_segments_per_access_lease() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; - let first = repair_context("lease-a", "000001"); - let second = repair_context("lease-a", "000002"); - - assert_eq!(selected_repair_mode(&manager, &first).await, Some(HlsSegmentRepairMode::Low)); - assert_eq!(selected_repair_mode(&manager, &first).await, None); - assert_eq!(selected_repair_mode(&manager, &second).await, None); - } - - #[tokio::test] - async fn background_candidate_without_access_lease_is_ignored_before_window_check() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; - let mut context = repair_context("lease-a", "000001"); - context.hls_access_lease_id = None; - - assert_eq!(selected_repair_mode(&manager, &context).await, None); - - let stats = manager.stats().await; - assert_eq!(stats.windows, 1); - assert_eq!(stats.checked_candidates, 0); - assert_eq!(stats.object_metadata, 0); - } - - #[tokio::test] - async fn repair_window_is_separate_per_access_lease() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Medium, 1)); - manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; - manager.start_access_lease_window(HlsAccessLeaseId("lease-b".to_string())).await; - - assert_eq!( - selected_repair_mode(&manager, &repair_context("lease-a", "000001")).await, - Some(HlsSegmentRepairMode::Medium) - ); - assert_eq!( - selected_repair_mode(&manager, &repair_context("lease-b", "000001")).await, - Some(HlsSegmentRepairMode::Medium) - ); - } - - #[tokio::test] - async fn known_normal_object_consumes_new_lease_window_without_rescan() { - let temp_dir = tempfile::tempdir().expect("temp dir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - let cache_key = SegmentCacheKey::new(ProxySessionId("proxy-session".to_string()), 1, "ts"); - let metadata = cache.write_bytes_and_commit(&cache_key, b"cached-normal-bytes").await.expect("commit"); - let committed_sha256 = sha256_file(&metadata.path).await.expect("hash"); - let previous_context = repair_context("lease-a", "1"); - manager - .record_object_metadata( - repair_object_metadata_key(&previous_context, HlsSegmentRepairMode::Low), - HlsRepairObjectMetadata { - committed_sha256, - raw_sha256: Some("previous-raw".to_string()), - status: RepairStatus::Clean, - raw_size: metadata.size, - final_size: metadata.size, - validation_reason: None, - }, - ) - .await; - - manager.start_access_lease_window(HlsAccessLeaseId("lease-b".to_string())).await; - - assert!(manager - .repair_ready_cache_hit(&cache, &cache_key, repair_context("lease-b", "1")) - .await - .expect("repair cache hit") - .is_none()); - assert_eq!(manager.stats().await.metadata, 0); - assert_eq!(selected_repair_mode(&manager, &repair_context("lease-b", "2")).await, None); - } - - #[tokio::test] - async fn known_transient_object_consumes_new_lease_window_without_rescan() { - let temp_dir = tempfile::tempdir().expect("temp dir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - let cache_key = TransientObjectCacheKey::new( - ProxySessionId("proxy-session".to_string()), - TransientResourceId("resource-a".to_string()), - "ts", - ); - let metadata = cache.write_bytes_and_commit(&cache_key, b"cached-transient-bytes").await.expect("commit"); - let committed_sha256 = sha256_file(&metadata.path).await.expect("hash"); - let mut previous_context = repair_context("lease-a", "1"); - previous_context.source = HlsSegmentRepairSource::Transient; - previous_context.rendered_object_id = - HlsRepairRenderedObjectId::Transient { resource_id: "resource-a".to_string() }; - previous_context.resource_id = "resource-a".to_string(); - manager - .record_object_metadata( - repair_object_metadata_key(&previous_context, HlsSegmentRepairMode::Low), - HlsRepairObjectMetadata { - committed_sha256, - raw_sha256: Some("previous-raw".to_string()), - status: RepairStatus::Clean, - raw_size: metadata.size, - final_size: metadata.size, - validation_reason: None, - }, - ) - .await; - - manager.start_access_lease_window(HlsAccessLeaseId("lease-b".to_string())).await; - let mut current_context = previous_context.clone(); - current_context.hls_access_lease_id = Some(HlsAccessLeaseId("lease-b".to_string())); - - assert!(manager - .repair_ready_cache_hit(&cache, &cache_key, current_context) - .await - .expect("repair cache hit") - .is_none()); - assert_eq!(manager.stats().await.metadata, 0); - let mut second_context = repair_context("lease-b", "2"); - second_context.source = HlsSegmentRepairSource::Transient; - second_context.rendered_object_id = - HlsRepairRenderedObjectId::Transient { resource_id: "resource-b".to_string() }; - second_context.resource_id = "resource-b".to_string(); - assert_eq!(selected_repair_mode(&manager, &second_context).await, None); - } - - #[tokio::test] - async fn object_metadata_hash_mismatch_does_not_skip_repair_evaluation() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - let context = repair_context("lease-a", "1"); - let object_key = repair_object_metadata_key(&context, HlsSegmentRepairMode::Low); - manager - .record_object_metadata( - object_key.clone(), - HlsRepairObjectMetadata { - committed_sha256: "old-hash".to_string(), - raw_sha256: Some("old-raw".to_string()), - status: RepairStatus::Clean, - raw_size: 1, - final_size: 1, - validation_reason: None, - }, - ) - .await; - - assert!(!manager.object_metadata_matches(&object_key, "new-hash").await); - } - - #[test] - fn repair_object_metadata_key_ignores_origin_fetch_uri_for_diagnostics() { - let mut first = repair_context("lease-a", "1"); - first.origin_fetch_uri_for_diagnostics = "http://mirror-a.example/live/1.ts".to_string(); - let mut second = first.clone(); - second.origin_fetch_uri_for_diagnostics = "http://redirect-b.example/cdn/path/1.ts".to_string(); - - assert_eq!( - repair_object_metadata_key(&first, HlsSegmentRepairMode::Low), - repair_object_metadata_key(&second, HlsSegmentRepairMode::Low) - ); - } - - #[tokio::test] - async fn repair_window_candidate_key_ignores_origin_fetch_uri_for_diagnostics() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 2)); - manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; - let mut first = repair_context("lease-a", "1"); - first.origin_fetch_uri_for_diagnostics = "http://mirror-a.example/live/1.ts".to_string(); - let mut same_rendered_object_other_fetch_uri = first.clone(); - same_rendered_object_other_fetch_uri.origin_fetch_uri_for_diagnostics = - "http://redirect-b.example/cdn/path/1.ts".to_string(); - let second_rendered_object = repair_context("lease-a", "2"); - - assert_eq!(selected_repair_mode(&manager, &first).await, Some(HlsSegmentRepairMode::Low)); - assert_eq!(selected_repair_mode(&manager, &same_rendered_object_other_fetch_uri).await, None); - assert_eq!(selected_repair_mode(&manager, &second_rendered_object).await, Some(HlsSegmentRepairMode::Low)); - } - - #[tokio::test] - async fn repair_disabled_does_not_track_candidates() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Off, 1)); - let context = repair_context("lease-a", "000001"); - - assert_eq!(selected_repair_mode(&manager, &context).await, None); - - let registry = manager.windows.read().await; - assert_eq!(registry.checked_candidates.len(), 0); - } - - #[tokio::test] - async fn transient_commit_and_cache_hit_consume_repair_window_once() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 2)); - manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; - let mut commit_context = repair_context("lease-a", "1"); - commit_context.source = HlsSegmentRepairSource::Transient; - commit_context.rendered_object_id = - HlsRepairRenderedObjectId::Transient { resource_id: "resource-a".to_string() }; - commit_context.resource_id = "resource-a".to_string(); - commit_context.origin_fetch_uri_for_diagnostics = "http://origin.example/live/resource-a.ts".to_string(); - commit_context.media_sequence = None; - commit_context.discontinuity_sequence = None; - - let mut cache_hit_context = commit_context.clone(); - cache_hit_context.origin_fetch_uri_for_diagnostics = "resource-a".to_string(); - - let mut second_context = commit_context.clone(); - second_context.rendered_object_id = - HlsRepairRenderedObjectId::Transient { resource_id: "resource-b".to_string() }; - second_context.resource_id = "resource-b".to_string(); - second_context.origin_fetch_uri_for_diagnostics = "resource-b".to_string(); - - assert_eq!(selected_repair_mode(&manager, &commit_context).await, Some(HlsSegmentRepairMode::Low)); - assert_eq!(selected_repair_mode(&manager, &cache_hit_context).await, None); - assert_eq!(selected_repair_mode(&manager, &second_context).await, Some(HlsSegmentRepairMode::Low)); - } - - #[tokio::test] - async fn normal_commit_and_cache_hit_consume_repair_window_once() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 2)); - manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; - let commit_context = repair_context("lease-a", "1"); - let mut cache_hit_context = commit_context.clone(); - cache_hit_context.origin_fetch_uri_for_diagnostics = "http://redirect.example/other-path.ts".to_string(); - cache_hit_context.media_sequence = Some(99); - cache_hit_context.discontinuity_sequence = Some(7); - let second_context = repair_context("lease-a", "2"); - - assert_eq!(selected_repair_mode(&manager, &commit_context).await, Some(HlsSegmentRepairMode::Low)); - assert_eq!(selected_repair_mode(&manager, &cache_hit_context).await, None); - assert_eq!(selected_repair_mode(&manager, &second_context).await, Some(HlsSegmentRepairMode::Low)); - } - - #[tokio::test] - async fn non_repairable_objects_do_not_consume_repair_window() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; - let mut partial = repair_context("lease-a", "1"); - partial.complete_object = false; - let repairable = repair_context("lease-a", "2"); - - assert_eq!(selected_repair_mode(&manager, &partial).await, None); - assert_eq!(selected_repair_mode(&manager, &repairable).await, Some(HlsSegmentRepairMode::Low)); - } - - #[tokio::test] - async fn new_repair_window_generation_allows_rechecking_candidate() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let context = repair_context("lease-a", "000001"); - - manager.start_access_lease_window(lease_id.clone()).await; - assert_eq!(selected_repair_mode(&manager, &context).await, Some(HlsSegmentRepairMode::Low)); - assert_eq!(manager.windows.read().await.checked_candidates.len(), 1); - - manager.start_access_lease_window(lease_id).await; - - assert_eq!(selected_repair_mode(&manager, &context).await, Some(HlsSegmentRepairMode::Low)); - assert_eq!(manager.windows.read().await.checked_candidates.len(), 2); - } - - #[tokio::test] - async fn ensuring_pending_repair_window_does_not_reset_consumed_candidate_budget() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - manager.ensure_access_lease_window(lease_id.clone()).await; - - assert_eq!( - selected_repair_mode(&manager, &repair_context("lease-a", "1")).await, - Some(HlsSegmentRepairMode::Low) - ); - manager.ensure_access_lease_window(lease_id).await; - - assert_eq!(selected_repair_mode(&manager, &repair_context("lease-a", "2")).await, None); - assert_eq!(manager.stats().await.generations, 1); - } - - #[tokio::test] - async fn on_demand_selection_joins_the_prewarm_candidate_identity() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - manager.ensure_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; - let context = repair_context("lease-a", "1"); - - let (prewarm_mode, _, prewarm_candidate) = - manager.try_select_candidate(&context).await.expect("prewarm candidate"); - let (demand_mode, _, demand_candidate) = - manager.try_select_or_join_candidate(&context).await.expect("demand joins prewarm"); - - assert_eq!(demand_mode, prewarm_mode); - assert_eq!(demand_candidate, prewarm_candidate); - assert_eq!( - manager.windows.read().await.windows[&HlsAccessLeaseId("lease-a".to_string())].remaining_segments, - 0 - ); - } - - #[tokio::test] - async fn exhausted_window_candidate_is_never_remembered_as_joinable() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - manager.ensure_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; - assert!(manager.try_select_candidate(&repair_context("lease-a", "1")).await.is_some()); - let skipped = repair_context("lease-a", "2"); - - assert!(manager.try_select_or_join_candidate(&skipped).await.is_none()); - assert!(manager.try_select_or_join_candidate(&skipped).await.is_none()); - assert_eq!(manager.stats().await.checked_candidates, 1); - } - - #[tokio::test] - async fn prewarmed_ready_object_reuses_metadata_on_later_demand() { - const CLEAN_TS: &[u8] = - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); - let temp_dir = tempfile::tempdir().expect("temp dir"); - let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let manager = Arc::new(HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1))); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let cache_key = SegmentCacheKey::new(ProxySessionId("proxy-session".to_string()), 1, "ts"); - cache.write_bytes_and_commit(&cache_key, CLEAN_TS).await.expect("cache fixture"); - manager.ensure_access_lease_window(lease_id).await; - let context = repair_context("lease-a", "1"); - - manager - .spawn_ready_cache_prewarm_inner(Arc::clone(&cache), vec![(cache_key.clone(), context.clone())], None) - .await; - tokio::time::timeout(Duration::from_secs(10), async { - loop { - if manager.stats().await.object_metadata == 1 { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("prewarm completes"); - let after_prewarm = manager.stats().await; - assert_eq!(after_prewarm.object_metadata, 1); - - assert!(manager - .repair_ready_cache_hit(&cache, &cache_key, context) - .await - .expect("demand reuses prewarm") - .is_none()); - assert_eq!(manager.stats().await, after_prewarm); - } - - #[tokio::test] - async fn remove_access_lease_window_clears_window_and_generation() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - manager.start_access_lease_window(lease_id.clone()).await; - - let before = manager.stats().await; - assert_eq!(before.windows, 1); - assert_eq!(before.generations, 1); - - manager.remove_access_lease_window(&lease_id).await; - - let after = manager.stats().await; - assert_eq!(after.windows, 0); - assert_eq!(after.generations, 0); - } - - #[tokio::test] - async fn remove_access_lease_window_clears_checked_candidates() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - - manager.start_access_lease_window(lease_id.clone()).await; - assert_eq!( - selected_repair_mode(&manager, &repair_context("lease-a", "000001")).await, - Some(HlsSegmentRepairMode::Low) - ); - assert_eq!(manager.windows.read().await.checked_candidates.len(), 1); - - manager.remove_access_lease_window(&lease_id).await; - - assert_eq!(manager.windows.read().await.checked_candidates.len(), 0); - } - - #[tokio::test] - async fn remove_proxy_session_state_clears_checked_candidates() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - manager.start_access_lease_window(lease_id.clone()).await; - let context = repair_context("lease-a", "000001"); - - assert_eq!(selected_repair_mode(&manager, &context).await, Some(HlsSegmentRepairMode::Low)); - assert_eq!(manager.windows.read().await.checked_candidates.len(), 1); - - manager.remove_proxy_session_state(&proxy_session_id, &[lease_id]).await; - - assert_eq!(manager.windows.read().await.checked_candidates.len(), 0); - } - - #[tokio::test] - async fn remove_proxy_session_state_clears_object_metadata() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - let lease_id = HlsAccessLeaseId("lease-a".to_string()); - let context = repair_context("lease-a", "000001"); - manager - .record_object_metadata( - repair_object_metadata_key(&context, HlsSegmentRepairMode::Low), - HlsRepairObjectMetadata { - committed_sha256: "hash".to_string(), - raw_sha256: Some("raw".to_string()), - status: RepairStatus::Clean, - raw_size: 1, - final_size: 1, - validation_reason: None, - }, - ) - .await; - - assert_eq!(manager.stats().await.object_metadata, 1); - - manager.remove_proxy_session_state(&proxy_session_id, &[lease_id]).await; - - assert_eq!(manager.stats().await.object_metadata, 0); - } - - #[tokio::test] - async fn repair_metadata_is_bounded() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - - for index in 0..REPAIR_METADATA_MAX_ENTRIES + 5 { - manager - .record_metadata( - RepairIdentity { - raw_sha256: format!("{index:064x}"), - repair_mode: HlsSegmentRepairMode::Low, - command_version: 1, - ffmpeg_version: "test".to_string(), - }, - RepairStatus::Clean, - 1, - 1, - None, - ) - .await; - } - - assert_eq!(manager.stats().await.metadata, REPAIR_METADATA_MAX_ENTRIES); - } - - #[tokio::test] - async fn repair_lock_cleanup_keeps_waited_lock_and_removes_unused_lock() { - let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); - let identity = RepairIdentity { - raw_sha256: "a".repeat(64), - repair_mode: HlsSegmentRepairMode::Low, - command_version: 1, - ffmpeg_version: "test".to_string(), - }; - let lock = manager.lock_for_identity(identity.clone()).await; - let waiter = Arc::clone(&lock); - - manager.remove_lock_if_unused(&identity, &lock).await; - assert_eq!(manager.stats().await.locks, 1); - - drop(waiter); - manager.remove_lock_if_unused(&identity, &lock).await; - assert_eq!(manager.stats().await.locks, 0); - } - - #[test] - fn repair_context_excludes_non_finite_origin_ts_objects() { - let mut context = repair_context("lease-a", "000001"); - - context.complete_object = false; - assert!(!context.is_repairable_ts()); - - context.complete_object = true; - context.encrypted = true; - assert!(!context.is_repairable_ts()); - - context.encrypted = false; - context.custom_response = true; - assert!(!context.is_repairable_ts()); - - context.custom_response = false; - context.file_ext = "m4s".to_string(); - assert!(!context.is_repairable_ts()); - } -} +mod tests; + +mod diagnostics; +mod manager; +mod probe; +mod processing; +mod registry; +mod window; +pub use diagnostics::{parse_ffmpeg_warnings, WarningCounters}; +use diagnostics::{HlsSegmentRepairTriggerSource, SegmentProbeStreamType}; +pub(super) use probe::{ffmpeg_identity_version, sha256_file}; +pub(super) use processing::run_command_with_deadline; +pub use registry::HlsSegmentRepairStats; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use window::{ + ready_segment_repair_prewarm_candidates, HlsRepairRenderedObjectId, HlsSegmentRepairObjectContext, + HlsSegmentRepairSource, +}; +use window::{RepairStatus, RepairVideoCodec}; diff --git a/backend/hls/src/segment_repair/diagnostics.rs b/backend/hls/src/segment_repair/diagnostics.rs new file mode 100644 index 000000000..0ebf12560 --- /dev/null +++ b/backend/hls/src/segment_repair/diagnostics.rs @@ -0,0 +1,577 @@ +use super::{ + window::RepairVideoCodec, HlsSegmentRepairDecision, HlsSegmentRepairObjectContext, RepairRemuxDroppedStream, +}; +#[cfg(any(test, feature = "test-support"))] +use super::{RepairRemuxStreamSelection, SegmentProbe}; +use log::debug; +use shared::model::HlsSegmentRepairMode; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +#[allow(dead_code)] +pub(super) enum HlsSegmentRepairWarningKind { + MissingPat, + MissingPmt, + MissingPmtPid, + MissingVideoPidInPmt, + MissingAudioPidInPmt, + MultiplePrograms, + PesPacketSizeMismatch, + PacketCorrupt, + ContinuityCheckFailed, + MissingVps, + MissingSps, + MissingPps, + VpsOutOfRange, + SpsOutOfRange, + PpsOutOfRange, + NoFrame, + DecodeSliceHeaderError, + MissingPicture, + InvalidNal, + InvalidData, + CodecParametersMissing, + MmcoUnrefShortFailure, + ReorderBufferIncrease, + PpsIdOutOfRange, + NoStartCode, + NalSplitError, + NalParseError, + InvalidVclNalu, + InvalidMetadataNalu, + MultipleDolbyVisionRpus, + AvDesync, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum HlsSegmentRepairTriggerSource { + Off, + CommonMpegTsLow, + H264Medium, + H264High, + HevcMedium, + HevcHigh, + UnsupportedCodec, +} + +impl HlsSegmentRepairTriggerSource { + pub(super) const fn as_log_value(self) -> &'static str { + match self { + Self::Off => "off", + Self::CommonMpegTsLow => "common-mpegts-low", + Self::H264Medium => "h264-medium", + Self::H264High => "h264-high", + Self::HevcMedium => "hevc-medium", + Self::HevcHigh => "hevc-high", + Self::UnsupportedCodec => "unsupported-codec", + } + } +} + +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct WarningCounters { + pub missing_pat: u32, + pub missing_pmt: u32, + pub missing_pmt_pid: u32, + pub missing_video_pid_in_pmt: u32, + pub missing_audio_pid_in_pmt: u32, + pub multiple_programs: u32, + pub pes_packet_size_mismatch: u32, + pub packet_corrupt: u32, + pub continuity_check_failed: u32, + pub missing_vps: u32, + pub missing_sps: u32, + pub missing_pps: u32, + pub vps_out_of_range: u32, + pub sps_out_of_range: u32, + pub pps_out_of_range: u32, + pub no_frame: u32, + pub decode_slice_header_error: u32, + pub missing_picture: u32, + pub invalid_nal: u32, + pub invalid_data: u32, + pub codec_parameters_missing: u32, + pub mmco_unref_short_failure: u32, + pub reorder_buffer: u32, + pub pps_id_out_of_range: u32, + pub no_start_code: u32, + pub nal_split_error: u32, + pub nal_parse_error: u32, + pub invalid_undecodable_nalu_total: u32, + pub invalid_undecodable_nalu_non_metadata: u32, + pub invalid_undecodable_nalu_keyframe: u32, + pub invalid_undecodable_nalu_metadata: u32, + pub dolby_vision_rpu: u32, + pub av_desync: u32, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum SegmentProbeStreamType { + Video, + Audio, + Other, +} + +#[cfg(any(test, feature = "test-support"))] +impl RepairRemuxStreamSelection { + pub(super) fn preserve_all(probe: &SegmentProbe) -> Self { + Self { mapped_streams: probe.streams.iter().map(|stream| stream.index).collect(), dropped_streams: Vec::new() } + } +} + +pub fn parse_ffmpeg_warnings(stderr: &str) -> WarningCounters { + let mut counters = WarningCounters::default(); + let mut last_increment: Option = None; + for line in stderr.lines() { + let trimmed = line.trim(); + if let Some(repeated) = parse_repeated_count(trimmed) { + if let Some(increment) = last_increment { + increment(&mut counters, repeated); + } + continue; + } + if let Some(increment) = warning_increment(trimmed) { + increment(&mut counters, 1); + last_increment = Some(increment); + } + } + counters +} + +fn parse_repeated_count(line: &str) -> Option { + let rest = line.strip_prefix("Last message repeated ")?; + let count = rest.strip_suffix(" times")?.parse().ok()?; + Some(count) +} + +#[allow(clippy::too_many_lines)] +fn warning_increment(line: &str) -> Option { + let lower = line.to_ascii_lowercase(); + if lower.contains("missing pat") { + return Some(|counters, count| counters.missing_pat = counters.missing_pat.saturating_add(count)); + } + if lower.contains("missing pmt pid") { + return Some(|counters, count| counters.missing_pmt_pid = counters.missing_pmt_pid.saturating_add(count)); + } + if lower.contains("missing pmt") { + return Some(|counters, count| counters.missing_pmt = counters.missing_pmt.saturating_add(count)); + } + if lower.contains("missing video pid") && lower.contains("pmt") { + return Some(|counters, count| { + counters.missing_video_pid_in_pmt = counters.missing_video_pid_in_pmt.saturating_add(count); + }); + } + if lower.contains("missing audio pid") && lower.contains("pmt") { + return Some(|counters, count| { + counters.missing_audio_pid_in_pmt = counters.missing_audio_pid_in_pmt.saturating_add(count); + }); + } + if lower.contains("multiple mpeg-ts programs") || lower.contains("multiple programs") { + return Some(|counters, count| counters.multiple_programs = counters.multiple_programs.saturating_add(count)); + } + if lower.contains("pes packet size mismatch") { + return Some(|counters, count| { + counters.pes_packet_size_mismatch = counters.pes_packet_size_mismatch.saturating_add(count); + }); + } + if lower.contains("packet corrupt") { + return Some(|counters, count| counters.packet_corrupt = counters.packet_corrupt.saturating_add(count)); + } + if lower.contains("continuity check failed") { + return Some(|counters, count| { + counters.continuity_check_failed = counters.continuity_check_failed.saturating_add(count); + }); + } + if lower.contains("non-existing vps") || lower.contains("missing vps") { + return Some(|counters, count| counters.missing_vps = counters.missing_vps.saturating_add(count)); + } + if lower.contains("non-existing sps") || lower.contains("missing sps") { + return Some(|counters, count| counters.missing_sps = counters.missing_sps.saturating_add(count)); + } + if lower.contains("non-existing pps") || lower.contains("missing pps") { + return Some(|counters, count| counters.missing_pps = counters.missing_pps.saturating_add(count)); + } + if lower.contains("vps id out of range") { + return Some(|counters, count| counters.vps_out_of_range = counters.vps_out_of_range.saturating_add(count)); + } + if lower.contains("sps id out of range") { + return Some(|counters, count| counters.sps_out_of_range = counters.sps_out_of_range.saturating_add(count)); + } + if lower.contains("pps id out of range") { + return Some(|counters, count| { + counters.pps_id_out_of_range = counters.pps_id_out_of_range.saturating_add(count); + counters.pps_out_of_range = counters.pps_out_of_range.saturating_add(count); + }); + } + if lower.contains("vps") && lower.contains("out of range") { + return Some(|counters, count| counters.vps_out_of_range = counters.vps_out_of_range.saturating_add(count)); + } + if lower.contains("sps") && lower.contains("out of range") { + return Some(|counters, count| counters.sps_out_of_range = counters.sps_out_of_range.saturating_add(count)); + } + if lower.contains("pps") && lower.contains("out of range") { + return Some(|counters, count| counters.pps_out_of_range = counters.pps_out_of_range.saturating_add(count)); + } + if lower.contains("no frame") { + return Some(|counters, count| counters.no_frame = counters.no_frame.saturating_add(count)); + } + if lower.contains("decode_slice_header error") { + return Some(|counters, count| { + counters.decode_slice_header_error = counters.decode_slice_header_error.saturating_add(count); + }); + } + if lower.contains("missing picture") { + return Some(|counters, count| counters.missing_picture = counters.missing_picture.saturating_add(count)); + } + if lower.contains("invalid nal") { + return Some(|counters, count| counters.invalid_nal = counters.invalid_nal.saturating_add(count)); + } + if lower.contains("invalid data found") { + return Some(|counters, count| counters.invalid_data = counters.invalid_data.saturating_add(count)); + } + if lower.contains("could not find codec parameters") { + return Some(|counters, count| { + counters.codec_parameters_missing = counters.codec_parameters_missing.saturating_add(count); + }); + } + if lower.contains("mmco: unref short failure") { + return Some(|counters, count| { + counters.mmco_unref_short_failure = counters.mmco_unref_short_failure.saturating_add(count); + }); + } + if lower.contains("increasing reorder buffer") { + return Some(|counters, count| counters.reorder_buffer = counters.reorder_buffer.saturating_add(count)); + } + if lower.contains("no start code is found") { + return Some(|counters, count| counters.no_start_code = counters.no_start_code.saturating_add(count)); + } + if lower.contains("error splitting") && lower.contains("nal") { + return Some(|counters, count| counters.nal_split_error = counters.nal_split_error.saturating_add(count)); + } + if lower.contains("error parsing nal unit") { + return Some(|counters, count| counters.nal_parse_error = counters.nal_parse_error.saturating_add(count)); + } + if let Some(nalu_type) = parse_invalid_undecodable_nalu_type(line) { + return Some(match nalu_type { + 0..=20 | 22..=31 => |counters: &mut WarningCounters, count| { + counters.invalid_undecodable_nalu_total = counters.invalid_undecodable_nalu_total.saturating_add(count); + counters.invalid_undecodable_nalu_non_metadata = + counters.invalid_undecodable_nalu_non_metadata.saturating_add(count); + }, + 21 => |counters: &mut WarningCounters, count| { + counters.invalid_undecodable_nalu_total = counters.invalid_undecodable_nalu_total.saturating_add(count); + counters.invalid_undecodable_nalu_keyframe = + counters.invalid_undecodable_nalu_keyframe.saturating_add(count); + }, + 39 => |counters: &mut WarningCounters, count| { + counters.invalid_undecodable_nalu_total = counters.invalid_undecodable_nalu_total.saturating_add(count); + counters.invalid_undecodable_nalu_metadata = + counters.invalid_undecodable_nalu_metadata.saturating_add(count); + }, + _ => |counters: &mut WarningCounters, count| { + counters.invalid_undecodable_nalu_total = counters.invalid_undecodable_nalu_total.saturating_add(count); + }, + }); + } + if lower.contains("multiple dolby vision rpus found in one au") { + return Some(|counters, count| counters.dolby_vision_rpu = counters.dolby_vision_rpu.saturating_add(count)); + } + if lower.contains("audio/video desynchronisation detected") { + return Some(|counters, count| counters.av_desync = counters.av_desync.saturating_add(count)); + } + None +} + +fn parse_invalid_undecodable_nalu_type(line: &str) -> Option { + let (_, rest) = line.split_once("Skipping invalid undecodable NALU:")?; + rest.trim().split(|ch: char| !ch.is_ascii_digit()).next().filter(|value| !value.is_empty())?.parse().ok() +} + +fn warning_count(warnings: &WarningCounters, kind: HlsSegmentRepairWarningKind) -> u32 { + match kind { + HlsSegmentRepairWarningKind::MissingPat => warnings.missing_pat, + HlsSegmentRepairWarningKind::MissingPmt => warnings.missing_pmt, + HlsSegmentRepairWarningKind::MissingPmtPid => warnings.missing_pmt_pid, + HlsSegmentRepairWarningKind::MissingVideoPidInPmt => warnings.missing_video_pid_in_pmt, + HlsSegmentRepairWarningKind::MissingAudioPidInPmt => warnings.missing_audio_pid_in_pmt, + HlsSegmentRepairWarningKind::MultiplePrograms => warnings.multiple_programs, + HlsSegmentRepairWarningKind::PesPacketSizeMismatch => warnings.pes_packet_size_mismatch, + HlsSegmentRepairWarningKind::PacketCorrupt => warnings.packet_corrupt, + HlsSegmentRepairWarningKind::ContinuityCheckFailed => warnings.continuity_check_failed, + HlsSegmentRepairWarningKind::MissingVps => warnings.missing_vps, + HlsSegmentRepairWarningKind::MissingSps => warnings.missing_sps, + HlsSegmentRepairWarningKind::MissingPps => warnings.missing_pps, + HlsSegmentRepairWarningKind::VpsOutOfRange => warnings.vps_out_of_range, + HlsSegmentRepairWarningKind::SpsOutOfRange => warnings.sps_out_of_range, + HlsSegmentRepairWarningKind::PpsOutOfRange => warnings.pps_out_of_range, + HlsSegmentRepairWarningKind::NoFrame => warnings.no_frame, + HlsSegmentRepairWarningKind::DecodeSliceHeaderError => warnings.decode_slice_header_error, + HlsSegmentRepairWarningKind::MissingPicture => warnings.missing_picture, + HlsSegmentRepairWarningKind::InvalidNal => warnings.invalid_nal, + HlsSegmentRepairWarningKind::InvalidData => warnings.invalid_data, + HlsSegmentRepairWarningKind::CodecParametersMissing => warnings.codec_parameters_missing, + HlsSegmentRepairWarningKind::MmcoUnrefShortFailure => warnings.mmco_unref_short_failure, + HlsSegmentRepairWarningKind::ReorderBufferIncrease => warnings.reorder_buffer, + HlsSegmentRepairWarningKind::PpsIdOutOfRange => warnings.pps_id_out_of_range, + HlsSegmentRepairWarningKind::NoStartCode => warnings.no_start_code, + HlsSegmentRepairWarningKind::NalSplitError => warnings.nal_split_error, + HlsSegmentRepairWarningKind::NalParseError => warnings.nal_parse_error, + HlsSegmentRepairWarningKind::InvalidVclNalu => { + warnings.invalid_undecodable_nalu_non_metadata.saturating_add(warnings.invalid_undecodable_nalu_keyframe) + } + HlsSegmentRepairWarningKind::InvalidMetadataNalu => warnings.invalid_undecodable_nalu_metadata, + HlsSegmentRepairWarningKind::MultipleDolbyVisionRpus => warnings.dolby_vision_rpu, + HlsSegmentRepairWarningKind::AvDesync => warnings.av_desync, + } +} + +fn has_any_warning(warnings: &WarningCounters, kinds: &[HlsSegmentRepairWarningKind]) -> bool { + kinds.iter().any(|kind| warning_count(warnings, *kind) > 0) +} + +fn common_mpegts_low_trigger(warnings: &WarningCounters) -> bool { + has_any_warning( + warnings, + &[ + HlsSegmentRepairWarningKind::MissingPat, + HlsSegmentRepairWarningKind::MissingPmt, + HlsSegmentRepairWarningKind::MissingPmtPid, + HlsSegmentRepairWarningKind::MissingVideoPidInPmt, + HlsSegmentRepairWarningKind::MissingAudioPidInPmt, + HlsSegmentRepairWarningKind::MultiplePrograms, + HlsSegmentRepairWarningKind::PesPacketSizeMismatch, + HlsSegmentRepairWarningKind::PacketCorrupt, + HlsSegmentRepairWarningKind::ContinuityCheckFailed, + HlsSegmentRepairWarningKind::CodecParametersMissing, + ], + ) +} + +fn h264_parameter_trigger(warnings: &WarningCounters) -> bool { + has_any_warning( + warnings, + &[ + HlsSegmentRepairWarningKind::MissingSps, + HlsSegmentRepairWarningKind::MissingPps, + HlsSegmentRepairWarningKind::SpsOutOfRange, + HlsSegmentRepairWarningKind::PpsOutOfRange, + ], + ) +} + +fn h264_medium_trigger(warnings: &WarningCounters) -> bool { + h264_parameter_trigger(warnings) + || warnings.decode_slice_header_error > 0 + || warnings.no_frame > 0 + || (warnings.missing_picture > 0 + && (warnings.missing_sps > 0 || warnings.missing_pps > 0 || warnings.decode_slice_header_error > 0)) + || (warnings.invalid_nal > 0 && h264_parameter_trigger(warnings)) + || (warnings.no_start_code > 0 && (warnings.missing_sps > 0 || warnings.missing_pps > 0)) + || (warnings.nal_split_error > 0 && (warnings.missing_sps > 0 || warnings.missing_pps > 0)) +} + +fn hevc_missing_parameter_trigger(warnings: &WarningCounters) -> bool { + warnings.missing_vps > 0 || warnings.missing_sps > 0 || warnings.missing_pps > 0 +} + +fn hevc_parameter_trigger(warnings: &WarningCounters) -> bool { + hevc_missing_parameter_trigger(warnings) + || warnings.vps_out_of_range > 0 + || warnings.sps_out_of_range > 0 + || warnings.pps_out_of_range > 0 + || warnings.pps_id_out_of_range > 0 +} + +fn hevc_invalid_vcl_nalu(warnings: &WarningCounters) -> bool { + warning_count(warnings, HlsSegmentRepairWarningKind::InvalidVclNalu) > 0 +} + +fn hevc_medium_trigger(warnings: &WarningCounters) -> bool { + hevc_parameter_trigger(warnings) + || (hevc_invalid_vcl_nalu(warnings) && hevc_parameter_trigger(warnings)) + || (warnings.nal_parse_error > 0 && hevc_parameter_trigger(warnings)) + || (warnings.invalid_nal > 0 && hevc_parameter_trigger(warnings)) + || (warnings.no_start_code > 0 && hevc_missing_parameter_trigger(warnings)) + || (warnings.nal_split_error > 0 && hevc_missing_parameter_trigger(warnings)) +} + +pub(super) fn decide_repair(codec: RepairVideoCodec, warnings: &WarningCounters) -> HlsSegmentRepairDecision { + let common_low_trigger = common_mpegts_low_trigger(warnings); + let codec_parameters_missing = warnings.codec_parameters_missing > 0; + let codec_medium_trigger = match codec { + RepairVideoCodec::H264 => h264_medium_trigger(warnings), + RepairVideoCodec::Hevc => hevc_medium_trigger(warnings), + RepairVideoCodec::Unsupported => false, + }; + match codec { + RepairVideoCodec::H264 if codec_medium_trigger && (common_low_trigger || codec_parameters_missing) => { + HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::High, + trigger_source: HlsSegmentRepairTriggerSource::H264High, + common_low_trigger, + codec_medium_trigger, + } + } + RepairVideoCodec::H264 if codec_medium_trigger => HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::Medium, + trigger_source: HlsSegmentRepairTriggerSource::H264Medium, + common_low_trigger, + codec_medium_trigger, + }, + RepairVideoCodec::Hevc + if (common_low_trigger && (codec_medium_trigger || hevc_invalid_vcl_nalu(warnings))) + || (codec_parameters_missing && codec_medium_trigger) => + { + HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::High, + trigger_source: HlsSegmentRepairTriggerSource::HevcHigh, + common_low_trigger, + codec_medium_trigger, + } + } + RepairVideoCodec::Hevc if codec_medium_trigger => HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::Medium, + trigger_source: HlsSegmentRepairTriggerSource::HevcMedium, + common_low_trigger, + codec_medium_trigger, + }, + RepairVideoCodec::H264 | RepairVideoCodec::Hevc if common_low_trigger => HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::Low, + trigger_source: HlsSegmentRepairTriggerSource::CommonMpegTsLow, + common_low_trigger, + codec_medium_trigger, + }, + RepairVideoCodec::Unsupported => HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::Off, + trigger_source: HlsSegmentRepairTriggerSource::UnsupportedCodec, + common_low_trigger, + codec_medium_trigger, + }, + _ => HlsSegmentRepairDecision { + codec, + required_level: HlsSegmentRepairMode::Off, + trigger_source: HlsSegmentRepairTriggerSource::Off, + common_low_trigger, + codec_medium_trigger, + }, + } +} + +pub(super) fn debug_repair_analysis( + context: &HlsSegmentRepairObjectContext, + configured_max_level: HlsSegmentRepairMode, + decision: HlsSegmentRepairDecision, + executed_level: Option, + warnings: &WarningCounters, +) { + let executed_level = executed_level.map_or("off", HlsSegmentRepairMode::as_log_value); + match decision.codec { + RepairVideoCodec::H264 => debug!( + "HLS segment repair analysis completed: {} source={} resource={} configured_max_level={} required_level={} executed_level={} codec={} trigger={} common_low={} codec_medium={} missing_sps={} missing_pps={} sps_out_of_range={} pps_out_of_range={} no_frame={} decode_slice_header_error={} missing_picture={} invalid_nal={} no_start_code={} nal_split_error={} packet_corrupt={} continuity_check_failed={} codec_parameters_missing={}", + context.log_identity_fields(), + context.source.as_log_value(), + context.resource_id, + configured_max_level.as_log_value(), + decision.required_level.as_log_value(), + executed_level, + decision.codec.as_log_value(), + decision.trigger_source.as_log_value(), + decision.common_low_trigger, + decision.codec_medium_trigger, + warnings.missing_sps, + warnings.missing_pps, + warnings.sps_out_of_range, + warnings.pps_out_of_range, + warnings.no_frame, + warnings.decode_slice_header_error, + warnings.missing_picture, + warnings.invalid_nal, + warnings.no_start_code, + warnings.nal_split_error, + warnings.packet_corrupt, + warnings.continuity_check_failed, + warnings.codec_parameters_missing + ), + RepairVideoCodec::Hevc => debug!( + "HLS segment repair analysis completed: {} source={} resource={} configured_max_level={} required_level={} executed_level={} codec={} trigger={} common_low={} codec_medium={} missing_vps={} missing_sps={} missing_pps={} vps_out_of_range={} sps_out_of_range={} pps_out_of_range={} pps_id_out_of_range={} invalid_vcl_nalu={} nal_parse_error={} invalid_nal={} no_start_code={} nal_split_error={} packet_corrupt={} continuity_check_failed={} codec_parameters_missing={}", + context.log_identity_fields(), + context.source.as_log_value(), + context.resource_id, + configured_max_level.as_log_value(), + decision.required_level.as_log_value(), + executed_level, + decision.codec.as_log_value(), + decision.trigger_source.as_log_value(), + decision.common_low_trigger, + decision.codec_medium_trigger, + warnings.missing_vps, + warnings.missing_sps, + warnings.missing_pps, + warnings.vps_out_of_range, + warnings.sps_out_of_range, + warnings.pps_out_of_range, + warnings.pps_id_out_of_range, + warning_count(warnings, HlsSegmentRepairWarningKind::InvalidVclNalu), + warnings.nal_parse_error, + warnings.invalid_nal, + warnings.no_start_code, + warnings.nal_split_error, + warnings.packet_corrupt, + warnings.continuity_check_failed, + warnings.codec_parameters_missing + ), + RepairVideoCodec::Unsupported => debug!( + "HLS segment repair analysis completed: {} source={} resource={} configured_max_level={} required_level={} executed_level={} codec={} trigger={}", + context.log_identity_fields(), + context.source.as_log_value(), + context.resource_id, + configured_max_level.as_log_value(), + decision.required_level.as_log_value(), + executed_level, + decision.codec.as_log_value(), + decision.trigger_source.as_log_value() + ), + } +} + +pub(super) fn debug_repair_event( + context: &HlsSegmentRepairObjectContext, + mode: HlsSegmentRepairMode, + event: &'static str, + reason: Option<&str>, +) { + if let Some(reason) = reason { + debug!( + "HLS segment repair {event}: {} source={} resource={} mode={} reason={}", + context.log_identity_fields(), + context.source.as_log_value(), + context.resource_id, + mode.as_log_value(), + reason + ); + } else { + debug!( + "HLS segment repair {event}: {} source={} resource={} mode={}", + context.log_identity_fields(), + context.source.as_log_value(), + context.resource_id, + mode.as_log_value() + ); + } +} + +pub(super) fn debug_repair_stream_dropped(context: &HlsSegmentRepairObjectContext, dropped: &RepairRemuxDroppedStream) { + debug!( + "HLS segment repair stream dropped: {} source={} resource={} stream={} reason={}", + context.log_identity_fields(), + context.source.as_log_value(), + context.resource_id, + dropped.index, + dropped.reason + ); +} diff --git a/backend/hls/src/segment_repair/manager.rs b/backend/hls/src/segment_repair/manager.rs new file mode 100644 index 000000000..f541f30a2 --- /dev/null +++ b/backend/hls/src/segment_repair/manager.rs @@ -0,0 +1,498 @@ +use super::{ + ffmpeg_identity_version, probe::adopt_repair_output, registry::repair_object_metadata_key, + safe_hls_access_lease_id, safe_proxy_session_id, sha256_file, window::RepairStatus, CachedSegmentMetadata, + HlsAccessLeaseId, HlsCacheObjectKey, HlsCorruptSegmentWatchdogManager, HlsPostProcessingDeadline, + HlsRepairObjectMetadata, HlsRepairObjectMetadataKey, HlsRepairPrewarmGuard, HlsRepairRenderedObjectId, + HlsRepairWindowRegistry, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, + HlsSegmentRepairRuntime, HlsSegmentRepairStats, HlsSelectedRepairCandidate, ProxySessionId, RepairIdentity, + SegmentRepairMetadata, COMMAND_VERSION, REPAIR_METADATA_MAX_ENTRIES, REPAIR_OBJECT_METADATA_MAX_ENTRIES, +}; +use arc_swap::ArcSwap; +use log::debug; +use shared::model::HlsSegmentRepairMode; +use std::{ + collections::{HashMap, VecDeque}, + io, + sync::Arc, +}; +use tokio::{ + fs, + io::AsyncRead, + sync::{Mutex, RwLock, Semaphore}, +}; +use tuliprox_core::model::{HlsCorruptSegmentWatchdogConfig, HlsSegmentRepairConfig}; + +impl HlsSegmentRepairRuntime { + pub(super) fn new(config: HlsSegmentRepairConfig) -> Self { + let semaphore = if config.max_parallel_repairs == 0 { + None + } else { + Some(Arc::new(Semaphore::new(config.max_parallel_repairs))) + }; + let watchdog_config = &config.corrupt_segment_watchdog; + let watchdog_semaphore = Arc::new(Semaphore::new(watchdog_config.max_parallel_jobs.max(1))); + Self { config, semaphore, watchdog_semaphore } + } + + pub(super) fn repair_enabled(&self) -> bool { + self.config.max_level != HlsSegmentRepairMode::Off && self.config.apply_to_first_segments > 0 + } + + pub(super) fn postprocessing_enabled(&self) -> bool { + self.repair_enabled() || self.config.corrupt_segment_watchdog.mode.is_enabled() + } + + pub(super) fn postprocess_timeout_ms(&self) -> u64 { self.config.postprocess_timeout_ms.get().max(100) } +} + +fn log_segment_repair_config(config: &HlsSegmentRepairConfig) { + debug!( + "HLS segment repair configured: max_level={} segments={} max_parallel={} postprocess_timeout_ms={}", + config.max_level.as_log_value(), + config.apply_to_first_segments, + config.max_parallel_repairs, + config.postprocess_timeout_ms + ); +} + +fn log_corrupt_segment_watchdog_config(config: &HlsCorruptSegmentWatchdogConfig) { + debug!( + "HLS corrupt segment watchdog configured: mode={} max_parallel_jobs={}", + config.mode.as_log_value(), + config.max_parallel_jobs + ); +} + +impl HlsSegmentRepairManager { + pub fn new(config: HlsSegmentRepairConfig) -> Self { + let watchdog_config = config.corrupt_segment_watchdog.clone(); + log_segment_repair_config(&config); + log_corrupt_segment_watchdog_config(&watchdog_config); + Self { + runtime: ArcSwap::from_pointee(HlsSegmentRepairRuntime::new(config)), + watchdog: HlsCorruptSegmentWatchdogManager::new(), + windows: RwLock::new(HlsRepairWindowRegistry::default()), + metadata: RwLock::new(HashMap::new()), + metadata_order: Mutex::new(VecDeque::new()), + object_metadata: RwLock::new(HashMap::new()), + object_metadata_order: Mutex::new(VecDeque::new()), + locks: Mutex::new(HashMap::new()), + } + } + + pub fn update_config(&self, config: HlsSegmentRepairConfig) { + let current = self.runtime.load(); + if current.config == config { + return; + } + let watchdog_config = config.corrupt_segment_watchdog.clone(); + log_segment_repair_config(&config); + log_corrupt_segment_watchdog_config(&watchdog_config); + self.runtime.store(Arc::new(HlsSegmentRepairRuntime::new(config))); + } + + pub async fn start_access_lease_window(&self, lease_id: HlsAccessLeaseId) { + let runtime = self.runtime.load_full(); + if !runtime.repair_enabled() { + return; + } + self.windows.write().await.start_window(lease_id.clone(), &runtime.config); + debug!( + "HLS segment repair window started: lease={} max_level={} segments={}", + safe_hls_access_lease_id(&lease_id), + runtime.config.max_level.as_log_value(), + runtime.config.apply_to_first_segments + ); + } + + pub async fn ensure_access_lease_window(&self, lease_id: HlsAccessLeaseId) { + let runtime = self.runtime.load_full(); + if !runtime.repair_enabled() { + return; + } + let started = self.windows.write().await.ensure_window(lease_id.clone(), &runtime.config); + if started { + debug!( + "HLS segment repair window started: lease={} max_level={} segments={}", + safe_hls_access_lease_id(&lease_id), + runtime.config.max_level.as_log_value(), + runtime.config.apply_to_first_segments + ); + } + } + + pub fn prewarm_candidate_limit(&self) -> usize { + let runtime = self.runtime.load(); + if runtime.repair_enabled() { + usize::from(runtime.config.apply_to_first_segments) + } else { + 0 + } + } + + pub async fn spawn_ready_cache_prewarm( + self: &Arc, + segment_cache: Arc, + candidates: Vec<(K, HlsSegmentRepairObjectContext)>, + guard: HlsRepairPrewarmGuard, + ) where + K: HlsCacheObjectKey + Send + Sync + 'static, + { + self.spawn_ready_cache_prewarm_inner(segment_cache, candidates, Some(guard)).await; + } + + pub(super) async fn spawn_ready_cache_prewarm_inner( + self: &Arc, + segment_cache: Arc, + candidates: Vec<(K, HlsSegmentRepairObjectContext)>, + guard: Option, + ) where + K: HlsCacheObjectKey + Send + Sync + 'static, + { + for (key, context) in candidates { + let Some((mode, runtime, candidate)) = self.try_select_candidate(&context).await else { + continue; + }; + let manager = Arc::clone(self); + let segment_cache = Arc::clone(&segment_cache); + let guard = guard.clone(); + tokio::spawn(async move { + let selection = HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard: guard.as_ref() }; + if let Err(err) = + manager.repair_ready_cache_hit_with_candidate(&segment_cache, &key, context, selection).await + { + debug!("HLS segment repair prewarm ended without a reusable decision: error_kind={:?}", err.kind()); + } + }); + } + } + + pub async fn remove_access_lease_window(&self, lease_id: &HlsAccessLeaseId) { + self.windows.write().await.remove_access_lease(lease_id); + } + + pub async fn remove_proxy_session_state(&self, proxy_session_id: &ProxySessionId, lease_ids: &[HlsAccessLeaseId]) { + self.windows.write().await.remove_proxy_session(proxy_session_id, lease_ids); + self.object_metadata.write().await.retain(|key, _| key.proxy_session_id != *proxy_session_id); + self.object_metadata_order.lock().await.retain(|key| key.proxy_session_id != *proxy_session_id); + } + + pub async fn clear_runtime_state(&self) { + self.windows.write().await.clear(); + self.metadata.write().await.clear(); + self.metadata_order.lock().await.clear(); + self.object_metadata.write().await.clear(); + self.object_metadata_order.lock().await.clear(); + self.locks.lock().await.clear(); + self.watchdog.clear_runtime_state().await; + } + + pub async fn stats(&self) -> HlsSegmentRepairStats { + let mut stats = self.windows.read().await.stats(); + stats.metadata = self.metadata.read().await.len(); + stats.object_metadata = self.object_metadata.read().await.len(); + stats.locks = self.locks.lock().await.len(); + let watchdog = self.watchdog.stats().await; + stats.watchdog_metadata = watchdog.metadata; + stats.watchdog_locks = watchdog.locks; + stats + } + + pub async fn commit_origin_response( + &self, + segment_cache: &HlsSegmentCache, + key: &K, + reader: R, + deadline: tokio::time::Instant, + context: HlsSegmentRepairObjectContext, + ) -> io::Result + where + K: HlsCacheObjectKey, + R: AsyncRead + Unpin, + { + let raw = segment_cache.stage_temp_with_deadline(key, reader, deadline).await?; + let runtime = self.runtime.load_full(); + if !runtime.postprocessing_enabled() { + return segment_cache.commit_staged(key, raw).await; + } + let postprocessing_deadline = HlsPostProcessingDeadline::new(runtime.postprocess_timeout_ms()); + self.process_staged_and_commit(segment_cache, key, raw, context, runtime, postprocessing_deadline).await + } + + #[allow(clippy::too_many_lines)] + pub async fn repair_ready_cache_hit( + &self, + segment_cache: &HlsSegmentCache, + key: &K, + context: HlsSegmentRepairObjectContext, + ) -> io::Result> + where + K: HlsCacheObjectKey, + { + let Some((mode, runtime, candidate)) = self.try_select_or_join_candidate(&context).await else { + return Ok(None); + }; + self.repair_ready_cache_hit_with_candidate( + segment_cache, + key, + context, + HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard: None }, + ) + .await + } + + #[allow(clippy::too_many_lines)] + pub(super) async fn repair_ready_cache_hit_with_candidate( + &self, + segment_cache: &HlsSegmentCache, + key: &K, + context: HlsSegmentRepairObjectContext, + selection: HlsSelectedRepairCandidate<'_>, + ) -> io::Result> + where + K: HlsCacheObjectKey, + { + let HlsSelectedRepairCandidate { mode, runtime, candidate, prewarm_guard } = selection; + let Some(metadata) = segment_cache.metadata(key).await? else { + return Ok(None); + }; + let raw_hash = sha256_file(&metadata.path).await?; + let object_key = repair_object_metadata_key(&context, mode); + if self.object_metadata_matches(&object_key, &raw_hash).await { + return Ok(None); + } + let identity = RepairIdentity { + raw_sha256: raw_hash.clone(), + repair_mode: mode, + command_version: COMMAND_VERSION, + ffmpeg_version: ffmpeg_identity_version(), + }; + if self.repair_metadata(&identity).await.is_some() { + self.record_object_metadata_from_repair_identity(object_key, raw_hash.clone(), Some(raw_hash), &identity) + .await; + return Ok(None); + } + let lock = self.lock_for_identity(identity.clone()).await; + let result = { + let _guard = lock.lock().await; + let prewarm_is_current = match prewarm_guard { + Some(prewarm_guard) => prewarm_guard.is_current().await, + None => true, + }; + if !prewarm_is_current { + Ok(None) + } else if let Some(current_metadata) = segment_cache.metadata(key).await? { + let current_hash = sha256_file(¤t_metadata.path).await?; + if current_hash != raw_hash { + Ok(None) + } else if self.repair_metadata(&identity).await.is_some() { + self.record_object_metadata_from_repair_identity( + object_key.clone(), + current_hash.clone(), + Some(current_hash), + &identity, + ) + .await; + Ok(None) + } else { + let deadline = HlsPostProcessingDeadline::new(runtime.postprocess_timeout_ms()); + if let Some(fixed_path) = self + .repair_file( + ¤t_metadata.path, + current_metadata.size, + &identity, + &context, + runtime.clone(), + &deadline, + ) + .await? + { + let candidate_current = self.windows.read().await.candidate_is_current(&candidate) + && match prewarm_guard { + Some(prewarm_guard) => prewarm_guard.is_current().await, + None => true, + }; + let latest_hash = match segment_cache.metadata(key).await? { + Some(latest_metadata) => Some(sha256_file(&latest_metadata.path).await?), + None => None, + }; + if !candidate_current || latest_hash.as_deref() != Some(current_hash.as_str()) { + let _ = fs::remove_file(&fixed_path).await; + Ok(None) + } else { + let fixed_size = fs::metadata(&fixed_path).await?.len(); + let staged = adopt_repair_output(segment_cache, fixed_path, fixed_size).await?; + let committed = segment_cache.commit_staged(key, staged).await?; + self.record_metadata( + identity.clone(), + RepairStatus::Fixed, + current_metadata.size, + committed.size, + None, + ) + .await; + let committed_hash = sha256_file(&committed.path).await?; + self.record_object_metadata( + object_key.clone(), + HlsRepairObjectMetadata { + committed_sha256: committed_hash, + raw_sha256: Some(current_hash), + status: RepairStatus::Fixed, + raw_size: current_metadata.size, + final_size: committed.size, + validation_reason: None, + }, + ) + .await; + Ok(Some(committed)) + } + } else { + self.record_object_metadata_from_repair_identity( + object_key.clone(), + current_hash.clone(), + Some(current_hash), + &identity, + ) + .await; + Ok(None) + } + } + } else { + Ok(None) + } + }; + self.remove_lock_if_unused(&identity, &lock).await; + result + } + + pub(super) async fn record_metadata( + &self, + identity: RepairIdentity, + status: RepairStatus, + raw_size: u64, + final_size: u64, + validation_reason: Option, + ) { + let inserted_new = { + let mut metadata = self.metadata.write().await; + let inserted_new = !metadata.contains_key(&identity); + metadata + .insert(identity.clone(), SegmentRepairMetadata { status, raw_size, final_size, validation_reason }); + inserted_new + }; + if inserted_new { + self.metadata_order.lock().await.push_back(identity); + } + self.prune_metadata().await; + } + + pub(super) async fn repair_metadata(&self, identity: &RepairIdentity) -> Option { + self.metadata.read().await.get(identity).cloned() + } + + pub(super) async fn object_metadata_matches( + &self, + key: &HlsRepairObjectMetadataKey, + committed_sha256: &str, + ) -> bool { + let matches = self + .object_metadata + .read() + .await + .get(key) + .is_some_and(|metadata| metadata.committed_sha256 == committed_sha256); + if matches { + let (source, resource) = match &key.rendered_object_id { + HlsRepairRenderedObjectId::Normal { proxy_seq } => ("normal", format!("{proxy_seq:06}")), + HlsRepairRenderedObjectId::Transient { resource_id } => ("transient", resource_id.clone()), + }; + debug!( + "HLS segment repair object metadata hit: proxy_session={} source={} resource={} mode={}", + safe_proxy_session_id(&key.proxy_session_id), + source, + resource, + key.repair_mode.as_log_value() + ); + } + matches + } + + pub(super) async fn record_object_metadata_from_repair_identity( + &self, + key: HlsRepairObjectMetadataKey, + committed_sha256: String, + raw_sha256: Option, + identity: &RepairIdentity, + ) { + let Some(metadata) = self.repair_metadata(identity).await else { + return; + }; + self.record_object_metadata( + key, + HlsRepairObjectMetadata { + committed_sha256, + raw_sha256, + status: metadata.status, + raw_size: metadata.raw_size, + final_size: metadata.final_size, + validation_reason: metadata.validation_reason, + }, + ) + .await; + } + + pub(super) async fn record_object_metadata( + &self, + key: HlsRepairObjectMetadataKey, + metadata: HlsRepairObjectMetadata, + ) { + let inserted_new = { + let mut object_metadata = self.object_metadata.write().await; + let inserted_new = !object_metadata.contains_key(&key); + object_metadata.insert(key.clone(), metadata); + inserted_new + }; + if inserted_new { + self.object_metadata_order.lock().await.push_back(key); + } + self.prune_object_metadata().await; + } + + pub(super) async fn lock_for_identity(&self, identity: RepairIdentity) -> Arc> { + let mut locks = self.locks.lock().await; + Arc::clone(locks.entry(identity).or_insert_with(|| Arc::new(Mutex::new(())))) + } + + pub(super) async fn remove_lock_if_unused(&self, identity: &RepairIdentity, lock: &Arc>) { + let mut locks = self.locks.lock().await; + if Arc::strong_count(lock) <= 2 && locks.get(identity).is_some_and(|current| Arc::ptr_eq(current, lock)) { + locks.remove(identity); + } + } + + pub(super) async fn prune_metadata(&self) { + loop { + let should_prune = self.metadata.read().await.len() > REPAIR_METADATA_MAX_ENTRIES; + if !should_prune { + return; + } + let Some(oldest) = self.metadata_order.lock().await.pop_front() else { + return; + }; + self.metadata.write().await.remove(&oldest); + } + } + + pub(super) async fn prune_object_metadata(&self) { + loop { + let should_prune = self.object_metadata.read().await.len() > REPAIR_OBJECT_METADATA_MAX_ENTRIES; + if !should_prune { + return; + } + let Some(oldest) = self.object_metadata_order.lock().await.pop_front() else { + return; + }; + self.object_metadata.write().await.remove(&oldest); + } + } +} diff --git a/backend/hls/src/segment_repair/probe.rs b/backend/hls/src/segment_repair/probe.rs new file mode 100644 index 000000000..83eccb9df --- /dev/null +++ b/backend/hls/src/segment_repair/probe.rs @@ -0,0 +1,277 @@ +use super::{ + diagnostics::{decide_repair, SegmentProbeStreamType}, + window::{RepairStatus, RepairVideoCodec}, + HlsSegmentCache, RepairRemuxDroppedStream, RepairRemuxStreamSelection, SegmentProbe, SegmentProbeStream, + StagedCacheObject, WarningCounters, +}; +use log::warn; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use shared::model::HlsSegmentRepairMode; +use std::{ + fmt, io, + path::{Path, PathBuf}, +}; +use tokio::{fs, fs::File, io::AsyncReadExt}; +use tuliprox_core::model::HlsSegmentRepairConfig; + +pub(super) fn parse_probe(json: &str, warnings: WarningCounters) -> Result { + let value = serde_json::from_str::(json).map_err(|_| "invalid_probe_json".to_string())?; + let streams = value.get("streams").and_then(Value::as_array).ok_or_else(|| "missing_streams".to_string())?; + let mut probe = SegmentProbe { stream_count: streams.len(), warnings, ..SegmentProbe::default() }; + if let Some(format) = value.get("format") { + probe.duration_ms = format.get("duration").and_then(Value::as_str).and_then(parse_seconds_ms_u64); + probe.size = + format.get("size").and_then(Value::as_str).and_then(|value| value.parse().ok()).unwrap_or_default(); + } + for stream in streams { + let codec_type = stream.get("codec_type").and_then(Value::as_str); + let codec_name = stream.get("codec_name").and_then(Value::as_str).map(ToOwned::to_owned); + let start_time_ms = stream.get("start_time").and_then(Value::as_str).and_then(parse_seconds_ms_i64); + let extradata_size = stream + .get("extradata_size") + .and_then(Value::as_u64) + .or_else(|| stream.get("extradata_size").and_then(Value::as_str).and_then(|value| value.parse().ok())); + let stream_index = + stream.get("index").and_then(parse_u32_value).map_or(probe.streams.len(), |value| value as usize); + let probe_stream = SegmentProbeStream { + index: stream_index, + stream_type: match codec_type { + Some("video") => SegmentProbeStreamType::Video, + Some("audio") => SegmentProbeStreamType::Audio, + _ => SegmentProbeStreamType::Other, + }, + codec_name: codec_name.clone(), + width: stream.get("width").and_then(parse_u32_value), + height: stream.get("height").and_then(parse_u32_value), + sample_rate: stream.get("sample_rate").and_then(parse_u32_value), + channels: stream.get("channels").and_then(parse_u32_value), + }; + match codec_type { + Some("video") if probe.primary_video_codec.is_none() => { + probe.primary_video_codec = codec_name; + probe.primary_video_start_time_ms = start_time_ms; + probe.primary_video_extradata_size = extradata_size; + } + Some("audio") if probe.primary_audio_codec.is_none() => { + probe.primary_audio_codec = codec_name; + probe.primary_audio_start_time_ms = start_time_ms; + } + _ => {} + } + probe.streams.push(probe_stream); + } + Ok(probe) +} + +fn parse_u32_value(value: &Value) -> Option { + value.as_u64().and_then(|value| u32::try_from(value).ok()).or_else(|| { + value + .as_str() + .map(str::trim) + .filter(|value| !value.is_empty() && *value != "N/A") + .and_then(|value| value.parse::().ok()) + }) +} + +pub(super) fn detect_video_codec(probe: &SegmentProbe) -> RepairVideoCodec { + match probe.primary_video_codec.as_deref() { + Some("h264") => RepairVideoCodec::H264, + Some("hevc" | "h265") => RepairVideoCodec::Hevc, + _ => RepairVideoCodec::Unsupported, + } +} + +pub(super) fn select_repair_remux_streams(probe: &SegmentProbe) -> Result { + let mut mapped_streams = Vec::new(); + let mut dropped_streams = Vec::new(); + let mut has_video = false; + for stream in &probe.streams { + match stream.stream_type { + SegmentProbeStreamType::Video if valid_video_stream(stream) => { + has_video = true; + mapped_streams.push(stream.index); + } + SegmentProbeStreamType::Audio if valid_audio_stream(stream) => { + mapped_streams.push(stream.index); + } + SegmentProbeStreamType::Video => dropped_streams + .push(RepairRemuxDroppedStream { index: stream.index, reason: "invalid-video-parameters" }), + SegmentProbeStreamType::Audio => dropped_streams + .push(RepairRemuxDroppedStream { index: stream.index, reason: "invalid-audio-parameters" }), + SegmentProbeStreamType::Other => dropped_streams + .push(RepairRemuxDroppedStream { index: stream.index, reason: "unsupported-stream-type" }), + } + } + if !has_video { + return Err("no_valid_video_stream".to_string()); + } + Ok(RepairRemuxStreamSelection { mapped_streams, dropped_streams }) +} + +fn valid_video_stream(stream: &SegmentProbeStream) -> bool { + stream.codec_name.as_deref().is_some_and(|codec| !codec.is_empty()) + && stream.width.unwrap_or_default() > 0 + && stream.height.unwrap_or_default() > 0 +} + +fn valid_audio_stream(stream: &SegmentProbeStream) -> bool { + stream.codec_name.as_deref().is_some_and(|codec| !codec.is_empty()) + && stream.sample_rate.unwrap_or_default() > 0 + && stream.channels.unwrap_or_default() > 0 +} + +pub(super) fn validate_repair( + config: &HlsSegmentRepairConfig, + codec: RepairVideoCodec, + raw: &SegmentProbe, + fixed: &SegmentProbe, + executed_level: HlsSegmentRepairMode, + stream_selection: &RepairRemuxStreamSelection, +) -> Result<(), String> { + if codec == RepairVideoCodec::Unsupported { + return Err("unsupported_codec".to_string()); + } + let expected_stream_count = if stream_selection.dropped_streams.is_empty() { + raw.stream_count + } else { + stream_selection.mapped_streams.len() + }; + if expected_stream_count != fixed.stream_count { + return Err("stream_count_changed".to_string()); + } + if raw.primary_video_codec != fixed.primary_video_codec { + return Err("video_codec_changed".to_string()); + } + if raw.primary_audio_codec != fixed.primary_audio_codec { + return Err("audio_codec_changed".to_string()); + } + if delta_u64(raw.duration_ms, fixed.duration_ms) > 250 { + return Err("duration_delta_too_large".to_string()); + } + if delta_i64(raw.primary_video_start_time_ms, fixed.primary_video_start_time_ms) > 250 { + return Err("video_start_time_delta_too_large".to_string()); + } + if delta_i64(raw.primary_audio_start_time_ms, fixed.primary_audio_start_time_ms) > 250 { + return Err("audio_start_time_delta_too_large".to_string()); + } + let raw_decision = decide_repair(codec, &raw.warnings); + let fixed_decision = decide_repair(codec, &fixed.warnings); + if raw_decision.required_level != HlsSegmentRepairMode::Off + && fixed_decision.required_level != HlsSegmentRepairMode::Off + { + return Err("repair_triggers_remaining".to_string()); + } + if raw.size > 0 { + let allowed = + raw.size.saturating_add(raw.size.saturating_mul(size_increase_percent(config, executed_level)) / 100); + if fixed.size > allowed { + return Err("size_increase_too_large".to_string()); + } + } + Ok(()) +} + +fn size_increase_percent(config: &HlsSegmentRepairConfig, level: HlsSegmentRepairMode) -> u64 { + match level { + HlsSegmentRepairMode::Off => 0, + HlsSegmentRepairMode::Low => u64::from(config.size_increase.low_percent), + HlsSegmentRepairMode::Medium => u64::from(config.size_increase.medium_percent), + HlsSegmentRepairMode::High => u64::from(config.size_increase.high_percent), + } +} + +fn parse_seconds_ms_u64(value: &str) -> Option { + let (whole, frac) = value.split_once('.').unwrap_or((value, "")); + let whole_ms = whole.parse::().ok()?.checked_mul(1_000)?; + let frac_ms = frac.chars().take(3).collect::(); + let frac_ms = format!("{frac_ms:0<3}").parse::().ok()?; + Some(whole_ms.saturating_add(frac_ms)) +} + +fn parse_seconds_ms_i64(value: &str) -> Option { + let negative = value.starts_with('-'); + let unsigned = value.strip_prefix('-').unwrap_or(value); + let parsed = i64::try_from(parse_seconds_ms_u64(unsigned)?).ok()?; + Some(if negative { -parsed } else { parsed }) +} + +fn delta_u64(lhs: Option, rhs: Option) -> u64 { + match (lhs, rhs) { + (Some(lhs), Some(rhs)) => lhs.abs_diff(rhs), + (None, None) => 0, + _ => u64::MAX, + } +} + +fn delta_i64(lhs: Option, rhs: Option) -> u64 { + match (lhs, rhs) { + (Some(lhs), Some(rhs)) => lhs.abs_diff(rhs), + (None, None) => 0, + _ => u64::MAX, + } +} + +pub(crate) async fn sha256_file(path: &Path) -> io::Result { + let mut file = File::open(path).await?; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 16 * 1024]; + loop { + let read = file.read(&mut buffer).await?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + let digest: [u8; 32] = hasher.finalize().into(); + let mut hex = String::with_capacity(64); + for byte in digest { + use std::fmt::Write as _; + let _ = write!(hex, "{byte:02x}"); + } + Ok(hex) +} + +pub(crate) fn ffmpeg_identity_version() -> String { "system".to_string() } + +pub(super) fn repair_output_path(raw_path: &Path) -> PathBuf { + let suffix = fastrand::u64(..); + let file_name = raw_path.file_name().and_then(|file_name| file_name.to_str()).unwrap_or("segment"); + raw_path.with_file_name(format!("{file_name}.repair.tmp.{suffix:016x}")) +} + +pub(super) async fn cleanup_repair_output(path: &Path) { + match fs::remove_file(path).await { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => warn!("HLS repair fixed staging cleanup deferred: error_kind={:?}", error.kind()), + } +} + +pub(super) async fn adopt_repair_output( + segment_cache: &HlsSegmentCache, + path: PathBuf, + size: u64, +) -> io::Result { + match segment_cache.adopt_staged_file(path.clone(), size) { + Ok(staged) => Ok(staged), + Err(error) => { + cleanup_repair_output(&path).await; + Err(error) + } + } +} + +impl fmt::Display for RepairStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Clean => f.write_str("clean"), + Self::Fixed => f.write_str("fixed"), + Self::PolicyLimited => f.write_str("policy_limited"), + Self::Unsupported => f.write_str("unsupported"), + Self::Timeout => f.write_str("timeout"), + Self::RemuxFailed => f.write_str("remux_failed"), + Self::ValidationFailed => f.write_str("validation_failed"), + } + } +} diff --git a/backend/hls/src/segment_repair/processing.rs b/backend/hls/src/segment_repair/processing.rs new file mode 100644 index 000000000..9106d4934 --- /dev/null +++ b/backend/hls/src/segment_repair/processing.rs @@ -0,0 +1,454 @@ +use super::{ + diagnostics::{debug_repair_analysis, debug_repair_event, debug_repair_stream_dropped, decide_repair}, + ffmpeg_identity_version, parse_ffmpeg_warnings, + probe::{ + adopt_repair_output, cleanup_repair_output, detect_video_codec, parse_probe, repair_output_path, + select_repair_remux_streams, validate_repair, + }, + registry::repair_object_metadata_key, + sha256_file, + window::{RepairCandidateSelection, RepairStatus}, + CachedSegmentMetadata, HlsCacheObjectKey, HlsPostProcessingDeadline, HlsRepairObjectMetadata, + HlsRepairWindowCandidateKey, HlsSegmentCache, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, + HlsSegmentRepairRuntime, RepairIdentity, RepairRemuxStreamSelection, SegmentProbe, StagedCacheObject, + COMMAND_VERSION, +}; +use log::{debug, warn}; +use shared::model::{HlsSegmentRepairExecutionPlan, HlsSegmentRepairMode}; +use std::{ + io, + path::{Path, PathBuf}, + sync::Arc, + time::Instant, +}; +use tokio::{fs, process::Command, time::timeout}; + +impl HlsSegmentRepairManager { + pub(super) async fn process_staged_and_commit( + &self, + segment_cache: &HlsSegmentCache, + key: &K, + raw: StagedCacheObject, + context: HlsSegmentRepairObjectContext, + runtime: Arc, + deadline: HlsPostProcessingDeadline, + ) -> io::Result + where + K: HlsCacheObjectKey, + { + let selected_repair = self.try_select_candidate_with_runtime(&context, &runtime).await; + if selected_repair.is_none() + && runtime.config.corrupt_segment_watchdog.mode.is_enabled() + && context.is_repairable_ts() + { + let raw_hash = sha256_file(&raw.path).await?; + return self + .watchdog + .process_staged_and_commit( + segment_cache, + key, + raw, + &context, + &runtime.config.corrupt_segment_watchdog, + &runtime.watchdog_semaphore, + raw_hash, + &deadline, + ) + .await; + } + let Some(mode) = selected_repair else { + return segment_cache.commit_staged(key, raw).await; + }; + let raw_hash = sha256_file(&raw.path).await?; + let object_key = repair_object_metadata_key(&context, mode); + if self.object_metadata_matches(&object_key, &raw_hash).await { + return segment_cache.commit_staged(key, raw).await; + } + let identity = RepairIdentity { + raw_sha256: raw_hash.clone(), + repair_mode: mode, + command_version: COMMAND_VERSION, + ffmpeg_version: ffmpeg_identity_version(), + }; + if self.repair_metadata(&identity).await.is_some() { + let committed = segment_cache.commit_staged(key, raw).await?; + self.record_object_metadata_from_repair_identity(object_key, raw_hash.clone(), Some(raw_hash), &identity) + .await; + return Ok(committed); + } + let lock = self.lock_for_identity(identity.clone()).await; + let result = { + let _guard = lock.lock().await; + if self.repair_metadata(&identity).await.is_some() { + let committed = segment_cache.commit_staged(key, raw).await?; + self.record_object_metadata_from_repair_identity( + object_key.clone(), + raw_hash.clone(), + Some(raw_hash.clone()), + &identity, + ) + .await; + Ok(committed) + } else { + let raw_size = raw.size; + if let Some(fixed_path) = + self.repair_file(&raw.path, raw.size, &identity, &context, runtime.clone(), &deadline).await? + { + let fixed_size = fs::metadata(&fixed_path).await?.len(); + if let Err(error) = segment_cache.remove_staged(raw.clone()).await { + warn!("HLS repair raw staging cleanup deferred: error_kind={:?}", error.kind()); + } + let fixed = adopt_repair_output(segment_cache, fixed_path, fixed_size).await?; + let committed = segment_cache.commit_staged(key, fixed).await?; + self.record_metadata(identity.clone(), RepairStatus::Fixed, raw.size, committed.size, None).await; + let committed_hash = sha256_file(&committed.path).await?; + self.record_object_metadata( + object_key.clone(), + HlsRepairObjectMetadata { + committed_sha256: committed_hash, + raw_sha256: Some(raw_hash.clone()), + status: RepairStatus::Fixed, + raw_size, + final_size: committed.size, + validation_reason: None, + }, + ) + .await; + Ok(committed) + } else { + let committed = segment_cache.commit_staged(key, raw).await?; + self.record_object_metadata_from_repair_identity( + object_key.clone(), + raw_hash.clone(), + Some(raw_hash.clone()), + &identity, + ) + .await; + Ok(committed) + } + } + }; + self.remove_lock_if_unused(&identity, &lock).await; + result + } + + pub(super) async fn try_select_candidate( + &self, + context: &HlsSegmentRepairObjectContext, + ) -> Option<(HlsSegmentRepairMode, Arc, HlsRepairWindowCandidateKey)> { + let runtime = self.runtime.load_full(); + match self.select_candidate(context, &runtime).await { + RepairCandidateSelection::Selected(mode, candidate) => Some((mode, runtime, candidate)), + RepairCandidateSelection::Skipped(_) | RepairCandidateSelection::AlreadyChecked(_, _) => None, + } + } + + pub(super) async fn try_select_or_join_candidate( + &self, + context: &HlsSegmentRepairObjectContext, + ) -> Option<(HlsSegmentRepairMode, Arc, HlsRepairWindowCandidateKey)> { + let runtime = self.runtime.load_full(); + match self.select_candidate(context, &runtime).await { + RepairCandidateSelection::Selected(mode, candidate) + | RepairCandidateSelection::AlreadyChecked(mode, candidate) => Some((mode, runtime, candidate)), + RepairCandidateSelection::Skipped(_) => None, + } + } + + pub(super) async fn try_select_candidate_with_runtime( + &self, + context: &HlsSegmentRepairObjectContext, + runtime: &Arc, + ) -> Option { + match self.select_candidate(context, runtime).await { + RepairCandidateSelection::Selected(mode, _) => Some(mode), + RepairCandidateSelection::Skipped(_) | RepairCandidateSelection::AlreadyChecked(_, _) => None, + } + } + + pub(super) async fn select_candidate( + &self, + context: &HlsSegmentRepairObjectContext, + runtime: &Arc, + ) -> RepairCandidateSelection { + if !runtime.repair_enabled() { + return RepairCandidateSelection::Skipped("disabled"); + } + if context.hls_access_lease_id.is_none() { + return RepairCandidateSelection::Skipped("missing-lease"); + } + let selection = self.windows.write().await.try_select_candidate(context); + match &selection { + RepairCandidateSelection::Selected(mode, _) => debug!( + "HLS segment repair candidate selected: {} source={} resource={} mode={}", + context.log_identity_fields(), + context.source.as_log_value(), + context.resource_id, + mode.as_log_value() + ), + RepairCandidateSelection::Skipped(reason) => debug!( + "HLS segment repair candidate skipped: {} source={} resource={} reason={}", + context.log_identity_fields(), + context.source.as_log_value(), + context.resource_id, + reason + ), + RepairCandidateSelection::AlreadyChecked(_, _) => {} + } + selection + } + + #[allow(clippy::too_many_lines)] + pub(super) async fn repair_file( + &self, + raw_path: &Path, + raw_size: u64, + identity: &RepairIdentity, + context: &HlsSegmentRepairObjectContext, + runtime: Arc, + deadline: &HlsPostProcessingDeadline, + ) -> io::Result> { + let _permit = match &runtime.semaphore { + Some(semaphore) => { + let Some(remaining) = deadline.remaining() else { + self.record_metadata( + identity.clone(), + RepairStatus::Timeout, + raw_size, + raw_size, + Some("timeout".to_string()), + ) + .await; + return Ok(None); + }; + Some( + timeout(remaining, semaphore.acquire()) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "repair semaphore timed out"))? + .map_err(|_| io::Error::other("repair semaphore closed"))?, + ) + } + None => None, + }; + let started = Instant::now(); + let raw_scan = match analyze_segment(raw_path, deadline).await { + Ok(scan) => scan, + Err(reason) => { + debug_repair_event(context, identity.repair_mode, "analysis skipped", Some(&reason)); + self.record_metadata(identity.clone(), RepairStatus::Unsupported, raw_size, raw_size, Some(reason)) + .await; + return Ok(None); + } + }; + let codec = detect_video_codec(&raw_scan); + let decision = decide_repair(codec, &raw_scan.warnings); + let execution_plan = identity.repair_mode.execution_plan(decision.required_level); + let executed_level = match execution_plan { + HlsSegmentRepairExecutionPlan::Repair(level) => Some(level), + HlsSegmentRepairExecutionPlan::SkipNoTrigger + | HlsSegmentRepairExecutionPlan::SkipConfiguredMaxBelowRequired => None, + }; + debug_repair_analysis(context, identity.repair_mode, decision, executed_level, &raw_scan.warnings); + let executed_level = match execution_plan { + HlsSegmentRepairExecutionPlan::Repair(level) => level, + HlsSegmentRepairExecutionPlan::SkipNoTrigger => { + self.record_metadata(identity.clone(), RepairStatus::Clean, raw_size, raw_size, None).await; + return Ok(None); + } + HlsSegmentRepairExecutionPlan::SkipConfiguredMaxBelowRequired => { + warn!( + "HLS segment repair required level exceeds configured max: {} source={} resource={} configured_max_level={} required_level={} trigger={} action=raw_commit", + context.log_identity_fields(), + context.source.as_log_value(), + context.resource_id, + identity.repair_mode.as_log_value(), + decision.required_level.as_log_value(), + decision.trigger_source.as_log_value() + ); + self.record_metadata( + identity.clone(), + RepairStatus::PolicyLimited, + raw_size, + raw_size, + Some("configured_max_level_below_required_level".to_string()), + ) + .await; + return Ok(None); + } + }; + let stream_selection = match select_repair_remux_streams(&raw_scan) { + Ok(selection) => selection, + Err(reason) => { + debug_repair_event(context, executed_level, "stream selection skipped", Some(&reason)); + self.record_metadata(identity.clone(), RepairStatus::Unsupported, raw_size, raw_size, Some(reason)) + .await; + return Ok(None); + } + }; + for dropped in &stream_selection.dropped_streams { + debug_repair_stream_dropped(context, dropped); + } + let fixed_path = repair_output_path(raw_path); + debug!( + "HLS segment repair remux started: {} source={} resource={} configured_max_level={} executed_level={}", + context.log_identity_fields(), + context.source.as_log_value(), + context.resource_id, + identity.repair_mode.as_log_value(), + executed_level.as_log_value() + ); + let remux = run_remux(raw_path, &fixed_path, executed_level, &stream_selection, deadline).await; + if let Err(reason) = remux { + debug_repair_event(context, executed_level, "remux failed", Some(&reason)); + let status = if reason == "timeout" { RepairStatus::Timeout } else { RepairStatus::RemuxFailed }; + cleanup_repair_output(&fixed_path).await; + self.record_metadata(identity.clone(), status, raw_size, raw_size, Some(reason)).await; + return Ok(None); + } + let fixed_scan = match analyze_segment(&fixed_path, deadline).await { + Ok(scan) => scan, + Err(reason) => { + debug_repair_event(context, executed_level, "validation probe failed", Some(&reason)); + cleanup_repair_output(&fixed_path).await; + self.record_metadata( + identity.clone(), + RepairStatus::ValidationFailed, + raw_size, + raw_size, + Some(reason), + ) + .await; + return Ok(None); + } + }; + let validation = + validate_repair(&runtime.config, codec, &raw_scan, &fixed_scan, executed_level, &stream_selection); + if let Err(reason) = validation { + debug_repair_event(context, executed_level, "validation failed", Some(&reason)); + cleanup_repair_output(&fixed_path).await; + self.record_metadata(identity.clone(), RepairStatus::ValidationFailed, raw_size, raw_size, Some(reason)) + .await; + return Ok(None); + } + debug!( + "HLS segment repair remux completed: {} source={} resource={} configured_max_level={} executed_level={} duration_ms={}", + context.log_identity_fields(), + context.source.as_log_value(), + context.resource_id, + identity.repair_mode.as_log_value(), + executed_level.as_log_value(), + started.elapsed().as_millis() + ); + Ok(Some(fixed_path)) + } +} + +async fn analyze_segment(path: &Path, deadline: &HlsPostProcessingDeadline) -> Result { + let probe_output = run_command_with_deadline( + "ffprobe", + &[ + "-hide_banner", + "-v", + "warning", + "-show_entries", + "format=duration,size,bit_rate", + "-show_streams", + "-of", + "json", + path.to_str().ok_or_else(|| "invalid_path".to_string())?, + ], + deadline, + ) + .await?; + let warnings_output = run_command_with_deadline( + "ffmpeg", + &[ + "-hide_banner", + "-nostdin", + "-v", + "warning", + "-i", + path.to_str().ok_or_else(|| "invalid_path".to_string())?, + "-map", + "0", + "-c", + "copy", + "-f", + "null", + "-", + ], + deadline, + ) + .await + .unwrap_or_else(|stderr| stderr); + parse_probe(&probe_output, parse_ffmpeg_warnings(&warnings_output)) +} + +async fn run_remux( + input_path: &Path, + output_path: &Path, + mode: HlsSegmentRepairMode, + stream_selection: &RepairRemuxStreamSelection, + deadline: &HlsPostProcessingDeadline, +) -> Result<(), String> { + let input = input_path.to_str().ok_or_else(|| "invalid_input_path".to_string())?; + let output = output_path.to_str().ok_or_else(|| "invalid_output_path".to_string())?; + let mut args = ["-hide_banner", "-nostdin", "-y", "-copyts", "-i", input] + .into_iter() + .map(ToOwned::to_owned) + .collect::>(); + for stream_index in &stream_selection.mapped_streams { + args.push("-map".to_string()); + args.push(format!("0:{stream_index}")); + } + args.extend(["-c", "copy"].into_iter().map(ToOwned::to_owned)); + if matches!(mode, HlsSegmentRepairMode::Medium | HlsSegmentRepairMode::High) { + args.push("-bsf:v".to_string()); + args.push("dump_extra=freq=keyframe".to_string()); + } + args.push("-mpegts_flags".to_string()); + args.push( + if mode == HlsSegmentRepairMode::High { "+resend_headers+pat_pmt_at_frames" } else { "+resend_headers" } + .to_string(), + ); + args.extend( + ["-mpegts_copyts", "1", "-muxpreload", "0", "-muxdelay", "0", "-f", "mpegts", output] + .into_iter() + .map(ToOwned::to_owned), + ); + let args = args.iter().map(String::as_str).collect::>(); + run_command_with_deadline("ffmpeg", &args, deadline).await.map(|_| ()) +} + +pub(crate) async fn run_command_with_deadline( + binary: &str, + args: &[&str], + deadline: &HlsPostProcessingDeadline, +) -> Result { + let Some(remaining) = deadline.remaining() else { + return Err("timeout".to_string()); + }; + let output = timeout(remaining, { + let mut command = Command::new(binary); + command.args(args).stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::piped()).kill_on_drop(true); + command.output() + }) + .await + .map_err(|_| "timeout".to_string())? + .map_err( + |err| { + if err.kind() == io::ErrorKind::NotFound { + "unsupported".to_string() + } else { + err.to_string() + } + }, + )?; + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + if output.status.success() { + Ok(if stdout.is_empty() { stderr } else { stdout }) + } else { + Err(if stderr.is_empty() { "command_failed".to_string() } else { stderr }) + } +} diff --git a/backend/hls/src/segment_repair/registry.rs b/backend/hls/src/segment_repair/registry.rs new file mode 100644 index 000000000..e7c840485 --- /dev/null +++ b/backend/hls/src/segment_repair/registry.rs @@ -0,0 +1,161 @@ +use super::{ + ffmpeg_identity_version, window::RepairCandidateSelection, HlsAccessLeaseId, HlsRepairObjectMetadataKey, + HlsRepairWindow, HlsRepairWindowCandidateKey, HlsRepairWindowRegistry, HlsSegmentRepairObjectContext, + ProxySessionId, COMMAND_VERSION, REPAIR_CANDIDATE_MAX_ENTRIES, +}; +use shared::model::HlsSegmentRepairMode; +use std::collections::HashSet; +use tuliprox_core::model::HlsSegmentRepairConfig; + +impl HlsRepairWindowRegistry { + pub(super) fn start_window(&mut self, lease_id: HlsAccessLeaseId, config: &HlsSegmentRepairConfig) { + if config.max_level == HlsSegmentRepairMode::Off || config.apply_to_first_segments == 0 { + self.windows.remove(&lease_id); + return; + } + let generation = self + .generations + .entry(lease_id.clone()) + .and_modify(|generation| *generation = generation.saturating_add(1)) + .or_insert(1); + self.windows.insert( + lease_id, + HlsRepairWindow { + mode: config.max_level, + activation_generation: *generation, + remaining_segments: config.apply_to_first_segments, + seen_candidates: HashSet::new(), + }, + ); + } + + pub(super) fn ensure_window(&mut self, lease_id: HlsAccessLeaseId, config: &HlsSegmentRepairConfig) -> bool { + if self.windows.contains_key(&lease_id) { + return false; + } + self.start_window(lease_id, config); + true + } + + pub(super) fn try_select_candidate(&mut self, context: &HlsSegmentRepairObjectContext) -> RepairCandidateSelection { + if let Some(reason) = context.repair_skip_reason() { + return RepairCandidateSelection::Skipped(reason); + } + let Some(lease_id) = context.hls_access_lease_id.as_ref() else { + return RepairCandidateSelection::Skipped("missing-lease"); + }; + let Some((activation_generation, mode)) = + self.windows.get(lease_id).map(|window| (window.activation_generation, window.mode)) + else { + return RepairCandidateSelection::Skipped("no-window"); + }; + let candidate_key = HlsRepairWindowCandidateKey { + proxy_session_id: context.proxy_session_id.clone(), + hls_access_lease_id: lease_id.clone(), + activation_generation, + object_id: context.rendered_object_id.clone(), + file_ext: context.file_ext.clone(), + }; + let Some(window) = self.windows.get(lease_id) else { + return RepairCandidateSelection::Skipped("no-window"); + }; + if window.seen_candidates.contains(&candidate_key) { + return RepairCandidateSelection::AlreadyChecked(mode, candidate_key); + } + if window.remaining_segments == 0 { + return RepairCandidateSelection::Skipped("window-exhausted"); + } + if !self.remember_candidate(candidate_key.clone()) { + return RepairCandidateSelection::Skipped("candidate-not-admitted"); + } + let Some(window) = self.windows.get_mut(lease_id) else { + return RepairCandidateSelection::Skipped("no-window"); + }; + window.seen_candidates.insert(candidate_key.clone()); + window.remaining_segments = window.remaining_segments.saturating_sub(1); + RepairCandidateSelection::Selected(window.mode, candidate_key) + } + + pub(super) fn candidate_is_current(&self, candidate: &HlsRepairWindowCandidateKey) -> bool { + self.windows.get(&candidate.hls_access_lease_id).is_some_and(|window| { + window.activation_generation == candidate.activation_generation + && window.seen_candidates.contains(candidate) + }) + } + + pub(super) fn remove_access_lease(&mut self, lease_id: &HlsAccessLeaseId) { + self.windows.remove(lease_id); + self.generations.remove(lease_id); + self.checked_candidates.retain(|key| key.hls_access_lease_id != *lease_id); + self.checked_candidate_order.retain(|key| key.hls_access_lease_id != *lease_id); + } + + pub(super) fn remove_proxy_session(&mut self, proxy_session_id: &ProxySessionId, lease_ids: &[HlsAccessLeaseId]) { + for lease_id in lease_ids { + self.windows.remove(lease_id); + self.generations.remove(lease_id); + } + self.checked_candidates.retain(|key| key.proxy_session_id != *proxy_session_id); + self.checked_candidate_order.retain(|key| key.proxy_session_id != *proxy_session_id); + } + + pub(super) fn clear(&mut self) { + self.windows.clear(); + self.generations.clear(); + self.checked_candidates.clear(); + self.checked_candidate_order.clear(); + } + + pub(super) fn stats(&self) -> HlsSegmentRepairStats { + HlsSegmentRepairStats { + windows: self.windows.len(), + generations: self.generations.len(), + checked_candidates: self.checked_candidates.len(), + ..HlsSegmentRepairStats::default() + } + } + + pub(super) fn remember_candidate(&mut self, key: HlsRepairWindowCandidateKey) -> bool { + if !self.checked_candidates.insert(key.clone()) { + return false; + } + self.checked_candidate_order.push_back(key); + self.prune_checked_candidates(); + true + } + + pub(super) fn prune_checked_candidates(&mut self) { + while self.checked_candidates.len() > REPAIR_CANDIDATE_MAX_ENTRIES { + let Some(oldest) = self.checked_candidate_order.pop_front() else { + return; + }; + self.checked_candidates.remove(&oldest); + } + } +} + +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] +pub struct HlsSegmentRepairStats { + pub windows: usize, + pub generations: usize, + pub checked_candidates: usize, + pub metadata: usize, + pub object_metadata: usize, + pub locks: usize, + pub watchdog_metadata: usize, + pub watchdog_locks: usize, +} + +pub(super) fn repair_object_metadata_key( + context: &HlsSegmentRepairObjectContext, + repair_mode: HlsSegmentRepairMode, +) -> HlsRepairObjectMetadataKey { + HlsRepairObjectMetadataKey { + proxy_session_id: context.proxy_session_id.clone(), + rendered_object_id: context.rendered_object_id.clone(), + file_ext: context.file_ext.to_ascii_lowercase(), + repair_mode, + command_version: COMMAND_VERSION, + ffmpeg_version: ffmpeg_identity_version(), + } +} diff --git a/backend/hls/src/segment_repair/tests.rs b/backend/hls/src/segment_repair/tests.rs new file mode 100644 index 000000000..b11399824 --- /dev/null +++ b/backend/hls/src/segment_repair/tests.rs @@ -0,0 +1,65 @@ +use super::{ + super::{ + media_reserve::{ + HlsLeaseManifestSegment, HlsLeaseManifestSnapshot, HlsManifestCommitIdentity, HlsManifestDeliveryMode, + }, + terminal_tail::HlsMediaContainer, + }, + parse_ffmpeg_warnings, + probe::{adopt_repair_output, detect_video_codec, parse_probe, select_repair_remux_streams, validate_repair}, + ready_segment_repair_prewarm_candidates, + registry::repair_object_metadata_key, + sha256_file, + window::{RepairStatus, RepairVideoCodec}, + HlsRepairObjectMetadata, HlsRepairRenderedObjectId, HlsSegmentRepairManager, HlsSegmentRepairObjectContext, + HlsSegmentRepairSource, RepairIdentity, RepairRemuxStreamSelection, WarningCounters, REPAIR_METADATA_MAX_ENTRIES, +}; +use crate::{ + timeline::HLS_PROVISIONING_ORIGIN_EPOCH, CacheAccessState, HlsAccessLeaseId, HlsSegmentCache, HlsSegmentEncryption, + HlsSession, HlsSessionKey, OriginSegmentKey, ProxySessionId, SegmentCacheKey, SegmentCacheStatus, SegmentEntry, + TransientObjectCacheKey, TransientResourceId, +}; +use shared::model::HlsSegmentRepairMode; +use std::{sync::Arc, time::Duration}; +use tuliprox_core::model::HlsSegmentRepairConfig; + +fn repair_config(mode: HlsSegmentRepairMode, apply_to_first_segments: u8) -> HlsSegmentRepairConfig { + HlsSegmentRepairConfig { max_level: mode, apply_to_first_segments, max_parallel_repairs: 1, ..Default::default() } +} + +fn should_repair(codec: RepairVideoCodec, warnings: &WarningCounters) -> bool { + super::diagnostics::decide_repair(codec, warnings).required_level != HlsSegmentRepairMode::Off +} + +fn repair_context(lease_id: &str, resource_id: &str) -> HlsSegmentRepairObjectContext { + HlsSegmentRepairObjectContext { + source: HlsSegmentRepairSource::Normal, + log_identity: super::HlsLogIdentity::for_test("input:1|hls|stream-a", "proxy-session"), + proxy_session_id: ProxySessionId("proxy-session".to_string()), + hls_access_lease_id: Some(HlsAccessLeaseId(lease_id.to_string())), + rendered_object_id: HlsRepairRenderedObjectId::Normal { proxy_seq: resource_id.parse().unwrap_or(1) }, + resource_id: resource_id.to_string(), + file_ext: "ts".to_string(), + origin_fetch_uri_for_diagnostics: format!("http://origin.example/{resource_id}.ts"), + media_sequence: Some(1), + discontinuity_sequence: Some(0), + complete_object: true, + encrypted: false, + custom_response: false, + } +} + +async fn selected_repair_mode( + manager: &HlsSegmentRepairManager, + context: &HlsSegmentRepairObjectContext, +) -> Option { + manager.try_select_candidate(context).await.map(|(mode, _, _)| mode) +} + +mod lifecycle; +mod policy; +mod publication; +mod storage; +mod streaming; +mod terminal; +mod transport; diff --git a/backend/hls/src/segment_repair/tests/lifecycle.rs b/backend/hls/src/segment_repair/tests/lifecycle.rs new file mode 100644 index 000000000..f1a1d2305 --- /dev/null +++ b/backend/hls/src/segment_repair/tests/lifecycle.rs @@ -0,0 +1,196 @@ +use super::*; + +#[tokio::test] +async fn failed_repair_output_adoption_removes_the_unowned_file() { + let cache_root = tempfile::tempdir().expect("cache root"); + let outside = tempfile::tempdir().expect("outside root"); + let output = outside.path().join("fixed.ts"); + tokio::fs::write(&output, b"fixed").await.expect("write output"); + let cache = HlsSegmentCache::with_cache_path(cache_root.path()); + + assert!(adopt_repair_output(&cache, output.clone(), 5).await.is_err()); + assert!(!output.exists()); +} + +#[test] +fn repair_remux_selection_drops_invalid_audio_side_stream() { + let probe = parse_probe( + r#"{ + "streams": [ + { "index": 0, "codec_type": "video", "codec_name": "hevc", "width": 1916, "height": 1080 }, + { "index": 1, "codec_type": "audio", "codec_name": "ac3", "sample_rate": "48000", "channels": 6 }, + { "index": 2, "codec_type": "audio", "codec_name": "ac3", "channels": 0 } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters::default(), + ) + .expect("probe should parse"); + + let selection = select_repair_remux_streams(&probe).expect("valid video should allow remux"); + + assert_eq!(selection.mapped_streams, vec![0, 1]); + assert_eq!(selection.dropped_streams.len(), 1); + assert_eq!(selection.dropped_streams[0].index, 2); + assert_eq!(selection.dropped_streams[0].reason, "invalid-audio-parameters"); +} + +#[test] +fn repair_validation_allows_configured_stream_drop() { + let raw = parse_probe( + r#"{ + "streams": [ + { "index": 0, "codec_type": "video", "codec_name": "hevc", "width": 1916, "height": 1080, "start_time": "0.000000" }, + { "index": 1, "codec_type": "audio", "codec_name": "ac3", "sample_rate": "48000", "channels": 6, "start_time": "0.000000" }, + { "index": 2, "codec_type": "audio", "codec_name": "ac3", "channels": 0, "start_time": "0.000000" } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters { codec_parameters_missing: 1, ..WarningCounters::default() }, + ) + .expect("raw probe should parse"); + let fixed = parse_probe( + r#"{ + "streams": [ + { "index": 0, "codec_type": "video", "codec_name": "hevc", "width": 1916, "height": 1080, "start_time": "0.000000" }, + { "index": 1, "codec_type": "audio", "codec_name": "ac3", "sample_rate": "48000", "channels": 6, "start_time": "0.000000" } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters::default(), + ) + .expect("fixed probe should parse"); + let selection = select_repair_remux_streams(&raw).expect("raw should select valid streams"); + + assert!(validate_repair( + &repair_config(HlsSegmentRepairMode::Low, 1), + RepairVideoCodec::Hevc, + &raw, + &fixed, + HlsSegmentRepairMode::Low, + &selection + ) + .is_ok()); +} + +#[test] +fn hevc_validation_accepts_when_repair_triggers_are_removed() { + let raw = parse_probe( + r#"{ + "streams": [ + { "codec_type": "video", "codec_name": "hevc", "start_time": "0.000000" }, + { "codec_type": "audio", "codec_name": "aac", "start_time": "0.000000" } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters { + pps_id_out_of_range: 1, + invalid_undecodable_nalu_non_metadata: 2, + invalid_undecodable_nalu_metadata: 1, + dolby_vision_rpu: 1, + ..WarningCounters::default() + }, + ) + .expect("raw probe should parse"); + let fixed = parse_probe( + r#"{ + "streams": [ + { "codec_type": "video", "codec_name": "hevc", "start_time": "0.000000" }, + { "codec_type": "audio", "codec_name": "aac", "start_time": "0.000000" } + ], + "format": { "duration": "2.000000", "size": "1010" } + }"#, + WarningCounters { invalid_undecodable_nalu_metadata: 1, dolby_vision_rpu: 1, ..WarningCounters::default() }, + ) + .expect("fixed probe should parse"); + + assert!(validate_repair( + &repair_config(HlsSegmentRepairMode::Medium, 1), + RepairVideoCodec::Hevc, + &raw, + &fixed, + HlsSegmentRepairMode::Medium, + &RepairRemuxStreamSelection::preserve_all(&raw) + ) + .is_ok()); +} + +#[tokio::test] +async fn remove_access_lease_window_clears_checked_candidates() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + + manager.start_access_lease_window(lease_id.clone()).await; + assert_eq!( + selected_repair_mode(&manager, &repair_context("lease-a", "000001")).await, + Some(HlsSegmentRepairMode::Low) + ); + assert_eq!(manager.windows.read().await.checked_candidates.len(), 1); + + manager.remove_access_lease_window(&lease_id).await; + + assert_eq!(manager.windows.read().await.checked_candidates.len(), 0); +} + +#[tokio::test] +async fn remove_proxy_session_state_clears_checked_candidates() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.start_access_lease_window(lease_id.clone()).await; + let context = repair_context("lease-a", "000001"); + + assert_eq!(selected_repair_mode(&manager, &context).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(manager.windows.read().await.checked_candidates.len(), 1); + + manager.remove_proxy_session_state(&proxy_session_id, &[lease_id]).await; + + assert_eq!(manager.windows.read().await.checked_candidates.len(), 0); +} + +#[tokio::test] +async fn remove_proxy_session_state_clears_object_metadata() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let context = repair_context("lease-a", "000001"); + manager + .record_object_metadata( + repair_object_metadata_key(&context, HlsSegmentRepairMode::Low), + HlsRepairObjectMetadata { + committed_sha256: "hash".to_string(), + raw_sha256: Some("raw".to_string()), + status: RepairStatus::Clean, + raw_size: 1, + final_size: 1, + validation_reason: None, + }, + ) + .await; + + assert_eq!(manager.stats().await.object_metadata, 1); + + manager.remove_proxy_session_state(&proxy_session_id, &[lease_id]).await; + + assert_eq!(manager.stats().await.object_metadata, 0); +} + +#[tokio::test] +async fn repair_lock_cleanup_keeps_waited_lock_and_removes_unused_lock() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let identity = RepairIdentity { + raw_sha256: "a".repeat(64), + repair_mode: HlsSegmentRepairMode::Low, + command_version: 1, + ffmpeg_version: "test".to_string(), + }; + let lock = manager.lock_for_identity(identity.clone()).await; + let waiter = Arc::clone(&lock); + + manager.remove_lock_if_unused(&identity, &lock).await; + assert_eq!(manager.stats().await.locks, 1); + + drop(waiter); + manager.remove_lock_if_unused(&identity, &lock).await; + assert_eq!(manager.stats().await.locks, 0); +} diff --git a/backend/hls/src/segment_repair/tests/policy.rs b/backend/hls/src/segment_repair/tests/policy.rs new file mode 100644 index 000000000..8f80afb8b --- /dev/null +++ b/backend/hls/src/segment_repair/tests/policy.rs @@ -0,0 +1,382 @@ +use super::*; + +#[tokio::test] +async fn update_config_applies_to_new_access_lease_windows() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Medium, 2)); + let mut updated = repair_config(HlsSegmentRepairMode::Medium, 3); + updated.max_parallel_repairs = 2; + manager.update_config(updated); + + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + + for resource_id in ["1", "2", "3"] { + assert!( + manager.try_select_candidate(&repair_context("lease-a", resource_id)).await.is_some(), + "resource {resource_id} should be inside updated repair window" + ); + } + assert!( + manager.try_select_candidate(&repair_context("lease-a", "4")).await.is_none(), + "fourth resource should be outside updated repair window" + ); +} + +#[test] +fn warning_parser_expands_repeated_messages() { + let warnings = parse_ffmpeg_warnings( + "non-existing SPS 0 referenced in buffering period\nLast message repeated 2 times\nno frame!\n", + ); + + assert_eq!(warnings.missing_sps, 3); + assert_eq!(warnings.no_frame, 1); +} + +#[test] +fn mmco_warning_alone_does_not_trigger_repair() { + let warnings = WarningCounters { mmco_unref_short_failure: 20, ..WarningCounters::default() }; + + assert!(!should_repair(RepairVideoCodec::H264, &warnings)); +} + +#[test] +fn critical_warning_triggers_repair() { + let warnings = WarningCounters { missing_sps: 1, ..WarningCounters::default() }; + + assert!(should_repair(RepairVideoCodec::H264, &warnings)); +} + +#[test] +fn hevc_pps_warning_triggers_repair() { + let warnings = parse_ffmpeg_warnings("[hevc @ 0x1] PPS id out of range: 0\n"); + + assert_eq!(warnings.pps_id_out_of_range, 1); + assert!(should_repair(RepairVideoCodec::Hevc, &warnings)); +} + +#[test] +fn hevc_invalid_slice_nalus_with_parameter_issue_trigger_repair() { + let warnings = parse_ffmpeg_warnings( + "[hevc @ 0x1] missing SPS\n[hevc @ 0x1] Skipping invalid undecodable NALU: 0\n[hevc @ 0x1] Skipping invalid undecodable NALU: 1\n", + ); + + assert_eq!(warnings.invalid_undecodable_nalu_total, 2); + assert_eq!(warnings.invalid_undecodable_nalu_non_metadata, 2); + assert!(should_repair(RepairVideoCodec::Hevc, &warnings)); +} + +#[test] +fn hevc_metadata_and_dolby_warnings_do_not_trigger_repair_alone() { + let warnings = parse_ffmpeg_warnings( + "[hevc @ 0x1] Skipping invalid undecodable NALU: 39\nMultiple Dolby Vision RPUs found in one AU. Skipping previous.\nAudio/Video desynchronisation detected!\n", + ); + + assert_eq!(warnings.invalid_undecodable_nalu_total, 1); + assert_eq!(warnings.invalid_undecodable_nalu_metadata, 1); + assert_eq!(warnings.dolby_vision_rpu, 1); + assert_eq!(warnings.av_desync, 1); + assert!(!should_repair(RepairVideoCodec::Hevc, &warnings)); +} + +#[test] +fn hevc_repeated_messages_expand_counters_without_triggering_alone() { + let warnings = + parse_ffmpeg_warnings("[hevc @ 0x1] Skipping invalid undecodable NALU: 0\nLast message repeated 2 times\n"); + + assert_eq!(warnings.invalid_undecodable_nalu_total, 3); + assert_eq!(warnings.invalid_undecodable_nalu_non_metadata, 3); + assert!(!should_repair(RepairVideoCodec::Hevc, &warnings)); +} + +#[test] +fn warning_parser_matches_trigger_patterns_case_insensitively() { + let warnings = parse_ffmpeg_warnings( + "NON-EXISTING SPS 0 referenced\ninvalid nal unit 1\ncould not find codec parameters for stream 0\n", + ); + + assert_eq!(warnings.missing_sps, 1); + assert_eq!(warnings.invalid_nal, 1); + assert_eq!(warnings.codec_parameters_missing, 1); + assert!(should_repair(RepairVideoCodec::H264, &warnings)); +} + +#[test] +fn unsupported_codec_never_triggers_repair() { + let warnings = WarningCounters { missing_sps: 1, pps_id_out_of_range: 1, ..WarningCounters::default() }; + + assert!(!should_repair(RepairVideoCodec::Unsupported, &warnings)); +} + +#[test] +fn configured_max_below_required_skips_repair() { + assert_eq!( + HlsSegmentRepairMode::Low.execution_plan(HlsSegmentRepairMode::High), + shared::model::HlsSegmentRepairExecutionPlan::SkipConfiguredMaxBelowRequired + ); + assert_eq!( + HlsSegmentRepairMode::Medium.execution_plan(HlsSegmentRepairMode::Low), + shared::model::HlsSegmentRepairExecutionPlan::Repair(HlsSegmentRepairMode::Low) + ); + assert_eq!( + HlsSegmentRepairMode::Off.execution_plan(HlsSegmentRepairMode::High), + shared::model::HlsSegmentRepairExecutionPlan::SkipNoTrigger + ); +} + +#[tokio::test] +async fn background_candidate_without_access_lease_is_ignored_before_window_check() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + let mut context = repair_context("lease-a", "000001"); + context.hls_access_lease_id = None; + + assert_eq!(selected_repair_mode(&manager, &context).await, None); + + let stats = manager.stats().await; + assert_eq!(stats.windows, 1); + assert_eq!(stats.checked_candidates, 0); + assert_eq!(stats.object_metadata, 0); +} + +#[tokio::test] +async fn repair_window_is_separate_per_access_lease() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Medium, 1)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + manager.start_access_lease_window(HlsAccessLeaseId("lease-b".to_string())).await; + + assert_eq!( + selected_repair_mode(&manager, &repair_context("lease-a", "000001")).await, + Some(HlsSegmentRepairMode::Medium) + ); + assert_eq!( + selected_repair_mode(&manager, &repair_context("lease-b", "000001")).await, + Some(HlsSegmentRepairMode::Medium) + ); +} + +#[tokio::test] +async fn known_normal_object_consumes_new_lease_window_without_rescan() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let cache_key = SegmentCacheKey::new(ProxySessionId("proxy-session".to_string()), 1, "ts"); + let metadata = cache.write_bytes_and_commit(&cache_key, b"cached-normal-bytes").await.expect("commit"); + let committed_sha256 = sha256_file(&metadata.path).await.expect("hash"); + let previous_context = repair_context("lease-a", "1"); + manager + .record_object_metadata( + repair_object_metadata_key(&previous_context, HlsSegmentRepairMode::Low), + HlsRepairObjectMetadata { + committed_sha256, + raw_sha256: Some("previous-raw".to_string()), + status: RepairStatus::Clean, + raw_size: metadata.size, + final_size: metadata.size, + validation_reason: None, + }, + ) + .await; + + manager.start_access_lease_window(HlsAccessLeaseId("lease-b".to_string())).await; + + assert!(manager + .repair_ready_cache_hit(&cache, &cache_key, repair_context("lease-b", "1")) + .await + .expect("repair cache hit") + .is_none()); + assert_eq!(manager.stats().await.metadata, 0); + assert_eq!(selected_repair_mode(&manager, &repair_context("lease-b", "2")).await, None); +} + +#[tokio::test] +async fn known_transient_object_consumes_new_lease_window_without_rescan() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let cache_key = TransientObjectCacheKey::new( + ProxySessionId("proxy-session".to_string()), + TransientResourceId("resource-a".to_string()), + "ts", + ); + let metadata = cache.write_bytes_and_commit(&cache_key, b"cached-transient-bytes").await.expect("commit"); + let committed_sha256 = sha256_file(&metadata.path).await.expect("hash"); + let mut previous_context = repair_context("lease-a", "1"); + previous_context.source = HlsSegmentRepairSource::Transient; + previous_context.rendered_object_id = + HlsRepairRenderedObjectId::Transient { resource_id: "resource-a".to_string() }; + previous_context.resource_id = "resource-a".to_string(); + manager + .record_object_metadata( + repair_object_metadata_key(&previous_context, HlsSegmentRepairMode::Low), + HlsRepairObjectMetadata { + committed_sha256, + raw_sha256: Some("previous-raw".to_string()), + status: RepairStatus::Clean, + raw_size: metadata.size, + final_size: metadata.size, + validation_reason: None, + }, + ) + .await; + + manager.start_access_lease_window(HlsAccessLeaseId("lease-b".to_string())).await; + let mut current_context = previous_context.clone(); + current_context.hls_access_lease_id = Some(HlsAccessLeaseId("lease-b".to_string())); + + assert!(manager + .repair_ready_cache_hit(&cache, &cache_key, current_context) + .await + .expect("repair cache hit") + .is_none()); + assert_eq!(manager.stats().await.metadata, 0); + let mut second_context = repair_context("lease-b", "2"); + second_context.source = HlsSegmentRepairSource::Transient; + second_context.rendered_object_id = HlsRepairRenderedObjectId::Transient { resource_id: "resource-b".to_string() }; + second_context.resource_id = "resource-b".to_string(); + assert_eq!(selected_repair_mode(&manager, &second_context).await, None); +} + +#[tokio::test] +async fn object_metadata_hash_mismatch_does_not_skip_repair_evaluation() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let context = repair_context("lease-a", "1"); + let object_key = repair_object_metadata_key(&context, HlsSegmentRepairMode::Low); + manager + .record_object_metadata( + object_key.clone(), + HlsRepairObjectMetadata { + committed_sha256: "old-hash".to_string(), + raw_sha256: Some("old-raw".to_string()), + status: RepairStatus::Clean, + raw_size: 1, + final_size: 1, + validation_reason: None, + }, + ) + .await; + + assert!(!manager.object_metadata_matches(&object_key, "new-hash").await); +} + +#[tokio::test] +async fn repair_disabled_does_not_track_candidates() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Off, 1)); + let context = repair_context("lease-a", "000001"); + + assert_eq!(selected_repair_mode(&manager, &context).await, None); + + let registry = manager.windows.read().await; + assert_eq!(registry.checked_candidates.len(), 0); +} + +#[tokio::test] +async fn transient_commit_and_cache_hit_consume_repair_window_once() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 2)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + let mut commit_context = repair_context("lease-a", "1"); + commit_context.source = HlsSegmentRepairSource::Transient; + commit_context.rendered_object_id = HlsRepairRenderedObjectId::Transient { resource_id: "resource-a".to_string() }; + commit_context.resource_id = "resource-a".to_string(); + commit_context.origin_fetch_uri_for_diagnostics = "http://origin.example/live/resource-a.ts".to_string(); + commit_context.media_sequence = None; + commit_context.discontinuity_sequence = None; + + let mut cache_hit_context = commit_context.clone(); + cache_hit_context.origin_fetch_uri_for_diagnostics = "resource-a".to_string(); + + let mut second_context = commit_context.clone(); + second_context.rendered_object_id = HlsRepairRenderedObjectId::Transient { resource_id: "resource-b".to_string() }; + second_context.resource_id = "resource-b".to_string(); + second_context.origin_fetch_uri_for_diagnostics = "resource-b".to_string(); + + assert_eq!(selected_repair_mode(&manager, &commit_context).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(selected_repair_mode(&manager, &cache_hit_context).await, None); + assert_eq!(selected_repair_mode(&manager, &second_context).await, Some(HlsSegmentRepairMode::Low)); +} + +#[tokio::test] +async fn normal_commit_and_cache_hit_consume_repair_window_once() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 2)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + let commit_context = repair_context("lease-a", "1"); + let mut cache_hit_context = commit_context.clone(); + cache_hit_context.origin_fetch_uri_for_diagnostics = "http://redirect.example/other-path.ts".to_string(); + cache_hit_context.media_sequence = Some(99); + cache_hit_context.discontinuity_sequence = Some(7); + let second_context = repair_context("lease-a", "2"); + + assert_eq!(selected_repair_mode(&manager, &commit_context).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(selected_repair_mode(&manager, &cache_hit_context).await, None); + assert_eq!(selected_repair_mode(&manager, &second_context).await, Some(HlsSegmentRepairMode::Low)); +} + +#[tokio::test] +async fn ensuring_pending_repair_window_does_not_reset_consumed_candidate_budget() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.ensure_access_lease_window(lease_id.clone()).await; + + assert_eq!(selected_repair_mode(&manager, &repair_context("lease-a", "1")).await, Some(HlsSegmentRepairMode::Low)); + manager.ensure_access_lease_window(lease_id).await; + + assert_eq!(selected_repair_mode(&manager, &repair_context("lease-a", "2")).await, None); + assert_eq!(manager.stats().await.generations, 1); +} + +#[tokio::test] +async fn prewarmed_ready_object_reuses_metadata_on_later_demand() { + const CLEAN_TS: &[u8] = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); + let temp_dir = tempfile::tempdir().expect("temp dir"); + let cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let manager = Arc::new(HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1))); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let cache_key = SegmentCacheKey::new(ProxySessionId("proxy-session".to_string()), 1, "ts"); + cache.write_bytes_and_commit(&cache_key, CLEAN_TS).await.expect("cache fixture"); + manager.ensure_access_lease_window(lease_id).await; + let context = repair_context("lease-a", "1"); + + manager.spawn_ready_cache_prewarm_inner(Arc::clone(&cache), vec![(cache_key.clone(), context.clone())], None).await; + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if manager.stats().await.object_metadata == 1 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("prewarm completes"); + let after_prewarm = manager.stats().await; + assert_eq!(after_prewarm.object_metadata, 1); + + assert!(manager + .repair_ready_cache_hit(&cache, &cache_key, context) + .await + .expect("demand reuses prewarm") + .is_none()); + assert_eq!(manager.stats().await, after_prewarm); +} + +#[tokio::test] +async fn repair_metadata_is_bounded() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + + for index in 0..REPAIR_METADATA_MAX_ENTRIES + 5 { + manager + .record_metadata( + RepairIdentity { + raw_sha256: format!("{index:064x}"), + repair_mode: HlsSegmentRepairMode::Low, + command_version: 1, + ffmpeg_version: "test".to_string(), + }, + RepairStatus::Clean, + 1, + 1, + None, + ) + .await; + } + + assert_eq!(manager.stats().await.metadata, REPAIR_METADATA_MAX_ENTRIES); +} diff --git a/backend/hls/src/segment_repair/tests/publication.rs b/backend/hls/src/segment_repair/tests/publication.rs new file mode 100644 index 000000000..8bbc5fb78 --- /dev/null +++ b/backend/hls/src/segment_repair/tests/publication.rs @@ -0,0 +1,34 @@ +use super::*; + +#[tokio::test] +async fn new_repair_window_generation_allows_rechecking_candidate() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let context = repair_context("lease-a", "000001"); + + manager.start_access_lease_window(lease_id.clone()).await; + assert_eq!(selected_repair_mode(&manager, &context).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(manager.windows.read().await.checked_candidates.len(), 1); + + manager.start_access_lease_window(lease_id).await; + + assert_eq!(selected_repair_mode(&manager, &context).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(manager.windows.read().await.checked_candidates.len(), 2); +} + +#[tokio::test] +async fn remove_access_lease_window_clears_window_and_generation() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + manager.start_access_lease_window(lease_id.clone()).await; + + let before = manager.stats().await; + assert_eq!(before.windows, 1); + assert_eq!(before.generations, 1); + + manager.remove_access_lease_window(&lease_id).await; + + let after = manager.stats().await; + assert_eq!(after.windows, 0); + assert_eq!(after.generations, 0); +} diff --git a/backend/hls/src/segment_repair/tests/storage.rs b/backend/hls/src/segment_repair/tests/storage.rs new file mode 100644 index 000000000..68b04b9b3 --- /dev/null +++ b/backend/hls/src/segment_repair/tests/storage.rs @@ -0,0 +1,33 @@ +use super::*; + +#[test] +fn repair_log_identity_fields_separate_content_proxy_and_lease_identities() { + let context = repair_context("lease-a", "000001"); + let fields = context.log_identity_fields(); + + assert_eq!( + fields, + format!( + "session={} proxy_session={} lease={}", + context.log_identity.session(), + context.log_identity.proxy_session(), + super::super::safe_hls_access_lease_id(context.hls_access_lease_id.as_ref().expect("test lease")) + ) + ); + assert_ne!(context.log_identity.session(), context.log_identity.proxy_session()); +} + +#[tokio::test] +async fn on_demand_selection_joins_the_prewarm_candidate_identity() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + manager.ensure_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + let context = repair_context("lease-a", "1"); + + let (prewarm_mode, _, prewarm_candidate) = manager.try_select_candidate(&context).await.expect("prewarm candidate"); + let (demand_mode, _, demand_candidate) = + manager.try_select_or_join_candidate(&context).await.expect("demand joins prewarm"); + + assert_eq!(demand_mode, prewarm_mode); + assert_eq!(demand_candidate, prewarm_candidate); + assert_eq!(manager.windows.read().await.windows[&HlsAccessLeaseId("lease-a".to_string())].remaining_segments, 0); +} diff --git a/backend/hls/src/segment_repair/tests/streaming.rs b/backend/hls/src/segment_repair/tests/streaming.rs new file mode 100644 index 000000000..2d27d6981 --- /dev/null +++ b/backend/hls/src/segment_repair/tests/streaming.rs @@ -0,0 +1,30 @@ +use super::*; + +#[test] +fn repair_object_metadata_key_ignores_origin_fetch_uri_for_diagnostics() { + let mut first = repair_context("lease-a", "1"); + first.origin_fetch_uri_for_diagnostics = "http://mirror-a.example/live/1.ts".to_string(); + let mut second = first.clone(); + second.origin_fetch_uri_for_diagnostics = "http://redirect-b.example/cdn/path/1.ts".to_string(); + + assert_eq!( + repair_object_metadata_key(&first, HlsSegmentRepairMode::Low), + repair_object_metadata_key(&second, HlsSegmentRepairMode::Low) + ); +} + +#[tokio::test] +async fn repair_window_candidate_key_ignores_origin_fetch_uri_for_diagnostics() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 2)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + let mut first = repair_context("lease-a", "1"); + first.origin_fetch_uri_for_diagnostics = "http://mirror-a.example/live/1.ts".to_string(); + let mut same_rendered_object_other_fetch_uri = first.clone(); + same_rendered_object_other_fetch_uri.origin_fetch_uri_for_diagnostics = + "http://redirect-b.example/cdn/path/1.ts".to_string(); + let second_rendered_object = repair_context("lease-a", "2"); + + assert_eq!(selected_repair_mode(&manager, &first).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(selected_repair_mode(&manager, &same_rendered_object_other_fetch_uri).await, None); + assert_eq!(selected_repair_mode(&manager, &second_rendered_object).await, Some(HlsSegmentRepairMode::Low)); +} diff --git a/backend/hls/src/segment_repair/tests/terminal.rs b/backend/hls/src/segment_repair/tests/terminal.rs new file mode 100644 index 000000000..9a94d3358 --- /dev/null +++ b/backend/hls/src/segment_repair/tests/terminal.rs @@ -0,0 +1,13 @@ +use super::*; + +#[tokio::test] +async fn exhausted_window_candidate_is_never_remembered_as_joinable() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + manager.ensure_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + assert!(manager.try_select_candidate(&repair_context("lease-a", "1")).await.is_some()); + let skipped = repair_context("lease-a", "2"); + + assert!(manager.try_select_or_join_candidate(&skipped).await.is_none()); + assert!(manager.try_select_or_join_candidate(&skipped).await.is_none()); + assert_eq!(manager.stats().await.checked_candidates, 1); +} diff --git a/backend/hls/src/segment_repair/tests/transport.rs b/backend/hls/src/segment_repair/tests/transport.rs new file mode 100644 index 000000000..199636976 --- /dev/null +++ b/backend/hls/src/segment_repair/tests/transport.rs @@ -0,0 +1,228 @@ +use super::*; + +#[test] +fn repair_prewarm_candidates_include_only_ready_clear_origin_ts_media() { + let lease_id = HlsAccessLeaseId("lease-a".to_string()); + let mut session = HlsSession::new(HlsSessionKey::new(1, "prewarm-candidates"), b"secret", 0); + let proxy_seq = 10; + let cache_key = SegmentCacheKey::new(session.proxy_session_id.clone(), proxy_seq, "ts"); + session.segments.insert( + proxy_seq, + SegmentEntry { + origin_key: OriginSegmentKey { + origin_epoch: 1, + effective_host_id: 1, + host_local_sequence: 100, + host_local_index: 0, + }, + proxy_seq, + duration_ms: 6_000, + proxy_file_ext: "ts".to_string(), + content_type: "video/mp2t".to_string(), + cache_key, + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + encryption: None, + origin_fetch_ref: None, + status: SegmentCacheStatus::Ready { content_length: 1_024, ready_at_ms: 1 }, + last_rendered_at_ms: Some(1), + access: Arc::new(CacheAccessState::new()), + }, + ); + let snapshot = HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(1), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 1, + delivered_at_ms: 1, + first_proxy_seq: proxy_seq, + last_proxy_seq: proxy_seq, + visible_segments: Arc::from([HlsLeaseManifestSegment { + proxy_seq, + duration_ms: 6_000, + uri: "/live/10.ts".to_string().into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }]), + discontinuity_sequence: 0, + target_duration_ms: 6_000, + playlist_duration_ms: 6_000, + last_visible_media_end_ms: 6_000, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + }; + + assert_eq!(ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, 1).len(), 1); + assert!(ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, 0).is_empty()); + + let entry = session.segments.get_mut(&proxy_seq).expect("candidate segment"); + entry.proxy_file_ext = "m4s".to_string(); + assert!(ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, 1).is_empty()); + let entry = session.segments.get_mut(&proxy_seq).expect("candidate segment"); + entry.proxy_file_ext = "ts".to_string(); + entry.encryption = Some(HlsSegmentEncryption { + resource_id: TransientResourceId("key-a".to_string()), + resource_extension: "key".to_string(), + iv: None, + key_format: None, + key_format_versions: None, + }); + assert!(ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, 1).is_empty()); + let entry = session.segments.get_mut(&proxy_seq).expect("candidate segment"); + entry.encryption = None; + entry.status = SegmentCacheStatus::Discovered; + assert!(ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, 1).is_empty()); + let entry = session.segments.get_mut(&proxy_seq).expect("candidate segment"); + entry.status = SegmentCacheStatus::Ready { content_length: 1_024, ready_at_ms: 1 }; + entry.origin_key.origin_epoch = HLS_PROVISIONING_ORIGIN_EPOCH; + assert!(ready_segment_repair_prewarm_candidates(&session, &lease_id, &snapshot, 1).is_empty()); +} + +#[test] +fn hevc_keyframe_nalu_counts_as_vcl_but_does_not_trigger_alone() { + let warnings = parse_ffmpeg_warnings("[hevc @ 0x1] Skipping invalid undecodable NALU: 21\n"); + + assert_eq!(warnings.invalid_undecodable_nalu_total, 1); + assert_eq!(warnings.invalid_undecodable_nalu_keyframe, 1); + assert!(!should_repair(RepairVideoCodec::Hevc, &warnings)); +} + +#[test] +fn hevc_invalid_undecodable_nalu_0_to_31_counts_as_vcl_trigger_input() { + let warnings = + parse_ffmpeg_warnings("[hevc @ 0x1] missing SPS\n[hevc @ 0x1] Skipping invalid undecodable NALU: 30\n"); + + assert_eq!(warnings.invalid_undecodable_nalu_total, 1); + assert_eq!(warnings.invalid_undecodable_nalu_non_metadata, 1); + assert!(should_repair(RepairVideoCodec::Hevc, &warnings)); +} + +#[test] +fn parse_probe_detects_hevc_codec_and_extradata() { + let probe = parse_probe( + r#"{ + "streams": [ + { + "index": 0, + "codec_type": "video", + "codec_name": "hevc", + "start_time": "1.250000", + "extradata_size": 96 + } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters::default(), + ) + .expect("probe should parse"); + + assert_eq!(detect_video_codec(&probe), RepairVideoCodec::Hevc); + assert_eq!(probe.primary_video_extradata_size, Some(96)); +} + +#[test] +fn repair_remux_selection_rejects_without_valid_video() { + let probe = parse_probe( + r#"{ + "streams": [ + { "index": 0, "codec_type": "video", "codec_name": "hevc", "height": 1080 }, + { "index": 1, "codec_type": "audio", "codec_name": "ac3", "sample_rate": "48000", "channels": 6 } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters::default(), + ) + .expect("probe should parse"); + + let err = select_repair_remux_streams(&probe).expect_err("missing width should reject video"); + + assert_eq!(err, "no_valid_video_stream"); +} + +#[test] +fn validation_rejects_remaining_repair_triggers_even_when_level_improves() { + let raw = parse_probe( + r#"{ + "streams": [ + { "codec_type": "video", "codec_name": "hevc", "start_time": "0.000000" }, + { "codec_type": "audio", "codec_name": "aac", "start_time": "0.000000" } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters { codec_parameters_missing: 1, missing_sps: 1, ..WarningCounters::default() }, + ) + .expect("raw probe should parse"); + let fixed = parse_probe( + r#"{ + "streams": [ + { "codec_type": "video", "codec_name": "hevc", "start_time": "0.000000" }, + { "codec_type": "audio", "codec_name": "aac", "start_time": "0.000000" } + ], + "format": { "duration": "2.000000", "size": "1000" } + }"#, + WarningCounters { missing_sps: 1, ..WarningCounters::default() }, + ) + .expect("fixed probe should parse"); + + let err = validate_repair( + &repair_config(HlsSegmentRepairMode::High, 1), + RepairVideoCodec::Hevc, + &raw, + &fixed, + HlsSegmentRepairMode::High, + &RepairRemuxStreamSelection::preserve_all(&raw), + ) + .expect_err("remaining medium trigger should fail validation"); + assert_eq!(err, "repair_triggers_remaining"); +} + +#[tokio::test] +async fn repair_window_selects_first_unique_segments_per_access_lease() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + let first = repair_context("lease-a", "000001"); + let second = repair_context("lease-a", "000002"); + + assert_eq!(selected_repair_mode(&manager, &first).await, Some(HlsSegmentRepairMode::Low)); + assert_eq!(selected_repair_mode(&manager, &first).await, None); + assert_eq!(selected_repair_mode(&manager, &second).await, None); +} + +#[tokio::test] +async fn non_repairable_objects_do_not_consume_repair_window() { + let manager = HlsSegmentRepairManager::new(repair_config(HlsSegmentRepairMode::Low, 1)); + manager.start_access_lease_window(HlsAccessLeaseId("lease-a".to_string())).await; + let mut partial = repair_context("lease-a", "1"); + partial.complete_object = false; + let repairable = repair_context("lease-a", "2"); + + assert_eq!(selected_repair_mode(&manager, &partial).await, None); + assert_eq!(selected_repair_mode(&manager, &repairable).await, Some(HlsSegmentRepairMode::Low)); +} + +#[test] +fn repair_context_excludes_non_finite_origin_ts_objects() { + let mut context = repair_context("lease-a", "000001"); + + context.complete_object = false; + assert!(!context.is_repairable_ts()); + + context.complete_object = true; + context.encrypted = true; + assert!(!context.is_repairable_ts()); + + context.encrypted = false; + context.custom_response = true; + assert!(!context.is_repairable_ts()); + + context.custom_response = false; + context.file_ext = "m4s".to_string(); + assert!(!context.is_repairable_ts()); +} diff --git a/backend/hls/src/segment_repair/window.rs b/backend/hls/src/segment_repair/window.rs new file mode 100644 index 000000000..dcb38a42e --- /dev/null +++ b/backend/hls/src/segment_repair/window.rs @@ -0,0 +1,188 @@ +use super::{ + is_hls_provisioning_segment, safe_hls_access_lease_id, HlsAccessLeaseId, HlsAccessLeaseStore, + HlsLeaseManifestSnapshot, HlsLogIdentity, HlsRepairPrewarmGuard, HlsRepairWindowCandidateKey, HlsSession, + ProxySessionId, SegmentCacheKey, SegmentCacheStatus, +}; +use shared::model::HlsSegmentRepairMode; +use std::sync::Arc; +use tokio::sync::RwLock; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub enum HlsSegmentRepairSource { + Normal, + Transient, +} + +impl HlsSegmentRepairSource { + pub(crate) const fn as_log_value(self) -> &'static str { + match self { + Self::Normal => "normal", + Self::Transient => "transient", + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum HlsRepairRenderedObjectId { + Normal { proxy_seq: u64 }, + Transient { resource_id: String }, +} + +#[derive(Debug, Clone)] +pub struct HlsSegmentRepairObjectContext { + pub source: HlsSegmentRepairSource, + pub log_identity: HlsLogIdentity, + pub proxy_session_id: ProxySessionId, + pub hls_access_lease_id: Option, + pub rendered_object_id: HlsRepairRenderedObjectId, + pub resource_id: String, + pub file_ext: String, + /// Concrete origin fetch URI retained for diagnostics and postprocess metadata. + /// + /// This may include a provider mirror or redirect/CDN host. It must not be used as HLS session identity, account + /// binding, provider-failover state, repair object identity, or repair-window candidate identity. + pub origin_fetch_uri_for_diagnostics: String, + pub media_sequence: Option, + pub discontinuity_sequence: Option, + pub complete_object: bool, + pub encrypted: bool, + pub custom_response: bool, +} + +pub fn ready_segment_repair_prewarm_candidates( + session: &HlsSession, + lease_id: &HlsAccessLeaseId, + snapshot: &HlsLeaseManifestSnapshot, + candidate_limit: usize, +) -> Vec<(SegmentCacheKey, HlsSegmentRepairObjectContext)> { + let identity = HlsLogIdentity::from_session(session); + snapshot + .visible_segments + .iter() + .take(candidate_limit) + .filter_map(|visible| { + let entry = session.segments.get(&visible.proxy_seq)?; + if entry.proxy_seq != visible.proxy_seq + || is_hls_provisioning_segment(entry) + || entry.proxy_file_ext != "ts" + || entry.origin_byte_range.is_some() + || entry.encryption.is_some() + || !matches!(entry.status, SegmentCacheStatus::Ready { .. }) + { + return None; + } + Some(( + entry.cache_key.clone(), + HlsSegmentRepairObjectContext { + source: HlsSegmentRepairSource::Normal, + log_identity: identity.clone(), + proxy_session_id: session.proxy_session_id.clone(), + hls_access_lease_id: Some(lease_id.clone()), + rendered_object_id: HlsRepairRenderedObjectId::Normal { proxy_seq: visible.proxy_seq }, + resource_id: format!("{:06}", visible.proxy_seq), + file_ext: entry.proxy_file_ext.clone(), + origin_fetch_uri_for_diagnostics: entry + .origin_fetch_ref + .as_ref() + .map(|fetch_ref| fetch_ref.resolved_origin_url.clone()) + .unwrap_or_default(), + media_sequence: Some(entry.origin_key.host_local_sequence), + discontinuity_sequence: Some(session.discontinuity_sequence), + complete_object: true, + encrypted: false, + custom_response: false, + }, + )) + }) + .collect() +} + +impl HlsRepairPrewarmGuard { + pub fn new( + access_leases: Arc>, + lease_id: HlsAccessLeaseId, + proxy_session_id: ProxySessionId, + issued_at_ms: u64, + snapshot_generation: u64, + ) -> Self { + Self { access_leases, lease_id, proxy_session_id, issued_at_ms, snapshot_generation } + } + + pub(super) async fn is_current(&self) -> bool { + self.access_leases.read().await.repair_prewarm_is_current( + &self.lease_id, + &self.proxy_session_id, + self.issued_at_ms, + self.snapshot_generation, + ) + } +} + +impl HlsSegmentRepairObjectContext { + pub(crate) fn is_repairable_ts(&self) -> bool { + self.file_ext.eq_ignore_ascii_case("ts") && self.complete_object && !self.encrypted && !self.custom_response + } + + pub(super) fn repair_skip_reason(&self) -> Option<&'static str> { + if self.is_repairable_ts() { + return None; + } + if !self.file_ext.eq_ignore_ascii_case("ts") { + return Some("not-ts"); + } + if !self.complete_object { + return Some("partial-object"); + } + if self.encrypted { + return Some("encrypted"); + } + if self.custom_response { + return Some("custom-response"); + } + None + } + + pub(super) fn log_identity_fields(&self) -> String { + format!( + "session={} proxy_session={} lease={}", + self.log_identity.session(), + self.log_identity.proxy_session(), + self.hls_access_lease_id.as_ref().map_or_else(|| "".to_string(), safe_hls_access_lease_id) + ) + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum RepairCandidateSelection { + Selected(HlsSegmentRepairMode, HlsRepairWindowCandidateKey), + Skipped(&'static str), + AlreadyChecked(HlsSegmentRepairMode, HlsRepairWindowCandidateKey), +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum RepairStatus { + Clean, + Fixed, + PolicyLimited, + Unsupported, + Timeout, + RemuxFailed, + ValidationFailed, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum RepairVideoCodec { + H264, + Hevc, + Unsupported, +} + +impl RepairVideoCodec { + pub(super) const fn as_log_value(self) -> &'static str { + match self { + Self::H264 => "h264", + Self::Hevc => "hevc", + Self::Unsupported => "unsupported", + } + } +} diff --git a/backend/hls/src/segment_revision.rs b/backend/hls/src/segment_revision.rs new file mode 100644 index 000000000..48fd1ed3d --- /dev/null +++ b/backend/hls/src/segment_revision.rs @@ -0,0 +1,350 @@ +use super::{CachedSegmentMetadata, HlsCacheObjectKey, ProxySessionId, SegmentCacheKey}; +use crate::sync_ext::MutexExt; +use shared::model::HlsStartupMode; +use std::{ + collections::BTreeMap, + fmt, io, + sync::{ + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, + Arc, Mutex, + }, +}; +use tokio::sync::watch; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd)] +pub enum SegmentRevisionKind { + Raw, + Processed, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd)] +pub struct SegmentRevisionId { + generation: u64, + ordinal: u64, +} + +#[derive(Clone, Debug)] +pub struct SegmentRevisionKey { + session: ProxySessionId, + pub proxy_seq: u64, + id: SegmentRevisionId, + pub kind: SegmentRevisionKind, +} + +impl HlsCacheObjectKey for SegmentRevisionKey { + fn proxy_session_id(&self) -> &ProxySessionId { &self.session } + fn session_path_component(&self) -> String { + SegmentCacheKey::new(self.session.clone(), self.proxy_seq, "ts").session_path_component() + } + fn file_name(&self) -> String { + let kind = match self.kind { + SegmentRevisionKind::Raw => "raw", + SegmentRevisionKind::Processed => "processed", + }; + format!("revision-{:016x}-{:016x}-{kind}.ts", self.id.generation, self.id.ordinal) + } +} + +#[derive(Debug, Clone)] +pub enum SegmentRevisionState { + Pending, + Complete(CachedSegmentMetadata), + Failed, +} + +pub struct SegmentRevision { + pub key: SegmentRevisionKey, + pins: AtomicUsize, + pub prefix_available: AtomicU64, + state: watch::Sender, + exposed: AtomicBool, + pub(crate) file_pin: Mutex>, + pub(crate) replay: Mutex>, +} + +impl fmt::Debug for SegmentRevision { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SegmentRevision") + .field("id", &self.key.id) + .field("kind", &self.key.kind) + .field("pins", &self.pins.load(Ordering::Acquire)) + .finish_non_exhaustive() + } +} + +impl SegmentRevision { + pub fn id(&self) -> SegmentRevisionId { self.key.id } + #[cfg(test)] + pub(crate) fn pin_count(&self) -> usize { self.pins.load(Ordering::Acquire) } + pub fn subscribe(&self) -> watch::Receiver { self.state.subscribe() } + pub fn complete(&self, metadata: CachedSegmentMetadata) -> bool { + self.state.send_if_modified(|state| { + if !matches!(state, SegmentRevisionState::Pending) { + return false; + } + *state = SegmentRevisionState::Complete(metadata); + true + }) + } + pub(crate) fn prefix_changed(&self) { self.state.send_modify(|_| {}); } + fn is_exposable(&self, state: &SegmentRevisionState) -> bool { + match state { + SegmentRevisionState::Complete(_) => true, + SegmentRevisionState::Pending => { + self.prefix_available.load(Ordering::Acquire) > 0 && self.replay.lock_unpoisoned().is_some() + } + SegmentRevisionState::Failed => false, + } + } + + /// Side-effect-free precheck for [`Self::expose`]. + pub(crate) fn can_expose(&self) -> bool { self.is_exposable(&self.state.borrow()) } + + pub(crate) fn expose(&self) -> bool { + let mut exposed = false; + self.state.send_if_modified(|state| { + exposed = self.is_exposable(state); + if exposed { + self.exposed.store(true, Ordering::Release); + } + false + }); + exposed + } + + pub(crate) fn replay_exhausted(&self) { + self.state.send_if_modified(|state| { + if !matches!(state, SegmentRevisionState::Pending) { + return false; + } + self.replay.lock_unpoisoned().take(); + if self.exposed.load(Ordering::Acquire) { + *state = SegmentRevisionState::Failed; + } else { + self.prefix_available.store(0, Ordering::Release); + } + true + }); + } + + pub fn fail(&self) { + self.state.send_if_modified(|state| { + if !matches!(state, SegmentRevisionState::Pending) { + return false; + } + *state = SegmentRevisionState::Failed; + true + }); + self.replay.lock_unpoisoned().take(); + } + pub fn state(&self) -> SegmentRevisionState { self.state.borrow().clone() } + pub fn is_complete(&self) -> bool { matches!(*self.state.borrow(), SegmentRevisionState::Complete(_)) } + + /// Whether this revision may open the startup window as its head. + pub fn is_startup_head_ready(&self, mode: HlsStartupMode) -> bool { + match mode { + HlsStartupMode::FirstReady => self.is_complete(), + HlsStartupMode::Progressive => self.is_publishable(), + HlsStartupMode::Conservative => false, + } + } + + pub fn is_publishable(&self) -> bool { + match &*self.state.borrow() { + SegmentRevisionState::Failed => false, + SegmentRevisionState::Complete(_) => true, + SegmentRevisionState::Pending => self.prefix_available.load(Ordering::Acquire) > 0, + } + } + pub async fn wait_complete(&self, deadline: tokio::time::Instant) -> io::Result { + let mut state = self.subscribe(); + loop { + match state.borrow_and_update().clone() { + SegmentRevisionState::Complete(metadata) => return Ok(metadata), + SegmentRevisionState::Failed => return Err(io::Error::other("published segment revision failed")), + SegmentRevisionState::Pending => {} + } + tokio::time::timeout_at(deadline, state.changed()) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "segment revision wait timed out"))? + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "segment revision producer ended"))?; + } + } +} + +#[derive(Debug)] +pub struct SegmentRevisionGuard { + revision: Arc, +} + +impl SegmentRevisionGuard { + pub fn revision(&self) -> &SegmentRevision { &self.revision } +} +impl Clone for SegmentRevisionGuard { + fn clone(&self) -> Self { + self.revision.pins.fetch_add(1, Ordering::AcqRel); + Self { revision: Arc::clone(&self.revision) } + } +} +impl Drop for SegmentRevisionGuard { + fn drop(&mut self) { self.revision.pins.fetch_sub(1, Ordering::AcqRel); } +} +impl PartialEq for SegmentRevisionGuard { + fn eq(&self, other: &Self) -> bool { self.revision.id() == other.revision.id() } +} +impl Eq for SegmentRevisionGuard {} + +#[derive(Debug)] +pub struct SegmentRevisionStore { + generation: u64, + next: AtomicU64, + entries: Mutex>>, +} + +impl Default for SegmentRevisionStore { + fn default() -> Self { + Self { generation: fastrand::u64(..), next: AtomicU64::new(0), entries: Mutex::new(BTreeMap::new()) } + } +} + +impl SegmentRevisionStore { + pub(crate) fn has_created_revisions(&self) -> bool { self.next.load(Ordering::Acquire) > 0 } + pub fn create( + &self, + session: ProxySessionId, + proxy_seq: u64, + kind: SegmentRevisionKind, + ) -> io::Result { + let mut ordinal = self.next.load(Ordering::Acquire); + loop { + let next = ordinal.checked_add(1).ok_or_else(|| io::Error::other("segment revision identity exhausted"))?; + match self.next.compare_exchange_weak(ordinal, next, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => break, + Err(current) => ordinal = current, + } + } + let id = SegmentRevisionId { generation: self.generation, ordinal }; + let revision = Arc::new(SegmentRevision { + key: SegmentRevisionKey { session, proxy_seq, id, kind }, + pins: AtomicUsize::new(1), + prefix_available: AtomicU64::new(0), + state: watch::channel(SegmentRevisionState::Pending).0, + exposed: AtomicBool::new(false), + file_pin: Mutex::new(None), + replay: Mutex::new(None), + }); + self.entries + .lock() + .map_err(|_| io::Error::other("revision registry poisoned"))? + .insert(id, Arc::clone(&revision)); + Ok(SegmentRevisionGuard { revision }) + } + + pub fn requeue_retired(&self, revision: Arc) -> io::Result<()> { + self.entries + .lock() + .map_err(|_| io::Error::other("revision registry poisoned"))? + .insert(revision.id(), revision); + Ok(()) + } + + pub fn retire_unpinned(&self) -> io::Result>> { + let mut entries = self.entries.lock().map_err(|_| io::Error::other("revision registry poisoned"))?; + let ids: Vec<_> = entries + .iter() + .filter_map(|(id, revision)| (revision.pins.load(Ordering::Acquire) == 0).then_some(*id)) + .take(128) + .collect(); + Ok(ids.into_iter().filter_map(|id| entries.remove(&id)).collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn published_owner_cannot_be_replaced_by_a_new_attempt() -> io::Result<()> { + let store = SegmentRevisionStore::default(); + let published = store.create(ProxySessionId("session".into()), 7, SegmentRevisionKind::Processed)?; + let next = store.create(ProxySessionId("session".into()), 7, SegmentRevisionKind::Processed)?; + assert_ne!(published.revision().id(), next.revision().id()); + published.revision().fail(); + assert!(next.revision().complete(CachedSegmentMetadata { path: "next.ts".into(), size: 10 })); + assert!(published.revision().wait_complete(tokio::time::Instant::now()).await.is_err()); + assert!(!published.revision().complete(CachedSegmentMetadata { path: "wrong.ts".into(), size: 10 })); + Ok(()) + } + + #[test] + fn replay_exhaustion_before_exposure_waits_for_the_identical_spool() -> io::Result<()> { + let store = SegmentRevisionStore::default(); + let owner = store.create(ProxySessionId("budget".into()), 0, SegmentRevisionKind::Raw)?; + let budget = super::super::ProgressiveBudgetManager::new(tuliprox_core::model::HlsStartupConfig::default()); + let mut replay = super::super::progressive_startup::ProgressiveReplay::new(&budget) + .ok_or_else(|| io::Error::other("slot"))?; + replay.push(b"prefix")?; + *owner.revision().replay.lock_unpoisoned() = Some(replay); + owner.revision().prefix_available.store(6, Ordering::Release); + owner.revision().replay_exhausted(); + assert!(matches!(*owner.revision().subscribe().borrow(), SegmentRevisionState::Pending)); + assert!(!owner.revision().is_publishable()); + assert!(!owner.revision().expose()); + assert_eq!(budget.usage(), (0, 0)); + assert!(owner.revision().complete(CachedSegmentMetadata { path: "raw.ts".into(), size: 6 })); + assert!(owner.revision().expose()); + Ok(()) + } + + #[test] + fn exposed_replay_exhaustion_is_terminal() -> io::Result<()> { + let store = SegmentRevisionStore::default(); + let owner = store.create(ProxySessionId("budget".into()), 0, SegmentRevisionKind::Raw)?; + let budget = super::super::ProgressiveBudgetManager::new(tuliprox_core::model::HlsStartupConfig::default()); + let mut replay = super::super::progressive_startup::ProgressiveReplay::new(&budget) + .ok_or_else(|| io::Error::other("slot"))?; + replay.push(b"prefix")?; + *owner.revision().replay.lock_unpoisoned() = Some(replay); + owner.revision().prefix_available.store(6, Ordering::Release); + assert!(owner.revision().expose()); + owner.revision().replay_exhausted(); + assert!(matches!(*owner.revision().subscribe().borrow(), SegmentRevisionState::Failed)); + assert!(!owner.revision().complete(CachedSegmentMetadata { path: "raw.ts".into(), size: 6 })); + assert_eq!(budget.usage(), (0, 0)); + Ok(()) + } + + #[test] + fn repeated_retention_and_reader_churn_leaves_no_revision_owners() -> io::Result<()> { + let store = SegmentRevisionStore::default(); + let mut window = std::collections::VecDeque::new(); + for seq in 0..10_000 { + let owner = store.create(ProxySessionId("churn".into()), seq, SegmentRevisionKind::Processed)?; + let readers: Vec<_> = (0..16).map(|_| owner.clone()).collect(); + window.push_back(owner); + if window.len() > 6 { + window.pop_front(); + } + drop(readers); + drop(store.retire_unpinned()?); + assert_eq!(store.entries.lock().map_err(|_| io::Error::other("registry"))?.len(), window.len()); + } + drop(window); + drop(store.retire_unpinned()?); + assert!(store.entries.lock().map_err(|_| io::Error::other("registry"))?.is_empty()); + Ok(()) + } + + #[test] + fn retirement_waits_for_all_explicit_guards() -> io::Result<()> { + let store = SegmentRevisionStore::default(); + let guard = store.create(ProxySessionId("session".into()), 7, SegmentRevisionKind::Raw)?; + let reader = guard.clone(); + drop(guard); + assert!(store.retire_unpinned()?.is_empty()); + drop(reader); + assert_eq!(store.retire_unpinned()?.len(), 1); + assert!(store.retire_unpinned()?.is_empty()); + Ok(()) + } +} diff --git a/backend/hls/src/session.rs b/backend/hls/src/session.rs index 08ac76ac8..2c64b4ba4 100644 --- a/backend/hls/src/session.rs +++ b/backend/hls/src/session.rs @@ -261,10 +261,12 @@ pub struct HlsSession { pub origin_refresh: OriginRefreshState, pub origin_control: HlsSessionOriginControl, pub render_policy: RenderPolicy, + pub startup: Option, pub last_rendered_manifest: Option, manifest_commit_generation: u64, last_normal_manifest_commit_identity: Option, pub(crate) published_live_origin_baseline: Option, + pub(crate) missing_head_limit: Option, pub longest_rendered_playlist_duration_ms: u64, pub initial_prefetch_gap_segments: usize, pub segment_prefetch_queue: SegmentPrefetchQueue, @@ -332,10 +334,12 @@ impl HlsSession { origin_refresh: OriginRefreshState::default(), origin_control: HlsSessionOriginControl::default(), render_policy: RenderPolicy::default(), + startup: None, last_rendered_manifest: None, manifest_commit_generation: 0, last_normal_manifest_commit_identity: None, published_live_origin_baseline: None, + missing_head_limit: None, longest_rendered_playlist_duration_ms: 0, initial_prefetch_gap_segments: 0, segment_prefetch_queue: SegmentPrefetchQueue::default(), @@ -811,6 +815,8 @@ impl fmt::Debug for HlsSession { .field("last_effective_manifest_host", &self.last_effective_manifest_host) .field("origin_refresh", &self.origin_refresh) .field("render_policy", &self.render_policy) + .field("startup", &self.startup) + .field("missing_head_limit", &self.missing_head_limit) .field("last_rendered_manifest", &self.last_rendered_manifest) .field("manifest_commit_generation", &self.manifest_commit_generation) .field("last_normal_manifest_commit_identity", &self.last_normal_manifest_commit_identity) diff --git a/backend/hls/src/session_store.rs b/backend/hls/src/session_store.rs index 42cb5d78e..a92ec4229 100644 --- a/backend/hls/src/session_store.rs +++ b/backend/hls/src/session_store.rs @@ -173,14 +173,26 @@ impl HlsSessionStore { origin_source: HlsOriginSource, reverse_proxy_rewrite_secret: &[u8], now_ms: u64, + ) -> (HlsSessionHandle, HlsSessionStoreOutcome) { + self.get_or_create_session_with_startup(key, origin_source, reverse_proxy_rewrite_secret, now_ms, None).await + } + + pub async fn get_or_create_session_with_startup( + &self, + key: HlsSessionKey, + origin_source: HlsOriginSource, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + startup: Option, ) -> (HlsSessionHandle, HlsSessionStoreOutcome) { let mut indexes = self.indexes.write().await; if let Some(session) = indexes.by_key.get(&key) { return (Arc::clone(session), HlsSessionStoreOutcome::Reused); } - let session = + let mut session = HlsSession::new_with_origin_source(key.clone(), origin_source, reverse_proxy_rewrite_secret, now_ms); + session.startup = startup; let proxy_session_id = session.proxy_session_id.clone(); let session = Arc::new(RwLock::new(session)); let _incarnation = self.register_session_incarnation(&session); diff --git a/backend/hls/src/startup_policy.rs b/backend/hls/src/startup_policy.rs new file mode 100644 index 000000000..b47470220 --- /dev/null +++ b/backend/hls/src/startup_policy.rs @@ -0,0 +1,54 @@ +use super::{HlsSession, ProgressiveBudgetManager, SegmentEntry, SegmentRevisionGuard, SegmentRevisionStore}; +use shared::model::HlsStartupMode; +use std::{collections::BTreeMap, sync::Arc}; +use tuliprox_core::model::HlsStartupConfig; + +/// Successors after the live head that fast start publishes and scans for missing segments. +pub(crate) const STARTUP_WINDOW_SUCCESSORS: u64 = 5; + +#[derive(Debug)] +pub struct HlsSessionStartup { + pub config: HlsStartupConfig, + pub first_data_timeout_ms: u64, + pub worker: std::sync::Weak, + pub access_leases: Arc>, + pub revisions: BTreeMap, + pub policy_fixed: bool, + pub store: Arc, + pub budget: Arc, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct HlsManifestRevisions { + pub mode: HlsStartupMode, + pub retained_start_seq: u64, + pub revisions: BTreeMap, +} + +impl HlsSession { + pub fn startup_mode(&self) -> HlsStartupMode { + self.startup.as_ref().map_or(HlsStartupMode::Conservative, |startup| startup.config.mode) + } + + pub(crate) fn reconcile_startup_eligibility(&mut self) { + if self.startup.is_none() { + return; + } + let eligible = self.origin_source.archive_reference.is_none() + && !self.segments.is_empty() + && self.segments.values().all(SegmentEntry::is_fast_start_eligible); + if !eligible && self.published_live_origin_baseline.is_none() { + self.startup = None; + } + } +} + +impl SegmentEntry { + /// Fast start serves only clear, whole MPEG-TS objects without an init map. + pub(crate) fn is_fast_start_eligible(&self) -> bool { + self.encryption.is_none() + && self.map_ref.is_none() + && self.origin_byte_range.is_none() + && self.proxy_file_ext.eq_ignore_ascii_case("ts") + } +} diff --git a/backend/hls/src/sync_ext.rs b/backend/hls/src/sync_ext.rs new file mode 100644 index 000000000..208c300cc --- /dev/null +++ b/backend/hls/src/sync_ext.rs @@ -0,0 +1,10 @@ +use std::sync::{Mutex, MutexGuard, PoisonError}; + +/// Lock access for state whose invariants survive a panicking holder. +pub(crate) trait MutexExt { + fn lock_unpoisoned(&self) -> MutexGuard<'_, T>; +} + +impl MutexExt for Mutex { + fn lock_unpoisoned(&self) -> MutexGuard<'_, T> { self.lock().unwrap_or_else(PoisonError::into_inner) } +} diff --git a/backend/hls/src/terminal_commit.rs b/backend/hls/src/terminal_commit.rs index 746c25617..b11f693a7 100644 --- a/backend/hls/src/terminal_commit.rs +++ b/backend/hls/src/terminal_commit.rs @@ -1362,6 +1362,7 @@ mod tests { terminal_media_preparation: HlsTerminalMediaPreparationState::Failed { key: None }, reserve, manifest_snapshot: HlsLeaseManifestSnapshot { + startup_revisions: None, delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, source_commit_identity: HlsManifestCommitIdentity::new(1), uri_materialization: None, diff --git a/backend/hls/src/terminal_tail.rs b/backend/hls/src/terminal_tail.rs index 02e5be25c..876d3e240 100644 --- a/backend/hls/src/terminal_tail.rs +++ b/backend/hls/src/terminal_tail.rs @@ -14,42 +14,20 @@ use super::{ HlsTrackEvidenceResolution, HlsTsProbeBudget, HlsTsProbeProtection, HlsTsSpliceEvidence, HlsTsTrackSignature, TransientObjectCacheKey, TransientPassthroughState, TransientResourceFile, TransientResourceKind, }; -use bytes::Bytes; -use log::debug; -use std::{ - collections::{hash_map::Entry, HashMap, HashSet}, - fmt::Write as _, - path::{Path, PathBuf}, - sync::Arc, -}; -use tokio::io::AsyncReadExt; -use tuliprox_core::utils::{format_hls_duration_ms, hls_target_duration_secs}; +use std::{path::PathBuf, sync::Arc}; use tuliprox_mpegts::transport_stream_buffer::{HlsTsTimestampProfile, TransportStreamBuffer}; use zeroize::Zeroizing; pub const HLS_TERMINAL_TAIL_SEGMENT_COUNT: u16 = 12; + const AES_128_BLOCK_BYTES: usize = 16; + const HLS_SHARED_LIVE_ROUTE_MARKER: &str = "/hls/shared/live/"; struct HlsTerminalKeyMaterial { bytes: Zeroizing<[u8; AES_128_BLOCK_BYTES]>, } -impl HlsTerminalKeyMaterial { - fn from_slice(bytes: &[u8]) -> Option { - let bytes: [u8; AES_128_BLOCK_BYTES] = bytes.try_into().ok()?; - Some(Self { bytes: Zeroizing::new(bytes) }) - } - - fn as_bytes(&self) -> &[u8; AES_128_BLOCK_BYTES] { &self.bytes } -} - -impl std::fmt::Debug for HlsTerminalKeyMaterial { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("HlsTerminalKeyMaterial()") - } -} - /// Immutable lease-bound evidence for one exact READY AES-128 key revision. /// /// The physical cache key proves which committed object supplied the frozen @@ -65,118 +43,9 @@ pub struct HlsTerminalKeyBinding { material: Arc, } -impl HlsTerminalKeyBinding { - fn new( - proxy_session_id: ProxySessionId, - resource_id: TransientResourceId, - route_extension: String, - source_cache_key: TransientObjectCacheKey, - content_type: String, - bytes: &[u8], - ) -> Option { - Some(Self { - proxy_session_id, - resource_id, - route_extension: Arc::from(route_extension), - source_cache_key, - content_type: Arc::from(content_type), - material: Arc::new(HlsTerminalKeyMaterial::from_slice(bytes)?), - }) - } - - fn matches_resource(&self, resource_file: &TransientResourceFile) -> bool { - self.resource_id == resource_file.resource_id && self.route_extension.as_ref() == resource_file.extension - } - - fn matches_route(&self, proxy_session_id: &ProxySessionId) -> bool { self.proxy_session_id == *proxy_session_id } - - pub fn content_type(&self) -> &str { &self.content_type } - - pub fn bytes(&self) -> Bytes { Bytes::copy_from_slice(self.material.as_bytes()) } - - #[cfg(any(test, feature = "test-support"))] - pub fn source_cache_key(&self) -> &TransientObjectCacheKey { &self.source_cache_key } - - #[cfg(any(test, feature = "test-support"))] - pub fn resource_id(&self) -> &TransientResourceId { &self.resource_id } -} - -impl std::fmt::Debug for HlsTerminalKeyBinding { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("HlsTerminalKeyBinding") - .field("proxy_session_id", &"") - .field("resource_id", &self.resource_id) - .field("route_extension", &self.route_extension) - .field("source_cache_key", &self.source_cache_key) - .field("content_type", &self.content_type) - .field("material", &"") - .finish() - } -} - -impl PartialEq for HlsTerminalKeyBinding { - fn eq(&self, other: &Self) -> bool { - self.proxy_session_id == other.proxy_session_id - && self.resource_id == other.resource_id - && self.route_extension == other.route_extension - && self.source_cache_key == other.source_cache_key - && self.content_type == other.content_type - && self.material.as_bytes() == other.material.as_bytes() - } -} - -impl Eq for HlsTerminalKeyBinding {} - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct HlsTerminalTailGeneration(pub u64); -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsMediaContainer { - MpegTs, - FragmentedMp4, - Unknown, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HlsMapSignature { - pub fingerprint: [u8; 32], - pub container: HlsMediaContainer, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HlsEncryptionSignature { - pub method: String, - pub key_uri: Option, - pub iv: Option, - pub key_format: Option, - pub key_format_versions: Option, - pub can_reset_to_clear: bool, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ParsedEncryptionMethod<'a> { - None, - Aes128, - Unsupported(&'a str), -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum HlsTerminalBaseScope { - SafeSuffix, - LastSafeSegment, -} - -fn parse_encryption_method(method: &str) -> ParsedEncryptionMethod<'_> { - if method.eq_ignore_ascii_case("NONE") { - ParsedEncryptionMethod::None - } else if method.eq_ignore_ascii_case("AES-128") { - ParsedEncryptionMethod::Aes128 - } else { - ParsedEncryptionMethod::Unsupported(method) - } -} - #[derive(Debug, Clone)] pub struct HlsTerminalMediaAsset { validated: Arc, @@ -195,121 +64,6 @@ struct HlsValidatedTerminalMediaAsset { renderer: Arc, } -impl PartialEq for HlsTerminalMediaAsset { - fn eq(&self, other: &Self) -> bool { - self.identity() == other.identity() - && self.container() == other.container() - && self.track_signature() == other.track_signature() - && self.duration_ms() == other.duration_ms() - && self.duration_ticks_90khz() == other.duration_ticks_90khz() - && self.timestamp_profile() == other.timestamp_profile() - && self.content_type() == other.content_type() - } -} - -impl Eq for HlsTerminalMediaAsset {} - -impl HlsTerminalMediaAsset { - fn identity(&self) -> HlsTerminalAssetIdentity { - HlsTerminalAssetIdentity { revision: self.validated.revision, fingerprint: self.validated.fingerprint } - } - - pub fn container(&self) -> HlsMediaContainer { self.validated.container } - - pub fn track_signature(&self) -> &HlsTsTrackSignature { &self.validated.track_signature } - - pub fn duration_ms(&self) -> u64 { self.validated.duration_ms } - - pub fn duration_ticks_90khz(&self) -> u64 { self.validated.duration_ticks_90khz } - - pub fn timestamp_profile(&self) -> Option { self.validated.timestamp_profile } - - pub fn content_type(&self) -> &'static str { self.validated.content_type } - - pub(super) fn renderer(&self) -> Arc { Arc::clone(&self.validated.renderer) } -} - -/// Captures and validates one immutable revision of the configured terminal TS asset. -pub fn snapshot_terminal_media_asset( - buffer: &TransportStreamBuffer, -) -> Result, HlsTerminalTailCompatibility> { - let (Some(duration_ms), Some(duration_ticks_90khz), Some(track_signature)) = - (buffer.duration_ms(), buffer.duration_ticks_90khz(), buffer.finite_hls_track_signature()) - else { - return Err(HlsTerminalTailCompatibility::InvalidAsset); - }; - let bytes = buffer.clone_bytes(); - if bytes.is_empty() { - return Err(HlsTerminalTailCompatibility::InvalidAsset); - } - let fingerprint = buffer.finite_hls_asset_fingerprint(); - let identity = terminal_asset_identity_from_fingerprint(fingerprint); - Ok(Arc::new(HlsTerminalMediaAsset { - validated: Arc::new(HlsValidatedTerminalMediaAsset { - revision: identity.revision, - fingerprint: identity.fingerprint, - container: HlsMediaContainer::MpegTs, - track_signature, - duration_ms, - duration_ticks_90khz, - timestamp_profile: buffer.finite_hls_timestamp_profile(), - content_type: "video/mp2t", - renderer: Arc::new(buffer.clone()), - }), - })) -} - -/// Reads only cached validation metadata from the configured buffer. This is -/// suitable for the final commit CAS and never parses, hashes, or clones media. -pub fn terminal_media_asset_identity(buffer: &TransportStreamBuffer) -> Option { - if buffer.as_bytes().is_empty() - || buffer.duration_ms().is_none() - || buffer.duration_ticks_90khz().is_none() - || !buffer.has_finite_hls_track_signature() - { - return None; - } - Some(terminal_asset_identity_from_fingerprint(buffer.finite_hls_asset_fingerprint())) -} - -fn terminal_asset_identity_from_fingerprint(fingerprint: [u8; 32]) -> HlsTerminalAssetIdentity { - let mut revision_bytes = [0_u8; std::mem::size_of::()]; - let revision_len = revision_bytes.len(); - revision_bytes.copy_from_slice(&fingerprint[..revision_len]); - HlsTerminalAssetIdentity { revision: u64::from_be_bytes(revision_bytes), fingerprint } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct HlsTerminalAssetIdentity { - pub revision: u64, - pub fingerprint: [u8; 32], -} - -impl HlsTerminalAssetIdentity { - pub fn from_asset(asset: &HlsTerminalMediaAsset) -> Self { asset.identity() } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsTerminalBaseMediaState { - Ready, - NotReady, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsTerminalBaseProtection { - Protectable, - Unavailable, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct HlsTerminalBaseSegmentAvailability { - pub proxy_seq: u64, - pub media_state: HlsTerminalBaseMediaState, - pub required_map_ready: bool, - pub required_key_ready: bool, - pub protection: HlsTerminalBaseProtection, -} - /// Immutable READY/cache evidence retained while a terminal plan is built and committed. /// /// The private reader pins close the gap between the session snapshot and publishing @@ -324,204 +78,20 @@ pub struct HlsTerminalBaseEvidence { read_protection: HlsTerminalBaseReadProtection, } -impl HlsTerminalBaseEvidence { - pub fn availability(&self) -> Arc<[HlsTerminalBaseSegmentAvailability]> { Arc::clone(&self.availability) } - - pub fn track_signature(&self) -> Option { - self.track_resolution.as_ref().and_then(HlsTrackEvidenceResolution::signature).cloned() - } - - pub fn track_resolution(&self) -> Option<&HlsTrackEvidenceResolution> { self.track_resolution.as_ref() } - - pub fn splice_evidence(&self) -> Option<&HlsTsSpliceEvidence> { self.splice_evidence.as_ref() } - - pub fn track_evidence_reason_code(&self) -> &'static str { - self.track_resolution.as_ref().map_or("base-not-ready", HlsTrackEvidenceResolution::reason_code) - } - - pub fn track_base(&self) -> Option<&HlsTerminalBaseTrackIdentity> { self.track_base.as_ref() } - - pub fn timing(&self) -> Option<&HlsTerminalBaseTimingEvidence> { self.timing.as_ref() } - - pub fn key_bindings(&self) -> Arc<[HlsTerminalKeyBinding]> { Arc::clone(&self.key_bindings) } - - /// Makes the intended guard lifetime explicit at the endpoint commit boundary. - #[cfg(any(test, feature = "test-support"))] - pub fn release(self) { drop(self.read_protection) } - - /// Transfers the READY-object reader pins to an autonomous terminal commit. - /// - /// The returned guard must stay alive until the lease/session transaction - /// either installs durable terminal-tail protection or reaches a terminal - /// rejection. This closes the request-cancellation and `LockBusy` gap. - pub fn into_commit_guard(self) -> HlsTerminalCommitMediaGuard { - HlsTerminalCommitMediaGuard { read_protection: self.read_protection } - } -} - /// Opaque ownership of the cache reader pins required by a pending terminal commit. pub struct HlsTerminalCommitMediaGuard { read_protection: HlsTerminalBaseReadProtection, } -#[cfg(any(test, feature = "test-support"))] -impl HlsTerminalCommitMediaGuard { - pub fn empty_for_test() -> Self { Self { read_protection: HlsTerminalBaseReadProtection { accesses: Vec::new() } } } -} - -impl std::fmt::Debug for HlsTerminalCommitMediaGuard { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("HlsTerminalCommitMediaGuard") - .field("pinned_objects", &self.read_protection.accesses.len()) - .finish() - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HlsTerminalBaseTrackIdentity { - pub proxy_seq: u64, - pub origin_epoch: u64, - pub cache_key: super::SegmentCacheKey, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct HlsTerminalBaseTimingEvidence { - pub base: HlsTerminalBaseTrackIdentity, - pub profile: HlsTsTimestampProfile, -} - struct HlsTerminalBaseReadProtection { accesses: Vec>, } -impl Drop for HlsTerminalBaseReadProtection { - fn drop(&mut self) { - for access in &self.accesses { - access.reader_finished(); - } - } -} - -/// Reads cache metadata and the last READY base segment without holding a session lock across I/O. -pub async fn prepare_terminal_base_evidence( - session: &HlsSessionHandle, - segment_cache: &HlsSegmentCache, - manifest: &HlsLeaseManifestSnapshot, - now_ms: u64, -) -> HlsTerminalBaseEvidence { - let preparation = { - let session = session.read().await; - pin_terminal_base_evidence(&session, manifest, now_ms) - }; - resolve_terminal_base_evidence(segment_cache, manifest, preparation).await -} - -/// Pins the exact READY media and key objects represented by one frozen lease -/// manifest. Callers can perform this while holding the same session read lock -/// used to select the lease, closing the selection-to-GC race without I/O. -pub fn pin_terminal_base_evidence( - session: &super::HlsSession, - manifest: &HlsLeaseManifestSnapshot, - now_ms: u64, -) -> HlsTerminalBaseEvidencePreparation { - snapshot_terminal_base_probes(session, manifest, now_ms) -} - -/// Resolves already pinned base objects and inspects the exact terminal track -/// basis. No session lock is held while cache metadata and bytes are read. -pub async fn resolve_terminal_base_evidence( - segment_cache: &HlsSegmentCache, - manifest: &HlsLeaseManifestSnapshot, - preparation: HlsTerminalBaseEvidencePreparation, -) -> HlsTerminalBaseEvidence { - let resolved = resolve_terminal_base_probes(segment_cache, manifest, preparation.probes).await; - let media_evidence = match resolved.last_media_probe { - Some(probe) => Some(terminal_base_media_evidence(probe).await), - None => None, - }; - let track_resolution = match media_evidence.as_ref() { - Some(evidence) => Some(evidence.track_resolution.clone()), - None => resolved.last_track_preprobe_resolution, - }; - let splice_evidence = media_evidence.as_ref().map(|evidence| evidence.splice_evidence.clone()); - if track_resolution.as_ref().and_then(HlsTrackEvidenceResolution::signature).is_none() { - debug!( - "HLS terminal base track evidence unavailable: reason={}", - track_resolution.as_ref().map_or("base-not-ready", HlsTrackEvidenceResolution::reason_code) - ); - } - HlsTerminalBaseEvidence { - availability: resolved.availability.into(), - track_resolution, - splice_evidence, - timing: media_evidence - .and_then(|evidence| evidence.timestamp_profile) - .zip(resolved.track_base.clone()) - .map(|(profile, base)| HlsTerminalBaseTimingEvidence { base, profile }), - track_base: resolved.track_base, - key_bindings: resolved.key_bindings.into(), - read_protection: preparation.read_protection, - } -} - pub struct HlsTerminalBaseEvidencePreparation { probes: Vec, read_protection: HlsTerminalBaseReadProtection, } -fn snapshot_terminal_base_probes( - session: &super::HlsSession, - manifest: &HlsLeaseManifestSnapshot, - now_ms: u64, -) -> HlsTerminalBaseEvidencePreparation { - let mut accesses = Vec::with_capacity(manifest.visible_segments.len()); - let mut pinned_key_objects = HashSet::new(); - let probes = manifest - .visible_segments - .iter() - .map(|segment| { - let entry = session.segments.get(&segment.proxy_seq); - let media_ready = entry.is_some_and(|entry| matches!(&entry.status, SegmentCacheStatus::Ready { .. })); - let required_map_ready = entry.is_some_and(|entry| { - entry.map_ref.is_none_or(|map_id| { - session - .maps - .get(&map_id) - .is_some_and(|map| matches!(&map.status, super::map::MapCacheStatus::Ready { .. })) - }) - }); - let key_evidence = terminal_base_key_evidence(session, entry, segment.encryption.as_deref()); - if let HlsTerminalBaseKeyEvidence::Aes128 { cache_key, object_access, resource_access, .. } = &key_evidence - { - if pinned_key_objects.insert(cache_key.clone()) { - object_access.reader_started(now_ms); - resource_access.reader_started(now_ms); - accesses.push(Arc::clone(object_access)); - accesses.push(Arc::clone(resource_access)); - } - } - let cache_key = media_ready.then(|| entry.map(|entry| entry.cache_key.clone())).flatten(); - if let Some(access) = media_ready.then(|| entry.map(|entry| Arc::clone(&entry.access))).flatten() { - // GC takes the session write lock before selecting removals, so acquiring - // the pin under this read lock cannot race behind an already queued removal. - access.reader_started(now_ms); - accesses.push(access); - } - HlsTerminalBaseSegmentProbe { - proxy_seq: segment.proxy_seq, - duration_ms: segment.duration_ms, - media_ready, - required_map_ready, - cache_key, - origin_epoch: media_ready.then(|| entry.map(|entry| entry.origin_key.origin_epoch)).flatten(), - key_evidence, - } - }) - .collect(); - HlsTerminalBaseEvidencePreparation { probes, read_protection: HlsTerminalBaseReadProtection { accesses } } -} - struct HlsResolvedTerminalBaseEvidence { availability: Vec, key_bindings: Vec, @@ -530,126 +100,6 @@ struct HlsResolvedTerminalBaseEvidence { track_base: Option, } -async fn resolve_terminal_base_probes( - segment_cache: &HlsSegmentCache, - manifest: &HlsLeaseManifestSnapshot, - probes: Vec, -) -> HlsResolvedTerminalBaseEvidence { - let mut availability = Vec::with_capacity(probes.len()); - let mut key_bindings = Vec::new(); - let mut key_binding_cache: HashMap> = HashMap::new(); - let mut last_media_probe = None; - let mut last_track_preprobe_resolution = None; - let mut track_base = None; - for probe in probes { - let metadata = if let Some(cache_key) = probe.cache_key.as_ref() { - segment_cache.metadata(cache_key).await.ok().flatten() - } else { - None - }; - let protectable = metadata.is_some(); - let (required_key_ready, track_encryption, key_failure) = match probe.key_evidence { - HlsTerminalBaseKeyEvidence::Clear => (true, Some(HlsTerminalTrackEncryption::Clear), None), - HlsTerminalBaseKeyEvidence::Aes128 { - proxy_session_id, - cache_key, - resource_id, - route_extension, - content_type, - iv, - .. - } => { - let binding = match key_binding_cache.entry(cache_key.clone()) { - Entry::Occupied(entry) => entry.get().clone(), - Entry::Vacant(entry) => { - let key_metadata = segment_cache.metadata(&cache_key).await.ok().flatten(); - let binding = if let Some(metadata) = - key_metadata.filter(|metadata| metadata.size == AES_128_BLOCK_BYTES as u64) - { - let bytes = read_bounded_file(&metadata.path, AES_128_BLOCK_BYTES as u64).await; - bytes.as_deref().and_then(|bytes| { - HlsTerminalKeyBinding::new( - proxy_session_id, - resource_id, - route_extension, - cache_key, - content_type, - bytes, - ) - }) - } else { - None - }; - if let Some(binding) = &binding { - key_bindings.push(binding.clone()); - } - entry.insert(binding).clone() - } - }; - let ready = binding.is_some(); - let encryption = binding.as_ref().map(|binding| HlsTerminalTrackEncryption::Aes128 { - key_material: Arc::clone(&binding.material), - iv, - }); - let failure = (!ready).then_some(HlsTrackEvidenceResolution::KeyUnavailable); - (ready, encryption, failure) - } - HlsTerminalBaseKeyEvidence::Unavailable { resolution } => (false, None, Some(resolution)), - }; - if probe.proxy_seq == manifest.last_proxy_seq && probe.media_ready && probe.required_map_ready { - if let (Some(cache_key), Some(origin_epoch)) = (probe.cache_key.as_ref(), probe.origin_epoch) { - track_base = Some(HlsTerminalBaseTrackIdentity { - proxy_seq: probe.proxy_seq, - origin_epoch, - cache_key: cache_key.clone(), - }); - if required_key_ready { - if let (Some(metadata), Some(encryption)) = (metadata.as_ref(), track_encryption) { - last_media_probe = Some(HlsTerminalMediaProbe { - segment_path: metadata.path.clone(), - source_size: metadata.size, - expected_duration_ticks_90khz: probe.duration_ms.saturating_mul(90), - encryption, - }); - } - } else { - last_track_preprobe_resolution = key_failure; - } - } - } - availability.push(HlsTerminalBaseSegmentAvailability { - proxy_seq: probe.proxy_seq, - media_state: terminal_base_media_state(probe.media_ready, protectable), - required_map_ready: probe.required_map_ready, - required_key_ready, - protection: terminal_base_protection(protectable), - }); - } - HlsResolvedTerminalBaseEvidence { - availability, - key_bindings, - last_media_probe, - last_track_preprobe_resolution, - track_base, - } -} - -const fn terminal_base_media_state(media_ready: bool, protectable: bool) -> HlsTerminalBaseMediaState { - if media_ready && protectable { - HlsTerminalBaseMediaState::Ready - } else { - HlsTerminalBaseMediaState::NotReady - } -} - -const fn terminal_base_protection(protectable: bool) -> HlsTerminalBaseProtection { - if protectable { - HlsTerminalBaseProtection::Protectable - } else { - HlsTerminalBaseProtection::Unavailable - } -} - struct HlsTerminalBaseSegmentProbe { proxy_seq: u64, duration_ms: u64, @@ -660,115 +110,6 @@ struct HlsTerminalBaseSegmentProbe { key_evidence: HlsTerminalBaseKeyEvidence, } -enum HlsTerminalBaseKeyEvidence { - Clear, - Aes128 { - proxy_session_id: ProxySessionId, - cache_key: TransientObjectCacheKey, - resource_id: TransientResourceId, - route_extension: String, - content_type: String, - iv: [u8; AES_128_BLOCK_BYTES], - object_access: Arc, - resource_access: Arc, - }, - Unavailable { - resolution: HlsTrackEvidenceResolution, - }, -} - -impl HlsTerminalBaseKeyEvidence { - fn unavailable(resolution: HlsTrackEvidenceResolution) -> Self { Self::Unavailable { resolution } } -} - -fn terminal_base_key_evidence( - session: &super::HlsSession, - timeline_entry: Option<&SegmentEntry>, - encryption: Option<&HlsEncryptionSignature>, -) -> HlsTerminalBaseKeyEvidence { - let Some(encryption) = encryption else { - return if timeline_entry.is_none_or(|entry| entry.encryption.is_none()) { - HlsTerminalBaseKeyEvidence::Clear - } else { - HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::IncompleteEvidence) - }; - }; - if parse_encryption_method(&encryption.method) != ParsedEncryptionMethod::Aes128 { - return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::UnsupportedProtection( - super::HlsTsProtectionReason::UnsupportedEncryption, - )); - } - let Some(resource_file) = encryption - .key_uri - .as_deref() - .and_then(|uri| uri.split(['?', '#']).next()) - .and_then(|path| path.rsplit('/').next()) - .and_then(TransientResourceFile::parse) - else { - return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); - }; - let Some(timeline_entry) = timeline_entry else { - return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); - }; - let Some(timeline_encryption) = timeline_entry.encryption.as_ref() else { - return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); - }; - if timeline_encryption.resource_id != resource_file.resource_id - || timeline_encryption.resource_extension != resource_file.extension - || timeline_encryption.iv != encryption.iv - || timeline_encryption.key_format != encryption.key_format - || timeline_encryption.key_format_versions != encryption.key_format_versions - { - return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); - } - let iv = match super::hls_aes128_cbc_iv(encryption.iv.as_deref(), timeline_entry.origin_key.host_local_sequence) { - Ok(iv) => iv, - Err(error) => { - return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::from(Err(error))); - } - }; - let Some(resource) = session.transient.resources.get(&resource_file.resource_id) else { - return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); - }; - if resource.kind != TransientResourceKind::Key - || resource.file_ext_hint.as_deref() != Some(resource_file.extension.as_str()) - { - return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); - } - let key = TransientPassthroughState::transient_object_key( - &session.proxy_session_id, - &resource_file.resource_id, - resource_file.extension.clone(), - ); - let Some(entry) = session.transient.object_cache.get(&key) else { - return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); - }; - // TTL controls admission to new live manifests, not whether exact READY key - // bytes referenced by this frozen lease snapshot may be pinned for terminal - // evidence. The reader pins acquired below close the GC race without any - // post-terminal origin fetch. - let ready = matches!(entry.status, super::TransientObjectCacheStatus::Ready { .. }); - if ready { - HlsTerminalBaseKeyEvidence::Aes128 { - proxy_session_id: session.proxy_session_id.clone(), - cache_key: entry.key.clone(), - resource_id: resource_file.resource_id, - route_extension: resource_file.extension, - content_type: entry.content_type.clone(), - iv, - object_access: Arc::clone(&entry.access), - resource_access: Arc::clone(&resource.access), - } - } else { - HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable) - } -} - -enum HlsTerminalTrackEncryption { - Clear, - Aes128 { key_material: Arc, iv: [u8; AES_128_BLOCK_BYTES] }, -} - struct HlsTerminalMediaProbe { segment_path: PathBuf, source_size: u64, @@ -782,152 +123,6 @@ struct HlsTerminalBaseMediaEvidence { splice_evidence: HlsTsSpliceEvidence, } -async fn terminal_base_media_evidence(probe: HlsTerminalMediaProbe) -> HlsTerminalBaseMediaEvidence { - let file = match tokio::fs::File::open(&probe.segment_path).await { - Ok(file) => file, - Err(error) => { - return HlsTerminalBaseMediaEvidence { - track_resolution: HlsTrackEvidenceResolution::Io(error.kind()), - timestamp_profile: None, - splice_evidence: HlsTsSpliceEvidence::Incompatible( - super::HlsTsSpliceIncompatibility::TopologyUnavailable, - ), - }; - } - }; - let budget = HlsTsProbeBudget { - max_bytes: probe.source_size.saturating_add(1), - max_packets: probe.source_size.saturating_add(187).saturating_div(188).saturating_add(1), - ..HlsTsProbeBudget::default() - }; - let outcome = match probe.encryption { - HlsTerminalTrackEncryption::Clear => { - super::inspect_mpeg_ts_media_evidence_async( - file, - HlsTsProbeProtection::Clear, - budget, - probe.expected_duration_ticks_90khz, - ) - .await - } - HlsTerminalTrackEncryption::Aes128 { key_material, iv } => { - super::inspect_mpeg_ts_media_evidence_async( - file, - HlsTsProbeProtection::Aes128Cbc { key: key_material.as_bytes(), iv }, - budget, - probe.expected_duration_ticks_90khz, - ) - .await - } - }; - match outcome { - Ok(evidence) => { - let super::HlsTsMediaEvidence { track_outcome, timestamp_profile, splice_evidence } = evidence; - HlsTerminalBaseMediaEvidence { - track_resolution: HlsTrackEvidenceResolution::from(Ok(track_outcome)), - timestamp_profile, - splice_evidence, - } - } - Err(error) => HlsTerminalBaseMediaEvidence { - track_resolution: HlsTrackEvidenceResolution::from(Err(error)), - timestamp_profile: None, - splice_evidence: HlsTsSpliceEvidence::Incompatible(super::HlsTsSpliceIncompatibility::TopologyUnavailable), - }, - } -} - -async fn read_bounded_file(path: &Path, max_bytes: u64) -> Option>> { - let file = tokio::fs::File::open(path).await.ok()?; - let mut limited = file.take(max_bytes.saturating_add(1)); - let mut bytes = Zeroizing::new(Vec::new()); - limited.read_to_end(&mut bytes).await.ok()?; - (u64::try_from(bytes.len()).unwrap_or(u64::MAX) <= max_bytes).then_some(bytes) -} - -#[derive(Clone)] -pub struct HlsTerminalTailBuildInput { - pub generation: HlsTerminalTailGeneration, - pub created_at_ms: u64, - pub base_manifest: HlsLeaseManifestSnapshot, - pub base_availability: Arc<[HlsTerminalBaseSegmentAvailability]>, - pub base_track_signature: Option, - pub base_splice_evidence: Option, - pub terminal_splice_evidence: Option, - pub base_timing: Option, - pub base_key_bindings: Arc<[HlsTerminalKeyBinding]>, - pub expected_asset: HlsRuntimeCustomTailAssetIdentity, - pub asset: Arc, - pub anchored_bundle: Arc, -} - -#[cfg(any(test, feature = "test-support"))] -impl HlsTerminalTailBuildInput { - /// # Panics - /// - /// Panics if the asset cannot produce a bundle, timing profile or splice - /// anchor. Test-support only: a fixture that cannot be built is a bug in - /// the test, not a runtime condition. - pub fn anchored_bundle_for_test( - asset: &HlsTerminalMediaAsset, - target_duration_ms: u64, - ) -> Arc { - let key = HlsPreparedTerminalBundleKey { - asset: HlsTerminalAssetIdentity::from_asset(asset), - target_duration_ms, - segment_count: HLS_TERMINAL_TAIL_SEGMENT_COUNT, - }; - let prepared = super::prepared_terminal_bundle::build_prepared_terminal_bundle(asset, key) - .expect("relative terminal test bundle"); - let profile = asset.timestamp_profile().expect("terminal test asset timing profile"); - let anchor = tuliprox_mpegts::transport_stream_buffer::HlsTsSpliceAnchor::between(profile, profile) - .expect("terminal test splice anchor"); - super::prepared_terminal_bundle::anchor_prepared_terminal_bundle(asset, &prepared, anchor) - .expect("anchored terminal test bundle") - } - - /// # Panics - /// - /// Panics if the manifest does not describe a usable base track. - /// Test-support only. - pub fn base_timing_for_test( - asset: &HlsTerminalMediaAsset, - manifest: &HlsLeaseManifestSnapshot, - ) -> HlsTerminalBaseTimingEvidence { - HlsTerminalBaseTimingEvidence { - base: HlsTerminalBaseTrackIdentity { - proxy_seq: manifest.last_proxy_seq, - origin_epoch: 1, - cache_key: super::SegmentCacheKey::new( - ProxySessionId("terminal-timing-test".to_string()), - manifest.last_proxy_seq, - "ts", - ), - }, - profile: asset.timestamp_profile().expect("terminal test asset timing profile"), - } - } - - pub fn compatible_splice_evidence_for_test(asset: &HlsTerminalMediaAsset) -> HlsTsSpliceEvidence { - HlsTsSpliceEvidence::compatible_for_test(asset.track_signature().clone()) - } -} - -#[derive(Clone, Copy)] -pub struct HlsTerminalTailCompatibilityInput<'a> { - pub manifest: &'a HlsLeaseManifestSnapshot, - pub base_track_signature: Option<&'a HlsTsTrackSignature>, - pub boundary_evidence: HlsTerminalTailBoundaryEvidence<'a>, - pub expected_asset: HlsTerminalAssetIdentity, - pub asset: &'a HlsTerminalMediaAsset, -} - -#[derive(Clone, Copy)] -pub enum HlsTerminalTailBoundaryEvidence<'a> { - StructuralOnly, - Exact { base: Option<&'a HlsTsSpliceEvidence>, terminal: Option<&'a HlsTsSpliceEvidence> }, -} - #[derive(Clone)] pub struct HlsTerminalTailPlan { pub generation: HlsTerminalTailGeneration, @@ -946,26 +141,6 @@ pub struct HlsTerminalTailPlan { segment_content_type: &'static str, } -impl std::fmt::Debug for HlsTerminalTailPlan { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("HlsTerminalTailPlan") - .field("generation", &self.generation) - .field("reason", &self.reason) - .field("created_at_ms", &self.created_at_ms) - .field("base_manifest_generation", &self.base_manifest.snapshot_generation) - .field("base_segment_count", &self.base_manifest.visible_segments.len()) - .field("protected_base_segment_count", &self.protected_base_proxy_seqs.len()) - .field("terminal_segment_count", &self.segment_count) - .field("segment_duration_ms", &self.segment_duration_ms) - .field("append_key_method_none", &self.append_key_method_none) - .field("key_binding_count", &self.key_bindings.len()) - .field("manifest_byte_len", &self.manifest_body.len()) - .field("segment_content_type", &self.segment_content_type) - .finish_non_exhaustive() - } -} - #[derive(Debug, Clone, PartialEq, Eq)] struct HlsTerminalTailRouteBinding { public_path_prefix: Arc, @@ -973,112 +148,6 @@ struct HlsTerminalTailRouteBinding { lease_id: HlsAccessLeaseId, } -impl PartialEq for HlsTerminalTailPlan { - fn eq(&self, other: &Self) -> bool { - self.generation == other.generation - && self.created_at_ms == other.created_at_ms - && self.base_manifest == other.base_manifest - && self.protected_base_proxy_seqs == other.protected_base_proxy_seqs - && self.reason == other.reason - && self.asset_identity == other.asset_identity - && self.segment_count == other.segment_count - && self.segment_duration_ms == other.segment_duration_ms - && self.append_key_method_none == other.append_key_method_none - && self.key_bindings == other.key_bindings - && self.route_binding == other.route_binding - && self.manifest_body == other.manifest_body - && self.anchored_bundle.prepared_key == other.anchored_bundle.prepared_key - && self.anchored_bundle.splice_anchor == other.anchored_bundle.splice_anchor - && self.segment_content_type == other.segment_content_type - } -} - -impl Eq for HlsTerminalTailPlan {} - -impl HlsTerminalTailPlan { - pub fn media_preparation_key(&self) -> super::recovery_timing::HlsTerminalMediaPreparationKey { - self.anchored_bundle.prepared_key - } - - pub fn matches_route(&self, proxy_session_id: &ProxySessionId, lease_id: &HlsAccessLeaseId) -> bool { - self.route_binding.proxy_session_id == *proxy_session_id && self.route_binding.lease_id == *lease_id - } - - pub fn segment_content_length(&self, path: HlsTerminalSegmentPath) -> Option { - self.prepared_segment(path).and_then(|segment| u64::try_from(segment.bytes.len()).ok()) - } - - pub fn segment_content_type(&self) -> &'static str { self.segment_content_type } - - /// Clones one immutable pre-rendered segment only for this plan generation. - pub fn segment_bytes(&self, path: HlsTerminalSegmentPath) -> Option { - self.prepared_segment(path).map(|segment| segment.bytes.clone()) - } - - fn prepared_segment(&self, path: HlsTerminalSegmentPath) -> Option<&HlsAnchoredTerminalSegment> { - if path.generation != self.generation || path.index >= self.segment_count { - return None; - } - let segment = self.anchored_bundle.segments.get(usize::from(path.index))?; - (segment.index == path.index).then_some(segment) - } - - pub fn terminal_key_binding( - &self, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, - resource_file: &TransientResourceFile, - ) -> Option { - self.matches_route(proxy_session_id, lease_id) - .then_some(()) - .and_then(|()| { - self.key_bindings - .iter() - .find(|binding| binding.matches_route(proxy_session_id) && binding.matches_resource(resource_file)) - }) - .cloned() - } - - pub fn key_bindings(&self) -> Arc<[HlsTerminalKeyBinding]> { Arc::clone(&self.key_bindings) } -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub enum HlsLeasePlaybackMode { - #[default] - Live, - TerminalTail(Arc), - TerminalUnavailable { - decision_generation: u64, - reason: HlsTerminalTailCompatibility, - }, - Ended, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsTerminalTailCompatibility { - Compatible, - MissingAsset, - TerminalMediaNotReady, - InvalidAsset, - AssetRevisionMismatch, - MissingSafeBase, - TargetDurationExceeded { asset_ms: u64, target_ms: u64 }, - ActiveMapRequiresCompatibleFallback, - UnsupportedEncryptionTransition, - ContainerMismatch, - MissingTrackSignature, - TrackLayoutMismatch, - MissingSpliceEvidence, - SpliceTransportFailure(super::HlsTsSpliceIncompatibility), - SpliceTopologyMismatch, - MissingTimestampAnchor, - InvalidTimestampTransition, - TransientPassthroughUnsupported, - ProtectionCapacityExceeded, - InvalidLeaseRoute, - ManifestRenderFailed, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct HlsTerminalSpliceDiagnostic { result: &'static str, @@ -1090,489 +159,6 @@ struct HlsTerminalSpliceDiagnostic { observed_pes_bytes: Option, } -impl HlsTerminalSpliceDiagnostic { - fn from_compatibility(compatibility: HlsTerminalTailCompatibility) -> Option { - let empty = |result| Self { - result, - pid: None, - packet_index: None, - expected_cc: None, - actual_cc: None, - declared_pes_bytes: None, - observed_pes_bytes: None, - }; - match compatibility { - HlsTerminalTailCompatibility::Compatible => Some(empty("compatible")), - HlsTerminalTailCompatibility::MissingTrackSignature - | HlsTerminalTailCompatibility::TrackLayoutMismatch - | HlsTerminalTailCompatibility::MissingSpliceEvidence - | HlsTerminalTailCompatibility::SpliceTopologyMismatch => Some(empty("topology-mismatch")), - HlsTerminalTailCompatibility::SpliceTransportFailure(reason) => { - let mut diagnostic = empty(reason.result_code()); - match reason { - super::HlsTsSpliceIncompatibility::InvalidPacket { packet_index } - | super::HlsTsSpliceIncompatibility::TransportError { packet_index, .. } => { - diagnostic.packet_index = Some(packet_index); - if let super::HlsTsSpliceIncompatibility::TransportError { pid, .. } = reason { - diagnostic.pid = Some(pid); - } - } - super::HlsTsSpliceIncompatibility::ContinuityFailure { pid, packet_index, expected, actual } => { - diagnostic.pid = Some(pid); - diagnostic.packet_index = Some(packet_index); - diagnostic.expected_cc = Some(expected); - diagnostic.actual_cc = Some(actual); - } - super::HlsTsSpliceIncompatibility::IncompletePes { - pid, - packet_index, - declared_bytes, - observed_bytes, - } => { - diagnostic.pid = Some(pid); - diagnostic.packet_index = Some(packet_index); - diagnostic.declared_pes_bytes = declared_bytes; - diagnostic.observed_pes_bytes = Some(observed_bytes); - } - super::HlsTsSpliceIncompatibility::InvalidPes { pid, packet_index } => { - diagnostic.pid = Some(pid); - diagnostic.packet_index = Some(packet_index); - } - super::HlsTsSpliceIncompatibility::InspectionBudgetExhausted - | super::HlsTsSpliceIncompatibility::TopologyUnavailable => {} - } - Some(diagnostic) - } - HlsTerminalTailCompatibility::MissingAsset - | HlsTerminalTailCompatibility::TerminalMediaNotReady - | HlsTerminalTailCompatibility::InvalidAsset - | HlsTerminalTailCompatibility::AssetRevisionMismatch - | HlsTerminalTailCompatibility::MissingSafeBase - | HlsTerminalTailCompatibility::TargetDurationExceeded { .. } - | HlsTerminalTailCompatibility::ActiveMapRequiresCompatibleFallback - | HlsTerminalTailCompatibility::UnsupportedEncryptionTransition - | HlsTerminalTailCompatibility::ContainerMismatch - | HlsTerminalTailCompatibility::MissingTimestampAnchor - | HlsTerminalTailCompatibility::InvalidTimestampTransition - | HlsTerminalTailCompatibility::TransientPassthroughUnsupported - | HlsTerminalTailCompatibility::ProtectionCapacityExceeded - | HlsTerminalTailCompatibility::InvalidLeaseRoute - | HlsTerminalTailCompatibility::ManifestRenderFailed => None, - } - } -} - -fn format_optional_diagnostic(value: Option) -> String { - value.map_or_else(|| "none".to_string(), |value| value.to_string()) -} - -fn log_terminal_splice_compatibility(base_proxy_seq: u64, compatibility: HlsTerminalTailCompatibility) { - let Some(diagnostic) = HlsTerminalSpliceDiagnostic::from_compatibility(compatibility) else { - return; - }; - debug!( - "HLS terminal TS splice eligibility: base_proxy_seq={} result={} pid={} packet_index={} \ - expected_cc={} actual_cc={} declared_pes_bytes={} observed_pes_bytes={}", - base_proxy_seq, - diagnostic.result, - format_optional_diagnostic(diagnostic.pid), - format_optional_diagnostic(diagnostic.packet_index), - format_optional_diagnostic(diagnostic.expected_cc), - format_optional_diagnostic(diagnostic.actual_cc), - format_optional_diagnostic(diagnostic.declared_pes_bytes), - format_optional_diagnostic(diagnostic.observed_pes_bytes), - ); -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct HlsTerminalSegmentPath { - pub generation: HlsTerminalTailGeneration, - pub index: u16, -} - -impl HlsTerminalSegmentPath { - pub fn parse(generation: &str, terminal_file: &str) -> Option { - let index = terminal_file.strip_suffix(".ts")?; - Some(Self { - generation: HlsTerminalTailGeneration(parse_canonical_decimal(generation)?), - index: parse_canonical_decimal(index)?, - }) - } -} - -fn parse_canonical_decimal(value: &str) -> Option -where - T: std::str::FromStr, -{ - if value.is_empty() - || (value.len() > 1 && value.starts_with('0')) - || !value.bytes().all(|byte| byte.is_ascii_digit()) - { - return None; - } - value.parse().ok() -} - -#[derive(Debug, thiserror::Error, PartialEq, Eq)] -pub enum HlsTerminalTailRenderError { - #[error("terminal tail has no safe base segments")] - MissingSafeBase, - #[error("terminal tail asset identity changed")] - AssetIdentityChanged, - #[error("terminal tail has an invalid encryption state")] - InvalidEncryptionState, - #[error("terminal tail formatting failed")] - Formatting, - #[error("terminal tail route does not match its frozen lease binding")] - RouteBindingMismatch, -} - -pub fn evaluate_terminal_tail_compatibility( - input: HlsTerminalTailCompatibilityInput<'_>, -) -> HlsTerminalTailCompatibility { - let manifest = input.manifest; - let asset = input.asset; - if manifest.delivery_mode == HlsManifestDeliveryMode::TransientPassthrough { - return HlsTerminalTailCompatibility::TransientPassthroughUnsupported; - } - if asset.duration_ms() == 0 || asset.duration_ticks_90khz() == 0 { - return HlsTerminalTailCompatibility::InvalidAsset; - } - if input.expected_asset != HlsTerminalAssetIdentity::from_asset(asset) { - return HlsTerminalTailCompatibility::AssetRevisionMismatch; - } - if asset.content_type() != "video/mp2t" { - return HlsTerminalTailCompatibility::InvalidAsset; - } - if manifest.active_map.is_some() { - return HlsTerminalTailCompatibility::ActiveMapRequiresCompatibleFallback; - } - if manifest.container != HlsMediaContainer::MpegTs || asset.container() != HlsMediaContainer::MpegTs { - return HlsTerminalTailCompatibility::ContainerMismatch; - } - let Some(base_track_signature) = input.base_track_signature else { - return HlsTerminalTailCompatibility::MissingTrackSignature; - }; - if base_track_signature != asset.track_signature() { - return HlsTerminalTailCompatibility::TrackLayoutMismatch; - } - if let HlsTerminalTailBoundaryEvidence::Exact { base, terminal } = input.boundary_evidence { - let (Some(base), Some(terminal)) = (base, terminal) else { - return HlsTerminalTailCompatibility::MissingSpliceEvidence; - }; - match super::evaluate_mpeg_ts_splice_boundary(base, terminal) { - Ok(()) => {} - Err(super::HlsTsSpliceBoundaryIncompatibility::Media(reason)) => { - return HlsTerminalTailCompatibility::SpliceTransportFailure(reason); - } - Err(super::HlsTsSpliceBoundaryIncompatibility::TopologyMismatch) => { - return HlsTerminalTailCompatibility::SpliceTopologyMismatch; - } - } - } - if encryption_reset_required(manifest.active_encryption.as_deref()).is_err() { - return HlsTerminalTailCompatibility::UnsupportedEncryptionTransition; - } - if !terminal_asset_fits_target_duration(asset.duration_ms(), manifest.target_duration_ms) { - return HlsTerminalTailCompatibility::TargetDurationExceeded { - asset_ms: asset.duration_ms(), - target_ms: manifest.target_duration_ms, - }; - } - HlsTerminalTailCompatibility::Compatible -} - -fn encryption_reset_required( - encryption: Option<&HlsEncryptionSignature>, -) -> Result { - let Some(encryption) = encryption else { - return Ok(false); - }; - match parse_encryption_method(&encryption.method) { - ParsedEncryptionMethod::None => Ok(false), - ParsedEncryptionMethod::Aes128 - if encryption.can_reset_to_clear - && encryption.key_format.as_deref().is_none_or(|format| format.eq_ignore_ascii_case("identity")) - && encryption.key_uri.as_deref().is_some_and(valid_quoted_attribute) - && encryption.iv.as_deref().is_none_or(valid_iv) - && encryption.key_format_versions.as_deref().is_none_or(valid_key_format_versions) - && (encryption.key_format_versions.is_none() - || encryption - .key_format - .as_deref() - .is_some_and(|format| format.eq_ignore_ascii_case("identity"))) => - { - Ok(true) - } - ParsedEncryptionMethod::Aes128 | ParsedEncryptionMethod::Unsupported(_) => { - Err(HlsTerminalTailCompatibility::UnsupportedEncryptionTransition) - } - } -} - -fn valid_quoted_attribute(value: &str) -> bool { - !value.is_empty() && value.chars().all(|character| character != '"' && !character.is_control()) -} - -fn valid_iv(value: &str) -> bool { super::hls_aes128_cbc_iv(Some(value), 0).is_ok() } - -fn valid_key_format_versions(value: &str) -> bool { - !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit() || byte == b'/') -} - -fn safe_terminal_base( - mut manifest: HlsLeaseManifestSnapshot, - availability: &[HlsTerminalBaseSegmentAvailability], - scope: HlsTerminalBaseScope, -) -> Option<(HlsLeaseManifestSnapshot, Arc<[u64]>)> { - let mut availability_by_seq = HashMap::>::new(); - for state in availability { - match availability_by_seq.entry(state.proxy_seq) { - Entry::Vacant(entry) => { - entry.insert(Some(state)); - } - Entry::Occupied(mut entry) => { - entry.insert(None); - } - } - } - - let mut start = manifest.visible_segments.len(); - let mut expected_next = None; - for (index, segment) in manifest.visible_segments.iter().enumerate().rev() { - if expected_next.is_some_and(|next| segment.proxy_seq.checked_add(1) != Some(next)) { - break; - } - let state = availability_by_seq.get(&segment.proxy_seq).copied().flatten(); - let safe = state.is_some_and(|state| { - state.media_state == HlsTerminalBaseMediaState::Ready - && state.required_map_ready - && state.required_key_ready - && state.protection == HlsTerminalBaseProtection::Protectable - && segment.map_ref_ready - && segment.encryption == manifest.active_encryption - }); - if !safe { - break; - } - start = index; - expected_next = Some(segment.proxy_seq); - } - if start == manifest.visible_segments.len() { - return None; - } - if scope == HlsTerminalBaseScope::LastSafeSegment { - start = manifest.visible_segments.len().saturating_sub(1); - } - - let mut selected = manifest.visible_segments[start..].to_vec(); - let removed_discontinuities = - manifest.visible_segments[..start].iter().filter(|segment| segment.discontinuity_before).count(); - if start > 0 && selected.first().is_some_and(|segment| segment.discontinuity_before) { - if let Some(first) = selected.first_mut() { - first.discontinuity_before = false; - } - manifest.discontinuity_sequence = manifest.discontinuity_sequence.saturating_add(1); - } - manifest.discontinuity_sequence = - manifest.discontinuity_sequence.saturating_add(u64::try_from(removed_discontinuities).unwrap_or(u64::MAX)); - let first_proxy_seq = selected.first()?.proxy_seq; - let last_proxy_seq = selected.last()?.proxy_seq; - let playlist_duration_ms = - selected.iter().fold(0u64, |duration, segment| duration.saturating_add(segment.duration_ms)); - let protected = Arc::from(selected.iter().map(|segment| segment.proxy_seq).collect::>()); - manifest.first_proxy_seq = first_proxy_seq; - manifest.last_proxy_seq = last_proxy_seq; - manifest.playlist_duration_ms = playlist_duration_ms; - manifest.last_visible_media_end_ms = playlist_duration_ms; - manifest.visible_segments = Arc::from(selected); - Some((manifest, protected)) -} - -fn terminal_tail_route_binding(manifest: &HlsLeaseManifestSnapshot) -> Option { - let first = manifest.visible_segments.first()?; - let first_uri = manifest.materialize_uri(&first.uri); - let binding = terminal_tail_route_binding_from_uri(&first_uri)?; - manifest - .visible_segments - .iter() - .all(|segment| { - terminal_tail_route_binding_from_uri(&manifest.materialize_uri(&segment.uri)).as_ref() == Some(&binding) - }) - .then_some(binding) -} - -fn terminal_tail_route_binding_from_uri(uri: &str) -> Option { - let path = uri.split(['?', '#']).next()?; - let marker_offset = path.find(HLS_SHARED_LIVE_ROUTE_MARKER)?; - let public_path_prefix = path.get(..marker_offset)?; - if !public_path_prefix.is_empty() && (!public_path_prefix.starts_with('/') || public_path_prefix.ends_with('/')) { - return None; - } - let route = path.get(marker_offset.saturating_add(HLS_SHARED_LIVE_ROUTE_MARKER.len())..)?; - let mut components = route.split('/'); - let proxy_session_id = components.next()?; - let lease_id = components.next()?; - let media_file = components.next()?; - if !valid_terminal_route_component(proxy_session_id) - || !valid_terminal_route_component(lease_id) - || media_file.is_empty() - || components.next().is_some() - || public_path_prefix.chars().any(char::is_control) - || media_file.chars().any(char::is_control) - { - return None; - } - Some(HlsTerminalTailRouteBinding { - public_path_prefix: Arc::from(public_path_prefix), - proxy_session_id: ProxySessionId(proxy_session_id.to_string()), - lease_id: HlsAccessLeaseId(lease_id.to_string()), - }) -} - -fn valid_terminal_route_component(value: &str) -> bool { - !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) -} - -pub fn build_terminal_tail_plan( - input: HlsTerminalTailBuildInput, -) -> Result { - let compatibility = evaluate_terminal_tail_compatibility(HlsTerminalTailCompatibilityInput { - manifest: &input.base_manifest, - base_track_signature: input.base_track_signature.as_ref(), - boundary_evidence: HlsTerminalTailBoundaryEvidence::Exact { - base: input.base_splice_evidence.as_ref(), - terminal: input.terminal_splice_evidence.as_ref(), - }, - expected_asset: input.expected_asset.media, - asset: &input.asset, - }); - log_terminal_splice_compatibility(input.base_manifest.last_proxy_seq, compatibility); - if compatibility != HlsTerminalTailCompatibility::Compatible { - return Err(compatibility); - } - let expected_bundle_key = HlsPreparedTerminalBundleKey { - asset: input.expected_asset.media, - target_duration_ms: input.base_manifest.target_duration_ms, - segment_count: HLS_TERMINAL_TAIL_SEGMENT_COUNT, - }; - if !input.anchored_bundle.matches_key_and_shape(expected_bundle_key, input.asset.duration_ticks_90khz()) { - return Err(HlsTerminalTailCompatibility::AssetRevisionMismatch); - } - let Some(base_timing) = input.base_timing.as_ref() else { - return Err(HlsTerminalTailCompatibility::MissingTimestampAnchor); - }; - let Some(asset_profile) = input.asset.timestamp_profile() else { - return Err(HlsTerminalTailCompatibility::InvalidTimestampTransition); - }; - let Some(expected_anchor) = - tuliprox_mpegts::transport_stream_buffer::HlsTsSpliceAnchor::between(base_timing.profile, asset_profile) - else { - return Err(HlsTerminalTailCompatibility::InvalidTimestampTransition); - }; - if base_timing.base.proxy_seq != input.base_manifest.last_proxy_seq - || input.anchored_bundle.splice_anchor != expected_anchor - { - return Err(HlsTerminalTailCompatibility::InvalidTimestampTransition); - } - let append_key_method_none = encryption_reset_required(input.base_manifest.active_encryption.as_deref())?; - let Some((base_manifest, protected_base_proxy_seqs)) = safe_terminal_base( - input.base_manifest, - &input.base_availability, - if append_key_method_none { HlsTerminalBaseScope::LastSafeSegment } else { HlsTerminalBaseScope::SafeSuffix }, - ) else { - return Err(HlsTerminalTailCompatibility::MissingSafeBase); - }; - let Some(route_binding) = terminal_tail_route_binding(&base_manifest) else { - return Err(HlsTerminalTailCompatibility::InvalidLeaseRoute); - }; - let key_bindings = terminal_key_bindings(&base_manifest, &input.base_key_bindings, &route_binding)?; - let segment_count = input.anchored_bundle.prepared_key.segment_count; - let segment_duration_ms = input.asset.duration_ms(); - let segment_content_type = input.asset.content_type(); - let manifest_body = render_terminal_tail_manifest_body(&HlsTerminalTailManifestRenderInput { - generation: input.generation, - base_manifest: &base_manifest, - protected_base_proxy_seqs: &protected_base_proxy_seqs, - asset: &input.asset, - asset_identity: input.expected_asset.media, - segment_count, - segment_duration_ms, - append_key_method_none, - route_binding: &route_binding, - }) - .map(Arc::from) - .map_err(|_| HlsTerminalTailCompatibility::ManifestRenderFailed)?; - Ok(HlsTerminalTailPlan { - generation: input.generation, - created_at_ms: input.created_at_ms, - segment_count, - segment_duration_ms, - append_key_method_none, - key_bindings, - base_manifest, - protected_base_proxy_seqs, - reason: input.expected_asset.reason, - asset_identity: input.expected_asset, - route_binding, - manifest_body, - anchored_bundle: input.anchored_bundle, - segment_content_type, - }) -} - -fn terminal_key_bindings( - manifest: &HlsLeaseManifestSnapshot, - evidence: &[HlsTerminalKeyBinding], - route_binding: &HlsTerminalTailRouteBinding, -) -> Result, HlsTerminalTailCompatibility> { - let mut selected = Vec::new(); - for encryption in manifest.visible_segments.iter().filter_map(|segment| segment.encryption.as_ref()) { - match parse_encryption_method(&encryption.method) { - ParsedEncryptionMethod::None => continue, - ParsedEncryptionMethod::Aes128 => {} - ParsedEncryptionMethod::Unsupported(_) => { - return Err(HlsTerminalTailCompatibility::UnsupportedEncryptionTransition); - } - } - let Some(resource_file) = terminal_key_resource_file(encryption) else { - return Err(HlsTerminalTailCompatibility::MissingSafeBase); - }; - let mut matches = evidence.iter().filter(|binding| { - binding.matches_route(&route_binding.proxy_session_id) && binding.matches_resource(&resource_file) - }); - let Some(binding) = matches.next().cloned() else { - return Err(HlsTerminalTailCompatibility::MissingSafeBase); - }; - if matches.any(|candidate| candidate != &binding) { - return Err(HlsTerminalTailCompatibility::MissingSafeBase); - } - if !selected.contains(&binding) { - selected.push(binding); - } - } - selected.sort_by(|left, right| { - (&left.resource_id.0, left.route_extension.as_ref()) - .cmp(&(&right.resource_id.0, right.route_extension.as_ref())) - }); - Ok(selected.into()) -} - -fn terminal_key_resource_file(encryption: &HlsEncryptionSignature) -> Option { - encryption.key_uri.as_deref()?.split(['?', '#']).next()?.rsplit('/').next().and_then(TransientResourceFile::parse) -} - -pub fn terminal_tail_manifest_body<'a>( - plan: &'a HlsTerminalTailPlan, - proxy_session_id: &ProxySessionId, - lease_id: &HlsAccessLeaseId, -) -> Result<&'a str, HlsTerminalTailRenderError> { - if !plan.matches_route(proxy_session_id, lease_id) { - return Err(HlsTerminalTailRenderError::RouteBindingMismatch); - } - Ok(&plan.manifest_body) -} - struct HlsTerminalTailManifestRenderInput<'a> { generation: HlsTerminalTailGeneration, base_manifest: &'a HlsLeaseManifestSnapshot, @@ -1585,883 +171,33 @@ struct HlsTerminalTailManifestRenderInput<'a> { route_binding: &'a HlsTerminalTailRouteBinding, } -fn render_terminal_tail_manifest_body( - input: &HlsTerminalTailManifestRenderInput<'_>, -) -> Result { - if input.asset_identity != HlsTerminalAssetIdentity::from_asset(input.asset) { - return Err(HlsTerminalTailRenderError::AssetIdentityChanged); - } - let Some(first) = input.base_manifest.visible_segments.first() else { - return Err(HlsTerminalTailRenderError::MissingSafeBase); - }; - if input.protected_base_proxy_seqs != input.base_manifest.visible_proxy_seqs().collect::>().as_slice() { - return Err(HlsTerminalTailRenderError::MissingSafeBase); - } - let mut body = String::new(); - let version = input.base_manifest.active_encryption.as_ref().map_or(3, |encryption| { - if encryption.key_format.is_some() || encryption.key_format_versions.is_some() { - 5 - } else { - 3 - } - }); - writeln!(body, "#EXTM3U\n#EXT-X-VERSION:{version}").map_err(|_| HlsTerminalTailRenderError::Formatting)?; - writeln!(body, "#EXT-X-TARGETDURATION:{}", hls_target_duration_secs(input.base_manifest.target_duration_ms)) - .map_err(|_| HlsTerminalTailRenderError::Formatting)?; - writeln!(body, "#EXT-X-MEDIA-SEQUENCE:{}", first.proxy_seq).map_err(|_| HlsTerminalTailRenderError::Formatting)?; - writeln!(body, "#EXT-X-DISCONTINUITY-SEQUENCE:{}", input.base_manifest.discontinuity_sequence) - .map_err(|_| HlsTerminalTailRenderError::Formatting)?; - render_active_base_encryption(&mut body, input.base_manifest)?; - for segment in input.base_manifest.visible_segments.iter() { - if segment.discontinuity_before { - body.push_str("#EXT-X-DISCONTINUITY\n"); - } - writeln!(body, "#EXTINF:{},", format_hls_duration_ms(segment.duration_ms)) - .map_err(|_| HlsTerminalTailRenderError::Formatting)?; - writeln!(body, "{}", input.base_manifest.materialize_uri(&segment.uri)) - .map_err(|_| HlsTerminalTailRenderError::Formatting)?; - } - if input.append_key_method_none { - body.push_str("#EXT-X-KEY:METHOD=NONE\n"); - } - body.push_str("#EXT-X-DISCONTINUITY\n"); - for index in 0..input.segment_count { - writeln!(body, "#EXTINF:{},", format_hls_duration_ms(input.segment_duration_ms)) - .map_err(|_| HlsTerminalTailRenderError::Formatting)?; - writeln!( - body, - "{}/hls/shared/live/{}/{}/terminal/{}/{}.ts", - input.route_binding.public_path_prefix, - input.route_binding.proxy_session_id.0, - input.route_binding.lease_id.0, - input.generation.0, - index - ) - .map_err(|_| HlsTerminalTailRenderError::Formatting)?; - } - body.push_str("#EXT-X-ENDLIST\n"); - Ok(body) -} - -fn render_active_base_encryption( - body: &mut String, - manifest: &HlsLeaseManifestSnapshot, -) -> Result<(), HlsTerminalTailRenderError> { - let encryption = manifest.active_encryption.as_deref(); - let Some(encryption) = encryption else { - return Ok(()); - }; - match parse_encryption_method(&encryption.method) { - ParsedEncryptionMethod::None => Ok(()), - ParsedEncryptionMethod::Aes128 => { - encryption_reset_required(Some(encryption)) - .map_err(|_| HlsTerminalTailRenderError::InvalidEncryptionState)?; - let Some(key_uri) = encryption.key_uri.as_deref() else { - return Err(HlsTerminalTailRenderError::InvalidEncryptionState); - }; - let key_uri = manifest.materialize_uri(key_uri); - write!(body, "#EXT-X-KEY:METHOD=AES-128,URI=\"{key_uri}\"") - .map_err(|_| HlsTerminalTailRenderError::Formatting)?; - if let Some(iv) = encryption.iv.as_deref() { - write!(body, ",IV={iv}").map_err(|_| HlsTerminalTailRenderError::Formatting)?; - } - if encryption.key_format.as_deref().is_some_and(|format| format.eq_ignore_ascii_case("identity")) { - body.push_str(",KEYFORMAT=\"identity\""); - } - if let Some(versions) = encryption.key_format_versions.as_deref() { - write!(body, ",KEYFORMATVERSIONS=\"{versions}\"") - .map_err(|_| HlsTerminalTailRenderError::Formatting)?; - } - body.push('\n'); - Ok(()) - } - ParsedEncryptionMethod::Unsupported(_) => Err(HlsTerminalTailRenderError::InvalidEncryptionState), - } -} - #[cfg(test)] -mod tests { - use super::{ - super::media_reserve::{HlsLeaseManifestSegment, HlsManifestCommitIdentity, HlsManifestDeliveryMode}, - *, - }; - use tuliprox_mpegts::transport_stream_buffer::{HlsFiniteTsRenderSpec, TransportStreamBuffer}; +mod tests; - const TERMINAL_ASSET_BYTES: &[u8] = - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); - - fn manifest() -> HlsLeaseManifestSnapshot { - HlsLeaseManifestSnapshot { - delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, - source_commit_identity: HlsManifestCommitIdentity::new(1), - uri_materialization: None, - finalized_transient_manifest_generation: None, - snapshot_generation: 1, - delivered_at_ms: 10, - first_proxy_seq: 193, - last_proxy_seq: 194, - visible_segments: Arc::from([ - HlsLeaseManifestSegment { - proxy_seq: 193, - duration_ms: 9_940, - uri: "/iptv/hls/shared/live/session/lease/193.ts".into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }, - HlsLeaseManifestSegment { - proxy_seq: 194, - duration_ms: 10_380, - uri: "/iptv/hls/shared/live/session/lease/194.ts".into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - }, - ]), - discontinuity_sequence: 7, - target_duration_ms: 12_000, - playlist_duration_ms: 20_320, - last_visible_media_end_ms: 20_320, - active_map: None, - active_encryption: None, - container: HlsMediaContainer::MpegTs, - } - } - - fn asset() -> Arc { - let bytes = Bytes::from_static(TERMINAL_ASSET_BYTES); - let buffer = TransportStreamBuffer::new(bytes.to_vec()); - snapshot_terminal_media_asset(&buffer).expect("valid terminal asset") - } - - fn availability(base: &HlsLeaseManifestSnapshot) -> Arc<[HlsTerminalBaseSegmentAvailability]> { - Arc::from( - base.visible_proxy_seqs() - .map(|proxy_seq| HlsTerminalBaseSegmentAvailability { - proxy_seq, - media_state: HlsTerminalBaseMediaState::Ready, - required_map_ready: true, - required_key_ready: true, - protection: HlsTerminalBaseProtection::Protectable, - }) - .collect::>(), - ) - } - - fn build_input( - generation: u64, - base_manifest: HlsLeaseManifestSnapshot, - asset: Arc, - ) -> HlsTerminalTailBuildInput { - let anchored_bundle = - HlsTerminalTailBuildInput::anchored_bundle_for_test(&asset, base_manifest.target_duration_ms); - let base_timing = Some(HlsTerminalTailBuildInput::base_timing_for_test(&asset, &base_manifest)); - let base_splice_evidence = Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); - let terminal_splice_evidence = base_splice_evidence.clone(); - HlsTerminalTailBuildInput { - generation: HlsTerminalTailGeneration(generation), - created_at_ms: 20, - base_availability: availability(&base_manifest), - base_track_signature: Some(asset.track_signature().clone()), - base_splice_evidence, - terminal_splice_evidence, - base_timing, - base_key_bindings: Arc::from([]), - expected_asset: HlsRuntimeCustomTailAssetIdentity { - reason: HlsRuntimeCustomTailReason::ChannelUnavailable, - media: HlsTerminalAssetIdentity::from_asset(&asset), - }, - base_manifest, - asset, - anchored_bundle, - } - } - - fn compatibility( - base: &HlsLeaseManifestSnapshot, - asset: &HlsTerminalMediaAsset, - base_track_signature: Option<&HlsTsTrackSignature>, - ) -> HlsTerminalTailCompatibility { - evaluate_terminal_tail_compatibility(HlsTerminalTailCompatibilityInput { - manifest: base, - base_track_signature, - boundary_evidence: HlsTerminalTailBoundaryEvidence::StructuralOnly, - expected_asset: HlsTerminalAssetIdentity::from_asset(asset), - asset, - }) - } - - fn resettable_aes128_encryption() -> Arc { - Arc::new(HlsEncryptionSignature { - method: "AES-128".into(), - key_uri: Some("/iptv/hls/shared/live/session/lease/r/aes-key.key".into()), - iv: Some("0x000000000000000000000000000000C2".into()), - key_format: Some("identity".into()), - key_format_versions: Some("1".into()), - can_reset_to_clear: true, - }) - } - - fn aes128_key_binding() -> HlsTerminalKeyBinding { - HlsTerminalKeyBinding::new( - ProxySessionId("session".into()), - TransientResourceId("aes-key".into()), - "key".into(), - TransientObjectCacheKey::new( - ProxySessionId("session".into()), - TransientResourceId("aes-key".into()), - "key.fill-0000000000000001", - ), - "application/octet-stream".into(), - b"0123456789abcdef", - ) - .expect("valid AES-128 binding") - } - - async fn unresolved_aes_terminal_base_evidence(iv: &str) -> HlsTerminalBaseEvidence { - let temp_dir = tempfile::tempdir().expect("terminal evidence cache tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let mut session = - super::super::HlsSession::new(super::super::HlsSessionKey::new(1, "terminal-evidence"), b"secret", 1); - let proxy_seq = 194; - let cache_key = super::super::SegmentCacheKey::new(session.proxy_session_id.clone(), proxy_seq, "ts"); - cache - .write_bytes_and_commit(&cache_key, TERMINAL_ASSET_BYTES) - .await - .expect("terminal evidence segment commits"); - let timeline_encryption = super::super::HlsSegmentEncryption { - resource_id: TransientResourceId("aes-key".into()), - resource_extension: "key".into(), - iv: Some(iv.into()), - key_format: Some("identity".into()), - key_format_versions: Some("1".into()), - }; - session.segments.insert( - proxy_seq, - SegmentEntry { - origin_key: super::super::OriginSegmentKey { - origin_epoch: 3, - effective_host_id: 7, - host_local_sequence: 900, - host_local_index: 0, - }, - proxy_seq, - duration_ms: 4_000, - proxy_file_ext: "ts".into(), - content_type: "video/mp2t".into(), - cache_key, - discontinuity_before: false, - program_date_time: None, - daterange_tags_before: Vec::new(), - origin_byte_range: None, - map_ref: None, - encryption: Some(timeline_encryption), - origin_fetch_ref: None, - status: SegmentCacheStatus::Ready { - content_length: u64::try_from(TERMINAL_ASSET_BYTES.len()).unwrap_or(u64::MAX), - ready_at_ms: 2, - }, - last_rendered_at_ms: Some(2), - access: Arc::new(CacheAccessState::new()), - }, - ); - let encryption = HlsEncryptionSignature { - method: "AES-128".into(), - key_uri: Some("/iptv/hls/shared/live/session/lease/r/aes-key.key".into()), - iv: Some(iv.into()), - key_format: Some("identity".into()), - key_format_versions: Some("1".into()), - can_reset_to_clear: true, - }; - let mut base = manifest(); - base.first_proxy_seq = proxy_seq; - base.last_proxy_seq = proxy_seq; - base.visible_segments = Arc::from([HlsLeaseManifestSegment { - proxy_seq, - duration_ms: 4_000, - uri: format!("/iptv/hls/shared/live/{}/lease/{proxy_seq}.ts", session.proxy_session_id.0).into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: Some(Arc::new(encryption.clone())), - }]); - base.active_encryption = Some(Arc::new(encryption)); - base.playlist_duration_ms = 4_000; - base.last_visible_media_end_ms = 4_000; - let session = Arc::new(tokio::sync::RwLock::new(session)); - - prepare_terminal_base_evidence(&session, &cache, &base, 3).await - } - - #[tokio::test] - async fn terminal_base_media_evidence_scans_complete_cache_file() { - let mut file = tempfile::NamedTempFile::new().expect("temporary cache file"); - std::io::Write::write_all(&mut file, TERMINAL_ASSET_BYTES).expect("write valid TS"); - let source_size = file.as_file().metadata().expect("cache metadata").len(); - let terminal_asset = asset(); - let probe = HlsTerminalMediaProbe { - segment_path: file.path().to_path_buf(), - source_size, - expected_duration_ticks_90khz: terminal_asset.duration_ticks_90khz(), - encryption: HlsTerminalTrackEncryption::Clear, - }; - - let evidence = terminal_base_media_evidence(probe).await; - let expected_signature = terminal_asset.track_signature().clone(); - - assert_eq!(evidence.track_resolution.signature(), Some(&expected_signature)); - assert_eq!(evidence.track_resolution.reason_code(), "found"); - assert_eq!(evidence.timestamp_profile, terminal_asset.timestamp_profile()); - assert!(matches!(evidence.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); - } - - async fn terminal_timing_session( - cache: &HlsSegmentCache, - terminal_asset: &HlsTerminalMediaAsset, - first_bytes: &[u8], - last_bytes: &[u8], - ) -> ( - Arc>, - HlsLeaseManifestSnapshot, - super::super::SegmentCacheKey, - u64, - ) { - let mut session = super::super::HlsSession::new( - super::super::HlsSessionKey::new(1, "terminal-timing-evidence"), - b"secret", - 1, - ); - let proxy_session_id = session.proxy_session_id.clone(); - let first_proxy_seq = 193; - let last_proxy_seq = 194; - let first_cache_key = super::super::SegmentCacheKey::new(proxy_session_id.clone(), first_proxy_seq, "ts"); - let last_cache_key = super::super::SegmentCacheKey::new(proxy_session_id.clone(), last_proxy_seq, "ts"); - cache.write_bytes_and_commit(&first_cache_key, first_bytes).await.expect("first segment commits"); - cache.write_bytes_and_commit(&last_cache_key, last_bytes).await.expect("last segment commits"); - for (proxy_seq, cache_key, content_length) in [ - (first_proxy_seq, first_cache_key, first_bytes.len()), - (last_proxy_seq, last_cache_key.clone(), last_bytes.len()), - ] { - session.segments.insert( - proxy_seq, - SegmentEntry { - origin_key: super::super::OriginSegmentKey { - origin_epoch: 7, - effective_host_id: 3, - host_local_sequence: proxy_seq, - host_local_index: 0, - }, - proxy_seq, - duration_ms: terminal_asset.duration_ms(), - proxy_file_ext: "ts".into(), - content_type: "video/mp2t".into(), - cache_key, - discontinuity_before: false, - program_date_time: None, - daterange_tags_before: Vec::new(), - origin_byte_range: None, - map_ref: None, - encryption: None, - origin_fetch_ref: None, - status: SegmentCacheStatus::Ready { - content_length: u64::try_from(content_length).unwrap_or(u64::MAX), - ready_at_ms: 2, - }, - last_rendered_at_ms: Some(2), - access: Arc::new(CacheAccessState::new()), - }, - ); - } - let mut base = manifest(); - base.first_proxy_seq = first_proxy_seq; - base.last_proxy_seq = last_proxy_seq; - base.visible_segments = Arc::from([first_proxy_seq, last_proxy_seq].map(|proxy_seq| HlsLeaseManifestSegment { - proxy_seq, - duration_ms: terminal_asset.duration_ms(), - uri: format!("/iptv/hls/shared/live/{}/lease/{proxy_seq}.ts", proxy_session_id.0).into(), - discontinuity_before: false, - map_ref_ready: true, - encryption: None, - })); - base.playlist_duration_ms = terminal_asset.duration_ms().saturating_mul(2); - base.last_visible_media_end_ms = base.playlist_duration_ms; - (Arc::new(tokio::sync::RwLock::new(session)), base, last_cache_key, last_proxy_seq) - } - - #[tokio::test] - async fn terminal_base_timestamp_profile_uses_exact_pinned_last_safe_segment() { - let temp_dir = tempfile::tempdir().expect("terminal timing cache tempdir"); - let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); - let terminal_asset = asset(); - let renderer = terminal_asset.renderer(); - let first_bytes = renderer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 0, - continuity_seed: 0, - logical_segment_index: 0, - }) - .expect("first cached segment"); - let last_offset = 9_000_000; - let last_bytes = renderer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: last_offset, - continuity_seed: 0, - logical_segment_index: 1, - }) - .expect("last cached segment"); - let expected_profile = TransportStreamBuffer::new(last_bytes.to_vec()) - .finite_hls_timestamp_profile() - .expect("last segment timestamp profile"); - let (session, base, last_cache_key, last_proxy_seq) = - terminal_timing_session(&cache, &terminal_asset, &first_bytes, &last_bytes).await; - - let evidence = prepare_terminal_base_evidence(&session, &cache, &base, 3).await; - let timing = evidence.timing().expect("exact last-base timing evidence"); - - assert_eq!(timing.base.proxy_seq, last_proxy_seq); - assert_eq!(timing.base.origin_epoch, 7); - assert_eq!(timing.base.cache_key, last_cache_key); - assert_eq!(timing.profile, expected_profile); - assert_ne!( - timing.profile, - TransportStreamBuffer::new(first_bytes.to_vec()) - .finite_hls_timestamp_profile() - .expect("first segment timestamp profile") - ); - } - - #[tokio::test] - async fn terminal_base_preserves_invalid_iv_and_missing_key_evidence() { - let invalid_iv = unresolved_aes_terminal_base_evidence("not-an-iv").await; - assert!(invalid_iv.track_base().is_some()); - assert_eq!(invalid_iv.track_resolution(), Some(&HlsTrackEvidenceResolution::InvalidIv)); - assert_eq!(invalid_iv.track_evidence_reason_code(), "invalid-iv"); - - let missing_key = unresolved_aes_terminal_base_evidence("0x00000000000000000000000000000384").await; - assert!(missing_key.track_base().is_some()); - assert_eq!(missing_key.track_resolution(), Some(&HlsTrackEvidenceResolution::KeyUnavailable)); - assert_eq!(missing_key.track_evidence_reason_code(), "key-unavailable"); - } - - #[test] - fn hls_terminal_response_manifest_keeps_live_tail_and_appends_finite_endlist_tail() { - let asset = asset(); - let plan = build_terminal_tail_plan(build_input(4, manifest(), Arc::clone(&asset))).expect("compatible tail"); - - let first = - terminal_tail_manifest_body(&plan, &ProxySessionId("session".into()), &HlsAccessLeaseId("lease".into())) - .expect("render tail"); - let second = - terminal_tail_manifest_body(&plan, &ProxySessionId("session".into()), &HlsAccessLeaseId("lease".into())) - .expect("render tail again"); - - assert_eq!(first, second); - assert_eq!(first.matches("#EXTM3U").count(), 1); - assert!(first.contains("/iptv/hls/shared/live/session/lease/193.ts")); - assert_eq!( - first.matches("/iptv/hls/shared/live/session/lease/terminal/4/").count(), - usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT) - ); - assert_eq!(first.matches("#EXT-X-DISCONTINUITY\n").count(), 1); - assert!(first.ends_with("#EXT-X-ENDLIST\n")); - assert!(first.contains("#EXT-X-TARGETDURATION:12\n")); - assert_eq!(plan.base_manifest.target_duration_ms, 12_000); - assert_eq!(plan.segment_duration_ms, asset.duration_ms()); - let terminal_media = first - .rsplit_once("#EXT-X-DISCONTINUITY\n") - .map(|(_, terminal_media)| terminal_media) - .expect("terminal discontinuity"); - assert_eq!( - terminal_media.matches(&format!("#EXTINF:{},", format_hls_duration_ms(asset.duration_ms()))).count(), - usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT) - ); - assert_eq!( - terminal_tail_manifest_body( - &plan, - &ProxySessionId("session".into()), - &HlsAccessLeaseId("other-lease".into()), - ), - Err(HlsTerminalTailRenderError::RouteBindingMismatch) - ); - } - - #[test] - fn terminal_tail_plan_serves_preanchored_bytes_without_request_time_rendering() { - let asset = asset(); - let renderer = asset.renderer(); - let first_input = build_input(4, manifest(), Arc::clone(&asset)); - let shared_bundle = Arc::clone(&first_input.anchored_bundle); - let mut second_input = build_input(5, manifest(), asset); - second_input.anchored_bundle = Arc::clone(&shared_bundle); - let plan = build_terminal_tail_plan(first_input).expect("compatible tail"); - let second_plan = build_terminal_tail_plan(second_input).expect("second compatible tail"); - let render_count_after_plan_commit = renderer.finite_hls_render_count(); - let finalize_count_after_plan_commit = renderer.finite_hls_finalize_count(); - assert!(Arc::ptr_eq(&plan.anchored_bundle, &second_plan.anchored_bundle)); - let first_path = HlsTerminalSegmentPath { generation: plan.generation, index: 0 }; - let second_path = HlsTerminalSegmentPath { generation: plan.generation, index: 1 }; - - let first = plan.segment_bytes(first_path).expect("first prepared segment exists"); - assert_eq!(plan.segment_content_length(first_path), u64::try_from(first.len()).ok()); - let first_again = plan.segment_bytes(first_path).expect("same prepared segment exists"); - assert_eq!(first, first_again); - assert_eq!(first.as_ptr(), first_again.as_ptr(), "Bytes clones share the prepared allocation"); - assert_ne!(plan.segment_bytes(second_path), Some(first)); - assert_eq!( - plan.segment_bytes(HlsTerminalSegmentPath { - generation: HlsTerminalTailGeneration(plan.generation.0.saturating_add(1)), - index: 0, - }), - None - ); - assert_eq!( - plan.segment_bytes(HlsTerminalSegmentPath { generation: plan.generation, index: plan.segment_count }), - None - ); - assert_eq!(renderer.finite_hls_render_count(), render_count_after_plan_commit); - assert_eq!(renderer.finite_hls_finalize_count(), finalize_count_after_plan_commit); - } - - #[test] - fn hls_terminal_response_path_parser_rejects_noncanonical_generation_and_file() { - assert_eq!( - HlsTerminalSegmentPath::parse("17", "2.ts"), - Some(HlsTerminalSegmentPath { generation: HlsTerminalTailGeneration(17), index: 2 }) - ); - for (generation, terminal_file) in [ - ("", "0.ts"), - ("01", "0.ts"), - ("+1", "0.ts"), - ("one", "0.ts"), - ("1", ""), - ("1", "00.ts"), - ("1", "+1.ts"), - ("1", "1.m4s"), - ("1", "65536.ts"), - ] { - assert_eq!(HlsTerminalSegmentPath::parse(generation, terminal_file), None); - } - } - - #[test] - fn hls_terminal_response_rejects_bundle_for_a_different_target_duration() { - let asset = asset(); - let mut input = build_input(4, manifest(), Arc::clone(&asset)); - input.anchored_bundle = HlsTerminalTailBuildInput::anchored_bundle_for_test(&asset, 13_000); - - assert_eq!(build_terminal_tail_plan(input), Err(HlsTerminalTailCompatibility::AssetRevisionMismatch)); - } - - #[test] - fn terminal_plan_rejects_unbound_and_mixed_lease_routes() { - let mut unbound = manifest(); - for segment in Arc::make_mut(&mut unbound.visible_segments) { - segment.uri = format!("/live/{}.ts", segment.proxy_seq).into(); - } - assert_eq!( - build_terminal_tail_plan(build_input(4, unbound, asset())), - Err(HlsTerminalTailCompatibility::InvalidLeaseRoute) - ); - - let mut mixed = manifest(); - Arc::make_mut(&mut mixed.visible_segments)[1].uri = - Arc::from("/iptv/hls/shared/live/session/other-lease/194.ts"); - assert_eq!( - build_terminal_tail_plan(build_input(4, mixed, asset())), - Err(HlsTerminalTailCompatibility::InvalidLeaseRoute) - ); - } - - #[test] - fn encrypted_base_announces_clear_key_transition() { - let mut base = manifest(); - let encryption = resettable_aes128_encryption(); - base.active_encryption = Some(encryption.clone()); - Arc::make_mut(&mut base.visible_segments)[1].encryption = Some(encryption); - let mut input = build_input(1, base, asset()); - input.base_key_bindings = Arc::from([aes128_key_binding()]); - let plan = build_terminal_tail_plan(input).expect("clear transition"); - let resource_file = TransientResourceFile::parse("aes-key.key").expect("terminal key route"); - let binding = plan - .terminal_key_binding(&ProxySessionId("session".into()), &HlsAccessLeaseId("lease".into()), &resource_file) - .expect("exact lease route resolves frozen key binding"); - assert_eq!(binding.bytes().as_ref(), b"0123456789abcdef"); - assert!(plan - .terminal_key_binding( - &ProxySessionId("session".into()), - &HlsAccessLeaseId("other-lease".into()), - &resource_file, - ) - .is_none()); - assert!(plan - .terminal_key_binding( - &ProxySessionId("session".into()), - &HlsAccessLeaseId("lease".into()), - &TransientResourceFile::parse("aes-key.bin").expect("different extension route"), - ) - .is_none()); - let rendered = - terminal_tail_manifest_body(&plan, &ProxySessionId("session".into()), &HlsAccessLeaseId("lease".into())) - .expect("render"); - - assert!( - !rendered.contains("/iptv/hls/shared/live/session/lease/193.ts"), - "only the segment covered by the active key is safe" - ); - let key = rendered.find("#EXT-X-KEY:METHOD=AES-128").expect("base key tag"); - let base_segment = rendered.find("/iptv/hls/shared/live/session/lease/194.ts").expect("encrypted base segment"); - let reset = rendered.find("#EXT-X-KEY:METHOD=NONE").expect("clear reset"); - assert!(key < base_segment && base_segment < reset); - assert!(rendered.contains( - "URI=\"/iptv/hls/shared/live/session/lease/r/aes-key.key\",IV=0x000000000000000000000000000000C2" - )); - assert!(rendered.contains("#EXT-X-VERSION:5\n")); - assert!(rendered.contains("#EXT-X-KEY:METHOD=NONE\n#EXT-X-DISCONTINUITY\n")); - } - - #[test] - fn method_none_boundary_keeps_only_homogeneous_clear_suffix() { - let mut base = manifest(); - Arc::make_mut(&mut base.visible_segments)[0].encryption = Some(resettable_aes128_encryption()); - - let plan = build_terminal_tail_plan(build_input(2, base, asset())).expect("clear suffix"); - let rendered = - terminal_tail_manifest_body(&plan, &ProxySessionId("session".into()), &HlsAccessLeaseId("lease".into())) - .expect("render"); - - assert_eq!(plan.protected_base_proxy_seqs.as_ref(), &[194]); - assert!(!rendered.contains("/iptv/hls/shared/live/session/lease/193.ts")); - assert!(!rendered.contains("#EXT-X-KEY:METHOD=AES-128")); - assert!(!rendered.contains("#EXT-X-KEY:METHOD=NONE")); - } - - #[test] - fn active_fmp4_map_rejects_mpeg_ts_splice() { - let mut base = manifest(); - base.container = HlsMediaContainer::FragmentedMp4; - base.active_map = Some(HlsMapSignature { fingerprint: [2; 32], container: HlsMediaContainer::FragmentedMp4 }); - let asset = asset(); - - assert_eq!( - compatibility(&base, &asset, Some(asset.track_signature())), - HlsTerminalTailCompatibility::ActiveMapRequiresCompatibleFallback - ); - } - - #[test] - fn mapless_non_ts_container_rejects_mpeg_ts_splice() { - let mut base = manifest(); - base.container = HlsMediaContainer::Unknown; - base.active_map = None; - let asset = asset(); - - assert_eq!( - compatibility(&base, &asset, Some(asset.track_signature())), - HlsTerminalTailCompatibility::ContainerMismatch - ); - } - - #[test] - fn transient_origin_backed_manifest_is_explicitly_incompatible() { - let asset = asset(); - let mut base = manifest(); - base.delivery_mode = HlsManifestDeliveryMode::TransientPassthrough; - - assert_eq!( - compatibility(&base, &asset, Some(asset.track_signature())), - HlsTerminalTailCompatibility::TransientPassthroughUnsupported - ); - } - - #[test] - fn asset_rounded_duration_must_not_exceed_target_duration() { - let asset = asset(); - let mut short_target = manifest(); - short_target.target_duration_ms = 9_000; - - assert_eq!( - compatibility(&short_target, &asset, Some(asset.track_signature())), - HlsTerminalTailCompatibility::TargetDurationExceeded { asset_ms: asset.duration_ms(), target_ms: 9_000 } - ); - } - - #[test] - fn terminal_plan_keeps_only_contiguous_ready_protectable_suffix() { - let asset = asset(); - let base = manifest(); - let mut input = build_input(1, base, Arc::clone(&asset)); - Arc::make_mut(&mut input.base_availability)[0].media_state = HlsTerminalBaseMediaState::NotReady; - - let plan = build_terminal_tail_plan(input).expect("safe suffix"); - - assert_eq!(plan.base_manifest.first_proxy_seq, 194); - assert_eq!(plan.base_manifest.last_proxy_seq, 194); - assert_eq!(plan.protected_base_proxy_seqs.as_ref(), &[194]); - assert_eq!(plan.base_manifest.visible_segments.len(), 1); - } - - #[test] - fn unsafe_last_advertised_segment_rejects_terminal_plan() { - let mut input = build_input(1, manifest(), asset()); - Arc::make_mut(&mut input.base_availability)[1].protection = HlsTerminalBaseProtection::Unavailable; - - assert_eq!(build_terminal_tail_plan(input), Err(HlsTerminalTailCompatibility::MissingSafeBase)); - } - - #[test] - fn unsafe_transport_evidence_cannot_commit_or_expose_any_runtime_custom_tail() { - let unsafe_reason = super::super::HlsTsSpliceIncompatibility::IncompletePes { - pid: 0x101, - packet_index: 27, - declared_bytes: Some(512), - observed_bytes: 384, - }; - for reason in [ - HlsRuntimeCustomTailReason::ChannelUnavailable, - HlsRuntimeCustomTailReason::LowPriorityPreempted, - HlsRuntimeCustomTailReason::UserConnectionsExhausted, - HlsRuntimeCustomTailReason::ProviderConnectionsExhausted, - HlsRuntimeCustomTailReason::UserAccountExpired, - HlsRuntimeCustomTailReason::SessionOrLeaseExpired, - ] { - let asset = asset(); - let mut input = build_input(1, manifest(), Arc::clone(&asset)); - input.expected_asset = - HlsRuntimeCustomTailAssetIdentity { reason, media: HlsTerminalAssetIdentity::from_asset(&asset) }; - input.base_splice_evidence = Some(HlsTsSpliceEvidence::Incompatible(unsafe_reason)); - - assert_eq!( - build_terminal_tail_plan(input), - Err(HlsTerminalTailCompatibility::SpliceTransportFailure(unsafe_reason)), - "reason {reason} must use the common exact splice gate" - ); - } - } - - #[test] - fn terminal_plan_rejects_missing_or_topologically_different_exact_evidence() { - let asset = asset(); - let mut missing = build_input(1, manifest(), Arc::clone(&asset)); - missing.terminal_splice_evidence = None; - assert_eq!(build_terminal_tail_plan(missing), Err(HlsTerminalTailCompatibility::MissingSpliceEvidence)); - - let mut different = build_input(2, manifest(), asset); - different.terminal_splice_evidence = Some(HlsTsSpliceEvidence::compatible_for_test( - HlsTsTrackSignature::from_stream_types(Arc::<[u8]>::from([0x1B])), - )); - assert_eq!(build_terminal_tail_plan(different), Err(HlsTerminalTailCompatibility::SpliceTopologyMismatch)); - } - - #[test] - fn terminal_splice_diagnostic_preserves_typed_boundary_fields() { - let compatibility = HlsTerminalTailCompatibility::SpliceTransportFailure( - super::super::HlsTsSpliceIncompatibility::ContinuityFailure { - pid: 0x102, - packet_index: 44, - expected: 9, - actual: 12, - }, - ); - - assert_eq!( - HlsTerminalSpliceDiagnostic::from_compatibility(compatibility), - Some(HlsTerminalSpliceDiagnostic { - result: "continuity-failure", - pid: Some(0x102), - packet_index: Some(44), - expected_cc: Some(9), - actual_cc: Some(12), - declared_pes_bytes: None, - observed_pes_bytes: None, - }) - ); - assert_eq!( - HlsTerminalSpliceDiagnostic::from_compatibility(HlsTerminalTailCompatibility::SpliceTopologyMismatch) - .map(|diagnostic| diagnostic.result), - Some("topology-mismatch") - ); - } - - #[test] - fn encrypted_base_without_ready_key_rejects_terminal_plan() { - let mut base = manifest(); - let encryption = resettable_aes128_encryption(); - base.active_encryption = Some(encryption.clone()); - Arc::make_mut(&mut base.visible_segments)[1].encryption = Some(encryption); - let mut input = build_input(1, base, asset()); - Arc::make_mut(&mut input.base_availability)[1].required_key_ready = false; - - assert_eq!(build_terminal_tail_plan(input), Err(HlsTerminalTailCompatibility::MissingSafeBase)); - } - - #[test] - fn duplicate_readiness_evidence_is_not_accepted_as_safe_base() { - let mut input = build_input(1, manifest(), asset()); - let duplicate = input.base_availability[1]; - let mut states = input.base_availability.to_vec(); - states.push(duplicate); - input.base_availability = Arc::from(states); - - assert_eq!(build_terminal_tail_plan(input), Err(HlsTerminalTailCompatibility::MissingSafeBase)); - } - - #[test] - fn unknown_or_different_track_layout_rejects_splice() { - let asset = asset(); - assert_eq!(compatibility(&manifest(), &asset, None), HlsTerminalTailCompatibility::MissingTrackSignature); - let different = HlsTsTrackSignature::from_stream_types(Arc::<[u8]>::from([0x1B])); - assert_eq!( - compatibility(&manifest(), &asset, Some(&different)), - HlsTerminalTailCompatibility::TrackLayoutMismatch - ); - } - - #[test] - fn unsupported_key_format_cannot_reset_to_clear_asset() { - let mut base = manifest(); - base.active_encryption = Some(Arc::new(HlsEncryptionSignature { - method: "SAMPLE-AES".into(), - key_uri: Some("/keys/drm.key".into()), - iv: None, - key_format: Some("com.example.drm".into()), - key_format_versions: None, - can_reset_to_clear: true, - })); - let asset = asset(); - - assert_eq!( - compatibility(&base, &asset, Some(asset.track_signature())), - HlsTerminalTailCompatibility::UnsupportedEncryptionTransition - ); - } - - #[test] - fn unsafe_key_uri_or_iv_cannot_be_rendered_into_terminal_manifest() { - let asset = asset(); - let mut unsafe_uri = manifest(); - let mut encryption = resettable_aes128_encryption(); - Arc::make_mut(&mut encryption).key_uri = Some("/key\"\n#EXT-X-ENDLIST".into()); - unsafe_uri.active_encryption = Some(encryption); - assert_eq!( - compatibility(&unsafe_uri, &asset, Some(asset.track_signature())), - HlsTerminalTailCompatibility::UnsupportedEncryptionTransition - ); - - let mut unsafe_iv = manifest(); - let mut encryption = resettable_aes128_encryption(); - Arc::make_mut(&mut encryption).iv = Some("not-hex".into()); - unsafe_iv.active_encryption = Some(encryption); - assert_eq!( - compatibility(&unsafe_iv, &asset, Some(asset.track_signature())), - HlsTerminalTailCompatibility::UnsupportedEncryptionTransition - ); - } - - #[test] - fn stale_asset_revision_cannot_build_terminal_plan() { - let asset = asset(); - let mut input = build_input(1, manifest(), asset); - input.expected_asset.media.revision = input.expected_asset.media.revision.saturating_add(1); - - assert_eq!(build_terminal_tail_plan(input), Err(HlsTerminalTailCompatibility::AssetRevisionMismatch)); - } -} +mod assets; +mod diagnostics; +mod evidence; +mod plan; +mod render; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use assets::{ + snapshot_terminal_media_asset, terminal_media_asset_identity, HlsEncryptionSignature, HlsMapSignature, + HlsMediaContainer, HlsTerminalAssetIdentity, +}; +#[cfg(test)] +use evidence::terminal_base_media_evidence; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use evidence::{ + pin_terminal_base_evidence, prepare_terminal_base_evidence, resolve_terminal_base_evidence, + HlsTerminalBaseMediaState, HlsTerminalBaseProtection, HlsTerminalBaseSegmentAvailability, + HlsTerminalBaseTimingEvidence, HlsTerminalBaseTrackIdentity, +}; +use evidence::{HlsTerminalBaseKeyEvidence, HlsTerminalTrackEncryption}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use plan::{ + build_terminal_tail_plan, evaluate_terminal_tail_compatibility, HlsLeasePlaybackMode, + HlsTerminalTailBoundaryEvidence, HlsTerminalTailBuildInput, HlsTerminalTailCompatibility, + HlsTerminalTailCompatibilityInput, +}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use render::{terminal_tail_manifest_body, HlsTerminalSegmentPath, HlsTerminalTailRenderError}; diff --git a/backend/hls/src/terminal_tail/assets.rs b/backend/hls/src/terminal_tail/assets.rs new file mode 100644 index 000000000..77768762e --- /dev/null +++ b/backend/hls/src/terminal_tail/assets.rs @@ -0,0 +1,229 @@ +use super::{ + HlsTerminalKeyBinding, HlsTerminalKeyMaterial, HlsTerminalMediaAsset, HlsTerminalTailCompatibility, + HlsTsTrackSignature, HlsValidatedTerminalMediaAsset, ProxySessionId, TransientObjectCacheKey, + TransientResourceFile, TransientResourceId, AES_128_BLOCK_BYTES, +}; +use bytes::Bytes; +use std::sync::Arc; +use tuliprox_mpegts::transport_stream_buffer::{HlsTsTimestampProfile, TransportStreamBuffer}; +use zeroize::Zeroizing; + +impl HlsTerminalKeyMaterial { + pub(super) fn from_slice(bytes: &[u8]) -> Option { + let bytes: [u8; AES_128_BLOCK_BYTES] = bytes.try_into().ok()?; + Some(Self { bytes: Zeroizing::new(bytes) }) + } + + pub(super) fn as_bytes(&self) -> &[u8; AES_128_BLOCK_BYTES] { &self.bytes } +} + +impl std::fmt::Debug for HlsTerminalKeyMaterial { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("HlsTerminalKeyMaterial()") + } +} + +impl HlsTerminalKeyBinding { + pub(super) fn new( + proxy_session_id: ProxySessionId, + resource_id: TransientResourceId, + route_extension: String, + source_cache_key: TransientObjectCacheKey, + content_type: String, + bytes: &[u8], + ) -> Option { + Some(Self { + proxy_session_id, + resource_id, + route_extension: Arc::from(route_extension), + source_cache_key, + content_type: Arc::from(content_type), + material: Arc::new(HlsTerminalKeyMaterial::from_slice(bytes)?), + }) + } + + pub(super) fn matches_resource(&self, resource_file: &TransientResourceFile) -> bool { + self.resource_id == resource_file.resource_id && self.route_extension.as_ref() == resource_file.extension + } + + pub(super) fn matches_route(&self, proxy_session_id: &ProxySessionId) -> bool { + self.proxy_session_id == *proxy_session_id + } + + pub fn content_type(&self) -> &str { &self.content_type } + + pub fn bytes(&self) -> Bytes { Bytes::copy_from_slice(self.material.as_bytes()) } + + #[cfg(any(test, feature = "test-support"))] + pub fn source_cache_key(&self) -> &TransientObjectCacheKey { &self.source_cache_key } + + #[cfg(any(test, feature = "test-support"))] + pub fn resource_id(&self) -> &TransientResourceId { &self.resource_id } +} + +impl std::fmt::Debug for HlsTerminalKeyBinding { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("HlsTerminalKeyBinding") + .field("proxy_session_id", &"") + .field("resource_id", &self.resource_id) + .field("route_extension", &self.route_extension) + .field("source_cache_key", &self.source_cache_key) + .field("content_type", &self.content_type) + .field("material", &"") + .finish() + } +} + +impl PartialEq for HlsTerminalKeyBinding { + fn eq(&self, other: &Self) -> bool { + self.proxy_session_id == other.proxy_session_id + && self.resource_id == other.resource_id + && self.route_extension == other.route_extension + && self.source_cache_key == other.source_cache_key + && self.content_type == other.content_type + && self.material.as_bytes() == other.material.as_bytes() + } +} + +impl Eq for HlsTerminalKeyBinding {} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsMediaContainer { + MpegTs, + FragmentedMp4, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsMapSignature { + pub fingerprint: [u8; 32], + pub container: HlsMediaContainer, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsEncryptionSignature { + pub method: String, + pub key_uri: Option, + pub iv: Option, + pub key_format: Option, + pub key_format_versions: Option, + pub can_reset_to_clear: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ParsedEncryptionMethod<'a> { + None, + Aes128, + Unsupported(&'a str), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum HlsTerminalBaseScope { + SafeSuffix, + LastSafeSegment, +} + +pub(super) fn parse_encryption_method(method: &str) -> ParsedEncryptionMethod<'_> { + if method.eq_ignore_ascii_case("NONE") { + ParsedEncryptionMethod::None + } else if method.eq_ignore_ascii_case("AES-128") { + ParsedEncryptionMethod::Aes128 + } else { + ParsedEncryptionMethod::Unsupported(method) + } +} + +impl PartialEq for HlsTerminalMediaAsset { + fn eq(&self, other: &Self) -> bool { + self.identity() == other.identity() + && self.container() == other.container() + && self.track_signature() == other.track_signature() + && self.duration_ms() == other.duration_ms() + && self.duration_ticks_90khz() == other.duration_ticks_90khz() + && self.timestamp_profile() == other.timestamp_profile() + && self.content_type() == other.content_type() + } +} + +impl Eq for HlsTerminalMediaAsset {} + +impl HlsTerminalMediaAsset { + pub(super) fn identity(&self) -> HlsTerminalAssetIdentity { + HlsTerminalAssetIdentity { revision: self.validated.revision, fingerprint: self.validated.fingerprint } + } + + pub fn container(&self) -> HlsMediaContainer { self.validated.container } + + pub fn track_signature(&self) -> &HlsTsTrackSignature { &self.validated.track_signature } + + pub fn duration_ms(&self) -> u64 { self.validated.duration_ms } + + pub fn duration_ticks_90khz(&self) -> u64 { self.validated.duration_ticks_90khz } + + pub fn timestamp_profile(&self) -> Option { self.validated.timestamp_profile } + + pub fn content_type(&self) -> &'static str { self.validated.content_type } + + pub(crate) fn renderer(&self) -> Arc { Arc::clone(&self.validated.renderer) } +} + +/// Captures and validates one immutable revision of the configured terminal TS asset. +pub fn snapshot_terminal_media_asset( + buffer: &TransportStreamBuffer, +) -> Result, HlsTerminalTailCompatibility> { + let (Some(duration_ms), Some(duration_ticks_90khz), Some(track_signature)) = + (buffer.duration_ms(), buffer.duration_ticks_90khz(), buffer.finite_hls_track_signature()) + else { + return Err(HlsTerminalTailCompatibility::InvalidAsset); + }; + let bytes = buffer.clone_bytes(); + if bytes.is_empty() { + return Err(HlsTerminalTailCompatibility::InvalidAsset); + } + let fingerprint = buffer.finite_hls_asset_fingerprint(); + let identity = terminal_asset_identity_from_fingerprint(fingerprint); + Ok(Arc::new(HlsTerminalMediaAsset { + validated: Arc::new(HlsValidatedTerminalMediaAsset { + revision: identity.revision, + fingerprint: identity.fingerprint, + container: HlsMediaContainer::MpegTs, + track_signature, + duration_ms, + duration_ticks_90khz, + timestamp_profile: buffer.finite_hls_timestamp_profile(), + content_type: "video/mp2t", + renderer: Arc::new(buffer.clone()), + }), + })) +} + +/// Reads only cached validation metadata from the configured buffer. This is +/// suitable for the final commit CAS and never parses, hashes, or clones media. +pub fn terminal_media_asset_identity(buffer: &TransportStreamBuffer) -> Option { + if buffer.as_bytes().is_empty() + || buffer.duration_ms().is_none() + || buffer.duration_ticks_90khz().is_none() + || !buffer.has_finite_hls_track_signature() + { + return None; + } + Some(terminal_asset_identity_from_fingerprint(buffer.finite_hls_asset_fingerprint())) +} + +fn terminal_asset_identity_from_fingerprint(fingerprint: [u8; 32]) -> HlsTerminalAssetIdentity { + let mut revision_bytes = [0_u8; std::mem::size_of::()]; + let revision_len = revision_bytes.len(); + revision_bytes.copy_from_slice(&fingerprint[..revision_len]); + HlsTerminalAssetIdentity { revision: u64::from_be_bytes(revision_bytes), fingerprint } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct HlsTerminalAssetIdentity { + pub revision: u64, + pub fingerprint: [u8; 32], +} + +impl HlsTerminalAssetIdentity { + pub fn from_asset(asset: &HlsTerminalMediaAsset) -> Self { asset.identity() } +} diff --git a/backend/hls/src/terminal_tail/diagnostics.rs b/backend/hls/src/terminal_tail/diagnostics.rs new file mode 100644 index 000000000..be75fabe7 --- /dev/null +++ b/backend/hls/src/terminal_tail/diagnostics.rs @@ -0,0 +1,101 @@ +use super::{HlsTerminalSpliceDiagnostic, HlsTerminalTailCompatibility}; +use log::debug; + +impl HlsTerminalSpliceDiagnostic { + pub(super) fn from_compatibility(compatibility: HlsTerminalTailCompatibility) -> Option { + let empty = |result| Self { + result, + pid: None, + packet_index: None, + expected_cc: None, + actual_cc: None, + declared_pes_bytes: None, + observed_pes_bytes: None, + }; + match compatibility { + HlsTerminalTailCompatibility::Compatible => Some(empty("compatible")), + HlsTerminalTailCompatibility::MissingTrackSignature + | HlsTerminalTailCompatibility::TrackLayoutMismatch + | HlsTerminalTailCompatibility::MissingSpliceEvidence + | HlsTerminalTailCompatibility::SpliceTopologyMismatch => Some(empty("topology-mismatch")), + HlsTerminalTailCompatibility::SpliceTransportFailure(reason) => { + let mut diagnostic = empty(reason.result_code()); + match reason { + super::super::HlsTsSpliceIncompatibility::InvalidPacket { packet_index } + | super::super::HlsTsSpliceIncompatibility::TransportError { packet_index, .. } => { + diagnostic.packet_index = Some(packet_index); + if let super::super::HlsTsSpliceIncompatibility::TransportError { pid, .. } = reason { + diagnostic.pid = Some(pid); + } + } + super::super::HlsTsSpliceIncompatibility::ContinuityFailure { + pid, + packet_index, + expected, + actual, + } => { + diagnostic.pid = Some(pid); + diagnostic.packet_index = Some(packet_index); + diagnostic.expected_cc = Some(expected); + diagnostic.actual_cc = Some(actual); + } + super::super::HlsTsSpliceIncompatibility::IncompletePes { + pid, + packet_index, + declared_bytes, + observed_bytes, + } => { + diagnostic.pid = Some(pid); + diagnostic.packet_index = Some(packet_index); + diagnostic.declared_pes_bytes = declared_bytes; + diagnostic.observed_pes_bytes = Some(observed_bytes); + } + super::super::HlsTsSpliceIncompatibility::InvalidPes { pid, packet_index } => { + diagnostic.pid = Some(pid); + diagnostic.packet_index = Some(packet_index); + } + super::super::HlsTsSpliceIncompatibility::InspectionBudgetExhausted + | super::super::HlsTsSpliceIncompatibility::TopologyUnavailable => {} + } + Some(diagnostic) + } + HlsTerminalTailCompatibility::MissingAsset + | HlsTerminalTailCompatibility::TerminalMediaNotReady + | HlsTerminalTailCompatibility::InvalidAsset + | HlsTerminalTailCompatibility::AssetRevisionMismatch + | HlsTerminalTailCompatibility::MissingSafeBase + | HlsTerminalTailCompatibility::TargetDurationExceeded { .. } + | HlsTerminalTailCompatibility::ActiveMapRequiresCompatibleFallback + | HlsTerminalTailCompatibility::UnsupportedEncryptionTransition + | HlsTerminalTailCompatibility::ContainerMismatch + | HlsTerminalTailCompatibility::MissingTimestampAnchor + | HlsTerminalTailCompatibility::InvalidTimestampTransition + | HlsTerminalTailCompatibility::TransientPassthroughUnsupported + | HlsTerminalTailCompatibility::ProtectionCapacityExceeded + | HlsTerminalTailCompatibility::InvalidLeaseRoute + | HlsTerminalTailCompatibility::ManifestRenderFailed => None, + } + } +} + +fn format_optional_diagnostic(value: Option) -> String { + value.map_or_else(|| "none".to_string(), |value| value.to_string()) +} + +pub(super) fn log_terminal_splice_compatibility(base_proxy_seq: u64, compatibility: HlsTerminalTailCompatibility) { + let Some(diagnostic) = HlsTerminalSpliceDiagnostic::from_compatibility(compatibility) else { + return; + }; + debug!( + "HLS terminal TS splice eligibility: base_proxy_seq={} result={} pid={} packet_index={} \ + expected_cc={} actual_cc={} declared_pes_bytes={} observed_pes_bytes={}", + base_proxy_seq, + diagnostic.result, + format_optional_diagnostic(diagnostic.pid), + format_optional_diagnostic(diagnostic.packet_index), + format_optional_diagnostic(diagnostic.expected_cc), + format_optional_diagnostic(diagnostic.actual_cc), + format_optional_diagnostic(diagnostic.declared_pes_bytes), + format_optional_diagnostic(diagnostic.observed_pes_bytes), + ); +} diff --git a/backend/hls/src/terminal_tail/evidence.rs b/backend/hls/src/terminal_tail/evidence.rs new file mode 100644 index 000000000..dc736b2d6 --- /dev/null +++ b/backend/hls/src/terminal_tail/evidence.rs @@ -0,0 +1,521 @@ +use super::{ + assets::{parse_encryption_method, ParsedEncryptionMethod}, + CacheAccessState, HlsEncryptionSignature, HlsLeaseManifestSnapshot, HlsResolvedTerminalBaseEvidence, + HlsSegmentCache, HlsSessionHandle, HlsTerminalBaseEvidence, HlsTerminalBaseEvidencePreparation, + HlsTerminalBaseMediaEvidence, HlsTerminalBaseReadProtection, HlsTerminalBaseSegmentProbe, + HlsTerminalCommitMediaGuard, HlsTerminalKeyBinding, HlsTerminalKeyMaterial, HlsTerminalMediaProbe, + HlsTrackEvidenceResolution, HlsTsProbeBudget, HlsTsProbeProtection, HlsTsSpliceEvidence, HlsTsTrackSignature, + ProxySessionId, SegmentCacheStatus, SegmentEntry, TransientObjectCacheKey, TransientPassthroughState, + TransientResourceFile, TransientResourceId, TransientResourceKind, AES_128_BLOCK_BYTES, +}; +use log::debug; +use std::{ + collections::{hash_map::Entry, HashMap, HashSet}, + path::Path, + sync::Arc, +}; +use tokio::io::AsyncReadExt; +use tuliprox_mpegts::transport_stream_buffer::HlsTsTimestampProfile; +use zeroize::Zeroizing; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsTerminalBaseMediaState { + Ready, + NotReady, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsTerminalBaseProtection { + Protectable, + Unavailable, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HlsTerminalBaseSegmentAvailability { + pub proxy_seq: u64, + pub media_state: HlsTerminalBaseMediaState, + pub required_map_ready: bool, + pub required_key_ready: bool, + pub protection: HlsTerminalBaseProtection, +} + +impl HlsTerminalBaseEvidence { + pub fn availability(&self) -> Arc<[HlsTerminalBaseSegmentAvailability]> { Arc::clone(&self.availability) } + + pub fn track_signature(&self) -> Option { + self.track_resolution.as_ref().and_then(HlsTrackEvidenceResolution::signature).cloned() + } + + pub fn track_resolution(&self) -> Option<&HlsTrackEvidenceResolution> { self.track_resolution.as_ref() } + + pub fn splice_evidence(&self) -> Option<&HlsTsSpliceEvidence> { self.splice_evidence.as_ref() } + + pub fn track_evidence_reason_code(&self) -> &'static str { + self.track_resolution.as_ref().map_or("base-not-ready", HlsTrackEvidenceResolution::reason_code) + } + + pub fn track_base(&self) -> Option<&HlsTerminalBaseTrackIdentity> { self.track_base.as_ref() } + + pub fn timing(&self) -> Option<&HlsTerminalBaseTimingEvidence> { self.timing.as_ref() } + + pub fn key_bindings(&self) -> Arc<[HlsTerminalKeyBinding]> { Arc::clone(&self.key_bindings) } + + /// Makes the intended guard lifetime explicit at the endpoint commit boundary. + #[cfg(any(test, feature = "test-support"))] + pub fn release(self) { drop(self.read_protection) } + + /// Transfers the READY-object reader pins to an autonomous terminal commit. + /// + /// The returned guard must stay alive until the lease/session transaction + /// either installs durable terminal-tail protection or reaches a terminal + /// rejection. This closes the request-cancellation and `LockBusy` gap. + pub fn into_commit_guard(self) -> HlsTerminalCommitMediaGuard { + HlsTerminalCommitMediaGuard { read_protection: self.read_protection } + } +} + +#[cfg(any(test, feature = "test-support"))] +impl HlsTerminalCommitMediaGuard { + pub fn empty_for_test() -> Self { Self { read_protection: HlsTerminalBaseReadProtection { accesses: Vec::new() } } } +} + +impl std::fmt::Debug for HlsTerminalCommitMediaGuard { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("HlsTerminalCommitMediaGuard") + .field("pinned_objects", &self.read_protection.accesses.len()) + .finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsTerminalBaseTrackIdentity { + pub proxy_seq: u64, + pub origin_epoch: u64, + pub cache_key: super::super::SegmentCacheKey, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct HlsTerminalBaseTimingEvidence { + pub base: HlsTerminalBaseTrackIdentity, + pub profile: HlsTsTimestampProfile, +} + +impl Drop for HlsTerminalBaseReadProtection { + fn drop(&mut self) { + for access in &self.accesses { + access.reader_finished(); + } + } +} + +/// Reads cache metadata and the last READY base segment without holding a session lock across I/O. +pub async fn prepare_terminal_base_evidence( + session: &HlsSessionHandle, + segment_cache: &HlsSegmentCache, + manifest: &HlsLeaseManifestSnapshot, + now_ms: u64, +) -> HlsTerminalBaseEvidence { + let preparation = { + let session = session.read().await; + pin_terminal_base_evidence(&session, manifest, now_ms) + }; + resolve_terminal_base_evidence(segment_cache, manifest, preparation).await +} + +/// Pins the exact READY media and key objects represented by one frozen lease +/// manifest. Callers can perform this while holding the same session read lock +/// used to select the lease, closing the selection-to-GC race without I/O. +pub fn pin_terminal_base_evidence( + session: &super::super::HlsSession, + manifest: &HlsLeaseManifestSnapshot, + now_ms: u64, +) -> HlsTerminalBaseEvidencePreparation { + snapshot_terminal_base_probes(session, manifest, now_ms) +} + +/// Resolves already pinned base objects and inspects the exact terminal track +/// basis. No session lock is held while cache metadata and bytes are read. +pub async fn resolve_terminal_base_evidence( + segment_cache: &HlsSegmentCache, + manifest: &HlsLeaseManifestSnapshot, + preparation: HlsTerminalBaseEvidencePreparation, +) -> HlsTerminalBaseEvidence { + let resolved = resolve_terminal_base_probes(segment_cache, manifest, preparation.probes).await; + let media_evidence = match resolved.last_media_probe { + Some(probe) => Some(terminal_base_media_evidence(probe).await), + None => None, + }; + let track_resolution = match media_evidence.as_ref() { + Some(evidence) => Some(evidence.track_resolution.clone()), + None => resolved.last_track_preprobe_resolution, + }; + let splice_evidence = media_evidence.as_ref().map(|evidence| evidence.splice_evidence.clone()); + if track_resolution.as_ref().and_then(HlsTrackEvidenceResolution::signature).is_none() { + debug!( + "HLS terminal base track evidence unavailable: reason={}", + track_resolution.as_ref().map_or("base-not-ready", HlsTrackEvidenceResolution::reason_code) + ); + } + HlsTerminalBaseEvidence { + availability: resolved.availability.into(), + track_resolution, + splice_evidence, + timing: media_evidence + .and_then(|evidence| evidence.timestamp_profile) + .zip(resolved.track_base.clone()) + .map(|(profile, base)| HlsTerminalBaseTimingEvidence { base, profile }), + track_base: resolved.track_base, + key_bindings: resolved.key_bindings.into(), + read_protection: preparation.read_protection, + } +} + +fn snapshot_terminal_base_probes( + session: &super::super::HlsSession, + manifest: &HlsLeaseManifestSnapshot, + now_ms: u64, +) -> HlsTerminalBaseEvidencePreparation { + let mut accesses = Vec::with_capacity(manifest.visible_segments.len()); + let mut pinned_key_objects = HashSet::new(); + let probes = manifest + .visible_segments + .iter() + .map(|segment| { + let entry = session.segments.get(&segment.proxy_seq); + let media_ready = entry.is_some_and(|entry| matches!(&entry.status, SegmentCacheStatus::Ready { .. })); + let required_map_ready = entry.is_some_and(|entry| { + entry.map_ref.is_none_or(|map_id| { + session + .maps + .get(&map_id) + .is_some_and(|map| matches!(&map.status, super::super::map::MapCacheStatus::Ready { .. })) + }) + }); + let key_evidence = terminal_base_key_evidence(session, entry, segment.encryption.as_deref()); + if let HlsTerminalBaseKeyEvidence::Aes128 { cache_key, object_access, resource_access, .. } = &key_evidence + { + if pinned_key_objects.insert(cache_key.clone()) { + object_access.reader_started(now_ms); + resource_access.reader_started(now_ms); + accesses.push(Arc::clone(object_access)); + accesses.push(Arc::clone(resource_access)); + } + } + let cache_key = media_ready.then(|| entry.map(|entry| entry.cache_key.clone())).flatten(); + if let Some(access) = media_ready.then(|| entry.map(|entry| Arc::clone(&entry.access))).flatten() { + // GC takes the session write lock before selecting removals, so acquiring + // the pin under this read lock cannot race behind an already queued removal. + access.reader_started(now_ms); + accesses.push(access); + } + HlsTerminalBaseSegmentProbe { + proxy_seq: segment.proxy_seq, + duration_ms: segment.duration_ms, + media_ready, + required_map_ready, + cache_key, + origin_epoch: media_ready.then(|| entry.map(|entry| entry.origin_key.origin_epoch)).flatten(), + key_evidence, + } + }) + .collect(); + HlsTerminalBaseEvidencePreparation { probes, read_protection: HlsTerminalBaseReadProtection { accesses } } +} + +async fn resolve_terminal_base_probes( + segment_cache: &HlsSegmentCache, + manifest: &HlsLeaseManifestSnapshot, + probes: Vec, +) -> HlsResolvedTerminalBaseEvidence { + let mut availability = Vec::with_capacity(probes.len()); + let mut key_bindings = Vec::new(); + let mut key_binding_cache: HashMap> = HashMap::new(); + let mut last_media_probe = None; + let mut last_track_preprobe_resolution = None; + let mut track_base = None; + for probe in probes { + let metadata = if let Some(cache_key) = probe.cache_key.as_ref() { + segment_cache.metadata(cache_key).await.ok().flatten() + } else { + None + }; + let protectable = metadata.is_some(); + let (required_key_ready, track_encryption, key_failure) = match probe.key_evidence { + HlsTerminalBaseKeyEvidence::Clear => (true, Some(HlsTerminalTrackEncryption::Clear), None), + HlsTerminalBaseKeyEvidence::Aes128 { + proxy_session_id, + cache_key, + resource_id, + route_extension, + content_type, + iv, + .. + } => { + let binding = match key_binding_cache.entry(cache_key.clone()) { + Entry::Occupied(entry) => entry.get().clone(), + Entry::Vacant(entry) => { + let key_metadata = segment_cache.metadata(&cache_key).await.ok().flatten(); + let binding = if let Some(metadata) = + key_metadata.filter(|metadata| metadata.size == AES_128_BLOCK_BYTES as u64) + { + let bytes = read_bounded_file(&metadata.path, AES_128_BLOCK_BYTES as u64).await; + bytes.as_deref().and_then(|bytes| { + HlsTerminalKeyBinding::new( + proxy_session_id, + resource_id, + route_extension, + cache_key, + content_type, + bytes, + ) + }) + } else { + None + }; + if let Some(binding) = &binding { + key_bindings.push(binding.clone()); + } + entry.insert(binding).clone() + } + }; + let ready = binding.is_some(); + let encryption = binding.as_ref().map(|binding| HlsTerminalTrackEncryption::Aes128 { + key_material: Arc::clone(&binding.material), + iv, + }); + let failure = (!ready).then_some(HlsTrackEvidenceResolution::KeyUnavailable); + (ready, encryption, failure) + } + HlsTerminalBaseKeyEvidence::Unavailable { resolution } => (false, None, Some(resolution)), + }; + if probe.proxy_seq == manifest.last_proxy_seq && probe.media_ready && probe.required_map_ready { + if let (Some(cache_key), Some(origin_epoch)) = (probe.cache_key.as_ref(), probe.origin_epoch) { + track_base = Some(HlsTerminalBaseTrackIdentity { + proxy_seq: probe.proxy_seq, + origin_epoch, + cache_key: cache_key.clone(), + }); + if required_key_ready { + if let (Some(metadata), Some(encryption)) = (metadata.as_ref(), track_encryption) { + last_media_probe = Some(HlsTerminalMediaProbe { + segment_path: metadata.path.clone(), + source_size: metadata.size, + expected_duration_ticks_90khz: probe.duration_ms.saturating_mul(90), + encryption, + }); + } + } else { + last_track_preprobe_resolution = key_failure; + } + } + } + availability.push(HlsTerminalBaseSegmentAvailability { + proxy_seq: probe.proxy_seq, + media_state: terminal_base_media_state(probe.media_ready, protectable), + required_map_ready: probe.required_map_ready, + required_key_ready, + protection: terminal_base_protection(protectable), + }); + } + HlsResolvedTerminalBaseEvidence { + availability, + key_bindings, + last_media_probe, + last_track_preprobe_resolution, + track_base, + } +} + +const fn terminal_base_media_state(media_ready: bool, protectable: bool) -> HlsTerminalBaseMediaState { + if media_ready && protectable { + HlsTerminalBaseMediaState::Ready + } else { + HlsTerminalBaseMediaState::NotReady + } +} + +const fn terminal_base_protection(protectable: bool) -> HlsTerminalBaseProtection { + if protectable { + HlsTerminalBaseProtection::Protectable + } else { + HlsTerminalBaseProtection::Unavailable + } +} + +pub(super) enum HlsTerminalBaseKeyEvidence { + Clear, + Aes128 { + proxy_session_id: ProxySessionId, + cache_key: TransientObjectCacheKey, + resource_id: TransientResourceId, + route_extension: String, + content_type: String, + iv: [u8; AES_128_BLOCK_BYTES], + object_access: Arc, + resource_access: Arc, + }, + Unavailable { + resolution: HlsTrackEvidenceResolution, + }, +} + +impl HlsTerminalBaseKeyEvidence { + pub(super) fn unavailable(resolution: HlsTrackEvidenceResolution) -> Self { Self::Unavailable { resolution } } +} + +fn terminal_base_key_evidence( + session: &super::super::HlsSession, + timeline_entry: Option<&SegmentEntry>, + encryption: Option<&HlsEncryptionSignature>, +) -> HlsTerminalBaseKeyEvidence { + let Some(encryption) = encryption else { + return if timeline_entry.is_none_or(|entry| entry.encryption.is_none()) { + HlsTerminalBaseKeyEvidence::Clear + } else { + HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::IncompleteEvidence) + }; + }; + if parse_encryption_method(&encryption.method) != ParsedEncryptionMethod::Aes128 { + return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::UnsupportedProtection( + super::super::HlsTsProtectionReason::UnsupportedEncryption, + )); + } + let Some(resource_file) = encryption + .key_uri + .as_deref() + .and_then(|uri| uri.split(['?', '#']).next()) + .and_then(|path| path.rsplit('/').next()) + .and_then(TransientResourceFile::parse) + else { + return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); + }; + let Some(timeline_entry) = timeline_entry else { + return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); + }; + let Some(timeline_encryption) = timeline_entry.encryption.as_ref() else { + return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); + }; + if timeline_encryption.resource_id != resource_file.resource_id + || timeline_encryption.resource_extension != resource_file.extension + || timeline_encryption.iv != encryption.iv + || timeline_encryption.key_format != encryption.key_format + || timeline_encryption.key_format_versions != encryption.key_format_versions + { + return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); + } + let iv = match super::super::hls_aes128_cbc_iv( + encryption.iv.as_deref(), + timeline_entry.origin_key.host_local_sequence, + ) { + Ok(iv) => iv, + Err(error) => { + return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::from(Err(error))); + } + }; + let Some(resource) = session.transient.resources.get(&resource_file.resource_id) else { + return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); + }; + if resource.kind != TransientResourceKind::Key + || resource.file_ext_hint.as_deref() != Some(resource_file.extension.as_str()) + { + return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); + } + let key = TransientPassthroughState::transient_object_key( + &session.proxy_session_id, + &resource_file.resource_id, + resource_file.extension.clone(), + ); + let Some(entry) = session.transient.object_cache.get(&key) else { + return HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable); + }; + // TTL controls admission to new live manifests, not whether exact READY key + // bytes referenced by this frozen lease snapshot may be pinned for terminal + // evidence. The reader pins acquired below close the GC race without any + // post-terminal origin fetch. + let ready = matches!(entry.status, super::super::TransientObjectCacheStatus::Ready { .. }); + if ready { + HlsTerminalBaseKeyEvidence::Aes128 { + proxy_session_id: session.proxy_session_id.clone(), + cache_key: entry.key.clone(), + resource_id: resource_file.resource_id, + route_extension: resource_file.extension, + content_type: entry.content_type.clone(), + iv, + object_access: Arc::clone(&entry.access), + resource_access: Arc::clone(&resource.access), + } + } else { + HlsTerminalBaseKeyEvidence::unavailable(HlsTrackEvidenceResolution::KeyUnavailable) + } +} + +pub(super) enum HlsTerminalTrackEncryption { + Clear, + Aes128 { key_material: Arc, iv: [u8; AES_128_BLOCK_BYTES] }, +} + +pub(super) async fn terminal_base_media_evidence(probe: HlsTerminalMediaProbe) -> HlsTerminalBaseMediaEvidence { + let file = match tokio::fs::File::open(&probe.segment_path).await { + Ok(file) => file, + Err(error) => { + return HlsTerminalBaseMediaEvidence { + track_resolution: HlsTrackEvidenceResolution::Io(error.kind()), + timestamp_profile: None, + splice_evidence: HlsTsSpliceEvidence::Incompatible( + super::super::HlsTsSpliceIncompatibility::TopologyUnavailable, + ), + }; + } + }; + let budget = HlsTsProbeBudget { + max_bytes: probe.source_size.saturating_add(1), + max_packets: probe.source_size.saturating_add(187).saturating_div(188).saturating_add(1), + ..HlsTsProbeBudget::default() + }; + let outcome = match probe.encryption { + HlsTerminalTrackEncryption::Clear => { + super::super::inspect_mpeg_ts_media_evidence_async( + file, + HlsTsProbeProtection::Clear, + budget, + probe.expected_duration_ticks_90khz, + ) + .await + } + HlsTerminalTrackEncryption::Aes128 { key_material, iv } => { + super::super::inspect_mpeg_ts_media_evidence_async( + file, + HlsTsProbeProtection::Aes128Cbc { key: key_material.as_bytes(), iv }, + budget, + probe.expected_duration_ticks_90khz, + ) + .await + } + }; + match outcome { + Ok(evidence) => { + let super::super::HlsTsMediaEvidence { track_outcome, timestamp_profile, splice_evidence } = evidence; + HlsTerminalBaseMediaEvidence { + track_resolution: HlsTrackEvidenceResolution::from(Ok(track_outcome)), + timestamp_profile, + splice_evidence, + } + } + Err(error) => HlsTerminalBaseMediaEvidence { + track_resolution: HlsTrackEvidenceResolution::from(Err(error)), + timestamp_profile: None, + splice_evidence: HlsTsSpliceEvidence::Incompatible( + super::super::HlsTsSpliceIncompatibility::TopologyUnavailable, + ), + }, + } +} + +async fn read_bounded_file(path: &Path, max_bytes: u64) -> Option>> { + let file = tokio::fs::File::open(path).await.ok()?; + let mut limited = file.take(max_bytes.saturating_add(1)); + let mut bytes = Zeroizing::new(Vec::new()); + limited.read_to_end(&mut bytes).await.ok()?; + (u64::try_from(bytes.len()).unwrap_or(u64::MAX) <= max_bytes).then_some(bytes) +} diff --git a/backend/hls/src/terminal_tail/plan.rs b/backend/hls/src/terminal_tail/plan.rs new file mode 100644 index 000000000..33ff5bd77 --- /dev/null +++ b/backend/hls/src/terminal_tail/plan.rs @@ -0,0 +1,435 @@ +#[cfg(any(test, feature = "test-support"))] +use super::HlsTerminalBaseTrackIdentity; +use super::{ + assets::{parse_encryption_method, HlsTerminalBaseScope, ParsedEncryptionMethod}, + diagnostics::log_terminal_splice_compatibility, + render::{ + render_terminal_tail_manifest_body, safe_terminal_base, terminal_key_resource_file, + terminal_tail_route_binding, valid_iv, valid_key_format_versions, valid_quoted_attribute, + }, + terminal_asset_fits_target_duration, HlsAccessLeaseId, HlsAnchoredTerminalBundle, HlsAnchoredTerminalSegment, + HlsEncryptionSignature, HlsLeaseManifestSnapshot, HlsManifestDeliveryMode, HlsMediaContainer, + HlsPreparedTerminalBundleKey, HlsRuntimeCustomTailAssetIdentity, HlsTerminalAssetIdentity, + HlsTerminalBaseSegmentAvailability, HlsTerminalBaseTimingEvidence, HlsTerminalKeyBinding, HlsTerminalMediaAsset, + HlsTerminalSegmentPath, HlsTerminalTailGeneration, HlsTerminalTailManifestRenderInput, HlsTerminalTailPlan, + HlsTerminalTailRouteBinding, HlsTsSpliceEvidence, HlsTsTrackSignature, ProxySessionId, TransientResourceFile, + HLS_TERMINAL_TAIL_SEGMENT_COUNT, +}; +use bytes::Bytes; +use std::sync::Arc; + +#[derive(Clone)] +pub struct HlsTerminalTailBuildInput { + pub generation: HlsTerminalTailGeneration, + pub created_at_ms: u64, + pub base_manifest: HlsLeaseManifestSnapshot, + pub base_availability: Arc<[HlsTerminalBaseSegmentAvailability]>, + pub base_track_signature: Option, + pub base_splice_evidence: Option, + pub terminal_splice_evidence: Option, + pub base_timing: Option, + pub base_key_bindings: Arc<[HlsTerminalKeyBinding]>, + pub expected_asset: HlsRuntimeCustomTailAssetIdentity, + pub asset: Arc, + pub anchored_bundle: Arc, +} + +#[cfg(any(test, feature = "test-support"))] +impl HlsTerminalTailBuildInput { + /// # Panics + /// + /// Panics if the asset cannot produce a bundle, timing profile or splice + /// anchor. Test-support only: a fixture that cannot be built is a bug in + /// the test, not a runtime condition. + pub fn anchored_bundle_for_test( + asset: &HlsTerminalMediaAsset, + target_duration_ms: u64, + ) -> Arc { + let key = HlsPreparedTerminalBundleKey { + asset: HlsTerminalAssetIdentity::from_asset(asset), + target_duration_ms, + segment_count: HLS_TERMINAL_TAIL_SEGMENT_COUNT, + }; + let prepared = super::super::prepared_terminal_bundle::build_prepared_terminal_bundle(asset, key) + .expect("relative terminal test bundle"); + let profile = asset.timestamp_profile().expect("terminal test asset timing profile"); + let anchor = tuliprox_mpegts::transport_stream_buffer::HlsTsSpliceAnchor::between(profile, profile) + .expect("terminal test splice anchor"); + super::super::prepared_terminal_bundle::anchor_prepared_terminal_bundle(asset, &prepared, anchor) + .expect("anchored terminal test bundle") + } + + /// # Panics + /// + /// Panics if the manifest does not describe a usable base track. + /// Test-support only. + pub fn base_timing_for_test( + asset: &HlsTerminalMediaAsset, + manifest: &HlsLeaseManifestSnapshot, + ) -> HlsTerminalBaseTimingEvidence { + HlsTerminalBaseTimingEvidence { + base: HlsTerminalBaseTrackIdentity { + proxy_seq: manifest.last_proxy_seq, + origin_epoch: 1, + cache_key: super::super::SegmentCacheKey::new( + ProxySessionId("terminal-timing-test".to_string()), + manifest.last_proxy_seq, + "ts", + ), + }, + profile: asset.timestamp_profile().expect("terminal test asset timing profile"), + } + } + + pub fn compatible_splice_evidence_for_test(asset: &HlsTerminalMediaAsset) -> HlsTsSpliceEvidence { + HlsTsSpliceEvidence::compatible_for_test(asset.track_signature().clone()) + } +} + +#[derive(Clone, Copy)] +pub struct HlsTerminalTailCompatibilityInput<'a> { + pub manifest: &'a HlsLeaseManifestSnapshot, + pub base_track_signature: Option<&'a HlsTsTrackSignature>, + pub boundary_evidence: HlsTerminalTailBoundaryEvidence<'a>, + pub expected_asset: HlsTerminalAssetIdentity, + pub asset: &'a HlsTerminalMediaAsset, +} + +#[derive(Clone, Copy)] +pub enum HlsTerminalTailBoundaryEvidence<'a> { + StructuralOnly, + Exact { base: Option<&'a HlsTsSpliceEvidence>, terminal: Option<&'a HlsTsSpliceEvidence> }, +} + +impl std::fmt::Debug for HlsTerminalTailPlan { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("HlsTerminalTailPlan") + .field("generation", &self.generation) + .field("reason", &self.reason) + .field("created_at_ms", &self.created_at_ms) + .field("base_manifest_generation", &self.base_manifest.snapshot_generation) + .field("base_segment_count", &self.base_manifest.visible_segments.len()) + .field("protected_base_segment_count", &self.protected_base_proxy_seqs.len()) + .field("terminal_segment_count", &self.segment_count) + .field("segment_duration_ms", &self.segment_duration_ms) + .field("append_key_method_none", &self.append_key_method_none) + .field("key_binding_count", &self.key_bindings.len()) + .field("manifest_byte_len", &self.manifest_body.len()) + .field("segment_content_type", &self.segment_content_type) + .finish_non_exhaustive() + } +} + +impl PartialEq for HlsTerminalTailPlan { + fn eq(&self, other: &Self) -> bool { + self.generation == other.generation + && self.created_at_ms == other.created_at_ms + && self.base_manifest == other.base_manifest + && self.protected_base_proxy_seqs == other.protected_base_proxy_seqs + && self.reason == other.reason + && self.asset_identity == other.asset_identity + && self.segment_count == other.segment_count + && self.segment_duration_ms == other.segment_duration_ms + && self.append_key_method_none == other.append_key_method_none + && self.key_bindings == other.key_bindings + && self.route_binding == other.route_binding + && self.manifest_body == other.manifest_body + && self.anchored_bundle.prepared_key == other.anchored_bundle.prepared_key + && self.anchored_bundle.splice_anchor == other.anchored_bundle.splice_anchor + && self.segment_content_type == other.segment_content_type + } +} + +impl Eq for HlsTerminalTailPlan {} + +impl HlsTerminalTailPlan { + pub fn media_preparation_key(&self) -> super::super::recovery_timing::HlsTerminalMediaPreparationKey { + self.anchored_bundle.prepared_key + } + + pub fn matches_route(&self, proxy_session_id: &ProxySessionId, lease_id: &HlsAccessLeaseId) -> bool { + self.route_binding.proxy_session_id == *proxy_session_id && self.route_binding.lease_id == *lease_id + } + + pub fn segment_content_length(&self, path: HlsTerminalSegmentPath) -> Option { + self.prepared_segment(path).and_then(|segment| u64::try_from(segment.bytes.len()).ok()) + } + + pub fn segment_content_type(&self) -> &'static str { self.segment_content_type } + + /// Clones one immutable pre-rendered segment only for this plan generation. + pub fn segment_bytes(&self, path: HlsTerminalSegmentPath) -> Option { + self.prepared_segment(path).map(|segment| segment.bytes.clone()) + } + + pub(super) fn prepared_segment(&self, path: HlsTerminalSegmentPath) -> Option<&HlsAnchoredTerminalSegment> { + if path.generation != self.generation || path.index >= self.segment_count { + return None; + } + let segment = self.anchored_bundle.segments.get(usize::from(path.index))?; + (segment.index == path.index).then_some(segment) + } + + pub fn terminal_key_binding( + &self, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, + resource_file: &TransientResourceFile, + ) -> Option { + self.matches_route(proxy_session_id, lease_id) + .then_some(()) + .and_then(|()| { + self.key_bindings + .iter() + .find(|binding| binding.matches_route(proxy_session_id) && binding.matches_resource(resource_file)) + }) + .cloned() + } + + pub fn key_bindings(&self) -> Arc<[HlsTerminalKeyBinding]> { Arc::clone(&self.key_bindings) } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum HlsLeasePlaybackMode { + #[default] + Live, + TerminalTail(Arc), + TerminalUnavailable { + decision_generation: u64, + reason: HlsTerminalTailCompatibility, + }, + Ended, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsTerminalTailCompatibility { + Compatible, + MissingAsset, + TerminalMediaNotReady, + InvalidAsset, + AssetRevisionMismatch, + MissingSafeBase, + TargetDurationExceeded { asset_ms: u64, target_ms: u64 }, + ActiveMapRequiresCompatibleFallback, + UnsupportedEncryptionTransition, + ContainerMismatch, + MissingTrackSignature, + TrackLayoutMismatch, + MissingSpliceEvidence, + SpliceTransportFailure(super::super::HlsTsSpliceIncompatibility), + SpliceTopologyMismatch, + MissingTimestampAnchor, + InvalidTimestampTransition, + TransientPassthroughUnsupported, + ProtectionCapacityExceeded, + InvalidLeaseRoute, + ManifestRenderFailed, +} + +pub fn evaluate_terminal_tail_compatibility( + input: HlsTerminalTailCompatibilityInput<'_>, +) -> HlsTerminalTailCompatibility { + let manifest = input.manifest; + let asset = input.asset; + if manifest.delivery_mode == HlsManifestDeliveryMode::TransientPassthrough { + return HlsTerminalTailCompatibility::TransientPassthroughUnsupported; + } + if asset.duration_ms() == 0 || asset.duration_ticks_90khz() == 0 { + return HlsTerminalTailCompatibility::InvalidAsset; + } + if input.expected_asset != HlsTerminalAssetIdentity::from_asset(asset) { + return HlsTerminalTailCompatibility::AssetRevisionMismatch; + } + if asset.content_type() != "video/mp2t" { + return HlsTerminalTailCompatibility::InvalidAsset; + } + if manifest.active_map.is_some() { + return HlsTerminalTailCompatibility::ActiveMapRequiresCompatibleFallback; + } + if manifest.container != HlsMediaContainer::MpegTs || asset.container() != HlsMediaContainer::MpegTs { + return HlsTerminalTailCompatibility::ContainerMismatch; + } + let Some(base_track_signature) = input.base_track_signature else { + return HlsTerminalTailCompatibility::MissingTrackSignature; + }; + if base_track_signature != asset.track_signature() { + return HlsTerminalTailCompatibility::TrackLayoutMismatch; + } + if let HlsTerminalTailBoundaryEvidence::Exact { base, terminal } = input.boundary_evidence { + let (Some(base), Some(terminal)) = (base, terminal) else { + return HlsTerminalTailCompatibility::MissingSpliceEvidence; + }; + match super::super::evaluate_mpeg_ts_splice_boundary(base, terminal) { + Ok(()) => {} + Err(super::super::HlsTsSpliceBoundaryIncompatibility::Media(reason)) => { + return HlsTerminalTailCompatibility::SpliceTransportFailure(reason); + } + Err(super::super::HlsTsSpliceBoundaryIncompatibility::TopologyMismatch) => { + return HlsTerminalTailCompatibility::SpliceTopologyMismatch; + } + } + } + if encryption_reset_required(manifest.active_encryption.as_deref()).is_err() { + return HlsTerminalTailCompatibility::UnsupportedEncryptionTransition; + } + if !terminal_asset_fits_target_duration(asset.duration_ms(), manifest.target_duration_ms) { + return HlsTerminalTailCompatibility::TargetDurationExceeded { + asset_ms: asset.duration_ms(), + target_ms: manifest.target_duration_ms, + }; + } + HlsTerminalTailCompatibility::Compatible +} + +pub(super) fn encryption_reset_required( + encryption: Option<&HlsEncryptionSignature>, +) -> Result { + let Some(encryption) = encryption else { + return Ok(false); + }; + match parse_encryption_method(&encryption.method) { + ParsedEncryptionMethod::None => Ok(false), + ParsedEncryptionMethod::Aes128 + if encryption.can_reset_to_clear + && encryption.key_format.as_deref().is_none_or(|format| format.eq_ignore_ascii_case("identity")) + && encryption.key_uri.as_deref().is_some_and(valid_quoted_attribute) + && encryption.iv.as_deref().is_none_or(valid_iv) + && encryption.key_format_versions.as_deref().is_none_or(valid_key_format_versions) + && (encryption.key_format_versions.is_none() + || encryption + .key_format + .as_deref() + .is_some_and(|format| format.eq_ignore_ascii_case("identity"))) => + { + Ok(true) + } + ParsedEncryptionMethod::Aes128 | ParsedEncryptionMethod::Unsupported(_) => { + Err(HlsTerminalTailCompatibility::UnsupportedEncryptionTransition) + } + } +} + +pub fn build_terminal_tail_plan( + input: HlsTerminalTailBuildInput, +) -> Result { + let compatibility = evaluate_terminal_tail_compatibility(HlsTerminalTailCompatibilityInput { + manifest: &input.base_manifest, + base_track_signature: input.base_track_signature.as_ref(), + boundary_evidence: HlsTerminalTailBoundaryEvidence::Exact { + base: input.base_splice_evidence.as_ref(), + terminal: input.terminal_splice_evidence.as_ref(), + }, + expected_asset: input.expected_asset.media, + asset: &input.asset, + }); + log_terminal_splice_compatibility(input.base_manifest.last_proxy_seq, compatibility); + if compatibility != HlsTerminalTailCompatibility::Compatible { + return Err(compatibility); + } + let expected_bundle_key = HlsPreparedTerminalBundleKey { + asset: input.expected_asset.media, + target_duration_ms: input.base_manifest.target_duration_ms, + segment_count: HLS_TERMINAL_TAIL_SEGMENT_COUNT, + }; + if !input.anchored_bundle.matches_key_and_shape(expected_bundle_key, input.asset.duration_ticks_90khz()) { + return Err(HlsTerminalTailCompatibility::AssetRevisionMismatch); + } + let Some(base_timing) = input.base_timing.as_ref() else { + return Err(HlsTerminalTailCompatibility::MissingTimestampAnchor); + }; + let Some(asset_profile) = input.asset.timestamp_profile() else { + return Err(HlsTerminalTailCompatibility::InvalidTimestampTransition); + }; + let Some(expected_anchor) = + tuliprox_mpegts::transport_stream_buffer::HlsTsSpliceAnchor::between(base_timing.profile, asset_profile) + else { + return Err(HlsTerminalTailCompatibility::InvalidTimestampTransition); + }; + if base_timing.base.proxy_seq != input.base_manifest.last_proxy_seq + || input.anchored_bundle.splice_anchor != expected_anchor + { + return Err(HlsTerminalTailCompatibility::InvalidTimestampTransition); + } + let append_key_method_none = encryption_reset_required(input.base_manifest.active_encryption.as_deref())?; + let Some((base_manifest, protected_base_proxy_seqs)) = safe_terminal_base( + input.base_manifest, + &input.base_availability, + if append_key_method_none { HlsTerminalBaseScope::LastSafeSegment } else { HlsTerminalBaseScope::SafeSuffix }, + ) else { + return Err(HlsTerminalTailCompatibility::MissingSafeBase); + }; + let Some(route_binding) = terminal_tail_route_binding(&base_manifest) else { + return Err(HlsTerminalTailCompatibility::InvalidLeaseRoute); + }; + let key_bindings = terminal_key_bindings(&base_manifest, &input.base_key_bindings, &route_binding)?; + let segment_count = input.anchored_bundle.prepared_key.segment_count; + let segment_duration_ms = input.asset.duration_ms(); + let segment_content_type = input.asset.content_type(); + let manifest_body = render_terminal_tail_manifest_body(&HlsTerminalTailManifestRenderInput { + generation: input.generation, + base_manifest: &base_manifest, + protected_base_proxy_seqs: &protected_base_proxy_seqs, + asset: &input.asset, + asset_identity: input.expected_asset.media, + segment_count, + segment_duration_ms, + append_key_method_none, + route_binding: &route_binding, + }) + .map(Arc::from) + .map_err(|_| HlsTerminalTailCompatibility::ManifestRenderFailed)?; + Ok(HlsTerminalTailPlan { + generation: input.generation, + created_at_ms: input.created_at_ms, + segment_count, + segment_duration_ms, + append_key_method_none, + key_bindings, + base_manifest, + protected_base_proxy_seqs, + reason: input.expected_asset.reason, + asset_identity: input.expected_asset, + route_binding, + manifest_body, + anchored_bundle: input.anchored_bundle, + segment_content_type, + }) +} + +fn terminal_key_bindings( + manifest: &HlsLeaseManifestSnapshot, + evidence: &[HlsTerminalKeyBinding], + route_binding: &HlsTerminalTailRouteBinding, +) -> Result, HlsTerminalTailCompatibility> { + let mut selected = Vec::new(); + for encryption in manifest.visible_segments.iter().filter_map(|segment| segment.encryption.as_ref()) { + match parse_encryption_method(&encryption.method) { + ParsedEncryptionMethod::None => continue, + ParsedEncryptionMethod::Aes128 => {} + ParsedEncryptionMethod::Unsupported(_) => { + return Err(HlsTerminalTailCompatibility::UnsupportedEncryptionTransition); + } + } + let Some(resource_file) = terminal_key_resource_file(encryption) else { + return Err(HlsTerminalTailCompatibility::MissingSafeBase); + }; + let mut matches = evidence.iter().filter(|binding| { + binding.matches_route(&route_binding.proxy_session_id) && binding.matches_resource(&resource_file) + }); + let Some(binding) = matches.next().cloned() else { + return Err(HlsTerminalTailCompatibility::MissingSafeBase); + }; + if matches.any(|candidate| candidate != &binding) { + return Err(HlsTerminalTailCompatibility::MissingSafeBase); + } + if !selected.contains(&binding) { + selected.push(binding); + } + } + selected.sort_by(|left, right| { + (&left.resource_id.0, left.route_extension.as_ref()) + .cmp(&(&right.resource_id.0, right.route_extension.as_ref())) + }); + Ok(selected.into()) +} diff --git a/backend/hls/src/terminal_tail/render.rs b/backend/hls/src/terminal_tail/render.rs new file mode 100644 index 000000000..feb38a2e7 --- /dev/null +++ b/backend/hls/src/terminal_tail/render.rs @@ -0,0 +1,290 @@ +use super::{ + assets::{parse_encryption_method, HlsTerminalBaseScope, ParsedEncryptionMethod}, + plan::encryption_reset_required, + HlsAccessLeaseId, HlsEncryptionSignature, HlsLeaseManifestSnapshot, HlsTerminalAssetIdentity, + HlsTerminalBaseMediaState, HlsTerminalBaseProtection, HlsTerminalBaseSegmentAvailability, + HlsTerminalTailGeneration, HlsTerminalTailManifestRenderInput, HlsTerminalTailPlan, HlsTerminalTailRouteBinding, + ProxySessionId, TransientResourceFile, HLS_SHARED_LIVE_ROUTE_MARKER, +}; +use std::{ + collections::{hash_map::Entry, HashMap}, + fmt::Write as _, + sync::Arc, +}; +use tuliprox_core::utils::{format_hls_duration_ms, hls_target_duration_secs}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HlsTerminalSegmentPath { + pub generation: HlsTerminalTailGeneration, + pub index: u16, +} + +impl HlsTerminalSegmentPath { + pub fn parse(generation: &str, terminal_file: &str) -> Option { + let index = terminal_file.strip_suffix(".ts")?; + Some(Self { + generation: HlsTerminalTailGeneration(parse_canonical_decimal(generation)?), + index: parse_canonical_decimal(index)?, + }) + } +} + +fn parse_canonical_decimal(value: &str) -> Option +where + T: std::str::FromStr, +{ + if value.is_empty() + || (value.len() > 1 && value.starts_with('0')) + || !value.bytes().all(|byte| byte.is_ascii_digit()) + { + return None; + } + value.parse().ok() +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum HlsTerminalTailRenderError { + #[error("terminal tail has no safe base segments")] + MissingSafeBase, + #[error("terminal tail asset identity changed")] + AssetIdentityChanged, + #[error("terminal tail has an invalid encryption state")] + InvalidEncryptionState, + #[error("terminal tail formatting failed")] + Formatting, + #[error("terminal tail route does not match its frozen lease binding")] + RouteBindingMismatch, +} + +pub(super) fn valid_quoted_attribute(value: &str) -> bool { + !value.is_empty() && value.chars().all(|character| character != '"' && !character.is_control()) +} + +pub(super) fn valid_iv(value: &str) -> bool { super::super::hls_aes128_cbc_iv(Some(value), 0).is_ok() } + +pub(super) fn valid_key_format_versions(value: &str) -> bool { + !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit() || byte == b'/') +} + +pub(super) fn safe_terminal_base( + mut manifest: HlsLeaseManifestSnapshot, + availability: &[HlsTerminalBaseSegmentAvailability], + scope: HlsTerminalBaseScope, +) -> Option<(HlsLeaseManifestSnapshot, Arc<[u64]>)> { + let mut availability_by_seq = HashMap::>::new(); + for state in availability { + match availability_by_seq.entry(state.proxy_seq) { + Entry::Vacant(entry) => { + entry.insert(Some(state)); + } + Entry::Occupied(mut entry) => { + entry.insert(None); + } + } + } + + let mut start = manifest.visible_segments.len(); + let mut expected_next = None; + for (index, segment) in manifest.visible_segments.iter().enumerate().rev() { + if expected_next.is_some_and(|next| segment.proxy_seq.checked_add(1) != Some(next)) { + break; + } + let state = availability_by_seq.get(&segment.proxy_seq).copied().flatten(); + let safe = state.is_some_and(|state| { + state.media_state == HlsTerminalBaseMediaState::Ready + && state.required_map_ready + && state.required_key_ready + && state.protection == HlsTerminalBaseProtection::Protectable + && segment.map_ref_ready + && segment.encryption == manifest.active_encryption + }); + if !safe { + break; + } + start = index; + expected_next = Some(segment.proxy_seq); + } + if start == manifest.visible_segments.len() { + return None; + } + if scope == HlsTerminalBaseScope::LastSafeSegment { + start = manifest.visible_segments.len().saturating_sub(1); + } + + let mut selected = manifest.visible_segments[start..].to_vec(); + let removed_discontinuities = + manifest.visible_segments[..start].iter().filter(|segment| segment.discontinuity_before).count(); + if start > 0 && selected.first().is_some_and(|segment| segment.discontinuity_before) { + if let Some(first) = selected.first_mut() { + first.discontinuity_before = false; + } + manifest.discontinuity_sequence = manifest.discontinuity_sequence.saturating_add(1); + } + manifest.discontinuity_sequence = + manifest.discontinuity_sequence.saturating_add(u64::try_from(removed_discontinuities).unwrap_or(u64::MAX)); + let first_proxy_seq = selected.first()?.proxy_seq; + let last_proxy_seq = selected.last()?.proxy_seq; + let playlist_duration_ms = + selected.iter().fold(0u64, |duration, segment| duration.saturating_add(segment.duration_ms)); + let protected = Arc::from(selected.iter().map(|segment| segment.proxy_seq).collect::>()); + manifest.first_proxy_seq = first_proxy_seq; + manifest.last_proxy_seq = last_proxy_seq; + manifest.playlist_duration_ms = playlist_duration_ms; + manifest.last_visible_media_end_ms = playlist_duration_ms; + manifest.visible_segments = Arc::from(selected); + Some((manifest, protected)) +} + +pub(super) fn terminal_tail_route_binding(manifest: &HlsLeaseManifestSnapshot) -> Option { + let first = manifest.visible_segments.first()?; + let first_uri = manifest.materialize_uri(&first.uri); + let binding = terminal_tail_route_binding_from_uri(&first_uri)?; + manifest + .visible_segments + .iter() + .all(|segment| { + terminal_tail_route_binding_from_uri(&manifest.materialize_uri(&segment.uri)).as_ref() == Some(&binding) + }) + .then_some(binding) +} + +fn terminal_tail_route_binding_from_uri(uri: &str) -> Option { + let path = uri.split(['?', '#']).next()?; + let marker_offset = path.find(HLS_SHARED_LIVE_ROUTE_MARKER)?; + let public_path_prefix = path.get(..marker_offset)?; + if !public_path_prefix.is_empty() && (!public_path_prefix.starts_with('/') || public_path_prefix.ends_with('/')) { + return None; + } + let route = path.get(marker_offset.saturating_add(HLS_SHARED_LIVE_ROUTE_MARKER.len())..)?; + let mut components = route.split('/'); + let proxy_session_id = components.next()?; + let lease_id = components.next()?; + let media_file = components.next()?; + if !valid_terminal_route_component(proxy_session_id) + || !valid_terminal_route_component(lease_id) + || media_file.is_empty() + || components.next().is_some() + || public_path_prefix.chars().any(char::is_control) + || media_file.chars().any(char::is_control) + { + return None; + } + Some(HlsTerminalTailRouteBinding { + public_path_prefix: Arc::from(public_path_prefix), + proxy_session_id: ProxySessionId(proxy_session_id.to_string()), + lease_id: HlsAccessLeaseId(lease_id.to_string()), + }) +} + +fn valid_terminal_route_component(value: &str) -> bool { + !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) +} + +pub(super) fn terminal_key_resource_file(encryption: &HlsEncryptionSignature) -> Option { + encryption.key_uri.as_deref()?.split(['?', '#']).next()?.rsplit('/').next().and_then(TransientResourceFile::parse) +} + +pub fn terminal_tail_manifest_body<'a>( + plan: &'a HlsTerminalTailPlan, + proxy_session_id: &ProxySessionId, + lease_id: &HlsAccessLeaseId, +) -> Result<&'a str, HlsTerminalTailRenderError> { + if !plan.matches_route(proxy_session_id, lease_id) { + return Err(HlsTerminalTailRenderError::RouteBindingMismatch); + } + Ok(&plan.manifest_body) +} + +pub(super) fn render_terminal_tail_manifest_body( + input: &HlsTerminalTailManifestRenderInput<'_>, +) -> Result { + if input.asset_identity != HlsTerminalAssetIdentity::from_asset(input.asset) { + return Err(HlsTerminalTailRenderError::AssetIdentityChanged); + } + let Some(first) = input.base_manifest.visible_segments.first() else { + return Err(HlsTerminalTailRenderError::MissingSafeBase); + }; + if input.protected_base_proxy_seqs != input.base_manifest.visible_proxy_seqs().collect::>().as_slice() { + return Err(HlsTerminalTailRenderError::MissingSafeBase); + } + let mut body = String::new(); + let version = input.base_manifest.active_encryption.as_ref().map_or(3, |encryption| { + if encryption.key_format.is_some() || encryption.key_format_versions.is_some() { + 5 + } else { + 3 + } + }); + writeln!(body, "#EXTM3U\n#EXT-X-VERSION:{version}").map_err(|_| HlsTerminalTailRenderError::Formatting)?; + writeln!(body, "#EXT-X-TARGETDURATION:{}", hls_target_duration_secs(input.base_manifest.target_duration_ms)) + .map_err(|_| HlsTerminalTailRenderError::Formatting)?; + writeln!(body, "#EXT-X-MEDIA-SEQUENCE:{}", first.proxy_seq).map_err(|_| HlsTerminalTailRenderError::Formatting)?; + writeln!(body, "#EXT-X-DISCONTINUITY-SEQUENCE:{}", input.base_manifest.discontinuity_sequence) + .map_err(|_| HlsTerminalTailRenderError::Formatting)?; + render_active_base_encryption(&mut body, input.base_manifest)?; + for segment in input.base_manifest.visible_segments.iter() { + if segment.discontinuity_before { + body.push_str("#EXT-X-DISCONTINUITY\n"); + } + writeln!(body, "#EXTINF:{},", format_hls_duration_ms(segment.duration_ms)) + .map_err(|_| HlsTerminalTailRenderError::Formatting)?; + writeln!(body, "{}", input.base_manifest.materialize_uri(&segment.uri)) + .map_err(|_| HlsTerminalTailRenderError::Formatting)?; + } + if input.append_key_method_none { + body.push_str("#EXT-X-KEY:METHOD=NONE\n"); + } + body.push_str("#EXT-X-DISCONTINUITY\n"); + for index in 0..input.segment_count { + writeln!(body, "#EXTINF:{},", format_hls_duration_ms(input.segment_duration_ms)) + .map_err(|_| HlsTerminalTailRenderError::Formatting)?; + writeln!( + body, + "{}/hls/shared/live/{}/{}/terminal/{}/{}.ts", + input.route_binding.public_path_prefix, + input.route_binding.proxy_session_id.0, + input.route_binding.lease_id.0, + input.generation.0, + index + ) + .map_err(|_| HlsTerminalTailRenderError::Formatting)?; + } + body.push_str("#EXT-X-ENDLIST\n"); + Ok(body) +} + +fn render_active_base_encryption( + body: &mut String, + manifest: &HlsLeaseManifestSnapshot, +) -> Result<(), HlsTerminalTailRenderError> { + let encryption = manifest.active_encryption.as_deref(); + let Some(encryption) = encryption else { + return Ok(()); + }; + match parse_encryption_method(&encryption.method) { + ParsedEncryptionMethod::None => Ok(()), + ParsedEncryptionMethod::Aes128 => { + encryption_reset_required(Some(encryption)) + .map_err(|_| HlsTerminalTailRenderError::InvalidEncryptionState)?; + let Some(key_uri) = encryption.key_uri.as_deref() else { + return Err(HlsTerminalTailRenderError::InvalidEncryptionState); + }; + let key_uri = manifest.materialize_uri(key_uri); + write!(body, "#EXT-X-KEY:METHOD=AES-128,URI=\"{key_uri}\"") + .map_err(|_| HlsTerminalTailRenderError::Formatting)?; + if let Some(iv) = encryption.iv.as_deref() { + write!(body, ",IV={iv}").map_err(|_| HlsTerminalTailRenderError::Formatting)?; + } + if encryption.key_format.as_deref().is_some_and(|format| format.eq_ignore_ascii_case("identity")) { + body.push_str(",KEYFORMAT=\"identity\""); + } + if let Some(versions) = encryption.key_format_versions.as_deref() { + write!(body, ",KEYFORMATVERSIONS=\"{versions}\"") + .map_err(|_| HlsTerminalTailRenderError::Formatting)?; + } + body.push('\n'); + Ok(()) + } + ParsedEncryptionMethod::Unsupported(_) => Err(HlsTerminalTailRenderError::InvalidEncryptionState), + } +} diff --git a/backend/hls/src/terminal_tail/tests.rs b/backend/hls/src/terminal_tail/tests.rs new file mode 100644 index 000000000..2eafb74ae --- /dev/null +++ b/backend/hls/src/terminal_tail/tests.rs @@ -0,0 +1,281 @@ +use super::{ + super::media_reserve::{HlsLeaseManifestSegment, HlsManifestCommitIdentity, HlsManifestDeliveryMode}, + *, +}; +use bytes::Bytes; +use tuliprox_core::utils::format_hls_duration_ms; +use tuliprox_mpegts::transport_stream_buffer::{HlsFiniteTsRenderSpec, TransportStreamBuffer}; + +const TERMINAL_ASSET_BYTES: &[u8] = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); + +fn manifest() -> HlsLeaseManifestSnapshot { + HlsLeaseManifestSnapshot { + startup_revisions: None, + delivery_mode: HlsManifestDeliveryMode::NormalCacheTimeline, + source_commit_identity: HlsManifestCommitIdentity::new(1), + uri_materialization: None, + finalized_transient_manifest_generation: None, + snapshot_generation: 1, + delivered_at_ms: 10, + first_proxy_seq: 193, + last_proxy_seq: 194, + visible_segments: Arc::from([ + HlsLeaseManifestSegment { + proxy_seq: 193, + duration_ms: 9_940, + uri: "/iptv/hls/shared/live/session/lease/193.ts".into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }, + HlsLeaseManifestSegment { + proxy_seq: 194, + duration_ms: 10_380, + uri: "/iptv/hls/shared/live/session/lease/194.ts".into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + }, + ]), + discontinuity_sequence: 7, + target_duration_ms: 12_000, + playlist_duration_ms: 20_320, + last_visible_media_end_ms: 20_320, + active_map: None, + active_encryption: None, + container: HlsMediaContainer::MpegTs, + } +} + +fn asset() -> Arc { + let bytes = Bytes::from_static(TERMINAL_ASSET_BYTES); + let buffer = TransportStreamBuffer::new(bytes.to_vec()); + snapshot_terminal_media_asset(&buffer).expect("valid terminal asset") +} + +fn availability(base: &HlsLeaseManifestSnapshot) -> Arc<[HlsTerminalBaseSegmentAvailability]> { + Arc::from( + base.visible_proxy_seqs() + .map(|proxy_seq| HlsTerminalBaseSegmentAvailability { + proxy_seq, + media_state: HlsTerminalBaseMediaState::Ready, + required_map_ready: true, + required_key_ready: true, + protection: HlsTerminalBaseProtection::Protectable, + }) + .collect::>(), + ) +} + +fn build_input( + generation: u64, + base_manifest: HlsLeaseManifestSnapshot, + asset: Arc, +) -> HlsTerminalTailBuildInput { + let anchored_bundle = HlsTerminalTailBuildInput::anchored_bundle_for_test(&asset, base_manifest.target_duration_ms); + let base_timing = Some(HlsTerminalTailBuildInput::base_timing_for_test(&asset, &base_manifest)); + let base_splice_evidence = Some(HlsTerminalTailBuildInput::compatible_splice_evidence_for_test(&asset)); + let terminal_splice_evidence = base_splice_evidence.clone(); + HlsTerminalTailBuildInput { + generation: HlsTerminalTailGeneration(generation), + created_at_ms: 20, + base_availability: availability(&base_manifest), + base_track_signature: Some(asset.track_signature().clone()), + base_splice_evidence, + terminal_splice_evidence, + base_timing, + base_key_bindings: Arc::from([]), + expected_asset: HlsRuntimeCustomTailAssetIdentity { + reason: HlsRuntimeCustomTailReason::ChannelUnavailable, + media: HlsTerminalAssetIdentity::from_asset(&asset), + }, + base_manifest, + asset, + anchored_bundle, + } +} + +fn compatibility( + base: &HlsLeaseManifestSnapshot, + asset: &HlsTerminalMediaAsset, + base_track_signature: Option<&HlsTsTrackSignature>, +) -> HlsTerminalTailCompatibility { + evaluate_terminal_tail_compatibility(HlsTerminalTailCompatibilityInput { + manifest: base, + base_track_signature, + boundary_evidence: HlsTerminalTailBoundaryEvidence::StructuralOnly, + expected_asset: HlsTerminalAssetIdentity::from_asset(asset), + asset, + }) +} + +fn resettable_aes128_encryption() -> Arc { + Arc::new(HlsEncryptionSignature { + method: "AES-128".into(), + key_uri: Some("/iptv/hls/shared/live/session/lease/r/aes-key.key".into()), + iv: Some("0x000000000000000000000000000000C2".into()), + key_format: Some("identity".into()), + key_format_versions: Some("1".into()), + can_reset_to_clear: true, + }) +} + +fn aes128_key_binding() -> HlsTerminalKeyBinding { + HlsTerminalKeyBinding::new( + ProxySessionId("session".into()), + TransientResourceId("aes-key".into()), + "key".into(), + TransientObjectCacheKey::new( + ProxySessionId("session".into()), + TransientResourceId("aes-key".into()), + "key.fill-0000000000000001", + ), + "application/octet-stream".into(), + b"0123456789abcdef", + ) + .expect("valid AES-128 binding") +} + +async fn unresolved_aes_terminal_base_evidence(iv: &str) -> HlsTerminalBaseEvidence { + let temp_dir = tempfile::tempdir().expect("terminal evidence cache tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let mut session = + super::super::HlsSession::new(super::super::HlsSessionKey::new(1, "terminal-evidence"), b"secret", 1); + let proxy_seq = 194; + let cache_key = super::super::SegmentCacheKey::new(session.proxy_session_id.clone(), proxy_seq, "ts"); + cache.write_bytes_and_commit(&cache_key, TERMINAL_ASSET_BYTES).await.expect("terminal evidence segment commits"); + let timeline_encryption = super::super::HlsSegmentEncryption { + resource_id: TransientResourceId("aes-key".into()), + resource_extension: "key".into(), + iv: Some(iv.into()), + key_format: Some("identity".into()), + key_format_versions: Some("1".into()), + }; + session.segments.insert( + proxy_seq, + SegmentEntry { + origin_key: super::super::OriginSegmentKey { + origin_epoch: 3, + effective_host_id: 7, + host_local_sequence: 900, + host_local_index: 0, + }, + proxy_seq, + duration_ms: 4_000, + proxy_file_ext: "ts".into(), + content_type: "video/mp2t".into(), + cache_key, + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + encryption: Some(timeline_encryption), + origin_fetch_ref: None, + status: SegmentCacheStatus::Ready { + content_length: u64::try_from(TERMINAL_ASSET_BYTES.len()).unwrap_or(u64::MAX), + ready_at_ms: 2, + }, + last_rendered_at_ms: Some(2), + access: Arc::new(CacheAccessState::new()), + }, + ); + let encryption = HlsEncryptionSignature { + method: "AES-128".into(), + key_uri: Some("/iptv/hls/shared/live/session/lease/r/aes-key.key".into()), + iv: Some(iv.into()), + key_format: Some("identity".into()), + key_format_versions: Some("1".into()), + can_reset_to_clear: true, + }; + let mut base = manifest(); + base.first_proxy_seq = proxy_seq; + base.last_proxy_seq = proxy_seq; + base.visible_segments = Arc::from([HlsLeaseManifestSegment { + proxy_seq, + duration_ms: 4_000, + uri: format!("/iptv/hls/shared/live/{}/lease/{proxy_seq}.ts", session.proxy_session_id.0).into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: Some(Arc::new(encryption.clone())), + }]); + base.active_encryption = Some(Arc::new(encryption)); + base.playlist_duration_ms = 4_000; + base.last_visible_media_end_ms = 4_000; + let session = Arc::new(tokio::sync::RwLock::new(session)); + + prepare_terminal_base_evidence(&session, &cache, &base, 3).await +} + +async fn terminal_timing_session( + cache: &HlsSegmentCache, + terminal_asset: &HlsTerminalMediaAsset, + first_bytes: &[u8], + last_bytes: &[u8], +) -> (Arc>, HlsLeaseManifestSnapshot, super::super::SegmentCacheKey, u64) +{ + let mut session = + super::super::HlsSession::new(super::super::HlsSessionKey::new(1, "terminal-timing-evidence"), b"secret", 1); + let proxy_session_id = session.proxy_session_id.clone(); + let first_proxy_seq = 193; + let last_proxy_seq = 194; + let first_cache_key = super::super::SegmentCacheKey::new(proxy_session_id.clone(), first_proxy_seq, "ts"); + let last_cache_key = super::super::SegmentCacheKey::new(proxy_session_id.clone(), last_proxy_seq, "ts"); + cache.write_bytes_and_commit(&first_cache_key, first_bytes).await.expect("first segment commits"); + cache.write_bytes_and_commit(&last_cache_key, last_bytes).await.expect("last segment commits"); + for (proxy_seq, cache_key, content_length) in [ + (first_proxy_seq, first_cache_key, first_bytes.len()), + (last_proxy_seq, last_cache_key.clone(), last_bytes.len()), + ] { + session.segments.insert( + proxy_seq, + SegmentEntry { + origin_key: super::super::OriginSegmentKey { + origin_epoch: 7, + effective_host_id: 3, + host_local_sequence: proxy_seq, + host_local_index: 0, + }, + proxy_seq, + duration_ms: terminal_asset.duration_ms(), + proxy_file_ext: "ts".into(), + content_type: "video/mp2t".into(), + cache_key, + discontinuity_before: false, + program_date_time: None, + daterange_tags_before: Vec::new(), + origin_byte_range: None, + map_ref: None, + encryption: None, + origin_fetch_ref: None, + status: SegmentCacheStatus::Ready { + content_length: u64::try_from(content_length).unwrap_or(u64::MAX), + ready_at_ms: 2, + }, + last_rendered_at_ms: Some(2), + access: Arc::new(CacheAccessState::new()), + }, + ); + } + let mut base = manifest(); + base.first_proxy_seq = first_proxy_seq; + base.last_proxy_seq = last_proxy_seq; + base.visible_segments = Arc::from([first_proxy_seq, last_proxy_seq].map(|proxy_seq| HlsLeaseManifestSegment { + proxy_seq, + duration_ms: terminal_asset.duration_ms(), + uri: format!("/iptv/hls/shared/live/{}/lease/{proxy_seq}.ts", proxy_session_id.0).into(), + discontinuity_before: false, + map_ref_ready: true, + encryption: None, + })); + base.playlist_duration_ms = terminal_asset.duration_ms().saturating_mul(2); + base.last_visible_media_end_ms = base.playlist_duration_ms; + (Arc::new(tokio::sync::RwLock::new(session)), base, last_cache_key, last_proxy_seq) +} + +mod policy; +mod publication; +mod storage; +mod terminal; +mod transport; diff --git a/backend/hls/src/terminal_tail/tests/policy.rs b/backend/hls/src/terminal_tail/tests/policy.rs new file mode 100644 index 000000000..d9b314050 --- /dev/null +++ b/backend/hls/src/terminal_tail/tests/policy.rs @@ -0,0 +1,48 @@ +use super::*; + +#[test] +fn method_none_boundary_keeps_only_homogeneous_clear_suffix() { + let mut base = manifest(); + Arc::make_mut(&mut base.visible_segments)[0].encryption = Some(resettable_aes128_encryption()); + + let plan = build_terminal_tail_plan(build_input(2, base, asset())).expect("clear suffix"); + let rendered = + terminal_tail_manifest_body(&plan, &ProxySessionId("session".into()), &HlsAccessLeaseId("lease".into())) + .expect("render"); + + assert_eq!(plan.protected_base_proxy_seqs.as_ref(), &[194]); + assert!(!rendered.contains("/iptv/hls/shared/live/session/lease/193.ts")); + assert!(!rendered.contains("#EXT-X-KEY:METHOD=AES-128")); + assert!(!rendered.contains("#EXT-X-KEY:METHOD=NONE")); +} + +#[test] +fn asset_rounded_duration_must_not_exceed_target_duration() { + let asset = asset(); + let mut short_target = manifest(); + short_target.target_duration_ms = 9_000; + + assert_eq!( + compatibility(&short_target, &asset, Some(asset.track_signature())), + HlsTerminalTailCompatibility::TargetDurationExceeded { asset_ms: asset.duration_ms(), target_ms: 9_000 } + ); +} + +#[test] +fn unsupported_key_format_cannot_reset_to_clear_asset() { + let mut base = manifest(); + base.active_encryption = Some(Arc::new(HlsEncryptionSignature { + method: "SAMPLE-AES".into(), + key_uri: Some("/keys/drm.key".into()), + iv: None, + key_format: Some("com.example.drm".into()), + key_format_versions: None, + can_reset_to_clear: true, + })); + let asset = asset(); + + assert_eq!( + compatibility(&base, &asset, Some(asset.track_signature())), + HlsTerminalTailCompatibility::UnsupportedEncryptionTransition + ); +} diff --git a/backend/hls/src/terminal_tail/tests/publication.rs b/backend/hls/src/terminal_tail/tests/publication.rs new file mode 100644 index 000000000..4020508dc --- /dev/null +++ b/backend/hls/src/terminal_tail/tests/publication.rs @@ -0,0 +1,103 @@ +use super::*; + +#[test] +fn hls_terminal_response_manifest_keeps_live_tail_and_appends_finite_endlist_tail() { + let asset = asset(); + let plan = build_terminal_tail_plan(build_input(4, manifest(), Arc::clone(&asset))).expect("compatible tail"); + + let first = + terminal_tail_manifest_body(&plan, &ProxySessionId("session".into()), &HlsAccessLeaseId("lease".into())) + .expect("render tail"); + let second = + terminal_tail_manifest_body(&plan, &ProxySessionId("session".into()), &HlsAccessLeaseId("lease".into())) + .expect("render tail again"); + + assert_eq!(first, second); + assert_eq!(first.matches("#EXTM3U").count(), 1); + assert!(first.contains("/iptv/hls/shared/live/session/lease/193.ts")); + assert_eq!( + first.matches("/iptv/hls/shared/live/session/lease/terminal/4/").count(), + usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT) + ); + assert_eq!(first.matches("#EXT-X-DISCONTINUITY\n").count(), 1); + assert!(first.ends_with("#EXT-X-ENDLIST\n")); + assert!(first.contains("#EXT-X-TARGETDURATION:12\n")); + assert_eq!(plan.base_manifest.target_duration_ms, 12_000); + assert_eq!(plan.segment_duration_ms, asset.duration_ms()); + let terminal_media = first + .rsplit_once("#EXT-X-DISCONTINUITY\n") + .map(|(_, terminal_media)| terminal_media) + .expect("terminal discontinuity"); + assert_eq!( + terminal_media.matches(&format!("#EXTINF:{},", format_hls_duration_ms(asset.duration_ms()))).count(), + usize::from(HLS_TERMINAL_TAIL_SEGMENT_COUNT) + ); + assert_eq!( + terminal_tail_manifest_body(&plan, &ProxySessionId("session".into()), &HlsAccessLeaseId("other-lease".into()),), + Err(HlsTerminalTailRenderError::RouteBindingMismatch) + ); +} + +#[test] +fn hls_terminal_response_path_parser_rejects_noncanonical_generation_and_file() { + assert_eq!( + HlsTerminalSegmentPath::parse("17", "2.ts"), + Some(HlsTerminalSegmentPath { generation: HlsTerminalTailGeneration(17), index: 2 }) + ); + for (generation, terminal_file) in [ + ("", "0.ts"), + ("01", "0.ts"), + ("+1", "0.ts"), + ("one", "0.ts"), + ("1", ""), + ("1", "00.ts"), + ("1", "+1.ts"), + ("1", "1.m4s"), + ("1", "65536.ts"), + ] { + assert_eq!(HlsTerminalSegmentPath::parse(generation, terminal_file), None); + } +} + +#[test] +fn transient_origin_backed_manifest_is_explicitly_incompatible() { + let asset = asset(); + let mut base = manifest(); + base.delivery_mode = HlsManifestDeliveryMode::TransientPassthrough; + + assert_eq!( + compatibility(&base, &asset, Some(asset.track_signature())), + HlsTerminalTailCompatibility::TransientPassthroughUnsupported + ); +} + +#[test] +fn unsafe_key_uri_or_iv_cannot_be_rendered_into_terminal_manifest() { + let asset = asset(); + let mut unsafe_uri = manifest(); + let mut encryption = resettable_aes128_encryption(); + Arc::make_mut(&mut encryption).key_uri = Some("/key\"\n#EXT-X-ENDLIST".into()); + unsafe_uri.active_encryption = Some(encryption); + assert_eq!( + compatibility(&unsafe_uri, &asset, Some(asset.track_signature())), + HlsTerminalTailCompatibility::UnsupportedEncryptionTransition + ); + + let mut unsafe_iv = manifest(); + let mut encryption = resettable_aes128_encryption(); + Arc::make_mut(&mut encryption).iv = Some("not-hex".into()); + unsafe_iv.active_encryption = Some(encryption); + assert_eq!( + compatibility(&unsafe_iv, &asset, Some(asset.track_signature())), + HlsTerminalTailCompatibility::UnsupportedEncryptionTransition + ); +} + +#[test] +fn stale_asset_revision_cannot_build_terminal_plan() { + let asset = asset(); + let mut input = build_input(1, manifest(), asset); + input.expected_asset.media.revision = input.expected_asset.media.revision.saturating_add(1); + + assert_eq!(build_terminal_tail_plan(input), Err(HlsTerminalTailCompatibility::AssetRevisionMismatch)); +} diff --git a/backend/hls/src/terminal_tail/tests/storage.rs b/backend/hls/src/terminal_tail/tests/storage.rs new file mode 100644 index 000000000..1f9b7a8f8 --- /dev/null +++ b/backend/hls/src/terminal_tail/tests/storage.rs @@ -0,0 +1,12 @@ +use super::*; + +#[test] +fn duplicate_readiness_evidence_is_not_accepted_as_safe_base() { + let mut input = build_input(1, manifest(), asset()); + let duplicate = input.base_availability[1]; + let mut states = input.base_availability.to_vec(); + states.push(duplicate); + input.base_availability = Arc::from(states); + + assert_eq!(build_terminal_tail_plan(input), Err(HlsTerminalTailCompatibility::MissingSafeBase)); +} diff --git a/backend/hls/src/terminal_tail/tests/terminal.rs b/backend/hls/src/terminal_tail/tests/terminal.rs new file mode 100644 index 000000000..183dc5ca7 --- /dev/null +++ b/backend/hls/src/terminal_tail/tests/terminal.rs @@ -0,0 +1,287 @@ +use super::*; + +#[tokio::test] +async fn terminal_base_media_evidence_scans_complete_cache_file() { + let mut file = tempfile::NamedTempFile::new().expect("temporary cache file"); + std::io::Write::write_all(&mut file, TERMINAL_ASSET_BYTES).expect("write valid TS"); + let source_size = file.as_file().metadata().expect("cache metadata").len(); + let terminal_asset = asset(); + let probe = HlsTerminalMediaProbe { + segment_path: file.path().to_path_buf(), + source_size, + expected_duration_ticks_90khz: terminal_asset.duration_ticks_90khz(), + encryption: HlsTerminalTrackEncryption::Clear, + }; + + let evidence = terminal_base_media_evidence(probe).await; + let expected_signature = terminal_asset.track_signature().clone(); + + assert_eq!(evidence.track_resolution.signature(), Some(&expected_signature)); + assert_eq!(evidence.track_resolution.reason_code(), "found"); + assert_eq!(evidence.timestamp_profile, terminal_asset.timestamp_profile()); + assert!(matches!(evidence.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); +} + +#[tokio::test] +async fn terminal_base_timestamp_profile_uses_exact_pinned_last_safe_segment() { + let temp_dir = tempfile::tempdir().expect("terminal timing cache tempdir"); + let cache = HlsSegmentCache::with_cache_path(temp_dir.path()); + let terminal_asset = asset(); + let renderer = terminal_asset.renderer(); + let first_bytes = renderer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 0, + continuity_seed: 0, + logical_segment_index: 0, + }) + .expect("first cached segment"); + let last_offset = 9_000_000; + let last_bytes = renderer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: last_offset, + continuity_seed: 0, + logical_segment_index: 1, + }) + .expect("last cached segment"); + let expected_profile = TransportStreamBuffer::new(last_bytes.to_vec()) + .finite_hls_timestamp_profile() + .expect("last segment timestamp profile"); + let (session, base, last_cache_key, last_proxy_seq) = + terminal_timing_session(&cache, &terminal_asset, &first_bytes, &last_bytes).await; + + let evidence = prepare_terminal_base_evidence(&session, &cache, &base, 3).await; + let timing = evidence.timing().expect("exact last-base timing evidence"); + + assert_eq!(timing.base.proxy_seq, last_proxy_seq); + assert_eq!(timing.base.origin_epoch, 7); + assert_eq!(timing.base.cache_key, last_cache_key); + assert_eq!(timing.profile, expected_profile); + assert_ne!( + timing.profile, + TransportStreamBuffer::new(first_bytes.to_vec()) + .finite_hls_timestamp_profile() + .expect("first segment timestamp profile") + ); +} + +#[tokio::test] +async fn terminal_base_preserves_invalid_iv_and_missing_key_evidence() { + let invalid_iv = unresolved_aes_terminal_base_evidence("not-an-iv").await; + assert!(invalid_iv.track_base().is_some()); + assert_eq!(invalid_iv.track_resolution(), Some(&HlsTrackEvidenceResolution::InvalidIv)); + assert_eq!(invalid_iv.track_evidence_reason_code(), "invalid-iv"); + + let missing_key = unresolved_aes_terminal_base_evidence("0x00000000000000000000000000000384").await; + assert!(missing_key.track_base().is_some()); + assert_eq!(missing_key.track_resolution(), Some(&HlsTrackEvidenceResolution::KeyUnavailable)); + assert_eq!(missing_key.track_evidence_reason_code(), "key-unavailable"); +} + +#[test] +fn terminal_tail_plan_serves_preanchored_bytes_without_request_time_rendering() { + let asset = asset(); + let renderer = asset.renderer(); + let first_input = build_input(4, manifest(), Arc::clone(&asset)); + let shared_bundle = Arc::clone(&first_input.anchored_bundle); + let mut second_input = build_input(5, manifest(), asset); + second_input.anchored_bundle = Arc::clone(&shared_bundle); + let plan = build_terminal_tail_plan(first_input).expect("compatible tail"); + let second_plan = build_terminal_tail_plan(second_input).expect("second compatible tail"); + let render_count_after_plan_commit = renderer.finite_hls_render_count(); + let finalize_count_after_plan_commit = renderer.finite_hls_finalize_count(); + assert!(Arc::ptr_eq(&plan.anchored_bundle, &second_plan.anchored_bundle)); + let first_path = HlsTerminalSegmentPath { generation: plan.generation, index: 0 }; + let second_path = HlsTerminalSegmentPath { generation: plan.generation, index: 1 }; + + let first = plan.segment_bytes(first_path).expect("first prepared segment exists"); + assert_eq!(plan.segment_content_length(first_path), u64::try_from(first.len()).ok()); + let first_again = plan.segment_bytes(first_path).expect("same prepared segment exists"); + assert_eq!(first, first_again); + assert_eq!(first.as_ptr(), first_again.as_ptr(), "Bytes clones share the prepared allocation"); + assert_ne!(plan.segment_bytes(second_path), Some(first)); + assert_eq!( + plan.segment_bytes(HlsTerminalSegmentPath { + generation: HlsTerminalTailGeneration(plan.generation.0.saturating_add(1)), + index: 0, + }), + None + ); + assert_eq!( + plan.segment_bytes(HlsTerminalSegmentPath { generation: plan.generation, index: plan.segment_count }), + None + ); + assert_eq!(renderer.finite_hls_render_count(), render_count_after_plan_commit); + assert_eq!(renderer.finite_hls_finalize_count(), finalize_count_after_plan_commit); +} + +#[test] +fn hls_terminal_response_rejects_bundle_for_a_different_target_duration() { + let asset = asset(); + let mut input = build_input(4, manifest(), Arc::clone(&asset)); + input.anchored_bundle = HlsTerminalTailBuildInput::anchored_bundle_for_test(&asset, 13_000); + + assert_eq!(build_terminal_tail_plan(input), Err(HlsTerminalTailCompatibility::AssetRevisionMismatch)); +} + +#[test] +fn terminal_plan_rejects_unbound_and_mixed_lease_routes() { + let mut unbound = manifest(); + for segment in Arc::make_mut(&mut unbound.visible_segments) { + segment.uri = format!("/live/{}.ts", segment.proxy_seq).into(); + } + assert_eq!( + build_terminal_tail_plan(build_input(4, unbound, asset())), + Err(HlsTerminalTailCompatibility::InvalidLeaseRoute) + ); + + let mut mixed = manifest(); + Arc::make_mut(&mut mixed.visible_segments)[1].uri = Arc::from("/iptv/hls/shared/live/session/other-lease/194.ts"); + assert_eq!( + build_terminal_tail_plan(build_input(4, mixed, asset())), + Err(HlsTerminalTailCompatibility::InvalidLeaseRoute) + ); +} + +#[test] +fn active_fmp4_map_rejects_mpeg_ts_splice() { + let mut base = manifest(); + base.container = HlsMediaContainer::FragmentedMp4; + base.active_map = Some(HlsMapSignature { fingerprint: [2; 32], container: HlsMediaContainer::FragmentedMp4 }); + let asset = asset(); + + assert_eq!( + compatibility(&base, &asset, Some(asset.track_signature())), + HlsTerminalTailCompatibility::ActiveMapRequiresCompatibleFallback + ); +} + +#[test] +fn mapless_non_ts_container_rejects_mpeg_ts_splice() { + let mut base = manifest(); + base.container = HlsMediaContainer::Unknown; + base.active_map = None; + let asset = asset(); + + assert_eq!( + compatibility(&base, &asset, Some(asset.track_signature())), + HlsTerminalTailCompatibility::ContainerMismatch + ); +} + +#[test] +fn terminal_plan_keeps_only_contiguous_ready_protectable_suffix() { + let asset = asset(); + let base = manifest(); + let mut input = build_input(1, base, Arc::clone(&asset)); + Arc::make_mut(&mut input.base_availability)[0].media_state = HlsTerminalBaseMediaState::NotReady; + + let plan = build_terminal_tail_plan(input).expect("safe suffix"); + + assert_eq!(plan.base_manifest.first_proxy_seq, 194); + assert_eq!(plan.base_manifest.last_proxy_seq, 194); + assert_eq!(plan.protected_base_proxy_seqs.as_ref(), &[194]); + assert_eq!(plan.base_manifest.visible_segments.len(), 1); +} + +#[test] +fn unsafe_last_advertised_segment_rejects_terminal_plan() { + let mut input = build_input(1, manifest(), asset()); + Arc::make_mut(&mut input.base_availability)[1].protection = HlsTerminalBaseProtection::Unavailable; + + assert_eq!(build_terminal_tail_plan(input), Err(HlsTerminalTailCompatibility::MissingSafeBase)); +} + +#[test] +fn unsafe_transport_evidence_cannot_commit_or_expose_any_runtime_custom_tail() { + let unsafe_reason = super::super::super::HlsTsSpliceIncompatibility::IncompletePes { + pid: 0x101, + packet_index: 27, + declared_bytes: Some(512), + observed_bytes: 384, + }; + for reason in [ + HlsRuntimeCustomTailReason::ChannelUnavailable, + HlsRuntimeCustomTailReason::LowPriorityPreempted, + HlsRuntimeCustomTailReason::UserConnectionsExhausted, + HlsRuntimeCustomTailReason::ProviderConnectionsExhausted, + HlsRuntimeCustomTailReason::UserAccountExpired, + HlsRuntimeCustomTailReason::SessionOrLeaseExpired, + ] { + let asset = asset(); + let mut input = build_input(1, manifest(), Arc::clone(&asset)); + input.expected_asset = + HlsRuntimeCustomTailAssetIdentity { reason, media: HlsTerminalAssetIdentity::from_asset(&asset) }; + input.base_splice_evidence = Some(HlsTsSpliceEvidence::Incompatible(unsafe_reason)); + + assert_eq!( + build_terminal_tail_plan(input), + Err(HlsTerminalTailCompatibility::SpliceTransportFailure(unsafe_reason)), + "reason {reason} must use the common exact splice gate" + ); + } +} + +#[test] +fn terminal_plan_rejects_missing_or_topologically_different_exact_evidence() { + let asset = asset(); + let mut missing = build_input(1, manifest(), Arc::clone(&asset)); + missing.terminal_splice_evidence = None; + assert_eq!(build_terminal_tail_plan(missing), Err(HlsTerminalTailCompatibility::MissingSpliceEvidence)); + + let mut different = build_input(2, manifest(), asset); + different.terminal_splice_evidence = + Some(HlsTsSpliceEvidence::compatible_for_test(HlsTsTrackSignature::from_stream_types(Arc::<[u8]>::from([ + 0x1B, + ])))); + assert_eq!(build_terminal_tail_plan(different), Err(HlsTerminalTailCompatibility::SpliceTopologyMismatch)); +} + +#[test] +fn terminal_splice_diagnostic_preserves_typed_boundary_fields() { + let compatibility = HlsTerminalTailCompatibility::SpliceTransportFailure( + super::super::super::HlsTsSpliceIncompatibility::ContinuityFailure { + pid: 0x102, + packet_index: 44, + expected: 9, + actual: 12, + }, + ); + + assert_eq!( + HlsTerminalSpliceDiagnostic::from_compatibility(compatibility), + Some(HlsTerminalSpliceDiagnostic { + result: "continuity-failure", + pid: Some(0x102), + packet_index: Some(44), + expected_cc: Some(9), + actual_cc: Some(12), + declared_pes_bytes: None, + observed_pes_bytes: None, + }) + ); + assert_eq!( + HlsTerminalSpliceDiagnostic::from_compatibility(HlsTerminalTailCompatibility::SpliceTopologyMismatch) + .map(|diagnostic| diagnostic.result), + Some("topology-mismatch") + ); +} + +#[test] +fn encrypted_base_without_ready_key_rejects_terminal_plan() { + let mut base = manifest(); + let encryption = resettable_aes128_encryption(); + base.active_encryption = Some(encryption.clone()); + Arc::make_mut(&mut base.visible_segments)[1].encryption = Some(encryption); + let mut input = build_input(1, base, asset()); + Arc::make_mut(&mut input.base_availability)[1].required_key_ready = false; + + assert_eq!(build_terminal_tail_plan(input), Err(HlsTerminalTailCompatibility::MissingSafeBase)); +} + +#[test] +fn unknown_or_different_track_layout_rejects_splice() { + let asset = asset(); + assert_eq!(compatibility(&manifest(), &asset, None), HlsTerminalTailCompatibility::MissingTrackSignature); + let different = HlsTsTrackSignature::from_stream_types(Arc::<[u8]>::from([0x1B])); + assert_eq!(compatibility(&manifest(), &asset, Some(&different)), HlsTerminalTailCompatibility::TrackLayoutMismatch); +} diff --git a/backend/hls/src/terminal_tail/tests/transport.rs b/backend/hls/src/terminal_tail/tests/transport.rs new file mode 100644 index 000000000..352115d82 --- /dev/null +++ b/backend/hls/src/terminal_tail/tests/transport.rs @@ -0,0 +1,47 @@ +use super::*; + +#[test] +fn encrypted_base_announces_clear_key_transition() { + let mut base = manifest(); + let encryption = resettable_aes128_encryption(); + base.active_encryption = Some(encryption.clone()); + Arc::make_mut(&mut base.visible_segments)[1].encryption = Some(encryption); + let mut input = build_input(1, base, asset()); + input.base_key_bindings = Arc::from([aes128_key_binding()]); + let plan = build_terminal_tail_plan(input).expect("clear transition"); + let resource_file = TransientResourceFile::parse("aes-key.key").expect("terminal key route"); + let binding = plan + .terminal_key_binding(&ProxySessionId("session".into()), &HlsAccessLeaseId("lease".into()), &resource_file) + .expect("exact lease route resolves frozen key binding"); + assert_eq!(binding.bytes().as_ref(), b"0123456789abcdef"); + assert!(plan + .terminal_key_binding( + &ProxySessionId("session".into()), + &HlsAccessLeaseId("other-lease".into()), + &resource_file, + ) + .is_none()); + assert!(plan + .terminal_key_binding( + &ProxySessionId("session".into()), + &HlsAccessLeaseId("lease".into()), + &TransientResourceFile::parse("aes-key.bin").expect("different extension route"), + ) + .is_none()); + let rendered = + terminal_tail_manifest_body(&plan, &ProxySessionId("session".into()), &HlsAccessLeaseId("lease".into())) + .expect("render"); + + assert!( + !rendered.contains("/iptv/hls/shared/live/session/lease/193.ts"), + "only the segment covered by the active key is safe" + ); + let key = rendered.find("#EXT-X-KEY:METHOD=AES-128").expect("base key tag"); + let base_segment = rendered.find("/iptv/hls/shared/live/session/lease/194.ts").expect("encrypted base segment"); + let reset = rendered.find("#EXT-X-KEY:METHOD=NONE").expect("clear reset"); + assert!(key < base_segment && base_segment < reset); + assert!(rendered + .contains("URI=\"/iptv/hls/shared/live/session/lease/r/aes-key.key\",IV=0x000000000000000000000000000000C2")); + assert!(rendered.contains("#EXT-X-VERSION:5\n")); + assert!(rendered.contains("#EXT-X-KEY:METHOD=NONE\n#EXT-X-DISCONTINUITY\n")); +} diff --git a/backend/hls/src/transient.rs b/backend/hls/src/transient.rs index 18ee34533..860cd4b36 100644 --- a/backend/hls/src/transient.rs +++ b/backend/hls/src/transient.rs @@ -11,21 +11,24 @@ use super::{ CacheAccessState, HlsAccessLeaseId, HlsSession, ProxySessionId, TransientObjectCacheKey, TransientResourceFile, }; use crate::transient_manifest::TransientRewriteCheckpoint; -use axum::http::StatusCode; -use base64::{engine::general_purpose, Engine as _}; use std::{ collections::{HashMap, HashSet}, fmt, sync::Arc, }; use tokio::sync::Notify; -use tuliprox_parser::hls::origin_manifest::{HlsManifestLifecycle, HlsManifestWindowPolicy, ParsedManifestSemantics}; +use tuliprox_parser::hls::origin_manifest::{HlsManifestLifecycle, ParsedManifestSemantics}; const TRANSIENT_RESOURCE_ID_LEN: usize = 16; + const DEFAULT_TRANSIENT_RESOURCE_TTL_MS: u64 = 300_000; + const MAX_FAILED_TRANSIENT_OBJECT_ENTRIES: usize = 256; + const TRANSIENT_RESOURCE_ID_KEY_CONTEXT: &str = "tuliprox:hls-cache:transient-resource-id-key:v1"; + const ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES: usize = 24; + const ESTIMATED_ARC_ALLOCATION_OVERHEAD_BYTES: usize = 16; pub(crate) const fn transient_object_expires_at(now_ms: u64, cache_duration_ms: u64) -> u64 { @@ -54,54 +57,10 @@ pub struct TransientResourceId(pub String); #[derive(Debug, Clone, Default, Eq, PartialEq)] pub struct HlsPublishedTransientResourceIds(Arc>); -impl HlsPublishedTransientResourceIds { - pub fn from_manifest_body(body: &str) -> Self { Self(Arc::new(extract_transient_resource_ids(body))) } - - fn from_shared(resource_ids: Arc>) -> Self { Self(resource_ids) } - - pub fn contains(&self, resource_id: &TransientResourceId) -> bool { self.0.contains(resource_id) } -} - /// Monotonic identity of one committed transient manifest body. #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd)] pub struct TransientManifestGeneration(u64); -impl TransientManifestGeneration { - #[cfg(test)] - pub(crate) const fn for_test(generation: u64) -> Self { Self(generation) } -} - -/// Exact lease incarnation that currently publishes one finalized transient manifest generation. -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub(crate) struct TransientManifestLeaseBinding { - pub(crate) lease_id: HlsAccessLeaseId, - pub(crate) lease_issued_at_ms: u64, - pub(crate) manifest_generation: TransientManifestGeneration, -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub(crate) struct HlsTransientManifestFootprint { - pub(crate) resources: usize, - pub(crate) resource_entries: usize, - pub(crate) rewritten_bytes: usize, - pub(crate) origin_uri_bytes: usize, - pub(crate) retained_finalized_generations: usize, - pub(crate) finalized_generation_memberships: usize, - pub(crate) estimated_metadata_bytes: usize, -} - -/// Rejects a transient commit before publication when its complete client representation is unavailable. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub(crate) enum HlsTransientManifestCommitError { - LocalRepresentationLimit(HlsManifestLimitViolation), - SnapshotUnavailable, - CommitGenerationExhausted, -} - -impl From for HlsTransientManifestCommitError { - fn from(violation: HlsManifestLimitViolation) -> Self { Self::LocalRepresentationLimit(violation) } -} - #[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] struct TransientManifestIdentity { body_hash: [u8; 32], @@ -146,101 +105,6 @@ struct TransientManifestFootprintCandidate<'a> { template: Option<&'a HlsTransientManifestTemplate>, } -pub(crate) struct RollingEventAppendContext { - pub(crate) suffix_offset: usize, - pub(crate) rewritten_prefix: Arc, - pub(crate) template: Arc, - pub(crate) checkpoint: TransientRewriteCheckpoint, -} - -impl TransientManifestLeaseBinding { - pub(crate) const fn new( - lease_id: HlsAccessLeaseId, - lease_issued_at_ms: u64, - manifest_generation: TransientManifestGeneration, - ) -> Self { - Self { lease_id, lease_issued_at_ms, manifest_generation } - } -} - -#[derive(Clone, Copy)] -enum TransientResourceValidityScope<'a> { - PublishedSession, - AccessLease { - lease_id: &'a HlsAccessLeaseId, - lease_issued_at_ms: u64, - published_resource_ids: &'a HlsPublishedTransientResourceIds, - }, -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum CommittedTransientManifestValidity { - Rolling { valid_until_ms: Option, window_policy: HlsManifestWindowPolicy }, - Finalized, -} - -impl CommittedTransientManifestValidity { - const fn from_semantics( - semantics: ParsedManifestSemantics, - rendered_at_ms: u64, - playlist_duration_ms: Option, - ) -> Self { - match semantics.lifecycle() { - HlsManifestLifecycle::Rolling => Self::Rolling { - valid_until_ms: match playlist_duration_ms { - Some(duration_ms) => Some(rendered_at_ms.saturating_add(duration_ms)), - None => None, - }, - window_policy: semantics.window_policy(), - }, - HlsManifestLifecycle::Finalized => Self::Finalized, - } - } - - const fn is_finalized(self) -> bool { matches!(self, Self::Finalized) } - - const fn window_policy(self) -> HlsManifestWindowPolicy { - match self { - Self::Rolling { window_policy, .. } => window_policy, - Self::Finalized => HlsManifestWindowPolicy::PreserveFullManifest, - } - } - - const fn valid_until_ms(self) -> Option { - match self { - Self::Rolling { valid_until_ms, .. } => valid_until_ms, - Self::Finalized => None, - } - } -} - -/// Builds a deterministic opaque transient resource ID from the concrete origin fetch URI. -/// -/// `resolved_origin_uri` must be the concrete resource URI after manifest-relative URL resolution against the final -/// manifest URL. In provider-url-failover/redirect flows this may intentionally include the selected mirror or final -/// CDN/origin host. Do not make this ID input host-neutral unless `TransientResourceRef` keeps a separate concrete fetch -/// URI and tests prove relative segment/MAP/key downloads still use that concrete URI. -/// -pub fn build_transient_resource_id( - resolved_origin_uri: &str, - reverse_proxy_rewrite_secret: &[u8], -) -> TransientResourceId { - let key = blake3::derive_key(TRANSIENT_RESOURCE_ID_KEY_CONTEXT, reverse_proxy_rewrite_secret); - let digest = blake3::keyed_hash(&key, resolved_origin_uri.as_bytes()); - let token = general_purpose::URL_SAFE_NO_PAD.encode(digest.as_bytes()); - TransientResourceId(token.chars().take(TRANSIENT_RESOURCE_ID_LEN).collect()) -} - -/// Transient origin resource category used for direct passthrough streaming. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum TransientResourceKind { - Segment, - Key, - Map, - Part, - Other, -} - /// Per-session mapping from an opaque transient ID to one resolved origin resource. #[derive(Clone, Eq, PartialEq)] pub struct TransientResourceRef { @@ -271,22 +135,6 @@ struct TransientObjectResourceBinding { file_extension: String, } -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum TransientObjectCacheStatus { - Fetching { started_at_ms: u64 }, - Ready { content_length: u64, ready_at_ms: u64 }, - FailedRetryable { failed_at_ms: u64, retry_after_ms: u64 }, - FailedPermanent { failed_at_ms: u64, status: Option }, -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum TransientObjectUnavailableState { - Missing, - Fetching, - FailedRetryable { retry_after_ms: u64 }, - FailedPermanent, -} - #[derive(Clone, Eq, PartialEq)] pub struct TransientObjectCacheEntry { pub key: TransientObjectCacheKey, @@ -299,13 +147,6 @@ pub struct TransientObjectCacheEntry { binding: TransientObjectResourceBinding, } -#[derive(Clone)] -pub enum TransientObjectFetchDecision { - Ready, - Fetch(Box), - Wait(Arc), -} - /// Generation-safe ownership of one transient full-object cache fill. #[derive(Clone)] pub struct TransientObjectFetchToken { @@ -316,160 +157,6 @@ pub struct TransientObjectFetchToken { superseded_object: Option, } -impl TransientObjectFetchToken { - pub fn cache_key(&self) -> &TransientObjectCacheKey { &self.cache_key } - - pub fn superseded_object(&self) -> Option<&TransientObjectRemoval> { self.superseded_object.as_ref() } -} - -impl TransientObjectResourceBinding { - fn from_resource(resource: &TransientResourceRef, file_extension: &str) -> Self { - Self { - resource_id: resource.id.clone(), - revision: resource.revision, - kind: resource.kind, - encrypted_media: resource.encrypted_media, - resolved_origin_uri: resource.resolved_origin_uri.clone(), - file_extension: file_extension.to_string(), - } - } - - fn matches_resource_identity(&self, resource: &TransientResourceRef) -> bool { - self.resource_id == resource.id - && self.revision == resource.revision - && self.kind == resource.kind - && self.encrypted_media == resource.encrypted_media - && self.resolved_origin_uri == resource.resolved_origin_uri - && resource.file_ext_hint.as_deref().is_none_or(|extension| extension == self.file_extension) - } -} - -impl TransientObjectCacheEntry { - fn new_fetching( - key: TransientObjectCacheKey, - binding: TransientObjectResourceBinding, - now_ms: u64, - expires_at_ms: u64, - content_type: String, - ) -> Self { - Self { - key, - status: TransientObjectCacheStatus::Fetching { started_at_ms: now_ms }, - content_type, - created_at_ms: now_ms, - last_accessed_at_ms: now_ms, - expires_at_ms, - access: Arc::new(CacheAccessState::new()), - binding, - } - } - - pub fn is_ready_at(&self, now_ms: u64) -> bool { - matches!(self.status, TransientObjectCacheStatus::Ready { .. }) && self.expires_at_ms >= now_ms - } - - pub fn ready_content_length(&self) -> Option { - match self.status { - TransientObjectCacheStatus::Ready { content_length, .. } => Some(content_length), - TransientObjectCacheStatus::Fetching { .. } - | TransientObjectCacheStatus::FailedRetryable { .. } - | TransientObjectCacheStatus::FailedPermanent { .. } => None, - } - } -} - -impl fmt::Debug for TransientObjectCacheEntry { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("TransientObjectCacheEntry") - .field("key", &self.key) - .field("status", &self.status) - .field("content_type", &self.content_type) - .field("created_at_ms", &self.created_at_ms) - .field("last_accessed_at_ms", &self.last_accessed_at_ms) - .field("expires_at_ms", &self.expires_at_ms) - .field("active_readers", &self.access.active_readers()) - .field("resource_revision", &self.binding.revision) - .finish() - } -} - -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct TransientObjectRemoval { - pub key: TransientObjectCacheKey, - pub content_length: u64, -} - -impl TransientResourceRef { - pub fn new( - kind: TransientResourceKind, - resolved_origin_uri: impl Into, - reverse_proxy_rewrite_secret: &[u8], - now_ms: u64, - ttl_ms: u64, - file_ext_hint: Option, - ) -> Self { - let resolved_origin_uri = resolved_origin_uri.into(); - let id = build_transient_resource_id(&resolved_origin_uri, reverse_proxy_rewrite_secret); - Self { - id, - kind, - encrypted_media: false, - resolved_origin_uri, - content_type_hint: file_ext_hint - .as_deref() - .and_then(default_content_type_for_transient_ext) - .map(str::to_string), - file_ext_hint, - created_at_ms: now_ms, - expires_at_ms: now_ms.saturating_add(ttl_ms), - access: Arc::new(CacheAccessState::new()), - revision: TransientResourceRevision::default(), - } - } - - pub fn is_valid_at(&self, now_ms: u64) -> bool { now_ms <= self.expires_at_ms } - - fn refresh_from(&mut self, next: TransientResourceRef) { - self.kind = next.kind; - // A URI reused across key transitions is ambiguous while an older cache object can - // still exist. Preserve the conservative encrypted classification until eviction. - self.encrypted_media |= next.encrypted_media; - self.resolved_origin_uri = next.resolved_origin_uri; - self.content_type_hint = next.content_type_hint; - self.file_ext_hint = next.file_ext_hint; - self.expires_at_ms = next.expires_at_ms; - self.revision = next.revision; - } - - fn has_same_cache_identity(&self, other: &Self) -> bool { - self.id == other.id - && self.kind == other.kind - && self.encrypted_media == (self.encrypted_media || other.encrypted_media) - && self.resolved_origin_uri == other.resolved_origin_uri - && self.content_type_hint == other.content_type_hint - && self.file_ext_hint == other.file_ext_hint - } - - pub fn active_readers(&self) -> u32 { self.access.active_readers() } -} - -impl fmt::Debug for TransientResourceRef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("TransientResourceRef") - .field("id", &self.id) - .field("kind", &self.kind) - .field("encrypted_media", &self.encrypted_media) - .field("resolved_origin_uri", &"") - .field("content_type_hint", &self.content_type_hint) - .field("file_ext_hint", &self.file_ext_hint) - .field("created_at_ms", &self.created_at_ms) - .field("expires_at_ms", &self.expires_at_ms) - .field("active_readers", &self.access.active_readers()) - .field("revision", &self.revision) - .finish() - } -} - /// Per-session transient passthrough manifest and resource mappings. #[derive(Clone)] pub struct TransientPassthroughState { @@ -495,1223 +182,13 @@ pub struct TransientPassthroughState { } impl TransientPassthroughState { - pub fn new(resource_ttl_ms: u64) -> Self { - Self { - resources: HashMap::new(), - object_cache: HashMap::new(), - object_fetch_notifiers: HashMap::new(), - next_resource_revision: 1, - next_object_fetch_generation: 0, - manifest_generation: 0, - last_manifest_commit_identity: None, - current_finalized_manifest_generation: None, - last_manifest_validity: CommittedTransientManifestValidity::Rolling { - valid_until_ms: None, - window_policy: HlsManifestWindowPolicy::ApplyLiveWindow, - }, - last_manifest_resource_ids: Arc::new(HashSet::new()), - last_manifest_template: None, - rolling_event_rewrite_seed: None, - finalized_manifest_generations: HashMap::new(), - finalized_manifest_lease_bindings: HashSet::new(), - protected_finalized_resource_refcounts: HashMap::new(), - last_manifest_body: None, - last_manifest_rendered_at_ms: None, - last_manifest_playlist_duration_ms: None, - resource_ttl_ms, - } - } - - pub fn set_resource_ttl_ms(&mut self, resource_ttl_ms: u64) { self.resource_ttl_ms = resource_ttl_ms; } - - #[cfg(any(test, feature = "test-support"))] - pub fn replace_manifest_with_semantics( - &mut self, - body: String, - rendered_at_ms: u64, - playlist_duration_ms: Option, - ) { - let semantics = tuliprox_parser::hls::origin_manifest::parse_manifest_semantics(&body); - if self.replace_manifest_common(body, rendered_at_ms, playlist_duration_ms, semantics).is_ok() { - self.last_manifest_commit_identity = Some(super::media_reserve::HlsManifestCommitIdentity::committed( - self.manifest_generation, - rendered_at_ms, - )); - } - } - - pub(crate) fn commit_rewritten_manifest_with_semantics( - &mut self, - body: String, - resources: Vec, - rendered_at_ms: u64, - playlist_duration_ms: Option, - semantics: ParsedManifestSemantics, - ) -> Result { - let manifest_resource_ids = Arc::new(extract_transient_resource_ids(&body)); - let manifest_identity = transient_manifest_identity(&body, &manifest_resource_ids); - let reusable_finalized_generation = semantics - .lifecycle() - .is_finalized() - .then(|| self.finalized_generation_for_identity(manifest_identity)) - .flatten(); - self.ensure_manifest_generation_available(reusable_finalized_generation)?; - let resource_count = resources.len().max(manifest_resource_ids.len()); - Self::validate_manifest_shape(body.len(), resource_count)?; - let template = if let Some(generation) = reusable_finalized_generation { - self.finalized_manifest_generations.get(&generation).and_then(|retained| retained.template.clone()) - } else { - parse_hls_transient_manifest_template(&body)? - } - .ok_or(HlsTransientManifestCommitError::SnapshotUnavailable)?; - let footprint = self.validate_manifest_footprint(&TransientManifestFootprintCandidate { - rewritten_bytes: body.len(), - resource_count, - resources: &resources, - lifecycle: semantics.lifecycle(), - resource_ids: &manifest_resource_ids, - reusable_finalized_generation, - template: Some(&template), - })?; - self.upsert_resources(resources); - self.replace_manifest_common_with_resource_ids(TransientManifestReplacement { - body: Arc::from(body), - rendered_at_ms, - playlist_duration_ms, - semantics, - resource_ids: manifest_resource_ids, - identity: manifest_identity, - reusable_finalized_generation, - template: Some(template), - })?; - Ok(footprint) - } - - pub(crate) fn commit_incremental_rewritten_manifest_with_semantics( - &mut self, - body: String, - appended_resources: Vec, - template: Arc, - rendered_at_ms: u64, - resource_ttl_ms: u64, - semantics: ParsedManifestSemantics, - ) -> Result { - self.ensure_manifest_generation_available(None)?; - let mut manifest_resource_ids = std::mem::take(&mut self.last_manifest_resource_ids); - let mut inserted_resource_ids = Vec::new(); - for resource in &appended_resources { - if Arc::make_mut(&mut manifest_resource_ids).insert(resource.id.clone()) { - inserted_resource_ids.push(resource.id.clone()); - } - } - let manifest_identity = if semantics.lifecycle().is_finalized() { - transient_manifest_identity(&body, &manifest_resource_ids) - } else { - TransientManifestIdentity::default() - }; - let reusable_finalized_generation = semantics - .lifecycle() - .is_finalized() - .then(|| self.finalized_generation_for_identity(manifest_identity)) - .flatten(); - let footprint = Self::validate_manifest_shape(body.len(), manifest_resource_ids.len()).and_then(|()| { - self.validate_manifest_footprint(&TransientManifestFootprintCandidate { - rewritten_bytes: body.len(), - resource_count: manifest_resource_ids.len(), - resources: &appended_resources, - lifecycle: semantics.lifecycle(), - resource_ids: &manifest_resource_ids, - reusable_finalized_generation, - template: Some(&template), - }) - }); - let footprint = match footprint { - Ok(footprint) => footprint, - Err(violation) => { - let resource_ids = Arc::make_mut(&mut manifest_resource_ids); - for resource_id in inserted_resource_ids { - resource_ids.remove(&resource_id); - } - self.last_manifest_resource_ids = manifest_resource_ids; - return Err(violation); - } - }; - self.upsert_resources(appended_resources); - let expires_at_ms = rendered_at_ms.saturating_add(resource_ttl_ms); - for resource_id in manifest_resource_ids.iter() { - if let Some(resource) = self.resources.get_mut(resource_id) { - resource.expires_at_ms = expires_at_ms; - } - } - let playlist_duration_ms = Some(template.playlist_duration_ms()); - self.replace_manifest_common_with_resource_ids(TransientManifestReplacement { - body: Arc::from(body), - rendered_at_ms, - playlist_duration_ms, - semantics, - resource_ids: manifest_resource_ids, - identity: manifest_identity, - reusable_finalized_generation, - template: Some(template), - })?; - Ok(footprint) - } - - pub(crate) fn rolling_event_append_context( - &self, - origin_body: &str, - final_manifest_url: &str, - rewrite_secret: &[u8], - ) -> Option { - let seed = self.rolling_event_rewrite_seed.as_ref()?; - if seed.final_manifest_url_hash != *blake3::hash(final_manifest_url.as_bytes()).as_bytes() - || seed.rewrite_secret_hash != *blake3::hash(rewrite_secret).as_bytes() - || !complete_manifest_append_boundary(&seed.origin_body) - || origin_body.len() <= seed.origin_body.len() - || !origin_body.starts_with(seed.origin_body.as_ref()) - { - return None; - } - Some(RollingEventAppendContext { - suffix_offset: seed.origin_body.len(), - rewritten_prefix: Arc::clone(&seed.rewritten_body), - template: Arc::clone(&seed.template), - checkpoint: seed.checkpoint.clone(), - }) - } - - pub(crate) fn extend_rolling_event_template( - previous: &Arc, - rewritten_suffix: &str, - ) -> Result>, HlsManifestLimitViolation> { - extend_hls_transient_manifest_template(previous, rewritten_suffix) - } - - pub(crate) fn record_rolling_event_rewrite_seed( - &mut self, - origin_body: &str, - final_manifest_url: &str, - rewrite_secret: &[u8], - checkpoint: TransientRewriteCheckpoint, - retained_metadata_bytes: usize, - eligible: bool, - ) { - let retained_with_origin_body = retained_metadata_bytes - .saturating_add(origin_body.len()) - .saturating_add(std::mem::size_of::()) - .saturating_add(ESTIMATED_ARC_ALLOCATION_OVERHEAD_BYTES); - if !eligible || retained_with_origin_body > MAX_ESTIMATED_TRANSIENT_METADATA_BYTES { - self.rolling_event_rewrite_seed = None; - return; - } - let (Some(rewritten_body), Some(template)) = (&self.last_manifest_body, &self.last_manifest_template) else { - self.rolling_event_rewrite_seed = None; - return; - }; - self.rolling_event_rewrite_seed = Some(RollingEventRewriteSeed { - origin_body: Arc::from(origin_body), - final_manifest_url_hash: *blake3::hash(final_manifest_url.as_bytes()).as_bytes(), - rewrite_secret_hash: *blake3::hash(rewrite_secret).as_bytes(), - rewritten_body: Arc::clone(rewritten_body), - template: Arc::clone(template), - checkpoint, - }); - } - - #[cfg(any(test, feature = "test-support"))] - fn replace_manifest_common( - &mut self, - body: String, - rendered_at_ms: u64, - playlist_duration_ms: Option, - semantics: ParsedManifestSemantics, - ) -> Result<(), HlsManifestLimitViolation> { - let manifest_resource_ids = Arc::new(extract_transient_resource_ids(&body)); - let manifest_identity = transient_manifest_identity(&body, &manifest_resource_ids); - let reusable_finalized_generation = semantics - .lifecycle() - .is_finalized() - .then(|| self.finalized_generation_for_identity(manifest_identity)) - .flatten(); - let template = parse_hls_transient_manifest_template(&body).ok().flatten(); - self.replace_manifest_common_with_resource_ids(TransientManifestReplacement { - body: Arc::from(body), - rendered_at_ms, - playlist_duration_ms, - semantics, - resource_ids: manifest_resource_ids, - identity: manifest_identity, - reusable_finalized_generation, - template, - }) - } - - fn replace_manifest_common_with_resource_ids( - &mut self, - replacement: TransientManifestReplacement, - ) -> Result<(), HlsManifestLimitViolation> { - let TransientManifestReplacement { - body, - rendered_at_ms, - playlist_duration_ms, - semantics, - resource_ids, - identity, - reusable_finalized_generation, - template, - } = replacement; - self.ensure_manifest_generation_available(reusable_finalized_generation)?; - let manifest_validity = - CommittedTransientManifestValidity::from_semantics(semantics, rendered_at_ms, playlist_duration_ms); - let manifest_generation = if let Some(generation) = reusable_finalized_generation { - generation - } else { - self.manifest_generation = self.manifest_generation.saturating_add(1); - TransientManifestGeneration(self.manifest_generation) - }; - if reusable_finalized_generation.is_none() { - self.last_manifest_resource_ids = Arc::clone(&resource_ids); - self.last_manifest_template.clone_from(&template); - } - self.last_manifest_validity = manifest_validity; - if manifest_validity.is_finalized() { - self.current_finalized_manifest_generation = Some(manifest_generation); - if reusable_finalized_generation.is_none() { - self.insert_finalized_manifest_generation( - manifest_generation, - RetainedFinalizedManifestGeneration { identity, resource_ids: Arc::clone(&resource_ids), template }, - ); - } - } else { - self.current_finalized_manifest_generation = None; - } - self.prune_unreferenced_finalized_manifest_generations(); - if reusable_finalized_generation.is_none() { - self.last_manifest_body = Some(body); - } - self.last_manifest_rendered_at_ms = Some(rendered_at_ms); - self.last_manifest_playlist_duration_ms = playlist_duration_ms; - Ok(()) - } - - fn validate_manifest_footprint( - &self, - candidate: &TransientManifestFootprintCandidate<'_>, - ) -> Result { - let origin_uri_bytes = self.prospective_origin_uri_bytes(candidate.resources); - ensure_manifest_limit( - HlsManifestLimitKind::TransientOriginUriBytes, - origin_uri_bytes, - MAX_TRANSIENT_ORIGIN_URI_BYTES_PER_SESSION, - )?; - let resource_entries = self.prospective_resource_entry_count(candidate.resources); - ensure_manifest_limit( - HlsManifestLimitKind::TransientResourceEntries, - resource_entries, - MAX_TRANSIENT_RESOURCE_ENTRIES_PER_SESSION, - )?; - let prospective_generation_sets = self.prospective_finalized_generation_sets( - candidate.lifecycle, - candidate.resource_ids, - candidate.reusable_finalized_generation, - ); - let retained_finalized_generations = prospective_generation_sets.len(); - ensure_manifest_limit( - HlsManifestLimitKind::FinalizedGenerations, - retained_finalized_generations, - MAX_RETAINED_FINALIZED_MANIFEST_GENERATIONS, - )?; - let finalized_generation_memberships = prospective_generation_sets - .iter() - .fold(0_usize, |total, resource_ids| total.saturating_add(resource_ids.len())); - ensure_manifest_limit( - HlsManifestLimitKind::TransientGenerationMemberships, - finalized_generation_memberships, - MAX_TRANSIENT_GENERATION_MEMBERSHIPS, - )?; - let estimated_metadata_bytes = - self.prospective_estimated_metadata_bytes(candidate, &prospective_generation_sets); - ensure_manifest_limit( - HlsManifestLimitKind::TransientEstimatedMetadataBytes, - estimated_metadata_bytes, - MAX_ESTIMATED_TRANSIENT_METADATA_BYTES, - )?; - Ok(HlsTransientManifestFootprint { - resources: candidate.resource_count, - resource_entries, - rewritten_bytes: candidate.rewritten_bytes, - origin_uri_bytes, - retained_finalized_generations, - finalized_generation_memberships, - estimated_metadata_bytes, - }) - } - - fn validate_manifest_shape(rewritten_bytes: usize, resource_count: usize) -> Result<(), HlsManifestLimitViolation> { - ensure_manifest_limit( - HlsManifestLimitKind::TransientResources, - resource_count, - MAX_TRANSIENT_MANIFEST_RESOURCES, - )?; - ensure_manifest_limit( - HlsManifestLimitKind::TransientRewrittenBytes, - rewritten_bytes, - MAX_TRANSIENT_REWRITTEN_MANIFEST_BYTES, - ) - } - - fn ensure_manifest_generation_available( - &self, - reusable_generation: Option, - ) -> Result<(), HlsManifestLimitViolation> { - if reusable_generation.is_none() && self.manifest_generation == u64::MAX { - return Err(HlsManifestLimitViolation::new( - HlsManifestLimitKind::ManifestCommitGeneration, - usize::MAX, - usize::MAX - 1, - )); - } - Ok(()) - } - - fn prospective_origin_uri_bytes(&self, resources: &[TransientResourceRef]) -> usize { - let incoming_uri_bytes = resources - .iter() - .map(|resource| (&resource.id, resource.resolved_origin_uri.len())) - .collect::>(); - let retained_uri_bytes = self - .resources - .iter() - .filter(|(resource_id, _)| !incoming_uri_bytes.contains_key(resource_id)) - .fold(0_usize, |total, (_, resource)| total.saturating_add(resource.resolved_origin_uri.len())); - incoming_uri_bytes.values().fold(retained_uri_bytes, |total, uri_bytes| total.saturating_add(*uri_bytes)) - } - - fn prospective_resource_entry_count(&self, resources: &[TransientResourceRef]) -> usize { - let incoming_ids = resources.iter().map(|resource| &resource.id).collect::>(); - incoming_ids.iter().fold(self.resources.len(), |count, resource_id| { - count.saturating_add(usize::from(!self.resources.contains_key(*resource_id))) - }) - } - - fn prospective_finalized_generation_sets<'a>( - &'a self, - lifecycle: HlsManifestLifecycle, - candidate_resource_ids: &'a HashSet, - reusable_generation: Option, - ) -> Vec<&'a HashSet> { - let mut sets = self - .finalized_manifest_generations - .iter() - .filter(|(generation, _)| { - Some(**generation) == reusable_generation || self.generation_is_lease_referenced(**generation) - }) - .map(|(_, generation)| generation.resource_ids.as_ref()) - .collect::>(); - if lifecycle.is_finalized() && reusable_generation.is_none() { - sets.push(candidate_resource_ids); - } - sets - } - - fn prospective_estimated_metadata_bytes( - &self, - candidate: &TransientManifestFootprintCandidate<'_>, - generation_sets: &[&HashSet], - ) -> usize { - let incoming = candidate.resources.iter().map(|resource| (&resource.id, resource)).collect::>(); - let retained_resource_bytes = self - .resources - .iter() - .filter(|(resource_id, _)| !incoming.contains_key(resource_id)) - .fold(0_usize, |total, (resource_id, resource)| { - total.saturating_add(estimated_transient_resource_entry_bytes(resource_id, resource)) - }); - let resource_bytes = incoming.values().fold(retained_resource_bytes, |total, resource| { - total.saturating_add(estimated_transient_resource_entry_bytes(&resource.id, resource)) - }); - let generation_bytes = generation_sets.iter().fold(0_usize, |total, resource_ids| { - total - .saturating_add(std::mem::size_of::()) - .saturating_add(ESTIMATED_ARC_ALLOCATION_OVERHEAD_BYTES) - .saturating_add(estimated_transient_resource_id_set_bytes(resource_ids)) - }); - let rolling_manifest_resource_bytes = if candidate.lifecycle.is_finalized() { - 0 - } else { - estimated_transient_resource_id_set_bytes(candidate.resource_ids) - }; - let retained_template_bytes = self - .finalized_manifest_generations - .iter() - .filter(|(generation, _)| { - Some(**generation) == candidate.reusable_finalized_generation - || self.generation_is_lease_referenced(**generation) - }) - .filter_map(|(_, generation)| generation.template.as_ref()) - .fold(0_usize, |total, template| total.saturating_add(template.estimated_metadata_bytes())); - let candidate_template_bytes = - if candidate.lifecycle.is_finalized() && candidate.reusable_finalized_generation.is_some() { - 0 - } else { - candidate.template.map_or(0, HlsTransientManifestTemplate::estimated_metadata_bytes) - }; - let mut protected_ids = HashSet::new(); - for resource_ids in generation_sets { - protected_ids.extend(resource_ids.iter()); - } - let protected_index_bytes = protected_ids.iter().fold(0_usize, |total, resource_id| { - total.saturating_add( - std::mem::size_of::<(TransientResourceId, u16)>() - .saturating_add(resource_id.0.len()) - .saturating_add(ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES), - ) - }); - let binding_bytes = self.finalized_manifest_lease_bindings.iter().fold(0_usize, |total, binding| { - total.saturating_add( - std::mem::size_of::() - .saturating_add(binding.lease_id.0.len()) - .saturating_add(ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES), - ) - }); - let object_bytes = self - .object_cache - .values() - .fold(0_usize, |total, entry| total.saturating_add(estimated_transient_object_entry_bytes(entry))); - candidate - .rewritten_bytes - .saturating_add(resource_bytes) - .saturating_add(generation_bytes) - .saturating_add(rolling_manifest_resource_bytes) - .saturating_add(retained_template_bytes) - .saturating_add(candidate_template_bytes) - .saturating_add(protected_index_bytes) - .saturating_add(binding_bytes) - .saturating_add(object_bytes) - .saturating_add( - self.object_fetch_notifiers.len().saturating_mul( - std::mem::size_of::<(TransientObjectCacheKey, Arc)>() - .saturating_add(ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES), - ), - ) - } - - #[cfg(test)] - pub(crate) const fn manifest_generation(&self) -> u64 { self.manifest_generation } - - pub const fn current_finalized_manifest_generation(&self) -> Option { - self.current_finalized_manifest_generation - } - - pub const fn last_manifest_commit_identity(&self) -> Option { - self.last_manifest_commit_identity - } - - pub(crate) fn record_manifest_commit_identity( - &mut self, - identity: super::media_reserve::HlsManifestCommitIdentity, - ) { - self.last_manifest_commit_identity = Some(identity); - } - - pub(crate) const fn last_manifest_finalized(&self) -> bool { self.last_manifest_validity.is_finalized() } - - pub fn last_manifest_template(&self) -> Option> { - self.last_manifest_template.clone() - } - - pub fn last_manifest_published_resource_ids(&self) -> HlsPublishedTransientResourceIds { - HlsPublishedTransientResourceIds::from_shared(Arc::clone(&self.last_manifest_resource_ids)) - } - - pub(crate) fn merge_current_published_resource_ids( - &self, - previous: &HlsPublishedTransientResourceIds, - next: HlsPublishedTransientResourceIds, - now_ms: u64, - ) -> HlsPublishedTransientResourceIds { - if Arc::ptr_eq(&previous.0, &next.0) - || previous.0.iter().all(|resource_id| { - next.contains(resource_id) - || !self - .resources - .get(resource_id) - .is_some_and(|resource| self.resource_is_valid_at(resource, now_ms)) - }) - { - return next; - } - let mut merged = next.0.as_ref().clone(); - merged.extend( - previous - .0 - .iter() - .filter(|resource_id| { - self.resources.get(*resource_id).is_some_and(|resource| self.resource_is_valid_at(resource, now_ms)) - }) - .cloned(), - ); - HlsPublishedTransientResourceIds(Arc::new(merged)) - } - - pub const fn last_manifest_window_policy(&self) -> HlsManifestWindowPolicy { - self.last_manifest_validity.window_policy() - } - - pub(crate) const fn last_manifest_valid_until_ms(&self) -> Option { - self.last_manifest_validity.valid_until_ms() - } - - #[cfg(test)] - pub(crate) fn current_manifest_resource_ids(&self) -> &HashSet { - self.last_manifest_resource_ids.as_ref() - } - - pub(crate) fn has_finalized_manifest_generation(&self, generation: TransientManifestGeneration) -> bool { - self.finalized_manifest_generations.contains_key(&generation) - } - - #[cfg(test)] - pub(crate) fn finalized_manifest_generation_count(&self) -> usize { self.finalized_manifest_generations.len() } - - #[cfg(test)] - pub(crate) fn finalized_manifest_lease_binding_count(&self) -> usize { - self.finalized_manifest_lease_bindings.len() - } - - pub(crate) fn bind_finalized_manifest_generation(&mut self, binding: TransientManifestLeaseBinding) -> bool { - if !self.finalized_manifest_generations.contains_key(&binding.manifest_generation) { - return false; - } - self.finalized_manifest_lease_bindings.insert(binding); - self.prune_unreferenced_finalized_manifest_generations(); - true - } - - pub(crate) fn release_finalized_manifest_generations( - &mut self, - lease_id: &HlsAccessLeaseId, - lease_issued_at_ms: u64, - ) -> bool { - let previous_len = self.finalized_manifest_lease_bindings.len(); - self.finalized_manifest_lease_bindings - .retain(|binding| binding.lease_id != *lease_id || binding.lease_issued_at_ms != lease_issued_at_ms); - let released = self.finalized_manifest_lease_bindings.len() != previous_len; - if !released { - return false; - } - self.prune_unreferenced_finalized_manifest_generations(); - true - } - - pub(crate) fn reconcile_finalized_manifest_lease_bindings(&mut self, bindings: &[TransientManifestLeaseBinding]) { - self.finalized_manifest_lease_bindings = bindings - .iter() - .filter(|binding| self.finalized_manifest_generations.contains_key(&binding.manifest_generation)) - .cloned() - .collect(); - self.prune_unreferenced_finalized_manifest_generations(); - } - - fn prune_unreferenced_finalized_manifest_generations(&mut self) { - let current_generation = self.current_finalized_manifest_generation(); - let referenced_generations = self - .finalized_manifest_lease_bindings - .iter() - .map(|binding| binding.manifest_generation) - .collect::>(); - let removed = self - .finalized_manifest_generations - .extract_if(|generation, _| { - Some(*generation) != current_generation && !referenced_generations.contains(generation) - }) - .map(|(_, generation)| generation) - .collect::>(); - for generation in removed { - self.decrement_protected_resource_refcounts(&generation.resource_ids); - } - } - - fn finalized_generation_for_identity( - &self, - identity: TransientManifestIdentity, - ) -> Option { - let generation = self.current_finalized_manifest_generation?; - self.finalized_manifest_generations - .get(&generation) - .is_some_and(|retained| retained.identity == identity) - .then_some(generation) - } - - fn generation_is_lease_referenced(&self, generation: TransientManifestGeneration) -> bool { - self.finalized_manifest_lease_bindings.iter().any(|binding| binding.manifest_generation == generation) - } - - fn insert_finalized_manifest_generation( - &mut self, - generation: TransientManifestGeneration, - retained: RetainedFinalizedManifestGeneration, - ) { - let resource_ids = Arc::clone(&retained.resource_ids); - if let Some(previous) = self.finalized_manifest_generations.insert(generation, retained) { - self.decrement_protected_resource_refcounts(&previous.resource_ids); - } - for resource_id in resource_ids.iter() { - let refcount = self.protected_finalized_resource_refcounts.entry(resource_id.clone()).or_default(); - *refcount = refcount.saturating_add(1); - } - } - - fn decrement_protected_resource_refcounts(&mut self, resource_ids: &HashSet) { - for resource_id in resource_ids { - let remove = self.protected_finalized_resource_refcounts.get_mut(resource_id).is_some_and(|refcount| { - *refcount = refcount.saturating_sub(1); - *refcount == 0 - }); - if remove { - self.protected_finalized_resource_refcounts.remove(resource_id); - } - } - } - - pub fn upsert_resources(&mut self, resources: I) - where - I: IntoIterator, - { - for mut resource in resources { - let preserve_revision = self - .resources - .get(&resource.id) - .filter(|existing| existing.has_same_cache_identity(&resource)) - .map(|existing| existing.revision); - resource.revision = if let Some(revision) = preserve_revision { - revision - } else { - let revision = TransientResourceRevision(self.next_resource_revision); - self.next_resource_revision = self.next_resource_revision.saturating_add(1); - revision - }; - match self.resources.get_mut(&resource.id) { - Some(existing) => existing.refresh_from(resource), - None => { - let _previous = self.resources.insert(resource.id.clone(), resource); - } - } - } - } - - fn resource_is_valid_at_for( - &self, - resource: &TransientResourceRef, - now_ms: u64, - scope: TransientResourceValidityScope<'_>, - ) -> bool { - match scope { - TransientResourceValidityScope::PublishedSession => { - resource.is_valid_at(now_ms) || self.protected_finalized_resource_refcounts.contains_key(&resource.id) - } - TransientResourceValidityScope::AccessLease { lease_id, lease_issued_at_ms, published_resource_ids } => { - (published_resource_ids.contains(&resource.id) && resource.is_valid_at(now_ms)) - || self.finalized_manifest_lease_bindings.iter().any(|binding| { - binding.lease_id == *lease_id - && binding.lease_issued_at_ms == lease_issued_at_ms - && self - .finalized_manifest_generations - .get(&binding.manifest_generation) - .is_some_and(|generation| generation.resource_ids.contains(&resource.id)) - }) - } - } - } - - pub(crate) fn resource_is_valid_at(&self, resource: &TransientResourceRef, now_ms: u64) -> bool { - self.resource_is_valid_at_for(resource, now_ms, TransientResourceValidityScope::PublishedSession) - } - - pub(crate) fn resource_matches_current(&self, resource: &TransientResourceRef, now_ms: u64) -> bool { - self.resources.get(&resource.id).is_some_and(|current| { - Arc::ptr_eq(¤t.access, &resource.access) - && current.has_same_cache_identity(resource) - && self.resource_is_valid_at(current, now_ms) - }) - } - - pub fn resolve_current_resource( - &self, - resource_id: &TransientResourceId, - now_ms: u64, - ) -> Option { - self.resources.get(resource_id).filter(|resource| self.resource_is_valid_at(resource, now_ms)).cloned() - } - - pub(crate) fn resolve_resource_for_lease( - &self, - resource_id: &TransientResourceId, - lease_id: &HlsAccessLeaseId, - lease_issued_at_ms: u64, - published_resource_ids: &HlsPublishedTransientResourceIds, - now_ms: u64, - ) -> Option { - self.resources - .get(resource_id) - .filter(|resource| { - self.resource_is_valid_at_for( - resource, - now_ms, - TransientResourceValidityScope::AccessLease { - lease_id, - lease_issued_at_ms, - published_resource_ids, - }, - ) - }) - .cloned() - } - - pub fn get_valid_resource( - &mut self, - resource_id: &TransientResourceId, - now_ms: u64, - ) -> Option { - self.prune_expired(now_ms); - self.resolve_current_resource(resource_id, now_ms) - } - - pub fn prune_expired(&mut self, now_ms: u64) { self.prune_expired_except(now_ms, &HashSet::new()); } - - pub fn prune_expired_except(&mut self, now_ms: u64, protected: &HashSet) { - let expired_resource_ids = self - .resources - .iter() - .filter(|(id, resource)| { - !protected.contains(*id) - && !self.resource_is_valid_at(resource, now_ms) - && resource.active_readers() == 0 - }) - .map(|(id, _)| id.clone()) - .collect::>(); - self.resources.retain(|id, _| !expired_resource_ids.contains(id)); - } - - pub fn active_resource_readers(&self) -> u32 { - self.resources.values().map(TransientResourceRef::active_readers).sum() - } - pub fn has_active_resource_readers(&self) -> bool { self.active_resource_readers() > 0 } - - /// Builds the object-cache key for an already registered transient resource. - /// - /// The key is intentionally based on the opaque resource ID. The concrete fetch URI remains in - /// `TransientResourceRef::resolved_origin_uri` and must not be reconstructed from this key. - pub fn transient_object_key( - proxy_session_id: &ProxySessionId, - resource_id: &TransientResourceId, - file_ext: impl Into, - ) -> TransientObjectCacheKey { - TransientObjectCacheKey::new(proxy_session_id.clone(), resource_id.clone(), file_ext) - } - - pub fn ready_object( - &mut self, - key: &TransientObjectCacheKey, - resource_kind: TransientResourceKind, - now_ms: u64, - ) -> Option { - let binding = { - let entry = self.object_cache.get(key)?; - let resource = self.resources.get(key.transient_resource_id())?; - if resource.kind != resource_kind { - return None; - } - let binding = TransientObjectResourceBinding::from_resource(resource, &entry.binding.file_extension); - (binding.matches_resource_identity(resource) && self.resource_is_valid_at(resource, now_ms)) - .then_some(binding)? - }; - let entry = self.object_cache.get_mut(key)?; - if !matches!(entry.status, TransientObjectCacheStatus::Ready { .. }) - || entry.binding != binding - || (resource_kind == TransientResourceKind::Key && entry.ready_content_length() != Some(16)) - || entry.expires_at_ms < now_ms - { - return None; - } - entry.last_accessed_at_ms = now_ms; - entry.access.reader_started(now_ms); - entry.access.reader_finished(); - Some(entry.clone()) - } - - fn current_resource_binding( - &self, - resource: &TransientResourceRef, - file_extension: &str, - now_ms: u64, - ) -> Option { - let current = self.resources.get(&resource.id)?; - let binding = TransientObjectResourceBinding::from_resource(current, file_extension); - (current.has_same_cache_identity(resource) - && binding.matches_resource_identity(current) - && self.resource_is_valid_at(current, now_ms)) - .then_some(binding) - } - - fn binding_is_current(&self, binding: &TransientObjectResourceBinding, now_ms: u64) -> bool { - self.resources.get(&binding.resource_id).is_some_and(|resource| { - binding.matches_resource_identity(resource) && self.resource_is_valid_at(resource, now_ms) - }) - } - - pub fn begin_object_fetch( - &mut self, - proxy_session_id: &ProxySessionId, - resource: &TransientResourceRef, - file_ext: &str, - now_ms: u64, - cache_duration_ms: u64, - ) -> TransientObjectFetchDecision { - let lookup_key = Self::transient_object_key(proxy_session_id, &resource.id, file_ext.to_string()); - let current_binding = self.current_resource_binding(resource, file_ext, now_ms); - let binding_is_current = current_binding.is_some(); - let binding = - current_binding.unwrap_or_else(|| TransientObjectResourceBinding::from_resource(resource, file_ext)); - match self.object_cache.get(&lookup_key) { - Some(entry) - if binding_is_current - && entry.is_ready_at(now_ms) - && entry.binding == binding - && (resource.kind != TransientResourceKind::Key || entry.ready_content_length() == Some(16)) => - { - return TransientObjectFetchDecision::Ready; - } - Some(entry) - if binding_is_current - && entry.binding == binding - && entry.expires_at_ms >= now_ms - && matches!(entry.status, TransientObjectCacheStatus::Fetching { .. }) => - { - let notifier = - self.object_fetch_notifiers.entry(lookup_key).or_insert_with(|| Arc::new(Notify::new())).clone(); - return TransientObjectFetchDecision::Wait(notifier); - } - Some(_) | None => {} - } - let expires_at_ms = transient_object_expires_at(now_ms, cache_duration_ms); - let content_type = resource.content_type_hint.clone().unwrap_or_else(|| "application/octet-stream".to_string()); - let fetch_generation = self.next_object_fetch_generation; - self.next_object_fetch_generation = self.next_object_fetch_generation.saturating_add(1); - let cache_key = Self::transient_object_key( - proxy_session_id, - &resource.id, - format!("{file_ext}.fill-{fetch_generation:016x}"), - ); - let superseded_object = self.remove_object_entry(&lookup_key); - let _previous = self.object_cache.insert( - lookup_key.clone(), - TransientObjectCacheEntry::new_fetching( - cache_key.clone(), - binding.clone(), - now_ms, - expires_at_ms, - content_type, - ), - ); - self.object_fetch_notifiers.entry(lookup_key.clone()).or_insert_with(|| Arc::new(Notify::new())); - let Some(entry) = self.object_cache.get(&lookup_key) else { - return TransientObjectFetchDecision::Wait( - self.object_fetch_notifiers.entry(lookup_key).or_insert_with(|| Arc::new(Notify::new())).clone(), - ); - }; - TransientObjectFetchDecision::Fetch(Box::new(TransientObjectFetchToken { - lookup_key, - cache_key, - entry_access: Arc::clone(&entry.access), - binding, - superseded_object, - })) - } - - fn object_fetch_entry_matches(&self, token: &TransientObjectFetchToken) -> bool { - self.object_cache.get(&token.lookup_key).is_some_and(|entry| { - matches!(entry.status, TransientObjectCacheStatus::Fetching { .. }) - && entry.key == token.cache_key - && Arc::ptr_eq(&entry.access, &token.entry_access) - && entry.binding == token.binding - }) - } - - pub fn object_fetch_token_matches(&self, token: &TransientObjectFetchToken) -> bool { - self.object_fetch_entry_matches(token) - } - - fn mark_object_ready( - &mut self, - token: &TransientObjectFetchToken, - content_type: String, - content_length: u64, - now_ms: u64, - expires_at_ms: u64, - ) { - let notify_waiters = self.object_fetch_notifiers.remove(&token.lookup_key); - if let Some(entry) = self.object_cache.get_mut(&token.lookup_key) { - entry.status = TransientObjectCacheStatus::Ready { content_length, ready_at_ms: now_ms }; - entry.content_type = content_type; - entry.last_accessed_at_ms = now_ms; - entry.expires_at_ms = expires_at_ms; - } - if let Some(notifier) = notify_waiters { - notifier.notify_waiters(); - } - } - - pub fn mark_object_ready_if_current( - &mut self, - token: &TransientObjectFetchToken, - content_type: String, - content_length: u64, - now_ms: u64, - expires_at_ms: u64, - ) -> bool { - if !self.object_fetch_entry_matches(token) { - return false; - } - if !self.binding_is_current(&token.binding, now_ms) { - let _removed = self.remove_object_entry(&token.lookup_key); - return false; - } - self.mark_object_ready(token, content_type, content_length, now_ms, expires_at_ms); - true - } - - fn mark_object_failed_retryable(&mut self, key: &TransientObjectCacheKey, now_ms: u64, retry_after_ms: u64) { - let failed_expires_at_ms = self.failed_object_metadata_expires_at(now_ms); - let notify_waiters = self.object_fetch_notifiers.remove(key); - if let Some(entry) = self.object_cache.get_mut(key) { - entry.status = TransientObjectCacheStatus::FailedRetryable { failed_at_ms: now_ms, retry_after_ms }; - entry.last_accessed_at_ms = now_ms; - entry.expires_at_ms = entry.expires_at_ms.min(failed_expires_at_ms); - } - if let Some(notifier) = notify_waiters { - notifier.notify_waiters(); - } - self.enforce_failed_object_metadata_bound(); - } - - fn mark_object_failed_permanent(&mut self, key: &TransientObjectCacheKey, now_ms: u64, status: Option) { - let failed_expires_at_ms = self.failed_object_metadata_expires_at(now_ms); - let notify_waiters = self.object_fetch_notifiers.remove(key); - if let Some(entry) = self.object_cache.get_mut(key) { - entry.status = TransientObjectCacheStatus::FailedPermanent { failed_at_ms: now_ms, status }; - entry.last_accessed_at_ms = now_ms; - entry.expires_at_ms = entry.expires_at_ms.min(failed_expires_at_ms); - } - if let Some(notifier) = notify_waiters { - notifier.notify_waiters(); - } - self.enforce_failed_object_metadata_bound(); - } - - fn failed_object_metadata_expires_at(&self, now_ms: u64) -> u64 { now_ms.saturating_add(self.resource_ttl_ms) } - - pub fn mark_object_failed_retryable_if_current( - &mut self, - token: &TransientObjectFetchToken, - now_ms: u64, - retry_after_ms: u64, - ) -> bool { - if !self.object_fetch_entry_matches(token) { - return false; - } - if !self.binding_is_current(&token.binding, now_ms) { - let _removed = self.remove_object_entry(&token.lookup_key); - return false; - } - self.mark_object_failed_retryable(&token.lookup_key, now_ms, retry_after_ms); - true - } - - pub fn mark_object_failed_permanent_if_current( - &mut self, - token: &TransientObjectFetchToken, - now_ms: u64, - status: Option, - ) -> bool { - if !self.object_fetch_entry_matches(token) { - return false; - } - if !self.binding_is_current(&token.binding, now_ms) { - let _removed = self.remove_object_entry(&token.lookup_key); - return false; - } - self.mark_object_failed_permanent(&token.lookup_key, now_ms, status); - true - } - - /// Returns the finite READY lifetime of an AES key dependency without touching access accounting. - pub fn ready_key_object_valid_until_ms( - &self, - proxy_session_id: &ProxySessionId, - resource_id: &TransientResourceId, - file_ext: &str, - now_ms: u64, - ) -> Option { - let resource = self.resources.get(resource_id)?; - if resource.kind != TransientResourceKind::Key - || resource.file_ext_hint.as_deref() != Some(file_ext) - || !self.resource_is_valid_at(resource, now_ms) - { - return None; - } - let key = Self::transient_object_key(proxy_session_id, resource_id, file_ext.to_string()); - let entry = self.object_cache.get(&key)?; - let binding = TransientObjectResourceBinding::from_resource(resource, file_ext); - let valid_aes128_key = matches!(entry.status, TransientObjectCacheStatus::Ready { content_length: 16, .. }); - let valid_until_ms = if self.protected_finalized_resource_refcounts.contains_key(resource_id) { - entry.expires_at_ms - } else { - entry.expires_at_ms.min(resource.expires_at_ms) - }; - (valid_aes128_key && entry.binding == binding && entry.is_ready_at(now_ms)).then_some(valid_until_ms) - } - - pub fn object_unavailable_state( - &self, - key: &TransientObjectCacheKey, - now_ms: u64, - ) -> TransientObjectUnavailableState { - let Some(entry) = self.object_cache.get(key) else { - return TransientObjectUnavailableState::Missing; - }; - if entry.expires_at_ms < now_ms || !self.binding_is_current(&entry.binding, now_ms) { - return TransientObjectUnavailableState::Missing; - } - match entry.status { - TransientObjectCacheStatus::Fetching { .. } => TransientObjectUnavailableState::Fetching, - TransientObjectCacheStatus::FailedRetryable { retry_after_ms, .. } => { - TransientObjectUnavailableState::FailedRetryable { retry_after_ms } - } - TransientObjectCacheStatus::FailedPermanent { .. } => TransientObjectUnavailableState::FailedPermanent, - TransientObjectCacheStatus::Ready { .. } => TransientObjectUnavailableState::Missing, - } - } - - pub fn ready_object_cache_size(&self) -> u64 { - self.object_cache.values().filter_map(TransientObjectCacheEntry::ready_content_length).sum() - } - - pub fn take_expired_object_removals_except( - &mut self, - now_ms: u64, - protected: &HashSet, - limit: usize, - ) -> Vec { - let mut keys = self - .object_cache - .iter() - .filter_map(|(key, entry)| { - let ready_is_protected = - entry.ready_content_length().is_some() && protected.contains(key.transient_resource_id()); - if !ready_is_protected && entry.access.active_readers() == 0 && entry.expires_at_ms < now_ms { - return Some((key.clone(), entry.expires_at_ms, entry.created_at_ms, key.stable_value())); - } - None - }) - .collect::>(); - keys.sort_by(|left, right| (left.1, left.2, &left.3).cmp(&(right.1, right.2, &right.3))); - keys.into_iter().take(limit).filter_map(|(key, _, _, _)| self.remove_object_entry(&key)).collect() - } - - pub fn remove_oldest_ready_object_except( - &mut self, - protected: &HashSet, - ) -> Option { - let candidate = self - .object_cache - .iter() - .filter_map(|(key, entry)| { - if !protected.contains(key.transient_resource_id()) && entry.access.active_readers() == 0 { - return entry.ready_content_length().map(|content_length| { - (key.clone(), content_length, entry.last_accessed_at_ms, entry.created_at_ms) - }); - } - None - }) - .min_by_key(|(_, _, last_accessed_at_ms, created_at_ms)| (*last_accessed_at_ms, *created_at_ms))?; - self.remove_object_entry(&candidate.0) - } - - fn remove_object_entry(&mut self, lookup_key: &TransientObjectCacheKey) -> Option { - let entry = self.object_cache.remove(lookup_key)?; - if let Some(notifier) = self.object_fetch_notifiers.remove(lookup_key) { - notifier.notify_waiters(); - } - let content_length = entry.ready_content_length().unwrap_or_default(); - Some(TransientObjectRemoval { key: entry.key, content_length }) - } - - fn enforce_failed_object_metadata_bound(&mut self) { - let mut failed = self - .object_cache - .iter() - .filter_map(|(key, entry)| match entry.status { - TransientObjectCacheStatus::FailedRetryable { failed_at_ms, .. } - | TransientObjectCacheStatus::FailedPermanent { failed_at_ms, .. } => { - Some((key.clone(), failed_at_ms, entry.created_at_ms, key.stable_value())) - } - TransientObjectCacheStatus::Fetching { .. } | TransientObjectCacheStatus::Ready { .. } => None, - }) - .collect::>(); - let overflow = failed.len().saturating_sub(MAX_FAILED_TRANSIENT_OBJECT_ENTRIES); - if overflow == 0 { - return; - } - failed.sort_by(|left, right| (left.1, left.2, &left.3).cmp(&(right.1, right.2, &right.3))); - for (key, _, _, _) in failed.into_iter().take(overflow) { - let _removed = self.remove_object_entry(&key); - } - } } impl Default for TransientPassthroughState { fn default() -> Self { Self::new(DEFAULT_TRANSIENT_RESOURCE_TTL_MS) } } -impl HlsSession { - pub fn commit_transient_object_ready_if_current( - &mut self, - resource_kind: TransientResourceKind, - token: &TransientObjectFetchToken, - content_type: String, - content_length: u64, - now_ms: u64, - expires_at_ms: u64, - ) -> bool { - let committed = - self.transient.mark_object_ready_if_current(token, content_type, content_length, now_ms, expires_at_ms); - if committed && resource_kind == TransientResourceKind::Key { - self.advance_media_readiness_generation(); - } - committed - } - - pub fn fail_transient_object_retryable_if_current( - &mut self, - token: &TransientObjectFetchToken, - now_ms: u64, - retry_after_ms: u64, - ) -> bool { - self.transient.mark_object_failed_retryable_if_current(token, now_ms, retry_after_ms) - } - - pub fn fail_transient_object_permanent_if_current( - &mut self, - token: &TransientObjectFetchToken, - now_ms: u64, - status: Option, - ) -> bool { - self.transient.mark_object_failed_permanent_if_current(token, now_ms, status) - } -} - impl fmt::Debug for TransientPassthroughState { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("TransientPassthroughState") @@ -1744,103 +221,6 @@ impl fmt::Debug for TransientPassthroughState { } } -fn transient_manifest_identity(body: &str, resource_ids: &HashSet) -> TransientManifestIdentity { - let body_hash = *blake3::hash(body.as_bytes()).as_bytes(); - let mut ordered_resource_ids = resource_ids.iter().map(|resource_id| resource_id.0.as_str()).collect::>(); - ordered_resource_ids.sort_unstable(); - let mut resource_hasher = blake3::Hasher::new(); - for resource_id in ordered_resource_ids { - resource_hasher.update(&u64::try_from(resource_id.len()).unwrap_or(u64::MAX).to_le_bytes()); - resource_hasher.update(resource_id.as_bytes()); - } - TransientManifestIdentity { body_hash, resource_set_hash: *resource_hasher.finalize().as_bytes() } -} - -fn complete_manifest_append_boundary(body: &str) -> bool { - body.ends_with('\n') - && body.lines().rev().map(str::trim).find(|line| !line.is_empty()).is_some_and(|line| !line.starts_with('#')) -} - -fn estimated_transient_resource_entry_bytes( - resource_id: &TransientResourceId, - resource: &TransientResourceRef, -) -> usize { - std::mem::size_of::<(TransientResourceId, TransientResourceRef)>() - .saturating_add(resource_id.0.len()) - .saturating_add(resource.id.0.len()) - .saturating_add(resource.resolved_origin_uri.len()) - .saturating_add(resource.content_type_hint.as_ref().map_or(0, String::len)) - .saturating_add(resource.file_ext_hint.as_ref().map_or(0, String::len)) - .saturating_add(std::mem::size_of::()) - .saturating_add(ESTIMATED_ARC_ALLOCATION_OVERHEAD_BYTES) - .saturating_add(ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES) -} - -fn estimated_transient_resource_id_set_bytes(resource_ids: &HashSet) -> usize { - resource_ids.iter().fold(0_usize, |total, resource_id| { - total.saturating_add( - std::mem::size_of::() - .saturating_add(resource_id.0.len()) - .saturating_add(ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES), - ) - }) -} - -fn estimated_transient_object_entry_bytes(entry: &TransientObjectCacheEntry) -> usize { - std::mem::size_of::<(TransientObjectCacheKey, TransientObjectCacheEntry)>() - .saturating_add(entry.key.stable_value().len()) - .saturating_add(entry.content_type.len()) - .saturating_add(entry.binding.resource_id.0.len()) - .saturating_add(entry.binding.resolved_origin_uri.len()) - .saturating_add(entry.binding.file_extension.len()) - .saturating_add(std::mem::size_of::()) - .saturating_add(ESTIMATED_ARC_ALLOCATION_OVERHEAD_BYTES) - .saturating_add(ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES) -} - -pub(crate) fn extract_transient_resource_ids(body: &str) -> HashSet { - let mut resource_ids = HashSet::new(); - for line in body.lines() { - let line = line.trim(); - if line.is_empty() { - continue; - } - if line.starts_with('#') { - for uri in tag_uri_attributes(line) { - if let Some(resource_id) = transient_resource_id_from_shared_route(uri) { - resource_ids.insert(resource_id); - } - } - } else if let Some(resource_id) = transient_resource_id_from_shared_route(line) { - resource_ids.insert(resource_id); - } - } - resource_ids -} - -/// Yields the quoted URI values carried by a tag line (`URI="..."`). -fn tag_uri_attributes(line: &str) -> impl Iterator { - line.split("URI=\"").skip(1).filter_map(|tail| tail.split('"').next()) -} - -/// Extracts one opaque transient resource ID from a canonical own `/r/{id}.{ext}` route. -/// -/// Unlike a naive `split("/r/")`, this only recognises proxy-owned shared-HLS locators: it -/// ignores `/r/` occurrences inside comments (non-URI tag content is never yielded as a URI), -/// query strings or fragments, and rejects absolute foreign URIs by requiring a path-only -/// `/hls/shared/live/` prefix before the route segment. -fn transient_resource_id_from_shared_route(uri: &str) -> Option { - let path = uri.split(['?', '#']).next().unwrap_or(""); - if path.contains("://") { - return None; - } - let (prefix, remainder) = path.rsplit_once("/r/")?; - if !prefix.contains("/hls/shared/live/") { - return None; - } - TransientResourceFile::parse(remainder).map(|file| file.resource_id) -} - /// Empty runtime store for future transient passthrough resources. #[derive(Default)] pub struct TransientResourceStore; @@ -1859,1209 +239,21 @@ fn default_content_type_for_transient_ext(extension: &str) -> Option<&'static st } #[cfg(test)] -mod tests { - use super::{ - build_transient_resource_id, extract_transient_resource_ids, transient_object_expires_at, - HlsPublishedTransientResourceIds, HlsTransientManifestCommitError, TransientManifestLeaseBinding, - TransientObjectCacheStatus, TransientPassthroughState, TransientResourceId, TransientResourceKind, - TransientResourceRef, MAX_FAILED_TRANSIENT_OBJECT_ENTRIES, - }; - use crate::{ - manifest_limits::{ - HlsManifestLimitKind, HlsManifestLimitViolation, MAX_ESTIMATED_TRANSIENT_METADATA_BYTES, - MAX_RETAINED_FINALIZED_MANIFEST_GENERATIONS, MAX_TRANSIENT_GENERATION_MEMBERSHIPS, - MAX_TRANSIENT_MANIFEST_RESOURCES, MAX_TRANSIENT_ORIGIN_URI_BYTES_PER_SESSION, - MAX_TRANSIENT_RESOURCE_ENTRIES_PER_SESSION, MAX_TRANSIENT_REWRITTEN_MANIFEST_BYTES, - }, - transient_manifest::{TransientManifestRewriter, TransientRewriteResult}, - HlsAccessLeaseId, ProxySessionId, - }; - use std::{collections::HashSet, fmt::Write as _, sync::Arc, task::Poll}; - use tuliprox_parser::hls::origin_manifest::{ - parse_manifest_semantics, HlsManifestLifecycle, HlsManifestWindowPolicy, - }; - - fn local_representation_limit(error: HlsTransientManifestCommitError) -> HlsManifestLimitViolation { - let HlsTransientManifestCommitError::LocalRepresentationLimit(violation) = error else { - panic!("expected local representation limit, got {error:?}"); - }; - violation - } - - #[test] - fn extract_transient_resource_ids_ignores_comment_and_foreign_uri_surfaces() { - let body = concat!( - "# /r/fake.ts\n", - "https://origin.example.com/r/fake.ts\n", - "#EXT-X-KEY:METHOD=AES-128,URI=\"https://origin.example.com/r/fake.key\"\n", - ); - - assert!(extract_transient_resource_ids(body).is_empty()); - } - - #[test] - fn extract_transient_resource_ids_ignores_query_values() { - let body = concat!( - "#EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/proxy/lease/r/AbCdEfGhIjKlMnOp.key?token=/r/fake.ts\"\n", - "/hls/shared/live/proxy/lease/r/AbCdEfGhIjKlMnOp.ts?redirect=/r/foreign.ts\n", - ); - - let resource_ids = extract_transient_resource_ids(body); - - assert_eq!(resource_ids.len(), 1); - assert!(resource_ids.contains(&TransientResourceId("AbCdEfGhIjKlMnOp".to_string()))); - } - - #[test] - fn extract_transient_resource_ids_rejects_noncanonical_resource_files() { - let body = concat!( - "/hls/shared/live/proxy/lease/r/AbCdEfGhIjKlMnOp.exe\n", - "/hls/shared/live/proxy/lease/r/AbCdEfGhIjKlMnOp.ts/extra\n", - "/hls/shared/live/proxy/lease/r/AbCdEfGhIjKlMnOp.tar.gz\n", - ); - - assert!(extract_transient_resource_ids(body).is_empty()); - } - - #[test] - fn extract_transient_resource_ids_keeps_segment_key_and_map_routes() { - let body = concat!( - "#EXTM3U\n", - "#EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/proxy/__hls_access_lease_id__/r/Key0000000000Id.key\"\n", - "#EXT-X-MAP:URI=\"/hls/shared/live/proxy/__hls_access_lease_id__/r/Map0000000000Id.mp4\"\n", - "#EXTINF:1.0,\n", - "/hls/shared/live/proxy/lease/r/Seg0000000000Id.ts\n", - ); - - let resource_ids = extract_transient_resource_ids(body); - - assert_eq!(resource_ids.len(), 3); - assert!(resource_ids.contains(&TransientResourceId("Key0000000000Id".to_string()))); - assert!(resource_ids.contains(&TransientResourceId("Map0000000000Id".to_string()))); - assert!(resource_ids.contains(&TransientResourceId("Seg0000000000Id".to_string()))); - } - - fn finalized_manifest_body(resource_id: &TransientResourceId, extension: &str) -> String { - format!( - "#EXTM3U\n#EXT-X-TARGETDURATION:1\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:1,\n/hls/shared/live/session/lease/r/{}.{extension}\n#EXT-X-ENDLIST\n", - resource_id.0 - ) - } - - fn replace_with_finalized_manifest( - state: &mut TransientPassthroughState, - resource_id: &TransientResourceId, - extension: &str, - rendered_at_ms: u64, - ) { - state.replace_manifest_with_semantics( - finalized_manifest_body(resource_id, extension), - rendered_at_ms, - Some(1_000), - ); - } - - #[test] - fn finalized_committed_manifest_has_full_window_and_no_deadline() { - let mut state = TransientPassthroughState::default(); - - state.replace_manifest_with_semantics("#EXTM3U\n#EXT-X-ENDLIST\n".to_string(), 100, Some(1_000)); - - assert!(state.last_manifest_finalized()); - assert_eq!(state.last_manifest_window_policy(), HlsManifestWindowPolicy::PreserveFullManifest); - assert_eq!(state.last_manifest_valid_until_ms(), None); - } - - #[test] - fn rolling_typed_committed_manifest_preserves_full_window_with_deadline() { - let mut state = TransientPassthroughState::default(); - - state.replace_manifest_with_semantics( - "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXTINF:1,\n/r/resource.ts\n".to_string(), - 100, - Some(1_000), - ); - - assert!(!state.last_manifest_finalized()); - assert_eq!(state.last_manifest_window_policy(), HlsManifestWindowPolicy::PreserveFullManifest); - assert_eq!(state.last_manifest_valid_until_ms(), Some(1_100)); - } - - fn rewritten_finalized_manifest(segment_count: usize, identity: &str) -> TransientRewriteResult { - let mut body = - String::from("#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:1\n"); - for index in 0..segment_count { - writeln!(body, "#EXTINF:6,\n{identity}-{index}.ts").expect("synthetic manifest renders"); - } - body.push_str("#EXT-X-ENDLIST\n"); - TransientManifestRewriter::rewrite( - &body, - "https://origin.example/archive/index.m3u8", - &ProxySessionId("proxy-session".to_string()), - b"secret", - 0, - 300_000, - ) - } - - #[test] - fn transient_manifest_resource_limit_rejects_without_mutating_current_manifest() { - let mut state = TransientPassthroughState::default(); - let baseline = rewritten_finalized_manifest(1, "baseline"); - let baseline_semantics = parse_manifest_semantics(&baseline.body); - state - .commit_rewritten_manifest_with_semantics( - baseline.body, - baseline.resources, - 1, - Some(6_000), - baseline_semantics, - ) - .expect("baseline manifest commits"); - let baseline_body = state.last_manifest_body.clone().expect("baseline body"); - let baseline_generation = state.manifest_generation(); - let baseline_resource_count = state.resources.len(); - let oversized = rewritten_finalized_manifest(MAX_TRANSIENT_MANIFEST_RESOURCES + 1, "oversized"); - let oversized_semantics = parse_manifest_semantics(&oversized.body); - - let violation = local_representation_limit( - state - .commit_rewritten_manifest_with_semantics( - oversized.body, - oversized.resources, - 2, - None, - oversized_semantics, - ) - .expect_err("resource overflow is rejected"), - ); - - assert_eq!(violation.kind, HlsManifestLimitKind::TransientResources); - assert_eq!(violation.actual, MAX_TRANSIENT_MANIFEST_RESOURCES + 1); - assert_eq!(state.manifest_generation(), baseline_generation); - assert_eq!(state.resources.len(), baseline_resource_count); - assert!(Arc::ptr_eq(state.last_manifest_body.as_ref().expect("current body remains"), &baseline_body)); - } - - #[test] - fn transient_rewritten_body_limit_rejects_before_commit() { - let mut state = TransientPassthroughState::default(); - let oversized_body = "x".repeat(MAX_TRANSIENT_REWRITTEN_MANIFEST_BYTES + 1); - - let violation = local_representation_limit( - state - .commit_rewritten_manifest_with_semantics( - oversized_body, - Vec::new(), - 1, - None, - parse_manifest_semantics("#EXTM3U\n"), - ) - .expect_err("rewritten body overflow is rejected"), - ); - - assert_eq!(violation.kind, HlsManifestLimitKind::TransientRewrittenBytes); - assert!(state.last_manifest_body.is_none()); - assert!(state.resources.is_empty()); - } - - #[test] - fn transient_session_origin_uri_byte_limit_is_enforced() { - let mut state = TransientPassthroughState::default(); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "x".repeat(MAX_TRANSIENT_ORIGIN_URI_BYTES_PER_SESSION + 1), - b"secret", - 0, - 300_000, - Some("ts".to_string()), - ); - - let violation = local_representation_limit( - state - .commit_rewritten_manifest_with_semantics( - finalized_manifest_body(&resource.id, "ts"), - vec![resource], - 1, - Some(1_000), - parse_manifest_semantics("#EXTM3U\n#EXT-X-ENDLIST\n"), - ) - .expect_err("origin URI byte overflow is rejected"), - ); - - assert_eq!(violation.kind, HlsManifestLimitKind::TransientOriginUriBytes); - assert!(state.resources.is_empty()); - } - - #[test] - fn prospective_origin_uri_bytes_counts_duplicate_resource_ids_once() { - let state = TransientPassthroughState::default(); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "https://origin.example/archive/segment.ts", - b"secret", - 0, - 300_000, - Some("ts".to_string()), - ); - - assert_eq!( - state.prospective_origin_uri_bytes(&[resource.clone(), resource.clone()]), - resource.resolved_origin_uri.len() - ); - } - - #[test] - fn retained_finalized_generation_limit_rejects_the_next_generation() { - let mut state = TransientPassthroughState::default(); - for index in 0..MAX_RETAINED_FINALIZED_MANIFEST_GENERATIONS { - let rewritten = rewritten_finalized_manifest(1, &format!("generation-{index}")); - let semantics = parse_manifest_semantics(&rewritten.body); - state - .commit_rewritten_manifest_with_semantics( - rewritten.body, - rewritten.resources, - u64::try_from(index).expect("test index fits u64"), - Some(6_000), - semantics, - ) - .expect("generation within retention limit commits"); - let generation = state.current_finalized_manifest_generation().expect("finalized generation"); - assert!(state.bind_finalized_manifest_generation(TransientManifestLeaseBinding::new( - HlsAccessLeaseId(format!("lease-{index}")), - u64::try_from(index).expect("test index fits u64"), - generation, - ))); - } - let retained_body = state.last_manifest_body.clone().expect("retained current body"); - let retained_generation = state.manifest_generation(); - let overflow = rewritten_finalized_manifest(1, "generation-overflow"); - let semantics = parse_manifest_semantics(&overflow.body); - - let violation = local_representation_limit( - state - .commit_rewritten_manifest_with_semantics( - overflow.body, - overflow.resources, - 100, - Some(6_000), - semantics, - ) - .expect_err("additional retained generation is rejected"), - ); - - assert_eq!(violation.kind, HlsManifestLimitKind::FinalizedGenerations); - assert_eq!(violation.actual, MAX_RETAINED_FINALIZED_MANIFEST_GENERATIONS + 1); - assert_eq!(state.manifest_generation(), retained_generation); - assert!(Arc::ptr_eq(state.last_manifest_body.as_ref().expect("current body remains"), &retained_body)); - } - - #[test] - fn byte_identical_finalized_refresh_reuses_generation_and_body_allocation() { - let mut state = TransientPassthroughState::default(); - let first = rewritten_finalized_manifest(3, "stable"); - let semantics = parse_manifest_semantics(&first.body); - state - .commit_rewritten_manifest_with_semantics(first.body, first.resources, 0, Some(18_000), semantics) - .expect("first finalized manifest commits"); - let generation = state.current_finalized_manifest_generation().expect("finalized generation"); - let generation_highwater = state.manifest_generation(); - let body = state.last_manifest_body.clone().expect("stored body"); - - let refreshed = TransientManifestRewriter::rewrite( - "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:1\n\ - #EXTINF:6,\nstable-0.ts\n#EXTINF:6,\nstable-1.ts\n#EXTINF:6,\nstable-2.ts\n#EXT-X-ENDLIST\n", - "https://origin.example/archive/index.m3u8", - &ProxySessionId("proxy-session".to_string()), - b"secret", - 400_000, - 300_000, - ); - let refreshed_resource_id = refreshed.resources[0].id.clone(); - state - .commit_rewritten_manifest_with_semantics( - refreshed.body, - refreshed.resources, - 400_000, - Some(18_000), - semantics, - ) - .expect("identical finalized refresh commits"); - - assert_eq!(state.current_finalized_manifest_generation(), Some(generation)); - assert_eq!(state.manifest_generation(), generation_highwater); - assert_eq!(state.finalized_manifest_generation_count(), 1); - assert!(Arc::ptr_eq(state.last_manifest_body.as_ref().expect("same body allocation"), &body)); - assert_eq!(state.resources[&refreshed_resource_id].expires_at_ms, 700_000); - } - - #[test] - fn rolling_event_append_requires_current_rewrite_secret_and_refreshes_existing_mapping_validity() { - let mut state = TransientPassthroughState::default(); - let first_origin = "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXT-X-PLAYLIST-TYPE:EVENT\n\ - #EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:6,\n0.ts\n#EXTINF:6,\n1.ts\n"; - let first = TransientManifestRewriter::rewrite( - first_origin, - "https://origin.example/event/index.m3u8", - &ProxySessionId("proxy-session".to_string()), - b"secret", - 0, - 300_000, - ); - let checkpoint = first.checkpoint.clone(); - let first_resource_id = first.resources[0].id.clone(); - let semantics = parse_manifest_semantics(first_origin); - let footprint = state - .commit_rewritten_manifest_with_semantics(first.body, first.resources, 0, Some(12_000), semantics) - .expect("initial EVENT commits"); - state.record_rolling_event_rewrite_seed( - first_origin, - "https://origin.example/event/index.m3u8", - b"secret", - checkpoint, - footprint.estimated_metadata_bytes, - true, - ); - let previous_template = state.last_manifest_template().expect("initial template"); - let second_origin = format!("{first_origin}#EXTINF:6,\n2.ts\n"); - assert!(state - .rolling_event_append_context(&second_origin, "https://origin.example/event/index.m3u8", b"rotated-secret",) - .is_none()); - let context = state - .rolling_event_append_context(&second_origin, "https://origin.example/event/index.m3u8", b"secret") - .expect("safe append is detected"); - let mut appended = TransientManifestRewriter::rewrite_append( - &second_origin[context.suffix_offset..], - "https://origin.example/event/index.m3u8", - &ProxySessionId("proxy-session".to_string()), - b"secret", - 400_000, - 300_000, - context.checkpoint, - ); - let template = TransientPassthroughState::extend_rolling_event_template(&context.template, &appended.body) - .expect("append remains within limits") - .expect("append contains a complete media unit"); - let mut body = String::with_capacity(context.rewritten_prefix.len().saturating_add(appended.body.len())); - body.push_str(&context.rewritten_prefix); - body.push_str(&appended.body); - appended.body = body; - - state - .commit_incremental_rewritten_manifest_with_semantics( - appended.body, - appended.resources, - Arc::clone(&template), - 400_000, - 300_000, - parse_manifest_semantics(&second_origin), - ) - .expect("append commits"); - - assert_eq!(template.segments().len(), 3); - assert!(Arc::ptr_eq(&previous_template.segments()[0].uri, &template.segments()[0].uri)); - assert_eq!(state.resources[&first_resource_id].expires_at_ms, 700_000); - assert_eq!(state.current_manifest_resource_ids().len(), 3); - } - - #[test] - fn rolling_lease_membership_retains_only_still_valid_previously_published_resources() { - let mut state = TransientPassthroughState::default(); - let previous_resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "https://origin.example/previous.ts", - b"secret", - 0, - 20, - Some("ts".to_string()), - ); - let next_resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "https://origin.example/next.ts", - b"secret", - 0, - 40, - Some("ts".to_string()), - ); - let previous_resource_id = previous_resource.id.clone(); - let next_resource_id = next_resource.id.clone(); - state.upsert_resources([previous_resource, next_resource]); - let previous = HlsPublishedTransientResourceIds::from_manifest_body(&format!( - "/hls/shared/live/proxy/lease/r/{}.ts", - previous_resource_id.0 - )); - let next = HlsPublishedTransientResourceIds::from_manifest_body(&format!( - "/hls/shared/live/proxy/lease/r/{}.ts", - next_resource_id.0 - )); - - let retained = state.merge_current_published_resource_ids(&previous, next.clone(), 10); - assert!(retained.contains(&previous_resource_id)); - assert!(retained.contains(&next_resource_id)); - - let pruned = state.merge_current_published_resource_ids(&retained, next, 21); - assert!(!pruned.contains(&previous_resource_id)); - assert!(pruned.contains(&next_resource_id)); - } - - #[test] - fn transient_session_resource_entry_limit_is_enforced() { - let mut state = TransientPassthroughState::default(); - for index in 0..MAX_TRANSIENT_RESOURCE_ENTRIES_PER_SESSION { - state.upsert_resources([TransientResourceRef::new( - TransientResourceKind::Segment, - format!("https://origin.example/{index}.ts"), - b"secret", - 0, - 300_000, - Some("ts".to_string()), - )]); - } - let incoming = TransientResourceRef::new( - TransientResourceKind::Segment, - "https://other-origin.example/overflow.ts".to_string(), - b"secret", - 0, - 300_000, - Some("ts".to_string()), - ); - - let violation = local_representation_limit( - state - .commit_rewritten_manifest_with_semantics( - finalized_manifest_body(&incoming.id, "ts"), - vec![incoming], - 1, - Some(1_000), - parse_manifest_semantics("#EXTM3U\n#EXT-X-ENDLIST\n"), - ) - .expect_err("resource entry overflow is rejected"), - ); - - assert_eq!(violation.kind, HlsManifestLimitKind::TransientResourceEntries); - assert_eq!(violation.actual, MAX_TRANSIENT_RESOURCE_ENTRIES_PER_SESSION + 1); - } - - #[test] - fn finalized_generation_membership_limit_counts_overlapping_sets() { - let mut state = TransientPassthroughState::default(); - let base = rewritten_finalized_manifest(MAX_TRANSIENT_MANIFEST_RESOURCES, "shared"); - let mut body = base.body; - let resources = base.resources; - let generations_within_limit = MAX_TRANSIENT_GENERATION_MEMBERSHIPS / MAX_TRANSIENT_MANIFEST_RESOURCES; - for index in 0..generations_within_limit { - let _ = writeln!(body, "# generation-{index}"); - state - .commit_rewritten_manifest_with_semantics( - body.clone(), - if index == 0 { resources.clone() } else { Vec::new() }, - u64::try_from(index).expect("index fits u64"), - None, - parse_manifest_semantics("#EXTM3U\n#EXT-X-ENDLIST\n"), - ) - .expect("membership count remains within limit"); - let generation = state.current_finalized_manifest_generation().expect("finalized generation"); - assert!(state.bind_finalized_manifest_generation(TransientManifestLeaseBinding::new( - HlsAccessLeaseId(format!("lease-{index}")), - u64::try_from(index).expect("index fits u64"), - generation, - ))); - } - body.push_str("# generation-overflow\n"); - - let violation = local_representation_limit( - state - .commit_rewritten_manifest_with_semantics( - body, - Vec::new(), - 100, - None, - parse_manifest_semantics("#EXTM3U\n#EXT-X-ENDLIST\n"), - ) - .expect_err("membership overflow is rejected"), - ); - - assert_eq!(violation.kind, HlsManifestLimitKind::TransientGenerationMemberships); - assert_eq!(violation.actual, MAX_TRANSIENT_GENERATION_MEMBERSHIPS + MAX_TRANSIENT_MANIFEST_RESOURCES); - } - - #[test] - fn estimated_transient_metadata_limit_counts_owned_hint_bytes() { - let mut state = TransientPassthroughState::default(); - let mut retained = TransientResourceRef::new( - TransientResourceKind::Segment, - "https://origin.example/retained.ts".to_string(), - b"secret", - 0, - 300_000, - Some("ts".to_string()), - ); - retained.content_type_hint = Some("x".repeat(MAX_ESTIMATED_TRANSIENT_METADATA_BYTES)); - state.upsert_resources([retained]); - let candidate = rewritten_finalized_manifest(1, "candidate"); - let semantics = parse_manifest_semantics(&candidate.body); - - let violation = local_representation_limit( - state - .commit_rewritten_manifest_with_semantics( - candidate.body, - candidate.resources, - 1, - Some(6_000), - semantics, - ) - .expect_err("estimated metadata overflow is rejected"), - ); - - assert_eq!(violation.kind, HlsManifestLimitKind::TransientEstimatedMetadataBytes); - assert!(violation.actual > MAX_ESTIMATED_TRANSIENT_METADATA_BYTES); - } - - #[test] - fn transient_resource_id_is_stable_and_opaque() { - let first = build_transient_resource_id("http://origin.example.com/live/key.bin", b"secret"); - let second = build_transient_resource_id("http://origin.example.com/live/key.bin", b"secret"); - let other = build_transient_resource_id("http://origin.example.com/live/seg.ts", b"secret"); - - assert_eq!(first, second); - assert_ne!(first, other); - assert_eq!(first.0.len(), 16); - assert!(!first.0.contains("origin")); - } - - #[test] - fn transient_state_updates_existing_resource_ttl() { - let mut state = TransientPassthroughState::default(); - let mut resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/seg.ts", - b"secret", - 10, - 100, - Some("ts".to_string()), - ); - resource.encrypted_media = true; - let resource_id = resource.id.clone(); - state.upsert_resources([resource]); - let updated = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/seg.ts", - b"secret", - 20, - 200, - Some("ts".to_string()), - ); - state.upsert_resources([updated]); - - assert_eq!(state.resources.len(), 1); - assert!(state.resources.get(&resource_id).expect("resource").encrypted_media); - assert_eq!(state.get_valid_resource(&resource_id, 150).expect("resource remains valid").expires_at_ms, 220); - assert!(state.get_valid_resource(&resource_id, 221).is_none()); - } - - #[test] - fn manifest_refresh_does_not_extend_transient_ready_object_ttl() { - let mut state = TransientPassthroughState::default(); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/seg.ts", - b"secret", - 10, - 100, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - let key = TransientPassthroughState::transient_object_key( - &ProxySessionId("proxy-session".to_string()), - &resource_id, - "ts", - ); - state.upsert_resources([resource.clone()]); - replace_with_finalized_manifest(&mut state, &resource_id, "ts", 10); - let token = - match state.begin_object_fetch(&ProxySessionId("proxy-session".to_string()), &resource, "ts", 20, 50) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("new resource starts a cache fetch") - } - }; - assert!(state.mark_object_ready_if_current(&token, "video/mp2t".to_string(), 7, 30, 110)); - assert_eq!(state.object_cache.get(&key).expect("object").expires_at_ms, 110); - - let updated = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/seg.ts", - b"secret", - 100, - 300, - Some("ts".to_string()), - ); - state.upsert_resources([updated]); - replace_with_finalized_manifest(&mut state, &resource_id, "ts", 100); - - let object = state.object_cache.get(&key).expect("object remains"); - assert!(matches!(object.status, TransientObjectCacheStatus::Ready { .. })); - assert_eq!(object.expires_at_ms, 110); - } - - #[test] - fn transient_resource_refresh_does_not_extend_an_inflight_fill_deadline() { - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - let mut state = TransientPassthroughState::default(); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/inflight.ts", - b"secret", - 10, - 100, - Some("ts".to_string()), - ); - state.upsert_resources([resource.clone()]); - let token = match state.begin_object_fetch(&proxy_session_id, &resource, "ts", 20, 50) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("new resource starts a cache fetch") - } - }; - - state.upsert_resources([TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/inflight.ts", - b"secret", - 100, - 300, - Some("ts".to_string()), - )]); - - assert_eq!(state.object_cache.get(&token.lookup_key).map(|entry| entry.expires_at_ms), Some(70)); - } - - #[test] - fn transient_media_resource_extensions_use_video_mp4_content_type() { - for extension in ["mp4", "m4s", "m4v"] { - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - format!("http://origin.example.com/live/seg.{extension}"), - b"secret", - 10, - 100, - Some(extension.to_string()), - ); - - assert_eq!(resource.content_type_hint.as_deref(), Some("video/mp4")); - } - } - - #[test] - fn aes_key_readiness_requires_exactly_sixteen_bytes_and_honors_the_inclusive_ttl_boundary() { - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - for invalid_size in [15, 17] { - let mut state = TransientPassthroughState::default(); - let resource = TransientResourceRef::new( - TransientResourceKind::Key, - format!("http://origin.example.com/live/key-{invalid_size}.key"), - b"secret", - 10, - 100, - Some("key".to_string()), - ); - let resource_id = resource.id.clone(); - state.upsert_resources([resource.clone()]); - let token = match state.begin_object_fetch(&proxy_session_id, &resource, "key", 20, 50) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("new AES key starts a cache fetch") - } - }; - assert!(state.mark_object_ready_if_current( - &token, - "application/octet-stream".to_string(), - invalid_size, - 30, - 110, - )); - assert_eq!(state.ready_key_object_valid_until_ms(&proxy_session_id, &resource_id, "key", 30), None); - } - - let mut state = TransientPassthroughState::default(); - let resource = TransientResourceRef::new( - TransientResourceKind::Key, - "http://origin.example.com/live/valid.key", - b"secret", - 10, - 100, - Some("key".to_string()), - ); - let resource_id = resource.id.clone(); - state.upsert_resources([resource.clone()]); - let token = match state.begin_object_fetch(&proxy_session_id, &resource, "key", 20, 50) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("new AES key starts a cache fetch") - } - }; - assert!(state.mark_object_ready_if_current(&token, "application/octet-stream".to_string(), 16, 30, 110,)); - - assert_eq!(state.ready_key_object_valid_until_ms(&proxy_session_id, &resource_id, "key", 110), Some(110)); - assert_eq!(state.ready_key_object_valid_until_ms(&proxy_session_id, &resource_id, "key", 111), None); - } - - #[test] - fn finalized_key_readiness_uses_object_ttl_after_mapping_ttl() { - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - let mut state = TransientPassthroughState::default(); - let resource = TransientResourceRef::new( - TransientResourceKind::Key, - "http://origin.example.com/archive/key.key", - b"secret", - 0, - 20, - Some("key".to_string()), - ); - let resource_id = resource.id.clone(); - state.upsert_resources([resource.clone()]); - replace_with_finalized_manifest(&mut state, &resource_id, "key", 0); - let token = match state.begin_object_fetch(&proxy_session_id, &resource, "key", 10, 100) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("new finalized AES key starts a cache fetch") - } - }; - assert!(state.mark_object_ready_if_current(&token, "application/octet-stream".to_string(), 16, 15, 110)); - - assert_eq!(state.ready_key_object_valid_until_ms(&proxy_session_id, &resource_id, "key", 30), Some(110)); - assert_eq!(state.ready_key_object_valid_until_ms(&proxy_session_id, &resource_id, "key", 111), None); - } - - #[test] - fn manifest_refresh_does_not_extend_a_ready_key_revision_and_rotation_reports_the_displaced_object() { - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - let mut state = TransientPassthroughState::default(); - let first_resource = TransientResourceRef::new( - TransientResourceKind::Key, - "http://origin.example.com/live/key.key", - b"secret", - 10, - 100, - Some("key".to_string()), - ); - let resource_id = first_resource.id.clone(); - state.upsert_resources([first_resource.clone()]); - let first = match state.begin_object_fetch(&proxy_session_id, &first_resource, "key", 20, 50) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("first key revision starts a fetch") - } - }; - assert!(state.mark_object_ready_if_current(&first, "application/octet-stream".to_string(), 16, 30, 110,)); - - let refreshed_resource = TransientResourceRef::new( - TransientResourceKind::Key, - "http://origin.example.com/live/key.key", - b"secret", - 100, - 100, - Some("key".to_string()), - ); - assert_eq!(refreshed_resource.id, resource_id); - state.upsert_resources([refreshed_resource.clone()]); - let lookup_key = - TransientPassthroughState::transient_object_key(&proxy_session_id, &resource_id, "key".to_string()); - assert_eq!(state.object_cache.get(&lookup_key).map(|entry| entry.expires_at_ms), Some(110)); - - let second = match state.begin_object_fetch(&proxy_session_id, &refreshed_resource, "key", 111, 50) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("expired key revision starts a replacement fetch") - } - }; - let displaced = second.superseded_object().expect("replacement owns one bounded displaced object"); - assert_eq!(&displaced.key, first.cache_key()); - assert_eq!(displaced.content_length, 16); - assert_ne!(second.cache_key(), first.cache_key()); - } - - #[test] - fn stale_transient_fetch_token_cannot_overwrite_or_fail_a_new_generation() { - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - let mut state = TransientPassthroughState::default(); - let resource = TransientResourceRef::new( - TransientResourceKind::Key, - "http://origin.example.com/live/key.key", - b"secret", - 10, - 100, - Some("key".to_string()), - ); - state.upsert_resources([resource.clone()]); - let first = match state.begin_object_fetch(&proxy_session_id, &resource, "key", 20, 50) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("first key fetch starts") - } - }; - assert!(state.mark_object_failed_retryable_if_current(&first, 21, 0)); - let second = match state.begin_object_fetch(&proxy_session_id, &resource, "key", 22, 50) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("replacement key fetch starts") - } - }; - assert_ne!(first.cache_key(), second.cache_key()); - - assert!(!state.mark_object_ready_if_current(&first, "application/octet-stream".to_string(), 16, 23, 110,)); - assert!(state.mark_object_ready_if_current(&second, "application/octet-stream".to_string(), 16, 24, 110,)); - assert!(!state.mark_object_failed_retryable_if_current(&first, 25, 0)); - assert!(matches!( - state.object_cache.get(&second.lookup_key).map(|entry| &entry.status), - Some(TransientObjectCacheStatus::Ready { content_length: 16, .. }) - )); - } - - #[tokio::test] - async fn controlled_resource_mapping_replacement_invalidates_the_fetch_token_and_wakes_its_waiter() { - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - let mut state = TransientPassthroughState::default(); - let original = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin-a.example.com/live/segment.ts", - b"secret", - 10, - 100, - Some("ts".to_string()), - ); - let resource_id = original.id.clone(); - state.upsert_resources([original]); - let original = state.resources.get(&resource_id).expect("registered original mapping").clone(); - let token = match state.begin_object_fetch(&proxy_session_id, &original, "ts", 20, 50) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("new resource starts a cache fetch") - } - }; - let notifier = match state.begin_object_fetch(&proxy_session_id, &original, "ts", 21, 50) { - super::TransientObjectFetchDecision::Wait(notifier) => notifier, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Fetch(_) => { - panic!("concurrent request waits for the owned fill") - } - }; - let waiter = notifier.notified(); - tokio::pin!(waiter); - assert!(matches!(futures::poll!(&mut waiter), Poll::Pending)); - - let mut replacement = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin-b.example.com/live/segment.ts", - b"secret", - 22, - 100, - Some("ts".to_string()), - ); - replacement.id = resource_id; - state.upsert_resources([replacement]); - - assert!(!state.mark_object_ready_if_current(&token, "video/mp2t".to_string(), 7, 23, 122)); - assert!(matches!(futures::poll!(&mut waiter), Poll::Ready(()))); - assert!(!state.object_cache.contains_key(&token.lookup_key)); - let replacement = state.resources.get(&token.binding.resource_id).expect("replacement mapping").clone(); - let replacement_token = match state.begin_object_fetch(&proxy_session_id, &replacement, "ts", 24, 50) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("replacement mapping owns a new cache fill") - } - }; - assert_ne!(token.cache_key(), replacement_token.cache_key()); - assert_ne!(token.binding.revision, replacement_token.binding.revision); - } - - #[test] - fn failed_transient_metadata_has_finite_ttl_and_a_hard_count_bound() { - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - let mut state = TransientPassthroughState::new(100); - let mut oldest_lookup_key = None; - for index in 0..=MAX_FAILED_TRANSIENT_OBJECT_ENTRIES { - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - format!("http://origin.example.com/live/{index}.ts"), - b"secret", - 0, - 10_000, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - state.upsert_resources([resource]); - let resource = state.resources.get(&resource_id).expect("registered resource").clone(); - let token = match state.begin_object_fetch(&proxy_session_id, &resource, "ts", index as u64, 10_000) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("unique resource starts a cache fill") - } - }; - if index == 0 { - oldest_lookup_key = Some(token.lookup_key.clone()); - } - assert!(state.mark_object_failed_permanent_if_current(&token, index as u64, None)); - } - - assert_eq!( - state - .object_cache - .values() - .filter(|entry| matches!(entry.status, TransientObjectCacheStatus::FailedPermanent { .. })) - .count(), - MAX_FAILED_TRANSIENT_OBJECT_ENTRIES - ); - assert!(!state.object_cache.contains_key(&oldest_lookup_key.expect("oldest lookup key"))); - - let mut state = TransientPassthroughState::new(100); - let expiring = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/expiring.ts", - b"secret", - 1_000, - 10_000, - Some("ts".to_string()), - ); - let expiring_id = expiring.id.clone(); - state.upsert_resources([expiring]); - let expiring = state.resources.get(&expiring_id).expect("registered expiring resource").clone(); - let token = match state.begin_object_fetch(&proxy_session_id, &expiring, "ts", 1_010, 10_000) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("expiring resource starts a cache fill") - } - }; - assert!(state.mark_object_failed_retryable_if_current(&token, 1_020, 10)); - assert_eq!(state.object_cache.get(&token.lookup_key).map(|entry| entry.expires_at_ms), Some(1_120)); - state.upsert_resources([TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/expiring.ts", - b"secret", - 1_100, - 20_000, - Some("ts".to_string()), - )]); - assert_eq!( - state.object_cache.get(&token.lookup_key).map(|entry| entry.expires_at_ms), - Some(1_120), - "manifest refresh must not make failed metadata immortal" - ); - let removals = state.take_expired_object_removals_except(1_121, &HashSet::from([expiring_id]), 1); - assert_eq!(removals.len(), 1); - assert_eq!(&removals[0].key, token.cache_key()); - assert!(!state.object_cache.contains_key(&token.lookup_key)); - } - - #[test] - fn expired_resource_validity_rejects_a_late_fetch_commit() { - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - let mut state = TransientPassthroughState::default(); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/short.ts", - b"secret", - 10, - 10, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - state.upsert_resources([resource]); - state.replace_manifest_with_semantics( - format!("#EXTM3U\n#EXTINF:1,\n/hls/shared/live/session/lease/r/{}.ts\n", resource_id.0), - 10, - None, - ); - let resource = state.resources.get(&resource_id).expect("registered resource").clone(); - let token = match state.begin_object_fetch(&proxy_session_id, &resource, "ts", 15, 100) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("valid resource starts a cache fill") - } - }; - - assert!(!state.mark_object_ready_if_current(&token, "video/mp2t".to_string(), 7, 21, 115)); - assert!(!state.object_cache.contains_key(&token.lookup_key)); - } - - #[test] - fn finalized_manifest_resource_accepts_late_fetch_commit() { - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - let mut state = TransientPassthroughState::default(); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/archive/short.ts", - b"secret", - 10, - 10, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - state.upsert_resources([resource]); - replace_with_finalized_manifest(&mut state, &resource_id, "ts", 10); - let resource = state.resolve_current_resource(&resource_id, 15).expect("finalized resource resolves"); - let token = match state.begin_object_fetch(&proxy_session_id, &resource, "ts", 15, 100) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("valid finalized resource starts a cache fill") - } - }; - - replace_with_finalized_manifest(&mut state, &resource_id, "ts", 18); - assert_eq!(state.manifest_generation(), 1); - assert!(state.mark_object_ready_if_current(&token, "video/mp2t".to_string(), 7, 21, 121)); - assert!(state.object_cache.contains_key(&token.lookup_key)); - } - - #[test] - fn finalized_resource_replacement_still_rejects_stale_fetch_commit() { - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - let mut state = TransientPassthroughState::default(); - let original = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/archive/original.ts", - b"secret", - 10, - 10, - Some("ts".to_string()), - ); - let resource_id = original.id.clone(); - state.upsert_resources([original]); - replace_with_finalized_manifest(&mut state, &resource_id, "ts", 10); - let original = state.resolve_current_resource(&resource_id, 15).expect("original mapping"); - let token = match state.begin_object_fetch(&proxy_session_id, &original, "ts", 15, 100) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("original finalized resource starts a cache fill") - } - }; - let mut replacement = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://replacement.example.com/archive/replacement.ts", - b"secret", - 16, - 100, - Some("ts".to_string()), - ); - replacement.id = resource_id.clone(); - state.upsert_resources([replacement]); - replace_with_finalized_manifest(&mut state, &resource_id, "ts", 16); - - assert!(!state.mark_object_ready_if_current(&token, "video/mp2t".to_string(), 7, 21, 121)); - assert!(!state.object_cache.contains_key(&token.lookup_key)); - } - - #[test] - fn expired_finalized_object_is_refetched_instead_of_served_stale() { - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - let mut state = TransientPassthroughState::default(); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/archive/refetch.ts", - b"secret", - 0, - 10, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - state.upsert_resources([resource]); - replace_with_finalized_manifest(&mut state, &resource_id, "ts", 0); - let resource = state.resolve_current_resource(&resource_id, 5).expect("finalized mapping"); - let first = match state.begin_object_fetch(&proxy_session_id, &resource, "ts", 5, 5) { - super::TransientObjectFetchDecision::Fetch(token) => token, - super::TransientObjectFetchDecision::Ready | super::TransientObjectFetchDecision::Wait(_) => { - panic!("first cache fill starts") - } - }; - assert!(state.mark_object_ready_if_current( - &first, - "video/mp2t".to_string(), - 7, - 6, - transient_object_expires_at(6, 5), - )); - let lookup_key = TransientPassthroughState::transient_object_key(&proxy_session_id, &resource_id, "ts"); - - assert!(state.ready_object(&lookup_key, TransientResourceKind::Segment, 11).is_some()); - assert!(state.ready_object(&lookup_key, TransientResourceKind::Segment, 12).is_none()); - let resource = state.resolve_current_resource(&resource_id, 12).expect("mapping outlives cached bytes"); - assert!(matches!( - state.begin_object_fetch(&proxy_session_id, &resource, "ts", 12, 5), - super::TransientObjectFetchDecision::Fetch(_) - )); - } - - #[test] - fn finalized_catchup_has_no_resource_boundary_at_cache_duration() { - let proxy_session_id = ProxySessionId("proxy-session".to_string()); - let mut origin_body = String::from("#EXTM3U\n#EXT-X-TARGETDURATION:7\n#EXT-X-PLAYLIST-TYPE:EVENT\n"); - let durations_ms = (0..46) - .map(|index| match index { - 0..44 => 6_700, - 44 => 6_720, - _ => 6_000, - }) - .collect::>(); - for (index, duration_ms) in durations_ms.iter().enumerate() { - writeln!(origin_body, "#EXTINF:{}.{:03},\n{index}.ts", duration_ms / 1_000, duration_ms % 1_000) - .expect("synthetic manifest renders"); - } - origin_body.push_str("#EXT-X-ENDLIST\n"); - assert_eq!(durations_ms.iter().take(45).sum::(), 301_520); - let lifecycle = parse_manifest_semantics(&origin_body).lifecycle(); - assert_eq!(lifecycle, HlsManifestLifecycle::Finalized); - let rewritten = TransientManifestRewriter::rewrite( - &origin_body, - "http://origin.example.com/archive/index.m3u8", - &proxy_session_id, - b"secret", - 0, - 300_000, - ); - let before_boundary = rewritten - .resources - .iter() - .find(|resource| resource.resolved_origin_uri.ends_with("/44.ts")) - .expect("segment before boundary") - .id - .clone(); - let after_boundary = rewritten - .resources - .iter() - .find(|resource| resource.resolved_origin_uri.ends_with("/45.ts")) - .expect("segment after boundary") - .id - .clone(); - let mut state = TransientPassthroughState::default(); - state.upsert_resources(rewritten.resources); - state.replace_manifest_with_semantics(rewritten.body, 0, Some(durations_ms.iter().sum())); - - assert!(state.resolve_current_resource(&before_boundary, 301_520).is_some()); - let after = state - .resolve_current_resource(&after_boundary, 301_520) - .expect("segment after cache-duration boundary remains resolvable"); - assert!(matches!( - state.begin_object_fetch(&proxy_session_id, &after, "ts", 301_520, 300_000), - super::TransientObjectFetchDecision::Fetch(_) - )); - } - - #[test] - fn finalized_prune_keeps_resources_referenced_by_current_manifest() { - let mut state = TransientPassthroughState::default(); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/seg.ts", - b"secret", - 0, - 10, - Some("ts".to_string()), - ); - let resource_id = resource.id.clone(); - state.upsert_resources([resource]); - replace_with_finalized_manifest(&mut state, &resource_id, "ts", 0); - - state.prune_expired(20); - - assert!(state.resources.contains_key(&resource_id)); - } -} +mod tests; + +mod generations; +mod identity; +mod manifests; +mod objects; +mod resources; +use identity::CommittedTransientManifestValidity; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use identity::{ + build_transient_resource_id, TransientObjectCacheStatus, TransientObjectFetchDecision, TransientObjectRemoval, + TransientObjectUnavailableState, TransientResourceKind, +}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(crate) use identity::{ + extract_transient_resource_ids, HlsTransientManifestCommitError, HlsTransientManifestFootprint, + RollingEventAppendContext, TransientManifestLeaseBinding, +}; diff --git a/backend/hls/src/transient/generations.rs b/backend/hls/src/transient/generations.rs new file mode 100644 index 000000000..e3a80e290 --- /dev/null +++ b/backend/hls/src/transient/generations.rs @@ -0,0 +1,228 @@ +use super::{ + CacheAccessState, HlsAccessLeaseId, HlsPublishedTransientResourceIds, HlsTransientManifestTemplate, + RetainedFinalizedManifestGeneration, TransientManifestGeneration, TransientManifestIdentity, + TransientManifestLeaseBinding, TransientObjectCacheEntry, TransientObjectCacheKey, TransientPassthroughState, + TransientResourceId, TransientResourceRef, ESTIMATED_ARC_ALLOCATION_OVERHEAD_BYTES, + ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES, +}; +use std::{collections::HashSet, sync::Arc}; +use tuliprox_parser::hls::origin_manifest::HlsManifestWindowPolicy; + +impl TransientPassthroughState { + #[cfg(test)] + pub(crate) const fn manifest_generation(&self) -> u64 { self.manifest_generation } + + pub const fn current_finalized_manifest_generation(&self) -> Option { + self.current_finalized_manifest_generation + } + + pub const fn last_manifest_commit_identity( + &self, + ) -> Option { + self.last_manifest_commit_identity + } + + pub(crate) fn record_manifest_commit_identity( + &mut self, + identity: super::super::media_reserve::HlsManifestCommitIdentity, + ) { + self.last_manifest_commit_identity = Some(identity); + } + + pub(crate) const fn last_manifest_finalized(&self) -> bool { self.last_manifest_validity.is_finalized() } + + pub fn last_manifest_template(&self) -> Option> { + self.last_manifest_template.clone() + } + + pub fn last_manifest_published_resource_ids(&self) -> HlsPublishedTransientResourceIds { + HlsPublishedTransientResourceIds::from_shared(Arc::clone(&self.last_manifest_resource_ids)) + } + + pub(crate) fn merge_current_published_resource_ids( + &self, + previous: &HlsPublishedTransientResourceIds, + next: HlsPublishedTransientResourceIds, + now_ms: u64, + ) -> HlsPublishedTransientResourceIds { + if Arc::ptr_eq(&previous.0, &next.0) + || previous.0.iter().all(|resource_id| { + next.contains(resource_id) + || !self + .resources + .get(resource_id) + .is_some_and(|resource| self.resource_is_valid_at(resource, now_ms)) + }) + { + return next; + } + let mut merged = next.0.as_ref().clone(); + merged.extend( + previous + .0 + .iter() + .filter(|resource_id| { + self.resources.get(*resource_id).is_some_and(|resource| self.resource_is_valid_at(resource, now_ms)) + }) + .cloned(), + ); + HlsPublishedTransientResourceIds(Arc::new(merged)) + } + + pub const fn last_manifest_window_policy(&self) -> HlsManifestWindowPolicy { + self.last_manifest_validity.window_policy() + } + + pub(crate) const fn last_manifest_valid_until_ms(&self) -> Option { + self.last_manifest_validity.valid_until_ms() + } + + #[cfg(test)] + pub(crate) fn current_manifest_resource_ids(&self) -> &HashSet { + self.last_manifest_resource_ids.as_ref() + } + + pub(crate) fn has_finalized_manifest_generation(&self, generation: TransientManifestGeneration) -> bool { + self.finalized_manifest_generations.contains_key(&generation) + } + + #[cfg(test)] + pub(crate) fn finalized_manifest_generation_count(&self) -> usize { self.finalized_manifest_generations.len() } + + #[cfg(test)] + pub(crate) fn finalized_manifest_lease_binding_count(&self) -> usize { + self.finalized_manifest_lease_bindings.len() + } + + pub(crate) fn bind_finalized_manifest_generation(&mut self, binding: TransientManifestLeaseBinding) -> bool { + if !self.finalized_manifest_generations.contains_key(&binding.manifest_generation) { + return false; + } + self.finalized_manifest_lease_bindings.insert(binding); + self.prune_unreferenced_finalized_manifest_generations(); + true + } + + pub(crate) fn release_finalized_manifest_generations( + &mut self, + lease_id: &HlsAccessLeaseId, + lease_issued_at_ms: u64, + ) -> bool { + let previous_len = self.finalized_manifest_lease_bindings.len(); + self.finalized_manifest_lease_bindings + .retain(|binding| binding.lease_id != *lease_id || binding.lease_issued_at_ms != lease_issued_at_ms); + let released = self.finalized_manifest_lease_bindings.len() != previous_len; + if !released { + return false; + } + self.prune_unreferenced_finalized_manifest_generations(); + true + } + + pub(crate) fn reconcile_finalized_manifest_lease_bindings(&mut self, bindings: &[TransientManifestLeaseBinding]) { + self.finalized_manifest_lease_bindings = bindings + .iter() + .filter(|binding| self.finalized_manifest_generations.contains_key(&binding.manifest_generation)) + .cloned() + .collect(); + self.prune_unreferenced_finalized_manifest_generations(); + } + + pub(super) fn prune_unreferenced_finalized_manifest_generations(&mut self) { + let current_generation = self.current_finalized_manifest_generation(); + let referenced_generations = self + .finalized_manifest_lease_bindings + .iter() + .map(|binding| binding.manifest_generation) + .collect::>(); + let removed = self + .finalized_manifest_generations + .extract_if(|generation, _| { + Some(*generation) != current_generation && !referenced_generations.contains(generation) + }) + .map(|(_, generation)| generation) + .collect::>(); + for generation in removed { + self.decrement_protected_resource_refcounts(&generation.resource_ids); + } + } + + pub(super) fn finalized_generation_for_identity( + &self, + identity: TransientManifestIdentity, + ) -> Option { + let generation = self.current_finalized_manifest_generation?; + self.finalized_manifest_generations + .get(&generation) + .is_some_and(|retained| retained.identity == identity) + .then_some(generation) + } + + pub(super) fn generation_is_lease_referenced(&self, generation: TransientManifestGeneration) -> bool { + self.finalized_manifest_lease_bindings.iter().any(|binding| binding.manifest_generation == generation) + } + + pub(super) fn insert_finalized_manifest_generation( + &mut self, + generation: TransientManifestGeneration, + retained: RetainedFinalizedManifestGeneration, + ) { + let resource_ids = Arc::clone(&retained.resource_ids); + if let Some(previous) = self.finalized_manifest_generations.insert(generation, retained) { + self.decrement_protected_resource_refcounts(&previous.resource_ids); + } + for resource_id in resource_ids.iter() { + let refcount = self.protected_finalized_resource_refcounts.entry(resource_id.clone()).or_default(); + *refcount = refcount.saturating_add(1); + } + } + + pub(super) fn decrement_protected_resource_refcounts(&mut self, resource_ids: &HashSet) { + for resource_id in resource_ids { + let remove = self.protected_finalized_resource_refcounts.get_mut(resource_id).is_some_and(|refcount| { + *refcount = refcount.saturating_sub(1); + *refcount == 0 + }); + if remove { + self.protected_finalized_resource_refcounts.remove(resource_id); + } + } + } +} + +pub(super) fn estimated_transient_resource_entry_bytes( + resource_id: &TransientResourceId, + resource: &TransientResourceRef, +) -> usize { + std::mem::size_of::<(TransientResourceId, TransientResourceRef)>() + .saturating_add(resource_id.0.len()) + .saturating_add(resource.id.0.len()) + .saturating_add(resource.resolved_origin_uri.len()) + .saturating_add(resource.content_type_hint.as_ref().map_or(0, String::len)) + .saturating_add(resource.file_ext_hint.as_ref().map_or(0, String::len)) + .saturating_add(std::mem::size_of::()) + .saturating_add(ESTIMATED_ARC_ALLOCATION_OVERHEAD_BYTES) + .saturating_add(ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES) +} + +pub(super) fn estimated_transient_resource_id_set_bytes(resource_ids: &HashSet) -> usize { + resource_ids.iter().fold(0_usize, |total, resource_id| { + total.saturating_add( + std::mem::size_of::() + .saturating_add(resource_id.0.len()) + .saturating_add(ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES), + ) + }) +} + +pub(super) fn estimated_transient_object_entry_bytes(entry: &TransientObjectCacheEntry) -> usize { + std::mem::size_of::<(TransientObjectCacheKey, TransientObjectCacheEntry)>() + .saturating_add(entry.key.stable_value().len()) + .saturating_add(entry.content_type.len()) + .saturating_add(entry.binding.resource_id.0.len()) + .saturating_add(entry.binding.resolved_origin_uri.len()) + .saturating_add(entry.binding.file_extension.len()) + .saturating_add(std::mem::size_of::()) + .saturating_add(ESTIMATED_ARC_ALLOCATION_OVERHEAD_BYTES) + .saturating_add(ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES) +} diff --git a/backend/hls/src/transient/identity.rs b/backend/hls/src/transient/identity.rs new file mode 100644 index 000000000..521426ce0 --- /dev/null +++ b/backend/hls/src/transient/identity.rs @@ -0,0 +1,392 @@ +use super::{ + default_content_type_for_transient_ext, CacheAccessState, HlsAccessLeaseId, HlsManifestLimitViolation, + HlsPublishedTransientResourceIds, HlsTransientManifestTemplate, TransientManifestGeneration, + TransientManifestIdentity, TransientObjectCacheEntry, TransientObjectCacheKey, TransientObjectFetchToken, + TransientObjectResourceBinding, TransientResourceFile, TransientResourceId, TransientResourceRef, + TransientResourceRevision, TRANSIENT_RESOURCE_ID_KEY_CONTEXT, TRANSIENT_RESOURCE_ID_LEN, +}; +use crate::transient_manifest::TransientRewriteCheckpoint; +use axum::http::StatusCode; +use base64::{engine::general_purpose, Engine as _}; +use std::{collections::HashSet, fmt, sync::Arc}; +use tokio::sync::Notify; +use tuliprox_parser::hls::origin_manifest::{HlsManifestLifecycle, HlsManifestWindowPolicy, ParsedManifestSemantics}; + +impl HlsPublishedTransientResourceIds { + pub fn from_manifest_body(body: &str) -> Self { Self(Arc::new(extract_transient_resource_ids(body))) } + + pub(super) fn from_shared(resource_ids: Arc>) -> Self { Self(resource_ids) } + + pub fn contains(&self, resource_id: &TransientResourceId) -> bool { self.0.contains(resource_id) } +} + +impl TransientManifestGeneration { + #[cfg(test)] + pub(crate) const fn for_test(generation: u64) -> Self { Self(generation) } +} + +/// Exact lease incarnation that currently publishes one finalized transient manifest generation. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub(crate) struct TransientManifestLeaseBinding { + pub(crate) lease_id: HlsAccessLeaseId, + pub(crate) lease_issued_at_ms: u64, + pub(crate) manifest_generation: TransientManifestGeneration, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) struct HlsTransientManifestFootprint { + pub(crate) resources: usize, + pub(crate) resource_entries: usize, + pub(crate) rewritten_bytes: usize, + pub(crate) origin_uri_bytes: usize, + pub(crate) retained_finalized_generations: usize, + pub(crate) finalized_generation_memberships: usize, + pub(crate) estimated_metadata_bytes: usize, +} + +/// Rejects a transient commit before publication when its complete client representation is unavailable. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) enum HlsTransientManifestCommitError { + LocalRepresentationLimit(HlsManifestLimitViolation), + SnapshotUnavailable, + CommitGenerationExhausted, +} + +impl From for HlsTransientManifestCommitError { + fn from(violation: HlsManifestLimitViolation) -> Self { Self::LocalRepresentationLimit(violation) } +} + +pub(crate) struct RollingEventAppendContext { + pub(crate) suffix_offset: usize, + pub(crate) rewritten_prefix: Arc, + pub(crate) template: Arc, + pub(crate) checkpoint: TransientRewriteCheckpoint, +} + +impl TransientManifestLeaseBinding { + pub(crate) const fn new( + lease_id: HlsAccessLeaseId, + lease_issued_at_ms: u64, + manifest_generation: TransientManifestGeneration, + ) -> Self { + Self { lease_id, lease_issued_at_ms, manifest_generation } + } +} + +#[derive(Clone, Copy)] +pub(super) enum TransientResourceValidityScope<'a> { + PublishedSession, + AccessLease { + lease_id: &'a HlsAccessLeaseId, + lease_issued_at_ms: u64, + published_resource_ids: &'a HlsPublishedTransientResourceIds, + }, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum CommittedTransientManifestValidity { + Rolling { valid_until_ms: Option, window_policy: HlsManifestWindowPolicy }, + Finalized, +} + +impl CommittedTransientManifestValidity { + pub(super) const fn from_semantics( + semantics: ParsedManifestSemantics, + rendered_at_ms: u64, + playlist_duration_ms: Option, + ) -> Self { + match semantics.lifecycle() { + HlsManifestLifecycle::Rolling => Self::Rolling { + valid_until_ms: match playlist_duration_ms { + Some(duration_ms) => Some(rendered_at_ms.saturating_add(duration_ms)), + None => None, + }, + window_policy: semantics.window_policy(), + }, + HlsManifestLifecycle::Finalized => Self::Finalized, + } + } + + pub(super) const fn is_finalized(self) -> bool { matches!(self, Self::Finalized) } + + pub(super) const fn window_policy(self) -> HlsManifestWindowPolicy { + match self { + Self::Rolling { window_policy, .. } => window_policy, + Self::Finalized => HlsManifestWindowPolicy::PreserveFullManifest, + } + } + + pub(super) const fn valid_until_ms(self) -> Option { + match self { + Self::Rolling { valid_until_ms, .. } => valid_until_ms, + Self::Finalized => None, + } + } +} + +/// Builds a deterministic opaque transient resource ID from the concrete origin fetch URI. +/// +/// `resolved_origin_uri` must be the concrete resource URI after manifest-relative URL resolution against the final +/// manifest URL. In provider-url-failover/redirect flows this may intentionally include the selected mirror or final +/// CDN/origin host. Do not make this ID input host-neutral unless `TransientResourceRef` keeps a separate concrete fetch +/// URI and tests prove relative segment/MAP/key downloads still use that concrete URI. +/// +pub fn build_transient_resource_id( + resolved_origin_uri: &str, + reverse_proxy_rewrite_secret: &[u8], +) -> TransientResourceId { + let key = blake3::derive_key(TRANSIENT_RESOURCE_ID_KEY_CONTEXT, reverse_proxy_rewrite_secret); + let digest = blake3::keyed_hash(&key, resolved_origin_uri.as_bytes()); + let token = general_purpose::URL_SAFE_NO_PAD.encode(digest.as_bytes()); + TransientResourceId(token.chars().take(TRANSIENT_RESOURCE_ID_LEN).collect()) +} + +/// Transient origin resource category used for direct passthrough streaming. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum TransientResourceKind { + Segment, + Key, + Map, + Part, + Other, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum TransientObjectCacheStatus { + Fetching { started_at_ms: u64 }, + Ready { content_length: u64, ready_at_ms: u64 }, + FailedRetryable { failed_at_ms: u64, retry_after_ms: u64 }, + FailedPermanent { failed_at_ms: u64, status: Option }, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum TransientObjectUnavailableState { + Missing, + Fetching, + FailedRetryable { retry_after_ms: u64 }, + FailedPermanent, +} + +#[derive(Clone)] +pub enum TransientObjectFetchDecision { + Ready, + Fetch(Box), + Wait(Arc), +} + +impl TransientObjectFetchToken { + pub fn cache_key(&self) -> &TransientObjectCacheKey { &self.cache_key } + + pub fn superseded_object(&self) -> Option<&TransientObjectRemoval> { self.superseded_object.as_ref() } +} + +impl TransientObjectResourceBinding { + pub(super) fn from_resource(resource: &TransientResourceRef, file_extension: &str) -> Self { + Self { + resource_id: resource.id.clone(), + revision: resource.revision, + kind: resource.kind, + encrypted_media: resource.encrypted_media, + resolved_origin_uri: resource.resolved_origin_uri.clone(), + file_extension: file_extension.to_string(), + } + } + + pub(super) fn matches_resource_identity(&self, resource: &TransientResourceRef) -> bool { + self.resource_id == resource.id + && self.revision == resource.revision + && self.kind == resource.kind + && self.encrypted_media == resource.encrypted_media + && self.resolved_origin_uri == resource.resolved_origin_uri + && resource.file_ext_hint.as_deref().is_none_or(|extension| extension == self.file_extension) + } +} + +impl TransientObjectCacheEntry { + pub(super) fn new_fetching( + key: TransientObjectCacheKey, + binding: TransientObjectResourceBinding, + now_ms: u64, + expires_at_ms: u64, + content_type: String, + ) -> Self { + Self { + key, + status: TransientObjectCacheStatus::Fetching { started_at_ms: now_ms }, + content_type, + created_at_ms: now_ms, + last_accessed_at_ms: now_ms, + expires_at_ms, + access: Arc::new(CacheAccessState::new()), + binding, + } + } + + pub fn is_ready_at(&self, now_ms: u64) -> bool { + matches!(self.status, TransientObjectCacheStatus::Ready { .. }) && self.expires_at_ms >= now_ms + } + + pub fn ready_content_length(&self) -> Option { + match self.status { + TransientObjectCacheStatus::Ready { content_length, .. } => Some(content_length), + TransientObjectCacheStatus::Fetching { .. } + | TransientObjectCacheStatus::FailedRetryable { .. } + | TransientObjectCacheStatus::FailedPermanent { .. } => None, + } + } +} + +impl fmt::Debug for TransientObjectCacheEntry { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TransientObjectCacheEntry") + .field("key", &self.key) + .field("status", &self.status) + .field("content_type", &self.content_type) + .field("created_at_ms", &self.created_at_ms) + .field("last_accessed_at_ms", &self.last_accessed_at_ms) + .field("expires_at_ms", &self.expires_at_ms) + .field("active_readers", &self.access.active_readers()) + .field("resource_revision", &self.binding.revision) + .finish() + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct TransientObjectRemoval { + pub key: TransientObjectCacheKey, + pub content_length: u64, +} + +impl TransientResourceRef { + pub fn new( + kind: TransientResourceKind, + resolved_origin_uri: impl Into, + reverse_proxy_rewrite_secret: &[u8], + now_ms: u64, + ttl_ms: u64, + file_ext_hint: Option, + ) -> Self { + let resolved_origin_uri = resolved_origin_uri.into(); + let id = build_transient_resource_id(&resolved_origin_uri, reverse_proxy_rewrite_secret); + Self { + id, + kind, + encrypted_media: false, + resolved_origin_uri, + content_type_hint: file_ext_hint + .as_deref() + .and_then(default_content_type_for_transient_ext) + .map(str::to_string), + file_ext_hint, + created_at_ms: now_ms, + expires_at_ms: now_ms.saturating_add(ttl_ms), + access: Arc::new(CacheAccessState::new()), + revision: TransientResourceRevision::default(), + } + } + + pub fn is_valid_at(&self, now_ms: u64) -> bool { now_ms <= self.expires_at_ms } + + pub(super) fn refresh_from(&mut self, next: TransientResourceRef) { + self.kind = next.kind; + // A URI reused across key transitions is ambiguous while an older cache object can + // still exist. Preserve the conservative encrypted classification until eviction. + self.encrypted_media |= next.encrypted_media; + self.resolved_origin_uri = next.resolved_origin_uri; + self.content_type_hint = next.content_type_hint; + self.file_ext_hint = next.file_ext_hint; + self.expires_at_ms = next.expires_at_ms; + self.revision = next.revision; + } + + pub(super) fn has_same_cache_identity(&self, other: &Self) -> bool { + self.id == other.id + && self.kind == other.kind + && self.encrypted_media == (self.encrypted_media || other.encrypted_media) + && self.resolved_origin_uri == other.resolved_origin_uri + && self.content_type_hint == other.content_type_hint + && self.file_ext_hint == other.file_ext_hint + } + + pub fn active_readers(&self) -> u32 { self.access.active_readers() } +} + +impl fmt::Debug for TransientResourceRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TransientResourceRef") + .field("id", &self.id) + .field("kind", &self.kind) + .field("encrypted_media", &self.encrypted_media) + .field("resolved_origin_uri", &"") + .field("content_type_hint", &self.content_type_hint) + .field("file_ext_hint", &self.file_ext_hint) + .field("created_at_ms", &self.created_at_ms) + .field("expires_at_ms", &self.expires_at_ms) + .field("active_readers", &self.access.active_readers()) + .field("revision", &self.revision) + .finish() + } +} + +pub(super) fn transient_manifest_identity( + body: &str, + resource_ids: &HashSet, +) -> TransientManifestIdentity { + let body_hash = *blake3::hash(body.as_bytes()).as_bytes(); + let mut ordered_resource_ids = resource_ids.iter().map(|resource_id| resource_id.0.as_str()).collect::>(); + ordered_resource_ids.sort_unstable(); + let mut resource_hasher = blake3::Hasher::new(); + for resource_id in ordered_resource_ids { + resource_hasher.update(&u64::try_from(resource_id.len()).unwrap_or(u64::MAX).to_le_bytes()); + resource_hasher.update(resource_id.as_bytes()); + } + TransientManifestIdentity { body_hash, resource_set_hash: *resource_hasher.finalize().as_bytes() } +} + +pub(super) fn complete_manifest_append_boundary(body: &str) -> bool { + body.ends_with('\n') + && body.lines().rev().map(str::trim).find(|line| !line.is_empty()).is_some_and(|line| !line.starts_with('#')) +} + +pub(crate) fn extract_transient_resource_ids(body: &str) -> HashSet { + let mut resource_ids = HashSet::new(); + for line in body.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + if line.starts_with('#') { + for uri in tag_uri_attributes(line) { + if let Some(resource_id) = transient_resource_id_from_shared_route(uri) { + resource_ids.insert(resource_id); + } + } + } else if let Some(resource_id) = transient_resource_id_from_shared_route(line) { + resource_ids.insert(resource_id); + } + } + resource_ids +} + +/// Yields the quoted URI values carried by a tag line (`URI="..."`). +fn tag_uri_attributes(line: &str) -> impl Iterator { + line.split("URI=\"").skip(1).filter_map(|tail| tail.split('"').next()) +} + +/// Extracts one opaque transient resource ID from a canonical own `/r/{id}.{ext}` route. +/// +/// Unlike a naive `split("/r/")`, this only recognises proxy-owned shared-HLS locators: it +/// ignores `/r/` occurrences inside comments (non-URI tag content is never yielded as a URI), +/// query strings or fragments, and rejects absolute foreign URIs by requiring a path-only +/// `/hls/shared/live/` prefix before the route segment. +fn transient_resource_id_from_shared_route(uri: &str) -> Option { + let path = uri.split(['?', '#']).next().unwrap_or(""); + if path.contains("://") { + return None; + } + let (prefix, remainder) = path.rsplit_once("/r/")?; + if !prefix.contains("/hls/shared/live/") { + return None; + } + TransientResourceFile::parse(remainder).map(|file| file.resource_id) +} diff --git a/backend/hls/src/transient/manifests.rs b/backend/hls/src/transient/manifests.rs new file mode 100644 index 000000000..dd82b9495 --- /dev/null +++ b/backend/hls/src/transient/manifests.rs @@ -0,0 +1,532 @@ +use super::{ + ensure_manifest_limit, extend_hls_transient_manifest_template, extract_transient_resource_ids, + generations::{ + estimated_transient_object_entry_bytes, estimated_transient_resource_entry_bytes, + estimated_transient_resource_id_set_bytes, + }, + identity::{complete_manifest_append_boundary, transient_manifest_identity, CommittedTransientManifestValidity}, + parse_hls_transient_manifest_template, HlsManifestLimitKind, HlsManifestLimitViolation, + HlsTransientManifestCommitError, HlsTransientManifestFootprint, HlsTransientManifestTemplate, + RetainedFinalizedManifestGeneration, RollingEventAppendContext, RollingEventRewriteSeed, + TransientManifestFootprintCandidate, TransientManifestGeneration, TransientManifestIdentity, + TransientManifestLeaseBinding, TransientManifestReplacement, TransientObjectCacheKey, TransientPassthroughState, + TransientResourceId, TransientResourceRef, ESTIMATED_ARC_ALLOCATION_OVERHEAD_BYTES, + ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES, MAX_ESTIMATED_TRANSIENT_METADATA_BYTES, + MAX_RETAINED_FINALIZED_MANIFEST_GENERATIONS, MAX_TRANSIENT_GENERATION_MEMBERSHIPS, + MAX_TRANSIENT_MANIFEST_RESOURCES, MAX_TRANSIENT_ORIGIN_URI_BYTES_PER_SESSION, + MAX_TRANSIENT_RESOURCE_ENTRIES_PER_SESSION, MAX_TRANSIENT_REWRITTEN_MANIFEST_BYTES, +}; +use crate::transient_manifest::TransientRewriteCheckpoint; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; +use tokio::sync::Notify; +use tuliprox_parser::hls::origin_manifest::{HlsManifestLifecycle, HlsManifestWindowPolicy, ParsedManifestSemantics}; + +impl TransientPassthroughState { + pub fn new(resource_ttl_ms: u64) -> Self { + Self { + resources: HashMap::new(), + object_cache: HashMap::new(), + object_fetch_notifiers: HashMap::new(), + next_resource_revision: 1, + next_object_fetch_generation: 0, + manifest_generation: 0, + last_manifest_commit_identity: None, + current_finalized_manifest_generation: None, + last_manifest_validity: CommittedTransientManifestValidity::Rolling { + valid_until_ms: None, + window_policy: HlsManifestWindowPolicy::ApplyLiveWindow, + }, + last_manifest_resource_ids: Arc::new(HashSet::new()), + last_manifest_template: None, + rolling_event_rewrite_seed: None, + finalized_manifest_generations: HashMap::new(), + finalized_manifest_lease_bindings: HashSet::new(), + protected_finalized_resource_refcounts: HashMap::new(), + last_manifest_body: None, + last_manifest_rendered_at_ms: None, + last_manifest_playlist_duration_ms: None, + resource_ttl_ms, + } + } + + pub fn set_resource_ttl_ms(&mut self, resource_ttl_ms: u64) { self.resource_ttl_ms = resource_ttl_ms; } + + #[cfg(any(test, feature = "test-support"))] + pub fn replace_manifest_with_semantics( + &mut self, + body: String, + rendered_at_ms: u64, + playlist_duration_ms: Option, + ) { + let semantics = tuliprox_parser::hls::origin_manifest::parse_manifest_semantics(&body); + if self.replace_manifest_common(body, rendered_at_ms, playlist_duration_ms, semantics).is_ok() { + self.last_manifest_commit_identity = + Some(super::super::media_reserve::HlsManifestCommitIdentity::committed( + self.manifest_generation, + rendered_at_ms, + )); + } + } + + pub(crate) fn commit_rewritten_manifest_with_semantics( + &mut self, + body: String, + resources: Vec, + rendered_at_ms: u64, + playlist_duration_ms: Option, + semantics: ParsedManifestSemantics, + ) -> Result { + let manifest_resource_ids = Arc::new(extract_transient_resource_ids(&body)); + let manifest_identity = transient_manifest_identity(&body, &manifest_resource_ids); + let reusable_finalized_generation = semantics + .lifecycle() + .is_finalized() + .then(|| self.finalized_generation_for_identity(manifest_identity)) + .flatten(); + self.ensure_manifest_generation_available(reusable_finalized_generation)?; + let resource_count = resources.len().max(manifest_resource_ids.len()); + Self::validate_manifest_shape(body.len(), resource_count)?; + let template = if let Some(generation) = reusable_finalized_generation { + self.finalized_manifest_generations.get(&generation).and_then(|retained| retained.template.clone()) + } else { + parse_hls_transient_manifest_template(&body)? + } + .ok_or(HlsTransientManifestCommitError::SnapshotUnavailable)?; + let footprint = self.validate_manifest_footprint(&TransientManifestFootprintCandidate { + rewritten_bytes: body.len(), + resource_count, + resources: &resources, + lifecycle: semantics.lifecycle(), + resource_ids: &manifest_resource_ids, + reusable_finalized_generation, + template: Some(&template), + })?; + self.upsert_resources(resources); + self.replace_manifest_common_with_resource_ids(TransientManifestReplacement { + body: Arc::from(body), + rendered_at_ms, + playlist_duration_ms, + semantics, + resource_ids: manifest_resource_ids, + identity: manifest_identity, + reusable_finalized_generation, + template: Some(template), + })?; + Ok(footprint) + } + + pub(crate) fn commit_incremental_rewritten_manifest_with_semantics( + &mut self, + body: String, + appended_resources: Vec, + template: Arc, + rendered_at_ms: u64, + resource_ttl_ms: u64, + semantics: ParsedManifestSemantics, + ) -> Result { + self.ensure_manifest_generation_available(None)?; + let mut manifest_resource_ids = std::mem::take(&mut self.last_manifest_resource_ids); + let mut inserted_resource_ids = Vec::new(); + for resource in &appended_resources { + if Arc::make_mut(&mut manifest_resource_ids).insert(resource.id.clone()) { + inserted_resource_ids.push(resource.id.clone()); + } + } + let manifest_identity = if semantics.lifecycle().is_finalized() { + transient_manifest_identity(&body, &manifest_resource_ids) + } else { + TransientManifestIdentity::default() + }; + let reusable_finalized_generation = semantics + .lifecycle() + .is_finalized() + .then(|| self.finalized_generation_for_identity(manifest_identity)) + .flatten(); + let footprint = Self::validate_manifest_shape(body.len(), manifest_resource_ids.len()).and_then(|()| { + self.validate_manifest_footprint(&TransientManifestFootprintCandidate { + rewritten_bytes: body.len(), + resource_count: manifest_resource_ids.len(), + resources: &appended_resources, + lifecycle: semantics.lifecycle(), + resource_ids: &manifest_resource_ids, + reusable_finalized_generation, + template: Some(&template), + }) + }); + let footprint = match footprint { + Ok(footprint) => footprint, + Err(violation) => { + let resource_ids = Arc::make_mut(&mut manifest_resource_ids); + for resource_id in inserted_resource_ids { + resource_ids.remove(&resource_id); + } + self.last_manifest_resource_ids = manifest_resource_ids; + return Err(violation); + } + }; + self.upsert_resources(appended_resources); + let expires_at_ms = rendered_at_ms.saturating_add(resource_ttl_ms); + for resource_id in manifest_resource_ids.iter() { + if let Some(resource) = self.resources.get_mut(resource_id) { + resource.expires_at_ms = expires_at_ms; + } + } + let playlist_duration_ms = Some(template.playlist_duration_ms()); + self.replace_manifest_common_with_resource_ids(TransientManifestReplacement { + body: Arc::from(body), + rendered_at_ms, + playlist_duration_ms, + semantics, + resource_ids: manifest_resource_ids, + identity: manifest_identity, + reusable_finalized_generation, + template: Some(template), + })?; + Ok(footprint) + } + + pub(crate) fn rolling_event_append_context( + &self, + origin_body: &str, + final_manifest_url: &str, + rewrite_secret: &[u8], + ) -> Option { + let seed = self.rolling_event_rewrite_seed.as_ref()?; + if seed.final_manifest_url_hash != *blake3::hash(final_manifest_url.as_bytes()).as_bytes() + || seed.rewrite_secret_hash != *blake3::hash(rewrite_secret).as_bytes() + || !complete_manifest_append_boundary(&seed.origin_body) + || origin_body.len() <= seed.origin_body.len() + || !origin_body.starts_with(seed.origin_body.as_ref()) + { + return None; + } + Some(RollingEventAppendContext { + suffix_offset: seed.origin_body.len(), + rewritten_prefix: Arc::clone(&seed.rewritten_body), + template: Arc::clone(&seed.template), + checkpoint: seed.checkpoint.clone(), + }) + } + + pub(crate) fn extend_rolling_event_template( + previous: &Arc, + rewritten_suffix: &str, + ) -> Result>, HlsManifestLimitViolation> { + extend_hls_transient_manifest_template(previous, rewritten_suffix) + } + + pub(crate) fn record_rolling_event_rewrite_seed( + &mut self, + origin_body: &str, + final_manifest_url: &str, + rewrite_secret: &[u8], + checkpoint: TransientRewriteCheckpoint, + retained_metadata_bytes: usize, + eligible: bool, + ) { + let retained_with_origin_body = retained_metadata_bytes + .saturating_add(origin_body.len()) + .saturating_add(std::mem::size_of::()) + .saturating_add(ESTIMATED_ARC_ALLOCATION_OVERHEAD_BYTES); + if !eligible || retained_with_origin_body > MAX_ESTIMATED_TRANSIENT_METADATA_BYTES { + self.rolling_event_rewrite_seed = None; + return; + } + let (Some(rewritten_body), Some(template)) = (&self.last_manifest_body, &self.last_manifest_template) else { + self.rolling_event_rewrite_seed = None; + return; + }; + self.rolling_event_rewrite_seed = Some(RollingEventRewriteSeed { + origin_body: Arc::from(origin_body), + final_manifest_url_hash: *blake3::hash(final_manifest_url.as_bytes()).as_bytes(), + rewrite_secret_hash: *blake3::hash(rewrite_secret).as_bytes(), + rewritten_body: Arc::clone(rewritten_body), + template: Arc::clone(template), + checkpoint, + }); + } + + #[cfg(any(test, feature = "test-support"))] + pub(super) fn replace_manifest_common( + &mut self, + body: String, + rendered_at_ms: u64, + playlist_duration_ms: Option, + semantics: ParsedManifestSemantics, + ) -> Result<(), HlsManifestLimitViolation> { + let manifest_resource_ids = Arc::new(extract_transient_resource_ids(&body)); + let manifest_identity = transient_manifest_identity(&body, &manifest_resource_ids); + let reusable_finalized_generation = semantics + .lifecycle() + .is_finalized() + .then(|| self.finalized_generation_for_identity(manifest_identity)) + .flatten(); + let template = parse_hls_transient_manifest_template(&body).ok().flatten(); + self.replace_manifest_common_with_resource_ids(TransientManifestReplacement { + body: Arc::from(body), + rendered_at_ms, + playlist_duration_ms, + semantics, + resource_ids: manifest_resource_ids, + identity: manifest_identity, + reusable_finalized_generation, + template, + }) + } + + pub(super) fn replace_manifest_common_with_resource_ids( + &mut self, + replacement: TransientManifestReplacement, + ) -> Result<(), HlsManifestLimitViolation> { + let TransientManifestReplacement { + body, + rendered_at_ms, + playlist_duration_ms, + semantics, + resource_ids, + identity, + reusable_finalized_generation, + template, + } = replacement; + self.ensure_manifest_generation_available(reusable_finalized_generation)?; + let manifest_validity = + CommittedTransientManifestValidity::from_semantics(semantics, rendered_at_ms, playlist_duration_ms); + let manifest_generation = if let Some(generation) = reusable_finalized_generation { + generation + } else { + self.manifest_generation = self.manifest_generation.saturating_add(1); + TransientManifestGeneration(self.manifest_generation) + }; + if reusable_finalized_generation.is_none() { + self.last_manifest_resource_ids = Arc::clone(&resource_ids); + self.last_manifest_template.clone_from(&template); + } + self.last_manifest_validity = manifest_validity; + if manifest_validity.is_finalized() { + self.current_finalized_manifest_generation = Some(manifest_generation); + if reusable_finalized_generation.is_none() { + self.insert_finalized_manifest_generation( + manifest_generation, + RetainedFinalizedManifestGeneration { identity, resource_ids: Arc::clone(&resource_ids), template }, + ); + } + } else { + self.current_finalized_manifest_generation = None; + } + self.prune_unreferenced_finalized_manifest_generations(); + if reusable_finalized_generation.is_none() { + self.last_manifest_body = Some(body); + } + self.last_manifest_rendered_at_ms = Some(rendered_at_ms); + self.last_manifest_playlist_duration_ms = playlist_duration_ms; + Ok(()) + } + + pub(super) fn validate_manifest_footprint( + &self, + candidate: &TransientManifestFootprintCandidate<'_>, + ) -> Result { + let origin_uri_bytes = self.prospective_origin_uri_bytes(candidate.resources); + ensure_manifest_limit( + HlsManifestLimitKind::TransientOriginUriBytes, + origin_uri_bytes, + MAX_TRANSIENT_ORIGIN_URI_BYTES_PER_SESSION, + )?; + let resource_entries = self.prospective_resource_entry_count(candidate.resources); + ensure_manifest_limit( + HlsManifestLimitKind::TransientResourceEntries, + resource_entries, + MAX_TRANSIENT_RESOURCE_ENTRIES_PER_SESSION, + )?; + let prospective_generation_sets = self.prospective_finalized_generation_sets( + candidate.lifecycle, + candidate.resource_ids, + candidate.reusable_finalized_generation, + ); + let retained_finalized_generations = prospective_generation_sets.len(); + ensure_manifest_limit( + HlsManifestLimitKind::FinalizedGenerations, + retained_finalized_generations, + MAX_RETAINED_FINALIZED_MANIFEST_GENERATIONS, + )?; + let finalized_generation_memberships = prospective_generation_sets + .iter() + .fold(0_usize, |total, resource_ids| total.saturating_add(resource_ids.len())); + ensure_manifest_limit( + HlsManifestLimitKind::TransientGenerationMemberships, + finalized_generation_memberships, + MAX_TRANSIENT_GENERATION_MEMBERSHIPS, + )?; + let estimated_metadata_bytes = + self.prospective_estimated_metadata_bytes(candidate, &prospective_generation_sets); + ensure_manifest_limit( + HlsManifestLimitKind::TransientEstimatedMetadataBytes, + estimated_metadata_bytes, + MAX_ESTIMATED_TRANSIENT_METADATA_BYTES, + )?; + Ok(HlsTransientManifestFootprint { + resources: candidate.resource_count, + resource_entries, + rewritten_bytes: candidate.rewritten_bytes, + origin_uri_bytes, + retained_finalized_generations, + finalized_generation_memberships, + estimated_metadata_bytes, + }) + } + + pub(super) fn validate_manifest_shape( + rewritten_bytes: usize, + resource_count: usize, + ) -> Result<(), HlsManifestLimitViolation> { + ensure_manifest_limit( + HlsManifestLimitKind::TransientResources, + resource_count, + MAX_TRANSIENT_MANIFEST_RESOURCES, + )?; + ensure_manifest_limit( + HlsManifestLimitKind::TransientRewrittenBytes, + rewritten_bytes, + MAX_TRANSIENT_REWRITTEN_MANIFEST_BYTES, + ) + } + + pub(super) fn ensure_manifest_generation_available( + &self, + reusable_generation: Option, + ) -> Result<(), HlsManifestLimitViolation> { + if reusable_generation.is_none() && self.manifest_generation == u64::MAX { + return Err(HlsManifestLimitViolation::new( + HlsManifestLimitKind::ManifestCommitGeneration, + usize::MAX, + usize::MAX - 1, + )); + } + Ok(()) + } + + pub(super) fn prospective_origin_uri_bytes(&self, resources: &[TransientResourceRef]) -> usize { + let incoming_uri_bytes = resources + .iter() + .map(|resource| (&resource.id, resource.resolved_origin_uri.len())) + .collect::>(); + let retained_uri_bytes = self + .resources + .iter() + .filter(|(resource_id, _)| !incoming_uri_bytes.contains_key(resource_id)) + .fold(0_usize, |total, (_, resource)| total.saturating_add(resource.resolved_origin_uri.len())); + incoming_uri_bytes.values().fold(retained_uri_bytes, |total, uri_bytes| total.saturating_add(*uri_bytes)) + } + + pub(super) fn prospective_resource_entry_count(&self, resources: &[TransientResourceRef]) -> usize { + let incoming_ids = resources.iter().map(|resource| &resource.id).collect::>(); + incoming_ids.iter().fold(self.resources.len(), |count, resource_id| { + count.saturating_add(usize::from(!self.resources.contains_key(*resource_id))) + }) + } + + pub(super) fn prospective_finalized_generation_sets<'a>( + &'a self, + lifecycle: HlsManifestLifecycle, + candidate_resource_ids: &'a HashSet, + reusable_generation: Option, + ) -> Vec<&'a HashSet> { + let mut sets = self + .finalized_manifest_generations + .iter() + .filter(|(generation, _)| { + Some(**generation) == reusable_generation || self.generation_is_lease_referenced(**generation) + }) + .map(|(_, generation)| generation.resource_ids.as_ref()) + .collect::>(); + if lifecycle.is_finalized() && reusable_generation.is_none() { + sets.push(candidate_resource_ids); + } + sets + } + + pub(super) fn prospective_estimated_metadata_bytes( + &self, + candidate: &TransientManifestFootprintCandidate<'_>, + generation_sets: &[&HashSet], + ) -> usize { + let incoming = candidate.resources.iter().map(|resource| (&resource.id, resource)).collect::>(); + let retained_resource_bytes = self + .resources + .iter() + .filter(|(resource_id, _)| !incoming.contains_key(resource_id)) + .fold(0_usize, |total, (resource_id, resource)| { + total.saturating_add(estimated_transient_resource_entry_bytes(resource_id, resource)) + }); + let resource_bytes = incoming.values().fold(retained_resource_bytes, |total, resource| { + total.saturating_add(estimated_transient_resource_entry_bytes(&resource.id, resource)) + }); + let generation_bytes = generation_sets.iter().fold(0_usize, |total, resource_ids| { + total + .saturating_add(std::mem::size_of::()) + .saturating_add(ESTIMATED_ARC_ALLOCATION_OVERHEAD_BYTES) + .saturating_add(estimated_transient_resource_id_set_bytes(resource_ids)) + }); + let rolling_manifest_resource_bytes = if candidate.lifecycle.is_finalized() { + 0 + } else { + estimated_transient_resource_id_set_bytes(candidate.resource_ids) + }; + let retained_template_bytes = self + .finalized_manifest_generations + .iter() + .filter(|(generation, _)| { + Some(**generation) == candidate.reusable_finalized_generation + || self.generation_is_lease_referenced(**generation) + }) + .filter_map(|(_, generation)| generation.template.as_ref()) + .fold(0_usize, |total, template| total.saturating_add(template.estimated_metadata_bytes())); + let candidate_template_bytes = + if candidate.lifecycle.is_finalized() && candidate.reusable_finalized_generation.is_some() { + 0 + } else { + candidate.template.map_or(0, HlsTransientManifestTemplate::estimated_metadata_bytes) + }; + let mut protected_ids = HashSet::new(); + for resource_ids in generation_sets { + protected_ids.extend(resource_ids.iter()); + } + let protected_index_bytes = protected_ids.iter().fold(0_usize, |total, resource_id| { + total.saturating_add( + std::mem::size_of::<(TransientResourceId, u16)>() + .saturating_add(resource_id.0.len()) + .saturating_add(ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES), + ) + }); + let binding_bytes = self.finalized_manifest_lease_bindings.iter().fold(0_usize, |total, binding| { + total.saturating_add( + std::mem::size_of::() + .saturating_add(binding.lease_id.0.len()) + .saturating_add(ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES), + ) + }); + let object_bytes = self + .object_cache + .values() + .fold(0_usize, |total, entry| total.saturating_add(estimated_transient_object_entry_bytes(entry))); + candidate + .rewritten_bytes + .saturating_add(resource_bytes) + .saturating_add(generation_bytes) + .saturating_add(rolling_manifest_resource_bytes) + .saturating_add(retained_template_bytes) + .saturating_add(candidate_template_bytes) + .saturating_add(protected_index_bytes) + .saturating_add(binding_bytes) + .saturating_add(object_bytes) + .saturating_add( + self.object_fetch_notifiers.len().saturating_mul( + std::mem::size_of::<(TransientObjectCacheKey, Arc)>() + .saturating_add(ESTIMATED_HASH_ENTRY_OVERHEAD_BYTES), + ), + ) + } +} diff --git a/backend/hls/src/transient/objects.rs b/backend/hls/src/transient/objects.rs new file mode 100644 index 000000000..2135b51b8 --- /dev/null +++ b/backend/hls/src/transient/objects.rs @@ -0,0 +1,413 @@ +use super::{ + transient_object_expires_at, HlsSession, ProxySessionId, TransientObjectCacheEntry, TransientObjectCacheKey, + TransientObjectCacheStatus, TransientObjectFetchDecision, TransientObjectFetchToken, TransientObjectRemoval, + TransientObjectResourceBinding, TransientPassthroughState, TransientResourceId, TransientResourceKind, + TransientResourceRef, MAX_FAILED_TRANSIENT_OBJECT_ENTRIES, +}; +use axum::http::StatusCode; +use std::{collections::HashSet, sync::Arc}; +use tokio::sync::Notify; + +impl TransientPassthroughState { + /// Builds the object-cache key for an already registered transient resource. + /// + /// The key is intentionally based on the opaque resource ID. The concrete fetch URI remains in + /// `TransientResourceRef::resolved_origin_uri` and must not be reconstructed from this key. + pub fn transient_object_key( + proxy_session_id: &ProxySessionId, + resource_id: &TransientResourceId, + file_ext: impl Into, + ) -> TransientObjectCacheKey { + TransientObjectCacheKey::new(proxy_session_id.clone(), resource_id.clone(), file_ext) + } + + pub fn ready_object( + &mut self, + key: &TransientObjectCacheKey, + resource_kind: TransientResourceKind, + now_ms: u64, + ) -> Option { + let binding = { + let entry = self.object_cache.get(key)?; + let resource = self.resources.get(key.transient_resource_id())?; + if resource.kind != resource_kind { + return None; + } + let binding = TransientObjectResourceBinding::from_resource(resource, &entry.binding.file_extension); + (binding.matches_resource_identity(resource) && self.resource_is_valid_at(resource, now_ms)) + .then_some(binding)? + }; + let entry = self.object_cache.get_mut(key)?; + if !matches!(entry.status, TransientObjectCacheStatus::Ready { .. }) + || entry.binding != binding + || (resource_kind == TransientResourceKind::Key && entry.ready_content_length() != Some(16)) + || entry.expires_at_ms < now_ms + { + return None; + } + entry.last_accessed_at_ms = now_ms; + entry.access.reader_started(now_ms); + entry.access.reader_finished(); + Some(entry.clone()) + } + + pub(super) fn current_resource_binding( + &self, + resource: &TransientResourceRef, + file_extension: &str, + now_ms: u64, + ) -> Option { + let current = self.resources.get(&resource.id)?; + let binding = TransientObjectResourceBinding::from_resource(current, file_extension); + (current.has_same_cache_identity(resource) + && binding.matches_resource_identity(current) + && self.resource_is_valid_at(current, now_ms)) + .then_some(binding) + } + + pub(super) fn binding_is_current(&self, binding: &TransientObjectResourceBinding, now_ms: u64) -> bool { + self.resources.get(&binding.resource_id).is_some_and(|resource| { + binding.matches_resource_identity(resource) && self.resource_is_valid_at(resource, now_ms) + }) + } + + pub fn begin_object_fetch( + &mut self, + proxy_session_id: &ProxySessionId, + resource: &TransientResourceRef, + file_ext: &str, + now_ms: u64, + cache_duration_ms: u64, + ) -> TransientObjectFetchDecision { + let lookup_key = Self::transient_object_key(proxy_session_id, &resource.id, file_ext.to_string()); + let current_binding = self.current_resource_binding(resource, file_ext, now_ms); + let binding_is_current = current_binding.is_some(); + let binding = + current_binding.unwrap_or_else(|| TransientObjectResourceBinding::from_resource(resource, file_ext)); + match self.object_cache.get(&lookup_key) { + Some(entry) + if binding_is_current + && entry.is_ready_at(now_ms) + && entry.binding == binding + && (resource.kind != TransientResourceKind::Key || entry.ready_content_length() == Some(16)) => + { + return TransientObjectFetchDecision::Ready; + } + Some(entry) + if binding_is_current + && entry.binding == binding + && entry.expires_at_ms >= now_ms + && matches!(entry.status, TransientObjectCacheStatus::Fetching { .. }) => + { + let notifier = + self.object_fetch_notifiers.entry(lookup_key).or_insert_with(|| Arc::new(Notify::new())).clone(); + return TransientObjectFetchDecision::Wait(notifier); + } + Some(_) | None => {} + } + let expires_at_ms = transient_object_expires_at(now_ms, cache_duration_ms); + let content_type = resource.content_type_hint.clone().unwrap_or_else(|| "application/octet-stream".to_string()); + let fetch_generation = self.next_object_fetch_generation; + self.next_object_fetch_generation = self.next_object_fetch_generation.saturating_add(1); + let cache_key = Self::transient_object_key( + proxy_session_id, + &resource.id, + format!("{file_ext}.fill-{fetch_generation:016x}"), + ); + let superseded_object = self.remove_object_entry(&lookup_key); + let _previous = self.object_cache.insert( + lookup_key.clone(), + TransientObjectCacheEntry::new_fetching( + cache_key.clone(), + binding.clone(), + now_ms, + expires_at_ms, + content_type, + ), + ); + self.object_fetch_notifiers.entry(lookup_key.clone()).or_insert_with(|| Arc::new(Notify::new())); + let Some(entry) = self.object_cache.get(&lookup_key) else { + return TransientObjectFetchDecision::Wait( + self.object_fetch_notifiers.entry(lookup_key).or_insert_with(|| Arc::new(Notify::new())).clone(), + ); + }; + TransientObjectFetchDecision::Fetch(Box::new(TransientObjectFetchToken { + lookup_key, + cache_key, + entry_access: Arc::clone(&entry.access), + binding, + superseded_object, + })) + } + + pub(super) fn object_fetch_entry_matches(&self, token: &TransientObjectFetchToken) -> bool { + self.object_cache.get(&token.lookup_key).is_some_and(|entry| { + matches!(entry.status, TransientObjectCacheStatus::Fetching { .. }) + && entry.key == token.cache_key + && Arc::ptr_eq(&entry.access, &token.entry_access) + && entry.binding == token.binding + }) + } + + pub fn object_fetch_token_matches(&self, token: &TransientObjectFetchToken) -> bool { + self.object_fetch_entry_matches(token) + } + + pub(super) fn mark_object_ready( + &mut self, + token: &TransientObjectFetchToken, + content_type: String, + content_length: u64, + now_ms: u64, + expires_at_ms: u64, + ) { + let notify_waiters = self.object_fetch_notifiers.remove(&token.lookup_key); + if let Some(entry) = self.object_cache.get_mut(&token.lookup_key) { + entry.status = TransientObjectCacheStatus::Ready { content_length, ready_at_ms: now_ms }; + entry.content_type = content_type; + entry.last_accessed_at_ms = now_ms; + entry.expires_at_ms = expires_at_ms; + } + if let Some(notifier) = notify_waiters { + notifier.notify_waiters(); + } + } + + pub fn mark_object_ready_if_current( + &mut self, + token: &TransientObjectFetchToken, + content_type: String, + content_length: u64, + now_ms: u64, + expires_at_ms: u64, + ) -> bool { + if !self.object_fetch_entry_matches(token) { + return false; + } + if !self.binding_is_current(&token.binding, now_ms) { + let _removed = self.remove_object_entry(&token.lookup_key); + return false; + } + self.mark_object_ready(token, content_type, content_length, now_ms, expires_at_ms); + true + } + + pub(super) fn mark_object_failed_retryable( + &mut self, + key: &TransientObjectCacheKey, + now_ms: u64, + retry_after_ms: u64, + ) { + let failed_expires_at_ms = self.failed_object_metadata_expires_at(now_ms); + let notify_waiters = self.object_fetch_notifiers.remove(key); + if let Some(entry) = self.object_cache.get_mut(key) { + entry.status = TransientObjectCacheStatus::FailedRetryable { failed_at_ms: now_ms, retry_after_ms }; + entry.last_accessed_at_ms = now_ms; + entry.expires_at_ms = entry.expires_at_ms.min(failed_expires_at_ms); + } + if let Some(notifier) = notify_waiters { + notifier.notify_waiters(); + } + self.enforce_failed_object_metadata_bound(); + } + + pub(super) fn mark_object_failed_permanent( + &mut self, + key: &TransientObjectCacheKey, + now_ms: u64, + status: Option, + ) { + let failed_expires_at_ms = self.failed_object_metadata_expires_at(now_ms); + let notify_waiters = self.object_fetch_notifiers.remove(key); + if let Some(entry) = self.object_cache.get_mut(key) { + entry.status = TransientObjectCacheStatus::FailedPermanent { failed_at_ms: now_ms, status }; + entry.last_accessed_at_ms = now_ms; + entry.expires_at_ms = entry.expires_at_ms.min(failed_expires_at_ms); + } + if let Some(notifier) = notify_waiters { + notifier.notify_waiters(); + } + self.enforce_failed_object_metadata_bound(); + } + + pub(super) fn failed_object_metadata_expires_at(&self, now_ms: u64) -> u64 { + now_ms.saturating_add(self.resource_ttl_ms) + } + + pub fn mark_object_failed_retryable_if_current( + &mut self, + token: &TransientObjectFetchToken, + now_ms: u64, + retry_after_ms: u64, + ) -> bool { + if !self.object_fetch_entry_matches(token) { + return false; + } + if !self.binding_is_current(&token.binding, now_ms) { + let _removed = self.remove_object_entry(&token.lookup_key); + return false; + } + self.mark_object_failed_retryable(&token.lookup_key, now_ms, retry_after_ms); + true + } + + pub fn mark_object_failed_permanent_if_current( + &mut self, + token: &TransientObjectFetchToken, + now_ms: u64, + status: Option, + ) -> bool { + if !self.object_fetch_entry_matches(token) { + return false; + } + if !self.binding_is_current(&token.binding, now_ms) { + let _removed = self.remove_object_entry(&token.lookup_key); + return false; + } + self.mark_object_failed_permanent(&token.lookup_key, now_ms, status); + true + } + + /// Returns the finite READY lifetime of an AES key dependency without touching access accounting. + pub fn ready_key_object_valid_until_ms( + &self, + proxy_session_id: &ProxySessionId, + resource_id: &TransientResourceId, + file_ext: &str, + now_ms: u64, + ) -> Option { + let resource = self.resources.get(resource_id)?; + if resource.kind != TransientResourceKind::Key + || resource.file_ext_hint.as_deref() != Some(file_ext) + || !self.resource_is_valid_at(resource, now_ms) + { + return None; + } + let key = Self::transient_object_key(proxy_session_id, resource_id, file_ext.to_string()); + let entry = self.object_cache.get(&key)?; + let binding = TransientObjectResourceBinding::from_resource(resource, file_ext); + let valid_aes128_key = matches!(entry.status, TransientObjectCacheStatus::Ready { content_length: 16, .. }); + let valid_until_ms = if self.protected_finalized_resource_refcounts.contains_key(resource_id) { + entry.expires_at_ms + } else { + entry.expires_at_ms.min(resource.expires_at_ms) + }; + (valid_aes128_key && entry.binding == binding && entry.is_ready_at(now_ms)).then_some(valid_until_ms) + } + + pub fn ready_object_cache_size(&self) -> u64 { + self.object_cache.values().filter_map(TransientObjectCacheEntry::ready_content_length).sum() + } + + pub fn take_expired_object_removals_except( + &mut self, + now_ms: u64, + protected: &HashSet, + limit: usize, + ) -> Vec { + let mut keys = self + .object_cache + .iter() + .filter_map(|(key, entry)| { + let ready_is_protected = + entry.ready_content_length().is_some() && protected.contains(key.transient_resource_id()); + if !ready_is_protected && entry.access.active_readers() == 0 && entry.expires_at_ms < now_ms { + return Some((key.clone(), entry.expires_at_ms, entry.created_at_ms, key.stable_value())); + } + None + }) + .collect::>(); + keys.sort_by(|left, right| (left.1, left.2, &left.3).cmp(&(right.1, right.2, &right.3))); + keys.into_iter().take(limit).filter_map(|(key, _, _, _)| self.remove_object_entry(&key)).collect() + } + + pub fn remove_oldest_ready_object_except( + &mut self, + protected: &HashSet, + ) -> Option { + let candidate = self + .object_cache + .iter() + .filter_map(|(key, entry)| { + if !protected.contains(key.transient_resource_id()) && entry.access.active_readers() == 0 { + return entry.ready_content_length().map(|content_length| { + (key.clone(), content_length, entry.last_accessed_at_ms, entry.created_at_ms) + }); + } + None + }) + .min_by_key(|(_, _, last_accessed_at_ms, created_at_ms)| (*last_accessed_at_ms, *created_at_ms))?; + self.remove_object_entry(&candidate.0) + } + + pub(super) fn remove_object_entry( + &mut self, + lookup_key: &TransientObjectCacheKey, + ) -> Option { + let entry = self.object_cache.remove(lookup_key)?; + if let Some(notifier) = self.object_fetch_notifiers.remove(lookup_key) { + notifier.notify_waiters(); + } + let content_length = entry.ready_content_length().unwrap_or_default(); + Some(TransientObjectRemoval { key: entry.key, content_length }) + } + + pub(super) fn enforce_failed_object_metadata_bound(&mut self) { + let mut failed = self + .object_cache + .iter() + .filter_map(|(key, entry)| match entry.status { + TransientObjectCacheStatus::FailedRetryable { failed_at_ms, .. } + | TransientObjectCacheStatus::FailedPermanent { failed_at_ms, .. } => { + Some((key.clone(), failed_at_ms, entry.created_at_ms, key.stable_value())) + } + TransientObjectCacheStatus::Fetching { .. } | TransientObjectCacheStatus::Ready { .. } => None, + }) + .collect::>(); + let overflow = failed.len().saturating_sub(MAX_FAILED_TRANSIENT_OBJECT_ENTRIES); + if overflow == 0 { + return; + } + failed.sort_by(|left, right| (left.1, left.2, &left.3).cmp(&(right.1, right.2, &right.3))); + for (key, _, _, _) in failed.into_iter().take(overflow) { + let _removed = self.remove_object_entry(&key); + } + } +} + +impl HlsSession { + pub fn commit_transient_object_ready_if_current( + &mut self, + resource_kind: TransientResourceKind, + token: &TransientObjectFetchToken, + content_type: String, + content_length: u64, + now_ms: u64, + expires_at_ms: u64, + ) -> bool { + let committed = + self.transient.mark_object_ready_if_current(token, content_type, content_length, now_ms, expires_at_ms); + if committed && resource_kind == TransientResourceKind::Key { + self.advance_media_readiness_generation(); + } + committed + } + + pub fn fail_transient_object_retryable_if_current( + &mut self, + token: &TransientObjectFetchToken, + now_ms: u64, + retry_after_ms: u64, + ) -> bool { + self.transient.mark_object_failed_retryable_if_current(token, now_ms, retry_after_ms) + } + + pub fn fail_transient_object_permanent_if_current( + &mut self, + token: &TransientObjectFetchToken, + now_ms: u64, + status: Option, + ) -> bool { + self.transient.mark_object_failed_permanent_if_current(token, now_ms, status) + } +} diff --git a/backend/hls/src/transient/resources.rs b/backend/hls/src/transient/resources.rs new file mode 100644 index 000000000..f40303d9c --- /dev/null +++ b/backend/hls/src/transient/resources.rs @@ -0,0 +1,152 @@ +use super::{ + identity::TransientResourceValidityScope, HlsAccessLeaseId, HlsPublishedTransientResourceIds, + TransientObjectCacheKey, TransientObjectCacheStatus, TransientObjectUnavailableState, TransientPassthroughState, + TransientResourceId, TransientResourceRef, TransientResourceRevision, +}; +use std::{collections::HashSet, sync::Arc}; + +impl TransientPassthroughState { + pub fn upsert_resources(&mut self, resources: I) + where + I: IntoIterator, + { + for mut resource in resources { + let preserve_revision = self + .resources + .get(&resource.id) + .filter(|existing| existing.has_same_cache_identity(&resource)) + .map(|existing| existing.revision); + resource.revision = if let Some(revision) = preserve_revision { + revision + } else { + let revision = TransientResourceRevision(self.next_resource_revision); + self.next_resource_revision = self.next_resource_revision.saturating_add(1); + revision + }; + match self.resources.get_mut(&resource.id) { + Some(existing) => existing.refresh_from(resource), + None => { + let _previous = self.resources.insert(resource.id.clone(), resource); + } + } + } + } + + pub(super) fn resource_is_valid_at_for( + &self, + resource: &TransientResourceRef, + now_ms: u64, + scope: TransientResourceValidityScope<'_>, + ) -> bool { + match scope { + TransientResourceValidityScope::PublishedSession => { + resource.is_valid_at(now_ms) || self.protected_finalized_resource_refcounts.contains_key(&resource.id) + } + TransientResourceValidityScope::AccessLease { lease_id, lease_issued_at_ms, published_resource_ids } => { + (published_resource_ids.contains(&resource.id) && resource.is_valid_at(now_ms)) + || self.finalized_manifest_lease_bindings.iter().any(|binding| { + binding.lease_id == *lease_id + && binding.lease_issued_at_ms == lease_issued_at_ms + && self + .finalized_manifest_generations + .get(&binding.manifest_generation) + .is_some_and(|generation| generation.resource_ids.contains(&resource.id)) + }) + } + } + } + + pub(crate) fn resource_is_valid_at(&self, resource: &TransientResourceRef, now_ms: u64) -> bool { + self.resource_is_valid_at_for(resource, now_ms, TransientResourceValidityScope::PublishedSession) + } + + pub(crate) fn resource_matches_current(&self, resource: &TransientResourceRef, now_ms: u64) -> bool { + self.resources.get(&resource.id).is_some_and(|current| { + Arc::ptr_eq(¤t.access, &resource.access) + && current.has_same_cache_identity(resource) + && self.resource_is_valid_at(current, now_ms) + }) + } + + pub fn resolve_current_resource( + &self, + resource_id: &TransientResourceId, + now_ms: u64, + ) -> Option { + self.resources.get(resource_id).filter(|resource| self.resource_is_valid_at(resource, now_ms)).cloned() + } + + pub(crate) fn resolve_resource_for_lease( + &self, + resource_id: &TransientResourceId, + lease_id: &HlsAccessLeaseId, + lease_issued_at_ms: u64, + published_resource_ids: &HlsPublishedTransientResourceIds, + now_ms: u64, + ) -> Option { + self.resources + .get(resource_id) + .filter(|resource| { + self.resource_is_valid_at_for( + resource, + now_ms, + TransientResourceValidityScope::AccessLease { + lease_id, + lease_issued_at_ms, + published_resource_ids, + }, + ) + }) + .cloned() + } + + pub fn get_valid_resource( + &mut self, + resource_id: &TransientResourceId, + now_ms: u64, + ) -> Option { + self.prune_expired(now_ms); + self.resolve_current_resource(resource_id, now_ms) + } + + pub fn prune_expired(&mut self, now_ms: u64) { self.prune_expired_except(now_ms, &HashSet::new()); } + + pub fn prune_expired_except(&mut self, now_ms: u64, protected: &HashSet) { + let expired_resource_ids = self + .resources + .iter() + .filter(|(id, resource)| { + !protected.contains(*id) + && !self.resource_is_valid_at(resource, now_ms) + && resource.active_readers() == 0 + }) + .map(|(id, _)| id.clone()) + .collect::>(); + self.resources.retain(|id, _| !expired_resource_ids.contains(id)); + } + + pub fn active_resource_readers(&self) -> u32 { + self.resources.values().map(TransientResourceRef::active_readers).sum() + } + + pub fn object_unavailable_state( + &self, + key: &TransientObjectCacheKey, + now_ms: u64, + ) -> TransientObjectUnavailableState { + let Some(entry) = self.object_cache.get(key) else { + return TransientObjectUnavailableState::Missing; + }; + if entry.expires_at_ms < now_ms || !self.binding_is_current(&entry.binding, now_ms) { + return TransientObjectUnavailableState::Missing; + } + match entry.status { + TransientObjectCacheStatus::Fetching { .. } => TransientObjectUnavailableState::Fetching, + TransientObjectCacheStatus::FailedRetryable { retry_after_ms, .. } => { + TransientObjectUnavailableState::FailedRetryable { retry_after_ms } + } + TransientObjectCacheStatus::FailedPermanent { .. } => TransientObjectUnavailableState::FailedPermanent, + TransientObjectCacheStatus::Ready { .. } => TransientObjectUnavailableState::Missing, + } + } +} diff --git a/backend/hls/src/transient/tests.rs b/backend/hls/src/transient/tests.rs new file mode 100644 index 000000000..efdd37e9c --- /dev/null +++ b/backend/hls/src/transient/tests.rs @@ -0,0 +1,66 @@ +use super::{ + build_transient_resource_id, extract_transient_resource_ids, transient_object_expires_at, + HlsPublishedTransientResourceIds, HlsTransientManifestCommitError, TransientManifestLeaseBinding, + TransientObjectCacheStatus, TransientPassthroughState, TransientResourceId, TransientResourceKind, + TransientResourceRef, MAX_FAILED_TRANSIENT_OBJECT_ENTRIES, +}; +use crate::{ + manifest_limits::{ + HlsManifestLimitKind, HlsManifestLimitViolation, MAX_ESTIMATED_TRANSIENT_METADATA_BYTES, + MAX_RETAINED_FINALIZED_MANIFEST_GENERATIONS, MAX_TRANSIENT_GENERATION_MEMBERSHIPS, + MAX_TRANSIENT_MANIFEST_RESOURCES, MAX_TRANSIENT_ORIGIN_URI_BYTES_PER_SESSION, + MAX_TRANSIENT_RESOURCE_ENTRIES_PER_SESSION, MAX_TRANSIENT_REWRITTEN_MANIFEST_BYTES, + }, + transient_manifest::{TransientManifestRewriter, TransientRewriteResult}, + HlsAccessLeaseId, ProxySessionId, +}; +use std::{collections::HashSet, fmt::Write as _, sync::Arc, task::Poll}; +use tuliprox_parser::hls::origin_manifest::{parse_manifest_semantics, HlsManifestLifecycle, HlsManifestWindowPolicy}; + +fn local_representation_limit(error: HlsTransientManifestCommitError) -> HlsManifestLimitViolation { + let HlsTransientManifestCommitError::LocalRepresentationLimit(violation) = error else { + panic!("expected local representation limit, got {error:?}"); + }; + violation +} + +fn finalized_manifest_body(resource_id: &TransientResourceId, extension: &str) -> String { + format!( + "#EXTM3U\n#EXT-X-TARGETDURATION:1\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:1,\n/hls/shared/live/session/lease/r/{}.{extension}\n#EXT-X-ENDLIST\n", + resource_id.0 + ) +} + +fn replace_with_finalized_manifest( + state: &mut TransientPassthroughState, + resource_id: &TransientResourceId, + extension: &str, + rendered_at_ms: u64, +) { + state.replace_manifest_with_semantics(finalized_manifest_body(resource_id, extension), rendered_at_ms, Some(1_000)); +} + +fn rewritten_finalized_manifest(segment_count: usize, identity: &str) -> TransientRewriteResult { + let mut body = + String::from("#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:1\n"); + for index in 0..segment_count { + writeln!(body, "#EXTINF:6,\n{identity}-{index}.ts").expect("synthetic manifest renders"); + } + body.push_str("#EXT-X-ENDLIST\n"); + TransientManifestRewriter::rewrite( + &body, + "https://origin.example/archive/index.m3u8", + &ProxySessionId("proxy-session".to_string()), + b"secret", + 0, + 300_000, + ) +} + +mod http; +mod playlist; +mod policy; +mod publication; +mod storage; +mod streaming; +mod transport; diff --git a/backend/hls/src/transient/tests/http.rs b/backend/hls/src/transient/tests/http.rs new file mode 100644 index 000000000..2c9ba702f --- /dev/null +++ b/backend/hls/src/transient/tests/http.rs @@ -0,0 +1,23 @@ +use super::*; + +#[test] +fn transient_rewritten_body_limit_rejects_before_commit() { + let mut state = TransientPassthroughState::default(); + let oversized_body = "x".repeat(MAX_TRANSIENT_REWRITTEN_MANIFEST_BYTES + 1); + + let violation = local_representation_limit( + state + .commit_rewritten_manifest_with_semantics( + oversized_body, + Vec::new(), + 1, + None, + parse_manifest_semantics("#EXTM3U\n"), + ) + .expect_err("rewritten body overflow is rejected"), + ); + + assert_eq!(violation.kind, HlsManifestLimitKind::TransientRewrittenBytes); + assert!(state.last_manifest_body.is_none()); + assert!(state.resources.is_empty()); +} diff --git a/backend/hls/src/transient/tests/playlist.rs b/backend/hls/src/transient/tests/playlist.rs new file mode 100644 index 000000000..d451fdb81 --- /dev/null +++ b/backend/hls/src/transient/tests/playlist.rs @@ -0,0 +1,226 @@ +use super::*; + +#[test] +fn extract_transient_resource_ids_ignores_comment_and_foreign_uri_surfaces() { + let body = concat!( + "# /r/fake.ts\n", + "https://origin.example.com/r/fake.ts\n", + "#EXT-X-KEY:METHOD=AES-128,URI=\"https://origin.example.com/r/fake.key\"\n", + ); + + assert!(extract_transient_resource_ids(body).is_empty()); +} + +#[test] +fn extract_transient_resource_ids_ignores_query_values() { + let body = concat!( + "#EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/proxy/lease/r/AbCdEfGhIjKlMnOp.key?token=/r/fake.ts\"\n", + "/hls/shared/live/proxy/lease/r/AbCdEfGhIjKlMnOp.ts?redirect=/r/foreign.ts\n", + ); + + let resource_ids = extract_transient_resource_ids(body); + + assert_eq!(resource_ids.len(), 1); + assert!(resource_ids.contains(&TransientResourceId("AbCdEfGhIjKlMnOp".to_string()))); +} + +#[test] +fn extract_transient_resource_ids_keeps_segment_key_and_map_routes() { + let body = concat!( + "#EXTM3U\n", + "#EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/proxy/__hls_access_lease_id__/r/Key0000000000Id.key\"\n", + "#EXT-X-MAP:URI=\"/hls/shared/live/proxy/__hls_access_lease_id__/r/Map0000000000Id.mp4\"\n", + "#EXTINF:1.0,\n", + "/hls/shared/live/proxy/lease/r/Seg0000000000Id.ts\n", + ); + + let resource_ids = extract_transient_resource_ids(body); + + assert_eq!(resource_ids.len(), 3); + assert!(resource_ids.contains(&TransientResourceId("Key0000000000Id".to_string()))); + assert!(resource_ids.contains(&TransientResourceId("Map0000000000Id".to_string()))); + assert!(resource_ids.contains(&TransientResourceId("Seg0000000000Id".to_string()))); +} + +#[test] +fn transient_session_resource_entry_limit_is_enforced() { + let mut state = TransientPassthroughState::default(); + for index in 0..MAX_TRANSIENT_RESOURCE_ENTRIES_PER_SESSION { + state.upsert_resources([TransientResourceRef::new( + TransientResourceKind::Segment, + format!("https://origin.example/{index}.ts"), + b"secret", + 0, + 300_000, + Some("ts".to_string()), + )]); + } + let incoming = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://other-origin.example/overflow.ts".to_string(), + b"secret", + 0, + 300_000, + Some("ts".to_string()), + ); + + let violation = local_representation_limit( + state + .commit_rewritten_manifest_with_semantics( + finalized_manifest_body(&incoming.id, "ts"), + vec![incoming], + 1, + Some(1_000), + parse_manifest_semantics("#EXTM3U\n#EXT-X-ENDLIST\n"), + ) + .expect_err("resource entry overflow is rejected"), + ); + + assert_eq!(violation.kind, HlsManifestLimitKind::TransientResourceEntries); + assert_eq!(violation.actual, MAX_TRANSIENT_RESOURCE_ENTRIES_PER_SESSION + 1); +} + +#[test] +fn transient_resource_id_is_stable_and_opaque() { + let first = build_transient_resource_id("http://origin.example.com/live/key.bin", b"secret"); + let second = build_transient_resource_id("http://origin.example.com/live/key.bin", b"secret"); + let other = build_transient_resource_id("http://origin.example.com/live/seg.ts", b"secret"); + + assert_eq!(first, second); + assert_ne!(first, other); + assert_eq!(first.0.len(), 16); + assert!(!first.0.contains("origin")); +} + +#[test] +fn transient_state_updates_existing_resource_ttl() { + let mut state = TransientPassthroughState::default(); + let mut resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 10, + 100, + Some("ts".to_string()), + ); + resource.encrypted_media = true; + let resource_id = resource.id.clone(); + state.upsert_resources([resource]); + let updated = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 20, + 200, + Some("ts".to_string()), + ); + state.upsert_resources([updated]); + + assert_eq!(state.resources.len(), 1); + assert!(state.resources.get(&resource_id).expect("resource").encrypted_media); + assert_eq!(state.get_valid_resource(&resource_id, 150).expect("resource remains valid").expires_at_ms, 220); + assert!(state.get_valid_resource(&resource_id, 221).is_none()); +} + +#[test] +fn transient_resource_refresh_does_not_extend_an_inflight_fill_deadline() { + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/inflight.ts", + b"secret", + 10, + 100, + Some("ts".to_string()), + ); + state.upsert_resources([resource.clone()]); + let token = match state.begin_object_fetch(&proxy_session_id, &resource, "ts", 20, 50) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("new resource starts a cache fetch") + } + }; + + state.upsert_resources([TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/inflight.ts", + b"secret", + 100, + 300, + Some("ts".to_string()), + )]); + + assert_eq!(state.object_cache.get(&token.lookup_key).map(|entry| entry.expires_at_ms), Some(70)); +} + +#[test] +fn transient_media_resource_extensions_use_video_mp4_content_type() { + for extension in ["mp4", "m4s", "m4v"] { + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + format!("http://origin.example.com/live/seg.{extension}"), + b"secret", + 10, + 100, + Some(extension.to_string()), + ); + + assert_eq!(resource.content_type_hint.as_deref(), Some("video/mp4")); + } +} + +#[test] +fn finalized_catchup_has_no_resource_boundary_at_cache_duration() { + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let mut origin_body = String::from("#EXTM3U\n#EXT-X-TARGETDURATION:7\n#EXT-X-PLAYLIST-TYPE:EVENT\n"); + let durations_ms = (0..46) + .map(|index| match index { + 0..44 => 6_700, + 44 => 6_720, + _ => 6_000, + }) + .collect::>(); + for (index, duration_ms) in durations_ms.iter().enumerate() { + writeln!(origin_body, "#EXTINF:{}.{:03},\n{index}.ts", duration_ms / 1_000, duration_ms % 1_000) + .expect("synthetic manifest renders"); + } + origin_body.push_str("#EXT-X-ENDLIST\n"); + assert_eq!(durations_ms.iter().take(45).sum::(), 301_520); + let lifecycle = parse_manifest_semantics(&origin_body).lifecycle(); + assert_eq!(lifecycle, HlsManifestLifecycle::Finalized); + let rewritten = TransientManifestRewriter::rewrite( + &origin_body, + "http://origin.example.com/archive/index.m3u8", + &proxy_session_id, + b"secret", + 0, + 300_000, + ); + let before_boundary = rewritten + .resources + .iter() + .find(|resource| resource.resolved_origin_uri.ends_with("/44.ts")) + .expect("segment before boundary") + .id + .clone(); + let after_boundary = rewritten + .resources + .iter() + .find(|resource| resource.resolved_origin_uri.ends_with("/45.ts")) + .expect("segment after boundary") + .id + .clone(); + let mut state = TransientPassthroughState::default(); + state.upsert_resources(rewritten.resources); + state.replace_manifest_with_semantics(rewritten.body, 0, Some(durations_ms.iter().sum())); + + assert!(state.resolve_current_resource(&before_boundary, 301_520).is_some()); + let after = state + .resolve_current_resource(&after_boundary, 301_520) + .expect("segment after cache-duration boundary remains resolvable"); + assert!(matches!( + state.begin_object_fetch(&proxy_session_id, &after, "ts", 301_520, 300_000), + super::super::TransientObjectFetchDecision::Fetch(_) + )); +} diff --git a/backend/hls/src/transient/tests/policy.rs b/backend/hls/src/transient/tests/policy.rs new file mode 100644 index 000000000..9ebf3bc93 --- /dev/null +++ b/backend/hls/src/transient/tests/policy.rs @@ -0,0 +1,175 @@ +use super::*; + +#[test] +fn rolling_event_append_requires_current_rewrite_secret_and_refreshes_existing_mapping_validity() { + let mut state = TransientPassthroughState::default(); + let first_origin = "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXT-X-PLAYLIST-TYPE:EVENT\n\ + #EXT-X-MEDIA-SEQUENCE:1\n#EXTINF:6,\n0.ts\n#EXTINF:6,\n1.ts\n"; + let first = TransientManifestRewriter::rewrite( + first_origin, + "https://origin.example/event/index.m3u8", + &ProxySessionId("proxy-session".to_string()), + b"secret", + 0, + 300_000, + ); + let checkpoint = first.checkpoint.clone(); + let first_resource_id = first.resources[0].id.clone(); + let semantics = parse_manifest_semantics(first_origin); + let footprint = state + .commit_rewritten_manifest_with_semantics(first.body, first.resources, 0, Some(12_000), semantics) + .expect("initial EVENT commits"); + state.record_rolling_event_rewrite_seed( + first_origin, + "https://origin.example/event/index.m3u8", + b"secret", + checkpoint, + footprint.estimated_metadata_bytes, + true, + ); + let previous_template = state.last_manifest_template().expect("initial template"); + let second_origin = format!("{first_origin}#EXTINF:6,\n2.ts\n"); + assert!(state + .rolling_event_append_context(&second_origin, "https://origin.example/event/index.m3u8", b"rotated-secret",) + .is_none()); + let context = state + .rolling_event_append_context(&second_origin, "https://origin.example/event/index.m3u8", b"secret") + .expect("safe append is detected"); + let mut appended = TransientManifestRewriter::rewrite_append( + &second_origin[context.suffix_offset..], + "https://origin.example/event/index.m3u8", + &ProxySessionId("proxy-session".to_string()), + b"secret", + 400_000, + 300_000, + context.checkpoint, + ); + let template = TransientPassthroughState::extend_rolling_event_template(&context.template, &appended.body) + .expect("append remains within limits") + .expect("append contains a complete media unit"); + let mut body = String::with_capacity(context.rewritten_prefix.len().saturating_add(appended.body.len())); + body.push_str(&context.rewritten_prefix); + body.push_str(&appended.body); + appended.body = body; + + state + .commit_incremental_rewritten_manifest_with_semantics( + appended.body, + appended.resources, + Arc::clone(&template), + 400_000, + 300_000, + parse_manifest_semantics(&second_origin), + ) + .expect("append commits"); + + assert_eq!(template.segments().len(), 3); + assert!(Arc::ptr_eq(&previous_template.segments()[0].uri, &template.segments()[0].uri)); + assert_eq!(state.resources[&first_resource_id].expires_at_ms, 700_000); + assert_eq!(state.current_manifest_resource_ids().len(), 3); +} + +#[test] +fn finalized_key_readiness_uses_object_ttl_after_mapping_ttl() { + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Key, + "http://origin.example.com/archive/key.key", + b"secret", + 0, + 20, + Some("key".to_string()), + ); + let resource_id = resource.id.clone(); + state.upsert_resources([resource.clone()]); + replace_with_finalized_manifest(&mut state, &resource_id, "key", 0); + let token = match state.begin_object_fetch(&proxy_session_id, &resource, "key", 10, 100) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("new finalized AES key starts a cache fetch") + } + }; + assert!(state.mark_object_ready_if_current(&token, "application/octet-stream".to_string(), 16, 15, 110)); + + assert_eq!(state.ready_key_object_valid_until_ms(&proxy_session_id, &resource_id, "key", 30), Some(110)); + assert_eq!(state.ready_key_object_valid_until_ms(&proxy_session_id, &resource_id, "key", 111), None); +} + +#[test] +fn failed_transient_metadata_has_finite_ttl_and_a_hard_count_bound() { + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let mut state = TransientPassthroughState::new(100); + let mut oldest_lookup_key = None; + for index in 0..=MAX_FAILED_TRANSIENT_OBJECT_ENTRIES { + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + format!("http://origin.example.com/live/{index}.ts"), + b"secret", + 0, + 10_000, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + state.upsert_resources([resource]); + let resource = state.resources.get(&resource_id).expect("registered resource").clone(); + let token = match state.begin_object_fetch(&proxy_session_id, &resource, "ts", index as u64, 10_000) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("unique resource starts a cache fill") + } + }; + if index == 0 { + oldest_lookup_key = Some(token.lookup_key.clone()); + } + assert!(state.mark_object_failed_permanent_if_current(&token, index as u64, None)); + } + + assert_eq!( + state + .object_cache + .values() + .filter(|entry| matches!(entry.status, TransientObjectCacheStatus::FailedPermanent { .. })) + .count(), + MAX_FAILED_TRANSIENT_OBJECT_ENTRIES + ); + assert!(!state.object_cache.contains_key(&oldest_lookup_key.expect("oldest lookup key"))); + + let mut state = TransientPassthroughState::new(100); + let expiring = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/expiring.ts", + b"secret", + 1_000, + 10_000, + Some("ts".to_string()), + ); + let expiring_id = expiring.id.clone(); + state.upsert_resources([expiring]); + let expiring = state.resources.get(&expiring_id).expect("registered expiring resource").clone(); + let token = match state.begin_object_fetch(&proxy_session_id, &expiring, "ts", 1_010, 10_000) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("expiring resource starts a cache fill") + } + }; + assert!(state.mark_object_failed_retryable_if_current(&token, 1_020, 10)); + assert_eq!(state.object_cache.get(&token.lookup_key).map(|entry| entry.expires_at_ms), Some(1_120)); + state.upsert_resources([TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/expiring.ts", + b"secret", + 1_100, + 20_000, + Some("ts".to_string()), + )]); + assert_eq!( + state.object_cache.get(&token.lookup_key).map(|entry| entry.expires_at_ms), + Some(1_120), + "manifest refresh must not make failed metadata immortal" + ); + let removals = state.take_expired_object_removals_except(1_121, &HashSet::from([expiring_id]), 1); + assert_eq!(removals.len(), 1); + assert_eq!(&removals[0].key, token.cache_key()); + assert!(!state.object_cache.contains_key(&token.lookup_key)); +} diff --git a/backend/hls/src/transient/tests/publication.rs b/backend/hls/src/transient/tests/publication.rs new file mode 100644 index 000000000..ae0ec54f7 --- /dev/null +++ b/backend/hls/src/transient/tests/publication.rs @@ -0,0 +1,387 @@ +use super::*; + +#[test] +fn finalized_committed_manifest_has_full_window_and_no_deadline() { + let mut state = TransientPassthroughState::default(); + + state.replace_manifest_with_semantics("#EXTM3U\n#EXT-X-ENDLIST\n".to_string(), 100, Some(1_000)); + + assert!(state.last_manifest_finalized()); + assert_eq!(state.last_manifest_window_policy(), HlsManifestWindowPolicy::PreserveFullManifest); + assert_eq!(state.last_manifest_valid_until_ms(), None); +} + +#[test] +fn rolling_typed_committed_manifest_preserves_full_window_with_deadline() { + let mut state = TransientPassthroughState::default(); + + state.replace_manifest_with_semantics( + "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXTINF:1,\n/r/resource.ts\n".to_string(), + 100, + Some(1_000), + ); + + assert!(!state.last_manifest_finalized()); + assert_eq!(state.last_manifest_window_policy(), HlsManifestWindowPolicy::PreserveFullManifest); + assert_eq!(state.last_manifest_valid_until_ms(), Some(1_100)); +} + +#[test] +fn transient_manifest_resource_limit_rejects_without_mutating_current_manifest() { + let mut state = TransientPassthroughState::default(); + let baseline = rewritten_finalized_manifest(1, "baseline"); + let baseline_semantics = parse_manifest_semantics(&baseline.body); + state + .commit_rewritten_manifest_with_semantics(baseline.body, baseline.resources, 1, Some(6_000), baseline_semantics) + .expect("baseline manifest commits"); + let baseline_body = state.last_manifest_body.clone().expect("baseline body"); + let baseline_generation = state.manifest_generation(); + let baseline_resource_count = state.resources.len(); + let oversized = rewritten_finalized_manifest(MAX_TRANSIENT_MANIFEST_RESOURCES + 1, "oversized"); + let oversized_semantics = parse_manifest_semantics(&oversized.body); + + let violation = local_representation_limit( + state + .commit_rewritten_manifest_with_semantics(oversized.body, oversized.resources, 2, None, oversized_semantics) + .expect_err("resource overflow is rejected"), + ); + + assert_eq!(violation.kind, HlsManifestLimitKind::TransientResources); + assert_eq!(violation.actual, MAX_TRANSIENT_MANIFEST_RESOURCES + 1); + assert_eq!(state.manifest_generation(), baseline_generation); + assert_eq!(state.resources.len(), baseline_resource_count); + assert!(Arc::ptr_eq(state.last_manifest_body.as_ref().expect("current body remains"), &baseline_body)); +} + +#[test] +fn retained_finalized_generation_limit_rejects_the_next_generation() { + let mut state = TransientPassthroughState::default(); + for index in 0..MAX_RETAINED_FINALIZED_MANIFEST_GENERATIONS { + let rewritten = rewritten_finalized_manifest(1, &format!("generation-{index}")); + let semantics = parse_manifest_semantics(&rewritten.body); + state + .commit_rewritten_manifest_with_semantics( + rewritten.body, + rewritten.resources, + u64::try_from(index).expect("test index fits u64"), + Some(6_000), + semantics, + ) + .expect("generation within retention limit commits"); + let generation = state.current_finalized_manifest_generation().expect("finalized generation"); + assert!(state.bind_finalized_manifest_generation(TransientManifestLeaseBinding::new( + HlsAccessLeaseId(format!("lease-{index}")), + u64::try_from(index).expect("test index fits u64"), + generation, + ))); + } + let retained_body = state.last_manifest_body.clone().expect("retained current body"); + let retained_generation = state.manifest_generation(); + let overflow = rewritten_finalized_manifest(1, "generation-overflow"); + let semantics = parse_manifest_semantics(&overflow.body); + + let violation = local_representation_limit( + state + .commit_rewritten_manifest_with_semantics(overflow.body, overflow.resources, 100, Some(6_000), semantics) + .expect_err("additional retained generation is rejected"), + ); + + assert_eq!(violation.kind, HlsManifestLimitKind::FinalizedGenerations); + assert_eq!(violation.actual, MAX_RETAINED_FINALIZED_MANIFEST_GENERATIONS + 1); + assert_eq!(state.manifest_generation(), retained_generation); + assert!(Arc::ptr_eq(state.last_manifest_body.as_ref().expect("current body remains"), &retained_body)); +} + +#[test] +fn byte_identical_finalized_refresh_reuses_generation_and_body_allocation() { + let mut state = TransientPassthroughState::default(); + let first = rewritten_finalized_manifest(3, "stable"); + let semantics = parse_manifest_semantics(&first.body); + state + .commit_rewritten_manifest_with_semantics(first.body, first.resources, 0, Some(18_000), semantics) + .expect("first finalized manifest commits"); + let generation = state.current_finalized_manifest_generation().expect("finalized generation"); + let generation_highwater = state.manifest_generation(); + let body = state.last_manifest_body.clone().expect("stored body"); + + let refreshed = TransientManifestRewriter::rewrite( + "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:1\n\ + #EXTINF:6,\nstable-0.ts\n#EXTINF:6,\nstable-1.ts\n#EXTINF:6,\nstable-2.ts\n#EXT-X-ENDLIST\n", + "https://origin.example/archive/index.m3u8", + &ProxySessionId("proxy-session".to_string()), + b"secret", + 400_000, + 300_000, + ); + let refreshed_resource_id = refreshed.resources[0].id.clone(); + state + .commit_rewritten_manifest_with_semantics(refreshed.body, refreshed.resources, 400_000, Some(18_000), semantics) + .expect("identical finalized refresh commits"); + + assert_eq!(state.current_finalized_manifest_generation(), Some(generation)); + assert_eq!(state.manifest_generation(), generation_highwater); + assert_eq!(state.finalized_manifest_generation_count(), 1); + assert!(Arc::ptr_eq(state.last_manifest_body.as_ref().expect("same body allocation"), &body)); + assert_eq!(state.resources[&refreshed_resource_id].expires_at_ms, 700_000); +} + +#[test] +fn rolling_lease_membership_retains_only_still_valid_previously_published_resources() { + let mut state = TransientPassthroughState::default(); + let previous_resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://origin.example/previous.ts", + b"secret", + 0, + 20, + Some("ts".to_string()), + ); + let next_resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://origin.example/next.ts", + b"secret", + 0, + 40, + Some("ts".to_string()), + ); + let previous_resource_id = previous_resource.id.clone(); + let next_resource_id = next_resource.id.clone(); + state.upsert_resources([previous_resource, next_resource]); + let previous = HlsPublishedTransientResourceIds::from_manifest_body(&format!( + "/hls/shared/live/proxy/lease/r/{}.ts", + previous_resource_id.0 + )); + let next = HlsPublishedTransientResourceIds::from_manifest_body(&format!( + "/hls/shared/live/proxy/lease/r/{}.ts", + next_resource_id.0 + )); + + let retained = state.merge_current_published_resource_ids(&previous, next.clone(), 10); + assert!(retained.contains(&previous_resource_id)); + assert!(retained.contains(&next_resource_id)); + + let pruned = state.merge_current_published_resource_ids(&retained, next, 21); + assert!(!pruned.contains(&previous_resource_id)); + assert!(pruned.contains(&next_resource_id)); +} + +#[test] +fn finalized_generation_membership_limit_counts_overlapping_sets() { + let mut state = TransientPassthroughState::default(); + let base = rewritten_finalized_manifest(MAX_TRANSIENT_MANIFEST_RESOURCES, "shared"); + let mut body = base.body; + let resources = base.resources; + let generations_within_limit = MAX_TRANSIENT_GENERATION_MEMBERSHIPS / MAX_TRANSIENT_MANIFEST_RESOURCES; + for index in 0..generations_within_limit { + let _ = writeln!(body, "# generation-{index}"); + state + .commit_rewritten_manifest_with_semantics( + body.clone(), + if index == 0 { resources.clone() } else { Vec::new() }, + u64::try_from(index).expect("index fits u64"), + None, + parse_manifest_semantics("#EXTM3U\n#EXT-X-ENDLIST\n"), + ) + .expect("membership count remains within limit"); + let generation = state.current_finalized_manifest_generation().expect("finalized generation"); + assert!(state.bind_finalized_manifest_generation(TransientManifestLeaseBinding::new( + HlsAccessLeaseId(format!("lease-{index}")), + u64::try_from(index).expect("index fits u64"), + generation, + ))); + } + body.push_str("# generation-overflow\n"); + + let violation = local_representation_limit( + state + .commit_rewritten_manifest_with_semantics( + body, + Vec::new(), + 100, + None, + parse_manifest_semantics("#EXTM3U\n#EXT-X-ENDLIST\n"), + ) + .expect_err("membership overflow is rejected"), + ); + + assert_eq!(violation.kind, HlsManifestLimitKind::TransientGenerationMemberships); + assert_eq!(violation.actual, MAX_TRANSIENT_GENERATION_MEMBERSHIPS + MAX_TRANSIENT_MANIFEST_RESOURCES); +} + +#[test] +fn manifest_refresh_does_not_extend_transient_ready_object_ttl() { + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 10, + 100, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + let key = TransientPassthroughState::transient_object_key( + &ProxySessionId("proxy-session".to_string()), + &resource_id, + "ts", + ); + state.upsert_resources([resource.clone()]); + replace_with_finalized_manifest(&mut state, &resource_id, "ts", 10); + let token = match state.begin_object_fetch(&ProxySessionId("proxy-session".to_string()), &resource, "ts", 20, 50) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("new resource starts a cache fetch") + } + }; + assert!(state.mark_object_ready_if_current(&token, "video/mp2t".to_string(), 7, 30, 110)); + assert_eq!(state.object_cache.get(&key).expect("object").expires_at_ms, 110); + + let updated = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 100, + 300, + Some("ts".to_string()), + ); + state.upsert_resources([updated]); + replace_with_finalized_manifest(&mut state, &resource_id, "ts", 100); + + let object = state.object_cache.get(&key).expect("object remains"); + assert!(matches!(object.status, TransientObjectCacheStatus::Ready { .. })); + assert_eq!(object.expires_at_ms, 110); +} + +#[test] +fn manifest_refresh_does_not_extend_a_ready_key_revision_and_rotation_reports_the_displaced_object() { + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let mut state = TransientPassthroughState::default(); + let first_resource = TransientResourceRef::new( + TransientResourceKind::Key, + "http://origin.example.com/live/key.key", + b"secret", + 10, + 100, + Some("key".to_string()), + ); + let resource_id = first_resource.id.clone(); + state.upsert_resources([first_resource.clone()]); + let first = match state.begin_object_fetch(&proxy_session_id, &first_resource, "key", 20, 50) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("first key revision starts a fetch") + } + }; + assert!(state.mark_object_ready_if_current(&first, "application/octet-stream".to_string(), 16, 30, 110,)); + + let refreshed_resource = TransientResourceRef::new( + TransientResourceKind::Key, + "http://origin.example.com/live/key.key", + b"secret", + 100, + 100, + Some("key".to_string()), + ); + assert_eq!(refreshed_resource.id, resource_id); + state.upsert_resources([refreshed_resource.clone()]); + let lookup_key = + TransientPassthroughState::transient_object_key(&proxy_session_id, &resource_id, "key".to_string()); + assert_eq!(state.object_cache.get(&lookup_key).map(|entry| entry.expires_at_ms), Some(110)); + + let second = match state.begin_object_fetch(&proxy_session_id, &refreshed_resource, "key", 111, 50) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("expired key revision starts a replacement fetch") + } + }; + let displaced = second.superseded_object().expect("replacement owns one bounded displaced object"); + assert_eq!(&displaced.key, first.cache_key()); + assert_eq!(displaced.content_length, 16); + assert_ne!(second.cache_key(), first.cache_key()); +} + +#[test] +fn stale_transient_fetch_token_cannot_overwrite_or_fail_a_new_generation() { + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Key, + "http://origin.example.com/live/key.key", + b"secret", + 10, + 100, + Some("key".to_string()), + ); + state.upsert_resources([resource.clone()]); + let first = match state.begin_object_fetch(&proxy_session_id, &resource, "key", 20, 50) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("first key fetch starts") + } + }; + assert!(state.mark_object_failed_retryable_if_current(&first, 21, 0)); + let second = match state.begin_object_fetch(&proxy_session_id, &resource, "key", 22, 50) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("replacement key fetch starts") + } + }; + assert_ne!(first.cache_key(), second.cache_key()); + + assert!(!state.mark_object_ready_if_current(&first, "application/octet-stream".to_string(), 16, 23, 110,)); + assert!(state.mark_object_ready_if_current(&second, "application/octet-stream".to_string(), 16, 24, 110,)); + assert!(!state.mark_object_failed_retryable_if_current(&first, 25, 0)); + assert!(matches!( + state.object_cache.get(&second.lookup_key).map(|entry| &entry.status), + Some(TransientObjectCacheStatus::Ready { content_length: 16, .. }) + )); +} + +#[test] +fn finalized_manifest_resource_accepts_late_fetch_commit() { + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/archive/short.ts", + b"secret", + 10, + 10, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + state.upsert_resources([resource]); + replace_with_finalized_manifest(&mut state, &resource_id, "ts", 10); + let resource = state.resolve_current_resource(&resource_id, 15).expect("finalized resource resolves"); + let token = match state.begin_object_fetch(&proxy_session_id, &resource, "ts", 15, 100) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("valid finalized resource starts a cache fill") + } + }; + + replace_with_finalized_manifest(&mut state, &resource_id, "ts", 18); + assert_eq!(state.manifest_generation(), 1); + assert!(state.mark_object_ready_if_current(&token, "video/mp2t".to_string(), 7, 21, 121)); + assert!(state.object_cache.contains_key(&token.lookup_key)); +} + +#[test] +fn finalized_prune_keeps_resources_referenced_by_current_manifest() { + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/seg.ts", + b"secret", + 0, + 10, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + state.upsert_resources([resource]); + replace_with_finalized_manifest(&mut state, &resource_id, "ts", 0); + + state.prune_expired(20); + + assert!(state.resources.contains_key(&resource_id)); +} diff --git a/backend/hls/src/transient/tests/storage.rs b/backend/hls/src/transient/tests/storage.rs new file mode 100644 index 000000000..2a6e8601d --- /dev/null +++ b/backend/hls/src/transient/tests/storage.rs @@ -0,0 +1,19 @@ +use super::*; + +#[test] +fn prospective_origin_uri_bytes_counts_duplicate_resource_ids_once() { + let state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://origin.example/archive/segment.ts", + b"secret", + 0, + 300_000, + Some("ts".to_string()), + ); + + assert_eq!( + state.prospective_origin_uri_bytes(&[resource.clone(), resource.clone()]), + resource.resolved_origin_uri.len() + ); +} diff --git a/backend/hls/src/transient/tests/streaming.rs b/backend/hls/src/transient/tests/streaming.rs new file mode 100644 index 000000000..d8ff5c2bb --- /dev/null +++ b/backend/hls/src/transient/tests/streaming.rs @@ -0,0 +1,69 @@ +use super::*; + +#[test] +fn transient_session_origin_uri_byte_limit_is_enforced() { + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "x".repeat(MAX_TRANSIENT_ORIGIN_URI_BYTES_PER_SESSION + 1), + b"secret", + 0, + 300_000, + Some("ts".to_string()), + ); + + let violation = local_representation_limit( + state + .commit_rewritten_manifest_with_semantics( + finalized_manifest_body(&resource.id, "ts"), + vec![resource], + 1, + Some(1_000), + parse_manifest_semantics("#EXTM3U\n#EXT-X-ENDLIST\n"), + ) + .expect_err("origin URI byte overflow is rejected"), + ); + + assert_eq!(violation.kind, HlsManifestLimitKind::TransientOriginUriBytes); + assert!(state.resources.is_empty()); +} + +#[test] +fn expired_finalized_object_is_refetched_instead_of_served_stale() { + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/archive/refetch.ts", + b"secret", + 0, + 10, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + state.upsert_resources([resource]); + replace_with_finalized_manifest(&mut state, &resource_id, "ts", 0); + let resource = state.resolve_current_resource(&resource_id, 5).expect("finalized mapping"); + let first = match state.begin_object_fetch(&proxy_session_id, &resource, "ts", 5, 5) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("first cache fill starts") + } + }; + assert!(state.mark_object_ready_if_current( + &first, + "video/mp2t".to_string(), + 7, + 6, + transient_object_expires_at(6, 5), + )); + let lookup_key = TransientPassthroughState::transient_object_key(&proxy_session_id, &resource_id, "ts"); + + assert!(state.ready_object(&lookup_key, TransientResourceKind::Segment, 11).is_some()); + assert!(state.ready_object(&lookup_key, TransientResourceKind::Segment, 12).is_none()); + let resource = state.resolve_current_resource(&resource_id, 12).expect("mapping outlives cached bytes"); + assert!(matches!( + state.begin_object_fetch(&proxy_session_id, &resource, "ts", 12, 5), + super::super::TransientObjectFetchDecision::Fetch(_) + )); +} diff --git a/backend/hls/src/transient/tests/transport.rs b/backend/hls/src/transient/tests/transport.rs new file mode 100644 index 000000000..467e17d7c --- /dev/null +++ b/backend/hls/src/transient/tests/transport.rs @@ -0,0 +1,217 @@ +use super::*; + +#[test] +fn extract_transient_resource_ids_rejects_noncanonical_resource_files() { + let body = concat!( + "/hls/shared/live/proxy/lease/r/AbCdEfGhIjKlMnOp.exe\n", + "/hls/shared/live/proxy/lease/r/AbCdEfGhIjKlMnOp.ts/extra\n", + "/hls/shared/live/proxy/lease/r/AbCdEfGhIjKlMnOp.tar.gz\n", + ); + + assert!(extract_transient_resource_ids(body).is_empty()); +} + +#[test] +fn estimated_transient_metadata_limit_counts_owned_hint_bytes() { + let mut state = TransientPassthroughState::default(); + let mut retained = TransientResourceRef::new( + TransientResourceKind::Segment, + "https://origin.example/retained.ts".to_string(), + b"secret", + 0, + 300_000, + Some("ts".to_string()), + ); + retained.content_type_hint = Some("x".repeat(MAX_ESTIMATED_TRANSIENT_METADATA_BYTES)); + state.upsert_resources([retained]); + let candidate = rewritten_finalized_manifest(1, "candidate"); + let semantics = parse_manifest_semantics(&candidate.body); + + let violation = local_representation_limit( + state + .commit_rewritten_manifest_with_semantics(candidate.body, candidate.resources, 1, Some(6_000), semantics) + .expect_err("estimated metadata overflow is rejected"), + ); + + assert_eq!(violation.kind, HlsManifestLimitKind::TransientEstimatedMetadataBytes); + assert!(violation.actual > MAX_ESTIMATED_TRANSIENT_METADATA_BYTES); +} + +#[test] +fn aes_key_readiness_requires_exactly_sixteen_bytes_and_honors_the_inclusive_ttl_boundary() { + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + for invalid_size in [15, 17] { + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Key, + format!("http://origin.example.com/live/key-{invalid_size}.key"), + b"secret", + 10, + 100, + Some("key".to_string()), + ); + let resource_id = resource.id.clone(); + state.upsert_resources([resource.clone()]); + let token = match state.begin_object_fetch(&proxy_session_id, &resource, "key", 20, 50) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("new AES key starts a cache fetch") + } + }; + assert!(state.mark_object_ready_if_current( + &token, + "application/octet-stream".to_string(), + invalid_size, + 30, + 110, + )); + assert_eq!(state.ready_key_object_valid_until_ms(&proxy_session_id, &resource_id, "key", 30), None); + } + + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Key, + "http://origin.example.com/live/valid.key", + b"secret", + 10, + 100, + Some("key".to_string()), + ); + let resource_id = resource.id.clone(); + state.upsert_resources([resource.clone()]); + let token = match state.begin_object_fetch(&proxy_session_id, &resource, "key", 20, 50) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("new AES key starts a cache fetch") + } + }; + assert!(state.mark_object_ready_if_current(&token, "application/octet-stream".to_string(), 16, 30, 110,)); + + assert_eq!(state.ready_key_object_valid_until_ms(&proxy_session_id, &resource_id, "key", 110), Some(110)); + assert_eq!(state.ready_key_object_valid_until_ms(&proxy_session_id, &resource_id, "key", 111), None); +} + +#[tokio::test] +async fn controlled_resource_mapping_replacement_invalidates_the_fetch_token_and_wakes_its_waiter() { + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let mut state = TransientPassthroughState::default(); + let original = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin-a.example.com/live/segment.ts", + b"secret", + 10, + 100, + Some("ts".to_string()), + ); + let resource_id = original.id.clone(); + state.upsert_resources([original]); + let original = state.resources.get(&resource_id).expect("registered original mapping").clone(); + let token = match state.begin_object_fetch(&proxy_session_id, &original, "ts", 20, 50) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("new resource starts a cache fetch") + } + }; + let notifier = match state.begin_object_fetch(&proxy_session_id, &original, "ts", 21, 50) { + super::super::TransientObjectFetchDecision::Wait(notifier) => notifier, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Fetch(_) => { + panic!("concurrent request waits for the owned fill") + } + }; + let waiter = notifier.notified(); + tokio::pin!(waiter); + assert!(matches!(futures::poll!(&mut waiter), Poll::Pending)); + + let mut replacement = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin-b.example.com/live/segment.ts", + b"secret", + 22, + 100, + Some("ts".to_string()), + ); + replacement.id = resource_id; + state.upsert_resources([replacement]); + + assert!(!state.mark_object_ready_if_current(&token, "video/mp2t".to_string(), 7, 23, 122)); + assert!(matches!(futures::poll!(&mut waiter), Poll::Ready(()))); + assert!(!state.object_cache.contains_key(&token.lookup_key)); + let replacement = state.resources.get(&token.binding.resource_id).expect("replacement mapping").clone(); + let replacement_token = match state.begin_object_fetch(&proxy_session_id, &replacement, "ts", 24, 50) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("replacement mapping owns a new cache fill") + } + }; + assert_ne!(token.cache_key(), replacement_token.cache_key()); + assert_ne!(token.binding.revision, replacement_token.binding.revision); +} + +#[test] +fn expired_resource_validity_rejects_a_late_fetch_commit() { + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let mut state = TransientPassthroughState::default(); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/short.ts", + b"secret", + 10, + 10, + Some("ts".to_string()), + ); + let resource_id = resource.id.clone(); + state.upsert_resources([resource]); + state.replace_manifest_with_semantics( + format!("#EXTM3U\n#EXTINF:1,\n/hls/shared/live/session/lease/r/{}.ts\n", resource_id.0), + 10, + None, + ); + let resource = state.resources.get(&resource_id).expect("registered resource").clone(); + let token = match state.begin_object_fetch(&proxy_session_id, &resource, "ts", 15, 100) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("valid resource starts a cache fill") + } + }; + + assert!(!state.mark_object_ready_if_current(&token, "video/mp2t".to_string(), 7, 21, 115)); + assert!(!state.object_cache.contains_key(&token.lookup_key)); +} + +#[test] +fn finalized_resource_replacement_still_rejects_stale_fetch_commit() { + let proxy_session_id = ProxySessionId("proxy-session".to_string()); + let mut state = TransientPassthroughState::default(); + let original = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/archive/original.ts", + b"secret", + 10, + 10, + Some("ts".to_string()), + ); + let resource_id = original.id.clone(); + state.upsert_resources([original]); + replace_with_finalized_manifest(&mut state, &resource_id, "ts", 10); + let original = state.resolve_current_resource(&resource_id, 15).expect("original mapping"); + let token = match state.begin_object_fetch(&proxy_session_id, &original, "ts", 15, 100) { + super::super::TransientObjectFetchDecision::Fetch(token) => token, + super::super::TransientObjectFetchDecision::Ready | super::super::TransientObjectFetchDecision::Wait(_) => { + panic!("original finalized resource starts a cache fill") + } + }; + let mut replacement = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://replacement.example.com/archive/replacement.ts", + b"secret", + 16, + 100, + Some("ts".to_string()), + ); + replacement.id = resource_id.clone(); + state.upsert_resources([replacement]); + replace_with_finalized_manifest(&mut state, &resource_id, "ts", 16); + + assert!(!state.mark_object_ready_if_current(&token, "video/mp2t".to_string(), 7, 21, 121)); + assert!(!state.object_cache.contains_key(&token.lookup_key)); +} diff --git a/backend/hls/src/transient_fetcher.rs b/backend/hls/src/transient_fetcher.rs index 0715d013f..808528a19 100644 --- a/backend/hls/src/transient_fetcher.rs +++ b/backend/hls/src/transient_fetcher.rs @@ -13,150 +13,8 @@ use super::{ TransientObjectFetchDecision, TransientObjectFetchToken, TransientPassthroughState, TransientResourceFile, TransientResourceKind, TransientResourceRef, }; -use axum::{ - body::Body, - http::{header, HeaderMap, HeaderValue, StatusCode}, - response::IntoResponse, -}; -use futures::{future::BoxFuture, FutureExt, StreamExt}; -use log::{debug, warn}; -use std::{io, sync::Arc}; -use tokio::{sync::Notify, time::sleep}; -use tokio_util::io::ReaderStream; -use tuliprox_core::{ - try_unwrap_body, - utils::{content_coding::DecodedHttpResponse, response_compression::mark_response_as_uncompressed}, -}; - -pub enum HlsTransientObjectFetchFailure { - Retryable, - Permanent { status: Option }, -} - -pub fn hls_transient_object_fetch_failure(error: &HlsOriginResourceFetchError) -> HlsTransientObjectFetchFailure { - if let Some(status) = error.permanent_status() { - return HlsTransientObjectFetchFailure::Permanent { status: Some(status) }; - } - if !error.retryable_failure() { - return HlsTransientObjectFetchFailure::Permanent { status: None }; - } - HlsTransientObjectFetchFailure::Retryable -} - -pub fn hls_transient_resource_fetch_kind(resource_kind: TransientResourceKind) -> HlsResourceFetchKind { - match resource_kind { - TransientResourceKind::Key => HlsResourceFetchKind::Key, - TransientResourceKind::Map => HlsResourceFetchKind::Map, - TransientResourceKind::Segment => HlsResourceFetchKind::Segment, - TransientResourceKind::Part => HlsResourceFetchKind::Part, - TransientResourceKind::Other => HlsResourceFetchKind::Other, - } -} - -fn build_hls_transient_resource_fetch_target( - resolved_origin_uri: &str, - origin_headers: &HeaderMap, - origin_provider_session_headers: &HeaderMap, - mode: HlsTransientOriginFetchMode, - resource_id: &str, - resource_kind: TransientResourceKind, -) -> Result { - let (range_header, byte_range_expectation) = match mode { - HlsTransientOriginFetchMode::CacheFullObject => (None, HlsOriginByteRangeExpectation::FullObject), - HlsTransientOriginFetchMode::DirectPassthrough { client_range } => { - (client_range, HlsOriginByteRangeExpectation::AnySuccess) - } - }; - Ok(HlsOriginResourceFetchTarget { - kind: hls_transient_resource_fetch_kind(resource_kind), - source: HlsResourceFetchSource::Transient, - object_id: resource_id.to_string(), - origin_url: resolved_origin_uri.to_string(), - headers: build_hls_origin_resource_headers_with_client_range( - origin_headers, - origin_provider_session_headers, - range_header, - )?, - byte_range_expectation, - }) -} - -/// Keeps full-object cache fills separate from decoded client-range passthrough. -enum HlsTransientOriginFetchMode { - CacheFullObject, - DirectPassthrough { client_range: Option }, -} - -pub struct HlsTransientOriginFetchRequest { - pub resolved_origin_uri: String, - pub origin_headers: HeaderMap, - pub origin_provider_session_headers: HeaderMap, - pub range_header: Option, - pub resource_file: TransientResourceFile, - pub resource_kind: TransientResourceKind, - pub clients: HlsOriginResourceClients, - pub policy: SegmentFetchPolicy, - pub log_identity: HlsLogIdentity, -} - -/// A decoded direct-origin response together with the attempt and guards that own its origin work. -pub struct HlsTransientDecodedOriginResponse { - pub decoded: DecodedHttpResponse, - pub body_deadline: HlsOriginResourceBodyDeadline, - pub attempt: HlsResourceFetchAttempt, - pub guard: G, -} - -pub async fn fetch_hls_transient_origin_response_with_attempt_prepare( - request: HlsTransientOriginFetchRequest, - prepare_attempt: P, -) -> Result, HlsOriginResourceFetchError> -where - G: Send + 'static, - P: FnMut(HlsResourceFetchAttempt) -> BoxFuture<'static, Result>, -{ - let target = build_hls_transient_resource_fetch_target( - &request.resolved_origin_uri, - &request.origin_headers, - &request.origin_provider_session_headers, - HlsTransientOriginFetchMode::DirectPassthrough { client_range: request.range_header }, - request.resource_file.resource_id.0.as_str(), - request.resource_kind, - )?; - run_hls_origin_resource_retry_loop_with_attempt_prepare( - target, - request.clients, - &request.policy, - &request.log_identity, - prepare_attempt, - |guard| async move { drop(guard) }.boxed(), - |decoded, attempt, body_deadline, guard| { - async move { Ok(HlsTransientDecodedOriginResponse { decoded, body_deadline, attempt, guard }) }.boxed() - }, - ) - .await -} - -pub enum HlsTransientObjectCacheAction { - ServeReady, - FetchAndCache(Box), - WaitForFetch(Arc), - PassthroughNoCache, -} - -pub struct HlsTransientObjectCacheResolution { - pub resource: TransientResourceRef, - pub origin_headers: HeaderMap, - pub origin_provider_session_headers: HeaderMap, - pub action: HlsTransientObjectCacheAction, -} - -#[derive(Clone, Copy)] -pub struct HlsTransientResourceLeaseContext<'a> { - pub access_lease_id: &'a HlsAccessLeaseId, - pub lease_issued_at_ms: u64, - pub published_resource_ids: &'a HlsPublishedTransientResourceIds, -} +use axum::http::HeaderValue; +use std::sync::Arc; #[derive(Clone, Copy)] struct HlsTransientObjectCacheActionInput<'a> { @@ -169,104 +27,6 @@ struct HlsTransientObjectCacheActionInput<'a> { key_object_cache_allowed: bool, } -pub async fn resolve_hls_transient_object_cache_action( - session: &HlsSessionHandle, - proxy_session_id: &ProxySessionId, - lease: HlsTransientResourceLeaseContext<'_>, - resource_file: &TransientResourceFile, - range_header: Option<&HeaderValue>, - now_ms: u64, - cache_duration_ms: u64, -) -> Result { - // `is_gc_marked_for_removal` is `&self` and is the dominant early-exit on a - // busy session (GC sweeps mark sessions on a timer). Resolve it under a read - // lock so we don't pay for the exclusive write-lock acquisition when the - // session is already doomed. - if session.read().await.is_gc_marked_for_removal() { - return Err(StatusCode::NOT_FOUND); - } - let mut session = session.write().await; - let Some(resource) = session.transient.resolve_resource_for_lease( - &resource_file.resource_id, - lease.access_lease_id, - lease.lease_issued_at_ms, - lease.published_resource_ids, - now_ms, - ) else { - return Err(StatusCode::NOT_FOUND); - }; - if resource.file_ext_hint.as_deref().is_some_and(|extension| extension != resource_file.extension) { - return Err(StatusCode::NOT_FOUND); - } - let key_object_cache_allowed = - resource.kind != TransientResourceKind::Key || session.mode == HlsSessionMode::NormalCacheTimeline; - let action = transient_object_cache_action( - &mut session, - HlsTransientObjectCacheActionInput { - proxy_session_id, - resource: &resource, - resource_file, - range_header, - now_ms, - cache_duration_ms, - key_object_cache_allowed, - }, - ); - Ok(HlsTransientObjectCacheResolution { - resource, - origin_headers: session.origin_request_headers.clone(), - origin_provider_session_headers: session.origin_provider_session_headers.clone(), - action, - }) -} - -fn transient_object_cache_action( - session: &mut super::HlsSession, - input: HlsTransientObjectCacheActionInput<'_>, -) -> HlsTransientObjectCacheAction { - if input.resource.kind == TransientResourceKind::Key && !input.key_object_cache_allowed { - return HlsTransientObjectCacheAction::PassthroughNoCache; - } - let cache_key = TransientPassthroughState::transient_object_key( - input.proxy_session_id, - &input.resource.id, - input.resource_file.extension.clone(), - ); - if session.transient.ready_object(&cache_key, input.resource.kind, input.now_ms).is_some() { - return HlsTransientObjectCacheAction::ServeReady; - } - if !is_hls_transient_full_object_cacheable_request(input.range_header) { - return HlsTransientObjectCacheAction::PassthroughNoCache; - } - session - .transient - .begin_object_fetch( - input.proxy_session_id, - input.resource, - &input.resource_file.extension, - input.now_ms, - input.cache_duration_ms, - ) - .into() -} - -impl From for HlsTransientObjectCacheAction { - fn from(decision: TransientObjectFetchDecision) -> Self { - match decision { - TransientObjectFetchDecision::Ready => Self::ServeReady, - TransientObjectFetchDecision::Fetch(cache_key) => Self::FetchAndCache(cache_key), - TransientObjectFetchDecision::Wait(notifier) => Self::WaitForFetch(notifier), - } - } -} - -pub fn is_hls_transient_full_object_cacheable_request(range_header: Option<&HeaderValue>) -> bool { - let Some(range_header) = range_header else { - return true; - }; - range_header.to_str().is_ok_and(|range| range.trim() == "bytes=0-") -} - pub struct HlsTransientObjectFetchFinalizer { session: HlsSessionHandle, segment_cache: Arc, @@ -275,206 +35,6 @@ pub struct HlsTransientObjectFetchFinalizer { retry_after_ms: u64, } -impl HlsTransientObjectFetchFinalizer { - pub fn new( - session: HlsSessionHandle, - segment_cache: Arc, - fetch_token: TransientObjectFetchToken, - retry_after_ms: u64, - ) -> Self { - Self { session, segment_cache, fetch_token, completed: false, retry_after_ms } - } - - pub fn complete(&mut self) { self.completed = true; } -} - -impl Drop for HlsTransientObjectFetchFinalizer { - fn drop(&mut self) { - if self.completed { - return; - } - let session = Arc::clone(&self.session); - let segment_cache = Arc::clone(&self.segment_cache); - let fetch_token = self.fetch_token.clone(); - let retry_after_ms = self.retry_after_ms; - tokio::spawn(async move { - if let Err(err) = segment_cache.delete(fetch_token.cache_key()).await { - debug!("HLS abandoned transient cache fill cleanup failed: error={err}"); - } - delete_superseded_transient_object(&segment_cache, &fetch_token).await; - session.write().await.fail_transient_object_retryable_if_current( - &fetch_token, - current_time_millis(), - retry_after_ms, - ); - }); - } -} - -pub(super) async fn delete_superseded_transient_object( - segment_cache: &HlsSegmentCache, - fetch_token: &TransientObjectFetchToken, -) { - let Some(superseded) = fetch_token.superseded_object() else { - return; - }; - if let Err(err) = segment_cache.delete(&superseded.key).await { - debug!("HLS superseded transient cache object cleanup failed: error={err}"); - } -} - -#[derive(Clone)] -pub struct HlsTransientCacheCommitContext { - pub segment_cache: Arc, - pub segment_repair: Arc, - pub session: HlsSessionHandle, - pub proxy_session_id: ProxySessionId, - pub log_identity: super::HlsLogIdentity, - pub access_lease_id: HlsAccessLeaseId, - pub resource: TransientResourceRef, - pub resource_file: TransientResourceFile, - pub fetch_token: TransientObjectFetchToken, - pub cache_duration_ms: u64, -} - -pub struct HlsTransientOriginCacheFetchRequest { - pub fetch: HlsTransientOriginFetchRequest, - pub commit: HlsTransientCacheCommitContext, -} - -pub async fn fetch_and_commit_hls_transient_origin_response_with_attempt_prepare( - request: HlsTransientOriginCacheFetchRequest, - prepare_attempt: P, -) -> Result<(), HlsOriginResourceFetchError> -where - G: Send + 'static, - P: FnMut(HlsResourceFetchAttempt) -> BoxFuture<'static, Result>, -{ - let target = build_hls_transient_resource_fetch_target( - &request.fetch.resolved_origin_uri, - &request.fetch.origin_headers, - &request.fetch.origin_provider_session_headers, - HlsTransientOriginFetchMode::CacheFullObject, - request.fetch.resource_file.resource_id.0.as_str(), - request.fetch.resource_kind, - )?; - let commit = request.commit; - run_hls_origin_resource_retry_loop_with_attempt_prepare( - target, - request.fetch.clients, - &request.fetch.policy, - &request.fetch.log_identity, - prepare_attempt, - |guard| async move { drop(guard) }.boxed(), - move |decoded, _attempt, body_deadline, guard| { - let commit = commit.clone(); - async move { - let result = commit_hls_transient_origin_response_attempt(commit, decoded, body_deadline).await; - drop(guard); - result - } - .boxed() - }, - ) - .await -} - -async fn commit_hls_transient_origin_response_attempt( - context: HlsTransientCacheCommitContext, - decoded: DecodedHttpResponse, - body_deadline: HlsOriginResourceBodyDeadline, -) -> Result<(), HlsOriginResourceFetchError> { - let content_type = decoded - .headers - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .map(str::to_string) - .or_else(|| context.resource.content_type_hint.clone()) - .unwrap_or_else(|| "application/octet-stream".to_string()); - let proxy_session_id = context.proxy_session_id.clone(); - let log_identity = context.log_identity.clone(); - let repair_context = HlsSegmentRepairObjectContext { - source: HlsSegmentRepairSource::Transient, - log_identity, - proxy_session_id, - hls_access_lease_id: Some(context.access_lease_id.clone()), - rendered_object_id: HlsRepairRenderedObjectId::Transient { - resource_id: context.resource_file.resource_id.0.clone(), - }, - resource_id: context.resource_file.resource_id.0.clone(), - file_ext: context.resource_file.extension.clone(), - origin_fetch_uri_for_diagnostics: context.resource.resolved_origin_uri.clone(), - media_sequence: None, - discontinuity_sequence: None, - complete_object: true, - encrypted: context.resource.encrypted_media || context.resource.kind == TransientResourceKind::Key, - custom_response: false, - }; - let cache_key = context.fetch_token.cache_key().clone(); - let commit = Box::pin(context.segment_repair.commit_origin_response( - &context.segment_cache, - &cache_key, - decoded.body, - body_deadline.deadline(), - repair_context, - )) - .await; - let ready_at_ms = current_time_millis(); - let metadata = match commit { - Ok(metadata) => metadata, - Err(err) => return Err(HlsOriginResourceFetchError::cache_body(&err)), - }; - let expires_at_ms = transient_object_expires_at(ready_at_ms, context.cache_duration_ms); - let mut session = context.session.write().await; - if context.resource.kind == TransientResourceKind::Key && metadata.size != 16 { - session.fail_transient_object_permanent_if_current( - &context.fetch_token, - ready_at_ms, - Some(StatusCode::BAD_GATEWAY), - ); - drop(session); - delete_stale_transient_fill(&context.segment_cache, &cache_key).await; - return Err(HlsOriginResourceFetchError::NonRetryableStatus(StatusCode::BAD_GATEWAY)); - } - let committed = session.commit_transient_object_ready_if_current( - context.resource.kind, - &context.fetch_token, - content_type, - metadata.size, - ready_at_ms, - expires_at_ms, - ); - drop(session); - if !committed { - delete_stale_transient_fill(&context.segment_cache, &cache_key).await; - return Err(HlsOriginResourceFetchError::Superseded); - } - delete_superseded_transient_object(&context.segment_cache, &context.fetch_token).await; - Ok(()) -} - -async fn delete_stale_transient_fill(segment_cache: &HlsSegmentCache, cache_key: &TransientObjectCacheKey) { - if let Err(err) = segment_cache.delete(cache_key).await { - debug!("HLS stale transient cache fill cleanup failed: error={err}"); - } -} - -/// Context retained until a direct transient response body reaches one terminal outcome. -pub struct HlsTransientDirectResponseContext { - pub session: HlsSessionHandle, - pub resource: TransientResourceRef, - pub policy: SegmentFetchPolicy, - pub now_ms: u64, - pub log_identity: HlsLogIdentity, -} - -#[derive(Clone, Copy)] -enum HlsTransientDirectStreamOutcome { - CleanEof, - OriginBodyFailure, - ClientAborted, -} - // Sole owner of the direct-body outcome: EOF is success, origin read failures // degrade affected media/dependencies, and downstream cancellation remains neutral. struct HlsTransientDirectResponseFinalizer { @@ -487,292 +47,10 @@ struct HlsTransientDirectResponseLifecycleContext { policy: SegmentFetchPolicy, } -impl HlsTransientDirectResponseFinalizer { - fn new(context: HlsTransientDirectResponseLifecycleContext) -> Self { - let context = transient_resource_affects_media_readiness(context.resource.kind).then_some(context); - Self { context } - } - - async fn finish(&mut self, outcome: HlsTransientDirectStreamOutcome) { - let Some(completion) = self.begin_finish(outcome) else { - return; - }; - if let Err(error) = completion.await { - warn!( - "HLS transient direct lifecycle task failed: cancelled={} panic={}", - error.is_cancelled(), - error.is_panic() - ); - } - } - - fn begin_finish(&mut self, outcome: HlsTransientDirectStreamOutcome) -> Option> { - let context = self.context.take()?; - if matches!(outcome, HlsTransientDirectStreamOutcome::ClientAborted) { - return None; - } - // Once EOF or an origin-body failure has been observed, its state transition - // must survive cancellation of the downstream body poll. - Some(tokio::spawn(async move { - match outcome { - HlsTransientDirectStreamOutcome::CleanEof => { - record_successful_transient_segment_fetch(&context.session, &context.resource).await; - } - HlsTransientDirectStreamOutcome::OriginBodyFailure => { - let failed_at_ms = current_time_millis(); - record_temporary_transient_segment_fetch_failure( - &context.session, - &context.resource, - &context.policy, - failed_at_ms, - ) - .await; - } - HlsTransientDirectStreamOutcome::ClientAborted => {} - } - })) - } - - fn finish_client_aborted(&mut self) { self.context.take(); } -} - -impl Drop for HlsTransientDirectResponseFinalizer { - fn drop(&mut self) { - // A body dropped before EOF is a downstream abort. It must release the - // retained guards without changing provider/segment failure state. - self.finish_client_aborted(); - } -} - -pub fn hls_transient_origin_response( - response: HlsTransientDecodedOriginResponse>, - context: HlsTransientDirectResponseContext, -) -> axum::response::Response { - let mut builder = axum::response::Response::builder().status(response.decoded.status); - for header_name in [ - header::CONTENT_TYPE, - header::CONTENT_LENGTH, - header::CONTENT_RANGE, - header::ACCEPT_RANGES, - header::CACHE_CONTROL, - header::ETAG, - header::LAST_MODIFIED, - ] { - if let Some(value) = response.decoded.headers.get(&header_name) { - builder = builder.header(header_name, value.clone()); - } - } - - let mut response = try_unwrap_body!(builder.body(hls_transient_direct_body(response, context))); - mark_response_as_uncompressed(&mut response); - response -} - -fn hls_transient_direct_body( - response: HlsTransientDecodedOriginResponse>, - context: HlsTransientDirectResponseContext, -) -> Body { - let HlsTransientDecodedOriginResponse { decoded, body_deadline, attempt, guard: origin_io_guard } = response; - let HlsTransientDirectResponseContext { session, resource, policy, now_ms, log_identity } = context; - - let guard = HlsTransientReadGuard::new(Arc::clone(&resource.access), now_ms); - let finalizer = HlsTransientDirectResponseFinalizer::new(HlsTransientDirectResponseLifecycleContext { - session, - resource: resource.clone(), - policy, - }); - let resource_id = resource.id.0.clone(); - let resource_kind = resource.kind; - let stream = futures::stream::unfold( - ( - ReaderStream::new(decoded.body), - Some(guard), - origin_io_guard, - finalizer, - Box::pin(sleep(hls_client_body_send_deadline())), - false, - ), - move |(mut stream, guard, mut origin_io_guard, mut finalizer, mut send_deadline, finished)| { - let log_identity = log_identity.clone(); - let resource_id = resource_id.clone(); - async move { - if finished { - return None; - } - let next_chunk = tokio::select! { - () = send_deadline.as_mut() => { - finalizer.finish(HlsTransientDirectStreamOutcome::ClientAborted).await; - return Some(( - Err(io::Error::new(io::ErrorKind::TimedOut, "hls client body send timed out")), - (stream, None, None, finalizer, send_deadline, true), - )); - } - // Decoder setup is bounded by the absolute attempt deadline before this - // response is built. Once handed to the client, retain the existing - // per-chunk origin-body idle timeout instead of imposing a total-body limit. - next_chunk = tokio::time::timeout(body_deadline.timeout(), stream.next()) => next_chunk, - }; - match next_chunk { - Ok(Some(Ok(chunk))) => { - refresh_hls_client_body_send_deadline(send_deadline.as_mut()); - Some((Ok(chunk), (stream, guard, origin_io_guard, finalizer, send_deadline, false))) - } - Ok(Some(Err(err))) => { - log_hls_resource_body_failure( - &log_identity, - hls_transient_direct_log_context(&resource_id, resource_kind), - attempt, - &err, - body_deadline.timeout().as_millis(), - ); - finalizer.finish(HlsTransientDirectStreamOutcome::OriginBodyFailure).await; - Some((Err(err), (stream, None, None, finalizer, send_deadline, true))) - } - Ok(None) => { - finalizer.finish(HlsTransientDirectStreamOutcome::CleanEof).await; - if let Some(guard) = origin_io_guard.take() { - guard.finish_clean().await; - } - None - } - Err(_) => { - let error = io::Error::new(io::ErrorKind::TimedOut, "transient passthrough body timed out"); - log_hls_resource_body_failure( - &log_identity, - hls_transient_direct_log_context(&resource_id, resource_kind), - attempt, - &error, - body_deadline.timeout().as_millis(), - ); - finalizer.finish(HlsTransientDirectStreamOutcome::OriginBodyFailure).await; - Some((Err(error), (stream, None, None, finalizer, send_deadline, true))) - } - } - } - }, - ); - Body::from_stream(stream) -} - -fn hls_transient_direct_log_context( - resource_id: &str, - resource_kind: TransientResourceKind, -) -> HlsResourceFetchLogContext<'_> { - HlsResourceFetchLogContext { - kind: hls_transient_resource_fetch_kind(resource_kind), - source: HlsResourceFetchSource::Transient, - object_id: resource_id, - origin_url: None, - } -} - -pub async fn record_successful_transient_segment_fetch(session: &HlsSessionHandle, resource: &TransientResourceRef) { - if !transient_resource_is_media(resource.kind) { - return; - } - let mut session = session.write().await; - if !transient_resource_is_current(&session, resource, current_time_millis()) { - return; - } - if let Some(reset_failures) = session.record_successful_segment_fetch() { - if log::log_enabled!(log::Level::Debug) { - let identity = HlsLogIdentity::from_session(&session); - debug!( - "HLS segment temporary failure counter reset: session={} proxy_session={} previous_failures={reset_failures}", - identity.session(), - identity.proxy_session() - ); - } - } -} - -pub async fn record_temporary_transient_segment_fetch_failure( - session: &HlsSessionHandle, - resource: &TransientResourceRef, - policy: &SegmentFetchPolicy, - now_ms: u64, -) -> bool { - if !transient_resource_affects_media_readiness(resource.kind) { - return false; - } - let mut session = session.write().await; - if !transient_resource_is_current(&session, resource, now_ms) { - return false; - } - session.origin_control.path_condition = super::origin_progress::HlsOriginPathCondition::SegmentReadinessFailure; - if !transient_resource_is_media(resource.kind) { - return false; - } - let threshold = policy.permanent_failure_segment_threshold.max(1); - match session.record_temporary_segment_fetch_failure( - now_ms, - HlsSegmentFailureObject::Transient { resource_id: resource.id.0.clone() }, - threshold, - ) { - HlsSegmentFailureTransition::StillRetryable { failures, threshold } => { - if log::log_enabled!(log::Level::Debug) { - let identity = HlsLogIdentity::from_session(&session); - debug!( - "HLS segment temporary failure counted: session={} proxy_session={} object={} failures={} threshold={}", - identity.session(), - identity.proxy_session(), - resource.id.0, - failures, - threshold - ); - } - false - } - HlsSegmentFailureTransition::BecamePermanentlyFailed { failures, threshold } => { - if log::log_enabled!(log::Level::Warn) { - let identity = HlsLogIdentity::from_session(&session); - warn!( - "HLS segment temporary failure threshold reached: session={} proxy_session={} failures={} threshold={}", - identity.session(), - identity.proxy_session(), - failures, - threshold - ); - } - session.invalidate_queued_origin_work(); - true - } - } -} - -const fn transient_resource_is_media(kind: TransientResourceKind) -> bool { - matches!(kind, TransientResourceKind::Segment | TransientResourceKind::Part) -} - -const fn transient_resource_affects_media_readiness(kind: TransientResourceKind) -> bool { - matches!( - kind, - TransientResourceKind::Segment - | TransientResourceKind::Part - | TransientResourceKind::Key - | TransientResourceKind::Map - ) -} - -fn transient_resource_is_current(session: &super::HlsSession, resource: &TransientResourceRef, now_ms: u64) -> bool { - session.transient.resource_matches_current(resource, now_ms) -} - struct HlsTransientReadGuard { access: Arc, } -impl HlsTransientReadGuard { - fn new(access: Arc, now_ms: u64) -> Self { - access.reader_started(now_ms); - Self { access } - } -} - -impl Drop for HlsTransientReadGuard { - fn drop(&mut self) { self.access.reader_finished(); } -} - pub struct HlsTransientOriginIoGuard { session: HlsSessionHandle, active_origin_work_count: std::sync::Arc, @@ -782,1441 +60,27 @@ pub struct HlsTransientOriginIoGuard { origin_work_finished: bool, } -impl HlsTransientOriginIoGuard { - pub fn new( - session: HlsSessionHandle, - active_origin_work_count: std::sync::Arc, - origin_io: HlsOriginIoContext, - lease_guard: HlsOriginAccountIoLeaseGuard, - started_generation: u64, - ) -> Self { - Self { - session, - active_origin_work_count, - origin_io, - lease_guard: Some(lease_guard), - started_generation, - origin_work_finished: false, - } - } - - pub async fn finish_clean(mut self) { - let generation_valid = { - let mut session = self.session.write().await; - let valid = session.finish_origin_work(self.started_generation); - self.origin_work_finished = true; - valid - }; - let refresh_reservation = if generation_valid { - self.session - .read() - .await - .should_refresh_origin_reservation(chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default()) - } else { - false - }; - if let Some(lease_guard) = self.lease_guard.take() { - crate::origin::finish_hls_origin_account_io( - &self.origin_io, - &self.session, - lease_guard, - refresh_reservation, - ) - .await; - } - } -} - -impl Drop for HlsTransientOriginIoGuard { - fn drop(&mut self) { - if self.origin_work_finished { - return; - } - self.origin_work_finished = true; - self.active_origin_work_count.fetch_sub(1, std::sync::atomic::Ordering::AcqRel); - } -} - -use tuliprox_core::utils::current_time_millis; - #[cfg(test)] -mod tests { - use super::{ - super::{ - origin_progress::{HlsOriginPathCondition, HlsOriginProgressPhase}, - resource_fetch::take_body_failure_log_attempts, - }, - fetch_and_commit_hls_transient_origin_response_with_attempt_prepare, - fetch_hls_transient_origin_response_with_attempt_prepare, hls_transient_object_fetch_failure, - hls_transient_origin_response, record_temporary_transient_segment_fetch_failure, - resolve_hls_transient_object_cache_action, HlsLogIdentity, HlsTransientCacheCommitContext, - HlsTransientDecodedOriginResponse, HlsTransientDirectResponseContext, HlsTransientDirectResponseFinalizer, - HlsTransientDirectResponseLifecycleContext, HlsTransientDirectStreamOutcome, HlsTransientObjectCacheAction, - HlsTransientObjectFetchFailure, HlsTransientOriginCacheFetchRequest, HlsTransientOriginFetchRequest, - HlsTransientOriginIoGuard, HlsTransientResourceLeaseContext, - }; - use crate::{ - origin::{HlsOriginAccountBinding, HlsOriginAccountIoLease, HlsOriginAccountIoLeaseGuard, HlsOriginIoContext}, - HlsAccessLeaseId, HlsOriginResourceClients, HlsOriginResourceFetchError, HlsPublishedTransientResourceIds, - HlsSegmentCache, HlsSegmentFailureObject, HlsSegmentRepairManager, HlsSession, HlsSessionHandle, HlsSessionKey, - HlsSessionStore, ProxySessionId, SegmentFetchPolicy, TransientObjectCacheKey, TransientObjectCacheStatus, - TransientObjectFetchDecision, TransientObjectFetchToken, TransientPassthroughState, TransientResourceFile, - TransientResourceKind, TransientResourceRef, - }; - use async_compression::tokio::write::{BrotliEncoder, GzipEncoder, ZstdEncoder}; - use axum::{ - body::to_bytes, - http::{header, HeaderMap, HeaderValue, StatusCode}, - }; - use futures::FutureExt; - use shared::model::HlsSegmentRepairMode; - use std::{ - io, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }, - time::Duration, - }; - use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpListener, - sync::{Mutex, RwLock}, - }; - use tuliprox_core::{model::HlsSegmentRepairConfig, utils::response_compression::should_compress_response}; - - struct TestOrigin { - base_url: String, - requests: Arc>>, - task: tokio::task::JoinHandle<()>, - } - - impl Drop for TestOrigin { - fn drop(&mut self) { self.task.abort(); } - } - - #[test] - fn storage_full_transient_cache_failure_is_permanent() { - let error = HlsOriginResourceFetchError::cache_commit(&io::Error::from_raw_os_error(28)); - - assert!(matches!( - hls_transient_object_fetch_failure(&error), - HlsTransientObjectFetchFailure::Permanent { status: None } - )); - } - - fn finalized_resource_resolution_fixture( - ) -> (HlsSessionHandle, ProxySessionId, HlsAccessLeaseId, TransientResourceFile, HlsPublishedTransientResourceIds) - { - let mut session = HlsSession::new(HlsSessionKey::new(1, "finalized-range"), b"rewrite-secret", 0); - let proxy_session_id = session.proxy_session_id.clone(); - let access_lease_id = HlsAccessLeaseId("lease".to_string()); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/archive/segment.ts", - b"rewrite-secret", - 0, - 300_000, - Some("ts".to_string()), - ); - let resource_file = TransientResourceFile { resource_id: resource.id.clone(), extension: "ts".to_string() }; - session.transient.upsert_resources([resource]); - let manifest_body = format!( - "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXTINF:1,\n/hls/shared/live/{}/lease/r/{}.ts\n#EXT-X-ENDLIST\n", - proxy_session_id.0, resource_file.resource_id.0 - ); - let published_resource_ids = HlsPublishedTransientResourceIds::from_manifest_body(&manifest_body); - session.transient.replace_manifest_with_semantics(manifest_body, 0, Some(1_000)); - let manifest_generation = - session.transient.current_finalized_manifest_generation().expect("finalized manifest generation"); - assert!(session.transient.bind_finalized_manifest_generation( - super::super::TransientManifestLeaseBinding::new(access_lease_id.clone(), 0, manifest_generation,) - )); - (Arc::new(RwLock::new(session)), proxy_session_id, access_lease_id, resource_file, published_resource_ids) - } - - #[tokio::test] - async fn raw_ttl_resource_requires_requesting_lease_manifest_membership_for_full_and_range() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "rolling-membership"), b"rewrite-secret", 0); - let proxy_session_id = session.proxy_session_id.clone(); - let authorized_lease_id = HlsAccessLeaseId("lease-a".to_string()); - let unpublished_lease_id = HlsAccessLeaseId("lease-b".to_string()); - let segment = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://origin.example.com/live/segment.ts", - b"rewrite-secret", - 0, - 300_000, - Some("ts".to_string()), - ); - let key = TransientResourceRef::new( - TransientResourceKind::Key, - "http://origin.example.com/live/key.bin", - b"rewrite-secret", - 0, - 300_000, - Some("key".to_string()), - ); - let resource_files = [ - TransientResourceFile { resource_id: segment.id.clone(), extension: "ts".to_string() }, - TransientResourceFile { resource_id: key.id.clone(), extension: "key".to_string() }, - ]; - let manifest_body = format!( - "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/{}/lease-a/r/{}.key\"\n\ - #EXTINF:6,\n/hls/shared/live/{}/lease-a/r/{}.ts\n", - proxy_session_id.0, key.id.0, proxy_session_id.0, segment.id.0 - ); - let authorized_resources = HlsPublishedTransientResourceIds::from_manifest_body(&manifest_body); - let unpublished_resources = HlsPublishedTransientResourceIds::default(); - session.transient.upsert_resources([segment, key]); - let session = Arc::new(RwLock::new(session)); - let ranges = [None, Some(HeaderValue::from_static("bytes=0-")), Some(HeaderValue::from_static("bytes=1000-"))]; - - for resource_file in &resource_files { - for range in &ranges { - let rejected = resolve_hls_transient_object_cache_action( - &session, - &proxy_session_id, - HlsTransientResourceLeaseContext { - access_lease_id: &unpublished_lease_id, - lease_issued_at_ms: 0, - published_resource_ids: &unpublished_resources, - }, - resource_file, - range.as_ref(), - 1, - 300_000, - ) - .await; - assert!(matches!(rejected, Err(StatusCode::NOT_FOUND))); - - let accepted = resolve_hls_transient_object_cache_action( - &session, - &proxy_session_id, - HlsTransientResourceLeaseContext { - access_lease_id: &authorized_lease_id, - lease_issued_at_ms: 0, - published_resource_ids: &authorized_resources, - }, - resource_file, - range.as_ref(), - 1, - 300_000, - ) - .await; - assert!(accepted.is_ok()); - } - } - } - - #[tokio::test] - async fn finalized_resource_validity_is_identical_for_full_and_range_requests() { - let (session, proxy_session_id, access_lease_id, resource_file, published_resource_ids) = - finalized_resource_resolution_fixture(); - let full = resolve_hls_transient_object_cache_action( - &session, - &proxy_session_id, - HlsTransientResourceLeaseContext { - access_lease_id: &access_lease_id, - lease_issued_at_ms: 0, - published_resource_ids: &published_resource_ids, - }, - &resource_file, - None, - 301_520, - 300_000, - ) - .await - .expect("full request resolves finalized mapping"); - assert!(matches!(full.action, HlsTransientObjectCacheAction::FetchAndCache(_))); - - let zero_range = HeaderValue::from_static("bytes=0-"); - let from_zero = resolve_hls_transient_object_cache_action( - &session, - &proxy_session_id, - HlsTransientResourceLeaseContext { - access_lease_id: &access_lease_id, - lease_issued_at_ms: 0, - published_resource_ids: &published_resource_ids, - }, - &resource_file, - Some(&zero_range), - 301_520, - 300_000, - ) - .await - .expect("zero range resolves finalized mapping"); - assert!(matches!(from_zero.action, HlsTransientObjectCacheAction::WaitForFetch(_))); - - let offset_range = HeaderValue::from_static("bytes=1000-"); - let from_offset = resolve_hls_transient_object_cache_action( - &session, - &proxy_session_id, - HlsTransientResourceLeaseContext { - access_lease_id: &access_lease_id, - lease_issued_at_ms: 0, - published_resource_ids: &published_resource_ids, - }, - &resource_file, - Some(&offset_range), - 301_520, - 300_000, - ) - .await - .expect("offset range resolves finalized mapping"); - assert!(matches!(from_offset.action, HlsTransientObjectCacheAction::PassthroughNoCache)); - } - - #[tokio::test] - async fn removed_finalized_mapping_is_rejected_for_full_and_range_requests() { - let (session, proxy_session_id, access_lease_id, resource_file, published_resource_ids) = - finalized_resource_resolution_fixture(); - { - let mut session = session.write().await; - session.transient.replace_manifest_with_semantics( - "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-ENDLIST\n".to_string(), - 1, - None, - ); - assert!(session.transient.release_finalized_manifest_generations(&access_lease_id, 0)); - } - let ranges = [None, Some(HeaderValue::from_static("bytes=0-")), Some(HeaderValue::from_static("bytes=1000-"))]; - - for range in &ranges { - let result = resolve_hls_transient_object_cache_action( - &session, - &proxy_session_id, - HlsTransientResourceLeaseContext { - access_lease_id: &access_lease_id, - lease_issued_at_ms: 0, - published_resource_ids: &published_resource_ids, - }, - &resource_file, - range.as_ref(), - 301_520, - 300_000, - ) - .await; - assert!(matches!(result, Err(StatusCode::NOT_FOUND))); - } - } - - async fn spawn_test_origin( - status_line: &'static str, - response_headers: Vec<(&'static str, &'static str)>, - body: Vec, - ) -> TestOrigin { - spawn_test_origin_in_chunks(status_line, response_headers, vec![body], Duration::ZERO).await - } - - async fn spawn_test_origin_in_chunks( - status_line: &'static str, - response_headers: Vec<(&'static str, &'static str)>, - body_chunks: Vec>, - inter_chunk_delay: Duration, - ) -> TestOrigin { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let addr = listener.local_addr().expect("test origin address"); - let requests = Arc::new(Mutex::new(Vec::new())); - let task_requests = Arc::clone(&requests); - let body_chunks = Arc::new(body_chunks); - let content_length = body_chunks.iter().map(Vec::len).sum::(); - let response_headers = Arc::new(response_headers); - let task = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let requests = Arc::clone(&task_requests); - let body_chunks = Arc::clone(&body_chunks); - let response_headers = Arc::clone(&response_headers); - tokio::spawn(async move { - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 2048]; - let Ok(read) = socket.read(&mut chunk).await else { - return; - }; - if read == 0 { - return; - } - request.extend_from_slice(&chunk[..read]); - } - requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); - let mut response = format!("HTTP/1.1 {status_line}\r\nContent-Length: {content_length}\r\n"); - for (name, value) in response_headers.iter() { - response.push_str(name); - response.push_str(": "); - response.push_str(value); - response.push_str("\r\n"); - } - response.push_str("Connection: close\r\n\r\n"); - let _ = socket.write_all(response.as_bytes()).await; - for (index, chunk) in body_chunks.iter().enumerate() { - if index > 0 && !inter_chunk_delay.is_zero() { - tokio::time::sleep(inter_chunk_delay).await; - } - if socket.write_all(chunk).await.is_err() { - return; - } - } - }); - } - }); - - TestOrigin { base_url: format!("http://{addr}"), requests, task } - } - - async fn spawn_retry_then_body_origin( - response_headers: Vec<(&'static str, &'static str)>, - body: Vec, - ) -> TestOrigin { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); - let addr = listener.local_addr().expect("test origin address"); - let requests = Arc::new(Mutex::new(Vec::new())); - let task_requests = Arc::clone(&requests); - let attempts = Arc::new(AtomicUsize::new(0)); - let task_attempts = Arc::clone(&attempts); - let body = Arc::new(body); - let response_headers = Arc::new(response_headers); - let task = tokio::spawn(async move { - loop { - let Ok((mut socket, _)) = listener.accept().await else { - break; - }; - let requests = Arc::clone(&task_requests); - let attempts = Arc::clone(&task_attempts); - let body = Arc::clone(&body); - let response_headers = Arc::clone(&response_headers); - tokio::spawn(async move { - let mut request = Vec::new(); - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let mut chunk = [0_u8; 2048]; - let Ok(read) = socket.read(&mut chunk).await else { - return; - }; - if read == 0 { - return; - } - request.extend_from_slice(&chunk[..read]); - } - requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); - if attempts.fetch_add(1, Ordering::Relaxed) == 0 { - let _ = socket - .write_all( - b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", - ) - .await; - return; - } - let mut response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n", body.len()); - for (name, value) in response_headers.iter() { - response.push_str(name); - response.push_str(": "); - response.push_str(value); - response.push_str("\r\n"); - } - response.push_str("Connection: close\r\n\r\n"); - let _ = socket.write_all(response.as_bytes()).await; - let _ = socket.write_all(&body).await; - }); - } - }); - - TestOrigin { base_url: format!("http://{addr}"), requests, task } - } - - async fn spawn_zstd_origin(body: Vec) -> TestOrigin { - spawn_test_origin( - "200 OK", - vec![("Content-Encoding", "zstd"), ("Content-Type", "application/octet-stream")], - body, - ) - .await - } - - async fn gzip_encode(body: &[u8]) -> Vec { - let mut encoder = GzipEncoder::new(Vec::new()); - encoder.write_all(body).await.expect("test body encodes"); - encoder.shutdown().await.expect("test encoder finishes"); - encoder.into_inner() - } - - async fn brotli_encode(body: &[u8]) -> Vec { - let mut encoder = BrotliEncoder::new(Vec::new()); - encoder.write_all(body).await.expect("test body encodes"); - encoder.shutdown().await.expect("test encoder finishes"); - encoder.into_inner() - } - - async fn zstd_encode(body: &[u8]) -> Vec { - let mut encoder = ZstdEncoder::new(Vec::new()); - encoder.write_all(body).await.expect("test body encodes"); - encoder.shutdown().await.expect("test encoder finishes"); - encoder.into_inner() - } - - async fn fetch_direct_decoded( - origin_url: String, - resource_kind: TransientResourceKind, - range_header: Option, - ) -> Result>, HlsOriginResourceFetchError> { - fetch_direct_decoded_with_timeout(origin_url, resource_kind, range_header, 1_000).await - } - - async fn fetch_direct_decoded_with_timeout( - origin_url: String, - resource_kind: TransientResourceKind, - range_header: Option, - origin_segment_timeout_ms: u64, - ) -> Result>, HlsOriginResourceFetchError> { - let resource = TransientResourceRef::new( - resource_kind, - origin_url, - b"rewrite-secret", - 10, - 60_000, - Some("bin".to_string()), - ); - let request = HlsTransientOriginFetchRequest { - resolved_origin_uri: resource.resolved_origin_uri.clone(), - origin_headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - range_header, - resource_file: TransientResourceFile { resource_id: resource.id, extension: "bin".to_string() }, - resource_kind, - clients: HlsOriginResourceClients { - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("no-redirect client builds"), - use_manual_redirects: false, - }, - policy: SegmentFetchPolicy { - origin_segment_timeout_ms, - retry_delays_ms: [0; 5], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }, - log_identity: HlsLogIdentity::for_test( - "direct-transient-content-session", - "direct-transient-proxy-session", - ), - }; - fetch_hls_transient_origin_response_with_attempt_prepare(request, |_| async { Ok(None) }.boxed()).await - } - - fn direct_client_response( - response: HlsTransientDecodedOriginResponse>, - resource_kind: TransientResourceKind, - ) -> axum::response::Response { - TestDirectResponseFixture::new(resource_kind, "http://127.0.0.1/origin").response(response) - } - - struct TestDirectResponseFixture { - session: HlsSessionHandle, - resource: TransientResourceRef, - policy: SegmentFetchPolicy, - log_identity: HlsLogIdentity, - } - - impl TestDirectResponseFixture { - fn new(resource_kind: TransientResourceKind, origin_url: impl Into) -> Self { - let resource = TransientResourceRef::new( - resource_kind, - origin_url, - b"rewrite-secret", - super::current_time_millis(), - 60_000, - Some("bin".to_string()), - ); - let policy = - SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; - let mut session = HlsSession::new(HlsSessionKey::new(1, "direct-test"), b"rewrite-secret", 10); - session.transient.upsert_resources([resource.clone()]); - let log_identity = HlsLogIdentity::from_session(&session); - Self { session: Arc::new(RwLock::new(session)), resource, policy, log_identity } - } - - fn response( - &self, - response: HlsTransientDecodedOriginResponse>, - ) -> axum::response::Response { - hls_transient_origin_response(response, self.response_context()) - } - - fn response_context(&self) -> HlsTransientDirectResponseContext { - HlsTransientDirectResponseContext { - session: Arc::clone(&self.session), - resource: self.resource.clone(), - policy: self.policy.clone(), - now_ms: 10, - log_identity: self.log_identity.clone(), - } - } - - fn lifecycle_context(&self) -> HlsTransientDirectResponseLifecycleContext { - HlsTransientDirectResponseLifecycleContext { - session: Arc::clone(&self.session), - resource: self.resource.clone(), - policy: self.policy.clone(), - } - } - - async fn seed_segment_failure(&self) { - assert!( - !record_temporary_transient_segment_fetch_failure(&self.session, &self.resource, &self.policy, 9).await - ); - } - - async fn segment_failure_count(&self) -> u32 { - self.session.read().await.segment_failure_tracker.consecutive_temporary_failures - } - - async fn wait_for_segment_failure_count(&self, expected: u32) { - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if self.segment_failure_count().await == expected { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("segment failure state reaches expected value"); - } - - async fn last_segment_failure(&self) -> Option { - self.session.read().await.segment_failure_tracker.last_failed_object.clone() - } - } - - struct TestTransientCacheFixture { - temp_dir: tempfile::TempDir, - segment_cache: Arc, - segment_repair: Arc, - session: HlsSessionHandle, - log_identity: HlsLogIdentity, - resource: TransientResourceRef, - resource_file: TransientResourceFile, - lookup_key: TransientObjectCacheKey, - cache_key: TransientObjectCacheKey, - fetch_token: TransientObjectFetchToken, - } - - impl TestTransientCacheFixture { - async fn new(origin_url: String) -> Self { - let now_ms = super::current_time_millis(); - let resource = TransientResourceRef::new( - TransientResourceKind::Segment, - origin_url, - b"rewrite-secret", - now_ms, - 60_000, - Some("ts".to_string()), - ); - let resource_file = TransientResourceFile { resource_id: resource.id.clone(), extension: "ts".to_string() }; - let store = HlsSessionStore::new(); - let session = store.get_or_create_session(HlsSessionKey::new(1, "1"), b"rewrite-secret", 10).await; - let (proxy_session_id, log_identity) = { - let session = session.read().await; - (session.proxy_session_id.clone(), HlsLogIdentity::from_session(&session)) - }; - let lookup_key = TransientPassthroughState::transient_object_key( - &proxy_session_id, - &resource.id, - resource_file.extension.clone(), - ); - let (resource, fetch_token) = { - let mut session = session.write().await; - session.transient.upsert_resources([resource]); - let resource = session - .transient - .resources - .get(&resource_file.resource_id) - .expect("registered transient resource") - .clone(); - match session.transient.begin_object_fetch(&proxy_session_id, &resource, "ts", now_ms, 60_000) { - TransientObjectFetchDecision::Fetch(fetch_token) => (resource, *fetch_token), - TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { - panic!("new transient resource should start a cache fetch") - } - } - }; - let cache_key = fetch_token.cache_key().clone(); - let temp_dir = tempfile::tempdir().expect("tempdir"); - let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); - let segment_repair = Arc::new(HlsSegmentRepairManager::new(HlsSegmentRepairConfig { - max_level: HlsSegmentRepairMode::Off, - apply_to_first_segments: 1, - max_parallel_repairs: 1, - ..Default::default() - })); - Self { - temp_dir, - segment_cache, - segment_repair, - session, - log_identity, - resource, - resource_file, - lookup_key, - cache_key, - fetch_token, - } - } - - fn request(&self, range_header: Option) -> HlsTransientOriginCacheFetchRequest { - HlsTransientOriginCacheFetchRequest { - fetch: HlsTransientOriginFetchRequest { - resolved_origin_uri: self.resource.resolved_origin_uri.clone(), - origin_headers: HeaderMap::new(), - origin_provider_session_headers: HeaderMap::new(), - range_header, - resource_file: self.resource_file.clone(), - resource_kind: self.resource.kind, - clients: HlsOriginResourceClients { - client: reqwest::Client::new(), - no_redirect_client: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("no-redirect client builds"), - use_manual_redirects: false, - }, - policy: SegmentFetchPolicy { - origin_segment_timeout_ms: 1_000, - retry_delays_ms: [0; 5], - retry_jitter_max_ms: 0, - ..SegmentFetchPolicy::default() - }, - log_identity: self.log_identity.clone(), - }, - commit: HlsTransientCacheCommitContext { - segment_cache: Arc::clone(&self.segment_cache), - segment_repair: Arc::clone(&self.segment_repair), - session: Arc::clone(&self.session), - proxy_session_id: ProxySessionId(String::from("test-proxy-session")), - log_identity: self.log_identity.clone(), - access_lease_id: HlsAccessLeaseId("transient-content-coding-test".to_string()), - resource: self.resource.clone(), - resource_file: self.resource_file.clone(), - fetch_token: self.fetch_token.clone(), - cache_duration_ms: 60_000, - }, - } - } - } - - struct DropCounter(Arc); - - impl Drop for DropCounter { - fn drop(&mut self) { self.0.fetch_add(1, Ordering::Relaxed); } - } - - #[tokio::test] - async fn direct_key_decodes_declared_gzip_brotli_and_zstd_to_exact_identity_bytes() { - let identity = b"\x00\xffdirect-key-identity\x10\x80"; - let encoded_bodies = [ - ("gzip", gzip_encode(identity).await), - ("br", brotli_encode(identity).await), - ("zstd", zstd_encode(identity).await), - ]; - - for (encoding, encoded) in encoded_bodies { - let origin = spawn_test_origin( - "200 OK", - vec![("Content-Encoding", encoding), ("Content-Type", "application/octet-stream")], - encoded, - ) - .await; - let decoded = - fetch_direct_decoded(format!("{}/key.bin", origin.base_url), TransientResourceKind::Key, None) - .await - .expect("declared direct key coding decodes"); - let response = direct_client_response(decoded, TransientResourceKind::Key); - - assert!(!should_compress_response(&response)); - assert!(response.headers().get(header::CONTENT_ENCODING).is_none()); - assert!(response.headers().get(header::CONTENT_LENGTH).is_none()); - assert_eq!( - to_bytes(response.into_body(), usize::MAX).await.expect("decoded key body streams"), - identity.as_slice() - ); - - let requests = origin.requests.lock().await; - assert_eq!(requests.len(), 1); - assert!(requests[0].to_ascii_lowercase().contains("accept-encoding: identity")); - } - } - - #[tokio::test] - async fn direct_map_and_part_or_other_resources_decode_to_identity() { - let identity = b"shared-direct-map-part-other"; - for (resource_kind, encoding, encoded) in [ - (TransientResourceKind::Map, "gzip", gzip_encode(identity).await), - (TransientResourceKind::Part, "br", brotli_encode(identity).await), - (TransientResourceKind::Other, "zstd", zstd_encode(identity).await), - ] { - let origin = spawn_test_origin( - "200 OK", - vec![("Content-Encoding", encoding), ("Content-Type", "application/octet-stream")], - encoded, - ) - .await; - let decoded = fetch_direct_decoded(format!("{}/object.bin", origin.base_url), resource_kind, None) - .await - .expect("declared direct resource coding decodes"); - let response = direct_client_response(decoded, resource_kind); - - assert_eq!( - to_bytes(response.into_body(), usize::MAX).await.expect("decoded resource body streams"), - identity.as_slice() - ); - assert_eq!(origin.requests.lock().await.len(), 1); - } - } - - #[tokio::test] - async fn direct_binary_declared_only_leaves_headerless_gzip_magic_unchanged() { - let identity = b"headerless-gzip-representation"; - let headerless_encoded = gzip_encode(identity).await; - let random_magic = vec![0x1f, 0x8b, 0x11, 0x00, 0xff, 0x42, 0x7e]; - - for body in [headerless_encoded, random_magic] { - let origin = - spawn_test_origin("200 OK", vec![("Content-Type", "application/octet-stream")], body.clone()).await; - let decoded = - fetch_direct_decoded(format!("{}/key.bin", origin.base_url), TransientResourceKind::Key, None) - .await - .expect("headerless binary is not inspected"); - let response = direct_client_response(decoded, TransientResourceKind::Key); - - assert_eq!(to_bytes(response.into_body(), usize::MAX).await.expect("headerless binary streams"), body); - assert_eq!(origin.requests.lock().await.len(), 1); - } - } - - #[tokio::test] - async fn decoded_direct_response_normalizes_headers_and_disables_tower_compression() { - let identity = b"normalized-direct-response"; - let encoded = gzip_encode(identity).await; - let origin = spawn_test_origin( - "200 OK", - vec![ - ("Content-Encoding", "gzip"), - ("Content-Type", "application/octet-stream"), - ("Content-Range", "bytes 0-9/10"), - ("Accept-Ranges", "bytes"), - ("Cache-Control", "private, max-age=5"), - ("ETag", "strong-origin-representation"), - ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"), - ("Set-Cookie", "provider_session=secret"), - ("X-Provider-Secret", "do-not-forward"), - ], - encoded, - ) - .await; - let decoded = fetch_direct_decoded(format!("{}/key.bin", origin.base_url), TransientResourceKind::Key, None) - .await - .expect("direct response decodes"); - let response = direct_client_response(decoded, TransientResourceKind::Key); - let headers = response.headers(); - - assert!(!should_compress_response(&response)); - for removed in [ - header::CONTENT_ENCODING, - header::CONTENT_LENGTH, - header::CONTENT_RANGE, - header::ACCEPT_RANGES, - header::ETAG, - header::TRANSFER_ENCODING, - header::SET_COOKIE, - ] { - assert!(headers.get(&removed).is_none(), "{removed} must not describe the decoded response"); - } - assert!(headers.get("x-provider-secret").is_none()); - assert_eq!(headers.get(header::CONTENT_TYPE).expect("content type"), "application/octet-stream"); - assert_eq!(headers.get(header::CACHE_CONTROL).expect("cache control"), "private, max-age=5"); - assert_eq!( - to_bytes(response.into_body(), usize::MAX).await.expect("normalized body streams"), - identity.as_slice() - ); - } - - #[tokio::test] - async fn direct_encoded_partial_content_is_rejected_before_client_response() { - let encoded = zstd_encode(b"encoded-range").await; - let origin = spawn_test_origin( - "206 Partial Content", - vec![ - ("Content-Encoding", "zstd"), - ("Content-Type", "application/octet-stream"), - ("Content-Range", "bytes 2-5/16"), - ], - encoded, - ) - .await; - - let result = fetch_direct_decoded( - format!("{}/key.bin", origin.base_url), - TransientResourceKind::Key, - Some(HeaderValue::from_static("bytes=2-5")), - ) - .await; - - assert!(matches!(result, Err(HlsOriginResourceFetchError::ContentCoding(_)))); - let requests = origin.requests.lock().await; - assert_eq!(requests.len(), 1); - let request = requests[0].to_ascii_lowercase(); - assert!(request.contains("accept-encoding: identity")); - assert!(request.contains("range: bytes=2-5")); - } - - #[tokio::test] - async fn direct_identity_partial_content_preserves_consistent_range_headers() { - let identity = b"part"; - let origin = spawn_test_origin( - "206 Partial Content", - vec![ - ("Content-Type", "application/octet-stream"), - ("Content-Range", "bytes 2-5/16"), - ("Accept-Ranges", "bytes"), - ], - identity.to_vec(), - ) - .await; - let decoded = fetch_direct_decoded( - format!("{}/key.bin", origin.base_url), - TransientResourceKind::Key, - Some(HeaderValue::from_static("bytes=2-5")), - ) - .await - .expect("identity partial response is allowed"); - let response = direct_client_response(decoded, TransientResourceKind::Key); - - assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - assert_eq!(response.headers().get(header::CONTENT_LENGTH).expect("content length"), "4"); - assert_eq!(response.headers().get(header::CONTENT_RANGE).expect("content range"), "bytes 2-5/16"); - assert_eq!(response.headers().get(header::ACCEPT_RANGES).expect("accept ranges"), "bytes"); - assert!(!should_compress_response(&response)); - assert_eq!( - to_bytes(response.into_body(), usize::MAX).await.expect("identity partial body streams"), - identity.as_slice() - ); - assert_eq!(origin.requests.lock().await.len(), 1); - } - - #[tokio::test] - async fn direct_fetch_propagates_retry_attempt_and_identity_to_client_response() { - let identity = b"successful-second-attempt"; - let origin = - spawn_retry_then_body_origin(vec![("Content-Type", "application/octet-stream")], identity.to_vec()).await; - let origin_url = format!("{}/segment.ts", origin.base_url); - let decoded = fetch_direct_decoded(origin_url.clone(), TransientResourceKind::Segment, None) - .await - .expect("second origin attempt succeeds"); - - assert_eq!(decoded.attempt.attempt_index, 1); - assert_eq!(decoded.attempt.attempts, 5); - - let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url); - let response = fixture.response(decoded); - assert_eq!( - to_bytes(response.into_body(), usize::MAX).await.expect("retried response body streams"), - identity.as_slice() - ); - - let requests = origin.requests.lock().await; - assert_eq!(requests.len(), 2); - assert!(requests.iter().all(|request| request.to_ascii_lowercase().contains("accept-encoding: identity"))); - } - - #[tokio::test] - async fn direct_segment_success_resets_failure_state_only_after_clean_eof() { - let identity = b"complete-segment"; - let origin = - spawn_test_origin("200 OK", vec![("Content-Type", "application/octet-stream")], identity.to_vec()).await; - let origin_url = format!("{}/segment.ts", origin.base_url); - let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url.clone()); - fixture.seed_segment_failure().await; - let decoded = fetch_direct_decoded(origin_url, TransientResourceKind::Segment, None) - .await - .expect("direct response setup succeeds"); - let response = fixture.response(decoded); - - assert_eq!(fixture.segment_failure_count().await, 1, "headers alone must not report body success"); - assert_eq!(fixture.resource.access.active_readers(), 1); - assert_eq!( - to_bytes(response.into_body(), usize::MAX).await.expect("complete segment streams"), - identity.as_slice() - ); - fixture.wait_for_segment_failure_count(0).await; - assert_eq!(fixture.resource.access.active_readers(), 0); - } - - #[tokio::test] - async fn direct_terminal_transition_survives_dropped_completion_waiter() { - let fixture = - TestDirectResponseFixture::new(TransientResourceKind::Segment, "http://127.0.0.1/cancelled-body-poll.ts"); - fixture.seed_segment_failure().await; - let session_lock = fixture.session.write().await; - let mut finalizer = HlsTransientDirectResponseFinalizer::new(fixture.lifecycle_context()); - - let transition = finalizer - .begin_finish(HlsTransientDirectStreamOutcome::CleanEof) - .expect("clean EOF starts an owned transition"); - drop(finalizer); - drop(transition); - drop(session_lock); - - fixture.wait_for_segment_failure_count(0).await; - } - - #[tokio::test] - async fn direct_segment_decoder_failure_after_retry_uses_selected_attempt_once() { - let _ = take_body_failure_log_attempts(); - let mut truncated = gzip_encode(b"decoder failure after response headers").await; - truncated.truncate(truncated.len().saturating_sub(8)); - let origin = spawn_retry_then_body_origin( - vec![("Content-Encoding", "gzip"), ("Content-Type", "application/octet-stream")], - truncated, - ) - .await; - let origin_url = format!("{}/segment.ts", origin.base_url); - let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url.clone()); - let decoded = fetch_direct_decoded(origin_url, TransientResourceKind::Segment, None) - .await - .expect("decoder setup succeeds before streaming"); - assert_eq!(decoded.attempt.attempt_index, 1); - assert_eq!(decoded.attempt.attempts, 5); - let response = fixture.response(decoded); - - assert!(to_bytes(response.into_body(), usize::MAX).await.is_err()); - fixture.wait_for_segment_failure_count(1).await; - assert_eq!( - fixture.last_segment_failure().await, - Some(HlsSegmentFailureObject::Transient { resource_id: fixture.resource.id.0.clone() }) - ); - tokio::task::yield_now().await; - assert_eq!(fixture.segment_failure_count().await, 1, "terminal body failure must not finalize twice"); - assert_eq!(fixture.resource.access.active_readers(), 0); - assert_eq!(origin.requests.lock().await.len(), 2, "streaming failure must not trigger a hidden retry"); - assert_eq!( - take_body_failure_log_attempts(), - vec![(1, 5)], - "test hook intentionally records the zero-based raw index of the selected second attempt" - ); - } - - #[tokio::test] - async fn unrelated_transient_objects_do_not_start_media_lifecycle_tasks() { - let kind = TransientResourceKind::Other; - for outcome in [HlsTransientDirectStreamOutcome::CleanEof, HlsTransientDirectStreamOutcome::OriginBodyFailure] { - let fixture = TestDirectResponseFixture::new(kind, "http://127.0.0.1/non-segment.bin"); - let mut finalizer = HlsTransientDirectResponseFinalizer::new(fixture.lifecycle_context()); - - assert!(finalizer.begin_finish(outcome).is_none(), "{kind:?} must not start a segment lifecycle task"); - } - } - - #[tokio::test] - async fn key_and_map_body_failures_degrade_readiness_without_incrementing_media_failure_count() { - for kind in [TransientResourceKind::Key, TransientResourceKind::Map] { - let fixture = TestDirectResponseFixture::new(kind, "http://127.0.0.1/media-dependency.bin"); - let mut finalizer = HlsTransientDirectResponseFinalizer::new(fixture.lifecycle_context()); - - finalizer - .begin_finish(HlsTransientDirectStreamOutcome::OriginBodyFailure) - .expect("media dependency starts a lifecycle task") - .await - .expect("media dependency lifecycle task joins"); - - let session = fixture.session.read().await; - assert_eq!(session.segment_failure_tracker.consecutive_temporary_failures, 0); - assert_eq!(session.origin_control.path_condition, HlsOriginPathCondition::SegmentReadinessFailure); - } - } - - #[tokio::test] - async fn direct_segment_origin_idle_timeout_is_counted_exactly_once() { - let origin = spawn_test_origin_in_chunks( - "200 OK", - vec![("Content-Type", "application/octet-stream")], - vec![b"first".to_vec(), b"late".to_vec()], - Duration::from_millis(200), - ) - .await; - let origin_url = format!("{}/segment.ts", origin.base_url); - let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url.clone()); - let decoded = fetch_direct_decoded_with_timeout(origin_url, TransientResourceKind::Segment, None, 50) - .await - .expect("direct response setup succeeds"); - let response = fixture.response(decoded); - - assert!(to_bytes(response.into_body(), usize::MAX).await.is_err()); - fixture.wait_for_segment_failure_count(1).await; - tokio::task::yield_now().await; - assert_eq!(fixture.segment_failure_count().await, 1); - assert_eq!(fixture.resource.access.active_readers(), 0); - } - - #[tokio::test] - async fn dropping_direct_segment_body_is_client_abort_without_state_transition() { - let _ = take_body_failure_log_attempts(); - let origin = spawn_test_origin( - "200 OK", - vec![("Content-Type", "application/octet-stream")], - b"unconsumed-segment".to_vec(), - ) - .await; - let origin_url = format!("{}/segment.ts", origin.base_url); - let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url.clone()); - fixture.seed_segment_failure().await; - let decoded = fetch_direct_decoded(origin_url, TransientResourceKind::Segment, None) - .await - .expect("direct response setup succeeds"); - let response = fixture.response(decoded); - assert_eq!(fixture.resource.access.active_readers(), 1); - - drop(response); - - assert_eq!(fixture.segment_failure_count().await, 1); - assert_eq!(fixture.resource.access.active_readers(), 0); - assert_eq!(origin.requests.lock().await.len(), 1); - assert!(take_body_failure_log_attempts().is_empty()); - } - - #[tokio::test] - async fn direct_non_media_body_failures_are_failure_tracker_neutral() { - for kind in [TransientResourceKind::Key, TransientResourceKind::Map, TransientResourceKind::Other] { - let mut truncated = gzip_encode(b"non-segment decoder failure").await; - truncated.truncate(truncated.len().saturating_sub(8)); - let origin = spawn_test_origin( - "200 OK", - vec![("Content-Encoding", "gzip"), ("Content-Type", "application/octet-stream")], - truncated, - ) - .await; - let origin_url = format!("{}/object.bin", origin.base_url); - let fixture = TestDirectResponseFixture::new(kind, origin_url.clone()); - let decoded = - fetch_direct_decoded(origin_url, kind, None).await.expect("decoder setup succeeds before streaming"); - let response = fixture.response(decoded); - - assert!(to_bytes(response.into_body(), usize::MAX).await.is_err()); - assert_eq!(fixture.segment_failure_count().await, 0, "{kind:?} must not affect segment failure state"); - let path_condition = fixture.session.read().await.origin_control.path_condition; - if matches!(kind, TransientResourceKind::Key | TransientResourceKind::Map) { - assert_eq!(path_condition, HlsOriginPathCondition::SegmentReadinessFailure); - } else { - assert_eq!(path_condition, HlsOriginPathCondition::ProgressExpected); - } - } - } - - #[tokio::test] - async fn transient_media_failure_degrades_availability_without_direct_terminal_transition() { - for kind in [TransientResourceKind::Segment, TransientResourceKind::Part] { - let fixture = TestDirectResponseFixture::new(kind, "http://127.0.0.1/media.bin"); - - assert!( - !record_temporary_transient_segment_fetch_failure( - &fixture.session, - &fixture.resource, - &fixture.policy, - 100, - ) - .await - ); - - let session = fixture.session.read().await; - assert_eq!(session.segment_failure_tracker.consecutive_temporary_failures, 1); - assert_eq!(session.origin_control.path_condition, HlsOriginPathCondition::SegmentReadinessFailure); - assert!(session.origin_control.acceptance_episode.is_none()); - assert!(!matches!( - session.origin_control.progress_phase, - HlsOriginProgressPhase::Terminal | HlsOriginProgressPhase::TerminalPartial - )); - drop(session); - - super::record_successful_transient_segment_fetch(&fixture.session, &fixture.resource).await; - let session = fixture.session.read().await; - assert_eq!(session.segment_failure_tracker.consecutive_temporary_failures, 0); - assert_eq!(session.origin_control.path_condition, HlsOriginPathCondition::SegmentReadinessFailure); - } - } - - #[tokio::test] - async fn direct_stream_decoder_failure_aborts_body_without_origin_retry() { - let mut truncated = gzip_encode(b"decoder failure after response headers").await; - truncated.truncate(truncated.len().saturating_sub(8)); - let origin = spawn_test_origin( - "200 OK", - vec![("Content-Encoding", "gzip"), ("Content-Type", "application/octet-stream")], - truncated, - ) - .await; - let decoded = fetch_direct_decoded(format!("{}/key.bin", origin.base_url), TransientResourceKind::Key, None) - .await - .expect("decoder setup succeeds before streaming"); - let response = direct_client_response(decoded, TransientResourceKind::Key); - - assert!(to_bytes(response.into_body(), usize::MAX).await.is_err()); - assert_eq!(origin.requests.lock().await.len(), 1, "streaming errors must not trigger a hidden retry"); - } - - #[tokio::test] - async fn direct_stream_refreshes_origin_body_idle_deadline_after_each_chunk() { - const BODY_IDLE_TIMEOUT_MS: u64 = 500; - let chunks = [b"slow-".to_vec(), b"but-".to_vec(), b"continuous-".to_vec(), b"body".to_vec()]; - let expected = chunks.concat(); - let origin = spawn_test_origin_in_chunks( - "200 OK", - vec![("Content-Type", "application/octet-stream")], - chunks.into(), - Duration::from_millis(200), - ) - .await; - let decoded = fetch_direct_decoded_with_timeout( - format!("{}/key.bin", origin.base_url), - TransientResourceKind::Key, - None, - BODY_IDLE_TIMEOUT_MS, - ) - .await - .expect("direct response setup succeeds"); - let response = direct_client_response(decoded, TransientResourceKind::Key); - - assert_eq!( - to_bytes(response.into_body(), usize::MAX).await.expect("progressing body outlives total timeout"), - expected - ); - assert_eq!(origin.requests.lock().await.len(), 1); - } - - #[tokio::test] - async fn cacheable_transient_rejects_identity_partial_response_before_commit_and_cleans_guard() { - let origin = spawn_test_origin( - "206 Partial Content", - vec![("Content-Type", "video/mp2t"), ("Content-Range", "bytes 0-3/10")], - b"part".to_vec(), - ) - .await; - let fixture = TestTransientCacheFixture::new(format!("{}/partial.ts", origin.base_url)).await; - let dropped_guards = Arc::new(AtomicUsize::new(0)); - let dropped_guards_for_prepare = Arc::clone(&dropped_guards); - - let result = - fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(fixture.request(None), move |_| { - let dropped_guards = Arc::clone(&dropped_guards_for_prepare); - async move { Ok(DropCounter(dropped_guards)) }.boxed() - }) - .await; - - assert!(matches!(result, Err(HlsOriginResourceFetchError::UnexpectedByteRangeStatus))); - assert_eq!(dropped_guards.load(Ordering::Relaxed), 1); - assert!(fixture.segment_cache.metadata(&fixture.cache_key).await.expect("cache metadata reads").is_none()); - assert!(!fixture.segment_cache.has_active_temp_files()); - assert_eq!(std::fs::read_dir(fixture.temp_dir.path()).expect("cache root reads").count(), 0); - let session = fixture.session.read().await; - let entry = session.transient.object_cache.get(&fixture.lookup_key).expect("fetching cache entry remains"); - assert!(!matches!(entry.status, TransientObjectCacheStatus::Ready { .. })); - drop(session); - - let requests = origin.requests.lock().await; - assert_eq!(requests.len(), 1); - let request = requests[0].to_ascii_lowercase(); - assert!(request.contains("accept-encoding: identity")); - assert!(!request.contains("\r\nrange:")); - } - - #[tokio::test] - async fn stale_resource_revision_commit_deletes_the_physical_fill_after_controlled_mapping_replacement() { - let origin = spawn_test_origin("200 OK", vec![("Content-Type", "video/mp2t")], b"stale-body".to_vec()).await; - let fixture = TestTransientCacheFixture::new(format!("{}/stale.ts", origin.base_url)).await; - { - let mut session = fixture.session.write().await; - let replacement_now_ms = super::current_time_millis(); - let mut replacement = TransientResourceRef::new( - TransientResourceKind::Segment, - "http://replacement.example.com/live/replacement.ts", - b"rewrite-secret", - replacement_now_ms, - 60_000, - Some("ts".to_string()), - ); - replacement.id = fixture.resource.id.clone(); - session.transient.upsert_resources([replacement]); - } - - let result = fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(fixture.request(None), |_| { - async { Ok(()) }.boxed() - }) - .await; - - assert!( - matches!(result, Err(HlsOriginResourceFetchError::Superseded)), - "the superseded resource revision aborts without a retryable timeout" - ); - assert!(fixture.segment_cache.metadata(&fixture.cache_key).await.expect("cache metadata reads").is_none()); - assert!(!fixture.segment_cache.has_active_temp_files()); - assert!(!fixture.session.read().await.transient.object_cache.contains_key(&fixture.lookup_key)); - assert_eq!(origin.requests.lock().await.len(), 1); - } - - #[tokio::test] - async fn cacheable_transient_without_range_decodes_to_identity_before_cache_and_range_reads() { - let ciphertext = - vec![0x00, 0xff, 0x92, 0x10, 0x7e, 0x33, 0xc4, 0x81, 0xa8, 0x09, 0x5d, 0xe2, 0x44, 0x17, 0xb0, 0x6f]; - assert!(std::str::from_utf8(&ciphertext).is_err()); - let encoded = zstd_encode(&ciphertext).await; - assert_ne!(encoded, ciphertext); - let origin = spawn_zstd_origin(encoded).await; - let fixture = TestTransientCacheFixture::new(format!("{}/cipher.ts", origin.base_url)).await; - - fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(fixture.request(None), |_| { - async { Ok(()) }.boxed() - }) - .await - .expect("encoded transient cache fetch succeeds"); - - let metadata = fixture - .segment_cache - .metadata(&fixture.cache_key) - .await - .expect("cache metadata reads") - .expect("decoded object is cached"); - assert_eq!(metadata.size, ciphertext.len() as u64); - assert_eq!(tokio::fs::read(&metadata.path).await.expect("cache body reads"), ciphertext); - let mut range = Vec::new(); - fixture - .segment_cache - .open_range(&fixture.cache_key, 5) - .await - .expect("decoded cache range opens") - .take(4) - .read_to_end(&mut range) - .await - .expect("decoded cache range reads"); - assert_eq!(range, ciphertext[5..9]); - - let session = fixture.session.read().await; - let entry = session.transient.object_cache.get(&fixture.lookup_key).expect("transient cache entry"); - assert!(matches!( - entry.status, - TransientObjectCacheStatus::Ready { content_length, .. } - if content_length == ciphertext.len() as u64 - )); - assert_eq!(entry.content_type, "application/octet-stream"); - drop(session); - - let requests = origin.requests.lock().await; - assert_eq!(requests.len(), 1); - let request = requests[0].to_ascii_lowercase(); - assert!(request.contains("accept-encoding: identity")); - assert!(!request.contains("\r\nrange:")); - } - - #[tokio::test] - async fn transient_origin_io_guard_finish_clean_decrements_work_once() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "clean-finish"), b"secret", 1_000); - let binding = - HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); - session.origin_account_io_lease = - Some(std::sync::Arc::new(HlsOriginAccountIoLease::active_for_test(&binding, 2))); - session.origin_account_binding = Some(binding.clone()); - session.activity.active_origin_work_count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(2)); - let started_gen = session.activity.origin_work_generation; - let session = Arc::new(tokio::sync::RwLock::new(session)); - let ctx = crate::hls_ctx::HlsCtx::for_test(tuliprox_core::model::Config::default()); - - let lease_guard = HlsOriginAccountIoLeaseGuard::new( - binding.clone(), - Arc::clone(session.try_read().unwrap().origin_account_io_lease.as_ref().unwrap()), - ); - let origin_io = HlsOriginIoContext { - ctx: ctx.clone(), - client_addr: "127.0.0.1:8080".parse().unwrap(), - allow_grace: false, - priority: 0, - connection_kind: tuliprox_session::ConnectionKind::Normal, - reservation_ttl_secs: 60, - preacquired_provider_handle: None, - started_generation: Some(started_gen), - }; - - let guard = HlsTransientOriginIoGuard::new( - Arc::clone(&session), - Arc::clone(&session.try_read().unwrap().activity.active_origin_work_count), - origin_io, - lease_guard, - started_gen, - ); - - guard.finish_clean().await; - assert_eq!( - session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), - 1, - "finish_clean and subsequent drop must decrement active_origin_work_count exactly once" - ); - } - - #[tokio::test] - async fn transient_origin_io_guard_drop_under_lock_contention_decrements_work() { - let mut session = HlsSession::new(HlsSessionKey::new(1, "contention-drop"), b"secret", 1_000); - let binding = - HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); - session.origin_account_io_lease = - Some(std::sync::Arc::new(HlsOriginAccountIoLease::active_for_test(&binding, 2))); - session.origin_account_binding = Some(binding.clone()); - session.activity.active_origin_work_count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(2)); - let started_gen = session.activity.origin_work_generation; - let session = Arc::new(tokio::sync::RwLock::new(session)); - let ctx = crate::hls_ctx::HlsCtx::for_test(tuliprox_core::model::Config::default()); - - let lease_guard = HlsOriginAccountIoLeaseGuard::new( - binding.clone(), - Arc::clone(session.try_read().unwrap().origin_account_io_lease.as_ref().unwrap()), - ); - let origin_io = HlsOriginIoContext { - ctx: ctx.clone(), - client_addr: "127.0.0.1:8080".parse().unwrap(), - allow_grace: false, - priority: 0, - connection_kind: tuliprox_session::ConnectionKind::Normal, - reservation_ttl_secs: 60, - preacquired_provider_handle: None, - started_generation: Some(started_gen), - }; - - let guard = HlsTransientOriginIoGuard::new( - Arc::clone(&session), - Arc::clone(&session.try_read().unwrap().activity.active_origin_work_count), - origin_io, - lease_guard, - started_gen, - ); - - let lock = session.read().await; - drop(guard); - assert_eq!( - lock.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), - 1, - "drop MUST synchronously decrement active_origin_work_count" - ); - drop(lock); - - for _ in 0..100 { - if session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) == 1 { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(5)).await; - } - assert_eq!( - session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), - 1, - "deferred cleanup must decrement active_origin_work_count under contention" - ); - } -} +mod tests; + +mod cache; +mod lifecycle; +mod origin; +mod response; +pub(super) use cache::delete_superseded_transient_object; +#[allow(unused_imports, reason = "Preserves the existing module interface.")] +pub use cache::{ + fetch_and_commit_hls_transient_origin_response_with_attempt_prepare, + is_hls_transient_full_object_cacheable_request, resolve_hls_transient_object_cache_action, + HlsTransientCacheCommitContext, HlsTransientObjectCacheAction, HlsTransientObjectCacheResolution, + HlsTransientOriginCacheFetchRequest, HlsTransientResourceLeaseContext, +}; +pub use lifecycle::{record_successful_transient_segment_fetch, record_temporary_transient_segment_fetch_failure}; +pub use origin::{ + fetch_hls_transient_origin_response_with_attempt_prepare, hls_transient_object_fetch_failure, + hls_transient_resource_fetch_kind, HlsTransientDecodedOriginResponse, HlsTransientObjectFetchFailure, + HlsTransientOriginFetchRequest, +}; +#[cfg(test)] +use response::HlsTransientDirectStreamOutcome; +pub use response::{hls_transient_origin_response, HlsTransientDirectResponseContext}; diff --git a/backend/hls/src/transient_fetcher/cache.rs b/backend/hls/src/transient_fetcher/cache.rs new file mode 100644 index 000000000..d10c91d13 --- /dev/null +++ b/backend/hls/src/transient_fetcher/cache.rs @@ -0,0 +1,319 @@ +use super::{ + origin::{build_hls_transient_resource_fetch_target, HlsTransientOriginFetchMode}, + run_hls_origin_resource_retry_loop_with_attempt_prepare, transient_object_expires_at, HlsAccessLeaseId, + HlsOriginResourceBodyDeadline, HlsOriginResourceFetchError, HlsPublishedTransientResourceIds, + HlsRepairRenderedObjectId, HlsResourceFetchAttempt, HlsSegmentCache, HlsSegmentRepairManager, + HlsSegmentRepairObjectContext, HlsSegmentRepairSource, HlsSessionHandle, HlsSessionMode, + HlsTransientObjectCacheActionInput, HlsTransientObjectFetchFinalizer, HlsTransientOriginFetchRequest, + ProxySessionId, TransientObjectCacheKey, TransientObjectFetchDecision, TransientObjectFetchToken, + TransientPassthroughState, TransientResourceFile, TransientResourceKind, TransientResourceRef, +}; +use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; +use futures::{future::BoxFuture, FutureExt}; +use log::debug; +use std::sync::Arc; +use tokio::sync::Notify; +use tuliprox_core::utils::{content_coding::DecodedHttpResponse, current_time_millis}; + +pub enum HlsTransientObjectCacheAction { + ServeReady, + FetchAndCache(Box), + WaitForFetch(Arc), + PassthroughNoCache, +} + +pub struct HlsTransientObjectCacheResolution { + pub resource: TransientResourceRef, + pub origin_headers: HeaderMap, + pub origin_provider_session_headers: HeaderMap, + pub action: HlsTransientObjectCacheAction, +} + +#[derive(Clone, Copy)] +pub struct HlsTransientResourceLeaseContext<'a> { + pub access_lease_id: &'a HlsAccessLeaseId, + pub lease_issued_at_ms: u64, + pub published_resource_ids: &'a HlsPublishedTransientResourceIds, +} + +pub async fn resolve_hls_transient_object_cache_action( + session: &HlsSessionHandle, + proxy_session_id: &ProxySessionId, + lease: HlsTransientResourceLeaseContext<'_>, + resource_file: &TransientResourceFile, + range_header: Option<&HeaderValue>, + now_ms: u64, + cache_duration_ms: u64, +) -> Result { + // `is_gc_marked_for_removal` is `&self` and is the dominant early-exit on a + // busy session (GC sweeps mark sessions on a timer). Resolve it under a read + // lock so we don't pay for the exclusive write-lock acquisition when the + // session is already doomed. + if session.read().await.is_gc_marked_for_removal() { + return Err(StatusCode::NOT_FOUND); + } + let mut session = session.write().await; + let Some(resource) = session.transient.resolve_resource_for_lease( + &resource_file.resource_id, + lease.access_lease_id, + lease.lease_issued_at_ms, + lease.published_resource_ids, + now_ms, + ) else { + return Err(StatusCode::NOT_FOUND); + }; + if resource.file_ext_hint.as_deref().is_some_and(|extension| extension != resource_file.extension) { + return Err(StatusCode::NOT_FOUND); + } + let key_object_cache_allowed = + resource.kind != TransientResourceKind::Key || session.mode == HlsSessionMode::NormalCacheTimeline; + let action = transient_object_cache_action( + &mut session, + HlsTransientObjectCacheActionInput { + proxy_session_id, + resource: &resource, + resource_file, + range_header, + now_ms, + cache_duration_ms, + key_object_cache_allowed, + }, + ); + Ok(HlsTransientObjectCacheResolution { + resource, + origin_headers: session.origin_request_headers.clone(), + origin_provider_session_headers: session.origin_provider_session_headers.clone(), + action, + }) +} + +fn transient_object_cache_action( + session: &mut super::super::HlsSession, + input: HlsTransientObjectCacheActionInput<'_>, +) -> HlsTransientObjectCacheAction { + if input.resource.kind == TransientResourceKind::Key && !input.key_object_cache_allowed { + return HlsTransientObjectCacheAction::PassthroughNoCache; + } + let cache_key = TransientPassthroughState::transient_object_key( + input.proxy_session_id, + &input.resource.id, + input.resource_file.extension.clone(), + ); + if session.transient.ready_object(&cache_key, input.resource.kind, input.now_ms).is_some() { + return HlsTransientObjectCacheAction::ServeReady; + } + if !is_hls_transient_full_object_cacheable_request(input.range_header) { + return HlsTransientObjectCacheAction::PassthroughNoCache; + } + session + .transient + .begin_object_fetch( + input.proxy_session_id, + input.resource, + &input.resource_file.extension, + input.now_ms, + input.cache_duration_ms, + ) + .into() +} + +impl From for HlsTransientObjectCacheAction { + fn from(decision: TransientObjectFetchDecision) -> Self { + match decision { + TransientObjectFetchDecision::Ready => Self::ServeReady, + TransientObjectFetchDecision::Fetch(cache_key) => Self::FetchAndCache(cache_key), + TransientObjectFetchDecision::Wait(notifier) => Self::WaitForFetch(notifier), + } + } +} + +pub fn is_hls_transient_full_object_cacheable_request(range_header: Option<&HeaderValue>) -> bool { + let Some(range_header) = range_header else { + return true; + }; + range_header.to_str().is_ok_and(|range| range.trim() == "bytes=0-") +} + +impl HlsTransientObjectFetchFinalizer { + pub fn new( + session: HlsSessionHandle, + segment_cache: Arc, + fetch_token: TransientObjectFetchToken, + retry_after_ms: u64, + ) -> Self { + Self { session, segment_cache, fetch_token, completed: false, retry_after_ms } + } + + pub fn complete(&mut self) { self.completed = true; } +} + +impl Drop for HlsTransientObjectFetchFinalizer { + fn drop(&mut self) { + if self.completed { + return; + } + let session = Arc::clone(&self.session); + let segment_cache = Arc::clone(&self.segment_cache); + let fetch_token = self.fetch_token.clone(); + let retry_after_ms = self.retry_after_ms; + tokio::spawn(async move { + if let Err(err) = segment_cache.delete(fetch_token.cache_key()).await { + debug!("HLS abandoned transient cache fill cleanup failed: error={err}"); + } + delete_superseded_transient_object(&segment_cache, &fetch_token).await; + session.write().await.fail_transient_object_retryable_if_current( + &fetch_token, + current_time_millis(), + retry_after_ms, + ); + }); + } +} + +pub(crate) async fn delete_superseded_transient_object( + segment_cache: &HlsSegmentCache, + fetch_token: &TransientObjectFetchToken, +) { + let Some(superseded) = fetch_token.superseded_object() else { + return; + }; + if let Err(err) = segment_cache.delete(&superseded.key).await { + debug!("HLS superseded transient cache object cleanup failed: error={err}"); + } +} + +#[derive(Clone)] +pub struct HlsTransientCacheCommitContext { + pub segment_cache: Arc, + pub segment_repair: Arc, + pub session: HlsSessionHandle, + pub proxy_session_id: ProxySessionId, + pub log_identity: super::super::HlsLogIdentity, + pub access_lease_id: HlsAccessLeaseId, + pub resource: TransientResourceRef, + pub resource_file: TransientResourceFile, + pub fetch_token: TransientObjectFetchToken, + pub cache_duration_ms: u64, +} + +pub struct HlsTransientOriginCacheFetchRequest { + pub fetch: HlsTransientOriginFetchRequest, + pub commit: HlsTransientCacheCommitContext, +} + +pub async fn fetch_and_commit_hls_transient_origin_response_with_attempt_prepare( + request: HlsTransientOriginCacheFetchRequest, + prepare_attempt: P, +) -> Result<(), HlsOriginResourceFetchError> +where + G: Send + 'static, + P: FnMut(HlsResourceFetchAttempt) -> BoxFuture<'static, Result>, +{ + let target = build_hls_transient_resource_fetch_target( + &request.fetch.resolved_origin_uri, + &request.fetch.origin_headers, + &request.fetch.origin_provider_session_headers, + HlsTransientOriginFetchMode::CacheFullObject, + request.fetch.resource_file.resource_id.0.as_str(), + request.fetch.resource_kind, + )?; + let commit = request.commit; + run_hls_origin_resource_retry_loop_with_attempt_prepare( + target, + request.fetch.clients, + &request.fetch.policy, + &request.fetch.log_identity, + prepare_attempt, + |guard| async move { drop(guard) }.boxed(), + move |decoded, _attempt, body_deadline, guard| { + let commit = commit.clone(); + async move { + let result = commit_hls_transient_origin_response_attempt(commit, decoded, body_deadline).await; + drop(guard); + result + } + .boxed() + }, + ) + .await +} + +async fn commit_hls_transient_origin_response_attempt( + context: HlsTransientCacheCommitContext, + decoded: DecodedHttpResponse, + body_deadline: HlsOriginResourceBodyDeadline, +) -> Result<(), HlsOriginResourceFetchError> { + let content_type = decoded + .headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_string) + .or_else(|| context.resource.content_type_hint.clone()) + .unwrap_or_else(|| "application/octet-stream".to_string()); + let proxy_session_id = context.proxy_session_id.clone(); + let log_identity = context.log_identity.clone(); + let repair_context = HlsSegmentRepairObjectContext { + source: HlsSegmentRepairSource::Transient, + log_identity, + proxy_session_id, + hls_access_lease_id: Some(context.access_lease_id.clone()), + rendered_object_id: HlsRepairRenderedObjectId::Transient { + resource_id: context.resource_file.resource_id.0.clone(), + }, + resource_id: context.resource_file.resource_id.0.clone(), + file_ext: context.resource_file.extension.clone(), + origin_fetch_uri_for_diagnostics: context.resource.resolved_origin_uri.clone(), + media_sequence: None, + discontinuity_sequence: None, + complete_object: true, + encrypted: context.resource.encrypted_media || context.resource.kind == TransientResourceKind::Key, + custom_response: false, + }; + let cache_key = context.fetch_token.cache_key().clone(); + let commit = Box::pin(context.segment_repair.commit_origin_response( + &context.segment_cache, + &cache_key, + decoded.body, + body_deadline.deadline(), + repair_context, + )) + .await; + let ready_at_ms = current_time_millis(); + let metadata = match commit { + Ok(metadata) => metadata, + Err(err) => return Err(HlsOriginResourceFetchError::cache_body(&err)), + }; + let expires_at_ms = transient_object_expires_at(ready_at_ms, context.cache_duration_ms); + let mut session = context.session.write().await; + if context.resource.kind == TransientResourceKind::Key && metadata.size != 16 { + session.fail_transient_object_permanent_if_current( + &context.fetch_token, + ready_at_ms, + Some(StatusCode::BAD_GATEWAY), + ); + drop(session); + delete_stale_transient_fill(&context.segment_cache, &cache_key).await; + return Err(HlsOriginResourceFetchError::NonRetryableStatus(StatusCode::BAD_GATEWAY)); + } + let committed = session.commit_transient_object_ready_if_current( + context.resource.kind, + &context.fetch_token, + content_type, + metadata.size, + ready_at_ms, + expires_at_ms, + ); + drop(session); + if !committed { + delete_stale_transient_fill(&context.segment_cache, &cache_key).await; + return Err(HlsOriginResourceFetchError::Superseded); + } + delete_superseded_transient_object(&context.segment_cache, &context.fetch_token).await; + Ok(()) +} + +async fn delete_stale_transient_fill(segment_cache: &HlsSegmentCache, cache_key: &TransientObjectCacheKey) { + if let Err(err) = segment_cache.delete(cache_key).await { + debug!("HLS stale transient cache fill cleanup failed: error={err}"); + } +} diff --git a/backend/hls/src/transient_fetcher/lifecycle.rs b/backend/hls/src/transient_fetcher/lifecycle.rs new file mode 100644 index 000000000..34857fd1e --- /dev/null +++ b/backend/hls/src/transient_fetcher/lifecycle.rs @@ -0,0 +1,171 @@ +use super::{ + CacheAccessState, HlsLogIdentity, HlsOriginAccountIoLeaseGuard, HlsOriginIoContext, HlsSegmentFailureObject, + HlsSegmentFailureTransition, HlsSessionHandle, HlsTransientOriginIoGuard, HlsTransientReadGuard, + SegmentFetchPolicy, TransientResourceKind, TransientResourceRef, +}; +use log::{debug, warn}; +use std::sync::Arc; +use tuliprox_core::utils::current_time_millis; + +pub async fn record_successful_transient_segment_fetch(session: &HlsSessionHandle, resource: &TransientResourceRef) { + if !transient_resource_is_media(resource.kind) { + return; + } + let mut session = session.write().await; + if !transient_resource_is_current(&session, resource, current_time_millis()) { + return; + } + if let Some(reset_failures) = session.record_successful_segment_fetch() { + if log::log_enabled!(log::Level::Debug) { + let identity = HlsLogIdentity::from_session(&session); + debug!( + "HLS segment temporary failure counter reset: session={} proxy_session={} previous_failures={reset_failures}", + identity.session(), + identity.proxy_session() + ); + } + } +} + +pub async fn record_temporary_transient_segment_fetch_failure( + session: &HlsSessionHandle, + resource: &TransientResourceRef, + policy: &SegmentFetchPolicy, + now_ms: u64, +) -> bool { + if !transient_resource_affects_media_readiness(resource.kind) { + return false; + } + let mut session = session.write().await; + if !transient_resource_is_current(&session, resource, now_ms) { + return false; + } + session.origin_control.path_condition = + super::super::origin_progress::HlsOriginPathCondition::SegmentReadinessFailure; + if !transient_resource_is_media(resource.kind) { + return false; + } + let threshold = policy.permanent_failure_segment_threshold.max(1); + match session.record_temporary_segment_fetch_failure( + now_ms, + HlsSegmentFailureObject::Transient { resource_id: resource.id.0.clone() }, + threshold, + ) { + HlsSegmentFailureTransition::StillRetryable { failures, threshold } => { + if log::log_enabled!(log::Level::Debug) { + let identity = HlsLogIdentity::from_session(&session); + debug!( + "HLS segment temporary failure counted: session={} proxy_session={} object={} failures={} threshold={}", + identity.session(), + identity.proxy_session(), + resource.id.0, + failures, + threshold + ); + } + false + } + HlsSegmentFailureTransition::BecamePermanentlyFailed { failures, threshold } => { + if log::log_enabled!(log::Level::Warn) { + let identity = HlsLogIdentity::from_session(&session); + warn!( + "HLS segment temporary failure threshold reached: session={} proxy_session={} failures={} threshold={}", + identity.session(), + identity.proxy_session(), + failures, + threshold + ); + } + session.invalidate_queued_origin_work(); + true + } + } +} + +const fn transient_resource_is_media(kind: TransientResourceKind) -> bool { + matches!(kind, TransientResourceKind::Segment | TransientResourceKind::Part) +} + +pub(super) const fn transient_resource_affects_media_readiness(kind: TransientResourceKind) -> bool { + matches!( + kind, + TransientResourceKind::Segment + | TransientResourceKind::Part + | TransientResourceKind::Key + | TransientResourceKind::Map + ) +} + +fn transient_resource_is_current( + session: &super::super::HlsSession, + resource: &TransientResourceRef, + now_ms: u64, +) -> bool { + session.transient.resource_matches_current(resource, now_ms) +} + +impl HlsTransientReadGuard { + pub(super) fn new(access: Arc, now_ms: u64) -> Self { + access.reader_started(now_ms); + Self { access } + } +} + +impl Drop for HlsTransientReadGuard { + fn drop(&mut self) { self.access.reader_finished(); } +} + +impl HlsTransientOriginIoGuard { + pub fn new( + session: HlsSessionHandle, + active_origin_work_count: std::sync::Arc, + origin_io: HlsOriginIoContext, + lease_guard: HlsOriginAccountIoLeaseGuard, + started_generation: u64, + ) -> Self { + Self { + session, + active_origin_work_count, + origin_io, + lease_guard: Some(lease_guard), + started_generation, + origin_work_finished: false, + } + } + + pub async fn finish_clean(mut self) { + let generation_valid = { + let mut session = self.session.write().await; + let valid = session.finish_origin_work(self.started_generation); + self.origin_work_finished = true; + valid + }; + let refresh_reservation = if generation_valid { + self.session + .read() + .await + .should_refresh_origin_reservation(chrono::Utc::now().timestamp_millis().try_into().unwrap_or_default()) + } else { + false + }; + if let Some(lease_guard) = self.lease_guard.take() { + crate::origin::finish_hls_origin_account_io( + &self.origin_io, + &self.session, + lease_guard, + refresh_reservation, + ) + .await; + } + } +} + +impl Drop for HlsTransientOriginIoGuard { + fn drop(&mut self) { + if self.origin_work_finished { + return; + } + self.origin_work_finished = true; + self.active_origin_work_count.fetch_sub(1, std::sync::atomic::Ordering::AcqRel); + } +} diff --git a/backend/hls/src/transient_fetcher/origin.rs b/backend/hls/src/transient_fetcher/origin.rs new file mode 100644 index 000000000..973bbbda9 --- /dev/null +++ b/backend/hls/src/transient_fetcher/origin.rs @@ -0,0 +1,118 @@ +use super::{ + build_hls_origin_resource_headers_with_client_range, run_hls_origin_resource_retry_loop_with_attempt_prepare, + HlsLogIdentity, HlsOriginByteRangeExpectation, HlsOriginResourceBodyDeadline, HlsOriginResourceClients, + HlsOriginResourceFetchError, HlsOriginResourceFetchTarget, HlsResourceFetchAttempt, HlsResourceFetchKind, + HlsResourceFetchSource, SegmentFetchPolicy, TransientResourceFile, TransientResourceKind, +}; +use axum::http::{HeaderMap, HeaderValue, StatusCode}; +use futures::{future::BoxFuture, FutureExt}; +use tuliprox_core::utils::content_coding::DecodedHttpResponse; + +pub enum HlsTransientObjectFetchFailure { + Retryable, + Permanent { status: Option }, +} + +pub fn hls_transient_object_fetch_failure(error: &HlsOriginResourceFetchError) -> HlsTransientObjectFetchFailure { + if let Some(status) = error.permanent_status() { + return HlsTransientObjectFetchFailure::Permanent { status: Some(status) }; + } + if !error.retryable_failure() { + return HlsTransientObjectFetchFailure::Permanent { status: None }; + } + HlsTransientObjectFetchFailure::Retryable +} + +pub fn hls_transient_resource_fetch_kind(resource_kind: TransientResourceKind) -> HlsResourceFetchKind { + match resource_kind { + TransientResourceKind::Key => HlsResourceFetchKind::Key, + TransientResourceKind::Map => HlsResourceFetchKind::Map, + TransientResourceKind::Segment => HlsResourceFetchKind::Segment, + TransientResourceKind::Part => HlsResourceFetchKind::Part, + TransientResourceKind::Other => HlsResourceFetchKind::Other, + } +} + +pub(super) fn build_hls_transient_resource_fetch_target( + resolved_origin_uri: &str, + origin_headers: &HeaderMap, + origin_provider_session_headers: &HeaderMap, + mode: HlsTransientOriginFetchMode, + resource_id: &str, + resource_kind: TransientResourceKind, +) -> Result { + let (range_header, byte_range_expectation) = match mode { + HlsTransientOriginFetchMode::CacheFullObject => (None, HlsOriginByteRangeExpectation::FullObject), + HlsTransientOriginFetchMode::DirectPassthrough { client_range } => { + (client_range, HlsOriginByteRangeExpectation::AnySuccess) + } + }; + Ok(HlsOriginResourceFetchTarget { + kind: hls_transient_resource_fetch_kind(resource_kind), + source: HlsResourceFetchSource::Transient, + object_id: resource_id.to_string(), + origin_url: resolved_origin_uri.to_string(), + headers: build_hls_origin_resource_headers_with_client_range( + origin_headers, + origin_provider_session_headers, + range_header, + )?, + byte_range_expectation, + }) +} + +/// Keeps full-object cache fills separate from decoded client-range passthrough. +pub(super) enum HlsTransientOriginFetchMode { + CacheFullObject, + DirectPassthrough { client_range: Option }, +} + +pub struct HlsTransientOriginFetchRequest { + pub resolved_origin_uri: String, + pub origin_headers: HeaderMap, + pub origin_provider_session_headers: HeaderMap, + pub range_header: Option, + pub resource_file: TransientResourceFile, + pub resource_kind: TransientResourceKind, + pub clients: HlsOriginResourceClients, + pub policy: SegmentFetchPolicy, + pub log_identity: HlsLogIdentity, +} + +/// A decoded direct-origin response together with the attempt and guards that own its origin work. +pub struct HlsTransientDecodedOriginResponse { + pub decoded: DecodedHttpResponse, + pub body_deadline: HlsOriginResourceBodyDeadline, + pub attempt: HlsResourceFetchAttempt, + pub guard: G, +} + +pub async fn fetch_hls_transient_origin_response_with_attempt_prepare( + request: HlsTransientOriginFetchRequest, + prepare_attempt: P, +) -> Result, HlsOriginResourceFetchError> +where + G: Send + 'static, + P: FnMut(HlsResourceFetchAttempt) -> BoxFuture<'static, Result>, +{ + let target = build_hls_transient_resource_fetch_target( + &request.resolved_origin_uri, + &request.origin_headers, + &request.origin_provider_session_headers, + HlsTransientOriginFetchMode::DirectPassthrough { client_range: request.range_header }, + request.resource_file.resource_id.0.as_str(), + request.resource_kind, + )?; + run_hls_origin_resource_retry_loop_with_attempt_prepare( + target, + request.clients, + &request.policy, + &request.log_identity, + prepare_attempt, + |guard| async move { drop(guard) }.boxed(), + |decoded, attempt, body_deadline, guard| { + async move { Ok(HlsTransientDecodedOriginResponse { decoded, body_deadline, attempt, guard }) }.boxed() + }, + ) + .await +} diff --git a/backend/hls/src/transient_fetcher/response.rs b/backend/hls/src/transient_fetcher/response.rs new file mode 100644 index 000000000..245060ae5 --- /dev/null +++ b/backend/hls/src/transient_fetcher/response.rs @@ -0,0 +1,217 @@ +use super::{ + hls_client_body_send_deadline, hls_transient_resource_fetch_kind, + lifecycle::transient_resource_affects_media_readiness, log_hls_resource_body_failure, + record_successful_transient_segment_fetch, record_temporary_transient_segment_fetch_failure, + refresh_hls_client_body_send_deadline, HlsLogIdentity, HlsResourceFetchLogContext, HlsResourceFetchSource, + HlsSessionHandle, HlsTransientDecodedOriginResponse, HlsTransientDirectResponseFinalizer, + HlsTransientDirectResponseLifecycleContext, HlsTransientOriginIoGuard, HlsTransientReadGuard, SegmentFetchPolicy, + TransientResourceKind, TransientResourceRef, +}; +use axum::{body::Body, http::header, response::IntoResponse}; +use futures::StreamExt; +use log::warn; +use std::{io, sync::Arc}; +use tokio::time::sleep; +use tokio_util::io::ReaderStream; +use tuliprox_core::{ + try_unwrap_body, + utils::{current_time_millis, response_compression::mark_response_as_uncompressed}, +}; + +/// Context retained until a direct transient response body reaches one terminal outcome. +pub struct HlsTransientDirectResponseContext { + pub session: HlsSessionHandle, + pub resource: TransientResourceRef, + pub policy: SegmentFetchPolicy, + pub now_ms: u64, + pub log_identity: HlsLogIdentity, +} + +#[derive(Clone, Copy)] +pub(super) enum HlsTransientDirectStreamOutcome { + CleanEof, + OriginBodyFailure, + ClientAborted, +} + +impl HlsTransientDirectResponseFinalizer { + pub(super) fn new(context: HlsTransientDirectResponseLifecycleContext) -> Self { + let context = transient_resource_affects_media_readiness(context.resource.kind).then_some(context); + Self { context } + } + + pub(super) async fn finish(&mut self, outcome: HlsTransientDirectStreamOutcome) { + let Some(completion) = self.begin_finish(outcome) else { + return; + }; + if let Err(error) = completion.await { + warn!( + "HLS transient direct lifecycle task failed: cancelled={} panic={}", + error.is_cancelled(), + error.is_panic() + ); + } + } + + pub(super) fn begin_finish( + &mut self, + outcome: HlsTransientDirectStreamOutcome, + ) -> Option> { + let context = self.context.take()?; + if matches!(outcome, HlsTransientDirectStreamOutcome::ClientAborted) { + return None; + } + // Once EOF or an origin-body failure has been observed, its state transition + // must survive cancellation of the downstream body poll. + Some(tokio::spawn(async move { + match outcome { + HlsTransientDirectStreamOutcome::CleanEof => { + record_successful_transient_segment_fetch(&context.session, &context.resource).await; + } + HlsTransientDirectStreamOutcome::OriginBodyFailure => { + let failed_at_ms = current_time_millis(); + record_temporary_transient_segment_fetch_failure( + &context.session, + &context.resource, + &context.policy, + failed_at_ms, + ) + .await; + } + HlsTransientDirectStreamOutcome::ClientAborted => {} + } + })) + } + + pub(super) fn finish_client_aborted(&mut self) { self.context.take(); } +} + +impl Drop for HlsTransientDirectResponseFinalizer { + fn drop(&mut self) { + // A body dropped before EOF is a downstream abort. It must release the + // retained guards without changing provider/segment failure state. + self.finish_client_aborted(); + } +} + +pub fn hls_transient_origin_response( + response: HlsTransientDecodedOriginResponse>, + context: HlsTransientDirectResponseContext, +) -> axum::response::Response { + let mut builder = axum::response::Response::builder().status(response.decoded.status); + for header_name in [ + header::CONTENT_TYPE, + header::CONTENT_LENGTH, + header::CONTENT_RANGE, + header::ACCEPT_RANGES, + header::CACHE_CONTROL, + header::ETAG, + header::LAST_MODIFIED, + ] { + if let Some(value) = response.decoded.headers.get(&header_name) { + builder = builder.header(header_name, value.clone()); + } + } + + let mut response = try_unwrap_body!(builder.body(hls_transient_direct_body(response, context))); + mark_response_as_uncompressed(&mut response); + response +} + +fn hls_transient_direct_body( + response: HlsTransientDecodedOriginResponse>, + context: HlsTransientDirectResponseContext, +) -> Body { + let HlsTransientDecodedOriginResponse { decoded, body_deadline, attempt, guard: origin_io_guard } = response; + let HlsTransientDirectResponseContext { session, resource, policy, now_ms, log_identity } = context; + + let guard = HlsTransientReadGuard::new(Arc::clone(&resource.access), now_ms); + let finalizer = HlsTransientDirectResponseFinalizer::new(HlsTransientDirectResponseLifecycleContext { + session, + resource: resource.clone(), + policy, + }); + let resource_id = resource.id.0.clone(); + let resource_kind = resource.kind; + let stream = futures::stream::unfold( + ( + ReaderStream::new(decoded.body), + Some(guard), + origin_io_guard, + finalizer, + Box::pin(sleep(hls_client_body_send_deadline())), + false, + ), + move |(mut stream, guard, mut origin_io_guard, mut finalizer, mut send_deadline, finished)| { + let log_identity = log_identity.clone(); + let resource_id = resource_id.clone(); + async move { + if finished { + return None; + } + let next_chunk = tokio::select! { + () = send_deadline.as_mut() => { + finalizer.finish(HlsTransientDirectStreamOutcome::ClientAborted).await; + return Some(( + Err(io::Error::new(io::ErrorKind::TimedOut, "hls client body send timed out")), + (stream, None, None, finalizer, send_deadline, true), + )); + } + // Decoder setup is bounded by the absolute attempt deadline before this + // response is built. Once handed to the client, retain the existing + // per-chunk origin-body idle timeout instead of imposing a total-body limit. + next_chunk = tokio::time::timeout(body_deadline.timeout(), stream.next()) => next_chunk, + }; + match next_chunk { + Ok(Some(Ok(chunk))) => { + refresh_hls_client_body_send_deadline(send_deadline.as_mut()); + Some((Ok(chunk), (stream, guard, origin_io_guard, finalizer, send_deadline, false))) + } + Ok(Some(Err(err))) => { + log_hls_resource_body_failure( + &log_identity, + hls_transient_direct_log_context(&resource_id, resource_kind), + attempt, + &err, + body_deadline.timeout().as_millis(), + ); + finalizer.finish(HlsTransientDirectStreamOutcome::OriginBodyFailure).await; + Some((Err(err), (stream, None, None, finalizer, send_deadline, true))) + } + Ok(None) => { + finalizer.finish(HlsTransientDirectStreamOutcome::CleanEof).await; + if let Some(guard) = origin_io_guard.take() { + guard.finish_clean().await; + } + None + } + Err(_) => { + let error = io::Error::new(io::ErrorKind::TimedOut, "transient passthrough body timed out"); + log_hls_resource_body_failure( + &log_identity, + hls_transient_direct_log_context(&resource_id, resource_kind), + attempt, + &error, + body_deadline.timeout().as_millis(), + ); + finalizer.finish(HlsTransientDirectStreamOutcome::OriginBodyFailure).await; + Some((Err(error), (stream, None, None, finalizer, send_deadline, true))) + } + } + } + }, + ); + Body::from_stream(stream) +} + +fn hls_transient_direct_log_context( + resource_id: &str, + resource_kind: TransientResourceKind, +) -> HlsResourceFetchLogContext<'_> { + HlsResourceFetchLogContext { + kind: hls_transient_resource_fetch_kind(resource_kind), + source: HlsResourceFetchSource::Transient, + object_id: resource_id, + origin_url: None, + } +} diff --git a/backend/hls/src/transient_fetcher/tests.rs b/backend/hls/src/transient_fetcher/tests.rs new file mode 100644 index 000000000..504d7ac96 --- /dev/null +++ b/backend/hls/src/transient_fetcher/tests.rs @@ -0,0 +1,488 @@ +use super::{ + super::{ + origin_progress::{HlsOriginPathCondition, HlsOriginProgressPhase}, + resource_fetch::take_body_failure_log_attempts, + }, + fetch_and_commit_hls_transient_origin_response_with_attempt_prepare, + fetch_hls_transient_origin_response_with_attempt_prepare, hls_transient_object_fetch_failure, + hls_transient_origin_response, record_temporary_transient_segment_fetch_failure, + resolve_hls_transient_object_cache_action, HlsLogIdentity, HlsTransientCacheCommitContext, + HlsTransientDecodedOriginResponse, HlsTransientDirectResponseContext, HlsTransientDirectResponseFinalizer, + HlsTransientDirectResponseLifecycleContext, HlsTransientDirectStreamOutcome, HlsTransientObjectCacheAction, + HlsTransientObjectFetchFailure, HlsTransientOriginCacheFetchRequest, HlsTransientOriginFetchRequest, + HlsTransientOriginIoGuard, HlsTransientResourceLeaseContext, +}; +use crate::{ + origin::{HlsOriginAccountBinding, HlsOriginAccountIoLease, HlsOriginAccountIoLeaseGuard, HlsOriginIoContext}, + HlsAccessLeaseId, HlsOriginResourceClients, HlsOriginResourceFetchError, HlsPublishedTransientResourceIds, + HlsSegmentCache, HlsSegmentFailureObject, HlsSegmentRepairManager, HlsSession, HlsSessionHandle, HlsSessionKey, + HlsSessionStore, ProxySessionId, SegmentFetchPolicy, TransientObjectCacheKey, TransientObjectCacheStatus, + TransientObjectFetchDecision, TransientObjectFetchToken, TransientPassthroughState, TransientResourceFile, + TransientResourceKind, TransientResourceRef, +}; +use async_compression::tokio::write::{BrotliEncoder, GzipEncoder, ZstdEncoder}; +use axum::{ + body::to_bytes, + http::{header, HeaderMap, HeaderValue, StatusCode}, +}; +use futures::FutureExt; +use shared::model::HlsSegmentRepairMode; +use std::{ + io, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::{Mutex, RwLock}, +}; +use tuliprox_core::{model::HlsSegmentRepairConfig, utils::response_compression::should_compress_response}; + +struct TestOrigin { + base_url: String, + requests: Arc>>, + task: tokio::task::JoinHandle<()>, +} + +impl Drop for TestOrigin { + fn drop(&mut self) { self.task.abort(); } +} + +fn finalized_resource_resolution_fixture( +) -> (HlsSessionHandle, ProxySessionId, HlsAccessLeaseId, TransientResourceFile, HlsPublishedTransientResourceIds) { + let mut session = HlsSession::new(HlsSessionKey::new(1, "finalized-range"), b"rewrite-secret", 0); + let proxy_session_id = session.proxy_session_id.clone(); + let access_lease_id = HlsAccessLeaseId("lease".to_string()); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/archive/segment.ts", + b"rewrite-secret", + 0, + 300_000, + Some("ts".to_string()), + ); + let resource_file = TransientResourceFile { resource_id: resource.id.clone(), extension: "ts".to_string() }; + session.transient.upsert_resources([resource]); + let manifest_body = format!( + "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXTINF:1,\n/hls/shared/live/{}/lease/r/{}.ts\n#EXT-X-ENDLIST\n", + proxy_session_id.0, resource_file.resource_id.0 + ); + let published_resource_ids = HlsPublishedTransientResourceIds::from_manifest_body(&manifest_body); + session.transient.replace_manifest_with_semantics(manifest_body, 0, Some(1_000)); + let manifest_generation = + session.transient.current_finalized_manifest_generation().expect("finalized manifest generation"); + assert!(session.transient.bind_finalized_manifest_generation(super::super::TransientManifestLeaseBinding::new( + access_lease_id.clone(), + 0, + manifest_generation, + ))); + (Arc::new(RwLock::new(session)), proxy_session_id, access_lease_id, resource_file, published_resource_ids) +} + +async fn spawn_test_origin( + status_line: &'static str, + response_headers: Vec<(&'static str, &'static str)>, + body: Vec, +) -> TestOrigin { + spawn_test_origin_in_chunks(status_line, response_headers, vec![body], Duration::ZERO).await +} + +async fn spawn_test_origin_in_chunks( + status_line: &'static str, + response_headers: Vec<(&'static str, &'static str)>, + body_chunks: Vec>, + inter_chunk_delay: Duration, +) -> TestOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("test origin address"); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let body_chunks = Arc::new(body_chunks); + let content_length = body_chunks.iter().map(Vec::len).sum::(); + let response_headers = Arc::new(response_headers); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + let body_chunks = Arc::clone(&body_chunks); + let response_headers = Arc::clone(&response_headers); + tokio::spawn(async move { + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 2048]; + let Ok(read) = socket.read(&mut chunk).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); + let mut response = format!("HTTP/1.1 {status_line}\r\nContent-Length: {content_length}\r\n"); + for (name, value) in response_headers.iter() { + response.push_str(name); + response.push_str(": "); + response.push_str(value); + response.push_str("\r\n"); + } + response.push_str("Connection: close\r\n\r\n"); + let _ = socket.write_all(response.as_bytes()).await; + for (index, chunk) in body_chunks.iter().enumerate() { + if index > 0 && !inter_chunk_delay.is_zero() { + tokio::time::sleep(inter_chunk_delay).await; + } + if socket.write_all(chunk).await.is_err() { + return; + } + } + }); + } + }); + + TestOrigin { base_url: format!("http://{addr}"), requests, task } +} + +async fn spawn_retry_then_body_origin( + response_headers: Vec<(&'static str, &'static str)>, + body: Vec, +) -> TestOrigin { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("test origin binds"); + let addr = listener.local_addr().expect("test origin address"); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let attempts = Arc::new(AtomicUsize::new(0)); + let task_attempts = Arc::clone(&attempts); + let body = Arc::new(body); + let response_headers = Arc::new(response_headers); + let task = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + break; + }; + let requests = Arc::clone(&task_requests); + let attempts = Arc::clone(&task_attempts); + let body = Arc::clone(&body); + let response_headers = Arc::clone(&response_headers); + tokio::spawn(async move { + let mut request = Vec::new(); + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 2048]; + let Ok(read) = socket.read(&mut chunk).await else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + requests.lock().await.push(String::from_utf8_lossy(&request).to_string()); + if attempts.fetch_add(1, Ordering::Relaxed) == 0 { + let _ = socket + .write_all( + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .await; + return; + } + let mut response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n", body.len()); + for (name, value) in response_headers.iter() { + response.push_str(name); + response.push_str(": "); + response.push_str(value); + response.push_str("\r\n"); + } + response.push_str("Connection: close\r\n\r\n"); + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.write_all(&body).await; + }); + } + }); + + TestOrigin { base_url: format!("http://{addr}"), requests, task } +} + +async fn spawn_zstd_origin(body: Vec) -> TestOrigin { + spawn_test_origin("200 OK", vec![("Content-Encoding", "zstd"), ("Content-Type", "application/octet-stream")], body) + .await +} + +async fn gzip_encode(body: &[u8]) -> Vec { + let mut encoder = GzipEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("test body encodes"); + encoder.shutdown().await.expect("test encoder finishes"); + encoder.into_inner() +} + +async fn brotli_encode(body: &[u8]) -> Vec { + let mut encoder = BrotliEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("test body encodes"); + encoder.shutdown().await.expect("test encoder finishes"); + encoder.into_inner() +} + +async fn zstd_encode(body: &[u8]) -> Vec { + let mut encoder = ZstdEncoder::new(Vec::new()); + encoder.write_all(body).await.expect("test body encodes"); + encoder.shutdown().await.expect("test encoder finishes"); + encoder.into_inner() +} + +async fn fetch_direct_decoded( + origin_url: String, + resource_kind: TransientResourceKind, + range_header: Option, +) -> Result>, HlsOriginResourceFetchError> { + fetch_direct_decoded_with_timeout(origin_url, resource_kind, range_header, 1_000).await +} + +async fn fetch_direct_decoded_with_timeout( + origin_url: String, + resource_kind: TransientResourceKind, + range_header: Option, + origin_segment_timeout_ms: u64, +) -> Result>, HlsOriginResourceFetchError> { + let resource = + TransientResourceRef::new(resource_kind, origin_url, b"rewrite-secret", 10, 60_000, Some("bin".to_string())); + let request = HlsTransientOriginFetchRequest { + resolved_origin_uri: resource.resolved_origin_uri.clone(), + origin_headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + range_header, + resource_file: TransientResourceFile { resource_id: resource.id, extension: "bin".to_string() }, + resource_kind, + clients: HlsOriginResourceClients { + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client builds"), + use_manual_redirects: false, + }, + policy: SegmentFetchPolicy { + origin_segment_timeout_ms, + retry_delays_ms: [0; 5], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }, + log_identity: HlsLogIdentity::for_test("direct-transient-content-session", "direct-transient-proxy-session"), + }; + fetch_hls_transient_origin_response_with_attempt_prepare(request, |_| async { Ok(None) }.boxed()).await +} + +fn direct_client_response( + response: HlsTransientDecodedOriginResponse>, + resource_kind: TransientResourceKind, +) -> axum::response::Response { + TestDirectResponseFixture::new(resource_kind, "http://127.0.0.1/origin").response(response) +} + +struct TestDirectResponseFixture { + session: HlsSessionHandle, + resource: TransientResourceRef, + policy: SegmentFetchPolicy, + log_identity: HlsLogIdentity, +} + +impl TestDirectResponseFixture { + fn new(resource_kind: TransientResourceKind, origin_url: impl Into) -> Self { + let resource = TransientResourceRef::new( + resource_kind, + origin_url, + b"rewrite-secret", + tuliprox_core::utils::current_time_millis(), + 60_000, + Some("bin".to_string()), + ); + let policy = + SegmentFetchPolicy { retry_delays_ms: [0; 5], retry_jitter_max_ms: 0, ..SegmentFetchPolicy::default() }; + let mut session = HlsSession::new(HlsSessionKey::new(1, "direct-test"), b"rewrite-secret", 10); + session.transient.upsert_resources([resource.clone()]); + let log_identity = HlsLogIdentity::from_session(&session); + Self { session: Arc::new(RwLock::new(session)), resource, policy, log_identity } + } + + fn response( + &self, + response: HlsTransientDecodedOriginResponse>, + ) -> axum::response::Response { + hls_transient_origin_response(response, self.response_context()) + } + + fn response_context(&self) -> HlsTransientDirectResponseContext { + HlsTransientDirectResponseContext { + session: Arc::clone(&self.session), + resource: self.resource.clone(), + policy: self.policy.clone(), + now_ms: 10, + log_identity: self.log_identity.clone(), + } + } + + fn lifecycle_context(&self) -> HlsTransientDirectResponseLifecycleContext { + HlsTransientDirectResponseLifecycleContext { + session: Arc::clone(&self.session), + resource: self.resource.clone(), + policy: self.policy.clone(), + } + } + + async fn seed_segment_failure(&self) { + assert!( + !record_temporary_transient_segment_fetch_failure(&self.session, &self.resource, &self.policy, 9).await + ); + } + + async fn segment_failure_count(&self) -> u32 { + self.session.read().await.segment_failure_tracker.consecutive_temporary_failures + } + + async fn wait_for_segment_failure_count(&self, expected: u32) { + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if self.segment_failure_count().await == expected { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("segment failure state reaches expected value"); + } + + async fn last_segment_failure(&self) -> Option { + self.session.read().await.segment_failure_tracker.last_failed_object.clone() + } +} + +struct TestTransientCacheFixture { + temp_dir: tempfile::TempDir, + segment_cache: Arc, + segment_repair: Arc, + session: HlsSessionHandle, + log_identity: HlsLogIdentity, + resource: TransientResourceRef, + resource_file: TransientResourceFile, + lookup_key: TransientObjectCacheKey, + cache_key: TransientObjectCacheKey, + fetch_token: TransientObjectFetchToken, +} + +impl TestTransientCacheFixture { + async fn new(origin_url: String) -> Self { + let now_ms = tuliprox_core::utils::current_time_millis(); + let resource = TransientResourceRef::new( + TransientResourceKind::Segment, + origin_url, + b"rewrite-secret", + now_ms, + 60_000, + Some("ts".to_string()), + ); + let resource_file = TransientResourceFile { resource_id: resource.id.clone(), extension: "ts".to_string() }; + let store = HlsSessionStore::new(); + let session = store.get_or_create_session(HlsSessionKey::new(1, "1"), b"rewrite-secret", 10).await; + let (proxy_session_id, log_identity) = { + let session = session.read().await; + (session.proxy_session_id.clone(), HlsLogIdentity::from_session(&session)) + }; + let lookup_key = TransientPassthroughState::transient_object_key( + &proxy_session_id, + &resource.id, + resource_file.extension.clone(), + ); + let (resource, fetch_token) = { + let mut session = session.write().await; + session.transient.upsert_resources([resource]); + let resource = session + .transient + .resources + .get(&resource_file.resource_id) + .expect("registered transient resource") + .clone(); + match session.transient.begin_object_fetch(&proxy_session_id, &resource, "ts", now_ms, 60_000) { + TransientObjectFetchDecision::Fetch(fetch_token) => (resource, *fetch_token), + TransientObjectFetchDecision::Ready | TransientObjectFetchDecision::Wait(_) => { + panic!("new transient resource should start a cache fetch") + } + } + }; + let cache_key = fetch_token.cache_key().clone(); + let temp_dir = tempfile::tempdir().expect("tempdir"); + let segment_cache = Arc::new(HlsSegmentCache::with_cache_path(temp_dir.path())); + let segment_repair = Arc::new(HlsSegmentRepairManager::new(HlsSegmentRepairConfig { + max_level: HlsSegmentRepairMode::Off, + apply_to_first_segments: 1, + max_parallel_repairs: 1, + ..Default::default() + })); + Self { + temp_dir, + segment_cache, + segment_repair, + session, + log_identity, + resource, + resource_file, + lookup_key, + cache_key, + fetch_token, + } + } + + fn request(&self, range_header: Option) -> HlsTransientOriginCacheFetchRequest { + HlsTransientOriginCacheFetchRequest { + fetch: HlsTransientOriginFetchRequest { + resolved_origin_uri: self.resource.resolved_origin_uri.clone(), + origin_headers: HeaderMap::new(), + origin_provider_session_headers: HeaderMap::new(), + range_header, + resource_file: self.resource_file.clone(), + resource_kind: self.resource.kind, + clients: HlsOriginResourceClients { + client: reqwest::Client::new(), + no_redirect_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("no-redirect client builds"), + use_manual_redirects: false, + }, + policy: SegmentFetchPolicy { + origin_segment_timeout_ms: 1_000, + retry_delays_ms: [0; 5], + retry_jitter_max_ms: 0, + ..SegmentFetchPolicy::default() + }, + log_identity: self.log_identity.clone(), + }, + commit: HlsTransientCacheCommitContext { + segment_cache: Arc::clone(&self.segment_cache), + segment_repair: Arc::clone(&self.segment_repair), + session: Arc::clone(&self.session), + proxy_session_id: ProxySessionId(String::from("test-proxy-session")), + log_identity: self.log_identity.clone(), + access_lease_id: HlsAccessLeaseId("transient-content-coding-test".to_string()), + resource: self.resource.clone(), + resource_file: self.resource_file.clone(), + fetch_token: self.fetch_token.clone(), + cache_duration_ms: 60_000, + }, + } + } +} + +struct DropCounter(Arc); + +impl Drop for DropCounter { + fn drop(&mut self) { self.0.fetch_add(1, Ordering::Relaxed); } +} + +mod cache; +mod decoding; +mod lifecycle; diff --git a/backend/hls/src/transient_fetcher/tests/cache.rs b/backend/hls/src/transient_fetcher/tests/cache.rs new file mode 100644 index 000000000..a8f86d4d5 --- /dev/null +++ b/backend/hls/src/transient_fetcher/tests/cache.rs @@ -0,0 +1,300 @@ +use super::*; + +#[test] +fn storage_full_transient_cache_failure_is_permanent() { + let error = HlsOriginResourceFetchError::cache_commit(&io::Error::from_raw_os_error(28)); + + assert!(matches!( + hls_transient_object_fetch_failure(&error), + HlsTransientObjectFetchFailure::Permanent { status: None } + )); +} + +#[tokio::test] +async fn raw_ttl_resource_requires_requesting_lease_manifest_membership_for_full_and_range() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "rolling-membership"), b"rewrite-secret", 0); + let proxy_session_id = session.proxy_session_id.clone(); + let authorized_lease_id = HlsAccessLeaseId("lease-a".to_string()); + let unpublished_lease_id = HlsAccessLeaseId("lease-b".to_string()); + let segment = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://origin.example.com/live/segment.ts", + b"rewrite-secret", + 0, + 300_000, + Some("ts".to_string()), + ); + let key = TransientResourceRef::new( + TransientResourceKind::Key, + "http://origin.example.com/live/key.bin", + b"rewrite-secret", + 0, + 300_000, + Some("key".to_string()), + ); + let resource_files = [ + TransientResourceFile { resource_id: segment.id.clone(), extension: "ts".to_string() }, + TransientResourceFile { resource_id: key.id.clone(), extension: "key".to_string() }, + ]; + let manifest_body = format!( + "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"/hls/shared/live/{}/lease-a/r/{}.key\"\n\ + #EXTINF:6,\n/hls/shared/live/{}/lease-a/r/{}.ts\n", + proxy_session_id.0, key.id.0, proxy_session_id.0, segment.id.0 + ); + let authorized_resources = HlsPublishedTransientResourceIds::from_manifest_body(&manifest_body); + let unpublished_resources = HlsPublishedTransientResourceIds::default(); + session.transient.upsert_resources([segment, key]); + let session = Arc::new(RwLock::new(session)); + let ranges = [None, Some(HeaderValue::from_static("bytes=0-")), Some(HeaderValue::from_static("bytes=1000-"))]; + + for resource_file in &resource_files { + for range in &ranges { + let rejected = resolve_hls_transient_object_cache_action( + &session, + &proxy_session_id, + HlsTransientResourceLeaseContext { + access_lease_id: &unpublished_lease_id, + lease_issued_at_ms: 0, + published_resource_ids: &unpublished_resources, + }, + resource_file, + range.as_ref(), + 1, + 300_000, + ) + .await; + assert!(matches!(rejected, Err(StatusCode::NOT_FOUND))); + + let accepted = resolve_hls_transient_object_cache_action( + &session, + &proxy_session_id, + HlsTransientResourceLeaseContext { + access_lease_id: &authorized_lease_id, + lease_issued_at_ms: 0, + published_resource_ids: &authorized_resources, + }, + resource_file, + range.as_ref(), + 1, + 300_000, + ) + .await; + assert!(accepted.is_ok()); + } + } +} + +#[tokio::test] +async fn finalized_resource_validity_is_identical_for_full_and_range_requests() { + let (session, proxy_session_id, access_lease_id, resource_file, published_resource_ids) = + finalized_resource_resolution_fixture(); + let full = resolve_hls_transient_object_cache_action( + &session, + &proxy_session_id, + HlsTransientResourceLeaseContext { + access_lease_id: &access_lease_id, + lease_issued_at_ms: 0, + published_resource_ids: &published_resource_ids, + }, + &resource_file, + None, + 301_520, + 300_000, + ) + .await + .expect("full request resolves finalized mapping"); + assert!(matches!(full.action, HlsTransientObjectCacheAction::FetchAndCache(_))); + + let zero_range = HeaderValue::from_static("bytes=0-"); + let from_zero = resolve_hls_transient_object_cache_action( + &session, + &proxy_session_id, + HlsTransientResourceLeaseContext { + access_lease_id: &access_lease_id, + lease_issued_at_ms: 0, + published_resource_ids: &published_resource_ids, + }, + &resource_file, + Some(&zero_range), + 301_520, + 300_000, + ) + .await + .expect("zero range resolves finalized mapping"); + assert!(matches!(from_zero.action, HlsTransientObjectCacheAction::WaitForFetch(_))); + + let offset_range = HeaderValue::from_static("bytes=1000-"); + let from_offset = resolve_hls_transient_object_cache_action( + &session, + &proxy_session_id, + HlsTransientResourceLeaseContext { + access_lease_id: &access_lease_id, + lease_issued_at_ms: 0, + published_resource_ids: &published_resource_ids, + }, + &resource_file, + Some(&offset_range), + 301_520, + 300_000, + ) + .await + .expect("offset range resolves finalized mapping"); + assert!(matches!(from_offset.action, HlsTransientObjectCacheAction::PassthroughNoCache)); +} + +#[tokio::test] +async fn removed_finalized_mapping_is_rejected_for_full_and_range_requests() { + let (session, proxy_session_id, access_lease_id, resource_file, published_resource_ids) = + finalized_resource_resolution_fixture(); + { + let mut session = session.write().await; + session.transient.replace_manifest_with_semantics( + "#EXTM3U\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-ENDLIST\n".to_string(), + 1, + None, + ); + assert!(session.transient.release_finalized_manifest_generations(&access_lease_id, 0)); + } + let ranges = [None, Some(HeaderValue::from_static("bytes=0-")), Some(HeaderValue::from_static("bytes=1000-"))]; + + for range in &ranges { + let result = resolve_hls_transient_object_cache_action( + &session, + &proxy_session_id, + HlsTransientResourceLeaseContext { + access_lease_id: &access_lease_id, + lease_issued_at_ms: 0, + published_resource_ids: &published_resource_ids, + }, + &resource_file, + range.as_ref(), + 301_520, + 300_000, + ) + .await; + assert!(matches!(result, Err(StatusCode::NOT_FOUND))); + } +} + +#[tokio::test] +async fn cacheable_transient_rejects_identity_partial_response_before_commit_and_cleans_guard() { + let origin = spawn_test_origin( + "206 Partial Content", + vec![("Content-Type", "video/mp2t"), ("Content-Range", "bytes 0-3/10")], + b"part".to_vec(), + ) + .await; + let fixture = TestTransientCacheFixture::new(format!("{}/partial.ts", origin.base_url)).await; + let dropped_guards = Arc::new(AtomicUsize::new(0)); + let dropped_guards_for_prepare = Arc::clone(&dropped_guards); + + let result = + fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(fixture.request(None), move |_| { + let dropped_guards = Arc::clone(&dropped_guards_for_prepare); + async move { Ok(DropCounter(dropped_guards)) }.boxed() + }) + .await; + + assert!(matches!(result, Err(HlsOriginResourceFetchError::UnexpectedByteRangeStatus))); + assert_eq!(dropped_guards.load(Ordering::Relaxed), 1); + assert!(fixture.segment_cache.metadata(&fixture.cache_key).await.expect("cache metadata reads").is_none()); + assert!(!fixture.segment_cache.has_active_temp_files()); + assert_eq!(std::fs::read_dir(fixture.temp_dir.path()).expect("cache root reads").count(), 0); + let session = fixture.session.read().await; + let entry = session.transient.object_cache.get(&fixture.lookup_key).expect("fetching cache entry remains"); + assert!(!matches!(entry.status, TransientObjectCacheStatus::Ready { .. })); + drop(session); + + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1); + let request = requests[0].to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity")); + assert!(!request.contains("\r\nrange:")); +} + +#[tokio::test] +async fn stale_resource_revision_commit_deletes_the_physical_fill_after_controlled_mapping_replacement() { + let origin = spawn_test_origin("200 OK", vec![("Content-Type", "video/mp2t")], b"stale-body".to_vec()).await; + let fixture = TestTransientCacheFixture::new(format!("{}/stale.ts", origin.base_url)).await; + { + let mut session = fixture.session.write().await; + let replacement_now_ms = tuliprox_core::utils::current_time_millis(); + let mut replacement = TransientResourceRef::new( + TransientResourceKind::Segment, + "http://replacement.example.com/live/replacement.ts", + b"rewrite-secret", + replacement_now_ms, + 60_000, + Some("ts".to_string()), + ); + replacement.id = fixture.resource.id.clone(); + session.transient.upsert_resources([replacement]); + } + + let result = fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(fixture.request(None), |_| { + async { Ok(()) }.boxed() + }) + .await; + + assert!( + matches!(result, Err(HlsOriginResourceFetchError::Superseded)), + "the superseded resource revision aborts without a retryable timeout" + ); + assert!(fixture.segment_cache.metadata(&fixture.cache_key).await.expect("cache metadata reads").is_none()); + assert!(!fixture.segment_cache.has_active_temp_files()); + assert!(!fixture.session.read().await.transient.object_cache.contains_key(&fixture.lookup_key)); + assert_eq!(origin.requests.lock().await.len(), 1); +} + +#[tokio::test] +async fn cacheable_transient_without_range_decodes_to_identity_before_cache_and_range_reads() { + let ciphertext = + vec![0x00, 0xff, 0x92, 0x10, 0x7e, 0x33, 0xc4, 0x81, 0xa8, 0x09, 0x5d, 0xe2, 0x44, 0x17, 0xb0, 0x6f]; + assert!(std::str::from_utf8(&ciphertext).is_err()); + let encoded = zstd_encode(&ciphertext).await; + assert_ne!(encoded, ciphertext); + let origin = spawn_zstd_origin(encoded).await; + let fixture = TestTransientCacheFixture::new(format!("{}/cipher.ts", origin.base_url)).await; + + fetch_and_commit_hls_transient_origin_response_with_attempt_prepare(fixture.request(None), |_| { + async { Ok(()) }.boxed() + }) + .await + .expect("encoded transient cache fetch succeeds"); + + let metadata = fixture + .segment_cache + .metadata(&fixture.cache_key) + .await + .expect("cache metadata reads") + .expect("decoded object is cached"); + assert_eq!(metadata.size, ciphertext.len() as u64); + assert_eq!(tokio::fs::read(&metadata.path).await.expect("cache body reads"), ciphertext); + let mut range = Vec::new(); + fixture + .segment_cache + .open_range(&fixture.cache_key, 5) + .await + .expect("decoded cache range opens") + .take(4) + .read_to_end(&mut range) + .await + .expect("decoded cache range reads"); + assert_eq!(range, ciphertext[5..9]); + + let session = fixture.session.read().await; + let entry = session.transient.object_cache.get(&fixture.lookup_key).expect("transient cache entry"); + assert!(matches!( + entry.status, + TransientObjectCacheStatus::Ready { content_length, .. } + if content_length == ciphertext.len() as u64 + )); + assert_eq!(entry.content_type, "application/octet-stream"); + drop(session); + + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1); + let request = requests[0].to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity")); + assert!(!request.contains("\r\nrange:")); +} diff --git a/backend/hls/src/transient_fetcher/tests/decoding.rs b/backend/hls/src/transient_fetcher/tests/decoding.rs new file mode 100644 index 000000000..df2be0a8b --- /dev/null +++ b/backend/hls/src/transient_fetcher/tests/decoding.rs @@ -0,0 +1,214 @@ +use super::*; + +#[tokio::test] +async fn direct_key_decodes_declared_gzip_brotli_and_zstd_to_exact_identity_bytes() { + let identity = b"\x00\xffdirect-key-identity\x10\x80"; + let encoded_bodies = [ + ("gzip", gzip_encode(identity).await), + ("br", brotli_encode(identity).await), + ("zstd", zstd_encode(identity).await), + ]; + + for (encoding, encoded) in encoded_bodies { + let origin = spawn_test_origin( + "200 OK", + vec![("Content-Encoding", encoding), ("Content-Type", "application/octet-stream")], + encoded, + ) + .await; + let decoded = fetch_direct_decoded(format!("{}/key.bin", origin.base_url), TransientResourceKind::Key, None) + .await + .expect("declared direct key coding decodes"); + let response = direct_client_response(decoded, TransientResourceKind::Key); + + assert!(!should_compress_response(&response)); + assert!(response.headers().get(header::CONTENT_ENCODING).is_none()); + assert!(response.headers().get(header::CONTENT_LENGTH).is_none()); + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.expect("decoded key body streams"), + identity.as_slice() + ); + + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1); + assert!(requests[0].to_ascii_lowercase().contains("accept-encoding: identity")); + } +} + +#[tokio::test] +async fn direct_map_and_part_or_other_resources_decode_to_identity() { + let identity = b"shared-direct-map-part-other"; + for (resource_kind, encoding, encoded) in [ + (TransientResourceKind::Map, "gzip", gzip_encode(identity).await), + (TransientResourceKind::Part, "br", brotli_encode(identity).await), + (TransientResourceKind::Other, "zstd", zstd_encode(identity).await), + ] { + let origin = spawn_test_origin( + "200 OK", + vec![("Content-Encoding", encoding), ("Content-Type", "application/octet-stream")], + encoded, + ) + .await; + let decoded = fetch_direct_decoded(format!("{}/object.bin", origin.base_url), resource_kind, None) + .await + .expect("declared direct resource coding decodes"); + let response = direct_client_response(decoded, resource_kind); + + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.expect("decoded resource body streams"), + identity.as_slice() + ); + assert_eq!(origin.requests.lock().await.len(), 1); + } +} + +#[tokio::test] +async fn direct_binary_declared_only_leaves_headerless_gzip_magic_unchanged() { + let identity = b"headerless-gzip-representation"; + let headerless_encoded = gzip_encode(identity).await; + let random_magic = vec![0x1f, 0x8b, 0x11, 0x00, 0xff, 0x42, 0x7e]; + + for body in [headerless_encoded, random_magic] { + let origin = + spawn_test_origin("200 OK", vec![("Content-Type", "application/octet-stream")], body.clone()).await; + let decoded = fetch_direct_decoded(format!("{}/key.bin", origin.base_url), TransientResourceKind::Key, None) + .await + .expect("headerless binary is not inspected"); + let response = direct_client_response(decoded, TransientResourceKind::Key); + + assert_eq!(to_bytes(response.into_body(), usize::MAX).await.expect("headerless binary streams"), body); + assert_eq!(origin.requests.lock().await.len(), 1); + } +} + +#[tokio::test] +async fn decoded_direct_response_normalizes_headers_and_disables_tower_compression() { + let identity = b"normalized-direct-response"; + let encoded = gzip_encode(identity).await; + let origin = spawn_test_origin( + "200 OK", + vec![ + ("Content-Encoding", "gzip"), + ("Content-Type", "application/octet-stream"), + ("Content-Range", "bytes 0-9/10"), + ("Accept-Ranges", "bytes"), + ("Cache-Control", "private, max-age=5"), + ("ETag", "strong-origin-representation"), + ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"), + ("Set-Cookie", "provider_session=secret"), + ("X-Provider-Secret", "do-not-forward"), + ], + encoded, + ) + .await; + let decoded = fetch_direct_decoded(format!("{}/key.bin", origin.base_url), TransientResourceKind::Key, None) + .await + .expect("direct response decodes"); + let response = direct_client_response(decoded, TransientResourceKind::Key); + let headers = response.headers(); + + assert!(!should_compress_response(&response)); + for removed in [ + header::CONTENT_ENCODING, + header::CONTENT_LENGTH, + header::CONTENT_RANGE, + header::ACCEPT_RANGES, + header::ETAG, + header::TRANSFER_ENCODING, + header::SET_COOKIE, + ] { + assert!(headers.get(&removed).is_none(), "{removed} must not describe the decoded response"); + } + assert!(headers.get("x-provider-secret").is_none()); + assert_eq!(headers.get(header::CONTENT_TYPE).expect("content type"), "application/octet-stream"); + assert_eq!(headers.get(header::CACHE_CONTROL).expect("cache control"), "private, max-age=5"); + assert_eq!(to_bytes(response.into_body(), usize::MAX).await.expect("normalized body streams"), identity.as_slice()); +} + +#[tokio::test] +async fn direct_encoded_partial_content_is_rejected_before_client_response() { + let encoded = zstd_encode(b"encoded-range").await; + let origin = spawn_test_origin( + "206 Partial Content", + vec![ + ("Content-Encoding", "zstd"), + ("Content-Type", "application/octet-stream"), + ("Content-Range", "bytes 2-5/16"), + ], + encoded, + ) + .await; + + let result = fetch_direct_decoded( + format!("{}/key.bin", origin.base_url), + TransientResourceKind::Key, + Some(HeaderValue::from_static("bytes=2-5")), + ) + .await; + + assert!(matches!(result, Err(HlsOriginResourceFetchError::ContentCoding(_)))); + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 1); + let request = requests[0].to_ascii_lowercase(); + assert!(request.contains("accept-encoding: identity")); + assert!(request.contains("range: bytes=2-5")); +} + +#[tokio::test] +async fn direct_identity_partial_content_preserves_consistent_range_headers() { + let identity = b"part"; + let origin = spawn_test_origin( + "206 Partial Content", + vec![ + ("Content-Type", "application/octet-stream"), + ("Content-Range", "bytes 2-5/16"), + ("Accept-Ranges", "bytes"), + ], + identity.to_vec(), + ) + .await; + let decoded = fetch_direct_decoded( + format!("{}/key.bin", origin.base_url), + TransientResourceKind::Key, + Some(HeaderValue::from_static("bytes=2-5")), + ) + .await + .expect("identity partial response is allowed"); + let response = direct_client_response(decoded, TransientResourceKind::Key); + + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers().get(header::CONTENT_LENGTH).expect("content length"), "4"); + assert_eq!(response.headers().get(header::CONTENT_RANGE).expect("content range"), "bytes 2-5/16"); + assert_eq!(response.headers().get(header::ACCEPT_RANGES).expect("accept ranges"), "bytes"); + assert!(!should_compress_response(&response)); + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.expect("identity partial body streams"), + identity.as_slice() + ); + assert_eq!(origin.requests.lock().await.len(), 1); +} + +#[tokio::test] +async fn direct_fetch_propagates_retry_attempt_and_identity_to_client_response() { + let identity = b"successful-second-attempt"; + let origin = + spawn_retry_then_body_origin(vec![("Content-Type", "application/octet-stream")], identity.to_vec()).await; + let origin_url = format!("{}/segment.ts", origin.base_url); + let decoded = fetch_direct_decoded(origin_url.clone(), TransientResourceKind::Segment, None) + .await + .expect("second origin attempt succeeds"); + + assert_eq!(decoded.attempt.attempt_index, 1); + assert_eq!(decoded.attempt.attempts, 5); + + let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url); + let response = fixture.response(decoded); + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.expect("retried response body streams"), + identity.as_slice() + ); + + let requests = origin.requests.lock().await; + assert_eq!(requests.len(), 2); + assert!(requests.iter().all(|request| request.to_ascii_lowercase().contains("accept-encoding: identity"))); +} diff --git a/backend/hls/src/transient_fetcher/tests/lifecycle.rs b/backend/hls/src/transient_fetcher/tests/lifecycle.rs new file mode 100644 index 000000000..e628a5335 --- /dev/null +++ b/backend/hls/src/transient_fetcher/tests/lifecycle.rs @@ -0,0 +1,361 @@ +use super::*; + +#[tokio::test] +async fn direct_segment_success_resets_failure_state_only_after_clean_eof() { + let identity = b"complete-segment"; + let origin = + spawn_test_origin("200 OK", vec![("Content-Type", "application/octet-stream")], identity.to_vec()).await; + let origin_url = format!("{}/segment.ts", origin.base_url); + let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url.clone()); + fixture.seed_segment_failure().await; + let decoded = fetch_direct_decoded(origin_url, TransientResourceKind::Segment, None) + .await + .expect("direct response setup succeeds"); + let response = fixture.response(decoded); + + assert_eq!(fixture.segment_failure_count().await, 1, "headers alone must not report body success"); + assert_eq!(fixture.resource.access.active_readers(), 1); + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.expect("complete segment streams"), + identity.as_slice() + ); + fixture.wait_for_segment_failure_count(0).await; + assert_eq!(fixture.resource.access.active_readers(), 0); +} + +#[tokio::test] +async fn direct_terminal_transition_survives_dropped_completion_waiter() { + let fixture = + TestDirectResponseFixture::new(TransientResourceKind::Segment, "http://127.0.0.1/cancelled-body-poll.ts"); + fixture.seed_segment_failure().await; + let session_lock = fixture.session.write().await; + let mut finalizer = HlsTransientDirectResponseFinalizer::new(fixture.lifecycle_context()); + + let transition = finalizer + .begin_finish(HlsTransientDirectStreamOutcome::CleanEof) + .expect("clean EOF starts an owned transition"); + drop(finalizer); + drop(transition); + drop(session_lock); + + fixture.wait_for_segment_failure_count(0).await; +} + +#[tokio::test] +async fn direct_segment_decoder_failure_after_retry_uses_selected_attempt_once() { + let _ = take_body_failure_log_attempts(); + let mut truncated = gzip_encode(b"decoder failure after response headers").await; + truncated.truncate(truncated.len().saturating_sub(8)); + let origin = spawn_retry_then_body_origin( + vec![("Content-Encoding", "gzip"), ("Content-Type", "application/octet-stream")], + truncated, + ) + .await; + let origin_url = format!("{}/segment.ts", origin.base_url); + let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url.clone()); + let decoded = fetch_direct_decoded(origin_url, TransientResourceKind::Segment, None) + .await + .expect("decoder setup succeeds before streaming"); + assert_eq!(decoded.attempt.attempt_index, 1); + assert_eq!(decoded.attempt.attempts, 5); + let response = fixture.response(decoded); + + assert!(to_bytes(response.into_body(), usize::MAX).await.is_err()); + fixture.wait_for_segment_failure_count(1).await; + assert_eq!( + fixture.last_segment_failure().await, + Some(HlsSegmentFailureObject::Transient { resource_id: fixture.resource.id.0.clone() }) + ); + tokio::task::yield_now().await; + assert_eq!(fixture.segment_failure_count().await, 1, "terminal body failure must not finalize twice"); + assert_eq!(fixture.resource.access.active_readers(), 0); + assert_eq!(origin.requests.lock().await.len(), 2, "streaming failure must not trigger a hidden retry"); + assert_eq!( + take_body_failure_log_attempts(), + vec![(1, 5)], + "test hook intentionally records the zero-based raw index of the selected second attempt" + ); +} + +#[tokio::test] +async fn unrelated_transient_objects_do_not_start_media_lifecycle_tasks() { + let kind = TransientResourceKind::Other; + for outcome in [HlsTransientDirectStreamOutcome::CleanEof, HlsTransientDirectStreamOutcome::OriginBodyFailure] { + let fixture = TestDirectResponseFixture::new(kind, "http://127.0.0.1/non-segment.bin"); + let mut finalizer = HlsTransientDirectResponseFinalizer::new(fixture.lifecycle_context()); + + assert!(finalizer.begin_finish(outcome).is_none(), "{kind:?} must not start a segment lifecycle task"); + } +} + +#[tokio::test] +async fn key_and_map_body_failures_degrade_readiness_without_incrementing_media_failure_count() { + for kind in [TransientResourceKind::Key, TransientResourceKind::Map] { + let fixture = TestDirectResponseFixture::new(kind, "http://127.0.0.1/media-dependency.bin"); + let mut finalizer = HlsTransientDirectResponseFinalizer::new(fixture.lifecycle_context()); + + finalizer + .begin_finish(HlsTransientDirectStreamOutcome::OriginBodyFailure) + .expect("media dependency starts a lifecycle task") + .await + .expect("media dependency lifecycle task joins"); + + let session = fixture.session.read().await; + assert_eq!(session.segment_failure_tracker.consecutive_temporary_failures, 0); + assert_eq!(session.origin_control.path_condition, HlsOriginPathCondition::SegmentReadinessFailure); + } +} + +#[tokio::test] +async fn direct_segment_origin_idle_timeout_is_counted_exactly_once() { + let origin = spawn_test_origin_in_chunks( + "200 OK", + vec![("Content-Type", "application/octet-stream")], + vec![b"first".to_vec(), b"late".to_vec()], + Duration::from_millis(200), + ) + .await; + let origin_url = format!("{}/segment.ts", origin.base_url); + let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url.clone()); + let decoded = fetch_direct_decoded_with_timeout(origin_url, TransientResourceKind::Segment, None, 50) + .await + .expect("direct response setup succeeds"); + let response = fixture.response(decoded); + + assert!(to_bytes(response.into_body(), usize::MAX).await.is_err()); + fixture.wait_for_segment_failure_count(1).await; + tokio::task::yield_now().await; + assert_eq!(fixture.segment_failure_count().await, 1); + assert_eq!(fixture.resource.access.active_readers(), 0); +} + +#[tokio::test] +async fn dropping_direct_segment_body_is_client_abort_without_state_transition() { + let _ = take_body_failure_log_attempts(); + let origin = + spawn_test_origin("200 OK", vec![("Content-Type", "application/octet-stream")], b"unconsumed-segment".to_vec()) + .await; + let origin_url = format!("{}/segment.ts", origin.base_url); + let fixture = TestDirectResponseFixture::new(TransientResourceKind::Segment, origin_url.clone()); + fixture.seed_segment_failure().await; + let decoded = fetch_direct_decoded(origin_url, TransientResourceKind::Segment, None) + .await + .expect("direct response setup succeeds"); + let response = fixture.response(decoded); + assert_eq!(fixture.resource.access.active_readers(), 1); + + drop(response); + + assert_eq!(fixture.segment_failure_count().await, 1); + assert_eq!(fixture.resource.access.active_readers(), 0); + assert_eq!(origin.requests.lock().await.len(), 1); + assert!(take_body_failure_log_attempts().is_empty()); +} + +#[tokio::test] +async fn direct_non_media_body_failures_are_failure_tracker_neutral() { + for kind in [TransientResourceKind::Key, TransientResourceKind::Map, TransientResourceKind::Other] { + let mut truncated = gzip_encode(b"non-segment decoder failure").await; + truncated.truncate(truncated.len().saturating_sub(8)); + let origin = spawn_test_origin( + "200 OK", + vec![("Content-Encoding", "gzip"), ("Content-Type", "application/octet-stream")], + truncated, + ) + .await; + let origin_url = format!("{}/object.bin", origin.base_url); + let fixture = TestDirectResponseFixture::new(kind, origin_url.clone()); + let decoded = + fetch_direct_decoded(origin_url, kind, None).await.expect("decoder setup succeeds before streaming"); + let response = fixture.response(decoded); + + assert!(to_bytes(response.into_body(), usize::MAX).await.is_err()); + assert_eq!(fixture.segment_failure_count().await, 0, "{kind:?} must not affect segment failure state"); + let path_condition = fixture.session.read().await.origin_control.path_condition; + if matches!(kind, TransientResourceKind::Key | TransientResourceKind::Map) { + assert_eq!(path_condition, HlsOriginPathCondition::SegmentReadinessFailure); + } else { + assert_eq!(path_condition, HlsOriginPathCondition::ProgressExpected); + } + } +} + +#[tokio::test] +async fn transient_media_failure_degrades_availability_without_direct_terminal_transition() { + for kind in [TransientResourceKind::Segment, TransientResourceKind::Part] { + let fixture = TestDirectResponseFixture::new(kind, "http://127.0.0.1/media.bin"); + + assert!( + !record_temporary_transient_segment_fetch_failure( + &fixture.session, + &fixture.resource, + &fixture.policy, + 100, + ) + .await + ); + + let session = fixture.session.read().await; + assert_eq!(session.segment_failure_tracker.consecutive_temporary_failures, 1); + assert_eq!(session.origin_control.path_condition, HlsOriginPathCondition::SegmentReadinessFailure); + assert!(session.origin_control.acceptance_episode.is_none()); + assert!(!matches!( + session.origin_control.progress_phase, + HlsOriginProgressPhase::Terminal | HlsOriginProgressPhase::TerminalPartial + )); + drop(session); + + super::super::record_successful_transient_segment_fetch(&fixture.session, &fixture.resource).await; + let session = fixture.session.read().await; + assert_eq!(session.segment_failure_tracker.consecutive_temporary_failures, 0); + assert_eq!(session.origin_control.path_condition, HlsOriginPathCondition::SegmentReadinessFailure); + } +} + +#[tokio::test] +async fn direct_stream_decoder_failure_aborts_body_without_origin_retry() { + let mut truncated = gzip_encode(b"decoder failure after response headers").await; + truncated.truncate(truncated.len().saturating_sub(8)); + let origin = spawn_test_origin( + "200 OK", + vec![("Content-Encoding", "gzip"), ("Content-Type", "application/octet-stream")], + truncated, + ) + .await; + let decoded = fetch_direct_decoded(format!("{}/key.bin", origin.base_url), TransientResourceKind::Key, None) + .await + .expect("decoder setup succeeds before streaming"); + let response = direct_client_response(decoded, TransientResourceKind::Key); + + assert!(to_bytes(response.into_body(), usize::MAX).await.is_err()); + assert_eq!(origin.requests.lock().await.len(), 1, "streaming errors must not trigger a hidden retry"); +} + +#[tokio::test] +async fn direct_stream_refreshes_origin_body_idle_deadline_after_each_chunk() { + const BODY_IDLE_TIMEOUT_MS: u64 = 500; + let chunks = [b"slow-".to_vec(), b"but-".to_vec(), b"continuous-".to_vec(), b"body".to_vec()]; + let expected = chunks.concat(); + let origin = spawn_test_origin_in_chunks( + "200 OK", + vec![("Content-Type", "application/octet-stream")], + chunks.into(), + Duration::from_millis(200), + ) + .await; + let decoded = fetch_direct_decoded_with_timeout( + format!("{}/key.bin", origin.base_url), + TransientResourceKind::Key, + None, + BODY_IDLE_TIMEOUT_MS, + ) + .await + .expect("direct response setup succeeds"); + let response = direct_client_response(decoded, TransientResourceKind::Key); + + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.expect("progressing body outlives total timeout"), + expected + ); + assert_eq!(origin.requests.lock().await.len(), 1); +} + +#[tokio::test] +async fn transient_origin_io_guard_finish_clean_decrements_work_once() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "clean-finish"), b"secret", 1_000); + let binding = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); + session.origin_account_io_lease = Some(std::sync::Arc::new(HlsOriginAccountIoLease::active_for_test(&binding, 2))); + session.origin_account_binding = Some(binding.clone()); + session.activity.active_origin_work_count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(2)); + let started_gen = session.activity.origin_work_generation; + let session = Arc::new(tokio::sync::RwLock::new(session)); + let ctx = crate::hls_ctx::HlsCtx::for_test(tuliprox_core::model::Config::default()); + + let lease_guard = HlsOriginAccountIoLeaseGuard::new( + binding.clone(), + Arc::clone(session.try_read().unwrap().origin_account_io_lease.as_ref().unwrap()), + ); + let origin_io = HlsOriginIoContext { + ctx: ctx.clone(), + client_addr: "127.0.0.1:8080".parse().unwrap(), + allow_grace: false, + priority: 0, + connection_kind: tuliprox_session::ConnectionKind::Normal, + reservation_ttl_secs: 60, + preacquired_provider_handle: None, + started_generation: Some(started_gen), + }; + + let guard = HlsTransientOriginIoGuard::new( + Arc::clone(&session), + Arc::clone(&session.try_read().unwrap().activity.active_origin_work_count), + origin_io, + lease_guard, + started_gen, + ); + + guard.finish_clean().await; + assert_eq!( + session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), + 1, + "finish_clean and subsequent drop must decrement active_origin_work_count exactly once" + ); +} + +#[tokio::test] +async fn transient_origin_io_guard_drop_under_lock_contention_decrements_work() { + let mut session = HlsSession::new(HlsSessionKey::new(1, "contention-drop"), b"secret", 1_000); + let binding = + HlsOriginAccountBinding::new(Arc::from("input"), Arc::from("account"), &session.proxy_session_id, 1_000); + session.origin_account_io_lease = Some(std::sync::Arc::new(HlsOriginAccountIoLease::active_for_test(&binding, 2))); + session.origin_account_binding = Some(binding.clone()); + session.activity.active_origin_work_count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(2)); + let started_gen = session.activity.origin_work_generation; + let session = Arc::new(tokio::sync::RwLock::new(session)); + let ctx = crate::hls_ctx::HlsCtx::for_test(tuliprox_core::model::Config::default()); + + let lease_guard = HlsOriginAccountIoLeaseGuard::new( + binding.clone(), + Arc::clone(session.try_read().unwrap().origin_account_io_lease.as_ref().unwrap()), + ); + let origin_io = HlsOriginIoContext { + ctx: ctx.clone(), + client_addr: "127.0.0.1:8080".parse().unwrap(), + allow_grace: false, + priority: 0, + connection_kind: tuliprox_session::ConnectionKind::Normal, + reservation_ttl_secs: 60, + preacquired_provider_handle: None, + started_generation: Some(started_gen), + }; + + let guard = HlsTransientOriginIoGuard::new( + Arc::clone(&session), + Arc::clone(&session.try_read().unwrap().activity.active_origin_work_count), + origin_io, + lease_guard, + started_gen, + ); + + let lock = session.read().await; + drop(guard); + assert_eq!( + lock.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), + 1, + "drop MUST synchronously decrement active_origin_work_count" + ); + drop(lock); + + for _ in 0..100 { + if session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire) == 1 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + assert_eq!( + session.read().await.activity.active_origin_work_count.load(std::sync::atomic::Ordering::Acquire), + 1, + "deferred cleanup must decrement active_origin_work_count under contention" + ); +} diff --git a/backend/library/src/ptt/handlers.rs b/backend/library/src/ptt/handlers.rs index 13b064461..cae92a272 100644 --- a/backend/library/src/ptt/handlers.rs +++ b/backend/library/src/ptt/handlers.rs @@ -1,12 +1,7 @@ use crate::ptt::{ models::PttMetadata, - parser::{handler_options, HandlerOptions, MatchInfo, ParseContext, PttParser}, - transformers::{ - boolean, date, first_uinteger, lowercase, none, range_i32, range_u32, transform_resolution, uinteger, - uppercase, value, - }, + parser::{handler_options, HandlerOptions, PttParser}, }; -use fancy_regex::Regex as FancyRegex; fn push_unique(values: &mut Vec, value: String) { if !values.contains(&value) { @@ -272,2908 +267,22 @@ fn parse_season_range(val: &str) -> Vec { nums } -#[allow(clippy::too_many_lines)] pub fn add_defaults(parser: &mut PttParser) { - parser.add_handler( - "tmdb", - FancyRegex::new(r"(?i)\btmdb\b[-=]\d+").unwrap(), - first_uinteger, - set_tmdb, - options_remove_skip_if_already_found(), - ); - parser.add_handler( - "tvdb", - FancyRegex::new(r"(?i)\btvdb\b[-=]\d+").unwrap(), - first_uinteger, - set_tvdb, - options_remove_skip_if_already_found(), - ); - - parser.add_handler( - "quality", - FancyRegex::new(r"(?i)\bPRE[- .]?HDRip\b").unwrap(), - boolean, - set_quality_to_scr_if_trash, - options_remove_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bE[- ]?Sub\b").unwrap(), - const_string("en"), - push_language, - options_remove_no_skip(), - ); - parser.add_handler( - "quality", - FancyRegex::new(r"(?i)\bTS-Screener\b").unwrap(), - boolean, - set_quality_to_tele_sync_if_trash, - options_remove_no_skip(), - ); - - parser.add_handler( - "year", - FancyRegex::new(r"\b19\d{2}\s?-\s?20\d{2}\b").unwrap(), - first_uinteger, - set_year, - options_keep(), - ); - - parser.add_handler( - "title_cleanup", - FancyRegex::new(r"(?i)\b(?:19|20)\d{2}\s*[-]\s*(?:(?:19|20)\d{2}|\d{2})\b").unwrap(), - none, - ignore, - options_remove(), - ); - - parser.add_handler( - "title_cleanup", - FancyRegex::new(r"(?i)\b100[ .-]*years?[ .-]*quest\b").unwrap(), - none, - ignore, - options_remove(), - ); - parser.add_handler( - "title_cleanup", - FancyRegex::new(r"(?i)\[?(\+.)?Extras\]?").unwrap(), - none, - ignore, - options_remove(), - ); - parser.add_handler( - "title_cleanup", - FancyRegex::new(r"(?i)(\+Movies)?\+Specials").unwrap(), - none, - ignore, - options_remove(), - ); - parser.add_handler( - "group", - FancyRegex::new(r"-?EDGE2020").unwrap(), - const_string("EDGE2020"), - set_group, - options_remove(), - ); - parser.add_handler("title_cleanup", FancyRegex::new(r"(?i)TV Money").unwrap(), none, ignore, options_remove()); - - parser.add_handler( - "container", - FancyRegex::new(r"(?i)\.?[\[(]?\b(MKV|AVI|MP4|WMV|MPG|MPEG)\b[\])]?").unwrap(), - lowercase, - set_container, - options_default(), - ); - - parser.add_handler("torrent", FancyRegex::new(r"\.torrent$").unwrap(), boolean, ignore, options_remove()); - - parser.add_handler("adult", FancyRegex::new(r"\b(XXX|xxx|Xxx)\b").unwrap(), boolean, set_adult, options_remove()); - - if let Ok(re) = FancyRegex::new(r"(?i)\b(18\+|adult|porn|xxx)\b") { - parser.add_handler("adult", re, boolean, mark_adult, options_default()); - } - - parser.add_handler( - "extras", - FancyRegex::new(r"(?i)\bOVA\b").unwrap(), - const_string("OVA"), - ignore, - options_remove(), - ); - - parser.add_handler( - "extras", - FancyRegex::new(r"(?i)\bOVA\b").unwrap(), - const_string("OVA"), - ignore, - options_remove(), - ); - - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)\[?\]?3840x\d{4}[\])?]?").unwrap(), - const_string("2160p"), - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)\[?\]?1920x\d{3,4}[\])?]?").unwrap(), - const_string("1080p"), - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)\[?\]?1280x\d{3}[\])?]?").unwrap(), - const_string("720p"), - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)\[?\]?(\d{3,4}x\d{3,4})[\])?]?p?").unwrap(), - append_p, - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)(480|720|1080)0[pi]").unwrap(), - append_p, - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)(?:QHD|QuadHD|WQHD|2560(\d+)?x(\d+)?1440p?)").unwrap(), - const_string("1440p"), - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)(?:Full HD|FHD|1920(\d+)?x(\d+)?1080p?)").unwrap(), - const_string("1080p"), - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)(?:BD|HD|M)(2160p?|4k)").unwrap(), - const_string("2160p"), - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)(?:BD|HD|M)1080p?").unwrap(), - const_string("1080p"), - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)(?:BD|HD|M)720p?").unwrap(), - const_string("720p"), - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)(?:BD|HD|M)480p?").unwrap(), - const_string("480p"), - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)\b(?:4k|2160p|1080p|720p|480p)(?!.*\b(?:4k|2160p|1080p|720p|480p)\b)").unwrap(), - transform_resolution, - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)\b4k|21600?[pi]\b").unwrap(), - const_string("2160p"), - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)(\d{3,4}[pi])").unwrap(), - lowercase, - set_resolution, - options_remove(), - ); - parser.add_handler( - "resolution", - FancyRegex::new(r"(?i)(240|360|480|576|720|1080|2160|3840)[pi]").unwrap(), - lowercase, - set_resolution, - options_remove(), - ); - - parser.add_handler( - "episode_code", - FancyRegex::new(r"[\[\()]([A-Fa-f0-9]{8})[\]\)]").unwrap(), - uppercase, - set_episode_code, - options_remove(), - ); - parser.add_handler( - "episode_code", - FancyRegex::new(r"[\[\()]([0-9]{8})[\]\)]").unwrap(), - uppercase, - set_episode_code, - options_remove_skip_if_already_found(), - ); - - parser.add_handler( - "trash", - FancyRegex::new( - r"(?i)\b(?:H[DQ][ .-]*)?(?().unwrap()), - set_year, - options_keep(), - ); - parser.add_handler( - "channels", - FancyRegex::new(r"(?i)\b(?:x[2-4]|5[\W]1(?:x[2-4])?)\b").unwrap(), - const_string("5.1"), - push_channels, - options_remove_no_skip(), - ); - parser.add_handler( - "channels", - FancyRegex::new(r"(?i)\b7[\.\- ]1(.?ch(annel)?)?\b").unwrap(), - const_string("7.1"), - push_channels, - options_remove_no_skip(), - ); - parser.add_handler( - "channels", - FancyRegex::new(r"(?i)\+?2[\.\s]0(?:x[2-4])?\b").unwrap(), - const_string("2.0"), - push_channels, - options_remove_no_skip(), - ); - - parser.add_handler( - "audio", - FancyRegex::new(r"(?i)\b(?!.+HR)(DTS.?HD.?Ma(ster)?|DTS.?X)\b").unwrap(), - const_string("DTS Lossless"), - push_audio, - options_remove_no_skip(), - ); - parser.add_handler( - "audio", - FancyRegex::new(r"(?i)\bDTS(?!(.?HD.?Ma(ster)?|.X)).?(HD.?HR|HD)?\b").unwrap(), - const_string("DTS Lossy"), - push_audio, - options_remove_no_skip(), - ); - parser.add_handler( - "audio", - FancyRegex::new(r"(?i)\b(Dolby.?)?Atmos\b").unwrap(), - const_string("Atmos"), - push_audio, - options_remove_no_skip(), - ); - parser.add_handler( - "audio", - FancyRegex::new(r"(?i)\b(True[ .-]?HD|\.True\.)\b").unwrap(), - const_string("TrueHD"), - push_audio, - handler_options! { - remove: true, - skip_if_already_found: false, - skip_from_title: true, - ..Default::default() - }, - ); - parser.add_handler( - "audio", - FancyRegex::new(r"(?i)\bTRUE\b").unwrap(), - const_string("TrueHD"), - push_audio, - handler_options! { - remove: true, - skip_if_already_found: false, - skip_from_title: true, - ..Default::default() - }, - ); - parser.add_handler( - "audio", - FancyRegex::new(r"(?i)\bFLAC(?:\d+(?:\.\d+)?)?(?:x\d+)?").unwrap(), - const_string("FLAC"), - push_audio, - options_remove_no_skip(), - ); - parser.add_handler( - "audio", - FancyRegex::new(r"(?i)DD2?[\+p]|DD Plus|Dolby Digital Plus|DDP(5[ \.\_]1)?|E-?AC-?3(?:-S\d+)?").unwrap(), - const_string("Dolby Digital Plus"), - push_audio, - options_remove_no_skip(), - ); - parser.add_handler( - "audio", - FancyRegex::new(r"(?i)\bddp(5.1)?").unwrap(), - const_string("Dolby Digital Plus"), - push_audio, - options_remove_skip_if_already_found(), - ); - parser.add_handler( - "audio", - FancyRegex::new(r"(?i)\bMP3\b").unwrap(), - const_string("MP3"), - push_audio, - options_remove_no_skip(), - ); - parser.add_handler( - "audio", - FancyRegex::new(r"(?i)\b(DD|Dolby.?Digital|DolbyD|AC-?3(x2)?(?:-S\d+)?)\b").unwrap(), - const_string("Dolby Digital"), - push_audio, - options_remove_no_skip(), - ); - parser.add_handler( - "audio", - FancyRegex::new(r"(?i)\bQ?Q?AAC(x?2)?\b").unwrap(), - const_string("AAC"), - push_audio, - options_remove_no_skip(), - ); - - parser.add_handler( - "group", - FancyRegex::new(r"(?i)- ?(?!\d+$|S\d+|\d+x|ep?\d+|[^\[]+]$)([^\-. \[]+[^\-. \[)\]\d][^\-. \[)\]]*)(?:\[[\w.-]+])?(?=\.\w{2,4}$|$)").unwrap(), - value, - set_group, - options_keep(), - ); - parser.add_handler( - "group", - FancyRegex::new(r"\(([\w-]+)\)(?:$|\.\w{2,4}$)").unwrap(), - value, - set_group, - options_default(), - ); - parser.add_handler("group", FancyRegex::new(r"^\[([^\[\]]+)\]").unwrap(), value, set_group, options_default()); - - parser.add_handler( - "volumes", - FancyRegex::new(r"(?i)\bvol(?:s|umes?)?[. -]*(?:\d{1,2}[., +/\\&-]+)+\d{1,2}\b").unwrap(), - range_i32, - set_volumes_if_present, - options_remove(), - ); - let volume_regex = FancyRegex::new(r"(?i)\bvol(?:ume)?[. -]*(\d{1,2})\b").unwrap(); - parser.add_handler_fn( - "volumes", - Box::new(move |context: &mut ParseContext| -> Option { - let title = &context.title; - let matched = &context.matched; - - let start_index = matched.get("year").map_or(0, |m| { - let mut idx = m.match_index.min(title.len()); - while idx > 0 && !title.is_char_boundary(idx) { - idx -= 1; - } - idx - }); - - if start_index >= title.len() { - return None; - } - - let search_slice = &title[start_index..]; - - if let Ok(Some(m)) = volume_regex.find(search_slice) { - let raw_match = m.as_str().to_string(); - let relative_start = m.start(); - - if let Ok(Some(cap)) = volume_regex.captures(search_slice) { - let volume_number = cap.get(1).map_or(0, |m| m.as_str().parse::().unwrap_or(0)); - - context.result.volumes = vec![volume_number]; - } - - let abs_start = start_index + relative_start; - - let info = MatchInfo { raw_match, match_index: abs_start, remove: true, skip_from_title: false }; - - context.matched.insert("volumes".to_string(), info.clone()); - return Some(info); - } - None - }), - ); - - parser.add_handler( - "seasons", - FancyRegex::new(r"(?i)(?:complete\W|seasons?\W|\W|^)((?:s\d{1,2}[., +/\\&-]+)+s\d{1,2}\b)").unwrap(), - parse_season_range, - extend_seasons, - options_remove(), - ); - parser.add_handler( - "seasons", - FancyRegex::new(r"(?i)(?:complete\W|seasons?\W|\W|^)[(\[]?(s\d{2,}-\d{2,}\b)[)\]]?").unwrap(), - parse_season_range, - extend_seasons, - options_remove(), - ); - parser.add_handler( - "seasons", - FancyRegex::new(r"(?i)(?:complete\W|seasons?\W|\W|^)[(\[]?(s[1-9]-[2-9])[)\]]?").unwrap(), - parse_season_range, - extend_seasons, - options_remove(), - ); - parser.add_handler( - "seasons", - FancyRegex::new(r"(?i)\d+ª(?:.+)?(?:a.?)?\d+ª(?:(?:.+)?(?:temporadas?))").unwrap(), - parse_season_range, - extend_seasons, - options_remove(), - ); - parser.add_handler( - "seasons", - FancyRegex::new(r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?(?:seasons?|[Сс]езони?|temporadas?)[. ]?[-:]?[. ]?[( \[]?((?:\d{1,2}[., /\\&]+)+\d{1,2}\b)[)\]]?").unwrap(), - parse_season_range, - extend_seasons, - options_remove(), - ); - parser.add_handler( - "seasons", - FancyRegex::new(r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?(?:seasons?|[Сс]езони?|temporadas?)[. ]?[-:]?[. ]?[( \[]?((?:\d{1,2}[.-]+)+[1-9]\d?\b)(?!\W*\d{4})[)\]]?").unwrap(), - parse_season_range, - extend_seasons, - options_remove(), - ); - parser.add_handler( - "seasons", - FancyRegex::new( - r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?season[. ]?[( \[]?((?:\d{1,2}[. -]+)+[1-9]\d?\b)[)\]]?(?!.*\.\w{2,4}$)", - ) - .unwrap(), - parse_season_range, - extend_seasons, - options_remove(), - ); - parser.add_handler( - "seasons", - FancyRegex::new( - r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?\bseasons?\b[. -]?(\d{1,2}[. -]?(?:to|thru|and|\+|:)[. -]?\d{1,2})\b", - ) - .unwrap(), - parse_season_range, - extend_seasons, - options_remove(), - ); - parser.add_handler( - "quality", - FancyRegex::new(r"(?i)\bDVB(?:\b|-)").unwrap(), - const_string("HDTV"), - set_quality, - options_remove(), - ); - parser.add_handler( - "seasons", - FancyRegex::new( - r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?(?:saison|seizoen|season|series|temp(?:orada)?):?[. ]?(\d{1,2})\b", - ) - .unwrap(), - |val| vec![val.parse().unwrap_or(0)], - extend_seasons, - options_remove(), - ); - parser.add_handler( - "seasons", - FancyRegex::new(r"(?i)(\d{1,2})(?:-?й)?[. _]?(?:[Сс]езон|sez(?:on)?)(?:\W?\D|$)").unwrap(), - |val| vec![val.parse().unwrap_or(0)], - extend_seasons, - options_remove(), - ); - parser.add_handler( - "seasons", - FancyRegex::new(r"(?i)[Сс]езон:?[. _]?№?(\d{1,2})(?!\d)").unwrap(), - |val| vec![val.parse().unwrap_or(0)], - extend_seasons, - options_remove(), - ); - parser.add_handler( - "seasons", - FancyRegex::new(r"(?i)(?:\D|^)(\d{1,2})Â?[°ºªa]?[. ]*temporada").unwrap(), - |val| vec![val.parse().unwrap_or(0)], - extend_seasons, - options_remove(), - ); - parser.add_handler( - "seasons", - FancyRegex::new(r"(?i)t(\d{1,3})(?:[ex]+|$)").unwrap(), - |val| vec![val.parse().unwrap_or(0)], - extend_seasons, - options_remove(), - ); - parser.add_handler( - "seasons", - FancyRegex::new(r"(?i)(?:(?:\bthe\W)?\bcomplete)?(?().unwrap_or(0)], - extend_seasons, - options_remove_skip_if_already_found(), - ); - - parser.add_handler( - "episodes", - FancyRegex::new( - r"(?i)(?:[\W\d]|^)e[ .]?[\[(]?(\d{1,3}(?:[ .-]*(?:[&+]|e|.){1,2}(?:[ .]*e)?[ .]?\d{1,3})+)(?:\W|$)", - ) - .unwrap(), - range_u32, - extend_episodes, - options_no_skip(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)(?:[\W\d]|^)ep[ .]?[\[(]?(\d{1,3}(?:[ .-]*(?:[&+]|ep){1,2}[ .]?\d{1,3})+)(?:\W|$)") - .unwrap(), - range_u32, - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)(?:[\W\d]|^)\d+[xх][ .]?[\[(]?(\d{1,3}(?:[ .]?[xх][ .]?\d{1,3})+)(?:\W|$)").unwrap(), - range_u32, - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)Серии:\s+(\d+)\s+(?:of|из|iz)\s+\d+\b").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new( - r"(?i)(?:[\W\d]|^)(?:episodes?|[Сс]ерии:?)[ .]?[\[(]?(\d{1,3}(?:[ .+]*[&+][ .]?\d{1,3})+)(?:\W|$)", - ) - .unwrap(), - range_u32, - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)[\[(]?(?:\D|^)(\d{1,3}[ .]?ao[ .]?\d{1,3})[)\]]?(?:\W|$)").unwrap(), - range_u32, - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new( - r"(?i)(?:[\W\d]|^)(?:e|eps?|episodes?|[Сс]ерии:?|\d+[xх])[ .]*[\[(]?(\d{1,3}(?:-\d{1,3})+)(?:\W|$)", - ) - .unwrap(), - range_u32, - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)(?:\W|^)(\d{1,3}(?:[ .]*~[ .]*\d{1,3})+)(?:\W|$)").unwrap(), - range_u32, - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)\bE\d{1,4}\s*à\s*E\d{1,4}\b").unwrap(), - range_u32, - extend_episodes, - options_remove_skip_if_already_found(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)[st]\d{1,2}[. ]?[xх-]?[. ]?(?:e|x|х|ep|-|\.)[. ]?(\d{1,4})(?:[abc]|v0?[1-4]|\D|$)") - .unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_remove(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)\b[st]\d{2}(\d{2})\b").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)-\s(\d{1,3}[ .]*-[ .]*\d{1,3})(?!-\d)(?:\W|$)").unwrap(), - range_u32, - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)s\d{1,2}\s?\((\d{1,3}[ .]*-[ .]*\d{1,3})\)").unwrap(), - range_u32, - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?:^|/)\d{1,2}-(\d{2})\b(?!-\d)").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)(?<=^\[.+].+)[. ]+-[. ]+(\d{1,4})[. ]+(?=\W)").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_remove(), - ); - parser.add_handler( - "episodes", - FancyRegex::new( - r"(?i)(?().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)Ep.\d+.-.\\d+").unwrap(), - range_u32, - extend_episodes, - options_remove(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)(?:\b[ée]p?(?:isode)?|[Ээ]пизод|[Сс]ер(?:ии|ия|\.)?|cap(?:itulo)?|epis[oó]dio)[. ]?[-:#№]?[. ]?(\d{1,4})(?:[abc]|v0?[1-4]|\W|$)").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)\b(\d{1,3})(?:-?я)?[ ._-]*(?:ser(?:i?[iyja]|\b)|[Сс]ер(?:ии|ия|\.)?)").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?:\D|^)\d{1,2}[. ]?[xх][. ]?(\d{1,3})(?:[abc]|v0?[1-4]|\D|$)").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)(?<=S\d{2}E)(\d+)").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"[\[(]\d{1,2}\.(\d{1,3})[)\]]").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"\b[Ss]\d{1,2}[ .](\d{1,2})\b").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"-\s?\d{1,2}\.(\d{2,3})\s?-").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)(?:\[|\()(\d+)\s(?:of|из|iz)\s\d+(?:\]|\))").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)(?<=\D|^)(\d{1,3})[. ]?(?:of|из|iz)[. ]?\d{1,3}(?=\D|$)").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"\b\d{2}[ ._-](\d{2})(?:.F)?\.\\w{2,4}$").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)(\d+)(?=.?\[([A-Z0-9]{8})\])").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_default(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)(?().unwrap_or(0)]), - extend_episodes, - options_remove_skip_if_already_found(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)(?:\W|^)(?:\d+)?(?:e|ep)(\d{1,3})(?:\W|$)").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - options_remove(), - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)E(\d+)\b").unwrap(), - |val| Some(vec![val.parse::().unwrap_or(0)]), - extend_episodes, - handler_options! { - remove: false, - skip_if_already_found: true, - ..Default::default() - }, - ); - parser.add_handler( - "episodes", - FancyRegex::new(r"(?i)\b(\d{1,4})-(\d{1,4})\b").unwrap(), - range_u32, - extend_episodes, - handler_options! { - remove: false, - skip_if_already_found: true, - ..Default::default() - }, - ); - - parser.add_handler( - "country", - FancyRegex::new(r"\b(US|UK|AU|NZ|CA)\b").unwrap(), - value, - set_country, - options_default(), - ); - - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bengl?(?:sub[A-Z]*)?\b").unwrap(), - const_string("en"), - push_language, - options_remove_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bEnglish[\. _-]*(?:subs?|sdh|hi)\b").unwrap(), - const_string("en"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(?:ingl[eéê]s|inglese?)\b").unwrap(), - const_string("en"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\[En\b").unwrap(), - const_string("en"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bIT\s+EN\b").unwrap(), - const_string("it"), - push_language_and_en, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bEng(?:,|\s)").unwrap(), - const_string("en"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bCze(?:ch)?\b").unwrap(), - const_string("cs"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bGer(?:,|\s|\b)").unwrap(), - const_string("de"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\[spanish\]").unwrap(), - const_string("es"), - push_language, - options_remove_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(?:español|espanhol)\b").unwrap(), - const_string("es"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bFR(?:a|e|anc[eê]s|VF[FQIB2]?)\b").unwrap(), - const_string("fr"), - push_language, - options_skip_from_title_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b\[?(VF[FQRIB2]?\]?\b|(VOST)?FR2?)\b").unwrap(), - const_string("fr"), - push_language, - options_remove_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(?:GERMAN|GER)\b|(?-i)\bDE\b").unwrap(), - const_string("de"), - push_language, - options_remove_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(TRUE|SUB).?FRENCH\b|\bFRENCH\b|\bFre?\b").unwrap(), - const_string("fr"), - push_language, - handler_options! { - remove: true, - skip_if_first: true, - skip_if_already_found: false, - ..Default::default() - }, - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(VOST(?:FR?|A)?)\b").unwrap(), - const_string("fr"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(VF[FQIB2]?|(TRUE|SUB).?FRENCH|(VOST)?FR2?)\b").unwrap(), - const_string("fr"), - push_language, - options_remove_skip_if_already_found(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bspanish\W?latin|american\W*(?:spa|esp?)").unwrap(), - const_string("la"), - push_language, - options_remove_skip_from_title_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(?:\bla\b.+(?:cia\b))").unwrap(), - const_string("es"), - push_language, - options_skip_from_title_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(?:audio.)?lat(?:in?|ino)?\b").unwrap(), - const_string("la"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(?:audio.)?(?:ESP?|spa|(en[ .]+)?espa[nñ]ola?|castellano)\b").unwrap(), - const_string("es"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bes(?=[ .,/-]+(?:[A-Z]{2}[ .,/-]+){2,})\b").unwrap(), - const_string("es"), - push_language, - options_skip_from_title_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(?<=[ .,/-]+(?:[A-Z]{2}[ .,/-]+){2,})es\b").unwrap(), - const_string("es"), - push_language, - options_skip_from_title_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(?<=[ .,/-]+[A-Z]{2}[ .,/-]+)es(?=[ .,/-]+[A-Z]{2}[ .,/-]+)\b").unwrap(), - const_string("es"), - push_language, - options_skip_from_title_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bes(?=\.(?:ass|ssa|srt|sub|idx)$)").unwrap(), - const_string("es"), - push_language, - options_skip_from_title_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(temporadas?|completa)\b").unwrap(), - const_string("es"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(?:INT[EÉ]GRALE?)\b").unwrap(), - const_string("fr"), - push_language, - handler_options! { - remove: false, - skip_if_already_found: false, - ..Default::default() - }, - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(?:Saison)\b").unwrap(), - const_string("fr"), - push_language, - handler_options! { - remove: false, - skip_if_already_found: false, - ..Default::default() - }, - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(?:p[rt]|en|port)[. (\\/-]*BR\b").unwrap(), - const_string("pt"), - push_language, - handler_options! { - skip_if_already_found: false, - remove: true, - ..Default::default() - }, - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bbr(?:a|azil|azilian)\W+(?:pt|por)\b").unwrap(), - const_string("pt"), - push_language, - handler_options! { - skip_if_already_found: false, - remove: true, - ..Default::default() - }, - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(?:leg(?:endado|endas?)?|dub(?:lado)?|portugu[eèê]se?)[. -]*BR\b").unwrap(), - const_string("pt"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bleg(?:endado|endas?)\b").unwrap(), - const_string("pt"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bportugu[eèê]s[ea]?\b").unwrap(), - const_string("pt"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bPT[. -]*(?:PT|ENG?|sub(?:s|titles?))\b").unwrap(), - const_string("pt"), - push_language, - options_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bpt(?=\.(?:ass|ssa|srt|sub|idx)$)").unwrap(), - const_string("pt"), - push_language, - options_skip_from_title_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bPT\b").unwrap(), - const_string("pt"), - push_language, - options_remove_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\bpor\b").unwrap(), - const_string("pt"), - push_language, - options_skip_from_title_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b-?ITA\b").unwrap(), - const_string("it"), - push_language, - options_remove_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(? Option { - let title = &context.title; - let matched = &context.matched; - - let start_index = matched.get("year").map_or(0, |m| { - let mut idx = m.match_index.min(title.len()); - while idx > 0 && !title.is_char_boundary(idx) { - idx -= 1; - } - idx - }); - - if start_index >= title.len() { - return None; - } - - let search_slice = &title[start_index..]; - - if let Ok(Some(m)) = volume_regex.find(search_slice) { - let raw_match = m.as_str().to_string(); - let relative_start = m.start(); - - if let Ok(Some(cap)) = volume_regex.captures(search_slice) { - let volume_number = cap.get(1).map_or(0, |m| m.as_str().parse::().unwrap_or(0)); - - context.result.volumes = vec![volume_number]; - } - - let abs_start = start_index + relative_start; - - let info = MatchInfo { raw_match, match_index: abs_start, remove: true, skip_from_title: false }; - - context.matched.insert("volumes".to_string(), info.clone()); - return Some(info); - } - None - }), - ); - let ep_regex = FancyRegex::new(r"\b\d{1,4}\b").unwrap(); - - parser.add_handler_fn( - "episodes", - Box::new(move |context: &mut ParseContext| -> Option { - if context.matched.contains_key("episodes") { - return None; - } - - let title = &context.title; - - if anime_regex.is_match(title).unwrap_or(false) { - if let Ok(Some(m)) = ep_regex.find(title) { - let raw_match = m.as_str().to_string(); - let val = raw_match.parse::().unwrap_or(0); - - context.result.episodes.push(val); - - let info = MatchInfo { raw_match, match_index: m.start(), remove: true, skip_from_title: true }; - context.matched.insert("episodes".to_string(), info.clone()); - return Some(info); - } - } - None - }), - ); - - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(PLDUB|PLSUB|DUBPL|DubbingPL|LekPL|LektorPL)\b").unwrap(), - const_string("pl"), - push_language, - options_remove_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(PLDUB|PLSUB|DUBPL|DubbingPL|LekPL|LektorPL)\b").unwrap(), - const_string("pl"), - push_language, - options_remove_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(PLDUB|PLSUB|DUBPL|DubbingPL|LekPL|LektorPL)\b").unwrap(), - const_string("pl"), - push_language, - options_remove_no_skip(), - ); - parser.add_handler( - "languages", - FancyRegex::new(r"(?i)\b(?:(? Option { + let title = &context.title; + let matched = &context.matched; + + let start_index = matched.get("year").map_or(0, |m| { + let mut idx = m.match_index.min(title.len()); + while idx > 0 && !title.is_char_boundary(idx) { + idx -= 1; + } + idx + }); + + if start_index >= title.len() { + return None; + } + + let search_slice = &title[start_index..]; + + if let Ok(Some(m)) = volume_regex.find(search_slice) { + let raw_match = m.as_str().to_string(); + let relative_start = m.start(); + + if let Ok(Some(cap)) = volume_regex.captures(search_slice) { + let volume_number = cap.get(1).map_or(0, |m| m.as_str().parse::().unwrap_or(0)); + + context.result.volumes = vec![volume_number]; + } + + let abs_start = start_index + relative_start; + + let info = MatchInfo { raw_match, match_index: abs_start, remove: true, skip_from_title: false }; + + context.matched.insert("volumes".to_string(), info.clone()); + return Some(info); + } + None + }), + ); + + let ep_regex = FancyRegex::new(r"\b\d{1,4}\b").unwrap(); + + parser.add_handler_fn( + "episodes", + Box::new(move |context: &mut ParseContext| -> Option { + if context.matched.contains_key("episodes") { + return None; + } + + let title = &context.title; + + if anime_regex.is_match(title).unwrap_or(false) { + if let Ok(Some(m)) = ep_regex.find(title) { + let raw_match = m.as_str().to_string(); + let val = raw_match.parse::().unwrap_or(0); + + context.result.episodes.push(val); + + let info = MatchInfo { raw_match, match_index: m.start(), remove: true, skip_from_title: true }; + context.matched.insert("episodes".to_string(), info.clone()); + return Some(info); + } + } + None + }), + ); +} diff --git a/backend/library/src/ptt/handlers/dates_release.rs b/backend/library/src/ptt/handlers/dates_release.rs new file mode 100644 index 000000000..14a236df3 --- /dev/null +++ b/backend/library/src/ptt/handlers/dates_release.rs @@ -0,0 +1,415 @@ +use super::{ + const_string, options_default, options_keep, options_no_skip, options_remove, options_remove_no_skip, + options_remove_skip_if_already_found, options_skip_from_title, push_episode, push_season, set_bitrate, + set_commentary, set_complete, set_convert, set_date, set_documentary, set_edition, set_hardcoded, set_proper, + set_quality, set_region, set_remastered, set_repack, set_retail, set_trash, set_uncensored, set_unrated, + set_upscaled, set_year, set_year_with_trace, +}; +use crate::ptt::{ + parser::{handler_options, PttParser}, + transformers::{boolean, date, lowercase, uinteger, uppercase}, +}; +use fancy_regex::Regex as FancyRegex; + +#[allow(clippy::too_many_lines)] +pub(super) fn register(parser: &mut PttParser) { + // parser.add_handler( + // "trash", + // FancyRegex::new(r"(?i)\bHDTV(?:Rip)?\b").unwrap(), + // boolean, + // |meta, val| { println!("TRASH MATCH HDTV: {}", val); meta.trash = val; }, + // handler_options! { remove: false, ..Default::default() } + // ); + + parser.add_handler( + "date", + FancyRegex::new(r"(?:\W|^)([\[(]?(?:19[6-9]|20[012])[0-9]([. \-/\\])(?:0[1-9]|1[012])\2(?:0[1-9]|[12][0-9]|3[01])[\])]?)(?:\W|$)").unwrap(), + |val| date(val, &["%Y-%m-%d", "%Y.%m.%d", "%Y %m %d"]).unwrap_or_default(), + set_date, + options_remove(), + ); + + parser.add_handler( + "date", + FancyRegex::new(r"(?:\W|^)(\[?\]?(?:0[1-9]|[12][0-9]|3[01])([. \-/\\])(?:0[1-9]|1[012])\2(?:19[6-9]|20[01])[0-9][\])]?)(?:\W|$)").unwrap(), + |val| date(val, &["%d-%m-%Y", "%d.%m.%Y", "%d %m %Y"]).unwrap_or_default(), + set_date, + options_remove(), + ); + + parser.add_handler( + "date", + FancyRegex::new(r"(?:\W)(\[?\]?(?:0[1-9]|1[012])([. \-/\\])(?:0[1-9]|[12][0-9]|3[01])\2(?:[0][1-9]|[0126789][0-9])[\])]?)(?:\W|$)").unwrap(), + |val| date(val, &["%m %d %y", "%m.%d.%y", "%m-%d-%y"]).unwrap_or_default(), + set_date, + options_remove(), + ); + + parser.add_handler( + "date", + FancyRegex::new(r"(?:\W)(\[?\]?(?:[0][1-9]|[12][0-9]|3[0-9])([. \-/\\])(?:0[1-9]|1[012])\2(?:0[1-9]|[12][0-9])[\])]?)(?:\W|$)").unwrap(), + |val| date(val, &["%y %m %d", "%y.%m.%d", "%y-%m-%d"]).unwrap_or_default(), + set_date, + options_remove(), + ); + + parser.add_handler( + "date", + FancyRegex::new(r"(?:\W)(\[?\]?(?:0[1-9]|[12][0-9]|3[01])([. \-/\\])(?:0[1-9]|1[012])\2(?:[0][1-9]|[0126789][0-9])[\])]?)(?:\W|$)").unwrap(), + |val| date(val, &["%d %m %y", "%d.%m.%y", "%d-%m-%y"]).unwrap_or_default(), + set_date, + options_remove(), + ); + + parser.add_handler( + "date", + FancyRegex::new(r"(?i)(?:\W|^)([(\[]?(?:0?[1-9]|[12][0-9]|3[01])[. ]?(?:st|nd|rd|th)?([. \-/\\])(?:feb(?:ruary)?|jan(?:uary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sept?(?:ember)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\2(?:19[7-9]|20[012])[0-9][)\]]?)(?=\W|$)").unwrap(), + |val| date(val, &["%d %b %Y", "%d %B %Y", "%d.%b.%Y", "%d.%B.%Y", "%d-%b-%Y", "%d-%B-%Y"]).unwrap_or_default(), + set_date, + options_remove(), + ); + + parser.add_handler( + "date", + FancyRegex::new(r"(?i)(?:\W|^)(\[?\]?(?:0?[1-9]|[12][0-9]|3[01])[. ]?(?:st|nd|rd|th)?([. \-\/\\])(?:feb(?:ruary)?|jan(?:uary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sept?(?:ember)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\2(?:0[1-9]|[0126789][0-9])[\])]?)(?:\W|$)").unwrap(), + |val| date(val, &["%d %b %y", "%d %B %y", "%d.%b.%y", "%d.%B.%y", "%d-%b-%y", "%d-%B-%y"]).unwrap_or_default(), + set_date, + options_remove(), + ); + + parser.add_handler( + "date", + FancyRegex::new(r"(?:\W|^)(\[?\]?20[012][0-9](?:0[1-9]|1[012])(?:0[1-9]|[12][0-9]|3[01])[\])]?)(?:\W|$)") + .unwrap(), + |val| date(val, &["%Y%m%d"]).unwrap_or_default(), + set_date, + options_remove(), + ); + + parser.add_handler( + "date", + FancyRegex::new(r"(?i)(?:\W|^)((?:0?[1-9]|[12][0-9]|3[01])(?:st|nd|rd|th)\s+(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|June?|July?|Aug(?:ust)?|Sept?(?:ember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+(?:19[7-9]|20[012])[0-9])(?=\W|$)").unwrap(), + |val| { + let clean = val.replace("st ", " ").replace("nd ", " ").replace("rd ", " ").replace("th ", " "); + date(&clean, &["%d %b %Y", "%d %B %Y"]).unwrap_or_default() + }, + set_date, + options_remove(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)\b((?:19\d|20[012])\d[ .]?-[ .]?(?:19\d|20[012])\d)\b").unwrap(), + boolean, + set_complete, + options_remove(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)[(\[][ .]?((?:19\d|20[012])\d[ .]?-[ .]?\d{2})[ .]?[)\]]").unwrap(), + boolean, + set_complete, + options_remove(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)\bcomplete\b").unwrap(), + boolean, + set_complete, + options_remove(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)\b(?:INTEGRALE?|INTÉGRALE?)\b").unwrap(), + boolean, + set_complete, + options_remove_no_skip(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)(Movie|Complete).Collection").unwrap(), + boolean, + set_complete, + options_remove_no_skip(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)Complete(.\d{1,2})").unwrap(), + boolean, + set_complete, + options_remove_no_skip(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)(?:\bthe\W)?(?:\bcomplete|collection|dvd)?\b[ .]?\bbox[ .-]?set\b").unwrap(), + boolean, + set_complete, + options_remove(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)(?:\bthe\W)?(?:\bcomplete|collection|dvd)?\b[ .]?\bmini[ .-]?series\b").unwrap(), + boolean, + set_complete, + options_default(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)(?:\bthe\W)?(?:\bcomplete\b|\bfull\b|\ball\b)\b.*\b(?:series|seasons|collection|episodes|set|pack|movies)\b").unwrap(), + boolean, + set_complete, + options_default(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)(Top\W+)?\d+\W+(movies?|series|seasons?)\W+Collection").unwrap(), + boolean, + set_complete, + options_remove(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)(?:\bthe\W)?\bultimate\b[ .]\bcollection\b").unwrap(), + boolean, + set_complete, + options_no_skip(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)\bcollection\b.*\b(?:set|pack|movies)\b").unwrap(), + boolean, + set_complete, + options_default(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)\bcollection(?:(\s\[|\s\())").unwrap(), + boolean, + set_complete, + options_remove(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)duology|trilogy|quadr[oi]logy|tetralogy|pentalogy|hexalogy|heptalogy|anthology").unwrap(), + boolean, + set_complete, + options_no_skip(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)\bcompleta\b").unwrap(), + boolean, + set_complete, + options_remove(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)\bsaga\b").unwrap(), + boolean, + set_complete, + options_skip_from_title(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)\b\[Complete\]\b").unwrap(), + boolean, + set_complete, + options_remove(), + ); + + parser.add_handler( + "complete", + FancyRegex::new(r"(?i)(?().unwrap()), + set_year, + options_keep(), + ); + + parser.add_handler( + "channels", + FancyRegex::new(r"(?i)\b(?:x[2-4]|5[\W]1(?:x[2-4])?)\b").unwrap(), + const_string("5.1"), + push_channels, + options_remove_no_skip(), + ); + + parser.add_handler( + "channels", + FancyRegex::new(r"(?i)\b7[\.\- ]1(.?ch(annel)?)?\b").unwrap(), + const_string("7.1"), + push_channels, + options_remove_no_skip(), + ); + + parser.add_handler( + "channels", + FancyRegex::new(r"(?i)\+?2[\.\s]0(?:x[2-4])?\b").unwrap(), + const_string("2.0"), + push_channels, + options_remove_no_skip(), + ); + + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\b(?!.+HR)(DTS.?HD.?Ma(ster)?|DTS.?X)\b").unwrap(), + const_string("DTS Lossless"), + push_audio, + options_remove_no_skip(), + ); + + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\bDTS(?!(.?HD.?Ma(ster)?|.X)).?(HD.?HR|HD)?\b").unwrap(), + const_string("DTS Lossy"), + push_audio, + options_remove_no_skip(), + ); + + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\b(Dolby.?)?Atmos\b").unwrap(), + const_string("Atmos"), + push_audio, + options_remove_no_skip(), + ); + + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\b(True[ .-]?HD|\.True\.)\b").unwrap(), + const_string("TrueHD"), + push_audio, + handler_options! { + remove: true, + skip_if_already_found: false, + skip_from_title: true, + ..Default::default() + }, + ); + + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\bTRUE\b").unwrap(), + const_string("TrueHD"), + push_audio, + handler_options! { + remove: true, + skip_if_already_found: false, + skip_from_title: true, + ..Default::default() + }, + ); + + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\bFLAC(?:\d+(?:\.\d+)?)?(?:x\d+)?").unwrap(), + const_string("FLAC"), + push_audio, + options_remove_no_skip(), + ); + + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)DD2?[\+p]|DD Plus|Dolby Digital Plus|DDP(5[ \.\_]1)?|E-?AC-?3(?:-S\d+)?").unwrap(), + const_string("Dolby Digital Plus"), + push_audio, + options_remove_no_skip(), + ); + + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\bddp(5.1)?").unwrap(), + const_string("Dolby Digital Plus"), + push_audio, + options_remove_skip_if_already_found(), + ); + + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\bMP3\b").unwrap(), + const_string("MP3"), + push_audio, + options_remove_no_skip(), + ); + + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\b(DD|Dolby.?Digital|DolbyD|AC-?3(x2)?(?:-S\d+)?)\b").unwrap(), + const_string("Dolby Digital"), + push_audio, + options_remove_no_skip(), + ); + + parser.add_handler( + "audio", + FancyRegex::new(r"(?i)\bQ?Q?AAC(x?2)?\b").unwrap(), + const_string("AAC"), + push_audio, + options_remove_no_skip(), + ); + + parser.add_handler( + "group", + FancyRegex::new(r"(?i)- ?(?!\d+$|S\d+|\d+x|ep?\d+|[^\[]+]$)([^\-. \[]+[^\-. \[)\]\d][^\-. \[)\]]*)(?:\[[\w.-]+])?(?=\.\w{2,4}$|$)").unwrap(), + value, + set_group, + options_keep(), + ); + + parser.add_handler( + "group", + FancyRegex::new(r"\(([\w-]+)\)(?:$|\.\w{2,4}$)").unwrap(), + value, + set_group, + options_default(), + ); + + parser.add_handler("group", FancyRegex::new(r"^\[([^\[\]]+)\]").unwrap(), value, set_group, options_default()); + + parser.add_handler( + "volumes", + FancyRegex::new(r"(?i)\bvol(?:s|umes?)?[. -]*(?:\d{1,2}[., +/\\&-]+)+\d{1,2}\b").unwrap(), + range_i32, + set_volumes_if_present, + options_remove(), + ); + + let volume_regex = FancyRegex::new(r"(?i)\bvol(?:ume)?[. -]*(\d{1,2})\b").unwrap(); + + parser.add_handler_fn( + "volumes", + Box::new(move |context: &mut ParseContext| -> Option { + let title = &context.title; + let matched = &context.matched; + + let start_index = matched.get("year").map_or(0, |m| { + let mut idx = m.match_index.min(title.len()); + while idx > 0 && !title.is_char_boundary(idx) { + idx -= 1; + } + idx + }); + + if start_index >= title.len() { + return None; + } + + let search_slice = &title[start_index..]; + + if let Ok(Some(m)) = volume_regex.find(search_slice) { + let raw_match = m.as_str().to_string(); + let relative_start = m.start(); + + if let Ok(Some(cap)) = volume_regex.captures(search_slice) { + let volume_number = cap.get(1).map_or(0, |m| m.as_str().parse::().unwrap_or(0)); + + context.result.volumes = vec![volume_number]; + } + + let abs_start = start_index + relative_start; + + let info = MatchInfo { raw_match, match_index: abs_start, remove: true, skip_from_title: false }; + + context.matched.insert("volumes".to_string(), info.clone()); + return Some(info); + } + None + }), + ); + + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:complete\W|seasons?\W|\W|^)((?:s\d{1,2}[., +/\\&-]+)+s\d{1,2}\b)").unwrap(), + parse_season_range, + extend_seasons, + options_remove(), + ); + + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:complete\W|seasons?\W|\W|^)[(\[]?(s\d{2,}-\d{2,}\b)[)\]]?").unwrap(), + parse_season_range, + extend_seasons, + options_remove(), + ); + + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:complete\W|seasons?\W|\W|^)[(\[]?(s[1-9]-[2-9])[)\]]?").unwrap(), + parse_season_range, + extend_seasons, + options_remove(), + ); + + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)\d+ª(?:.+)?(?:a.?)?\d+ª(?:(?:.+)?(?:temporadas?))").unwrap(), + parse_season_range, + extend_seasons, + options_remove(), + ); + + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?(?:seasons?|[Сс]езони?|temporadas?)[. ]?[-:]?[. ]?[( \[]?((?:\d{1,2}[., /\\&]+)+\d{1,2}\b)[)\]]?").unwrap(), + parse_season_range, + extend_seasons, + options_remove(), + ); + + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?(?:seasons?|[Сс]езони?|temporadas?)[. ]?[-:]?[. ]?[( \[]?((?:\d{1,2}[.-]+)+[1-9]\d?\b)(?!\W*\d{4})[)\]]?").unwrap(), + parse_season_range, + extend_seasons, + options_remove(), + ); + + parser.add_handler( + "seasons", + FancyRegex::new( + r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?season[. ]?[( \[]?((?:\d{1,2}[. -]+)+[1-9]\d?\b)[)\]]?(?!.*\.\w{2,4}$)", + ) + .unwrap(), + parse_season_range, + extend_seasons, + options_remove(), + ); + + parser.add_handler( + "seasons", + FancyRegex::new( + r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?\bseasons?\b[. -]?(\d{1,2}[. -]?(?:to|thru|and|\+|:)[. -]?\d{1,2})\b", + ) + .unwrap(), + parse_season_range, + extend_seasons, + options_remove(), + ); + + parser.add_handler( + "quality", + FancyRegex::new(r"(?i)\bDVB(?:\b|-)").unwrap(), + const_string("HDTV"), + set_quality, + options_remove(), + ); + + parser.add_handler( + "seasons", + FancyRegex::new( + r"(?i)(?:(?:\bthe\W)?\bcomplete\W)?(?:saison|seizoen|season|series|temp(?:orada)?):?[. ]?(\d{1,2})\b", + ) + .unwrap(), + |val| vec![val.parse().unwrap_or(0)], + extend_seasons, + options_remove(), + ); + + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(\d{1,2})(?:-?й)?[. _]?(?:[Сс]езон|sez(?:on)?)(?:\W?\D|$)").unwrap(), + |val| vec![val.parse().unwrap_or(0)], + extend_seasons, + options_remove(), + ); + + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)[Сс]езон:?[. _]?№?(\d{1,2})(?!\d)").unwrap(), + |val| vec![val.parse().unwrap_or(0)], + extend_seasons, + options_remove(), + ); + + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:\D|^)(\d{1,2})Â?[°ºªa]?[. ]*temporada").unwrap(), + |val| vec![val.parse().unwrap_or(0)], + extend_seasons, + options_remove(), + ); + + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)t(\d{1,3})(?:[ex]+|$)").unwrap(), + |val| vec![val.parse().unwrap_or(0)], + extend_seasons, + options_remove(), + ); + + parser.add_handler( + "seasons", + FancyRegex::new(r"(?i)(?:(?:\bthe\W)?\bcomplete)?(?().unwrap_or(0)], + extend_seasons, + options_remove_skip_if_already_found(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new( + r"(?i)(?:[\W\d]|^)e[ .]?[\[(]?(\d{1,3}(?:[ .-]*(?:[&+]|e|.){1,2}(?:[ .]*e)?[ .]?\d{1,3})+)(?:\W|$)", + ) + .unwrap(), + range_u32, + extend_episodes, + options_no_skip(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?:[\W\d]|^)ep[ .]?[\[(]?(\d{1,3}(?:[ .-]*(?:[&+]|ep){1,2}[ .]?\d{1,3})+)(?:\W|$)") + .unwrap(), + range_u32, + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?:[\W\d]|^)\d+[xх][ .]?[\[(]?(\d{1,3}(?:[ .]?[xх][ .]?\d{1,3})+)(?:\W|$)").unwrap(), + range_u32, + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)Серии:\s+(\d+)\s+(?:of|из|iz)\s+\d+\b").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new( + r"(?i)(?:[\W\d]|^)(?:episodes?|[Сс]ерии:?)[ .]?[\[(]?(\d{1,3}(?:[ .+]*[&+][ .]?\d{1,3})+)(?:\W|$)", + ) + .unwrap(), + range_u32, + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)[\[(]?(?:\D|^)(\d{1,3}[ .]?ao[ .]?\d{1,3})[)\]]?(?:\W|$)").unwrap(), + range_u32, + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new( + r"(?i)(?:[\W\d]|^)(?:e|eps?|episodes?|[Сс]ерии:?|\d+[xх])[ .]*[\[(]?(\d{1,3}(?:-\d{1,3})+)(?:\W|$)", + ) + .unwrap(), + range_u32, + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?:\W|^)(\d{1,3}(?:[ .]*~[ .]*\d{1,3})+)(?:\W|$)").unwrap(), + range_u32, + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)\bE\d{1,4}\s*à\s*E\d{1,4}\b").unwrap(), + range_u32, + extend_episodes, + options_remove_skip_if_already_found(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)[st]\d{1,2}[. ]?[xх-]?[. ]?(?:e|x|х|ep|-|\.)[. ]?(\d{1,4})(?:[abc]|v0?[1-4]|\D|$)") + .unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_remove(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)\b[st]\d{2}(\d{2})\b").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)-\s(\d{1,3}[ .]*-[ .]*\d{1,3})(?!-\d)(?:\W|$)").unwrap(), + range_u32, + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)s\d{1,2}\s?\((\d{1,3}[ .]*-[ .]*\d{1,3})\)").unwrap(), + range_u32, + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?:^|/)\d{1,2}-(\d{2})\b(?!-\d)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?<=^\[.+].+)[. ]+-[. ]+(\d{1,4})[. ]+(?=\W)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_remove(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new( + r"(?i)(?().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)Ep.\d+.-.\\d+").unwrap(), + range_u32, + extend_episodes, + options_remove(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?:\b[ée]p?(?:isode)?|[Ээ]пизод|[Сс]ер(?:ии|ия|\.)?|cap(?:itulo)?|epis[oó]dio)[. ]?[-:#№]?[. ]?(\d{1,4})(?:[abc]|v0?[1-4]|\W|$)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)\b(\d{1,3})(?:-?я)?[ ._-]*(?:ser(?:i?[iyja]|\b)|[Сс]ер(?:ии|ия|\.)?)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?:\D|^)\d{1,2}[. ]?[xх][. ]?(\d{1,3})(?:[abc]|v0?[1-4]|\D|$)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?<=S\d{2}E)(\d+)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"[\[(]\d{1,2}\.(\d{1,3})[)\]]").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"\b[Ss]\d{1,2}[ .](\d{1,2})\b").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"-\s?\d{1,2}\.(\d{2,3})\s?-").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?:\[|\()(\d+)\s(?:of|из|iz)\s\d+(?:\]|\))").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?<=\D|^)(\d{1,3})[. ]?(?:of|из|iz)[. ]?\d{1,3}(?=\D|$)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"\b\d{2}[ ._-](\d{2})(?:.F)?\.\\w{2,4}$").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(\d+)(?=.?\[([A-Z0-9]{8})\])").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_default(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?().unwrap_or(0)]), + extend_episodes, + options_remove_skip_if_already_found(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)(?:\W|^)(?:\d+)?(?:e|ep)(\d{1,3})(?:\W|$)").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + options_remove(), + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)E(\d+)\b").unwrap(), + |val| Some(vec![val.parse::().unwrap_or(0)]), + extend_episodes, + handler_options! { + remove: false, + skip_if_already_found: true, + ..Default::default() + }, + ); + + parser.add_handler( + "episodes", + FancyRegex::new(r"(?i)\b(\d{1,4})-(\d{1,4})\b").unwrap(), + range_u32, + extend_episodes, + handler_options! { + remove: false, + skip_if_already_found: true, + ..Default::default() + }, + ); + + parser.add_handler( + "country", + FancyRegex::new(r"\b(US|UK|AU|NZ|CA)\b").unwrap(), + value, + set_country, + options_default(), + ); +} diff --git a/backend/library/src/ptt/handlers/identity.rs b/backend/library/src/ptt/handlers/identity.rs new file mode 100644 index 000000000..d652b3136 --- /dev/null +++ b/backend/library/src/ptt/handlers/identity.rs @@ -0,0 +1,342 @@ +use super::{ + append_p, const_string, ignore, mark_adult, options_default, options_keep, options_keep_no_skip, options_remove, + options_remove_no_skip, options_remove_skip_if_already_found, options_skip_from_title_no_skip, push_language, + set_adult, set_container, set_episode_code, set_group, set_quality_to_scr_if_trash, + set_quality_to_tele_cine_if_trash, set_quality_to_tele_sync_if_trash, set_quality_to_vhs_if_trash, + set_quality_to_vhsrip_if_trash, set_resolution, set_tmdb, set_trash, set_tvdb, set_year, +}; +use crate::ptt::{ + parser::PttParser, + transformers::{boolean, first_uinteger, lowercase, none, transform_resolution, uppercase}, +}; +use fancy_regex::Regex as FancyRegex; + +#[allow(clippy::too_many_lines)] +pub(super) fn register(parser: &mut PttParser) { + parser.add_handler( + "tmdb", + FancyRegex::new(r"(?i)\btmdb\b[-=]\d+").unwrap(), + first_uinteger, + set_tmdb, + options_remove_skip_if_already_found(), + ); + + parser.add_handler( + "tvdb", + FancyRegex::new(r"(?i)\btvdb\b[-=]\d+").unwrap(), + first_uinteger, + set_tvdb, + options_remove_skip_if_already_found(), + ); + + parser.add_handler( + "quality", + FancyRegex::new(r"(?i)\bPRE[- .]?HDRip\b").unwrap(), + boolean, + set_quality_to_scr_if_trash, + options_remove_no_skip(), + ); + + parser.add_handler( + "languages", + FancyRegex::new(r"(?i)\bE[- ]?Sub\b").unwrap(), + const_string("en"), + push_language, + options_remove_no_skip(), + ); + + parser.add_handler( + "quality", + FancyRegex::new(r"(?i)\bTS-Screener\b").unwrap(), + boolean, + set_quality_to_tele_sync_if_trash, + options_remove_no_skip(), + ); + + parser.add_handler( + "year", + FancyRegex::new(r"\b19\d{2}\s?-\s?20\d{2}\b").unwrap(), + first_uinteger, + set_year, + options_keep(), + ); + + parser.add_handler( + "title_cleanup", + FancyRegex::new(r"(?i)\b(?:19|20)\d{2}\s*[-]\s*(?:(?:19|20)\d{2}|\d{2})\b").unwrap(), + none, + ignore, + options_remove(), + ); + + parser.add_handler( + "title_cleanup", + FancyRegex::new(r"(?i)\b100[ .-]*years?[ .-]*quest\b").unwrap(), + none, + ignore, + options_remove(), + ); + + parser.add_handler( + "title_cleanup", + FancyRegex::new(r"(?i)\[?(\+.)?Extras\]?").unwrap(), + none, + ignore, + options_remove(), + ); + + parser.add_handler( + "title_cleanup", + FancyRegex::new(r"(?i)(\+Movies)?\+Specials").unwrap(), + none, + ignore, + options_remove(), + ); + + parser.add_handler( + "group", + FancyRegex::new(r"-?EDGE2020").unwrap(), + const_string("EDGE2020"), + set_group, + options_remove(), + ); + + parser.add_handler("title_cleanup", FancyRegex::new(r"(?i)TV Money").unwrap(), none, ignore, options_remove()); + + parser.add_handler( + "container", + FancyRegex::new(r"(?i)\.?[\[(]?\b(MKV|AVI|MP4|WMV|MPG|MPEG)\b[\])]?").unwrap(), + lowercase, + set_container, + options_default(), + ); + + parser.add_handler("torrent", FancyRegex::new(r"\.torrent$").unwrap(), boolean, ignore, options_remove()); + + parser.add_handler("adult", FancyRegex::new(r"\b(XXX|xxx|Xxx)\b").unwrap(), boolean, set_adult, options_remove()); + + if let Ok(re) = FancyRegex::new(r"(?i)\b(18\+|adult|porn|xxx)\b") { + parser.add_handler("adult", re, boolean, mark_adult, options_default()); + } + + parser.add_handler( + "extras", + FancyRegex::new(r"(?i)\bOVA\b").unwrap(), + const_string("OVA"), + ignore, + options_remove(), + ); + + parser.add_handler( + "extras", + FancyRegex::new(r"(?i)\bOVA\b").unwrap(), + const_string("OVA"), + ignore, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)\[?\]?3840x\d{4}[\])?]?").unwrap(), + const_string("2160p"), + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)\[?\]?1920x\d{3,4}[\])?]?").unwrap(), + const_string("1080p"), + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)\[?\]?1280x\d{3}[\])?]?").unwrap(), + const_string("720p"), + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)\[?\]?(\d{3,4}x\d{3,4})[\])?]?p?").unwrap(), + append_p, + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(480|720|1080)0[pi]").unwrap(), + append_p, + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(?:QHD|QuadHD|WQHD|2560(\d+)?x(\d+)?1440p?)").unwrap(), + const_string("1440p"), + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(?:Full HD|FHD|1920(\d+)?x(\d+)?1080p?)").unwrap(), + const_string("1080p"), + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(?:BD|HD|M)(2160p?|4k)").unwrap(), + const_string("2160p"), + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(?:BD|HD|M)1080p?").unwrap(), + const_string("1080p"), + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(?:BD|HD|M)720p?").unwrap(), + const_string("720p"), + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(?:BD|HD|M)480p?").unwrap(), + const_string("480p"), + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)\b(?:4k|2160p|1080p|720p|480p)(?!.*\b(?:4k|2160p|1080p|720p|480p)\b)").unwrap(), + transform_resolution, + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)\b4k|21600?[pi]\b").unwrap(), + const_string("2160p"), + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(\d{3,4}[pi])").unwrap(), + lowercase, + set_resolution, + options_remove(), + ); + + parser.add_handler( + "resolution", + FancyRegex::new(r"(?i)(240|360|480|576|720|1080|2160|3840)[pi]").unwrap(), + lowercase, + set_resolution, + options_remove(), + ); + + parser.add_handler( + "episode_code", + FancyRegex::new(r"[\[\()]([A-Fa-f0-9]{8})[\]\)]").unwrap(), + uppercase, + set_episode_code, + options_remove(), + ); + + parser.add_handler( + "episode_code", + FancyRegex::new(r"[\[\()]([0-9]{8})[\]\)]").unwrap(), + uppercase, + set_episode_code, + options_remove_skip_if_already_found(), + ); + + parser.add_handler( + "trash", + FancyRegex::new( + r"(?i)\b(?:H[DQ][ .-]*)?(? usize { - let parallelism = - std::thread::available_parallelism().map_or(BLOCKING_DB_MIN_CONCURRENCY, std::num::NonZeroUsize::get); - parallelism.saturating_mul(2).clamp(BLOCKING_DB_MIN_CONCURRENCY, BLOCKING_DB_MAX_CONCURRENCY) -} - -fn metadata_blocking_semaphore() -> &'static Semaphore { - static BLOCKING_SEMAPHORE: OnceLock = OnceLock::new(); - BLOCKING_SEMAPHORE.get_or_init(|| Semaphore::new(metadata_blocking_concurrency_limit())) -} - -async fn spawn_blocking_limited(task: F) -> Result -where - F: FnOnce() -> R + Send + 'static, - R: Send + 'static, -{ - // Throttle B+Tree and file-heavy blocking tasks to avoid saturating Tokio's blocking pool. - let permit = metadata_blocking_semaphore().acquire().await.ok(); - let result = tokio::task::spawn_blocking(task).await; - drop(permit); - result -} - -#[derive(Debug, Clone)] -struct MetadataUpdateRuntimeSettings { - queue_log_interval: Duration, - progress_log_interval: Duration, - max_resolve_retry_backoff_secs: u64, - resolve_min_retry_base_secs: u64, - max_attempts_resolve: u8, - max_attempts_probe: u8, - resolve_exhaustion_reset_gap_secs: i64, - probe_cooldown_secs: i64, - tmdb_cooldown_secs: i64, - retry_delay_secs: u64, - metadata_retry_load_retry_delay_secs: i64, - worker_idle_timeout_secs: u64, - max_queue_size: usize, - no_change_cache_ttl_secs: u64, - probe_fairness_resolve_burst: usize, - probe_retry_backoff_step_1_secs: u64, - probe_retry_backoff_step_2_secs: u64, - probe_retry_backoff_step_3_secs: u64, - backoff_jitter_percent: u8, -} - -impl Default for MetadataUpdateRuntimeSettings { - fn default() -> Self { - let defaults = MetadataUpdateConfig::default(); - Self::from_metadata_update(&defaults) - } -} - -impl MetadataUpdateRuntimeSettings { - fn from_ctx(ctx: Option<&MetadataUpdateCtx>) -> Self { - let metadata_update = ctx.map_or_else(MetadataUpdateConfig::default, |ctx| { - ctx.app_config - .config - .load() - .metadata_update - .as_ref() - .map_or_else(MetadataUpdateConfig::default, Clone::clone) - }); - Self::from_metadata_update(&metadata_update) - } - - fn from_metadata_update(cfg: &MetadataUpdateConfig) -> Self { - let to_i64 = |v: u64| i64::try_from(v.max(1)).unwrap_or(i64::MAX); - Self { - queue_log_interval: Duration::from_secs(cfg.log.queue_interval_secs.max(1)), - progress_log_interval: Duration::from_secs(cfg.log.progress_interval_secs.max(1)), - max_resolve_retry_backoff_secs: cfg.resolve.max_retry_backoff_secs.max(1), - resolve_min_retry_base_secs: cfg.resolve.min_retry_base_secs.max(1), - max_attempts_resolve: cfg.resolve.max_attempts.max(1), - max_attempts_probe: cfg.probe.max_attempts.max(1), - resolve_exhaustion_reset_gap_secs: to_i64(cfg.resolve.exhaustion_reset_gap_secs), - probe_cooldown_secs: to_i64(cfg.probe.cooldown_secs), - tmdb_cooldown_secs: to_i64(cfg.tmdb.cooldown_secs), - retry_delay_secs: cfg.retry_delay_secs.max(1), - metadata_retry_load_retry_delay_secs: to_i64(cfg.probe.retry_load_retry_delay_secs), - worker_idle_timeout_secs: cfg.worker_idle_timeout_secs.max(1), - max_queue_size: cfg.max_queue_size.max(1), - no_change_cache_ttl_secs: cfg.no_change_cache_ttl_secs.max(1), - probe_fairness_resolve_burst: cfg.probe_fairness_resolve_burst.max(1), - probe_retry_backoff_step_1_secs: cfg.probe.retry_backoff_step_1_secs.max(1), - probe_retry_backoff_step_2_secs: cfg.probe.retry_backoff_step_2_secs.max(1), - probe_retry_backoff_step_3_secs: cfg.probe.retry_backoff_step_3_secs.max(1), - backoff_jitter_percent: cfg.probe.backoff_jitter_percent.min(95), - } - } -} - #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum TaskKey { Vod(u32), @@ -159,278 +42,6 @@ pub enum TaskKey { Stream { scope: Arc, id: Arc }, } -impl TaskKey { - pub fn from_task(task: &UpdateTask) -> Self { - match task { - UpdateTask::ResolveVod { id, .. } => match id { - ProviderIdType::Id(val) => TaskKey::Vod(*val), - ProviderIdType::Text(val) => TaskKey::VodStr(val.clone()), - }, - UpdateTask::ResolveSeries { id, .. } => match id { - ProviderIdType::Id(val) => TaskKey::Series(*val), - ProviderIdType::Text(val) => TaskKey::SeriesStr(val.clone()), - }, - UpdateTask::ProbeLive { id, .. } => match id { - ProviderIdType::Id(val) => TaskKey::Live(*val), - ProviderIdType::Text(val) => TaskKey::LiveStr(val.clone()), - }, - UpdateTask::ProbeStream { probe_scope, unique_id, url, .. } => { - if unique_id.trim().is_empty() { - TaskKey::Stream { scope: probe_scope.clone(), id: Arc::from(url.as_str()) } - } else { - TaskKey::Stream { scope: probe_scope.clone(), id: Arc::from(unique_id.as_str()) } - } - } - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct ScopedTaskKey { - input_name: Arc, - task_key: TaskKey, -} - -impl ScopedTaskKey { - fn new(input_name: Arc, task_key: TaskKey) -> Self { Self { input_name, task_key } } -} - -#[derive(Debug, Clone)] -struct RetryState { - attempts: u8, - next_allowed_at_ts: i64, - cooldown_until_ts: Option, - last_error: Option, - source_last_modified: Option, -} - -impl RetryState { - fn new() -> Self { - Self { - attempts: 0, - next_allowed_at_ts: 0, - cooldown_until_ts: None, - last_error: None, - source_last_modified: None, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum RetryDomain { - Resolve, - Probe, - Tmdb, -} - -#[derive(Debug, Clone, Default)] -struct TaskRetryState { - resolve: Option, - probe: Option, - tmdb: Option, - updated_at_ts: i64, -} - -impl TaskRetryState { - fn is_empty(&self) -> bool { self.resolve.is_none() && self.probe.is_none() && self.tmdb.is_none() } - - fn touch(&mut self, now_ts: i64) { self.updated_at_ts = now_ts.max(1); } - - fn max_domain_timestamp(&self) -> i64 { - let domain_max = |state: &RetryState| state.next_allowed_at_ts.max(state.cooldown_until_ts.unwrap_or(0)); - self.resolve - .as_ref() - .map_or(0, domain_max) - .max(self.probe.as_ref().map_or(0, domain_max)) - .max(self.tmdb.as_ref().map_or(0, domain_max)) - } - - fn is_stale(&self, now_ts: i64, ttl_secs: i64) -> bool { - let ttl_secs = ttl_secs.max(1); - let anchor_ts = self.updated_at_ts.max(self.max_domain_timestamp()); - now_ts >= anchor_ts.saturating_add(ttl_secs) - } - - fn get(&self, domain: RetryDomain) -> Option<&RetryState> { - match domain { - RetryDomain::Resolve => self.resolve.as_ref(), - RetryDomain::Probe => self.probe.as_ref(), - RetryDomain::Tmdb => self.tmdb.as_ref(), - } - } - - fn get_mut_or_insert(&mut self, domain: RetryDomain) -> &mut RetryState { - let slot = match domain { - RetryDomain::Resolve => &mut self.resolve, - RetryDomain::Probe => &mut self.probe, - RetryDomain::Tmdb => &mut self.tmdb, - }; - slot.get_or_insert_with(RetryState::new) - } - - fn clear_domain(&mut self, domain: RetryDomain) { - match domain { - RetryDomain::Resolve => self.resolve = None, - RetryDomain::Probe => self.probe = None, - RetryDomain::Tmdb => self.tmdb = None, - } - } -} - -fn metadata_retry_key_from_task_key(task_key: &TaskKey) -> MetadataRetryDbKey { - match task_key { - TaskKey::Vod(id) => MetadataRetryDbKey::VodId(*id), - TaskKey::VodStr(id) => MetadataRetryDbKey::VodText(id.as_ref().to_owned()), - TaskKey::Series(id) => MetadataRetryDbKey::SeriesId(*id), - TaskKey::SeriesStr(id) => MetadataRetryDbKey::SeriesText(id.as_ref().to_owned()), - TaskKey::Live(id) => MetadataRetryDbKey::LiveId(*id), - TaskKey::LiveStr(id) => MetadataRetryDbKey::LiveText(id.as_ref().to_owned()), - TaskKey::Stream { scope, id } => { - MetadataRetryDbKey::Stream { scope: scope.as_ref().to_owned(), id: id.as_ref().to_owned() } - } - } -} - -fn metadata_retry_key_into_task_key(value: MetadataRetryDbKey) -> TaskKey { - match value { - MetadataRetryDbKey::VodId(id) => TaskKey::Vod(id), - MetadataRetryDbKey::VodText(id) => TaskKey::VodStr(Arc::from(id)), - MetadataRetryDbKey::SeriesId(id) => TaskKey::Series(id), - MetadataRetryDbKey::SeriesText(id) => TaskKey::SeriesStr(Arc::from(id)), - MetadataRetryDbKey::LiveId(id) => TaskKey::Live(id), - MetadataRetryDbKey::LiveText(id) => TaskKey::LiveStr(Arc::from(id)), - MetadataRetryDbKey::Stream { scope, id } => TaskKey::Stream { scope: Arc::from(scope), id: Arc::from(id) }, - } -} - -fn retry_state_db_from_retry_state(state: &RetryState) -> RetryStateDbValue { - RetryStateDbValue { - attempts: state.attempts, - next_allowed_at_ts: state.next_allowed_at_ts, - cooldown_until_ts: state.cooldown_until_ts, - last_error: state.last_error.clone(), - source_last_modified: state.source_last_modified, - } -} - -fn retry_state_db_into_retry_state(value: RetryStateDbValue) -> Option { - if value.attempts == 0 - && value.next_allowed_at_ts <= 0 - && value.cooldown_until_ts.is_none() - && value.source_last_modified.is_none() - { - return None; - } - Some(RetryState { - attempts: value.attempts, - next_allowed_at_ts: value.next_allowed_at_ts, - cooldown_until_ts: value.cooldown_until_ts, - last_error: value.last_error, - source_last_modified: value.source_last_modified, - }) -} - -fn metadata_retry_value_from_task_retry_state(state: &TaskRetryState, updated_at_ts: i64) -> MetadataRetryDbValue { - MetadataRetryDbValue { - resolve: state.resolve.as_ref().map(retry_state_db_from_retry_state), - probe: state.probe.as_ref().map(retry_state_db_from_retry_state), - tmdb: state.tmdb.as_ref().map(retry_state_db_from_retry_state), - updated_at_ts, - } -} - -fn metadata_retry_value_into_task_retry_state(value: MetadataRetryDbValue) -> Option { - let mut state = TaskRetryState { - resolve: value.resolve.and_then(retry_state_db_into_retry_state), - probe: value.probe.and_then(retry_state_db_into_retry_state), - tmdb: value.tmdb.and_then(retry_state_db_into_retry_state), - updated_at_ts: value.updated_at_ts, - }; - if state.is_empty() { - return None; - } - if state.updated_at_ts <= 0 { - state.updated_at_ts = state.max_domain_timestamp(); - } - Some(state) -} - -fn ensure_metadata_retry_db(path: &Path) -> io::Result<()> { - if path.exists() { - return Ok(()); - } - let mut tree = BPlusTree::::new(); - tree.store(path).map(|_| ()) -} - -fn load_metadata_retry_states_from_disk(path: &Path) -> io::Result> { - ensure_metadata_retry_db(path)?; - - let mut result = HashMap::new(); - let mut stale_keys: Vec = Vec::new(); - let mut query = BPlusTreeQuery::::try_new(path)?; - for entry in query.iter() { - let (key, value) = entry?; - if let Some(state) = metadata_retry_value_into_task_retry_state(value.clone()) { - result.insert(metadata_retry_key_into_task_key(key), state); - } else { - stale_keys.push(key); - } - } - drop(query); - - if !stale_keys.is_empty() { - let delete_refs: Vec<&MetadataRetryDbKey> = stale_keys.iter().collect(); - let mut update = BPlusTreeUpdate::::try_new_with_backoff(path)?; - update - .delete_batch(&delete_refs) - .map_err(|e| io::Error::other(format!("cleanup metadata retry tombstones failed: {e}")))?; - } - - Ok(result) -} - -fn persist_metadata_retry_state_to_disk( - path: &Path, - task_key: &TaskKey, - state: Option<&TaskRetryState>, -) -> io::Result<()> { - let db_key = metadata_retry_key_from_task_key(task_key); - - ensure_metadata_retry_db(path)?; - - let mut update = BPlusTreeUpdate::::try_new_with_backoff(path)?; - if let Some(retry_state) = state { - let now_ts = chrono::Utc::now().timestamp(); - let value = metadata_retry_value_from_task_retry_state(retry_state, now_ts); - update - .upsert_batch(&[(&db_key, &value)]) - .map_err(|e| io::Error::other(format!("persist metadata retry state failed: {e}")))?; - } else { - update.delete(&db_key).map_err(|e| io::Error::other(format!("delete metadata retry state failed: {e}")))?; - } - Ok(()) -} - -/// Per-input worker context. Each input has its own worker -/// that processes tasks sequentially with rate limiting. -#[derive(Clone)] -struct InputWorkerContext { - worker_id: u64, - sender: mpsc::Sender, - pending_tasks: Arc>, - pending_task_count: Arc, -} - -struct PendingTask { - task: ParkingMutex, - generation: AtomicU64, -} - -impl PendingTask { - fn new(task: UpdateTask) -> Self { Self { task: ParkingMutex::new(task), generation: AtomicU64::new(0) } } -} - /// Manager for background metadata resolution tasks. /// /// Architecture: Per-Input Worker Pattern @@ -470,628 +81,13 @@ pub struct MetadataUpdateManager { last_resolve_enqueue_suppression_prune_at_ts: AtomicI64, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum SubmitTaskResult { - QueuedOrMerged, - QueueFull, - ChannelClosed, -} - -impl MetadataUpdateManager { - pub fn new(cancel_token: CancellationToken) -> Self { - Self { - workers: DashMap::new(), - worker_lifecycle_lock: ParkingMutex::new(()), - is_shutdown_flag: AtomicBool::new(false), - ctx: tokio::sync::Mutex::new(None), - update_pause_gate: Arc::new(RwLock::new(())), - cancel_token: ArcSwap::from_pointee(cancel_token), - next_worker_id: AtomicU64::new(1), - resolve_enqueue_suppressions: Arc::new(DashMap::new()), - tmdb_source_markers: Arc::new(DashMap::new()), - enqueue_state_loaded_inputs: Arc::new(DashMap::new()), - enqueue_state_load_retry_at_ts: Arc::new(DashMap::new()), - last_resolve_enqueue_suppression_prune_at_ts: AtomicI64::new(0), - } - } - - /// Requests graceful shutdown for all metadata workers and delayed requeue tasks. - /// This operation is idempotent. - pub fn shutdown(&self) { - let _guard = self.worker_lifecycle_lock.lock(); - self.is_shutdown_flag.store(true, Ordering::Release); - self.cancel_token.load_full().cancel(); - self.resolve_enqueue_suppressions.clear(); - self.tmdb_source_markers.clear(); - self.enqueue_state_loaded_inputs.clear(); - self.enqueue_state_load_retry_at_ts.clear(); - } - - /// Returns true once shutdown has been requested. - pub fn is_shutdown(&self) -> bool { self.is_shutdown_flag.load(Ordering::Acquire) } - - /// Rotates the cancellation token for metadata workers. - /// Existing workers are cancelled and removed so new tasks start with fresh runtime state. - pub fn rotate_cancel_token(&self, cancel_token: CancellationToken) { - let old_token = { - let _guard = self.worker_lifecycle_lock.lock(); - if self.is_shutdown_flag.load(Ordering::Acquire) { - self.workers.clear(); - cancel_token.cancel(); - return; - } - - let old_token = self.cancel_token.swap(Arc::new(cancel_token)); - self.workers.clear(); - self.resolve_enqueue_suppressions.clear(); - self.tmdb_source_markers.clear(); - self.enqueue_state_loaded_inputs.clear(); - self.enqueue_state_load_retry_at_ts.clear(); - old_token - }; - old_token.cancel(); - } - - /// Acquire exclusive gate for a foreground playlist update. - /// While this guard is held, background workers wait before starting heavy metadata/probe steps. - pub async fn acquire_update_pause_guard(&self) -> tokio::sync::OwnedRwLockWriteGuard<()> { - self.update_pause_gate.clone().write_owned().await - } - - /// Bind the handles the worker reads. Called once, after the server's root - /// state is built - the worker itself is constructed before it. - pub async fn set_ctx(&self, ctx: BoundMetadataUpdateCtx) { - let mut guard = self.ctx.lock().await; - *guard = Some(ctx); - } - - fn scoped_task_key(input_name: &str, task: &UpdateTask) -> ScopedTaskKey { - ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(task)) - } - - fn has_pending_task(&self, input_name: &str, task: &UpdateTask) -> bool { - self.workers - .get(input_name) - .is_some_and(|ctx| !ctx.sender.is_closed() && ctx.pending_tasks.contains_key(&TaskKey::from_task(task))) - } - - fn prune_resolve_enqueue_suppressions(&self, now_ts: i64) { - let last_pruned_at = self.last_resolve_enqueue_suppression_prune_at_ts.load(Ordering::Relaxed); - if last_pruned_at != 0 && now_ts.saturating_sub(last_pruned_at) < RETRY_STATE_PRUNE_INTERVAL_SECS { - return; - } - self.last_resolve_enqueue_suppression_prune_at_ts.store(now_ts, Ordering::Relaxed); - self.resolve_enqueue_suppressions.retain(|_, suppressed_until_ts| *suppressed_until_ts > now_ts); - } - - fn should_skip_enqueue_cached(&self, input_name: &str, task: &UpdateTask) -> bool { - let now_ts = chrono::Utc::now().timestamp(); - self.prune_resolve_enqueue_suppressions(now_ts); - - if self.is_redundant_with_pending_task(input_name, task) { - return true; - } - - if InputWorker::retry_domain_for_task(task) != RetryDomain::Resolve { - return false; - } - - let scoped_key = Self::scoped_task_key(input_name, task); - if let Some(entry) = self.resolve_enqueue_suppressions.get(&scoped_key) { - let suppressed_until_ts = *entry; - drop(entry); - if now_ts < suppressed_until_ts { - if self.has_pending_task(input_name, task) { - return false; - } - debug!( - "[Task] Skipping enqueue (resolve suppression) for input {}: {} (until_ts={}, remaining={}s)", - input_name, - task, - suppressed_until_ts, - suppressed_until_ts.saturating_sub(now_ts) - ); - return true; - } - self.resolve_enqueue_suppressions.remove(&scoped_key); - } - - false - } - - async fn ensure_enqueue_state_loaded_for_input(&self, input_name: &Arc) { - if self.enqueue_state_loaded_inputs.contains_key(input_name) { - return; - } - let now_ts = chrono::Utc::now().timestamp(); - if self.enqueue_state_load_retry_at_ts.get(input_name).is_some_and(|retry_at_ts| now_ts < *retry_at_ts) { - return; - } - - let bound_ctx = { - let guard = self.ctx.lock().await; - guard.clone() - }; - let runtime_settings = MetadataUpdateRuntimeSettings::from_ctx(bound_ctx.as_ref()); - let Some(ctx) = bound_ctx else { - return; - }; - let retry_at_ts = now_ts.saturating_add(runtime_settings.metadata_retry_load_retry_delay_secs); - - let storage_dir = ctx.app_config.config.load().storage_dir.clone(); - let Ok(storage_path) = get_input_storage_path(input_name, &storage_dir).await else { - self.enqueue_state_load_retry_at_ts.insert(input_name.clone(), retry_at_ts); - return; - }; - let retry_path = storage_path.join(METADATA_RETRY_STATE_FILE); - let input_name_cloned = input_name.clone(); - let loaded = spawn_blocking_limited(move || load_metadata_retry_states_from_disk(&retry_path)) - .await - .ok() - .and_then(Result::ok); - let Some(states) = loaded else { - self.enqueue_state_load_retry_at_ts.insert(input_name.clone(), retry_at_ts); - return; - }; - - for (task_key, state) in states { - if let Some(resolve_state) = state.resolve.as_ref() { - let suppressed_until_ts = resolve_state.cooldown_until_ts.unwrap_or(resolve_state.next_allowed_at_ts); - if suppressed_until_ts > now_ts { - let scoped_key = ScopedTaskKey::new(input_name_cloned.clone(), task_key.clone()); - self.resolve_enqueue_suppressions.insert(scoped_key, suppressed_until_ts); - } - } - if let Some(source_last_modified) = - state.tmdb.as_ref().and_then(|tmdb_state| tmdb_state.source_last_modified) - { - let scoped_key = ScopedTaskKey::new(input_name_cloned.clone(), task_key.clone()); - self.tmdb_source_markers.insert(scoped_key, source_last_modified); - } - } - - self.enqueue_state_load_retry_at_ts.remove(input_name); - self.enqueue_state_loaded_inputs.insert(input_name.clone(), ()); - } - - fn strip_tmdb_reasons_for_enqueue(&self, input_name: &str, task: UpdateTask) -> Option { - let scoped_key = Self::scoped_task_key(input_name, &task); - let current_last_modified = InputWorker::task_source_last_modified(&task); - let previous_last_modified = self.tmdb_source_markers.get(&scoped_key).map(|entry| *entry); - if (current_last_modified.is_some() || previous_last_modified.is_some()) - && current_last_modified == previous_last_modified - && InputWorker::task_has_tmdb_reason(&task) - { - InputWorker::strip_tmdb_reasons(&task) - } else { - Some(task) - } - } - - async fn prepare_task_for_enqueue(&self, input_name: Arc, task: UpdateTask) -> Option { - self.ensure_enqueue_state_loaded_for_input(&input_name).await; - let prepared_task = self.strip_tmdb_reasons_for_enqueue(input_name.as_ref(), task)?; - if self.should_skip_enqueue_cached(input_name.as_ref(), &prepared_task) { - None - } else { - Some(prepared_task) - } - } - - /// Spawn a background task to queue the update. - /// This is a fire-and-forget method that returns immediately. - pub fn queue_task_background(self: &Arc, input_name: Arc, task: UpdateTask) { - let this = self.clone(); - tokio::spawn(async move { - this.queue_task(input_name, task).await; - }); - } - - /// Queue a task for background processing. - /// - /// If a worker exists for the input, the task is sent to it. - /// If no worker exists, a new one is spawned. - /// - /// # Arguments - /// * `input_name` - The input this task belongs to - /// * `task` - The task to process - #[allow(clippy::too_many_lines)] - pub async fn queue_task(&self, input_name: Arc, task: UpdateTask) { - debug!("[Task] Queuing task for input {input_name}: {task}"); - let Some(task_to_queue) = self.prepare_task_for_enqueue(input_name.clone(), task).await else { - return; - }; - - // Read app state once and reuse for worker creation when needed. - let bound_ctx = { - let guard = self.ctx.lock().await; - guard.clone() - }; - let runtime_settings = MetadataUpdateRuntimeSettings::from_ctx(bound_ctx.as_ref()); - let max_queue_size = runtime_settings.max_queue_size; - - let mut channel_closed_attempt: u32 = 0; - loop { - // Atomically ensure there is exactly one worker context per input. - let mut worker_to_spawn: Option<(u64, InputWorker)> = None; - let (cancel_token, ctx) = { - let _guard = self.worker_lifecycle_lock.lock(); - let cancel_token = self.cancel_token.load_full(); - if cancel_token.is_cancelled() { - debug_if_enabled!( - "Aborting metadata enqueue loop for input {input_name} because cancellation was requested: {task_to_queue}" - ); - return; - } - - let ctx = match self.workers.entry(input_name.clone()) { - Entry::Occupied(entry) => entry.get().clone(), - Entry::Vacant(entry) => { - let (tx, rx) = mpsc::channel::(max_queue_size); - let pending_tasks = Arc::new(DashMap::new()); - let pending_task_count = Arc::new(AtomicUsize::new(0)); - let worker_id = self.next_worker_id.fetch_add(1, Ordering::Relaxed); - - let ctx = InputWorkerContext { - worker_id, - sender: tx.clone(), - pending_tasks: pending_tasks.clone(), - pending_task_count: pending_task_count.clone(), - }; - entry.insert(ctx.clone()); - - worker_to_spawn = Some(( - worker_id, - InputWorker { - input_name: input_name.clone(), - sender: tx, - receiver: rx, - pending_tasks, - pending_task_count, - bound_ctx: bound_ctx.clone(), - update_pause_gate: Arc::clone(&self.update_pause_gate), - cancel_token: (*cancel_token).clone(), - batch_buffer: BatchResultCollector::new(), - db_handles: HashMap::new(), - failed_clusters: HashSet::new(), - retry_states: HashMap::new(), - resolve_exhausted: HashMap::new(), - last_cycle_completed_at_ts: None, - metadata_retry_state_path: None, - metadata_retry_loaded: false, - metadata_retry_load_retry_at_ts: None, - last_retry_state_prune_at_ts: None, - scheduled_requeues: Arc::new(DashMap::new()), - recently_completed_no_change: HashMap::new(), - resolve_enqueue_suppressions: Arc::clone(&self.resolve_enqueue_suppressions), - tmdb_source_markers: Arc::clone(&self.tmdb_source_markers), - dirty_retry_state_keys: HashSet::new(), - }, - )); - - ctx - } - }; - - (cancel_token, ctx) - }; - - if let Some((worker_id, worker)) = worker_to_spawn { - let workers_ref = self.workers.clone(); - let input_name_for_cleanup = input_name.clone(); - tokio::spawn(async move { - worker.run().await; - - // Cleanup only if this exact worker context is still active. - if let Entry::Occupied(entry) = workers_ref.entry(input_name_for_cleanup.clone()) { - if entry.get().worker_id == worker_id { - entry.remove(); - } - } - }); - } - - let sender_still_current = { - let _guard = self.worker_lifecycle_lock.lock(); - let token_matches = Arc::ptr_eq(&self.cancel_token.load_full(), &cancel_token); - let worker_matches = - self.workers.get(&input_name).is_some_and(|current| current.worker_id == ctx.worker_id); - token_matches && worker_matches - }; - if !sender_still_current { - continue; - } - - match Self::submit_task( - ctx.sender.clone(), - ctx.pending_tasks.clone(), - ctx.pending_task_count.clone(), - &input_name, - max_queue_size, - task_to_queue.clone(), - ) - .await - { - SubmitTaskResult::QueuedOrMerged => return, - SubmitTaskResult::QueueFull => { - error!( - "Metadata queue full for input {input_name} (max_queue_size={max_queue_size}), \ - dropping task: {task_to_queue}; consider increasing max_queue_size or reducing \ - probe frequency" - ); - return; - } - SubmitTaskResult::ChannelClosed => { - channel_closed_attempt = channel_closed_attempt.saturating_add(1); - if channel_closed_attempt.is_multiple_of(10) { - warn!( - "Metadata enqueue channel still closed for input {input_name} after {channel_closed_attempt} retries; continuing recovery" - ); - } - debug_if_enabled!( - "Detected closed metadata worker channel for input {}, recreating worker context (attempt {})", - input_name, - channel_closed_attempt - ); - Self::remove_worker_context_if_id(&self.workers, &input_name, ctx.worker_id); - let exp = channel_closed_attempt.saturating_sub(1).min(6); - let factor = 1_u64.checked_shl(exp).unwrap_or(u64::MAX); - let backoff_ms = 25_u64.saturating_mul(factor).min(2_000); - let backoff = Duration::from_millis(backoff_ms); - tokio::select! { - () = cancel_token.cancelled() => { - warn!( - "Aborting metadata task enqueue for input {input_name} because cancellation was requested: {task_to_queue}" - ); - return; - } - () = tokio::time::sleep(backoff) => {} - } - } - } - } - } - - fn remove_worker_context_if_id( - workers: &DashMap, InputWorkerContext>, - input_name: &Arc, - worker_id: u64, - ) { - if let Entry::Occupied(entry) = workers.entry(input_name.clone()) { - if entry.get().worker_id == worker_id { - entry.remove(); - } - } - } - - async fn submit_task( - sender: mpsc::Sender, - pending_tasks: Arc>, - pending_task_count: Arc, - input_name: &str, - max_queue_size: usize, - task: UpdateTask, - ) -> SubmitTaskResult { - let key = TaskKey::from_task(&task); - let task_to_submit = task; - - if let Some(entry) = pending_tasks.get(&key) { - if sender.is_closed() { - drop(entry); - if pending_tasks.remove(&key).is_some() { - Self::decrement_pending_task_count(&pending_task_count); - } - } else { - let mut existing = entry.task.lock(); - let before = format!("{existing}"); - if Self::merge_task_payload(&mut existing, task_to_submit) { - entry.generation.fetch_add(1, Ordering::Relaxed); - debug!("[Task] Merged task for input {input_name}: before={before}, after={existing}"); - } else { - debug!("[Task] Task already pending for input {input_name} (no merge needed): {existing}"); - } - return SubmitTaskResult::QueuedOrMerged; - } - } - - // Lock-free admission with CAS: reserve one queue slot only if capacity allows. - if pending_task_count - .try_update(Ordering::AcqRel, Ordering::Relaxed, |current| { - if current < max_queue_size { - Some(current + 1) - } else { - None - } - }) - .is_err() - { - return SubmitTaskResult::QueueFull; - } - - loop { - match pending_tasks.entry(key.clone()) { - Entry::Occupied(entry) => { - // Another producer inserted this key after our fast-path `get`. - if sender.is_closed() { - entry.remove(); - Self::decrement_pending_task_count(&pending_task_count); - continue; - } - - // Release reserved capacity and merge into the existing task. - Self::decrement_pending_task_count(&pending_task_count); - let mut existing = entry.get().task.lock(); - if Self::merge_task_payload(&mut existing, task_to_submit) { - entry.get().generation.fetch_add(1, Ordering::Relaxed); - } - return SubmitTaskResult::QueuedOrMerged; - } - Entry::Vacant(entry) => { - entry.insert(PendingTask::new(task_to_submit)); - break; - } - } - } - - if sender.send(key.clone()).await.is_err() { - if pending_tasks.remove(&key).is_some() { - Self::decrement_pending_task_count(&pending_task_count); - } - warn!("Failed to send task signal for input {input_name}"); - return SubmitTaskResult::ChannelClosed; - } - SubmitTaskResult::QueuedOrMerged - } - - #[inline] - fn decrement_pending_task_count(pending_task_count: &AtomicUsize) { - // Guard against accidental underflow in edge/error paths. - let _ = pending_task_count.try_update(Ordering::AcqRel, Ordering::Relaxed, |current| current.checked_sub(1)); - } - - fn merge_task_payload(existing: &mut UpdateTask, task: UpdateTask) -> bool { - let mut changed = false; - // Merge logic - match (existing, task) { - ( - UpdateTask::ResolveVod { reason: r1, delay: d1, source_last_modified: lm1, .. }, - UpdateTask::ResolveVod { reason: r2, delay: d2, source_last_modified: lm2, .. }, - ) - | ( - UpdateTask::ResolveSeries { reason: r1, delay: d1, source_last_modified: lm1, .. }, - UpdateTask::ResolveSeries { reason: r2, delay: d2, source_last_modified: lm2, .. }, - ) => { - let previous_reason = *r1; - let previous_delay = *d1; - let previous_last_modified = *lm1; - *r1 |= r2; - *d1 = min(*d1, d2); - *lm1 = match (*lm1, lm2) { - (Some(left), Some(right)) => Some(left.max(right)), - _ => None, - }; - changed = *r1 != previous_reason || *d1 != previous_delay || *lm1 != previous_last_modified; - } - ( - UpdateTask::ProbeStream { reason: r1, delay: d1, url: url1, item_type: item_type1, .. }, - UpdateTask::ProbeStream { reason: r2, delay: d2, url: url2, item_type: item_type2, .. }, - ) => { - let previous_reason = *r1; - let previous_delay = *d1; - let previous_url = url1.clone(); - let previous_item_type = *item_type1; - *r1 |= r2; - *d1 = min(*d1, d2); - // Keep the existing payload by default; only fill it from the incoming - // task when the destination payload is empty. - if url1.is_empty() && !url2.is_empty() { - *url1 = url2; - *item_type1 = item_type2; - } - changed = *r1 != previous_reason - || *d1 != previous_delay - || *url1 != previous_url - || *item_type1 != previous_item_type; - } - ( - UpdateTask::ProbeLive { reason: r1, delay: d1, interval: i1, .. }, - UpdateTask::ProbeLive { reason: r2, delay: d2, interval: i2, .. }, - ) => { - let previous_reason = *r1; - let previous_delay = *d1; - let previous_interval = *i1; - *r1 |= r2; - *d1 = min(*d1, d2); - *i1 = min(*i1, i2); - changed = *r1 != previous_reason || *d1 != previous_delay || *i1 != previous_interval; - } - _ => {} // Mismatched types, should not happen due to TaskKey - } - - changed - } - - /// Queue a task using the legacy API (for backward compatibility). - /// Uses default delay of 50ms. - pub async fn queue_task_legacy(&self, input_name: Arc, task: UpdateTask) { - self.queue_task(input_name, task).await; - } - - /// Get the number of active workers (for monitoring/debugging) - pub fn active_worker_count(&self) -> usize { self.workers.len() } - - /// Returns `true` when an equivalent task is already pending for this input and - /// submitting `task` would not change the queued payload after merge semantics. - pub fn is_redundant_with_pending_task(&self, input_name: &str, task: &UpdateTask) -> bool { - let Some(ctx) = self.workers.get(input_name) else { - return false; - }; - - // If the worker channel is already closed, let normal enqueue recovery run. - if ctx.sender.is_closed() { - return false; - } - - let key = TaskKey::from_task(task); - let Some(entry) = ctx.pending_tasks.get(&key) else { - return false; - }; - - let mut merged = entry.task.lock().clone(); - !Self::merge_task_payload(&mut merged, task.clone()) - } -} - -struct DbHandle { - _guard: FileReadGuard, - query: Arc>>, -} - -struct InputWorker { - input_name: Arc, - sender: mpsc::Sender, - receiver: mpsc::Receiver, - pending_tasks: Arc>, - pending_task_count: Arc, - bound_ctx: Option, - update_pause_gate: Arc>, - cancel_token: CancellationToken, - batch_buffer: BatchResultCollector, - db_handles: HashMap, - failed_clusters: HashSet, - retry_states: HashMap, - resolve_exhausted: HashMap, - last_cycle_completed_at_ts: Option, - metadata_retry_state_path: Option, - metadata_retry_loaded: bool, - metadata_retry_load_retry_at_ts: Option, - last_retry_state_prune_at_ts: Option, - // Shared with detached delayed requeue tasks spawned in `schedule_requeue_at`. - // A plain HashMap cannot be moved safely into those `'static` tasks. - scheduled_requeues: Arc>, - // Cache of tasks that recently completed with no changes (Ok(None)). - // Prevents repeated resolution of already-resolved items across playlist refreshes. - // Stores the reason set so that tasks with new/different reasons are not wrongly skipped. - recently_completed_no_change: HashMap, - resolve_enqueue_suppressions: Arc>, - tmdb_source_markers: Arc>, - dirty_retry_state_keys: HashSet, -} - -#[derive(Debug, Clone, Copy)] -struct ProcessTaskOutcome { - task_changed: bool, - tmdb_pending: bool, - probe_pending: bool, -} - /// Generate `collect_{vod,series,live}_virtual_updates` from a single body. /// The three methods differ only in the `BatchResultCollector` field they /// walk (`vod` / `series` / `live`), the per-kind stream properties type, /// and the `PlaylistItemType` discriminator used to build provider UUIDs. macro_rules! collect_virtual_updates { ($fn_name:ident, $field:ident, $props_ty:ty, $item_type:expr) => { - fn $fn_name<'a>( + pub(super) fn $fn_name<'a>( mapping: &TargetIdMapping, input_name: &str, batch: &'a BatchResultCollector, @@ -1134,7 +130,7 @@ macro_rules! collect_virtual_updates { /// used in error and log messages. macro_rules! apply_cascade_updates { ($fn_name:ident, $props_ty:ty, $cluster:expr, $variant:ident, $log_label:literal) => { - async fn $fn_name( + pub(super) async fn $fn_name( ctx: &MetadataUpdateCtx, target: &tuliprox_core::model::ConfigTarget, storage_path: &std::path::Path, @@ -1200,3293 +196,24 @@ macro_rules! apply_cascade_updates { }; } -impl InputWorker { - #[allow(clippy::too_many_lines)] - async fn run(mut self) { - debug!("Metadata worker started for input {}", self.input_name); - - let mut processed_vod_count: usize = 0; - let mut processed_series_count: usize = 0; - let mut last_queue_log_at = Instant::now(); - let mut last_progress_log_at = Instant::now(); - let mut queue_cycle_active = false; - let mut cycle_had_changes = false; - // Tasks that burned through their retries this cycle. A cycle that - // exhausts work is the only signal an operator gets that an input has - // stopped resolving - `InputMetadataUpdatesCompleted` fires only when - // something actually changed, so a permanently broken input would - // otherwise emit a start and then nothing at all. - let mut cycle_failed_tasks: usize = 0; - let mut cycle_last_error: Option = None; - let mut consecutive_resolve_tasks = 0_usize; - - let input_name = self.input_name.clone(); - let bound_ctx = self.bound_ctx.clone(); - - let mut runtime_settings = self.runtime_settings(); - let mut last_runtime_settings_refresh_at = Instant::now(); - self.ensure_metadata_retry_state_loaded(&input_name, bound_ctx.as_ref(), &runtime_settings).await; - - // Keep one prefetched task to minimize channel waits/lock churn. - let mut next_task: Option<(TaskKey, UpdateTask, u64)> = None; - - loop { - if last_runtime_settings_refresh_at.elapsed() >= Duration::from_secs(RUNTIME_SETTINGS_REFRESH_INTERVAL_SECS) - { - runtime_settings = self.runtime_settings(); - last_runtime_settings_refresh_at = Instant::now(); - } - let prefer_probe = consecutive_resolve_tasks >= runtime_settings.probe_fairness_resolve_burst; - let task_data = if let Some(prefetched) = next_task.take() { - if prefer_probe && Self::retry_domain_for_task(&prefetched.1) == RetryDomain::Resolve { - if let Some(probe_task) = self.take_pending_probe_task_snapshot() { - next_task = Some(prefetched); - Some(probe_task) - } else { - Some(prefetched) - } - } else { - Some(prefetched) - } - } else if prefer_probe { - if let Some(probe_task) = self.take_pending_probe_task_snapshot() { - Some(probe_task) - } else { - self.recv_task_fast_or_wait(&runtime_settings).await - } - } else { - self.recv_task_fast_or_wait(&runtime_settings).await - }; - - let Some((current_key, current_task, current_generation)) = task_data else { break }; - if self.cancel_token.is_cancelled() { - break; - } - if !self.metadata_retry_loaded { - self.ensure_metadata_retry_state_loaded(&input_name, bound_ctx.as_ref(), &runtime_settings).await; - } - let now_ts = chrono::Utc::now().timestamp(); - self.prune_retry_tracking_maps_if_needed(now_ts, &runtime_settings).await; - - if !queue_cycle_active { - // First entry of a new processing cycle. - queue_cycle_active = true; - cycle_had_changes = false; - cycle_failed_tasks = 0; - cycle_last_error = None; - processed_vod_count = 0; - processed_series_count = 0; - last_progress_log_at = Instant::now(); - // Emit queue logs promptly for the new cycle. - last_queue_log_at = Instant::now() - .checked_sub(runtime_settings.queue_log_interval + Duration::from_secs(1)) - .unwrap_or_else(Instant::now); - if let Some(ctx) = bound_ctx.clone() { - ctx.events.emit(EventMessage::InputMetadataUpdatesStarted(input_name.clone())); - } - if self.last_cycle_completed_at_ts.is_some_and(|last| { - now_ts.saturating_sub(last) >= runtime_settings.resolve_exhaustion_reset_gap_secs - }) { - self.resolve_exhausted.clear(); - } - debug!("Background metadata update queue has entries for input {input_name}; starting processing"); - } - - let current_retry_domain = Self::retry_domain_for_task(¤t_task); - let delay_secs = current_task.delay(); - let mut schedule_requeue_at_ts: Option = None; - let mut remove_current_task = false; - let mut apply_rate_limit = false; - let mut metadata_persist_state: Option> = None; - let mut skip_execution = false; - let mut task_for_execution = current_task.clone(); - - if Self::is_resolve_task(&task_for_execution) && self.resolve_exhausted.contains_key(¤t_key) { - debug!( - "[Metadata-Task] Skipping task (resolve exhausted) for input {}: {} (reset window: {}s)", - input_name, task_for_execution, runtime_settings.resolve_exhaustion_reset_gap_secs - ); - self.scheduled_requeues.remove(¤t_key); - remove_current_task = true; - skip_execution = true; - } - - // Skip tasks that recently completed with no changes (already resolved in DB). - // Only skip when the incoming reason set exactly matches the cached reason set. - if !skip_execution - && self.should_skip_recent_no_change_task(¤t_key, &task_for_execution, &runtime_settings) - { - debug!( - "[Metadata-Task] Skipping task (recently completed with no changes) for input {input_name}: {task_for_execution}", - ); - remove_current_task = true; - skip_execution = true; - } - - if !skip_execution { - let mut clear_tmdb_state = false; - if let Some(state_bundle) = self.retry_states.get(¤t_key) { - if let Some(tmdb_state) = state_bundle.get(RetryDomain::Tmdb) { - let task_has_tmdb_reason = Self::task_has_tmdb_reason(&task_for_execution); - let source_unchanged = tmdb_state.source_last_modified.is_some() - && tmdb_state.source_last_modified == Self::task_source_last_modified(&task_for_execution); - if source_unchanged && task_has_tmdb_reason { - if let Some(stripped_task) = Self::strip_tmdb_reasons(&task_for_execution) { - debug!( - "[Task] TMDB/date unchanged for input {}: {}, continuing with non-TMDB reasons (last_modified={:?})", - input_name, - task_for_execution, - tmdb_state.source_last_modified - ); - task_for_execution = stripped_task; - } else { - debug!( - "[Metadata-Task] Skipping task (TMDB/date unresolved and source unchanged) for input {}: {} (last_modified={:?})", - input_name, - task_for_execution, - tmdb_state.source_last_modified - ); - self.scheduled_requeues.remove(¤t_key); - remove_current_task = true; - skip_execution = true; - } - } else if task_has_tmdb_reason { - if let Some(cooldown_until_ts) = tmdb_state.cooldown_until_ts { - if now_ts < cooldown_until_ts { - if let Some(stripped_task) = Self::strip_tmdb_reasons(&task_for_execution) { - debug!( - "[Task] TMDB cooldown active for input {}: {}, continuing with non-TMDB reasons (cooldown_until={}, remaining={}s)", - input_name, - task_for_execution, - cooldown_until_ts, - cooldown_until_ts.saturating_sub(now_ts) - ); - task_for_execution = stripped_task; - } else { - debug!( - "[Metadata-Task] Skipping task (TMDB-only in cooldown) for input {}: {} (cooldown_until={}, remaining={}s)", - input_name, - task_for_execution, - cooldown_until_ts, - cooldown_until_ts.saturating_sub(now_ts) - ); - self.scheduled_requeues.remove(¤t_key); - remove_current_task = true; - skip_execution = true; - } - } else { - clear_tmdb_state = true; - } - } - } - } - } - - let active_retry_domain = Self::retry_domain_for_task(&task_for_execution); - let mut clear_active_retry_state = false; - - if !skip_execution { - if let Some(state_bundle) = self.retry_states.get(¤t_key) { - if let Some(active_state) = state_bundle.get(active_retry_domain) { - if let Some(cooldown_until_ts) = active_state.cooldown_until_ts { - if now_ts < cooldown_until_ts { - let cooldown_label = - if active_retry_domain == RetryDomain::Probe { "probe" } else { "resolve" }; - debug!( - "[Metadata-Task] Skipping task ({} cooldown) for input {}: {} (cooldown_until={}, remaining={}s)", - cooldown_label, - input_name, - task_for_execution, - cooldown_until_ts, - cooldown_until_ts.saturating_sub(now_ts) - ); - self.scheduled_requeues.remove(¤t_key); - remove_current_task = true; - skip_execution = true; - } else { - clear_active_retry_state = true; - } - } - - if !skip_execution && active_state.next_allowed_at_ts > now_ts { - debug!( - "[Task] Deferring task (retry backoff) for input {}: {} (next_allowed_at={}, wait={}s, attempts={})", - input_name, - task_for_execution, - active_state.next_allowed_at_ts, - active_state.next_allowed_at_ts.saturating_sub(now_ts), - active_state.attempts - ); - schedule_requeue_at_ts = Some(active_state.next_allowed_at_ts); - skip_execution = true; - } - } - } - } - - if clear_active_retry_state || clear_tmdb_state { - let mut should_remove_retry_entry = false; - let mut state_after_clear: Option = None; - - if let Some(state_bundle) = self.retry_states.get_mut(¤t_key) { - if clear_active_retry_state { - state_bundle.clear_domain(active_retry_domain); - } - if clear_tmdb_state { - state_bundle.clear_domain(RetryDomain::Tmdb); - } - if state_bundle.is_empty() { - should_remove_retry_entry = true; - } else { - state_bundle.touch(now_ts); - state_after_clear = Some(state_bundle.clone()); - } - } - - if should_remove_retry_entry { - self.retry_states.remove(¤t_key); - } - if clear_active_retry_state && active_retry_domain == RetryDomain::Resolve { - self.clear_resolve_enqueue_suppression(¤t_key); - } - if clear_tmdb_state { - self.clear_tmdb_source_marker(¤t_key); - } - metadata_persist_state = Some(state_after_clear); - } - } - - if !skip_execution { - debug!( - "[Task] Executing task for input {}: {} (retry_domain={:?})", - input_name, - task_for_execution, - Self::retry_domain_for_task(&task_for_execution) - ); - let task_result = { - let Some(_pause_guard) = self.wait_for_update_pause_window().await else { - break; - }; - Self::process_task_static( - &input_name, - bound_ctx.as_ref(), - &task_for_execution, - &mut self.batch_buffer, - &mut self.db_handles, - &mut self.failed_clusters, - ) - .await - }; - - match task_result { - Ok(task_outcome) => { - if Self::is_vod_task_key(¤t_key) { - processed_vod_count += 1; - } else if Self::is_series_task_key(¤t_key) { - processed_series_count += 1; - } - let trigger_playlist_update = Self::should_trigger_playlist_update_for_task( - &task_for_execution, - task_outcome.task_changed, - ); - cycle_had_changes |= trigger_playlist_update; - debug!( - "[Metadata-Task] Task succeeded for input {input_name}: {task_for_execution} (changed={}, trigger_playlist_update={}, tmdb_pending={}, probe_pending={})", - task_outcome.task_changed, - trigger_playlist_update, - task_outcome.tmdb_pending, - task_outcome.probe_pending - ); - - let active_retry_domain = Self::retry_domain_for_task(&task_for_execution); - let task_has_tmdb_reason = Self::task_has_tmdb_reason(&task_for_execution); - let task_source_last_modified = Self::task_source_last_modified(&task_for_execution); - let mut should_remove_retry_entry = false; - let mut clear_tmdb_marker = false; - let mut state_after_success: Option = None; - - if let Some(state_bundle) = self.retry_states.get_mut(¤t_key) { - state_bundle.clear_domain(active_retry_domain); - if task_has_tmdb_reason { - if task_outcome.tmdb_pending { - let cooldown_until_ts = now_ts.saturating_add(runtime_settings.tmdb_cooldown_secs); - let tmdb_state = state_bundle.get_mut_or_insert(RetryDomain::Tmdb); - tmdb_state.attempts = 0; - tmdb_state.next_allowed_at_ts = cooldown_until_ts; - tmdb_state.cooldown_until_ts = Some(cooldown_until_ts); - tmdb_state.last_error = - Some("TMDB lookup completed without matching result".to_string()); - tmdb_state.source_last_modified = task_source_last_modified; - debug!( - "[Metadata-Task] TMDB resolve produced no match (existing retry state), entering cooldown for input {}: {} (cooldown_until={}, cooldown_duration={}s)", - input_name, - task_for_execution, - cooldown_until_ts, - runtime_settings.tmdb_cooldown_secs - ); - } else { - state_bundle.clear_domain(RetryDomain::Tmdb); - clear_tmdb_marker = true; - } - } - - if state_bundle.is_empty() { - should_remove_retry_entry = true; - } else { - state_bundle.touch(now_ts); - state_after_success = Some(state_bundle.clone()); - } - } else if task_has_tmdb_reason && task_outcome.tmdb_pending { - let cooldown_until_ts = now_ts.saturating_add(runtime_settings.tmdb_cooldown_secs); - let state_bundle = TaskRetryState { - resolve: None, - probe: None, - tmdb: Some(RetryState { - attempts: 0, - next_allowed_at_ts: cooldown_until_ts, - cooldown_until_ts: Some(cooldown_until_ts), - last_error: Some("TMDB lookup completed without matching result".to_string()), - source_last_modified: task_source_last_modified, - }), - updated_at_ts: now_ts.max(1), - }; - self.retry_states.insert(current_key.clone(), state_bundle.clone()); - state_after_success = Some(state_bundle); - debug!( - "[Metadata-Task] TMDB resolve produced no match (new retry state), entering cooldown for input {}: {} (cooldown_until={}, cooldown_duration={}s)", - input_name, - task_for_execution, - cooldown_until_ts, - runtime_settings.tmdb_cooldown_secs - ); - } else if task_has_tmdb_reason { - self.clear_tmdb_source_marker(¤t_key); - } - - if should_remove_retry_entry { - self.retry_states.remove(¤t_key); - } - if clear_tmdb_marker { - self.clear_tmdb_source_marker(¤t_key); - } - if task_outcome.tmdb_pending { - if let Some(source_last_modified) = task_source_last_modified { - self.set_tmdb_source_marker(¤t_key, source_last_modified); - } else { - self.clear_tmdb_source_marker(¤t_key); - } - } - if should_remove_retry_entry || state_after_success.is_some() { - metadata_persist_state = Some(state_after_success); - } - - self.resolve_exhausted.remove(¤t_key); - if active_retry_domain == RetryDomain::Resolve { - self.clear_resolve_enqueue_suppression(¤t_key); - } - self.scheduled_requeues.remove(¤t_key); - - // Cache tasks that completed with no changes to skip redundant re-resolution. - if !task_outcome.task_changed - && Self::is_resolve_task(&task_for_execution) - && !task_outcome.tmdb_pending - && !task_outcome.probe_pending - { - let reasons = Self::task_reason(&task_for_execution); - debug!( - "[Task] Caching no-change result for input {}: {} (reasons={}, ttl={}s)", - input_name, task_for_execution, reasons, runtime_settings.no_change_cache_ttl_secs - ); - self.recently_completed_no_change.insert(current_key.clone(), (Instant::now(), reasons)); - // Set producer-side enqueue suppression so that future playlist - // processing cycles do not re-queue this task during the TTL. - let suppressed_until_ts = now_ts.saturating_add( - i64::try_from(runtime_settings.no_change_cache_ttl_secs).unwrap_or(i64::MAX), - ); - self.set_resolve_enqueue_suppression(¤t_key, suppressed_until_ts); - } else { - self.recently_completed_no_change.remove(¤t_key); - } - - if last_progress_log_at.elapsed() >= runtime_settings.progress_log_interval { - // current_key is removed from pending_tasks later in this loop iteration; - // subtract it here so "remaining" reflects the post-success queue size. - let (mut remaining_vod, mut remaining_series) = - Self::queue_resolve_counts(&self.pending_tasks); - if Self::is_vod_task_key(¤t_key) { - remaining_vod = remaining_vod.saturating_sub(1); - } else if Self::is_series_task_key(¤t_key) { - remaining_series = remaining_series.saturating_sub(1); - } - - let total_vod = processed_vod_count.saturating_add(remaining_vod); - let total_series = processed_series_count.saturating_add(remaining_series); - let resolved_total = processed_vod_count.saturating_add(processed_series_count); - let total_resolve = total_vod.saturating_add(total_series); - - info!("Background metadata update: {resolved_total} / {total_resolve} resolved for input {input_name} (vod: {processed_vod_count}/{total_vod}, series: {processed_series_count}/{total_series})"); - last_progress_log_at = Instant::now(); - } - - remove_current_task = true; - apply_rate_limit = true; - } - Err(e) => { - if Self::is_permanent_not_found_error(e.message()) { - debug!( - "[Task] Task failed with permanent not-found for input {}: {} (error={})", - input_name, - task_for_execution, - e.message() - ); - let retry_domain = Self::retry_domain_for_task(&task_for_execution); - self.scheduled_requeues.remove(¤t_key); - if retry_domain == RetryDomain::Probe { - let cooldown_until_ts = now_ts.saturating_add(runtime_settings.probe_cooldown_secs); - let state_bundle_after_update = { - let state_bundle = self.retry_states.entry(current_key.clone()).or_default(); - let state = state_bundle.get_mut_or_insert(RetryDomain::Probe); - state.attempts = runtime_settings.max_attempts_probe; - state.next_allowed_at_ts = cooldown_until_ts; - state.cooldown_until_ts = Some(cooldown_until_ts); - state.last_error = Some(e.message().to_string()); - state_bundle.touch(now_ts); - state_bundle.clone() - }; - metadata_persist_state = Some(Some(state_bundle_after_update)); - remove_current_task = true; - debug!( - "[Metadata-Task] Probe task entering cooldown after permanent not-found for input {}: {} (cooldown_until={}, cooldown_duration={}s)", - input_name, - task_for_execution, - cooldown_until_ts, - runtime_settings.probe_cooldown_secs - ); - } else { - let cooldown_until_ts = - now_ts.saturating_add(runtime_settings.resolve_exhaustion_reset_gap_secs); - let state_bundle_after_update = { - let state_bundle = self.retry_states.entry(current_key.clone()).or_default(); - let state = state_bundle.get_mut_or_insert(RetryDomain::Resolve); - state.attempts = runtime_settings.max_attempts_resolve; - state.next_allowed_at_ts = cooldown_until_ts; - state.cooldown_until_ts = Some(cooldown_until_ts); - state.last_error = Some(e.message().to_string()); - state_bundle.touch(now_ts); - state_bundle.clone() - }; - self.set_resolve_enqueue_suppression(¤t_key, cooldown_until_ts); - self.resolve_exhausted.insert(current_key.clone(), now_ts); - metadata_persist_state = Some(Some(state_bundle_after_update)); - remove_current_task = true; - debug!( - "[Metadata-Task] Resolve task entering cooldown after permanent not-found for input {}: {} (cooldown_until={}, cooldown_duration={}s, error={})", - input_name, - task_for_execution, - cooldown_until_ts, - runtime_settings.resolve_exhaustion_reset_gap_secs, - e.message() - ); - } - } else if Self::is_transient_worker_error(e.message()) { - if e.message() == TASK_ERR_UPDATE_IN_PROGRESS { - // Drop cached readers quickly so foreground writer can progress. - self.release_db_handles(); - } - - let retry_delay_secs = - Self::compute_retry_delay_secs(current_task.delay(), &runtime_settings); - let retry_delay_i64 = i64::try_from(retry_delay_secs).unwrap_or(i64::MAX); - schedule_requeue_at_ts = Some(now_ts.saturating_add(retry_delay_i64)); - debug!( - "[Metadata-Task] Task deferred (transient error) for input {}: {} (retry_in={}s, error={})", - input_name, - task_for_execution, - retry_delay_secs, - e.message() - ); - } else { - let retry_domain = Self::retry_domain_for_task(&task_for_execution); - let max_attempts = if retry_domain == RetryDomain::Probe { - runtime_settings.max_attempts_probe - } else { - runtime_settings.max_attempts_resolve - }; - - let (state_after_update, state_bundle_after_update) = { - let state_bundle = self.retry_states.entry(current_key.clone()).or_default(); - let state_after_update = { - let state = state_bundle.get_mut_or_insert(retry_domain); - state.attempts = state.attempts.saturating_add(1); - state.last_error = Some(e.message().to_string()); - - if state.attempts < max_attempts { - let backoff_secs = if retry_domain == RetryDomain::Probe { - Self::compute_probe_retry_backoff_secs(state.attempts, &runtime_settings) - } else { - Self::compute_resolve_retry_backoff_secs( - task_for_execution.delay(), - state.attempts, - &runtime_settings, - ) - }; - let backoff_i64 = i64::try_from(backoff_secs).unwrap_or(i64::MAX); - state.next_allowed_at_ts = now_ts.saturating_add(backoff_i64); - state.cooldown_until_ts = None; - } else if retry_domain == RetryDomain::Probe { - state.cooldown_until_ts = - Some(now_ts.saturating_add(runtime_settings.probe_cooldown_secs)); - state.next_allowed_at_ts = state.cooldown_until_ts.unwrap_or(now_ts); - } - state.clone() - }; - state_bundle.touch(now_ts); - - (state_after_update, state_bundle.clone()) - }; - - let attempts = state_after_update.attempts; - - if attempts >= max_attempts { - cycle_failed_tasks = cycle_failed_tasks.saturating_add(1); - cycle_last_error = Some(sanitize_sensitive_info(e.message()).into_owned()); - self.scheduled_requeues.remove(¤t_key); - if retry_domain == RetryDomain::Probe { - remove_current_task = true; - metadata_persist_state = Some(Some(state_bundle_after_update.clone())); - let cooldown_until = state_after_update - .cooldown_until_ts - .map_or_else(|| "none".to_string(), |ts| ts.to_string()); - debug!( - "Metadata-[Task] Probe task exhausted (max attempts reached) for input {}: {} (attempts={}/{}, cooldown_until={}, error={})", - input_name, - task_for_execution, - state_after_update.attempts, - max_attempts, - cooldown_until, - e.message() - ); - } else { - let cooldown_until_ts = - now_ts.saturating_add(runtime_settings.resolve_exhaustion_reset_gap_secs); - let state_bundle_after_update = { - let state_bundle = self.retry_states.entry(current_key.clone()).or_default(); - let state = state_bundle.get_mut_or_insert(RetryDomain::Resolve); - state.attempts = max_attempts; - state.next_allowed_at_ts = cooldown_until_ts; - state.cooldown_until_ts = Some(cooldown_until_ts); - state.last_error = Some(e.message().to_string()); - state_bundle.touch(now_ts); - state_bundle.clone() - }; - self.set_resolve_enqueue_suppression(¤t_key, cooldown_until_ts); - self.resolve_exhausted.insert(current_key.clone(), now_ts); - metadata_persist_state = Some(Some(state_bundle_after_update)); - remove_current_task = true; - debug!( - "[Metadata-Task] Resolve task exhausted (max attempts reached) for input {}: {} (attempts={}/{}, cooldown_duration={}s, error={})", - input_name, - task_for_execution, - attempts, - max_attempts, - runtime_settings.resolve_exhaustion_reset_gap_secs, - e.message() - ); - } - } else { - schedule_requeue_at_ts = Some(state_after_update.next_allowed_at_ts); - metadata_persist_state = Some(Some(state_bundle_after_update)); - if retry_domain == RetryDomain::Resolve { - self.set_resolve_enqueue_suppression( - ¤t_key, - state_after_update.next_allowed_at_ts, - ); - } - debug!( - "[Metadata-Task] Task failed, scheduling retry for input {}: {} (attempt={}/{}, next_allowed_at={}, backoff={}s, error={})", - input_name, - task_for_execution, - attempts, - max_attempts, - state_after_update.next_allowed_at_ts, - state_after_update.next_allowed_at_ts.saturating_sub(now_ts), - e.message() - ); - } - } - } - } - } - - if let Some(state) = metadata_persist_state { - self.dirty_retry_state_keys.insert(current_key.clone()); - self.persist_metadata_retry_state(¤t_key, state.as_ref()).await; - } - - // Check and flush batch - if self.batch_buffer.should_flush() { - self.release_db_handles(); - let Some(_pause_guard) = self.wait_for_update_pause_window().await else { - break; - }; - Self::flush_batch_static(&input_name, bound_ctx.as_ref(), &mut self.batch_buffer).await; - } - - if let Some(retry_at_ts) = schedule_requeue_at_ts { - debug!( - "[Task] Scheduling requeue for input {}: {:?} (retry_at_ts={}, in={}s)", - input_name, - current_key, - retry_at_ts, - retry_at_ts.saturating_sub(chrono::Utc::now().timestamp()) - ); - self.schedule_requeue_at(current_key.clone(), retry_at_ts); - } - - if remove_current_task { - self.finalize_processed_task_success(¤t_key, current_generation, &input_name).await; - } - - if apply_rate_limit { - // Rate limiting - if delay_secs > 0 - && Self::sleep_or_cancel(&self.cancel_token, Duration::from_secs(u64::from(delay_secs))).await - { - break; - } - } - - if current_retry_domain == RetryDomain::Resolve { - consecutive_resolve_tasks = consecutive_resolve_tasks.saturating_add(1); - } else { - consecutive_resolve_tasks = 0; - } - - // Try to get the next task immediately to keep locks open. - // Ignore phantom signals (channel key without pending map entry). - if next_task.is_none() { - if self.cancel_token.is_cancelled() { - break; - } - while let Ok(key) = self.receiver.try_recv() { - if self.cancel_token.is_cancelled() { - break; - } - if let Some(snapshot) = self.load_task_snapshot(key) { - next_task = Some(snapshot); - break; - } - } - } - - let channel_has_work = next_task.is_some() || !self.receiver.is_empty(); - let queue_completely_empty = - !channel_has_work && self.pending_tasks.is_empty() && self.scheduled_requeues.is_empty(); - - // Avoid O(n) queue scans per task; report queue status periodically. - if (channel_has_work || !self.pending_tasks.is_empty()) - && last_queue_log_at.elapsed() >= runtime_settings.queue_log_interval - { - let queue_counts = Self::queue_resolve_counts(&self.pending_tasks); - debug!("In queue to resolve vod: {}, series: {} (input: {input_name})", queue_counts.0, queue_counts.1); - last_queue_log_at = Instant::now(); - } - - // If no immediate work is available, flush buffered results now even when delayed retries remain pending. - if !channel_has_work && !self.batch_buffer.is_empty() { - self.release_db_handles(); - let Some(_pause_guard) = self.wait_for_update_pause_window().await else { - break; - }; - Self::flush_batch_static(&input_name, bound_ctx.as_ref(), &mut self.batch_buffer).await; - } - - if queue_cycle_active && queue_completely_empty { - self.last_cycle_completed_at_ts = Some(chrono::Utc::now().timestamp()); - queue_cycle_active = false; - processed_vod_count = 0; - processed_series_count = 0; - consecutive_resolve_tasks = 0; - if cycle_had_changes { - info!("All pending metadata resolves completed for input {input_name} (with changes)"); - if let Some(ctx) = bound_ctx.clone() { - ctx.events.emit(EventMessage::InputMetadataUpdatesCompleted(input_name.clone())); - } - } else { - debug!("All pending metadata resolves completed for input {input_name} (no changes, skipping playlist update trigger)"); - } - // Reported alongside the completion rather than instead of it: - // a cycle can both produce changes and exhaust tasks, and - // suppressing the completion would also suppress the playlist - // update it triggers. - if cycle_failed_tasks > 0 { - warn!( - "Metadata update cycle for input {input_name} exhausted {cycle_failed_tasks} task(s) without resolving them" - ); - if let Some(ctx) = bound_ctx.clone() { - ctx.events.emit(EventMessage::InputMetadataUpdatesFailed(MetadataUpdateFailure::new( - input_name.clone(), - cycle_failed_tasks, - cycle_had_changes, - cycle_last_error.clone(), - ))); - } - } - cycle_had_changes = false; - cycle_failed_tasks = 0; - cycle_last_error = None; - } - } - - // Final flush - self.release_db_handles(); - if !self.batch_buffer.is_empty() { - if let Some(_pause_guard) = self.wait_for_update_pause_window().await { - Self::flush_batch_static(&input_name, bound_ctx.as_ref(), &mut self.batch_buffer).await; - } - } - self.flush_dirty_retry_states().await; - - debug!("Metadata worker stopped for input {input_name}"); - } - - async fn ensure_metadata_retry_state_loaded( - &mut self, - input_name: &str, - bound_ctx: Option<&MetadataUpdateCtx>, - runtime_settings: &MetadataUpdateRuntimeSettings, - ) { - if self.metadata_retry_loaded { - return; - } - let now_ts = chrono::Utc::now().timestamp(); - if self.metadata_retry_load_retry_at_ts.is_some_and(|retry_at_ts| now_ts < retry_at_ts) { - return; - } - - let Some(ctx) = bound_ctx else { - self.metadata_retry_load_retry_at_ts = - Some(now_ts.saturating_add(runtime_settings.metadata_retry_load_retry_delay_secs)); - return; - }; - - let storage_dir = ctx.app_config.config.load().storage_dir.clone(); - let Ok(storage_path) = get_input_storage_path(input_name, &storage_dir).await else { - warn!("Could not resolve storage path for metadata retry state on input {input_name}"); - self.metadata_retry_load_retry_at_ts = - Some(now_ts.saturating_add(runtime_settings.metadata_retry_load_retry_delay_secs)); - return; - }; - - let retry_path = storage_path.join(METADATA_RETRY_STATE_FILE); - self.metadata_retry_state_path = Some(retry_path.clone()); - - let loaded = spawn_blocking_limited(move || load_metadata_retry_states_from_disk(&retry_path)) - .await - .map_err(|err| err.to_string()) - .and_then(|result| result.map_err(|err| err.to_string())); - - let loaded = match loaded { - Ok(states) => states, - Err(err) => { - warn!("Failed to load metadata retry state for input {input_name}: {err}"); - self.metadata_retry_load_retry_at_ts = - Some(now_ts.saturating_add(runtime_settings.metadata_retry_load_retry_delay_secs)); - return; - } - }; - - // Intentionally do not resurrect pending tasks solely from persisted retry state. - // The persisted state is applied once the corresponding task is naturally queued again - // (for example by the next playlist update), because state alone does not contain the - // full `UpdateTask` payload for all variants. - for (key, state) in loaded { - self.sync_resolve_enqueue_suppression_from_retry_state(&key, &state, now_ts); - self.retry_states.insert(key, state); - } - self.prune_retry_tracking_maps(now_ts, runtime_settings).await; - self.last_retry_state_prune_at_ts = Some(now_ts); - self.metadata_retry_loaded = true; - self.metadata_retry_load_retry_at_ts = None; - } - - async fn persist_metadata_retry_state(&mut self, key: &TaskKey, state: Option<&TaskRetryState>) { - let Some(path) = self.metadata_retry_state_path.clone() else { - return; - }; - if !self.dirty_retry_state_keys.contains(key) { - return; - } - - let key_for_persist = key.clone(); - let key_for_dirty = key.clone(); - let state = state.cloned(); - let input_name = self.input_name.clone(); - let persist_result = spawn_blocking_limited(move || { - persist_metadata_retry_state_to_disk(&path, &key_for_persist, state.as_ref()) - }) - .await; - - match persist_result { - Ok(Ok(())) => { - self.dirty_retry_state_keys.remove(&key_for_dirty); - } - Ok(Err(err)) => warn!("Failed to persist metadata retry state for input {input_name}: {err}"), - Err(err) => warn!("Failed to persist metadata retry state for input {input_name}: {err}"), - } - } - - async fn flush_dirty_retry_states(&mut self) { - if self.dirty_retry_state_keys.is_empty() { - return; - } - - let dirty_keys: Vec = self.dirty_retry_state_keys.iter().cloned().collect(); - for key in dirty_keys { - let state = self.retry_states.get(&key).cloned(); - self.persist_metadata_retry_state(&key, state.as_ref()).await; - } - } - - fn schedule_requeue_at(&self, key: TaskKey, retry_at_ts: i64) { - let now_ts = chrono::Utc::now().timestamp(); - let retry_at_ts = retry_at_ts.max(now_ts); - - if self.scheduled_requeues.get(&key).is_some_and(|existing| *existing == retry_at_ts) { - return; - } - - self.scheduled_requeues.insert(key.clone(), retry_at_ts); - - let sender = self.sender.clone(); - let pending_tasks = Arc::clone(&self.pending_tasks); - let pending_task_count = Arc::clone(&self.pending_task_count); - let scheduled = Arc::clone(&self.scheduled_requeues); - let cancel_token = self.cancel_token.clone(); - let input_name = self.input_name.clone(); - - tokio::spawn(async move { - let delay_secs = retry_at_ts.saturating_sub(chrono::Utc::now().timestamp()); - if delay_secs > 0 { - let delay = Duration::from_secs(u64::try_from(delay_secs).unwrap_or(u64::MAX)); - tokio::select! { - () = cancel_token.cancelled() => return, - () = tokio::time::sleep(delay) => {} - } - } - - let should_send = scheduled.get(&key).is_some_and(|scheduled_at| *scheduled_at == retry_at_ts); - if !should_send { - return; - } - scheduled.remove(&key); - - if !pending_tasks.contains_key(&key) { - return; - } - - if sender.send(key.clone()).await.is_err() { - if pending_tasks.remove(&key).is_some() { - MetadataUpdateManager::decrement_pending_task_count(&pending_task_count); - } - warn!("Failed to schedule delayed retry task for input {input_name}"); - } - }); - } - - async fn finalize_processed_task_success( - &mut self, - current_key: &TaskKey, - current_generation: u64, - input_name: &str, - ) -> bool { - // Atomically remove processed key and reinsert it if it changed while in-flight. - if let Some((_k, removed_task)) = self.pending_tasks.remove(current_key) { - let latest_generation = removed_task.generation.load(Ordering::Relaxed); - if latest_generation != current_generation { - self.pending_tasks.insert(current_key.clone(), removed_task); - if self.sender.send(current_key.clone()).await.is_err() { - if self.pending_tasks.remove(current_key).is_some() { - MetadataUpdateManager::decrement_pending_task_count(&self.pending_task_count); - } - warn!("Failed to schedule merged task replay for input {input_name}"); - return false; - } - return true; - } - // Task finished and is not reinserted. - MetadataUpdateManager::decrement_pending_task_count(&self.pending_task_count); - } - false - } - - async fn recv_task_fast_or_wait( - &mut self, - runtime_settings: &MetadataUpdateRuntimeSettings, - ) -> Option<(TaskKey, UpdateTask, u64)> { - if self.cancel_token.is_cancelled() { - return None; - } - - // Fast path: drain immediate signals until we find a real pending task. - loop { - if self.cancel_token.is_cancelled() { - return None; - } - match self.receiver.try_recv() { - Ok(key) => { - if self.cancel_token.is_cancelled() { - return None; - } - if let Some(snapshot) = self.load_task_snapshot(key) { - return Some(snapshot); - } - } - Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => { - return None; - } - Err(tokio::sync::mpsc::error::TryRecvError::Empty) => { - break; - } - } - } - - // When idle, release read handles to avoid writer starvation. - self.release_db_handles(); - - loop { - tokio::select! { - biased; - () = self.cancel_token.cancelled() => return None, - res = tokio::time::timeout(Duration::from_secs(runtime_settings.worker_idle_timeout_secs), self.receiver.recv()) => { - match res { - Ok(Some(key)) => { - if self.cancel_token.is_cancelled() { - return None; - } - if let Some(snapshot) = self.load_task_snapshot(key) { - return Some(snapshot); - } - } - Ok(None) => return None, - Err(_) => { - loop { - if self.cancel_token.is_cancelled() { - return None; - } - match self.receiver.try_recv() { - Ok(key) => { - if self.cancel_token.is_cancelled() { - return None; - } - if let Some(snapshot) = self.load_task_snapshot(key) { - return Some(snapshot); - } - } - Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => return None, - Err(tokio::sync::mpsc::error::TryRecvError::Empty) => break, - } - } - if self.pending_tasks.is_empty() - && self.receiver.is_empty() - && self.scheduled_requeues.is_empty() - { - return None; - } - } - } - } - } - } - } - - fn load_task_snapshot(&self, key: TaskKey) -> Option<(TaskKey, UpdateTask, u64)> { - let entry = self.pending_tasks.get(&key)?; - let generation = entry.generation.load(Ordering::Relaxed); - let task = entry.task.lock().clone(); - Some((key, task, generation)) - } - - fn take_pending_probe_task_snapshot(&self) -> Option<(TaskKey, UpdateTask, u64)> { - for entry in self.pending_tasks.iter() { - if self.scheduled_requeues.contains_key(entry.key()) { - continue; - } - - let generation = entry.generation.load(Ordering::Relaxed); - let task = entry.task.lock().clone(); - if Self::retry_domain_for_task(&task) == RetryDomain::Probe { - return Some((entry.key().clone(), task, generation)); - } - } - - None - } - - fn runtime_settings(&self) -> MetadataUpdateRuntimeSettings { - MetadataUpdateRuntimeSettings::from_ctx(self.bound_ctx.as_ref()) - } - - async fn wait_for_update_pause_window(&mut self) -> Option> { - if let Ok(guard) = self.update_pause_gate.clone().try_read_owned() { - return Some(guard); - } - - // Foreground writer is active or queued. Release cached handles before waiting to avoid AB-BA patterns. - self.release_db_handles(); - tokio::select! { - () = self.cancel_token.cancelled() => None, - guard = self.update_pause_gate.clone().read_owned() => { - Some(guard) - } - } - } - - fn retry_state_ttl_secs(runtime_settings: &MetadataUpdateRuntimeSettings) -> i64 { - let max_resolve_backoff = i64::try_from(runtime_settings.max_resolve_retry_backoff_secs).unwrap_or(i64::MAX); - runtime_settings - .tmdb_cooldown_secs - .max(runtime_settings.probe_cooldown_secs) - .max(runtime_settings.resolve_exhaustion_reset_gap_secs) - .max(max_resolve_backoff) - .saturating_mul(6) - .max(RETRY_STATE_MIN_TTL_SECS) - } - - fn resolve_exhausted_ttl_secs(runtime_settings: &MetadataUpdateRuntimeSettings) -> i64 { - runtime_settings.resolve_exhaustion_reset_gap_secs.saturating_mul(6).max(RETRY_STATE_MIN_TTL_SECS) - } - - async fn prune_retry_tracking_maps_if_needed( - &mut self, - now_ts: i64, - runtime_settings: &MetadataUpdateRuntimeSettings, - ) { - let due = self - .last_retry_state_prune_at_ts - .is_none_or(|last| now_ts.saturating_sub(last) >= RETRY_STATE_PRUNE_INTERVAL_SECS); - if !due { - return; - } - self.last_retry_state_prune_at_ts = Some(now_ts); - self.prune_retry_tracking_maps(now_ts, runtime_settings).await; - } - - async fn prune_retry_tracking_maps(&mut self, now_ts: i64, runtime_settings: &MetadataUpdateRuntimeSettings) { - let retry_state_ttl_secs = Self::retry_state_ttl_secs(runtime_settings); - let resolve_exhausted_ttl_secs = Self::resolve_exhausted_ttl_secs(runtime_settings); - let stale_retry_keys: Vec = self - .retry_states - .iter() - .filter_map( - |(key, state)| { - if state.is_stale(now_ts, retry_state_ttl_secs) { - Some(key.clone()) - } else { - None - } - }, - ) - .collect(); - - for key in stale_retry_keys { - if self.retry_states.remove(&key).is_some() { - self.clear_resolve_enqueue_suppression(&key); - self.clear_tmdb_source_marker(&key); - self.dirty_retry_state_keys.insert(key.clone()); - debug_if_enabled!("Pruned stale metadata retry state for input {}: {:?}", self.input_name, key); - self.persist_metadata_retry_state(&key, None).await; - } - } - - let stale_resolve_exhausted_keys: Vec = self - .resolve_exhausted - .iter() - .filter_map(|(key, exhausted_at_ts)| { - if now_ts.saturating_sub(*exhausted_at_ts) >= resolve_exhausted_ttl_secs { - Some(key.clone()) - } else { - None - } - }) - .collect(); - for key in stale_resolve_exhausted_keys { - self.resolve_exhausted.remove(&key); - self.clear_resolve_enqueue_suppression(&key); - } - - let no_change_ttl = Duration::from_secs(runtime_settings.no_change_cache_ttl_secs); - self.recently_completed_no_change.retain(|_, (completed_at, _)| completed_at.elapsed() < no_change_ttl); - } - - fn release_db_handles(&mut self) { - if !self.db_handles.is_empty() { - self.db_handles.clear(); - } - if !self.failed_clusters.is_empty() { - self.failed_clusters.clear(); - } - } - - fn compute_resolve_retry_backoff_secs( - base_delay_secs: u16, - attempts: u8, - runtime_settings: &MetadataUpdateRuntimeSettings, - ) -> u64 { - let base_delay = u64::from(base_delay_secs).max(runtime_settings.resolve_min_retry_base_secs); - let exp = u32::from(attempts.saturating_sub(1).min(6)); - let without_jitter = - base_delay.saturating_mul(2_u64.saturating_pow(exp)).min(runtime_settings.max_resolve_retry_backoff_secs); - Self::apply_jitter(without_jitter, runtime_settings.backoff_jitter_percent) - } - - fn compute_retry_delay_secs(base_delay_secs: u16, runtime_settings: &MetadataUpdateRuntimeSettings) -> u64 { - u64::from(base_delay_secs).max(runtime_settings.retry_delay_secs) - } - - fn compute_probe_retry_backoff_secs(attempts: u8, runtime_settings: &MetadataUpdateRuntimeSettings) -> u64 { - let base_secs = match attempts { - 1 => runtime_settings.probe_retry_backoff_step_1_secs, - 2 => runtime_settings.probe_retry_backoff_step_2_secs, - _ => runtime_settings.probe_retry_backoff_step_3_secs, - }; - Self::apply_jitter(base_secs, runtime_settings.backoff_jitter_percent) - } - - fn apply_jitter(base_secs: u64, jitter_percent: u8) -> u64 { - let jitter_percent = i64::from(jitter_percent); - let jitter_percent = fastrand::i64(-jitter_percent..=jitter_percent); - let base_i64 = i64::try_from(base_secs).unwrap_or(i64::MAX); - let jitter_delta = base_i64.saturating_mul(jitter_percent).saturating_div(100); - let jittered = base_i64.saturating_add(jitter_delta); - u64::try_from(jittered.max(1)).unwrap_or(1) - } - - async fn sleep_or_cancel(cancel_token: &CancellationToken, duration: Duration) -> bool { - tokio::select! { - () = cancel_token.cancelled() => true, - () = tokio::time::sleep(duration) => false, - } - } - - fn queue_resolve_counts(pending_tasks: &DashMap) -> (usize, usize) { - let mut vod_count = 0_usize; - let mut series_count = 0_usize; - - for entry in pending_tasks { - match entry.key() { - TaskKey::Vod(_) | TaskKey::VodStr(_) => vod_count += 1, - TaskKey::Series(_) | TaskKey::SeriesStr(_) => series_count += 1, - _ => {} - } - } - - (vod_count, series_count) - } - - #[inline] - fn is_vod_task_key(key: &TaskKey) -> bool { matches!(key, TaskKey::Vod(_) | TaskKey::VodStr(_)) } - - #[inline] - fn is_series_task_key(key: &TaskKey) -> bool { matches!(key, TaskKey::Series(_) | TaskKey::SeriesStr(_)) } - - #[inline] - fn is_probe_task(task: &UpdateTask) -> bool { - matches!(task, UpdateTask::ProbeLive { .. } | UpdateTask::ProbeStream { .. }) - } - - #[inline] - fn is_probe_only_resolve_task(task: &UpdateTask) -> bool { - match task { - UpdateTask::ResolveVod { reason, .. } | UpdateTask::ResolveSeries { reason, .. } => { - reason.contains(ResolveReason::Probe) - && !reason.contains(ResolveReason::Info) - && !reason.contains(ResolveReason::Tmdb) - && !reason.contains(ResolveReason::Date) - } - _ => false, - } - } - - #[inline] - fn retry_domain_for_task(task: &UpdateTask) -> RetryDomain { - if Self::is_probe_task(task) || Self::is_probe_only_resolve_task(task) { - RetryDomain::Probe - } else { - RetryDomain::Resolve - } - } - - #[inline] - fn is_resolve_task(task: &UpdateTask) -> bool { - matches!(task, UpdateTask::ResolveVod { .. } | UpdateTask::ResolveSeries { .. }) - } - - #[inline] - fn task_reason(task: &UpdateTask) -> ResolveReasonSet { - match task { - UpdateTask::ResolveVod { reason, .. } - | UpdateTask::ResolveSeries { reason, .. } - | UpdateTask::ProbeLive { reason, .. } - | UpdateTask::ProbeStream { reason, .. } => *reason, - } - } - - fn should_skip_recent_no_change_task( - &mut self, - current_key: &TaskKey, - task_for_execution: &UpdateTask, - runtime_settings: &MetadataUpdateRuntimeSettings, - ) -> bool { - let Some((completed_at, cached_reasons)) = self.recently_completed_no_change.get(current_key).copied() else { - return false; - }; - - let ttl = Duration::from_secs(runtime_settings.no_change_cache_ttl_secs); - let current_reasons = Self::task_reason(task_for_execution); - if completed_at.elapsed() < ttl && current_reasons == cached_reasons { - self.scheduled_requeues.remove(current_key); - return true; - } - - self.recently_completed_no_change.remove(current_key); - false - } - - fn set_resolve_enqueue_suppression(&self, current_key: &TaskKey, until_ts: i64) { - let scoped_key = ScopedTaskKey::new(self.input_name.clone(), current_key.clone()); - self.resolve_enqueue_suppressions.insert(scoped_key, until_ts); - } - - fn clear_resolve_enqueue_suppression(&self, current_key: &TaskKey) { - let scoped_key = ScopedTaskKey::new(self.input_name.clone(), current_key.clone()); - self.resolve_enqueue_suppressions.remove(&scoped_key); - } - - fn set_tmdb_source_marker(&self, current_key: &TaskKey, source_last_modified: u64) { - let scoped_key = ScopedTaskKey::new(self.input_name.clone(), current_key.clone()); - self.tmdb_source_markers.insert(scoped_key, source_last_modified); - } - - fn clear_tmdb_source_marker(&self, current_key: &TaskKey) { - let scoped_key = ScopedTaskKey::new(self.input_name.clone(), current_key.clone()); - self.tmdb_source_markers.remove(&scoped_key); - } - - fn sync_resolve_enqueue_suppression_from_retry_state( - &self, - current_key: &TaskKey, - state: &TaskRetryState, - now_ts: i64, - ) { - let Some(resolve_state) = state.resolve.as_ref() else { - self.clear_resolve_enqueue_suppression(current_key); - return; - }; - - let suppressed_until_ts = resolve_state.cooldown_until_ts.unwrap_or(resolve_state.next_allowed_at_ts); - if suppressed_until_ts > now_ts { - self.set_resolve_enqueue_suppression(current_key, suppressed_until_ts); - } else { - self.clear_resolve_enqueue_suppression(current_key); - } - } - - #[inline] - fn should_trigger_playlist_update_for_task(task: &UpdateTask, task_changed: bool) -> bool { - task_changed && !Self::is_probe_task(task) && !Self::is_probe_only_resolve_task(task) - } - - #[inline] - fn task_has_tmdb_reason(task: &UpdateTask) -> bool { - match task { - UpdateTask::ResolveVod { reason, .. } | UpdateTask::ResolveSeries { reason, .. } => { - reason.contains(ResolveReason::Tmdb) || reason.contains(ResolveReason::Date) - } - _ => false, - } - } - - #[inline] - fn task_source_last_modified(task: &UpdateTask) -> Option { - match task { - UpdateTask::ResolveVod { source_last_modified, .. } - | UpdateTask::ResolveSeries { source_last_modified, .. } => *source_last_modified, - _ => None, - } - } - - fn strip_tmdb_reasons(task: &UpdateTask) -> Option { - match task { - UpdateTask::ResolveVod { id, reason, delay, source_last_modified } => { - let mut next_reason = *reason; - next_reason.unset(ResolveReason::Tmdb); - next_reason.unset(ResolveReason::Date); - if next_reason.is_empty() { - None - } else { - Some(UpdateTask::ResolveVod { - id: id.clone(), - reason: next_reason, - delay: *delay, - source_last_modified: *source_last_modified, - }) - } - } - UpdateTask::ResolveSeries { id, reason, delay, source_last_modified } => { - let mut next_reason = *reason; - next_reason.unset(ResolveReason::Tmdb); - next_reason.unset(ResolveReason::Date); - if next_reason.is_empty() { - None - } else { - Some(UpdateTask::ResolveSeries { - id: id.clone(), - reason: next_reason, - delay: *delay, - source_last_modified: *source_last_modified, - }) - } - } - _ => Some(task.clone()), - } - } - - fn is_transient_worker_error(message: &str) -> bool { - message == TASK_ERR_UPDATE_IN_PROGRESS || message == TASK_ERR_PREEMPTED || message == TASK_ERR_NO_CONNECTION - } - - #[inline] - fn is_permanent_not_found_error(message: &str) -> bool { - let normalized = message.to_ascii_lowercase(); - Self::contains_standalone_fragment(&normalized, "404") - || Self::contains_standalone_fragment(&normalized, "not found") - } - - #[inline] - fn is_word_byte(byte: u8) -> bool { byte.is_ascii_alphanumeric() || byte == b'_' } - - fn contains_standalone_fragment(haystack: &str, fragment: &str) -> bool { - if fragment.is_empty() || haystack.len() < fragment.len() { - return false; - } - - let bytes = haystack.as_bytes(); - let mut search_from = 0usize; - - while let Some(relative_idx) = haystack[search_from..].find(fragment) { - let start = search_from + relative_idx; - let end = start + fragment.len(); - let before_is_word = start > 0 && Self::is_word_byte(bytes[start - 1]); - let after_is_word = end < bytes.len() && Self::is_word_byte(bytes[end]); - - if !before_is_word && !after_is_word { - return true; - } - search_from = end; - } - - false - } - - async fn flush_batch_static( - input_name: &str, - bound_ctx: Option<&MetadataUpdateCtx>, - batch_buffer: &mut BatchResultCollector, - ) { - if batch_buffer.is_empty() { - return; - } - - let Some(ctx) = bound_ctx else { return }; - let app_config = &ctx.app_config; - let cfg = app_config.config.load(); - let vod_updates = batch_buffer.take_vod_updates(); - let series_updates = batch_buffer.take_series_updates(); - let live_updates = batch_buffer.take_live_updates(); - - if vod_updates.is_empty() && series_updates.is_empty() && live_updates.is_empty() { - return; - } - - if let Ok(storage_path) = get_input_storage_path(input_name, &cfg.storage_dir).await { - if !vod_updates.is_empty() { - let mut updates: Vec<(u32, VideoStreamProperties)> = Vec::with_capacity(vod_updates.len()); - for (id, props) in &vod_updates { - if let ProviderIdType::Id(vid) = id { - updates.push((*vid, props.clone())); - } - } - - if !updates.is_empty() { - if let Err(e) = persist_input_vod_info_batch( - app_config, - &storage_path, - XtreamCluster::Video, - input_name, - updates, - ) - .await - { - error!("Failed to flush VOD batch for input {input_name}: {e}"); - } - } - } - - if !series_updates.is_empty() { - let mut updates: Vec<(u32, SeriesStreamProperties)> = Vec::with_capacity(series_updates.len()); - for (id, props) in &series_updates { - if let ProviderIdType::Id(vid) = id { - updates.push((*vid, props.clone())); - } - } - - if !updates.is_empty() { - if let Err(e) = persist_input_series_info_batch( - app_config, - &storage_path, - XtreamCluster::Series, - input_name, - updates, - ) - .await - { - error!("Failed to flush Series batch for input {input_name}: {e}"); - } - } - } - - if !live_updates.is_empty() { - let mut updates: Vec<(u32, LiveStreamProperties)> = Vec::with_capacity(live_updates.len()); - for (id, props) in &live_updates { - if let ProviderIdType::Id(vid) = id { - updates.push((*vid, props.clone())); - } - } - - if !updates.is_empty() { - if let Err(e) = persist_input_live_info_batch( - app_config, - &storage_path, - XtreamCluster::Live, - input_name, - updates, - ) - .await - { - error!("Failed to flush Live batch for input {input_name}: {e}"); - } - } - } - } - - let cascade_batch = BatchResultCollector { vod: vod_updates, series: series_updates, live: live_updates }; - - Self::cascade_updates(ctx, &app_config.config.load(), input_name, &cascade_batch).await; - } - - #[allow(clippy::too_many_lines)] - async fn cascade_updates( - ctx: &MetadataUpdateCtx, - config: &tuliprox_core::model::Config, - input_name: &str, - batch: &BatchResultCollector, - ) { - if batch.is_empty() { - return; - } - - // Find targets affected by this input. - let targets = { - let sources = ctx.app_config.sources.load(); - let mut affected_targets = Vec::new(); - - for source in &sources.sources { - if source.inputs.iter().any(|i_name| i_name.as_ref() == input_name) { - for t_def in &source.targets { - affected_targets.push(t_def.clone()); - } - } - } - affected_targets - }; - - if targets.is_empty() { - return; - } - - for target in targets { - let target_name = &target.name; - let Some(target_path) = tuliprox_repository::get_target_storage_path(config, target_name) else { - continue; - }; - let Some(storage_path) = tuliprox_repository::xtream_get_storage_path(config, target_name) else { - continue; - }; - let mapping_file = get_target_id_mapping_file(&target_path); - - let mapping = { - // Scope read lock strictly to mapping load. - let _file_lock = ctx.app_config.file_locks.read_lock(&mapping_file).await; - let mapping_file_clone = mapping_file.clone(); - match spawn_blocking_limited(move || TargetIdMapping::new(&mapping_file_clone, false)).await { - Ok(Ok(mapping)) => mapping, - Ok(Err(e)) => { - error!("Failed to open ID mapping for target {target_name}: {e}"); - continue; - } - Err(err) => { - error!("Failed to open ID mapping for target {target_name}: {err}"); - continue; - } - } - }; - - let mut provider_virtual_ids: HashMap> = HashMap::new(); - let mut uuid_virtual_ids: HashMap> = HashMap::new(); - - let vod_virtual_updates = Self::collect_vod_virtual_updates( - &mapping, - input_name, - batch, - &mut provider_virtual_ids, - &mut uuid_virtual_ids, - ); - Self::apply_vod_cascade_updates(ctx, &target, &storage_path, vod_virtual_updates).await; - - let series_virtual_updates = Self::collect_series_virtual_updates( - &mapping, - input_name, - batch, - &mut provider_virtual_ids, - &mut uuid_virtual_ids, - ); - Self::apply_series_cascade_updates(ctx, &target, &storage_path, series_virtual_updates).await; - - let live_virtual_updates = Self::collect_live_virtual_updates( - &mapping, - input_name, - batch, - &mut provider_virtual_ids, - &mut uuid_virtual_ids, - ); - Self::apply_live_cascade_updates(ctx, &target, &storage_path, live_virtual_updates).await; - } - } - - fn get_cached_uuid_virtual_id( - mapping: &TargetIdMapping, - cache: &mut HashMap>, - uuid: UUIDType, - ) -> Option { - if let Some(cached) = cache.get(&uuid) { - return *cached; - } - let resolved = mapping.get_virtual_id_by_uuid(&uuid); - cache.insert(uuid, resolved); - resolved - } - - // Three associated functions generated by the `collect_virtual_updates!` - // macro below. The macro body walks a single `BatchResultCollector` field - // and builds a `HashMap` of pending per-virtual updates. - collect_virtual_updates!(collect_vod_virtual_updates, vod, VideoStreamProperties, PlaylistItemType::Video); - collect_virtual_updates!( - collect_series_virtual_updates, - series, - SeriesStreamProperties, - PlaylistItemType::SeriesInfo - ); - collect_virtual_updates!(collect_live_virtual_updates, live, LiveStreamProperties, PlaylistItemType::Live); - - apply_cascade_updates!(apply_vod_cascade_updates, VideoStreamProperties, XtreamCluster::Video, Video, "VOD"); - apply_cascade_updates!( - apply_series_cascade_updates, - SeriesStreamProperties, - XtreamCluster::Series, - Series, - "Series" - ); - apply_cascade_updates!(apply_live_cascade_updates, LiveStreamProperties, XtreamCluster::Live, Live, "Live"); - - async fn update_memory_cache( - ctx: &MetadataUpdateCtx, - target_name: &str, - cluster: XtreamCluster, - updates: Vec, - ) { - let mut playlists = ctx.playlists.data.write().await; - if let Some(playlist) = playlists.get_mut(target_name) { - if let Some(xtream_storage) = &mut playlist.xtream { - let storage = match cluster { - XtreamCluster::Live => &mut xtream_storage.live, - XtreamCluster::Video => &mut xtream_storage.vod, - XtreamCluster::Series => &mut xtream_storage.series, - }; - for item in updates { - storage.insert(item.virtual_id.get(), item); - } - } - } - } - async fn get_or_open_query( - input_name: &str, - ctx: &MetadataUpdateCtx, - cluster: XtreamCluster, - db_handles: &mut HashMap, - failed_clusters: &mut HashSet, - ) -> Option>>> { - if failed_clusters.contains(&cluster) { - return None; - } - - if let std::collections::hash_map::Entry::Vacant(entry) = db_handles.entry(cluster) { - let cfg = ctx.app_config.config.load(); - if let Ok(storage_path) = get_input_storage_path(input_name, &cfg.storage_dir).await { - let file_path = xtream_get_file_path(&storage_path, cluster); - if file_path.exists() { - let lock = ctx.app_config.file_locks.read_lock(&file_path).await; - let file_path = file_path.clone(); - let query = match spawn_blocking_limited(move || { - BPlusTreeQuery::::try_new(&file_path) - }) - .await - { - Ok(Ok(query)) => Some(query), - Ok(Err(err)) => { - error!("Failed to open BPlusTreeQuery for {cluster}: {err}"); - None - } - Err(err) => { - error!("Failed to open BPlusTreeQuery for {cluster}: {err}"); - None - } - }; - - if let Some(query) = query { - entry.insert(DbHandle { _guard: lock, query: Arc::new(ParkingMutex::new(query)) }); - } else { - failed_clusters.insert(cluster); - } - } else { - // File doesn't exist; do not mark as failure to allow future creation. - } - } - } - - db_handles.get(&cluster).map(|h| Arc::clone(&h.query)) - } - - // Helper for get_item_name with caching - async fn get_item_name_static( - input_name: &str, - ctx: &MetadataUpdateCtx, - task: &UpdateTask, - db_handles: &mut HashMap, - failed_clusters: &mut HashSet, - ) -> Option { - let (id, cluster) = match task { - UpdateTask::ResolveVod { id, .. } => (id, XtreamCluster::Video), - UpdateTask::ResolveSeries { id, .. } => (id, XtreamCluster::Series), - UpdateTask::ProbeLive { id, .. } => (id, XtreamCluster::Live), - UpdateTask::ProbeStream { .. } => return None, - }; - - if let ProviderIdType::Id(vid) = id { - let stream_id = *vid; - if let Some(query) = Self::get_or_open_query(input_name, ctx, cluster, db_handles, failed_clusters).await { - let query = Arc::clone(&query); - let item = match spawn_blocking_limited(move || { - let mut guard = query.lock(); - guard.query_zero_copy(&stream_id).ok().flatten() - }) - .await - { - Ok(item) => item, - Err(err) => { - error!("Failed to query item name for {stream_id}: {err}"); - None - } - }; - - if let Some(item) = item { - return Some(if item.title.is_empty() { item.name.to_string() } else { item.title.to_string() }); - } - } - } - None - } - - #[allow(clippy::too_many_lines)] - async fn process_task_static( - input_name: &Arc, - bound_ctx: Option<&MetadataUpdateCtx>, - task: &UpdateTask, - collector: &mut BatchResultCollector, - db_handles: &mut HashMap, - failed_clusters: &mut HashSet, - ) -> Result { - let ctx = - bound_ctx.ok_or_else(|| shared::error::TuliproxError::Config("metadata context not bound".to_string()))?; - - let Some(input_base) = ctx.app_config.get_input_by_name(input_name) else { - return Err(shared::error::TuliproxError::Config(format!("Input {input_name} not found"))); - }; - - if !input_base.enabled { - return Err(shared::error::TuliproxError::Config(format!("Input {input_name} is disabled"))); - } - - // Background metadata/probe tasks are low-priority. - // Never run them while a foreground playlist update is active. - if let Some(guard) = ctx.update_guard.try_playlist() { - drop(guard); - } else { - return Err(shared::error::TuliproxError::Config(TASK_ERR_UPDATE_IN_PROGRESS.to_string())); - } - - let needs_probe_connection = Self::task_needs_provider_connection(task, input_base.input_type); - - let probe_priority = ctx - .app_config - .config - .load() - .metadata_update - .as_ref() - .map_or(default_probe_user_priority(), |cfg| cfg.probe.user_priority); - - // Reserve provider capacity only for actual probe work (ffprobe paths). - let provider_handle = if needs_probe_connection { - let Some(handle) = ctx.active_provider.acquire_connection_for_probe(input_name, probe_priority) else { - debug_if_enabled!("No provider connection available for background task {}, skipping...", task); - return Err(shared::error::TuliproxError::Config(TASK_ERR_NO_CONNECTION.to_string())); - }; - Some(handle) - } else { - None - }; - - let item_title = Self::get_item_name_static(input_name, ctx, task, db_handles, failed_clusters).await; - - let config_to_use = provider_handle.as_ref().and_then(|handle| handle.allocation.get_provider_config()); - let name_display = item_title.as_deref().map_or(String::new(), |n| format!(" \"{n}\"")); - - debug!("Processing task for {input_name}: {task}{name_display}"); - - let pre_vod_updates = collector.vod.len(); - let pre_series_updates = collector.series.len(); - let pre_live_updates = collector.live.len(); - - // Determine input to use (may be alias) - let input_to_use = config_to_use - .filter(|alloc| alloc.name != input_base.name) - .and_then(|alloc| input_base.aliases.as_ref()?.iter().find(|a| a.enabled && a.name == alloc.name)) - .map(|alias_def| { - let mut temp_input = (*input_base).clone(); - temp_input.url.clone_from(&alias_def.url); - temp_input.username.clone_from(&alias_def.username); - temp_input.password.clone_from(&alias_def.password); - Arc::new(temp_input) - }) - .unwrap_or(input_base); - - let client = if tuliprox_core::model::should_use_manual_redirects(&ctx.app_config) { - ctx.http_clients.no_redirect.load() - } else { - ctx.http_clients.default.load() - }; - - // Execute task; probe tasks get a reserved provider handle, resolve tasks don't. - // The hard timeout is applied only to tasks that perform network probing — Probe* - // variants and Resolve tasks whose reason includes ResolveReason::Probe — because - // those are the only ones that can stall on an unresponsive provider. Pure metadata - // resolve tasks (Info / TMDB / Date only) run without a timeout so they are not - // subject to the probe hard limit. - let exec_fut = async { - if let Some(handle) = provider_handle.as_ref() { - if let Some(token) = &handle.cancel_token { - tokio::select! { - biased; - - () = token.cancelled() => { - debug_if_enabled!("Metadata update task preempted by user request for input {}", input_name); - Err(shared::error::TuliproxError::Config(TASK_ERR_PREEMPTED.to_string())) - } - - res = Self::execute_task_inner_static(ctx, &client, &input_to_use, task, item_title.as_deref(), Some(handle), probe_priority, collector, db_handles, failed_clusters) => { - res - } - } - } else { - Self::execute_task_inner_static( - ctx, - &client, - &input_to_use, - task, - item_title.as_deref(), - Some(handle), - probe_priority, - collector, - db_handles, - failed_clusters, - ) - .await - } - } else { - Self::execute_task_inner_static( - ctx, - &client, - &input_to_use, - task, - item_title.as_deref(), - None, - probe_priority, - collector, - db_handles, - failed_clusters, - ) - .await - } - }; - let needs_probe_timeout = Self::is_probe_task(task) - || (Self::is_resolve_task(task) && Self::task_reason(task).contains(ResolveReason::Probe)); - let res = if needs_probe_timeout { - match tokio::time::timeout(Duration::from_secs(PROBE_TASK_TIMEOUT_SECS), exec_fut).await { - Ok(result) => result, - Err(_elapsed) => { - error!( - "Metadata probe task timed out after {PROBE_TASK_TIMEOUT_SECS}s for input {input_name}: {task}; \ - releasing provider handle and skipping task" - ); - Err(shared::error::TuliproxError::Config(format!( - "Task timed out after {PROBE_TASK_TIMEOUT_SECS}s for input {input_name}: {task}" - ))) - } - } - } else { - exec_fut.await - }; - - if provider_handle.is_some() { - ctx.connection_manager.release_provider_handle(provider_handle); - } - match res { - Ok((tmdb_and_date_present, probe_pending)) => { - let task_changed = match task { - UpdateTask::ResolveVod { .. } => collector.vod.len() > pre_vod_updates, - UpdateTask::ResolveSeries { .. } => collector.series.len() > pre_series_updates, - UpdateTask::ProbeLive { .. } => collector.live.len() > pre_live_updates, - UpdateTask::ProbeStream { .. } => true, - }; - let tmdb_pending = match task { - UpdateTask::ResolveVod { reason, .. } | UpdateTask::ResolveSeries { reason, .. } => { - if reason.contains(ResolveReason::Tmdb) || reason.contains(ResolveReason::Date) { - !tmdb_and_date_present - } else { - false - } - } - UpdateTask::ProbeLive { .. } | UpdateTask::ProbeStream { .. } => false, - }; - Ok(ProcessTaskOutcome { task_changed, tmdb_pending, probe_pending }) - } - Err(e) => Err(e), - } - } - - fn task_needs_provider_connection(task: &UpdateTask, input_type: InputType) -> bool { - match task { - UpdateTask::ProbeLive { .. } => true, - // Local library and media-server probing must not depend on IPTV provider capacity. - UpdateTask::ProbeStream { .. } => { - !(matches!(input_type, InputType::Library) || input_type.is_media_server()) - } - // Resolve tasks handle their own probe connection acquisition internally. - // This avoids holding a provider connection for the entire duration of - // info fetch + TMDB resolve + probe, reducing "provider exhausted" errors. - UpdateTask::ResolveVod { .. } | UpdateTask::ResolveSeries { .. } => false, - } - } - - fn batchable_generic_probe_xtream_cluster(item_type: PlaylistItemType) -> Option { - if item_type.is_live() { - Some(XtreamCluster::Live) - } else if item_type.is_video() { - Some(XtreamCluster::Video) - } else { - None - } - } - - fn apply_pending_generic_probe_base( - collector: &BatchResultCollector, - cluster: XtreamCluster, - provider_id: u32, - item: &mut XtreamPlaylistItem, - ) { - match cluster { - XtreamCluster::Video => { - if let Some((_, props)) = collector - .vod - .iter() - .rev() - .find(|(id, _)| matches!(id, ProviderIdType::Id(id) if *id == provider_id)) - { - item.additional_properties = Some(StreamProperties::Video(Box::new(props.clone()))); - } - } - XtreamCluster::Live => { - if let Some((_, props)) = collector - .live - .iter() - .rev() - .find(|(id, _)| matches!(id, ProviderIdType::Id(id) if *id == provider_id)) - { - item.additional_properties = Some(StreamProperties::Live(Box::new(props.clone()))); - } - } - XtreamCluster::Series => {} - } - } - - #[allow(clippy::too_many_arguments, clippy::too_many_lines)] - async fn execute_task_inner_static( - ctx: &MetadataUpdateCtx, - client: &reqwest::Client, - input: &Arc, - task: &UpdateTask, - item_title: Option<&str>, - active_handle: Option<&ProviderHandle>, - probe_priority: i8, - collector: &mut BatchResultCollector, - db_handles: &mut HashMap, - failed_clusters: &mut HashSet, - ) -> Result<(bool, bool), TuliproxError> { - // The returned tuple is `(tmdb_and_date_present, probe_pending)`. - // `tmdb_and_date_present` avoids false-positive TMDB "no match" cooldowns. - // `probe_pending` prevents skipped/aborted probes from being cached as no-op. - match task { - UpdateTask::ResolveVod { id, reason, .. } => { - let fetch_info = reason.contains(ResolveReason::Info); - let resolve_tmdb = - fetch_info || reason.contains(ResolveReason::Tmdb) || reason.contains(ResolveReason::Date); - let will_probe = reason.contains(ResolveReason::Probe); - - // If we are going to probe, release the cached handle to avoid holding a READ lock - // for along time (blocks writers) and also to avoid potential deadlocks if - // the probe function itself tries to acquire a WRITE lock later. - if will_probe { - db_handles.remove(&XtreamCluster::Video); - } - - let query_opt = if will_probe { - None - } else { - Self::get_or_open_query(&input.name, ctx, XtreamCluster::Video, db_handles, failed_clusters).await - }; - - let tmdb_and_date_present = AtomicBool::new(false); - let probe_pending = AtomicBool::new(false); - match update_vod_metadata( - &ctx.app_config, - client, - input, - id.clone(), - active_handle, - &ctx.active_provider, - item_title, - false, // Batch collect - fetch_info, - resolve_tmdb, - will_probe, - query_opt, - Some(&tmdb_and_date_present), - Some(&probe_pending), - ) - .await - { - Ok(Some(props)) => { - collector.add_vod(id.clone(), props); - Ok((tmdb_and_date_present.load(Ordering::Relaxed), probe_pending.load(Ordering::Relaxed))) - } - Ok(None) => { - Ok((tmdb_and_date_present.load(Ordering::Relaxed), probe_pending.load(Ordering::Relaxed))) - } - Err(e) => Err(e), - } - } - UpdateTask::ResolveSeries { id, reason, .. } => { - let fetch_info = reason.contains(ResolveReason::Info); - let resolve_tmdb = reason.contains(ResolveReason::Tmdb) || reason.contains(ResolveReason::Date); - let will_probe = reason.contains(ResolveReason::Probe); - let series_probe_settings = { - let config = ctx.app_config.config.load(); - SeriesProbeSettings::from_metadata_update(config.metadata_update.as_ref()) - }; - - if will_probe { - db_handles.remove(&XtreamCluster::Series); - } - - // Get handle for Series - let query_opt = if will_probe { - None - } else { - Self::get_or_open_query(&input.name, ctx, XtreamCluster::Series, db_handles, failed_clusters).await - }; - - let tmdb_and_date_present = AtomicBool::new(false); - let probe_pending = AtomicBool::new(false); - match update_series_metadata( - &ctx.app_config, - client, - input, - id.clone(), - &ctx.active_provider, - active_handle, - item_title, - false, // Batch collect - fetch_info, - resolve_tmdb, - will_probe, - series_probe_settings, - query_opt, - Some(&tmdb_and_date_present), - Some(&probe_pending), - ) - .await - { - Ok(Some(props)) => { - collector.add_series(id.clone(), props); - Ok((tmdb_and_date_present.load(Ordering::Relaxed), probe_pending.load(Ordering::Relaxed))) - } - Ok(None) => { - Ok((tmdb_and_date_present.load(Ordering::Relaxed), probe_pending.load(Ordering::Relaxed))) - } - Err(e) => Err(e), - } - } - UpdateTask::ProbeLive { id, .. } => { - // ProbeLive always probes, so we must never use a cached handle here. - db_handles.remove(&XtreamCluster::Live); - - match update_live_stream_metadata( - &ctx.app_config, - client, - input, - id.clone(), - false, - None, - active_handle, - &ctx.active_provider, - ) - .await - { - Ok(Some(props)) => { - collector.add_live(id.clone(), props); - Ok((false, false)) - } - Ok(None) => Ok((false, false)), - Err(e) => Err(e), - } - } - UpdateTask::ProbeStream { unique_id, url, item_type, .. } => { - let task_key = TaskKey::from_task(task); - let probe_identifier = if unique_id.trim().is_empty() { url.as_str() } else { unique_id.as_str() }; - - if input.input_type.is_xtream() { - if let Some(cluster) = Self::batchable_generic_probe_xtream_cluster(*item_type) { - let Ok(provider_id) = unique_id.parse::() else { - warn!("Skipping xtream generic probe update with non-numeric id: {unique_id}"); - return Ok((false, false)); - }; - - let outcome = probe_generic_stream_metadata( - &ctx.app_config, - client, - input.as_ref(), - unique_id, - url, - *item_type, - &ctx.active_provider, - active_handle, - probe_priority, - &ctx.events, - ) - .await?; - - let metadata = match outcome { - GenericProbeMetadataOutcome::Metadata(metadata) => metadata, - GenericProbeMetadataOutcome::Noop => return Ok((false, false)), - GenericProbeMetadataOutcome::ProbeFailed => { - return Err(shared::error::TuliproxError::Config(format!( - "Probe stream task failed for key {task_key:?} ({probe_identifier})" - ))); - } - }; - - let Some(query) = - Self::get_or_open_query(&input.name, ctx, cluster, db_handles, failed_clusters).await - else { - warn!("Item not found in Xtream DB for generic probe: {unique_id}"); - return Ok((false, false)); - }; - - let query = Arc::clone(&query); - let mut item = match spawn_blocking_limited(move || { - let mut guard = query.lock(); - guard.query_zero_copy(&provider_id).ok().flatten() - }) - .await - { - Ok(Some(item)) => item, - Ok(None) => { - warn!("Item not found in Xtream DB: {unique_id}"); - return Ok((false, false)); - } - Err(err) => { - return Err(shared::error::TuliproxError::Config(format!( - "Failed to query generic probe item {unique_id}: {err}" - ))); - } - }; - - Self::apply_pending_generic_probe_base(collector, cluster, provider_id, &mut item); - - update_properties( - &mut item.additional_properties, - *item_type, - &item.name, - item.virtual_id.get(), - metadata.raw_video, - metadata.raw_audio, - metadata.stats, - ); - - match item.additional_properties { - Some(StreamProperties::Live(props)) => collector.add_live(provider_id.into(), *props), - Some(StreamProperties::Video(props)) => collector.add_vod(provider_id.into(), *props), - _ => {} - } - - return Ok((false, false)); - } - } - - // Non-Xtream generic probe still writes directly and can target - // M3U or library storage, so release cached Xtream read handles. - if !db_handles.is_empty() { - db_handles.clear(); - } - let outcome = update_generic_stream_metadata( - &ctx.app_config, - client, - input.as_ref(), - unique_id, - url, - *item_type, - &ctx.active_provider, - active_handle, - probe_priority, - &ctx.events, - ) - .await?; - - match outcome { - GenericProbeOutcome::Updated | GenericProbeOutcome::Noop => Ok((false, false)), - GenericProbeOutcome::ProbeFailed => Err(shared::error::TuliproxError::Config(format!( - "Probe stream task failed for key {task_key:?} ({probe_identifier})" - ))), - } - } - } - } -} - -/// The pipeline's view of this manager. -/// -/// `tuliprox-processing` needs three operations here and nothing else; the trait -/// lives there so the pipeline does not have to name this type. The impl lives -/// here because the type does. -impl tuliprox_processing::metadata_sink::MetadataUpdateSink for MetadataUpdateManager { - async fn acquire_update_pause_guard(&self) -> tokio::sync::OwnedRwLockWriteGuard<()> { - MetadataUpdateManager::acquire_update_pause_guard(self).await - } - - async fn prepare_enqueue_state(&self, input_name: Arc) { - self.ensure_enqueue_state_loaded_for_input(&input_name).await; - } - - fn should_skip_enqueue(&self, input_name: &str, task: &UpdateTask) -> bool { - self.should_skip_enqueue_cached(input_name, task) - } - - fn queue_task_background(self: Arc, input_name: Arc, task: UpdateTask) { - MetadataUpdateManager::queue_task_background(&self, input_name, task); - } -} +#[cfg(test)] +mod tests; + +mod publication; +mod retry; +mod retry_store; +mod settings; +mod task_processing; +mod task_queue; +mod worker; #[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - use tokio::sync::mpsc; - - fn create_test_worker( - input_name: &str, - sender: mpsc::Sender, - receiver: mpsc::Receiver, - pending_tasks: Arc>, - pending_task_count: Arc, - ) -> InputWorker { - InputWorker { - input_name: Arc::from(input_name), - sender, - receiver, - pending_tasks, - pending_task_count, - bound_ctx: None, - update_pause_gate: Arc::new(RwLock::new(())), - cancel_token: CancellationToken::new(), - batch_buffer: BatchResultCollector::new(), - db_handles: HashMap::new(), - failed_clusters: HashSet::new(), - retry_states: HashMap::new(), - resolve_exhausted: HashMap::new(), - last_cycle_completed_at_ts: None, - metadata_retry_state_path: None, - metadata_retry_loaded: false, - metadata_retry_load_retry_at_ts: None, - last_retry_state_prune_at_ts: None, - scheduled_requeues: Arc::new(DashMap::new()), - recently_completed_no_change: HashMap::new(), - resolve_enqueue_suppressions: Arc::new(DashMap::new()), - tmdb_source_markers: Arc::new(DashMap::new()), - dirty_retry_state_keys: HashSet::new(), - } - } - - #[tokio::test] - async fn queue_task_creates_single_worker_per_input_under_concurrency() { - let cancel_token = CancellationToken::new(); - let manager = Arc::new(MetadataUpdateManager::new(cancel_token)); - let input_name: Arc = Arc::from("race_input"); - - let mut joins = Vec::new(); - for id in 0..32u32 { - let manager_cloned = manager.clone(); - let input_cloned = input_name.clone(); - joins.push(tokio::spawn(async move { - manager_cloned - .queue_task( - input_cloned, - UpdateTask::ResolveVod { - id: ProviderIdType::Id(id), - reason: ResolveReasonSet::default(), - delay: 0, - source_last_modified: None, - }, - ) - .await; - })); - } - - for join in joins { - join.await.expect("queue task spawn should complete"); - } - - // Allow spawned worker startup to settle. - tokio::time::sleep(Duration::from_millis(50)).await; - assert_eq!(manager.active_worker_count(), 1); - - manager.shutdown(); - } - - #[test] - fn should_skip_enqueue_respects_resolve_suppression() { - let manager = MetadataUpdateManager::new(CancellationToken::new()); - let input_name = "input_suppressed"; - let task = UpdateTask::ResolveVod { - id: ProviderIdType::Id(42), - reason: ResolveReason::Info.into(), - delay: 1, - source_last_modified: None, - }; - let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); - manager - .resolve_enqueue_suppressions - .insert(scoped_key.clone(), chrono::Utc::now().timestamp().saturating_add(60)); - - assert!(manager.should_skip_enqueue_cached(input_name, &task)); - - manager - .resolve_enqueue_suppressions - .insert(scoped_key.clone(), chrono::Utc::now().timestamp().saturating_sub(1)); - - assert!(!manager.should_skip_enqueue_cached(input_name, &task)); - assert!(!manager.resolve_enqueue_suppressions.contains_key(&scoped_key)); - } - - #[test] - fn strip_tmdb_reasons_for_enqueue_skips_unchanged_series_tmdb_only_task() { - let manager = MetadataUpdateManager::new(CancellationToken::new()); - let input_name = "input_tmdb_marker"; - let task = UpdateTask::ResolveSeries { - id: ProviderIdType::Id(42), - reason: ResolveReason::Tmdb | ResolveReason::Date, - delay: 1, - source_last_modified: Some(777), - }; - let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); - manager.tmdb_source_markers.insert(scoped_key, 777); - - assert!(manager.strip_tmdb_reasons_for_enqueue(input_name, task).is_none()); - } - - #[test] - fn strip_tmdb_reasons_for_enqueue_keeps_non_tmdb_reasons_for_unchanged_series_source() { - let manager = MetadataUpdateManager::new(CancellationToken::new()); - let input_name = "input_tmdb_probe"; - let task = UpdateTask::ResolveSeries { - id: ProviderIdType::Id(42), - reason: ResolveReason::Tmdb | ResolveReason::Probe, - delay: 1, - source_last_modified: Some(777), - }; - let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); - manager.tmdb_source_markers.insert(scoped_key, 777); - - let prepared = manager.strip_tmdb_reasons_for_enqueue(input_name, task).expect("probe reason should remain"); - match prepared { - UpdateTask::ResolveSeries { reason, source_last_modified, .. } => { - assert_eq!(source_last_modified, Some(777)); - assert!(!reason.contains(ResolveReason::Tmdb)); - assert!(!reason.contains(ResolveReason::Date)); - assert!(reason.contains(ResolveReason::Probe)); - } - other => panic!("unexpected task type after enqueue strip: {other:?}"), - } - } - - #[test] - fn strip_tmdb_reasons_for_enqueue_keeps_unknown_series_timestamp_and_tmdb_reason() { - let manager = MetadataUpdateManager::new(CancellationToken::new()); - let input_name = "input_tmdb_unknown_series"; - let task = UpdateTask::ResolveSeries { - id: ProviderIdType::Id(42), - reason: ResolveReason::Tmdb | ResolveReason::Date, - delay: 1, - source_last_modified: None, - }; - let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); - - assert!(!manager.tmdb_source_markers.contains_key(&scoped_key)); - - let prepared = manager - .strip_tmdb_reasons_for_enqueue(input_name, task) - .expect("unknown timestamps should not be suppressed"); - match prepared { - UpdateTask::ResolveSeries { reason, source_last_modified, .. } => { - assert_eq!(source_last_modified, None); - assert!(reason.contains(ResolveReason::Tmdb)); - assert!(reason.contains(ResolveReason::Date)); - } - other => panic!("unexpected task type after enqueue strip: {other:?}"), - } - } - - #[test] - fn strip_tmdb_reasons_for_enqueue_skips_unchanged_vod_tmdb_only_task() { - let manager = MetadataUpdateManager::new(CancellationToken::new()); - let input_name = "input_vod_tmdb_marker"; - let task = UpdateTask::ResolveVod { - id: ProviderIdType::Id(42), - reason: ResolveReason::Tmdb | ResolveReason::Date, - delay: 1, - source_last_modified: Some(777), - }; - let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); - manager.tmdb_source_markers.insert(scoped_key, 777); - - assert!(manager.strip_tmdb_reasons_for_enqueue(input_name, task).is_none()); - } - - #[test] - fn strip_tmdb_reasons_for_enqueue_keeps_non_tmdb_reasons_for_unchanged_vod_source() { - let manager = MetadataUpdateManager::new(CancellationToken::new()); - let input_name = "input_vod_tmdb_probe"; - let task = UpdateTask::ResolveVod { - id: ProviderIdType::Id(42), - reason: ResolveReason::Tmdb | ResolveReason::Probe, - delay: 1, - source_last_modified: Some(777), - }; - let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); - manager.tmdb_source_markers.insert(scoped_key, 777); - - let prepared = manager.strip_tmdb_reasons_for_enqueue(input_name, task).expect("probe reason should remain"); - match prepared { - UpdateTask::ResolveVod { reason, source_last_modified, .. } => { - assert_eq!(source_last_modified, Some(777)); - assert!(!reason.contains(ResolveReason::Tmdb)); - assert!(!reason.contains(ResolveReason::Date)); - assert!(reason.contains(ResolveReason::Probe)); - } - other => panic!("unexpected task type after enqueue strip: {other:?}"), - } - } - - #[test] - fn strip_tmdb_reasons_for_enqueue_keeps_unknown_vod_timestamp_and_tmdb_reason() { - let manager = MetadataUpdateManager::new(CancellationToken::new()); - let input_name = "input_tmdb_unknown_vod"; - let task = UpdateTask::ResolveVod { - id: ProviderIdType::Id(42), - reason: ResolveReason::Tmdb | ResolveReason::Date, - delay: 1, - source_last_modified: None, - }; - let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); - - assert!(!manager.tmdb_source_markers.contains_key(&scoped_key)); - - let prepared = manager - .strip_tmdb_reasons_for_enqueue(input_name, task) - .expect("unknown timestamps should not be suppressed"); - match prepared { - UpdateTask::ResolveVod { reason, source_last_modified, .. } => { - assert_eq!(source_last_modified, None); - assert!(reason.contains(ResolveReason::Tmdb)); - assert!(reason.contains(ResolveReason::Date)); - } - other => panic!("unexpected task type after enqueue strip: {other:?}"), - } - } - - #[tokio::test] - async fn submit_task_merges_existing_task_and_increments_generation() { - let (tx, mut rx) = mpsc::channel::(8); - let pending_tasks = Arc::new(DashMap::new()); - let pending_task_count = Arc::new(AtomicUsize::new(0)); - - let task_initial = UpdateTask::ResolveVod { - id: ProviderIdType::Id(42), - reason: ResolveReason::Info.into(), - delay: 10, - source_last_modified: None, - }; - let queue_size = MetadataUpdateRuntimeSettings::default().max_queue_size; - MetadataUpdateManager::submit_task( - tx.clone(), - pending_tasks.clone(), - pending_task_count.clone(), - "input_a", - queue_size, - task_initial, - ) - .await; - - let task_merge = UpdateTask::ResolveVod { - id: ProviderIdType::Id(42), - reason: ResolveReason::Probe.into(), - delay: 2, - source_last_modified: None, - }; - MetadataUpdateManager::submit_task( - tx, - pending_tasks.clone(), - pending_task_count.clone(), - "input_a", - queue_size, - task_merge, - ) - .await; - - let first_signal = rx.try_recv().expect("first signal should be queued"); - assert_eq!(first_signal, TaskKey::Vod(42)); - assert!(matches!( - rx.try_recv(), - Err(tokio::sync::mpsc::error::TryRecvError::Empty | tokio::sync::mpsc::error::TryRecvError::Disconnected) - )); - - let entry = pending_tasks.get(&TaskKey::Vod(42)).expect("pending entry should exist"); - assert_eq!(entry.generation.load(Ordering::Relaxed), 1); - - let merged = entry.task.lock().clone(); - match merged { - UpdateTask::ResolveVod { reason, delay, .. } => { - assert!(reason.contains(ResolveReason::Info)); - assert!(reason.contains(ResolveReason::Probe)); - assert_eq!(delay, 2); - } - other => panic!("unexpected task type after merge: {other:?}"), - } - } - - #[tokio::test] - async fn submit_task_identical_resolve_merge_keeps_generation() { - let (tx, mut rx) = mpsc::channel::(8); - let pending_tasks = Arc::new(DashMap::new()); - let pending_task_count = Arc::new(AtomicUsize::new(0)); - let queue_size = MetadataUpdateRuntimeSettings::default().max_queue_size; - - let initial = UpdateTask::ResolveVod { - id: ProviderIdType::Id(42), - reason: ResolveReason::Tmdb.into(), - delay: 10, - source_last_modified: None, - }; - MetadataUpdateManager::submit_task( - tx.clone(), - pending_tasks.clone(), - pending_task_count.clone(), - "input_a", - queue_size, - initial, - ) - .await; - - let identical_merge = UpdateTask::ResolveVod { - id: ProviderIdType::Id(42), - reason: ResolveReason::Tmdb.into(), - delay: 10, - source_last_modified: None, - }; - MetadataUpdateManager::submit_task( - tx, - pending_tasks.clone(), - pending_task_count.clone(), - "input_a", - queue_size, - identical_merge, - ) - .await; - - let first_signal = rx.try_recv().expect("first signal should be queued"); - assert_eq!(first_signal, TaskKey::Vod(42)); - assert!(matches!( - rx.try_recv(), - Err(tokio::sync::mpsc::error::TryRecvError::Empty | tokio::sync::mpsc::error::TryRecvError::Disconnected) - )); - - let entry = pending_tasks.get(&TaskKey::Vod(42)).expect("pending entry should exist"); - assert_eq!(entry.generation.load(Ordering::Relaxed), 0); - } - - #[tokio::test] - async fn submit_task_probe_stream_merge_keeps_existing_payload_when_present() { - let (tx, mut rx) = mpsc::channel::(8); - let pending_tasks = Arc::new(DashMap::new()); - let pending_task_count = Arc::new(AtomicUsize::new(0)); - let queue_size = MetadataUpdateRuntimeSettings::default().max_queue_size; - - let initial = UpdateTask::ProbeStream { - probe_scope: Arc::from("scope_a"), - unique_id: "uid_1".to_string(), - url: "http://old.example/stream".to_string(), - item_type: PlaylistItemType::Video, - reason: ResolveReason::MissingDetails.into(), - delay: 10, - }; - MetadataUpdateManager::submit_task( - tx.clone(), - pending_tasks.clone(), - pending_task_count.clone(), - "input_a", - queue_size, - initial, - ) - .await; - - let merged_in = UpdateTask::ProbeStream { - probe_scope: Arc::from("scope_a"), - unique_id: "uid_1".to_string(), - url: "http://new.example/stream".to_string(), - item_type: PlaylistItemType::LocalVideo, - reason: ResolveReason::Probe.into(), - delay: 2, - }; - MetadataUpdateManager::submit_task( - tx, - pending_tasks.clone(), - pending_task_count.clone(), - "input_a", - queue_size, - merged_in, - ) - .await; - - let first_signal = rx.try_recv().expect("first signal should be queued"); - assert_eq!(first_signal, TaskKey::Stream { scope: Arc::from("scope_a"), id: Arc::from("uid_1") }); - assert!(matches!( - rx.try_recv(), - Err(tokio::sync::mpsc::error::TryRecvError::Empty | tokio::sync::mpsc::error::TryRecvError::Disconnected) - )); - - let key = TaskKey::Stream { scope: Arc::from("scope_a"), id: Arc::from("uid_1") }; - let entry = pending_tasks.get(&key).expect("pending entry should exist"); - assert_eq!(entry.generation.load(Ordering::Relaxed), 1); - - let merged = entry.task.lock().clone(); - match merged { - UpdateTask::ProbeStream { reason, delay, url, item_type, .. } => { - assert!(reason.contains(ResolveReason::MissingDetails)); - assert!(reason.contains(ResolveReason::Probe)); - assert_eq!(delay, 2); - assert_eq!(url, "http://old.example/stream"); - assert_eq!(item_type, PlaylistItemType::Video); - } - other => panic!("unexpected task type after merge: {other:?}"), - } - } - - #[tokio::test] - async fn submit_task_with_closed_sender_does_not_report_merged_for_existing_pending_entry() { - let (tx, rx) = mpsc::channel::(8); - drop(rx); - - let pending_tasks = Arc::new(DashMap::new()); - let pending_task_count = Arc::new(AtomicUsize::new(1)); - let key = TaskKey::Vod(42); - let existing_task = UpdateTask::ResolveVod { - id: ProviderIdType::Id(42), - reason: ResolveReason::Info.into(), - delay: 10, - source_last_modified: None, - }; - pending_tasks.insert(key.clone(), PendingTask::new(existing_task)); - - let incoming_task = UpdateTask::ResolveVod { - id: ProviderIdType::Id(42), - reason: ResolveReason::Probe.into(), - delay: 2, - source_last_modified: None, - }; - - let result = MetadataUpdateManager::submit_task( - tx, - pending_tasks.clone(), - pending_task_count.clone(), - "input_a", - MetadataUpdateRuntimeSettings::default().max_queue_size, - incoming_task, - ) - .await; - - assert_eq!(result, SubmitTaskResult::ChannelClosed); - assert!(!pending_tasks.contains_key(&key)); - assert_eq!(pending_task_count.load(Ordering::Relaxed), 0); - } - - #[tokio::test] - async fn finalize_processed_task_success_requeues_when_generation_changed() { - let (tx, rx) = mpsc::channel::(8); - let pending_tasks = Arc::new(DashMap::new()); - let pending_task_count = Arc::new(AtomicUsize::new(1)); - let key = TaskKey::Vod(7); - - pending_tasks.insert( - key.clone(), - PendingTask::new(UpdateTask::ResolveVod { - id: ProviderIdType::Id(7), - reason: ResolveReason::Info.into(), - delay: 0, - source_last_modified: None, - }), - ); - if let Some(entry) = pending_tasks.get(&key) { - entry.generation.store(1, Ordering::Relaxed); - } - - let mut worker = create_test_worker("input_a", tx, rx, pending_tasks.clone(), pending_task_count); - - let requeued = worker.finalize_processed_task_success(&key, 0, "input_a").await; - assert!(requeued); - assert!(pending_tasks.contains_key(&key)); - assert_eq!(worker.receiver.try_recv().expect("requeued signal should be present"), key); - } - - #[tokio::test] - async fn finalize_processed_task_success_removes_when_unchanged() { - let (tx, rx) = mpsc::channel::(8); - let pending_tasks = Arc::new(DashMap::new()); - let pending_task_count = Arc::new(AtomicUsize::new(1)); - let key = TaskKey::Vod(9); - let task = UpdateTask::ResolveVod { - id: ProviderIdType::Id(9), - reason: ResolveReason::Info.into(), - delay: 0, - source_last_modified: None, - }; - - pending_tasks.insert(key.clone(), PendingTask::new(task.clone())); - - let mut worker = create_test_worker("input_b", tx, rx, pending_tasks.clone(), pending_task_count); - let runtime_settings = MetadataUpdateRuntimeSettings::default(); - - worker.scheduled_requeues.insert(key.clone(), chrono::Utc::now().timestamp().saturating_add(30)); - worker.recently_completed_no_change.insert(key.clone(), (Instant::now(), ResolveReason::Info.into())); - assert!(worker.should_skip_recent_no_change_task(&key, &task, &runtime_settings)); - assert!(worker.recently_completed_no_change.contains_key(&key)); - assert!(!worker.scheduled_requeues.contains_key(&key)); - - let requeued = worker.finalize_processed_task_success(&key, 0, "input_b").await; - assert!(!requeued); - assert!(!pending_tasks.contains_key(&key)); - assert!(matches!(worker.receiver.try_recv(), Err(tokio::sync::mpsc::error::TryRecvError::Empty))); - } - - #[test] - fn recent_no_change_skip_requires_exact_reason_match() { - let (tx, rx) = mpsc::channel::(8); - let pending_tasks = Arc::new(DashMap::new()); - let pending_task_count = Arc::new(AtomicUsize::new(0)); - let key = TaskKey::Vod(10); - let task = UpdateTask::ResolveVod { - id: ProviderIdType::Id(10), - reason: ResolveReason::Info.into(), - delay: 0, - source_last_modified: None, - }; - let mut worker = create_test_worker("input_c", tx, rx, pending_tasks, pending_task_count); - let runtime_settings = MetadataUpdateRuntimeSettings::default(); - - worker - .recently_completed_no_change - .insert(key.clone(), (Instant::now(), ResolveReason::Info | ResolveReason::Probe)); - assert!(!worker.should_skip_recent_no_change_task(&key, &task, &runtime_settings)); - assert!(!worker.recently_completed_no_change.contains_key(&key)); - - worker.recently_completed_no_change.insert(key.clone(), (Instant::now(), ResolveReason::Info.into())); - assert!(worker.should_skip_recent_no_change_task(&key, &task, &runtime_settings)); - assert!(worker.recently_completed_no_change.contains_key(&key)); - } - - #[tokio::test] - async fn recent_no_change_skip_ttl_expiry_allows_requeue() { - let (tx, rx) = mpsc::channel::(8); - let pending_tasks = Arc::new(DashMap::new()); - let pending_task_count = Arc::new(AtomicUsize::new(1)); - let key = TaskKey::Vod(11); - let task = UpdateTask::ResolveVod { - id: ProviderIdType::Id(11), - reason: ResolveReason::Info.into(), - delay: 0, - source_last_modified: None, - }; - - pending_tasks.insert(key.clone(), PendingTask::new(task.clone())); - if let Some(entry) = pending_tasks.get(&key) { - entry.generation.store(1, Ordering::Relaxed); - } - - let mut worker = create_test_worker("input_d", tx, rx, pending_tasks.clone(), pending_task_count); - let runtime_settings = - MetadataUpdateRuntimeSettings { no_change_cache_ttl_secs: 1, ..MetadataUpdateRuntimeSettings::default() }; - - let stale_instant = Instant::now().checked_sub(Duration::from_secs(2)).unwrap_or_else(Instant::now); - worker.recently_completed_no_change.insert(key.clone(), (stale_instant, ResolveReason::Info.into())); - assert!(!worker.should_skip_recent_no_change_task(&key, &task, &runtime_settings)); - assert!(!worker.recently_completed_no_change.contains_key(&key)); - - let requeued = worker.finalize_processed_task_success(&key, 0, "input_d").await; - assert!(requeued); - assert!(pending_tasks.contains_key(&key)); - assert_eq!(worker.receiver.try_recv().expect("requeued signal should be present"), key); - } - - #[test] - fn task_needs_provider_connection_skips_library_probe_stream() { - let task = UpdateTask::ProbeStream { - probe_scope: Arc::from("input_a"), - unique_id: "u1".to_string(), - url: "file:///movie.mkv".to_string(), - item_type: PlaylistItemType::LocalVideo, - reason: ResolveReason::MissingDetails.into(), - delay: 0, - }; - - assert!(!InputWorker::task_needs_provider_connection(&task, InputType::Library)); - } - - #[test] - fn task_needs_provider_connection_skips_media_server_probe_stream() { - let task = UpdateTask::ProbeStream { - probe_scope: Arc::from("input_remote"), - unique_id: "u1".to_string(), - url: "http://example.com/stream.mkv".to_string(), - item_type: PlaylistItemType::Video, - reason: ResolveReason::MissingDetails.into(), - delay: 0, - }; - - for input_type in [InputType::Emby, InputType::Jellyfin, InputType::Plex] { - assert!(!InputWorker::task_needs_provider_connection(&task, input_type)); - } - assert!(InputWorker::task_needs_provider_connection(&task, InputType::M3u)); - } - - #[test] - fn task_needs_provider_connection_keeps_non_library_probe_stream() { - let task = UpdateTask::ProbeStream { - probe_scope: Arc::from("input_a"), - unique_id: "u1".to_string(), - url: "http://example.com/stream.m3u8".to_string(), - item_type: PlaylistItemType::Video, - reason: ResolveReason::MissingDetails.into(), - delay: 0, - }; - - assert!(InputWorker::task_needs_provider_connection(&task, InputType::M3u)); - assert!(InputWorker::task_needs_provider_connection(&task, InputType::Xtream)); - } - - #[test] - fn task_needs_provider_connection_keeps_live_probe() { - let task = UpdateTask::ProbeLive { - id: ProviderIdType::Id(1), - reason: ResolveReason::Probe.into(), - delay: 0, - interval: 60, - }; - - assert!(InputWorker::task_needs_provider_connection(&task, InputType::Library)); - } - - /// Run the `collect_*_virtual_updates` text-id resolution test for any - /// cluster (vod / series / live). The macro emits the shared tempdir + - /// mapping + batch + collect + assert scaffolding; the cluster-specific - /// bits (`$fn_name`, `$item_type`, `$input_name`, `$text_id`, - /// `$add_method`, `$props`) are passed by the caller. - macro_rules! assert_collect_resolves_text_id { - ($fn_name:ident, $item_type:expr, $input_name:expr, $text_id:expr, $add_method:ident, $props:expr) => { - let dir = tempdir().expect("tempdir should be created"); - let mapping_path = dir.path().join("target_id_mapping.db"); - let mut mapping = TargetIdMapping::new(&mapping_path, false).expect("mapping should be created"); - let uuid = generate_provider_playlist_uuid($input_name, $text_id, $item_type); - let virtual_id = mapping.get_and_update_virtual_id(&uuid, 0, $item_type, VirtualId::default()); - mapping.persist().expect("mapping should persist"); - - let mut batch = BatchResultCollector::new(); - batch.$add_method(ProviderIdType::from($text_id), $props); - - let mut provider_virtual_ids = HashMap::new(); - let mut uuid_virtual_ids = HashMap::new(); - let updates = - InputWorker::$fn_name(&mapping, $input_name, &batch, &mut provider_virtual_ids, &mut uuid_virtual_ids); - - assert!(updates.contains_key(&virtual_id)); - }; - } - - #[test] - fn collect_series_virtual_updates_resolves_text_ids_via_series_info_uuid() { - assert_collect_resolves_text_id!( - collect_series_virtual_updates, - PlaylistItemType::SeriesInfo, - "input_series", - "series-text-id", - add_series, - SeriesStreamProperties::default() - ); - } - - #[test] - fn collect_vod_virtual_updates_resolves_text_ids_via_video_uuid() { - assert_collect_resolves_text_id!( - collect_vod_virtual_updates, - PlaylistItemType::Video, - "input_vod", - "vod-text-id", - add_vod, - VideoStreamProperties::default() - ); - } - - #[test] - fn collect_live_virtual_updates_resolves_text_ids_via_live_uuid() { - assert_collect_resolves_text_id!( - collect_live_virtual_updates, - PlaylistItemType::Live, - "input_live", - "live-text-id", - add_live, - LiveStreamProperties::default() - ); - } - - #[test] - fn generic_probe_uses_pending_vod_batch_as_update_base() { - let mut batch = BatchResultCollector::new(); - let pending_props = - VideoStreamProperties { container_extension: Arc::from("mkv"), ..VideoStreamProperties::default() }; - batch.add_vod(ProviderIdType::Id(42), pending_props); - - let mut item = XtreamPlaylistItem { - virtual_id: VirtualId::new(7), - provider_id: 42, - name: Arc::from("Movie"), - logo: Arc::from(""), - logo_small: Arc::from(""), - group: Arc::from(""), - title: Arc::from(""), - parent_code: Arc::from(""), - rec: Arc::from(""), - url: Arc::from(""), - epg_channel_id: None, - xtream_cluster: XtreamCluster::Video, - additional_properties: None, - item_type: PlaylistItemType::Video, - category_id: 0, - input_name: Arc::from("input"), - channel_no: 0, - source_ordinal: 0, - input_stream_id: Arc::from("42"), - upstream_user_agent: None, - }; - - InputWorker::apply_pending_generic_probe_base(&batch, XtreamCluster::Video, 42, &mut item); - - let Some(StreamProperties::Video(props)) = item.additional_properties else { - panic!("expected pending video properties"); - }; - assert_eq!(props.container_extension.as_ref(), "mkv"); - } - - #[test] - fn strip_tmdb_reasons_returns_none_for_tmdb_only_resolve_task() { - let task = UpdateTask::ResolveVod { - id: ProviderIdType::Id(100), - reason: ResolveReason::Tmdb | ResolveReason::Date, - delay: 5, - source_last_modified: None, - }; - - assert!(InputWorker::strip_tmdb_reasons(&task).is_none()); - } - - #[test] - fn strip_tmdb_reasons_keeps_non_tmdb_reasons() { - let task = UpdateTask::ResolveSeries { - id: ProviderIdType::Id(5), - reason: ResolveReason::Tmdb | ResolveReason::Probe | ResolveReason::Info, - delay: 1, - source_last_modified: Some(123), - }; - - let stripped = InputWorker::strip_tmdb_reasons(&task).expect("task should keep non-tmdb reasons"); - match stripped { - UpdateTask::ResolveSeries { reason, .. } => { - assert!(!reason.contains(ResolveReason::Tmdb)); - assert!(!reason.contains(ResolveReason::Date)); - assert!(reason.contains(ResolveReason::Probe)); - assert!(reason.contains(ResolveReason::Info)); - } - other => panic!("unexpected task type after strip: {other:?}"), - } - } - - #[test] - fn merge_task_payload_resolve_vod_unknown_last_modified_wins() { - let mut existing = UpdateTask::ResolveVod { - id: ProviderIdType::Id(7), - reason: ResolveReason::Info.into(), - delay: 5, - source_last_modified: Some(123), - }; - let changed = MetadataUpdateManager::merge_task_payload( - &mut existing, - UpdateTask::ResolveVod { - id: ProviderIdType::Id(7), - reason: ResolveReason::Probe.into(), - delay: 3, - source_last_modified: None, - }, - ); - - assert!(changed); - match existing { - UpdateTask::ResolveVod { reason, delay, source_last_modified, .. } => { - assert!(reason.contains(ResolveReason::Info)); - assert!(reason.contains(ResolveReason::Probe)); - assert_eq!(delay, 3); - assert_eq!(source_last_modified, None); - } - other => panic!("unexpected merged task type: {other:?}"), - } - } - - #[test] - fn merge_task_payload_resolve_series_unknown_last_modified_wins() { - let mut existing = UpdateTask::ResolveSeries { - id: ProviderIdType::Id(9), - reason: ResolveReason::Info.into(), - delay: 2, - source_last_modified: None, - }; - let changed = MetadataUpdateManager::merge_task_payload( - &mut existing, - UpdateTask::ResolveSeries { - id: ProviderIdType::Id(9), - reason: ResolveReason::Tmdb.into(), - delay: 2, - source_last_modified: Some(456), - }, - ); - - assert!(changed); - match existing { - UpdateTask::ResolveSeries { reason, delay, source_last_modified, .. } => { - assert!(reason.contains(ResolveReason::Info)); - assert!(reason.contains(ResolveReason::Tmdb)); - assert_eq!(delay, 2); - assert_eq!(source_last_modified, None); - } - other => panic!("unexpected merged task type: {other:?}"), - } - } - - #[tokio::test] - async fn flush_dirty_retry_states_retries_persist_after_transient_failure() { - let dir = tempdir().expect("tempdir should be created"); - let failing_path = dir.path().join("missing").join("metadata_retry_state.db"); - let success_path = dir.path().join("metadata_retry_state.db"); - let key = TaskKey::Vod(42); - let state = TaskRetryState { - resolve: Some(RetryState { - attempts: 2, - next_allowed_at_ts: 1_700_043_200, - cooldown_until_ts: None, - last_error: Some("temporary failure".to_string()), - source_last_modified: None, - }), - probe: None, - tmdb: None, - updated_at_ts: 1_700_000_000, - }; - let (tx, rx) = mpsc::channel::(8); - let pending_tasks = Arc::new(DashMap::new()); - let pending_task_count = Arc::new(AtomicUsize::new(0)); - let mut worker = create_test_worker("input_retry_flush", tx, rx, pending_tasks, pending_task_count); - - worker.metadata_retry_state_path = Some(failing_path); - worker.retry_states.insert(key.clone(), state.clone()); - worker.dirty_retry_state_keys.insert(key.clone()); - - worker.persist_metadata_retry_state(&key, Some(&state)).await; - assert!(worker.dirty_retry_state_keys.contains(&key)); - - worker.metadata_retry_state_path = Some(success_path.clone()); - worker.flush_dirty_retry_states().await; - - assert!(!worker.dirty_retry_state_keys.contains(&key)); - let loaded = - load_metadata_retry_states_from_disk(&success_path).expect("state load should succeed after retry"); - let loaded_state = loaded.get(&key).expect("retry state should be persisted on flush"); - let loaded_resolve = loaded_state.resolve.as_ref().expect("resolve retry state should be present"); - assert_eq!(loaded_resolve.attempts, 2); - assert_eq!(loaded_resolve.last_error.as_deref(), Some("temporary failure")); - } - - #[test] - fn metadata_retry_state_disk_roundtrip() { - let dir = tempdir().expect("tempdir should be created"); - let path = dir.path().join("metadata_retry_state.db"); - let key = TaskKey::Stream { scope: Arc::from("input_a"), id: Arc::from("stream_1") }; - let state = TaskRetryState { - resolve: Some(RetryState { - attempts: 4, - next_allowed_at_ts: 1_700_043_200, - cooldown_until_ts: Some(1_700_043_200), - last_error: Some("resolve exhausted".to_string()), - source_last_modified: None, - }), - probe: Some(RetryState { - attempts: 3, - next_allowed_at_ts: 1_700_000_000, - cooldown_until_ts: Some(1_700_086_400), - last_error: Some("probe timeout".to_string()), - source_last_modified: None, - }), - tmdb: Some(RetryState { - attempts: 0, - next_allowed_at_ts: 1_700_172_800, - cooldown_until_ts: Some(1_700_172_800), - last_error: Some("tmdb no match".to_string()), - source_last_modified: Some(999), - }), - updated_at_ts: 1_700_000_000, - }; - - persist_metadata_retry_state_to_disk(&path, &key, Some(&state)).expect("state persistence should succeed"); - let loaded = load_metadata_retry_states_from_disk(&path).expect("state load should succeed"); - let loaded_state = loaded.get(&key).expect("probe key should be present"); - - let loaded_resolve = loaded_state.resolve.as_ref().expect("resolve retry state should be present"); - assert_eq!(loaded_resolve.attempts, 4); - assert_eq!(loaded_resolve.cooldown_until_ts, Some(1_700_043_200)); - assert_eq!(loaded_resolve.last_error.as_deref(), Some("resolve exhausted")); - let loaded_probe = loaded_state.probe.as_ref().expect("probe retry state should be present"); - assert_eq!(loaded_probe.attempts, 3); - assert_eq!(loaded_probe.cooldown_until_ts, Some(1_700_086_400)); - assert_eq!(loaded_probe.last_error.as_deref(), Some("probe timeout")); - let loaded_tmdb = loaded_state.tmdb.as_ref().expect("tmdb retry state should be present"); - assert_eq!(loaded_tmdb.cooldown_until_ts, Some(1_700_172_800)); - assert_eq!(loaded_tmdb.last_error.as_deref(), Some("tmdb no match")); - assert_eq!(loaded_tmdb.source_last_modified, Some(999)); - - persist_metadata_retry_state_to_disk(&path, &key, None).expect("state clear should succeed"); - let cleared = load_metadata_retry_states_from_disk(&path).expect("state reload should succeed"); - assert!(!cleared.contains_key(&key)); - } - - #[test] - fn msgpack_backward_compat_old_4field_retry_state() { - // Simulate old 4-field format (before source_last_modified was added) - #[derive(Debug, serde::Serialize)] - struct OldRetryStateDbValue { - attempts: u8, - next_allowed_at_ts: i64, - cooldown_until_ts: Option, - last_error: Option, - } - - let old = OldRetryStateDbValue { - attempts: 3, - next_allowed_at_ts: 1_700_000_000, - cooldown_until_ts: Some(1_700_043_200), - last_error: Some("HTTP 404".to_string()), - }; - - let bytes = rmp_serde::to_vec(&old).expect("old format should serialize"); - let result = rmp_serde::from_slice::(&bytes); - assert!( - result.is_ok(), - "deserializing old 4-field format into new 5-field struct should succeed, got: {result:?}" - ); - let loaded = result.unwrap(); - assert_eq!(loaded.attempts, 3); - assert_eq!(loaded.cooldown_until_ts, Some(1_700_043_200)); - assert_eq!(loaded.source_last_modified, None); - } - - #[test] - fn probe_backoff_steps_follow_expected_windows() { - let runtime_settings = MetadataUpdateRuntimeSettings::default(); - let first = InputWorker::compute_probe_retry_backoff_secs(1, &runtime_settings); - let second = InputWorker::compute_probe_retry_backoff_secs(2, &runtime_settings); - let third = InputWorker::compute_probe_retry_backoff_secs(3, &runtime_settings); - - assert!((480..=720).contains(&first), "expected ~10m with jitter, got {first}"); - assert!((1_440..=2_160).contains(&second), "expected ~30m with jitter, got {second}"); - assert!((2_880..=4_320).contains(&third), "expected ~60m with jitter, got {third}"); - } - - #[test] - fn transient_worker_errors_include_connection_unavailable() { - assert!(InputWorker::is_transient_worker_error(TASK_ERR_UPDATE_IN_PROGRESS)); - assert!(InputWorker::is_transient_worker_error(TASK_ERR_PREEMPTED)); - assert!(InputWorker::is_transient_worker_error(TASK_ERR_NO_CONNECTION)); - assert!(!InputWorker::is_transient_worker_error("permanent error")); - } - - #[test] - fn permanent_not_found_error_matches_standalone_markers() { - assert!(InputWorker::is_permanent_not_found_error("HTTP 404 Not Found")); - assert!(InputWorker::is_permanent_not_found_error("probe failed: 404: stream unavailable")); - assert!(InputWorker::is_permanent_not_found_error("resource not found on provider")); - } - - #[test] - fn permanent_not_found_error_ignores_partial_markers() { - assert!(!InputWorker::is_permanent_not_found_error("error code 1404 while probing")); - assert!(!InputWorker::is_permanent_not_found_error("status404unexpected")); - assert!(!InputWorker::is_permanent_not_found_error("movie not foundry metadata mismatch")); - } - - #[test] - fn retry_domain_uses_probe_for_probe_only_resolve_tasks() { - let task = UpdateTask::ResolveVod { - id: ProviderIdType::Id(7), - reason: ResolveReason::Probe.into(), - delay: 0, - source_last_modified: None, - }; - assert_eq!(InputWorker::retry_domain_for_task(&task), RetryDomain::Probe); - } - - #[test] - fn retry_domain_keeps_resolve_for_mixed_resolve_tasks() { - let task = UpdateTask::ResolveSeries { - id: ProviderIdType::Id(11), - reason: ResolveReason::Probe | ResolveReason::Info, - delay: 0, - source_last_modified: None, - }; - assert_eq!(InputWorker::retry_domain_for_task(&task), RetryDomain::Resolve); - } - - #[test] - fn take_pending_probe_task_snapshot_skips_scheduled_requeues() { - let (tx, rx) = mpsc::channel::(8); - let pending_tasks = Arc::new(DashMap::new()); - let pending_task_count = Arc::new(AtomicUsize::new(1)); - let worker = create_test_worker("input_probe_skip", tx, rx, pending_tasks.clone(), pending_task_count); - - let key = TaskKey::Live(100); - let task = UpdateTask::ProbeLive { - id: ProviderIdType::Id(100), - reason: ResolveReason::Probe.into(), - delay: 0, - interval: 60, - }; - pending_tasks.insert(key.clone(), PendingTask::new(task)); - worker.scheduled_requeues.insert(key, chrono::Utc::now().timestamp().saturating_add(30)); - - assert!(worker.take_pending_probe_task_snapshot().is_none()); - } - - #[test] - fn take_pending_probe_task_snapshot_accepts_probe_only_resolve() { - let (tx, rx) = mpsc::channel::(8); - let pending_tasks = Arc::new(DashMap::new()); - let pending_task_count = Arc::new(AtomicUsize::new(1)); - let worker = create_test_worker("input_probe_only_resolve", tx, rx, pending_tasks.clone(), pending_task_count); - - let key = TaskKey::Vod(77); - let task = UpdateTask::ResolveVod { - id: ProviderIdType::Id(77), - reason: ResolveReason::Probe.into(), - delay: 0, - source_last_modified: None, - }; - pending_tasks.insert(key.clone(), PendingTask::new(task)); - - let snapshot = worker.take_pending_probe_task_snapshot().expect("expected probe-domain snapshot"); - assert_eq!(snapshot.0, key); - assert_eq!(InputWorker::retry_domain_for_task(&snapshot.1), RetryDomain::Probe); - } - - #[test] - fn playlist_trigger_ignores_probe_only_changes() { - let task = UpdateTask::ProbeLive { - id: ProviderIdType::Id(22), - reason: ResolveReason::Probe.into(), - delay: 0, - interval: 60, - }; - assert!(!InputWorker::should_trigger_playlist_update_for_task(&task, true)); - } - - #[test] - fn playlist_trigger_keeps_info_changes() { - let task = UpdateTask::ResolveSeries { - id: ProviderIdType::Id(33), - reason: ResolveReason::Info.into(), - delay: 0, - source_last_modified: None, - }; - assert!(InputWorker::should_trigger_playlist_update_for_task(&task, true)); - assert!(!InputWorker::should_trigger_playlist_update_for_task(&task, false)); - } -} +use self::task_queue::SubmitTaskResult; +use self::{ + publication::DbHandle, + retry::{RetryDomain, RetryState, TaskRetryState}, + retry_store::{load_metadata_retry_states_from_disk, persist_metadata_retry_state_to_disk}, + settings::{spawn_blocking_limited, MetadataUpdateRuntimeSettings}, + task_queue::{PendingTask, ScopedTaskKey}, + worker::{InputWorker, InputWorkerContext}, +}; diff --git a/backend/metadata/src/manager/publication.rs b/backend/metadata/src/manager/publication.rs new file mode 100644 index 000000000..e6720e815 --- /dev/null +++ b/backend/metadata/src/manager/publication.rs @@ -0,0 +1,308 @@ +use super::{spawn_blocking_limited, InputWorker, ScopedTaskKey, TaskKey}; +use crate::ctx::MetadataUpdateCtx; +use log::error; +use parking_lot::Mutex as ParkingMutex; +use shared::model::{ + EventSink, LiveStreamProperties, SeriesStreamProperties, UUIDType, VideoStreamProperties, VirtualId, XtreamCluster, + XtreamPlaylistItem, +}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; +use tuliprox_core::{ + model::{BatchResultCollector, ProviderIdType, UpdateTask}, + utils::FileReadGuard, +}; +use tuliprox_repository::{ + get_input_storage_path, get_target_id_mapping_file, persist_input_live_info_batch, persist_input_series_info_batch, + persist_input_vod_info_batch, xtream_get_file_path, BPlusTreeQuery, TargetIdMapping, +}; + +pub(super) struct DbHandle { + pub(super) _guard: FileReadGuard, + pub(super) query: Arc>>, +} + +impl InputWorker { + pub(super) fn set_tmdb_source_marker(&self, current_key: &TaskKey, source_last_modified: u64) { + let scoped_key = ScopedTaskKey::new(self.input_name.clone(), current_key.clone()); + self.tmdb_source_markers.insert(scoped_key, source_last_modified); + } + + pub(super) fn clear_tmdb_source_marker(&self, current_key: &TaskKey) { + let scoped_key = ScopedTaskKey::new(self.input_name.clone(), current_key.clone()); + self.tmdb_source_markers.remove(&scoped_key); + } + + #[inline] + pub(super) fn should_trigger_playlist_update_for_task(task: &UpdateTask, task_changed: bool) -> bool { + task_changed && !Self::is_probe_task(task) && !Self::is_probe_only_resolve_task(task) + } + + pub(super) async fn flush_batch_static( + input_name: &str, + bound_ctx: Option<&MetadataUpdateCtx>, + batch_buffer: &mut BatchResultCollector, + ) { + if batch_buffer.is_empty() { + return; + } + + let Some(ctx) = bound_ctx else { return }; + let app_config = &ctx.app_config; + let cfg = app_config.config.load(); + let vod_updates = batch_buffer.take_vod_updates(); + let series_updates = batch_buffer.take_series_updates(); + let live_updates = batch_buffer.take_live_updates(); + + if vod_updates.is_empty() && series_updates.is_empty() && live_updates.is_empty() { + return; + } + + if let Ok(storage_path) = get_input_storage_path(input_name, &cfg.storage_dir).await { + if !vod_updates.is_empty() { + let mut updates: Vec<(u32, VideoStreamProperties)> = Vec::with_capacity(vod_updates.len()); + for (id, props) in &vod_updates { + if let ProviderIdType::Id(vid) = id { + updates.push((*vid, props.clone())); + } + } + + if !updates.is_empty() { + if let Err(e) = persist_input_vod_info_batch( + app_config, + &storage_path, + XtreamCluster::Video, + input_name, + updates, + ) + .await + { + error!("Failed to flush VOD batch for input {input_name}: {e}"); + } + } + } + + if !series_updates.is_empty() { + let mut updates: Vec<(u32, SeriesStreamProperties)> = Vec::with_capacity(series_updates.len()); + for (id, props) in &series_updates { + if let ProviderIdType::Id(vid) = id { + updates.push((*vid, props.clone())); + } + } + + if !updates.is_empty() { + if let Err(e) = persist_input_series_info_batch( + app_config, + &storage_path, + XtreamCluster::Series, + input_name, + updates, + ) + .await + { + error!("Failed to flush Series batch for input {input_name}: {e}"); + } + } + } + + if !live_updates.is_empty() { + let mut updates: Vec<(u32, LiveStreamProperties)> = Vec::with_capacity(live_updates.len()); + for (id, props) in &live_updates { + if let ProviderIdType::Id(vid) = id { + updates.push((*vid, props.clone())); + } + } + + if !updates.is_empty() { + if let Err(e) = persist_input_live_info_batch( + app_config, + &storage_path, + XtreamCluster::Live, + input_name, + updates, + ) + .await + { + error!("Failed to flush Live batch for input {input_name}: {e}"); + } + } + } + } + + let cascade_batch = BatchResultCollector { vod: vod_updates, series: series_updates, live: live_updates }; + + Self::cascade_updates(ctx, &app_config.config.load(), input_name, &cascade_batch).await; + } + + #[allow(clippy::too_many_lines)] + async fn cascade_updates( + ctx: &MetadataUpdateCtx, + config: &tuliprox_core::model::Config, + input_name: &str, + batch: &BatchResultCollector, + ) { + if batch.is_empty() { + return; + } + + // Find targets affected by this input. + let targets = { + let sources = ctx.app_config.sources.load(); + let mut affected_targets = Vec::new(); + + for source in &sources.sources { + if source.inputs.iter().any(|i_name| i_name.as_ref() == input_name) { + for t_def in &source.targets { + affected_targets.push(t_def.clone()); + } + } + } + affected_targets + }; + + if targets.is_empty() { + return; + } + + for target in targets { + let target_name = &target.name; + let Some(target_path) = tuliprox_repository::get_target_storage_path(config, target_name) else { + continue; + }; + let Some(storage_path) = tuliprox_repository::xtream_get_storage_path(config, target_name) else { + continue; + }; + let mapping_file = get_target_id_mapping_file(&target_path); + + let mapping = { + // Scope read lock strictly to mapping load. + let _file_lock = ctx.app_config.file_locks.read_lock(&mapping_file).await; + let mapping_file_clone = mapping_file.clone(); + match spawn_blocking_limited(move || TargetIdMapping::new(&mapping_file_clone, false)).await { + Ok(Ok(mapping)) => mapping, + Ok(Err(e)) => { + error!("Failed to open ID mapping for target {target_name}: {e}"); + continue; + } + Err(err) => { + error!("Failed to open ID mapping for target {target_name}: {err}"); + continue; + } + } + }; + + let mut provider_virtual_ids: HashMap> = HashMap::new(); + let mut uuid_virtual_ids: HashMap> = HashMap::new(); + + let vod_virtual_updates = Self::collect_vod_virtual_updates( + &mapping, + input_name, + batch, + &mut provider_virtual_ids, + &mut uuid_virtual_ids, + ); + Self::apply_vod_cascade_updates(ctx, &target, &storage_path, vod_virtual_updates).await; + + let series_virtual_updates = Self::collect_series_virtual_updates( + &mapping, + input_name, + batch, + &mut provider_virtual_ids, + &mut uuid_virtual_ids, + ); + Self::apply_series_cascade_updates(ctx, &target, &storage_path, series_virtual_updates).await; + + let live_virtual_updates = Self::collect_live_virtual_updates( + &mapping, + input_name, + batch, + &mut provider_virtual_ids, + &mut uuid_virtual_ids, + ); + Self::apply_live_cascade_updates(ctx, &target, &storage_path, live_virtual_updates).await; + } + } + + pub(super) fn get_cached_uuid_virtual_id( + mapping: &TargetIdMapping, + cache: &mut HashMap>, + uuid: UUIDType, + ) -> Option { + if let Some(cached) = cache.get(&uuid) { + return *cached; + } + let resolved = mapping.get_virtual_id_by_uuid(&uuid); + cache.insert(uuid, resolved); + resolved + } + + pub(super) async fn update_memory_cache( + ctx: &MetadataUpdateCtx, + target_name: &str, + cluster: XtreamCluster, + updates: Vec, + ) { + let mut playlists = ctx.playlists.data.write().await; + if let Some(playlist) = playlists.get_mut(target_name) { + if let Some(xtream_storage) = &mut playlist.xtream { + let storage = match cluster { + XtreamCluster::Live => &mut xtream_storage.live, + XtreamCluster::Video => &mut xtream_storage.vod, + XtreamCluster::Series => &mut xtream_storage.series, + }; + for item in updates { + storage.insert(item.virtual_id.get(), item); + } + } + } + } + pub(super) async fn get_or_open_query( + input_name: &str, + ctx: &MetadataUpdateCtx, + cluster: XtreamCluster, + db_handles: &mut HashMap, + failed_clusters: &mut HashSet, + ) -> Option>>> { + if failed_clusters.contains(&cluster) { + return None; + } + + if let std::collections::hash_map::Entry::Vacant(entry) = db_handles.entry(cluster) { + let cfg = ctx.app_config.config.load(); + if let Ok(storage_path) = get_input_storage_path(input_name, &cfg.storage_dir).await { + let file_path = xtream_get_file_path(&storage_path, cluster); + if file_path.exists() { + let lock = ctx.app_config.file_locks.read_lock(&file_path).await; + let file_path = file_path.clone(); + let query = match spawn_blocking_limited(move || { + BPlusTreeQuery::::try_new(&file_path) + }) + .await + { + Ok(Ok(query)) => Some(query), + Ok(Err(err)) => { + error!("Failed to open BPlusTreeQuery for {cluster}: {err}"); + None + } + Err(err) => { + error!("Failed to open BPlusTreeQuery for {cluster}: {err}"); + None + } + }; + + if let Some(query) = query { + entry.insert(DbHandle { _guard: lock, query: Arc::new(ParkingMutex::new(query)) }); + } else { + failed_clusters.insert(cluster); + } + } else { + // File doesn't exist; do not mark as failure to allow future creation. + } + } + } + + db_handles.get(&cluster).map(|h| Arc::clone(&h.query)) + } +} diff --git a/backend/metadata/src/manager/retry.rs b/backend/metadata/src/manager/retry.rs new file mode 100644 index 000000000..12322eec1 --- /dev/null +++ b/backend/metadata/src/manager/retry.rs @@ -0,0 +1,489 @@ +use super::{ + load_metadata_retry_states_from_disk, persist_metadata_retry_state_to_disk, spawn_blocking_limited, InputWorker, + MetadataUpdateManager, MetadataUpdateRuntimeSettings, ScopedTaskKey, TaskKey, METADATA_RETRY_STATE_FILE, + RETRY_STATE_MIN_TTL_SECS, RETRY_STATE_PRUNE_INTERVAL_SECS, TASK_ERR_NO_CONNECTION, TASK_ERR_PREEMPTED, + TASK_ERR_UPDATE_IN_PROGRESS, +}; +use crate::ctx::MetadataUpdateCtx; +use log::warn; +use shared::model::EventSink; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; +use tuliprox_core::{model::UpdateTask, utils::debug_if_enabled}; +use tuliprox_repository::get_input_storage_path; + +#[derive(Debug, Clone)] +pub(super) struct RetryState { + pub(super) attempts: u8, + pub(super) next_allowed_at_ts: i64, + pub(super) cooldown_until_ts: Option, + pub(super) last_error: Option, + pub(super) source_last_modified: Option, +} + +impl RetryState { + pub(super) fn new() -> Self { + Self { + attempts: 0, + next_allowed_at_ts: 0, + cooldown_until_ts: None, + last_error: None, + source_last_modified: None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum RetryDomain { + Resolve, + Probe, + Tmdb, +} + +#[derive(Debug, Clone, Default)] +pub(super) struct TaskRetryState { + pub(super) resolve: Option, + pub(super) probe: Option, + pub(super) tmdb: Option, + pub(super) updated_at_ts: i64, +} + +impl TaskRetryState { + pub(super) fn is_empty(&self) -> bool { self.resolve.is_none() && self.probe.is_none() && self.tmdb.is_none() } + + pub(super) fn touch(&mut self, now_ts: i64) { self.updated_at_ts = now_ts.max(1); } + + pub(super) fn max_domain_timestamp(&self) -> i64 { + let domain_max = |state: &RetryState| state.next_allowed_at_ts.max(state.cooldown_until_ts.unwrap_or(0)); + self.resolve + .as_ref() + .map_or(0, domain_max) + .max(self.probe.as_ref().map_or(0, domain_max)) + .max(self.tmdb.as_ref().map_or(0, domain_max)) + } + + fn is_stale(&self, now_ts: i64, ttl_secs: i64) -> bool { + let ttl_secs = ttl_secs.max(1); + let anchor_ts = self.updated_at_ts.max(self.max_domain_timestamp()); + now_ts >= anchor_ts.saturating_add(ttl_secs) + } + + pub(super) fn get(&self, domain: RetryDomain) -> Option<&RetryState> { + match domain { + RetryDomain::Resolve => self.resolve.as_ref(), + RetryDomain::Probe => self.probe.as_ref(), + RetryDomain::Tmdb => self.tmdb.as_ref(), + } + } + + pub(super) fn get_mut_or_insert(&mut self, domain: RetryDomain) -> &mut RetryState { + let slot = match domain { + RetryDomain::Resolve => &mut self.resolve, + RetryDomain::Probe => &mut self.probe, + RetryDomain::Tmdb => &mut self.tmdb, + }; + slot.get_or_insert_with(RetryState::new) + } + + pub(super) fn clear_domain(&mut self, domain: RetryDomain) { + match domain { + RetryDomain::Resolve => self.resolve = None, + RetryDomain::Probe => self.probe = None, + RetryDomain::Tmdb => self.tmdb = None, + } + } +} + +impl MetadataUpdateManager { + pub(super) fn prune_resolve_enqueue_suppressions(&self, now_ts: i64) { + let last_pruned_at = self.last_resolve_enqueue_suppression_prune_at_ts.load(Ordering::Relaxed); + if last_pruned_at != 0 && now_ts.saturating_sub(last_pruned_at) < RETRY_STATE_PRUNE_INTERVAL_SECS { + return; + } + self.last_resolve_enqueue_suppression_prune_at_ts.store(now_ts, Ordering::Relaxed); + self.resolve_enqueue_suppressions.retain(|_, suppressed_until_ts| *suppressed_until_ts > now_ts); + } +} + +impl InputWorker { + pub(super) async fn ensure_metadata_retry_state_loaded( + &mut self, + input_name: &str, + bound_ctx: Option<&MetadataUpdateCtx>, + runtime_settings: &MetadataUpdateRuntimeSettings, + ) { + if self.metadata_retry_loaded { + return; + } + let now_ts = chrono::Utc::now().timestamp(); + if self.metadata_retry_load_retry_at_ts.is_some_and(|retry_at_ts| now_ts < retry_at_ts) { + return; + } + + let Some(ctx) = bound_ctx else { + self.metadata_retry_load_retry_at_ts = + Some(now_ts.saturating_add(runtime_settings.metadata_retry_load_retry_delay_secs)); + return; + }; + + let storage_dir = ctx.app_config.config.load().storage_dir.clone(); + let Ok(storage_path) = get_input_storage_path(input_name, &storage_dir).await else { + warn!("Could not resolve storage path for metadata retry state on input {input_name}"); + self.metadata_retry_load_retry_at_ts = + Some(now_ts.saturating_add(runtime_settings.metadata_retry_load_retry_delay_secs)); + return; + }; + + let retry_path = storage_path.join(METADATA_RETRY_STATE_FILE); + self.metadata_retry_state_path = Some(retry_path.clone()); + + let loaded = spawn_blocking_limited(move || load_metadata_retry_states_from_disk(&retry_path)) + .await + .map_err(|err| err.to_string()) + .and_then(|result| result.map_err(|err| err.to_string())); + + let loaded = match loaded { + Ok(states) => states, + Err(err) => { + warn!("Failed to load metadata retry state for input {input_name}: {err}"); + self.metadata_retry_load_retry_at_ts = + Some(now_ts.saturating_add(runtime_settings.metadata_retry_load_retry_delay_secs)); + return; + } + }; + + // Intentionally do not resurrect pending tasks solely from persisted retry state. + // The persisted state is applied once the corresponding task is naturally queued again + // (for example by the next playlist update), because state alone does not contain the + // full `UpdateTask` payload for all variants. + for (key, state) in loaded { + self.sync_resolve_enqueue_suppression_from_retry_state(&key, &state, now_ts); + self.retry_states.insert(key, state); + } + self.prune_retry_tracking_maps(now_ts, runtime_settings).await; + self.last_retry_state_prune_at_ts = Some(now_ts); + self.metadata_retry_loaded = true; + self.metadata_retry_load_retry_at_ts = None; + } + + pub(super) async fn persist_metadata_retry_state(&mut self, key: &TaskKey, state: Option<&TaskRetryState>) { + let Some(path) = self.metadata_retry_state_path.clone() else { + return; + }; + if !self.dirty_retry_state_keys.contains(key) { + return; + } + + let key_for_persist = key.clone(); + let key_for_dirty = key.clone(); + let state = state.cloned(); + let input_name = self.input_name.clone(); + let persist_result = spawn_blocking_limited(move || { + persist_metadata_retry_state_to_disk(&path, &key_for_persist, state.as_ref()) + }) + .await; + + match persist_result { + Ok(Ok(())) => { + self.dirty_retry_state_keys.remove(&key_for_dirty); + } + Ok(Err(err)) => warn!("Failed to persist metadata retry state for input {input_name}: {err}"), + Err(err) => warn!("Failed to persist metadata retry state for input {input_name}: {err}"), + } + } + + pub(super) async fn flush_dirty_retry_states(&mut self) { + if self.dirty_retry_state_keys.is_empty() { + return; + } + + let dirty_keys: Vec = self.dirty_retry_state_keys.iter().cloned().collect(); + for key in dirty_keys { + let state = self.retry_states.get(&key).cloned(); + self.persist_metadata_retry_state(&key, state.as_ref()).await; + } + } + + pub(super) fn schedule_requeue_at(&self, key: TaskKey, retry_at_ts: i64) { + let now_ts = chrono::Utc::now().timestamp(); + let retry_at_ts = retry_at_ts.max(now_ts); + + if self.scheduled_requeues.get(&key).is_some_and(|existing| *existing == retry_at_ts) { + return; + } + + self.scheduled_requeues.insert(key.clone(), retry_at_ts); + + let sender = self.sender.clone(); + let pending_tasks = Arc::clone(&self.pending_tasks); + let pending_task_count = Arc::clone(&self.pending_task_count); + let scheduled = Arc::clone(&self.scheduled_requeues); + let cancel_token = self.cancel_token.clone(); + let input_name = self.input_name.clone(); + + tokio::spawn(async move { + let delay_secs = retry_at_ts.saturating_sub(chrono::Utc::now().timestamp()); + if delay_secs > 0 { + let delay = Duration::from_secs(u64::try_from(delay_secs).unwrap_or(u64::MAX)); + tokio::select! { + () = cancel_token.cancelled() => return, + () = tokio::time::sleep(delay) => {} + } + } + + let should_send = scheduled.get(&key).is_some_and(|scheduled_at| *scheduled_at == retry_at_ts); + if !should_send { + return; + } + scheduled.remove(&key); + + if !pending_tasks.contains_key(&key) { + return; + } + + if sender.send(key.clone()).await.is_err() { + if pending_tasks.remove(&key).is_some() { + MetadataUpdateManager::decrement_pending_task_count(&pending_task_count); + } + warn!("Failed to schedule delayed retry task for input {input_name}"); + } + }); + } + + pub(super) async fn finalize_processed_task_success( + &mut self, + current_key: &TaskKey, + current_generation: u64, + input_name: &str, + ) -> bool { + // Atomically remove processed key and reinsert it if it changed while in-flight. + if let Some((_k, removed_task)) = self.pending_tasks.remove(current_key) { + let latest_generation = removed_task.generation.load(Ordering::Relaxed); + if latest_generation != current_generation { + self.pending_tasks.insert(current_key.clone(), removed_task); + if self.sender.send(current_key.clone()).await.is_err() { + if self.pending_tasks.remove(current_key).is_some() { + MetadataUpdateManager::decrement_pending_task_count(&self.pending_task_count); + } + warn!("Failed to schedule merged task replay for input {input_name}"); + return false; + } + return true; + } + // Task finished and is not reinserted. + MetadataUpdateManager::decrement_pending_task_count(&self.pending_task_count); + } + false + } + + fn retry_state_ttl_secs(runtime_settings: &MetadataUpdateRuntimeSettings) -> i64 { + let max_resolve_backoff = i64::try_from(runtime_settings.max_resolve_retry_backoff_secs).unwrap_or(i64::MAX); + runtime_settings + .tmdb_cooldown_secs + .max(runtime_settings.probe_cooldown_secs) + .max(runtime_settings.resolve_exhaustion_reset_gap_secs) + .max(max_resolve_backoff) + .saturating_mul(6) + .max(RETRY_STATE_MIN_TTL_SECS) + } + + pub(super) async fn prune_retry_tracking_maps_if_needed( + &mut self, + now_ts: i64, + runtime_settings: &MetadataUpdateRuntimeSettings, + ) { + let due = self + .last_retry_state_prune_at_ts + .is_none_or(|last| now_ts.saturating_sub(last) >= RETRY_STATE_PRUNE_INTERVAL_SECS); + if !due { + return; + } + self.last_retry_state_prune_at_ts = Some(now_ts); + self.prune_retry_tracking_maps(now_ts, runtime_settings).await; + } + + async fn prune_retry_tracking_maps(&mut self, now_ts: i64, runtime_settings: &MetadataUpdateRuntimeSettings) { + let retry_state_ttl_secs = Self::retry_state_ttl_secs(runtime_settings); + let resolve_exhausted_ttl_secs = Self::resolve_exhausted_ttl_secs(runtime_settings); + let stale_retry_keys: Vec = self + .retry_states + .iter() + .filter_map( + |(key, state)| { + if state.is_stale(now_ts, retry_state_ttl_secs) { + Some(key.clone()) + } else { + None + } + }, + ) + .collect(); + + for key in stale_retry_keys { + if self.retry_states.remove(&key).is_some() { + self.clear_resolve_enqueue_suppression(&key); + self.clear_tmdb_source_marker(&key); + self.dirty_retry_state_keys.insert(key.clone()); + debug_if_enabled!("Pruned stale metadata retry state for input {}: {:?}", self.input_name, key); + self.persist_metadata_retry_state(&key, None).await; + } + } + + let stale_resolve_exhausted_keys: Vec = self + .resolve_exhausted + .iter() + .filter_map(|(key, exhausted_at_ts)| { + if now_ts.saturating_sub(*exhausted_at_ts) >= resolve_exhausted_ttl_secs { + Some(key.clone()) + } else { + None + } + }) + .collect(); + for key in stale_resolve_exhausted_keys { + self.resolve_exhausted.remove(&key); + self.clear_resolve_enqueue_suppression(&key); + } + + let no_change_ttl = Duration::from_secs(runtime_settings.no_change_cache_ttl_secs); + self.recently_completed_no_change.retain(|_, (completed_at, _)| completed_at.elapsed() < no_change_ttl); + } + + pub(super) fn compute_resolve_retry_backoff_secs( + base_delay_secs: u16, + attempts: u8, + runtime_settings: &MetadataUpdateRuntimeSettings, + ) -> u64 { + let base_delay = u64::from(base_delay_secs).max(runtime_settings.resolve_min_retry_base_secs); + let exp = u32::from(attempts.saturating_sub(1).min(6)); + let without_jitter = + base_delay.saturating_mul(2_u64.saturating_pow(exp)).min(runtime_settings.max_resolve_retry_backoff_secs); + Self::apply_jitter(without_jitter, runtime_settings.backoff_jitter_percent) + } + + pub(super) fn compute_retry_delay_secs( + base_delay_secs: u16, + runtime_settings: &MetadataUpdateRuntimeSettings, + ) -> u64 { + u64::from(base_delay_secs).max(runtime_settings.retry_delay_secs) + } + + pub(super) fn compute_probe_retry_backoff_secs( + attempts: u8, + runtime_settings: &MetadataUpdateRuntimeSettings, + ) -> u64 { + let base_secs = match attempts { + 1 => runtime_settings.probe_retry_backoff_step_1_secs, + 2 => runtime_settings.probe_retry_backoff_step_2_secs, + _ => runtime_settings.probe_retry_backoff_step_3_secs, + }; + Self::apply_jitter(base_secs, runtime_settings.backoff_jitter_percent) + } + + fn apply_jitter(base_secs: u64, jitter_percent: u8) -> u64 { + let jitter_percent = i64::from(jitter_percent); + let jitter_percent = fastrand::i64(-jitter_percent..=jitter_percent); + let base_i64 = i64::try_from(base_secs).unwrap_or(i64::MAX); + let jitter_delta = base_i64.saturating_mul(jitter_percent).saturating_div(100); + let jittered = base_i64.saturating_add(jitter_delta); + u64::try_from(jittered.max(1)).unwrap_or(1) + } + + #[inline] + pub(super) fn retry_domain_for_task(task: &UpdateTask) -> RetryDomain { + if Self::is_probe_task(task) || Self::is_probe_only_resolve_task(task) { + RetryDomain::Probe + } else { + RetryDomain::Resolve + } + } + + pub(super) fn should_skip_recent_no_change_task( + &mut self, + current_key: &TaskKey, + task_for_execution: &UpdateTask, + runtime_settings: &MetadataUpdateRuntimeSettings, + ) -> bool { + let Some((completed_at, cached_reasons)) = self.recently_completed_no_change.get(current_key).copied() else { + return false; + }; + + let ttl = Duration::from_secs(runtime_settings.no_change_cache_ttl_secs); + let current_reasons = Self::task_reason(task_for_execution); + if completed_at.elapsed() < ttl && current_reasons == cached_reasons { + self.scheduled_requeues.remove(current_key); + return true; + } + + self.recently_completed_no_change.remove(current_key); + false + } + + pub(super) fn set_resolve_enqueue_suppression(&self, current_key: &TaskKey, until_ts: i64) { + let scoped_key = ScopedTaskKey::new(self.input_name.clone(), current_key.clone()); + self.resolve_enqueue_suppressions.insert(scoped_key, until_ts); + } + + pub(super) fn clear_resolve_enqueue_suppression(&self, current_key: &TaskKey) { + let scoped_key = ScopedTaskKey::new(self.input_name.clone(), current_key.clone()); + self.resolve_enqueue_suppressions.remove(&scoped_key); + } + + fn sync_resolve_enqueue_suppression_from_retry_state( + &self, + current_key: &TaskKey, + state: &TaskRetryState, + now_ts: i64, + ) { + let Some(resolve_state) = state.resolve.as_ref() else { + self.clear_resolve_enqueue_suppression(current_key); + return; + }; + + let suppressed_until_ts = resolve_state.cooldown_until_ts.unwrap_or(resolve_state.next_allowed_at_ts); + if suppressed_until_ts > now_ts { + self.set_resolve_enqueue_suppression(current_key, suppressed_until_ts); + } else { + self.clear_resolve_enqueue_suppression(current_key); + } + } + + pub(super) fn is_transient_worker_error(message: &str) -> bool { + message == TASK_ERR_UPDATE_IN_PROGRESS || message == TASK_ERR_PREEMPTED || message == TASK_ERR_NO_CONNECTION + } + + #[inline] + pub(super) fn is_permanent_not_found_error(message: &str) -> bool { + let normalized = message.to_ascii_lowercase(); + Self::contains_standalone_fragment(&normalized, "404") + || Self::contains_standalone_fragment(&normalized, "not found") + } + + #[inline] + fn is_word_byte(byte: u8) -> bool { byte.is_ascii_alphanumeric() || byte == b'_' } + + fn contains_standalone_fragment(haystack: &str, fragment: &str) -> bool { + if fragment.is_empty() || haystack.len() < fragment.len() { + return false; + } + + let bytes = haystack.as_bytes(); + let mut search_from = 0usize; + + while let Some(relative_idx) = haystack[search_from..].find(fragment) { + let start = search_from + relative_idx; + let end = start + fragment.len(); + let before_is_word = start > 0 && Self::is_word_byte(bytes[start - 1]); + let after_is_word = end < bytes.len() && Self::is_word_byte(bytes[end]); + + if !before_is_word && !after_is_word { + return true; + } + search_from = end; + } + + false + } +} diff --git a/backend/metadata/src/manager/retry_store.rs b/backend/metadata/src/manager/retry_store.rs new file mode 100644 index 000000000..5f005ea19 --- /dev/null +++ b/backend/metadata/src/manager/retry_store.rs @@ -0,0 +1,140 @@ +use super::{RetryState, TaskKey, TaskRetryState}; +use std::{collections::HashMap, io, path::Path, sync::Arc}; +use tuliprox_repository::{ + BPlusTree, BPlusTreeQuery, BPlusTreeUpdate, MetadataRetryDbKey, MetadataRetryDbValue, RetryStateDbValue, +}; + +fn metadata_retry_key_from_task_key(task_key: &TaskKey) -> MetadataRetryDbKey { + match task_key { + TaskKey::Vod(id) => MetadataRetryDbKey::VodId(*id), + TaskKey::VodStr(id) => MetadataRetryDbKey::VodText(id.as_ref().to_owned()), + TaskKey::Series(id) => MetadataRetryDbKey::SeriesId(*id), + TaskKey::SeriesStr(id) => MetadataRetryDbKey::SeriesText(id.as_ref().to_owned()), + TaskKey::Live(id) => MetadataRetryDbKey::LiveId(*id), + TaskKey::LiveStr(id) => MetadataRetryDbKey::LiveText(id.as_ref().to_owned()), + TaskKey::Stream { scope, id } => { + MetadataRetryDbKey::Stream { scope: scope.as_ref().to_owned(), id: id.as_ref().to_owned() } + } + } +} + +fn metadata_retry_key_into_task_key(value: MetadataRetryDbKey) -> TaskKey { + match value { + MetadataRetryDbKey::VodId(id) => TaskKey::Vod(id), + MetadataRetryDbKey::VodText(id) => TaskKey::VodStr(Arc::from(id)), + MetadataRetryDbKey::SeriesId(id) => TaskKey::Series(id), + MetadataRetryDbKey::SeriesText(id) => TaskKey::SeriesStr(Arc::from(id)), + MetadataRetryDbKey::LiveId(id) => TaskKey::Live(id), + MetadataRetryDbKey::LiveText(id) => TaskKey::LiveStr(Arc::from(id)), + MetadataRetryDbKey::Stream { scope, id } => TaskKey::Stream { scope: Arc::from(scope), id: Arc::from(id) }, + } +} + +fn retry_state_db_from_retry_state(state: &RetryState) -> RetryStateDbValue { + RetryStateDbValue { + attempts: state.attempts, + next_allowed_at_ts: state.next_allowed_at_ts, + cooldown_until_ts: state.cooldown_until_ts, + last_error: state.last_error.clone(), + source_last_modified: state.source_last_modified, + } +} + +fn retry_state_db_into_retry_state(value: RetryStateDbValue) -> Option { + if value.attempts == 0 + && value.next_allowed_at_ts <= 0 + && value.cooldown_until_ts.is_none() + && value.source_last_modified.is_none() + { + return None; + } + Some(RetryState { + attempts: value.attempts, + next_allowed_at_ts: value.next_allowed_at_ts, + cooldown_until_ts: value.cooldown_until_ts, + last_error: value.last_error, + source_last_modified: value.source_last_modified, + }) +} + +fn metadata_retry_value_from_task_retry_state(state: &TaskRetryState, updated_at_ts: i64) -> MetadataRetryDbValue { + MetadataRetryDbValue { + resolve: state.resolve.as_ref().map(retry_state_db_from_retry_state), + probe: state.probe.as_ref().map(retry_state_db_from_retry_state), + tmdb: state.tmdb.as_ref().map(retry_state_db_from_retry_state), + updated_at_ts, + } +} + +fn metadata_retry_value_into_task_retry_state(value: MetadataRetryDbValue) -> Option { + let mut state = TaskRetryState { + resolve: value.resolve.and_then(retry_state_db_into_retry_state), + probe: value.probe.and_then(retry_state_db_into_retry_state), + tmdb: value.tmdb.and_then(retry_state_db_into_retry_state), + updated_at_ts: value.updated_at_ts, + }; + if state.is_empty() { + return None; + } + if state.updated_at_ts <= 0 { + state.updated_at_ts = state.max_domain_timestamp(); + } + Some(state) +} + +fn ensure_metadata_retry_db(path: &Path) -> io::Result<()> { + if path.exists() { + return Ok(()); + } + let mut tree = BPlusTree::::new(); + tree.store(path).map(|_| ()) +} + +pub(super) fn load_metadata_retry_states_from_disk(path: &Path) -> io::Result> { + ensure_metadata_retry_db(path)?; + + let mut result = HashMap::new(); + let mut stale_keys: Vec = Vec::new(); + let mut query = BPlusTreeQuery::::try_new(path)?; + for entry in query.iter() { + let (key, value) = entry?; + if let Some(state) = metadata_retry_value_into_task_retry_state(value.clone()) { + result.insert(metadata_retry_key_into_task_key(key), state); + } else { + stale_keys.push(key); + } + } + drop(query); + + if !stale_keys.is_empty() { + let delete_refs: Vec<&MetadataRetryDbKey> = stale_keys.iter().collect(); + let mut update = BPlusTreeUpdate::::try_new_with_backoff(path)?; + update + .delete_batch(&delete_refs) + .map_err(|e| io::Error::other(format!("cleanup metadata retry tombstones failed: {e}")))?; + } + + Ok(result) +} + +pub(super) fn persist_metadata_retry_state_to_disk( + path: &Path, + task_key: &TaskKey, + state: Option<&TaskRetryState>, +) -> io::Result<()> { + let db_key = metadata_retry_key_from_task_key(task_key); + + ensure_metadata_retry_db(path)?; + + let mut update = BPlusTreeUpdate::::try_new_with_backoff(path)?; + if let Some(retry_state) = state { + let now_ts = chrono::Utc::now().timestamp(); + let value = metadata_retry_value_from_task_retry_state(retry_state, now_ts); + update + .upsert_batch(&[(&db_key, &value)]) + .map_err(|e| io::Error::other(format!("persist metadata retry state failed: {e}")))?; + } else { + update.delete(&db_key).map_err(|e| io::Error::other(format!("delete metadata retry state failed: {e}")))?; + } + Ok(()) +} diff --git a/backend/metadata/src/manager/settings.rs b/backend/metadata/src/manager/settings.rs new file mode 100644 index 000000000..cb8b39cef --- /dev/null +++ b/backend/metadata/src/manager/settings.rs @@ -0,0 +1,98 @@ +use super::{BLOCKING_DB_MAX_CONCURRENCY, BLOCKING_DB_MIN_CONCURRENCY}; +use crate::ctx::MetadataUpdateCtx; +use shared::model::EventSink; +use std::{sync::OnceLock, time::Duration}; +use tokio::sync::Semaphore; +use tuliprox_core::model::MetadataUpdateConfig; + +fn metadata_blocking_concurrency_limit() -> usize { + let parallelism = + std::thread::available_parallelism().map_or(BLOCKING_DB_MIN_CONCURRENCY, std::num::NonZeroUsize::get); + parallelism.saturating_mul(2).clamp(BLOCKING_DB_MIN_CONCURRENCY, BLOCKING_DB_MAX_CONCURRENCY) +} + +fn metadata_blocking_semaphore() -> &'static Semaphore { + static BLOCKING_SEMAPHORE: OnceLock = OnceLock::new(); + BLOCKING_SEMAPHORE.get_or_init(|| Semaphore::new(metadata_blocking_concurrency_limit())) +} + +pub(super) async fn spawn_blocking_limited(task: F) -> Result +where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, +{ + // Throttle B+Tree and file-heavy blocking tasks to avoid saturating Tokio's blocking pool. + let permit = metadata_blocking_semaphore().acquire().await.ok(); + let result = tokio::task::spawn_blocking(task).await; + drop(permit); + result +} + +#[derive(Debug, Clone)] +pub(super) struct MetadataUpdateRuntimeSettings { + pub(super) queue_log_interval: Duration, + pub(super) progress_log_interval: Duration, + pub(super) max_resolve_retry_backoff_secs: u64, + pub(super) resolve_min_retry_base_secs: u64, + pub(super) max_attempts_resolve: u8, + pub(super) max_attempts_probe: u8, + pub(super) resolve_exhaustion_reset_gap_secs: i64, + pub(super) probe_cooldown_secs: i64, + pub(super) tmdb_cooldown_secs: i64, + pub(super) retry_delay_secs: u64, + pub(super) metadata_retry_load_retry_delay_secs: i64, + pub(super) worker_idle_timeout_secs: u64, + pub(super) max_queue_size: usize, + pub(super) no_change_cache_ttl_secs: u64, + pub(super) probe_fairness_resolve_burst: usize, + pub(super) probe_retry_backoff_step_1_secs: u64, + pub(super) probe_retry_backoff_step_2_secs: u64, + pub(super) probe_retry_backoff_step_3_secs: u64, + pub(super) backoff_jitter_percent: u8, +} + +impl Default for MetadataUpdateRuntimeSettings { + fn default() -> Self { + let defaults = MetadataUpdateConfig::default(); + Self::from_metadata_update(&defaults) + } +} + +impl MetadataUpdateRuntimeSettings { + pub(super) fn from_ctx(ctx: Option<&MetadataUpdateCtx>) -> Self { + let metadata_update = ctx.map_or_else(MetadataUpdateConfig::default, |ctx| { + ctx.app_config + .config + .load() + .metadata_update + .as_ref() + .map_or_else(MetadataUpdateConfig::default, Clone::clone) + }); + Self::from_metadata_update(&metadata_update) + } + + pub(super) fn from_metadata_update(cfg: &MetadataUpdateConfig) -> Self { + let to_i64 = |v: u64| i64::try_from(v.max(1)).unwrap_or(i64::MAX); + Self { + queue_log_interval: Duration::from_secs(cfg.log.queue_interval_secs.max(1)), + progress_log_interval: Duration::from_secs(cfg.log.progress_interval_secs.max(1)), + max_resolve_retry_backoff_secs: cfg.resolve.max_retry_backoff_secs.max(1), + resolve_min_retry_base_secs: cfg.resolve.min_retry_base_secs.max(1), + max_attempts_resolve: cfg.resolve.max_attempts.max(1), + max_attempts_probe: cfg.probe.max_attempts.max(1), + resolve_exhaustion_reset_gap_secs: to_i64(cfg.resolve.exhaustion_reset_gap_secs), + probe_cooldown_secs: to_i64(cfg.probe.cooldown_secs), + tmdb_cooldown_secs: to_i64(cfg.tmdb.cooldown_secs), + retry_delay_secs: cfg.retry_delay_secs.max(1), + metadata_retry_load_retry_delay_secs: to_i64(cfg.probe.retry_load_retry_delay_secs), + worker_idle_timeout_secs: cfg.worker_idle_timeout_secs.max(1), + max_queue_size: cfg.max_queue_size.max(1), + no_change_cache_ttl_secs: cfg.no_change_cache_ttl_secs.max(1), + probe_fairness_resolve_burst: cfg.probe_fairness_resolve_burst.max(1), + probe_retry_backoff_step_1_secs: cfg.probe.retry_backoff_step_1_secs.max(1), + probe_retry_backoff_step_2_secs: cfg.probe.retry_backoff_step_2_secs.max(1), + probe_retry_backoff_step_3_secs: cfg.probe.retry_backoff_step_3_secs.max(1), + backoff_jitter_percent: cfg.probe.backoff_jitter_percent.min(95), + } + } +} diff --git a/backend/metadata/src/manager/task_processing.rs b/backend/metadata/src/manager/task_processing.rs new file mode 100644 index 000000000..4f3b1a701 --- /dev/null +++ b/backend/metadata/src/manager/task_processing.rs @@ -0,0 +1,584 @@ +use super::{ + spawn_blocking_limited, DbHandle, InputWorker, RetryDomain, TaskKey, PROBE_TASK_TIMEOUT_SECS, + TASK_ERR_NO_CONNECTION, TASK_ERR_PREEMPTED, TASK_ERR_UPDATE_IN_PROGRESS, +}; +use crate::ctx::MetadataUpdateCtx; +use log::{debug, error, warn}; +use shared::{ + defaults::default_probe_user_priority, + error::TuliproxError, + model::{EventSink, InputType, PlaylistItemType, StreamProperties, XtreamCluster, XtreamPlaylistItem}, +}; +use std::{ + collections::{HashMap, HashSet}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; +use tuliprox_core::{ + model::{BatchResultCollector, ProviderHandle, ProviderIdType, ResolveReason, UpdateTask}, + utils::debug_if_enabled, +}; +use tuliprox_processing::processor::{ + probe_generic_stream_metadata, update_generic_stream_metadata, update_live_stream_metadata, update_properties, + update_series_metadata, update_vod_metadata, GenericProbeMetadataOutcome, GenericProbeOutcome, SeriesProbeSettings, +}; + +#[derive(Debug, Clone, Copy)] +pub(super) struct ProcessTaskOutcome { + pub(super) task_changed: bool, + pub(super) tmdb_pending: bool, + pub(super) probe_pending: bool, +} + +impl InputWorker { + pub(super) fn load_task_snapshot(&self, key: TaskKey) -> Option<(TaskKey, UpdateTask, u64)> { + let entry = self.pending_tasks.get(&key)?; + let generation = entry.generation.load(Ordering::Relaxed); + let task = entry.task.lock().clone(); + Some((key, task, generation)) + } + + pub(super) fn take_pending_probe_task_snapshot(&self) -> Option<(TaskKey, UpdateTask, u64)> { + for entry in self.pending_tasks.iter() { + if self.scheduled_requeues.contains_key(entry.key()) { + continue; + } + + let generation = entry.generation.load(Ordering::Relaxed); + let task = entry.task.lock().clone(); + if Self::retry_domain_for_task(&task) == RetryDomain::Probe { + return Some((entry.key().clone(), task, generation)); + } + } + + None + } + + // Helper for get_item_name with caching + async fn get_item_name_static( + input_name: &str, + ctx: &MetadataUpdateCtx, + task: &UpdateTask, + db_handles: &mut HashMap, + failed_clusters: &mut HashSet, + ) -> Option { + let (id, cluster) = match task { + UpdateTask::ResolveVod { id, .. } => (id, XtreamCluster::Video), + UpdateTask::ResolveSeries { id, .. } => (id, XtreamCluster::Series), + UpdateTask::ProbeLive { id, .. } => (id, XtreamCluster::Live), + UpdateTask::ProbeStream { .. } => return None, + }; + + if let ProviderIdType::Id(vid) = id { + let stream_id = *vid; + if let Some(query) = Self::get_or_open_query(input_name, ctx, cluster, db_handles, failed_clusters).await { + let query = Arc::clone(&query); + let item = match spawn_blocking_limited(move || { + let mut guard = query.lock(); + guard.query_zero_copy(&stream_id).ok().flatten() + }) + .await + { + Ok(item) => item, + Err(err) => { + error!("Failed to query item name for {stream_id}: {err}"); + None + } + }; + + if let Some(item) = item { + return Some(if item.title.is_empty() { item.name.to_string() } else { item.title.to_string() }); + } + } + } + None + } + + #[allow(clippy::too_many_lines)] + pub(super) async fn process_task_static( + input_name: &Arc, + bound_ctx: Option<&MetadataUpdateCtx>, + task: &UpdateTask, + collector: &mut BatchResultCollector, + db_handles: &mut HashMap, + failed_clusters: &mut HashSet, + ) -> Result { + let ctx = + bound_ctx.ok_or_else(|| shared::error::TuliproxError::Config("metadata context not bound".to_string()))?; + + let Some(input_base) = ctx.app_config.get_input_by_name(input_name) else { + return Err(shared::error::TuliproxError::Config(format!("Input {input_name} not found"))); + }; + + if !input_base.enabled { + return Err(shared::error::TuliproxError::Config(format!("Input {input_name} is disabled"))); + } + + // Background metadata/probe tasks are low-priority. + // Never run them while a foreground playlist update is active. + if let Some(guard) = ctx.update_guard.try_playlist() { + drop(guard); + } else { + return Err(shared::error::TuliproxError::Config(TASK_ERR_UPDATE_IN_PROGRESS.to_string())); + } + + let needs_probe_connection = Self::task_needs_provider_connection(task, input_base.input_type); + + let probe_priority = ctx + .app_config + .config + .load() + .metadata_update + .as_ref() + .map_or(default_probe_user_priority(), |cfg| cfg.probe.user_priority); + + // Reserve provider capacity only for actual probe work (ffprobe paths). + let provider_handle = if needs_probe_connection { + let Some(handle) = ctx.active_provider.acquire_connection_for_probe(input_name, probe_priority) else { + debug_if_enabled!("No provider connection available for background task {}, skipping...", task); + return Err(shared::error::TuliproxError::Config(TASK_ERR_NO_CONNECTION.to_string())); + }; + Some(handle) + } else { + None + }; + + let item_title = Self::get_item_name_static(input_name, ctx, task, db_handles, failed_clusters).await; + + let config_to_use = provider_handle.as_ref().and_then(|handle| handle.allocation.get_provider_config()); + let name_display = item_title.as_deref().map_or(String::new(), |n| format!(" \"{n}\"")); + + debug!("Processing task for {input_name}: {task}{name_display}"); + + let pre_vod_updates = collector.vod.len(); + let pre_series_updates = collector.series.len(); + let pre_live_updates = collector.live.len(); + + // Determine input to use (may be alias) + let input_to_use = config_to_use + .filter(|alloc| alloc.name != input_base.name) + .and_then(|alloc| input_base.aliases.as_ref()?.iter().find(|a| a.enabled && a.name == alloc.name)) + .map(|alias_def| { + let mut temp_input = (*input_base).clone(); + temp_input.url.clone_from(&alias_def.url); + temp_input.username.clone_from(&alias_def.username); + temp_input.password.clone_from(&alias_def.password); + Arc::new(temp_input) + }) + .unwrap_or(input_base); + + let client = if tuliprox_core::model::should_use_manual_redirects(&ctx.app_config) { + ctx.http_clients.no_redirect.load() + } else { + ctx.http_clients.default.load() + }; + + // Execute task; probe tasks get a reserved provider handle, resolve tasks don't. + // The hard timeout is applied only to tasks that perform network probing — Probe* + // variants and Resolve tasks whose reason includes ResolveReason::Probe — because + // those are the only ones that can stall on an unresponsive provider. Pure metadata + // resolve tasks (Info / TMDB / Date only) run without a timeout so they are not + // subject to the probe hard limit. + let exec_fut = async { + if let Some(handle) = provider_handle.as_ref() { + if let Some(token) = &handle.cancel_token { + tokio::select! { + biased; + + () = token.cancelled() => { + debug_if_enabled!("Metadata update task preempted by user request for input {}", input_name); + Err(shared::error::TuliproxError::Config(TASK_ERR_PREEMPTED.to_string())) + } + + res = Self::execute_task_inner_static(ctx, &client, &input_to_use, task, item_title.as_deref(), Some(handle), probe_priority, collector, db_handles, failed_clusters) => { + res + } + } + } else { + Self::execute_task_inner_static( + ctx, + &client, + &input_to_use, + task, + item_title.as_deref(), + Some(handle), + probe_priority, + collector, + db_handles, + failed_clusters, + ) + .await + } + } else { + Self::execute_task_inner_static( + ctx, + &client, + &input_to_use, + task, + item_title.as_deref(), + None, + probe_priority, + collector, + db_handles, + failed_clusters, + ) + .await + } + }; + let needs_probe_timeout = Self::is_probe_task(task) + || (Self::is_resolve_task(task) && Self::task_reason(task).contains(ResolveReason::Probe)); + let res = if needs_probe_timeout { + match tokio::time::timeout(Duration::from_secs(PROBE_TASK_TIMEOUT_SECS), exec_fut).await { + Ok(result) => result, + Err(_elapsed) => { + error!( + "Metadata probe task timed out after {PROBE_TASK_TIMEOUT_SECS}s for input {input_name}: {task}; \ + releasing provider handle and skipping task" + ); + Err(shared::error::TuliproxError::Config(format!( + "Task timed out after {PROBE_TASK_TIMEOUT_SECS}s for input {input_name}: {task}" + ))) + } + } + } else { + exec_fut.await + }; + + if provider_handle.is_some() { + ctx.connection_manager.release_provider_handle(provider_handle); + } + match res { + Ok((tmdb_and_date_present, probe_pending)) => { + let task_changed = match task { + UpdateTask::ResolveVod { .. } => collector.vod.len() > pre_vod_updates, + UpdateTask::ResolveSeries { .. } => collector.series.len() > pre_series_updates, + UpdateTask::ProbeLive { .. } => collector.live.len() > pre_live_updates, + UpdateTask::ProbeStream { .. } => true, + }; + let tmdb_pending = match task { + UpdateTask::ResolveVod { reason, .. } | UpdateTask::ResolveSeries { reason, .. } => { + if reason.contains(ResolveReason::Tmdb) || reason.contains(ResolveReason::Date) { + !tmdb_and_date_present + } else { + false + } + } + UpdateTask::ProbeLive { .. } | UpdateTask::ProbeStream { .. } => false, + }; + Ok(ProcessTaskOutcome { task_changed, tmdb_pending, probe_pending }) + } + Err(e) => Err(e), + } + } + + pub(super) fn task_needs_provider_connection(task: &UpdateTask, input_type: InputType) -> bool { + match task { + UpdateTask::ProbeLive { .. } => true, + // Local library and media-server probing must not depend on IPTV provider capacity. + UpdateTask::ProbeStream { .. } => { + !(matches!(input_type, InputType::Library) || input_type.is_media_server()) + } + // Resolve tasks handle their own probe connection acquisition internally. + // This avoids holding a provider connection for the entire duration of + // info fetch + TMDB resolve + probe, reducing "provider exhausted" errors. + UpdateTask::ResolveVod { .. } | UpdateTask::ResolveSeries { .. } => false, + } + } + + fn batchable_generic_probe_xtream_cluster(item_type: PlaylistItemType) -> Option { + if item_type.is_live() { + Some(XtreamCluster::Live) + } else if item_type.is_video() { + Some(XtreamCluster::Video) + } else { + None + } + } + + pub(super) fn apply_pending_generic_probe_base( + collector: &BatchResultCollector, + cluster: XtreamCluster, + provider_id: u32, + item: &mut XtreamPlaylistItem, + ) { + match cluster { + XtreamCluster::Video => { + if let Some((_, props)) = collector + .vod + .iter() + .rev() + .find(|(id, _)| matches!(id, ProviderIdType::Id(id) if *id == provider_id)) + { + item.additional_properties = Some(StreamProperties::Video(Box::new(props.clone()))); + } + } + XtreamCluster::Live => { + if let Some((_, props)) = collector + .live + .iter() + .rev() + .find(|(id, _)| matches!(id, ProviderIdType::Id(id) if *id == provider_id)) + { + item.additional_properties = Some(StreamProperties::Live(Box::new(props.clone()))); + } + } + XtreamCluster::Series => {} + } + } + + #[allow(clippy::too_many_arguments, clippy::too_many_lines)] + async fn execute_task_inner_static( + ctx: &MetadataUpdateCtx, + client: &reqwest::Client, + input: &Arc, + task: &UpdateTask, + item_title: Option<&str>, + active_handle: Option<&ProviderHandle>, + probe_priority: i8, + collector: &mut BatchResultCollector, + db_handles: &mut HashMap, + failed_clusters: &mut HashSet, + ) -> Result<(bool, bool), TuliproxError> { + // The returned tuple is `(tmdb_and_date_present, probe_pending)`. + // `tmdb_and_date_present` avoids false-positive TMDB "no match" cooldowns. + // `probe_pending` prevents skipped/aborted probes from being cached as no-op. + match task { + UpdateTask::ResolveVod { id, reason, .. } => { + let fetch_info = reason.contains(ResolveReason::Info); + let resolve_tmdb = + fetch_info || reason.contains(ResolveReason::Tmdb) || reason.contains(ResolveReason::Date); + let will_probe = reason.contains(ResolveReason::Probe); + + // If we are going to probe, release the cached handle to avoid holding a READ lock + // for along time (blocks writers) and also to avoid potential deadlocks if + // the probe function itself tries to acquire a WRITE lock later. + if will_probe { + db_handles.remove(&XtreamCluster::Video); + } + + let query_opt = if will_probe { + None + } else { + Self::get_or_open_query(&input.name, ctx, XtreamCluster::Video, db_handles, failed_clusters).await + }; + + let tmdb_and_date_present = AtomicBool::new(false); + let probe_pending = AtomicBool::new(false); + match update_vod_metadata( + &ctx.app_config, + client, + input, + id.clone(), + active_handle, + &ctx.active_provider, + item_title, + false, // Batch collect + fetch_info, + resolve_tmdb, + will_probe, + query_opt, + Some(&tmdb_and_date_present), + Some(&probe_pending), + ) + .await + { + Ok(Some(props)) => { + collector.add_vod(id.clone(), props); + Ok((tmdb_and_date_present.load(Ordering::Relaxed), probe_pending.load(Ordering::Relaxed))) + } + Ok(None) => { + Ok((tmdb_and_date_present.load(Ordering::Relaxed), probe_pending.load(Ordering::Relaxed))) + } + Err(e) => Err(e), + } + } + UpdateTask::ResolveSeries { id, reason, .. } => { + let fetch_info = reason.contains(ResolveReason::Info); + let resolve_tmdb = reason.contains(ResolveReason::Tmdb) || reason.contains(ResolveReason::Date); + let will_probe = reason.contains(ResolveReason::Probe); + let series_probe_settings = { + let config = ctx.app_config.config.load(); + SeriesProbeSettings::from_metadata_update(config.metadata_update.as_ref()) + }; + + if will_probe { + db_handles.remove(&XtreamCluster::Series); + } + + // Get handle for Series + let query_opt = if will_probe { + None + } else { + Self::get_or_open_query(&input.name, ctx, XtreamCluster::Series, db_handles, failed_clusters).await + }; + + let tmdb_and_date_present = AtomicBool::new(false); + let probe_pending = AtomicBool::new(false); + match update_series_metadata( + &ctx.app_config, + client, + input, + id.clone(), + &ctx.active_provider, + active_handle, + item_title, + false, // Batch collect + fetch_info, + resolve_tmdb, + will_probe, + series_probe_settings, + query_opt, + Some(&tmdb_and_date_present), + Some(&probe_pending), + ) + .await + { + Ok(Some(props)) => { + collector.add_series(id.clone(), props); + Ok((tmdb_and_date_present.load(Ordering::Relaxed), probe_pending.load(Ordering::Relaxed))) + } + Ok(None) => { + Ok((tmdb_and_date_present.load(Ordering::Relaxed), probe_pending.load(Ordering::Relaxed))) + } + Err(e) => Err(e), + } + } + UpdateTask::ProbeLive { id, .. } => { + // ProbeLive always probes, so we must never use a cached handle here. + db_handles.remove(&XtreamCluster::Live); + + match update_live_stream_metadata( + &ctx.app_config, + client, + input, + id.clone(), + false, + None, + active_handle, + &ctx.active_provider, + ) + .await + { + Ok(Some(props)) => { + collector.add_live(id.clone(), props); + Ok((false, false)) + } + Ok(None) => Ok((false, false)), + Err(e) => Err(e), + } + } + UpdateTask::ProbeStream { unique_id, url, item_type, .. } => { + let task_key = TaskKey::from_task(task); + let probe_identifier = if unique_id.trim().is_empty() { url.as_str() } else { unique_id.as_str() }; + + if input.input_type.is_xtream() { + if let Some(cluster) = Self::batchable_generic_probe_xtream_cluster(*item_type) { + let Ok(provider_id) = unique_id.parse::() else { + warn!("Skipping xtream generic probe update with non-numeric id: {unique_id}"); + return Ok((false, false)); + }; + + let outcome = probe_generic_stream_metadata( + &ctx.app_config, + client, + input.as_ref(), + unique_id, + url, + *item_type, + &ctx.active_provider, + active_handle, + probe_priority, + &ctx.events, + ) + .await?; + + let metadata = match outcome { + GenericProbeMetadataOutcome::Metadata(metadata) => metadata, + GenericProbeMetadataOutcome::Noop => return Ok((false, false)), + GenericProbeMetadataOutcome::ProbeFailed => { + return Err(shared::error::TuliproxError::Config(format!( + "Probe stream task failed for key {task_key:?} ({probe_identifier})" + ))); + } + }; + + let Some(query) = + Self::get_or_open_query(&input.name, ctx, cluster, db_handles, failed_clusters).await + else { + warn!("Item not found in Xtream DB for generic probe: {unique_id}"); + return Ok((false, false)); + }; + + let query = Arc::clone(&query); + let mut item = match spawn_blocking_limited(move || { + let mut guard = query.lock(); + guard.query_zero_copy(&provider_id).ok().flatten() + }) + .await + { + Ok(Some(item)) => item, + Ok(None) => { + warn!("Item not found in Xtream DB: {unique_id}"); + return Ok((false, false)); + } + Err(err) => { + return Err(shared::error::TuliproxError::Config(format!( + "Failed to query generic probe item {unique_id}: {err}" + ))); + } + }; + + Self::apply_pending_generic_probe_base(collector, cluster, provider_id, &mut item); + + update_properties( + &mut item.additional_properties, + *item_type, + &item.name, + item.virtual_id.get(), + metadata.raw_video, + metadata.raw_audio, + metadata.stats, + ); + + match item.additional_properties { + Some(StreamProperties::Live(props)) => collector.add_live(provider_id.into(), *props), + Some(StreamProperties::Video(props)) => collector.add_vod(provider_id.into(), *props), + _ => {} + } + + return Ok((false, false)); + } + } + + // Non-Xtream generic probe still writes directly and can target + // M3U or library storage, so release cached Xtream read handles. + if !db_handles.is_empty() { + db_handles.clear(); + } + let outcome = update_generic_stream_metadata( + &ctx.app_config, + client, + input.as_ref(), + unique_id, + url, + *item_type, + &ctx.active_provider, + active_handle, + probe_priority, + &ctx.events, + ) + .await?; + + match outcome { + GenericProbeOutcome::Updated | GenericProbeOutcome::Noop => Ok((false, false)), + GenericProbeOutcome::ProbeFailed => Err(shared::error::TuliproxError::Config(format!( + "Probe stream task failed for key {task_key:?} ({probe_identifier})" + ))), + } + } + } + } +} diff --git a/backend/metadata/src/manager/task_queue.rs b/backend/metadata/src/manager/task_queue.rs new file mode 100644 index 000000000..ce9332ba9 --- /dev/null +++ b/backend/metadata/src/manager/task_queue.rs @@ -0,0 +1,722 @@ +use super::{ + load_metadata_retry_states_from_disk, spawn_blocking_limited, InputWorker, InputWorkerContext, + MetadataUpdateManager, MetadataUpdateRuntimeSettings, RetryDomain, TaskKey, METADATA_RETRY_STATE_FILE, + RETRY_STATE_MIN_TTL_SECS, +}; +use crate::ctx::BoundMetadataUpdateCtx; +use arc_swap::ArcSwap; +use dashmap::{mapref::entry::Entry, DashMap}; +use log::{debug, error, warn}; +use parking_lot::Mutex as ParkingMutex; +use std::{ + cmp::min, + collections::{HashMap, HashSet}, + sync::{ + atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::sync::{mpsc, RwLock}; +use tokio_util::sync::CancellationToken; +use tuliprox_core::{ + model::{BatchResultCollector, ProviderIdType, ResolveReason, ResolveReasonSet, UpdateTask}, + utils::debug_if_enabled, +}; +use tuliprox_repository::get_input_storage_path; + +impl TaskKey { + pub fn from_task(task: &UpdateTask) -> Self { + match task { + UpdateTask::ResolveVod { id, .. } => match id { + ProviderIdType::Id(val) => TaskKey::Vod(*val), + ProviderIdType::Text(val) => TaskKey::VodStr(val.clone()), + }, + UpdateTask::ResolveSeries { id, .. } => match id { + ProviderIdType::Id(val) => TaskKey::Series(*val), + ProviderIdType::Text(val) => TaskKey::SeriesStr(val.clone()), + }, + UpdateTask::ProbeLive { id, .. } => match id { + ProviderIdType::Id(val) => TaskKey::Live(*val), + ProviderIdType::Text(val) => TaskKey::LiveStr(val.clone()), + }, + UpdateTask::ProbeStream { probe_scope, unique_id, url, .. } => { + if unique_id.trim().is_empty() { + TaskKey::Stream { scope: probe_scope.clone(), id: Arc::from(url.as_str()) } + } else { + TaskKey::Stream { scope: probe_scope.clone(), id: Arc::from(unique_id.as_str()) } + } + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub(super) struct ScopedTaskKey { + pub(super) input_name: Arc, + pub(super) task_key: TaskKey, +} + +impl ScopedTaskKey { + pub(super) fn new(input_name: Arc, task_key: TaskKey) -> Self { Self { input_name, task_key } } +} + +pub(super) struct PendingTask { + pub(super) task: ParkingMutex, + pub(super) generation: AtomicU64, +} + +impl PendingTask { + pub(super) fn new(task: UpdateTask) -> Self { + Self { task: ParkingMutex::new(task), generation: AtomicU64::new(0) } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum SubmitTaskResult { + QueuedOrMerged, + QueueFull, + ChannelClosed, +} + +impl MetadataUpdateManager { + pub fn new(cancel_token: CancellationToken) -> Self { + Self { + workers: DashMap::new(), + worker_lifecycle_lock: ParkingMutex::new(()), + is_shutdown_flag: AtomicBool::new(false), + ctx: tokio::sync::Mutex::new(None), + update_pause_gate: Arc::new(RwLock::new(())), + cancel_token: ArcSwap::from_pointee(cancel_token), + next_worker_id: AtomicU64::new(1), + resolve_enqueue_suppressions: Arc::new(DashMap::new()), + tmdb_source_markers: Arc::new(DashMap::new()), + enqueue_state_loaded_inputs: Arc::new(DashMap::new()), + enqueue_state_load_retry_at_ts: Arc::new(DashMap::new()), + last_resolve_enqueue_suppression_prune_at_ts: AtomicI64::new(0), + } + } + + /// Bind the handles the worker reads. Called once, after the server's root + /// state is built - the worker itself is constructed before it. + pub async fn set_ctx(&self, ctx: BoundMetadataUpdateCtx) { + let mut guard = self.ctx.lock().await; + *guard = Some(ctx); + } + + fn scoped_task_key(input_name: &str, task: &UpdateTask) -> ScopedTaskKey { + ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(task)) + } + + fn has_pending_task(&self, input_name: &str, task: &UpdateTask) -> bool { + self.workers + .get(input_name) + .is_some_and(|ctx| !ctx.sender.is_closed() && ctx.pending_tasks.contains_key(&TaskKey::from_task(task))) + } + + pub(super) fn should_skip_enqueue_cached(&self, input_name: &str, task: &UpdateTask) -> bool { + let now_ts = chrono::Utc::now().timestamp(); + self.prune_resolve_enqueue_suppressions(now_ts); + + if self.is_redundant_with_pending_task(input_name, task) { + return true; + } + + if InputWorker::retry_domain_for_task(task) != RetryDomain::Resolve { + return false; + } + + let scoped_key = Self::scoped_task_key(input_name, task); + if let Some(entry) = self.resolve_enqueue_suppressions.get(&scoped_key) { + let suppressed_until_ts = *entry; + drop(entry); + if now_ts < suppressed_until_ts { + if self.has_pending_task(input_name, task) { + return false; + } + debug!( + "[Task] Skipping enqueue (resolve suppression) for input {}: {} (until_ts={}, remaining={}s)", + input_name, + task, + suppressed_until_ts, + suppressed_until_ts.saturating_sub(now_ts) + ); + return true; + } + self.resolve_enqueue_suppressions.remove(&scoped_key); + } + + false + } + + async fn ensure_enqueue_state_loaded_for_input(&self, input_name: &Arc) { + if self.enqueue_state_loaded_inputs.contains_key(input_name) { + return; + } + let now_ts = chrono::Utc::now().timestamp(); + if self.enqueue_state_load_retry_at_ts.get(input_name).is_some_and(|retry_at_ts| now_ts < *retry_at_ts) { + return; + } + + let bound_ctx = { + let guard = self.ctx.lock().await; + guard.clone() + }; + let runtime_settings = MetadataUpdateRuntimeSettings::from_ctx(bound_ctx.as_ref()); + let Some(ctx) = bound_ctx else { + return; + }; + let retry_at_ts = now_ts.saturating_add(runtime_settings.metadata_retry_load_retry_delay_secs); + + let storage_dir = ctx.app_config.config.load().storage_dir.clone(); + let Ok(storage_path) = get_input_storage_path(input_name, &storage_dir).await else { + self.enqueue_state_load_retry_at_ts.insert(input_name.clone(), retry_at_ts); + return; + }; + let retry_path = storage_path.join(METADATA_RETRY_STATE_FILE); + let input_name_cloned = input_name.clone(); + let loaded = spawn_blocking_limited(move || load_metadata_retry_states_from_disk(&retry_path)) + .await + .ok() + .and_then(Result::ok); + let Some(states) = loaded else { + self.enqueue_state_load_retry_at_ts.insert(input_name.clone(), retry_at_ts); + return; + }; + + for (task_key, state) in states { + if let Some(resolve_state) = state.resolve.as_ref() { + let suppressed_until_ts = resolve_state.cooldown_until_ts.unwrap_or(resolve_state.next_allowed_at_ts); + if suppressed_until_ts > now_ts { + let scoped_key = ScopedTaskKey::new(input_name_cloned.clone(), task_key.clone()); + self.resolve_enqueue_suppressions.insert(scoped_key, suppressed_until_ts); + } + } + if let Some(source_last_modified) = + state.tmdb.as_ref().and_then(|tmdb_state| tmdb_state.source_last_modified) + { + let scoped_key = ScopedTaskKey::new(input_name_cloned.clone(), task_key.clone()); + self.tmdb_source_markers.insert(scoped_key, source_last_modified); + } + } + + self.enqueue_state_load_retry_at_ts.remove(input_name); + self.enqueue_state_loaded_inputs.insert(input_name.clone(), ()); + } + + pub(super) fn strip_tmdb_reasons_for_enqueue(&self, input_name: &str, task: UpdateTask) -> Option { + let scoped_key = Self::scoped_task_key(input_name, &task); + let current_last_modified = InputWorker::task_source_last_modified(&task); + let previous_last_modified = self.tmdb_source_markers.get(&scoped_key).map(|entry| *entry); + if (current_last_modified.is_some() || previous_last_modified.is_some()) + && current_last_modified == previous_last_modified + && InputWorker::task_has_tmdb_reason(&task) + { + InputWorker::strip_tmdb_reasons(&task) + } else { + Some(task) + } + } + + async fn prepare_task_for_enqueue(&self, input_name: Arc, task: UpdateTask) -> Option { + self.ensure_enqueue_state_loaded_for_input(&input_name).await; + let prepared_task = self.strip_tmdb_reasons_for_enqueue(input_name.as_ref(), task)?; + if self.should_skip_enqueue_cached(input_name.as_ref(), &prepared_task) { + None + } else { + Some(prepared_task) + } + } + + /// Spawn a background task to queue the update. + /// This is a fire-and-forget method that returns immediately. + pub fn queue_task_background(self: &Arc, input_name: Arc, task: UpdateTask) { + let this = self.clone(); + tokio::spawn(async move { + this.queue_task(input_name, task).await; + }); + } + + /// Queue a task for background processing. + /// + /// If a worker exists for the input, the task is sent to it. + /// If no worker exists, a new one is spawned. + /// + /// # Arguments + /// * `input_name` - The input this task belongs to + /// * `task` - The task to process + #[allow(clippy::too_many_lines)] + pub async fn queue_task(&self, input_name: Arc, task: UpdateTask) { + debug!("[Task] Queuing task for input {input_name}: {task}"); + let Some(task_to_queue) = self.prepare_task_for_enqueue(input_name.clone(), task).await else { + return; + }; + + // Read app state once and reuse for worker creation when needed. + let bound_ctx = { + let guard = self.ctx.lock().await; + guard.clone() + }; + let runtime_settings = MetadataUpdateRuntimeSettings::from_ctx(bound_ctx.as_ref()); + let max_queue_size = runtime_settings.max_queue_size; + + let mut channel_closed_attempt: u32 = 0; + loop { + // Atomically ensure there is exactly one worker context per input. + let mut worker_to_spawn: Option<(u64, InputWorker)> = None; + let (cancel_token, ctx) = { + let _guard = self.worker_lifecycle_lock.lock(); + let cancel_token = self.cancel_token.load_full(); + if cancel_token.is_cancelled() { + debug_if_enabled!( + "Aborting metadata enqueue loop for input {input_name} because cancellation was requested: {task_to_queue}" + ); + return; + } + + let ctx = match self.workers.entry(input_name.clone()) { + Entry::Occupied(entry) => entry.get().clone(), + Entry::Vacant(entry) => { + let (tx, rx) = mpsc::channel::(max_queue_size); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(0)); + let worker_id = self.next_worker_id.fetch_add(1, Ordering::Relaxed); + + let ctx = InputWorkerContext { + worker_id, + sender: tx.clone(), + pending_tasks: pending_tasks.clone(), + pending_task_count: pending_task_count.clone(), + }; + entry.insert(ctx.clone()); + + worker_to_spawn = Some(( + worker_id, + InputWorker { + input_name: input_name.clone(), + sender: tx, + receiver: rx, + pending_tasks, + pending_task_count, + bound_ctx: bound_ctx.clone(), + update_pause_gate: Arc::clone(&self.update_pause_gate), + cancel_token: (*cancel_token).clone(), + batch_buffer: BatchResultCollector::new(), + db_handles: HashMap::new(), + failed_clusters: HashSet::new(), + retry_states: HashMap::new(), + resolve_exhausted: HashMap::new(), + last_cycle_completed_at_ts: None, + metadata_retry_state_path: None, + metadata_retry_loaded: false, + metadata_retry_load_retry_at_ts: None, + last_retry_state_prune_at_ts: None, + scheduled_requeues: Arc::new(DashMap::new()), + recently_completed_no_change: HashMap::new(), + resolve_enqueue_suppressions: Arc::clone(&self.resolve_enqueue_suppressions), + tmdb_source_markers: Arc::clone(&self.tmdb_source_markers), + dirty_retry_state_keys: HashSet::new(), + }, + )); + + ctx + } + }; + + (cancel_token, ctx) + }; + + if let Some((worker_id, worker)) = worker_to_spawn { + let workers_ref = self.workers.clone(); + let input_name_for_cleanup = input_name.clone(); + tokio::spawn(async move { + worker.run().await; + + // Cleanup only if this exact worker context is still active. + if let Entry::Occupied(entry) = workers_ref.entry(input_name_for_cleanup.clone()) { + if entry.get().worker_id == worker_id { + entry.remove(); + } + } + }); + } + + let sender_still_current = { + let _guard = self.worker_lifecycle_lock.lock(); + let token_matches = Arc::ptr_eq(&self.cancel_token.load_full(), &cancel_token); + let worker_matches = + self.workers.get(&input_name).is_some_and(|current| current.worker_id == ctx.worker_id); + token_matches && worker_matches + }; + if !sender_still_current { + continue; + } + + match Self::submit_task( + ctx.sender.clone(), + ctx.pending_tasks.clone(), + ctx.pending_task_count.clone(), + &input_name, + max_queue_size, + task_to_queue.clone(), + ) + .await + { + SubmitTaskResult::QueuedOrMerged => return, + SubmitTaskResult::QueueFull => { + error!( + "Metadata queue full for input {input_name} (max_queue_size={max_queue_size}), \ + dropping task: {task_to_queue}; consider increasing max_queue_size or reducing \ + probe frequency" + ); + return; + } + SubmitTaskResult::ChannelClosed => { + channel_closed_attempt = channel_closed_attempt.saturating_add(1); + if channel_closed_attempt.is_multiple_of(10) { + warn!( + "Metadata enqueue channel still closed for input {input_name} after {channel_closed_attempt} retries; continuing recovery" + ); + } + debug_if_enabled!( + "Detected closed metadata worker channel for input {}, recreating worker context (attempt {})", + input_name, + channel_closed_attempt + ); + Self::remove_worker_context_if_id(&self.workers, &input_name, ctx.worker_id); + let exp = channel_closed_attempt.saturating_sub(1).min(6); + let factor = 1_u64.checked_shl(exp).unwrap_or(u64::MAX); + let backoff_ms = 25_u64.saturating_mul(factor).min(2_000); + let backoff = Duration::from_millis(backoff_ms); + tokio::select! { + () = cancel_token.cancelled() => { + warn!( + "Aborting metadata task enqueue for input {input_name} because cancellation was requested: {task_to_queue}" + ); + return; + } + () = tokio::time::sleep(backoff) => {} + } + } + } + } + } + + pub(super) async fn submit_task( + sender: mpsc::Sender, + pending_tasks: Arc>, + pending_task_count: Arc, + input_name: &str, + max_queue_size: usize, + task: UpdateTask, + ) -> SubmitTaskResult { + let key = TaskKey::from_task(&task); + let task_to_submit = task; + + if let Some(entry) = pending_tasks.get(&key) { + if sender.is_closed() { + drop(entry); + if pending_tasks.remove(&key).is_some() { + Self::decrement_pending_task_count(&pending_task_count); + } + } else { + let mut existing = entry.task.lock(); + let before = format!("{existing}"); + if Self::merge_task_payload(&mut existing, task_to_submit) { + entry.generation.fetch_add(1, Ordering::Relaxed); + debug!("[Task] Merged task for input {input_name}: before={before}, after={existing}"); + } else { + debug!("[Task] Task already pending for input {input_name} (no merge needed): {existing}"); + } + return SubmitTaskResult::QueuedOrMerged; + } + } + + // Lock-free admission with CAS: reserve one queue slot only if capacity allows. + if pending_task_count + .try_update(Ordering::AcqRel, Ordering::Relaxed, |current| { + if current < max_queue_size { + Some(current + 1) + } else { + None + } + }) + .is_err() + { + return SubmitTaskResult::QueueFull; + } + + loop { + match pending_tasks.entry(key.clone()) { + Entry::Occupied(entry) => { + // Another producer inserted this key after our fast-path `get`. + if sender.is_closed() { + entry.remove(); + Self::decrement_pending_task_count(&pending_task_count); + continue; + } + + // Release reserved capacity and merge into the existing task. + Self::decrement_pending_task_count(&pending_task_count); + let mut existing = entry.get().task.lock(); + if Self::merge_task_payload(&mut existing, task_to_submit) { + entry.get().generation.fetch_add(1, Ordering::Relaxed); + } + return SubmitTaskResult::QueuedOrMerged; + } + Entry::Vacant(entry) => { + entry.insert(PendingTask::new(task_to_submit)); + break; + } + } + } + + if sender.send(key.clone()).await.is_err() { + if pending_tasks.remove(&key).is_some() { + Self::decrement_pending_task_count(&pending_task_count); + } + warn!("Failed to send task signal for input {input_name}"); + return SubmitTaskResult::ChannelClosed; + } + SubmitTaskResult::QueuedOrMerged + } + + #[inline] + pub(super) fn decrement_pending_task_count(pending_task_count: &AtomicUsize) { + // Guard against accidental underflow in edge/error paths. + let _ = pending_task_count.try_update(Ordering::AcqRel, Ordering::Relaxed, |current| current.checked_sub(1)); + } + + pub(super) fn merge_task_payload(existing: &mut UpdateTask, task: UpdateTask) -> bool { + let mut changed = false; + // Merge logic + match (existing, task) { + ( + UpdateTask::ResolveVod { reason: r1, delay: d1, source_last_modified: lm1, .. }, + UpdateTask::ResolveVod { reason: r2, delay: d2, source_last_modified: lm2, .. }, + ) + | ( + UpdateTask::ResolveSeries { reason: r1, delay: d1, source_last_modified: lm1, .. }, + UpdateTask::ResolveSeries { reason: r2, delay: d2, source_last_modified: lm2, .. }, + ) => { + let previous_reason = *r1; + let previous_delay = *d1; + let previous_last_modified = *lm1; + *r1 |= r2; + *d1 = min(*d1, d2); + *lm1 = match (*lm1, lm2) { + (Some(left), Some(right)) => Some(left.max(right)), + _ => None, + }; + changed = *r1 != previous_reason || *d1 != previous_delay || *lm1 != previous_last_modified; + } + ( + UpdateTask::ProbeStream { reason: r1, delay: d1, url: url1, item_type: item_type1, .. }, + UpdateTask::ProbeStream { reason: r2, delay: d2, url: url2, item_type: item_type2, .. }, + ) => { + let previous_reason = *r1; + let previous_delay = *d1; + let previous_url = url1.clone(); + let previous_item_type = *item_type1; + *r1 |= r2; + *d1 = min(*d1, d2); + // Keep the existing payload by default; only fill it from the incoming + // task when the destination payload is empty. + if url1.is_empty() && !url2.is_empty() { + *url1 = url2; + *item_type1 = item_type2; + } + changed = *r1 != previous_reason + || *d1 != previous_delay + || *url1 != previous_url + || *item_type1 != previous_item_type; + } + ( + UpdateTask::ProbeLive { reason: r1, delay: d1, interval: i1, .. }, + UpdateTask::ProbeLive { reason: r2, delay: d2, interval: i2, .. }, + ) => { + let previous_reason = *r1; + let previous_delay = *d1; + let previous_interval = *i1; + *r1 |= r2; + *d1 = min(*d1, d2); + *i1 = min(*i1, i2); + changed = *r1 != previous_reason || *d1 != previous_delay || *i1 != previous_interval; + } + _ => {} // Mismatched types, should not happen due to TaskKey + } + + changed + } + + /// Queue a task using the legacy API (for backward compatibility). + /// Uses default delay of 50ms. + pub async fn queue_task_legacy(&self, input_name: Arc, task: UpdateTask) { + self.queue_task(input_name, task).await; + } + + /// Returns `true` when an equivalent task is already pending for this input and + /// submitting `task` would not change the queued payload after merge semantics. + pub fn is_redundant_with_pending_task(&self, input_name: &str, task: &UpdateTask) -> bool { + let Some(ctx) = self.workers.get(input_name) else { + return false; + }; + + // If the worker channel is already closed, let normal enqueue recovery run. + if ctx.sender.is_closed() { + return false; + } + + let key = TaskKey::from_task(task); + let Some(entry) = ctx.pending_tasks.get(&key) else { + return false; + }; + + let mut merged = entry.task.lock().clone(); + !Self::merge_task_payload(&mut merged, task.clone()) + } +} + +impl InputWorker { + pub(super) fn runtime_settings(&self) -> MetadataUpdateRuntimeSettings { + MetadataUpdateRuntimeSettings::from_ctx(self.bound_ctx.as_ref()) + } + + pub(super) fn resolve_exhausted_ttl_secs(runtime_settings: &MetadataUpdateRuntimeSettings) -> i64 { + runtime_settings.resolve_exhaustion_reset_gap_secs.saturating_mul(6).max(RETRY_STATE_MIN_TTL_SECS) + } + + pub(super) fn queue_resolve_counts(pending_tasks: &DashMap) -> (usize, usize) { + let mut vod_count = 0_usize; + let mut series_count = 0_usize; + + for entry in pending_tasks { + match entry.key() { + TaskKey::Vod(_) | TaskKey::VodStr(_) => vod_count += 1, + TaskKey::Series(_) | TaskKey::SeriesStr(_) => series_count += 1, + _ => {} + } + } + + (vod_count, series_count) + } + + #[inline] + pub(super) fn is_vod_task_key(key: &TaskKey) -> bool { matches!(key, TaskKey::Vod(_) | TaskKey::VodStr(_)) } + + #[inline] + pub(super) fn is_series_task_key(key: &TaskKey) -> bool { + matches!(key, TaskKey::Series(_) | TaskKey::SeriesStr(_)) + } + + #[inline] + pub(super) fn is_probe_task(task: &UpdateTask) -> bool { + matches!(task, UpdateTask::ProbeLive { .. } | UpdateTask::ProbeStream { .. }) + } + + #[inline] + pub(super) fn is_probe_only_resolve_task(task: &UpdateTask) -> bool { + match task { + UpdateTask::ResolveVod { reason, .. } | UpdateTask::ResolveSeries { reason, .. } => { + reason.contains(ResolveReason::Probe) + && !reason.contains(ResolveReason::Info) + && !reason.contains(ResolveReason::Tmdb) + && !reason.contains(ResolveReason::Date) + } + _ => false, + } + } + + #[inline] + pub(super) fn is_resolve_task(task: &UpdateTask) -> bool { + matches!(task, UpdateTask::ResolveVod { .. } | UpdateTask::ResolveSeries { .. }) + } + + #[inline] + pub(super) fn task_reason(task: &UpdateTask) -> ResolveReasonSet { + match task { + UpdateTask::ResolveVod { reason, .. } + | UpdateTask::ResolveSeries { reason, .. } + | UpdateTask::ProbeLive { reason, .. } + | UpdateTask::ProbeStream { reason, .. } => *reason, + } + } + + #[inline] + pub(super) fn task_has_tmdb_reason(task: &UpdateTask) -> bool { + match task { + UpdateTask::ResolveVod { reason, .. } | UpdateTask::ResolveSeries { reason, .. } => { + reason.contains(ResolveReason::Tmdb) || reason.contains(ResolveReason::Date) + } + _ => false, + } + } + + #[inline] + pub(super) fn task_source_last_modified(task: &UpdateTask) -> Option { + match task { + UpdateTask::ResolveVod { source_last_modified, .. } + | UpdateTask::ResolveSeries { source_last_modified, .. } => *source_last_modified, + _ => None, + } + } + + pub(super) fn strip_tmdb_reasons(task: &UpdateTask) -> Option { + match task { + UpdateTask::ResolveVod { id, reason, delay, source_last_modified } => { + let mut next_reason = *reason; + next_reason.unset(ResolveReason::Tmdb); + next_reason.unset(ResolveReason::Date); + if next_reason.is_empty() { + None + } else { + Some(UpdateTask::ResolveVod { + id: id.clone(), + reason: next_reason, + delay: *delay, + source_last_modified: *source_last_modified, + }) + } + } + UpdateTask::ResolveSeries { id, reason, delay, source_last_modified } => { + let mut next_reason = *reason; + next_reason.unset(ResolveReason::Tmdb); + next_reason.unset(ResolveReason::Date); + if next_reason.is_empty() { + None + } else { + Some(UpdateTask::ResolveSeries { + id: id.clone(), + reason: next_reason, + delay: *delay, + source_last_modified: *source_last_modified, + }) + } + } + _ => Some(task.clone()), + } + } +} + +/// The pipeline's view of this manager. +/// +/// `tuliprox-processing` needs three operations here and nothing else; the trait +/// lives there so the pipeline does not have to name this type. The impl lives +/// here because the type does. +impl tuliprox_processing::metadata_sink::MetadataUpdateSink for MetadataUpdateManager { + async fn acquire_update_pause_guard(&self) -> tokio::sync::OwnedRwLockWriteGuard<()> { + MetadataUpdateManager::acquire_update_pause_guard(self).await + } + + async fn prepare_enqueue_state(&self, input_name: Arc) { + self.ensure_enqueue_state_loaded_for_input(&input_name).await; + } + + fn should_skip_enqueue(&self, input_name: &str, task: &UpdateTask) -> bool { + self.should_skip_enqueue_cached(input_name, task) + } + + fn queue_task_background(self: Arc, input_name: Arc, task: UpdateTask) { + MetadataUpdateManager::queue_task_background(&self, input_name, task); + } +} diff --git a/backend/metadata/src/manager/tests.rs b/backend/metadata/src/manager/tests.rs new file mode 100644 index 000000000..c67184b46 --- /dev/null +++ b/backend/metadata/src/manager/tests.rs @@ -0,0 +1,37 @@ +use super::*; + +/// Run the `collect_*_virtual_updates` text-id resolution test for any +/// cluster (vod / series / live). The macro emits the shared tempdir + +/// mapping + batch + collect + assert scaffolding; the cluster-specific +/// bits (`$fn_name`, `$item_type`, `$input_name`, `$text_id`, +/// `$add_method`, `$props`) are passed by the caller. +macro_rules! assert_collect_resolves_text_id { + ($fn_name:ident, $item_type:expr, $input_name:expr, $text_id:expr, $add_method:ident, $props:expr) => { + let dir = tempdir().expect("tempdir should be created"); + let mapping_path = dir.path().join("target_id_mapping.db"); + let mut mapping = TargetIdMapping::new(&mapping_path, false).expect("mapping should be created"); + let uuid = generate_provider_playlist_uuid($input_name, $text_id, $item_type); + let virtual_id = mapping.get_and_update_virtual_id(&uuid, 0, $item_type, VirtualId::default()); + mapping.persist().expect("mapping should persist"); + + let mut batch = BatchResultCollector::new(); + batch.$add_method(ProviderIdType::from($text_id), $props); + + let mut provider_virtual_ids = HashMap::new(); + let mut uuid_virtual_ids = HashMap::new(); + let updates = + InputWorker::$fn_name(&mapping, $input_name, &batch, &mut provider_virtual_ids, &mut uuid_virtual_ids); + + assert!(updates.contains_key(&virtual_id)); + }; +} + +mod behavior; + +mod lifecycle; + +mod retry; + +mod support; + +use self::support::create_test_worker; diff --git a/backend/metadata/src/manager/tests/behavior.rs b/backend/metadata/src/manager/tests/behavior.rs new file mode 100644 index 000000000..c8daed1de --- /dev/null +++ b/backend/metadata/src/manager/tests/behavior.rs @@ -0,0 +1,826 @@ +use super::{ + create_test_worker, InputWorker, MetadataUpdateManager, MetadataUpdateRuntimeSettings, PendingTask, RetryDomain, + ScopedTaskKey, SubmitTaskResult, TaskKey, TASK_ERR_NO_CONNECTION, TASK_ERR_PREEMPTED, TASK_ERR_UPDATE_IN_PROGRESS, +}; +use dashmap::DashMap; +use shared::{ + model::{ + InputType, LiveStreamProperties, PlaylistItemType, SeriesStreamProperties, StreamProperties, + VideoStreamProperties, VirtualId, XtreamCluster, XtreamPlaylistItem, + }, + utils::generate_provider_playlist_uuid, +}; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; +use tempfile::tempdir; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; +use tuliprox_core::model::{BatchResultCollector, ProviderIdType, ResolveReason, ResolveReasonSet, UpdateTask}; +use tuliprox_repository::TargetIdMapping; + +#[tokio::test] +async fn queue_task_creates_single_worker_per_input_under_concurrency() { + let cancel_token = CancellationToken::new(); + let manager = Arc::new(MetadataUpdateManager::new(cancel_token)); + let input_name: Arc = Arc::from("race_input"); + + let mut joins = Vec::new(); + for id in 0..32u32 { + let manager_cloned = manager.clone(); + let input_cloned = input_name.clone(); + joins.push(tokio::spawn(async move { + manager_cloned + .queue_task( + input_cloned, + UpdateTask::ResolveVod { + id: ProviderIdType::Id(id), + reason: ResolveReasonSet::default(), + delay: 0, + source_last_modified: None, + }, + ) + .await; + })); + } + + for join in joins { + join.await.expect("queue task spawn should complete"); + } + + // Allow spawned worker startup to settle. + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(manager.active_worker_count(), 1); + + manager.shutdown(); +} + +#[test] +fn should_skip_enqueue_respects_resolve_suppression() { + let manager = MetadataUpdateManager::new(CancellationToken::new()); + let input_name = "input_suppressed"; + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReason::Info.into(), + delay: 1, + source_last_modified: None, + }; + let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); + manager.resolve_enqueue_suppressions.insert(scoped_key.clone(), chrono::Utc::now().timestamp().saturating_add(60)); + + assert!(manager.should_skip_enqueue_cached(input_name, &task)); + + manager.resolve_enqueue_suppressions.insert(scoped_key.clone(), chrono::Utc::now().timestamp().saturating_sub(1)); + + assert!(!manager.should_skip_enqueue_cached(input_name, &task)); + assert!(!manager.resolve_enqueue_suppressions.contains_key(&scoped_key)); +} + +#[test] +fn strip_tmdb_reasons_for_enqueue_skips_unchanged_series_tmdb_only_task() { + let manager = MetadataUpdateManager::new(CancellationToken::new()); + let input_name = "input_tmdb_marker"; + let task = UpdateTask::ResolveSeries { + id: ProviderIdType::Id(42), + reason: ResolveReason::Tmdb | ResolveReason::Date, + delay: 1, + source_last_modified: Some(777), + }; + let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); + manager.tmdb_source_markers.insert(scoped_key, 777); + + assert!(manager.strip_tmdb_reasons_for_enqueue(input_name, task).is_none()); +} + +#[test] +fn strip_tmdb_reasons_for_enqueue_keeps_non_tmdb_reasons_for_unchanged_series_source() { + let manager = MetadataUpdateManager::new(CancellationToken::new()); + let input_name = "input_tmdb_probe"; + let task = UpdateTask::ResolveSeries { + id: ProviderIdType::Id(42), + reason: ResolveReason::Tmdb | ResolveReason::Probe, + delay: 1, + source_last_modified: Some(777), + }; + let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); + manager.tmdb_source_markers.insert(scoped_key, 777); + + let prepared = manager.strip_tmdb_reasons_for_enqueue(input_name, task).expect("probe reason should remain"); + match prepared { + UpdateTask::ResolveSeries { reason, source_last_modified, .. } => { + assert_eq!(source_last_modified, Some(777)); + assert!(!reason.contains(ResolveReason::Tmdb)); + assert!(!reason.contains(ResolveReason::Date)); + assert!(reason.contains(ResolveReason::Probe)); + } + other => panic!("unexpected task type after enqueue strip: {other:?}"), + } +} + +#[test] +fn strip_tmdb_reasons_for_enqueue_keeps_unknown_series_timestamp_and_tmdb_reason() { + let manager = MetadataUpdateManager::new(CancellationToken::new()); + let input_name = "input_tmdb_unknown_series"; + let task = UpdateTask::ResolveSeries { + id: ProviderIdType::Id(42), + reason: ResolveReason::Tmdb | ResolveReason::Date, + delay: 1, + source_last_modified: None, + }; + let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); + + assert!(!manager.tmdb_source_markers.contains_key(&scoped_key)); + + let prepared = + manager.strip_tmdb_reasons_for_enqueue(input_name, task).expect("unknown timestamps should not be suppressed"); + match prepared { + UpdateTask::ResolveSeries { reason, source_last_modified, .. } => { + assert_eq!(source_last_modified, None); + assert!(reason.contains(ResolveReason::Tmdb)); + assert!(reason.contains(ResolveReason::Date)); + } + other => panic!("unexpected task type after enqueue strip: {other:?}"), + } +} + +#[test] +fn strip_tmdb_reasons_for_enqueue_skips_unchanged_vod_tmdb_only_task() { + let manager = MetadataUpdateManager::new(CancellationToken::new()); + let input_name = "input_vod_tmdb_marker"; + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReason::Tmdb | ResolveReason::Date, + delay: 1, + source_last_modified: Some(777), + }; + let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); + manager.tmdb_source_markers.insert(scoped_key, 777); + + assert!(manager.strip_tmdb_reasons_for_enqueue(input_name, task).is_none()); +} + +#[test] +fn strip_tmdb_reasons_for_enqueue_keeps_non_tmdb_reasons_for_unchanged_vod_source() { + let manager = MetadataUpdateManager::new(CancellationToken::new()); + let input_name = "input_vod_tmdb_probe"; + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReason::Tmdb | ResolveReason::Probe, + delay: 1, + source_last_modified: Some(777), + }; + let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); + manager.tmdb_source_markers.insert(scoped_key, 777); + + let prepared = manager.strip_tmdb_reasons_for_enqueue(input_name, task).expect("probe reason should remain"); + match prepared { + UpdateTask::ResolveVod { reason, source_last_modified, .. } => { + assert_eq!(source_last_modified, Some(777)); + assert!(!reason.contains(ResolveReason::Tmdb)); + assert!(!reason.contains(ResolveReason::Date)); + assert!(reason.contains(ResolveReason::Probe)); + } + other => panic!("unexpected task type after enqueue strip: {other:?}"), + } +} + +#[test] +fn strip_tmdb_reasons_for_enqueue_keeps_unknown_vod_timestamp_and_tmdb_reason() { + let manager = MetadataUpdateManager::new(CancellationToken::new()); + let input_name = "input_tmdb_unknown_vod"; + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReason::Tmdb | ResolveReason::Date, + delay: 1, + source_last_modified: None, + }; + let scoped_key = ScopedTaskKey::new(Arc::from(input_name), TaskKey::from_task(&task)); + + assert!(!manager.tmdb_source_markers.contains_key(&scoped_key)); + + let prepared = + manager.strip_tmdb_reasons_for_enqueue(input_name, task).expect("unknown timestamps should not be suppressed"); + match prepared { + UpdateTask::ResolveVod { reason, source_last_modified, .. } => { + assert_eq!(source_last_modified, None); + assert!(reason.contains(ResolveReason::Tmdb)); + assert!(reason.contains(ResolveReason::Date)); + } + other => panic!("unexpected task type after enqueue strip: {other:?}"), + } +} + +#[tokio::test] +async fn submit_task_merges_existing_task_and_increments_generation() { + let (tx, mut rx) = mpsc::channel::(8); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(0)); + + let task_initial = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReason::Info.into(), + delay: 10, + source_last_modified: None, + }; + let queue_size = MetadataUpdateRuntimeSettings::default().max_queue_size; + MetadataUpdateManager::submit_task( + tx.clone(), + pending_tasks.clone(), + pending_task_count.clone(), + "input_a", + queue_size, + task_initial, + ) + .await; + + let task_merge = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReason::Probe.into(), + delay: 2, + source_last_modified: None, + }; + MetadataUpdateManager::submit_task( + tx, + pending_tasks.clone(), + pending_task_count.clone(), + "input_a", + queue_size, + task_merge, + ) + .await; + + let first_signal = rx.try_recv().expect("first signal should be queued"); + assert_eq!(first_signal, TaskKey::Vod(42)); + assert!(matches!( + rx.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty | tokio::sync::mpsc::error::TryRecvError::Disconnected) + )); + + let entry = pending_tasks.get(&TaskKey::Vod(42)).expect("pending entry should exist"); + assert_eq!(entry.generation.load(Ordering::Relaxed), 1); + + let merged = entry.task.lock().clone(); + match merged { + UpdateTask::ResolveVod { reason, delay, .. } => { + assert!(reason.contains(ResolveReason::Info)); + assert!(reason.contains(ResolveReason::Probe)); + assert_eq!(delay, 2); + } + other => panic!("unexpected task type after merge: {other:?}"), + } +} + +#[tokio::test] +async fn submit_task_identical_resolve_merge_keeps_generation() { + let (tx, mut rx) = mpsc::channel::(8); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(0)); + let queue_size = MetadataUpdateRuntimeSettings::default().max_queue_size; + + let initial = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReason::Tmdb.into(), + delay: 10, + source_last_modified: None, + }; + MetadataUpdateManager::submit_task( + tx.clone(), + pending_tasks.clone(), + pending_task_count.clone(), + "input_a", + queue_size, + initial, + ) + .await; + + let identical_merge = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReason::Tmdb.into(), + delay: 10, + source_last_modified: None, + }; + MetadataUpdateManager::submit_task( + tx, + pending_tasks.clone(), + pending_task_count.clone(), + "input_a", + queue_size, + identical_merge, + ) + .await; + + let first_signal = rx.try_recv().expect("first signal should be queued"); + assert_eq!(first_signal, TaskKey::Vod(42)); + assert!(matches!( + rx.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty | tokio::sync::mpsc::error::TryRecvError::Disconnected) + )); + + let entry = pending_tasks.get(&TaskKey::Vod(42)).expect("pending entry should exist"); + assert_eq!(entry.generation.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn submit_task_probe_stream_merge_keeps_existing_payload_when_present() { + let (tx, mut rx) = mpsc::channel::(8); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(0)); + let queue_size = MetadataUpdateRuntimeSettings::default().max_queue_size; + + let initial = UpdateTask::ProbeStream { + probe_scope: Arc::from("scope_a"), + unique_id: "uid_1".to_string(), + url: "http://old.example/stream".to_string(), + item_type: PlaylistItemType::Video, + reason: ResolveReason::MissingDetails.into(), + delay: 10, + }; + MetadataUpdateManager::submit_task( + tx.clone(), + pending_tasks.clone(), + pending_task_count.clone(), + "input_a", + queue_size, + initial, + ) + .await; + + let merged_in = UpdateTask::ProbeStream { + probe_scope: Arc::from("scope_a"), + unique_id: "uid_1".to_string(), + url: "http://new.example/stream".to_string(), + item_type: PlaylistItemType::LocalVideo, + reason: ResolveReason::Probe.into(), + delay: 2, + }; + MetadataUpdateManager::submit_task( + tx, + pending_tasks.clone(), + pending_task_count.clone(), + "input_a", + queue_size, + merged_in, + ) + .await; + + let first_signal = rx.try_recv().expect("first signal should be queued"); + assert_eq!(first_signal, TaskKey::Stream { scope: Arc::from("scope_a"), id: Arc::from("uid_1") }); + assert!(matches!( + rx.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty | tokio::sync::mpsc::error::TryRecvError::Disconnected) + )); + + let key = TaskKey::Stream { scope: Arc::from("scope_a"), id: Arc::from("uid_1") }; + let entry = pending_tasks.get(&key).expect("pending entry should exist"); + assert_eq!(entry.generation.load(Ordering::Relaxed), 1); + + let merged = entry.task.lock().clone(); + match merged { + UpdateTask::ProbeStream { reason, delay, url, item_type, .. } => { + assert!(reason.contains(ResolveReason::MissingDetails)); + assert!(reason.contains(ResolveReason::Probe)); + assert_eq!(delay, 2); + assert_eq!(url, "http://old.example/stream"); + assert_eq!(item_type, PlaylistItemType::Video); + } + other => panic!("unexpected task type after merge: {other:?}"), + } +} + +#[tokio::test] +async fn submit_task_with_closed_sender_does_not_report_merged_for_existing_pending_entry() { + let (tx, rx) = mpsc::channel::(8); + drop(rx); + + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(1)); + let key = TaskKey::Vod(42); + let existing_task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReason::Info.into(), + delay: 10, + source_last_modified: None, + }; + pending_tasks.insert(key.clone(), PendingTask::new(existing_task)); + + let incoming_task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(42), + reason: ResolveReason::Probe.into(), + delay: 2, + source_last_modified: None, + }; + + let result = MetadataUpdateManager::submit_task( + tx, + pending_tasks.clone(), + pending_task_count.clone(), + "input_a", + MetadataUpdateRuntimeSettings::default().max_queue_size, + incoming_task, + ) + .await; + + assert_eq!(result, SubmitTaskResult::ChannelClosed); + assert!(!pending_tasks.contains_key(&key)); + assert_eq!(pending_task_count.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn finalize_processed_task_success_requeues_when_generation_changed() { + let (tx, rx) = mpsc::channel::(8); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(1)); + let key = TaskKey::Vod(7); + + pending_tasks.insert( + key.clone(), + PendingTask::new(UpdateTask::ResolveVod { + id: ProviderIdType::Id(7), + reason: ResolveReason::Info.into(), + delay: 0, + source_last_modified: None, + }), + ); + if let Some(entry) = pending_tasks.get(&key) { + entry.generation.store(1, Ordering::Relaxed); + } + + let mut worker = create_test_worker("input_a", tx, rx, pending_tasks.clone(), pending_task_count); + + let requeued = worker.finalize_processed_task_success(&key, 0, "input_a").await; + assert!(requeued); + assert!(pending_tasks.contains_key(&key)); + assert_eq!(worker.receiver.try_recv().expect("requeued signal should be present"), key); +} + +#[tokio::test] +async fn finalize_processed_task_success_removes_when_unchanged() { + let (tx, rx) = mpsc::channel::(8); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(1)); + let key = TaskKey::Vod(9); + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(9), + reason: ResolveReason::Info.into(), + delay: 0, + source_last_modified: None, + }; + + pending_tasks.insert(key.clone(), PendingTask::new(task.clone())); + + let mut worker = create_test_worker("input_b", tx, rx, pending_tasks.clone(), pending_task_count); + let runtime_settings = MetadataUpdateRuntimeSettings::default(); + + worker.scheduled_requeues.insert(key.clone(), chrono::Utc::now().timestamp().saturating_add(30)); + worker.recently_completed_no_change.insert(key.clone(), (Instant::now(), ResolveReason::Info.into())); + assert!(worker.should_skip_recent_no_change_task(&key, &task, &runtime_settings)); + assert!(worker.recently_completed_no_change.contains_key(&key)); + assert!(!worker.scheduled_requeues.contains_key(&key)); + + let requeued = worker.finalize_processed_task_success(&key, 0, "input_b").await; + assert!(!requeued); + assert!(!pending_tasks.contains_key(&key)); + assert!(matches!(worker.receiver.try_recv(), Err(tokio::sync::mpsc::error::TryRecvError::Empty))); +} + +#[test] +fn recent_no_change_skip_requires_exact_reason_match() { + let (tx, rx) = mpsc::channel::(8); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(0)); + let key = TaskKey::Vod(10); + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(10), + reason: ResolveReason::Info.into(), + delay: 0, + source_last_modified: None, + }; + let mut worker = create_test_worker("input_c", tx, rx, pending_tasks, pending_task_count); + let runtime_settings = MetadataUpdateRuntimeSettings::default(); + + worker + .recently_completed_no_change + .insert(key.clone(), (Instant::now(), ResolveReason::Info | ResolveReason::Probe)); + assert!(!worker.should_skip_recent_no_change_task(&key, &task, &runtime_settings)); + assert!(!worker.recently_completed_no_change.contains_key(&key)); + + worker.recently_completed_no_change.insert(key.clone(), (Instant::now(), ResolveReason::Info.into())); + assert!(worker.should_skip_recent_no_change_task(&key, &task, &runtime_settings)); + assert!(worker.recently_completed_no_change.contains_key(&key)); +} + +#[test] +fn task_needs_provider_connection_skips_library_probe_stream() { + let task = UpdateTask::ProbeStream { + probe_scope: Arc::from("input_a"), + unique_id: "u1".to_string(), + url: "file:///movie.mkv".to_string(), + item_type: PlaylistItemType::LocalVideo, + reason: ResolveReason::MissingDetails.into(), + delay: 0, + }; + + assert!(!InputWorker::task_needs_provider_connection(&task, InputType::Library)); +} + +#[test] +fn task_needs_provider_connection_skips_media_server_probe_stream() { + let task = UpdateTask::ProbeStream { + probe_scope: Arc::from("input_remote"), + unique_id: "u1".to_string(), + url: "http://example.com/stream.mkv".to_string(), + item_type: PlaylistItemType::Video, + reason: ResolveReason::MissingDetails.into(), + delay: 0, + }; + + for input_type in [InputType::Emby, InputType::Jellyfin, InputType::Plex] { + assert!(!InputWorker::task_needs_provider_connection(&task, input_type)); + } + assert!(InputWorker::task_needs_provider_connection(&task, InputType::M3u)); +} + +#[test] +fn task_needs_provider_connection_keeps_non_library_probe_stream() { + let task = UpdateTask::ProbeStream { + probe_scope: Arc::from("input_a"), + unique_id: "u1".to_string(), + url: "http://example.com/stream.m3u8".to_string(), + item_type: PlaylistItemType::Video, + reason: ResolveReason::MissingDetails.into(), + delay: 0, + }; + + assert!(InputWorker::task_needs_provider_connection(&task, InputType::M3u)); + assert!(InputWorker::task_needs_provider_connection(&task, InputType::Xtream)); +} + +#[test] +fn task_needs_provider_connection_keeps_live_probe() { + let task = UpdateTask::ProbeLive { + id: ProviderIdType::Id(1), + reason: ResolveReason::Probe.into(), + delay: 0, + interval: 60, + }; + + assert!(InputWorker::task_needs_provider_connection(&task, InputType::Library)); +} + +#[test] +fn collect_series_virtual_updates_resolves_text_ids_via_series_info_uuid() { + assert_collect_resolves_text_id!( + collect_series_virtual_updates, + PlaylistItemType::SeriesInfo, + "input_series", + "series-text-id", + add_series, + SeriesStreamProperties::default() + ); +} + +#[test] +fn collect_vod_virtual_updates_resolves_text_ids_via_video_uuid() { + assert_collect_resolves_text_id!( + collect_vod_virtual_updates, + PlaylistItemType::Video, + "input_vod", + "vod-text-id", + add_vod, + VideoStreamProperties::default() + ); +} + +#[test] +fn collect_live_virtual_updates_resolves_text_ids_via_live_uuid() { + assert_collect_resolves_text_id!( + collect_live_virtual_updates, + PlaylistItemType::Live, + "input_live", + "live-text-id", + add_live, + LiveStreamProperties::default() + ); +} + +#[test] +fn generic_probe_uses_pending_vod_batch_as_update_base() { + let mut batch = BatchResultCollector::new(); + let pending_props = + VideoStreamProperties { container_extension: Arc::from("mkv"), ..VideoStreamProperties::default() }; + batch.add_vod(ProviderIdType::Id(42), pending_props); + + let mut item = XtreamPlaylistItem { + virtual_id: VirtualId::new(7), + provider_id: 42, + name: Arc::from("Movie"), + logo: Arc::from(""), + logo_small: Arc::from(""), + group: Arc::from(""), + title: Arc::from(""), + parent_code: Arc::from(""), + rec: Arc::from(""), + url: Arc::from(""), + epg_channel_id: None, + xtream_cluster: XtreamCluster::Video, + additional_properties: None, + item_type: PlaylistItemType::Video, + category_id: 0, + input_name: Arc::from("input"), + channel_no: 0, + source_ordinal: 0, + input_stream_id: Arc::from("42"), + upstream_user_agent: None, + }; + + InputWorker::apply_pending_generic_probe_base(&batch, XtreamCluster::Video, 42, &mut item); + + let Some(StreamProperties::Video(props)) = item.additional_properties else { + panic!("expected pending video properties"); + }; + assert_eq!(props.container_extension.as_ref(), "mkv"); +} + +#[test] +fn strip_tmdb_reasons_returns_none_for_tmdb_only_resolve_task() { + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(100), + reason: ResolveReason::Tmdb | ResolveReason::Date, + delay: 5, + source_last_modified: None, + }; + + assert!(InputWorker::strip_tmdb_reasons(&task).is_none()); +} + +#[test] +fn strip_tmdb_reasons_keeps_non_tmdb_reasons() { + let task = UpdateTask::ResolveSeries { + id: ProviderIdType::Id(5), + reason: ResolveReason::Tmdb | ResolveReason::Probe | ResolveReason::Info, + delay: 1, + source_last_modified: Some(123), + }; + + let stripped = InputWorker::strip_tmdb_reasons(&task).expect("task should keep non-tmdb reasons"); + match stripped { + UpdateTask::ResolveSeries { reason, .. } => { + assert!(!reason.contains(ResolveReason::Tmdb)); + assert!(!reason.contains(ResolveReason::Date)); + assert!(reason.contains(ResolveReason::Probe)); + assert!(reason.contains(ResolveReason::Info)); + } + other => panic!("unexpected task type after strip: {other:?}"), + } +} + +#[test] +fn merge_task_payload_resolve_vod_unknown_last_modified_wins() { + let mut existing = UpdateTask::ResolveVod { + id: ProviderIdType::Id(7), + reason: ResolveReason::Info.into(), + delay: 5, + source_last_modified: Some(123), + }; + let changed = MetadataUpdateManager::merge_task_payload( + &mut existing, + UpdateTask::ResolveVod { + id: ProviderIdType::Id(7), + reason: ResolveReason::Probe.into(), + delay: 3, + source_last_modified: None, + }, + ); + + assert!(changed); + match existing { + UpdateTask::ResolveVod { reason, delay, source_last_modified, .. } => { + assert!(reason.contains(ResolveReason::Info)); + assert!(reason.contains(ResolveReason::Probe)); + assert_eq!(delay, 3); + assert_eq!(source_last_modified, None); + } + other => panic!("unexpected merged task type: {other:?}"), + } +} + +#[test] +fn merge_task_payload_resolve_series_unknown_last_modified_wins() { + let mut existing = UpdateTask::ResolveSeries { + id: ProviderIdType::Id(9), + reason: ResolveReason::Info.into(), + delay: 2, + source_last_modified: None, + }; + let changed = MetadataUpdateManager::merge_task_payload( + &mut existing, + UpdateTask::ResolveSeries { + id: ProviderIdType::Id(9), + reason: ResolveReason::Tmdb.into(), + delay: 2, + source_last_modified: Some(456), + }, + ); + + assert!(changed); + match existing { + UpdateTask::ResolveSeries { reason, delay, source_last_modified, .. } => { + assert!(reason.contains(ResolveReason::Info)); + assert!(reason.contains(ResolveReason::Tmdb)); + assert_eq!(delay, 2); + assert_eq!(source_last_modified, None); + } + other => panic!("unexpected merged task type: {other:?}"), + } +} + +#[test] +fn transient_worker_errors_include_connection_unavailable() { + assert!(InputWorker::is_transient_worker_error(TASK_ERR_UPDATE_IN_PROGRESS)); + assert!(InputWorker::is_transient_worker_error(TASK_ERR_PREEMPTED)); + assert!(InputWorker::is_transient_worker_error(TASK_ERR_NO_CONNECTION)); + assert!(!InputWorker::is_transient_worker_error("permanent error")); +} + +#[test] +fn permanent_not_found_error_matches_standalone_markers() { + assert!(InputWorker::is_permanent_not_found_error("HTTP 404 Not Found")); + assert!(InputWorker::is_permanent_not_found_error("probe failed: 404: stream unavailable")); + assert!(InputWorker::is_permanent_not_found_error("resource not found on provider")); +} + +#[test] +fn permanent_not_found_error_ignores_partial_markers() { + assert!(!InputWorker::is_permanent_not_found_error("error code 1404 while probing")); + assert!(!InputWorker::is_permanent_not_found_error("status404unexpected")); + assert!(!InputWorker::is_permanent_not_found_error("movie not foundry metadata mismatch")); +} + +#[test] +fn take_pending_probe_task_snapshot_skips_scheduled_requeues() { + let (tx, rx) = mpsc::channel::(8); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(1)); + let worker = create_test_worker("input_probe_skip", tx, rx, pending_tasks.clone(), pending_task_count); + + let key = TaskKey::Live(100); + let task = UpdateTask::ProbeLive { + id: ProviderIdType::Id(100), + reason: ResolveReason::Probe.into(), + delay: 0, + interval: 60, + }; + pending_tasks.insert(key.clone(), PendingTask::new(task)); + worker.scheduled_requeues.insert(key, chrono::Utc::now().timestamp().saturating_add(30)); + + assert!(worker.take_pending_probe_task_snapshot().is_none()); +} + +#[test] +fn take_pending_probe_task_snapshot_accepts_probe_only_resolve() { + let (tx, rx) = mpsc::channel::(8); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(1)); + let worker = create_test_worker("input_probe_only_resolve", tx, rx, pending_tasks.clone(), pending_task_count); + + let key = TaskKey::Vod(77); + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(77), + reason: ResolveReason::Probe.into(), + delay: 0, + source_last_modified: None, + }; + pending_tasks.insert(key.clone(), PendingTask::new(task)); + + let snapshot = worker.take_pending_probe_task_snapshot().expect("expected probe-domain snapshot"); + assert_eq!(snapshot.0, key); + assert_eq!(InputWorker::retry_domain_for_task(&snapshot.1), RetryDomain::Probe); +} + +#[test] +fn playlist_trigger_ignores_probe_only_changes() { + let task = UpdateTask::ProbeLive { + id: ProviderIdType::Id(22), + reason: ResolveReason::Probe.into(), + delay: 0, + interval: 60, + }; + assert!(!InputWorker::should_trigger_playlist_update_for_task(&task, true)); +} + +#[test] +fn playlist_trigger_keeps_info_changes() { + let task = UpdateTask::ResolveSeries { + id: ProviderIdType::Id(33), + reason: ResolveReason::Info.into(), + delay: 0, + source_last_modified: None, + }; + assert!(InputWorker::should_trigger_playlist_update_for_task(&task, true)); + assert!(!InputWorker::should_trigger_playlist_update_for_task(&task, false)); +} diff --git a/backend/metadata/src/manager/tests/lifecycle.rs b/backend/metadata/src/manager/tests/lifecycle.rs new file mode 100644 index 000000000..982b2cd56 --- /dev/null +++ b/backend/metadata/src/manager/tests/lifecycle.rs @@ -0,0 +1,44 @@ +use super::{create_test_worker, MetadataUpdateRuntimeSettings, PendingTask, TaskKey}; +use dashmap::DashMap; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; +use tokio::sync::mpsc; +use tuliprox_core::model::{ProviderIdType, ResolveReason, UpdateTask}; + +#[tokio::test] +async fn recent_no_change_skip_ttl_expiry_allows_requeue() { + let (tx, rx) = mpsc::channel::(8); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(1)); + let key = TaskKey::Vod(11); + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(11), + reason: ResolveReason::Info.into(), + delay: 0, + source_last_modified: None, + }; + + pending_tasks.insert(key.clone(), PendingTask::new(task.clone())); + if let Some(entry) = pending_tasks.get(&key) { + entry.generation.store(1, Ordering::Relaxed); + } + + let mut worker = create_test_worker("input_d", tx, rx, pending_tasks.clone(), pending_task_count); + let runtime_settings = + MetadataUpdateRuntimeSettings { no_change_cache_ttl_secs: 1, ..MetadataUpdateRuntimeSettings::default() }; + + let stale_instant = Instant::now().checked_sub(Duration::from_secs(2)).unwrap_or_else(Instant::now); + worker.recently_completed_no_change.insert(key.clone(), (stale_instant, ResolveReason::Info.into())); + assert!(!worker.should_skip_recent_no_change_task(&key, &task, &runtime_settings)); + assert!(!worker.recently_completed_no_change.contains_key(&key)); + + let requeued = worker.finalize_processed_task_success(&key, 0, "input_d").await; + assert!(requeued); + assert!(pending_tasks.contains_key(&key)); + assert_eq!(worker.receiver.try_recv().expect("requeued signal should be present"), key); +} diff --git a/backend/metadata/src/manager/tests/retry.rs b/backend/metadata/src/manager/tests/retry.rs new file mode 100644 index 000000000..31a662196 --- /dev/null +++ b/backend/metadata/src/manager/tests/retry.rs @@ -0,0 +1,164 @@ +use super::{ + create_test_worker, load_metadata_retry_states_from_disk, persist_metadata_retry_state_to_disk, InputWorker, + MetadataUpdateRuntimeSettings, RetryDomain, RetryState, TaskKey, TaskRetryState, +}; +use dashmap::DashMap; +use std::sync::{atomic::AtomicUsize, Arc}; +use tempfile::tempdir; +use tokio::sync::mpsc; +use tuliprox_core::model::{ProviderIdType, ResolveReason, UpdateTask}; +use tuliprox_repository::RetryStateDbValue; + +#[tokio::test] +async fn flush_dirty_retry_states_retries_persist_after_transient_failure() { + let dir = tempdir().expect("tempdir should be created"); + let failing_path = dir.path().join("missing").join("metadata_retry_state.db"); + let success_path = dir.path().join("metadata_retry_state.db"); + let key = TaskKey::Vod(42); + let state = TaskRetryState { + resolve: Some(RetryState { + attempts: 2, + next_allowed_at_ts: 1_700_043_200, + cooldown_until_ts: None, + last_error: Some("temporary failure".to_string()), + source_last_modified: None, + }), + probe: None, + tmdb: None, + updated_at_ts: 1_700_000_000, + }; + let (tx, rx) = mpsc::channel::(8); + let pending_tasks = Arc::new(DashMap::new()); + let pending_task_count = Arc::new(AtomicUsize::new(0)); + let mut worker = create_test_worker("input_retry_flush", tx, rx, pending_tasks, pending_task_count); + + worker.metadata_retry_state_path = Some(failing_path); + worker.retry_states.insert(key.clone(), state.clone()); + worker.dirty_retry_state_keys.insert(key.clone()); + + worker.persist_metadata_retry_state(&key, Some(&state)).await; + assert!(worker.dirty_retry_state_keys.contains(&key)); + + worker.metadata_retry_state_path = Some(success_path.clone()); + worker.flush_dirty_retry_states().await; + + assert!(!worker.dirty_retry_state_keys.contains(&key)); + let loaded = load_metadata_retry_states_from_disk(&success_path).expect("state load should succeed after retry"); + let loaded_state = loaded.get(&key).expect("retry state should be persisted on flush"); + let loaded_resolve = loaded_state.resolve.as_ref().expect("resolve retry state should be present"); + assert_eq!(loaded_resolve.attempts, 2); + assert_eq!(loaded_resolve.last_error.as_deref(), Some("temporary failure")); +} + +#[test] +fn metadata_retry_state_disk_roundtrip() { + let dir = tempdir().expect("tempdir should be created"); + let path = dir.path().join("metadata_retry_state.db"); + let key = TaskKey::Stream { scope: Arc::from("input_a"), id: Arc::from("stream_1") }; + let state = TaskRetryState { + resolve: Some(RetryState { + attempts: 4, + next_allowed_at_ts: 1_700_043_200, + cooldown_until_ts: Some(1_700_043_200), + last_error: Some("resolve exhausted".to_string()), + source_last_modified: None, + }), + probe: Some(RetryState { + attempts: 3, + next_allowed_at_ts: 1_700_000_000, + cooldown_until_ts: Some(1_700_086_400), + last_error: Some("probe timeout".to_string()), + source_last_modified: None, + }), + tmdb: Some(RetryState { + attempts: 0, + next_allowed_at_ts: 1_700_172_800, + cooldown_until_ts: Some(1_700_172_800), + last_error: Some("tmdb no match".to_string()), + source_last_modified: Some(999), + }), + updated_at_ts: 1_700_000_000, + }; + + persist_metadata_retry_state_to_disk(&path, &key, Some(&state)).expect("state persistence should succeed"); + let loaded = load_metadata_retry_states_from_disk(&path).expect("state load should succeed"); + let loaded_state = loaded.get(&key).expect("probe key should be present"); + + let loaded_resolve = loaded_state.resolve.as_ref().expect("resolve retry state should be present"); + assert_eq!(loaded_resolve.attempts, 4); + assert_eq!(loaded_resolve.cooldown_until_ts, Some(1_700_043_200)); + assert_eq!(loaded_resolve.last_error.as_deref(), Some("resolve exhausted")); + let loaded_probe = loaded_state.probe.as_ref().expect("probe retry state should be present"); + assert_eq!(loaded_probe.attempts, 3); + assert_eq!(loaded_probe.cooldown_until_ts, Some(1_700_086_400)); + assert_eq!(loaded_probe.last_error.as_deref(), Some("probe timeout")); + let loaded_tmdb = loaded_state.tmdb.as_ref().expect("tmdb retry state should be present"); + assert_eq!(loaded_tmdb.cooldown_until_ts, Some(1_700_172_800)); + assert_eq!(loaded_tmdb.last_error.as_deref(), Some("tmdb no match")); + assert_eq!(loaded_tmdb.source_last_modified, Some(999)); + + persist_metadata_retry_state_to_disk(&path, &key, None).expect("state clear should succeed"); + let cleared = load_metadata_retry_states_from_disk(&path).expect("state reload should succeed"); + assert!(!cleared.contains_key(&key)); +} + +#[test] +fn msgpack_backward_compat_old_4field_retry_state() { + // Simulate old 4-field format (before source_last_modified was added) + #[derive(Debug, serde::Serialize)] + struct OldRetryStateDbValue { + attempts: u8, + next_allowed_at_ts: i64, + cooldown_until_ts: Option, + last_error: Option, + } + + let old = OldRetryStateDbValue { + attempts: 3, + next_allowed_at_ts: 1_700_000_000, + cooldown_until_ts: Some(1_700_043_200), + last_error: Some("HTTP 404".to_string()), + }; + + let bytes = rmp_serde::to_vec(&old).expect("old format should serialize"); + let result = rmp_serde::from_slice::(&bytes); + assert!(result.is_ok(), "deserializing old 4-field format into new 5-field struct should succeed, got: {result:?}"); + let loaded = result.unwrap(); + assert_eq!(loaded.attempts, 3); + assert_eq!(loaded.cooldown_until_ts, Some(1_700_043_200)); + assert_eq!(loaded.source_last_modified, None); +} + +#[test] +fn probe_backoff_steps_follow_expected_windows() { + let runtime_settings = MetadataUpdateRuntimeSettings::default(); + let first = InputWorker::compute_probe_retry_backoff_secs(1, &runtime_settings); + let second = InputWorker::compute_probe_retry_backoff_secs(2, &runtime_settings); + let third = InputWorker::compute_probe_retry_backoff_secs(3, &runtime_settings); + + assert!((480..=720).contains(&first), "expected ~10m with jitter, got {first}"); + assert!((1_440..=2_160).contains(&second), "expected ~30m with jitter, got {second}"); + assert!((2_880..=4_320).contains(&third), "expected ~60m with jitter, got {third}"); +} + +#[test] +fn retry_domain_uses_probe_for_probe_only_resolve_tasks() { + let task = UpdateTask::ResolveVod { + id: ProviderIdType::Id(7), + reason: ResolveReason::Probe.into(), + delay: 0, + source_last_modified: None, + }; + assert_eq!(InputWorker::retry_domain_for_task(&task), RetryDomain::Probe); +} + +#[test] +fn retry_domain_keeps_resolve_for_mixed_resolve_tasks() { + let task = UpdateTask::ResolveSeries { + id: ProviderIdType::Id(11), + reason: ResolveReason::Probe | ResolveReason::Info, + delay: 0, + source_last_modified: None, + }; + assert_eq!(InputWorker::retry_domain_for_task(&task), RetryDomain::Resolve); +} diff --git a/backend/metadata/src/manager/tests/support.rs b/backend/metadata/src/manager/tests/support.rs new file mode 100644 index 000000000..d4603ae1f --- /dev/null +++ b/backend/metadata/src/manager/tests/support.rs @@ -0,0 +1,43 @@ +use super::{InputWorker, PendingTask, TaskKey}; +use dashmap::DashMap; +use std::{ + collections::{HashMap, HashSet}, + sync::{atomic::AtomicUsize, Arc}, +}; +use tokio::sync::{mpsc, RwLock}; +use tokio_util::sync::CancellationToken; +use tuliprox_core::model::BatchResultCollector; + +pub(in crate::manager::tests) fn create_test_worker( + input_name: &str, + sender: mpsc::Sender, + receiver: mpsc::Receiver, + pending_tasks: Arc>, + pending_task_count: Arc, +) -> InputWorker { + InputWorker { + input_name: Arc::from(input_name), + sender, + receiver, + pending_tasks, + pending_task_count, + bound_ctx: None, + update_pause_gate: Arc::new(RwLock::new(())), + cancel_token: CancellationToken::new(), + batch_buffer: BatchResultCollector::new(), + db_handles: HashMap::new(), + failed_clusters: HashSet::new(), + retry_states: HashMap::new(), + resolve_exhausted: HashMap::new(), + last_cycle_completed_at_ts: None, + metadata_retry_state_path: None, + metadata_retry_loaded: false, + metadata_retry_load_retry_at_ts: None, + last_retry_state_prune_at_ts: None, + scheduled_requeues: Arc::new(DashMap::new()), + recently_completed_no_change: HashMap::new(), + resolve_enqueue_suppressions: Arc::new(DashMap::new()), + tmdb_source_markers: Arc::new(DashMap::new()), + dirty_retry_state_keys: HashSet::new(), + } +} diff --git a/backend/metadata/src/manager/worker.rs b/backend/metadata/src/manager/worker.rs new file mode 100644 index 000000000..cddd7073d --- /dev/null +++ b/backend/metadata/src/manager/worker.rs @@ -0,0 +1,1013 @@ +use super::{ + spawn_blocking_limited, DbHandle, MetadataUpdateManager, MetadataUpdateRuntimeSettings, PendingTask, RetryDomain, + RetryState, ScopedTaskKey, TaskKey, TaskRetryState, RUNTIME_SETTINGS_REFRESH_INTERVAL_SECS, + TASK_ERR_UPDATE_IN_PROGRESS, +}; +use crate::ctx::{BoundMetadataUpdateCtx, MetadataUpdateCtx}; +use dashmap::{mapref::entry::Entry, DashMap}; +use log::{debug, error, info, warn}; +use shared::{ + model::{ + EventMessage, EventSink, LiveStreamProperties, MetadataUpdateFailure, PlaylistItemType, SeriesStreamProperties, + UUIDType, VideoStreamProperties, VirtualId, XtreamCluster, XtreamPlaylistItem, + }, + utils::{generate_provider_playlist_uuid, sanitize_sensitive_info}, +}; +use std::{ + collections::{HashMap, HashSet}, + path::PathBuf, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; +use tokio::sync::{mpsc, RwLock}; +use tokio_util::sync::CancellationToken; +use tuliprox_core::model::{BatchResultCollector, ProviderIdType, ResolveReasonSet, UpdateTask}; +use tuliprox_repository::{write_playlist_batch_item_upsert, xtream_get_file_path, BPlusTreeQuery, TargetIdMapping}; + +/// Per-input worker context. Each input has its own worker +/// that processes tasks sequentially with rate limiting. +#[derive(Clone)] +pub(super) struct InputWorkerContext { + pub(super) worker_id: u64, + pub(super) sender: mpsc::Sender, + pub(super) pending_tasks: Arc>, + pub(super) pending_task_count: Arc, +} + +impl MetadataUpdateManager { + /// Requests graceful shutdown for all metadata workers and delayed requeue tasks. + /// This operation is idempotent. + pub fn shutdown(&self) { + let _guard = self.worker_lifecycle_lock.lock(); + self.is_shutdown_flag.store(true, Ordering::Release); + self.cancel_token.load_full().cancel(); + self.resolve_enqueue_suppressions.clear(); + self.tmdb_source_markers.clear(); + self.enqueue_state_loaded_inputs.clear(); + self.enqueue_state_load_retry_at_ts.clear(); + } + + /// Returns true once shutdown has been requested. + pub fn is_shutdown(&self) -> bool { self.is_shutdown_flag.load(Ordering::Acquire) } + + /// Rotates the cancellation token for metadata workers. + /// Existing workers are cancelled and removed so new tasks start with fresh runtime state. + pub fn rotate_cancel_token(&self, cancel_token: CancellationToken) { + let old_token = { + let _guard = self.worker_lifecycle_lock.lock(); + if self.is_shutdown_flag.load(Ordering::Acquire) { + self.workers.clear(); + cancel_token.cancel(); + return; + } + + let old_token = self.cancel_token.swap(Arc::new(cancel_token)); + self.workers.clear(); + self.resolve_enqueue_suppressions.clear(); + self.tmdb_source_markers.clear(); + self.enqueue_state_loaded_inputs.clear(); + self.enqueue_state_load_retry_at_ts.clear(); + old_token + }; + old_token.cancel(); + } + + /// Acquire exclusive gate for a foreground playlist update. + /// While this guard is held, background workers wait before starting heavy metadata/probe steps. + pub async fn acquire_update_pause_guard(&self) -> tokio::sync::OwnedRwLockWriteGuard<()> { + self.update_pause_gate.clone().write_owned().await + } + + pub(super) fn remove_worker_context_if_id( + workers: &DashMap, InputWorkerContext>, + input_name: &Arc, + worker_id: u64, + ) { + if let Entry::Occupied(entry) = workers.entry(input_name.clone()) { + if entry.get().worker_id == worker_id { + entry.remove(); + } + } + } + + /// Get the number of active workers (for monitoring/debugging) + pub fn active_worker_count(&self) -> usize { self.workers.len() } +} + +pub(super) struct InputWorker { + pub(super) input_name: Arc, + pub(super) sender: mpsc::Sender, + pub(super) receiver: mpsc::Receiver, + pub(super) pending_tasks: Arc>, + pub(super) pending_task_count: Arc, + pub(super) bound_ctx: Option, + pub(super) update_pause_gate: Arc>, + pub(super) cancel_token: CancellationToken, + pub(super) batch_buffer: BatchResultCollector, + pub(super) db_handles: HashMap, + pub(super) failed_clusters: HashSet, + pub(super) retry_states: HashMap, + pub(super) resolve_exhausted: HashMap, + pub(super) last_cycle_completed_at_ts: Option, + pub(super) metadata_retry_state_path: Option, + pub(super) metadata_retry_loaded: bool, + pub(super) metadata_retry_load_retry_at_ts: Option, + pub(super) last_retry_state_prune_at_ts: Option, + // Shared with detached delayed requeue tasks spawned in `schedule_requeue_at`. + // A plain HashMap cannot be moved safely into those `'static` tasks. + pub(super) scheduled_requeues: Arc>, + // Cache of tasks that recently completed with no changes (Ok(None)). + // Prevents repeated resolution of already-resolved items across playlist refreshes. + // Stores the reason set so that tasks with new/different reasons are not wrongly skipped. + pub(super) recently_completed_no_change: HashMap, + pub(super) resolve_enqueue_suppressions: Arc>, + pub(super) tmdb_source_markers: Arc>, + pub(super) dirty_retry_state_keys: HashSet, +} + +impl InputWorker { + #[allow(clippy::too_many_lines)] + pub(super) async fn run(mut self) { + debug!("Metadata worker started for input {}", self.input_name); + + let mut processed_vod_count: usize = 0; + let mut processed_series_count: usize = 0; + let mut last_queue_log_at = Instant::now(); + let mut last_progress_log_at = Instant::now(); + let mut queue_cycle_active = false; + let mut cycle_had_changes = false; + // Tasks that burned through their retries this cycle. A cycle that + // exhausts work is the only signal an operator gets that an input has + // stopped resolving - `InputMetadataUpdatesCompleted` fires only when + // something actually changed, so a permanently broken input would + // otherwise emit a start and then nothing at all. + let mut cycle_failed_tasks: usize = 0; + let mut cycle_last_error: Option = None; + let mut consecutive_resolve_tasks = 0_usize; + + let input_name = self.input_name.clone(); + let bound_ctx = self.bound_ctx.clone(); + + let mut runtime_settings = self.runtime_settings(); + let mut last_runtime_settings_refresh_at = Instant::now(); + self.ensure_metadata_retry_state_loaded(&input_name, bound_ctx.as_ref(), &runtime_settings).await; + + // Keep one prefetched task to minimize channel waits/lock churn. + let mut next_task: Option<(TaskKey, UpdateTask, u64)> = None; + + loop { + if last_runtime_settings_refresh_at.elapsed() >= Duration::from_secs(RUNTIME_SETTINGS_REFRESH_INTERVAL_SECS) + { + runtime_settings = self.runtime_settings(); + last_runtime_settings_refresh_at = Instant::now(); + } + let prefer_probe = consecutive_resolve_tasks >= runtime_settings.probe_fairness_resolve_burst; + let task_data = if let Some(prefetched) = next_task.take() { + if prefer_probe && Self::retry_domain_for_task(&prefetched.1) == RetryDomain::Resolve { + if let Some(probe_task) = self.take_pending_probe_task_snapshot() { + next_task = Some(prefetched); + Some(probe_task) + } else { + Some(prefetched) + } + } else { + Some(prefetched) + } + } else if prefer_probe { + if let Some(probe_task) = self.take_pending_probe_task_snapshot() { + Some(probe_task) + } else { + self.recv_task_fast_or_wait(&runtime_settings).await + } + } else { + self.recv_task_fast_or_wait(&runtime_settings).await + }; + + let Some((current_key, current_task, current_generation)) = task_data else { break }; + if self.cancel_token.is_cancelled() { + break; + } + if !self.metadata_retry_loaded { + self.ensure_metadata_retry_state_loaded(&input_name, bound_ctx.as_ref(), &runtime_settings).await; + } + let now_ts = chrono::Utc::now().timestamp(); + self.prune_retry_tracking_maps_if_needed(now_ts, &runtime_settings).await; + + if !queue_cycle_active { + // First entry of a new processing cycle. + queue_cycle_active = true; + cycle_had_changes = false; + cycle_failed_tasks = 0; + cycle_last_error = None; + processed_vod_count = 0; + processed_series_count = 0; + last_progress_log_at = Instant::now(); + // Emit queue logs promptly for the new cycle. + last_queue_log_at = Instant::now() + .checked_sub(runtime_settings.queue_log_interval + Duration::from_secs(1)) + .unwrap_or_else(Instant::now); + if let Some(ctx) = bound_ctx.clone() { + ctx.events.emit(EventMessage::InputMetadataUpdatesStarted(input_name.clone())); + } + if self.last_cycle_completed_at_ts.is_some_and(|last| { + now_ts.saturating_sub(last) >= runtime_settings.resolve_exhaustion_reset_gap_secs + }) { + self.resolve_exhausted.clear(); + } + debug!("Background metadata update queue has entries for input {input_name}; starting processing"); + } + + let current_retry_domain = Self::retry_domain_for_task(¤t_task); + let delay_secs = current_task.delay(); + let mut schedule_requeue_at_ts: Option = None; + let mut remove_current_task = false; + let mut apply_rate_limit = false; + let mut metadata_persist_state: Option> = None; + let mut skip_execution = false; + let mut task_for_execution = current_task.clone(); + + if Self::is_resolve_task(&task_for_execution) && self.resolve_exhausted.contains_key(¤t_key) { + debug!( + "[Metadata-Task] Skipping task (resolve exhausted) for input {}: {} (reset window: {}s)", + input_name, task_for_execution, runtime_settings.resolve_exhaustion_reset_gap_secs + ); + self.scheduled_requeues.remove(¤t_key); + remove_current_task = true; + skip_execution = true; + } + + // Skip tasks that recently completed with no changes (already resolved in DB). + // Only skip when the incoming reason set exactly matches the cached reason set. + if !skip_execution + && self.should_skip_recent_no_change_task(¤t_key, &task_for_execution, &runtime_settings) + { + debug!( + "[Metadata-Task] Skipping task (recently completed with no changes) for input {input_name}: {task_for_execution}", + ); + remove_current_task = true; + skip_execution = true; + } + + if !skip_execution { + let mut clear_tmdb_state = false; + if let Some(state_bundle) = self.retry_states.get(¤t_key) { + if let Some(tmdb_state) = state_bundle.get(RetryDomain::Tmdb) { + let task_has_tmdb_reason = Self::task_has_tmdb_reason(&task_for_execution); + let source_unchanged = tmdb_state.source_last_modified.is_some() + && tmdb_state.source_last_modified == Self::task_source_last_modified(&task_for_execution); + if source_unchanged && task_has_tmdb_reason { + if let Some(stripped_task) = Self::strip_tmdb_reasons(&task_for_execution) { + debug!( + "[Task] TMDB/date unchanged for input {}: {}, continuing with non-TMDB reasons (last_modified={:?})", + input_name, + task_for_execution, + tmdb_state.source_last_modified + ); + task_for_execution = stripped_task; + } else { + debug!( + "[Metadata-Task] Skipping task (TMDB/date unresolved and source unchanged) for input {}: {} (last_modified={:?})", + input_name, + task_for_execution, + tmdb_state.source_last_modified + ); + self.scheduled_requeues.remove(¤t_key); + remove_current_task = true; + skip_execution = true; + } + } else if task_has_tmdb_reason { + if let Some(cooldown_until_ts) = tmdb_state.cooldown_until_ts { + if now_ts < cooldown_until_ts { + if let Some(stripped_task) = Self::strip_tmdb_reasons(&task_for_execution) { + debug!( + "[Task] TMDB cooldown active for input {}: {}, continuing with non-TMDB reasons (cooldown_until={}, remaining={}s)", + input_name, + task_for_execution, + cooldown_until_ts, + cooldown_until_ts.saturating_sub(now_ts) + ); + task_for_execution = stripped_task; + } else { + debug!( + "[Metadata-Task] Skipping task (TMDB-only in cooldown) for input {}: {} (cooldown_until={}, remaining={}s)", + input_name, + task_for_execution, + cooldown_until_ts, + cooldown_until_ts.saturating_sub(now_ts) + ); + self.scheduled_requeues.remove(¤t_key); + remove_current_task = true; + skip_execution = true; + } + } else { + clear_tmdb_state = true; + } + } + } + } + } + + let active_retry_domain = Self::retry_domain_for_task(&task_for_execution); + let mut clear_active_retry_state = false; + + if !skip_execution { + if let Some(state_bundle) = self.retry_states.get(¤t_key) { + if let Some(active_state) = state_bundle.get(active_retry_domain) { + if let Some(cooldown_until_ts) = active_state.cooldown_until_ts { + if now_ts < cooldown_until_ts { + let cooldown_label = + if active_retry_domain == RetryDomain::Probe { "probe" } else { "resolve" }; + debug!( + "[Metadata-Task] Skipping task ({} cooldown) for input {}: {} (cooldown_until={}, remaining={}s)", + cooldown_label, + input_name, + task_for_execution, + cooldown_until_ts, + cooldown_until_ts.saturating_sub(now_ts) + ); + self.scheduled_requeues.remove(¤t_key); + remove_current_task = true; + skip_execution = true; + } else { + clear_active_retry_state = true; + } + } + + if !skip_execution && active_state.next_allowed_at_ts > now_ts { + debug!( + "[Task] Deferring task (retry backoff) for input {}: {} (next_allowed_at={}, wait={}s, attempts={})", + input_name, + task_for_execution, + active_state.next_allowed_at_ts, + active_state.next_allowed_at_ts.saturating_sub(now_ts), + active_state.attempts + ); + schedule_requeue_at_ts = Some(active_state.next_allowed_at_ts); + skip_execution = true; + } + } + } + } + + if clear_active_retry_state || clear_tmdb_state { + let mut should_remove_retry_entry = false; + let mut state_after_clear: Option = None; + + if let Some(state_bundle) = self.retry_states.get_mut(¤t_key) { + if clear_active_retry_state { + state_bundle.clear_domain(active_retry_domain); + } + if clear_tmdb_state { + state_bundle.clear_domain(RetryDomain::Tmdb); + } + if state_bundle.is_empty() { + should_remove_retry_entry = true; + } else { + state_bundle.touch(now_ts); + state_after_clear = Some(state_bundle.clone()); + } + } + + if should_remove_retry_entry { + self.retry_states.remove(¤t_key); + } + if clear_active_retry_state && active_retry_domain == RetryDomain::Resolve { + self.clear_resolve_enqueue_suppression(¤t_key); + } + if clear_tmdb_state { + self.clear_tmdb_source_marker(¤t_key); + } + metadata_persist_state = Some(state_after_clear); + } + } + + if !skip_execution { + debug!( + "[Task] Executing task for input {}: {} (retry_domain={:?})", + input_name, + task_for_execution, + Self::retry_domain_for_task(&task_for_execution) + ); + let task_result = { + let Some(_pause_guard) = self.wait_for_update_pause_window().await else { + break; + }; + Self::process_task_static( + &input_name, + bound_ctx.as_ref(), + &task_for_execution, + &mut self.batch_buffer, + &mut self.db_handles, + &mut self.failed_clusters, + ) + .await + }; + + match task_result { + Ok(task_outcome) => { + if Self::is_vod_task_key(¤t_key) { + processed_vod_count += 1; + } else if Self::is_series_task_key(¤t_key) { + processed_series_count += 1; + } + let trigger_playlist_update = Self::should_trigger_playlist_update_for_task( + &task_for_execution, + task_outcome.task_changed, + ); + cycle_had_changes |= trigger_playlist_update; + debug!( + "[Metadata-Task] Task succeeded for input {input_name}: {task_for_execution} (changed={}, trigger_playlist_update={}, tmdb_pending={}, probe_pending={})", + task_outcome.task_changed, + trigger_playlist_update, + task_outcome.tmdb_pending, + task_outcome.probe_pending + ); + + let active_retry_domain = Self::retry_domain_for_task(&task_for_execution); + let task_has_tmdb_reason = Self::task_has_tmdb_reason(&task_for_execution); + let task_source_last_modified = Self::task_source_last_modified(&task_for_execution); + let mut should_remove_retry_entry = false; + let mut clear_tmdb_marker = false; + let mut state_after_success: Option = None; + + if let Some(state_bundle) = self.retry_states.get_mut(¤t_key) { + state_bundle.clear_domain(active_retry_domain); + if task_has_tmdb_reason { + if task_outcome.tmdb_pending { + let cooldown_until_ts = now_ts.saturating_add(runtime_settings.tmdb_cooldown_secs); + let tmdb_state = state_bundle.get_mut_or_insert(RetryDomain::Tmdb); + tmdb_state.attempts = 0; + tmdb_state.next_allowed_at_ts = cooldown_until_ts; + tmdb_state.cooldown_until_ts = Some(cooldown_until_ts); + tmdb_state.last_error = + Some("TMDB lookup completed without matching result".to_string()); + tmdb_state.source_last_modified = task_source_last_modified; + debug!( + "[Metadata-Task] TMDB resolve produced no match (existing retry state), entering cooldown for input {}: {} (cooldown_until={}, cooldown_duration={}s)", + input_name, + task_for_execution, + cooldown_until_ts, + runtime_settings.tmdb_cooldown_secs + ); + } else { + state_bundle.clear_domain(RetryDomain::Tmdb); + clear_tmdb_marker = true; + } + } + + if state_bundle.is_empty() { + should_remove_retry_entry = true; + } else { + state_bundle.touch(now_ts); + state_after_success = Some(state_bundle.clone()); + } + } else if task_has_tmdb_reason && task_outcome.tmdb_pending { + let cooldown_until_ts = now_ts.saturating_add(runtime_settings.tmdb_cooldown_secs); + let state_bundle = TaskRetryState { + resolve: None, + probe: None, + tmdb: Some(RetryState { + attempts: 0, + next_allowed_at_ts: cooldown_until_ts, + cooldown_until_ts: Some(cooldown_until_ts), + last_error: Some("TMDB lookup completed without matching result".to_string()), + source_last_modified: task_source_last_modified, + }), + updated_at_ts: now_ts.max(1), + }; + self.retry_states.insert(current_key.clone(), state_bundle.clone()); + state_after_success = Some(state_bundle); + debug!( + "[Metadata-Task] TMDB resolve produced no match (new retry state), entering cooldown for input {}: {} (cooldown_until={}, cooldown_duration={}s)", + input_name, + task_for_execution, + cooldown_until_ts, + runtime_settings.tmdb_cooldown_secs + ); + } else if task_has_tmdb_reason { + self.clear_tmdb_source_marker(¤t_key); + } + + if should_remove_retry_entry { + self.retry_states.remove(¤t_key); + } + if clear_tmdb_marker { + self.clear_tmdb_source_marker(¤t_key); + } + if task_outcome.tmdb_pending { + if let Some(source_last_modified) = task_source_last_modified { + self.set_tmdb_source_marker(¤t_key, source_last_modified); + } else { + self.clear_tmdb_source_marker(¤t_key); + } + } + if should_remove_retry_entry || state_after_success.is_some() { + metadata_persist_state = Some(state_after_success); + } + + self.resolve_exhausted.remove(¤t_key); + if active_retry_domain == RetryDomain::Resolve { + self.clear_resolve_enqueue_suppression(¤t_key); + } + self.scheduled_requeues.remove(¤t_key); + + // Cache tasks that completed with no changes to skip redundant re-resolution. + if !task_outcome.task_changed + && Self::is_resolve_task(&task_for_execution) + && !task_outcome.tmdb_pending + && !task_outcome.probe_pending + { + let reasons = Self::task_reason(&task_for_execution); + debug!( + "[Task] Caching no-change result for input {}: {} (reasons={}, ttl={}s)", + input_name, task_for_execution, reasons, runtime_settings.no_change_cache_ttl_secs + ); + self.recently_completed_no_change.insert(current_key.clone(), (Instant::now(), reasons)); + // Set producer-side enqueue suppression so that future playlist + // processing cycles do not re-queue this task during the TTL. + let suppressed_until_ts = now_ts.saturating_add( + i64::try_from(runtime_settings.no_change_cache_ttl_secs).unwrap_or(i64::MAX), + ); + self.set_resolve_enqueue_suppression(¤t_key, suppressed_until_ts); + } else { + self.recently_completed_no_change.remove(¤t_key); + } + + if last_progress_log_at.elapsed() >= runtime_settings.progress_log_interval { + // current_key is removed from pending_tasks later in this loop iteration; + // subtract it here so "remaining" reflects the post-success queue size. + let (mut remaining_vod, mut remaining_series) = + Self::queue_resolve_counts(&self.pending_tasks); + if Self::is_vod_task_key(¤t_key) { + remaining_vod = remaining_vod.saturating_sub(1); + } else if Self::is_series_task_key(¤t_key) { + remaining_series = remaining_series.saturating_sub(1); + } + + let total_vod = processed_vod_count.saturating_add(remaining_vod); + let total_series = processed_series_count.saturating_add(remaining_series); + let resolved_total = processed_vod_count.saturating_add(processed_series_count); + let total_resolve = total_vod.saturating_add(total_series); + + info!("Background metadata update: {resolved_total} / {total_resolve} resolved for input {input_name} (vod: {processed_vod_count}/{total_vod}, series: {processed_series_count}/{total_series})"); + last_progress_log_at = Instant::now(); + } + + remove_current_task = true; + apply_rate_limit = true; + } + Err(e) => { + if Self::is_permanent_not_found_error(e.message()) { + debug!( + "[Task] Task failed with permanent not-found for input {}: {} (error={})", + input_name, + task_for_execution, + e.message() + ); + let retry_domain = Self::retry_domain_for_task(&task_for_execution); + self.scheduled_requeues.remove(¤t_key); + if retry_domain == RetryDomain::Probe { + let cooldown_until_ts = now_ts.saturating_add(runtime_settings.probe_cooldown_secs); + let state_bundle_after_update = { + let state_bundle = self.retry_states.entry(current_key.clone()).or_default(); + let state = state_bundle.get_mut_or_insert(RetryDomain::Probe); + state.attempts = runtime_settings.max_attempts_probe; + state.next_allowed_at_ts = cooldown_until_ts; + state.cooldown_until_ts = Some(cooldown_until_ts); + state.last_error = Some(e.message().to_string()); + state_bundle.touch(now_ts); + state_bundle.clone() + }; + metadata_persist_state = Some(Some(state_bundle_after_update)); + remove_current_task = true; + debug!( + "[Metadata-Task] Probe task entering cooldown after permanent not-found for input {}: {} (cooldown_until={}, cooldown_duration={}s)", + input_name, + task_for_execution, + cooldown_until_ts, + runtime_settings.probe_cooldown_secs + ); + } else { + let cooldown_until_ts = + now_ts.saturating_add(runtime_settings.resolve_exhaustion_reset_gap_secs); + let state_bundle_after_update = { + let state_bundle = self.retry_states.entry(current_key.clone()).or_default(); + let state = state_bundle.get_mut_or_insert(RetryDomain::Resolve); + state.attempts = runtime_settings.max_attempts_resolve; + state.next_allowed_at_ts = cooldown_until_ts; + state.cooldown_until_ts = Some(cooldown_until_ts); + state.last_error = Some(e.message().to_string()); + state_bundle.touch(now_ts); + state_bundle.clone() + }; + self.set_resolve_enqueue_suppression(¤t_key, cooldown_until_ts); + self.resolve_exhausted.insert(current_key.clone(), now_ts); + metadata_persist_state = Some(Some(state_bundle_after_update)); + remove_current_task = true; + debug!( + "[Metadata-Task] Resolve task entering cooldown after permanent not-found for input {}: {} (cooldown_until={}, cooldown_duration={}s, error={})", + input_name, + task_for_execution, + cooldown_until_ts, + runtime_settings.resolve_exhaustion_reset_gap_secs, + e.message() + ); + } + } else if Self::is_transient_worker_error(e.message()) { + if e.message() == TASK_ERR_UPDATE_IN_PROGRESS { + // Drop cached readers quickly so foreground writer can progress. + self.release_db_handles(); + } + + let retry_delay_secs = + Self::compute_retry_delay_secs(current_task.delay(), &runtime_settings); + let retry_delay_i64 = i64::try_from(retry_delay_secs).unwrap_or(i64::MAX); + schedule_requeue_at_ts = Some(now_ts.saturating_add(retry_delay_i64)); + debug!( + "[Metadata-Task] Task deferred (transient error) for input {}: {} (retry_in={}s, error={})", + input_name, + task_for_execution, + retry_delay_secs, + e.message() + ); + } else { + let retry_domain = Self::retry_domain_for_task(&task_for_execution); + let max_attempts = if retry_domain == RetryDomain::Probe { + runtime_settings.max_attempts_probe + } else { + runtime_settings.max_attempts_resolve + }; + + let (state_after_update, state_bundle_after_update) = { + let state_bundle = self.retry_states.entry(current_key.clone()).or_default(); + let state_after_update = { + let state = state_bundle.get_mut_or_insert(retry_domain); + state.attempts = state.attempts.saturating_add(1); + state.last_error = Some(e.message().to_string()); + + if state.attempts < max_attempts { + let backoff_secs = if retry_domain == RetryDomain::Probe { + Self::compute_probe_retry_backoff_secs(state.attempts, &runtime_settings) + } else { + Self::compute_resolve_retry_backoff_secs( + task_for_execution.delay(), + state.attempts, + &runtime_settings, + ) + }; + let backoff_i64 = i64::try_from(backoff_secs).unwrap_or(i64::MAX); + state.next_allowed_at_ts = now_ts.saturating_add(backoff_i64); + state.cooldown_until_ts = None; + } else if retry_domain == RetryDomain::Probe { + state.cooldown_until_ts = + Some(now_ts.saturating_add(runtime_settings.probe_cooldown_secs)); + state.next_allowed_at_ts = state.cooldown_until_ts.unwrap_or(now_ts); + } + state.clone() + }; + state_bundle.touch(now_ts); + + (state_after_update, state_bundle.clone()) + }; + + let attempts = state_after_update.attempts; + + if attempts >= max_attempts { + cycle_failed_tasks = cycle_failed_tasks.saturating_add(1); + cycle_last_error = Some(sanitize_sensitive_info(e.message()).into_owned()); + self.scheduled_requeues.remove(¤t_key); + if retry_domain == RetryDomain::Probe { + remove_current_task = true; + metadata_persist_state = Some(Some(state_bundle_after_update.clone())); + let cooldown_until = state_after_update + .cooldown_until_ts + .map_or_else(|| "none".to_string(), |ts| ts.to_string()); + debug!( + "Metadata-[Task] Probe task exhausted (max attempts reached) for input {}: {} (attempts={}/{}, cooldown_until={}, error={})", + input_name, + task_for_execution, + state_after_update.attempts, + max_attempts, + cooldown_until, + e.message() + ); + } else { + let cooldown_until_ts = + now_ts.saturating_add(runtime_settings.resolve_exhaustion_reset_gap_secs); + let state_bundle_after_update = { + let state_bundle = self.retry_states.entry(current_key.clone()).or_default(); + let state = state_bundle.get_mut_or_insert(RetryDomain::Resolve); + state.attempts = max_attempts; + state.next_allowed_at_ts = cooldown_until_ts; + state.cooldown_until_ts = Some(cooldown_until_ts); + state.last_error = Some(e.message().to_string()); + state_bundle.touch(now_ts); + state_bundle.clone() + }; + self.set_resolve_enqueue_suppression(¤t_key, cooldown_until_ts); + self.resolve_exhausted.insert(current_key.clone(), now_ts); + metadata_persist_state = Some(Some(state_bundle_after_update)); + remove_current_task = true; + debug!( + "[Metadata-Task] Resolve task exhausted (max attempts reached) for input {}: {} (attempts={}/{}, cooldown_duration={}s, error={})", + input_name, + task_for_execution, + attempts, + max_attempts, + runtime_settings.resolve_exhaustion_reset_gap_secs, + e.message() + ); + } + } else { + schedule_requeue_at_ts = Some(state_after_update.next_allowed_at_ts); + metadata_persist_state = Some(Some(state_bundle_after_update)); + if retry_domain == RetryDomain::Resolve { + self.set_resolve_enqueue_suppression( + ¤t_key, + state_after_update.next_allowed_at_ts, + ); + } + debug!( + "[Metadata-Task] Task failed, scheduling retry for input {}: {} (attempt={}/{}, next_allowed_at={}, backoff={}s, error={})", + input_name, + task_for_execution, + attempts, + max_attempts, + state_after_update.next_allowed_at_ts, + state_after_update.next_allowed_at_ts.saturating_sub(now_ts), + e.message() + ); + } + } + } + } + } + + if let Some(state) = metadata_persist_state { + self.dirty_retry_state_keys.insert(current_key.clone()); + self.persist_metadata_retry_state(¤t_key, state.as_ref()).await; + } + + // Check and flush batch + if self.batch_buffer.should_flush() { + self.release_db_handles(); + let Some(_pause_guard) = self.wait_for_update_pause_window().await else { + break; + }; + Self::flush_batch_static(&input_name, bound_ctx.as_ref(), &mut self.batch_buffer).await; + } + + if let Some(retry_at_ts) = schedule_requeue_at_ts { + debug!( + "[Task] Scheduling requeue for input {}: {:?} (retry_at_ts={}, in={}s)", + input_name, + current_key, + retry_at_ts, + retry_at_ts.saturating_sub(chrono::Utc::now().timestamp()) + ); + self.schedule_requeue_at(current_key.clone(), retry_at_ts); + } + + if remove_current_task { + self.finalize_processed_task_success(¤t_key, current_generation, &input_name).await; + } + + if apply_rate_limit { + // Rate limiting + if delay_secs > 0 + && Self::sleep_or_cancel(&self.cancel_token, Duration::from_secs(u64::from(delay_secs))).await + { + break; + } + } + + if current_retry_domain == RetryDomain::Resolve { + consecutive_resolve_tasks = consecutive_resolve_tasks.saturating_add(1); + } else { + consecutive_resolve_tasks = 0; + } + + // Try to get the next task immediately to keep locks open. + // Ignore phantom signals (channel key without pending map entry). + if next_task.is_none() { + if self.cancel_token.is_cancelled() { + break; + } + while let Ok(key) = self.receiver.try_recv() { + if self.cancel_token.is_cancelled() { + break; + } + if let Some(snapshot) = self.load_task_snapshot(key) { + next_task = Some(snapshot); + break; + } + } + } + + let channel_has_work = next_task.is_some() || !self.receiver.is_empty(); + let queue_completely_empty = + !channel_has_work && self.pending_tasks.is_empty() && self.scheduled_requeues.is_empty(); + + // Avoid O(n) queue scans per task; report queue status periodically. + if (channel_has_work || !self.pending_tasks.is_empty()) + && last_queue_log_at.elapsed() >= runtime_settings.queue_log_interval + { + let queue_counts = Self::queue_resolve_counts(&self.pending_tasks); + debug!("In queue to resolve vod: {}, series: {} (input: {input_name})", queue_counts.0, queue_counts.1); + last_queue_log_at = Instant::now(); + } + + // If no immediate work is available, flush buffered results now even when delayed retries remain pending. + if !channel_has_work && !self.batch_buffer.is_empty() { + self.release_db_handles(); + let Some(_pause_guard) = self.wait_for_update_pause_window().await else { + break; + }; + Self::flush_batch_static(&input_name, bound_ctx.as_ref(), &mut self.batch_buffer).await; + } + + if queue_cycle_active && queue_completely_empty { + self.last_cycle_completed_at_ts = Some(chrono::Utc::now().timestamp()); + queue_cycle_active = false; + processed_vod_count = 0; + processed_series_count = 0; + consecutive_resolve_tasks = 0; + if cycle_had_changes { + info!("All pending metadata resolves completed for input {input_name} (with changes)"); + if let Some(ctx) = bound_ctx.clone() { + ctx.events.emit(EventMessage::InputMetadataUpdatesCompleted(input_name.clone())); + } + } else { + debug!("All pending metadata resolves completed for input {input_name} (no changes, skipping playlist update trigger)"); + } + // Reported alongside the completion rather than instead of it: + // a cycle can both produce changes and exhaust tasks, and + // suppressing the completion would also suppress the playlist + // update it triggers. + if cycle_failed_tasks > 0 { + warn!( + "Metadata update cycle for input {input_name} exhausted {cycle_failed_tasks} task(s) without resolving them" + ); + if let Some(ctx) = bound_ctx.clone() { + ctx.events.emit(EventMessage::InputMetadataUpdatesFailed(MetadataUpdateFailure::new( + input_name.clone(), + cycle_failed_tasks, + cycle_had_changes, + cycle_last_error.clone(), + ))); + } + } + cycle_had_changes = false; + cycle_failed_tasks = 0; + cycle_last_error = None; + } + } + + // Final flush + self.release_db_handles(); + if !self.batch_buffer.is_empty() { + if let Some(_pause_guard) = self.wait_for_update_pause_window().await { + Self::flush_batch_static(&input_name, bound_ctx.as_ref(), &mut self.batch_buffer).await; + } + } + self.flush_dirty_retry_states().await; + + debug!("Metadata worker stopped for input {input_name}"); + } + + async fn recv_task_fast_or_wait( + &mut self, + runtime_settings: &MetadataUpdateRuntimeSettings, + ) -> Option<(TaskKey, UpdateTask, u64)> { + if self.cancel_token.is_cancelled() { + return None; + } + + // Fast path: drain immediate signals until we find a real pending task. + loop { + if self.cancel_token.is_cancelled() { + return None; + } + match self.receiver.try_recv() { + Ok(key) => { + if self.cancel_token.is_cancelled() { + return None; + } + if let Some(snapshot) = self.load_task_snapshot(key) { + return Some(snapshot); + } + } + Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => { + return None; + } + Err(tokio::sync::mpsc::error::TryRecvError::Empty) => { + break; + } + } + } + + // When idle, release read handles to avoid writer starvation. + self.release_db_handles(); + + loop { + tokio::select! { + biased; + () = self.cancel_token.cancelled() => return None, + res = tokio::time::timeout(Duration::from_secs(runtime_settings.worker_idle_timeout_secs), self.receiver.recv()) => { + match res { + Ok(Some(key)) => { + if self.cancel_token.is_cancelled() { + return None; + } + if let Some(snapshot) = self.load_task_snapshot(key) { + return Some(snapshot); + } + } + Ok(None) => return None, + Err(_) => { + loop { + if self.cancel_token.is_cancelled() { + return None; + } + match self.receiver.try_recv() { + Ok(key) => { + if self.cancel_token.is_cancelled() { + return None; + } + if let Some(snapshot) = self.load_task_snapshot(key) { + return Some(snapshot); + } + } + Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => return None, + Err(tokio::sync::mpsc::error::TryRecvError::Empty) => break, + } + } + if self.pending_tasks.is_empty() + && self.receiver.is_empty() + && self.scheduled_requeues.is_empty() + { + return None; + } + } + } + } + } + } + } + + async fn wait_for_update_pause_window(&mut self) -> Option> { + if let Ok(guard) = self.update_pause_gate.clone().try_read_owned() { + return Some(guard); + } + + // Foreground writer is active or queued. Release cached handles before waiting to avoid AB-BA patterns. + self.release_db_handles(); + tokio::select! { + () = self.cancel_token.cancelled() => None, + guard = self.update_pause_gate.clone().read_owned() => { + Some(guard) + } + } + } + + fn release_db_handles(&mut self) { + if !self.db_handles.is_empty() { + self.db_handles.clear(); + } + if !self.failed_clusters.is_empty() { + self.failed_clusters.clear(); + } + } + + async fn sleep_or_cancel(cancel_token: &CancellationToken, duration: Duration) -> bool { + tokio::select! { + () = cancel_token.cancelled() => true, + () = tokio::time::sleep(duration) => false, + } + } + + // Three associated functions generated by the `collect_virtual_updates!` + // macro below. The macro body walks a single `BatchResultCollector` field + // and builds a `HashMap` of pending per-virtual updates. + collect_virtual_updates!(collect_vod_virtual_updates, vod, VideoStreamProperties, PlaylistItemType::Video); + collect_virtual_updates!( + collect_series_virtual_updates, + series, + SeriesStreamProperties, + PlaylistItemType::SeriesInfo + ); + collect_virtual_updates!(collect_live_virtual_updates, live, LiveStreamProperties, PlaylistItemType::Live); + + apply_cascade_updates!(apply_vod_cascade_updates, VideoStreamProperties, XtreamCluster::Video, Video, "VOD"); + apply_cascade_updates!( + apply_series_cascade_updates, + SeriesStreamProperties, + XtreamCluster::Series, + Series, + "Series" + ); + apply_cascade_updates!(apply_live_cascade_updates, LiveStreamProperties, XtreamCluster::Live, Live, "Live"); +} diff --git a/backend/mpegts/src/transport_stream_buffer.rs b/backend/mpegts/src/transport_stream_buffer.rs index d4813d478..05f3adfdf 100644 --- a/backend/mpegts/src/transport_stream_buffer.rs +++ b/backend/mpegts/src/transport_stream_buffer.rs @@ -1,150 +1,6 @@ -use bytes::{Bytes, BytesMut}; +use bytes::Bytes; use futures::task::AtomicWaker; -use sha2::{Digest, Sha256}; -use std::{ - collections::{HashMap, HashSet}, - sync::Arc, - task::Waker, -}; - -// PCR wraps at 2^33 * 300 (base is 33-bit, multiplied by 300 to get 27 MHz units). -// Using 1<<42 was slightly too large and could cause strict-decoder issues on hardware -// that computes modulo 2^33 on the base before multiplying. -const MAX_PCR: u64 = (1u64 << 33) * 300; -const MAX_PTS_DTS: u64 = 1 << 33; // 33 bit PTS/DTS cycle -const HLS_TS_SPLICE_MIN_GAP_TICKS_90KHZ: u64 = 90; -const HLS_TS_PROFILE_MIN_TOLERANCE_TICKS_90KHZ: u64 = 2 * 90_000; - -#[inline] -#[allow(clippy::cast_possible_truncation)] -fn add_pts_dts_offset(timestamp: u64, offset_90khz: u64) -> u64 { - ((u128::from(timestamp) + u128::from(offset_90khz)) % u128::from(MAX_PTS_DTS)) as u64 -} - -#[inline] -fn forward_clock_distance_90khz(start: u64, end: u64) -> u64 { - end.wrapping_add(MAX_PTS_DTS).wrapping_sub(start) % MAX_PTS_DTS -} - -#[inline] -#[allow(clippy::cast_possible_truncation)] -fn pcr_offset_27mhz(offset_90khz: u64) -> u64 { (u128::from(offset_90khz) * 300_u128 % u128::from(MAX_PCR)) as u64 } - -#[inline] -#[allow(clippy::cast_possible_truncation)] -fn add_pcr_offset_27mhz(timestamp_27mhz: u64, offset_27mhz: u64) -> u64 { - ((u128::from(timestamp_27mhz) + u128::from(offset_27mhz)) % u128::from(MAX_PCR)) as u64 -} - -const TS_PACKET_SIZE: usize = 188; -const SYNC_BYTE: u8 = 0x47; -const PACKET_COUNT: usize = 7; // Reduced from 250 to 7 (1316 bytes) to prevent latency/timeout on low-bitrate streams -const MAX_PACKET_COUNT: usize = 250; - -/// Packets per emitted chunk; overridable via `TULIPROX_TS_CHUNK_PACKETS` (1-250). -/// Larger chunks raise throughput, smaller chunks lower latency on low-bitrate streams. -fn ts_chunk_packet_count() -> usize { - static PACKET_COUNT_OVERRIDE: std::sync::LazyLock = std::sync::LazyLock::new(|| { - std::env::var("TULIPROX_TS_CHUNK_PACKETS") - .ok() - .and_then(|value| value.trim().parse::().ok()) - .filter(|count| (1..=MAX_PACKET_COUNT).contains(count)) - .unwrap_or(PACKET_COUNT) - }); - *PACKET_COUNT_OVERRIDE -} - -const ADAPTATION_FIELD_FLAG_PCR: u8 = 0x10; // PCR flag bit in adaptation field flags -const NULL_PID: u16 = 0x1FFF; -const HLS_TS_MAX_PENDING_PES_HEADERS: usize = 64; -const HLS_TS_TIMESTAMP_HEADER_BYTES: usize = 19; - -/// Byte offset of PTS within a PES payload (after the 3-byte start code, `stream_id`, length, flags). -const PES_PTS_OFFSET: usize = 9; -/// Byte offset of DTS within a PES payload when both PTS and DTS are present. -const PES_DTS_OFFSET: usize = 14; - -/// Decodes a 5-byte DTS/PTS field from PES header into u64 timestamp. -#[inline] -fn decode_timestamp(ts_bytes: &[u8]) -> u64 { - (((u64::from(ts_bytes[0]) >> 1) & 0x07) << 30) - | (u64::from(ts_bytes[1]) << 22) - | (((u64::from(ts_bytes[2]) >> 1) & 0x7F) << 15) - | (u64::from(ts_bytes[3]) << 7) - | ((u64::from(ts_bytes[4]) >> 1) & 0x7F) -} - -/// Encodes a u64 timestamp into 5-byte PES DTS/PTS field -#[inline] -fn encode_timestamp(ts: u64) -> [u8; 5] { - [ - 0x20 | ((((ts >> 30) & 0x07) as u8) << 1) | 1, - ((ts >> 22) & 0xFF) as u8, - ((((ts >> 15) & 0x7F) as u8) << 1) | 1, - ((ts >> 7) & 0xFF) as u8, - (((ts & 0x7F) as u8) << 1) | 1, - ] -} - -/// Decode PCR from 6 bytes (adaptation field) into 42-bit PCR base + 9-bit extension as u64 -#[inline] -fn decode_pcr(pcr_bytes: &[u8]) -> u64 { - let pcr_base = (u64::from(pcr_bytes[0]) << 25) - | ((u64::from(pcr_bytes[1])) << 17) - | ((u64::from(pcr_bytes[2])) << 9) - | ((u64::from(pcr_bytes[3])) << 1) - | ((u64::from(pcr_bytes[4])) >> 7); - let pcr_ext = ((u64::from(pcr_bytes[4]) & 1) << 8) | u64::from(pcr_bytes[5]); - pcr_base * 300 + pcr_ext -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum HlsTsTimestampFieldKind { - Pts, - Dts, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct HlsTsTimestampFieldLocation { - pub pid: u16, - pub kind: HlsTsTimestampFieldKind, - /// Absolute byte offsets in the immutable aligned TS buffer. One field may span packets. - pub byte_offsets: [usize; 5], -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct HlsTsPcrFieldLocation { - pub pid: u16, - pub byte_offset: usize, -} - -#[derive(Debug, thiserror::Error, Clone, Copy, PartialEq, Eq)] -enum HlsFiniteTsLayoutError { - #[error("transport stream asset is empty or invalid")] - InvalidAsset, - #[error("transport stream packet layout is invalid")] - InvalidTransportPacket, - #[error("transport stream timestamp location is invalid")] - InvalidTimestampLocation, - #[error("PES timestamp header for PID {pid} uses an unsupported layout")] - UnsupportedPesTimestampHeader { pid: u16 }, - #[error("PES {kind:?} field for PID {pid} is invalid")] - InvalidPesTimestampField { pid: u16, kind: HlsTsTimestampFieldKind }, - #[error("PES timestamp header for PID {pid} was interrupted by a new payload unit")] - PesTimestampHeaderInterrupted { pid: u16 }, - #[error("PES timestamp header continuity failed for PID {pid}: expected {expected:?}, actual {actual}")] - PesTimestampContinuityDiscontinuity { pid: u16, expected: Option, actual: u8 }, - #[error("PES timestamp header for PID {pid} is incomplete")] - IncompletePesTimestampHeader { pid: u16 }, - #[error("too many concurrent split PES timestamp headers")] - TooManyPendingPesTimestampHeaders, - #[error("transport stream contains no presentation clock")] - PresentationClockUnavailable, - #[error("PID {pid} has no defensible presentation cadence")] - PresentationCadenceUnavailable { pid: u16 }, - #[error("transport stream presentation duration overflows its clock domain")] - PresentationDurationOverflow, -} +use std::{collections::HashMap, sync::Arc}; #[derive(Clone, Debug)] pub struct HlsPendingPesHeader { @@ -156,19 +12,6 @@ pub struct HlsPendingPesHeader { last_payload_continuity_counter: u8, } -impl HlsPendingPesHeader { - fn new(pid: u16, continuity_counter: u8) -> Self { - Self { - pid, - bytes: [0; HLS_TS_TIMESTAMP_HEADER_BYTES], - byte_offsets: [0; HLS_TS_TIMESTAMP_HEADER_BYTES], - len: 0, - expected_len: None, - last_payload_continuity_counter: continuity_counter, - } - } -} - #[derive(Clone, Copy)] struct HlsCompletedTimestampField { location: HlsTsTimestampFieldLocation, @@ -190,267 +33,11 @@ struct HlsTsPacketEvidence { pcr_field: Option, } -impl HlsTsPacketEvidence { - const fn has_payload(self) -> bool { self.payload_offset.is_some() } -} - -fn inspect_hls_ts_packet(packet: &[u8], packet_start: usize) -> Result { - if packet.len() != TS_PACKET_SIZE || packet[0] != SYNC_BYTE || packet[1] & 0x80 != 0 || packet[3] & 0xC0 != 0 { - return Err(HlsFiniteTsLayoutError::InvalidTransportPacket); - } - let pid = ts_packet_pid(packet); - let adaptation_field_control = (packet[3] >> 4) & 0b11; - let continuity_counter = packet[3] & 0x0F; - let (payload_offset, adaptation_length) = match adaptation_field_control { - 0b01 => (Some(4), None), - 0b10 if packet[4] == 183 => (None, Some(183)), - 0b11 if packet[4] <= 182 => { - let adaptation_length = usize::from(packet[4]); - (Some(5usize.saturating_add(adaptation_length)), Some(adaptation_length)) - } - _ => return Err(HlsFiniteTsLayoutError::InvalidTransportPacket), - }; - let mut discontinuity = false; - let mut pcr_field = None; - if let Some(adaptation_length) = adaptation_length { - if adaptation_length > 0 { - discontinuity = packet[5] & 0x80 != 0; - if packet[5] & ADAPTATION_FIELD_FLAG_PCR != 0 { - if adaptation_length < 7 { - return Err(HlsFiniteTsLayoutError::InvalidTransportPacket); - } - pcr_field = Some(HlsTsPcrFieldLocation { - pid, - byte_offset: packet_start.checked_add(6).ok_or(HlsFiniteTsLayoutError::InvalidTransportPacket)?, - }); - } - } - } - Ok(HlsTsPacketEvidence { - pid, - payload_unit_start: packet[1] & 0x40 != 0, - payload_offset, - continuity_counter, - discontinuity, - pcr_field, - }) -} - -fn pes_stream_id_has_optional_header(stream_id: u8) -> bool { - !matches!( - stream_id, - 0xBC | // program_stream_map - 0xBE | // padding_stream - 0xBF | // private_stream_2 - 0xF0 | // ECM - 0xF1 | // EMM - 0xFF | // program_stream_directory - 0xF2 | // DSM-CC - 0xF8 // ITU-T Rec. H.222.1 type E - ) -} - -fn timestamp_field_from_pending( - pending: &HlsPendingPesHeader, - kind: HlsTsTimestampFieldKind, - start: usize, - prefix: u8, -) -> Result { - let end = - start.checked_add(5).ok_or(HlsFiniteTsLayoutError::InvalidPesTimestampField { pid: pending.pid, kind })?; - let bytes: [u8; 5] = pending - .bytes - .get(start..end) - .and_then(|bytes| bytes.try_into().ok()) - .ok_or(HlsFiniteTsLayoutError::InvalidPesTimestampField { pid: pending.pid, kind })?; - let byte_offsets: [usize; 5] = pending - .byte_offsets - .get(start..end) - .and_then(|offsets| offsets.try_into().ok()) - .ok_or(HlsFiniteTsLayoutError::InvalidPesTimestampField { pid: pending.pid, kind })?; - if bytes[0] >> 4 != prefix || bytes[0] & 1 == 0 || bytes[2] & 1 == 0 || bytes[4] & 1 == 0 { - return Err(HlsFiniteTsLayoutError::InvalidPesTimestampField { pid: pending.pid, kind }); - } - Ok(HlsCompletedTimestampField { - location: HlsTsTimestampFieldLocation { pid: pending.pid, kind, byte_offsets }, - bytes, - }) -} - -fn complete_pes_timestamps(pending: &HlsPendingPesHeader) -> Result { - let pts_dts_flags = (pending.bytes[7] >> 6) & 0b11; - match pts_dts_flags { - 0b00 => Ok(HlsCompletedPesTimestamps::default()), - 0b10 => Ok(HlsCompletedPesTimestamps { - fields: [ - Some(timestamp_field_from_pending(pending, HlsTsTimestampFieldKind::Pts, PES_PTS_OFFSET, 0x02)?), - None, - ], - }), - 0b11 => Ok(HlsCompletedPesTimestamps { - fields: [ - Some(timestamp_field_from_pending(pending, HlsTsTimestampFieldKind::Pts, PES_PTS_OFFSET, 0x03)?), - Some(timestamp_field_from_pending(pending, HlsTsTimestampFieldKind::Dts, PES_DTS_OFFSET, 0x01)?), - ], - }), - _ => Err(HlsFiniteTsLayoutError::UnsupportedPesTimestampHeader { pid: pending.pid }), - } -} - -fn append_pending_pes_header( - pending: &mut HlsPendingPesHeader, - payload: &[u8], - payload_start: usize, -) -> Result, HlsFiniteTsLayoutError> { - for (payload_index, byte) in payload.iter().copied().enumerate() { - if pending.expected_len.is_some_and(|expected| pending.len >= expected) { - break; - } - if pending.len >= HLS_TS_TIMESTAMP_HEADER_BYTES { - return Err(HlsFiniteTsLayoutError::UnsupportedPesTimestampHeader { pid: pending.pid }); - } - pending.bytes[pending.len] = byte; - pending.byte_offsets[pending.len] = - payload_start.checked_add(payload_index).ok_or(HlsFiniteTsLayoutError::InvalidTransportPacket)?; - pending.len = pending.len.saturating_add(1); - - if pending.len == 4 && !pes_stream_id_has_optional_header(pending.bytes[3]) { - pending.expected_len = Some(4); - } - if pending.len == 7 && (pending.bytes[6] & 0xC0) != 0x80 { - return Err(HlsFiniteTsLayoutError::UnsupportedPesTimestampHeader { pid: pending.pid }); - } - if pending.len == PES_PTS_OFFSET { - let header_data_length = usize::from(pending.bytes[8]); - pending.expected_len = match (pending.bytes[7] >> 6) & 0b11 { - 0b00 => Some(PES_PTS_OFFSET), - 0b10 if header_data_length >= 5 => Some(PES_PTS_OFFSET + 5), - 0b11 if header_data_length >= 10 => Some(PES_DTS_OFFSET + 5), - _ => return Err(HlsFiniteTsLayoutError::UnsupportedPesTimestampHeader { pid: pending.pid }), - }; - } - } - - if pending.expected_len.is_some_and(|expected| pending.len >= expected) { - return complete_pes_timestamps(pending).map(Some); - } - Ok(None) -} - #[derive(Debug)] struct HlsPesHeaderAssembler { pending: HashMap, } -impl HlsPesHeaderAssembler { - fn new() -> Self { Self { pending: HashMap::with_capacity(8) } } - - fn push_packet( - &mut self, - packet: &[u8], - packet_start: usize, - evidence: HlsTsPacketEvidence, - ) -> Result { - if evidence.payload_unit_start && self.pending.contains_key(&evidence.pid) { - return Err(HlsFiniteTsLayoutError::PesTimestampHeaderInterrupted { pid: evidence.pid }); - } - if evidence.discontinuity && self.pending.contains_key(&evidence.pid) { - return Err(HlsFiniteTsLayoutError::PesTimestampContinuityDiscontinuity { - pid: evidence.pid, - expected: None, - actual: evidence.continuity_counter, - }); - } - let Some(payload_offset) = evidence.payload_offset else { - return Ok(HlsCompletedPesTimestamps::default()); - }; - let payload = packet.get(payload_offset..).ok_or(HlsFiniteTsLayoutError::InvalidTransportPacket)?; - let payload_start = - packet_start.checked_add(payload_offset).ok_or(HlsFiniteTsLayoutError::InvalidTransportPacket)?; - - if let Some(mut pending) = self.pending.remove(&evidence.pid) { - let expected = pending.last_payload_continuity_counter.wrapping_add(1) & 0x0F; - if evidence.continuity_counter != expected { - return Err(HlsFiniteTsLayoutError::PesTimestampContinuityDiscontinuity { - pid: evidence.pid, - expected: Some(expected), - actual: evidence.continuity_counter, - }); - } - pending.last_payload_continuity_counter = evidence.continuity_counter; - if let Some(completed) = append_pending_pes_header(&mut pending, payload, payload_start)? { - return Ok(completed); - } - self.pending.insert(evidence.pid, pending); - return Ok(HlsCompletedPesTimestamps::default()); - } - - if !evidence.payload_unit_start || payload.len() < 3 || !payload.starts_with(&[0x00, 0x00, 0x01]) { - return Ok(HlsCompletedPesTimestamps::default()); - } - if self.pending.len() >= HLS_TS_MAX_PENDING_PES_HEADERS { - return Err(HlsFiniteTsLayoutError::TooManyPendingPesTimestampHeaders); - } - let mut pending = HlsPendingPesHeader::new(evidence.pid, evidence.continuity_counter); - if let Some(completed) = append_pending_pes_header(&mut pending, payload, payload_start)? { - return Ok(completed); - } - self.pending.insert(evidence.pid, pending); - Ok(HlsCompletedPesTimestamps::default()) - } - - fn finish(self) -> Result<(), HlsFiniteTsLayoutError> { - match self.pending.keys().copied().min() { - Some(pid) => Err(HlsFiniteTsLayoutError::IncompletePesTimestampHeader { pid }), - None => Ok(()), - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct HlsTsTimestampProfile { - pub first_clock_90khz: u64, - pub last_clock_90khz: u64, - pub span_ticks_90khz: u64, - pub observed_pts_or_dts: bool, - pub observed_pcr: bool, -} - -/// MPEG-TS live-to-terminal timestamp anchor expressed in 90 kHz clock ticks. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct HlsTsSpliceAnchor { - pub live_last_clock: u64, - pub terminal_first_clock: u64, - pub timestamp_delta_ticks: u64, -} - -impl HlsTsSpliceAnchor { - pub fn between(live_tail: HlsTsTimestampProfile, terminal_asset: HlsTsTimestampProfile) -> Option { - if !timestamp_profile_is_coherent(live_tail) || !timestamp_profile_is_coherent(terminal_asset) { - return None; - } - let terminal_first_clock_90khz = - add_pts_dts_offset(live_tail.last_clock_90khz, HLS_TS_SPLICE_MIN_GAP_TICKS_90KHZ); - let timestamp_delta_ticks_90khz = - terminal_first_clock_90khz.wrapping_add(MAX_PTS_DTS).wrapping_sub(terminal_asset.first_clock_90khz) - % MAX_PTS_DTS; - let rebased_first = add_pts_dts_offset(terminal_asset.first_clock_90khz, timestamp_delta_ticks_90khz); - (rebased_first == terminal_first_clock_90khz).then_some(Self { - live_last_clock: live_tail.last_clock_90khz, - terminal_first_clock: terminal_first_clock_90khz, - timestamp_delta_ticks: timestamp_delta_ticks_90khz, - }) - } -} - -fn timestamp_profile_is_coherent(profile: HlsTsTimestampProfile) -> bool { - (profile.observed_pts_or_dts || profile.observed_pcr) - && profile.first_clock_90khz < MAX_PTS_DTS - && profile.last_clock_90khz < MAX_PTS_DTS - && profile.span_ticks_90khz < MAX_PTS_DTS - && forward_clock_distance_90khz(profile.first_clock_90khz, profile.last_clock_90khz) == profile.span_ticks_90khz -} - #[derive(Debug)] struct HlsTsTimestampProfileAccumulator { reference_clock_90khz: Option, @@ -463,71 +50,6 @@ struct HlsTsTimestampProfileAccumulator { invalid: bool, } -impl HlsTsTimestampProfileAccumulator { - fn new(expected_duration_ticks_90khz: u64) -> Self { - let tolerance = (expected_duration_ticks_90khz / 2).max(HLS_TS_PROFILE_MIN_TOLERANCE_TICKS_90KHZ); - Self { - reference_clock_90khz: None, - earliest_relative_ticks: 0, - latest_relative_ticks: 0, - maximum_span_ticks_90khz: expected_duration_ticks_90khz.saturating_add(tolerance), - observed_pts_or_dts: false, - observed_pcr: false, - observation_count: 0, - invalid: expected_duration_ticks_90khz == 0, - } - } - - fn observe_clock(&mut self, clock_90khz: u64, kind: HlsTsTimestampKind) { - let clock_90khz = clock_90khz % MAX_PTS_DTS; - match kind { - HlsTsTimestampKind::PtsOrDts => self.observed_pts_or_dts = true, - HlsTsTimestampKind::Pcr => self.observed_pcr = true, - } - self.observation_count = self.observation_count.saturating_add(1); - let Some(reference) = self.reference_clock_90khz else { - self.reference_clock_90khz = Some(clock_90khz); - return; - }; - let forward = clock_90khz.wrapping_add(MAX_PTS_DTS).wrapping_sub(reference) % MAX_PTS_DTS; - let relative = if forward <= MAX_PTS_DTS / 2 { - i64::try_from(forward).unwrap_or(i64::MAX) - } else { - -i64::try_from(MAX_PTS_DTS.saturating_sub(forward)).unwrap_or(i64::MAX) - }; - self.earliest_relative_ticks = self.earliest_relative_ticks.min(relative); - self.latest_relative_ticks = self.latest_relative_ticks.max(relative); - let span = self.latest_relative_ticks.saturating_sub(self.earliest_relative_ticks); - if u64::try_from(span).unwrap_or(u64::MAX) > self.maximum_span_ticks_90khz { - self.invalid = true; - } - } - - fn finish(self) -> Option { - let reference = self.reference_clock_90khz?; - if self.invalid || self.observation_count < 2 || (!self.observed_pts_or_dts && !self.observed_pcr) { - return None; - } - let span_ticks_90khz = - u64::try_from(self.latest_relative_ticks.saturating_sub(self.earliest_relative_ticks)).ok()?; - if span_ticks_90khz == 0 { - return None; - } - let cycle = i128::from(MAX_PTS_DTS); - let first_clock_90khz = - u64::try_from((i128::from(reference) + i128::from(self.earliest_relative_ticks)).rem_euclid(cycle)).ok()?; - let last_clock_90khz = - u64::try_from((i128::from(reference) + i128::from(self.latest_relative_ticks)).rem_euclid(cycle)).ok()?; - Some(HlsTsTimestampProfile { - first_clock_90khz, - last_clock_90khz, - span_ticks_90khz, - observed_pts_or_dts: self.observed_pts_or_dts, - observed_pcr: self.observed_pcr, - }) - } -} - #[derive(Debug)] pub struct HlsTsTimestampProfileScanner { assembler: HlsPesHeaderAssembler, @@ -536,132 +58,6 @@ pub struct HlsTsTimestampProfileScanner { invalid: bool, } -impl HlsTsTimestampProfileScanner { - pub fn new(expected_duration_ticks_90khz: u64) -> Self { - Self { - assembler: HlsPesHeaderAssembler::new(), - accumulator: HlsTsTimestampProfileAccumulator::new(expected_duration_ticks_90khz), - next_packet_start: 0, - invalid: false, - } - } - - pub fn push_aligned_packet(&mut self, packet: &[u8]) { - if self.invalid { - return; - } - let packet_start = self.next_packet_start; - let Some(next_packet_start) = packet_start.checked_add(TS_PACKET_SIZE) else { - self.invalid = true; - return; - }; - self.next_packet_start = next_packet_start; - let Ok(evidence) = inspect_hls_ts_packet(packet, packet_start) else { - self.invalid = true; - return; - }; - if let Some(pcr) = evidence.pcr_field { - let Some(relative_offset) = pcr.byte_offset.checked_sub(packet_start) else { - self.invalid = true; - return; - }; - let Some(bytes) = packet.get(relative_offset..relative_offset.saturating_add(6)) else { - self.invalid = true; - return; - }; - self.accumulator.observe_clock(decode_pcr(bytes) / 300, HlsTsTimestampKind::Pcr); - } - let Ok(completed) = self.assembler.push_packet(packet, packet_start, evidence) else { - self.invalid = true; - return; - }; - for field in completed.fields.into_iter().flatten() { - self.accumulator.observe_clock(decode_timestamp(&field.bytes), HlsTsTimestampKind::PtsOrDts); - } - } - - pub fn finish(self) -> Option { - if self.invalid || self.assembler.finish().is_err() { - return None; - } - self.accumulator.finish() - } -} - -#[derive(Clone, Copy)] -enum HlsTsTimestampKind { - PtsOrDts, - Pcr, -} - -#[inline] -fn ts_packet_pid(packet: &[u8]) -> u16 { (u16::from(packet[1] & 0x1F) << 8) | u16::from(packet[2]) } - -fn same_finite_ts_packet_layout(source: &[u8], prepared: &[u8]) -> bool { - if source.len() != TS_PACKET_SIZE - || prepared.len() != TS_PACKET_SIZE - || source[0] != SYNC_BYTE - || prepared[0] != SYNC_BYTE - || source[1] & 0x7F != prepared[1] & 0x7F - || source[2] != prepared[2] - || source[3] & 0xF0 != prepared[3] & 0xF0 - { - return false; - } - let adaptation_field_control = (source[3] >> 4) & 0b11; - !matches!(adaptation_field_control, 0b10 | 0b11) || source[4] == prepared[4] -} - -fn append_finite_discontinuity_packet(following_packet: &[u8], output: &mut BytesMut) { - let following_cc = following_packet[3] & 0x0F; - let marker_cc = if following_packet[3] & 0x10 != 0 { following_cc.wrapping_sub(1) & 0x0F } else { following_cc }; - let start = output.len(); - output.resize(start.saturating_add(TS_PACKET_SIZE), 0xFF); - let marker = &mut output[start..start + TS_PACKET_SIZE]; - marker[0] = SYNC_BYTE; - marker[1] = following_packet[1] & 0x1F; - marker[2] = following_packet[2]; - // FFmpeg retains this adaptation-only marker as the per-PID CC baseline. - marker[3] = 0x20 | marker_cc; - marker[4] = 183; - marker[5] = 0x80; -} - -/// Encode PCR timestamp (u64) back into 6 bytes -#[inline] -#[allow(clippy::cast_possible_truncation)] -fn encode_pcr(pcr: u64) -> [u8; 6] { - let pcr_base = pcr / 300; - let pcr_ext = pcr % 300; - - [ - ((pcr_base >> 25) & 0xFF) as u8, - ((pcr_base >> 17) & 0xFF) as u8, - ((pcr_base >> 9) & 0xFF) as u8, - ((pcr_base >> 1) & 0xFF) as u8, - // Bit 7 = bit0 of pcr_base, Bits 6-1 reserved '111111', Bit 0 = high bit of pcr_ext - (((pcr_base & 1) << 7) as u8) | 0x7E | (((pcr_ext >> 8) & 1) as u8), - (pcr_ext & 0xFF) as u8, - ] -} - -/// Finds TS alignment by checking for 0x47 sync byte every 188 bytes -fn find_ts_alignment(buf: &[u8]) -> Option { - for offset in 0..TS_PACKET_SIZE { - let mut valid = true; - for i in 0..5 { - if buf.get(offset + i * TS_PACKET_SIZE) != Some(&SYNC_BYTE) { - valid = false; - break; - } - } - if valid { - return Some(offset); - } - } - None -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct HlsFiniteTsPacketLayout { packet_start: usize, @@ -680,342 +76,6 @@ pub struct HlsFiniteTsLayout { packet_timestamp_field_indices: Arc<[usize]>, } -fn build_hls_finite_ts_layout(buffer: &[u8]) -> Result { - if buffer.is_empty() || !buffer.len().is_multiple_of(TS_PACKET_SIZE) { - return Err(HlsFiniteTsLayoutError::InvalidAsset); - } - let packet_count = buffer.len() / TS_PACKET_SIZE; - let mut packet_evidence = Vec::with_capacity(packet_count); - let mut pcr_fields = Vec::new(); - let mut packet_pcr_indices = Vec::with_capacity(packet_count); - let mut timestamp_fields = Vec::new(); - let mut assembler = HlsPesHeaderAssembler::new(); - - for (packet_index, packet) in buffer.as_chunks::().0.iter().enumerate() { - let packet_start = - packet_index.checked_mul(TS_PACKET_SIZE).ok_or(HlsFiniteTsLayoutError::InvalidTransportPacket)?; - let evidence = inspect_hls_ts_packet(packet, packet_start)?; - let pcr_field_index = evidence.pcr_field.map(|field| { - let index = pcr_fields.len(); - pcr_fields.push(field); - index - }); - let completed = assembler.push_packet(packet, packet_start, evidence)?; - timestamp_fields.extend(completed.fields.into_iter().flatten().map(|field| field.location)); - packet_evidence.push(evidence); - packet_pcr_indices.push(pcr_field_index); - } - assembler.finish()?; - timestamp_fields.sort_unstable_by_key(|field| field.byte_offsets[0]); - - let mut timestamp_field_counts = vec![0usize; packet_count]; - for field in ×tamp_fields { - let mut previous_packet = None; - for offset in field.byte_offsets { - if offset >= buffer.len() { - return Err(HlsFiniteTsLayoutError::InvalidTimestampLocation); - } - let packet_index = offset / TS_PACKET_SIZE; - if previous_packet != Some(packet_index) { - timestamp_field_counts[packet_index] = timestamp_field_counts[packet_index] - .checked_add(1) - .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - previous_packet = Some(packet_index); - } - } - } - let mut timestamp_field_starts = Vec::with_capacity(packet_count.saturating_add(1)); - timestamp_field_starts.push(0usize); - for count in ×tamp_field_counts { - let next = timestamp_field_starts - .last() - .copied() - .and_then(|start| start.checked_add(*count)) - .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - timestamp_field_starts.push(next); - } - let timestamp_membership_count = timestamp_field_starts.last().copied().unwrap_or(0); - let mut packet_timestamp_field_indices = vec![0usize; timestamp_membership_count]; - let mut packet_cursors = timestamp_field_starts[..packet_count].to_vec(); - for (field_index, field) in timestamp_fields.iter().enumerate() { - let mut previous_packet = None; - for offset in field.byte_offsets { - let packet_index = offset / TS_PACKET_SIZE; - if previous_packet == Some(packet_index) { - continue; - } - let cursor = packet_cursors[packet_index]; - packet_timestamp_field_indices[cursor] = field_index; - packet_cursors[packet_index] = cursor.saturating_add(1); - previous_packet = Some(packet_index); - } - } - let packets = packet_evidence - .into_iter() - .enumerate() - .map(|(packet_index, evidence)| HlsFiniteTsPacketLayout { - packet_start: packet_index.saturating_mul(TS_PACKET_SIZE), - pid: evidence.pid, - has_payload: evidence.has_payload(), - timestamp_field_indices_start: timestamp_field_starts[packet_index], - timestamp_field_indices_end: timestamp_field_starts[packet_index.saturating_add(1)], - pcr_field_index: packet_pcr_indices[packet_index], - }) - .collect::>(); - Ok(HlsFiniteTsLayout { - packets: Arc::from(packets), - timestamp_fields: Arc::from(timestamp_fields), - pcr_fields: Arc::from(pcr_fields), - packet_timestamp_field_indices: Arc::from(packet_timestamp_field_indices), - }) -} - -fn gather_timestamp_bytes( - buffer: &[u8], - location: HlsTsTimestampFieldLocation, -) -> Result<[u8; 5], HlsFiniteTsLayoutError> { - let mut bytes = [0u8; 5]; - for (destination, offset) in bytes.iter_mut().zip(location.byte_offsets) { - *destination = buffer.get(offset).copied().ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - } - Ok(bytes) -} - -fn decode_timestamp_at_location( - buffer: &[u8], - location: HlsTsTimestampFieldLocation, -) -> Result { - gather_timestamp_bytes(buffer, location).map(|bytes| decode_timestamp(&bytes)) -} - -fn decode_pcr_at_location(buffer: &[u8], location: HlsTsPcrFieldLocation) -> Result { - let end = location.byte_offset.checked_add(6).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - let bytes = buffer.get(location.byte_offset..end).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - Ok(decode_pcr(bytes)) -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum HlsTsPresentationClockSource { - Pts, - PcrFallback, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct HlsTsPidPresentationTimeline { - pub pid: u16, - pub first_pts_90khz: u64, - pub last_pts_90khz: u64, - pub cadence_ticks_90khz: u64, - pub end_exclusive_90khz: u64, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct HlsTsPresentationDuration { - pub first_presentation_clock_90khz: u64, - pub end_exclusive_clock_90khz: u64, - pub duration_ticks_90khz: u64, - pub timelines: Arc<[HlsTsPidPresentationTimeline]>, - pub source: HlsTsPresentationClockSource, -} - -fn stable_presentation_cadence(mut deltas: Vec) -> Option { - if deltas.is_empty() { - return None; - } - deltas.sort_unstable(); - let mut dominant_delta = deltas[0]; - let mut dominant_count = 1usize; - let mut current_delta = deltas[0]; - let mut current_count = 1usize; - for delta in deltas.iter().copied().skip(1) { - if delta == current_delta { - current_count = current_count.saturating_add(1); - } else { - if current_count > dominant_count { - dominant_delta = current_delta; - dominant_count = current_count; - } - current_delta = delta; - current_count = 1; - } - } - if current_count > dominant_count { - dominant_delta = current_delta; - dominant_count = current_count; - } - if dominant_count.saturating_mul(2) > deltas.len() { - Some(dominant_delta) - } else { - deltas.get(deltas.len() / 2).copied() - } -} - -fn presentation_timeline_for_pid( - pid: u16, - timestamps: &[u64], -) -> Result { - let Some(first_raw) = timestamps.first().copied() else { - return Err(HlsFiniteTsLayoutError::PresentationCadenceUnavailable { pid }); - }; - let cycle = i128::from(MAX_PTS_DTS); - let mut previous_raw = first_raw % MAX_PTS_DTS; - let mut previous_unwrapped = i128::from(previous_raw); - let mut first_unwrapped = previous_unwrapped; - let mut last_unwrapped = previous_unwrapped; - let mut seen_raw_timestamps = HashSet::with_capacity(timestamps.len()); - seen_raw_timestamps.insert(previous_raw); - let mut deltas = Vec::with_capacity(timestamps.len().saturating_sub(1)); - - for raw in timestamps.iter().copied().skip(1).map(|value| value % MAX_PTS_DTS) { - if !seen_raw_timestamps.insert(raw) { - continue; - } - let forward = raw.wrapping_add(MAX_PTS_DTS).wrapping_sub(previous_raw) % MAX_PTS_DTS; - let signed_delta = if forward <= MAX_PTS_DTS / 2 { - i128::from(forward) - } else { - -i128::from(MAX_PTS_DTS.saturating_sub(forward)) - }; - let unwrapped = - previous_unwrapped.checked_add(signed_delta).ok_or(HlsFiniteTsLayoutError::PresentationDurationOverflow)?; - if unwrapped > previous_unwrapped { - deltas.push( - u64::try_from(unwrapped - previous_unwrapped) - .map_err(|_| HlsFiniteTsLayoutError::PresentationDurationOverflow)?, - ); - } - first_unwrapped = first_unwrapped.min(unwrapped); - last_unwrapped = last_unwrapped.max(unwrapped); - previous_raw = raw; - previous_unwrapped = unwrapped; - } - let cadence_ticks_90khz = stable_presentation_cadence(deltas) - .filter(|cadence| *cadence > 0 && *cadence < MAX_PTS_DTS / 2) - .ok_or(HlsFiniteTsLayoutError::PresentationCadenceUnavailable { pid })?; - let normalization = if first_unwrapped < 0 { - (-first_unwrapped) - .checked_add(cycle.saturating_sub(1)) - .and_then(|value| value.checked_div(cycle)) - .and_then(|cycles| cycles.checked_mul(cycle)) - .ok_or(HlsFiniteTsLayoutError::PresentationDurationOverflow)? - } else { - 0 - }; - let first_pts_90khz = u64::try_from( - first_unwrapped.checked_add(normalization).ok_or(HlsFiniteTsLayoutError::PresentationDurationOverflow)?, - ) - .map_err(|_| HlsFiniteTsLayoutError::PresentationDurationOverflow)?; - let last_pts_90khz = u64::try_from( - last_unwrapped.checked_add(normalization).ok_or(HlsFiniteTsLayoutError::PresentationDurationOverflow)?, - ) - .map_err(|_| HlsFiniteTsLayoutError::PresentationDurationOverflow)?; - let end_exclusive_90khz = - last_pts_90khz.checked_add(cadence_ticks_90khz).ok_or(HlsFiniteTsLayoutError::PresentationDurationOverflow)?; - Ok(HlsTsPidPresentationTimeline { pid, first_pts_90khz, last_pts_90khz, cadence_ticks_90khz, end_exclusive_90khz }) -} - -fn presentation_duration_from_pid_clocks( - clocks: HashMap>, - source: HlsTsPresentationClockSource, -) -> Result { - if clocks.is_empty() { - return Err(HlsFiniteTsLayoutError::PresentationClockUnavailable); - } - let mut timelines = clocks - .into_iter() - .map(|(pid, timestamps)| presentation_timeline_for_pid(pid, ×tamps)) - .collect::, _>>()?; - timelines.sort_unstable_by_key(|timeline| timeline.pid); - let first_presentation_clock_90khz = timelines - .iter() - .map(|timeline| timeline.first_pts_90khz) - .min() - .ok_or(HlsFiniteTsLayoutError::PresentationClockUnavailable)?; - let end_exclusive_clock_90khz = timelines - .iter() - .map(|timeline| timeline.end_exclusive_90khz) - .max() - .ok_or(HlsFiniteTsLayoutError::PresentationClockUnavailable)?; - let duration_ticks_90khz = end_exclusive_clock_90khz - .checked_sub(first_presentation_clock_90khz) - .filter(|duration| *duration > 0 && *duration < MAX_PTS_DTS) - .ok_or(HlsFiniteTsLayoutError::PresentationDurationOverflow)?; - Ok(HlsTsPresentationDuration { - first_presentation_clock_90khz, - end_exclusive_clock_90khz, - duration_ticks_90khz, - timelines: Arc::from(timelines), - source, - }) -} - -fn finite_hls_presentation_duration( - buffer: &[u8], - layout: &HlsFiniteTsLayout, -) -> Result { - let mut pts_by_pid = HashMap::>::new(); - for location in - layout.timestamp_fields.iter().copied().filter(|location| location.kind == HlsTsTimestampFieldKind::Pts) - { - pts_by_pid.entry(location.pid).or_default().push(decode_timestamp_at_location(buffer, location)?); - } - if !pts_by_pid.is_empty() { - return presentation_duration_from_pid_clocks(pts_by_pid, HlsTsPresentationClockSource::Pts); - } - - let mut pcr_by_pid = HashMap::>::new(); - for location in layout.pcr_fields.iter().copied() { - pcr_by_pid.entry(location.pid).or_default().push(decode_pcr_at_location(buffer, location)? / 300); - } - presentation_duration_from_pid_clocks(pcr_by_pid, HlsTsPresentationClockSource::PcrFallback) -} - -fn timestamp_profile_from_finite_layout( - buffer: &[u8], - layout: &HlsFiniteTsLayout, - expected_duration_ticks_90khz: u64, -) -> Option { - let mut accumulator = HlsTsTimestampProfileAccumulator::new(expected_duration_ticks_90khz); - for location in layout.timestamp_fields.iter().copied() { - accumulator.observe_clock(decode_timestamp_at_location(buffer, location).ok()?, HlsTsTimestampKind::PtsOrDts); - } - for location in layout.pcr_fields.iter().copied() { - accumulator.observe_clock(decode_pcr_at_location(buffer, location).ok()? / 300, HlsTsTimestampKind::Pcr); - } - accumulator.finish() -} - -fn ticks_90khz_to_rounded_millis(ticks: u64) -> Option { ticks.checked_add(45)?.checked_div(90) } - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct HlsFiniteTsRenderSpec { - pub timestamp_offset_ticks_90khz: u64, - /// Seed used by logical segment zero. Later segments derive each PID's starting counter from - /// the number of payload packets for that PID in one immutable asset cycle. - pub continuity_seed: u8, - pub logical_segment_index: u16, -} - -#[derive(Debug, thiserror::Error, Clone, Copy, PartialEq, Eq)] -pub enum HlsFiniteTsRenderError { - #[error("transport stream asset is empty or invalid")] - InvalidAsset, - #[error("prepared transport stream does not match the immutable asset packet layout")] - PreparedLayoutMismatch, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum HlsFiniteTsDiscontinuityMode { - None, - FirstPacketPerPid, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct HlsFiniteTsFinalizeSpec { - pub additional_timestamp_offset_ticks_90khz: u64, - pub discontinuity: HlsFiniteTsDiscontinuityMode, -} - pub struct TransportStreamBuffer { // `Bytes` instead of `Arc>`: cloning the inner payload (which // happens once per HLS-CVS fallback response) is then a refcount bump @@ -1044,1679 +104,23 @@ pub struct TransportStreamBuffer { force_discontinuity_on_wrap: bool, } -impl std::fmt::Debug for TransportStreamBuffer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TransportStreamBuffer") - .field("length", &self.length) - .field("current_pos", &self.current_pos) - .field("current_dts", &self.current_dts) - .field("timestamp_offset", &self.timestamp_offset) - .field("finite_hls_presentation_duration", &self.finite_hls_presentation_duration) - .field("first_pcr", &self.first_pcr) - .field("finite_hls_timestamp_profile", &self.finite_hls_timestamp_profile) - .field("force_discontinuity_on_wrap", &self.force_discontinuity_on_wrap) - .finish_non_exhaustive() - } -} - -impl Clone for TransportStreamBuffer { - fn clone(&self) -> Self { - Self { - buffer: self.buffer.clone(), - packet_starts: Arc::clone(&self.packet_starts), - finite_hls_layout: self.finite_hls_layout.clone(), - finite_hls_presentation_duration: self.finite_hls_presentation_duration.clone(), - current_pos: 0, - current_dts: 0, - timestamp_offset: 0, - length: self.length, - // Each clone starts with a fresh CC state; the discontinuity packets at the first - // loop boundary will signal decoders to reset their CC expectations. - cc_entries: Box::new([None; 8192]), - waker: Arc::clone(&self.waker), - first_pcr: self.first_pcr, - finite_hls_timestamp_profile: self.finite_hls_timestamp_profile, - finite_hls_track_signature: self.finite_hls_track_signature.clone(), - finite_hls_asset_fingerprint: self.finite_hls_asset_fingerprint, - #[cfg(any(test, feature = "test-support"))] - finite_hls_render_count: Arc::clone(&self.finite_hls_render_count), - #[cfg(any(test, feature = "test-support"))] - finite_hls_finalize_count: Arc::clone(&self.finite_hls_finalize_count), - // Start each clone with one initial discontinuity marker to keep - // continuity behavior consistent for fresh consumers. - force_discontinuity_on_wrap: true, - } - } -} - -impl TransportStreamBuffer { - pub fn new(mut raw: Vec) -> Self { - let offset = find_ts_alignment(&raw).unwrap_or(0); - raw.drain(..offset); - - // Remove trailing partial packets - let valid_length = (raw.len() / TS_PACKET_SIZE) * TS_PACKET_SIZE; - raw.truncate(valid_length); - - let length = raw.len() / TS_PACKET_SIZE; - let packet_starts = - (0..length).map(|packet_index| packet_index.saturating_mul(TS_PACKET_SIZE)).collect::>(); - let finite_hls_layout = build_hls_finite_ts_layout(&raw).map(Arc::new); - let finite_hls_presentation_duration = - finite_hls_layout.as_ref().ok().and_then(|layout| finite_hls_presentation_duration(&raw, layout).ok()); - let finite_hls_track_signature = match crate::ts_inspector::inspect_mpeg_ts( - std::io::Cursor::new(&raw), - crate::ts_inspector::HlsTsProbeProtection::Clear, - crate::ts_inspector::HlsTsProbeBudget::default(), - ) { - Ok(crate::ts_inspector::HlsTsProbeOutcome::Found(signature)) => Some(signature), - Ok( - crate::ts_inspector::HlsTsProbeOutcome::ProbeBudgetExhausted { .. } - | crate::ts_inspector::HlsTsProbeOutcome::Malformed(_) - | crate::ts_inspector::HlsTsProbeOutcome::UnsupportedProtection(_), - ) - | Err(_) => None, - }; - let finite_hls_asset_fingerprint = Sha256::digest(&raw).into(); - let finite_hls_timestamp_profile = finite_hls_layout.as_ref().ok().and_then(|layout| { - finite_hls_presentation_duration - .as_ref() - .and_then(|duration| timestamp_profile_from_finite_layout(&raw, layout, duration.duration_ticks_90khz)) - }); - let first_pcr = finite_hls_layout - .as_ref() - .ok() - .and_then(|layout| layout.pcr_fields.first().copied()) - .and_then(|location| decode_pcr_at_location(&raw, location).ok()); - - Self { - buffer: Bytes::from(raw), - current_pos: 0, - current_dts: 0, - timestamp_offset: 0, - length, - packet_starts, - finite_hls_layout, - finite_hls_presentation_duration, - cc_entries: Box::new([None; 8192]), - waker: Arc::new(AtomicWaker::new()), - first_pcr, - finite_hls_timestamp_profile, - finite_hls_track_signature, - finite_hls_asset_fingerprint, - #[cfg(any(test, feature = "test-support"))] - finite_hls_render_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)), - #[cfg(any(test, feature = "test-support"))] - finite_hls_finalize_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)), - // Emit a discontinuity packet on startup. Subsequent injections are - // governed by wrap handling and duration/discontinuity logic. - force_discontinuity_on_wrap: true, - } - } - - pub fn as_bytes(&self) -> &[u8] { &self.buffer } - - /// Cheap clone of the underlying buffer as `Bytes` (refcount bump). - /// Use this from response builders to avoid `Bytes::copy_from_slice(&[u8])`. - pub fn clone_bytes(&self) -> Bytes { self.buffer.clone() } - - pub fn duration_ms(&self) -> Option { self.duration_ticks_90khz().and_then(ticks_90khz_to_rounded_millis) } - - pub fn duration_ticks_90khz(&self) -> Option { - self.finite_hls_presentation_duration.as_ref().map(|duration| duration.duration_ticks_90khz) - } - - pub const fn finite_hls_timestamp_profile(&self) -> Option { - self.finite_hls_timestamp_profile - } - - pub fn finite_hls_track_signature(&self) -> Option { - self.finite_hls_track_signature.clone() - } - - pub const fn has_finite_hls_track_signature(&self) -> bool { self.finite_hls_track_signature.is_some() } - - pub const fn finite_hls_asset_fingerprint(&self) -> [u8; 32] { self.finite_hls_asset_fingerprint } - - #[cfg(any(test, feature = "test-support"))] - pub fn finite_hls_render_count(&self) -> usize { - self.finite_hls_render_count.load(std::sync::atomic::Ordering::Acquire) - } - - #[cfg(any(test, feature = "test-support"))] - pub fn finite_hls_finalize_count(&self) -> usize { - self.finite_hls_finalize_count.load(std::sync::atomic::Ordering::Acquire) - } - - /// Renders one immutable, finite HLS media segment without mutating the looping stream state. - pub fn render_finite_hls_segment(&self, spec: HlsFiniteTsRenderSpec) -> Result { - #[cfg(any(test, feature = "test-support"))] - self.finite_hls_render_count.fetch_add(1, std::sync::atomic::Ordering::AcqRel); - let layout = self.finite_hls_layout.as_ref().map_err(|_| HlsFiniteTsRenderError::InvalidAsset)?; - if self.buffer.is_empty() || layout.packets.is_empty() { - return Err(HlsFiniteTsRenderError::InvalidAsset); - } - let mut output = BytesMut::with_capacity(self.buffer.len()); - let mut payload_counts = [0_u8; 8192]; - for packet_layout in layout.packets.iter().copied() { - if packet_layout.has_payload { - let count = &mut payload_counts[usize::from(packet_layout.pid)]; - *count = count.wrapping_add(1) & 0x0F; - } - } - let mut continuity = [None; 8192]; - for packet_layout in layout.packets.iter().copied() { - let packet_end = packet_layout.packet_start.saturating_add(TS_PACKET_SIZE); - let Some(packet) = self.buffer.get(packet_layout.packet_start..packet_end) else { - return Err(HlsFiniteTsRenderError::InvalidAsset); - }; - let output_start = output.len(); - output.extend_from_slice(packet); - let counter = continuity[usize::from(packet_layout.pid)].get_or_insert_with(|| { - let cycle_advance = payload_counts[usize::from(packet_layout.pid)] - .wrapping_mul((spec.logical_segment_index & 0x0F) as u8) - & 0x0F; - spec.continuity_seed.wrapping_add(cycle_advance) & 0x0F - }); - output[output_start + 3] = (output[output_start + 3] & 0xF0) | *counter; - if packet_layout.has_payload { - *counter = counter.wrapping_add(1) & 0x0F; - } - } - Self::rewrite_layout_timestamps(&mut output, layout, spec.timestamp_offset_ticks_90khz) - .map_err(|_| HlsFiniteTsRenderError::InvalidAsset)?; - Ok(output.freeze()) - } - - /// Applies one lease-specific timestamp anchor to an already prepared relative segment. - /// - /// The immutable source packet layout is verified before any bytes are published. Optional - /// splice markers are adaptation-only packets and therefore do not advance payload CC. - pub fn finalize_prepared_finite_hls_segment( - &self, - prepared: &Bytes, - spec: HlsFiniteTsFinalizeSpec, - ) -> Result { - #[cfg(any(test, feature = "test-support"))] - self.finite_hls_finalize_count.fetch_add(1, std::sync::atomic::Ordering::AcqRel); - let layout = self.finite_hls_layout.as_ref().map_err(|_| HlsFiniteTsRenderError::PreparedLayoutMismatch)?; - if self.buffer.is_empty() - || layout.packets.is_empty() - || prepared.len() != layout.packets.len().saturating_mul(TS_PACKET_SIZE) - { - return Err(HlsFiniteTsRenderError::PreparedLayoutMismatch); - } - let mut first_pid_seen = [false; 8192]; - let mut marker_count = 0usize; - for (position, packet_layout) in layout.packets.iter().enumerate() { - let prepared_start = position.saturating_mul(TS_PACKET_SIZE); - let Some(source) = - self.buffer.get(packet_layout.packet_start..packet_layout.packet_start.saturating_add(TS_PACKET_SIZE)) - else { - return Err(HlsFiniteTsRenderError::PreparedLayoutMismatch); - }; - let Some(candidate) = prepared.get(prepared_start..prepared_start.saturating_add(TS_PACKET_SIZE)) else { - return Err(HlsFiniteTsRenderError::PreparedLayoutMismatch); - }; - if !same_finite_ts_packet_layout(source, candidate) { - return Err(HlsFiniteTsRenderError::PreparedLayoutMismatch); - } - let pid = ts_packet_pid(candidate); - if spec.discontinuity == HlsFiniteTsDiscontinuityMode::FirstPacketPerPid - && pid != NULL_PID - && !first_pid_seen[usize::from(pid)] - { - first_pid_seen[usize::from(pid)] = true; - marker_count = marker_count.saturating_add(1); - } - } - - let mut rewritten = BytesMut::from(prepared.as_ref()); - Self::rewrite_layout_timestamps(&mut rewritten, layout, spec.additional_timestamp_offset_ticks_90khz) - .map_err(|_| HlsFiniteTsRenderError::PreparedLayoutMismatch)?; - first_pid_seen.fill(false); - let additional_bytes = marker_count.saturating_mul(TS_PACKET_SIZE); - let mut output = BytesMut::with_capacity(prepared.len().saturating_add(additional_bytes)); - for position in 0..layout.packets.len() { - let prepared_start = position.saturating_mul(TS_PACKET_SIZE); - let packet = &rewritten[prepared_start..prepared_start + TS_PACKET_SIZE]; - let pid = ts_packet_pid(packet); - if spec.discontinuity == HlsFiniteTsDiscontinuityMode::FirstPacketPerPid - && pid != NULL_PID - && !first_pid_seen[usize::from(pid)] - { - append_finite_discontinuity_packet(packet, &mut output); - first_pid_seen[usize::from(pid)] = true; - } - output.extend_from_slice(packet); - } - Ok(output.freeze()) - } - - #[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation, clippy::cast_sign_loss)] - pub fn estimated_bitrate_kbps(&self) -> Option { - let stream_duration_90khz = self.duration_ticks_90khz()?; - if self.buffer.is_empty() { - return None; - } - let duration_secs = stream_duration_90khz as f64 / 90_000.0; - if duration_secs <= 0.0 { - return None; - } - let kbps = ((self.buffer.len() as f64 * 8.0) / duration_secs / 1_000.0).round(); - if !kbps.is_finite() || kbps <= 0.0 { - return None; - } - Some(kbps as usize) - } - - pub fn register_waker(&self, waker: &Waker) { self.waker.register(waker); } - - /// Sets the timestamp offset used when rewriting PTS/DTS/PCR values. - pub fn set_timestamp_offset(&mut self, offset: u64) { self.timestamp_offset = offset; } - - /// Generates a Discontinuity packet for the given packet/PID state, writing it directly into `out`. - fn generate_discontinuity_packet( - new_packet: &[u8], - cc: u8, - first_pcr: Option, - timestamp_offset: u64, - out: &mut BytesMut, - ) { - let start = out.len(); - out.resize(start + TS_PACKET_SIZE, 0xFF); - let pkt = &mut out[start..start + TS_PACKET_SIZE]; - - pkt[0] = SYNC_BYTE; - pkt[1] = new_packet[1] & 0x1F; - pkt[2] = new_packet[2]; - - // Check if the current packet has a PCR (need at least 7 adaptation bytes: 1 flag + 6 PCR). - let new_pkt_has_pcr = { - let afc = (new_packet[3] >> 4) & 0b11; - if afc == 2 || afc == 3 { - let adaptation_len = new_packet[4] as usize; - adaptation_len >= 7 && (new_packet[5] & ADAPTATION_FIELD_FLAG_PCR) != 0 - } else { - false - } - }; - - // AFC=2 (Adaptation Only), Scrambling=00 (Unscrambled), CC=cc - pkt[3] = 0x20 | (cc & 0x0F); - - // Adaptation Field covers rest of packet (183 bytes) - pkt[4] = 183; - - // If we contain a PCR, inject it. Otherwise just Discontinuity. - if new_pkt_has_pcr { - if let Some(base_pcr) = first_pcr { - pkt[5] = 0x80 | 0x10; // Discontinuity (0x80) + PCR Flag (0x10) - let offset_27mhz = pcr_offset_27mhz(timestamp_offset); - let new_pcr = add_pcr_offset_27mhz(base_pcr, offset_27mhz); - let pcr_bytes = encode_pcr(new_pcr); - pkt[6..12].copy_from_slice(&pcr_bytes); - } else { - pkt[5] = 0x80; - } - } else { - pkt[5] = 0x80; // Discontinuity Indicator Only - } - } - - fn rewrite_layout_timestamps( - bytes: &mut BytesMut, - layout: &HlsFiniteTsLayout, - timestamp_offset: u64, - ) -> Result<(), HlsFiniteTsLayoutError> { - if timestamp_offset == 0 { - return Ok(()); - } - for location in layout.pcr_fields.iter().copied() { - let original = decode_pcr_at_location(bytes, location)?; - let adjusted = add_pcr_offset_27mhz(original, pcr_offset_27mhz(timestamp_offset)); - let end = location.byte_offset.checked_add(6).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - let destination = - bytes.get_mut(location.byte_offset..end).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - destination.copy_from_slice(&encode_pcr(adjusted)); - } - for location in layout.timestamp_fields.iter().copied() { - let original = gather_timestamp_bytes(bytes, location)?; - let prefix = original[0] & 0xF0; - let mut encoded = encode_timestamp(add_pts_dts_offset(decode_timestamp(&original), timestamp_offset)); - encoded[0] = (encoded[0] & 0x0F) | prefix; - for (marker_index, (source_offset, value)) in location.byte_offsets.into_iter().zip(encoded).enumerate() { - let destination = - bytes.get_mut(source_offset).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - *destination = value; - if matches!(marker_index, 0 | 2 | 4) && value & 1 == 0 { - return Err(HlsFiniteTsLayoutError::InvalidPesTimestampField { - pid: location.pid, - kind: location.kind, - }); - } - } - } - Ok(()) - } - - fn rewrite_source_packet_timestamps( - &self, - bytes: &mut BytesMut, - output_start: usize, - layout: &HlsFiniteTsLayout, - packet_layout: HlsFiniteTsPacketLayout, - timestamp_offset: u64, - ) -> Result<(), HlsFiniteTsLayoutError> { - if timestamp_offset == 0 { - return Ok(()); - } - let packet_end = packet_layout.packet_start.saturating_add(TS_PACKET_SIZE); - if let Some(pcr_field_index) = packet_layout.pcr_field_index { - let location = layout - .pcr_fields - .get(pcr_field_index) - .copied() - .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - let original = decode_pcr_at_location(&self.buffer, location)?; - let adjusted = add_pcr_offset_27mhz(original, pcr_offset_27mhz(timestamp_offset)); - let relative_offset = location - .byte_offset - .checked_sub(packet_layout.packet_start) - .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - let destination_start = - output_start.checked_add(relative_offset).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - let destination_end = - destination_start.checked_add(6).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - bytes - .get_mut(destination_start..destination_end) - .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)? - .copy_from_slice(&encode_pcr(adjusted)); - } - let field_indices = layout - .packet_timestamp_field_indices - .get(packet_layout.timestamp_field_indices_start..packet_layout.timestamp_field_indices_end) - .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - for field_index in field_indices.iter().copied() { - let location = layout - .timestamp_fields - .get(field_index) - .copied() - .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - let original = gather_timestamp_bytes(&self.buffer, location)?; - let prefix = original[0] & 0xF0; - let mut encoded = encode_timestamp(add_pts_dts_offset(decode_timestamp(&original), timestamp_offset)); - encoded[0] = (encoded[0] & 0x0F) | prefix; - for (source_offset, value) in location.byte_offsets.into_iter().zip(encoded) { - if !(packet_layout.packet_start..packet_end).contains(&source_offset) { - continue; - } - let relative_offset = source_offset - .checked_sub(packet_layout.packet_start) - .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - let destination_offset = output_start - .checked_add(relative_offset) - .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; - *bytes.get_mut(destination_offset).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)? = value; - } - } - Ok(()) - } - - /// Returns next chunks with adjusted PTS/DTS and PCR. - /// All timestamp rewrites are performed in-place on the `BytesMut` output buffer to avoid - /// per-packet heap allocations. PID continuity-counter lookup is O(1) via a fixed 8192-entry array. - pub fn next_chunk(&mut self) -> Option { - if self.length == 0 { - return None; - } - let packet_count = ts_chunk_packet_count(); - let mut bytes = BytesMut::with_capacity(TS_PACKET_SIZE * packet_count); - let mut packets_remaining = packet_count; - - while packets_remaining > 0 { - if self.current_pos >= self.length { - // Loop back — advance timestamp offset so PTS/DTS/PCR remain monotonically - // increasing across loops. Resetting to 0 causes decoders (MPV, ffmpeg) to see - // a backward timestamp jump and treat the loop as end-of-stream or corrupt data. - self.current_pos = 0; - // Advance timestamps by one full source duration per loop so output time is - // monotonic for clients. Resetting to zero causes backward jumps that some - // players interpret as stream end/corruption after the first cycle. - if let Some(stream_duration_90khz) = self.duration_ticks_90khz() { - self.timestamp_offset = add_pts_dts_offset(self.timestamp_offset, stream_duration_90khz); - self.current_dts = add_pts_dts_offset(self.current_dts, stream_duration_90khz); - } else { - // PCR-only (or malformed) assets may not expose PTS/DTS-derived duration. - // Force one discontinuity marker after wrap so decoders do not see identical - // timestamp cycles as a continuous timeline. - self.force_discontinuity_on_wrap = true; - } - - // Reset only the discontinuity-sent flag so injection packets are emitted at the - // start of the next loop. Continuity counter values keep running so CC remains - // globally monotonic across loops. - for entry in self.cc_entries.iter_mut().flatten() { - entry.1 = false; - } - } - - let current_pos = self.current_pos; - let packet_start = self.packet_starts[current_pos]; - let packet = &self.buffer[packet_start..packet_start + TS_PACKET_SIZE]; - let packet_layout = - self.finite_hls_layout.as_ref().ok().and_then(|layout| layout.packets.get(current_pos)).copied(); - let packet_has_payload = packet_layout - .map_or_else(|| matches!((packet[3] >> 4) & 0b11, 0b01 | 0b11), |layout| layout.has_payload); - - // O(1) PID lookup — PID is at most 13 bits (0–8191). - let pid = (u16::from(packet[1] & 0x1F) << 8) | u16::from(packet[2]); - let entry = &mut self.cc_entries[pid as usize]; - // Normalize payload continuity per PID to a clean local sequence. - let (counter, discontinuity_sent) = entry.get_or_insert((0, false)); - - // Disable synthetic discontinuity packet insertion for looped custom streams. - // In practice this can produce demuxer corruption on some clients (PES mismatch). - // Monotonic timestamps + stable continuity counters are sufficient here. - let inject_discontinuity = self.force_discontinuity_on_wrap && !*discontinuity_sent; - if inject_discontinuity { - let extra_packet_cc = if packet_has_payload { *counter } else { packet[3] & 0x0F }; - Self::generate_discontinuity_packet( - packet, - extra_packet_cc, - self.first_pcr, - self.timestamp_offset, - &mut bytes, - ); - self.force_discontinuity_on_wrap = false; - } - *discontinuity_sent = true; - let payload_packet_cc = if packet_has_payload { *counter } else { packet[3] & 0x0F }; - - // TS continuity counter increments only when payload is present (AFC=01/11). - if packet_has_payload { - *counter = (*counter + 1) % 16; - } - - // Append the original packet into `bytes`, then mutate the appended slice in-place. - let pkt_start = bytes.len(); - bytes.extend_from_slice(packet); - - // Apply the computed CC to the payload packet. - bytes[pkt_start + 3] = (bytes[pkt_start + 3] & 0xF0) | (payload_packet_cc & 0x0F); - - if let (Ok(layout), Some(packet_layout)) = (&self.finite_hls_layout, packet_layout) { - if self - .rewrite_source_packet_timestamps( - &mut bytes, - pkt_start, - layout, - packet_layout, - self.timestamp_offset, - ) - .is_err() - { - return None; - } - } - - self.current_pos += 1; - packets_remaining -= 1; - } - - Some(bytes.freeze()) - } -} - #[cfg(test)] -mod tests { - use super::*; - - fn build_pts_dts_payload_packet(pid: u16, cc: u8, pts: u64, dts: u64) -> [u8; TS_PACKET_SIZE] { - let mut packet = [0xFF_u8; TS_PACKET_SIZE]; - packet[0] = SYNC_BYTE; - packet[1] = 0x40 | ((pid >> 8) as u8 & 0x1F); // PUSI + PID high bits - packet[2] = (pid & 0xFF) as u8; - packet[3] = 0x10 | (cc & 0x0F); // payload only - - let payload = &mut packet[4..]; - payload[0] = 0x00; - payload[1] = 0x00; - payload[2] = 0x01; - payload[3] = 0xE0; - payload[4] = 0x00; - payload[5] = 0x00; - payload[6] = 0x80; - payload[7] = 0xC0; // PTS + DTS present - payload[8] = 0x0A; - - let mut pts_bytes = encode_timestamp(pts); - pts_bytes[0] = (pts_bytes[0] & 0x0F) | 0x30; - payload[9..14].copy_from_slice(&pts_bytes); - - let mut dts_bytes = encode_timestamp(dts); - dts_bytes[0] = (dts_bytes[0] & 0x0F) | 0x10; - payload[14..19].copy_from_slice(&dts_bytes); - - packet - } - - fn build_pts_only_payload_packet(pid: u16, cc: u8, pts: u64) -> [u8; TS_PACKET_SIZE] { - let mut packet = [0xFF_u8; TS_PACKET_SIZE]; - packet[0] = SYNC_BYTE; - packet[1] = 0x40 | ((pid >> 8) as u8 & 0x1F); - packet[2] = (pid & 0xFF) as u8; - packet[3] = 0x10 | (cc & 0x0F); - let payload = &mut packet[4..]; - payload[..9].copy_from_slice(&[0x00, 0x00, 0x01, 0xE0, 0x00, 0x00, 0x80, 0x80, 0x05]); - payload[9..14].copy_from_slice(&encode_timestamp(pts)); - packet - } - - fn pts_only_pes_header(pts: u64) -> [u8; 14] { - let mut header = [0u8; 14]; - header[..9].copy_from_slice(&[0x00, 0x00, 0x01, 0xE0, 0x00, 0x00, 0x80, 0x80, 0x05]); - header[9..14].copy_from_slice(&encode_timestamp(pts)); - header - } - - fn pts_dts_pes_header(pts: u64, dts: u64) -> [u8; 19] { - let mut header = [0u8; 19]; - header[..9].copy_from_slice(&[0x00, 0x00, 0x01, 0xE0, 0x00, 0x00, 0x80, 0xC0, 0x0A]); - let mut pts_bytes = encode_timestamp(pts); - pts_bytes[0] = (pts_bytes[0] & 0x0F) | 0x30; - header[9..14].copy_from_slice(&pts_bytes); - let mut dts_bytes = encode_timestamp(dts); - dts_bytes[0] = (dts_bytes[0] & 0x0F) | 0x10; - header[14..19].copy_from_slice(&dts_bytes); - header - } - - fn build_split_pes_header_packets( - pid: u16, - first_cc: u8, - header: &[u8], - payload_lengths: &[usize], - ) -> Vec<[u8; TS_PACKET_SIZE]> { - let mut header_offset = 0usize; - payload_lengths - .iter() - .copied() - .enumerate() - .map(|(packet_index, payload_length)| { - assert!((1..=184).contains(&payload_length)); - let mut packet = [0xFF_u8; TS_PACKET_SIZE]; - let packet_index = u8::try_from(packet_index).expect("test packet index fits"); - packet[0] = SYNC_BYTE; - packet[1] = ((pid >> 8) as u8 & 0x1F) | (u8::from(packet_index == 0) * 0x40); - packet[2] = (pid & 0xFF) as u8; - packet[3] = if payload_length == 184 { - 0x10 | (first_cc.wrapping_add(packet_index) & 0x0F) - } else { - 0x30 | (first_cc.wrapping_add(packet_index) & 0x0F) - }; - let payload_offset = if payload_length == 184 { - 4 - } else { - let adaptation_length = 183usize.saturating_sub(payload_length); - packet[4] = u8::try_from(adaptation_length).expect("test adaptation length fits"); - packet[5] = 0; - 5 + adaptation_length - }; - let remaining = header.len().saturating_sub(header_offset); - let copied = remaining.min(payload_length); - packet[payload_offset..payload_offset + copied] - .copy_from_slice(&header[header_offset..header_offset + copied]); - header_offset = header_offset.saturating_add(copied); - packet - }) - .collect() - } - - fn build_pts_dts_pcr_packet(pid: u16, cc: u8, pts: u64, dts: u64, pcr_90khz: u64) -> [u8; TS_PACKET_SIZE] { - let mut packet = [0xFF_u8; TS_PACKET_SIZE]; - packet[0] = SYNC_BYTE; - packet[1] = 0x40 | ((pid >> 8) as u8 & 0x1F); - packet[2] = (pid & 0xFF) as u8; - packet[3] = 0x30 | (cc & 0x0F); - packet[4] = 7; - packet[5] = ADAPTATION_FIELD_FLAG_PCR; - packet[6..12].copy_from_slice(&encode_pcr(pcr_90khz.saturating_mul(300))); - - let payload = &mut packet[12..]; - payload[0] = 0x00; - payload[1] = 0x00; - payload[2] = 0x01; - payload[3] = 0xE0; - payload[4] = 0x00; - payload[5] = 0x00; - payload[6] = 0x80; - payload[7] = 0xC0; - payload[8] = 0x0A; - - let mut pts_bytes = encode_timestamp(pts); - pts_bytes[0] = (pts_bytes[0] & 0x0F) | 0x30; - payload[9..14].copy_from_slice(&pts_bytes); - let mut dts_bytes = encode_timestamp(dts); - dts_bytes[0] = (dts_bytes[0] & 0x0F) | 0x10; - payload[14..19].copy_from_slice(&dts_bytes); - packet - } - - fn packet_timestamps(packet: &[u8]) -> (u64, u64, u64) { - let layout = build_hls_finite_ts_layout(packet).expect("test packet layout"); - let pts = layout - .timestamp_fields - .iter() - .copied() - .find(|field| field.kind == HlsTsTimestampFieldKind::Pts) - .and_then(|field| decode_timestamp_at_location(packet, field).ok()) - .expect("test packet PTS"); - let dts = layout - .timestamp_fields - .iter() - .copied() - .find(|field| field.kind == HlsTsTimestampFieldKind::Dts) - .and_then(|field| decode_timestamp_at_location(packet, field).ok()) - .expect("test packet DTS"); - let pcr = layout - .pcr_fields - .first() - .copied() - .and_then(|field| decode_pcr_at_location(packet, field).ok()) - .expect("test packet PCR") - / 300; - (pts, dts, pcr) - } - - fn build_adaptation_only_packet(pid: u16, cc: u8) -> [u8; TS_PACKET_SIZE] { - let mut packet = [0xFF_u8; TS_PACKET_SIZE]; - packet[0] = SYNC_BYTE; - packet[1] = (pid >> 8) as u8 & 0x1F; - packet[2] = (pid & 0xFF) as u8; - packet[3] = 0x20 | (cc & 0x0F); // adaptation only - packet[4] = 183; - packet[5] = 0; - packet - } - - fn build_pcr_only_packet(pid: u16, cc: u8, pcr_90khz: u64) -> [u8; TS_PACKET_SIZE] { - let mut packet = build_adaptation_only_packet(pid, cc); - packet[5] = ADAPTATION_FIELD_FLAG_PCR; - packet[6..12].copy_from_slice(&encode_pcr(pcr_90khz.saturating_mul(300))); - packet - } - - fn build_payload_packet(pid: u16, cc: u8) -> [u8; TS_PACKET_SIZE] { - let mut packet = [0xFF_u8; TS_PACKET_SIZE]; - packet[0] = SYNC_BYTE; - packet[1] = (pid >> 8) as u8 & 0x1F; - packet[2] = (pid & 0xFF) as u8; - packet[3] = 0x10 | (cc & 0x0F); - packet - } - - fn assert_ffmpeg_compatible_continuity(bytes: &[u8]) { - assert_eq!(bytes.len() % TS_PACKET_SIZE, 0, "unaligned MPEG-TS fixture"); - let mut last_continuity: [Option; 8192] = [None; 8192]; - for (packet_index, packet) in bytes.as_chunks::().0.iter().enumerate() { - let evidence = inspect_hls_ts_packet(packet, packet_index.saturating_mul(TS_PACKET_SIZE)) - .expect("valid MPEG-TS fixture packet"); - if evidence.pid == NULL_PID { - continue; - } - if let Some(previous) = last_continuity[usize::from(evidence.pid)] { - let expected = if evidence.has_payload() { previous.wrapping_add(1) & 0x0F } else { previous }; - assert!( - evidence.discontinuity || evidence.continuity_counter == expected, - "PID {} continuity failed at packet {packet_index}: expected {expected}, actual {}", - evidence.pid, - evidence.continuity_counter - ); - } - last_continuity[usize::from(evidence.pid)] = Some(evidence.continuity_counter); - } - } - - #[test] - fn finite_discontinuity_marker_sets_ffmpeg_compatible_counter_baseline() { - let cases = [ - ("payload wrap", build_payload_packet(0x0100, 0), 15), - ("payload increment", build_payload_packet(0x0101, 7), 6), - ("adaptation only", build_adaptation_only_packet(0x0102, 5), 5), - ]; - for (name, following, expected_marker_cc) in cases { - let mut output = BytesMut::new(); - append_finite_discontinuity_packet(&following, &mut output); - let marker = &output[..TS_PACKET_SIZE]; - - assert_eq!((marker[3] >> 4) & 0b11, 0b10, "{name} marker AFC"); - assert_eq!(marker[5] & 0x80, 0x80, "{name} discontinuity"); - assert_eq!(marker[3] & 0x0F, expected_marker_cc, "{name} marker CC"); - - output.extend_from_slice(&following); - assert_ffmpeg_compatible_continuity(&output); - } - } - - #[test] - fn discontinuity_packet_does_not_advance_payload_cc() { - let packet = build_pts_dts_payload_packet(0x0100, 7, 90_000, 87_000); - let mut buf = TransportStreamBuffer::new(packet.to_vec()); - let chunk = buf.next_chunk().expect("expected chunk"); - assert!(chunk.len() >= TS_PACKET_SIZE * 2); - - // First emitted packet is injected discontinuity (adaptation-only), - // second is the actual payload packet for the same PID. - let disc_cc = chunk[3] & 0x0F; - let disc_afc = (chunk[3] >> 4) & 0b11; - let payload_cc = chunk[TS_PACKET_SIZE + 3] & 0x0F; - assert_eq!(disc_afc, 0b10); - assert_eq!(disc_cc, payload_cc); - } - - #[test] - fn adaptation_only_packets_keep_same_continuity_counter() { - let packet = build_adaptation_only_packet(0x0011, 5); - let mut buf = TransportStreamBuffer::new(packet.to_vec()); - let chunk = buf.next_chunk().expect("expected chunk"); - // Each of the 7 loop iterations emits a discontinuity packet + the actual packet - // because the single-packet buffer loops on every iteration. - let total_packets = PACKET_COUNT * 2; - assert_eq!(chunk.len(), TS_PACKET_SIZE * total_packets); - - for i in 0..total_packets { - let cc = chunk[i * TS_PACKET_SIZE + 3] & 0x0F; - assert_eq!(cc, 5, "packet {i} CC mismatch"); - } - } - - #[test] - fn runtime_custom_assets_use_exact_per_pid_presentation_duration() { - let fixtures: [(&str, &[u8]); 6] = [ - ( - "channel_unavailable.ts", - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) - .as_slice(), - ), - ( - "hls_session_or_lease_expired.ts", - include_bytes!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../test/fixtures/hls/hls_session_or_lease_expired.ts" - )) - .as_slice(), - ), - ( - "low_priority_preempted.ts", - include_bytes!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../test/fixtures/hls/low_priority_preempted.ts" - )) - .as_slice(), - ), - ( - "provider_connections_exhausted.ts", - include_bytes!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../test/fixtures/hls/provider_connections_exhausted.ts" - )) - .as_slice(), - ), - ( - "user_account_expired.ts", - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/user_account_expired.ts")) - .as_slice(), - ), - ( - "user_connections_exhausted.ts", - include_bytes!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../test/fixtures/hls/user_connections_exhausted.ts" - )) - .as_slice(), - ), - ]; - for (name, bytes) in fixtures { - let buffer = TransportStreamBuffer::new(bytes.to_vec()); - assert_eq!(buffer.duration_ticks_90khz(), Some(902_400), "{name} tick duration"); - assert_eq!(buffer.duration_ms(), Some(10_027), "{name} rounded HLS duration"); - } - } - - #[test] - fn provisioning_asset_uses_exact_per_pid_presentation_duration() { - let buffer = TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/panel_api_provisioning.ts")) - .to_vec(), - ); - - assert_eq!(buffer.duration_ticks_90khz(), Some(182_400)); - assert_eq!(buffer.duration_ms(), Some(2_027)); - } - - #[test] - fn custom_asset_stride_does_not_overlap_previous_audio_presentation() { - let buffer = TransportStreamBuffer::new( - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) - .to_vec(), - ); - let duration = buffer.duration_ticks_90khz().expect("exact custom asset duration"); - let segment_zero = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 0, - continuity_seed: 0, - logical_segment_index: 0, - }) - .expect("segment zero"); - let segment_one = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: duration, - continuity_seed: 0, - logical_segment_index: 1, - }) - .expect("segment one"); - let zero = TransportStreamBuffer::new(segment_zero.to_vec()); - let one = TransportStreamBuffer::new(segment_one.to_vec()); - let zero_audio = zero - .finite_hls_presentation_duration - .as_ref() - .and_then(|duration| duration.timelines.iter().find(|timeline| timeline.cadence_ticks_90khz == 1_920)) - .expect("segment zero audio timeline"); - let one_audio = one - .finite_hls_presentation_duration - .as_ref() - .and_then(|duration| duration.timelines.iter().find(|timeline| timeline.cadence_ticks_90khz == 1_920)) - .expect("segment one audio timeline"); - - assert_eq!(duration, 902_400); - assert_ne!(duration, 901_650); - assert!(one_audio.first_pts_90khz >= zero_audio.end_exclusive_90khz); - assert!( - zero_audio.first_pts_90khz.saturating_add(901_650) < zero_audio.end_exclusive_90khz, - "the previous mixed-clock stride would overlap the final audio presentation" - ); - } - - #[test] - fn finite_hls_segments_are_deterministic_aligned_and_timestamp_shifted() { - let first = build_pts_dts_payload_packet(0x0100, 7, 90_000, 87_000); - let second = build_pts_dts_payload_packet(0x0100, 8, 180_000, 177_000); - let mut raw = Vec::new(); - raw.extend_from_slice(&first); - raw.extend_from_slice(&second); - let buffer = TransportStreamBuffer::new(raw); - let spec = HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 900_000, - continuity_seed: 3, - logical_segment_index: 0, - }; - - let rendered = buffer.render_finite_hls_segment(spec).expect("finite render"); - let repeated = buffer.render_finite_hls_segment(spec).expect("repeated finite render"); - - assert_eq!(rendered, repeated); - assert_eq!(rendered.len() % TS_PACKET_SIZE, 0); - assert_eq!(rendered[3] & 0x0F, 3); - assert_eq!(rendered[TS_PACKET_SIZE + 3] & 0x0F, 4); - assert_eq!(decode_timestamp(&rendered[13..18]), 990_000); - assert_eq!(decode_timestamp(&rendered[18..23]), 987_000); - } - - #[test] - fn consecutive_finite_hls_segments_advance_cc_pts_dts_and_pcr() { - let first = build_pts_dts_pcr_packet(0x0100, 7, 90_000, 87_000, 90_000); - let second = build_pts_dts_pcr_packet(0x0100, 8, 180_000, 177_000, 180_000); - let mut raw = Vec::new(); - raw.extend_from_slice(&first); - raw.extend_from_slice(&second); - let buffer = TransportStreamBuffer::new(raw); - let duration = buffer.duration_ticks_90khz().expect("transportable duration"); - - let segment_zero = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 0, - continuity_seed: 3, - logical_segment_index: 0, - }) - .expect("first segment"); - let segment_one = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: duration, - continuity_seed: 3, - logical_segment_index: 1, - }) - .expect("second segment"); - - assert_eq!(segment_zero[TS_PACKET_SIZE + 3] & 0x0F, 4); - assert_eq!(segment_one[3] & 0x0F, 5); - assert_eq!(segment_one[TS_PACKET_SIZE + 3] & 0x0F, 6); - - let zero_last = packet_timestamps(&segment_zero[TS_PACKET_SIZE..]); - let one_first = packet_timestamps(&segment_one[..TS_PACKET_SIZE]); - assert!(one_first.0 > zero_last.0, "PTS must advance across segment boundary"); - assert!(one_first.1 > zero_last.1, "DTS must advance across segment boundary"); - assert!(one_first.2 > zero_last.2, "PCR must advance across segment boundary"); - } - - #[test] - fn finite_hls_timestamp_rewrite_wraps_pts_dts_and_pcr_without_overflow() { - let packet = build_pts_dts_pcr_packet( - 0x0100, - 7, - MAX_PTS_DTS.saturating_sub(10), - MAX_PTS_DTS.saturating_sub(20), - MAX_PTS_DTS.saturating_sub(10), - ); - let buffer = TransportStreamBuffer::new(packet.to_vec()); - - let rendered = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 30, - continuity_seed: 0, - logical_segment_index: 0, - }) - .expect("finite wrapped segment"); - - assert_eq!(packet_timestamps(&rendered), (20, 10, 20)); - } - - #[test] - fn split_pts_layout_profile_relative_and_anchored_rewrite_are_exact() { - let mut packets = build_split_pes_header_packets(0x0101, 0, &pts_only_pes_header(90_000), &[12, 2]); - packets.push(build_pts_only_payload_packet(0x0101, 2, 93_000)); - let raw = packets.into_iter().flatten().collect::>(); - let buffer = TransportStreamBuffer::new(raw); - let layout = buffer.finite_hls_layout.as_ref().expect("split PTS layout"); - let pts_fields = layout - .timestamp_fields - .iter() - .copied() - .filter(|field| field.kind == HlsTsTimestampFieldKind::Pts) - .collect::>(); - assert_eq!(pts_fields.len(), 2); - let split_field = pts_fields[0]; - assert_ne!(split_field.byte_offsets[0] / TS_PACKET_SIZE, split_field.byte_offsets[4] / TS_PACKET_SIZE); - assert_eq!( - buffer.finite_hls_timestamp_profile(), - Some(HlsTsTimestampProfile { - first_clock_90khz: 90_000, - last_clock_90khz: 93_000, - span_ticks_90khz: 3_000, - observed_pts_or_dts: true, - observed_pcr: false, - }) - ); - - let relative = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 7_000, - continuity_seed: 0, - logical_segment_index: 0, - }) - .expect("relative split PTS"); - assert_eq!(decode_timestamp_at_location(&relative, split_field).expect("relative split PTS value"), 97_000); - let anchored = buffer - .finalize_prepared_finite_hls_segment( - &relative, - HlsFiniteTsFinalizeSpec { - additional_timestamp_offset_ticks_90khz: 11_000, - discontinuity: HlsFiniteTsDiscontinuityMode::None, - }, - ) - .expect("anchored split PTS"); - assert_eq!(decode_timestamp_at_location(&anchored, split_field).expect("anchored split PTS value"), 108_000); - assert_ne!(decode_timestamp_at_location(&anchored, split_field).expect("rewritten split PTS"), 90_000); - let encoded = gather_timestamp_bytes(&anchored, split_field).expect("anchored split PTS bytes"); - assert!(encoded[0] & 1 != 0 && encoded[2] & 1 != 0 && encoded[4] & 1 != 0); - } - - #[test] - fn split_pts_dts_layout_rewrites_both_fields_across_distinct_packet_boundaries() { - let packets = build_split_pes_header_packets(0x0101, 5, &pts_dts_pes_header(90_000, 87_000), &[11, 5, 3]); - let raw = packets.into_iter().flatten().collect::>(); - let buffer = TransportStreamBuffer::new(raw); - let layout = buffer.finite_hls_layout.as_ref().expect("split PTS+DTS layout"); - let pts = layout - .timestamp_fields - .iter() - .copied() - .find(|field| field.kind == HlsTsTimestampFieldKind::Pts) - .expect("split PTS location"); - let dts = layout - .timestamp_fields - .iter() - .copied() - .find(|field| field.kind == HlsTsTimestampFieldKind::Dts) - .expect("split DTS location"); - assert_ne!(pts.byte_offsets[0] / TS_PACKET_SIZE, pts.byte_offsets[4] / TS_PACKET_SIZE); - assert_ne!(dts.byte_offsets[0] / TS_PACKET_SIZE, dts.byte_offsets[4] / TS_PACKET_SIZE); - assert_ne!(pts.byte_offsets[0] / TS_PACKET_SIZE, dts.byte_offsets[0] / TS_PACKET_SIZE); - - let relative = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 1_000, - continuity_seed: 0, - logical_segment_index: 0, - }) - .expect("relative split PTS+DTS"); - let anchored = buffer - .finalize_prepared_finite_hls_segment( - &relative, - HlsFiniteTsFinalizeSpec { - additional_timestamp_offset_ticks_90khz: 2_000, - discontinuity: HlsFiniteTsDiscontinuityMode::None, - }, - ) - .expect("anchored split PTS+DTS"); - assert_eq!(decode_timestamp_at_location(&anchored, pts), Ok(93_000)); - assert_eq!(decode_timestamp_at_location(&anchored, dts), Ok(90_000)); - for field in [pts, dts] { - let encoded = gather_timestamp_bytes(&anchored, field).expect("split timestamp bytes"); - assert!(encoded[0] & 1 != 0 && encoded[2] & 1 != 0 && encoded[4] & 1 != 0); - } - } - - #[test] - fn timestamp_profile_scanner_reads_split_pts_across_packets() { - let mut packets = build_split_pes_header_packets(0x0101, 0, &pts_only_pes_header(90_000), &[12, 2]); - packets.push(build_pts_only_payload_packet(0x0101, 2, 93_000)); - let mut scanner = HlsTsTimestampProfileScanner::new(10_000); - for packet in packets { - scanner.push_aligned_packet(&packet); - } - - assert_eq!( - scanner.finish(), - Some(HlsTsTimestampProfile { - first_clock_90khz: 90_000, - last_clock_90khz: 93_000, - span_ticks_90khz: 3_000, - observed_pts_or_dts: true, - observed_pcr: false, - }) - ); - } - - #[test] - fn timestamp_profile_scanner_rejects_one_clock_sample() { - let packet = build_pts_only_payload_packet(0x0101, 0, 90_000); - let mut scanner = HlsTsTimestampProfileScanner::new(10_000); - scanner.push_aligned_packet(&packet); - - assert_eq!(scanner.finish(), None); - } - - #[test] - fn split_pts_incomplete_at_eof_fails_with_typed_layout_error() { - let packets = build_split_pes_header_packets(0x0101, 0, &pts_only_pes_header(90_000), &[12, 2]); - let buffer = TransportStreamBuffer::new(packets[0].to_vec()); - - assert!(matches!( - buffer.finite_hls_layout.as_ref(), - Err(HlsFiniteTsLayoutError::IncompletePesTimestampHeader { pid: 0x0101 }) - )); - assert_eq!( - buffer.render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 1, - continuity_seed: 0, - logical_segment_index: 0, - }), - Err(HlsFiniteTsRenderError::InvalidAsset) - ); - assert_eq!(buffer.duration_ticks_90khz(), None); - } - - #[test] - fn split_pts_continuity_discontinuity_fails_with_typed_layout_error() { - let mut packets = build_split_pes_header_packets(0x0101, 0, &pts_only_pes_header(90_000), &[12, 2]); - packets[1][3] = (packets[1][3] & 0xF0) | 3; - let raw = packets.into_iter().flatten().collect::>(); - let buffer = TransportStreamBuffer::new(raw); - - assert!(matches!( - buffer.finite_hls_layout.as_ref(), - Err(HlsFiniteTsLayoutError::PesTimestampContinuityDiscontinuity { - pid: 0x0101, - expected: Some(1), - actual: 3, - }) - )); - } - - #[test] - fn per_pid_presentation_duration_unwraps_33_bit_audio_and_video_clocks() { - let video_pid = 0x0101; - let audio_pid = 0x0102; - let packets = [ - build_pts_only_payload_packet(video_pid, 0, MAX_PTS_DTS - 1_080), - build_pts_only_payload_packet(audio_pid, 0, MAX_PTS_DTS - 1_000), - build_pts_only_payload_packet(video_pid, 1, 1_920), - build_pts_only_payload_packet(audio_pid, 1, 920), - build_pts_only_payload_packet(video_pid, 2, 4_920), - build_pts_only_payload_packet(audio_pid, 2, 2_840), - ]; - let raw = packets.into_iter().flatten().collect::>(); - let buffer = TransportStreamBuffer::new(raw); - let duration = buffer.finite_hls_presentation_duration.as_ref().expect("per-PID wrapped duration"); - - assert_eq!(duration.source, HlsTsPresentationClockSource::Pts); - assert_eq!(duration.first_presentation_clock_90khz, MAX_PTS_DTS - 1_080); - assert_eq!(duration.end_exclusive_clock_90khz, MAX_PTS_DTS + 7_920); - assert_eq!(duration.duration_ticks_90khz, 9_000); - assert_eq!( - duration.timelines.as_ref(), - [ - HlsTsPidPresentationTimeline { - pid: video_pid, - first_pts_90khz: MAX_PTS_DTS - 1_080, - last_pts_90khz: MAX_PTS_DTS + 4_920, - cadence_ticks_90khz: 3_000, - end_exclusive_90khz: MAX_PTS_DTS + 7_920, - }, - HlsTsPidPresentationTimeline { - pid: audio_pid, - first_pts_90khz: MAX_PTS_DTS - 1_000, - last_pts_90khz: MAX_PTS_DTS + 2_840, - cadence_ticks_90khz: 1_920, - end_exclusive_90khz: MAX_PTS_DTS + 4_760, - }, - ] - ); - } - - #[test] - fn pcr_fallback_uses_observed_cadence_and_rejects_a_single_sample() { - let pid = 0x0101; - let packets = [build_pcr_only_packet(pid, 0, 90_000), build_pcr_only_packet(pid, 0, 91_920)]; - let buffer = TransportStreamBuffer::new(packets.into_iter().flatten().collect()); - let duration = buffer.finite_hls_presentation_duration.as_ref().expect("PCR fallback duration"); - - assert_eq!(duration.source, HlsTsPresentationClockSource::PcrFallback); - assert_eq!(duration.first_presentation_clock_90khz, 90_000); - assert_eq!(duration.end_exclusive_clock_90khz, 93_840); - assert_eq!(duration.duration_ticks_90khz, 3_840); - assert_eq!( - duration.timelines.as_ref(), - [HlsTsPidPresentationTimeline { - pid, - first_pts_90khz: 90_000, - last_pts_90khz: 91_920, - cadence_ticks_90khz: 1_920, - end_exclusive_90khz: 93_840, - }] - ); - - let single = TransportStreamBuffer::new(build_pcr_only_packet(pid, 0, 90_000).to_vec()); - assert_eq!(single.duration_ticks_90khz(), None); - } - - #[test] - fn timestamp_profile_scanner_unwraps_one_33_bit_boundary() { - let before_wrap = MAX_PTS_DTS - 1_000; - let packets = [ - build_pts_dts_pcr_packet(0x0100, 0, before_wrap + 200, before_wrap + 100, before_wrap), - build_pts_dts_pcr_packet(0x0100, 1, 1_200, 1_100, 1_000), - ]; - let mut scanner = HlsTsTimestampProfileScanner::new(5_000); - for packet in packets { - scanner.push_aligned_packet(&packet); - } - - assert_eq!( - scanner.finish(), - Some(HlsTsTimestampProfile { - first_clock_90khz: before_wrap, - last_clock_90khz: 1_200, - span_ticks_90khz: 2_200, - observed_pts_or_dts: true, - observed_pcr: true, - }) - ); - } - - #[test] - fn timestamp_profile_scanner_rejects_implausible_backward_segment_jump() { - let packets = [ - build_pts_dts_pcr_packet(0x0100, 0, 500_200, 500_100, 500_000), - build_pts_dts_pcr_packet(0x0100, 1, 100_200, 100_100, 100_000), - ]; - let mut scanner = HlsTsTimestampProfileScanner::new(90_000); - for packet in packets { - scanner.push_aligned_packet(&packet); - } - - assert_eq!(scanner.finish(), None); - } - - #[test] - fn terminal_splice_anchor_places_first_terminal_clock_after_live_tail() { - let live_last_clock_90khz = 1_003_618_800; - let live = HlsTsTimestampProfile { - first_clock_90khz: live_last_clock_90khz - 900_000, - last_clock_90khz: live_last_clock_90khz, - span_ticks_90khz: 900_000, - observed_pts_or_dts: true, - observed_pcr: true, - }; - let terminal = HlsTsTimestampProfile { - first_clock_90khz: 0, - last_clock_90khz: 902_400, - span_ticks_90khz: 902_400, - observed_pts_or_dts: true, - observed_pcr: true, - }; - - let anchor = HlsTsSpliceAnchor::between(live, terminal).expect("valid modular splice"); - - assert_eq!( - forward_clock_distance_90khz(live_last_clock_90khz, anchor.terminal_first_clock), - HLS_TS_SPLICE_MIN_GAP_TICKS_90KHZ - ); - assert_eq!(anchor.timestamp_delta_ticks, anchor.terminal_first_clock); - assert!( - forward_clock_distance_90khz(live_last_clock_90khz, anchor.terminal_first_clock) < 90_000, - "splice must not contain the observed near-full-cycle jump" - ); - } - - #[test] - fn terminal_splice_preserves_terminal_asset_audio_video_offset() { - let audio = build_pts_dts_payload_packet(0x0102, 0, 0, 0); - let video = build_pts_dts_payload_packet(0x0101, 0, 1_920, 1_920); - let mut raw = Vec::new(); - raw.extend_from_slice(&audio); - raw.extend_from_slice(&video); - raw.extend_from_slice(&build_pts_dts_payload_packet(0x0102, 1, 1_920, 1_920)); - raw.extend_from_slice(&build_pts_dts_payload_packet(0x0101, 1, 4_920, 4_920)); - let buffer = TransportStreamBuffer::new(raw); - let asset_profile = buffer.finite_hls_timestamp_profile().expect("asset timestamp profile"); - let live = HlsTsTimestampProfile { - first_clock_90khz: 1_003_000_000, - last_clock_90khz: 1_003_618_800, - span_ticks_90khz: 618_800, - observed_pts_or_dts: true, - observed_pcr: false, - }; - let anchor = HlsTsSpliceAnchor::between(live, asset_profile).expect("splice anchor"); - let prepared = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 0, - continuity_seed: 0, - logical_segment_index: 0, - }) - .expect("relative segment"); - let anchored = buffer - .finalize_prepared_finite_hls_segment( - &prepared, - HlsFiniteTsFinalizeSpec { - additional_timestamp_offset_ticks_90khz: anchor.timestamp_delta_ticks, - discontinuity: HlsFiniteTsDiscontinuityMode::None, - }, - ) - .expect("anchored segment"); - let audio_pts = decode_timestamp(&anchored[13..18]); - let video_pts = decode_timestamp(&anchored[TS_PACKET_SIZE + 13..TS_PACKET_SIZE + 18]); - - assert_eq!(forward_clock_distance_90khz(audio_pts, video_pts), 1_920); - } - - #[test] - fn terminal_splice_anchor_wraps_pts_dts_and_pcr_exactly() { - let original_clock = 10; - let packet = build_pts_dts_pcr_packet(0x0100, 7, original_clock, original_clock, original_clock); - let next_packet = build_pts_dts_pcr_packet(0x0100, 8, 3_010, 3_010, 3_010); - let mut raw = packet.to_vec(); - raw.extend_from_slice(&next_packet); - let buffer = TransportStreamBuffer::new(raw); - let asset_profile = buffer.finite_hls_timestamp_profile().expect("asset timestamp profile"); - let live_last_clock_90khz = MAX_PTS_DTS - 40; - let live = HlsTsTimestampProfile { - first_clock_90khz: live_last_clock_90khz - 90_000, - last_clock_90khz: live_last_clock_90khz, - span_ticks_90khz: 90_000, - observed_pts_or_dts: true, - observed_pcr: true, - }; - let anchor = HlsTsSpliceAnchor::between(live, asset_profile).expect("wrapped splice anchor"); - let prepared = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 0, - continuity_seed: 0, - logical_segment_index: 0, - }) - .expect("relative segment"); - let anchored = buffer - .finalize_prepared_finite_hls_segment( - &prepared, - HlsFiniteTsFinalizeSpec { - additional_timestamp_offset_ticks_90khz: anchor.timestamp_delta_ticks, - discontinuity: HlsFiniteTsDiscontinuityMode::None, - }, - ) - .expect("wrapped anchored segment"); - - assert_eq!(anchor.terminal_first_clock, 50); - assert_eq!(packet_timestamps(&anchored), (50, 50, 50)); - assert_eq!(forward_clock_distance_90khz(live_last_clock_90khz, 50), 90); - } - - #[test] - fn first_terminal_segment_marks_discontinuity_for_each_non_null_pid() { - let mut raw = Vec::new(); - raw.extend_from_slice(&build_payload_packet(0x0000, 9)); - raw.extend_from_slice(&build_payload_packet(0x0100, 3)); - raw.extend_from_slice(&build_pts_dts_payload_packet(0x0101, 7, 90_000, 87_000)); - raw.extend_from_slice(&build_pts_dts_payload_packet(0x0102, 5, 90_000, 90_000)); - raw.extend_from_slice(&build_pts_dts_pcr_packet(0x0103, 4, 90_000, 90_000, 90_000)); - raw.extend_from_slice(&build_payload_packet(NULL_PID, 12)); - let buffer = TransportStreamBuffer::new(raw); - let prepared = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 0, - continuity_seed: 2, - logical_segment_index: 0, - }) - .expect("relative segment"); - let finalized = buffer - .finalize_prepared_finite_hls_segment( - &prepared, - HlsFiniteTsFinalizeSpec { - additional_timestamp_offset_ticks_90khz: 90, - discontinuity: HlsFiniteTsDiscontinuityMode::FirstPacketPerPid, - }, - ) - .expect("splice segment"); - - let packets = finalized.as_chunks::().0.iter().collect::>(); - let mut marked_pids = Vec::new(); - for pair in packets.windows(2) { - let marker = pair[0]; - let following = pair[1]; - if (marker[3] >> 4) & 0b11 == 0b10 && marker[4] == 183 && marker[5] & 0x80 != 0 { - assert_eq!(ts_packet_pid(marker), ts_packet_pid(following)); - assert_eq!((marker[3] & 0x0F).wrapping_add(1) & 0x0F, following[3] & 0x0F); - marked_pids.push(ts_packet_pid(marker)); - } - } - marked_pids.sort_unstable(); - - assert_eq!(marked_pids, vec![0x0000, 0x0100, 0x0101, 0x0102, 0x0103]); - assert!(!marked_pids.contains(&NULL_PID)); - } - - #[test] - fn finalized_terminal_asset_preserves_tracks_and_ffmpeg_continuity() { - const TERMINAL_ASSET_BYTES: &[u8] = - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); - let buffer = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); - let expected_signature = buffer.finite_hls_track_signature().expect("terminal asset track signature"); - let prepared = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 0, - continuity_seed: 0, - logical_segment_index: 0, - }) - .expect("relative terminal asset"); - let finalized = buffer - .finalize_prepared_finite_hls_segment( - &prepared, - HlsFiniteTsFinalizeSpec { - additional_timestamp_offset_ticks_90khz: 1_003_618_890, - discontinuity: HlsFiniteTsDiscontinuityMode::FirstPacketPerPid, - }, - ) - .expect("anchored terminal asset"); - - assert_ffmpeg_compatible_continuity(&finalized); - let outcome = crate::ts_inspector::inspect_mpeg_ts( - std::io::Cursor::new(finalized), - crate::ts_inspector::HlsTsProbeProtection::Clear, - crate::ts_inspector::HlsTsProbeBudget::default(), - ) - .expect("finalized terminal asset remains inspectable"); - assert_eq!(outcome, crate::ts_inspector::HlsTsProbeOutcome::Found(expected_signature)); - } - - #[test] - fn prepared_terminal_finalization_rejects_packet_layout_mismatch() { - let packet = build_pts_dts_pcr_packet(0x0100, 0, 90_000, 87_000, 86_000); - let buffer = TransportStreamBuffer::new(packet.to_vec()); - let prepared = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 0, - continuity_seed: 0, - logical_segment_index: 0, - }) - .expect("relative terminal segment"); - let mut mismatched = BytesMut::from(prepared.as_ref()); - mismatched[2] ^= 1; - - assert_eq!( - buffer.finalize_prepared_finite_hls_segment( - &mismatched.freeze(), - HlsFiniteTsFinalizeSpec { - additional_timestamp_offset_ticks_90khz: 1, - discontinuity: HlsFiniteTsDiscontinuityMode::FirstPacketPerPid, - }, - ), - Err(HlsFiniteTsRenderError::PreparedLayoutMismatch) - ); - } - - #[test] - fn later_terminal_segments_continue_cc_without_repeating_splice_marker() { - let first = build_pts_dts_pcr_packet(0x0101, 7, 90_000, 87_000, 90_000); - let second = build_pts_dts_payload_packet(0x0102, 5, 90_000, 90_000); - let mut raw = Vec::new(); - raw.extend_from_slice(&first); - raw.extend_from_slice(&second); - raw.extend_from_slice(&build_pts_dts_pcr_packet(0x0101, 8, 180_000, 177_000, 180_000)); - raw.extend_from_slice(&build_pts_dts_payload_packet(0x0102, 6, 180_000, 180_000)); - let buffer = TransportStreamBuffer::new(raw); - let duration = buffer.duration_ticks_90khz().expect("asset duration"); - let relative_zero = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 0, - continuity_seed: 0, - logical_segment_index: 0, - }) - .expect("relative zero"); - let relative_one = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: duration, - continuity_seed: 0, - logical_segment_index: 1, - }) - .expect("relative one"); - let finalized_zero = buffer - .finalize_prepared_finite_hls_segment( - &relative_zero, - HlsFiniteTsFinalizeSpec { - additional_timestamp_offset_ticks_90khz: 1_000_000, - discontinuity: HlsFiniteTsDiscontinuityMode::FirstPacketPerPid, - }, - ) - .expect("anchored zero"); - let finalized_one = buffer - .finalize_prepared_finite_hls_segment( - &relative_one, - HlsFiniteTsFinalizeSpec { - additional_timestamp_offset_ticks_90khz: 1_000_000, - discontinuity: HlsFiniteTsDiscontinuityMode::None, - }, - ) - .expect("anchored one"); - - assert_eq!(finalized_one.len(), relative_one.len()); - assert!(finalized_one - .as_chunks::() - .0 - .iter() - .all(|packet| !((packet[3] >> 4) & 0b11 == 0b10 && packet[4] == 183 && packet[5] & 0x80 != 0))); - let zero_payload = finalized_zero - .as_chunks::() - .0 - .iter() - .filter(|packet| (packet[3] >> 4) & 0b11 != 0b10) - .collect::>(); - let one_payload = finalized_one.as_chunks::().0.iter().collect::>(); - for (zero, one) in zero_payload.into_iter().zip(one_payload) { - assert_eq!(ts_packet_pid(zero), ts_packet_pid(one)); - assert_eq!((zero[3] & 0x0F).wrapping_add(2) & 0x0F, one[3] & 0x0F); - } - let mut concatenated = finalized_zero.to_vec(); - concatenated.extend_from_slice(&finalized_one); - assert_ffmpeg_compatible_continuity(&concatenated); - } - - #[test] - fn live_to_terminal_splice_has_monotone_pts_dts_pcr_and_valid_cc_reset() { - let live_first_clock = 1_003_528_800; - let live_last_clock = 1_003_618_800; - let mut live_bytes = Vec::new(); - live_bytes.extend_from_slice(&build_pts_dts_pcr_packet( - 0x0101, - 13, - live_first_clock, - live_first_clock, - live_first_clock, - )); - live_bytes.extend_from_slice(&build_pts_dts_pcr_packet( - 0x0101, - 14, - live_last_clock, - live_last_clock, - live_last_clock, - )); - let live_buffer = TransportStreamBuffer::new(live_bytes.clone()); - let live_profile = live_buffer.finite_hls_timestamp_profile().expect("live tail profile"); - - let mut terminal_bytes = Vec::new(); - terminal_bytes.extend_from_slice(&build_pts_dts_pcr_packet(0x0101, 0, 0, 0, 0)); - terminal_bytes.extend_from_slice(&build_pts_dts_pcr_packet(0x0101, 1, 90_000, 90_000, 90_000)); - let terminal_buffer = TransportStreamBuffer::new(terminal_bytes); - let terminal_profile = terminal_buffer.finite_hls_timestamp_profile().expect("terminal asset profile"); - let anchor = HlsTsSpliceAnchor::between(live_profile, terminal_profile).expect("splice anchor"); - let duration = terminal_buffer.duration_ticks_90khz().expect("terminal duration"); - let relative_zero = terminal_buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 0, - continuity_seed: 0, - logical_segment_index: 0, - }) - .expect("terminal zero"); - let relative_one = terminal_buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: duration, - continuity_seed: 0, - logical_segment_index: 1, - }) - .expect("terminal one"); - let terminal_zero = terminal_buffer - .finalize_prepared_finite_hls_segment( - &relative_zero, - HlsFiniteTsFinalizeSpec { - additional_timestamp_offset_ticks_90khz: anchor.timestamp_delta_ticks, - discontinuity: HlsFiniteTsDiscontinuityMode::FirstPacketPerPid, - }, - ) - .expect("anchored terminal zero"); - let terminal_one = terminal_buffer - .finalize_prepared_finite_hls_segment( - &relative_one, - HlsFiniteTsFinalizeSpec { - additional_timestamp_offset_ticks_90khz: anchor.timestamp_delta_ticks, - discontinuity: HlsFiniteTsDiscontinuityMode::None, - }, - ) - .expect("anchored terminal one"); - - let zero_packets = terminal_zero.as_chunks::().0.iter().collect::>(); - assert_eq!((zero_packets[0][3] >> 4) & 0b11, 0b10); - assert_eq!(zero_packets[0][5] & 0x80, 0x80); - assert_eq!((zero_packets[0][3] & 0x0F).wrapping_add(1) & 0x0F, zero_packets[1][3] & 0x0F); - let terminal_one_packets = terminal_one.as_chunks::().0; - assert!(terminal_one_packets.iter().all(|packet| !((packet[3] >> 4) & 0b11 == 0b10 && packet[5] & 0x80 != 0))); - - let zero_first = packet_timestamps(zero_packets[1]); - let zero_last = packet_timestamps(zero_packets[2]); - let one_first = packet_timestamps(&terminal_one[..TS_PACKET_SIZE]); - assert_eq!(forward_clock_distance_90khz(live_last_clock, zero_first.0), 90); - assert!(forward_clock_distance_90khz(zero_first.0, zero_last.0) < MAX_PTS_DTS / 2); - assert!(forward_clock_distance_90khz(zero_last.0, one_first.0) < MAX_PTS_DTS / 2); - assert_eq!(forward_clock_distance_90khz(zero_last.0, one_first.0), duration.saturating_sub(90_000)); - assert_eq!(zero_first.0, zero_first.1); - assert_eq!(zero_first.0, zero_first.2); - assert_eq!(one_first.0, one_first.1); - assert_eq!(one_first.0, one_first.2); - - let mut concatenated = live_bytes; - concatenated.extend_from_slice(&terminal_zero); - concatenated.extend_from_slice(&terminal_one); - assert_ffmpeg_compatible_continuity(&concatenated); - let mut scanner = HlsTsTimestampProfileScanner::new(duration.saturating_mul(3)); - for packet in concatenated.as_chunks::().0 { - scanner.push_aligned_packet(packet); - } - let combined = scanner.finish().expect("combined splice profile"); - assert!(combined.span_ticks_90khz < duration.saturating_mul(3)); - } - - #[test] - fn hls_prepared_terminal_bundle_pcr_uses_300_factor_and_large_offsets_wrap_exactly() { - let base_pcr_90khz = 123_456; - let large_offset_90khz = u64::MAX - 37; - let packet = build_pts_dts_pcr_packet(0x0100, 7, 90_000, 87_000, base_pcr_90khz); - let buffer = TransportStreamBuffer::new(packet.to_vec()); - - let rendered = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: large_offset_90khz, - continuity_seed: 0, - logical_segment_index: 11, - }) - .expect("finite segment with a large timestamp offset"); - - let actual_pcr_27mhz = decode_pcr(&rendered[6..12]); - let expected_offset_27mhz = (u128::from(large_offset_90khz) * 300_u128) % u128::from(MAX_PCR); - let expected_pcr_27mhz = - u64::try_from((u128::from(base_pcr_90khz) * 300_u128 + expected_offset_27mhz) % u128::from(MAX_PCR)) - .expect("PCR modulo fits in u64"); - let legacy_wrapped_offset = large_offset_90khz.wrapping_mul(300) % MAX_PCR; - - assert_eq!(pcr_offset_27mhz(1), 300); - assert_ne!( - pcr_offset_27mhz(large_offset_90khz), - legacy_wrapped_offset, - "large offsets must be multiplied before modulo without u64 wrap" - ); - assert_eq!(actual_pcr_27mhz, expected_pcr_27mhz); - } - - #[test] - fn hls_prepared_terminal_bundle_pts_and_dts_large_offsets_wrap_modulo_33_bits() { - let presentation_timestamp = MAX_PTS_DTS - 123; - let decoding_timestamp = MAX_PTS_DTS - 456; - let large_offset_90khz = u64::MAX - 37; - let packet = build_pts_dts_pcr_packet(0x0100, 7, presentation_timestamp, decoding_timestamp, 90_000); - let buffer = TransportStreamBuffer::new(packet.to_vec()); - - let rendered = buffer - .render_finite_hls_segment(HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: large_offset_90khz, - continuity_seed: 0, - logical_segment_index: 11, - }) - .expect("finite segment with wrapped PTS and DTS"); - - let expected_timestamp = |timestamp| { - u64::try_from((u128::from(timestamp) + u128::from(large_offset_90khz)) % u128::from(MAX_PTS_DTS)) - .expect("PTS/DTS modulo fits in u64") - }; - let (rendered_presentation_timestamp, rendered_decoding_timestamp, _) = packet_timestamps(&rendered); - - assert_eq!(rendered_presentation_timestamp, expected_timestamp(presentation_timestamp)); - assert_eq!(rendered_decoding_timestamp, expected_timestamp(decoding_timestamp)); - } -} +mod tests; + +mod buffer; +mod clock; +mod layout; +mod pes; +mod profile; +mod render; +use clock::HLS_TS_TIMESTAMP_HEADER_BYTES; +#[cfg(test)] +use clock::{MAX_PCR, PACKET_COUNT}; +pub use layout::{HlsTsPidPresentationTimeline, HlsTsPresentationClockSource, HlsTsPresentationDuration}; +use pes::HlsFiniteTsLayoutError; +pub use pes::{HlsTsPcrFieldLocation, HlsTsTimestampFieldKind, HlsTsTimestampFieldLocation}; +pub use profile::{HlsTsSpliceAnchor, HlsTsTimestampProfile}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use render::{ + HlsFiniteTsDiscontinuityMode, HlsFiniteTsFinalizeSpec, HlsFiniteTsRenderError, HlsFiniteTsRenderSpec, +}; diff --git a/backend/mpegts/src/transport_stream_buffer/buffer.rs b/backend/mpegts/src/transport_stream_buffer/buffer.rs new file mode 100644 index 000000000..9be7b754a --- /dev/null +++ b/backend/mpegts/src/transport_stream_buffer/buffer.rs @@ -0,0 +1,262 @@ +use super::{ + clock::{ + add_pts_dts_offset, decode_pcr_at_location, ticks_90khz_to_rounded_millis, ts_chunk_packet_count, + TS_PACKET_SIZE, + }, + layout::{build_hls_finite_ts_layout, find_ts_alignment, finite_hls_presentation_duration}, + profile::timestamp_profile_from_finite_layout, + HlsTsTimestampProfile, TransportStreamBuffer, +}; +use bytes::{Bytes, BytesMut}; +use futures::task::AtomicWaker; +use sha2::{Digest, Sha256}; +use std::sync::Arc; + +impl std::fmt::Debug for TransportStreamBuffer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TransportStreamBuffer") + .field("length", &self.length) + .field("current_pos", &self.current_pos) + .field("current_dts", &self.current_dts) + .field("timestamp_offset", &self.timestamp_offset) + .field("finite_hls_presentation_duration", &self.finite_hls_presentation_duration) + .field("first_pcr", &self.first_pcr) + .field("finite_hls_timestamp_profile", &self.finite_hls_timestamp_profile) + .field("force_discontinuity_on_wrap", &self.force_discontinuity_on_wrap) + .finish_non_exhaustive() + } +} + +impl Clone for TransportStreamBuffer { + fn clone(&self) -> Self { + Self { + buffer: self.buffer.clone(), + packet_starts: Arc::clone(&self.packet_starts), + finite_hls_layout: self.finite_hls_layout.clone(), + finite_hls_presentation_duration: self.finite_hls_presentation_duration.clone(), + current_pos: 0, + current_dts: 0, + timestamp_offset: 0, + length: self.length, + // Each clone starts with a fresh CC state; the discontinuity packets at the first + // loop boundary will signal decoders to reset their CC expectations. + cc_entries: Box::new([None; 8192]), + waker: Arc::clone(&self.waker), + first_pcr: self.first_pcr, + finite_hls_timestamp_profile: self.finite_hls_timestamp_profile, + finite_hls_track_signature: self.finite_hls_track_signature.clone(), + finite_hls_asset_fingerprint: self.finite_hls_asset_fingerprint, + #[cfg(any(test, feature = "test-support"))] + finite_hls_render_count: Arc::clone(&self.finite_hls_render_count), + #[cfg(any(test, feature = "test-support"))] + finite_hls_finalize_count: Arc::clone(&self.finite_hls_finalize_count), + // Start each clone with one initial discontinuity marker to keep + // continuity behavior consistent for fresh consumers. + force_discontinuity_on_wrap: true, + } + } +} + +impl TransportStreamBuffer { + pub fn new(mut raw: Vec) -> Self { + let offset = find_ts_alignment(&raw).unwrap_or(0); + raw.drain(..offset); + + // Remove trailing partial packets + let valid_length = (raw.len() / TS_PACKET_SIZE) * TS_PACKET_SIZE; + raw.truncate(valid_length); + + let length = raw.len() / TS_PACKET_SIZE; + let packet_starts = + (0..length).map(|packet_index| packet_index.saturating_mul(TS_PACKET_SIZE)).collect::>(); + let finite_hls_layout = build_hls_finite_ts_layout(&raw).map(Arc::new); + let finite_hls_presentation_duration = + finite_hls_layout.as_ref().ok().and_then(|layout| finite_hls_presentation_duration(&raw, layout).ok()); + let finite_hls_track_signature = match crate::ts_inspector::inspect_mpeg_ts( + std::io::Cursor::new(&raw), + crate::ts_inspector::HlsTsProbeProtection::Clear, + crate::ts_inspector::HlsTsProbeBudget::default(), + ) { + Ok(crate::ts_inspector::HlsTsProbeOutcome::Found(signature)) => Some(signature), + Ok( + crate::ts_inspector::HlsTsProbeOutcome::ProbeBudgetExhausted { .. } + | crate::ts_inspector::HlsTsProbeOutcome::Malformed(_) + | crate::ts_inspector::HlsTsProbeOutcome::UnsupportedProtection(_), + ) + | Err(_) => None, + }; + let finite_hls_asset_fingerprint = Sha256::digest(&raw).into(); + let finite_hls_timestamp_profile = finite_hls_layout.as_ref().ok().and_then(|layout| { + finite_hls_presentation_duration + .as_ref() + .and_then(|duration| timestamp_profile_from_finite_layout(&raw, layout, duration.duration_ticks_90khz)) + }); + let first_pcr = finite_hls_layout + .as_ref() + .ok() + .and_then(|layout| layout.pcr_fields.first().copied()) + .and_then(|location| decode_pcr_at_location(&raw, location).ok()); + + Self { + buffer: Bytes::from(raw), + current_pos: 0, + current_dts: 0, + timestamp_offset: 0, + length, + packet_starts, + finite_hls_layout, + finite_hls_presentation_duration, + cc_entries: Box::new([None; 8192]), + waker: Arc::new(AtomicWaker::new()), + first_pcr, + finite_hls_timestamp_profile, + finite_hls_track_signature, + finite_hls_asset_fingerprint, + #[cfg(any(test, feature = "test-support"))] + finite_hls_render_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + #[cfg(any(test, feature = "test-support"))] + finite_hls_finalize_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + // Emit a discontinuity packet on startup. Subsequent injections are + // governed by wrap handling and duration/discontinuity logic. + force_discontinuity_on_wrap: true, + } + } + + pub fn as_bytes(&self) -> &[u8] { &self.buffer } + + /// Cheap clone of the underlying buffer as `Bytes` (refcount bump). + /// Use this from response builders to avoid `Bytes::copy_from_slice(&[u8])`. + pub fn clone_bytes(&self) -> Bytes { self.buffer.clone() } + + pub fn duration_ms(&self) -> Option { self.duration_ticks_90khz().and_then(ticks_90khz_to_rounded_millis) } + + pub fn duration_ticks_90khz(&self) -> Option { + self.finite_hls_presentation_duration.as_ref().map(|duration| duration.duration_ticks_90khz) + } + + pub const fn finite_hls_timestamp_profile(&self) -> Option { + self.finite_hls_timestamp_profile + } + + pub fn finite_hls_track_signature(&self) -> Option { + self.finite_hls_track_signature.clone() + } + + pub const fn has_finite_hls_track_signature(&self) -> bool { self.finite_hls_track_signature.is_some() } + + pub const fn finite_hls_asset_fingerprint(&self) -> [u8; 32] { self.finite_hls_asset_fingerprint } + + #[cfg(any(test, feature = "test-support"))] + pub fn finite_hls_render_count(&self) -> usize { + self.finite_hls_render_count.load(std::sync::atomic::Ordering::Acquire) + } + + #[cfg(any(test, feature = "test-support"))] + pub fn finite_hls_finalize_count(&self) -> usize { + self.finite_hls_finalize_count.load(std::sync::atomic::Ordering::Acquire) + } + + /// Returns next chunks with adjusted PTS/DTS and PCR. + /// All timestamp rewrites are performed in-place on the `BytesMut` output buffer to avoid + /// per-packet heap allocations. PID continuity-counter lookup is O(1) via a fixed 8192-entry array. + pub fn next_chunk(&mut self) -> Option { + if self.length == 0 { + return None; + } + let packet_count = ts_chunk_packet_count(); + let mut bytes = BytesMut::with_capacity(TS_PACKET_SIZE * packet_count); + let mut packets_remaining = packet_count; + + while packets_remaining > 0 { + if self.current_pos >= self.length { + // Loop back — advance timestamp offset so PTS/DTS/PCR remain monotonically + // increasing across loops. Resetting to 0 causes decoders (MPV, ffmpeg) to see + // a backward timestamp jump and treat the loop as end-of-stream or corrupt data. + self.current_pos = 0; + // Advance timestamps by one full source duration per loop so output time is + // monotonic for clients. Resetting to zero causes backward jumps that some + // players interpret as stream end/corruption after the first cycle. + if let Some(stream_duration_90khz) = self.duration_ticks_90khz() { + self.timestamp_offset = add_pts_dts_offset(self.timestamp_offset, stream_duration_90khz); + self.current_dts = add_pts_dts_offset(self.current_dts, stream_duration_90khz); + } else { + // PCR-only (or malformed) assets may not expose PTS/DTS-derived duration. + // Force one discontinuity marker after wrap so decoders do not see identical + // timestamp cycles as a continuous timeline. + self.force_discontinuity_on_wrap = true; + } + + // Reset only the discontinuity-sent flag so injection packets are emitted at the + // start of the next loop. Continuity counter values keep running so CC remains + // globally monotonic across loops. + for entry in self.cc_entries.iter_mut().flatten() { + entry.1 = false; + } + } + + let current_pos = self.current_pos; + let packet_start = self.packet_starts[current_pos]; + let packet = &self.buffer[packet_start..packet_start + TS_PACKET_SIZE]; + let packet_layout = + self.finite_hls_layout.as_ref().ok().and_then(|layout| layout.packets.get(current_pos)).copied(); + let packet_has_payload = packet_layout + .map_or_else(|| matches!((packet[3] >> 4) & 0b11, 0b01 | 0b11), |layout| layout.has_payload); + + // O(1) PID lookup — PID is at most 13 bits (0–8191). + let pid = (u16::from(packet[1] & 0x1F) << 8) | u16::from(packet[2]); + let entry = &mut self.cc_entries[pid as usize]; + // Normalize payload continuity per PID to a clean local sequence. + let (counter, discontinuity_sent) = entry.get_or_insert((0, false)); + + // Disable synthetic discontinuity packet insertion for looped custom streams. + // In practice this can produce demuxer corruption on some clients (PES mismatch). + // Monotonic timestamps + stable continuity counters are sufficient here. + let inject_discontinuity = self.force_discontinuity_on_wrap && !*discontinuity_sent; + if inject_discontinuity { + let extra_packet_cc = if packet_has_payload { *counter } else { packet[3] & 0x0F }; + Self::generate_discontinuity_packet( + packet, + extra_packet_cc, + self.first_pcr, + self.timestamp_offset, + &mut bytes, + ); + self.force_discontinuity_on_wrap = false; + } + *discontinuity_sent = true; + let payload_packet_cc = if packet_has_payload { *counter } else { packet[3] & 0x0F }; + + // TS continuity counter increments only when payload is present (AFC=01/11). + if packet_has_payload { + *counter = (*counter + 1) % 16; + } + + // Append the original packet into `bytes`, then mutate the appended slice in-place. + let pkt_start = bytes.len(); + bytes.extend_from_slice(packet); + + // Apply the computed CC to the payload packet. + bytes[pkt_start + 3] = (bytes[pkt_start + 3] & 0xF0) | (payload_packet_cc & 0x0F); + + if let (Ok(layout), Some(packet_layout)) = (&self.finite_hls_layout, packet_layout) { + if self + .rewrite_source_packet_timestamps( + &mut bytes, + pkt_start, + layout, + packet_layout, + self.timestamp_offset, + ) + .is_err() + { + return None; + } + } + + self.current_pos += 1; + packets_remaining -= 1; + } + + Some(bytes.freeze()) + } +} diff --git a/backend/mpegts/src/transport_stream_buffer/clock.rs b/backend/mpegts/src/transport_stream_buffer/clock.rs new file mode 100644 index 000000000..5d795da2d --- /dev/null +++ b/backend/mpegts/src/transport_stream_buffer/clock.rs @@ -0,0 +1,194 @@ +use super::{pes::HlsFiniteTsLayoutError, HlsTsPcrFieldLocation, HlsTsTimestampFieldLocation}; +use bytes::BytesMut; + +// PCR wraps at 2^33 * 300 (base is 33-bit, multiplied by 300 to get 27 MHz units). +// Using 1<<42 was slightly too large and could cause strict-decoder issues on hardware +// that computes modulo 2^33 on the base before multiplying. +pub(super) const MAX_PCR: u64 = (1u64 << 33) * 300; + +pub(super) const MAX_PTS_DTS: u64 = 1 << 33; + +// 33 bit PTS/DTS cycle +pub(super) const HLS_TS_SPLICE_MIN_GAP_TICKS_90KHZ: u64 = 90; + +pub(super) const HLS_TS_PROFILE_MIN_TOLERANCE_TICKS_90KHZ: u64 = 2 * 90_000; + +#[inline] +#[allow(clippy::cast_possible_truncation)] +pub(super) fn add_pts_dts_offset(timestamp: u64, offset_90khz: u64) -> u64 { + ((u128::from(timestamp) + u128::from(offset_90khz)) % u128::from(MAX_PTS_DTS)) as u64 +} + +#[inline] +pub(super) fn forward_clock_distance_90khz(start: u64, end: u64) -> u64 { + end.wrapping_add(MAX_PTS_DTS).wrapping_sub(start) % MAX_PTS_DTS +} + +#[inline] +#[allow(clippy::cast_possible_truncation)] +pub(super) fn pcr_offset_27mhz(offset_90khz: u64) -> u64 { + (u128::from(offset_90khz) * 300_u128 % u128::from(MAX_PCR)) as u64 +} + +#[inline] +#[allow(clippy::cast_possible_truncation)] +pub(super) fn add_pcr_offset_27mhz(timestamp_27mhz: u64, offset_27mhz: u64) -> u64 { + ((u128::from(timestamp_27mhz) + u128::from(offset_27mhz)) % u128::from(MAX_PCR)) as u64 +} + +pub(super) const TS_PACKET_SIZE: usize = 188; + +pub(super) const SYNC_BYTE: u8 = 0x47; + +pub(super) const PACKET_COUNT: usize = 7; + +// Reduced from 250 to 7 (1316 bytes) to prevent latency/timeout on low-bitrate streams +const MAX_PACKET_COUNT: usize = 250; + +/// Packets per emitted chunk; overridable via `TULIPROX_TS_CHUNK_PACKETS` (1-250). +/// Larger chunks raise throughput, smaller chunks lower latency on low-bitrate streams. +pub(super) fn ts_chunk_packet_count() -> usize { + static PACKET_COUNT_OVERRIDE: std::sync::LazyLock = std::sync::LazyLock::new(|| { + std::env::var("TULIPROX_TS_CHUNK_PACKETS") + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|count| (1..=MAX_PACKET_COUNT).contains(count)) + .unwrap_or(PACKET_COUNT) + }); + *PACKET_COUNT_OVERRIDE +} + +pub(super) const ADAPTATION_FIELD_FLAG_PCR: u8 = 0x10; + +// PCR flag bit in adaptation field flags +pub(super) const NULL_PID: u16 = 0x1FFF; + +pub(super) const HLS_TS_MAX_PENDING_PES_HEADERS: usize = 64; + +pub(super) const HLS_TS_TIMESTAMP_HEADER_BYTES: usize = 19; + +/// Byte offset of PTS within a PES payload (after the 3-byte start code, `stream_id`, length, flags). +pub(super) const PES_PTS_OFFSET: usize = 9; + +/// Byte offset of DTS within a PES payload when both PTS and DTS are present. +pub(super) const PES_DTS_OFFSET: usize = 14; + +/// Decodes a 5-byte DTS/PTS field from PES header into u64 timestamp. +#[inline] +pub(super) fn decode_timestamp(ts_bytes: &[u8]) -> u64 { + (((u64::from(ts_bytes[0]) >> 1) & 0x07) << 30) + | (u64::from(ts_bytes[1]) << 22) + | (((u64::from(ts_bytes[2]) >> 1) & 0x7F) << 15) + | (u64::from(ts_bytes[3]) << 7) + | ((u64::from(ts_bytes[4]) >> 1) & 0x7F) +} + +/// Encodes a u64 timestamp into 5-byte PES DTS/PTS field +#[inline] +pub(super) fn encode_timestamp(ts: u64) -> [u8; 5] { + [ + 0x20 | ((((ts >> 30) & 0x07) as u8) << 1) | 1, + ((ts >> 22) & 0xFF) as u8, + ((((ts >> 15) & 0x7F) as u8) << 1) | 1, + ((ts >> 7) & 0xFF) as u8, + (((ts & 0x7F) as u8) << 1) | 1, + ] +} + +/// Decode PCR from 6 bytes (adaptation field) into 42-bit PCR base + 9-bit extension as u64 +#[inline] +pub(super) fn decode_pcr(pcr_bytes: &[u8]) -> u64 { + let pcr_base = (u64::from(pcr_bytes[0]) << 25) + | ((u64::from(pcr_bytes[1])) << 17) + | ((u64::from(pcr_bytes[2])) << 9) + | ((u64::from(pcr_bytes[3])) << 1) + | ((u64::from(pcr_bytes[4])) >> 7); + let pcr_ext = ((u64::from(pcr_bytes[4]) & 1) << 8) | u64::from(pcr_bytes[5]); + pcr_base * 300 + pcr_ext +} + +#[derive(Clone, Copy)] +pub(super) enum HlsTsTimestampKind { + PtsOrDts, + Pcr, +} + +#[inline] +pub(super) fn ts_packet_pid(packet: &[u8]) -> u16 { (u16::from(packet[1] & 0x1F) << 8) | u16::from(packet[2]) } + +pub(super) fn same_finite_ts_packet_layout(source: &[u8], prepared: &[u8]) -> bool { + if source.len() != TS_PACKET_SIZE + || prepared.len() != TS_PACKET_SIZE + || source[0] != SYNC_BYTE + || prepared[0] != SYNC_BYTE + || source[1] & 0x7F != prepared[1] & 0x7F + || source[2] != prepared[2] + || source[3] & 0xF0 != prepared[3] & 0xF0 + { + return false; + } + let adaptation_field_control = (source[3] >> 4) & 0b11; + !matches!(adaptation_field_control, 0b10 | 0b11) || source[4] == prepared[4] +} + +pub(super) fn append_finite_discontinuity_packet(following_packet: &[u8], output: &mut BytesMut) { + let following_cc = following_packet[3] & 0x0F; + let marker_cc = if following_packet[3] & 0x10 != 0 { following_cc.wrapping_sub(1) & 0x0F } else { following_cc }; + let start = output.len(); + output.resize(start.saturating_add(TS_PACKET_SIZE), 0xFF); + let marker = &mut output[start..start + TS_PACKET_SIZE]; + marker[0] = SYNC_BYTE; + marker[1] = following_packet[1] & 0x1F; + marker[2] = following_packet[2]; + // FFmpeg retains this adaptation-only marker as the per-PID CC baseline. + marker[3] = 0x20 | marker_cc; + marker[4] = 183; + marker[5] = 0x80; +} + +/// Encode PCR timestamp (u64) back into 6 bytes +#[inline] +#[allow(clippy::cast_possible_truncation)] +pub(super) fn encode_pcr(pcr: u64) -> [u8; 6] { + let pcr_base = pcr / 300; + let pcr_ext = pcr % 300; + + [ + ((pcr_base >> 25) & 0xFF) as u8, + ((pcr_base >> 17) & 0xFF) as u8, + ((pcr_base >> 9) & 0xFF) as u8, + ((pcr_base >> 1) & 0xFF) as u8, + // Bit 7 = bit0 of pcr_base, Bits 6-1 reserved '111111', Bit 0 = high bit of pcr_ext + (((pcr_base & 1) << 7) as u8) | 0x7E | (((pcr_ext >> 8) & 1) as u8), + (pcr_ext & 0xFF) as u8, + ] +} + +pub(super) fn gather_timestamp_bytes( + buffer: &[u8], + location: HlsTsTimestampFieldLocation, +) -> Result<[u8; 5], HlsFiniteTsLayoutError> { + let mut bytes = [0u8; 5]; + for (destination, offset) in bytes.iter_mut().zip(location.byte_offsets) { + *destination = buffer.get(offset).copied().ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + } + Ok(bytes) +} + +pub(super) fn decode_timestamp_at_location( + buffer: &[u8], + location: HlsTsTimestampFieldLocation, +) -> Result { + gather_timestamp_bytes(buffer, location).map(|bytes| decode_timestamp(&bytes)) +} + +pub(super) fn decode_pcr_at_location( + buffer: &[u8], + location: HlsTsPcrFieldLocation, +) -> Result { + let end = location.byte_offset.checked_add(6).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + let bytes = buffer.get(location.byte_offset..end).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + Ok(decode_pcr(bytes)) +} + +pub(super) fn ticks_90khz_to_rounded_millis(ticks: u64) -> Option { ticks.checked_add(45)?.checked_div(90) } diff --git a/backend/mpegts/src/transport_stream_buffer/layout.rs b/backend/mpegts/src/transport_stream_buffer/layout.rs new file mode 100644 index 000000000..5a2bff05a --- /dev/null +++ b/backend/mpegts/src/transport_stream_buffer/layout.rs @@ -0,0 +1,292 @@ +use super::{ + clock::{decode_pcr_at_location, decode_timestamp_at_location, MAX_PTS_DTS, SYNC_BYTE, TS_PACKET_SIZE}, + pes::{inspect_hls_ts_packet, HlsFiniteTsLayoutError}, + HlsFiniteTsLayout, HlsFiniteTsPacketLayout, HlsPesHeaderAssembler, HlsTsTimestampFieldKind, +}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; + +/// Finds TS alignment by checking for 0x47 sync byte every 188 bytes +pub(super) fn find_ts_alignment(buf: &[u8]) -> Option { + for offset in 0..TS_PACKET_SIZE { + let mut valid = true; + for i in 0..5 { + if buf.get(offset + i * TS_PACKET_SIZE) != Some(&SYNC_BYTE) { + valid = false; + break; + } + } + if valid { + return Some(offset); + } + } + None +} + +pub(super) fn build_hls_finite_ts_layout(buffer: &[u8]) -> Result { + if buffer.is_empty() || !buffer.len().is_multiple_of(TS_PACKET_SIZE) { + return Err(HlsFiniteTsLayoutError::InvalidAsset); + } + let packet_count = buffer.len() / TS_PACKET_SIZE; + let mut packet_evidence = Vec::with_capacity(packet_count); + let mut pcr_fields = Vec::new(); + let mut packet_pcr_indices = Vec::with_capacity(packet_count); + let mut timestamp_fields = Vec::new(); + let mut assembler = HlsPesHeaderAssembler::new(); + + for (packet_index, packet) in buffer.as_chunks::().0.iter().enumerate() { + let packet_start = + packet_index.checked_mul(TS_PACKET_SIZE).ok_or(HlsFiniteTsLayoutError::InvalidTransportPacket)?; + let evidence = inspect_hls_ts_packet(packet, packet_start)?; + let pcr_field_index = evidence.pcr_field.map(|field| { + let index = pcr_fields.len(); + pcr_fields.push(field); + index + }); + let completed = assembler.push_packet(packet, packet_start, evidence)?; + timestamp_fields.extend(completed.fields.into_iter().flatten().map(|field| field.location)); + packet_evidence.push(evidence); + packet_pcr_indices.push(pcr_field_index); + } + assembler.finish()?; + timestamp_fields.sort_unstable_by_key(|field| field.byte_offsets[0]); + + let mut timestamp_field_counts = vec![0usize; packet_count]; + for field in ×tamp_fields { + let mut previous_packet = None; + for offset in field.byte_offsets { + if offset >= buffer.len() { + return Err(HlsFiniteTsLayoutError::InvalidTimestampLocation); + } + let packet_index = offset / TS_PACKET_SIZE; + if previous_packet != Some(packet_index) { + timestamp_field_counts[packet_index] = timestamp_field_counts[packet_index] + .checked_add(1) + .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + previous_packet = Some(packet_index); + } + } + } + let mut timestamp_field_starts = Vec::with_capacity(packet_count.saturating_add(1)); + timestamp_field_starts.push(0usize); + for count in ×tamp_field_counts { + let next = timestamp_field_starts + .last() + .copied() + .and_then(|start| start.checked_add(*count)) + .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + timestamp_field_starts.push(next); + } + let timestamp_membership_count = timestamp_field_starts.last().copied().unwrap_or(0); + let mut packet_timestamp_field_indices = vec![0usize; timestamp_membership_count]; + let mut packet_cursors = timestamp_field_starts[..packet_count].to_vec(); + for (field_index, field) in timestamp_fields.iter().enumerate() { + let mut previous_packet = None; + for offset in field.byte_offsets { + let packet_index = offset / TS_PACKET_SIZE; + if previous_packet == Some(packet_index) { + continue; + } + let cursor = packet_cursors[packet_index]; + packet_timestamp_field_indices[cursor] = field_index; + packet_cursors[packet_index] = cursor.saturating_add(1); + previous_packet = Some(packet_index); + } + } + let packets = packet_evidence + .into_iter() + .enumerate() + .map(|(packet_index, evidence)| HlsFiniteTsPacketLayout { + packet_start: packet_index.saturating_mul(TS_PACKET_SIZE), + pid: evidence.pid, + has_payload: evidence.has_payload(), + timestamp_field_indices_start: timestamp_field_starts[packet_index], + timestamp_field_indices_end: timestamp_field_starts[packet_index.saturating_add(1)], + pcr_field_index: packet_pcr_indices[packet_index], + }) + .collect::>(); + Ok(HlsFiniteTsLayout { + packets: Arc::from(packets), + timestamp_fields: Arc::from(timestamp_fields), + pcr_fields: Arc::from(pcr_fields), + packet_timestamp_field_indices: Arc::from(packet_timestamp_field_indices), + }) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HlsTsPresentationClockSource { + Pts, + PcrFallback, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct HlsTsPidPresentationTimeline { + pub pid: u16, + pub first_pts_90khz: u64, + pub last_pts_90khz: u64, + pub cadence_ticks_90khz: u64, + pub end_exclusive_90khz: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct HlsTsPresentationDuration { + pub first_presentation_clock_90khz: u64, + pub end_exclusive_clock_90khz: u64, + pub duration_ticks_90khz: u64, + pub timelines: Arc<[HlsTsPidPresentationTimeline]>, + pub source: HlsTsPresentationClockSource, +} + +fn stable_presentation_cadence(mut deltas: Vec) -> Option { + if deltas.is_empty() { + return None; + } + deltas.sort_unstable(); + let mut dominant_delta = deltas[0]; + let mut dominant_count = 1usize; + let mut current_delta = deltas[0]; + let mut current_count = 1usize; + for delta in deltas.iter().copied().skip(1) { + if delta == current_delta { + current_count = current_count.saturating_add(1); + } else { + if current_count > dominant_count { + dominant_delta = current_delta; + dominant_count = current_count; + } + current_delta = delta; + current_count = 1; + } + } + if current_count > dominant_count { + dominant_delta = current_delta; + dominant_count = current_count; + } + if dominant_count.saturating_mul(2) > deltas.len() { + Some(dominant_delta) + } else { + deltas.get(deltas.len() / 2).copied() + } +} + +fn presentation_timeline_for_pid( + pid: u16, + timestamps: &[u64], +) -> Result { + let Some(first_raw) = timestamps.first().copied() else { + return Err(HlsFiniteTsLayoutError::PresentationCadenceUnavailable { pid }); + }; + let cycle = i128::from(MAX_PTS_DTS); + let mut previous_raw = first_raw % MAX_PTS_DTS; + let mut previous_unwrapped = i128::from(previous_raw); + let mut first_unwrapped = previous_unwrapped; + let mut last_unwrapped = previous_unwrapped; + let mut seen_raw_timestamps = HashSet::with_capacity(timestamps.len()); + seen_raw_timestamps.insert(previous_raw); + let mut deltas = Vec::with_capacity(timestamps.len().saturating_sub(1)); + + for raw in timestamps.iter().copied().skip(1).map(|value| value % MAX_PTS_DTS) { + if !seen_raw_timestamps.insert(raw) { + continue; + } + let forward = raw.wrapping_add(MAX_PTS_DTS).wrapping_sub(previous_raw) % MAX_PTS_DTS; + let signed_delta = if forward <= MAX_PTS_DTS / 2 { + i128::from(forward) + } else { + -i128::from(MAX_PTS_DTS.saturating_sub(forward)) + }; + let unwrapped = + previous_unwrapped.checked_add(signed_delta).ok_or(HlsFiniteTsLayoutError::PresentationDurationOverflow)?; + if unwrapped > previous_unwrapped { + deltas.push( + u64::try_from(unwrapped - previous_unwrapped) + .map_err(|_| HlsFiniteTsLayoutError::PresentationDurationOverflow)?, + ); + } + first_unwrapped = first_unwrapped.min(unwrapped); + last_unwrapped = last_unwrapped.max(unwrapped); + previous_raw = raw; + previous_unwrapped = unwrapped; + } + let cadence_ticks_90khz = stable_presentation_cadence(deltas) + .filter(|cadence| *cadence > 0 && *cadence < MAX_PTS_DTS / 2) + .ok_or(HlsFiniteTsLayoutError::PresentationCadenceUnavailable { pid })?; + let normalization = if first_unwrapped < 0 { + (-first_unwrapped) + .checked_add(cycle.saturating_sub(1)) + .and_then(|value| value.checked_div(cycle)) + .and_then(|cycles| cycles.checked_mul(cycle)) + .ok_or(HlsFiniteTsLayoutError::PresentationDurationOverflow)? + } else { + 0 + }; + let first_pts_90khz = u64::try_from( + first_unwrapped.checked_add(normalization).ok_or(HlsFiniteTsLayoutError::PresentationDurationOverflow)?, + ) + .map_err(|_| HlsFiniteTsLayoutError::PresentationDurationOverflow)?; + let last_pts_90khz = u64::try_from( + last_unwrapped.checked_add(normalization).ok_or(HlsFiniteTsLayoutError::PresentationDurationOverflow)?, + ) + .map_err(|_| HlsFiniteTsLayoutError::PresentationDurationOverflow)?; + let end_exclusive_90khz = + last_pts_90khz.checked_add(cadence_ticks_90khz).ok_or(HlsFiniteTsLayoutError::PresentationDurationOverflow)?; + Ok(HlsTsPidPresentationTimeline { pid, first_pts_90khz, last_pts_90khz, cadence_ticks_90khz, end_exclusive_90khz }) +} + +fn presentation_duration_from_pid_clocks( + clocks: HashMap>, + source: HlsTsPresentationClockSource, +) -> Result { + if clocks.is_empty() { + return Err(HlsFiniteTsLayoutError::PresentationClockUnavailable); + } + let mut timelines = clocks + .into_iter() + .map(|(pid, timestamps)| presentation_timeline_for_pid(pid, ×tamps)) + .collect::, _>>()?; + timelines.sort_unstable_by_key(|timeline| timeline.pid); + let first_presentation_clock_90khz = timelines + .iter() + .map(|timeline| timeline.first_pts_90khz) + .min() + .ok_or(HlsFiniteTsLayoutError::PresentationClockUnavailable)?; + let end_exclusive_clock_90khz = timelines + .iter() + .map(|timeline| timeline.end_exclusive_90khz) + .max() + .ok_or(HlsFiniteTsLayoutError::PresentationClockUnavailable)?; + let duration_ticks_90khz = end_exclusive_clock_90khz + .checked_sub(first_presentation_clock_90khz) + .filter(|duration| *duration > 0 && *duration < MAX_PTS_DTS) + .ok_or(HlsFiniteTsLayoutError::PresentationDurationOverflow)?; + Ok(HlsTsPresentationDuration { + first_presentation_clock_90khz, + end_exclusive_clock_90khz, + duration_ticks_90khz, + timelines: Arc::from(timelines), + source, + }) +} + +pub(super) fn finite_hls_presentation_duration( + buffer: &[u8], + layout: &HlsFiniteTsLayout, +) -> Result { + let mut pts_by_pid = HashMap::>::new(); + for location in + layout.timestamp_fields.iter().copied().filter(|location| location.kind == HlsTsTimestampFieldKind::Pts) + { + pts_by_pid.entry(location.pid).or_default().push(decode_timestamp_at_location(buffer, location)?); + } + if !pts_by_pid.is_empty() { + return presentation_duration_from_pid_clocks(pts_by_pid, HlsTsPresentationClockSource::Pts); + } + + let mut pcr_by_pid = HashMap::>::new(); + for location in layout.pcr_fields.iter().copied() { + pcr_by_pid.entry(location.pid).or_default().push(decode_pcr_at_location(buffer, location)? / 300); + } + presentation_duration_from_pid_clocks(pcr_by_pid, HlsTsPresentationClockSource::PcrFallback) +} diff --git a/backend/mpegts/src/transport_stream_buffer/pes.rs b/backend/mpegts/src/transport_stream_buffer/pes.rs new file mode 100644 index 000000000..804563eb7 --- /dev/null +++ b/backend/mpegts/src/transport_stream_buffer/pes.rs @@ -0,0 +1,285 @@ +use super::{ + clock::{ + ts_packet_pid, ADAPTATION_FIELD_FLAG_PCR, HLS_TS_MAX_PENDING_PES_HEADERS, HLS_TS_TIMESTAMP_HEADER_BYTES, + PES_DTS_OFFSET, PES_PTS_OFFSET, SYNC_BYTE, TS_PACKET_SIZE, + }, + HlsCompletedPesTimestamps, HlsCompletedTimestampField, HlsPendingPesHeader, HlsPesHeaderAssembler, + HlsTsPacketEvidence, +}; +use std::collections::HashMap; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HlsTsTimestampFieldKind { + Pts, + Dts, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct HlsTsTimestampFieldLocation { + pub pid: u16, + pub kind: HlsTsTimestampFieldKind, + /// Absolute byte offsets in the immutable aligned TS buffer. One field may span packets. + pub byte_offsets: [usize; 5], +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct HlsTsPcrFieldLocation { + pub pid: u16, + pub byte_offset: usize, +} + +#[derive(Debug, thiserror::Error, Clone, Copy, PartialEq, Eq)] +pub(super) enum HlsFiniteTsLayoutError { + #[error("transport stream asset is empty or invalid")] + InvalidAsset, + #[error("transport stream packet layout is invalid")] + InvalidTransportPacket, + #[error("transport stream timestamp location is invalid")] + InvalidTimestampLocation, + #[error("PES timestamp header for PID {pid} uses an unsupported layout")] + UnsupportedPesTimestampHeader { pid: u16 }, + #[error("PES {kind:?} field for PID {pid} is invalid")] + InvalidPesTimestampField { pid: u16, kind: HlsTsTimestampFieldKind }, + #[error("PES timestamp header for PID {pid} was interrupted by a new payload unit")] + PesTimestampHeaderInterrupted { pid: u16 }, + #[error("PES timestamp header continuity failed for PID {pid}: expected {expected:?}, actual {actual}")] + PesTimestampContinuityDiscontinuity { pid: u16, expected: Option, actual: u8 }, + #[error("PES timestamp header for PID {pid} is incomplete")] + IncompletePesTimestampHeader { pid: u16 }, + #[error("too many concurrent split PES timestamp headers")] + TooManyPendingPesTimestampHeaders, + #[error("transport stream contains no presentation clock")] + PresentationClockUnavailable, + #[error("PID {pid} has no defensible presentation cadence")] + PresentationCadenceUnavailable { pid: u16 }, + #[error("transport stream presentation duration overflows its clock domain")] + PresentationDurationOverflow, +} + +impl HlsPendingPesHeader { + pub(super) fn new(pid: u16, continuity_counter: u8) -> Self { + Self { + pid, + bytes: [0; HLS_TS_TIMESTAMP_HEADER_BYTES], + byte_offsets: [0; HLS_TS_TIMESTAMP_HEADER_BYTES], + len: 0, + expected_len: None, + last_payload_continuity_counter: continuity_counter, + } + } +} + +impl HlsTsPacketEvidence { + pub(super) const fn has_payload(self) -> bool { self.payload_offset.is_some() } +} + +pub(super) fn inspect_hls_ts_packet( + packet: &[u8], + packet_start: usize, +) -> Result { + if packet.len() != TS_PACKET_SIZE || packet[0] != SYNC_BYTE || packet[1] & 0x80 != 0 || packet[3] & 0xC0 != 0 { + return Err(HlsFiniteTsLayoutError::InvalidTransportPacket); + } + let pid = ts_packet_pid(packet); + let adaptation_field_control = (packet[3] >> 4) & 0b11; + let continuity_counter = packet[3] & 0x0F; + let (payload_offset, adaptation_length) = match adaptation_field_control { + 0b01 => (Some(4), None), + 0b10 if packet[4] == 183 => (None, Some(183)), + 0b11 if packet[4] <= 182 => { + let adaptation_length = usize::from(packet[4]); + (Some(5usize.saturating_add(adaptation_length)), Some(adaptation_length)) + } + _ => return Err(HlsFiniteTsLayoutError::InvalidTransportPacket), + }; + let mut discontinuity = false; + let mut pcr_field = None; + if let Some(adaptation_length) = adaptation_length { + if adaptation_length > 0 { + discontinuity = packet[5] & 0x80 != 0; + if packet[5] & ADAPTATION_FIELD_FLAG_PCR != 0 { + if adaptation_length < 7 { + return Err(HlsFiniteTsLayoutError::InvalidTransportPacket); + } + pcr_field = Some(HlsTsPcrFieldLocation { + pid, + byte_offset: packet_start.checked_add(6).ok_or(HlsFiniteTsLayoutError::InvalidTransportPacket)?, + }); + } + } + } + Ok(HlsTsPacketEvidence { + pid, + payload_unit_start: packet[1] & 0x40 != 0, + payload_offset, + continuity_counter, + discontinuity, + pcr_field, + }) +} + +fn pes_stream_id_has_optional_header(stream_id: u8) -> bool { + !matches!( + stream_id, + 0xBC | // program_stream_map + 0xBE | // padding_stream + 0xBF | // private_stream_2 + 0xF0 | // ECM + 0xF1 | // EMM + 0xFF | // program_stream_directory + 0xF2 | // DSM-CC + 0xF8 // ITU-T Rec. H.222.1 type E + ) +} + +fn timestamp_field_from_pending( + pending: &HlsPendingPesHeader, + kind: HlsTsTimestampFieldKind, + start: usize, + prefix: u8, +) -> Result { + let end = + start.checked_add(5).ok_or(HlsFiniteTsLayoutError::InvalidPesTimestampField { pid: pending.pid, kind })?; + let bytes: [u8; 5] = pending + .bytes + .get(start..end) + .and_then(|bytes| bytes.try_into().ok()) + .ok_or(HlsFiniteTsLayoutError::InvalidPesTimestampField { pid: pending.pid, kind })?; + let byte_offsets: [usize; 5] = pending + .byte_offsets + .get(start..end) + .and_then(|offsets| offsets.try_into().ok()) + .ok_or(HlsFiniteTsLayoutError::InvalidPesTimestampField { pid: pending.pid, kind })?; + if bytes[0] >> 4 != prefix || bytes[0] & 1 == 0 || bytes[2] & 1 == 0 || bytes[4] & 1 == 0 { + return Err(HlsFiniteTsLayoutError::InvalidPesTimestampField { pid: pending.pid, kind }); + } + Ok(HlsCompletedTimestampField { + location: HlsTsTimestampFieldLocation { pid: pending.pid, kind, byte_offsets }, + bytes, + }) +} + +fn complete_pes_timestamps(pending: &HlsPendingPesHeader) -> Result { + let pts_dts_flags = (pending.bytes[7] >> 6) & 0b11; + match pts_dts_flags { + 0b00 => Ok(HlsCompletedPesTimestamps::default()), + 0b10 => Ok(HlsCompletedPesTimestamps { + fields: [ + Some(timestamp_field_from_pending(pending, HlsTsTimestampFieldKind::Pts, PES_PTS_OFFSET, 0x02)?), + None, + ], + }), + 0b11 => Ok(HlsCompletedPesTimestamps { + fields: [ + Some(timestamp_field_from_pending(pending, HlsTsTimestampFieldKind::Pts, PES_PTS_OFFSET, 0x03)?), + Some(timestamp_field_from_pending(pending, HlsTsTimestampFieldKind::Dts, PES_DTS_OFFSET, 0x01)?), + ], + }), + _ => Err(HlsFiniteTsLayoutError::UnsupportedPesTimestampHeader { pid: pending.pid }), + } +} + +fn append_pending_pes_header( + pending: &mut HlsPendingPesHeader, + payload: &[u8], + payload_start: usize, +) -> Result, HlsFiniteTsLayoutError> { + for (payload_index, byte) in payload.iter().copied().enumerate() { + if pending.expected_len.is_some_and(|expected| pending.len >= expected) { + break; + } + if pending.len >= HLS_TS_TIMESTAMP_HEADER_BYTES { + return Err(HlsFiniteTsLayoutError::UnsupportedPesTimestampHeader { pid: pending.pid }); + } + pending.bytes[pending.len] = byte; + pending.byte_offsets[pending.len] = + payload_start.checked_add(payload_index).ok_or(HlsFiniteTsLayoutError::InvalidTransportPacket)?; + pending.len = pending.len.saturating_add(1); + + if pending.len == 4 && !pes_stream_id_has_optional_header(pending.bytes[3]) { + pending.expected_len = Some(4); + } + if pending.len == 7 && (pending.bytes[6] & 0xC0) != 0x80 { + return Err(HlsFiniteTsLayoutError::UnsupportedPesTimestampHeader { pid: pending.pid }); + } + if pending.len == PES_PTS_OFFSET { + let header_data_length = usize::from(pending.bytes[8]); + pending.expected_len = match (pending.bytes[7] >> 6) & 0b11 { + 0b00 => Some(PES_PTS_OFFSET), + 0b10 if header_data_length >= 5 => Some(PES_PTS_OFFSET + 5), + 0b11 if header_data_length >= 10 => Some(PES_DTS_OFFSET + 5), + _ => return Err(HlsFiniteTsLayoutError::UnsupportedPesTimestampHeader { pid: pending.pid }), + }; + } + } + + if pending.expected_len.is_some_and(|expected| pending.len >= expected) { + return complete_pes_timestamps(pending).map(Some); + } + Ok(None) +} + +impl HlsPesHeaderAssembler { + pub(super) fn new() -> Self { Self { pending: HashMap::with_capacity(8) } } + + pub(super) fn push_packet( + &mut self, + packet: &[u8], + packet_start: usize, + evidence: HlsTsPacketEvidence, + ) -> Result { + if evidence.payload_unit_start && self.pending.contains_key(&evidence.pid) { + return Err(HlsFiniteTsLayoutError::PesTimestampHeaderInterrupted { pid: evidence.pid }); + } + if evidence.discontinuity && self.pending.contains_key(&evidence.pid) { + return Err(HlsFiniteTsLayoutError::PesTimestampContinuityDiscontinuity { + pid: evidence.pid, + expected: None, + actual: evidence.continuity_counter, + }); + } + let Some(payload_offset) = evidence.payload_offset else { + return Ok(HlsCompletedPesTimestamps::default()); + }; + let payload = packet.get(payload_offset..).ok_or(HlsFiniteTsLayoutError::InvalidTransportPacket)?; + let payload_start = + packet_start.checked_add(payload_offset).ok_or(HlsFiniteTsLayoutError::InvalidTransportPacket)?; + + if let Some(mut pending) = self.pending.remove(&evidence.pid) { + let expected = pending.last_payload_continuity_counter.wrapping_add(1) & 0x0F; + if evidence.continuity_counter != expected { + return Err(HlsFiniteTsLayoutError::PesTimestampContinuityDiscontinuity { + pid: evidence.pid, + expected: Some(expected), + actual: evidence.continuity_counter, + }); + } + pending.last_payload_continuity_counter = evidence.continuity_counter; + if let Some(completed) = append_pending_pes_header(&mut pending, payload, payload_start)? { + return Ok(completed); + } + self.pending.insert(evidence.pid, pending); + return Ok(HlsCompletedPesTimestamps::default()); + } + + if !evidence.payload_unit_start || payload.len() < 3 || !payload.starts_with(&[0x00, 0x00, 0x01]) { + return Ok(HlsCompletedPesTimestamps::default()); + } + if self.pending.len() >= HLS_TS_MAX_PENDING_PES_HEADERS { + return Err(HlsFiniteTsLayoutError::TooManyPendingPesTimestampHeaders); + } + let mut pending = HlsPendingPesHeader::new(evidence.pid, evidence.continuity_counter); + if let Some(completed) = append_pending_pes_header(&mut pending, payload, payload_start)? { + return Ok(completed); + } + self.pending.insert(evidence.pid, pending); + Ok(HlsCompletedPesTimestamps::default()) + } + + pub(super) fn finish(self) -> Result<(), HlsFiniteTsLayoutError> { + match self.pending.keys().copied().min() { + Some(pid) => Err(HlsFiniteTsLayoutError::IncompletePesTimestampHeader { pid }), + None => Ok(()), + } + } +} diff --git a/backend/mpegts/src/transport_stream_buffer/profile.rs b/backend/mpegts/src/transport_stream_buffer/profile.rs new file mode 100644 index 000000000..7d8466e90 --- /dev/null +++ b/backend/mpegts/src/transport_stream_buffer/profile.rs @@ -0,0 +1,185 @@ +use super::{ + clock::{ + add_pts_dts_offset, decode_pcr, decode_pcr_at_location, decode_timestamp, decode_timestamp_at_location, + forward_clock_distance_90khz, HlsTsTimestampKind, HLS_TS_PROFILE_MIN_TOLERANCE_TICKS_90KHZ, + HLS_TS_SPLICE_MIN_GAP_TICKS_90KHZ, MAX_PTS_DTS, TS_PACKET_SIZE, + }, + pes::inspect_hls_ts_packet, + HlsFiniteTsLayout, HlsPesHeaderAssembler, HlsTsTimestampProfileAccumulator, HlsTsTimestampProfileScanner, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct HlsTsTimestampProfile { + pub first_clock_90khz: u64, + pub last_clock_90khz: u64, + pub span_ticks_90khz: u64, + pub observed_pts_or_dts: bool, + pub observed_pcr: bool, +} + +/// MPEG-TS live-to-terminal timestamp anchor expressed in 90 kHz clock ticks. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct HlsTsSpliceAnchor { + pub live_last_clock: u64, + pub terminal_first_clock: u64, + pub timestamp_delta_ticks: u64, +} + +impl HlsTsSpliceAnchor { + pub fn between(live_tail: HlsTsTimestampProfile, terminal_asset: HlsTsTimestampProfile) -> Option { + if !timestamp_profile_is_coherent(live_tail) || !timestamp_profile_is_coherent(terminal_asset) { + return None; + } + let terminal_first_clock_90khz = + add_pts_dts_offset(live_tail.last_clock_90khz, HLS_TS_SPLICE_MIN_GAP_TICKS_90KHZ); + let timestamp_delta_ticks_90khz = + terminal_first_clock_90khz.wrapping_add(MAX_PTS_DTS).wrapping_sub(terminal_asset.first_clock_90khz) + % MAX_PTS_DTS; + let rebased_first = add_pts_dts_offset(terminal_asset.first_clock_90khz, timestamp_delta_ticks_90khz); + (rebased_first == terminal_first_clock_90khz).then_some(Self { + live_last_clock: live_tail.last_clock_90khz, + terminal_first_clock: terminal_first_clock_90khz, + timestamp_delta_ticks: timestamp_delta_ticks_90khz, + }) + } +} + +fn timestamp_profile_is_coherent(profile: HlsTsTimestampProfile) -> bool { + (profile.observed_pts_or_dts || profile.observed_pcr) + && profile.first_clock_90khz < MAX_PTS_DTS + && profile.last_clock_90khz < MAX_PTS_DTS + && profile.span_ticks_90khz < MAX_PTS_DTS + && forward_clock_distance_90khz(profile.first_clock_90khz, profile.last_clock_90khz) == profile.span_ticks_90khz +} + +impl HlsTsTimestampProfileAccumulator { + pub(super) fn new(expected_duration_ticks_90khz: u64) -> Self { + let tolerance = (expected_duration_ticks_90khz / 2).max(HLS_TS_PROFILE_MIN_TOLERANCE_TICKS_90KHZ); + Self { + reference_clock_90khz: None, + earliest_relative_ticks: 0, + latest_relative_ticks: 0, + maximum_span_ticks_90khz: expected_duration_ticks_90khz.saturating_add(tolerance), + observed_pts_or_dts: false, + observed_pcr: false, + observation_count: 0, + invalid: expected_duration_ticks_90khz == 0, + } + } + + pub(super) fn observe_clock(&mut self, clock_90khz: u64, kind: HlsTsTimestampKind) { + let clock_90khz = clock_90khz % MAX_PTS_DTS; + match kind { + HlsTsTimestampKind::PtsOrDts => self.observed_pts_or_dts = true, + HlsTsTimestampKind::Pcr => self.observed_pcr = true, + } + self.observation_count = self.observation_count.saturating_add(1); + let Some(reference) = self.reference_clock_90khz else { + self.reference_clock_90khz = Some(clock_90khz); + return; + }; + let forward = clock_90khz.wrapping_add(MAX_PTS_DTS).wrapping_sub(reference) % MAX_PTS_DTS; + let relative = if forward <= MAX_PTS_DTS / 2 { + i64::try_from(forward).unwrap_or(i64::MAX) + } else { + -i64::try_from(MAX_PTS_DTS.saturating_sub(forward)).unwrap_or(i64::MAX) + }; + self.earliest_relative_ticks = self.earliest_relative_ticks.min(relative); + self.latest_relative_ticks = self.latest_relative_ticks.max(relative); + let span = self.latest_relative_ticks.saturating_sub(self.earliest_relative_ticks); + if u64::try_from(span).unwrap_or(u64::MAX) > self.maximum_span_ticks_90khz { + self.invalid = true; + } + } + + pub(super) fn finish(self) -> Option { + let reference = self.reference_clock_90khz?; + if self.invalid || self.observation_count < 2 || (!self.observed_pts_or_dts && !self.observed_pcr) { + return None; + } + let span_ticks_90khz = + u64::try_from(self.latest_relative_ticks.saturating_sub(self.earliest_relative_ticks)).ok()?; + if span_ticks_90khz == 0 { + return None; + } + let cycle = i128::from(MAX_PTS_DTS); + let first_clock_90khz = + u64::try_from((i128::from(reference) + i128::from(self.earliest_relative_ticks)).rem_euclid(cycle)).ok()?; + let last_clock_90khz = + u64::try_from((i128::from(reference) + i128::from(self.latest_relative_ticks)).rem_euclid(cycle)).ok()?; + Some(HlsTsTimestampProfile { + first_clock_90khz, + last_clock_90khz, + span_ticks_90khz, + observed_pts_or_dts: self.observed_pts_or_dts, + observed_pcr: self.observed_pcr, + }) + } +} + +impl HlsTsTimestampProfileScanner { + pub fn new(expected_duration_ticks_90khz: u64) -> Self { + Self { + assembler: HlsPesHeaderAssembler::new(), + accumulator: HlsTsTimestampProfileAccumulator::new(expected_duration_ticks_90khz), + next_packet_start: 0, + invalid: false, + } + } + + pub fn push_aligned_packet(&mut self, packet: &[u8]) { + if self.invalid { + return; + } + let packet_start = self.next_packet_start; + let Some(next_packet_start) = packet_start.checked_add(TS_PACKET_SIZE) else { + self.invalid = true; + return; + }; + self.next_packet_start = next_packet_start; + let Ok(evidence) = inspect_hls_ts_packet(packet, packet_start) else { + self.invalid = true; + return; + }; + if let Some(pcr) = evidence.pcr_field { + let Some(relative_offset) = pcr.byte_offset.checked_sub(packet_start) else { + self.invalid = true; + return; + }; + let Some(bytes) = packet.get(relative_offset..relative_offset.saturating_add(6)) else { + self.invalid = true; + return; + }; + self.accumulator.observe_clock(decode_pcr(bytes) / 300, HlsTsTimestampKind::Pcr); + } + let Ok(completed) = self.assembler.push_packet(packet, packet_start, evidence) else { + self.invalid = true; + return; + }; + for field in completed.fields.into_iter().flatten() { + self.accumulator.observe_clock(decode_timestamp(&field.bytes), HlsTsTimestampKind::PtsOrDts); + } + } + + pub fn finish(self) -> Option { + if self.invalid || self.assembler.finish().is_err() { + return None; + } + self.accumulator.finish() + } +} + +pub(super) fn timestamp_profile_from_finite_layout( + buffer: &[u8], + layout: &HlsFiniteTsLayout, + expected_duration_ticks_90khz: u64, +) -> Option { + let mut accumulator = HlsTsTimestampProfileAccumulator::new(expected_duration_ticks_90khz); + for location in layout.timestamp_fields.iter().copied() { + accumulator.observe_clock(decode_timestamp_at_location(buffer, location).ok()?, HlsTsTimestampKind::PtsOrDts); + } + for location in layout.pcr_fields.iter().copied() { + accumulator.observe_clock(decode_pcr_at_location(buffer, location).ok()? / 300, HlsTsTimestampKind::Pcr); + } + accumulator.finish() +} diff --git a/backend/mpegts/src/transport_stream_buffer/render.rs b/backend/mpegts/src/transport_stream_buffer/render.rs new file mode 100644 index 000000000..0046f0f23 --- /dev/null +++ b/backend/mpegts/src/transport_stream_buffer/render.rs @@ -0,0 +1,317 @@ +use super::{ + clock::{ + add_pcr_offset_27mhz, add_pts_dts_offset, append_finite_discontinuity_packet, decode_pcr_at_location, + decode_timestamp, encode_pcr, encode_timestamp, gather_timestamp_bytes, pcr_offset_27mhz, + same_finite_ts_packet_layout, ts_packet_pid, ADAPTATION_FIELD_FLAG_PCR, NULL_PID, SYNC_BYTE, TS_PACKET_SIZE, + }, + pes::HlsFiniteTsLayoutError, + HlsFiniteTsLayout, HlsFiniteTsPacketLayout, TransportStreamBuffer, +}; +use bytes::{Bytes, BytesMut}; +use std::task::Waker; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HlsFiniteTsRenderSpec { + pub timestamp_offset_ticks_90khz: u64, + /// Seed used by logical segment zero. Later segments derive each PID's starting counter from + /// the number of payload packets for that PID in one immutable asset cycle. + pub continuity_seed: u8, + pub logical_segment_index: u16, +} + +#[derive(Debug, thiserror::Error, Clone, Copy, PartialEq, Eq)] +pub enum HlsFiniteTsRenderError { + #[error("transport stream asset is empty or invalid")] + InvalidAsset, + #[error("prepared transport stream does not match the immutable asset packet layout")] + PreparedLayoutMismatch, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HlsFiniteTsDiscontinuityMode { + None, + FirstPacketPerPid, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct HlsFiniteTsFinalizeSpec { + pub additional_timestamp_offset_ticks_90khz: u64, + pub discontinuity: HlsFiniteTsDiscontinuityMode, +} + +impl TransportStreamBuffer { + /// Renders one immutable, finite HLS media segment without mutating the looping stream state. + pub fn render_finite_hls_segment(&self, spec: HlsFiniteTsRenderSpec) -> Result { + #[cfg(any(test, feature = "test-support"))] + self.finite_hls_render_count.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + let layout = self.finite_hls_layout.as_ref().map_err(|_| HlsFiniteTsRenderError::InvalidAsset)?; + if self.buffer.is_empty() || layout.packets.is_empty() { + return Err(HlsFiniteTsRenderError::InvalidAsset); + } + let mut output = BytesMut::with_capacity(self.buffer.len()); + let mut payload_counts = [0_u8; 8192]; + for packet_layout in layout.packets.iter().copied() { + if packet_layout.has_payload { + let count = &mut payload_counts[usize::from(packet_layout.pid)]; + *count = count.wrapping_add(1) & 0x0F; + } + } + let mut continuity = [None; 8192]; + for packet_layout in layout.packets.iter().copied() { + let packet_end = packet_layout.packet_start.saturating_add(TS_PACKET_SIZE); + let Some(packet) = self.buffer.get(packet_layout.packet_start..packet_end) else { + return Err(HlsFiniteTsRenderError::InvalidAsset); + }; + let output_start = output.len(); + output.extend_from_slice(packet); + let counter = continuity[usize::from(packet_layout.pid)].get_or_insert_with(|| { + let cycle_advance = payload_counts[usize::from(packet_layout.pid)] + .wrapping_mul((spec.logical_segment_index & 0x0F) as u8) + & 0x0F; + spec.continuity_seed.wrapping_add(cycle_advance) & 0x0F + }); + output[output_start + 3] = (output[output_start + 3] & 0xF0) | *counter; + if packet_layout.has_payload { + *counter = counter.wrapping_add(1) & 0x0F; + } + } + Self::rewrite_layout_timestamps(&mut output, layout, spec.timestamp_offset_ticks_90khz) + .map_err(|_| HlsFiniteTsRenderError::InvalidAsset)?; + Ok(output.freeze()) + } + + /// Applies one lease-specific timestamp anchor to an already prepared relative segment. + /// + /// The immutable source packet layout is verified before any bytes are published. Optional + /// splice markers are adaptation-only packets and therefore do not advance payload CC. + pub fn finalize_prepared_finite_hls_segment( + &self, + prepared: &Bytes, + spec: HlsFiniteTsFinalizeSpec, + ) -> Result { + #[cfg(any(test, feature = "test-support"))] + self.finite_hls_finalize_count.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + let layout = self.finite_hls_layout.as_ref().map_err(|_| HlsFiniteTsRenderError::PreparedLayoutMismatch)?; + if self.buffer.is_empty() + || layout.packets.is_empty() + || prepared.len() != layout.packets.len().saturating_mul(TS_PACKET_SIZE) + { + return Err(HlsFiniteTsRenderError::PreparedLayoutMismatch); + } + let mut first_pid_seen = [false; 8192]; + let mut marker_count = 0usize; + for (position, packet_layout) in layout.packets.iter().enumerate() { + let prepared_start = position.saturating_mul(TS_PACKET_SIZE); + let Some(source) = + self.buffer.get(packet_layout.packet_start..packet_layout.packet_start.saturating_add(TS_PACKET_SIZE)) + else { + return Err(HlsFiniteTsRenderError::PreparedLayoutMismatch); + }; + let Some(candidate) = prepared.get(prepared_start..prepared_start.saturating_add(TS_PACKET_SIZE)) else { + return Err(HlsFiniteTsRenderError::PreparedLayoutMismatch); + }; + if !same_finite_ts_packet_layout(source, candidate) { + return Err(HlsFiniteTsRenderError::PreparedLayoutMismatch); + } + let pid = ts_packet_pid(candidate); + if spec.discontinuity == HlsFiniteTsDiscontinuityMode::FirstPacketPerPid + && pid != NULL_PID + && !first_pid_seen[usize::from(pid)] + { + first_pid_seen[usize::from(pid)] = true; + marker_count = marker_count.saturating_add(1); + } + } + + let mut rewritten = BytesMut::from(prepared.as_ref()); + Self::rewrite_layout_timestamps(&mut rewritten, layout, spec.additional_timestamp_offset_ticks_90khz) + .map_err(|_| HlsFiniteTsRenderError::PreparedLayoutMismatch)?; + first_pid_seen.fill(false); + let additional_bytes = marker_count.saturating_mul(TS_PACKET_SIZE); + let mut output = BytesMut::with_capacity(prepared.len().saturating_add(additional_bytes)); + for position in 0..layout.packets.len() { + let prepared_start = position.saturating_mul(TS_PACKET_SIZE); + let packet = &rewritten[prepared_start..prepared_start + TS_PACKET_SIZE]; + let pid = ts_packet_pid(packet); + if spec.discontinuity == HlsFiniteTsDiscontinuityMode::FirstPacketPerPid + && pid != NULL_PID + && !first_pid_seen[usize::from(pid)] + { + append_finite_discontinuity_packet(packet, &mut output); + first_pid_seen[usize::from(pid)] = true; + } + output.extend_from_slice(packet); + } + Ok(output.freeze()) + } + + #[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation, clippy::cast_sign_loss)] + pub fn estimated_bitrate_kbps(&self) -> Option { + let stream_duration_90khz = self.duration_ticks_90khz()?; + if self.buffer.is_empty() { + return None; + } + let duration_secs = stream_duration_90khz as f64 / 90_000.0; + if duration_secs <= 0.0 { + return None; + } + let kbps = ((self.buffer.len() as f64 * 8.0) / duration_secs / 1_000.0).round(); + if !kbps.is_finite() || kbps <= 0.0 { + return None; + } + Some(kbps as usize) + } + + pub fn register_waker(&self, waker: &Waker) { self.waker.register(waker); } + + /// Sets the timestamp offset used when rewriting PTS/DTS/PCR values. + pub fn set_timestamp_offset(&mut self, offset: u64) { self.timestamp_offset = offset; } + + /// Generates a Discontinuity packet for the given packet/PID state, writing it directly into `out`. + pub(super) fn generate_discontinuity_packet( + new_packet: &[u8], + cc: u8, + first_pcr: Option, + timestamp_offset: u64, + out: &mut BytesMut, + ) { + let start = out.len(); + out.resize(start + TS_PACKET_SIZE, 0xFF); + let pkt = &mut out[start..start + TS_PACKET_SIZE]; + + pkt[0] = SYNC_BYTE; + pkt[1] = new_packet[1] & 0x1F; + pkt[2] = new_packet[2]; + + // Check if the current packet has a PCR (need at least 7 adaptation bytes: 1 flag + 6 PCR). + let new_pkt_has_pcr = { + let afc = (new_packet[3] >> 4) & 0b11; + if afc == 2 || afc == 3 { + let adaptation_len = new_packet[4] as usize; + adaptation_len >= 7 && (new_packet[5] & ADAPTATION_FIELD_FLAG_PCR) != 0 + } else { + false + } + }; + + // AFC=2 (Adaptation Only), Scrambling=00 (Unscrambled), CC=cc + pkt[3] = 0x20 | (cc & 0x0F); + + // Adaptation Field covers rest of packet (183 bytes) + pkt[4] = 183; + + // If we contain a PCR, inject it. Otherwise just Discontinuity. + if new_pkt_has_pcr { + if let Some(base_pcr) = first_pcr { + pkt[5] = 0x80 | 0x10; // Discontinuity (0x80) + PCR Flag (0x10) + let offset_27mhz = pcr_offset_27mhz(timestamp_offset); + let new_pcr = add_pcr_offset_27mhz(base_pcr, offset_27mhz); + let pcr_bytes = encode_pcr(new_pcr); + pkt[6..12].copy_from_slice(&pcr_bytes); + } else { + pkt[5] = 0x80; + } + } else { + pkt[5] = 0x80; // Discontinuity Indicator Only + } + } + + pub(super) fn rewrite_layout_timestamps( + bytes: &mut BytesMut, + layout: &HlsFiniteTsLayout, + timestamp_offset: u64, + ) -> Result<(), HlsFiniteTsLayoutError> { + if timestamp_offset == 0 { + return Ok(()); + } + for location in layout.pcr_fields.iter().copied() { + let original = decode_pcr_at_location(bytes, location)?; + let adjusted = add_pcr_offset_27mhz(original, pcr_offset_27mhz(timestamp_offset)); + let end = location.byte_offset.checked_add(6).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + let destination = + bytes.get_mut(location.byte_offset..end).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + destination.copy_from_slice(&encode_pcr(adjusted)); + } + for location in layout.timestamp_fields.iter().copied() { + let original = gather_timestamp_bytes(bytes, location)?; + let prefix = original[0] & 0xF0; + let mut encoded = encode_timestamp(add_pts_dts_offset(decode_timestamp(&original), timestamp_offset)); + encoded[0] = (encoded[0] & 0x0F) | prefix; + for (marker_index, (source_offset, value)) in location.byte_offsets.into_iter().zip(encoded).enumerate() { + let destination = + bytes.get_mut(source_offset).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + *destination = value; + if matches!(marker_index, 0 | 2 | 4) && value & 1 == 0 { + return Err(HlsFiniteTsLayoutError::InvalidPesTimestampField { + pid: location.pid, + kind: location.kind, + }); + } + } + } + Ok(()) + } + + pub(super) fn rewrite_source_packet_timestamps( + &self, + bytes: &mut BytesMut, + output_start: usize, + layout: &HlsFiniteTsLayout, + packet_layout: HlsFiniteTsPacketLayout, + timestamp_offset: u64, + ) -> Result<(), HlsFiniteTsLayoutError> { + if timestamp_offset == 0 { + return Ok(()); + } + let packet_end = packet_layout.packet_start.saturating_add(TS_PACKET_SIZE); + if let Some(pcr_field_index) = packet_layout.pcr_field_index { + let location = layout + .pcr_fields + .get(pcr_field_index) + .copied() + .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + let original = decode_pcr_at_location(&self.buffer, location)?; + let adjusted = add_pcr_offset_27mhz(original, pcr_offset_27mhz(timestamp_offset)); + let relative_offset = location + .byte_offset + .checked_sub(packet_layout.packet_start) + .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + let destination_start = + output_start.checked_add(relative_offset).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + let destination_end = + destination_start.checked_add(6).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + bytes + .get_mut(destination_start..destination_end) + .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)? + .copy_from_slice(&encode_pcr(adjusted)); + } + let field_indices = layout + .packet_timestamp_field_indices + .get(packet_layout.timestamp_field_indices_start..packet_layout.timestamp_field_indices_end) + .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + for field_index in field_indices.iter().copied() { + let location = layout + .timestamp_fields + .get(field_index) + .copied() + .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + let original = gather_timestamp_bytes(&self.buffer, location)?; + let prefix = original[0] & 0xF0; + let mut encoded = encode_timestamp(add_pts_dts_offset(decode_timestamp(&original), timestamp_offset)); + encoded[0] = (encoded[0] & 0x0F) | prefix; + for (source_offset, value) in location.byte_offsets.into_iter().zip(encoded) { + if !(packet_layout.packet_start..packet_end).contains(&source_offset) { + continue; + } + let relative_offset = source_offset + .checked_sub(packet_layout.packet_start) + .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + let destination_offset = output_start + .checked_add(relative_offset) + .ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)?; + *bytes.get_mut(destination_offset).ok_or(HlsFiniteTsLayoutError::InvalidTimestampLocation)? = value; + } + } + Ok(()) + } +} diff --git a/backend/mpegts/src/transport_stream_buffer/tests.rs b/backend/mpegts/src/transport_stream_buffer/tests.rs new file mode 100644 index 000000000..fac39525b --- /dev/null +++ b/backend/mpegts/src/transport_stream_buffer/tests.rs @@ -0,0 +1,222 @@ +use super::{ + clock::{ + append_finite_discontinuity_packet, decode_pcr, decode_pcr_at_location, decode_timestamp, + decode_timestamp_at_location, encode_pcr, encode_timestamp, forward_clock_distance_90khz, + gather_timestamp_bytes, pcr_offset_27mhz, ts_packet_pid, ADAPTATION_FIELD_FLAG_PCR, + HLS_TS_SPLICE_MIN_GAP_TICKS_90KHZ, MAX_PTS_DTS, NULL_PID, SYNC_BYTE, TS_PACKET_SIZE, + }, + layout::build_hls_finite_ts_layout, + pes::inspect_hls_ts_packet, + *, +}; +use bytes::BytesMut; + +fn build_pts_dts_payload_packet(pid: u16, cc: u8, pts: u64, dts: u64) -> [u8; TS_PACKET_SIZE] { + let mut packet = [0xFF_u8; TS_PACKET_SIZE]; + packet[0] = SYNC_BYTE; + packet[1] = 0x40 | ((pid >> 8) as u8 & 0x1F); // PUSI + PID high bits + packet[2] = (pid & 0xFF) as u8; + packet[3] = 0x10 | (cc & 0x0F); // payload only + + let payload = &mut packet[4..]; + payload[0] = 0x00; + payload[1] = 0x00; + payload[2] = 0x01; + payload[3] = 0xE0; + payload[4] = 0x00; + payload[5] = 0x00; + payload[6] = 0x80; + payload[7] = 0xC0; // PTS + DTS present + payload[8] = 0x0A; + + let mut pts_bytes = encode_timestamp(pts); + pts_bytes[0] = (pts_bytes[0] & 0x0F) | 0x30; + payload[9..14].copy_from_slice(&pts_bytes); + + let mut dts_bytes = encode_timestamp(dts); + dts_bytes[0] = (dts_bytes[0] & 0x0F) | 0x10; + payload[14..19].copy_from_slice(&dts_bytes); + + packet +} + +fn build_pts_only_payload_packet(pid: u16, cc: u8, pts: u64) -> [u8; TS_PACKET_SIZE] { + let mut packet = [0xFF_u8; TS_PACKET_SIZE]; + packet[0] = SYNC_BYTE; + packet[1] = 0x40 | ((pid >> 8) as u8 & 0x1F); + packet[2] = (pid & 0xFF) as u8; + packet[3] = 0x10 | (cc & 0x0F); + let payload = &mut packet[4..]; + payload[..9].copy_from_slice(&[0x00, 0x00, 0x01, 0xE0, 0x00, 0x00, 0x80, 0x80, 0x05]); + payload[9..14].copy_from_slice(&encode_timestamp(pts)); + packet +} + +fn pts_only_pes_header(pts: u64) -> [u8; 14] { + let mut header = [0u8; 14]; + header[..9].copy_from_slice(&[0x00, 0x00, 0x01, 0xE0, 0x00, 0x00, 0x80, 0x80, 0x05]); + header[9..14].copy_from_slice(&encode_timestamp(pts)); + header +} + +fn pts_dts_pes_header(pts: u64, dts: u64) -> [u8; 19] { + let mut header = [0u8; 19]; + header[..9].copy_from_slice(&[0x00, 0x00, 0x01, 0xE0, 0x00, 0x00, 0x80, 0xC0, 0x0A]); + let mut pts_bytes = encode_timestamp(pts); + pts_bytes[0] = (pts_bytes[0] & 0x0F) | 0x30; + header[9..14].copy_from_slice(&pts_bytes); + let mut dts_bytes = encode_timestamp(dts); + dts_bytes[0] = (dts_bytes[0] & 0x0F) | 0x10; + header[14..19].copy_from_slice(&dts_bytes); + header +} + +fn build_split_pes_header_packets( + pid: u16, + first_cc: u8, + header: &[u8], + payload_lengths: &[usize], +) -> Vec<[u8; TS_PACKET_SIZE]> { + let mut header_offset = 0usize; + payload_lengths + .iter() + .copied() + .enumerate() + .map(|(packet_index, payload_length)| { + assert!((1..=184).contains(&payload_length)); + let mut packet = [0xFF_u8; TS_PACKET_SIZE]; + let packet_index = u8::try_from(packet_index).expect("test packet index fits"); + packet[0] = SYNC_BYTE; + packet[1] = ((pid >> 8) as u8 & 0x1F) | (u8::from(packet_index == 0) * 0x40); + packet[2] = (pid & 0xFF) as u8; + packet[3] = if payload_length == 184 { + 0x10 | (first_cc.wrapping_add(packet_index) & 0x0F) + } else { + 0x30 | (first_cc.wrapping_add(packet_index) & 0x0F) + }; + let payload_offset = if payload_length == 184 { + 4 + } else { + let adaptation_length = 183usize.saturating_sub(payload_length); + packet[4] = u8::try_from(adaptation_length).expect("test adaptation length fits"); + packet[5] = 0; + 5 + adaptation_length + }; + let remaining = header.len().saturating_sub(header_offset); + let copied = remaining.min(payload_length); + packet[payload_offset..payload_offset + copied] + .copy_from_slice(&header[header_offset..header_offset + copied]); + header_offset = header_offset.saturating_add(copied); + packet + }) + .collect() +} + +fn build_pts_dts_pcr_packet(pid: u16, cc: u8, pts: u64, dts: u64, pcr_90khz: u64) -> [u8; TS_PACKET_SIZE] { + let mut packet = [0xFF_u8; TS_PACKET_SIZE]; + packet[0] = SYNC_BYTE; + packet[1] = 0x40 | ((pid >> 8) as u8 & 0x1F); + packet[2] = (pid & 0xFF) as u8; + packet[3] = 0x30 | (cc & 0x0F); + packet[4] = 7; + packet[5] = ADAPTATION_FIELD_FLAG_PCR; + packet[6..12].copy_from_slice(&encode_pcr(pcr_90khz.saturating_mul(300))); + + let payload = &mut packet[12..]; + payload[0] = 0x00; + payload[1] = 0x00; + payload[2] = 0x01; + payload[3] = 0xE0; + payload[4] = 0x00; + payload[5] = 0x00; + payload[6] = 0x80; + payload[7] = 0xC0; + payload[8] = 0x0A; + + let mut pts_bytes = encode_timestamp(pts); + pts_bytes[0] = (pts_bytes[0] & 0x0F) | 0x30; + payload[9..14].copy_from_slice(&pts_bytes); + let mut dts_bytes = encode_timestamp(dts); + dts_bytes[0] = (dts_bytes[0] & 0x0F) | 0x10; + payload[14..19].copy_from_slice(&dts_bytes); + packet +} + +fn packet_timestamps(packet: &[u8]) -> (u64, u64, u64) { + let layout = build_hls_finite_ts_layout(packet).expect("test packet layout"); + let pts = layout + .timestamp_fields + .iter() + .copied() + .find(|field| field.kind == HlsTsTimestampFieldKind::Pts) + .and_then(|field| decode_timestamp_at_location(packet, field).ok()) + .expect("test packet PTS"); + let dts = layout + .timestamp_fields + .iter() + .copied() + .find(|field| field.kind == HlsTsTimestampFieldKind::Dts) + .and_then(|field| decode_timestamp_at_location(packet, field).ok()) + .expect("test packet DTS"); + let pcr = layout + .pcr_fields + .first() + .copied() + .and_then(|field| decode_pcr_at_location(packet, field).ok()) + .expect("test packet PCR") + / 300; + (pts, dts, pcr) +} + +fn build_adaptation_only_packet(pid: u16, cc: u8) -> [u8; TS_PACKET_SIZE] { + let mut packet = [0xFF_u8; TS_PACKET_SIZE]; + packet[0] = SYNC_BYTE; + packet[1] = (pid >> 8) as u8 & 0x1F; + packet[2] = (pid & 0xFF) as u8; + packet[3] = 0x20 | (cc & 0x0F); // adaptation only + packet[4] = 183; + packet[5] = 0; + packet +} + +fn build_pcr_only_packet(pid: u16, cc: u8, pcr_90khz: u64) -> [u8; TS_PACKET_SIZE] { + let mut packet = build_adaptation_only_packet(pid, cc); + packet[5] = ADAPTATION_FIELD_FLAG_PCR; + packet[6..12].copy_from_slice(&encode_pcr(pcr_90khz.saturating_mul(300))); + packet +} + +fn build_payload_packet(pid: u16, cc: u8) -> [u8; TS_PACKET_SIZE] { + let mut packet = [0xFF_u8; TS_PACKET_SIZE]; + packet[0] = SYNC_BYTE; + packet[1] = (pid >> 8) as u8 & 0x1F; + packet[2] = (pid & 0xFF) as u8; + packet[3] = 0x10 | (cc & 0x0F); + packet +} + +fn assert_ffmpeg_compatible_continuity(bytes: &[u8]) { + assert_eq!(bytes.len() % TS_PACKET_SIZE, 0, "unaligned MPEG-TS fixture"); + let mut last_continuity: [Option; 8192] = [None; 8192]; + for (packet_index, packet) in bytes.as_chunks::().0.iter().enumerate() { + let evidence = inspect_hls_ts_packet(packet, packet_index.saturating_mul(TS_PACKET_SIZE)) + .expect("valid MPEG-TS fixture packet"); + if evidence.pid == NULL_PID { + continue; + } + if let Some(previous) = last_continuity[usize::from(evidence.pid)] { + let expected = if evidence.has_payload() { previous.wrapping_add(1) & 0x0F } else { previous }; + assert!( + evidence.discontinuity || evidence.continuity_counter == expected, + "PID {} continuity failed at packet {packet_index}: expected {expected}, actual {}", + evidence.pid, + evidence.continuity_counter + ); + } + last_continuity[usize::from(evidence.pid)] = Some(evidence.continuity_counter); + } +} + +mod policy; +mod terminal; +mod transport; diff --git a/backend/mpegts/src/transport_stream_buffer/tests/policy.rs b/backend/mpegts/src/transport_stream_buffer/tests/policy.rs new file mode 100644 index 000000000..0b63a82e0 --- /dev/null +++ b/backend/mpegts/src/transport_stream_buffer/tests/policy.rs @@ -0,0 +1,53 @@ +use super::*; + +#[test] +fn provisioning_asset_uses_exact_per_pid_presentation_duration() { + let buffer = TransportStreamBuffer::new( + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/panel_api_provisioning.ts")) + .to_vec(), + ); + + assert_eq!(buffer.duration_ticks_90khz(), Some(182_400)); + assert_eq!(buffer.duration_ms(), Some(2_027)); +} + +#[test] +fn per_pid_presentation_duration_unwraps_33_bit_audio_and_video_clocks() { + let video_pid = 0x0101; + let audio_pid = 0x0102; + let packets = [ + build_pts_only_payload_packet(video_pid, 0, MAX_PTS_DTS - 1_080), + build_pts_only_payload_packet(audio_pid, 0, MAX_PTS_DTS - 1_000), + build_pts_only_payload_packet(video_pid, 1, 1_920), + build_pts_only_payload_packet(audio_pid, 1, 920), + build_pts_only_payload_packet(video_pid, 2, 4_920), + build_pts_only_payload_packet(audio_pid, 2, 2_840), + ]; + let raw = packets.into_iter().flatten().collect::>(); + let buffer = TransportStreamBuffer::new(raw); + let duration = buffer.finite_hls_presentation_duration.as_ref().expect("per-PID wrapped duration"); + + assert_eq!(duration.source, HlsTsPresentationClockSource::Pts); + assert_eq!(duration.first_presentation_clock_90khz, MAX_PTS_DTS - 1_080); + assert_eq!(duration.end_exclusive_clock_90khz, MAX_PTS_DTS + 7_920); + assert_eq!(duration.duration_ticks_90khz, 9_000); + assert_eq!( + duration.timelines.as_ref(), + [ + HlsTsPidPresentationTimeline { + pid: video_pid, + first_pts_90khz: MAX_PTS_DTS - 1_080, + last_pts_90khz: MAX_PTS_DTS + 4_920, + cadence_ticks_90khz: 3_000, + end_exclusive_90khz: MAX_PTS_DTS + 7_920, + }, + HlsTsPidPresentationTimeline { + pid: audio_pid, + first_pts_90khz: MAX_PTS_DTS - 1_000, + last_pts_90khz: MAX_PTS_DTS + 2_840, + cadence_ticks_90khz: 1_920, + end_exclusive_90khz: MAX_PTS_DTS + 4_760, + }, + ] + ); +} diff --git a/backend/mpegts/src/transport_stream_buffer/tests/terminal.rs b/backend/mpegts/src/transport_stream_buffer/tests/terminal.rs new file mode 100644 index 000000000..d840e7f36 --- /dev/null +++ b/backend/mpegts/src/transport_stream_buffer/tests/terminal.rs @@ -0,0 +1,524 @@ +use super::*; + +#[test] +fn runtime_custom_assets_use_exact_per_pid_presentation_duration() { + let fixtures: [(&str, &[u8]); 6] = [ + ( + "channel_unavailable.ts", + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")) + .as_slice(), + ), + ( + "hls_session_or_lease_expired.ts", + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/hls_session_or_lease_expired.ts" + )) + .as_slice(), + ), + ( + "low_priority_preempted.ts", + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/low_priority_preempted.ts")) + .as_slice(), + ), + ( + "provider_connections_exhausted.ts", + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/provider_connections_exhausted.ts" + )) + .as_slice(), + ), + ( + "user_account_expired.ts", + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/user_account_expired.ts")) + .as_slice(), + ), + ( + "user_connections_exhausted.ts", + include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../test/fixtures/hls/user_connections_exhausted.ts" + )) + .as_slice(), + ), + ]; + for (name, bytes) in fixtures { + let buffer = TransportStreamBuffer::new(bytes.to_vec()); + assert_eq!(buffer.duration_ticks_90khz(), Some(902_400), "{name} tick duration"); + assert_eq!(buffer.duration_ms(), Some(10_027), "{name} rounded HLS duration"); + } +} + +#[test] +fn custom_asset_stride_does_not_overlap_previous_audio_presentation() { + let buffer = TransportStreamBuffer::new( + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")).to_vec(), + ); + let duration = buffer.duration_ticks_90khz().expect("exact custom asset duration"); + let segment_zero = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 0, + continuity_seed: 0, + logical_segment_index: 0, + }) + .expect("segment zero"); + let segment_one = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: duration, + continuity_seed: 0, + logical_segment_index: 1, + }) + .expect("segment one"); + let zero = TransportStreamBuffer::new(segment_zero.to_vec()); + let one = TransportStreamBuffer::new(segment_one.to_vec()); + let zero_audio = zero + .finite_hls_presentation_duration + .as_ref() + .and_then(|duration| duration.timelines.iter().find(|timeline| timeline.cadence_ticks_90khz == 1_920)) + .expect("segment zero audio timeline"); + let one_audio = one + .finite_hls_presentation_duration + .as_ref() + .and_then(|duration| duration.timelines.iter().find(|timeline| timeline.cadence_ticks_90khz == 1_920)) + .expect("segment one audio timeline"); + + assert_eq!(duration, 902_400); + assert_ne!(duration, 901_650); + assert!(one_audio.first_pts_90khz >= zero_audio.end_exclusive_90khz); + assert!( + zero_audio.first_pts_90khz.saturating_add(901_650) < zero_audio.end_exclusive_90khz, + "the previous mixed-clock stride would overlap the final audio presentation" + ); +} + +#[test] +fn terminal_splice_anchor_places_first_terminal_clock_after_live_tail() { + let live_last_clock_90khz = 1_003_618_800; + let live = HlsTsTimestampProfile { + first_clock_90khz: live_last_clock_90khz - 900_000, + last_clock_90khz: live_last_clock_90khz, + span_ticks_90khz: 900_000, + observed_pts_or_dts: true, + observed_pcr: true, + }; + let terminal = HlsTsTimestampProfile { + first_clock_90khz: 0, + last_clock_90khz: 902_400, + span_ticks_90khz: 902_400, + observed_pts_or_dts: true, + observed_pcr: true, + }; + + let anchor = HlsTsSpliceAnchor::between(live, terminal).expect("valid modular splice"); + + assert_eq!( + forward_clock_distance_90khz(live_last_clock_90khz, anchor.terminal_first_clock), + HLS_TS_SPLICE_MIN_GAP_TICKS_90KHZ + ); + assert_eq!(anchor.timestamp_delta_ticks, anchor.terminal_first_clock); + assert!( + forward_clock_distance_90khz(live_last_clock_90khz, anchor.terminal_first_clock) < 90_000, + "splice must not contain the observed near-full-cycle jump" + ); +} + +#[test] +fn terminal_splice_preserves_terminal_asset_audio_video_offset() { + let audio = build_pts_dts_payload_packet(0x0102, 0, 0, 0); + let video = build_pts_dts_payload_packet(0x0101, 0, 1_920, 1_920); + let mut raw = Vec::new(); + raw.extend_from_slice(&audio); + raw.extend_from_slice(&video); + raw.extend_from_slice(&build_pts_dts_payload_packet(0x0102, 1, 1_920, 1_920)); + raw.extend_from_slice(&build_pts_dts_payload_packet(0x0101, 1, 4_920, 4_920)); + let buffer = TransportStreamBuffer::new(raw); + let asset_profile = buffer.finite_hls_timestamp_profile().expect("asset timestamp profile"); + let live = HlsTsTimestampProfile { + first_clock_90khz: 1_003_000_000, + last_clock_90khz: 1_003_618_800, + span_ticks_90khz: 618_800, + observed_pts_or_dts: true, + observed_pcr: false, + }; + let anchor = HlsTsSpliceAnchor::between(live, asset_profile).expect("splice anchor"); + let prepared = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 0, + continuity_seed: 0, + logical_segment_index: 0, + }) + .expect("relative segment"); + let anchored = buffer + .finalize_prepared_finite_hls_segment( + &prepared, + HlsFiniteTsFinalizeSpec { + additional_timestamp_offset_ticks_90khz: anchor.timestamp_delta_ticks, + discontinuity: HlsFiniteTsDiscontinuityMode::None, + }, + ) + .expect("anchored segment"); + let audio_pts = decode_timestamp(&anchored[13..18]); + let video_pts = decode_timestamp(&anchored[TS_PACKET_SIZE + 13..TS_PACKET_SIZE + 18]); + + assert_eq!(forward_clock_distance_90khz(audio_pts, video_pts), 1_920); +} + +#[test] +fn terminal_splice_anchor_wraps_pts_dts_and_pcr_exactly() { + let original_clock = 10; + let packet = build_pts_dts_pcr_packet(0x0100, 7, original_clock, original_clock, original_clock); + let next_packet = build_pts_dts_pcr_packet(0x0100, 8, 3_010, 3_010, 3_010); + let mut raw = packet.to_vec(); + raw.extend_from_slice(&next_packet); + let buffer = TransportStreamBuffer::new(raw); + let asset_profile = buffer.finite_hls_timestamp_profile().expect("asset timestamp profile"); + let live_last_clock_90khz = MAX_PTS_DTS - 40; + let live = HlsTsTimestampProfile { + first_clock_90khz: live_last_clock_90khz - 90_000, + last_clock_90khz: live_last_clock_90khz, + span_ticks_90khz: 90_000, + observed_pts_or_dts: true, + observed_pcr: true, + }; + let anchor = HlsTsSpliceAnchor::between(live, asset_profile).expect("wrapped splice anchor"); + let prepared = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 0, + continuity_seed: 0, + logical_segment_index: 0, + }) + .expect("relative segment"); + let anchored = buffer + .finalize_prepared_finite_hls_segment( + &prepared, + HlsFiniteTsFinalizeSpec { + additional_timestamp_offset_ticks_90khz: anchor.timestamp_delta_ticks, + discontinuity: HlsFiniteTsDiscontinuityMode::None, + }, + ) + .expect("wrapped anchored segment"); + + assert_eq!(anchor.terminal_first_clock, 50); + assert_eq!(packet_timestamps(&anchored), (50, 50, 50)); + assert_eq!(forward_clock_distance_90khz(live_last_clock_90khz, 50), 90); +} + +#[test] +fn first_terminal_segment_marks_discontinuity_for_each_non_null_pid() { + let mut raw = Vec::new(); + raw.extend_from_slice(&build_payload_packet(0x0000, 9)); + raw.extend_from_slice(&build_payload_packet(0x0100, 3)); + raw.extend_from_slice(&build_pts_dts_payload_packet(0x0101, 7, 90_000, 87_000)); + raw.extend_from_slice(&build_pts_dts_payload_packet(0x0102, 5, 90_000, 90_000)); + raw.extend_from_slice(&build_pts_dts_pcr_packet(0x0103, 4, 90_000, 90_000, 90_000)); + raw.extend_from_slice(&build_payload_packet(NULL_PID, 12)); + let buffer = TransportStreamBuffer::new(raw); + let prepared = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 0, + continuity_seed: 2, + logical_segment_index: 0, + }) + .expect("relative segment"); + let finalized = buffer + .finalize_prepared_finite_hls_segment( + &prepared, + HlsFiniteTsFinalizeSpec { + additional_timestamp_offset_ticks_90khz: 90, + discontinuity: HlsFiniteTsDiscontinuityMode::FirstPacketPerPid, + }, + ) + .expect("splice segment"); + + let packets = finalized.as_chunks::().0.iter().collect::>(); + let mut marked_pids = Vec::new(); + for pair in packets.windows(2) { + let marker = pair[0]; + let following = pair[1]; + if (marker[3] >> 4) & 0b11 == 0b10 && marker[4] == 183 && marker[5] & 0x80 != 0 { + assert_eq!(ts_packet_pid(marker), ts_packet_pid(following)); + assert_eq!((marker[3] & 0x0F).wrapping_add(1) & 0x0F, following[3] & 0x0F); + marked_pids.push(ts_packet_pid(marker)); + } + } + marked_pids.sort_unstable(); + + assert_eq!(marked_pids, vec![0x0000, 0x0100, 0x0101, 0x0102, 0x0103]); + assert!(!marked_pids.contains(&NULL_PID)); +} + +#[test] +fn finalized_terminal_asset_preserves_tracks_and_ffmpeg_continuity() { + const TERMINAL_ASSET_BYTES: &[u8] = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); + let buffer = TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); + let expected_signature = buffer.finite_hls_track_signature().expect("terminal asset track signature"); + let prepared = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 0, + continuity_seed: 0, + logical_segment_index: 0, + }) + .expect("relative terminal asset"); + let finalized = buffer + .finalize_prepared_finite_hls_segment( + &prepared, + HlsFiniteTsFinalizeSpec { + additional_timestamp_offset_ticks_90khz: 1_003_618_890, + discontinuity: HlsFiniteTsDiscontinuityMode::FirstPacketPerPid, + }, + ) + .expect("anchored terminal asset"); + + assert_ffmpeg_compatible_continuity(&finalized); + let outcome = crate::ts_inspector::inspect_mpeg_ts( + std::io::Cursor::new(finalized), + crate::ts_inspector::HlsTsProbeProtection::Clear, + crate::ts_inspector::HlsTsProbeBudget::default(), + ) + .expect("finalized terminal asset remains inspectable"); + assert_eq!(outcome, crate::ts_inspector::HlsTsProbeOutcome::Found(expected_signature)); +} + +#[test] +fn prepared_terminal_finalization_rejects_packet_layout_mismatch() { + let packet = build_pts_dts_pcr_packet(0x0100, 0, 90_000, 87_000, 86_000); + let buffer = TransportStreamBuffer::new(packet.to_vec()); + let prepared = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 0, + continuity_seed: 0, + logical_segment_index: 0, + }) + .expect("relative terminal segment"); + let mut mismatched = BytesMut::from(prepared.as_ref()); + mismatched[2] ^= 1; + + assert_eq!( + buffer.finalize_prepared_finite_hls_segment( + &mismatched.freeze(), + HlsFiniteTsFinalizeSpec { + additional_timestamp_offset_ticks_90khz: 1, + discontinuity: HlsFiniteTsDiscontinuityMode::FirstPacketPerPid, + }, + ), + Err(HlsFiniteTsRenderError::PreparedLayoutMismatch) + ); +} + +#[test] +fn later_terminal_segments_continue_cc_without_repeating_splice_marker() { + let first = build_pts_dts_pcr_packet(0x0101, 7, 90_000, 87_000, 90_000); + let second = build_pts_dts_payload_packet(0x0102, 5, 90_000, 90_000); + let mut raw = Vec::new(); + raw.extend_from_slice(&first); + raw.extend_from_slice(&second); + raw.extend_from_slice(&build_pts_dts_pcr_packet(0x0101, 8, 180_000, 177_000, 180_000)); + raw.extend_from_slice(&build_pts_dts_payload_packet(0x0102, 6, 180_000, 180_000)); + let buffer = TransportStreamBuffer::new(raw); + let duration = buffer.duration_ticks_90khz().expect("asset duration"); + let relative_zero = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 0, + continuity_seed: 0, + logical_segment_index: 0, + }) + .expect("relative zero"); + let relative_one = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: duration, + continuity_seed: 0, + logical_segment_index: 1, + }) + .expect("relative one"); + let finalized_zero = buffer + .finalize_prepared_finite_hls_segment( + &relative_zero, + HlsFiniteTsFinalizeSpec { + additional_timestamp_offset_ticks_90khz: 1_000_000, + discontinuity: HlsFiniteTsDiscontinuityMode::FirstPacketPerPid, + }, + ) + .expect("anchored zero"); + let finalized_one = buffer + .finalize_prepared_finite_hls_segment( + &relative_one, + HlsFiniteTsFinalizeSpec { + additional_timestamp_offset_ticks_90khz: 1_000_000, + discontinuity: HlsFiniteTsDiscontinuityMode::None, + }, + ) + .expect("anchored one"); + + assert_eq!(finalized_one.len(), relative_one.len()); + assert!(finalized_one + .as_chunks::() + .0 + .iter() + .all(|packet| !((packet[3] >> 4) & 0b11 == 0b10 && packet[4] == 183 && packet[5] & 0x80 != 0))); + let zero_payload = finalized_zero + .as_chunks::() + .0 + .iter() + .filter(|packet| (packet[3] >> 4) & 0b11 != 0b10) + .collect::>(); + let one_payload = finalized_one.as_chunks::().0.iter().collect::>(); + for (zero, one) in zero_payload.into_iter().zip(one_payload) { + assert_eq!(ts_packet_pid(zero), ts_packet_pid(one)); + assert_eq!((zero[3] & 0x0F).wrapping_add(2) & 0x0F, one[3] & 0x0F); + } + let mut concatenated = finalized_zero.to_vec(); + concatenated.extend_from_slice(&finalized_one); + assert_ffmpeg_compatible_continuity(&concatenated); +} + +#[test] +fn live_to_terminal_splice_has_monotone_pts_dts_pcr_and_valid_cc_reset() { + let live_first_clock = 1_003_528_800; + let live_last_clock = 1_003_618_800; + let mut live_bytes = Vec::new(); + live_bytes.extend_from_slice(&build_pts_dts_pcr_packet( + 0x0101, + 13, + live_first_clock, + live_first_clock, + live_first_clock, + )); + live_bytes.extend_from_slice(&build_pts_dts_pcr_packet( + 0x0101, + 14, + live_last_clock, + live_last_clock, + live_last_clock, + )); + let live_buffer = TransportStreamBuffer::new(live_bytes.clone()); + let live_profile = live_buffer.finite_hls_timestamp_profile().expect("live tail profile"); + + let mut terminal_bytes = Vec::new(); + terminal_bytes.extend_from_slice(&build_pts_dts_pcr_packet(0x0101, 0, 0, 0, 0)); + terminal_bytes.extend_from_slice(&build_pts_dts_pcr_packet(0x0101, 1, 90_000, 90_000, 90_000)); + let terminal_buffer = TransportStreamBuffer::new(terminal_bytes); + let terminal_profile = terminal_buffer.finite_hls_timestamp_profile().expect("terminal asset profile"); + let anchor = HlsTsSpliceAnchor::between(live_profile, terminal_profile).expect("splice anchor"); + let duration = terminal_buffer.duration_ticks_90khz().expect("terminal duration"); + let relative_zero = terminal_buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 0, + continuity_seed: 0, + logical_segment_index: 0, + }) + .expect("terminal zero"); + let relative_one = terminal_buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: duration, + continuity_seed: 0, + logical_segment_index: 1, + }) + .expect("terminal one"); + let terminal_zero = terminal_buffer + .finalize_prepared_finite_hls_segment( + &relative_zero, + HlsFiniteTsFinalizeSpec { + additional_timestamp_offset_ticks_90khz: anchor.timestamp_delta_ticks, + discontinuity: HlsFiniteTsDiscontinuityMode::FirstPacketPerPid, + }, + ) + .expect("anchored terminal zero"); + let terminal_one = terminal_buffer + .finalize_prepared_finite_hls_segment( + &relative_one, + HlsFiniteTsFinalizeSpec { + additional_timestamp_offset_ticks_90khz: anchor.timestamp_delta_ticks, + discontinuity: HlsFiniteTsDiscontinuityMode::None, + }, + ) + .expect("anchored terminal one"); + + let zero_packets = terminal_zero.as_chunks::().0.iter().collect::>(); + assert_eq!((zero_packets[0][3] >> 4) & 0b11, 0b10); + assert_eq!(zero_packets[0][5] & 0x80, 0x80); + assert_eq!((zero_packets[0][3] & 0x0F).wrapping_add(1) & 0x0F, zero_packets[1][3] & 0x0F); + let terminal_one_packets = terminal_one.as_chunks::().0; + assert!(terminal_one_packets.iter().all(|packet| !((packet[3] >> 4) & 0b11 == 0b10 && packet[5] & 0x80 != 0))); + + let zero_first = packet_timestamps(zero_packets[1]); + let zero_last = packet_timestamps(zero_packets[2]); + let one_first = packet_timestamps(&terminal_one[..TS_PACKET_SIZE]); + assert_eq!(forward_clock_distance_90khz(live_last_clock, zero_first.0), 90); + assert!(forward_clock_distance_90khz(zero_first.0, zero_last.0) < MAX_PTS_DTS / 2); + assert!(forward_clock_distance_90khz(zero_last.0, one_first.0) < MAX_PTS_DTS / 2); + assert_eq!(forward_clock_distance_90khz(zero_last.0, one_first.0), duration.saturating_sub(90_000)); + assert_eq!(zero_first.0, zero_first.1); + assert_eq!(zero_first.0, zero_first.2); + assert_eq!(one_first.0, one_first.1); + assert_eq!(one_first.0, one_first.2); + + let mut concatenated = live_bytes; + concatenated.extend_from_slice(&terminal_zero); + concatenated.extend_from_slice(&terminal_one); + assert_ffmpeg_compatible_continuity(&concatenated); + let mut scanner = HlsTsTimestampProfileScanner::new(duration.saturating_mul(3)); + for packet in concatenated.as_chunks::().0 { + scanner.push_aligned_packet(packet); + } + let combined = scanner.finish().expect("combined splice profile"); + assert!(combined.span_ticks_90khz < duration.saturating_mul(3)); +} + +#[test] +fn hls_prepared_terminal_bundle_pcr_uses_300_factor_and_large_offsets_wrap_exactly() { + let base_pcr_90khz = 123_456; + let large_offset_90khz = u64::MAX - 37; + let packet = build_pts_dts_pcr_packet(0x0100, 7, 90_000, 87_000, base_pcr_90khz); + let buffer = TransportStreamBuffer::new(packet.to_vec()); + + let rendered = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: large_offset_90khz, + continuity_seed: 0, + logical_segment_index: 11, + }) + .expect("finite segment with a large timestamp offset"); + + let actual_pcr_27mhz = decode_pcr(&rendered[6..12]); + let expected_offset_27mhz = (u128::from(large_offset_90khz) * 300_u128) % u128::from(MAX_PCR); + let expected_pcr_27mhz = + u64::try_from((u128::from(base_pcr_90khz) * 300_u128 + expected_offset_27mhz) % u128::from(MAX_PCR)) + .expect("PCR modulo fits in u64"); + let legacy_wrapped_offset = large_offset_90khz.wrapping_mul(300) % MAX_PCR; + + assert_eq!(pcr_offset_27mhz(1), 300); + assert_ne!( + pcr_offset_27mhz(large_offset_90khz), + legacy_wrapped_offset, + "large offsets must be multiplied before modulo without u64 wrap" + ); + assert_eq!(actual_pcr_27mhz, expected_pcr_27mhz); +} + +#[test] +fn hls_prepared_terminal_bundle_pts_and_dts_large_offsets_wrap_modulo_33_bits() { + let presentation_timestamp = MAX_PTS_DTS - 123; + let decoding_timestamp = MAX_PTS_DTS - 456; + let large_offset_90khz = u64::MAX - 37; + let packet = build_pts_dts_pcr_packet(0x0100, 7, presentation_timestamp, decoding_timestamp, 90_000); + let buffer = TransportStreamBuffer::new(packet.to_vec()); + + let rendered = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: large_offset_90khz, + continuity_seed: 0, + logical_segment_index: 11, + }) + .expect("finite segment with wrapped PTS and DTS"); + + let expected_timestamp = |timestamp| { + u64::try_from((u128::from(timestamp) + u128::from(large_offset_90khz)) % u128::from(MAX_PTS_DTS)) + .expect("PTS/DTS modulo fits in u64") + }; + let (rendered_presentation_timestamp, rendered_decoding_timestamp, _) = packet_timestamps(&rendered); + + assert_eq!(rendered_presentation_timestamp, expected_timestamp(presentation_timestamp)); + assert_eq!(rendered_decoding_timestamp, expected_timestamp(decoding_timestamp)); +} diff --git a/backend/mpegts/src/transport_stream_buffer/tests/transport.rs b/backend/mpegts/src/transport_stream_buffer/tests/transport.rs new file mode 100644 index 000000000..320c7fe46 --- /dev/null +++ b/backend/mpegts/src/transport_stream_buffer/tests/transport.rs @@ -0,0 +1,357 @@ +use super::*; + +#[test] +fn finite_discontinuity_marker_sets_ffmpeg_compatible_counter_baseline() { + let cases = [ + ("payload wrap", build_payload_packet(0x0100, 0), 15), + ("payload increment", build_payload_packet(0x0101, 7), 6), + ("adaptation only", build_adaptation_only_packet(0x0102, 5), 5), + ]; + for (name, following, expected_marker_cc) in cases { + let mut output = BytesMut::new(); + append_finite_discontinuity_packet(&following, &mut output); + let marker = &output[..TS_PACKET_SIZE]; + + assert_eq!((marker[3] >> 4) & 0b11, 0b10, "{name} marker AFC"); + assert_eq!(marker[5] & 0x80, 0x80, "{name} discontinuity"); + assert_eq!(marker[3] & 0x0F, expected_marker_cc, "{name} marker CC"); + + output.extend_from_slice(&following); + assert_ffmpeg_compatible_continuity(&output); + } +} + +#[test] +fn discontinuity_packet_does_not_advance_payload_cc() { + let packet = build_pts_dts_payload_packet(0x0100, 7, 90_000, 87_000); + let mut buf = TransportStreamBuffer::new(packet.to_vec()); + let chunk = buf.next_chunk().expect("expected chunk"); + assert!(chunk.len() >= TS_PACKET_SIZE * 2); + + // First emitted packet is injected discontinuity (adaptation-only), + // second is the actual payload packet for the same PID. + let disc_cc = chunk[3] & 0x0F; + let disc_afc = (chunk[3] >> 4) & 0b11; + let payload_cc = chunk[TS_PACKET_SIZE + 3] & 0x0F; + assert_eq!(disc_afc, 0b10); + assert_eq!(disc_cc, payload_cc); +} + +#[test] +fn adaptation_only_packets_keep_same_continuity_counter() { + let packet = build_adaptation_only_packet(0x0011, 5); + let mut buf = TransportStreamBuffer::new(packet.to_vec()); + let chunk = buf.next_chunk().expect("expected chunk"); + // Each of the 7 loop iterations emits a discontinuity packet + the actual packet + // because the single-packet buffer loops on every iteration. + let total_packets = PACKET_COUNT * 2; + assert_eq!(chunk.len(), TS_PACKET_SIZE * total_packets); + + for i in 0..total_packets { + let cc = chunk[i * TS_PACKET_SIZE + 3] & 0x0F; + assert_eq!(cc, 5, "packet {i} CC mismatch"); + } +} + +#[test] +fn finite_hls_segments_are_deterministic_aligned_and_timestamp_shifted() { + let first = build_pts_dts_payload_packet(0x0100, 7, 90_000, 87_000); + let second = build_pts_dts_payload_packet(0x0100, 8, 180_000, 177_000); + let mut raw = Vec::new(); + raw.extend_from_slice(&first); + raw.extend_from_slice(&second); + let buffer = TransportStreamBuffer::new(raw); + let spec = + HlsFiniteTsRenderSpec { timestamp_offset_ticks_90khz: 900_000, continuity_seed: 3, logical_segment_index: 0 }; + + let rendered = buffer.render_finite_hls_segment(spec).expect("finite render"); + let repeated = buffer.render_finite_hls_segment(spec).expect("repeated finite render"); + + assert_eq!(rendered, repeated); + assert_eq!(rendered.len() % TS_PACKET_SIZE, 0); + assert_eq!(rendered[3] & 0x0F, 3); + assert_eq!(rendered[TS_PACKET_SIZE + 3] & 0x0F, 4); + assert_eq!(decode_timestamp(&rendered[13..18]), 990_000); + assert_eq!(decode_timestamp(&rendered[18..23]), 987_000); +} + +#[test] +fn consecutive_finite_hls_segments_advance_cc_pts_dts_and_pcr() { + let first = build_pts_dts_pcr_packet(0x0100, 7, 90_000, 87_000, 90_000); + let second = build_pts_dts_pcr_packet(0x0100, 8, 180_000, 177_000, 180_000); + let mut raw = Vec::new(); + raw.extend_from_slice(&first); + raw.extend_from_slice(&second); + let buffer = TransportStreamBuffer::new(raw); + let duration = buffer.duration_ticks_90khz().expect("transportable duration"); + + let segment_zero = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 0, + continuity_seed: 3, + logical_segment_index: 0, + }) + .expect("first segment"); + let segment_one = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: duration, + continuity_seed: 3, + logical_segment_index: 1, + }) + .expect("second segment"); + + assert_eq!(segment_zero[TS_PACKET_SIZE + 3] & 0x0F, 4); + assert_eq!(segment_one[3] & 0x0F, 5); + assert_eq!(segment_one[TS_PACKET_SIZE + 3] & 0x0F, 6); + + let zero_last = packet_timestamps(&segment_zero[TS_PACKET_SIZE..]); + let one_first = packet_timestamps(&segment_one[..TS_PACKET_SIZE]); + assert!(one_first.0 > zero_last.0, "PTS must advance across segment boundary"); + assert!(one_first.1 > zero_last.1, "DTS must advance across segment boundary"); + assert!(one_first.2 > zero_last.2, "PCR must advance across segment boundary"); +} + +#[test] +fn finite_hls_timestamp_rewrite_wraps_pts_dts_and_pcr_without_overflow() { + let packet = build_pts_dts_pcr_packet( + 0x0100, + 7, + MAX_PTS_DTS.saturating_sub(10), + MAX_PTS_DTS.saturating_sub(20), + MAX_PTS_DTS.saturating_sub(10), + ); + let buffer = TransportStreamBuffer::new(packet.to_vec()); + + let rendered = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 30, + continuity_seed: 0, + logical_segment_index: 0, + }) + .expect("finite wrapped segment"); + + assert_eq!(packet_timestamps(&rendered), (20, 10, 20)); +} + +#[test] +fn split_pts_layout_profile_relative_and_anchored_rewrite_are_exact() { + let mut packets = build_split_pes_header_packets(0x0101, 0, &pts_only_pes_header(90_000), &[12, 2]); + packets.push(build_pts_only_payload_packet(0x0101, 2, 93_000)); + let raw = packets.into_iter().flatten().collect::>(); + let buffer = TransportStreamBuffer::new(raw); + let layout = buffer.finite_hls_layout.as_ref().expect("split PTS layout"); + let pts_fields = layout + .timestamp_fields + .iter() + .copied() + .filter(|field| field.kind == HlsTsTimestampFieldKind::Pts) + .collect::>(); + assert_eq!(pts_fields.len(), 2); + let split_field = pts_fields[0]; + assert_ne!(split_field.byte_offsets[0] / TS_PACKET_SIZE, split_field.byte_offsets[4] / TS_PACKET_SIZE); + assert_eq!( + buffer.finite_hls_timestamp_profile(), + Some(HlsTsTimestampProfile { + first_clock_90khz: 90_000, + last_clock_90khz: 93_000, + span_ticks_90khz: 3_000, + observed_pts_or_dts: true, + observed_pcr: false, + }) + ); + + let relative = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 7_000, + continuity_seed: 0, + logical_segment_index: 0, + }) + .expect("relative split PTS"); + assert_eq!(decode_timestamp_at_location(&relative, split_field).expect("relative split PTS value"), 97_000); + let anchored = buffer + .finalize_prepared_finite_hls_segment( + &relative, + HlsFiniteTsFinalizeSpec { + additional_timestamp_offset_ticks_90khz: 11_000, + discontinuity: HlsFiniteTsDiscontinuityMode::None, + }, + ) + .expect("anchored split PTS"); + assert_eq!(decode_timestamp_at_location(&anchored, split_field).expect("anchored split PTS value"), 108_000); + assert_ne!(decode_timestamp_at_location(&anchored, split_field).expect("rewritten split PTS"), 90_000); + let encoded = gather_timestamp_bytes(&anchored, split_field).expect("anchored split PTS bytes"); + assert!(encoded[0] & 1 != 0 && encoded[2] & 1 != 0 && encoded[4] & 1 != 0); +} + +#[test] +fn split_pts_dts_layout_rewrites_both_fields_across_distinct_packet_boundaries() { + let packets = build_split_pes_header_packets(0x0101, 5, &pts_dts_pes_header(90_000, 87_000), &[11, 5, 3]); + let raw = packets.into_iter().flatten().collect::>(); + let buffer = TransportStreamBuffer::new(raw); + let layout = buffer.finite_hls_layout.as_ref().expect("split PTS+DTS layout"); + let pts = layout + .timestamp_fields + .iter() + .copied() + .find(|field| field.kind == HlsTsTimestampFieldKind::Pts) + .expect("split PTS location"); + let dts = layout + .timestamp_fields + .iter() + .copied() + .find(|field| field.kind == HlsTsTimestampFieldKind::Dts) + .expect("split DTS location"); + assert_ne!(pts.byte_offsets[0] / TS_PACKET_SIZE, pts.byte_offsets[4] / TS_PACKET_SIZE); + assert_ne!(dts.byte_offsets[0] / TS_PACKET_SIZE, dts.byte_offsets[4] / TS_PACKET_SIZE); + assert_ne!(pts.byte_offsets[0] / TS_PACKET_SIZE, dts.byte_offsets[0] / TS_PACKET_SIZE); + + let relative = buffer + .render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 1_000, + continuity_seed: 0, + logical_segment_index: 0, + }) + .expect("relative split PTS+DTS"); + let anchored = buffer + .finalize_prepared_finite_hls_segment( + &relative, + HlsFiniteTsFinalizeSpec { + additional_timestamp_offset_ticks_90khz: 2_000, + discontinuity: HlsFiniteTsDiscontinuityMode::None, + }, + ) + .expect("anchored split PTS+DTS"); + assert_eq!(decode_timestamp_at_location(&anchored, pts), Ok(93_000)); + assert_eq!(decode_timestamp_at_location(&anchored, dts), Ok(90_000)); + for field in [pts, dts] { + let encoded = gather_timestamp_bytes(&anchored, field).expect("split timestamp bytes"); + assert!(encoded[0] & 1 != 0 && encoded[2] & 1 != 0 && encoded[4] & 1 != 0); + } +} + +#[test] +fn timestamp_profile_scanner_reads_split_pts_across_packets() { + let mut packets = build_split_pes_header_packets(0x0101, 0, &pts_only_pes_header(90_000), &[12, 2]); + packets.push(build_pts_only_payload_packet(0x0101, 2, 93_000)); + let mut scanner = HlsTsTimestampProfileScanner::new(10_000); + for packet in packets { + scanner.push_aligned_packet(&packet); + } + + assert_eq!( + scanner.finish(), + Some(HlsTsTimestampProfile { + first_clock_90khz: 90_000, + last_clock_90khz: 93_000, + span_ticks_90khz: 3_000, + observed_pts_or_dts: true, + observed_pcr: false, + }) + ); +} + +#[test] +fn timestamp_profile_scanner_rejects_one_clock_sample() { + let packet = build_pts_only_payload_packet(0x0101, 0, 90_000); + let mut scanner = HlsTsTimestampProfileScanner::new(10_000); + scanner.push_aligned_packet(&packet); + + assert_eq!(scanner.finish(), None); +} + +#[test] +fn split_pts_incomplete_at_eof_fails_with_typed_layout_error() { + let packets = build_split_pes_header_packets(0x0101, 0, &pts_only_pes_header(90_000), &[12, 2]); + let buffer = TransportStreamBuffer::new(packets[0].to_vec()); + + assert!(matches!( + buffer.finite_hls_layout.as_ref(), + Err(HlsFiniteTsLayoutError::IncompletePesTimestampHeader { pid: 0x0101 }) + )); + assert_eq!( + buffer.render_finite_hls_segment(HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 1, + continuity_seed: 0, + logical_segment_index: 0, + }), + Err(HlsFiniteTsRenderError::InvalidAsset) + ); + assert_eq!(buffer.duration_ticks_90khz(), None); +} + +#[test] +fn split_pts_continuity_discontinuity_fails_with_typed_layout_error() { + let mut packets = build_split_pes_header_packets(0x0101, 0, &pts_only_pes_header(90_000), &[12, 2]); + packets[1][3] = (packets[1][3] & 0xF0) | 3; + let raw = packets.into_iter().flatten().collect::>(); + let buffer = TransportStreamBuffer::new(raw); + + assert!(matches!( + buffer.finite_hls_layout.as_ref(), + Err(HlsFiniteTsLayoutError::PesTimestampContinuityDiscontinuity { pid: 0x0101, expected: Some(1), actual: 3 }) + )); +} + +#[test] +fn pcr_fallback_uses_observed_cadence_and_rejects_a_single_sample() { + let pid = 0x0101; + let packets = [build_pcr_only_packet(pid, 0, 90_000), build_pcr_only_packet(pid, 0, 91_920)]; + let buffer = TransportStreamBuffer::new(packets.into_iter().flatten().collect()); + let duration = buffer.finite_hls_presentation_duration.as_ref().expect("PCR fallback duration"); + + assert_eq!(duration.source, HlsTsPresentationClockSource::PcrFallback); + assert_eq!(duration.first_presentation_clock_90khz, 90_000); + assert_eq!(duration.end_exclusive_clock_90khz, 93_840); + assert_eq!(duration.duration_ticks_90khz, 3_840); + assert_eq!( + duration.timelines.as_ref(), + [HlsTsPidPresentationTimeline { + pid, + first_pts_90khz: 90_000, + last_pts_90khz: 91_920, + cadence_ticks_90khz: 1_920, + end_exclusive_90khz: 93_840, + }] + ); + + let single = TransportStreamBuffer::new(build_pcr_only_packet(pid, 0, 90_000).to_vec()); + assert_eq!(single.duration_ticks_90khz(), None); +} + +#[test] +fn timestamp_profile_scanner_unwraps_one_33_bit_boundary() { + let before_wrap = MAX_PTS_DTS - 1_000; + let packets = [ + build_pts_dts_pcr_packet(0x0100, 0, before_wrap + 200, before_wrap + 100, before_wrap), + build_pts_dts_pcr_packet(0x0100, 1, 1_200, 1_100, 1_000), + ]; + let mut scanner = HlsTsTimestampProfileScanner::new(5_000); + for packet in packets { + scanner.push_aligned_packet(&packet); + } + + assert_eq!( + scanner.finish(), + Some(HlsTsTimestampProfile { + first_clock_90khz: before_wrap, + last_clock_90khz: 1_200, + span_ticks_90khz: 2_200, + observed_pts_or_dts: true, + observed_pcr: true, + }) + ); +} + +#[test] +fn timestamp_profile_scanner_rejects_implausible_backward_segment_jump() { + let packets = [ + build_pts_dts_pcr_packet(0x0100, 0, 500_200, 500_100, 500_000), + build_pts_dts_pcr_packet(0x0100, 1, 100_200, 100_100, 100_000), + ]; + let mut scanner = HlsTsTimestampProfileScanner::new(90_000); + for packet in packets { + scanner.push_aligned_packet(&packet); + } + + assert_eq!(scanner.finish(), None); +} diff --git a/backend/mpegts/src/ts_inspector.rs b/backend/mpegts/src/ts_inspector.rs index 62e0e10f4..af3e13bed 100644 --- a/backend/mpegts/src/ts_inspector.rs +++ b/backend/mpegts/src/ts_inspector.rs @@ -1,61 +1,32 @@ -use crate::transport_stream_buffer::{HlsTsTimestampProfile, HlsTsTimestampProfileScanner}; -use aes::{ - cipher::{Block, BlockDecrypt, KeyInit}, - Aes128, -}; +use crate::transport_stream_buffer::HlsTsTimestampProfileScanner; +use aes::Aes128; use mpeg2ts_reader::{ - demultiplex::{self, DemuxContext, FilterChangeset, FilterRequest, PacketFilter}, - mpegts_crc, + demultiplex::{self, FilterChangeset}, packet::{Packet, Pid}, - psi::{ - self, - pat::{PatSection, ProgramDescriptor}, - pmt::PmtSection, - BufferSectionSyntaxParser, CurrentNext, SectionPacketConsumer, SectionProcessor, SectionSyntaxSectionProcessor, - WholeSectionSyntaxPayloadParser, - }, + psi::{self, BufferSectionSyntaxParser, SectionPacketConsumer, SectionProcessor, SectionSyntaxSectionProcessor}, }; use std::{ collections::{BTreeMap, BTreeSet}, - io::Read, sync::Arc, }; -use tokio::io::{AsyncRead, AsyncReadExt}; -use zeroize::{Zeroize, Zeroizing}; +use zeroize::Zeroizing; const AES_128_BLOCK_BYTES: usize = 16; + const TS_PACKET_BYTES: usize = Packet::SIZE; + const TS_ALIGNMENT_CONFIRMATION_PACKETS: usize = 2; + const HLS_TS_PROBE_MAX_BYTES: u64 = 2 * 1024 * 1024; + const HLS_TS_PROBE_MAX_PACKETS: u64 = 8_192; + const HLS_TS_PROBE_READ_CHUNK_BYTES: usize = 64 * 1024; + const HLS_TS_PROBE_MAX_RESYNC_BYTES: usize = TS_PACKET_BYTES * 4; + const PSI_SYNTAX_HEADER_BYTES: usize = psi::SectionCommonHeader::SIZE + psi::TableSyntaxHeader::SIZE; -/// Hard limits for one read-only MPEG-TS compatibility probe. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct HlsTsProbeBudget { - pub max_bytes: u64, - pub max_packets: u64, - pub read_chunk_bytes: usize, - pub max_resync_bytes: usize, -} - -impl Default for HlsTsProbeBudget { - fn default() -> Self { - Self { - max_bytes: HLS_TS_PROBE_MAX_BYTES, - max_packets: HLS_TS_PROBE_MAX_PACKETS, - read_chunk_bytes: HLS_TS_PROBE_READ_CHUNK_BYTES, - max_resync_bytes: HLS_TS_PROBE_MAX_RESYNC_BYTES, - } - } -} - -impl HlsTsProbeBudget { - fn bounded_read_chunk_bytes(self) -> usize { self.read_chunk_bytes.clamp(1, HLS_TS_PROBE_READ_CHUNK_BYTES) } -} - /// Stable PAT/PMT compatibility evidence. It is not cross-host content identity. #[derive(Debug, Clone, PartialEq, Eq)] pub struct HlsTsTrackSignature { @@ -65,368 +36,12 @@ pub struct HlsTsTrackSignature { programs: Arc<[HlsTsProgramTopology]>, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct HlsTsElementaryStreamBinding { - pub stream_type: u8, - pub elementary_pid: u16, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HlsTsProgramTopology { - pub transport_stream_id: u16, - pub program_number: u16, - pub pmt_pid: u16, - pub pcr_pid: u16, - pub streams: Arc<[HlsTsElementaryStreamBinding]>, -} - -impl HlsTsTrackSignature { - #[cfg(any(test, feature = "test-support"))] - pub fn from_stream_types(stream_types: impl Into>) -> Self { - Self { program_count: 1, has_pcr: true, stream_types: stream_types.into(), programs: Arc::from([]) } - } -} - -fn is_audio_stream_type(stream_type: u8) -> bool { matches!(stream_type, 0x03 | 0x04 | 0x0F | 0x11 | 0x81 | 0x87) } - -fn is_video_stream_type(stream_type: u8) -> bool { matches!(stream_type, 0x01 | 0x02 | 0x10 | 0x1B | 0x24 | 0x42) } - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsTsMalformedReason { - InvalidSynchronization, - InvalidPacketHeader, - TransportError, - InvalidPsiPointer, - InvalidPat, - InvalidPmt, - InvalidPsiCrc, - IncompletePacket, - IncompleteProgramMetadata, -} - -impl HlsTsMalformedReason { - const fn reason_code(self) -> &'static str { - match self { - Self::InvalidSynchronization => "invalid-synchronization", - Self::InvalidPacketHeader => "invalid-packet-header", - Self::TransportError => "transport-error", - Self::InvalidPsiPointer => "invalid-psi-pointer", - Self::InvalidPat => "invalid-pat", - Self::InvalidPmt => "invalid-pmt", - Self::InvalidPsiCrc => "invalid-psi-crc", - Self::IncompletePacket => "incomplete-packet", - Self::IncompleteProgramMetadata => "incomplete-program-metadata", - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsTsProtectionReason { - TransportScrambling, - UnsupportedEncryption, -} - -impl HlsTsProtectionReason { - const fn reason_code(self) -> &'static str { - match self { - Self::TransportScrambling => "transport-scrambling", - Self::UnsupportedEncryption => "unsupported-encryption", - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum HlsTsProbeOutcome { - Found(HlsTsTrackSignature), - ProbeBudgetExhausted { bytes_examined: u64, packets_examined: u64 }, - Malformed(HlsTsMalformedReason), - UnsupportedProtection(HlsTsProtectionReason), -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HlsTsMediaEvidence { - pub track_outcome: HlsTsProbeOutcome, - pub timestamp_profile: Option, - pub splice_evidence: HlsTsSpliceEvidence, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct HlsTsPidBoundaryEvidence { - pub pid: u16, - pub first_packet_index: u64, - pub first_continuity_counter: u8, - pub first_has_payload: bool, - pub first_discontinuity: bool, - pub last_continuity_counter: u8, -} - #[derive(Debug, Clone, PartialEq, Eq)] pub struct HlsTsCompatibleSpliceEvidence { topology: HlsTsTrackSignature, pid_boundaries: Arc<[HlsTsPidBoundaryEvidence]>, } -impl HlsTsCompatibleSpliceEvidence { - fn pid_boundary(&self, pid: u16) -> Option<&HlsTsPidBoundaryEvidence> { - self.pid_boundaries - .binary_search_by_key(&pid, |boundary| boundary.pid) - .ok() - .and_then(|index| self.pid_boundaries.get(index)) - } - - #[cfg(any(test, feature = "test-support"))] - pub fn for_test(topology: HlsTsTrackSignature) -> Self { Self { topology, pid_boundaries: Arc::from([]) } } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsTsSpliceIncompatibility { - InvalidPacket { packet_index: u64 }, - TransportError { pid: u16, packet_index: u64 }, - ContinuityFailure { pid: u16, packet_index: u64, expected: u8, actual: u8 }, - IncompletePes { pid: u16, packet_index: u64, declared_bytes: Option, observed_bytes: u64 }, - InvalidPes { pid: u16, packet_index: u64 }, - InspectionBudgetExhausted, - TopologyUnavailable, -} - -impl HlsTsSpliceIncompatibility { - pub const fn result_code(self) -> &'static str { - match self { - Self::InvalidPacket { .. } | Self::TransportError { .. } | Self::ContinuityFailure { .. } => { - "continuity-failure" - } - Self::IncompletePes { .. } | Self::InvalidPes { .. } => "incomplete-pes", - Self::InspectionBudgetExhausted | Self::TopologyUnavailable => "topology-mismatch", - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum HlsTsSpliceEvidence { - Compatible(HlsTsCompatibleSpliceEvidence), - Incompatible(HlsTsSpliceIncompatibility), -} - -impl HlsTsSpliceEvidence { - #[cfg(any(test, feature = "test-support"))] - pub fn compatible_for_test(topology: HlsTsTrackSignature) -> Self { - Self::Compatible(HlsTsCompatibleSpliceEvidence::for_test(topology)) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HlsTsSpliceBoundaryIncompatibility { - Media(HlsTsSpliceIncompatibility), - TopologyMismatch, -} - -pub fn evaluate_mpeg_ts_splice_boundary( - base: &HlsTsSpliceEvidence, - terminal: &HlsTsSpliceEvidence, -) -> Result<(), HlsTsSpliceBoundaryIncompatibility> { - let base = match base { - HlsTsSpliceEvidence::Compatible(evidence) => evidence, - HlsTsSpliceEvidence::Incompatible(reason) => { - return Err(HlsTsSpliceBoundaryIncompatibility::Media(*reason)); - } - }; - let terminal = match terminal { - HlsTsSpliceEvidence::Compatible(evidence) => evidence, - HlsTsSpliceEvidence::Incompatible(reason) => { - return Err(HlsTsSpliceBoundaryIncompatibility::Media(*reason)); - } - }; - if base.topology != terminal.topology { - return Err(HlsTsSpliceBoundaryIncompatibility::TopologyMismatch); - } - for boundary in terminal.pid_boundaries.iter() { - if boundary.first_discontinuity { - continue; - } - let Some(base_boundary) = base.pid_boundary(boundary.pid) else { - continue; - }; - let expected = if boundary.first_has_payload { - base_boundary.last_continuity_counter.wrapping_add(1) & 0x0F - } else { - base_boundary.last_continuity_counter - }; - if boundary.first_continuity_counter != expected { - return Err(HlsTsSpliceBoundaryIncompatibility::Media(HlsTsSpliceIncompatibility::ContinuityFailure { - pid: boundary.pid, - packet_index: boundary.first_packet_index, - expected, - actual: boundary.first_continuity_counter, - })); - } - } - Ok(()) -} - -#[derive(Debug, thiserror::Error)] -pub enum HlsTsProbeError { - #[error("MPEG-TS probe I/O failed")] - Io(#[source] std::io::Error), - #[error("MPEG-TS probe key is unavailable")] - KeyUnavailable, - #[error("MPEG-TS probe IV is invalid")] - InvalidIv, - #[error("MPEG-TS probe decryption failed")] - DecryptionFailed, -} - -/// Policy-facing track evidence with stable diagnostics and no parser-crate types. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum HlsTrackEvidenceResolution { - Found(HlsTsTrackSignature), - InsufficientEvidence { bytes_examined: u64, packets_examined: u64 }, - IncompleteEvidence, - Malformed(HlsTsMalformedReason), - UnsupportedProtection(HlsTsProtectionReason), - KeyUnavailable, - InvalidIv, - DecryptionFailed, - Io(std::io::ErrorKind), -} - -impl HlsTrackEvidenceResolution { - pub fn signature(&self) -> Option<&HlsTsTrackSignature> { - match self { - Self::Found(signature) => Some(signature), - Self::InsufficientEvidence { .. } - | Self::IncompleteEvidence - | Self::Malformed(_) - | Self::UnsupportedProtection(_) - | Self::KeyUnavailable - | Self::InvalidIv - | Self::DecryptionFailed - | Self::Io(_) => None, - } - } - - pub const fn reason_code(&self) -> &'static str { - match self { - Self::Found(_) => "found", - Self::InsufficientEvidence { .. } => "insufficient-evidence", - Self::IncompleteEvidence => "incomplete-evidence", - Self::Malformed(reason) => reason.reason_code(), - Self::UnsupportedProtection(reason) => reason.reason_code(), - Self::KeyUnavailable => "key-unavailable", - Self::InvalidIv => "invalid-iv", - Self::DecryptionFailed => "decryption-failed", - // ErrorKind remains available on the typed value; logs deliberately use - // one stable non-sensitive code rather than exposing platform wording. - Self::Io(_) => "io", - } - } -} - -impl From> for HlsTrackEvidenceResolution { - fn from(result: Result) -> Self { - match result { - Ok(HlsTsProbeOutcome::Found(signature)) => Self::Found(signature), - Ok(HlsTsProbeOutcome::ProbeBudgetExhausted { bytes_examined, packets_examined }) => { - Self::InsufficientEvidence { bytes_examined, packets_examined } - } - Ok(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::IncompleteProgramMetadata)) => { - Self::IncompleteEvidence - } - Ok(HlsTsProbeOutcome::Malformed(reason)) => Self::Malformed(reason), - Ok(HlsTsProbeOutcome::UnsupportedProtection(reason)) => Self::UnsupportedProtection(reason), - Err(HlsTsProbeError::Io(error)) => Self::Io(error.kind()), - Err(HlsTsProbeError::KeyUnavailable) => Self::KeyUnavailable, - Err(HlsTsProbeError::InvalidIv) => Self::InvalidIv, - Err(HlsTsProbeError::DecryptionFailed) => Self::DecryptionFailed, - } - } -} - -#[derive(Clone, Copy)] -pub enum HlsTsProbeProtection<'a> { - Clear, - Aes128Cbc { key: &'a [u8], iv: [u8; AES_128_BLOCK_BYTES] }, -} - -/// Implements the existing HLS AES-128 explicit-IV and sequence-derived-IV rules. -pub fn hls_aes128_cbc_iv( - explicit_iv: Option<&str>, - media_sequence: u64, -) -> Result<[u8; AES_128_BLOCK_BYTES], HlsTsProbeError> { - let mut iv = [0_u8; AES_128_BLOCK_BYTES]; - let Some(explicit_iv) = explicit_iv else { - iv[AES_128_BLOCK_BYTES - std::mem::size_of::()..].copy_from_slice(&media_sequence.to_be_bytes()); - return Ok(iv); - }; - let hex = explicit_iv.strip_prefix("0x").ok_or(HlsTsProbeError::InvalidIv)?; - if hex.is_empty() || hex.len() > AES_128_BLOCK_BYTES * 2 || !hex.bytes().all(|byte| byte.is_ascii_hexdigit()) { - return Err(HlsTsProbeError::InvalidIv); - } - let mut output_index = AES_128_BLOCK_BYTES; - let mut input_index = hex.len(); - while input_index > 0 { - let low = hex_digit(hex.as_bytes()[input_index - 1]).ok_or(HlsTsProbeError::InvalidIv)?; - input_index = input_index.saturating_sub(1); - let high = if input_index > 0 { - let value = hex_digit(hex.as_bytes()[input_index - 1]).ok_or(HlsTsProbeError::InvalidIv)?; - input_index = input_index.saturating_sub(1); - value - } else { - 0 - }; - output_index = output_index.checked_sub(1).ok_or(HlsTsProbeError::InvalidIv)?; - iv[output_index] = (high << 4) | low; - } - Ok(iv) -} - -fn hex_digit(byte: u8) -> Option { - match byte { - b'0'..=b'9' => Some(byte - b'0'), - b'a'..=b'f' => Some(byte - b'a' + 10), - b'A'..=b'F' => Some(byte - b'A' + 10), - _ => None, - } -} - -#[derive(Clone, Copy)] -enum HlsPsiTableKind { - Pat, - Pmt, -} - -impl HlsPsiTableKind { - const fn malformed_reason(self) -> HlsTsMalformedReason { - match self { - Self::Pat => HlsTsMalformedReason::InvalidPat, - Self::Pmt => HlsTsMalformedReason::InvalidPmt, - } - } - - const fn expected_table_id(self) -> u8 { - match self { - Self::Pat => 0x00, - Self::Pmt => 0x02, - } - } - - const fn minimum_section_length(self) -> usize { - match self { - Self::Pat => 9, - Self::Pmt => 13, - } - } - - fn validate_common_header(self, header: &[u8]) -> bool { - header.len() == psi::SectionCommonHeader::SIZE - && header[0] == self.expected_table_id() - && header[1] & 0x80 != 0 - && header[1] & 0x30 == 0x30 - && (self.minimum_section_length()..=1_021).contains(&psi_section_length(header)) - } -} - -fn psi_section_length(header: &[u8]) -> usize { (usize::from(header[1] & 0x0F) << 8) | usize::from(header[2]) } - /// Thin adapter for the crate's documented syntax-header buffering gap. The /// crate still owns section framing, continuation, CRC payload collection and /// PAT/PMT parsing; this adapter retains at most the first seven header bytes. @@ -439,235 +54,14 @@ where pending_syntax_header: Option>, } -impl

HlsPsiSectionConsumer

-where - P: SectionProcessor, -{ - fn new(consumer: SectionPacketConsumer

, table_kind: HlsPsiTableKind) -> Self { - Self { consumer, table_kind, pending_syntax_header: None } - } - - fn consume(&mut self, ctx: &mut HlsTsDemuxContext, packet: &Packet<'_>) { - if let Some(mut pending) = self.pending_syntax_header.take() { - let Some(payload) = packet.payload() else { - self.pending_syntax_header = Some(pending); - return; - }; - if packet.payload_unit_start_indicator() { - let Some((&pointer, section_data)) = payload.split_first() else { - ctx.record_malformed(HlsTsMalformedReason::InvalidPsiPointer); - return; - }; - let pointer = usize::from(pointer); - if pointer >= section_data.len() { - ctx.record_malformed(HlsTsMalformedReason::InvalidPsiPointer); - return; - } - let (previous_section, section_start) = section_data.split_at(pointer); - pending.extend_from_slice(previous_section); - if pending.len() < psi::SectionCommonHeader::SIZE - || !self.table_kind.validate_common_header(&pending[..psi::SectionCommonHeader::SIZE]) - { - ctx.record_malformed(self.table_kind.malformed_reason()); - return; - } - let section_bytes = psi::SectionCommonHeader::SIZE - .saturating_add(psi_section_length(&pending[..psi::SectionCommonHeader::SIZE])); - if pending.len() < section_bytes { - ctx.record_malformed(self.table_kind.malformed_reason()); - return; - } - self.consume_synthetic_payload(ctx, packet.pid(), &pending[..section_bytes], true); - if ctx.malformed.is_none() { - self.consume_new_section(ctx, packet.pid(), section_start); - } - return; - } - pending.extend_from_slice(payload); - if pending.len() < psi::SectionCommonHeader::SIZE { - self.pending_syntax_header = Some(pending); - return; - } - if !self.table_kind.validate_common_header(&pending[..psi::SectionCommonHeader::SIZE]) { - ctx.record_malformed(self.table_kind.malformed_reason()); - return; - } - if pending.len() < PSI_SYNTAX_HEADER_BYTES { - self.pending_syntax_header = Some(pending); - return; - } - let section_bytes = psi::SectionCommonHeader::SIZE - .saturating_add(psi_section_length(&pending[..psi::SectionCommonHeader::SIZE])) - .min(pending.len()); - self.consume_synthetic_payload(ctx, packet.pid(), &pending[..section_bytes], true); - return; - } - - let Some(payload) = packet.payload() else { - self.consumer.consume(ctx, packet); - return; - }; - if !packet.payload_unit_start_indicator() { - self.consumer.consume(ctx, packet); - return; - } - let Some((&pointer, section_data)) = payload.split_first() else { - ctx.record_malformed(HlsTsMalformedReason::InvalidPsiPointer); - return; - }; - let pointer = usize::from(pointer); - if pointer >= section_data.len() { - ctx.record_malformed(HlsTsMalformedReason::InvalidPsiPointer); - return; - } - let section_start = §ion_data[pointer..]; - if section_start.len() < PSI_SYNTAX_HEADER_BYTES { - if let Some(previous_section) = section_data.get(..pointer).filter(|bytes| !bytes.is_empty()) { - self.consume_synthetic_payload(ctx, packet.pid(), previous_section, false); - } - if ctx.malformed.is_none() { - self.consume_new_section(ctx, packet.pid(), section_start); - } - return; - } - if !self.table_kind.validate_common_header(§ion_start[..psi::SectionCommonHeader::SIZE]) { - ctx.record_malformed(self.table_kind.malformed_reason()); - return; - } - self.consumer.consume(ctx, packet); - } - - fn consume_new_section(&mut self, ctx: &mut HlsTsDemuxContext, pid: Pid, section_start: &[u8]) { - if section_start.len() < PSI_SYNTAX_HEADER_BYTES { - if section_start.len() >= psi::SectionCommonHeader::SIZE - && !self.table_kind.validate_common_header(§ion_start[..psi::SectionCommonHeader::SIZE]) - { - ctx.record_malformed(self.table_kind.malformed_reason()); - return; - } - self.pending_syntax_header = Some(section_start.to_vec()); - return; - } - if !self.table_kind.validate_common_header(§ion_start[..psi::SectionCommonHeader::SIZE]) { - ctx.record_malformed(self.table_kind.malformed_reason()); - return; - } - self.consume_synthetic_payload(ctx, pid, section_start, true); - } - - fn consume_synthetic_payload(&mut self, ctx: &mut HlsTsDemuxContext, pid: Pid, bytes: &[u8], start: bool) { - let mut offset = 0usize; - let mut first = true; - while offset < bytes.len() { - let starts_section = start && first; - let capacity = if starts_section { 183 } else { 184 }; - let copied = bytes.len().saturating_sub(offset).min(capacity); - let payload = &bytes[offset..offset.saturating_add(copied)]; - let packet_bytes = synthetic_psi_packet(pid, starts_section, payload); - if let Some(packet) = Packet::try_new(&packet_bytes) { - self.consumer.consume(ctx, &packet); - } - offset = offset.saturating_add(copied); - first = false; - } - } -} - -fn synthetic_psi_packet(pid: Pid, starts_section: bool, payload: &[u8]) -> [u8; TS_PACKET_BYTES] { - let pointer_bytes = usize::from(starts_section); - let payload_length = payload.len().saturating_add(pointer_bytes).min(184); - let pid = u16::from(pid); - let mut packet = [0xFF_u8; TS_PACKET_BYTES]; - packet[0] = Packet::SYNC_BYTE; - packet[1] = (u8::try_from(pid >> 8).unwrap_or(0) & 0x1F) | if starts_section { 0x40 } else { 0 }; - packet[2] = pid.to_be_bytes()[1]; - let payload_offset = if payload_length == 184 { - packet[3] = 0x10; - 4 - } else { - packet[3] = 0x30; - let adaptation_length = 183usize.saturating_sub(payload_length); - packet[4] = u8::try_from(adaptation_length).unwrap_or(182); - if adaptation_length > 0 { - packet[5] = 0; - } - 5usize.saturating_add(adaptation_length) - }; - let data_offset = if starts_section { - packet[payload_offset] = 0; - payload_offset.saturating_add(1) - } else { - payload_offset - }; - let copy_length = payload.len().min(TS_PACKET_BYTES.saturating_sub(data_offset)); - packet[data_offset..data_offset.saturating_add(copy_length)].copy_from_slice(&payload[..copy_length]); - packet -} - struct HlsPatPacketFilter { consumer: HlsPsiSectionConsumer>>, } -impl Default for HlsPatPacketFilter { - fn default() -> Self { - Self { - consumer: HlsPsiSectionConsumer::new( - SectionPacketConsumer::new(SectionSyntaxSectionProcessor::new(BufferSectionSyntaxParser::new( - HlsPatSectionParser, - ))), - HlsPsiTableKind::Pat, - ), - } - } -} - -impl PacketFilter for HlsPatPacketFilter { - type Ctx = HlsTsDemuxContext; - - fn consume(&mut self, ctx: &mut Self::Ctx, packet: &Packet<'_>) { self.consumer.consume(ctx, packet); } -} - struct HlsPmtPacketFilter { consumer: HlsPsiSectionConsumer>>, } -impl HlsPmtPacketFilter { - fn new(pid: Pid, program_number: u16) -> Self { - Self { - consumer: HlsPsiSectionConsumer::new( - SectionPacketConsumer::new(SectionSyntaxSectionProcessor::new(BufferSectionSyntaxParser::new( - HlsPmtSectionParser { pid, program_number }, - ))), - HlsPsiTableKind::Pmt, - ), - } - } -} - -impl PacketFilter for HlsPmtPacketFilter { - type Ctx = HlsTsDemuxContext; - - fn consume(&mut self, ctx: &mut Self::Ctx, packet: &Packet<'_>) { self.consumer.consume(ctx, packet); } -} - -enum HlsTsPacketFilter { - Pat(HlsPatPacketFilter), - Pmt(HlsPmtPacketFilter), - Ignore(demultiplex::NullPacketFilter), -} - -impl PacketFilter for HlsTsPacketFilter { - type Ctx = HlsTsDemuxContext; - - fn consume(&mut self, ctx: &mut Self::Ctx, packet: &Packet<'_>) { - match self { - Self::Pat(filter) => filter.consume(ctx, packet), - Self::Pmt(filter) => filter.consume(ctx, packet), - Self::Ignore(filter) => filter.consume(ctx, packet), - } - } -} - #[derive(Debug, Clone, PartialEq, Eq)] struct HlsPmtSectionEvidence { pcr_pid: u16, @@ -684,64 +78,6 @@ struct HlsPmtEvidence { stream_types: BTreeSet, } -impl HlsPmtEvidence { - fn complete(&self) -> bool { - self.last_section_number.is_some_and(|last| (0..=last).all(|section| self.sections.contains_key(§ion))) - } - - fn restart_version(&mut self, version: u8, last_section_number: u8) { - self.table_version = Some(version); - self.last_section_number = Some(last_section_number); - self.sections.clear(); - self.has_pcr = false; - self.stream_types.clear(); - } - - fn record_section( - &mut self, - section_number: u8, - section: HlsPmtSectionEvidence, - ) -> Result<(), HlsTsMalformedReason> { - if self.sections.get(§ion_number).is_some_and(|current| current != §ion) { - return Err(HlsTsMalformedReason::InvalidPmt); - } - self.sections.insert(section_number, section); - self.has_pcr = self.sections.values().any(|section| section.pcr_pid != u16::from(mpeg2ts_reader::STUFFING_PID)); - self.stream_types = self - .sections - .values() - .flat_map(|section| section.streams.iter().map(|stream| stream.stream_type)) - .collect(); - Ok(()) - } - - fn topology(&self, transport_stream_id: u16, pmt_pid: u16) -> Option { - if !self.complete() { - return None; - } - let mut sections = self.sections.values(); - let pcr_pid = sections.next()?.pcr_pid; - if self.sections.values().any(|section| section.pcr_pid != pcr_pid) { - return None; - } - let streams = self.sections.values().flat_map(|section| section.streams.iter().copied()).collect::>(); - if streams.is_empty() { - return None; - } - let mut elementary_pids = BTreeSet::new(); - if streams.iter().any(|stream| !elementary_pids.insert(stream.elementary_pid)) { - return None; - } - Some(HlsTsProgramTopology { - transport_stream_id, - program_number: self.program_number, - pmt_pid, - pcr_pid, - streams: streams.into(), - }) - } -} - struct HlsTsDemuxContext { changeset: FilterChangeset, pat_table_identity: Option<(u16, u8)>, @@ -752,355 +88,11 @@ struct HlsTsDemuxContext { malformed: Option, } -impl HlsTsDemuxContext { - fn new() -> Self { - Self { - changeset: FilterChangeset::default(), - pat_table_identity: None, - pat_last_section_number: None, - pat_sections: BTreeMap::new(), - programs_by_pid: BTreeMap::new(), - pmt_evidence: BTreeMap::new(), - malformed: None, - } - } - - fn record_malformed(&mut self, reason: HlsTsMalformedReason) { - if self.malformed.is_none() { - self.malformed = Some(reason); - } - } - - fn register_program(&mut self, program_number: u16, pid: Pid) { - let pid_value = u16::from(pid); - if self.programs_by_pid.get(&pid_value).is_some_and(|registered_program| *registered_program != program_number) - || self.programs_by_pid.iter().any(|(registered_pid, registered_program)| { - *registered_program == program_number && *registered_pid != pid_value - }) - { - self.record_malformed(HlsTsMalformedReason::InvalidPat); - return; - } - if self.programs_by_pid.insert(pid_value, program_number).is_none() { - self.pmt_evidence.insert(pid_value, HlsPmtEvidence { program_number, ..HlsPmtEvidence::default() }); - self.changeset.insert(pid, HlsTsPacketFilter::Pmt(HlsPmtPacketFilter::new(pid, program_number))); - } - } - - fn restart_pat_version(&mut self, table_identity: (u16, u8)) { - let mut invalid_registered_pid = false; - for pid in self.programs_by_pid.keys().copied() { - match Pid::try_from(pid) { - Ok(pid) => self.changeset.remove(pid), - Err(()) => invalid_registered_pid = true, - } - } - self.pat_table_identity = Some(table_identity); - self.pat_last_section_number = None; - self.pat_sections.clear(); - self.programs_by_pid.clear(); - self.pmt_evidence.clear(); - if invalid_registered_pid { - self.record_malformed(HlsTsMalformedReason::InvalidPat); - } - } - - fn reconcile_pat_programs(&mut self) -> Result<(), HlsTsMalformedReason> { - let mut programs_by_pid = BTreeMap::new(); - let mut pids_by_program = BTreeMap::new(); - for section in self.pat_sections.values() { - for (pid, program_number) in section { - if programs_by_pid.insert(*pid, *program_number).is_some_and(|current| current != *program_number) - || pids_by_program.insert(*program_number, *pid).is_some_and(|current| current != *pid) - { - return Err(HlsTsMalformedReason::InvalidPat); - } - } - } - for (pid, program_number) in programs_by_pid { - match self.programs_by_pid.get(&pid) { - Some(current) if *current == program_number => {} - Some(_) => return Err(HlsTsMalformedReason::InvalidPat), - None => { - let pid = Pid::try_from(pid).map_err(|()| HlsTsMalformedReason::InvalidPat)?; - self.register_program(program_number, pid); - if self.malformed.is_some() { - return Err(HlsTsMalformedReason::InvalidPat); - } - } - } - } - Ok(()) - } - - fn is_psi_pid(&self, pid: u16) -> bool { - pid == u16::from(psi::pat::PAT_PID) || self.programs_by_pid.contains_key(&pid) - } - - fn signature(&self) -> Option { - let pat_complete = self - .pat_last_section_number - .is_some_and(|last| (0..=last).all(|section| self.pat_sections.contains_key(§ion))); - if !pat_complete || self.programs_by_pid.is_empty() || self.pmt_evidence.values().any(|pmt| !pmt.complete()) { - return None; - } - let transport_stream_id = self.pat_table_identity?.0; - let programs = self - .pmt_evidence - .iter() - .map(|(pmt_pid, pmt)| pmt.topology(transport_stream_id, *pmt_pid)) - .collect::>>()?; - let stream_types = self - .pmt_evidence - .values() - .flat_map(|pmt| pmt.stream_types.iter().copied()) - .collect::>() - .into_iter() - .collect::>(); - if !stream_types - .iter() - .copied() - .any(|stream_type| is_audio_stream_type(stream_type) || is_video_stream_type(stream_type)) - { - return None; - } - Some(HlsTsTrackSignature { - program_count: u16::try_from(self.programs_by_pid.len()).unwrap_or(u16::MAX), - has_pcr: self.pmt_evidence.values().any(|pmt| pmt.has_pcr), - stream_types: Arc::from(stream_types), - programs: programs.into(), - }) - } -} - -impl DemuxContext for HlsTsDemuxContext { - type F = HlsTsPacketFilter; - - fn filter_changeset(&mut self) -> &mut FilterChangeset { &mut self.changeset } - - fn construct(&mut self, request: FilterRequest<'_, '_>) -> Self::F { - match request { - FilterRequest::ByPid(pid) if pid == psi::pat::PAT_PID => { - HlsTsPacketFilter::Pat(HlsPatPacketFilter::default()) - } - FilterRequest::Pmt { pid, program_number } => { - HlsTsPacketFilter::Pmt(HlsPmtPacketFilter::new(pid, program_number)) - } - FilterRequest::ByPid(_) | FilterRequest::ByStream { .. } | FilterRequest::Nit { .. } => { - HlsTsPacketFilter::Ignore(demultiplex::NullPacketFilter::default()) - } - } - } -} - -struct HlsPatSectionParser; - -impl WholeSectionSyntaxPayloadParser for HlsPatSectionParser { - type Context = HlsTsDemuxContext; - - fn section<'a>( - &mut self, - ctx: &mut Self::Context, - header: &psi::SectionCommonHeader, - table_header: &psi::TableSyntaxHeader<'a>, - data: &'a [u8], - ) { - if header.table_id != 0x00 || data.len() < 12 { - ctx.record_malformed(HlsTsMalformedReason::InvalidPat); - return; - } - if mpegts_crc::sum32(data) != 0 { - ctx.record_malformed(HlsTsMalformedReason::InvalidPsiCrc); - return; - } - if table_header.current_next_indicator() == CurrentNext::Next { - return; - } - if table_header.section_number() > table_header.last_section_number() { - ctx.record_malformed(HlsTsMalformedReason::InvalidPat); - return; - } - let table_identity = (table_header.id(), table_header.version()); - match ctx.pat_table_identity { - Some((table_id, _)) if table_id != table_header.id() => { - ctx.record_malformed(HlsTsMalformedReason::InvalidPat); - return; - } - Some(current) if current != table_identity => ctx.restart_pat_version(table_identity), - Some(_) => {} - None => ctx.pat_table_identity = Some(table_identity), - } - if ctx.pat_last_section_number.is_some_and(|last| last != table_header.last_section_number()) { - ctx.record_malformed(HlsTsMalformedReason::InvalidPat); - return; - } - let payload_start = psi::SectionCommonHeader::SIZE + psi::TableSyntaxHeader::SIZE; - let payload_end = data.len().saturating_sub(4); - let Some(payload) = data.get(payload_start..payload_end) else { - ctx.record_malformed(HlsTsMalformedReason::InvalidPat); - return; - }; - if !payload.len().is_multiple_of(4) { - ctx.record_malformed(HlsTsMalformedReason::InvalidPat); - return; - } - let mut programs = BTreeMap::new(); - let mut pids_by_program = BTreeMap::new(); - for descriptor in PatSection::new(payload).programs() { - if let ProgramDescriptor::Program { program_number, pid } = descriptor { - let pid = u16::from(pid); - if programs.insert(pid, program_number).is_some_and(|current| current != program_number) - || pids_by_program.insert(program_number, pid).is_some_and(|current| current != pid) - { - ctx.record_malformed(HlsTsMalformedReason::InvalidPat); - return; - } - } - } - if ctx.pat_sections.get(&table_header.section_number()).is_some_and(|current| current != &programs) { - ctx.record_malformed(HlsTsMalformedReason::InvalidPat); - return; - } - ctx.pat_last_section_number = Some(table_header.last_section_number()); - ctx.pat_sections.insert(table_header.section_number(), programs); - if let Err(reason) = ctx.reconcile_pat_programs() { - ctx.record_malformed(reason); - } - } -} - struct HlsPmtSectionParser { pid: Pid, program_number: u16, } -impl WholeSectionSyntaxPayloadParser for HlsPmtSectionParser { - type Context = HlsTsDemuxContext; - - fn section<'a>( - &mut self, - ctx: &mut Self::Context, - header: &psi::SectionCommonHeader, - table_header: &psi::TableSyntaxHeader<'a>, - data: &'a [u8], - ) { - if header.table_id != 0x02 || data.len() < 16 || table_header.id() != self.program_number { - ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); - return; - } - if mpegts_crc::sum32(data) != 0 { - ctx.record_malformed(HlsTsMalformedReason::InvalidPsiCrc); - return; - } - if table_header.current_next_indicator() == CurrentNext::Next { - return; - } - if table_header.section_number() > table_header.last_section_number() { - ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); - return; - } - let payload_start = psi::SectionCommonHeader::SIZE + psi::TableSyntaxHeader::SIZE; - let payload_end = data.len().saturating_sub(4); - let Some(payload) = data.get(payload_start..payload_end) else { - ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); - return; - }; - if !pmt_stream_loop_is_well_formed(payload) { - ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); - return; - } - let Ok(section) = PmtSection::from_bytes(payload) else { - ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); - return; - }; - let pid = u16::from(self.pid); - let Some(evidence) = ctx.pmt_evidence.get_mut(&pid) else { - ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); - return; - }; - if evidence.program_number != self.program_number { - ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); - return; - } - match evidence.table_version { - Some(version) if version != table_header.version() => { - evidence.restart_version(table_header.version(), table_header.last_section_number()); - } - Some(_) => { - if evidence.last_section_number != Some(table_header.last_section_number()) { - ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); - return; - } - } - None => evidence.restart_version(table_header.version(), table_header.last_section_number()), - } - let section_evidence = HlsPmtSectionEvidence { - pcr_pid: u16::from(section.pcr_pid()), - streams: section - .streams() - .map(|stream| HlsTsElementaryStreamBinding { - stream_type: stream.stream_type().0, - elementary_pid: u16::from(stream.elementary_pid()), - }) - .collect(), - }; - if let Err(reason) = evidence.record_section(table_header.section_number(), section_evidence) { - ctx.record_malformed(reason); - } - } -} - -/// `mpeg2ts-reader` intentionally stops its stream iterator at a truncated -/// descriptor. Validate both descriptor loops and ES framing so truncation is -/// a typed malformed outcome instead of a partial, falsely compatible signature. -fn pmt_stream_loop_is_well_formed(payload: &[u8]) -> bool { - let Some(program_info_length_bytes) = payload.get(2..4) else { - return false; - }; - let program_info_length = - (usize::from(program_info_length_bytes[0] & 0x0F) << 8) | usize::from(program_info_length_bytes[1]); - let Some(mut offset) = 4usize.checked_add(program_info_length).filter(|offset| *offset <= payload.len()) else { - return false; - }; - if !descriptor_loop_is_well_formed(&payload[4..offset]) { - return false; - } - while offset < payload.len() { - let Some(header) = payload.get(offset..offset.saturating_add(5)) else { - return false; - }; - let es_info_length = (usize::from(header[3] & 0x0F) << 8) | usize::from(header[4]); - let Some(next) = offset.checked_add(5).and_then(|value| value.checked_add(es_info_length)) else { - return false; - }; - if next > payload.len() { - return false; - } - if !descriptor_loop_is_well_formed(&payload[offset.saturating_add(5)..next]) { - return false; - } - offset = next; - } - true -} - -fn descriptor_loop_is_well_formed(mut descriptors: &[u8]) -> bool { - while !descriptors.is_empty() { - let Some(header) = descriptors.get(..2) else { - return false; - }; - let descriptor_length = usize::from(header[1]); - let Some(next) = 2usize.checked_add(descriptor_length) else { - return false; - }; - let Some(remaining) = descriptors.get(next..) else { - return false; - }; - descriptors = remaining; - } - true -} - #[derive(Clone, Copy)] struct HlsTsPidContinuityState { first_packet_index: u64, @@ -1110,12 +102,6 @@ struct HlsTsPidContinuityState { last_continuity_counter: u8, } -enum HlsTsPesInspectionState { - Header { bytes: [u8; 6], len: usize, started_at_packet_index: u64 }, - Finite { declared_bytes: u16, observed_bytes: u64, started_at_packet_index: u64 }, - UnboundedVideo, -} - struct HlsTsTransportStreamInspector { continuity: Vec>, pes: Vec>, @@ -1123,290 +109,6 @@ struct HlsTsTransportStreamInspector { incompatible: Option, } -fn validated_adaptation_length(packet: &Packet<'_>, bytes: &[u8]) -> Result, ()> { - let adaptation_control = packet.adaptation_control(); - if !adaptation_control.has_payload() && !adaptation_control.has_adaptation_field() { - return Err(()); - } - let adaptation_length = adaptation_control.has_adaptation_field().then(|| usize::from(bytes[4])); - match (adaptation_control.has_adaptation_field(), adaptation_control.has_payload(), adaptation_length) { - (false, true, None) | (true, false, Some(183)) => Ok(adaptation_length), - (true, true, Some(length)) if length <= 182 => Ok(adaptation_length), - _ => Err(()), - } -} - -impl HlsTsTransportStreamInspector { - fn new() -> Self { - let pid_count = usize::from(u16::from(mpeg2ts_reader::STUFFING_PID)).saturating_add(1); - Self { - continuity: vec![None; pid_count], - pes: (0..pid_count).map(|_| None).collect(), - invalid_pes_starts: vec![None; pid_count], - incompatible: None, - } - } - - fn push_packet(&mut self, bytes: &[u8], packet_index: u64) { - if self.incompatible.is_some() { - return; - } - if bytes.len() != TS_PACKET_BYTES { - self.incompatible = Some(HlsTsSpliceIncompatibility::InvalidPacket { packet_index }); - return; - } - let Some(packet) = Packet::try_new(bytes) else { - self.incompatible = Some(HlsTsSpliceIncompatibility::InvalidPacket { packet_index }); - return; - }; - let pid = u16::from(packet.pid()); - if packet.transport_error_indicator() { - self.incompatible = Some(HlsTsSpliceIncompatibility::TransportError { pid, packet_index }); - return; - } - if packet.transport_scrambling_control().is_scrambled() { - self.incompatible = Some(HlsTsSpliceIncompatibility::InvalidPacket { packet_index }); - return; - } - let Ok(adaptation_length) = validated_adaptation_length(&packet, bytes) else { - self.incompatible = Some(HlsTsSpliceIncompatibility::InvalidPacket { packet_index }); - return; - }; - let has_payload = packet.adaptation_control().has_payload(); - let discontinuity = adaptation_length.is_some_and(|length| length > 0 && bytes[5] & 0x80 != 0); - let continuity_counter = packet.continuity_counter().count(); - if let Err(reason) = self.inspect_continuity(pid, packet_index, continuity_counter, has_payload, discontinuity) - { - self.incompatible = Some(reason); - return; - } - if discontinuity { - if let Some(reason) = self.take_discontinuity_pes_incompatibility(pid) { - self.incompatible = Some(reason); - return; - } - } - let Some(payload) = packet.payload() else { - return; - }; - let previous = self.pes[usize::from(pid)].take(); - match advance_ts_pes_inspection( - pid, - packet.payload_unit_start_indicator(), - payload, - packet_index, - previous, - &mut self.invalid_pes_starts[usize::from(pid)], - ) { - Ok(next) => self.pes[usize::from(pid)] = next, - Err(reason) => self.incompatible = Some(reason), - } - } - - fn inspect_continuity( - &mut self, - pid: u16, - packet_index: u64, - continuity_counter: u8, - has_payload: bool, - discontinuity: bool, - ) -> Result<(), HlsTsSpliceIncompatibility> { - if pid == u16::from(mpeg2ts_reader::STUFFING_PID) { - return Ok(()); - } - let state = &mut self.continuity[usize::from(pid)]; - let Some(previous) = state else { - *state = Some(HlsTsPidContinuityState { - first_packet_index: packet_index, - first_continuity_counter: continuity_counter, - first_has_payload: has_payload, - first_discontinuity: discontinuity, - last_continuity_counter: continuity_counter, - }); - return Ok(()); - }; - let expected = if has_payload { - previous.last_continuity_counter.wrapping_add(1) & 0x0F - } else { - previous.last_continuity_counter - }; - if !discontinuity && continuity_counter != expected { - return Err(HlsTsSpliceIncompatibility::ContinuityFailure { - pid, - packet_index, - expected, - actual: continuity_counter, - }); - } - previous.last_continuity_counter = continuity_counter; - Ok(()) - } - - fn take_discontinuity_pes_incompatibility(&mut self, pid: u16) -> Option { - match self.pes[usize::from(pid)].take() { - Some(HlsTsPesInspectionState::Header { started_at_packet_index, .. }) => { - Some(HlsTsSpliceIncompatibility::IncompletePes { - pid, - packet_index: started_at_packet_index, - declared_bytes: None, - observed_bytes: 0, - }) - } - Some(HlsTsPesInspectionState::Finite { declared_bytes, observed_bytes, started_at_packet_index }) => { - Some(HlsTsSpliceIncompatibility::IncompletePes { - pid, - packet_index: started_at_packet_index, - declared_bytes: Some(declared_bytes), - observed_bytes, - }) - } - Some(HlsTsPesInspectionState::UnboundedVideo) | None => None, - } - } - - fn finish(mut self, topology: Option) -> HlsTsSpliceEvidence { - if let Some(reason) = self.incompatible { - return HlsTsSpliceEvidence::Incompatible(reason); - } - for (pid, state) in self.pes.drain(..).enumerate() { - let Some(state) = state else { - continue; - }; - let pid = u16::try_from(pid).unwrap_or(u16::MAX); - match state { - HlsTsPesInspectionState::Header { started_at_packet_index, .. } => { - return HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::IncompletePes { - pid, - packet_index: started_at_packet_index, - declared_bytes: None, - observed_bytes: 0, - }); - } - HlsTsPesInspectionState::Finite { declared_bytes, observed_bytes, started_at_packet_index } => { - return HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::IncompletePes { - pid, - packet_index: started_at_packet_index, - declared_bytes: Some(declared_bytes), - observed_bytes, - }); - } - HlsTsPesInspectionState::UnboundedVideo => {} - } - } - let Some(topology) = topology else { - return HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::TopologyUnavailable); - }; - for stream in topology.programs.iter().flat_map(|program| program.streams.iter()) { - if let Some(packet_index) = self.invalid_pes_starts[usize::from(stream.elementary_pid)] { - return HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::InvalidPes { - pid: stream.elementary_pid, - packet_index, - }); - } - } - let pid_boundaries = self - .continuity - .into_iter() - .enumerate() - .filter_map(|(pid, state)| { - let state = state?; - Some(HlsTsPidBoundaryEvidence { - pid: u16::try_from(pid).unwrap_or(u16::MAX), - first_packet_index: state.first_packet_index, - first_continuity_counter: state.first_continuity_counter, - first_has_payload: state.first_has_payload, - first_discontinuity: state.first_discontinuity, - last_continuity_counter: state.last_continuity_counter, - }) - }) - .collect::>(); - HlsTsSpliceEvidence::Compatible(HlsTsCompatibleSpliceEvidence { - topology, - pid_boundaries: pid_boundaries.into(), - }) - } -} - -fn advance_ts_pes_inspection( - pid: u16, - payload_unit_start: bool, - payload: &[u8], - packet_index: u64, - previous: Option, - invalid_pes_start: &mut Option, -) -> Result, HlsTsSpliceIncompatibility> { - let state = if payload_unit_start { - match previous { - Some(HlsTsPesInspectionState::Header { started_at_packet_index, .. }) => { - return Err(HlsTsSpliceIncompatibility::IncompletePes { - pid, - packet_index: started_at_packet_index, - declared_bytes: None, - observed_bytes: 0, - }); - } - Some(HlsTsPesInspectionState::Finite { declared_bytes, observed_bytes, started_at_packet_index }) => { - return Err(HlsTsSpliceIncompatibility::IncompletePes { - pid, - packet_index: started_at_packet_index, - declared_bytes: Some(declared_bytes), - observed_bytes, - }); - } - Some(HlsTsPesInspectionState::UnboundedVideo) | None => {} - } - Some(HlsTsPesInspectionState::Header { bytes: [0; 6], len: 0, started_at_packet_index: packet_index }) - } else { - previous - }; - let (mut bytes, mut len, started_at_packet_index) = match state { - Some(HlsTsPesInspectionState::Header { bytes, len, started_at_packet_index }) => { - (bytes, len, started_at_packet_index) - } - Some(HlsTsPesInspectionState::Finite { declared_bytes, observed_bytes, started_at_packet_index }) => { - let available = u64::try_from(payload.len()).unwrap_or(u64::MAX); - let observed_bytes = observed_bytes.saturating_add(available).min(u64::from(declared_bytes)); - return if observed_bytes == u64::from(declared_bytes) { - Ok(None) - } else { - Ok(Some(HlsTsPesInspectionState::Finite { declared_bytes, observed_bytes, started_at_packet_index })) - }; - } - Some(HlsTsPesInspectionState::UnboundedVideo) => { - return Ok(Some(HlsTsPesInspectionState::UnboundedVideo)); - } - None => return Ok(None), - }; - let copied = payload.len().min(6usize.saturating_sub(len)); - bytes[len..len.saturating_add(copied)].copy_from_slice(&payload[..copied]); - len = len.saturating_add(copied); - if len >= 3 && bytes[..3] != [0, 0, 1] { - if invalid_pes_start.is_none() { - *invalid_pes_start = Some(started_at_packet_index); - } - return Ok(None); - } - if len < 6 { - return Ok(Some(HlsTsPesInspectionState::Header { bytes, len, started_at_packet_index })); - } - let stream_id = bytes[3]; - let declared_bytes = u16::from_be_bytes([bytes[4], bytes[5]]); - if declared_bytes == 0 { - return if (0xE0..=0xEF).contains(&stream_id) { - Ok(Some(HlsTsPesInspectionState::UnboundedVideo)) - } else { - Err(HlsTsSpliceIncompatibility::InvalidPes { pid, packet_index }) - }; - } - let observed_bytes = - u64::try_from(payload.len().saturating_sub(copied)).unwrap_or(u64::MAX).min(u64::from(declared_bytes)); - if observed_bytes == u64::from(declared_bytes) { - Ok(None) - } else { - Ok(Some(HlsTsPesInspectionState::Finite { declared_bytes, observed_bytes, started_at_packet_index })) - } -} - struct HlsTsMediaStreamInspector { budget: HlsTsProbeBudget, timestamp_scanner: HlsTsTimestampProfileScanner, @@ -1417,104 +119,6 @@ struct HlsTsMediaStreamInspector { invalid: bool, } -#[derive(Clone, Copy)] -enum HlsTsPlaintextRemainder { - ExactPackets, - Aes128Pkcs7, -} - -impl HlsTsMediaStreamInspector { - fn new(budget: HlsTsProbeBudget, expected_duration_ticks_90khz: u64) -> Self { - Self { - budget, - timestamp_scanner: HlsTsTimestampProfileScanner::new(expected_duration_ticks_90khz), - transport_scanner: HlsTsTransportStreamInspector::new(), - pending: Vec::with_capacity(budget.bounded_read_chunk_bytes().saturating_add(TS_PACKET_BYTES)), - aligned: false, - packets_examined: 0, - invalid: false, - } - } - - fn feed_plaintext(&mut self, bytes: &[u8]) { - if self.invalid { - return; - } - self.pending.extend_from_slice(bytes); - if !self.aligned { - let confirmation_bytes = TS_PACKET_BYTES.saturating_mul(TS_ALIGNMENT_CONFIRMATION_PACKETS); - if self.pending.len() < confirmation_bytes { - return; - } - let available_search = self.pending.len().saturating_sub(confirmation_bytes); - let search_end = available_search.min(self.budget.max_resync_bytes); - let alignment = (0..=search_end).find(|offset| { - (0..TS_ALIGNMENT_CONFIRMATION_PACKETS).all(|index| { - self.pending[offset.saturating_add(index.saturating_mul(TS_PACKET_BYTES))] == Packet::SYNC_BYTE - }) - }); - let Some(alignment) = alignment else { - if available_search >= self.budget.max_resync_bytes { - self.invalid = true; - } - return; - }; - self.pending.drain(..alignment); - self.aligned = true; - } - - let mut consumed = 0usize; - while self.pending.len().saturating_sub(consumed) >= TS_PACKET_BYTES { - if self.packets_examined >= self.budget.max_packets { - self.invalid = true; - break; - } - let packet_end = consumed.saturating_add(TS_PACKET_BYTES); - let packet = &self.pending[consumed..packet_end]; - self.timestamp_scanner.push_aligned_packet(packet); - self.transport_scanner.push_packet(packet, self.packets_examined); - self.packets_examined = self.packets_examined.saturating_add(1); - consumed = packet_end; - } - self.pending.drain(..consumed); - } - - fn finish( - self, - remainder: HlsTsPlaintextRemainder, - topology: Option, - ) -> (Option, HlsTsSpliceEvidence) { - let valid_remainder = match remainder { - HlsTsPlaintextRemainder::ExactPackets => self.pending.is_empty(), - HlsTsPlaintextRemainder::Aes128Pkcs7 => { - let Some(&padding) = self.pending.last() else { - return ( - None, - HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::InvalidPacket { - packet_index: self.packets_examined, - }), - ); - }; - let padding = usize::from(padding); - (1..=AES_128_BLOCK_BYTES).contains(&padding) - && self.pending.len() == padding - && self.pending.iter().all(|byte| usize::from(*byte) == padding) - } - }; - if self.invalid || !self.aligned || !valid_remainder { - return ( - None, - HlsTsSpliceEvidence::Incompatible(if self.packets_examined >= self.budget.max_packets { - HlsTsSpliceIncompatibility::InspectionBudgetExhausted - } else { - HlsTsSpliceIncompatibility::InvalidPacket { packet_index: self.packets_examined } - }), - ); - } - (self.timestamp_scanner.finish(), self.transport_scanner.finish(topology)) - } -} - struct HlsTsInspector { budget: HlsTsProbeBudget, demux: demultiplex::Demultiplex, @@ -1525,1520 +129,32 @@ struct HlsTsInspector { packets_examined: u64, } -#[derive(Clone, Copy, PartialEq, Eq)] -enum HlsTsInspectionMode { - Prefix, - Complete, -} - -impl HlsTsInspector { - fn new(budget: HlsTsProbeBudget) -> Self { - let mut context = HlsTsDemuxContext::new(); - let demux = demultiplex::Demultiplex::new(&mut context); - Self { - budget, - demux, - context, - pending: Vec::with_capacity(budget.bounded_read_chunk_bytes().saturating_add(TS_PACKET_BYTES)), - aligned: false, - source_bytes_examined: 0, - packets_examined: 0, - } - } - - fn record_source_bytes(&mut self, bytes: usize) { - self.source_bytes_examined = - self.source_bytes_examined.saturating_add(u64::try_from(bytes).unwrap_or(u64::MAX)); - } - - fn budget_exhausted(&self) -> HlsTsProbeOutcome { - HlsTsProbeOutcome::ProbeBudgetExhausted { - bytes_examined: self.source_bytes_examined, - packets_examined: self.packets_examined, - } - } - - fn feed_plaintext(&mut self, bytes: &[u8]) -> Option { - self.feed_plaintext_with_mode(bytes, HlsTsInspectionMode::Prefix) - } - - fn feed_plaintext_complete(&mut self, bytes: &[u8]) -> Option { - self.feed_plaintext_with_mode(bytes, HlsTsInspectionMode::Complete) - } - - fn feed_plaintext_with_mode(&mut self, bytes: &[u8], mode: HlsTsInspectionMode) -> Option { - self.pending.extend_from_slice(bytes); - if !self.aligned { - let confirmation_bytes = TS_PACKET_BYTES.saturating_mul(TS_ALIGNMENT_CONFIRMATION_PACKETS); - if self.pending.len() < confirmation_bytes { - return None; - } - let available_search = self.pending.len().saturating_sub(confirmation_bytes); - let search_end = available_search.min(self.budget.max_resync_bytes); - let alignment = (0..=search_end).find(|offset| { - (0..TS_ALIGNMENT_CONFIRMATION_PACKETS).all(|index| { - self.pending[offset.saturating_add(index.saturating_mul(TS_PACKET_BYTES))] == Packet::SYNC_BYTE - }) - }); - let Some(alignment) = alignment else { - if available_search >= self.budget.max_resync_bytes { - return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidSynchronization)); - } - return None; - }; - self.pending.drain(..alignment); - self.aligned = true; - } - - let mut consumed = 0usize; - while self.pending.len().saturating_sub(consumed) >= TS_PACKET_BYTES { - if self.packets_examined >= self.budget.max_packets { - self.pending.drain(..consumed); - return Some(self.budget_exhausted()); - } - let mut packet_bytes = [0_u8; TS_PACKET_BYTES]; - packet_bytes.copy_from_slice(&self.pending[consumed..consumed.saturating_add(TS_PACKET_BYTES)]); - consumed = consumed.saturating_add(TS_PACKET_BYTES); - self.packets_examined = self.packets_examined.saturating_add(1); - if let Some(outcome) = self.validate_packet(&packet_bytes) { - self.pending.drain(..consumed); - return Some(outcome); - } - self.demux.push(&mut self.context, &packet_bytes); - if let Some(reason) = self.context.malformed { - self.pending.drain(..consumed); - return Some(HlsTsProbeOutcome::Malformed(reason)); - } - if mode == HlsTsInspectionMode::Prefix { - if let Some(signature) = self.context.signature() { - self.pending.drain(..consumed); - return Some(HlsTsProbeOutcome::Found(signature)); - } - } - } - self.pending.drain(..consumed); - None - } - - fn validate_packet(&self, bytes: &[u8; TS_PACKET_BYTES]) -> Option { - if bytes[0] != Packet::SYNC_BYTE { - return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidSynchronization)); - } - if bytes[1] & 0x80 != 0 { - return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::TransportError)); - } - if bytes[3] & 0xC0 != 0 { - return Some(HlsTsProbeOutcome::UnsupportedProtection(HlsTsProtectionReason::TransportScrambling)); - } - let adaptation_control = (bytes[3] >> 4) & 0b11; - let payload_offset = match adaptation_control { - 0b01 => 4, - 0b10 if bytes[4] == 183 => TS_PACKET_BYTES, - 0b11 if bytes[4] <= 182 => 5usize.saturating_add(usize::from(bytes[4])), - 0b00 | 0b10 | 0b11 => { - return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPacketHeader)); - } - _ => return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPacketHeader)), - }; - let pid = (u16::from(bytes[1] & 0x1F) << 8) | u16::from(bytes[2]); - if self.context.is_psi_pid(pid) && bytes[1] & 0x40 != 0 { - let Some(payload) = bytes.get(payload_offset..) else { - return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPsiPointer)); - }; - let Some(pointer) = payload.first().map(|value| usize::from(*value)) else { - return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPsiPointer)); - }; - let section_start = pointer.saturating_add(1); - if section_start >= payload.len() { - return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPsiPointer)); - } - } - None - } - - fn finish(self) -> HlsTsProbeOutcome { - if let Some(reason) = self.context.malformed { - return HlsTsProbeOutcome::Malformed(reason); - } - if let Some(signature) = self.context.signature() { - return HlsTsProbeOutcome::Found(signature); - } - if !self.aligned { - return HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidSynchronization); - } - if !self.pending.is_empty() { - return HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::IncompletePacket); - } - HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::IncompleteProgramMetadata) - } - - fn finish_complete(mut self, remainder: HlsTsPlaintextRemainder) -> HlsTsProbeOutcome { - if matches!(remainder, HlsTsPlaintextRemainder::Aes128Pkcs7) { - if let Some(&padding) = self.pending.last() { - let padding = usize::from(padding); - if (1..=AES_128_BLOCK_BYTES).contains(&padding) - && self.pending.len() == padding - && self.pending.iter().all(|byte| usize::from(*byte) == padding) - { - self.pending.clear(); - } - } - } - self.finish() - } -} - struct HlsAes128CbcPrefixDecoder { cipher: Aes128, previous_ciphertext: [u8; AES_128_BLOCK_BYTES], carry: Zeroizing>, } -impl HlsAes128CbcPrefixDecoder { - fn new(key: &[u8], iv: [u8; AES_128_BLOCK_BYTES]) -> Result { - if key.len() != AES_128_BLOCK_BYTES { - return Err(HlsTsProbeError::KeyUnavailable); - } - let cipher = Aes128::new_from_slice(key).map_err(|_| HlsTsProbeError::KeyUnavailable)?; - Ok(Self { cipher, previous_ciphertext: iv, carry: Zeroizing::new(Vec::with_capacity(AES_128_BLOCK_BYTES)) }) - } - - fn push(&mut self, ciphertext: &[u8]) -> Zeroizing> { - let mut plaintext = Zeroizing::new(Vec::with_capacity( - self.carry.len().saturating_add(ciphertext.len()) / AES_128_BLOCK_BYTES * AES_128_BLOCK_BYTES, - )); - let mut input = ciphertext; - if !self.carry.is_empty() { - let required = AES_128_BLOCK_BYTES.saturating_sub(self.carry.len()); - let copied = required.min(input.len()); - self.carry.extend_from_slice(&input[..copied]); - input = &input[copied..]; - if self.carry.len() == AES_128_BLOCK_BYTES { - let mut block = [0_u8; AES_128_BLOCK_BYTES]; - block.copy_from_slice(&self.carry); - self.decrypt_block(block, &mut plaintext); - self.carry.clear(); - } - } - let (chunks, remainder) = input.as_chunks::(); - for chunk in chunks { - let block = *chunk; - self.decrypt_block(block, &mut plaintext); - } - self.carry.extend_from_slice(remainder); - plaintext - } - - fn decrypt_block(&mut self, ciphertext: [u8; AES_128_BLOCK_BYTES], plaintext: &mut Vec) { - let mut decrypted = Block::::default(); - decrypted.copy_from_slice(&ciphertext); - self.cipher.decrypt_block(&mut decrypted); - plaintext.extend(decrypted.into_iter().zip(self.previous_ciphertext).map(|(byte, previous)| byte ^ previous)); - self.previous_ciphertext = ciphertext; - } - - fn finish(&self) -> Result<(), HlsTsProbeError> { - self.carry.is_empty().then_some(()).ok_or(HlsTsProbeError::DecryptionFailed) - } -} - -impl Drop for HlsAes128CbcPrefixDecoder { - fn drop(&mut self) { self.previous_ciphertext.zeroize(); } -} - -enum HlsTsSourceDecoder { - Clear, - Aes128Cbc(Box), -} - -impl HlsTsSourceDecoder { - fn new(protection: HlsTsProbeProtection<'_>) -> Result { - match protection { - HlsTsProbeProtection::Clear => Ok(Self::Clear), - HlsTsProbeProtection::Aes128Cbc { key, iv } => { - HlsAes128CbcPrefixDecoder::new(key, iv).map(Box::new).map(Self::Aes128Cbc) - } - } - } - - fn with_plaintext(&mut self, bytes: &[u8], consume: impl FnOnce(&[u8]) -> T) -> T { - match self { - Self::Clear => consume(bytes), - Self::Aes128Cbc(decoder) => { - let plaintext = decoder.push(bytes); - consume(&plaintext) - } - } - } - - const fn plaintext_remainder(&self) -> HlsTsPlaintextRemainder { - match self { - Self::Clear => HlsTsPlaintextRemainder::ExactPackets, - Self::Aes128Cbc(_) => HlsTsPlaintextRemainder::Aes128Pkcs7, - } - } - - fn finish(&self) -> Result<(), HlsTsProbeError> { - match self { - Self::Clear => Ok(()), - Self::Aes128Cbc(decoder) => decoder.finish(), - } - } -} - -/// Inspects a blocking reader without retaining or mutating source media bytes. -pub fn inspect_mpeg_ts( - mut reader: R, - protection: HlsTsProbeProtection<'_>, - budget: HlsTsProbeBudget, -) -> Result { - let mut decoder = HlsTsSourceDecoder::new(protection)?; - let chunk_bytes = budget.bounded_read_chunk_bytes(); - let mut buffer = Zeroizing::new(vec![0_u8; chunk_bytes]); - let mut inspector = HlsTsInspector::new(budget); - loop { - let remaining = budget.max_bytes.saturating_sub(inspector.source_bytes_examined); - if remaining == 0 { - return Ok(inspector.budget_exhausted()); - } - let read_limit = usize::try_from(remaining).unwrap_or(usize::MAX).min(buffer.len()); - let read = match reader.read(&mut buffer[..read_limit]) { - Ok(0) => break, - Ok(read) => read, - Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, - Err(error) => return Err(HlsTsProbeError::Io(error)), - }; - inspector.record_source_bytes(read); - if let Some(outcome) = decoder.with_plaintext(&buffer[..read], |plaintext| inspector.feed_plaintext(plaintext)) - { - return Ok(outcome); - } - } - decoder.finish()?; - Ok(inspector.finish()) -} - -/// Async cache-file entry point backed by the same parser and CBC state machine. -pub async fn inspect_mpeg_ts_async( - mut reader: R, - protection: HlsTsProbeProtection<'_>, - budget: HlsTsProbeBudget, -) -> Result { - let mut decoder = HlsTsSourceDecoder::new(protection)?; - let chunk_bytes = budget.bounded_read_chunk_bytes(); - let mut buffer = Zeroizing::new(vec![0_u8; chunk_bytes]); - let mut inspector = HlsTsInspector::new(budget); - loop { - let remaining = budget.max_bytes.saturating_sub(inspector.source_bytes_examined); - if remaining == 0 { - return Ok(inspector.budget_exhausted()); - } - let read_limit = usize::try_from(remaining).unwrap_or(usize::MAX).min(buffer.len()); - let read = reader.read(&mut buffer[..read_limit]).await.map_err(HlsTsProbeError::Io)?; - if read == 0 { - break; - } - inspector.record_source_bytes(read); - if let Some(outcome) = decoder.with_plaintext(&buffer[..read], |plaintext| inspector.feed_plaintext(plaintext)) - { - return Ok(outcome); - } - } - decoder.finish()?; - Ok(inspector.finish()) -} - -/// Collects track compatibility and complete clock evidence from one exact cache reader. -/// -/// PAT/PMT discovery may settle from the prefix, while timestamp collection continues to -/// EOF in bounded chunks. Valid AES-CBC padding is excluded from the aligned TS stream. -pub async fn inspect_mpeg_ts_media_evidence_async( - mut reader: R, - protection: HlsTsProbeProtection<'_>, - budget: HlsTsProbeBudget, - expected_duration_ticks_90khz: u64, -) -> Result { - let mut decoder = HlsTsSourceDecoder::new(protection)?; - let plaintext_remainder = decoder.plaintext_remainder(); - let chunk_bytes = budget.bounded_read_chunk_bytes(); - let mut buffer = Zeroizing::new(vec![0_u8; chunk_bytes]); - let mut track_inspector = HlsTsInspector::new(budget); - let mut media_inspector = HlsTsMediaStreamInspector::new(budget, expected_duration_ticks_90khz); - let mut track_outcome = None; - let mut source_bytes_examined = 0_u64; - loop { - let remaining = budget.max_bytes.saturating_sub(source_bytes_examined); - if remaining == 0 { - return Ok(HlsTsMediaEvidence { - track_outcome: track_outcome.unwrap_or_else(|| track_inspector.budget_exhausted()), - timestamp_profile: None, - splice_evidence: HlsTsSpliceEvidence::Incompatible( - HlsTsSpliceIncompatibility::InspectionBudgetExhausted, - ), - }); - } - let read_limit = usize::try_from(remaining).unwrap_or(usize::MAX).min(buffer.len()); - let read = reader.read(&mut buffer[..read_limit]).await.map_err(HlsTsProbeError::Io)?; - if read == 0 { - break; - } - source_bytes_examined = source_bytes_examined.saturating_add(u64::try_from(read).unwrap_or(u64::MAX)); - if track_outcome.is_none() { - track_inspector.record_source_bytes(read); - } - decoder.with_plaintext(&buffer[..read], |plaintext| { - media_inspector.feed_plaintext(plaintext); - if track_outcome.is_none() { - track_outcome = track_inspector.feed_plaintext_complete(plaintext); - } - }); - } - decoder.finish()?; - let track_outcome = track_outcome.unwrap_or_else(|| track_inspector.finish_complete(plaintext_remainder)); - let topology = match &track_outcome { - HlsTsProbeOutcome::Found(signature) => Some(signature.clone()), - HlsTsProbeOutcome::ProbeBudgetExhausted { .. } - | HlsTsProbeOutcome::Malformed(_) - | HlsTsProbeOutcome::UnsupportedProtection(_) => None, - }; - let (timestamp_profile, splice_evidence) = media_inspector.finish(plaintext_remainder, topology); - Ok(HlsTsMediaEvidence { track_outcome, timestamp_profile, splice_evidence }) -} - #[cfg(test)] -mod tests { - use super::*; - use aes::cipher::BlockEncrypt; - use std::{ - cell::Cell, - io::Cursor, - pin::Pin, - rc::Rc, - sync::atomic::{AtomicU64, Ordering}, - task::{Context, Poll}, - }; - use tokio::io::ReadBuf; - - fn append_crc(mut section: Vec) -> Vec { - let crc = mpegts_crc::sum32(§ion); - section.extend_from_slice(&crc.to_be_bytes()); - section - } - - fn psi_version_byte(version: u8, current: bool) -> u8 { 0xC0 | ((version & 0x1F) << 1) | u8::from(current) } - - fn pat_section_with_header( - programs: &[(u16, u16)], - version: u8, - current: bool, - section_number: u8, - last_section_number: u8, - ) -> Vec { - let section_length = 5usize.saturating_add(programs.len().saturating_mul(4)).saturating_add(4); - let mut section = vec![ - 0x00, - 0xB0 | u8::try_from(section_length >> 8).unwrap_or(0), - u8::try_from(section_length & 0xFF).unwrap_or(0), - ]; - section.extend_from_slice(&[ - 0x00, - 0x01, - psi_version_byte(version, current), - section_number, - last_section_number, - ]); - for (program_number, pid) in programs { - section.extend_from_slice(&program_number.to_be_bytes()); - section.extend_from_slice(&(0xE000 | (pid & 0x1FFF)).to_be_bytes()); - } - append_crc(section) - } - - fn pat_section(programs: &[(u16, u16)]) -> Vec { pat_section_with_header(programs, 0, true, 0, 0) } - - #[derive(Clone, Copy)] - struct PmtSectionHeader { - version: u8, - current: bool, - section_number: u8, - last_section_number: u8, - } - - fn pmt_section_with_header( - program_number: u16, - pcr_pid: u16, - streams: &[(u8, u16)], - descriptor_bytes: usize, - header: PmtSectionHeader, - ) -> Vec { - let program_info_length = descriptor_bytes.min(0x0FFF) & !1; - let section_length = 9usize - .saturating_add(program_info_length) - .saturating_add(streams.len().saturating_mul(5)) - .saturating_add(4); - let mut section = vec![ - 0x02, - 0xB0 | u8::try_from(section_length >> 8).unwrap_or(0), - u8::try_from(section_length & 0xFF).unwrap_or(0), - ]; - section.extend_from_slice(&program_number.to_be_bytes()); - section.extend_from_slice(&[ - psi_version_byte(header.version, header.current), - header.section_number, - header.last_section_number, - ]); - section.extend_from_slice(&(0xE000 | (pcr_pid & 0x1FFF)).to_be_bytes()); - section.extend_from_slice(&(0xF000 | u16::try_from(program_info_length).unwrap_or(0)).to_be_bytes()); - for _ in 0..program_info_length / 2 { - section.extend_from_slice(&[0x80, 0x00]); - } - for (stream_type, pid) in streams { - section.push(*stream_type); - section.extend_from_slice(&(0xE000 | (pid & 0x1FFF)).to_be_bytes()); - section.extend_from_slice(&[0xF0, 0x00]); - } - append_crc(section) - } - - fn pmt_section(program_number: u16, pcr_pid: u16, streams: &[(u8, u16)], descriptor_bytes: usize) -> Vec { - pmt_section_with_header( - program_number, - pcr_pid, - streams, - descriptor_bytes, - PmtSectionHeader { version: 0, current: true, section_number: 0, last_section_number: 0 }, - ) - } - - fn packetize_section(pid: u16, section: &[u8], first_counter: u8) -> Vec { - let mut output = Vec::new(); - let mut offset = 0usize; - let mut counter = first_counter & 0x0F; - let mut first = true; - while offset < section.len() { - let mut packet = [0xFF_u8; TS_PACKET_BYTES]; - packet[0] = Packet::SYNC_BYTE; - packet[1] = u8::try_from(pid >> 8).unwrap_or(0) & 0x1F; - packet[2] = u8::try_from(pid & 0xFF).unwrap_or(0); - packet[3] = 0x10 | counter; - let payload_start = if first { - packet[1] |= 0x40; - packet[4] = 0; - 5 - } else { - 4 - }; - let copied = section.len().saturating_sub(offset).min(TS_PACKET_BYTES.saturating_sub(payload_start)); - packet[payload_start..payload_start.saturating_add(copied)] - .copy_from_slice(§ion[offset..offset.saturating_add(copied)]); - output.extend_from_slice(&packet); - offset = offset.saturating_add(copied); - counter = counter.wrapping_add(1) & 0x0F; - first = false; - } - output - } - - fn packetize_section_with_split_syntax_header(pid: u16, section: &[u8], first_header_bytes: usize) -> Vec { - let first_header_bytes = first_header_bytes.min(PSI_SYNTAX_HEADER_BYTES.saturating_sub(1)); - let pid = Pid::new(pid); - let mut output = synthetic_psi_packet(pid, true, §ion[..first_header_bytes]).to_vec(); - for chunk in section[first_header_bytes..].chunks(184) { - output.extend_from_slice(&synthetic_psi_packet(pid, false, chunk)); - } - output - } - - fn psi_start_packet_with_pointer(pid: u16, previous_section: &[u8], section_start: &[u8]) -> [u8; TS_PACKET_BYTES] { - let payload_length = 1usize.saturating_add(previous_section.len()).saturating_add(section_start.len()).min(184); - let mut packet = [0xFF_u8; TS_PACKET_BYTES]; - packet[0] = Packet::SYNC_BYTE; - packet[1] = (u8::try_from(pid >> 8).unwrap_or(0) & 0x1F) | 0x40; - packet[2] = u8::try_from(pid & 0xFF).unwrap_or(0); - packet[3] = 0x30; - let adaptation_length = 183usize.saturating_sub(payload_length); - packet[4] = u8::try_from(adaptation_length).unwrap_or(182); - if adaptation_length > 0 { - packet[5] = 0; - } - let payload_offset = 5usize.saturating_add(adaptation_length); - packet[payload_offset] = u8::try_from(previous_section.len()).unwrap_or(u8::MAX); - let previous_offset = payload_offset.saturating_add(1); - let previous_end = previous_offset.saturating_add(previous_section.len()); - packet[previous_offset..previous_end].copy_from_slice(previous_section); - let section_end = previous_end.saturating_add(section_start.len()); - packet[previous_end..section_end].copy_from_slice(section_start); - packet - } - - fn null_packet() -> [u8; TS_PACKET_BYTES] { - let mut packet = [0xFF_u8; TS_PACKET_BYTES]; - packet[0] = Packet::SYNC_BYTE; - packet[1] = 0x1F; - packet[2] = 0xFF; - packet[3] = 0x10; - packet - } - - fn track_stream(descriptor_bytes: usize) -> Vec { - let mut stream = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); - stream.extend_from_slice(&packetize_section( - 0x100, - &pmt_section(1, 0x101, &[(0x1B, 0x101), (0x0F, 0x102), (0x0F, 0x103)], descriptor_bytes), - 0, - )); - stream.extend_from_slice(&null_packet()); - stream - } - - fn media_payload_packet( - pid: u16, - continuity_counter: u8, - payload_unit_start: bool, - discontinuity: bool, - payload: &[u8], - ) -> [u8; TS_PACKET_BYTES] { - assert!(!payload.is_empty() && payload.len() <= 182); - let mut packet = [0xFF_u8; TS_PACKET_BYTES]; - packet[0] = Packet::SYNC_BYTE; - packet[1] = u8::try_from(pid >> 8).unwrap_or(0) & 0x1F; - if payload_unit_start { - packet[1] |= 0x40; - } - packet[2] = u8::try_from(pid & 0xFF).unwrap_or(0); - packet[3] = 0x30 | (continuity_counter & 0x0F); - let adaptation_length = 183usize.saturating_sub(payload.len()); - packet[4] = u8::try_from(adaptation_length).unwrap_or(182); - if adaptation_length > 0 { - packet[5] = if discontinuity { 0x80 } else { 0 }; - } - let payload_offset = 5usize.saturating_add(adaptation_length); - packet[payload_offset..].copy_from_slice(payload); - packet - } - - fn media_adaptation_only_packet(pid: u16, continuity_counter: u8, discontinuity: bool) -> [u8; TS_PACKET_BYTES] { - let mut packet = [0xFF_u8; TS_PACKET_BYTES]; - packet[0] = Packet::SYNC_BYTE; - packet[1] = u8::try_from(pid >> 8).unwrap_or(0) & 0x1F; - packet[2] = u8::try_from(pid & 0xFF).unwrap_or(0); - packet[3] = 0x20 | (continuity_counter & 0x0F); - packet[4] = 183; - packet[5] = if discontinuity { 0x80 } else { 0 }; - packet - } - - fn pes_bytes(stream_id: u8, declared_bytes: u16, payload_after_length: &[u8]) -> Vec { - let mut pes = vec![0, 0, 1, stream_id]; - pes.extend_from_slice(&declared_bytes.to_be_bytes()); - pes.extend_from_slice(payload_after_length); - pes - } - - async fn complete_media_evidence(bytes: &[u8]) -> HlsTsMediaEvidence { - complete_media_evidence_with_duration(bytes, 90_000).await - } - - async fn complete_media_evidence_with_duration( - bytes: &[u8], - expected_duration_ticks_90khz: u64, - ) -> HlsTsMediaEvidence { - let source_size = u64::try_from(bytes.len()).unwrap_or(u64::MAX); - inspect_mpeg_ts_media_evidence_async( - bytes, - HlsTsProbeProtection::Clear, - HlsTsProbeBudget { - max_bytes: source_size.saturating_add(1), - max_packets: source_size.saturating_add(187).saturating_div(188).saturating_add(1), - ..HlsTsProbeBudget::default() - }, - expected_duration_ticks_90khz, - ) - .await - .expect("complete media evidence") - } - - fn valid_track_prefix(minimum_bytes: usize) -> Vec { - let mut stream = track_stream(0); - let null_packet = null_packet(); - while stream.len() < minimum_bytes { - stream.extend_from_slice(&null_packet); - } - stream - } - - fn multi_packet_pat_stream() -> Vec { - let programs = - (0_u16..46).map(|index| (index.saturating_add(1), 0x100_u16.saturating_add(index))).collect::>(); - let mut stream = packetize_section(0, &pat_section(&programs), 0); - for (program_number, pmt_pid) in programs { - stream.extend_from_slice(&packetize_section( - pmt_pid, - &pmt_section(program_number, 0x400, &[(0x1B, 0x400), (0x0F, 0x401)], 0), - 0, - )); - } - stream.extend_from_slice(&null_packet()); - stream - } - - fn found(outcome: HlsTsProbeOutcome) -> HlsTsTrackSignature { - match outcome { - HlsTsProbeOutcome::Found(signature) => signature, - other => panic!("expected signature, got {other:?}"), - } - } - - #[test] - fn ts_inspector_reads_single_packet_pat_and_pmt() { - let signature = found( - inspect_mpeg_ts(Cursor::new(track_stream(0)), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe succeeds"), - ); - assert_eq!(signature.program_count, 1); - assert!(signature.has_pcr); - assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x1B]); - assert_eq!( - signature.programs.as_ref(), - &[HlsTsProgramTopology { - transport_stream_id: 1, - program_number: 1, - pmt_pid: 0x100, - pcr_pid: 0x101, - streams: Arc::from([ - HlsTsElementaryStreamBinding { stream_type: 0x1B, elementary_pid: 0x101 }, - HlsTsElementaryStreamBinding { stream_type: 0x0F, elementary_pid: 0x102 }, - HlsTsElementaryStreamBinding { stream_type: 0x0F, elementary_pid: 0x103 }, - ]), - }] - ); - } - - #[test] - fn equal_stream_types_with_different_pid_topology_are_not_compatible() { - let mut first = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); - first.extend_from_slice(&packetize_section( - 0x100, - &pmt_section(1, 0x101, &[(0x1B, 0x101), (0x0F, 0x102)], 0), - 0, - )); - let mut second = packetize_section(0, &pat_section(&[(1, 0x200)]), 0); - second.extend_from_slice(&packetize_section( - 0x200, - &pmt_section(1, 0x201, &[(0x1B, 0x201), (0x0F, 0x202)], 0), - 0, - )); - - let first = found( - inspect_mpeg_ts(Cursor::new(first), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("first topology"), - ); - let second = found( - inspect_mpeg_ts(Cursor::new(second), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("second topology"), - ); - - assert_eq!(first.stream_types, second.stream_types); - assert_ne!(first, second); - assert_eq!( - evaluate_mpeg_ts_splice_boundary( - &HlsTsSpliceEvidence::compatible_for_test(first), - &HlsTsSpliceEvidence::compatible_for_test(second), - ), - Err(HlsTsSpliceBoundaryIncompatibility::TopologyMismatch) - ); - } - - #[tokio::test] - async fn complete_media_evidence_enforces_ffmpeg_continuity_semantics() { - let pid = 0x101; - let exact_pes = pes_bytes(0xE0, 4, &[1, 2, 3, 4]); - let next_pes = pes_bytes(0xE0, 0, &[0x80, 0, 0]); - let mut valid = track_stream(0); - valid.extend_from_slice(&media_payload_packet(pid, 3, true, false, &exact_pes)); - valid.extend_from_slice(&media_adaptation_only_packet(pid, 3, false)); - valid.extend_from_slice(&media_payload_packet(pid, 4, true, false, &next_pes)); - assert!(matches!(complete_media_evidence(&valid).await.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); - - let mut discontinuity = track_stream(0); - discontinuity.extend_from_slice(&media_payload_packet(pid, 3, true, false, &exact_pes)); - discontinuity.extend_from_slice(&media_adaptation_only_packet(pid, 11, true)); - discontinuity.extend_from_slice(&media_payload_packet(pid, 12, true, false, &next_pes)); - assert!(matches!( - complete_media_evidence(&discontinuity).await.splice_evidence, - HlsTsSpliceEvidence::Compatible(_) - )); - - let mut invalid = track_stream(0); - invalid.extend_from_slice(&media_payload_packet(pid, 3, true, false, &exact_pes)); - invalid.extend_from_slice(&media_adaptation_only_packet(pid, 4, false)); - assert!(matches!( - complete_media_evidence(&invalid).await.splice_evidence, - HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::ContinuityFailure { - pid: 0x101, - expected: 3, - actual: 4, - .. - }) - )); - } - - #[tokio::test] - async fn complete_media_evidence_rejects_tei_and_internal_payload_jump() { - let pid = 0x101; - let first = pes_bytes(0xE0, 200, &[0x11; 100]); - let continuation = [0x22; 100]; - let mut jumped = track_stream(0); - jumped.extend_from_slice(&media_payload_packet(pid, 5, true, false, &first)); - jumped.extend_from_slice(&media_payload_packet(pid, 9, false, false, &continuation)); - assert!(matches!( - complete_media_evidence(&jumped).await.splice_evidence, - HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::ContinuityFailure { - pid: 0x101, - expected: 6, - actual: 9, - .. - }) - )); - - let mut tei_packet = media_payload_packet(pid, 5, true, false, &pes_bytes(0xE0, 0, &[0x80])); - tei_packet[1] |= 0x80; - let mut tei = track_stream(0); - tei.extend_from_slice(&tei_packet); - assert!(matches!( - complete_media_evidence(&tei).await.splice_evidence, - HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::TransportError { pid: 0x101, .. }) - )); - } - - #[tokio::test] - async fn complete_media_evidence_accounts_finite_split_and_unbounded_pes() { - let pid = 0x101; - let mut truncated = track_stream(0); - truncated.extend_from_slice(&media_payload_packet(pid, 0, true, false, &pes_bytes(0xE0, 20, &[0xAA; 8]))); - assert!(matches!( - complete_media_evidence(&truncated).await.splice_evidence, - HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::IncompletePes { - pid: 0x101, - declared_bytes: Some(20), - observed_bytes: 8, - .. - }) - )); - - let mut exact = track_stream(0); - exact.extend_from_slice(&media_payload_packet(pid, 0, true, false, &pes_bytes(0xE0, 8, &[0xAA; 8]))); - assert!(matches!(complete_media_evidence(&exact).await.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); - - let header = pes_bytes(0xE0, 5, &[1, 2, 3, 4, 5]); - let mut split = track_stream(0); - split.extend_from_slice(&media_payload_packet(pid, 0, true, false, &header[..4])); - split.extend_from_slice(&media_payload_packet(pid, 1, false, false, &header[4..])); - assert!(matches!(complete_media_evidence(&split).await.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); - - let mut unbounded = track_stream(0); - unbounded.extend_from_slice(&media_payload_packet(pid, 0, true, false, &pes_bytes(0xE0, 0, &[0x80, 0, 0, 1]))); - assert!(matches!( - complete_media_evidence(&unbounded).await.splice_evidence, - HlsTsSpliceEvidence::Compatible(_) - )); - - let mut invalid = track_stream(0); - invalid.extend_from_slice(&media_payload_packet(pid, 0, true, false, &[0x12, 0x34, 0x56])); - assert!(matches!( - complete_media_evidence(&invalid).await.splice_evidence, - HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::InvalidPes { pid: 0x101, .. }) - )); - } - - #[tokio::test] - async fn current_terminal_asset_and_finalized_segment_zero_have_complete_splice_evidence() { - const TERMINAL_ASSET_BYTES: &[u8] = - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); - let renderer = crate::transport_stream_buffer::TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); - let duration_ticks = renderer.duration_ticks_90khz().expect("terminal asset duration"); - let asset_profile = renderer.finite_hls_timestamp_profile().expect("terminal asset profile"); - let prepared = renderer - .render_finite_hls_segment(crate::transport_stream_buffer::HlsFiniteTsRenderSpec { - timestamp_offset_ticks_90khz: 0, - continuity_seed: 0, - logical_segment_index: 0, - }) - .expect("prepared terminal segment zero"); - let finalized = renderer - .finalize_prepared_finite_hls_segment( - &prepared, - crate::transport_stream_buffer::HlsFiniteTsFinalizeSpec { - additional_timestamp_offset_ticks_90khz: 90_000, - discontinuity: crate::transport_stream_buffer::HlsFiniteTsDiscontinuityMode::FirstPacketPerPid, - }, - ) - .expect("finalized terminal segment zero"); - - let base = complete_media_evidence_with_duration(TERMINAL_ASSET_BYTES, duration_ticks).await; - let terminal = complete_media_evidence_with_duration(&finalized, duration_ticks).await; - - assert_eq!(base.timestamp_profile, Some(asset_profile)); - assert!(matches!(base.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); - assert!(matches!(terminal.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); - assert_eq!(evaluate_mpeg_ts_splice_boundary(&base.splice_evidence, &terminal.splice_evidence), Ok(())); - assert_eq!( - terminal.timestamp_profile.map(|profile| profile.span_ticks_90khz), - Some(asset_profile.span_ticks_90khz) - ); - assert!(duration_ticks > asset_profile.span_ticks_90khz); - } - - #[test] - fn ts_inspector_reassembles_multi_packet_and_cross_chunk_psi() { - let budget = HlsTsProbeBudget { read_chunk_bytes: 191, ..HlsTsProbeBudget::default() }; - let track_signature = found( - inspect_mpeg_ts(Cursor::new(track_stream(400)), HlsTsProbeProtection::Clear, budget) - .expect("probe succeeds"), - ); - assert_eq!(track_signature.stream_types.as_ref(), &[0x0F, 0x1B]); - - let program_signature = found( - inspect_mpeg_ts(Cursor::new(multi_packet_pat_stream()), HlsTsProbeProtection::Clear, budget) - .expect("probe succeeds"), - ); - assert_eq!(program_signature.program_count, 46); - assert_eq!(program_signature.stream_types.as_ref(), &[0x0F, 0x1B]); - } - - #[test] - fn ts_inspector_restarts_incomplete_pat_on_new_current_version() { - let mut stream = packetize_section(0, &pat_section_with_header(&[(1, 0x100)], 0, true, 0, 1), 0); - stream.extend_from_slice(&packetize_section( - 0x100, - &pmt_section_with_header( - 1, - 0x101, - &[(0x02, 0x101)], - 0, - PmtSectionHeader { version: 0, current: true, section_number: 0, last_section_number: 1 }, - ), - 0, - )); - stream.extend_from_slice(&packetize_section(0, &pat_section_with_header(&[(2, 0x200)], 1, true, 0, 0), 1)); - stream.extend_from_slice(&packetize_section( - 0x200, - &pmt_section(2, 0x201, &[(0x1B, 0x201), (0x0F, 0x202)], 0), - 0, - )); - stream.extend_from_slice(&null_packet()); - - let signature = found( - inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe succeeds"), - ); - - assert_eq!(signature.program_count, 1); - assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x1B]); - } - - #[test] - fn ts_inspector_restarts_incomplete_pmt_on_new_current_version() { - let mut stream = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); - stream.extend_from_slice(&packetize_section( - 0x100, - &pmt_section_with_header( - 1, - 0x101, - &[(0x1B, 0x101)], - 0, - PmtSectionHeader { version: 0, current: true, section_number: 0, last_section_number: 1 }, - ), - 0, - )); - stream.extend_from_slice(&packetize_section( - 0x100, - &pmt_section_with_header( - 1, - 0x201, - &[(0x24, 0x201), (0x0F, 0x202)], - 0, - PmtSectionHeader { version: 1, current: true, section_number: 0, last_section_number: 0 }, - ), - 1, - )); - stream.extend_from_slice(&null_packet()); - - let signature = found( - inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe succeeds"), - ); - - assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x24]); - } - - #[test] - fn ts_inspector_never_mixes_pat_or_pmt_sections_across_versions() { - let mut pat_version_mix = packetize_section(0, &pat_section_with_header(&[(1, 0x100)], 0, true, 0, 1), 0); - pat_version_mix.extend_from_slice(&packetize_section( - 0, - &pat_section_with_header(&[(2, 0x200)], 1, true, 1, 1), - 1, - )); - pat_version_mix.extend_from_slice(&packetize_section(0x200, &pmt_section(2, 0x201, &[(0x1B, 0x201)], 0), 0)); - assert_eq!( - inspect_mpeg_ts(Cursor::new(pat_version_mix), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default(),) - .expect("probe completes"), - HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::IncompleteProgramMetadata) - ); - - let mut program_map_version_mix = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); - program_map_version_mix.extend_from_slice(&packetize_section( - 0x100, - &pmt_section_with_header( - 1, - 0x101, - &[(0x1B, 0x101)], - 0, - PmtSectionHeader { version: 0, current: true, section_number: 0, last_section_number: 1 }, - ), - 0, - )); - program_map_version_mix.extend_from_slice(&packetize_section( - 0x100, - &pmt_section_with_header( - 1, - 0x102, - &[(0x0F, 0x102)], - 0, - PmtSectionHeader { version: 1, current: true, section_number: 1, last_section_number: 1 }, - ), - 1, - )); - assert_eq!( - inspect_mpeg_ts( - Cursor::new(program_map_version_mix), - HlsTsProbeProtection::Clear, - HlsTsProbeBudget::default(), - ) - .expect("probe completes"), - HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::IncompleteProgramMetadata) - ); - } - - #[test] - fn ts_inspector_rejects_same_version_pat_and_pmt_section_contradictions() { - let mut contradictory_pat = packetize_section(0, &pat_section_with_header(&[(1, 0x100)], 0, true, 0, 1), 0); - contradictory_pat.extend_from_slice(&packetize_section( - 0, - &pat_section_with_header(&[(1, 0x101)], 0, true, 0, 1), - 1, - )); - assert_eq!( - inspect_mpeg_ts(Cursor::new(contradictory_pat), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default(),) - .expect("probe completes"), - HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPat) - ); - - let mut stream = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); - for (counter, stream_type) in [(0, 0x1B), (1, 0x0F)] { - stream.extend_from_slice(&packetize_section( - 0x100, - &pmt_section_with_header( - 1, - 0x101, - &[(stream_type, 0x101)], - 0, - PmtSectionHeader { version: 0, current: true, section_number: 0, last_section_number: 1 }, - ), - counter, - )); - } - - assert_eq!( - inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe completes"), - HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPmt) - ); - } - - #[test] - fn ts_inspector_next_pat_or_pmt_does_not_change_current_evidence() { - let mut stream = packetize_section(0, &pat_section_with_header(&[(1, 0x100)], 0, true, 0, 1), 0); - stream.extend_from_slice(&packetize_section(0, &pat_section_with_header(&[(2, 0x200)], 1, false, 1, 1), 1)); - stream.extend_from_slice(&packetize_section(0, &pat_section_with_header(&[], 0, true, 1, 1), 2)); - for (counter, header, stream_type, pcr_pid, elementary_pid) in [ - ( - 0, - PmtSectionHeader { version: 0, current: true, section_number: 0, last_section_number: 1 }, - 0x1B, - 0x101, - 0x101, - ), - ( - 1, - PmtSectionHeader { version: 1, current: false, section_number: 1, last_section_number: 1 }, - 0x24, - 0x201, - 0x201, - ), - ( - 2, - PmtSectionHeader { version: 0, current: true, section_number: 1, last_section_number: 1 }, - 0x0F, - 0x101, - 0x102, - ), - ] { - stream.extend_from_slice(&packetize_section( - 0x100, - &pmt_section_with_header(1, pcr_pid, &[(stream_type, elementary_pid)], 0, header), - counter, - )); - } - - let signature = found( - inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe succeeds"), - ); - - assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x1B]); - } - - #[test] - fn ts_inspector_reassembles_syntax_headers_split_across_packets() { - let mut stream = packetize_section_with_split_syntax_header(0, &pat_section(&[(1, 0x100)]), 2); - stream.extend_from_slice(&packetize_section_with_split_syntax_header( - 0x100, - &pmt_section(1, 0x101, &[(0x1B, 0x101), (0x0F, 0x102)], 0), - 1, - )); - stream.extend_from_slice(&null_packet()); - let budget = HlsTsProbeBudget { read_chunk_bytes: 191, ..HlsTsProbeBudget::default() }; - - let signature = - found(inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, budget).expect("probe succeeds")); - - assert_eq!(signature.program_count, 1); - assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x1B]); - - let mut stream = packetize_section_with_split_syntax_header(0, &pat_section(&[(1, 0x100)]), 5); - stream.extend_from_slice(&packetize_section_with_split_syntax_header( - 0x100, - &pmt_section(1, 0x101, &[(0x1B, 0x101), (0x0F, 0x102)], 0), - 7, - )); - stream.extend_from_slice(&null_packet()); - let signature = - found(inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, budget).expect("probe succeeds")); - assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x1B]); - } - - #[test] - fn ts_inspector_accepts_pusi_pointer_that_completes_pending_syntax_header() { - let pat = pat_section(&[(1, 0x100)]); - let pmt = pmt_section(1, 0x101, &[(0x1B, 0x101), (0x0F, 0x102)], 0); - let mut stream = synthetic_psi_packet(Pid::new(0), true, &pat[..2]).to_vec(); - stream.extend_from_slice(&psi_start_packet_with_pointer(0, &pat[2..], &pat)); - stream.extend_from_slice(&synthetic_psi_packet(Pid::new(0x100), true, &pmt[..5])); - stream.extend_from_slice(&psi_start_packet_with_pointer(0x100, &pmt[5..], &pmt)); - stream.extend_from_slice(&null_packet()); - - let signature = found( - inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe succeeds"), - ); - - assert_eq!(signature.program_count, 1); - assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x1B]); - } - - #[test] - fn ts_inspector_rejects_pusi_without_section_start_bytes() { - let mut stream = psi_start_packet_with_pointer(0, &[], &[]).to_vec(); - stream.extend_from_slice(&null_packet()); - - let outcome = inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe completes"); - - assert!(matches!(outcome, HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPsiPointer))); - } - - struct CountingReader { - inner: Cursor>, - bytes_read: Rc>, - } - - impl Read for CountingReader { - fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { - let read = std::io::Read::read(&mut self.inner, buffer)?; - self.bytes_read.set(self.bytes_read.get().saturating_add(read)); - Ok(read) - } - } - - struct VirtualTailReader { - prefix: Arc<[u8]>, - logical_len: u64, - position: u64, - bytes_read: Arc, - } - - impl VirtualTailReader { - fn new(prefix: Arc<[u8]>, logical_len: u64, bytes_read: Arc) -> Self { - Self { prefix, logical_len, position: 0, bytes_read } - } - - fn read_into(&mut self, output: &mut [u8]) -> usize { - let remaining = self.logical_len.saturating_sub(self.position); - let read = usize::try_from(remaining).unwrap_or(usize::MAX).min(output.len()); - if read == 0 { - return 0; - } - let position = usize::try_from(self.position).unwrap_or(usize::MAX); - let prefix_read = self.prefix.len().saturating_sub(position).min(read); - if prefix_read > 0 { - output[..prefix_read].copy_from_slice(&self.prefix[position..position.saturating_add(prefix_read)]); - } - output[prefix_read..read].fill(0); - self.position = self.position.saturating_add(u64::try_from(read).unwrap_or(u64::MAX)); - self.bytes_read.fetch_add(u64::try_from(read).unwrap_or(u64::MAX), Ordering::Relaxed); - read - } - } - - impl Read for VirtualTailReader { - fn read(&mut self, output: &mut [u8]) -> std::io::Result { Ok(self.read_into(output)) } - } - - impl AsyncRead for VirtualTailReader { - fn poll_read( - mut self: Pin<&mut Self>, - _context: &mut Context<'_>, - output: &mut ReadBuf<'_>, - ) -> Poll> { - let read = self.as_mut().get_mut().read_into(output.initialize_unfilled()); - output.advance(read); - Poll::Ready(Ok(())) - } - } - - #[tokio::test] - async fn ts_inspector_async_clear_stops_in_small_prefix_of_logical_large_segment() { - const LOGICAL_SEGMENT_BYTES: u64 = 20 * 1024 * 1024; - let prefix: Arc<[u8]> = valid_track_prefix(HLS_TS_PROBE_READ_CHUNK_BYTES).into(); - let sync_read = Arc::new(AtomicU64::new(0)); - let async_read = Arc::new(AtomicU64::new(0)); - - let sync_signature = found( - inspect_mpeg_ts( - VirtualTailReader::new(Arc::clone(&prefix), LOGICAL_SEGMENT_BYTES, Arc::clone(&sync_read)), - HlsTsProbeProtection::Clear, - HlsTsProbeBudget::default(), - ) - .expect("sync probe succeeds"), - ); - let async_signature = found( - inspect_mpeg_ts_async( - VirtualTailReader::new(prefix, LOGICAL_SEGMENT_BYTES, Arc::clone(&async_read)), - HlsTsProbeProtection::Clear, - HlsTsProbeBudget::default(), - ) - .await - .expect("async probe succeeds"), - ); - - assert_eq!(async_signature, sync_signature); - assert_eq!(async_signature.stream_types.as_ref(), &[0x0F, 0x1B]); - assert!(sync_read.load(Ordering::Relaxed) <= HLS_TS_PROBE_READ_CHUNK_BYTES as u64); - assert!(async_read.load(Ordering::Relaxed) <= HLS_TS_PROBE_READ_CHUNK_BYTES as u64); - } - - fn encrypt_aes128_cbc(plaintext: &[u8], key: &[u8; AES_128_BLOCK_BYTES], iv: [u8; AES_128_BLOCK_BYTES]) -> Vec { - let cipher = Aes128::new_from_slice(key).expect("valid test key"); - let mut previous = iv; - let mut output = Vec::with_capacity(plaintext.len()); - for chunk in plaintext.as_chunks::().0 { - let mut block = Block::::default(); - for ((byte, plaintext), previous) in block.iter_mut().zip(chunk).zip(previous) { - *byte = plaintext ^ previous; - } - cipher.encrypt_block(&mut block); - previous.copy_from_slice(&block); - output.extend_from_slice(&block); - } - output - } - - fn encrypt_aes128_cbc_pkcs7( - plaintext: &[u8], - key: &[u8; AES_128_BLOCK_BYTES], - iv: [u8; AES_128_BLOCK_BYTES], - ) -> Vec { - let padding = AES_128_BLOCK_BYTES - plaintext.len() % AES_128_BLOCK_BYTES; - let mut padded = plaintext.to_vec(); - padded.resize(padded.len().saturating_add(padding), u8::try_from(padding).unwrap_or(0)); - encrypt_aes128_cbc(&padded, key, iv) - } - - #[tokio::test] - async fn aes128_terminal_base_timestamp_profile_uses_decrypted_cached_bytes() { - const TERMINAL_ASSET_BYTES: &[u8] = - include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); - let asset = crate::transport_stream_buffer::TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); - let expected_profile = asset.finite_hls_timestamp_profile().expect("fixture timestamp profile"); - let expected_signature = asset.finite_hls_track_signature().expect("fixture track signature"); - let key = *b"0123456789abcdef"; - let iv = [0xA5; AES_128_BLOCK_BYTES]; - let ciphertext = encrypt_aes128_cbc_pkcs7(TERMINAL_ASSET_BYTES, &key, iv); - let source_size = u64::try_from(ciphertext.len()).expect("ciphertext size"); - let budget = HlsTsProbeBudget { - max_bytes: source_size.saturating_add(1), - max_packets: source_size.saturating_add(187) / 188 + 1, - ..HlsTsProbeBudget::default() - }; - - let evidence = inspect_mpeg_ts_media_evidence_async( - &ciphertext[..], - HlsTsProbeProtection::Aes128Cbc { key: &key, iv }, - budget, - asset.duration_ticks_90khz().expect("fixture duration"), - ) - .await - .expect("AES media evidence"); - - assert_eq!(evidence.track_outcome, HlsTsProbeOutcome::Found(expected_signature)); - assert_eq!(evidence.timestamp_profile, Some(expected_profile)); - assert!(matches!(evidence.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); - } - - #[tokio::test] - async fn ts_inspector_async_aes_stops_in_small_prefix_of_logical_large_segment() { - const LOGICAL_SEGMENT_BYTES: u64 = 20 * 1024 * 1024; - let key = *b"0123456789abcdef"; - let iv = [0x5A; AES_128_BLOCK_BYTES]; - let mut plaintext = valid_track_prefix(HLS_TS_PROBE_READ_CHUNK_BYTES); - plaintext.resize(plaintext.len().next_multiple_of(AES_128_BLOCK_BYTES), 0xFF); - let ciphertext: Arc<[u8]> = encrypt_aes128_cbc(&plaintext, &key, iv).into(); - let sync_read = Arc::new(AtomicU64::new(0)); - let async_read = Arc::new(AtomicU64::new(0)); - - let sync_signature = found( - inspect_mpeg_ts( - VirtualTailReader::new(Arc::clone(&ciphertext), LOGICAL_SEGMENT_BYTES, Arc::clone(&sync_read)), - HlsTsProbeProtection::Aes128Cbc { key: &key, iv }, - HlsTsProbeBudget::default(), - ) - .expect("sync AES probe succeeds"), - ); - let async_signature = found( - inspect_mpeg_ts_async( - VirtualTailReader::new(ciphertext, LOGICAL_SEGMENT_BYTES, Arc::clone(&async_read)), - HlsTsProbeProtection::Aes128Cbc { key: &key, iv }, - HlsTsProbeBudget::default(), - ) - .await - .expect("async AES probe succeeds"), - ); - - assert_eq!(async_signature, sync_signature); - assert_eq!(async_signature.stream_types.as_ref(), &[0x0F, 0x1B]); - assert!(sync_read.load(Ordering::Relaxed) <= HLS_TS_PROBE_READ_CHUNK_BYTES as u64); - assert!(async_read.load(Ordering::Relaxed) <= HLS_TS_PROBE_READ_CHUNK_BYTES as u64); - } - - #[tokio::test] - async fn ts_inspector_sync_async_and_aes_prefix_paths_share_signature() { - let budget = HlsTsProbeBudget { read_chunk_bytes: 191, ..HlsTsProbeBudget::default() }; - let mut plaintext = track_stream(400); - let clear_signature = found( - inspect_mpeg_ts(Cursor::new(&plaintext), HlsTsProbeProtection::Clear, budget).expect("sync probe succeeds"), - ); - let async_clear_signature = found( - inspect_mpeg_ts_async(&plaintext[..], HlsTsProbeProtection::Clear, budget) - .await - .expect("async clear probe succeeds"), - ); - plaintext.resize(plaintext.len().next_multiple_of(AES_128_BLOCK_BYTES), 0xFF); - let key = *b"0123456789abcdef"; - let iv = [0xA5; AES_128_BLOCK_BYTES]; - let ciphertext = encrypt_aes128_cbc(&plaintext, &key, iv); - let async_aes_signature = found( - inspect_mpeg_ts_async(&ciphertext[..], HlsTsProbeProtection::Aes128Cbc { key: &key, iv }, budget) - .await - .expect("async AES probe succeeds"), - ); - - assert_eq!(async_clear_signature, clear_signature); - assert_eq!(async_aes_signature, clear_signature); - } - - #[test] - fn ts_inspector_reports_probe_budget_without_reading_whole_source() { - let mut bytes = Vec::new(); - for _ in 0..100 { - bytes.extend_from_slice(&null_packet()); - } - let count = Rc::new(Cell::new(0)); - let reader = CountingReader { inner: Cursor::new(bytes), bytes_read: Rc::clone(&count) }; - let budget = HlsTsProbeBudget { max_bytes: (TS_PACKET_BYTES * 4) as u64, ..HlsTsProbeBudget::default() }; - let outcome = inspect_mpeg_ts(reader, HlsTsProbeProtection::Clear, budget).expect("probe succeeds"); - assert!(matches!(outcome, HlsTsProbeOutcome::ProbeBudgetExhausted { .. })); - assert_eq!(count.get(), TS_PACKET_BYTES * 4); - } - - #[test] - fn ts_inspector_policy_resolution_preserves_all_probe_reasons() { - assert_eq!( - HlsTrackEvidenceResolution::from(Ok(HlsTsProbeOutcome::ProbeBudgetExhausted { - bytes_examined: 512, - packets_examined: 2, - })), - HlsTrackEvidenceResolution::InsufficientEvidence { bytes_examined: 512, packets_examined: 2 } - ); - assert_eq!( - HlsTrackEvidenceResolution::from(Ok(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPmt,))) - .reason_code(), - "invalid-pmt" - ); - assert_eq!( - HlsTrackEvidenceResolution::from(Ok(HlsTsProbeOutcome::Malformed( - HlsTsMalformedReason::IncompleteProgramMetadata, - ))), - HlsTrackEvidenceResolution::IncompleteEvidence - ); - let unsupported = HlsTrackEvidenceResolution::from(Ok(HlsTsProbeOutcome::UnsupportedProtection( - HlsTsProtectionReason::TransportScrambling, - ))); - assert_eq!( - unsupported, - HlsTrackEvidenceResolution::UnsupportedProtection(HlsTsProtectionReason::TransportScrambling) - ); - assert_eq!(unsupported.reason_code(), "transport-scrambling"); - let key_unavailable = HlsTrackEvidenceResolution::from(Err(HlsTsProbeError::KeyUnavailable)); - assert_eq!(key_unavailable, HlsTrackEvidenceResolution::KeyUnavailable); - assert_eq!(key_unavailable.reason_code(), "key-unavailable"); - let invalid_iv = HlsTrackEvidenceResolution::from(Err(HlsTsProbeError::InvalidIv)); - assert_eq!(invalid_iv, HlsTrackEvidenceResolution::InvalidIv); - assert_eq!(invalid_iv.reason_code(), "invalid-iv"); - let decryption_failed = HlsTrackEvidenceResolution::from(Err(HlsTsProbeError::DecryptionFailed)); - assert_eq!(decryption_failed, HlsTrackEvidenceResolution::DecryptionFailed); - assert_eq!(decryption_failed.reason_code(), "decryption-failed"); - let io = HlsTrackEvidenceResolution::from(Err(HlsTsProbeError::Io(std::io::Error::from( - std::io::ErrorKind::PermissionDenied, - )))); - assert_eq!(io, HlsTrackEvidenceResolution::Io(std::io::ErrorKind::PermissionDenied)); - assert_eq!(io.reason_code(), "io"); - } - - #[test] - fn ts_inspector_resynchronizes_only_within_named_budget() { - let budget = HlsTsProbeBudget { max_resync_bytes: 16, ..HlsTsProbeBudget::default() }; - let mut within_budget = vec![0xAA; 16]; - within_budget.extend_from_slice(&track_stream(0)); - assert!(matches!( - inspect_mpeg_ts(Cursor::new(within_budget), HlsTsProbeProtection::Clear, budget) - .expect("bounded resync probe completes"), - HlsTsProbeOutcome::Found(_) - )); - - let mut outside_budget = vec![0xAA; 17]; - outside_budget.extend_from_slice(&track_stream(0)); - assert_eq!( - inspect_mpeg_ts(Cursor::new(outside_budget), HlsTsProbeProtection::Clear, budget) - .expect("out-of-budget resync probe completes"), - HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidSynchronization) - ); - } - - #[test] - fn ts_inspector_types_framing_crc_and_pmt_errors() { - let invalid_sync = vec![0_u8; TS_PACKET_BYTES * 3]; - assert_eq!( - inspect_mpeg_ts(Cursor::new(invalid_sync), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe completes"), - HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidSynchronization) - ); - - let mut invalid_crc = track_stream(0); - invalid_crc[10] ^= 0x01; - assert_eq!( - inspect_mpeg_ts(Cursor::new(invalid_crc), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe completes"), - HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPsiCrc) - ); - - let mut invalid_pmt = pmt_section(1, 0x101, &[(0x1B, 0x101)], 0); - invalid_pmt[15] = 0xFF; - invalid_pmt[16] = 0xFF; - let crc_start = invalid_pmt.len().saturating_sub(4); - invalid_pmt.truncate(crc_start); - invalid_pmt = append_crc(invalid_pmt); - let mut stream = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); - stream.extend_from_slice(&packetize_section(0x100, &invalid_pmt, 0)); - stream.extend_from_slice(&null_packet()); - assert_eq!( - inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe completes"), - HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPmt) - ); - - let mut malformed_descriptors = pmt_section(1, 0x101, &[(0x1B, 0x101)], 4); - malformed_descriptors[13] = 5; - let crc_start = malformed_descriptors.len().saturating_sub(4); - malformed_descriptors.truncate(crc_start); - malformed_descriptors = append_crc(malformed_descriptors); - let mut stream = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); - stream.extend_from_slice(&packetize_section(0x100, &malformed_descriptors, 0)); - stream.extend_from_slice(&null_packet()); - assert_eq!( - inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe completes"), - HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPmt) - ); - } - - #[test] - fn ts_inspector_rejects_invalid_psi_header_before_exhausting_probe_budget() { - let mut invalid_pat = pat_section(&[(1, 0x100)]); - invalid_pat[1] &= 0x7F; - invalid_pat[1] |= 0x0F; - invalid_pat[2] = 0xFF; - let mut stream = packetize_section(0, &invalid_pat, 0); - let null_packet = null_packet(); - while stream.len() <= usize::try_from(HLS_TS_PROBE_MAX_BYTES).unwrap_or(usize::MAX) { - stream.extend_from_slice(&null_packet); - } - - assert_eq!( - inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe completes"), - HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPat) - ); - } - - #[test] - fn ts_inspector_types_scrambled_transport_as_unsupported_protection() { - let mut stream = track_stream(0); - stream[3] |= 0x80; - - assert_eq!( - inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe completes"), - HlsTsProbeOutcome::UnsupportedProtection(HlsTsProtectionReason::TransportScrambling) - ); - } - - #[test] - fn ts_inspector_applies_explicit_and_sequence_derived_hls_ivs() { - let sequence_iv = hls_aes128_cbc_iv(None, 0x0102_0304_0506_0708).expect("sequence IV"); - assert_eq!(sequence_iv, [0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8]); - let explicit_iv = hls_aes128_cbc_iv(Some("0x10203"), 0).expect("explicit IV"); - assert_eq!(explicit_iv, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3]); - assert!(matches!(hls_aes128_cbc_iv(Some("10203"), 0), Err(HlsTsProbeError::InvalidIv))); - } - - #[test] - fn ts_inspector_does_not_mutate_source_bytes() { - let bytes = track_stream(0); - let before = bytes.clone(); - let _ = inspect_mpeg_ts(Cursor::new(&bytes), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) - .expect("probe succeeds"); - assert_eq!(bytes, before); - } -} +mod tests; + +mod demux; +mod evidence; +mod inspector; +mod protection; +mod sections; +mod transport; +use demux::HlsPatSectionParser; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use evidence::{ + evaluate_mpeg_ts_splice_boundary, HlsTrackEvidenceResolution, HlsTsElementaryStreamBinding, HlsTsMalformedReason, + HlsTsMediaEvidence, HlsTsPidBoundaryEvidence, HlsTsProbeBudget, HlsTsProbeError, HlsTsProbeOutcome, + HlsTsProbeProtection, HlsTsProgramTopology, HlsTsProtectionReason, HlsTsSpliceBoundaryIncompatibility, + HlsTsSpliceEvidence, HlsTsSpliceIncompatibility, +}; +pub use inspector::{inspect_mpeg_ts, inspect_mpeg_ts_async, inspect_mpeg_ts_media_evidence_async}; +pub use protection::hls_aes128_cbc_iv; +#[cfg(test)] +use sections::synthetic_psi_packet; +use sections::{HlsPsiTableKind, HlsTsPacketFilter}; +use transport::HlsTsPesInspectionState; diff --git a/backend/mpegts/src/ts_inspector/demux.rs b/backend/mpegts/src/ts_inspector/demux.rs new file mode 100644 index 000000000..b03a003c4 --- /dev/null +++ b/backend/mpegts/src/ts_inspector/demux.rs @@ -0,0 +1,366 @@ +use super::{ + evidence::{is_audio_stream_type, is_video_stream_type}, + sections::HlsTsPacketFilter, + HlsPatPacketFilter, HlsPmtEvidence, HlsPmtPacketFilter, HlsPmtSectionEvidence, HlsPmtSectionParser, + HlsTsDemuxContext, HlsTsElementaryStreamBinding, HlsTsMalformedReason, HlsTsTrackSignature, +}; +use mpeg2ts_reader::{ + demultiplex, + demultiplex::{DemuxContext, FilterChangeset, FilterRequest}, + mpegts_crc, + packet::Pid, + psi, + psi::{ + pat::{PatSection, ProgramDescriptor}, + pmt::PmtSection, + CurrentNext, WholeSectionSyntaxPayloadParser, + }, +}; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, +}; + +impl HlsTsDemuxContext { + pub(super) fn new() -> Self { + Self { + changeset: FilterChangeset::default(), + pat_table_identity: None, + pat_last_section_number: None, + pat_sections: BTreeMap::new(), + programs_by_pid: BTreeMap::new(), + pmt_evidence: BTreeMap::new(), + malformed: None, + } + } + + pub(super) fn record_malformed(&mut self, reason: HlsTsMalformedReason) { + if self.malformed.is_none() { + self.malformed = Some(reason); + } + } + + pub(super) fn register_program(&mut self, program_number: u16, pid: Pid) { + let pid_value = u16::from(pid); + if self.programs_by_pid.get(&pid_value).is_some_and(|registered_program| *registered_program != program_number) + || self.programs_by_pid.iter().any(|(registered_pid, registered_program)| { + *registered_program == program_number && *registered_pid != pid_value + }) + { + self.record_malformed(HlsTsMalformedReason::InvalidPat); + return; + } + if self.programs_by_pid.insert(pid_value, program_number).is_none() { + self.pmt_evidence.insert(pid_value, HlsPmtEvidence { program_number, ..HlsPmtEvidence::default() }); + self.changeset.insert(pid, HlsTsPacketFilter::Pmt(HlsPmtPacketFilter::new(pid, program_number))); + } + } + + pub(super) fn restart_pat_version(&mut self, table_identity: (u16, u8)) { + let mut invalid_registered_pid = false; + for pid in self.programs_by_pid.keys().copied() { + match Pid::try_from(pid) { + Ok(pid) => self.changeset.remove(pid), + Err(()) => invalid_registered_pid = true, + } + } + self.pat_table_identity = Some(table_identity); + self.pat_last_section_number = None; + self.pat_sections.clear(); + self.programs_by_pid.clear(); + self.pmt_evidence.clear(); + if invalid_registered_pid { + self.record_malformed(HlsTsMalformedReason::InvalidPat); + } + } + + pub(super) fn reconcile_pat_programs(&mut self) -> Result<(), HlsTsMalformedReason> { + let mut programs_by_pid = BTreeMap::new(); + let mut pids_by_program = BTreeMap::new(); + for section in self.pat_sections.values() { + for (pid, program_number) in section { + if programs_by_pid.insert(*pid, *program_number).is_some_and(|current| current != *program_number) + || pids_by_program.insert(*program_number, *pid).is_some_and(|current| current != *pid) + { + return Err(HlsTsMalformedReason::InvalidPat); + } + } + } + for (pid, program_number) in programs_by_pid { + match self.programs_by_pid.get(&pid) { + Some(current) if *current == program_number => {} + Some(_) => return Err(HlsTsMalformedReason::InvalidPat), + None => { + let pid = Pid::try_from(pid).map_err(|()| HlsTsMalformedReason::InvalidPat)?; + self.register_program(program_number, pid); + if self.malformed.is_some() { + return Err(HlsTsMalformedReason::InvalidPat); + } + } + } + } + Ok(()) + } + + pub(super) fn is_psi_pid(&self, pid: u16) -> bool { + pid == u16::from(psi::pat::PAT_PID) || self.programs_by_pid.contains_key(&pid) + } + + pub(super) fn signature(&self) -> Option { + let pat_complete = self + .pat_last_section_number + .is_some_and(|last| (0..=last).all(|section| self.pat_sections.contains_key(§ion))); + if !pat_complete || self.programs_by_pid.is_empty() || self.pmt_evidence.values().any(|pmt| !pmt.complete()) { + return None; + } + let transport_stream_id = self.pat_table_identity?.0; + let programs = self + .pmt_evidence + .iter() + .map(|(pmt_pid, pmt)| pmt.topology(transport_stream_id, *pmt_pid)) + .collect::>>()?; + let stream_types = self + .pmt_evidence + .values() + .flat_map(|pmt| pmt.stream_types.iter().copied()) + .collect::>() + .into_iter() + .collect::>(); + if !stream_types + .iter() + .copied() + .any(|stream_type| is_audio_stream_type(stream_type) || is_video_stream_type(stream_type)) + { + return None; + } + Some(HlsTsTrackSignature { + program_count: u16::try_from(self.programs_by_pid.len()).unwrap_or(u16::MAX), + has_pcr: self.pmt_evidence.values().any(|pmt| pmt.has_pcr), + stream_types: Arc::from(stream_types), + programs: programs.into(), + }) + } +} + +impl DemuxContext for HlsTsDemuxContext { + type F = HlsTsPacketFilter; + + fn filter_changeset(&mut self) -> &mut FilterChangeset { &mut self.changeset } + + fn construct(&mut self, request: FilterRequest<'_, '_>) -> Self::F { + match request { + FilterRequest::ByPid(pid) if pid == psi::pat::PAT_PID => { + HlsTsPacketFilter::Pat(HlsPatPacketFilter::default()) + } + FilterRequest::Pmt { pid, program_number } => { + HlsTsPacketFilter::Pmt(HlsPmtPacketFilter::new(pid, program_number)) + } + FilterRequest::ByPid(_) | FilterRequest::ByStream { .. } | FilterRequest::Nit { .. } => { + HlsTsPacketFilter::Ignore(demultiplex::NullPacketFilter::default()) + } + } + } +} + +pub(super) struct HlsPatSectionParser; + +impl WholeSectionSyntaxPayloadParser for HlsPatSectionParser { + type Context = HlsTsDemuxContext; + + fn section<'a>( + &mut self, + ctx: &mut Self::Context, + header: &psi::SectionCommonHeader, + table_header: &psi::TableSyntaxHeader<'a>, + data: &'a [u8], + ) { + if header.table_id != 0x00 || data.len() < 12 { + ctx.record_malformed(HlsTsMalformedReason::InvalidPat); + return; + } + if mpegts_crc::sum32(data) != 0 { + ctx.record_malformed(HlsTsMalformedReason::InvalidPsiCrc); + return; + } + if table_header.current_next_indicator() == CurrentNext::Next { + return; + } + if table_header.section_number() > table_header.last_section_number() { + ctx.record_malformed(HlsTsMalformedReason::InvalidPat); + return; + } + let table_identity = (table_header.id(), table_header.version()); + match ctx.pat_table_identity { + Some((table_id, _)) if table_id != table_header.id() => { + ctx.record_malformed(HlsTsMalformedReason::InvalidPat); + return; + } + Some(current) if current != table_identity => ctx.restart_pat_version(table_identity), + Some(_) => {} + None => ctx.pat_table_identity = Some(table_identity), + } + if ctx.pat_last_section_number.is_some_and(|last| last != table_header.last_section_number()) { + ctx.record_malformed(HlsTsMalformedReason::InvalidPat); + return; + } + let payload_start = psi::SectionCommonHeader::SIZE + psi::TableSyntaxHeader::SIZE; + let payload_end = data.len().saturating_sub(4); + let Some(payload) = data.get(payload_start..payload_end) else { + ctx.record_malformed(HlsTsMalformedReason::InvalidPat); + return; + }; + if !payload.len().is_multiple_of(4) { + ctx.record_malformed(HlsTsMalformedReason::InvalidPat); + return; + } + let mut programs = BTreeMap::new(); + let mut pids_by_program = BTreeMap::new(); + for descriptor in PatSection::new(payload).programs() { + if let ProgramDescriptor::Program { program_number, pid } = descriptor { + let pid = u16::from(pid); + if programs.insert(pid, program_number).is_some_and(|current| current != program_number) + || pids_by_program.insert(program_number, pid).is_some_and(|current| current != pid) + { + ctx.record_malformed(HlsTsMalformedReason::InvalidPat); + return; + } + } + } + if ctx.pat_sections.get(&table_header.section_number()).is_some_and(|current| current != &programs) { + ctx.record_malformed(HlsTsMalformedReason::InvalidPat); + return; + } + ctx.pat_last_section_number = Some(table_header.last_section_number()); + ctx.pat_sections.insert(table_header.section_number(), programs); + if let Err(reason) = ctx.reconcile_pat_programs() { + ctx.record_malformed(reason); + } + } +} + +impl WholeSectionSyntaxPayloadParser for HlsPmtSectionParser { + type Context = HlsTsDemuxContext; + + fn section<'a>( + &mut self, + ctx: &mut Self::Context, + header: &psi::SectionCommonHeader, + table_header: &psi::TableSyntaxHeader<'a>, + data: &'a [u8], + ) { + if header.table_id != 0x02 || data.len() < 16 || table_header.id() != self.program_number { + ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); + return; + } + if mpegts_crc::sum32(data) != 0 { + ctx.record_malformed(HlsTsMalformedReason::InvalidPsiCrc); + return; + } + if table_header.current_next_indicator() == CurrentNext::Next { + return; + } + if table_header.section_number() > table_header.last_section_number() { + ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); + return; + } + let payload_start = psi::SectionCommonHeader::SIZE + psi::TableSyntaxHeader::SIZE; + let payload_end = data.len().saturating_sub(4); + let Some(payload) = data.get(payload_start..payload_end) else { + ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); + return; + }; + if !pmt_stream_loop_is_well_formed(payload) { + ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); + return; + } + let Ok(section) = PmtSection::from_bytes(payload) else { + ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); + return; + }; + let pid = u16::from(self.pid); + let Some(evidence) = ctx.pmt_evidence.get_mut(&pid) else { + ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); + return; + }; + if evidence.program_number != self.program_number { + ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); + return; + } + match evidence.table_version { + Some(version) if version != table_header.version() => { + evidence.restart_version(table_header.version(), table_header.last_section_number()); + } + Some(_) => { + if evidence.last_section_number != Some(table_header.last_section_number()) { + ctx.record_malformed(HlsTsMalformedReason::InvalidPmt); + return; + } + } + None => evidence.restart_version(table_header.version(), table_header.last_section_number()), + } + let section_evidence = HlsPmtSectionEvidence { + pcr_pid: u16::from(section.pcr_pid()), + streams: section + .streams() + .map(|stream| HlsTsElementaryStreamBinding { + stream_type: stream.stream_type().0, + elementary_pid: u16::from(stream.elementary_pid()), + }) + .collect(), + }; + if let Err(reason) = evidence.record_section(table_header.section_number(), section_evidence) { + ctx.record_malformed(reason); + } + } +} + +/// `mpeg2ts-reader` intentionally stops its stream iterator at a truncated +/// descriptor. Validate both descriptor loops and ES framing so truncation is +/// a typed malformed outcome instead of a partial, falsely compatible signature. +fn pmt_stream_loop_is_well_formed(payload: &[u8]) -> bool { + let Some(program_info_length_bytes) = payload.get(2..4) else { + return false; + }; + let program_info_length = + (usize::from(program_info_length_bytes[0] & 0x0F) << 8) | usize::from(program_info_length_bytes[1]); + let Some(mut offset) = 4usize.checked_add(program_info_length).filter(|offset| *offset <= payload.len()) else { + return false; + }; + if !descriptor_loop_is_well_formed(&payload[4..offset]) { + return false; + } + while offset < payload.len() { + let Some(header) = payload.get(offset..offset.saturating_add(5)) else { + return false; + }; + let es_info_length = (usize::from(header[3] & 0x0F) << 8) | usize::from(header[4]); + let Some(next) = offset.checked_add(5).and_then(|value| value.checked_add(es_info_length)) else { + return false; + }; + if next > payload.len() { + return false; + } + if !descriptor_loop_is_well_formed(&payload[offset.saturating_add(5)..next]) { + return false; + } + offset = next; + } + true +} + +fn descriptor_loop_is_well_formed(mut descriptors: &[u8]) -> bool { + while !descriptors.is_empty() { + let Some(header) = descriptors.get(..2) else { + return false; + }; + let descriptor_length = usize::from(header[1]); + let Some(next) = 2usize.checked_add(descriptor_length) else { + return false; + }; + let Some(remaining) = descriptors.get(next..) else { + return false; + }; + descriptors = remaining; + } + true +} diff --git a/backend/mpegts/src/ts_inspector/evidence.rs b/backend/mpegts/src/ts_inspector/evidence.rs new file mode 100644 index 000000000..1db9b4c61 --- /dev/null +++ b/backend/mpegts/src/ts_inspector/evidence.rs @@ -0,0 +1,312 @@ +use super::{ + HlsTsCompatibleSpliceEvidence, HlsTsTrackSignature, AES_128_BLOCK_BYTES, HLS_TS_PROBE_MAX_BYTES, + HLS_TS_PROBE_MAX_PACKETS, HLS_TS_PROBE_MAX_RESYNC_BYTES, HLS_TS_PROBE_READ_CHUNK_BYTES, +}; +use crate::transport_stream_buffer::HlsTsTimestampProfile; +use std::sync::Arc; + +/// Hard limits for one read-only MPEG-TS compatibility probe. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HlsTsProbeBudget { + pub max_bytes: u64, + pub max_packets: u64, + pub read_chunk_bytes: usize, + pub max_resync_bytes: usize, +} + +impl Default for HlsTsProbeBudget { + fn default() -> Self { + Self { + max_bytes: HLS_TS_PROBE_MAX_BYTES, + max_packets: HLS_TS_PROBE_MAX_PACKETS, + read_chunk_bytes: HLS_TS_PROBE_READ_CHUNK_BYTES, + max_resync_bytes: HLS_TS_PROBE_MAX_RESYNC_BYTES, + } + } +} + +impl HlsTsProbeBudget { + pub(super) fn bounded_read_chunk_bytes(self) -> usize { + self.read_chunk_bytes.clamp(1, HLS_TS_PROBE_READ_CHUNK_BYTES) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HlsTsElementaryStreamBinding { + pub stream_type: u8, + pub elementary_pid: u16, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsTsProgramTopology { + pub transport_stream_id: u16, + pub program_number: u16, + pub pmt_pid: u16, + pub pcr_pid: u16, + pub streams: Arc<[HlsTsElementaryStreamBinding]>, +} + +impl HlsTsTrackSignature { + #[cfg(any(test, feature = "test-support"))] + pub fn from_stream_types(stream_types: impl Into>) -> Self { + Self { program_count: 1, has_pcr: true, stream_types: stream_types.into(), programs: Arc::from([]) } + } +} + +pub(super) fn is_audio_stream_type(stream_type: u8) -> bool { + matches!(stream_type, 0x03 | 0x04 | 0x0F | 0x11 | 0x81 | 0x87) +} + +pub(super) fn is_video_stream_type(stream_type: u8) -> bool { + matches!(stream_type, 0x01 | 0x02 | 0x10 | 0x1B | 0x24 | 0x42) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsTsMalformedReason { + InvalidSynchronization, + InvalidPacketHeader, + TransportError, + InvalidPsiPointer, + InvalidPat, + InvalidPmt, + InvalidPsiCrc, + IncompletePacket, + IncompleteProgramMetadata, +} + +impl HlsTsMalformedReason { + pub(super) const fn reason_code(self) -> &'static str { + match self { + Self::InvalidSynchronization => "invalid-synchronization", + Self::InvalidPacketHeader => "invalid-packet-header", + Self::TransportError => "transport-error", + Self::InvalidPsiPointer => "invalid-psi-pointer", + Self::InvalidPat => "invalid-pat", + Self::InvalidPmt => "invalid-pmt", + Self::InvalidPsiCrc => "invalid-psi-crc", + Self::IncompletePacket => "incomplete-packet", + Self::IncompleteProgramMetadata => "incomplete-program-metadata", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsTsProtectionReason { + TransportScrambling, + UnsupportedEncryption, +} + +impl HlsTsProtectionReason { + pub(super) const fn reason_code(self) -> &'static str { + match self { + Self::TransportScrambling => "transport-scrambling", + Self::UnsupportedEncryption => "unsupported-encryption", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HlsTsProbeOutcome { + Found(HlsTsTrackSignature), + ProbeBudgetExhausted { bytes_examined: u64, packets_examined: u64 }, + Malformed(HlsTsMalformedReason), + UnsupportedProtection(HlsTsProtectionReason), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HlsTsMediaEvidence { + pub track_outcome: HlsTsProbeOutcome, + pub timestamp_profile: Option, + pub splice_evidence: HlsTsSpliceEvidence, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HlsTsPidBoundaryEvidence { + pub pid: u16, + pub first_packet_index: u64, + pub first_continuity_counter: u8, + pub first_has_payload: bool, + pub first_discontinuity: bool, + pub last_continuity_counter: u8, +} + +impl HlsTsCompatibleSpliceEvidence { + pub(super) fn pid_boundary(&self, pid: u16) -> Option<&HlsTsPidBoundaryEvidence> { + self.pid_boundaries + .binary_search_by_key(&pid, |boundary| boundary.pid) + .ok() + .and_then(|index| self.pid_boundaries.get(index)) + } + + #[cfg(any(test, feature = "test-support"))] + pub fn for_test(topology: HlsTsTrackSignature) -> Self { Self { topology, pid_boundaries: Arc::from([]) } } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsTsSpliceIncompatibility { + InvalidPacket { packet_index: u64 }, + TransportError { pid: u16, packet_index: u64 }, + ContinuityFailure { pid: u16, packet_index: u64, expected: u8, actual: u8 }, + IncompletePes { pid: u16, packet_index: u64, declared_bytes: Option, observed_bytes: u64 }, + InvalidPes { pid: u16, packet_index: u64 }, + InspectionBudgetExhausted, + TopologyUnavailable, +} + +impl HlsTsSpliceIncompatibility { + pub const fn result_code(self) -> &'static str { + match self { + Self::InvalidPacket { .. } | Self::TransportError { .. } | Self::ContinuityFailure { .. } => { + "continuity-failure" + } + Self::IncompletePes { .. } | Self::InvalidPes { .. } => "incomplete-pes", + Self::InspectionBudgetExhausted | Self::TopologyUnavailable => "topology-mismatch", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HlsTsSpliceEvidence { + Compatible(HlsTsCompatibleSpliceEvidence), + Incompatible(HlsTsSpliceIncompatibility), +} + +impl HlsTsSpliceEvidence { + #[cfg(any(test, feature = "test-support"))] + pub fn compatible_for_test(topology: HlsTsTrackSignature) -> Self { + Self::Compatible(HlsTsCompatibleSpliceEvidence::for_test(topology)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HlsTsSpliceBoundaryIncompatibility { + Media(HlsTsSpliceIncompatibility), + TopologyMismatch, +} + +pub fn evaluate_mpeg_ts_splice_boundary( + base: &HlsTsSpliceEvidence, + terminal: &HlsTsSpliceEvidence, +) -> Result<(), HlsTsSpliceBoundaryIncompatibility> { + let base = match base { + HlsTsSpliceEvidence::Compatible(evidence) => evidence, + HlsTsSpliceEvidence::Incompatible(reason) => { + return Err(HlsTsSpliceBoundaryIncompatibility::Media(*reason)); + } + }; + let terminal = match terminal { + HlsTsSpliceEvidence::Compatible(evidence) => evidence, + HlsTsSpliceEvidence::Incompatible(reason) => { + return Err(HlsTsSpliceBoundaryIncompatibility::Media(*reason)); + } + }; + if base.topology != terminal.topology { + return Err(HlsTsSpliceBoundaryIncompatibility::TopologyMismatch); + } + for boundary in terminal.pid_boundaries.iter() { + if boundary.first_discontinuity { + continue; + } + let Some(base_boundary) = base.pid_boundary(boundary.pid) else { + continue; + }; + let expected = if boundary.first_has_payload { + base_boundary.last_continuity_counter.wrapping_add(1) & 0x0F + } else { + base_boundary.last_continuity_counter + }; + if boundary.first_continuity_counter != expected { + return Err(HlsTsSpliceBoundaryIncompatibility::Media(HlsTsSpliceIncompatibility::ContinuityFailure { + pid: boundary.pid, + packet_index: boundary.first_packet_index, + expected, + actual: boundary.first_continuity_counter, + })); + } + } + Ok(()) +} + +#[derive(Debug, thiserror::Error)] +pub enum HlsTsProbeError { + #[error("MPEG-TS probe I/O failed")] + Io(#[source] std::io::Error), + #[error("MPEG-TS probe key is unavailable")] + KeyUnavailable, + #[error("MPEG-TS probe IV is invalid")] + InvalidIv, + #[error("MPEG-TS probe decryption failed")] + DecryptionFailed, +} + +/// Policy-facing track evidence with stable diagnostics and no parser-crate types. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HlsTrackEvidenceResolution { + Found(HlsTsTrackSignature), + InsufficientEvidence { bytes_examined: u64, packets_examined: u64 }, + IncompleteEvidence, + Malformed(HlsTsMalformedReason), + UnsupportedProtection(HlsTsProtectionReason), + KeyUnavailable, + InvalidIv, + DecryptionFailed, + Io(std::io::ErrorKind), +} + +impl HlsTrackEvidenceResolution { + pub fn signature(&self) -> Option<&HlsTsTrackSignature> { + match self { + Self::Found(signature) => Some(signature), + Self::InsufficientEvidence { .. } + | Self::IncompleteEvidence + | Self::Malformed(_) + | Self::UnsupportedProtection(_) + | Self::KeyUnavailable + | Self::InvalidIv + | Self::DecryptionFailed + | Self::Io(_) => None, + } + } + + pub const fn reason_code(&self) -> &'static str { + match self { + Self::Found(_) => "found", + Self::InsufficientEvidence { .. } => "insufficient-evidence", + Self::IncompleteEvidence => "incomplete-evidence", + Self::Malformed(reason) => reason.reason_code(), + Self::UnsupportedProtection(reason) => reason.reason_code(), + Self::KeyUnavailable => "key-unavailable", + Self::InvalidIv => "invalid-iv", + Self::DecryptionFailed => "decryption-failed", + // ErrorKind remains available on the typed value; logs deliberately use + // one stable non-sensitive code rather than exposing platform wording. + Self::Io(_) => "io", + } + } +} + +impl From> for HlsTrackEvidenceResolution { + fn from(result: Result) -> Self { + match result { + Ok(HlsTsProbeOutcome::Found(signature)) => Self::Found(signature), + Ok(HlsTsProbeOutcome::ProbeBudgetExhausted { bytes_examined, packets_examined }) => { + Self::InsufficientEvidence { bytes_examined, packets_examined } + } + Ok(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::IncompleteProgramMetadata)) => { + Self::IncompleteEvidence + } + Ok(HlsTsProbeOutcome::Malformed(reason)) => Self::Malformed(reason), + Ok(HlsTsProbeOutcome::UnsupportedProtection(reason)) => Self::UnsupportedProtection(reason), + Err(HlsTsProbeError::Io(error)) => Self::Io(error.kind()), + Err(HlsTsProbeError::KeyUnavailable) => Self::KeyUnavailable, + Err(HlsTsProbeError::InvalidIv) => Self::InvalidIv, + Err(HlsTsProbeError::DecryptionFailed) => Self::DecryptionFailed, + } + } +} + +#[derive(Clone, Copy)] +pub enum HlsTsProbeProtection<'a> { + Clear, + Aes128Cbc { key: &'a [u8], iv: [u8; AES_128_BLOCK_BYTES] }, +} diff --git a/backend/mpegts/src/ts_inspector/inspector.rs b/backend/mpegts/src/ts_inspector/inspector.rs new file mode 100644 index 000000000..4c12ced29 --- /dev/null +++ b/backend/mpegts/src/ts_inspector/inspector.rs @@ -0,0 +1,395 @@ +use super::{ + protection::HlsTsSourceDecoder, HlsTsDemuxContext, HlsTsInspector, HlsTsMalformedReason, HlsTsMediaEvidence, + HlsTsMediaStreamInspector, HlsTsProbeBudget, HlsTsProbeError, HlsTsProbeOutcome, HlsTsProbeProtection, + HlsTsProtectionReason, HlsTsSpliceEvidence, HlsTsSpliceIncompatibility, HlsTsTrackSignature, + HlsTsTransportStreamInspector, AES_128_BLOCK_BYTES, TS_ALIGNMENT_CONFIRMATION_PACKETS, TS_PACKET_BYTES, +}; +use crate::transport_stream_buffer::{HlsTsTimestampProfile, HlsTsTimestampProfileScanner}; +use mpeg2ts_reader::{demultiplex, packet::Packet}; +use std::io::Read; +use tokio::io::{AsyncRead, AsyncReadExt}; +use zeroize::Zeroizing; + +#[derive(Clone, Copy)] +pub(super) enum HlsTsPlaintextRemainder { + ExactPackets, + Aes128Pkcs7, +} + +impl HlsTsMediaStreamInspector { + pub(super) fn new(budget: HlsTsProbeBudget, expected_duration_ticks_90khz: u64) -> Self { + Self { + budget, + timestamp_scanner: HlsTsTimestampProfileScanner::new(expected_duration_ticks_90khz), + transport_scanner: HlsTsTransportStreamInspector::new(), + pending: Vec::with_capacity(budget.bounded_read_chunk_bytes().saturating_add(TS_PACKET_BYTES)), + aligned: false, + packets_examined: 0, + invalid: false, + } + } + + pub(super) fn feed_plaintext(&mut self, bytes: &[u8]) { + if self.invalid { + return; + } + self.pending.extend_from_slice(bytes); + if !self.aligned { + let confirmation_bytes = TS_PACKET_BYTES.saturating_mul(TS_ALIGNMENT_CONFIRMATION_PACKETS); + if self.pending.len() < confirmation_bytes { + return; + } + let available_search = self.pending.len().saturating_sub(confirmation_bytes); + let search_end = available_search.min(self.budget.max_resync_bytes); + let alignment = (0..=search_end).find(|offset| { + (0..TS_ALIGNMENT_CONFIRMATION_PACKETS).all(|index| { + self.pending[offset.saturating_add(index.saturating_mul(TS_PACKET_BYTES))] == Packet::SYNC_BYTE + }) + }); + let Some(alignment) = alignment else { + if available_search >= self.budget.max_resync_bytes { + self.invalid = true; + } + return; + }; + self.pending.drain(..alignment); + self.aligned = true; + } + + let mut consumed = 0usize; + while self.pending.len().saturating_sub(consumed) >= TS_PACKET_BYTES { + if self.packets_examined >= self.budget.max_packets { + self.invalid = true; + break; + } + let packet_end = consumed.saturating_add(TS_PACKET_BYTES); + let packet = &self.pending[consumed..packet_end]; + self.timestamp_scanner.push_aligned_packet(packet); + self.transport_scanner.push_packet(packet, self.packets_examined); + self.packets_examined = self.packets_examined.saturating_add(1); + consumed = packet_end; + } + self.pending.drain(..consumed); + } + + pub(super) fn finish( + self, + remainder: HlsTsPlaintextRemainder, + topology: Option, + ) -> (Option, HlsTsSpliceEvidence) { + let valid_remainder = match remainder { + HlsTsPlaintextRemainder::ExactPackets => self.pending.is_empty(), + HlsTsPlaintextRemainder::Aes128Pkcs7 => { + let Some(&padding) = self.pending.last() else { + return ( + None, + HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::InvalidPacket { + packet_index: self.packets_examined, + }), + ); + }; + let padding = usize::from(padding); + (1..=AES_128_BLOCK_BYTES).contains(&padding) + && self.pending.len() == padding + && self.pending.iter().all(|byte| usize::from(*byte) == padding) + } + }; + if self.invalid || !self.aligned || !valid_remainder { + return ( + None, + HlsTsSpliceEvidence::Incompatible(if self.packets_examined >= self.budget.max_packets { + HlsTsSpliceIncompatibility::InspectionBudgetExhausted + } else { + HlsTsSpliceIncompatibility::InvalidPacket { packet_index: self.packets_examined } + }), + ); + } + (self.timestamp_scanner.finish(), self.transport_scanner.finish(topology)) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum HlsTsInspectionMode { + Prefix, + Complete, +} + +impl HlsTsInspector { + pub(super) fn new(budget: HlsTsProbeBudget) -> Self { + let mut context = HlsTsDemuxContext::new(); + let demux = demultiplex::Demultiplex::new(&mut context); + Self { + budget, + demux, + context, + pending: Vec::with_capacity(budget.bounded_read_chunk_bytes().saturating_add(TS_PACKET_BYTES)), + aligned: false, + source_bytes_examined: 0, + packets_examined: 0, + } + } + + pub(super) fn record_source_bytes(&mut self, bytes: usize) { + self.source_bytes_examined = + self.source_bytes_examined.saturating_add(u64::try_from(bytes).unwrap_or(u64::MAX)); + } + + pub(super) fn budget_exhausted(&self) -> HlsTsProbeOutcome { + HlsTsProbeOutcome::ProbeBudgetExhausted { + bytes_examined: self.source_bytes_examined, + packets_examined: self.packets_examined, + } + } + + pub(super) fn feed_plaintext(&mut self, bytes: &[u8]) -> Option { + self.feed_plaintext_with_mode(bytes, HlsTsInspectionMode::Prefix) + } + + pub(super) fn feed_plaintext_complete(&mut self, bytes: &[u8]) -> Option { + self.feed_plaintext_with_mode(bytes, HlsTsInspectionMode::Complete) + } + + pub(super) fn feed_plaintext_with_mode( + &mut self, + bytes: &[u8], + mode: HlsTsInspectionMode, + ) -> Option { + self.pending.extend_from_slice(bytes); + if !self.aligned { + let confirmation_bytes = TS_PACKET_BYTES.saturating_mul(TS_ALIGNMENT_CONFIRMATION_PACKETS); + if self.pending.len() < confirmation_bytes { + return None; + } + let available_search = self.pending.len().saturating_sub(confirmation_bytes); + let search_end = available_search.min(self.budget.max_resync_bytes); + let alignment = (0..=search_end).find(|offset| { + (0..TS_ALIGNMENT_CONFIRMATION_PACKETS).all(|index| { + self.pending[offset.saturating_add(index.saturating_mul(TS_PACKET_BYTES))] == Packet::SYNC_BYTE + }) + }); + let Some(alignment) = alignment else { + if available_search >= self.budget.max_resync_bytes { + return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidSynchronization)); + } + return None; + }; + self.pending.drain(..alignment); + self.aligned = true; + } + + let mut consumed = 0usize; + while self.pending.len().saturating_sub(consumed) >= TS_PACKET_BYTES { + if self.packets_examined >= self.budget.max_packets { + self.pending.drain(..consumed); + return Some(self.budget_exhausted()); + } + let mut packet_bytes = [0_u8; TS_PACKET_BYTES]; + packet_bytes.copy_from_slice(&self.pending[consumed..consumed.saturating_add(TS_PACKET_BYTES)]); + consumed = consumed.saturating_add(TS_PACKET_BYTES); + self.packets_examined = self.packets_examined.saturating_add(1); + if let Some(outcome) = self.validate_packet(&packet_bytes) { + self.pending.drain(..consumed); + return Some(outcome); + } + self.demux.push(&mut self.context, &packet_bytes); + if let Some(reason) = self.context.malformed { + self.pending.drain(..consumed); + return Some(HlsTsProbeOutcome::Malformed(reason)); + } + if mode == HlsTsInspectionMode::Prefix { + if let Some(signature) = self.context.signature() { + self.pending.drain(..consumed); + return Some(HlsTsProbeOutcome::Found(signature)); + } + } + } + self.pending.drain(..consumed); + None + } + + pub(super) fn validate_packet(&self, bytes: &[u8; TS_PACKET_BYTES]) -> Option { + if bytes[0] != Packet::SYNC_BYTE { + return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidSynchronization)); + } + if bytes[1] & 0x80 != 0 { + return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::TransportError)); + } + if bytes[3] & 0xC0 != 0 { + return Some(HlsTsProbeOutcome::UnsupportedProtection(HlsTsProtectionReason::TransportScrambling)); + } + let adaptation_control = (bytes[3] >> 4) & 0b11; + let payload_offset = match adaptation_control { + 0b01 => 4, + 0b10 if bytes[4] == 183 => TS_PACKET_BYTES, + 0b11 if bytes[4] <= 182 => 5usize.saturating_add(usize::from(bytes[4])), + 0b00 | 0b10 | 0b11 => { + return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPacketHeader)); + } + _ => return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPacketHeader)), + }; + let pid = (u16::from(bytes[1] & 0x1F) << 8) | u16::from(bytes[2]); + if self.context.is_psi_pid(pid) && bytes[1] & 0x40 != 0 { + let Some(payload) = bytes.get(payload_offset..) else { + return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPsiPointer)); + }; + let Some(pointer) = payload.first().map(|value| usize::from(*value)) else { + return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPsiPointer)); + }; + let section_start = pointer.saturating_add(1); + if section_start >= payload.len() { + return Some(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPsiPointer)); + } + } + None + } + + pub(super) fn finish(self) -> HlsTsProbeOutcome { + if let Some(reason) = self.context.malformed { + return HlsTsProbeOutcome::Malformed(reason); + } + if let Some(signature) = self.context.signature() { + return HlsTsProbeOutcome::Found(signature); + } + if !self.aligned { + return HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidSynchronization); + } + if !self.pending.is_empty() { + return HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::IncompletePacket); + } + HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::IncompleteProgramMetadata) + } + + pub(super) fn finish_complete(mut self, remainder: HlsTsPlaintextRemainder) -> HlsTsProbeOutcome { + if matches!(remainder, HlsTsPlaintextRemainder::Aes128Pkcs7) { + if let Some(&padding) = self.pending.last() { + let padding = usize::from(padding); + if (1..=AES_128_BLOCK_BYTES).contains(&padding) + && self.pending.len() == padding + && self.pending.iter().all(|byte| usize::from(*byte) == padding) + { + self.pending.clear(); + } + } + } + self.finish() + } +} + +/// Inspects a blocking reader without retaining or mutating source media bytes. +pub fn inspect_mpeg_ts( + mut reader: R, + protection: HlsTsProbeProtection<'_>, + budget: HlsTsProbeBudget, +) -> Result { + let mut decoder = HlsTsSourceDecoder::new(protection)?; + let chunk_bytes = budget.bounded_read_chunk_bytes(); + let mut buffer = Zeroizing::new(vec![0_u8; chunk_bytes]); + let mut inspector = HlsTsInspector::new(budget); + loop { + let remaining = budget.max_bytes.saturating_sub(inspector.source_bytes_examined); + if remaining == 0 { + return Ok(inspector.budget_exhausted()); + } + let read_limit = usize::try_from(remaining).unwrap_or(usize::MAX).min(buffer.len()); + let read = match reader.read(&mut buffer[..read_limit]) { + Ok(0) => break, + Ok(read) => read, + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) => return Err(HlsTsProbeError::Io(error)), + }; + inspector.record_source_bytes(read); + if let Some(outcome) = decoder.with_plaintext(&buffer[..read], |plaintext| inspector.feed_plaintext(plaintext)) + { + return Ok(outcome); + } + } + decoder.finish()?; + Ok(inspector.finish()) +} + +/// Async cache-file entry point backed by the same parser and CBC state machine. +pub async fn inspect_mpeg_ts_async( + mut reader: R, + protection: HlsTsProbeProtection<'_>, + budget: HlsTsProbeBudget, +) -> Result { + let mut decoder = HlsTsSourceDecoder::new(protection)?; + let chunk_bytes = budget.bounded_read_chunk_bytes(); + let mut buffer = Zeroizing::new(vec![0_u8; chunk_bytes]); + let mut inspector = HlsTsInspector::new(budget); + loop { + let remaining = budget.max_bytes.saturating_sub(inspector.source_bytes_examined); + if remaining == 0 { + return Ok(inspector.budget_exhausted()); + } + let read_limit = usize::try_from(remaining).unwrap_or(usize::MAX).min(buffer.len()); + let read = reader.read(&mut buffer[..read_limit]).await.map_err(HlsTsProbeError::Io)?; + if read == 0 { + break; + } + inspector.record_source_bytes(read); + if let Some(outcome) = decoder.with_plaintext(&buffer[..read], |plaintext| inspector.feed_plaintext(plaintext)) + { + return Ok(outcome); + } + } + decoder.finish()?; + Ok(inspector.finish()) +} + +/// Collects track compatibility and complete clock evidence from one exact cache reader. +/// +/// PAT/PMT discovery may settle from the prefix, while timestamp collection continues to +/// EOF in bounded chunks. Valid AES-CBC padding is excluded from the aligned TS stream. +pub async fn inspect_mpeg_ts_media_evidence_async( + mut reader: R, + protection: HlsTsProbeProtection<'_>, + budget: HlsTsProbeBudget, + expected_duration_ticks_90khz: u64, +) -> Result { + let mut decoder = HlsTsSourceDecoder::new(protection)?; + let plaintext_remainder = decoder.plaintext_remainder(); + let chunk_bytes = budget.bounded_read_chunk_bytes(); + let mut buffer = Zeroizing::new(vec![0_u8; chunk_bytes]); + let mut track_inspector = HlsTsInspector::new(budget); + let mut media_inspector = HlsTsMediaStreamInspector::new(budget, expected_duration_ticks_90khz); + let mut track_outcome = None; + let mut source_bytes_examined = 0_u64; + loop { + let remaining = budget.max_bytes.saturating_sub(source_bytes_examined); + if remaining == 0 { + return Ok(HlsTsMediaEvidence { + track_outcome: track_outcome.unwrap_or_else(|| track_inspector.budget_exhausted()), + timestamp_profile: None, + splice_evidence: HlsTsSpliceEvidence::Incompatible( + HlsTsSpliceIncompatibility::InspectionBudgetExhausted, + ), + }); + } + let read_limit = usize::try_from(remaining).unwrap_or(usize::MAX).min(buffer.len()); + let read = reader.read(&mut buffer[..read_limit]).await.map_err(HlsTsProbeError::Io)?; + if read == 0 { + break; + } + source_bytes_examined = source_bytes_examined.saturating_add(u64::try_from(read).unwrap_or(u64::MAX)); + if track_outcome.is_none() { + track_inspector.record_source_bytes(read); + } + decoder.with_plaintext(&buffer[..read], |plaintext| { + media_inspector.feed_plaintext(plaintext); + if track_outcome.is_none() { + track_outcome = track_inspector.feed_plaintext_complete(plaintext); + } + }); + } + decoder.finish()?; + let track_outcome = track_outcome.unwrap_or_else(|| track_inspector.finish_complete(plaintext_remainder)); + let topology = match &track_outcome { + HlsTsProbeOutcome::Found(signature) => Some(signature.clone()), + HlsTsProbeOutcome::ProbeBudgetExhausted { .. } + | HlsTsProbeOutcome::Malformed(_) + | HlsTsProbeOutcome::UnsupportedProtection(_) => None, + }; + let (timestamp_profile, splice_evidence) = media_inspector.finish(plaintext_remainder, topology); + Ok(HlsTsMediaEvidence { track_outcome, timestamp_profile, splice_evidence }) +} diff --git a/backend/mpegts/src/ts_inspector/protection.rs b/backend/mpegts/src/ts_inspector/protection.rs new file mode 100644 index 000000000..a42414107 --- /dev/null +++ b/backend/mpegts/src/ts_inspector/protection.rs @@ -0,0 +1,142 @@ +use super::{ + inspector::HlsTsPlaintextRemainder, HlsAes128CbcPrefixDecoder, HlsTsProbeError, HlsTsProbeProtection, + AES_128_BLOCK_BYTES, +}; +use aes::{ + cipher::{Block, BlockDecrypt, KeyInit}, + Aes128, +}; +use zeroize::{Zeroize, Zeroizing}; + +/// Implements the existing HLS AES-128 explicit-IV and sequence-derived-IV rules. +pub fn hls_aes128_cbc_iv( + explicit_iv: Option<&str>, + media_sequence: u64, +) -> Result<[u8; AES_128_BLOCK_BYTES], HlsTsProbeError> { + let mut iv = [0_u8; AES_128_BLOCK_BYTES]; + let Some(explicit_iv) = explicit_iv else { + iv[AES_128_BLOCK_BYTES - std::mem::size_of::()..].copy_from_slice(&media_sequence.to_be_bytes()); + return Ok(iv); + }; + let hex = explicit_iv.strip_prefix("0x").ok_or(HlsTsProbeError::InvalidIv)?; + if hex.is_empty() || hex.len() > AES_128_BLOCK_BYTES * 2 || !hex.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(HlsTsProbeError::InvalidIv); + } + let mut output_index = AES_128_BLOCK_BYTES; + let mut input_index = hex.len(); + while input_index > 0 { + let low = hex_digit(hex.as_bytes()[input_index - 1]).ok_or(HlsTsProbeError::InvalidIv)?; + input_index = input_index.saturating_sub(1); + let high = if input_index > 0 { + let value = hex_digit(hex.as_bytes()[input_index - 1]).ok_or(HlsTsProbeError::InvalidIv)?; + input_index = input_index.saturating_sub(1); + value + } else { + 0 + }; + output_index = output_index.checked_sub(1).ok_or(HlsTsProbeError::InvalidIv)?; + iv[output_index] = (high << 4) | low; + } + Ok(iv) +} + +fn hex_digit(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +impl HlsAes128CbcPrefixDecoder { + pub(super) fn new(key: &[u8], iv: [u8; AES_128_BLOCK_BYTES]) -> Result { + if key.len() != AES_128_BLOCK_BYTES { + return Err(HlsTsProbeError::KeyUnavailable); + } + let cipher = Aes128::new_from_slice(key).map_err(|_| HlsTsProbeError::KeyUnavailable)?; + Ok(Self { cipher, previous_ciphertext: iv, carry: Zeroizing::new(Vec::with_capacity(AES_128_BLOCK_BYTES)) }) + } + + pub(super) fn push(&mut self, ciphertext: &[u8]) -> Zeroizing> { + let mut plaintext = Zeroizing::new(Vec::with_capacity( + self.carry.len().saturating_add(ciphertext.len()) / AES_128_BLOCK_BYTES * AES_128_BLOCK_BYTES, + )); + let mut input = ciphertext; + if !self.carry.is_empty() { + let required = AES_128_BLOCK_BYTES.saturating_sub(self.carry.len()); + let copied = required.min(input.len()); + self.carry.extend_from_slice(&input[..copied]); + input = &input[copied..]; + if self.carry.len() == AES_128_BLOCK_BYTES { + let mut block = [0_u8; AES_128_BLOCK_BYTES]; + block.copy_from_slice(&self.carry); + self.decrypt_block(block, &mut plaintext); + self.carry.clear(); + } + } + let (chunks, remainder) = input.as_chunks::(); + for chunk in chunks { + let block = *chunk; + self.decrypt_block(block, &mut plaintext); + } + self.carry.extend_from_slice(remainder); + plaintext + } + + pub(super) fn decrypt_block(&mut self, ciphertext: [u8; AES_128_BLOCK_BYTES], plaintext: &mut Vec) { + let mut decrypted = Block::::default(); + decrypted.copy_from_slice(&ciphertext); + self.cipher.decrypt_block(&mut decrypted); + plaintext.extend(decrypted.into_iter().zip(self.previous_ciphertext).map(|(byte, previous)| byte ^ previous)); + self.previous_ciphertext = ciphertext; + } + + pub(super) fn finish(&self) -> Result<(), HlsTsProbeError> { + self.carry.is_empty().then_some(()).ok_or(HlsTsProbeError::DecryptionFailed) + } +} + +impl Drop for HlsAes128CbcPrefixDecoder { + fn drop(&mut self) { self.previous_ciphertext.zeroize(); } +} + +pub(super) enum HlsTsSourceDecoder { + Clear, + Aes128Cbc(Box), +} + +impl HlsTsSourceDecoder { + pub(super) fn new(protection: HlsTsProbeProtection<'_>) -> Result { + match protection { + HlsTsProbeProtection::Clear => Ok(Self::Clear), + HlsTsProbeProtection::Aes128Cbc { key, iv } => { + HlsAes128CbcPrefixDecoder::new(key, iv).map(Box::new).map(Self::Aes128Cbc) + } + } + } + + pub(super) fn with_plaintext(&mut self, bytes: &[u8], consume: impl FnOnce(&[u8]) -> T) -> T { + match self { + Self::Clear => consume(bytes), + Self::Aes128Cbc(decoder) => { + let plaintext = decoder.push(bytes); + consume(&plaintext) + } + } + } + + pub(super) const fn plaintext_remainder(&self) -> HlsTsPlaintextRemainder { + match self { + Self::Clear => HlsTsPlaintextRemainder::ExactPackets, + Self::Aes128Cbc(_) => HlsTsPlaintextRemainder::Aes128Pkcs7, + } + } + + pub(super) fn finish(&self) -> Result<(), HlsTsProbeError> { + match self { + Self::Clear => Ok(()), + Self::Aes128Cbc(decoder) => decoder.finish(), + } + } +} diff --git a/backend/mpegts/src/ts_inspector/sections.rs b/backend/mpegts/src/ts_inspector/sections.rs new file mode 100644 index 000000000..e75c5507e --- /dev/null +++ b/backend/mpegts/src/ts_inspector/sections.rs @@ -0,0 +1,337 @@ +use super::{ + demux::HlsPatSectionParser, HlsPatPacketFilter, HlsPmtEvidence, HlsPmtPacketFilter, HlsPmtSectionEvidence, + HlsPmtSectionParser, HlsPsiSectionConsumer, HlsTsDemuxContext, HlsTsMalformedReason, HlsTsProgramTopology, + PSI_SYNTAX_HEADER_BYTES, TS_PACKET_BYTES, +}; +use mpeg2ts_reader::{ + demultiplex, + demultiplex::PacketFilter, + packet::{Packet, Pid}, + psi, + psi::{BufferSectionSyntaxParser, SectionPacketConsumer, SectionProcessor, SectionSyntaxSectionProcessor}, +}; +use std::collections::BTreeSet; + +#[derive(Clone, Copy)] +pub(super) enum HlsPsiTableKind { + Pat, + Pmt, +} + +impl HlsPsiTableKind { + pub(super) const fn malformed_reason(self) -> HlsTsMalformedReason { + match self { + Self::Pat => HlsTsMalformedReason::InvalidPat, + Self::Pmt => HlsTsMalformedReason::InvalidPmt, + } + } + + pub(super) const fn expected_table_id(self) -> u8 { + match self { + Self::Pat => 0x00, + Self::Pmt => 0x02, + } + } + + pub(super) const fn minimum_section_length(self) -> usize { + match self { + Self::Pat => 9, + Self::Pmt => 13, + } + } + + pub(super) fn validate_common_header(self, header: &[u8]) -> bool { + header.len() == psi::SectionCommonHeader::SIZE + && header[0] == self.expected_table_id() + && header[1] & 0x80 != 0 + && header[1] & 0x30 == 0x30 + && (self.minimum_section_length()..=1_021).contains(&psi_section_length(header)) + } +} + +fn psi_section_length(header: &[u8]) -> usize { (usize::from(header[1] & 0x0F) << 8) | usize::from(header[2]) } + +impl

HlsPsiSectionConsumer

+where + P: SectionProcessor, +{ + pub(super) fn new(consumer: SectionPacketConsumer

, table_kind: HlsPsiTableKind) -> Self { + Self { consumer, table_kind, pending_syntax_header: None } + } + + pub(super) fn consume(&mut self, ctx: &mut HlsTsDemuxContext, packet: &Packet<'_>) { + if let Some(mut pending) = self.pending_syntax_header.take() { + let Some(payload) = packet.payload() else { + self.pending_syntax_header = Some(pending); + return; + }; + if packet.payload_unit_start_indicator() { + let Some((&pointer, section_data)) = payload.split_first() else { + ctx.record_malformed(HlsTsMalformedReason::InvalidPsiPointer); + return; + }; + let pointer = usize::from(pointer); + if pointer >= section_data.len() { + ctx.record_malformed(HlsTsMalformedReason::InvalidPsiPointer); + return; + } + let (previous_section, section_start) = section_data.split_at(pointer); + pending.extend_from_slice(previous_section); + if pending.len() < psi::SectionCommonHeader::SIZE + || !self.table_kind.validate_common_header(&pending[..psi::SectionCommonHeader::SIZE]) + { + ctx.record_malformed(self.table_kind.malformed_reason()); + return; + } + let section_bytes = psi::SectionCommonHeader::SIZE + .saturating_add(psi_section_length(&pending[..psi::SectionCommonHeader::SIZE])); + if pending.len() < section_bytes { + ctx.record_malformed(self.table_kind.malformed_reason()); + return; + } + self.consume_synthetic_payload(ctx, packet.pid(), &pending[..section_bytes], true); + if ctx.malformed.is_none() { + self.consume_new_section(ctx, packet.pid(), section_start); + } + return; + } + pending.extend_from_slice(payload); + if pending.len() < psi::SectionCommonHeader::SIZE { + self.pending_syntax_header = Some(pending); + return; + } + if !self.table_kind.validate_common_header(&pending[..psi::SectionCommonHeader::SIZE]) { + ctx.record_malformed(self.table_kind.malformed_reason()); + return; + } + if pending.len() < PSI_SYNTAX_HEADER_BYTES { + self.pending_syntax_header = Some(pending); + return; + } + let section_bytes = psi::SectionCommonHeader::SIZE + .saturating_add(psi_section_length(&pending[..psi::SectionCommonHeader::SIZE])) + .min(pending.len()); + self.consume_synthetic_payload(ctx, packet.pid(), &pending[..section_bytes], true); + return; + } + + let Some(payload) = packet.payload() else { + self.consumer.consume(ctx, packet); + return; + }; + if !packet.payload_unit_start_indicator() { + self.consumer.consume(ctx, packet); + return; + } + let Some((&pointer, section_data)) = payload.split_first() else { + ctx.record_malformed(HlsTsMalformedReason::InvalidPsiPointer); + return; + }; + let pointer = usize::from(pointer); + if pointer >= section_data.len() { + ctx.record_malformed(HlsTsMalformedReason::InvalidPsiPointer); + return; + } + let section_start = §ion_data[pointer..]; + if section_start.len() < PSI_SYNTAX_HEADER_BYTES { + if let Some(previous_section) = section_data.get(..pointer).filter(|bytes| !bytes.is_empty()) { + self.consume_synthetic_payload(ctx, packet.pid(), previous_section, false); + } + if ctx.malformed.is_none() { + self.consume_new_section(ctx, packet.pid(), section_start); + } + return; + } + if !self.table_kind.validate_common_header(§ion_start[..psi::SectionCommonHeader::SIZE]) { + ctx.record_malformed(self.table_kind.malformed_reason()); + return; + } + self.consumer.consume(ctx, packet); + } + + pub(super) fn consume_new_section(&mut self, ctx: &mut HlsTsDemuxContext, pid: Pid, section_start: &[u8]) { + if section_start.len() < PSI_SYNTAX_HEADER_BYTES { + if section_start.len() >= psi::SectionCommonHeader::SIZE + && !self.table_kind.validate_common_header(§ion_start[..psi::SectionCommonHeader::SIZE]) + { + ctx.record_malformed(self.table_kind.malformed_reason()); + return; + } + self.pending_syntax_header = Some(section_start.to_vec()); + return; + } + if !self.table_kind.validate_common_header(§ion_start[..psi::SectionCommonHeader::SIZE]) { + ctx.record_malformed(self.table_kind.malformed_reason()); + return; + } + self.consume_synthetic_payload(ctx, pid, section_start, true); + } + + pub(super) fn consume_synthetic_payload( + &mut self, + ctx: &mut HlsTsDemuxContext, + pid: Pid, + bytes: &[u8], + start: bool, + ) { + let mut offset = 0usize; + let mut first = true; + while offset < bytes.len() { + let starts_section = start && first; + let capacity = if starts_section { 183 } else { 184 }; + let copied = bytes.len().saturating_sub(offset).min(capacity); + let payload = &bytes[offset..offset.saturating_add(copied)]; + let packet_bytes = synthetic_psi_packet(pid, starts_section, payload); + if let Some(packet) = Packet::try_new(&packet_bytes) { + self.consumer.consume(ctx, &packet); + } + offset = offset.saturating_add(copied); + first = false; + } + } +} + +pub(super) fn synthetic_psi_packet(pid: Pid, starts_section: bool, payload: &[u8]) -> [u8; TS_PACKET_BYTES] { + let pointer_bytes = usize::from(starts_section); + let payload_length = payload.len().saturating_add(pointer_bytes).min(184); + let pid = u16::from(pid); + let mut packet = [0xFF_u8; TS_PACKET_BYTES]; + packet[0] = Packet::SYNC_BYTE; + packet[1] = (u8::try_from(pid >> 8).unwrap_or(0) & 0x1F) | if starts_section { 0x40 } else { 0 }; + packet[2] = pid.to_be_bytes()[1]; + let payload_offset = if payload_length == 184 { + packet[3] = 0x10; + 4 + } else { + packet[3] = 0x30; + let adaptation_length = 183usize.saturating_sub(payload_length); + packet[4] = u8::try_from(adaptation_length).unwrap_or(182); + if adaptation_length > 0 { + packet[5] = 0; + } + 5usize.saturating_add(adaptation_length) + }; + let data_offset = if starts_section { + packet[payload_offset] = 0; + payload_offset.saturating_add(1) + } else { + payload_offset + }; + let copy_length = payload.len().min(TS_PACKET_BYTES.saturating_sub(data_offset)); + packet[data_offset..data_offset.saturating_add(copy_length)].copy_from_slice(&payload[..copy_length]); + packet +} + +impl Default for HlsPatPacketFilter { + fn default() -> Self { + Self { + consumer: HlsPsiSectionConsumer::new( + SectionPacketConsumer::new(SectionSyntaxSectionProcessor::new(BufferSectionSyntaxParser::new( + HlsPatSectionParser, + ))), + HlsPsiTableKind::Pat, + ), + } + } +} + +impl PacketFilter for HlsPatPacketFilter { + type Ctx = HlsTsDemuxContext; + + fn consume(&mut self, ctx: &mut Self::Ctx, packet: &Packet<'_>) { self.consumer.consume(ctx, packet); } +} + +impl HlsPmtPacketFilter { + pub(super) fn new(pid: Pid, program_number: u16) -> Self { + Self { + consumer: HlsPsiSectionConsumer::new( + SectionPacketConsumer::new(SectionSyntaxSectionProcessor::new(BufferSectionSyntaxParser::new( + HlsPmtSectionParser { pid, program_number }, + ))), + HlsPsiTableKind::Pmt, + ), + } + } +} + +impl PacketFilter for HlsPmtPacketFilter { + type Ctx = HlsTsDemuxContext; + + fn consume(&mut self, ctx: &mut Self::Ctx, packet: &Packet<'_>) { self.consumer.consume(ctx, packet); } +} + +pub(super) enum HlsTsPacketFilter { + Pat(HlsPatPacketFilter), + Pmt(HlsPmtPacketFilter), + Ignore(demultiplex::NullPacketFilter), +} + +impl PacketFilter for HlsTsPacketFilter { + type Ctx = HlsTsDemuxContext; + + fn consume(&mut self, ctx: &mut Self::Ctx, packet: &Packet<'_>) { + match self { + Self::Pat(filter) => filter.consume(ctx, packet), + Self::Pmt(filter) => filter.consume(ctx, packet), + Self::Ignore(filter) => filter.consume(ctx, packet), + } + } +} + +impl HlsPmtEvidence { + pub(super) fn complete(&self) -> bool { + self.last_section_number.is_some_and(|last| (0..=last).all(|section| self.sections.contains_key(§ion))) + } + + pub(super) fn restart_version(&mut self, version: u8, last_section_number: u8) { + self.table_version = Some(version); + self.last_section_number = Some(last_section_number); + self.sections.clear(); + self.has_pcr = false; + self.stream_types.clear(); + } + + pub(super) fn record_section( + &mut self, + section_number: u8, + section: HlsPmtSectionEvidence, + ) -> Result<(), HlsTsMalformedReason> { + if self.sections.get(§ion_number).is_some_and(|current| current != §ion) { + return Err(HlsTsMalformedReason::InvalidPmt); + } + self.sections.insert(section_number, section); + self.has_pcr = self.sections.values().any(|section| section.pcr_pid != u16::from(mpeg2ts_reader::STUFFING_PID)); + self.stream_types = self + .sections + .values() + .flat_map(|section| section.streams.iter().map(|stream| stream.stream_type)) + .collect(); + Ok(()) + } + + pub(super) fn topology(&self, transport_stream_id: u16, pmt_pid: u16) -> Option { + if !self.complete() { + return None; + } + let mut sections = self.sections.values(); + let pcr_pid = sections.next()?.pcr_pid; + if self.sections.values().any(|section| section.pcr_pid != pcr_pid) { + return None; + } + let streams = self.sections.values().flat_map(|section| section.streams.iter().copied()).collect::>(); + if streams.is_empty() { + return None; + } + let mut elementary_pids = BTreeSet::new(); + if streams.iter().any(|stream| !elementary_pids.insert(stream.elementary_pid)) { + return None; + } + Some(HlsTsProgramTopology { + transport_stream_id, + program_number: self.program_number, + pmt_pid, + pcr_pid, + streams: streams.into(), + }) + } +} diff --git a/backend/mpegts/src/ts_inspector/tests.rs b/backend/mpegts/src/ts_inspector/tests.rs new file mode 100644 index 000000000..ced1f700a --- /dev/null +++ b/backend/mpegts/src/ts_inspector/tests.rs @@ -0,0 +1,362 @@ +use super::*; +use aes::cipher::{Block, BlockEncrypt, KeyInit}; +use mpeg2ts_reader::mpegts_crc; +use std::{ + cell::Cell, + io::{Cursor, Read}, + pin::Pin, + rc::Rc, + sync::atomic::{AtomicU64, Ordering}, + task::{Context, Poll}, +}; +use tokio::io::{AsyncRead, ReadBuf}; + +fn append_crc(mut section: Vec) -> Vec { + let crc = mpegts_crc::sum32(§ion); + section.extend_from_slice(&crc.to_be_bytes()); + section +} + +fn psi_version_byte(version: u8, current: bool) -> u8 { 0xC0 | ((version & 0x1F) << 1) | u8::from(current) } + +fn pat_section_with_header( + programs: &[(u16, u16)], + version: u8, + current: bool, + section_number: u8, + last_section_number: u8, +) -> Vec { + let section_length = 5usize.saturating_add(programs.len().saturating_mul(4)).saturating_add(4); + let mut section = vec![ + 0x00, + 0xB0 | u8::try_from(section_length >> 8).unwrap_or(0), + u8::try_from(section_length & 0xFF).unwrap_or(0), + ]; + section.extend_from_slice(&[0x00, 0x01, psi_version_byte(version, current), section_number, last_section_number]); + for (program_number, pid) in programs { + section.extend_from_slice(&program_number.to_be_bytes()); + section.extend_from_slice(&(0xE000 | (pid & 0x1FFF)).to_be_bytes()); + } + append_crc(section) +} + +fn pat_section(programs: &[(u16, u16)]) -> Vec { pat_section_with_header(programs, 0, true, 0, 0) } + +#[derive(Clone, Copy)] +struct PmtSectionHeader { + version: u8, + current: bool, + section_number: u8, + last_section_number: u8, +} + +fn pmt_section_with_header( + program_number: u16, + pcr_pid: u16, + streams: &[(u8, u16)], + descriptor_bytes: usize, + header: PmtSectionHeader, +) -> Vec { + let program_info_length = descriptor_bytes.min(0x0FFF) & !1; + let section_length = + 9usize.saturating_add(program_info_length).saturating_add(streams.len().saturating_mul(5)).saturating_add(4); + let mut section = vec![ + 0x02, + 0xB0 | u8::try_from(section_length >> 8).unwrap_or(0), + u8::try_from(section_length & 0xFF).unwrap_or(0), + ]; + section.extend_from_slice(&program_number.to_be_bytes()); + section.extend_from_slice(&[ + psi_version_byte(header.version, header.current), + header.section_number, + header.last_section_number, + ]); + section.extend_from_slice(&(0xE000 | (pcr_pid & 0x1FFF)).to_be_bytes()); + section.extend_from_slice(&(0xF000 | u16::try_from(program_info_length).unwrap_or(0)).to_be_bytes()); + for _ in 0..program_info_length / 2 { + section.extend_from_slice(&[0x80, 0x00]); + } + for (stream_type, pid) in streams { + section.push(*stream_type); + section.extend_from_slice(&(0xE000 | (pid & 0x1FFF)).to_be_bytes()); + section.extend_from_slice(&[0xF0, 0x00]); + } + append_crc(section) +} + +fn pmt_section(program_number: u16, pcr_pid: u16, streams: &[(u8, u16)], descriptor_bytes: usize) -> Vec { + pmt_section_with_header( + program_number, + pcr_pid, + streams, + descriptor_bytes, + PmtSectionHeader { version: 0, current: true, section_number: 0, last_section_number: 0 }, + ) +} + +fn packetize_section(pid: u16, section: &[u8], first_counter: u8) -> Vec { + let mut output = Vec::new(); + let mut offset = 0usize; + let mut counter = first_counter & 0x0F; + let mut first = true; + while offset < section.len() { + let mut packet = [0xFF_u8; TS_PACKET_BYTES]; + packet[0] = Packet::SYNC_BYTE; + packet[1] = u8::try_from(pid >> 8).unwrap_or(0) & 0x1F; + packet[2] = u8::try_from(pid & 0xFF).unwrap_or(0); + packet[3] = 0x10 | counter; + let payload_start = if first { + packet[1] |= 0x40; + packet[4] = 0; + 5 + } else { + 4 + }; + let copied = section.len().saturating_sub(offset).min(TS_PACKET_BYTES.saturating_sub(payload_start)); + packet[payload_start..payload_start.saturating_add(copied)] + .copy_from_slice(§ion[offset..offset.saturating_add(copied)]); + output.extend_from_slice(&packet); + offset = offset.saturating_add(copied); + counter = counter.wrapping_add(1) & 0x0F; + first = false; + } + output +} + +fn packetize_section_with_split_syntax_header(pid: u16, section: &[u8], first_header_bytes: usize) -> Vec { + let first_header_bytes = first_header_bytes.min(PSI_SYNTAX_HEADER_BYTES.saturating_sub(1)); + let pid = Pid::new(pid); + let mut output = synthetic_psi_packet(pid, true, §ion[..first_header_bytes]).to_vec(); + for chunk in section[first_header_bytes..].chunks(184) { + output.extend_from_slice(&synthetic_psi_packet(pid, false, chunk)); + } + output +} + +fn psi_start_packet_with_pointer(pid: u16, previous_section: &[u8], section_start: &[u8]) -> [u8; TS_PACKET_BYTES] { + let payload_length = 1usize.saturating_add(previous_section.len()).saturating_add(section_start.len()).min(184); + let mut packet = [0xFF_u8; TS_PACKET_BYTES]; + packet[0] = Packet::SYNC_BYTE; + packet[1] = (u8::try_from(pid >> 8).unwrap_or(0) & 0x1F) | 0x40; + packet[2] = u8::try_from(pid & 0xFF).unwrap_or(0); + packet[3] = 0x30; + let adaptation_length = 183usize.saturating_sub(payload_length); + packet[4] = u8::try_from(adaptation_length).unwrap_or(182); + if adaptation_length > 0 { + packet[5] = 0; + } + let payload_offset = 5usize.saturating_add(adaptation_length); + packet[payload_offset] = u8::try_from(previous_section.len()).unwrap_or(u8::MAX); + let previous_offset = payload_offset.saturating_add(1); + let previous_end = previous_offset.saturating_add(previous_section.len()); + packet[previous_offset..previous_end].copy_from_slice(previous_section); + let section_end = previous_end.saturating_add(section_start.len()); + packet[previous_end..section_end].copy_from_slice(section_start); + packet +} + +fn null_packet() -> [u8; TS_PACKET_BYTES] { + let mut packet = [0xFF_u8; TS_PACKET_BYTES]; + packet[0] = Packet::SYNC_BYTE; + packet[1] = 0x1F; + packet[2] = 0xFF; + packet[3] = 0x10; + packet +} + +fn track_stream(descriptor_bytes: usize) -> Vec { + let mut stream = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); + stream.extend_from_slice(&packetize_section( + 0x100, + &pmt_section(1, 0x101, &[(0x1B, 0x101), (0x0F, 0x102), (0x0F, 0x103)], descriptor_bytes), + 0, + )); + stream.extend_from_slice(&null_packet()); + stream +} + +fn media_payload_packet( + pid: u16, + continuity_counter: u8, + payload_unit_start: bool, + discontinuity: bool, + payload: &[u8], +) -> [u8; TS_PACKET_BYTES] { + assert!(!payload.is_empty() && payload.len() <= 182); + let mut packet = [0xFF_u8; TS_PACKET_BYTES]; + packet[0] = Packet::SYNC_BYTE; + packet[1] = u8::try_from(pid >> 8).unwrap_or(0) & 0x1F; + if payload_unit_start { + packet[1] |= 0x40; + } + packet[2] = u8::try_from(pid & 0xFF).unwrap_or(0); + packet[3] = 0x30 | (continuity_counter & 0x0F); + let adaptation_length = 183usize.saturating_sub(payload.len()); + packet[4] = u8::try_from(adaptation_length).unwrap_or(182); + if adaptation_length > 0 { + packet[5] = if discontinuity { 0x80 } else { 0 }; + } + let payload_offset = 5usize.saturating_add(adaptation_length); + packet[payload_offset..].copy_from_slice(payload); + packet +} + +fn media_adaptation_only_packet(pid: u16, continuity_counter: u8, discontinuity: bool) -> [u8; TS_PACKET_BYTES] { + let mut packet = [0xFF_u8; TS_PACKET_BYTES]; + packet[0] = Packet::SYNC_BYTE; + packet[1] = u8::try_from(pid >> 8).unwrap_or(0) & 0x1F; + packet[2] = u8::try_from(pid & 0xFF).unwrap_or(0); + packet[3] = 0x20 | (continuity_counter & 0x0F); + packet[4] = 183; + packet[5] = if discontinuity { 0x80 } else { 0 }; + packet +} + +fn pes_bytes(stream_id: u8, declared_bytes: u16, payload_after_length: &[u8]) -> Vec { + let mut pes = vec![0, 0, 1, stream_id]; + pes.extend_from_slice(&declared_bytes.to_be_bytes()); + pes.extend_from_slice(payload_after_length); + pes +} + +async fn complete_media_evidence(bytes: &[u8]) -> HlsTsMediaEvidence { + complete_media_evidence_with_duration(bytes, 90_000).await +} + +async fn complete_media_evidence_with_duration(bytes: &[u8], expected_duration_ticks_90khz: u64) -> HlsTsMediaEvidence { + let source_size = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + inspect_mpeg_ts_media_evidence_async( + bytes, + HlsTsProbeProtection::Clear, + HlsTsProbeBudget { + max_bytes: source_size.saturating_add(1), + max_packets: source_size.saturating_add(187).saturating_div(188).saturating_add(1), + ..HlsTsProbeBudget::default() + }, + expected_duration_ticks_90khz, + ) + .await + .expect("complete media evidence") +} + +fn valid_track_prefix(minimum_bytes: usize) -> Vec { + let mut stream = track_stream(0); + let null_packet = null_packet(); + while stream.len() < minimum_bytes { + stream.extend_from_slice(&null_packet); + } + stream +} + +fn multi_packet_pat_stream() -> Vec { + let programs = + (0_u16..46).map(|index| (index.saturating_add(1), 0x100_u16.saturating_add(index))).collect::>(); + let mut stream = packetize_section(0, &pat_section(&programs), 0); + for (program_number, pmt_pid) in programs { + stream.extend_from_slice(&packetize_section( + pmt_pid, + &pmt_section(program_number, 0x400, &[(0x1B, 0x400), (0x0F, 0x401)], 0), + 0, + )); + } + stream.extend_from_slice(&null_packet()); + stream +} + +fn found(outcome: HlsTsProbeOutcome) -> HlsTsTrackSignature { + match outcome { + HlsTsProbeOutcome::Found(signature) => signature, + other => panic!("expected signature, got {other:?}"), + } +} + +struct CountingReader { + inner: Cursor>, + bytes_read: Rc>, +} + +impl Read for CountingReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let read = std::io::Read::read(&mut self.inner, buffer)?; + self.bytes_read.set(self.bytes_read.get().saturating_add(read)); + Ok(read) + } +} + +struct VirtualTailReader { + prefix: Arc<[u8]>, + logical_len: u64, + position: u64, + bytes_read: Arc, +} + +impl VirtualTailReader { + fn new(prefix: Arc<[u8]>, logical_len: u64, bytes_read: Arc) -> Self { + Self { prefix, logical_len, position: 0, bytes_read } + } + + fn read_into(&mut self, output: &mut [u8]) -> usize { + let remaining = self.logical_len.saturating_sub(self.position); + let read = usize::try_from(remaining).unwrap_or(usize::MAX).min(output.len()); + if read == 0 { + return 0; + } + let position = usize::try_from(self.position).unwrap_or(usize::MAX); + let prefix_read = self.prefix.len().saturating_sub(position).min(read); + if prefix_read > 0 { + output[..prefix_read].copy_from_slice(&self.prefix[position..position.saturating_add(prefix_read)]); + } + output[prefix_read..read].fill(0); + self.position = self.position.saturating_add(u64::try_from(read).unwrap_or(u64::MAX)); + self.bytes_read.fetch_add(u64::try_from(read).unwrap_or(u64::MAX), Ordering::Relaxed); + read + } +} + +impl Read for VirtualTailReader { + fn read(&mut self, output: &mut [u8]) -> std::io::Result { Ok(self.read_into(output)) } +} + +impl AsyncRead for VirtualTailReader { + fn poll_read( + mut self: Pin<&mut Self>, + _context: &mut Context<'_>, + output: &mut ReadBuf<'_>, + ) -> Poll> { + let read = self.as_mut().get_mut().read_into(output.initialize_unfilled()); + output.advance(read); + Poll::Ready(Ok(())) + } +} + +fn encrypt_aes128_cbc(plaintext: &[u8], key: &[u8; AES_128_BLOCK_BYTES], iv: [u8; AES_128_BLOCK_BYTES]) -> Vec { + let cipher = Aes128::new_from_slice(key).expect("valid test key"); + let mut previous = iv; + let mut output = Vec::with_capacity(plaintext.len()); + for chunk in plaintext.as_chunks::().0 { + let mut block = Block::::default(); + for ((byte, plaintext), previous) in block.iter_mut().zip(chunk).zip(previous) { + *byte = plaintext ^ previous; + } + cipher.encrypt_block(&mut block); + previous.copy_from_slice(&block); + output.extend_from_slice(&block); + } + output +} + +fn encrypt_aes128_cbc_pkcs7( + plaintext: &[u8], + key: &[u8; AES_128_BLOCK_BYTES], + iv: [u8; AES_128_BLOCK_BYTES], +) -> Vec { + let padding = AES_128_BLOCK_BYTES - plaintext.len() % AES_128_BLOCK_BYTES; + let mut padded = plaintext.to_vec(); + padded.resize(padded.len().saturating_add(padding), u8::try_from(padding).unwrap_or(0)); + encrypt_aes128_cbc(&padded, key, iv) +} + +mod recovery; +mod storage; +mod terminal; +mod transport; diff --git a/backend/mpegts/src/ts_inspector/tests/recovery.rs b/backend/mpegts/src/ts_inspector/tests/recovery.rs new file mode 100644 index 000000000..e1adee61d --- /dev/null +++ b/backend/mpegts/src/ts_inspector/tests/recovery.rs @@ -0,0 +1,63 @@ +use super::*; + +#[test] +fn ts_inspector_restarts_incomplete_pat_on_new_current_version() { + let mut stream = packetize_section(0, &pat_section_with_header(&[(1, 0x100)], 0, true, 0, 1), 0); + stream.extend_from_slice(&packetize_section( + 0x100, + &pmt_section_with_header( + 1, + 0x101, + &[(0x02, 0x101)], + 0, + PmtSectionHeader { version: 0, current: true, section_number: 0, last_section_number: 1 }, + ), + 0, + )); + stream.extend_from_slice(&packetize_section(0, &pat_section_with_header(&[(2, 0x200)], 1, true, 0, 0), 1)); + stream.extend_from_slice(&packetize_section(0x200, &pmt_section(2, 0x201, &[(0x1B, 0x201), (0x0F, 0x202)], 0), 0)); + stream.extend_from_slice(&null_packet()); + + let signature = found( + inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe succeeds"), + ); + + assert_eq!(signature.program_count, 1); + assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x1B]); +} + +#[test] +fn ts_inspector_restarts_incomplete_pmt_on_new_current_version() { + let mut stream = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); + stream.extend_from_slice(&packetize_section( + 0x100, + &pmt_section_with_header( + 1, + 0x101, + &[(0x1B, 0x101)], + 0, + PmtSectionHeader { version: 0, current: true, section_number: 0, last_section_number: 1 }, + ), + 0, + )); + stream.extend_from_slice(&packetize_section( + 0x100, + &pmt_section_with_header( + 1, + 0x201, + &[(0x24, 0x201), (0x0F, 0x202)], + 0, + PmtSectionHeader { version: 1, current: true, section_number: 0, last_section_number: 0 }, + ), + 1, + )); + stream.extend_from_slice(&null_packet()); + + let signature = found( + inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe succeeds"), + ); + + assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x24]); +} diff --git a/backend/mpegts/src/ts_inspector/tests/storage.rs b/backend/mpegts/src/ts_inspector/tests/storage.rs new file mode 100644 index 000000000..837c43c2a --- /dev/null +++ b/backend/mpegts/src/ts_inspector/tests/storage.rs @@ -0,0 +1,36 @@ +use super::*; + +#[tokio::test] +async fn ts_inspector_sync_async_and_aes_prefix_paths_share_signature() { + let budget = HlsTsProbeBudget { read_chunk_bytes: 191, ..HlsTsProbeBudget::default() }; + let mut plaintext = track_stream(400); + let clear_signature = found( + inspect_mpeg_ts(Cursor::new(&plaintext), HlsTsProbeProtection::Clear, budget).expect("sync probe succeeds"), + ); + let async_clear_signature = found( + inspect_mpeg_ts_async(&plaintext[..], HlsTsProbeProtection::Clear, budget) + .await + .expect("async clear probe succeeds"), + ); + plaintext.resize(plaintext.len().next_multiple_of(AES_128_BLOCK_BYTES), 0xFF); + let key = *b"0123456789abcdef"; + let iv = [0xA5; AES_128_BLOCK_BYTES]; + let ciphertext = encrypt_aes128_cbc(&plaintext, &key, iv); + let async_aes_signature = found( + inspect_mpeg_ts_async(&ciphertext[..], HlsTsProbeProtection::Aes128Cbc { key: &key, iv }, budget) + .await + .expect("async AES probe succeeds"), + ); + + assert_eq!(async_clear_signature, clear_signature); + assert_eq!(async_aes_signature, clear_signature); +} + +#[test] +fn ts_inspector_does_not_mutate_source_bytes() { + let bytes = track_stream(0); + let before = bytes.clone(); + let _ = inspect_mpeg_ts(Cursor::new(&bytes), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe succeeds"); + assert_eq!(bytes, before); +} diff --git a/backend/mpegts/src/ts_inspector/tests/terminal.rs b/backend/mpegts/src/ts_inspector/tests/terminal.rs new file mode 100644 index 000000000..878dc91ca --- /dev/null +++ b/backend/mpegts/src/ts_inspector/tests/terminal.rs @@ -0,0 +1,89 @@ +use super::*; + +#[tokio::test] +async fn current_terminal_asset_and_finalized_segment_zero_have_complete_splice_evidence() { + const TERMINAL_ASSET_BYTES: &[u8] = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); + let renderer = crate::transport_stream_buffer::TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); + let duration_ticks = renderer.duration_ticks_90khz().expect("terminal asset duration"); + let asset_profile = renderer.finite_hls_timestamp_profile().expect("terminal asset profile"); + let prepared = renderer + .render_finite_hls_segment(crate::transport_stream_buffer::HlsFiniteTsRenderSpec { + timestamp_offset_ticks_90khz: 0, + continuity_seed: 0, + logical_segment_index: 0, + }) + .expect("prepared terminal segment zero"); + let finalized = renderer + .finalize_prepared_finite_hls_segment( + &prepared, + crate::transport_stream_buffer::HlsFiniteTsFinalizeSpec { + additional_timestamp_offset_ticks_90khz: 90_000, + discontinuity: crate::transport_stream_buffer::HlsFiniteTsDiscontinuityMode::FirstPacketPerPid, + }, + ) + .expect("finalized terminal segment zero"); + + let base = complete_media_evidence_with_duration(TERMINAL_ASSET_BYTES, duration_ticks).await; + let terminal = complete_media_evidence_with_duration(&finalized, duration_ticks).await; + + assert_eq!(base.timestamp_profile, Some(asset_profile)); + assert!(matches!(base.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); + assert!(matches!(terminal.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); + assert_eq!(evaluate_mpeg_ts_splice_boundary(&base.splice_evidence, &terminal.splice_evidence), Ok(())); + assert_eq!( + terminal.timestamp_profile.map(|profile| profile.span_ticks_90khz), + Some(asset_profile.span_ticks_90khz) + ); + assert!(duration_ticks > asset_profile.span_ticks_90khz); +} + +#[tokio::test] +async fn aes128_terminal_base_timestamp_profile_uses_decrypted_cached_bytes() { + const TERMINAL_ASSET_BYTES: &[u8] = + include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test/fixtures/hls/channel_unavailable.ts")); + let asset = crate::transport_stream_buffer::TransportStreamBuffer::new(TERMINAL_ASSET_BYTES.to_vec()); + let expected_profile = asset.finite_hls_timestamp_profile().expect("fixture timestamp profile"); + let expected_signature = asset.finite_hls_track_signature().expect("fixture track signature"); + let key = *b"0123456789abcdef"; + let iv = [0xA5; AES_128_BLOCK_BYTES]; + let ciphertext = encrypt_aes128_cbc_pkcs7(TERMINAL_ASSET_BYTES, &key, iv); + let source_size = u64::try_from(ciphertext.len()).expect("ciphertext size"); + let budget = HlsTsProbeBudget { + max_bytes: source_size.saturating_add(1), + max_packets: source_size.saturating_add(187) / 188 + 1, + ..HlsTsProbeBudget::default() + }; + + let evidence = inspect_mpeg_ts_media_evidence_async( + &ciphertext[..], + HlsTsProbeProtection::Aes128Cbc { key: &key, iv }, + budget, + asset.duration_ticks_90khz().expect("fixture duration"), + ) + .await + .expect("AES media evidence"); + + assert_eq!(evidence.track_outcome, HlsTsProbeOutcome::Found(expected_signature)); + assert_eq!(evidence.timestamp_profile, Some(expected_profile)); + assert!(matches!(evidence.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); +} + +#[test] +fn ts_inspector_rejects_invalid_psi_header_before_exhausting_probe_budget() { + let mut invalid_pat = pat_section(&[(1, 0x100)]); + invalid_pat[1] &= 0x7F; + invalid_pat[1] |= 0x0F; + invalid_pat[2] = 0xFF; + let mut stream = packetize_section(0, &invalid_pat, 0); + let null_packet = null_packet(); + while stream.len() <= usize::try_from(HLS_TS_PROBE_MAX_BYTES).unwrap_or(usize::MAX) { + stream.extend_from_slice(&null_packet); + } + + assert_eq!( + inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe completes"), + HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPat) + ); +} diff --git a/backend/mpegts/src/ts_inspector/tests/transport.rs b/backend/mpegts/src/ts_inspector/tests/transport.rs new file mode 100644 index 000000000..a71e49227 --- /dev/null +++ b/backend/mpegts/src/ts_inspector/tests/transport.rs @@ -0,0 +1,563 @@ +use super::*; + +#[test] +fn ts_inspector_reads_single_packet_pat_and_pmt() { + let signature = found( + inspect_mpeg_ts(Cursor::new(track_stream(0)), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe succeeds"), + ); + assert_eq!(signature.program_count, 1); + assert!(signature.has_pcr); + assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x1B]); + assert_eq!( + signature.programs.as_ref(), + &[HlsTsProgramTopology { + transport_stream_id: 1, + program_number: 1, + pmt_pid: 0x100, + pcr_pid: 0x101, + streams: Arc::from([ + HlsTsElementaryStreamBinding { stream_type: 0x1B, elementary_pid: 0x101 }, + HlsTsElementaryStreamBinding { stream_type: 0x0F, elementary_pid: 0x102 }, + HlsTsElementaryStreamBinding { stream_type: 0x0F, elementary_pid: 0x103 }, + ]), + }] + ); +} + +#[test] +fn equal_stream_types_with_different_pid_topology_are_not_compatible() { + let mut first = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); + first.extend_from_slice(&packetize_section(0x100, &pmt_section(1, 0x101, &[(0x1B, 0x101), (0x0F, 0x102)], 0), 0)); + let mut second = packetize_section(0, &pat_section(&[(1, 0x200)]), 0); + second.extend_from_slice(&packetize_section(0x200, &pmt_section(1, 0x201, &[(0x1B, 0x201), (0x0F, 0x202)], 0), 0)); + + let first = found( + inspect_mpeg_ts(Cursor::new(first), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("first topology"), + ); + let second = found( + inspect_mpeg_ts(Cursor::new(second), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("second topology"), + ); + + assert_eq!(first.stream_types, second.stream_types); + assert_ne!(first, second); + assert_eq!( + evaluate_mpeg_ts_splice_boundary( + &HlsTsSpliceEvidence::compatible_for_test(first), + &HlsTsSpliceEvidence::compatible_for_test(second), + ), + Err(HlsTsSpliceBoundaryIncompatibility::TopologyMismatch) + ); +} + +#[tokio::test] +async fn complete_media_evidence_enforces_ffmpeg_continuity_semantics() { + let pid = 0x101; + let exact_pes = pes_bytes(0xE0, 4, &[1, 2, 3, 4]); + let next_pes = pes_bytes(0xE0, 0, &[0x80, 0, 0]); + let mut valid = track_stream(0); + valid.extend_from_slice(&media_payload_packet(pid, 3, true, false, &exact_pes)); + valid.extend_from_slice(&media_adaptation_only_packet(pid, 3, false)); + valid.extend_from_slice(&media_payload_packet(pid, 4, true, false, &next_pes)); + assert!(matches!(complete_media_evidence(&valid).await.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); + + let mut discontinuity = track_stream(0); + discontinuity.extend_from_slice(&media_payload_packet(pid, 3, true, false, &exact_pes)); + discontinuity.extend_from_slice(&media_adaptation_only_packet(pid, 11, true)); + discontinuity.extend_from_slice(&media_payload_packet(pid, 12, true, false, &next_pes)); + assert!(matches!( + complete_media_evidence(&discontinuity).await.splice_evidence, + HlsTsSpliceEvidence::Compatible(_) + )); + + let mut invalid = track_stream(0); + invalid.extend_from_slice(&media_payload_packet(pid, 3, true, false, &exact_pes)); + invalid.extend_from_slice(&media_adaptation_only_packet(pid, 4, false)); + assert!(matches!( + complete_media_evidence(&invalid).await.splice_evidence, + HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::ContinuityFailure { + pid: 0x101, + expected: 3, + actual: 4, + .. + }) + )); +} + +#[tokio::test] +async fn complete_media_evidence_rejects_tei_and_internal_payload_jump() { + let pid = 0x101; + let first = pes_bytes(0xE0, 200, &[0x11; 100]); + let continuation = [0x22; 100]; + let mut jumped = track_stream(0); + jumped.extend_from_slice(&media_payload_packet(pid, 5, true, false, &first)); + jumped.extend_from_slice(&media_payload_packet(pid, 9, false, false, &continuation)); + assert!(matches!( + complete_media_evidence(&jumped).await.splice_evidence, + HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::ContinuityFailure { + pid: 0x101, + expected: 6, + actual: 9, + .. + }) + )); + + let mut tei_packet = media_payload_packet(pid, 5, true, false, &pes_bytes(0xE0, 0, &[0x80])); + tei_packet[1] |= 0x80; + let mut tei = track_stream(0); + tei.extend_from_slice(&tei_packet); + assert!(matches!( + complete_media_evidence(&tei).await.splice_evidence, + HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::TransportError { pid: 0x101, .. }) + )); +} + +#[tokio::test] +async fn complete_media_evidence_accounts_finite_split_and_unbounded_pes() { + let pid = 0x101; + let mut truncated = track_stream(0); + truncated.extend_from_slice(&media_payload_packet(pid, 0, true, false, &pes_bytes(0xE0, 20, &[0xAA; 8]))); + assert!(matches!( + complete_media_evidence(&truncated).await.splice_evidence, + HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::IncompletePes { + pid: 0x101, + declared_bytes: Some(20), + observed_bytes: 8, + .. + }) + )); + + let mut exact = track_stream(0); + exact.extend_from_slice(&media_payload_packet(pid, 0, true, false, &pes_bytes(0xE0, 8, &[0xAA; 8]))); + assert!(matches!(complete_media_evidence(&exact).await.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); + + let header = pes_bytes(0xE0, 5, &[1, 2, 3, 4, 5]); + let mut split = track_stream(0); + split.extend_from_slice(&media_payload_packet(pid, 0, true, false, &header[..4])); + split.extend_from_slice(&media_payload_packet(pid, 1, false, false, &header[4..])); + assert!(matches!(complete_media_evidence(&split).await.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); + + let mut unbounded = track_stream(0); + unbounded.extend_from_slice(&media_payload_packet(pid, 0, true, false, &pes_bytes(0xE0, 0, &[0x80, 0, 0, 1]))); + assert!(matches!(complete_media_evidence(&unbounded).await.splice_evidence, HlsTsSpliceEvidence::Compatible(_))); + + let mut invalid = track_stream(0); + invalid.extend_from_slice(&media_payload_packet(pid, 0, true, false, &[0x12, 0x34, 0x56])); + assert!(matches!( + complete_media_evidence(&invalid).await.splice_evidence, + HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::InvalidPes { pid: 0x101, .. }) + )); +} + +#[test] +fn ts_inspector_reassembles_multi_packet_and_cross_chunk_psi() { + let budget = HlsTsProbeBudget { read_chunk_bytes: 191, ..HlsTsProbeBudget::default() }; + let track_signature = found( + inspect_mpeg_ts(Cursor::new(track_stream(400)), HlsTsProbeProtection::Clear, budget).expect("probe succeeds"), + ); + assert_eq!(track_signature.stream_types.as_ref(), &[0x0F, 0x1B]); + + let program_signature = found( + inspect_mpeg_ts(Cursor::new(multi_packet_pat_stream()), HlsTsProbeProtection::Clear, budget) + .expect("probe succeeds"), + ); + assert_eq!(program_signature.program_count, 46); + assert_eq!(program_signature.stream_types.as_ref(), &[0x0F, 0x1B]); +} + +#[test] +fn ts_inspector_never_mixes_pat_or_pmt_sections_across_versions() { + let mut pat_version_mix = packetize_section(0, &pat_section_with_header(&[(1, 0x100)], 0, true, 0, 1), 0); + pat_version_mix.extend_from_slice(&packetize_section(0, &pat_section_with_header(&[(2, 0x200)], 1, true, 1, 1), 1)); + pat_version_mix.extend_from_slice(&packetize_section(0x200, &pmt_section(2, 0x201, &[(0x1B, 0x201)], 0), 0)); + assert_eq!( + inspect_mpeg_ts(Cursor::new(pat_version_mix), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default(),) + .expect("probe completes"), + HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::IncompleteProgramMetadata) + ); + + let mut program_map_version_mix = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); + program_map_version_mix.extend_from_slice(&packetize_section( + 0x100, + &pmt_section_with_header( + 1, + 0x101, + &[(0x1B, 0x101)], + 0, + PmtSectionHeader { version: 0, current: true, section_number: 0, last_section_number: 1 }, + ), + 0, + )); + program_map_version_mix.extend_from_slice(&packetize_section( + 0x100, + &pmt_section_with_header( + 1, + 0x102, + &[(0x0F, 0x102)], + 0, + PmtSectionHeader { version: 1, current: true, section_number: 1, last_section_number: 1 }, + ), + 1, + )); + assert_eq!( + inspect_mpeg_ts( + Cursor::new(program_map_version_mix), + HlsTsProbeProtection::Clear, + HlsTsProbeBudget::default(), + ) + .expect("probe completes"), + HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::IncompleteProgramMetadata) + ); +} + +#[test] +fn ts_inspector_rejects_same_version_pat_and_pmt_section_contradictions() { + let mut contradictory_pat = packetize_section(0, &pat_section_with_header(&[(1, 0x100)], 0, true, 0, 1), 0); + contradictory_pat.extend_from_slice(&packetize_section( + 0, + &pat_section_with_header(&[(1, 0x101)], 0, true, 0, 1), + 1, + )); + assert_eq!( + inspect_mpeg_ts(Cursor::new(contradictory_pat), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default(),) + .expect("probe completes"), + HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPat) + ); + + let mut stream = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); + for (counter, stream_type) in [(0, 0x1B), (1, 0x0F)] { + stream.extend_from_slice(&packetize_section( + 0x100, + &pmt_section_with_header( + 1, + 0x101, + &[(stream_type, 0x101)], + 0, + PmtSectionHeader { version: 0, current: true, section_number: 0, last_section_number: 1 }, + ), + counter, + )); + } + + assert_eq!( + inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe completes"), + HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPmt) + ); +} + +#[test] +fn ts_inspector_next_pat_or_pmt_does_not_change_current_evidence() { + let mut stream = packetize_section(0, &pat_section_with_header(&[(1, 0x100)], 0, true, 0, 1), 0); + stream.extend_from_slice(&packetize_section(0, &pat_section_with_header(&[(2, 0x200)], 1, false, 1, 1), 1)); + stream.extend_from_slice(&packetize_section(0, &pat_section_with_header(&[], 0, true, 1, 1), 2)); + for (counter, header, stream_type, pcr_pid, elementary_pid) in [ + ( + 0, + PmtSectionHeader { version: 0, current: true, section_number: 0, last_section_number: 1 }, + 0x1B, + 0x101, + 0x101, + ), + ( + 1, + PmtSectionHeader { version: 1, current: false, section_number: 1, last_section_number: 1 }, + 0x24, + 0x201, + 0x201, + ), + ( + 2, + PmtSectionHeader { version: 0, current: true, section_number: 1, last_section_number: 1 }, + 0x0F, + 0x101, + 0x102, + ), + ] { + stream.extend_from_slice(&packetize_section( + 0x100, + &pmt_section_with_header(1, pcr_pid, &[(stream_type, elementary_pid)], 0, header), + counter, + )); + } + + let signature = found( + inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe succeeds"), + ); + + assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x1B]); +} + +#[test] +fn ts_inspector_reassembles_syntax_headers_split_across_packets() { + let mut stream = packetize_section_with_split_syntax_header(0, &pat_section(&[(1, 0x100)]), 2); + stream.extend_from_slice(&packetize_section_with_split_syntax_header( + 0x100, + &pmt_section(1, 0x101, &[(0x1B, 0x101), (0x0F, 0x102)], 0), + 1, + )); + stream.extend_from_slice(&null_packet()); + let budget = HlsTsProbeBudget { read_chunk_bytes: 191, ..HlsTsProbeBudget::default() }; + + let signature = + found(inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, budget).expect("probe succeeds")); + + assert_eq!(signature.program_count, 1); + assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x1B]); + + let mut stream = packetize_section_with_split_syntax_header(0, &pat_section(&[(1, 0x100)]), 5); + stream.extend_from_slice(&packetize_section_with_split_syntax_header( + 0x100, + &pmt_section(1, 0x101, &[(0x1B, 0x101), (0x0F, 0x102)], 0), + 7, + )); + stream.extend_from_slice(&null_packet()); + let signature = + found(inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, budget).expect("probe succeeds")); + assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x1B]); +} + +#[test] +fn ts_inspector_accepts_pusi_pointer_that_completes_pending_syntax_header() { + let pat = pat_section(&[(1, 0x100)]); + let pmt = pmt_section(1, 0x101, &[(0x1B, 0x101), (0x0F, 0x102)], 0); + let mut stream = synthetic_psi_packet(Pid::new(0), true, &pat[..2]).to_vec(); + stream.extend_from_slice(&psi_start_packet_with_pointer(0, &pat[2..], &pat)); + stream.extend_from_slice(&synthetic_psi_packet(Pid::new(0x100), true, &pmt[..5])); + stream.extend_from_slice(&psi_start_packet_with_pointer(0x100, &pmt[5..], &pmt)); + stream.extend_from_slice(&null_packet()); + + let signature = found( + inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe succeeds"), + ); + + assert_eq!(signature.program_count, 1); + assert_eq!(signature.stream_types.as_ref(), &[0x0F, 0x1B]); +} + +#[test] +fn ts_inspector_rejects_pusi_without_section_start_bytes() { + let mut stream = psi_start_packet_with_pointer(0, &[], &[]).to_vec(); + stream.extend_from_slice(&null_packet()); + + let outcome = inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe completes"); + + assert!(matches!(outcome, HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPsiPointer))); +} + +#[tokio::test] +async fn ts_inspector_async_clear_stops_in_small_prefix_of_logical_large_segment() { + const LOGICAL_SEGMENT_BYTES: u64 = 20 * 1024 * 1024; + let prefix: Arc<[u8]> = valid_track_prefix(HLS_TS_PROBE_READ_CHUNK_BYTES).into(); + let sync_read = Arc::new(AtomicU64::new(0)); + let async_read = Arc::new(AtomicU64::new(0)); + + let sync_signature = found( + inspect_mpeg_ts( + VirtualTailReader::new(Arc::clone(&prefix), LOGICAL_SEGMENT_BYTES, Arc::clone(&sync_read)), + HlsTsProbeProtection::Clear, + HlsTsProbeBudget::default(), + ) + .expect("sync probe succeeds"), + ); + let async_signature = found( + inspect_mpeg_ts_async( + VirtualTailReader::new(prefix, LOGICAL_SEGMENT_BYTES, Arc::clone(&async_read)), + HlsTsProbeProtection::Clear, + HlsTsProbeBudget::default(), + ) + .await + .expect("async probe succeeds"), + ); + + assert_eq!(async_signature, sync_signature); + assert_eq!(async_signature.stream_types.as_ref(), &[0x0F, 0x1B]); + assert!(sync_read.load(Ordering::Relaxed) <= HLS_TS_PROBE_READ_CHUNK_BYTES as u64); + assert!(async_read.load(Ordering::Relaxed) <= HLS_TS_PROBE_READ_CHUNK_BYTES as u64); +} + +#[tokio::test] +async fn ts_inspector_async_aes_stops_in_small_prefix_of_logical_large_segment() { + const LOGICAL_SEGMENT_BYTES: u64 = 20 * 1024 * 1024; + let key = *b"0123456789abcdef"; + let iv = [0x5A; AES_128_BLOCK_BYTES]; + let mut plaintext = valid_track_prefix(HLS_TS_PROBE_READ_CHUNK_BYTES); + plaintext.resize(plaintext.len().next_multiple_of(AES_128_BLOCK_BYTES), 0xFF); + let ciphertext: Arc<[u8]> = encrypt_aes128_cbc(&plaintext, &key, iv).into(); + let sync_read = Arc::new(AtomicU64::new(0)); + let async_read = Arc::new(AtomicU64::new(0)); + + let sync_signature = found( + inspect_mpeg_ts( + VirtualTailReader::new(Arc::clone(&ciphertext), LOGICAL_SEGMENT_BYTES, Arc::clone(&sync_read)), + HlsTsProbeProtection::Aes128Cbc { key: &key, iv }, + HlsTsProbeBudget::default(), + ) + .expect("sync AES probe succeeds"), + ); + let async_signature = found( + inspect_mpeg_ts_async( + VirtualTailReader::new(ciphertext, LOGICAL_SEGMENT_BYTES, Arc::clone(&async_read)), + HlsTsProbeProtection::Aes128Cbc { key: &key, iv }, + HlsTsProbeBudget::default(), + ) + .await + .expect("async AES probe succeeds"), + ); + + assert_eq!(async_signature, sync_signature); + assert_eq!(async_signature.stream_types.as_ref(), &[0x0F, 0x1B]); + assert!(sync_read.load(Ordering::Relaxed) <= HLS_TS_PROBE_READ_CHUNK_BYTES as u64); + assert!(async_read.load(Ordering::Relaxed) <= HLS_TS_PROBE_READ_CHUNK_BYTES as u64); +} + +#[test] +fn ts_inspector_reports_probe_budget_without_reading_whole_source() { + let mut bytes = Vec::new(); + for _ in 0..100 { + bytes.extend_from_slice(&null_packet()); + } + let count = Rc::new(Cell::new(0)); + let reader = CountingReader { inner: Cursor::new(bytes), bytes_read: Rc::clone(&count) }; + let budget = HlsTsProbeBudget { max_bytes: (TS_PACKET_BYTES * 4) as u64, ..HlsTsProbeBudget::default() }; + let outcome = inspect_mpeg_ts(reader, HlsTsProbeProtection::Clear, budget).expect("probe succeeds"); + assert!(matches!(outcome, HlsTsProbeOutcome::ProbeBudgetExhausted { .. })); + assert_eq!(count.get(), TS_PACKET_BYTES * 4); +} + +#[test] +fn ts_inspector_policy_resolution_preserves_all_probe_reasons() { + assert_eq!( + HlsTrackEvidenceResolution::from(Ok(HlsTsProbeOutcome::ProbeBudgetExhausted { + bytes_examined: 512, + packets_examined: 2, + })), + HlsTrackEvidenceResolution::InsufficientEvidence { bytes_examined: 512, packets_examined: 2 } + ); + assert_eq!( + HlsTrackEvidenceResolution::from(Ok(HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPmt,))) + .reason_code(), + "invalid-pmt" + ); + assert_eq!( + HlsTrackEvidenceResolution::from(Ok(HlsTsProbeOutcome::Malformed( + HlsTsMalformedReason::IncompleteProgramMetadata, + ))), + HlsTrackEvidenceResolution::IncompleteEvidence + ); + let unsupported = HlsTrackEvidenceResolution::from(Ok(HlsTsProbeOutcome::UnsupportedProtection( + HlsTsProtectionReason::TransportScrambling, + ))); + assert_eq!( + unsupported, + HlsTrackEvidenceResolution::UnsupportedProtection(HlsTsProtectionReason::TransportScrambling) + ); + assert_eq!(unsupported.reason_code(), "transport-scrambling"); + let key_unavailable = HlsTrackEvidenceResolution::from(Err(HlsTsProbeError::KeyUnavailable)); + assert_eq!(key_unavailable, HlsTrackEvidenceResolution::KeyUnavailable); + assert_eq!(key_unavailable.reason_code(), "key-unavailable"); + let invalid_iv = HlsTrackEvidenceResolution::from(Err(HlsTsProbeError::InvalidIv)); + assert_eq!(invalid_iv, HlsTrackEvidenceResolution::InvalidIv); + assert_eq!(invalid_iv.reason_code(), "invalid-iv"); + let decryption_failed = HlsTrackEvidenceResolution::from(Err(HlsTsProbeError::DecryptionFailed)); + assert_eq!(decryption_failed, HlsTrackEvidenceResolution::DecryptionFailed); + assert_eq!(decryption_failed.reason_code(), "decryption-failed"); + let io = HlsTrackEvidenceResolution::from(Err(HlsTsProbeError::Io(std::io::Error::from( + std::io::ErrorKind::PermissionDenied, + )))); + assert_eq!(io, HlsTrackEvidenceResolution::Io(std::io::ErrorKind::PermissionDenied)); + assert_eq!(io.reason_code(), "io"); +} + +#[test] +fn ts_inspector_resynchronizes_only_within_named_budget() { + let budget = HlsTsProbeBudget { max_resync_bytes: 16, ..HlsTsProbeBudget::default() }; + let mut within_budget = vec![0xAA; 16]; + within_budget.extend_from_slice(&track_stream(0)); + assert!(matches!( + inspect_mpeg_ts(Cursor::new(within_budget), HlsTsProbeProtection::Clear, budget) + .expect("bounded resync probe completes"), + HlsTsProbeOutcome::Found(_) + )); + + let mut outside_budget = vec![0xAA; 17]; + outside_budget.extend_from_slice(&track_stream(0)); + assert_eq!( + inspect_mpeg_ts(Cursor::new(outside_budget), HlsTsProbeProtection::Clear, budget) + .expect("out-of-budget resync probe completes"), + HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidSynchronization) + ); +} + +#[test] +fn ts_inspector_types_framing_crc_and_pmt_errors() { + let invalid_sync = vec![0_u8; TS_PACKET_BYTES * 3]; + assert_eq!( + inspect_mpeg_ts(Cursor::new(invalid_sync), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe completes"), + HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidSynchronization) + ); + + let mut invalid_crc = track_stream(0); + invalid_crc[10] ^= 0x01; + assert_eq!( + inspect_mpeg_ts(Cursor::new(invalid_crc), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe completes"), + HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPsiCrc) + ); + + let mut invalid_pmt = pmt_section(1, 0x101, &[(0x1B, 0x101)], 0); + invalid_pmt[15] = 0xFF; + invalid_pmt[16] = 0xFF; + let crc_start = invalid_pmt.len().saturating_sub(4); + invalid_pmt.truncate(crc_start); + invalid_pmt = append_crc(invalid_pmt); + let mut stream = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); + stream.extend_from_slice(&packetize_section(0x100, &invalid_pmt, 0)); + stream.extend_from_slice(&null_packet()); + assert_eq!( + inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe completes"), + HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPmt) + ); + + let mut malformed_descriptors = pmt_section(1, 0x101, &[(0x1B, 0x101)], 4); + malformed_descriptors[13] = 5; + let crc_start = malformed_descriptors.len().saturating_sub(4); + malformed_descriptors.truncate(crc_start); + malformed_descriptors = append_crc(malformed_descriptors); + let mut stream = packetize_section(0, &pat_section(&[(1, 0x100)]), 0); + stream.extend_from_slice(&packetize_section(0x100, &malformed_descriptors, 0)); + stream.extend_from_slice(&null_packet()); + assert_eq!( + inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe completes"), + HlsTsProbeOutcome::Malformed(HlsTsMalformedReason::InvalidPmt) + ); +} + +#[test] +fn ts_inspector_types_scrambled_transport_as_unsupported_protection() { + let mut stream = track_stream(0); + stream[3] |= 0x80; + + assert_eq!( + inspect_mpeg_ts(Cursor::new(stream), HlsTsProbeProtection::Clear, HlsTsProbeBudget::default()) + .expect("probe completes"), + HlsTsProbeOutcome::UnsupportedProtection(HlsTsProtectionReason::TransportScrambling) + ); +} + +#[test] +fn ts_inspector_applies_explicit_and_sequence_derived_hls_ivs() { + let sequence_iv = hls_aes128_cbc_iv(None, 0x0102_0304_0506_0708).expect("sequence IV"); + assert_eq!(sequence_iv, [0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8]); + let explicit_iv = hls_aes128_cbc_iv(Some("0x10203"), 0).expect("explicit IV"); + assert_eq!(explicit_iv, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3]); + assert!(matches!(hls_aes128_cbc_iv(Some("10203"), 0), Err(HlsTsProbeError::InvalidIv))); +} diff --git a/backend/mpegts/src/ts_inspector/transport.rs b/backend/mpegts/src/ts_inspector/transport.rs new file mode 100644 index 000000000..605f9ffe9 --- /dev/null +++ b/backend/mpegts/src/ts_inspector/transport.rs @@ -0,0 +1,295 @@ +use super::{ + HlsTsCompatibleSpliceEvidence, HlsTsPidBoundaryEvidence, HlsTsPidContinuityState, HlsTsSpliceEvidence, + HlsTsSpliceIncompatibility, HlsTsTrackSignature, HlsTsTransportStreamInspector, TS_PACKET_BYTES, +}; +use mpeg2ts_reader::packet::Packet; + +pub(super) enum HlsTsPesInspectionState { + Header { bytes: [u8; 6], len: usize, started_at_packet_index: u64 }, + Finite { declared_bytes: u16, observed_bytes: u64, started_at_packet_index: u64 }, + UnboundedVideo, +} + +fn validated_adaptation_length(packet: &Packet<'_>, bytes: &[u8]) -> Result, ()> { + let adaptation_control = packet.adaptation_control(); + if !adaptation_control.has_payload() && !adaptation_control.has_adaptation_field() { + return Err(()); + } + let adaptation_length = adaptation_control.has_adaptation_field().then(|| usize::from(bytes[4])); + match (adaptation_control.has_adaptation_field(), adaptation_control.has_payload(), adaptation_length) { + (false, true, None) | (true, false, Some(183)) => Ok(adaptation_length), + (true, true, Some(length)) if length <= 182 => Ok(adaptation_length), + _ => Err(()), + } +} + +impl HlsTsTransportStreamInspector { + pub(super) fn new() -> Self { + let pid_count = usize::from(u16::from(mpeg2ts_reader::STUFFING_PID)).saturating_add(1); + Self { + continuity: vec![None; pid_count], + pes: (0..pid_count).map(|_| None).collect(), + invalid_pes_starts: vec![None; pid_count], + incompatible: None, + } + } + + pub(super) fn push_packet(&mut self, bytes: &[u8], packet_index: u64) { + if self.incompatible.is_some() { + return; + } + if bytes.len() != TS_PACKET_BYTES { + self.incompatible = Some(HlsTsSpliceIncompatibility::InvalidPacket { packet_index }); + return; + } + let Some(packet) = Packet::try_new(bytes) else { + self.incompatible = Some(HlsTsSpliceIncompatibility::InvalidPacket { packet_index }); + return; + }; + let pid = u16::from(packet.pid()); + if packet.transport_error_indicator() { + self.incompatible = Some(HlsTsSpliceIncompatibility::TransportError { pid, packet_index }); + return; + } + if packet.transport_scrambling_control().is_scrambled() { + self.incompatible = Some(HlsTsSpliceIncompatibility::InvalidPacket { packet_index }); + return; + } + let Ok(adaptation_length) = validated_adaptation_length(&packet, bytes) else { + self.incompatible = Some(HlsTsSpliceIncompatibility::InvalidPacket { packet_index }); + return; + }; + let has_payload = packet.adaptation_control().has_payload(); + let discontinuity = adaptation_length.is_some_and(|length| length > 0 && bytes[5] & 0x80 != 0); + let continuity_counter = packet.continuity_counter().count(); + if let Err(reason) = self.inspect_continuity(pid, packet_index, continuity_counter, has_payload, discontinuity) + { + self.incompatible = Some(reason); + return; + } + if discontinuity { + if let Some(reason) = self.take_discontinuity_pes_incompatibility(pid) { + self.incompatible = Some(reason); + return; + } + } + let Some(payload) = packet.payload() else { + return; + }; + let previous = self.pes[usize::from(pid)].take(); + match advance_ts_pes_inspection( + pid, + packet.payload_unit_start_indicator(), + payload, + packet_index, + previous, + &mut self.invalid_pes_starts[usize::from(pid)], + ) { + Ok(next) => self.pes[usize::from(pid)] = next, + Err(reason) => self.incompatible = Some(reason), + } + } + + pub(super) fn inspect_continuity( + &mut self, + pid: u16, + packet_index: u64, + continuity_counter: u8, + has_payload: bool, + discontinuity: bool, + ) -> Result<(), HlsTsSpliceIncompatibility> { + if pid == u16::from(mpeg2ts_reader::STUFFING_PID) { + return Ok(()); + } + let state = &mut self.continuity[usize::from(pid)]; + let Some(previous) = state else { + *state = Some(HlsTsPidContinuityState { + first_packet_index: packet_index, + first_continuity_counter: continuity_counter, + first_has_payload: has_payload, + first_discontinuity: discontinuity, + last_continuity_counter: continuity_counter, + }); + return Ok(()); + }; + let expected = if has_payload { + previous.last_continuity_counter.wrapping_add(1) & 0x0F + } else { + previous.last_continuity_counter + }; + if !discontinuity && continuity_counter != expected { + return Err(HlsTsSpliceIncompatibility::ContinuityFailure { + pid, + packet_index, + expected, + actual: continuity_counter, + }); + } + previous.last_continuity_counter = continuity_counter; + Ok(()) + } + + pub(super) fn take_discontinuity_pes_incompatibility(&mut self, pid: u16) -> Option { + match self.pes[usize::from(pid)].take() { + Some(HlsTsPesInspectionState::Header { started_at_packet_index, .. }) => { + Some(HlsTsSpliceIncompatibility::IncompletePes { + pid, + packet_index: started_at_packet_index, + declared_bytes: None, + observed_bytes: 0, + }) + } + Some(HlsTsPesInspectionState::Finite { declared_bytes, observed_bytes, started_at_packet_index }) => { + Some(HlsTsSpliceIncompatibility::IncompletePes { + pid, + packet_index: started_at_packet_index, + declared_bytes: Some(declared_bytes), + observed_bytes, + }) + } + Some(HlsTsPesInspectionState::UnboundedVideo) | None => None, + } + } + + pub(super) fn finish(mut self, topology: Option) -> HlsTsSpliceEvidence { + if let Some(reason) = self.incompatible { + return HlsTsSpliceEvidence::Incompatible(reason); + } + for (pid, state) in self.pes.drain(..).enumerate() { + let Some(state) = state else { + continue; + }; + let pid = u16::try_from(pid).unwrap_or(u16::MAX); + match state { + HlsTsPesInspectionState::Header { started_at_packet_index, .. } => { + return HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::IncompletePes { + pid, + packet_index: started_at_packet_index, + declared_bytes: None, + observed_bytes: 0, + }); + } + HlsTsPesInspectionState::Finite { declared_bytes, observed_bytes, started_at_packet_index } => { + return HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::IncompletePes { + pid, + packet_index: started_at_packet_index, + declared_bytes: Some(declared_bytes), + observed_bytes, + }); + } + HlsTsPesInspectionState::UnboundedVideo => {} + } + } + let Some(topology) = topology else { + return HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::TopologyUnavailable); + }; + for stream in topology.programs.iter().flat_map(|program| program.streams.iter()) { + if let Some(packet_index) = self.invalid_pes_starts[usize::from(stream.elementary_pid)] { + return HlsTsSpliceEvidence::Incompatible(HlsTsSpliceIncompatibility::InvalidPes { + pid: stream.elementary_pid, + packet_index, + }); + } + } + let pid_boundaries = self + .continuity + .into_iter() + .enumerate() + .filter_map(|(pid, state)| { + let state = state?; + Some(HlsTsPidBoundaryEvidence { + pid: u16::try_from(pid).unwrap_or(u16::MAX), + first_packet_index: state.first_packet_index, + first_continuity_counter: state.first_continuity_counter, + first_has_payload: state.first_has_payload, + first_discontinuity: state.first_discontinuity, + last_continuity_counter: state.last_continuity_counter, + }) + }) + .collect::>(); + HlsTsSpliceEvidence::Compatible(HlsTsCompatibleSpliceEvidence { + topology, + pid_boundaries: pid_boundaries.into(), + }) + } +} + +fn advance_ts_pes_inspection( + pid: u16, + payload_unit_start: bool, + payload: &[u8], + packet_index: u64, + previous: Option, + invalid_pes_start: &mut Option, +) -> Result, HlsTsSpliceIncompatibility> { + let state = if payload_unit_start { + match previous { + Some(HlsTsPesInspectionState::Header { started_at_packet_index, .. }) => { + return Err(HlsTsSpliceIncompatibility::IncompletePes { + pid, + packet_index: started_at_packet_index, + declared_bytes: None, + observed_bytes: 0, + }); + } + Some(HlsTsPesInspectionState::Finite { declared_bytes, observed_bytes, started_at_packet_index }) => { + return Err(HlsTsSpliceIncompatibility::IncompletePes { + pid, + packet_index: started_at_packet_index, + declared_bytes: Some(declared_bytes), + observed_bytes, + }); + } + Some(HlsTsPesInspectionState::UnboundedVideo) | None => {} + } + Some(HlsTsPesInspectionState::Header { bytes: [0; 6], len: 0, started_at_packet_index: packet_index }) + } else { + previous + }; + let (mut bytes, mut len, started_at_packet_index) = match state { + Some(HlsTsPesInspectionState::Header { bytes, len, started_at_packet_index }) => { + (bytes, len, started_at_packet_index) + } + Some(HlsTsPesInspectionState::Finite { declared_bytes, observed_bytes, started_at_packet_index }) => { + let available = u64::try_from(payload.len()).unwrap_or(u64::MAX); + let observed_bytes = observed_bytes.saturating_add(available).min(u64::from(declared_bytes)); + return if observed_bytes == u64::from(declared_bytes) { + Ok(None) + } else { + Ok(Some(HlsTsPesInspectionState::Finite { declared_bytes, observed_bytes, started_at_packet_index })) + }; + } + Some(HlsTsPesInspectionState::UnboundedVideo) => { + return Ok(Some(HlsTsPesInspectionState::UnboundedVideo)); + } + None => return Ok(None), + }; + let copied = payload.len().min(6usize.saturating_sub(len)); + bytes[len..len.saturating_add(copied)].copy_from_slice(&payload[..copied]); + len = len.saturating_add(copied); + if len >= 3 && bytes[..3] != [0, 0, 1] { + if invalid_pes_start.is_none() { + *invalid_pes_start = Some(started_at_packet_index); + } + return Ok(None); + } + if len < 6 { + return Ok(Some(HlsTsPesInspectionState::Header { bytes, len, started_at_packet_index })); + } + let stream_id = bytes[3]; + let declared_bytes = u16::from_be_bytes([bytes[4], bytes[5]]); + if declared_bytes == 0 { + return if (0xE0..=0xEF).contains(&stream_id) { + Ok(Some(HlsTsPesInspectionState::UnboundedVideo)) + } else { + Err(HlsTsSpliceIncompatibility::InvalidPes { pid, packet_index }) + }; + } + let observed_bytes = + u64::try_from(payload.len().saturating_sub(copied)).unwrap_or(u64::MAX).min(u64::from(declared_bytes)); + if observed_bytes == u64::from(declared_bytes) { + Ok(None) + } else { + Ok(Some(HlsTsPesInspectionState::Finite { declared_bytes, observed_bytes, started_at_packet_index })) + } +} diff --git a/backend/parser/src/m3u_format/catchup.rs b/backend/parser/src/m3u_format/catchup.rs index aeadbc041..8cfc79cfd 100644 --- a/backend/parser/src/m3u_format/catchup.rs +++ b/backend/parser/src/m3u_format/catchup.rs @@ -803,7 +803,7 @@ mod tests { let expected = resolve_m3u_catchup_url(live_url, &catchup, Some(&format!("v0={start}&v1=14400")))?; for query in [ format!("utc={start}&lutc={end}"), - format!("utc={start}&lutc={}&duration=99999999", start + 265000), + format!("utc={start}&lutc={}&duration=99999999", start + 265_000), format!("v0={start}&v1=864000"), ] { let resolved = resolve_m3u_catchup_url(live_url, &catchup, Some(&query))?; diff --git a/backend/processing/src/parser/xmltv.rs b/backend/processing/src/parser/xmltv.rs index eb0abe10e..df4cdb0c7 100644 --- a/backend/processing/src/parser/xmltv.rs +++ b/backend/processing/src/parser/xmltv.rs @@ -1,34 +1,9 @@ -use crate::processor::EpgIdCache; -use log::error; -use quick_xml::events::{BytesStart, BytesText, Event}; -use serde::{Deserialize, Serialize}; -use shared::{ - concat_string, - model::{EpgCategory, EpgChannel, EpgNamePrefix, EpgProgramme}, - utils::{deunicode_string, Internable, CONSTANTS}, -}; -use std::{ - borrow::Cow, - collections::{HashMap, HashSet}, - io, - path::PathBuf, - sync::Arc, -}; -use tokio::io::AsyncRead; +use shared::model::{EpgChannel, EpgProgramme}; +use std::{collections::HashMap, path::PathBuf, sync::Arc}; use tuliprox_core::{ - model::{ - Epg, EpgSmartMatchConfig, IcsDummyPolicy, IcsEpgSourceConfig, PersistedEpgSource, PersistedEpgSourceKind, - XmlTag, XmlTagIcon, EPG_ATTRIB_CHANNEL, EPG_ATTRIB_ID, EPG_ATTRIB_LANG, EPG_TAG_CATEGORY, EPG_TAG_CHANNEL, - EPG_TAG_DESC, EPG_TAG_DISPLAY_NAME, EPG_TAG_ICON, EPG_TAG_LIVE, EPG_TAG_NEW, EPG_TAG_PREVIOUSLY_SHOWN, - EPG_TAG_PROGRAMME, EPG_TAG_TITLE, EPG_TAG_TV, - }, - utils::{ - arc_str_serde, async_file_reader, compressed_file_reader_async::CompressedFileReaderAsync, parse_xmltv_time, - with_folded_epg_id, FileLockManager, - }, + model::{IcsDummyPolicy, IcsEpgSourceConfig, PersistedEpgSource}, + utils::FileLockManager, }; -use tuliprox_parser::ics; -use tuliprox_repository::{BPlusTree, BPlusTreeQuery, BPlusTreeUpdate, FlushPolicy}; struct IcsPersistedSource<'a> { channel_id: &'a Arc, @@ -37,101 +12,6 @@ struct IcsPersistedSource<'a> { config: &'a IcsEpgSourceConfig, } -/// Splits a string at the first delimiter if the prefix matches a known country code. -/// -/// Returns a tuple containing the country code prefix (if found) and the remainder of the string, both trimmed. If no valid prefix is found, returns `None` and the original input. -/// -/// # Examples -/// -/// ```text -/// let delimiters = vec!['.', '-', '_']; -/// let (prefix, rest) = split_by_first_match("US.HBO", &delimiters); -/// assert_eq!(prefix, Some("US")); -/// assert_eq!(rest, "HBO"); -/// -/// let (prefix, rest) = split_by_first_match("HBO", &delimiters); -/// assert_eq!(prefix, None); -/// assert_eq!(rest, "HBO"); -/// ``` -fn split_by_first_match<'a>(input: &'a str, delimiters: &[char]) -> (Option<&'a str>, &'a str) { - let content = input.trim_start_matches(|c: char| !c.is_alphanumeric()); - - for delim in delimiters { - if let Some(index) = content.find(*delim) { - let (left, right) = content.split_at(index); - let right = &right[delim.len_utf8()..].trim(); - if !right.is_empty() { - let prefix = left.trim(); - if CONSTANTS.country_codes.contains(&prefix) { - return (Some(prefix), right.trim()); - } - } - } - } - (None, input) -} - -fn name_prefix<'a>(name: &'a str, smart_config: &EpgSmartMatchConfig) -> (&'a str, Option<&'a str>) { - if smart_config.name_prefix != EpgNamePrefix::Ignore { - let (prefix, suffix) = split_by_first_match(name, &smart_config.name_prefix_separator); - if prefix.is_some() { - return (suffix, prefix); - } - } - (name, None) -} - -fn combine(join: &str, left: &str, right: &str) -> String { - let mut combined = String::with_capacity(left.len() + join.len() + right.len()); - combined.push_str(left); - combined.push_str(join); - combined.push_str(right); - combined -} - -fn strip_markers(input: &str, markers: &[String]) -> String { - let mut output = String::with_capacity(input.len()); - let mut cursor = 0; - while cursor < input.len() { - let left_is_boundary = cursor == 0 || !input.as_bytes()[cursor - 1].is_ascii_alphanumeric(); - let matched_end = left_is_boundary - .then(|| { - markers.iter().filter(|marker| !marker.is_empty()).find_map(|marker| { - let end = cursor + marker.len(); - let candidate = input.get(cursor..end)?; - (candidate.eq_ignore_ascii_case(marker) - && input.as_bytes().get(end).is_none_or(|byte| !byte.is_ascii_alphanumeric())) - .then_some(end) - }) - }) - .flatten(); - if let Some(end) = matched_end { - cursor = end; - } else { - let next = input[cursor..].chars().next().expect("cursor is within the string"); - output.push(next); - cursor += next.len_utf8(); - } - } - output -} - -/// # Panics -pub fn normalize_channel_name(name: &str, normalize_config: &EpgSmartMatchConfig) -> String { - let normalized = deunicode_string(name.trim()).to_lowercase(); - let (channel_name, suffix) = name_prefix(&normalized, normalize_config); - let stripped_name = strip_markers(channel_name, &normalize_config.strip); - let reconstructed = match suffix { - None => stripped_name, - Some(sfx) => match &normalize_config.name_prefix { - EpgNamePrefix::Ignore => stripped_name, - EpgNamePrefix::Suffix(separator) => combine(separator, &stripped_name, sfx), - EpgNamePrefix::Prefix(separator) => combine(separator, sfx, &stripped_name), - }, - }; - normalize_config.normalize_regex.replace_all(&reconstructed, "").into_owned() -} - /// A set of persisted EPG sources, read and merged into one guide. /// /// The type lives beside the XMLTV parsing that gives it behaviour: its @@ -162,525 +42,6 @@ impl TVGuide { pub fn get_file_locks(&self) -> Option<&FileLockManager> { self.file_locks.as_deref() } } -impl TVGuide { - fn prepare_tag(id_cache: &mut EpgIdCache, tag: &mut XmlTag, smart_match: bool) { - { - let maybe_epg_id = { tag.get_attribute_value(&EPG_ATTRIB_ID.intern()).cloned() }; - if let Some(epg_id) = maybe_epg_id { - tag.normalized_epg_ids - .get_or_insert_with(Vec::new) - .push(normalize_channel_name(&epg_id, &id_cache.smart_match_config).intern()); - } - } - - if let Some(children) = &tag.children { - let src = "src".intern(); - for child in children { - match child.name.as_ref() { - EPG_TAG_DISPLAY_NAME if smart_match => { - if let Some(name) = &child.value { - tag.normalized_epg_ids - .get_or_insert_with(Vec::new) - .push(normalize_channel_name(name, &id_cache.smart_match_config).intern()); - } - } - EPG_TAG_ICON => { - if let Some(src) = child.get_attribute_value(&src) { - if !src.is_empty() { - tag.icon = XmlTagIcon::Src(src.clone()); - // We cannot easily modify the child icon since it's inside Arc, - // but we already set the tag.icon, which is what matters. - } - } - } - _ => {} - } - } - } - } - - fn channel_display_name(tag: &XmlTag) -> Option> { - tag.children.as_ref().and_then(|children| { - children - .iter() - .find(|child| child.name.as_ref() == EPG_TAG_DISPLAY_NAME) - .and_then(|child| child.value.clone()) - }) - } - - fn channel_icon(tag: &XmlTag) -> Option> { - match &tag.icon { - XmlTagIcon::Src(src) => Some(Arc::clone(src)), - XmlTagIcon::Undefined | XmlTagIcon::Exists => None, - } - } - - fn extract_programme( - tag: &XmlTag, - epg_id: &Arc, - start_attrib: &Arc, - stop_attrib: &Arc, - catchup_id_attrib: &Arc, - ) -> Option { - let Some((Some(start), Some(stop))) = - tag.attributes.as_ref().map(|a| (a.get(start_attrib), a.get(stop_attrib))) - else { - error!("Missing start or stop attribute in programme tag, skipping"); - return None; - }; - - let (Some(start_time), Some(stop_time)) = (parse_xmltv_time(start), parse_xmltv_time(stop)) else { - error!("Failed to parse epg programme time {start} - {stop}"); - return None; - }; - - let mut title = None; - let mut desc = None; - let mut icon = None; - let mut categories = Vec::new(); - let mut is_live = false; - let mut is_new = false; - let mut previously_shown = false; - if let Some(children) = tag.children.as_ref() { - for child in children { - match child.name.as_ref() { - EPG_TAG_TITLE => title.clone_from(&child.value), - EPG_TAG_DESC => desc.clone_from(&child.value), - EPG_TAG_ICON => { - if let Some(src) = child - .attributes - .as_ref() - .and_then(|attributes| attributes.get("src")) - .filter(|src| !src.is_empty()) - { - icon = Some(Arc::clone(src)); - } - } - EPG_TAG_CATEGORY => { - if let Some(value) = child.value.as_ref().filter(|value| !value.is_empty()) { - categories.push(EpgCategory { - value: Arc::clone(value), - lang: child - .attributes - .as_ref() - .and_then(|attributes| attributes.get(EPG_ATTRIB_LANG)) - .cloned(), - }); - } - } - EPG_TAG_LIVE => is_live = true, - EPG_TAG_NEW => is_new = true, - EPG_TAG_PREVIOUSLY_SHOWN => previously_shown = true, - _ => {} - } - } - } - - let catchup_id = tag.attributes.as_ref().and_then(|attributes| attributes.get(catchup_id_attrib)).cloned(); - - let mut programme = EpgProgramme::new_all(start_time, stop_time, Arc::clone(epg_id), title, desc, catchup_id); - programme.icon = icon; - programme.categories = categories; - programme.is_live = is_live; - programme.is_new = is_new; - programme.previously_shown = previously_shown; - Some(programme) - } - - /// Parses and filters a compressed EPG XML file, extracting relevant channel and program tags based on smart and fuzzy matching criteria. - /// - /// Returns an `Epg` containing filtered tags and TV attributes if any matching channels are found; otherwise, returns `None`. - /// The returned `Epg` will include the priority from the source, which is used for merging multiple EPG sources. - /// - /// # Examples - /// - /// ```text - /// let mut id_cache = EpgIdCache::default(); - /// let epg_source = PersistedEpgSource { file_path: Path::new("guide.xml.gz"), priority: 0 }; - /// if let Some(epg) = process_epg_file(&mut id_cache, &epg_source) { - /// assert!(!epg.children.is_empty()); - /// } - /// ``` - async fn process_epg_file( - id_cache: &mut EpgIdCache, - epg_source: &PersistedEpgSource, - source_order: usize, - accumulator: &mut EpgMergeAccumulator, - ) -> bool { - let epg_attrib_id = EPG_ATTRIB_ID.intern(); - let epg_attrib_channel = EPG_ATTRIB_CHANNEL.intern(); - let start_attrib = "start".intern(); - let stop_attrib = "stop".intern(); - let catchup_id_attrib = "catchup-id".intern(); - - match CompressedFileReaderAsync::new(&epg_source.file_path).await { - Ok(mut reader) => { - let mut source_processed: HashSet> = HashSet::with_capacity(5000); - let mut accepted_channels = 0usize; - let smart_match = id_cache.smart_match_config.enabled; - let mut filter_tags = |mut tag: XmlTag| { - match tag.name.as_ref() { - EPG_TAG_CHANNEL => { - let tag_epg_id = - tag.get_attribute_value(&epg_attrib_id).map_or_else(|| "".intern(), Internable::intern); - if tag_epg_id.is_empty() { - return; - } - - Self::prepare_tag(id_cache, &mut tag, smart_match); - if smart_match && id_cache.needs_guide_names(&tag_epg_id) { - id_cache.register_guide_names( - &tag_epg_id, - tag.children.iter().flatten().filter_map(|child| { - (child.name.as_ref() == EPG_TAG_DISPLAY_NAME) - .then_some(child.value.as_ref()) - .flatten() - }), - ); - } - // Case-insensitive (ASCII) membership: fold the guide id for the - // lookup only; `tag_epg_id` keeps its original case for output. - let add_channel = if smart_match { - let direct_match = id_cache.contains_channel_epg_id(&tag_epg_id); - let normalized_match = tag.normalized_epg_ids.as_ref().is_some_and(|candidates| { - id_cache.match_epg_channel_candidates( - &tag_epg_id, - candidates, - epg_source.priority, - source_order, - ) - }); - direct_match || normalized_match - } else { - id_cache.contains_channel_epg_id(&tag_epg_id) - }; - - if add_channel { - with_folded_epg_id(&tag_epg_id, |folded| source_processed.insert(folded.intern())); - id_cache.insert_processed_epg_id(&tag_epg_id); - accumulator.upsert_channel( - epg_source.priority, - source_order, - epg_source.logo_override, - EpgChannel { - id: Arc::clone(&tag_epg_id), - title: Self::channel_display_name(&tag), - icon: Self::channel_icon(&tag), - programmes: vec![], - }, - ); - accepted_channels += 1; - } - } - EPG_TAG_PROGRAMME => { - if let Some(epg_id) = tag.get_attribute_value(&epg_attrib_channel) { - if with_folded_epg_id(epg_id, |folded| source_processed.contains(folded)) { - if let Some(programme) = Self::extract_programme( - &tag, - epg_id, - &start_attrib, - &stop_attrib, - &catchup_id_attrib, - ) { - accumulator.push_programme(epg_source.priority, source_order, programme); - } - } - } - } - EPG_TAG_TV => { - accumulator.set_attributes_if_preferred(epg_source.priority, source_order, tag.attributes); - } - _ => {} - } - }; - - parse_tvguide(&mut reader, &mut filter_tags).await; - accepted_channels > 0 - } - Err(e) => { - log::warn!("Failed to process EPG file {}: {e}", epg_source.file_path.display()); - false - } - } - } - - async fn process_ics_file( - id_cache: &mut EpgIdCache, - epg_source: &PersistedEpgSource, - source_order: usize, - accumulator: &mut EpgMergeAccumulator, - source: IcsPersistedSource<'_>, - ) -> bool { - let IcsPersistedSource { channel_id, channel_title, match_names, config } = source; - let mut candidates = Vec::with_capacity(2 + match_names.len()); - candidates.push(channel_id.to_string()); - if let Some(title) = channel_title { - candidates.push(title.to_string()); - } - candidates.extend(match_names.iter().map(ToString::to_string)); - let normalized_candidates = id_cache.normalize_candidates(candidates); - - let add_channel = if id_cache.smart_match_enabled { - let direct_match = id_cache.contains_channel_epg_id(channel_id); - let normalized_match = id_cache.match_epg_channel_candidates( - channel_id, - &normalized_candidates, - epg_source.priority, - source_order, - ); - direct_match || normalized_match - } else { - id_cache.contains_channel_epg_id(channel_id) - }; - - if !add_channel { - return false; - } - - match ics::parse_ics_file_to_channel( - &epg_source.file_path, - Arc::clone(channel_id), - channel_title.cloned(), - config, - ) - .await - { - Ok(channel) => { - id_cache.insert_processed_epg_id(channel_id); - accumulator.add_channel_with_programmes( - epg_source.priority, - source_order, - epg_source.logo_override, - channel, - ); - accumulator.register_dummy_policy( - channel_id, - epg_source.priority, - source_order, - IcsDummyPolicy { timezone: config.timezone.clone(), config: config.dummy.clone() }, - ); - true - } - Err(err) => { - log::warn!("Failed to process ICS EPG file {}: {err}", epg_source.file_path.display()); - false - } - } - } - - // Exercised only by this module's own tests. - #[cfg(test)] - pub async fn filter(&self, id_cache: &mut EpgIdCache) -> Option> { - self.filter_merged(id_cache).await.map(|epg| vec![epg]) - } - - // Exercised only by this module's own tests. - #[cfg(test)] - pub async fn filter_merged(&self, id_cache: &mut EpgIdCache) -> Option { - self.filter_merged_with_icon_overrides(id_cache).await.map(|(epg, _)| epg) - } - - pub async fn filter_merged_with_icon_overrides( - &self, - id_cache: &mut EpgIdCache, - ) -> Option { - if id_cache.channel_epg_id.is_empty() && id_cache.normalized.is_empty() { - return None; - } - let mut accumulator = EpgMergeAccumulator::new(); - for (source_order, epg_source) in self.get_epg_sources().iter().enumerate() { - let _source_read_lock = match self.get_file_locks() { - Some(file_locks) => Some(file_locks.read_lock(&epg_source.file_path).await), - None => None, - }; - match &epg_source.kind { - PersistedEpgSourceKind::Xmltv => { - Self::process_epg_file(id_cache, epg_source, source_order, &mut accumulator).await; - } - PersistedEpgSourceKind::Ics { channel_id, channel_title, match_names, config } => { - Self::process_ics_file( - id_cache, - epg_source, - source_order, - &mut accumulator, - IcsPersistedSource { - channel_id, - channel_title: channel_title.as_ref(), - match_names, - config: config.as_ref(), - }, - ) - .await; - } - } - } - let available_epg_ids = accumulator.channel_ids_with_programmes(); - id_cache.finalize_matches(&available_epg_ids); - let mut selected_epg_ids = id_cache.selected_epg_ids(&available_epg_ids); - selected_epg_ids.extend(id_cache.channel_epg_id.iter().cloned()); - let retained_epg_ids = accumulator.retain_channels(&selected_epg_ids); - id_cache.replace_processed_epg_ids(retained_epg_ids.intersection(&available_epg_ids).cloned().collect()); - accumulator.finish_epg_with_icon_overrides() - } -} - -fn handle_tag_start(callback: &mut F, stack: &mut Vec, e: &BytesStart) -where - F: FnMut(XmlTag), -{ - let binding = e.name(); - let name_raw = String::from_utf8_lossy(binding.as_ref()); - let name = name_raw.intern(); - let tag_type = get_tag_type(&name); - let attributes = collect_tag_attributes(e); - let attribs = if attributes.is_empty() { None } else { Some(attributes) }; - let tag = XmlTag::new(name, attribs); - - if tag_type.is_tv() { - callback(tag); - } else { - stack.push(tag); - } -} - -fn handle_tag_end(callback: &mut F, stack: &mut Vec) -where - F: FnMut(XmlTag), -{ - if !stack.is_empty() { - if let Some(tag) = stack.pop() { - if tag.name.as_ref() == EPG_TAG_CHANNEL { - if let Some(chan_id) = tag.get_attribute_value(&EPG_ATTRIB_ID.intern()) { - if !chan_id.is_empty() { - callback(tag); - } - } - } else if tag.name.as_ref() == EPG_TAG_PROGRAMME { - if let Some(chan_id) = tag.get_attribute_value(&EPG_ATTRIB_CHANNEL.intern()) { - if !chan_id.is_empty() { - callback(tag); - } - } - } else if !stack.is_empty() { - let tag_arc = Arc::new(tag); - if let Some(mut parent) = stack.pop() { - parent.children.get_or_insert_with(Vec::new).push(tag_arc); - stack.push(parent); - } - } - } - } -} - -fn handle_text_tag(stack: &mut [XmlTag], e: &BytesText) { - if let Some(tag) = stack.last_mut() { - if let Ok(text) = e.decode() { - let t = text.trim(); - if !t.is_empty() { - let t_fixed: Cow = if t.ends_with('\\') { - let mut owned = t.to_string(); - owned.pop(); - owned.push_str("' "); - Cow::Owned(owned) - } else { - Cow::Borrowed(t) - }; - - tag.value = Some(match tag.value.take() { - None => t_fixed.intern(), - Some(old) => concat_string!(old.as_ref(), t_fixed.as_ref()).intern(), - }); - } - } - } -} - -pub async fn parse_tvguide(content: R, callback: &mut F) -where - R: AsyncRead + Unpin, - F: FnMut(XmlTag), -{ - let mut stack: Vec = vec![]; - let mut xml_reader = quick_xml::reader::Reader::from_reader(async_file_reader(content)); - // Pre-allocate so the first giant `` block does not trigger a - // chain of `Vec::grow` reallocs. The buffer is monotonically grown by - // quick_xml (it is reset by the caller, see below), so the eventual size - // is the largest single event — for XMLTV that is the biggest programme - // description. Starting at 0 capacity makes that growth ~25 doubling copies. - let mut buf = Vec::::with_capacity(64 * 1024); - loop { - match xml_reader.read_event_into_async(&mut buf).await { - Ok(Event::Eof) => break, - Ok(Event::Start(e)) => handle_tag_start(callback, &mut stack, &e), - Ok(Event::Empty(e)) => { - handle_tag_start(callback, &mut stack, &e); - handle_tag_end(callback, &mut stack); - } - Ok(Event::End(_e)) => handle_tag_end(callback, &mut stack), - Ok(Event::Text(e)) => handle_text_tag(&mut stack, &e), - _ => {} - } - // quick_xml does not clear the buffer between events — the borrow - // returned by `read_event_into_async` is dropped at the end of each - // match arm, so we can reclaim the capacity here without invalidating - // any handler output. Without this, `buf` grows monotonically over the - // whole file, defeating the 64 KiB pre-allocation above. - buf.clear(); - } -} - -/// Channels per `BPlusTree` write batch. Sized so the in-flight batch + its -/// prepared references stay under 1 MiB on the typical 100-KiB-per-channel -/// EPG feed. This is an educated guess; profile-driven tuning is fine. -const EPG_DISK_BATCH_SIZE: usize = 100; - -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] -enum XmlTagType { - Ignored, - Tv, - Channel, - Programme, -} - -impl XmlTagType { - #[inline] - pub fn is_tv(self) -> bool { self == XmlTagType::Tv } -} - -fn get_tag_type(name: &str) -> XmlTagType { - match name { - EPG_TAG_TV => XmlTagType::Tv, - EPG_TAG_CHANNEL => XmlTagType::Channel, - EPG_TAG_PROGRAMME => XmlTagType::Programme, - _ => XmlTagType::Ignored, - } -} - -fn collect_tag_attributes(e: &BytesStart) -> HashMap, Arc> { - let attributes = e - .attributes() - .filter_map(Result::ok) - .filter_map(|a| { - let key_binding = a.key; - let key_raw = String::from_utf8_lossy(key_binding.as_ref()); - let key = key_raw.intern(); - if let Ok(value) = a.normalized_value(quick_xml::XmlVersion::Implicit1_0).as_ref() { - if value.is_empty() { - None - } else { - // Ids are no longer lowercased when parsed; EPG matching folds case - // at the comparison instead, so the guide's original-case - // and are preserved in the output. - Some((key, value.intern())) - } - } else { - None - } - }) - .collect::, Arc>>(); - attributes -} - #[derive(Debug)] struct PreferredAttributes { priority: i16, @@ -695,18 +56,6 @@ struct PreferredDummyPolicy { policy: IcsDummyPolicy, } -/// Carries the source rank required to select a preview dummy policy exactly like the main EPG merge. -#[derive(Debug)] -pub struct EpgDummyPolicySource { - pub priority: i16, - pub source_order: usize, - pub channel_id: Arc, - pub policy: IcsDummyPolicy, -} - -type FinishedEpgChannels = (Option, Arc>>, Vec); -pub type MergedEpgWithIconOverrides = (Epg, HashSet>); - #[derive(Debug)] struct ProgrammeMergeEntry { priority: i16, @@ -732,1587 +81,30 @@ pub struct EpgMergeAccumulator { dummy_policies: HashMap, PreferredDummyPolicy>, } -impl EpgMergeAccumulator { - pub fn new() -> Self { Self::default() } - - fn channel_ids_with_programmes(&self) -> HashSet> { - self.channels - .iter() - .filter(|(id, channel)| !channel.programmes.is_empty() || self.dummy_policies.contains_key(*id)) - .map(|(id, _)| Arc::clone(id)) - .collect() - } - - fn retain_channels(&mut self, selected_epg_ids: &HashSet>) -> HashSet> { - self.channels.retain(|id, _| selected_epg_ids.contains(id)); - self.dummy_policies.retain(|id, _| selected_epg_ids.contains(id)); - self.channels.keys().cloned().collect() - } - - pub fn set_attributes_if_preferred( - &mut self, - priority: i16, - source_order: usize, - attributes: Option, Arc>>, - ) { - let Some(attributes) = attributes else { - return; - }; - let replace = self - .attributes - .as_ref() - .is_none_or(|current| (priority, source_order) < (current.priority, current.source_order)); - if replace { - self.attributes = Some(PreferredAttributes { priority, source_order, attributes }); - } - } - - fn upsert_channel(&mut self, priority: i16, source_order: usize, logo_override: bool, mut channel: EpgChannel) { - let channel_key = with_folded_epg_id(&channel.id, |folded| folded.intern()); - match self.channels.entry(channel_key) { - std::collections::hash_map::Entry::Occupied(mut entry) => { - let acc = entry.get_mut(); - acc.needs_programme_merge = true; - if (priority, source_order) < (acc.priority, acc.source_order) { - if channel.title.is_none() { - channel.title = acc.channel.title.take(); - } - if channel.icon.is_none() { - channel.icon = acc.channel.icon.take(); - } else { - acc.icon_logo_override = logo_override; - } - acc.priority = priority; - acc.source_order = source_order; - acc.logo_override = logo_override; - acc.channel.title = channel.title; - acc.channel.icon = channel.icon; - } else { - if acc.channel.title.is_none() { - acc.channel.title = channel.title.take(); - } - if acc.channel.icon.is_none() { - acc.channel.icon = channel.icon.take(); - acc.icon_logo_override = logo_override; - } - } - } - std::collections::hash_map::Entry::Vacant(entry) => { - let icon_logo_override = channel.icon.is_some() && logo_override; - entry.insert(ChannelMergeAcc { - priority, - source_order, - logo_override, - icon_logo_override, - needs_programme_merge: false, - channel, - programmes: Vec::new(), - }); - } - } - } - - fn push_programme(&mut self, priority: i16, source_order: usize, programme: EpgProgramme) { - if let Some(channel) = - with_folded_epg_id(programme.get_transient_channel_id(), |folded| self.channels.get_mut(folded)) - { - channel.programmes.push(ProgrammeMergeEntry { priority, source_order, programme }); - } else { - error!("Channel {} not found in EPG, dangling programme", programme.get_transient_channel_id()); - } - } - - fn register_dummy_policy( - &mut self, - channel_id: &Arc, - priority: i16, - source_order: usize, - policy: IcsDummyPolicy, - ) { - if !policy.config.enabled { - return; - } - let key = with_folded_epg_id(channel_id, |folded| folded.intern()); - let replace = self - .dummy_policies - .get(&key) - .is_none_or(|current| (priority, source_order) < (current.priority, current.source_order)); - if replace { - self.dummy_policies.insert(key, PreferredDummyPolicy { priority, source_order, policy }); - } - } - - pub fn add_channel_with_programmes( - &mut self, - priority: i16, - source_order: usize, - logo_override: bool, - mut channel: EpgChannel, - ) { - let programmes = std::mem::take(&mut channel.programmes); - let channel_key = with_folded_epg_id(&channel.id, |folded| folded.intern()); - self.upsert_channel(priority, source_order, logo_override, channel); - if let Some(acc) = self.channels.get_mut(&channel_key) { - if !acc.programmes.is_empty() { - acc.needs_programme_merge = true; - } - acc.programmes.extend(programmes.into_iter().map(|programme| ProgrammeMergeEntry { - priority, - source_order, - programme, - })); - } - } - - fn finish_channels(mut self) -> Option { - if self.channels.is_empty() { - return None; - } - - let mut channels = self - .channels - .drain() - .map(|(_, mut acc)| { - normalize_channel_programmes(&mut acc); - acc - }) - .collect::>(); - - channels.sort_by(|left, right| left.channel.id.cmp(&right.channel.id)); - let channels = channels.into_iter().map(|acc| acc.channel).collect(); - Some((self.attributes.map(|attributes| attributes.attributes), channels)) - } - - fn finish(self) -> Option { - self.finish_channels().map(|(attributes, channels)| Epg { - logo_override: false, - priority: 0, - attributes, - children: channels.into_iter().map(Arc::new).collect(), - }) - } - - fn finish_epg_with_icon_overrides(self) -> Option { - let EpgMergeAccumulator { attributes, channels, dummy_policies } = self; - let mut channels = channels.into_values().collect::>(); - if channels.is_empty() { - return None; - } - - for acc in &mut channels { - normalize_channel_programmes(acc); - } - - apply_dummy_policies(&mut channels, &dummy_policies); - - channels.sort_by(|left, right| left.channel.id.cmp(&right.channel.id)); - let icon_override_channels = channels - .iter() - .filter(|acc| acc.icon_logo_override) - .map(|acc| Arc::clone(&acc.channel.id)) - .collect::>(); - let children = channels.into_iter().map(|acc| Arc::new(acc.channel)).collect(); - - Some(( - Epg { - logo_override: false, - priority: 0, - attributes: attributes.map(|attributes| attributes.attributes), - children, - }, - icon_override_channels, - )) - } - - /// Drain the accumulator directly into a temp `BPlusTree` on disk. Peak RAM - /// here is `EPG_DISK_BATCH_SIZE × max_channel_size` — channels never sit - /// in a `Vec`. The returned `DiskEpgSource` removes the file in its - /// `Drop`. - /// - /// `source_priority` and `source_order` are the per-source values used - /// for `set_attributes_if_preferred` at merge time. The accumulator - /// itself does not aggregate them, so the caller — which knows the - /// original Epg's `priority` and `source_order` — must hand them in. - /// - /// # Panics - /// - /// Panics if `source_order` exceeds `u32`, which would mean four billion - /// sources in one import. - pub fn finish_into_disk(self, path: PathBuf, source_priority: i16, source_order: u32) -> io::Result { - // Fresh tree at the temp path. `store` creates the file; the - // subsequent updater opens it for batched writes. - BPlusTree::::new() - .store(&path) - .map_err(|e| io::Error::other(format!("create temp EPG tree: {e}")))?; - // The temp file exists from here until `DiskEpgSource::new` takes - // ownership at the end. Any `?` in between would otherwise leak the - // file because no `Drop` is wired up yet. The local guard covers the - // fallible window and is disarmed just before we hand the path off. - let mut temp_guard = TempFileGuard(Some(path.clone())); - - let mut updater = BPlusTreeUpdate::::try_new_with_backoff(&path) - .map_err(|e| io::Error::other(format!("open temp EPG tree: {e}")))?; - updater.set_flush_policy(FlushPolicy::Batch); - - let EpgMergeAccumulator { attributes, channels, dummy_policies: _ } = self; - let total = channels.len(); - let mut batch: Vec<(EpgDiskChannelKey, EpgChannel)> = Vec::with_capacity(EPG_DISK_BATCH_SIZE); - let mut written = 0usize; - - for (_, mut acc) in channels { - normalize_channel_programmes(&mut acc); - let folded = with_folded_epg_id(&acc.channel.id, |folded| folded.intern()); - batch.push(( - EpgDiskChannelKey { - folded_id: folded, - priority: acc.priority, - // `usize -> u32` overflow requires > 4 billion sources, which - // is not representable in the file format anyway; treat it - // as a programmer error rather than a recoverable I/O - // failure. - source_order: u32::try_from(acc.source_order) - .expect("source_order exceeds u32 (4 billion sources in one import)"), - }, - acc.channel, - )); - if batch.len() >= EPG_DISK_BATCH_SIZE { - flush_batch(&mut updater, &mut batch, &mut written)?; - } - } - flush_batch(&mut updater, &mut batch, &mut written)?; - updater.commit()?; - // Reclaim space wasted by priority-override entries (multiple keys per - // channel). Without compact, the file is ~2× its eventual size. - updater.compact()?; - - log::debug!( - "Drained {written} channels ({total} total before normalize) into temp EPG tree at {}", - path.display() - ); - - let source = DiskEpgSource::new(path, attributes.map(|a| a.attributes), source_priority, source_order); - // DiskEpgSource now owns the path; disarm the guard so its Drop does - // not double-remove the file. `mem::forget` would also work, but - // `take()` keeps the guard in scope and is auditable. - temp_guard.0.take(); - Ok(source) - } -} - /// Removes the wrapped file path on drop unless `take()` was called first. /// Mirrors the cleanup logic of `DiskEpgSource::Drop` for the fallible window /// inside `finish_into_disk` where no `DiskEpgSource` exists yet. struct TempFileGuard(Option); -impl Drop for TempFileGuard { - fn drop(&mut self) { - if let Some(path) = self.0.take() { - if let Err(err) = std::fs::remove_file(&path) { - if err.kind() != std::io::ErrorKind::NotFound { - log::warn!("Failed to remove temp EPG tree at {}: {err}", path.display()); - } - } - } - } -} - -/// Flush a non-empty batch and bump the written counter. No-op for empty -/// batches, so the caller does not need a guard around the trailing flush. -fn flush_batch( - updater: &mut BPlusTreeUpdate, - batch: &mut Vec<(EpgDiskChannelKey, EpgChannel)>, - written: &mut usize, -) -> io::Result<()> { - if batch.is_empty() { - return Ok(()); - } - let items: Vec<(&EpgDiskChannelKey, &EpgChannel)> = batch.iter().map(|(k, v)| (k, v)).collect(); - let prepared = BPlusTreeUpdate::::prepare_upsert_batch(&items)?; - updater.upsert_batch_encoded(prepared)?; - *written += batch.len(); - batch.clear(); - Ok(()) -} - -/// Sort key for the per-source temp `BPlusTree`. Order: folded channel id ascending, -/// then priority ascending, then source order ascending. This matches the existing -/// rule in `EpgMergeAccumulator::upsert_channel` (`(priority, source_order) < (acc.priority, acc.source_order)`) -/// — lower numbers win. Sorted iteration over the tree therefore yields channels in -/// the right order for the multi-way merge downstream. -/// -/// The `Ord` implementation below is hand-written to make the value-based, -/// deterministic ordering explicit (`folded_id` → `priority` → `source_order`). -/// `Arc` already compares by value when the inner `str` is `Ord`, so a -/// derived `Ord` would produce the same total order — the custom impl exists -/// to lock the contract in source rather than rely on a derived behaviour. -/// Same key type serialises through `rmp_serde` as a record. -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct EpgDiskChannelKey { - #[serde(with = "arc_str_serde")] - pub folded_id: Arc, - pub priority: i16, - pub source_order: u32, -} - -// Explicit value-based ordering. `#[derive(Ord)]` would derive the same total -// order via `Arc`'s value-based `Ord` impl, but writing it out documents -// the contract: `folded_id` ascending, then `priority` ascending, then -// `source_order` ascending. Sorted iteration in `merge_epg_trees` depends on -// this exact order for deterministic multi-way merge results. -impl Ord for EpgDiskChannelKey { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.folded_id - .as_ref() - .cmp(other.folded_id.as_ref()) - .then(self.priority.cmp(&other.priority)) - .then(self.source_order.cmp(&other.source_order)) - } -} -impl PartialOrd for EpgDiskChannelKey { - fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } -} - -/// Handle for a temp `BPlusTree` that holds the channels of one EPG source. -/// `Drop` removes the file — even on panic — so the temp dir never accumulates -/// stale trees. The caller is expected to hand ownership to `merge_epg_trees`, -/// which opens a query handle and drains the file before letting the guard drop. -/// -/// `source_priority` and `source_order` are the per-source values used at -/// merge time by `set_attributes_if_preferred`. They live on the source -/// rather than being reconstructed from the disk tree, because the disk -/// tree only stores per-channel priorities (which can differ across -/// channels if sources overlap), not a single source-level value. -pub struct DiskEpgSource { - pub(super) path: PathBuf, - pub(super) attributes: Option, Arc>>, - pub(super) source_priority: i16, - pub(super) source_order: u32, -} - -impl DiskEpgSource { - pub fn new( - path: PathBuf, - attributes: Option, Arc>>, - source_priority: i16, - source_order: u32, - ) -> Self { - Self { path, attributes, source_priority, source_order } - } -} - -impl Drop for DiskEpgSource { - fn drop(&mut self) { - if let Err(err) = std::fs::remove_file(&self.path) { - // Missing-file is fine (already cleaned up). Anything else is worth - // a warning — it leaks the temp file until the OS clears /tmp. - if err.kind() != std::io::ErrorKind::NotFound { - log::warn!("Failed to remove temp EPG tree at {}: {err}", self.path.display()); - } - } - } -} - -fn apply_dummy_policies(channels: &mut [ChannelMergeAcc], dummy_policies: &HashMap, PreferredDummyPolicy>) { - let now = chrono::Utc::now(); - for acc in channels { - let key = with_folded_epg_id(&acc.channel.id, |folded| folded.intern()); - if let Some(preferred) = dummy_policies.get(&key) { - let policy = &preferred.policy; - if let Err(err) = ics::fill_dummy_gaps( - &mut acc.channel.programmes, - &acc.channel.id, - &policy.timezone, - &policy.config, - now, - ) { - log::warn!("Failed to apply ICS dummy policy for {}: {err}", acc.channel.id); - } - } - } -} - -fn backfill_programme_metadata(existing: &mut EpgProgramme, incoming: EpgProgramme) { - if existing.title.is_none() { - existing.title = incoming.title; - } - if existing.desc.is_none() { - existing.desc = incoming.desc; - } - if existing.icon.is_none() { - existing.icon = incoming.icon; - } - if existing.catchup_id.is_none() { - existing.catchup_id = incoming.catchup_id; - } - if existing.categories.is_empty() { - existing.categories = incoming.categories; - } - existing.is_live |= incoming.is_live; - existing.is_new |= incoming.is_new; -} - -fn normalize_channel_programmes(acc: &mut ChannelMergeAcc) { - // `acc.programmes` is the cross-source list of `ProgrammeMergeEntry` records - // (populated by `add_channel_with_programmes` and `push_programme`). - // When the list is empty, the channel's own programmes — set by the - // vacant path of `upsert_channel` — are authoritative; the previous - // implementation unconditionally overwrote `acc.channel.programmes` with - // an empty vector here, which silently dropped every programme on the - // single-source and disk-spill paths. - if acc.programmes.is_empty() { - return; - } - acc.programmes - .sort_by_key(|entry| (entry.programme.start, entry.programme.stop, entry.priority, entry.source_order)); - - let programme_entries = std::mem::take(&mut acc.programmes); - let mut merged_programmes = Vec::with_capacity(programme_entries.len()); - let mut entries = programme_entries.into_iter(); - if let Some(first_entry) = entries.next() { - let mut current = first_entry.programme; - for entry in entries { - if entry.programme.start == current.start && entry.programme.stop == current.stop { - backfill_programme_metadata(&mut current, entry.programme); - } else { - merged_programmes.push(current); - current = entry.programme; - } - } - merged_programmes.push(current); - } - - merged_programmes.sort_by_key(|programme| (programme.start, programme.stop)); - acc.channel.programmes = merged_programmes; -} - #[cfg(test)] -pub fn merge_epg_channels_by_priority(channels_by_source: Vec<(i16, Vec)>) -> Vec { - let mut accumulator = EpgMergeAccumulator::new(); - for (source_order, (priority, channels)) in channels_by_source.into_iter().enumerate() { - for channel in channels { - accumulator.add_channel_with_programmes(priority, source_order, false, channel); - } - } - accumulator.finish_channels().map(|(_, channels)| channels).unwrap_or_default() -} +mod tests; -pub fn merge_epg_channels_by_priority_with_dummy_policies( - channels_by_source: Vec<(i16, Vec)>, - dummy_policies: Vec, -) -> Vec { - let mut accumulator = EpgMergeAccumulator::new(); - for (source_order, (priority, channels)) in channels_by_source.into_iter().enumerate() { - for channel in channels { - accumulator.add_channel_with_programmes(priority, source_order, false, channel); - } - } - for source in dummy_policies { - accumulator.register_dummy_policy(&source.channel_id, source.priority, source.source_order, source.policy); - } - accumulator - .finish_epg_with_icon_overrides() - .map(|(epg, _)| epg.children.into_iter().map(Arc::unwrap_or_clone).collect()) - .unwrap_or_default() -} - -pub fn flatten_tvguide(tv_guides: Vec) -> Option { - let mut accumulator = EpgMergeAccumulator::new(); - for (source_order, guide) in tv_guides.into_iter().enumerate() { - accumulator.set_attributes_if_preferred(guide.priority, source_order, guide.attributes); - for channel in guide.children { - accumulator.add_channel_with_programmes( - guide.priority, - source_order, - guide.logo_override, - Arc::unwrap_or_clone(channel), - ); - } - } - accumulator.finish() -} - -/// Merge N temp `BPlusTree`s (one per EPG source) into a single `Epg`, -/// applying dummy policies and the existing priority/sort rules. Sources -/// are consumed one at a time — each `DiskEpgSource`'s temp file is -/// removed by its `Drop` as soon as its iterator is exhausted, so peak RAM -/// is bounded by the largest single source plus the accumulator's -/// per-channel `HashMap`, not the total feed size. -/// -/// Complexity: `O(n_sources + total_channels)`. The per-channel priority -/// resolution happens inside `EpgMergeAccumulator::upsert_channel` -/// (`HashMap` insert), not in a heap. Sources are processed in iteration -/// order, so earlier sources win priority ties for shared channels. -/// -/// The constant-memory guarantee lives on the write side -/// (`EpgMergeAccumulator::finish_into_disk`); this function is bounded by -/// the union's metadata `HashMap` in the accumulator, which is the same -/// shape the in-memory path already carries. -/// -/// The icon-override channel set returned by `finish_epg_with_icon_overrides` -/// is intentionally dropped — the wire-up path persists channels directly, -/// not the override metadata. If a caller later needs that set, the -/// `MergedEpgWithIconOverrides` tuple is already in scope via this -/// function's return type. -pub fn merge_epg_trees(sources: Vec) -> io::Result> { - let mut accumulator = EpgMergeAccumulator::new(); - for source in sources { - if let Some(attrs) = &source.attributes { - accumulator.set_attributes_if_preferred( - source.source_priority, - source.source_order as usize, - Some(attrs.clone()), - ); - } - let mut query = BPlusTreeQuery::::try_new(&source.path) - .map_err(|e| io::Error::other(format!("open temp EPG tree at {}: {e}", source.path.display())))?; - for entry in query.iter() { - let (key, channel) = entry.map_err(|e| io::Error::other(format!("read temp EPG entry: {e}")))?; - // `add_channel_with_programmes` preserves the channel's programmes - // on the merge accumulator; plain `upsert_channel` would drop them - // because the vacant-entry path constructs a fresh `ChannelMergeAcc` - // with `programmes: Vec::new()`. - accumulator.add_channel_with_programmes(key.priority, key.source_order as usize, false, channel); - } - // `source` drops at the end of this iteration, which removes the - // temp file. The query's mmap/buffer is closed first because `query` - // goes out of scope before `source`. - } - Ok(accumulator.finish_epg_with_icon_overrides()) -} +mod calendar; +mod disk; +mod filter; +mod merge; +mod normalize; +mod xml; +pub use disk::{DiskEpgSource, EpgDiskChannelKey}; #[cfg(test)] -mod tests { - use super::TVGuide; - use crate::parser::xmltv::{ - flatten_tvguide, merge_epg_channels_by_priority, merge_epg_channels_by_priority_with_dummy_policies, - normalize_channel_name, EpgDummyPolicySource, EpgMergeAccumulator, - }; - use shared::model::{EpgCategory, EpgChannel, EpgProgramme}; - use std::{collections::HashSet, fs, path::PathBuf, sync::Arc}; - use tempfile::tempdir; - use tuliprox_core::{ - model::{ - Epg, EpgSmartMatchConfig, IcsDummyConfig, IcsEpgSourceConfig, PersistedEpgSource, PersistedEpgSourceKind, - }, - utils::FileLockManager, - }; - - /// Run an async test body on a freshly-created multi-threaded tokio - /// runtime. Centralizes the `Runtime::new()...block_on(...)` boilerplate - /// shared by every test in this module that exercises async EPG code. - fn run_async_test(future: F) - where - F: std::future::Future, - { - tokio::runtime::Runtime::new().unwrap().block_on(future); - } - - fn xmltv_source(file_path: PathBuf, priority: i16, logo_override: bool) -> PersistedEpgSource { - PersistedEpgSource { file_path, priority, logo_override, kind: PersistedEpgSourceKind::Xmltv } - } - - fn dummy_policy_source(priority: i16, source_order: usize, title: &str) -> EpgDummyPolicySource { - EpgDummyPolicySource { - priority, - source_order, - channel_id: "f1.calendar".intern(), - policy: tuliprox_core::model::IcsDummyPolicy { - timezone: "UTC".to_string(), - config: IcsDummyConfig { - enabled: true, - title: title.to_string(), - description: String::new(), - days_past: 0, - days_future: 0, - block_hours: 24, - min_gap_minutes: 1, - }, - }, - } - } - - #[test] - /// Tests normalization of a channel name using the default smart match configuration. - /// - /// # Examples - /// - /// ``` - /// parse_normalize().unwrap(); - /// ``` - fn parse_normalize() { - let epg_normalize_dto = EpgSmartMatchConfigDto { ..Default::default() }; - let epg_normalize = EpgSmartMatchConfig::from(epg_normalize_dto); - let normalized = normalize_channel_name("Love Nature", &epg_normalize); - assert_eq!(normalized, "lovenature".to_string()); - } - - #[test] - fn normalization_keeps_country_suffixes_consistent() { - let mut dto = EpgSmartMatchConfigDto { - enabled: true, - name_prefix: EpgNamePrefix::Suffix(".".to_string()), - ..Default::default() - }; - dto.prepare().expect("valid smart-match config"); - let config = EpgSmartMatchConfig::from(dto); - - assert_eq!(normalize_channel_name("FR: TF1 FHD", &config), "tf1.fr"); - assert_eq!(normalize_channel_name("TF1.fr", &config), "tf1.fr"); - } - - #[test] - fn normalization_strips_quality_markers_without_corrupting_names() { - let mut dto = EpgSmartMatchConfigDto { enabled: true, ..Default::default() }; - dto.prepare().expect("valid smart-match config"); - let config = EpgSmartMatchConfig::from(dto); - - assert_eq!(normalize_channel_name("RMC Story H265 50FPS", &config), "rmcstory"); - assert_eq!(normalize_channel_name("Ashdod TV HD", &config), "ashdodtv"); - } - - #[test] - fn marker_stripping_skips_non_utf8_boundaries() { - assert_eq!(super::strip_markers("éclair", &["x".to_string()]), "éclair"); - } - - fn epg_channel(id: &str, title: Option<&str>, icon: Option<&str>, programmes: Vec) -> EpgChannel { - EpgChannel { - id: id.intern(), - title: title.map(Internable::intern), - icon: icon.map(Internable::intern), - programmes, - } - } - - fn epg_programme(id: &str, start: i64, stop: i64, title: Option<&str>, desc: Option<&str>) -> EpgProgramme { - EpgProgramme::new_all( - start, - stop, - id.intern(), - title.map(Internable::intern), - desc.map(Internable::intern), - None, - ) - } - - #[test] - fn epg_priority_merge_preserves_high_priority_metadata_and_fills_programme_gaps() { - let merged = merge_epg_channels_by_priority(vec![ - ( - 10, - vec![epg_channel( - "demo.channel", - Some("Low"), - Some("http://low/icon.png"), - vec![epg_programme("demo.channel", 10, 20, Some("Low Show"), None)], - )], - ), - ( - 0, - vec![epg_channel( - "demo.channel", - Some("High"), - Some("http://high/icon.png"), - vec![epg_programme("demo.channel", 20, 30, Some("High Show"), None)], - )], - ), - ]); - - assert_eq!(merged.len(), 1); - assert_eq!(merged[0].title.as_deref(), Some("High")); - assert_eq!(merged[0].icon.as_deref(), Some("http://high/icon.png")); - assert_eq!( - merged[0].programmes.iter().map(|programme| (programme.start, programme.stop)).collect::>(), - vec![(10, 20), (20, 30)], - ); - } - - #[test] - fn epg_priority_merge_backfills_duplicate_programme_metadata_without_overwriting() { - let merged = merge_epg_channels_by_priority(vec![ - ( - 0, - vec![epg_channel( - "demo.channel", - Some("High"), - None, - vec![epg_programme("demo.channel", 10, 20, Some("High Title"), None)], - )], - ), - ( - 10, - vec![epg_channel( - "demo.channel", - None, - Some("http://fallback/icon.png"), - vec![epg_programme("demo.channel", 10, 20, Some("Low Title"), Some("Recovered desc"))], - )], - ), - ]); - - assert_eq!(merged.len(), 1); - assert_eq!(merged[0].title.as_deref(), Some("High")); - assert_eq!(merged[0].icon.as_deref(), Some("http://fallback/icon.png")); - assert_eq!(merged[0].programmes.len(), 1); - assert_eq!(merged[0].programmes[0].title.as_deref(), Some("High Title")); - assert_eq!(merged[0].programmes[0].desc.as_deref(), Some("Recovered desc")); - } - - #[test] - fn epg_priority_merge_backfills_programme_tags() { - let low_priority_categories = vec![ - EpgCategory { value: "Sports".intern(), lang: None }, - EpgCategory { value: "Live".intern(), lang: Some("en".intern()) }, - ]; - let mut low_programme = epg_programme("demo.channel", 10, 20, None, None); - low_programme.icon = Some("https://example.com/programme.jpg".intern()); - low_programme.categories = low_priority_categories.clone(); - low_programme.is_live = true; - low_programme.is_new = true; - - let merged = merge_epg_channels_by_priority(vec![ - ( - 0, - vec![epg_channel( - "demo.channel", - Some("High"), - None, - vec![epg_programme("demo.channel", 10, 20, Some("High Title"), None)], - )], - ), - (10, vec![epg_channel("demo.channel", None, None, vec![low_programme])]), - ]); - - assert_eq!(merged.len(), 1); - assert_eq!(merged[0].programmes.len(), 1); - assert_eq!(merged[0].programmes[0].icon.as_deref(), Some("https://example.com/programme.jpg")); - assert_eq!(merged[0].programmes[0].categories, low_priority_categories); - assert!(merged[0].programmes[0].is_live); - assert!(merged[0].programmes[0].is_new); - - let high_programme_with_categories = { - let mut programme = epg_programme("demo.channel", 30, 40, Some("High Title 2"), None); - programme.categories = vec![EpgCategory { value: "Drama".intern(), lang: None }]; - programme.is_new = true; - programme - }; - let low_programme_extra = { - let mut programme = epg_programme("demo.channel", 30, 40, None, None); - programme.categories = vec![EpgCategory { value: "ShouldNotWin".intern(), lang: None }]; - programme.is_live = true; - programme - }; - let merged = merge_epg_channels_by_priority(vec![ - (0, vec![epg_channel("demo.channel", None, None, vec![high_programme_with_categories])]), - (10, vec![epg_channel("demo.channel", None, None, vec![low_programme_extra])]), - ]); - - assert_eq!(merged.len(), 1); - let programme = &merged[0].programmes[0]; - assert_eq!(programme.categories, vec![EpgCategory { value: "Drama".intern(), lang: None }],); - assert!(programme.is_live); - assert!(programme.is_new); - } - - #[test] - fn epg_priority_merge_keeps_same_priority_deterministic_order() { - let merged = merge_epg_channels_by_priority(vec![ - ( - 0, - vec![epg_channel( - "demo.channel", - Some("First"), - None, - vec![epg_programme("demo.channel", 10, 20, Some("First Title"), None)], - )], - ), - ( - 0, - vec![epg_channel( - "demo.channel", - Some("Second"), - Some("http://second/icon.png"), - vec![ - epg_programme("demo.channel", 10, 20, Some("Second Title"), Some("Second desc")), - epg_programme("demo.channel", 20, 30, Some("Second Programme"), None), - ], - )], - ), - ]); - - assert_eq!(merged[0].title.as_deref(), Some("First")); - assert_eq!(merged[0].icon.as_deref(), Some("http://second/icon.png")); - assert_eq!(merged[0].programmes[0].title.as_deref(), Some("First Title")); - assert_eq!(merged[0].programmes[0].desc.as_deref(), Some("Second desc")); - assert_eq!(merged[0].programmes.len(), 2); - } - - #[test] - fn epg_priority_merge_flatten_uses_same_backfill_rules() { - let flattened = flatten_tvguide(vec![ - Epg { - logo_override: false, - priority: 0, - attributes: None, - children: vec![Arc::new(epg_channel( - "demo.channel", - Some("High"), - None, - vec![epg_programme("demo.channel", 10, 20, Some("High Show"), None)], - ))], - }, - Epg { - logo_override: false, - priority: 10, - attributes: None, - children: vec![Arc::new(epg_channel( - "demo.channel", - Some("Low"), - Some("http://low/icon.png"), - vec![epg_programme("demo.channel", 20, 30, Some("Low Show"), None)], - ))], - }, - ]) - .expect("flattened epg"); - - assert_eq!(flattened.children.len(), 1); - assert_eq!(flattened.children[0].title.as_deref(), Some("High")); - assert_eq!(flattened.children[0].icon.as_deref(), Some("http://low/icon.png")); - assert_eq!(flattened.children[0].programmes.len(), 2); - } - - #[test] - fn epg_priority_merge_flatten_keeps_shared_arc_channels() { - let shared_channel = Arc::new(epg_channel( - "demo.channel", - Some("Shared"), - Some("http://shared/icon.png"), - vec![epg_programme("demo.channel", 10, 20, Some("Shared Show"), None)], - )); - - let flattened = flatten_tvguide(vec![Epg { - logo_override: false, - priority: 0, - attributes: None, - children: vec![Arc::clone(&shared_channel)], - }]) - .expect("flattened epg"); - - assert_eq!(flattened.children.len(), 1); - assert_eq!(flattened.children[0].title.as_deref(), Some("Shared")); - assert_eq!(flattened.children[0].icon.as_deref(), Some("http://shared/icon.png")); - } - - #[test] - fn epg_priority_merge_tracks_icon_override_per_channel() { - let mut accumulator = EpgMergeAccumulator::new(); - accumulator.add_channel_with_programmes( - 0, - 0, - false, - epg_channel("demo.keep", Some("Keep"), Some("http://keep/icon.png"), vec![]), - ); - accumulator.add_channel_with_programmes( - 1, - 1, - true, - epg_channel("demo.override", Some("Override"), Some("http://override/icon.png"), vec![]), - ); - - let (_epg, icon_override_channels) = accumulator.finish_epg_with_icon_overrides().expect("merged epg"); - - let keep_id: Arc = "demo.keep".intern(); - let override_id: Arc = "demo.override".intern(); - - assert!(!icon_override_channels.contains(&keep_id)); - assert!(icon_override_channels.contains(&override_id)); - } - - #[test] - fn epg_priority_merge_normalizes_duplicates_within_single_source() { - let merged = merge_epg_channels_by_priority(vec![( - 0, - vec![epg_channel( - "demo.channel", - Some("Demo"), - None, - vec![ - epg_programme("demo.channel", 20, 30, Some("Later"), None), - epg_programme("demo.channel", 10, 20, Some("First"), None), - epg_programme("demo.channel", 10, 20, None, Some("Recovered desc")), - ], - )], - )]); - - assert_eq!(merged.len(), 1); - assert_eq!( - merged[0].programmes.iter().map(|programme| (programme.start, programme.stop)).collect::>(), - vec![(10, 20), (20, 30)], - ); - assert_eq!(merged[0].programmes[0].title.as_deref(), Some("First")); - assert_eq!(merged[0].programmes[0].desc.as_deref(), Some("Recovered desc")); - } - - #[test] - fn epg_priority_merge_preserves_exact_id_match_when_smart_match_is_enabled() { - run_async_test(async move { - let dir = tempdir().unwrap(); - let epg_path = dir.path().join("smart-exact.xml"); - - fs::write( - &epg_path, - r#" - - - Completely Different Name - - - Exact Match Source - -"#, - ) - .unwrap(); - - let mut smart_cfg = EpgSmartMatchConfigDto { enabled: true, ..Default::default() }; - smart_cfg.prepare().expect("smart match config"); - let tv_guide = TVGuide::new(vec![xmltv_source(epg_path, 0, false)]); - let mut id_cache = EpgIdCache::new(Some(&tuliprox_core::model::EpgConfig { - sources: vec![], - smart_match: Some(EpgSmartMatchConfig::from(smart_cfg)), - })); - id_cache.insert_channel_epg_id("demo.channel"); - - let merged = tv_guide.filter_merged(&mut id_cache).await.expect("merged epg"); - - assert_eq!(merged.children.len(), 1); - assert_eq!(merged.children[0].id.as_ref(), "demo.channel"); - assert_eq!(merged.children[0].programmes.len(), 1); - }); - } - - #[test] - fn persisted_epg_source_read_uses_shared_file_lock() { - run_async_test(async move { - let dir = tempdir().expect("temp dir"); - let epg_path = dir.path().join("locked.xml"); - fs::write( - &epg_path, - r#" - Demo - - Locked read - -"#, - ) - .expect("write XMLTV fixture"); - - let file_locks = Arc::new(FileLockManager::new()); - let guide = - TVGuide::new(vec![xmltv_source(epg_path.clone(), 0, false)]).with_file_locks(Arc::clone(&file_locks)); - let mut id_cache = EpgIdCache::new(None); - id_cache.insert_channel_epg_id("demo.channel"); - let write_guard = file_locks.write_lock(&epg_path).await; - let filter = guide.filter_merged(&mut id_cache); - tokio::pin!(filter); - - assert!(tokio::time::timeout(std::time::Duration::from_millis(25), filter.as_mut()).await.is_err()); - drop(write_guard); - - let merged = filter.await.expect("EPG parse after write lock release"); - assert_eq!(merged.children[0].programmes[0].title.as_deref(), Some("Locked read")); - }); - } - - #[test] - fn epg_channel_id_match_is_case_insensitive_and_preserves_guide_case() { - run_async_test(async move { - let dir = tempdir().unwrap(); - let epg_path = dir.path().join("mixed-case.xml"); - - // Guide channel ids are MixedCase and differ in case from the playlist ids. - fs::write( - &epg_path, - r#" - - - Sport Extra - - - CNN - - - Match A - - - Match B - -"#, - ) - .unwrap(); - - let tv_guide = TVGuide::new(vec![xmltv_source(epg_path, 0, false)]); - let mut id_cache = EpgIdCache::new(None); - // Playlist epg ids arrive in a *different* case than the guide. Both origins - // (an Xtream source and a mapper-script literal) go through the same - // `insert_channel_epg_id`, which folds the membership key. - id_cache.insert_channel_epg_id("sport.EXTRA1.de"); // e.g. from an Xtream source - id_cache.insert_channel_epg_id("cnn.us"); // e.g. set by a mapper literal @epg_channel_id = "cnn.us" - - let merged = tv_guide.filter_merged(&mut id_cache).await.expect("merged epg"); - - // Both MixedCase guide channels matched despite the case difference. - assert_eq!(merged.children.len(), 2); - let ids: HashSet<&str> = merged.children.iter().map(|c| c.id.as_ref()).collect(); - // Emitted preserves the guide's ORIGINAL case (not folded). - assert!(ids.contains("Sport.Extra1.DE"), "emitted ids: {ids:?}"); - assert!(ids.contains("CNN.US"), "emitted ids: {ids:?}"); - for channel in &merged.children { - assert_eq!(channel.programmes.len(), 1, "channel {} programmes", channel.id); - } - }); - } - - #[test] - fn epg_programme_channel_match_is_case_insensitive_within_source() { - run_async_test(async move { - let dir = tempdir().unwrap(); - let epg_path = dir.path().join("mixed-case-programme.xml"); - - fs::write( - &epg_path, - r#" - - - Demo - - - Case Variant Programme - -"#, - ) - .unwrap(); - - let tv_guide = TVGuide::new(vec![xmltv_source(epg_path, 0, false)]); - let mut id_cache = EpgIdCache::new(None); - id_cache.insert_channel_epg_id("demo.channel"); - - let merged = tv_guide.filter_merged(&mut id_cache).await.expect("merged epg"); - - assert_eq!(merged.children.len(), 1); - assert_eq!(merged.children[0].id.as_ref(), "Demo.Channel"); - assert_eq!(merged.children[0].programmes.len(), 1); - assert_eq!(merged.children[0].programmes[0].get_transient_channel_id().as_ref(), "demo.channel"); - }); - } - - #[test] - fn epg_priority_merge_filter_accepts_later_source_for_already_processed_channel() { - run_async_test(async move { - let dir = tempdir().unwrap(); - let high_path = dir.path().join("high.xml"); - let low_path = dir.path().join("low.xml"); - - fs::write( - &high_path, - r#" - - - High Channel - - - High Source - -"#, - ) - .unwrap(); - fs::write( - &low_path, - r#" - - - Low Channel - - - Low Source - -"#, - ) - .unwrap(); - - let tv_guide = TVGuide::new(vec![xmltv_source(high_path, 0, false), xmltv_source(low_path, 10, false)]); - let mut id_cache = EpgIdCache::new(None); - id_cache.insert_channel_epg_id("demo.channel"); - - let merged = tv_guide.filter_merged(&mut id_cache).await.expect("merged epg"); - - assert!(id_cache.contains_processed_epg_id("demo.channel")); - assert_eq!(merged.children.len(), 1); - assert_eq!(merged.children[0].title.as_deref(), Some("High Channel")); - assert_eq!(merged.children[0].programmes.len(), 2); - assert_eq!( - merged.children[0] - .programmes - .iter() - .filter_map(|programme| programme.title.as_deref()) - .collect::>(), - vec!["High Source", "Low Source"], - ); - }); - } - - #[test] - fn ics_dummy_policy_fills_after_real_programme_merge_without_overwriting_xmltv() { - run_async_test(async move { - use chrono::{Datelike, TimeZone, Utc}; - - let dir = tempdir().unwrap(); - let xml_path = dir.path().join("guide.xml"); - let ics_path = dir.path().join("empty.ics"); - let now = Utc::now(); - let real_start = Utc.with_ymd_and_hms(now.year(), now.month(), now.day(), 4, 0, 0).single().expect("start"); - let real_stop = Utc.with_ymd_and_hms(now.year(), now.month(), now.day(), 6, 0, 0).single().expect("stop"); - - fs::write( - &xml_path, - format!( - r#" - - - Formula 1 - - - Real Show - -"#, - start = real_start.format("%Y%m%d%H%M%S %z"), - stop = real_stop.format("%Y%m%d%H%M%S %z"), - ), - ) - .unwrap(); - fs::write(&ics_path, "BEGIN:VCALENDAR\nEND:VCALENDAR\n").unwrap(); - - let guide = TVGuide::new(vec![ - xmltv_source(xml_path, 0, false), - PersistedEpgSource { - file_path: ics_path, - priority: 1, - logo_override: false, - kind: PersistedEpgSourceKind::Ics { - channel_id: "f1.calendar".intern(), - channel_title: Some("Formula 1".intern()), - match_names: Vec::new(), - config: Box::new(IcsEpgSourceConfig { - timezone: "UTC".to_string(), - dummy: IcsDummyConfig { - enabled: true, - title: "No programme".to_string(), - description: String::new(), - days_past: 0, - days_future: 0, - block_hours: 4, - min_gap_minutes: 1, - }, - ..IcsEpgSourceConfig::default() - }), - }, - }, - ]); - let mut id_cache = EpgIdCache::new(None); - id_cache.insert_channel_epg_id("f1.calendar"); - - let merged = guide.filter_merged(&mut id_cache).await.expect("merged"); - let channel = &merged.children[0]; - assert_eq!(channel.id.as_ref(), "f1.calendar"); - assert_eq!( - channel.programmes.iter().filter(|programme| programme.title.as_deref() == Some("Real Show")).count(), - 1 - ); - let dummy_programmes = channel - .programmes - .iter() - .filter(|programme| programme.title.as_deref() == Some("No programme")) - .collect::>(); - assert!(!dummy_programmes.is_empty()); - for dummy in dummy_programmes { - assert!( - dummy.stop <= real_start.timestamp() || dummy.start >= real_stop.timestamp(), - "dummy overlaps real programme: {}-{}", - dummy.start, - dummy.stop - ); - } - }); - } - - #[test] - fn dummy_policy_selection_uses_priority_then_source_order() { - let channel = || EpgChannel { - id: "f1.calendar".intern(), - title: Some("Formula 1".intern()), - icon: None, - programmes: Vec::new(), - }; - let merge = |policies| { - merge_epg_channels_by_priority_with_dummy_policies(vec![(0, vec![channel()])], policies) - .into_iter() - .next() - .expect("merged channel") - }; - - let priority_winner = - merge(vec![dummy_policy_source(10, 0, "Low priority"), dummy_policy_source(-10, 1, "High priority")]); - assert!(!priority_winner.programmes.is_empty()); - assert!(priority_winner.programmes.iter().all(|programme| programme.title.as_deref() == Some("High priority"))); - - let source_order_winner = - merge(vec![dummy_policy_source(0, 2, "Later source"), dummy_policy_source(0, 1, "Earlier source")]); - assert!(!source_order_winner.programmes.is_empty()); - assert!(source_order_winner - .programmes - .iter() - .all(|programme| programme.title.as_deref() == Some("Earlier source"))); - } - - #[ignore = "requires a local XMLTV fixture under /tmp"] - #[test] - fn parse_test() { - let run_test = async move || { - //let file_path = PathBuf::from("/tmp/epg.xml.gz"); - let file_path = PathBuf::from("/tmp/invalid_epg.xml"); - - if file_path.exists() { - let tv_guide = TVGuide::new(vec![xmltv_source(file_path, 0, false)]); - - let mut id_cache = EpgIdCache::new(None); - id_cache.insert_channel_epg_id("342"); - //id_cache.collect_epg_id(fp); - - let channel_ids = HashSet::from([342u32.intern()]); - match tv_guide.filter(&mut id_cache).await { - None => panic!("No epg filtered"), - Some(epgs) => { - for epg in epgs { - assert_eq!(epg.children.len(), channel_ids.len() * 2, "Epg size does not match"); - } - } - } - } - }; - tokio::runtime::Runtime::new().unwrap().block_on(run_test()); - } - - #[test] - /// Tests normalization of channel names with various prefixes, suffixes, and special characters using a configured `EpgSmartMatchConfig`. - /// - /// # Examples - /// - /// ``` - /// normalize(); - /// // This will assert that various channel names are normalized as expected. - /// ``` - fn normalize() { - let mut epg_smart_cfg_dto = EpgSmartMatchConfigDto { - enabled: true, - name_prefix: EpgNamePrefix::Suffix(".".to_string()), - ..Default::default() - }; - let _ = epg_smart_cfg_dto.prepare(); - let epg_smart_cfg = EpgSmartMatchConfig::from(epg_smart_cfg_dto); - println!("{epg_smart_cfg:?}"); - assert_eq!("supersport6.ru", normalize_channel_name("RU: SUPERSPORT 6 ᴿᴬᵂ", &epg_smart_cfg)); - assert_eq!("odisea.sat", normalize_channel_name("SAT: ODISEA ᴿᴬᵂ", &epg_smart_cfg)); - assert_eq!("odisea.4k", normalize_channel_name("4K: ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg)); - assert_eq!("odisea", normalize_channel_name("ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg)); - assert_eq!("odisea.bu", normalize_channel_name("BU | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg)); - assert_eq!("odisea.bg", normalize_channel_name("BG | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg)); - } - - use crate::processor::EpgIdCache; - use rphonetic::{Encoder, Metaphone}; - use shared::{ - model::{EpgNamePrefix, EpgSmartMatchConfigDto}, - utils::Internable, - }; - - #[test] - /// Demonstrates phonetic encoding (Metaphone) of normalized channel names with various prefixes and suffixes. - /// - /// This test prints the Metaphone-encoded representations of several normalized channel names using a configured `EpgSmartMatchConfig`. - /// - /// # Examples - /// - /// ``` - /// test_metaphone(); - /// // Output will show the Metaphone encodings for different channel name variants. - /// ``` - fn test_metaphone() { - let metaphone = Metaphone::default(); - let mut epg_smart_cfg_dto = EpgSmartMatchConfigDto { - enabled: true, - name_prefix: EpgNamePrefix::Suffix(".".to_string()), - ..Default::default() - }; - let _ = epg_smart_cfg_dto.prepare(); - let epg_smart_cfg = EpgSmartMatchConfig::from(epg_smart_cfg_dto); - println!("{epg_smart_cfg:?}"); - // assert_eq!("supersport6.ru", metaphone.encode(&normalize_channel_name("RU: SUPERSPORT 6 ᴿᴬᵂ", &epg_normalize_cfg))); - // assert_eq!("odisea.sat", metaphone.encode(&normalize_channel_name("SAT: ODISEA ᴿᴬᵂ", &epg_normalize_cfg))); - // assert_eq!("odisea", metaphone.encode(&normalize_channel_name("4K: ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); - // assert_eq!("odisea", metaphone.encode(&normalize_channel_name("ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); - // assert_eq!("odisea.bu", metaphone.encode(&normalize_channel_name("BU | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); - // assert_eq!("odisea.bg", metaphone.encode(&normalize_channel_name("BG | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); - - println!("{}", metaphone.encode(&normalize_channel_name("RU: SUPERSPORT 6 ᴿᴬᵂ", &epg_smart_cfg))); - println!("{}", metaphone.encode(&normalize_channel_name("SAT: ODISEA ᴿᴬᵂ", &epg_smart_cfg))); - println!("{}", metaphone.encode(&normalize_channel_name("4K: ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg))); - println!("{}", metaphone.encode(&normalize_channel_name("ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg))); - println!("{}", metaphone.encode(&normalize_channel_name("BU | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg))); - println!("{}", metaphone.encode(&normalize_channel_name("BG | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg))); - } - - #[test] - fn xmltv_programme_tags_are_extracted() { - run_async_test(async move { - let dir = tempdir().expect("temp dir"); - let epg_path = dir.path().join("programme-tags.xml"); - fs::write( - &epg_path, - r#" - ESPN - - Softball - - - Softball - Sports - - - -"#, - ) - .expect("write XMLTV fixture"); - - let guide = TVGuide::new(vec![xmltv_source(epg_path, 0, false)]); - let mut id_cache = EpgIdCache::new(None); - id_cache.insert_channel_epg_id("ESPN.us"); - let merged = guide.filter_merged(&mut id_cache).await.expect("merged EPG"); - let programme = &merged.children[0].programmes[0]; - - assert_eq!( - programme.categories, - vec![ - EpgCategory { value: "Softball".intern(), lang: Some("en".intern()) }, - EpgCategory { value: "Sports".intern(), lang: None }, - ], - ); - assert_eq!(programme.icon.as_deref(), Some("https://example.com/softball.jpg")); - assert!(programme.is_live); - assert!(programme.is_new); - }); - } - - /// Regression guard for the buffer-preallocation in `parse_tvguide`. - /// - /// `quick_xml` uses the caller-provided `Vec` as a monotonically growing - /// read buffer (it does not call `.clear()` between events; the caller - /// slices the new portion). For an XMLTV feed with a single giant - /// `` description, that buffer used to start at 0 capacity and - /// double ~25 times on the way to ~163 MiB — every doubling being a - /// copying realloc. Starting with `Vec::with_capacity(64 * 1024)` removes - /// the realloc chain. This test feeds a programme whose description is - /// bigger than that initial capacity and checks that parsing still - /// succeeds; the absence of the test would have allowed a silent - /// regression to `Vec::new()` without breaking any visible behaviour. - #[test] - fn parse_tvguide_handles_giant_programme_description() { - use crate::parser::xmltv::{parse_tvguide, EPG_TAG_DESC, EPG_TAG_PROGRAMME}; - - run_async_test(async { - // 200 KiB of text content — well beyond the 64 KiB preallocation. - let big_text = "x".repeat(200 * 1024); - // `channel` attribute is required for the parser to fire the - // callback on a tag (see `handle_tag_end`). - let xml = format!( - r#"t{big_text}"# - ); - let mut emitted_tags: Vec = Vec::new(); - parse_tvguide(xml.as_bytes(), &mut |tag| { - emitted_tags.push(tag); - }) - .await; - - // The whole point of the preallocation is that the full payload - // survives the round trip — not just the presence of a tag. - let programme = emitted_tags - .iter() - .find(|tag| tag.name.as_ref() == EPG_TAG_PROGRAMME) - .expect("parser emitted no tag"); - let children = programme.children.as_deref().expect(" has no children"); - let desc = children - .iter() - .find(|child| child.name.as_ref() == EPG_TAG_DESC) - .expect(" emitted no child"); - assert_eq!( - desc.value.as_deref().map(str::len), - Some(big_text.len()), - " payload was truncated; got {} bytes, expected {}", - desc.value.as_deref().map_or(0, str::len), - big_text.len() - ); - }); - } - - /// Smallest testable piece of the disk-spilling path. The Drop guard is - /// what keeps a panic or early return from leaking temp files into /tmp. - /// If this fails, every other disk-spilling test is built on sand. - #[test] - fn disk_epg_source_removes_its_temp_file_on_drop() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("epg-source.db"); - std::fs::write(&path, b"placeholder").unwrap(); - assert!(path.exists(), "precondition: temp file must exist"); - - { - let _src = super::DiskEpgSource::new(path.clone(), None, 0, 0); - } // _src dropped here - - assert!(!path.exists(), "DiskEpgSource::Drop must remove its temp file"); - } - - /// Builds an accumulator with N channels and drains it to disk. Asserts: - /// (a) the temp file exists while the guard is alive, (b) it has - /// non-trivial size, (c) the file is removed after Drop. Together these - /// prove the writer runs end-to-end without panicking. - #[test] - fn finish_into_disk_writes_a_real_temp_tree() { - use super::EpgMergeAccumulator; - - let mut acc = EpgMergeAccumulator::new(); - for i in 0..250u32 { - let id: Arc = format!("channel-{i:04}").into(); - let ch = shared::model::EpgChannel { - id: Arc::clone(&id), - title: Some(format!("title {i}").into()), - icon: None, - programmes: vec![shared::model::EpgProgramme::new(i64::from(i), i64::from(i + 1), id)], - }; - acc.upsert_channel(i16::try_from(i).unwrap_or(0), 0, false, ch); - } - - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("epg-source.db"); - let src = acc.finish_into_disk(path.clone(), 5, 0).unwrap(); - assert!(path.exists(), "temp tree file must exist while guard is alive"); - let size = std::fs::metadata(&path).unwrap().len(); - assert!(size > 1024, "expected non-trivial size, got {size}"); - drop(src); - assert!(!path.exists(), "Drop must remove the temp file"); - } - - /// Contract test: two temp trees written from two accumulators must - /// merge into the same `Epg` as the in-memory `finish_epg_with_icon_overrides` - /// would produce from the same inputs. Without this, every other - /// optimisation is built on sand. - #[test] - fn disk_path_matches_in_memory_finish() { - use super::{merge_epg_trees, EpgMergeAccumulator}; - - // Build one source by appending one programme per channel through the - // accumulator's primary entry point. `add_channel_with_programmes` is - // the same call the in-memory `merge_epg_channels_by_priority` uses, - // so the in-memory reference and the disk path go through identical - // APIs. The two sources use disjoint programme intervals - // (source 0: 0..1, source 1: 100..101) so the merged channel should - // retain both programmes end-to-end. - fn build_acc(source: usize, channels: std::ops::Range) -> EpgMergeAccumulator { - let mut acc = EpgMergeAccumulator::new(); - for i in channels { - let id: Arc = format!("ch-{i:04}").into(); - let priority = if source == 0 { 5 } else { 3 }; // source 1 wins - let (start, stop) = if source == 0 { - (i64::from(i), i64::from(i + 1)) - } else { - (100 + i64::from(i), 101 + i64::from(i)) - }; - acc.add_channel_with_programmes( - i16::try_from(priority).unwrap_or(0), - source, - false, - shared::model::EpgChannel { - id: Arc::clone(&id), - title: Some(format!("title-{source}-{i}").into()), - icon: None, - programmes: vec![shared::model::EpgProgramme::new(start, stop, id)], - }, - ); - } - acc - } - - // Reference: in-memory merge of the same two sources. - let ref_acc_a = build_acc(0, 0..50); - let ref_acc_b = build_acc(1, 0..50); - // `EpgMergeAccumulator` is single-shot, so the reference merge has to - // rebuild a fresh accumulator. The two halves contribute disjoint - // programme intervals, so a single accumulator sees both. - let mut ref_acc = EpgMergeAccumulator::new(); - for i in 0..50 { - let id: Arc = format!("ch-{i:04}").into(); - ref_acc.add_channel_with_programmes( - 5, - 0, - false, - shared::model::EpgChannel { - id: Arc::clone(&id), - title: Some(format!("title-0-{i}").into()), - icon: None, - programmes: vec![shared::model::EpgProgramme::new(i64::from(i), i64::from(i + 1), id)], - }, - ); - } - for i in 0..50 { - let id: Arc = format!("ch-{i:04}").into(); - ref_acc.add_channel_with_programmes( - 3, - 1, - false, - shared::model::EpgChannel { - id: Arc::clone(&id), - title: Some(format!("title-1-{i}").into()), - icon: None, - programmes: vec![shared::model::EpgProgramme::new(100 + i64::from(i), 101 + i64::from(i), id)], - }, - ); - } - let reference = ref_acc.finish_epg_with_icon_overrides().unwrap().0; - let _ = (ref_acc_a, ref_acc_b); // keep the per-source builder API in the test - - // Disk path: two temp trees, then merge. - let dir = tempfile::tempdir().unwrap(); - let src_a = build_acc(0, 0..50).finish_into_disk(dir.path().join("a.db"), 5, 0).unwrap(); - let src_b = build_acc(1, 0..50).finish_into_disk(dir.path().join("b.db"), 3, 1).unwrap(); - let merged = merge_epg_trees(vec![src_a, src_b]).unwrap().unwrap().0; - - assert_eq!(reference.children.len(), merged.children.len(), "channel counts must match"); - for (left, right) in reference.children.iter().zip(merged.children.iter()) { - assert_eq!(left.id, right.id, "channel order must match"); - // Source 1 had priority 3 (lower = wins) so its title should win. - assert_eq!(left.title, right.title, "priority winner's title must propagate"); - // Both sources contribute distinct, non-overlapping programmes. - // The merged channel must keep both — the disk path uses - // `add_channel_with_programmes` so nothing is dropped on the way in. - assert_eq!( - left.programmes.len(), - 2, - "channel {} should retain both source programmes, got {}", - left.id, - left.programmes.len() - ); - assert_eq!(right.programmes.len(), 2, "disk-merged channel {} lost programmes", right.id); - for (lp, rp) in left.programmes.iter().zip(right.programmes.iter()) { - assert_eq!(lp.start, rp.start, "programme start must match for channel {}", left.id); - assert_eq!(lp.stop, rp.stop, "programme stop must match for channel {}", left.id); - } - } - } -} +pub use merge::merge_epg_channels_by_priority; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use merge::{ + flatten_tvguide, merge_epg_channels_by_priority_with_dummy_policies, merge_epg_trees, EpgDummyPolicySource, + MergedEpgWithIconOverrides, +}; +pub use normalize::normalize_channel_name; +#[cfg(test)] +use normalize::strip_markers; +pub use xml::parse_tvguide; diff --git a/backend/processing/src/parser/xmltv/calendar.rs b/backend/processing/src/parser/xmltv/calendar.rs new file mode 100644 index 000000000..42352928a --- /dev/null +++ b/backend/processing/src/parser/xmltv/calendar.rs @@ -0,0 +1,71 @@ +use super::{EpgMergeAccumulator, IcsPersistedSource, TVGuide}; +use crate::processor::EpgIdCache; +use std::sync::Arc; +use tuliprox_core::model::{IcsDummyPolicy, PersistedEpgSource}; +use tuliprox_parser::ics; + +impl TVGuide { + pub(super) async fn process_ics_file( + id_cache: &mut EpgIdCache, + epg_source: &PersistedEpgSource, + source_order: usize, + accumulator: &mut EpgMergeAccumulator, + source: IcsPersistedSource<'_>, + ) -> bool { + let IcsPersistedSource { channel_id, channel_title, match_names, config } = source; + let mut candidates = Vec::with_capacity(2 + match_names.len()); + candidates.push(channel_id.to_string()); + if let Some(title) = channel_title { + candidates.push(title.to_string()); + } + candidates.extend(match_names.iter().map(ToString::to_string)); + let normalized_candidates = id_cache.normalize_candidates(candidates); + + let add_channel = if id_cache.smart_match_enabled { + let direct_match = id_cache.contains_channel_epg_id(channel_id); + let normalized_match = id_cache.match_epg_channel_candidates( + channel_id, + &normalized_candidates, + epg_source.priority, + source_order, + ); + direct_match || normalized_match + } else { + id_cache.contains_channel_epg_id(channel_id) + }; + + if !add_channel { + return false; + } + + match ics::parse_ics_file_to_channel( + &epg_source.file_path, + Arc::clone(channel_id), + channel_title.cloned(), + config, + ) + .await + { + Ok(channel) => { + id_cache.insert_processed_epg_id(channel_id); + accumulator.add_channel_with_programmes( + epg_source.priority, + source_order, + epg_source.logo_override, + channel, + ); + accumulator.register_dummy_policy( + channel_id, + epg_source.priority, + source_order, + IcsDummyPolicy { timezone: config.timezone.clone(), config: config.dummy.clone() }, + ); + true + } + Err(err) => { + log::warn!("Failed to process ICS EPG file {}: {err}", epg_source.file_path.display()); + false + } + } + } +} diff --git a/backend/processing/src/parser/xmltv/disk.rs b/backend/processing/src/parser/xmltv/disk.rs new file mode 100644 index 000000000..05a6d418b --- /dev/null +++ b/backend/processing/src/parser/xmltv/disk.rs @@ -0,0 +1,194 @@ +use super::{ + merge::{normalize_channel_programmes, EPG_DISK_BATCH_SIZE}, + EpgMergeAccumulator, TempFileGuard, +}; +use serde::{Deserialize, Serialize}; +use shared::{model::EpgChannel, utils::Internable}; +use std::{collections::HashMap, io, path::PathBuf, sync::Arc}; +use tuliprox_core::utils::{arc_str_serde, with_folded_epg_id}; +use tuliprox_repository::{BPlusTree, BPlusTreeUpdate, FlushPolicy}; + +impl EpgMergeAccumulator { + /// Drain the accumulator directly into a temp `BPlusTree` on disk. Peak RAM + /// here is `EPG_DISK_BATCH_SIZE × max_channel_size` — channels never sit + /// in a `Vec`. The returned `DiskEpgSource` removes the file in its + /// `Drop`. + /// + /// `source_priority` and `source_order` are the per-source values used + /// for `set_attributes_if_preferred` at merge time. The accumulator + /// itself does not aggregate them, so the caller — which knows the + /// original Epg's `priority` and `source_order` — must hand them in. + /// + /// # Panics + /// + /// Panics if `source_order` exceeds `u32`, which would mean four billion + /// sources in one import. + pub fn finish_into_disk(self, path: PathBuf, source_priority: i16, source_order: u32) -> io::Result { + // Fresh tree at the temp path. `store` creates the file; the + // subsequent updater opens it for batched writes. + BPlusTree::::new() + .store(&path) + .map_err(|e| io::Error::other(format!("create temp EPG tree: {e}")))?; + // The temp file exists from here until `DiskEpgSource::new` takes + // ownership at the end. Any `?` in between would otherwise leak the + // file because no `Drop` is wired up yet. The local guard covers the + // fallible window and is disarmed just before we hand the path off. + let mut temp_guard = TempFileGuard(Some(path.clone())); + + let mut updater = BPlusTreeUpdate::::try_new_with_backoff(&path) + .map_err(|e| io::Error::other(format!("open temp EPG tree: {e}")))?; + updater.set_flush_policy(FlushPolicy::Batch); + + let EpgMergeAccumulator { attributes, channels, dummy_policies: _ } = self; + let total = channels.len(); + let mut batch: Vec<(EpgDiskChannelKey, EpgChannel)> = Vec::with_capacity(EPG_DISK_BATCH_SIZE); + let mut written = 0usize; + + for (_, mut acc) in channels { + normalize_channel_programmes(&mut acc); + let folded = with_folded_epg_id(&acc.channel.id, |folded| folded.intern()); + batch.push(( + EpgDiskChannelKey { + folded_id: folded, + priority: acc.priority, + // `usize -> u32` overflow requires > 4 billion sources, which + // is not representable in the file format anyway; treat it + // as a programmer error rather than a recoverable I/O + // failure. + source_order: u32::try_from(acc.source_order) + .expect("source_order exceeds u32 (4 billion sources in one import)"), + }, + acc.channel, + )); + if batch.len() >= EPG_DISK_BATCH_SIZE { + flush_batch(&mut updater, &mut batch, &mut written)?; + } + } + flush_batch(&mut updater, &mut batch, &mut written)?; + updater.commit()?; + // Reclaim space wasted by priority-override entries (multiple keys per + // channel). Without compact, the file is ~2× its eventual size. + updater.compact()?; + + log::debug!( + "Drained {written} channels ({total} total before normalize) into temp EPG tree at {}", + path.display() + ); + + let source = DiskEpgSource::new(path, attributes.map(|a| a.attributes), source_priority, source_order); + // DiskEpgSource now owns the path; disarm the guard so its Drop does + // not double-remove the file. `mem::forget` would also work, but + // `take()` keeps the guard in scope and is auditable. + temp_guard.0.take(); + Ok(source) + } +} + +impl Drop for TempFileGuard { + fn drop(&mut self) { + if let Some(path) = self.0.take() { + if let Err(err) = std::fs::remove_file(&path) { + if err.kind() != std::io::ErrorKind::NotFound { + log::warn!("Failed to remove temp EPG tree at {}: {err}", path.display()); + } + } + } + } +} + +/// Flush a non-empty batch and bump the written counter. No-op for empty +/// batches, so the caller does not need a guard around the trailing flush. +fn flush_batch( + updater: &mut BPlusTreeUpdate, + batch: &mut Vec<(EpgDiskChannelKey, EpgChannel)>, + written: &mut usize, +) -> io::Result<()> { + if batch.is_empty() { + return Ok(()); + } + let items: Vec<(&EpgDiskChannelKey, &EpgChannel)> = batch.iter().map(|(k, v)| (k, v)).collect(); + let prepared = BPlusTreeUpdate::::prepare_upsert_batch(&items)?; + updater.upsert_batch_encoded(prepared)?; + *written += batch.len(); + batch.clear(); + Ok(()) +} + +/// Sort key for the per-source temp `BPlusTree`. Order: folded channel id ascending, +/// then priority ascending, then source order ascending. This matches the existing +/// rule in `EpgMergeAccumulator::upsert_channel` (`(priority, source_order) < (acc.priority, acc.source_order)`) +/// — lower numbers win. Sorted iteration over the tree therefore yields channels in +/// the right order for the multi-way merge downstream. +/// +/// The `Ord` implementation below is hand-written to make the value-based, +/// deterministic ordering explicit (`folded_id` → `priority` → `source_order`). +/// `Arc` already compares by value when the inner `str` is `Ord`, so a +/// derived `Ord` would produce the same total order — the custom impl exists +/// to lock the contract in source rather than rely on a derived behaviour. +/// Same key type serialises through `rmp_serde` as a record. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct EpgDiskChannelKey { + #[serde(with = "arc_str_serde")] + pub folded_id: Arc, + pub priority: i16, + pub source_order: u32, +} + +// Explicit value-based ordering. `#[derive(Ord)]` would derive the same total +// order via `Arc`'s value-based `Ord` impl, but writing it out documents +// the contract: `folded_id` ascending, then `priority` ascending, then +// `source_order` ascending. Sorted iteration in `merge_epg_trees` depends on +// this exact order for deterministic multi-way merge results. +impl Ord for EpgDiskChannelKey { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.folded_id + .as_ref() + .cmp(other.folded_id.as_ref()) + .then(self.priority.cmp(&other.priority)) + .then(self.source_order.cmp(&other.source_order)) + } +} + +impl PartialOrd for EpgDiskChannelKey { + fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } +} + +/// Handle for a temp `BPlusTree` that holds the channels of one EPG source. +/// `Drop` removes the file — even on panic — so the temp dir never accumulates +/// stale trees. The caller is expected to hand ownership to `merge_epg_trees`, +/// which opens a query handle and drains the file before letting the guard drop. +/// +/// `source_priority` and `source_order` are the per-source values used at +/// merge time by `set_attributes_if_preferred`. They live on the source +/// rather than being reconstructed from the disk tree, because the disk +/// tree only stores per-channel priorities (which can differ across +/// channels if sources overlap), not a single source-level value. +pub struct DiskEpgSource { + pub(in crate::parser) path: PathBuf, + pub(in crate::parser) attributes: Option, Arc>>, + pub(in crate::parser) source_priority: i16, + pub(in crate::parser) source_order: u32, +} + +impl DiskEpgSource { + pub fn new( + path: PathBuf, + attributes: Option, Arc>>, + source_priority: i16, + source_order: u32, + ) -> Self { + Self { path, attributes, source_priority, source_order } + } +} + +impl Drop for DiskEpgSource { + fn drop(&mut self) { + if let Err(err) = std::fs::remove_file(&self.path) { + // Missing-file is fine (already cleaned up). Anything else is worth + // a warning — it leaks the temp file until the OS clears /tmp. + if err.kind() != std::io::ErrorKind::NotFound { + log::warn!("Failed to remove temp EPG tree at {}: {err}", self.path.display()); + } + } + } +} diff --git a/backend/processing/src/parser/xmltv/filter.rs b/backend/processing/src/parser/xmltv/filter.rs new file mode 100644 index 000000000..5181b5238 --- /dev/null +++ b/backend/processing/src/parser/xmltv/filter.rs @@ -0,0 +1,62 @@ +use super::{EpgMergeAccumulator, IcsPersistedSource, MergedEpgWithIconOverrides, TVGuide}; +use crate::processor::EpgIdCache; +#[cfg(test)] +use tuliprox_core::model::Epg; +use tuliprox_core::model::PersistedEpgSourceKind; + +impl TVGuide { + // Exercised only by this module's own tests. + #[cfg(test)] + pub async fn filter(&self, id_cache: &mut EpgIdCache) -> Option> { + self.filter_merged(id_cache).await.map(|epg| vec![epg]) + } + + // Exercised only by this module's own tests. + #[cfg(test)] + pub async fn filter_merged(&self, id_cache: &mut EpgIdCache) -> Option { + self.filter_merged_with_icon_overrides(id_cache).await.map(|(epg, _)| epg) + } + + pub async fn filter_merged_with_icon_overrides( + &self, + id_cache: &mut EpgIdCache, + ) -> Option { + if id_cache.channel_epg_id.is_empty() && id_cache.normalized.is_empty() { + return None; + } + let mut accumulator = EpgMergeAccumulator::new(); + for (source_order, epg_source) in self.get_epg_sources().iter().enumerate() { + let _source_read_lock = match self.get_file_locks() { + Some(file_locks) => Some(file_locks.read_lock(&epg_source.file_path).await), + None => None, + }; + match &epg_source.kind { + PersistedEpgSourceKind::Xmltv => { + Self::process_epg_file(id_cache, epg_source, source_order, &mut accumulator).await; + } + PersistedEpgSourceKind::Ics { channel_id, channel_title, match_names, config } => { + Self::process_ics_file( + id_cache, + epg_source, + source_order, + &mut accumulator, + IcsPersistedSource { + channel_id, + channel_title: channel_title.as_ref(), + match_names, + config: config.as_ref(), + }, + ) + .await; + } + } + } + let available_epg_ids = accumulator.channel_ids_with_programmes(); + id_cache.finalize_matches(&available_epg_ids); + let mut selected_epg_ids = id_cache.selected_epg_ids(&available_epg_ids); + selected_epg_ids.extend(id_cache.channel_epg_id.iter().cloned()); + let retained_epg_ids = accumulator.retain_channels(&selected_epg_ids); + id_cache.replace_processed_epg_ids(retained_epg_ids.intersection(&available_epg_ids).cloned().collect()); + accumulator.finish_epg_with_icon_overrides() + } +} diff --git a/backend/processing/src/parser/xmltv/merge.rs b/backend/processing/src/parser/xmltv/merge.rs new file mode 100644 index 000000000..94b16bb25 --- /dev/null +++ b/backend/processing/src/parser/xmltv/merge.rs @@ -0,0 +1,453 @@ +use super::{ + ChannelMergeAcc, DiskEpgSource, EpgDiskChannelKey, EpgMergeAccumulator, PreferredAttributes, PreferredDummyPolicy, + ProgrammeMergeEntry, +}; +use log::error; +use quick_xml::events::BytesStart; +use shared::{ + model::{EpgChannel, EpgProgramme}, + utils::Internable, +}; +use std::{ + collections::{HashMap, HashSet}, + io, + sync::Arc, +}; +use tuliprox_core::{ + model::{Epg, IcsDummyPolicy, EPG_TAG_CHANNEL, EPG_TAG_PROGRAMME, EPG_TAG_TV}, + utils::with_folded_epg_id, +}; +use tuliprox_parser::ics; +use tuliprox_repository::BPlusTreeQuery; + +/// Channels per `BPlusTree` write batch. Sized so the in-flight batch + its +/// prepared references stay under 1 MiB on the typical 100-KiB-per-channel +/// EPG feed. This is an educated guess; profile-driven tuning is fine. +pub(super) const EPG_DISK_BATCH_SIZE: usize = 100; + +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +pub(super) enum XmlTagType { + Ignored, + Tv, + Channel, + Programme, +} + +impl XmlTagType { + #[inline] + pub fn is_tv(self) -> bool { self == XmlTagType::Tv } +} + +pub(super) fn get_tag_type(name: &str) -> XmlTagType { + match name { + EPG_TAG_TV => XmlTagType::Tv, + EPG_TAG_CHANNEL => XmlTagType::Channel, + EPG_TAG_PROGRAMME => XmlTagType::Programme, + _ => XmlTagType::Ignored, + } +} + +pub(super) fn collect_tag_attributes(e: &BytesStart) -> HashMap, Arc> { + let attributes = e + .attributes() + .filter_map(Result::ok) + .filter_map(|a| { + let key_binding = a.key; + let key_raw = String::from_utf8_lossy(key_binding.as_ref()); + let key = key_raw.intern(); + if let Ok(value) = a.normalized_value(quick_xml::XmlVersion::Implicit1_0).as_ref() { + if value.is_empty() { + None + } else { + // Ids are no longer lowercased when parsed; EPG matching folds case + // at the comparison instead, so the guide's original-case + // and are preserved in the output. + Some((key, value.intern())) + } + } else { + None + } + }) + .collect::, Arc>>(); + attributes +} + +/// Carries the source rank required to select a preview dummy policy exactly like the main EPG merge. +#[derive(Debug)] +pub struct EpgDummyPolicySource { + pub priority: i16, + pub source_order: usize, + pub channel_id: Arc, + pub policy: IcsDummyPolicy, +} + +pub(super) type FinishedEpgChannels = (Option, Arc>>, Vec); + +pub type MergedEpgWithIconOverrides = (Epg, HashSet>); + +impl EpgMergeAccumulator { + pub fn new() -> Self { Self::default() } + + pub(super) fn channel_ids_with_programmes(&self) -> HashSet> { + self.channels + .iter() + .filter(|(id, channel)| !channel.programmes.is_empty() || self.dummy_policies.contains_key(*id)) + .map(|(id, _)| Arc::clone(id)) + .collect() + } + + pub(super) fn retain_channels(&mut self, selected_epg_ids: &HashSet>) -> HashSet> { + self.channels.retain(|id, _| selected_epg_ids.contains(id)); + self.dummy_policies.retain(|id, _| selected_epg_ids.contains(id)); + self.channels.keys().cloned().collect() + } + + pub fn set_attributes_if_preferred( + &mut self, + priority: i16, + source_order: usize, + attributes: Option, Arc>>, + ) { + let Some(attributes) = attributes else { + return; + }; + let replace = self + .attributes + .as_ref() + .is_none_or(|current| (priority, source_order) < (current.priority, current.source_order)); + if replace { + self.attributes = Some(PreferredAttributes { priority, source_order, attributes }); + } + } + + pub(super) fn upsert_channel( + &mut self, + priority: i16, + source_order: usize, + logo_override: bool, + mut channel: EpgChannel, + ) { + let channel_key = with_folded_epg_id(&channel.id, |folded| folded.intern()); + match self.channels.entry(channel_key) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + let acc = entry.get_mut(); + acc.needs_programme_merge = true; + if (priority, source_order) < (acc.priority, acc.source_order) { + if channel.title.is_none() { + channel.title = acc.channel.title.take(); + } + if channel.icon.is_none() { + channel.icon = acc.channel.icon.take(); + } else { + acc.icon_logo_override = logo_override; + } + acc.priority = priority; + acc.source_order = source_order; + acc.logo_override = logo_override; + acc.channel.title = channel.title; + acc.channel.icon = channel.icon; + } else { + if acc.channel.title.is_none() { + acc.channel.title = channel.title.take(); + } + if acc.channel.icon.is_none() { + acc.channel.icon = channel.icon.take(); + acc.icon_logo_override = logo_override; + } + } + } + std::collections::hash_map::Entry::Vacant(entry) => { + let icon_logo_override = channel.icon.is_some() && logo_override; + entry.insert(ChannelMergeAcc { + priority, + source_order, + logo_override, + icon_logo_override, + needs_programme_merge: false, + channel, + programmes: Vec::new(), + }); + } + } + } + + pub(super) fn push_programme(&mut self, priority: i16, source_order: usize, programme: EpgProgramme) { + if let Some(channel) = + with_folded_epg_id(programme.get_transient_channel_id(), |folded| self.channels.get_mut(folded)) + { + channel.programmes.push(ProgrammeMergeEntry { priority, source_order, programme }); + } else { + error!("Channel {} not found in EPG, dangling programme", programme.get_transient_channel_id()); + } + } + + pub(super) fn register_dummy_policy( + &mut self, + channel_id: &Arc, + priority: i16, + source_order: usize, + policy: IcsDummyPolicy, + ) { + if !policy.config.enabled { + return; + } + let key = with_folded_epg_id(channel_id, |folded| folded.intern()); + let replace = self + .dummy_policies + .get(&key) + .is_none_or(|current| (priority, source_order) < (current.priority, current.source_order)); + if replace { + self.dummy_policies.insert(key, PreferredDummyPolicy { priority, source_order, policy }); + } + } + + pub fn add_channel_with_programmes( + &mut self, + priority: i16, + source_order: usize, + logo_override: bool, + mut channel: EpgChannel, + ) { + let programmes = std::mem::take(&mut channel.programmes); + let channel_key = with_folded_epg_id(&channel.id, |folded| folded.intern()); + self.upsert_channel(priority, source_order, logo_override, channel); + if let Some(acc) = self.channels.get_mut(&channel_key) { + if !acc.programmes.is_empty() { + acc.needs_programme_merge = true; + } + acc.programmes.extend(programmes.into_iter().map(|programme| ProgrammeMergeEntry { + priority, + source_order, + programme, + })); + } + } + + pub(super) fn finish_channels(mut self) -> Option { + if self.channels.is_empty() { + return None; + } + + let mut channels = self + .channels + .drain() + .map(|(_, mut acc)| { + normalize_channel_programmes(&mut acc); + acc + }) + .collect::>(); + + channels.sort_by(|left, right| left.channel.id.cmp(&right.channel.id)); + let channels = channels.into_iter().map(|acc| acc.channel).collect(); + Some((self.attributes.map(|attributes| attributes.attributes), channels)) + } + + pub(super) fn finish(self) -> Option { + self.finish_channels().map(|(attributes, channels)| Epg { + logo_override: false, + priority: 0, + attributes, + children: channels.into_iter().map(Arc::new).collect(), + }) + } + + pub(super) fn finish_epg_with_icon_overrides(self) -> Option { + let EpgMergeAccumulator { attributes, channels, dummy_policies } = self; + let mut channels = channels.into_values().collect::>(); + if channels.is_empty() { + return None; + } + + for acc in &mut channels { + normalize_channel_programmes(acc); + } + + apply_dummy_policies(&mut channels, &dummy_policies); + + channels.sort_by(|left, right| left.channel.id.cmp(&right.channel.id)); + let icon_override_channels = channels + .iter() + .filter(|acc| acc.icon_logo_override) + .map(|acc| Arc::clone(&acc.channel.id)) + .collect::>(); + let children = channels.into_iter().map(|acc| Arc::new(acc.channel)).collect(); + + Some(( + Epg { + logo_override: false, + priority: 0, + attributes: attributes.map(|attributes| attributes.attributes), + children, + }, + icon_override_channels, + )) + } +} + +fn apply_dummy_policies(channels: &mut [ChannelMergeAcc], dummy_policies: &HashMap, PreferredDummyPolicy>) { + let now = chrono::Utc::now(); + for acc in channels { + let key = with_folded_epg_id(&acc.channel.id, |folded| folded.intern()); + if let Some(preferred) = dummy_policies.get(&key) { + let policy = &preferred.policy; + if let Err(err) = ics::fill_dummy_gaps( + &mut acc.channel.programmes, + &acc.channel.id, + &policy.timezone, + &policy.config, + now, + ) { + log::warn!("Failed to apply ICS dummy policy for {}: {err}", acc.channel.id); + } + } + } +} + +fn backfill_programme_metadata(existing: &mut EpgProgramme, incoming: EpgProgramme) { + if existing.title.is_none() { + existing.title = incoming.title; + } + if existing.desc.is_none() { + existing.desc = incoming.desc; + } + if existing.icon.is_none() { + existing.icon = incoming.icon; + } + if existing.catchup_id.is_none() { + existing.catchup_id = incoming.catchup_id; + } + if existing.categories.is_empty() { + existing.categories = incoming.categories; + } + existing.is_live |= incoming.is_live; + existing.is_new |= incoming.is_new; +} + +pub(super) fn normalize_channel_programmes(acc: &mut ChannelMergeAcc) { + // `acc.programmes` is the cross-source list of `ProgrammeMergeEntry` records + // (populated by `add_channel_with_programmes` and `push_programme`). + // When the list is empty, the channel's own programmes — set by the + // vacant path of `upsert_channel` — are authoritative; the previous + // implementation unconditionally overwrote `acc.channel.programmes` with + // an empty vector here, which silently dropped every programme on the + // single-source and disk-spill paths. + if acc.programmes.is_empty() { + return; + } + acc.programmes + .sort_by_key(|entry| (entry.programme.start, entry.programme.stop, entry.priority, entry.source_order)); + + let programme_entries = std::mem::take(&mut acc.programmes); + let mut merged_programmes = Vec::with_capacity(programme_entries.len()); + let mut entries = programme_entries.into_iter(); + if let Some(first_entry) = entries.next() { + let mut current = first_entry.programme; + for entry in entries { + if entry.programme.start == current.start && entry.programme.stop == current.stop { + backfill_programme_metadata(&mut current, entry.programme); + } else { + merged_programmes.push(current); + current = entry.programme; + } + } + merged_programmes.push(current); + } + + merged_programmes.sort_by_key(|programme| (programme.start, programme.stop)); + acc.channel.programmes = merged_programmes; +} + +#[cfg(test)] +pub fn merge_epg_channels_by_priority(channels_by_source: Vec<(i16, Vec)>) -> Vec { + let mut accumulator = EpgMergeAccumulator::new(); + for (source_order, (priority, channels)) in channels_by_source.into_iter().enumerate() { + for channel in channels { + accumulator.add_channel_with_programmes(priority, source_order, false, channel); + } + } + accumulator.finish_channels().map(|(_, channels)| channels).unwrap_or_default() +} + +pub fn merge_epg_channels_by_priority_with_dummy_policies( + channels_by_source: Vec<(i16, Vec)>, + dummy_policies: Vec, +) -> Vec { + let mut accumulator = EpgMergeAccumulator::new(); + for (source_order, (priority, channels)) in channels_by_source.into_iter().enumerate() { + for channel in channels { + accumulator.add_channel_with_programmes(priority, source_order, false, channel); + } + } + for source in dummy_policies { + accumulator.register_dummy_policy(&source.channel_id, source.priority, source.source_order, source.policy); + } + accumulator + .finish_epg_with_icon_overrides() + .map(|(epg, _)| epg.children.into_iter().map(Arc::unwrap_or_clone).collect()) + .unwrap_or_default() +} + +pub fn flatten_tvguide(tv_guides: Vec) -> Option { + let mut accumulator = EpgMergeAccumulator::new(); + for (source_order, guide) in tv_guides.into_iter().enumerate() { + accumulator.set_attributes_if_preferred(guide.priority, source_order, guide.attributes); + for channel in guide.children { + accumulator.add_channel_with_programmes( + guide.priority, + source_order, + guide.logo_override, + Arc::unwrap_or_clone(channel), + ); + } + } + accumulator.finish() +} + +/// Merge N temp `BPlusTree`s (one per EPG source) into a single `Epg`, +/// applying dummy policies and the existing priority/sort rules. Sources +/// are consumed one at a time — each `DiskEpgSource`'s temp file is +/// removed by its `Drop` as soon as its iterator is exhausted, so peak RAM +/// is bounded by the largest single source plus the accumulator's +/// per-channel `HashMap`, not the total feed size. +/// +/// Complexity: `O(n_sources + total_channels)`. The per-channel priority +/// resolution happens inside `EpgMergeAccumulator::upsert_channel` +/// (`HashMap` insert), not in a heap. Sources are processed in iteration +/// order, so earlier sources win priority ties for shared channels. +/// +/// The constant-memory guarantee lives on the write side +/// (`EpgMergeAccumulator::finish_into_disk`); this function is bounded by +/// the union's metadata `HashMap` in the accumulator, which is the same +/// shape the in-memory path already carries. +/// +/// The icon-override channel set returned by `finish_epg_with_icon_overrides` +/// is intentionally dropped — the wire-up path persists channels directly, +/// not the override metadata. If a caller later needs that set, the +/// `MergedEpgWithIconOverrides` tuple is already in scope via this +/// function's return type. +pub fn merge_epg_trees(sources: Vec) -> io::Result> { + let mut accumulator = EpgMergeAccumulator::new(); + for source in sources { + if let Some(attrs) = &source.attributes { + accumulator.set_attributes_if_preferred( + source.source_priority, + source.source_order as usize, + Some(attrs.clone()), + ); + } + let mut query = BPlusTreeQuery::::try_new(&source.path) + .map_err(|e| io::Error::other(format!("open temp EPG tree at {}: {e}", source.path.display())))?; + for entry in query.iter() { + let (key, channel) = entry.map_err(|e| io::Error::other(format!("read temp EPG entry: {e}")))?; + // `add_channel_with_programmes` preserves the channel's programmes + // on the merge accumulator; plain `upsert_channel` would drop them + // because the vacant-entry path constructs a fresh `ChannelMergeAcc` + // with `programmes: Vec::new()`. + accumulator.add_channel_with_programmes(key.priority, key.source_order as usize, false, channel); + } + // `source` drops at the end of this iteration, which removes the + // temp file. The query's mmap/buffer is closed first because `query` + // goes out of scope before `source`. + } + Ok(accumulator.finish_epg_with_icon_overrides()) +} diff --git a/backend/processing/src/parser/xmltv/normalize.rs b/backend/processing/src/parser/xmltv/normalize.rs new file mode 100644 index 000000000..5331e7a7f --- /dev/null +++ b/backend/processing/src/parser/xmltv/normalize.rs @@ -0,0 +1,100 @@ +use shared::{ + model::EpgNamePrefix, + utils::{deunicode_string, CONSTANTS}, +}; +use tuliprox_core::model::EpgSmartMatchConfig; + +/// Splits a string at the first delimiter if the prefix matches a known country code. +/// +/// Returns a tuple containing the country code prefix (if found) and the remainder of the string, both trimmed. If no valid prefix is found, returns `None` and the original input. +/// +/// # Examples +/// +/// ```text +/// let delimiters = vec!['.', '-', '_']; +/// let (prefix, rest) = split_by_first_match("US.HBO", &delimiters); +/// assert_eq!(prefix, Some("US")); +/// assert_eq!(rest, "HBO"); +/// +/// let (prefix, rest) = split_by_first_match("HBO", &delimiters); +/// assert_eq!(prefix, None); +/// assert_eq!(rest, "HBO"); +/// ``` +fn split_by_first_match<'a>(input: &'a str, delimiters: &[char]) -> (Option<&'a str>, &'a str) { + let content = input.trim_start_matches(|c: char| !c.is_alphanumeric()); + + for delim in delimiters { + if let Some(index) = content.find(*delim) { + let (left, right) = content.split_at(index); + let right = &right[delim.len_utf8()..].trim(); + if !right.is_empty() { + let prefix = left.trim(); + if CONSTANTS.country_codes.contains(&prefix) { + return (Some(prefix), right.trim()); + } + } + } + } + (None, input) +} + +pub(super) fn name_prefix<'a>(name: &'a str, smart_config: &EpgSmartMatchConfig) -> (&'a str, Option<&'a str>) { + if smart_config.name_prefix != EpgNamePrefix::Ignore { + let (prefix, suffix) = split_by_first_match(name, &smart_config.name_prefix_separator); + if prefix.is_some() { + return (suffix, prefix); + } + } + (name, None) +} + +fn combine(join: &str, left: &str, right: &str) -> String { + let mut combined = String::with_capacity(left.len() + join.len() + right.len()); + combined.push_str(left); + combined.push_str(join); + combined.push_str(right); + combined +} + +pub(super) fn strip_markers(input: &str, markers: &[String]) -> String { + let mut output = String::with_capacity(input.len()); + let mut cursor = 0; + while cursor < input.len() { + let left_is_boundary = cursor == 0 || !input.as_bytes()[cursor - 1].is_ascii_alphanumeric(); + let matched_end = left_is_boundary + .then(|| { + markers.iter().filter(|marker| !marker.is_empty()).find_map(|marker| { + let end = cursor + marker.len(); + let candidate = input.get(cursor..end)?; + (candidate.eq_ignore_ascii_case(marker) + && input.as_bytes().get(end).is_none_or(|byte| !byte.is_ascii_alphanumeric())) + .then_some(end) + }) + }) + .flatten(); + if let Some(end) = matched_end { + cursor = end; + } else { + let next = input[cursor..].chars().next().expect("cursor is within the string"); + output.push(next); + cursor += next.len_utf8(); + } + } + output +} + +/// # Panics +pub fn normalize_channel_name(name: &str, normalize_config: &EpgSmartMatchConfig) -> String { + let normalized = deunicode_string(name.trim()).to_lowercase(); + let (channel_name, suffix) = name_prefix(&normalized, normalize_config); + let stripped_name = strip_markers(channel_name, &normalize_config.strip); + let reconstructed = match suffix { + None => stripped_name, + Some(sfx) => match &normalize_config.name_prefix { + EpgNamePrefix::Ignore => stripped_name, + EpgNamePrefix::Suffix(separator) => combine(separator, &stripped_name, sfx), + EpgNamePrefix::Prefix(separator) => combine(separator, sfx, &stripped_name), + }, + }; + normalize_config.normalize_regex.replace_all(&reconstructed, "").into_owned() +} diff --git a/backend/processing/src/parser/xmltv/tests.rs b/backend/processing/src/parser/xmltv/tests.rs new file mode 100644 index 000000000..c0e585da6 --- /dev/null +++ b/backend/processing/src/parser/xmltv/tests.rs @@ -0,0 +1,67 @@ +use super::TVGuide; +use crate::parser::xmltv::{ + flatten_tvguide, merge_epg_channels_by_priority, merge_epg_channels_by_priority_with_dummy_policies, + normalize_channel_name, EpgDummyPolicySource, EpgMergeAccumulator, +}; +use shared::model::{EpgCategory, EpgChannel, EpgProgramme}; +use std::{collections::HashSet, fs, path::PathBuf, sync::Arc}; +use tempfile::tempdir; +use tuliprox_core::{ + model::{Epg, EpgSmartMatchConfig, IcsDummyConfig, IcsEpgSourceConfig, PersistedEpgSource, PersistedEpgSourceKind}, + utils::FileLockManager, +}; + +/// Run an async test body on a freshly-created multi-threaded tokio +/// runtime. Centralizes the `Runtime::new()...block_on(...)` boilerplate +/// shared by every test in this module that exercises async EPG code. +fn run_async_test(future: F) +where + F: std::future::Future, +{ + tokio::runtime::Runtime::new().unwrap().block_on(future); +} + +fn xmltv_source(file_path: PathBuf, priority: i16, logo_override: bool) -> PersistedEpgSource { + PersistedEpgSource { file_path, priority, logo_override, kind: PersistedEpgSourceKind::Xmltv } +} + +fn dummy_policy_source(priority: i16, source_order: usize, title: &str) -> EpgDummyPolicySource { + EpgDummyPolicySource { + priority, + source_order, + channel_id: "f1.calendar".intern(), + policy: tuliprox_core::model::IcsDummyPolicy { + timezone: "UTC".to_string(), + config: IcsDummyConfig { + enabled: true, + title: title.to_string(), + description: String::new(), + days_past: 0, + days_future: 0, + block_hours: 24, + min_gap_minutes: 1, + }, + }, + } +} + +fn epg_channel(id: &str, title: Option<&str>, icon: Option<&str>, programmes: Vec) -> EpgChannel { + EpgChannel { id: id.intern(), title: title.map(Internable::intern), icon: icon.map(Internable::intern), programmes } +} + +fn epg_programme(id: &str, start: i64, stop: i64, title: Option<&str>, desc: Option<&str>) -> EpgProgramme { + EpgProgramme::new_all(start, stop, id.intern(), title.map(Internable::intern), desc.map(Internable::intern), None) +} + +use crate::processor::EpgIdCache; +use rphonetic::{Encoder, Metaphone}; +use shared::{ + model::{EpgNamePrefix, EpgSmartMatchConfigDto}, + utils::Internable, +}; + +mod lifecycle; +mod playlist; +mod policy; +mod storage; +mod transport; diff --git a/backend/processing/src/parser/xmltv/tests/lifecycle.rs b/backend/processing/src/parser/xmltv/tests/lifecycle.rs new file mode 100644 index 000000000..4fda6dc79 --- /dev/null +++ b/backend/processing/src/parser/xmltv/tests/lifecycle.rs @@ -0,0 +1,16 @@ +/// Smallest testable piece of the disk-spilling path. The Drop guard is +/// what keeps a panic or early return from leaking temp files into /tmp. +/// If this fails, every other disk-spilling test is built on sand. +#[test] +fn disk_epg_source_removes_its_temp_file_on_drop() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("epg-source.db"); + std::fs::write(&path, b"placeholder").unwrap(); + assert!(path.exists(), "precondition: temp file must exist"); + + { + let _src = super::super::DiskEpgSource::new(path.clone(), None, 0, 0); + } // _src dropped here + + assert!(!path.exists(), "DiskEpgSource::Drop must remove its temp file"); +} diff --git a/backend/processing/src/parser/xmltv/tests/playlist.rs b/backend/processing/src/parser/xmltv/tests/playlist.rs new file mode 100644 index 000000000..bbfd84342 --- /dev/null +++ b/backend/processing/src/parser/xmltv/tests/playlist.rs @@ -0,0 +1,528 @@ +use super::*; + +#[test] +fn epg_priority_merge_preserves_high_priority_metadata_and_fills_programme_gaps() { + let merged = merge_epg_channels_by_priority(vec![ + ( + 10, + vec![epg_channel( + "demo.channel", + Some("Low"), + Some("http://low/icon.png"), + vec![epg_programme("demo.channel", 10, 20, Some("Low Show"), None)], + )], + ), + ( + 0, + vec![epg_channel( + "demo.channel", + Some("High"), + Some("http://high/icon.png"), + vec![epg_programme("demo.channel", 20, 30, Some("High Show"), None)], + )], + ), + ]); + + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].title.as_deref(), Some("High")); + assert_eq!(merged[0].icon.as_deref(), Some("http://high/icon.png")); + assert_eq!( + merged[0].programmes.iter().map(|programme| (programme.start, programme.stop)).collect::>(), + vec![(10, 20), (20, 30)], + ); +} + +#[test] +fn epg_priority_merge_backfills_programme_tags() { + let low_priority_categories = vec![ + EpgCategory { value: "Sports".intern(), lang: None }, + EpgCategory { value: "Live".intern(), lang: Some("en".intern()) }, + ]; + let mut low_programme = epg_programme("demo.channel", 10, 20, None, None); + low_programme.icon = Some("https://example.com/programme.jpg".intern()); + low_programme.categories = low_priority_categories.clone(); + low_programme.is_live = true; + low_programme.is_new = true; + + let merged = merge_epg_channels_by_priority(vec![ + ( + 0, + vec![epg_channel( + "demo.channel", + Some("High"), + None, + vec![epg_programme("demo.channel", 10, 20, Some("High Title"), None)], + )], + ), + (10, vec![epg_channel("demo.channel", None, None, vec![low_programme])]), + ]); + + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].programmes.len(), 1); + assert_eq!(merged[0].programmes[0].icon.as_deref(), Some("https://example.com/programme.jpg")); + assert_eq!(merged[0].programmes[0].categories, low_priority_categories); + assert!(merged[0].programmes[0].is_live); + assert!(merged[0].programmes[0].is_new); + + let high_programme_with_categories = { + let mut programme = epg_programme("demo.channel", 30, 40, Some("High Title 2"), None); + programme.categories = vec![EpgCategory { value: "Drama".intern(), lang: None }]; + programme.is_new = true; + programme + }; + let low_programme_extra = { + let mut programme = epg_programme("demo.channel", 30, 40, None, None); + programme.categories = vec![EpgCategory { value: "ShouldNotWin".intern(), lang: None }]; + programme.is_live = true; + programme + }; + let merged = merge_epg_channels_by_priority(vec![ + (0, vec![epg_channel("demo.channel", None, None, vec![high_programme_with_categories])]), + (10, vec![epg_channel("demo.channel", None, None, vec![low_programme_extra])]), + ]); + + assert_eq!(merged.len(), 1); + let programme = &merged[0].programmes[0]; + assert_eq!(programme.categories, vec![EpgCategory { value: "Drama".intern(), lang: None }],); + assert!(programme.is_live); + assert!(programme.is_new); +} + +#[test] +fn epg_priority_merge_keeps_same_priority_deterministic_order() { + let merged = merge_epg_channels_by_priority(vec![ + ( + 0, + vec![epg_channel( + "demo.channel", + Some("First"), + None, + vec![epg_programme("demo.channel", 10, 20, Some("First Title"), None)], + )], + ), + ( + 0, + vec![epg_channel( + "demo.channel", + Some("Second"), + Some("http://second/icon.png"), + vec![ + epg_programme("demo.channel", 10, 20, Some("Second Title"), Some("Second desc")), + epg_programme("demo.channel", 20, 30, Some("Second Programme"), None), + ], + )], + ), + ]); + + assert_eq!(merged[0].title.as_deref(), Some("First")); + assert_eq!(merged[0].icon.as_deref(), Some("http://second/icon.png")); + assert_eq!(merged[0].programmes[0].title.as_deref(), Some("First Title")); + assert_eq!(merged[0].programmes[0].desc.as_deref(), Some("Second desc")); + assert_eq!(merged[0].programmes.len(), 2); +} + +#[test] +fn epg_priority_merge_flatten_uses_same_backfill_rules() { + let flattened = flatten_tvguide(vec![ + Epg { + logo_override: false, + priority: 0, + attributes: None, + children: vec![Arc::new(epg_channel( + "demo.channel", + Some("High"), + None, + vec![epg_programme("demo.channel", 10, 20, Some("High Show"), None)], + ))], + }, + Epg { + logo_override: false, + priority: 10, + attributes: None, + children: vec![Arc::new(epg_channel( + "demo.channel", + Some("Low"), + Some("http://low/icon.png"), + vec![epg_programme("demo.channel", 20, 30, Some("Low Show"), None)], + ))], + }, + ]) + .expect("flattened epg"); + + assert_eq!(flattened.children.len(), 1); + assert_eq!(flattened.children[0].title.as_deref(), Some("High")); + assert_eq!(flattened.children[0].icon.as_deref(), Some("http://low/icon.png")); + assert_eq!(flattened.children[0].programmes.len(), 2); +} + +#[test] +fn epg_priority_merge_flatten_keeps_shared_arc_channels() { + let shared_channel = Arc::new(epg_channel( + "demo.channel", + Some("Shared"), + Some("http://shared/icon.png"), + vec![epg_programme("demo.channel", 10, 20, Some("Shared Show"), None)], + )); + + let flattened = flatten_tvguide(vec![Epg { + logo_override: false, + priority: 0, + attributes: None, + children: vec![Arc::clone(&shared_channel)], + }]) + .expect("flattened epg"); + + assert_eq!(flattened.children.len(), 1); + assert_eq!(flattened.children[0].title.as_deref(), Some("Shared")); + assert_eq!(flattened.children[0].icon.as_deref(), Some("http://shared/icon.png")); +} + +#[test] +fn epg_priority_merge_tracks_icon_override_per_channel() { + let mut accumulator = EpgMergeAccumulator::new(); + accumulator.add_channel_with_programmes( + 0, + 0, + false, + epg_channel("demo.keep", Some("Keep"), Some("http://keep/icon.png"), vec![]), + ); + accumulator.add_channel_with_programmes( + 1, + 1, + true, + epg_channel("demo.override", Some("Override"), Some("http://override/icon.png"), vec![]), + ); + + let (_epg, icon_override_channels) = accumulator.finish_epg_with_icon_overrides().expect("merged epg"); + + let keep_id: Arc = "demo.keep".intern(); + let override_id: Arc = "demo.override".intern(); + + assert!(!icon_override_channels.contains(&keep_id)); + assert!(icon_override_channels.contains(&override_id)); +} + +#[test] +fn epg_priority_merge_preserves_exact_id_match_when_smart_match_is_enabled() { + run_async_test(async move { + let dir = tempdir().unwrap(); + let epg_path = dir.path().join("smart-exact.xml"); + + fs::write( + &epg_path, + r#" + + + Completely Different Name + + + Exact Match Source + +"#, + ) + .unwrap(); + + let mut smart_cfg = EpgSmartMatchConfigDto { enabled: true, ..Default::default() }; + smart_cfg.prepare().expect("smart match config"); + let tv_guide = TVGuide::new(vec![xmltv_source(epg_path, 0, false)]); + let mut id_cache = EpgIdCache::new(Some(&tuliprox_core::model::EpgConfig { + sources: vec![], + smart_match: Some(EpgSmartMatchConfig::from(smart_cfg)), + })); + id_cache.insert_channel_epg_id("demo.channel"); + + let merged = tv_guide.filter_merged(&mut id_cache).await.expect("merged epg"); + + assert_eq!(merged.children.len(), 1); + assert_eq!(merged.children[0].id.as_ref(), "demo.channel"); + assert_eq!(merged.children[0].programmes.len(), 1); + }); +} + +#[test] +fn epg_channel_id_match_is_case_insensitive_and_preserves_guide_case() { + run_async_test(async move { + let dir = tempdir().unwrap(); + let epg_path = dir.path().join("mixed-case.xml"); + + // Guide channel ids are MixedCase and differ in case from the playlist ids. + fs::write( + &epg_path, + r#" + + + Sport Extra + + + CNN + + + Match A + + + Match B + +"#, + ) + .unwrap(); + + let tv_guide = TVGuide::new(vec![xmltv_source(epg_path, 0, false)]); + let mut id_cache = EpgIdCache::new(None); + // Playlist epg ids arrive in a *different* case than the guide. Both origins + // (an Xtream source and a mapper-script literal) go through the same + // `insert_channel_epg_id`, which folds the membership key. + id_cache.insert_channel_epg_id("sport.EXTRA1.de"); // e.g. from an Xtream source + id_cache.insert_channel_epg_id("cnn.us"); // e.g. set by a mapper literal @epg_channel_id = "cnn.us" + + let merged = tv_guide.filter_merged(&mut id_cache).await.expect("merged epg"); + + // Both MixedCase guide channels matched despite the case difference. + assert_eq!(merged.children.len(), 2); + let ids: HashSet<&str> = merged.children.iter().map(|c| c.id.as_ref()).collect(); + // Emitted preserves the guide's ORIGINAL case (not folded). + assert!(ids.contains("Sport.Extra1.DE"), "emitted ids: {ids:?}"); + assert!(ids.contains("CNN.US"), "emitted ids: {ids:?}"); + for channel in &merged.children { + assert_eq!(channel.programmes.len(), 1, "channel {} programmes", channel.id); + } + }); +} + +#[test] +fn epg_programme_channel_match_is_case_insensitive_within_source() { + run_async_test(async move { + let dir = tempdir().unwrap(); + let epg_path = dir.path().join("mixed-case-programme.xml"); + + fs::write( + &epg_path, + r#" + + + Demo + + + Case Variant Programme + +"#, + ) + .unwrap(); + + let tv_guide = TVGuide::new(vec![xmltv_source(epg_path, 0, false)]); + let mut id_cache = EpgIdCache::new(None); + id_cache.insert_channel_epg_id("demo.channel"); + + let merged = tv_guide.filter_merged(&mut id_cache).await.expect("merged epg"); + + assert_eq!(merged.children.len(), 1); + assert_eq!(merged.children[0].id.as_ref(), "Demo.Channel"); + assert_eq!(merged.children[0].programmes.len(), 1); + assert_eq!(merged.children[0].programmes[0].get_transient_channel_id().as_ref(), "demo.channel"); + }); +} + +#[test] +fn ics_dummy_policy_fills_after_real_programme_merge_without_overwriting_xmltv() { + run_async_test(async move { + use chrono::{Datelike, TimeZone, Utc}; + + let dir = tempdir().unwrap(); + let xml_path = dir.path().join("guide.xml"); + let ics_path = dir.path().join("empty.ics"); + let now = Utc::now(); + let real_start = Utc.with_ymd_and_hms(now.year(), now.month(), now.day(), 4, 0, 0).single().expect("start"); + let real_stop = Utc.with_ymd_and_hms(now.year(), now.month(), now.day(), 6, 0, 0).single().expect("stop"); + + fs::write( + &xml_path, + format!( + r#" + + + Formula 1 + + + Real Show + +"#, + start = real_start.format("%Y%m%d%H%M%S %z"), + stop = real_stop.format("%Y%m%d%H%M%S %z"), + ), + ) + .unwrap(); + fs::write(&ics_path, "BEGIN:VCALENDAR\nEND:VCALENDAR\n").unwrap(); + + let guide = TVGuide::new(vec![ + xmltv_source(xml_path, 0, false), + PersistedEpgSource { + file_path: ics_path, + priority: 1, + logo_override: false, + kind: PersistedEpgSourceKind::Ics { + channel_id: "f1.calendar".intern(), + channel_title: Some("Formula 1".intern()), + match_names: Vec::new(), + config: Box::new(IcsEpgSourceConfig { + timezone: "UTC".to_string(), + dummy: IcsDummyConfig { + enabled: true, + title: "No programme".to_string(), + description: String::new(), + days_past: 0, + days_future: 0, + block_hours: 4, + min_gap_minutes: 1, + }, + ..IcsEpgSourceConfig::default() + }), + }, + }, + ]); + let mut id_cache = EpgIdCache::new(None); + id_cache.insert_channel_epg_id("f1.calendar"); + + let merged = guide.filter_merged(&mut id_cache).await.expect("merged"); + let channel = &merged.children[0]; + assert_eq!(channel.id.as_ref(), "f1.calendar"); + assert_eq!( + channel.programmes.iter().filter(|programme| programme.title.as_deref() == Some("Real Show")).count(), + 1 + ); + let dummy_programmes = channel + .programmes + .iter() + .filter(|programme| programme.title.as_deref() == Some("No programme")) + .collect::>(); + assert!(!dummy_programmes.is_empty()); + for dummy in dummy_programmes { + assert!( + dummy.stop <= real_start.timestamp() || dummy.start >= real_stop.timestamp(), + "dummy overlaps real programme: {}-{}", + dummy.start, + dummy.stop + ); + } + }); +} + +#[test] +fn dummy_policy_selection_uses_priority_then_source_order() { + let channel = || EpgChannel { + id: "f1.calendar".intern(), + title: Some("Formula 1".intern()), + icon: None, + programmes: Vec::new(), + }; + let merge = |policies| { + merge_epg_channels_by_priority_with_dummy_policies(vec![(0, vec![channel()])], policies) + .into_iter() + .next() + .expect("merged channel") + }; + + let priority_winner = + merge(vec![dummy_policy_source(10, 0, "Low priority"), dummy_policy_source(-10, 1, "High priority")]); + assert!(!priority_winner.programmes.is_empty()); + assert!(priority_winner.programmes.iter().all(|programme| programme.title.as_deref() == Some("High priority"))); + + let source_order_winner = + merge(vec![dummy_policy_source(0, 2, "Later source"), dummy_policy_source(0, 1, "Earlier source")]); + assert!(!source_order_winner.programmes.is_empty()); + assert!(source_order_winner + .programmes + .iter() + .all(|programme| programme.title.as_deref() == Some("Earlier source"))); +} + +#[test] +fn xmltv_programme_tags_are_extracted() { + run_async_test(async move { + let dir = tempdir().expect("temp dir"); + let epg_path = dir.path().join("programme-tags.xml"); + fs::write( + &epg_path, + r#" + ESPN + + Softball + + + Softball + Sports + + + +"#, + ) + .expect("write XMLTV fixture"); + + let guide = TVGuide::new(vec![xmltv_source(epg_path, 0, false)]); + let mut id_cache = EpgIdCache::new(None); + id_cache.insert_channel_epg_id("ESPN.us"); + let merged = guide.filter_merged(&mut id_cache).await.expect("merged EPG"); + let programme = &merged.children[0].programmes[0]; + + assert_eq!( + programme.categories, + vec![ + EpgCategory { value: "Softball".intern(), lang: Some("en".intern()) }, + EpgCategory { value: "Sports".intern(), lang: None }, + ], + ); + assert_eq!(programme.icon.as_deref(), Some("https://example.com/softball.jpg")); + assert!(programme.is_live); + assert!(programme.is_new); + }); +} + +/// Regression guard for the buffer-preallocation in `parse_tvguide`. +/// +/// `quick_xml` uses the caller-provided `Vec` as a monotonically growing +/// read buffer (it does not call `.clear()` between events; the caller +/// slices the new portion). For an XMLTV feed with a single giant +/// `` description, that buffer used to start at 0 capacity and +/// double ~25 times on the way to ~163 MiB — every doubling being a +/// copying realloc. Starting with `Vec::with_capacity(64 * 1024)` removes +/// the realloc chain. This test feeds a programme whose description is +/// bigger than that initial capacity and checks that parsing still +/// succeeds; the absence of the test would have allowed a silent +/// regression to `Vec::new()` without breaking any visible behaviour. +#[test] +fn parse_tvguide_handles_giant_programme_description() { + use crate::parser::xmltv::parse_tvguide; + use tuliprox_core::model::{EPG_TAG_DESC, EPG_TAG_PROGRAMME}; + + run_async_test(async { + // 200 KiB of text content — well beyond the 64 KiB preallocation. + let big_text = "x".repeat(200 * 1024); + // `channel` attribute is required for the parser to fire the + // callback on a tag (see `handle_tag_end`). + let xml = format!( + r#"t{big_text}"# + ); + let mut emitted_tags: Vec = Vec::new(); + parse_tvguide(xml.as_bytes(), &mut |tag| { + emitted_tags.push(tag); + }) + .await; + + // The whole point of the preallocation is that the full payload + // survives the round trip — not just the presence of a tag. + let programme = emitted_tags + .iter() + .find(|tag| tag.name.as_ref() == EPG_TAG_PROGRAMME) + .expect("parser emitted no tag"); + let children = programme.children.as_deref().expect(" has no children"); + let desc = children + .iter() + .find(|child| child.name.as_ref() == EPG_TAG_DESC) + .expect(" emitted no child"); + assert_eq!( + desc.value.as_deref().map(str::len), + Some(big_text.len()), + " payload was truncated; got {} bytes, expected {}", + desc.value.as_deref().map_or(0, str::len), + big_text.len() + ); + }); +} diff --git a/backend/processing/src/parser/xmltv/tests/policy.rs b/backend/processing/src/parser/xmltv/tests/policy.rs new file mode 100644 index 000000000..97ee83e39 --- /dev/null +++ b/backend/processing/src/parser/xmltv/tests/policy.rs @@ -0,0 +1,135 @@ +use super::*; + +#[test] +/// Tests normalization of a channel name using the default smart match configuration. +/// +/// # Examples +/// +/// ``` +/// parse_normalize().unwrap(); +/// ``` +fn parse_normalize() { + let epg_normalize_dto = EpgSmartMatchConfigDto { ..Default::default() }; + let epg_normalize = EpgSmartMatchConfig::from(epg_normalize_dto); + let normalized = normalize_channel_name("Love Nature", &epg_normalize); + assert_eq!(normalized, "lovenature".to_string()); +} + +#[test] +fn normalization_keeps_country_suffixes_consistent() { + let mut dto = EpgSmartMatchConfigDto { + enabled: true, + name_prefix: EpgNamePrefix::Suffix(".".to_string()), + ..Default::default() + }; + dto.prepare().expect("valid smart-match config"); + let config = EpgSmartMatchConfig::from(dto); + + assert_eq!(normalize_channel_name("FR: TF1 FHD", &config), "tf1.fr"); + assert_eq!(normalize_channel_name("TF1.fr", &config), "tf1.fr"); +} + +#[test] +fn normalization_strips_quality_markers_without_corrupting_names() { + let mut dto = EpgSmartMatchConfigDto { enabled: true, ..Default::default() }; + dto.prepare().expect("valid smart-match config"); + let config = EpgSmartMatchConfig::from(dto); + + assert_eq!(normalize_channel_name("RMC Story H265 50FPS", &config), "rmcstory"); + assert_eq!(normalize_channel_name("Ashdod TV HD", &config), "ashdodtv"); +} + +#[test] +fn marker_stripping_skips_non_utf8_boundaries() { + assert_eq!(super::super::strip_markers("éclair", &["x".to_string()]), "éclair"); +} + +#[ignore = "requires a local XMLTV fixture under /tmp"] +#[test] +fn parse_test() { + let run_test = async move || { + //let file_path = PathBuf::from("/tmp/epg.xml.gz"); + let file_path = PathBuf::from("/tmp/invalid_epg.xml"); + + if file_path.exists() { + let tv_guide = TVGuide::new(vec![xmltv_source(file_path, 0, false)]); + + let mut id_cache = EpgIdCache::new(None); + id_cache.insert_channel_epg_id("342"); + //id_cache.collect_epg_id(fp); + + let channel_ids = HashSet::from([342u32.intern()]); + match tv_guide.filter(&mut id_cache).await { + None => panic!("No epg filtered"), + Some(epgs) => { + for epg in epgs { + assert_eq!(epg.children.len(), channel_ids.len() * 2, "Epg size does not match"); + } + } + } + } + }; + tokio::runtime::Runtime::new().unwrap().block_on(run_test()); +} + +#[test] +/// Tests normalization of channel names with various prefixes, suffixes, and special characters using a configured `EpgSmartMatchConfig`. +/// +/// # Examples +/// +/// ``` +/// normalize(); +/// // This will assert that various channel names are normalized as expected. +/// ``` +fn normalize() { + let mut epg_smart_cfg_dto = EpgSmartMatchConfigDto { + enabled: true, + name_prefix: EpgNamePrefix::Suffix(".".to_string()), + ..Default::default() + }; + let _ = epg_smart_cfg_dto.prepare(); + let epg_smart_cfg = EpgSmartMatchConfig::from(epg_smart_cfg_dto); + println!("{epg_smart_cfg:?}"); + assert_eq!("supersport6.ru", normalize_channel_name("RU: SUPERSPORT 6 ᴿᴬᵂ", &epg_smart_cfg)); + assert_eq!("odisea.sat", normalize_channel_name("SAT: ODISEA ᴿᴬᵂ", &epg_smart_cfg)); + assert_eq!("odisea.4k", normalize_channel_name("4K: ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg)); + assert_eq!("odisea", normalize_channel_name("ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg)); + assert_eq!("odisea.bu", normalize_channel_name("BU | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg)); + assert_eq!("odisea.bg", normalize_channel_name("BG | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg)); +} + +#[test] +/// Demonstrates phonetic encoding (Metaphone) of normalized channel names with various prefixes and suffixes. +/// +/// This test prints the Metaphone-encoded representations of several normalized channel names using a configured `EpgSmartMatchConfig`. +/// +/// # Examples +/// +/// ``` +/// test_metaphone(); +/// // Output will show the Metaphone encodings for different channel name variants. +/// ``` +fn test_metaphone() { + let metaphone = Metaphone::default(); + let mut epg_smart_cfg_dto = EpgSmartMatchConfigDto { + enabled: true, + name_prefix: EpgNamePrefix::Suffix(".".to_string()), + ..Default::default() + }; + let _ = epg_smart_cfg_dto.prepare(); + let epg_smart_cfg = EpgSmartMatchConfig::from(epg_smart_cfg_dto); + println!("{epg_smart_cfg:?}"); + // assert_eq!("supersport6.ru", metaphone.encode(&normalize_channel_name("RU: SUPERSPORT 6 ᴿᴬᵂ", &epg_normalize_cfg))); + // assert_eq!("odisea.sat", metaphone.encode(&normalize_channel_name("SAT: ODISEA ᴿᴬᵂ", &epg_normalize_cfg))); + // assert_eq!("odisea", metaphone.encode(&normalize_channel_name("4K: ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); + // assert_eq!("odisea", metaphone.encode(&normalize_channel_name("ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); + // assert_eq!("odisea.bu", metaphone.encode(&normalize_channel_name("BU | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); + // assert_eq!("odisea.bg", metaphone.encode(&normalize_channel_name("BG | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_normalize_cfg))); + + println!("{}", metaphone.encode(&normalize_channel_name("RU: SUPERSPORT 6 ᴿᴬᵂ", &epg_smart_cfg))); + println!("{}", metaphone.encode(&normalize_channel_name("SAT: ODISEA ᴿᴬᵂ", &epg_smart_cfg))); + println!("{}", metaphone.encode(&normalize_channel_name("4K: ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg))); + println!("{}", metaphone.encode(&normalize_channel_name("ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg))); + println!("{}", metaphone.encode(&normalize_channel_name("BU | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg))); + println!("{}", metaphone.encode(&normalize_channel_name("BG | ODISEA ᵁᴴᴰ ³⁸⁴⁰ᴾ", &epg_smart_cfg))); +} diff --git a/backend/processing/src/parser/xmltv/tests/storage.rs b/backend/processing/src/parser/xmltv/tests/storage.rs new file mode 100644 index 000000000..1d56e0d81 --- /dev/null +++ b/backend/processing/src/parser/xmltv/tests/storage.rs @@ -0,0 +1,224 @@ +use super::*; + +#[test] +fn epg_priority_merge_backfills_duplicate_programme_metadata_without_overwriting() { + let merged = merge_epg_channels_by_priority(vec![ + ( + 0, + vec![epg_channel( + "demo.channel", + Some("High"), + None, + vec![epg_programme("demo.channel", 10, 20, Some("High Title"), None)], + )], + ), + ( + 10, + vec![epg_channel( + "demo.channel", + None, + Some("http://fallback/icon.png"), + vec![epg_programme("demo.channel", 10, 20, Some("Low Title"), Some("Recovered desc"))], + )], + ), + ]); + + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].title.as_deref(), Some("High")); + assert_eq!(merged[0].icon.as_deref(), Some("http://fallback/icon.png")); + assert_eq!(merged[0].programmes.len(), 1); + assert_eq!(merged[0].programmes[0].title.as_deref(), Some("High Title")); + assert_eq!(merged[0].programmes[0].desc.as_deref(), Some("Recovered desc")); +} + +#[test] +fn epg_priority_merge_normalizes_duplicates_within_single_source() { + let merged = merge_epg_channels_by_priority(vec![( + 0, + vec![epg_channel( + "demo.channel", + Some("Demo"), + None, + vec![ + epg_programme("demo.channel", 20, 30, Some("Later"), None), + epg_programme("demo.channel", 10, 20, Some("First"), None), + epg_programme("demo.channel", 10, 20, None, Some("Recovered desc")), + ], + )], + )]); + + assert_eq!(merged.len(), 1); + assert_eq!( + merged[0].programmes.iter().map(|programme| (programme.start, programme.stop)).collect::>(), + vec![(10, 20), (20, 30)], + ); + assert_eq!(merged[0].programmes[0].title.as_deref(), Some("First")); + assert_eq!(merged[0].programmes[0].desc.as_deref(), Some("Recovered desc")); +} + +#[test] +fn persisted_epg_source_read_uses_shared_file_lock() { + run_async_test(async move { + let dir = tempdir().expect("temp dir"); + let epg_path = dir.path().join("locked.xml"); + fs::write( + &epg_path, + r#" + Demo + + Locked read + +"#, + ) + .expect("write XMLTV fixture"); + + let file_locks = Arc::new(FileLockManager::new()); + let guide = + TVGuide::new(vec![xmltv_source(epg_path.clone(), 0, false)]).with_file_locks(Arc::clone(&file_locks)); + let mut id_cache = EpgIdCache::new(None); + id_cache.insert_channel_epg_id("demo.channel"); + let write_guard = file_locks.write_lock(&epg_path).await; + let filter = guide.filter_merged(&mut id_cache); + tokio::pin!(filter); + + assert!(tokio::time::timeout(std::time::Duration::from_millis(25), filter.as_mut()).await.is_err()); + drop(write_guard); + + let merged = filter.await.expect("EPG parse after write lock release"); + assert_eq!(merged.children[0].programmes[0].title.as_deref(), Some("Locked read")); + }); +} + +/// Builds an accumulator with N channels and drains it to disk. Asserts: +/// (a) the temp file exists while the guard is alive, (b) it has +/// non-trivial size, (c) the file is removed after Drop. Together these +/// prove the writer runs end-to-end without panicking. +#[test] +fn finish_into_disk_writes_a_real_temp_tree() { + use super::super::EpgMergeAccumulator; + + let mut acc = EpgMergeAccumulator::new(); + for i in 0..250u32 { + let id: Arc = format!("channel-{i:04}").into(); + let ch = shared::model::EpgChannel { + id: Arc::clone(&id), + title: Some(format!("title {i}").into()), + icon: None, + programmes: vec![shared::model::EpgProgramme::new(i64::from(i), i64::from(i + 1), id)], + }; + acc.upsert_channel(i16::try_from(i).unwrap_or(0), 0, false, ch); + } + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("epg-source.db"); + let src = acc.finish_into_disk(path.clone(), 5, 0).unwrap(); + assert!(path.exists(), "temp tree file must exist while guard is alive"); + let size = std::fs::metadata(&path).unwrap().len(); + assert!(size > 1024, "expected non-trivial size, got {size}"); + drop(src); + assert!(!path.exists(), "Drop must remove the temp file"); +} + +/// Contract test: two temp trees written from two accumulators must +/// merge into the same `Epg` as the in-memory `finish_epg_with_icon_overrides` +/// would produce from the same inputs. Without this, every other +/// optimisation is built on sand. +#[test] +fn disk_path_matches_in_memory_finish() { + use super::super::{merge_epg_trees, EpgMergeAccumulator}; + + // Build one source by appending one programme per channel through the + // accumulator's primary entry point. `add_channel_with_programmes` is + // the same call the in-memory `merge_epg_channels_by_priority` uses, + // so the in-memory reference and the disk path go through identical + // APIs. The two sources use disjoint programme intervals + // (source 0: 0..1, source 1: 100..101) so the merged channel should + // retain both programmes end-to-end. + fn build_acc(source: usize, channels: std::ops::Range) -> EpgMergeAccumulator { + let mut acc = EpgMergeAccumulator::new(); + for i in channels { + let id: Arc = format!("ch-{i:04}").into(); + let priority = if source == 0 { 5 } else { 3 }; // source 1 wins + let (start, stop) = + if source == 0 { (i64::from(i), i64::from(i + 1)) } else { (100 + i64::from(i), 101 + i64::from(i)) }; + acc.add_channel_with_programmes( + i16::try_from(priority).unwrap_or(0), + source, + false, + shared::model::EpgChannel { + id: Arc::clone(&id), + title: Some(format!("title-{source}-{i}").into()), + icon: None, + programmes: vec![shared::model::EpgProgramme::new(start, stop, id)], + }, + ); + } + acc + } + + // Reference: in-memory merge of the same two sources. + let ref_acc_a = build_acc(0, 0..50); + let ref_acc_b = build_acc(1, 0..50); + // `EpgMergeAccumulator` is single-shot, so the reference merge has to + // rebuild a fresh accumulator. The two halves contribute disjoint + // programme intervals, so a single accumulator sees both. + let mut ref_acc = EpgMergeAccumulator::new(); + for i in 0..50 { + let id: Arc = format!("ch-{i:04}").into(); + ref_acc.add_channel_with_programmes( + 5, + 0, + false, + shared::model::EpgChannel { + id: Arc::clone(&id), + title: Some(format!("title-0-{i}").into()), + icon: None, + programmes: vec![shared::model::EpgProgramme::new(i64::from(i), i64::from(i + 1), id)], + }, + ); + } + for i in 0..50 { + let id: Arc = format!("ch-{i:04}").into(); + ref_acc.add_channel_with_programmes( + 3, + 1, + false, + shared::model::EpgChannel { + id: Arc::clone(&id), + title: Some(format!("title-1-{i}").into()), + icon: None, + programmes: vec![shared::model::EpgProgramme::new(100 + i64::from(i), 101 + i64::from(i), id)], + }, + ); + } + let reference = ref_acc.finish_epg_with_icon_overrides().unwrap().0; + let _ = (ref_acc_a, ref_acc_b); // keep the per-source builder API in the test + + // Disk path: two temp trees, then merge. + let dir = tempfile::tempdir().unwrap(); + let src_a = build_acc(0, 0..50).finish_into_disk(dir.path().join("a.db"), 5, 0).unwrap(); + let src_b = build_acc(1, 0..50).finish_into_disk(dir.path().join("b.db"), 3, 1).unwrap(); + let merged = merge_epg_trees(vec![src_a, src_b]).unwrap().unwrap().0; + + assert_eq!(reference.children.len(), merged.children.len(), "channel counts must match"); + for (left, right) in reference.children.iter().zip(merged.children.iter()) { + assert_eq!(left.id, right.id, "channel order must match"); + // Source 1 had priority 3 (lower = wins) so its title should win. + assert_eq!(left.title, right.title, "priority winner's title must propagate"); + // Both sources contribute distinct, non-overlapping programmes. + // The merged channel must keep both — the disk path uses + // `add_channel_with_programmes` so nothing is dropped on the way in. + assert_eq!( + left.programmes.len(), + 2, + "channel {} should retain both source programmes, got {}", + left.id, + left.programmes.len() + ); + assert_eq!(right.programmes.len(), 2, "disk-merged channel {} lost programmes", right.id); + for (lp, rp) in left.programmes.iter().zip(right.programmes.iter()) { + assert_eq!(lp.start, rp.start, "programme start must match for channel {}", left.id); + assert_eq!(lp.stop, rp.stop, "programme stop must match for channel {}", left.id); + } + } +} diff --git a/backend/processing/src/parser/xmltv/tests/transport.rs b/backend/processing/src/parser/xmltv/tests/transport.rs new file mode 100644 index 000000000..dbd38eee8 --- /dev/null +++ b/backend/processing/src/parser/xmltv/tests/transport.rs @@ -0,0 +1,52 @@ +use super::*; + +#[test] +fn epg_priority_merge_filter_accepts_later_source_for_already_processed_channel() { + run_async_test(async move { + let dir = tempdir().unwrap(); + let high_path = dir.path().join("high.xml"); + let low_path = dir.path().join("low.xml"); + + fs::write( + &high_path, + r#" + + + High Channel + + + High Source + +"#, + ) + .unwrap(); + fs::write( + &low_path, + r#" + + + Low Channel + + + Low Source + +"#, + ) + .unwrap(); + + let tv_guide = TVGuide::new(vec![xmltv_source(high_path, 0, false), xmltv_source(low_path, 10, false)]); + let mut id_cache = EpgIdCache::new(None); + id_cache.insert_channel_epg_id("demo.channel"); + + let merged = tv_guide.filter_merged(&mut id_cache).await.expect("merged epg"); + + assert!(id_cache.contains_processed_epg_id("demo.channel")); + assert_eq!(merged.children.len(), 1); + assert_eq!(merged.children[0].title.as_deref(), Some("High Channel")); + assert_eq!(merged.children[0].programmes.len(), 2); + assert_eq!( + merged.children[0].programmes.iter().filter_map(|programme| programme.title.as_deref()).collect::>(), + vec!["High Source", "Low Source"], + ); + }); +} diff --git a/backend/processing/src/parser/xmltv/xml.rs b/backend/processing/src/parser/xmltv/xml.rs new file mode 100644 index 000000000..b91fabd89 --- /dev/null +++ b/backend/processing/src/parser/xmltv/xml.rs @@ -0,0 +1,373 @@ +use super::{ + merge::{collect_tag_attributes, get_tag_type}, + normalize_channel_name, EpgMergeAccumulator, TVGuide, +}; +use crate::processor::EpgIdCache; +use log::error; +use quick_xml::events::{BytesStart, BytesText, Event}; +use shared::{ + concat_string, + model::{EpgCategory, EpgChannel, EpgProgramme}, + utils::Internable, +}; +use std::{borrow::Cow, collections::HashSet, sync::Arc}; +use tokio::io::AsyncRead; +use tuliprox_core::{ + model::{ + PersistedEpgSource, XmlTag, XmlTagIcon, EPG_ATTRIB_CHANNEL, EPG_ATTRIB_ID, EPG_ATTRIB_LANG, EPG_TAG_CATEGORY, + EPG_TAG_CHANNEL, EPG_TAG_DESC, EPG_TAG_DISPLAY_NAME, EPG_TAG_ICON, EPG_TAG_LIVE, EPG_TAG_NEW, + EPG_TAG_PREVIOUSLY_SHOWN, EPG_TAG_PROGRAMME, EPG_TAG_TITLE, EPG_TAG_TV, + }, + utils::{ + async_file_reader, compressed_file_reader_async::CompressedFileReaderAsync, parse_xmltv_time, + with_folded_epg_id, + }, +}; + +impl TVGuide { + pub(super) fn prepare_tag(id_cache: &mut EpgIdCache, tag: &mut XmlTag, smart_match: bool) { + { + let maybe_epg_id = { tag.get_attribute_value(&EPG_ATTRIB_ID.intern()).cloned() }; + if let Some(epg_id) = maybe_epg_id { + tag.normalized_epg_ids + .get_or_insert_with(Vec::new) + .push(normalize_channel_name(&epg_id, &id_cache.smart_match_config).intern()); + } + } + + if let Some(children) = &tag.children { + let src = "src".intern(); + for child in children { + match child.name.as_ref() { + EPG_TAG_DISPLAY_NAME if smart_match => { + if let Some(name) = &child.value { + tag.normalized_epg_ids + .get_or_insert_with(Vec::new) + .push(normalize_channel_name(name, &id_cache.smart_match_config).intern()); + } + } + EPG_TAG_ICON => { + if let Some(src) = child.get_attribute_value(&src) { + if !src.is_empty() { + tag.icon = XmlTagIcon::Src(src.clone()); + // We cannot easily modify the child icon since it's inside Arc, + // but we already set the tag.icon, which is what matters. + } + } + } + _ => {} + } + } + } + } + + pub(super) fn channel_display_name(tag: &XmlTag) -> Option> { + tag.children.as_ref().and_then(|children| { + children + .iter() + .find(|child| child.name.as_ref() == EPG_TAG_DISPLAY_NAME) + .and_then(|child| child.value.clone()) + }) + } + + pub(super) fn channel_icon(tag: &XmlTag) -> Option> { + match &tag.icon { + XmlTagIcon::Src(src) => Some(Arc::clone(src)), + XmlTagIcon::Undefined | XmlTagIcon::Exists => None, + } + } + + pub(super) fn extract_programme( + tag: &XmlTag, + epg_id: &Arc, + start_attrib: &Arc, + stop_attrib: &Arc, + catchup_id_attrib: &Arc, + ) -> Option { + let Some((Some(start), Some(stop))) = + tag.attributes.as_ref().map(|a| (a.get(start_attrib), a.get(stop_attrib))) + else { + error!("Missing start or stop attribute in programme tag, skipping"); + return None; + }; + + let (Some(start_time), Some(stop_time)) = (parse_xmltv_time(start), parse_xmltv_time(stop)) else { + error!("Failed to parse epg programme time {start} - {stop}"); + return None; + }; + + let mut title = None; + let mut desc = None; + let mut icon = None; + let mut categories = Vec::new(); + let mut is_live = false; + let mut is_new = false; + let mut previously_shown = false; + if let Some(children) = tag.children.as_ref() { + for child in children { + match child.name.as_ref() { + EPG_TAG_TITLE => title.clone_from(&child.value), + EPG_TAG_DESC => desc.clone_from(&child.value), + EPG_TAG_ICON => { + if let Some(src) = child + .attributes + .as_ref() + .and_then(|attributes| attributes.get("src")) + .filter(|src| !src.is_empty()) + { + icon = Some(Arc::clone(src)); + } + } + EPG_TAG_CATEGORY => { + if let Some(value) = child.value.as_ref().filter(|value| !value.is_empty()) { + categories.push(EpgCategory { + value: Arc::clone(value), + lang: child + .attributes + .as_ref() + .and_then(|attributes| attributes.get(EPG_ATTRIB_LANG)) + .cloned(), + }); + } + } + EPG_TAG_LIVE => is_live = true, + EPG_TAG_NEW => is_new = true, + EPG_TAG_PREVIOUSLY_SHOWN => previously_shown = true, + _ => {} + } + } + } + + let catchup_id = tag.attributes.as_ref().and_then(|attributes| attributes.get(catchup_id_attrib)).cloned(); + + let mut programme = EpgProgramme::new_all(start_time, stop_time, Arc::clone(epg_id), title, desc, catchup_id); + programme.icon = icon; + programme.categories = categories; + programme.is_live = is_live; + programme.is_new = is_new; + programme.previously_shown = previously_shown; + Some(programme) + } + + /// Parses and filters a compressed EPG XML file, extracting relevant channel and program tags based on smart and fuzzy matching criteria. + /// + /// Returns an `Epg` containing filtered tags and TV attributes if any matching channels are found; otherwise, returns `None`. + /// The returned `Epg` will include the priority from the source, which is used for merging multiple EPG sources. + /// + /// # Examples + /// + /// ```text + /// let mut id_cache = EpgIdCache::default(); + /// let epg_source = PersistedEpgSource { file_path: Path::new("guide.xml.gz"), priority: 0 }; + /// if let Some(epg) = process_epg_file(&mut id_cache, &epg_source) { + /// assert!(!epg.children.is_empty()); + /// } + /// ``` + pub(super) async fn process_epg_file( + id_cache: &mut EpgIdCache, + epg_source: &PersistedEpgSource, + source_order: usize, + accumulator: &mut EpgMergeAccumulator, + ) -> bool { + let epg_attrib_id = EPG_ATTRIB_ID.intern(); + let epg_attrib_channel = EPG_ATTRIB_CHANNEL.intern(); + let start_attrib = "start".intern(); + let stop_attrib = "stop".intern(); + let catchup_id_attrib = "catchup-id".intern(); + + match CompressedFileReaderAsync::new(&epg_source.file_path).await { + Ok(mut reader) => { + let mut source_processed: HashSet> = HashSet::with_capacity(5000); + let mut accepted_channels = 0usize; + let smart_match = id_cache.smart_match_config.enabled; + let mut filter_tags = |mut tag: XmlTag| { + match tag.name.as_ref() { + EPG_TAG_CHANNEL => { + let tag_epg_id = + tag.get_attribute_value(&epg_attrib_id).map_or_else(|| "".intern(), Internable::intern); + if tag_epg_id.is_empty() { + return; + } + + Self::prepare_tag(id_cache, &mut tag, smart_match); + if smart_match && id_cache.needs_guide_names(&tag_epg_id) { + id_cache.register_guide_names( + &tag_epg_id, + tag.children.iter().flatten().filter_map(|child| { + (child.name.as_ref() == EPG_TAG_DISPLAY_NAME) + .then_some(child.value.as_ref()) + .flatten() + }), + ); + } + // Case-insensitive (ASCII) membership: fold the guide id for the + // lookup only; `tag_epg_id` keeps its original case for output. + let add_channel = if smart_match { + let direct_match = id_cache.contains_channel_epg_id(&tag_epg_id); + let normalized_match = tag.normalized_epg_ids.as_ref().is_some_and(|candidates| { + id_cache.match_epg_channel_candidates( + &tag_epg_id, + candidates, + epg_source.priority, + source_order, + ) + }); + direct_match || normalized_match + } else { + id_cache.contains_channel_epg_id(&tag_epg_id) + }; + + if add_channel { + with_folded_epg_id(&tag_epg_id, |folded| source_processed.insert(folded.intern())); + id_cache.insert_processed_epg_id(&tag_epg_id); + accumulator.upsert_channel( + epg_source.priority, + source_order, + epg_source.logo_override, + EpgChannel { + id: Arc::clone(&tag_epg_id), + title: Self::channel_display_name(&tag), + icon: Self::channel_icon(&tag), + programmes: vec![], + }, + ); + accepted_channels += 1; + } + } + EPG_TAG_PROGRAMME => { + if let Some(epg_id) = tag.get_attribute_value(&epg_attrib_channel) { + if with_folded_epg_id(epg_id, |folded| source_processed.contains(folded)) { + if let Some(programme) = Self::extract_programme( + &tag, + epg_id, + &start_attrib, + &stop_attrib, + &catchup_id_attrib, + ) { + accumulator.push_programme(epg_source.priority, source_order, programme); + } + } + } + } + EPG_TAG_TV => { + accumulator.set_attributes_if_preferred(epg_source.priority, source_order, tag.attributes); + } + _ => {} + } + }; + + parse_tvguide(&mut reader, &mut filter_tags).await; + accepted_channels > 0 + } + Err(e) => { + log::warn!("Failed to process EPG file {}: {e}", epg_source.file_path.display()); + false + } + } + } +} + +fn handle_tag_start(callback: &mut F, stack: &mut Vec, e: &BytesStart) +where + F: FnMut(XmlTag), +{ + let binding = e.name(); + let name_raw = String::from_utf8_lossy(binding.as_ref()); + let name = name_raw.intern(); + let tag_type = get_tag_type(&name); + let attributes = collect_tag_attributes(e); + let attribs = if attributes.is_empty() { None } else { Some(attributes) }; + let tag = XmlTag::new(name, attribs); + + if tag_type.is_tv() { + callback(tag); + } else { + stack.push(tag); + } +} + +pub(super) fn handle_tag_end(callback: &mut F, stack: &mut Vec) +where + F: FnMut(XmlTag), +{ + if !stack.is_empty() { + if let Some(tag) = stack.pop() { + if tag.name.as_ref() == EPG_TAG_CHANNEL { + if let Some(chan_id) = tag.get_attribute_value(&EPG_ATTRIB_ID.intern()) { + if !chan_id.is_empty() { + callback(tag); + } + } + } else if tag.name.as_ref() == EPG_TAG_PROGRAMME { + if let Some(chan_id) = tag.get_attribute_value(&EPG_ATTRIB_CHANNEL.intern()) { + if !chan_id.is_empty() { + callback(tag); + } + } + } else if !stack.is_empty() { + let tag_arc = Arc::new(tag); + if let Some(mut parent) = stack.pop() { + parent.children.get_or_insert_with(Vec::new).push(tag_arc); + stack.push(parent); + } + } + } + } +} + +fn handle_text_tag(stack: &mut [XmlTag], e: &BytesText) { + if let Some(tag) = stack.last_mut() { + if let Ok(text) = e.decode() { + let t = text.trim(); + if !t.is_empty() { + let t_fixed: Cow = if t.ends_with('\\') { + let mut owned = t.to_string(); + owned.pop(); + owned.push_str("' "); + Cow::Owned(owned) + } else { + Cow::Borrowed(t) + }; + + tag.value = Some(match tag.value.take() { + None => t_fixed.intern(), + Some(old) => concat_string!(old.as_ref(), t_fixed.as_ref()).intern(), + }); + } + } + } +} + +pub async fn parse_tvguide(content: R, callback: &mut F) +where + R: AsyncRead + Unpin, + F: FnMut(XmlTag), +{ + let mut stack: Vec = vec![]; + let mut xml_reader = quick_xml::reader::Reader::from_reader(async_file_reader(content)); + // Pre-allocate so the first giant `` block does not trigger a + // chain of `Vec::grow` reallocs. The buffer is monotonically grown by + // quick_xml (it is reset by the caller, see below), so the eventual size + // is the largest single event — for XMLTV that is the biggest programme + // description. Starting at 0 capacity makes that growth ~25 doubling copies. + let mut buf = Vec::::with_capacity(64 * 1024); + loop { + match xml_reader.read_event_into_async(&mut buf).await { + Ok(Event::Eof) => break, + Ok(Event::Start(e)) => handle_tag_start(callback, &mut stack, &e), + Ok(Event::Empty(e)) => { + handle_tag_start(callback, &mut stack, &e); + handle_tag_end(callback, &mut stack); + } + Ok(Event::End(_e)) => handle_tag_end(callback, &mut stack), + Ok(Event::Text(e)) => handle_text_tag(&mut stack, &e), + _ => {} + } + // quick_xml does not clear the buffer between events — the borrow + // returned by `read_event_into_async` is dropped at the end of each + // match arm, so we can reclaim the capacity here without invalidating + // any handler output. Without this, `buf` grows monotonically over the + // whole file, defeating the 64 KiB pre-allocation above. + buf.clear(); + } +} diff --git a/backend/processing/src/processor/playlist/ingest.rs b/backend/processing/src/processor/playlist/ingest.rs index 1e220dc11..10afe8381 100644 --- a/backend/processing/src/processor/playlist/ingest.rs +++ b/backend/processing/src/processor/playlist/ingest.rs @@ -1,33 +1,10 @@ #![allow(clippy::wildcard_imports)] + use super::{ fetch_outcome::{apply_playlist_fetch_outcome, report_force_updates, CacheStatusScope}, *, }; -// Inputs disabled in the config are always disabled. -// Command-line targets can only restrict enabled inputs, never enable them. -pub(crate) fn is_input_enabled(input: &ConfigInput, user_targets: &ProcessTargets) -> bool { - input.enabled && (!user_targets.enabled || user_targets.has_input(input.id)) -} - -pub(crate) async fn with_sequential_group( - file_locks: &tuliprox_core::utils::FileLockManager, - group: Option, - process_parallel: bool, - future: impl std::future::Future, -) -> T { - let _guard = if process_parallel { - if let Some(group) = group { - Some(file_locks.write_lock_str(&format!("sequential_group:{group}")).await) - } else { - None - } - } else { - None - }; - future.await -} - pub(crate) struct PlaylistDownloadResult { pub downloaded_playlist: Vec, pub download_err: Vec, @@ -48,247 +25,6 @@ struct PendingInputStatusUpdate { cluster_status_source: ClusterStatusSource, } -#[derive(Clone, Copy)] -enum ClusterStatusSource { - PerCluster, - Default, -} - -#[derive(Clone, Copy, Default)] -enum ProviderFailureEvidence { - #[default] - None, - TechnicalFailure, -} - -impl PlaylistDownloadResult { - pub fn new( - downloaded_playlist: Vec, - download_err: Vec, - was_cached: bool, - persisted: bool, - ) -> Self { - Self { - downloaded_playlist, - download_err, - quality_rejections: Vec::new(), - force_updates: Vec::new(), - was_cached, - persisted, - partial: false, - input_telemetry: None, - pending_status: None, - provider_failure_evidence: ProviderFailureEvidence::None, - } - } - - fn with_input_telemetry(mut self, input_telemetry: PlaylistUpdateInputTelemetry) -> Self { - self.input_telemetry = Some(input_telemetry); - self - } - - fn with_pending_status(mut self, pending_status: PendingInputStatusUpdate) -> Self { - self.pending_status = Some(pending_status); - self - } -} - -impl From for PlaylistDownloadResult { - fn from(fetch: PlaylistFetch) -> Self { - let provider_failure_evidence = if fetch.partial || !fetch.errors.is_empty() { - ProviderFailureEvidence::TechnicalFailure - } else { - ProviderFailureEvidence::None - }; - Self { - downloaded_playlist: fetch.groups, - download_err: fetch.errors, - quality_rejections: fetch.quality_rejections, - force_updates: fetch.force_updates, - was_cached: false, - persisted: fetch.persisted, - partial: fetch.partial, - input_telemetry: None, - pending_status: None, - provider_failure_evidence, - } - } -} - -pub(super) const PIPELINE_TRANSPARENCY_CLUSTERS: [XtreamCluster; 3] = - [XtreamCluster::Live, XtreamCluster::Video, XtreamCluster::Series]; - -fn is_cluster_input(input_type: InputType) -> bool { - matches!(input_type, InputType::Xtream | InputType::Stalker | InputType::M3u) -} - -pub(super) fn cluster_is_configured(input: &ConfigInput, cluster: XtreamCluster) -> bool { - let skip_flag = match cluster { - XtreamCluster::Live => ConfigInputFlags::SkipLive, - XtreamCluster::Video => ConfigInputFlags::SkipVod, - XtreamCluster::Series => ConfigInputFlags::SkipSeries, - }; - !input.has_flag(skip_flag) -} - -fn input_telemetry_for_fetch( - input: &ConfigInput, - refresh_policy: InputRefreshPolicy, - source: PlaylistUpdateDataSource, - provider_clusters: &[XtreamCluster], - fetch: Option<&PlaylistFetch>, -) -> PlaylistUpdateInputTelemetry { - let input_type = input.get_download_input_type(); - if !is_cluster_input(input_type) { - return PlaylistUpdateInputTelemetry { refresh_policy, source: Some(source), clusters: Vec::new() }; - } - - let mut candidate_counts = HashMap::::new(); - if let Some(fetch) = fetch { - for group in &fetch.groups { - *candidate_counts.entry(group.xtream_cluster).or_default() += group.channels.len(); - } - if fetch.errors.is_empty() && !fetch.partial && !fetch.persisted { - for cluster in provider_clusters { - candidate_counts.entry(*cluster).or_default(); - } - } - } - - let clusters = PIPELINE_TRANSPARENCY_CLUSTERS - .into_iter() - .map(|cluster| { - let requested = cluster_is_configured(input, cluster); - let source = requested.then(|| { - if provider_clusters.contains(&cluster) { - PlaylistUpdateDataSource::Provider - } else { - PlaylistUpdateDataSource::Cache - } - }); - let configured_threshold = - input.options.as_ref().map_or(0, |options| options.update_quality.threshold(cluster)); - let mut telemetry = PlaylistUpdateClusterTelemetry { - cluster, - requested, - source, - baseline_count: None, - candidate_count: candidate_counts.get(&cluster).copied(), - active_count: None, - threshold: (configured_threshold > 0).then_some(configured_threshold), - quality: None, - decision: None, - technical_state: None, - }; - - if let Some(fetch) = fetch { - if requested && fetch.failed_clusters.contains(&cluster) { - telemetry.technical_state = Some(PersistedPlaylistUpdateTechnicalState::Failed); - } - if let Some(rejection) = fetch.quality_rejections.iter().find(|item| item.cluster == cluster) { - telemetry.baseline_count = Some(rejection.current_count); - telemetry.candidate_count = Some(rejection.candidate_count); - telemetry.active_count = Some(rejection.current_count); - telemetry.threshold = Some(rejection.threshold); - telemetry.quality = Some(rejection.quality); - telemetry.decision = Some(PlaylistUpdateClusterDecision::Rejected); - } else if let Some(acceptance) = fetch.quality_acceptances.iter().find(|item| item.cluster == cluster) { - telemetry.baseline_count = acceptance.current_count; - telemetry.candidate_count = Some(acceptance.candidate_count); - telemetry.threshold = Some(acceptance.threshold); - telemetry.quality = acceptance.quality; - telemetry.decision = Some(PlaylistUpdateClusterDecision::Accepted); - } else if let Some(force_update) = fetch.force_updates.iter().find(|item| item.cluster == cluster) { - telemetry.candidate_count = Some(force_update.candidate_count); - telemetry.threshold = - (force_update.configured_threshold > 0).then_some(force_update.configured_threshold); - telemetry.decision = Some(PlaylistUpdateClusterDecision::Accepted); - } else if source == Some(PlaylistUpdateDataSource::Provider) { - if configured_threshold == 0 - && (candidate_counts.contains_key(&cluster) || (fetch.errors.is_empty() && !fetch.partial)) - { - telemetry.decision = Some(PlaylistUpdateClusterDecision::Accepted); - } else if (provider_clusters == [cluster] || input_type == InputType::M3u) - && fetch.groups.is_empty() - && !fetch.persisted - && !fetch.errors.is_empty() - { - telemetry.decision = Some(PlaylistUpdateClusterDecision::TechnicalError); - } - } - } - telemetry - }) - .collect(); - - PlaylistUpdateInputTelemetry { refresh_policy, source: None, clusters } -} - -fn confirm_published_cluster_counts(input_telemetry: &mut PlaylistUpdateInputTelemetry) { - for cluster in &mut input_telemetry.clusters { - if cluster.decision == Some(PlaylistUpdateClusterDecision::Accepted) { - cluster.active_count = cluster.candidate_count; - } - } -} - -fn finalize_input_telemetry( - playlist_download_result: &mut PlaylistDownloadResult, - storage_error: Option<&TuliproxError>, -) { - let activation_is_uncertain = storage_error.is_some() - || playlist_download_result.partial - || !playlist_download_result.download_err.is_empty(); - if let Some(input_telemetry) = playlist_download_result.input_telemetry.as_mut() { - if activation_is_uncertain { - for cluster in &mut input_telemetry.clusters { - cluster.active_count = None; - } - } else { - confirm_published_cluster_counts(input_telemetry); - } - } -} - -pub(crate) fn neutralize_overlaid_cluster_facts( - input_telemetry: &mut PlaylistUpdateInputTelemetry, - overlaid_clusters: ClusterFlags, -) { - for cluster in &mut input_telemetry.clusters { - if cluster_selected(cluster.cluster, overlaid_clusters) { - cluster.source = None; - cluster.baseline_count = None; - cluster.candidate_count = None; - cluster.active_count = None; - cluster.quality = None; - cluster.decision = None; - cluster.technical_state = None; - } - } -} - -fn persisted_quality_snapshot( - cluster: &PlaylistUpdateClusterTelemetry, - effective_policy: InputRefreshPolicy, -) -> Option { - if effective_policy.bypasses_quality() { - return None; - } - let threshold = cluster.threshold?; - let decision = match cluster.decision? { - PlaylistUpdateClusterDecision::Accepted => PersistedPlaylistUpdateQualityDecision::Accepted, - PlaylistUpdateClusterDecision::Rejected => PersistedPlaylistUpdateQualityDecision::Rejected, - PlaylistUpdateClusterDecision::TechnicalError => return None, - }; - Some(PersistedPlaylistUpdateQualitySnapshot { - threshold, - baseline_count: cluster.baseline_count, - candidate_count: cluster.candidate_count, - achieved_quality: cluster.quality, - decision, - }) -} - #[derive(Clone, Copy)] struct PersistedSnapshotContext { effective_policy: InputRefreshPolicy, @@ -299,403 +35,6 @@ struct PersistedSnapshotContext { provider_cluster_count: usize, } -fn persisted_technical_state( - cluster: &PlaylistUpdateClusterTelemetry, - context: PersistedSnapshotContext, -) -> Option { - let source = cluster.source?; - if cluster.technical_state == Some(PersistedPlaylistUpdateTechnicalState::Failed) { - return cluster.technical_state; - } - if cluster.decision == Some(PlaylistUpdateClusterDecision::TechnicalError) { - return Some(PersistedPlaylistUpdateTechnicalState::Failed); - } - if context.storage_failure && source == PlaylistUpdateDataSource::Provider { - return Some(PersistedPlaylistUpdateTechnicalState::Failed); - } - if !context.technical_failure { - return Some(PersistedPlaylistUpdateTechnicalState::Succeeded); - } - (source == PlaylistUpdateDataSource::Provider - && context.provider_cluster_count == 1 - && matches!(context.provider_failure_evidence, ProviderFailureEvidence::TechnicalFailure)) - .then_some(PersistedPlaylistUpdateTechnicalState::Failed) -} - -fn persisted_cluster_snapshot( - cluster: &PlaylistUpdateClusterTelemetry, - context: PersistedSnapshotContext, -) -> PersistedPlaylistUpdateClusterSnapshot { - let quality = persisted_quality_snapshot(cluster, context.effective_policy); - PersistedPlaylistUpdateClusterSnapshot { - policy: cluster.source.and(context.request_policy), - source: cluster.source, - quality_guard_threshold: (quality.is_none() && cluster.source.is_some()) - .then_some(cluster.threshold.unwrap_or(0)), - quality, - active_count: cluster.active_count, - technical_state: persisted_technical_state(cluster, context), - } -} - -fn persist_finalized_cluster_snapshots( - playlist_download_result: &mut PlaylistDownloadResult, - request_policy: Option, - storage_error: Option<&TuliproxError>, -) { - let technical_failure = storage_error.is_some() - || playlist_download_result.partial - || !playlist_download_result.download_err.is_empty(); - let provider_failure_evidence = playlist_download_result.provider_failure_evidence; - let snapshots = playlist_download_result.input_telemetry.as_ref().map_or_else(Vec::new, |telemetry| { - let provider_cluster_count = telemetry - .clusters - .iter() - .filter(|cluster| cluster.requested && cluster.source == Some(PlaylistUpdateDataSource::Provider)) - .count(); - telemetry - .clusters - .iter() - .filter(|cluster| cluster.requested) - .map(|cluster| { - let snapshot = persisted_cluster_snapshot( - cluster, - PersistedSnapshotContext { - effective_policy: telemetry.refresh_policy, - request_policy, - technical_failure, - storage_failure: storage_error.is_some(), - provider_failure_evidence, - provider_cluster_count, - }, - ); - (cluster.cluster, snapshot) - }) - .collect() - }); - - let Some(mut pending_status) = playlist_download_result.pending_status.take() else { - return; - }; - let mut changed = pending_status.status_changed; - for (cluster, snapshot) in snapshots { - changed |= match pending_status.cluster_status_source { - ClusterStatusSource::PerCluster => { - input_cache::replace_cluster_snapshot(&mut pending_status.status, cluster.as_ref(), snapshot) - } - ClusterStatusSource::Default => input_cache::replace_cluster_snapshot_from_default_status( - &mut pending_status.status, - cluster.as_ref(), - snapshot, - ), - }; - } - if changed { - input_cache::save_input_status(&pending_status.storage_path, &pending_status.status); - } -} - -fn cached_playlist_download_result(input: &ConfigInput, refresh_policy: InputRefreshPolicy) -> PlaylistDownloadResult { - PlaylistDownloadResult::new(vec![], vec![], true, false).with_input_telemetry(input_telemetry_for_fetch( - input, - refresh_policy, - PlaylistUpdateDataSource::Cache, - &[], - None, - )) -} - -/// Loads an input already acquired in this run without reporting a second acquisition. -fn in_run_reuse_playlist_download_result() -> PlaylistDownloadResult { - PlaylistDownloadResult::new(vec![], vec![], true, false) -} - -fn forced_empty_cluster_flags(force_updates: &[ClusterForceUpdate]) -> ClusterFlags { - force_updates.iter().filter(|update| update.candidate_count == 0).fold( - ClusterFlags::empty(), - |mut clusters, update| { - clusters.insert(match update.cluster { - XtreamCluster::Live => ClusterFlags::Live, - XtreamCluster::Video => ClusterFlags::Vod, - XtreamCluster::Series => ClusterFlags::Series, - }); - clusters - }, - ) -} - -pub(crate) fn collect_effective_skip_clusters(input: &ConfigInput) -> Vec { - if !input.input_type.is_xtream() && input.input_type != InputType::M3u { - return vec![]; - } - xtream::get_skip_cluster(input) -} - -fn report_forced_update_request( - ctx: &PlaylistProcessingContext, - input: &ConfigInput, - refresh_policy: InputRefreshPolicy, -) { - if !refresh_policy.bypasses_quality() { - return; - } - let input_type = input.get_download_input_type(); - if !is_cluster_input(input_type) { - return; - } - let clusters = [ - (ConfigInputFlags::SkipLive, "live"), - (ConfigInputFlags::SkipVod, "vod"), - (ConfigInputFlags::SkipSeries, "series"), - ] - .into_iter() - .filter_map(|(skip_flag, name)| (!input.has_flag(skip_flag)).then_some(name)) - .collect::>() - .join(","); - let message = - format!("Input '{}': forced update requested; cache and update-quality bypassed for {clusters}", input.name); - info!("{message}"); - ctx.events.emit(EventMessage::PlaylistUpdateProgress( - PlaylistUpdateProgressEvent::for_run_input( - ctx.run_id.clone(), - ctx.execution_order, - input.id, - input.name.to_string(), - message, - ) - .with_detail(shared::model::PlaylistUpdateProgressDetail::ForcedUpdateRequested), - )); -} - -pub(crate) fn filter_skipped_clusters_from_source(source: PlaylistSource, input: &ConfigInput) -> PlaylistSource { - let skip_clusters = collect_effective_skip_clusters(input); - if skip_clusters.is_empty() { - return source; - } - - let skip_set: HashSet = skip_clusters.into_iter().collect(); - PlaylistSource::filtered(source, skip_set) -} - -pub(crate) fn cluster_selected(cluster: XtreamCluster, clusters: ClusterFlags) -> bool { - match cluster { - XtreamCluster::Live => clusters.contains(ClusterFlags::Live), - XtreamCluster::Video => clusters.contains(ClusterFlags::Vod), - XtreamCluster::Series => clusters.contains(ClusterFlags::Series), - } -} - -fn prepare_staged_xtream_group( - mut group: PlaylistGroup, - provider: &ConfigInput, - provider_urls: &HashMap<(XtreamCluster, u32), &Arc>, - username: &str, - password: &str, - live_stream_use_prefix: bool, - live_stream_without_extension: bool, -) -> PlaylistGroup { - let initial_count = group.channels.len(); - group.channels.retain_mut(|item| { - // The staged item's ID identifies its original Xtream stream. - let Ok(stream_id) = item.header.id.parse::() else { - return false; - }; - // Existing streams keep the provider's playback URL, including a direct source URL. - item.header.url = provider_urls.get(&(group.xtream_cluster, stream_id)).map_or_else( - || { - let metadata_extension = item.header.get_container_extension(); - let container_extension = if group.xtream_cluster == XtreamCluster::Video { - metadata_extension.as_deref().or_else(|| { - shared::utils::extract_extension_from_url(&item.header.url) - .and_then(|extension| extension.strip_prefix('.')) - }) - } else { - None - }; - tuliprox_parser::xtream::get_xtream_url( - group.xtream_cluster, - &provider.url, - username, - password, - stream_id, - container_extension, - live_stream_use_prefix, - live_stream_without_extension, - ) - .into() - }, - |url| Arc::clone(url), - ); - item.header.input_name = Arc::clone(&provider.name); - true - }); - let dropped = initial_count - group.channels.len(); - if dropped > 0 { - warn!("Skipped {dropped} staged channel(s) in group '{}' without a numeric Xtream stream ID", group.title); - } - group -} - -/// Overlays the staged source on the provider playlist of the clusters it covers. -/// -/// The staged playlist is the provider playlist restructured by an external tool: group and channel -/// names may differ, the ids do not. Staged groups are therefore resolved against the provider groups -/// by id, never by name ([`match_staged_groups`]). An overlaid group keeps the provider category id; -/// a staged group that introduces a new category keeps its own id only when that id is still free. A -/// staged group whose channels carry no numeric stream id leaves the provider category untouched for -/// Xtream staged input, while an M3U staged input remains authoritative for the selected clusters. -pub(crate) fn apply_staged_overlay_groups( - provider: &ConfigInput, - staged_type: StagedInputType, - clusters: ClusterFlags, - provider_groups: Vec, - staged_groups: Vec, -) -> Vec { - if provider.input_type != InputType::Xtream { - let mut groups: Vec = - provider_groups.into_iter().filter(|group| !cluster_selected(group.xtream_cluster, clusters)).collect(); - groups.extend(staged_groups.into_iter().filter(|group| cluster_selected(group.xtream_cluster, clusters)).map( - |mut group| { - for item in &mut group.channels { - item.header.input_name = Arc::clone(&provider.name); - } - group - }, - )); - return groups; - } - - let (Some(username), Some(password)) = (provider.username.as_deref(), provider.password.as_deref()) else { - warn!("Skipping staged channels for Xtream input '{}' without credentials", provider.name); - return provider_groups; - }; - let live_stream_use_prefix = provider.has_flag(ConfigInputFlags::XtreamLiveStreamUsePrefix); - let live_stream_without_extension = provider.has_flag(ConfigInputFlags::XtreamLiveStreamWithoutExtension); - - let staged_stream_ids: HashSet<(XtreamCluster, u32)> = staged_groups - .iter() - .filter(|group| cluster_selected(group.xtream_cluster, clusters)) - .flat_map(|group| { - group.channels.iter().filter_map(|item| { - item.header.id.parse::().ok().map(|stream_id| (group.xtream_cluster, stream_id)) - }) - }) - .collect(); - let mut provider_urls = HashMap::with_capacity(staged_stream_ids.len()); - for group in &provider_groups { - if !cluster_selected(group.xtream_cluster, clusters) { - continue; - } - for item in &group.channels { - if let Ok(stream_id) = item.header.id.parse::() { - let key = (group.xtream_cluster, stream_id); - if staged_stream_ids.contains(&key) { - provider_urls.entry(key).or_insert(&item.header.url); - } - } - } - } - - let mut staged_selected: Vec> = staged_groups - .into_iter() - .filter(|group| cluster_selected(group.xtream_cluster, clusters)) - .map(|group| { - Some(prepare_staged_xtream_group( - group, - provider, - &provider_urls, - username, - password, - live_stream_use_prefix, - live_stream_without_extension, - )) - }) - .collect(); - - // Category ids of the provider playlist are authoritative and stay reserved across the overlay: - // persistence keys input groups by `(cluster, id)`, so two groups of one cluster sharing an id - // would be collapsed into a single category. Only staged ids are renumbered; a provider playlist - // that already repeats a category id is passed through unchanged. - let mut used_ids = GroupCategoryIds::from_groups(&provider_groups); - let assignment = match_staged_groups(&provider_groups, &staged_selected, staged_type, clusters); - - if staged_type == StagedInputType::M3u { - return apply_staged_m3u_template(provider_groups, staged_selected, &assignment, clusters, used_ids); - } - - let mut result: Vec = Vec::with_capacity(provider_groups.len() + staged_selected.len()); - - for (provider_idx, provider_group) in provider_groups.into_iter().enumerate() { - if !cluster_selected(provider_group.xtream_cluster, clusters) { - result.push(provider_group); - continue; - } - - let matched = assignment.staged_of(provider_idx).and_then(|staged_idx| staged_selected[staged_idx].take()); - let Some(mut staged) = matched.filter(|staged| !staged.channels.is_empty()) else { - result.push(provider_group); - continue; - }; - - // The provider category id is the identity of the overlaid category: a staged group matched - // by stream ids or title takes it over instead of keeping its own numbering. - staged.id = - if provider_group.id == 0 { used_ids.allocate(provider_group.xtream_cluster) } else { provider_group.id }; - sync_channel_category_id(&mut staged); - result.push(staged); - } - - // Append newly introduced staged groups that had no corresponding provider group. - for mut staged in staged_selected.into_iter().flatten().filter(|staged| !staged.channels.is_empty()) { - used_ids.assign_new_group(&mut staged); - sync_channel_category_id(&mut staged); - result.push(staged); - } - - result -} - -fn apply_staged_m3u_template( - provider_groups: Vec, - staged_groups: Vec>, - assignment: &StagedOverlayAssignment, - clusters: ClusterFlags, - mut used_ids: GroupCategoryIds, -) -> Vec { - let mut result = Vec::with_capacity(provider_groups.len() + staged_groups.len()); - for (staged_idx, staged) in staged_groups.into_iter().enumerate() { - let Some(mut staged) = staged.filter(|group| !group.channels.is_empty()) else { - continue; - }; - if let Some(provider_idx) = assignment.provider_by_staged[staged_idx] { - let provider_group = &provider_groups[provider_idx]; - staged.id = if provider_group.id == 0 { - used_ids.allocate(provider_group.xtream_cluster) - } else { - provider_group.id - }; - } else { - used_ids.assign_new_group(&mut staged); - } - sync_channel_category_id(&mut staged); - result.push(staged); - } - result.extend(provider_groups.into_iter().filter(|group| !cluster_selected(group.xtream_cluster, clusters))); - result -} - -/// Keeps the channels of a group consistent with its final category id. -/// -/// Persistence re-syncs this through [`PlaylistGroup::on_load`], but the overlay result is consumed as -/// a whole: a reader must not see the category id of the staged source on a group that was just given -/// the provider category id. -fn sync_channel_category_id(group: &mut PlaylistGroup) { - let category_id = group.id; - for item in &mut group.channels { - item.header.category_id = category_id; - } -} - /// Staged groups resolved against the provider groups they overlay. /// /// Both directions are kept in sync, so one staged group can never overlay two provider categories @@ -706,122 +45,6 @@ struct StagedOverlayAssignment { has_stream_id_evidence: Vec, } -impl StagedOverlayAssignment { - fn new(provider_len: usize, staged_len: usize) -> Self { - Self { - staged_by_provider: vec![None; provider_len], - provider_by_staged: vec![None; staged_len], - has_stream_id_evidence: vec![false; staged_len], - } - } - - /// Assigns `staged_idx` to `provider_idx`; returns `false` when either side is already taken. - fn assign(&mut self, provider_idx: usize, staged_idx: usize) -> bool { - if self.staged_by_provider[provider_idx].is_some() || self.provider_by_staged[staged_idx].is_some() { - return false; - } - self.staged_by_provider[provider_idx] = Some(staged_idx); - self.provider_by_staged[staged_idx] = Some(provider_idx); - true - } - - fn staged_of(&self, provider_idx: usize) -> Option { self.staged_by_provider[provider_idx] } - - fn is_assigned(&self, staged_idx: usize) -> bool { self.provider_by_staged[staged_idx].is_some() } -} - -/// Resolves every staged group against the provider groups it overlays. -/// -/// The staged playlist is the provider playlist restructured by an external tool: group and channel -/// names may be rewritten, the ids do not. The passes therefore rank the id evidence above the name: -/// -/// 1. the channel stream ids decide. A staged channel carries the provider stream id, and the provider -/// playlist itself states which category owns a stream, so this also covers staged sources whose -/// group ids are not provider category ids (a `m3u` staged playlist numbers its groups positionally). -/// 2. the category id decides for Xtream staged groups whose channels the provider playlist does not know. -/// M3U staged group ids are positional and cannot identify provider categories. -/// 3. the title remains the last resort for staged groups without provider stream-id evidence. -fn match_staged_groups( - provider_groups: &[PlaylistGroup], - staged_groups: &[Option], - staged_type: StagedInputType, - clusters: ClusterFlags, -) -> StagedOverlayAssignment { - let mut assignment = StagedOverlayAssignment::new(provider_groups.len(), staged_groups.len()); - match_staged_by_stream_id(provider_groups, staged_groups, clusters, &mut assignment); - if staged_type == StagedInputType::Xtream { - match_staged_by_category_id(provider_groups, staged_groups, clusters, &mut assignment); - } - match_staged_by_title(provider_groups, staged_groups, clusters, &mut assignment); - assignment -} - -/// Pass 1: a staged group overlays the provider category owning its streams, strongest overlap first. -fn match_staged_by_stream_id( - provider_groups: &[PlaylistGroup], - staged_groups: &[Option], - clusters: ClusterFlags, - assignment: &mut StagedOverlayAssignment, -) { - // Only the staged stream ids are indexed, so the memory cost is bounded by the overlay and not - // by the size of the provider playlist. - let staged_stream_ids: HashSet<(XtreamCluster, &str)> = staged_groups - .iter() - .flatten() - .flat_map(|staged| staged.channels.iter().map(|item| (staged.xtream_cluster, item.header.id.as_ref()))) - .filter(|(_, stream_id)| !stream_id.is_empty()) - .collect(); - if staged_stream_ids.is_empty() { - return; - } - - let mut provider_idx_by_stream_id: HashMap<(XtreamCluster, &str), usize> = HashMap::new(); - for (provider_idx, provider_group) in provider_groups.iter().enumerate() { - if !cluster_selected(provider_group.xtream_cluster, clusters) { - continue; - } - for item in &provider_group.channels { - let key = (provider_group.xtream_cluster, item.header.id.as_ref()); - if staged_stream_ids.contains(&key) { - provider_idx_by_stream_id.entry(key).or_insert(provider_idx); - } - } - } - if provider_idx_by_stream_id.is_empty() { - return; - } - - // Every staged group is offered all provider categories owning its streams, ranked by overlap: when - // a stronger overlap takes the best-matching category, the next candidate can still claim the group. - let mut candidates: Vec = Vec::new(); - let mut hits_by_provider: HashMap = HashMap::new(); - for (staged_idx, staged) in staged_groups.iter().enumerate() { - let Some(staged) = staged else { - continue; - }; - hits_by_provider.clear(); - for item in &staged.channels { - if let Some(provider_idx) = provider_idx_by_stream_id.get(&(staged.xtream_cluster, item.header.id.as_ref())) - { - *hits_by_provider.entry(*provider_idx).or_insert(0) += 1; - } - } - assignment.has_stream_id_evidence[staged_idx] = !hits_by_provider.is_empty(); - candidates.extend(hits_by_provider.iter().map(|(provider_idx, hits)| StreamOverlap { - hits: *hits, - staged_idx, - provider_idx: *provider_idx, - })); - } - - // The strongest overlap wins first, so a staged group never loses its category to a weaker candidate. - candidates - .sort_unstable_by_key(|overlap| (std::cmp::Reverse(overlap.hits), overlap.staged_idx, overlap.provider_idx)); - for overlap in candidates { - assignment.assign(overlap.provider_idx, overlap.staged_idx); - } -} - /// One provider category a staged group shares streams with. struct StreamOverlap { hits: usize, @@ -829,67 +52,6 @@ struct StreamOverlap { provider_idx: usize, } -/// Pass 2: a staged group overlays the provider category carrying its id. -fn match_staged_by_category_id( - provider_groups: &[PlaylistGroup], - staged_groups: &[Option], - clusters: ClusterFlags, - assignment: &mut StagedOverlayAssignment, -) { - let mut provider_idx_by_id: HashMap<(XtreamCluster, u32), usize> = HashMap::new(); - for (provider_idx, provider_group) in provider_groups.iter().enumerate() { - if provider_group.id != 0 && cluster_selected(provider_group.xtream_cluster, clusters) { - provider_idx_by_id.entry((provider_group.xtream_cluster, provider_group.id)).or_insert(provider_idx); - } - } - - for (staged_idx, staged) in staged_groups.iter().enumerate() { - let Some(staged) = staged else { - continue; - }; - if staged.id == 0 || assignment.is_assigned(staged_idx) || assignment.has_stream_id_evidence[staged_idx] { - continue; - } - if let Some(provider_idx) = provider_idx_by_id.get(&(staged.xtream_cluster, staged.id)).copied() { - assignment.assign(provider_idx, staged_idx); - } - } -} - -/// Pass 3: a staged group overlays the provider category of the same name. -fn match_staged_by_title( - provider_groups: &[PlaylistGroup], - staged_groups: &[Option], - clusters: ClusterFlags, - assignment: &mut StagedOverlayAssignment, -) { - // Staged groups of the same name are interchangeable, so one lookup per provider category is enough. - let mut staged_by_title: HashMap<(XtreamCluster, &str), Vec> = HashMap::new(); - for (staged_idx, staged) in staged_groups.iter().enumerate() { - if assignment.is_assigned(staged_idx) || assignment.has_stream_id_evidence[staged_idx] { - continue; - } - if let Some(staged) = staged { - staged_by_title.entry((staged.xtream_cluster, staged.title.as_ref())).or_default().push(staged_idx); - } - } - - for (provider_idx, provider_group) in provider_groups.iter().enumerate() { - if assignment.staged_of(provider_idx).is_some() || !cluster_selected(provider_group.xtream_cluster, clusters) { - continue; - } - let key = (provider_group.xtream_cluster, provider_group.title.as_ref()); - let Some(free_groups) = staged_by_title.get_mut(&key) else { - continue; - }; - if !free_groups.is_empty() { - // Buckets stay tiny, so taking the first free group keeps the earlier first-match semantics. - let staged_idx = free_groups.remove(0); - assignment.assign(provider_idx, staged_idx); - } - } -} - /// Cluster-scoped category ids of a merged playlist. /// /// Provider category ids are reserved up front because they are authoritative: an overlaid group @@ -900,330 +62,12 @@ struct GroupCategoryIds { next_free: HashMap, } -impl GroupCategoryIds { - fn from_groups(groups: &[PlaylistGroup]) -> Self { - let mut ids = Self::default(); - for group in groups { - if group.id != 0 { - ids.used.entry(group.xtream_cluster).or_default().insert(group.id); - } - } - for (cluster, used) in &ids.used { - ids.next_free.insert(*cluster, used.iter().copied().max().unwrap_or(0).saturating_add(1)); - } - ids - } - - /// Claims `id` for `cluster`; a zero id or an id already in use is not claimable. - fn claim(&mut self, cluster: XtreamCluster, id: u32) -> bool { - id != 0 && self.used.entry(cluster).or_default().insert(id) - } - - fn assign_new_group(&mut self, group: &mut PlaylistGroup) { - if group.id == 0 { - group.id = self.allocate(group.xtream_cluster); - } else if !self.claim(group.xtream_cluster, group.id) { - let allocated = self.allocate(group.xtream_cluster); - warn!( - "Staged group '{}' uses category id {} which is already taken in {}; renumbered to {allocated}", - group.title, group.id, group.xtream_cluster - ); - group.id = allocated; - } - } - - /// Reserves and returns the next free id of `cluster`. - fn allocate(&mut self, cluster: XtreamCluster) -> u32 { - let used = self.used.entry(cluster).or_default(); - let mut candidate = self.next_free.get(&cluster).copied().unwrap_or(1); - // Zero is never a valid category id. The loop wraps at the u32 edge and ends at the latest - // when it reaches the first free id, which the finite id set always contains. - while candidate == 0 || used.contains(&candidate) { - candidate = candidate.checked_add(1).unwrap_or(1); - } - used.insert(candidate); - self.next_free.insert(cluster, candidate.saturating_add(1)); - candidate - } -} - -pub(crate) fn should_apply_staged_overlay(download_result: &PlaylistDownloadResult) -> bool { - !download_result.was_cached -} - #[derive(Clone, Copy)] struct PlaylistUpdateExecutionRef<'a> { run_id: &'a PlaylistUpdateRunId, execution_order: PlaylistUpdateRunOrder, } -/// Local discovery is not a provider Refresh policy. Both can require a fresh read. -#[derive(Clone, Copy)] -enum InputAcquisition { - Provider(InputRefreshPolicy), - RescannedLibrary, -} - -impl InputAcquisition { - fn refresh_policy(self) -> InputRefreshPolicy { - match self { - Self::Provider(policy) => policy, - Self::RescannedLibrary => InputRefreshPolicy::NORMAL, - } - } - - fn bypasses_cache(self) -> bool { - match self { - Self::Provider(policy) => policy.bypasses_cache(), - Self::RescannedLibrary => true, - } - } -} - -#[allow(clippy::too_many_lines)] -async fn playlist_download_from_input( - client: &reqwest::Client, - app_config: &Arc, - events: &E, - execution: PlaylistUpdateExecutionRef<'_>, - input: &ConfigInput, - stalker_refresh_mode: StalkerRefreshMode, - acquisition: InputAcquisition, -) -> PlaylistDownloadResult { - let refresh_policy = acquisition.refresh_policy(); - let config = &*app_config.config.load(); - let storage_dir = &config.storage_dir; - - // Check Status - let storage_path = input_cache::resolve_input_storage_path(storage_dir, &input.name).await; - let mut status = input_cache::load_input_status(&storage_path); - let cache_duration = input.cache_duration_seconds; - - // Ensure data directory exists - match tokio::fs::try_exists(&storage_path).await { - Ok(false) => { - if let Err(err) = tokio::fs::create_dir_all(&storage_path).await { - warn!("Failed to create input storage directory '{}': {err}", storage_path.display()); - } - } - Err(err) => { - warn!("Failed to check existence of input storage directory '{}': {err}", storage_path.display()); - } - Ok(true) => {} - } - - let download_input_type = input.get_download_input_type(); - // Use per-cluster cache for effective Xtream downloads. - let use_per_cluster_cache = download_input_type.is_xtream(); - - let mut xtream_clusters_to_download = Vec::new(); - let fully_cached = if use_per_cluster_cache { - let skip_cluster = collect_effective_skip_clusters(input); - let xtream_cache_candidates = xtream::requested_clusters(None, &skip_cluster); - - for cluster in xtream_cache_candidates { - if acquisition.bypasses_cache() || !input_cache::is_cache_valid(&status, cluster.as_ref(), cache_duration) { - xtream_clusters_to_download.push(cluster); - } - } - - xtream_clusters_to_download.is_empty() - } else { - !acquisition.bypasses_cache() && input_cache::is_cache_valid(&status, "default", cache_duration) - }; - let cluster_status_source = if use_per_cluster_cache || download_input_type == InputType::M3u { - ClusterStatusSource::PerCluster - } else { - ClusterStatusSource::Default - }; - - if fully_cached { - return cached_playlist_download_result(input, refresh_policy).with_pending_status(PendingInputStatusUpdate { - storage_path, - status, - status_changed: false, - cluster_status_source, - }); - } - - let provider_clusters = if is_cluster_input(download_input_type) { - if use_per_cluster_cache { - xtream_clusters_to_download.clone() - } else { - PIPELINE_TRANSPARENCY_CLUSTERS - .into_iter() - .filter(|cluster| cluster_is_configured(input, *cluster)) - .collect() - } - } else { - Vec::new() - }; - - let request = PlaylistFetchRequest { - app_config, - config: &app_config.config.load(), - client, - input, - xtream_clusters: Some(xtream_clusters_to_download.as_slice()), - update_quality: refresh_policy.quality, - }; - - // Each arm builds the provider its input type needs and awaits it in place: the - // provider types share no supertype, and building one is free, so this stays a match - // and stays statically dispatched. What changed is the result - one named - // `PlaylistFetch` instead of a six-element tuple assembled by position. - let fetch = match download_input_type { - InputType::M3u => { - super::m3u_quality::effective_m3u_fetch( - app_config, - input, - refresh_policy.quality, - M3uProvider.fetch(&request).await, - ) - .await - } - InputType::Xtream => XtreamProvider::new(events).fetch(&request).await, - InputType::M3uBatch | InputType::XtreamBatch | InputType::StalkerBatch => { - BatchContainerProvider.fetch(&request).await - } - InputType::Stalker => { - StalkerProvider::new(stalker_refresh_mode, !config.disk_based_processing).fetch(&request).await - } - InputType::Library => LibraryProvider.fetch(&request).await, - InputType::Plex => PlexProvider.fetch(&request).await, - InputType::Emby | InputType::Jellyfin => { - UnsupportedProvider::new( - "media-server", - format!("media-server input '{}' is configured but catalog import is not implemented yet", input.name), - ) - .fetch(&request) - .await - } - InputType::Staged => { - UnsupportedProvider::new( - "staged", - format!("staged input '{}' was not resolved against a parent input", input.name), - ) - .fetch(&request) - .await - } - }; - // `ProviderErrorKind` has always been able to answer "is this worth - // retrying, and does it need a human" - `needs_operator()` is exactly that - // question - and nothing consumed the answer. Every fetch failure was - // counted, logged and treated identically. - if let Some(kind) = fetch.error_kind() { - let worst = fetch - .errors - .iter() - .max_by_key(|error| ProviderErrorKind::of_tuliprox(error)) - .map(|error| sanitize_sensitive_info(&error.to_string()).into_owned()); - events.emit(EventMessage::ProviderFetchFailed(ProviderFetchFailure { - input: sanitize_sensitive_info(&input.name).into_owned().into(), - provider: download_input_type.to_string().into(), - kind: kind.into(), - error_count: fetch.errors.len(), - message: worst, - retryable: kind.is_retryable(), - needs_operator: kind.needs_operator(), - partial: fetch.partial, - })); - } - - let cache_scope = if use_per_cluster_cache { - CacheStatusScope::RequestedClusters(&xtream_clusters_to_download) - } else { - CacheStatusScope::Default - }; - let save_status = apply_playlist_fetch_outcome( - events, - execution.run_id, - execution.execution_order, - input, - &mut status, - cache_scope, - &fetch, - ); - - let input_telemetry = input_telemetry_for_fetch( - input, - refresh_policy, - PlaylistUpdateDataSource::Provider, - &provider_clusters, - Some(&fetch), - ); - PlaylistDownloadResult::from(fetch).with_input_telemetry(input_telemetry).with_pending_status( - PendingInputStatusUpdate { storage_path, status, status_changed: save_status, cluster_status_source }, - ) -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum InputJobState { - Ready, - Pending, - Failed, -} - -pub(crate) struct InputDownloadResult { - /// Only a completely read Library catalog (or its persisted input) may be empty. - pub(crate) authoritative_library: bool, - pub(crate) errors: Vec, - pub(crate) source: PlaylistSource, - pub(crate) storage_error: Option, - pub(crate) partial: bool, - pub(crate) quality_rejections: Vec, - pub(crate) accepted_empty_clusters: ClusterFlags, - pub(crate) input_telemetry: Option, -} - -impl InputDownloadResult { - pub(super) fn update_state(&mut self) -> PlaylistUpdateState { - InputCompletionFacts { - job_state: self.job_state(), - had_errors: !self.errors.is_empty() || self.storage_error.is_some(), - had_quality_rejections: !self.quality_rejections.is_empty(), - } - .update_state() - } - - pub(crate) fn job_state(&mut self) -> InputJobState { - if self.partial { - InputJobState::Pending - } else if self.storage_error.is_some() - || (self.source.is_empty() && self.accepted_empty_clusters.is_empty() && !self.authoritative_library) - { - InputJobState::Failed - } else { - InputJobState::Ready - } - } -} - -pub(crate) struct InputJobResult { - pub(crate) index: usize, - pub(crate) input_id: u16, - pub(crate) input_name: Arc, - pub(crate) state: InputJobState, - pub(crate) source: Option, - pub(crate) epg: Option, - pub(crate) stat: InputStats, - pub(crate) errors: Vec, - pub(crate) accepted_empty_clusters: ClusterFlags, - pub(crate) had_quality_rejections: bool, - pub(crate) input_telemetry: Option, -} - -impl InputJobResult { - pub(crate) fn update_state(&self) -> PlaylistUpdateState { - InputCompletionFacts { - job_state: self.state, - had_errors: !self.errors.is_empty(), - had_quality_rejections: self.had_quality_rejections, - } - .update_state() - } -} - // The same completion contract applies to a source job (including EPG errors) // and an indirect staged download, which has no separate source job. struct InputCompletionFacts { @@ -1232,649 +76,6 @@ struct InputCompletionFacts { had_quality_rejections: bool, } -impl InputCompletionFacts { - fn update_state(self) -> PlaylistUpdateState { - if self.job_state == InputJobState::Failed || self.had_errors { - PlaylistUpdateState::Failure - } else if self.job_state == InputJobState::Pending || self.had_quality_rejections { - PlaylistUpdateState::Partial - } else { - PlaylistUpdateState::Success - } - } -} - -pub(crate) async fn process_input_job( - index: usize, - ctx: &PlaylistProcessingContext, - input: &Arc, - process_parallel: bool, -) -> InputJobResult { - with_sequential_group( - &ctx.config.file_locks, - input.sequential_group, - process_parallel, - process_input_job_inner(index, ctx, input), - ) - .await -} - -pub(crate) async fn process_input_job_inner( - index: usize, - ctx: &PlaylistProcessingContext, - input: &Arc, -) -> InputJobResult { - let start_time = Instant::now(); - let input_type = input.get_download_input_type(); - let broadcast_step = create_input_broadcast_callback(&ctx.events, &ctx.run_id, ctx.execution_order, input.id); - broadcast_step("Playlist download", &format!("Downloading input '{}'", input.name)); - - let mut download = download_input(ctx, input, false).await; - let state = download.job_state(); - let storage_failed = download.storage_error.is_some(); - let had_quality_rejections = !download.quality_rejections.is_empty(); - if had_quality_rejections { - ctx.had_quality_rejections.store(true, std::sync::atomic::Ordering::Release); - } - if let Some(err) = download.storage_error.take() { - broadcast_step("Playlist download", &format!("Failed to persist/load input '{}' playlist", input.name)); - error!("Failed to persist input playlist {}", input.name); - download.errors.push(err); - } - let epg = if input_type == InputType::Library || download.partial || storage_failed { - None - } else { - download_input_epg(ctx, input, &mut download.errors).await - }; - let group_count = download.source.get_group_count(); - let channel_count = download.source.get_channel_count(); - if state == InputJobState::Failed && download.source.is_empty() && download.accepted_empty_clusters.is_empty() { - broadcast_step("Playlist download", &format!("Input '{}' playlist is empty", input.name)); - download.errors.push(TuliproxError::RepositoryPlaylist(format!("Source is empty {}", input.name))); - } - let stat = create_input_stat( - group_count, - channel_count, - download.errors.len(), - input_type, - &input.name, - start_time.elapsed().as_secs(), - ); - - InputJobResult { - index, - input_id: input.id, - input_name: input.name.clone(), - state, - source: (state == InputJobState::Ready).then_some(download.source), - epg, - stat, - errors: download.errors, - accepted_empty_clusters: download.accepted_empty_clusters, - had_quality_rejections, - input_telemetry: download.input_telemetry, - } -} - -pub(crate) fn panicked_input_job(index: usize, input: &ConfigInput) -> InputJobResult { - let error = TuliproxError::RepositoryPlaylist(format!("Input '{}' processing panicked", input.name)); - InputJobResult { - index, - input_id: input.id, - input_name: input.name.clone(), - state: InputJobState::Failed, - source: None, - epg: None, - stat: create_input_stat(0, 0, 1, input.get_download_input_type(), &input.name, 0), - errors: vec![error], - accepted_empty_clusters: ClusterFlags::empty(), - had_quality_rejections: false, - input_telemetry: None, - } -} - -pub(crate) fn report_input_job_completion( - events: &E, - run_id: &PlaylistUpdateRunId, - execution_order: PlaylistUpdateRunOrder, - result: &InputJobResult, -) { - report_input_completion( - events, - run_id, - execution_order, - result.input_id, - &result.input_name, - result.update_state(), - result.input_telemetry.as_ref(), - ); -} - -pub(super) fn report_input_completion( - events: &E, - run_id: &PlaylistUpdateRunId, - execution_order: PlaylistUpdateRunOrder, - input_id: u16, - input_name: &str, - state: PlaylistUpdateState, - input_telemetry: Option<&PlaylistUpdateInputTelemetry>, -) { - let input_name = sanitize_sensitive_info(input_name).into_owned(); - let message = match state { - PlaylistUpdateState::Success => format!("Input '{input_name}' completed successfully"), - PlaylistUpdateState::Partial => format!("Input '{input_name}' completed partially"), - PlaylistUpdateState::Failure => format!("Input '{input_name}' failed during update"), - }; - let progress = PlaylistUpdateProgressEvent::input_completed( - run_id.clone(), - execution_order, - input_id, - state, - input_name, - message, - ); - let progress = match input_telemetry { - Some(input_telemetry) => progress.with_input_telemetry(input_telemetry.clone()), - None => progress, - }; - events.emit(EventMessage::PlaylistUpdateProgress(progress)); -} - -#[allow(clippy::too_many_lines)] -pub(crate) async fn process_source( - source_idx: usize, - ctx: Arc>, -) -> (Vec, Vec, Vec) { - log_memory_snapshot(format!("source[{source_idx}] start").as_str()); - let sources = ctx.config.sources.load(); - let mut errors = vec![]; - let mut input_stats = HashMap::, InputStats>::new(); - let mut target_stats = Vec::::new(); - if let Some(source) = sources.get_source_at(source_idx) { - let mut source_playlists = Vec::with_capacity(source.inputs.len()); - let broadcast_step = create_broadcast_callback(&ctx.events, &ctx.run_id, ctx.execution_order); - let process_parallel = ctx.config.config.load().process_parallel; - let mut disabled_inputs: Vec> = vec![]; - let mut enabled_inputs = Vec::with_capacity(source.inputs.len()); - for (index, input_name) in source.inputs.iter().enumerate() { - let Some(input) = sources.get_input_by_name(input_name) else { - error!("Input {input_name} referenced by source {source_idx} does not exist"); - continue; - }; - if is_input_enabled(input, &ctx.user_targets) { - enabled_inputs.push((index, input)); - } else { - disabled_inputs.push(input.name.clone()); - } - } - - let source_downloaded = !enabled_inputs.is_empty(); - let mut job_results = Vec::with_capacity(enabled_inputs.len()); - if process_parallel { - let mut jobs = futures::stream::FuturesUnordered::new(); - for &(index, input) in &enabled_inputs { - let job = std::panic::AssertUnwindSafe(process_input_job(index, &ctx, input, true)).catch_unwind(); - jobs.push(async move { - match job.await { - Ok(result) => result, - Err(_) => panicked_input_job(index, input), - } - }); - } - while let Some(result) = jobs.next().await { - job_results.push(result); - } - } else { - for &(index, input) in &enabled_inputs { - job_results.push(process_input_job(index, &ctx, input, false).await); - } - } - job_results.sort_by_key(|result| result.index); - - let mut blockers = Vec::new(); - let mut accepted_empty_clusters = ClusterFlags::empty(); - for mut result in job_results { - super::input_status::persist_input_job_result(&ctx, &result).await; - report_input_job_completion(&ctx.events, &ctx.run_id, ctx.execution_order, &result); - errors.append(&mut result.errors); - input_stats.insert(result.input_name.clone(), result.stat); - if result.state == InputJobState::Ready { - accepted_empty_clusters |= result.accepted_empty_clusters; - if let (Some(input), Some(source)) = - (sources.get_input_by_name(&result.input_name), result.source.take()) - { - source_playlists.push(FetchedPlaylist { input, source, epg: result.epg }); - } - } else { - blockers.push(result.input_name); - } - } - - if !disabled_inputs.is_empty() && !source_downloaded { - warn!( - "Source at index {source_idx} has no enabled inputs for the given targets. Disabled: {}", - join_arc_strs(&disabled_inputs, ", ") - ); - } - if source_downloaded { - if !blockers.is_empty() { - for target in source.targets.iter().filter(|target| is_target_enabled(target, &ctx.user_targets)) { - for input_name in &blockers { - broadcast_step("Playlist download", &target_waiting_message(&target.name, input_name)); - } - } - } else if source_playlists.is_empty() { - debug!("Source at index {source_idx} is empty"); - errors.push(TuliproxError::RepositoryPlaylist(format!( - "Source at index {source_idx} is empty: {}", - join_arc_strs(&source.inputs, ", ") - ))); - } else { - debug_if_enabled!( - "Source has {} groups", - source_playlists.iter_mut().map(FetchedPlaylist::get_channel_count).sum::() - ); - let enabled_targets: Vec<_> = - source.targets.iter().filter(|target| is_target_enabled(target, &ctx.user_targets)).collect(); - target_stats = process_targets( - &ctx, - &mut source_playlists, - &enabled_targets, - &mut input_stats, - &mut errors, - accepted_empty_clusters, - process_parallel, - ) - .await; - } - } - } - log_memory_snapshot(format!("source[{source_idx}] end").as_str()); - let ordered_input_stats = sources - .get_source_at(source_idx) - .map_or_else(Vec::new, |source| source.inputs.iter().filter_map(|name| input_stats.remove(name)).collect()); - (ordered_input_stats, target_stats, errors) -} - -pub(crate) async fn download_input_epg( - ctx: &PlaylistProcessingContext, - input: &Arc, - error_list: &mut Vec, -) -> Option { - // A failed playlist download makes the EPG moot: the channels it would annotate are - // not there. - if !error_list.is_empty() { - return None; - } - let provider = XmltvEpgProvider::new(ctx); - // The XMLTV path produces documents, not programme records, so nothing reaches the - // sink. It is here because the same call answers for a record-streaming provider. - let mut discarded = CountingEpgSink::new(); - let outcome = provider.fetch(&EpgFetchRequest::new(input), &mut discarded).await; - error_list.extend(provider.take_errors()); - match outcome { - Ok(outcome) => outcome.into_guide(), - Err(err) => { - error_list.push(err); - None - } - } -} - -/// `invalidate_input_cache_status` performs a non-atomic file I/O sequence -/// (`input_cache::load_input_status` + `input_cache::save_input_status`). -/// Call this only while holding the per-input lock from -/// `PlaylistProcessingContext::get_input_lock` (as done in `download_input`). -pub(crate) async fn invalidate_input_cache_status( - ctx: &PlaylistProcessingContext, - input: &ConfigInput, -) { - let storage_dir = { ctx.config.config.load().storage_dir.clone() }; - let storage_path = input_cache::resolve_input_storage_path(&storage_dir, &input.name).await; - let mut status = input_cache::load_input_status(&storage_path); - if !status.clusters.is_empty() { - status.clusters.clear(); - input_cache::save_input_status(&storage_path, &status); - } -} - -pub(crate) async fn load_cached_input_playlist( - ctx: &PlaylistProcessingContext, - input: &Arc, -) -> (PlaylistSource, Option) { - match load_input_playlist(&ctx.config, input, None).await { - Ok(pl_source) => (pl_source, None), - Err(err) => (MemoryPlaylistSource::default().into_source(), Some(err)), - } -} - -#[allow(clippy::too_many_lines)] -pub(crate) async fn download_input( - ctx: &PlaylistProcessingContext, - input: &Arc, - allow_staged_input: bool, -) -> InputDownloadResult { - if input.staged.is_some() && !allow_staged_input { - return InputDownloadResult { - authoritative_library: false, - errors: Vec::new(), - source: MemoryPlaylistSource::default().into_source(), - storage_error: None, - partial: false, - quality_rejections: Vec::new(), - accepted_empty_clusters: ClusterFlags::empty(), - input_telemetry: None, - }; - } - - let staged_overlay = if input.staged.is_none() { - let sources = ctx.config.sources.load(); - sources.get_staged_input_for_provider(&input.name).cloned() - } else { - None - }; - - // Coordination Logic - let need_download = !ctx.is_input_downloaded(&input.name).await; - // Keep this lock for the whole critical section (download + persist/load + mark processed) - // so parallel sources sharing the same input cannot observe a half-written state. - let mut input_lock = if need_download { Some(ctx.get_input_lock(&input.name).await) } else { None }; - let mut mark_as_processed = false; - let refresh_policy = ctx.refresh_policy(input.id); - - let mut playlist_download_result = if need_download { - // Check again after lock - let already_processed = ctx.is_input_downloaded(&input.name).await; - - if already_processed { - // Use empty results, will load from disk below - in_run_reuse_playlist_download_result() - } else if ctx.pre_processed_inputs.as_ref().is_some_and(|s| s.contains(&input.name)) { - // Input was already processed in a prior session; skip download and load from disk. - // Mark only after load succeeds (or fails) to avoid exposing a half-ready state. - mark_as_processed = true; - cached_playlist_download_result(input, refresh_policy) - } else { - mark_as_processed = true; - report_forced_update_request(ctx, input, refresh_policy); - playlist_download_from_input( - &ctx.client, - &ctx.config, - &ctx.events, - PlaylistUpdateExecutionRef { run_id: &ctx.run_id, execution_order: ctx.execution_order }, - input, - ctx.stalker_refresh_mode, - ctx.acquisition(input.id), - ) - .await - } - } else { - in_run_reuse_playlist_download_result() - }; - - let mut preloaded_playlist: Option<(PlaylistSource, Option)> = None; - if playlist_download_result.was_cached { - let (cached_playlist, cached_error) = load_cached_input_playlist(ctx, input).await; - // Defensive fallback: if cache metadata says "valid" but persisted data is unreadable, - // retry once before forcing a refresh. - let must_force_refresh = cached_error.is_some(); - if must_force_refresh { - warn!("Input '{}' cache hit produced unreadable playlist; retrying cached load once", input.name); - let (retry_playlist, retry_error) = load_cached_input_playlist(ctx, input).await; - if retry_error.is_none() { - preloaded_playlist = Some((retry_playlist, None)); - } else { - if input_lock.is_none() { - input_lock = Some(ctx.get_input_lock(&input.name).await); - } - // Re-check immediately after locking to avoid duplicate refreshes when another worker - // repaired the cache between our earlier retry and lock acquisition. - let (locked_retry_playlist, locked_retry_error) = load_cached_input_playlist(ctx, input).await; - if locked_retry_error.is_none() { - warn!("Input '{}' cache became readable after lock re-check; skipping refresh", input.name); - preloaded_playlist = Some((locked_retry_playlist, None)); - } else { - warn!( - "Input '{}' cached playlist remained unreadable after retry and lock re-check; invalidating cache and forcing refresh", - input.name - ); - invalidate_input_cache_status(ctx, input).await; - playlist_download_result = playlist_download_from_input( - &ctx.client, - &ctx.config, - &ctx.events, - PlaylistUpdateExecutionRef { run_id: &ctx.run_id, execution_order: ctx.execution_order }, - input, - ctx.stalker_refresh_mode, - ctx.acquisition(input.id), - ) - .await; - } - } - } else { - preloaded_playlist = Some((cached_playlist, None)); - } - } - if playlist_download_result.partial { - ctx.partial_refresh.store(true, std::sync::atomic::Ordering::Release); - ctx.events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent::for_run_input( - ctx.run_id.clone(), - ctx.execution_order, - input.id, - input.name.to_string(), - stalker_checkpoint_message(&input.name), - ))); - } - let apply_staged_overlay = should_apply_staged_overlay(&playlist_download_result); - let authoritative_library = input.input_type == InputType::Library - && playlist_download_result.download_err.is_empty() - && !playlist_download_result.partial; - let mut accepted_empty_clusters = forced_empty_cluster_flags(&playlist_download_result.force_updates); - let reuse_persisted_after_quality_rejection = !playlist_download_result.quality_rejections.is_empty() - && playlist_download_result.downloaded_playlist.is_empty() - && !playlist_download_result.persisted; - - let (mut playlist, mut error) = if input.input_type == InputType::Library && !authoritative_library { - // A failed catalog read must not replace an existing Library input with an empty tree. - (MemoryPlaylistSource::default().into_source(), None) - } else if let Some(preloaded) = preloaded_playlist { - preloaded - } else if playlist_download_result.was_cached - || playlist_download_result.persisted - || reuse_persisted_after_quality_rejection - { - match load_input_playlist(&ctx.config, input, None).await { - Ok(pl_source) => (pl_source, None), - Err(e) => (MemoryPlaylistSource::default().into_source(), Some(e)), - } - } else { - debug!("Persisting input '{}' playlist", input.name); - let (pl, err) = persist_input_playlist_with_options( - &ctx.config, - input, - std::mem::take(&mut playlist_download_result.downloaded_playlist), - InputPlaylistPersistOptions { accepted_empty_clusters }, - ) - .await; - (MemoryPlaylistSource::new(pl).into_source(), err) - }; - - playlist = filter_skipped_clusters_from_source(playlist, input); - - if let Some(staged_input) = staged_overlay.filter(|_| apply_staged_overlay) { - let clusters = staged_input.staged.as_ref().map_or_else(ClusterFlags::all, |staged| staged.clusters); - let mut staged_result = Box::pin(download_input(ctx, &staged_input, true)).await; - // download_input has released the staged input's lock. Record its own - // facts before merging them into the parent's outcome or consuming groups. - super::input_status::complete_staged_input(ctx, &staged_input, &mut staged_result).await; - playlist_download_result.partial |= staged_result.partial; - playlist_download_result.download_err.append(&mut staged_result.errors); - playlist_download_result.quality_rejections.append(&mut staged_result.quality_rejections); - accepted_empty_clusters |= staged_result.accepted_empty_clusters; - if let Some(staged_error) = staged_result.storage_error { - playlist_download_result.download_err.push(staged_error); - } else { - let provider_groups = playlist.take_groups(); - let staged_groups = staged_result.source.take_groups(); - let merged_groups = - apply_staged_overlay_groups(input, staged_input.staged_type, clusters, provider_groups, staged_groups); - if let Some(input_telemetry) = playlist_download_result.input_telemetry.as_mut() { - neutralize_overlaid_cluster_facts(input_telemetry, clusters); - } - let (merged_playlist, persist_error) = persist_input_playlist_with_options( - &ctx.config, - input, - merged_groups, - InputPlaylistPersistOptions { accepted_empty_clusters }, - ) - .await; - playlist = MemoryPlaylistSource::new(merged_playlist).into_source(); - if error.is_none() { - error = persist_error; - } else if let Some(persist_error) = persist_error { - playlist_download_result.download_err.push(persist_error); - } - } - } - - if input.input_type == InputType::M3u { - let alias_errors = download_m3u_alias_playlists(ctx, input).await; - playlist_download_result.download_err.extend(alias_errors); - } - - if mark_as_processed - && !playlist_download_result.partial - && error.is_none() - && (!playlist.is_empty() || !accepted_empty_clusters.is_empty() || authoritative_library) - { - // Mark after persist/load so other workers only see this input as ready when data is usable. - ctx.mark_input_downloaded(input.name.clone()).await; - } - - if !playlist_download_result.persisted && error.is_none() { - report_force_updates( - &ctx.events, - &ctx.run_id, - ctx.execution_order, - input, - &playlist_download_result.force_updates, - ); - } - - finalize_input_telemetry(&mut playlist_download_result, error.as_ref()); - let request_policy = ctx - .input_refresh - .filter(|input_refresh| input_refresh.input_id == input.id) - .map(|input_refresh| input_refresh.policy); - persist_finalized_cluster_snapshots(&mut playlist_download_result, request_policy, error.as_ref()); - - // Explicitly release per-input lock after load/persist/mark steps are completed. - drop(input_lock); - - InputDownloadResult { - authoritative_library, - errors: playlist_download_result.download_err, - source: playlist, - storage_error: error, - partial: playlist_download_result.partial, - quality_rejections: playlist_download_result.quality_rejections, - accepted_empty_clusters, - input_telemetry: playlist_download_result.input_telemetry, - } -} - -async fn download_m3u_alias_playlists( - ctx: &PlaylistProcessingContext, - input: &ConfigInput, -) -> Vec { - let Some(aliases) = input.get_enabled_aliases() else { return vec![] }; - let mut errors = Vec::new(); - - for alias in aliases { - if ctx.is_input_downloaded(&alias.name).await { - continue; - } - - let mut alias_input = input.as_input(alias); - // A user-provided raw-playlist persist path belongs to the primary input. Alias - // snapshots use their own internal storage so accounts never overwrite each other. - alias_input.persist = None; - alias_input.epg = None; - let alias_input = Arc::new(alias_input); - - let mut alias_result = Box::pin(download_input(ctx, &alias_input, false)).await; - let alias_had_errors = !alias_result.errors.is_empty() || alias_result.storage_error.is_some(); - errors.append(&mut alias_result.errors); - if let Some(storage_error) = alias_result.storage_error { - errors.push(storage_error); - } - if alias_result.partial { - errors.push(TuliproxError::RepositoryPlaylist(format!( - "M3U alias '{}' returned a partial playlist", - alias.name - ))); - } else if alias_result.source.is_empty() && !alias_had_errors { - errors.push(TuliproxError::RepositoryPlaylist(format!("M3U alias '{}' playlist is empty", alias.name))); - } - } - - errors -} - -pub(crate) fn create_broadcast_callback( - events: &E, - run_id: &PlaylistUpdateRunId, - execution_order: PlaylistUpdateRunOrder, -) -> StepMeasureCallback { - let events = events.clone(); - let run_id = run_id.clone(); - Box::new(move |context: &str, msg: &str| { - events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent::for_run_global( - run_id.clone(), - execution_order, - context, - msg, - ))); - }) -} - -fn create_input_broadcast_callback( - events: &E, - run_id: &PlaylistUpdateRunId, - execution_order: PlaylistUpdateRunOrder, - input_id: u16, -) -> StepMeasureCallback { - let events = events.clone(); - let run_id = run_id.clone(); - Box::new(move |context: &str, msg: &str| { - events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent::for_run_input( - run_id.clone(), - execution_order, - input_id, - context, - msg, - ))); - }) -} - -pub(crate) fn create_input_stat( - group_count: usize, - channel_count: usize, - error_count: usize, - input_type: InputType, - input_name: &str, - secs_took: u64, -) -> InputStats { - InputStats { - name: input_name.to_string(), - input_type, - error_count, - raw_stats: PlaylistStats { group_count, channel_count }, - processed_stats: PlaylistStats { group_count: 0, channel_count: 0 }, - secs_took, - } -} - pub struct PlaylistProcessingContext { pub client: reqwest::Client, pub run_id: PlaylistUpdateRunId, @@ -1915,779 +116,31 @@ pub struct PlaylistProcessingContext Clone for PlaylistProcessingContext { - fn clone(&self) -> Self { - Self { - client: self.client.clone(), - run_id: self.run_id.clone(), - execution_order: self.execution_order, - config: Arc::clone(&self.config), - user_targets: Arc::clone(&self.user_targets), - events: self.events.clone(), - playlist_state: self.playlist_state.clone(), - disabled_headers: self.disabled_headers.clone(), - processed_inputs: Arc::clone(&self.processed_inputs), - input_completions: Arc::clone(&self.input_completions), - input_locks: Arc::clone(&self.input_locks), - provider_manager: self.provider_manager.clone(), - metadata_manager: self.metadata_manager.clone(), - pre_processed_inputs: self.pre_processed_inputs.clone(), - stalker_refresh_mode: self.stalker_refresh_mode, - partial_refresh: Arc::clone(&self.partial_refresh), - had_quality_rejections: Arc::clone(&self.had_quality_rejections), - input_refresh: self.input_refresh, - library_update_mode: self.library_update_mode, - } - } -} - -impl PlaylistProcessingContext { - #[must_use] - pub fn refresh_policy(&self, input_id: u16) -> InputRefreshPolicy { - self.input_refresh - .filter(|input_refresh| input_refresh.input_id == input_id) - .map_or(InputRefreshPolicy::NORMAL, |input_refresh| input_refresh.policy) - } - - fn acquisition(&self, input_id: u16) -> InputAcquisition { - if self.library_update_mode.reloads_input(input_id) { - InputAcquisition::RescannedLibrary - } else { - InputAcquisition::Provider(self.refresh_policy(input_id)) - } - } - - pub async fn is_input_downloaded(&self, input_name: &str) -> bool { - let processed = self.processed_inputs.lock().await; - processed.contains(input_name) - } - pub async fn mark_input_downloaded(&self, input_name: Arc) -> bool { - let mut processed = self.processed_inputs.lock().await; - processed.insert(input_name) - } - - pub async fn get_input_lock(&self, input_name: &Arc) -> OwnedRwLockWriteGuard<()> { - let mut locks = self.input_locks.lock().await; - // Try to upgrade the existing weak reference - let lock = locks.get(input_name).and_then(Weak::upgrade).unwrap_or_else(|| { - let new_lock = Arc::new(RwLock::new(())); - locks.insert(input_name.clone(), Arc::downgrade(&new_lock)); - new_lock - }); - - // Clean up stale references periodically - locks.retain(|_, weak| weak.strong_count() > 0); - - drop(locks); // Release mutex before awaiting write lock - lock.write_owned().await - } -} - -pub(crate) async fn process_sources( - processing_ctx: &PlaylistProcessingContext, -) -> (Vec, Vec) { - let mut async_tasks = JoinSet::new(); - let sources = processing_ctx.config.sources.load(); - let process_parallel = processing_ctx.config.config.load().process_parallel; - if process_parallel && log_enabled!(Level::Debug) { - debug!("Parallel processing enabled"); - } - - let mut source_results = Vec::new(); - let mut errors = Vec::new(); - let mut processed_any = false; - - for (index, source) in sources.sources.iter().enumerate() { - if !source.should_process_for_user_targets(&processing_ctx.user_targets) { - continue; - } - - // We're using the file lock this way on purpose - let source_lock_path = PathBuf::from(concat_string!("source_", &index.to_string())); - let Ok(update_lock) = processing_ctx.config.file_locks.try_write_lock(&source_lock_path).await else { - warn!( - "The update operation for the source at index {index} was skipped because an update is already in progress." - ); - continue; - }; - - let ctx = Arc::new(processing_ctx.clone()); - - processed_any = true; - if process_parallel { - async_tasks.spawn(async move { - let _update_lock = update_lock; - (index, process_source(index, ctx).await) - }); - } else { - source_results.push((index, process_source(index, ctx).await)); - drop(update_lock); - } - } - if !processed_any { - warn!( - "No sources were processed for the given targets. Check that:\n\ - - Sources have enabled targets matching your target selection\n\ - - CLI -t filter or schedule.targets are correct\n\ - - No playlist lock is blocking updates" - ); - } - while let Some(result) = async_tasks.join_next().await { - match result { - Ok(result) => source_results.push(result), - Err(err) => { - error!("Playlist processing task failed: {err:?}"); - errors - .push(TuliproxError::RepositoryPlaylist(format!("Playlist source processing task failed: {err}"))); - } - } - } - - source_results.sort_by_key(|(index, _)| *index); - let mut stats = Vec::with_capacity(source_results.len()); - for (_, (input_stats, target_stats, mut source_errors)) in source_results { - errors.append(&mut source_errors); - if let Some(source_stats) = SourceStats::try_new(input_stats, target_stats) { - stats.push(source_stats); - } - } - (stats, errors) -} - #[cfg(test)] -mod pipeline_transparency_tests { - use super::*; - use shared::model::{ConfigInputOptionsDto, ConfigInputUpdateQualityDto}; - use tuliprox_core::model::ClusterUpdateAcceptance; +mod pipeline_transparency_tests; - fn input(input_type: InputType) -> ConfigInput { - ConfigInput { id: 17, name: Arc::from("provider-a"), input_type, enabled: true, ..ConfigInput::default() } - } - - fn cluster(telemetry: &PlaylistUpdateInputTelemetry, cluster: XtreamCluster) -> &PlaylistUpdateClusterTelemetry { - telemetry.clusters.iter().find(|item| item.cluster == cluster).expect("configured cluster telemetry") - } - - const fn completed_snapshot_context( - effective_policy: InputRefreshPolicy, - request_policy: Option, - provider_cluster_count: usize, - ) -> PersistedSnapshotContext { - PersistedSnapshotContext { - effective_policy, - request_policy, - technical_failure: false, - storage_failure: false, - provider_failure_evidence: ProviderFailureEvidence::None, - provider_cluster_count, - } - } - - const fn failed_provider_snapshot_context( - effective_policy: InputRefreshPolicy, - request_policy: Option, - provider_cluster_count: usize, - ) -> PersistedSnapshotContext { - PersistedSnapshotContext { - effective_policy, - request_policy, - technical_failure: true, - storage_failure: false, - provider_failure_evidence: ProviderFailureEvidence::TechnicalFailure, - provider_cluster_count, - } - } - - #[test] - fn pipeline_transparency_telemetry_preserves_cache_rejection_and_force_decisions() { - let input = input(InputType::Xtream); - let fetch = PlaylistFetch::groups(Vec::new()) - .with_quality_rejections(vec![ClusterUpdateRejection { - cluster: XtreamCluster::Video, - current_count: 12_543, - candidate_count: 217, - threshold: 90, - quality: 1, - }]) - .with_force_updates(vec![ClusterForceUpdate { - cluster: XtreamCluster::Series, - candidate_count: 8_412, - configured_threshold: 100, - }]); - - let telemetry = input_telemetry_for_fetch( - &input, - InputRefreshPolicy::FORCE, - PlaylistUpdateDataSource::Provider, - &[XtreamCluster::Video, XtreamCluster::Series], - Some(&fetch), - ); - - let live = cluster(&telemetry, XtreamCluster::Live); - assert!(live.requested); - assert_eq!(live.source, Some(PlaylistUpdateDataSource::Cache)); - assert_eq!(live.decision, None); - - let video = cluster(&telemetry, XtreamCluster::Video); - assert_eq!(video.source, Some(PlaylistUpdateDataSource::Provider)); - assert_eq!(video.decision, Some(PlaylistUpdateClusterDecision::Rejected)); - assert_eq!( - (video.baseline_count, video.candidate_count, video.active_count), - (Some(12_543), Some(217), Some(12_543)) - ); - assert_eq!((video.threshold, video.quality), (Some(90), Some(1))); - - let series = cluster(&telemetry, XtreamCluster::Series); - assert_eq!(series.source, Some(PlaylistUpdateDataSource::Provider)); - assert_eq!(series.decision, Some(PlaylistUpdateClusterDecision::Accepted)); - assert_eq!((series.candidate_count, series.active_count), (Some(8_412), None)); - assert_eq!(series.threshold, Some(100)); - - let mut published = telemetry; - confirm_published_cluster_counts(&mut published); - assert_eq!(cluster(&published, XtreamCluster::Series).active_count, Some(8_412)); - } - - #[test] - fn pipeline_transparency_telemetry_preserves_accepted_quality_and_bootstrap_facts() { - let mut input = input(InputType::Xtream); - input.options = Some(ConfigInputOptions::from(&ConfigInputOptionsDto { - update_quality: ConfigInputUpdateQualityDto { live: 90, vod: 90, series: 0 }, - ..ConfigInputOptionsDto::default() - })); - let fetch = PlaylistFetch::groups(Vec::new()).with_quality_acceptances(vec![ - ClusterUpdateAcceptance { - cluster: XtreamCluster::Live, - current_count: Some(12_543), - candidate_count: 12_000, - threshold: 90, - quality: Some(95), - }, - ClusterUpdateAcceptance { - cluster: XtreamCluster::Video, - current_count: None, - candidate_count: 217, - threshold: 90, - quality: None, - }, - ]); - - let mut telemetry = input_telemetry_for_fetch( - &input, - InputRefreshPolicy::NORMAL, - PlaylistUpdateDataSource::Provider, - &PIPELINE_TRANSPARENCY_CLUSTERS, - Some(&fetch), - ); - - let live = cluster(&telemetry, XtreamCluster::Live); - assert_eq!(live.decision, Some(PlaylistUpdateClusterDecision::Accepted)); - assert_eq!((live.baseline_count, live.candidate_count), (Some(12_543), Some(12_000))); - assert_eq!((live.threshold, live.quality, live.active_count), (Some(90), Some(95), None)); - - let video = cluster(&telemetry, XtreamCluster::Video); - assert_eq!(video.decision, Some(PlaylistUpdateClusterDecision::Accepted)); - assert_eq!((video.baseline_count, video.candidate_count), (None, Some(217))); - assert_eq!((video.threshold, video.quality, video.active_count), (Some(90), None, None)); - - confirm_published_cluster_counts(&mut telemetry); - assert_eq!(cluster(&telemetry, XtreamCluster::Live).active_count, Some(12_000)); - assert_eq!(cluster(&telemetry, XtreamCluster::Video).active_count, Some(217)); - } - - #[test] - fn pipeline_transparency_accepted_quality_survives_fetch_error_without_active_claim() { - let mut input = input(InputType::Xtream); - input.options = Some(ConfigInputOptions::from(&ConfigInputOptionsDto { - update_quality: ConfigInputUpdateQualityDto { live: 90, ..ConfigInputUpdateQualityDto::default() }, - ..ConfigInputOptionsDto::default() - })); - let acceptance = ClusterUpdateAcceptance { - cluster: XtreamCluster::Live, - current_count: Some(100), - candidate_count: 95, - threshold: 90, - quality: Some(95), - }; - let fetch = PlaylistFetch::groups(Vec::new()) - .with_quality_acceptances(vec![acceptance]) - .with_errors(vec![TuliproxError::RepositoryXtream("post-quality publish failure".to_string())]); - let telemetry = input_telemetry_for_fetch( - &input, - InputRefreshPolicy::NORMAL, - PlaylistUpdateDataSource::Provider, - &[XtreamCluster::Live], - Some(&fetch), - ); - let mut download_result = PlaylistDownloadResult::from(fetch).with_input_telemetry(telemetry); - - finalize_input_telemetry(&mut download_result, None); - - let telemetry = download_result.input_telemetry.as_ref().expect("input telemetry"); - let live = cluster(telemetry, XtreamCluster::Live); - assert_eq!(live.decision, Some(PlaylistUpdateClusterDecision::Accepted)); - assert_eq!((live.baseline_count, live.candidate_count), (Some(100), Some(95))); - assert_eq!((live.threshold, live.quality), (Some(90), Some(95))); - assert_eq!(live.active_count, None); - assert_eq!(download_result.download_err.len(), 1); - } - - #[test] - fn pipeline_transparency_telemetry_keeps_zero_threshold_quality_disabled() { - let input = input(InputType::Xtream); - let fetch = PlaylistFetch::groups(Vec::new()); - - let telemetry = input_telemetry_for_fetch( - &input, - InputRefreshPolicy::NORMAL, - PlaylistUpdateDataSource::Provider, - &[XtreamCluster::Live], - Some(&fetch), - ); - - let live = cluster(&telemetry, XtreamCluster::Live); - assert_eq!(live.decision, Some(PlaylistUpdateClusterDecision::Accepted)); - assert_eq!(live.threshold, None); - assert_eq!(live.quality, None); - assert_eq!(live.active_count, None); - } - - #[test] - fn pipeline_transparency_telemetry_reports_non_cluster_cache_without_synthetic_clusters() { - let telemetry = input_telemetry_for_fetch( - &input(InputType::Plex), - InputRefreshPolicy::NORMAL, - PlaylistUpdateDataSource::Cache, - &[], - None, - ); - - assert_eq!(telemetry.refresh_policy, InputRefreshPolicy::NORMAL); - assert_eq!(telemetry.source, Some(PlaylistUpdateDataSource::Cache)); - assert!(telemetry.clusters.is_empty()); - } - - #[test] - fn pipeline_transparency_known_cluster_failure_keeps_quality_decision_and_neutralizes_overlay() { - let input = input(InputType::Xtream); - let acceptance = ClusterUpdateAcceptance { - cluster: XtreamCluster::Series, - current_count: Some(100), - candidate_count: 95, - threshold: 90, - quality: Some(95), - }; - let mut fetch = PlaylistFetch::groups(Vec::new()).with_quality_acceptances(vec![acceptance]); - fetch.record_cluster_error( - XtreamCluster::Series, - TuliproxError::RepositoryXtream("publish failure".to_string()), - ); - let telemetry = input_telemetry_for_fetch( - &input, - InputRefreshPolicy::NORMAL, - PlaylistUpdateDataSource::Provider, - &PIPELINE_TRANSPARENCY_CLUSTERS, - Some(&fetch), - ); - let mut result = PlaylistDownloadResult::from(fetch).with_input_telemetry(telemetry); - finalize_input_telemetry(&mut result, None); - let telemetry = result.input_telemetry.as_mut().unwrap(); - let shows = cluster(telemetry, XtreamCluster::Series); - assert_eq!(shows.decision, Some(PlaylistUpdateClusterDecision::Accepted)); - assert_eq!(shows.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Failed)); - assert_eq!(shows.active_count, None); - let context = failed_provider_snapshot_context(InputRefreshPolicy::NORMAL, None, 3); - let snapshot = persisted_cluster_snapshot(shows, context); - assert_eq!(snapshot.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Failed)); - assert_eq!(snapshot.quality.unwrap().decision, PersistedPlaylistUpdateQualityDecision::Accepted); - neutralize_overlaid_cluster_facts(telemetry, ClusterFlags::Series); - let overlaid = cluster(telemetry, XtreamCluster::Series); - assert_eq!(overlaid.technical_state, None); - assert_eq!(persisted_cluster_snapshot(overlaid, context).technical_state, None); - } - - #[test] - fn pipeline_transparency_telemetry_types_complete_provider_failure_without_guessing_metrics() { - let mut input = input(InputType::Xtream); - input.options = Some(ConfigInputOptions::from(&ConfigInputOptionsDto { - skip_vod: true, - skip_series: true, - ..ConfigInputOptionsDto::default() - })); - let fetch = PlaylistFetch::failed(TuliproxError::Download("provider unavailable".to_string())); - - let telemetry = input_telemetry_for_fetch( - &input, - InputRefreshPolicy::REFRESH, - PlaylistUpdateDataSource::Provider, - &[XtreamCluster::Live], - Some(&fetch), - ); - - assert_eq!(telemetry.refresh_policy, InputRefreshPolicy::REFRESH); - let live = cluster(&telemetry, XtreamCluster::Live); - assert!(live.requested); - assert_eq!(live.source, Some(PlaylistUpdateDataSource::Provider)); - assert_eq!(live.decision, Some(PlaylistUpdateClusterDecision::TechnicalError)); - assert!(live.baseline_count.is_none()); - assert!(live.candidate_count.is_none()); - assert!(live.active_count.is_none()); - assert!(live.quality.is_none()); - } - - #[test] - fn pipeline_transparency_telemetry_does_not_assign_an_unscoped_error_to_multiple_clusters() { - let input = input(InputType::Stalker); - let fetch = PlaylistFetch::failed(TuliproxError::Download("provider unavailable".to_string())); - - let telemetry = input_telemetry_for_fetch( - &input, - InputRefreshPolicy::NORMAL, - PlaylistUpdateDataSource::Provider, - &PIPELINE_TRANSPARENCY_CLUSTERS, - Some(&fetch), - ); - - assert!(telemetry.clusters.iter().all(|item| item.requested && item.decision.is_none())); - } - - #[test] - fn pipeline_transparency_telemetry_does_not_infer_an_unreported_quality_acceptance() { - let mut input = input(InputType::Xtream); - input.options = Some(ConfigInputOptions::from(&ConfigInputOptionsDto { - skip_vod: true, - update_quality: ConfigInputUpdateQualityDto { live: 90, ..ConfigInputUpdateQualityDto::default() }, - ..ConfigInputOptionsDto::default() - })); - let fetch = PlaylistFetch::groups(Vec::new()); - - let telemetry = input_telemetry_for_fetch( - &input, - InputRefreshPolicy::NORMAL, - PlaylistUpdateDataSource::Provider, - &[XtreamCluster::Live], - Some(&fetch), - ); - - let live = cluster(&telemetry, XtreamCluster::Live); - assert_eq!(live.threshold, Some(90)); - assert_eq!(live.decision, None, "an accepted quality decision was not retained by the provider result"); - let video = cluster(&telemetry, XtreamCluster::Video); - assert!(!video.requested); - assert_eq!(video.source, None); - assert_eq!(video.decision, None); - } - - #[test] - fn persisted_cluster_snapshot_preserves_quality_facts_without_merging_technical_state() { - let accepted = PlaylistUpdateClusterTelemetry { - cluster: XtreamCluster::Live, - requested: true, - source: Some(PlaylistUpdateDataSource::Provider), - baseline_count: Some(1_000), - candidate_count: Some(950), - active_count: Some(950), - threshold: Some(90), - quality: Some(95), - decision: Some(PlaylistUpdateClusterDecision::Accepted), - technical_state: None, - }; - - let completed = persisted_cluster_snapshot( - &accepted, - completed_snapshot_context(InputRefreshPolicy::NORMAL, Some(InputRefreshPolicy::REFRESH), 1), - ); - assert_eq!(completed.policy, Some(InputRefreshPolicy::REFRESH)); - assert_eq!(completed.source, Some(PlaylistUpdateDataSource::Provider)); - assert_eq!(completed.quality_guard_threshold, None, "an evaluation already owns its threshold"); - assert_eq!( - completed.quality, - Some(PersistedPlaylistUpdateQualitySnapshot { - threshold: 90, - baseline_count: Some(1_000), - candidate_count: Some(950), - achieved_quality: Some(95), - decision: PersistedPlaylistUpdateQualityDecision::Accepted, - }) - ); - assert_eq!(completed.active_count, Some(950)); - assert_eq!(completed.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Succeeded)); - - let mut bootstrap = accepted.clone(); - bootstrap.baseline_count = None; - bootstrap.candidate_count = Some(42); - bootstrap.active_count = Some(42); - bootstrap.quality = None; - let bootstrap = - persisted_cluster_snapshot(&bootstrap, completed_snapshot_context(InputRefreshPolicy::NORMAL, None, 1)); - assert_eq!(bootstrap.policy, None, "an automatic run must not acquire a default policy"); - assert_eq!(bootstrap.quality_guard_threshold, None, "bootstrap evaluation already owns its threshold"); - assert_eq!( - bootstrap.quality, - Some(PersistedPlaylistUpdateQualitySnapshot { - threshold: 90, - baseline_count: None, - candidate_count: Some(42), - achieved_quality: None, - decision: PersistedPlaylistUpdateQualityDecision::Accepted, - }), - "bootstrap acceptance must not invent achieved Quality" - ); - - let mut rejected = accepted.clone(); - rejected.candidate_count = Some(217); - rejected.active_count = Some(1_000); - rejected.quality = Some(21); - rejected.decision = Some(PlaylistUpdateClusterDecision::Rejected); - let rejected = - persisted_cluster_snapshot(&rejected, completed_snapshot_context(InputRefreshPolicy::NORMAL, None, 1)); - assert_eq!( - rejected.quality.map(|quality| quality.decision), - Some(PersistedPlaylistUpdateQualityDecision::Rejected) - ); - assert_eq!(rejected.active_count, Some(1_000), "only the typed active baseline is retained"); - assert_eq!(rejected.quality_guard_threshold, None, "a rejection evaluation already owns its threshold"); - assert_eq!(rejected.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Succeeded)); - - let mut accepted_then_failed = accepted; - accepted_then_failed.active_count = None; - let accepted_then_failed = persisted_cluster_snapshot( - &accepted_then_failed, - PersistedSnapshotContext { - effective_policy: InputRefreshPolicy::NORMAL, - request_policy: None, - technical_failure: true, - storage_failure: true, - provider_failure_evidence: ProviderFailureEvidence::None, - provider_cluster_count: 1, - }, - ); - assert_eq!( - accepted_then_failed.quality.map(|quality| quality.decision), - Some(PersistedPlaylistUpdateQualityDecision::Accepted) - ); - assert_eq!(accepted_then_failed.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Failed)); - assert_eq!(accepted_then_failed.active_count, None); - } - - #[test] - fn persisted_cluster_snapshot_keeps_policy_source_and_disabled_guard_independent() { - let forced = PlaylistUpdateClusterTelemetry { - cluster: XtreamCluster::Video, - requested: true, - source: Some(PlaylistUpdateDataSource::Provider), - baseline_count: None, - candidate_count: Some(73), - active_count: Some(73), - threshold: Some(95), - quality: None, - decision: Some(PlaylistUpdateClusterDecision::Accepted), - technical_state: None, - }; - - let forced = persisted_cluster_snapshot( - &forced, - completed_snapshot_context(InputRefreshPolicy::FORCE, Some(InputRefreshPolicy::FORCE), 1), - ); - assert_eq!(forced.policy, Some(InputRefreshPolicy::FORCE)); - assert_eq!(forced.source, Some(PlaylistUpdateDataSource::Provider)); - assert_eq!(forced.quality_guard_threshold, Some(95)); - assert_eq!(forced.quality, None, "a Force bypass is not a Quality evaluation"); - - let disabled = PlaylistUpdateClusterTelemetry { - cluster: XtreamCluster::Series, - requested: true, - source: Some(PlaylistUpdateDataSource::Provider), - baseline_count: None, - candidate_count: Some(12), - active_count: Some(12), - threshold: None, - quality: None, - decision: Some(PlaylistUpdateClusterDecision::Accepted), - technical_state: None, - }; - let disabled = persisted_cluster_snapshot( - &disabled, - completed_snapshot_context(InputRefreshPolicy::FORCE, Some(InputRefreshPolicy::FORCE), 1), - ); - assert_eq!(disabled.quality_guard_threshold, Some(0)); - assert_eq!(disabled.quality, None, "threshold zero stays disabled even during Force"); - - let cached = PlaylistUpdateClusterTelemetry { - cluster: XtreamCluster::Live, - requested: true, - source: Some(PlaylistUpdateDataSource::Cache), - baseline_count: None, - candidate_count: None, - active_count: None, - threshold: Some(90), - quality: None, - decision: None, - technical_state: None, - }; - let cached = persisted_cluster_snapshot( - &cached, - completed_snapshot_context(InputRefreshPolicy::REFRESH, Some(InputRefreshPolicy::REFRESH), 0), - ); - assert_eq!(cached.policy, Some(InputRefreshPolicy::REFRESH)); - assert_eq!(cached.quality_guard_threshold, Some(90)); - assert_eq!( - cached.source, - Some(PlaylistUpdateDataSource::Cache), - "the actual source remains stronger than request policy" - ); - } - - #[test] - fn persisted_cluster_snapshot_final_status_boundary_replaces_provider_with_cache_without_history() { - let temp = tempfile::tempdir().expect("temporary status directory"); - let mut status = input_cache::InputStatus::default(); - status.clusters.insert( - XtreamCluster::Live.as_ref().to_string(), - input_cache::ClusterStatus { status: input_cache::ClusterState::Ok, timestamp: 17, last_update: None }, - ); - let provider_telemetry = PlaylistUpdateInputTelemetry { - refresh_policy: InputRefreshPolicy::REFRESH, - source: None, - clusters: vec![PlaylistUpdateClusterTelemetry { - cluster: XtreamCluster::Live, - requested: true, - source: Some(PlaylistUpdateDataSource::Provider), - baseline_count: Some(100), - candidate_count: Some(95), - active_count: Some(95), - threshold: Some(90), - quality: Some(95), - decision: Some(PlaylistUpdateClusterDecision::Accepted), - technical_state: None, - }], - }; - let mut provider_result = PlaylistDownloadResult::new(Vec::new(), Vec::new(), false, false) - .with_input_telemetry(provider_telemetry) - .with_pending_status(PendingInputStatusUpdate { - storage_path: temp.path().to_path_buf(), - status, - status_changed: true, - cluster_status_source: ClusterStatusSource::PerCluster, - }); - - persist_finalized_cluster_snapshots(&mut provider_result, Some(InputRefreshPolicy::REFRESH), None); - - let provider_status = input_cache::load_input_status(temp.path()); - let provider_snapshot = provider_status.clusters["live"].last_update.expect("provider snapshot"); - assert_eq!(provider_snapshot.source, Some(PlaylistUpdateDataSource::Provider)); - assert!(provider_snapshot.quality.is_some()); - - let cache_telemetry = PlaylistUpdateInputTelemetry { - refresh_policy: InputRefreshPolicy::NORMAL, - source: None, - clusters: vec![PlaylistUpdateClusterTelemetry { - cluster: XtreamCluster::Live, - requested: true, - source: Some(PlaylistUpdateDataSource::Cache), - baseline_count: None, - candidate_count: None, - active_count: None, - threshold: Some(90), - quality: None, - decision: None, - technical_state: None, - }], - }; - let mut cache_result = PlaylistDownloadResult::new(Vec::new(), Vec::new(), true, false) - .with_input_telemetry(cache_telemetry) - .with_pending_status(PendingInputStatusUpdate { - storage_path: temp.path().to_path_buf(), - status: provider_status, - status_changed: false, - cluster_status_source: ClusterStatusSource::PerCluster, - }); - - persist_finalized_cluster_snapshots(&mut cache_result, Some(InputRefreshPolicy::NORMAL), None); - - let cached_status = input_cache::load_input_status(temp.path()); - assert_eq!(cached_status.clusters.len(), 1); - let cached_snapshot = cached_status.clusters["live"].last_update.expect("replacement cache snapshot"); - assert_eq!(cached_snapshot.policy, Some(InputRefreshPolicy::NORMAL)); - assert_eq!(cached_snapshot.source, Some(PlaylistUpdateDataSource::Cache)); - assert_eq!(cached_snapshot.quality_guard_threshold, Some(90)); - assert_eq!(cached_snapshot.quality, None); - assert_eq!(cached_snapshot.active_count, None); - assert_eq!(cached_snapshot.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Succeeded)); - let encoded = - std::fs::read_to_string(temp.path().join(input_cache::STATUS_FILE)).expect("persisted status JSON"); - assert_eq!(encoded.matches(r#""last_update""#).count(), 1); - assert!(!encoded.contains("history")); - } - - #[test] - fn persisted_cluster_snapshot_discards_unproven_staged_overlay_facts() { - let mut telemetry = PlaylistUpdateInputTelemetry { - refresh_policy: InputRefreshPolicy::FORCE, - source: None, - clusters: vec![PlaylistUpdateClusterTelemetry { - cluster: XtreamCluster::Video, - requested: true, - source: Some(PlaylistUpdateDataSource::Provider), - baseline_count: Some(100), - candidate_count: Some(95), - active_count: Some(95), - threshold: Some(90), - quality: Some(95), - decision: Some(PlaylistUpdateClusterDecision::Accepted), - technical_state: None, - }], - }; - neutralize_overlaid_cluster_facts(&mut telemetry, ClusterFlags::Vod); - - let snapshot = persisted_cluster_snapshot( - cluster(&telemetry, XtreamCluster::Video), - completed_snapshot_context(telemetry.refresh_policy, Some(InputRefreshPolicy::FORCE), 0), - ); - - assert_eq!(snapshot, PersistedPlaylistUpdateClusterSnapshot::default()); - } - - #[test] - fn persisted_cluster_snapshot_leaves_unscoped_multi_cluster_failure_unknown() { - let cluster = PlaylistUpdateClusterTelemetry { - cluster: XtreamCluster::Live, - requested: true, - source: Some(PlaylistUpdateDataSource::Provider), - baseline_count: Some(100), - candidate_count: Some(95), - active_count: None, - threshold: Some(90), - quality: Some(95), - decision: Some(PlaylistUpdateClusterDecision::Accepted), - technical_state: None, - }; - - let snapshot = - persisted_cluster_snapshot(&cluster, failed_provider_snapshot_context(InputRefreshPolicy::NORMAL, None, 2)); - - assert_eq!(snapshot.technical_state, None); - assert_eq!( - snapshot.quality.map(|quality| quality.decision), - Some(PersistedPlaylistUpdateQualityDecision::Accepted) - ); - - let unscoped_ancillary_failure = persisted_cluster_snapshot( - &cluster, - PersistedSnapshotContext { - effective_policy: InputRefreshPolicy::NORMAL, - request_policy: None, - technical_failure: true, - storage_failure: false, - provider_failure_evidence: ProviderFailureEvidence::None, - provider_cluster_count: 1, - }, - ); - assert_eq!( - unscoped_ancillary_failure.technical_state, None, - "an input-wide ancillary failure is not invented as a cluster-scoped provider failure" - ); - } -} +mod acquisition; +mod jobs; +mod overlay; +mod status; +mod telemetry; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(crate) use acquisition::{download_input, invalidate_input_cache_status, load_cached_input_playlist}; +pub(super) use jobs::report_input_completion; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(crate) use jobs::{ + create_broadcast_callback, create_input_stat, download_input_epg, panicked_input_job, process_input_job, + process_input_job_inner, process_source, process_sources, report_input_job_completion, InputDownloadResult, + InputJobResult, InputJobState, +}; +pub(crate) use overlay::{apply_staged_overlay_groups, should_apply_staged_overlay}; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub(crate) use status::{ + cluster_selected, collect_effective_skip_clusters, filter_skipped_clusters_from_source, is_input_enabled, + with_sequential_group, +}; +use status::{ClusterStatusSource, ProviderFailureEvidence}; +pub(crate) use telemetry::neutralize_overlaid_cluster_facts; +pub(super) use telemetry::{cluster_is_configured, PIPELINE_TRANSPARENCY_CLUSTERS}; +#[cfg(test)] +use telemetry::{confirm_published_cluster_counts, persisted_cluster_snapshot}; diff --git a/backend/processing/src/processor/playlist/ingest/acquisition.rs b/backend/processing/src/processor/playlist/ingest/acquisition.rs new file mode 100644 index 000000000..eead204c1 --- /dev/null +++ b/backend/processing/src/processor/playlist/ingest/acquisition.rs @@ -0,0 +1,539 @@ +use super::{ + apply_playlist_fetch_outcome, apply_staged_overlay_groups, cluster_is_configured, collect_effective_skip_clusters, + filter_skipped_clusters_from_source, neutralize_overlaid_cluster_facts, report_force_updates, + should_apply_staged_overlay, stalker_checkpoint_message, + status::{ + cached_playlist_download_result, forced_empty_cluster_flags, in_run_reuse_playlist_download_result, + report_forced_update_request, ClusterStatusSource, + }, + telemetry::{ + finalize_input_telemetry, input_telemetry_for_fetch, is_cluster_input, persist_finalized_cluster_snapshots, + }, + CacheStatusScope, InputDownloadResult, LibraryProvider, PendingInputStatusUpdate, PlaylistDownloadResult, + PlaylistProcessingContext, PlaylistUpdateExecutionRef, PlexProvider, StalkerProvider, + PIPELINE_TRANSPARENCY_CLUSTERS, +}; +use crate::{input_cache, metadata_sink::MetadataUpdateSink, processor::StalkerRefreshMode}; +use log::{debug, warn}; +use shared::{ + error::TuliproxError, + model::{ + ClusterFlags, EventMessage, EventSink, InputRefreshPolicy, InputType, PlaylistUpdateDataSource, + PlaylistUpdateProgressEvent, ProviderFetchFailure, + }, + utils::sanitize_sensitive_info, +}; +use std::sync::Arc; +use tuliprox_core::model::{AppConfig, ConfigInput}; +use tuliprox_iptv::{ + error::ProviderErrorKind, + provider::{ + BatchContainerProvider, M3uProvider, PlaylistFetchRequest, PlaylistProvider, UnsupportedProvider, + XtreamProvider, + }, + xtream, +}; +use tuliprox_repository::{ + load_input_playlist, persist_input_playlist_with_options, InputPlaylistPersistOptions, MemoryPlaylistSource, + PlaylistSource, +}; + +/// Local discovery is not a provider Refresh policy. Both can require a fresh read. +#[derive(Clone, Copy)] +pub(super) enum InputAcquisition { + Provider(InputRefreshPolicy), + RescannedLibrary, +} + +impl InputAcquisition { + pub(super) fn refresh_policy(self) -> InputRefreshPolicy { + match self { + Self::Provider(policy) => policy, + Self::RescannedLibrary => InputRefreshPolicy::NORMAL, + } + } + + pub(super) fn bypasses_cache(self) -> bool { + match self { + Self::Provider(policy) => policy.bypasses_cache(), + Self::RescannedLibrary => true, + } + } +} + +#[allow(clippy::too_many_lines)] +async fn playlist_download_from_input( + client: &reqwest::Client, + app_config: &Arc, + events: &E, + execution: PlaylistUpdateExecutionRef<'_>, + input: &ConfigInput, + stalker_refresh_mode: StalkerRefreshMode, + acquisition: InputAcquisition, +) -> PlaylistDownloadResult { + let refresh_policy = acquisition.refresh_policy(); + let config = &*app_config.config.load(); + let storage_dir = &config.storage_dir; + + // Check Status + let storage_path = input_cache::resolve_input_storage_path(storage_dir, &input.name).await; + let mut status = input_cache::load_input_status(&storage_path); + let cache_duration = input.cache_duration_seconds; + + // Ensure data directory exists + match tokio::fs::try_exists(&storage_path).await { + Ok(false) => { + if let Err(err) = tokio::fs::create_dir_all(&storage_path).await { + warn!("Failed to create input storage directory '{}': {err}", storage_path.display()); + } + } + Err(err) => { + warn!("Failed to check existence of input storage directory '{}': {err}", storage_path.display()); + } + Ok(true) => {} + } + + let download_input_type = input.get_download_input_type(); + // Use per-cluster cache for effective Xtream downloads. + let use_per_cluster_cache = download_input_type.is_xtream(); + + let mut xtream_clusters_to_download = Vec::new(); + let fully_cached = if use_per_cluster_cache { + let skip_cluster = collect_effective_skip_clusters(input); + let xtream_cache_candidates = xtream::requested_clusters(None, &skip_cluster); + + for cluster in xtream_cache_candidates { + if acquisition.bypasses_cache() || !input_cache::is_cache_valid(&status, cluster.as_ref(), cache_duration) { + xtream_clusters_to_download.push(cluster); + } + } + + xtream_clusters_to_download.is_empty() + } else { + !acquisition.bypasses_cache() && input_cache::is_cache_valid(&status, "default", cache_duration) + }; + let cluster_status_source = if use_per_cluster_cache || download_input_type == InputType::M3u { + ClusterStatusSource::PerCluster + } else { + ClusterStatusSource::Default + }; + + if fully_cached { + return cached_playlist_download_result(input, refresh_policy).with_pending_status(PendingInputStatusUpdate { + storage_path, + status, + status_changed: false, + cluster_status_source, + }); + } + + let provider_clusters = if is_cluster_input(download_input_type) { + if use_per_cluster_cache { + xtream_clusters_to_download.clone() + } else { + PIPELINE_TRANSPARENCY_CLUSTERS + .into_iter() + .filter(|cluster| cluster_is_configured(input, *cluster)) + .collect() + } + } else { + Vec::new() + }; + + let request = PlaylistFetchRequest { + app_config, + config: &app_config.config.load(), + client, + input, + xtream_clusters: Some(xtream_clusters_to_download.as_slice()), + update_quality: refresh_policy.quality, + }; + + // Each arm builds the provider its input type needs and awaits it in place: the + // provider types share no supertype, and building one is free, so this stays a match + // and stays statically dispatched. What changed is the result - one named + // `PlaylistFetch` instead of a six-element tuple assembled by position. + let fetch = match download_input_type { + InputType::M3u => { + super::super::m3u_quality::effective_m3u_fetch( + app_config, + input, + refresh_policy.quality, + M3uProvider.fetch(&request).await, + ) + .await + } + InputType::Xtream => XtreamProvider::new(events).fetch(&request).await, + InputType::M3uBatch | InputType::XtreamBatch | InputType::StalkerBatch => { + BatchContainerProvider.fetch(&request).await + } + InputType::Stalker => { + StalkerProvider::new(stalker_refresh_mode, !config.disk_based_processing).fetch(&request).await + } + InputType::Library => LibraryProvider.fetch(&request).await, + InputType::Plex => PlexProvider.fetch(&request).await, + InputType::Emby | InputType::Jellyfin => { + UnsupportedProvider::new( + "media-server", + format!("media-server input '{}' is configured but catalog import is not implemented yet", input.name), + ) + .fetch(&request) + .await + } + InputType::Staged => { + UnsupportedProvider::new( + "staged", + format!("staged input '{}' was not resolved against a parent input", input.name), + ) + .fetch(&request) + .await + } + }; + // `ProviderErrorKind` has always been able to answer "is this worth + // retrying, and does it need a human" - `needs_operator()` is exactly that + // question - and nothing consumed the answer. Every fetch failure was + // counted, logged and treated identically. + if let Some(kind) = fetch.error_kind() { + let worst = fetch + .errors + .iter() + .max_by_key(|error| ProviderErrorKind::of_tuliprox(error)) + .map(|error| sanitize_sensitive_info(&error.to_string()).into_owned()); + events.emit(EventMessage::ProviderFetchFailed(ProviderFetchFailure { + input: sanitize_sensitive_info(&input.name).into_owned().into(), + provider: download_input_type.to_string().into(), + kind: kind.into(), + error_count: fetch.errors.len(), + message: worst, + retryable: kind.is_retryable(), + needs_operator: kind.needs_operator(), + partial: fetch.partial, + })); + } + + let cache_scope = if use_per_cluster_cache { + CacheStatusScope::RequestedClusters(&xtream_clusters_to_download) + } else { + CacheStatusScope::Default + }; + let save_status = apply_playlist_fetch_outcome( + events, + execution.run_id, + execution.execution_order, + input, + &mut status, + cache_scope, + &fetch, + ); + + let input_telemetry = input_telemetry_for_fetch( + input, + refresh_policy, + PlaylistUpdateDataSource::Provider, + &provider_clusters, + Some(&fetch), + ); + PlaylistDownloadResult::from(fetch).with_input_telemetry(input_telemetry).with_pending_status( + PendingInputStatusUpdate { storage_path, status, status_changed: save_status, cluster_status_source }, + ) +} + +/// `invalidate_input_cache_status` performs a non-atomic file I/O sequence +/// (`input_cache::load_input_status` + `input_cache::save_input_status`). +/// Call this only while holding the per-input lock from +/// `PlaylistProcessingContext::get_input_lock` (as done in `download_input`). +pub(crate) async fn invalidate_input_cache_status( + ctx: &PlaylistProcessingContext, + input: &ConfigInput, +) { + let storage_dir = { ctx.config.config.load().storage_dir.clone() }; + let storage_path = input_cache::resolve_input_storage_path(&storage_dir, &input.name).await; + let mut status = input_cache::load_input_status(&storage_path); + if !status.clusters.is_empty() { + status.clusters.clear(); + input_cache::save_input_status(&storage_path, &status); + } +} + +pub(crate) async fn load_cached_input_playlist( + ctx: &PlaylistProcessingContext, + input: &Arc, +) -> (PlaylistSource, Option) { + match load_input_playlist(&ctx.config, input, None).await { + Ok(pl_source) => (pl_source, None), + Err(err) => (MemoryPlaylistSource::default().into_source(), Some(err)), + } +} + +#[allow(clippy::too_many_lines)] +pub(crate) async fn download_input( + ctx: &PlaylistProcessingContext, + input: &Arc, + allow_staged_input: bool, +) -> InputDownloadResult { + if input.staged.is_some() && !allow_staged_input { + return InputDownloadResult { + authoritative_library: false, + errors: Vec::new(), + source: MemoryPlaylistSource::default().into_source(), + storage_error: None, + partial: false, + quality_rejections: Vec::new(), + accepted_empty_clusters: ClusterFlags::empty(), + input_telemetry: None, + }; + } + + let staged_overlay = if input.staged.is_none() { + let sources = ctx.config.sources.load(); + sources.get_staged_input_for_provider(&input.name).cloned() + } else { + None + }; + + // Coordination Logic + let need_download = !ctx.is_input_downloaded(&input.name).await; + // Keep this lock for the whole critical section (download + persist/load + mark processed) + // so parallel sources sharing the same input cannot observe a half-written state. + let mut input_lock = if need_download { Some(ctx.get_input_lock(&input.name).await) } else { None }; + let mut mark_as_processed = false; + let refresh_policy = ctx.refresh_policy(input.id); + + let mut playlist_download_result = if need_download { + // Check again after lock + let already_processed = ctx.is_input_downloaded(&input.name).await; + + if already_processed { + // Use empty results, will load from disk below + in_run_reuse_playlist_download_result() + } else if ctx.pre_processed_inputs.as_ref().is_some_and(|s| s.contains(&input.name)) { + // Input was already processed in a prior session; skip download and load from disk. + // Mark only after load succeeds (or fails) to avoid exposing a half-ready state. + mark_as_processed = true; + cached_playlist_download_result(input, refresh_policy) + } else { + mark_as_processed = true; + report_forced_update_request(ctx, input, refresh_policy); + playlist_download_from_input( + &ctx.client, + &ctx.config, + &ctx.events, + PlaylistUpdateExecutionRef { run_id: &ctx.run_id, execution_order: ctx.execution_order }, + input, + ctx.stalker_refresh_mode, + ctx.acquisition(input.id), + ) + .await + } + } else { + in_run_reuse_playlist_download_result() + }; + + let mut preloaded_playlist: Option<(PlaylistSource, Option)> = None; + if playlist_download_result.was_cached { + let (cached_playlist, cached_error) = load_cached_input_playlist(ctx, input).await; + // Defensive fallback: if cache metadata says "valid" but persisted data is unreadable, + // retry once before forcing a refresh. + let must_force_refresh = cached_error.is_some(); + if must_force_refresh { + warn!("Input '{}' cache hit produced unreadable playlist; retrying cached load once", input.name); + let (retry_playlist, retry_error) = load_cached_input_playlist(ctx, input).await; + if retry_error.is_none() { + preloaded_playlist = Some((retry_playlist, None)); + } else { + if input_lock.is_none() { + input_lock = Some(ctx.get_input_lock(&input.name).await); + } + // Re-check immediately after locking to avoid duplicate refreshes when another worker + // repaired the cache between our earlier retry and lock acquisition. + let (locked_retry_playlist, locked_retry_error) = load_cached_input_playlist(ctx, input).await; + if locked_retry_error.is_none() { + warn!("Input '{}' cache became readable after lock re-check; skipping refresh", input.name); + preloaded_playlist = Some((locked_retry_playlist, None)); + } else { + warn!( + "Input '{}' cached playlist remained unreadable after retry and lock re-check; invalidating cache and forcing refresh", + input.name + ); + invalidate_input_cache_status(ctx, input).await; + playlist_download_result = playlist_download_from_input( + &ctx.client, + &ctx.config, + &ctx.events, + PlaylistUpdateExecutionRef { run_id: &ctx.run_id, execution_order: ctx.execution_order }, + input, + ctx.stalker_refresh_mode, + ctx.acquisition(input.id), + ) + .await; + } + } + } else { + preloaded_playlist = Some((cached_playlist, None)); + } + } + if playlist_download_result.partial { + ctx.partial_refresh.store(true, std::sync::atomic::Ordering::Release); + ctx.events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent::for_run_input( + ctx.run_id.clone(), + ctx.execution_order, + input.id, + input.name.to_string(), + stalker_checkpoint_message(&input.name), + ))); + } + let apply_staged_overlay = should_apply_staged_overlay(&playlist_download_result); + let authoritative_library = input.input_type == InputType::Library + && playlist_download_result.download_err.is_empty() + && !playlist_download_result.partial; + let mut accepted_empty_clusters = forced_empty_cluster_flags(&playlist_download_result.force_updates); + let reuse_persisted_after_quality_rejection = !playlist_download_result.quality_rejections.is_empty() + && playlist_download_result.downloaded_playlist.is_empty() + && !playlist_download_result.persisted; + + let (mut playlist, mut error) = if input.input_type == InputType::Library && !authoritative_library { + // A failed catalog read must not replace an existing Library input with an empty tree. + (MemoryPlaylistSource::default().into_source(), None) + } else if let Some(preloaded) = preloaded_playlist { + preloaded + } else if playlist_download_result.was_cached + || playlist_download_result.persisted + || reuse_persisted_after_quality_rejection + { + match load_input_playlist(&ctx.config, input, None).await { + Ok(pl_source) => (pl_source, None), + Err(e) => (MemoryPlaylistSource::default().into_source(), Some(e)), + } + } else { + debug!("Persisting input '{}' playlist", input.name); + let (pl, err) = persist_input_playlist_with_options( + &ctx.config, + input, + std::mem::take(&mut playlist_download_result.downloaded_playlist), + InputPlaylistPersistOptions { accepted_empty_clusters }, + ) + .await; + (MemoryPlaylistSource::new(pl).into_source(), err) + }; + + playlist = filter_skipped_clusters_from_source(playlist, input); + + if let Some(staged_input) = staged_overlay.filter(|_| apply_staged_overlay) { + let clusters = staged_input.staged.as_ref().map_or_else(ClusterFlags::all, |staged| staged.clusters); + let mut staged_result = Box::pin(download_input(ctx, &staged_input, true)).await; + // download_input has released the staged input's lock. Record its own + // facts before merging them into the parent's outcome or consuming groups. + super::super::input_status::complete_staged_input(ctx, &staged_input, &mut staged_result).await; + playlist_download_result.partial |= staged_result.partial; + playlist_download_result.download_err.append(&mut staged_result.errors); + playlist_download_result.quality_rejections.append(&mut staged_result.quality_rejections); + accepted_empty_clusters |= staged_result.accepted_empty_clusters; + if let Some(staged_error) = staged_result.storage_error { + playlist_download_result.download_err.push(staged_error); + } else { + let provider_groups = playlist.take_groups(); + let staged_groups = staged_result.source.take_groups(); + let merged_groups = + apply_staged_overlay_groups(input, staged_input.staged_type, clusters, provider_groups, staged_groups); + if let Some(input_telemetry) = playlist_download_result.input_telemetry.as_mut() { + neutralize_overlaid_cluster_facts(input_telemetry, clusters); + } + let (merged_playlist, persist_error) = persist_input_playlist_with_options( + &ctx.config, + input, + merged_groups, + InputPlaylistPersistOptions { accepted_empty_clusters }, + ) + .await; + playlist = MemoryPlaylistSource::new(merged_playlist).into_source(); + if error.is_none() { + error = persist_error; + } else if let Some(persist_error) = persist_error { + playlist_download_result.download_err.push(persist_error); + } + } + } + + if input.input_type == InputType::M3u { + let alias_errors = download_m3u_alias_playlists(ctx, input).await; + playlist_download_result.download_err.extend(alias_errors); + } + + if mark_as_processed + && !playlist_download_result.partial + && error.is_none() + && (!playlist.is_empty() || !accepted_empty_clusters.is_empty() || authoritative_library) + { + // Mark after persist/load so other workers only see this input as ready when data is usable. + ctx.mark_input_downloaded(input.name.clone()).await; + } + + if !playlist_download_result.persisted && error.is_none() { + report_force_updates( + &ctx.events, + &ctx.run_id, + ctx.execution_order, + input, + &playlist_download_result.force_updates, + ); + } + + finalize_input_telemetry(&mut playlist_download_result, error.as_ref()); + let request_policy = ctx + .input_refresh + .filter(|input_refresh| input_refresh.input_id == input.id) + .map(|input_refresh| input_refresh.policy); + persist_finalized_cluster_snapshots(&mut playlist_download_result, request_policy, error.as_ref()); + + // Explicitly release per-input lock after load/persist/mark steps are completed. + drop(input_lock); + + InputDownloadResult { + authoritative_library, + errors: playlist_download_result.download_err, + source: playlist, + storage_error: error, + partial: playlist_download_result.partial, + quality_rejections: playlist_download_result.quality_rejections, + accepted_empty_clusters, + input_telemetry: playlist_download_result.input_telemetry, + } +} + +async fn download_m3u_alias_playlists( + ctx: &PlaylistProcessingContext, + input: &ConfigInput, +) -> Vec { + let Some(aliases) = input.get_enabled_aliases() else { return vec![] }; + let mut errors = Vec::new(); + + for alias in aliases { + if ctx.is_input_downloaded(&alias.name).await { + continue; + } + + let mut alias_input = input.as_input(alias); + // A user-provided raw-playlist persist path belongs to the primary input. Alias + // snapshots use their own internal storage so accounts never overwrite each other. + alias_input.persist = None; + alias_input.epg = None; + let alias_input = Arc::new(alias_input); + + let mut alias_result = Box::pin(download_input(ctx, &alias_input, false)).await; + let alias_had_errors = !alias_result.errors.is_empty() || alias_result.storage_error.is_some(); + errors.append(&mut alias_result.errors); + if let Some(storage_error) = alias_result.storage_error { + errors.push(storage_error); + } + if alias_result.partial { + errors.push(TuliproxError::RepositoryPlaylist(format!( + "M3U alias '{}' returned a partial playlist", + alias.name + ))); + } else if alias_result.source.is_empty() && !alias_had_errors { + errors.push(TuliproxError::RepositoryPlaylist(format!("M3U alias '{}' playlist is empty", alias.name))); + } + } + + errors +} diff --git a/backend/processing/src/processor/playlist/ingest/jobs.rs b/backend/processing/src/processor/playlist/ingest/jobs.rs new file mode 100644 index 000000000..a5a375ccf --- /dev/null +++ b/backend/processing/src/processor/playlist/ingest/jobs.rs @@ -0,0 +1,584 @@ +use super::{ + acquisition::InputAcquisition, download_input, is_input_enabled, is_target_enabled, join_arc_strs, process_targets, + target_waiting_message, with_sequential_group, InputCompletionFacts, PlaylistProcessingContext, XmltvEpgProvider, +}; +use crate::{fetched_playlist::FetchedPlaylist, metadata_sink::MetadataUpdateSink, parser::xmltv::TVGuide}; +use futures::{FutureExt, StreamExt}; +use log::{debug, error, log_enabled, warn, Level}; +use shared::{ + concat_string, + error::TuliproxError, + model::{ + ClusterFlags, EventMessage, EventSink, InputRefreshPolicy, InputStats, InputType, PlaylistStats, + PlaylistUpdateInputTelemetry, PlaylistUpdateProgressEvent, PlaylistUpdateRunId, PlaylistUpdateRunOrder, + PlaylistUpdateState, SourceStats, TargetStats, + }, + utils::sanitize_sensitive_info, +}; +use std::{ + collections::HashMap, + path::PathBuf, + sync::{Arc, Weak}, + time::Instant, +}; +use tokio::{ + sync::{OwnedRwLockWriteGuard, RwLock}, + task::JoinSet, +}; +use tuliprox_core::{ + model::{ClusterUpdateRejection, ConfigInput}, + utils::{debug_if_enabled, log_memory_snapshot, StepMeasureCallback}, +}; +use tuliprox_iptv::epg::{CountingEpgSink, EpgFetchRequest, EpgProvider}; +use tuliprox_repository::PlaylistSource; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum InputJobState { + Ready, + Pending, + Failed, +} + +pub(crate) struct InputDownloadResult { + /// Only a completely read Library catalog (or its persisted input) may be empty. + pub(crate) authoritative_library: bool, + pub(crate) errors: Vec, + pub(crate) source: PlaylistSource, + pub(crate) storage_error: Option, + pub(crate) partial: bool, + pub(crate) quality_rejections: Vec, + pub(crate) accepted_empty_clusters: ClusterFlags, + pub(crate) input_telemetry: Option, +} + +impl InputDownloadResult { + pub(in crate::processor::playlist) fn update_state(&mut self) -> PlaylistUpdateState { + InputCompletionFacts { + job_state: self.job_state(), + had_errors: !self.errors.is_empty() || self.storage_error.is_some(), + had_quality_rejections: !self.quality_rejections.is_empty(), + } + .update_state() + } + + pub(crate) fn job_state(&mut self) -> InputJobState { + if self.partial { + InputJobState::Pending + } else if self.storage_error.is_some() + || (self.source.is_empty() && self.accepted_empty_clusters.is_empty() && !self.authoritative_library) + { + InputJobState::Failed + } else { + InputJobState::Ready + } + } +} + +pub(crate) struct InputJobResult { + pub(crate) index: usize, + pub(crate) input_id: u16, + pub(crate) input_name: Arc, + pub(crate) state: InputJobState, + pub(crate) source: Option, + pub(crate) epg: Option, + pub(crate) stat: InputStats, + pub(crate) errors: Vec, + pub(crate) accepted_empty_clusters: ClusterFlags, + pub(crate) had_quality_rejections: bool, + pub(crate) input_telemetry: Option, +} + +impl InputJobResult { + pub(crate) fn update_state(&self) -> PlaylistUpdateState { + InputCompletionFacts { + job_state: self.state, + had_errors: !self.errors.is_empty(), + had_quality_rejections: self.had_quality_rejections, + } + .update_state() + } +} + +impl InputCompletionFacts { + pub(super) fn update_state(self) -> PlaylistUpdateState { + if self.job_state == InputJobState::Failed || self.had_errors { + PlaylistUpdateState::Failure + } else if self.job_state == InputJobState::Pending || self.had_quality_rejections { + PlaylistUpdateState::Partial + } else { + PlaylistUpdateState::Success + } + } +} + +pub(crate) async fn process_input_job( + index: usize, + ctx: &PlaylistProcessingContext, + input: &Arc, + process_parallel: bool, +) -> InputJobResult { + with_sequential_group( + &ctx.config.file_locks, + input.sequential_group, + process_parallel, + process_input_job_inner(index, ctx, input), + ) + .await +} + +pub(crate) async fn process_input_job_inner( + index: usize, + ctx: &PlaylistProcessingContext, + input: &Arc, +) -> InputJobResult { + let start_time = Instant::now(); + let input_type = input.get_download_input_type(); + let broadcast_step = create_input_broadcast_callback(&ctx.events, &ctx.run_id, ctx.execution_order, input.id); + broadcast_step("Playlist download", &format!("Downloading input '{}'", input.name)); + + let mut download = download_input(ctx, input, false).await; + let state = download.job_state(); + let storage_failed = download.storage_error.is_some(); + let had_quality_rejections = !download.quality_rejections.is_empty(); + if had_quality_rejections { + ctx.had_quality_rejections.store(true, std::sync::atomic::Ordering::Release); + } + if let Some(err) = download.storage_error.take() { + broadcast_step("Playlist download", &format!("Failed to persist/load input '{}' playlist", input.name)); + error!("Failed to persist input playlist {}", input.name); + download.errors.push(err); + } + let epg = if input_type == InputType::Library || download.partial || storage_failed { + None + } else { + download_input_epg(ctx, input, &mut download.errors).await + }; + let group_count = download.source.get_group_count(); + let channel_count = download.source.get_channel_count(); + if state == InputJobState::Failed && download.source.is_empty() && download.accepted_empty_clusters.is_empty() { + broadcast_step("Playlist download", &format!("Input '{}' playlist is empty", input.name)); + download.errors.push(TuliproxError::RepositoryPlaylist(format!("Source is empty {}", input.name))); + } + let stat = create_input_stat( + group_count, + channel_count, + download.errors.len(), + input_type, + &input.name, + start_time.elapsed().as_secs(), + ); + + InputJobResult { + index, + input_id: input.id, + input_name: input.name.clone(), + state, + source: (state == InputJobState::Ready).then_some(download.source), + epg, + stat, + errors: download.errors, + accepted_empty_clusters: download.accepted_empty_clusters, + had_quality_rejections, + input_telemetry: download.input_telemetry, + } +} + +pub(crate) fn panicked_input_job(index: usize, input: &ConfigInput) -> InputJobResult { + let error = TuliproxError::RepositoryPlaylist(format!("Input '{}' processing panicked", input.name)); + InputJobResult { + index, + input_id: input.id, + input_name: input.name.clone(), + state: InputJobState::Failed, + source: None, + epg: None, + stat: create_input_stat(0, 0, 1, input.get_download_input_type(), &input.name, 0), + errors: vec![error], + accepted_empty_clusters: ClusterFlags::empty(), + had_quality_rejections: false, + input_telemetry: None, + } +} + +pub(crate) fn report_input_job_completion( + events: &E, + run_id: &PlaylistUpdateRunId, + execution_order: PlaylistUpdateRunOrder, + result: &InputJobResult, +) { + report_input_completion( + events, + run_id, + execution_order, + result.input_id, + &result.input_name, + result.update_state(), + result.input_telemetry.as_ref(), + ); +} + +pub(in crate::processor::playlist) fn report_input_completion( + events: &E, + run_id: &PlaylistUpdateRunId, + execution_order: PlaylistUpdateRunOrder, + input_id: u16, + input_name: &str, + state: PlaylistUpdateState, + input_telemetry: Option<&PlaylistUpdateInputTelemetry>, +) { + let input_name = sanitize_sensitive_info(input_name).into_owned(); + let message = match state { + PlaylistUpdateState::Success => format!("Input '{input_name}' completed successfully"), + PlaylistUpdateState::Partial => format!("Input '{input_name}' completed partially"), + PlaylistUpdateState::Failure => format!("Input '{input_name}' failed during update"), + }; + let progress = PlaylistUpdateProgressEvent::input_completed( + run_id.clone(), + execution_order, + input_id, + state, + input_name, + message, + ); + let progress = match input_telemetry { + Some(input_telemetry) => progress.with_input_telemetry(input_telemetry.clone()), + None => progress, + }; + events.emit(EventMessage::PlaylistUpdateProgress(progress)); +} + +#[allow(clippy::too_many_lines)] +pub(crate) async fn process_source( + source_idx: usize, + ctx: Arc>, +) -> (Vec, Vec, Vec) { + log_memory_snapshot(format!("source[{source_idx}] start").as_str()); + let sources = ctx.config.sources.load(); + let mut errors = vec![]; + let mut input_stats = HashMap::, InputStats>::new(); + let mut target_stats = Vec::::new(); + if let Some(source) = sources.get_source_at(source_idx) { + let mut source_playlists = Vec::with_capacity(source.inputs.len()); + let broadcast_step = create_broadcast_callback(&ctx.events, &ctx.run_id, ctx.execution_order); + let process_parallel = ctx.config.config.load().process_parallel; + let mut disabled_inputs: Vec> = vec![]; + let mut enabled_inputs = Vec::with_capacity(source.inputs.len()); + for (index, input_name) in source.inputs.iter().enumerate() { + let Some(input) = sources.get_input_by_name(input_name) else { + error!("Input {input_name} referenced by source {source_idx} does not exist"); + continue; + }; + if is_input_enabled(input, &ctx.user_targets) { + enabled_inputs.push((index, input)); + } else { + disabled_inputs.push(input.name.clone()); + } + } + + let source_downloaded = !enabled_inputs.is_empty(); + let mut job_results = Vec::with_capacity(enabled_inputs.len()); + if process_parallel { + let mut jobs = futures::stream::FuturesUnordered::new(); + for &(index, input) in &enabled_inputs { + let job = std::panic::AssertUnwindSafe(process_input_job(index, &ctx, input, true)).catch_unwind(); + jobs.push(async move { + match job.await { + Ok(result) => result, + Err(_) => panicked_input_job(index, input), + } + }); + } + while let Some(result) = jobs.next().await { + job_results.push(result); + } + } else { + for &(index, input) in &enabled_inputs { + job_results.push(process_input_job(index, &ctx, input, false).await); + } + } + job_results.sort_by_key(|result| result.index); + + let mut blockers = Vec::new(); + let mut accepted_empty_clusters = ClusterFlags::empty(); + for mut result in job_results { + super::super::input_status::persist_input_job_result(&ctx, &result).await; + report_input_job_completion(&ctx.events, &ctx.run_id, ctx.execution_order, &result); + errors.append(&mut result.errors); + input_stats.insert(result.input_name.clone(), result.stat); + if result.state == InputJobState::Ready { + accepted_empty_clusters |= result.accepted_empty_clusters; + if let (Some(input), Some(source)) = + (sources.get_input_by_name(&result.input_name), result.source.take()) + { + source_playlists.push(FetchedPlaylist { input, source, epg: result.epg }); + } + } else { + blockers.push(result.input_name); + } + } + + if !disabled_inputs.is_empty() && !source_downloaded { + warn!( + "Source at index {source_idx} has no enabled inputs for the given targets. Disabled: {}", + join_arc_strs(&disabled_inputs, ", ") + ); + } + if source_downloaded { + if !blockers.is_empty() { + for target in source.targets.iter().filter(|target| is_target_enabled(target, &ctx.user_targets)) { + for input_name in &blockers { + broadcast_step("Playlist download", &target_waiting_message(&target.name, input_name)); + } + } + } else if source_playlists.is_empty() { + debug!("Source at index {source_idx} is empty"); + errors.push(TuliproxError::RepositoryPlaylist(format!( + "Source at index {source_idx} is empty: {}", + join_arc_strs(&source.inputs, ", ") + ))); + } else { + debug_if_enabled!( + "Source has {} groups", + source_playlists.iter_mut().map(FetchedPlaylist::get_channel_count).sum::() + ); + let enabled_targets: Vec<_> = + source.targets.iter().filter(|target| is_target_enabled(target, &ctx.user_targets)).collect(); + target_stats = process_targets( + &ctx, + &mut source_playlists, + &enabled_targets, + &mut input_stats, + &mut errors, + accepted_empty_clusters, + process_parallel, + ) + .await; + } + } + } + log_memory_snapshot(format!("source[{source_idx}] end").as_str()); + let ordered_input_stats = sources + .get_source_at(source_idx) + .map_or_else(Vec::new, |source| source.inputs.iter().filter_map(|name| input_stats.remove(name)).collect()); + (ordered_input_stats, target_stats, errors) +} + +pub(crate) async fn download_input_epg( + ctx: &PlaylistProcessingContext, + input: &Arc, + error_list: &mut Vec, +) -> Option { + // A failed playlist download makes the EPG moot: the channels it would annotate are + // not there. + if !error_list.is_empty() { + return None; + } + let provider = XmltvEpgProvider::new(ctx); + // The XMLTV path produces documents, not programme records, so nothing reaches the + // sink. It is here because the same call answers for a record-streaming provider. + let mut discarded = CountingEpgSink::new(); + let outcome = provider.fetch(&EpgFetchRequest::new(input), &mut discarded).await; + error_list.extend(provider.take_errors()); + match outcome { + Ok(outcome) => outcome.into_guide(), + Err(err) => { + error_list.push(err); + None + } + } +} + +pub(crate) fn create_broadcast_callback( + events: &E, + run_id: &PlaylistUpdateRunId, + execution_order: PlaylistUpdateRunOrder, +) -> StepMeasureCallback { + let events = events.clone(); + let run_id = run_id.clone(); + Box::new(move |context: &str, msg: &str| { + events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent::for_run_global( + run_id.clone(), + execution_order, + context, + msg, + ))); + }) +} + +fn create_input_broadcast_callback( + events: &E, + run_id: &PlaylistUpdateRunId, + execution_order: PlaylistUpdateRunOrder, + input_id: u16, +) -> StepMeasureCallback { + let events = events.clone(); + let run_id = run_id.clone(); + Box::new(move |context: &str, msg: &str| { + events.emit(EventMessage::PlaylistUpdateProgress(PlaylistUpdateProgressEvent::for_run_input( + run_id.clone(), + execution_order, + input_id, + context, + msg, + ))); + }) +} + +pub(crate) fn create_input_stat( + group_count: usize, + channel_count: usize, + error_count: usize, + input_type: InputType, + input_name: &str, + secs_took: u64, +) -> InputStats { + InputStats { + name: input_name.to_string(), + input_type, + error_count, + raw_stats: PlaylistStats { group_count, channel_count }, + processed_stats: PlaylistStats { group_count: 0, channel_count: 0 }, + secs_took, + } +} + +// Written out rather than derived: `#[derive(Clone)]` would demand `M: Clone`, +// but the sink is held behind an `Arc` and is cloneable whatever `M` is. +impl Clone for PlaylistProcessingContext { + fn clone(&self) -> Self { + Self { + client: self.client.clone(), + run_id: self.run_id.clone(), + execution_order: self.execution_order, + config: Arc::clone(&self.config), + user_targets: Arc::clone(&self.user_targets), + events: self.events.clone(), + playlist_state: self.playlist_state.clone(), + disabled_headers: self.disabled_headers.clone(), + processed_inputs: Arc::clone(&self.processed_inputs), + input_completions: Arc::clone(&self.input_completions), + input_locks: Arc::clone(&self.input_locks), + provider_manager: self.provider_manager.clone(), + metadata_manager: self.metadata_manager.clone(), + pre_processed_inputs: self.pre_processed_inputs.clone(), + stalker_refresh_mode: self.stalker_refresh_mode, + partial_refresh: Arc::clone(&self.partial_refresh), + had_quality_rejections: Arc::clone(&self.had_quality_rejections), + input_refresh: self.input_refresh, + library_update_mode: self.library_update_mode, + } + } +} + +impl PlaylistProcessingContext { + #[must_use] + pub fn refresh_policy(&self, input_id: u16) -> InputRefreshPolicy { + self.input_refresh + .filter(|input_refresh| input_refresh.input_id == input_id) + .map_or(InputRefreshPolicy::NORMAL, |input_refresh| input_refresh.policy) + } + + pub(super) fn acquisition(&self, input_id: u16) -> InputAcquisition { + if self.library_update_mode.reloads_input(input_id) { + InputAcquisition::RescannedLibrary + } else { + InputAcquisition::Provider(self.refresh_policy(input_id)) + } + } + + pub async fn is_input_downloaded(&self, input_name: &str) -> bool { + let processed = self.processed_inputs.lock().await; + processed.contains(input_name) + } + pub async fn mark_input_downloaded(&self, input_name: Arc) -> bool { + let mut processed = self.processed_inputs.lock().await; + processed.insert(input_name) + } + + pub async fn get_input_lock(&self, input_name: &Arc) -> OwnedRwLockWriteGuard<()> { + let mut locks = self.input_locks.lock().await; + // Try to upgrade the existing weak reference + let lock = locks.get(input_name).and_then(Weak::upgrade).unwrap_or_else(|| { + let new_lock = Arc::new(RwLock::new(())); + locks.insert(input_name.clone(), Arc::downgrade(&new_lock)); + new_lock + }); + + // Clean up stale references periodically + locks.retain(|_, weak| weak.strong_count() > 0); + + drop(locks); // Release mutex before awaiting write lock + lock.write_owned().await + } +} + +pub(crate) async fn process_sources( + processing_ctx: &PlaylistProcessingContext, +) -> (Vec, Vec) { + let mut async_tasks = JoinSet::new(); + let sources = processing_ctx.config.sources.load(); + let process_parallel = processing_ctx.config.config.load().process_parallel; + if process_parallel && log_enabled!(Level::Debug) { + debug!("Parallel processing enabled"); + } + + let mut source_results = Vec::new(); + let mut errors = Vec::new(); + let mut processed_any = false; + + for (index, source) in sources.sources.iter().enumerate() { + if !source.should_process_for_user_targets(&processing_ctx.user_targets) { + continue; + } + + // We're using the file lock this way on purpose + let source_lock_path = PathBuf::from(concat_string!("source_", &index.to_string())); + let Ok(update_lock) = processing_ctx.config.file_locks.try_write_lock(&source_lock_path).await else { + warn!( + "The update operation for the source at index {index} was skipped because an update is already in progress." + ); + continue; + }; + + let ctx = Arc::new(processing_ctx.clone()); + + processed_any = true; + if process_parallel { + async_tasks.spawn(async move { + let _update_lock = update_lock; + (index, process_source(index, ctx).await) + }); + } else { + source_results.push((index, process_source(index, ctx).await)); + drop(update_lock); + } + } + if !processed_any { + warn!( + "No sources were processed for the given targets. Check that:\n\ + - Sources have enabled targets matching your target selection\n\ + - CLI -t filter or schedule.targets are correct\n\ + - No playlist lock is blocking updates" + ); + } + while let Some(result) = async_tasks.join_next().await { + match result { + Ok(result) => source_results.push(result), + Err(err) => { + error!("Playlist processing task failed: {err:?}"); + errors + .push(TuliproxError::RepositoryPlaylist(format!("Playlist source processing task failed: {err}"))); + } + } + } + + source_results.sort_by_key(|(index, _)| *index); + let mut stats = Vec::with_capacity(source_results.len()); + for (_, (input_stats, target_stats, mut source_errors)) in source_results { + errors.append(&mut source_errors); + if let Some(source_stats) = SourceStats::try_new(input_stats, target_stats) { + stats.push(source_stats); + } + } + (stats, errors) +} diff --git a/backend/processing/src/processor/playlist/ingest/overlay.rs b/backend/processing/src/processor/playlist/ingest/overlay.rs new file mode 100644 index 000000000..43e73a7e9 --- /dev/null +++ b/backend/processing/src/processor/playlist/ingest/overlay.rs @@ -0,0 +1,447 @@ +use super::{cluster_selected, GroupCategoryIds, PlaylistDownloadResult, StagedOverlayAssignment, StreamOverlap}; +use log::warn; +use shared::model::{ClusterFlags, InputType, PlaylistGroup, StagedInputType, XtreamCluster}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; +use tuliprox_core::model::{ConfigInput, ConfigInputFlags}; + +fn prepare_staged_xtream_group( + mut group: PlaylistGroup, + provider: &ConfigInput, + provider_urls: &HashMap<(XtreamCluster, u32), &Arc>, + username: &str, + password: &str, + live_stream_use_prefix: bool, + live_stream_without_extension: bool, +) -> PlaylistGroup { + let initial_count = group.channels.len(); + group.channels.retain_mut(|item| { + // The staged item's ID identifies its original Xtream stream. + let Ok(stream_id) = item.header.id.parse::() else { + return false; + }; + // Existing streams keep the provider's playback URL, including a direct source URL. + item.header.url = provider_urls.get(&(group.xtream_cluster, stream_id)).map_or_else( + || { + let metadata_extension = item.header.get_container_extension(); + let container_extension = if group.xtream_cluster == XtreamCluster::Video { + metadata_extension.as_deref().or_else(|| { + shared::utils::extract_extension_from_url(&item.header.url) + .and_then(|extension| extension.strip_prefix('.')) + }) + } else { + None + }; + tuliprox_parser::xtream::get_xtream_url( + group.xtream_cluster, + &provider.url, + username, + password, + stream_id, + container_extension, + live_stream_use_prefix, + live_stream_without_extension, + ) + .into() + }, + |url| Arc::clone(url), + ); + item.header.input_name = Arc::clone(&provider.name); + true + }); + let dropped = initial_count - group.channels.len(); + if dropped > 0 { + warn!("Skipped {dropped} staged channel(s) in group '{}' without a numeric Xtream stream ID", group.title); + } + group +} + +/// Overlays the staged source on the provider playlist of the clusters it covers. +/// +/// The staged playlist is the provider playlist restructured by an external tool: group and channel +/// names may differ, the ids do not. Staged groups are therefore resolved against the provider groups +/// by id, never by name ([`match_staged_groups`]). An overlaid group keeps the provider category id; +/// a staged group that introduces a new category keeps its own id only when that id is still free. A +/// staged group whose channels carry no numeric stream id leaves the provider category untouched for +/// Xtream staged input, while an M3U staged input remains authoritative for the selected clusters. +pub(crate) fn apply_staged_overlay_groups( + provider: &ConfigInput, + staged_type: StagedInputType, + clusters: ClusterFlags, + provider_groups: Vec, + staged_groups: Vec, +) -> Vec { + if provider.input_type != InputType::Xtream { + let mut groups: Vec = + provider_groups.into_iter().filter(|group| !cluster_selected(group.xtream_cluster, clusters)).collect(); + groups.extend(staged_groups.into_iter().filter(|group| cluster_selected(group.xtream_cluster, clusters)).map( + |mut group| { + for item in &mut group.channels { + item.header.input_name = Arc::clone(&provider.name); + } + group + }, + )); + return groups; + } + + let (Some(username), Some(password)) = (provider.username.as_deref(), provider.password.as_deref()) else { + warn!("Skipping staged channels for Xtream input '{}' without credentials", provider.name); + return provider_groups; + }; + let live_stream_use_prefix = provider.has_flag(ConfigInputFlags::XtreamLiveStreamUsePrefix); + let live_stream_without_extension = provider.has_flag(ConfigInputFlags::XtreamLiveStreamWithoutExtension); + + let staged_stream_ids: HashSet<(XtreamCluster, u32)> = staged_groups + .iter() + .filter(|group| cluster_selected(group.xtream_cluster, clusters)) + .flat_map(|group| { + group.channels.iter().filter_map(|item| { + item.header.id.parse::().ok().map(|stream_id| (group.xtream_cluster, stream_id)) + }) + }) + .collect(); + let mut provider_urls = HashMap::with_capacity(staged_stream_ids.len()); + for group in &provider_groups { + if !cluster_selected(group.xtream_cluster, clusters) { + continue; + } + for item in &group.channels { + if let Ok(stream_id) = item.header.id.parse::() { + let key = (group.xtream_cluster, stream_id); + if staged_stream_ids.contains(&key) { + provider_urls.entry(key).or_insert(&item.header.url); + } + } + } + } + + let mut staged_selected: Vec> = staged_groups + .into_iter() + .filter(|group| cluster_selected(group.xtream_cluster, clusters)) + .map(|group| { + Some(prepare_staged_xtream_group( + group, + provider, + &provider_urls, + username, + password, + live_stream_use_prefix, + live_stream_without_extension, + )) + }) + .collect(); + + // Category ids of the provider playlist are authoritative and stay reserved across the overlay: + // persistence keys input groups by `(cluster, id)`, so two groups of one cluster sharing an id + // would be collapsed into a single category. Only staged ids are renumbered; a provider playlist + // that already repeats a category id is passed through unchanged. + let mut used_ids = GroupCategoryIds::from_groups(&provider_groups); + let assignment = match_staged_groups(&provider_groups, &staged_selected, staged_type, clusters); + + if staged_type == StagedInputType::M3u { + return apply_staged_m3u_template(provider_groups, staged_selected, &assignment, clusters, used_ids); + } + + let mut result: Vec = Vec::with_capacity(provider_groups.len() + staged_selected.len()); + + for (provider_idx, provider_group) in provider_groups.into_iter().enumerate() { + if !cluster_selected(provider_group.xtream_cluster, clusters) { + result.push(provider_group); + continue; + } + + let matched = assignment.staged_of(provider_idx).and_then(|staged_idx| staged_selected[staged_idx].take()); + let Some(mut staged) = matched.filter(|staged| !staged.channels.is_empty()) else { + result.push(provider_group); + continue; + }; + + // The provider category id is the identity of the overlaid category: a staged group matched + // by stream ids or title takes it over instead of keeping its own numbering. + staged.id = + if provider_group.id == 0 { used_ids.allocate(provider_group.xtream_cluster) } else { provider_group.id }; + sync_channel_category_id(&mut staged); + result.push(staged); + } + + // Append newly introduced staged groups that had no corresponding provider group. + for mut staged in staged_selected.into_iter().flatten().filter(|staged| !staged.channels.is_empty()) { + used_ids.assign_new_group(&mut staged); + sync_channel_category_id(&mut staged); + result.push(staged); + } + + result +} + +fn apply_staged_m3u_template( + provider_groups: Vec, + staged_groups: Vec>, + assignment: &StagedOverlayAssignment, + clusters: ClusterFlags, + mut used_ids: GroupCategoryIds, +) -> Vec { + let mut result = Vec::with_capacity(provider_groups.len() + staged_groups.len()); + for (staged_idx, staged) in staged_groups.into_iter().enumerate() { + let Some(mut staged) = staged.filter(|group| !group.channels.is_empty()) else { + continue; + }; + if let Some(provider_idx) = assignment.provider_by_staged[staged_idx] { + let provider_group = &provider_groups[provider_idx]; + staged.id = if provider_group.id == 0 { + used_ids.allocate(provider_group.xtream_cluster) + } else { + provider_group.id + }; + } else { + used_ids.assign_new_group(&mut staged); + } + sync_channel_category_id(&mut staged); + result.push(staged); + } + result.extend(provider_groups.into_iter().filter(|group| !cluster_selected(group.xtream_cluster, clusters))); + result +} + +/// Keeps the channels of a group consistent with its final category id. +/// +/// Persistence re-syncs this through [`PlaylistGroup::on_load`], but the overlay result is consumed as +/// a whole: a reader must not see the category id of the staged source on a group that was just given +/// the provider category id. +fn sync_channel_category_id(group: &mut PlaylistGroup) { + let category_id = group.id; + for item in &mut group.channels { + item.header.category_id = category_id; + } +} + +impl StagedOverlayAssignment { + pub(super) fn new(provider_len: usize, staged_len: usize) -> Self { + Self { + staged_by_provider: vec![None; provider_len], + provider_by_staged: vec![None; staged_len], + has_stream_id_evidence: vec![false; staged_len], + } + } + + /// Assigns `staged_idx` to `provider_idx`; returns `false` when either side is already taken. + pub(super) fn assign(&mut self, provider_idx: usize, staged_idx: usize) -> bool { + if self.staged_by_provider[provider_idx].is_some() || self.provider_by_staged[staged_idx].is_some() { + return false; + } + self.staged_by_provider[provider_idx] = Some(staged_idx); + self.provider_by_staged[staged_idx] = Some(provider_idx); + true + } + + pub(super) fn staged_of(&self, provider_idx: usize) -> Option { self.staged_by_provider[provider_idx] } + + pub(super) fn is_assigned(&self, staged_idx: usize) -> bool { self.provider_by_staged[staged_idx].is_some() } +} + +/// Resolves every staged group against the provider groups it overlays. +/// +/// The staged playlist is the provider playlist restructured by an external tool: group and channel +/// names may be rewritten, the ids do not. The passes therefore rank the id evidence above the name: +/// +/// 1. the channel stream ids decide. A staged channel carries the provider stream id, and the provider +/// playlist itself states which category owns a stream, so this also covers staged sources whose +/// group ids are not provider category ids (a `m3u` staged playlist numbers its groups positionally). +/// 2. the category id decides for Xtream staged groups whose channels the provider playlist does not know. +/// M3U staged group ids are positional and cannot identify provider categories. +/// 3. the title remains the last resort for staged groups without provider stream-id evidence. +fn match_staged_groups( + provider_groups: &[PlaylistGroup], + staged_groups: &[Option], + staged_type: StagedInputType, + clusters: ClusterFlags, +) -> StagedOverlayAssignment { + let mut assignment = StagedOverlayAssignment::new(provider_groups.len(), staged_groups.len()); + match_staged_by_stream_id(provider_groups, staged_groups, clusters, &mut assignment); + if staged_type == StagedInputType::Xtream { + match_staged_by_category_id(provider_groups, staged_groups, clusters, &mut assignment); + } + match_staged_by_title(provider_groups, staged_groups, clusters, &mut assignment); + assignment +} + +/// Pass 1: a staged group overlays the provider category owning its streams, strongest overlap first. +fn match_staged_by_stream_id( + provider_groups: &[PlaylistGroup], + staged_groups: &[Option], + clusters: ClusterFlags, + assignment: &mut StagedOverlayAssignment, +) { + // Only the staged stream ids are indexed, so the memory cost is bounded by the overlay and not + // by the size of the provider playlist. + let staged_stream_ids: HashSet<(XtreamCluster, &str)> = staged_groups + .iter() + .flatten() + .flat_map(|staged| staged.channels.iter().map(|item| (staged.xtream_cluster, item.header.id.as_ref()))) + .filter(|(_, stream_id)| !stream_id.is_empty()) + .collect(); + if staged_stream_ids.is_empty() { + return; + } + + let mut provider_idx_by_stream_id: HashMap<(XtreamCluster, &str), usize> = HashMap::new(); + for (provider_idx, provider_group) in provider_groups.iter().enumerate() { + if !cluster_selected(provider_group.xtream_cluster, clusters) { + continue; + } + for item in &provider_group.channels { + let key = (provider_group.xtream_cluster, item.header.id.as_ref()); + if staged_stream_ids.contains(&key) { + provider_idx_by_stream_id.entry(key).or_insert(provider_idx); + } + } + } + if provider_idx_by_stream_id.is_empty() { + return; + } + + // Every staged group is offered all provider categories owning its streams, ranked by overlap: when + // a stronger overlap takes the best-matching category, the next candidate can still claim the group. + let mut candidates: Vec = Vec::new(); + let mut hits_by_provider: HashMap = HashMap::new(); + for (staged_idx, staged) in staged_groups.iter().enumerate() { + let Some(staged) = staged else { + continue; + }; + hits_by_provider.clear(); + for item in &staged.channels { + if let Some(provider_idx) = provider_idx_by_stream_id.get(&(staged.xtream_cluster, item.header.id.as_ref())) + { + *hits_by_provider.entry(*provider_idx).or_insert(0) += 1; + } + } + assignment.has_stream_id_evidence[staged_idx] = !hits_by_provider.is_empty(); + candidates.extend(hits_by_provider.iter().map(|(provider_idx, hits)| StreamOverlap { + hits: *hits, + staged_idx, + provider_idx: *provider_idx, + })); + } + + // The strongest overlap wins first, so a staged group never loses its category to a weaker candidate. + candidates + .sort_unstable_by_key(|overlap| (std::cmp::Reverse(overlap.hits), overlap.staged_idx, overlap.provider_idx)); + for overlap in candidates { + assignment.assign(overlap.provider_idx, overlap.staged_idx); + } +} + +/// Pass 2: a staged group overlays the provider category carrying its id. +fn match_staged_by_category_id( + provider_groups: &[PlaylistGroup], + staged_groups: &[Option], + clusters: ClusterFlags, + assignment: &mut StagedOverlayAssignment, +) { + let mut provider_idx_by_id: HashMap<(XtreamCluster, u32), usize> = HashMap::new(); + for (provider_idx, provider_group) in provider_groups.iter().enumerate() { + if provider_group.id != 0 && cluster_selected(provider_group.xtream_cluster, clusters) { + provider_idx_by_id.entry((provider_group.xtream_cluster, provider_group.id)).or_insert(provider_idx); + } + } + + for (staged_idx, staged) in staged_groups.iter().enumerate() { + let Some(staged) = staged else { + continue; + }; + if staged.id == 0 || assignment.is_assigned(staged_idx) || assignment.has_stream_id_evidence[staged_idx] { + continue; + } + if let Some(provider_idx) = provider_idx_by_id.get(&(staged.xtream_cluster, staged.id)).copied() { + assignment.assign(provider_idx, staged_idx); + } + } +} + +/// Pass 3: a staged group overlays the provider category of the same name. +fn match_staged_by_title( + provider_groups: &[PlaylistGroup], + staged_groups: &[Option], + clusters: ClusterFlags, + assignment: &mut StagedOverlayAssignment, +) { + // Staged groups of the same name are interchangeable, so one lookup per provider category is enough. + let mut staged_by_title: HashMap<(XtreamCluster, &str), Vec> = HashMap::new(); + for (staged_idx, staged) in staged_groups.iter().enumerate() { + if assignment.is_assigned(staged_idx) || assignment.has_stream_id_evidence[staged_idx] { + continue; + } + if let Some(staged) = staged { + staged_by_title.entry((staged.xtream_cluster, staged.title.as_ref())).or_default().push(staged_idx); + } + } + + for (provider_idx, provider_group) in provider_groups.iter().enumerate() { + if assignment.staged_of(provider_idx).is_some() || !cluster_selected(provider_group.xtream_cluster, clusters) { + continue; + } + let key = (provider_group.xtream_cluster, provider_group.title.as_ref()); + let Some(free_groups) = staged_by_title.get_mut(&key) else { + continue; + }; + if !free_groups.is_empty() { + // Buckets stay tiny, so taking the first free group keeps the earlier first-match semantics. + let staged_idx = free_groups.remove(0); + assignment.assign(provider_idx, staged_idx); + } + } +} + +impl GroupCategoryIds { + pub(super) fn from_groups(groups: &[PlaylistGroup]) -> Self { + let mut ids = Self::default(); + for group in groups { + if group.id != 0 { + ids.used.entry(group.xtream_cluster).or_default().insert(group.id); + } + } + for (cluster, used) in &ids.used { + ids.next_free.insert(*cluster, used.iter().copied().max().unwrap_or(0).saturating_add(1)); + } + ids + } + + /// Claims `id` for `cluster`; a zero id or an id already in use is not claimable. + pub(super) fn claim(&mut self, cluster: XtreamCluster, id: u32) -> bool { + id != 0 && self.used.entry(cluster).or_default().insert(id) + } + + pub(super) fn assign_new_group(&mut self, group: &mut PlaylistGroup) { + if group.id == 0 { + group.id = self.allocate(group.xtream_cluster); + } else if !self.claim(group.xtream_cluster, group.id) { + let allocated = self.allocate(group.xtream_cluster); + warn!( + "Staged group '{}' uses category id {} which is already taken in {}; renumbered to {allocated}", + group.title, group.id, group.xtream_cluster + ); + group.id = allocated; + } + } + + /// Reserves and returns the next free id of `cluster`. + pub(super) fn allocate(&mut self, cluster: XtreamCluster) -> u32 { + let used = self.used.entry(cluster).or_default(); + let mut candidate = self.next_free.get(&cluster).copied().unwrap_or(1); + // Zero is never a valid category id. The loop wraps at the u32 edge and ends at the latest + // when it reaches the first free id, which the finite id set always contains. + while candidate == 0 || used.contains(&candidate) { + candidate = candidate.checked_add(1).unwrap_or(1); + } + used.insert(candidate); + self.next_free.insert(cluster, candidate.saturating_add(1)); + candidate + } +} + +pub(crate) fn should_apply_staged_overlay(download_result: &PlaylistDownloadResult) -> bool { + !download_result.was_cached +} diff --git a/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests.rs b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests.rs new file mode 100644 index 000000000..120df816c --- /dev/null +++ b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests.rs @@ -0,0 +1,56 @@ +use super::{ + telemetry::{finalize_input_telemetry, input_telemetry_for_fetch, persist_finalized_cluster_snapshots}, + *, +}; +use shared::model::{ + ConfigInputOptionsDto, ConfigInputUpdateQualityDto, PersistedPlaylistUpdateClusterSnapshot, + PersistedPlaylistUpdateQualityDecision, PersistedPlaylistUpdateQualitySnapshot, + PersistedPlaylistUpdateTechnicalState, PlaylistUpdateClusterDecision, PlaylistUpdateClusterTelemetry, + PlaylistUpdateDataSource, +}; +use tuliprox_core::model::{ClusterUpdateAcceptance, ConfigInput}; +use tuliprox_iptv::provider::PlaylistFetch; + +fn input(input_type: InputType) -> ConfigInput { + ConfigInput { id: 17, name: Arc::from("provider-a"), input_type, enabled: true, ..ConfigInput::default() } +} + +fn cluster(telemetry: &PlaylistUpdateInputTelemetry, cluster: XtreamCluster) -> &PlaylistUpdateClusterTelemetry { + telemetry.clusters.iter().find(|item| item.cluster == cluster).expect("configured cluster telemetry") +} + +const fn completed_snapshot_context( + effective_policy: InputRefreshPolicy, + request_policy: Option, + provider_cluster_count: usize, +) -> PersistedSnapshotContext { + PersistedSnapshotContext { + effective_policy, + request_policy, + technical_failure: false, + storage_failure: false, + provider_failure_evidence: ProviderFailureEvidence::None, + provider_cluster_count, + } +} + +const fn failed_provider_snapshot_context( + effective_policy: InputRefreshPolicy, + request_policy: Option, + provider_cluster_count: usize, +) -> PersistedSnapshotContext { + PersistedSnapshotContext { + effective_policy, + request_policy, + technical_failure: true, + storage_failure: false, + provider_failure_evidence: ProviderFailureEvidence::TechnicalFailure, + provider_cluster_count, + } +} + +mod playlist; +mod policy; +mod storage; +mod streaming; +mod transport; diff --git a/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/playlist.rs b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/playlist.rs new file mode 100644 index 000000000..7ead372e3 --- /dev/null +++ b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/playlist.rs @@ -0,0 +1,53 @@ +use super::*; + +#[test] +fn pipeline_transparency_known_cluster_failure_keeps_quality_decision_and_neutralizes_overlay() { + let input = input(InputType::Xtream); + let acceptance = ClusterUpdateAcceptance { + cluster: XtreamCluster::Series, + current_count: Some(100), + candidate_count: 95, + threshold: 90, + quality: Some(95), + }; + let mut fetch = PlaylistFetch::groups(Vec::new()).with_quality_acceptances(vec![acceptance]); + fetch.record_cluster_error(XtreamCluster::Series, TuliproxError::RepositoryXtream("publish failure".to_string())); + let telemetry = input_telemetry_for_fetch( + &input, + InputRefreshPolicy::NORMAL, + PlaylistUpdateDataSource::Provider, + &PIPELINE_TRANSPARENCY_CLUSTERS, + Some(&fetch), + ); + let mut result = PlaylistDownloadResult::from(fetch).with_input_telemetry(telemetry); + finalize_input_telemetry(&mut result, None); + let telemetry = result.input_telemetry.as_mut().unwrap(); + let shows = cluster(telemetry, XtreamCluster::Series); + assert_eq!(shows.decision, Some(PlaylistUpdateClusterDecision::Accepted)); + assert_eq!(shows.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Failed)); + assert_eq!(shows.active_count, None); + let context = failed_provider_snapshot_context(InputRefreshPolicy::NORMAL, None, 3); + let snapshot = persisted_cluster_snapshot(shows, context); + assert_eq!(snapshot.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Failed)); + assert_eq!(snapshot.quality.unwrap().decision, PersistedPlaylistUpdateQualityDecision::Accepted); + neutralize_overlaid_cluster_facts(telemetry, ClusterFlags::Series); + let overlaid = cluster(telemetry, XtreamCluster::Series); + assert_eq!(overlaid.technical_state, None); + assert_eq!(persisted_cluster_snapshot(overlaid, context).technical_state, None); +} + +#[test] +fn pipeline_transparency_telemetry_does_not_assign_an_unscoped_error_to_multiple_clusters() { + let input = input(InputType::Stalker); + let fetch = PlaylistFetch::failed(TuliproxError::Download("provider unavailable".to_string())); + + let telemetry = input_telemetry_for_fetch( + &input, + InputRefreshPolicy::NORMAL, + PlaylistUpdateDataSource::Provider, + &PIPELINE_TRANSPARENCY_CLUSTERS, + Some(&fetch), + ); + + assert!(telemetry.clusters.iter().all(|item| item.requested && item.decision.is_none())); +} diff --git a/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/policy.rs b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/policy.rs new file mode 100644 index 000000000..ee0c67273 --- /dev/null +++ b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/policy.rs @@ -0,0 +1,145 @@ +use super::*; + +#[test] +fn pipeline_transparency_telemetry_preserves_cache_rejection_and_force_decisions() { + let input = input(InputType::Xtream); + let fetch = PlaylistFetch::groups(Vec::new()) + .with_quality_rejections(vec![ClusterUpdateRejection { + cluster: XtreamCluster::Video, + current_count: 12_543, + candidate_count: 217, + threshold: 90, + quality: 1, + }]) + .with_force_updates(vec![ClusterForceUpdate { + cluster: XtreamCluster::Series, + candidate_count: 8_412, + configured_threshold: 100, + }]); + + let telemetry = input_telemetry_for_fetch( + &input, + InputRefreshPolicy::FORCE, + PlaylistUpdateDataSource::Provider, + &[XtreamCluster::Video, XtreamCluster::Series], + Some(&fetch), + ); + + let live = cluster(&telemetry, XtreamCluster::Live); + assert!(live.requested); + assert_eq!(live.source, Some(PlaylistUpdateDataSource::Cache)); + assert_eq!(live.decision, None); + + let video = cluster(&telemetry, XtreamCluster::Video); + assert_eq!(video.source, Some(PlaylistUpdateDataSource::Provider)); + assert_eq!(video.decision, Some(PlaylistUpdateClusterDecision::Rejected)); + assert_eq!( + (video.baseline_count, video.candidate_count, video.active_count), + (Some(12_543), Some(217), Some(12_543)) + ); + assert_eq!((video.threshold, video.quality), (Some(90), Some(1))); + + let series = cluster(&telemetry, XtreamCluster::Series); + assert_eq!(series.source, Some(PlaylistUpdateDataSource::Provider)); + assert_eq!(series.decision, Some(PlaylistUpdateClusterDecision::Accepted)); + assert_eq!((series.candidate_count, series.active_count), (Some(8_412), None)); + assert_eq!(series.threshold, Some(100)); + + let mut published = telemetry; + confirm_published_cluster_counts(&mut published); + assert_eq!(cluster(&published, XtreamCluster::Series).active_count, Some(8_412)); +} + +#[test] +fn pipeline_transparency_telemetry_preserves_accepted_quality_and_bootstrap_facts() { + let mut input = input(InputType::Xtream); + input.options = Some(ConfigInputOptions::from(&ConfigInputOptionsDto { + update_quality: ConfigInputUpdateQualityDto { live: 90, vod: 90, series: 0 }, + ..ConfigInputOptionsDto::default() + })); + let fetch = PlaylistFetch::groups(Vec::new()).with_quality_acceptances(vec![ + ClusterUpdateAcceptance { + cluster: XtreamCluster::Live, + current_count: Some(12_543), + candidate_count: 12_000, + threshold: 90, + quality: Some(95), + }, + ClusterUpdateAcceptance { + cluster: XtreamCluster::Video, + current_count: None, + candidate_count: 217, + threshold: 90, + quality: None, + }, + ]); + + let mut telemetry = input_telemetry_for_fetch( + &input, + InputRefreshPolicy::NORMAL, + PlaylistUpdateDataSource::Provider, + &PIPELINE_TRANSPARENCY_CLUSTERS, + Some(&fetch), + ); + + let live = cluster(&telemetry, XtreamCluster::Live); + assert_eq!(live.decision, Some(PlaylistUpdateClusterDecision::Accepted)); + assert_eq!((live.baseline_count, live.candidate_count), (Some(12_543), Some(12_000))); + assert_eq!((live.threshold, live.quality, live.active_count), (Some(90), Some(95), None)); + + let video = cluster(&telemetry, XtreamCluster::Video); + assert_eq!(video.decision, Some(PlaylistUpdateClusterDecision::Accepted)); + assert_eq!((video.baseline_count, video.candidate_count), (None, Some(217))); + assert_eq!((video.threshold, video.quality, video.active_count), (Some(90), None, None)); + + confirm_published_cluster_counts(&mut telemetry); + assert_eq!(cluster(&telemetry, XtreamCluster::Live).active_count, Some(12_000)); + assert_eq!(cluster(&telemetry, XtreamCluster::Video).active_count, Some(217)); +} + +#[test] +fn pipeline_transparency_telemetry_keeps_zero_threshold_quality_disabled() { + let input = input(InputType::Xtream); + let fetch = PlaylistFetch::groups(Vec::new()); + + let telemetry = input_telemetry_for_fetch( + &input, + InputRefreshPolicy::NORMAL, + PlaylistUpdateDataSource::Provider, + &[XtreamCluster::Live], + Some(&fetch), + ); + + let live = cluster(&telemetry, XtreamCluster::Live); + assert_eq!(live.decision, Some(PlaylistUpdateClusterDecision::Accepted)); + assert_eq!(live.threshold, None); + assert_eq!(live.quality, None); + assert_eq!(live.active_count, None); +} + +#[test] +fn pipeline_transparency_telemetry_does_not_infer_an_unreported_quality_acceptance() { + let mut input = input(InputType::Xtream); + input.options = Some(ConfigInputOptions::from(&ConfigInputOptionsDto { + skip_vod: true, + update_quality: ConfigInputUpdateQualityDto { live: 90, ..ConfigInputUpdateQualityDto::default() }, + ..ConfigInputOptionsDto::default() + })); + let fetch = PlaylistFetch::groups(Vec::new()); + + let telemetry = input_telemetry_for_fetch( + &input, + InputRefreshPolicy::NORMAL, + PlaylistUpdateDataSource::Provider, + &[XtreamCluster::Live], + Some(&fetch), + ); + + let live = cluster(&telemetry, XtreamCluster::Live); + assert_eq!(live.threshold, Some(90)); + assert_eq!(live.decision, None, "an accepted quality decision was not retained by the provider result"); + let video = cluster(&telemetry, XtreamCluster::Video); + assert!(!video.requested); + assert_eq!(video.source, None); + assert_eq!(video.decision, None); +} diff --git a/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/storage.rs b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/storage.rs new file mode 100644 index 000000000..59d0a4691 --- /dev/null +++ b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/storage.rs @@ -0,0 +1,311 @@ +use super::*; + +#[test] +fn persisted_cluster_snapshot_preserves_quality_facts_without_merging_technical_state() { + let accepted = PlaylistUpdateClusterTelemetry { + cluster: XtreamCluster::Live, + requested: true, + source: Some(PlaylistUpdateDataSource::Provider), + baseline_count: Some(1_000), + candidate_count: Some(950), + active_count: Some(950), + threshold: Some(90), + quality: Some(95), + decision: Some(PlaylistUpdateClusterDecision::Accepted), + technical_state: None, + }; + + let completed = persisted_cluster_snapshot( + &accepted, + completed_snapshot_context(InputRefreshPolicy::NORMAL, Some(InputRefreshPolicy::REFRESH), 1), + ); + assert_eq!(completed.policy, Some(InputRefreshPolicy::REFRESH)); + assert_eq!(completed.source, Some(PlaylistUpdateDataSource::Provider)); + assert_eq!(completed.quality_guard_threshold, None, "an evaluation already owns its threshold"); + assert_eq!( + completed.quality, + Some(PersistedPlaylistUpdateQualitySnapshot { + threshold: 90, + baseline_count: Some(1_000), + candidate_count: Some(950), + achieved_quality: Some(95), + decision: PersistedPlaylistUpdateQualityDecision::Accepted, + }) + ); + assert_eq!(completed.active_count, Some(950)); + assert_eq!(completed.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Succeeded)); + + let mut bootstrap = accepted.clone(); + bootstrap.baseline_count = None; + bootstrap.candidate_count = Some(42); + bootstrap.active_count = Some(42); + bootstrap.quality = None; + let bootstrap = + persisted_cluster_snapshot(&bootstrap, completed_snapshot_context(InputRefreshPolicy::NORMAL, None, 1)); + assert_eq!(bootstrap.policy, None, "an automatic run must not acquire a default policy"); + assert_eq!(bootstrap.quality_guard_threshold, None, "bootstrap evaluation already owns its threshold"); + assert_eq!( + bootstrap.quality, + Some(PersistedPlaylistUpdateQualitySnapshot { + threshold: 90, + baseline_count: None, + candidate_count: Some(42), + achieved_quality: None, + decision: PersistedPlaylistUpdateQualityDecision::Accepted, + }), + "bootstrap acceptance must not invent achieved Quality" + ); + + let mut rejected = accepted.clone(); + rejected.candidate_count = Some(217); + rejected.active_count = Some(1_000); + rejected.quality = Some(21); + rejected.decision = Some(PlaylistUpdateClusterDecision::Rejected); + let rejected = + persisted_cluster_snapshot(&rejected, completed_snapshot_context(InputRefreshPolicy::NORMAL, None, 1)); + assert_eq!( + rejected.quality.map(|quality| quality.decision), + Some(PersistedPlaylistUpdateQualityDecision::Rejected) + ); + assert_eq!(rejected.active_count, Some(1_000), "only the typed active baseline is retained"); + assert_eq!(rejected.quality_guard_threshold, None, "a rejection evaluation already owns its threshold"); + assert_eq!(rejected.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Succeeded)); + + let mut accepted_then_failed = accepted; + accepted_then_failed.active_count = None; + let accepted_then_failed = persisted_cluster_snapshot( + &accepted_then_failed, + PersistedSnapshotContext { + effective_policy: InputRefreshPolicy::NORMAL, + request_policy: None, + technical_failure: true, + storage_failure: true, + provider_failure_evidence: ProviderFailureEvidence::None, + provider_cluster_count: 1, + }, + ); + assert_eq!( + accepted_then_failed.quality.map(|quality| quality.decision), + Some(PersistedPlaylistUpdateQualityDecision::Accepted) + ); + assert_eq!(accepted_then_failed.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Failed)); + assert_eq!(accepted_then_failed.active_count, None); +} + +#[test] +fn persisted_cluster_snapshot_keeps_policy_source_and_disabled_guard_independent() { + let forced = PlaylistUpdateClusterTelemetry { + cluster: XtreamCluster::Video, + requested: true, + source: Some(PlaylistUpdateDataSource::Provider), + baseline_count: None, + candidate_count: Some(73), + active_count: Some(73), + threshold: Some(95), + quality: None, + decision: Some(PlaylistUpdateClusterDecision::Accepted), + technical_state: None, + }; + + let forced = persisted_cluster_snapshot( + &forced, + completed_snapshot_context(InputRefreshPolicy::FORCE, Some(InputRefreshPolicy::FORCE), 1), + ); + assert_eq!(forced.policy, Some(InputRefreshPolicy::FORCE)); + assert_eq!(forced.source, Some(PlaylistUpdateDataSource::Provider)); + assert_eq!(forced.quality_guard_threshold, Some(95)); + assert_eq!(forced.quality, None, "a Force bypass is not a Quality evaluation"); + + let disabled = PlaylistUpdateClusterTelemetry { + cluster: XtreamCluster::Series, + requested: true, + source: Some(PlaylistUpdateDataSource::Provider), + baseline_count: None, + candidate_count: Some(12), + active_count: Some(12), + threshold: None, + quality: None, + decision: Some(PlaylistUpdateClusterDecision::Accepted), + technical_state: None, + }; + let disabled = persisted_cluster_snapshot( + &disabled, + completed_snapshot_context(InputRefreshPolicy::FORCE, Some(InputRefreshPolicy::FORCE), 1), + ); + assert_eq!(disabled.quality_guard_threshold, Some(0)); + assert_eq!(disabled.quality, None, "threshold zero stays disabled even during Force"); + + let cached = PlaylistUpdateClusterTelemetry { + cluster: XtreamCluster::Live, + requested: true, + source: Some(PlaylistUpdateDataSource::Cache), + baseline_count: None, + candidate_count: None, + active_count: None, + threshold: Some(90), + quality: None, + decision: None, + technical_state: None, + }; + let cached = persisted_cluster_snapshot( + &cached, + completed_snapshot_context(InputRefreshPolicy::REFRESH, Some(InputRefreshPolicy::REFRESH), 0), + ); + assert_eq!(cached.policy, Some(InputRefreshPolicy::REFRESH)); + assert_eq!(cached.quality_guard_threshold, Some(90)); + assert_eq!( + cached.source, + Some(PlaylistUpdateDataSource::Cache), + "the actual source remains stronger than request policy" + ); +} + +#[test] +fn persisted_cluster_snapshot_final_status_boundary_replaces_provider_with_cache_without_history() { + let temp = tempfile::tempdir().expect("temporary status directory"); + let mut status = input_cache::InputStatus::default(); + status.clusters.insert( + XtreamCluster::Live.as_ref().to_string(), + input_cache::ClusterStatus { status: input_cache::ClusterState::Ok, timestamp: 17, last_update: None }, + ); + let provider_telemetry = PlaylistUpdateInputTelemetry { + refresh_policy: InputRefreshPolicy::REFRESH, + source: None, + clusters: vec![PlaylistUpdateClusterTelemetry { + cluster: XtreamCluster::Live, + requested: true, + source: Some(PlaylistUpdateDataSource::Provider), + baseline_count: Some(100), + candidate_count: Some(95), + active_count: Some(95), + threshold: Some(90), + quality: Some(95), + decision: Some(PlaylistUpdateClusterDecision::Accepted), + technical_state: None, + }], + }; + let mut provider_result = PlaylistDownloadResult::new(Vec::new(), Vec::new(), false, false) + .with_input_telemetry(provider_telemetry) + .with_pending_status(PendingInputStatusUpdate { + storage_path: temp.path().to_path_buf(), + status, + status_changed: true, + cluster_status_source: ClusterStatusSource::PerCluster, + }); + + persist_finalized_cluster_snapshots(&mut provider_result, Some(InputRefreshPolicy::REFRESH), None); + + let provider_status = input_cache::load_input_status(temp.path()); + let provider_snapshot = provider_status.clusters["live"].last_update.expect("provider snapshot"); + assert_eq!(provider_snapshot.source, Some(PlaylistUpdateDataSource::Provider)); + assert!(provider_snapshot.quality.is_some()); + + let cache_telemetry = PlaylistUpdateInputTelemetry { + refresh_policy: InputRefreshPolicy::NORMAL, + source: None, + clusters: vec![PlaylistUpdateClusterTelemetry { + cluster: XtreamCluster::Live, + requested: true, + source: Some(PlaylistUpdateDataSource::Cache), + baseline_count: None, + candidate_count: None, + active_count: None, + threshold: Some(90), + quality: None, + decision: None, + technical_state: None, + }], + }; + let mut cache_result = PlaylistDownloadResult::new(Vec::new(), Vec::new(), true, false) + .with_input_telemetry(cache_telemetry) + .with_pending_status(PendingInputStatusUpdate { + storage_path: temp.path().to_path_buf(), + status: provider_status, + status_changed: false, + cluster_status_source: ClusterStatusSource::PerCluster, + }); + + persist_finalized_cluster_snapshots(&mut cache_result, Some(InputRefreshPolicy::NORMAL), None); + + let cached_status = input_cache::load_input_status(temp.path()); + assert_eq!(cached_status.clusters.len(), 1); + let cached_snapshot = cached_status.clusters["live"].last_update.expect("replacement cache snapshot"); + assert_eq!(cached_snapshot.policy, Some(InputRefreshPolicy::NORMAL)); + assert_eq!(cached_snapshot.source, Some(PlaylistUpdateDataSource::Cache)); + assert_eq!(cached_snapshot.quality_guard_threshold, Some(90)); + assert_eq!(cached_snapshot.quality, None); + assert_eq!(cached_snapshot.active_count, None); + assert_eq!(cached_snapshot.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Succeeded)); + let encoded = std::fs::read_to_string(temp.path().join(input_cache::STATUS_FILE)).expect("persisted status JSON"); + assert_eq!(encoded.matches(r#""last_update""#).count(), 1); + assert!(!encoded.contains("history")); +} + +#[test] +fn persisted_cluster_snapshot_discards_unproven_staged_overlay_facts() { + let mut telemetry = PlaylistUpdateInputTelemetry { + refresh_policy: InputRefreshPolicy::FORCE, + source: None, + clusters: vec![PlaylistUpdateClusterTelemetry { + cluster: XtreamCluster::Video, + requested: true, + source: Some(PlaylistUpdateDataSource::Provider), + baseline_count: Some(100), + candidate_count: Some(95), + active_count: Some(95), + threshold: Some(90), + quality: Some(95), + decision: Some(PlaylistUpdateClusterDecision::Accepted), + technical_state: None, + }], + }; + neutralize_overlaid_cluster_facts(&mut telemetry, ClusterFlags::Vod); + + let snapshot = persisted_cluster_snapshot( + cluster(&telemetry, XtreamCluster::Video), + completed_snapshot_context(telemetry.refresh_policy, Some(InputRefreshPolicy::FORCE), 0), + ); + + assert_eq!(snapshot, PersistedPlaylistUpdateClusterSnapshot::default()); +} + +#[test] +fn persisted_cluster_snapshot_leaves_unscoped_multi_cluster_failure_unknown() { + let cluster = PlaylistUpdateClusterTelemetry { + cluster: XtreamCluster::Live, + requested: true, + source: Some(PlaylistUpdateDataSource::Provider), + baseline_count: Some(100), + candidate_count: Some(95), + active_count: None, + threshold: Some(90), + quality: Some(95), + decision: Some(PlaylistUpdateClusterDecision::Accepted), + technical_state: None, + }; + + let snapshot = + persisted_cluster_snapshot(&cluster, failed_provider_snapshot_context(InputRefreshPolicy::NORMAL, None, 2)); + + assert_eq!(snapshot.technical_state, None); + assert_eq!( + snapshot.quality.map(|quality| quality.decision), + Some(PersistedPlaylistUpdateQualityDecision::Accepted) + ); + + let unscoped_ancillary_failure = persisted_cluster_snapshot( + &cluster, + PersistedSnapshotContext { + effective_policy: InputRefreshPolicy::NORMAL, + request_policy: None, + technical_failure: true, + storage_failure: false, + provider_failure_evidence: ProviderFailureEvidence::None, + provider_cluster_count: 1, + }, + ); + assert_eq!( + unscoped_ancillary_failure.technical_state, None, + "an input-wide ancillary failure is not invented as a cluster-scoped provider failure" + ); +} diff --git a/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/streaming.rs b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/streaming.rs new file mode 100644 index 000000000..6bb126108 --- /dev/null +++ b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/streaming.rs @@ -0,0 +1,38 @@ +use super::*; + +#[test] +fn pipeline_transparency_accepted_quality_survives_fetch_error_without_active_claim() { + let mut input = input(InputType::Xtream); + input.options = Some(ConfigInputOptions::from(&ConfigInputOptionsDto { + update_quality: ConfigInputUpdateQualityDto { live: 90, ..ConfigInputUpdateQualityDto::default() }, + ..ConfigInputOptionsDto::default() + })); + let acceptance = ClusterUpdateAcceptance { + cluster: XtreamCluster::Live, + current_count: Some(100), + candidate_count: 95, + threshold: 90, + quality: Some(95), + }; + let fetch = PlaylistFetch::groups(Vec::new()) + .with_quality_acceptances(vec![acceptance]) + .with_errors(vec![TuliproxError::RepositoryXtream("post-quality publish failure".to_string())]); + let telemetry = input_telemetry_for_fetch( + &input, + InputRefreshPolicy::NORMAL, + PlaylistUpdateDataSource::Provider, + &[XtreamCluster::Live], + Some(&fetch), + ); + let mut download_result = PlaylistDownloadResult::from(fetch).with_input_telemetry(telemetry); + + finalize_input_telemetry(&mut download_result, None); + + let telemetry = download_result.input_telemetry.as_ref().expect("input telemetry"); + let live = cluster(telemetry, XtreamCluster::Live); + assert_eq!(live.decision, Some(PlaylistUpdateClusterDecision::Accepted)); + assert_eq!((live.baseline_count, live.candidate_count), (Some(100), Some(95))); + assert_eq!((live.threshold, live.quality), (Some(90), Some(95))); + assert_eq!(live.active_count, None); + assert_eq!(download_result.download_err.len(), 1); +} diff --git a/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/transport.rs b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/transport.rs new file mode 100644 index 000000000..1df0ebae8 --- /dev/null +++ b/backend/processing/src/processor/playlist/ingest/pipeline_transparency_tests/transport.rs @@ -0,0 +1,45 @@ +use super::*; + +#[test] +fn pipeline_transparency_telemetry_reports_non_cluster_cache_without_synthetic_clusters() { + let telemetry = input_telemetry_for_fetch( + &input(InputType::Plex), + InputRefreshPolicy::NORMAL, + PlaylistUpdateDataSource::Cache, + &[], + None, + ); + + assert_eq!(telemetry.refresh_policy, InputRefreshPolicy::NORMAL); + assert_eq!(telemetry.source, Some(PlaylistUpdateDataSource::Cache)); + assert!(telemetry.clusters.is_empty()); +} + +#[test] +fn pipeline_transparency_telemetry_types_complete_provider_failure_without_guessing_metrics() { + let mut input = input(InputType::Xtream); + input.options = Some(ConfigInputOptions::from(&ConfigInputOptionsDto { + skip_vod: true, + skip_series: true, + ..ConfigInputOptionsDto::default() + })); + let fetch = PlaylistFetch::failed(TuliproxError::Download("provider unavailable".to_string())); + + let telemetry = input_telemetry_for_fetch( + &input, + InputRefreshPolicy::REFRESH, + PlaylistUpdateDataSource::Provider, + &[XtreamCluster::Live], + Some(&fetch), + ); + + assert_eq!(telemetry.refresh_policy, InputRefreshPolicy::REFRESH); + let live = cluster(&telemetry, XtreamCluster::Live); + assert!(live.requested); + assert_eq!(live.source, Some(PlaylistUpdateDataSource::Provider)); + assert_eq!(live.decision, Some(PlaylistUpdateClusterDecision::TechnicalError)); + assert!(live.baseline_count.is_none()); + assert!(live.candidate_count.is_none()); + assert!(live.active_count.is_none()); + assert!(live.quality.is_none()); +} diff --git a/backend/processing/src/processor/playlist/ingest/status.rs b/backend/processing/src/processor/playlist/ingest/status.rs new file mode 100644 index 000000000..96577120f --- /dev/null +++ b/backend/processing/src/processor/playlist/ingest/status.rs @@ -0,0 +1,201 @@ +use super::{ + telemetry::{input_telemetry_for_fetch, is_cluster_input}, + PendingInputStatusUpdate, PlaylistDownloadResult, PlaylistProcessingContext, +}; +use crate::metadata_sink::MetadataUpdateSink; +use log::info; +use shared::{ + error::TuliproxError, + model::{ + ClusterFlags, EventMessage, EventSink, InputRefreshPolicy, InputType, PlaylistGroup, PlaylistUpdateDataSource, + PlaylistUpdateInputTelemetry, PlaylistUpdateProgressEvent, XtreamCluster, + }, +}; +use std::collections::HashSet; +use tuliprox_core::model::{ClusterForceUpdate, ConfigInput, ConfigInputFlags, ProcessTargets}; +use tuliprox_iptv::{provider::PlaylistFetch, xtream}; +use tuliprox_repository::PlaylistSource; + +// Inputs disabled in the config are always disabled. +// Command-line targets can only restrict enabled inputs, never enable them. +pub(crate) fn is_input_enabled(input: &ConfigInput, user_targets: &ProcessTargets) -> bool { + input.enabled && (!user_targets.enabled || user_targets.has_input(input.id)) +} + +pub(crate) async fn with_sequential_group( + file_locks: &tuliprox_core::utils::FileLockManager, + group: Option, + process_parallel: bool, + future: impl std::future::Future, +) -> T { + let _guard = if process_parallel { + if let Some(group) = group { + Some(file_locks.write_lock_str(&format!("sequential_group:{group}")).await) + } else { + None + } + } else { + None + }; + future.await +} + +#[derive(Clone, Copy)] +pub(super) enum ClusterStatusSource { + PerCluster, + Default, +} + +#[derive(Clone, Copy, Default)] +pub(super) enum ProviderFailureEvidence { + #[default] + None, + TechnicalFailure, +} + +impl PlaylistDownloadResult { + pub fn new( + downloaded_playlist: Vec, + download_err: Vec, + was_cached: bool, + persisted: bool, + ) -> Self { + Self { + downloaded_playlist, + download_err, + quality_rejections: Vec::new(), + force_updates: Vec::new(), + was_cached, + persisted, + partial: false, + input_telemetry: None, + pending_status: None, + provider_failure_evidence: ProviderFailureEvidence::None, + } + } + + pub(super) fn with_input_telemetry(mut self, input_telemetry: PlaylistUpdateInputTelemetry) -> Self { + self.input_telemetry = Some(input_telemetry); + self + } + + pub(super) fn with_pending_status(mut self, pending_status: PendingInputStatusUpdate) -> Self { + self.pending_status = Some(pending_status); + self + } +} + +impl From for PlaylistDownloadResult { + fn from(fetch: PlaylistFetch) -> Self { + let provider_failure_evidence = if fetch.partial || !fetch.errors.is_empty() { + ProviderFailureEvidence::TechnicalFailure + } else { + ProviderFailureEvidence::None + }; + Self { + downloaded_playlist: fetch.groups, + download_err: fetch.errors, + quality_rejections: fetch.quality_rejections, + force_updates: fetch.force_updates, + was_cached: false, + persisted: fetch.persisted, + partial: fetch.partial, + input_telemetry: None, + pending_status: None, + provider_failure_evidence, + } + } +} + +pub(super) fn cached_playlist_download_result( + input: &ConfigInput, + refresh_policy: InputRefreshPolicy, +) -> PlaylistDownloadResult { + PlaylistDownloadResult::new(vec![], vec![], true, false).with_input_telemetry(input_telemetry_for_fetch( + input, + refresh_policy, + PlaylistUpdateDataSource::Cache, + &[], + None, + )) +} + +/// Loads an input already acquired in this run without reporting a second acquisition. +pub(super) fn in_run_reuse_playlist_download_result() -> PlaylistDownloadResult { + PlaylistDownloadResult::new(vec![], vec![], true, false) +} + +pub(super) fn forced_empty_cluster_flags(force_updates: &[ClusterForceUpdate]) -> ClusterFlags { + force_updates.iter().filter(|update| update.candidate_count == 0).fold( + ClusterFlags::empty(), + |mut clusters, update| { + clusters.insert(match update.cluster { + XtreamCluster::Live => ClusterFlags::Live, + XtreamCluster::Video => ClusterFlags::Vod, + XtreamCluster::Series => ClusterFlags::Series, + }); + clusters + }, + ) +} + +pub(crate) fn collect_effective_skip_clusters(input: &ConfigInput) -> Vec { + if !input.input_type.is_xtream() && input.input_type != InputType::M3u { + return vec![]; + } + xtream::get_skip_cluster(input) +} + +pub(super) fn report_forced_update_request( + ctx: &PlaylistProcessingContext, + input: &ConfigInput, + refresh_policy: InputRefreshPolicy, +) { + if !refresh_policy.bypasses_quality() { + return; + } + let input_type = input.get_download_input_type(); + if !is_cluster_input(input_type) { + return; + } + let clusters = [ + (ConfigInputFlags::SkipLive, "live"), + (ConfigInputFlags::SkipVod, "vod"), + (ConfigInputFlags::SkipSeries, "series"), + ] + .into_iter() + .filter_map(|(skip_flag, name)| (!input.has_flag(skip_flag)).then_some(name)) + .collect::>() + .join(","); + let message = + format!("Input '{}': forced update requested; cache and update-quality bypassed for {clusters}", input.name); + info!("{message}"); + ctx.events.emit(EventMessage::PlaylistUpdateProgress( + PlaylistUpdateProgressEvent::for_run_input( + ctx.run_id.clone(), + ctx.execution_order, + input.id, + input.name.to_string(), + message, + ) + .with_detail(shared::model::PlaylistUpdateProgressDetail::ForcedUpdateRequested), + )); +} + +pub(crate) fn filter_skipped_clusters_from_source(source: PlaylistSource, input: &ConfigInput) -> PlaylistSource { + let skip_clusters = collect_effective_skip_clusters(input); + if skip_clusters.is_empty() { + return source; + } + + let skip_set: HashSet = skip_clusters.into_iter().collect(); + PlaylistSource::filtered(source, skip_set) +} + +pub(crate) fn cluster_selected(cluster: XtreamCluster, clusters: ClusterFlags) -> bool { + match cluster { + XtreamCluster::Live => clusters.contains(ClusterFlags::Live), + XtreamCluster::Video => clusters.contains(ClusterFlags::Vod), + XtreamCluster::Series => clusters.contains(ClusterFlags::Series), + } +} diff --git a/backend/processing/src/processor/playlist/ingest/telemetry.rs b/backend/processing/src/processor/playlist/ingest/telemetry.rs new file mode 100644 index 000000000..1562dd0da --- /dev/null +++ b/backend/processing/src/processor/playlist/ingest/telemetry.rs @@ -0,0 +1,288 @@ +use super::{ + cluster_selected, + status::{ClusterStatusSource, ProviderFailureEvidence}, + PersistedSnapshotContext, PlaylistDownloadResult, +}; +use crate::input_cache; +use shared::{ + error::TuliproxError, + model::{ + ClusterFlags, InputRefreshPolicy, InputType, PersistedPlaylistUpdateClusterSnapshot, + PersistedPlaylistUpdateQualityDecision, PersistedPlaylistUpdateQualitySnapshot, + PersistedPlaylistUpdateTechnicalState, PlaylistUpdateClusterDecision, PlaylistUpdateClusterTelemetry, + PlaylistUpdateDataSource, PlaylistUpdateInputTelemetry, XtreamCluster, + }, +}; +use std::collections::HashMap; +use tuliprox_core::model::{ConfigInput, ConfigInputFlags}; +use tuliprox_iptv::provider::PlaylistFetch; + +pub(in crate::processor::playlist) const PIPELINE_TRANSPARENCY_CLUSTERS: [XtreamCluster; 3] = + [XtreamCluster::Live, XtreamCluster::Video, XtreamCluster::Series]; + +pub(super) fn is_cluster_input(input_type: InputType) -> bool { + matches!(input_type, InputType::Xtream | InputType::Stalker | InputType::M3u) +} + +pub(in crate::processor::playlist) fn cluster_is_configured(input: &ConfigInput, cluster: XtreamCluster) -> bool { + let skip_flag = match cluster { + XtreamCluster::Live => ConfigInputFlags::SkipLive, + XtreamCluster::Video => ConfigInputFlags::SkipVod, + XtreamCluster::Series => ConfigInputFlags::SkipSeries, + }; + !input.has_flag(skip_flag) +} + +pub(super) fn input_telemetry_for_fetch( + input: &ConfigInput, + refresh_policy: InputRefreshPolicy, + source: PlaylistUpdateDataSource, + provider_clusters: &[XtreamCluster], + fetch: Option<&PlaylistFetch>, +) -> PlaylistUpdateInputTelemetry { + let input_type = input.get_download_input_type(); + if !is_cluster_input(input_type) { + return PlaylistUpdateInputTelemetry { refresh_policy, source: Some(source), clusters: Vec::new() }; + } + + let mut candidate_counts = HashMap::::new(); + if let Some(fetch) = fetch { + for group in &fetch.groups { + *candidate_counts.entry(group.xtream_cluster).or_default() += group.channels.len(); + } + if fetch.errors.is_empty() && !fetch.partial && !fetch.persisted { + for cluster in provider_clusters { + candidate_counts.entry(*cluster).or_default(); + } + } + } + + let clusters = PIPELINE_TRANSPARENCY_CLUSTERS + .into_iter() + .map(|cluster| { + let requested = cluster_is_configured(input, cluster); + let source = requested.then(|| { + if provider_clusters.contains(&cluster) { + PlaylistUpdateDataSource::Provider + } else { + PlaylistUpdateDataSource::Cache + } + }); + let configured_threshold = + input.options.as_ref().map_or(0, |options| options.update_quality.threshold(cluster)); + let mut telemetry = PlaylistUpdateClusterTelemetry { + cluster, + requested, + source, + baseline_count: None, + candidate_count: candidate_counts.get(&cluster).copied(), + active_count: None, + threshold: (configured_threshold > 0).then_some(configured_threshold), + quality: None, + decision: None, + technical_state: None, + }; + + if let Some(fetch) = fetch { + if requested && fetch.failed_clusters.contains(&cluster) { + telemetry.technical_state = Some(PersistedPlaylistUpdateTechnicalState::Failed); + } + if let Some(rejection) = fetch.quality_rejections.iter().find(|item| item.cluster == cluster) { + telemetry.baseline_count = Some(rejection.current_count); + telemetry.candidate_count = Some(rejection.candidate_count); + telemetry.active_count = Some(rejection.current_count); + telemetry.threshold = Some(rejection.threshold); + telemetry.quality = Some(rejection.quality); + telemetry.decision = Some(PlaylistUpdateClusterDecision::Rejected); + } else if let Some(acceptance) = fetch.quality_acceptances.iter().find(|item| item.cluster == cluster) { + telemetry.baseline_count = acceptance.current_count; + telemetry.candidate_count = Some(acceptance.candidate_count); + telemetry.threshold = Some(acceptance.threshold); + telemetry.quality = acceptance.quality; + telemetry.decision = Some(PlaylistUpdateClusterDecision::Accepted); + } else if let Some(force_update) = fetch.force_updates.iter().find(|item| item.cluster == cluster) { + telemetry.candidate_count = Some(force_update.candidate_count); + telemetry.threshold = + (force_update.configured_threshold > 0).then_some(force_update.configured_threshold); + telemetry.decision = Some(PlaylistUpdateClusterDecision::Accepted); + } else if source == Some(PlaylistUpdateDataSource::Provider) { + if configured_threshold == 0 + && (candidate_counts.contains_key(&cluster) || (fetch.errors.is_empty() && !fetch.partial)) + { + telemetry.decision = Some(PlaylistUpdateClusterDecision::Accepted); + } else if (provider_clusters == [cluster] || input_type == InputType::M3u) + && fetch.groups.is_empty() + && !fetch.persisted + && !fetch.errors.is_empty() + { + telemetry.decision = Some(PlaylistUpdateClusterDecision::TechnicalError); + } + } + } + telemetry + }) + .collect(); + + PlaylistUpdateInputTelemetry { refresh_policy, source: None, clusters } +} + +pub(super) fn confirm_published_cluster_counts(input_telemetry: &mut PlaylistUpdateInputTelemetry) { + for cluster in &mut input_telemetry.clusters { + if cluster.decision == Some(PlaylistUpdateClusterDecision::Accepted) { + cluster.active_count = cluster.candidate_count; + } + } +} + +pub(super) fn finalize_input_telemetry( + playlist_download_result: &mut PlaylistDownloadResult, + storage_error: Option<&TuliproxError>, +) { + let activation_is_uncertain = storage_error.is_some() + || playlist_download_result.partial + || !playlist_download_result.download_err.is_empty(); + if let Some(input_telemetry) = playlist_download_result.input_telemetry.as_mut() { + if activation_is_uncertain { + for cluster in &mut input_telemetry.clusters { + cluster.active_count = None; + } + } else { + confirm_published_cluster_counts(input_telemetry); + } + } +} + +pub(crate) fn neutralize_overlaid_cluster_facts( + input_telemetry: &mut PlaylistUpdateInputTelemetry, + overlaid_clusters: ClusterFlags, +) { + for cluster in &mut input_telemetry.clusters { + if cluster_selected(cluster.cluster, overlaid_clusters) { + cluster.source = None; + cluster.baseline_count = None; + cluster.candidate_count = None; + cluster.active_count = None; + cluster.quality = None; + cluster.decision = None; + cluster.technical_state = None; + } + } +} + +fn persisted_quality_snapshot( + cluster: &PlaylistUpdateClusterTelemetry, + effective_policy: InputRefreshPolicy, +) -> Option { + if effective_policy.bypasses_quality() { + return None; + } + let threshold = cluster.threshold?; + let decision = match cluster.decision? { + PlaylistUpdateClusterDecision::Accepted => PersistedPlaylistUpdateQualityDecision::Accepted, + PlaylistUpdateClusterDecision::Rejected => PersistedPlaylistUpdateQualityDecision::Rejected, + PlaylistUpdateClusterDecision::TechnicalError => return None, + }; + Some(PersistedPlaylistUpdateQualitySnapshot { + threshold, + baseline_count: cluster.baseline_count, + candidate_count: cluster.candidate_count, + achieved_quality: cluster.quality, + decision, + }) +} + +fn persisted_technical_state( + cluster: &PlaylistUpdateClusterTelemetry, + context: PersistedSnapshotContext, +) -> Option { + let source = cluster.source?; + if cluster.technical_state == Some(PersistedPlaylistUpdateTechnicalState::Failed) { + return cluster.technical_state; + } + if cluster.decision == Some(PlaylistUpdateClusterDecision::TechnicalError) { + return Some(PersistedPlaylistUpdateTechnicalState::Failed); + } + if context.storage_failure && source == PlaylistUpdateDataSource::Provider { + return Some(PersistedPlaylistUpdateTechnicalState::Failed); + } + if !context.technical_failure { + return Some(PersistedPlaylistUpdateTechnicalState::Succeeded); + } + (source == PlaylistUpdateDataSource::Provider + && context.provider_cluster_count == 1 + && matches!(context.provider_failure_evidence, ProviderFailureEvidence::TechnicalFailure)) + .then_some(PersistedPlaylistUpdateTechnicalState::Failed) +} + +pub(super) fn persisted_cluster_snapshot( + cluster: &PlaylistUpdateClusterTelemetry, + context: PersistedSnapshotContext, +) -> PersistedPlaylistUpdateClusterSnapshot { + let quality = persisted_quality_snapshot(cluster, context.effective_policy); + PersistedPlaylistUpdateClusterSnapshot { + policy: cluster.source.and(context.request_policy), + source: cluster.source, + quality_guard_threshold: (quality.is_none() && cluster.source.is_some()) + .then_some(cluster.threshold.unwrap_or(0)), + quality, + active_count: cluster.active_count, + technical_state: persisted_technical_state(cluster, context), + } +} + +pub(super) fn persist_finalized_cluster_snapshots( + playlist_download_result: &mut PlaylistDownloadResult, + request_policy: Option, + storage_error: Option<&TuliproxError>, +) { + let technical_failure = storage_error.is_some() + || playlist_download_result.partial + || !playlist_download_result.download_err.is_empty(); + let provider_failure_evidence = playlist_download_result.provider_failure_evidence; + let snapshots = playlist_download_result.input_telemetry.as_ref().map_or_else(Vec::new, |telemetry| { + let provider_cluster_count = telemetry + .clusters + .iter() + .filter(|cluster| cluster.requested && cluster.source == Some(PlaylistUpdateDataSource::Provider)) + .count(); + telemetry + .clusters + .iter() + .filter(|cluster| cluster.requested) + .map(|cluster| { + let snapshot = persisted_cluster_snapshot( + cluster, + PersistedSnapshotContext { + effective_policy: telemetry.refresh_policy, + request_policy, + technical_failure, + storage_failure: storage_error.is_some(), + provider_failure_evidence, + provider_cluster_count, + }, + ); + (cluster.cluster, snapshot) + }) + .collect() + }); + + let Some(mut pending_status) = playlist_download_result.pending_status.take() else { + return; + }; + let mut changed = pending_status.status_changed; + for (cluster, snapshot) in snapshots { + changed |= match pending_status.cluster_status_source { + ClusterStatusSource::PerCluster => { + input_cache::replace_cluster_snapshot(&mut pending_status.status, cluster.as_ref(), snapshot) + } + ClusterStatusSource::Default => input_cache::replace_cluster_snapshot_from_default_status( + &mut pending_status.status, + cluster.as_ref(), + snapshot, + ), + }; + } + if changed { + input_cache::save_input_status(&pending_status.storage_path, &pending_status.status); + } +} diff --git a/backend/processing/src/processor/playlist/input_status_tests.rs b/backend/processing/src/processor/playlist/input_status_tests.rs deleted file mode 100644 index 01cfeb650..000000000 --- a/backend/processing/src/processor/playlist/input_status_tests.rs +++ /dev/null @@ -1,404 +0,0 @@ -mod persisted_input_status { - use super::*; - use crate::processor::playlist::input_status::{persist_input_completion, persist_input_job_result}; - use shared::model::PersistedPlaylistUpdateInputResult; - - fn acquired_result(state: PlaylistUpdateState) -> InputJobResult { - let mut result = super::super::playlist_update_run_result( - 7, - InputJobState::Ready, - vec![], - state == PlaylistUpdateState::Partial, - ); - result.input_telemetry = Some(PlaylistUpdateInputTelemetry { - refresh_policy: InputRefreshPolicy::REFRESH, - source: Some(PlaylistUpdateDataSource::Provider), - clusters: Vec::new(), - }); - result - } - - #[tokio::test] - async fn playlist_update_status_run_precedence_is_isolated_by_stable_input_id() { - let temp = tempfile::tempdir().unwrap(); - let ctx = processing_context(temp.path()); - let failed = super::super::playlist_update_run_result(7, InputJobState::Failed, vec![], false); - persist_input_job_result(&ctx, &failed).await; - let mut independent = acquired_result(PlaylistUpdateState::Success); - independent.input_id = 8; - independent.input_name = "independent".intern(); - persist_input_job_result(&ctx, &independent).await; - for (result, state) in [(failed, PlaylistUpdateState::Failure), (independent, PlaylistUpdateState::Success)] { - let path = - input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &result.input_name).await; - assert_eq!(input_cache::load_input_status(&path).last_input_update.unwrap().state, state); - } - } - - #[tokio::test] - async fn m3u_update_quality_playlist_update_status_staged_completion_is_independent_of_parent_failure() { - for (parent_fails, staged_fails) in [(false, false), (true, false), (false, true)] { - let temp = tempfile::tempdir().unwrap(); - let events = CollectSink::default(); - let ctx = processing_context_with_events(temp.path(), events.clone()); - let parent_path = temp.path().join("parent.m3u"); - let staged_path = temp.path().join("overlay.m3u"); - for (path, fails) in [(&parent_path, parent_fails), (&staged_path, staged_fails)] { - if !fails { - std::fs::write(path, "#EXTM3U\n#EXTINF:-1,Fixture\nhttp://stream.example/fixture\n").unwrap(); - } - } - let parent = Arc::new(ConfigInput { - id: 7, - name: "parent".intern(), - input_type: InputType::M3u, - url: parent_path.to_string_lossy().into_owned(), - enabled: true, - ..ConfigInput::default() - }); - let mut staged = ConfigInput { - id: 8, - name: "staged".intern(), - input_type: InputType::Staged, - url: staged_path.to_string_lossy().into_owned(), - enabled: true, - staged: Some(tuliprox_core::model::ConfigInputStaged { - for_input: Some(parent.name.clone()), - clusters: ClusterFlags::Live, - }), - ..ConfigInput::default() - }; - staged.resolve_staged_download_type(); - let staged = Arc::new(staged); - ctx.config.sources.store(Arc::new(SourcesConfig { - inputs: vec![parent.clone(), staged.clone()], - ..SourcesConfig::default() - })); - let result = download_input(&ctx, &parent, false).await; - assert_eq!(!result.errors.is_empty(), parent_fails || staged_fails); - let path = input_storage_path(&ctx, &staged).await; - let persisted = input_cache::load_input_status(&path); - assert_eq!( - persisted.last_input_update.unwrap().state, - if staged_fails { PlaylistUpdateState::Failure } else { PlaylistUpdateState::Success } - ); - assert_eq!(persisted.clusters.len(), 4); - assert!(persisted.clusters["default"].last_update.is_none(), "document cache and cluster facts stay separate"); - for cluster in XTREAM_CLUSTER { - assert!(persisted.clusters[cluster.as_ref()].last_update.is_some()); - } - let completions = staged_completion::input_completions(&events, staged.id); - assert_eq!(completions.len(), 1, "real staged download must report its own completion"); - assert_eq!(completions[0].state, persisted.last_input_update.map(|result| result.state)); - assert_eq!(completions[0].run_id, Some(ctx.run_id.clone())); - assert_eq!(completions[0].execution_order, Some(ctx.execution_order)); - let telemetry = completions[0].input_telemetry.as_ref().unwrap(); - assert_eq!(telemetry.source, None); - assert!(telemetry.clusters.iter().all(|cluster| cluster.source == Some(PlaylistUpdateDataSource::Provider))); - assert!(staged_completion::input_completions(&events, parent.id).is_empty()); - } - } - - #[tokio::test] - async fn playlist_update_status_delayed_source_results_keep_strongest_completion_until_next_run() { - for weaker in [PlaylistUpdateState::Success, PlaylistUpdateState::Partial] { - for failure_first in [false, true] { - let temp = tempfile::tempdir().unwrap(); - let events = CollectSink::default(); - let ctx = processing_context_with_events(temp.path(), events.clone()); - let acquired = acquired_result(weaker); - let failed_reuse = super::super::playlist_update_run_result(7, InputJobState::Failed, vec![], false); - let (first, delayed) = if failure_first { (failed_reuse, acquired) } else { (acquired, failed_reuse) }; - let path = - input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &first.input_name).await; - let mut original = input_cache::InputStatus::default(); - original.clusters.insert( - "live".into(), - input_cache::ClusterStatus { - status: input_cache::ClusterState::Failed, - timestamp: 123, - last_update: Some(PersistedPlaylistUpdateClusterSnapshot { - source: Some(PlaylistUpdateDataSource::Cache), - policy: Some(InputRefreshPolicy::FORCE), - quality_guard_threshold: Some(95), - ..PersistedPlaylistUpdateClusterSnapshot::default() - }), - }, - ); - input_cache::save_input_status(&path, &original); - let (release, other_input_finished) = tokio::sync::oneshot::channel(); - let delayed_ctx = ctx.clone(); - // Source A already holds its shared-input result, but cannot drain - // its sorted result list until another input job has completed. - let delayed_source = async move { - let source_results = vec![delayed]; - other_input_finished.await.unwrap(); - for result in source_results { - persist_input_job_result(&delayed_ctx, &result).await; - report_input_job_completion( - &delayed_ctx.events, - &delayed_ctx.run_id, - delayed_ctx.execution_order, - &result, - ); - } - }; - let first_source = async { - persist_input_job_result(&ctx, &first).await; - report_input_job_completion(&ctx.events, &ctx.run_id, ctx.execution_order, &first); - let mut persisted = input_cache::load_input_status(&path); - assert_eq!(persisted.last_input_update.unwrap().state, first.update_state()); - // Distinct sentinel proves a delayed weaker result does not - // advance the stronger completion's timestamp, without sleeps. - persisted.last_input_update.as_mut().unwrap().timestamp = 456; - input_cache::save_input_status(&path, &persisted); - release.send(()).unwrap(); - }; - tokio::join!(delayed_source, first_source); - let final_status = input_cache::load_input_status(&path); - let completion = final_status.last_input_update.unwrap(); - assert_eq!( - completion.state, - PlaylistUpdateState::Failure, - "weaker={weaker:?}, failure_first={failure_first}" - ); - if failure_first { - assert_eq!(completion.timestamp, 456); - } - assert_eq!(final_status.clusters, original.clusters); - let states: Vec<_> = events - .0 - .lock() - .unwrap() - .iter() - .filter_map(|event| match event { - EventMessage::PlaylistUpdateProgress(progress) => progress.state, - _ => None, - }) - .collect(); - assert_eq!( - states, - if failure_first { - vec![PlaylistUpdateState::Failure, weaker] - } else { - vec![weaker, PlaylistUpdateState::Failure] - } - ); - - let mut next = processing_context(temp.path()); - next.run_id = "next-successful-run".into(); - next.execution_order = 2.into(); - let success = acquired_result(PlaylistUpdateState::Success); - persist_input_job_result(&next, &success).await; - let replacement = input_cache::load_input_status(&path); - assert_eq!(replacement.last_input_update.unwrap().state, PlaylistUpdateState::Success); - assert_eq!(replacement.clusters, original.clusters); - } - } - } - - #[tokio::test] - async fn playlist_update_status_parent_download_persists_staged_own_per_cluster_success_and_failure() { - for fails in [false, true] { - let temp = tempfile::tempdir().unwrap(); - let mut responses = fixture_responses([1, 0, 0]); - if fails { - responses.insert("get_live_streams".into(), "not JSON".into()); - } - let server = TestXtreamServer::start_with_responses(responses); - let events = CollectSink::default(); - let ctx = processing_context_with_events(temp.path(), events.clone()); - let parent_file = temp.path().join("parent.m3u"); - std::fs::write(&parent_file, "#EXTM3U\n#EXTINF:-1,Parent\nhttp://stream.example/parent\n").unwrap(); - let parent = Arc::new(ConfigInput { - id: 7, - name: "parent".intern(), - input_type: InputType::M3u, - url: parent_file.to_string_lossy().into_owned(), - enabled: true, - ..ConfigInput::default() - }); - let mut staged = - (*test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 95, vod: 0, series: 0 }, 0, &[])) - .clone(); - staged.id = 8; - staged.name = "own-staged".intern(); - staged.input_type = InputType::Staged; - staged.staged_type = shared::model::StagedInputType::Xtream; - staged.staged = Some(tuliprox_core::model::ConfigInputStaged { - for_input: Some(parent.name.clone()), - clusters: ClusterFlags::Live, - }); - staged.options = Some(ConfigInputOptions::from(&ConfigInputOptionsDto { - skip_vod: true, - skip_series: true, - update_quality: ConfigInputUpdateQualityDto { live: 95, vod: 0, series: 0 }, - ..ConfigInputOptionsDto::default() - })); - staged.resolve_staged_download_type(); // Same resolver used when SourcesConfig is built. - let staged = Arc::new(staged); - ctx.config.sources.store(Arc::new(SourcesConfig { - inputs: vec![parent.clone(), staged.clone()], - ..SourcesConfig::default() - })); - let staged_path = input_storage_path(&ctx, &staged).await; - // Referencing or directly skipping a staged input is not an acquisition. - let skipped = download_input(&ctx, &staged, false).await; - assert!(skipped.input_telemetry.is_none()); - assert!(input_cache::load_input_status(&staged_path).last_input_update.is_none()); - assert!(staged_completion::input_completions(&events, staged.id).is_empty()); - - let result = download_input(&ctx, &parent, false).await; - let status = input_cache::load_input_status(&staged_path); - assert!(!status.clusters.contains_key("default")); - assert_eq!(status.clusters.len(), 1); - let completion = - status.last_input_update.expect("actual indirect staged acquisition has its own completion"); - assert_eq!( - completion.state, - if fails { PlaylistUpdateState::Failure } else { PlaylistUpdateState::Success } - ); - assert!(completion.timestamp > 0); - let completions = staged_completion::input_completions(&events, staged.id); - assert_eq!(completions.len(), 1); - assert_eq!(completions[0].state, Some(completion.state)); - assert_eq!(completions[0].run_id, Some(ctx.run_id.clone())); - assert_eq!(completions[0].execution_order, Some(ctx.execution_order)); - let snapshot = status.clusters["live"].last_update.unwrap(); - assert_eq!(snapshot.source, Some(PlaylistUpdateDataSource::Provider)); - assert_eq!(snapshot.policy, None); - if fails { - assert_eq!(snapshot.quality_guard_threshold, Some(95)); - assert_eq!(snapshot.quality, None); - } else { - assert_eq!(snapshot.quality_guard_threshold, None, "the evaluation already owns its threshold"); - let quality = snapshot.quality.unwrap(); - assert_eq!(quality.threshold, 95); - assert_eq!(quality.decision, PersistedPlaylistUpdateQualityDecision::Accepted); - assert_eq!(quality.achieved_quality, None, "bootstrap has no invented Quality"); - } - assert_eq!(result.errors.is_empty(), !fails); - let parent_path = input_storage_path(&ctx, &parent).await; - assert_ne!(parent_path, staged_path); - assert!( - input_cache::load_input_status(&parent_path).last_input_update.is_none(), - "staged result does not invent a parent job completion" - ); - if !fails { - let bytes = std::fs::read(staged_path.join(input_cache::STATUS_FILE)).unwrap(); - let reuse = download_input(&ctx, &parent, false).await; - assert!(reuse.input_telemetry.is_none()); - let mut staged_reuse = download_input(&ctx, &staged, true).await; - assert!(staged_reuse.input_telemetry.is_none()); - super::super::input_status::complete_staged_input(&ctx, &staged, &mut staged_reuse).await; - assert_eq!(std::fs::read(staged_path.join(input_cache::STATUS_FILE)).unwrap(), bytes); - assert_eq!(staged_completion::input_completions(&events, staged.id), completions); - } - let requests = server.finish(); - assert_eq!( - requests.iter().filter(|action| action.as_str() == "get_live_streams").count(), - 1, - "no extra staged acquisition on parent reuse" - ); - } - } - - #[tokio::test] - async fn playlist_update_status_completion_replaces_one_result_without_touching_cache_or_cluster_snapshots() { - let temp = tempfile::tempdir().unwrap(); - let ctx = processing_context(temp.path()); - let input_name: Arc = "result-input".intern(); - let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &input_name).await; - let mut original = input_cache::InputStatus::default(); - input_cache::update_cluster_status(&mut original, "default", input_cache::ClusterState::Ok); - original.clusters.get_mut("default").unwrap().timestamp = 1; // Expired cache must stay expired. - input_cache::replace_cluster_snapshot( - &mut original, - "default", - PersistedPlaylistUpdateClusterSnapshot { - source: Some(PlaylistUpdateDataSource::Cache), - ..PersistedPlaylistUpdateClusterSnapshot::default() - }, - ); - input_cache::save_input_status(&path, &original); - for state in [PlaylistUpdateState::Success, PlaylistUpdateState::Partial, PlaylistUpdateState::Failure] { - persist_input_completion(&ctx, 7, &input_name, state).await; - let reloaded = input_cache::load_input_status(&path); - let result = reloaded.last_input_update.unwrap(); - assert_eq!(result.state, state); - assert!(result.timestamp > 1); - assert_eq!(reloaded.clusters, original.clusters); - assert!(!input_cache::is_cache_valid(&reloaded, "default", 3600)); - let json: serde_json::Value = - serde_json::from_slice(&std::fs::read(path.join(input_cache::STATUS_FILE)).unwrap()).unwrap(); - assert!(json["last_input_update"].is_object()); - assert_eq!(json.as_object().unwrap().len(), 2, "only clusters + one result, no history"); - } - } - - #[tokio::test] - async fn playlist_update_status_in_run_reuse_preserves_original_partial_result_and_timestamp() { - let temp = tempfile::tempdir().unwrap(); - let ctx = processing_context(temp.path()); - let mut result = super::super::playlist_update_run_result(7, InputJobState::Ready, vec![], true); - result.input_telemetry = Some(PlaylistUpdateInputTelemetry { - refresh_policy: InputRefreshPolicy::NORMAL, - source: Some(PlaylistUpdateDataSource::Provider), - clusters: Vec::new(), - }); - persist_input_job_result(&ctx, &result).await; - let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &result.input_name).await; - let mut original = input_cache::load_input_status(&path); - assert_eq!(original.last_input_update.unwrap().state, PlaylistUpdateState::Partial); - original.last_input_update.as_mut().unwrap().timestamp = 123; - input_cache::save_input_status(&path, &original); - let before = std::fs::read(path.join(input_cache::STATUS_FILE)).unwrap(); - result.had_quality_rejections = false; - result.input_telemetry = None; // Same existing absence of acquisition facts as a real in-run reuse. - persist_input_job_result(&ctx, &result).await; - assert_eq!(std::fs::read(path.join(input_cache::STATUS_FILE)).unwrap(), before); - result.state = InputJobState::Failed; // A genuine technical completion must still replace the prior result. - persist_input_job_result(&ctx, &result).await; - assert_eq!( - input_cache::load_input_status(&path).last_input_update.unwrap().state, - PlaylistUpdateState::Failure - ); - } - - #[tokio::test] - async fn playlist_update_status_completion_covers_every_input_type_without_artificial_clusters() { - let temp = tempfile::tempdir().unwrap(); - let ctx = processing_context(temp.path()); - for (id, input_type) in [ - InputType::Xtream, - InputType::XtreamBatch, - InputType::Stalker, - InputType::StalkerBatch, - InputType::M3u, - InputType::M3uBatch, - InputType::Library, - InputType::Plex, - InputType::Emby, - InputType::Jellyfin, - InputType::Staged, - ] - .into_iter() - .enumerate() - { - let input = ConfigInput { - id: u16::try_from(id).unwrap(), - input_type, - name: format!("input-{id}").intern(), - ..ConfigInput::default() - }; - let result = panicked_input_job(id, &input); - persist_input_job_result(&ctx, &result).await; - let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &input.name).await; - let persisted = input_cache::load_input_status(&path); - assert!(persisted.clusters.is_empty()); - assert!( - matches!(persisted.last_input_update, Some(PersistedPlaylistUpdateInputResult {state: PlaylistUpdateState::Failure, timestamp}) if timestamp > 0) - ); - } - } -} diff --git a/backend/processing/src/processor/playlist/m3u_quality_tests.rs b/backend/processing/src/processor/playlist/m3u_quality_tests.rs deleted file mode 100644 index 05c7ceabd..000000000 --- a/backend/processing/src/processor/playlist/m3u_quality_tests.rs +++ /dev/null @@ -1,520 +0,0 @@ -mod m3u_update_quality { - use super::*; - use std::fmt::Write; - use tuliprox_repository::{load_input_m3u_playlist, raw_group_catalog_path, RawInputGroupCatalog}; - - fn document(prefix: &str, counts: [usize; 3]) -> String { - let mut text = String::from("#EXTM3U\n"); - for (cluster, count) in XTREAM_CLUSTER.into_iter().zip(counts) { - for index in 0..count { - let (kind, group, title) = match cluster { - XtreamCluster::Live => ("live", format!("{prefix}-live"), format!("Channel {index}")), - XtreamCluster::Video => ("movie", format!("{prefix}-vod"), format!("Movie {index}")), - XtreamCluster::Series => { - ("series", format!("{prefix}-show-{index}"), format!("Show {index} S01E01")) - } - }; - let id = match cluster { - XtreamCluster::Live => 1000, - XtreamCluster::Video => 2000, - XtreamCluster::Series => 3000, - } + index; - writeln!(text, "#EXTGRP:{prefix}-{cluster}\n#EXTINF:-1 xui-id=\"{id}\" tvg-type=\"{kind}\" group-title=\"{group}\",{title}\nhttp://stream.example/{prefix}/{cluster}/{id}").unwrap(); - } - } - text - } - - async fn fixture( - root: &Path, - previous: Option<[usize; 3]>, - thresholds: [u8; 3], - ) -> (PlaylistProcessingContext, Arc) { - let ctx = processing_context(root); - let input = Arc::new(ConfigInput { - id: 27, - name: "m3u-quality".intern(), - input_type: InputType::M3u, - enabled: true, - url: root.join("provider.m3u").to_string_lossy().into_owned(), - options: Some(ConfigInputOptions::from(&ConfigInputOptionsDto { - update_quality: ConfigInputUpdateQualityDto { - live: thresholds[0], - vod: thresholds[1], - series: thresholds[2], - }, - ..ConfigInputOptionsDto::default() - })), - ..ConfigInput::default() - }); - if let Some(counts) = previous { - tokio::fs::write(&input.url, document("previous", counts)).await.unwrap(); - let (groups, errors) = - tuliprox_iptv::m3u::download_m3u_playlist(&ctx.config, &ctx.client, &ctx.config.config.load(), &input) - .await; - assert!(errors.is_empty()); - let (_, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, groups).await; - assert!(error.is_none(), "{error:?}"); - } - (ctx, input) - } - - async fn run( - ctx: &PlaylistProcessingContext, - input: &Arc, - counts: [usize; 3], - ) -> InputDownloadResult { - tokio::fs::write(&input.url, document("candidate", counts)).await.unwrap(); - download_input(ctx, input, false).await - } - - fn counts(groups: &[PlaylistGroup]) -> [usize; 3] { - XTREAM_CLUSTER - .map(|cluster| groups.iter().filter(|g| g.xtream_cluster == cluster).map(|g| g.channels.len()).sum()) - } - - async fn verify_effective( - ctx: &PlaylistProcessingContext, - input: &ConfigInput, - result: &mut InputDownloadResult, - expected: [usize; 3], - ) { - assert!(result.errors.is_empty(), "{:?}", result.errors); - assert!(result.storage_error.is_none(), "{:?}", result.storage_error); - assert_eq!(result.job_state(), InputJobState::Ready); - let effective = result.source.take_groups(); - assert_eq!(counts(&effective), expected); - let storage = tuliprox_repository::get_input_storage_path(&input.name, &ctx.config.config.load().storage_dir) - .await - .unwrap(); - let saved = load_input_m3u_playlist( - &ctx.config, - &tuliprox_repository::get_input_m3u_playlist_file_path(&storage, &input.name), - ) - .await - .unwrap(); - assert_eq!(counts(&saved), expected); - for cluster in XTREAM_CLUSTER { - let catalog: RawInputGroupCatalog = - serde_json::from_slice(&tokio::fs::read(raw_group_catalog_path(&storage, cluster)).await.unwrap()) - .unwrap(); - let mut expected_groups: Vec<_> = - effective.iter().filter(|g| g.xtream_cluster == cluster).map(|g| g.title.to_string()).collect(); - expected_groups.sort(); - expected_groups.dedup(); - assert_eq!(catalog.groups, expected_groups); - } - } - - #[tokio::test] - async fn m3u_update_quality_all_clusters_accepted() { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; - let mut result = run(&ctx, &input, [9, 11, 10]).await; - assert!(result.quality_rejections.is_empty()); - assert_eq!(result.update_state(), PlaylistUpdateState::Success); - verify_effective(&ctx, &input, &mut result, [9, 11, 10]).await; - } - - async fn order_candidate( - ctx: &PlaylistProcessingContext, - input: &ConfigInput, - prefix: &str, - ) -> tuliprox_iptv::provider::PlaylistFetch { - let mut body = String::from("#EXTM3U\n"); - for (id, kind, group) in [ - (2000, "movie", "obsolete"), - (2000, "movie", "movies-a"), - (1000, "live", "live-a"), - (2001, "movie", "movies-b"), - (1001, "live", "live-b"), - (3000, "series", "shows-a"), - (3001, "series", "shows-b"), - ] { - writeln!(body, "#EXTGRP:{prefix}-{group}\n#EXTINF:-1 tvg-type=\"{kind}\" group-title=\"{prefix}-{group}\",Item {id} S01E01\nhttp://stream.example/{kind}/{id}").unwrap(); - } - tokio::fs::write(&input.url, body).await.unwrap(); - let (groups, errors) = - tuliprox_iptv::m3u::download_m3u_playlist(&ctx.config, &ctx.client, &ctx.config.config.load(), input).await; - assert!(errors.is_empty()); - assert_eq!( - group_names(&groups), - ["obsolete", "movies-a", "live-a", "movies-b", "live-b", "shows-a", "shows-b"] - .map(|name| format!("{prefix}-{name}")) - ); - tuliprox_iptv::provider::PlaylistFetch::groups(groups) - } - - fn group_names(groups: &[PlaylistGroup]) -> Vec { - groups.iter().map(|group| group.title.to_string()).collect() - } - - #[tokio::test] - async fn m3u_update_quality_preserves_candidate_group_order_when_disabled() { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), None, [0, 0, 0]).await; - let candidate = order_candidate(&ctx, &input, "candidate").await; - let effective = super::super::super::m3u_quality::effective_m3u_fetch( - &ctx.config, - &input, - shared::model::UpdateQualityPolicy::Enforce, - candidate, - ) - .await; - assert!(effective.errors.is_empty()); - assert!(effective.quality_acceptances.is_empty() && effective.quality_rejections.is_empty()); - assert_eq!( - group_names(&effective.groups), - ["movies-a", "live-a", "movies-b", "live-b", "shows-a", "shows-b"] - .map(|name| format!("candidate-{name}")) - ); - assert_eq!(counts(&effective.groups), [2, 2, 2], "disabled quality processing must collapse duplicate keys"); - } - - #[tokio::test] - async fn m3u_update_quality_preserves_candidate_group_order_when_all_clusters_accepted() { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), Some([2, 2, 2]), [100, 100, 100]).await; - let candidate = order_candidate(&ctx, &input, "candidate").await; - let effective = super::super::super::m3u_quality::effective_m3u_fetch( - &ctx.config, - &input, - shared::model::UpdateQualityPolicy::Enforce, - candidate, - ) - .await; - assert!(effective.errors.is_empty() && effective.quality_rejections.is_empty()); - assert_eq!(effective.quality_acceptances.len(), 3); - assert_eq!( - group_names(&effective.groups), - ["movies-a", "live-a", "movies-b", "live-b", "shows-a", "shows-b"].map(|name| format!("candidate-{name}")) - ); - } - - #[tokio::test] - async fn m3u_update_quality_mixed_fallback_preserves_relative_group_order() { - for _ in 0..3 { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), None, [100, 100, 100]).await; - let mut previous = order_candidate(&ctx, &input, "previous").await.groups; - previous.retain(|group| group.title.as_ref() != "previous-obsolete"); - let (_, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, previous).await; - assert!(error.is_none()); - let mut previous = load_input_playlist(&ctx.config, &input, None).await.unwrap(); - let previous = previous.take_groups(); - assert_eq!( - previous - .iter() - .filter(|group| group.xtream_cluster == XtreamCluster::Series) - .map(|group| group.title.to_string()) - .collect::>(), - ["previous-shows-a", "previous-shows-b"] - ); - let mut candidate = order_candidate(&ctx, &input, "candidate").await; - candidate.groups.retain(|group| group.title.as_ref() != "candidate-shows-b"); - let effective = super::super::super::m3u_quality::effective_m3u_fetch( - &ctx.config, - &input, - shared::model::UpdateQualityPolicy::Enforce, - candidate, - ) - .await; - assert!(effective.errors.is_empty()); - assert_eq!(effective.quality_rejections.len(), 1); - assert_eq!(effective.quality_rejections[0].cluster, XtreamCluster::Series); - assert_eq!( - group_names(&effective.groups), - [ - "candidate-movies-a", - "candidate-live-a", - "candidate-movies-b", - "candidate-live-b", - "previous-shows-a", - "previous-shows-b" - ] - ); - } - } - - #[tokio::test] - async fn m3u_update_quality_retains_only_rejected_cluster() { - for disk in [false, true] { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; - let mut config = (**ctx.config.config.load()).clone(); - config.disk_based_processing = disk; - ctx.config.config.store(Arc::new(config)); - let mut result = run(&ctx, &input, [9, 11, 8]).await; - assert_eq!(result.update_state(), PlaylistUpdateState::Partial); - assert_eq!(result.quality_rejections.len(), 1); - assert_eq!(result.quality_rejections[0].cluster, XtreamCluster::Series); - verify_effective(&ctx, &input, &mut result, [9, 11, 10]).await; - } - } - - #[tokio::test] - async fn m3u_update_quality_zero_candidate_retains_previous_cluster() { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; - let mut result = run(&ctx, &input, [10, 10, 0]).await; - assert_eq!(result.quality_rejections[0].candidate_count, 0); - verify_effective(&ctx, &input, &mut result, [10, 10, 10]).await; - } - - #[tokio::test] - async fn m3u_update_quality_disabled() { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [0, 0, 0]).await; - let mut result = run(&ctx, &input, [1, 1, 0]).await; - assert!(result.quality_rejections.is_empty()); - assert!(result - .input_telemetry - .as_ref() - .unwrap() - .clusters - .iter() - .all(|c| c.quality.is_none() && c.threshold.is_none())); - verify_effective(&ctx, &input, &mut result, [1, 1, 0]).await; - } - - #[tokio::test] - async fn m3u_update_quality_first_import() { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), None, [90, 90, 90]).await; - let mut result = run(&ctx, &input, [1, 1, 1]).await; - assert!(result.quality_rejections.is_empty()); - assert!(result.input_telemetry.as_ref().unwrap().clusters.iter().all(|c| c.quality.is_none() - && c.baseline_count.is_none() - && c.decision == Some(PlaylistUpdateClusterDecision::Accepted))); - verify_effective(&ctx, &input, &mut result, [1, 1, 1]).await; - } - - #[tokio::test] - async fn m3u_update_quality_download_or_parse_failure() { - for body in [None, Some("provider unavailable"), Some("#EXTM3U\n#EXTINF:-1,Truncated\n")] { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; - if let Some(body) = body { - tokio::fs::write(&input.url, body).await.unwrap(); - } else { - tokio::fs::remove_file(&input.url).await.unwrap(); - } - let mut result = download_input(&ctx, &input, false).await; - assert!(!result.errors.is_empty()); - assert!(result.quality_rejections.is_empty()); - assert_eq!(result.update_state(), PlaylistUpdateState::Failure); - let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); - assert_eq!(counts(&saved.take_groups()), [10, 10, 10]); - } - } - - #[tokio::test] - async fn m3u_update_quality_empty_bootstrap_and_structural_empty_use_existing_domain() { - for previous in [None, Some([10, 10, 0])] { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), previous, [90, 90, 90]).await; - let mut result = run(&ctx, &input, if previous.is_some() { [10, 10, 0] } else { [0, 0, 0] }).await; - if previous.is_none() { - assert_eq!(result.update_state(), PlaylistUpdateState::Failure); - assert_eq!(result.quality_rejections.len(), 3); - } else { - assert_eq!(result.update_state(), PlaylistUpdateState::Partial); - assert_eq!(result.quality_rejections.len(), 1); - assert_eq!(result.quality_rejections[0].current_count, 0); - assert_eq!(result.quality_rejections[0].cluster, XtreamCluster::Series); - } - } - } - - #[tokio::test] - async fn m3u_update_quality_skip_preserves_previous_and_reuse_emits_no_new_facts() { - let temp = tempfile::tempdir().unwrap(); - let (ctx, mut input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; - Arc::make_mut(&mut input).options.as_mut().unwrap().flags.set(ConfigInputFlags::SkipSeries); - let mut result = run(&ctx, &input, [10, 10, 0]).await; - assert_eq!(result.update_state(), PlaylistUpdateState::Success); - assert!(result.quality_rejections.is_empty()); - let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); - assert_eq!(counts(&result.source.take_groups()), [10, 10, 0]); - assert_eq!(counts(&saved.take_groups()), [10, 10, 10]); - let telemetry = result.input_telemetry.unwrap(); - assert!(!telemetry.clusters.iter().find(|c| c.cluster == XtreamCluster::Series).unwrap().requested); - tokio::fs::remove_file(&input.url).await.unwrap(); - let reused = download_input(&ctx, &input, false).await; - assert!(reused.errors.is_empty()); - assert!(reused.input_telemetry.is_none()); - } - - #[tokio::test] - async fn m3u_update_quality_force_bypasses_guard_and_cache_without_inventing_quality() { - for threshold in [0, 95] { - for policy in [InputRefreshPolicy::REFRESH, InputRefreshPolicy::FORCE] { - let temp = tempfile::tempdir().unwrap(); - let (mut ctx, mut input) = fixture(temp.path(), Some([10, 10, 10]), [threshold; 3]).await; - Arc::make_mut(&mut input).cache_duration_seconds = 3_600; - let storage = input_storage_path(&ctx, &input).await; - let mut status = input_cache::InputStatus::default(); - input_cache::update_cluster_status(&mut status, "default", input_cache::ClusterState::Ok); - input_cache::save_input_status(&storage, &status); - set_refresh_policy(&mut ctx, &input, policy); - let mut result = run(&ctx, &input, [1, 1, 1]).await; - let rejected = threshold > 0 && policy == InputRefreshPolicy::REFRESH; - assert_eq!( - result.update_state(), - if rejected { PlaylistUpdateState::Partial } else { PlaylistUpdateState::Success } - ); - assert_eq!(counts(&result.source.take_groups()), if rejected { [10; 3] } else { [1; 3] }); - let telemetry = result.input_telemetry.as_ref().unwrap(); - assert_eq!(telemetry.refresh_policy, policy); - assert!(telemetry - .clusters - .iter() - .all(|cluster| cluster.source == Some(PlaylistUpdateDataSource::Provider))); - let saved = input_cache::load_input_status(&storage); - for cluster in &telemetry.clusters { - if policy == InputRefreshPolicy::FORCE { - assert_eq!(cluster.decision, Some(PlaylistUpdateClusterDecision::Accepted)); - assert_eq!((cluster.baseline_count, cluster.quality), (None, None)); - assert_eq!((cluster.candidate_count, cluster.active_count), (Some(1), Some(1))); - let snapshot = saved.clusters[cluster.cluster.as_ref()].last_update.unwrap(); - assert_eq!(snapshot.quality, None); - assert_eq!(snapshot.quality_guard_threshold, Some(threshold)); - assert_eq!(snapshot.policy, Some(policy)); - } - } - let reused = download_input(&ctx, &input, false).await; - assert!(reused.input_telemetry.is_none(), "in-run reuse must not create another Force acquisition"); - } - } - } - - #[tokio::test] - async fn m3u_update_quality_force_keeps_parse_and_empty_publication_errors() { - for body in ["#EXTM3U\n", "#EXTM3U\n#EXTINF:-1,Truncated\n"] { - let temp = tempfile::tempdir().unwrap(); - let (mut ctx, input) = fixture(temp.path(), Some([10; 3]), [95; 3]).await; - set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); - tokio::fs::write(&input.url, body).await.unwrap(); - let mut result = download_input(&ctx, &input, false).await; - assert_eq!(result.update_state(), PlaylistUpdateState::Failure); - assert!(result.quality_rejections.is_empty()); - let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); - assert_eq!(counts(&saved.take_groups()), [10; 3]); - } - } - - #[tokio::test] - async fn m3u_update_quality_target_processing_uses_retained_and_accepted_groups() { - for policy in [InputRefreshPolicy::NORMAL, InputRefreshPolicy::FORCE] { - let temp = tempfile::tempdir().unwrap(); - let (mut ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; - set_refresh_policy(&mut ctx, &input, policy); - tokio::fs::write(&input.url, document("candidate", [10, 0, 10])).await.unwrap(); - let output = temp.path().join("client.m3u"); - let mut target = ConfigTarget::from(&ConfigTargetDto { - id: 1, - name: "m3u-target".into(), - enabled: true, - ..ConfigTargetDto::default() - }); - target.filter = get_filter(r#"Name ~ ".*""#, None).unwrap().into(); - target.output = vec![TargetOutput::M3u(tuliprox_core::model::M3uTargetOutput { - filename: Some(output.to_string_lossy().into_owned()), - include_type_in_url: false, - mask_redirect_url: false, - filter: None, - })]; - ctx.config.sources.store(Arc::new(SourcesConfig { - inputs: vec![input.clone()], - sources: vec![ConfigSource { inputs: vec![input.name.clone()], targets: vec![Arc::new(target)] }], - ..SourcesConfig::default() - })); - let (_, targets, errors) = process_source(0, Arc::new(ctx)).await; - assert!(errors.is_empty(), "{errors:?}"); - assert_eq!(targets.len(), 1); - let published = tokio::fs::read_to_string(output).await.unwrap(); - assert!(published.contains("/candidate/live/") && published.contains("/candidate/series/"), "{published}"); - assert_eq!(published.contains("/previous/video/"), policy == InputRefreshPolicy::NORMAL, "{published}"); - assert!(!published.contains("/previous/live/") && !published.contains("/candidate/video/")); - } - } - - #[tokio::test] - async fn m3u_update_quality_raw_group_catalog_uses_effective_state() { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; - let mut result = run(&ctx, &input, [10, 10, 0]).await; - let storage = input_cache::resolve_input_storage_path(&ctx.config.config.load().storage_dir, &input.name).await; - let status = input_cache::load_input_status(&storage); - for cluster in XTREAM_CLUSTER { - let saved = &status.clusters[cluster.as_ref()]; - let rejected = cluster == XtreamCluster::Series; - assert_eq!( - saved.status, - if rejected { input_cache::ClusterState::Failed } else { input_cache::ClusterState::Ok } - ); - let snapshot = saved.last_update.unwrap(); - assert_eq!( - snapshot.quality.unwrap().decision, - if rejected { - PersistedPlaylistUpdateQualityDecision::Rejected - } else { - PersistedPlaylistUpdateQualityDecision::Accepted - } - ); - assert_eq!(snapshot.active_count, Some(10)); - } - assert_eq!(status.clusters["default"].status, input_cache::ClusterState::Failed); - let series: RawInputGroupCatalog = serde_json::from_slice( - &tokio::fs::read(raw_group_catalog_path(&storage, XtreamCluster::Series)).await.unwrap(), - ) - .unwrap(); - assert_eq!(series.groups, ["previous-series"]); - verify_effective(&ctx, &input, &mut result, [10, 10, 10]).await; - } - - #[tokio::test] - async fn m3u_update_quality_duplicate_keys_match_the_existing_persisted_population() { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), Some([1, 1, 0]), [100, 100, 0]).await; - let candidate = document("superseded", [1, 1, 0]) + &document("last", [1, 1, 0]); - tokio::fs::write(&input.url, candidate).await.unwrap(); - let mut result = download_input(&ctx, &input, false).await; - assert_eq!(result.update_state(), PlaylistUpdateState::Success); - assert!(result.source.take_groups().iter().all(|group| group.title.starts_with("last"))); - let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); - assert_eq!(counts(&saved.take_groups()), [1, 1, 0]); - } - - #[tokio::test] - async fn m3u_update_quality_duplicate_keys_match_the_existing_persisted_population_when_disabled() { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), None, [0, 0, 0]).await; - let candidate = document("superseded", [1, 1, 0]) + &document("last", [1, 1, 0]); - tokio::fs::write(&input.url, candidate).await.unwrap(); - let mut result = download_input(&ctx, &input, false).await; - assert_eq!(result.update_state(), PlaylistUpdateState::Success); - assert!(result.source.take_groups().iter().all(|group| group.title.starts_with("last"))); - let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); - assert_eq!(counts(&saved.take_groups()), [1, 1, 0]); - } - - #[tokio::test] - async fn m3u_update_quality_conflicting_retained_key_cannot_diverge_from_storage() { - let temp = tempfile::tempdir().unwrap(); - let (ctx, input) = fixture(temp.path(), Some([1, 1, 0]), [90, 0, 0]).await; - let candidate = document("candidate", [0, 1, 0]).replace("2000", "1000"); - tokio::fs::write(&input.url, candidate).await.unwrap(); - let mut result = download_input(&ctx, &input, false).await; - assert_eq!(result.update_state(), PlaylistUpdateState::Failure); - assert!(!result.errors.is_empty()); - let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); - assert_eq!(counts(&saved.take_groups()), [1, 1, 0]); - let storage = input_storage_path(&ctx, &input).await; - let catalog: RawInputGroupCatalog = serde_json::from_slice( - &tokio::fs::read(raw_group_catalog_path(&storage, XtreamCluster::Live)).await.unwrap(), - ) - .unwrap(); - assert_eq!(catalog.groups, ["previous-live"]); - } -} diff --git a/backend/processing/src/processor/playlist/manual_update_integration_tests.rs b/backend/processing/src/processor/playlist/manual_update_integration_tests.rs deleted file mode 100644 index 87cc1672e..000000000 --- a/backend/processing/src/processor/playlist/manual_update_integration_tests.rs +++ /dev/null @@ -1,725 +0,0 @@ -mod manual_update_integration { - use super::*; - use shared::model::{ConfigRenameDto, ItemField, LibraryConfigDto, LibraryContentType, LibraryScanResult}; - use tuliprox_core::model::{LibraryConfig, LibraryScanDirectory, M3uTargetOutput, MetadataUpdateConfig}; - - async fn library_context(root: &Path, order: ProcessingOrder) -> PlaylistProcessingContext { - let ctx = processing_context_with_events(root, CollectSink::default()); - let media = root.join("media"); - tokio::fs::create_dir_all(&media).await.unwrap(); - tokio::fs::write(media.join("Library Movie (2025).mp4"), b"fixture").await.unwrap(); - let companion_path = root.join("companion.m3u"); - tokio::fs::write(&companion_path, "#EXTM3U\n#EXTINF:-1,Companion\nhttp://stream.example/companion\n") - .await - .unwrap(); - let mut config = (**ctx.config.config.load()).clone(); - let mut library = LibraryConfig::from(&LibraryConfigDto { enabled: true, ..LibraryConfigDto::default() }); - library.metadata.fallback_to_filename = true; - library.thumbnails.enabled = false; - library.scan_directories = vec![LibraryScanDirectory { - enabled: true, - path: media.to_string_lossy().into_owned(), - content_type: LibraryContentType::Movie, - recursive: true, - }]; - config.library = Some(library); - let mut metadata = MetadataUpdateConfig::default(); - metadata.ffprobe.enabled = false; - metadata.tmdb.enabled = false; - config.metadata_update = Some(metadata); - ctx.config.config.store(Arc::new(config)); - let library_input = Arc::new(ConfigInput { - id: 2, - name: "local".intern(), - input_type: InputType::Library, - enabled: true, - cache_duration_seconds: 3600, - ..ConfigInput::default() - }); - let companion = Arc::new(ConfigInput { - id: 3, - name: "companion".intern(), - input_type: InputType::M3u, - enabled: true, - url: companion_path.to_string_lossy().into_owned(), - ..ConfigInput::default() - }); - let mut target = ConfigTarget::from(&ConfigTargetDto { - id: 20, - enabled: true, - name: "selected".into(), - processing_order: order, - rename: Some(vec![ConfigRenameDto { - field: ItemField::Name, - pattern: "^MAPPED$".into(), - new_name: "RENAMED".into(), - t_pattern: None, - }]), - ..ConfigTargetDto::default() - }); - target.output = vec![TargetOutput::M3u(M3uTargetOutput { - filename: Some(root.join("selected.m3u").to_string_lossy().into_owned()), - include_type_in_url: false, - mask_redirect_url: false, - filter: None, - })]; - target.filter = get_filter(r#"Name ~ ".*""#, None).unwrap().into(); - let mapped = super::super::mapping_stage::build_target( - vec![super::super::mapping_stage::build_mapping("manual", MappingStage::Processing, r#"@Name = "MAPPED""#)], - false, - ); - target.mapping = mapped.mapping; - let mut unselected = target.clone(); - unselected.id = 21; - unselected.name = "unselected".into(); - unselected.output = vec![TargetOutput::M3u(M3uTargetOutput { - filename: Some(root.join("unselected.m3u").to_string_lossy().into_owned()), - include_type_in_url: false, - mask_redirect_url: false, - filter: None, - })]; - ctx.config.sources.store(Arc::new(SourcesConfig { - inputs: vec![library_input.clone(), companion.clone()], - sources: vec![ConfigSource { - inputs: vec![library_input.name.clone(), companion.name.clone()], - targets: vec![Arc::new(target), Arc::new(unselected)], - }], - ..SourcesConfig::default() - })); - ctx - } - - fn rescan_run(ctx: &PlaylistProcessingContext, guard: UpdateGuard) -> ProcessingRun { - ProcessingRun::for_run( - "manual-library-run".into(), - ctx.client.clone(), - ctx.config.clone(), - Arc::new(ProcessTargets { - enabled: true, - inputs: vec![2, 3], - targets: vec![20], - target_names: vec!["selected".into()], - }), - ctx.events.clone(), - ) - .with_update_guard(guard) - .with_manual_input_update(Some(InputUpdateRequest { input_id: 2, action: InputUpdateAction::Rescan })) - } - - fn scan_results(ctx: &PlaylistProcessingContext) -> Vec { - ctx.events - .0 - .lock() - .unwrap() - .iter() - .filter_map(|event| match event { - EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(2) => { - if progress.library_scan_result.is_some() { - assert_eq!( - progress.run_id.as_ref().map(PlaylistUpdateRunId::as_ref), - Some("manual-library-run") - ); - assert!(progress.execution_order.is_some()); - } - progress.library_scan_result.clone() - } - _ => None, - }) - .collect() - } - - fn assert_run_state(ctx: &PlaylistProcessingContext, state: PlaylistUpdateState) { - let events = ctx.events.0.lock().unwrap(); - assert!( - events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) if summary.state == state)), - "{events:?}" - ); - assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(2) && progress.state == Some(state))), "{events:?}"); - } - - #[tokio::test] - async fn manual_update_library_empty_rescan_persists_zero_to_zero_and_nonzero_to_zero_with_companion() { - for disk in [false, true] { - for previous_population in [false, true] { - for companion in [false, true] { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; - let mut config = (**ctx.config.config.load()).clone(); - config.disk_based_processing = disk; - ctx.config.config.store(Arc::new(config)); - if !companion { - let mut sources = (**ctx.config.sources.load()).clone(); - sources.sources[0].inputs.retain(|name| name.as_ref() == "local"); - ctx.config.sources.store(Arc::new(sources)); - } - if previous_population { - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - assert_run_state(&ctx, PlaylistUpdateState::Success); - ctx.events.0.lock().unwrap().clear(); - } - tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap(); - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - assert_run_state(&ctx, PlaylistUpdateState::Success); - for name in std::iter::once("local").chain(companion.then_some("companion")) { - let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), name).await; - let status = input_cache::load_input_status(&path); - assert_eq!(status.last_input_update.unwrap().state, PlaylistUpdateState::Success, "{name}"); - } - assert_eq!( - scan_results(&ctx), - vec![LibraryScanResult { - files_scanned: 0, - groups_scanned: 0, - files_added: 0, - files_updated: 0, - files_removed: usize::from(previous_population), - errors: 0, - }] - ); - let output = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap(); - assert_eq!(output.matches("#EXTINF").count(), usize::from(companion), "{output}"); - assert_eq!(output.contains("http://stream.example/companion"), companion); - assert!(!output.contains("Library%20Movie") && !output.contains("file://")); - assert!(!temp.path().join("unselected.m3u").exists()); - let input = ctx.config.sources.load().inputs[0].clone(); - let mut persisted = load_input_playlist(&ctx.config, &input, None).await.unwrap(); - assert!(persisted.is_empty(), "Empty Library must replace its old persisted input"); - assert!(!ctx.events.0.lock().unwrap().iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.message.contains("playlist is empty")))); - // Same canonical empty input remains readable on subsequent cache/reuse paths. - let first = download_input(&ctx, &input, false).await; - assert!(first.errors.is_empty() && first.storage_error.is_none()); - let mut reuse = download_input(&ctx, &input, false).await; - assert_eq!(reuse.job_state(), InputJobState::Ready); - assert_eq!(reuse.input_telemetry, None, "In-run reuse must not emit another acquisition"); - assert_eq!(scan_results(&ctx).len(), 1, "Reuse must not manufacture a second rescan"); - } - } - } - } - - #[tokio::test] - async fn coderabbit_library_unavailable_is_not_an_authoritative_empty_catalog() { - for disabled in [false, true] { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - assert_run_state(&ctx, PlaylistUpdateState::Success); - let mut input = (*ctx.config.sources.load().inputs[0]).clone(); - input.cache_duration_seconds = 0; - let input = Arc::new(input); - let storage = input_storage_path(&ctx, &input).await; - let file = tuliprox_repository::get_input_local_library_playlist_file_path(&storage, &input.name); - let before = tokio::fs::read(&file).await.unwrap(); - let output = tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap(); - let mut config = (**ctx.config.config.load()).clone(); - if disabled { - config.library.as_mut().unwrap().enabled = false; - } else { - config.library = None; - } - ctx.config.config.store(Arc::new(config)); - - let mut result = download_input(&ctx, &input, false).await; - assert_eq!(result.job_state(), InputJobState::Failed); - assert_eq!(result.update_state(), PlaylistUpdateState::Failure); - assert!(!result.authoritative_library); - assert!(!result.errors.is_empty()); - assert_eq!(tokio::fs::read(&file).await.unwrap(), before); - assert_eq!(tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap(), output); - } - } - - #[tokio::test] - async fn coderabbit_library_missing_input_triggers_catalog_recovery_not_empty_success() { - for disk in [false, true] { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; - let mut config = (**ctx.config.config.load()).clone(); - config.disk_based_processing = disk; - ctx.config.config.store(Arc::new(config)); - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - assert_run_state(&ctx, PlaylistUpdateState::Success); - let input = ctx.config.sources.load().inputs[0].clone(); - let storage = input_storage_path(&ctx, &input).await; - let file = tuliprox_repository::get_input_local_library_playlist_file_path(&storage, &input.name); - tokio::fs::remove_file(&file).await.unwrap(); - - assert!(load_input_playlist(&ctx.config, &input, None).await.is_err()); - let mut recovered = download_input(&ctx, &input, false).await; - assert_eq!(recovered.job_state(), InputJobState::Ready); - assert!(recovered.errors.is_empty() && recovered.storage_error.is_none()); - assert!(!recovered.source.is_empty(), "missing cache must be repaired from the populated catalog"); - assert!(file.exists()); - let mut persisted = load_input_playlist(&ctx.config, &input, None).await.unwrap(); - assert!(!persisted.is_empty()); - - let media_server = ConfigInput { input_type: InputType::Plex, ..(*input).clone() }; - assert!( - load_input_playlist(&ctx.config, &media_server, None).await.is_err(), - "the shared loader must not turn a missing Plex cache into a successful empty input" - ); - } - } - - #[tokio::test] - async fn manual_update_library_scan_errors_keep_real_result_and_block_rebuild() { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; - let mut config = (**ctx.config.config.load()).clone(); - config.library.as_mut().unwrap().metadata.fallback_to_filename = false; - ctx.config.config.store(Arc::new(config)); - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - assert_run_state(&ctx, PlaylistUpdateState::Failure); - let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "local").await; - assert_eq!( - input_cache::load_input_status(&path).last_input_update.unwrap().state, - PlaylistUpdateState::Failure - ); - assert_eq!( - scan_results(&ctx), - vec![LibraryScanResult { - files_scanned: 1, - groups_scanned: 1, - files_added: 0, - files_updated: 0, - files_removed: 0, - errors: 1, - }] - ); - assert!(!temp.path().join("selected.m3u").exists()); - } - - #[tokio::test] - async fn manual_update_library_persisted_input_status_keeps_failed_companion_separate() { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "companion").await; - assert_eq!( - input_cache::load_input_status(&path).last_input_update.unwrap().state, - PlaylistUpdateState::Success - ); - let before = std::fs::read(temp.path().join("selected.m3u")).unwrap(); - tokio::fs::write(temp.path().join("companion.m3u"), "#EXTM3U\n").await.unwrap(); - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - assert_eq!( - input_cache::load_input_status(&path).last_input_update.unwrap().state, - PlaylistUpdateState::Failure - ); - let library_path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "local").await; - assert_eq!( - input_cache::load_input_status(&library_path).last_input_update.unwrap().state, - PlaylistUpdateState::Success - ); - assert_eq!(std::fs::read(temp.path().join("selected.m3u")).unwrap(), before); - } - - #[tokio::test] - async fn manual_update_library_empty_xtream_and_strm_use_existing_publish_and_keep_companions() { - use tuliprox_repository::{xtream_get_file_path, xtream_get_storage_path, BPlusTreeQuery}; - for companion in [false, true] { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; - tokio::fs::write( - temp.path().join("companion.m3u"), - "#EXTM3U\n#EXTINF:-1 type=\"vod\",Companion\nhttp://stream.example/companion.mp4\n", - ) - .await - .unwrap(); - let mut sources = (**ctx.config.sources.load()).clone(); - if !companion { - sources.sources[0].inputs.retain(|name| name.as_ref() == "local"); - } - let target = Arc::make_mut(&mut sources.sources[0].targets[0]); - target.output.extend([ - TargetOutput::Xtream(XtreamTargetOutput { - flags: XtreamTargetFlagsSet::new(), - trakt: None, - filter: None, - }), - TargetOutput::Strm(tuliprox_core::model::StrmTargetOutput::from(&shared::model::StrmTargetOutputDto { - directory: temp.path().join("strm").to_string_lossy().into_owned(), - flat: true, - ..shared::model::StrmTargetOutputDto::default() - })), - ]); - ctx.config.sources.store(Arc::new(sources)); - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - assert_run_state(&ctx, PlaylistUpdateState::Success); - ctx.events.0.lock().unwrap().clear(); - tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap(); - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - assert_run_state(&ctx, PlaylistUpdateState::Success); - let config = ctx.config.config.load(); - let xtream = xtream_get_storage_path(&config, "selected").unwrap(); - for (cluster, expected) in - [(XtreamCluster::Live, 0), (XtreamCluster::Video, usize::from(companion)), (XtreamCluster::Series, 0)] - { - let path = xtream_get_file_path(&xtream, cluster); - if !path.exists() { - assert_eq!(expected, 0); - continue; - } - let mut query = BPlusTreeQuery::::try_new(&path).unwrap(); - assert_eq!(query.iter().count(), expected, "{cluster:?}"); - } - let entries = strm_artifacts(&temp.path().join("strm")).await; - assert_eq!(entries.len(), usize::from(companion), "{entries:?}"); - if companion { - assert!(entries[0].1.contains("http://stream.example/companion")); - } - } - } - - #[tokio::test] - async fn manual_update_library_empty_permission_does_not_loosen_other_provider_inputs() { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; - for input_type in [InputType::M3u, InputType::Xtream, InputType::Stalker, InputType::Plex] { - let input = - ConfigInput { input_type, name: format!("empty-{input_type:?}").intern(), ..ConfigInput::default() }; - let (_, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, Vec::new()).await; - assert!(error.is_some(), "{input_type:?} must retain its generic empty-input guard"); - } - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - let before = tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap(); - ctx.events.0.lock().unwrap().clear(); - tokio::fs::write(temp.path().join("companion.m3u"), "#EXTM3U\n").await.unwrap(); - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - { - let events = ctx.events.0.lock().unwrap(); - assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) if summary.state == PlaylistUpdateState::Failure))); - assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(3) && progress.state == Some(PlaylistUpdateState::Failure)))); - } - assert_eq!(before, tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap()); - // An authoritatively empty Library must not mask a failed companion either. - ctx.events.0.lock().unwrap().clear(); - tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap(); - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - { - let events = ctx.events.0.lock().unwrap(); - assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) if summary.state == PlaylistUpdateState::Failure))); - assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(3) && progress.state == Some(PlaylistUpdateState::Failure)))); - } - assert_eq!(before, tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap()); - } - - #[derive(Clone, Copy, Debug)] - enum LibraryFilterBoundary { - Processing, - Persist, - M3u, - Strm, - } - - async fn strm_artifacts(root: &Path) -> Vec<(PathBuf, String)> { - let mut directories = vec![root.to_path_buf()]; - let mut artifacts = Vec::new(); - while let Some(directory) = directories.pop() { - let mut entries = tokio::fs::read_dir(directory).await.unwrap(); - while let Some(entry) = entries.next_entry().await.unwrap() { - if entry.file_type().await.unwrap().is_dir() { - directories.push(entry.path()); - } else if entry.path().extension().is_some_and(|extension| extension == "strm") { - artifacts.push((entry.path(), tokio::fs::read_to_string(entry.path()).await.unwrap())); - } - } - } - artifacts.sort(); - artifacts - } - - async fn assert_library_filtered_empty_publication(boundary: LibraryFilterBoundary) { - for disk in [false, true] { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; - let mut config = (**ctx.config.config.load()).clone(); - config.disk_based_processing = disk; - ctx.config.config.store(Arc::new(config)); - tokio::fs::write( - temp.path().join("companion.m3u"), - "#EXTM3U\n#EXTINF:-1 type=\"vod\",Companion\nhttp://stream.example/companion.mp4\n", - ) - .await - .unwrap(); - let mut sources = (**ctx.config.sources.load()).clone(); - let target = Arc::make_mut(&mut sources.sources[0].targets[0]); - target.rename = None; - target.mapping = Arc::default(); - let library_only = get_filter(r#"Name ~ "^Library Movie.*""#, None).unwrap(); - if matches!(boundary, LibraryFilterBoundary::Processing) { - target.filter = library_only.clone().into(); - } - if matches!(boundary, LibraryFilterBoundary::Persist) { - target.filter.persist = Some(library_only.clone()); - } - if let TargetOutput::M3u(output) = &mut target.output[0] { - output.filter = matches!(boundary, LibraryFilterBoundary::M3u).then(|| library_only.clone()); - } - let strm_root = temp.path().join("strm"); - let mut strm = tuliprox_core::model::StrmTargetOutput::from(&shared::model::StrmTargetOutputDto { - directory: strm_root.to_string_lossy().into_owned(), - flat: true, - cleanup: true, - ..shared::model::StrmTargetOutputDto::default() - }); - strm.filter = matches!(boundary, LibraryFilterBoundary::Strm).then_some(library_only); - target.output.push(TargetOutput::Strm(strm)); - ctx.config.sources.store(Arc::new(sources)); - - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - assert_run_state(&ctx, PlaylistUpdateState::Success); - let m3u_path = temp.path().join("selected.m3u"); - let before_m3u = tokio::fs::read_to_string(&m3u_path).await.unwrap(); - let before_strm = strm_artifacts(&strm_root).await; - let m3u_keeps_companion = matches!(boundary, LibraryFilterBoundary::Strm); - let strm_keeps_companion = matches!(boundary, LibraryFilterBoundary::M3u); - assert_eq!(before_m3u.matches("#EXTINF").count(), 1 + usize::from(m3u_keeps_companion)); - assert_eq!(before_strm.len(), 1 + usize::from(strm_keeps_companion)); - assert_eq!(before_m3u.contains("http://stream.example/companion"), m3u_keeps_companion); - assert_eq!( - before_strm.iter().any(|(_, body)| body.contains("http://stream.example/companion")), - strm_keeps_companion - ); - - ctx.events.0.lock().unwrap().clear(); - tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap(); - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - let after_m3u = tokio::fs::read_to_string(&m3u_path).await.unwrap(); - let after_strm = strm_artifacts(&strm_root).await; - println!("Library filter {boundary:?}, disk={disk}: M3U before={before_m3u:?}, after={after_m3u:?}; STRM before={before_strm:?}, after={after_strm:?}"); - assert_run_state(&ctx, PlaylistUpdateState::Success); - assert_eq!( - scan_results(&ctx), - vec![LibraryScanResult { - files_scanned: 0, - groups_scanned: 0, - files_added: 0, - files_updated: 0, - files_removed: 1, - errors: 0 - }] - ); - assert_ne!(before_m3u, after_m3u, "old Library entries must really be replaced"); - assert_eq!(after_m3u.matches("#EXTINF").count(), usize::from(m3u_keeps_companion)); - assert_eq!(after_m3u.contains("http://stream.example/companion"), m3u_keeps_companion); - assert_eq!(after_strm.len(), usize::from(strm_keeps_companion)); - for (path, contents) in &before_strm { - if contents.contains("http://stream.example/companion") { - assert!( - after_strm.contains(&(path.clone(), contents.clone())), - "valid companion must be retained byte-for-byte" - ); - } else { - assert!(!path.exists(), "cleanup must remove old Library artifact {path:?}"); - } - } - // The existing M3U database is refreshed too, not only the client text file. - let target_path = - tuliprox_repository::get_target_storage_path(&ctx.config.config.load(), "selected").unwrap(); - let mut db = tuliprox_repository::BPlusTreeQuery::::try_new( - &tuliprox_repository::m3u_get_file_path_for_db(&target_path), - ) - .unwrap(); - assert_eq!(db.iter().count(), usize::from(m3u_keeps_companion)); - assert!(!temp.path().join("unselected.m3u").exists()); - } - } - - #[tokio::test] - async fn manual_update_library_processing_filter_replaces_old_output_after_last_movie_removed() { - assert_library_filtered_empty_publication(LibraryFilterBoundary::Processing).await; - } - - #[tokio::test] - async fn manual_update_library_persist_filter_replaces_old_output_after_last_movie_removed() { - assert_library_filtered_empty_publication(LibraryFilterBoundary::Persist).await; - } - - #[tokio::test] - async fn manual_update_library_m3u_output_filter_replaces_old_output_and_preserves_unfiltered_strm() { - assert_library_filtered_empty_publication(LibraryFilterBoundary::M3u).await; - } - - #[tokio::test] - async fn manual_update_library_strm_output_filter_cleans_old_files_and_preserves_unfiltered_m3u() { - assert_library_filtered_empty_publication(LibraryFilterBoundary::Strm).await; - } - - #[tokio::test] - async fn manual_update_library_empty_does_not_authorize_foreign_forced_empty_target() { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; - let sources = ctx.config.sources.load_full(); - let mut playlists = sources - .inputs - .iter() - .map(|input| FetchedPlaylist { input, source: MemoryPlaylistSource::default().into_source(), epg: None }) - .collect::>(); - let prepared = prepare_playlist_for_target( - &ctx, - &mut playlists, - &sources.sources[0].targets[0], - &mut HashMap::new(), - &mut Vec::new(), - ClusterFlags::Vod, - false, - ) - .await - .unwrap(); - assert_eq!(prepared.library_empty, tuliprox_repository::LibraryEmptyPublication::Contribution); - let (result, _) = finalize_prepared_target(Arc::new(ctx), prepared).await; - assert!(result.is_err(), "Library must not relax the existing non-Xtream force-empty output guard"); - assert!(!temp.path().join("selected.m3u").exists()); - } - - #[tokio::test] - async fn manual_update_library_incomplete_discovery_has_no_invented_scan_result_or_rebuild() { - for failure in ["missing", "not-directory", "partial", "no-directories", "corrupt-catalog"] { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - let before = tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap(); - let status_path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "local").await; - let cache_before = input_cache::load_input_status(&status_path).clusters; - ctx.events.0.lock().unwrap().clear(); - let mut config = (**ctx.config.config.load()).clone(); - let library = config.library.as_mut().unwrap(); - match failure { - "missing" => { - library.scan_directories[0].path = temp.path().join("missing").to_string_lossy().into_owned(); - } - "not-directory" => { - library.scan_directories[0].path = - temp.path().join("media/Library Movie (2025).mp4").to_string_lossy().into_owned(); - } - "partial" => { - let mut missing = library.scan_directories[0].clone(); - missing.path = temp.path().join("missing").to_string_lossy().into_owned(); - library.scan_directories.push(missing); - } - "no-directories" => library.scan_directories.clear(), - "corrupt-catalog" => { - let path = tuliprox_library::library::resolve_metadata_storage_path( - config.metadata_update.as_ref(), - &config.storage_dir, - ) - .join("library/broken.json"); - tokio::fs::write(path, "invalid json").await.unwrap(); - } - _ => unreachable!(), - } - ctx.config.config.store(Arc::new(config)); - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - assert_run_state(&ctx, PlaylistUpdateState::Failure); - assert!(scan_results(&ctx).is_empty(), "{failure}"); - let status = input_cache::load_input_status(&status_path); - assert_eq!(status.last_input_update.unwrap().state, PlaylistUpdateState::Failure, "{failure}"); - assert_eq!(status.clusters, cache_before, "failed rescan must not mutate cache validity"); - assert_eq!(before, tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap()); - } - } - - #[tokio::test] - async fn manual_update_library_selected_target_keeps_companion_and_real_processing_order() { - for (order, expected) in [(ProcessingOrder::Frm, "MAPPED"), (ProcessingOrder::Fmr, "RENAMED")] { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), order).await; - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - let output = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap(); - assert_eq!(output.matches("#EXTINF").count(), 2, "{output}"); - assert!(output.contains("http://stream.example/companion"), "{output}"); - assert!(output.contains(expected), "{output}"); - assert!(!temp.path().join("unselected.m3u").exists()); - let events = ctx.events.0.lock().unwrap(); - let scan = events - .iter() - .position(|event| { - matches!(event, EventMessage::PlaylistUpdateProgress(progress) - if progress.input_id == Some(2) && progress.message.starts_with("Library rescan completed")) - }) - .unwrap(); - let acquired = events - .iter() - .position(|event| { - matches!(event, EventMessage::PlaylistUpdateProgress(progress) - if progress.input_id == Some(2) && progress.state == Some(PlaylistUpdateState::Success)) - }) - .unwrap(); - let published = events - .iter() - .position(|event| { - matches!(event, EventMessage::PlaylistUpdate(summary) - if summary.state == PlaylistUpdateState::Success) - }) - .unwrap(); - assert!(scan < acquired && acquired < published); - assert!(events.iter().all(|event| !matches!(event, EventMessage::PlaylistUpdateProgress(progress) - if progress.input_id == Some(2) && progress.input_telemetry.as_ref().is_some_and(|telemetry| - !telemetry.clusters.is_empty() || telemetry.refresh_policy != InputRefreshPolicy::NORMAL)))); - } - } - - #[tokio::test] - async fn manual_update_library_unreadable_catalog_and_busy_scan_do_not_publish() { - for busy in [false, true] { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; - let guard = UpdateGuard::new(); - let _busy = busy.then(|| guard.try_library().unwrap()); - if !busy { - let mut config = (**ctx.config.config.load()).clone(); - config.library.as_mut().unwrap().scan_directories[0].path = - temp.path().join("missing").to_string_lossy().into_owned(); - ctx.config.config.store(Arc::new(config)); - } - exec_processing(rescan_run(&ctx, guard.clone())).await; - assert!(!temp.path().join("selected.m3u").exists()); - assert!(!temp.path().join("unselected.m3u").exists()); - assert!(ctx - .events - .0 - .lock() - .unwrap() - .iter() - .any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) - if summary.state == PlaylistUpdateState::Failure))); - assert!(guard.try_playlist().is_some()); - } - } - - #[tokio::test] - async fn manual_update_library_input_success_remains_separate_from_target_publish_failure() { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; - tokio::fs::create_dir(temp.path().join("selected.m3u")).await.unwrap(); - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - let events = ctx.events.0.lock().unwrap(); - assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) - if progress.input_id == Some(2) && progress.state == Some(PlaylistUpdateState::Success)))); - assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) - if summary.state == PlaylistUpdateState::Failure))); - } - - #[tokio::test] - async fn manual_update_library_rescan_reloads_valid_cache_but_bulk_keeps_catalog() { - let temp = tempfile::tempdir().unwrap(); - let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; - exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; - tokio::fs::write(temp.path().join("media/Second Movie (2025).mp4"), b"fixture").await.unwrap(); - // Ordinary bulk does not discover the new file and does not acquire a library permit. - let guard = UpdateGuard::new(); - let standalone_scan = guard.try_library().unwrap(); - exec_processing( - ProcessingRun::new(ctx.client.clone(), ctx.config.clone(), ctx.user_targets.clone(), ctx.events.clone()) - .with_update_guard(guard.clone()), - ) - .await; - let bulk = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap(); - assert_eq!(bulk.matches("#EXTINF").count(), 2); - drop(standalone_scan); - exec_processing(rescan_run(&ctx, guard)).await; - let rescanned = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap(); - assert_eq!(rescanned.matches("#EXTINF").count(), 3, "{rescanned}"); - } -} diff --git a/backend/processing/src/processor/playlist/mod.rs b/backend/processing/src/processor/playlist/mod.rs index 31a9239cb..185198724 100644 --- a/backend/processing/src/processor/playlist/mod.rs +++ b/backend/processing/src/processor/playlist/mod.rs @@ -3,7 +3,7 @@ use crate::{ fetched_playlist::FetchedPlaylist, input_cache, metadata_sink::{MetadataUpdateSink, NoopMetadataSink}, - parser::xmltv::{flatten_tvguide, merge_epg_trees, EpgMergeAccumulator, TVGuide}, + parser::xmltv::{flatten_tvguide, merge_epg_trees, EpgMergeAccumulator}, playlist_watch::{process_group_watch, process_target_groups_watch}, processor::{ epg::{clear_invalid_live_epg_ids, process_playlist_epg, retain_epg_referenced_by_groups}, @@ -18,22 +18,18 @@ use indexmap::IndexMap; use log::{debug, error, info, log_enabled, warn, Level}; use path_clean::PathClean; use shared::{ - concat_string, defaults::{default_as_default, default_probe_delay_secs, default_probe_live_interval}, error::{get_errors_notify_message, TuliproxError}, foundation::{get_field_value, set_field_value, Filter, ValueAccessor, ValueProvider}, model::{ ClusterFlags, ConfigTargetOptions, CounterModifier, EventMessage, EventSink, FieldGet, FieldSet, InputRefreshOverride, InputRefreshPolicy, InputStats, InputType, InputUpdateAction, InputUpdateRequest, - MappingStage, PersistedPlaylistUpdateClusterSnapshot, PersistedPlaylistUpdateQualityDecision, - PersistedPlaylistUpdateQualitySnapshot, PersistedPlaylistUpdateTechnicalState, PipelineStats, PlaylistEntry, - PlaylistGroup, PlaylistItem, PlaylistItemType, PlaylistStats, PlaylistUpdateClusterDecision, - PlaylistUpdateClusterTelemetry, PlaylistUpdateDataSource, PlaylistUpdateInputTelemetry, - PlaylistUpdateProgressEvent, PlaylistUpdateRunId, PlaylistUpdateRunOrder, PlaylistUpdateState, - PlaylistUpdateSummary, ProviderFetchFailure, SourceStats, StagedInputType, StreamProperties, TargetStats, - UUIDType, WatchDisabled, WatchDisabledReason, WatchUnmatched, XtreamCluster, + MappingStage, PipelineStats, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemType, + PlaylistUpdateInputTelemetry, PlaylistUpdateProgressEvent, PlaylistUpdateRunId, PlaylistUpdateRunOrder, + PlaylistUpdateState, PlaylistUpdateSummary, StreamProperties, TargetStats, UUIDType, WatchDisabled, + WatchDisabledReason, WatchUnmatched, XtreamCluster, }, - utils::{create_alias_uuid, interner_gc, sanitize_sensitive_info, Internable}, + utils::{create_alias_uuid, interner_gc, Internable}, }; use std::{ collections::{HashMap, HashSet}, @@ -43,35 +39,25 @@ use std::{ time::{Duration, Instant}, }; use tokio::{ - sync::{watch, Mutex, OwnedRwLockWriteGuard, RwLock}, + sync::{watch, Mutex, RwLock}, task::JoinSet, }; use tuliprox_core::{ model::{ is_valid, retain_filtered_playlist, AppConfig, ClusterForceUpdate, ClusterUpdateRejection, CompiledMapping, - ConfigFavourites, ConfigInput, ConfigInputFlags, ConfigInputOptions, ConfigRename, ConfigTarget, - CurationConfig, Epg, FilterOutcome, MappingProgram, ProcessTargets, ProviderIdType, ResolveReason, - ReverseProxyDisabledHeaderConfig, TransformStage, UpdateGuard, UpdateTask, + ConfigFavourites, ConfigInputFlags, ConfigInputOptions, ConfigRename, ConfigTarget, CurationConfig, Epg, + FilterOutcome, MappingProgram, ProcessTargets, ProviderIdType, ResolveReason, ReverseProxyDisabledHeaderConfig, + TransformStage, UpdateGuard, UpdateTask, }, - utils::{debug_if_enabled, log_memory_snapshot, trace_if_enabled, StepMeasure, StepMeasureCallback}, + utils::{debug_if_enabled, log_memory_snapshot, trace_if_enabled, StepMeasure}, }; use tuliprox_curation::{ evaluate_curation, project_curation_categories, CurationEvaluation, CurationFailure, CurationRunOutcome, SelectorOutcome, }; -use tuliprox_iptv::{ - epg::{CountingEpgSink, EpgFetchRequest, EpgProvider}, - error::ProviderErrorKind, - provider::{ - BatchContainerProvider, M3uProvider, PlaylistFetch, PlaylistFetchRequest, PlaylistProvider, - UnsupportedProvider, XtreamProvider, - }, - xtream, -}; use tuliprox_repository::{ - load_input_playlist, persist_input_playlist_with_options, persist_playlist_views, CategoryKey, - InputPlaylistPersistOptions, MemoryPlaylistSource, PlaylistPublicationPlan, PlaylistSource, PlaylistStorageState, - TargetPlaylistPersistOptions, + persist_playlist_views, CategoryKey, MemoryPlaylistSource, PlaylistPublicationPlan, PlaylistSource, + PlaylistStorageState, TargetPlaylistPersistOptions, }; use tuliprox_session::ActiveProviderManager; diff --git a/backend/processing/src/processor/playlist/staged_completion_tests.rs b/backend/processing/src/processor/playlist/staged_completion_tests.rs deleted file mode 100644 index 1ca760e69..000000000 --- a/backend/processing/src/processor/playlist/staged_completion_tests.rs +++ /dev/null @@ -1,166 +0,0 @@ -mod staged_completion { - use super::*; - - pub(super) fn input_completions(events: &CollectSink, input_id: u16) -> Vec { - events - .0 - .lock() - .unwrap() - .iter() - .filter_map(|event| match event { - EventMessage::PlaylistUpdateProgress(progress) - if progress.input_id == Some(input_id) && progress.state.is_some() => - { - Some(progress.clone()) - } - _ => None, - }) - .collect() - } - - async fn staged_run_context( - root: &Path, - base_url: &str, - ) -> (PlaylistProcessingContext, Arc) { - let ctx = processing_context_with_events(root, CollectSink::default()); - let parent_file = root.join("parent.m3u"); - std::fs::write(&parent_file, "#EXTM3U\n#EXTINF:-1,Parent\nhttp://stream.example/parent\n").unwrap(); - let parent = Arc::new(ConfigInput { - id: 7, - name: "parent".intern(), - input_type: InputType::M3u, - url: parent_file.to_string_lossy().into_owned(), - enabled: true, - ..ConfigInput::default() - }); - let mut staged = (*test_input( - base_url, - ConfigInputUpdateQualityDto { live: 95, vod: 0, series: 0 }, - 0, - &[XtreamCluster::Video, XtreamCluster::Series], - )) - .clone(); - staged.id = 8; - staged.name = "staged".intern(); - staged.input_type = InputType::Staged; - staged.staged_type = shared::model::StagedInputType::Xtream; - staged.staged = Some(tuliprox_core::model::ConfigInputStaged { - for_input: Some(parent.name.clone()), - clusters: ClusterFlags::Live, - }); - staged.resolve_staged_download_type(); - let staged = Arc::new(staged); - seed_live_baseline(&ctx, &staged, 100).await; - let failed_companion = Arc::new(ConfigInput { - id: 9, - name: "failed-companion".intern(), - input_type: InputType::M3u, - url: root.join("missing.m3u").to_string_lossy().into_owned(), - enabled: true, - ..ConfigInput::default() - }); - ctx.config.sources.store(Arc::new(SourcesConfig { - inputs: vec![parent.clone(), staged.clone(), failed_companion.clone()], - sources: vec![ConfigSource { - inputs: vec![parent.name.clone(), failed_companion.name.clone()], - targets: vec![xtream_target(true)], - }], - ..SourcesConfig::default() - })); - (ctx, staged) - } - - fn normalized_trace( - events: &[EventMessage], - persisted: shared::model::PersistedPlaylistUpdateInputResult, - ) -> serde_json::Value { - let summary = events - .iter() - .find_map(|event| match event { - EventMessage::PlaylistUpdate(summary) => Some(summary), - _ => None, - }) - .unwrap(); - assert_eq!(summary.run_id, Some("staged-completion-run".into())); - assert!(summary.execution_order.is_some()); - assert_eq!(summary.state, PlaylistUpdateState::Failure); - let progress: Vec<_> = events - .iter() - .filter_map(|event| match event { - EventMessage::PlaylistUpdateProgress(progress) - if progress.input_id == Some(8) || progress.state.is_some() => - { - assert_eq!(progress.run_id, summary.run_id); - assert_eq!(progress.execution_order, summary.execution_order); - let mut progress = progress.clone(); - // Only the globally allocated test-run order is normalized. All - // input identities, states, messages and telemetry remain exact. - progress.execution_order = Some(1.into()); - Some(progress) - } - _ => None, - }) - .collect(); - assert!(persisted.timestamp > 0); - serde_json::json!({ - "progress": progress, - "completed": {"run_id": summary.run_id, "execution_order": 1, "state": summary.state}, - "persisted_input_result": {"state": persisted.state, "timestamp": 1} - }) - } - - #[tokio::test] - async fn playlist_update_status_staged_reporter_trace_matches_persistence_before_foreign_run_failure() { - let mut traces = Vec::new(); - for expected in [PlaylistUpdateState::Partial, PlaylistUpdateState::Success, PlaylistUpdateState::Failure] { - let temp = tempfile::tempdir().unwrap(); - let mut responses = - fixture_responses([if expected == PlaylistUpdateState::Success { 100 } else { 1 }, 0, 0]); - if expected == PlaylistUpdateState::Failure { - responses.insert("get_live_streams".into(), "not JSON".into()); - } - let server = TestXtreamServer::start_with_responses(responses); - let (ctx, staged) = staged_run_context(temp.path(), &server.base_url).await; - exec_processing(ProcessingRun::for_run( - "staged-completion-run".into(), - ctx.client.clone(), - ctx.config.clone(), - ctx.user_targets.clone(), - ctx.events.clone(), - )) - .await; - let path = input_storage_path(&ctx, &staged).await; - let persisted = input_cache::load_input_status(&path); - assert!(!persisted.clusters.contains_key("default")); - assert_eq!(persisted.last_input_update.unwrap().state, expected); - let completions = input_completions(&ctx.events, staged.id); - assert_eq!(completions.len(), 1, "real indirect staged input must emit one correlated completion"); - assert_eq!(completions[0].state, Some(expected)); - let telemetry = completions[0].input_telemetry.as_ref().unwrap(); - assert_eq!(telemetry.refresh_policy, InputRefreshPolicy::NORMAL); - assert_eq!(telemetry.source, None); - assert_eq!(telemetry.clusters[0].source, Some(PlaylistUpdateDataSource::Provider)); - assert_eq!(telemetry.clusters[0].threshold, Some(95)); - if expected == PlaylistUpdateState::Partial { - assert_eq!(telemetry.clusters[0].decision, Some(PlaylistUpdateClusterDecision::Rejected)); - assert_eq!(telemetry.clusters[0].active_count, Some(100)); - } - let emitted = ctx.events.0.lock().unwrap().clone(); - let own_index = emitted.iter().position(|event| matches!(event, EventMessage::PlaylistUpdateProgress(p) if p.input_id == Some(8) && p.state.is_some())).unwrap(); - let parent_index = emitted.iter().position(|event| matches!(event, EventMessage::PlaylistUpdateProgress(p) if p.input_id == Some(7) && p.state.is_some())).unwrap(); - let completed_index = - emitted.iter().position(|event| matches!(event, EventMessage::PlaylistUpdate(_))).unwrap(); - assert!(own_index < parent_index && parent_index < completed_index); - assert_eq!(input_completions(&ctx.events, 9)[0].state, Some(PlaylistUpdateState::Failure)); - traces.push(normalized_trace(&emitted, persisted.last_input_update.unwrap())); - assert_eq!(server.finish().iter().filter(|action| action.as_str() == "get_live_streams").count(), 1); - } - let actual = serde_json::Value::Array(traces); - let fixture: serde_json::Value = serde_json::from_str(include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../frontend/src/model/test_data/staged_completions.json" - ))) - .unwrap(); - assert_eq!(actual, fixture, "actual reporter trace: {}", serde_json::to_string_pretty(&actual).unwrap()); - } -} diff --git a/backend/processing/src/processor/playlist/tests.rs b/backend/processing/src/processor/playlist/tests.rs index a177324aa..6976c0b9a 100644 --- a/backend/processing/src/processor/playlist/tests.rs +++ b/backend/processing/src/processor/playlist/tests.rs @@ -1,5231 +1,29 @@ #![allow(clippy::wildcard_imports)] use super::*; +use crate::parser::xmltv::TVGuide; use shared::{ - foundation::{get_filter, MapperScript, ValueProvider}, + foundation::{get_filter, MapperScript}, model::{ - ClusterFlags, ConfigInputDto, ConfigRenameDto, ConfigTargetDto, ConfigTargetOptions, FieldSetAccessor, - ItemField, M3uPlaylistItem, M3uTargetOutputDto, MappingStage, PlaylistEntry, PlaylistItem, PlaylistItemHeader, - PlaylistItemType, PlaylistUpdateRunOrder, TargetOutputDto, TraktApiConfigDto, TraktCatalogSelection, - TraktConfigDto, TraktContentType, TraktListConfigDto, UUIDType, XtreamCluster, XtreamPlaylistItem, - XtreamTargetOutputDto, + ClusterFlags, ConfigInputDto, ConfigTargetDto, ConfigTargetOptions, M3uTargetOutputDto, MappingStage, + PersistedPlaylistUpdateClusterSnapshot, PersistedPlaylistUpdateQualityDecision, + PersistedPlaylistUpdateTechnicalState, PlaylistItem, PlaylistItemHeader, PlaylistItemType, + PlaylistUpdateClusterDecision, PlaylistUpdateDataSource, PlaylistUpdateRunOrder, TargetOutputDto, + TraktApiConfigDto, TraktCatalogSelection, TraktConfigDto, TraktContentType, TraktListConfigDto, UUIDType, + XtreamCluster, XtreamPlaylistItem, XtreamTargetOutputDto, }, utils::Internable, }; -use tuliprox_core::model::{CompiledMappingRule, CompiledTargetMappings, Config, ConfigInputAlias, TraktConfig}; -use tuliprox_curation::{ - CurationEvaluation, CurationMediaKind, CurationMembership, CurationSelectorKey, CurationSelectorSummary, -}; +use tuliprox_core::model::{CompiledMappingRule, CompiledTargetMappings, Config, ConfigInput, ConfigInputAlias}; +use tuliprox_repository::load_input_playlist; mod curation_publication; -fn apply_staged_overlay_groups( - provider: &ConfigInput, - clusters: ClusterFlags, - provider_groups: Vec, - staged_groups: Vec, -) -> Vec { - super::apply_staged_overlay_groups(provider, StagedInputType::Xtream, clusters, provider_groups, staged_groups) -} +mod mapping_stage; -#[derive(Clone, Default)] -struct PlaylistRunCollectSink(Arc>>); - -impl EventSink for PlaylistRunCollectSink { - fn emit(&self, event: EventMessage) { - self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push(event); - } -} - -fn playlist_update_run_result( - input_id: u16, - state: InputJobState, - errors: Vec, - had_quality_rejections: bool, -) -> InputJobResult { - let input_name = format!("input-{input_id}").intern(); - InputJobResult { - index: usize::from(input_id), - input_id, - input_name: Arc::clone(&input_name), - state, - source: None, - epg: None, - stat: create_input_stat(0, 0, errors.len(), InputType::Xtream, &input_name, 0), - errors, - accepted_empty_clusters: ClusterFlags::empty(), - had_quality_rejections, - input_telemetry: None, - } -} - -#[test] -fn playlist_update_run_input_completion_uses_existing_success_partial_failure_classification() { - let success = playlist_update_run_result(1, InputJobState::Ready, Vec::new(), false); - let rejection = playlist_update_run_result(2, InputJobState::Ready, Vec::new(), true); - let resumable = playlist_update_run_result(3, InputJobState::Pending, Vec::new(), false); - let failure = playlist_update_run_result( - 4, - InputJobState::Ready, - vec![TuliproxError::RepositoryPlaylist("technical".to_string())], - false, - ); - - assert_eq!(success.update_state(), PlaylistUpdateState::Success); - assert_eq!(rejection.update_state(), PlaylistUpdateState::Partial); - assert_eq!(resumable.update_state(), PlaylistUpdateState::Partial); - assert_eq!(failure.update_state(), PlaylistUpdateState::Failure); -} - -#[test] -fn pipeline_transparency_input_completion_reuses_correlated_progress_event_for_typed_telemetry() { - let events = PlaylistRunCollectSink::default(); - let run_id = PlaylistUpdateRunId::from("pipeline-transparency-run"); - let execution_order = PlaylistUpdateRunOrder::from(19); - let mut result = playlist_update_run_result(7, InputJobState::Ready, Vec::new(), true); - let telemetry = PlaylistUpdateInputTelemetry { - refresh_policy: InputRefreshPolicy::NORMAL, - source: None, - clusters: vec![PlaylistUpdateClusterTelemetry { - cluster: XtreamCluster::Video, - requested: true, - source: Some(PlaylistUpdateDataSource::Provider), - baseline_count: Some(12_543), - candidate_count: Some(217), - active_count: Some(12_543), - threshold: Some(90), - quality: Some(1), - decision: Some(PlaylistUpdateClusterDecision::Rejected), - technical_state: None, - }], - }; - result.input_telemetry = Some(telemetry.clone()); - - report_input_job_completion(&events, &run_id, execution_order, &result); - - let emitted = events.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let EventMessage::PlaylistUpdateProgress(progress) = &emitted[0] else { - panic!("expected existing playlist progress event"); - }; - assert_eq!(progress.run_id.as_ref(), Some(&run_id)); - assert_eq!(progress.execution_order, Some(execution_order)); - assert_eq!(progress.input_id, Some(7)); - assert_eq!(progress.state, Some(PlaylistUpdateState::Partial)); - assert_eq!(progress.input_telemetry, Some(telemetry)); -} - -#[test] -fn playlist_update_run_technical_failure_progress_is_correlated_sanitized_and_input_scoped() { - let events = PlaylistRunCollectSink::default(); - let run_id = PlaylistUpdateRunId::from("processing-run"); - let execution_order = PlaylistUpdateRunOrder::from(17); - let failed = playlist_update_run_result( - 7, - InputJobState::Failed, - vec![TuliproxError::RepositoryPlaylist("https://user:secret@provider.example/playlist".to_string())], - false, - ); - let succeeded = playlist_update_run_result(8, InputJobState::Ready, Vec::new(), false); - - report_input_job_completion(&events, &run_id, execution_order, &failed); - report_input_job_completion(&events, &run_id, execution_order, &succeeded); - - let emitted = events.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let progress = emitted - .iter() - .filter_map(|event| match event { - EventMessage::PlaylistUpdateProgress(progress) => Some(progress), - _ => None, - }) - .collect::>(); - assert_eq!(progress.len(), 2); - assert!(progress.iter().all(|event| event.run_id.as_ref() == Some(&run_id))); - assert!(progress.iter().all(|event| event.execution_order == Some(execution_order))); - assert_eq!(progress[0].input_id, Some(7)); - assert_eq!(progress[0].state, Some(PlaylistUpdateState::Failure)); - assert_eq!(progress[0].message, "Input 'input-7' failed during update"); - assert!(!progress[0].message.contains("secret")); - assert!(!progress[0].message.contains("previous accepted data")); - assert_eq!(progress[1].input_id, Some(8)); - assert_eq!(progress[1].state, Some(PlaylistUpdateState::Success)); - assert!(!progress[1].message.contains("failed during update")); -} - -#[test] -fn playlist_update_run_failure_text_makes_no_unverified_previous_data_claim() { - let events = PlaylistRunCollectSink::default(); - let run_id = PlaylistUpdateRunId::from("neutral-failure-run"); - let execution_order = PlaylistUpdateRunOrder::from(18); - let failed_first_run = playlist_update_run_result( - 7, - InputJobState::Failed, - vec![TuliproxError::RepositoryPlaylist("https://first:secret@provider.example/playlist".to_string())], - false, - ); - let persisted_then_failed = playlist_update_run_result( - 8, - InputJobState::Ready, - vec![TuliproxError::RepositoryPlaylist("https://epg:secret@provider.example/guide".to_string())], - false, - ); - - report_input_job_completion(&events, &run_id, execution_order, &failed_first_run); - report_input_job_completion(&events, &run_id, execution_order, &persisted_then_failed); - - let emitted = events.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let progress = emitted - .iter() - .filter_map(|event| match event { - EventMessage::PlaylistUpdateProgress(progress) => Some(progress), - _ => None, - }) - .collect::>(); - assert_eq!(progress.len(), 2); - assert_eq!(progress[0].input_id, Some(7)); - assert_eq!(progress[0].message, "Input 'input-7' failed during update"); - assert_eq!(progress[1].input_id, Some(8)); - assert_eq!(progress[1].message, "Input 'input-8' failed during update"); - assert!(progress.iter().all(|event| event.run_id.as_ref() == Some(&run_id))); - assert!(progress.iter().all(|event| event.execution_order == Some(execution_order))); - assert!(progress.iter().all(|event| event.state == Some(PlaylistUpdateState::Failure))); - assert!(progress.iter().all(|event| !event.message.contains("secret"))); - assert!(progress.iter().all(|event| !event.message.contains("previous accepted data"))); -} - -fn serialize_without_trailing_fields(value: &T, trailing_fields: &[u8]) -> Vec { - let mut encoded = rmp_serde::to_vec(value).expect("playlist item should serialize"); - for expected in trailing_fields { - assert_eq!(encoded.pop(), Some(*expected), "unexpected trailing MessagePack field"); - } - let removed = trailing_fields.len(); - match encoded[0] { - marker @ 0x92..=0x9f => { - let len = usize::from(marker - 0x90); - assert!(len >= removed, "trailing field count exceeds MessagePack sequence length"); - encoded[0] = 0x90 + u8::try_from(len - removed).unwrap_or_default(); - } - 0xdc => { - let len = u16::from_be_bytes([encoded[1], encoded[2]]); - let removed = u16::try_from(removed).unwrap_or(u16::MAX); - assert!(len >= removed, "trailing field count exceeds MessagePack sequence length"); - encoded[1..3].copy_from_slice(&(len - removed).to_be_bytes()); - } - 0xdd => { - let len = u32::from_be_bytes([encoded[1], encoded[2], encoded[3], encoded[4]]); - let removed = u32::try_from(removed).unwrap_or(u32::MAX); - assert!(len >= removed, "trailing field count exceeds MessagePack sequence length"); - encoded[1..5].copy_from_slice(&(len - removed).to_be_bytes()); - } - marker => panic!("unexpected MessagePack sequence marker {marker:#x}"), - } - encoded -} - -fn item_with_props(props: StreamProperties) -> PlaylistItem { - let header = shared::model::PlaylistItemHeader { additional_properties: Some(props), ..Default::default() }; - PlaylistItem { header } -} - -fn live_item_with_probe_timestamp_and_bitrate(last_probed_timestamp: i64, bitrate: u32) -> PlaylistItem { - item_with_props(StreamProperties::Live(Box::new(shared::model::LiveStreamProperties { - video: Some("{\"codec_name\":\"h264\"}".intern()), - audio: Some("{\"codec_name\":\"aac\"}".intern()), - bitrate, - last_probed_timestamp: Some(last_probed_timestamp), - ..Default::default() - }))) -} - -#[test] -fn rename_preserves_input_stream_id_captured_at_target_boundary() { - let mut item = PlaylistItem { - header: PlaylistItemHeader { - id: "origin-alpha".intern(), - url: "http://provider.example/channel.m3u8".intern(), - ..Default::default() - }, - }; - item.header.freeze_input_stream_id(); - let rename = ConfigRename::from(&ConfigRenameDto { - field: ItemField::Url, - pattern: "provider".to_string(), - new_name: "target".to_string(), - t_pattern: None, - }); - - exec_rename(&mut item, Some(&vec![rename])); - - assert_eq!(item.header.url.as_ref(), "http://target.example/channel.m3u8"); - assert_eq!(item.header.input_stream_id.as_ref(), "origin-alpha"); -} - -#[test] -fn mapper_changes_id_without_changing_frozen_input_stream_id() { - let mut item = PlaylistItem { - header: PlaylistItemHeader { id: "origin-alpha".intern(), name: "Channel".intern(), ..Default::default() }, - }; - item.header.freeze_input_stream_id(); - let mapping = CompiledMapping { - rules: vec![CompiledMappingRule { - name: None, - filter: get_filter(r#"name ~ ".*""#, None).expect("filter should parse"), - program: MappingProgram::Script( - MapperScript::parse(r#"@id = "target-id""#, None).expect("mapper should parse"), - ), - }], - ..Default::default() - }; - - let outcome = map_channel(item, &mapping); - - assert_eq!(outcome.matched_rules, 1); - assert_eq!(outcome.channel.header.id.as_ref(), "target-id"); - assert_eq!(outcome.channel.header.input_stream_id.as_ref(), "origin-alpha"); -} - -#[test] -fn mapper_cannot_resurrect_missing_legacy_input_stream_id_from_target_id() { - let mut source = PlaylistItem { - header: PlaylistItemHeader { - id: "80510".intern(), - url: "http://provider.example/live/user/pass/80510.ts".intern(), - input_name: "input".intern(), - item_type: PlaylistItemType::Live, - xtream_cluster: XtreamCluster::Live, - ..Default::default() - }, - }; - source.header.freeze_input_stream_id(); - let mut legacy_xtream = XtreamPlaylistItem::from(&source); - legacy_xtream.provider_id = 0; - legacy_xtream.input_stream_id = "".intern(); - legacy_xtream.url = "http://provider.example/live/channel.m3u8".intern(); - let mut legacy_item = PlaylistItem::from(&legacy_xtream); - legacy_item.header.freeze_input_stream_id(); - let mapping = CompiledMapping { - rules: vec![CompiledMappingRule { - name: None, - filter: get_filter(r#"name ~ ".*""#, None).expect("filter should parse"), - program: MappingProgram::Script( - MapperScript::parse(r#"@id = "target-id""#, None).expect("mapper should parse"), - ), - }], - ..Default::default() - }; - - let outcome = map_channel(legacy_item, &mapping); - let materialized_m3u = M3uPlaylistItem::from(&outcome.channel); - let materialized_xtream = XtreamPlaylistItem::from(&outcome.channel); - - assert_eq!(outcome.matched_rules, 1); - assert_eq!(outcome.channel.header.id.as_ref(), "target-id"); - assert_eq!(outcome.channel.get_input_stream_id(), None); - assert!(materialized_m3u.provider_id.is_empty()); - assert_eq!(materialized_m3u.get_input_stream_id(), None); - assert_eq!(materialized_xtream.provider_id, 0); - assert_eq!(materialized_xtream.get_input_stream_id(), None); -} - -#[test] -fn execute_pipe_freezes_input_stream_id_without_rename_or_mapper() { - let input = ConfigInput::default(); - let item = PlaylistItem { header: PlaylistItemHeader { id: "origin-alpha".intern(), ..Default::default() } }; - let source = MemoryPlaylistSource::new(vec![PlaylistGroup { - id: 1, - title: "Group".intern(), - channels: vec![item], - xtream_cluster: XtreamCluster::Live, - }]) - .into_source(); - let mut fetched = FetchedPlaylist { input: &input, source, epg: None }; - let mut duplicates = HashSet::new(); - let target = ConfigTarget::from(&ConfigTargetDto::default()); - - let (mut processed, _outcome) = execute_pipe(&target, &vec![], &mut fetched, &mut duplicates, false, None) - .expect("target processing should succeed"); - let mut groups = processed.source.take_groups(); - - assert_eq!(groups[0].channels[0].header.input_stream_id.as_ref(), "origin-alpha"); - assert!(groups[0].channels[0].header.set_field("id", "late-target-id")); - assert_eq!(groups[0].channels[0].header.input_stream_id.as_ref(), "origin-alpha"); -} - -#[test] -fn execute_pipe_applies_target_bouquet_prefilter() { - let input = ConfigInput::default(); - let item1 = PlaylistItem { - header: PlaylistItemHeader { - id: "ch-1".intern(), - group: "Kids".intern(), - xtream_cluster: XtreamCluster::Live, - ..Default::default() - }, - }; - let item2 = PlaylistItem { - header: PlaylistItemHeader { - id: "ch-2".intern(), - group: "Adults".intern(), - xtream_cluster: XtreamCluster::Live, - ..Default::default() - }, - }; - let source = MemoryPlaylistSource::new(vec![ - PlaylistGroup { id: 1, title: "Kids".intern(), channels: vec![item1], xtream_cluster: XtreamCluster::Live }, - PlaylistGroup { id: 2, title: "Adults".intern(), channels: vec![item2], xtream_cluster: XtreamCluster::Live }, - ]) - .into_source(); - let mut fetched = FetchedPlaylist { input: &input, source, epg: None }; - let mut duplicates = HashSet::new(); - let target = ConfigTarget::from(&ConfigTargetDto::default()); - - let bouquet_dto = - shared::model::PlaylistClusterBouquetDto { live: Some(vec!["Kids".to_string()]), vod: None, series: None }; - let filter = - tuliprox_core::model::TargetBouquetFilter::from_dto(shared::model::TargetBouquetDto::whitelist(bouquet_dto)) - .unwrap(); - - let (mut processed, _outcome) = execute_pipe(&target, &vec![], &mut fetched, &mut duplicates, false, Some(&filter)) - .expect("target processing should succeed"); - let groups = processed.source.take_groups(); - - assert_eq!(groups.len(), 1); - assert_eq!(groups[0].title.as_ref(), "Kids"); - assert_eq!(groups[0].channels.len(), 1); - assert_eq!(groups[0].channels[0].header.id.as_ref(), "ch-1"); -} - -#[test] -fn execute_pipe_prefilter_does_not_suppress_allowed_duplicate() { - let input = ConfigInput::default(); - // Two items with the same URL (same UUID): one in disallowed group, one in allowed group. - let item_disallowed = PlaylistItem { - header: PlaylistItemHeader { - id: "ch-1".intern(), - url: "http://provider.example/stream.ts".intern(), - group: "Adults".intern(), - xtream_cluster: XtreamCluster::Live, - ..Default::default() - }, - }; - let item_allowed = PlaylistItem { - header: PlaylistItemHeader { - id: "ch-2".intern(), - url: "http://provider.example/stream.ts".intern(), - group: "Kids".intern(), - xtream_cluster: XtreamCluster::Live, - ..Default::default() - }, - }; - let source = MemoryPlaylistSource::new(vec![ - PlaylistGroup { - id: 1, - title: "Adults".intern(), - channels: vec![item_disallowed], - xtream_cluster: XtreamCluster::Live, - }, - PlaylistGroup { - id: 2, - title: "Kids".intern(), - channels: vec![item_allowed], - xtream_cluster: XtreamCluster::Live, - }, - ]) - .into_source(); - let mut fetched = FetchedPlaylist { input: &input, source, epg: None }; - let mut duplicates = HashSet::new(); - let target = ConfigTarget::from(&ConfigTargetDto { - options: Some(ConfigTargetOptions { remove_duplicates: true, ..Default::default() }), - ..Default::default() - }); - - let bouquet_dto = - shared::model::PlaylistClusterBouquetDto { live: Some(vec!["Kids".to_string()]), vod: None, series: None }; - let filter = - tuliprox_core::model::TargetBouquetFilter::from_dto(shared::model::TargetBouquetDto::whitelist(bouquet_dto)) - .unwrap(); - - let (mut processed, _outcome) = execute_pipe(&target, &vec![], &mut fetched, &mut duplicates, false, Some(&filter)) - .expect("target processing should succeed"); - let groups = processed.source.take_groups(); - - // The allowed item must be retained because the rejected item did not consume the duplicate UUID slot. - assert_eq!(groups.len(), 1); - assert_eq!(groups[0].title.as_ref(), "Kids"); - assert_eq!(groups[0].channels.len(), 1); - assert_eq!(groups[0].channels[0].header.id.as_ref(), "ch-2"); -} - -#[test] -fn legacy_messagepack_playlist_items_default_missing_input_stream_id() { - let mut source = PlaylistItem { - header: PlaylistItemHeader { - id: "origin-alpha".intern(), - url: "http://provider.example/live/user/pass/80510.ts".intern(), - input_name: "input".intern(), - item_type: PlaylistItemType::Live, - xtream_cluster: XtreamCluster::Live, - ..Default::default() - }, - }; - - let header_bytes = serialize_without_trailing_fields(&source.header, &[0xc0, 0xa0]); - let decoded_header: PlaylistItemHeader = - rmp_serde::from_slice(&header_bytes).expect("legacy header should deserialize"); - assert!(decoded_header.input_stream_id.is_empty()); - assert_eq!(decoded_header.get_input_stream_id(), None); - assert_eq!(decoded_header.upstream_user_agent, None); - let mut decoded_header = decoded_header; - decoded_header.freeze_input_stream_id(); - assert_eq!(decoded_header.get_input_stream_id().as_deref(), Some("origin-alpha")); - - source.header.freeze_input_stream_id(); - let mut m3u_item = M3uPlaylistItem::from(&source); - m3u_item.input_stream_id = "".intern(); - let m3u_bytes = serialize_without_trailing_fields(&m3u_item, &[0xc0, 0xa0]); - let decoded_m3u: M3uPlaylistItem = rmp_serde::from_slice(&m3u_bytes).expect("legacy M3U item should deserialize"); - assert!(decoded_m3u.input_stream_id.is_empty()); - assert_eq!(decoded_m3u.get_input_stream_id().as_deref(), Some("origin-alpha")); - assert_eq!(decoded_m3u.upstream_user_agent, None); - - let mut xtream_item = XtreamPlaylistItem::from(&source); - xtream_item.input_stream_id = "".intern(); - let xtream_bytes = serialize_without_trailing_fields(&xtream_item, &[0xc0, 0xa0]); - let decoded_xtream: XtreamPlaylistItem = - rmp_serde::from_slice(&xtream_bytes).expect("legacy Xtream item should deserialize"); - assert!(decoded_xtream.input_stream_id.is_empty()); - assert_eq!(decoded_xtream.get_input_stream_id().as_deref(), Some("80510")); - assert_eq!(decoded_xtream.upstream_user_agent, None); -} - -#[test] -fn previous_messagepack_playlist_items_default_missing_upstream_user_agent() { - let source = PlaylistItem { - header: PlaylistItemHeader { - id: "80510".intern(), - input_stream_id: "origin-alpha".intern(), - ..Default::default() - }, - }; - - let header: PlaylistItemHeader = rmp_serde::from_slice(&serialize_without_trailing_fields(&source.header, &[0xc0])) - .expect("previous header should deserialize"); - let m3u: M3uPlaylistItem = - rmp_serde::from_slice(&serialize_without_trailing_fields(&M3uPlaylistItem::from(&source), &[0xc0])) - .expect("previous M3U item should deserialize"); - let xtream: XtreamPlaylistItem = - rmp_serde::from_slice(&serialize_without_trailing_fields(&XtreamPlaylistItem::from(&source), &[0xc0])) - .expect("previous Xtream item should deserialize"); - - assert_eq!(header.input_stream_id.as_ref(), "origin-alpha"); - assert_eq!(m3u.input_stream_id.as_ref(), "origin-alpha"); - assert_eq!(xtream.input_stream_id.as_ref(), "origin-alpha"); - assert_eq!(header.upstream_user_agent, None); - assert_eq!(m3u.upstream_user_agent, None); - assert_eq!(xtream.upstream_user_agent, None); -} - -#[test] -fn messagepack_playlist_items_preserve_upstream_user_agent() -> Result<(), Box> { - let source = PlaylistItem { - header: PlaylistItemHeader { upstream_user_agent: Some("Provider-UA".intern()), ..Default::default() }, - }; - - let header: PlaylistItemHeader = rmp_serde::from_slice(&rmp_serde::to_vec(&source.header)?)?; - let m3u: M3uPlaylistItem = rmp_serde::from_slice(&rmp_serde::to_vec(&M3uPlaylistItem::from(&source))?)?; - let xtream: XtreamPlaylistItem = rmp_serde::from_slice(&rmp_serde::to_vec(&XtreamPlaylistItem::from(&source))?)?; - - assert_eq!(header.upstream_user_agent.as_deref(), Some("Provider-UA")); - assert_eq!(m3u.upstream_user_agent.as_deref(), Some("Provider-UA")); - assert_eq!(xtream.upstream_user_agent.as_deref(), Some("Provider-UA")); - Ok(()) -} - -#[test] -fn has_probe_details_requires_video_and_audio_for_video() { - let video = shared::model::VideoStreamProperties { - details: Some(shared::model::VideoStreamDetailProperties { - video: Some("{\"codec_name\":\"h264\"}".intern()), - audio: None, - ..Default::default() - }), - ..Default::default() - }; - let item_missing_audio = item_with_props(StreamProperties::Video(Box::new(video))); - assert!(!has_probe_details(&item_missing_audio)); - - let video_complete = shared::model::VideoStreamProperties { - details: Some(shared::model::VideoStreamDetailProperties { - video: Some("{\"codec_name\":\"h264\"}".intern()), - audio: Some("{\"codec_name\":\"aac\"}".intern()), - ..Default::default() - }), - ..Default::default() - }; - let item_complete = item_with_props(StreamProperties::Video(Box::new(video_complete))); - assert!(has_probe_details(&item_complete)); -} - -#[test] -fn has_probe_details_requires_video_audio_and_bitrate_for_live() { - let live_missing_audio = shared::model::LiveStreamProperties { - video: Some("{\"codec_name\":\"h264\"}".intern()), - audio: None, - ..Default::default() - }; - let item_missing_audio = item_with_props(StreamProperties::Live(Box::new(live_missing_audio))); - assert!(!has_probe_details(&item_missing_audio)); - - let live_missing_bitrate = shared::model::LiveStreamProperties { - video: Some("{\"codec_name\":\"h264\"}".intern()), - audio: Some("{\"codec_name\":\"aac\"}".intern()), - ..Default::default() - }; - let item_missing_bitrate = item_with_props(StreamProperties::Live(Box::new(live_missing_bitrate))); - assert!(!has_probe_details(&item_missing_bitrate)); - - let live_complete = shared::model::LiveStreamProperties { - video: Some("{\"codec_name\":\"h264\"}".intern()), - audio: Some("{\"codec_name\":\"aac\"}".intern()), - bitrate: 2_500_000, - ..Default::default() - }; - let item_complete = item_with_props(StreamProperties::Live(Box::new(live_complete))); - assert!(has_probe_details(&item_complete)); -} - -#[test] -fn needs_live_probe_when_fresh_probe_has_no_bitrate() { - let item = live_item_with_probe_timestamp_and_bitrate(101, 0); - - assert!(needs_live_probe(&item, 100)); -} - -#[test] -fn does_not_need_live_probe_when_fresh_probe_has_positive_bitrate() { - let item = live_item_with_probe_timestamp_and_bitrate(101, 2_500_000); - - assert!(!needs_live_probe(&item, 100)); -} - -#[test] -fn needs_live_probe_when_positive_bitrate_probe_is_older_than_cutoff() { - let item = live_item_with_probe_timestamp_and_bitrate(99, 2_500_000); - - assert!(needs_live_probe(&item, 100)); -} - -#[test] -fn has_probe_details_is_false_for_series() { - let series = shared::model::SeriesStreamProperties::default(); - let item = item_with_props(StreamProperties::Series(Box::new(series))); - assert!(!has_probe_details(&item)); -} - -#[test] -fn collect_effective_skip_clusters_uses_input_skip_flags() { - use tuliprox_core::model::{ConfigInputFlags, ConfigInputOptions}; - let input = ConfigInput { - name: "skip_live".intern(), - input_type: InputType::Xtream, - options: Some(ConfigInputOptions { - flags: ConfigInputFlags::SkipLive.into(), - ..ConfigInputOptions::defaults().clone() - }), - ..ConfigInput::default() - }; - let skip = collect_effective_skip_clusters(&input); - assert!(skip.contains(&XtreamCluster::Live)); - assert!(!skip.contains(&XtreamCluster::Video)); - assert!(!skip.contains(&XtreamCluster::Series)); -} - -#[test] -fn filter_skipped_clusters_removes_cached_groups() { - use tuliprox_core::model::{ConfigInputFlags, ConfigInputOptions}; - let live_item = PlaylistItem { - header: shared::model::PlaylistItemHeader { xtream_cluster: XtreamCluster::Live, ..Default::default() }, - }; - let vod_item = PlaylistItem { - header: shared::model::PlaylistItemHeader { xtream_cluster: XtreamCluster::Video, ..Default::default() }, - }; - - let groups = vec![ - PlaylistGroup { id: 1, title: "Live".intern(), channels: vec![live_item], xtream_cluster: XtreamCluster::Live }, - PlaylistGroup { id: 2, title: "Vod".intern(), channels: vec![vod_item], xtream_cluster: XtreamCluster::Video }, - ]; - - let source = MemoryPlaylistSource::new(groups).into_source(); - let input = ConfigInput { - name: "skip_live".intern(), - input_type: InputType::Xtream, - options: Some(ConfigInputOptions { - flags: ConfigInputFlags::SkipLive.into(), - ..ConfigInputOptions::defaults().clone() - }), - ..ConfigInput::default() - }; - - let mut filtered = filter_skipped_clusters_from_source(source, &input); - let filtered_groups = filtered.take_groups(); - assert_eq!(filtered_groups.len(), 1); - assert_eq!(filtered_groups[0].xtream_cluster, XtreamCluster::Video); -} - -fn test_group(cluster: XtreamCluster, item_name: &str, input_name: &str) -> PlaylistGroup { - PlaylistGroup { - id: 1, - title: item_name.intern(), - xtream_cluster: cluster, - channels: vec![PlaylistItem { - header: PlaylistItemHeader { - name: item_name.intern(), - input_name: input_name.intern(), - xtream_cluster: cluster, - item_type: match cluster { - XtreamCluster::Live => PlaylistItemType::Live, - XtreamCluster::Video => PlaylistItemType::Video, - XtreamCluster::Series => PlaylistItemType::Series, - }, - ..Default::default() - }, - }], - } -} - -#[test] -fn pipeline_transparency_staged_overlay_replaces_groups_and_neutralizes_overlaid_runtime_facts() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - options: Some(ConfigInputOptions::defaults().clone()), - ..Default::default() - }; - let provider_groups = vec![ - test_group(XtreamCluster::Live, "provider-live", "provider"), - test_group(XtreamCluster::Video, "provider-vod", "provider"), - ]; - let mut staged_live = test_group(XtreamCluster::Live, "staged-live", "staged"); - staged_live.channels[0].header.id = "11203".intern(); - staged_live.channels[0].header.url = "http://iptvhost.example/live/fake-user/fake-pass/11203.ts".intern(); - let mut invalid = test_group(XtreamCluster::Live, "invalid", "staged").channels.remove(0); - invalid.header.id = "invalid".intern(); - invalid.header.url = "http://iptvhost.example/live/fake-user/fake-pass/invalid.ts".intern(); - staged_live.channels.push(invalid); - let staged_groups = vec![staged_live, test_group(XtreamCluster::Series, "staged-series", "staged")]; - - let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Live, provider_groups, staged_groups); - - assert_eq!(groups.len(), 2); - assert_eq!(groups[0].title.as_ref(), "staged-live"); - assert_eq!(groups[0].channels.len(), 1); - assert_eq!(groups[0].channels[0].header.input_name.as_ref(), "provider"); - assert_eq!(groups[0].channels[0].header.url.as_ref(), "http://provider.example/live/real-user/real-pass/11203.ts"); - assert_eq!(groups[1].title.as_ref(), "provider-vod"); - assert_eq!(groups[1].channels[0].header.input_name.as_ref(), "provider"); - - let mut telemetry = PlaylistUpdateInputTelemetry { - refresh_policy: InputRefreshPolicy::NORMAL, - source: None, - clusters: vec![ - PlaylistUpdateClusterTelemetry { - cluster: XtreamCluster::Live, - requested: true, - source: Some(PlaylistUpdateDataSource::Provider), - baseline_count: Some(100), - candidate_count: Some(95), - active_count: Some(95), - threshold: Some(90), - quality: Some(95), - decision: Some(PlaylistUpdateClusterDecision::Accepted), - technical_state: None, - }, - PlaylistUpdateClusterTelemetry { - cluster: XtreamCluster::Video, - requested: true, - source: Some(PlaylistUpdateDataSource::Provider), - baseline_count: Some(200), - candidate_count: Some(200), - active_count: Some(200), - threshold: Some(100), - quality: Some(100), - decision: Some(PlaylistUpdateClusterDecision::Accepted), - technical_state: None, - }, - ], - }; - - neutralize_overlaid_cluster_facts(&mut telemetry, ClusterFlags::Live); - - let live = - telemetry.clusters.iter().find(|cluster| cluster.cluster == XtreamCluster::Live).expect("Live telemetry"); - assert!(live.requested); - assert_eq!(live.threshold, Some(90)); - assert_eq!(live.source, None); - assert_eq!(live.baseline_count, None); - assert_eq!(live.candidate_count, None); - assert_eq!(live.active_count, None); - assert_eq!(live.quality, None); - assert_eq!(live.decision, None); - - let video = - telemetry.clusters.iter().find(|cluster| cluster.cluster == XtreamCluster::Video).expect("Video telemetry"); - assert_eq!(video.source, Some(PlaylistUpdateDataSource::Provider)); - assert_eq!(video.baseline_count, Some(200)); - assert_eq!(video.candidate_count, Some(200)); - assert_eq!(video.active_count, Some(200)); - assert_eq!(video.quality, Some(100)); - assert_eq!(video.decision, Some(PlaylistUpdateClusterDecision::Accepted)); -} - -#[test] -fn staged_xtream_vod_url_uses_provider_credentials_and_one_extension() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut staged_vod = test_group(XtreamCluster::Video, "staged-vod", "staged"); - staged_vod.channels[0].header.id = "310".intern(); - staged_vod.channels[0].header.url = "http://iptvhost.example/movie/fake-user/fake-pass/310.mkv".intern(); - - let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Vod, Vec::new(), vec![staged_vod]); - - assert_eq!(groups[0].channels[0].header.url.as_ref(), "http://provider.example/movie/real-user/real-pass/310.mkv"); -} - -#[test] -fn staged_xtream_overlay_keeps_known_provider_stream_urls() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_vod = test_group(XtreamCluster::Video, "Movies", "provider"); - provider_vod.channels[0].header.id = "310".intern(); - provider_vod.channels[0].header.url = "https://cdn.example/video?id=310".intern(); - let mut second_provider = provider_vod.channels[0].clone(); - second_provider.header.id = "311".intern(); - second_provider.header.url = "http://provider.example/movie/real-user/real-pass/311.mp4".intern(); - provider_vod.channels.push(second_provider); - - let mut staged_vod = test_group(XtreamCluster::Video, "Edited Movies", "staged"); - staged_vod.channels[0].header.id = "310".intern(); - staged_vod.channels[0].header.url = "http://editor.example/movie/310.mkv".intern(); - let mut second_staged = staged_vod.channels[0].clone(); - second_staged.header.id = "311".intern(); - second_staged.header.url = "http://editor.example/movie/311.mkv".intern(); - staged_vod.channels.push(second_staged); - - let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Vod, vec![provider_vod], vec![staged_vod]); - - assert_eq!(groups[0].title.as_ref(), "Edited Movies"); - assert_eq!(groups[0].channels[0].header.url.as_ref(), "https://cdn.example/video?id=310"); - assert_eq!(groups[0].channels[1].header.url.as_ref(), "http://provider.example/movie/real-user/real-pass/311.mp4"); -} - -#[test] -fn staged_xtream_new_vod_stream_uses_metadata_extension() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut staged_vod = test_group(XtreamCluster::Video, "New Movies", "staged"); - staged_vod.channels[0].header.id = "310".intern(); - staged_vod.channels[0].header.url = "http://editor.example/movie/310".intern(); - staged_vod.channels[0].header.additional_properties = - Some(StreamProperties::Video(Box::new(shared::model::VideoStreamProperties { - container_extension: "mkv".intern(), - ..Default::default() - }))); - - let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Vod, Vec::new(), vec![staged_vod]); - - assert_eq!(groups[0].channels[0].header.url.as_ref(), "http://provider.example/movie/real-user/real-pass/310.mkv"); -} - -#[test] -fn staged_xtream_live_url_respects_prefix_and_without_extension_flags() { - let provider_prefix = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - options: Some(ConfigInputOptions { - flags: ConfigInputFlags::XtreamLiveStreamUsePrefix.into(), - ..ConfigInputOptions::defaults().clone() - }), - ..Default::default() - }; - let mut staged_live_prefix = test_group(XtreamCluster::Live, "staged-live", "staged"); - staged_live_prefix.channels[0].header.id = "11203".intern(); - staged_live_prefix.channels[0].header.url = "http://iptvhost.example/fake-user/fake-pass/11203.ts".intern(); - - let groups_prefix = - apply_staged_overlay_groups(&provider_prefix, ClusterFlags::Live, Vec::new(), vec![staged_live_prefix]); - - assert_eq!( - groups_prefix[0].channels[0].header.url.as_ref(), - "http://provider.example/live/real-user/real-pass/11203.ts" - ); - - let provider_no_ext = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - options: Some(ConfigInputOptions { - flags: ConfigInputFlags::XtreamLiveStreamWithoutExtension.into(), - ..ConfigInputOptions::defaults().clone() - }), - ..Default::default() - }; - let mut staged_live_no_ext = test_group(XtreamCluster::Live, "staged-live", "staged"); - staged_live_no_ext.channels[0].header.id = "11203".intern(); - staged_live_no_ext.channels[0].header.url = "http://iptvhost.example/fake-user/fake-pass/11203.ts".intern(); - - let groups_no_ext = - apply_staged_overlay_groups(&provider_no_ext, ClusterFlags::Live, Vec::new(), vec![staged_live_no_ext]); - - assert_eq!(groups_no_ext[0].channels[0].header.url.as_ref(), "http://provider.example/real-user/real-pass/11203"); -} - -#[test] -fn staged_xtream_overlay_recomputes_uuid_on_group_load() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - options: Some(ConfigInputOptions { - flags: ConfigInputFlags::XtreamLiveStreamUsePrefix.into(), - ..ConfigInputOptions::defaults().clone() - }), - ..Default::default() - }; - let mut staged_live = test_group(XtreamCluster::Live, "staged-live", "staged"); - staged_live.channels[0].header.id = "11203".intern(); - staged_live.channels[0].header.freeze_input_stream_id(); - staged_live.channels[0].header.url = "http://iptvhost.example/live/fake-user/fake-pass/11203.ts".intern(); - staged_live.channels[0].header.gen_uuid(); - let old_uuid = staged_live.channels[0].header.get_uuid().clone(); - - let mut groups = apply_staged_overlay_groups(&provider, ClusterFlags::Live, Vec::new(), vec![staged_live]); - - assert_eq!(groups[0].channels[0].header.input_name.as_ref(), "provider"); - let reconstructed_url = "http://provider.example/live/real-user/real-pass/11203.ts"; - assert_eq!(groups[0].channels[0].header.url.as_ref(), reconstructed_url); - assert_eq!(groups[0].channels[0].header.get_input_stream_id().as_deref(), Some("11203")); - - groups[0].on_load(); - - let expected_uuid = shared::utils::generate_runtime_playlist_uuid( - "provider", - "11203", - shared::model::PlaylistItemType::Live, - reconstructed_url, - ); - assert_ne!(groups[0].channels[0].header.get_uuid(), &old_uuid); - assert_eq!(groups[0].channels[0].header.get_uuid(), &expected_uuid); - assert_eq!(groups[0].channels[0].header.get_input_stream_id().as_deref(), Some("11203")); -} - -#[test] -fn staged_xtream_overlay_full_cluster_staging_replaces_all_matching_groups() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_action = test_group(XtreamCluster::Video, "Action", "provider"); - provider_action.channels[0].header.id = "10".intern(); - let mut provider_comedy = test_group(XtreamCluster::Video, "Comedy", "provider"); - provider_comedy.channels[0].header.id = "20".intern(); - - let mut staged_action = test_group(XtreamCluster::Video, "Action", "staged"); - staged_action.channels[0].header.id = "11".intern(); - let mut staged_comedy = test_group(XtreamCluster::Video, "Comedy", "staged"); - staged_comedy.channels[0].header.id = "21".intern(); - - let groups = apply_staged_overlay_groups( - &provider, - ClusterFlags::Vod, - vec![provider_action, provider_comedy], - vec![staged_action, staged_comedy], - ); - - assert_eq!(groups.len(), 2); - assert_eq!(groups[0].title.as_ref(), "Action"); - assert_eq!(groups[0].channels[0].header.id.as_ref(), "11"); - assert_eq!(groups[0].channels[0].header.input_name.as_ref(), "provider"); - assert_eq!(groups[1].title.as_ref(), "Comedy"); - assert_eq!(groups[1].channels[0].header.id.as_ref(), "21"); - assert_eq!(groups[1].channels[0].header.input_name.as_ref(), "provider"); -} - -#[test] -fn staged_xtream_overlay_partial_cluster_staging_preserves_unstaged_provider_groups() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_action = test_group(XtreamCluster::Video, "Action", "provider"); - provider_action.channels[0].header.id = "10".intern(); - let mut provider_comedy = test_group(XtreamCluster::Video, "Comedy", "provider"); - provider_comedy.channels[0].header.id = "20".intern(); - let mut provider_drama = test_group(XtreamCluster::Video, "Drama", "provider"); - provider_drama.channels[0].header.id = "30".intern(); - - let mut staged_action = test_group(XtreamCluster::Video, "Action", "staged"); - staged_action.channels[0].header.id = "11".intern(); - - let groups = apply_staged_overlay_groups( - &provider, - ClusterFlags::Vod, - vec![provider_action, provider_comedy, provider_drama], - vec![staged_action], - ); - - assert_eq!(groups.len(), 3); - assert_eq!(groups[0].title.as_ref(), "Action"); - assert_eq!(groups[0].channels[0].header.id.as_ref(), "11"); - assert_eq!(groups[1].title.as_ref(), "Comedy"); - assert_eq!(groups[1].channels[0].header.id.as_ref(), "20"); - assert_eq!(groups[2].title.as_ref(), "Drama"); - assert_eq!(groups[2].channels[0].header.id.as_ref(), "30"); -} - -#[test] -fn staged_xtream_overlay_no_staged_groups_for_selected_cluster_preserves_all_provider_groups() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let provider_news = test_group(XtreamCluster::Live, "News", "provider"); - let provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); - let staged_series = test_group(XtreamCluster::Series, "Shows", "staged"); - - let groups = apply_staged_overlay_groups( - &provider, - ClusterFlags::Live, - vec![provider_news, provider_sports], - vec![staged_series], - ); - - assert_eq!(groups.len(), 2); - assert_eq!(groups[0].title.as_ref(), "News"); - assert_eq!(groups[1].title.as_ref(), "Sports"); -} - -#[test] -fn staged_xtream_overlay_all_invalid_ids_falls_back_to_provider_group() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); - provider_sports.channels[0].header.id = "501".intern(); - - let mut staged_sports = test_group(XtreamCluster::Live, "Sports", "staged"); - staged_sports.channels[0].header.id = "non-numeric-a".intern(); - let mut second_invalid = staged_sports.channels[0].clone(); - second_invalid.header.id = "non-numeric-b".intern(); - staged_sports.channels.push(second_invalid); - - let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Live, vec![provider_sports], vec![staged_sports]); - - assert_eq!(groups.len(), 1); - assert_eq!(groups[0].title.as_ref(), "Sports"); - assert_eq!(groups[0].channels.len(), 1); - assert_eq!(groups[0].channels[0].header.id.as_ref(), "501"); -} - -#[test] -fn staged_xtream_overlay_partially_valid_ids_keeps_valid_and_drops_invalid() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); - provider_sports.channels[0].header.id = "501".intern(); - - let mut staged_sports = test_group(XtreamCluster::Live, "Sports", "staged"); - staged_sports.channels[0].header.id = "502".intern(); - let mut invalid = staged_sports.channels[0].clone(); - invalid.header.id = "invalid-stream".intern(); - staged_sports.channels.push(invalid); - - let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Live, vec![provider_sports], vec![staged_sports]); - - assert_eq!(groups.len(), 1); - assert_eq!(groups[0].title.as_ref(), "Sports"); - assert_eq!(groups[0].channels.len(), 1); - assert_eq!(groups[0].channels[0].header.id.as_ref(), "502"); -} - -#[test] -fn staged_xtream_overlay_missing_credentials_preserves_all_provider_groups() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: None, - password: None, - ..Default::default() - }; - let provider_news = test_group(XtreamCluster::Live, "News", "provider"); - let provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); - let mut staged_sports = test_group(XtreamCluster::Live, "Sports", "staged"); - staged_sports.channels[0].header.id = "501".intern(); - - let groups = apply_staged_overlay_groups( - &provider, - ClusterFlags::Live, - vec![provider_news, provider_sports], - vec![staged_sports], - ); - - assert_eq!(groups.len(), 2); - assert_eq!(groups[0].title.as_ref(), "News"); - assert_eq!(groups[1].title.as_ref(), "Sports"); -} - -#[test] -fn staged_xtream_overlay_multiple_groups_with_different_clusters() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_live = test_group(XtreamCluster::Live, "News", "provider"); - provider_live.channels[0].header.id = "100".intern(); - let mut provider_action = test_group(XtreamCluster::Video, "Action", "provider"); - provider_action.channels[0].header.id = "200".intern(); - let mut provider_comedy = test_group(XtreamCluster::Video, "Comedy", "provider"); - provider_comedy.channels[0].header.id = "300".intern(); - let mut provider_series = test_group(XtreamCluster::Series, "Shows", "provider"); - provider_series.channels[0].header.id = "400".intern(); - - let mut staged_live = test_group(XtreamCluster::Live, "News", "staged"); - staged_live.channels[0].header.id = "101".intern(); - let mut staged_action = test_group(XtreamCluster::Video, "Action", "staged"); - staged_action.channels[0].header.id = "201".intern(); - - let groups = apply_staged_overlay_groups( - &provider, - ClusterFlags::Live | ClusterFlags::Vod, - vec![provider_live, provider_action, provider_comedy, provider_series], - vec![staged_live, staged_action], - ); - - assert_eq!(groups.len(), 4); - assert_eq!(groups[0].title.as_ref(), "News"); - assert_eq!(groups[0].channels[0].header.id.as_ref(), "101"); - assert_eq!(groups[1].title.as_ref(), "Action"); - assert_eq!(groups[1].channels[0].header.id.as_ref(), "201"); - assert_eq!(groups[2].title.as_ref(), "Comedy"); - assert_eq!(groups[2].channels[0].header.id.as_ref(), "300"); - assert_eq!(groups[3].title.as_ref(), "Shows"); - assert_eq!(groups[3].channels[0].header.id.as_ref(), "400"); -} - -#[test] -fn staged_xtream_overlay_preserves_original_group_order() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut live_1 = test_group(XtreamCluster::Live, "Live1", "provider"); - live_1.channels[0].header.id = "1".intern(); - let mut vod_1 = test_group(XtreamCluster::Video, "Vod1", "provider"); - vod_1.channels[0].header.id = "2".intern(); - let mut live_2 = test_group(XtreamCluster::Live, "Live2", "provider"); - live_2.channels[0].header.id = "3".intern(); - let mut vod_2 = test_group(XtreamCluster::Video, "Vod2", "provider"); - vod_2.channels[0].header.id = "4".intern(); - - let mut staged_live_1 = test_group(XtreamCluster::Live, "Live1", "staged"); - staged_live_1.channels[0].header.id = "11".intern(); - let mut staged_live_2 = test_group(XtreamCluster::Live, "Live2", "staged"); - staged_live_2.channels[0].header.id = "33".intern(); - - let groups = apply_staged_overlay_groups( - &provider, - ClusterFlags::Live, - vec![live_1, vod_1, live_2, vod_2], - vec![staged_live_1, staged_live_2], - ); - - assert_eq!(groups.len(), 4); - assert_eq!(groups[0].title.as_ref(), "Live1"); - assert_eq!(groups[0].channels[0].header.id.as_ref(), "11"); - assert_eq!(groups[1].title.as_ref(), "Vod1"); - assert_eq!(groups[1].channels[0].header.id.as_ref(), "2"); - assert_eq!(groups[2].title.as_ref(), "Live2"); - assert_eq!(groups[2].channels[0].header.id.as_ref(), "33"); - assert_eq!(groups[3].title.as_ref(), "Vod2"); - assert_eq!(groups[3].channels[0].header.id.as_ref(), "4"); -} - -/// The merged playlist reaches persistence keyed by `(cluster, id)`, which collapses groups that -/// share an id within one cluster. -fn assert_unique_category_ids(groups: &[PlaylistGroup]) { - let mut seen = std::collections::HashSet::new(); - for group in groups { - assert!( - seen.insert((group.xtream_cluster, group.id)), - "category id {} is used by more than one {} group", - group.id, - group.xtream_cluster - ); - } -} - -#[test] -fn staged_overlay_syncs_channel_category_ids_with_the_final_group_id() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_news = test_group(XtreamCluster::Live, "News", "provider"); - provider_news.id = 5; - provider_news.channels[0].header.id = "100".intern(); - - // Matched by stream id, so the group takes over the provider category id. - let mut staged_rewritten = test_group(XtreamCluster::Live, "Rewritten", "staged"); - staged_rewritten.id = 7; - staged_rewritten.channels[0].header.id = "100".intern(); - staged_rewritten.channels[0].header.category_id = 7; - // No provider counterpart: the group becomes a new category with an allocated id. - let mut staged_kids = test_group(XtreamCluster::Live, "Kids", "staged"); - staged_kids.id = 0; - staged_kids.channels[0].header.id = "300".intern(); - - let groups = apply_staged_overlay_groups( - &provider, - ClusterFlags::Live, - vec![provider_news], - vec![staged_rewritten, staged_kids], - ); - - assert_eq!(groups.len(), 2); - assert_eq!(groups[0].id, 5); - assert_eq!(groups[0].channels[0].header.category_id, 5); - assert_eq!(groups[1].id, 6); - assert_eq!(groups[1].channels[0].header.category_id, 6); - assert_unique_category_ids(&groups); -} - -#[test] -fn staged_overlay_on_non_xtream_provider_replaces_cluster_and_rewrites_input_name() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::M3u, - url: "http://provider.example/list.m3u".to_string(), - ..Default::default() - }; - let provider_live = test_group(XtreamCluster::Live, "provider-live", "provider"); - let provider_vod = test_group(XtreamCluster::Video, "provider-vod", "provider"); - let mut staged_live = test_group(XtreamCluster::Live, "staged-live", "staged"); - staged_live.channels[0].header.url = "http://editor.example/live/1.ts".intern(); - - let groups = apply_staged_overlay_groups( - &provider, - ClusterFlags::Live, - vec![provider_live, provider_vod], - vec![staged_live], - ); - - assert_eq!(groups.len(), 2); - assert_eq!(groups[0].title.as_ref(), "provider-vod"); - assert_eq!(groups[0].channels[0].header.input_name.as_ref(), "provider"); - assert_eq!(groups[1].title.as_ref(), "staged-live"); - // A non-Xtream provider cannot rebuild Xtream stream URLs, so the staged URL and its group id stay. - assert_eq!(groups[1].channels[0].header.input_name.as_ref(), "provider"); - assert_eq!(groups[1].channels[0].header.url.as_ref(), "http://editor.example/live/1.ts"); - assert_eq!(groups[1].id, 1); -} - -#[test] -fn staged_xtream_overlay_matches_by_category_id_not_by_title() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - // The staged tool renamed the provider category 7 and moved its streams; the title now collides - // with the unrelated provider category 5. - let mut provider_kids = test_group(XtreamCluster::Live, "Kids", "provider"); - provider_kids.id = 5; - provider_kids.channels[0].header.id = "105".intern(); - let mut provider_news = test_group(XtreamCluster::Live, "News", "provider"); - provider_news.id = 7; - provider_news.channels[0].header.id = "107".intern(); - - let mut staged_renamed = test_group(XtreamCluster::Live, "Kids", "staged"); - staged_renamed.id = 7; - staged_renamed.channels[0].header.id = "207".intern(); - - let groups = apply_staged_overlay_groups( - &provider, - ClusterFlags::Live, - vec![provider_kids, provider_news], - vec![staged_renamed], - ); - - assert_eq!(groups.len(), 2); - // Category 5 keeps its content: the staged group belongs to category 7. - assert_eq!(groups[0].title.as_ref(), "Kids"); - assert_eq!(groups[0].id, 5); - assert_eq!(groups[0].channels[0].header.id.as_ref(), "105"); - // Category 7 carries the staged rename and the staged stream, under its own id. - assert_eq!(groups[1].title.as_ref(), "Kids"); - assert_eq!(groups[1].id, 7); - assert_eq!(groups[1].channels[0].header.id.as_ref(), "207"); - assert_unique_category_ids(&groups); -} - -#[test] -fn staged_xtream_overlay_matches_by_stream_id_over_group_id() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_news = test_group(XtreamCluster::Live, "News", "provider"); - provider_news.id = 5; - provider_news.channels[0].header.id = "100".intern(); - let mut provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); - provider_sports.id = 7; - provider_sports.channels[0].header.id = "200".intern(); - - // A m3u staged playlist numbers its groups on its own, so the group id points at another - // provider category. The stream id of the overlaid channel proves which category it replaces. - let mut staged_sports = test_group(XtreamCluster::Live, "Sports Rewritten", "staged"); - staged_sports.id = 5; - staged_sports.channels[0].header.id = "200".intern(); - - let groups = apply_staged_overlay_groups( - &provider, - ClusterFlags::Live, - vec![provider_news, provider_sports], - vec![staged_sports], - ); - - assert_eq!(groups.len(), 2); - // Category 5 keeps its content: no staged channel belongs to it. - assert_eq!(groups[0].title.as_ref(), "News"); - assert_eq!(groups[0].id, 5); - assert_eq!(groups[0].channels[0].header.id.as_ref(), "100"); - // Category 7 carries the rewritten staged group, under its own id. - assert_eq!(groups[1].title.as_ref(), "Sports Rewritten"); - assert_eq!(groups[1].id, 7); - assert_eq!(groups[1].channels[0].header.id.as_ref(), "200"); - assert_unique_category_ids(&groups); -} - -#[test] -fn staged_xtream_split_group_does_not_replace_an_unrelated_category() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); - provider_sports.id = 5; - provider_sports.channels[0].header.id = "100".intern(); - let mut second_sport = provider_sports.channels[0].clone(); - second_sport.header.id = "101".intern(); - provider_sports.channels.push(second_sport); - let mut provider_news = test_group(XtreamCluster::Live, "News", "provider"); - provider_news.id = 7; - provider_news.channels[0].header.id = "200".intern(); - - let mut staged_first = test_group(XtreamCluster::Live, "Sports A", "staged"); - staged_first.id = 5; - staged_first.channels[0].header.id = "100".intern(); - let mut staged_second = test_group(XtreamCluster::Live, "News", "staged"); - staged_second.id = 7; - staged_second.channels[0].header.id = "101".intern(); - - let groups = apply_staged_overlay_groups( - &provider, - ClusterFlags::Live, - vec![provider_sports, provider_news], - vec![staged_first, staged_second], - ); - - assert_eq!(groups.len(), 3); - assert_eq!(groups[0].id, 5); - assert_eq!(groups[0].title.as_ref(), "Sports A"); - assert_eq!(groups[1].id, 7); - assert_eq!(groups[1].channels[0].header.id.as_ref(), "200"); - assert_eq!(groups[2].title.as_ref(), "News"); - assert_eq!(groups[2].channels[0].header.id.as_ref(), "101"); - assert_unique_category_ids(&groups); -} - -#[test] -fn staged_m3u_unknown_streams_do_not_match_positional_category_id() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); - provider_sports.id = 5; - provider_sports.channels[0].header.id = "100".intern(); - let mut staged_new = test_group(XtreamCluster::Live, "New", "staged"); - staged_new.id = 5; - staged_new.channels[0].header.id = "300".intern(); - - let groups = super::apply_staged_overlay_groups( - &provider, - StagedInputType::M3u, - ClusterFlags::Live, - vec![provider_sports], - vec![staged_new], - ); - - assert_eq!(groups.len(), 1); - assert_eq!(groups[0].title.as_ref(), "New"); - assert_eq!(groups[0].channels[0].header.id.as_ref(), "300"); - assert_unique_category_ids(&groups); -} - -#[test] -fn staged_m3u_replaces_selected_xtream_groups_and_keeps_other_clusters() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_news = test_group(XtreamCluster::Live, "Provider News", "provider"); - provider_news.id = 5; - provider_news.channels[0].header.id = "100".intern(); - let mut provider_sports = test_group(XtreamCluster::Live, "Provider Sports", "provider"); - provider_sports.id = 7; - provider_sports.channels[0].header.id = "200".intern(); - provider_sports.channels[0].header.url = "http://provider.example/direct/200.ts".intern(); - let mut provider_kids = test_group(XtreamCluster::Live, "Provider Kids", "provider"); - provider_kids.id = 9; - provider_kids.channels[0].header.id = "300".intern(); - let provider_vod = test_group(XtreamCluster::Video, "Provider VOD", "provider"); - let mut staged_custom = test_group(XtreamCluster::Live, "My Channels", "staged"); - staged_custom.channels[0].header.id = "200".intern(); - let mut staged_news = test_group(XtreamCluster::Live, "My News", "staged"); - staged_news.channels[0].header.id = "100".intern(); - - let groups = super::apply_staged_overlay_groups( - &provider, - StagedInputType::M3u, - ClusterFlags::Live, - vec![provider_news, provider_sports, provider_kids, provider_vod], - vec![staged_custom, staged_news], - ); - - assert_eq!(groups.len(), 3); - assert_eq!(groups[0].title.as_ref(), "My Channels"); - assert_eq!(groups[0].channels[0].header.id.as_ref(), "200"); - assert_eq!(groups[0].channels[0].header.input_name.as_ref(), "provider"); - assert_eq!(groups[0].channels[0].header.url.as_ref(), "http://provider.example/direct/200.ts"); - assert_eq!(groups[1].title.as_ref(), "My News"); - assert_eq!(groups[1].channels[0].header.id.as_ref(), "100"); - assert_eq!(groups[2].title.as_ref(), "Provider VOD"); -} - -#[test] -fn staged_xtream_overlay_falls_back_to_the_secondary_stream_overlap() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_alpha = test_group(XtreamCluster::Live, "Alpha", "provider"); - provider_alpha.id = 5; - provider_alpha.channels[0].header.id = "100".intern(); - let mut provider_beta = test_group(XtreamCluster::Live, "Beta", "provider"); - provider_beta.id = 7; - provider_beta.channels[0].header.id = "200".intern(); - let mut beta_second = provider_beta.channels[0].clone(); - beta_second.header.id = "201".intern(); - provider_beta.channels.push(beta_second); - - // Both staged groups overlap Beta; the stronger one takes it, so the other has to fall back to the - // provider category it also owns streams of instead of becoming a new category. - let mut staged_strong = test_group(XtreamCluster::Live, "Beta Rewritten", "staged"); - staged_strong.id = 0; - staged_strong.channels[0].header.id = "200".intern(); - let mut strong_second = staged_strong.channels[0].clone(); - strong_second.header.id = "201".intern(); - staged_strong.channels.push(strong_second); - - let mut staged_weak = test_group(XtreamCluster::Live, "Alpha Rewritten", "staged"); - staged_weak.id = 0; - staged_weak.channels[0].header.id = "200".intern(); - let mut weak_second = staged_weak.channels[0].clone(); - weak_second.header.id = "201".intern(); - staged_weak.channels.push(weak_second); - let mut weak_third = staged_weak.channels[0].clone(); - weak_third.header.id = "100".intern(); - staged_weak.channels.push(weak_third); - - let groups = apply_staged_overlay_groups( - &provider, - ClusterFlags::Live, - vec![provider_alpha, provider_beta], - vec![staged_strong, staged_weak], - ); - - assert_eq!(groups.len(), 2); - assert_eq!(groups[0].title.as_ref(), "Alpha Rewritten"); - assert_eq!(groups[0].id, 5); - assert_eq!(groups[0].channels.len(), 3); - assert_eq!(groups[1].title.as_ref(), "Beta Rewritten"); - assert_eq!(groups[1].id, 7); - assert_eq!(groups[1].channels.len(), 2); - assert_unique_category_ids(&groups); -} - -#[test] -fn staged_xtream_overlay_gives_a_new_category_a_free_category_id() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); - provider_sports.id = 2; - provider_sports.channels[0].header.id = "200".intern(); - - // Staged groups without a category id must not reach persistence as a shared zero key. - let mut staged_kids = test_group(XtreamCluster::Live, "Kids", "staged"); - staged_kids.id = 0; - staged_kids.channels[0].header.id = "300".intern(); - let mut staged_movies = test_group(XtreamCluster::Live, "Movies", "staged"); - staged_movies.id = 0; - staged_movies.channels[0].header.id = "400".intern(); - - let groups = apply_staged_overlay_groups( - &provider, - ClusterFlags::Live, - vec![provider_sports], - vec![staged_kids, staged_movies], - ); - - assert_eq!(groups.len(), 3); - assert_eq!(groups[0].title.as_ref(), "Sports"); - assert_eq!(groups[0].id, 2); - assert_eq!(groups[1].title.as_ref(), "Kids"); - assert_eq!(groups[1].id, 3); - assert_eq!(groups[2].title.as_ref(), "Movies"); - assert_eq!(groups[2].id, 4); - assert_unique_category_ids(&groups); -} - -#[test] -fn staged_xtream_overlay_empty_staged_group_without_provider_fallback_is_omitted() { - let provider = ConfigInput { - name: "provider".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("real-user".to_string()), - password: Some("real-pass".to_string()), - ..Default::default() - }; - let mut provider_action = test_group(XtreamCluster::Video, "Action", "provider"); - provider_action.channels[0].header.id = "10".intern(); - - let mut staged_horror = test_group(XtreamCluster::Video, "Horror", "staged"); - staged_horror.id = 2; - staged_horror.channels[0].header.id = "non-numeric".intern(); - - let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Vod, vec![provider_action], vec![staged_horror]); - - assert_eq!(groups.len(), 1); - assert_eq!(groups[0].title.as_ref(), "Action"); -} - -#[test] -fn staged_overlay_is_skipped_when_provider_playlist_is_cached() { - let result = PlaylistDownloadResult::new(vec![], vec![], true, false); - - assert!(!should_apply_staged_overlay(&result)); -} - -#[test] -fn quality_rejection_keeps_a_usable_input_ready() { - let mut result = InputDownloadResult { - authoritative_library: false, - errors: Vec::new(), - source: MemoryPlaylistSource::new(vec![test_group(XtreamCluster::Video, "retained-vod", "provider-a")]) - .into_source(), - storage_error: None, - partial: false, - quality_rejections: vec![ClusterUpdateRejection { - cluster: XtreamCluster::Video, - current_count: 12_543, - candidate_count: 217, - threshold: 90, - quality: 1, - }], - accepted_empty_clusters: ClusterFlags::empty(), - input_telemetry: None, - }; - - assert_eq!(result.job_state(), InputJobState::Ready); - assert!(result.errors.is_empty()); - assert!(!result.partial); - assert_eq!(result.quality_rejections.len(), 1); -} - -#[test] -fn quality_rejection_marks_the_run_partial_without_reusing_stalker_partial() { - let quality_rejection = PlaylistRunSignals { has_quality_rejections: true, ..PlaylistRunSignals::default() }; - assert!(!quality_rejection.has_pending_stalker_refresh); - assert_eq!(quality_rejection.state(), PlaylistUpdateState::Partial); - - let stalker_partial = PlaylistRunSignals { has_pending_stalker_refresh: true, ..PlaylistRunSignals::default() }; - assert!(!stalker_partial.has_quality_rejections); - assert_eq!(stalker_partial.state(), PlaylistUpdateState::Partial); - - let technical_failure = PlaylistRunSignals { has_error: true, ..quality_rejection }; - assert_eq!(technical_failure.state(), PlaylistUpdateState::Failure); -} - -fn make_test_item(name: &str, item_type: PlaylistItemType) -> PlaylistItem { - let header = - PlaylistItemHeader { name: name.into(), group: "Test Group".intern(), item_type, ..Default::default() }; - PlaylistItem { header } -} - -#[test] -fn test_filter_evalutes_correctly() { - let filter = get_filter(r#"name ~ "Allowed""#, None).unwrap(); - - let allowed_item = make_test_item("Allowed Channel", PlaylistItemType::Live); - let denied_item = make_test_item("Denied Channel", PlaylistItemType::Live); - - let allowed_provider = ValueProvider { pli: &allowed_item, match_as_ascii: false }; - let denied_provider = ValueProvider { pli: &denied_item, match_as_ascii: false }; - - assert!(filter.filter(&allowed_provider)); - assert!(!filter.filter(&denied_provider)); -} - -#[test] -fn test_filter_with_type_comparison() { - let filter = get_filter("type = vod", None).unwrap(); - - let vod_item = make_test_item("Test Movie", PlaylistItemType::Video); - let live_item = make_test_item("Test Channel", PlaylistItemType::Live); - - let vod_provider = ValueProvider { pli: &vod_item, match_as_ascii: false }; - let live_provider = ValueProvider { pli: &live_item, match_as_ascii: false }; - - assert!(filter.filter(&vod_provider)); - assert!(!filter.filter(&live_provider)); -} - -#[test] -fn playlist_retention_reports_filter_counts() { - let groups = vec![PlaylistGroup { - id: 1, - title: "Test Group".intern(), - channels: vec![ - make_test_item("Allowed", PlaylistItemType::Live), - make_test_item("Denied", PlaylistItemType::Live), - ], - xtream_cluster: XtreamCluster::Live, - }]; - let mut source = MemoryPlaylistSource::new(groups).into_source(); - - let (filtered, outcome) = retain_playlist_items(&mut source, |item| item.header.name.as_ref() == "Allowed"); - - assert_eq!(outcome, FilterOutcome { inspected: 2, retained: 1, removed: 1 }); - assert_eq!(filtered.expect("one item should remain")[0].channels[0].header.name.as_ref(), "Allowed"); -} - -#[test] -fn filter_stage_can_remove_every_item() { - let groups = vec![PlaylistGroup { - id: 1, - title: "Test Group".intern(), - channels: vec![make_test_item("Denied", PlaylistItemType::Live)], - xtream_cluster: XtreamCluster::Live, - }]; - let mut target = ConfigTarget::from(&ConfigTargetDto::default()); - target.filter = get_filter(r#"name ~ "Allowed""#, None).expect("filter should parse").into(); - - let (groups, outcome) = execute_pipeline_on_groups(groups, &target, &[TransformStage::Filter]); - - assert!(groups.is_empty()); - assert_eq!(outcome.filter, Some(FilterOutcome { inspected: 1, retained: 0, removed: 1 })); -} - -#[test] -fn missing_processing_filter_skips_filter_stage() { - let groups = vec![PlaylistGroup { - id: 1, - title: "Test Group".intern(), - channels: vec![make_test_item("Allowed", PlaylistItemType::Live)], - xtream_cluster: XtreamCluster::Live, - }]; - let target = ConfigTarget::from(&ConfigTargetDto::default()); - - let (groups, outcome) = execute_pipeline_on_groups(groups, &target, &[TransformStage::Filter]); - - assert_eq!(groups.len(), 1); - assert_eq!(groups[0].channels.len(), 1); - assert!(outcome.filter.is_none()); -} - -#[test] -fn missing_processing_filter_preserves_filter_stage_group_normalization() { - let mut first = make_test_item("One", PlaylistItemType::Live); - first.header.group = "News".intern(); - let mut second = make_test_item("Two", PlaylistItemType::Live); - second.header.group = "news".intern(); - let groups = vec![ - PlaylistGroup { id: 1, title: "News".intern(), channels: vec![first], xtream_cluster: XtreamCluster::Live }, - PlaylistGroup { id: 2, title: "news".intern(), channels: vec![second], xtream_cluster: XtreamCluster::Live }, - ]; - let target = ConfigTarget::from(&ConfigTargetDto::default()); - - let (groups, outcome) = execute_pipeline_on_groups(groups, &target, &[TransformStage::Filter]); - - assert_eq!(groups.len(), 1); - assert_eq!(groups[0].channels.len(), 2); - assert!(outcome.filter.is_none()); -} - -#[test] -fn pipeline_reports_filter_and_rename_outcomes() { - let groups = vec![PlaylistGroup { - id: 1, - title: "Test Group".intern(), - channels: vec![ - make_test_item("Allowed", PlaylistItemType::Live), - make_test_item("Denied", PlaylistItemType::Live), - ], - xtream_cluster: XtreamCluster::Live, - }]; - let mut target = ConfigTarget::from(&ConfigTargetDto::default()); - target.filter = get_filter(r#"name ~ "Allowed""#, None).expect("filter should parse").into(); - target.rename = Some(vec![ConfigRename::from(&ConfigRenameDto { - field: ItemField::Name, - pattern: "Allowed".to_string(), - new_name: "Renamed".to_string(), - t_pattern: None, - })]); - - let (groups, outcome) = - execute_pipeline_on_groups(groups, &target, &[TransformStage::Filter, TransformStage::Rename]); - - assert_eq!(groups[0].channels[0].header.name.as_ref(), "Renamed"); - assert_eq!(outcome.filter, Some(FilterOutcome { inspected: 2, retained: 1, removed: 1 })); - assert_eq!(outcome.rename, Some(RenameOutcome { inspected: 1, changed_items: 1, changed_fields: 1 })); -} - -#[test] -fn assign_channel_no_playlist_preserves_non_zero_chno() { - let mut groups = vec![ - PlaylistGroup { - id: 1, - title: "Group A".intern(), - channels: vec![ - PlaylistItem { header: PlaylistItemHeader { name: "A".intern(), chno: 10, ..Default::default() } }, - PlaylistItem { header: PlaylistItemHeader { name: "B".intern(), chno: 0, ..Default::default() } }, - ], - xtream_cluster: XtreamCluster::Live, - }, - PlaylistGroup { - id: 2, - title: "Group C".intern(), - channels: vec![ - PlaylistItem { header: PlaylistItemHeader { name: "C".intern(), chno: 1, ..Default::default() } }, - PlaylistItem { header: PlaylistItemHeader { name: "D".intern(), chno: 0, ..Default::default() } }, - ], - xtream_cluster: XtreamCluster::Live, - }, - ]; - - assign_channel_no_playlist(&mut groups); - - // Non-zero chno values must be preserved - assert_eq!(groups[0].channels[0].header.chno, 10); - assert_eq!(groups[1].channels[0].header.chno, 1); -} - -#[test] -fn assign_channel_no_playlist_assigns_zero_chno_only() { - let mut groups = vec![PlaylistGroup { - id: 1, - title: "Group A".intern(), - channels: vec![ - PlaylistItem { header: PlaylistItemHeader { name: "A".intern(), chno: 0, ..Default::default() } }, - PlaylistItem { header: PlaylistItemHeader { name: "B".intern(), chno: 0, ..Default::default() } }, - PlaylistItem { header: PlaylistItemHeader { name: "C".intern(), chno: 0, ..Default::default() } }, - ], - xtream_cluster: XtreamCluster::Live, - }]; - - assign_channel_no_playlist(&mut groups); - - // All zero-chno channels should get assigned numbers starting at 1 - assert_eq!(groups[0].channels[0].header.chno, 1); - assert_eq!(groups[0].channels[1].header.chno, 2); - assert_eq!(groups[0].channels[2].header.chno, 3); -} - -#[test] -fn assign_channel_no_playlist_skips_existing_nonzero_numbers() { - let mut groups = vec![PlaylistGroup { - id: 1, - title: "Group A".intern(), - channels: vec![ - PlaylistItem { header: PlaylistItemHeader { name: "A".intern(), chno: 5, ..Default::default() } }, - PlaylistItem { header: PlaylistItemHeader { name: "B".intern(), chno: 0, ..Default::default() } }, - PlaylistItem { header: PlaylistItemHeader { name: "C".intern(), chno: 2, ..Default::default() } }, - PlaylistItem { header: PlaylistItemHeader { name: "D".intern(), chno: 0, ..Default::default() } }, - ], - xtream_cluster: XtreamCluster::Live, - }]; - - assign_channel_no_playlist(&mut groups); - - // Existing non-zero numbers (2, 5) must be skipped when assigning new numbers - assert_eq!(groups[0].channels[0].header.chno, 5); // preserved - assert_eq!(groups[0].channels[2].header.chno, 2); // preserved - // B gets 1 (smallest available), D gets 3 (next available after 1 and existing 2) - assert_eq!(groups[0].channels[1].header.chno, 1); - assert_eq!(groups[0].channels[3].header.chno, 3); -} - -#[test] -fn assign_channel_no_playlist_assigns_following_group_order() { - let mut groups = vec![ - PlaylistGroup { - id: 1, - title: "Group 1".intern(), - channels: vec![ - PlaylistItem { header: PlaylistItemHeader { name: "A".intern(), chno: 0, ..Default::default() } }, - PlaylistItem { header: PlaylistItemHeader { name: "B".intern(), chno: 0, ..Default::default() } }, - ], - xtream_cluster: XtreamCluster::Live, - }, - PlaylistGroup { - id: 2, - title: "Group 2".intern(), - channels: vec![PlaylistItem { - header: PlaylistItemHeader { name: "C".intern(), chno: 0, ..Default::default() }, - }], - xtream_cluster: XtreamCluster::Live, - }, - ]; - - assign_channel_no_playlist(&mut groups); - - // Numbers should follow iteration order across groups: A=1, B=2, C=3 - assert_eq!(groups[0].channels[0].header.chno, 1); - assert_eq!(groups[0].channels[1].header.chno, 2); - assert_eq!(groups[1].channels[0].header.chno, 3); -} - -#[tokio::test] -async fn parallel_input_scheduler_serializes_equal_groups_and_overlaps_distinct_groups() { - use std::sync::atomic::{AtomicUsize, Ordering}; - - async fn observe(active: &AtomicUsize, maximum: &AtomicUsize) { - let current = active.fetch_add(1, Ordering::SeqCst) + 1; - maximum.fetch_max(current, Ordering::SeqCst); - tokio::time::sleep(Duration::from_millis(20)).await; - active.fetch_sub(1, Ordering::SeqCst); - } - - let locks = tuliprox_core::utils::FileLockManager::default(); - let active = AtomicUsize::new(0); - let maximum = AtomicUsize::new(0); - tokio::join!( - with_sequential_group(&locks, Some(7), true, observe(&active, &maximum)), - with_sequential_group(&locks, Some(7), true, observe(&active, &maximum)), - ); - assert_eq!(maximum.load(Ordering::SeqCst), 1); - - maximum.store(0, Ordering::SeqCst); - tokio::join!( - with_sequential_group(&locks, Some(7), true, observe(&active, &maximum)), - with_sequential_group(&locks, Some(8), true, observe(&active, &maximum)), - ); - assert_eq!(maximum.load(Ordering::SeqCst), 2); -} - -#[tokio::test] -async fn parallel_input_scheduler_releases_group_after_abort() { - let locks = Arc::new(tuliprox_core::utils::FileLockManager::default()); - let task_locks = Arc::clone(&locks); - let task = tokio::spawn(async move { - with_sequential_group(&task_locks, Some(7), true, std::future::pending::<()>()).await; - }); - tokio::task::yield_now().await; - task.abort(); - let _ = task.await; - - tokio::time::timeout(Duration::from_secs(1), with_sequential_group(&locks, Some(7), true, std::future::ready(()))) - .await - .expect("aborting an input job must release its sequential group"); -} - -#[test] -fn input_progress_message_contains_each_target_and_blocking_input() { - let targets = ["target-a", "target-b"]; - let inputs = ["input-a", "input-b"]; - let messages: Vec<_> = targets - .iter() - .flat_map(|target| inputs.iter().map(move |input| target_waiting_message(target, input))) - .collect(); - - assert_eq!(messages.len(), 4); - for target in targets { - for input in inputs { - assert!(messages.contains(&format!("Target '{target}' is waiting for input '{input}'"))); - } - } - assert!(stalker_checkpoint_message("portal-a").contains("portal-a")); -} - -#[tokio::test] -async fn parallel_target_pipeline_bounds_active_finalizers() { - use std::sync::atomic::{AtomicUsize, Ordering}; - - let active = Arc::new(AtomicUsize::new(0)); - let maximum = Arc::new(AtomicUsize::new(0)); - let mut tasks = JoinSet::new(); - let mut results = Vec::new(); - let mut errors = Vec::new(); - - for index in 0..6 { - wait_for_target_finalizer_slot(&mut tasks, &mut results, &mut errors).await; - let active = Arc::clone(&active); - let maximum = Arc::clone(&maximum); - tasks.spawn(async move { - let current = active.fetch_add(1, Ordering::SeqCst) + 1; - maximum.fetch_max(current, Ordering::SeqCst); - tokio::time::sleep(Duration::from_millis(10)).await; - active.fetch_sub(1, Ordering::SeqCst); - TargetJobResult { - index, - name: format!("target-{index}"), - result: Ok(()), - errors: Vec::new(), - processing: PipelineStats::default(), - } - }); - } - while let Some(result) = tasks.join_next().await { - collect_target_task_result(result, &mut results, &mut errors); - } - - assert!(maximum.load(Ordering::SeqCst) <= MAX_CONCURRENT_TARGET_FINALIZERS); - assert_eq!(results.len(), 6); - assert!(errors.is_empty()); -} - -#[test] -fn parallel_target_pipeline_normalizes_conflicting_output_resources() { - let config = Config { storage_dir: "/tmp/tuliprox-target-resources".to_string(), ..Config::default() }; - - let mut spaced = ConfigTarget::from(&ConfigTargetDto::default()); - spaced.name = "A B".to_string(); - let mut underscored = ConfigTarget::from(&ConfigTargetDto::default()); - underscored.name = "A_B".to_string(); - assert!(!target_mutated_resources(&config, &spaced).is_disjoint(&target_mutated_resources(&config, &underscored))); - - spaced.name = "one".to_string(); - spaced.output = vec![tuliprox_core::model::TargetOutput::M3u(tuliprox_core::model::M3uTargetOutput { - filename: Some("out/../x.m3u".to_string()), - include_type_in_url: false, - mask_redirect_url: false, - filter: None, - })]; - underscored.name = "two".to_string(); - underscored.output = vec![tuliprox_core::model::TargetOutput::M3u(tuliprox_core::model::M3uTargetOutput { - filename: Some("x.m3u".to_string()), - include_type_in_url: false, - mask_redirect_url: false, - filter: None, - })]; - assert!(!target_mutated_resources(&config, &spaced).is_disjoint(&target_mutated_resources(&config, &underscored))); -} - -mod mapping_stage { - use super::*; - use arc_swap::{ArcSwap, ArcSwapOption}; - use shared::model::{ConfigPaths, EpgSmartMatchConfigDto}; - use std::sync::Arc; - use tempfile::tempdir; - use tokio::runtime::Runtime; - use tuliprox_core::{ - model::{ - EpgConfig, EpgSmartMatchConfig, IcsEpgSourceConfig, MediaToolCapabilities, PersistedEpgSource, - PersistedEpgSourceKind, SourcesConfig, - }, - utils::FileLockManager, - }; - - pub(super) fn build_mapping(id: &str, stage: MappingStage, script: &str) -> CompiledMapping { - CompiledMapping { - id: id.to_string(), - match_as_ascii: false, - stage, - rules: vec![CompiledMappingRule { - name: None, - filter: get_filter(r#"name ~ ".*""#, None).expect("filter parses"), - program: MappingProgram::Script(MapperScript::parse(script, None).expect("script parses")), - }], - counters: vec![], - templates: None, - } - } - - pub(super) fn build_target(mappings: Vec, remove_duplicates: bool) -> ConfigTarget { - let dto = ConfigTargetDto { - options: if remove_duplicates { - Some(ConfigTargetOptions { remove_duplicates, ..Default::default() }) - } else { - None - }, - ..Default::default() - }; - let mut target = ConfigTarget::from(&dto); - target.mapping = Arc::new(ArcSwapOption::from(Some(Arc::new(CompiledTargetMappings::new( - mappings.into_iter().map(Arc::new).collect(), - ))))); - target - } - - /// Pinned to `NoopSink` rather than staying generic: these tests - /// exercise the pipeline, not the bus, and an inferred sink type - /// would just make every call site name one. - fn processing_context() -> PlaylistProcessingContext { - let paths = ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - }; - let config = AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config::default())), - sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), - hdhomerun: Arc::new(ArcSwapOption::default()), - api_proxy: Arc::new(ArcSwapOption::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(paths)), - custom_stream_response: Arc::new(ArcSwapOption::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - }; - PlaylistProcessingContext { - client: reqwest::Client::new(), - run_id: "m3u-alias-test-run".into(), - execution_order: PlaylistUpdateRunOrder::from(1), - config: Arc::new(config), - user_targets: Arc::new(ProcessTargets { - enabled: false, - inputs: Vec::new(), - targets: Vec::new(), - target_names: Vec::new(), - }), - events: shared::model::NoopSink, - playlist_state: None, - disabled_headers: None, - processed_inputs: Arc::new(Mutex::new(HashSet::new())), - input_completions: Arc::new(Mutex::new(HashMap::new())), - input_locks: Arc::new(Mutex::new(HashMap::new())), - provider_manager: None, - metadata_manager: None, - pre_processed_inputs: None, - input_refresh: None, - library_update_mode: LibraryUpdateMode::ExistingCatalog, - stalker_refresh_mode: StalkerRefreshMode::Complete, - partial_refresh: Arc::new(std::sync::atomic::AtomicBool::new(false)), - had_quality_rejections: Arc::new(std::sync::atomic::AtomicBool::new(false)), - } - } - - #[tokio::test] - async fn m3u_alias_playlist_is_downloaded_and_indexed_separately() { - let temp = tempfile::tempdir().expect("temp dir should be created"); - let primary_playlist_path = temp.path().join("primary.m3u"); - let alias_playlist_path = temp.path().join("backup.m3u"); - tokio::fs::write( - &primary_playlist_path, - "#EXTM3U\n#EXTINF:-1 tvg-id=\"323\",Channel\nhttp://stream.example:4000/323/mono.m3u8?token=primary-stream-token\n", - ) - .await - .expect("primary fixture should be written"); - tokio::fs::write( - &alias_playlist_path, - "#EXTM3U\n#EXTINF:-1 tvg-id=\"323\",Channel\nhttp://stream.example:4000/323/mono.m3u8?token=backup-stream-token\n", - ) - .await - .expect("alias fixture should be written"); - - let ctx = processing_context(); - let config = - Config { storage_dir: temp.path().join("storage").to_string_lossy().into_owned(), ..Config::default() }; - ctx.config.config.store(Arc::new(config)); - let input = Arc::new(ConfigInput { - id: 1, - name: "primary-account".intern(), - input_type: InputType::M3u, - url: primary_playlist_path.to_string_lossy().into_owned(), - enabled: true, - aliases: Some(vec![ConfigInputAlias { - id: 2, - name: "backup-account".intern(), - url: alias_playlist_path.to_string_lossy().into_owned(), - username: None, - password: None, - priority: 1, - max_connections: 1, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }); - - let InputDownloadResult { - errors, - source: mut primary_playlist, - storage_error, - partial, - quality_rejections, - .. - } = download_input(&ctx, &input, false).await; - - assert!(errors.is_empty(), "unexpected download errors: {errors:?}"); - assert!(storage_error.is_none(), "unexpected primary storage error: {storage_error:?}"); - assert!(!partial); - assert!(quality_rejections.is_empty()); - assert!(!primary_playlist.is_empty()); - let alias_url = tuliprox_repository::load_input_m3u_stream_url( - &ctx.config, - &"backup-account".intern(), - "http://stream.example:4000/323/mono.m3u8?token=primary-stream-token", - ) - .await - .expect("alias URL lookup should succeed"); - assert_eq!(alias_url.as_deref(), Some("http://stream.example:4000/323/mono.m3u8?token=backup-stream-token")); - } - - #[tokio::test] - async fn failed_m3u_alias_is_retried_after_primary_input_is_processed() { - let temp = tempfile::tempdir().expect("temp dir should be created"); - let primary_playlist_path = temp.path().join("main.m3u"); - let alias_playlist_path = temp.path().join("retry.m3u"); - tokio::fs::write( - &primary_playlist_path, - "#EXTM3U\n#EXTINF:-1 tvg-id=\"323\",Channel\nhttp://stream.example:4000/323/mono.m3u8?token=main-stream-token\n", - ) - .await - .expect("primary fixture should be written"); - - let ctx = processing_context(); - let config = - Config { storage_dir: temp.path().join("storage").to_string_lossy().into_owned(), ..Config::default() }; - ctx.config.config.store(Arc::new(config)); - let input = Arc::new(ConfigInput { - id: 1, - name: "main-account".intern(), - input_type: InputType::M3u, - url: primary_playlist_path.to_string_lossy().into_owned(), - enabled: true, - aliases: Some(vec![ConfigInputAlias { - id: 2, - name: "retry-account".intern(), - url: alias_playlist_path.to_string_lossy().into_owned(), - username: None, - password: None, - priority: 1, - max_connections: 1, - exp_date: None, - enabled: true, - stalker: None, - }]), - ..ConfigInput::default() - }); - - let InputDownloadResult { - errors: first_errors, - source: mut first_playlist, - storage_error: first_storage_error, - partial: first_partial, - quality_rejections: first_quality_rejections, - .. - } = download_input(&ctx, &input, false).await; - - assert!(!first_errors.is_empty(), "missing alias should report an error"); - assert!(first_storage_error.is_none(), "primary storage should succeed: {first_storage_error:?}"); - assert!(!first_partial); - assert!(first_quality_rejections.is_empty()); - assert!(!first_playlist.is_empty()); - assert!(ctx.is_input_downloaded("main-account").await); - assert!(!ctx.is_input_downloaded("retry-account").await); - - tokio::fs::write( - &alias_playlist_path, - "#EXTM3U\n#EXTINF:-1 tvg-id=\"323\",Channel\nhttp://stream.example:4000/323/mono.m3u8?token=retry-stream-token\n", - ) - .await - .expect("alias fixture should be written"); - - let InputDownloadResult { - errors: second_errors, - source: mut second_playlist, - storage_error: second_storage_error, - partial: second_partial, - quality_rejections: second_quality_rejections, - .. - } = download_input(&ctx, &input, false).await; - - assert!(second_errors.is_empty(), "unexpected retry errors: {second_errors:?}"); - assert!(second_storage_error.is_none(), "primary storage should remain readable: {second_storage_error:?}"); - assert!(!second_partial); - assert!(second_quality_rejections.is_empty()); - assert!(!second_playlist.is_empty()); - assert!(ctx.is_input_downloaded("retry-account").await); - let alias_url = tuliprox_repository::load_input_m3u_stream_url( - &ctx.config, - &"retry-account".intern(), - "http://stream.example:4000/323/mono.m3u8?token=main-stream-token", - ) - .await - .expect("retried alias URL lookup should succeed"); - assert_eq!(alias_url.as_deref(), Some("http://stream.example:4000/323/mono.m3u8?token=retry-stream-token")); - } - - #[test] - fn persist_filter_runs_after_after_epg_mapping() { - let runtime = Runtime::new().expect("runtime"); - runtime.block_on(async { - let mut input = ConfigInput::from(ConfigInputDto::default()); - input.name = "input".intern(); - let groups = vec![PlaylistGroup { - id: 1, - title: "Live".intern(), - channels: vec![PlaylistItem { - header: PlaylistItemHeader { - name: "Before".intern(), - group: "Live".intern(), - xtream_cluster: XtreamCluster::Live, - item_type: PlaylistItemType::Live, - ..Default::default() - }, - }], - xtream_cluster: XtreamCluster::Live, - }]; - let mut playlist = - FetchedPlaylist { input: &input, source: MemoryPlaylistSource::new(groups).into_source(), epg: None }; - let rename = build_mapping("rename", MappingStage::AfterEpg, r#"@Name = "After""#); - let mut target = build_target(vec![rename], false); - target.filter.persist = Some(get_filter(r#"Name = "After""#, None).expect("filter parses")); - let mut stats = HashMap::from([( - Arc::clone(&input.name), - create_input_stat(1, 1, 0, input.input_type, &input.name, 0), - )]); - let mut errors = Vec::new(); - - let mut prepared = prepare_playlist_for_target( - &processing_context(), - std::slice::from_mut(&mut playlist), - &target, - &mut stats, - &mut errors, - ClusterFlags::empty(), - false, - ) - .await - .expect("target preparation"); - - assert!(errors.is_empty()); - apply_persist_filter(&target, &mut prepared.playlist); - let item = &prepared.playlist[0].channels[0]; - assert_eq!(item.header.name.as_ref(), "After"); - }); - } - - fn make_channel(name: &str) -> PlaylistItem { - let mut item = PlaylistItem { - header: PlaylistItemHeader { - name: name.intern(), - group: "Originals".intern(), - xtream_cluster: XtreamCluster::Live, - item_type: PlaylistItemType::Live, - ..Default::default() - }, - }; - item.header.freeze_input_stream_id(); - item - } - - fn memory_source(channels: Vec) -> PlaylistSource { - MemoryPlaylistSource::new(vec![PlaylistGroup { - id: 1, - title: "Live".intern(), - channels, - xtream_cluster: XtreamCluster::Live, - }]) - .into_source() - } - - fn channel_count(source: &mut PlaylistSource) -> usize { - source.take_groups().iter().map(|g| g.channels.len()).sum() - } - - #[test] - fn map_playlist_applies_only_the_requested_stage() { - let processing = build_mapping("processing", MappingStage::Processing, r#"@name = concat(@Name, "-P")"#); - let after_epg = build_mapping("after_epg", MappingStage::AfterEpg, r#"@name = concat(@Name, "-E")"#); - let target = build_target(vec![processing, after_epg], false); - - let mut source = memory_source(vec![make_channel("Alpha")]); - let (groups, _) = execute_pipeline_on_groups(source.take_groups(), &target, &[TransformStage::Map]); - assert_eq!(groups[0].channels[0].header.name.as_ref(), "Alpha-P"); - - let mut source = MemoryPlaylistSource::new(groups).into_source(); - let groups = map_playlist_at_stage(&mut source, &target, MappingStage::AfterEpg, None) - .expect("after_epg mapping should run"); - assert_eq!(groups[0].channels[0].header.name.as_ref(), "Alpha-P-E"); - } - - #[test] - fn map_playlist_at_stage_returns_none_without_consuming_source_when_no_match() { - let target = build_target(Vec::new(), false); - let mut source = memory_source(vec![make_channel("Alpha")]); - - let result = map_playlist_at_stage(&mut source, &target, MappingStage::AfterEpg, None); - assert!(result.is_none(), "no matching stage must return None"); - assert_eq!(channel_count(&mut source), 1, "source must remain intact"); - } - - #[test] - fn prepare_target_applies_after_epg_mapping_before_sampling_stats() { - let runtime = Runtime::new().expect("runtime"); - runtime.block_on(async { - let dir = tempdir().expect("tempdir"); - let ics_path = dir.path().join("bbc.ics"); - std::fs::write( - &ics_path, - "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:News\nDTSTART:20260306T120000Z\nDTEND:20260306T130000Z\nEND:VEVENT\nEND:VCALENDAR", - ) - .expect("write ics"); - - let mut smart_dto = EpgSmartMatchConfigDto { - enabled: true, - fuzzy_matching: false, - ..EpgSmartMatchConfigDto::default() - }; - smart_dto.prepare().expect("smart config"); - let mut input = ConfigInput::from(ConfigInputDto::default()); - input.name = "input".intern(); - input.epg = Some(EpgConfig { - sources: vec![], - smart_match: Some(EpgSmartMatchConfig::from(smart_dto)), - }); - - let channels = vec![live_item_for_epg("BBC One")]; - let groups = vec![PlaylistGroup { - id: 1, - title: "Live".intern(), - channels, - xtream_cluster: XtreamCluster::Live, - }]; - let tv_guide = TVGuide::new(vec![PersistedEpgSource { - file_path: ics_path, - priority: 0, - logo_override: false, - kind: PersistedEpgSourceKind::Ics { - channel_id: "bbc.one".intern(), - channel_title: Some("BBC One".intern()), - match_names: vec!["BBC One".intern()], - config: Box::new(IcsEpgSourceConfig::default()), - }, - }]); - - let mut playlist = FetchedPlaylist { - input: &input, - source: MemoryPlaylistSource::new(groups).into_source(), - epg: Some(tv_guide), - }; - - let rename_from_epg = build_mapping( - "rename", - MappingStage::AfterEpg, - r#"epg = @epg_channel_id ~ "(.+)" -match { - epg => @Name = epg.1 -}"#, - ); - let add_virtual = build_mapping("virtual", MappingStage::AfterEpg, r#"add_favourite("Echo")"#); - let target = build_target(vec![rename_from_epg, add_virtual], false); - let mut stats = HashMap::from([( - Arc::clone(&input.name), - create_input_stat(1, 1, 0, input.input_type, &input.name, 0), - )]); - let mut errors = Vec::new(); - let prepared = prepare_playlist_for_target( - &processing_context(), - std::slice::from_mut(&mut playlist), - &target, - &mut stats, - &mut errors, - ClusterFlags::empty(), - false, - ) - .await - .expect("target preparation"); - - assert!(errors.is_empty()); - assert_eq!(prepared.playlist.iter().map(|group| group.channels.len()).sum::(), 2); - let channel = prepared - .playlist - .iter() - .flat_map(|group| &group.channels) - .find(|channel| channel.header.group.as_ref() != "Echo") - .expect("original channel"); - assert_eq!(channel.header.epg_channel_id.as_deref(), Some("bbc.one")); - assert_eq!( - channel.header.name.as_ref(), - "bbc.one", - "after_epg mapper must consume the EPG-enriched field" - ); - let processed_stats = &stats[&input.name].processed_stats; - assert_eq!(processed_stats.group_count, 2); - assert_eq!(processed_stats.channel_count, 2); - }); - } - - #[test] - fn clear_invalid_epg_ids_clears_ids_invalidated_by_after_epg_mapping() { - let runtime = Runtime::new().expect("runtime"); - runtime.block_on(async { - let dir = tempdir().expect("tempdir"); - let ics_path = dir.path().join("bbc.ics"); - std::fs::write( - &ics_path, - "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:News\nDTSTART:20260306T120000Z\nDTEND:20260306T130000Z\nEND:VEVENT\nEND:VCALENDAR", - ) - .expect("write ics"); - - let mut input = ConfigInput::from(ConfigInputDto::default()); - input.name = "input".intern(); - input.epg = Some(EpgConfig { sources: vec![], smart_match: None }); - let groups = vec![PlaylistGroup { - id: 1, - title: "Live".intern(), - channels: vec![PlaylistItem { - header: PlaylistItemHeader { - name: "BBC One".intern(), - epg_channel_id: Some("bbc.one".intern()), - group: "Live".intern(), - xtream_cluster: XtreamCluster::Live, - item_type: PlaylistItemType::Live, - ..Default::default() - }, - }], - xtream_cluster: XtreamCluster::Live, - }]; - let tv_guide = TVGuide::new(vec![PersistedEpgSource { - file_path: ics_path, - priority: 0, - logo_override: false, - kind: PersistedEpgSourceKind::Ics { - channel_id: "bbc.one".intern(), - channel_title: Some("BBC One".intern()), - match_names: vec![], - config: Box::new(IcsEpgSourceConfig::default()), - }, - }]); - let mut playlist = FetchedPlaylist { - input: &input, - source: MemoryPlaylistSource::new(groups).into_source(), - epg: Some(tv_guide), - }; - - let rewrite_epg = - build_mapping("rewrite", MappingStage::AfterEpg, r#"@epg_channel_id = "missing.epg""#); - let add_virtual = build_mapping("virtual", MappingStage::AfterEpg, r#"add_favourite("Echo")"#); - let mut target = build_target(vec![rewrite_epg, add_virtual], false); - target.options = Some(ConfigTargetOptions { clear_invalid_epg_ids: true, ..Default::default() }); - let mut stats = HashMap::from([( - Arc::clone(&input.name), - create_input_stat(1, 1, 0, input.input_type, &input.name, 0), - )]); - let mut errors = Vec::new(); - - let prepared = prepare_playlist_for_target( - &processing_context(), - std::slice::from_mut(&mut playlist), - &target, - &mut stats, - &mut errors, - ClusterFlags::empty(), - false, - ) - .await - .expect("target preparation"); - - assert!(errors.is_empty()); - assert!(!prepared.playlist.is_empty()); - assert!(prepared - .playlist - .iter() - .flat_map(|group| &group.channels) - .all(|channel| channel.header.epg_channel_id.is_none())); - assert_eq!(stats[&input.name].processed_stats.channel_count, 2); - }); - } - - fn live_item_for_epg(name: &str) -> PlaylistItem { - PlaylistItem { - header: PlaylistItemHeader { - name: name.intern(), - group: "Live".intern(), - xtream_cluster: XtreamCluster::Live, - item_type: PlaylistItemType::Live, - ..Default::default() - }, - } - } - - #[test] - fn after_epg_hook_runs_on_source_already_deduplicated_by_processing_pipe() { - let processing = build_mapping("processing", MappingStage::Processing, r#"@group = "PROCESSED""#); - let after_epg = build_mapping("after_epg", MappingStage::AfterEpg, r#"add_favourite("Echo")"#); - let target = build_target(vec![processing, after_epg], true); - - let input = ConfigInput::default(); - let channel = make_channel("Alpha"); - let mut fetched = - FetchedPlaylist { input: &input, source: memory_source(vec![channel.clone(), channel]), epg: None }; - let mut duplicates = HashSet::new(); - let (mut processed, _outcome) = - execute_pipe(&target, &get_processing_pipe(&target), &mut fetched, &mut duplicates, false, None) - .expect("processing pipe must run"); - assert_eq!(processed.get_channel_count(), 1, "processing pipe must remove the duplicate"); - - let groups = map_playlist_at_stage(&mut processed.source, &target, MappingStage::AfterEpg, None) - .expect("after_epg hook must run"); - - assert_eq!(groups.len(), 2); - assert_eq!(groups[0].title.as_ref(), "PROCESSED"); - assert_eq!(groups[0].channels.len(), 1); - assert_eq!(groups[1].title.as_ref(), "Echo"); - assert_eq!(groups[1].channels.len(), 1); - } - - #[test] - fn prepare_target_deduplicates_virtual_items_created_by_after_epg_mappings() { - let runtime = Runtime::new().expect("runtime"); - runtime.block_on(async { - let first = build_mapping("first", MappingStage::AfterEpg, r#"add_favourite("Echo")"#); - let second = build_mapping("second", MappingStage::AfterEpg, r#"add_favourite("Echo")"#); - let target = build_target(vec![first, second], true); - let input = ConfigInput { name: "input".intern(), ..Default::default() }; - let mut playlist = - FetchedPlaylist { input: &input, source: memory_source(vec![make_channel("Alpha")]), epg: None }; - let mut stats = HashMap::from([( - Arc::clone(&input.name), - create_input_stat(1, 1, 0, input.input_type, &input.name, 0), - )]); - let mut errors = Vec::new(); - - let prepared = prepare_playlist_for_target( - &processing_context(), - std::slice::from_mut(&mut playlist), - &target, - &mut stats, - &mut errors, - ClusterFlags::empty(), - false, - ) - .await - .expect("target preparation"); - - assert!(errors.is_empty()); - assert_eq!(prepared.playlist.iter().map(|group| group.channels.len()).sum::(), 3); - assert_eq!(stats[&input.name].processed_stats.channel_count, 3); - }); - } -} - -#[test] -fn current_target_finalization_order_merges_before_dedup_and_presentation() { - assert_eq!( - FINALIZATION_ORDER, - [ - FinalizationStage::Merge, - FinalizationStage::Deduplicate, - FinalizationStage::Sort, - FinalizationStage::AssignChannelNumbers, - FinalizationStage::AssignCounters, - ] - ); -} - -#[test] -fn curation_eligible_catalog_is_merged_and_deduplicated_before_matching() { - let mut target = ConfigTarget::from(&ConfigTargetDto::default()); - target.execution_plan.post_merge_content_dedup = Some(shared::model::DeduplicateConfig::default()); - let losing_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000051"); - let winning_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000052"); - let playlist = vec![ - PlaylistGroup { - id: 1, - title: "Movies".intern(), - channels: vec![catalog_test_item( - "Movie HD", - losing_uuid, - PlaylistItemType::Video, - XtreamCluster::Video, - None, - )], - xtream_cluster: XtreamCluster::Video, - }, - PlaylistGroup { - id: 2, - title: "movies".intern(), - channels: vec![catalog_test_item( - "Movie 4K", - winning_uuid, - PlaylistItemType::Video, - XtreamCluster::Video, - None, - )], - xtream_cluster: XtreamCluster::Video, - }, - ]; - let mut step = StepMeasure::new("test", |_, _| {}); - - let eligible = prepare_eligible_catalog(&target, playlist, &mut step); - - assert_eq!(eligible.len(), 1); - assert_eq!(eligible[0].channels.len(), 1); - assert_eq!(eligible[0].channels[0].header.uuid, winning_uuid); -} - -#[test] -fn persist_filter_can_select_a_generated_curation_group() { - let mut target = ConfigTarget::from(&ConfigTargetDto::default()); - target.filter.persist = Some(get_filter(r#"Group = "Trending""#, None).expect("persist filter")); - let mut playlist = vec![ - PlaylistGroup { - id: 1, - title: "Base".intern(), - channels: vec![PlaylistItem { - header: PlaylistItemHeader { group: "Base".intern(), ..Default::default() }, - }], - xtream_cluster: XtreamCluster::Video, - }, - PlaylistGroup { - id: 2, - title: "Trending".intern(), - channels: vec![PlaylistItem { - header: PlaylistItemHeader { group: "Trending".intern(), ..Default::default() }, - }], - xtream_cluster: XtreamCluster::Video, - }, - ]; - - apply_persist_filter(&target, &mut playlist); - - assert_eq!(playlist.len(), 1); - assert_eq!(playlist[0].title.as_ref(), "Trending"); -} - -#[test] -fn persist_filter_can_select_a_base_group() { - let mut target = ConfigTarget::from(&ConfigTargetDto::default()); - target.filter.persist = Some(get_filter(r#"Group = "Base""#, None).expect("persist filter")); - let mut playlist = vec![ - PlaylistGroup { - id: 1, - title: "Base".intern(), - channels: vec![PlaylistItem { - header: PlaylistItemHeader { group: "Base".intern(), ..Default::default() }, - }], - xtream_cluster: XtreamCluster::Video, - }, - PlaylistGroup { - id: 2, - title: "Curated".intern(), - channels: vec![PlaylistItem { - header: PlaylistItemHeader { group: "Curated".intern(), ..Default::default() }, - }], - xtream_cluster: XtreamCluster::Video, - }, - ]; - - apply_persist_filter(&target, &mut playlist); - - assert_eq!(playlist.len(), 1); - assert_eq!(playlist[0].title.as_ref(), "Base"); -} - -#[tokio::test] -async fn trakt_target_curation_is_a_noop_without_xtream_configuration() { - let target = ConfigTarget::from(&ConfigTargetDto::default()); - - let views = prepare_target_playlist_views(&reqwest::Client::new(), None, &target, Vec::new()).await; - - assert!(views.base.is_empty()); - assert!(views.xtream.is_none()); - assert_eq!(views.publication_plan, PlaylistPublicationPlan::Ordinary); -} - -#[tokio::test] -async fn unavailable_required_selector_continues_with_the_base_catalog() { - let target = ConfigTarget::from(&ConfigTargetDto { - name: "curation-failure".to_string(), - output: vec![TargetOutputDto::Xtream(XtreamTargetOutputDto { - trakt: Some(TraktConfigDto { - lists: vec![TraktListConfigDto { - user: "alice".to_string(), - list_slug: "watchlist".to_string(), - category_name: Some("Watchlist".to_string()), - create_xtream_category: true, - content_type: TraktContentType::Vod, - tmdb_only: true, - fuzzy_match_threshold: 100, - }], - ..TraktConfigDto::default() - }), - ..XtreamTargetOutputDto::default() - })], - ..ConfigTargetDto::default() - }); - let base = vec![PlaylistGroup { - id: 1, - title: "Movies".intern(), - channels: vec![catalog_test_item( - "Base movie", - UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000061"), - PlaylistItemType::Video, - XtreamCluster::Video, - None, - )], - xtream_cluster: XtreamCluster::Video, - }]; - - let views = prepare_target_playlist_views(&reqwest::Client::new(), None, &target, base).await; - assert_eq!(views.base.len(), 1); - assert_eq!(views.base[0].channels[0].header.title.as_ref(), "Base movie"); - assert!(views.xtream.is_none()); - assert_eq!(views.publication_plan, PlaylistPublicationPlan::Ordinary); -} - -mod curation_effect_gate { - use super::*; - use arc_swap::{ArcSwap, ArcSwapOption}; - use shared::model::{ConfigPaths, NoopSink}; - use std::{ - collections::{BTreeMap, BTreeSet}, - path::Path, - }; - use tempfile::tempdir; - use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpListener, - task::JoinHandle, - }; - use tuliprox_core::{ - model::{ApiProxyConfig, CustomStreamResponse, HdHomeRunConfig, MediaToolCapabilities, SourcesConfig}, - utils::FileLockManager, - }; - - pub(super) fn app_config(storage_dir: &Path) -> Arc { - Arc::new(AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config { - storage_dir: storage_dir.to_string_lossy().into_owned(), - ..Config::default() - })), - sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), - hdhomerun: Arc::new(ArcSwapOption::::default()), - api_proxy: Arc::new(ArcSwapOption::::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - }) - } - - pub(super) fn processing_context( - app_config: Arc, - playlist_state: Option>, - ) -> PlaylistProcessingContext { - PlaylistProcessingContext { - client: reqwest::Client::new(), - run_id: "curation-effect-gate-run".into(), - execution_order: PlaylistUpdateRunOrder::from(1), - config: app_config, - user_targets: Arc::new(ProcessTargets { - enabled: false, - inputs: Vec::new(), - targets: Vec::new(), - target_names: Vec::new(), - }), - events: NoopSink, - playlist_state, - disabled_headers: None, - processed_inputs: Arc::new(Mutex::new(HashSet::new())), - input_completions: Arc::new(Mutex::new(HashMap::new())), - input_locks: Arc::new(Mutex::new(HashMap::new())), - provider_manager: None, - metadata_manager: None, - pre_processed_inputs: None, - stalker_refresh_mode: StalkerRefreshMode::Complete, - partial_refresh: Arc::new(std::sync::atomic::AtomicBool::new(false)), - had_quality_rejections: Arc::new(std::sync::atomic::AtomicBool::new(false)), - input_refresh: None, - library_update_mode: LibraryUpdateMode::ExistingCatalog, - } - } - - fn curation_persist_options(publication_plan: PlaylistPublicationPlan) -> TargetPlaylistPersistOptions { - TargetPlaylistPersistOptions { publication_plan, ..TargetPlaylistPersistOptions::default() } - } - - pub(super) fn file_snapshot(root: &Path) -> BTreeMap> { - fn collect(root: &Path, path: &Path, snapshot: &mut BTreeMap>) { - let Ok(entries) = std::fs::read_dir(path) else { return }; - for entry in entries.flatten() { - let entry_path = entry.path(); - if entry_path.is_dir() { - collect(root, &entry_path, snapshot); - } else { - snapshot.insert( - entry_path.strip_prefix(root).expect("snapshot path under root").to_path_buf(), - std::fs::read(&entry_path).expect("snapshot file"), - ); - } - } - } - - let mut snapshot = BTreeMap::new(); - collect(root, root, &mut snapshot); - snapshot - } - - async fn empty_trakt_server() -> (String, JoinHandle<()>) { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind Trakt test server"); - let address = listener.local_addr().expect("Trakt test address"); - let server = tokio::spawn(async move { - let (mut stream, _) = listener.accept().await.expect("accept Trakt request"); - let mut request = Vec::new(); - loop { - let mut buffer = [0u8; 1024]; - let read = stream.read(&mut buffer).await.expect("read Trakt request"); - if read == 0 { - break; - } - request.extend_from_slice(&buffer[..read]); - if request.windows(4).any(|window| window == b"\r\n\r\n") { - break; - } - } - stream - .write_all(b"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 2\r\nconnection: close\r\n\r\n[]") - .await - .expect("write Trakt response"); - }); - (format!("http://{address}"), server) - } - - #[tokio::test] - async fn unavailable_curation_preserves_the_empty_mixed_output_guard() { - assert_curation_preserves_empty_mixed_output_guard(None).await; - } - - #[tokio::test] - async fn tmdb_and_mixed_failures_preserve_the_empty_mixed_output_guard() { - for policy in ["full", "curated"] { - for mixed in [false, true] { - let (url, server) = empty_trakt_server().await; - let mut value = serde_json::json!({"catalog_selection": policy, "tmdb": {"trending": [{"kind": "movie", "time_window": "week", "limit": 100, "category_name": "TMDB"}]}}); - if mixed { - value["trakt"] = serde_json::json!({"api": {"api_key": "test-client", "url": url}, "charts": [{"kind": "movies", "chart": "popular", "category_name": "Trakt"}]}); - } - let mut dto: shared::model::CurationConfigDto = serde_json::from_value(value).unwrap(); - dto.prepare(&[TargetOutputDto::Xtream(XtreamTargetOutputDto::default())]).unwrap(); - assert_curation_preserves_empty_mixed_output_guard(Some(dto)).await; - if mixed { - server.await.unwrap(); - } else { - server.abort(); - } - } - } - } - - async fn assert_curation_preserves_empty_mixed_output_guard(canonical: Option) { - let directory = tempdir().expect("tempdir"); - let app_config = app_config(directory.path()); - let playlist_state = Arc::new(PlaylistStorageState::new()); - let mut target = ConfigTarget::from(&ConfigTargetDto { - name: "curation-effect-gate".to_string(), - output: vec![ - TargetOutputDto::Xtream(XtreamTargetOutputDto { - trakt: Some(TraktConfigDto { - lists: vec![TraktListConfigDto { - user: "alice".to_string(), - list_slug: "watchlist".to_string(), - category_name: Some("Watchlist".to_string()), - create_xtream_category: true, - content_type: TraktContentType::Vod, - tmdb_only: true, - fuzzy_match_threshold: 100, - }], - ..TraktConfigDto::default() - }), - ..XtreamTargetOutputDto::default() - }), - TargetOutputDto::M3u(M3uTargetOutputDto { - filename: Some("curation-effect-gate.m3u".to_string()), - ..M3uTargetOutputDto::default() - }), - ], - watch: Some(vec![".*".to_string()]), - use_memory_cache: true, - ..ConfigTargetDto::default() - }); - if let Some(canonical) = canonical { - target.curation = Some(CurationConfig::from(&canonical)); - if let tuliprox_core::model::TargetOutput::Xtream(output) = &mut target.output[0] { - output.trakt = None; - } - } - let mut seeded = vec![PlaylistGroup { - id: 1, - title: "Movies".intern(), - channels: vec![catalog_test_item( - "Seeded movie", - UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000071"), - PlaylistItemType::Video, - XtreamCluster::Video, - None, - )], - xtream_cluster: XtreamCluster::Video, - }]; - let seeded_result = persist_playlist_views( - &app_config, - &mut seeded, - None, - None, - &target, - Some(&playlist_state), - curation_persist_options(PlaylistPublicationPlan::Ordinary), - ) - .await; - assert!(seeded_result.is_ok(), "seed persist failed: {seeded_result:?}"); - assert!(process_watch(&app_config, &NoopSink, &target, &seeded).await); - let before_files = file_snapshot(directory.path()); - let before_cache_len = playlist_state - .data - .read() - .await - .get(&target.name) - .and_then(|storage| storage.xtream.as_ref()) - .map_or(0, |storage| storage.vod.len()); - - let context = processing_context(Arc::clone(&app_config), Some(Arc::clone(&playlist_state))); - let prepared = PreparedTarget { - target, - playlist: Vec::new(), - epg: Vec::new(), - processing: PipelineStats::default(), - accepted_empty_clusters: ClusterFlags::Vod, - library_empty: tuliprox_repository::LibraryEmptyPublication::None, - }; - - let (result, errors) = finalize_prepared_target(Arc::new(context), prepared).await; - - assert!(result.is_err_and(|errors| errors.iter().any(|error| { - error.message().contains("non-Xtream outputs that cannot safely publish a fully empty forced result") - }))); - assert!(errors.is_empty()); - assert_eq!(file_snapshot(directory.path()), before_files); - let after_cache_len = playlist_state - .data - .read() - .await - .get("curation-effect-gate") - .and_then(|storage| storage.xtream.as_ref()) - .map_or(0, |storage| storage.vod.len()); - assert!(before_cache_len > 0); - assert_eq!(after_cache_len, before_cache_len); - } - - #[tokio::test] - async fn complete_empty_curation_publishes_empty_watch_group_state() { - let directory = tempdir().expect("tempdir"); - let app_config = app_config(directory.path()); - let (base_url, server) = empty_trakt_server().await; - let target_name = "curation-empty-watch"; - let target = ConfigTarget::from(&ConfigTargetDto { - name: target_name.to_string(), - output: vec![TargetOutputDto::Xtream(XtreamTargetOutputDto { - trakt: Some(TraktConfigDto { - catalog_selection: TraktCatalogSelection::Curated, - api: TraktApiConfigDto { - api_key: "test-client-id".to_string(), - version: "2".to_string(), - url: base_url, - user_agent: "tuliprox-test".to_string(), - }, - lists: vec![TraktListConfigDto { - user: "alice".to_string(), - list_slug: "watchlist".to_string(), - category_name: Some("Watchlist".to_string()), - create_xtream_category: true, - content_type: TraktContentType::Vod, - tmdb_only: true, - fuzzy_match_threshold: 100, - }], - ..TraktConfigDto::default() - }), - ..XtreamTargetOutputDto::default() - })], - watch: Some(vec![".*".to_string()]), - ..ConfigTargetDto::default() - }); - let mut seeded = vec![PlaylistGroup { - id: 1, - title: "Movies".intern(), - channels: vec![catalog_test_item( - "Seeded movie", - UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000072"), - PlaylistItemType::Video, - XtreamCluster::Video, - None, - )], - xtream_cluster: XtreamCluster::Video, - }]; - let seed_result = persist_playlist_views( - &app_config, - &mut seeded, - None, - None, - &target, - None, - curation_persist_options(PlaylistPublicationPlan::Ordinary), - ) - .await; - assert!(seed_result.is_ok(), "seed persist failed: {seed_result:?}"); - assert!(process_watch(&app_config, &NoopSink, &target, &seeded).await); - let watch_index = directory.path().join(format!("{target_name}.groups.bin")); - let before: BTreeSet> = - tuliprox_core::utils::binary_deserialize(&std::fs::read(&watch_index).expect("seeded watch index")) - .expect("decode seeded watch index"); - assert_eq!(before.len(), 1); - - let context = processing_context(Arc::clone(&app_config), None); - let prepared = PreparedTarget { - target, - playlist: seeded, - epg: Vec::new(), - processing: PipelineStats::default(), - accepted_empty_clusters: ClusterFlags::empty(), - library_empty: tuliprox_repository::LibraryEmptyPublication::None, - }; - let (result, errors) = finalize_prepared_target(Arc::new(context), prepared).await; - server.await.expect("Trakt server should finish"); - - assert!(result.is_ok(), "complete empty finalization failed: {result:?}"); - assert!(errors.is_empty()); - let after: BTreeSet> = - tuliprox_core::utils::binary_deserialize(&std::fs::read(watch_index).expect("empty watch index")) - .expect("decode empty watch index"); - assert!(after.is_empty()); - } -} - -#[test] -fn tmdb_complete_target_selection_preserves_live_and_keeps_xtream_aliases_out_of_normal_outputs() { - for has_xtream in [false, true] { - let mut value = serde_json::json!({"name": "discovery", "output": [{"type": "m3u"}], - "curation": {"catalog_selection": "curated", "tmdb": {"trending": [{"kind": "movie", "time_window": "week", "limit": 100, "create_xtream_category": has_xtream, "category_name": "TMDB"}]}}}); - if has_xtream { - value["output"].as_array_mut().unwrap().push(serde_json::json!({"type": "xtream"})); - } - let mut dto: ConfigTargetDto = serde_json::from_value(value).unwrap(); - dto.prepare(1, None, None).unwrap(); - let target = ConfigTarget::from(&dto); - let id = shared::utils::hash_string("selected"); - let live_id = shared::utils::hash_string("live"); - let playlist = vec![ - PlaylistGroup { - id: 1, - title: "Live".intern(), - xtream_cluster: XtreamCluster::Live, - channels: vec![catalog_test_item("Live", live_id, PlaylistItemType::Live, XtreamCluster::Live, None)], - }, - PlaylistGroup { - id: 2, - title: "Movies".intern(), - xtream_cluster: XtreamCluster::Video, - channels: vec![ - catalog_test_item("Selected", id, PlaylistItemType::Video, XtreamCluster::Video, None), - catalog_test_item( - "Other", - shared::utils::hash_string("other"), - PlaylistItemType::Video, - XtreamCluster::Video, - None, - ), - ], - }, - ]; - let config = target.effective_curation().unwrap(); - let outcome = CurationRunOutcome::Complete(complete_catalog_evaluation(vec![catalog_membership( - id, - CurationMediaKind::Movie, - 0, - )])); - let views = target::curation_playlist_views(&target, playlist.clone(), &config, outcome); - assert_eq!( - views.base.iter().flat_map(|g| &g.channels).map(|i| i.header.uuid).collect::>(), - [live_id, id] - ); - if has_xtream { - let groups = views.xtream.unwrap(); - let projected = groups.iter().find(|g| g.title.as_ref() == "TMDB").unwrap(); - assert_ne!(projected.channels[0].header.uuid, id); - } else { - assert!(views.xtream.is_none()); - } - let empty = CurationRunOutcome::Complete(complete_catalog_evaluation(Vec::new())); - let empty_views = target::curation_playlist_views(&target, playlist, &config, empty); - assert_eq!(empty_views.base.len(), 1); - assert_eq!(empty_views.base[0].xtream_cluster, XtreamCluster::Live); - assert_ne!(empty_views.publication_plan, PlaylistPublicationPlan::Ordinary); - } -} - -fn catalog_test_item( - title: &str, - uuid: UUIDType, - item_type: PlaylistItemType, - cluster: XtreamCluster, - parent_code: Option<&str>, -) -> PlaylistItem { - PlaylistItem { - header: PlaylistItemHeader { - id: title.intern(), - name: title.intern(), - title: title.intern(), - group: match cluster { - XtreamCluster::Live => "Live".intern(), - XtreamCluster::Video => "Movies".intern(), - XtreamCluster::Series => "Series".intern(), - }, - uuid, - item_type, - xtream_cluster: cluster, - parent_code: parent_code.unwrap_or_default().intern(), - ..PlaylistItemHeader::default() - }, - } -} - -fn complete_catalog_evaluation(memberships: Vec) -> CurationEvaluation { - CurationEvaluation { - selectors: vec![CurationSelectorSummary { - key: CurationSelectorKey(0), - reference_count: memberships.len(), - membership_count: memberships.len(), - }], - memberships, - } -} - -fn catalog_membership(uuid: UUIDType, media_kind: CurationMediaKind, order: usize) -> CurationMembership { - CurationMembership { - selector_key: CurationSelectorKey(0), - subject_uuid: uuid, - media_kind, - rank: Some(u32::try_from(order + 1).expect("test rank")), - title_tiebreak: format!("item-{order}"), - candidate_order: order, - } -} - -#[test] -fn catalog_selection_preserves_live_and_selected_series_children() { - let live_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000001"); - let selected_movie_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000002"); - let rejected_movie_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000003"); - let series_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000004"); - let episode_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000005"); - let rejected_series_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000006"); - let rejected_episode_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000007"); - let playlist = vec![ - PlaylistGroup { - id: 1, - title: "Live".intern(), - channels: vec![catalog_test_item( - "Live channel", - live_uuid, - PlaylistItemType::Live, - XtreamCluster::Live, - None, - )], - xtream_cluster: XtreamCluster::Live, - }, - PlaylistGroup { - id: 2, - title: "Movies".intern(), - channels: vec![ - catalog_test_item( - "Selected movie", - selected_movie_uuid, - PlaylistItemType::Video, - XtreamCluster::Video, - None, - ), - catalog_test_item( - "Rejected movie", - rejected_movie_uuid, - PlaylistItemType::Video, - XtreamCluster::Video, - None, - ), - ], - xtream_cluster: XtreamCluster::Video, - }, - PlaylistGroup { - id: 3, - title: "Series".intern(), - channels: vec![ - catalog_test_item( - "Selected series", - series_uuid, - PlaylistItemType::SeriesInfo, - XtreamCluster::Series, - None, - ), - catalog_test_item( - "Selected episode", - episode_uuid, - PlaylistItemType::Series, - XtreamCluster::Series, - Some(&series_uuid.to_string()), - ), - catalog_test_item( - "Rejected series", - rejected_series_uuid, - PlaylistItemType::SeriesInfo, - XtreamCluster::Series, - None, - ), - catalog_test_item( - "Rejected episode", - rejected_episode_uuid, - PlaylistItemType::Series, - XtreamCluster::Series, - Some(&rejected_series_uuid.to_string()), - ), - ], - xtream_cluster: XtreamCluster::Series, - }, - ]; - let evaluation = complete_catalog_evaluation(vec![ - catalog_membership(selected_movie_uuid, CurationMediaKind::Movie, 0), - catalog_membership(series_uuid, CurationMediaKind::Series, 1), - ]); - - let selected = select_target_catalog(playlist.clone(), &evaluation, true); - let titles = - selected.iter().flat_map(|group| &group.channels).map(|item| item.header.title.as_ref()).collect::>(); - - assert_eq!(titles, ["Live channel", "Selected movie", "Selected series", "Selected episode"]); - assert_eq!( - select_target_catalog(playlist.clone(), &evaluation, false).iter().flat_map(|group| &group.channels).count(), - 7 - ); - assert_eq!(selected[0].channels[0].header.uuid, live_uuid, "target-wide selection must not rewrite Live"); - - let remote_empty = select_target_catalog(playlist, &complete_catalog_evaluation(Vec::new()), true); - assert_eq!(remote_empty.len(), 1); - assert_eq!(remote_empty[0].xtream_cluster, XtreamCluster::Live); - assert_eq!(remote_empty[0].channels[0].header.uuid, live_uuid); -} - -#[test] -fn xtream_base_and_selector_category_projection_are_independent() { - let selected_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000011"); - let rejected_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000012"); - let playlist = vec![PlaylistGroup { - id: 1, - title: "Movies".intern(), - channels: vec![ - catalog_test_item("Selected", selected_uuid, PlaylistItemType::Video, XtreamCluster::Video, None), - catalog_test_item("Rejected", rejected_uuid, PlaylistItemType::Video, XtreamCluster::Video, None), - ], - xtream_cluster: XtreamCluster::Video, - }]; - let evaluation = complete_catalog_evaluation(vec![catalog_membership(selected_uuid, CurationMediaKind::Movie, 0)]); - let dto = TraktConfigDto { - enabled: true, - catalog_selection: TraktCatalogSelection::Curated, - include_xtream_base_categories: false, - api: TraktApiConfigDto::default(), - lists: vec![TraktListConfigDto { - user: "alice".to_string(), - list_slug: "watchlist".to_string(), - category_name: Some("Curated".to_string()), - create_xtream_category: true, - content_type: TraktContentType::Vod, - tmdb_only: true, - fuzzy_match_threshold: 100, - }], - charts: Vec::new(), - }; - let config = TraktConfig::from(&dto); - - let views = - build_curated_playlist_views(playlist.clone(), &evaluation, &CurationConfig::from(&config), false, true); - - assert_eq!(views.base.len(), 1); - assert_eq!(views.base[0].channels.len(), 1); - assert!( - target_watch_view(&views.base, views.xtream.as_deref()).iter().any(|group| group.title.as_ref() == "Curated"), - "configured Trakt watches observe the Xtream category appearance" - ); - let xtream = views.xtream.expect("complete curation has an Xtream view"); - assert_eq!(xtream.len(), 1); - assert_eq!(xtream[0].title.as_ref(), "Curated"); - assert_eq!(xtream[0].channels.len(), 1); - assert_ne!(xtream[0].channels[0].header.uuid, selected_uuid); - - let mut compatible_dto = dto; - compatible_dto.catalog_selection = TraktCatalogSelection::Full; - compatible_dto.include_xtream_base_categories = true; - compatible_dto.lists[0].create_xtream_category = false; - compatible_dto.lists[0].category_name = None; - let compatible = build_curated_playlist_views( - playlist, - &evaluation, - &CurationConfig::from(&TraktConfig::from(&compatible_dto)), - false, - true, - ); - assert_eq!(compatible.base[0].channels.len(), 2); - assert_eq!(compatible.xtream.expect("Xtream view").len(), 1, "selection-only selector creates no category"); -} - -#[test] -fn curation_policy_truth_table_covers_a_through_h() { - let selected_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000041"); - let rejected_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000042"); - let playlist = vec![PlaylistGroup { - id: 1, - title: "Movies".intern(), - channels: vec![ - catalog_test_item("Selected", selected_uuid, PlaylistItemType::Video, XtreamCluster::Video, None), - catalog_test_item("Rejected", rejected_uuid, PlaylistItemType::Video, XtreamCluster::Video, None), - ], - xtream_cluster: XtreamCluster::Video, - }]; - let evaluation = complete_catalog_evaluation(vec![catalog_membership(selected_uuid, CurationMediaKind::Movie, 0)]); - - let cases = [ - ("A", TraktCatalogSelection::Full, true, true), - ("B", TraktCatalogSelection::Curated, true, true), - ("C", TraktCatalogSelection::Curated, true, false), - ("D", TraktCatalogSelection::Curated, false, true), - ("E", TraktCatalogSelection::Full, true, false), - ("F", TraktCatalogSelection::Full, false, true), - ("G", TraktCatalogSelection::Full, false, false), - ("H", TraktCatalogSelection::Curated, false, false), - ]; - - for (case, catalog_selection, include_base, create_category) in cases { - let config = TraktConfig::from(&TraktConfigDto { - enabled: true, - catalog_selection, - include_xtream_base_categories: include_base, - api: TraktApiConfigDto::default(), - lists: vec![TraktListConfigDto { - user: "alice".to_string(), - list_slug: "watchlist".to_string(), - category_name: Some("Curated".to_string()), - create_xtream_category: create_category, - content_type: TraktContentType::Vod, - tmdb_only: true, - fuzzy_match_threshold: 100, - }], - charts: Vec::new(), - }); - - let views = - build_curated_playlist_views(playlist.clone(), &evaluation, &CurationConfig::from(&config), false, true); - let base_items = views.base.iter().flat_map(|group| &group.channels).collect::>(); - let expected_base_count = if catalog_selection == TraktCatalogSelection::Full { 2 } else { 1 }; - assert_eq!(base_items.len(), expected_base_count, "case {case} selected catalog"); - assert!(base_items.iter().any(|item| item.header.uuid == selected_uuid), "case {case} selected subject"); - - let xtream = views.xtream.expect("complete curation Xtream view"); - let base_groups = xtream.iter().filter(|group| group.title.as_ref() == "Movies").count(); - let category_groups = xtream.iter().filter(|group| group.title.as_ref() == "Curated").count(); - assert_eq!(base_groups, usize::from(include_base), "case {case} base appearance"); - assert_eq!(category_groups, usize::from(create_category), "case {case} category appearance"); - let expected_unfiltered_items = usize::from(include_base) * expected_base_count + usize::from(create_category); - assert_eq!( - xtream.iter().map(|group| group.channels.len()).sum::(), - expected_unfiltered_items, - "case {case} unfiltered Xtream catalog" - ); - if let Some(alias) = - xtream.iter().find(|group| group.title.as_ref() == "Curated").and_then(|group| group.channels.first()) - { - assert_ne!(alias.header.uuid, selected_uuid, "case {case} alias identity"); - } - } -} - -#[test] -fn large_catalog_projection_smoke_keeps_two_explicit_views_bounded() { - const CATALOG_SIZE: usize = 10_000; - const SELECTOR_COUNT: usize = 4; - const MEMBERSHIP_STRIDE: usize = 10; - - let uuid_for = |index: usize| { - let mut bytes = [0u8; 32]; - bytes[..8].copy_from_slice(&u64::try_from(index + 1).expect("test index").to_be_bytes()); - UUIDType(bytes) - }; - let playlist = vec![PlaylistGroup { - id: 1, - title: "Movies".intern(), - channels: (0..CATALOG_SIZE) - .map(|index| { - catalog_test_item( - &format!("Movie {index}"), - uuid_for(index), - PlaylistItemType::Video, - XtreamCluster::Video, - None, - ) - }) - .collect(), - xtream_cluster: XtreamCluster::Video, - }]; - let mut memberships = Vec::with_capacity(CATALOG_SIZE / MEMBERSHIP_STRIDE * SELECTOR_COUNT); - let mut selectors = Vec::with_capacity(SELECTOR_COUNT); - for selector in 0..SELECTOR_COUNT { - let key = CurationSelectorKey(selector); - let start = memberships.len(); - for index in (selector..CATALOG_SIZE).step_by(MEMBERSHIP_STRIDE) { - memberships.push(CurationMembership { - selector_key: key, - subject_uuid: uuid_for(index), - media_kind: CurationMediaKind::Movie, - rank: Some(u32::try_from(index).expect("test rank")), - title_tiebreak: format!("movie-{index}"), - candidate_order: index, - }); - } - selectors.push(CurationSelectorSummary { - key, - reference_count: memberships.len() - start, - membership_count: memberships.len() - start, - }); - } - let evaluation = CurationEvaluation { selectors, memberships }; - let config = TraktConfig::from(&TraktConfigDto { - lists: (0..SELECTOR_COUNT) - .map(|selector| TraktListConfigDto { - user: "alice".to_string(), - list_slug: format!("list-{selector}"), - category_name: Some(format!("Curated {selector}")), - create_xtream_category: true, - content_type: TraktContentType::Vod, - tmdb_only: true, - fuzzy_match_threshold: 100, - }) - .collect(), - ..TraktConfigDto::default() - }); - - let views = build_curated_playlist_views(playlist, &evaluation, &CurationConfig::from(&config), false, true); - - assert_eq!(views.base[0].channels.len(), CATALOG_SIZE); - let xtream = views.xtream.expect("Xtream view"); - assert_eq!(xtream.len(), SELECTOR_COUNT + 1); - assert_eq!(xtream.iter().map(|group| group.channels.len()).sum::(), CATALOG_SIZE + 4_000); -} +mod curation_effect_gate; #[cfg(test)] -mod quality_rejection_fallback { - include!("m3u_quality_tests.rs"); - include!("manual_update_integration_tests.rs"); - include!("input_status_tests.rs"); - include!("staged_completion_tests.rs"); - use super::*; - use arc_swap::{ArcSwap, ArcSwapOption}; - use shared::model::{ - provider_saturation::build_group_lookup, xtream_const::XTREAM_CLUSTER, ConfigInputOptionsDto, - ConfigInputUpdateQualityDto, ConfigPaths, EventMessage, EventSink, PlaylistUpdateState, ProcessingOrder, - SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, SeriesStreamProperties, UpdateQualityPolicy, - }; - use std::{ - io::{ErrorKind, Read, Write}, - net::{TcpListener, TcpStream}, - path::Path, - sync::{ - atomic::{AtomicBool, Ordering}, - Mutex as StdMutex, - }, - thread::{self, JoinHandle}, - time::{Duration, SystemTime, UNIX_EPOCH}, - }; - use tuliprox_core::{ - model::{ - ConfigSource, MediaToolCapabilities, SourcesConfig, StagedFilter, TargetExecutionPlan, TargetOutput, - XtreamTargetFlagsSet, XtreamTargetOutput, - }, - utils::FileLockManager, - }; - use tuliprox_repository::{ - count_input_xtream_cluster, get_input_storage_path, get_live_cat_collection_path, - get_series_cat_collection_path, get_vod_cat_collection_path, load_xtream_target_storage, persist_playlist, - xtream_get_item_for_stream_id, xtream_get_storage_path, TargetPlaylistPersistOptions, - }; - - struct TestXtreamServer { - base_url: String, - stop: Arc, - requests: Arc>>, - errors: Arc>>, - worker: Option>, - } - - impl TestXtreamServer { - fn start(candidate_counts: [usize; 3]) -> Self { - Self::start_with_responses(fixture_responses(candidate_counts)) - } - - fn start_with_responses(responses: HashMap) -> Self { - Self::start_with_statuses(responses, HashMap::new()) - } - - fn start_with_statuses( - responses: HashMap, - statuses: HashMap, - ) -> Self { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind Xtream fixture server"); - listener.set_nonblocking(true).expect("configure Xtream fixture listener"); - let address = listener.local_addr().expect("Xtream fixture address"); - let stop = Arc::new(AtomicBool::new(false)); - let requests = Arc::new(StdMutex::new(Vec::new())); - let errors = Arc::new(StdMutex::new(Vec::new())); - let worker_stop = Arc::clone(&stop); - let worker_requests = Arc::clone(&requests); - let worker_errors = Arc::clone(&errors); - let worker = thread::spawn(move || { - while !worker_stop.load(Ordering::Acquire) { - match listener.accept() { - Ok((stream, _)) => match serve_fixture_request(stream, &responses, &statuses) { - Ok(action) => worker_requests.lock().expect("request log lock").push(action), - Err(err) => worker_errors.lock().expect("server error lock").push(err.to_string()), - }, - Err(err) if err.kind() == ErrorKind::WouldBlock => thread::sleep(Duration::from_millis(1)), - Err(err) => { - worker_errors.lock().expect("server error lock").push(err.to_string()); - break; - } - } - } - }); - - Self { base_url: format!("http://{address}"), stop, requests, errors, worker: Some(worker) } - } - - fn finish(mut self) -> Vec { - self.stop_worker(); - let errors = self.errors.lock().expect("server error lock").clone(); - assert!(errors.is_empty(), "Xtream fixture server errors: {errors:?}"); - self.requests.lock().expect("request log lock").clone() - } - - fn stop_worker(&mut self) { - self.stop.store(true, Ordering::Release); - if let Some(worker) = self.worker.take() { - worker.join().expect("Xtream fixture server should stop cleanly"); - } - } - } - - impl Drop for TestXtreamServer { - fn drop(&mut self) { self.stop_worker(); } - } - - #[derive(Clone, Default)] - struct CollectSink(Arc>>); - - impl EventSink for CollectSink { - fn emit(&self, event: EventMessage) { self.0.lock().expect("event sink lock").push(event); } - } - - fn fixture_responses(candidate_counts: [usize; 3]) -> HashMap { - let mut responses = - HashMap::from([("login".to_string(), serde_json::json!({"user_info": {"status": "Active"}}).to_string())]); - for (cluster, count) in XTREAM_CLUSTER.into_iter().zip(candidate_counts) { - let (category_action, stream_action, category_id, id_field, id_base) = match cluster { - XtreamCluster::Live => ("get_live_categories", "get_live_streams", 1_u32, "stream_id", 1_000_u32), - XtreamCluster::Video => ("get_vod_categories", "get_vod_streams", 2, "stream_id", 2_000), - XtreamCluster::Series => ("get_series_categories", "get_series", 3, "series_id", 3_000), - }; - responses.insert( - category_action.to_string(), - serde_json::json!([{"category_id": category_id, "category_name": format!("candidate-{cluster}")}]) - .to_string(), - ); - let streams: Vec<_> = (0..count) - .map(|offset| { - serde_json::json!({ - "name": format!("candidate-{cluster}-{offset}"), - id_field: id_base + u32::try_from(offset).expect("fixture count fits u32"), - "category_id": category_id, - }) - }) - .collect(); - responses.insert(stream_action.to_string(), serde_json::Value::Array(streams).to_string()); - } - responses - } - - fn live_fixture_responses( - categories: &serde_json::Value, - streams: Vec, - ) -> HashMap { - HashMap::from([ - ("login".to_string(), serde_json::json!({"user_info": {"status": "Active"}}).to_string()), - ("get_live_categories".to_string(), categories.to_string()), - ("get_live_streams".to_string(), serde_json::Value::Array(streams).to_string()), - ]) - } - - fn live_fixture_stream(provider_id: u32, category_id: u32, name: &str) -> serde_json::Value { - serde_json::json!({ - "name": name, - "stream_id": provider_id, - "category_id": category_id, - }) - } - - fn serve_fixture_request( - mut stream: TcpStream, - responses: &HashMap, - statuses: &HashMap, - ) -> std::io::Result { - stream.set_nonblocking(false)?; - stream.set_read_timeout(Some(Duration::from_secs(1)))?; - let mut request = Vec::with_capacity(1_024); - let mut buffer = [0_u8; 1_024]; - loop { - let read = stream.read(&mut buffer)?; - if read == 0 { - break; - } - request.extend_from_slice(&buffer[..read]); - if request.windows(4).any(|window| window == b"\r\n\r\n") { - break; - } - } - let request = String::from_utf8_lossy(&request); - let action = request - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .and_then(|target| target.split_once("action=").map(|(_, value)| value)) - .map_or_else(|| "login".to_string(), |value| value.split('&').next().unwrap_or(value).to_string()); - let body = responses - .get(&action) - .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, format!("unexpected action {action}")))?; - let status = statuses.get(&action).copied().unwrap_or(reqwest::StatusCode::OK); - let response = format!( - "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - stream.write_all(response.as_bytes())?; - Ok(action) - } - - fn processing_context(storage_dir: &Path) -> PlaylistProcessingContext { - processing_context_with_events(storage_dir, shared::model::NoopSink) - } - - fn processing_context_with_events(storage_dir: &Path, events: E) -> PlaylistProcessingContext { - let paths = ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - }; - let config = AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config { - storage_dir: storage_dir.to_string_lossy().into_owned(), - disk_based_processing: false, - ..Config::default() - })), - sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), - hdhomerun: Arc::new(ArcSwapOption::default()), - api_proxy: Arc::new(ArcSwapOption::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(paths)), - custom_stream_response: Arc::new(ArcSwapOption::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - }; - PlaylistProcessingContext { - client: reqwest::Client::new(), - run_id: "xtream-quality-test-run".into(), - execution_order: PlaylistUpdateRunOrder::from(1), - config: Arc::new(config), - user_targets: Arc::new(ProcessTargets { - enabled: false, - inputs: Vec::new(), - targets: Vec::new(), - target_names: Vec::new(), - }), - events, - playlist_state: None, - disabled_headers: None, - processed_inputs: Arc::new(Mutex::new(HashSet::new())), - input_completions: Arc::new(Mutex::new(HashMap::new())), - input_locks: Arc::new(Mutex::new(HashMap::new())), - provider_manager: None, - metadata_manager: None, - pre_processed_inputs: None, - input_refresh: None, - library_update_mode: LibraryUpdateMode::ExistingCatalog, - stalker_refresh_mode: StalkerRefreshMode::Complete, - partial_refresh: Arc::new(AtomicBool::new(false)), - had_quality_rejections: Arc::new(AtomicBool::new(false)), - } - } - - #[test] - fn playlist_update_run_constructor_preserves_queued_identity_and_generates_unique_automatic_ids() { - let temp = tempfile::tempdir().expect("temp dir"); - let ctx = processing_context(temp.path()); - let queued_id = PlaylistUpdateRunId::from("queued-run"); - let queued = ProcessingRun::for_run( - queued_id.clone(), - ctx.client.clone(), - Arc::clone(&ctx.config), - Arc::clone(&ctx.user_targets), - shared::model::NoopSink, - ); - let automatic_a = ProcessingRun::new( - ctx.client.clone(), - Arc::clone(&ctx.config), - Arc::clone(&ctx.user_targets), - shared::model::NoopSink, - ); - let automatic_b = ProcessingRun::new(ctx.client, ctx.config, ctx.user_targets, shared::model::NoopSink); - - assert_eq!(queued.run_id, queued_id); - assert_ne!(automatic_a.run_id, automatic_b.run_id); - } - - #[tokio::test] - async fn playlist_update_run_manual_and_automatic_execution_order_is_generated_and_preserved() { - let temp = tempfile::tempdir().expect("temp dir"); - let events = CollectSink::default(); - let ctx = processing_context_with_events(temp.path(), events.clone()); - let manual_run_id = PlaylistUpdateRunId::from("manual-run"); - - exec_processing(ProcessingRun::for_run( - manual_run_id.clone(), - ctx.client.clone(), - Arc::clone(&ctx.config), - Arc::clone(&ctx.user_targets), - events.clone(), - )) - .await; - let automatic_run = ProcessingRun::new(ctx.client, ctx.config, ctx.user_targets, events.clone()); - let automatic_run_id = automatic_run.run_id.clone(); - exec_processing(automatic_run).await; - - let emitted = events.0.lock().expect("event sink lock"); - let terminal_for = |run_id: &PlaylistUpdateRunId| { - emitted.iter().find_map(|event| match event { - EventMessage::PlaylistUpdate(summary) if summary.run_id.as_ref() == Some(run_id) => Some(summary), - _ => None, - }) - }; - let manual_terminal = terminal_for(&manual_run_id).expect("manual terminal event"); - let automatic_terminal = terminal_for(&automatic_run_id).expect("automatic terminal event"); - let manual_order = manual_terminal.execution_order.expect("manual execution order"); - let automatic_order = automatic_terminal.execution_order.expect("automatic execution order"); - assert!(automatic_order > manual_order); - - let progress_for = |run_id: &PlaylistUpdateRunId| { - emitted - .iter() - .filter_map(|event| match event { - EventMessage::PlaylistUpdateProgress(progress) if progress.run_id.as_ref() == Some(run_id) => { - Some(progress) - } - _ => None, - }) - .collect::>() - }; - let manual_progress = progress_for(&manual_run_id); - let automatic_progress = progress_for(&automatic_run_id); - assert!(!manual_progress.is_empty()); - assert!(!automatic_progress.is_empty()); - assert!(manual_progress.iter().all(|progress| progress.execution_order == Some(manual_order))); - assert!(automatic_progress.iter().all(|progress| progress.execution_order == Some(automatic_order))); - } - - fn test_input( - base_url: &str, - update_quality: ConfigInputUpdateQualityDto, - cache_duration_seconds: u64, - skipped_clusters: &[XtreamCluster], - ) -> Arc { - let options = ConfigInputOptionsDto { - skip_live: skipped_clusters.contains(&XtreamCluster::Live), - skip_vod: skipped_clusters.contains(&XtreamCluster::Video), - skip_series: skipped_clusters.contains(&XtreamCluster::Series), - update_quality, - ..ConfigInputOptionsDto::default() - }; - Arc::new(ConfigInput { - id: 1, - name: "quality-provider".intern(), - input_type: InputType::Xtream, - url: base_url.to_string(), - username: Some("user".to_string()), - password: Some("password".to_string()), - enabled: true, - options: Some(ConfigInputOptions::from(&options)), - cache_duration_seconds, - ..ConfigInput::default() - }) - } - - fn baseline_group(cluster: XtreamCluster, category_id: u32, title: &str, first_item_id: u32) -> PlaylistGroup { - baseline_group_with_count(cluster, category_id, title, first_item_id, 2) - } - - fn baseline_group_with_count( - cluster: XtreamCluster, - category_id: u32, - title: &str, - first_item_id: u32, - count: usize, - ) -> PlaylistGroup { - let title = title.intern(); - let item_type = PlaylistItemType::from(cluster); - let channels = (0..count) - .map(|offset| { - let offset = u32::try_from(offset).expect("baseline fixture count fits u32"); - let item_id = (first_item_id + offset).to_string().intern(); - let mut header = PlaylistItemHeader { - id: Arc::clone(&item_id), - input_stream_id: item_id, - name: format!("{title}-{offset}").intern(), - title: format!("{title}-{offset}").intern(), - group: Arc::clone(&title), - url: format!("http://old.example/{cluster}/{first_item_id}/{offset}").intern(), - input_name: "quality-provider".intern(), - item_type, - xtream_cluster: cluster, - category_id, - ..PlaylistItemHeader::default() - }; - header.gen_uuid(); - PlaylistItem { header } - }) - .collect(); - PlaylistGroup { id: category_id, title, channels, xtream_cluster: cluster } - } - - async fn seed_live_baseline( - ctx: &PlaylistProcessingContext, - input: &ConfigInput, - first_item_id: u32, - ) { - let baseline = baseline_group_with_count(XtreamCluster::Live, 1, "old-live", first_item_id, 100); - let (persisted, error) = tuliprox_repository::persist_input_playlist(&ctx.config, input, vec![baseline]).await; - assert!(error.is_none(), "Live baseline persistence failed: {error:?}"); - assert_eq!(persisted.iter().map(|group| group.channels.len()).sum::(), 100); - } - - fn baseline_groups() -> Vec { - vec![ - baseline_group(XtreamCluster::Live, 1, "old-live", 100), - baseline_group(XtreamCluster::Video, 2, "old-vod", 200), - baseline_group(XtreamCluster::Series, 3, "old-series", 300), - ] - } - - async fn seed_baseline(ctx: &PlaylistProcessingContext, input: &ConfigInput) { - let (persisted, error) = - tuliprox_repository::persist_input_playlist(&ctx.config, input, baseline_groups()).await; - assert!(error.is_none(), "baseline persistence failed: {error:?}"); - assert_baseline(&persisted); - } - - fn assert_baseline(groups: &[PlaylistGroup]) { - for (cluster, category_id, title, item_ids) in [ - (XtreamCluster::Live, 1, "old-live", ["100", "101"]), - (XtreamCluster::Video, 2, "old-vod", ["200", "201"]), - (XtreamCluster::Series, 3, "old-series", ["300", "301"]), - ] { - let group = groups - .iter() - .find(|group| group.xtream_cluster == cluster && group.id == category_id) - .unwrap_or_else(|| panic!("missing persisted {cluster} group")); - assert_eq!(group.title.as_ref(), title); - assert_eq!(group.channels.len(), 2); - assert!(group.channels.iter().all(|item| item.header.xtream_cluster == cluster)); - assert_eq!(group.channels[0].header.id.as_ref(), item_ids[0]); - assert_eq!(group.channels[1].header.id.as_ref(), item_ids[1]); - } - } - - async fn input_storage_path(ctx: &PlaylistProcessingContext, input: &ConfigInput) -> PathBuf { - let storage_dir = ctx.config.config.load().storage_dir.clone(); - get_input_storage_path(&input.name, &storage_dir).await.expect("input storage path") - } - - async fn seed_valid_cluster_cache(ctx: &PlaylistProcessingContext, input: &ConfigInput) -> u64 { - let storage_path = input_storage_path(ctx, input).await; - let now = SystemTime::now().duration_since(UNIX_EPOCH).expect("current time").as_secs(); - let mut status = input_cache::InputStatus::default(); - for cluster in XTREAM_CLUSTER { - status.clusters.insert( - cluster.as_ref().to_string(), - input_cache::ClusterStatus { status: input_cache::ClusterState::Ok, timestamp: now, last_update: None }, - ); - } - input_cache::save_input_status(&storage_path, &status); - now - } - - fn set_refresh_policy( - ctx: &mut PlaylistProcessingContext, - input: &ConfigInput, - policy: InputRefreshPolicy, - ) { - ctx.input_refresh = Some(InputRefreshOverride { input_id: input.id, policy }); - } - - fn xtream_target(use_memory_cache: bool) -> Arc { - Arc::new(ConfigTarget { - curation: None, - id: 1, - enabled: true, - name: "quality-target".to_string(), - options: None, - sort: None, - filter: StagedFilter::default(), - output: vec![TargetOutput::Xtream(XtreamTargetOutput { - flags: XtreamTargetFlagsSet::new(), - trakt: None, - filter: None, - })], - rename: None, - mapping_ids: None, - mapping: Arc::new(ArcSwapOption::new(None)), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: TargetExecutionPlan::default(), - watch: None, - use_memory_cache, - }) - } - - fn install_source( - ctx: &PlaylistProcessingContext, - input: &Arc, - target: &Arc, - ) { - let inputs = vec![Arc::clone(input)]; - ctx.config.sources.store(Arc::new(SourcesConfig { - batch_files: Vec::new(), - templates: None, - provider: Vec::new(), - group_lookup: build_group_lookup(&inputs), - inputs, - sources: vec![ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![Arc::clone(target)] }], - })); - } - - async fn seed_xtream_target( - ctx: &PlaylistProcessingContext, - target: &ConfigTarget, - playlist_state: &Arc, - ) -> u32 { - let mut groups = baseline_groups(); - persist_playlist( - &ctx.config, - &mut groups, - None, - target, - Some(playlist_state), - TargetPlaylistPersistOptions::default(), - ) - .await - .expect("target baseline should persist"); - groups - .iter() - .find(|group| group.xtream_cluster == XtreamCluster::Video) - .and_then(|group| group.channels.first()) - .map(|item| item.header.virtual_id.get()) - .expect("target baseline VOD id") - } - - async fn assert_target_counts(app_config: &Arc, target: &ConfigTarget, expected: [usize; 3]) { - let storage = load_xtream_target_storage(app_config, target).await.expect("target storage should load"); - assert_eq!([storage.live.len(), storage.vod.len(), storage.series.len()], expected); - } - - async fn assert_empty_target_categories( - app_config: &Arc, - target: &ConfigTarget, - clusters: &[XtreamCluster], - ) { - let storage_path = { - let config = app_config.config.load(); - xtream_get_storage_path(&config, &target.name).expect("Xtream target storage path") - }; - for cluster in clusters { - let path = match cluster { - XtreamCluster::Live => get_live_cat_collection_path(&storage_path), - XtreamCluster::Video => get_vod_cat_collection_path(&storage_path), - XtreamCluster::Series => get_series_cat_collection_path(&storage_path), - }; - let categories: Vec = serde_json::from_slice( - &tokio::fs::read(&path).await.unwrap_or_else(|error| panic!("read {}: {error}", path.display())), - ) - .unwrap_or_else(|error| panic!("parse {}: {error}", path.display())); - assert!(categories.is_empty(), "{} should contain no categories", path.display()); - } - } - - #[test] - fn refresh_policy_override_is_scoped_to_the_selected_input_id() { - let temp = tempfile::tempdir().expect("temporary storage"); - let mut ctx = processing_context(temp.path()); - let selected = ConfigInput { id: 17, ..ConfigInput::default() }; - set_refresh_policy(&mut ctx, &selected, InputRefreshPolicy::FORCE); - - assert_eq!(ctx.refresh_policy(17), InputRefreshPolicy::FORCE); - assert_eq!(ctx.refresh_policy(18), InputRefreshPolicy::NORMAL); - } - - fn assert_requested_actions(mut actual: Vec, expected: &[&str]) { - actual.sort(); - let mut expected: Vec<_> = expected.iter().map(ToString::to_string).collect(); - expected.sort(); - assert_eq!(actual, expected); - } - - #[tokio::test] - async fn mixed_in_memory_update_keeps_rejected_vod_ready_and_marks_the_run_partial() { - let temp = tempfile::tempdir().expect("temporary storage"); - let server = TestXtreamServer::start([2, 1, 2]); - let events = CollectSink::default(); - let ctx = processing_context_with_events(temp.path(), events.clone()); - let input = - test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 0, &[]); - seed_baseline(&ctx, &input).await; - - let mut result = process_input_job_inner(0, &ctx, &input).await; - let requests = server.finish(); - - assert_eq!(result.state, InputJobState::Ready); - assert!(result.errors.is_empty(), "quality rejection must not become an error: {:?}", result.errors); - assert!(ctx.had_quality_rejections.load(Ordering::Acquire)); - assert_eq!( - PlaylistRunSignals { has_quality_rejections: true, ..PlaylistRunSignals::default() }.state(), - PlaylistUpdateState::Partial - ); - - let groups = result.source.take().expect("ready input source").take_groups(); - let live = - groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Live).expect("accepted Live group"); - let vod = groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Video).expect("retained VOD group"); - let series = - groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Series).expect("accepted Series group"); - assert_eq!(live.title.as_ref(), "candidate-live"); - assert_eq!(live.channels.len(), 2); - assert_eq!(vod.title.as_ref(), "old-vod"); - assert_eq!(vod.channels.len(), 2); - assert_eq!(series.title.as_ref(), "candidate-series"); - assert_eq!(series.channels.len(), 2); - - let storage_path = input_storage_path(&ctx, &input).await; - let status = input_cache::load_input_status(&storage_path); - assert_eq!( - status.clusters.get(XtreamCluster::Live.as_ref()).map(|entry| &entry.status), - Some(&input_cache::ClusterState::Ok) - ); - assert_eq!( - status.clusters.get(XtreamCluster::Video.as_ref()).map(|entry| &entry.status), - Some(&input_cache::ClusterState::Failed) - ); - assert_eq!( - status.clusters.get(XtreamCluster::Series.as_ref()).map(|entry| &entry.status), - Some(&input_cache::ClusterState::Ok) - ); - - let emitted = events.0.lock().expect("event sink lock"); - let rejection_events = emitted - .iter() - .filter_map(|event| match event { - EventMessage::PlaylistUpdateProgress(progress) - if progress.message.contains("cluster 'vod' rejected") => - { - Some(progress) - } - _ => None, - }) - .collect::>(); - assert_eq!(rejection_events.len(), 1); - assert_eq!(rejection_events[0].target, "quality-provider"); - assert_eq!( - rejection_events[0].message, - "Input 'quality-provider' cluster 'vod' rejected: current=2 candidate=1 threshold=100 quality=50; retaining previous cluster" - ); - assert_requested_actions( - requests, - &[ - "login", - "get_live_categories", - "get_live_streams", - "get_vod_categories", - "get_vod_streams", - "get_series_categories", - "get_series", - ], - ); - } - - #[tokio::test] - async fn coderabbit_stalker_global_failures_mark_only_requested_non_skipped_clusters() { - use crate::processor::stalker::{download_stalker_playlist, StalkerCluster}; - - #[derive(Clone, Copy, Debug)] - enum FailurePoint { - Portal, - Client, - Storage, - Handshake, - } - - for failure in [FailurePoint::Portal, FailurePoint::Client, FailurePoint::Storage, FailurePoint::Handshake] { - for requested in [None, Some(vec![StalkerCluster::Live, StalkerCluster::Series])] { - let temp = tempfile::tempdir().unwrap(); - let ctx = processing_context(temp.path()); - let server = TestXtreamServer::start_with_responses(HashMap::from([("handshake".into(), "{}".into())])); - let mut input = - (*test_input(&server.base_url, ConfigInputUpdateQualityDto::default(), 0, &[XtreamCluster::Live])) - .clone(); - input.input_type = InputType::Stalker; - input.stalker = Some(tuliprox_core::model::StalkerInputConfig::default()); - match failure { - FailurePoint::Portal => input.url.clear(), - FailurePoint::Client => input.url = "://invalid-url".into(), - FailurePoint::Storage => { - let path = input_storage_path(&ctx, &input).await; - tokio::fs::write(path.join("stalker"), b"not a directory").await.unwrap(); - } - FailurePoint::Handshake => {} - } - let fetch = download_stalker_playlist( - &ctx.config, - &ctx.client, - &input, - requested.as_deref(), - StalkerRefreshMode::Complete, - true, - UpdateQualityPolicy::Enforce, - ) - .await; - let expected = if requested.is_some() { - vec![XtreamCluster::Series] - } else { - vec![XtreamCluster::Video, XtreamCluster::Series] - }; - assert_eq!(fetch.failed_clusters, expected, "{failure:?}"); - assert_eq!(fetch.errors.len(), 1, "{failure:?}"); - assert!(!fetch.is_ok()); - assert!(fetch.quality_acceptances.is_empty() && fetch.quality_rejections.is_empty()); - assert!(fetch.force_updates.is_empty()); - let requests = server.finish(); - match failure { - FailurePoint::Handshake => assert!(!requests.is_empty()), - FailurePoint::Portal | FailurePoint::Client | FailurePoint::Storage => assert!(requests.is_empty()), - } - } - } - } - - #[tokio::test] - async fn coderabbit_xtream_http_failure_preserves_successful_disk_clusters() { - for quality in [UpdateQualityPolicy::Enforce, UpdateQualityPolicy::Bypass] { - let temp = tempfile::tempdir().unwrap(); - let server = TestXtreamServer::start_with_statuses( - fixture_responses([2, 2, 2]), - HashMap::from([("get_vod_streams".to_owned(), reqwest::StatusCode::BAD_REQUEST)]), - ); - let ctx = processing_context(temp.path()); - let mut config = (**ctx.config.config.load()).clone(); - config.disk_based_processing = true; - ctx.config.config.store(Arc::new(config)); - let input = - test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 0, &[]); - seed_baseline(&ctx, &input).await; - - let fetch = tuliprox_iptv::xtream::download_xtream_playlist( - &ctx.config, - &ctx.client, - &ctx.events, - &input, - None, - quality, - ) - .await; - assert_eq!(fetch.failed_clusters, vec![XtreamCluster::Video]); - assert!(!fetch.is_ok(), "the failed cluster must remain a technical failure"); - assert!(fetch.persisted); - assert!(fetch.quality_rejections.is_empty()); - let accepted: Vec<_> = match quality { - UpdateQualityPolicy::Enforce => fetch.quality_acceptances.iter().map(|value| value.cluster).collect(), - UpdateQualityPolicy::Bypass => fetch.force_updates.iter().map(|value| value.cluster).collect(), - }; - assert_eq!(accepted, vec![XtreamCluster::Live, XtreamCluster::Series]); - - let storage = input_storage_path(&ctx, &input).await; - let groups = - tuliprox_repository::load_input_xtream_playlist(&ctx.config, &storage, &XTREAM_CLUSTER).await.unwrap(); - for (cluster, title) in [ - (XtreamCluster::Live, "candidate-live"), - (XtreamCluster::Video, "old-vod"), - (XtreamCluster::Series, "candidate-series"), - ] { - let group = groups.iter().find(|group| group.xtream_cluster == cluster).unwrap(); - assert_eq!(group.title.as_ref(), title); - assert_eq!(group.channels.len(), 2); - } - assert_eq!(server.finish().len(), 7); - } - } - - #[tokio::test] - async fn pipeline_transparency_shows_failure_survives_xtream_fetch_and_reload() { - for disk in [false, true] { - let temp = tempfile::tempdir().unwrap(); - let mut responses = fixture_responses([2, 2, 2]); - responses.insert("get_series".to_owned(), "{".to_owned()); - let server = TestXtreamServer::start_with_responses(responses); - let mut ctx = processing_context(temp.path()); - let mut config = ctx.config.config.load().as_ref().clone(); - config.disk_based_processing = disk; - ctx.config.config.store(Arc::new(config)); - let input = - test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 0 }, 0, &[]); - seed_baseline(&ctx, &input).await; - set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); - let mut result = download_input(&ctx, &input, false).await; - assert_eq!(result.update_state(), PlaylistUpdateState::Failure); - assert!(!result.errors.is_empty()); - let telemetry = result.input_telemetry.as_ref().unwrap(); - for cluster in &telemetry.clusters { - if cluster.cluster == XtreamCluster::Series { - assert_eq!(cluster.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Failed)); - assert_eq!(cluster.decision, None); - assert_eq!(cluster.quality, None); - } else { - assert_eq!(cluster.technical_state, None, "do not attribute Shows failure to a sibling"); - assert_eq!(cluster.decision, Some(PlaylistUpdateClusterDecision::Accepted)); - } - assert_eq!(cluster.active_count, None, "failed batch does not prove final activation"); - } - let storage = input_storage_path(&ctx, &input).await; - let saved = input_cache::load_input_status(&storage); - let shows = saved.clusters["series"].last_update.unwrap(); - assert_eq!(shows.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Failed)); - assert_eq!(shows.quality, None); - assert_eq!(shows.quality_guard_threshold, Some(0)); - assert_eq!(shows.policy, Some(InputRefreshPolicy::FORCE)); - assert!(server.finish().iter().any(|action| action == "get_series")); - } - } - - #[tokio::test] - async fn pipeline_transparency_normal_policy_reuses_a_valid_cluster_cache() { - let temp = tempfile::tempdir().expect("temporary storage"); - let server = TestXtreamServer::start([2, 2, 2]); - let mut ctx = processing_context(temp.path()); - let input = - test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 3_600, &[]); - seed_baseline(&ctx, &input).await; - seed_valid_cluster_cache(&ctx, &input).await; - set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::NORMAL); - - let mut result = download_input(&ctx, &input, false).await; - let requests = server.finish(); - - assert!(result.errors.is_empty()); - assert!(result.storage_error.is_none()); - assert!(result.quality_rejections.is_empty()); - assert_eq!(result.job_state(), InputJobState::Ready); - assert_baseline(&result.source.take_groups()); - assert!(requests.is_empty(), "valid cache should avoid provider requests: {requests:?}"); - let telemetry = result.input_telemetry.as_ref().expect("real cache acquisition telemetry"); - assert_eq!(telemetry.refresh_policy, InputRefreshPolicy::NORMAL); - assert!(telemetry - .clusters - .iter() - .all(|cluster| { cluster.requested && cluster.source == Some(PlaylistUpdateDataSource::Cache) })); - } - - async fn assert_provider_acquisition_survives_in_run_reuse(policy: InputRefreshPolicy) { - let temp = tempfile::tempdir().expect("temporary storage"); - let server = TestXtreamServer::start([2, 2, 2]); - let mut ctx = processing_context(temp.path()); - let input = test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 95, vod: 95, series: 95 }, 0, &[]); - seed_baseline(&ctx, &input).await; - set_refresh_policy(&mut ctx, &input, policy); - - let mut first = download_input(&ctx, &input, false).await; - - assert!(first.errors.is_empty(), "first provider acquisition failed: {:?}", first.errors); - assert!(first.storage_error.is_none()); - assert_eq!(first.source.get_channel_count(), 6); - let first_telemetry = first.input_telemetry.as_ref().expect("provider acquisition telemetry"); - assert_eq!(first_telemetry.refresh_policy, policy); - assert!(first_telemetry - .clusters - .iter() - .all(|cluster| { cluster.requested && cluster.source == Some(PlaylistUpdateDataSource::Provider) })); - - let storage_path = input_storage_path(&ctx, &input).await; - let status_after_acquisition = input_cache::load_input_status(&storage_path); - assert!(XTREAM_CLUSTER.iter().all(|cluster| { - status_after_acquisition.clusters[cluster.as_ref()].last_update.as_ref().is_some_and(|snapshot| { - snapshot.policy == Some(policy) && snapshot.source == Some(PlaylistUpdateDataSource::Provider) - }) - })); - - // A second source/target in the same run reuses the persisted input. It is not a - // second acquisition and therefore must not emit replacement cache telemetry. - let mut reused = download_input(&ctx, &input, false).await; - - assert!(reused.errors.is_empty(), "in-run reuse failed: {:?}", reused.errors); - assert!(reused.storage_error.is_none()); - assert_eq!(reused.source.get_channel_count(), 6); - assert_eq!(reused.input_telemetry, None); - assert_eq!(input_cache::load_input_status(&storage_path), status_after_acquisition); - assert_requested_actions( - server.finish(), - &[ - "login", - "get_live_categories", - "get_live_streams", - "get_vod_categories", - "get_vod_streams", - "get_series_categories", - "get_series", - ], - ); - } - - #[tokio::test] - async fn in_run_reuse_after_force_keeps_provider_acquisition_and_snapshot() { - assert_provider_acquisition_survives_in_run_reuse(InputRefreshPolicy::FORCE).await; - } - - #[tokio::test] - async fn in_run_reuse_after_refresh_keeps_provider_acquisition_and_snapshot() { - assert_provider_acquisition_survives_in_run_reuse(InputRefreshPolicy::REFRESH).await; - } - - #[tokio::test] - async fn in_run_reuse_after_normal_provider_fetch_keeps_provider_acquisition_and_snapshot() { - assert_provider_acquisition_survives_in_run_reuse(InputRefreshPolicy::NORMAL).await; - } - - #[tokio::test] - async fn in_run_reuse_for_parallel_sources_emits_exactly_one_provider_acquisition() { - let temp = tempfile::tempdir().expect("temporary storage"); - let server = TestXtreamServer::start([2, 2, 2]); - let mut ctx = processing_context(temp.path()); - let input = test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 95, vod: 95, series: 95 }, 0, &[]); - seed_baseline(&ctx, &input).await; - set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); - - let (mut first_source, mut second_source) = - tokio::join!(download_input(&ctx, &input, false), download_input(&ctx, &input, false)); - - assert!(first_source.errors.is_empty(), "first source failed: {:?}", first_source.errors); - assert!(second_source.errors.is_empty(), "second source failed: {:?}", second_source.errors); - assert_eq!(first_source.source.get_channel_count(), 6); - assert_eq!(second_source.source.get_channel_count(), 6); - let acquisition_telemetry = [first_source.input_telemetry.as_ref(), second_source.input_telemetry.as_ref()]; - assert_eq!(acquisition_telemetry.iter().filter(|telemetry| telemetry.is_some()).count(), 1); - let acquisition = acquisition_telemetry.into_iter().flatten().next().expect("one provider acquisition"); - assert_eq!(acquisition.refresh_policy, InputRefreshPolicy::FORCE); - assert!(acquisition - .clusters - .iter() - .all(|cluster| { cluster.requested && cluster.source == Some(PlaylistUpdateDataSource::Provider) })); - assert_requested_actions( - server.finish(), - &[ - "login", - "get_live_categories", - "get_live_streams", - "get_vod_categories", - "get_vod_streams", - "get_series_categories", - "get_series", - ], - ); - } - - #[tokio::test] - async fn refresh_policy_bypasses_valid_cache_and_keeps_quality_enforced() { - let temp = tempfile::tempdir().expect("temporary storage"); - let server = TestXtreamServer::start([2, 1, 2]); - let mut ctx = processing_context(temp.path()); - let input = - test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 3_600, &[]); - seed_baseline(&ctx, &input).await; - seed_valid_cluster_cache(&ctx, &input).await; - set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::REFRESH); - - let mut result = download_input(&ctx, &input, false).await; - let requests = server.finish(); - - assert!(result.errors.is_empty()); - assert!(result.storage_error.is_none()); - assert_eq!(result.job_state(), InputJobState::Ready); - assert_eq!(result.quality_rejections.len(), 1); - assert_eq!(result.quality_rejections[0].cluster, XtreamCluster::Video); - let groups = result.source.take_groups(); - assert_eq!( - groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Video).map(|group| group.title.as_ref()), - Some("old-vod") - ); - assert_requested_actions( - requests, - &[ - "login", - "get_live_categories", - "get_live_streams", - "get_vod_categories", - "get_vod_streams", - "get_series_categories", - "get_series", - ], - ); - } - - #[tokio::test] - async fn force_policy_bypasses_valid_cache_and_quality_and_accepts_an_empty_cluster() { - let temp = tempfile::tempdir().expect("temporary storage"); - let server = TestXtreamServer::start([2, 0, 1]); - let events = CollectSink::default(); - let mut ctx = processing_context_with_events(temp.path(), events.clone()); - let input = - test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 95, series: 100 }, 3_600, &[]); - seed_baseline(&ctx, &input).await; - seed_valid_cluster_cache(&ctx, &input).await; - set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); - - let mut result = process_input_job_inner(0, &ctx, &input).await; - let requests = server.finish(); - - assert!(result.errors.is_empty(), "forced update failed: {:?}", result.errors); - assert_eq!(result.state, InputJobState::Ready); - let groups = result.source.take().expect("forced input source").take_groups(); - assert_eq!( - groups - .iter() - .filter(|group| group.xtream_cluster == XtreamCluster::Live) - .map(|group| group.channels.len()) - .sum::(), - 2 - ); - assert!(groups.iter().all(|group| group.xtream_cluster != XtreamCluster::Video)); - assert_eq!( - groups - .iter() - .filter(|group| group.xtream_cluster == XtreamCluster::Series) - .map(|group| group.channels.len()) - .sum::(), - 1 - ); - assert_eq!( - PlaylistRunSignals { - has_error: !result.errors.is_empty(), - has_pending_stalker_refresh: ctx.partial_refresh.load(Ordering::Acquire), - has_quality_rejections: ctx.had_quality_rejections.load(Ordering::Acquire), - } - .state(), - PlaylistUpdateState::Success - ); - - let storage_path = input_storage_path(&ctx, &input).await; - assert_eq!( - count_input_xtream_cluster(&ctx.config, &input, XtreamCluster::Video) - .await - .expect("forced empty VOD should remain countable"), - Some(0) - ); - let status = input_cache::load_input_status(&storage_path); - assert!(XTREAM_CLUSTER.iter().all(|cluster| { - status.clusters.get(cluster.as_ref()).map(|entry| &entry.status) == Some(&input_cache::ClusterState::Ok) - })); - let messages = events - .0 - .lock() - .expect("event sink lock") - .iter() - .filter_map(|event| match event { - EventMessage::PlaylistUpdateProgress(progress) => Some(progress.message.clone()), - _ => None, - }) - .collect::>(); - assert!(messages.iter().any(|message| { - message - == "Input 'quality-provider': forced update requested; cache and update-quality bypassed for live,vod,series" - })); - assert!(messages.iter().any(|message| { - message == "Input 'quality-provider' cluster 'vod' force-published: candidate=0 configured_threshold=95" - })); - assert_requested_actions( - requests, - &[ - "login", - "get_live_categories", - "get_live_streams", - "get_vod_categories", - "get_vod_streams", - "get_series_categories", - "get_series", - ], - ); - } - - #[tokio::test] - async fn force_empty_vod_is_published_to_xtream_target_and_memory_cache() { - let temp = tempfile::tempdir().expect("temporary storage"); - let server = TestXtreamServer::start([2, 0, 1]); - let mut ctx = processing_context(temp.path()); - let input = - test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 95, series: 100 }, 0, &[]); - let target = xtream_target(true); - let playlist_state = Arc::new(PlaylistStorageState::new()); - ctx.playlist_state = Some(Arc::clone(&playlist_state)); - seed_baseline(&ctx, &input).await; - let old_vod_id = seed_xtream_target(&ctx, &target, &playlist_state).await; - install_source(&ctx, &input, &target); - set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); - let app_config = Arc::clone(&ctx.config); - let pending_refresh = Arc::clone(&ctx.partial_refresh); - let quality_rejections = Arc::clone(&ctx.had_quality_rejections); - - let (_, target_stats, errors) = process_source(0, Arc::new(ctx)).await; - server.finish(); - - assert!(errors.is_empty(), "forced target update failed: {errors:?}"); - assert_eq!(target_stats.len(), 1); - assert!(target_stats[0].success); - assert_eq!( - PlaylistRunSignals { - has_error: false, - has_pending_stalker_refresh: pending_refresh.load(Ordering::Acquire), - has_quality_rejections: quality_rejections.load(Ordering::Acquire), - } - .state(), - PlaylistUpdateState::Success - ); - assert_target_counts(&app_config, &target, [2, 0, 1]).await; - assert_empty_target_categories(&app_config, &target, &[XtreamCluster::Video]).await; - assert!( - xtream_get_item_for_stream_id( - old_vod_id, - &app_config, - &playlist_state, - &target, - Some(XtreamCluster::Video), - ) - .await - .is_err(), - "old VOD id must not remain client-visible" - ); - let cache = playlist_state.data.read().await; - let cached = - cache.get(&target.name).and_then(|storage| storage.xtream.as_ref()).expect("Xtream target memory cache"); - assert_eq!([cached.live.len(), cached.vod.len(), cached.series.len()], [2, 0, 1]); - } - - #[tokio::test] - async fn force_fully_empty_result_clears_the_xtream_target() { - let temp = tempfile::tempdir().expect("temporary storage"); - let server = TestXtreamServer::start([0, 0, 0]); - let mut ctx = processing_context(temp.path()); - let input = test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 90, vod: 90, series: 90 }, 0, &[]); - let target = xtream_target(true); - let playlist_state = Arc::new(PlaylistStorageState::new()); - ctx.playlist_state = Some(Arc::clone(&playlist_state)); - seed_baseline(&ctx, &input).await; - let old_vod_id = seed_xtream_target(&ctx, &target, &playlist_state).await; - install_source(&ctx, &input, &target); - set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); - let app_config = Arc::clone(&ctx.config); - let pending_refresh = Arc::clone(&ctx.partial_refresh); - let quality_rejections = Arc::clone(&ctx.had_quality_rejections); - - let (_, target_stats, errors) = process_source(0, Arc::new(ctx)).await; - server.finish(); - - assert!(errors.is_empty(), "fully empty forced target update failed: {errors:?}"); - assert_eq!(target_stats.len(), 1); - assert!(target_stats[0].success); - assert_eq!( - PlaylistRunSignals { - has_error: false, - has_pending_stalker_refresh: pending_refresh.load(Ordering::Acquire), - has_quality_rejections: quality_rejections.load(Ordering::Acquire), - } - .state(), - PlaylistUpdateState::Success - ); - assert_target_counts(&app_config, &target, [0, 0, 0]).await; - assert_empty_target_categories(&app_config, &target, &XTREAM_CLUSTER).await; - assert!( - xtream_get_item_for_stream_id( - old_vod_id, - &app_config, - &playlist_state, - &target, - Some(XtreamCluster::Video), - ) - .await - .is_err(), - "fully empty target must not resolve an old VOD id" - ); - let cache = playlist_state.data.read().await; - let cached = - cache.get(&target.name).and_then(|storage| storage.xtream.as_ref()).expect("Xtream target memory cache"); - assert_eq!([cached.live.len(), cached.vod.len(), cached.series.len()], [0, 0, 0]); - } - - #[tokio::test] - async fn force_technical_parse_failure_retains_the_target_cluster() { - let temp = tempfile::tempdir().expect("temporary storage"); - let mut responses = fixture_responses([2, 1, 2]); - responses.insert("get_vod_streams".to_string(), "{".to_string()); - let server = TestXtreamServer::start_with_responses(responses); - let mut ctx = processing_context(temp.path()); - let input = - test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 0, &[]); - let target = xtream_target(true); - let playlist_state = Arc::new(PlaylistStorageState::new()); - ctx.playlist_state = Some(Arc::clone(&playlist_state)); - seed_baseline(&ctx, &input).await; - let old_vod_id = seed_xtream_target(&ctx, &target, &playlist_state).await; - install_source(&ctx, &input, &target); - set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); - let app_config = Arc::clone(&ctx.config); - - let (_, target_stats, errors) = process_source(0, Arc::new(ctx)).await; - server.finish(); - - assert!(!errors.is_empty(), "malformed provider response must remain a technical error"); - assert_eq!(target_stats.len(), 1); - assert!(target_stats[0].success, "usable fallback should still be published"); - assert_target_counts(&app_config, &target, [2, 2, 2]).await; - let retained_vod = xtream_get_item_for_stream_id( - old_vod_id, - &app_config, - &playlist_state, - &target, - Some(XtreamCluster::Video), - ) - .await - .expect("old VOD should remain client-visible after technical failure"); - assert!(retained_vod.name.starts_with("old-vod")); - } - - #[tokio::test] - async fn normal_empty_candidate_retains_the_xtream_target_without_empty_authorization() { - let temp = tempfile::tempdir().expect("temporary storage"); - let server = TestXtreamServer::start([0, 0, 0]); - let mut ctx = processing_context(temp.path()); - let input = - test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 0, &[]); - let target = xtream_target(true); - let playlist_state = Arc::new(PlaylistStorageState::new()); - ctx.playlist_state = Some(Arc::clone(&playlist_state)); - seed_baseline(&ctx, &input).await; - let old_vod_id = seed_xtream_target(&ctx, &target, &playlist_state).await; - install_source(&ctx, &input, &target); - let app_config = Arc::clone(&ctx.config); - let quality_rejections = Arc::clone(&ctx.had_quality_rejections); - - let (_, target_stats, errors) = process_source(0, Arc::new(ctx)).await; - server.finish(); - - assert!(errors.is_empty(), "quality rejection must not become a technical error: {errors:?}"); - assert!(quality_rejections.load(Ordering::Acquire)); - assert_eq!(target_stats.len(), 1); - assert!(target_stats[0].success); - assert_target_counts(&app_config, &target, [2, 2, 2]).await; - let retained_vod = xtream_get_item_for_stream_id( - old_vod_id, - &app_config, - &playlist_state, - &target, - Some(XtreamCluster::Video), - ) - .await - .expect("normal empty candidate must retain the target VOD"); - assert!(retained_vod.name.starts_with("old-vod")); - } - - #[tokio::test] - async fn force_policy_retains_a_cluster_after_a_technical_parse_failure() { - let temp = tempfile::tempdir().expect("temporary storage"); - let mut responses = fixture_responses([2, 1, 2]); - responses.insert("get_vod_streams".to_string(), "{".to_string()); - let server = TestXtreamServer::start_with_responses(responses); - let mut ctx = processing_context(temp.path()); - let input = - test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 0, &[]); - seed_baseline(&ctx, &input).await; - set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); - - let mut result = download_input(&ctx, &input, false).await; - let requests = server.finish(); - - assert!(!result.errors.is_empty(), "malformed VOD response must remain a technical error"); - assert!(result.storage_error.is_none()); - assert!(result.quality_rejections.is_empty()); - let groups = result.source.take_groups(); - let vod = groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Video).expect("retained VOD"); - assert_eq!(vod.title.as_ref(), "old-vod"); - assert_eq!(vod.channels.len(), 2); - assert_eq!( - groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Live).map(|group| group.title.as_ref()), - Some("candidate-live") - ); - assert_eq!( - groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Series).map(|group| group.title.as_ref()), - Some("candidate-series") - ); - assert_requested_actions( - requests, - &[ - "login", - "get_live_categories", - "get_live_streams", - "get_vod_categories", - "get_vod_streams", - "get_series_categories", - "get_series", - ], - ); - } - - #[tokio::test] - async fn in_memory_series_quality_compares_catalog_rows_with_an_enriched_baseline() { - let temp = tempfile::tempdir().expect("temporary storage"); - let server = TestXtreamServer::start([0, 0, 1]); - let ctx = processing_context(temp.path()); - let input = test_input( - &server.base_url, - ConfigInputUpdateQualityDto { series: 100, ..ConfigInputUpdateQualityDto::default() }, - 0, - &[XtreamCluster::Live, XtreamCluster::Video], - ); - let mut baseline = baseline_group(XtreamCluster::Series, 3, "old-series", 3_000); - baseline.channels.truncate(1); - let episodes = (0..200_u32) - .map(|id| SeriesStreamDetailEpisodeProperties { id, ..SeriesStreamDetailEpisodeProperties::default() }) - .collect(); - baseline.channels[0].header.additional_properties = - Some(StreamProperties::Series(Box::new(SeriesStreamProperties { - series_id: 3_000, - details: Some(SeriesStreamDetailProperties::new(None, Vec::new(), Some(episodes))), - ..SeriesStreamProperties::default() - }))); - let (persisted, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, vec![baseline]).await; - assert!(error.is_none(), "enriched baseline persistence failed: {error:?}"); - assert_eq!(persisted.iter().map(|group| group.channels.len()).sum::(), 1); - assert_eq!( - count_input_xtream_cluster(&ctx.config, &input, XtreamCluster::Series) - .await - .expect("enriched baseline should be countable"), - Some(1) - ); - - let mut result = download_input(&ctx, &input, false).await; - let requests = server.finish(); - - assert!(result.errors.is_empty(), "Series update failed: {:?}", result.errors); - assert!(result.quality_rejections.is_empty()); - let groups = result.source.take_groups(); - assert_eq!(groups.len(), 1); - assert_eq!(groups[0].title.as_ref(), "candidate-series"); - assert_eq!(groups[0].channels.len(), 1); - let episode_count = groups[0].channels[0] - .header - .additional_properties - .as_ref() - .and_then(|properties| match properties { - StreamProperties::Series(series) => series.details.as_ref(), - _ => None, - }) - .and_then(|details| details.episodes.as_ref()) - .map(Vec::len); - assert_eq!(episode_count, Some(200)); - assert_requested_actions(requests, &["login", "get_series_categories", "get_series"]); - } - - #[tokio::test] - async fn in_memory_quality_rejection_counts_duplicate_provider_ids_once_and_retains_the_baseline() { - let temp = tempfile::tempdir().expect("temporary storage"); - let first_provider_id = 1_000_u32; - let streams = (0..50_u32) - .flat_map(|offset| { - let provider_id = first_provider_id + offset; - [ - live_fixture_stream(provider_id, 1, &format!("candidate-{provider_id}")), - live_fixture_stream(provider_id, 1, &format!("duplicate-{provider_id}")), - ] - }) - .collect(); - let responses = live_fixture_responses( - &serde_json::json!([{"category_id": 1, "category_name": "candidate-live"}]), - streams, - ); - let server = TestXtreamServer::start_with_responses(responses); - let ctx = processing_context(temp.path()); - let input = test_input( - &server.base_url, - ConfigInputUpdateQualityDto { live: 100, ..ConfigInputUpdateQualityDto::default() }, - 0, - &[XtreamCluster::Video, XtreamCluster::Series], - ); - seed_live_baseline(&ctx, &input, first_provider_id).await; - - let mut result = download_input(&ctx, &input, false).await; - let requests = server.finish(); - - assert!(result.errors.is_empty(), "duplicate rejection must not become an error: {:?}", result.errors); - assert!(result.storage_error.is_none()); - assert_eq!( - result.quality_rejections, - vec![ClusterUpdateRejection { - cluster: XtreamCluster::Live, - current_count: 100, - candidate_count: 50, - threshold: 100, - quality: 50, - }] - ); - let groups = result.source.take_groups(); - assert_eq!(groups.iter().map(|group| group.channels.len()).sum::(), 100); - assert_eq!(groups[0].title.as_ref(), "old-live"); - - let mut reloaded = load_input_playlist(&ctx.config, &input, None).await.expect("reload retained Live baseline"); - let reloaded_groups = reloaded.take_groups(); - assert_eq!(reloaded_groups.iter().map(|group| group.channels.len()).sum::(), 100); - assert_eq!(reloaded_groups[0].title.as_ref(), "old-live"); - assert_eq!( - count_input_xtream_cluster(&ctx.config, &input, XtreamCluster::Live) - .await - .expect("retained baseline should be countable"), - Some(100) - ); - assert_requested_actions(requests, &["login", "get_live_categories", "get_live_streams"]); - } - - #[tokio::test] - async fn in_memory_quality_accepts_unique_population_and_persists_the_last_duplicate_row() { - let temp = tempfile::tempdir().expect("temporary storage"); - let first_provider_id = 1_000_u32; - let winning_provider_id = first_provider_id + 42; - let mut streams = (0..100_u32) - .map(|offset| { - let provider_id = first_provider_id + offset; - live_fixture_stream(provider_id, 1, &format!("candidate-{provider_id}")) - }) - .collect::>(); - streams.push(live_fixture_stream(winning_provider_id, 2, "winning-duplicate")); - let responses = live_fixture_responses( - &serde_json::json!([ - {"category_id": 1, "category_name": "candidate-live"}, - {"category_id": 2, "category_name": "winning-category"} - ]), - streams, - ); - let server = TestXtreamServer::start_with_responses(responses); - let ctx = processing_context(temp.path()); - let input = test_input( - &server.base_url, - ConfigInputUpdateQualityDto { live: 100, ..ConfigInputUpdateQualityDto::default() }, - 0, - &[XtreamCluster::Video, XtreamCluster::Series], - ); - seed_live_baseline(&ctx, &input, first_provider_id).await; - - let mut result = download_input(&ctx, &input, false).await; - let requests = server.finish(); - - assert!(result.errors.is_empty(), "deduplicated acceptance failed: {:?}", result.errors); - assert!(result.storage_error.is_none()); - assert!(result.quality_rejections.is_empty()); - let groups = result.source.take_groups(); - assert_eq!(groups.iter().map(|group| group.channels.len()).sum::(), 100); - let winner_group = groups.iter().find(|group| group.id == 2).expect("winning category"); - assert_eq!(winner_group.title.as_ref(), "winning-category"); - assert_eq!(winner_group.channels.len(), 1); - assert_eq!(winner_group.channels[0].header.id.as_ref(), winning_provider_id.to_string()); - assert_eq!(winner_group.channels[0].header.name.as_ref(), "winning-duplicate"); - assert_eq!(winner_group.channels[0].header.category_id, 2); - - let mut reloaded = - load_input_playlist(&ctx.config, &input, None).await.expect("reload accepted Live candidate"); - let reloaded_groups = reloaded.take_groups(); - assert_eq!(reloaded_groups.iter().map(|group| group.channels.len()).sum::(), 100); - let reloaded_winner = reloaded_groups.iter().find(|group| group.id == 2).expect("persisted winning category"); - assert_eq!(reloaded_winner.channels.len(), 1); - assert_eq!(reloaded_winner.channels[0].header.id.as_ref(), winning_provider_id.to_string()); - assert_eq!(reloaded_winner.channels[0].header.name.as_ref(), "winning-duplicate"); - assert_eq!(reloaded_winner.channels[0].header.category_id, 2); - assert_eq!( - count_input_xtream_cluster(&ctx.config, &input, XtreamCluster::Live) - .await - .expect("accepted candidate should be countable"), - Some(100) - ); - assert_requested_actions(requests, &["login", "get_live_categories", "get_live_streams"]); - } - - #[tokio::test] - async fn m3u_update_quality_enforces_the_existing_cluster_threshold() { - let temp = tempfile::tempdir().expect("temporary storage"); - let playlist_path = temp.path().join("input.m3u"); - tokio::fs::write( - &playlist_path, - "#EXTM3U\n#EXTINF:-1,Old One\nhttp://old.example/1\n#EXTINF:-1,Old Two\nhttp://old.example/2\n", - ) - .await - .expect("initial M3U fixture"); - let options = ConfigInputOptionsDto { - update_quality: ConfigInputUpdateQualityDto { live: 100, vod: 0, series: 0 }, - ..ConfigInputOptionsDto::default() - }; - let input = Arc::new(ConfigInput { - id: 2, - name: "m3u-quality-guard".intern(), - input_type: InputType::M3u, - url: playlist_path.to_string_lossy().into_owned(), - enabled: true, - options: Some(ConfigInputOptions::from(&options)), - ..ConfigInput::default() - }); - let initial_ctx = processing_context(temp.path()); - - let mut initial = download_input(&initial_ctx, &input, false).await; - - assert!(initial.errors.is_empty(), "initial M3U download failed: {:?}", initial.errors); - assert!(initial.quality_rejections.is_empty()); - assert_eq!(initial.source.get_channel_count(), 2); - - tokio::fs::write(&playlist_path, "#EXTM3U\n#EXTINF:-1,New\nhttp://new.example/1\n") - .await - .expect("replacement M3U fixture"); - let refreshed_ctx = processing_context(temp.path()); - let mut refreshed = download_input(&refreshed_ctx, &input, false).await; - - assert!(refreshed.errors.is_empty(), "replacement M3U download failed: {:?}", refreshed.errors); - assert!(refreshed.storage_error.is_none()); - assert_eq!(refreshed.quality_rejections.len(), 1); - assert_eq!(refreshed.quality_rejections[0].cluster, XtreamCluster::Live); - assert_eq!(refreshed.source.get_channel_count(), 2); - let mut reloaded = load_input_playlist(&refreshed_ctx.config, &input, None) - .await - .expect("replacement M3U should be persisted"); - assert_eq!(reloaded.get_channel_count(), 2); - } - - #[tokio::test] - async fn quality_rejection_for_only_requested_vod_loads_complete_persisted_input() { - let temp = tempfile::tempdir().expect("temporary storage"); - let server = TestXtreamServer::start([0, 1, 0]); - let ctx = processing_context(temp.path()); - let input = test_input( - &server.base_url, - ConfigInputUpdateQualityDto { vod: 100, ..ConfigInputUpdateQualityDto::default() }, - 3_600, - &[], - ); - seed_baseline(&ctx, &input).await; - let storage_path = input_storage_path(&ctx, &input).await; - let now = SystemTime::now().duration_since(UNIX_EPOCH).expect("current time").as_secs(); - let mut status = input_cache::InputStatus::default(); - status.clusters.insert( - XtreamCluster::Live.as_ref().to_string(), - input_cache::ClusterStatus { status: input_cache::ClusterState::Ok, timestamp: now, last_update: None }, - ); - status.clusters.insert( - XtreamCluster::Video.as_ref().to_string(), - input_cache::ClusterStatus { - status: input_cache::ClusterState::Ok, - timestamp: now.saturating_sub(3_601), - last_update: None, - }, - ); - status.clusters.insert( - XtreamCluster::Series.as_ref().to_string(), - input_cache::ClusterStatus { status: input_cache::ClusterState::Ok, timestamp: now, last_update: None }, - ); - input_cache::save_input_status(&storage_path, &status); - - let mut result = download_input(&ctx, &input, false).await; - let requests = server.finish(); - - assert!(result.errors.is_empty(), "quality rejection must not become an error: {:?}", result.errors); - assert!(result.storage_error.is_none()); - assert!(!result.partial); - assert_eq!(result.job_state(), InputJobState::Ready); - assert_eq!(result.quality_rejections.len(), 1); - assert_eq!( - result.quality_rejections[0], - ClusterUpdateRejection { - cluster: XtreamCluster::Video, - current_count: 2, - candidate_count: 1, - threshold: 100, - quality: 50, - } - ); - assert_eq!( - PlaylistRunSignals { - has_quality_rejections: !result.quality_rejections.is_empty(), - ..PlaylistRunSignals::default() - } - .state(), - PlaylistUpdateState::Partial - ); - assert_baseline(&result.source.take_groups()); - - let status = input_cache::load_input_status(&storage_path); - let live = status.clusters.get(XtreamCluster::Live.as_ref()).expect("Live cache status"); - let vod = status.clusters.get(XtreamCluster::Video.as_ref()).expect("VOD cache status"); - let series = status.clusters.get(XtreamCluster::Series.as_ref()).expect("Series cache status"); - assert_eq!(live.status, input_cache::ClusterState::Ok); - assert_eq!(live.timestamp, now); - assert_eq!(vod.status, input_cache::ClusterState::Failed); - assert_eq!(series.status, input_cache::ClusterState::Ok); - assert_eq!(series.timestamp, now); - assert_requested_actions(requests, &["login", "get_vod_categories", "get_vod_streams"]); - } - - #[tokio::test] - async fn all_requested_clusters_rejected_load_the_complete_unchanged_baseline() { - let temp = tempfile::tempdir().expect("temporary storage"); - let server = TestXtreamServer::start([1, 1, 1]); - let ctx = processing_context(temp.path()); - let input = - test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 0, &[]); - seed_baseline(&ctx, &input).await; - - let mut result = download_input(&ctx, &input, false).await; - let requests = server.finish(); - - assert!(result.errors.is_empty(), "quality rejections must not become errors: {:?}", result.errors); - assert!(result.storage_error.is_none()); - assert_eq!(result.job_state(), InputJobState::Ready); - assert_eq!( - result.quality_rejections.iter().map(|rejection| rejection.cluster).collect::>(), - XTREAM_CLUSTER - ); - assert!(result.quality_rejections.iter().all(|rejection| { - rejection.current_count == 2 - && rejection.candidate_count == 1 - && rejection.threshold == 100 - && rejection.quality == 50 - })); - assert_baseline(&result.source.take_groups()); - - let mut reloaded = load_input_playlist(&ctx.config, &input, None).await.expect("reload persisted baseline"); - assert_baseline(&reloaded.take_groups()); - let storage_path = input_storage_path(&ctx, &input).await; - let status = input_cache::load_input_status(&storage_path); - assert!(XTREAM_CLUSTER.iter().all(|cluster| { - status.clusters.get(cluster.as_ref()).map(|entry| &entry.status) == Some(&input_cache::ClusterState::Failed) - })); - assert_requested_actions( - requests, - &[ - "login", - "get_live_categories", - "get_live_streams", - "get_vod_categories", - "get_vod_streams", - "get_series_categories", - "get_series", - ], - ); - } - - #[tokio::test] - async fn quality_rejection_for_empty_bootstrap_does_not_invent_a_baseline() { - let temp = tempfile::tempdir().expect("temporary storage"); - let server = TestXtreamServer::start([0, 0, 0]); - let ctx = processing_context(temp.path()); - let input = test_input( - &server.base_url, - ConfigInputUpdateQualityDto { vod: 100, ..ConfigInputUpdateQualityDto::default() }, - 0, - &[XtreamCluster::Live, XtreamCluster::Series], - ); - - let mut result = download_input(&ctx, &input, false).await; - let requests = server.finish(); - - assert!(result.errors.is_empty(), "unexpected bootstrap errors: {:?}", result.errors); - assert!(result.storage_error.is_none()); - assert_eq!(result.quality_rejections.len(), 1); - assert_eq!( - result.quality_rejections[0], - ClusterUpdateRejection { - cluster: XtreamCluster::Video, - current_count: 0, - candidate_count: 0, - threshold: 100, - quality: 0, - } - ); - assert!(result.source.is_empty()); - assert_eq!(result.job_state(), InputJobState::Failed); - let mut reloaded = load_input_playlist(&ctx.config, &input, None).await.expect("empty bootstrap reload"); - assert!(reloaded.take_groups().is_empty()); - assert_requested_actions(requests, &["login", "get_vod_categories", "get_vod_streams"]); - } -} +mod quality_rejection_fallback; #[cfg(test)] mod disk_epg_wireup_tests { @@ -5328,3 +126,24 @@ mod disk_epg_wireup_tests { assert!(merged.is_none()); } } + +mod curation_selection; + +mod filter_rename; + +mod identity_serialization; + +mod pipeline_transparency; + +mod probe_details; + +mod scheduling; + +mod staged_overlay; + +mod support; + +use self::support::{ + apply_staged_overlay_groups, catalog_membership, catalog_test_item, complete_catalog_evaluation, + playlist_update_run_result, test_group, PlaylistRunCollectSink, +}; diff --git a/backend/processing/src/processor/playlist/tests/curation_effect_gate.rs b/backend/processing/src/processor/playlist/tests/curation_effect_gate.rs new file mode 100644 index 000000000..c54ac32d4 --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/curation_effect_gate.rs @@ -0,0 +1,336 @@ +use super::*; +use arc_swap::{ArcSwap, ArcSwapOption}; +use shared::model::{ConfigPaths, NoopSink}; +use std::{ + collections::{BTreeMap, BTreeSet}, + path::Path, +}; +use tempfile::tempdir; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + task::JoinHandle, +}; +use tuliprox_core::{ + model::{ApiProxyConfig, CustomStreamResponse, HdHomeRunConfig, MediaToolCapabilities, SourcesConfig}, + utils::FileLockManager, +}; + +pub(super) fn app_config(storage_dir: &Path) -> Arc { + Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config { + storage_dir: storage_dir.to_string_lossy().into_owned(), + ..Config::default() + })), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::::default()), + api_proxy: Arc::new(ArcSwapOption::::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + }) +} + +pub(super) fn processing_context( + app_config: Arc, + playlist_state: Option>, +) -> PlaylistProcessingContext { + PlaylistProcessingContext { + client: reqwest::Client::new(), + run_id: "curation-effect-gate-run".into(), + execution_order: PlaylistUpdateRunOrder::from(1), + config: app_config, + user_targets: Arc::new(ProcessTargets { + enabled: false, + inputs: Vec::new(), + targets: Vec::new(), + target_names: Vec::new(), + }), + events: NoopSink, + playlist_state, + disabled_headers: None, + processed_inputs: Arc::new(Mutex::new(HashSet::new())), + input_completions: Arc::new(Mutex::new(HashMap::new())), + input_locks: Arc::new(Mutex::new(HashMap::new())), + provider_manager: None, + metadata_manager: None, + pre_processed_inputs: None, + stalker_refresh_mode: StalkerRefreshMode::Complete, + partial_refresh: Arc::new(std::sync::atomic::AtomicBool::new(false)), + had_quality_rejections: Arc::new(std::sync::atomic::AtomicBool::new(false)), + input_refresh: None, + library_update_mode: LibraryUpdateMode::ExistingCatalog, + } +} + +fn curation_persist_options(publication_plan: PlaylistPublicationPlan) -> TargetPlaylistPersistOptions { + TargetPlaylistPersistOptions { publication_plan, ..TargetPlaylistPersistOptions::default() } +} + +pub(super) fn file_snapshot(root: &Path) -> BTreeMap> { + fn collect(root: &Path, path: &Path, snapshot: &mut BTreeMap>) { + let Ok(entries) = std::fs::read_dir(path) else { return }; + for entry in entries.flatten() { + let entry_path = entry.path(); + if entry_path.is_dir() { + collect(root, &entry_path, snapshot); + } else { + snapshot.insert( + entry_path.strip_prefix(root).expect("snapshot path under root").to_path_buf(), + std::fs::read(&entry_path).expect("snapshot file"), + ); + } + } + } + + let mut snapshot = BTreeMap::new(); + collect(root, root, &mut snapshot); + snapshot +} + +async fn empty_trakt_server() -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind Trakt test server"); + let address = listener.local_addr().expect("Trakt test address"); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("accept Trakt request"); + let mut request = Vec::new(); + loop { + let mut buffer = [0u8; 1024]; + let read = stream.read(&mut buffer).await.expect("read Trakt request"); + if read == 0 { + break; + } + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + stream + .write_all(b"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 2\r\nconnection: close\r\n\r\n[]") + .await + .expect("write Trakt response"); + }); + (format!("http://{address}"), server) +} + +#[tokio::test] +async fn unavailable_curation_preserves_the_empty_mixed_output_guard() { + assert_curation_preserves_empty_mixed_output_guard(None).await; +} + +#[tokio::test] +async fn tmdb_and_mixed_failures_preserve_the_empty_mixed_output_guard() { + for policy in ["full", "curated"] { + for mixed in [false, true] { + let (url, server) = empty_trakt_server().await; + let mut value = serde_json::json!({"catalog_selection": policy, "tmdb": {"trending": [{"kind": "movie", "time_window": "week", "limit": 100, "category_name": "TMDB"}]}}); + if mixed { + value["trakt"] = serde_json::json!({"api": {"api_key": "test-client", "url": url}, "charts": [{"kind": "movies", "chart": "popular", "category_name": "Trakt"}]}); + } + let mut dto: shared::model::CurationConfigDto = serde_json::from_value(value).unwrap(); + dto.prepare(&[TargetOutputDto::Xtream(XtreamTargetOutputDto::default())]).unwrap(); + assert_curation_preserves_empty_mixed_output_guard(Some(dto)).await; + if mixed { + server.await.unwrap(); + } else { + server.abort(); + } + } + } +} + +async fn assert_curation_preserves_empty_mixed_output_guard(canonical: Option) { + let directory = tempdir().expect("tempdir"); + let app_config = app_config(directory.path()); + let playlist_state = Arc::new(PlaylistStorageState::new()); + let mut target = ConfigTarget::from(&ConfigTargetDto { + name: "curation-effect-gate".to_string(), + output: vec![ + TargetOutputDto::Xtream(XtreamTargetOutputDto { + trakt: Some(TraktConfigDto { + lists: vec![TraktListConfigDto { + user: "alice".to_string(), + list_slug: "watchlist".to_string(), + category_name: Some("Watchlist".to_string()), + create_xtream_category: true, + content_type: TraktContentType::Vod, + tmdb_only: true, + fuzzy_match_threshold: 100, + }], + ..TraktConfigDto::default() + }), + ..XtreamTargetOutputDto::default() + }), + TargetOutputDto::M3u(M3uTargetOutputDto { + filename: Some("curation-effect-gate.m3u".to_string()), + ..M3uTargetOutputDto::default() + }), + ], + watch: Some(vec![".*".to_string()]), + use_memory_cache: true, + ..ConfigTargetDto::default() + }); + if let Some(canonical) = canonical { + target.curation = Some(CurationConfig::from(&canonical)); + if let tuliprox_core::model::TargetOutput::Xtream(output) = &mut target.output[0] { + output.trakt = None; + } + } + let mut seeded = vec![PlaylistGroup { + id: 1, + title: "Movies".intern(), + channels: vec![catalog_test_item( + "Seeded movie", + UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000071"), + PlaylistItemType::Video, + XtreamCluster::Video, + None, + )], + xtream_cluster: XtreamCluster::Video, + }]; + let seeded_result = persist_playlist_views( + &app_config, + &mut seeded, + None, + None, + &target, + Some(&playlist_state), + curation_persist_options(PlaylistPublicationPlan::Ordinary), + ) + .await; + assert!(seeded_result.is_ok(), "seed persist failed: {seeded_result:?}"); + assert!(process_watch(&app_config, &NoopSink, &target, &seeded).await); + let before_files = file_snapshot(directory.path()); + let before_cache_len = playlist_state + .data + .read() + .await + .get(&target.name) + .and_then(|storage| storage.xtream.as_ref()) + .map_or(0, |storage| storage.vod.len()); + + let context = processing_context(Arc::clone(&app_config), Some(Arc::clone(&playlist_state))); + let prepared = PreparedTarget { + target, + playlist: Vec::new(), + epg: Vec::new(), + processing: PipelineStats::default(), + accepted_empty_clusters: ClusterFlags::Vod, + library_empty: tuliprox_repository::LibraryEmptyPublication::None, + }; + + let (result, errors) = finalize_prepared_target(Arc::new(context), prepared).await; + + assert!(result.is_err_and(|errors| errors.iter().any(|error| { + error.message().contains("non-Xtream outputs that cannot safely publish a fully empty forced result") + }))); + assert!(errors.is_empty()); + assert_eq!(file_snapshot(directory.path()), before_files); + let after_cache_len = playlist_state + .data + .read() + .await + .get("curation-effect-gate") + .and_then(|storage| storage.xtream.as_ref()) + .map_or(0, |storage| storage.vod.len()); + assert!(before_cache_len > 0); + assert_eq!(after_cache_len, before_cache_len); +} + +#[tokio::test] +async fn complete_empty_curation_publishes_empty_watch_group_state() { + let directory = tempdir().expect("tempdir"); + let app_config = app_config(directory.path()); + let (base_url, server) = empty_trakt_server().await; + let target_name = "curation-empty-watch"; + let target = ConfigTarget::from(&ConfigTargetDto { + name: target_name.to_string(), + output: vec![TargetOutputDto::Xtream(XtreamTargetOutputDto { + trakt: Some(TraktConfigDto { + catalog_selection: TraktCatalogSelection::Curated, + api: TraktApiConfigDto { + api_key: "test-client-id".to_string(), + version: "2".to_string(), + url: base_url, + user_agent: "tuliprox-test".to_string(), + }, + lists: vec![TraktListConfigDto { + user: "alice".to_string(), + list_slug: "watchlist".to_string(), + category_name: Some("Watchlist".to_string()), + create_xtream_category: true, + content_type: TraktContentType::Vod, + tmdb_only: true, + fuzzy_match_threshold: 100, + }], + ..TraktConfigDto::default() + }), + ..XtreamTargetOutputDto::default() + })], + watch: Some(vec![".*".to_string()]), + ..ConfigTargetDto::default() + }); + let mut seeded = vec![PlaylistGroup { + id: 1, + title: "Movies".intern(), + channels: vec![catalog_test_item( + "Seeded movie", + UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000072"), + PlaylistItemType::Video, + XtreamCluster::Video, + None, + )], + xtream_cluster: XtreamCluster::Video, + }]; + let seed_result = persist_playlist_views( + &app_config, + &mut seeded, + None, + None, + &target, + None, + curation_persist_options(PlaylistPublicationPlan::Ordinary), + ) + .await; + assert!(seed_result.is_ok(), "seed persist failed: {seed_result:?}"); + assert!(process_watch(&app_config, &NoopSink, &target, &seeded).await); + let watch_index = directory.path().join(format!("{target_name}.groups.bin")); + let before: BTreeSet> = + tuliprox_core::utils::binary_deserialize(&std::fs::read(&watch_index).expect("seeded watch index")) + .expect("decode seeded watch index"); + assert_eq!(before.len(), 1); + + let context = processing_context(Arc::clone(&app_config), None); + let prepared = PreparedTarget { + target, + playlist: seeded, + epg: Vec::new(), + processing: PipelineStats::default(), + accepted_empty_clusters: ClusterFlags::empty(), + library_empty: tuliprox_repository::LibraryEmptyPublication::None, + }; + let (result, errors) = finalize_prepared_target(Arc::new(context), prepared).await; + server.await.expect("Trakt server should finish"); + + assert!(result.is_ok(), "complete empty finalization failed: {result:?}"); + assert!(errors.is_empty()); + let after: BTreeSet> = + tuliprox_core::utils::binary_deserialize(&std::fs::read(watch_index).expect("empty watch index")) + .expect("decode empty watch index"); + assert!(after.is_empty()); +} diff --git a/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs b/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs index 52cff6edd..bf9907230 100644 --- a/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs +++ b/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs @@ -15,10 +15,14 @@ fn fail_late(server: &DiscoveryServer, failure: &str) { match failure { "status" => server.reply(MOVIES_2, 503, "sensitive-body-must-not-appear"), "wire" => { - server.reply(MOVIES_2, 200, r#"{"page":2,"total_pages":2,"total_results":2,"results":[{"id":7},{"id":0}]}"#) + server.reply( + MOVIES_2, + 200, + r#"{"page":2,"total_pages":2,"total_results":2,"results":[{"id":7},{"id":0}]}"#, + ); } "no-progress" => { - server.reply(MOVIES_2, 200, r#"{"page":2,"total_pages":2,"total_results":1,"results":[{"id":8}]}"#) + server.reply(MOVIES_2, 200, r#"{"page":2,"total_pages":2,"total_results":1,"results":[{"id":8}]}"#); } "empty" => server.reply(MOVIES_2, 200, r#"{"page":2,"total_pages":2,"total_results":0,"results":[]}"#), "body" => server.reply_raw( @@ -40,7 +44,7 @@ fn fail_late(server: &DiscoveryServer, failure: &str) { server.reply_raw(MOVIES_2, response); } "bytes" => { - server.reply(MOVIES_2, 200, &format!("{MOVIE_PAGE_2}{}", " ".repeat(1024 * 1024 + 1 - MOVIE_PAGE_2.len()))) + server.reply(MOVIES_2, 200, &format!("{MOVIE_PAGE_2}{}", " ".repeat(1024 * 1024 + 1 - MOVIE_PAGE_2.len()))); } "requests" => { for p in 1..=33 { diff --git a/backend/processing/src/processor/playlist/tests/curation_selection.rs b/backend/processing/src/processor/playlist/tests/curation_selection.rs new file mode 100644 index 000000000..fb4c5cedc --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/curation_selection.rs @@ -0,0 +1,401 @@ +use super::{ + apply_persist_filter, build_curated_playlist_views, catalog_membership, catalog_test_item, + complete_catalog_evaluation, prepare_eligible_catalog, prepare_target_playlist_views, select_target_catalog, + FinalizationStage, FINALIZATION_ORDER, +}; +use shared::{ + foundation::get_filter, + model::{ + ConfigTargetDto, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, TargetOutputDto, + TraktApiConfigDto, TraktCatalogSelection, TraktConfigDto, TraktContentType, TraktListConfigDto, UUIDType, + XtreamCluster, XtreamTargetOutputDto, + }, + utils::Internable, +}; +use tuliprox_core::{ + model::{ConfigTarget, CurationConfig, TraktConfig}, + utils::StepMeasure, +}; +use tuliprox_curation::{ + CurationEvaluation, CurationMediaKind, CurationMembership, CurationSelectorKey, CurationSelectorSummary, +}; +use tuliprox_repository::PlaylistPublicationPlan; + +#[test] +fn current_target_finalization_order_merges_before_dedup_and_presentation() { + assert_eq!( + FINALIZATION_ORDER, + [ + FinalizationStage::Merge, + FinalizationStage::Deduplicate, + FinalizationStage::Sort, + FinalizationStage::AssignChannelNumbers, + FinalizationStage::AssignCounters, + ] + ); +} + +#[test] +fn curation_eligible_catalog_is_merged_and_deduplicated_before_matching() { + let mut target = ConfigTarget::from(&ConfigTargetDto::default()); + target.execution_plan.post_merge_content_dedup = Some(shared::model::DeduplicateConfig::default()); + let losing_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000051"); + let winning_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000052"); + let playlist = vec![ + PlaylistGroup { + id: 1, + title: "Movies".intern(), + channels: vec![catalog_test_item( + "Movie HD", + losing_uuid, + PlaylistItemType::Video, + XtreamCluster::Video, + None, + )], + xtream_cluster: XtreamCluster::Video, + }, + PlaylistGroup { + id: 2, + title: "movies".intern(), + channels: vec![catalog_test_item( + "Movie 4K", + winning_uuid, + PlaylistItemType::Video, + XtreamCluster::Video, + None, + )], + xtream_cluster: XtreamCluster::Video, + }, + ]; + let mut step = StepMeasure::new("test", |_, _| {}); + + let eligible = prepare_eligible_catalog(&target, playlist, &mut step); + + assert_eq!(eligible.len(), 1); + assert_eq!(eligible[0].channels.len(), 1); + assert_eq!(eligible[0].channels[0].header.uuid, winning_uuid); +} + +#[test] +fn persist_filter_can_select_a_generated_curation_group() { + let mut target = ConfigTarget::from(&ConfigTargetDto::default()); + target.filter.persist = Some(get_filter(r#"Group = "Trending""#, None).expect("persist filter")); + let mut playlist = vec![ + PlaylistGroup { + id: 1, + title: "Base".intern(), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { group: "Base".intern(), ..Default::default() }, + }], + xtream_cluster: XtreamCluster::Video, + }, + PlaylistGroup { + id: 2, + title: "Trending".intern(), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { group: "Trending".intern(), ..Default::default() }, + }], + xtream_cluster: XtreamCluster::Video, + }, + ]; + + apply_persist_filter(&target, &mut playlist); + + assert_eq!(playlist.len(), 1); + assert_eq!(playlist[0].title.as_ref(), "Trending"); +} + +#[tokio::test] +async fn trakt_target_curation_is_a_noop_without_xtream_configuration() { + let target = ConfigTarget::from(&ConfigTargetDto::default()); + + let views = prepare_target_playlist_views(&reqwest::Client::new(), None, &target, Vec::new()).await; + + assert!(views.base.is_empty()); + assert!(views.xtream.is_none()); + assert_eq!(views.publication_plan, PlaylistPublicationPlan::Ordinary); +} + +#[tokio::test] +async fn unavailable_required_selector_continues_with_the_base_catalog() { + let target = ConfigTarget::from(&ConfigTargetDto { + name: "curation-failure".to_string(), + output: vec![TargetOutputDto::Xtream(XtreamTargetOutputDto { + trakt: Some(TraktConfigDto { + lists: vec![TraktListConfigDto { + user: "alice".to_string(), + list_slug: "watchlist".to_string(), + category_name: Some("Watchlist".to_string()), + create_xtream_category: true, + content_type: TraktContentType::Vod, + tmdb_only: true, + fuzzy_match_threshold: 100, + }], + ..TraktConfigDto::default() + }), + ..XtreamTargetOutputDto::default() + })], + ..ConfigTargetDto::default() + }); + let base = vec![PlaylistGroup { + id: 1, + title: "Movies".intern(), + channels: vec![catalog_test_item( + "Base movie", + UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000061"), + PlaylistItemType::Video, + XtreamCluster::Video, + None, + )], + xtream_cluster: XtreamCluster::Video, + }]; + + let views = prepare_target_playlist_views(&reqwest::Client::new(), None, &target, base).await; + assert_eq!(views.base.len(), 1); + assert_eq!(views.base[0].channels[0].header.title.as_ref(), "Base movie"); + assert!(views.xtream.is_none()); + assert_eq!(views.publication_plan, PlaylistPublicationPlan::Ordinary); +} + +#[test] +fn catalog_selection_preserves_live_and_selected_series_children() { + let live_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000001"); + let selected_movie_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000002"); + let rejected_movie_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000003"); + let series_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000004"); + let episode_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000005"); + let rejected_series_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000006"); + let rejected_episode_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000007"); + let playlist = vec![ + PlaylistGroup { + id: 1, + title: "Live".intern(), + channels: vec![catalog_test_item( + "Live channel", + live_uuid, + PlaylistItemType::Live, + XtreamCluster::Live, + None, + )], + xtream_cluster: XtreamCluster::Live, + }, + PlaylistGroup { + id: 2, + title: "Movies".intern(), + channels: vec![ + catalog_test_item( + "Selected movie", + selected_movie_uuid, + PlaylistItemType::Video, + XtreamCluster::Video, + None, + ), + catalog_test_item( + "Rejected movie", + rejected_movie_uuid, + PlaylistItemType::Video, + XtreamCluster::Video, + None, + ), + ], + xtream_cluster: XtreamCluster::Video, + }, + PlaylistGroup { + id: 3, + title: "Series".intern(), + channels: vec![ + catalog_test_item( + "Selected series", + series_uuid, + PlaylistItemType::SeriesInfo, + XtreamCluster::Series, + None, + ), + catalog_test_item( + "Selected episode", + episode_uuid, + PlaylistItemType::Series, + XtreamCluster::Series, + Some(&series_uuid.to_string()), + ), + catalog_test_item( + "Rejected series", + rejected_series_uuid, + PlaylistItemType::SeriesInfo, + XtreamCluster::Series, + None, + ), + catalog_test_item( + "Rejected episode", + rejected_episode_uuid, + PlaylistItemType::Series, + XtreamCluster::Series, + Some(&rejected_series_uuid.to_string()), + ), + ], + xtream_cluster: XtreamCluster::Series, + }, + ]; + let evaluation = complete_catalog_evaluation(vec![ + catalog_membership(selected_movie_uuid, CurationMediaKind::Movie, 0), + catalog_membership(series_uuid, CurationMediaKind::Series, 1), + ]); + + let selected = select_target_catalog(playlist.clone(), &evaluation, true); + let titles = + selected.iter().flat_map(|group| &group.channels).map(|item| item.header.title.as_ref()).collect::>(); + + assert_eq!(titles, ["Live channel", "Selected movie", "Selected series", "Selected episode"]); + assert_eq!( + select_target_catalog(playlist.clone(), &evaluation, false).iter().flat_map(|group| &group.channels).count(), + 7 + ); + assert_eq!(selected[0].channels[0].header.uuid, live_uuid, "target-wide selection must not rewrite Live"); + + let remote_empty = select_target_catalog(playlist, &complete_catalog_evaluation(Vec::new()), true); + assert_eq!(remote_empty.len(), 1); + assert_eq!(remote_empty[0].xtream_cluster, XtreamCluster::Live); + assert_eq!(remote_empty[0].channels[0].header.uuid, live_uuid); +} + +#[test] +fn curation_policy_truth_table_covers_a_through_h() { + let selected_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000041"); + let rejected_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000042"); + let playlist = vec![PlaylistGroup { + id: 1, + title: "Movies".intern(), + channels: vec![ + catalog_test_item("Selected", selected_uuid, PlaylistItemType::Video, XtreamCluster::Video, None), + catalog_test_item("Rejected", rejected_uuid, PlaylistItemType::Video, XtreamCluster::Video, None), + ], + xtream_cluster: XtreamCluster::Video, + }]; + let evaluation = complete_catalog_evaluation(vec![catalog_membership(selected_uuid, CurationMediaKind::Movie, 0)]); + + let cases = [ + ("A", TraktCatalogSelection::Full, true, true), + ("B", TraktCatalogSelection::Curated, true, true), + ("C", TraktCatalogSelection::Curated, true, false), + ("D", TraktCatalogSelection::Curated, false, true), + ("E", TraktCatalogSelection::Full, true, false), + ("F", TraktCatalogSelection::Full, false, true), + ("G", TraktCatalogSelection::Full, false, false), + ("H", TraktCatalogSelection::Curated, false, false), + ]; + + for (case, catalog_selection, include_base, create_category) in cases { + let config = TraktConfig::from(&TraktConfigDto { + enabled: true, + catalog_selection, + include_xtream_base_categories: include_base, + api: TraktApiConfigDto::default(), + lists: vec![TraktListConfigDto { + user: "alice".to_string(), + list_slug: "watchlist".to_string(), + category_name: Some("Curated".to_string()), + create_xtream_category: create_category, + content_type: TraktContentType::Vod, + tmdb_only: true, + fuzzy_match_threshold: 100, + }], + charts: Vec::new(), + }); + + let views = + build_curated_playlist_views(playlist.clone(), &evaluation, &CurationConfig::from(&config), false, true); + let base_items = views.base.iter().flat_map(|group| &group.channels).collect::>(); + let expected_base_count = if catalog_selection == TraktCatalogSelection::Full { 2 } else { 1 }; + assert_eq!(base_items.len(), expected_base_count, "case {case} selected catalog"); + assert!(base_items.iter().any(|item| item.header.uuid == selected_uuid), "case {case} selected subject"); + + let xtream = views.xtream.expect("complete curation Xtream view"); + let base_groups = xtream.iter().filter(|group| group.title.as_ref() == "Movies").count(); + let category_groups = xtream.iter().filter(|group| group.title.as_ref() == "Curated").count(); + assert_eq!(base_groups, usize::from(include_base), "case {case} base appearance"); + assert_eq!(category_groups, usize::from(create_category), "case {case} category appearance"); + let expected_unfiltered_items = usize::from(include_base) * expected_base_count + usize::from(create_category); + assert_eq!( + xtream.iter().map(|group| group.channels.len()).sum::(), + expected_unfiltered_items, + "case {case} unfiltered Xtream catalog" + ); + if let Some(alias) = + xtream.iter().find(|group| group.title.as_ref() == "Curated").and_then(|group| group.channels.first()) + { + assert_ne!(alias.header.uuid, selected_uuid, "case {case} alias identity"); + } + } +} + +#[test] +fn large_catalog_projection_smoke_keeps_two_explicit_views_bounded() { + const CATALOG_SIZE: usize = 10_000; + const SELECTOR_COUNT: usize = 4; + const MEMBERSHIP_STRIDE: usize = 10; + + let uuid_for = |index: usize| { + let mut bytes = [0u8; 32]; + bytes[..8].copy_from_slice(&u64::try_from(index + 1).expect("test index").to_be_bytes()); + UUIDType(bytes) + }; + let playlist = vec![PlaylistGroup { + id: 1, + title: "Movies".intern(), + channels: (0..CATALOG_SIZE) + .map(|index| { + catalog_test_item( + &format!("Movie {index}"), + uuid_for(index), + PlaylistItemType::Video, + XtreamCluster::Video, + None, + ) + }) + .collect(), + xtream_cluster: XtreamCluster::Video, + }]; + let mut memberships = Vec::with_capacity(CATALOG_SIZE / MEMBERSHIP_STRIDE * SELECTOR_COUNT); + let mut selectors = Vec::with_capacity(SELECTOR_COUNT); + for selector in 0..SELECTOR_COUNT { + let key = CurationSelectorKey(selector); + let start = memberships.len(); + for index in (selector..CATALOG_SIZE).step_by(MEMBERSHIP_STRIDE) { + memberships.push(CurationMembership { + selector_key: key, + subject_uuid: uuid_for(index), + media_kind: CurationMediaKind::Movie, + rank: Some(u32::try_from(index).expect("test rank")), + title_tiebreak: format!("movie-{index}"), + candidate_order: index, + }); + } + selectors.push(CurationSelectorSummary { + key, + reference_count: memberships.len() - start, + membership_count: memberships.len() - start, + }); + } + let evaluation = CurationEvaluation { selectors, memberships }; + let config = TraktConfig::from(&TraktConfigDto { + lists: (0..SELECTOR_COUNT) + .map(|selector| TraktListConfigDto { + user: "alice".to_string(), + list_slug: format!("list-{selector}"), + category_name: Some(format!("Curated {selector}")), + create_xtream_category: true, + content_type: TraktContentType::Vod, + tmdb_only: true, + fuzzy_match_threshold: 100, + }) + .collect(), + ..TraktConfigDto::default() + }); + + let views = build_curated_playlist_views(playlist, &evaluation, &CurationConfig::from(&config), false, true); + + assert_eq!(views.base[0].channels.len(), CATALOG_SIZE); + let xtream = views.xtream.expect("Xtream view"); + assert_eq!(xtream.len(), SELECTOR_COUNT + 1); + assert_eq!(xtream.iter().map(|group| group.channels.len()).sum::(), CATALOG_SIZE + 4_000); +} diff --git a/backend/processing/src/processor/playlist/tests/filter_rename.rs b/backend/processing/src/processor/playlist/tests/filter_rename.rs new file mode 100644 index 000000000..54db77cd0 --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/filter_rename.rs @@ -0,0 +1,527 @@ +use super::{ + apply_persist_filter, assign_channel_no_playlist, catalog_membership, catalog_test_item, + complete_catalog_evaluation, execute_pipe, execute_pipeline_on_groups, filter_skipped_clusters_from_source, + retain_playlist_items, target, test_group, InputDownloadResult, InputJobState, PlaylistRunSignals, RenameOutcome, +}; +use crate::fetched_playlist::FetchedPlaylist; +use shared::{ + foundation::{get_filter, ValueProvider}, + model::{ + ClusterFlags, ConfigRenameDto, ConfigTargetDto, ConfigTargetOptions, InputType, ItemField, PlaylistGroup, + PlaylistItem, PlaylistItemHeader, PlaylistItemType, PlaylistUpdateState, XtreamCluster, + }, + utils::Internable, +}; +use std::collections::HashSet; +use tuliprox_core::model::{ + ClusterUpdateRejection, ConfigInput, ConfigRename, ConfigTarget, FilterOutcome, TransformStage, +}; +use tuliprox_curation::{CurationMediaKind, CurationRunOutcome}; +use tuliprox_repository::{MemoryPlaylistSource, PlaylistPublicationPlan}; + +#[test] +fn execute_pipe_applies_target_bouquet_prefilter() { + let input = ConfigInput::default(); + let item1 = PlaylistItem { + header: PlaylistItemHeader { + id: "ch-1".intern(), + group: "Kids".intern(), + xtream_cluster: XtreamCluster::Live, + ..Default::default() + }, + }; + let item2 = PlaylistItem { + header: PlaylistItemHeader { + id: "ch-2".intern(), + group: "Adults".intern(), + xtream_cluster: XtreamCluster::Live, + ..Default::default() + }, + }; + let source = MemoryPlaylistSource::new(vec![ + PlaylistGroup { id: 1, title: "Kids".intern(), channels: vec![item1], xtream_cluster: XtreamCluster::Live }, + PlaylistGroup { id: 2, title: "Adults".intern(), channels: vec![item2], xtream_cluster: XtreamCluster::Live }, + ]) + .into_source(); + let mut fetched = FetchedPlaylist { input: &input, source, epg: None }; + let mut duplicates = HashSet::new(); + let target = ConfigTarget::from(&ConfigTargetDto::default()); + + let bouquet_dto = + shared::model::PlaylistClusterBouquetDto { live: Some(vec!["Kids".to_string()]), vod: None, series: None }; + let filter = + tuliprox_core::model::TargetBouquetFilter::from_dto(shared::model::TargetBouquetDto::whitelist(bouquet_dto)) + .unwrap(); + + let (mut processed, _outcome) = execute_pipe(&target, &vec![], &mut fetched, &mut duplicates, false, Some(&filter)) + .expect("target processing should succeed"); + let groups = processed.source.take_groups(); + + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].title.as_ref(), "Kids"); + assert_eq!(groups[0].channels.len(), 1); + assert_eq!(groups[0].channels[0].header.id.as_ref(), "ch-1"); +} + +#[test] +fn execute_pipe_prefilter_does_not_suppress_allowed_duplicate() { + let input = ConfigInput::default(); + // Two items with the same URL (same UUID): one in disallowed group, one in allowed group. + let item_disallowed = PlaylistItem { + header: PlaylistItemHeader { + id: "ch-1".intern(), + url: "http://provider.example/stream.ts".intern(), + group: "Adults".intern(), + xtream_cluster: XtreamCluster::Live, + ..Default::default() + }, + }; + let item_allowed = PlaylistItem { + header: PlaylistItemHeader { + id: "ch-2".intern(), + url: "http://provider.example/stream.ts".intern(), + group: "Kids".intern(), + xtream_cluster: XtreamCluster::Live, + ..Default::default() + }, + }; + let source = MemoryPlaylistSource::new(vec![ + PlaylistGroup { + id: 1, + title: "Adults".intern(), + channels: vec![item_disallowed], + xtream_cluster: XtreamCluster::Live, + }, + PlaylistGroup { + id: 2, + title: "Kids".intern(), + channels: vec![item_allowed], + xtream_cluster: XtreamCluster::Live, + }, + ]) + .into_source(); + let mut fetched = FetchedPlaylist { input: &input, source, epg: None }; + let mut duplicates = HashSet::new(); + let target = ConfigTarget::from(&ConfigTargetDto { + options: Some(ConfigTargetOptions { remove_duplicates: true, ..Default::default() }), + ..Default::default() + }); + + let bouquet_dto = + shared::model::PlaylistClusterBouquetDto { live: Some(vec!["Kids".to_string()]), vod: None, series: None }; + let filter = + tuliprox_core::model::TargetBouquetFilter::from_dto(shared::model::TargetBouquetDto::whitelist(bouquet_dto)) + .unwrap(); + + let (mut processed, _outcome) = execute_pipe(&target, &vec![], &mut fetched, &mut duplicates, false, Some(&filter)) + .expect("target processing should succeed"); + let groups = processed.source.take_groups(); + + // The allowed item must be retained because the rejected item did not consume the duplicate UUID slot. + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].title.as_ref(), "Kids"); + assert_eq!(groups[0].channels.len(), 1); + assert_eq!(groups[0].channels[0].header.id.as_ref(), "ch-2"); +} + +#[test] +fn filter_skipped_clusters_removes_cached_groups() { + use tuliprox_core::model::{ConfigInputFlags, ConfigInputOptions}; + let live_item = PlaylistItem { + header: shared::model::PlaylistItemHeader { xtream_cluster: XtreamCluster::Live, ..Default::default() }, + }; + let vod_item = PlaylistItem { + header: shared::model::PlaylistItemHeader { xtream_cluster: XtreamCluster::Video, ..Default::default() }, + }; + + let groups = vec![ + PlaylistGroup { id: 1, title: "Live".intern(), channels: vec![live_item], xtream_cluster: XtreamCluster::Live }, + PlaylistGroup { id: 2, title: "Vod".intern(), channels: vec![vod_item], xtream_cluster: XtreamCluster::Video }, + ]; + + let source = MemoryPlaylistSource::new(groups).into_source(); + let input = ConfigInput { + name: "skip_live".intern(), + input_type: InputType::Xtream, + options: Some(ConfigInputOptions { + flags: ConfigInputFlags::SkipLive.into(), + ..ConfigInputOptions::defaults().clone() + }), + ..ConfigInput::default() + }; + + let mut filtered = filter_skipped_clusters_from_source(source, &input); + let filtered_groups = filtered.take_groups(); + assert_eq!(filtered_groups.len(), 1); + assert_eq!(filtered_groups[0].xtream_cluster, XtreamCluster::Video); +} + +#[test] +fn quality_rejection_keeps_a_usable_input_ready() { + let mut result = InputDownloadResult { + authoritative_library: false, + errors: Vec::new(), + source: MemoryPlaylistSource::new(vec![test_group(XtreamCluster::Video, "retained-vod", "provider-a")]) + .into_source(), + storage_error: None, + partial: false, + quality_rejections: vec![ClusterUpdateRejection { + cluster: XtreamCluster::Video, + current_count: 12_543, + candidate_count: 217, + threshold: 90, + quality: 1, + }], + accepted_empty_clusters: ClusterFlags::empty(), + input_telemetry: None, + }; + + assert_eq!(result.job_state(), InputJobState::Ready); + assert!(result.errors.is_empty()); + assert!(!result.partial); + assert_eq!(result.quality_rejections.len(), 1); +} + +#[test] +fn quality_rejection_marks_the_run_partial_without_reusing_stalker_partial() { + let quality_rejection = PlaylistRunSignals { has_quality_rejections: true, ..PlaylistRunSignals::default() }; + assert!(!quality_rejection.has_pending_stalker_refresh); + assert_eq!(quality_rejection.state(), PlaylistUpdateState::Partial); + + let stalker_partial = PlaylistRunSignals { has_pending_stalker_refresh: true, ..PlaylistRunSignals::default() }; + assert!(!stalker_partial.has_quality_rejections); + assert_eq!(stalker_partial.state(), PlaylistUpdateState::Partial); + + let technical_failure = PlaylistRunSignals { has_error: true, ..quality_rejection }; + assert_eq!(technical_failure.state(), PlaylistUpdateState::Failure); +} + +pub(in crate::processor::playlist::tests) fn make_test_item(name: &str, item_type: PlaylistItemType) -> PlaylistItem { + let header = + PlaylistItemHeader { name: name.into(), group: "Test Group".intern(), item_type, ..Default::default() }; + PlaylistItem { header } +} + +#[test] +fn test_filter_evalutes_correctly() { + let filter = get_filter(r#"name ~ "Allowed""#, None).unwrap(); + + let allowed_item = make_test_item("Allowed Channel", PlaylistItemType::Live); + let denied_item = make_test_item("Denied Channel", PlaylistItemType::Live); + + let allowed_provider = ValueProvider { pli: &allowed_item, match_as_ascii: false }; + let denied_provider = ValueProvider { pli: &denied_item, match_as_ascii: false }; + + assert!(filter.filter(&allowed_provider)); + assert!(!filter.filter(&denied_provider)); +} + +#[test] +fn test_filter_with_type_comparison() { + let filter = get_filter("type = vod", None).unwrap(); + + let vod_item = make_test_item("Test Movie", PlaylistItemType::Video); + let live_item = make_test_item("Test Channel", PlaylistItemType::Live); + + let vod_provider = ValueProvider { pli: &vod_item, match_as_ascii: false }; + let live_provider = ValueProvider { pli: &live_item, match_as_ascii: false }; + + assert!(filter.filter(&vod_provider)); + assert!(!filter.filter(&live_provider)); +} + +#[test] +fn playlist_retention_reports_filter_counts() { + let groups = vec![PlaylistGroup { + id: 1, + title: "Test Group".intern(), + channels: vec![ + make_test_item("Allowed", PlaylistItemType::Live), + make_test_item("Denied", PlaylistItemType::Live), + ], + xtream_cluster: XtreamCluster::Live, + }]; + let mut source = MemoryPlaylistSource::new(groups).into_source(); + + let (filtered, outcome) = retain_playlist_items(&mut source, |item| item.header.name.as_ref() == "Allowed"); + + assert_eq!(outcome, FilterOutcome { inspected: 2, retained: 1, removed: 1 }); + assert_eq!(filtered.expect("one item should remain")[0].channels[0].header.name.as_ref(), "Allowed"); +} + +#[test] +fn filter_stage_can_remove_every_item() { + let groups = vec![PlaylistGroup { + id: 1, + title: "Test Group".intern(), + channels: vec![make_test_item("Denied", PlaylistItemType::Live)], + xtream_cluster: XtreamCluster::Live, + }]; + let mut target = ConfigTarget::from(&ConfigTargetDto::default()); + target.filter = get_filter(r#"name ~ "Allowed""#, None).expect("filter should parse").into(); + + let (groups, outcome) = execute_pipeline_on_groups(groups, &target, &[TransformStage::Filter]); + + assert!(groups.is_empty()); + assert_eq!(outcome.filter, Some(FilterOutcome { inspected: 1, retained: 0, removed: 1 })); +} + +#[test] +fn missing_processing_filter_skips_filter_stage() { + let groups = vec![PlaylistGroup { + id: 1, + title: "Test Group".intern(), + channels: vec![make_test_item("Allowed", PlaylistItemType::Live)], + xtream_cluster: XtreamCluster::Live, + }]; + let target = ConfigTarget::from(&ConfigTargetDto::default()); + + let (groups, outcome) = execute_pipeline_on_groups(groups, &target, &[TransformStage::Filter]); + + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].channels.len(), 1); + assert!(outcome.filter.is_none()); +} + +#[test] +fn missing_processing_filter_preserves_filter_stage_group_normalization() { + let mut first = make_test_item("One", PlaylistItemType::Live); + first.header.group = "News".intern(); + let mut second = make_test_item("Two", PlaylistItemType::Live); + second.header.group = "news".intern(); + let groups = vec![ + PlaylistGroup { id: 1, title: "News".intern(), channels: vec![first], xtream_cluster: XtreamCluster::Live }, + PlaylistGroup { id: 2, title: "news".intern(), channels: vec![second], xtream_cluster: XtreamCluster::Live }, + ]; + let target = ConfigTarget::from(&ConfigTargetDto::default()); + + let (groups, outcome) = execute_pipeline_on_groups(groups, &target, &[TransformStage::Filter]); + + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].channels.len(), 2); + assert!(outcome.filter.is_none()); +} + +#[test] +fn pipeline_reports_filter_and_rename_outcomes() { + let groups = vec![PlaylistGroup { + id: 1, + title: "Test Group".intern(), + channels: vec![ + make_test_item("Allowed", PlaylistItemType::Live), + make_test_item("Denied", PlaylistItemType::Live), + ], + xtream_cluster: XtreamCluster::Live, + }]; + let mut target = ConfigTarget::from(&ConfigTargetDto::default()); + target.filter = get_filter(r#"name ~ "Allowed""#, None).expect("filter should parse").into(); + target.rename = Some(vec![ConfigRename::from(&ConfigRenameDto { + field: ItemField::Name, + pattern: "Allowed".to_string(), + new_name: "Renamed".to_string(), + t_pattern: None, + })]); + + let (groups, outcome) = + execute_pipeline_on_groups(groups, &target, &[TransformStage::Filter, TransformStage::Rename]); + + assert_eq!(groups[0].channels[0].header.name.as_ref(), "Renamed"); + assert_eq!(outcome.filter, Some(FilterOutcome { inspected: 2, retained: 1, removed: 1 })); + assert_eq!(outcome.rename, Some(RenameOutcome { inspected: 1, changed_items: 1, changed_fields: 1 })); +} + +#[test] +fn assign_channel_no_playlist_preserves_non_zero_chno() { + let mut groups = vec![ + PlaylistGroup { + id: 1, + title: "Group A".intern(), + channels: vec![ + PlaylistItem { header: PlaylistItemHeader { name: "A".intern(), chno: 10, ..Default::default() } }, + PlaylistItem { header: PlaylistItemHeader { name: "B".intern(), chno: 0, ..Default::default() } }, + ], + xtream_cluster: XtreamCluster::Live, + }, + PlaylistGroup { + id: 2, + title: "Group C".intern(), + channels: vec![ + PlaylistItem { header: PlaylistItemHeader { name: "C".intern(), chno: 1, ..Default::default() } }, + PlaylistItem { header: PlaylistItemHeader { name: "D".intern(), chno: 0, ..Default::default() } }, + ], + xtream_cluster: XtreamCluster::Live, + }, + ]; + + assign_channel_no_playlist(&mut groups); + + // Non-zero chno values must be preserved + assert_eq!(groups[0].channels[0].header.chno, 10); + assert_eq!(groups[1].channels[0].header.chno, 1); +} + +#[test] +fn assign_channel_no_playlist_assigns_zero_chno_only() { + let mut groups = vec![PlaylistGroup { + id: 1, + title: "Group A".intern(), + channels: vec![ + PlaylistItem { header: PlaylistItemHeader { name: "A".intern(), chno: 0, ..Default::default() } }, + PlaylistItem { header: PlaylistItemHeader { name: "B".intern(), chno: 0, ..Default::default() } }, + PlaylistItem { header: PlaylistItemHeader { name: "C".intern(), chno: 0, ..Default::default() } }, + ], + xtream_cluster: XtreamCluster::Live, + }]; + + assign_channel_no_playlist(&mut groups); + + // All zero-chno channels should get assigned numbers starting at 1 + assert_eq!(groups[0].channels[0].header.chno, 1); + assert_eq!(groups[0].channels[1].header.chno, 2); + assert_eq!(groups[0].channels[2].header.chno, 3); +} + +#[test] +fn assign_channel_no_playlist_skips_existing_nonzero_numbers() { + let mut groups = vec![PlaylistGroup { + id: 1, + title: "Group A".intern(), + channels: vec![ + PlaylistItem { header: PlaylistItemHeader { name: "A".intern(), chno: 5, ..Default::default() } }, + PlaylistItem { header: PlaylistItemHeader { name: "B".intern(), chno: 0, ..Default::default() } }, + PlaylistItem { header: PlaylistItemHeader { name: "C".intern(), chno: 2, ..Default::default() } }, + PlaylistItem { header: PlaylistItemHeader { name: "D".intern(), chno: 0, ..Default::default() } }, + ], + xtream_cluster: XtreamCluster::Live, + }]; + + assign_channel_no_playlist(&mut groups); + + // Existing non-zero numbers (2, 5) must be skipped when assigning new numbers + assert_eq!(groups[0].channels[0].header.chno, 5); // preserved + assert_eq!(groups[0].channels[2].header.chno, 2); // preserved + // B gets 1 (smallest available), D gets 3 (next available after 1 and existing 2) + assert_eq!(groups[0].channels[1].header.chno, 1); + assert_eq!(groups[0].channels[3].header.chno, 3); +} + +#[test] +fn assign_channel_no_playlist_assigns_following_group_order() { + let mut groups = vec![ + PlaylistGroup { + id: 1, + title: "Group 1".intern(), + channels: vec![ + PlaylistItem { header: PlaylistItemHeader { name: "A".intern(), chno: 0, ..Default::default() } }, + PlaylistItem { header: PlaylistItemHeader { name: "B".intern(), chno: 0, ..Default::default() } }, + ], + xtream_cluster: XtreamCluster::Live, + }, + PlaylistGroup { + id: 2, + title: "Group 2".intern(), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { name: "C".intern(), chno: 0, ..Default::default() }, + }], + xtream_cluster: XtreamCluster::Live, + }, + ]; + + assign_channel_no_playlist(&mut groups); + + // Numbers should follow iteration order across groups: A=1, B=2, C=3 + assert_eq!(groups[0].channels[0].header.chno, 1); + assert_eq!(groups[0].channels[1].header.chno, 2); + assert_eq!(groups[1].channels[0].header.chno, 3); +} + +#[test] +fn persist_filter_can_select_a_base_group() { + let mut target = ConfigTarget::from(&ConfigTargetDto::default()); + target.filter.persist = Some(get_filter(r#"Group = "Base""#, None).expect("persist filter")); + let mut playlist = vec![ + PlaylistGroup { + id: 1, + title: "Base".intern(), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { group: "Base".intern(), ..Default::default() }, + }], + xtream_cluster: XtreamCluster::Video, + }, + PlaylistGroup { + id: 2, + title: "Curated".intern(), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { group: "Curated".intern(), ..Default::default() }, + }], + xtream_cluster: XtreamCluster::Video, + }, + ]; + + apply_persist_filter(&target, &mut playlist); + + assert_eq!(playlist.len(), 1); + assert_eq!(playlist[0].title.as_ref(), "Base"); +} + +#[test] +fn tmdb_complete_target_selection_preserves_live_and_keeps_xtream_aliases_out_of_normal_outputs() { + for has_xtream in [false, true] { + let mut value = serde_json::json!({"name": "discovery", "output": [{"type": "m3u"}], + "curation": {"catalog_selection": "curated", "tmdb": {"trending": [{"kind": "movie", "time_window": "week", "limit": 100, "create_xtream_category": has_xtream, "category_name": "TMDB"}]}}}); + if has_xtream { + value["output"].as_array_mut().unwrap().push(serde_json::json!({"type": "xtream"})); + } + let mut dto: ConfigTargetDto = serde_json::from_value(value).unwrap(); + dto.prepare(1, None, None).unwrap(); + let target = ConfigTarget::from(&dto); + let id = shared::utils::hash_string("selected"); + let live_id = shared::utils::hash_string("live"); + let playlist = vec![ + PlaylistGroup { + id: 1, + title: "Live".intern(), + xtream_cluster: XtreamCluster::Live, + channels: vec![catalog_test_item("Live", live_id, PlaylistItemType::Live, XtreamCluster::Live, None)], + }, + PlaylistGroup { + id: 2, + title: "Movies".intern(), + xtream_cluster: XtreamCluster::Video, + channels: vec![ + catalog_test_item("Selected", id, PlaylistItemType::Video, XtreamCluster::Video, None), + catalog_test_item( + "Other", + shared::utils::hash_string("other"), + PlaylistItemType::Video, + XtreamCluster::Video, + None, + ), + ], + }, + ]; + let config = target.effective_curation().unwrap(); + let outcome = CurationRunOutcome::Complete(complete_catalog_evaluation(vec![catalog_membership( + id, + CurationMediaKind::Movie, + 0, + )])); + let views = target::curation_playlist_views(&target, playlist.clone(), &config, outcome); + assert_eq!( + views.base.iter().flat_map(|g| &g.channels).map(|i| i.header.uuid).collect::>(), + [live_id, id] + ); + if has_xtream { + let groups = views.xtream.unwrap(); + let projected = groups.iter().find(|g| g.title.as_ref() == "TMDB").unwrap(); + assert_ne!(projected.channels[0].header.uuid, id); + } else { + assert!(views.xtream.is_none()); + } + let empty = CurationRunOutcome::Complete(complete_catalog_evaluation(Vec::new())); + let empty_views = target::curation_playlist_views(&target, playlist, &config, empty); + assert_eq!(empty_views.base.len(), 1); + assert_eq!(empty_views.base[0].xtream_cluster, XtreamCluster::Live); + assert_ne!(empty_views.publication_plan, PlaylistPublicationPlan::Ordinary); + } +} diff --git a/backend/processing/src/processor/playlist/tests/identity_serialization.rs b/backend/processing/src/processor/playlist/tests/identity_serialization.rs new file mode 100644 index 000000000..6d8374a1c --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/identity_serialization.rs @@ -0,0 +1,334 @@ +use super::{ + apply_staged_overlay_groups, exec_rename, execute_pipe, map_channel, playlist_update_run_result, + report_input_job_completion, test_group, InputJobState, PlaylistRunCollectSink, +}; +use crate::fetched_playlist::FetchedPlaylist; +use shared::{ + error::TuliproxError, + foundation::{get_filter, MapperScript}, + model::{ + ClusterFlags, ConfigRenameDto, ConfigTargetDto, EventMessage, FieldSetAccessor, InputType, ItemField, + M3uPlaylistItem, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, + PlaylistUpdateRunId, PlaylistUpdateRunOrder, PlaylistUpdateState, XtreamCluster, XtreamPlaylistItem, + }, + utils::Internable, +}; +use std::collections::HashSet; +use tuliprox_core::model::{ + CompiledMapping, CompiledMappingRule, ConfigInput, ConfigInputFlags, ConfigInputOptions, ConfigRename, + ConfigTarget, MappingProgram, +}; +use tuliprox_repository::MemoryPlaylistSource; + +#[test] +fn playlist_update_run_failure_text_makes_no_unverified_previous_data_claim() { + let events = PlaylistRunCollectSink::default(); + let run_id = PlaylistUpdateRunId::from("neutral-failure-run"); + let execution_order = PlaylistUpdateRunOrder::from(18); + let failed_first_run = playlist_update_run_result( + 7, + InputJobState::Failed, + vec![TuliproxError::RepositoryPlaylist("https://first:secret@provider.example/playlist".to_string())], + false, + ); + let persisted_then_failed = playlist_update_run_result( + 8, + InputJobState::Ready, + vec![TuliproxError::RepositoryPlaylist("https://epg:secret@provider.example/guide".to_string())], + false, + ); + + report_input_job_completion(&events, &run_id, execution_order, &failed_first_run); + report_input_job_completion(&events, &run_id, execution_order, &persisted_then_failed); + + let emitted = events.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let progress = emitted + .iter() + .filter_map(|event| match event { + EventMessage::PlaylistUpdateProgress(progress) => Some(progress), + _ => None, + }) + .collect::>(); + assert_eq!(progress.len(), 2); + assert_eq!(progress[0].input_id, Some(7)); + assert_eq!(progress[0].message, "Input 'input-7' failed during update"); + assert_eq!(progress[1].input_id, Some(8)); + assert_eq!(progress[1].message, "Input 'input-8' failed during update"); + assert!(progress.iter().all(|event| event.run_id.as_ref() == Some(&run_id))); + assert!(progress.iter().all(|event| event.execution_order == Some(execution_order))); + assert!(progress.iter().all(|event| event.state == Some(PlaylistUpdateState::Failure))); + assert!(progress.iter().all(|event| !event.message.contains("secret"))); + assert!(progress.iter().all(|event| !event.message.contains("previous accepted data"))); +} + +pub(in crate::processor::playlist::tests) fn serialize_without_trailing_fields( + value: &T, + trailing_fields: &[u8], +) -> Vec { + let mut encoded = rmp_serde::to_vec(value).expect("playlist item should serialize"); + for expected in trailing_fields { + assert_eq!(encoded.pop(), Some(*expected), "unexpected trailing MessagePack field"); + } + let removed = trailing_fields.len(); + match encoded[0] { + marker @ 0x92..=0x9f => { + let len = usize::from(marker - 0x90); + assert!(len >= removed, "trailing field count exceeds MessagePack sequence length"); + encoded[0] = 0x90 + u8::try_from(len - removed).unwrap_or_default(); + } + 0xdc => { + let len = u16::from_be_bytes([encoded[1], encoded[2]]); + let removed = u16::try_from(removed).unwrap_or(u16::MAX); + assert!(len >= removed, "trailing field count exceeds MessagePack sequence length"); + encoded[1..3].copy_from_slice(&(len - removed).to_be_bytes()); + } + 0xdd => { + let len = u32::from_be_bytes([encoded[1], encoded[2], encoded[3], encoded[4]]); + let removed = u32::try_from(removed).unwrap_or(u32::MAX); + assert!(len >= removed, "trailing field count exceeds MessagePack sequence length"); + encoded[1..5].copy_from_slice(&(len - removed).to_be_bytes()); + } + marker => panic!("unexpected MessagePack sequence marker {marker:#x}"), + } + encoded +} + +#[test] +fn rename_preserves_input_stream_id_captured_at_target_boundary() { + let mut item = PlaylistItem { + header: PlaylistItemHeader { + id: "origin-alpha".intern(), + url: "http://provider.example/channel.m3u8".intern(), + ..Default::default() + }, + }; + item.header.freeze_input_stream_id(); + let rename = ConfigRename::from(&ConfigRenameDto { + field: ItemField::Url, + pattern: "provider".to_string(), + new_name: "target".to_string(), + t_pattern: None, + }); + + exec_rename(&mut item, Some(&vec![rename])); + + assert_eq!(item.header.url.as_ref(), "http://target.example/channel.m3u8"); + assert_eq!(item.header.input_stream_id.as_ref(), "origin-alpha"); +} + +#[test] +fn mapper_changes_id_without_changing_frozen_input_stream_id() { + let mut item = PlaylistItem { + header: PlaylistItemHeader { id: "origin-alpha".intern(), name: "Channel".intern(), ..Default::default() }, + }; + item.header.freeze_input_stream_id(); + let mapping = CompiledMapping { + rules: vec![CompiledMappingRule { + name: None, + filter: get_filter(r#"name ~ ".*""#, None).expect("filter should parse"), + program: MappingProgram::Script( + MapperScript::parse(r#"@id = "target-id""#, None).expect("mapper should parse"), + ), + }], + ..Default::default() + }; + + let outcome = map_channel(item, &mapping); + + assert_eq!(outcome.matched_rules, 1); + assert_eq!(outcome.channel.header.id.as_ref(), "target-id"); + assert_eq!(outcome.channel.header.input_stream_id.as_ref(), "origin-alpha"); +} + +#[test] +fn mapper_cannot_resurrect_missing_legacy_input_stream_id_from_target_id() { + let mut source = PlaylistItem { + header: PlaylistItemHeader { + id: "80510".intern(), + url: "http://provider.example/live/user/pass/80510.ts".intern(), + input_name: "input".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..Default::default() + }, + }; + source.header.freeze_input_stream_id(); + let mut legacy_xtream = XtreamPlaylistItem::from(&source); + legacy_xtream.provider_id = 0; + legacy_xtream.input_stream_id = "".intern(); + legacy_xtream.url = "http://provider.example/live/channel.m3u8".intern(); + let mut legacy_item = PlaylistItem::from(&legacy_xtream); + legacy_item.header.freeze_input_stream_id(); + let mapping = CompiledMapping { + rules: vec![CompiledMappingRule { + name: None, + filter: get_filter(r#"name ~ ".*""#, None).expect("filter should parse"), + program: MappingProgram::Script( + MapperScript::parse(r#"@id = "target-id""#, None).expect("mapper should parse"), + ), + }], + ..Default::default() + }; + + let outcome = map_channel(legacy_item, &mapping); + let materialized_m3u = M3uPlaylistItem::from(&outcome.channel); + let materialized_xtream = XtreamPlaylistItem::from(&outcome.channel); + + assert_eq!(outcome.matched_rules, 1); + assert_eq!(outcome.channel.header.id.as_ref(), "target-id"); + assert_eq!(outcome.channel.get_input_stream_id(), None); + assert!(materialized_m3u.provider_id.is_empty()); + assert_eq!(materialized_m3u.get_input_stream_id(), None); + assert_eq!(materialized_xtream.provider_id, 0); + assert_eq!(materialized_xtream.get_input_stream_id(), None); +} + +#[test] +fn execute_pipe_freezes_input_stream_id_without_rename_or_mapper() { + let input = ConfigInput::default(); + let item = PlaylistItem { header: PlaylistItemHeader { id: "origin-alpha".intern(), ..Default::default() } }; + let source = MemoryPlaylistSource::new(vec![PlaylistGroup { + id: 1, + title: "Group".intern(), + channels: vec![item], + xtream_cluster: XtreamCluster::Live, + }]) + .into_source(); + let mut fetched = FetchedPlaylist { input: &input, source, epg: None }; + let mut duplicates = HashSet::new(); + let target = ConfigTarget::from(&ConfigTargetDto::default()); + + let (mut processed, _outcome) = execute_pipe(&target, &vec![], &mut fetched, &mut duplicates, false, None) + .expect("target processing should succeed"); + let mut groups = processed.source.take_groups(); + + assert_eq!(groups[0].channels[0].header.input_stream_id.as_ref(), "origin-alpha"); + assert!(groups[0].channels[0].header.set_field("id", "late-target-id")); + assert_eq!(groups[0].channels[0].header.input_stream_id.as_ref(), "origin-alpha"); +} + +#[test] +fn legacy_messagepack_playlist_items_default_missing_input_stream_id() { + let mut source = PlaylistItem { + header: PlaylistItemHeader { + id: "origin-alpha".intern(), + url: "http://provider.example/live/user/pass/80510.ts".intern(), + input_name: "input".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..Default::default() + }, + }; + + let header_bytes = serialize_without_trailing_fields(&source.header, &[0xc0, 0xa0]); + let decoded_header: PlaylistItemHeader = + rmp_serde::from_slice(&header_bytes).expect("legacy header should deserialize"); + assert!(decoded_header.input_stream_id.is_empty()); + assert_eq!(decoded_header.get_input_stream_id(), None); + assert_eq!(decoded_header.upstream_user_agent, None); + let mut decoded_header = decoded_header; + decoded_header.freeze_input_stream_id(); + assert_eq!(decoded_header.get_input_stream_id().as_deref(), Some("origin-alpha")); + + source.header.freeze_input_stream_id(); + let mut m3u_item = M3uPlaylistItem::from(&source); + m3u_item.input_stream_id = "".intern(); + let m3u_bytes = serialize_without_trailing_fields(&m3u_item, &[0xc0, 0xa0]); + let decoded_m3u: M3uPlaylistItem = rmp_serde::from_slice(&m3u_bytes).expect("legacy M3U item should deserialize"); + assert!(decoded_m3u.input_stream_id.is_empty()); + assert_eq!(decoded_m3u.get_input_stream_id().as_deref(), Some("origin-alpha")); + assert_eq!(decoded_m3u.upstream_user_agent, None); + + let mut xtream_item = XtreamPlaylistItem::from(&source); + xtream_item.input_stream_id = "".intern(); + let xtream_bytes = serialize_without_trailing_fields(&xtream_item, &[0xc0, 0xa0]); + let decoded_xtream: XtreamPlaylistItem = + rmp_serde::from_slice(&xtream_bytes).expect("legacy Xtream item should deserialize"); + assert!(decoded_xtream.input_stream_id.is_empty()); + assert_eq!(decoded_xtream.get_input_stream_id().as_deref(), Some("80510")); + assert_eq!(decoded_xtream.upstream_user_agent, None); +} + +#[test] +fn previous_messagepack_playlist_items_default_missing_upstream_user_agent() { + let source = PlaylistItem { + header: PlaylistItemHeader { + id: "80510".intern(), + input_stream_id: "origin-alpha".intern(), + ..Default::default() + }, + }; + + let header: PlaylistItemHeader = rmp_serde::from_slice(&serialize_without_trailing_fields(&source.header, &[0xc0])) + .expect("previous header should deserialize"); + let m3u: M3uPlaylistItem = + rmp_serde::from_slice(&serialize_without_trailing_fields(&M3uPlaylistItem::from(&source), &[0xc0])) + .expect("previous M3U item should deserialize"); + let xtream: XtreamPlaylistItem = + rmp_serde::from_slice(&serialize_without_trailing_fields(&XtreamPlaylistItem::from(&source), &[0xc0])) + .expect("previous Xtream item should deserialize"); + + assert_eq!(header.input_stream_id.as_ref(), "origin-alpha"); + assert_eq!(m3u.input_stream_id.as_ref(), "origin-alpha"); + assert_eq!(xtream.input_stream_id.as_ref(), "origin-alpha"); + assert_eq!(header.upstream_user_agent, None); + assert_eq!(m3u.upstream_user_agent, None); + assert_eq!(xtream.upstream_user_agent, None); +} + +#[test] +fn messagepack_playlist_items_preserve_upstream_user_agent() -> Result<(), Box> { + let source = PlaylistItem { + header: PlaylistItemHeader { upstream_user_agent: Some("Provider-UA".intern()), ..Default::default() }, + }; + + let header: PlaylistItemHeader = rmp_serde::from_slice(&rmp_serde::to_vec(&source.header)?)?; + let m3u: M3uPlaylistItem = rmp_serde::from_slice(&rmp_serde::to_vec(&M3uPlaylistItem::from(&source))?)?; + let xtream: XtreamPlaylistItem = rmp_serde::from_slice(&rmp_serde::to_vec(&XtreamPlaylistItem::from(&source))?)?; + + assert_eq!(header.upstream_user_agent.as_deref(), Some("Provider-UA")); + assert_eq!(m3u.upstream_user_agent.as_deref(), Some("Provider-UA")); + assert_eq!(xtream.upstream_user_agent.as_deref(), Some("Provider-UA")); + Ok(()) +} + +#[test] +fn staged_xtream_overlay_recomputes_uuid_on_group_load() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + options: Some(ConfigInputOptions { + flags: ConfigInputFlags::XtreamLiveStreamUsePrefix.into(), + ..ConfigInputOptions::defaults().clone() + }), + ..Default::default() + }; + let mut staged_live = test_group(XtreamCluster::Live, "staged-live", "staged"); + staged_live.channels[0].header.id = "11203".intern(); + staged_live.channels[0].header.freeze_input_stream_id(); + staged_live.channels[0].header.url = "http://iptvhost.example/live/fake-user/fake-pass/11203.ts".intern(); + staged_live.channels[0].header.gen_uuid(); + let old_uuid = *staged_live.channels[0].header.get_uuid(); + + let mut groups = apply_staged_overlay_groups(&provider, ClusterFlags::Live, Vec::new(), vec![staged_live]); + + assert_eq!(groups[0].channels[0].header.input_name.as_ref(), "provider"); + let reconstructed_url = "http://provider.example/live/real-user/real-pass/11203.ts"; + assert_eq!(groups[0].channels[0].header.url.as_ref(), reconstructed_url); + assert_eq!(groups[0].channels[0].header.get_input_stream_id().as_deref(), Some("11203")); + + groups[0].on_load(); + + let expected_uuid = shared::utils::generate_runtime_playlist_uuid( + "provider", + "11203", + shared::model::PlaylistItemType::Live, + reconstructed_url, + ); + assert_ne!(groups[0].channels[0].header.get_uuid(), &old_uuid); + assert_eq!(groups[0].channels[0].header.get_uuid(), &expected_uuid); + assert_eq!(groups[0].channels[0].header.get_input_stream_id().as_deref(), Some("11203")); +} diff --git a/backend/processing/src/processor/playlist/tests/mapping_stage.rs b/backend/processing/src/processor/playlist/tests/mapping_stage.rs new file mode 100644 index 000000000..72edad42c --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/mapping_stage.rs @@ -0,0 +1,597 @@ +use super::*; +use arc_swap::{ArcSwap, ArcSwapOption}; +use shared::model::{ConfigPaths, EpgSmartMatchConfigDto}; +use std::sync::Arc; +use tempfile::tempdir; +use tokio::runtime::Runtime; +use tuliprox_core::{ + model::{ + EpgConfig, EpgSmartMatchConfig, IcsEpgSourceConfig, MediaToolCapabilities, PersistedEpgSource, + PersistedEpgSourceKind, SourcesConfig, + }, + utils::FileLockManager, +}; + +pub(super) fn build_mapping(id: &str, stage: MappingStage, script: &str) -> CompiledMapping { + CompiledMapping { + id: id.to_string(), + match_as_ascii: false, + stage, + rules: vec![CompiledMappingRule { + name: None, + filter: get_filter(r#"name ~ ".*""#, None).expect("filter parses"), + program: MappingProgram::Script(MapperScript::parse(script, None).expect("script parses")), + }], + counters: vec![], + templates: None, + } +} + +pub(super) fn build_target(mappings: Vec, remove_duplicates: bool) -> ConfigTarget { + let dto = ConfigTargetDto { + options: if remove_duplicates { + Some(ConfigTargetOptions { remove_duplicates, ..Default::default() }) + } else { + None + }, + ..Default::default() + }; + let mut target = ConfigTarget::from(&dto); + target.mapping = Arc::new(ArcSwapOption::from(Some(Arc::new(CompiledTargetMappings::new( + mappings.into_iter().map(Arc::new).collect(), + ))))); + target +} + +/// Pinned to `NoopSink` rather than staying generic: these tests +/// exercise the pipeline, not the bus, and an inferred sink type +/// would just make every call site name one. +fn processing_context() -> PlaylistProcessingContext { + let paths = ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + }; + let config = AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(paths)), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + }; + PlaylistProcessingContext { + client: reqwest::Client::new(), + run_id: "m3u-alias-test-run".into(), + execution_order: PlaylistUpdateRunOrder::from(1), + config: Arc::new(config), + user_targets: Arc::new(ProcessTargets { + enabled: false, + inputs: Vec::new(), + targets: Vec::new(), + target_names: Vec::new(), + }), + events: shared::model::NoopSink, + playlist_state: None, + disabled_headers: None, + processed_inputs: Arc::new(Mutex::new(HashSet::new())), + input_completions: Arc::new(Mutex::new(HashMap::new())), + input_locks: Arc::new(Mutex::new(HashMap::new())), + provider_manager: None, + metadata_manager: None, + pre_processed_inputs: None, + input_refresh: None, + library_update_mode: LibraryUpdateMode::ExistingCatalog, + stalker_refresh_mode: StalkerRefreshMode::Complete, + partial_refresh: Arc::new(std::sync::atomic::AtomicBool::new(false)), + had_quality_rejections: Arc::new(std::sync::atomic::AtomicBool::new(false)), + } +} + +#[tokio::test] +async fn m3u_alias_playlist_is_downloaded_and_indexed_separately() { + let temp = tempfile::tempdir().expect("temp dir should be created"); + let primary_playlist_path = temp.path().join("primary.m3u"); + let alias_playlist_path = temp.path().join("backup.m3u"); + tokio::fs::write( + &primary_playlist_path, + "#EXTM3U\n#EXTINF:-1 tvg-id=\"323\",Channel\nhttp://stream.example:4000/323/mono.m3u8?token=primary-stream-token\n", + ) + .await + .expect("primary fixture should be written"); + tokio::fs::write( + &alias_playlist_path, + "#EXTM3U\n#EXTINF:-1 tvg-id=\"323\",Channel\nhttp://stream.example:4000/323/mono.m3u8?token=backup-stream-token\n", + ) + .await + .expect("alias fixture should be written"); + + let ctx = processing_context(); + let config = + Config { storage_dir: temp.path().join("storage").to_string_lossy().into_owned(), ..Config::default() }; + ctx.config.config.store(Arc::new(config)); + let input = Arc::new(ConfigInput { + id: 1, + name: "primary-account".intern(), + input_type: InputType::M3u, + url: primary_playlist_path.to_string_lossy().into_owned(), + enabled: true, + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "backup-account".intern(), + url: alias_playlist_path.to_string_lossy().into_owned(), + username: None, + password: None, + priority: 1, + max_connections: 1, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }); + + let InputDownloadResult { + errors, source: mut primary_playlist, storage_error, partial, quality_rejections, .. + } = download_input(&ctx, &input, false).await; + + assert!(errors.is_empty(), "unexpected download errors: {errors:?}"); + assert!(storage_error.is_none(), "unexpected primary storage error: {storage_error:?}"); + assert!(!partial); + assert!(quality_rejections.is_empty()); + assert!(!primary_playlist.is_empty()); + let alias_url = tuliprox_repository::load_input_m3u_stream_url( + &ctx.config, + &"backup-account".intern(), + "http://stream.example:4000/323/mono.m3u8?token=primary-stream-token", + ) + .await + .expect("alias URL lookup should succeed"); + assert_eq!(alias_url.as_deref(), Some("http://stream.example:4000/323/mono.m3u8?token=backup-stream-token")); +} + +#[tokio::test] +async fn failed_m3u_alias_is_retried_after_primary_input_is_processed() { + let temp = tempfile::tempdir().expect("temp dir should be created"); + let primary_playlist_path = temp.path().join("main.m3u"); + let alias_playlist_path = temp.path().join("retry.m3u"); + tokio::fs::write( + &primary_playlist_path, + "#EXTM3U\n#EXTINF:-1 tvg-id=\"323\",Channel\nhttp://stream.example:4000/323/mono.m3u8?token=main-stream-token\n", + ) + .await + .expect("primary fixture should be written"); + + let ctx = processing_context(); + let config = + Config { storage_dir: temp.path().join("storage").to_string_lossy().into_owned(), ..Config::default() }; + ctx.config.config.store(Arc::new(config)); + let input = Arc::new(ConfigInput { + id: 1, + name: "main-account".intern(), + input_type: InputType::M3u, + url: primary_playlist_path.to_string_lossy().into_owned(), + enabled: true, + aliases: Some(vec![ConfigInputAlias { + id: 2, + name: "retry-account".intern(), + url: alias_playlist_path.to_string_lossy().into_owned(), + username: None, + password: None, + priority: 1, + max_connections: 1, + exp_date: None, + enabled: true, + stalker: None, + }]), + ..ConfigInput::default() + }); + + let InputDownloadResult { + errors: first_errors, + source: mut first_playlist, + storage_error: first_storage_error, + partial: first_partial, + quality_rejections: first_quality_rejections, + .. + } = download_input(&ctx, &input, false).await; + + assert!(!first_errors.is_empty(), "missing alias should report an error"); + assert!(first_storage_error.is_none(), "primary storage should succeed: {first_storage_error:?}"); + assert!(!first_partial); + assert!(first_quality_rejections.is_empty()); + assert!(!first_playlist.is_empty()); + assert!(ctx.is_input_downloaded("main-account").await); + assert!(!ctx.is_input_downloaded("retry-account").await); + + tokio::fs::write( + &alias_playlist_path, + "#EXTM3U\n#EXTINF:-1 tvg-id=\"323\",Channel\nhttp://stream.example:4000/323/mono.m3u8?token=retry-stream-token\n", + ) + .await + .expect("alias fixture should be written"); + + let InputDownloadResult { + errors: second_errors, + source: mut second_playlist, + storage_error: second_storage_error, + partial: second_partial, + quality_rejections: second_quality_rejections, + .. + } = download_input(&ctx, &input, false).await; + + assert!(second_errors.is_empty(), "unexpected retry errors: {second_errors:?}"); + assert!(second_storage_error.is_none(), "primary storage should remain readable: {second_storage_error:?}"); + assert!(!second_partial); + assert!(second_quality_rejections.is_empty()); + assert!(!second_playlist.is_empty()); + assert!(ctx.is_input_downloaded("retry-account").await); + let alias_url = tuliprox_repository::load_input_m3u_stream_url( + &ctx.config, + &"retry-account".intern(), + "http://stream.example:4000/323/mono.m3u8?token=main-stream-token", + ) + .await + .expect("retried alias URL lookup should succeed"); + assert_eq!(alias_url.as_deref(), Some("http://stream.example:4000/323/mono.m3u8?token=retry-stream-token")); +} + +#[test] +fn persist_filter_runs_after_after_epg_mapping() { + let runtime = Runtime::new().expect("runtime"); + runtime.block_on(async { + let mut input = ConfigInput::from(ConfigInputDto::default()); + input.name = "input".intern(); + let groups = vec![PlaylistGroup { + id: 1, + title: "Live".intern(), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { + name: "Before".intern(), + group: "Live".intern(), + xtream_cluster: XtreamCluster::Live, + item_type: PlaylistItemType::Live, + ..Default::default() + }, + }], + xtream_cluster: XtreamCluster::Live, + }]; + let mut playlist = + FetchedPlaylist { input: &input, source: MemoryPlaylistSource::new(groups).into_source(), epg: None }; + let rename = build_mapping("rename", MappingStage::AfterEpg, r#"@Name = "After""#); + let mut target = build_target(vec![rename], false); + target.filter.persist = Some(get_filter(r#"Name = "After""#, None).expect("filter parses")); + let mut stats = + HashMap::from([(Arc::clone(&input.name), create_input_stat(1, 1, 0, input.input_type, &input.name, 0))]); + let mut errors = Vec::new(); + + let mut prepared = prepare_playlist_for_target( + &processing_context(), + std::slice::from_mut(&mut playlist), + &target, + &mut stats, + &mut errors, + ClusterFlags::empty(), + false, + ) + .await + .expect("target preparation"); + + assert!(errors.is_empty()); + apply_persist_filter(&target, &mut prepared.playlist); + let item = &prepared.playlist[0].channels[0]; + assert_eq!(item.header.name.as_ref(), "After"); + }); +} + +fn make_channel(name: &str) -> PlaylistItem { + let mut item = PlaylistItem { + header: PlaylistItemHeader { + name: name.intern(), + group: "Originals".intern(), + xtream_cluster: XtreamCluster::Live, + item_type: PlaylistItemType::Live, + ..Default::default() + }, + }; + item.header.freeze_input_stream_id(); + item +} + +fn memory_source(channels: Vec) -> PlaylistSource { + MemoryPlaylistSource::new(vec![PlaylistGroup { + id: 1, + title: "Live".intern(), + channels, + xtream_cluster: XtreamCluster::Live, + }]) + .into_source() +} + +fn channel_count(source: &mut PlaylistSource) -> usize { source.take_groups().iter().map(|g| g.channels.len()).sum() } + +#[test] +fn map_playlist_applies_only_the_requested_stage() { + let processing = build_mapping("processing", MappingStage::Processing, r#"@name = concat(@Name, "-P")"#); + let after_epg = build_mapping("after_epg", MappingStage::AfterEpg, r#"@name = concat(@Name, "-E")"#); + let target = build_target(vec![processing, after_epg], false); + + let mut source = memory_source(vec![make_channel("Alpha")]); + let (groups, _) = execute_pipeline_on_groups(source.take_groups(), &target, &[TransformStage::Map]); + assert_eq!(groups[0].channels[0].header.name.as_ref(), "Alpha-P"); + + let mut source = MemoryPlaylistSource::new(groups).into_source(); + let groups = map_playlist_at_stage(&mut source, &target, MappingStage::AfterEpg, None) + .expect("after_epg mapping should run"); + assert_eq!(groups[0].channels[0].header.name.as_ref(), "Alpha-P-E"); +} + +#[test] +fn map_playlist_at_stage_returns_none_without_consuming_source_when_no_match() { + let target = build_target(Vec::new(), false); + let mut source = memory_source(vec![make_channel("Alpha")]); + + let result = map_playlist_at_stage(&mut source, &target, MappingStage::AfterEpg, None); + assert!(result.is_none(), "no matching stage must return None"); + assert_eq!(channel_count(&mut source), 1, "source must remain intact"); +} + +#[test] +fn prepare_target_applies_after_epg_mapping_before_sampling_stats() { + let runtime = Runtime::new().expect("runtime"); + runtime.block_on(async { + let dir = tempdir().expect("tempdir"); + let ics_path = dir.path().join("bbc.ics"); + std::fs::write( + &ics_path, + "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:News\nDTSTART:20260306T120000Z\nDTEND:20260306T130000Z\nEND:VEVENT\nEND:VCALENDAR", + ) + .expect("write ics"); + + let mut smart_dto = EpgSmartMatchConfigDto { + enabled: true, + fuzzy_matching: false, + ..EpgSmartMatchConfigDto::default() + }; + smart_dto.prepare().expect("smart config"); + let mut input = ConfigInput::from(ConfigInputDto::default()); + input.name = "input".intern(); + input.epg = Some(EpgConfig { + sources: vec![], + smart_match: Some(EpgSmartMatchConfig::from(smart_dto)), + }); + + let channels = vec![live_item_for_epg("BBC One")]; + let groups = vec![PlaylistGroup { + id: 1, + title: "Live".intern(), + channels, + xtream_cluster: XtreamCluster::Live, + }]; + let tv_guide = TVGuide::new(vec![PersistedEpgSource { + file_path: ics_path, + priority: 0, + logo_override: false, + kind: PersistedEpgSourceKind::Ics { + channel_id: "bbc.one".intern(), + channel_title: Some("BBC One".intern()), + match_names: vec!["BBC One".intern()], + config: Box::new(IcsEpgSourceConfig::default()), + }, + }]); + + let mut playlist = FetchedPlaylist { + input: &input, + source: MemoryPlaylistSource::new(groups).into_source(), + epg: Some(tv_guide), + }; + + let rename_from_epg = build_mapping( + "rename", + MappingStage::AfterEpg, + r#"epg = @epg_channel_id ~ "(.+)" +match { + epg => @Name = epg.1 +}"#, + ); + let add_virtual = build_mapping("virtual", MappingStage::AfterEpg, r#"add_favourite("Echo")"#); + let target = build_target(vec![rename_from_epg, add_virtual], false); + let mut stats = HashMap::from([( + Arc::clone(&input.name), + create_input_stat(1, 1, 0, input.input_type, &input.name, 0), + )]); + let mut errors = Vec::new(); + let prepared = prepare_playlist_for_target( + &processing_context(), + std::slice::from_mut(&mut playlist), + &target, + &mut stats, + &mut errors, + ClusterFlags::empty(), + false, + ) + .await + .expect("target preparation"); + + assert!(errors.is_empty()); + assert_eq!(prepared.playlist.iter().map(|group| group.channels.len()).sum::(), 2); + let channel = prepared + .playlist + .iter() + .flat_map(|group| &group.channels) + .find(|channel| channel.header.group.as_ref() != "Echo") + .expect("original channel"); + assert_eq!(channel.header.epg_channel_id.as_deref(), Some("bbc.one")); + assert_eq!( + channel.header.name.as_ref(), + "bbc.one", + "after_epg mapper must consume the EPG-enriched field" + ); + let processed_stats = &stats[&input.name].processed_stats; + assert_eq!(processed_stats.group_count, 2); + assert_eq!(processed_stats.channel_count, 2); + }); +} + +#[test] +fn clear_invalid_epg_ids_clears_ids_invalidated_by_after_epg_mapping() { + let runtime = Runtime::new().expect("runtime"); + runtime.block_on(async { + let dir = tempdir().expect("tempdir"); + let ics_path = dir.path().join("bbc.ics"); + std::fs::write( + &ics_path, + "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:News\nDTSTART:20260306T120000Z\nDTEND:20260306T130000Z\nEND:VEVENT\nEND:VCALENDAR", + ) + .expect("write ics"); + + let mut input = ConfigInput::from(ConfigInputDto::default()); + input.name = "input".intern(); + input.epg = Some(EpgConfig { sources: vec![], smart_match: None }); + let groups = vec![PlaylistGroup { + id: 1, + title: "Live".intern(), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { + name: "BBC One".intern(), + epg_channel_id: Some("bbc.one".intern()), + group: "Live".intern(), + xtream_cluster: XtreamCluster::Live, + item_type: PlaylistItemType::Live, + ..Default::default() + }, + }], + xtream_cluster: XtreamCluster::Live, + }]; + let tv_guide = TVGuide::new(vec![PersistedEpgSource { + file_path: ics_path, + priority: 0, + logo_override: false, + kind: PersistedEpgSourceKind::Ics { + channel_id: "bbc.one".intern(), + channel_title: Some("BBC One".intern()), + match_names: vec![], + config: Box::new(IcsEpgSourceConfig::default()), + }, + }]); + let mut playlist = FetchedPlaylist { + input: &input, + source: MemoryPlaylistSource::new(groups).into_source(), + epg: Some(tv_guide), + }; + + let rewrite_epg = + build_mapping("rewrite", MappingStage::AfterEpg, r#"@epg_channel_id = "missing.epg""#); + let add_virtual = build_mapping("virtual", MappingStage::AfterEpg, r#"add_favourite("Echo")"#); + let mut target = build_target(vec![rewrite_epg, add_virtual], false); + target.options = Some(ConfigTargetOptions { clear_invalid_epg_ids: true, ..Default::default() }); + let mut stats = HashMap::from([( + Arc::clone(&input.name), + create_input_stat(1, 1, 0, input.input_type, &input.name, 0), + )]); + let mut errors = Vec::new(); + + let prepared = prepare_playlist_for_target( + &processing_context(), + std::slice::from_mut(&mut playlist), + &target, + &mut stats, + &mut errors, + ClusterFlags::empty(), + false, + ) + .await + .expect("target preparation"); + + assert!(errors.is_empty()); + assert!(!prepared.playlist.is_empty()); + assert!(prepared + .playlist + .iter() + .flat_map(|group| &group.channels) + .all(|channel| channel.header.epg_channel_id.is_none())); + assert_eq!(stats[&input.name].processed_stats.channel_count, 2); + }); +} + +fn live_item_for_epg(name: &str) -> PlaylistItem { + PlaylistItem { + header: PlaylistItemHeader { + name: name.intern(), + group: "Live".intern(), + xtream_cluster: XtreamCluster::Live, + item_type: PlaylistItemType::Live, + ..Default::default() + }, + } +} + +#[test] +fn after_epg_hook_runs_on_source_already_deduplicated_by_processing_pipe() { + let processing = build_mapping("processing", MappingStage::Processing, r#"@group = "PROCESSED""#); + let after_epg = build_mapping("after_epg", MappingStage::AfterEpg, r#"add_favourite("Echo")"#); + let target = build_target(vec![processing, after_epg], true); + + let input = ConfigInput::default(); + let channel = make_channel("Alpha"); + let mut fetched = + FetchedPlaylist { input: &input, source: memory_source(vec![channel.clone(), channel]), epg: None }; + let mut duplicates = HashSet::new(); + let (mut processed, _outcome) = + execute_pipe(&target, &get_processing_pipe(&target), &mut fetched, &mut duplicates, false, None) + .expect("processing pipe must run"); + assert_eq!(processed.get_channel_count(), 1, "processing pipe must remove the duplicate"); + + let groups = map_playlist_at_stage(&mut processed.source, &target, MappingStage::AfterEpg, None) + .expect("after_epg hook must run"); + + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].title.as_ref(), "PROCESSED"); + assert_eq!(groups[0].channels.len(), 1); + assert_eq!(groups[1].title.as_ref(), "Echo"); + assert_eq!(groups[1].channels.len(), 1); +} + +#[test] +fn prepare_target_deduplicates_virtual_items_created_by_after_epg_mappings() { + let runtime = Runtime::new().expect("runtime"); + runtime.block_on(async { + let first = build_mapping("first", MappingStage::AfterEpg, r#"add_favourite("Echo")"#); + let second = build_mapping("second", MappingStage::AfterEpg, r#"add_favourite("Echo")"#); + let target = build_target(vec![first, second], true); + let input = ConfigInput { name: "input".intern(), ..Default::default() }; + let mut playlist = + FetchedPlaylist { input: &input, source: memory_source(vec![make_channel("Alpha")]), epg: None }; + let mut stats = + HashMap::from([(Arc::clone(&input.name), create_input_stat(1, 1, 0, input.input_type, &input.name, 0))]); + let mut errors = Vec::new(); + + let prepared = prepare_playlist_for_target( + &processing_context(), + std::slice::from_mut(&mut playlist), + &target, + &mut stats, + &mut errors, + ClusterFlags::empty(), + false, + ) + .await + .expect("target preparation"); + + assert!(errors.is_empty()); + assert_eq!(prepared.playlist.iter().map(|group| group.channels.len()).sum::(), 3); + assert_eq!(stats[&input.name].processed_stats.channel_count, 3); + }); +} diff --git a/backend/processing/src/processor/playlist/tests/pipeline_transparency.rs b/backend/processing/src/processor/playlist/tests/pipeline_transparency.rs new file mode 100644 index 000000000..699205a30 --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/pipeline_transparency.rs @@ -0,0 +1,194 @@ +use super::{ + apply_staged_overlay_groups, neutralize_overlaid_cluster_facts, playlist_update_run_result, + report_input_job_completion, test_group, InputJobState, PlaylistRunCollectSink, +}; +use shared::{ + error::TuliproxError, + model::{ + ClusterFlags, EventMessage, InputRefreshPolicy, InputType, PlaylistUpdateClusterDecision, + PlaylistUpdateClusterTelemetry, PlaylistUpdateDataSource, PlaylistUpdateInputTelemetry, PlaylistUpdateRunId, + PlaylistUpdateRunOrder, PlaylistUpdateState, XtreamCluster, + }, + utils::Internable, +}; +use tuliprox_core::model::{ConfigInput, ConfigInputOptions}; + +#[test] +fn playlist_update_run_input_completion_uses_existing_success_partial_failure_classification() { + let success = playlist_update_run_result(1, InputJobState::Ready, Vec::new(), false); + let rejection = playlist_update_run_result(2, InputJobState::Ready, Vec::new(), true); + let resumable = playlist_update_run_result(3, InputJobState::Pending, Vec::new(), false); + let failure = playlist_update_run_result( + 4, + InputJobState::Ready, + vec![TuliproxError::RepositoryPlaylist("technical".to_string())], + false, + ); + + assert_eq!(success.update_state(), PlaylistUpdateState::Success); + assert_eq!(rejection.update_state(), PlaylistUpdateState::Partial); + assert_eq!(resumable.update_state(), PlaylistUpdateState::Partial); + assert_eq!(failure.update_state(), PlaylistUpdateState::Failure); +} + +#[test] +fn pipeline_transparency_input_completion_reuses_correlated_progress_event_for_typed_telemetry() { + let events = PlaylistRunCollectSink::default(); + let run_id = PlaylistUpdateRunId::from("pipeline-transparency-run"); + let execution_order = PlaylistUpdateRunOrder::from(19); + let mut result = playlist_update_run_result(7, InputJobState::Ready, Vec::new(), true); + let telemetry = PlaylistUpdateInputTelemetry { + refresh_policy: InputRefreshPolicy::NORMAL, + source: None, + clusters: vec![PlaylistUpdateClusterTelemetry { + cluster: XtreamCluster::Video, + requested: true, + source: Some(PlaylistUpdateDataSource::Provider), + baseline_count: Some(12_543), + candidate_count: Some(217), + active_count: Some(12_543), + threshold: Some(90), + quality: Some(1), + decision: Some(PlaylistUpdateClusterDecision::Rejected), + technical_state: None, + }], + }; + result.input_telemetry = Some(telemetry.clone()); + + report_input_job_completion(&events, &run_id, execution_order, &result); + + let emitted = events.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let EventMessage::PlaylistUpdateProgress(progress) = &emitted[0] else { + panic!("expected existing playlist progress event"); + }; + assert_eq!(progress.run_id.as_ref(), Some(&run_id)); + assert_eq!(progress.execution_order, Some(execution_order)); + assert_eq!(progress.input_id, Some(7)); + assert_eq!(progress.state, Some(PlaylistUpdateState::Partial)); + assert_eq!(progress.input_telemetry, Some(telemetry)); +} + +#[test] +fn playlist_update_run_technical_failure_progress_is_correlated_sanitized_and_input_scoped() { + let events = PlaylistRunCollectSink::default(); + let run_id = PlaylistUpdateRunId::from("processing-run"); + let execution_order = PlaylistUpdateRunOrder::from(17); + let failed = playlist_update_run_result( + 7, + InputJobState::Failed, + vec![TuliproxError::RepositoryPlaylist("https://user:secret@provider.example/playlist".to_string())], + false, + ); + let succeeded = playlist_update_run_result(8, InputJobState::Ready, Vec::new(), false); + + report_input_job_completion(&events, &run_id, execution_order, &failed); + report_input_job_completion(&events, &run_id, execution_order, &succeeded); + + let emitted = events.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let progress = emitted + .iter() + .filter_map(|event| match event { + EventMessage::PlaylistUpdateProgress(progress) => Some(progress), + _ => None, + }) + .collect::>(); + assert_eq!(progress.len(), 2); + assert!(progress.iter().all(|event| event.run_id.as_ref() == Some(&run_id))); + assert!(progress.iter().all(|event| event.execution_order == Some(execution_order))); + assert_eq!(progress[0].input_id, Some(7)); + assert_eq!(progress[0].state, Some(PlaylistUpdateState::Failure)); + assert_eq!(progress[0].message, "Input 'input-7' failed during update"); + assert!(!progress[0].message.contains("secret")); + assert!(!progress[0].message.contains("previous accepted data")); + assert_eq!(progress[1].input_id, Some(8)); + assert_eq!(progress[1].state, Some(PlaylistUpdateState::Success)); + assert!(!progress[1].message.contains("failed during update")); +} + +#[test] +fn pipeline_transparency_staged_overlay_replaces_groups_and_neutralizes_overlaid_runtime_facts() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + options: Some(ConfigInputOptions::defaults().clone()), + ..Default::default() + }; + let provider_groups = vec![ + test_group(XtreamCluster::Live, "provider-live", "provider"), + test_group(XtreamCluster::Video, "provider-vod", "provider"), + ]; + let mut staged_live = test_group(XtreamCluster::Live, "staged-live", "staged"); + staged_live.channels[0].header.id = "11203".intern(); + staged_live.channels[0].header.url = "http://iptvhost.example/live/fake-user/fake-pass/11203.ts".intern(); + let mut invalid = test_group(XtreamCluster::Live, "invalid", "staged").channels.remove(0); + invalid.header.id = "invalid".intern(); + invalid.header.url = "http://iptvhost.example/live/fake-user/fake-pass/invalid.ts".intern(); + staged_live.channels.push(invalid); + let staged_groups = vec![staged_live, test_group(XtreamCluster::Series, "staged-series", "staged")]; + + let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Live, provider_groups, staged_groups); + + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].title.as_ref(), "staged-live"); + assert_eq!(groups[0].channels.len(), 1); + assert_eq!(groups[0].channels[0].header.input_name.as_ref(), "provider"); + assert_eq!(groups[0].channels[0].header.url.as_ref(), "http://provider.example/live/real-user/real-pass/11203.ts"); + assert_eq!(groups[1].title.as_ref(), "provider-vod"); + assert_eq!(groups[1].channels[0].header.input_name.as_ref(), "provider"); + + let mut telemetry = PlaylistUpdateInputTelemetry { + refresh_policy: InputRefreshPolicy::NORMAL, + source: None, + clusters: vec![ + PlaylistUpdateClusterTelemetry { + cluster: XtreamCluster::Live, + requested: true, + source: Some(PlaylistUpdateDataSource::Provider), + baseline_count: Some(100), + candidate_count: Some(95), + active_count: Some(95), + threshold: Some(90), + quality: Some(95), + decision: Some(PlaylistUpdateClusterDecision::Accepted), + technical_state: None, + }, + PlaylistUpdateClusterTelemetry { + cluster: XtreamCluster::Video, + requested: true, + source: Some(PlaylistUpdateDataSource::Provider), + baseline_count: Some(200), + candidate_count: Some(200), + active_count: Some(200), + threshold: Some(100), + quality: Some(100), + decision: Some(PlaylistUpdateClusterDecision::Accepted), + technical_state: None, + }, + ], + }; + + neutralize_overlaid_cluster_facts(&mut telemetry, ClusterFlags::Live); + + let live = + telemetry.clusters.iter().find(|cluster| cluster.cluster == XtreamCluster::Live).expect("Live telemetry"); + assert!(live.requested); + assert_eq!(live.threshold, Some(90)); + assert_eq!(live.source, None); + assert_eq!(live.baseline_count, None); + assert_eq!(live.candidate_count, None); + assert_eq!(live.active_count, None); + assert_eq!(live.quality, None); + assert_eq!(live.decision, None); + + let video = + telemetry.clusters.iter().find(|cluster| cluster.cluster == XtreamCluster::Video).expect("Video telemetry"); + assert_eq!(video.source, Some(PlaylistUpdateDataSource::Provider)); + assert_eq!(video.baseline_count, Some(200)); + assert_eq!(video.candidate_count, Some(200)); + assert_eq!(video.active_count, Some(200)); + assert_eq!(video.quality, Some(100)); + assert_eq!(video.decision, Some(PlaylistUpdateClusterDecision::Accepted)); +} diff --git a/backend/processing/src/processor/playlist/tests/probe_details.rs b/backend/processing/src/processor/playlist/tests/probe_details.rs new file mode 100644 index 000000000..025fc1b72 --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/probe_details.rs @@ -0,0 +1,123 @@ +use super::{collect_effective_skip_clusters, has_probe_details, needs_live_probe}; +use shared::{ + model::{InputType, PlaylistItem, StreamProperties, XtreamCluster}, + utils::Internable, +}; +use tuliprox_core::model::ConfigInput; + +pub(in crate::processor::playlist::tests) fn item_with_props(props: StreamProperties) -> PlaylistItem { + let header = shared::model::PlaylistItemHeader { additional_properties: Some(props), ..Default::default() }; + PlaylistItem { header } +} + +pub(in crate::processor::playlist::tests) fn live_item_with_probe_timestamp_and_bitrate( + last_probed_timestamp: i64, + bitrate: u32, +) -> PlaylistItem { + item_with_props(StreamProperties::Live(Box::new(shared::model::LiveStreamProperties { + video: Some("{\"codec_name\":\"h264\"}".intern()), + audio: Some("{\"codec_name\":\"aac\"}".intern()), + bitrate, + last_probed_timestamp: Some(last_probed_timestamp), + ..Default::default() + }))) +} + +#[test] +fn has_probe_details_requires_video_and_audio_for_video() { + let video = shared::model::VideoStreamProperties { + details: Some(shared::model::VideoStreamDetailProperties { + video: Some("{\"codec_name\":\"h264\"}".intern()), + audio: None, + ..Default::default() + }), + ..Default::default() + }; + let item_missing_audio = item_with_props(StreamProperties::Video(Box::new(video))); + assert!(!has_probe_details(&item_missing_audio)); + + let video_complete = shared::model::VideoStreamProperties { + details: Some(shared::model::VideoStreamDetailProperties { + video: Some("{\"codec_name\":\"h264\"}".intern()), + audio: Some("{\"codec_name\":\"aac\"}".intern()), + ..Default::default() + }), + ..Default::default() + }; + let item_complete = item_with_props(StreamProperties::Video(Box::new(video_complete))); + assert!(has_probe_details(&item_complete)); +} + +#[test] +fn has_probe_details_requires_video_audio_and_bitrate_for_live() { + let live_missing_audio = shared::model::LiveStreamProperties { + video: Some("{\"codec_name\":\"h264\"}".intern()), + audio: None, + ..Default::default() + }; + let item_missing_audio = item_with_props(StreamProperties::Live(Box::new(live_missing_audio))); + assert!(!has_probe_details(&item_missing_audio)); + + let live_missing_bitrate = shared::model::LiveStreamProperties { + video: Some("{\"codec_name\":\"h264\"}".intern()), + audio: Some("{\"codec_name\":\"aac\"}".intern()), + ..Default::default() + }; + let item_missing_bitrate = item_with_props(StreamProperties::Live(Box::new(live_missing_bitrate))); + assert!(!has_probe_details(&item_missing_bitrate)); + + let live_complete = shared::model::LiveStreamProperties { + video: Some("{\"codec_name\":\"h264\"}".intern()), + audio: Some("{\"codec_name\":\"aac\"}".intern()), + bitrate: 2_500_000, + ..Default::default() + }; + let item_complete = item_with_props(StreamProperties::Live(Box::new(live_complete))); + assert!(has_probe_details(&item_complete)); +} + +#[test] +fn needs_live_probe_when_fresh_probe_has_no_bitrate() { + let item = live_item_with_probe_timestamp_and_bitrate(101, 0); + + assert!(needs_live_probe(&item, 100)); +} + +#[test] +fn does_not_need_live_probe_when_fresh_probe_has_positive_bitrate() { + let item = live_item_with_probe_timestamp_and_bitrate(101, 2_500_000); + + assert!(!needs_live_probe(&item, 100)); +} + +#[test] +fn needs_live_probe_when_positive_bitrate_probe_is_older_than_cutoff() { + let item = live_item_with_probe_timestamp_and_bitrate(99, 2_500_000); + + assert!(needs_live_probe(&item, 100)); +} + +#[test] +fn has_probe_details_is_false_for_series() { + let series = shared::model::SeriesStreamProperties::default(); + let item = item_with_props(StreamProperties::Series(Box::new(series))); + assert!(!has_probe_details(&item)); +} + +#[test] +fn collect_effective_skip_clusters_uses_input_skip_flags() { + use tuliprox_core::model::{ConfigInputFlags, ConfigInputOptions}; + let input = ConfigInput { + name: "skip_live".intern(), + input_type: InputType::Xtream, + options: Some(ConfigInputOptions { + flags: ConfigInputFlags::SkipLive.into(), + ..ConfigInputOptions::defaults().clone() + }), + ..ConfigInput::default() + }; + let skip = collect_effective_skip_clusters(&input); + assert!(skip.contains(&XtreamCluster::Live)); + assert!(!skip.contains(&XtreamCluster::Video)); + assert!(!skip.contains(&XtreamCluster::Series)); +} diff --git a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs new file mode 100644 index 000000000..2d8c4c923 --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback.rs @@ -0,0 +1,1619 @@ +use super::*; +use arc_swap::{ArcSwap, ArcSwapOption}; +use shared::model::{ + provider_saturation::build_group_lookup, xtream_const::XTREAM_CLUSTER, ConfigInputOptionsDto, + ConfigInputUpdateQualityDto, ConfigPaths, EventMessage, EventSink, PlaylistUpdateState, ProcessingOrder, + SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, SeriesStreamProperties, UpdateQualityPolicy, +}; +use std::{ + io::{ErrorKind, Read, Write}, + net::{TcpListener, TcpStream}, + path::Path, + sync::{ + atomic::{AtomicBool, Ordering}, + Mutex as StdMutex, + }, + thread::{self, JoinHandle}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tuliprox_core::{ + model::{ + ConfigSource, MediaToolCapabilities, SourcesConfig, StagedFilter, TargetExecutionPlan, TargetOutput, + XtreamTargetFlagsSet, XtreamTargetOutput, + }, + utils::FileLockManager, +}; +use tuliprox_repository::{ + count_input_xtream_cluster, get_input_storage_path, get_live_cat_collection_path, get_series_cat_collection_path, + get_vod_cat_collection_path, load_xtream_target_storage, persist_playlist, xtream_get_item_for_stream_id, + xtream_get_storage_path, TargetPlaylistPersistOptions, +}; + +struct TestXtreamServer { + base_url: String, + stop: Arc, + requests: Arc>>, + errors: Arc>>, + worker: Option>, +} + +impl TestXtreamServer { + fn start(candidate_counts: [usize; 3]) -> Self { Self::start_with_responses(fixture_responses(candidate_counts)) } + + fn start_with_responses(responses: HashMap) -> Self { + Self::start_with_statuses(responses, HashMap::new()) + } + + fn start_with_statuses(responses: HashMap, statuses: HashMap) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind Xtream fixture server"); + listener.set_nonblocking(true).expect("configure Xtream fixture listener"); + let address = listener.local_addr().expect("Xtream fixture address"); + let stop = Arc::new(AtomicBool::new(false)); + let requests = Arc::new(StdMutex::new(Vec::new())); + let errors = Arc::new(StdMutex::new(Vec::new())); + let worker_stop = Arc::clone(&stop); + let worker_requests = Arc::clone(&requests); + let worker_errors = Arc::clone(&errors); + let worker = thread::spawn(move || { + while !worker_stop.load(Ordering::Acquire) { + match listener.accept() { + Ok((stream, _)) => match serve_fixture_request(stream, &responses, &statuses) { + Ok(action) => worker_requests.lock().expect("request log lock").push(action), + Err(err) => worker_errors.lock().expect("server error lock").push(err.to_string()), + }, + Err(err) if err.kind() == ErrorKind::WouldBlock => thread::sleep(Duration::from_millis(1)), + Err(err) => { + worker_errors.lock().expect("server error lock").push(err.to_string()); + break; + } + } + } + }); + + Self { base_url: format!("http://{address}"), stop, requests, errors, worker: Some(worker) } + } + + fn finish(mut self) -> Vec { + self.stop_worker(); + let errors = self.errors.lock().expect("server error lock").clone(); + assert!(errors.is_empty(), "Xtream fixture server errors: {errors:?}"); + self.requests.lock().expect("request log lock").clone() + } + + fn stop_worker(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(worker) = self.worker.take() { + worker.join().expect("Xtream fixture server should stop cleanly"); + } + } +} + +impl Drop for TestXtreamServer { + fn drop(&mut self) { self.stop_worker(); } +} + +#[derive(Clone, Default)] +struct CollectSink(Arc>>); + +impl EventSink for CollectSink { + fn emit(&self, event: EventMessage) { self.0.lock().expect("event sink lock").push(event); } +} + +fn fixture_responses(candidate_counts: [usize; 3]) -> HashMap { + let mut responses = + HashMap::from([("login".to_string(), serde_json::json!({"user_info": {"status": "Active"}}).to_string())]); + for (cluster, count) in XTREAM_CLUSTER.into_iter().zip(candidate_counts) { + let (category_action, stream_action, category_id, id_field, id_base) = match cluster { + XtreamCluster::Live => ("get_live_categories", "get_live_streams", 1_u32, "stream_id", 1_000_u32), + XtreamCluster::Video => ("get_vod_categories", "get_vod_streams", 2, "stream_id", 2_000), + XtreamCluster::Series => ("get_series_categories", "get_series", 3, "series_id", 3_000), + }; + responses.insert( + category_action.to_string(), + serde_json::json!([{"category_id": category_id, "category_name": format!("candidate-{cluster}")}]) + .to_string(), + ); + let streams: Vec<_> = (0..count) + .map(|offset| { + serde_json::json!({ + "name": format!("candidate-{cluster}-{offset}"), + id_field: id_base + u32::try_from(offset).expect("fixture count fits u32"), + "category_id": category_id, + }) + }) + .collect(); + responses.insert(stream_action.to_string(), serde_json::Value::Array(streams).to_string()); + } + responses +} + +fn live_fixture_responses(categories: &serde_json::Value, streams: Vec) -> HashMap { + HashMap::from([ + ("login".to_string(), serde_json::json!({"user_info": {"status": "Active"}}).to_string()), + ("get_live_categories".to_string(), categories.to_string()), + ("get_live_streams".to_string(), serde_json::Value::Array(streams).to_string()), + ]) +} + +fn live_fixture_stream(provider_id: u32, category_id: u32, name: &str) -> serde_json::Value { + serde_json::json!({ + "name": name, + "stream_id": provider_id, + "category_id": category_id, + }) +} + +fn serve_fixture_request( + mut stream: TcpStream, + responses: &HashMap, + statuses: &HashMap, +) -> std::io::Result { + stream.set_nonblocking(false)?; + stream.set_read_timeout(Some(Duration::from_secs(1)))?; + let mut request = Vec::with_capacity(1_024); + let mut buffer = [0_u8; 1_024]; + loop { + let read = stream.read(&mut buffer)?; + if read == 0 { + break; + } + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + let request = String::from_utf8_lossy(&request); + let action = request + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .and_then(|target| target.split_once("action=").map(|(_, value)| value)) + .map_or_else(|| "login".to_string(), |value| value.split('&').next().unwrap_or(value).to_string()); + let body = responses + .get(&action) + .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, format!("unexpected action {action}")))?; + let status = statuses.get(&action).copied().unwrap_or(reqwest::StatusCode::OK); + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes())?; + Ok(action) +} + +fn processing_context(storage_dir: &Path) -> PlaylistProcessingContext { + processing_context_with_events(storage_dir, shared::model::NoopSink) +} + +fn processing_context_with_events(storage_dir: &Path, events: E) -> PlaylistProcessingContext { + let paths = ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + }; + let config = AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config { + storage_dir: storage_dir.to_string_lossy().into_owned(), + disk_based_processing: false, + ..Config::default() + })), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(paths)), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + }; + PlaylistProcessingContext { + client: reqwest::Client::new(), + run_id: "xtream-quality-test-run".into(), + execution_order: PlaylistUpdateRunOrder::from(1), + config: Arc::new(config), + user_targets: Arc::new(ProcessTargets { + enabled: false, + inputs: Vec::new(), + targets: Vec::new(), + target_names: Vec::new(), + }), + events, + playlist_state: None, + disabled_headers: None, + processed_inputs: Arc::new(Mutex::new(HashSet::new())), + input_completions: Arc::new(Mutex::new(HashMap::new())), + input_locks: Arc::new(Mutex::new(HashMap::new())), + provider_manager: None, + metadata_manager: None, + pre_processed_inputs: None, + input_refresh: None, + library_update_mode: LibraryUpdateMode::ExistingCatalog, + stalker_refresh_mode: StalkerRefreshMode::Complete, + partial_refresh: Arc::new(AtomicBool::new(false)), + had_quality_rejections: Arc::new(AtomicBool::new(false)), + } +} + +#[test] +fn playlist_update_run_constructor_preserves_queued_identity_and_generates_unique_automatic_ids() { + let temp = tempfile::tempdir().expect("temp dir"); + let ctx = processing_context(temp.path()); + let queued_id = PlaylistUpdateRunId::from("queued-run"); + let queued = ProcessingRun::for_run( + queued_id.clone(), + ctx.client.clone(), + Arc::clone(&ctx.config), + Arc::clone(&ctx.user_targets), + shared::model::NoopSink, + ); + let automatic_a = ProcessingRun::new( + ctx.client.clone(), + Arc::clone(&ctx.config), + Arc::clone(&ctx.user_targets), + shared::model::NoopSink, + ); + let automatic_b = ProcessingRun::new(ctx.client, ctx.config, ctx.user_targets, shared::model::NoopSink); + + assert_eq!(queued.run_id, queued_id); + assert_ne!(automatic_a.run_id, automatic_b.run_id); +} + +#[tokio::test] +async fn playlist_update_run_manual_and_automatic_execution_order_is_generated_and_preserved() { + let temp = tempfile::tempdir().expect("temp dir"); + let events = CollectSink::default(); + let ctx = processing_context_with_events(temp.path(), events.clone()); + let manual_run_id = PlaylistUpdateRunId::from("manual-run"); + + exec_processing(ProcessingRun::for_run( + manual_run_id.clone(), + ctx.client.clone(), + Arc::clone(&ctx.config), + Arc::clone(&ctx.user_targets), + events.clone(), + )) + .await; + let automatic_run = ProcessingRun::new(ctx.client, ctx.config, ctx.user_targets, events.clone()); + let automatic_run_id = automatic_run.run_id.clone(); + exec_processing(automatic_run).await; + + let emitted = events.0.lock().expect("event sink lock"); + let terminal_for = |run_id: &PlaylistUpdateRunId| { + emitted.iter().find_map(|event| match event { + EventMessage::PlaylistUpdate(summary) if summary.run_id.as_ref() == Some(run_id) => Some(summary), + _ => None, + }) + }; + let manual_terminal = terminal_for(&manual_run_id).expect("manual terminal event"); + let automatic_terminal = terminal_for(&automatic_run_id).expect("automatic terminal event"); + let manual_order = manual_terminal.execution_order.expect("manual execution order"); + let automatic_order = automatic_terminal.execution_order.expect("automatic execution order"); + assert!(automatic_order > manual_order); + + let progress_for = |run_id: &PlaylistUpdateRunId| { + emitted + .iter() + .filter_map(|event| match event { + EventMessage::PlaylistUpdateProgress(progress) if progress.run_id.as_ref() == Some(run_id) => { + Some(progress) + } + _ => None, + }) + .collect::>() + }; + let manual_progress = progress_for(&manual_run_id); + let automatic_progress = progress_for(&automatic_run_id); + assert!(!manual_progress.is_empty()); + assert!(!automatic_progress.is_empty()); + assert!(manual_progress.iter().all(|progress| progress.execution_order == Some(manual_order))); + assert!(automatic_progress.iter().all(|progress| progress.execution_order == Some(automatic_order))); +} + +fn test_input( + base_url: &str, + update_quality: ConfigInputUpdateQualityDto, + cache_duration_seconds: u64, + skipped_clusters: &[XtreamCluster], +) -> Arc { + let options = ConfigInputOptionsDto { + skip_live: skipped_clusters.contains(&XtreamCluster::Live), + skip_vod: skipped_clusters.contains(&XtreamCluster::Video), + skip_series: skipped_clusters.contains(&XtreamCluster::Series), + update_quality, + ..ConfigInputOptionsDto::default() + }; + Arc::new(ConfigInput { + id: 1, + name: "quality-provider".intern(), + input_type: InputType::Xtream, + url: base_url.to_string(), + username: Some("user".to_string()), + password: Some("password".to_string()), + enabled: true, + options: Some(ConfigInputOptions::from(&options)), + cache_duration_seconds, + ..ConfigInput::default() + }) +} + +fn baseline_group(cluster: XtreamCluster, category_id: u32, title: &str, first_item_id: u32) -> PlaylistGroup { + baseline_group_with_count(cluster, category_id, title, first_item_id, 2) +} + +fn baseline_group_with_count( + cluster: XtreamCluster, + category_id: u32, + title: &str, + first_item_id: u32, + count: usize, +) -> PlaylistGroup { + let title = title.intern(); + let item_type = PlaylistItemType::from(cluster); + let channels = (0..count) + .map(|offset| { + let offset = u32::try_from(offset).expect("baseline fixture count fits u32"); + let item_id = (first_item_id + offset).to_string().intern(); + let mut header = PlaylistItemHeader { + id: Arc::clone(&item_id), + input_stream_id: item_id, + name: format!("{title}-{offset}").intern(), + title: format!("{title}-{offset}").intern(), + group: Arc::clone(&title), + url: format!("http://old.example/{cluster}/{first_item_id}/{offset}").intern(), + input_name: "quality-provider".intern(), + item_type, + xtream_cluster: cluster, + category_id, + ..PlaylistItemHeader::default() + }; + header.gen_uuid(); + PlaylistItem { header } + }) + .collect(); + PlaylistGroup { id: category_id, title, channels, xtream_cluster: cluster } +} + +async fn seed_live_baseline(ctx: &PlaylistProcessingContext, input: &ConfigInput, first_item_id: u32) { + let baseline = baseline_group_with_count(XtreamCluster::Live, 1, "old-live", first_item_id, 100); + let (persisted, error) = tuliprox_repository::persist_input_playlist(&ctx.config, input, vec![baseline]).await; + assert!(error.is_none(), "Live baseline persistence failed: {error:?}"); + assert_eq!(persisted.iter().map(|group| group.channels.len()).sum::(), 100); +} + +fn baseline_groups() -> Vec { + vec![ + baseline_group(XtreamCluster::Live, 1, "old-live", 100), + baseline_group(XtreamCluster::Video, 2, "old-vod", 200), + baseline_group(XtreamCluster::Series, 3, "old-series", 300), + ] +} + +async fn seed_baseline(ctx: &PlaylistProcessingContext, input: &ConfigInput) { + let (persisted, error) = tuliprox_repository::persist_input_playlist(&ctx.config, input, baseline_groups()).await; + assert!(error.is_none(), "baseline persistence failed: {error:?}"); + assert_baseline(&persisted); +} + +fn assert_baseline(groups: &[PlaylistGroup]) { + for (cluster, category_id, title, item_ids) in [ + (XtreamCluster::Live, 1, "old-live", ["100", "101"]), + (XtreamCluster::Video, 2, "old-vod", ["200", "201"]), + (XtreamCluster::Series, 3, "old-series", ["300", "301"]), + ] { + let group = groups + .iter() + .find(|group| group.xtream_cluster == cluster && group.id == category_id) + .unwrap_or_else(|| panic!("missing persisted {cluster} group")); + assert_eq!(group.title.as_ref(), title); + assert_eq!(group.channels.len(), 2); + assert!(group.channels.iter().all(|item| item.header.xtream_cluster == cluster)); + assert_eq!(group.channels[0].header.id.as_ref(), item_ids[0]); + assert_eq!(group.channels[1].header.id.as_ref(), item_ids[1]); + } +} + +async fn input_storage_path(ctx: &PlaylistProcessingContext, input: &ConfigInput) -> PathBuf { + let storage_dir = ctx.config.config.load().storage_dir.clone(); + get_input_storage_path(&input.name, &storage_dir).await.expect("input storage path") +} + +async fn seed_valid_cluster_cache(ctx: &PlaylistProcessingContext, input: &ConfigInput) -> u64 { + let storage_path = input_storage_path(ctx, input).await; + let now = SystemTime::now().duration_since(UNIX_EPOCH).expect("current time").as_secs(); + let mut status = input_cache::InputStatus::default(); + for cluster in XTREAM_CLUSTER { + status.clusters.insert( + cluster.as_ref().to_string(), + input_cache::ClusterStatus { status: input_cache::ClusterState::Ok, timestamp: now, last_update: None }, + ); + } + input_cache::save_input_status(&storage_path, &status); + now +} + +fn set_refresh_policy( + ctx: &mut PlaylistProcessingContext, + input: &ConfigInput, + policy: InputRefreshPolicy, +) { + ctx.input_refresh = Some(InputRefreshOverride { input_id: input.id, policy }); +} + +fn xtream_target(use_memory_cache: bool) -> Arc { + Arc::new(ConfigTarget { + curation: None, + id: 1, + enabled: true, + name: "quality-target".to_string(), + options: None, + sort: None, + filter: StagedFilter::default(), + output: vec![TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::new(), + trakt: None, + filter: None, + })], + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::new(None)), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: TargetExecutionPlan::default(), + watch: None, + use_memory_cache, + }) +} + +fn install_source( + ctx: &PlaylistProcessingContext, + input: &Arc, + target: &Arc, +) { + let inputs = vec![Arc::clone(input)]; + ctx.config.sources.store(Arc::new(SourcesConfig { + batch_files: Vec::new(), + templates: None, + provider: Vec::new(), + group_lookup: build_group_lookup(&inputs), + inputs, + sources: vec![ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![Arc::clone(target)] }], + })); +} + +async fn seed_xtream_target( + ctx: &PlaylistProcessingContext, + target: &ConfigTarget, + playlist_state: &Arc, +) -> u32 { + let mut groups = baseline_groups(); + persist_playlist( + &ctx.config, + &mut groups, + None, + target, + Some(playlist_state), + TargetPlaylistPersistOptions::default(), + ) + .await + .expect("target baseline should persist"); + groups + .iter() + .find(|group| group.xtream_cluster == XtreamCluster::Video) + .and_then(|group| group.channels.first()) + .map(|item| item.header.virtual_id.get()) + .expect("target baseline VOD id") +} + +async fn assert_target_counts(app_config: &Arc, target: &ConfigTarget, expected: [usize; 3]) { + let storage = load_xtream_target_storage(app_config, target).await.expect("target storage should load"); + assert_eq!([storage.live.len(), storage.vod.len(), storage.series.len()], expected); +} + +async fn assert_empty_target_categories( + app_config: &Arc, + target: &ConfigTarget, + clusters: &[XtreamCluster], +) { + let storage_path = { + let config = app_config.config.load(); + xtream_get_storage_path(&config, &target.name).expect("Xtream target storage path") + }; + for cluster in clusters { + let path = match cluster { + XtreamCluster::Live => get_live_cat_collection_path(&storage_path), + XtreamCluster::Video => get_vod_cat_collection_path(&storage_path), + XtreamCluster::Series => get_series_cat_collection_path(&storage_path), + }; + let categories: Vec = serde_json::from_slice( + &tokio::fs::read(&path).await.unwrap_or_else(|error| panic!("read {}: {error}", path.display())), + ) + .unwrap_or_else(|error| panic!("parse {}: {error}", path.display())); + assert!(categories.is_empty(), "{} should contain no categories", path.display()); + } +} + +#[test] +fn refresh_policy_override_is_scoped_to_the_selected_input_id() { + let temp = tempfile::tempdir().expect("temporary storage"); + let mut ctx = processing_context(temp.path()); + let selected = ConfigInput { id: 17, ..ConfigInput::default() }; + set_refresh_policy(&mut ctx, &selected, InputRefreshPolicy::FORCE); + + assert_eq!(ctx.refresh_policy(17), InputRefreshPolicy::FORCE); + assert_eq!(ctx.refresh_policy(18), InputRefreshPolicy::NORMAL); +} + +fn assert_requested_actions(mut actual: Vec, expected: &[&str]) { + actual.sort(); + let mut expected: Vec<_> = expected.iter().map(ToString::to_string).collect(); + expected.sort(); + assert_eq!(actual, expected); +} + +#[tokio::test] +async fn mixed_in_memory_update_keeps_rejected_vod_ready_and_marks_the_run_partial() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([2, 1, 2]); + let events = CollectSink::default(); + let ctx = processing_context_with_events(temp.path(), events.clone()); + let input = test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 0, &[]); + seed_baseline(&ctx, &input).await; + + let mut result = process_input_job_inner(0, &ctx, &input).await; + let requests = server.finish(); + + assert_eq!(result.state, InputJobState::Ready); + assert!(result.errors.is_empty(), "quality rejection must not become an error: {:?}", result.errors); + assert!(ctx.had_quality_rejections.load(Ordering::Acquire)); + assert_eq!( + PlaylistRunSignals { has_quality_rejections: true, ..PlaylistRunSignals::default() }.state(), + PlaylistUpdateState::Partial + ); + + let groups = result.source.take().expect("ready input source").take_groups(); + let live = groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Live).expect("accepted Live group"); + let vod = groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Video).expect("retained VOD group"); + let series = + groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Series).expect("accepted Series group"); + assert_eq!(live.title.as_ref(), "candidate-live"); + assert_eq!(live.channels.len(), 2); + assert_eq!(vod.title.as_ref(), "old-vod"); + assert_eq!(vod.channels.len(), 2); + assert_eq!(series.title.as_ref(), "candidate-series"); + assert_eq!(series.channels.len(), 2); + + let storage_path = input_storage_path(&ctx, &input).await; + let status = input_cache::load_input_status(&storage_path); + assert_eq!( + status.clusters.get(XtreamCluster::Live.as_ref()).map(|entry| &entry.status), + Some(&input_cache::ClusterState::Ok) + ); + assert_eq!( + status.clusters.get(XtreamCluster::Video.as_ref()).map(|entry| &entry.status), + Some(&input_cache::ClusterState::Failed) + ); + assert_eq!( + status.clusters.get(XtreamCluster::Series.as_ref()).map(|entry| &entry.status), + Some(&input_cache::ClusterState::Ok) + ); + + let emitted = events.0.lock().expect("event sink lock"); + let rejection_events = emitted + .iter() + .filter_map(|event| match event { + EventMessage::PlaylistUpdateProgress(progress) if progress.message.contains("cluster 'vod' rejected") => { + Some(progress) + } + _ => None, + }) + .collect::>(); + assert_eq!(rejection_events.len(), 1); + assert_eq!(rejection_events[0].target, "quality-provider"); + assert_eq!( + rejection_events[0].message, + "Input 'quality-provider' cluster 'vod' rejected: current=2 candidate=1 threshold=100 quality=50; retaining previous cluster" + ); + assert_requested_actions( + requests, + &[ + "login", + "get_live_categories", + "get_live_streams", + "get_vod_categories", + "get_vod_streams", + "get_series_categories", + "get_series", + ], + ); +} + +#[tokio::test] +async fn coderabbit_stalker_global_failures_mark_only_requested_non_skipped_clusters() { + use crate::processor::stalker::{download_stalker_playlist, StalkerCluster}; + + #[derive(Clone, Copy, Debug)] + enum FailurePoint { + Portal, + Client, + Storage, + Handshake, + } + + for failure in [FailurePoint::Portal, FailurePoint::Client, FailurePoint::Storage, FailurePoint::Handshake] { + for requested in [None, Some(vec![StalkerCluster::Live, StalkerCluster::Series])] { + let temp = tempfile::tempdir().unwrap(); + let ctx = processing_context(temp.path()); + let server = TestXtreamServer::start_with_responses(HashMap::from([("handshake".into(), "{}".into())])); + let mut input = + (*test_input(&server.base_url, ConfigInputUpdateQualityDto::default(), 0, &[XtreamCluster::Live])) + .clone(); + input.input_type = InputType::Stalker; + input.stalker = Some(tuliprox_core::model::StalkerInputConfig::default()); + match failure { + FailurePoint::Portal => input.url.clear(), + FailurePoint::Client => input.url = "://invalid-url".into(), + FailurePoint::Storage => { + let path = input_storage_path(&ctx, &input).await; + tokio::fs::write(path.join("stalker"), b"not a directory").await.unwrap(); + } + FailurePoint::Handshake => {} + } + let fetch = download_stalker_playlist( + &ctx.config, + &ctx.client, + &input, + requested.as_deref(), + StalkerRefreshMode::Complete, + true, + UpdateQualityPolicy::Enforce, + ) + .await; + let expected = if requested.is_some() { + vec![XtreamCluster::Series] + } else { + vec![XtreamCluster::Video, XtreamCluster::Series] + }; + assert_eq!(fetch.failed_clusters, expected, "{failure:?}"); + assert_eq!(fetch.errors.len(), 1, "{failure:?}"); + assert!(!fetch.is_ok()); + assert!(fetch.quality_acceptances.is_empty() && fetch.quality_rejections.is_empty()); + assert!(fetch.force_updates.is_empty()); + let requests = server.finish(); + match failure { + FailurePoint::Handshake => assert!(!requests.is_empty()), + FailurePoint::Portal | FailurePoint::Client | FailurePoint::Storage => assert!(requests.is_empty()), + } + } + } +} + +#[tokio::test] +async fn coderabbit_xtream_http_failure_preserves_successful_disk_clusters() { + for quality in [UpdateQualityPolicy::Enforce, UpdateQualityPolicy::Bypass] { + let temp = tempfile::tempdir().unwrap(); + let server = TestXtreamServer::start_with_statuses( + fixture_responses([2, 2, 2]), + HashMap::from([("get_vod_streams".to_owned(), reqwest::StatusCode::BAD_REQUEST)]), + ); + let ctx = processing_context(temp.path()); + let mut config = (**ctx.config.config.load()).clone(); + config.disk_based_processing = true; + ctx.config.config.store(Arc::new(config)); + let input = + test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 0, &[]); + seed_baseline(&ctx, &input).await; + + let fetch = tuliprox_iptv::xtream::download_xtream_playlist( + &ctx.config, + &ctx.client, + &ctx.events, + &input, + None, + quality, + ) + .await; + assert_eq!(fetch.failed_clusters, vec![XtreamCluster::Video]); + assert!(!fetch.is_ok(), "the failed cluster must remain a technical failure"); + assert!(fetch.persisted); + assert!(fetch.quality_rejections.is_empty()); + let accepted: Vec<_> = match quality { + UpdateQualityPolicy::Enforce => fetch.quality_acceptances.iter().map(|value| value.cluster).collect(), + UpdateQualityPolicy::Bypass => fetch.force_updates.iter().map(|value| value.cluster).collect(), + }; + assert_eq!(accepted, vec![XtreamCluster::Live, XtreamCluster::Series]); + + let storage = input_storage_path(&ctx, &input).await; + let groups = + tuliprox_repository::load_input_xtream_playlist(&ctx.config, &storage, &XTREAM_CLUSTER).await.unwrap(); + for (cluster, title) in [ + (XtreamCluster::Live, "candidate-live"), + (XtreamCluster::Video, "old-vod"), + (XtreamCluster::Series, "candidate-series"), + ] { + let group = groups.iter().find(|group| group.xtream_cluster == cluster).unwrap(); + assert_eq!(group.title.as_ref(), title); + assert_eq!(group.channels.len(), 2); + } + assert_eq!(server.finish().len(), 7); + } +} + +#[tokio::test] +async fn pipeline_transparency_shows_failure_survives_xtream_fetch_and_reload() { + for disk in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let mut responses = fixture_responses([2, 2, 2]); + responses.insert("get_series".to_owned(), "{".to_owned()); + let server = TestXtreamServer::start_with_responses(responses); + let mut ctx = processing_context(temp.path()); + let mut config = ctx.config.config.load().as_ref().clone(); + config.disk_based_processing = disk; + ctx.config.config.store(Arc::new(config)); + let input = + test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 0 }, 0, &[]); + seed_baseline(&ctx, &input).await; + set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); + let mut result = download_input(&ctx, &input, false).await; + assert_eq!(result.update_state(), PlaylistUpdateState::Failure); + assert!(!result.errors.is_empty()); + let telemetry = result.input_telemetry.as_ref().unwrap(); + for cluster in &telemetry.clusters { + if cluster.cluster == XtreamCluster::Series { + assert_eq!(cluster.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Failed)); + assert_eq!(cluster.decision, None); + assert_eq!(cluster.quality, None); + } else { + assert_eq!(cluster.technical_state, None, "do not attribute Shows failure to a sibling"); + assert_eq!(cluster.decision, Some(PlaylistUpdateClusterDecision::Accepted)); + } + assert_eq!(cluster.active_count, None, "failed batch does not prove final activation"); + } + let storage = input_storage_path(&ctx, &input).await; + let saved = input_cache::load_input_status(&storage); + let shows = saved.clusters["series"].last_update.unwrap(); + assert_eq!(shows.technical_state, Some(PersistedPlaylistUpdateTechnicalState::Failed)); + assert_eq!(shows.quality, None); + assert_eq!(shows.quality_guard_threshold, Some(0)); + assert_eq!(shows.policy, Some(InputRefreshPolicy::FORCE)); + assert!(server.finish().iter().any(|action| action == "get_series")); + } +} + +#[tokio::test] +async fn pipeline_transparency_normal_policy_reuses_a_valid_cluster_cache() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([2, 2, 2]); + let mut ctx = processing_context(temp.path()); + let input = + test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 3_600, &[]); + seed_baseline(&ctx, &input).await; + seed_valid_cluster_cache(&ctx, &input).await; + set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::NORMAL); + + let mut result = download_input(&ctx, &input, false).await; + let requests = server.finish(); + + assert!(result.errors.is_empty()); + assert!(result.storage_error.is_none()); + assert!(result.quality_rejections.is_empty()); + assert_eq!(result.job_state(), InputJobState::Ready); + assert_baseline(&result.source.take_groups()); + assert!(requests.is_empty(), "valid cache should avoid provider requests: {requests:?}"); + let telemetry = result.input_telemetry.as_ref().expect("real cache acquisition telemetry"); + assert_eq!(telemetry.refresh_policy, InputRefreshPolicy::NORMAL); + assert!(telemetry + .clusters + .iter() + .all(|cluster| { cluster.requested && cluster.source == Some(PlaylistUpdateDataSource::Cache) })); +} + +async fn assert_provider_acquisition_survives_in_run_reuse(policy: InputRefreshPolicy) { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([2, 2, 2]); + let mut ctx = processing_context(temp.path()); + let input = test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 95, vod: 95, series: 95 }, 0, &[]); + seed_baseline(&ctx, &input).await; + set_refresh_policy(&mut ctx, &input, policy); + + let mut first = download_input(&ctx, &input, false).await; + + assert!(first.errors.is_empty(), "first provider acquisition failed: {:?}", first.errors); + assert!(first.storage_error.is_none()); + assert_eq!(first.source.get_channel_count(), 6); + let first_telemetry = first.input_telemetry.as_ref().expect("provider acquisition telemetry"); + assert_eq!(first_telemetry.refresh_policy, policy); + assert!(first_telemetry + .clusters + .iter() + .all(|cluster| { cluster.requested && cluster.source == Some(PlaylistUpdateDataSource::Provider) })); + + let storage_path = input_storage_path(&ctx, &input).await; + let status_after_acquisition = input_cache::load_input_status(&storage_path); + assert!(XTREAM_CLUSTER.iter().all(|cluster| { + status_after_acquisition.clusters[cluster.as_ref()].last_update.as_ref().is_some_and(|snapshot| { + snapshot.policy == Some(policy) && snapshot.source == Some(PlaylistUpdateDataSource::Provider) + }) + })); + + // A second source/target in the same run reuses the persisted input. It is not a + // second acquisition and therefore must not emit replacement cache telemetry. + let mut reused = download_input(&ctx, &input, false).await; + + assert!(reused.errors.is_empty(), "in-run reuse failed: {:?}", reused.errors); + assert!(reused.storage_error.is_none()); + assert_eq!(reused.source.get_channel_count(), 6); + assert_eq!(reused.input_telemetry, None); + assert_eq!(input_cache::load_input_status(&storage_path), status_after_acquisition); + assert_requested_actions( + server.finish(), + &[ + "login", + "get_live_categories", + "get_live_streams", + "get_vod_categories", + "get_vod_streams", + "get_series_categories", + "get_series", + ], + ); +} + +#[tokio::test] +async fn in_run_reuse_after_force_keeps_provider_acquisition_and_snapshot() { + assert_provider_acquisition_survives_in_run_reuse(InputRefreshPolicy::FORCE).await; +} + +#[tokio::test] +async fn in_run_reuse_after_refresh_keeps_provider_acquisition_and_snapshot() { + assert_provider_acquisition_survives_in_run_reuse(InputRefreshPolicy::REFRESH).await; +} + +#[tokio::test] +async fn in_run_reuse_after_normal_provider_fetch_keeps_provider_acquisition_and_snapshot() { + assert_provider_acquisition_survives_in_run_reuse(InputRefreshPolicy::NORMAL).await; +} + +#[tokio::test] +async fn in_run_reuse_for_parallel_sources_emits_exactly_one_provider_acquisition() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([2, 2, 2]); + let mut ctx = processing_context(temp.path()); + let input = test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 95, vod: 95, series: 95 }, 0, &[]); + seed_baseline(&ctx, &input).await; + set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); + + let (mut first_source, mut second_source) = + tokio::join!(download_input(&ctx, &input, false), download_input(&ctx, &input, false)); + + assert!(first_source.errors.is_empty(), "first source failed: {:?}", first_source.errors); + assert!(second_source.errors.is_empty(), "second source failed: {:?}", second_source.errors); + assert_eq!(first_source.source.get_channel_count(), 6); + assert_eq!(second_source.source.get_channel_count(), 6); + let acquisition_telemetry = [first_source.input_telemetry.as_ref(), second_source.input_telemetry.as_ref()]; + assert_eq!(acquisition_telemetry.iter().filter(|telemetry| telemetry.is_some()).count(), 1); + let acquisition = acquisition_telemetry.into_iter().flatten().next().expect("one provider acquisition"); + assert_eq!(acquisition.refresh_policy, InputRefreshPolicy::FORCE); + assert!(acquisition + .clusters + .iter() + .all(|cluster| { cluster.requested && cluster.source == Some(PlaylistUpdateDataSource::Provider) })); + assert_requested_actions( + server.finish(), + &[ + "login", + "get_live_categories", + "get_live_streams", + "get_vod_categories", + "get_vod_streams", + "get_series_categories", + "get_series", + ], + ); +} + +#[tokio::test] +async fn refresh_policy_bypasses_valid_cache_and_keeps_quality_enforced() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([2, 1, 2]); + let mut ctx = processing_context(temp.path()); + let input = + test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 3_600, &[]); + seed_baseline(&ctx, &input).await; + seed_valid_cluster_cache(&ctx, &input).await; + set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::REFRESH); + + let mut result = download_input(&ctx, &input, false).await; + let requests = server.finish(); + + assert!(result.errors.is_empty()); + assert!(result.storage_error.is_none()); + assert_eq!(result.job_state(), InputJobState::Ready); + assert_eq!(result.quality_rejections.len(), 1); + assert_eq!(result.quality_rejections[0].cluster, XtreamCluster::Video); + let groups = result.source.take_groups(); + assert_eq!( + groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Video).map(|group| group.title.as_ref()), + Some("old-vod") + ); + assert_requested_actions( + requests, + &[ + "login", + "get_live_categories", + "get_live_streams", + "get_vod_categories", + "get_vod_streams", + "get_series_categories", + "get_series", + ], + ); +} + +#[tokio::test] +async fn force_policy_bypasses_valid_cache_and_quality_and_accepts_an_empty_cluster() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([2, 0, 1]); + let events = CollectSink::default(); + let mut ctx = processing_context_with_events(temp.path(), events.clone()); + let input = + test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 95, series: 100 }, 3_600, &[]); + seed_baseline(&ctx, &input).await; + seed_valid_cluster_cache(&ctx, &input).await; + set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); + + let mut result = process_input_job_inner(0, &ctx, &input).await; + let requests = server.finish(); + + assert!(result.errors.is_empty(), "forced update failed: {:?}", result.errors); + assert_eq!(result.state, InputJobState::Ready); + let groups = result.source.take().expect("forced input source").take_groups(); + assert_eq!( + groups + .iter() + .filter(|group| group.xtream_cluster == XtreamCluster::Live) + .map(|group| group.channels.len()) + .sum::(), + 2 + ); + assert!(groups.iter().all(|group| group.xtream_cluster != XtreamCluster::Video)); + assert_eq!( + groups + .iter() + .filter(|group| group.xtream_cluster == XtreamCluster::Series) + .map(|group| group.channels.len()) + .sum::(), + 1 + ); + assert_eq!( + PlaylistRunSignals { + has_error: !result.errors.is_empty(), + has_pending_stalker_refresh: ctx.partial_refresh.load(Ordering::Acquire), + has_quality_rejections: ctx.had_quality_rejections.load(Ordering::Acquire), + } + .state(), + PlaylistUpdateState::Success + ); + + let storage_path = input_storage_path(&ctx, &input).await; + assert_eq!( + count_input_xtream_cluster(&ctx.config, &input, XtreamCluster::Video) + .await + .expect("forced empty VOD should remain countable"), + Some(0) + ); + let status = input_cache::load_input_status(&storage_path); + assert!(XTREAM_CLUSTER.iter().all(|cluster| { + status.clusters.get(cluster.as_ref()).map(|entry| &entry.status) == Some(&input_cache::ClusterState::Ok) + })); + let messages = events + .0 + .lock() + .expect("event sink lock") + .iter() + .filter_map(|event| match event { + EventMessage::PlaylistUpdateProgress(progress) => Some(progress.message.clone()), + _ => None, + }) + .collect::>(); + assert!(messages.iter().any(|message| { + message + == "Input 'quality-provider': forced update requested; cache and update-quality bypassed for live,vod,series" + })); + assert!(messages.iter().any(|message| { + message == "Input 'quality-provider' cluster 'vod' force-published: candidate=0 configured_threshold=95" + })); + assert_requested_actions( + requests, + &[ + "login", + "get_live_categories", + "get_live_streams", + "get_vod_categories", + "get_vod_streams", + "get_series_categories", + "get_series", + ], + ); +} + +#[tokio::test] +async fn force_empty_vod_is_published_to_xtream_target_and_memory_cache() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([2, 0, 1]); + let mut ctx = processing_context(temp.path()); + let input = test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 95, series: 100 }, 0, &[]); + let target = xtream_target(true); + let playlist_state = Arc::new(PlaylistStorageState::new()); + ctx.playlist_state = Some(Arc::clone(&playlist_state)); + seed_baseline(&ctx, &input).await; + let old_vod_id = seed_xtream_target(&ctx, &target, &playlist_state).await; + install_source(&ctx, &input, &target); + set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); + let app_config = Arc::clone(&ctx.config); + let pending_refresh = Arc::clone(&ctx.partial_refresh); + let quality_rejections = Arc::clone(&ctx.had_quality_rejections); + + let (_, target_stats, errors) = process_source(0, Arc::new(ctx)).await; + server.finish(); + + assert!(errors.is_empty(), "forced target update failed: {errors:?}"); + assert_eq!(target_stats.len(), 1); + assert!(target_stats[0].success); + assert_eq!( + PlaylistRunSignals { + has_error: false, + has_pending_stalker_refresh: pending_refresh.load(Ordering::Acquire), + has_quality_rejections: quality_rejections.load(Ordering::Acquire), + } + .state(), + PlaylistUpdateState::Success + ); + assert_target_counts(&app_config, &target, [2, 0, 1]).await; + assert_empty_target_categories(&app_config, &target, &[XtreamCluster::Video]).await; + assert!( + xtream_get_item_for_stream_id(old_vod_id, &app_config, &playlist_state, &target, Some(XtreamCluster::Video),) + .await + .is_err(), + "old VOD id must not remain client-visible" + ); + let cache = playlist_state.data.read().await; + let cached = + cache.get(&target.name).and_then(|storage| storage.xtream.as_ref()).expect("Xtream target memory cache"); + assert_eq!([cached.live.len(), cached.vod.len(), cached.series.len()], [2, 0, 1]); +} + +#[tokio::test] +async fn force_fully_empty_result_clears_the_xtream_target() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([0, 0, 0]); + let mut ctx = processing_context(temp.path()); + let input = test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 90, vod: 90, series: 90 }, 0, &[]); + let target = xtream_target(true); + let playlist_state = Arc::new(PlaylistStorageState::new()); + ctx.playlist_state = Some(Arc::clone(&playlist_state)); + seed_baseline(&ctx, &input).await; + let old_vod_id = seed_xtream_target(&ctx, &target, &playlist_state).await; + install_source(&ctx, &input, &target); + set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); + let app_config = Arc::clone(&ctx.config); + let pending_refresh = Arc::clone(&ctx.partial_refresh); + let quality_rejections = Arc::clone(&ctx.had_quality_rejections); + + let (_, target_stats, errors) = process_source(0, Arc::new(ctx)).await; + server.finish(); + + assert!(errors.is_empty(), "fully empty forced target update failed: {errors:?}"); + assert_eq!(target_stats.len(), 1); + assert!(target_stats[0].success); + assert_eq!( + PlaylistRunSignals { + has_error: false, + has_pending_stalker_refresh: pending_refresh.load(Ordering::Acquire), + has_quality_rejections: quality_rejections.load(Ordering::Acquire), + } + .state(), + PlaylistUpdateState::Success + ); + assert_target_counts(&app_config, &target, [0, 0, 0]).await; + assert_empty_target_categories(&app_config, &target, &XTREAM_CLUSTER).await; + assert!( + xtream_get_item_for_stream_id(old_vod_id, &app_config, &playlist_state, &target, Some(XtreamCluster::Video),) + .await + .is_err(), + "fully empty target must not resolve an old VOD id" + ); + let cache = playlist_state.data.read().await; + let cached = + cache.get(&target.name).and_then(|storage| storage.xtream.as_ref()).expect("Xtream target memory cache"); + assert_eq!([cached.live.len(), cached.vod.len(), cached.series.len()], [0, 0, 0]); +} + +#[tokio::test] +async fn force_technical_parse_failure_retains_the_target_cluster() { + let temp = tempfile::tempdir().expect("temporary storage"); + let mut responses = fixture_responses([2, 1, 2]); + responses.insert("get_vod_streams".to_string(), "{".to_string()); + let server = TestXtreamServer::start_with_responses(responses); + let mut ctx = processing_context(temp.path()); + let input = test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 0, &[]); + let target = xtream_target(true); + let playlist_state = Arc::new(PlaylistStorageState::new()); + ctx.playlist_state = Some(Arc::clone(&playlist_state)); + seed_baseline(&ctx, &input).await; + let old_vod_id = seed_xtream_target(&ctx, &target, &playlist_state).await; + install_source(&ctx, &input, &target); + set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); + let app_config = Arc::clone(&ctx.config); + + let (_, target_stats, errors) = process_source(0, Arc::new(ctx)).await; + server.finish(); + + assert!(!errors.is_empty(), "malformed provider response must remain a technical error"); + assert_eq!(target_stats.len(), 1); + assert!(target_stats[0].success, "usable fallback should still be published"); + assert_target_counts(&app_config, &target, [2, 2, 2]).await; + let retained_vod = + xtream_get_item_for_stream_id(old_vod_id, &app_config, &playlist_state, &target, Some(XtreamCluster::Video)) + .await + .expect("old VOD should remain client-visible after technical failure"); + assert!(retained_vod.name.starts_with("old-vod")); +} + +#[tokio::test] +async fn normal_empty_candidate_retains_the_xtream_target_without_empty_authorization() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([0, 0, 0]); + let mut ctx = processing_context(temp.path()); + let input = test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 0, &[]); + let target = xtream_target(true); + let playlist_state = Arc::new(PlaylistStorageState::new()); + ctx.playlist_state = Some(Arc::clone(&playlist_state)); + seed_baseline(&ctx, &input).await; + let old_vod_id = seed_xtream_target(&ctx, &target, &playlist_state).await; + install_source(&ctx, &input, &target); + let app_config = Arc::clone(&ctx.config); + let quality_rejections = Arc::clone(&ctx.had_quality_rejections); + + let (_, target_stats, errors) = process_source(0, Arc::new(ctx)).await; + server.finish(); + + assert!(errors.is_empty(), "quality rejection must not become a technical error: {errors:?}"); + assert!(quality_rejections.load(Ordering::Acquire)); + assert_eq!(target_stats.len(), 1); + assert!(target_stats[0].success); + assert_target_counts(&app_config, &target, [2, 2, 2]).await; + let retained_vod = + xtream_get_item_for_stream_id(old_vod_id, &app_config, &playlist_state, &target, Some(XtreamCluster::Video)) + .await + .expect("normal empty candidate must retain the target VOD"); + assert!(retained_vod.name.starts_with("old-vod")); +} + +#[tokio::test] +async fn force_policy_retains_a_cluster_after_a_technical_parse_failure() { + let temp = tempfile::tempdir().expect("temporary storage"); + let mut responses = fixture_responses([2, 1, 2]); + responses.insert("get_vod_streams".to_string(), "{".to_string()); + let server = TestXtreamServer::start_with_responses(responses); + let mut ctx = processing_context(temp.path()); + let input = test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 0, &[]); + seed_baseline(&ctx, &input).await; + set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); + + let mut result = download_input(&ctx, &input, false).await; + let requests = server.finish(); + + assert!(!result.errors.is_empty(), "malformed VOD response must remain a technical error"); + assert!(result.storage_error.is_none()); + assert!(result.quality_rejections.is_empty()); + let groups = result.source.take_groups(); + let vod = groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Video).expect("retained VOD"); + assert_eq!(vod.title.as_ref(), "old-vod"); + assert_eq!(vod.channels.len(), 2); + assert_eq!( + groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Live).map(|group| group.title.as_ref()), + Some("candidate-live") + ); + assert_eq!( + groups.iter().find(|group| group.xtream_cluster == XtreamCluster::Series).map(|group| group.title.as_ref()), + Some("candidate-series") + ); + assert_requested_actions( + requests, + &[ + "login", + "get_live_categories", + "get_live_streams", + "get_vod_categories", + "get_vod_streams", + "get_series_categories", + "get_series", + ], + ); +} + +#[tokio::test] +async fn in_memory_series_quality_compares_catalog_rows_with_an_enriched_baseline() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([0, 0, 1]); + let ctx = processing_context(temp.path()); + let input = test_input( + &server.base_url, + ConfigInputUpdateQualityDto { series: 100, ..ConfigInputUpdateQualityDto::default() }, + 0, + &[XtreamCluster::Live, XtreamCluster::Video], + ); + let mut baseline = baseline_group(XtreamCluster::Series, 3, "old-series", 3_000); + baseline.channels.truncate(1); + let episodes = (0..200_u32) + .map(|id| SeriesStreamDetailEpisodeProperties { id, ..SeriesStreamDetailEpisodeProperties::default() }) + .collect(); + baseline.channels[0].header.additional_properties = + Some(StreamProperties::Series(Box::new(SeriesStreamProperties { + series_id: 3_000, + details: Some(SeriesStreamDetailProperties::new(None, Vec::new(), Some(episodes))), + ..SeriesStreamProperties::default() + }))); + let (persisted, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, vec![baseline]).await; + assert!(error.is_none(), "enriched baseline persistence failed: {error:?}"); + assert_eq!(persisted.iter().map(|group| group.channels.len()).sum::(), 1); + assert_eq!( + count_input_xtream_cluster(&ctx.config, &input, XtreamCluster::Series) + .await + .expect("enriched baseline should be countable"), + Some(1) + ); + + let mut result = download_input(&ctx, &input, false).await; + let requests = server.finish(); + + assert!(result.errors.is_empty(), "Series update failed: {:?}", result.errors); + assert!(result.quality_rejections.is_empty()); + let groups = result.source.take_groups(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].title.as_ref(), "candidate-series"); + assert_eq!(groups[0].channels.len(), 1); + let episode_count = groups[0].channels[0] + .header + .additional_properties + .as_ref() + .and_then(|properties| match properties { + StreamProperties::Series(series) => series.details.as_ref(), + _ => None, + }) + .and_then(|details| details.episodes.as_ref()) + .map(Vec::len); + assert_eq!(episode_count, Some(200)); + assert_requested_actions(requests, &["login", "get_series_categories", "get_series"]); +} + +#[tokio::test] +async fn in_memory_quality_rejection_counts_duplicate_provider_ids_once_and_retains_the_baseline() { + let temp = tempfile::tempdir().expect("temporary storage"); + let first_provider_id = 1_000_u32; + let streams = (0..50_u32) + .flat_map(|offset| { + let provider_id = first_provider_id + offset; + [ + live_fixture_stream(provider_id, 1, &format!("candidate-{provider_id}")), + live_fixture_stream(provider_id, 1, &format!("duplicate-{provider_id}")), + ] + }) + .collect(); + let responses = + live_fixture_responses(&serde_json::json!([{"category_id": 1, "category_name": "candidate-live"}]), streams); + let server = TestXtreamServer::start_with_responses(responses); + let ctx = processing_context(temp.path()); + let input = test_input( + &server.base_url, + ConfigInputUpdateQualityDto { live: 100, ..ConfigInputUpdateQualityDto::default() }, + 0, + &[XtreamCluster::Video, XtreamCluster::Series], + ); + seed_live_baseline(&ctx, &input, first_provider_id).await; + + let mut result = download_input(&ctx, &input, false).await; + let requests = server.finish(); + + assert!(result.errors.is_empty(), "duplicate rejection must not become an error: {:?}", result.errors); + assert!(result.storage_error.is_none()); + assert_eq!( + result.quality_rejections, + vec![ClusterUpdateRejection { + cluster: XtreamCluster::Live, + current_count: 100, + candidate_count: 50, + threshold: 100, + quality: 50, + }] + ); + let groups = result.source.take_groups(); + assert_eq!(groups.iter().map(|group| group.channels.len()).sum::(), 100); + assert_eq!(groups[0].title.as_ref(), "old-live"); + + let mut reloaded = load_input_playlist(&ctx.config, &input, None).await.expect("reload retained Live baseline"); + let reloaded_groups = reloaded.take_groups(); + assert_eq!(reloaded_groups.iter().map(|group| group.channels.len()).sum::(), 100); + assert_eq!(reloaded_groups[0].title.as_ref(), "old-live"); + assert_eq!( + count_input_xtream_cluster(&ctx.config, &input, XtreamCluster::Live) + .await + .expect("retained baseline should be countable"), + Some(100) + ); + assert_requested_actions(requests, &["login", "get_live_categories", "get_live_streams"]); +} + +#[tokio::test] +async fn in_memory_quality_accepts_unique_population_and_persists_the_last_duplicate_row() { + let temp = tempfile::tempdir().expect("temporary storage"); + let first_provider_id = 1_000_u32; + let winning_provider_id = first_provider_id + 42; + let mut streams = (0..100_u32) + .map(|offset| { + let provider_id = first_provider_id + offset; + live_fixture_stream(provider_id, 1, &format!("candidate-{provider_id}")) + }) + .collect::>(); + streams.push(live_fixture_stream(winning_provider_id, 2, "winning-duplicate")); + let responses = live_fixture_responses( + &serde_json::json!([ + {"category_id": 1, "category_name": "candidate-live"}, + {"category_id": 2, "category_name": "winning-category"} + ]), + streams, + ); + let server = TestXtreamServer::start_with_responses(responses); + let ctx = processing_context(temp.path()); + let input = test_input( + &server.base_url, + ConfigInputUpdateQualityDto { live: 100, ..ConfigInputUpdateQualityDto::default() }, + 0, + &[XtreamCluster::Video, XtreamCluster::Series], + ); + seed_live_baseline(&ctx, &input, first_provider_id).await; + + let mut result = download_input(&ctx, &input, false).await; + let requests = server.finish(); + + assert!(result.errors.is_empty(), "deduplicated acceptance failed: {:?}", result.errors); + assert!(result.storage_error.is_none()); + assert!(result.quality_rejections.is_empty()); + let groups = result.source.take_groups(); + assert_eq!(groups.iter().map(|group| group.channels.len()).sum::(), 100); + let winner_group = groups.iter().find(|group| group.id == 2).expect("winning category"); + assert_eq!(winner_group.title.as_ref(), "winning-category"); + assert_eq!(winner_group.channels.len(), 1); + assert_eq!(winner_group.channels[0].header.id.as_ref(), winning_provider_id.to_string()); + assert_eq!(winner_group.channels[0].header.name.as_ref(), "winning-duplicate"); + assert_eq!(winner_group.channels[0].header.category_id, 2); + + let mut reloaded = load_input_playlist(&ctx.config, &input, None).await.expect("reload accepted Live candidate"); + let reloaded_groups = reloaded.take_groups(); + assert_eq!(reloaded_groups.iter().map(|group| group.channels.len()).sum::(), 100); + let reloaded_winner = reloaded_groups.iter().find(|group| group.id == 2).expect("persisted winning category"); + assert_eq!(reloaded_winner.channels.len(), 1); + assert_eq!(reloaded_winner.channels[0].header.id.as_ref(), winning_provider_id.to_string()); + assert_eq!(reloaded_winner.channels[0].header.name.as_ref(), "winning-duplicate"); + assert_eq!(reloaded_winner.channels[0].header.category_id, 2); + assert_eq!( + count_input_xtream_cluster(&ctx.config, &input, XtreamCluster::Live) + .await + .expect("accepted candidate should be countable"), + Some(100) + ); + assert_requested_actions(requests, &["login", "get_live_categories", "get_live_streams"]); +} + +#[tokio::test] +async fn m3u_update_quality_enforces_the_existing_cluster_threshold() { + let temp = tempfile::tempdir().expect("temporary storage"); + let playlist_path = temp.path().join("input.m3u"); + tokio::fs::write( + &playlist_path, + "#EXTM3U\n#EXTINF:-1,Old One\nhttp://old.example/1\n#EXTINF:-1,Old Two\nhttp://old.example/2\n", + ) + .await + .expect("initial M3U fixture"); + let options = ConfigInputOptionsDto { + update_quality: ConfigInputUpdateQualityDto { live: 100, vod: 0, series: 0 }, + ..ConfigInputOptionsDto::default() + }; + let input = Arc::new(ConfigInput { + id: 2, + name: "m3u-quality-guard".intern(), + input_type: InputType::M3u, + url: playlist_path.to_string_lossy().into_owned(), + enabled: true, + options: Some(ConfigInputOptions::from(&options)), + ..ConfigInput::default() + }); + let initial_ctx = processing_context(temp.path()); + + let mut initial = download_input(&initial_ctx, &input, false).await; + + assert!(initial.errors.is_empty(), "initial M3U download failed: {:?}", initial.errors); + assert!(initial.quality_rejections.is_empty()); + assert_eq!(initial.source.get_channel_count(), 2); + + tokio::fs::write(&playlist_path, "#EXTM3U\n#EXTINF:-1,New\nhttp://new.example/1\n") + .await + .expect("replacement M3U fixture"); + let refreshed_ctx = processing_context(temp.path()); + let mut refreshed = download_input(&refreshed_ctx, &input, false).await; + + assert!(refreshed.errors.is_empty(), "replacement M3U download failed: {:?}", refreshed.errors); + assert!(refreshed.storage_error.is_none()); + assert_eq!(refreshed.quality_rejections.len(), 1); + assert_eq!(refreshed.quality_rejections[0].cluster, XtreamCluster::Live); + assert_eq!(refreshed.source.get_channel_count(), 2); + let mut reloaded = + load_input_playlist(&refreshed_ctx.config, &input, None).await.expect("replacement M3U should be persisted"); + assert_eq!(reloaded.get_channel_count(), 2); +} + +#[tokio::test] +async fn quality_rejection_for_only_requested_vod_loads_complete_persisted_input() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([0, 1, 0]); + let ctx = processing_context(temp.path()); + let input = test_input( + &server.base_url, + ConfigInputUpdateQualityDto { vod: 100, ..ConfigInputUpdateQualityDto::default() }, + 3_600, + &[], + ); + seed_baseline(&ctx, &input).await; + let storage_path = input_storage_path(&ctx, &input).await; + let now = SystemTime::now().duration_since(UNIX_EPOCH).expect("current time").as_secs(); + let mut status = input_cache::InputStatus::default(); + status.clusters.insert( + XtreamCluster::Live.as_ref().to_string(), + input_cache::ClusterStatus { status: input_cache::ClusterState::Ok, timestamp: now, last_update: None }, + ); + status.clusters.insert( + XtreamCluster::Video.as_ref().to_string(), + input_cache::ClusterStatus { + status: input_cache::ClusterState::Ok, + timestamp: now.saturating_sub(3_601), + last_update: None, + }, + ); + status.clusters.insert( + XtreamCluster::Series.as_ref().to_string(), + input_cache::ClusterStatus { status: input_cache::ClusterState::Ok, timestamp: now, last_update: None }, + ); + input_cache::save_input_status(&storage_path, &status); + + let mut result = download_input(&ctx, &input, false).await; + let requests = server.finish(); + + assert!(result.errors.is_empty(), "quality rejection must not become an error: {:?}", result.errors); + assert!(result.storage_error.is_none()); + assert!(!result.partial); + assert_eq!(result.job_state(), InputJobState::Ready); + assert_eq!(result.quality_rejections.len(), 1); + assert_eq!( + result.quality_rejections[0], + ClusterUpdateRejection { + cluster: XtreamCluster::Video, + current_count: 2, + candidate_count: 1, + threshold: 100, + quality: 50, + } + ); + assert_eq!( + PlaylistRunSignals { + has_quality_rejections: !result.quality_rejections.is_empty(), + ..PlaylistRunSignals::default() + } + .state(), + PlaylistUpdateState::Partial + ); + assert_baseline(&result.source.take_groups()); + + let status = input_cache::load_input_status(&storage_path); + let live = status.clusters.get(XtreamCluster::Live.as_ref()).expect("Live cache status"); + let vod = status.clusters.get(XtreamCluster::Video.as_ref()).expect("VOD cache status"); + let series = status.clusters.get(XtreamCluster::Series.as_ref()).expect("Series cache status"); + assert_eq!(live.status, input_cache::ClusterState::Ok); + assert_eq!(live.timestamp, now); + assert_eq!(vod.status, input_cache::ClusterState::Failed); + assert_eq!(series.status, input_cache::ClusterState::Ok); + assert_eq!(series.timestamp, now); + assert_requested_actions(requests, &["login", "get_vod_categories", "get_vod_streams"]); +} + +#[tokio::test] +async fn all_requested_clusters_rejected_load_the_complete_unchanged_baseline() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([1, 1, 1]); + let ctx = processing_context(temp.path()); + let input = test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 100, vod: 100, series: 100 }, 0, &[]); + seed_baseline(&ctx, &input).await; + + let mut result = download_input(&ctx, &input, false).await; + let requests = server.finish(); + + assert!(result.errors.is_empty(), "quality rejections must not become errors: {:?}", result.errors); + assert!(result.storage_error.is_none()); + assert_eq!(result.job_state(), InputJobState::Ready); + assert_eq!(result.quality_rejections.iter().map(|rejection| rejection.cluster).collect::>(), XTREAM_CLUSTER); + assert!(result.quality_rejections.iter().all(|rejection| { + rejection.current_count == 2 + && rejection.candidate_count == 1 + && rejection.threshold == 100 + && rejection.quality == 50 + })); + assert_baseline(&result.source.take_groups()); + + let mut reloaded = load_input_playlist(&ctx.config, &input, None).await.expect("reload persisted baseline"); + assert_baseline(&reloaded.take_groups()); + let storage_path = input_storage_path(&ctx, &input).await; + let status = input_cache::load_input_status(&storage_path); + assert!(XTREAM_CLUSTER.iter().all(|cluster| { + status.clusters.get(cluster.as_ref()).map(|entry| &entry.status) == Some(&input_cache::ClusterState::Failed) + })); + assert_requested_actions( + requests, + &[ + "login", + "get_live_categories", + "get_live_streams", + "get_vod_categories", + "get_vod_streams", + "get_series_categories", + "get_series", + ], + ); +} + +#[tokio::test] +async fn quality_rejection_for_empty_bootstrap_does_not_invent_a_baseline() { + let temp = tempfile::tempdir().expect("temporary storage"); + let server = TestXtreamServer::start([0, 0, 0]); + let ctx = processing_context(temp.path()); + let input = test_input( + &server.base_url, + ConfigInputUpdateQualityDto { vod: 100, ..ConfigInputUpdateQualityDto::default() }, + 0, + &[XtreamCluster::Live, XtreamCluster::Series], + ); + + let mut result = download_input(&ctx, &input, false).await; + let requests = server.finish(); + + assert!(result.errors.is_empty(), "unexpected bootstrap errors: {:?}", result.errors); + assert!(result.storage_error.is_none()); + assert_eq!(result.quality_rejections.len(), 1); + assert_eq!( + result.quality_rejections[0], + ClusterUpdateRejection { + cluster: XtreamCluster::Video, + current_count: 0, + candidate_count: 0, + threshold: 100, + quality: 0, + } + ); + assert!(result.source.is_empty()); + assert_eq!(result.job_state(), InputJobState::Failed); + let mut reloaded = load_input_playlist(&ctx.config, &input, None).await.expect("empty bootstrap reload"); + assert!(reloaded.take_groups().is_empty()); + assert_requested_actions(requests, &["login", "get_vod_categories", "get_vod_streams"]); +} + +mod m3u_update_quality; +mod manual_update_integration; +mod persisted_input_status; +mod staged_completion; diff --git a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/m3u_update_quality.rs b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/m3u_update_quality.rs new file mode 100644 index 000000000..7e93b080d --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/m3u_update_quality.rs @@ -0,0 +1,509 @@ +use super::*; +use std::fmt::Write; +use tuliprox_repository::{load_input_m3u_playlist, raw_group_catalog_path, RawInputGroupCatalog}; + +fn document(prefix: &str, counts: [usize; 3]) -> String { + let mut text = String::from("#EXTM3U\n"); + for (cluster, count) in XTREAM_CLUSTER.into_iter().zip(counts) { + for index in 0..count { + let (kind, group, title) = match cluster { + XtreamCluster::Live => ("live", format!("{prefix}-live"), format!("Channel {index}")), + XtreamCluster::Video => ("movie", format!("{prefix}-vod"), format!("Movie {index}")), + XtreamCluster::Series => ("series", format!("{prefix}-show-{index}"), format!("Show {index} S01E01")), + }; + let id = match cluster { + XtreamCluster::Live => 1000, + XtreamCluster::Video => 2000, + XtreamCluster::Series => 3000, + } + index; + writeln!(text, "#EXTGRP:{prefix}-{cluster}\n#EXTINF:-1 xui-id=\"{id}\" tvg-type=\"{kind}\" group-title=\"{group}\",{title}\nhttp://stream.example/{prefix}/{cluster}/{id}").unwrap(); + } + } + text +} + +async fn fixture( + root: &Path, + previous: Option<[usize; 3]>, + thresholds: [u8; 3], +) -> (PlaylistProcessingContext, Arc) { + let ctx = processing_context(root); + let input = Arc::new(ConfigInput { + id: 27, + name: "m3u-quality".intern(), + input_type: InputType::M3u, + enabled: true, + url: root.join("provider.m3u").to_string_lossy().into_owned(), + options: Some(ConfigInputOptions::from(&ConfigInputOptionsDto { + update_quality: ConfigInputUpdateQualityDto { + live: thresholds[0], + vod: thresholds[1], + series: thresholds[2], + }, + ..ConfigInputOptionsDto::default() + })), + ..ConfigInput::default() + }); + if let Some(counts) = previous { + tokio::fs::write(&input.url, document("previous", counts)).await.unwrap(); + let (groups, errors) = + tuliprox_iptv::m3u::download_m3u_playlist(&ctx.config, &ctx.client, &ctx.config.config.load(), &input) + .await; + assert!(errors.is_empty()); + let (_, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, groups).await; + assert!(error.is_none(), "{error:?}"); + } + (ctx, input) +} + +async fn run( + ctx: &PlaylistProcessingContext, + input: &Arc, + counts: [usize; 3], +) -> InputDownloadResult { + tokio::fs::write(&input.url, document("candidate", counts)).await.unwrap(); + download_input(ctx, input, false).await +} + +fn counts(groups: &[PlaylistGroup]) -> [usize; 3] { + XTREAM_CLUSTER.map(|cluster| groups.iter().filter(|g| g.xtream_cluster == cluster).map(|g| g.channels.len()).sum()) +} + +async fn verify_effective( + ctx: &PlaylistProcessingContext, + input: &ConfigInput, + result: &mut InputDownloadResult, + expected: [usize; 3], +) { + assert!(result.errors.is_empty(), "{:?}", result.errors); + assert!(result.storage_error.is_none(), "{:?}", result.storage_error); + assert_eq!(result.job_state(), InputJobState::Ready); + let effective = result.source.take_groups(); + assert_eq!(counts(&effective), expected); + let storage = + tuliprox_repository::get_input_storage_path(&input.name, &ctx.config.config.load().storage_dir).await.unwrap(); + let saved = load_input_m3u_playlist( + &ctx.config, + &tuliprox_repository::get_input_m3u_playlist_file_path(&storage, &input.name), + ) + .await + .unwrap(); + assert_eq!(counts(&saved), expected); + for cluster in XTREAM_CLUSTER { + let catalog: RawInputGroupCatalog = + serde_json::from_slice(&tokio::fs::read(raw_group_catalog_path(&storage, cluster)).await.unwrap()).unwrap(); + let mut expected_groups: Vec<_> = + effective.iter().filter(|g| g.xtream_cluster == cluster).map(|g| g.title.to_string()).collect(); + expected_groups.sort(); + expected_groups.dedup(); + assert_eq!(catalog.groups, expected_groups); + } +} + +#[tokio::test] +async fn m3u_update_quality_all_clusters_accepted() { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; + let mut result = run(&ctx, &input, [9, 11, 10]).await; + assert!(result.quality_rejections.is_empty()); + assert_eq!(result.update_state(), PlaylistUpdateState::Success); + verify_effective(&ctx, &input, &mut result, [9, 11, 10]).await; +} + +async fn order_candidate( + ctx: &PlaylistProcessingContext, + input: &ConfigInput, + prefix: &str, +) -> tuliprox_iptv::provider::PlaylistFetch { + let mut body = String::from("#EXTM3U\n"); + for (id, kind, group) in [ + (2000, "movie", "obsolete"), + (2000, "movie", "movies-a"), + (1000, "live", "live-a"), + (2001, "movie", "movies-b"), + (1001, "live", "live-b"), + (3000, "series", "shows-a"), + (3001, "series", "shows-b"), + ] { + writeln!(body, "#EXTGRP:{prefix}-{group}\n#EXTINF:-1 tvg-type=\"{kind}\" group-title=\"{prefix}-{group}\",Item {id} S01E01\nhttp://stream.example/{kind}/{id}").unwrap(); + } + tokio::fs::write(&input.url, body).await.unwrap(); + let (groups, errors) = + tuliprox_iptv::m3u::download_m3u_playlist(&ctx.config, &ctx.client, &ctx.config.config.load(), input).await; + assert!(errors.is_empty()); + assert_eq!( + group_names(&groups), + ["obsolete", "movies-a", "live-a", "movies-b", "live-b", "shows-a", "shows-b"] + .map(|name| format!("{prefix}-{name}")) + ); + tuliprox_iptv::provider::PlaylistFetch::groups(groups) +} + +fn group_names(groups: &[PlaylistGroup]) -> Vec { groups.iter().map(|group| group.title.to_string()).collect() } + +#[tokio::test] +async fn m3u_update_quality_preserves_candidate_group_order_when_disabled() { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), None, [0, 0, 0]).await; + let candidate = order_candidate(&ctx, &input, "candidate").await; + let effective = super::super::super::m3u_quality::effective_m3u_fetch( + &ctx.config, + &input, + shared::model::UpdateQualityPolicy::Enforce, + candidate, + ) + .await; + assert!(effective.errors.is_empty()); + assert!(effective.quality_acceptances.is_empty() && effective.quality_rejections.is_empty()); + assert_eq!( + group_names(&effective.groups), + ["movies-a", "live-a", "movies-b", "live-b", "shows-a", "shows-b"].map(|name| format!("candidate-{name}")) + ); + assert_eq!(counts(&effective.groups), [2, 2, 2], "disabled quality processing must collapse duplicate keys"); +} + +#[tokio::test] +async fn m3u_update_quality_preserves_candidate_group_order_when_all_clusters_accepted() { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), Some([2, 2, 2]), [100, 100, 100]).await; + let candidate = order_candidate(&ctx, &input, "candidate").await; + let effective = super::super::super::m3u_quality::effective_m3u_fetch( + &ctx.config, + &input, + shared::model::UpdateQualityPolicy::Enforce, + candidate, + ) + .await; + assert!(effective.errors.is_empty() && effective.quality_rejections.is_empty()); + assert_eq!(effective.quality_acceptances.len(), 3); + assert_eq!( + group_names(&effective.groups), + ["movies-a", "live-a", "movies-b", "live-b", "shows-a", "shows-b"].map(|name| format!("candidate-{name}")) + ); +} + +#[tokio::test] +async fn m3u_update_quality_mixed_fallback_preserves_relative_group_order() { + for _ in 0..3 { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), None, [100, 100, 100]).await; + let mut previous = order_candidate(&ctx, &input, "previous").await.groups; + previous.retain(|group| group.title.as_ref() != "previous-obsolete"); + let (_, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, previous).await; + assert!(error.is_none()); + let mut previous = load_input_playlist(&ctx.config, &input, None).await.unwrap(); + let previous = previous.take_groups(); + assert_eq!( + previous + .iter() + .filter(|group| group.xtream_cluster == XtreamCluster::Series) + .map(|group| group.title.to_string()) + .collect::>(), + ["previous-shows-a", "previous-shows-b"] + ); + let mut candidate = order_candidate(&ctx, &input, "candidate").await; + candidate.groups.retain(|group| group.title.as_ref() != "candidate-shows-b"); + let effective = super::super::super::m3u_quality::effective_m3u_fetch( + &ctx.config, + &input, + shared::model::UpdateQualityPolicy::Enforce, + candidate, + ) + .await; + assert!(effective.errors.is_empty()); + assert_eq!(effective.quality_rejections.len(), 1); + assert_eq!(effective.quality_rejections[0].cluster, XtreamCluster::Series); + assert_eq!( + group_names(&effective.groups), + [ + "candidate-movies-a", + "candidate-live-a", + "candidate-movies-b", + "candidate-live-b", + "previous-shows-a", + "previous-shows-b" + ] + ); + } +} + +#[tokio::test] +async fn m3u_update_quality_retains_only_rejected_cluster() { + for disk in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; + let mut config = (**ctx.config.config.load()).clone(); + config.disk_based_processing = disk; + ctx.config.config.store(Arc::new(config)); + let mut result = run(&ctx, &input, [9, 11, 8]).await; + assert_eq!(result.update_state(), PlaylistUpdateState::Partial); + assert_eq!(result.quality_rejections.len(), 1); + assert_eq!(result.quality_rejections[0].cluster, XtreamCluster::Series); + verify_effective(&ctx, &input, &mut result, [9, 11, 10]).await; + } +} + +#[tokio::test] +async fn m3u_update_quality_zero_candidate_retains_previous_cluster() { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; + let mut result = run(&ctx, &input, [10, 10, 0]).await; + assert_eq!(result.quality_rejections[0].candidate_count, 0); + verify_effective(&ctx, &input, &mut result, [10, 10, 10]).await; +} + +#[tokio::test] +async fn m3u_update_quality_disabled() { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [0, 0, 0]).await; + let mut result = run(&ctx, &input, [1, 1, 0]).await; + assert!(result.quality_rejections.is_empty()); + assert!(result + .input_telemetry + .as_ref() + .unwrap() + .clusters + .iter() + .all(|c| c.quality.is_none() && c.threshold.is_none())); + verify_effective(&ctx, &input, &mut result, [1, 1, 0]).await; +} + +#[tokio::test] +async fn m3u_update_quality_first_import() { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), None, [90, 90, 90]).await; + let mut result = run(&ctx, &input, [1, 1, 1]).await; + assert!(result.quality_rejections.is_empty()); + assert!(result.input_telemetry.as_ref().unwrap().clusters.iter().all(|c| c.quality.is_none() + && c.baseline_count.is_none() + && c.decision == Some(PlaylistUpdateClusterDecision::Accepted))); + verify_effective(&ctx, &input, &mut result, [1, 1, 1]).await; +} + +#[tokio::test] +async fn m3u_update_quality_download_or_parse_failure() { + for body in [None, Some("provider unavailable"), Some("#EXTM3U\n#EXTINF:-1,Truncated\n")] { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; + if let Some(body) = body { + tokio::fs::write(&input.url, body).await.unwrap(); + } else { + tokio::fs::remove_file(&input.url).await.unwrap(); + } + let mut result = download_input(&ctx, &input, false).await; + assert!(!result.errors.is_empty()); + assert!(result.quality_rejections.is_empty()); + assert_eq!(result.update_state(), PlaylistUpdateState::Failure); + let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); + assert_eq!(counts(&saved.take_groups()), [10, 10, 10]); + } +} + +#[tokio::test] +async fn m3u_update_quality_empty_bootstrap_and_structural_empty_use_existing_domain() { + for previous in [None, Some([10, 10, 0])] { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), previous, [90, 90, 90]).await; + let mut result = run(&ctx, &input, if previous.is_some() { [10, 10, 0] } else { [0, 0, 0] }).await; + if previous.is_none() { + assert_eq!(result.update_state(), PlaylistUpdateState::Failure); + assert_eq!(result.quality_rejections.len(), 3); + } else { + assert_eq!(result.update_state(), PlaylistUpdateState::Partial); + assert_eq!(result.quality_rejections.len(), 1); + assert_eq!(result.quality_rejections[0].current_count, 0); + assert_eq!(result.quality_rejections[0].cluster, XtreamCluster::Series); + } + } +} + +#[tokio::test] +async fn m3u_update_quality_skip_preserves_previous_and_reuse_emits_no_new_facts() { + let temp = tempfile::tempdir().unwrap(); + let (ctx, mut input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; + Arc::make_mut(&mut input).options.as_mut().unwrap().flags.set(ConfigInputFlags::SkipSeries); + let mut result = run(&ctx, &input, [10, 10, 0]).await; + assert_eq!(result.update_state(), PlaylistUpdateState::Success); + assert!(result.quality_rejections.is_empty()); + let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); + assert_eq!(counts(&result.source.take_groups()), [10, 10, 0]); + assert_eq!(counts(&saved.take_groups()), [10, 10, 10]); + let telemetry = result.input_telemetry.unwrap(); + assert!(!telemetry.clusters.iter().find(|c| c.cluster == XtreamCluster::Series).unwrap().requested); + tokio::fs::remove_file(&input.url).await.unwrap(); + let reused = download_input(&ctx, &input, false).await; + assert!(reused.errors.is_empty()); + assert!(reused.input_telemetry.is_none()); +} + +#[tokio::test] +async fn m3u_update_quality_force_bypasses_guard_and_cache_without_inventing_quality() { + for threshold in [0, 95] { + for policy in [InputRefreshPolicy::REFRESH, InputRefreshPolicy::FORCE] { + let temp = tempfile::tempdir().unwrap(); + let (mut ctx, mut input) = fixture(temp.path(), Some([10, 10, 10]), [threshold; 3]).await; + Arc::make_mut(&mut input).cache_duration_seconds = 3_600; + let storage = input_storage_path(&ctx, &input).await; + let mut status = input_cache::InputStatus::default(); + input_cache::update_cluster_status(&mut status, "default", input_cache::ClusterState::Ok); + input_cache::save_input_status(&storage, &status); + set_refresh_policy(&mut ctx, &input, policy); + let mut result = run(&ctx, &input, [1, 1, 1]).await; + let rejected = threshold > 0 && policy == InputRefreshPolicy::REFRESH; + assert_eq!( + result.update_state(), + if rejected { PlaylistUpdateState::Partial } else { PlaylistUpdateState::Success } + ); + assert_eq!(counts(&result.source.take_groups()), if rejected { [10; 3] } else { [1; 3] }); + let telemetry = result.input_telemetry.as_ref().unwrap(); + assert_eq!(telemetry.refresh_policy, policy); + assert!(telemetry + .clusters + .iter() + .all(|cluster| cluster.source == Some(PlaylistUpdateDataSource::Provider))); + let saved = input_cache::load_input_status(&storage); + for cluster in &telemetry.clusters { + if policy == InputRefreshPolicy::FORCE { + assert_eq!(cluster.decision, Some(PlaylistUpdateClusterDecision::Accepted)); + assert_eq!((cluster.baseline_count, cluster.quality), (None, None)); + assert_eq!((cluster.candidate_count, cluster.active_count), (Some(1), Some(1))); + let snapshot = saved.clusters[cluster.cluster.as_ref()].last_update.unwrap(); + assert_eq!(snapshot.quality, None); + assert_eq!(snapshot.quality_guard_threshold, Some(threshold)); + assert_eq!(snapshot.policy, Some(policy)); + } + } + let reused = download_input(&ctx, &input, false).await; + assert!(reused.input_telemetry.is_none(), "in-run reuse must not create another Force acquisition"); + } + } +} + +#[tokio::test] +async fn m3u_update_quality_force_keeps_parse_and_empty_publication_errors() { + for body in ["#EXTM3U\n", "#EXTM3U\n#EXTINF:-1,Truncated\n"] { + let temp = tempfile::tempdir().unwrap(); + let (mut ctx, input) = fixture(temp.path(), Some([10; 3]), [95; 3]).await; + set_refresh_policy(&mut ctx, &input, InputRefreshPolicy::FORCE); + tokio::fs::write(&input.url, body).await.unwrap(); + let mut result = download_input(&ctx, &input, false).await; + assert_eq!(result.update_state(), PlaylistUpdateState::Failure); + assert!(result.quality_rejections.is_empty()); + let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); + assert_eq!(counts(&saved.take_groups()), [10; 3]); + } +} + +#[tokio::test] +async fn m3u_update_quality_target_processing_uses_retained_and_accepted_groups() { + for policy in [InputRefreshPolicy::NORMAL, InputRefreshPolicy::FORCE] { + let temp = tempfile::tempdir().unwrap(); + let (mut ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; + set_refresh_policy(&mut ctx, &input, policy); + tokio::fs::write(&input.url, document("candidate", [10, 0, 10])).await.unwrap(); + let output = temp.path().join("client.m3u"); + let mut target = ConfigTarget::from(&ConfigTargetDto { + id: 1, + name: "m3u-target".into(), + enabled: true, + ..ConfigTargetDto::default() + }); + target.filter = get_filter(r#"Name ~ ".*""#, None).unwrap().into(); + target.output = vec![TargetOutput::M3u(tuliprox_core::model::M3uTargetOutput { + filename: Some(output.to_string_lossy().into_owned()), + include_type_in_url: false, + mask_redirect_url: false, + filter: None, + })]; + ctx.config.sources.store(Arc::new(SourcesConfig { + inputs: vec![input.clone()], + sources: vec![ConfigSource { inputs: vec![input.name.clone()], targets: vec![Arc::new(target)] }], + ..SourcesConfig::default() + })); + let (_, targets, errors) = process_source(0, Arc::new(ctx)).await; + assert!(errors.is_empty(), "{errors:?}"); + assert_eq!(targets.len(), 1); + let published = tokio::fs::read_to_string(output).await.unwrap(); + assert!(published.contains("/candidate/live/") && published.contains("/candidate/series/"), "{published}"); + assert_eq!(published.contains("/previous/video/"), policy == InputRefreshPolicy::NORMAL, "{published}"); + assert!(!published.contains("/previous/live/") && !published.contains("/candidate/video/")); + } +} + +#[tokio::test] +async fn m3u_update_quality_raw_group_catalog_uses_effective_state() { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), Some([10, 10, 10]), [90, 90, 90]).await; + let mut result = run(&ctx, &input, [10, 10, 0]).await; + let storage = input_cache::resolve_input_storage_path(&ctx.config.config.load().storage_dir, &input.name).await; + let status = input_cache::load_input_status(&storage); + for cluster in XTREAM_CLUSTER { + let saved = &status.clusters[cluster.as_ref()]; + let rejected = cluster == XtreamCluster::Series; + assert_eq!( + saved.status, + if rejected { input_cache::ClusterState::Failed } else { input_cache::ClusterState::Ok } + ); + let snapshot = saved.last_update.unwrap(); + assert_eq!( + snapshot.quality.unwrap().decision, + if rejected { + PersistedPlaylistUpdateQualityDecision::Rejected + } else { + PersistedPlaylistUpdateQualityDecision::Accepted + } + ); + assert_eq!(snapshot.active_count, Some(10)); + } + assert_eq!(status.clusters["default"].status, input_cache::ClusterState::Failed); + let series: RawInputGroupCatalog = serde_json::from_slice( + &tokio::fs::read(raw_group_catalog_path(&storage, XtreamCluster::Series)).await.unwrap(), + ) + .unwrap(); + assert_eq!(series.groups, ["previous-series"]); + verify_effective(&ctx, &input, &mut result, [10, 10, 10]).await; +} + +#[tokio::test] +async fn m3u_update_quality_duplicate_keys_match_the_existing_persisted_population() { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), Some([1, 1, 0]), [100, 100, 0]).await; + let candidate = document("superseded", [1, 1, 0]) + &document("last", [1, 1, 0]); + tokio::fs::write(&input.url, candidate).await.unwrap(); + let mut result = download_input(&ctx, &input, false).await; + assert_eq!(result.update_state(), PlaylistUpdateState::Success); + assert!(result.source.take_groups().iter().all(|group| group.title.starts_with("last"))); + let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); + assert_eq!(counts(&saved.take_groups()), [1, 1, 0]); +} + +#[tokio::test] +async fn m3u_update_quality_duplicate_keys_match_the_existing_persisted_population_when_disabled() { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), None, [0, 0, 0]).await; + let candidate = document("superseded", [1, 1, 0]) + &document("last", [1, 1, 0]); + tokio::fs::write(&input.url, candidate).await.unwrap(); + let mut result = download_input(&ctx, &input, false).await; + assert_eq!(result.update_state(), PlaylistUpdateState::Success); + assert!(result.source.take_groups().iter().all(|group| group.title.starts_with("last"))); + let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); + assert_eq!(counts(&saved.take_groups()), [1, 1, 0]); +} + +#[tokio::test] +async fn m3u_update_quality_conflicting_retained_key_cannot_diverge_from_storage() { + let temp = tempfile::tempdir().unwrap(); + let (ctx, input) = fixture(temp.path(), Some([1, 1, 0]), [90, 0, 0]).await; + let candidate = document("candidate", [0, 1, 0]).replace("2000", "1000"); + tokio::fs::write(&input.url, candidate).await.unwrap(); + let mut result = download_input(&ctx, &input, false).await; + assert_eq!(result.update_state(), PlaylistUpdateState::Failure); + assert!(!result.errors.is_empty()); + let mut saved = load_input_playlist(&ctx.config, &input, None).await.unwrap(); + assert_eq!(counts(&saved.take_groups()), [1, 1, 0]); + let storage = input_storage_path(&ctx, &input).await; + let catalog: RawInputGroupCatalog = + serde_json::from_slice(&tokio::fs::read(raw_group_catalog_path(&storage, XtreamCluster::Live)).await.unwrap()) + .unwrap(); + assert_eq!(catalog.groups, ["previous-live"]); +} diff --git a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/manual_update_integration.rs b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/manual_update_integration.rs new file mode 100644 index 000000000..ae7a0bb90 --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/manual_update_integration.rs @@ -0,0 +1,700 @@ +use super::*; +use shared::model::{ConfigRenameDto, ItemField, LibraryConfigDto, LibraryContentType, LibraryScanResult}; +use tuliprox_core::model::{LibraryConfig, LibraryScanDirectory, M3uTargetOutput, MetadataUpdateConfig}; + +async fn library_context(root: &Path, order: ProcessingOrder) -> PlaylistProcessingContext { + let ctx = processing_context_with_events(root, CollectSink::default()); + let media = root.join("media"); + tokio::fs::create_dir_all(&media).await.unwrap(); + tokio::fs::write(media.join("Library Movie (2025).mp4"), b"fixture").await.unwrap(); + let companion_path = root.join("companion.m3u"); + tokio::fs::write(&companion_path, "#EXTM3U\n#EXTINF:-1,Companion\nhttp://stream.example/companion\n") + .await + .unwrap(); + let mut config = (**ctx.config.config.load()).clone(); + let mut library = LibraryConfig::from(&LibraryConfigDto { enabled: true, ..LibraryConfigDto::default() }); + library.metadata.fallback_to_filename = true; + library.thumbnails.enabled = false; + library.scan_directories = vec![LibraryScanDirectory { + enabled: true, + path: media.to_string_lossy().into_owned(), + content_type: LibraryContentType::Movie, + recursive: true, + }]; + config.library = Some(library); + let mut metadata = MetadataUpdateConfig::default(); + metadata.ffprobe.enabled = false; + metadata.tmdb.enabled = false; + config.metadata_update = Some(metadata); + ctx.config.config.store(Arc::new(config)); + let library_input = Arc::new(ConfigInput { + id: 2, + name: "local".intern(), + input_type: InputType::Library, + enabled: true, + cache_duration_seconds: 3600, + ..ConfigInput::default() + }); + let companion = Arc::new(ConfigInput { + id: 3, + name: "companion".intern(), + input_type: InputType::M3u, + enabled: true, + url: companion_path.to_string_lossy().into_owned(), + ..ConfigInput::default() + }); + let mut target = ConfigTarget::from(&ConfigTargetDto { + id: 20, + enabled: true, + name: "selected".into(), + processing_order: order, + rename: Some(vec![ConfigRenameDto { + field: ItemField::Name, + pattern: "^MAPPED$".into(), + new_name: "RENAMED".into(), + t_pattern: None, + }]), + ..ConfigTargetDto::default() + }); + target.output = vec![TargetOutput::M3u(M3uTargetOutput { + filename: Some(root.join("selected.m3u").to_string_lossy().into_owned()), + include_type_in_url: false, + mask_redirect_url: false, + filter: None, + })]; + target.filter = get_filter(r#"Name ~ ".*""#, None).unwrap().into(); + let mapped = super::super::mapping_stage::build_target( + vec![super::super::mapping_stage::build_mapping("manual", MappingStage::Processing, r#"@Name = "MAPPED""#)], + false, + ); + target.mapping = mapped.mapping; + let mut unselected = target.clone(); + unselected.id = 21; + unselected.name = "unselected".into(); + unselected.output = vec![TargetOutput::M3u(M3uTargetOutput { + filename: Some(root.join("unselected.m3u").to_string_lossy().into_owned()), + include_type_in_url: false, + mask_redirect_url: false, + filter: None, + })]; + ctx.config.sources.store(Arc::new(SourcesConfig { + inputs: vec![library_input.clone(), companion.clone()], + sources: vec![ConfigSource { + inputs: vec![library_input.name.clone(), companion.name.clone()], + targets: vec![Arc::new(target), Arc::new(unselected)], + }], + ..SourcesConfig::default() + })); + ctx +} + +fn rescan_run(ctx: &PlaylistProcessingContext, guard: UpdateGuard) -> ProcessingRun { + ProcessingRun::for_run( + "manual-library-run".into(), + ctx.client.clone(), + ctx.config.clone(), + Arc::new(ProcessTargets { + enabled: true, + inputs: vec![2, 3], + targets: vec![20], + target_names: vec!["selected".into()], + }), + ctx.events.clone(), + ) + .with_update_guard(guard) + .with_manual_input_update(Some(InputUpdateRequest { input_id: 2, action: InputUpdateAction::Rescan })) +} + +fn scan_results(ctx: &PlaylistProcessingContext) -> Vec { + ctx.events + .0 + .lock() + .unwrap() + .iter() + .filter_map(|event| match event { + EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(2) => { + if progress.library_scan_result.is_some() { + assert_eq!(progress.run_id.as_ref().map(PlaylistUpdateRunId::as_ref), Some("manual-library-run")); + assert!(progress.execution_order.is_some()); + } + progress.library_scan_result.clone() + } + _ => None, + }) + .collect() +} + +fn assert_run_state(ctx: &PlaylistProcessingContext, state: PlaylistUpdateState) { + let events = ctx.events.0.lock().unwrap(); + assert!( + events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) if summary.state == state)), + "{events:?}" + ); + assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(2) && progress.state == Some(state))), "{events:?}"); +} + +#[tokio::test] +async fn manual_update_library_empty_rescan_persists_zero_to_zero_and_nonzero_to_zero_with_companion() { + for disk in [false, true] { + for previous_population in [false, true] { + for companion in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; + let mut config = (**ctx.config.config.load()).clone(); + config.disk_based_processing = disk; + ctx.config.config.store(Arc::new(config)); + if !companion { + let mut sources = (**ctx.config.sources.load()).clone(); + sources.sources[0].inputs.retain(|name| name.as_ref() == "local"); + ctx.config.sources.store(Arc::new(sources)); + } + if previous_population { + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + assert_run_state(&ctx, PlaylistUpdateState::Success); + ctx.events.0.lock().unwrap().clear(); + } + tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap(); + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + assert_run_state(&ctx, PlaylistUpdateState::Success); + for name in std::iter::once("local").chain(companion.then_some("companion")) { + let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), name).await; + let status = input_cache::load_input_status(&path); + assert_eq!(status.last_input_update.unwrap().state, PlaylistUpdateState::Success, "{name}"); + } + assert_eq!( + scan_results(&ctx), + vec![LibraryScanResult { + files_scanned: 0, + groups_scanned: 0, + files_added: 0, + files_updated: 0, + files_removed: usize::from(previous_population), + errors: 0, + }] + ); + let output = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap(); + assert_eq!(output.matches("#EXTINF").count(), usize::from(companion), "{output}"); + assert_eq!(output.contains("http://stream.example/companion"), companion); + assert!(!output.contains("Library%20Movie") && !output.contains("file://")); + assert!(!temp.path().join("unselected.m3u").exists()); + let input = ctx.config.sources.load().inputs[0].clone(); + let mut persisted = load_input_playlist(&ctx.config, &input, None).await.unwrap(); + assert!(persisted.is_empty(), "Empty Library must replace its old persisted input"); + assert!(!ctx.events.0.lock().unwrap().iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.message.contains("playlist is empty")))); + // Same canonical empty input remains readable on subsequent cache/reuse paths. + let first = download_input(&ctx, &input, false).await; + assert!(first.errors.is_empty() && first.storage_error.is_none()); + let mut reuse = download_input(&ctx, &input, false).await; + assert_eq!(reuse.job_state(), InputJobState::Ready); + assert_eq!(reuse.input_telemetry, None, "In-run reuse must not emit another acquisition"); + assert_eq!(scan_results(&ctx).len(), 1, "Reuse must not manufacture a second rescan"); + } + } + } +} + +#[tokio::test] +async fn coderabbit_library_unavailable_is_not_an_authoritative_empty_catalog() { + for disabled in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + assert_run_state(&ctx, PlaylistUpdateState::Success); + let mut input = (*ctx.config.sources.load().inputs[0]).clone(); + input.cache_duration_seconds = 0; + let input = Arc::new(input); + let storage = input_storage_path(&ctx, &input).await; + let file = tuliprox_repository::get_input_local_library_playlist_file_path(&storage, &input.name); + let before = tokio::fs::read(&file).await.unwrap(); + let output = tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap(); + let mut config = (**ctx.config.config.load()).clone(); + if disabled { + config.library.as_mut().unwrap().enabled = false; + } else { + config.library = None; + } + ctx.config.config.store(Arc::new(config)); + + let mut result = download_input(&ctx, &input, false).await; + assert_eq!(result.job_state(), InputJobState::Failed); + assert_eq!(result.update_state(), PlaylistUpdateState::Failure); + assert!(!result.authoritative_library); + assert!(!result.errors.is_empty()); + assert_eq!(tokio::fs::read(&file).await.unwrap(), before); + assert_eq!(tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap(), output); + } +} + +#[tokio::test] +async fn coderabbit_library_missing_input_triggers_catalog_recovery_not_empty_success() { + for disk in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; + let mut config = (**ctx.config.config.load()).clone(); + config.disk_based_processing = disk; + ctx.config.config.store(Arc::new(config)); + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + assert_run_state(&ctx, PlaylistUpdateState::Success); + let input = ctx.config.sources.load().inputs[0].clone(); + let storage = input_storage_path(&ctx, &input).await; + let file = tuliprox_repository::get_input_local_library_playlist_file_path(&storage, &input.name); + tokio::fs::remove_file(&file).await.unwrap(); + + assert!(load_input_playlist(&ctx.config, &input, None).await.is_err()); + let mut recovered = download_input(&ctx, &input, false).await; + assert_eq!(recovered.job_state(), InputJobState::Ready); + assert!(recovered.errors.is_empty() && recovered.storage_error.is_none()); + assert!(!recovered.source.is_empty(), "missing cache must be repaired from the populated catalog"); + assert!(file.exists()); + let mut persisted = load_input_playlist(&ctx.config, &input, None).await.unwrap(); + assert!(!persisted.is_empty()); + + let media_server = ConfigInput { input_type: InputType::Plex, ..(*input).clone() }; + assert!( + load_input_playlist(&ctx.config, &media_server, None).await.is_err(), + "the shared loader must not turn a missing Plex cache into a successful empty input" + ); + } +} + +#[tokio::test] +async fn manual_update_library_scan_errors_keep_real_result_and_block_rebuild() { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; + let mut config = (**ctx.config.config.load()).clone(); + config.library.as_mut().unwrap().metadata.fallback_to_filename = false; + ctx.config.config.store(Arc::new(config)); + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + assert_run_state(&ctx, PlaylistUpdateState::Failure); + let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "local").await; + assert_eq!(input_cache::load_input_status(&path).last_input_update.unwrap().state, PlaylistUpdateState::Failure); + assert_eq!( + scan_results(&ctx), + vec![LibraryScanResult { + files_scanned: 1, + groups_scanned: 1, + files_added: 0, + files_updated: 0, + files_removed: 0, + errors: 1, + }] + ); + assert!(!temp.path().join("selected.m3u").exists()); +} + +#[tokio::test] +async fn manual_update_library_persisted_input_status_keeps_failed_companion_separate() { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "companion").await; + assert_eq!(input_cache::load_input_status(&path).last_input_update.unwrap().state, PlaylistUpdateState::Success); + let before = std::fs::read(temp.path().join("selected.m3u")).unwrap(); + tokio::fs::write(temp.path().join("companion.m3u"), "#EXTM3U\n").await.unwrap(); + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + assert_eq!(input_cache::load_input_status(&path).last_input_update.unwrap().state, PlaylistUpdateState::Failure); + let library_path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "local").await; + assert_eq!( + input_cache::load_input_status(&library_path).last_input_update.unwrap().state, + PlaylistUpdateState::Success + ); + assert_eq!(std::fs::read(temp.path().join("selected.m3u")).unwrap(), before); +} + +#[tokio::test] +async fn manual_update_library_empty_xtream_and_strm_use_existing_publish_and_keep_companions() { + use tuliprox_repository::{xtream_get_file_path, xtream_get_storage_path, BPlusTreeQuery}; + for companion in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; + tokio::fs::write( + temp.path().join("companion.m3u"), + "#EXTM3U\n#EXTINF:-1 type=\"vod\",Companion\nhttp://stream.example/companion.mp4\n", + ) + .await + .unwrap(); + let mut sources = (**ctx.config.sources.load()).clone(); + if !companion { + sources.sources[0].inputs.retain(|name| name.as_ref() == "local"); + } + let target = Arc::make_mut(&mut sources.sources[0].targets[0]); + target.output.extend([ + TargetOutput::Xtream(XtreamTargetOutput { flags: XtreamTargetFlagsSet::new(), trakt: None, filter: None }), + TargetOutput::Strm(tuliprox_core::model::StrmTargetOutput::from(&shared::model::StrmTargetOutputDto { + directory: temp.path().join("strm").to_string_lossy().into_owned(), + flat: true, + ..shared::model::StrmTargetOutputDto::default() + })), + ]); + ctx.config.sources.store(Arc::new(sources)); + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + assert_run_state(&ctx, PlaylistUpdateState::Success); + ctx.events.0.lock().unwrap().clear(); + tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap(); + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + assert_run_state(&ctx, PlaylistUpdateState::Success); + let config = ctx.config.config.load(); + let xtream = xtream_get_storage_path(&config, "selected").unwrap(); + for (cluster, expected) in + [(XtreamCluster::Live, 0), (XtreamCluster::Video, usize::from(companion)), (XtreamCluster::Series, 0)] + { + let path = xtream_get_file_path(&xtream, cluster); + if !path.exists() { + assert_eq!(expected, 0); + continue; + } + let mut query = BPlusTreeQuery::::try_new(&path).unwrap(); + assert_eq!(query.iter().count(), expected, "{cluster:?}"); + } + let entries = strm_artifacts(&temp.path().join("strm")).await; + assert_eq!(entries.len(), usize::from(companion), "{entries:?}"); + if companion { + assert!(entries[0].1.contains("http://stream.example/companion")); + } + } +} + +#[tokio::test] +async fn manual_update_library_empty_permission_does_not_loosen_other_provider_inputs() { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; + for input_type in [InputType::M3u, InputType::Xtream, InputType::Stalker, InputType::Plex] { + let input = + ConfigInput { input_type, name: format!("empty-{input_type:?}").intern(), ..ConfigInput::default() }; + let (_, error) = tuliprox_repository::persist_input_playlist(&ctx.config, &input, Vec::new()).await; + assert!(error.is_some(), "{input_type:?} must retain its generic empty-input guard"); + } + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + let before = tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap(); + ctx.events.0.lock().unwrap().clear(); + tokio::fs::write(temp.path().join("companion.m3u"), "#EXTM3U\n").await.unwrap(); + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + { + let events = ctx.events.0.lock().unwrap(); + assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) if summary.state == PlaylistUpdateState::Failure))); + assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(3) && progress.state == Some(PlaylistUpdateState::Failure)))); + } + assert_eq!(before, tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap()); + // An authoritatively empty Library must not mask a failed companion either. + ctx.events.0.lock().unwrap().clear(); + tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap(); + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + { + let events = ctx.events.0.lock().unwrap(); + assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) if summary.state == PlaylistUpdateState::Failure))); + assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) if progress.input_id == Some(3) && progress.state == Some(PlaylistUpdateState::Failure)))); + } + assert_eq!(before, tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap()); +} + +#[derive(Clone, Copy, Debug)] +enum LibraryFilterBoundary { + Processing, + Persist, + M3u, + Strm, +} + +async fn strm_artifacts(root: &Path) -> Vec<(PathBuf, String)> { + let mut directories = vec![root.to_path_buf()]; + let mut artifacts = Vec::new(); + while let Some(directory) = directories.pop() { + let mut entries = tokio::fs::read_dir(directory).await.unwrap(); + while let Some(entry) = entries.next_entry().await.unwrap() { + if entry.file_type().await.unwrap().is_dir() { + directories.push(entry.path()); + } else if entry.path().extension().is_some_and(|extension| extension == "strm") { + artifacts.push((entry.path(), tokio::fs::read_to_string(entry.path()).await.unwrap())); + } + } + } + artifacts.sort(); + artifacts +} + +async fn assert_library_filtered_empty_publication(boundary: LibraryFilterBoundary) { + for disk in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; + let mut config = (**ctx.config.config.load()).clone(); + config.disk_based_processing = disk; + ctx.config.config.store(Arc::new(config)); + tokio::fs::write( + temp.path().join("companion.m3u"), + "#EXTM3U\n#EXTINF:-1 type=\"vod\",Companion\nhttp://stream.example/companion.mp4\n", + ) + .await + .unwrap(); + let mut sources = (**ctx.config.sources.load()).clone(); + let target = Arc::make_mut(&mut sources.sources[0].targets[0]); + target.rename = None; + target.mapping = Arc::default(); + let library_only = get_filter(r#"Name ~ "^Library Movie.*""#, None).unwrap(); + if matches!(boundary, LibraryFilterBoundary::Processing) { + target.filter = library_only.clone().into(); + } + if matches!(boundary, LibraryFilterBoundary::Persist) { + target.filter.persist = Some(library_only.clone()); + } + if let TargetOutput::M3u(output) = &mut target.output[0] { + output.filter = matches!(boundary, LibraryFilterBoundary::M3u).then(|| library_only.clone()); + } + let strm_root = temp.path().join("strm"); + let mut strm = tuliprox_core::model::StrmTargetOutput::from(&shared::model::StrmTargetOutputDto { + directory: strm_root.to_string_lossy().into_owned(), + flat: true, + cleanup: true, + ..shared::model::StrmTargetOutputDto::default() + }); + strm.filter = matches!(boundary, LibraryFilterBoundary::Strm).then_some(library_only); + target.output.push(TargetOutput::Strm(strm)); + ctx.config.sources.store(Arc::new(sources)); + + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + assert_run_state(&ctx, PlaylistUpdateState::Success); + let m3u_path = temp.path().join("selected.m3u"); + let before_m3u = tokio::fs::read_to_string(&m3u_path).await.unwrap(); + let before_strm = strm_artifacts(&strm_root).await; + let m3u_keeps_companion = matches!(boundary, LibraryFilterBoundary::Strm); + let strm_keeps_companion = matches!(boundary, LibraryFilterBoundary::M3u); + assert_eq!(before_m3u.matches("#EXTINF").count(), 1 + usize::from(m3u_keeps_companion)); + assert_eq!(before_strm.len(), 1 + usize::from(strm_keeps_companion)); + assert_eq!(before_m3u.contains("http://stream.example/companion"), m3u_keeps_companion); + assert_eq!( + before_strm.iter().any(|(_, body)| body.contains("http://stream.example/companion")), + strm_keeps_companion + ); + + ctx.events.0.lock().unwrap().clear(); + tokio::fs::remove_file(temp.path().join("media/Library Movie (2025).mp4")).await.unwrap(); + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + let after_m3u = tokio::fs::read_to_string(&m3u_path).await.unwrap(); + let after_strm = strm_artifacts(&strm_root).await; + println!("Library filter {boundary:?}, disk={disk}: M3U before={before_m3u:?}, after={after_m3u:?}; STRM before={before_strm:?}, after={after_strm:?}"); + assert_run_state(&ctx, PlaylistUpdateState::Success); + assert_eq!( + scan_results(&ctx), + vec![LibraryScanResult { + files_scanned: 0, + groups_scanned: 0, + files_added: 0, + files_updated: 0, + files_removed: 1, + errors: 0 + }] + ); + assert_ne!(before_m3u, after_m3u, "old Library entries must really be replaced"); + assert_eq!(after_m3u.matches("#EXTINF").count(), usize::from(m3u_keeps_companion)); + assert_eq!(after_m3u.contains("http://stream.example/companion"), m3u_keeps_companion); + assert_eq!(after_strm.len(), usize::from(strm_keeps_companion)); + for (path, contents) in &before_strm { + if contents.contains("http://stream.example/companion") { + assert!( + after_strm.contains(&(path.clone(), contents.clone())), + "valid companion must be retained byte-for-byte" + ); + } else { + assert!(!path.exists(), "cleanup must remove old Library artifact {path:?}"); + } + } + // The existing M3U database is refreshed too, not only the client text file. + let target_path = tuliprox_repository::get_target_storage_path(&ctx.config.config.load(), "selected").unwrap(); + let mut db = tuliprox_repository::BPlusTreeQuery::::try_new( + &tuliprox_repository::m3u_get_file_path_for_db(&target_path), + ) + .unwrap(); + assert_eq!(db.iter().count(), usize::from(m3u_keeps_companion)); + assert!(!temp.path().join("unselected.m3u").exists()); + } +} + +#[tokio::test] +async fn manual_update_library_processing_filter_replaces_old_output_after_last_movie_removed() { + assert_library_filtered_empty_publication(LibraryFilterBoundary::Processing).await; +} + +#[tokio::test] +async fn manual_update_library_persist_filter_replaces_old_output_after_last_movie_removed() { + assert_library_filtered_empty_publication(LibraryFilterBoundary::Persist).await; +} + +#[tokio::test] +async fn manual_update_library_m3u_output_filter_replaces_old_output_and_preserves_unfiltered_strm() { + assert_library_filtered_empty_publication(LibraryFilterBoundary::M3u).await; +} + +#[tokio::test] +async fn manual_update_library_strm_output_filter_cleans_old_files_and_preserves_unfiltered_m3u() { + assert_library_filtered_empty_publication(LibraryFilterBoundary::Strm).await; +} + +#[tokio::test] +async fn manual_update_library_empty_does_not_authorize_foreign_forced_empty_target() { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; + let sources = ctx.config.sources.load_full(); + let mut playlists = sources + .inputs + .iter() + .map(|input| FetchedPlaylist { input, source: MemoryPlaylistSource::default().into_source(), epg: None }) + .collect::>(); + let prepared = prepare_playlist_for_target( + &ctx, + &mut playlists, + &sources.sources[0].targets[0], + &mut HashMap::new(), + &mut Vec::new(), + ClusterFlags::Vod, + false, + ) + .await + .unwrap(); + assert_eq!(prepared.library_empty, tuliprox_repository::LibraryEmptyPublication::Contribution); + let (result, _) = finalize_prepared_target(Arc::new(ctx), prepared).await; + assert!(result.is_err(), "Library must not relax the existing non-Xtream force-empty output guard"); + assert!(!temp.path().join("selected.m3u").exists()); +} + +#[tokio::test] +async fn manual_update_library_incomplete_discovery_has_no_invented_scan_result_or_rebuild() { + for failure in ["missing", "not-directory", "partial", "no-directories", "corrupt-catalog"] { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + let before = tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap(); + let status_path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), "local").await; + let cache_before = input_cache::load_input_status(&status_path).clusters; + ctx.events.0.lock().unwrap().clear(); + let mut config = (**ctx.config.config.load()).clone(); + let library = config.library.as_mut().unwrap(); + match failure { + "missing" => { + library.scan_directories[0].path = temp.path().join("missing").to_string_lossy().into_owned(); + } + "not-directory" => { + library.scan_directories[0].path = + temp.path().join("media/Library Movie (2025).mp4").to_string_lossy().into_owned(); + } + "partial" => { + let mut missing = library.scan_directories[0].clone(); + missing.path = temp.path().join("missing").to_string_lossy().into_owned(); + library.scan_directories.push(missing); + } + "no-directories" => library.scan_directories.clear(), + "corrupt-catalog" => { + let path = tuliprox_library::library::resolve_metadata_storage_path( + config.metadata_update.as_ref(), + &config.storage_dir, + ) + .join("library/broken.json"); + tokio::fs::write(path, "invalid json").await.unwrap(); + } + _ => unreachable!(), + } + ctx.config.config.store(Arc::new(config)); + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + assert_run_state(&ctx, PlaylistUpdateState::Failure); + assert!(scan_results(&ctx).is_empty(), "{failure}"); + let status = input_cache::load_input_status(&status_path); + assert_eq!(status.last_input_update.unwrap().state, PlaylistUpdateState::Failure, "{failure}"); + assert_eq!(status.clusters, cache_before, "failed rescan must not mutate cache validity"); + assert_eq!(before, tokio::fs::read(temp.path().join("selected.m3u")).await.unwrap()); + } +} + +#[tokio::test] +async fn manual_update_library_selected_target_keeps_companion_and_real_processing_order() { + for (order, expected) in [(ProcessingOrder::Frm, "MAPPED"), (ProcessingOrder::Fmr, "RENAMED")] { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), order).await; + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + let output = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap(); + assert_eq!(output.matches("#EXTINF").count(), 2, "{output}"); + assert!(output.contains("http://stream.example/companion"), "{output}"); + assert!(output.contains(expected), "{output}"); + assert!(!temp.path().join("unselected.m3u").exists()); + let events = ctx.events.0.lock().unwrap(); + let scan = events + .iter() + .position(|event| { + matches!(event, EventMessage::PlaylistUpdateProgress(progress) + if progress.input_id == Some(2) && progress.message.starts_with("Library rescan completed")) + }) + .unwrap(); + let acquired = events + .iter() + .position(|event| { + matches!(event, EventMessage::PlaylistUpdateProgress(progress) + if progress.input_id == Some(2) && progress.state == Some(PlaylistUpdateState::Success)) + }) + .unwrap(); + let published = events + .iter() + .position(|event| { + matches!(event, EventMessage::PlaylistUpdate(summary) + if summary.state == PlaylistUpdateState::Success) + }) + .unwrap(); + assert!(scan < acquired && acquired < published); + assert!(events.iter().all(|event| !matches!(event, EventMessage::PlaylistUpdateProgress(progress) + if progress.input_id == Some(2) && progress.input_telemetry.as_ref().is_some_and(|telemetry| + !telemetry.clusters.is_empty() || telemetry.refresh_policy != InputRefreshPolicy::NORMAL)))); + } +} + +#[tokio::test] +async fn manual_update_library_unreadable_catalog_and_busy_scan_do_not_publish() { + for busy in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; + let guard = UpdateGuard::new(); + let _busy = busy.then(|| guard.try_library().unwrap()); + if !busy { + let mut config = (**ctx.config.config.load()).clone(); + config.library.as_mut().unwrap().scan_directories[0].path = + temp.path().join("missing").to_string_lossy().into_owned(); + ctx.config.config.store(Arc::new(config)); + } + exec_processing(rescan_run(&ctx, guard.clone())).await; + assert!(!temp.path().join("selected.m3u").exists()); + assert!(!temp.path().join("unselected.m3u").exists()); + assert!(ctx.events.0.lock().unwrap().iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) + if summary.state == PlaylistUpdateState::Failure))); + assert!(guard.try_playlist().is_some()); + } +} + +#[tokio::test] +async fn manual_update_library_input_success_remains_separate_from_target_publish_failure() { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; + tokio::fs::create_dir(temp.path().join("selected.m3u")).await.unwrap(); + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + let events = ctx.events.0.lock().unwrap(); + assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdateProgress(progress) + if progress.input_id == Some(2) && progress.state == Some(PlaylistUpdateState::Success)))); + assert!(events.iter().any(|event| matches!(event, EventMessage::PlaylistUpdate(summary) + if summary.state == PlaylistUpdateState::Failure))); +} + +#[tokio::test] +async fn manual_update_library_rescan_reloads_valid_cache_but_bulk_keeps_catalog() { + let temp = tempfile::tempdir().unwrap(); + let ctx = library_context(temp.path(), ProcessingOrder::Frm).await; + exec_processing(rescan_run(&ctx, UpdateGuard::new())).await; + tokio::fs::write(temp.path().join("media/Second Movie (2025).mp4"), b"fixture").await.unwrap(); + // Ordinary bulk does not discover the new file and does not acquire a library permit. + let guard = UpdateGuard::new(); + let standalone_scan = guard.try_library().unwrap(); + exec_processing( + ProcessingRun::new(ctx.client.clone(), ctx.config.clone(), ctx.user_targets.clone(), ctx.events.clone()) + .with_update_guard(guard.clone()), + ) + .await; + let bulk = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap(); + assert_eq!(bulk.matches("#EXTINF").count(), 2); + drop(standalone_scan); + exec_processing(rescan_run(&ctx, guard)).await; + let rescanned = tokio::fs::read_to_string(temp.path().join("selected.m3u")).await.unwrap(); + assert_eq!(rescanned.matches("#EXTINF").count(), 3, "{rescanned}"); +} diff --git a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/persisted_input_status.rs b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/persisted_input_status.rs new file mode 100644 index 000000000..0d2bd45fe --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/persisted_input_status.rs @@ -0,0 +1,393 @@ +use super::*; +use crate::processor::playlist::input_status::{persist_input_completion, persist_input_job_result}; +use shared::model::PersistedPlaylistUpdateInputResult; + +fn acquired_result(state: PlaylistUpdateState) -> InputJobResult { + let mut result = super::super::playlist_update_run_result( + 7, + InputJobState::Ready, + vec![], + state == PlaylistUpdateState::Partial, + ); + result.input_telemetry = Some(PlaylistUpdateInputTelemetry { + refresh_policy: InputRefreshPolicy::REFRESH, + source: Some(PlaylistUpdateDataSource::Provider), + clusters: Vec::new(), + }); + result +} + +#[tokio::test] +async fn playlist_update_status_run_precedence_is_isolated_by_stable_input_id() { + let temp = tempfile::tempdir().unwrap(); + let ctx = processing_context(temp.path()); + let failed = super::super::playlist_update_run_result(7, InputJobState::Failed, vec![], false); + persist_input_job_result(&ctx, &failed).await; + let mut independent = acquired_result(PlaylistUpdateState::Success); + independent.input_id = 8; + independent.input_name = "independent".intern(); + persist_input_job_result(&ctx, &independent).await; + for (result, state) in [(failed, PlaylistUpdateState::Failure), (independent, PlaylistUpdateState::Success)] { + let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &result.input_name).await; + assert_eq!(input_cache::load_input_status(&path).last_input_update.unwrap().state, state); + } +} + +#[tokio::test] +async fn m3u_update_quality_playlist_update_status_staged_completion_is_independent_of_parent_failure() { + for (parent_fails, staged_fails) in [(false, false), (true, false), (false, true)] { + let temp = tempfile::tempdir().unwrap(); + let events = CollectSink::default(); + let ctx = processing_context_with_events(temp.path(), events.clone()); + let parent_path = temp.path().join("parent.m3u"); + let staged_path = temp.path().join("overlay.m3u"); + for (path, fails) in [(&parent_path, parent_fails), (&staged_path, staged_fails)] { + if !fails { + std::fs::write(path, "#EXTM3U\n#EXTINF:-1,Fixture\nhttp://stream.example/fixture\n").unwrap(); + } + } + let parent = Arc::new(ConfigInput { + id: 7, + name: "parent".intern(), + input_type: InputType::M3u, + url: parent_path.to_string_lossy().into_owned(), + enabled: true, + ..ConfigInput::default() + }); + let mut staged = ConfigInput { + id: 8, + name: "staged".intern(), + input_type: InputType::Staged, + url: staged_path.to_string_lossy().into_owned(), + enabled: true, + staged: Some(tuliprox_core::model::ConfigInputStaged { + for_input: Some(parent.name.clone()), + clusters: ClusterFlags::Live, + }), + ..ConfigInput::default() + }; + staged.resolve_staged_download_type(); + let staged = Arc::new(staged); + ctx.config.sources.store(Arc::new(SourcesConfig { + inputs: vec![parent.clone(), staged.clone()], + ..SourcesConfig::default() + })); + let result = download_input(&ctx, &parent, false).await; + assert_eq!(!result.errors.is_empty(), parent_fails || staged_fails); + let path = input_storage_path(&ctx, &staged).await; + let persisted = input_cache::load_input_status(&path); + assert_eq!( + persisted.last_input_update.unwrap().state, + if staged_fails { PlaylistUpdateState::Failure } else { PlaylistUpdateState::Success } + ); + assert_eq!(persisted.clusters.len(), 4); + assert!(persisted.clusters["default"].last_update.is_none(), "document cache and cluster facts stay separate"); + for cluster in XTREAM_CLUSTER { + assert!(persisted.clusters[cluster.as_ref()].last_update.is_some()); + } + let completions = staged_completion::input_completions(&events, staged.id); + assert_eq!(completions.len(), 1, "real staged download must report its own completion"); + assert_eq!(completions[0].state, persisted.last_input_update.map(|result| result.state)); + assert_eq!(completions[0].run_id, Some(ctx.run_id.clone())); + assert_eq!(completions[0].execution_order, Some(ctx.execution_order)); + let telemetry = completions[0].input_telemetry.as_ref().unwrap(); + assert_eq!(telemetry.source, None); + assert!(telemetry.clusters.iter().all(|cluster| cluster.source == Some(PlaylistUpdateDataSource::Provider))); + assert!(staged_completion::input_completions(&events, parent.id).is_empty()); + } +} + +#[tokio::test] +async fn playlist_update_status_delayed_source_results_keep_strongest_completion_until_next_run() { + for weaker in [PlaylistUpdateState::Success, PlaylistUpdateState::Partial] { + for failure_first in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let events = CollectSink::default(); + let ctx = processing_context_with_events(temp.path(), events.clone()); + let acquired = acquired_result(weaker); + let failed_reuse = super::super::playlist_update_run_result(7, InputJobState::Failed, vec![], false); + let (first, delayed) = if failure_first { (failed_reuse, acquired) } else { (acquired, failed_reuse) }; + let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &first.input_name).await; + let mut original = input_cache::InputStatus::default(); + original.clusters.insert( + "live".into(), + input_cache::ClusterStatus { + status: input_cache::ClusterState::Failed, + timestamp: 123, + last_update: Some(PersistedPlaylistUpdateClusterSnapshot { + source: Some(PlaylistUpdateDataSource::Cache), + policy: Some(InputRefreshPolicy::FORCE), + quality_guard_threshold: Some(95), + ..PersistedPlaylistUpdateClusterSnapshot::default() + }), + }, + ); + input_cache::save_input_status(&path, &original); + let (release, other_input_finished) = tokio::sync::oneshot::channel(); + let delayed_ctx = ctx.clone(); + // Source A already holds its shared-input result, but cannot drain + // its sorted result list until another input job has completed. + let delayed_source = async move { + let source_results = vec![delayed]; + other_input_finished.await.unwrap(); + for result in source_results { + persist_input_job_result(&delayed_ctx, &result).await; + report_input_job_completion( + &delayed_ctx.events, + &delayed_ctx.run_id, + delayed_ctx.execution_order, + &result, + ); + } + }; + let first_source = async { + persist_input_job_result(&ctx, &first).await; + report_input_job_completion(&ctx.events, &ctx.run_id, ctx.execution_order, &first); + let mut persisted = input_cache::load_input_status(&path); + assert_eq!(persisted.last_input_update.unwrap().state, first.update_state()); + // Distinct sentinel proves a delayed weaker result does not + // advance the stronger completion's timestamp, without sleeps. + persisted.last_input_update.as_mut().unwrap().timestamp = 456; + input_cache::save_input_status(&path, &persisted); + release.send(()).unwrap(); + }; + tokio::join!(delayed_source, first_source); + let final_status = input_cache::load_input_status(&path); + let completion = final_status.last_input_update.unwrap(); + assert_eq!( + completion.state, + PlaylistUpdateState::Failure, + "weaker={weaker:?}, failure_first={failure_first}" + ); + if failure_first { + assert_eq!(completion.timestamp, 456); + } + assert_eq!(final_status.clusters, original.clusters); + let states: Vec<_> = events + .0 + .lock() + .unwrap() + .iter() + .filter_map(|event| match event { + EventMessage::PlaylistUpdateProgress(progress) => progress.state, + _ => None, + }) + .collect(); + assert_eq!( + states, + if failure_first { + vec![PlaylistUpdateState::Failure, weaker] + } else { + vec![weaker, PlaylistUpdateState::Failure] + } + ); + + let mut next = processing_context(temp.path()); + next.run_id = "next-successful-run".into(); + next.execution_order = 2.into(); + let success = acquired_result(PlaylistUpdateState::Success); + persist_input_job_result(&next, &success).await; + let replacement = input_cache::load_input_status(&path); + assert_eq!(replacement.last_input_update.unwrap().state, PlaylistUpdateState::Success); + assert_eq!(replacement.clusters, original.clusters); + } + } +} + +#[tokio::test] +async fn playlist_update_status_parent_download_persists_staged_own_per_cluster_success_and_failure() { + for fails in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let mut responses = fixture_responses([1, 0, 0]); + if fails { + responses.insert("get_live_streams".into(), "not JSON".into()); + } + let server = TestXtreamServer::start_with_responses(responses); + let events = CollectSink::default(); + let ctx = processing_context_with_events(temp.path(), events.clone()); + let parent_file = temp.path().join("parent.m3u"); + std::fs::write(&parent_file, "#EXTM3U\n#EXTINF:-1,Parent\nhttp://stream.example/parent\n").unwrap(); + let parent = Arc::new(ConfigInput { + id: 7, + name: "parent".intern(), + input_type: InputType::M3u, + url: parent_file.to_string_lossy().into_owned(), + enabled: true, + ..ConfigInput::default() + }); + let mut staged = + (*test_input(&server.base_url, ConfigInputUpdateQualityDto { live: 95, vod: 0, series: 0 }, 0, &[])) + .clone(); + staged.id = 8; + staged.name = "own-staged".intern(); + staged.input_type = InputType::Staged; + staged.staged_type = shared::model::StagedInputType::Xtream; + staged.staged = Some(tuliprox_core::model::ConfigInputStaged { + for_input: Some(parent.name.clone()), + clusters: ClusterFlags::Live, + }); + staged.options = Some(ConfigInputOptions::from(&ConfigInputOptionsDto { + skip_vod: true, + skip_series: true, + update_quality: ConfigInputUpdateQualityDto { live: 95, vod: 0, series: 0 }, + ..ConfigInputOptionsDto::default() + })); + staged.resolve_staged_download_type(); // Same resolver used when SourcesConfig is built. + let staged = Arc::new(staged); + ctx.config.sources.store(Arc::new(SourcesConfig { + inputs: vec![parent.clone(), staged.clone()], + ..SourcesConfig::default() + })); + let staged_path = input_storage_path(&ctx, &staged).await; + // Referencing or directly skipping a staged input is not an acquisition. + let skipped = download_input(&ctx, &staged, false).await; + assert!(skipped.input_telemetry.is_none()); + assert!(input_cache::load_input_status(&staged_path).last_input_update.is_none()); + assert!(staged_completion::input_completions(&events, staged.id).is_empty()); + + let result = download_input(&ctx, &parent, false).await; + let status = input_cache::load_input_status(&staged_path); + assert!(!status.clusters.contains_key("default")); + assert_eq!(status.clusters.len(), 1); + let completion = status.last_input_update.expect("actual indirect staged acquisition has its own completion"); + assert_eq!(completion.state, if fails { PlaylistUpdateState::Failure } else { PlaylistUpdateState::Success }); + assert!(completion.timestamp > 0); + let completions = staged_completion::input_completions(&events, staged.id); + assert_eq!(completions.len(), 1); + assert_eq!(completions[0].state, Some(completion.state)); + assert_eq!(completions[0].run_id, Some(ctx.run_id.clone())); + assert_eq!(completions[0].execution_order, Some(ctx.execution_order)); + let snapshot = status.clusters["live"].last_update.unwrap(); + assert_eq!(snapshot.source, Some(PlaylistUpdateDataSource::Provider)); + assert_eq!(snapshot.policy, None); + if fails { + assert_eq!(snapshot.quality_guard_threshold, Some(95)); + assert_eq!(snapshot.quality, None); + } else { + assert_eq!(snapshot.quality_guard_threshold, None, "the evaluation already owns its threshold"); + let quality = snapshot.quality.unwrap(); + assert_eq!(quality.threshold, 95); + assert_eq!(quality.decision, PersistedPlaylistUpdateQualityDecision::Accepted); + assert_eq!(quality.achieved_quality, None, "bootstrap has no invented Quality"); + } + assert_eq!(result.errors.is_empty(), !fails); + let parent_path = input_storage_path(&ctx, &parent).await; + assert_ne!(parent_path, staged_path); + assert!( + input_cache::load_input_status(&parent_path).last_input_update.is_none(), + "staged result does not invent a parent job completion" + ); + if !fails { + let bytes = std::fs::read(staged_path.join(input_cache::STATUS_FILE)).unwrap(); + let reuse = download_input(&ctx, &parent, false).await; + assert!(reuse.input_telemetry.is_none()); + let mut staged_reuse = download_input(&ctx, &staged, true).await; + assert!(staged_reuse.input_telemetry.is_none()); + super::super::input_status::complete_staged_input(&ctx, &staged, &mut staged_reuse).await; + assert_eq!(std::fs::read(staged_path.join(input_cache::STATUS_FILE)).unwrap(), bytes); + assert_eq!(staged_completion::input_completions(&events, staged.id), completions); + } + let requests = server.finish(); + assert_eq!( + requests.iter().filter(|action| action.as_str() == "get_live_streams").count(), + 1, + "no extra staged acquisition on parent reuse" + ); + } +} + +#[tokio::test] +async fn playlist_update_status_completion_replaces_one_result_without_touching_cache_or_cluster_snapshots() { + let temp = tempfile::tempdir().unwrap(); + let ctx = processing_context(temp.path()); + let input_name: Arc = "result-input".intern(); + let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &input_name).await; + let mut original = input_cache::InputStatus::default(); + input_cache::update_cluster_status(&mut original, "default", input_cache::ClusterState::Ok); + original.clusters.get_mut("default").unwrap().timestamp = 1; // Expired cache must stay expired. + input_cache::replace_cluster_snapshot( + &mut original, + "default", + PersistedPlaylistUpdateClusterSnapshot { + source: Some(PlaylistUpdateDataSource::Cache), + ..PersistedPlaylistUpdateClusterSnapshot::default() + }, + ); + input_cache::save_input_status(&path, &original); + for state in [PlaylistUpdateState::Success, PlaylistUpdateState::Partial, PlaylistUpdateState::Failure] { + persist_input_completion(&ctx, 7, &input_name, state).await; + let reloaded = input_cache::load_input_status(&path); + let result = reloaded.last_input_update.unwrap(); + assert_eq!(result.state, state); + assert!(result.timestamp > 1); + assert_eq!(reloaded.clusters, original.clusters); + assert!(!input_cache::is_cache_valid(&reloaded, "default", 3600)); + let json: serde_json::Value = + serde_json::from_slice(&std::fs::read(path.join(input_cache::STATUS_FILE)).unwrap()).unwrap(); + assert!(json["last_input_update"].is_object()); + assert_eq!(json.as_object().unwrap().len(), 2, "only clusters + one result, no history"); + } +} + +#[tokio::test] +async fn playlist_update_status_in_run_reuse_preserves_original_partial_result_and_timestamp() { + let temp = tempfile::tempdir().unwrap(); + let ctx = processing_context(temp.path()); + let mut result = super::super::playlist_update_run_result(7, InputJobState::Ready, vec![], true); + result.input_telemetry = Some(PlaylistUpdateInputTelemetry { + refresh_policy: InputRefreshPolicy::NORMAL, + source: Some(PlaylistUpdateDataSource::Provider), + clusters: Vec::new(), + }); + persist_input_job_result(&ctx, &result).await; + let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &result.input_name).await; + let mut original = input_cache::load_input_status(&path); + assert_eq!(original.last_input_update.unwrap().state, PlaylistUpdateState::Partial); + original.last_input_update.as_mut().unwrap().timestamp = 123; + input_cache::save_input_status(&path, &original); + let before = std::fs::read(path.join(input_cache::STATUS_FILE)).unwrap(); + result.had_quality_rejections = false; + result.input_telemetry = None; // Same existing absence of acquisition facts as a real in-run reuse. + persist_input_job_result(&ctx, &result).await; + assert_eq!(std::fs::read(path.join(input_cache::STATUS_FILE)).unwrap(), before); + result.state = InputJobState::Failed; // A genuine technical completion must still replace the prior result. + persist_input_job_result(&ctx, &result).await; + assert_eq!(input_cache::load_input_status(&path).last_input_update.unwrap().state, PlaylistUpdateState::Failure); +} + +#[tokio::test] +async fn playlist_update_status_completion_covers_every_input_type_without_artificial_clusters() { + let temp = tempfile::tempdir().unwrap(); + let ctx = processing_context(temp.path()); + for (id, input_type) in [ + InputType::Xtream, + InputType::XtreamBatch, + InputType::Stalker, + InputType::StalkerBatch, + InputType::M3u, + InputType::M3uBatch, + InputType::Library, + InputType::Plex, + InputType::Emby, + InputType::Jellyfin, + InputType::Staged, + ] + .into_iter() + .enumerate() + { + let input = ConfigInput { + id: u16::try_from(id).unwrap(), + input_type, + name: format!("input-{id}").intern(), + ..ConfigInput::default() + }; + let result = panicked_input_job(id, &input); + persist_input_job_result(&ctx, &result).await; + let path = input_cache::resolve_input_storage_path(&temp.path().to_string_lossy(), &input.name).await; + let persisted = input_cache::load_input_status(&path); + assert!(persisted.clusters.is_empty()); + assert!( + matches!(persisted.last_input_update, Some(PersistedPlaylistUpdateInputResult {state: PlaylistUpdateState::Failure, timestamp}) if timestamp > 0) + ); + } +} diff --git a/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/staged_completion.rs b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/staged_completion.rs new file mode 100644 index 000000000..867bd84a3 --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/quality_rejection_fallback/staged_completion.rs @@ -0,0 +1,160 @@ +use super::*; + +pub(super) fn input_completions(events: &CollectSink, input_id: u16) -> Vec { + events + .0 + .lock() + .unwrap() + .iter() + .filter_map(|event| match event { + EventMessage::PlaylistUpdateProgress(progress) + if progress.input_id == Some(input_id) && progress.state.is_some() => + { + Some(progress.clone()) + } + _ => None, + }) + .collect() +} + +async fn staged_run_context(root: &Path, base_url: &str) -> (PlaylistProcessingContext, Arc) { + let ctx = processing_context_with_events(root, CollectSink::default()); + let parent_file = root.join("parent.m3u"); + std::fs::write(&parent_file, "#EXTM3U\n#EXTINF:-1,Parent\nhttp://stream.example/parent\n").unwrap(); + let parent = Arc::new(ConfigInput { + id: 7, + name: "parent".intern(), + input_type: InputType::M3u, + url: parent_file.to_string_lossy().into_owned(), + enabled: true, + ..ConfigInput::default() + }); + let mut staged = (*test_input( + base_url, + ConfigInputUpdateQualityDto { live: 95, vod: 0, series: 0 }, + 0, + &[XtreamCluster::Video, XtreamCluster::Series], + )) + .clone(); + staged.id = 8; + staged.name = "staged".intern(); + staged.input_type = InputType::Staged; + staged.staged_type = shared::model::StagedInputType::Xtream; + staged.staged = Some(tuliprox_core::model::ConfigInputStaged { + for_input: Some(parent.name.clone()), + clusters: ClusterFlags::Live, + }); + staged.resolve_staged_download_type(); + let staged = Arc::new(staged); + seed_live_baseline(&ctx, &staged, 100).await; + let failed_companion = Arc::new(ConfigInput { + id: 9, + name: "failed-companion".intern(), + input_type: InputType::M3u, + url: root.join("missing.m3u").to_string_lossy().into_owned(), + enabled: true, + ..ConfigInput::default() + }); + ctx.config.sources.store(Arc::new(SourcesConfig { + inputs: vec![parent.clone(), staged.clone(), failed_companion.clone()], + sources: vec![ConfigSource { + inputs: vec![parent.name.clone(), failed_companion.name.clone()], + targets: vec![xtream_target(true)], + }], + ..SourcesConfig::default() + })); + (ctx, staged) +} + +fn normalized_trace( + events: &[EventMessage], + persisted: shared::model::PersistedPlaylistUpdateInputResult, +) -> serde_json::Value { + let summary = events + .iter() + .find_map(|event| match event { + EventMessage::PlaylistUpdate(summary) => Some(summary), + _ => None, + }) + .unwrap(); + assert_eq!(summary.run_id, Some("staged-completion-run".into())); + assert!(summary.execution_order.is_some()); + assert_eq!(summary.state, PlaylistUpdateState::Failure); + let progress: Vec<_> = events + .iter() + .filter_map(|event| match event { + EventMessage::PlaylistUpdateProgress(progress) + if progress.input_id == Some(8) || progress.state.is_some() => + { + assert_eq!(progress.run_id, summary.run_id); + assert_eq!(progress.execution_order, summary.execution_order); + let mut progress = progress.clone(); + // Only the globally allocated test-run order is normalized. All + // input identities, states, messages and telemetry remain exact. + progress.execution_order = Some(1.into()); + Some(progress) + } + _ => None, + }) + .collect(); + assert!(persisted.timestamp > 0); + serde_json::json!({ + "progress": progress, + "completed": {"run_id": summary.run_id, "execution_order": 1, "state": summary.state}, + "persisted_input_result": {"state": persisted.state, "timestamp": 1} + }) +} + +#[tokio::test] +async fn playlist_update_status_staged_reporter_trace_matches_persistence_before_foreign_run_failure() { + let mut traces = Vec::new(); + for expected in [PlaylistUpdateState::Partial, PlaylistUpdateState::Success, PlaylistUpdateState::Failure] { + let temp = tempfile::tempdir().unwrap(); + let mut responses = fixture_responses([if expected == PlaylistUpdateState::Success { 100 } else { 1 }, 0, 0]); + if expected == PlaylistUpdateState::Failure { + responses.insert("get_live_streams".into(), "not JSON".into()); + } + let server = TestXtreamServer::start_with_responses(responses); + let (ctx, staged) = staged_run_context(temp.path(), &server.base_url).await; + exec_processing(ProcessingRun::for_run( + "staged-completion-run".into(), + ctx.client.clone(), + ctx.config.clone(), + ctx.user_targets.clone(), + ctx.events.clone(), + )) + .await; + let path = input_storage_path(&ctx, &staged).await; + let persisted = input_cache::load_input_status(&path); + assert!(!persisted.clusters.contains_key("default")); + assert_eq!(persisted.last_input_update.unwrap().state, expected); + let completions = input_completions(&ctx.events, staged.id); + assert_eq!(completions.len(), 1, "real indirect staged input must emit one correlated completion"); + assert_eq!(completions[0].state, Some(expected)); + let telemetry = completions[0].input_telemetry.as_ref().unwrap(); + assert_eq!(telemetry.refresh_policy, InputRefreshPolicy::NORMAL); + assert_eq!(telemetry.source, None); + assert_eq!(telemetry.clusters[0].source, Some(PlaylistUpdateDataSource::Provider)); + assert_eq!(telemetry.clusters[0].threshold, Some(95)); + if expected == PlaylistUpdateState::Partial { + assert_eq!(telemetry.clusters[0].decision, Some(PlaylistUpdateClusterDecision::Rejected)); + assert_eq!(telemetry.clusters[0].active_count, Some(100)); + } + let emitted = ctx.events.0.lock().unwrap().clone(); + let own_index = emitted.iter().position(|event| matches!(event, EventMessage::PlaylistUpdateProgress(p) if p.input_id == Some(8) && p.state.is_some())).unwrap(); + let parent_index = emitted.iter().position(|event| matches!(event, EventMessage::PlaylistUpdateProgress(p) if p.input_id == Some(7) && p.state.is_some())).unwrap(); + let completed_index = + emitted.iter().position(|event| matches!(event, EventMessage::PlaylistUpdate(_))).unwrap(); + assert!(own_index < parent_index && parent_index < completed_index); + assert_eq!(input_completions(&ctx.events, 9)[0].state, Some(PlaylistUpdateState::Failure)); + traces.push(normalized_trace(&emitted, persisted.last_input_update.unwrap())); + assert_eq!(server.finish().iter().filter(|action| action.as_str() == "get_live_streams").count(), 1); + } + let actual = serde_json::Value::Array(traces); + let fixture: serde_json::Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../frontend/src/model/test_data/staged_completions.json" + ))) + .unwrap(); + assert_eq!(actual, fixture, "actual reporter trace: {}", serde_json::to_string_pretty(&actual).unwrap()); +} diff --git a/backend/processing/src/processor/playlist/tests/scheduling.rs b/backend/processing/src/processor/playlist/tests/scheduling.rs new file mode 100644 index 000000000..4e02fe11e --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/scheduling.rs @@ -0,0 +1,134 @@ +use super::{ + collect_target_task_result, stalker_checkpoint_message, target_mutated_resources, target_waiting_message, + wait_for_target_finalizer_slot, with_sequential_group, TargetJobResult, MAX_CONCURRENT_TARGET_FINALIZERS, +}; +use shared::model::{ConfigTargetDto, PipelineStats}; +use std::{sync::Arc, time::Duration}; +use tokio::task::JoinSet; +use tuliprox_core::model::{Config, ConfigTarget}; + +#[tokio::test] +async fn parallel_input_scheduler_serializes_equal_groups_and_overlaps_distinct_groups() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + async fn observe(active: &AtomicUsize, maximum: &AtomicUsize) { + let current = active.fetch_add(1, Ordering::SeqCst) + 1; + maximum.fetch_max(current, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(20)).await; + active.fetch_sub(1, Ordering::SeqCst); + } + + let locks = tuliprox_core::utils::FileLockManager::default(); + let active = AtomicUsize::new(0); + let maximum = AtomicUsize::new(0); + tokio::join!( + with_sequential_group(&locks, Some(7), true, observe(&active, &maximum)), + with_sequential_group(&locks, Some(7), true, observe(&active, &maximum)), + ); + assert_eq!(maximum.load(Ordering::SeqCst), 1); + + maximum.store(0, Ordering::SeqCst); + tokio::join!( + with_sequential_group(&locks, Some(7), true, observe(&active, &maximum)), + with_sequential_group(&locks, Some(8), true, observe(&active, &maximum)), + ); + assert_eq!(maximum.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn parallel_input_scheduler_releases_group_after_abort() { + let locks = Arc::new(tuliprox_core::utils::FileLockManager::default()); + let task_locks = Arc::clone(&locks); + let task = tokio::spawn(async move { + with_sequential_group(&task_locks, Some(7), true, std::future::pending::<()>()).await; + }); + tokio::task::yield_now().await; + task.abort(); + let _ = task.await; + + tokio::time::timeout(Duration::from_secs(1), with_sequential_group(&locks, Some(7), true, std::future::ready(()))) + .await + .expect("aborting an input job must release its sequential group"); +} + +#[test] +fn input_progress_message_contains_each_target_and_blocking_input() { + let targets = ["target-a", "target-b"]; + let inputs = ["input-a", "input-b"]; + let messages: Vec<_> = targets + .iter() + .flat_map(|target| inputs.iter().map(move |input| target_waiting_message(target, input))) + .collect(); + + assert_eq!(messages.len(), 4); + for target in targets { + for input in inputs { + assert!(messages.contains(&format!("Target '{target}' is waiting for input '{input}'"))); + } + } + assert!(stalker_checkpoint_message("portal-a").contains("portal-a")); +} + +#[tokio::test] +async fn parallel_target_pipeline_bounds_active_finalizers() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let active = Arc::new(AtomicUsize::new(0)); + let maximum = Arc::new(AtomicUsize::new(0)); + let mut tasks = JoinSet::new(); + let mut results = Vec::new(); + let mut errors = Vec::new(); + + for index in 0..6 { + wait_for_target_finalizer_slot(&mut tasks, &mut results, &mut errors).await; + let active = Arc::clone(&active); + let maximum = Arc::clone(&maximum); + tasks.spawn(async move { + let current = active.fetch_add(1, Ordering::SeqCst) + 1; + maximum.fetch_max(current, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(10)).await; + active.fetch_sub(1, Ordering::SeqCst); + TargetJobResult { + index, + name: format!("target-{index}"), + result: Ok(()), + errors: Vec::new(), + processing: PipelineStats::default(), + } + }); + } + while let Some(result) = tasks.join_next().await { + collect_target_task_result(result, &mut results, &mut errors); + } + + assert!(maximum.load(Ordering::SeqCst) <= MAX_CONCURRENT_TARGET_FINALIZERS); + assert_eq!(results.len(), 6); + assert!(errors.is_empty()); +} + +#[test] +fn parallel_target_pipeline_normalizes_conflicting_output_resources() { + let config = Config { storage_dir: "/tmp/tuliprox-target-resources".to_string(), ..Config::default() }; + + let mut spaced = ConfigTarget::from(&ConfigTargetDto::default()); + spaced.name = "A B".to_string(); + let mut underscored = ConfigTarget::from(&ConfigTargetDto::default()); + underscored.name = "A_B".to_string(); + assert!(!target_mutated_resources(&config, &spaced).is_disjoint(&target_mutated_resources(&config, &underscored))); + + spaced.name = "one".to_string(); + spaced.output = vec![tuliprox_core::model::TargetOutput::M3u(tuliprox_core::model::M3uTargetOutput { + filename: Some("out/../x.m3u".to_string()), + include_type_in_url: false, + mask_redirect_url: false, + filter: None, + })]; + underscored.name = "two".to_string(); + underscored.output = vec![tuliprox_core::model::TargetOutput::M3u(tuliprox_core::model::M3uTargetOutput { + filename: Some("x.m3u".to_string()), + include_type_in_url: false, + mask_redirect_url: false, + filter: None, + })]; + assert!(!target_mutated_resources(&config, &spaced).is_disjoint(&target_mutated_resources(&config, &underscored))); +} diff --git a/backend/processing/src/processor/playlist/tests/staged_overlay.rs b/backend/processing/src/processor/playlist/tests/staged_overlay.rs new file mode 100644 index 000000000..9d12c1f02 --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/staged_overlay.rs @@ -0,0 +1,874 @@ +use super::{ + apply_staged_overlay_groups, build_curated_playlist_views, catalog_membership, catalog_test_item, + complete_catalog_evaluation, should_apply_staged_overlay, target_watch_view, test_group, PlaylistDownloadResult, +}; +use shared::{ + model::{ + ClusterFlags, InputType, PlaylistGroup, PlaylistItemType, StagedInputType, StreamProperties, TraktApiConfigDto, + TraktCatalogSelection, TraktConfigDto, TraktContentType, TraktListConfigDto, UUIDType, XtreamCluster, + }, + utils::Internable, +}; +use tuliprox_core::model::{ConfigInput, ConfigInputFlags, ConfigInputOptions, CurationConfig, TraktConfig}; +use tuliprox_curation::CurationMediaKind; + +#[test] +fn staged_xtream_vod_url_uses_provider_credentials_and_one_extension() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut staged_vod = test_group(XtreamCluster::Video, "staged-vod", "staged"); + staged_vod.channels[0].header.id = "310".intern(); + staged_vod.channels[0].header.url = "http://iptvhost.example/movie/fake-user/fake-pass/310.mkv".intern(); + + let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Vod, Vec::new(), vec![staged_vod]); + + assert_eq!(groups[0].channels[0].header.url.as_ref(), "http://provider.example/movie/real-user/real-pass/310.mkv"); +} + +#[test] +fn staged_xtream_overlay_keeps_known_provider_stream_urls() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_vod = test_group(XtreamCluster::Video, "Movies", "provider"); + provider_vod.channels[0].header.id = "310".intern(); + provider_vod.channels[0].header.url = "https://cdn.example/video?id=310".intern(); + let mut second_provider = provider_vod.channels[0].clone(); + second_provider.header.id = "311".intern(); + second_provider.header.url = "http://provider.example/movie/real-user/real-pass/311.mp4".intern(); + provider_vod.channels.push(second_provider); + + let mut staged_vod = test_group(XtreamCluster::Video, "Edited Movies", "staged"); + staged_vod.channels[0].header.id = "310".intern(); + staged_vod.channels[0].header.url = "http://editor.example/movie/310.mkv".intern(); + let mut second_staged = staged_vod.channels[0].clone(); + second_staged.header.id = "311".intern(); + second_staged.header.url = "http://editor.example/movie/311.mkv".intern(); + staged_vod.channels.push(second_staged); + + let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Vod, vec![provider_vod], vec![staged_vod]); + + assert_eq!(groups[0].title.as_ref(), "Edited Movies"); + assert_eq!(groups[0].channels[0].header.url.as_ref(), "https://cdn.example/video?id=310"); + assert_eq!(groups[0].channels[1].header.url.as_ref(), "http://provider.example/movie/real-user/real-pass/311.mp4"); +} + +#[test] +fn staged_xtream_new_vod_stream_uses_metadata_extension() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut staged_vod = test_group(XtreamCluster::Video, "New Movies", "staged"); + staged_vod.channels[0].header.id = "310".intern(); + staged_vod.channels[0].header.url = "http://editor.example/movie/310".intern(); + staged_vod.channels[0].header.additional_properties = + Some(StreamProperties::Video(Box::new(shared::model::VideoStreamProperties { + container_extension: "mkv".intern(), + ..Default::default() + }))); + + let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Vod, Vec::new(), vec![staged_vod]); + + assert_eq!(groups[0].channels[0].header.url.as_ref(), "http://provider.example/movie/real-user/real-pass/310.mkv"); +} + +#[test] +fn staged_xtream_live_url_respects_prefix_and_without_extension_flags() { + let provider_prefix = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + options: Some(ConfigInputOptions { + flags: ConfigInputFlags::XtreamLiveStreamUsePrefix.into(), + ..ConfigInputOptions::defaults().clone() + }), + ..Default::default() + }; + let mut staged_live_prefix = test_group(XtreamCluster::Live, "staged-live", "staged"); + staged_live_prefix.channels[0].header.id = "11203".intern(); + staged_live_prefix.channels[0].header.url = "http://iptvhost.example/fake-user/fake-pass/11203.ts".intern(); + + let groups_prefix = + apply_staged_overlay_groups(&provider_prefix, ClusterFlags::Live, Vec::new(), vec![staged_live_prefix]); + + assert_eq!( + groups_prefix[0].channels[0].header.url.as_ref(), + "http://provider.example/live/real-user/real-pass/11203.ts" + ); + + let provider_no_ext = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + options: Some(ConfigInputOptions { + flags: ConfigInputFlags::XtreamLiveStreamWithoutExtension.into(), + ..ConfigInputOptions::defaults().clone() + }), + ..Default::default() + }; + let mut staged_live_no_ext = test_group(XtreamCluster::Live, "staged-live", "staged"); + staged_live_no_ext.channels[0].header.id = "11203".intern(); + staged_live_no_ext.channels[0].header.url = "http://iptvhost.example/fake-user/fake-pass/11203.ts".intern(); + + let groups_no_ext = + apply_staged_overlay_groups(&provider_no_ext, ClusterFlags::Live, Vec::new(), vec![staged_live_no_ext]); + + assert_eq!(groups_no_ext[0].channels[0].header.url.as_ref(), "http://provider.example/real-user/real-pass/11203"); +} + +#[test] +fn staged_xtream_overlay_full_cluster_staging_replaces_all_matching_groups() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_action = test_group(XtreamCluster::Video, "Action", "provider"); + provider_action.channels[0].header.id = "10".intern(); + let mut provider_comedy = test_group(XtreamCluster::Video, "Comedy", "provider"); + provider_comedy.channels[0].header.id = "20".intern(); + + let mut staged_action = test_group(XtreamCluster::Video, "Action", "staged"); + staged_action.channels[0].header.id = "11".intern(); + let mut staged_comedy = test_group(XtreamCluster::Video, "Comedy", "staged"); + staged_comedy.channels[0].header.id = "21".intern(); + + let groups = apply_staged_overlay_groups( + &provider, + ClusterFlags::Vod, + vec![provider_action, provider_comedy], + vec![staged_action, staged_comedy], + ); + + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].title.as_ref(), "Action"); + assert_eq!(groups[0].channels[0].header.id.as_ref(), "11"); + assert_eq!(groups[0].channels[0].header.input_name.as_ref(), "provider"); + assert_eq!(groups[1].title.as_ref(), "Comedy"); + assert_eq!(groups[1].channels[0].header.id.as_ref(), "21"); + assert_eq!(groups[1].channels[0].header.input_name.as_ref(), "provider"); +} + +#[test] +fn staged_xtream_overlay_partial_cluster_staging_preserves_unstaged_provider_groups() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_action = test_group(XtreamCluster::Video, "Action", "provider"); + provider_action.channels[0].header.id = "10".intern(); + let mut provider_comedy = test_group(XtreamCluster::Video, "Comedy", "provider"); + provider_comedy.channels[0].header.id = "20".intern(); + let mut provider_drama = test_group(XtreamCluster::Video, "Drama", "provider"); + provider_drama.channels[0].header.id = "30".intern(); + + let mut staged_action = test_group(XtreamCluster::Video, "Action", "staged"); + staged_action.channels[0].header.id = "11".intern(); + + let groups = apply_staged_overlay_groups( + &provider, + ClusterFlags::Vod, + vec![provider_action, provider_comedy, provider_drama], + vec![staged_action], + ); + + assert_eq!(groups.len(), 3); + assert_eq!(groups[0].title.as_ref(), "Action"); + assert_eq!(groups[0].channels[0].header.id.as_ref(), "11"); + assert_eq!(groups[1].title.as_ref(), "Comedy"); + assert_eq!(groups[1].channels[0].header.id.as_ref(), "20"); + assert_eq!(groups[2].title.as_ref(), "Drama"); + assert_eq!(groups[2].channels[0].header.id.as_ref(), "30"); +} + +#[test] +fn staged_xtream_overlay_no_staged_groups_for_selected_cluster_preserves_all_provider_groups() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let provider_news = test_group(XtreamCluster::Live, "News", "provider"); + let provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); + let staged_series = test_group(XtreamCluster::Series, "Shows", "staged"); + + let groups = apply_staged_overlay_groups( + &provider, + ClusterFlags::Live, + vec![provider_news, provider_sports], + vec![staged_series], + ); + + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].title.as_ref(), "News"); + assert_eq!(groups[1].title.as_ref(), "Sports"); +} + +#[test] +fn staged_xtream_overlay_all_invalid_ids_falls_back_to_provider_group() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); + provider_sports.channels[0].header.id = "501".intern(); + + let mut staged_sports = test_group(XtreamCluster::Live, "Sports", "staged"); + staged_sports.channels[0].header.id = "non-numeric-a".intern(); + let mut second_invalid = staged_sports.channels[0].clone(); + second_invalid.header.id = "non-numeric-b".intern(); + staged_sports.channels.push(second_invalid); + + let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Live, vec![provider_sports], vec![staged_sports]); + + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].title.as_ref(), "Sports"); + assert_eq!(groups[0].channels.len(), 1); + assert_eq!(groups[0].channels[0].header.id.as_ref(), "501"); +} + +#[test] +fn staged_xtream_overlay_partially_valid_ids_keeps_valid_and_drops_invalid() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); + provider_sports.channels[0].header.id = "501".intern(); + + let mut staged_sports = test_group(XtreamCluster::Live, "Sports", "staged"); + staged_sports.channels[0].header.id = "502".intern(); + let mut invalid = staged_sports.channels[0].clone(); + invalid.header.id = "invalid-stream".intern(); + staged_sports.channels.push(invalid); + + let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Live, vec![provider_sports], vec![staged_sports]); + + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].title.as_ref(), "Sports"); + assert_eq!(groups[0].channels.len(), 1); + assert_eq!(groups[0].channels[0].header.id.as_ref(), "502"); +} + +#[test] +fn staged_xtream_overlay_missing_credentials_preserves_all_provider_groups() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: None, + password: None, + ..Default::default() + }; + let provider_news = test_group(XtreamCluster::Live, "News", "provider"); + let provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); + let mut staged_sports = test_group(XtreamCluster::Live, "Sports", "staged"); + staged_sports.channels[0].header.id = "501".intern(); + + let groups = apply_staged_overlay_groups( + &provider, + ClusterFlags::Live, + vec![provider_news, provider_sports], + vec![staged_sports], + ); + + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].title.as_ref(), "News"); + assert_eq!(groups[1].title.as_ref(), "Sports"); +} + +#[test] +fn staged_xtream_overlay_multiple_groups_with_different_clusters() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_live = test_group(XtreamCluster::Live, "News", "provider"); + provider_live.channels[0].header.id = "100".intern(); + let mut provider_action = test_group(XtreamCluster::Video, "Action", "provider"); + provider_action.channels[0].header.id = "200".intern(); + let mut provider_comedy = test_group(XtreamCluster::Video, "Comedy", "provider"); + provider_comedy.channels[0].header.id = "300".intern(); + let mut provider_series = test_group(XtreamCluster::Series, "Shows", "provider"); + provider_series.channels[0].header.id = "400".intern(); + + let mut staged_live = test_group(XtreamCluster::Live, "News", "staged"); + staged_live.channels[0].header.id = "101".intern(); + let mut staged_action = test_group(XtreamCluster::Video, "Action", "staged"); + staged_action.channels[0].header.id = "201".intern(); + + let groups = apply_staged_overlay_groups( + &provider, + ClusterFlags::Live | ClusterFlags::Vod, + vec![provider_live, provider_action, provider_comedy, provider_series], + vec![staged_live, staged_action], + ); + + assert_eq!(groups.len(), 4); + assert_eq!(groups[0].title.as_ref(), "News"); + assert_eq!(groups[0].channels[0].header.id.as_ref(), "101"); + assert_eq!(groups[1].title.as_ref(), "Action"); + assert_eq!(groups[1].channels[0].header.id.as_ref(), "201"); + assert_eq!(groups[2].title.as_ref(), "Comedy"); + assert_eq!(groups[2].channels[0].header.id.as_ref(), "300"); + assert_eq!(groups[3].title.as_ref(), "Shows"); + assert_eq!(groups[3].channels[0].header.id.as_ref(), "400"); +} + +#[test] +fn staged_xtream_overlay_preserves_original_group_order() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut live_1 = test_group(XtreamCluster::Live, "Live1", "provider"); + live_1.channels[0].header.id = "1".intern(); + let mut vod_1 = test_group(XtreamCluster::Video, "Vod1", "provider"); + vod_1.channels[0].header.id = "2".intern(); + let mut live_2 = test_group(XtreamCluster::Live, "Live2", "provider"); + live_2.channels[0].header.id = "3".intern(); + let mut vod_2 = test_group(XtreamCluster::Video, "Vod2", "provider"); + vod_2.channels[0].header.id = "4".intern(); + + let mut staged_live_1 = test_group(XtreamCluster::Live, "Live1", "staged"); + staged_live_1.channels[0].header.id = "11".intern(); + let mut staged_live_2 = test_group(XtreamCluster::Live, "Live2", "staged"); + staged_live_2.channels[0].header.id = "33".intern(); + + let groups = apply_staged_overlay_groups( + &provider, + ClusterFlags::Live, + vec![live_1, vod_1, live_2, vod_2], + vec![staged_live_1, staged_live_2], + ); + + assert_eq!(groups.len(), 4); + assert_eq!(groups[0].title.as_ref(), "Live1"); + assert_eq!(groups[0].channels[0].header.id.as_ref(), "11"); + assert_eq!(groups[1].title.as_ref(), "Vod1"); + assert_eq!(groups[1].channels[0].header.id.as_ref(), "2"); + assert_eq!(groups[2].title.as_ref(), "Live2"); + assert_eq!(groups[2].channels[0].header.id.as_ref(), "33"); + assert_eq!(groups[3].title.as_ref(), "Vod2"); + assert_eq!(groups[3].channels[0].header.id.as_ref(), "4"); +} + +/// The merged playlist reaches persistence keyed by `(cluster, id)`, which collapses groups that +/// share an id within one cluster. +pub(in crate::processor::playlist::tests) fn assert_unique_category_ids(groups: &[PlaylistGroup]) { + let mut seen = std::collections::HashSet::new(); + for group in groups { + assert!( + seen.insert((group.xtream_cluster, group.id)), + "category id {} is used by more than one {} group", + group.id, + group.xtream_cluster + ); + } +} + +#[test] +fn staged_overlay_syncs_channel_category_ids_with_the_final_group_id() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_news = test_group(XtreamCluster::Live, "News", "provider"); + provider_news.id = 5; + provider_news.channels[0].header.id = "100".intern(); + + // Matched by stream id, so the group takes over the provider category id. + let mut staged_rewritten = test_group(XtreamCluster::Live, "Rewritten", "staged"); + staged_rewritten.id = 7; + staged_rewritten.channels[0].header.id = "100".intern(); + staged_rewritten.channels[0].header.category_id = 7; + // No provider counterpart: the group becomes a new category with an allocated id. + let mut staged_kids = test_group(XtreamCluster::Live, "Kids", "staged"); + staged_kids.id = 0; + staged_kids.channels[0].header.id = "300".intern(); + + let groups = apply_staged_overlay_groups( + &provider, + ClusterFlags::Live, + vec![provider_news], + vec![staged_rewritten, staged_kids], + ); + + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].id, 5); + assert_eq!(groups[0].channels[0].header.category_id, 5); + assert_eq!(groups[1].id, 6); + assert_eq!(groups[1].channels[0].header.category_id, 6); + assert_unique_category_ids(&groups); +} + +#[test] +fn staged_overlay_on_non_xtream_provider_replaces_cluster_and_rewrites_input_name() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::M3u, + url: "http://provider.example/list.m3u".to_string(), + ..Default::default() + }; + let provider_live = test_group(XtreamCluster::Live, "provider-live", "provider"); + let provider_vod = test_group(XtreamCluster::Video, "provider-vod", "provider"); + let mut staged_live = test_group(XtreamCluster::Live, "staged-live", "staged"); + staged_live.channels[0].header.url = "http://editor.example/live/1.ts".intern(); + + let groups = apply_staged_overlay_groups( + &provider, + ClusterFlags::Live, + vec![provider_live, provider_vod], + vec![staged_live], + ); + + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].title.as_ref(), "provider-vod"); + assert_eq!(groups[0].channels[0].header.input_name.as_ref(), "provider"); + assert_eq!(groups[1].title.as_ref(), "staged-live"); + // A non-Xtream provider cannot rebuild Xtream stream URLs, so the staged URL and its group id stay. + assert_eq!(groups[1].channels[0].header.input_name.as_ref(), "provider"); + assert_eq!(groups[1].channels[0].header.url.as_ref(), "http://editor.example/live/1.ts"); + assert_eq!(groups[1].id, 1); +} + +#[test] +fn staged_xtream_overlay_matches_by_category_id_not_by_title() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + // The staged tool renamed the provider category 7 and moved its streams; the title now collides + // with the unrelated provider category 5. + let mut provider_kids = test_group(XtreamCluster::Live, "Kids", "provider"); + provider_kids.id = 5; + provider_kids.channels[0].header.id = "105".intern(); + let mut provider_news = test_group(XtreamCluster::Live, "News", "provider"); + provider_news.id = 7; + provider_news.channels[0].header.id = "107".intern(); + + let mut staged_renamed = test_group(XtreamCluster::Live, "Kids", "staged"); + staged_renamed.id = 7; + staged_renamed.channels[0].header.id = "207".intern(); + + let groups = apply_staged_overlay_groups( + &provider, + ClusterFlags::Live, + vec![provider_kids, provider_news], + vec![staged_renamed], + ); + + assert_eq!(groups.len(), 2); + // Category 5 keeps its content: the staged group belongs to category 7. + assert_eq!(groups[0].title.as_ref(), "Kids"); + assert_eq!(groups[0].id, 5); + assert_eq!(groups[0].channels[0].header.id.as_ref(), "105"); + // Category 7 carries the staged rename and the staged stream, under its own id. + assert_eq!(groups[1].title.as_ref(), "Kids"); + assert_eq!(groups[1].id, 7); + assert_eq!(groups[1].channels[0].header.id.as_ref(), "207"); + assert_unique_category_ids(&groups); +} + +#[test] +fn staged_xtream_overlay_matches_by_stream_id_over_group_id() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_news = test_group(XtreamCluster::Live, "News", "provider"); + provider_news.id = 5; + provider_news.channels[0].header.id = "100".intern(); + let mut provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); + provider_sports.id = 7; + provider_sports.channels[0].header.id = "200".intern(); + + // A m3u staged playlist numbers its groups on its own, so the group id points at another + // provider category. The stream id of the overlaid channel proves which category it replaces. + let mut staged_sports = test_group(XtreamCluster::Live, "Sports Rewritten", "staged"); + staged_sports.id = 5; + staged_sports.channels[0].header.id = "200".intern(); + + let groups = apply_staged_overlay_groups( + &provider, + ClusterFlags::Live, + vec![provider_news, provider_sports], + vec![staged_sports], + ); + + assert_eq!(groups.len(), 2); + // Category 5 keeps its content: no staged channel belongs to it. + assert_eq!(groups[0].title.as_ref(), "News"); + assert_eq!(groups[0].id, 5); + assert_eq!(groups[0].channels[0].header.id.as_ref(), "100"); + // Category 7 carries the rewritten staged group, under its own id. + assert_eq!(groups[1].title.as_ref(), "Sports Rewritten"); + assert_eq!(groups[1].id, 7); + assert_eq!(groups[1].channels[0].header.id.as_ref(), "200"); + assert_unique_category_ids(&groups); +} + +#[test] +fn staged_xtream_split_group_does_not_replace_an_unrelated_category() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); + provider_sports.id = 5; + provider_sports.channels[0].header.id = "100".intern(); + let mut second_sport = provider_sports.channels[0].clone(); + second_sport.header.id = "101".intern(); + provider_sports.channels.push(second_sport); + let mut provider_news = test_group(XtreamCluster::Live, "News", "provider"); + provider_news.id = 7; + provider_news.channels[0].header.id = "200".intern(); + + let mut staged_first = test_group(XtreamCluster::Live, "Sports A", "staged"); + staged_first.id = 5; + staged_first.channels[0].header.id = "100".intern(); + let mut staged_second = test_group(XtreamCluster::Live, "News", "staged"); + staged_second.id = 7; + staged_second.channels[0].header.id = "101".intern(); + + let groups = apply_staged_overlay_groups( + &provider, + ClusterFlags::Live, + vec![provider_sports, provider_news], + vec![staged_first, staged_second], + ); + + assert_eq!(groups.len(), 3); + assert_eq!(groups[0].id, 5); + assert_eq!(groups[0].title.as_ref(), "Sports A"); + assert_eq!(groups[1].id, 7); + assert_eq!(groups[1].channels[0].header.id.as_ref(), "200"); + assert_eq!(groups[2].title.as_ref(), "News"); + assert_eq!(groups[2].channels[0].header.id.as_ref(), "101"); + assert_unique_category_ids(&groups); +} + +#[test] +fn staged_m3u_unknown_streams_do_not_match_positional_category_id() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); + provider_sports.id = 5; + provider_sports.channels[0].header.id = "100".intern(); + let mut staged_new = test_group(XtreamCluster::Live, "New", "staged"); + staged_new.id = 5; + staged_new.channels[0].header.id = "300".intern(); + + let groups = super::super::apply_staged_overlay_groups( + &provider, + StagedInputType::M3u, + ClusterFlags::Live, + vec![provider_sports], + vec![staged_new], + ); + + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].title.as_ref(), "New"); + assert_eq!(groups[0].channels[0].header.id.as_ref(), "300"); + assert_unique_category_ids(&groups); +} + +#[test] +fn staged_m3u_replaces_selected_xtream_groups_and_keeps_other_clusters() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_news = test_group(XtreamCluster::Live, "Provider News", "provider"); + provider_news.id = 5; + provider_news.channels[0].header.id = "100".intern(); + let mut provider_sports = test_group(XtreamCluster::Live, "Provider Sports", "provider"); + provider_sports.id = 7; + provider_sports.channels[0].header.id = "200".intern(); + provider_sports.channels[0].header.url = "http://provider.example/direct/200.ts".intern(); + let mut provider_kids = test_group(XtreamCluster::Live, "Provider Kids", "provider"); + provider_kids.id = 9; + provider_kids.channels[0].header.id = "300".intern(); + let provider_vod = test_group(XtreamCluster::Video, "Provider VOD", "provider"); + let mut staged_custom = test_group(XtreamCluster::Live, "My Channels", "staged"); + staged_custom.channels[0].header.id = "200".intern(); + let mut staged_news = test_group(XtreamCluster::Live, "My News", "staged"); + staged_news.channels[0].header.id = "100".intern(); + + let groups = super::super::apply_staged_overlay_groups( + &provider, + StagedInputType::M3u, + ClusterFlags::Live, + vec![provider_news, provider_sports, provider_kids, provider_vod], + vec![staged_custom, staged_news], + ); + + assert_eq!(groups.len(), 3); + assert_eq!(groups[0].title.as_ref(), "My Channels"); + assert_eq!(groups[0].channels[0].header.id.as_ref(), "200"); + assert_eq!(groups[0].channels[0].header.input_name.as_ref(), "provider"); + assert_eq!(groups[0].channels[0].header.url.as_ref(), "http://provider.example/direct/200.ts"); + assert_eq!(groups[1].title.as_ref(), "My News"); + assert_eq!(groups[1].channels[0].header.id.as_ref(), "100"); + assert_eq!(groups[2].title.as_ref(), "Provider VOD"); +} + +#[test] +fn staged_xtream_overlay_falls_back_to_the_secondary_stream_overlap() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_alpha = test_group(XtreamCluster::Live, "Alpha", "provider"); + provider_alpha.id = 5; + provider_alpha.channels[0].header.id = "100".intern(); + let mut provider_beta = test_group(XtreamCluster::Live, "Beta", "provider"); + provider_beta.id = 7; + provider_beta.channels[0].header.id = "200".intern(); + let mut beta_second = provider_beta.channels[0].clone(); + beta_second.header.id = "201".intern(); + provider_beta.channels.push(beta_second); + + // Both staged groups overlap Beta; the stronger one takes it, so the other has to fall back to the + // provider category it also owns streams of instead of becoming a new category. + let mut staged_strong = test_group(XtreamCluster::Live, "Beta Rewritten", "staged"); + staged_strong.id = 0; + staged_strong.channels[0].header.id = "200".intern(); + let mut strong_second = staged_strong.channels[0].clone(); + strong_second.header.id = "201".intern(); + staged_strong.channels.push(strong_second); + + let mut staged_weak = test_group(XtreamCluster::Live, "Alpha Rewritten", "staged"); + staged_weak.id = 0; + staged_weak.channels[0].header.id = "200".intern(); + let mut weak_second = staged_weak.channels[0].clone(); + weak_second.header.id = "201".intern(); + staged_weak.channels.push(weak_second); + let mut weak_third = staged_weak.channels[0].clone(); + weak_third.header.id = "100".intern(); + staged_weak.channels.push(weak_third); + + let groups = apply_staged_overlay_groups( + &provider, + ClusterFlags::Live, + vec![provider_alpha, provider_beta], + vec![staged_strong, staged_weak], + ); + + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].title.as_ref(), "Alpha Rewritten"); + assert_eq!(groups[0].id, 5); + assert_eq!(groups[0].channels.len(), 3); + assert_eq!(groups[1].title.as_ref(), "Beta Rewritten"); + assert_eq!(groups[1].id, 7); + assert_eq!(groups[1].channels.len(), 2); + assert_unique_category_ids(&groups); +} + +#[test] +fn staged_xtream_overlay_gives_a_new_category_a_free_category_id() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_sports = test_group(XtreamCluster::Live, "Sports", "provider"); + provider_sports.id = 2; + provider_sports.channels[0].header.id = "200".intern(); + + // Staged groups without a category id must not reach persistence as a shared zero key. + let mut staged_kids = test_group(XtreamCluster::Live, "Kids", "staged"); + staged_kids.id = 0; + staged_kids.channels[0].header.id = "300".intern(); + let mut staged_movies = test_group(XtreamCluster::Live, "Movies", "staged"); + staged_movies.id = 0; + staged_movies.channels[0].header.id = "400".intern(); + + let groups = apply_staged_overlay_groups( + &provider, + ClusterFlags::Live, + vec![provider_sports], + vec![staged_kids, staged_movies], + ); + + assert_eq!(groups.len(), 3); + assert_eq!(groups[0].title.as_ref(), "Sports"); + assert_eq!(groups[0].id, 2); + assert_eq!(groups[1].title.as_ref(), "Kids"); + assert_eq!(groups[1].id, 3); + assert_eq!(groups[2].title.as_ref(), "Movies"); + assert_eq!(groups[2].id, 4); + assert_unique_category_ids(&groups); +} + +#[test] +fn staged_xtream_overlay_empty_staged_group_without_provider_fallback_is_omitted() { + let provider = ConfigInput { + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("real-user".to_string()), + password: Some("real-pass".to_string()), + ..Default::default() + }; + let mut provider_action = test_group(XtreamCluster::Video, "Action", "provider"); + provider_action.channels[0].header.id = "10".intern(); + + let mut staged_horror = test_group(XtreamCluster::Video, "Horror", "staged"); + staged_horror.id = 2; + staged_horror.channels[0].header.id = "non-numeric".intern(); + + let groups = apply_staged_overlay_groups(&provider, ClusterFlags::Vod, vec![provider_action], vec![staged_horror]); + + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].title.as_ref(), "Action"); +} + +#[test] +fn staged_overlay_is_skipped_when_provider_playlist_is_cached() { + let result = PlaylistDownloadResult::new(vec![], vec![], true, false); + + assert!(!should_apply_staged_overlay(&result)); +} + +#[test] +fn xtream_base_and_selector_category_projection_are_independent() { + let selected_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000011"); + let rejected_uuid = UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000012"); + let playlist = vec![PlaylistGroup { + id: 1, + title: "Movies".intern(), + channels: vec![ + catalog_test_item("Selected", selected_uuid, PlaylistItemType::Video, XtreamCluster::Video, None), + catalog_test_item("Rejected", rejected_uuid, PlaylistItemType::Video, XtreamCluster::Video, None), + ], + xtream_cluster: XtreamCluster::Video, + }]; + let evaluation = complete_catalog_evaluation(vec![catalog_membership(selected_uuid, CurationMediaKind::Movie, 0)]); + let dto = TraktConfigDto { + enabled: true, + catalog_selection: TraktCatalogSelection::Curated, + include_xtream_base_categories: false, + api: TraktApiConfigDto::default(), + lists: vec![TraktListConfigDto { + user: "alice".to_string(), + list_slug: "watchlist".to_string(), + category_name: Some("Curated".to_string()), + create_xtream_category: true, + content_type: TraktContentType::Vod, + tmdb_only: true, + fuzzy_match_threshold: 100, + }], + charts: Vec::new(), + }; + let config = TraktConfig::from(&dto); + + let views = + build_curated_playlist_views(playlist.clone(), &evaluation, &CurationConfig::from(&config), false, true); + + assert_eq!(views.base.len(), 1); + assert_eq!(views.base[0].channels.len(), 1); + assert!( + target_watch_view(&views.base, views.xtream.as_deref()).iter().any(|group| group.title.as_ref() == "Curated"), + "configured Trakt watches observe the Xtream category appearance" + ); + let xtream = views.xtream.expect("complete curation has an Xtream view"); + assert_eq!(xtream.len(), 1); + assert_eq!(xtream[0].title.as_ref(), "Curated"); + assert_eq!(xtream[0].channels.len(), 1); + assert_ne!(xtream[0].channels[0].header.uuid, selected_uuid); + + let mut compatible_dto = dto; + compatible_dto.catalog_selection = TraktCatalogSelection::Full; + compatible_dto.include_xtream_base_categories = true; + compatible_dto.lists[0].create_xtream_category = false; + compatible_dto.lists[0].category_name = None; + let compatible = build_curated_playlist_views( + playlist, + &evaluation, + &CurationConfig::from(&TraktConfig::from(&compatible_dto)), + false, + true, + ); + assert_eq!(compatible.base[0].channels.len(), 2); + assert_eq!(compatible.xtream.expect("Xtream view").len(), 1, "selection-only selector creates no category"); +} diff --git a/backend/processing/src/processor/playlist/tests/support.rs b/backend/processing/src/processor/playlist/tests/support.rs new file mode 100644 index 000000000..8dcc9575b --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/support.rs @@ -0,0 +1,141 @@ +use super::{create_input_stat, InputJobResult, InputJobState}; +use shared::{ + error::TuliproxError, + model::{ + ClusterFlags, EventMessage, EventSink, InputType, PlaylistGroup, PlaylistItem, PlaylistItemHeader, + PlaylistItemType, StagedInputType, UUIDType, XtreamCluster, + }, + utils::Internable, +}; +use std::sync::Arc; +use tuliprox_core::model::ConfigInput; +use tuliprox_curation::{ + CurationEvaluation, CurationMediaKind, CurationMembership, CurationSelectorKey, CurationSelectorSummary, +}; + +pub(in crate::processor::playlist::tests) fn apply_staged_overlay_groups( + provider: &ConfigInput, + clusters: ClusterFlags, + provider_groups: Vec, + staged_groups: Vec, +) -> Vec { + super::super::apply_staged_overlay_groups( + provider, + StagedInputType::Xtream, + clusters, + provider_groups, + staged_groups, + ) +} + +#[derive(Clone, Default)] +pub(in crate::processor::playlist::tests) struct PlaylistRunCollectSink( + pub(in crate::processor::playlist::tests) Arc>>, +); + +impl EventSink for PlaylistRunCollectSink { + fn emit(&self, event: EventMessage) { + self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push(event); + } +} + +pub(in crate::processor::playlist::tests) fn playlist_update_run_result( + input_id: u16, + state: InputJobState, + errors: Vec, + had_quality_rejections: bool, +) -> InputJobResult { + let input_name = format!("input-{input_id}").intern(); + InputJobResult { + index: usize::from(input_id), + input_id, + input_name: Arc::clone(&input_name), + state, + source: None, + epg: None, + stat: create_input_stat(0, 0, errors.len(), InputType::Xtream, &input_name, 0), + errors, + accepted_empty_clusters: ClusterFlags::empty(), + had_quality_rejections, + input_telemetry: None, + } +} + +pub(in crate::processor::playlist::tests) fn test_group( + cluster: XtreamCluster, + item_name: &str, + input_name: &str, +) -> PlaylistGroup { + PlaylistGroup { + id: 1, + title: item_name.intern(), + xtream_cluster: cluster, + channels: vec![PlaylistItem { + header: PlaylistItemHeader { + name: item_name.intern(), + input_name: input_name.intern(), + xtream_cluster: cluster, + item_type: match cluster { + XtreamCluster::Live => PlaylistItemType::Live, + XtreamCluster::Video => PlaylistItemType::Video, + XtreamCluster::Series => PlaylistItemType::Series, + }, + ..Default::default() + }, + }], + } +} + +pub(in crate::processor::playlist::tests) fn catalog_test_item( + title: &str, + uuid: UUIDType, + item_type: PlaylistItemType, + cluster: XtreamCluster, + parent_code: Option<&str>, +) -> PlaylistItem { + PlaylistItem { + header: PlaylistItemHeader { + id: title.intern(), + name: title.intern(), + title: title.intern(), + group: match cluster { + XtreamCluster::Live => "Live".intern(), + XtreamCluster::Video => "Movies".intern(), + XtreamCluster::Series => "Series".intern(), + }, + uuid, + item_type, + xtream_cluster: cluster, + parent_code: parent_code.unwrap_or_default().intern(), + ..PlaylistItemHeader::default() + }, + } +} + +pub(in crate::processor::playlist::tests) fn complete_catalog_evaluation( + memberships: Vec, +) -> CurationEvaluation { + CurationEvaluation { + selectors: vec![CurationSelectorSummary { + key: CurationSelectorKey(0), + reference_count: memberships.len(), + membership_count: memberships.len(), + }], + memberships, + } +} + +pub(in crate::processor::playlist::tests) fn catalog_membership( + uuid: UUIDType, + media_kind: CurationMediaKind, + order: usize, +) -> CurationMembership { + CurationMembership { + selector_key: CurationSelectorKey(0), + subject_uuid: uuid, + media_kind, + rank: Some(u32::try_from(order + 1).expect("test rank")), + title_tiebreak: format!("item-{order}"), + candidate_order: order, + } +} diff --git a/backend/repository/src/playlist_repository.rs b/backend/repository/src/playlist_repository.rs index ed313484d..f79cc62e4 100644 --- a/backend/repository/src/playlist_repository.rs +++ b/backend/repository/src/playlist_repository.rs @@ -1,2772 +1,47 @@ use super::playlist_mem_cache::{ PlaylistM3uStorage, PlaylistStorageState, PlaylistXtreamStorage, TargetPlaylistStorage, }; -use crate::{ - ensure_target_storage_path, epg_write_for_target, get_input_storage_path, get_target_id_mapping_file, - get_target_storage_path, load_input_local_library_playlist, load_input_m3u_playlist, load_input_xtream_playlist, - m3u_get_file_path_for_db, m3u_write_playlist, persist_input_library_playlist, persist_input_m3u_playlist, - stalker_repository::get_stalker_storage_path, write_strm_playlist, xtream_get_file_path, xtream_get_storage_path, - xtream_write_playlist, BPlusTree, LocalLibraryDiskPlaylistSource, M3uDiskPlaylistSource, - MediaServerDiskPlaylistSource, MemoryPlaylistSource, PlaylistSource, StalkerDiskPlaylistSource, TargetIdMapping, - VirtualIdRecord, XtreamDiskPlaylistSource, FILE_SUFFIX_DB, -}; -use log::{debug, warn}; -use shared::{ - error::TuliproxError, - model::{ - xtream_const::XTREAM_CLUSTER, ClusterFlags, ConfigTargetOptions, InputPersistence, M3uPlaylistItem, - PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, - SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, StreamProperties, UUIDType, VirtualId, - XtreamCluster, XtreamPlaylistItem, - }, - utils::{is_dash_url, is_hls_url, Internable}, -}; -use std::{ - collections::{HashMap, HashSet}, - path::{Path, PathBuf}, - sync::Arc, -}; -use tuliprox_core::{ - model::{apply_filter_to_playlist, AppConfig, ConfigInput, ConfigTarget, Epg, TargetOutput}, - utils::{self, fold_epg_id_arc, normalized_source_ordinal}, -}; +use std::sync::Arc; struct LocalEpisodeKey { path: Arc, virtual_id: u32, } -fn playlist_has_items(playlist: &[PlaylistGroup]) -> bool { playlist.iter().any(|group| !group.channels.is_empty()) } - -#[derive(Debug, Clone, Copy, Default)] -enum TargetPersistenceMode { - #[default] - Persist, - #[cfg(test)] - FailCacheReloadAt(TargetCacheReloadStage), - #[cfg(test)] - FailEmptyReplacementAt(crate::TargetEmptyReplacementFailure), -} - #[cfg(test)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum TargetCacheReloadStage { - IdMapping, - XtreamStorage, -} - -/// Scoped persistence behavior derived from technically successful forced clusters. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct InputPlaylistPersistOptions { - pub accepted_empty_clusters: ClusterFlags, -} - -impl Default for InputPlaylistPersistOptions { - fn default() -> Self { Self { accepted_empty_clusters: ClusterFlags::empty() } } -} - -/// Empty Library contribution proven by successful input jobs, before target transforms. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub enum LibraryEmptyPublication { - #[default] - None, - /// Empty Library contribution without permission to clear the complete target/output. - Contribution, - /// All inputs are ready, non-Library inputs are populated, and no forced-empty cluster is involved. - /// Configured transformations may remove the remaining entries from a target or individual output. - FilterableContribution, - /// All required inputs are empty Library catalogs, or the proven complete inputs - /// have been reduced to an empty target/output by the configured transformations. - CompleteTarget, -} - -impl LibraryEmptyPublication { - #[must_use] - pub const fn replaces_empty_target(self) -> bool { matches!(self, Self::CompleteTarget) } - - /// Resolve the input-level authorization against the actual prepared target/output. - /// A plain contribution never authorizes a fully empty result from a foreign input. - #[must_use] - pub fn for_filtered_playlist(self, playlist: &[PlaylistGroup]) -> Self { - match self { - Self::FilterableContribution if !playlist_has_items(playlist) => Self::CompleteTarget, - Self::None | Self::Contribution | Self::FilterableContribution | Self::CompleteTarget => self, - } - } - - fn replacement_clusters(self) -> ClusterFlags { - match self { - Self::None => ClusterFlags::empty(), - Self::Contribution | Self::FilterableContribution => ClusterFlags::Vod | ClusterFlags::Series, - Self::CompleteTarget => ClusterFlags::all(), - } - } -} - -/// Output replacement authority derived only from a complete curation run. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PlaylistPublicationPlan { - Ordinary, - CompleteCuration { - intentionally_empty_base_vod: bool, - intentionally_empty_base_series: bool, - intentionally_empty_xtream_vod: bool, - intentionally_empty_xtream_series: bool, - }, -} - -impl PlaylistPublicationPlan { - #[must_use] - pub const fn complete_curation(curated_catalog: bool, xtream_base_suppressed: bool) -> Self { - Self::complete_curation_with_filter(curated_catalog, xtream_base_suppressed, false) - } - - #[must_use] - pub const fn complete_curation_with_filter( - curated_catalog: bool, - xtream_base_suppressed: bool, - appearance_filter_configured: bool, - ) -> Self { - Self::CompleteCuration { - intentionally_empty_base_vod: curated_catalog || appearance_filter_configured, - intentionally_empty_base_series: curated_catalog || appearance_filter_configured, - intentionally_empty_xtream_vod: curated_catalog || xtream_base_suppressed || appearance_filter_configured, - intentionally_empty_xtream_series: curated_catalog - || xtream_base_suppressed - || appearance_filter_configured, - } - } - - #[must_use] - pub const fn with_output_filter(self, output_filter_configured: bool) -> Self { - if !output_filter_configured { - return self; - } - match self { - Self::Ordinary => Self::Ordinary, - Self::CompleteCuration { .. } => Self::CompleteCuration { - intentionally_empty_base_vod: true, - intentionally_empty_base_series: true, - intentionally_empty_xtream_vod: true, - intentionally_empty_xtream_series: true, - }, - } - } - - #[must_use] - pub const fn allows_empty_xtream_cluster(self, cluster: XtreamCluster) -> bool { - matches!( - (self, cluster), - (Self::CompleteCuration { intentionally_empty_xtream_vod: true, .. }, XtreamCluster::Video) - | (Self::CompleteCuration { intentionally_empty_xtream_series: true, .. }, XtreamCluster::Series) - ) - } - - #[must_use] - pub const fn allows_empty_base_output(self) -> bool { - matches!( - self, - Self::CompleteCuration { intentionally_empty_base_vod: true, .. } - | Self::CompleteCuration { intentionally_empty_base_series: true, .. } - ) - } - - #[must_use] - pub const fn allows_any_empty_output(self) -> bool { - self.allows_empty_base_output() - || self.allows_empty_xtream_cluster(XtreamCluster::Video) - || self.allows_empty_xtream_cluster(XtreamCluster::Series) - } - - fn replacement_clusters(self) -> ClusterFlags { - let mut clusters = ClusterFlags::empty(); - if self.allows_empty_xtream_cluster(XtreamCluster::Video) { - clusters |= ClusterFlags::Vod; - } - if self.allows_empty_xtream_cluster(XtreamCluster::Series) { - clusters |= ClusterFlags::Series; - } - clusters - } -} - -/// Target persistence behavior authorized by successful input and curation results, not by Quality inference. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct TargetPlaylistPersistOptions { - pub accepted_empty_clusters: ClusterFlags, - pub library_empty: LibraryEmptyPublication, - pub publication_plan: PlaylistPublicationPlan, -} - -impl Default for TargetPlaylistPersistOptions { - fn default() -> Self { - Self { - accepted_empty_clusters: ClusterFlags::empty(), - library_empty: LibraryEmptyPublication::None, - publication_plan: PlaylistPublicationPlan::Ordinary, - } - } -} - -fn validate_target_playlist_persistence( - target: &ConfigTarget, - playlist_is_empty: bool, - options: TargetPlaylistPersistOptions, -) -> Result<(), TuliproxError> { - if !playlist_is_empty - || options.library_empty.replaces_empty_target() - || options.publication_plan.allows_any_empty_output() - { - return Ok(()); - } - if options.accepted_empty_clusters.is_empty() { - return Err(TuliproxError::RepositoryPlaylist(format!( - "Refusing to persist empty playlist for target '{}'; existing data was retained", - target.name - ))); - } - if target.output.iter().any(|output| !matches!(output, TargetOutput::Xtream(_))) { - return Err(TuliproxError::RepositoryPlaylist(format!( - "Target '{}' has non-Xtream outputs that cannot safely publish a fully empty forced result; existing data was retained", - target.name - ))); - } - Ok(()) -} - -fn prepare_target_output_playlists( - target: &ConfigTarget, - base_playlist: &[PlaylistGroup], - xtream_playlist: Option<&[PlaylistGroup]>, -) -> Vec>> { - target - .output - .iter() - .map(|output| { - let source_playlist = match output { - TargetOutput::Xtream(_) => xtream_playlist.unwrap_or(base_playlist), - _ => base_playlist, - }; - output.filter().map(|filter| apply_filter_to_playlist(source_playlist, filter)) - }) - .collect() -} - -fn validate_force_empty_output_filters( - target: &ConfigTarget, - prepared_outputs: &[Option>], - options: TargetPlaylistPersistOptions, -) -> Result<(), TuliproxError> { - if options.accepted_empty_clusters.is_empty() { - return Ok(()); - } - - debug_assert_eq!(target.output.len(), prepared_outputs.len()); - for (output, prepared_output) in target.output.iter().zip(prepared_outputs) { - let output_name = match output { - TargetOutput::M3u(_) => "M3U", - TargetOutput::Strm(_) => "STRM", - TargetOutput::Xtream(_) | TargetOutput::HdHomeRun(_) => continue, - }; - let Some(filtered_playlist) = prepared_output else { - continue; - }; - let curation_allows_empty = - options.publication_plan.with_output_filter(output.filter().is_some()).allows_empty_base_output(); - if !playlist_has_items(filtered_playlist) && !curation_allows_empty { - return Err(TuliproxError::RepositoryPlaylist(format!( - "Refusing to publish force-empty {output_name} output for target '{}' after its output filter; existing data was retained", - target.name - ))); - } - } - - Ok(()) -} - -async fn persist_xtream_target_playlist( - app_config: &Arc, - target: &ConfigTarget, - playlist: &mut [PlaylistGroup], - accepted_empty_clusters: ClusterFlags, - mode: TargetPersistenceMode, -) -> Result<(), TuliproxError> { - #[cfg(test)] - if let TargetPersistenceMode::FailEmptyReplacementAt(failure) = mode { - return crate::xtream_write_playlist_with_injected_empty_replacement_failure( - app_config, - target, - playlist, - accepted_empty_clusters, - failure, - ) - .await; - } - #[cfg(not(test))] - let _ = mode; - - xtream_write_playlist(app_config, target, playlist, accepted_empty_clusters).await -} - -pub struct ProviderEpisodeKey { - pub provider_id: u32, - pub virtual_id: u32, -} - -fn normalize_target_playlist_epg_ids(playlist: &mut [PlaylistGroup], target_options: Option<&ConfigTargetOptions>) { - if !target_options.is_some_and(ConfigTargetOptions::lowercase_epg_ids) { - return; - } - - for group in playlist { - for channel in &mut group.channels { - let Some(epg_id) = channel.header.epg_channel_id.as_mut() else { - continue; - }; - if epg_id.is_empty() { - continue; - } - - *epg_id = fold_epg_id_arc(epg_id); - } - } -} - -fn prepare_target_playlist_for_persistence( - playlist: &mut [PlaylistGroup], - target: &ConfigTarget, - target_id_mapping: &mut TargetIdMapping, -) { - let mut source_ordinal: u32 = 0; - for group in playlist.iter_mut() { - for channel in &mut group.channels { - let header = &mut channel.header; - source_ordinal += 1; - header.source_ordinal = source_ordinal; - let provider_id = header.get_provider_id().unwrap_or_default(); - if provider_id == 0 { - header.item_type = match (is_hls_url(&header.url), header.item_type) { - (true, _) => PlaylistItemType::LiveHls, - (false, PlaylistItemType::Live) => { - if is_dash_url(&header.url) { - PlaylistItemType::LiveDash - } else { - PlaylistItemType::LiveUnknown - } - } - _ => header.item_type, - }; - } - - let uuid = header.get_uuid(); - let item_type = header.item_type; - let parent_virtual_id = if item_type.is_series() { - target_id_mapping.get_parent_virtual_id_by_uuid(uuid).unwrap_or_default() - } else { - VirtualId::default() - }; - header.virtual_id = - target_id_mapping.get_and_update_virtual_id(uuid, provider_id, item_type, parent_virtual_id); - } - } - - rewrite_series_episode_parent_virtual_ids(playlist, target_id_mapping); - - let mut local_library_series = HashMap::, Vec>::new(); - let mut provider_series = HashMap::, Vec>::new(); - let mut media_server_series = HashMap::, Vec>::new(); - for group in playlist.iter_mut() { - for channel in &mut group.channels { - let header = &mut channel.header; - let item_type = header.item_type; - if item_type == PlaylistItemType::LocalSeries { - assign_local_series_info_episode_key(&mut local_library_series, header, item_type); - } else if is_media_server_series_episode_header(header) { - assign_media_server_series_info_episode(&mut media_server_series, header); - } else if item_type == PlaylistItemType::Series { - assign_provider_series_info_episode_key(&mut provider_series, header, item_type); - } - } - } - - materialize_media_server_series_info_episodes(playlist, &media_server_series); - rewrite_series_info_episode_virtual_id(playlist, &local_library_series, &provider_series); - normalize_target_playlist_epg_ids(playlist, target.options.as_ref()); -} - -pub async fn persist_playlist( - app_config: &Arc, - playlist: &mut [PlaylistGroup], - epg: Option<&Epg>, - target: &ConfigTarget, - playlist_state: Option<&Arc>, - options: TargetPlaylistPersistOptions, -) -> Result<(), Vec> { - persist_playlist_with_mode( - app_config, - playlist, - epg, - target, - playlist_state, - options, - TargetPersistenceMode::Persist, - ) - .await -} - -pub async fn persist_playlist_views( - app_config: &Arc, - base_playlist: &mut [PlaylistGroup], - xtream_playlist: Option<&mut [PlaylistGroup]>, - epg: Option<&Epg>, - target: &ConfigTarget, - playlist_state: Option<&Arc>, - options: TargetPlaylistPersistOptions, -) -> Result<(), Vec> { - persist_playlist_views_with_mode( - app_config, - base_playlist, - xtream_playlist, - epg, - target, - playlist_state, - options, - TargetPersistenceMode::Persist, - ) - .await -} - -async fn persist_playlist_with_mode( - app_config: &Arc, - playlist: &mut [PlaylistGroup], - epg: Option<&Epg>, - target: &ConfigTarget, - playlist_state: Option<&Arc>, - options: TargetPlaylistPersistOptions, - persistence_mode: TargetPersistenceMode, -) -> Result<(), Vec> { - persist_playlist_views_with_mode(app_config, playlist, None, epg, target, playlist_state, options, persistence_mode) - .await -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -async fn persist_playlist_views_with_mode( - app_config: &Arc, - base_playlist: &mut [PlaylistGroup], - mut xtream_playlist: Option<&mut [PlaylistGroup]>, - epg: Option<&Epg>, - target: &ConfigTarget, - playlist_state: Option<&Arc>, - options: TargetPlaylistPersistOptions, - persistence_mode: TargetPersistenceMode, -) -> Result<(), Vec> { - let playlist_is_empty = !playlist_has_items(base_playlist) - && xtream_playlist.as_deref().is_none_or(|playlist| !playlist_has_items(playlist)); - if let Err(error) = validate_target_playlist_persistence(target, playlist_is_empty, options) { - return Err(vec![error]); - } - let mut errors = vec![]; - let config = &app_config.config.load(); - let target_path = match ensure_target_storage_path(config, &target.name).await { - Ok(path) => path, - Err(err) => return Err(vec![err]), - }; - - let (mut target_id_mapping, file_lock) = - match get_target_id_mapping(app_config, &target_path, target.use_memory_cache).await { - Ok(result) => result, - Err(err) => return Err(vec![err]), - }; - - prepare_target_playlist_for_persistence(base_playlist, target, &mut target_id_mapping); - if let Some(xtream_view) = xtream_playlist.as_deref_mut() { - prepare_target_playlist_for_persistence(xtream_view, target, &mut target_id_mapping); - } - - let mut prepared_outputs = prepare_target_output_playlists(target, base_playlist, xtream_playlist.as_deref()); - if let Err(error) = validate_force_empty_output_filters(target, &prepared_outputs, options) { - target_id_mapping.discard_unpersisted_changes(); - drop(target_id_mapping); - drop(file_lock); - return Err(vec![error]); - } - - for (output, prepared_output) in target.output.iter().zip(&mut prepared_outputs) { - let output_publication_plan = options.publication_plan.with_output_filter(output.filter().is_some()); - let source_playlist = match output { - TargetOutput::Xtream(_) => xtream_playlist.as_deref_mut().unwrap_or(base_playlist), - _ => &mut *base_playlist, - }; - let pl: &mut [PlaylistGroup] = if let Some(filtered_playlist) = prepared_output.as_mut() { - filtered_playlist.as_mut_slice() - } else { - source_playlist - }; - let library_empty = options.library_empty.for_filtered_playlist(pl); - let curation_empty_clusters = output_publication_plan.replacement_clusters(); - let allows_empty_base = - library_empty.replaces_empty_target() || output_publication_plan.allows_empty_base_output(); - - let result = match output { - TargetOutput::Xtream(_xtream_output) => { - persist_xtream_target_playlist( - app_config, - target, - pl, - options.accepted_empty_clusters | library_empty.replacement_clusters() | curation_empty_clusters, - persistence_mode, - ) - .await - } - TargetOutput::M3u(m3u_output) => { - m3u_write_playlist(app_config, target, m3u_output, &target_path, pl, allows_empty_base).await - } - TargetOutput::Strm(strm_output) => { - write_strm_playlist(app_config, target, strm_output, pl, allows_empty_base).await - } - TargetOutput::HdHomeRun(_hdhomerun_output) => Ok(()), - }; - - match result { - Ok(()) => { - let allows_empty_output = match output { - TargetOutput::Xtream(_) => { - !curation_empty_clusters.is_empty() - || !library_empty.replacement_clusters().is_empty() - || !options.accepted_empty_clusters.is_empty() - } - _ => allows_empty_base, - }; - if !pl.is_empty() || allows_empty_output { - let epg_pl: &[PlaylistGroup] = pl; - if let Err(err) = - epg_write_for_target(config, target, &target_path, epg, output, Some(epg_pl)).await - { - errors.push(err); - } - } - } - Err(err) => errors.push(err), - } - } - - if let Err(err) = target_id_mapping.persist() { - errors.push(TuliproxError::Config(format!("{err}"))); - } - // Keep lock until all outputs are persisted to prevent concurrent writers - // from interleaving mapping and output state for the same target. - // We must release it before loading caches below (which may acquire read locks). - drop(target_id_mapping); - drop(file_lock); - - if errors.is_empty() && target.use_memory_cache { - match playlist_state { - Some(playlist_storage) => { - match load_target_memory_cache_snapshot(app_config, target, persistence_mode).await { - Ok(storage) => playlist_storage.replace_target(&target.name, storage).await, - Err(error) => errors.push(error), - } - } - None => errors.push(TuliproxError::RepositoryPlaylist(format!( - "Target '{}' was persisted but its configured memory cache is unavailable", - target.name - ))), - } - } - - if errors.is_empty() { - Ok(()) - } else { - Err(errors) - } -} - -fn assign_local_series_info_episode_key( - local_library_series: &mut HashMap, Vec>, - header: &mut PlaylistItemHeader, - item_type: PlaylistItemType, -) { - // we need to rewrite local series info with the new virtual ids - if item_type == PlaylistItemType::LocalSeries { - local_library_series - .entry(header.parent_code.clone()) - .or_default() - .push(LocalEpisodeKey { path: header.url.clone(), virtual_id: header.virtual_id.get() }); - } -} - -fn assign_provider_series_info_episode_key( - provider_series: &mut HashMap, Vec>, - header: &mut PlaylistItemHeader, - item_type: PlaylistItemType, -) { - // we need to rewrite local series info with the new virtual ids - if item_type == PlaylistItemType::Series { - provider_series.entry(header.parent_code.clone()).or_default().push(ProviderEpisodeKey { - provider_id: header.get_provider_id().unwrap_or_default(), - virtual_id: header.virtual_id.get(), - }); - } -} - -fn is_media_server_item_header(header: &PlaylistItemHeader) -> bool { - header.id.starts_with("media-server:") || header.url.starts_with("media-server://") -} - -fn is_media_server_series_info_header(header: &PlaylistItemHeader) -> bool { - header.item_type == PlaylistItemType::SeriesInfo && is_media_server_item_header(header) -} - -fn is_media_server_series_episode_header(header: &PlaylistItemHeader) -> bool { - header.item_type == PlaylistItemType::Series - && !header.parent_code.is_empty() - && is_media_server_item_header(header) -} - -fn assign_media_server_series_info_episode( - media_server_series: &mut HashMap, Vec>, - header: &PlaylistItemHeader, -) { - if let Some(episode) = media_server_series_episode_detail(header) { - media_server_series.entry(header.parent_code.clone()).or_default().push(episode); - } -} - -fn media_server_series_episode_detail(header: &PlaylistItemHeader) -> Option { - if !is_media_server_series_episode_header(header) { - return None; - } - - let Some(StreamProperties::Episode(episode)) = header.additional_properties.as_ref() else { - return None; - }; - let episode = episode.as_ref(); - - Some(SeriesStreamDetailEpisodeProperties { - id: header.virtual_id.get(), - episode_num: episode.episode, - season: episode.season, - title: non_blank_arc(&header.title) - .unwrap_or_else(|| non_blank_arc(&header.name).unwrap_or_else(|| "Episode".intern())), - container_extension: episode.container_extension.clone(), - custom_sid: None, - added: episode.added.clone().unwrap_or_else(|| "".intern()), - direct_source: "".intern(), - tmdb: episode.tmdb, - release_date: episode.release_date.clone().unwrap_or_else(|| "".intern()), - series_release_date: episode.series_release_date.clone(), - plot: episode.plot.clone(), - crew: None, - duration_secs: 0, - duration: "".intern(), - movie_image: episode.movie_image.clone(), - bitrate: 0, - rating: None, - video: episode.video.clone(), - audio: episode.audio.clone(), - }) -} - -fn non_blank_arc(value: &Arc) -> Option> { (!value.trim().is_empty()).then(|| Arc::clone(value)) } - -fn source_series_info_episode_key(channel: &PlaylistItem) -> Option> { - match channel.header.item_type { - PlaylistItemType::SeriesInfo => Some(channel.get_uuid().intern()), - PlaylistItemType::LocalSeriesInfo => Some(channel.header.id.clone()), - _ => None, - } -} - -fn header_uuid_episode_key(header: &PlaylistItemHeader) -> Option> { - (header.uuid != UUIDType::default()).then(|| header.uuid.intern()) -} - -fn push_unique_key(keys: &mut Vec>, key: Arc) { - if !key.is_empty() && !keys.iter().any(|existing| existing.as_ref() == key.as_ref()) { - keys.push(key); - } -} - -fn series_info_episode_lookup_keys(channel: &PlaylistItem) -> Vec> { - let mut keys = Vec::with_capacity(2); - if let Some(alias_key) = header_uuid_episode_key(&channel.header) { - push_unique_key(&mut keys, alias_key); - } - if let Some(source_key) = source_series_info_episode_key(channel) { - push_unique_key(&mut keys, source_key); - } - keys -} - -fn series_info_parent_keys(channel: &PlaylistItem) -> Vec<(Arc, bool)> { - let Some(source_key) = source_series_info_episode_key(channel) else { return Vec::new() }; - let Some(alias_key) = header_uuid_episode_key(&channel.header) else { return vec![(source_key, true)] }; - if alias_key.as_ref() == source_key.as_ref() { - vec![(source_key, true)] - } else { - vec![(alias_key, true), (source_key, false)] - } -} - -#[allow(clippy::implicit_hasher)] -fn materialize_media_server_series_info_episodes( - playlist: &mut [PlaylistGroup], - media_server_series: &HashMap, Vec>, -) { - if media_server_series.is_empty() { - return; - } - - for group in playlist.iter_mut() { - for channel in &mut group.channels { - if !is_media_server_series_info_header(&channel.header) { - continue; - } - let lookup_keys = series_info_episode_lookup_keys(channel); - let Some(episodes) = lookup_keys.iter().find_map(|key| media_server_series.get(key)) else { continue }; - let Some(StreamProperties::Series(series)) = channel.header.additional_properties.as_mut() else { - continue; - }; - let details = series.details.get_or_insert(SeriesStreamDetailProperties { - year: None, - seasons: None, - episodes: None, - }); - let mut episodes = episodes.clone(); - episodes.sort_by_key(|episode| (episode.season, episode.episode_num, episode.id)); - details.episodes = Some(episodes); - } - } -} - -#[allow(clippy::implicit_hasher)] -fn rewrite_local_series_info_episode_virtual_id( - pli: &mut PlaylistItem, - local_library_series: &HashMap, Vec>, -) { - // local_library_series keys are the Series UUID or a category alias UUID. - // For LocalSeriesInfo items, header.id is the source Series UUID; category - // aliases use header.uuid so cloned episode rows can point at the alias. - let lookup_keys = if pli.header.item_type == PlaylistItemType::LocalSeries { - vec![pli.header.parent_code.clone()] - } else { - series_info_episode_lookup_keys(pli) - }; - - if let Some(episode_keys) = lookup_keys.iter().find_map(|key| local_library_series.get(key)) { - if let Some(StreamProperties::Series(series)) = pli.header.additional_properties.as_mut() { - if let Some(episodes) = series.details.as_mut().and_then(|d| d.episodes.as_mut()) { - for episode in episodes.iter_mut() { - for episode_key in episode_keys { - if episode.direct_source == episode_key.path { - episode.id = episode_key.virtual_id; - break; - } - } - } - } - } - } -} - -#[allow(clippy::implicit_hasher)] -pub fn rewrite_provider_series_info_episode_virtual_id

( - pli: &mut P, - provider_series: &HashMap, Vec>, -) where - P: PlaylistEntry, -{ - let lookup_key = pli.get_uuid().intern(); - if let Some(episode_keys) = provider_series.get(&lookup_key) { - if let Some(properties) = pli.get_additional_properties_mut() { - apply_provider_episode_keys(properties, episode_keys); - } - } -} - -#[allow(clippy::implicit_hasher)] -fn rewrite_provider_playlist_item_series_info_episode_virtual_id( - pli: &mut PlaylistItem, - provider_series: &HashMap, Vec>, -) { - let lookup_keys = series_info_episode_lookup_keys(pli); - if let Some(episode_keys) = lookup_keys.iter().find_map(|key| provider_series.get(key)) { - if let Some(properties) = pli.get_additional_properties_mut() { - apply_provider_episode_keys(properties, episode_keys); - } - } -} - -fn apply_provider_episode_keys(properties: &mut StreamProperties, episode_keys: &[ProviderEpisodeKey]) { - if let StreamProperties::Series(series) = properties { - if let Some(episodes) = series.details.as_mut().and_then(|d| d.episodes.as_mut()) { - for episode in episodes.iter_mut() { - for episode_key in episode_keys { - if episode.id == episode_key.provider_id { - episode.id = episode_key.virtual_id; - break; - } - } - } - } - } -} - -fn rewrite_series_info_episode_virtual_id( - playlist: &mut [PlaylistGroup], - local_library_series: &HashMap, Vec>, - provider_series: &HashMap, Vec>, -) { - if local_library_series.is_empty() && provider_series.is_empty() { - return; - } - for group in playlist.iter_mut() { - for channel in &mut group.channels { - let item_type = channel.header.item_type; - if item_type == PlaylistItemType::SeriesInfo { - rewrite_provider_playlist_item_series_info_episode_virtual_id(channel, provider_series); - } else if item_type == PlaylistItemType::LocalSeriesInfo { - rewrite_local_series_info_episode_virtual_id(channel, local_library_series); - } else if item_type == PlaylistItemType::LocalSeries { - channel.header.parent_code = "".intern(); - } - } - } -} - -fn rewrite_series_episode_parent_virtual_ids(playlist: &mut [PlaylistGroup], target_id_mapping: &mut TargetIdMapping) { - let mut series_parent_virtual_ids = HashMap::, u32>::new(); - - for group in playlist.iter() { - for channel in &group.channels { - for (parent_key, overwrite) in series_info_parent_keys(channel) { - if overwrite { - series_parent_virtual_ids.insert(parent_key, channel.header.virtual_id.get()); - } else { - series_parent_virtual_ids.entry(parent_key).or_insert(channel.header.virtual_id.get()); - } - } - } - } - - if series_parent_virtual_ids.is_empty() { - return; - } - - for group in playlist.iter_mut() { - for channel in &mut group.channels { - let header = &mut channel.header; - if header.item_type.is_series() { - if let Some(parent_virtual_id) = series_parent_virtual_ids.get(&header.parent_code) { - let provider_id = header.get_provider_id().unwrap_or_default(); - let item_type = header.item_type; - let uuid = header.get_uuid(); - header.virtual_id = target_id_mapping.get_and_update_virtual_id( - uuid, - provider_id, - item_type, - VirtualId::new(*parent_virtual_id), - ); - } - } - } - } -} - -pub async fn get_target_id_mapping( - cfg: &AppConfig, - target_path: &Path, - use_memory_cache: bool, -) -> Result<(TargetIdMapping, utils::FileWriteGuard), TuliproxError> { - let target_id_mapping_file = get_target_id_mapping_file(target_path); - let file_lock = cfg.file_locks.write_lock(&target_id_mapping_file).await; - let mapping_path = target_id_mapping_file.clone(); - let mapping = - tokio::task::spawn_blocking(move || TargetIdMapping::new(&mapping_path, use_memory_cache)).await.map_err( - |err| TuliproxError::Config(format!("spawn_blocking failed while creating TargetIdMapping: {err}")), - )??; - - Ok((mapping, file_lock)) -} - -async fn load_target_id_mapping_as_tree( - app_config: &AppConfig, - target_path: &Path, - target: &ConfigTarget, -) -> Result, TuliproxError> { - let target_id_mapping_file = get_target_id_mapping_file(target_path); - let _file_lock = app_config.file_locks.read_lock(&target_id_mapping_file).await; - - // Move B+Tree load to spawn_blocking to avoid blocking tokio runtime - let path_clone = target_id_mapping_file.clone(); - let target_name = target.name.clone(); - tokio::task::spawn_blocking(move || BPlusTree::::load(&path_clone)) - .await - .map_err(|e| TuliproxError::Config(format!("Blocking task failed: {e}")))? - .map_err(|err| TuliproxError::Config(format!("Could not find path for target {target_name} err:{err}"))) -} - -async fn load_xtream_playlist_as_tree( - app_config: &AppConfig, - storage_path: &Path, - cluster: XtreamCluster, -) -> Result, TuliproxError> { - let xtream_path = xtream_get_file_path(storage_path, cluster); - let file_lock = app_config.file_locks.read_lock(&xtream_path).await; - // Move B+Tree query and iteration to spawn_blocking to avoid blocking tokio runtime - let path_clone = xtream_path.clone(); - match tokio::task::spawn_blocking(move || { - let _guard = file_lock; - BPlusTree::::load(&path_clone) - }) - .await - { - Ok(Ok(tree)) => Ok(tree), - Ok(Err(err)) if err.kind() == std::io::ErrorKind::NotFound => { - debug!("No xtream {cluster} storage at {}, serving empty playlist", xtream_path.display()); - Ok(BPlusTree::new()) - } - Ok(Err(err)) => Err(TuliproxError::RepositoryXtream(format!( - "Failed to load xtream {cluster} storage {}: {err}", - xtream_path.display() - ))), - Err(join_err) => Err(TuliproxError::RepositoryXtream(format!( - "Failed to join xtream {cluster} storage load task {}: {join_err}", - xtream_path.display() - ))), - } -} - -async fn load_id_mapping_target_storage( - app_config: &AppConfig, - target: &ConfigTarget, -) -> Result, TuliproxError> { - let config = app_config.config.load(); - let target_path = get_target_storage_path(&config, target.name.as_str()) - .ok_or_else(|| TuliproxError::Config(format!("Could not find path for target {}", target.name)))?; - - load_target_id_mapping_as_tree(app_config, &target_path, target).await -} - -pub async fn load_xtream_target_storage( - app_config: &AppConfig, - target: &ConfigTarget, -) -> Result { - let config = app_config.config.load(); - - let storage_path = xtream_get_storage_path(&config, target.name.as_str()).ok_or_else(|| { - TuliproxError::Config(format!("Could not find path for target {} xtream output", target.name)) - })?; - - let live_storage = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Live).await?; - let vod_storage = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Video).await?; - let series_storage = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Series).await?; - - Ok(PlaylistXtreamStorage { live: live_storage, vod: vod_storage, series: series_storage }) -} - -pub async fn load_m3u_target_storage( - app_config: &AppConfig, - target: &ConfigTarget, -) -> Result { - let config = app_config.config.load(); - let target_path = get_target_storage_path(&config, target.name.as_str()) - .ok_or_else(|| TuliproxError::Config(format!("Could not find path for target {}", target.name)))?; - - let m3u_path = m3u_get_file_path_for_db(&target_path); - let file_lock = app_config.file_locks.read_lock(&m3u_path).await; - - let path_clone = m3u_path.clone(); - match tokio::task::spawn_blocking(move || { - let _guard = file_lock; - BPlusTree::::load(&path_clone) - }) - .await - { - Ok(Ok(tree)) => Ok(tree), - Ok(Err(err)) if err.kind() == std::io::ErrorKind::NotFound => { - debug!("No m3u storage at {}, serving empty playlist", m3u_path.display()); - Ok(BPlusTree::new()) - } - Ok(Err(err)) => { - Err(TuliproxError::RepositoryM3u(format!("Failed to load m3u storage {}: {err}", m3u_path.display()))) - } - Err(join_err) => Err(TuliproxError::RepositoryM3u(format!( - "Failed to join m3u storage load task {}: {join_err}", - m3u_path.display() - ))), - } -} - -fn target_cache_reload_error(target: &ConfigTarget, component: &str, error: impl std::fmt::Display) -> TuliproxError { - TuliproxError::RepositoryPlaylist(format!( - "Target '{}' was persisted but its {component} could not be reloaded into the memory cache: {error}", - target.name - )) -} - -async fn load_target_memory_cache_snapshot( - app_config: &AppConfig, - target: &ConfigTarget, - mode: TargetPersistenceMode, -) -> Result { - #[cfg(not(test))] - let _ = mode; - - #[cfg(test)] - if matches!(mode, TargetPersistenceMode::FailCacheReloadAt(TargetCacheReloadStage::IdMapping)) { - return Err(target_cache_reload_error(target, "ID mapping", "injected reload failure")); - } - let id_mapping = load_id_mapping_target_storage(app_config, target) - .await - .map_err(|error| target_cache_reload_error(target, "ID mapping", error))?; - - let xtream = if target.output.iter().any(|output| matches!(output, TargetOutput::Xtream(_))) { - #[cfg(test)] - if matches!(mode, TargetPersistenceMode::FailCacheReloadAt(TargetCacheReloadStage::XtreamStorage)) { - return Err(target_cache_reload_error(target, "Xtream storage", "injected reload failure")); - } - Some( - load_xtream_target_storage(app_config, target) - .await - .map_err(|error| target_cache_reload_error(target, "Xtream storage", error))?, - ) - } else { - None - }; - - let m3u = if target.output.iter().any(|output| matches!(output, TargetOutput::M3u(_))) { - Some( - load_m3u_target_storage(app_config, target) - .await - .map_err(|error| target_cache_reload_error(target, "M3U storage", error))?, - ) - } else { - None - }; - - Ok(TargetPlaylistStorage { xtream, m3u, id_mapping: Some(id_mapping) }) -} - -async fn publish_all_cluster_catalogs( - app_config: &AppConfig, - storage_path: &Path, - input_name: &str, - persisted_playlist: Vec, -) -> (Vec, Option) { - let mut live_groups = Vec::new(); - let mut vod_groups = Vec::new(); - let mut series_groups = Vec::new(); - for group in &persisted_playlist { - match group.xtream_cluster { - XtreamCluster::Live => live_groups.push(group.title.to_string()), - XtreamCluster::Video => vod_groups.push(group.title.to_string()), - XtreamCluster::Series => series_groups.push(group.title.to_string()), - } - } - for (cluster, groups) in - [(XtreamCluster::Live, live_groups), (XtreamCluster::Video, vod_groups), (XtreamCluster::Series, series_groups)] - { - if let Err(publish_err) = - crate::publish_raw_group_catalog(storage_path, input_name, cluster, groups, &app_config.file_locks).await - { - warn!( - "Playlist data for input '{input_name}' was persisted, but publishing its raw group catalog for cluster {cluster:?} failed: {publish_err}" - ); - } - } - (persisted_playlist, None) -} - -pub async fn persist_input_playlist( - app_config: &Arc, - input: &ConfigInput, - playlist: Vec, -) -> (Vec, Option) { - persist_input_playlist_with_options(app_config, input, playlist, InputPlaylistPersistOptions::default()).await -} - -pub async fn persist_input_playlist_with_options( - app_config: &Arc, - input: &ConfigInput, - mut playlist: Vec, - options: InputPlaylistPersistOptions, -) -> (Vec, Option) { - let persistence = input.get_download_input_type().persistence(); - let accepts_empty = persistence == InputPersistence::Library - || (!options.accepted_empty_clusters.is_empty() - && matches!(persistence, InputPersistence::Xtream | InputPersistence::Stalker)); - if !playlist_has_items(&playlist) && !accepts_empty { - let empty_error = TuliproxError::RepositoryPlaylist(format!( - "Refusing to persist empty playlist for input '{}'; existing data was retained", - input.name - )); - warn!("{empty_error}"); - return match load_input_playlist(app_config, input, None).await { - Ok(mut previous) => { - if previous.is_empty() { - (playlist, Some(empty_error)) - } else { - (previous.take_groups(), Some(empty_error)) - } - } - Err(load_err) => ( - playlist, - Some(TuliproxError::RepositoryPlaylist(format!( - "{empty_error}; failed to load the retained input playlist: {load_err}" - ))), - ), - }; - } - if persistence == InputPersistence::Stalker { - // The Stalker processor (`processor::stalker::download_stalker_playlist`) - // is the single writer of the per-cluster B+Tree and raw group catalogs. - // Re-encoding the `PlaylistItem` runtime projection back into `StalkerPlaylistItem` - // would destroy the canonical `cmd`/`playback_descriptor`/capability - // flags the processor just persisted — including the field the - // runtime 4xx-re-resolve hook relies on. The disk layout is - // already in sync; nothing to do here. - return (playlist, None); - } - playlist.iter_mut().for_each(PlaylistGroup::on_load); - let cfg = app_config.config.load(); - let storage_path = match get_input_storage_path(&input.name, &cfg.storage_dir).await { - Ok(storage_path) => storage_path, - Err(err) => { - return ( - playlist, - Some(TuliproxError::Config(format!( - "Error creating input storage directory for input '{}' failed: {err}", - input.name - ))), - ); - } - }; - - let (persisted_playlist, err) = match persistence { - InputPersistence::Xtream => { - crate::persist_input_xtream_playlist_with_empty_replacements( - app_config, - &storage_path, - playlist, - options.accepted_empty_clusters, - ) - .await - } - - InputPersistence::M3u => { - // Persist M3U - let file_path = get_input_m3u_playlist_file_path(&storage_path, &input.name); - if let Err(err) = persist_input_m3u_playlist(app_config, &file_path, &playlist).await { - return (playlist, Some(err)); - } - (playlist, None) - } - InputPersistence::Library => { - // Persist local library playlist - let file_path = get_input_local_library_playlist_file_path(&storage_path, &input.name); - let (playlist, result) = persist_input_library_playlist(app_config, &file_path, playlist).await; - if let Err(err) = result { - return (playlist, Some(err)); - } - (playlist, None) - } - InputPersistence::MediaServer => { - let file_path = get_input_media_server_playlist_file_path(&storage_path, &input.name); - let (playlist, result) = persist_input_media_server_playlist(app_config, &file_path, playlist).await; - if let Err(err) = result { - return (playlist, Some(err)); - } - (playlist, None) - } - InputPersistence::Stalker => unreachable!("handled above"), - }; - - if err.is_none() { - return publish_all_cluster_catalogs(app_config, &storage_path, &input.name, persisted_playlist).await; - } - - (persisted_playlist, err) -} - -pub async fn load_input_playlist( - app_config: &Arc, - input: &ConfigInput, - clusters: Option<&[XtreamCluster]>, -) -> Result { - let cfg = app_config.config.load(); - let storage_path = get_input_storage_path(&input.name, &cfg.storage_dir) - .await - .map_err(|e| TuliproxError::Config(format!("Error getting input path: {e}")))?; - let disk_based_processing = cfg.disk_based_processing; - - match input.get_download_input_type().persistence() { - InputPersistence::Xtream => { - let clusters_to_load = clusters.unwrap_or(&XTREAM_CLUSTER); - if disk_based_processing { - let source = - PlaylistSource::xtream_disk(XtreamDiskPlaylistSource::new(app_config, &storage_path).await?); - Ok(PlaylistSource::filtered(source, skipped_clusters(clusters_to_load))) - } else { - let groups = load_input_xtream_playlist(app_config, &storage_path, clusters_to_load).await?; - Ok(MemoryPlaylistSource::new(groups).into_source()) - } - } - InputPersistence::M3u => { - // Load M3U - let file_path = get_input_m3u_playlist_file_path(&storage_path, &input.name); - if disk_based_processing && file_path.exists() { - Ok(PlaylistSource::m3u_disk(M3uDiskPlaylistSource::new(app_config, &file_path).await?)) - } else { - let groups = load_input_m3u_playlist(app_config, &file_path).await?; - Ok(MemoryPlaylistSource::new(groups).into_source()) - } - } - InputPersistence::Library => { - let file_path = get_input_local_library_playlist_file_path(&storage_path, &input.name); - if disk_based_processing && file_path.exists() { - Ok(PlaylistSource::local_library_disk( - LocalLibraryDiskPlaylistSource::new(app_config, &file_path).await?, - )) - } else { - let groups = load_input_local_library_playlist(app_config, &file_path).await?; - Ok(MemoryPlaylistSource::new(groups).into_source()) - } - } - InputPersistence::MediaServer => { - let file_path = get_input_media_server_playlist_file_path(&storage_path, &input.name); - if disk_based_processing && file_path.exists() { - Ok(PlaylistSource::media_server_disk(MediaServerDiskPlaylistSource::new(app_config, &file_path).await?)) - } else { - let groups = load_input_media_server_playlist(app_config, &file_path).await?; - Ok(MemoryPlaylistSource::new(groups).into_source()) - } - } - InputPersistence::Stalker => { - let clusters_to_load = clusters.unwrap_or(&XTREAM_CLUSTER); - let stalker_path = get_stalker_storage_path(&storage_path); - let stalker_config = input.stalker.as_ref().ok_or_else(|| { - TuliproxError::ConfigInput(format!("Stalker input '{}' has no Stalker configuration", input.name)) - })?; - let portal_url = input.resolve_url(&input.url)?.into_owned(); - // A read path: when the published manifest belongs to a different identity the - // input simply has nothing to serve yet, so fall back to an empty manifest - // instead of replacing the publication state of the refresh that owns it. - let (manifest, published) = crate::stalker_generation_repository::readable_active_manifest( - &stalker_path, - stalker_config.identity_fingerprint(&portal_url), - ) - .await?; - if !published { - debug!( - "Stalker input '{}' has no published catalog for its current identity; serving an empty playlist", - input.name - ); - } - if disk_based_processing { - let source = PlaylistSource::stalker_disk( - StalkerDiskPlaylistSource::new(app_config, &stalker_path, Arc::clone(&input.name), manifest) - .await?, - ); - Ok(PlaylistSource::filtered(source, skipped_clusters(clusters_to_load))) - } else { - let groups = load_input_stalker_playlist(app_config, &input.name, clusters_to_load, &manifest).await?; - Ok(MemoryPlaylistSource::new(groups).into_source()) - } - } - } -} - -fn skipped_clusters(clusters_to_load: &[XtreamCluster]) -> HashSet { - XTREAM_CLUSTER.iter().copied().filter(|cluster| !clusters_to_load.contains(cluster)).collect() -} - -pub fn get_input_m3u_playlist_file_path(storage_path: &Path, input_name: &Arc) -> PathBuf { - let sanitized_input_name: String = input_name.chars().map(|c| if c.is_alphanumeric() { c } else { '_' }).collect(); - storage_path.join(format!("m3u_{sanitized_input_name}.{FILE_SUFFIX_DB}")) -} - -pub fn get_input_local_library_playlist_file_path(storage_path: &Path, input_name: &Arc) -> PathBuf { - let sanitized_input_name: String = input_name.chars().map(|c| if c.is_alphanumeric() { c } else { '_' }).collect(); - storage_path.join(format!("lib_{sanitized_input_name}.{FILE_SUFFIX_DB}")) -} - -pub fn get_input_media_server_playlist_file_path(storage_path: &Path, input_name: &Arc) -> PathBuf { - let sanitized_input_name: String = input_name.chars().map(|c| if c.is_alphanumeric() { c } else { '_' }).collect(); - storage_path.join(format!("media_server_{sanitized_input_name}.{FILE_SUFFIX_DB}")) -} - -/// Load a Stalker input's playlist into memory. The on-disk B+Tree is the -/// source of truth; we stream every per-cluster tree and bucket the items -/// by cluster to build the runtime `PlaylistGroup`s. `input_name` seeds the -/// canonical `PlaylistItem::from_stalker` conversion so item identity matches -/// the download path. -pub async fn load_input_stalker_playlist( - app_config: &Arc, - input_name: &str, - clusters: &[XtreamCluster], - manifest: &crate::stalker_generation_repository::StalkerActiveManifest, -) -> Result, TuliproxError> { - let mut groups_map: indexmap::IndexMap<(XtreamCluster, u32), PlaylistGroup> = indexmap::IndexMap::new(); - for &cluster in clusters { - let mut batches = Vec::new(); - match cluster { - XtreamCluster::Live => { - if let Some(files) = manifest.live.as_ref() { - batches.push(crate::stalker_repository::load_stalker_items_at(app_config, &files.data).await?); - } - } - XtreamCluster::Video => { - if let Some(files) = manifest.vod.as_ref() { - batches.push(crate::stalker_repository::load_stalker_items_at(app_config, &files.data).await?); - } - } - XtreamCluster::Series => { - if let Some(files) = manifest.series.as_ref() { - batches.push(crate::stalker_repository::load_stalker_items_at(app_config, &files.roots).await?); - batches.push(crate::stalker_repository::load_stalker_items_at(app_config, &files.episodes).await?); - } - } - } - for batch in batches { - for item in batch { - let category_id = item.category_id; - let playlist_item = PlaylistItem::from_stalker(&item, input_name); - groups_map - .entry((cluster, category_id)) - .or_insert_with(|| PlaylistGroup { - id: category_id, - title: Arc::clone(&playlist_item.header.group), - channels: Vec::new(), - xtream_cluster: cluster, - }) - .channels - .push(playlist_item); - } - } - } - let mut groups: Vec = groups_map.into_values().collect(); - for group in &mut groups { - group.channels.sort_by_key(|item| normalized_source_ordinal(item.header.source_ordinal)); - } - groups.sort_by_key(|group| { - group.channels.first().map_or(u32::MAX, |c| normalized_source_ordinal(c.header.source_ordinal)) - }); - Ok(groups) -} - -pub async fn persist_input_media_server_playlist( - app_config: &Arc, - file_path: &Path, - playlist: Vec, -) -> (Vec, Result<(), TuliproxError>) { - persist_input_library_playlist(app_config, file_path, playlist).await -} - -pub async fn load_input_media_server_playlist( - app_config: &Arc, - file_path: &Path, -) -> Result, TuliproxError> { - load_input_local_library_playlist(app_config, file_path).await -} +mod tests; +mod episode; +mod input; +mod persist; +mod publication; +mod target; #[cfg(test)] -mod tests { - use super::{ - assign_local_series_info_episode_key, assign_media_server_series_info_episode, - get_input_media_server_playlist_file_path, materialize_media_server_series_info_episodes, - normalize_target_playlist_epg_ids, persist_playlist_views, persist_playlist_with_mode, playlist_has_items, - rewrite_local_series_info_episode_virtual_id, rewrite_series_episode_parent_virtual_ids, - rewrite_series_info_episode_virtual_id, skipped_clusters, validate_target_playlist_persistence, - LocalEpisodeKey, PlaylistPublicationPlan, ProviderEpisodeKey, TargetCacheReloadStage, TargetPersistenceMode, - TargetPlaylistPersistOptions, - }; - use crate::{ - get_series_cat_collection_path, get_target_storage_path, get_vod_cat_collection_path, load_m3u_target_storage, - load_xtream_target_storage, strm_get_file_paths, xtream_get_storage_path, BPlusTreeQuery, PlaylistStorageState, - TargetIdMapping, VirtualIdRecord, - }; - use arc_swap::{ArcSwap, ArcSwapOption}; - use shared::{ - foundation::get_filter, - model::{ - ClusterFlags, ConfigPaths, ConfigTargetDto, ConfigTargetOptions, EpgOutputOptions, EpisodeStreamProperties, - M3uPlaylistItem, M3uTargetOutputDto, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemHeader, - PlaylistItemType, ProcessingOrder, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, - SeriesStreamDetailSeasonProperties, SeriesStreamProperties, StreamProperties, StrmExportStyle, - StrmTargetOutputDto, TargetOutputDto, UUIDType, VirtualId, XtreamCluster, XtreamPlaylistItem, - XtreamTargetOutputDto, - }, - utils::{hash_string_as_hex, Internable}, - }; - use std::{collections::HashMap, path::Path, sync::Arc}; - use tuliprox_core::{ - model::{ - ApiProxyConfig, AppConfig, Config, ConfigTarget, CustomStreamResponse, HdHomeRunConfig, M3uTargetOutput, - MediaToolCapabilities, SourcesConfig, StagedFilter, StrmTargetFlagsSet, StrmTargetOutput, - TargetExecutionPlan, TargetOutput, XtreamTargetFlagsSet, XtreamTargetOutput, - }, - utils::{normalize_string_path, FileLockManager}, - }; - - fn target_with_outputs(output: Vec) -> ConfigTarget { - target_with_options("target-empty-guard", output, false) - } - - fn target_with_options(name: &str, output: Vec, use_memory_cache: bool) -> ConfigTarget { - ConfigTarget { - curation: None, - id: 1, - enabled: true, - name: name.to_string(), - options: None, - sort: None, - filter: StagedFilter::default(), - output, - rename: None, - mapping_ids: None, - mapping: Arc::new(ArcSwapOption::new(None)), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: TargetExecutionPlan::default(), - watch: None, - use_memory_cache, - } - } - - fn test_app_config(storage_dir: &Path) -> Arc { - Arc::new(AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config { - storage_dir: storage_dir.to_string_lossy().into_owned(), - ..Config::default() - })), - sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), - hdhomerun: Arc::new(ArcSwapOption::::default()), - api_proxy: Arc::new(ArcSwapOption::::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - }) - } - - fn target_group(cluster: XtreamCluster, id: u32, name: &str) -> PlaylistGroup { - let name = name.intern(); - let mut header = PlaylistItemHeader { - id: id.to_string().intern(), - input_stream_id: id.to_string().intern(), - name: Arc::clone(&name), - title: Arc::clone(&name), - group: Arc::clone(&name), - input_name: "target-test-input".intern(), - item_type: PlaylistItemType::from(cluster), - xtream_cluster: cluster, - category_id: id, - ..PlaylistItemHeader::default() - }; - header.gen_uuid(); - PlaylistGroup { id, title: name, channels: vec![PlaylistItem { header }], xtream_cluster: cluster } - } - - fn xtream_target(name: &str, use_memory_cache: bool) -> ConfigTarget { - target_with_options( - name, - vec![TargetOutput::Xtream(XtreamTargetOutput { - flags: XtreamTargetFlagsSet::new(), - trakt: None, - filter: None, - })], - use_memory_cache, - ) - } - - async fn cached_target_signature( - playlist_state: &PlaylistStorageState, - target_name: &str, - ) -> (usize, Vec<(XtreamCluster, u32, Arc)>) { - let cache = playlist_state.data.read().await; - let target = cache.get(target_name).expect("target cache"); - let mapping_len = target.id_mapping.as_ref().expect("cached ID mapping").len(); - let xtream = target.xtream.as_ref().expect("cached Xtream storage"); - let mut items = Vec::new(); - for (cluster, storage) in [ - (XtreamCluster::Live, &xtream.live), - (XtreamCluster::Video, &xtream.vod), - (XtreamCluster::Series, &xtream.series), - ] { - for virtual_id in 1..=16 { - if let Some(item) = storage.query(&virtual_id) { - items.push((cluster, virtual_id, Arc::clone(&item.name))); - } - } - } - (mapping_len, items) - } - - async fn seed_memory_cached_xtream_target( - app_config: &Arc, - playlist_state: &Arc, - target: &ConfigTarget, - ) { - let mut baseline = vec![ - target_group(XtreamCluster::Live, 101, "old-live"), - target_group(XtreamCluster::Video, 201, "old-vod"), - target_group(XtreamCluster::Series, 301, "old-series"), - ]; - persist_playlist_with_mode( - app_config, - &mut baseline, - None, - target, - Some(playlist_state), - TargetPlaylistPersistOptions::default(), - TargetPersistenceMode::Persist, - ) - .await - .expect("baseline target persistence"); - } - - #[test] - fn target_force_empty_authorization_is_limited_to_xtream_outputs() { - let xtream_target = target_with_outputs(vec![TargetOutput::Xtream(XtreamTargetOutput { - flags: XtreamTargetFlagsSet::new(), - trakt: None, - filter: None, - })]); - let options = TargetPlaylistPersistOptions { - accepted_empty_clusters: ClusterFlags::Vod, - ..TargetPlaylistPersistOptions::default() - }; - assert!(validate_target_playlist_persistence(&xtream_target, true, options).is_ok()); - - let mixed_target = target_with_outputs(vec![ - TargetOutput::Xtream(XtreamTargetOutput { flags: XtreamTargetFlagsSet::new(), trakt: None, filter: None }), - TargetOutput::M3u(M3uTargetOutput { - filename: None, - include_type_in_url: false, - mask_redirect_url: false, - filter: None, - }), - ]); - let error = validate_target_playlist_persistence(&mixed_target, true, options) - .expect_err("fully empty mixed output must fail before persistence"); - assert!(error.to_string().contains("non-Xtream outputs")); - } - - #[test] - fn target_empty_playlist_without_force_authorization_remains_rejected() { - let target = target_with_outputs(vec![TargetOutput::Xtream(XtreamTargetOutput { - flags: XtreamTargetFlagsSet::new(), - trakt: None, - filter: None, - })]); - let error = validate_target_playlist_persistence(&target, true, TargetPlaylistPersistOptions::default()) - .expect_err("unauthorized empty target must remain rejected"); - assert!(error.to_string().contains("Refusing to persist empty playlist")); - } - - #[tokio::test] - async fn target_cache_reload_failures_return_errors_without_mutating_the_previous_cache() { - for stage in [TargetCacheReloadStage::IdMapping, TargetCacheReloadStage::XtreamStorage] { - let directory = tempdir().expect("temporary storage"); - let app_config = test_app_config(directory.path()); - let playlist_state = Arc::new(PlaylistStorageState::new()); - let target = xtream_target(&format!("cache-reload-{stage:?}"), true); - seed_memory_cached_xtream_target(&app_config, &playlist_state, &target).await; - let before = cached_target_signature(&playlist_state, &target.name).await; - let mut candidate = vec![ - target_group(XtreamCluster::Live, 102, "new-live"), - target_group(XtreamCluster::Series, 302, "new-series"), - ]; - - let errors = persist_playlist_with_mode( - &app_config, - &mut candidate, - None, - &target, - Some(&playlist_state), - TargetPlaylistPersistOptions { - accepted_empty_clusters: ClusterFlags::Vod, - ..TargetPlaylistPersistOptions::default() - }, - TargetPersistenceMode::FailCacheReloadAt(stage), - ) - .await - .expect_err("injected target cache reload must fail"); - - assert!(errors.iter().any(|error| error.to_string().contains("could not be reloaded"))); - assert_eq!(cached_target_signature(&playlist_state, &target.name).await, before); - let persisted = load_xtream_target_storage(&app_config, &target).await.expect("persisted target storage"); - assert_eq!([persisted.live.len(), persisted.vod.len(), persisted.series.len()], [1, 0, 1]); - } - } - - #[tokio::test] - async fn target_force_empty_reload_replaces_the_complete_memory_cache() { - let directory = tempdir().expect("temporary storage"); - let app_config = test_app_config(directory.path()); - let playlist_state = Arc::new(PlaylistStorageState::new()); - let target = xtream_target("force-empty-cache-success", true); - seed_memory_cached_xtream_target(&app_config, &playlist_state, &target).await; - let mut candidate = vec![ - target_group(XtreamCluster::Live, 102, "new-live"), - target_group(XtreamCluster::Series, 302, "new-series"), - ]; - - persist_playlist_with_mode( - &app_config, - &mut candidate, - None, - &target, - Some(&playlist_state), - TargetPlaylistPersistOptions { - accepted_empty_clusters: ClusterFlags::Vod, - ..TargetPlaylistPersistOptions::default() - }, - TargetPersistenceMode::Persist, - ) - .await - .expect("force-empty target persistence and cache reload"); - - let (_, cached_items) = cached_target_signature(&playlist_state, &target.name).await; - assert_eq!(cached_items.iter().filter(|(cluster, _, _)| *cluster == XtreamCluster::Live).count(), 1); - assert!(!cached_items.iter().any(|(cluster, _, _)| *cluster == XtreamCluster::Video)); - assert_eq!(cached_items.iter().filter(|(cluster, _, _)| *cluster == XtreamCluster::Series).count(), 1); - assert!(cached_items.iter().any(|(_, _, name)| name.as_ref() == "new-live")); - assert!(cached_items.iter().any(|(_, _, name)| name.as_ref() == "new-series")); - } - - #[tokio::test] - async fn target_empty_replacement_failures_leave_the_memory_cache_unchanged() { - for failure in [ - crate::TargetEmptyReplacementFailure::CategoryPersistence, - crate::TargetEmptyReplacementFailure::BTreePersistence, - crate::TargetEmptyReplacementFailure::Publication, - ] { - let directory = tempdir().expect("temporary storage"); - let app_config = test_app_config(directory.path()); - let playlist_state = Arc::new(PlaylistStorageState::new()); - let target = xtream_target(&format!("empty-persist-{failure:?}"), true); - seed_memory_cached_xtream_target(&app_config, &playlist_state, &target).await; - let before = cached_target_signature(&playlist_state, &target.name).await; - let mut candidate = vec![ - target_group(XtreamCluster::Live, 102, "new-live"), - target_group(XtreamCluster::Series, 302, "new-series"), - ]; - - let errors = persist_playlist_with_mode( - &app_config, - &mut candidate, - None, - &target, - Some(&playlist_state), - TargetPlaylistPersistOptions { - accepted_empty_clusters: ClusterFlags::Vod, - ..TargetPlaylistPersistOptions::default() - }, - TargetPersistenceMode::FailEmptyReplacementAt(failure), - ) - .await - .expect_err("injected empty replacement must fail"); - - assert!(errors.iter().any(|error| error.to_string().contains("target cluster failed"))); - assert_eq!(cached_target_signature(&playlist_state, &target.name).await, before); - let persisted = load_xtream_target_storage(&app_config, &target).await.expect("retained target storage"); - assert_eq!(persisted.vod.len(), 1); - } - } - - #[tokio::test] - async fn force_empty_output_filters_reject_m3u_and_strm_before_mutation() { - let filter = get_filter(r#"EpgId ~ "^Mixed\.Case$""#, None).expect("case-sensitive EPG-ID filter"); - let cases = [ - ( - "filtered-m3u", - TargetOutput::M3u(M3uTargetOutput { - filename: Some("client.m3u".to_string()), - include_type_in_url: false, - mask_redirect_url: false, - filter: Some(filter.clone()), - }), - "client.m3u", - ), - ( - "filtered-strm", - TargetOutput::Strm(StrmTargetOutput { - directory: "client-strm".to_string(), - username: None, - style: StrmExportStyle::Jellyfin, - flags: StrmTargetFlagsSet::new(), - strm_props: None, - filter: Some(filter), - probe_probe_size_bytes: None, - probe_analyze_duration: None, - }), - "client-strm/old.strm", - ), - ]; - - for (target_name, output, artifact) in cases { - let directory = tempdir().expect("temporary storage"); - let app_config = test_app_config(directory.path()); - let artifact_path = directory.path().join(artifact); - tokio::fs::create_dir_all(artifact_path.parent().expect("artifact parent")) - .await - .expect("artifact directory"); - tokio::fs::write(&artifact_path, b"previous-client-state").await.expect("previous client artifact"); - let mut target = target_with_options(target_name, vec![output], false); - target.options = Some(epg_normalization_options(true)); - let target_path = { - let config = app_config.config.load(); - crate::get_target_storage_path(&config, &target.name).expect("target storage path") - }; - tokio::fs::create_dir_all(&target_path).await.expect("target storage directory"); - let mapping_path = crate::get_target_id_mapping_file(&target_path); - let mapping_uuid_path = mapping_path.with_extension("uuid.db"); - let seed_group = target_group(XtreamCluster::Live, 100, "mapping-seed"); - { - let seed_header = &seed_group.channels[0].header; - let mut mapping = TargetIdMapping::new(&mapping_path, false).expect("seed target ID mapping"); - mapping.get_and_update_virtual_id( - seed_header.get_uuid(), - 0, - seed_header.item_type, - VirtualId::default(), - ); - mapping.persist().expect("persist target ID mapping seed"); - } - let mapping_before = [ - tokio::fs::read(&mapping_path).await.expect("target ID mapping"), - tokio::fs::read(&mapping_uuid_path).await.expect("target UUID mapping"), - ]; - let mut candidate = vec![target_group(XtreamCluster::Live, 101, "filtered-after-normalization")]; - candidate[0].channels[0].header.epg_channel_id = Some("Mixed.Case".intern()); - - let errors = persist_playlist_with_mode( - &app_config, - &mut candidate, - None, - &target, - None, - TargetPlaylistPersistOptions { - accepted_empty_clusters: ClusterFlags::Vod, - ..TargetPlaylistPersistOptions::default() - }, - TargetPersistenceMode::Persist, - ) - .await - .expect_err("force-empty filtered output must be rejected"); - - assert!(errors.iter().any(|error| error.to_string().contains("after its output filter"))); - assert_eq!(candidate[0].channels[0].header.epg_channel_id.as_deref(), Some("mixed.case")); - assert_eq!( - tokio::fs::read(&artifact_path).await.expect("retained client artifact"), - b"previous-client-state" - ); - assert_eq!(tokio::fs::read(&mapping_path).await.expect("retained target ID mapping"), mapping_before[0]); - assert_eq!( - tokio::fs::read(&mapping_uuid_path).await.expect("retained target UUID mapping"), - mapping_before[1] - ); - } - } - - #[test] - fn playlist_without_channels_is_empty_for_persistence() { - assert!(!playlist_has_items(&[])); - assert!(!playlist_has_items(&[PlaylistGroup { - id: 1, - title: "empty".intern(), - channels: Vec::new(), - xtream_cluster: XtreamCluster::Live, - }])); - } - use tempfile::tempdir; - - #[test] - fn complete_curation_empty_authorization_is_projection_and_cluster_scoped() { - let compatibility = PlaylistPublicationPlan::complete_curation(false, false); - assert!(!compatibility.allows_any_empty_output()); - - let suppressed_xtream_base = PlaylistPublicationPlan::complete_curation(false, true); - assert!(!suppressed_xtream_base.allows_empty_base_output()); - assert!(!suppressed_xtream_base.allows_empty_xtream_cluster(XtreamCluster::Live)); - assert!(suppressed_xtream_base.allows_empty_xtream_cluster(XtreamCluster::Video)); - assert!(suppressed_xtream_base.allows_empty_xtream_cluster(XtreamCluster::Series)); - - let persist_filtered = PlaylistPublicationPlan::complete_curation_with_filter(false, false, true); - assert!(persist_filtered.allows_empty_base_output()); - assert!(persist_filtered.allows_empty_xtream_cluster(XtreamCluster::Video)); - let output_filtered = compatibility.with_output_filter(true); - assert!(output_filtered.allows_empty_base_output()); - assert_eq!(PlaylistPublicationPlan::Ordinary.with_output_filter(true), PlaylistPublicationPlan::Ordinary); - } - - fn curation_persist_options(publication_plan: PlaylistPublicationPlan) -> TargetPlaylistPersistOptions { - TargetPlaylistPersistOptions { publication_plan, ..TargetPlaylistPersistOptions::default() } - } - - fn target_test_app_config(storage_dir: &Path) -> Arc { - Arc::new(AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config { - storage_dir: storage_dir.to_string_lossy().into_owned(), - ..Config::default() - })), - sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), - hdhomerun: Arc::new(ArcSwapOption::::default()), - api_proxy: Arc::new(ArcSwapOption::::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - }) - } - - fn mixed_output_target() -> ConfigTarget { - let mut xtream = XtreamTargetOutputDto::default(); - xtream.t_filter = - Some(shared::foundation::get_filter(r#"Group = "Curated alias""#, None).expect("Xtream output filter")); - let mut m3u = M3uTargetOutputDto { filename: Some("curated.m3u".to_string()), ..Default::default() }; - m3u.t_filter = - Some(shared::foundation::get_filter(r#"Group = "Base movie""#, None).expect("M3U output filter")); - ConfigTarget::from(&ConfigTargetDto { - name: "curated-output-test".to_string(), - output: vec![ - TargetOutputDto::Xtream(xtream), - TargetOutputDto::M3u(m3u), - TargetOutputDto::Strm(StrmTargetOutputDto { - directory: "strm".to_string(), - flat: true, - cleanup: true, - ..StrmTargetOutputDto::default() - }), - ], - use_memory_cache: true, - ..ConfigTargetDto::default() - }) - } - - fn target_video_group(title: &str, uuid: UUIDType) -> PlaylistGroup { - PlaylistGroup { - id: 1, - title: title.intern(), - channels: vec![PlaylistItem { - header: PlaylistItemHeader { - id: "1".intern(), - name: title.intern(), - title: title.intern(), - group: title.intern(), - url: format!("http://example.invalid/{title}").intern(), - uuid, - item_type: PlaylistItemType::Video, - xtream_cluster: XtreamCluster::Video, - ..PlaylistItemHeader::default() - }, - }], - xtream_cluster: XtreamCluster::Video, - } - } - - fn strm_files_below(path: &Path) -> Vec { - let mut files = Vec::new(); - let Ok(entries) = std::fs::read_dir(path) else { return files }; - for entry in entries.flatten() { - let entry_path = entry.path(); - if entry_path.is_dir() { - files.extend(strm_files_below(&entry_path)); - } else if entry_path.extension().is_some_and(|extension| extension == "strm") { - files.push(entry_path); - } - } - files - } - - #[tokio::test] - async fn intentional_empty_output_views_clear_managed_artifacts_without_alias_leakage() { - let directory = tempdir().expect("tempdir"); - let app_config = target_test_app_config(directory.path()); - let target = mixed_output_target(); - let playlist_state = Arc::new(crate::PlaylistStorageState::new()); - let mut standard = - vec![target_video_group("Base movie", UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000021"))]; - let mut xtream = vec![target_video_group( - "Curated alias", - UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000022"), - )]; - - let first = persist_playlist_views( - &app_config, - &mut standard, - Some(&mut xtream), - None, - &target, - Some(&playlist_state), - curation_persist_options(PlaylistPublicationPlan::complete_curation(true, false)), - ) - .await; - assert!(first.is_ok(), "initial mixed-output persist failed: {first:?}"); - - let m3u = load_m3u_target_storage(&app_config, &target).await.expect("M3U storage"); - let xtream = load_xtream_target_storage(&app_config, &target).await.expect("Xtream storage"); - assert_eq!(m3u.len(), 1); - assert_eq!(m3u.iter().next().expect("M3U item").1.title.as_ref(), "Base movie"); - assert_eq!(xtream.vod.len(), 1); - let xtream_item = xtream.vod.iter().next().expect("Xtream item").1; - assert_eq!(xtream_item.title.as_ref(), "Curated alias"); - assert_ne!(xtream_item.category_id, 0, "base-suppressed unfiltered rows remain category-backed aliases"); - { - let cache = playlist_state.data.read().await; - let cached = cache.get(&target.name).expect("target cache"); - assert_eq!(cached.m3u.as_ref().expect("M3U cache").len(), 1); - assert_eq!(cached.xtream.as_ref().expect("Xtream cache").vod.len(), 1); - } - let m3u_text_path = directory.path().join("curated.m3u"); - assert!(std::fs::read_to_string(&m3u_text_path).expect("M3U text").contains("Base movie")); - let strm_root = directory.path().join("strm"); - let strm_files = strm_files_below(&strm_root); - assert_eq!(strm_files.len(), 1); - assert!(std::fs::read_to_string(&strm_files[0]).expect("STRM content").contains("Base movie")); - - let mut empty_standard = Vec::new(); - let mut empty_xtream = Vec::new(); - let retained = persist_playlist_views( - &app_config, - &mut empty_standard, - Some(&mut empty_xtream), - None, - &target, - Some(&playlist_state), - curation_persist_options(PlaylistPublicationPlan::Ordinary), - ) - .await; - assert!(retained.is_err(), "untrusted empty input must retain the published snapshot"); - assert_eq!(load_m3u_target_storage(&app_config, &target).await.expect("retained M3U").len(), 1); - assert_eq!(load_xtream_target_storage(&app_config, &target).await.expect("retained Xtream").vod.len(), 1); - { - let cache = playlist_state.data.read().await; - let cached = cache.get(&target.name).expect("retained target cache"); - assert_eq!(cached.m3u.as_ref().expect("retained M3U cache").len(), 1); - assert_eq!(cached.xtream.as_ref().expect("retained Xtream cache").vod.len(), 1); - } - assert_eq!(strm_files_below(&strm_root).len(), 1, "untrusted empty refresh must retain STRM files"); - - let published = persist_playlist_views( - &app_config, - &mut empty_standard, - Some(&mut empty_xtream), - None, - &target, - Some(&playlist_state), - curation_persist_options(PlaylistPublicationPlan::complete_curation(true, false)), - ) - .await; - assert!(published.is_ok(), "trusted empty snapshot failed: {published:?}"); - assert!(load_m3u_target_storage(&app_config, &target).await.expect("empty M3U").is_empty()); - assert_eq!(std::fs::read_to_string(&m3u_text_path).expect("empty M3U text"), "#EXTM3U\n"); - let empty_xtream = load_xtream_target_storage(&app_config, &target).await.expect("empty Xtream"); - assert!(empty_xtream.live.is_empty()); - assert!(empty_xtream.vod.is_empty()); - assert!(empty_xtream.series.is_empty()); - let target_storage = { - let config = app_config.config.load(); - get_target_storage_path(&config, &target.name).expect("target storage") - }; - let xtream_storage = { - let config = app_config.config.load(); - xtream_get_storage_path(&config, &target.name).expect("Xtream storage path") - }; - assert_eq!( - std::fs::read_to_string(get_vod_cat_collection_path(&xtream_storage)).expect("empty VOD categories"), - "[]" - ); - assert_eq!( - std::fs::read_to_string(get_series_cat_collection_path(&xtream_storage)).expect("empty series categories"), - "[]" - ); - let strm_index = strm_get_file_paths(&hash_string_as_hex(&normalize_string_path("strm")), &target_storage); - assert!(std::fs::read_to_string(strm_index).expect("empty STRM index").is_empty()); - { - let cache = playlist_state.data.read().await; - let cached = cache.get(&target.name).expect("empty target cache"); - assert!(cached.m3u.as_ref().expect("empty M3U cache").is_empty()); - assert!(cached.xtream.as_ref().expect("empty Xtream cache").vod.is_empty()); - } - assert!(strm_files_below(&strm_root).is_empty(), "trusted empty refresh must clean stale STRM files"); - } - - #[tokio::test] - async fn intentional_empty_strm_cleanup_false_removes_indexed_files_but_keeps_unmanaged_files() { - let directory = tempdir().expect("tempdir"); - let app_config = target_test_app_config(directory.path()); - let target = ConfigTarget::from(&ConfigTargetDto { - name: "curated-strm-retention-test".to_string(), - output: vec![TargetOutputDto::Strm(StrmTargetOutputDto { - directory: "strm-retained".to_string(), - flat: true, - cleanup: false, - ..StrmTargetOutputDto::default() - })], - ..ConfigTargetDto::default() - }); - let mut seeded = vec![target_video_group( - "Retained movie", - UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000029"), - )]; - let initial = persist_playlist_views( - &app_config, - &mut seeded, - None, - None, - &target, - None, - curation_persist_options(PlaylistPublicationPlan::Ordinary), - ) - .await; - assert!(initial.is_ok(), "initial STRM persist failed: {initial:?}"); - let strm_root = directory.path().join("strm-retained"); - let managed_files = strm_files_below(&strm_root); - assert_eq!(managed_files.len(), 1); - let managed_file = managed_files[0].clone(); - let unmanaged_file = strm_root.join("unmanaged.strm"); - std::fs::write(&unmanaged_file, "unmanaged").expect("unmanaged STRM fixture"); - - let mut empty = Vec::new(); - let published = persist_playlist_views( - &app_config, - &mut empty, - None, - None, - &target, - None, - curation_persist_options(PlaylistPublicationPlan::complete_curation(true, false)), - ) - .await; - assert!(published.is_ok(), "empty STRM persist failed: {published:?}"); - - assert!(!managed_file.exists(), "cleanup=false removes files tracked by the managed index"); - assert!(unmanaged_file.exists(), "cleanup=false does not scan and remove unmanaged files"); - let target_storage = { - let config = app_config.config.load(); - get_target_storage_path(&config, &target.name).expect("target storage") - }; - let index = strm_get_file_paths(&hash_string_as_hex(&normalize_string_path("strm-retained")), &target_storage); - assert!(std::fs::read_to_string(index).expect("empty STRM index").is_empty()); - } - - #[tokio::test] - async fn intentional_empty_curation_never_authorizes_empty_live_cluster_replacement() { - let directory = tempdir().expect("tempdir"); - let app_config = target_test_app_config(directory.path()); - let target = ConfigTarget::from(&ConfigTargetDto { - name: "curated-live-retention-test".to_string(), - output: vec![TargetOutputDto::Xtream(XtreamTargetOutputDto::default())], - ..ConfigTargetDto::default() - }); - let mut live = vec![ - PlaylistGroup { - id: 1, - title: "Live".intern(), - channels: vec![PlaylistItem { - header: PlaylistItemHeader { - id: "1".intern(), - name: "Live channel".intern(), - title: "Live channel".intern(), - group: "Live".intern(), - url: "http://example.invalid/live".intern(), - uuid: UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000031"), - item_type: PlaylistItemType::Live, - xtream_cluster: XtreamCluster::Live, - ..PlaylistItemHeader::default() - }, - }], - xtream_cluster: XtreamCluster::Live, - }, - target_video_group( - "Previously published movie", - UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000032"), - ), - ]; - - let initial = persist_playlist_views( - &app_config, - &mut live, - None, - None, - &target, - None, - curation_persist_options(PlaylistPublicationPlan::Ordinary), - ) - .await; - assert!(initial.is_ok(), "initial Live persist failed: {initial:?}"); - - let mut ordinary_live_only = vec![live[0].clone()]; - let ordinary = persist_playlist_views( - &app_config, - &mut ordinary_live_only, - None, - None, - &target, - None, - curation_persist_options(PlaylistPublicationPlan::Ordinary), - ) - .await; - assert!(ordinary.is_ok(), "ordinary Live-only persist failed: {ordinary:?}"); - let ordinary_storage = load_xtream_target_storage(&app_config, &target).await.expect("ordinary storage"); - assert_eq!(ordinary_storage.live.len(), 1); - assert_eq!(ordinary_storage.vod.len(), 1, "ordinary refresh retains an absent cluster"); - - let mut empty_standard = Vec::new(); - let mut empty_xtream = Vec::new(); - let curated_empty = persist_playlist_views( - &app_config, - &mut empty_standard, - Some(&mut empty_xtream), - None, - &target, - None, - curation_persist_options(PlaylistPublicationPlan::complete_curation(true, false)), - ) - .await; - assert!(curated_empty.is_ok(), "curated empty persist failed: {curated_empty:?}"); - - let storage = load_xtream_target_storage(&app_config, &target).await.expect("Xtream storage"); - assert_eq!(storage.live.len(), 1); - assert!(storage.vod.is_empty()); - assert!(storage.series.is_empty()); - } - - #[test] - fn media_server_playlist_file_path_uses_separate_prefix() { - let dir = tempdir().expect("tempdir"); - let path = get_input_media_server_playlist_file_path(dir.path(), &"Media Server Input".intern()); - - assert!(path.ends_with("media_server_Media_Server_Input.db")); - } - - #[test] - fn skipped_clusters_converts_loaded_clusters_to_exclusions() { - let skipped = skipped_clusters(&[XtreamCluster::Live, XtreamCluster::Series]); - - assert_eq!(skipped.len(), 1); - assert!(skipped.contains(&XtreamCluster::Video)); - } - - fn epg_normalization_options(enabled: bool) -> ConfigTargetOptions { - ConfigTargetOptions { - epg_output: EpgOutputOptions { lowercase_ids: enabled, ..EpgOutputOptions::default() }, - ..ConfigTargetOptions::default() - } - } - - fn epg_normalization_playlist() -> Vec { - let channel = |epg_channel_id: Option<&str>, name: &str| PlaylistItem { - header: PlaylistItemHeader { - name: name.intern(), - title: "Visible Title".intern(), - group: "Visible Group".intern(), - epg_channel_id: epg_channel_id.map(Internable::intern), - item_type: PlaylistItemType::Live, - xtream_cluster: XtreamCluster::Live, - ..PlaylistItemHeader::default() - }, - }; - - vec![PlaylistGroup { - id: 1, - title: "Live".intern(), - channels: vec![ - channel(Some("Example.Channel"), "Mixed Case"), - channel(Some("already.lower"), "Lowercase"), - channel(Some(""), "Empty"), - channel(None, "Missing"), - ], - xtream_cluster: XtreamCluster::Live, - }] - } - - #[test] - fn target_playlist_epg_normalization_is_consistent_for_m3u_and_xtream() { - let options = epg_normalization_options(true); - let mut playlist = epg_normalization_playlist(); - let lowercase_id = - Arc::clone(playlist[0].channels[1].header.epg_channel_id.as_ref().expect("lowercase EPG ID should exist")); - - normalize_target_playlist_epg_ids(&mut playlist, Some(&options)); - - let mixed = &playlist[0].channels[0]; - assert_eq!(mixed.header.epg_channel_id.as_deref(), Some("example.channel")); - assert_eq!(mixed.header.name.as_ref(), "Mixed Case"); - assert_eq!(mixed.header.title.as_ref(), "Visible Title"); - assert_eq!(mixed.header.group.as_ref(), "Visible Group"); - assert!(Arc::ptr_eq( - playlist[0].channels[1].header.epg_channel_id.as_ref().expect("lowercase EPG ID should remain"), - &lowercase_id, - )); - assert_eq!(playlist[0].channels[2].header.epg_channel_id.as_deref(), Some("")); - assert!(playlist[0].channels[3].header.epg_channel_id.is_none()); - - let m3u = M3uPlaylistItem::from(mixed); - let xtream = XtreamPlaylistItem::from(mixed); - assert_eq!(m3u.epg_channel_id.as_deref(), Some("example.channel")); - assert_eq!(xtream.epg_channel_id.as_deref(), Some("example.channel")); - assert!(m3u.to_m3u(None, false).contains(r#"tvg-id="example.channel""#)); - } - - #[test] - fn target_playlist_epg_normalization_preserves_ids_when_disabled() { - let options = epg_normalization_options(false); - let mut playlist = epg_normalization_playlist(); - - normalize_target_playlist_epg_ids(&mut playlist, Some(&options)); - - assert_eq!(playlist[0].channels[0].header.epg_channel_id.as_deref(), Some("Example.Channel")); - } - - fn make_local_series_info(series_uuid: &str, episodes: Vec<(u32, &str, &str)>) -> PlaylistItem { - let episode_props = episodes - .into_iter() - .map(|(id, title, direct_source)| SeriesStreamDetailEpisodeProperties { - id, - episode_num: 0, - season: 0, - title: title.intern(), - container_extension: "".intern(), - custom_sid: None, - added: "".intern(), - direct_source: direct_source.intern(), - tmdb: None, - release_date: "".intern(), - series_release_date: None, - plot: None, - crew: None, - duration_secs: 0, - duration: "".intern(), - movie_image: "".intern(), - bitrate: 0, - rating: None, - video: None, - audio: None, - }) - .collect(); - - PlaylistItem { - header: PlaylistItemHeader { - id: series_uuid.intern(), - item_type: PlaylistItemType::LocalSeriesInfo, - xtream_cluster: XtreamCluster::Series, - additional_properties: Some(StreamProperties::Series(Box::new(SeriesStreamProperties { - name: "Series".intern(), - details: Some(SeriesStreamDetailProperties { - year: None, - seasons: None, - episodes: Some(episode_props), - }), - ..SeriesStreamProperties::default() - }))), - ..PlaylistItemHeader::default() - }, - } - } - - fn make_local_series_episode(series_uuid: &str, direct_source: &str, virtual_id: u32) -> PlaylistItemHeader { - PlaylistItemHeader { - parent_code: series_uuid.intern(), - url: direct_source.intern(), - item_type: PlaylistItemType::LocalSeries, - xtream_cluster: XtreamCluster::Series, - virtual_id: VirtualId::new(virtual_id), - ..PlaylistItemHeader::default() - } - } - - fn make_media_server_episode( - series_uuid: &str, - item_id: &str, - virtual_id: u32, - season: u32, - episode: u32, - ) -> PlaylistItemHeader { - PlaylistItemHeader { - id: format!("media-server:server:shows:episode:{item_id}").intern(), - name: format!("Episode {episode}").intern(), - title: format!("Episode {episode}").intern(), - parent_code: series_uuid.intern(), - url: format!("media-server://plex/server/{item_id}?part_key=%2Flibrary%2Fparts%2Fredacted").intern(), - item_type: PlaylistItemType::Series, - xtream_cluster: XtreamCluster::Series, - virtual_id: VirtualId::new(virtual_id), - additional_properties: Some(StreamProperties::Episode(Box::new(EpisodeStreamProperties { - episode_id: 0, - episode, - season, - added: Some("1700000000".intern()), - release_date: Some("2024-02-03".intern()), - series_release_date: Some("2024-01-01".intern()), - plot: Some("Episode summary".intern()), - tmdb: Some(67890), - movie_image: - "media-server://image/plex/server/episode?image_path=%2Flibrary%2Fmetadata%2Fredacted%2Fthumb" - .intern(), - container_extension: "mkv".intern(), - video: Some(r#"{"codec_name":"h264"}"#.intern()), - audio: Some(r#"{"codec_name":"aac"}"#.intern()), - }))), - ..PlaylistItemHeader::default() - } - } - - #[test] - // Unchanged by the move; rustfmt reflowed it past the 100-line threshold. - #[allow(clippy::too_many_lines)] - fn materializes_media_server_series_info_episodes_after_target_virtual_ids_are_assigned() { - let series_uuid = "123e4567-e89b-12d3-a456-426614174111"; - let mut series_info = PlaylistItem { - header: PlaylistItemHeader { - uuid: UUIDType::from_valid_uuid(series_uuid), - id: "media-server:server:shows:series:series".intern(), - name: "Media Server Series".intern(), - title: "Media Server Series".intern(), - url: "media-server://unavailable/server/shows/series".intern(), - item_type: PlaylistItemType::SeriesInfo, - xtream_cluster: XtreamCluster::Series, - additional_properties: Some(StreamProperties::Series(Box::new(SeriesStreamProperties { - name: "Media Server Series".intern(), - details: Some(SeriesStreamDetailProperties { - year: Some(2024), - seasons: Some(vec![SeriesStreamDetailSeasonProperties { - name: "Season 1".intern(), - season_number: 1, - episode_count: 2, - overview: Some("season summary".intern()), - air_date: Some("2024-01-01".intern()), - cover: None, - cover_tmdb: None, - cover_big: None, - duration: None, - }]), - episodes: None, - }), - ..SeriesStreamProperties::default() - }))), - ..PlaylistItemHeader::default() - }, - }; - let series_parent_code = series_info.header.uuid.to_string(); - let media_episode_two = - PlaylistItem { header: make_media_server_episode(&series_parent_code, "episode-two", 7002, 1, 2) }; - let media_episode_one = - PlaylistItem { header: make_media_server_episode(&series_parent_code, "episode-one", 7001, 1, 1) }; - let malformed_media_episode = PlaylistItem { - header: PlaylistItemHeader { - id: "media-server:server:shows:episode:malformed".intern(), - parent_code: series_parent_code.clone().intern(), - url: "media-server://plex/server/malformed?part_key=%2Flibrary%2Fparts%2Fredacted".intern(), - item_type: PlaylistItemType::Series, - xtream_cluster: XtreamCluster::Series, - virtual_id: VirtualId::new(7003), - additional_properties: None, - ..PlaylistItemHeader::default() - }, - }; - let provider_episode = PlaylistItem { - header: PlaylistItemHeader { - id: "999".intern(), - parent_code: series_parent_code.clone().intern(), - url: "http://provider.example.invalid/series/999.mkv".intern(), - item_type: PlaylistItemType::Series, - xtream_cluster: XtreamCluster::Series, - virtual_id: VirtualId::new(7999), - ..PlaylistItemHeader::default() - }, - }; - - let mut media_server_series = HashMap::, Vec>::new(); - assign_media_server_series_info_episode(&mut media_server_series, &media_episode_two.header); - assign_media_server_series_info_episode(&mut media_server_series, &provider_episode.header); - assign_media_server_series_info_episode(&mut media_server_series, &malformed_media_episode.header); - assign_media_server_series_info_episode(&mut media_server_series, &media_episode_one.header); - - let mut playlist = vec![PlaylistGroup { - id: 1, - title: "Media Server Series".intern(), - channels: vec![ - series_info, - media_episode_two, - provider_episode, - malformed_media_episode, - media_episode_one, - ], - xtream_cluster: XtreamCluster::Series, - }]; - - materialize_media_server_series_info_episodes(&mut playlist, &media_server_series); - series_info = playlist[0].channels[0].clone(); - - let Some(StreamProperties::Series(series)) = series_info.header.additional_properties.as_ref() else { - panic!("missing series properties"); - }; - let details = series.details.as_ref().expect("series details should be present"); - assert_eq!(details.year, Some(2024)); - assert_eq!(details.seasons.as_ref().expect("seasons should be preserved")[0].episode_count, 2); - let episodes = details.episodes.as_ref().expect("media-server episodes should be materialized"); - assert_eq!(episodes.len(), 2); - assert_eq!(episodes[0].id, 7001); - assert_eq!(episodes[0].episode_num, 1); - assert_eq!(episodes[0].season, 1); - assert_eq!(episodes[0].title.as_ref(), "Episode 1"); - assert_eq!(episodes[0].container_extension.as_ref(), "mkv"); - assert_eq!(episodes[0].release_date.as_ref(), "2024-02-03"); - assert_eq!(episodes[0].series_release_date.as_deref(), Some("2024-01-01")); - assert_eq!(episodes[0].plot.as_deref(), Some("Episode summary")); - assert_eq!(episodes[0].tmdb, Some(67890)); - assert_eq!(episodes[0].direct_source.as_ref(), ""); - assert_eq!( - episodes[0].movie_image.as_ref(), - "media-server://image/plex/server/episode?image_path=%2Flibrary%2Fmetadata%2Fredacted%2Fthumb" - ); - assert!(episodes[0].video.as_deref().is_some_and(|video| video.contains("h264"))); - assert!(episodes[0].audio.as_deref().is_some_and(|audio| audio.contains("aac"))); - assert_eq!(episodes[1].id, 7002); - assert!(!episodes.iter().any(|episode| episode.id == 7003)); - } - - #[test] - fn rewrite_series_episode_parent_virtual_ids_updates_local_episode_mapping() { - let series_uuid = "123e4567-e89b-12d3-a456-426614174000"; - let series_uuid_type = UUIDType::from_valid_uuid(series_uuid); - let episode_uuid = UUIDType::from_valid_uuid("123e4567-e89b-12d3-a456-426614174001"); - let dir = tempdir().expect("tempdir"); - let mapping_path = dir.path().join("id_mapping.db"); - let mut target_id_mapping = TargetIdMapping::new(&mapping_path, false).expect("mapping"); - - let mut playlist = vec![PlaylistGroup { - id: 1, - title: "Series".intern(), - channels: vec![ - PlaylistItem { - header: PlaylistItemHeader { - uuid: episode_uuid, - id: "101".intern(), - parent_code: series_uuid.intern(), - url: "/library/episode1.mkv".intern(), - item_type: PlaylistItemType::LocalSeries, - xtream_cluster: XtreamCluster::Series, - ..PlaylistItemHeader::default() - }, - }, - PlaylistItem { - header: PlaylistItemHeader { - uuid: series_uuid_type, - id: series_uuid.intern(), - item_type: PlaylistItemType::LocalSeriesInfo, - xtream_cluster: XtreamCluster::Series, - ..PlaylistItemHeader::default() - }, - }, - ], - xtream_cluster: XtreamCluster::Series, - }]; - - for (idx, channel) in playlist[0].channels.iter_mut().enumerate() { - let uuid = channel.header.uuid; - let provider_id = channel.header.get_provider_id().unwrap_or_default(); - let item_type = channel.header.item_type; - channel.header.virtual_id = - target_id_mapping.get_and_update_virtual_id(&uuid, provider_id, item_type, VirtualId::new(0)); - channel.header.source_ordinal = u32::try_from(idx + 1).expect("ordinal"); - } - - let series_virtual_id = playlist[0].channels[1].header.virtual_id; - let episode_virtual_id = playlist[0].channels[0].header.virtual_id; - - rewrite_series_episode_parent_virtual_ids(&mut playlist, &mut target_id_mapping); - target_id_mapping.persist().expect("persist"); - - let mut query = BPlusTreeQuery::::try_new(&mapping_path).expect("query"); - let record = query.query_zero_copy(&episode_virtual_id.get()).expect("query ok").expect("record missing"); - - assert_eq!(record.parent_virtual_id, series_virtual_id); - assert_eq!(playlist[0].channels[0].header.virtual_id, episode_virtual_id); - } - - #[test] - fn rewrite_series_episode_parent_virtual_ids_updates_provider_episode_mapping_using_series_info_uuid() { - let dir = tempdir().expect("tempdir"); - let mapping_path = dir.path().join("id_mapping.db"); - let mut target_id_mapping = TargetIdMapping::new(&mapping_path, false).expect("mapping"); - - let input_name = "provider-input".intern(); - let xtream_series_info = XtreamPlaylistItem { - virtual_id: VirtualId::new(0), - provider_id: 9001, - name: "Provider Series".intern(), - logo: "".intern(), - logo_small: "".intern(), - group: "Provider Series".intern(), - title: "Provider Series".intern(), - parent_code: "".intern(), - rec: "".intern(), - url: "http://provider.example.com/series/user/pass/9001".intern(), - epg_channel_id: None, - xtream_cluster: XtreamCluster::Series, - additional_properties: None, - item_type: PlaylistItemType::SeriesInfo, - category_id: 0, - input_name: Arc::clone(&input_name), - channel_no: 0, - source_ordinal: 0, - input_stream_id: "9001".intern(), - upstream_user_agent: None, - }; - let provider_parent_code = xtream_series_info.get_uuid().intern(); - let xtream_provider_episode = XtreamPlaylistItem { - virtual_id: VirtualId::new(0), - provider_id: 201, - name: "Episode 1".intern(), - logo: "".intern(), - logo_small: "".intern(), - group: "Provider Series".intern(), - title: "Episode 1".intern(), - parent_code: provider_parent_code, - rec: "".intern(), - url: "http://provider.example.com/series/user/pass/201.mkv".intern(), - epg_channel_id: None, - xtream_cluster: XtreamCluster::Series, - additional_properties: None, - item_type: PlaylistItemType::Series, - category_id: 0, - input_name, - channel_no: 0, - source_ordinal: 0, - input_stream_id: "201".intern(), - upstream_user_agent: None, - }; - let provider_episode = PlaylistItem::from(&xtream_provider_episode); - let mut series_info = PlaylistItem::from(&xtream_series_info); - series_info.header.uuid = UUIDType::from_valid_uuid("123e4567-e89b-12d3-a456-426614174099"); - - let mut playlist = vec![PlaylistGroup { - id: 1, - title: "Provider Series".intern(), - channels: vec![provider_episode, series_info], - xtream_cluster: XtreamCluster::Series, - }]; - - for (idx, channel) in playlist[0].channels.iter_mut().enumerate() { - let uuid = channel.get_uuid(); - let provider_id = channel.header.get_provider_id().unwrap_or_default(); - let item_type = channel.header.item_type; - channel.header.virtual_id = - target_id_mapping.get_and_update_virtual_id(&uuid, provider_id, item_type, VirtualId::new(0)); - channel.header.source_ordinal = u32::try_from(idx + 1).expect("ordinal"); - } - - let series_virtual_id = playlist[0].channels[1].header.virtual_id; - let episode_virtual_id = playlist[0].channels[0].header.virtual_id; - - rewrite_series_episode_parent_virtual_ids(&mut playlist, &mut target_id_mapping); - target_id_mapping.persist().expect("persist"); - - let mut query = BPlusTreeQuery::::try_new(&mapping_path).expect("query"); - let record = query.query_zero_copy(&episode_virtual_id.get()).expect("query ok").expect("record missing"); - - assert_eq!(record.parent_virtual_id, series_virtual_id); - assert_eq!(playlist[0].channels[0].header.virtual_id, episode_virtual_id); - } - - #[test] - fn initial_virtual_id_assignment_preserves_existing_parent_for_series_episode_without_parent_match() { - let dir = tempdir().expect("tempdir"); - let mapping_path = dir.path().join("id_mapping.db"); - let mut target_id_mapping = TargetIdMapping::new(&mapping_path, false).expect("mapping"); - - let input_name = "provider-input".intern(); - let mut episode = PlaylistItem { - header: PlaylistItemHeader { - id: "201".intern(), - url: "http://provider.example.com/series/user/pass/201.mkv".intern(), - input_name, - item_type: PlaylistItemType::Series, - xtream_cluster: XtreamCluster::Series, - ..PlaylistItemHeader::default() - }, - }; - - let provider_id = episode.header.get_provider_id().unwrap_or_default(); - let uuid = *episode.header.get_uuid(); - let original_virtual_id = target_id_mapping.get_and_update_virtual_id( - &uuid, - provider_id, - episode.header.item_type, - VirtualId::new(77), - ); - - let preserved_parent_virtual_id = target_id_mapping.get_parent_virtual_id_by_uuid(&uuid).unwrap_or_default(); - episode.header.virtual_id = target_id_mapping.get_and_update_virtual_id( - &uuid, - provider_id, - episode.header.item_type, - preserved_parent_virtual_id, - ); - target_id_mapping.persist().expect("persist"); - - let mut query = BPlusTreeQuery::::try_new(&mapping_path).expect("query"); - let record = query.query_zero_copy(&original_virtual_id.get()).expect("query ok").expect("record missing"); - - assert_eq!(record.parent_virtual_id, VirtualId::new(77)); - assert_eq!(episode.header.virtual_id, original_virtual_id); - } - - #[test] - fn rewrite_local_series_info_uses_series_uuid_lookup_and_updates_episode_virtual_ids() { - let series_uuid = "series-uuid"; - let mut series_info = make_local_series_info( - series_uuid, - vec![(101, "Episode 1", "/library/episode1.mkv"), (202, "Episode 2", "/library/episode2.mkv")], - ); - let mut local_library_series = HashMap::, Vec>::new(); - local_library_series.insert( - series_uuid.intern(), - vec![ - LocalEpisodeKey { path: "/library/episode1.mkv".intern(), virtual_id: 7001 }, - LocalEpisodeKey { path: "/library/episode2.mkv".intern(), virtual_id: 7002 }, - ], - ); - - rewrite_local_series_info_episode_virtual_id(&mut series_info, &local_library_series); - - let Some(StreamProperties::Series(series)) = series_info.header.additional_properties.as_ref() else { - panic!("missing series properties"); - }; - let episodes = series.details.as_ref().and_then(|details| details.episodes.as_ref()).expect("missing episodes"); - assert_eq!(episodes[0].id, 7001); - assert_eq!(episodes[1].id, 7002); - } - - #[test] - fn rewrite_series_info_updates_local_episode_ids_before_parent_code_is_cleared() { - let series_uuid = "series-uuid"; - let mut episode_one = make_local_series_episode(series_uuid, "/library/episode1.mkv", 7001); - let mut episode_two = make_local_series_episode(series_uuid, "/library/episode2.mkv", 7002); - - let mut local_library_series = HashMap::, Vec>::new(); - assign_local_series_info_episode_key( - &mut local_library_series, - &mut episode_one, - PlaylistItemType::LocalSeries, - ); - assign_local_series_info_episode_key( - &mut local_library_series, - &mut episode_two, - PlaylistItemType::LocalSeries, - ); - - let series_info = make_local_series_info( - series_uuid, - vec![(101, "Episode 1", "/library/episode1.mkv"), (202, "Episode 2", "/library/episode2.mkv")], - ); - let local_episode_one = PlaylistItem { header: episode_one }; - let local_episode_two = PlaylistItem { header: episode_two }; - let mut playlist = vec![PlaylistGroup { - id: 1, - title: "Series".intern(), - channels: vec![series_info, local_episode_one, local_episode_two], - xtream_cluster: XtreamCluster::Series, - }]; - - rewrite_series_info_episode_virtual_id( - &mut playlist, - &local_library_series, - &HashMap::, Vec>::new(), - ); - - let Some(StreamProperties::Series(series)) = playlist[0].channels[0].header.additional_properties.as_ref() - else { - panic!("missing series properties"); - }; - let episodes = series.details.as_ref().and_then(|details| details.episodes.as_ref()).expect("missing episodes"); - assert_eq!(episodes[0].id, 7001); - assert_eq!(episodes[1].id, 7002); - assert!(playlist[0].channels[1].header.parent_code.is_empty()); - assert!(playlist[0].channels[2].header.parent_code.is_empty()); - } - - #[test] - fn rewrite_series_info_updates_local_episode_ids_when_episodes_come_first() { - // Test with episodes BEFORE series_info to verify iteration-order doesn't matter - let series_uuid = "series-uuid"; - let mut episode_one = make_local_series_episode(series_uuid, "/library/episode1.mkv", 7001); - let mut episode_two = make_local_series_episode(series_uuid, "/library/episode2.mkv", 7002); - - let mut local_library_series = HashMap::, Vec>::new(); - assign_local_series_info_episode_key( - &mut local_library_series, - &mut episode_one, - PlaylistItemType::LocalSeries, - ); - assign_local_series_info_episode_key( - &mut local_library_series, - &mut episode_two, - PlaylistItemType::LocalSeries, - ); - - let series_info = make_local_series_info( - series_uuid, - vec![(101, "Episode 1", "/library/episode1.mkv"), (202, "Episode 2", "/library/episode2.mkv")], - ); - let local_episode_one = PlaylistItem { header: episode_one }; - let local_episode_two = PlaylistItem { header: episode_two }; - - // Episodes FIRST, then series_info (reversed order) - let mut playlist = vec![PlaylistGroup { - id: 1, - title: "Series".intern(), - channels: vec![local_episode_one, local_episode_two, series_info], - xtream_cluster: XtreamCluster::Series, - }]; - - rewrite_series_info_episode_virtual_id( - &mut playlist, - &local_library_series, - &HashMap::, Vec>::new(), - ); - - let Some(StreamProperties::Series(series)) = playlist[0].channels[2].header.additional_properties.as_ref() - else { - panic!("missing series properties"); - }; - let episodes = series.details.as_ref().and_then(|details| details.episodes.as_ref()).expect("missing episodes"); - assert_eq!(episodes[0].id, 7001); - assert_eq!(episodes[1].id, 7002); - assert!(playlist[0].channels[0].header.parent_code.is_empty()); - assert!(playlist[0].channels[1].header.parent_code.is_empty()); - } -} +use episode::{ + assign_local_series_info_episode_key, assign_media_server_series_info_episode, + materialize_media_server_series_info_episodes, normalize_target_playlist_epg_ids, + rewrite_local_series_info_episode_virtual_id, rewrite_series_episode_parent_virtual_ids, + rewrite_series_info_episode_virtual_id, +}; +pub use episode::{rewrite_provider_series_info_episode_virtual_id, ProviderEpisodeKey}; +#[cfg(test)] +use input::skipped_clusters; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use input::{ + get_input_local_library_playlist_file_path, get_input_m3u_playlist_file_path, + get_input_media_server_playlist_file_path, load_input_playlist, load_input_stalker_playlist, + persist_input_playlist, persist_input_playlist_with_options, +}; +#[cfg(test)] +use persist::persist_playlist_with_mode; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use persist::{ + load_input_media_server_playlist, persist_input_media_server_playlist, persist_playlist, persist_playlist_views, +}; +#[cfg(test)] +use publication::TargetCacheReloadStage; +#[allow(unused_imports, reason = "Retains the existing module interface in production and test builds.")] +pub use publication::{ + InputPlaylistPersistOptions, LibraryEmptyPublication, PlaylistPublicationPlan, TargetPlaylistPersistOptions, +}; +pub use target::{get_target_id_mapping, load_m3u_target_storage, load_xtream_target_storage}; diff --git a/backend/repository/src/playlist_repository/episode.rs b/backend/repository/src/playlist_repository/episode.rs new file mode 100644 index 000000000..3fd5338d0 --- /dev/null +++ b/backend/repository/src/playlist_repository/episode.rs @@ -0,0 +1,395 @@ +use super::LocalEpisodeKey; +use crate::TargetIdMapping; +use shared::{ + model::{ + ConfigTargetOptions, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, + SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, StreamProperties, UUIDType, VirtualId, + }, + utils::{is_dash_url, is_hls_url, Internable}, +}; +use std::{collections::HashMap, sync::Arc}; +use tuliprox_core::{model::ConfigTarget, utils::fold_epg_id_arc}; + +pub struct ProviderEpisodeKey { + pub provider_id: u32, + pub virtual_id: u32, +} + +pub(super) fn normalize_target_playlist_epg_ids( + playlist: &mut [PlaylistGroup], + target_options: Option<&ConfigTargetOptions>, +) { + if !target_options.is_some_and(ConfigTargetOptions::lowercase_epg_ids) { + return; + } + + for group in playlist { + for channel in &mut group.channels { + let Some(epg_id) = channel.header.epg_channel_id.as_mut() else { + continue; + }; + if epg_id.is_empty() { + continue; + } + + *epg_id = fold_epg_id_arc(epg_id); + } + } +} + +pub(super) fn prepare_target_playlist_for_persistence( + playlist: &mut [PlaylistGroup], + target: &ConfigTarget, + target_id_mapping: &mut TargetIdMapping, +) { + let mut source_ordinal: u32 = 0; + for group in playlist.iter_mut() { + for channel in &mut group.channels { + let header = &mut channel.header; + source_ordinal += 1; + header.source_ordinal = source_ordinal; + let provider_id = header.get_provider_id().unwrap_or_default(); + if provider_id == 0 { + header.item_type = match (is_hls_url(&header.url), header.item_type) { + (true, _) => PlaylistItemType::LiveHls, + (false, PlaylistItemType::Live) => { + if is_dash_url(&header.url) { + PlaylistItemType::LiveDash + } else { + PlaylistItemType::LiveUnknown + } + } + _ => header.item_type, + }; + } + + let uuid = header.get_uuid(); + let item_type = header.item_type; + let parent_virtual_id = if item_type.is_series() { + target_id_mapping.get_parent_virtual_id_by_uuid(uuid).unwrap_or_default() + } else { + VirtualId::default() + }; + header.virtual_id = + target_id_mapping.get_and_update_virtual_id(uuid, provider_id, item_type, parent_virtual_id); + } + } + + rewrite_series_episode_parent_virtual_ids(playlist, target_id_mapping); + + let mut local_library_series = HashMap::, Vec>::new(); + let mut provider_series = HashMap::, Vec>::new(); + let mut media_server_series = HashMap::, Vec>::new(); + for group in playlist.iter_mut() { + for channel in &mut group.channels { + let header = &mut channel.header; + let item_type = header.item_type; + if item_type == PlaylistItemType::LocalSeries { + assign_local_series_info_episode_key(&mut local_library_series, header, item_type); + } else if is_media_server_series_episode_header(header) { + assign_media_server_series_info_episode(&mut media_server_series, header); + } else if item_type == PlaylistItemType::Series { + assign_provider_series_info_episode_key(&mut provider_series, header, item_type); + } + } + } + + materialize_media_server_series_info_episodes(playlist, &media_server_series); + rewrite_series_info_episode_virtual_id(playlist, &local_library_series, &provider_series); + normalize_target_playlist_epg_ids(playlist, target.options.as_ref()); +} + +pub(super) fn assign_local_series_info_episode_key( + local_library_series: &mut HashMap, Vec>, + header: &mut PlaylistItemHeader, + item_type: PlaylistItemType, +) { + // we need to rewrite local series info with the new virtual ids + if item_type == PlaylistItemType::LocalSeries { + local_library_series + .entry(header.parent_code.clone()) + .or_default() + .push(LocalEpisodeKey { path: header.url.clone(), virtual_id: header.virtual_id.get() }); + } +} + +fn assign_provider_series_info_episode_key( + provider_series: &mut HashMap, Vec>, + header: &mut PlaylistItemHeader, + item_type: PlaylistItemType, +) { + // we need to rewrite local series info with the new virtual ids + if item_type == PlaylistItemType::Series { + provider_series.entry(header.parent_code.clone()).or_default().push(ProviderEpisodeKey { + provider_id: header.get_provider_id().unwrap_or_default(), + virtual_id: header.virtual_id.get(), + }); + } +} + +fn is_media_server_item_header(header: &PlaylistItemHeader) -> bool { + header.id.starts_with("media-server:") || header.url.starts_with("media-server://") +} + +fn is_media_server_series_info_header(header: &PlaylistItemHeader) -> bool { + header.item_type == PlaylistItemType::SeriesInfo && is_media_server_item_header(header) +} + +fn is_media_server_series_episode_header(header: &PlaylistItemHeader) -> bool { + header.item_type == PlaylistItemType::Series + && !header.parent_code.is_empty() + && is_media_server_item_header(header) +} + +pub(super) fn assign_media_server_series_info_episode( + media_server_series: &mut HashMap, Vec>, + header: &PlaylistItemHeader, +) { + if let Some(episode) = media_server_series_episode_detail(header) { + media_server_series.entry(header.parent_code.clone()).or_default().push(episode); + } +} + +fn media_server_series_episode_detail(header: &PlaylistItemHeader) -> Option { + if !is_media_server_series_episode_header(header) { + return None; + } + + let Some(StreamProperties::Episode(episode)) = header.additional_properties.as_ref() else { + return None; + }; + let episode = episode.as_ref(); + + Some(SeriesStreamDetailEpisodeProperties { + id: header.virtual_id.get(), + episode_num: episode.episode, + season: episode.season, + title: non_blank_arc(&header.title) + .unwrap_or_else(|| non_blank_arc(&header.name).unwrap_or_else(|| "Episode".intern())), + container_extension: episode.container_extension.clone(), + custom_sid: None, + added: episode.added.clone().unwrap_or_else(|| "".intern()), + direct_source: "".intern(), + tmdb: episode.tmdb, + release_date: episode.release_date.clone().unwrap_or_else(|| "".intern()), + series_release_date: episode.series_release_date.clone(), + plot: episode.plot.clone(), + crew: None, + duration_secs: 0, + duration: "".intern(), + movie_image: episode.movie_image.clone(), + bitrate: 0, + rating: None, + video: episode.video.clone(), + audio: episode.audio.clone(), + }) +} + +fn non_blank_arc(value: &Arc) -> Option> { (!value.trim().is_empty()).then(|| Arc::clone(value)) } + +fn source_series_info_episode_key(channel: &PlaylistItem) -> Option> { + match channel.header.item_type { + PlaylistItemType::SeriesInfo => Some(channel.get_uuid().intern()), + PlaylistItemType::LocalSeriesInfo => Some(channel.header.id.clone()), + _ => None, + } +} + +fn header_uuid_episode_key(header: &PlaylistItemHeader) -> Option> { + (header.uuid != UUIDType::default()).then(|| header.uuid.intern()) +} + +fn push_unique_key(keys: &mut Vec>, key: Arc) { + if !key.is_empty() && !keys.iter().any(|existing| existing.as_ref() == key.as_ref()) { + keys.push(key); + } +} + +fn series_info_episode_lookup_keys(channel: &PlaylistItem) -> Vec> { + let mut keys = Vec::with_capacity(2); + if let Some(alias_key) = header_uuid_episode_key(&channel.header) { + push_unique_key(&mut keys, alias_key); + } + if let Some(source_key) = source_series_info_episode_key(channel) { + push_unique_key(&mut keys, source_key); + } + keys +} + +fn series_info_parent_keys(channel: &PlaylistItem) -> Vec<(Arc, bool)> { + let Some(source_key) = source_series_info_episode_key(channel) else { return Vec::new() }; + let Some(alias_key) = header_uuid_episode_key(&channel.header) else { return vec![(source_key, true)] }; + if alias_key.as_ref() == source_key.as_ref() { + vec![(source_key, true)] + } else { + vec![(alias_key, true), (source_key, false)] + } +} + +#[allow(clippy::implicit_hasher)] +pub(super) fn materialize_media_server_series_info_episodes( + playlist: &mut [PlaylistGroup], + media_server_series: &HashMap, Vec>, +) { + if media_server_series.is_empty() { + return; + } + + for group in playlist.iter_mut() { + for channel in &mut group.channels { + if !is_media_server_series_info_header(&channel.header) { + continue; + } + let lookup_keys = series_info_episode_lookup_keys(channel); + let Some(episodes) = lookup_keys.iter().find_map(|key| media_server_series.get(key)) else { continue }; + let Some(StreamProperties::Series(series)) = channel.header.additional_properties.as_mut() else { + continue; + }; + let details = series.details.get_or_insert(SeriesStreamDetailProperties { + year: None, + seasons: None, + episodes: None, + }); + let mut episodes = episodes.clone(); + episodes.sort_by_key(|episode| (episode.season, episode.episode_num, episode.id)); + details.episodes = Some(episodes); + } + } +} + +#[allow(clippy::implicit_hasher)] +pub(super) fn rewrite_local_series_info_episode_virtual_id( + pli: &mut PlaylistItem, + local_library_series: &HashMap, Vec>, +) { + // local_library_series keys are the Series UUID or a category alias UUID. + // For LocalSeriesInfo items, header.id is the source Series UUID; category + // aliases use header.uuid so cloned episode rows can point at the alias. + let lookup_keys = if pli.header.item_type == PlaylistItemType::LocalSeries { + vec![pli.header.parent_code.clone()] + } else { + series_info_episode_lookup_keys(pli) + }; + + if let Some(episode_keys) = lookup_keys.iter().find_map(|key| local_library_series.get(key)) { + if let Some(StreamProperties::Series(series)) = pli.header.additional_properties.as_mut() { + if let Some(episodes) = series.details.as_mut().and_then(|d| d.episodes.as_mut()) { + for episode in episodes.iter_mut() { + for episode_key in episode_keys { + if episode.direct_source == episode_key.path { + episode.id = episode_key.virtual_id; + break; + } + } + } + } + } + } +} + +#[allow(clippy::implicit_hasher)] +pub fn rewrite_provider_series_info_episode_virtual_id

( + pli: &mut P, + provider_series: &HashMap, Vec>, +) where + P: PlaylistEntry, +{ + let lookup_key = pli.get_uuid().intern(); + if let Some(episode_keys) = provider_series.get(&lookup_key) { + if let Some(properties) = pli.get_additional_properties_mut() { + apply_provider_episode_keys(properties, episode_keys); + } + } +} + +#[allow(clippy::implicit_hasher)] +fn rewrite_provider_playlist_item_series_info_episode_virtual_id( + pli: &mut PlaylistItem, + provider_series: &HashMap, Vec>, +) { + let lookup_keys = series_info_episode_lookup_keys(pli); + if let Some(episode_keys) = lookup_keys.iter().find_map(|key| provider_series.get(key)) { + if let Some(properties) = pli.get_additional_properties_mut() { + apply_provider_episode_keys(properties, episode_keys); + } + } +} + +fn apply_provider_episode_keys(properties: &mut StreamProperties, episode_keys: &[ProviderEpisodeKey]) { + if let StreamProperties::Series(series) = properties { + if let Some(episodes) = series.details.as_mut().and_then(|d| d.episodes.as_mut()) { + for episode in episodes.iter_mut() { + for episode_key in episode_keys { + if episode.id == episode_key.provider_id { + episode.id = episode_key.virtual_id; + break; + } + } + } + } + } +} + +pub(super) fn rewrite_series_info_episode_virtual_id( + playlist: &mut [PlaylistGroup], + local_library_series: &HashMap, Vec>, + provider_series: &HashMap, Vec>, +) { + if local_library_series.is_empty() && provider_series.is_empty() { + return; + } + for group in playlist.iter_mut() { + for channel in &mut group.channels { + let item_type = channel.header.item_type; + if item_type == PlaylistItemType::SeriesInfo { + rewrite_provider_playlist_item_series_info_episode_virtual_id(channel, provider_series); + } else if item_type == PlaylistItemType::LocalSeriesInfo { + rewrite_local_series_info_episode_virtual_id(channel, local_library_series); + } else if item_type == PlaylistItemType::LocalSeries { + channel.header.parent_code = "".intern(); + } + } + } +} + +pub(super) fn rewrite_series_episode_parent_virtual_ids( + playlist: &mut [PlaylistGroup], + target_id_mapping: &mut TargetIdMapping, +) { + let mut series_parent_virtual_ids = HashMap::, u32>::new(); + + for group in playlist.iter() { + for channel in &group.channels { + for (parent_key, overwrite) in series_info_parent_keys(channel) { + if overwrite { + series_parent_virtual_ids.insert(parent_key, channel.header.virtual_id.get()); + } else { + series_parent_virtual_ids.entry(parent_key).or_insert(channel.header.virtual_id.get()); + } + } + } + } + + if series_parent_virtual_ids.is_empty() { + return; + } + + for group in playlist.iter_mut() { + for channel in &mut group.channels { + let header = &mut channel.header; + if header.item_type.is_series() { + if let Some(parent_virtual_id) = series_parent_virtual_ids.get(&header.parent_code) { + let provider_id = header.get_provider_id().unwrap_or_default(); + let item_type = header.item_type; + let uuid = header.get_uuid(); + header.virtual_id = target_id_mapping.get_and_update_virtual_id( + uuid, + provider_id, + item_type, + VirtualId::new(*parent_virtual_id), + ); + } + } + } + } +} diff --git a/backend/repository/src/playlist_repository/input.rs b/backend/repository/src/playlist_repository/input.rs new file mode 100644 index 000000000..4cb41b1b1 --- /dev/null +++ b/backend/repository/src/playlist_repository/input.rs @@ -0,0 +1,301 @@ +use super::{ + load_input_media_server_playlist, persist_input_media_server_playlist, publication::playlist_has_items, + target::publish_all_cluster_catalogs, InputPlaylistPersistOptions, +}; +use crate::{ + get_input_storage_path, load_input_local_library_playlist, load_input_m3u_playlist, load_input_xtream_playlist, + persist_input_library_playlist, persist_input_m3u_playlist, stalker_repository::get_stalker_storage_path, + LocalLibraryDiskPlaylistSource, M3uDiskPlaylistSource, MediaServerDiskPlaylistSource, MemoryPlaylistSource, + PlaylistSource, StalkerDiskPlaylistSource, XtreamDiskPlaylistSource, FILE_SUFFIX_DB, +}; +use log::{debug, warn}; +use shared::{ + error::TuliproxError, + model::{xtream_const::XTREAM_CLUSTER, InputPersistence, PlaylistGroup, PlaylistItem, XtreamCluster}, +}; +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + sync::Arc, +}; +use tuliprox_core::{ + model::{AppConfig, ConfigInput}, + utils::normalized_source_ordinal, +}; + +pub async fn persist_input_playlist( + app_config: &Arc, + input: &ConfigInput, + playlist: Vec, +) -> (Vec, Option) { + persist_input_playlist_with_options(app_config, input, playlist, InputPlaylistPersistOptions::default()).await +} + +pub async fn persist_input_playlist_with_options( + app_config: &Arc, + input: &ConfigInput, + mut playlist: Vec, + options: InputPlaylistPersistOptions, +) -> (Vec, Option) { + let persistence = input.get_download_input_type().persistence(); + let accepts_empty = persistence == InputPersistence::Library + || (!options.accepted_empty_clusters.is_empty() + && matches!(persistence, InputPersistence::Xtream | InputPersistence::Stalker)); + if !playlist_has_items(&playlist) && !accepts_empty { + let empty_error = TuliproxError::RepositoryPlaylist(format!( + "Refusing to persist empty playlist for input '{}'; existing data was retained", + input.name + )); + warn!("{empty_error}"); + return match load_input_playlist(app_config, input, None).await { + Ok(mut previous) => { + if previous.is_empty() { + (playlist, Some(empty_error)) + } else { + (previous.take_groups(), Some(empty_error)) + } + } + Err(load_err) => ( + playlist, + Some(TuliproxError::RepositoryPlaylist(format!( + "{empty_error}; failed to load the retained input playlist: {load_err}" + ))), + ), + }; + } + if persistence == InputPersistence::Stalker { + // The Stalker processor (`processor::stalker::download_stalker_playlist`) + // is the single writer of the per-cluster B+Tree and raw group catalogs. + // Re-encoding the `PlaylistItem` runtime projection back into `StalkerPlaylistItem` + // would destroy the canonical `cmd`/`playback_descriptor`/capability + // flags the processor just persisted — including the field the + // runtime 4xx-re-resolve hook relies on. The disk layout is + // already in sync; nothing to do here. + return (playlist, None); + } + playlist.iter_mut().for_each(PlaylistGroup::on_load); + let cfg = app_config.config.load(); + let storage_path = match get_input_storage_path(&input.name, &cfg.storage_dir).await { + Ok(storage_path) => storage_path, + Err(err) => { + return ( + playlist, + Some(TuliproxError::Config(format!( + "Error creating input storage directory for input '{}' failed: {err}", + input.name + ))), + ); + } + }; + + let (persisted_playlist, err) = match persistence { + InputPersistence::Xtream => { + crate::persist_input_xtream_playlist_with_empty_replacements( + app_config, + &storage_path, + playlist, + options.accepted_empty_clusters, + ) + .await + } + + InputPersistence::M3u => { + // Persist M3U + let file_path = get_input_m3u_playlist_file_path(&storage_path, &input.name); + if let Err(err) = persist_input_m3u_playlist(app_config, &file_path, &playlist).await { + return (playlist, Some(err)); + } + (playlist, None) + } + InputPersistence::Library => { + // Persist local library playlist + let file_path = get_input_local_library_playlist_file_path(&storage_path, &input.name); + let (playlist, result) = persist_input_library_playlist(app_config, &file_path, playlist).await; + if let Err(err) = result { + return (playlist, Some(err)); + } + (playlist, None) + } + InputPersistence::MediaServer => { + let file_path = get_input_media_server_playlist_file_path(&storage_path, &input.name); + let (playlist, result) = persist_input_media_server_playlist(app_config, &file_path, playlist).await; + if let Err(err) = result { + return (playlist, Some(err)); + } + (playlist, None) + } + InputPersistence::Stalker => unreachable!("handled above"), + }; + + if err.is_none() { + return publish_all_cluster_catalogs(app_config, &storage_path, &input.name, persisted_playlist).await; + } + + (persisted_playlist, err) +} + +pub async fn load_input_playlist( + app_config: &Arc, + input: &ConfigInput, + clusters: Option<&[XtreamCluster]>, +) -> Result { + let cfg = app_config.config.load(); + let storage_path = get_input_storage_path(&input.name, &cfg.storage_dir) + .await + .map_err(|e| TuliproxError::Config(format!("Error getting input path: {e}")))?; + let disk_based_processing = cfg.disk_based_processing; + + match input.get_download_input_type().persistence() { + InputPersistence::Xtream => { + let clusters_to_load = clusters.unwrap_or(&XTREAM_CLUSTER); + if disk_based_processing { + let source = + PlaylistSource::xtream_disk(XtreamDiskPlaylistSource::new(app_config, &storage_path).await?); + Ok(PlaylistSource::filtered(source, skipped_clusters(clusters_to_load))) + } else { + let groups = load_input_xtream_playlist(app_config, &storage_path, clusters_to_load).await?; + Ok(MemoryPlaylistSource::new(groups).into_source()) + } + } + InputPersistence::M3u => { + // Load M3U + let file_path = get_input_m3u_playlist_file_path(&storage_path, &input.name); + if disk_based_processing && file_path.exists() { + Ok(PlaylistSource::m3u_disk(M3uDiskPlaylistSource::new(app_config, &file_path).await?)) + } else { + let groups = load_input_m3u_playlist(app_config, &file_path).await?; + Ok(MemoryPlaylistSource::new(groups).into_source()) + } + } + InputPersistence::Library => { + let file_path = get_input_local_library_playlist_file_path(&storage_path, &input.name); + if disk_based_processing && file_path.exists() { + Ok(PlaylistSource::local_library_disk( + LocalLibraryDiskPlaylistSource::new(app_config, &file_path).await?, + )) + } else { + let groups = load_input_local_library_playlist(app_config, &file_path).await?; + Ok(MemoryPlaylistSource::new(groups).into_source()) + } + } + InputPersistence::MediaServer => { + let file_path = get_input_media_server_playlist_file_path(&storage_path, &input.name); + if disk_based_processing && file_path.exists() { + Ok(PlaylistSource::media_server_disk(MediaServerDiskPlaylistSource::new(app_config, &file_path).await?)) + } else { + let groups = load_input_media_server_playlist(app_config, &file_path).await?; + Ok(MemoryPlaylistSource::new(groups).into_source()) + } + } + InputPersistence::Stalker => { + let clusters_to_load = clusters.unwrap_or(&XTREAM_CLUSTER); + let stalker_path = get_stalker_storage_path(&storage_path); + let stalker_config = input.stalker.as_ref().ok_or_else(|| { + TuliproxError::ConfigInput(format!("Stalker input '{}' has no Stalker configuration", input.name)) + })?; + let portal_url = input.resolve_url(&input.url)?.into_owned(); + // A read path: when the published manifest belongs to a different identity the + // input simply has nothing to serve yet, so fall back to an empty manifest + // instead of replacing the publication state of the refresh that owns it. + let (manifest, published) = crate::stalker_generation_repository::readable_active_manifest( + &stalker_path, + stalker_config.identity_fingerprint(&portal_url), + ) + .await?; + if !published { + debug!( + "Stalker input '{}' has no published catalog for its current identity; serving an empty playlist", + input.name + ); + } + if disk_based_processing { + let source = PlaylistSource::stalker_disk( + StalkerDiskPlaylistSource::new(app_config, &stalker_path, Arc::clone(&input.name), manifest) + .await?, + ); + Ok(PlaylistSource::filtered(source, skipped_clusters(clusters_to_load))) + } else { + let groups = load_input_stalker_playlist(app_config, &input.name, clusters_to_load, &manifest).await?; + Ok(MemoryPlaylistSource::new(groups).into_source()) + } + } + } +} + +pub(super) fn skipped_clusters(clusters_to_load: &[XtreamCluster]) -> HashSet { + XTREAM_CLUSTER.iter().copied().filter(|cluster| !clusters_to_load.contains(cluster)).collect() +} + +pub fn get_input_m3u_playlist_file_path(storage_path: &Path, input_name: &Arc) -> PathBuf { + let sanitized_input_name: String = input_name.chars().map(|c| if c.is_alphanumeric() { c } else { '_' }).collect(); + storage_path.join(format!("m3u_{sanitized_input_name}.{FILE_SUFFIX_DB}")) +} + +pub fn get_input_local_library_playlist_file_path(storage_path: &Path, input_name: &Arc) -> PathBuf { + let sanitized_input_name: String = input_name.chars().map(|c| if c.is_alphanumeric() { c } else { '_' }).collect(); + storage_path.join(format!("lib_{sanitized_input_name}.{FILE_SUFFIX_DB}")) +} + +pub fn get_input_media_server_playlist_file_path(storage_path: &Path, input_name: &Arc) -> PathBuf { + let sanitized_input_name: String = input_name.chars().map(|c| if c.is_alphanumeric() { c } else { '_' }).collect(); + storage_path.join(format!("media_server_{sanitized_input_name}.{FILE_SUFFIX_DB}")) +} + +/// Load a Stalker input's playlist into memory. The on-disk B+Tree is the +/// source of truth; we stream every per-cluster tree and bucket the items +/// by cluster to build the runtime `PlaylistGroup`s. `input_name` seeds the +/// canonical `PlaylistItem::from_stalker` conversion so item identity matches +/// the download path. +pub async fn load_input_stalker_playlist( + app_config: &Arc, + input_name: &str, + clusters: &[XtreamCluster], + manifest: &crate::stalker_generation_repository::StalkerActiveManifest, +) -> Result, TuliproxError> { + let mut groups_map: indexmap::IndexMap<(XtreamCluster, u32), PlaylistGroup> = indexmap::IndexMap::new(); + for &cluster in clusters { + let mut batches = Vec::new(); + match cluster { + XtreamCluster::Live => { + if let Some(files) = manifest.live.as_ref() { + batches.push(crate::stalker_repository::load_stalker_items_at(app_config, &files.data).await?); + } + } + XtreamCluster::Video => { + if let Some(files) = manifest.vod.as_ref() { + batches.push(crate::stalker_repository::load_stalker_items_at(app_config, &files.data).await?); + } + } + XtreamCluster::Series => { + if let Some(files) = manifest.series.as_ref() { + batches.push(crate::stalker_repository::load_stalker_items_at(app_config, &files.roots).await?); + batches.push(crate::stalker_repository::load_stalker_items_at(app_config, &files.episodes).await?); + } + } + } + for batch in batches { + for item in batch { + let category_id = item.category_id; + let playlist_item = PlaylistItem::from_stalker(&item, input_name); + groups_map + .entry((cluster, category_id)) + .or_insert_with(|| PlaylistGroup { + id: category_id, + title: Arc::clone(&playlist_item.header.group), + channels: Vec::new(), + xtream_cluster: cluster, + }) + .channels + .push(playlist_item); + } + } + } + let mut groups: Vec = groups_map.into_values().collect(); + for group in &mut groups { + group.channels.sort_by_key(|item| normalized_source_ordinal(item.header.source_ordinal)); + } + groups.sort_by_key(|group| { + group.channels.first().map_or(u32::MAX, |c| normalized_source_ordinal(c.header.source_ordinal)) + }); + Ok(groups) +} diff --git a/backend/repository/src/playlist_repository/persist.rs b/backend/repository/src/playlist_repository/persist.rs new file mode 100644 index 000000000..4c4ac8543 --- /dev/null +++ b/backend/repository/src/playlist_repository/persist.rs @@ -0,0 +1,219 @@ +use super::{ + episode::prepare_target_playlist_for_persistence, + get_target_id_mapping, + publication::{ + persist_xtream_target_playlist, playlist_has_items, prepare_target_output_playlists, + validate_force_empty_output_filters, validate_target_playlist_persistence, TargetPersistenceMode, + }, + target::load_target_memory_cache_snapshot, + PlaylistStorageState, TargetPlaylistPersistOptions, +}; +use crate::{ + ensure_target_storage_path, epg_write_for_target, load_input_local_library_playlist, m3u_write_playlist, + persist_input_library_playlist, write_strm_playlist, +}; +use shared::{error::TuliproxError, model::PlaylistGroup}; +use std::{path::Path, sync::Arc}; +use tuliprox_core::model::{AppConfig, ConfigTarget, Epg, TargetOutput}; + +pub async fn persist_playlist( + app_config: &Arc, + playlist: &mut [PlaylistGroup], + epg: Option<&Epg>, + target: &ConfigTarget, + playlist_state: Option<&Arc>, + options: TargetPlaylistPersistOptions, +) -> Result<(), Vec> { + persist_playlist_with_mode( + app_config, + playlist, + epg, + target, + playlist_state, + options, + TargetPersistenceMode::Persist, + ) + .await +} + +pub async fn persist_playlist_views( + app_config: &Arc, + base_playlist: &mut [PlaylistGroup], + xtream_playlist: Option<&mut [PlaylistGroup]>, + epg: Option<&Epg>, + target: &ConfigTarget, + playlist_state: Option<&Arc>, + options: TargetPlaylistPersistOptions, +) -> Result<(), Vec> { + persist_playlist_views_with_mode( + app_config, + base_playlist, + xtream_playlist, + epg, + target, + playlist_state, + options, + TargetPersistenceMode::Persist, + ) + .await +} + +pub(super) async fn persist_playlist_with_mode( + app_config: &Arc, + playlist: &mut [PlaylistGroup], + epg: Option<&Epg>, + target: &ConfigTarget, + playlist_state: Option<&Arc>, + options: TargetPlaylistPersistOptions, + persistence_mode: TargetPersistenceMode, +) -> Result<(), Vec> { + persist_playlist_views_with_mode(app_config, playlist, None, epg, target, playlist_state, options, persistence_mode) + .await +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn persist_playlist_views_with_mode( + app_config: &Arc, + base_playlist: &mut [PlaylistGroup], + mut xtream_playlist: Option<&mut [PlaylistGroup]>, + epg: Option<&Epg>, + target: &ConfigTarget, + playlist_state: Option<&Arc>, + options: TargetPlaylistPersistOptions, + persistence_mode: TargetPersistenceMode, +) -> Result<(), Vec> { + let playlist_is_empty = !playlist_has_items(base_playlist) + && xtream_playlist.as_deref().is_none_or(|playlist| !playlist_has_items(playlist)); + if let Err(error) = validate_target_playlist_persistence(target, playlist_is_empty, options) { + return Err(vec![error]); + } + let mut errors = vec![]; + let config = &app_config.config.load(); + let target_path = match ensure_target_storage_path(config, &target.name).await { + Ok(path) => path, + Err(err) => return Err(vec![err]), + }; + + let (mut target_id_mapping, file_lock) = + match get_target_id_mapping(app_config, &target_path, target.use_memory_cache).await { + Ok(result) => result, + Err(err) => return Err(vec![err]), + }; + + prepare_target_playlist_for_persistence(base_playlist, target, &mut target_id_mapping); + if let Some(xtream_view) = xtream_playlist.as_deref_mut() { + prepare_target_playlist_for_persistence(xtream_view, target, &mut target_id_mapping); + } + + let mut prepared_outputs = prepare_target_output_playlists(target, base_playlist, xtream_playlist.as_deref()); + if let Err(error) = validate_force_empty_output_filters(target, &prepared_outputs, options) { + target_id_mapping.discard_unpersisted_changes(); + drop(target_id_mapping); + drop(file_lock); + return Err(vec![error]); + } + + for (output, prepared_output) in target.output.iter().zip(&mut prepared_outputs) { + let output_publication_plan = options.publication_plan.with_output_filter(output.filter().is_some()); + let source_playlist = match output { + TargetOutput::Xtream(_) => xtream_playlist.as_deref_mut().unwrap_or(base_playlist), + _ => &mut *base_playlist, + }; + let pl: &mut [PlaylistGroup] = if let Some(filtered_playlist) = prepared_output.as_mut() { + filtered_playlist.as_mut_slice() + } else { + source_playlist + }; + let library_empty = options.library_empty.for_filtered_playlist(pl); + let curation_empty_clusters = output_publication_plan.replacement_clusters(); + let allows_empty_base = + library_empty.replaces_empty_target() || output_publication_plan.allows_empty_base_output(); + + let result = match output { + TargetOutput::Xtream(_xtream_output) => { + persist_xtream_target_playlist( + app_config, + target, + pl, + options.accepted_empty_clusters | library_empty.replacement_clusters() | curation_empty_clusters, + persistence_mode, + ) + .await + } + TargetOutput::M3u(m3u_output) => { + m3u_write_playlist(app_config, target, m3u_output, &target_path, pl, allows_empty_base).await + } + TargetOutput::Strm(strm_output) => { + write_strm_playlist(app_config, target, strm_output, pl, allows_empty_base).await + } + TargetOutput::HdHomeRun(_hdhomerun_output) => Ok(()), + }; + + match result { + Ok(()) => { + let allows_empty_output = match output { + TargetOutput::Xtream(_) => { + !curation_empty_clusters.is_empty() + || !library_empty.replacement_clusters().is_empty() + || !options.accepted_empty_clusters.is_empty() + } + _ => allows_empty_base, + }; + if !pl.is_empty() || allows_empty_output { + let epg_pl: &[PlaylistGroup] = pl; + if let Err(err) = + epg_write_for_target(config, target, &target_path, epg, output, Some(epg_pl)).await + { + errors.push(err); + } + } + } + Err(err) => errors.push(err), + } + } + + if let Err(err) = target_id_mapping.persist() { + errors.push(TuliproxError::Config(format!("{err}"))); + } + // Keep lock until all outputs are persisted to prevent concurrent writers + // from interleaving mapping and output state for the same target. + // We must release it before loading caches below (which may acquire read locks). + drop(target_id_mapping); + drop(file_lock); + + if errors.is_empty() && target.use_memory_cache { + match playlist_state { + Some(playlist_storage) => { + match load_target_memory_cache_snapshot(app_config, target, persistence_mode).await { + Ok(storage) => playlist_storage.replace_target(&target.name, storage).await, + Err(error) => errors.push(error), + } + } + None => errors.push(TuliproxError::RepositoryPlaylist(format!( + "Target '{}' was persisted but its configured memory cache is unavailable", + target.name + ))), + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } +} + +pub async fn persist_input_media_server_playlist( + app_config: &Arc, + file_path: &Path, + playlist: Vec, +) -> (Vec, Result<(), TuliproxError>) { + persist_input_library_playlist(app_config, file_path, playlist).await +} + +pub async fn load_input_media_server_playlist( + app_config: &Arc, + file_path: &Path, +) -> Result, TuliproxError> { + load_input_local_library_playlist(app_config, file_path).await +} diff --git a/backend/repository/src/playlist_repository/publication.rs b/backend/repository/src/playlist_repository/publication.rs new file mode 100644 index 000000000..76931c1d7 --- /dev/null +++ b/backend/repository/src/playlist_repository/publication.rs @@ -0,0 +1,281 @@ +use crate::xtream_write_playlist; +use shared::{ + error::TuliproxError, + model::{ClusterFlags, PlaylistGroup, XtreamCluster}, +}; +use std::sync::Arc; +use tuliprox_core::model::{apply_filter_to_playlist, AppConfig, ConfigTarget, TargetOutput}; + +pub(super) fn playlist_has_items(playlist: &[PlaylistGroup]) -> bool { + playlist.iter().any(|group| !group.channels.is_empty()) +} + +#[derive(Debug, Clone, Copy, Default)] +pub(super) enum TargetPersistenceMode { + #[default] + Persist, + #[cfg(test)] + FailCacheReloadAt(TargetCacheReloadStage), + #[cfg(test)] + FailEmptyReplacementAt(crate::TargetEmptyReplacementFailure), +} + +#[cfg(test)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum TargetCacheReloadStage { + IdMapping, + XtreamStorage, +} + +/// Scoped persistence behavior derived from technically successful forced clusters. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InputPlaylistPersistOptions { + pub accepted_empty_clusters: ClusterFlags, +} + +impl Default for InputPlaylistPersistOptions { + fn default() -> Self { Self { accepted_empty_clusters: ClusterFlags::empty() } } +} + +/// Empty Library contribution proven by successful input jobs, before target transforms. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum LibraryEmptyPublication { + #[default] + None, + /// Empty Library contribution without permission to clear the complete target/output. + Contribution, + /// All inputs are ready, non-Library inputs are populated, and no forced-empty cluster is involved. + /// Configured transformations may remove the remaining entries from a target or individual output. + FilterableContribution, + /// All required inputs are empty Library catalogs, or the proven complete inputs + /// have been reduced to an empty target/output by the configured transformations. + CompleteTarget, +} + +impl LibraryEmptyPublication { + #[must_use] + pub const fn replaces_empty_target(self) -> bool { matches!(self, Self::CompleteTarget) } + + /// Resolve the input-level authorization against the actual prepared target/output. + /// A plain contribution never authorizes a fully empty result from a foreign input. + #[must_use] + pub fn for_filtered_playlist(self, playlist: &[PlaylistGroup]) -> Self { + match self { + Self::FilterableContribution if !playlist_has_items(playlist) => Self::CompleteTarget, + Self::None | Self::Contribution | Self::FilterableContribution | Self::CompleteTarget => self, + } + } + + pub(super) fn replacement_clusters(self) -> ClusterFlags { + match self { + Self::None => ClusterFlags::empty(), + Self::Contribution | Self::FilterableContribution => ClusterFlags::Vod | ClusterFlags::Series, + Self::CompleteTarget => ClusterFlags::all(), + } + } +} + +/// Output replacement authority derived only from a complete curation run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PlaylistPublicationPlan { + Ordinary, + CompleteCuration { + intentionally_empty_base_vod: bool, + intentionally_empty_base_series: bool, + intentionally_empty_xtream_vod: bool, + intentionally_empty_xtream_series: bool, + }, +} + +impl PlaylistPublicationPlan { + #[must_use] + pub const fn complete_curation(curated_catalog: bool, xtream_base_suppressed: bool) -> Self { + Self::complete_curation_with_filter(curated_catalog, xtream_base_suppressed, false) + } + + #[must_use] + pub const fn complete_curation_with_filter( + curated_catalog: bool, + xtream_base_suppressed: bool, + appearance_filter_configured: bool, + ) -> Self { + Self::CompleteCuration { + intentionally_empty_base_vod: curated_catalog || appearance_filter_configured, + intentionally_empty_base_series: curated_catalog || appearance_filter_configured, + intentionally_empty_xtream_vod: curated_catalog || xtream_base_suppressed || appearance_filter_configured, + intentionally_empty_xtream_series: curated_catalog + || xtream_base_suppressed + || appearance_filter_configured, + } + } + + #[must_use] + pub const fn with_output_filter(self, output_filter_configured: bool) -> Self { + if !output_filter_configured { + return self; + } + match self { + Self::Ordinary => Self::Ordinary, + Self::CompleteCuration { .. } => Self::CompleteCuration { + intentionally_empty_base_vod: true, + intentionally_empty_base_series: true, + intentionally_empty_xtream_vod: true, + intentionally_empty_xtream_series: true, + }, + } + } + + #[must_use] + pub const fn allows_empty_xtream_cluster(self, cluster: XtreamCluster) -> bool { + matches!( + (self, cluster), + (Self::CompleteCuration { intentionally_empty_xtream_vod: true, .. }, XtreamCluster::Video) + | (Self::CompleteCuration { intentionally_empty_xtream_series: true, .. }, XtreamCluster::Series) + ) + } + + #[must_use] + pub const fn allows_empty_base_output(self) -> bool { + matches!( + self, + Self::CompleteCuration { intentionally_empty_base_vod: true, .. } + | Self::CompleteCuration { intentionally_empty_base_series: true, .. } + ) + } + + #[must_use] + pub const fn allows_any_empty_output(self) -> bool { + self.allows_empty_base_output() + || self.allows_empty_xtream_cluster(XtreamCluster::Video) + || self.allows_empty_xtream_cluster(XtreamCluster::Series) + } + + pub(super) fn replacement_clusters(self) -> ClusterFlags { + let mut clusters = ClusterFlags::empty(); + if self.allows_empty_xtream_cluster(XtreamCluster::Video) { + clusters |= ClusterFlags::Vod; + } + if self.allows_empty_xtream_cluster(XtreamCluster::Series) { + clusters |= ClusterFlags::Series; + } + clusters + } +} + +/// Target persistence behavior authorized by successful input and curation results, not by Quality inference. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TargetPlaylistPersistOptions { + pub accepted_empty_clusters: ClusterFlags, + pub library_empty: LibraryEmptyPublication, + pub publication_plan: PlaylistPublicationPlan, +} + +impl Default for TargetPlaylistPersistOptions { + fn default() -> Self { + Self { + accepted_empty_clusters: ClusterFlags::empty(), + library_empty: LibraryEmptyPublication::None, + publication_plan: PlaylistPublicationPlan::Ordinary, + } + } +} + +pub(super) fn validate_target_playlist_persistence( + target: &ConfigTarget, + playlist_is_empty: bool, + options: TargetPlaylistPersistOptions, +) -> Result<(), TuliproxError> { + if !playlist_is_empty + || options.library_empty.replaces_empty_target() + || options.publication_plan.allows_any_empty_output() + { + return Ok(()); + } + if options.accepted_empty_clusters.is_empty() { + return Err(TuliproxError::RepositoryPlaylist(format!( + "Refusing to persist empty playlist for target '{}'; existing data was retained", + target.name + ))); + } + if target.output.iter().any(|output| !matches!(output, TargetOutput::Xtream(_))) { + return Err(TuliproxError::RepositoryPlaylist(format!( + "Target '{}' has non-Xtream outputs that cannot safely publish a fully empty forced result; existing data was retained", + target.name + ))); + } + Ok(()) +} + +pub(super) fn prepare_target_output_playlists( + target: &ConfigTarget, + base_playlist: &[PlaylistGroup], + xtream_playlist: Option<&[PlaylistGroup]>, +) -> Vec>> { + target + .output + .iter() + .map(|output| { + let source_playlist = match output { + TargetOutput::Xtream(_) => xtream_playlist.unwrap_or(base_playlist), + _ => base_playlist, + }; + output.filter().map(|filter| apply_filter_to_playlist(source_playlist, filter)) + }) + .collect() +} + +pub(super) fn validate_force_empty_output_filters( + target: &ConfigTarget, + prepared_outputs: &[Option>], + options: TargetPlaylistPersistOptions, +) -> Result<(), TuliproxError> { + if options.accepted_empty_clusters.is_empty() { + return Ok(()); + } + + debug_assert_eq!(target.output.len(), prepared_outputs.len()); + for (output, prepared_output) in target.output.iter().zip(prepared_outputs) { + let output_name = match output { + TargetOutput::M3u(_) => "M3U", + TargetOutput::Strm(_) => "STRM", + TargetOutput::Xtream(_) | TargetOutput::HdHomeRun(_) => continue, + }; + let Some(filtered_playlist) = prepared_output else { + continue; + }; + let curation_allows_empty = + options.publication_plan.with_output_filter(output.filter().is_some()).allows_empty_base_output(); + if !playlist_has_items(filtered_playlist) && !curation_allows_empty { + return Err(TuliproxError::RepositoryPlaylist(format!( + "Refusing to publish force-empty {output_name} output for target '{}' after its output filter; existing data was retained", + target.name + ))); + } + } + + Ok(()) +} + +pub(super) async fn persist_xtream_target_playlist( + app_config: &Arc, + target: &ConfigTarget, + playlist: &mut [PlaylistGroup], + accepted_empty_clusters: ClusterFlags, + mode: TargetPersistenceMode, +) -> Result<(), TuliproxError> { + #[cfg(test)] + if let TargetPersistenceMode::FailEmptyReplacementAt(failure) = mode { + return crate::xtream_write_playlist_with_injected_empty_replacement_failure( + app_config, + target, + playlist, + accepted_empty_clusters, + failure, + ) + .await; + } + #[cfg(not(test))] + let _ = mode; + + xtream_write_playlist(app_config, target, playlist, accepted_empty_clusters).await +} diff --git a/backend/repository/src/playlist_repository/target.rs b/backend/repository/src/playlist_repository/target.rs new file mode 100644 index 000000000..b97d5fdc1 --- /dev/null +++ b/backend/repository/src/playlist_repository/target.rs @@ -0,0 +1,222 @@ +#[cfg(test)] +use super::publication::TargetCacheReloadStage; +use super::{publication::TargetPersistenceMode, PlaylistM3uStorage, PlaylistXtreamStorage, TargetPlaylistStorage}; +use crate::{ + get_target_id_mapping_file, get_target_storage_path, m3u_get_file_path_for_db, xtream_get_file_path, + xtream_get_storage_path, BPlusTree, TargetIdMapping, VirtualIdRecord, +}; +use log::{debug, warn}; +use shared::{ + error::TuliproxError, + model::{M3uPlaylistItem, PlaylistGroup, VirtualId, XtreamCluster, XtreamPlaylistItem}, +}; +use std::path::Path; +use tuliprox_core::{ + model::{AppConfig, ConfigTarget, TargetOutput}, + utils, +}; + +pub async fn get_target_id_mapping( + cfg: &AppConfig, + target_path: &Path, + use_memory_cache: bool, +) -> Result<(TargetIdMapping, utils::FileWriteGuard), TuliproxError> { + let target_id_mapping_file = get_target_id_mapping_file(target_path); + let file_lock = cfg.file_locks.write_lock(&target_id_mapping_file).await; + let mapping_path = target_id_mapping_file.clone(); + let mapping = + tokio::task::spawn_blocking(move || TargetIdMapping::new(&mapping_path, use_memory_cache)).await.map_err( + |err| TuliproxError::Config(format!("spawn_blocking failed while creating TargetIdMapping: {err}")), + )??; + + Ok((mapping, file_lock)) +} + +async fn load_target_id_mapping_as_tree( + app_config: &AppConfig, + target_path: &Path, + target: &ConfigTarget, +) -> Result, TuliproxError> { + let target_id_mapping_file = get_target_id_mapping_file(target_path); + let _file_lock = app_config.file_locks.read_lock(&target_id_mapping_file).await; + + // Move B+Tree load to spawn_blocking to avoid blocking tokio runtime + let path_clone = target_id_mapping_file.clone(); + let target_name = target.name.clone(); + tokio::task::spawn_blocking(move || BPlusTree::::load(&path_clone)) + .await + .map_err(|e| TuliproxError::Config(format!("Blocking task failed: {e}")))? + .map_err(|err| TuliproxError::Config(format!("Could not find path for target {target_name} err:{err}"))) +} + +async fn load_xtream_playlist_as_tree( + app_config: &AppConfig, + storage_path: &Path, + cluster: XtreamCluster, +) -> Result, TuliproxError> { + let xtream_path = xtream_get_file_path(storage_path, cluster); + let file_lock = app_config.file_locks.read_lock(&xtream_path).await; + // Move B+Tree query and iteration to spawn_blocking to avoid blocking tokio runtime + let path_clone = xtream_path.clone(); + match tokio::task::spawn_blocking(move || { + let _guard = file_lock; + BPlusTree::::load(&path_clone) + }) + .await + { + Ok(Ok(tree)) => Ok(tree), + Ok(Err(err)) if err.kind() == std::io::ErrorKind::NotFound => { + debug!("No xtream {cluster} storage at {}, serving empty playlist", xtream_path.display()); + Ok(BPlusTree::new()) + } + Ok(Err(err)) => Err(TuliproxError::RepositoryXtream(format!( + "Failed to load xtream {cluster} storage {}: {err}", + xtream_path.display() + ))), + Err(join_err) => Err(TuliproxError::RepositoryXtream(format!( + "Failed to join xtream {cluster} storage load task {}: {join_err}", + xtream_path.display() + ))), + } +} + +async fn load_id_mapping_target_storage( + app_config: &AppConfig, + target: &ConfigTarget, +) -> Result, TuliproxError> { + let config = app_config.config.load(); + let target_path = get_target_storage_path(&config, target.name.as_str()) + .ok_or_else(|| TuliproxError::Config(format!("Could not find path for target {}", target.name)))?; + + load_target_id_mapping_as_tree(app_config, &target_path, target).await +} + +pub async fn load_xtream_target_storage( + app_config: &AppConfig, + target: &ConfigTarget, +) -> Result { + let config = app_config.config.load(); + + let storage_path = xtream_get_storage_path(&config, target.name.as_str()).ok_or_else(|| { + TuliproxError::Config(format!("Could not find path for target {} xtream output", target.name)) + })?; + + let live_storage = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Live).await?; + let vod_storage = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Video).await?; + let series_storage = load_xtream_playlist_as_tree(app_config, &storage_path, XtreamCluster::Series).await?; + + Ok(PlaylistXtreamStorage { live: live_storage, vod: vod_storage, series: series_storage }) +} + +pub async fn load_m3u_target_storage( + app_config: &AppConfig, + target: &ConfigTarget, +) -> Result { + let config = app_config.config.load(); + let target_path = get_target_storage_path(&config, target.name.as_str()) + .ok_or_else(|| TuliproxError::Config(format!("Could not find path for target {}", target.name)))?; + + let m3u_path = m3u_get_file_path_for_db(&target_path); + let file_lock = app_config.file_locks.read_lock(&m3u_path).await; + + let path_clone = m3u_path.clone(); + match tokio::task::spawn_blocking(move || { + let _guard = file_lock; + BPlusTree::::load(&path_clone) + }) + .await + { + Ok(Ok(tree)) => Ok(tree), + Ok(Err(err)) if err.kind() == std::io::ErrorKind::NotFound => { + debug!("No m3u storage at {}, serving empty playlist", m3u_path.display()); + Ok(BPlusTree::new()) + } + Ok(Err(err)) => { + Err(TuliproxError::RepositoryM3u(format!("Failed to load m3u storage {}: {err}", m3u_path.display()))) + } + Err(join_err) => Err(TuliproxError::RepositoryM3u(format!( + "Failed to join m3u storage load task {}: {join_err}", + m3u_path.display() + ))), + } +} + +fn target_cache_reload_error(target: &ConfigTarget, component: &str, error: impl std::fmt::Display) -> TuliproxError { + TuliproxError::RepositoryPlaylist(format!( + "Target '{}' was persisted but its {component} could not be reloaded into the memory cache: {error}", + target.name + )) +} + +pub(super) async fn load_target_memory_cache_snapshot( + app_config: &AppConfig, + target: &ConfigTarget, + mode: TargetPersistenceMode, +) -> Result { + #[cfg(not(test))] + let _ = mode; + + #[cfg(test)] + if matches!(mode, TargetPersistenceMode::FailCacheReloadAt(TargetCacheReloadStage::IdMapping)) { + return Err(target_cache_reload_error(target, "ID mapping", "injected reload failure")); + } + let id_mapping = load_id_mapping_target_storage(app_config, target) + .await + .map_err(|error| target_cache_reload_error(target, "ID mapping", error))?; + + let xtream = if target.output.iter().any(|output| matches!(output, TargetOutput::Xtream(_))) { + #[cfg(test)] + if matches!(mode, TargetPersistenceMode::FailCacheReloadAt(TargetCacheReloadStage::XtreamStorage)) { + return Err(target_cache_reload_error(target, "Xtream storage", "injected reload failure")); + } + Some( + load_xtream_target_storage(app_config, target) + .await + .map_err(|error| target_cache_reload_error(target, "Xtream storage", error))?, + ) + } else { + None + }; + + let m3u = if target.output.iter().any(|output| matches!(output, TargetOutput::M3u(_))) { + Some( + load_m3u_target_storage(app_config, target) + .await + .map_err(|error| target_cache_reload_error(target, "M3U storage", error))?, + ) + } else { + None + }; + + Ok(TargetPlaylistStorage { xtream, m3u, id_mapping: Some(id_mapping) }) +} + +pub(super) async fn publish_all_cluster_catalogs( + app_config: &AppConfig, + storage_path: &Path, + input_name: &str, + persisted_playlist: Vec, +) -> (Vec, Option) { + let mut live_groups = Vec::new(); + let mut vod_groups = Vec::new(); + let mut series_groups = Vec::new(); + for group in &persisted_playlist { + match group.xtream_cluster { + XtreamCluster::Live => live_groups.push(group.title.to_string()), + XtreamCluster::Video => vod_groups.push(group.title.to_string()), + XtreamCluster::Series => series_groups.push(group.title.to_string()), + } + } + for (cluster, groups) in + [(XtreamCluster::Live, live_groups), (XtreamCluster::Video, vod_groups), (XtreamCluster::Series, series_groups)] + { + if let Err(publish_err) = + crate::publish_raw_group_catalog(storage_path, input_name, cluster, groups, &app_config.file_locks).await + { + warn!( + "Playlist data for input '{input_name}' was persisted, but publishing its raw group catalog for cluster {cluster:?} failed: {publish_err}" + ); + } + } + (persisted_playlist, None) +} diff --git a/backend/repository/src/playlist_repository/tests.rs b/backend/repository/src/playlist_repository/tests.rs new file mode 100644 index 000000000..15c30194b --- /dev/null +++ b/backend/repository/src/playlist_repository/tests.rs @@ -0,0 +1,396 @@ +use super::{ + assign_local_series_info_episode_key, assign_media_server_series_info_episode, + get_input_media_server_playlist_file_path, materialize_media_server_series_info_episodes, + normalize_target_playlist_epg_ids, persist_playlist_views, persist_playlist_with_mode, + publication::{playlist_has_items, validate_target_playlist_persistence, TargetPersistenceMode}, + rewrite_local_series_info_episode_virtual_id, rewrite_series_episode_parent_virtual_ids, + rewrite_series_info_episode_virtual_id, skipped_clusters, LocalEpisodeKey, PlaylistPublicationPlan, + ProviderEpisodeKey, TargetCacheReloadStage, TargetPlaylistPersistOptions, +}; +use crate::{ + get_series_cat_collection_path, get_target_storage_path, get_vod_cat_collection_path, load_m3u_target_storage, + load_xtream_target_storage, strm_get_file_paths, xtream_get_storage_path, BPlusTreeQuery, PlaylistStorageState, + TargetIdMapping, VirtualIdRecord, +}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use shared::{ + foundation::get_filter, + model::{ + ClusterFlags, ConfigPaths, ConfigTargetDto, ConfigTargetOptions, EpgOutputOptions, EpisodeStreamProperties, + M3uPlaylistItem, M3uTargetOutputDto, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemHeader, + PlaylistItemType, ProcessingOrder, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, + SeriesStreamDetailSeasonProperties, SeriesStreamProperties, StreamProperties, StrmExportStyle, + StrmTargetOutputDto, TargetOutputDto, UUIDType, VirtualId, XtreamCluster, XtreamPlaylistItem, + XtreamTargetOutputDto, + }, + utils::{hash_string_as_hex, Internable}, +}; +use std::{collections::HashMap, path::Path, sync::Arc}; +use tuliprox_core::{ + model::{ + ApiProxyConfig, AppConfig, Config, ConfigTarget, CustomStreamResponse, HdHomeRunConfig, M3uTargetOutput, + MediaToolCapabilities, SourcesConfig, StagedFilter, StrmTargetFlagsSet, StrmTargetOutput, TargetExecutionPlan, + TargetOutput, XtreamTargetFlagsSet, XtreamTargetOutput, + }, + utils::{normalize_string_path, FileLockManager}, +}; + +fn target_with_outputs(output: Vec) -> ConfigTarget { + target_with_options("target-empty-guard", output, false) +} + +fn target_with_options(name: &str, output: Vec, use_memory_cache: bool) -> ConfigTarget { + ConfigTarget { + curation: None, + id: 1, + enabled: true, + name: name.to_string(), + options: None, + sort: None, + filter: StagedFilter::default(), + output, + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::new(None)), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: TargetExecutionPlan::default(), + watch: None, + use_memory_cache, + } +} + +fn test_app_config(storage_dir: &Path) -> Arc { + Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config { + storage_dir: storage_dir.to_string_lossy().into_owned(), + ..Config::default() + })), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::::default()), + api_proxy: Arc::new(ArcSwapOption::::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + }) +} + +fn target_group(cluster: XtreamCluster, id: u32, name: &str) -> PlaylistGroup { + let name = name.intern(); + let mut header = PlaylistItemHeader { + id: id.to_string().intern(), + input_stream_id: id.to_string().intern(), + name: Arc::clone(&name), + title: Arc::clone(&name), + group: Arc::clone(&name), + input_name: "target-test-input".intern(), + item_type: PlaylistItemType::from(cluster), + xtream_cluster: cluster, + category_id: id, + ..PlaylistItemHeader::default() + }; + header.gen_uuid(); + PlaylistGroup { id, title: name, channels: vec![PlaylistItem { header }], xtream_cluster: cluster } +} + +fn xtream_target(name: &str, use_memory_cache: bool) -> ConfigTarget { + target_with_options( + name, + vec![TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::new(), + trakt: None, + filter: None, + })], + use_memory_cache, + ) +} + +async fn cached_target_signature( + playlist_state: &PlaylistStorageState, + target_name: &str, +) -> (usize, Vec<(XtreamCluster, u32, Arc)>) { + let cache = playlist_state.data.read().await; + let target = cache.get(target_name).expect("target cache"); + let mapping_len = target.id_mapping.as_ref().expect("cached ID mapping").len(); + let xtream = target.xtream.as_ref().expect("cached Xtream storage"); + let mut items = Vec::new(); + for (cluster, storage) in [ + (XtreamCluster::Live, &xtream.live), + (XtreamCluster::Video, &xtream.vod), + (XtreamCluster::Series, &xtream.series), + ] { + for virtual_id in 1..=16 { + if let Some(item) = storage.query(&virtual_id) { + items.push((cluster, virtual_id, Arc::clone(&item.name))); + } + } + } + (mapping_len, items) +} + +async fn seed_memory_cached_xtream_target( + app_config: &Arc, + playlist_state: &Arc, + target: &ConfigTarget, +) { + let mut baseline = vec![ + target_group(XtreamCluster::Live, 101, "old-live"), + target_group(XtreamCluster::Video, 201, "old-vod"), + target_group(XtreamCluster::Series, 301, "old-series"), + ]; + persist_playlist_with_mode( + app_config, + &mut baseline, + None, + target, + Some(playlist_state), + TargetPlaylistPersistOptions::default(), + TargetPersistenceMode::Persist, + ) + .await + .expect("baseline target persistence"); +} + +use tempfile::tempdir; + +fn curation_persist_options(publication_plan: PlaylistPublicationPlan) -> TargetPlaylistPersistOptions { + TargetPlaylistPersistOptions { publication_plan, ..TargetPlaylistPersistOptions::default() } +} + +fn target_test_app_config(storage_dir: &Path) -> Arc { + Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config { + storage_dir: storage_dir.to_string_lossy().into_owned(), + ..Config::default() + })), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::::default()), + api_proxy: Arc::new(ArcSwapOption::::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + }) +} + +fn mixed_output_target() -> ConfigTarget { + let xtream = XtreamTargetOutputDto { + t_filter: Some( + shared::foundation::get_filter(r#"Group = "Curated alias""#, None).expect("Xtream output filter"), + ), + ..Default::default() + }; + let mut m3u = M3uTargetOutputDto { filename: Some("curated.m3u".to_string()), ..Default::default() }; + m3u.t_filter = Some(shared::foundation::get_filter(r#"Group = "Base movie""#, None).expect("M3U output filter")); + ConfigTarget::from(&ConfigTargetDto { + name: "curated-output-test".to_string(), + output: vec![ + TargetOutputDto::Xtream(xtream), + TargetOutputDto::M3u(m3u), + TargetOutputDto::Strm(StrmTargetOutputDto { + directory: "strm".to_string(), + flat: true, + cleanup: true, + ..StrmTargetOutputDto::default() + }), + ], + use_memory_cache: true, + ..ConfigTargetDto::default() + }) +} + +fn target_video_group(title: &str, uuid: UUIDType) -> PlaylistGroup { + PlaylistGroup { + id: 1, + title: title.intern(), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { + id: "1".intern(), + name: title.intern(), + title: title.intern(), + group: title.intern(), + url: format!("http://example.invalid/{title}").intern(), + uuid, + item_type: PlaylistItemType::Video, + xtream_cluster: XtreamCluster::Video, + ..PlaylistItemHeader::default() + }, + }], + xtream_cluster: XtreamCluster::Video, + } +} + +fn strm_files_below(path: &Path) -> Vec { + let mut files = Vec::new(); + let Ok(entries) = std::fs::read_dir(path) else { return files }; + for entry in entries.flatten() { + let entry_path = entry.path(); + if entry_path.is_dir() { + files.extend(strm_files_below(&entry_path)); + } else if entry_path.extension().is_some_and(|extension| extension == "strm") { + files.push(entry_path); + } + } + files +} + +fn epg_normalization_options(enabled: bool) -> ConfigTargetOptions { + ConfigTargetOptions { + epg_output: EpgOutputOptions { lowercase_ids: enabled, ..EpgOutputOptions::default() }, + ..ConfigTargetOptions::default() + } +} + +fn epg_normalization_playlist() -> Vec { + let channel = |epg_channel_id: Option<&str>, name: &str| PlaylistItem { + header: PlaylistItemHeader { + name: name.intern(), + title: "Visible Title".intern(), + group: "Visible Group".intern(), + epg_channel_id: epg_channel_id.map(Internable::intern), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + + vec![PlaylistGroup { + id: 1, + title: "Live".intern(), + channels: vec![ + channel(Some("Example.Channel"), "Mixed Case"), + channel(Some("already.lower"), "Lowercase"), + channel(Some(""), "Empty"), + channel(None, "Missing"), + ], + xtream_cluster: XtreamCluster::Live, + }] +} + +fn make_local_series_info(series_uuid: &str, episodes: Vec<(u32, &str, &str)>) -> PlaylistItem { + let episode_props = episodes + .into_iter() + .map(|(id, title, direct_source)| SeriesStreamDetailEpisodeProperties { + id, + episode_num: 0, + season: 0, + title: title.intern(), + container_extension: "".intern(), + custom_sid: None, + added: "".intern(), + direct_source: direct_source.intern(), + tmdb: None, + release_date: "".intern(), + series_release_date: None, + plot: None, + crew: None, + duration_secs: 0, + duration: "".intern(), + movie_image: "".intern(), + bitrate: 0, + rating: None, + video: None, + audio: None, + }) + .collect(); + + PlaylistItem { + header: PlaylistItemHeader { + id: series_uuid.intern(), + item_type: PlaylistItemType::LocalSeriesInfo, + xtream_cluster: XtreamCluster::Series, + additional_properties: Some(StreamProperties::Series(Box::new(SeriesStreamProperties { + name: "Series".intern(), + details: Some(SeriesStreamDetailProperties { + year: None, + seasons: None, + episodes: Some(episode_props), + }), + ..SeriesStreamProperties::default() + }))), + ..PlaylistItemHeader::default() + }, + } +} + +fn make_local_series_episode(series_uuid: &str, direct_source: &str, virtual_id: u32) -> PlaylistItemHeader { + PlaylistItemHeader { + parent_code: series_uuid.intern(), + url: direct_source.intern(), + item_type: PlaylistItemType::LocalSeries, + xtream_cluster: XtreamCluster::Series, + virtual_id: VirtualId::new(virtual_id), + ..PlaylistItemHeader::default() + } +} + +fn make_media_server_episode( + series_uuid: &str, + item_id: &str, + virtual_id: u32, + season: u32, + episode: u32, +) -> PlaylistItemHeader { + PlaylistItemHeader { + id: format!("media-server:server:shows:episode:{item_id}").intern(), + name: format!("Episode {episode}").intern(), + title: format!("Episode {episode}").intern(), + parent_code: series_uuid.intern(), + url: format!("media-server://plex/server/{item_id}?part_key=%2Flibrary%2Fparts%2Fredacted").intern(), + item_type: PlaylistItemType::Series, + xtream_cluster: XtreamCluster::Series, + virtual_id: VirtualId::new(virtual_id), + additional_properties: Some(StreamProperties::Episode(Box::new(EpisodeStreamProperties { + episode_id: 0, + episode, + season, + added: Some("1700000000".intern()), + release_date: Some("2024-02-03".intern()), + series_release_date: Some("2024-01-01".intern()), + plot: Some("Episode summary".intern()), + tmdb: Some(67890), + movie_image: "media-server://image/plex/server/episode?image_path=%2Flibrary%2Fmetadata%2Fredacted%2Fthumb" + .intern(), + container_extension: "mkv".intern(), + video: Some(r#"{"codec_name":"h264"}"#.intern()), + audio: Some(r#"{"codec_name":"aac"}"#.intern()), + }))), + ..PlaylistItemHeader::default() + } +} + +mod lifecycle; +mod playlist; +mod policy; +mod recovery; +mod storage; +mod transport; diff --git a/backend/repository/src/playlist_repository/tests/lifecycle.rs b/backend/repository/src/playlist_repository/tests/lifecycle.rs new file mode 100644 index 000000000..355e9b3f4 --- /dev/null +++ b/backend/repository/src/playlist_repository/tests/lifecycle.rs @@ -0,0 +1,58 @@ +use super::*; + +#[tokio::test] +async fn intentional_empty_strm_cleanup_false_removes_indexed_files_but_keeps_unmanaged_files() { + let directory = tempdir().expect("tempdir"); + let app_config = target_test_app_config(directory.path()); + let target = ConfigTarget::from(&ConfigTargetDto { + name: "curated-strm-retention-test".to_string(), + output: vec![TargetOutputDto::Strm(StrmTargetOutputDto { + directory: "strm-retained".to_string(), + flat: true, + cleanup: false, + ..StrmTargetOutputDto::default() + })], + ..ConfigTargetDto::default() + }); + let mut seeded = + vec![target_video_group("Retained movie", UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000029"))]; + let initial = persist_playlist_views( + &app_config, + &mut seeded, + None, + None, + &target, + None, + curation_persist_options(PlaylistPublicationPlan::Ordinary), + ) + .await; + assert!(initial.is_ok(), "initial STRM persist failed: {initial:?}"); + let strm_root = directory.path().join("strm-retained"); + let managed_files = strm_files_below(&strm_root); + assert_eq!(managed_files.len(), 1); + let managed_file = managed_files[0].clone(); + let unmanaged_file = strm_root.join("unmanaged.strm"); + std::fs::write(&unmanaged_file, "unmanaged").expect("unmanaged STRM fixture"); + + let mut empty = Vec::new(); + let published = persist_playlist_views( + &app_config, + &mut empty, + None, + None, + &target, + None, + curation_persist_options(PlaylistPublicationPlan::complete_curation(true, false)), + ) + .await; + assert!(published.is_ok(), "empty STRM persist failed: {published:?}"); + + assert!(!managed_file.exists(), "cleanup=false removes files tracked by the managed index"); + assert!(unmanaged_file.exists(), "cleanup=false does not scan and remove unmanaged files"); + let target_storage = { + let config = app_config.config.load(); + get_target_storage_path(&config, &target.name).expect("target storage") + }; + let index = strm_get_file_paths(&hash_string_as_hex(&normalize_string_path("strm-retained")), &target_storage); + assert!(std::fs::read_to_string(index).expect("empty STRM index").is_empty()); +} diff --git a/backend/repository/src/playlist_repository/tests/playlist.rs b/backend/repository/src/playlist_repository/tests/playlist.rs new file mode 100644 index 000000000..197ddd82a --- /dev/null +++ b/backend/repository/src/playlist_repository/tests/playlist.rs @@ -0,0 +1,152 @@ +use super::*; + +#[test] +fn target_empty_playlist_without_force_authorization_remains_rejected() { + let target = target_with_outputs(vec![TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::new(), + trakt: None, + filter: None, + })]); + let error = validate_target_playlist_persistence(&target, true, TargetPlaylistPersistOptions::default()) + .expect_err("unauthorized empty target must remain rejected"); + assert!(error.to_string().contains("Refusing to persist empty playlist")); +} + +#[test] +fn complete_curation_empty_authorization_is_projection_and_cluster_scoped() { + let compatibility = PlaylistPublicationPlan::complete_curation(false, false); + assert!(!compatibility.allows_any_empty_output()); + + let suppressed_xtream_base = PlaylistPublicationPlan::complete_curation(false, true); + assert!(!suppressed_xtream_base.allows_empty_base_output()); + assert!(!suppressed_xtream_base.allows_empty_xtream_cluster(XtreamCluster::Live)); + assert!(suppressed_xtream_base.allows_empty_xtream_cluster(XtreamCluster::Video)); + assert!(suppressed_xtream_base.allows_empty_xtream_cluster(XtreamCluster::Series)); + + let persist_filtered = PlaylistPublicationPlan::complete_curation_with_filter(false, false, true); + assert!(persist_filtered.allows_empty_base_output()); + assert!(persist_filtered.allows_empty_xtream_cluster(XtreamCluster::Video)); + let output_filtered = compatibility.with_output_filter(true); + assert!(output_filtered.allows_empty_base_output()); + assert_eq!(PlaylistPublicationPlan::Ordinary.with_output_filter(true), PlaylistPublicationPlan::Ordinary); +} + +#[tokio::test] +async fn intentional_empty_curation_never_authorizes_empty_live_cluster_replacement() { + let directory = tempdir().expect("tempdir"); + let app_config = target_test_app_config(directory.path()); + let target = ConfigTarget::from(&ConfigTargetDto { + name: "curated-live-retention-test".to_string(), + output: vec![TargetOutputDto::Xtream(XtreamTargetOutputDto::default())], + ..ConfigTargetDto::default() + }); + let mut live = vec![ + PlaylistGroup { + id: 1, + title: "Live".intern(), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { + id: "1".intern(), + name: "Live channel".intern(), + title: "Live channel".intern(), + group: "Live".intern(), + url: "http://example.invalid/live".intern(), + uuid: UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000031"), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }], + xtream_cluster: XtreamCluster::Live, + }, + target_video_group( + "Previously published movie", + UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000032"), + ), + ]; + + let initial = persist_playlist_views( + &app_config, + &mut live, + None, + None, + &target, + None, + curation_persist_options(PlaylistPublicationPlan::Ordinary), + ) + .await; + assert!(initial.is_ok(), "initial Live persist failed: {initial:?}"); + + let mut ordinary_live_only = vec![live[0].clone()]; + let ordinary = persist_playlist_views( + &app_config, + &mut ordinary_live_only, + None, + None, + &target, + None, + curation_persist_options(PlaylistPublicationPlan::Ordinary), + ) + .await; + assert!(ordinary.is_ok(), "ordinary Live-only persist failed: {ordinary:?}"); + let ordinary_storage = load_xtream_target_storage(&app_config, &target).await.expect("ordinary storage"); + assert_eq!(ordinary_storage.live.len(), 1); + assert_eq!(ordinary_storage.vod.len(), 1, "ordinary refresh retains an absent cluster"); + + let mut empty_standard = Vec::new(); + let mut empty_xtream = Vec::new(); + let curated_empty = persist_playlist_views( + &app_config, + &mut empty_standard, + Some(&mut empty_xtream), + None, + &target, + None, + curation_persist_options(PlaylistPublicationPlan::complete_curation(true, false)), + ) + .await; + assert!(curated_empty.is_ok(), "curated empty persist failed: {curated_empty:?}"); + + let storage = load_xtream_target_storage(&app_config, &target).await.expect("Xtream storage"); + assert_eq!(storage.live.len(), 1); + assert!(storage.vod.is_empty()); + assert!(storage.series.is_empty()); +} + +#[test] +fn target_playlist_epg_normalization_is_consistent_for_m3u_and_xtream() { + let options = epg_normalization_options(true); + let mut playlist = epg_normalization_playlist(); + let lowercase_id = + Arc::clone(playlist[0].channels[1].header.epg_channel_id.as_ref().expect("lowercase EPG ID should exist")); + + normalize_target_playlist_epg_ids(&mut playlist, Some(&options)); + + let mixed = &playlist[0].channels[0]; + assert_eq!(mixed.header.epg_channel_id.as_deref(), Some("example.channel")); + assert_eq!(mixed.header.name.as_ref(), "Mixed Case"); + assert_eq!(mixed.header.title.as_ref(), "Visible Title"); + assert_eq!(mixed.header.group.as_ref(), "Visible Group"); + assert!(Arc::ptr_eq( + playlist[0].channels[1].header.epg_channel_id.as_ref().expect("lowercase EPG ID should remain"), + &lowercase_id, + )); + assert_eq!(playlist[0].channels[2].header.epg_channel_id.as_deref(), Some("")); + assert!(playlist[0].channels[3].header.epg_channel_id.is_none()); + + let m3u = M3uPlaylistItem::from(mixed); + let xtream = XtreamPlaylistItem::from(mixed); + assert_eq!(m3u.epg_channel_id.as_deref(), Some("example.channel")); + assert_eq!(xtream.epg_channel_id.as_deref(), Some("example.channel")); + assert!(m3u.to_m3u(None, false).contains(r#"tvg-id="example.channel""#)); +} + +#[test] +fn target_playlist_epg_normalization_preserves_ids_when_disabled() { + let options = epg_normalization_options(false); + let mut playlist = epg_normalization_playlist(); + + normalize_target_playlist_epg_ids(&mut playlist, Some(&options)); + + assert_eq!(playlist[0].channels[0].header.epg_channel_id.as_deref(), Some("Example.Channel")); +} diff --git a/backend/repository/src/playlist_repository/tests/policy.rs b/backend/repository/src/playlist_repository/tests/policy.rs new file mode 100644 index 000000000..78f2a9937 --- /dev/null +++ b/backend/repository/src/playlist_repository/tests/policy.rs @@ -0,0 +1,467 @@ +use super::*; + +#[test] +fn target_force_empty_authorization_is_limited_to_xtream_outputs() { + let xtream_target = target_with_outputs(vec![TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::new(), + trakt: None, + filter: None, + })]); + let options = TargetPlaylistPersistOptions { + accepted_empty_clusters: ClusterFlags::Vod, + ..TargetPlaylistPersistOptions::default() + }; + assert!(validate_target_playlist_persistence(&xtream_target, true, options).is_ok()); + + let mixed_target = target_with_outputs(vec![ + TargetOutput::Xtream(XtreamTargetOutput { flags: XtreamTargetFlagsSet::new(), trakt: None, filter: None }), + TargetOutput::M3u(M3uTargetOutput { + filename: None, + include_type_in_url: false, + mask_redirect_url: false, + filter: None, + }), + ]); + let error = validate_target_playlist_persistence(&mixed_target, true, options) + .expect_err("fully empty mixed output must fail before persistence"); + assert!(error.to_string().contains("non-Xtream outputs")); +} + +#[tokio::test] +async fn target_empty_replacement_failures_leave_the_memory_cache_unchanged() { + for failure in [ + crate::TargetEmptyReplacementFailure::CategoryPersistence, + crate::TargetEmptyReplacementFailure::BTreePersistence, + crate::TargetEmptyReplacementFailure::Publication, + ] { + let directory = tempdir().expect("temporary storage"); + let app_config = test_app_config(directory.path()); + let playlist_state = Arc::new(PlaylistStorageState::new()); + let target = xtream_target(&format!("empty-persist-{failure:?}"), true); + seed_memory_cached_xtream_target(&app_config, &playlist_state, &target).await; + let before = cached_target_signature(&playlist_state, &target.name).await; + let mut candidate = vec![ + target_group(XtreamCluster::Live, 102, "new-live"), + target_group(XtreamCluster::Series, 302, "new-series"), + ]; + + let errors = persist_playlist_with_mode( + &app_config, + &mut candidate, + None, + &target, + Some(&playlist_state), + TargetPlaylistPersistOptions { + accepted_empty_clusters: ClusterFlags::Vod, + ..TargetPlaylistPersistOptions::default() + }, + TargetPersistenceMode::FailEmptyReplacementAt(failure), + ) + .await + .expect_err("injected empty replacement must fail"); + + assert!(errors.iter().any(|error| error.to_string().contains("target cluster failed"))); + assert_eq!(cached_target_signature(&playlist_state, &target.name).await, before); + let persisted = load_xtream_target_storage(&app_config, &target).await.expect("retained target storage"); + assert_eq!(persisted.vod.len(), 1); + } +} + +#[test] +// Unchanged by the move; rustfmt reflowed it past the 100-line threshold. +#[allow(clippy::too_many_lines)] +fn materializes_media_server_series_info_episodes_after_target_virtual_ids_are_assigned() { + let series_uuid = "123e4567-e89b-12d3-a456-426614174111"; + let mut series_info = PlaylistItem { + header: PlaylistItemHeader { + uuid: UUIDType::from_valid_uuid(series_uuid), + id: "media-server:server:shows:series:series".intern(), + name: "Media Server Series".intern(), + title: "Media Server Series".intern(), + url: "media-server://unavailable/server/shows/series".intern(), + item_type: PlaylistItemType::SeriesInfo, + xtream_cluster: XtreamCluster::Series, + additional_properties: Some(StreamProperties::Series(Box::new(SeriesStreamProperties { + name: "Media Server Series".intern(), + details: Some(SeriesStreamDetailProperties { + year: Some(2024), + seasons: Some(vec![SeriesStreamDetailSeasonProperties { + name: "Season 1".intern(), + season_number: 1, + episode_count: 2, + overview: Some("season summary".intern()), + air_date: Some("2024-01-01".intern()), + cover: None, + cover_tmdb: None, + cover_big: None, + duration: None, + }]), + episodes: None, + }), + ..SeriesStreamProperties::default() + }))), + ..PlaylistItemHeader::default() + }, + }; + let series_parent_code = series_info.header.uuid.to_string(); + let media_episode_two = + PlaylistItem { header: make_media_server_episode(&series_parent_code, "episode-two", 7002, 1, 2) }; + let media_episode_one = + PlaylistItem { header: make_media_server_episode(&series_parent_code, "episode-one", 7001, 1, 1) }; + let malformed_media_episode = PlaylistItem { + header: PlaylistItemHeader { + id: "media-server:server:shows:episode:malformed".intern(), + parent_code: series_parent_code.clone().intern(), + url: "media-server://plex/server/malformed?part_key=%2Flibrary%2Fparts%2Fredacted".intern(), + item_type: PlaylistItemType::Series, + xtream_cluster: XtreamCluster::Series, + virtual_id: VirtualId::new(7003), + additional_properties: None, + ..PlaylistItemHeader::default() + }, + }; + let provider_episode = PlaylistItem { + header: PlaylistItemHeader { + id: "999".intern(), + parent_code: series_parent_code.clone().intern(), + url: "http://provider.example.invalid/series/999.mkv".intern(), + item_type: PlaylistItemType::Series, + xtream_cluster: XtreamCluster::Series, + virtual_id: VirtualId::new(7999), + ..PlaylistItemHeader::default() + }, + }; + + let mut media_server_series = HashMap::, Vec>::new(); + assign_media_server_series_info_episode(&mut media_server_series, &media_episode_two.header); + assign_media_server_series_info_episode(&mut media_server_series, &provider_episode.header); + assign_media_server_series_info_episode(&mut media_server_series, &malformed_media_episode.header); + assign_media_server_series_info_episode(&mut media_server_series, &media_episode_one.header); + + let mut playlist = vec![PlaylistGroup { + id: 1, + title: "Media Server Series".intern(), + channels: vec![series_info, media_episode_two, provider_episode, malformed_media_episode, media_episode_one], + xtream_cluster: XtreamCluster::Series, + }]; + + materialize_media_server_series_info_episodes(&mut playlist, &media_server_series); + series_info = playlist[0].channels[0].clone(); + + let Some(StreamProperties::Series(series)) = series_info.header.additional_properties.as_ref() else { + panic!("missing series properties"); + }; + let details = series.details.as_ref().expect("series details should be present"); + assert_eq!(details.year, Some(2024)); + assert_eq!(details.seasons.as_ref().expect("seasons should be preserved")[0].episode_count, 2); + let episodes = details.episodes.as_ref().expect("media-server episodes should be materialized"); + assert_eq!(episodes.len(), 2); + assert_eq!(episodes[0].id, 7001); + assert_eq!(episodes[0].episode_num, 1); + assert_eq!(episodes[0].season, 1); + assert_eq!(episodes[0].title.as_ref(), "Episode 1"); + assert_eq!(episodes[0].container_extension.as_ref(), "mkv"); + assert_eq!(episodes[0].release_date.as_ref(), "2024-02-03"); + assert_eq!(episodes[0].series_release_date.as_deref(), Some("2024-01-01")); + assert_eq!(episodes[0].plot.as_deref(), Some("Episode summary")); + assert_eq!(episodes[0].tmdb, Some(67890)); + assert_eq!(episodes[0].direct_source.as_ref(), ""); + assert_eq!( + episodes[0].movie_image.as_ref(), + "media-server://image/plex/server/episode?image_path=%2Flibrary%2Fmetadata%2Fredacted%2Fthumb" + ); + assert!(episodes[0].video.as_deref().is_some_and(|video| video.contains("h264"))); + assert!(episodes[0].audio.as_deref().is_some_and(|audio| audio.contains("aac"))); + assert_eq!(episodes[1].id, 7002); + assert!(!episodes.iter().any(|episode| episode.id == 7003)); +} + +#[test] +fn rewrite_series_episode_parent_virtual_ids_updates_local_episode_mapping() { + let series_uuid = "123e4567-e89b-12d3-a456-426614174000"; + let series_uuid_type = UUIDType::from_valid_uuid(series_uuid); + let episode_uuid = UUIDType::from_valid_uuid("123e4567-e89b-12d3-a456-426614174001"); + let dir = tempdir().expect("tempdir"); + let mapping_path = dir.path().join("id_mapping.db"); + let mut target_id_mapping = TargetIdMapping::new(&mapping_path, false).expect("mapping"); + + let mut playlist = vec![PlaylistGroup { + id: 1, + title: "Series".intern(), + channels: vec![ + PlaylistItem { + header: PlaylistItemHeader { + uuid: episode_uuid, + id: "101".intern(), + parent_code: series_uuid.intern(), + url: "/library/episode1.mkv".intern(), + item_type: PlaylistItemType::LocalSeries, + xtream_cluster: XtreamCluster::Series, + ..PlaylistItemHeader::default() + }, + }, + PlaylistItem { + header: PlaylistItemHeader { + uuid: series_uuid_type, + id: series_uuid.intern(), + item_type: PlaylistItemType::LocalSeriesInfo, + xtream_cluster: XtreamCluster::Series, + ..PlaylistItemHeader::default() + }, + }, + ], + xtream_cluster: XtreamCluster::Series, + }]; + + for (idx, channel) in playlist[0].channels.iter_mut().enumerate() { + let uuid = channel.header.uuid; + let provider_id = channel.header.get_provider_id().unwrap_or_default(); + let item_type = channel.header.item_type; + channel.header.virtual_id = + target_id_mapping.get_and_update_virtual_id(&uuid, provider_id, item_type, VirtualId::new(0)); + channel.header.source_ordinal = u32::try_from(idx + 1).expect("ordinal"); + } + + let series_virtual_id = playlist[0].channels[1].header.virtual_id; + let episode_virtual_id = playlist[0].channels[0].header.virtual_id; + + rewrite_series_episode_parent_virtual_ids(&mut playlist, &mut target_id_mapping); + target_id_mapping.persist().expect("persist"); + + let mut query = BPlusTreeQuery::::try_new(&mapping_path).expect("query"); + let record = query.query_zero_copy(&episode_virtual_id.get()).expect("query ok").expect("record missing"); + + assert_eq!(record.parent_virtual_id, series_virtual_id); + assert_eq!(playlist[0].channels[0].header.virtual_id, episode_virtual_id); +} + +#[test] +fn rewrite_series_episode_parent_virtual_ids_updates_provider_episode_mapping_using_series_info_uuid() { + let dir = tempdir().expect("tempdir"); + let mapping_path = dir.path().join("id_mapping.db"); + let mut target_id_mapping = TargetIdMapping::new(&mapping_path, false).expect("mapping"); + + let input_name = "provider-input".intern(); + let xtream_series_info = XtreamPlaylistItem { + virtual_id: VirtualId::new(0), + provider_id: 9001, + name: "Provider Series".intern(), + logo: "".intern(), + logo_small: "".intern(), + group: "Provider Series".intern(), + title: "Provider Series".intern(), + parent_code: "".intern(), + rec: "".intern(), + url: "http://provider.example.com/series/user/pass/9001".intern(), + epg_channel_id: None, + xtream_cluster: XtreamCluster::Series, + additional_properties: None, + item_type: PlaylistItemType::SeriesInfo, + category_id: 0, + input_name: Arc::clone(&input_name), + channel_no: 0, + source_ordinal: 0, + input_stream_id: "9001".intern(), + upstream_user_agent: None, + }; + let provider_parent_code = xtream_series_info.get_uuid().intern(); + let xtream_provider_episode = XtreamPlaylistItem { + virtual_id: VirtualId::new(0), + provider_id: 201, + name: "Episode 1".intern(), + logo: "".intern(), + logo_small: "".intern(), + group: "Provider Series".intern(), + title: "Episode 1".intern(), + parent_code: provider_parent_code, + rec: "".intern(), + url: "http://provider.example.com/series/user/pass/201.mkv".intern(), + epg_channel_id: None, + xtream_cluster: XtreamCluster::Series, + additional_properties: None, + item_type: PlaylistItemType::Series, + category_id: 0, + input_name, + channel_no: 0, + source_ordinal: 0, + input_stream_id: "201".intern(), + upstream_user_agent: None, + }; + let provider_episode = PlaylistItem::from(&xtream_provider_episode); + let mut series_info = PlaylistItem::from(&xtream_series_info); + series_info.header.uuid = UUIDType::from_valid_uuid("123e4567-e89b-12d3-a456-426614174099"); + + let mut playlist = vec![PlaylistGroup { + id: 1, + title: "Provider Series".intern(), + channels: vec![provider_episode, series_info], + xtream_cluster: XtreamCluster::Series, + }]; + + for (idx, channel) in playlist[0].channels.iter_mut().enumerate() { + let uuid = channel.get_uuid(); + let provider_id = channel.header.get_provider_id().unwrap_or_default(); + let item_type = channel.header.item_type; + channel.header.virtual_id = + target_id_mapping.get_and_update_virtual_id(&uuid, provider_id, item_type, VirtualId::new(0)); + channel.header.source_ordinal = u32::try_from(idx + 1).expect("ordinal"); + } + + let series_virtual_id = playlist[0].channels[1].header.virtual_id; + let episode_virtual_id = playlist[0].channels[0].header.virtual_id; + + rewrite_series_episode_parent_virtual_ids(&mut playlist, &mut target_id_mapping); + target_id_mapping.persist().expect("persist"); + + let mut query = BPlusTreeQuery::::try_new(&mapping_path).expect("query"); + let record = query.query_zero_copy(&episode_virtual_id.get()).expect("query ok").expect("record missing"); + + assert_eq!(record.parent_virtual_id, series_virtual_id); + assert_eq!(playlist[0].channels[0].header.virtual_id, episode_virtual_id); +} + +#[test] +fn initial_virtual_id_assignment_preserves_existing_parent_for_series_episode_without_parent_match() { + let dir = tempdir().expect("tempdir"); + let mapping_path = dir.path().join("id_mapping.db"); + let mut target_id_mapping = TargetIdMapping::new(&mapping_path, false).expect("mapping"); + + let input_name = "provider-input".intern(); + let mut episode = PlaylistItem { + header: PlaylistItemHeader { + id: "201".intern(), + url: "http://provider.example.com/series/user/pass/201.mkv".intern(), + input_name, + item_type: PlaylistItemType::Series, + xtream_cluster: XtreamCluster::Series, + ..PlaylistItemHeader::default() + }, + }; + + let provider_id = episode.header.get_provider_id().unwrap_or_default(); + let uuid = *episode.header.get_uuid(); + let original_virtual_id = + target_id_mapping.get_and_update_virtual_id(&uuid, provider_id, episode.header.item_type, VirtualId::new(77)); + + let preserved_parent_virtual_id = target_id_mapping.get_parent_virtual_id_by_uuid(&uuid).unwrap_or_default(); + episode.header.virtual_id = target_id_mapping.get_and_update_virtual_id( + &uuid, + provider_id, + episode.header.item_type, + preserved_parent_virtual_id, + ); + target_id_mapping.persist().expect("persist"); + + let mut query = BPlusTreeQuery::::try_new(&mapping_path).expect("query"); + let record = query.query_zero_copy(&original_virtual_id.get()).expect("query ok").expect("record missing"); + + assert_eq!(record.parent_virtual_id, VirtualId::new(77)); + assert_eq!(episode.header.virtual_id, original_virtual_id); +} + +#[test] +fn rewrite_local_series_info_uses_series_uuid_lookup_and_updates_episode_virtual_ids() { + let series_uuid = "series-uuid"; + let mut series_info = make_local_series_info( + series_uuid, + vec![(101, "Episode 1", "/library/episode1.mkv"), (202, "Episode 2", "/library/episode2.mkv")], + ); + let mut local_library_series = HashMap::, Vec>::new(); + local_library_series.insert( + series_uuid.intern(), + vec![ + LocalEpisodeKey { path: "/library/episode1.mkv".intern(), virtual_id: 7001 }, + LocalEpisodeKey { path: "/library/episode2.mkv".intern(), virtual_id: 7002 }, + ], + ); + + rewrite_local_series_info_episode_virtual_id(&mut series_info, &local_library_series); + + let Some(StreamProperties::Series(series)) = series_info.header.additional_properties.as_ref() else { + panic!("missing series properties"); + }; + let episodes = series.details.as_ref().and_then(|details| details.episodes.as_ref()).expect("missing episodes"); + assert_eq!(episodes[0].id, 7001); + assert_eq!(episodes[1].id, 7002); +} + +#[test] +fn rewrite_series_info_updates_local_episode_ids_before_parent_code_is_cleared() { + let series_uuid = "series-uuid"; + let mut episode_one = make_local_series_episode(series_uuid, "/library/episode1.mkv", 7001); + let mut episode_two = make_local_series_episode(series_uuid, "/library/episode2.mkv", 7002); + + let mut local_library_series = HashMap::, Vec>::new(); + assign_local_series_info_episode_key(&mut local_library_series, &mut episode_one, PlaylistItemType::LocalSeries); + assign_local_series_info_episode_key(&mut local_library_series, &mut episode_two, PlaylistItemType::LocalSeries); + + let series_info = make_local_series_info( + series_uuid, + vec![(101, "Episode 1", "/library/episode1.mkv"), (202, "Episode 2", "/library/episode2.mkv")], + ); + let local_episode_one = PlaylistItem { header: episode_one }; + let local_episode_two = PlaylistItem { header: episode_two }; + let mut playlist = vec![PlaylistGroup { + id: 1, + title: "Series".intern(), + channels: vec![series_info, local_episode_one, local_episode_two], + xtream_cluster: XtreamCluster::Series, + }]; + + rewrite_series_info_episode_virtual_id( + &mut playlist, + &local_library_series, + &HashMap::, Vec>::new(), + ); + + let Some(StreamProperties::Series(series)) = playlist[0].channels[0].header.additional_properties.as_ref() else { + panic!("missing series properties"); + }; + let episodes = series.details.as_ref().and_then(|details| details.episodes.as_ref()).expect("missing episodes"); + assert_eq!(episodes[0].id, 7001); + assert_eq!(episodes[1].id, 7002); + assert!(playlist[0].channels[1].header.parent_code.is_empty()); + assert!(playlist[0].channels[2].header.parent_code.is_empty()); +} + +#[test] +fn rewrite_series_info_updates_local_episode_ids_when_episodes_come_first() { + // Test with episodes BEFORE series_info to verify iteration-order doesn't matter + let series_uuid = "series-uuid"; + let mut episode_one = make_local_series_episode(series_uuid, "/library/episode1.mkv", 7001); + let mut episode_two = make_local_series_episode(series_uuid, "/library/episode2.mkv", 7002); + + let mut local_library_series = HashMap::, Vec>::new(); + assign_local_series_info_episode_key(&mut local_library_series, &mut episode_one, PlaylistItemType::LocalSeries); + assign_local_series_info_episode_key(&mut local_library_series, &mut episode_two, PlaylistItemType::LocalSeries); + + let series_info = make_local_series_info( + series_uuid, + vec![(101, "Episode 1", "/library/episode1.mkv"), (202, "Episode 2", "/library/episode2.mkv")], + ); + let local_episode_one = PlaylistItem { header: episode_one }; + let local_episode_two = PlaylistItem { header: episode_two }; + + // Episodes FIRST, then series_info (reversed order) + let mut playlist = vec![PlaylistGroup { + id: 1, + title: "Series".intern(), + channels: vec![local_episode_one, local_episode_two, series_info], + xtream_cluster: XtreamCluster::Series, + }]; + + rewrite_series_info_episode_virtual_id( + &mut playlist, + &local_library_series, + &HashMap::, Vec>::new(), + ); + + let Some(StreamProperties::Series(series)) = playlist[0].channels[2].header.additional_properties.as_ref() else { + panic!("missing series properties"); + }; + let episodes = series.details.as_ref().and_then(|details| details.episodes.as_ref()).expect("missing episodes"); + assert_eq!(episodes[0].id, 7001); + assert_eq!(episodes[1].id, 7002); + assert!(playlist[0].channels[0].header.parent_code.is_empty()); + assert!(playlist[0].channels[1].header.parent_code.is_empty()); +} diff --git a/backend/repository/src/playlist_repository/tests/recovery.rs b/backend/repository/src/playlist_repository/tests/recovery.rs new file mode 100644 index 000000000..d1db867e1 --- /dev/null +++ b/backend/repository/src/playlist_repository/tests/recovery.rs @@ -0,0 +1,72 @@ +use super::*; + +#[tokio::test] +async fn target_cache_reload_failures_return_errors_without_mutating_the_previous_cache() { + for stage in [TargetCacheReloadStage::IdMapping, TargetCacheReloadStage::XtreamStorage] { + let directory = tempdir().expect("temporary storage"); + let app_config = test_app_config(directory.path()); + let playlist_state = Arc::new(PlaylistStorageState::new()); + let target = xtream_target(&format!("cache-reload-{stage:?}"), true); + seed_memory_cached_xtream_target(&app_config, &playlist_state, &target).await; + let before = cached_target_signature(&playlist_state, &target.name).await; + let mut candidate = vec![ + target_group(XtreamCluster::Live, 102, "new-live"), + target_group(XtreamCluster::Series, 302, "new-series"), + ]; + + let errors = persist_playlist_with_mode( + &app_config, + &mut candidate, + None, + &target, + Some(&playlist_state), + TargetPlaylistPersistOptions { + accepted_empty_clusters: ClusterFlags::Vod, + ..TargetPlaylistPersistOptions::default() + }, + TargetPersistenceMode::FailCacheReloadAt(stage), + ) + .await + .expect_err("injected target cache reload must fail"); + + assert!(errors.iter().any(|error| error.to_string().contains("could not be reloaded"))); + assert_eq!(cached_target_signature(&playlist_state, &target.name).await, before); + let persisted = load_xtream_target_storage(&app_config, &target).await.expect("persisted target storage"); + assert_eq!([persisted.live.len(), persisted.vod.len(), persisted.series.len()], [1, 0, 1]); + } +} + +#[tokio::test] +async fn target_force_empty_reload_replaces_the_complete_memory_cache() { + let directory = tempdir().expect("temporary storage"); + let app_config = test_app_config(directory.path()); + let playlist_state = Arc::new(PlaylistStorageState::new()); + let target = xtream_target("force-empty-cache-success", true); + seed_memory_cached_xtream_target(&app_config, &playlist_state, &target).await; + let mut candidate = vec![ + target_group(XtreamCluster::Live, 102, "new-live"), + target_group(XtreamCluster::Series, 302, "new-series"), + ]; + + persist_playlist_with_mode( + &app_config, + &mut candidate, + None, + &target, + Some(&playlist_state), + TargetPlaylistPersistOptions { + accepted_empty_clusters: ClusterFlags::Vod, + ..TargetPlaylistPersistOptions::default() + }, + TargetPersistenceMode::Persist, + ) + .await + .expect("force-empty target persistence and cache reload"); + + let (_, cached_items) = cached_target_signature(&playlist_state, &target.name).await; + assert_eq!(cached_items.iter().filter(|(cluster, _, _)| *cluster == XtreamCluster::Live).count(), 1); + assert!(!cached_items.iter().any(|(cluster, _, _)| *cluster == XtreamCluster::Video)); + assert_eq!(cached_items.iter().filter(|(cluster, _, _)| *cluster == XtreamCluster::Series).count(), 1); + assert!(cached_items.iter().any(|(_, _, name)| name.as_ref() == "new-live")); + assert!(cached_items.iter().any(|(_, _, name)| name.as_ref() == "new-series")); +} diff --git a/backend/repository/src/playlist_repository/tests/storage.rs b/backend/repository/src/playlist_repository/tests/storage.rs new file mode 100644 index 000000000..7442addba --- /dev/null +++ b/backend/repository/src/playlist_repository/tests/storage.rs @@ -0,0 +1,102 @@ +use super::*; + +#[tokio::test] +async fn force_empty_output_filters_reject_m3u_and_strm_before_mutation() { + let filter = get_filter(r#"EpgId ~ "^Mixed\.Case$""#, None).expect("case-sensitive EPG-ID filter"); + let cases = [ + ( + "filtered-m3u", + TargetOutput::M3u(M3uTargetOutput { + filename: Some("client.m3u".to_string()), + include_type_in_url: false, + mask_redirect_url: false, + filter: Some(filter.clone()), + }), + "client.m3u", + ), + ( + "filtered-strm", + TargetOutput::Strm(StrmTargetOutput { + directory: "client-strm".to_string(), + username: None, + style: StrmExportStyle::Jellyfin, + flags: StrmTargetFlagsSet::new(), + strm_props: None, + filter: Some(filter), + probe_probe_size_bytes: None, + probe_analyze_duration: None, + }), + "client-strm/old.strm", + ), + ]; + + for (target_name, output, artifact) in cases { + let directory = tempdir().expect("temporary storage"); + let app_config = test_app_config(directory.path()); + let artifact_path = directory.path().join(artifact); + tokio::fs::create_dir_all(artifact_path.parent().expect("artifact parent")).await.expect("artifact directory"); + tokio::fs::write(&artifact_path, b"previous-client-state").await.expect("previous client artifact"); + let mut target = target_with_options(target_name, vec![output], false); + target.options = Some(epg_normalization_options(true)); + let target_path = { + let config = app_config.config.load(); + crate::get_target_storage_path(&config, &target.name).expect("target storage path") + }; + tokio::fs::create_dir_all(&target_path).await.expect("target storage directory"); + let mapping_path = crate::get_target_id_mapping_file(&target_path); + let mapping_uuid_path = mapping_path.with_extension("uuid.db"); + let seed_group = target_group(XtreamCluster::Live, 100, "mapping-seed"); + { + let seed_header = &seed_group.channels[0].header; + let mut mapping = TargetIdMapping::new(&mapping_path, false).expect("seed target ID mapping"); + mapping.get_and_update_virtual_id(seed_header.get_uuid(), 0, seed_header.item_type, VirtualId::default()); + mapping.persist().expect("persist target ID mapping seed"); + } + let mapping_before = [ + tokio::fs::read(&mapping_path).await.expect("target ID mapping"), + tokio::fs::read(&mapping_uuid_path).await.expect("target UUID mapping"), + ]; + let mut candidate = vec![target_group(XtreamCluster::Live, 101, "filtered-after-normalization")]; + candidate[0].channels[0].header.epg_channel_id = Some("Mixed.Case".intern()); + + let errors = persist_playlist_with_mode( + &app_config, + &mut candidate, + None, + &target, + None, + TargetPlaylistPersistOptions { + accepted_empty_clusters: ClusterFlags::Vod, + ..TargetPlaylistPersistOptions::default() + }, + TargetPersistenceMode::Persist, + ) + .await + .expect_err("force-empty filtered output must be rejected"); + + assert!(errors.iter().any(|error| error.to_string().contains("after its output filter"))); + assert_eq!(candidate[0].channels[0].header.epg_channel_id.as_deref(), Some("mixed.case")); + assert_eq!(tokio::fs::read(&artifact_path).await.expect("retained client artifact"), b"previous-client-state"); + assert_eq!(tokio::fs::read(&mapping_path).await.expect("retained target ID mapping"), mapping_before[0]); + assert_eq!(tokio::fs::read(&mapping_uuid_path).await.expect("retained target UUID mapping"), mapping_before[1]); + } +} + +#[test] +fn playlist_without_channels_is_empty_for_persistence() { + assert!(!playlist_has_items(&[])); + assert!(!playlist_has_items(&[PlaylistGroup { + id: 1, + title: "empty".intern(), + channels: Vec::new(), + xtream_cluster: XtreamCluster::Live, + }])); +} + +#[test] +fn media_server_playlist_file_path_uses_separate_prefix() { + let dir = tempdir().expect("tempdir"); + let path = get_input_media_server_playlist_file_path(dir.path(), &"Media Server Input".intern()); + + assert!(path.ends_with("media_server_Media_Server_Input.db")); +} diff --git a/backend/repository/src/playlist_repository/tests/transport.rs b/backend/repository/src/playlist_repository/tests/transport.rs new file mode 100644 index 000000000..7b4837dd1 --- /dev/null +++ b/backend/repository/src/playlist_repository/tests/transport.rs @@ -0,0 +1,121 @@ +use super::*; + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn intentional_empty_output_views_clear_managed_artifacts_without_alias_leakage() { + let directory = tempdir().expect("tempdir"); + let app_config = target_test_app_config(directory.path()); + let target = mixed_output_target(); + let playlist_state = Arc::new(crate::PlaylistStorageState::new()); + let mut standard = + vec![target_video_group("Base movie", UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000021"))]; + let mut xtream = + vec![target_video_group("Curated alias", UUIDType::from_valid_uuid("00000000-0000-4000-8000-000000000022"))]; + + let first = persist_playlist_views( + &app_config, + &mut standard, + Some(&mut xtream), + None, + &target, + Some(&playlist_state), + curation_persist_options(PlaylistPublicationPlan::complete_curation(true, false)), + ) + .await; + assert!(first.is_ok(), "initial mixed-output persist failed: {first:?}"); + + let m3u = load_m3u_target_storage(&app_config, &target).await.expect("M3U storage"); + let xtream = load_xtream_target_storage(&app_config, &target).await.expect("Xtream storage"); + assert_eq!(m3u.len(), 1); + assert_eq!(m3u.iter().next().expect("M3U item").1.title.as_ref(), "Base movie"); + assert_eq!(xtream.vod.len(), 1); + let xtream_item = xtream.vod.iter().next().expect("Xtream item").1; + assert_eq!(xtream_item.title.as_ref(), "Curated alias"); + assert_ne!(xtream_item.category_id, 0, "base-suppressed unfiltered rows remain category-backed aliases"); + { + let cache = playlist_state.data.read().await; + let cached = cache.get(&target.name).expect("target cache"); + assert_eq!(cached.m3u.as_ref().expect("M3U cache").len(), 1); + assert_eq!(cached.xtream.as_ref().expect("Xtream cache").vod.len(), 1); + } + let m3u_text_path = directory.path().join("curated.m3u"); + assert!(std::fs::read_to_string(&m3u_text_path).expect("M3U text").contains("Base movie")); + let strm_root = directory.path().join("strm"); + let strm_files = strm_files_below(&strm_root); + assert_eq!(strm_files.len(), 1); + assert!(std::fs::read_to_string(&strm_files[0]).expect("STRM content").contains("Base movie")); + + let mut empty_standard = Vec::new(); + let mut empty_xtream = Vec::new(); + let retained = persist_playlist_views( + &app_config, + &mut empty_standard, + Some(&mut empty_xtream), + None, + &target, + Some(&playlist_state), + curation_persist_options(PlaylistPublicationPlan::Ordinary), + ) + .await; + assert!(retained.is_err(), "untrusted empty input must retain the published snapshot"); + assert_eq!(load_m3u_target_storage(&app_config, &target).await.expect("retained M3U").len(), 1); + assert_eq!(load_xtream_target_storage(&app_config, &target).await.expect("retained Xtream").vod.len(), 1); + { + let cache = playlist_state.data.read().await; + let cached = cache.get(&target.name).expect("retained target cache"); + assert_eq!(cached.m3u.as_ref().expect("retained M3U cache").len(), 1); + assert_eq!(cached.xtream.as_ref().expect("retained Xtream cache").vod.len(), 1); + } + assert_eq!(strm_files_below(&strm_root).len(), 1, "untrusted empty refresh must retain STRM files"); + + let published = persist_playlist_views( + &app_config, + &mut empty_standard, + Some(&mut empty_xtream), + None, + &target, + Some(&playlist_state), + curation_persist_options(PlaylistPublicationPlan::complete_curation(true, false)), + ) + .await; + assert!(published.is_ok(), "trusted empty snapshot failed: {published:?}"); + assert!(load_m3u_target_storage(&app_config, &target).await.expect("empty M3U").is_empty()); + assert_eq!(std::fs::read_to_string(&m3u_text_path).expect("empty M3U text"), "#EXTM3U\n"); + let empty_xtream = load_xtream_target_storage(&app_config, &target).await.expect("empty Xtream"); + assert!(empty_xtream.live.is_empty()); + assert!(empty_xtream.vod.is_empty()); + assert!(empty_xtream.series.is_empty()); + let target_storage = { + let config = app_config.config.load(); + get_target_storage_path(&config, &target.name).expect("target storage") + }; + let xtream_storage = { + let config = app_config.config.load(); + xtream_get_storage_path(&config, &target.name).expect("Xtream storage path") + }; + assert_eq!( + std::fs::read_to_string(get_vod_cat_collection_path(&xtream_storage)).expect("empty VOD categories"), + "[]" + ); + assert_eq!( + std::fs::read_to_string(get_series_cat_collection_path(&xtream_storage)).expect("empty series categories"), + "[]" + ); + let strm_index = strm_get_file_paths(&hash_string_as_hex(&normalize_string_path("strm")), &target_storage); + assert!(std::fs::read_to_string(strm_index).expect("empty STRM index").is_empty()); + { + let cache = playlist_state.data.read().await; + let cached = cache.get(&target.name).expect("empty target cache"); + assert!(cached.m3u.as_ref().expect("empty M3U cache").is_empty()); + assert!(cached.xtream.as_ref().expect("empty Xtream cache").vod.is_empty()); + } + assert!(strm_files_below(&strm_root).is_empty(), "trusted empty refresh must clean stale STRM files"); +} + +#[test] +fn skipped_clusters_converts_loaded_clusters_to_exclusions() { + let skipped = skipped_clusters(&[XtreamCluster::Live, XtreamCluster::Series]); + + assert_eq!(skipped.len(), 1); + assert!(skipped.contains(&XtreamCluster::Video)); +} diff --git a/backend/repository/src/xtream_repository.rs b/backend/repository/src/xtream_repository.rs index e6e5ccd62..aeab674a9 100644 --- a/backend/repository/src/xtream_repository.rs +++ b/backend/repository/src/xtream_repository.rs @@ -1,151 +1,11 @@ use super::playlist_mem_cache::PlaylistStorageState; -use crate::{ - bplustree::{ - ensure_distinct_sidecar_lock_domains, get_file_path_for_db_index, publish_staged_database, BPlusTree, - BPlusTreeError, BPlusTreeQuery, BPlusTreeStagingArtifacts, BPlusTreeUpdate, FlushPolicy, - }, - error_macros::{cant_read_result, cant_write_result}, - playlist_backend::{ensure_storage_path, iter_raw_playlist, PlaylistBackend, PlaylistKey, Xtream}, - playlist_scratch::PlaylistScratch, - storage::{ - ensure_input_storage_path, get_input_storage_path, get_target_id_mapping_file, get_target_storage_path, - XtreamRefreshGenerationGuard, - }, - storage_const, - target_id_mapping::VirtualIdRecord, - xtream_playlist_iterator::XtreamPlaylistJsonIterator, - LockedReceiverStream, -}; -use bytes::Bytes; -use futures::{stream, Stream, StreamExt}; -use indexmap::IndexMap; -use log::{error, warn}; use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; -use shared::{ - concat_string, - error::{string_to_io_error, TuliproxError}, - model::{ - xtream_const::XTREAM_CLUSTER, ClusterFlags, LiveStreamProperties, PlaylistGroup, PlaylistItem, - PlaylistItemType, ProviderId, SeriesStreamProperties, StreamProperties, VideoStreamProperties, VirtualId, - XtreamCluster, XtreamPlaylistItem, - }, - utils::{arc_str_serde, get_u32_from_serde_value, Internable}, -}; -#[cfg(unix)] -use std::fs; -use std::{ - collections::{HashMap, HashSet}, - ffi::OsString, - fs::File, - io::{self, Error, ErrorKind}, - path::{Path, PathBuf}, - sync::Arc, -}; -#[cfg(not(windows))] -use tuliprox_core::utils::parent_or_dot; +use shared::{error::TuliproxError, model::XtreamCluster, utils::arc_str_serde}; +use std::sync::Arc; use tuliprox_core::{ - model::{ - evaluate_update_quality, AppConfig, ClusterForceUpdate, ClusterUpdateAcceptance, ClusterUpdateRejection, - Config, ConfigInput, ConfigInputFlags, ConfigTarget, PlaylistXtreamCategory, ProxyUserCredentials, - UpdateQualityDecision, XtreamCategory, - }, - utils::{ - file_exists_async, file_reader, json_write_documents_to_file, remove_file_if_exists, request::DynReader, - require_same_parent_directory, FileReadGuard, FileWriteGuard, - }, + model::{ClusterForceUpdate, ClusterUpdateAcceptance, ClusterUpdateRejection}, + utils::request::DynReader, }; -use tuliprox_parser::xtream; -use uuid::Uuid; - -#[inline] -pub fn get_collection_path(path: &Path, collection: &str) -> PathBuf { path.join(format!("{collection}.json")) } - -/// Returns the category-collection base name for an [`XtreamCluster`]. -/// -/// Centralizes the per-cluster `cat_live` / `cat_vod` / `cat_series` mapping so the -/// path-deriving call sites read a single property instead of re-matching the cluster. -#[inline] -pub const fn xtream_cluster_category_collection(cluster: XtreamCluster) -> &'static str { - match cluster { - XtreamCluster::Live => storage_const::COL_CAT_LIVE, - XtreamCluster::Video => storage_const::COL_CAT_VOD, - XtreamCluster::Series => storage_const::COL_CAT_SERIES, - } -} - -#[inline] -pub fn get_live_cat_collection_path(path: &Path) -> PathBuf { get_collection_path(path, storage_const::COL_CAT_LIVE) } - -#[inline] -pub fn get_vod_cat_collection_path(path: &Path) -> PathBuf { get_collection_path(path, storage_const::COL_CAT_VOD) } - -#[inline] -pub fn get_series_cat_collection_path(path: &Path) -> PathBuf { - get_collection_path(path, storage_const::COL_CAT_SERIES) -} - -fn target_category_lock_path(category_path: &Path) -> PathBuf { - category_path.with_extension("json.target-category.lock") -} - -#[inline] -pub async fn ensure_xtream_storage_path(cfg: &Config, target_name: &str) -> Result { - ensure_storage_path::(cfg, target_name).await -} - -/// Persist `collections`, keyed by whatever `key_of` extracts. -/// -/// The key space is a type parameter rather than a runtime tag. These stores are -/// keyed by [`VirtualId`] on the target path and by [`ProviderId`] on the input -/// path -- the same file layout and value type, two different id spaces -- and a -/// `StorageKey` enum matched per item used to be the only thing recording which. -/// Both keys are `#[serde(transparent)]` over `u32`, so the on-disk encoding is -/// unchanged either way (see the codec test in `backend/btree`). -async fn write_playlists_to_file( - app_config: &Arc, - storage_path: &Path, - with_index: bool, - key_of: F, - collections: Vec<(XtreamCluster, Vec)>, - replace_empty_clusters: ClusterFlags, -) -> Result<(), TuliproxError> -where - K: PlaylistKey, - F: Fn(&XtreamPlaylistItem) -> K + Copy + Send + 'static, -{ - for (cluster, playlist) in collections { - if playlist.is_empty() && !replace_empty_clusters.contains(cluster_flag(cluster)) { - continue; - } - let xtream_path = xtream_get_file_path(storage_path, cluster); - - // Acquire FileLockManager lock (async, in-process coordination) - let file_lock = app_config.file_locks.write_lock(&xtream_path).await; - - // Move all B+Tree building and I/O to spawn_blocking - // We take ownership of `playlist` here (no cloning needed) - let path_clone = xtream_path.clone(); - tokio::task::spawn_blocking(move || -> Result<(), std::io::Error> { - let _guard = file_lock; - let mut tree = BPlusTree::::new(); - for item in playlist { - let key = key_of(&item); - tree.insert(key, item); - } - if with_index { - tree.store_with_index(&path_clone, |pli| pli.source_ordinal)?; - } else { - tree.store(&path_clone)?; - } - Ok(()) - }) - .await - .map_err(|e| TuliproxError::RepositoryXtream(format!("Blocking task failed: {e}")))? - .map_err(|err| cant_write_result!(RepositoryXtream, "xtream", &xtream_path, err))?; - } - Ok(()) -} #[cfg(test)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -155,553 +15,9 @@ pub(crate) enum TargetEmptyReplacementFailure { Publication, } -#[derive(Debug, Clone, Default)] -enum TargetEmptyReplacementMode { - #[default] - Persist, - #[cfg(test)] - FailAt(TargetEmptyReplacementFailure), - #[cfg(test)] - PauseDuringPublication(TargetEmptyPublicationHook), -} - -#[cfg(test)] -#[derive(Debug, Clone)] -struct TargetEmptyPublicationHook { - backup_window_entered: Arc, - resume_publication: Arc, -} - -#[cfg(test)] -impl TargetEmptyPublicationHook { - fn new() -> Self { - Self { - backup_window_entered: Arc::new(std::sync::Barrier::new(2)), - resume_publication: Arc::new(std::sync::Barrier::new(2)), - } - } -} - -struct TargetEmptyClusterPaths { - published_database: PathBuf, - published_index: PathBuf, - published_categories: PathBuf, - staging_database: PathBuf, - staging_index: PathBuf, - staging_categories: PathBuf, -} - -impl TargetEmptyClusterPaths { - fn new(storage_path: &Path, cluster: XtreamCluster) -> Self { - let token = Uuid::new_v4().simple(); - let cluster_name = cluster.as_str().to_lowercase(); - let published_database = xtream_get_file_path(storage_path, cluster); - let staging_database = storage_path.join(format!(".{cluster_name}.force-empty-{token}.db")); - Self { - published_index: get_file_path_for_db_index(&published_database), - published_categories: get_collection_path(storage_path, xtream_cluster_category_collection(cluster)), - staging_index: get_file_path_for_db_index(&staging_database), - staging_categories: storage_path.join(format!(".{cluster_name}.force-empty-{token}.json")), - published_database, - staging_database, - } - } -} - -struct TargetFileReplacement { - published: PathBuf, - staging: PathBuf, - backup: PathBuf, - previous_moved: bool, - replacement_published: bool, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum TargetFileMoveMode { - PreserveDestination, - ReplaceDestination, -} - -#[cfg(unix)] -fn move_target_file_platform(source: &Path, destination: &Path, mode: TargetFileMoveMode) -> io::Result<()> { - require_same_parent_directory(source, destination)?; - if mode == TargetFileMoveMode::PreserveDestination && destination.exists() { - return Err(io::Error::new( - io::ErrorKind::AlreadyExists, - format!("target transaction destination already exists: {}", destination.display()), - )); - } - fs::rename(source, destination) -} - -#[cfg(windows)] -fn move_target_file_platform(source: &Path, destination: &Path, mode: TargetFileMoveMode) -> io::Result<()> { - use windows_sys::Win32::Storage::FileSystem::{MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH}; - - require_same_parent_directory(source, destination)?; - let source_encoded = encode_windows_path(source)?; - let destination_encoded = encode_windows_path(destination)?; - let flags = MOVEFILE_WRITE_THROUGH - | if mode == TargetFileMoveMode::ReplaceDestination { MOVEFILE_REPLACE_EXISTING } else { 0 }; - - // SAFETY: both buffers are live, immutable, and NUL-terminated for the - // duration of the call. The same-directory check prevents a cross-volume - // move from degrading into a copy. - let result = unsafe { MoveFileExW(source_encoded.as_ptr(), destination_encoded.as_ptr(), flags) }; - if result == 0 { - let error = io::Error::last_os_error(); - return Err(io::Error::new( - error.kind(), - format!( - "failed to move target transaction file {} to {} with Windows write-through semantics: {error}", - source.display(), - destination.display() - ), - )); - } - Ok(()) -} - -#[cfg(all(not(unix), not(windows)))] -fn move_target_file_platform(source: &Path, destination: &Path, _mode: TargetFileMoveMode) -> io::Result<()> { - Err(io::Error::new( - io::ErrorKind::Unsupported, - format!( - "durable target transaction moves are unsupported on this platform: {} -> {}", - source.display(), - destination.display() - ), - )) -} - -impl TargetFileReplacement { - fn new(published: &Path, staging: &Path, token: uuid::fmt::Simple) -> Self { - let filename = - published.file_name().map_or_else(|| "target-artifact".into(), |name| name.to_string_lossy().into_owned()); - Self { - published: published.to_path_buf(), - staging: staging.to_path_buf(), - backup: published.with_file_name(format!(".{filename}.force-empty-backup-{token}")), - previous_moved: false, - replacement_published: false, - } - } -} - -fn cleanup_target_empty_staging( - staging_artifacts: &BPlusTreeStagingArtifacts, - staging_categories: &Path, -) -> io::Result<()> { - let database_result = staging_artifacts.remove_owned_staging_artifacts(); - let category_result = remove_file_if_exists(staging_categories); - match (database_result, category_result) { - (Ok(()), Ok(())) => Ok(()), - (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Err(database_error), Err(category_error)) => Err(io::Error::new( - database_error.kind(), - format!("{database_error}; category staging cleanup also failed: {category_error}"), - )), - } -} - -fn rollback_target_file_replacements(replacements: &mut [TargetFileReplacement]) -> io::Result<()> { - let mut errors = Vec::new(); - for replacement in replacements.iter_mut().rev() { - if replacement.previous_moved { - if let Err(error) = move_target_file_platform( - &replacement.backup, - &replacement.published, - TargetFileMoveMode::ReplaceDestination, - ) { - errors.push(format!( - "failed to restore {} from {}: {error}", - replacement.published.display(), - replacement.backup.display() - )); - } - } else if replacement.replacement_published { - if let Err(error) = move_target_file_platform( - &replacement.published, - &replacement.backup, - TargetFileMoveMode::ReplaceDestination, - ) { - errors.push(format!( - "failed to withdraw replacement {} during rollback: {error}", - replacement.published.display() - )); - } else if let Err(error) = remove_file_if_exists(&replacement.backup) { - errors - .push(format!("failed to remove withdrawn replacement {}: {error}", replacement.backup.display())); - } - } - } - - if errors.is_empty() { - Ok(()) - } else { - Err(io::Error::other(errors.join("; "))) - } -} - -fn publish_target_file_replacements( - paths: &TargetEmptyClusterPaths, - mode: &TargetEmptyReplacementMode, -) -> io::Result<()> { - #[cfg(not(test))] - let _ = &mode; - let staging_artifacts = BPlusTreeStagingArtifacts::new(&paths.published_database, &paths.staging_database)?; - let token = Uuid::new_v4().simple(); - let mut replacements = vec![ - TargetFileReplacement::new(&paths.published_database, &paths.staging_database, token), - TargetFileReplacement::new(&paths.published_index, &paths.staging_index, token), - TargetFileReplacement::new(&paths.published_categories, &paths.staging_categories, token), - ]; - - let publication = (|| -> io::Result<()> { - for replacement in &replacements { - require_same_parent_directory(&replacement.staging, &replacement.published)?; - if !replacement.staging.is_file() { - return Err(io::Error::new( - io::ErrorKind::NotFound, - format!("missing prepared target artifact {}", replacement.staging.display()), - )); - } - } - - for replacement in &mut replacements { - if replacement.published.exists() { - move_target_file_platform( - &replacement.published, - &replacement.backup, - TargetFileMoveMode::PreserveDestination, - )?; - replacement.previous_moved = true; - } - } - - #[cfg(test)] - if let TargetEmptyReplacementMode::PauseDuringPublication(hook) = mode { - hook.backup_window_entered.wait(); - hook.resume_publication.wait(); - } - - for (index, replacement) in replacements.iter_mut().enumerate() { - move_target_file_platform( - &replacement.staging, - &replacement.published, - TargetFileMoveMode::ReplaceDestination, - )?; - replacement.replacement_published = true; - #[cfg(test)] - if matches!(mode, TargetEmptyReplacementMode::FailAt(TargetEmptyReplacementFailure::Publication)) - && index == 0 - { - return Err(io::Error::other("injected target empty-replacement publication failure")); - } - #[cfg(not(test))] - let _ = index; - } - sync_published_file_parent(&paths.published_database)?; - cleanup_target_empty_staging(&staging_artifacts, &paths.staging_categories) - })(); - - if let Err(publication_error) = publication { - let rollback_result = rollback_target_file_replacements(&mut replacements); - let durability_result = sync_published_file_parent(&paths.published_database); - return match (rollback_result, durability_result) { - (Ok(()), Ok(())) => Err(publication_error), - (rollback, durability) => Err(io::Error::new( - publication_error.kind(), - format!( - "{publication_error}; rollback result: {}; rollback directory sync result: {}", - rollback.map_or_else(|error| error.to_string(), |()| "ok".to_string()), - durability.map_or_else(|error| error.to_string(), |()| "ok".to_string()) - ), - )), - }; - } - - for replacement in replacements { - if replacement.previous_moved { - if let Err(error) = remove_file_if_exists(&replacement.backup) { - warn!( - "Target empty replacement was published, but backup cleanup failed for {}: {error}", - replacement.backup.display() - ); - } - } - } - Ok(()) -} - -async fn replace_target_xtream_cluster_with_empty( - app_config: &Arc, - storage_path: &Path, - cluster: XtreamCluster, - mode: TargetEmptyReplacementMode, -) -> Result<(), TuliproxError> { - #[cfg(not(test))] - let _ = &mode; - let paths = TargetEmptyClusterPaths::new(storage_path, cluster); - let staging_artifacts = BPlusTreeStagingArtifacts::new(&paths.published_database, &paths.staging_database) - .map_err(|error| { - TuliproxError::RepositoryXtream(format!("Failed to prepare empty {cluster} target: {error}")) - })?; - - let operation = async { - #[cfg(test)] - if matches!(&mode, TargetEmptyReplacementMode::FailAt(TargetEmptyReplacementFailure::CategoryPersistence)) { - return Err(TuliproxError::RepositoryXtream("injected target category persistence failure".to_string())); - } - json_write_documents_to_file(&paths.staging_categories, &Vec::::new()).await.map_err( - |error| { - TuliproxError::RepositoryXtream(format!( - "Failed to prepare empty {cluster} target categories {}: {error}", - paths.staging_categories.display() - )) - }, - )?; - let staging_category_file = tokio::fs::File::open(&paths.staging_categories).await.map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to reopen empty {cluster} target categories {}: {error}", - paths.staging_categories.display() - )) - })?; - staging_category_file.sync_all().await.map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to synchronize empty {cluster} target categories {}: {error}", - paths.staging_categories.display() - )) - })?; - - #[cfg(test)] - if matches!(&mode, TargetEmptyReplacementMode::FailAt(TargetEmptyReplacementFailure::BTreePersistence)) { - return Err(TuliproxError::RepositoryXtream("injected target BTree/index persistence failure".to_string())); - } - let staging_database = paths.staging_database.clone(); - tokio::task::spawn_blocking(move || { - BPlusTree::::new().store_with_index(&staging_database, |item| item.source_ordinal) - }) - .await - .map_err(|error| TuliproxError::RepositoryXtream(format!("Empty target BTree task failed: {error}")))? - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to prepare empty {cluster} target BTree/index {}: {error}", - paths.staging_database.display() - )) - })?; - - let category_lock_path = target_category_lock_path(&paths.published_categories); - let mut lock_paths = [&paths.published_database, &category_lock_path]; - lock_paths.sort_unstable(); - let mut file_locks = Vec::with_capacity(lock_paths.len()); - for path in lock_paths { - file_locks.push(app_config.file_locks.write_lock(path).await); - } - let publication_paths = TargetEmptyClusterPaths { - published_database: paths.published_database.clone(), - published_index: paths.published_index.clone(), - published_categories: paths.published_categories.clone(), - staging_database: paths.staging_database.clone(), - staging_index: paths.staging_index.clone(), - staging_categories: paths.staging_categories.clone(), - }; - let publication_mode = mode.clone(); - tokio::task::spawn_blocking(move || { - let result = publish_target_file_replacements(&publication_paths, &publication_mode); - drop(file_locks); - result - }) - .await - .map_err(|error| TuliproxError::RepositoryXtream(format!("Empty target publish task failed: {error}")))? - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to publish empty {cluster} target cluster atomically: {error}" - )) - }) - } - .await; - - let cleanup = cleanup_target_empty_staging(&staging_artifacts, &paths.staging_categories); - match (operation, cleanup) { - (Ok(()), Ok(())) => Ok(()), - (Err(error), Ok(())) => Err(error), - (Ok(()), Err(cleanup_error)) => Err(TuliproxError::RepositoryXtream(format!( - "Empty {cluster} target cluster was published, but staging cleanup failed: {cleanup_error}" - ))), - (Err(error), Err(cleanup_error)) => { - Err(TuliproxError::RepositoryXtream(format!("{error}; staging cleanup also failed: {cleanup_error}"))) - } - } -} - -pub async fn write_playlist_item_update( - app_config: &Arc, - target_name: &str, - pli: &XtreamPlaylistItem, -) -> Result<(), TuliproxError> { - let storage_path = { - let config = app_config.config.load(); - ensure_xtream_storage_path(&config, target_name).await? - }; - let xtream_path = xtream_get_file_path(&storage_path, pli.xtream_cluster); - - if !file_exists_async(&xtream_path).await { - return Err(TuliproxError::RepositoryXtream(format!( - "BPlusTree file not found for update {}", - xtream_path.display() - ))); - } - - // Prepare encoded payload before opening the writer lock. - let prepared_items = - BPlusTreeUpdate::::prepare_upsert_batch(&[(&pli.virtual_id.get(), pli)]) - .map_err(|e| TuliproxError::RepositoryXtream(format!("Failed to serialize value: {e}")))?; - - // Keep FileLockManager lock for cross-operation coordination (e.g. swap + update). - let file_lock = app_config.file_locks.write_lock(&xtream_path).await; - - let xtream_path_clone = xtream_path.clone(); - tokio::task::spawn_blocking(move || -> Result<(), std::io::Error> { - let _guard = file_lock; - let mut tree = BPlusTreeUpdate::::try_new_with_backoff(&xtream_path_clone)?; - tree.upsert_batch_encoded(prepared_items)?; - Ok(()) - }) - .await - .map_err(|e| TuliproxError::RepositoryXtream(format!("Blocking task failed: {e}")))? - .map_err(|err| cant_write_result!(RepositoryXtream, "xtream", &xtream_path, err))?; - - Ok(()) -} - -pub async fn write_playlist_batch_item_upsert( - app_config: &Arc, - target_name: &str, - xtream_cluster: XtreamCluster, - pli_list: &[XtreamPlaylistItem], -) -> Result<(), TuliproxError> { - if pli_list.is_empty() { - return Ok(()); - } - - let storage_path = { - let config = app_config.config.load(); - ensure_xtream_storage_path(&config, target_name).await? - }; - let xtream_path = xtream_get_file_path(&storage_path, xtream_cluster); - - if !file_exists_async(&xtream_path).await { - return Err(TuliproxError::RepositoryXtream(format!( - "BPlusTree file not found for upsert {}", - xtream_path.display() - ))); - } - - // Prepare encoded payload before opening the writer lock. - let virtual_ids: Vec = pli_list.iter().map(|pli| pli.virtual_id.get()).collect(); - let batch_refs: Vec<(&u32, &XtreamPlaylistItem)> = virtual_ids.iter().zip(pli_list.iter()).collect(); - let prepared_items = BPlusTreeUpdate::::prepare_upsert_batch(&batch_refs) - .map_err(|e| TuliproxError::RepositoryXtream(format!("Failed to serialize value: {e}")))?; - - // Keep FileLockManager lock for cross-operation coordination (e.g. swap + update). - let file_lock = app_config.file_locks.write_lock(&xtream_path).await; - - let xtream_path_clone = xtream_path.clone(); - tokio::task::spawn_blocking(move || -> Result<(), std::io::Error> { - let _guard = file_lock; - let mut tree = BPlusTreeUpdate::::try_new_with_backoff(&xtream_path_clone)?; - tree.upsert_batch_encoded(prepared_items)?; - Ok(()) - }) - .await - .map_err(|e| TuliproxError::RepositoryXtream(format!("Blocking task failed: {e}")))? - .map_err(|err| cant_write_result!(RepositoryXtream, "xtream", &xtream_path, err))?; - - Ok(()) -} - -fn get_map_item_as_str(map: &serde_json::Map, key: &str) -> Option { - if let Some(value) = map.get(key) { - if let Some(result) = value.as_str() { - return Some(result.to_string()); - } - } - None -} - // `CategoryKey` lives in `model`; re-exported for this layer's call sites. pub use tuliprox_core::model::CategoryKey; -// Because interner is not thread safe we can't use it currently for interning. -// We leave the argument for later optimizations. -async fn load_old_category_ids(path: &Path) -> (u32, HashMap) { - let old_path = path.to_path_buf(); - tokio::task::spawn_blocking(move || { - let mut result: HashMap = HashMap::new(); - let mut max_id: u32 = 0; - for (cluster, cat) in [ - (XtreamCluster::Live, storage_const::COL_CAT_LIVE), - (XtreamCluster::Video, storage_const::COL_CAT_VOD), - (XtreamCluster::Series, storage_const::COL_CAT_SERIES), - ] { - let col_path = get_collection_path(&old_path, cat); - if col_path.exists() { - if let Ok(file) = File::open(&col_path) { - let reader = file_reader(file); - match serde_json::from_reader(reader) { - Ok(value) => { - if let Value::Array(list) = value { - for entry in list { - if let Some(category_id) = entry - .get(tuliprox_core::model::XC_TAG_CATEGORY_ID) - .and_then(get_u32_from_serde_value) - { - if let Value::Object(item) = entry { - if let Some(category_name) = - get_map_item_as_str(&item, tuliprox_core::model::XC_TAG_CATEGORY_NAME) - { - result.insert( - (cluster, /*interner.*/ category_name.intern()), - category_id, - ); - max_id = max_id.max(category_id); - } - } - } - } - } - } - Err(err) => { - log::warn!("Failed to parse category file {}: {err}", col_path.display()); - } - } - } - } - } - (max_id, result) - }) - .await - .unwrap_or_else(|_| (0, HashMap::new())) -} - -#[inline] -pub fn xtream_get_storage_path(cfg: &Config, target_name: &str) -> Option { - Xtream::storage_path(cfg, target_name) -} - -pub fn xtream_get_epg_file_path_for_target(path: &Path) -> PathBuf { - path.join(concat_string!("epg.", storage_const::FILE_SUFFIX_DB)) -} - -fn xtream_get_file_path_for_name(storage_path: &Path, name: &str) -> PathBuf { - storage_path.join(concat_string!(name, ".", storage_const::FILE_SUFFIX_DB)) -} - -pub fn xtream_get_file_path(storage_path: &Path, cluster: XtreamCluster) -> PathBuf { - xtream_get_file_path_for_name(storage_path, &cluster.as_str().to_lowercase()) -} - #[derive(Serialize, Deserialize)] pub struct CategoryEntry { pub category_id: u32, @@ -710,235 +26,6 @@ pub struct CategoryEntry { pub parent_id: u32, } -pub async fn xtream_write_playlist( - app_cfg: &Arc, - target: &ConfigTarget, - playlist: &mut [PlaylistGroup], - replace_empty_clusters: ClusterFlags, -) -> Result<(), TuliproxError> { - xtream_write_playlist_with_mode( - app_cfg, - target, - playlist, - replace_empty_clusters, - TargetEmptyReplacementMode::Persist, - ) - .await -} - -#[cfg(test)] -pub(crate) async fn xtream_write_playlist_with_injected_empty_replacement_failure( - app_cfg: &Arc, - target: &ConfigTarget, - playlist: &mut [PlaylistGroup], - replace_empty_clusters: ClusterFlags, - failure: TargetEmptyReplacementFailure, -) -> Result<(), TuliproxError> { - xtream_write_playlist_with_mode( - app_cfg, - target, - playlist, - replace_empty_clusters, - TargetEmptyReplacementMode::FailAt(failure), - ) - .await -} - -async fn xtream_write_playlist_with_mode( - app_cfg: &Arc, - target: &ConfigTarget, - playlist: &mut [PlaylistGroup], - replace_empty_clusters: ClusterFlags, - empty_replacement_mode: TargetEmptyReplacementMode, -) -> Result<(), TuliproxError> { - let path = { - let config = app_cfg.config.load(); - ensure_xtream_storage_path(&config, target.name.as_str()).await? - }; - let mut errors = Vec::new(); - let mut cat_live_col = Vec::with_capacity(1_000); - let mut cat_series_col = Vec::with_capacity(1_000); - let mut cat_vod_col = Vec::with_capacity(1_000); - let mut live_col = Vec::with_capacity(50_000); - let mut series_col = Vec::with_capacity(50_000); - let mut vod_col = Vec::with_capacity(50_000); - - let categories = create_categories(playlist, &path).await; - { - for (xtream_cluster, category) in categories { - match xtream_cluster { - XtreamCluster::Live => &mut cat_live_col, - XtreamCluster::Series => &mut cat_series_col, - XtreamCluster::Video => &mut cat_vod_col, - } - .push(category); - } - } - - for plg in playlist.iter_mut() { - if plg.channels.is_empty() { - continue; - } - - for pli in &plg.channels { - let col = match pli.header.xtream_cluster { - XtreamCluster::Live => &mut live_col, - XtreamCluster::Series => &mut series_col, - XtreamCluster::Video => &mut vod_col, - }; - col.push(pli); - } - } - - let root_path = path.clone(); - let app_config = app_cfg.clone(); - for (cluster, col_path, data) in [ - (XtreamCluster::Live, get_live_cat_collection_path(&root_path), &cat_live_col), - (XtreamCluster::Video, get_vod_cat_collection_path(&root_path), &cat_vod_col), - (XtreamCluster::Series, get_series_cat_collection_path(&root_path), &cat_series_col), - ] { - if data.is_empty() { - if replace_empty_clusters.contains(cluster_flag(cluster)) { - continue; - } - if file_exists_async(&col_path).await { - continue; - } - } - let category_lock_path = target_category_lock_path(&col_path); - let lock = app_config.file_locks.write_lock(&category_lock_path).await; - match json_write_documents_to_file(&col_path, data).await { - Ok(()) => {} - Err(err) => { - errors.push(format!("Persisting collection failed: {}: {err}", col_path.display())); - } - } - drop(lock); - } - - // Process each cluster sequentially to avoid holding multiple fully - // materialized Xtream collections in memory at the same time. - for (cluster, col) in - [(XtreamCluster::Live, &live_col), (XtreamCluster::Video, &vod_col), (XtreamCluster::Series, &series_col)] - { - if col.is_empty() && replace_empty_clusters.contains(cluster_flag(cluster)) { - if let Err(error) = - replace_target_xtream_cluster_with_empty(app_cfg, &path, cluster, empty_replacement_mode.clone()).await - { - errors.push(format!("Persisting empty {cluster} target cluster failed: {error}")); - } - continue; - } - let data = col.iter().map(|item| XtreamPlaylistItem::from(&**item)).collect::>(); - if let Err(err) = write_playlists_to_file( - app_cfg, - &path, - true, - |item| item.virtual_id, - vec![(cluster, data)], - replace_empty_clusters, - ) - .await - { - errors.push(format!("Persisting collection failed:{err}")); - } - } - - if !errors.is_empty() { - return Err(TuliproxError::Config(errors.join("\n"))); - } - - Ok(()) -} - -async fn create_categories(playlist: &mut [PlaylistGroup], path: &Path) -> Vec<(XtreamCluster, CategoryEntry)> { - // preserve category_ids - let (max_cat_id, existing_cat_ids) = load_old_category_ids(path).await; - let mut cat_id_counter = max_cat_id; - - let mut new_categories: IndexMap = IndexMap::new(); - - for plg in playlist.iter_mut() { - if plg.channels.is_empty() { - continue; - } - - for channel in &mut plg.channels { - let cluster = channel.header.xtream_cluster; - let group = &channel.header.group; - - let entry = new_categories.entry((cluster, group.clone())).or_insert_with(|| { - let cat_id = existing_cat_ids.get(&(cluster, group.clone())).copied().unwrap_or_else(|| { - cat_id_counter += 1; - cat_id_counter - }); - - CategoryEntry { category_id: cat_id, category_name: group.clone(), parent_id: 0 } - }); - - channel.header.category_id = entry.category_id; - } - } - - new_categories - .into_iter() - .map(|((cluster, _group), value)| (cluster, value)) - .collect::>() -} - -pub fn xtream_get_collection_path(cfg: &Config, target_name: &str, collection_name: &str) -> Result { - if let Some(path) = xtream_get_storage_path(cfg, target_name) { - let col_path = get_collection_path(&path, collection_name); - if col_path.exists() { - return Ok(col_path); - } - } - Err(string_to_io_error(format!("Can't find collection: {target_name}/{collection_name}"))) -} - -async fn xtream_read_item_for_stream_id( - cfg: &AppConfig, - stream_id: u32, - storage_path: &Path, - cluster: XtreamCluster, -) -> Result { - let xtream_path = xtream_get_file_path(storage_path, cluster); - let file_lock = cfg.file_locks.read_lock(&xtream_path).await; - let xtream_path_clone = xtream_path.clone(); - tokio::task::spawn_blocking(move || -> Result { - let _guard = file_lock; - let mut query = BPlusTreeQuery::::try_new(&xtream_path_clone)?; - match query.query_zero_copy(&stream_id) { - Ok(Some(item)) => Ok(item), - Ok(None) => Err(Error::new(ErrorKind::NotFound, format!("Item {stream_id} not found in {cluster}"))), - Err(err) => Err(Error::other(format!("Query failed for {stream_id} in {cluster}: {err}"))), - } - }) - .await - .map_err(|err| Error::other(format!("Query task failed for {stream_id} in {cluster}: {err}")))? -} - -async fn xtream_read_series_item_for_stream_id( - cfg: &AppConfig, - stream_id: u32, - storage_path: &Path, -) -> Result { - let xtream_path = xtream_get_file_path(storage_path, XtreamCluster::Series); - let file_lock = cfg.file_locks.read_lock(&xtream_path).await; - let xtream_path_clone = xtream_path.clone(); - tokio::task::spawn_blocking(move || -> Result { - let _guard = file_lock; - let mut query = BPlusTreeQuery::::try_new(&xtream_path_clone)?; - match query.query_zero_copy(&stream_id) { - Ok(Some(item)) => Ok(item), - Ok(None) => Err(Error::new(ErrorKind::NotFound, format!("Item {stream_id} not found in series"))), - Err(err) => Err(Error::other(format!("Query failed for {stream_id} in series: {err}"))), - } - }) - .await - .map_err(|err| Error::other(format!("Query task failed for {stream_id} in series: {err}")))? -} - /// The stored cluster if the mapping has one, otherwise the item type's own. /// /// Was a `try_cluster!` returning a `Result` whose error arm was unreachable: @@ -950,300 +37,6 @@ macro_rules! cluster_or_item_type { }; } -async fn xtream_get_item_for_stream_id_from_memory( - virtual_id: u32, - playlists: &PlaylistStorageState, - target: &ConfigTarget, - xtream_cluster: Option, -) -> Result, Error> { - if let Some(playlist) = playlists.data.read().await.get(target.name.as_str()) { - return match (playlist.xtream.as_ref(), playlist.id_mapping.as_ref()) { - (Some(xtream_storage), Some(id_mapping)) => { - let mapping = id_mapping - .query(&VirtualId::new(virtual_id)) - .ok_or_else(|| { - string_to_io_error(format!( - "Could not find mapping for target {} and id {}", - target.name, virtual_id - )) - })? - .clone(); - let result = match mapping.item_type { - PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo => Ok(xtream_storage - .series - .query(&mapping.virtual_id.get()) - .ok_or_else(|| string_to_io_error(format!("Failed to read xtream item for id {virtual_id}")))? - .clone()), - PlaylistItemType::Series | PlaylistItemType::LocalSeries => { - log::debug!("In-memory series item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", virtual_id, mapping.parent_virtual_id, mapping.provider_id); - - if let Some(item) = xtream_storage.series.query(&virtual_id) { - Ok(item.clone()) - } else if let Some(item) = xtream_storage.series.query(&mapping.parent_virtual_id.get()) { - let mut xc_item = item.clone(); - xc_item.provider_id = mapping.provider_id; - xc_item.item_type = PlaylistItemType::Series; - xc_item.virtual_id = mapping.virtual_id; - Ok(xc_item) - } else { - Err(string_to_io_error(format!("Failed to read xtream item for id {virtual_id}"))) - } - } - PlaylistItemType::Catchup => { - log::debug!("In-memory catchup item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", virtual_id, mapping.parent_virtual_id, mapping.provider_id); - let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); - let item = match cluster { - XtreamCluster::Live => xtream_storage.live.query(&mapping.parent_virtual_id.get()), - XtreamCluster::Video => xtream_storage.vod.query(&mapping.parent_virtual_id.get()), - XtreamCluster::Series => xtream_storage.series.query(&mapping.parent_virtual_id.get()), - }; - - if let Some(pl_item) = item { - let mut xc_item = pl_item.clone(); - xc_item.provider_id = mapping.provider_id; - xc_item.item_type = PlaylistItemType::Catchup; - xc_item.virtual_id = mapping.virtual_id; - Ok(xc_item) - } else { - Err(string_to_io_error(format!("Failed to read xtream item for id {virtual_id}"))) - } - } - _ => { - let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); - Ok((match cluster { - XtreamCluster::Live => xtream_storage.live.query(&virtual_id), - XtreamCluster::Video => xtream_storage.vod.query(&virtual_id), - XtreamCluster::Series => xtream_storage.series.query(&virtual_id), - }) - .ok_or_else(|| string_to_io_error(format!("Failed to read xtream item for id {virtual_id}")))? - .clone()) - } - }; - - result.map(|xpli| Some((xpli, mapping))) - } - _ => Ok(None), - }; - } - //Err(string_to_io_error(format!("Failed to read xtream item for id {virtual_id}. No entry found."))) - Ok(None) -} - -pub async fn xtream_get_item_for_stream_id( - virtual_id: u32, - app_config: &Arc, - playlists: &PlaylistStorageState, - target: &ConfigTarget, - xtream_cluster: Option, -) -> Result { - if target.use_memory_cache { - if let Ok(Some((playlist_item, _virtual_record))) = - xtream_get_item_for_stream_id_from_memory(virtual_id, playlists, target, xtream_cluster).await - { - return Ok(playlist_item); - } - // fall through to disk lookup on cache miss - } - - let config = app_config.config.load(); - let target_path = get_target_storage_path(&config, target.name.as_str()) - .ok_or_else(|| string_to_io_error(format!("Could not find path for target {}", target.name)))?; - let storage_path = xtream_get_storage_path(&config, target.name.as_str()) - .ok_or_else(|| string_to_io_error(format!("Could not find path for target {} xtream output", target.name)))?; - { - let result = if let Some(cluster) = xtream_cluster { - xtream_read_item_for_stream_id(app_config, virtual_id, &storage_path, cluster).await - } else { - let target_id_mapping_file = get_target_id_mapping_file(&target_path); - let target_name = target.name.clone(); - let file_lock = app_config.file_locks.read_lock(&target_id_mapping_file).await; - let target_id_mapping_file_clone = target_id_mapping_file.clone(); - let mapping = tokio::task::spawn_blocking(move || -> Result { - let _guard = file_lock; - let mut target_id_mapping = - BPlusTreeQuery::::try_new(&target_id_mapping_file_clone).map_err(|err| { - string_to_io_error(format!("Could not load id mapping for target {target_name} err:{err}")) - })?; - match target_id_mapping.query_zero_copy(&virtual_id) { - Ok(Some(record)) => Ok(record), - Ok(None) => Err(string_to_io_error(format!( - "Could not find mapping for target {target_name} and id {virtual_id}" - ))), - Err(err) => Err(string_to_io_error(format!("Query failed for id {virtual_id}: {err}"))), - } - }) - .await - .map_err(|err| string_to_io_error(format!("Mapping query task failed for id {virtual_id}: {err}")))??; - match mapping.item_type { - PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo => { - xtream_read_series_item_for_stream_id(app_config, virtual_id, &storage_path).await - } - PlaylistItemType::Series | PlaylistItemType::LocalSeries => { - log::debug!( - "Disk series item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", - virtual_id, - mapping.parent_virtual_id, - mapping.provider_id - ); - - if let Ok(episode) = - xtream_read_item_for_stream_id(app_config, virtual_id, &storage_path, XtreamCluster::Series) - .await - { - return Ok(episode); - } - - if let Ok(mut item) = xtream_read_series_item_for_stream_id( - app_config, - mapping.parent_virtual_id.get(), - &storage_path, - ) - .await - { - item.provider_id = mapping.provider_id; - item.item_type = PlaylistItemType::Series; - item.virtual_id = mapping.virtual_id; - return Ok(item); - } - - return Err(Error::other(format!("Failed to find episode item with virtual-id {virtual_id}"))); - } - PlaylistItemType::Catchup => { - log::debug!( - "Disk catchup item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", - virtual_id, - mapping.parent_virtual_id, - mapping.provider_id - ); - let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); - let mut item = xtream_read_item_for_stream_id( - app_config, - mapping.parent_virtual_id.get(), - &storage_path, - cluster, - ) - .await?; - item.provider_id = mapping.provider_id; - item.item_type = PlaylistItemType::Catchup; - item.virtual_id = mapping.virtual_id; - Ok(item) - } - _ => { - let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); - xtream_read_item_for_stream_id(app_config, virtual_id, &storage_path, cluster).await - } - } - }; - - result - } -} - -pub async fn xtream_load_rewrite_playlist( - cluster: XtreamCluster, - app_config: &Arc, - target: &ConfigTarget, - category_id: Option, - user: &ProxyUserCredentials, -) -> Result { - XtreamPlaylistJsonIterator::new(cluster, app_config, target, category_id, user).await -} - -pub async fn iter_raw_xtream_target_playlist( - app_config: &AppConfig, - target: &ConfigTarget, - cluster: XtreamCluster, -) -> Option>> { - let config = app_config.config.load(); - let storage_path = xtream_get_storage_path(&config, target.name.as_str())?; - let xtream_path = xtream_get_file_path(&storage_path, cluster); - - // Xtream partitions by cluster at the file level, so every item in this - // database already belongs to `cluster` and no per-item filter is needed. - iter_raw_playlist::(app_config, &xtream_path, |_| true).await -} - -pub async fn iter_raw_xtream_input_playlist( - app_config: &AppConfig, - input: &ConfigInput, - cluster: XtreamCluster, -) -> Option>> { - let config = app_config.config.load(); - let storage_dir = &config.storage_dir; - let storage_path = get_input_storage_path(&input.name, storage_dir).await.ok()?; - let xtream_path = xtream_get_file_path(&storage_path, cluster); - - iter_raw_playlist::(app_config, &xtream_path, |_| true).await -} - -/// Counts entries in one active persisted raw input cluster without materializing them. -pub async fn count_input_xtream_cluster( - app_config: &AppConfig, - input: &ConfigInput, - cluster: XtreamCluster, -) -> Result, TuliproxError> { - let storage_dir = app_config.config.load().storage_dir.clone(); - let storage_path = get_input_storage_path(&input.name, &storage_dir).await.map_err(|err| { - TuliproxError::RepositoryXtream(format!("Failed to resolve active input storage for {}: {err}", input.name)) - })?; - let xtream_path = xtream_get_file_path(&storage_path, cluster); - let file_lock = app_config.file_locks.read_lock(&xtream_path).await; - let query_path = xtream_path.clone(); - let count = tokio::task::spawn_blocking(move || -> io::Result> { - let _guard = file_lock; - let mut query = match BPlusTreeQuery::::try_new(&query_path) { - Ok(query) => query, - Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), - Err(err) => return Err(err), - }; - query.len().map(Some).map_err(BPlusTreeError::to_io) - }) - .await - .map_err(|err| cant_read_result!(RepositoryXtream, "xtream", &xtream_path, err))? - .map_err(|err| cant_read_result!(RepositoryXtream, "xtream", &xtream_path, err))?; - - Ok(count) -} - -pub fn playlist_iter_to_stream(channels: Option<(FileReadGuard, I)>) -> impl Stream> -where - I: Iterator + 'static, - P: Serialize, -{ - match channels { - Some((_, chans)) => { - // Convert iterator items to Result with minimal allocations - let mapped = chans.map(move |(item, has_next)| match serde_json::to_string(&item) { - Ok(mut content) => { - if has_next { - content.push(','); - } - Ok(Bytes::from(content)) - } - Err(_) => Ok(Bytes::from("")), - }); - stream::iter(mapped).left_stream() - } - None => stream::once(async { Ok(Bytes::from("")) }).right_stream(), - } -} - -pub async fn xtream_get_playlist_categories( - app_config: &AppConfig, - target_name: &str, - cluster: XtreamCluster, -) -> Option> { - let file_path = { - let config = app_config.config.load(); - let storage_path = xtream_get_storage_path(&config, target_name)?; - get_collection_path(&storage_path, xtream_cluster_category_collection(cluster)) - }; - let category_lock_path = target_category_lock_path(&file_path); - let _file_lock = app_config.file_locks.read_lock(&category_lock_path).await; - let content = tokio::fs::read_to_string(&file_path).await.ok()?; - serde_json::from_str::>(&content).ok() -} - const BATCH_SIZE: usize = 1000; /// Result of publishing one fully staged Xtream input cluster. @@ -1282,25 +75,6 @@ pub struct XtreamClusterPublishBatchResult { pub failed_cluster: Option, } -impl XtreamClusterPublishBatchResult { - fn record_cluster_error(&mut self, cluster: XtreamCluster, error: TuliproxError) { - self.failed_cluster = Some(cluster); - self.errors.push(error); - } - - fn record_evaluation(&mut self, evaluation: XtreamClusterEvaluationReport) { - if let Some(acceptance) = evaluation.quality_acceptance { - self.quality_acceptances.push(acceptance); - } - if let Some(rejection) = evaluation.quality_rejection { - self.quality_rejections.push(rejection); - } - if let Some(force_update) = evaluation.force_update { - self.force_updates.push(force_update); - } - } -} - /// Quality behavior for one staged Xtream cluster publication. #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum XtreamClusterQualityPolicy { @@ -1316,3621 +90,73 @@ pub struct XtreamClusterRefreshRequest { pub streams: DynReader, } -#[derive(Clone, Copy, Debug, Default)] -struct XtreamClusterEvaluationReport { - quality_acceptance: Option, - quality_rejection: Option, - force_update: Option, -} +#[cfg(test)] +mod tests; -#[derive(Debug, Clone, Eq, PartialEq)] -struct XtreamRefreshPaths { - generation: Uuid, - published_database: PathBuf, - staging_database: PathBuf, - published_categories: PathBuf, - staging_categories: PathBuf, -} - -impl XtreamRefreshPaths { - fn new(storage_path: &Path, cluster: XtreamCluster) -> Result { - Self::for_generation(storage_path, cluster, Uuid::new_v4()) - } - - fn for_generation(storage_path: &Path, cluster: XtreamCluster, generation: Uuid) -> Result { - let published_database = xtream_get_file_path(storage_path, cluster); - let published_categories = get_collection_path(storage_path, xtream_cluster_category_collection(cluster)); - let staging_database = refresh_staging_path(&published_database, generation)?; - // The lock-domain check is repeated by `XtreamRefreshLease::new` with a stricter - // aliasing scan; doing it here too would canonicalize the same paths twice. - Ok(Self { - generation, - staging_database, - staging_categories: refresh_staging_path(&published_categories, generation)?, - published_database, - published_categories, - }) - } -} - -fn refresh_staging_path(path: &Path, generation: Uuid) -> Result { - let stem = path - .file_stem() - .ok_or_else(|| TuliproxError::RepositoryXtream(format!("Refresh path has no file stem: {}", path.display())))?; - let extension = path - .extension() - .ok_or_else(|| TuliproxError::RepositoryXtream(format!("Refresh path has no extension: {}", path.display())))?; - let mut filename = OsString::from(stem); - filename.push(".refresh-"); - filename.push(generation.simple().to_string()); - filename.push("."); - filename.push(extension); - Ok(path.with_file_name(filename)) -} - -#[derive(Debug, Clone)] -struct XtreamRefreshLease(Arc); - -#[derive(Debug)] -struct XtreamRefreshLeaseInner { - paths: XtreamRefreshPaths, - database_artifacts: BPlusTreeStagingArtifacts, - _generation_guard: XtreamRefreshGenerationGuard, -} - -impl XtreamRefreshLease { - fn new(paths: XtreamRefreshPaths) -> Result { - let database_artifacts = BPlusTreeStagingArtifacts::new(&paths.published_database, &paths.staging_database) - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Invalid Xtream staging artifacts for generation {}: {error}", - paths.generation - )) - })?; - let storage_path = paths.staging_database.parent().ok_or_else(|| { - TuliproxError::RepositoryXtream(format!( - "Xtream staging database has no storage directory: {}", - paths.staging_database.display() - )) - })?; - let generation_guard = - XtreamRefreshGenerationGuard::acquire(storage_path, paths.generation).map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to acquire Xtream refresh generation guard {} in {}: {error}", - paths.generation, - storage_path.display() - )) - })?; - Ok(Self(Arc::new(XtreamRefreshLeaseInner { paths, database_artifacts, _generation_guard: generation_guard }))) - } - - fn paths(&self) -> &XtreamRefreshPaths { &self.0.paths } - - fn cleanup_staging_artifacts(&self) -> io::Result<()> { - let database_result = self.0.database_artifacts.remove_owned_staging_artifacts(); - let categories_result = remove_file_if_exists(&self.0.paths.staging_categories); - match (database_result, categories_result) { - (Ok(()), Ok(())) => Ok(()), - (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Err(database_error), Err(categories_error)) => Err(io::Error::new( - database_error.kind(), - format!("{database_error}; category staging cleanup also failed: {categories_error}"), - )), - } - } -} - -impl Drop for XtreamRefreshLeaseInner { - fn drop(&mut self) { - let database_result = self.database_artifacts.remove_owned_staging_artifacts(); - let categories_result = remove_file_if_exists(&self.paths.staging_categories); - if let Err(error) = database_result { - log::warn!( - "Failed to clean Xtream staging database artifacts for generation {}: {error}", - self.paths.generation - ); - } - if let Err(error) = categories_result { - log::warn!( - "Failed to clean Xtream staging categories for generation {} at {}: {error}", - self.paths.generation, - self.paths.staging_categories.display() - ); - } - } -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum PreserveDetailsOutcome { - SourceMissing, - Merged { scanned: usize, updated: usize }, -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum DetailPreservationOperation { - Query, - BatchWrite, - Commit, -} - -fn write_preserved_detail_batch( - staging_tree: &mut BPlusTreeUpdate, - staging_path: &Path, - updates: &mut Vec<(u32, XtreamPlaylistItem)>, - before_operation: &mut F, -) -> Result -where - F: FnMut(DetailPreservationOperation) -> io::Result<()>, -{ - if updates.is_empty() { - return Ok(0); - } - let batch_len = updates.len(); - let refs: Vec<(&u32, &XtreamPlaylistItem)> = updates.iter().map(|(id, item)| (id, item)).collect(); - before_operation(DetailPreservationOperation::BatchWrite) - .and_then(|()| staging_tree.update_batch(&refs).map(|_| ()).map_err(BPlusTreeError::to_io)) - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to update staging Xtream tree {} during detail preservation: {error}", - staging_path.display() - )) - })?; - updates.clear(); - Ok(batch_len) -} - -fn preserve_details_input_xtream_playlist_cluster_to_disk( - published_path: &Path, - staging_path: &Path, -) -> Result { - preserve_details_input_xtream_playlist_cluster_to_disk_with_hook(published_path, staging_path, |_| Ok(())) -} - -fn preserve_details_input_xtream_playlist_cluster_to_disk_with_hook( - published_path: &Path, - staging_path: &Path, - mut before_operation: F, -) -> Result -where - F: FnMut(DetailPreservationOperation) -> io::Result<()>, -{ - ensure_distinct_sidecar_lock_domains(published_path, staging_path) - .map_err(|error| TuliproxError::RepositoryXtream(error.to_string()))?; - - let mut published_tree = match BPlusTreeQuery::::try_new(published_path) { - Ok(tree) => tree, - Err(error) if error.kind() == io::ErrorKind::NotFound => { - return Ok(PreserveDetailsOutcome::SourceMissing); - } - Err(error) => { - return Err(TuliproxError::RepositoryXtream(format!( - "Failed to open published Xtream tree {} for detail preservation: {error}", - published_path.display() - ))); - } - }; - - let mut staging_tree = - BPlusTreeUpdate::::try_new_with_backoff(staging_path).map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to open staging Xtream tree {} for detail preservation: {error}", - staging_path.display() - )) - })?; - - let mut pending_updates: Vec<(u32, XtreamPlaylistItem)> = Vec::with_capacity(BATCH_SIZE); - let mut scanned_count = 0usize; - let mut updated_count = 0usize; - for entry in published_tree.iter() { - let (_, old_item) = entry.map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to read published Xtream tree {} during detail preservation: {error}", - published_path.display() - )) - })?; - scanned_count = scanned_count.saturating_add(1); - if let Some(old_props) = old_item.additional_properties.as_ref() { - if old_props.has_details() { - let staging_item = before_operation(DetailPreservationOperation::Query) - .and_then(|()| staging_tree.query(&old_item.provider_id).map_err(BPlusTreeError::to_io)) - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to query staging Xtream tree {} for provider {}: {error}", - staging_path.display(), - old_item.provider_id - )) - })?; - if let Some(mut new_item) = staging_item { - if let Some(new_props) = new_item.additional_properties.as_mut() { - if merge_preserved_stream_properties(new_props, old_props) { - pending_updates.push((new_item.provider_id, new_item)); - if pending_updates.len() >= BATCH_SIZE { - updated_count = updated_count.saturating_add(write_preserved_detail_batch( - &mut staging_tree, - staging_path, - &mut pending_updates, - &mut before_operation, - )?); - } - } - } - } - } - } - } - - updated_count = updated_count.saturating_add(write_preserved_detail_batch( - &mut staging_tree, - staging_path, - &mut pending_updates, - &mut before_operation, - )?); - before_operation(DetailPreservationOperation::Commit).and_then(|()| staging_tree.commit()).map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to commit staging Xtream tree {} after detail preservation: {error}", - staging_path.display() - )) - })?; - - Ok(PreserveDetailsOutcome::Merged { scanned: scanned_count, updated: updated_count }) -} +mod details; +mod input_refresh; +mod paths; +mod publish; +mod read; +mod target_publish; #[cfg(test)] -fn preserve_details_with_injected_operation_failure( - published_path: &Path, - staging_path: &Path, - failure: DetailPreservationOperation, -) -> Result { - preserve_details_input_xtream_playlist_cluster_to_disk_with_hook(published_path, staging_path, move |operation| { - if operation == failure { - Err(io::Error::other(format!("injected {failure:?} failure"))) - } else { - Ok(()) - } - }) -} - -#[derive(Clone, Copy)] -struct XtreamClusterStageOperations { - preserve_details: fn(&Path, &Path) -> Result, -} - -impl Default for XtreamClusterStageOperations { - fn default() -> Self { Self { preserve_details: preserve_details_input_xtream_playlist_cluster_to_disk } } -} - -struct StagedXtreamClusterRefresh { - refresh_lease: XtreamRefreshLease, - publish_lock: Arc, - storage_path: PathBuf, - input_name: Arc, - cluster: XtreamCluster, - raw_groups: Vec, - item_count: usize, - evaluation: XtreamClusterEvaluationReport, -} - -enum StagedXtreamClusterOutcome { - Ready(StagedXtreamClusterRefresh), - RetainedPrevious(ClusterUpdateRejection), - Failed { evaluation: XtreamClusterEvaluationReport, error: TuliproxError }, -} - -impl StagedXtreamClusterOutcome { - const fn evaluation(&self) -> XtreamClusterEvaluationReport { - match self { - Self::Ready(refresh) => refresh.evaluation, - Self::RetainedPrevious(rejection) => XtreamClusterEvaluationReport { - quality_acceptance: None, - quality_rejection: Some(*rejection), - force_update: None, - }, - Self::Failed { evaluation, .. } => *evaluation, - } - } -} - -fn count_xtream_tree_entries(path: &Path) -> io::Result> { - let mut query = match BPlusTreeQuery::::try_new(path) { - Ok(query) => query, - Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(error), - }; - query.len().map(Some).map_err(BPlusTreeError::to_io) -} - -fn evaluate_staged_xtream_cluster_quality( - paths: &XtreamRefreshPaths, - cluster: XtreamCluster, - threshold: u8, -) -> Result { - if threshold == 0 { - return Ok(UpdateQualityDecision::Disabled); - } - - let candidate_count = count_xtream_tree_entries(&paths.staging_database) - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to count staging Xtream tree {} for {cluster} quality evaluation: {error}", - paths.staging_database.display() - )) - })? - .ok_or_else(|| { - TuliproxError::RepositoryXtream(format!( - "Staging Xtream tree {} disappeared before {cluster} quality evaluation", - paths.staging_database.display() - )) - })?; - let current_count = count_xtream_tree_entries(&paths.published_database).map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to count published Xtream tree {} for {cluster} quality evaluation: {error}", - paths.published_database.display() - )) - })?; - - Ok(evaluate_update_quality(current_count, candidate_count, threshold)) -} - -#[allow(clippy::too_many_lines)] -async fn stage_input_xtream_playlist_cluster_to_disk( - app_config: &Arc, - input: &ConfigInput, - request: XtreamClusterRefreshRequest, - operations: XtreamClusterStageOperations, -) -> Result { - let XtreamClusterRefreshRequest { cluster, quality: quality_policy, categories, streams } = request; - let cfg = app_config.config.load(); - let storage_path = ensure_input_storage_path(&cfg, &input.name).await?; - drop(cfg); - let refresh_lease = XtreamRefreshLease::new(XtreamRefreshPaths::new(&storage_path, cluster)?)?; - - // Channel for transferring items from Parser (Async Task) to Consumer (Blocking Task) - let (tx, mut rx) = tokio::sync::mpsc::channel::(BATCH_SIZE * 2); - let input_clone = input.clone(); - - // 1. Parser Task: Runs the async parsing logic - // We move the readers into this task. - let parse_task = tokio::spawn(async move { - let tx_for_closure = tx.clone(); - let res = xtream::parse_xtream_streaming(&input_clone, cluster, categories, streams, move |item| { - // Copy needed data before moving the item into the channel. - let item_id = item.virtual_id; - - // We use blocking_send because the closure provided by the parser library is synchronous. - // This is safe here because it runs within its own tokio::spawn task. - if let Err(e) = tx_for_closure.blocking_send(item) { - error!("Channel closed while processing {cluster} for item {item_id}: {e}"); - return Err(TuliproxError::RepositoryXtream(format!("Channel closed while processing {cluster}"))); - } - Ok(()) - }) - .await; - - // CRITICAL: Explicitly drop the sender to signal rx.blocking_recv() to stop. - // This prevents the consumer from waiting forever if the parser fails. - drop(tx); - res - }); - - // 2. Consumer Task: Handles heavy Disk I/O (BPlusTree updates) - let consumer_lease = refresh_lease.clone(); - let consumer_task = tokio::task::spawn_blocking(move || { - let staging_path = &consumer_lease.paths().staging_database; - - BPlusTree::::new().store(staging_path).map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to initialize staging Xtream tree {} for {cluster}: {error}", - staging_path.display() - )) - })?; - - let mut tree: BPlusTreeUpdate = BPlusTreeUpdate::try_new_with_backoff(staging_path) - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to open staging Xtream tree {} for {cluster}: {error}", - staging_path.display() - )) - })?; - tree.set_flush_policy(FlushPolicy::Batch); - - let mut buffer = Vec::with_capacity(BATCH_SIZE); - let mut seen_groups: HashSet = HashSet::new(); - let mut item_count = 0_usize; - - // This loop exits when all 'tx' clones are dropped (signaling end of stream) - while let Some(item) = rx.blocking_recv() { - item_count = item_count.saturating_add(1); - if !seen_groups.contains(item.group.as_ref()) { - seen_groups.insert(item.group.to_string()); - } - buffer.push(item); - if buffer.len() >= BATCH_SIZE { - let batch: Vec<(&u32, &XtreamPlaylistItem)> = buffer.iter().map(|i| (&i.provider_id, i)).collect(); - let prepared = - BPlusTreeUpdate::::prepare_upsert_batch(&batch).map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to prepare staging batch for {cluster} at {}: {error}", - staging_path.display() - )) - })?; - tree.upsert_batch_encoded(prepared).map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to write staging batch for {cluster} at {}: {error}", - staging_path.display() - )) - })?; - // Commit per batch so the write transaction's dirty-page map stays bounded. - // Holding it open for the whole cluster buffered ~48k pages (196 MB); the - // import writes into a .tmp file that is renamed on success, so atomicity - // comes from the rename, not from a single transaction. - tree.commit().map_err(|e| { - error!("Batch commit failed for cluster {cluster} at {}: {e}", staging_path.display()); - TuliproxError::RepositoryXtream(format!( - "Failed to commit staging batch for {cluster} at {}: {e}", - staging_path.display() - )) - })?; - buffer.clear(); - } - } - - // Final batch processing - if !buffer.is_empty() { - let batch: Vec<(&u32, &XtreamPlaylistItem)> = buffer.iter().map(|i| (&i.provider_id, i)).collect(); - let prepared = - BPlusTreeUpdate::::prepare_upsert_batch(&batch).map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to prepare final staging batch for {cluster} at {}: {error}", - staging_path.display() - )) - })?; - tree.upsert_batch_encoded(prepared).map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to write final staging batch for {cluster} at {}: {error}", - staging_path.display() - )) - })?; - } - - tree.commit().map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to commit staging Xtream tree for {cluster} at {}: {error}", - staging_path.display() - )) - })?; - Ok::<(Vec, usize), TuliproxError>((seen_groups.into_iter().collect(), item_count)) - }); - - // 3. Robust Joining of both tasks - // try_join! returns immediately if any task returns an error or panics. - let (parse_res, consumer_res) = tokio::try_join!(parse_task, consumer_task).map_err(|e| { - TuliproxError::RepositoryXtream(format!("Task join error during cluster {cluster} update: {e}")) - })?; - - // Handle internal errors from the tasks - let parsed_categories = parse_res?; - let (raw_groups, item_count) = consumer_res?; - - save_xtream_categories_to_file(refresh_lease.clone(), &parsed_categories).await?; - - // Lock order for the publish phase is always FileLockManager(final) followed by B+Tree sidecars. No B+Tree - // handle escapes its blocking closure, so none is held while this async lock is acquired. - let publish_lock = Arc::new(app_config.file_locks.write_lock(&refresh_lease.paths().published_database).await); - - let quality_lease = refresh_lease.clone(); - let quality_lock = Arc::clone(&publish_lock); - let (evaluation, rejection_cleanup_error) = tokio::task::spawn_blocking(move || { - let _publish_guard = quality_lock; - let evaluation = match quality_policy { - XtreamClusterQualityPolicy::Enforce { threshold } => { - let decision = evaluate_staged_xtream_cluster_quality(quality_lease.paths(), cluster, threshold)?; - XtreamClusterEvaluationReport { - quality_acceptance: decision.acceptance(cluster), - quality_rejection: decision.rejection(cluster), - force_update: None, - } - } - XtreamClusterQualityPolicy::Bypass { configured_threshold } => { - let candidate_count = count_xtream_tree_entries(&quality_lease.paths().staging_database) - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to count staging Xtream tree {} for forced {cluster} publication: {error}", - quality_lease.paths().staging_database.display() - )) - })? - .ok_or_else(|| { - TuliproxError::RepositoryXtream(format!( - "Staging Xtream tree {} disappeared before forced {cluster} publication", - quality_lease.paths().staging_database.display() - )) - })?; - XtreamClusterEvaluationReport { - quality_acceptance: None, - quality_rejection: None, - force_update: Some(ClusterForceUpdate { cluster, candidate_count, configured_threshold }), - } - } - }; - let rejection_cleanup_error = if evaluation.quality_rejection.is_some() { - quality_lease.cleanup_staging_artifacts().err().map(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to clean rejected Xtream staging artifacts for {cluster}: {error}" - )) - }) - } else { - None - }; - Ok::<_, TuliproxError>((evaluation, rejection_cleanup_error)) - }) - .await - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Quality-evaluation task failed to join during {cluster} refresh: {error}" - )) - })??; - - if let Some(error) = rejection_cleanup_error { - return Ok(StagedXtreamClusterOutcome::Failed { evaluation, error }); - } - - if let Some(rejection) = evaluation.quality_rejection { - drop(publish_lock); - log::debug!( - "Xtream cluster candidate rejected; retained active cluster: cluster={cluster} generation={} rejection={rejection:?}", - refresh_lease.paths().generation - ); - return Ok(StagedXtreamClusterOutcome::RetainedPrevious(rejection)); - } - - let merge_lease = refresh_lease.clone(); - let merge_lock = Arc::clone(&publish_lock); - let preserve_details = operations.preserve_details; - let merge_result = tokio::task::spawn_blocking(move || { - let _publish_guard = merge_lock; - preserve_details(&merge_lease.paths().published_database, &merge_lease.paths().staging_database) - }) - .await - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Detail-preservation task failed to join during {cluster} refresh: {error}" - )) - }); - let merge_outcome = match merge_result { - Ok(Ok(outcome)) => outcome, - Ok(Err(error)) | Err(error) => return Ok(StagedXtreamClusterOutcome::Failed { evaluation, error }), - }; - log::debug!( - "Xtream cluster detail preservation completed: cluster={cluster} generation={} outcome={merge_outcome:?}", - refresh_lease.paths().generation - ); - - let compact_lease = refresh_lease.clone(); - let compact_lock = Arc::clone(&publish_lock); - let compact_result = tokio::task::spawn_blocking(move || { - let _publish_guard = compact_lock; - let staging_path = &compact_lease.paths().staging_database; - let mut tree = - BPlusTreeUpdate::::try_new_with_backoff(staging_path).map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to open staging Xtream tree {} for compaction: {error}", - staging_path.display() - )) - })?; - tree.compact().map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to compact staging Xtream tree {}: {error}", - staging_path.display() - )) - }) - }) - .await - .map_err(|error| { - TuliproxError::RepositoryXtream(format!("Compaction task failed to join during {cluster} refresh: {error}")) - }); - match compact_result { - Ok(Ok(())) => {} - Ok(Err(error)) | Err(error) => return Ok(StagedXtreamClusterOutcome::Failed { evaluation, error }), - } - - Ok(StagedXtreamClusterOutcome::Ready(StagedXtreamClusterRefresh { - refresh_lease, - publish_lock, - storage_path, - input_name: Arc::clone(&input.name), - cluster, - raw_groups, - item_count, - evaluation, - })) -} - -async fn publish_staged_xtream_cluster( - app_config: &AppConfig, - staged: StagedXtreamClusterRefresh, -) -> Result<(), TuliproxError> { - let StagedXtreamClusterRefresh { - refresh_lease, - publish_lock, - storage_path, - input_name, - cluster, - raw_groups, - item_count: _, - evaluation: _, - } = staged; - - let database_publish_lease = refresh_lease.clone(); - let database_publish_lock = Arc::clone(&publish_lock); - tokio::task::spawn_blocking(move || { - let _publish_guard = database_publish_lock; - publish_staged_database::( - &database_publish_lease.paths().staging_database, - &database_publish_lease.paths().published_database, - ) - .map_err(|error| { - TuliproxError::RepositoryXtream(format!("Failed to publish staging Xtream database for {cluster}: {error}")) - }) - }) - .await - .map_err(|error| { - TuliproxError::RepositoryXtream(format!("Database publish task failed to join for {cluster}: {error}")) - })??; - - let category_publish_lease = refresh_lease.clone(); - let category_publish_lock = Arc::clone(&publish_lock); - tokio::task::spawn_blocking(move || { - let _publish_guard = category_publish_lock; - publish_staged_file_same_directory( - &category_publish_lease.paths().staging_categories, - &category_publish_lease.paths().published_categories, - ) - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Xtream database for {cluster} was published, but category publication failed: {error}" - )) - }) - }) - .await - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Xtream database for {cluster} was published, but the category publish task failed to join: {error}" - )) - })??; - - if let Err(publish_err) = - crate::publish_raw_group_catalog(&storage_path, &input_name, cluster, raw_groups, &app_config.file_locks).await - { - log::warn!( - "Xtream data for input '{input_name}' cluster {cluster} was published, but its raw group catalog could not be published: {publish_err}" - ); - } - - let cleanup_lease = refresh_lease.clone(); - let cleanup_lock = Arc::clone(&publish_lock); - tokio::task::spawn_blocking(move || { - let _publish_guard = cleanup_lock; - cleanup_lease.cleanup_staging_artifacts() - }) - .await - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Xtream refresh for {cluster} was published, but cleanup task failed to join: {error}" - )) - })? - .map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Xtream refresh for {cluster} was published, but staging cleanup failed: {error}" - )) - })?; - - drop(publish_lock); - log::debug!( - "Xtream cluster updated successfully: cluster={cluster} generation={}", - refresh_lease.paths().generation - ); - Ok(()) -} - -fn input_cluster_enabled(input: &ConfigInput, cluster: XtreamCluster) -> bool { - match cluster { - XtreamCluster::Live => !input.has_flag(ConfigInputFlags::SkipLive), - XtreamCluster::Video => !input.has_flag(ConfigInputFlags::SkipVod), - XtreamCluster::Series => !input.has_flag(ConfigInputFlags::SkipSeries), - } -} - -async fn published_xtream_cluster_has_items( - app_config: &AppConfig, - storage_path: &Path, - cluster: XtreamCluster, -) -> Result { - let path = xtream_get_file_path(storage_path, cluster); - let lock = app_config.file_locks.read_lock(&path).await; - tokio::task::spawn_blocking(move || { - let _guard = lock; - match BPlusTreeQuery::::try_new(&path) { - Ok(mut query) => query.iter().next().transpose().map(|entry| entry.is_some()).map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to inspect published Xtream cluster {cluster} at {}: {error}", - path.display() - )) - }), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), - Err(error) => Err(TuliproxError::RepositoryXtream(format!( - "Failed to open published Xtream cluster {cluster} at {}: {error}", - path.display() - ))), - } - }) - .await - .map_err(|error| TuliproxError::Task(format!("Failed to join Xtream cluster inspection: {error}")))? -} - -pub async fn persist_input_xtream_playlist_clusters_to_disk( - app_config: &Arc, - input: &ConfigInput, - cluster_readers: Vec, -) -> XtreamClusterPublishBatchResult { - persist_input_xtream_playlist_clusters_to_disk_with_operations( - app_config, - input, - cluster_readers, - XtreamClusterStageOperations::default(), - ) - .await -} - -async fn persist_input_xtream_playlist_clusters_to_disk_with_operations( - app_config: &Arc, - input: &ConfigInput, - cluster_readers: Vec, - operations: XtreamClusterStageOperations, -) -> XtreamClusterPublishBatchResult { - let mut result = XtreamClusterPublishBatchResult::default(); - let mut staged = Vec::with_capacity(cluster_readers.len()); - for request in cluster_readers { - let cluster = request.cluster; - let outcome = match stage_input_xtream_playlist_cluster_to_disk(app_config, input, request, operations).await { - Ok(outcome) => outcome, - Err(error) => { - result.record_cluster_error(cluster, error); - return result; - } - }; - result.record_evaluation(outcome.evaluation()); - match outcome { - StagedXtreamClusterOutcome::Failed { error, .. } => { - result.record_cluster_error(cluster, error); - return result; - } - ready_or_retained => staged.push(ready_or_retained), - } - } - - let staged_clusters: HashSet = staged - .iter() - .filter_map(|outcome| match outcome { - StagedXtreamClusterOutcome::Ready(refresh) => Some(refresh.cluster), - StagedXtreamClusterOutcome::RetainedPrevious(_) | StagedXtreamClusterOutcome::Failed { .. } => None, - }) - .collect(); - let has_publishable_clusters = !staged_clusters.is_empty(); - let mut has_items = staged - .iter() - .any(|outcome| matches!(outcome, StagedXtreamClusterOutcome::Ready(refresh) if refresh.item_count > 0)); - if has_publishable_clusters && !has_items { - let cfg = app_config.config.load(); - let storage_path = match ensure_input_storage_path(&cfg, &input.name).await { - Ok(storage_path) => storage_path, - Err(error) => { - drop(cfg); - result.errors.push(error); - return result; - } - }; - drop(cfg); - for cluster in XTREAM_CLUSTER { - if input_cluster_enabled(input, cluster) && !staged_clusters.contains(&cluster) { - match published_xtream_cluster_has_items(app_config, &storage_path, cluster).await { - Ok(true) => { - has_items = true; - break; - } - Ok(false) => {} - Err(error) => { - result.errors.push(error); - return result; - } - } - } - } - } - - let all_publishable_clusters_are_forced = has_publishable_clusters - && staged.iter().all( - |outcome| matches!(outcome, StagedXtreamClusterOutcome::Ready(refresh) if refresh.evaluation.force_update.is_some()), - ); - if has_publishable_clusters && !has_items && !all_publishable_clusters_are_forced { - result.errors.push(TuliproxError::RepositoryPlaylist(format!( - "Refusing to publish empty disk-based Xtream playlist for input '{}'; existing data was retained", - input.name - ))); - return result; - } - - for outcome in staged { - match outcome { - StagedXtreamClusterOutcome::Ready(refresh) => { - let evaluation = refresh.evaluation; - let cluster = refresh.cluster; - if let Err(error) = publish_staged_xtream_cluster(app_config, refresh).await { - result.record_cluster_error(cluster, error); - break; - } - result.outcomes.push(if let Some(force_update) = evaluation.force_update { - XtreamClusterPublishOutcome::ForcePublished(force_update) - } else if let Some(quality_acceptance) = evaluation.quality_acceptance { - XtreamClusterPublishOutcome::QualityAccepted(quality_acceptance) - } else { - XtreamClusterPublishOutcome::Published - }); - } - StagedXtreamClusterOutcome::RetainedPrevious(rejection) => { - result.outcomes.push(XtreamClusterPublishOutcome::RetainedPrevious(rejection)); - } - StagedXtreamClusterOutcome::Failed { error, .. } => { - result.errors.push(error); - break; - } - } - } - result -} - -pub async fn persist_input_xtream_playlist_cluster_to_disk( - app_config: &Arc, - input: &ConfigInput, - cluster: XtreamCluster, - quality_threshold: u8, - categories: DynReader, - streams: DynReader, -) -> Result { - let mut result = persist_input_xtream_playlist_clusters_to_disk( - app_config, - input, - vec![XtreamClusterRefreshRequest { - cluster, - quality: XtreamClusterQualityPolicy::Enforce { threshold: quality_threshold }, - categories, - streams, - }], - ) - .await; - if let Some(error) = result.errors.pop() { - return Err(error); - } - result.outcomes.pop().ok_or_else(|| { - TuliproxError::RepositoryXtream(format!( - "Missing Xtream publish outcome for input '{}' cluster {cluster}", - input.name - )) - }) -} - -fn publish_staged_file_same_directory(staging: &Path, published: &Path) -> io::Result<()> { - require_same_parent_directory(staging, published)?; - let staging_path = tempfile::TempPath::try_from_path(staging).map_err(|error| { - io::Error::new( - error.kind(), - format!("failed to prepare staging file {} for publication: {error}", staging.display()), - ) - })?; - publish_staged_file_platform(staging_path, published) -} - -#[cfg(not(windows))] -fn publish_staged_file_platform(staging_path: tempfile::TempPath, published: &Path) -> io::Result<()> { - publish_staged_file_with_parent_sync(staging_path, published, sync_published_file_parent) -} - -#[cfg(not(windows))] -fn publish_staged_file_with_parent_sync( - staging_path: tempfile::TempPath, - published: &Path, - sync_parent: impl FnOnce(&Path) -> io::Result<()>, -) -> io::Result<()> { - staging_path.persist(published).map_err(io::Error::from)?; - sync_parent(published).map_err(|error| { - io::Error::new( - error.kind(), - format!( - "file {} was published, but its parent directory {} could not be synchronized: {error}", - published.display(), - parent_or_dot(published).display() - ), - ) - }) -} - +use self::details::preserve_details_with_injected_operation_failure; +#[cfg(test)] +use self::details::DetailPreservationOperation; +#[cfg(test)] +use self::input_refresh::{ + count_xtream_tree_entries, evaluate_staged_xtream_cluster_quality, + persist_input_xtream_playlist_clusters_to_disk_with_operations, XtreamClusterStageOperations, +}; #[cfg(windows)] -fn publish_staged_file_platform(mut staging_path: tempfile::TempPath, published: &Path) -> io::Result<()> { - move_target_file_platform(staging_path.as_ref(), published, TargetFileMoveMode::ReplaceDestination)?; - - // MoveFileExW consumed the source path. Prevent TempPath from issuing a - // redundant delete for a path that no longer exists. - staging_path.disable_cleanup(true); - Ok(()) -} - -#[cfg(windows)] -fn encode_windows_path(path: &Path) -> io::Result> { - use std::os::windows::ffi::OsStrExt; - - let mut encoded = path.as_os_str().encode_wide().collect::>(); - if encoded.contains(&0) { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("Windows path contains an embedded NUL: {}", path.display()), - )); - } - encoded.push(0); - Ok(encoded) -} - -#[cfg(unix)] -fn sync_published_file_parent(path: &Path) -> io::Result<()> { File::open(parent_or_dot(path))?.sync_all() } - -/// Every Windows transaction move uses `MOVEFILE_WRITE_THROUGH`; reaching -/// this barrier therefore means all preceding backup or publication moves -/// have completed durably without a second raw rename. -#[cfg(windows)] -fn sync_published_file_parent(_path: &Path) -> io::Result<()> { Ok(()) } - -/// There is no supported directory durability barrier for other targets. -/// Callers report this only after the atomic rename has completed. +use self::paths::encode_windows_path; +#[cfg(all(test, unix))] +use self::paths::refresh_staging_path; +#[cfg(all(test, not(windows)))] +use self::publish::publish_staged_file_with_parent_sync; #[cfg(all(not(unix), not(windows)))] -fn sync_published_file_parent(_path: &Path) -> io::Result<()> { - Err(io::Error::new(io::ErrorKind::Unsupported, "parent-directory synchronization is unsupported on this platform")) -} - -/// Owns the staging category file plus its `flock`, so the lock is released -/// even when a later step (`serde_json::to_writer`, `sync_all`) returns an -/// error. The unlock runs in `Drop` and is logged on failure; closing the -/// underlying `File` releases the OS-level lock either way. -struct LockedCategoryFile { - file: File, - path: PathBuf, -} - -impl LockedCategoryFile { - fn create(path: &Path) -> io::Result { - let file = File::create(path)?; - file.lock()?; - Ok(Self { file, path: path.to_path_buf() }) - } - - fn sync_all(&self) -> io::Result<()> { self.file.sync_all() } -} - -impl Drop for LockedCategoryFile { - fn drop(&mut self) { - if let Err(error) = self.file.unlock() { - log::warn!( - "Failed to unlock staging category file {}: {error}; the OS will release it on close", - self.path.display() - ); - } - } -} - -async fn save_xtream_categories_to_file( - refresh_lease: XtreamRefreshLease, - categories: &[XtreamCategory], -) -> Result<(), TuliproxError> { - let cat_entries: Vec = categories - .iter() - .map(|c| CategoryEntry { category_id: c.category_id, category_name: c.category_name.clone(), parent_id: 0 }) - .collect(); - - tokio::task::spawn_blocking(move || { - let staging_path = &refresh_lease.paths().staging_categories; - let locked = LockedCategoryFile::create(staging_path).map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to create or lock staging category file {}: {error}", - staging_path.display() - )) - })?; - serde_json::to_writer(&locked.file, &cat_entries).map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to write staging category file {}: {error}", - staging_path.display() - )) - })?; - locked.sync_all().map_err(|error| { - TuliproxError::RepositoryXtream(format!( - "Failed to synchronize staging category file {}: {error}", - staging_path.display() - )) - })?; - Ok(()) - }) - .await - .map_err(|e| TuliproxError::RepositoryXtream(format!("Spawn error {e}")))? -} - -pub async fn persist_input_xtream_playlist( - app_config: &Arc, - storage_path: &Path, - playlist: Vec, -) -> (Vec, Option) { - persist_input_xtream_playlist_with_empty_replacements(app_config, storage_path, playlist, ClusterFlags::empty()) - .await -} - -#[allow(clippy::too_many_lines)] -pub async fn persist_input_xtream_playlist_with_empty_replacements( - app_config: &Arc, - storage_path: &Path, - playlist: Vec, - replace_empty_clusters: ClusterFlags, -) -> (Vec, Option) { - let mut errors = Vec::new(); - - let mut fetched_categories = PlaylistScratch::>::new(1_000); - let mut fetched_scratch = PlaylistScratch::>::new(50_000); - let mut stored_scratch = PlaylistScratch::>::new(50_000); - - // load - for cluster in XTREAM_CLUSTER { - let xtream_path = xtream_get_file_path(storage_path, cluster); - if file_exists_async(&xtream_path).await { - let file_lock = app_config.file_locks.read_lock(&xtream_path).await; - let xtream_path = xtream_path.clone(); - let stored_entries = match tokio::task::spawn_blocking(move || { - let _guard = file_lock; - let mut entries = IndexMap::new(); - let mut query = BPlusTreeQuery::::try_new(&xtream_path)?; - for entry in query.iter() { - let (_, doc) = entry?; - entries.insert(doc.provider_id, doc); - } - Ok::<_, std::io::Error>(entries) - }) - .await - { - Ok(Ok(entries)) => Some(entries), - Ok(Err(err)) => { - errors.push(format!("Failed to read stored xtream playlist entries for {cluster}: {err}")); - None - } - Err(err) => { - errors.push(format!("Failed to load stored xtream playlist entries for {cluster}: {err}")); - None - } - }; - - if let Some(entries) = stored_entries { - *stored_scratch.get_mut(cluster) = entries; - } - } - } - - if !errors.is_empty() { - return (playlist, Some(TuliproxError::RepositoryXtream(errors.join("\n")))); - } - - let mut groups = IndexMap::new(); - - for mut plg in playlist { - if !&plg.channels.is_empty() { - fetched_categories.get_mut(plg.xtream_cluster).push(json!(CategoryEntry { - category_id: plg.id, - category_name: plg.title.clone(), - parent_id: 0 - })); - - let channels = std::mem::take(&mut plg.channels); - for mut pli in channels { - let stored_col = stored_scratch.get_mut(plg.xtream_cluster); - let fetched_col = fetched_scratch.get_mut(plg.xtream_cluster); - - if let Ok(provider_id) = pli.header.id.parse::() { - if let Some(stored_pli) = stored_col.get_mut(&provider_id) { - if let (Some(new_stream_props), Some(old_stream_props)) = - (&mut pli.header.additional_properties, stored_pli.additional_properties.take()) - { - merge_preserved_stream_properties(new_stream_props, &old_stream_props); - } - } - } - fetched_col.push(pli); - } - groups.insert((plg.xtream_cluster, plg.id), plg); - } - } - - let mut processed_scratch = PlaylistScratch::>::new(0); - for xc in XTREAM_CLUSTER { - processed_scratch.set( - xc, - if !replace_empty_clusters.contains(cluster_flag(xc)) - && !stored_scratch.is_empty(xc) - && fetched_scratch.is_empty(xc) - { - stored_scratch.take(xc).iter().map(|(_, item)| PlaylistItem::from(item)).collect::>() - } else { - fetched_scratch.take(xc) - }, - ); - } - drop(stored_scratch); - drop(fetched_scratch); - - let root_path = storage_path.to_path_buf(); - let app_cfg = app_config.clone(); - for cluster in XTREAM_CLUSTER { - let col_path = get_collection_path(&root_path, xtream_cluster_category_collection(cluster)); - let data = fetched_categories.get_mut(cluster); - // if there is no data save only if no file exists! Prevent data loss from failed download attempt - if !data.is_empty() - || replace_empty_clusters.contains(cluster_flag(cluster)) - || !file_exists_async(&col_path).await - { - let lock = app_cfg.file_locks.write_lock(&col_path).await; - if let Err(err) = json_write_documents_to_file(&col_path, data).await { - errors.push(format!("Persisting collection failed: {}: {err}", col_path.display())); - } - drop(lock); - } - } - - for cluster in XTREAM_CLUSTER { - let col = processed_scratch.take(cluster); - - // persist playlist - if let Err(err) = write_playlists_to_file( - app_config, - storage_path, - false, - |item| ProviderId::new(item.provider_id), - vec![(cluster, col.iter().map(Into::into).collect::>())], - replace_empty_clusters, - ) - .await - { - errors.push(format!("Persisting collection failed:{err}")); - } - - for item in col { - let group_key = (item.header.xtream_cluster, item.header.category_id); - groups - .entry(group_key) - .or_insert_with(|| PlaylistGroup { - id: item.header.category_id, - title: item.header.group.clone(), - channels: Vec::new(), - xtream_cluster: item.header.xtream_cluster, - }) - .channels - .push(item); - } - } - - let result = groups.into_values().collect(); - - let err = if errors.is_empty() { None } else { Some(TuliproxError::RepositoryXtream(errors.join("\n"))) }; - - (result, err) -} - -const fn cluster_flag(cluster: XtreamCluster) -> ClusterFlags { - match cluster { - XtreamCluster::Live => ClusterFlags::Live, - XtreamCluster::Video => ClusterFlags::Vod, - XtreamCluster::Series => ClusterFlags::Series, - } -} - -// Checks if the info has changed after the last update -pub fn needs_update_info_details(new_stream_props: &StreamProperties, old_stream_props: &StreamProperties) -> bool { - let new_modified = new_stream_props.get_last_modified(); - let old_modified = old_stream_props.get_last_modified(); - - match (new_modified, old_modified) { - (Some(new_ts), Some(old_ts)) => new_ts > old_ts, - (Some(_), None) => true, - _ => false, - } -} - -/// Merges persisted fields from old stream properties into freshly fetched properties. -/// -/// This keeps long-lived metadata stable across full playlist rewrites: -/// - VOD/Series `details` are preserved when incoming provider metadata is not newer. -/// - Learned Live fields are merged through [`LiveStreamProperties::merge_learned_metadata_from`]. -/// - Live catchup remains separate provider metadata and is copied only when missing. -pub fn merge_preserved_stream_properties( - new_stream_props: &mut StreamProperties, - old_stream_props: &StreamProperties, -) -> bool { - let preserve_info_details = - old_stream_props.has_details() && !needs_update_info_details(new_stream_props, old_stream_props); - - match (new_stream_props, old_stream_props) { - (StreamProperties::Video(v_new), StreamProperties::Video(v_old)) => { - let mut changed = false; - - if preserve_info_details && v_old.details.is_some() && v_new.details != v_old.details { - v_new.details.clone_from(&v_old.details); - changed = true; - } - - if v_new.tmdb.is_none() && v_old.tmdb.is_some() { - v_new.tmdb = v_old.tmdb; - changed = true; - } - - changed - } - (StreamProperties::Series(s_new), StreamProperties::Series(s_old)) => { - let mut changed = false; - - if preserve_info_details && s_old.details.is_some() && s_new.details != s_old.details { - s_new.details.clone_from(&s_old.details); - changed = true; - } - - if s_new.tmdb.is_none() && s_old.tmdb.is_some() { - s_new.tmdb = s_old.tmdb; - changed = true; - } - - if s_new.release_date.is_none() && s_old.release_date.is_some() { - s_new.release_date.clone_from(&s_old.release_date); - changed = true; - } - - changed - } - (StreamProperties::Live(l_new), StreamProperties::Live(l_old)) => { - let mut changed = l_new.merge_learned_metadata_from(l_old); - - if l_new.catchup.is_none() && l_old.catchup.is_some() { - l_new.catchup.clone_from(&l_old.catchup); - changed = true; - } - - changed - } - _ => false, - } -} - -async fn persist_input_info( - app_config: &Arc, - storage_path: &Path, - cluster: XtreamCluster, - input_name: &str, - provider_id: u32, - props: StreamProperties, -) -> Result<(), Error> { - let xtream_path = xtream_get_file_path(storage_path, cluster); - if xtream_path.exists() { - let file_lock = app_config.file_locks.write_lock(&xtream_path).await; - let xtream_path_clone = xtream_path.clone(); - let input_name_owned = input_name.to_string(); - tokio::task::spawn_blocking(move || -> Result<(), Error> { - let _guard = file_lock; - let mut tree: BPlusTreeUpdate = - BPlusTreeUpdate::try_new_with_backoff(&xtream_path_clone).map_err(|err| { - Error::other(format!("failed to open BPlusTree for input {input_name_owned}: {err}")) - })?; - match tree.query(&provider_id) { - Ok(Some(mut pli)) => { - pli.additional_properties = Some(props); - tree.update(&provider_id, pli).map_err(|err| { - Error::other(format!("failed to write {cluster} info for input {input_name_owned}: {err}")) - })?; - //rebuild_source_ordinal_index_if_present(&xtream_path_clone) - // .map_err(|err| Error::other(format!("failed to rebuild sorted index for input {input_name_owned}: {err}")))?; - } - Ok(None) => { - error!("Could not find input entry for provider_id: {provider_id} and input: {input_name_owned}"); - } - Err(err) => { - error!( - "Failed to query BPlusTree for provider_id: {provider_id} and input: {input_name_owned}: {err}" - ); - } - } - Ok(()) - }) - .await - .map_err(|err| Error::other(format!("failed to join blocking input info persist for {input_name}: {err}")))??; - } - Ok(()) -} - -pub async fn persist_input_info_batch( - app_config: &Arc, - storage_path: &Path, - cluster: XtreamCluster, - input_name: &str, - updates: Vec<(u32, StreamProperties)>, -) -> Result<(), Error> { - if updates.is_empty() { - return Ok(()); - } - let xtream_path = xtream_get_file_path(storage_path, cluster); - if xtream_path.exists() { - let file_lock = app_config.file_locks.write_lock(&xtream_path).await; - let xtream_path_clone = xtream_path.clone(); - let input_name_owned = input_name.to_string(); - tokio::task::spawn_blocking(move || -> Result<(), Error> { - let _guard = file_lock; - let mut tree: BPlusTreeUpdate = BPlusTreeUpdate::try_new_with_backoff(&xtream_path_clone) - .map_err(|err| Error::other(format!("failed to open BPlusTree for input {input_name_owned}: {err}")))?; - - // Keep only the latest update per provider id to avoid duplicate reads/writes. - let mut deduped_updates: HashMap = HashMap::with_capacity(updates.len()); - for (provider_id, props) in updates { - deduped_updates.insert(provider_id, props); - } - - let mut updated_plis = Vec::with_capacity(deduped_updates.len()); - for (provider_id, props) in deduped_updates { - match tree.query(&provider_id) { - Ok(Some(mut pli)) => { - pli.additional_properties = Some(props); - updated_plis.push((provider_id, pli)); - } - Ok(None) => { - error!("Could not find input entry for provider_id: {provider_id} and input: {input_name_owned}"); - } - Err(err) => { - error!("Failed to query BPlusTree for provider_id: {provider_id} and input: {input_name_owned}: {err}"); - } - } - } - - if !updated_plis.is_empty() { - let refs: Vec<(&u32, &XtreamPlaylistItem)> = updated_plis.iter() - .map(|(id, pli)| (id, pli)) - .collect(); - tree.update_batch(&refs).map_err(|err| Error::other(format!("failed to write batch {cluster} info for input {input_name_owned}: {err}")))?; - //rebuild_source_ordinal_index_if_present(&xtream_path_clone) - // .map_err(|err| Error::other(format!("failed to rebuild sorted index for input {input_name_owned}: {err}")))?; - } - Ok(()) - }).await.map_err(|err| Error::other(format!("failed to join blocking input info batch persist for {input_name}: {err}")))??; - } - Ok(()) -} - -pub async fn persist_input_vod_info( - app_config: &Arc, - storage_path: &Path, - cluster: XtreamCluster, - input_name: &str, - provider_id: u32, - props: &VideoStreamProperties, -) -> Result<(), Error> { - persist_input_info( - app_config, - storage_path, - cluster, - input_name, - provider_id, - StreamProperties::Video(Box::new(props.clone())), - ) - .await -} - -pub async fn persist_input_live_info( - app_config: &Arc, - storage_path: &Path, - cluster: XtreamCluster, - input_name: &str, - provider_id: u32, - props: &LiveStreamProperties, -) -> Result<(), Error> { - persist_input_info( - app_config, - storage_path, - cluster, - input_name, - provider_id, - StreamProperties::Live(Box::new(props.clone())), - ) - .await -} - -pub async fn persist_input_live_info_batch( - app_config: &Arc, - storage_path: &Path, - cluster: XtreamCluster, - input_name: &str, - updates: Vec<(u32, LiveStreamProperties)>, -) -> Result<(), Error> { - let batch = updates.into_iter().map(|(id, props)| (id, StreamProperties::Live(Box::new(props)))).collect(); - persist_input_info_batch(app_config, storage_path, cluster, input_name, batch).await -} - -pub async fn persist_input_vod_info_batch( - app_config: &Arc, - storage_path: &Path, - cluster: XtreamCluster, - input_name: &str, - updates: Vec<(u32, VideoStreamProperties)>, -) -> Result<(), Error> { - let batch = updates.into_iter().map(|(id, props)| (id, StreamProperties::Video(Box::new(props)))).collect(); - persist_input_info_batch(app_config, storage_path, cluster, input_name, batch).await -} - -pub async fn persists_input_series_info( - app_config: &Arc, - storage_path: &Path, - cluster: XtreamCluster, - input_name: &str, - provider_id: u32, - props: &SeriesStreamProperties, -) -> Result<(), Error> { - persist_input_info( - app_config, - storage_path, - cluster, - input_name, - provider_id, - StreamProperties::Series(Box::new(props.clone())), - ) - .await -} - -pub async fn persist_input_series_info_batch( - app_config: &Arc, - storage_path: &Path, - cluster: XtreamCluster, - input_name: &str, - updates: Vec<(u32, SeriesStreamProperties)>, -) -> Result<(), Error> { - let batch = updates.into_iter().map(|(id, props)| (id, StreamProperties::Series(Box::new(props)))).collect(); - persist_input_info_batch(app_config, storage_path, cluster, input_name, batch).await -} - -pub async fn load_input_xtream_playlist( - app_config: &Arc, - storage_path: &Path, - clusters: &[XtreamCluster], -) -> Result, TuliproxError> { - let mut groups: IndexMap<(XtreamCluster, u32), PlaylistGroup> = IndexMap::new(); - - for &cluster in clusters { - let xtream_path = xtream_get_file_path(storage_path, cluster); - if xtream_path.exists() { - let cat_col_name = xtream_cluster_category_collection(cluster); - let cat_path = get_collection_path(storage_path, cat_col_name); - - if cat_path.exists() { - if let Ok(content) = tokio::fs::read_to_string(&cat_path).await { - if let Ok(cats) = serde_json::from_str::>(&content) { - for cat in cats { - groups.insert( - (cluster, cat.category_id), - PlaylistGroup { - id: cat.category_id, - title: cat.category_name, - channels: Vec::new(), - xtream_cluster: cluster, - }, - ); - } - } - } - } - - // Load Items - let file_lock = app_config.file_locks.read_lock(&xtream_path).await; - let xtream_path_err = xtream_path.clone(); - let items = tokio::task::spawn_blocking(move || -> Result, TuliproxError> { - let _guard = file_lock; - let mut items = Vec::new(); - let mut query = BPlusTreeQuery::::try_new(&xtream_path) - .map_err(|error| TuliproxError::RepositoryXtream(error.to_string()))?; - for entry in query.iter() { - let (_, item) = entry.map_err(|error| TuliproxError::RepositoryXtream(error.to_string()))?; - items.push(item); - } - Ok(items) - }) - .await - .map_err(|err| cant_read_result!(RepositoryXtream, "xtream", &xtream_path_err, err))??; - - for item in items { - let cat_id = item.category_id; - groups - .entry((cluster, cat_id)) - .or_insert_with(|| PlaylistGroup { - id: cat_id, - title: "Unknown".intern(), - channels: Vec::new(), - xtream_cluster: cluster, - }) - .channels - .push(PlaylistItem::from(&item)); - } - } - } - - Ok(groups.into_values().collect()) -} - +use self::publish::sync_published_file_parent; +#[cfg(unix)] +use self::publish::sync_published_file_parent; +#[cfg(windows)] +use self::publish::sync_published_file_parent; +#[cfg(all(not(unix), not(windows)))] +use self::target_publish::move_target_file_platform; +#[cfg(windows)] +use self::target_publish::move_target_file_platform; #[cfg(test)] -mod tests { - #[cfg(unix)] - use super::refresh_staging_path; - use super::{ - count_input_xtream_cluster, get_collection_path, load_input_xtream_playlist, merge_preserved_stream_properties, - needs_update_info_details, persist_input_xtream_playlist, persist_input_xtream_playlist_cluster_to_disk, - persist_input_xtream_playlist_clusters_to_disk, persist_input_xtream_playlist_clusters_to_disk_with_operations, - persists_input_series_info, preserve_details_input_xtream_playlist_cluster_to_disk, - preserve_details_with_injected_operation_failure, publish_staged_file_same_directory, - target_category_lock_path, xtream_cluster_category_collection, xtream_get_playlist_categories, - xtream_write_playlist, xtream_write_playlist_with_injected_empty_replacement_failure, - xtream_write_playlist_with_mode, DetailPreservationOperation, PreserveDetailsOutcome, - TargetEmptyPublicationHook, TargetEmptyReplacementFailure, TargetEmptyReplacementMode, - XtreamClusterPublishOutcome, XtreamClusterQualityPolicy, XtreamClusterRefreshRequest, - XtreamClusterStageOperations, XtreamRefreshLease, XtreamRefreshPaths, - }; - use crate::{ - bplustree::{ensure_distinct_sidecar_lock_domains, sidecar_lock_path}, - build_input_storage_path, cleanup_orphaned_staging_artifacts, get_file_path_for_db_index, - get_input_storage_path, refresh_generation_guard_path, BPlusTreeQuery, BPlusTreeUpdate, - }; - use arc_swap::{ArcSwap, ArcSwapOption}; - use shared::{ - error::TuliproxError, - model::{ - CatchupProperties, ClusterFlags, ConfigPaths, InputType, LiveStreamProperties, PlaylistGroup, PlaylistItem, - PlaylistItemHeader, PlaylistItemType, ProcessingOrder, SeriesStreamDetailEpisodeProperties, - SeriesStreamDetailProperties, SeriesStreamProperties, StreamProperties, VideoStreamProperties, VirtualId, - XtreamCluster, XtreamPlaylistItem, - }, - utils::Internable, - }; - use std::{ - env, fs, io, - path::Path, - process::{Child, Command, ExitStatus, Stdio}, - sync::Arc, - thread, - time::{Duration, Instant}, - }; - use tempfile::tempdir; - use tokio::io::AsyncWriteExt; - use tuliprox_core::{ - model::{ - ApiProxyConfig, AppConfig, ClusterForceUpdate, ClusterUpdateAcceptance, ClusterUpdateRejection, Config, - ConfigInput, ConfigTarget, CustomStreamResponse, HdHomeRunConfig, MediaToolCapabilities, SourcesConfig, - StagedFilter, TargetExecutionPlan, TargetOutput, UpdateQualityDecision, XtreamTargetFlagsSet, - XtreamTargetOutput, - }, - utils::{request::DynReader, FileLockManager}, - }; - use uuid::Uuid; - - fn test_app_config(storage_dir: &Path) -> Arc { - Arc::new(AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config { - storage_dir: storage_dir.to_string_lossy().into_owned(), - ..Config::default() - })), - sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), - hdhomerun: Arc::new(ArcSwapOption::::default()), - api_proxy: Arc::new(ArcSwapOption::::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - }) - } - - fn target_writer_config() -> ConfigTarget { - ConfigTarget { - curation: None, - id: 1, - enabled: true, - name: "target-empty-cluster-guard".to_string(), - options: None, - sort: None, - filter: StagedFilter::default(), - output: vec![TargetOutput::Xtream(XtreamTargetOutput { - flags: XtreamTargetFlagsSet::new(), - trakt: None, - filter: None, - })], - rename: None, - mapping_ids: None, - mapping: Arc::new(ArcSwapOption::new(None)), - favourites: None, - processing_order: ProcessingOrder::default(), - execution_plan: TargetExecutionPlan::default(), - watch: None, - use_memory_cache: false, - } - } - - fn target_writer_group(cluster: XtreamCluster, category_id: u32, virtual_id: u32) -> PlaylistGroup { - let title = format!("target-{cluster}").intern(); - PlaylistGroup { - id: category_id, - title: Arc::clone(&title), - channels: vec![PlaylistItem { - header: PlaylistItemHeader { - id: virtual_id.to_string().intern(), - input_stream_id: virtual_id.to_string().intern(), - virtual_id: VirtualId::new(virtual_id), - name: Arc::clone(&title), - title: Arc::clone(&title), - group: title, - input_name: "target-writer-input".intern(), - item_type: PlaylistItemType::from(cluster), - xtream_cluster: cluster, - category_id, - ..PlaylistItemHeader::default() - }, - }], - xtream_cluster: cluster, - } - } - - #[tokio::test] - async fn target_writer_creates_missing_empty_category_files_without_force() -> Result<(), Box> - { - let directory = tempfile::tempdir()?; - let app_config = test_app_config(directory.path()); - let target = target_writer_config(); - let mut live_only = vec![target_writer_group(XtreamCluster::Live, 1, 101)]; - - xtream_write_playlist(&app_config, &target, &mut live_only, ClusterFlags::empty()).await?; - - let storage_path = { - let config = app_config.config.load(); - super::xtream_get_storage_path(&config, &target.name).expect("target Xtream storage") - }; - for cluster in [XtreamCluster::Video, XtreamCluster::Series] { - let categories = get_collection_path(&storage_path, xtream_cluster_category_collection(cluster)); - assert_eq!(tokio::fs::read(categories).await?, b"[]"); - assert!(!super::xtream_get_file_path(&storage_path, cluster).exists()); - } - Ok(()) - } - - #[tokio::test] - async fn target_writer_preserves_an_unauthorized_empty_cluster() -> Result<(), Box> { - let directory = tempfile::tempdir()?; - let app_config = test_app_config(directory.path()); - let target = target_writer_config(); - let mut baseline = vec![ - target_writer_group(XtreamCluster::Live, 1, 101), - target_writer_group(XtreamCluster::Video, 2, 201), - target_writer_group(XtreamCluster::Series, 3, 301), - ]; - xtream_write_playlist(&app_config, &target, &mut baseline, ClusterFlags::empty()).await?; - - let storage_path = { - let config = app_config.config.load(); - super::xtream_get_storage_path(&config, &target.name).expect("target Xtream storage") - }; - let retained_paths = [XtreamCluster::Video, XtreamCluster::Series].map(|cluster| { - ( - super::xtream_get_file_path(&storage_path, cluster), - get_collection_path(&storage_path, xtream_cluster_category_collection(cluster)), - ) - }); - let mut retained_before = Vec::new(); - for (database, categories) in &retained_paths { - retained_before.push((tokio::fs::read(database).await?, tokio::fs::read(categories).await?)); - } - - let mut live_only = vec![target_writer_group(XtreamCluster::Live, 1, 102)]; - xtream_write_playlist(&app_config, &target, &mut live_only, ClusterFlags::empty()).await?; - - for ((database, categories), (database_before, categories_before)) in retained_paths.iter().zip(retained_before) - { - assert_eq!(tokio::fs::read(database).await?, database_before); - assert_eq!(tokio::fs::read(categories).await?, categories_before); - } - Ok(()) - } - - #[tokio::test] - async fn target_force_empty_failures_restore_database_index_and_categories( - ) -> Result<(), Box> { - for failure in [ - TargetEmptyReplacementFailure::CategoryPersistence, - TargetEmptyReplacementFailure::BTreePersistence, - TargetEmptyReplacementFailure::Publication, - ] { - let directory = tempfile::tempdir()?; - let app_config = test_app_config(directory.path()); - let target = target_writer_config(); - let mut baseline = vec![ - target_writer_group(XtreamCluster::Live, 1, 101), - target_writer_group(XtreamCluster::Video, 2, 201), - target_writer_group(XtreamCluster::Series, 3, 301), - ]; - xtream_write_playlist(&app_config, &target, &mut baseline, ClusterFlags::empty()).await?; - let storage_path = { - let config = app_config.config.load(); - super::xtream_get_storage_path(&config, &target.name).expect("target Xtream storage") - }; - let database = super::xtream_get_file_path(&storage_path, XtreamCluster::Video); - let index = get_file_path_for_db_index(&database); - let categories = super::get_vod_cat_collection_path(&storage_path); - let before = [ - tokio::fs::read(&database).await?, - tokio::fs::read(&index).await?, - tokio::fs::read(&categories).await?, - ]; - let mut candidate = vec![ - target_writer_group(XtreamCluster::Live, 1, 102), - target_writer_group(XtreamCluster::Series, 3, 302), - ]; - - let error = xtream_write_playlist_with_injected_empty_replacement_failure( - &app_config, - &target, - &mut candidate, - ClusterFlags::Vod, - failure, - ) - .await - .expect_err("injected empty replacement must fail"); - - assert!(error.to_string().contains("target cluster failed")); - assert_eq!(tokio::fs::read(&database).await?, before[0]); - assert_eq!(tokio::fs::read(&index).await?, before[1]); - assert_eq!(tokio::fs::read(&categories).await?, before[2]); - let leaked = fs::read_dir(&storage_path)? - .filter_map(Result::ok) - .map(|entry| entry.file_name().to_string_lossy().into_owned()) - .filter(|name| name.contains("force-empty")) - .collect::>(); - assert!(leaked.is_empty(), "staging or backup files leaked after {failure:?}: {leaked:?}"); - } - Ok(()) - } - - #[tokio::test] - async fn target_force_empty_category_reader_waits_through_the_backup_window( - ) -> Result<(), Box> { - let directory = tempfile::tempdir()?; - let app_config = test_app_config(directory.path()); - let target = target_writer_config(); - let mut baseline = vec![ - target_writer_group(XtreamCluster::Live, 1, 101), - target_writer_group(XtreamCluster::Video, 2, 201), - target_writer_group(XtreamCluster::Series, 3, 301), - ]; - xtream_write_playlist(&app_config, &target, &mut baseline, ClusterFlags::empty()).await?; - let storage_path = { - let config = app_config.config.load(); - super::xtream_get_storage_path(&config, &target.name).expect("target Xtream storage") - }; - let category_path = super::get_vod_cat_collection_path(&storage_path); - let hook = TargetEmptyPublicationHook::new(); - let entered = Arc::clone(&hook.backup_window_entered); - let resume = Arc::clone(&hook.resume_publication); - let writer_app_config = Arc::clone(&app_config); - let writer_target = target.clone(); - let writer = tokio::spawn(async move { - let mut candidate = vec![ - target_writer_group(XtreamCluster::Live, 1, 102), - target_writer_group(XtreamCluster::Series, 3, 302), - ]; - xtream_write_playlist_with_mode( - &writer_app_config, - &writer_target, - &mut candidate, - ClusterFlags::Vod, - TargetEmptyReplacementMode::PauseDuringPublication(hook), - ) - .await - }); - - tokio::task::spawn_blocking(move || entered.wait()).await?; - let category_was_temporarily_backed_up = !category_path.exists(); - let category_lock_path = target_category_lock_path(&category_path); - let category_write_lock_was_held = app_config.file_locks.try_write_lock(&category_lock_path).await.is_err(); - - let category_read = xtream_get_playlist_categories(&app_config, &target.name, XtreamCluster::Video); - tokio::pin!(category_read); - let category_read_poll = futures::poll!(category_read.as_mut()); - let category_reader_state = match &category_read_poll { - std::task::Poll::Pending => "waiting", - std::task::Poll::Ready(None) => "missing", - std::task::Poll::Ready(Some(categories)) if categories.is_empty() => "new", - std::task::Poll::Ready(Some(_)) => "old-or-partial", - }; - - tokio::task::spawn_blocking(move || resume.wait()).await?; - writer.await??; - assert!(category_was_temporarily_backed_up, "failure hook must expose the internal backup window"); - assert!(category_write_lock_was_held, "writer must hold the target category lock during publication"); - assert_eq!(category_reader_state, "waiting", "the category reader must wait on the writer's category lock"); - let categories = category_read.await.expect("published VOD category catalog"); - assert!(categories.is_empty(), "reader must observe the complete force-empty category catalog"); - Ok(()) - } - - fn json_reader(content: &str) -> DynReader { - let (mut writer, reader) = tokio::io::duplex(4096); - let content = content.as_bytes().to_vec(); - tokio::spawn(async move { - writer.write_all(&content).await.expect("fixture should fit into duplex reader"); - writer.shutdown().await.expect("fixture writer should shut down"); - }); - Box::pin(reader) - } - - #[tokio::test] - async fn empty_disk_refresh_retains_published_playlist_and_catalog() -> Result<(), Box> { - let directory = tempfile::tempdir()?; - let app_config = test_app_config(directory.path()); - let input = ConfigInput { name: "empty-guard".intern(), input_type: InputType::Xtream, ..Default::default() }; - - persist_input_xtream_playlist_cluster_to_disk( - &app_config, - &input, - XtreamCluster::Live, - 0, - json_reader(r#"[{"category_id":"1","category_name":"News"}]"#), - json_reader(r#"[{"name":"Channel","stream_id":7,"category_id":"1","added":"0"}]"#), - ) - .await?; - - let storage = build_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()); - let database = super::xtream_get_file_path(&storage, XtreamCluster::Live); - let catalog = crate::raw_group_catalog_path(&storage, XtreamCluster::Live); - let database_before = tokio::fs::read(&database).await?; - let catalog_before = tokio::fs::read(&catalog).await?; - - let result = persist_input_xtream_playlist_cluster_to_disk( - &app_config, - &input, - XtreamCluster::Live, - 0, - json_reader("[]"), - json_reader("[]"), - ) - .await; - - assert!(result.is_err()); - assert_eq!(tokio::fs::read(database).await?, database_before); - assert_eq!(tokio::fs::read(catalog).await?, catalog_before); - Ok(()) - } - - #[tokio::test] - async fn empty_cluster_is_published_when_another_cluster_still_has_items() -> Result<(), Box> - { - let directory = tempfile::tempdir()?; - let app_config = test_app_config(directory.path()); - let input = ConfigInput { name: "empty-cluster".intern(), input_type: InputType::Xtream, ..Default::default() }; - - for (cluster, category, stream) in [ - ( - XtreamCluster::Live, - r#"[{"category_id":"1","category_name":"News"}]"#, - r#"[{"name":"Channel","stream_id":7,"category_id":"1","added":"0"}]"#, - ), - ( - XtreamCluster::Video, - r#"[{"category_id":"2","category_name":"Movies"}]"#, - r#"[{"name":"Movie","stream_id":8,"category_id":"2","added":"0"}]"#, - ), - ] { - persist_input_xtream_playlist_cluster_to_disk( - &app_config, - &input, - cluster, - 0, - json_reader(category), - json_reader(stream), - ) - .await?; - } - - persist_input_xtream_playlist_cluster_to_disk( - &app_config, - &input, - XtreamCluster::Live, - 0, - json_reader("[]"), - json_reader("[]"), - ) - .await?; - - let storage = build_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()); - let playlist = - super::load_input_xtream_playlist(&app_config, &storage, &[XtreamCluster::Live, XtreamCluster::Video]) - .await?; - assert!(playlist.iter().all(|group| group.xtream_cluster != XtreamCluster::Live)); - assert!(playlist.iter().any(|group| group.xtream_cluster == XtreamCluster::Video)); - Ok(()) - } - - fn disk_test_input(name: &str) -> ConfigInput { - ConfigInput { - name: name.intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("user".to_string()), - password: Some("password".to_string()), - ..ConfigInput::default() - } - } - - fn cluster_fixture_readers( - cluster: XtreamCluster, - category_name: &str, - count: usize, - first_provider_id: u32, - ) -> (DynReader, DynReader) { - let category_id = match cluster { - XtreamCluster::Live => 1_u32, - XtreamCluster::Video => 2, - XtreamCluster::Series => 3, - }; - let categories = serde_json::json!([{ - "category_id": category_id, - "category_name": category_name, - }]) - .to_string(); - let streams = (0..count) - .map(|offset| { - let provider_id = - first_provider_id + u32::try_from(offset).expect("fixture item count should fit into u32"); - let name = format!("{category_name}-{provider_id}"); - match cluster { - XtreamCluster::Live => serde_json::json!({ - "name": name, - "stream_id": provider_id, - "category_id": category_id, - "added": "0", - }), - XtreamCluster::Video => serde_json::json!({ - "name": name, - "stream_id": provider_id, - "category_id": category_id, - "added": "0", - "container_extension": "mp4", - }), - XtreamCluster::Series => serde_json::json!({ - "name": name, - "series_id": provider_id, - "category_id": category_id, - "last_modified": "0", - }), - } - }) - .collect::>(); - let streams = serde_json::Value::Array(streams).to_string(); - (json_reader(&categories), json_reader(&streams)) - } - - fn live_fixture_stream(provider_id: u32, category_id: u32, name: &str) -> serde_json::Value { - serde_json::json!({ - "name": name, - "stream_id": provider_id, - "category_id": category_id, - "added": "0", - }) - } - - async fn publish_live_fixture_rows( - app_config: &Arc, - input: &ConfigInput, - threshold: u8, - categories: serde_json::Value, - streams: Vec, - ) -> XtreamClusterPublishOutcome { - persist_input_xtream_playlist_cluster_to_disk( - app_config, - input, - XtreamCluster::Live, - threshold, - json_reader(&categories.to_string()), - json_reader(&serde_json::Value::Array(streams).to_string()), - ) - .await - .expect("Live fixture refresh should complete") - } - - async fn publish_test_cluster( - app_config: &Arc, - input: &ConfigInput, - cluster: XtreamCluster, - threshold: u8, - category_name: &str, - count: usize, - first_provider_id: u32, - ) -> XtreamClusterPublishOutcome { - let (categories, streams) = cluster_fixture_readers(cluster, category_name, count, first_provider_id); - persist_input_xtream_playlist_cluster_to_disk(app_config, input, cluster, threshold, categories, streams) - .await - .expect("test cluster refresh should complete") - } - - fn active_cluster_count(storage_path: &Path, cluster: XtreamCluster) -> usize { - super::count_xtream_tree_entries(&super::xtream_get_file_path(storage_path, cluster)) - .expect("active cluster should be countable") - .expect("active cluster should exist") - } - - fn active_category_bytes(storage_path: &Path, cluster: XtreamCluster) -> Vec { - fs::read(get_collection_path(storage_path, xtream_cluster_category_collection(cluster))) - .expect("active categories should be readable") - } - - #[derive(Debug, Eq, PartialEq)] - struct ActiveClusterSnapshot { - database: Vec, - categories: Vec, - } - - fn active_cluster_snapshot(storage_path: &Path, cluster: XtreamCluster) -> ActiveClusterSnapshot { - let database_path = super::xtream_get_file_path(storage_path, cluster); - ActiveClusterSnapshot { - database: fs::read(&database_path).expect("active database should be readable"), - categories: active_category_bytes(storage_path, cluster), - } - } - - fn fail_detail_preservation_after_quality( - published_path: &Path, - staging_path: &Path, - ) -> Result { - preserve_details_with_injected_operation_failure( - published_path, - staging_path, - DetailPreservationOperation::Commit, - ) - } - - fn read_series_props(path: &Path, provider_id: u32) -> SeriesStreamProperties { - let mut query = BPlusTreeQuery::::try_new(path).expect("query open should succeed"); - let item = query.query_zero_copy(&provider_id).expect("query should succeed").expect("item should exist"); - match item.additional_properties { - Some(StreamProperties::Series(series)) => *series, - other => panic!("expected series stream properties, got {other:?}"), - } - } - - fn assert_no_refresh_artifacts(storage_path: &Path) { - let entries = fs::read_dir(storage_path) - .expect("input storage should be readable") - .collect::>>() - .expect("input storage entries should be readable"); - let refresh_artifacts = entries - .into_iter() - .map(|entry| entry.file_name().to_string_lossy().into_owned()) - .filter(|name| name.contains("refresh-")) - .collect::>(); - assert!(refresh_artifacts.is_empty(), "staging artifacts survived: {refresh_artifacts:?}"); - } - - #[test] - fn disk_quality_disabled_does_not_read_staging_or_baseline() { - let directory = tempdir().expect("temp directory"); - let paths = fixed_refresh_paths(directory.path(), 18); - - let decision = super::evaluate_staged_xtream_cluster_quality(&paths, XtreamCluster::Live, 0) - .expect("disabled quality evaluation should not access missing files"); - - assert_eq!(decision, UpdateQualityDecision::Disabled); - } - - #[tokio::test] - async fn disk_force_publishes_an_empty_cluster_and_cleans_staging() { - let directory = tempdir().expect("temporary directory"); - let app_config = test_app_config(directory.path()); - let input = disk_test_input("force-empty"); - assert_eq!( - publish_test_cluster(&app_config, &input, XtreamCluster::Video, 0, "old-vod", 3, 2_000).await, - XtreamClusterPublishOutcome::Published - ); - let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) - .await - .expect("force storage"); - let before = active_cluster_snapshot(&storage_path, XtreamCluster::Video); - - let result = persist_input_xtream_playlist_clusters_to_disk( - &app_config, - &input, - vec![XtreamClusterRefreshRequest { - cluster: XtreamCluster::Video, - quality: XtreamClusterQualityPolicy::Bypass { configured_threshold: 95 }, - categories: json_reader("[]"), - streams: json_reader("[]"), - }], - ) - .await; - - assert!(result.errors.is_empty()); - assert_eq!( - result.outcomes, - vec![XtreamClusterPublishOutcome::ForcePublished(ClusterForceUpdate { - cluster: XtreamCluster::Video, - candidate_count: 0, - configured_threshold: 95, - })] - ); - assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Video), 0); - assert_ne!(active_cluster_snapshot(&storage_path, XtreamCluster::Video), before); - assert_eq!( - serde_json::from_slice::(&active_category_bytes(&storage_path, XtreamCluster::Video,)) - .expect("empty category JSON"), - serde_json::json!([]) - ); - assert_no_refresh_artifacts(&storage_path); - } - - #[tokio::test] - async fn disk_force_keeps_active_files_after_a_technical_staging_failure() { - let directory = tempdir().expect("temporary directory"); - let app_config = test_app_config(directory.path()); - let input = disk_test_input("force-technical-error"); - assert_eq!( - publish_test_cluster(&app_config, &input, XtreamCluster::Live, 0, "old-live", 3, 1_000).await, - XtreamClusterPublishOutcome::Published - ); - let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) - .await - .expect("force storage"); - let before = active_cluster_snapshot(&storage_path, XtreamCluster::Live); - - let result = persist_input_xtream_playlist_clusters_to_disk( - &app_config, - &input, - vec![XtreamClusterRefreshRequest { - cluster: XtreamCluster::Live, - quality: XtreamClusterQualityPolicy::Bypass { configured_threshold: 100 }, - categories: json_reader(r#"[{"category_id":"1","category_name":"new-live"}]"#), - streams: json_reader("{"), - }], - ) - .await; - - assert_eq!(result.errors.len(), 1); - assert!(result.outcomes.is_empty()); - assert!(result.force_updates.is_empty()); - assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Live), before); - assert_no_refresh_artifacts(&storage_path); - } - - #[tokio::test] - async fn pipeline_transparency_disk_quality_survives_post_evaluation_staging_failure() { - let directory = tempdir().expect("temporary directory"); - let app_config = test_app_config(directory.path()); - let input = disk_test_input("accepted-then-detail-error"); - assert_eq!( - publish_test_cluster(&app_config, &input, XtreamCluster::Live, 0, "old-live", 100, 1_000).await, - XtreamClusterPublishOutcome::Published - ); - let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) - .await - .expect("storage path"); - let before = active_cluster_snapshot(&storage_path, XtreamCluster::Live); - let (categories, streams) = cluster_fixture_readers(XtreamCluster::Live, "new-live", 95, 10_000); - - let result = persist_input_xtream_playlist_clusters_to_disk_with_operations( - &app_config, - &input, - vec![XtreamClusterRefreshRequest { - cluster: XtreamCluster::Live, - quality: XtreamClusterQualityPolicy::Enforce { threshold: 90 }, - categories, - streams, - }], - XtreamClusterStageOperations { preserve_details: fail_detail_preservation_after_quality }, - ) - .await; - - assert_eq!( - result.quality_acceptances, - vec![ClusterUpdateAcceptance { - cluster: XtreamCluster::Live, - current_count: Some(100), - candidate_count: 95, - threshold: 90, - quality: Some(95), - }] - ); - assert!(result.outcomes.is_empty()); - assert_eq!(result.errors.len(), 1); - assert_eq!(result.failed_cluster, Some(XtreamCluster::Live)); - assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Live), before); - assert_no_refresh_artifacts(&storage_path); - } - - #[tokio::test] - async fn pipeline_transparency_disk_batch_keeps_prior_publish_and_all_evaluated_quality_on_later_error() { - let directory = tempdir().expect("temporary directory"); - let app_config = test_app_config(directory.path()); - let input = disk_test_input("later-cluster-publish-error"); - assert_eq!( - publish_test_cluster(&app_config, &input, XtreamCluster::Live, 0, "old-live", 100, 1_000).await, - XtreamClusterPublishOutcome::Published - ); - assert_eq!( - publish_test_cluster(&app_config, &input, XtreamCluster::Video, 0, "old-vod", 100, 2_000).await, - XtreamClusterPublishOutcome::Published - ); - let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) - .await - .expect("storage path"); - let blocked_category_path = - get_collection_path(&storage_path, xtream_cluster_category_collection(XtreamCluster::Video)); - fs::remove_file(&blocked_category_path).expect("replace published VOD categories with a directory"); - fs::create_dir(&blocked_category_path).expect("create category publication blocker"); - let (live_categories, live_streams) = cluster_fixture_readers(XtreamCluster::Live, "new-live", 90, 10_000); - let (vod_categories, vod_streams) = cluster_fixture_readers(XtreamCluster::Video, "new-vod", 90, 20_000); - - let result = persist_input_xtream_playlist_clusters_to_disk( - &app_config, - &input, - vec![ - XtreamClusterRefreshRequest { - cluster: XtreamCluster::Live, - quality: XtreamClusterQualityPolicy::Enforce { threshold: 90 }, - categories: live_categories, - streams: live_streams, - }, - XtreamClusterRefreshRequest { - cluster: XtreamCluster::Video, - quality: XtreamClusterQualityPolicy::Enforce { threshold: 90 }, - categories: vod_categories, - streams: vod_streams, - }, - ], - ) - .await; - - assert_eq!(result.quality_acceptances.len(), 2); - assert_eq!(result.quality_acceptances[0].cluster, XtreamCluster::Live); - assert_eq!(result.quality_acceptances[1].cluster, XtreamCluster::Video); - assert_eq!(result.outcomes, vec![XtreamClusterPublishOutcome::QualityAccepted(result.quality_acceptances[0])]); - assert_eq!(result.errors.len(), 1); - assert_eq!(result.failed_cluster, Some(XtreamCluster::Video)); - assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Live), 90); - assert_no_refresh_artifacts(&storage_path); - } - - #[tokio::test] - async fn disk_quality_bootstrap_publishes_nonempty_candidate_and_rejects_empty_without_baseline() { - let populated_directory = tempdir().expect("populated bootstrap directory"); - let populated_config = test_app_config(populated_directory.path()); - let populated_input = disk_test_input("bootstrap-populated"); - - let populated_outcome = publish_test_cluster( - &populated_config, - &populated_input, - XtreamCluster::Live, - 90, - "bootstrap-live", - 3, - 1_000, - ) - .await; - let populated_storage = - get_input_storage_path(&populated_input.name, populated_directory.path().to_string_lossy().as_ref()) - .await - .expect("populated bootstrap storage"); - - assert_eq!( - populated_outcome, - XtreamClusterPublishOutcome::QualityAccepted(ClusterUpdateAcceptance { - cluster: XtreamCluster::Live, - current_count: None, - candidate_count: 3, - threshold: 90, - quality: None, - }) - ); - assert_eq!(active_cluster_count(&populated_storage, XtreamCluster::Live), 3); - assert!(String::from_utf8_lossy(&active_category_bytes(&populated_storage, XtreamCluster::Live)) - .contains("bootstrap-live")); - assert_no_refresh_artifacts(&populated_storage); - - let empty_directory = tempdir().expect("empty bootstrap directory"); - let empty_config = test_app_config(empty_directory.path()); - let empty_input = disk_test_input("bootstrap-empty"); - let empty_outcome = - publish_test_cluster(&empty_config, &empty_input, XtreamCluster::Video, 90, "empty-vod", 0, 2_000).await; - let empty_storage = - get_input_storage_path(&empty_input.name, empty_directory.path().to_string_lossy().as_ref()) - .await - .expect("empty bootstrap storage"); - - assert_eq!( - empty_outcome, - XtreamClusterPublishOutcome::RetainedPrevious(ClusterUpdateRejection { - cluster: XtreamCluster::Video, - current_count: 0, - candidate_count: 0, - threshold: 90, - quality: 0, - }) - ); - assert!(!super::xtream_get_file_path(&empty_storage, XtreamCluster::Video).exists()); - assert!(!get_collection_path(&empty_storage, xtream_cluster_category_collection(XtreamCluster::Video)).exists()); - assert_no_refresh_artifacts(&empty_storage); - } - - #[tokio::test] - async fn disk_quality_enforces_exact_90_percent_boundaries_and_retains_rejected_files() { - for (name, candidate_count, accepted) in [ - ("lower-boundary", 90, true), - ("below-lower-boundary", 89, false), - ("upper-boundary", 110, true), - ("above-upper-boundary", 111, false), - ] { - let directory = tempdir().expect("boundary directory"); - let app_config = test_app_config(directory.path()); - let input = disk_test_input(name); - assert_eq!( - publish_test_cluster(&app_config, &input, XtreamCluster::Live, 0, "old-live", 100, 10_000,).await, - XtreamClusterPublishOutcome::Published, - "baseline publish failed for {name}" - ); - let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) - .await - .expect("boundary storage"); - let before = active_cluster_snapshot(&storage_path, XtreamCluster::Live); - - let outcome = - publish_test_cluster(&app_config, &input, XtreamCluster::Live, 90, "new-live", candidate_count, 20_000) - .await; - - if accepted { - assert_eq!( - outcome, - XtreamClusterPublishOutcome::QualityAccepted(ClusterUpdateAcceptance { - cluster: XtreamCluster::Live, - current_count: Some(100), - candidate_count, - threshold: 90, - quality: Some(90), - }), - "case: {name}" - ); - assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Live), candidate_count); - assert!(String::from_utf8_lossy(&active_category_bytes(&storage_path, XtreamCluster::Live)) - .contains("new-live")); - } else { - assert_eq!( - outcome, - XtreamClusterPublishOutcome::RetainedPrevious(ClusterUpdateRejection { - cluster: XtreamCluster::Live, - current_count: 100, - candidate_count, - threshold: 90, - quality: 89, - }), - "case: {name}" - ); - assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Live), before, "case: {name}"); - } - assert_no_refresh_artifacts(&storage_path); - } - } - - #[tokio::test] - async fn disk_quality_rejects_duplicate_rows_that_represent_only_half_the_provider_ids() { - let directory = tempdir().expect("duplicate rejection directory"); - let app_config = test_app_config(directory.path()); - let input = disk_test_input("duplicate-rejection"); - let first_provider_id = 1_000_u32; - assert_eq!( - publish_test_cluster(&app_config, &input, XtreamCluster::Live, 0, "old-live", 100, first_provider_id,) - .await, - XtreamClusterPublishOutcome::Published - ); - let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) - .await - .expect("duplicate rejection storage"); - let before = active_cluster_snapshot(&storage_path, XtreamCluster::Live); - let streams = (0..50_u32) - .flat_map(|offset| { - let provider_id = first_provider_id + offset; - [ - live_fixture_stream(provider_id, 1, &format!("candidate-{provider_id}")), - live_fixture_stream(provider_id, 1, &format!("duplicate-{provider_id}")), - ] - }) - .collect(); - - let outcome = publish_live_fixture_rows( - &app_config, - &input, - 100, - serde_json::json!([{"category_id": 1, "category_name": "candidate-live"}]), - streams, - ) - .await; - - assert_eq!( - outcome, - XtreamClusterPublishOutcome::RetainedPrevious(ClusterUpdateRejection { - cluster: XtreamCluster::Live, - current_count: 100, - candidate_count: 50, - threshold: 100, - quality: 50, - }) - ); - assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Live), before); - assert_no_refresh_artifacts(&storage_path); - } - - #[tokio::test] - async fn pipeline_transparency_disk_quality_preserves_accepted_publish_facts() { - let directory = tempdir().expect("duplicate acceptance directory"); - let app_config = test_app_config(directory.path()); - let input = disk_test_input("duplicate-acceptance"); - let first_provider_id = 1_000_u32; - let winning_provider_id = first_provider_id + 42; - assert_eq!( - publish_test_cluster(&app_config, &input, XtreamCluster::Live, 0, "old-live", 100, first_provider_id,) - .await, - XtreamClusterPublishOutcome::Published - ); - let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) - .await - .expect("duplicate acceptance storage"); - let mut streams = (0..100_u32) - .map(|offset| { - let provider_id = first_provider_id + offset; - live_fixture_stream(provider_id, 1, &format!("candidate-{provider_id}")) - }) - .collect::>(); - streams.push(live_fixture_stream(winning_provider_id, 2, "winning-duplicate")); - - let outcome = publish_live_fixture_rows( - &app_config, - &input, - 100, - serde_json::json!([ - {"category_id": 1, "category_name": "candidate-live"}, - {"category_id": 2, "category_name": "winning-category"} - ]), - streams, - ) - .await; - - assert_eq!( - outcome, - XtreamClusterPublishOutcome::QualityAccepted(ClusterUpdateAcceptance { - cluster: XtreamCluster::Live, - current_count: Some(100), - candidate_count: 100, - threshold: 100, - quality: Some(100), - }) - ); - assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Live), 100); - let active_path = super::xtream_get_file_path(&storage_path, XtreamCluster::Live); - let mut query = BPlusTreeQuery::::try_new(&active_path) - .expect("accepted Live cluster should open"); - let winner = query - .query_zero_copy(&winning_provider_id) - .expect("winner lookup should succeed") - .expect("winner should exist"); - assert_eq!(winner.name.as_ref(), "winning-duplicate"); - assert_eq!(winner.category_id, 2); - assert!(String::from_utf8_lossy(&active_category_bytes(&storage_path, XtreamCluster::Live)) - .contains("winning-category")); - assert_no_refresh_artifacts(&storage_path); - } - - #[tokio::test] - async fn disk_series_quality_compares_catalog_rows_and_ignores_embedded_episode_details() { - let directory = tempdir().expect("series population directory"); - let app_config = test_app_config(directory.path()); - let input = disk_test_input("series-logical-population"); - let provider_id = 30_000; - assert_eq!( - publish_test_cluster(&app_config, &input, XtreamCluster::Series, 0, "old-series", 1, provider_id,).await, - XtreamClusterPublishOutcome::Published - ); - let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) - .await - .expect("series population storage"); - let episodes = (0..200_u32) - .map(|id| SeriesStreamDetailEpisodeProperties { id, ..SeriesStreamDetailEpisodeProperties::default() }) - .collect(); - let enriched = SeriesStreamProperties { - series_id: provider_id, - details: Some(SeriesStreamDetailProperties::new(None, Vec::new(), Some(episodes))), - ..SeriesStreamProperties::default() - }; - persists_input_series_info( - &app_config, - &storage_path, - XtreamCluster::Series, - &input.name, - provider_id, - &enriched, - ) - .await - .expect("series enrichment should persist"); - let active_path = super::xtream_get_file_path(&storage_path, XtreamCluster::Series); - assert_eq!( - read_series_props(&active_path, provider_id) - .details - .and_then(|details| details.episodes) - .map(|episodes| episodes.len()), - Some(200) - ); - - let equal_catalog = - publish_test_cluster(&app_config, &input, XtreamCluster::Series, 100, "equal-series", 1, provider_id).await; - - assert_eq!( - equal_catalog, - XtreamClusterPublishOutcome::QualityAccepted(ClusterUpdateAcceptance { - cluster: XtreamCluster::Series, - current_count: Some(1), - candidate_count: 1, - threshold: 100, - quality: Some(100), - }) - ); - assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Series), 1); - let accepted_snapshot = active_cluster_snapshot(&storage_path, XtreamCluster::Series); - - let different_catalog = - publish_test_cluster(&app_config, &input, XtreamCluster::Series, 100, "different-series", 2, 40_000).await; - - assert_eq!( - different_catalog, - XtreamClusterPublishOutcome::RetainedPrevious(ClusterUpdateRejection { - cluster: XtreamCluster::Series, - current_count: 1, - candidate_count: 2, - threshold: 100, - quality: 0, - }) - ); - assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Series), accepted_snapshot); - assert_no_refresh_artifacts(&storage_path); - } - - #[tokio::test] - async fn disk_quality_publishes_accepted_clusters_and_retains_rejected_cluster_independently() { - let directory = tempdir().expect("mixed cluster directory"); - let app_config = test_app_config(directory.path()); - let input = disk_test_input("mixed-clusters"); - - for (cluster, category_name, first_provider_id) in [ - (XtreamCluster::Live, "old-live", 10_000), - (XtreamCluster::Video, "old-vod", 20_000), - (XtreamCluster::Series, "old-series", 30_000), - ] { - assert_eq!( - publish_test_cluster(&app_config, &input, cluster, 0, category_name, 100, first_provider_id).await, - XtreamClusterPublishOutcome::Published - ); - } - let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) - .await - .expect("mixed cluster storage"); - let previous_vod = active_cluster_snapshot(&storage_path, XtreamCluster::Video); - - let live_outcome = - publish_test_cluster(&app_config, &input, XtreamCluster::Live, 90, "new-live", 90, 40_000).await; - let vod_outcome = - publish_test_cluster(&app_config, &input, XtreamCluster::Video, 90, "new-vod", 89, 50_000).await; - let series_outcome = - publish_test_cluster(&app_config, &input, XtreamCluster::Series, 90, "new-series", 110, 60_000).await; - - assert_eq!( - live_outcome, - XtreamClusterPublishOutcome::QualityAccepted(ClusterUpdateAcceptance { - cluster: XtreamCluster::Live, - current_count: Some(100), - candidate_count: 90, - threshold: 90, - quality: Some(90), - }) - ); - assert_eq!( - vod_outcome, - XtreamClusterPublishOutcome::RetainedPrevious(ClusterUpdateRejection { - cluster: XtreamCluster::Video, - current_count: 100, - candidate_count: 89, - threshold: 90, - quality: 89, - }) - ); - assert_eq!( - series_outcome, - XtreamClusterPublishOutcome::QualityAccepted(ClusterUpdateAcceptance { - cluster: XtreamCluster::Series, - current_count: Some(100), - candidate_count: 110, - threshold: 90, - quality: Some(90), - }) - ); - assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Live), 90); - assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Video), 100); - assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Series), 110); - assert!( - String::from_utf8_lossy(&active_category_bytes(&storage_path, XtreamCluster::Live)).contains("new-live") - ); - assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Video), previous_vod); - assert!(String::from_utf8_lossy(&active_category_bytes(&storage_path, XtreamCluster::Series)) - .contains("new-series")); - assert_no_refresh_artifacts(&storage_path); - } - - fn wait_for_child(mut child: Child, timeout: Duration) -> io::Result { - let deadline = Instant::now() + timeout; - loop { - match child.try_wait() { - Ok(Some(status)) => return Ok(status), - Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(10)), - Ok(None) => { - let _ = child.kill(); - let _ = child.wait(); - return Err(io::Error::new(io::ErrorKind::TimedOut, "Xtream refresh child timed out")); - } - Err(error) => { - let _ = child.kill(); - let _ = child.wait(); - return Err(error); - } - } - } - } - - #[test] - fn keeps_existing_details_when_new_timestamp_is_missing() { - let new_props = StreamProperties::Video(Box::new(VideoStreamProperties { - added: "".into(), - ..VideoStreamProperties::default() - })); - let old_props = StreamProperties::Video(Box::new(VideoStreamProperties { - added: "1700000000".into(), - ..VideoStreamProperties::default() - })); - - assert!(!needs_update_info_details(&new_props, &old_props)); - } - - #[test] - fn updates_details_when_new_timestamp_is_newer() { - let new_props = StreamProperties::Series(Box::new(SeriesStreamProperties { - last_modified: Some("200".into()), - ..SeriesStreamProperties::default() - })); - let old_props = StreamProperties::Series(Box::new(SeriesStreamProperties { - last_modified: Some("100".into()), - ..SeriesStreamProperties::default() - })); - - assert!(needs_update_info_details(&new_props, &old_props)); - } - - #[test] - fn does_not_update_details_when_new_timestamp_is_older() { - let new_props = StreamProperties::Series(Box::new(SeriesStreamProperties { - last_modified: Some("100".into()), - ..SeriesStreamProperties::default() - })); - let old_props = StreamProperties::Series(Box::new(SeriesStreamProperties { - last_modified: Some("200".into()), - ..SeriesStreamProperties::default() - })); - - assert!(!needs_update_info_details(&new_props, &old_props)); - } - - #[test] - fn merge_preserves_missing_live_probe_timestamps() { - let mut new_props = - StreamProperties::Live(Box::new(LiveStreamProperties { stream_id: 1, ..LiveStreamProperties::default() })); - let old_props = StreamProperties::Live(Box::new(LiveStreamProperties { - stream_id: 1, - last_probed_timestamp: Some(1_700_000_000), - last_success_timestamp: Some(1_700_000_100), - ..LiveStreamProperties::default() - })); - - let changed = merge_preserved_stream_properties(&mut new_props, &old_props); - assert!(changed); - - match new_props { - StreamProperties::Live(live) => { - assert_eq!(live.last_probed_timestamp, Some(1_700_000_000)); - assert_eq!(live.last_success_timestamp, Some(1_700_000_100)); - } - _ => panic!("expected live properties"), - } - } - - #[test] - fn merge_does_not_override_existing_live_probe_timestamps() { - let mut new_props = StreamProperties::Live(Box::new(LiveStreamProperties { - stream_id: 1, - last_probed_timestamp: Some(1_800_000_000), - last_success_timestamp: Some(1_800_000_100), - ..LiveStreamProperties::default() - })); - let old_props = StreamProperties::Live(Box::new(LiveStreamProperties { - stream_id: 1, - last_probed_timestamp: Some(1_700_000_000), - last_success_timestamp: Some(1_700_000_100), - ..LiveStreamProperties::default() - })); - - let changed = merge_preserved_stream_properties(&mut new_props, &old_props); - assert!(!changed); - - match new_props { - StreamProperties::Live(live) => { - assert_eq!(live.last_probed_timestamp, Some(1_800_000_000)); - assert_eq!(live.last_success_timestamp, Some(1_800_000_100)); - } - _ => panic!("expected live properties"), - } - } - - #[test] - fn merge_preserves_higher_learned_live_bitrate() { - let mut new_props = StreamProperties::Live(Box::new(LiveStreamProperties { - stream_id: 1, - bitrate: 1_500_000, - ..LiveStreamProperties::default() - })); - let old_props = StreamProperties::Live(Box::new(LiveStreamProperties { - stream_id: 1, - bitrate: 2_500_000, - ..LiveStreamProperties::default() - })); - - assert!(merge_preserved_stream_properties(&mut new_props, &old_props)); - match new_props { - StreamProperties::Live(live) => assert_eq!(live.bitrate, 2_500_000), - _ => panic!("expected live properties"), - } - } - - #[test] - fn merge_preserves_missing_live_catchup_properties() { - let mut new_props = - StreamProperties::Live(Box::new(LiveStreamProperties { stream_id: 1, ..LiveStreamProperties::default() })); - let old_props = StreamProperties::Live(Box::new(LiveStreamProperties { - stream_id: 1, - catchup: Some(CatchupProperties { - mode: Some("append".into()), - source: Some("?offset=-${offset}".into()), - ..CatchupProperties::default() - }), - ..LiveStreamProperties::default() - })); - - let changed = merge_preserved_stream_properties(&mut new_props, &old_props); - assert!(changed); - match new_props { - StreamProperties::Live(live) => { - let catchup = live.catchup.expect("catchup should be preserved"); - assert_eq!(catchup.mode.as_deref(), Some("append")); - assert_eq!(catchup.source.as_deref(), Some("?offset=-${offset}")); - } - _ => panic!("expected live properties"), - } - } - - #[test] - fn merge_preserves_missing_video_tmdb() { - let mut new_props = - StreamProperties::Video(Box::new(VideoStreamProperties { tmdb: None, ..VideoStreamProperties::default() })); - let old_props = StreamProperties::Video(Box::new(VideoStreamProperties { - tmdb: Some(317_981), - ..VideoStreamProperties::default() - })); - - let changed = merge_preserved_stream_properties(&mut new_props, &old_props); - assert!(changed); - match new_props { - StreamProperties::Video(video) => assert_eq!(video.tmdb, Some(317_981)), - _ => panic!("expected video properties"), - } - } - - #[test] - fn merge_preserves_missing_series_tmdb_and_release_date() { - let mut new_props = StreamProperties::Series(Box::new(SeriesStreamProperties { - tmdb: None, - release_date: None, - ..SeriesStreamProperties::default() - })); - let old_props = StreamProperties::Series(Box::new(SeriesStreamProperties { - tmdb: Some(12345), - release_date: Some("2015-01-01".into()), - ..SeriesStreamProperties::default() - })); - - let changed = merge_preserved_stream_properties(&mut new_props, &old_props); - assert!(changed); - match new_props { - StreamProperties::Series(series) => { - assert_eq!(series.tmdb, Some(12345)); - assert_eq!(series.release_date.as_deref(), Some("2015-01-01")); - } - _ => panic!("expected series properties"), - } - } - - fn make_live_item( - provider_id: u32, - video: Option<&str>, - audio: Option<&str>, - last_probed_timestamp: Option, - last_success_timestamp: Option, - bitrate: u32, - ) -> XtreamPlaylistItem { - XtreamPlaylistItem { - virtual_id: VirtualId::new(provider_id), - provider_id, - name: "Live".intern(), - logo: "".intern(), - logo_small: "".intern(), - group: "group".intern(), - title: "".intern(), - parent_code: "".intern(), - rec: "".intern(), - url: "http://example.com/live.ts".intern(), - epg_channel_id: None, - xtream_cluster: XtreamCluster::Live, - additional_properties: Some(StreamProperties::Live(Box::new(LiveStreamProperties { - video: video.map(Internable::intern), - audio: audio.map(Internable::intern), - last_probed_timestamp, - last_success_timestamp, - bitrate, - ..Default::default() - }))), - item_type: shared::model::PlaylistItemType::Live, - category_id: 1, - input_name: "input_a".intern(), - channel_no: 0, - source_ordinal: 0, - input_stream_id: provider_id.to_string().intern(), - upstream_user_agent: None, - } - } - - fn write_single_item(path: &Path, item: &XtreamPlaylistItem) { - crate::BPlusTree::::new().store(path).expect("tree creation should succeed"); - let mut tree = - BPlusTreeUpdate::::try_new_with_backoff(path).expect("tree open should succeed"); - let batch: Vec<(&u32, &XtreamPlaylistItem)> = vec![(&item.provider_id, item)]; - let prepared = BPlusTreeUpdate::::prepare_upsert_batch(&batch) - .expect("batch preparation should succeed"); - tree.upsert_batch_encoded(prepared).expect("batch upsert should succeed"); - tree.commit().expect("tree commit should succeed"); - } - - fn make_input_group( - cluster: XtreamCluster, - category_id: u32, - category_name: &str, - provider_id: u32, - ) -> PlaylistGroup { - let stream_id = provider_id.to_string().intern(); - let category_name = category_name.intern(); - PlaylistGroup { - id: category_id, - title: Arc::clone(&category_name), - channels: vec![PlaylistItem { - header: PlaylistItemHeader { - id: Arc::clone(&stream_id), - input_stream_id: stream_id, - name: format!("stream-{provider_id}").intern(), - title: format!("stream-{provider_id}").intern(), - group: category_name, - url: format!("http://provider.example/{cluster}/{provider_id}").intern(), - item_type: PlaylistItemType::from(cluster), - xtream_cluster: cluster, - category_id, - input_name: "provider-a".intern(), - ..PlaylistItemHeader::default() - }, - }], - xtream_cluster: cluster, - } - } - - #[tokio::test] - async fn input_cluster_count_returns_none_for_missing_baseline() { - let directory = tempdir().expect("temp directory"); - let app_config = test_app_config(directory.path()); - let input = - ConfigInput { name: "provider-a".intern(), input_type: InputType::Xtream, ..ConfigInput::default() }; - - assert_eq!( - count_input_xtream_cluster(&app_config, &input, XtreamCluster::Live) - .await - .expect("missing baseline should be readable"), - None - ); - } - - #[tokio::test] - async fn input_cluster_count_reads_the_canonical_active_raw_cluster() { - let directory = tempdir().expect("temp directory"); - let app_config = test_app_config(directory.path()); - let input = - ConfigInput { name: "provider-a".intern(), input_type: InputType::Xtream, ..ConfigInput::default() }; - - let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) - .await - .expect("canonical input storage"); - write_single_item( - &super::xtream_get_file_path(&storage_path, XtreamCluster::Live), - &make_live_item(700, None, None, None, None, 0), - ); - - assert_eq!( - count_input_xtream_cluster(&app_config, &input, XtreamCluster::Live) - .await - .expect("active baseline should be readable"), - Some(1) - ); - assert_eq!( - count_input_xtream_cluster(&app_config, &input, XtreamCluster::Video) - .await - .expect("other cluster should remain absent"), - None - ); - } - - #[tokio::test] - async fn in_memory_fallback_keeps_colliding_category_ids_separate_by_cluster() { - let directory = tempdir().expect("temp directory"); - let app_config = test_app_config(directory.path()); - let storage_path = get_input_storage_path("provider-a", directory.path().to_string_lossy().as_ref()) - .await - .expect("canonical input storage"); - - let (_, seed_error) = persist_input_xtream_playlist( - &app_config, - &storage_path, - vec![ - make_input_group(XtreamCluster::Live, 1, "Previous Live", 100), - make_input_group(XtreamCluster::Video, 2, "Previous VOD", 200), - make_input_group(XtreamCluster::Series, 3, "Previous Series", 300), - ], - ) - .await; - assert!(seed_error.is_none(), "failed to seed persisted VOD: {seed_error:?}"); - let previous_vod = load_input_xtream_playlist(&app_config, &storage_path, &[XtreamCluster::Video]) - .await - .expect("seeded VOD should load"); - let category_path = - get_collection_path(&storage_path, xtream_cluster_category_collection(XtreamCluster::Video)); - let previous_categories = fs::read(&category_path).expect("persisted VOD categories"); - - let (merged, persist_error) = persist_input_xtream_playlist( - &app_config, - &storage_path, - vec![ - make_input_group(XtreamCluster::Live, 1, "New Live", 101), - make_input_group(XtreamCluster::Series, 2, "New Series", 301), - ], - ) - .await; - - assert!(persist_error.is_none(), "failed to persist accepted clusters: {persist_error:?}"); - assert_eq!(fs::read(&category_path).expect("retained VOD categories"), previous_categories); - assert!(merged - .iter() - .all(|group| { group.channels.iter().all(|item| item.header.xtream_cluster == group.xtream_cluster) })); - - let retained_vod = merged - .iter() - .find(|group| group.xtream_cluster == XtreamCluster::Video && group.id == 2) - .expect("persisted VOD fallback"); - assert_eq!(retained_vod.title.as_ref(), "Previous VOD"); - assert_eq!(retained_vod.channels.len(), 1); - assert_eq!(retained_vod.channels[0].header.id.as_ref(), "200"); - - let accepted_live = merged - .iter() - .find(|group| group.xtream_cluster == XtreamCluster::Live && group.id == 1) - .expect("accepted Live candidate"); - assert_eq!(accepted_live.title.as_ref(), "New Live"); - assert_eq!(accepted_live.channels[0].header.id.as_ref(), "101"); - - let accepted_series = merged - .iter() - .find(|group| group.xtream_cluster == XtreamCluster::Series && group.id == 2) - .expect("accepted Series candidate sharing VOD category id"); - assert_eq!(accepted_series.title.as_ref(), "New Series"); - assert_eq!(accepted_series.channels[0].header.id.as_ref(), "301"); - - let loaded = load_input_xtream_playlist(&app_config, &storage_path, &[XtreamCluster::Video]) - .await - .expect("retained VOD should load"); - assert_eq!(loaded.len(), previous_vod.len()); - assert_eq!(loaded[0].id, previous_vod[0].id); - assert_eq!(loaded[0].title, previous_vod[0].title); - assert_eq!(loaded[0].xtream_cluster, previous_vod[0].xtream_cluster); - assert_eq!(loaded[0].channels.len(), previous_vod[0].channels.len()); - assert_eq!(loaded[0].channels[0].header.id, previous_vod[0].channels[0].header.id); - assert_eq!(loaded[0].channels[0].header.name, previous_vod[0].channels[0].header.name); - assert_eq!(loaded[0].channels[0].header.category_id, previous_vod[0].channels[0].header.category_id); - assert_eq!(loaded[0].channels[0].header.xtream_cluster, previous_vod[0].channels[0].header.xtream_cluster); - } - - fn read_live_props(path: &Path, provider_id: u32) -> LiveStreamProperties { - let mut query = BPlusTreeQuery::::try_new(path).expect("query open should succeed"); - let item = query.query_zero_copy(&provider_id).expect("query should succeed").expect("item should exist"); - match item.additional_properties { - Some(StreamProperties::Live(live)) => *live, - other => panic!("expected live stream properties, got {other:?}"), - } - } - - fn fixed_refresh_paths(path: &Path, generation: u128) -> XtreamRefreshPaths { - XtreamRefreshPaths::for_generation(path, XtreamCluster::Live, Uuid::from_u128(generation)) - .expect("fixed refresh paths should be valid") - } - - fn write_detail_preservation_fixture(paths: &XtreamRefreshPaths, provider_id: u32) { - write_single_item( - &paths.published_database, - &make_live_item( - provider_id, - Some("{\"codec_name\":\"h264\"}"), - Some("{\"codec_name\":\"aac\"}"), - Some(1_700_000_000), - Some(1_700_000_100), - 2_500_000, - ), - ); - write_single_item(&paths.staging_database, &make_live_item(provider_id, None, None, None, None, 0)); - } - - #[test] - fn refresh_staging_database_uses_distinct_published_lock_domain() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 1); - - assert_eq!(paths.published_database.file_name().and_then(|name| name.to_str()), Some("live.db")); - assert_ne!(sidecar_lock_path(&paths.published_database), sidecar_lock_path(&paths.staging_database)); - } - - #[test] - fn refresh_staging_database_and_index_share_one_generation_lock_domain() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 2); - let staging_index = get_file_path_for_db_index(&paths.staging_database); - - assert_eq!(sidecar_lock_path(&paths.staging_database), sidecar_lock_path(&staging_index)); - } - - #[test] - fn colliding_staging_path_is_rejected_before_lock_acquisition() { - let dir = tempdir().expect("temp dir should be created"); - let published = dir.path().join("live.db"); - let colliding = dir.path().join("live.tmp"); - - let error = ensure_distinct_sidecar_lock_domains(&published, &colliding) - .expect_err("colliding sidecar domains must be rejected"); - assert_eq!(error.kind(), io::ErrorKind::InvalidInput); - } - - #[test] - fn preserve_details_for_disk_cluster_copies_missing_live_probe_fields() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 3); - let provider_id = 100_u32; - - write_single_item( - &paths.published_database, - &make_live_item( - provider_id, - Some("{\"codec_name\":\"h264\"}"), - Some("{\"codec_name\":\"aac\"}"), - Some(1_700_000_000), - Some(1_700_000_100), - 2_500_000, - ), - ); - write_single_item(&paths.staging_database, &make_live_item(provider_id, None, None, None, None, 0)); - - let outcome = - preserve_details_input_xtream_playlist_cluster_to_disk(&paths.published_database, &paths.staging_database) - .expect("merge should succeed"); - assert_eq!(outcome, PreserveDetailsOutcome::Merged { scanned: 1, updated: 1 }); - - let merged = read_live_props(&paths.staging_database, provider_id); - assert_eq!(merged.video, Some("{\"codec_name\":\"h264\"}".intern())); - assert_eq!(merged.audio, Some("{\"codec_name\":\"aac\"}".intern())); - assert_eq!(merged.last_probed_timestamp, Some(1_700_000_000)); - assert_eq!(merged.last_success_timestamp, Some(1_700_000_100)); - assert_eq!(merged.bitrate, 2_500_000); - } - - #[test] - fn preserve_details_reports_missing_published_database_explicitly() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 4); - write_single_item(&paths.staging_database, &make_live_item(101, None, None, None, None, 0)); - - let outcome = - preserve_details_input_xtream_playlist_cluster_to_disk(&paths.published_database, &paths.staging_database) - .expect("a missing published database should not fail the refresh"); - - assert_eq!(outcome, PreserveDetailsOutcome::SourceMissing); - } - - #[test] - fn preserve_details_propagates_corrupt_published_database() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 5); - fs::write(&paths.published_database, b"corrupt").expect("corrupt fixture should be written"); - write_single_item(&paths.staging_database, &make_live_item(102, None, None, None, None, 0)); - - let error = - preserve_details_input_xtream_playlist_cluster_to_disk(&paths.published_database, &paths.staging_database) - .expect_err("corrupt published data must fail"); - - assert!(error.to_string().contains(&paths.published_database.display().to_string())); - } - - #[test] - fn preserve_details_propagates_corrupt_staging_database() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 6); - write_single_item(&paths.published_database, &make_live_item(103, None, None, None, None, 0)); - fs::write(&paths.staging_database, b"corrupt").expect("corrupt fixture should be written"); - - let error = - preserve_details_input_xtream_playlist_cluster_to_disk(&paths.published_database, &paths.staging_database) - .expect_err("corrupt staging data must fail"); - - assert!(error.to_string().contains(&paths.staging_database.display().to_string())); - } - - #[test] - fn preserve_details_propagates_missing_staging_database() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 14); - write_single_item(&paths.published_database, &make_live_item(105, None, None, None, None, 0)); - - let error = - preserve_details_input_xtream_playlist_cluster_to_disk(&paths.published_database, &paths.staging_database) - .expect_err("a missing staging database must fail"); - let message = error.to_string(); - - assert!(message.contains("Failed to open staging Xtream tree")); - assert!(message.contains(&paths.staging_database.display().to_string())); - } - - #[test] - fn preserve_details_propagates_staging_query_failure() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 15); - write_detail_preservation_fixture(&paths, 106); - - let error = preserve_details_with_injected_operation_failure( - &paths.published_database, - &paths.staging_database, - DetailPreservationOperation::Query, - ) - .expect_err("a staging query failure must fail the merge"); - let message = error.to_string(); - - assert!(message.contains("Failed to query staging Xtream tree")); - assert!(message.contains(&paths.staging_database.display().to_string())); - assert!(message.contains("injected Query failure")); - } - - #[test] - fn preserve_details_propagates_staging_batch_write_failure() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 16); - write_detail_preservation_fixture(&paths, 107); - - let error = preserve_details_with_injected_operation_failure( - &paths.published_database, - &paths.staging_database, - DetailPreservationOperation::BatchWrite, - ) - .expect_err("a staging batch write failure must fail the merge"); - let message = error.to_string(); - - assert!(message.contains("Failed to update staging Xtream tree")); - assert!(message.contains(&paths.staging_database.display().to_string())); - assert!(message.contains("injected BatchWrite failure")); - } - - #[test] - fn preserve_details_propagates_staging_commit_failure() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 17); - write_detail_preservation_fixture(&paths, 108); - - let error = preserve_details_with_injected_operation_failure( - &paths.published_database, - &paths.staging_database, - DetailPreservationOperation::Commit, - ) - .expect_err("a staging commit failure must fail the merge"); - let message = error.to_string(); - - assert!(message.contains("Failed to commit staging Xtream tree")); - assert!(message.contains(&paths.staging_database.display().to_string())); - assert!(message.contains("injected Commit failure")); - } - - #[test] - fn preserve_details_empty_merge_reports_zero_updates() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 7); - let item = make_live_item(104, None, None, None, None, 0); - write_single_item(&paths.published_database, &item); - write_single_item(&paths.staging_database, &item); - - let outcome = - preserve_details_input_xtream_playlist_cluster_to_disk(&paths.published_database, &paths.staging_database) - .expect("empty merge should succeed"); - - assert_eq!(outcome, PreserveDetailsOutcome::Merged { scanned: 1, updated: 0 }); - } - - #[test] - fn refresh_lease_cleanup_is_idempotent_and_generation_local() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 8); - let other_paths = fixed_refresh_paths(dir.path(), 9); - let lease = XtreamRefreshLease::new(paths.clone()).expect("refresh lease should be valid"); - - fs::write(&paths.published_database, b"published").expect("published fixture should be written"); - let published_lock = sidecar_lock_path(&paths.published_database); - fs::write(&published_lock, b"").expect("published lock fixture should be written"); - for artifact in lease.0.database_artifacts.owned_paths() { - fs::write(artifact, b"staging").expect("staging artifact should be written"); - } - fs::write(&paths.staging_categories, b"staging").expect("staging category should be written"); - fs::write(&other_paths.staging_database, b"other generation") - .expect("other generation fixture should be written"); - - lease.cleanup_staging_artifacts().expect("first cleanup should succeed"); - lease.cleanup_staging_artifacts().expect("second cleanup should be idempotent"); - - assert!(paths.published_database.exists()); - assert!(published_lock.exists()); - assert!(other_paths.staging_database.exists()); - assert!(!paths.staging_database.exists()); - assert!(!paths.staging_categories.exists()); - } - - #[test] - fn refresh_lease_defers_cleanup_until_last_worker_clone_drops() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 13); - let guard_path = refresh_generation_guard_path(dir.path(), paths.generation); - let parent_lease = XtreamRefreshLease::new(paths.clone()).expect("refresh lease should be valid"); - let worker_lease = parent_lease.clone(); - fs::write(&paths.staging_database, b"staging").expect("staging fixture should be written"); - fs::write(&paths.staging_categories, b"categories").expect("category fixture should be written"); - - drop(parent_lease); - assert!(paths.staging_database.exists()); - assert!(paths.staging_categories.exists()); - assert!(guard_path.exists()); - - drop(worker_lease); - assert!(!paths.staging_database.exists()); - assert!(!paths.staging_categories.exists()); - assert!(!guard_path.exists()); - } - - #[test] - fn active_refresh_between_btree_batches_survives_orphan_cleanup() { - let dir = tempdir().expect("temp dir should be created"); - let paths = fixed_refresh_paths(dir.path(), 14); - let guard_path = refresh_generation_guard_path(dir.path(), paths.generation); - let lease = XtreamRefreshLease::new(paths.clone()).expect("refresh lease should be valid"); - let sidecar = sidecar_lock_path(&paths.staging_database); - fs::write(&paths.staging_database, b"staging").expect("staging fixture should be written"); - fs::write(&paths.staging_categories, b"categories").expect("category fixture should be written"); - fs::write(&sidecar, b"").expect("sidecar fixture should be written"); - - let between_batch_probe = - fs::OpenOptions::new().read(true).write(true).open(&sidecar).expect("open staging sidecar"); - between_batch_probe.try_lock().expect("staging sidecar should be unlocked between batches"); - between_batch_probe.unlock().expect("release between-batch probe"); - drop(between_batch_probe); - - cleanup_orphaned_staging_artifacts(dir.path(), Duration::ZERO); - - assert!(paths.staging_database.exists()); - assert!(paths.staging_categories.exists()); - assert!(sidecar.exists()); - assert!(guard_path.exists()); - - drop(lease); - assert!(!paths.staging_database.exists()); - assert!(!paths.staging_categories.exists()); - assert!(!sidecar.exists()); - assert!(!guard_path.exists()); - } - - #[test] - fn sequential_refresh_generations_use_distinct_stems() { - let dir = tempdir().expect("temp dir should be created"); - let first = fixed_refresh_paths(dir.path(), 10); - let second = fixed_refresh_paths(dir.path(), 11); - - assert_ne!(first.staging_database.file_stem(), second.staging_database.file_stem()); - assert_ne!(sidecar_lock_path(&first.staging_database), sidecar_lock_path(&second.staging_database)); - } - - #[test] - fn category_publish_atomically_replaces_same_directory_file() { - let dir = tempdir().expect("temp dir should be created"); - let published = dir.path().join("cat_live.json"); - let staging = dir.path().join("cat_live.refresh-fixed.json"); - fs::write(&published, b"old").expect("published fixture should be written"); - fs::write(&staging, b"new").expect("staging fixture should be written"); - - publish_staged_file_same_directory(&staging, &published).expect("category publish should succeed"); - - assert_eq!(fs::read(&published).expect("published categories should be readable"), b"new"); - assert!(!staging.exists()); - } - - #[test] - fn category_publish_creates_missing_same_directory_file() { - let dir = tempdir().expect("temp dir should be created"); - let published = dir.path().join("cat_live.json"); - let staging = dir.path().join("cat_live.refresh-fixed.json"); - fs::write(&staging, b"new").expect("staging fixture should be written"); - - publish_staged_file_same_directory(&staging, &published).expect("category publish should succeed"); - - assert_eq!(fs::read(&published).expect("published categories should be readable"), b"new"); - assert!(!staging.exists()); - } - - #[test] - fn category_publish_rejects_different_parent_before_replace() { - let dir = tempdir().expect("temp dir should be created"); - let staging_dir = dir.path().join("staging"); - let published_dir = dir.path().join("published"); - fs::create_dir_all(&staging_dir).expect("staging directory should be created"); - fs::create_dir_all(&published_dir).expect("published directory should be created"); - let staging = staging_dir.join("cat_live.refresh-fixed.json"); - let published = published_dir.join("cat_live.json"); - fs::write(&staging, b"new").expect("staging fixture should be written"); - fs::write(&published, b"old").expect("published fixture should be written"); - - let error = publish_staged_file_same_directory(&staging, &published) - .expect_err("cross-directory category publication should fail"); - - assert_eq!(error.kind(), io::ErrorKind::InvalidInput); - assert_eq!(fs::read(&staging).expect("staging fixture should remain"), b"new"); - assert_eq!(fs::read(&published).expect("published fixture should remain"), b"old"); - } - - #[cfg(not(windows))] - #[test] - fn category_publish_reports_post_rename_barrier_failure_truthfully() { - let dir = tempdir().expect("temp dir should be created"); - let published = dir.path().join("cat_live.json"); - let staging = dir.path().join("cat_live.refresh-fixed.json"); - fs::write(&published, b"old").expect("published fixture should be written"); - fs::write(&staging, b"new").expect("staging fixture should be written"); - let staging_path = - tempfile::TempPath::try_from_path(&staging).expect("staging fixture should become an owned temporary path"); - - let error = super::publish_staged_file_with_parent_sync(staging_path, &published, |_| { - Err(io::Error::other("injected parent synchronization failure")) - }) - .expect_err("post-rename synchronization failure should be reported"); - - assert_eq!(error.kind(), io::ErrorKind::Other); - assert!(error.to_string().contains("was published, but its parent directory")); - assert_eq!(fs::read(&published).expect("published categories should be readable"), b"new"); - assert!(!staging.exists()); - } - - #[cfg(unix)] - #[test] - fn refresh_staging_path_preserves_non_utf8_stem() { - use std::os::unix::ffi::{OsStrExt, OsStringExt}; - - let mut input_name = std::ffi::OsString::from_vec(vec![b'l', b'i', b'v', b'e', 0xff]); - input_name.push(".db"); - let published = Path::new("/tmp").join(input_name); - let generation = Uuid::from_u128(12); - - let staging = refresh_staging_path(&published, generation).expect("non-UTF-8 path should be supported"); - let bytes = staging.file_name().expect("staging file name").as_bytes(); - assert!(bytes.starts_with(&[b'l', b'i', b'v', b'e', 0xff])); - assert!(bytes.ends_with(b".db")); - assert!(bytes.windows(b".refresh-".len()).any(|window| window == b".refresh-")); - } - - #[test] - fn xtream_refresh_end_to_end_child() -> io::Result<()> { - let Some(storage_root) = env::var_os("TULIPROX_XTREAM_REFRESH_TEST_ROOT") else { - return Ok(()); - }; - let storage_root = Path::new(&storage_root); - let app_config = test_app_config(storage_root); - let input = - ConfigInput { name: "deadlock-test".intern(), input_type: InputType::Xtream, ..ConfigInput::default() }; - let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; - runtime.block_on(async { - for _ in 0..2 { - persist_input_xtream_playlist_cluster_to_disk( - &app_config, - &input, - XtreamCluster::Live, - 0, - json_reader(r#"[{"category_id":"1","category_name":"Sports"}]"#), - json_reader(r#"[{"name":"Live","stream_id":700,"category_id":"1","added":"0"}]"#), - ) - .await - .map_err(|error| io::Error::other(error.to_string()))?; - } - Ok::<(), io::Error>(()) - })?; - - let input_storage = build_input_storage_path(&input.name, storage_root.to_string_lossy().as_ref()); - let published = super::xtream_get_file_path(&input_storage, XtreamCluster::Live); - let learned = read_live_props(&published, 700); - assert_eq!(learned.video, Some("{\"codec_name\":\"h264\"}".intern())); - assert_eq!(learned.bitrate, 2_500_000); - Ok(()) - } - - #[test] - fn two_sequential_cluster_refreshes_complete_without_generation_artifacts() -> io::Result<()> { - let directory = tempfile::tempdir()?; - let input_name = "deadlock-test".intern(); - let input_storage = build_input_storage_path(&input_name, directory.path().to_string_lossy().as_ref()); - fs::create_dir_all(&input_storage)?; - let published = super::xtream_get_file_path(&input_storage, XtreamCluster::Live); - write_single_item( - &published, - &make_live_item( - 700, - Some("{\"codec_name\":\"h264\"}"), - Some("{\"codec_name\":\"aac\"}"), - Some(1_700_000_000), - Some(1_700_000_100), - 2_500_000, - ), - ); - - let child = Command::new(env::current_exe()?) - .arg("--exact") - .arg("xtream_repository::tests::xtream_refresh_end_to_end_child") - .arg("--nocapture") - .env("TULIPROX_XTREAM_REFRESH_TEST_ROOT", directory.path()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn()?; - let status = wait_for_child(child, Duration::from_secs(20))?; - assert!(status.success(), "Xtream refresh child failed with {status}"); - - let learned = read_live_props(&published, 700); - assert_eq!(learned.video, Some("{\"codec_name\":\"h264\"}".intern())); - assert_eq!(learned.bitrate, 2_500_000); - let entries = fs::read_dir(&input_storage)?.collect::>>()?; - let generation_artifacts = entries - .into_iter() - .filter(|entry| entry.file_name().to_string_lossy().contains(".refresh-")) - .collect::>(); - assert!(generation_artifacts.is_empty(), "generation artifacts survived successful refreshes"); - assert!(sidecar_lock_path(&published).exists()); - Ok(()) - } - - #[test] - fn preserve_details_for_disk_cluster_does_not_override_existing_live_probe_fields() { - let dir = tempdir().expect("temp dir should be created"); - let old_path = dir.path().join("old_live_existing.db"); - let tmp_path = dir.path().join("tmp_live_existing.db"); - let provider_id = 200_u32; - - write_single_item( - &old_path, - &make_live_item( - provider_id, - Some("{\"codec_name\":\"h264\"}"), - Some("{\"codec_name\":\"aac\"}"), - Some(1_700_000_000), - Some(1_700_000_100), - 2_500_000, - ), - ); - write_single_item( - &tmp_path, - &make_live_item( - provider_id, - Some("{\"codec_name\":\"hevc\"}"), - Some("{\"codec_name\":\"ac3\"}"), - Some(1_800_000_000), - Some(1_800_000_100), - 3_500_000, - ), - ); - - preserve_details_input_xtream_playlist_cluster_to_disk(&old_path, &tmp_path).expect("merge should succeed"); - - let merged = read_live_props(&tmp_path, provider_id); - assert_eq!(merged.video, Some("{\"codec_name\":\"hevc\"}".intern())); - assert_eq!(merged.audio, Some("{\"codec_name\":\"ac3\"}".intern())); - assert_eq!(merged.last_probed_timestamp, Some(1_800_000_000)); - assert_eq!(merged.last_success_timestamp, Some(1_800_000_100)); - assert_eq!(merged.bitrate, 3_500_000); - } - - #[test] - fn preserve_details_for_disk_cluster_fills_only_missing_live_probe_fields() { - let dir = tempdir().expect("temp dir should be created"); - let old_path = dir.path().join("old_live_partial.db"); - let tmp_path = dir.path().join("tmp_live_partial.db"); - let provider_id = 300_u32; - - write_single_item( - &old_path, - &make_live_item( - provider_id, - Some("{\"codec_name\":\"h264\"}"), - Some("{\"codec_name\":\"aac\"}"), - Some(1_700_000_000), - Some(1_700_000_100), - 2_500_000, - ), - ); - write_single_item( - &tmp_path, - &make_live_item(provider_id, Some("{\"codec_name\":\"hevc\"}"), None, Some(1_800_000_000), None, 3_500_000), - ); - - preserve_details_input_xtream_playlist_cluster_to_disk(&old_path, &tmp_path).expect("merge should succeed"); - - let merged = read_live_props(&tmp_path, provider_id); - assert_eq!(merged.video, Some("{\"codec_name\":\"hevc\"}".intern())); - assert_eq!(merged.audio, Some("{\"codec_name\":\"aac\"}".intern())); - assert_eq!(merged.last_probed_timestamp, Some(1_800_000_000)); - assert_eq!(merged.last_success_timestamp, Some(1_700_000_100)); - } -} +pub(crate) use self::target_publish::xtream_write_playlist_with_injected_empty_replacement_failure; +#[cfg(test)] +use self::target_publish::TargetEmptyPublicationHook; +#[cfg(test)] +use self::target_publish::{xtream_write_playlist_with_mode, TargetEmptyReplacementMode}; +use self::{ + details::{cluster_flag, preserve_details_input_xtream_playlist_cluster_to_disk, PreserveDetailsOutcome}, + input_refresh::{XtreamClusterEvaluationReport, XtreamRefreshLease}, + paths::{target_category_lock_path, XtreamRefreshPaths}, + publish::{publish_staged_file_same_directory, save_xtream_categories_to_file}, + read::{get_map_item_as_str, xtream_get_file_path_for_name}, + target_publish::write_playlists_to_file, +}; +pub use self::{ + details::{ + merge_preserved_stream_properties, needs_update_info_details, persist_input_info_batch, + persist_input_live_info, persist_input_live_info_batch, persist_input_series_info_batch, + persist_input_vod_info, persist_input_vod_info_batch, persists_input_series_info, + }, + input_refresh::{persist_input_xtream_playlist_cluster_to_disk, persist_input_xtream_playlist_clusters_to_disk}, + paths::{ + ensure_xtream_storage_path, get_collection_path, get_live_cat_collection_path, get_series_cat_collection_path, + get_vod_cat_collection_path, xtream_cluster_category_collection, xtream_get_collection_path, + xtream_get_file_path, xtream_get_storage_path, + }, + read::{ + count_input_xtream_cluster, iter_raw_xtream_input_playlist, iter_raw_xtream_target_playlist, + load_input_xtream_playlist, persist_input_xtream_playlist, + persist_input_xtream_playlist_with_empty_replacements, playlist_iter_to_stream, + xtream_get_epg_file_path_for_target, xtream_get_item_for_stream_id, xtream_get_playlist_categories, + xtream_load_rewrite_playlist, + }, + target_publish::{write_playlist_batch_item_upsert, write_playlist_item_update, xtream_write_playlist}, +}; diff --git a/backend/repository/src/xtream_repository/details.rs b/backend/repository/src/xtream_repository/details.rs new file mode 100644 index 000000000..9cfd31b59 --- /dev/null +++ b/backend/repository/src/xtream_repository/details.rs @@ -0,0 +1,436 @@ +use super::{xtream_get_file_path, BATCH_SIZE}; +use crate::bplustree::{ensure_distinct_sidecar_lock_domains, BPlusTreeError, BPlusTreeQuery, BPlusTreeUpdate}; +use log::error; +use shared::{ + error::TuliproxError, + model::{ + ClusterFlags, LiveStreamProperties, SeriesStreamProperties, StreamProperties, VideoStreamProperties, + XtreamCluster, XtreamPlaylistItem, + }, +}; +use std::{collections::HashMap, io, io::Error, path::Path, sync::Arc}; +use tuliprox_core::model::AppConfig; + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum PreserveDetailsOutcome { + SourceMissing, + Merged { scanned: usize, updated: usize }, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum DetailPreservationOperation { + Query, + BatchWrite, + Commit, +} + +fn write_preserved_detail_batch( + staging_tree: &mut BPlusTreeUpdate, + staging_path: &Path, + updates: &mut Vec<(u32, XtreamPlaylistItem)>, + before_operation: &mut F, +) -> Result +where + F: FnMut(DetailPreservationOperation) -> io::Result<()>, +{ + if updates.is_empty() { + return Ok(0); + } + let batch_len = updates.len(); + let refs: Vec<(&u32, &XtreamPlaylistItem)> = updates.iter().map(|(id, item)| (id, item)).collect(); + before_operation(DetailPreservationOperation::BatchWrite) + .and_then(|()| staging_tree.update_batch(&refs).map(|_| ()).map_err(BPlusTreeError::to_io)) + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to update staging Xtream tree {} during detail preservation: {error}", + staging_path.display() + )) + })?; + updates.clear(); + Ok(batch_len) +} + +pub(super) fn preserve_details_input_xtream_playlist_cluster_to_disk( + published_path: &Path, + staging_path: &Path, +) -> Result { + preserve_details_input_xtream_playlist_cluster_to_disk_with_hook(published_path, staging_path, |_| Ok(())) +} + +fn preserve_details_input_xtream_playlist_cluster_to_disk_with_hook( + published_path: &Path, + staging_path: &Path, + mut before_operation: F, +) -> Result +where + F: FnMut(DetailPreservationOperation) -> io::Result<()>, +{ + ensure_distinct_sidecar_lock_domains(published_path, staging_path) + .map_err(|error| TuliproxError::RepositoryXtream(error.to_string()))?; + + let mut published_tree = match BPlusTreeQuery::::try_new(published_path) { + Ok(tree) => tree, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return Ok(PreserveDetailsOutcome::SourceMissing); + } + Err(error) => { + return Err(TuliproxError::RepositoryXtream(format!( + "Failed to open published Xtream tree {} for detail preservation: {error}", + published_path.display() + ))); + } + }; + + let mut staging_tree = + BPlusTreeUpdate::::try_new_with_backoff(staging_path).map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to open staging Xtream tree {} for detail preservation: {error}", + staging_path.display() + )) + })?; + + let mut pending_updates: Vec<(u32, XtreamPlaylistItem)> = Vec::with_capacity(BATCH_SIZE); + let mut scanned_count = 0usize; + let mut updated_count = 0usize; + for entry in published_tree.iter() { + let (_, old_item) = entry.map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to read published Xtream tree {} during detail preservation: {error}", + published_path.display() + )) + })?; + scanned_count = scanned_count.saturating_add(1); + if let Some(old_props) = old_item.additional_properties.as_ref() { + if old_props.has_details() { + let staging_item = before_operation(DetailPreservationOperation::Query) + .and_then(|()| staging_tree.query(&old_item.provider_id).map_err(BPlusTreeError::to_io)) + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to query staging Xtream tree {} for provider {}: {error}", + staging_path.display(), + old_item.provider_id + )) + })?; + if let Some(mut new_item) = staging_item { + if let Some(new_props) = new_item.additional_properties.as_mut() { + if merge_preserved_stream_properties(new_props, old_props) { + pending_updates.push((new_item.provider_id, new_item)); + if pending_updates.len() >= BATCH_SIZE { + updated_count = updated_count.saturating_add(write_preserved_detail_batch( + &mut staging_tree, + staging_path, + &mut pending_updates, + &mut before_operation, + )?); + } + } + } + } + } + } + } + + updated_count = updated_count.saturating_add(write_preserved_detail_batch( + &mut staging_tree, + staging_path, + &mut pending_updates, + &mut before_operation, + )?); + before_operation(DetailPreservationOperation::Commit).and_then(|()| staging_tree.commit()).map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to commit staging Xtream tree {} after detail preservation: {error}", + staging_path.display() + )) + })?; + + Ok(PreserveDetailsOutcome::Merged { scanned: scanned_count, updated: updated_count }) +} + +#[cfg(test)] +pub(super) fn preserve_details_with_injected_operation_failure( + published_path: &Path, + staging_path: &Path, + failure: DetailPreservationOperation, +) -> Result { + preserve_details_input_xtream_playlist_cluster_to_disk_with_hook(published_path, staging_path, move |operation| { + if operation == failure { + Err(io::Error::other(format!("injected {failure:?} failure"))) + } else { + Ok(()) + } + }) +} + +pub(super) const fn cluster_flag(cluster: XtreamCluster) -> ClusterFlags { + match cluster { + XtreamCluster::Live => ClusterFlags::Live, + XtreamCluster::Video => ClusterFlags::Vod, + XtreamCluster::Series => ClusterFlags::Series, + } +} + +// Checks if the info has changed after the last update +pub fn needs_update_info_details(new_stream_props: &StreamProperties, old_stream_props: &StreamProperties) -> bool { + let new_modified = new_stream_props.get_last_modified(); + let old_modified = old_stream_props.get_last_modified(); + + match (new_modified, old_modified) { + (Some(new_ts), Some(old_ts)) => new_ts > old_ts, + (Some(_), None) => true, + _ => false, + } +} + +/// Merges persisted fields from old stream properties into freshly fetched properties. +/// +/// This keeps long-lived metadata stable across full playlist rewrites: +/// - VOD/Series `details` are preserved when incoming provider metadata is not newer. +/// - Learned Live fields are merged through [`LiveStreamProperties::merge_learned_metadata_from`]. +/// - Live catchup remains separate provider metadata and is copied only when missing. +pub fn merge_preserved_stream_properties( + new_stream_props: &mut StreamProperties, + old_stream_props: &StreamProperties, +) -> bool { + let preserve_info_details = + old_stream_props.has_details() && !needs_update_info_details(new_stream_props, old_stream_props); + + match (new_stream_props, old_stream_props) { + (StreamProperties::Video(v_new), StreamProperties::Video(v_old)) => { + let mut changed = false; + + if preserve_info_details && v_old.details.is_some() && v_new.details != v_old.details { + v_new.details.clone_from(&v_old.details); + changed = true; + } + + if v_new.tmdb.is_none() && v_old.tmdb.is_some() { + v_new.tmdb = v_old.tmdb; + changed = true; + } + + changed + } + (StreamProperties::Series(s_new), StreamProperties::Series(s_old)) => { + let mut changed = false; + + if preserve_info_details && s_old.details.is_some() && s_new.details != s_old.details { + s_new.details.clone_from(&s_old.details); + changed = true; + } + + if s_new.tmdb.is_none() && s_old.tmdb.is_some() { + s_new.tmdb = s_old.tmdb; + changed = true; + } + + if s_new.release_date.is_none() && s_old.release_date.is_some() { + s_new.release_date.clone_from(&s_old.release_date); + changed = true; + } + + changed + } + (StreamProperties::Live(l_new), StreamProperties::Live(l_old)) => { + let mut changed = l_new.merge_learned_metadata_from(l_old); + + if l_new.catchup.is_none() && l_old.catchup.is_some() { + l_new.catchup.clone_from(&l_old.catchup); + changed = true; + } + + changed + } + _ => false, + } +} + +async fn persist_input_info( + app_config: &Arc, + storage_path: &Path, + cluster: XtreamCluster, + input_name: &str, + provider_id: u32, + props: StreamProperties, +) -> Result<(), Error> { + let xtream_path = xtream_get_file_path(storage_path, cluster); + if xtream_path.exists() { + let file_lock = app_config.file_locks.write_lock(&xtream_path).await; + let xtream_path_clone = xtream_path.clone(); + let input_name_owned = input_name.to_string(); + tokio::task::spawn_blocking(move || -> Result<(), Error> { + let _guard = file_lock; + let mut tree: BPlusTreeUpdate = + BPlusTreeUpdate::try_new_with_backoff(&xtream_path_clone).map_err(|err| { + Error::other(format!("failed to open BPlusTree for input {input_name_owned}: {err}")) + })?; + match tree.query(&provider_id) { + Ok(Some(mut pli)) => { + pli.additional_properties = Some(props); + tree.update(&provider_id, pli).map_err(|err| { + Error::other(format!("failed to write {cluster} info for input {input_name_owned}: {err}")) + })?; + //rebuild_source_ordinal_index_if_present(&xtream_path_clone) + // .map_err(|err| Error::other(format!("failed to rebuild sorted index for input {input_name_owned}: {err}")))?; + } + Ok(None) => { + error!("Could not find input entry for provider_id: {provider_id} and input: {input_name_owned}"); + } + Err(err) => { + error!( + "Failed to query BPlusTree for provider_id: {provider_id} and input: {input_name_owned}: {err}" + ); + } + } + Ok(()) + }) + .await + .map_err(|err| Error::other(format!("failed to join blocking input info persist for {input_name}: {err}")))??; + } + Ok(()) +} + +pub async fn persist_input_info_batch( + app_config: &Arc, + storage_path: &Path, + cluster: XtreamCluster, + input_name: &str, + updates: Vec<(u32, StreamProperties)>, +) -> Result<(), Error> { + if updates.is_empty() { + return Ok(()); + } + let xtream_path = xtream_get_file_path(storage_path, cluster); + if xtream_path.exists() { + let file_lock = app_config.file_locks.write_lock(&xtream_path).await; + let xtream_path_clone = xtream_path.clone(); + let input_name_owned = input_name.to_string(); + tokio::task::spawn_blocking(move || -> Result<(), Error> { + let _guard = file_lock; + let mut tree: BPlusTreeUpdate = BPlusTreeUpdate::try_new_with_backoff(&xtream_path_clone) + .map_err(|err| Error::other(format!("failed to open BPlusTree for input {input_name_owned}: {err}")))?; + + // Keep only the latest update per provider id to avoid duplicate reads/writes. + let mut deduped_updates: HashMap = HashMap::with_capacity(updates.len()); + for (provider_id, props) in updates { + deduped_updates.insert(provider_id, props); + } + + let mut updated_plis = Vec::with_capacity(deduped_updates.len()); + for (provider_id, props) in deduped_updates { + match tree.query(&provider_id) { + Ok(Some(mut pli)) => { + pli.additional_properties = Some(props); + updated_plis.push((provider_id, pli)); + } + Ok(None) => { + error!("Could not find input entry for provider_id: {provider_id} and input: {input_name_owned}"); + } + Err(err) => { + error!("Failed to query BPlusTree for provider_id: {provider_id} and input: {input_name_owned}: {err}"); + } + } + } + + if !updated_plis.is_empty() { + let refs: Vec<(&u32, &XtreamPlaylistItem)> = updated_plis.iter() + .map(|(id, pli)| (id, pli)) + .collect(); + tree.update_batch(&refs).map_err(|err| Error::other(format!("failed to write batch {cluster} info for input {input_name_owned}: {err}")))?; + //rebuild_source_ordinal_index_if_present(&xtream_path_clone) + // .map_err(|err| Error::other(format!("failed to rebuild sorted index for input {input_name_owned}: {err}")))?; + } + Ok(()) + }).await.map_err(|err| Error::other(format!("failed to join blocking input info batch persist for {input_name}: {err}")))??; + } + Ok(()) +} + +pub async fn persist_input_vod_info( + app_config: &Arc, + storage_path: &Path, + cluster: XtreamCluster, + input_name: &str, + provider_id: u32, + props: &VideoStreamProperties, +) -> Result<(), Error> { + persist_input_info( + app_config, + storage_path, + cluster, + input_name, + provider_id, + StreamProperties::Video(Box::new(props.clone())), + ) + .await +} + +pub async fn persist_input_live_info( + app_config: &Arc, + storage_path: &Path, + cluster: XtreamCluster, + input_name: &str, + provider_id: u32, + props: &LiveStreamProperties, +) -> Result<(), Error> { + persist_input_info( + app_config, + storage_path, + cluster, + input_name, + provider_id, + StreamProperties::Live(Box::new(props.clone())), + ) + .await +} + +pub async fn persist_input_live_info_batch( + app_config: &Arc, + storage_path: &Path, + cluster: XtreamCluster, + input_name: &str, + updates: Vec<(u32, LiveStreamProperties)>, +) -> Result<(), Error> { + let batch = updates.into_iter().map(|(id, props)| (id, StreamProperties::Live(Box::new(props)))).collect(); + persist_input_info_batch(app_config, storage_path, cluster, input_name, batch).await +} + +pub async fn persist_input_vod_info_batch( + app_config: &Arc, + storage_path: &Path, + cluster: XtreamCluster, + input_name: &str, + updates: Vec<(u32, VideoStreamProperties)>, +) -> Result<(), Error> { + let batch = updates.into_iter().map(|(id, props)| (id, StreamProperties::Video(Box::new(props)))).collect(); + persist_input_info_batch(app_config, storage_path, cluster, input_name, batch).await +} + +pub async fn persists_input_series_info( + app_config: &Arc, + storage_path: &Path, + cluster: XtreamCluster, + input_name: &str, + provider_id: u32, + props: &SeriesStreamProperties, +) -> Result<(), Error> { + persist_input_info( + app_config, + storage_path, + cluster, + input_name, + provider_id, + StreamProperties::Series(Box::new(props.clone())), + ) + .await +} + +pub async fn persist_input_series_info_batch( + app_config: &Arc, + storage_path: &Path, + cluster: XtreamCluster, + input_name: &str, + updates: Vec<(u32, SeriesStreamProperties)>, +) -> Result<(), Error> { + let batch = updates.into_iter().map(|(id, props)| (id, StreamProperties::Series(Box::new(props)))).collect(); + persist_input_info_batch(app_config, storage_path, cluster, input_name, batch).await +} diff --git a/backend/repository/src/xtream_repository/input_refresh.rs b/backend/repository/src/xtream_repository/input_refresh.rs new file mode 100644 index 000000000..6c8e6d47e --- /dev/null +++ b/backend/repository/src/xtream_repository/input_refresh.rs @@ -0,0 +1,739 @@ +use super::{ + preserve_details_input_xtream_playlist_cluster_to_disk, publish_staged_file_same_directory, + save_xtream_categories_to_file, xtream_get_file_path, PreserveDetailsOutcome, XtreamClusterPublishBatchResult, + XtreamClusterPublishOutcome, XtreamClusterQualityPolicy, XtreamClusterRefreshRequest, XtreamRefreshPaths, + BATCH_SIZE, +}; +use crate::{ + bplustree::{ + publish_staged_database, BPlusTree, BPlusTreeError, BPlusTreeQuery, BPlusTreeStagingArtifacts, BPlusTreeUpdate, + FlushPolicy, + }, + storage::{ensure_input_storage_path, XtreamRefreshGenerationGuard}, +}; +use log::error; +use shared::{ + error::TuliproxError, + model::{xtream_const::XTREAM_CLUSTER, XtreamCluster, XtreamPlaylistItem}, +}; +use std::{ + collections::HashSet, + io, + io::ErrorKind, + path::{Path, PathBuf}, + sync::Arc, +}; +use tuliprox_core::{ + model::{ + evaluate_update_quality, AppConfig, ClusterForceUpdate, ClusterUpdateAcceptance, ClusterUpdateRejection, + ConfigInput, ConfigInputFlags, UpdateQualityDecision, + }, + utils::{remove_file_if_exists, request::DynReader, FileWriteGuard}, +}; +use tuliprox_parser::xtream; + +#[derive(Clone, Copy, Debug, Default)] +pub(super) struct XtreamClusterEvaluationReport { + pub(super) quality_acceptance: Option, + pub(super) quality_rejection: Option, + pub(super) force_update: Option, +} + +#[derive(Debug, Clone)] +pub(super) struct XtreamRefreshLease(pub(super) Arc); + +#[derive(Debug)] +pub(super) struct XtreamRefreshLeaseInner { + paths: XtreamRefreshPaths, + pub(super) database_artifacts: BPlusTreeStagingArtifacts, + _generation_guard: XtreamRefreshGenerationGuard, +} + +impl XtreamRefreshLease { + pub(super) fn new(paths: XtreamRefreshPaths) -> Result { + let database_artifacts = BPlusTreeStagingArtifacts::new(&paths.published_database, &paths.staging_database) + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Invalid Xtream staging artifacts for generation {}: {error}", + paths.generation + )) + })?; + let storage_path = paths.staging_database.parent().ok_or_else(|| { + TuliproxError::RepositoryXtream(format!( + "Xtream staging database has no storage directory: {}", + paths.staging_database.display() + )) + })?; + let generation_guard = + XtreamRefreshGenerationGuard::acquire(storage_path, paths.generation).map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to acquire Xtream refresh generation guard {} in {}: {error}", + paths.generation, + storage_path.display() + )) + })?; + Ok(Self(Arc::new(XtreamRefreshLeaseInner { paths, database_artifacts, _generation_guard: generation_guard }))) + } + + pub(super) fn paths(&self) -> &XtreamRefreshPaths { &self.0.paths } + + pub(super) fn cleanup_staging_artifacts(&self) -> io::Result<()> { + let database_result = self.0.database_artifacts.remove_owned_staging_artifacts(); + let categories_result = remove_file_if_exists(&self.0.paths.staging_categories); + match (database_result, categories_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(database_error), Err(categories_error)) => Err(io::Error::new( + database_error.kind(), + format!("{database_error}; category staging cleanup also failed: {categories_error}"), + )), + } + } +} + +impl Drop for XtreamRefreshLeaseInner { + fn drop(&mut self) { + let database_result = self.database_artifacts.remove_owned_staging_artifacts(); + let categories_result = remove_file_if_exists(&self.paths.staging_categories); + if let Err(error) = database_result { + log::warn!( + "Failed to clean Xtream staging database artifacts for generation {}: {error}", + self.paths.generation + ); + } + if let Err(error) = categories_result { + log::warn!( + "Failed to clean Xtream staging categories for generation {} at {}: {error}", + self.paths.generation, + self.paths.staging_categories.display() + ); + } + } +} + +#[derive(Clone, Copy)] +pub(super) struct XtreamClusterStageOperations { + pub(super) preserve_details: fn(&Path, &Path) -> Result, +} + +impl Default for XtreamClusterStageOperations { + fn default() -> Self { Self { preserve_details: preserve_details_input_xtream_playlist_cluster_to_disk } } +} + +struct StagedXtreamClusterRefresh { + refresh_lease: XtreamRefreshLease, + publish_lock: Arc, + storage_path: PathBuf, + input_name: Arc, + cluster: XtreamCluster, + raw_groups: Vec, + item_count: usize, + evaluation: XtreamClusterEvaluationReport, +} + +enum StagedXtreamClusterOutcome { + Ready(StagedXtreamClusterRefresh), + RetainedPrevious(ClusterUpdateRejection), + Failed { evaluation: XtreamClusterEvaluationReport, error: TuliproxError }, +} + +impl StagedXtreamClusterOutcome { + pub(super) const fn evaluation(&self) -> XtreamClusterEvaluationReport { + match self { + Self::Ready(refresh) => refresh.evaluation, + Self::RetainedPrevious(rejection) => XtreamClusterEvaluationReport { + quality_acceptance: None, + quality_rejection: Some(*rejection), + force_update: None, + }, + Self::Failed { evaluation, .. } => *evaluation, + } + } +} + +pub(super) fn count_xtream_tree_entries(path: &Path) -> io::Result> { + let mut query = match BPlusTreeQuery::::try_new(path) { + Ok(query) => query, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + query.len().map(Some).map_err(BPlusTreeError::to_io) +} + +pub(super) fn evaluate_staged_xtream_cluster_quality( + paths: &XtreamRefreshPaths, + cluster: XtreamCluster, + threshold: u8, +) -> Result { + if threshold == 0 { + return Ok(UpdateQualityDecision::Disabled); + } + + let candidate_count = count_xtream_tree_entries(&paths.staging_database) + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to count staging Xtream tree {} for {cluster} quality evaluation: {error}", + paths.staging_database.display() + )) + })? + .ok_or_else(|| { + TuliproxError::RepositoryXtream(format!( + "Staging Xtream tree {} disappeared before {cluster} quality evaluation", + paths.staging_database.display() + )) + })?; + let current_count = count_xtream_tree_entries(&paths.published_database).map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to count published Xtream tree {} for {cluster} quality evaluation: {error}", + paths.published_database.display() + )) + })?; + + Ok(evaluate_update_quality(current_count, candidate_count, threshold)) +} + +#[allow(clippy::too_many_lines)] +async fn stage_input_xtream_playlist_cluster_to_disk( + app_config: &Arc, + input: &ConfigInput, + request: XtreamClusterRefreshRequest, + operations: XtreamClusterStageOperations, +) -> Result { + let XtreamClusterRefreshRequest { cluster, quality: quality_policy, categories, streams } = request; + let cfg = app_config.config.load(); + let storage_path = ensure_input_storage_path(&cfg, &input.name).await?; + drop(cfg); + let refresh_lease = XtreamRefreshLease::new(XtreamRefreshPaths::new(&storage_path, cluster)?)?; + + // Channel for transferring items from Parser (Async Task) to Consumer (Blocking Task) + let (tx, mut rx) = tokio::sync::mpsc::channel::(BATCH_SIZE * 2); + let input_clone = input.clone(); + + // 1. Parser Task: Runs the async parsing logic + // We move the readers into this task. + let parse_task = tokio::spawn(async move { + let tx_for_closure = tx.clone(); + let res = xtream::parse_xtream_streaming(&input_clone, cluster, categories, streams, move |item| { + // Copy needed data before moving the item into the channel. + let item_id = item.virtual_id; + + // We use blocking_send because the closure provided by the parser library is synchronous. + // This is safe here because it runs within its own tokio::spawn task. + if let Err(e) = tx_for_closure.blocking_send(item) { + error!("Channel closed while processing {cluster} for item {item_id}: {e}"); + return Err(TuliproxError::RepositoryXtream(format!("Channel closed while processing {cluster}"))); + } + Ok(()) + }) + .await; + + // CRITICAL: Explicitly drop the sender to signal rx.blocking_recv() to stop. + // This prevents the consumer from waiting forever if the parser fails. + drop(tx); + res + }); + + // 2. Consumer Task: Handles heavy Disk I/O (BPlusTree updates) + let consumer_lease = refresh_lease.clone(); + let consumer_task = tokio::task::spawn_blocking(move || { + let staging_path = &consumer_lease.paths().staging_database; + + BPlusTree::::new().store(staging_path).map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to initialize staging Xtream tree {} for {cluster}: {error}", + staging_path.display() + )) + })?; + + let mut tree: BPlusTreeUpdate = BPlusTreeUpdate::try_new_with_backoff(staging_path) + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to open staging Xtream tree {} for {cluster}: {error}", + staging_path.display() + )) + })?; + tree.set_flush_policy(FlushPolicy::Batch); + + let mut buffer = Vec::with_capacity(BATCH_SIZE); + let mut seen_groups: HashSet = HashSet::new(); + let mut item_count = 0_usize; + + // This loop exits when all 'tx' clones are dropped (signaling end of stream) + while let Some(item) = rx.blocking_recv() { + item_count = item_count.saturating_add(1); + if !seen_groups.contains(item.group.as_ref()) { + seen_groups.insert(item.group.to_string()); + } + buffer.push(item); + if buffer.len() >= BATCH_SIZE { + let batch: Vec<(&u32, &XtreamPlaylistItem)> = buffer.iter().map(|i| (&i.provider_id, i)).collect(); + let prepared = + BPlusTreeUpdate::::prepare_upsert_batch(&batch).map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to prepare staging batch for {cluster} at {}: {error}", + staging_path.display() + )) + })?; + tree.upsert_batch_encoded(prepared).map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to write staging batch for {cluster} at {}: {error}", + staging_path.display() + )) + })?; + // Commit per batch so the write transaction's dirty-page map stays bounded. + // Holding it open for the whole cluster buffered ~48k pages (196 MB); the + // import writes into a .tmp file that is renamed on success, so atomicity + // comes from the rename, not from a single transaction. + tree.commit().map_err(|e| { + error!("Batch commit failed for cluster {cluster} at {}: {e}", staging_path.display()); + TuliproxError::RepositoryXtream(format!( + "Failed to commit staging batch for {cluster} at {}: {e}", + staging_path.display() + )) + })?; + buffer.clear(); + } + } + + // Final batch processing + if !buffer.is_empty() { + let batch: Vec<(&u32, &XtreamPlaylistItem)> = buffer.iter().map(|i| (&i.provider_id, i)).collect(); + let prepared = + BPlusTreeUpdate::::prepare_upsert_batch(&batch).map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to prepare final staging batch for {cluster} at {}: {error}", + staging_path.display() + )) + })?; + tree.upsert_batch_encoded(prepared).map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to write final staging batch for {cluster} at {}: {error}", + staging_path.display() + )) + })?; + } + + tree.commit().map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to commit staging Xtream tree for {cluster} at {}: {error}", + staging_path.display() + )) + })?; + Ok::<(Vec, usize), TuliproxError>((seen_groups.into_iter().collect(), item_count)) + }); + + // 3. Robust Joining of both tasks + // try_join! returns immediately if any task returns an error or panics. + let (parse_res, consumer_res) = tokio::try_join!(parse_task, consumer_task).map_err(|e| { + TuliproxError::RepositoryXtream(format!("Task join error during cluster {cluster} update: {e}")) + })?; + + // Handle internal errors from the tasks + let parsed_categories = parse_res?; + let (raw_groups, item_count) = consumer_res?; + + save_xtream_categories_to_file(refresh_lease.clone(), &parsed_categories).await?; + + // Lock order for the publish phase is always FileLockManager(final) followed by B+Tree sidecars. No B+Tree + // handle escapes its blocking closure, so none is held while this async lock is acquired. + let publish_lock = Arc::new(app_config.file_locks.write_lock(&refresh_lease.paths().published_database).await); + + let quality_lease = refresh_lease.clone(); + let quality_lock = Arc::clone(&publish_lock); + let (evaluation, rejection_cleanup_error) = tokio::task::spawn_blocking(move || { + let _publish_guard = quality_lock; + let evaluation = match quality_policy { + XtreamClusterQualityPolicy::Enforce { threshold } => { + let decision = evaluate_staged_xtream_cluster_quality(quality_lease.paths(), cluster, threshold)?; + XtreamClusterEvaluationReport { + quality_acceptance: decision.acceptance(cluster), + quality_rejection: decision.rejection(cluster), + force_update: None, + } + } + XtreamClusterQualityPolicy::Bypass { configured_threshold } => { + let candidate_count = count_xtream_tree_entries(&quality_lease.paths().staging_database) + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to count staging Xtream tree {} for forced {cluster} publication: {error}", + quality_lease.paths().staging_database.display() + )) + })? + .ok_or_else(|| { + TuliproxError::RepositoryXtream(format!( + "Staging Xtream tree {} disappeared before forced {cluster} publication", + quality_lease.paths().staging_database.display() + )) + })?; + XtreamClusterEvaluationReport { + quality_acceptance: None, + quality_rejection: None, + force_update: Some(ClusterForceUpdate { cluster, candidate_count, configured_threshold }), + } + } + }; + let rejection_cleanup_error = if evaluation.quality_rejection.is_some() { + quality_lease.cleanup_staging_artifacts().err().map(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to clean rejected Xtream staging artifacts for {cluster}: {error}" + )) + }) + } else { + None + }; + Ok::<_, TuliproxError>((evaluation, rejection_cleanup_error)) + }) + .await + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Quality-evaluation task failed to join during {cluster} refresh: {error}" + )) + })??; + + if let Some(error) = rejection_cleanup_error { + return Ok(StagedXtreamClusterOutcome::Failed { evaluation, error }); + } + + if let Some(rejection) = evaluation.quality_rejection { + drop(publish_lock); + log::debug!( + "Xtream cluster candidate rejected; retained active cluster: cluster={cluster} generation={} rejection={rejection:?}", + refresh_lease.paths().generation + ); + return Ok(StagedXtreamClusterOutcome::RetainedPrevious(rejection)); + } + + let merge_lease = refresh_lease.clone(); + let merge_lock = Arc::clone(&publish_lock); + let preserve_details = operations.preserve_details; + let merge_result = tokio::task::spawn_blocking(move || { + let _publish_guard = merge_lock; + preserve_details(&merge_lease.paths().published_database, &merge_lease.paths().staging_database) + }) + .await + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Detail-preservation task failed to join during {cluster} refresh: {error}" + )) + }); + let merge_outcome = match merge_result { + Ok(Ok(outcome)) => outcome, + Ok(Err(error)) | Err(error) => return Ok(StagedXtreamClusterOutcome::Failed { evaluation, error }), + }; + log::debug!( + "Xtream cluster detail preservation completed: cluster={cluster} generation={} outcome={merge_outcome:?}", + refresh_lease.paths().generation + ); + + let compact_lease = refresh_lease.clone(); + let compact_lock = Arc::clone(&publish_lock); + let compact_result = tokio::task::spawn_blocking(move || { + let _publish_guard = compact_lock; + let staging_path = &compact_lease.paths().staging_database; + let mut tree = + BPlusTreeUpdate::::try_new_with_backoff(staging_path).map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to open staging Xtream tree {} for compaction: {error}", + staging_path.display() + )) + })?; + tree.compact().map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to compact staging Xtream tree {}: {error}", + staging_path.display() + )) + }) + }) + .await + .map_err(|error| { + TuliproxError::RepositoryXtream(format!("Compaction task failed to join during {cluster} refresh: {error}")) + }); + match compact_result { + Ok(Ok(())) => {} + Ok(Err(error)) | Err(error) => return Ok(StagedXtreamClusterOutcome::Failed { evaluation, error }), + } + + Ok(StagedXtreamClusterOutcome::Ready(StagedXtreamClusterRefresh { + refresh_lease, + publish_lock, + storage_path, + input_name: Arc::clone(&input.name), + cluster, + raw_groups, + item_count, + evaluation, + })) +} + +async fn publish_staged_xtream_cluster( + app_config: &AppConfig, + staged: StagedXtreamClusterRefresh, +) -> Result<(), TuliproxError> { + let StagedXtreamClusterRefresh { + refresh_lease, + publish_lock, + storage_path, + input_name, + cluster, + raw_groups, + item_count: _, + evaluation: _, + } = staged; + + let database_publish_lease = refresh_lease.clone(); + let database_publish_lock = Arc::clone(&publish_lock); + tokio::task::spawn_blocking(move || { + let _publish_guard = database_publish_lock; + publish_staged_database::( + &database_publish_lease.paths().staging_database, + &database_publish_lease.paths().published_database, + ) + .map_err(|error| { + TuliproxError::RepositoryXtream(format!("Failed to publish staging Xtream database for {cluster}: {error}")) + }) + }) + .await + .map_err(|error| { + TuliproxError::RepositoryXtream(format!("Database publish task failed to join for {cluster}: {error}")) + })??; + + let category_publish_lease = refresh_lease.clone(); + let category_publish_lock = Arc::clone(&publish_lock); + tokio::task::spawn_blocking(move || { + let _publish_guard = category_publish_lock; + publish_staged_file_same_directory( + &category_publish_lease.paths().staging_categories, + &category_publish_lease.paths().published_categories, + ) + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Xtream database for {cluster} was published, but category publication failed: {error}" + )) + }) + }) + .await + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Xtream database for {cluster} was published, but the category publish task failed to join: {error}" + )) + })??; + + if let Err(publish_err) = + crate::publish_raw_group_catalog(&storage_path, &input_name, cluster, raw_groups, &app_config.file_locks).await + { + log::warn!( + "Xtream data for input '{input_name}' cluster {cluster} was published, but its raw group catalog could not be published: {publish_err}" + ); + } + + let cleanup_lease = refresh_lease.clone(); + let cleanup_lock = Arc::clone(&publish_lock); + tokio::task::spawn_blocking(move || { + let _publish_guard = cleanup_lock; + cleanup_lease.cleanup_staging_artifacts() + }) + .await + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Xtream refresh for {cluster} was published, but cleanup task failed to join: {error}" + )) + })? + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Xtream refresh for {cluster} was published, but staging cleanup failed: {error}" + )) + })?; + + drop(publish_lock); + log::debug!( + "Xtream cluster updated successfully: cluster={cluster} generation={}", + refresh_lease.paths().generation + ); + Ok(()) +} + +fn input_cluster_enabled(input: &ConfigInput, cluster: XtreamCluster) -> bool { + match cluster { + XtreamCluster::Live => !input.has_flag(ConfigInputFlags::SkipLive), + XtreamCluster::Video => !input.has_flag(ConfigInputFlags::SkipVod), + XtreamCluster::Series => !input.has_flag(ConfigInputFlags::SkipSeries), + } +} + +async fn published_xtream_cluster_has_items( + app_config: &AppConfig, + storage_path: &Path, + cluster: XtreamCluster, +) -> Result { + let path = xtream_get_file_path(storage_path, cluster); + let lock = app_config.file_locks.read_lock(&path).await; + tokio::task::spawn_blocking(move || { + let _guard = lock; + match BPlusTreeQuery::::try_new(&path) { + Ok(mut query) => query.iter().next().transpose().map(|entry| entry.is_some()).map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to inspect published Xtream cluster {cluster} at {}: {error}", + path.display() + )) + }), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(TuliproxError::RepositoryXtream(format!( + "Failed to open published Xtream cluster {cluster} at {}: {error}", + path.display() + ))), + } + }) + .await + .map_err(|error| TuliproxError::Task(format!("Failed to join Xtream cluster inspection: {error}")))? +} + +pub async fn persist_input_xtream_playlist_clusters_to_disk( + app_config: &Arc, + input: &ConfigInput, + cluster_readers: Vec, +) -> XtreamClusterPublishBatchResult { + persist_input_xtream_playlist_clusters_to_disk_with_operations( + app_config, + input, + cluster_readers, + XtreamClusterStageOperations::default(), + ) + .await +} + +pub(super) async fn persist_input_xtream_playlist_clusters_to_disk_with_operations( + app_config: &Arc, + input: &ConfigInput, + cluster_readers: Vec, + operations: XtreamClusterStageOperations, +) -> XtreamClusterPublishBatchResult { + let mut result = XtreamClusterPublishBatchResult::default(); + let mut staged = Vec::with_capacity(cluster_readers.len()); + for request in cluster_readers { + let cluster = request.cluster; + let outcome = match stage_input_xtream_playlist_cluster_to_disk(app_config, input, request, operations).await { + Ok(outcome) => outcome, + Err(error) => { + result.record_cluster_error(cluster, error); + return result; + } + }; + result.record_evaluation(outcome.evaluation()); + match outcome { + StagedXtreamClusterOutcome::Failed { error, .. } => { + result.record_cluster_error(cluster, error); + return result; + } + ready_or_retained => staged.push(ready_or_retained), + } + } + + let staged_clusters: HashSet = staged + .iter() + .filter_map(|outcome| match outcome { + StagedXtreamClusterOutcome::Ready(refresh) => Some(refresh.cluster), + StagedXtreamClusterOutcome::RetainedPrevious(_) | StagedXtreamClusterOutcome::Failed { .. } => None, + }) + .collect(); + let has_publishable_clusters = !staged_clusters.is_empty(); + let mut has_items = staged + .iter() + .any(|outcome| matches!(outcome, StagedXtreamClusterOutcome::Ready(refresh) if refresh.item_count > 0)); + if has_publishable_clusters && !has_items { + let cfg = app_config.config.load(); + let storage_path = match ensure_input_storage_path(&cfg, &input.name).await { + Ok(storage_path) => storage_path, + Err(error) => { + drop(cfg); + result.errors.push(error); + return result; + } + }; + drop(cfg); + for cluster in XTREAM_CLUSTER { + if input_cluster_enabled(input, cluster) && !staged_clusters.contains(&cluster) { + match published_xtream_cluster_has_items(app_config, &storage_path, cluster).await { + Ok(true) => { + has_items = true; + break; + } + Ok(false) => {} + Err(error) => { + result.errors.push(error); + return result; + } + } + } + } + } + + let all_publishable_clusters_are_forced = has_publishable_clusters + && staged.iter().all( + |outcome| matches!(outcome, StagedXtreamClusterOutcome::Ready(refresh) if refresh.evaluation.force_update.is_some()), + ); + if has_publishable_clusters && !has_items && !all_publishable_clusters_are_forced { + result.errors.push(TuliproxError::RepositoryPlaylist(format!( + "Refusing to publish empty disk-based Xtream playlist for input '{}'; existing data was retained", + input.name + ))); + return result; + } + + for outcome in staged { + match outcome { + StagedXtreamClusterOutcome::Ready(refresh) => { + let evaluation = refresh.evaluation; + let cluster = refresh.cluster; + if let Err(error) = publish_staged_xtream_cluster(app_config, refresh).await { + result.record_cluster_error(cluster, error); + break; + } + result.outcomes.push(if let Some(force_update) = evaluation.force_update { + XtreamClusterPublishOutcome::ForcePublished(force_update) + } else if let Some(quality_acceptance) = evaluation.quality_acceptance { + XtreamClusterPublishOutcome::QualityAccepted(quality_acceptance) + } else { + XtreamClusterPublishOutcome::Published + }); + } + StagedXtreamClusterOutcome::RetainedPrevious(rejection) => { + result.outcomes.push(XtreamClusterPublishOutcome::RetainedPrevious(rejection)); + } + StagedXtreamClusterOutcome::Failed { error, .. } => { + result.errors.push(error); + break; + } + } + } + result +} + +pub async fn persist_input_xtream_playlist_cluster_to_disk( + app_config: &Arc, + input: &ConfigInput, + cluster: XtreamCluster, + quality_threshold: u8, + categories: DynReader, + streams: DynReader, +) -> Result { + let mut result = persist_input_xtream_playlist_clusters_to_disk( + app_config, + input, + vec![XtreamClusterRefreshRequest { + cluster, + quality: XtreamClusterQualityPolicy::Enforce { threshold: quality_threshold }, + categories, + streams, + }], + ) + .await; + if let Some(error) = result.errors.pop() { + return Err(error); + } + result.outcomes.pop().ok_or_else(|| { + TuliproxError::RepositoryXtream(format!( + "Missing Xtream publish outcome for input '{}' cluster {cluster}", + input.name + )) + }) +} diff --git a/backend/repository/src/xtream_repository/paths.rs b/backend/repository/src/xtream_repository/paths.rs new file mode 100644 index 000000000..11dbf7674 --- /dev/null +++ b/backend/repository/src/xtream_repository/paths.rs @@ -0,0 +1,135 @@ +use super::xtream_get_file_path_for_name; +use crate::{ + playlist_backend::{ensure_storage_path, PlaylistBackend, Xtream}, + storage_const, +}; +use shared::{ + error::{string_to_io_error, TuliproxError}, + model::XtreamCluster, +}; +use std::{ + ffi::OsString, + io::Error, + path::{Path, PathBuf}, +}; +use tuliprox_core::model::Config; +use uuid::Uuid; + +#[inline] +pub fn get_collection_path(path: &Path, collection: &str) -> PathBuf { path.join(format!("{collection}.json")) } + +/// Returns the category-collection base name for an [`XtreamCluster`]. +/// +/// Centralizes the per-cluster `cat_live` / `cat_vod` / `cat_series` mapping so the +/// path-deriving call sites read a single property instead of re-matching the cluster. +#[inline] +pub const fn xtream_cluster_category_collection(cluster: XtreamCluster) -> &'static str { + match cluster { + XtreamCluster::Live => storage_const::COL_CAT_LIVE, + XtreamCluster::Video => storage_const::COL_CAT_VOD, + XtreamCluster::Series => storage_const::COL_CAT_SERIES, + } +} + +#[inline] +pub fn get_live_cat_collection_path(path: &Path) -> PathBuf { get_collection_path(path, storage_const::COL_CAT_LIVE) } + +#[inline] +pub fn get_vod_cat_collection_path(path: &Path) -> PathBuf { get_collection_path(path, storage_const::COL_CAT_VOD) } + +#[inline] +pub fn get_series_cat_collection_path(path: &Path) -> PathBuf { + get_collection_path(path, storage_const::COL_CAT_SERIES) +} + +pub(super) fn target_category_lock_path(category_path: &Path) -> PathBuf { + category_path.with_extension("json.target-category.lock") +} + +#[inline] +pub async fn ensure_xtream_storage_path(cfg: &Config, target_name: &str) -> Result { + ensure_storage_path::(cfg, target_name).await +} + +#[inline] +pub fn xtream_get_storage_path(cfg: &Config, target_name: &str) -> Option { + Xtream::storage_path(cfg, target_name) +} + +pub fn xtream_get_file_path(storage_path: &Path, cluster: XtreamCluster) -> PathBuf { + xtream_get_file_path_for_name(storage_path, &cluster.as_str().to_lowercase()) +} + +pub fn xtream_get_collection_path(cfg: &Config, target_name: &str, collection_name: &str) -> Result { + if let Some(path) = xtream_get_storage_path(cfg, target_name) { + let col_path = get_collection_path(&path, collection_name); + if col_path.exists() { + return Ok(col_path); + } + } + Err(string_to_io_error(format!("Can't find collection: {target_name}/{collection_name}"))) +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) struct XtreamRefreshPaths { + pub(super) generation: Uuid, + pub(super) published_database: PathBuf, + pub(super) staging_database: PathBuf, + pub(super) published_categories: PathBuf, + pub(super) staging_categories: PathBuf, +} + +impl XtreamRefreshPaths { + pub(super) fn new(storage_path: &Path, cluster: XtreamCluster) -> Result { + Self::for_generation(storage_path, cluster, Uuid::new_v4()) + } + + pub(super) fn for_generation( + storage_path: &Path, + cluster: XtreamCluster, + generation: Uuid, + ) -> Result { + let published_database = xtream_get_file_path(storage_path, cluster); + let published_categories = get_collection_path(storage_path, xtream_cluster_category_collection(cluster)); + let staging_database = refresh_staging_path(&published_database, generation)?; + // The lock-domain check is repeated by `XtreamRefreshLease::new` with a stricter + // aliasing scan; doing it here too would canonicalize the same paths twice. + Ok(Self { + generation, + staging_database, + staging_categories: refresh_staging_path(&published_categories, generation)?, + published_database, + published_categories, + }) + } +} + +pub(super) fn refresh_staging_path(path: &Path, generation: Uuid) -> Result { + let stem = path + .file_stem() + .ok_or_else(|| TuliproxError::RepositoryXtream(format!("Refresh path has no file stem: {}", path.display())))?; + let extension = path + .extension() + .ok_or_else(|| TuliproxError::RepositoryXtream(format!("Refresh path has no extension: {}", path.display())))?; + let mut filename = OsString::from(stem); + filename.push(".refresh-"); + filename.push(generation.simple().to_string()); + filename.push("."); + filename.push(extension); + Ok(path.with_file_name(filename)) +} + +#[cfg(windows)] +pub(super) fn encode_windows_path(path: &Path) -> std::io::Result> { + use std::os::windows::ffi::OsStrExt; + + let mut encoded = path.as_os_str().encode_wide().collect::>(); + if encoded.contains(&0) { + return Err(Error::new( + std::io::ErrorKind::InvalidInput, + format!("Windows path contains an embedded NUL: {}", path.display()), + )); + } + encoded.push(0); + Ok(encoded) +} diff --git a/backend/repository/src/xtream_repository/publish.rs b/backend/repository/src/xtream_repository/publish.rs new file mode 100644 index 000000000..1260a0bcc --- /dev/null +++ b/backend/repository/src/xtream_repository/publish.rs @@ -0,0 +1,139 @@ +#[cfg(windows)] +use super::{move_target_file_platform, target_publish::TargetFileMoveMode}; +use super::{CategoryEntry, XtreamRefreshLease}; +use shared::error::TuliproxError; +use std::{ + fs::File, + io, + path::{Path, PathBuf}, +}; +#[cfg(not(windows))] +use tuliprox_core::utils::parent_or_dot; +use tuliprox_core::{model::XtreamCategory, utils::require_same_parent_directory}; + +pub(super) fn publish_staged_file_same_directory(staging: &Path, published: &Path) -> io::Result<()> { + require_same_parent_directory(staging, published)?; + let staging_path = tempfile::TempPath::try_from_path(staging).map_err(|error| { + io::Error::new( + error.kind(), + format!("failed to prepare staging file {} for publication: {error}", staging.display()), + ) + })?; + publish_staged_file_platform(staging_path, published) +} + +#[cfg(not(windows))] +fn publish_staged_file_platform(staging_path: tempfile::TempPath, published: &Path) -> io::Result<()> { + publish_staged_file_with_parent_sync(staging_path, published, sync_published_file_parent) +} + +#[cfg(not(windows))] +pub(super) fn publish_staged_file_with_parent_sync( + staging_path: tempfile::TempPath, + published: &Path, + sync_parent: impl FnOnce(&Path) -> io::Result<()>, +) -> io::Result<()> { + staging_path.persist(published).map_err(io::Error::from)?; + sync_parent(published).map_err(|error| { + io::Error::new( + error.kind(), + format!( + "file {} was published, but its parent directory {} could not be synchronized: {error}", + published.display(), + parent_or_dot(published).display() + ), + ) + }) +} + +#[cfg(windows)] +fn publish_staged_file_platform(mut staging_path: tempfile::TempPath, published: &Path) -> io::Result<()> { + move_target_file_platform(staging_path.as_ref(), published, TargetFileMoveMode::ReplaceDestination)?; + + // MoveFileExW consumed the source path. Prevent TempPath from issuing a + // redundant delete for a path that no longer exists. + staging_path.disable_cleanup(true); + Ok(()) +} + +#[cfg(unix)] +pub(super) fn sync_published_file_parent(path: &Path) -> io::Result<()> { File::open(parent_or_dot(path))?.sync_all() } + +/// Every Windows transaction move uses `MOVEFILE_WRITE_THROUGH`; reaching +/// this barrier therefore means all preceding backup or publication moves +/// have completed durably without a second raw rename. +#[cfg(windows)] +#[allow(clippy::unnecessary_wraps)] // keeps the signature identical across platforms +pub(super) fn sync_published_file_parent(_path: &Path) -> io::Result<()> { Ok(()) } + +/// There is no supported directory durability barrier for other targets. +/// Callers report this only after the atomic rename has completed. +#[cfg(all(not(unix), not(windows)))] +pub(super) fn sync_published_file_parent(_path: &Path) -> io::Result<()> { + Err(io::Error::new(io::ErrorKind::Unsupported, "parent-directory synchronization is unsupported on this platform")) +} + +/// Owns the staging category file plus its `flock`, so the lock is released +/// even when a later step (`serde_json::to_writer`, `sync_all`) returns an +/// error. The unlock runs in `Drop` and is logged on failure; closing the +/// underlying `File` releases the OS-level lock either way. +struct LockedCategoryFile { + file: File, + path: PathBuf, +} + +impl LockedCategoryFile { + fn create(path: &Path) -> io::Result { + let file = File::create(path)?; + file.lock()?; + Ok(Self { file, path: path.to_path_buf() }) + } + + pub(super) fn sync_all(&self) -> io::Result<()> { self.file.sync_all() } +} + +impl Drop for LockedCategoryFile { + fn drop(&mut self) { + if let Err(error) = self.file.unlock() { + log::warn!( + "Failed to unlock staging category file {}: {error}; the OS will release it on close", + self.path.display() + ); + } + } +} + +pub(super) async fn save_xtream_categories_to_file( + refresh_lease: XtreamRefreshLease, + categories: &[XtreamCategory], +) -> Result<(), TuliproxError> { + let cat_entries: Vec = categories + .iter() + .map(|c| CategoryEntry { category_id: c.category_id, category_name: c.category_name.clone(), parent_id: 0 }) + .collect(); + + tokio::task::spawn_blocking(move || { + let staging_path = &refresh_lease.paths().staging_categories; + let locked = LockedCategoryFile::create(staging_path).map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to create or lock staging category file {}: {error}", + staging_path.display() + )) + })?; + serde_json::to_writer(&locked.file, &cat_entries).map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to write staging category file {}: {error}", + staging_path.display() + )) + })?; + locked.sync_all().map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to synchronize staging category file {}: {error}", + staging_path.display() + )) + })?; + Ok(()) + }) + .await + .map_err(|e| TuliproxError::RepositoryXtream(format!("Spawn error {e}")))? +} diff --git a/backend/repository/src/xtream_repository/read.rs b/backend/repository/src/xtream_repository/read.rs new file mode 100644 index 000000000..d21d6042b --- /dev/null +++ b/backend/repository/src/xtream_repository/read.rs @@ -0,0 +1,645 @@ +use super::{ + cluster_flag, get_collection_path, merge_preserved_stream_properties, target_category_lock_path, + write_playlists_to_file, xtream_cluster_category_collection, xtream_get_file_path, xtream_get_storage_path, + CategoryEntry, PlaylistStorageState, XtreamClusterEvaluationReport, XtreamClusterPublishBatchResult, +}; +use crate::{ + bplustree::{BPlusTreeError, BPlusTreeQuery}, + error_macros::cant_read_result, + playlist_backend::{iter_raw_playlist, Xtream}, + playlist_scratch::PlaylistScratch, + storage::{get_input_storage_path, get_target_id_mapping_file, get_target_storage_path}, + storage_const, + target_id_mapping::VirtualIdRecord, + xtream_playlist_iterator::XtreamPlaylistJsonIterator, + LockedReceiverStream, +}; +use bytes::Bytes; +use futures::{stream, Stream, StreamExt}; +use indexmap::IndexMap; +use serde::Serialize; +use serde_json::{json, Value}; +use shared::{ + concat_string, + error::{string_to_io_error, TuliproxError}, + model::{ + xtream_const::XTREAM_CLUSTER, ClusterFlags, PlaylistGroup, PlaylistItem, PlaylistItemType, ProviderId, + VirtualId, XtreamCluster, XtreamPlaylistItem, + }, + utils::Internable, +}; +use std::{ + io, + io::{Error, ErrorKind}, + path::{Path, PathBuf}, + sync::Arc, +}; +use tuliprox_core::{ + model::{AppConfig, ConfigInput, ConfigTarget, PlaylistXtreamCategory, ProxyUserCredentials}, + utils::{file_exists_async, json_write_documents_to_file, FileReadGuard}, +}; + +pub(super) fn get_map_item_as_str(map: &serde_json::Map, key: &str) -> Option { + if let Some(value) = map.get(key) { + if let Some(result) = value.as_str() { + return Some(result.to_string()); + } + } + None +} + +pub fn xtream_get_epg_file_path_for_target(path: &Path) -> PathBuf { + path.join(concat_string!("epg.", storage_const::FILE_SUFFIX_DB)) +} + +pub(super) fn xtream_get_file_path_for_name(storage_path: &Path, name: &str) -> PathBuf { + storage_path.join(concat_string!(name, ".", storage_const::FILE_SUFFIX_DB)) +} + +async fn xtream_read_item_for_stream_id( + cfg: &AppConfig, + stream_id: u32, + storage_path: &Path, + cluster: XtreamCluster, +) -> Result { + let xtream_path = xtream_get_file_path(storage_path, cluster); + let file_lock = cfg.file_locks.read_lock(&xtream_path).await; + let xtream_path_clone = xtream_path.clone(); + tokio::task::spawn_blocking(move || -> Result { + let _guard = file_lock; + let mut query = BPlusTreeQuery::::try_new(&xtream_path_clone)?; + match query.query_zero_copy(&stream_id) { + Ok(Some(item)) => Ok(item), + Ok(None) => Err(Error::new(ErrorKind::NotFound, format!("Item {stream_id} not found in {cluster}"))), + Err(err) => Err(Error::other(format!("Query failed for {stream_id} in {cluster}: {err}"))), + } + }) + .await + .map_err(|err| Error::other(format!("Query task failed for {stream_id} in {cluster}: {err}")))? +} + +async fn xtream_read_series_item_for_stream_id( + cfg: &AppConfig, + stream_id: u32, + storage_path: &Path, +) -> Result { + let xtream_path = xtream_get_file_path(storage_path, XtreamCluster::Series); + let file_lock = cfg.file_locks.read_lock(&xtream_path).await; + let xtream_path_clone = xtream_path.clone(); + tokio::task::spawn_blocking(move || -> Result { + let _guard = file_lock; + let mut query = BPlusTreeQuery::::try_new(&xtream_path_clone)?; + match query.query_zero_copy(&stream_id) { + Ok(Some(item)) => Ok(item), + Ok(None) => Err(Error::new(ErrorKind::NotFound, format!("Item {stream_id} not found in series"))), + Err(err) => Err(Error::other(format!("Query failed for {stream_id} in series: {err}"))), + } + }) + .await + .map_err(|err| Error::other(format!("Query task failed for {stream_id} in series: {err}")))? +} + +async fn xtream_get_item_for_stream_id_from_memory( + virtual_id: u32, + playlists: &PlaylistStorageState, + target: &ConfigTarget, + xtream_cluster: Option, +) -> Result, Error> { + if let Some(playlist) = playlists.data.read().await.get(target.name.as_str()) { + return match (playlist.xtream.as_ref(), playlist.id_mapping.as_ref()) { + (Some(xtream_storage), Some(id_mapping)) => { + let mapping = id_mapping + .query(&VirtualId::new(virtual_id)) + .ok_or_else(|| { + string_to_io_error(format!( + "Could not find mapping for target {} and id {}", + target.name, virtual_id + )) + })? + .clone(); + let result = match mapping.item_type { + PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo => Ok(xtream_storage + .series + .query(&mapping.virtual_id.get()) + .ok_or_else(|| string_to_io_error(format!("Failed to read xtream item for id {virtual_id}")))? + .clone()), + PlaylistItemType::Series | PlaylistItemType::LocalSeries => { + log::debug!("In-memory series item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", virtual_id, mapping.parent_virtual_id, mapping.provider_id); + + if let Some(item) = xtream_storage.series.query(&virtual_id) { + Ok(item.clone()) + } else if let Some(item) = xtream_storage.series.query(&mapping.parent_virtual_id.get()) { + let mut xc_item = item.clone(); + xc_item.provider_id = mapping.provider_id; + xc_item.item_type = PlaylistItemType::Series; + xc_item.virtual_id = mapping.virtual_id; + Ok(xc_item) + } else { + Err(string_to_io_error(format!("Failed to read xtream item for id {virtual_id}"))) + } + } + PlaylistItemType::Catchup => { + log::debug!("In-memory catchup item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", virtual_id, mapping.parent_virtual_id, mapping.provider_id); + let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); + let item = match cluster { + XtreamCluster::Live => xtream_storage.live.query(&mapping.parent_virtual_id.get()), + XtreamCluster::Video => xtream_storage.vod.query(&mapping.parent_virtual_id.get()), + XtreamCluster::Series => xtream_storage.series.query(&mapping.parent_virtual_id.get()), + }; + + if let Some(pl_item) = item { + let mut xc_item = pl_item.clone(); + xc_item.provider_id = mapping.provider_id; + xc_item.item_type = PlaylistItemType::Catchup; + xc_item.virtual_id = mapping.virtual_id; + Ok(xc_item) + } else { + Err(string_to_io_error(format!("Failed to read xtream item for id {virtual_id}"))) + } + } + _ => { + let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); + Ok((match cluster { + XtreamCluster::Live => xtream_storage.live.query(&virtual_id), + XtreamCluster::Video => xtream_storage.vod.query(&virtual_id), + XtreamCluster::Series => xtream_storage.series.query(&virtual_id), + }) + .ok_or_else(|| string_to_io_error(format!("Failed to read xtream item for id {virtual_id}")))? + .clone()) + } + }; + + result.map(|xpli| Some((xpli, mapping))) + } + _ => Ok(None), + }; + } + //Err(string_to_io_error(format!("Failed to read xtream item for id {virtual_id}. No entry found."))) + Ok(None) +} + +pub async fn xtream_get_item_for_stream_id( + virtual_id: u32, + app_config: &Arc, + playlists: &PlaylistStorageState, + target: &ConfigTarget, + xtream_cluster: Option, +) -> Result { + if target.use_memory_cache { + if let Ok(Some((playlist_item, _virtual_record))) = + xtream_get_item_for_stream_id_from_memory(virtual_id, playlists, target, xtream_cluster).await + { + return Ok(playlist_item); + } + // fall through to disk lookup on cache miss + } + + let config = app_config.config.load(); + let target_path = get_target_storage_path(&config, target.name.as_str()) + .ok_or_else(|| string_to_io_error(format!("Could not find path for target {}", target.name)))?; + let storage_path = xtream_get_storage_path(&config, target.name.as_str()) + .ok_or_else(|| string_to_io_error(format!("Could not find path for target {} xtream output", target.name)))?; + { + let result = if let Some(cluster) = xtream_cluster { + xtream_read_item_for_stream_id(app_config, virtual_id, &storage_path, cluster).await + } else { + let target_id_mapping_file = get_target_id_mapping_file(&target_path); + let target_name = target.name.clone(); + let file_lock = app_config.file_locks.read_lock(&target_id_mapping_file).await; + let target_id_mapping_file_clone = target_id_mapping_file.clone(); + let mapping = tokio::task::spawn_blocking(move || -> Result { + let _guard = file_lock; + let mut target_id_mapping = + BPlusTreeQuery::::try_new(&target_id_mapping_file_clone).map_err(|err| { + string_to_io_error(format!("Could not load id mapping for target {target_name} err:{err}")) + })?; + match target_id_mapping.query_zero_copy(&virtual_id) { + Ok(Some(record)) => Ok(record), + Ok(None) => Err(string_to_io_error(format!( + "Could not find mapping for target {target_name} and id {virtual_id}" + ))), + Err(err) => Err(string_to_io_error(format!("Query failed for id {virtual_id}: {err}"))), + } + }) + .await + .map_err(|err| string_to_io_error(format!("Mapping query task failed for id {virtual_id}: {err}")))??; + match mapping.item_type { + PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo => { + xtream_read_series_item_for_stream_id(app_config, virtual_id, &storage_path).await + } + PlaylistItemType::Series | PlaylistItemType::LocalSeries => { + log::debug!( + "Disk series item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", + virtual_id, + mapping.parent_virtual_id, + mapping.provider_id + ); + + if let Ok(episode) = + xtream_read_item_for_stream_id(app_config, virtual_id, &storage_path, XtreamCluster::Series) + .await + { + return Ok(episode); + } + + if let Ok(mut item) = xtream_read_series_item_for_stream_id( + app_config, + mapping.parent_virtual_id.get(), + &storage_path, + ) + .await + { + item.provider_id = mapping.provider_id; + item.item_type = PlaylistItemType::Series; + item.virtual_id = mapping.virtual_id; + return Ok(item); + } + + return Err(Error::other(format!("Failed to find episode item with virtual-id {virtual_id}"))); + } + PlaylistItemType::Catchup => { + log::debug!( + "Disk catchup item requested. VirtualID: {}, ParentVirtualID: {}, MappingProviderID: {}", + virtual_id, + mapping.parent_virtual_id, + mapping.provider_id + ); + let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); + let mut item = xtream_read_item_for_stream_id( + app_config, + mapping.parent_virtual_id.get(), + &storage_path, + cluster, + ) + .await?; + item.provider_id = mapping.provider_id; + item.item_type = PlaylistItemType::Catchup; + item.virtual_id = mapping.virtual_id; + Ok(item) + } + _ => { + let cluster = cluster_or_item_type!(xtream_cluster, mapping.item_type); + xtream_read_item_for_stream_id(app_config, virtual_id, &storage_path, cluster).await + } + } + }; + + result + } +} + +pub async fn xtream_load_rewrite_playlist( + cluster: XtreamCluster, + app_config: &Arc, + target: &ConfigTarget, + category_id: Option, + user: &ProxyUserCredentials, +) -> Result { + XtreamPlaylistJsonIterator::new(cluster, app_config, target, category_id, user).await +} + +pub async fn iter_raw_xtream_target_playlist( + app_config: &AppConfig, + target: &ConfigTarget, + cluster: XtreamCluster, +) -> Option>> { + let config = app_config.config.load(); + let storage_path = xtream_get_storage_path(&config, target.name.as_str())?; + let xtream_path = xtream_get_file_path(&storage_path, cluster); + + // Xtream partitions by cluster at the file level, so every item in this + // database already belongs to `cluster` and no per-item filter is needed. + iter_raw_playlist::(app_config, &xtream_path, |_| true).await +} + +pub async fn iter_raw_xtream_input_playlist( + app_config: &AppConfig, + input: &ConfigInput, + cluster: XtreamCluster, +) -> Option>> { + let config = app_config.config.load(); + let storage_dir = &config.storage_dir; + let storage_path = get_input_storage_path(&input.name, storage_dir).await.ok()?; + let xtream_path = xtream_get_file_path(&storage_path, cluster); + + iter_raw_playlist::(app_config, &xtream_path, |_| true).await +} + +/// Counts entries in one active persisted raw input cluster without materializing them. +pub async fn count_input_xtream_cluster( + app_config: &AppConfig, + input: &ConfigInput, + cluster: XtreamCluster, +) -> Result, TuliproxError> { + let storage_dir = app_config.config.load().storage_dir.clone(); + let storage_path = get_input_storage_path(&input.name, &storage_dir).await.map_err(|err| { + TuliproxError::RepositoryXtream(format!("Failed to resolve active input storage for {}: {err}", input.name)) + })?; + let xtream_path = xtream_get_file_path(&storage_path, cluster); + let file_lock = app_config.file_locks.read_lock(&xtream_path).await; + let query_path = xtream_path.clone(); + let count = tokio::task::spawn_blocking(move || -> io::Result> { + let _guard = file_lock; + let mut query = match BPlusTreeQuery::::try_new(&query_path) { + Ok(query) => query, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(err), + }; + query.len().map(Some).map_err(BPlusTreeError::to_io) + }) + .await + .map_err(|err| cant_read_result!(RepositoryXtream, "xtream", &xtream_path, err))? + .map_err(|err| cant_read_result!(RepositoryXtream, "xtream", &xtream_path, err))?; + + Ok(count) +} + +pub fn playlist_iter_to_stream(channels: Option<(FileReadGuard, I)>) -> impl Stream> +where + I: Iterator + 'static, + P: Serialize, +{ + match channels { + Some((_, chans)) => { + // Convert iterator items to Result with minimal allocations + let mapped = chans.map(move |(item, has_next)| match serde_json::to_string(&item) { + Ok(mut content) => { + if has_next { + content.push(','); + } + Ok(Bytes::from(content)) + } + Err(_) => Ok(Bytes::from("")), + }); + stream::iter(mapped).left_stream() + } + None => stream::once(async { Ok(Bytes::from("")) }).right_stream(), + } +} + +pub async fn xtream_get_playlist_categories( + app_config: &AppConfig, + target_name: &str, + cluster: XtreamCluster, +) -> Option> { + let file_path = { + let config = app_config.config.load(); + let storage_path = xtream_get_storage_path(&config, target_name)?; + get_collection_path(&storage_path, xtream_cluster_category_collection(cluster)) + }; + let category_lock_path = target_category_lock_path(&file_path); + let _file_lock = app_config.file_locks.read_lock(&category_lock_path).await; + let content = tokio::fs::read_to_string(&file_path).await.ok()?; + serde_json::from_str::>(&content).ok() +} + +impl XtreamClusterPublishBatchResult { + pub(super) fn record_cluster_error(&mut self, cluster: XtreamCluster, error: TuliproxError) { + self.failed_cluster = Some(cluster); + self.errors.push(error); + } + + pub(super) fn record_evaluation(&mut self, evaluation: XtreamClusterEvaluationReport) { + if let Some(acceptance) = evaluation.quality_acceptance { + self.quality_acceptances.push(acceptance); + } + if let Some(rejection) = evaluation.quality_rejection { + self.quality_rejections.push(rejection); + } + if let Some(force_update) = evaluation.force_update { + self.force_updates.push(force_update); + } + } +} + +pub async fn persist_input_xtream_playlist( + app_config: &Arc, + storage_path: &Path, + playlist: Vec, +) -> (Vec, Option) { + persist_input_xtream_playlist_with_empty_replacements(app_config, storage_path, playlist, ClusterFlags::empty()) + .await +} + +#[allow(clippy::too_many_lines)] +pub async fn persist_input_xtream_playlist_with_empty_replacements( + app_config: &Arc, + storage_path: &Path, + playlist: Vec, + replace_empty_clusters: ClusterFlags, +) -> (Vec, Option) { + let mut errors = Vec::new(); + + let mut fetched_categories = PlaylistScratch::>::new(1_000); + let mut fetched_scratch = PlaylistScratch::>::new(50_000); + let mut stored_scratch = PlaylistScratch::>::new(50_000); + + // load + for cluster in XTREAM_CLUSTER { + let xtream_path = xtream_get_file_path(storage_path, cluster); + if file_exists_async(&xtream_path).await { + let file_lock = app_config.file_locks.read_lock(&xtream_path).await; + let xtream_path = xtream_path.clone(); + let stored_entries = match tokio::task::spawn_blocking(move || { + let _guard = file_lock; + let mut entries = IndexMap::new(); + let mut query = BPlusTreeQuery::::try_new(&xtream_path)?; + for entry in query.iter() { + let (_, doc) = entry?; + entries.insert(doc.provider_id, doc); + } + Ok::<_, std::io::Error>(entries) + }) + .await + { + Ok(Ok(entries)) => Some(entries), + Ok(Err(err)) => { + errors.push(format!("Failed to read stored xtream playlist entries for {cluster}: {err}")); + None + } + Err(err) => { + errors.push(format!("Failed to load stored xtream playlist entries for {cluster}: {err}")); + None + } + }; + + if let Some(entries) = stored_entries { + *stored_scratch.get_mut(cluster) = entries; + } + } + } + + if !errors.is_empty() { + return (playlist, Some(TuliproxError::RepositoryXtream(errors.join("\n")))); + } + + let mut groups = IndexMap::new(); + + for mut plg in playlist { + if !&plg.channels.is_empty() { + fetched_categories.get_mut(plg.xtream_cluster).push(json!(CategoryEntry { + category_id: plg.id, + category_name: plg.title.clone(), + parent_id: 0 + })); + + let channels = std::mem::take(&mut plg.channels); + for mut pli in channels { + let stored_col = stored_scratch.get_mut(plg.xtream_cluster); + let fetched_col = fetched_scratch.get_mut(plg.xtream_cluster); + + if let Ok(provider_id) = pli.header.id.parse::() { + if let Some(stored_pli) = stored_col.get_mut(&provider_id) { + if let (Some(new_stream_props), Some(old_stream_props)) = + (&mut pli.header.additional_properties, stored_pli.additional_properties.take()) + { + merge_preserved_stream_properties(new_stream_props, &old_stream_props); + } + } + } + fetched_col.push(pli); + } + groups.insert((plg.xtream_cluster, plg.id), plg); + } + } + + let mut processed_scratch = PlaylistScratch::>::new(0); + for xc in XTREAM_CLUSTER { + processed_scratch.set( + xc, + if !replace_empty_clusters.contains(cluster_flag(xc)) + && !stored_scratch.is_empty(xc) + && fetched_scratch.is_empty(xc) + { + stored_scratch.take(xc).iter().map(|(_, item)| PlaylistItem::from(item)).collect::>() + } else { + fetched_scratch.take(xc) + }, + ); + } + drop(stored_scratch); + drop(fetched_scratch); + + let root_path = storage_path.to_path_buf(); + let app_cfg = app_config.clone(); + for cluster in XTREAM_CLUSTER { + let col_path = get_collection_path(&root_path, xtream_cluster_category_collection(cluster)); + let data = fetched_categories.get_mut(cluster); + // if there is no data save only if no file exists! Prevent data loss from failed download attempt + if !data.is_empty() + || replace_empty_clusters.contains(cluster_flag(cluster)) + || !file_exists_async(&col_path).await + { + let lock = app_cfg.file_locks.write_lock(&col_path).await; + if let Err(err) = json_write_documents_to_file(&col_path, data).await { + errors.push(format!("Persisting collection failed: {}: {err}", col_path.display())); + } + drop(lock); + } + } + + for cluster in XTREAM_CLUSTER { + let col = processed_scratch.take(cluster); + + // persist playlist + if let Err(err) = write_playlists_to_file( + app_config, + storage_path, + false, + |item| ProviderId::new(item.provider_id), + vec![(cluster, col.iter().map(Into::into).collect::>())], + replace_empty_clusters, + ) + .await + { + errors.push(format!("Persisting collection failed:{err}")); + } + + for item in col { + let group_key = (item.header.xtream_cluster, item.header.category_id); + groups + .entry(group_key) + .or_insert_with(|| PlaylistGroup { + id: item.header.category_id, + title: item.header.group.clone(), + channels: Vec::new(), + xtream_cluster: item.header.xtream_cluster, + }) + .channels + .push(item); + } + } + + let result = groups.into_values().collect(); + + let err = if errors.is_empty() { None } else { Some(TuliproxError::RepositoryXtream(errors.join("\n"))) }; + + (result, err) +} + +pub async fn load_input_xtream_playlist( + app_config: &Arc, + storage_path: &Path, + clusters: &[XtreamCluster], +) -> Result, TuliproxError> { + let mut groups: IndexMap<(XtreamCluster, u32), PlaylistGroup> = IndexMap::new(); + + for &cluster in clusters { + let xtream_path = xtream_get_file_path(storage_path, cluster); + if xtream_path.exists() { + let cat_col_name = xtream_cluster_category_collection(cluster); + let cat_path = get_collection_path(storage_path, cat_col_name); + + if cat_path.exists() { + if let Ok(content) = tokio::fs::read_to_string(&cat_path).await { + if let Ok(cats) = serde_json::from_str::>(&content) { + for cat in cats { + groups.insert( + (cluster, cat.category_id), + PlaylistGroup { + id: cat.category_id, + title: cat.category_name, + channels: Vec::new(), + xtream_cluster: cluster, + }, + ); + } + } + } + } + + // Load Items + let file_lock = app_config.file_locks.read_lock(&xtream_path).await; + let xtream_path_err = xtream_path.clone(); + let items = tokio::task::spawn_blocking(move || -> Result, TuliproxError> { + let _guard = file_lock; + let mut items = Vec::new(); + let mut query = BPlusTreeQuery::::try_new(&xtream_path) + .map_err(|error| TuliproxError::RepositoryXtream(error.to_string()))?; + for entry in query.iter() { + let (_, item) = entry.map_err(|error| TuliproxError::RepositoryXtream(error.to_string()))?; + items.push(item); + } + Ok(items) + }) + .await + .map_err(|err| cant_read_result!(RepositoryXtream, "xtream", &xtream_path_err, err))??; + + for item in items { + let cat_id = item.category_id; + groups + .entry((cluster, cat_id)) + .or_insert_with(|| PlaylistGroup { + id: cat_id, + title: "Unknown".intern(), + channels: Vec::new(), + xtream_cluster: cluster, + }) + .channels + .push(PlaylistItem::from(&item)); + } + } + } + + Ok(groups.into_values().collect()) +} diff --git a/backend/repository/src/xtream_repository/target_publish.rs b/backend/repository/src/xtream_repository/target_publish.rs new file mode 100644 index 000000000..2fffa74ac --- /dev/null +++ b/backend/repository/src/xtream_repository/target_publish.rs @@ -0,0 +1,795 @@ +#[cfg(all(not(unix), not(windows)))] +use super::sync_published_file_parent; +#[cfg(unix)] +use super::sync_published_file_parent; +#[cfg(test)] +use super::TargetEmptyReplacementFailure; +use super::{ + cluster_flag, ensure_xtream_storage_path, get_collection_path, get_live_cat_collection_path, get_map_item_as_str, + get_series_cat_collection_path, get_vod_cat_collection_path, target_category_lock_path, + xtream_cluster_category_collection, xtream_get_file_path, CategoryEntry, CategoryKey, +}; +#[cfg(windows)] +use super::{encode_windows_path, sync_published_file_parent}; +use crate::{ + bplustree::{get_file_path_for_db_index, BPlusTree, BPlusTreeStagingArtifacts, BPlusTreeUpdate}, + error_macros::cant_write_result, + playlist_backend::PlaylistKey, + storage_const, +}; +use indexmap::IndexMap; +use log::warn; +use serde_json::Value; +use shared::{ + error::TuliproxError, + model::{ClusterFlags, PlaylistGroup, XtreamCluster, XtreamPlaylistItem}, + utils::{get_u32_from_serde_value, Internable}, +}; +#[cfg(unix)] +use std::fs; +use std::{ + collections::HashMap, + fs::File, + io, + path::{Path, PathBuf}, + sync::Arc, +}; +use tuliprox_core::{ + model::{AppConfig, ConfigTarget}, + utils::{ + file_exists_async, file_reader, json_write_documents_to_file, remove_file_if_exists, + require_same_parent_directory, + }, +}; +use uuid::Uuid; + +/// Persist `collections`, keyed by whatever `key_of` extracts. +/// +/// The key space is a type parameter rather than a runtime tag. These stores are +/// keyed by [`VirtualId`] on the target path and by [`ProviderId`] on the input +/// path -- the same file layout and value type, two different id spaces -- and a +/// `StorageKey` enum matched per item used to be the only thing recording which. +/// Both keys are `#[serde(transparent)]` over `u32`, so the on-disk encoding is +/// unchanged either way (see the codec test in `backend/btree`). +pub(super) async fn write_playlists_to_file( + app_config: &Arc, + storage_path: &Path, + with_index: bool, + key_of: F, + collections: Vec<(XtreamCluster, Vec)>, + replace_empty_clusters: ClusterFlags, +) -> Result<(), TuliproxError> +where + K: PlaylistKey, + F: Fn(&XtreamPlaylistItem) -> K + Copy + Send + 'static, +{ + for (cluster, playlist) in collections { + if playlist.is_empty() && !replace_empty_clusters.contains(cluster_flag(cluster)) { + continue; + } + let xtream_path = xtream_get_file_path(storage_path, cluster); + + // Acquire FileLockManager lock (async, in-process coordination) + let file_lock = app_config.file_locks.write_lock(&xtream_path).await; + + // Move all B+Tree building and I/O to spawn_blocking + // We take ownership of `playlist` here (no cloning needed) + let path_clone = xtream_path.clone(); + tokio::task::spawn_blocking(move || -> Result<(), std::io::Error> { + let _guard = file_lock; + let mut tree = BPlusTree::::new(); + for item in playlist { + let key = key_of(&item); + tree.insert(key, item); + } + if with_index { + tree.store_with_index(&path_clone, |pli| pli.source_ordinal)?; + } else { + tree.store(&path_clone)?; + } + Ok(()) + }) + .await + .map_err(|e| TuliproxError::RepositoryXtream(format!("Blocking task failed: {e}")))? + .map_err(|err| cant_write_result!(RepositoryXtream, "xtream", &xtream_path, err))?; + } + Ok(()) +} + +#[derive(Debug, Clone, Default)] +pub(super) enum TargetEmptyReplacementMode { + #[default] + Persist, + #[cfg(test)] + FailAt(TargetEmptyReplacementFailure), + #[cfg(test)] + PauseDuringPublication(TargetEmptyPublicationHook), +} + +#[cfg(test)] +#[derive(Debug, Clone)] +pub(super) struct TargetEmptyPublicationHook { + pub(super) backup_window_entered: Arc, + pub(super) resume_publication: Arc, +} + +#[cfg(test)] +impl TargetEmptyPublicationHook { + pub(super) fn new() -> Self { + Self { + backup_window_entered: Arc::new(std::sync::Barrier::new(2)), + resume_publication: Arc::new(std::sync::Barrier::new(2)), + } + } +} + +struct TargetEmptyClusterPaths { + published_database: PathBuf, + published_index: PathBuf, + published_categories: PathBuf, + staging_database: PathBuf, + staging_index: PathBuf, + staging_categories: PathBuf, +} + +impl TargetEmptyClusterPaths { + pub(super) fn new(storage_path: &Path, cluster: XtreamCluster) -> Self { + let token = Uuid::new_v4().simple(); + let cluster_name = cluster.as_str().to_lowercase(); + let published_database = xtream_get_file_path(storage_path, cluster); + let staging_database = storage_path.join(format!(".{cluster_name}.force-empty-{token}.db")); + Self { + published_index: get_file_path_for_db_index(&published_database), + published_categories: get_collection_path(storage_path, xtream_cluster_category_collection(cluster)), + staging_index: get_file_path_for_db_index(&staging_database), + staging_categories: storage_path.join(format!(".{cluster_name}.force-empty-{token}.json")), + published_database, + staging_database, + } + } +} + +struct TargetFileReplacement { + published: PathBuf, + staging: PathBuf, + backup: PathBuf, + previous_moved: bool, + replacement_published: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum TargetFileMoveMode { + PreserveDestination, + ReplaceDestination, +} + +#[cfg(unix)] +pub(super) fn move_target_file_platform(source: &Path, destination: &Path, mode: TargetFileMoveMode) -> io::Result<()> { + require_same_parent_directory(source, destination)?; + if mode == TargetFileMoveMode::PreserveDestination && destination.exists() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!("target transaction destination already exists: {}", destination.display()), + )); + } + fs::rename(source, destination) +} + +#[cfg(windows)] +pub(super) fn move_target_file_platform(source: &Path, destination: &Path, mode: TargetFileMoveMode) -> io::Result<()> { + use windows_sys::Win32::Storage::FileSystem::{MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH}; + + require_same_parent_directory(source, destination)?; + let source_encoded = encode_windows_path(source)?; + let destination_encoded = encode_windows_path(destination)?; + let flags = MOVEFILE_WRITE_THROUGH + | if mode == TargetFileMoveMode::ReplaceDestination { MOVEFILE_REPLACE_EXISTING } else { 0 }; + + // SAFETY: both buffers are live, immutable, and NUL-terminated for the + // duration of the call. The same-directory check prevents a cross-volume + // move from degrading into a copy. + let result = unsafe { MoveFileExW(source_encoded.as_ptr(), destination_encoded.as_ptr(), flags) }; + if result == 0 { + let error = io::Error::last_os_error(); + return Err(io::Error::new( + error.kind(), + format!( + "failed to move target transaction file {} to {} with Windows write-through semantics: {error}", + source.display(), + destination.display() + ), + )); + } + Ok(()) +} + +#[cfg(all(not(unix), not(windows)))] +pub(super) fn move_target_file_platform( + source: &Path, + destination: &Path, + _mode: TargetFileMoveMode, +) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + format!( + "durable target transaction moves are unsupported on this platform: {} -> {}", + source.display(), + destination.display() + ), + )) +} + +impl TargetFileReplacement { + pub(super) fn new(published: &Path, staging: &Path, token: uuid::fmt::Simple) -> Self { + let filename = + published.file_name().map_or_else(|| "target-artifact".into(), |name| name.to_string_lossy().into_owned()); + Self { + published: published.to_path_buf(), + staging: staging.to_path_buf(), + backup: published.with_file_name(format!(".{filename}.force-empty-backup-{token}")), + previous_moved: false, + replacement_published: false, + } + } +} + +fn cleanup_target_empty_staging( + staging_artifacts: &BPlusTreeStagingArtifacts, + staging_categories: &Path, +) -> io::Result<()> { + let database_result = staging_artifacts.remove_owned_staging_artifacts(); + let category_result = remove_file_if_exists(staging_categories); + match (database_result, category_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(database_error), Err(category_error)) => Err(io::Error::new( + database_error.kind(), + format!("{database_error}; category staging cleanup also failed: {category_error}"), + )), + } +} + +fn rollback_target_file_replacements(replacements: &mut [TargetFileReplacement]) -> io::Result<()> { + let mut errors = Vec::new(); + for replacement in replacements.iter_mut().rev() { + if replacement.previous_moved { + if let Err(error) = move_target_file_platform( + &replacement.backup, + &replacement.published, + TargetFileMoveMode::ReplaceDestination, + ) { + errors.push(format!( + "failed to restore {} from {}: {error}", + replacement.published.display(), + replacement.backup.display() + )); + } + } else if replacement.replacement_published { + if let Err(error) = move_target_file_platform( + &replacement.published, + &replacement.backup, + TargetFileMoveMode::ReplaceDestination, + ) { + errors.push(format!( + "failed to withdraw replacement {} during rollback: {error}", + replacement.published.display() + )); + } else if let Err(error) = remove_file_if_exists(&replacement.backup) { + errors + .push(format!("failed to remove withdrawn replacement {}: {error}", replacement.backup.display())); + } + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(io::Error::other(errors.join("; "))) + } +} + +fn publish_target_file_replacements( + paths: &TargetEmptyClusterPaths, + mode: &TargetEmptyReplacementMode, +) -> io::Result<()> { + #[cfg(not(test))] + let _ = &mode; + let staging_artifacts = BPlusTreeStagingArtifacts::new(&paths.published_database, &paths.staging_database)?; + let token = Uuid::new_v4().simple(); + let mut replacements = vec![ + TargetFileReplacement::new(&paths.published_database, &paths.staging_database, token), + TargetFileReplacement::new(&paths.published_index, &paths.staging_index, token), + TargetFileReplacement::new(&paths.published_categories, &paths.staging_categories, token), + ]; + + let publication = (|| -> io::Result<()> { + for replacement in &replacements { + require_same_parent_directory(&replacement.staging, &replacement.published)?; + if !replacement.staging.is_file() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + format!("missing prepared target artifact {}", replacement.staging.display()), + )); + } + } + + for replacement in &mut replacements { + if replacement.published.exists() { + move_target_file_platform( + &replacement.published, + &replacement.backup, + TargetFileMoveMode::PreserveDestination, + )?; + replacement.previous_moved = true; + } + } + + #[cfg(test)] + if let TargetEmptyReplacementMode::PauseDuringPublication(hook) = mode { + hook.backup_window_entered.wait(); + hook.resume_publication.wait(); + } + + for (index, replacement) in replacements.iter_mut().enumerate() { + move_target_file_platform( + &replacement.staging, + &replacement.published, + TargetFileMoveMode::ReplaceDestination, + )?; + replacement.replacement_published = true; + #[cfg(test)] + if matches!(mode, TargetEmptyReplacementMode::FailAt(TargetEmptyReplacementFailure::Publication)) + && index == 0 + { + return Err(io::Error::other("injected target empty-replacement publication failure")); + } + #[cfg(not(test))] + let _ = index; + } + sync_published_file_parent(&paths.published_database)?; + cleanup_target_empty_staging(&staging_artifacts, &paths.staging_categories) + })(); + + if let Err(publication_error) = publication { + let rollback_result = rollback_target_file_replacements(&mut replacements); + let durability_result = sync_published_file_parent(&paths.published_database); + return match (rollback_result, durability_result) { + (Ok(()), Ok(())) => Err(publication_error), + (rollback, durability) => Err(io::Error::new( + publication_error.kind(), + format!( + "{publication_error}; rollback result: {}; rollback directory sync result: {}", + rollback.map_or_else(|error| error.to_string(), |()| "ok".to_string()), + durability.map_or_else(|error| error.to_string(), |()| "ok".to_string()) + ), + )), + }; + } + + for replacement in replacements { + if replacement.previous_moved { + if let Err(error) = remove_file_if_exists(&replacement.backup) { + warn!( + "Target empty replacement was published, but backup cleanup failed for {}: {error}", + replacement.backup.display() + ); + } + } + } + Ok(()) +} + +async fn replace_target_xtream_cluster_with_empty( + app_config: &Arc, + storage_path: &Path, + cluster: XtreamCluster, + mode: TargetEmptyReplacementMode, +) -> Result<(), TuliproxError> { + #[cfg(not(test))] + let _ = &mode; + let paths = TargetEmptyClusterPaths::new(storage_path, cluster); + let staging_artifacts = BPlusTreeStagingArtifacts::new(&paths.published_database, &paths.staging_database) + .map_err(|error| { + TuliproxError::RepositoryXtream(format!("Failed to prepare empty {cluster} target: {error}")) + })?; + + let operation = async { + #[cfg(test)] + if matches!(&mode, TargetEmptyReplacementMode::FailAt(TargetEmptyReplacementFailure::CategoryPersistence)) { + return Err(TuliproxError::RepositoryXtream("injected target category persistence failure".to_string())); + } + json_write_documents_to_file(&paths.staging_categories, &Vec::::new()).await.map_err( + |error| { + TuliproxError::RepositoryXtream(format!( + "Failed to prepare empty {cluster} target categories {}: {error}", + paths.staging_categories.display() + )) + }, + )?; + let staging_category_file = tokio::fs::File::open(&paths.staging_categories).await.map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to reopen empty {cluster} target categories {}: {error}", + paths.staging_categories.display() + )) + })?; + staging_category_file.sync_all().await.map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to synchronize empty {cluster} target categories {}: {error}", + paths.staging_categories.display() + )) + })?; + + #[cfg(test)] + if matches!(&mode, TargetEmptyReplacementMode::FailAt(TargetEmptyReplacementFailure::BTreePersistence)) { + return Err(TuliproxError::RepositoryXtream("injected target BTree/index persistence failure".to_string())); + } + let staging_database = paths.staging_database.clone(); + tokio::task::spawn_blocking(move || { + BPlusTree::::new().store_with_index(&staging_database, |item| item.source_ordinal) + }) + .await + .map_err(|error| TuliproxError::RepositoryXtream(format!("Empty target BTree task failed: {error}")))? + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to prepare empty {cluster} target BTree/index {}: {error}", + paths.staging_database.display() + )) + })?; + + let category_lock_path = target_category_lock_path(&paths.published_categories); + let mut lock_paths = [&paths.published_database, &category_lock_path]; + lock_paths.sort_unstable(); + let mut file_locks = Vec::with_capacity(lock_paths.len()); + for path in lock_paths { + file_locks.push(app_config.file_locks.write_lock(path).await); + } + let publication_paths = TargetEmptyClusterPaths { + published_database: paths.published_database.clone(), + published_index: paths.published_index.clone(), + published_categories: paths.published_categories.clone(), + staging_database: paths.staging_database.clone(), + staging_index: paths.staging_index.clone(), + staging_categories: paths.staging_categories.clone(), + }; + let publication_mode = mode.clone(); + tokio::task::spawn_blocking(move || { + let result = publish_target_file_replacements(&publication_paths, &publication_mode); + drop(file_locks); + result + }) + .await + .map_err(|error| TuliproxError::RepositoryXtream(format!("Empty target publish task failed: {error}")))? + .map_err(|error| { + TuliproxError::RepositoryXtream(format!( + "Failed to publish empty {cluster} target cluster atomically: {error}" + )) + }) + } + .await; + + let cleanup = cleanup_target_empty_staging(&staging_artifacts, &paths.staging_categories); + match (operation, cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) => Err(error), + (Ok(()), Err(cleanup_error)) => Err(TuliproxError::RepositoryXtream(format!( + "Empty {cluster} target cluster was published, but staging cleanup failed: {cleanup_error}" + ))), + (Err(error), Err(cleanup_error)) => { + Err(TuliproxError::RepositoryXtream(format!("{error}; staging cleanup also failed: {cleanup_error}"))) + } + } +} + +pub async fn write_playlist_item_update( + app_config: &Arc, + target_name: &str, + pli: &XtreamPlaylistItem, +) -> Result<(), TuliproxError> { + let storage_path = { + let config = app_config.config.load(); + ensure_xtream_storage_path(&config, target_name).await? + }; + let xtream_path = xtream_get_file_path(&storage_path, pli.xtream_cluster); + + if !file_exists_async(&xtream_path).await { + return Err(TuliproxError::RepositoryXtream(format!( + "BPlusTree file not found for update {}", + xtream_path.display() + ))); + } + + // Prepare encoded payload before opening the writer lock. + let prepared_items = + BPlusTreeUpdate::::prepare_upsert_batch(&[(&pli.virtual_id.get(), pli)]) + .map_err(|e| TuliproxError::RepositoryXtream(format!("Failed to serialize value: {e}")))?; + + // Keep FileLockManager lock for cross-operation coordination (e.g. swap + update). + let file_lock = app_config.file_locks.write_lock(&xtream_path).await; + + let xtream_path_clone = xtream_path.clone(); + tokio::task::spawn_blocking(move || -> Result<(), std::io::Error> { + let _guard = file_lock; + let mut tree = BPlusTreeUpdate::::try_new_with_backoff(&xtream_path_clone)?; + tree.upsert_batch_encoded(prepared_items)?; + Ok(()) + }) + .await + .map_err(|e| TuliproxError::RepositoryXtream(format!("Blocking task failed: {e}")))? + .map_err(|err| cant_write_result!(RepositoryXtream, "xtream", &xtream_path, err))?; + + Ok(()) +} + +pub async fn write_playlist_batch_item_upsert( + app_config: &Arc, + target_name: &str, + xtream_cluster: XtreamCluster, + pli_list: &[XtreamPlaylistItem], +) -> Result<(), TuliproxError> { + if pli_list.is_empty() { + return Ok(()); + } + + let storage_path = { + let config = app_config.config.load(); + ensure_xtream_storage_path(&config, target_name).await? + }; + let xtream_path = xtream_get_file_path(&storage_path, xtream_cluster); + + if !file_exists_async(&xtream_path).await { + return Err(TuliproxError::RepositoryXtream(format!( + "BPlusTree file not found for upsert {}", + xtream_path.display() + ))); + } + + // Prepare encoded payload before opening the writer lock. + let virtual_ids: Vec = pli_list.iter().map(|pli| pli.virtual_id.get()).collect(); + let batch_refs: Vec<(&u32, &XtreamPlaylistItem)> = virtual_ids.iter().zip(pli_list.iter()).collect(); + let prepared_items = BPlusTreeUpdate::::prepare_upsert_batch(&batch_refs) + .map_err(|e| TuliproxError::RepositoryXtream(format!("Failed to serialize value: {e}")))?; + + // Keep FileLockManager lock for cross-operation coordination (e.g. swap + update). + let file_lock = app_config.file_locks.write_lock(&xtream_path).await; + + let xtream_path_clone = xtream_path.clone(); + tokio::task::spawn_blocking(move || -> Result<(), std::io::Error> { + let _guard = file_lock; + let mut tree = BPlusTreeUpdate::::try_new_with_backoff(&xtream_path_clone)?; + tree.upsert_batch_encoded(prepared_items)?; + Ok(()) + }) + .await + .map_err(|e| TuliproxError::RepositoryXtream(format!("Blocking task failed: {e}")))? + .map_err(|err| cant_write_result!(RepositoryXtream, "xtream", &xtream_path, err))?; + + Ok(()) +} + +// Because interner is not thread safe we can't use it currently for interning. +// We leave the argument for later optimizations. +async fn load_old_category_ids(path: &Path) -> (u32, HashMap) { + let old_path = path.to_path_buf(); + tokio::task::spawn_blocking(move || { + let mut result: HashMap = HashMap::new(); + let mut max_id: u32 = 0; + for (cluster, cat) in [ + (XtreamCluster::Live, storage_const::COL_CAT_LIVE), + (XtreamCluster::Video, storage_const::COL_CAT_VOD), + (XtreamCluster::Series, storage_const::COL_CAT_SERIES), + ] { + let col_path = get_collection_path(&old_path, cat); + if col_path.exists() { + if let Ok(file) = File::open(&col_path) { + let reader = file_reader(file); + match serde_json::from_reader(reader) { + Ok(value) => { + if let Value::Array(list) = value { + for entry in list { + if let Some(category_id) = entry + .get(tuliprox_core::model::XC_TAG_CATEGORY_ID) + .and_then(get_u32_from_serde_value) + { + if let Value::Object(item) = entry { + if let Some(category_name) = + get_map_item_as_str(&item, tuliprox_core::model::XC_TAG_CATEGORY_NAME) + { + result.insert( + (cluster, /*interner.*/ category_name.intern()), + category_id, + ); + max_id = max_id.max(category_id); + } + } + } + } + } + } + Err(err) => { + log::warn!("Failed to parse category file {}: {err}", col_path.display()); + } + } + } + } + } + (max_id, result) + }) + .await + .unwrap_or_else(|_| (0, HashMap::new())) +} + +pub async fn xtream_write_playlist( + app_cfg: &Arc, + target: &ConfigTarget, + playlist: &mut [PlaylistGroup], + replace_empty_clusters: ClusterFlags, +) -> Result<(), TuliproxError> { + xtream_write_playlist_with_mode( + app_cfg, + target, + playlist, + replace_empty_clusters, + TargetEmptyReplacementMode::Persist, + ) + .await +} + +#[cfg(test)] +pub(crate) async fn xtream_write_playlist_with_injected_empty_replacement_failure( + app_cfg: &Arc, + target: &ConfigTarget, + playlist: &mut [PlaylistGroup], + replace_empty_clusters: ClusterFlags, + failure: TargetEmptyReplacementFailure, +) -> Result<(), TuliproxError> { + xtream_write_playlist_with_mode( + app_cfg, + target, + playlist, + replace_empty_clusters, + TargetEmptyReplacementMode::FailAt(failure), + ) + .await +} + +pub(super) async fn xtream_write_playlist_with_mode( + app_cfg: &Arc, + target: &ConfigTarget, + playlist: &mut [PlaylistGroup], + replace_empty_clusters: ClusterFlags, + empty_replacement_mode: TargetEmptyReplacementMode, +) -> Result<(), TuliproxError> { + let path = { + let config = app_cfg.config.load(); + ensure_xtream_storage_path(&config, target.name.as_str()).await? + }; + let mut errors = Vec::new(); + let mut cat_live_col = Vec::with_capacity(1_000); + let mut cat_series_col = Vec::with_capacity(1_000); + let mut cat_vod_col = Vec::with_capacity(1_000); + let mut live_col = Vec::with_capacity(50_000); + let mut series_col = Vec::with_capacity(50_000); + let mut vod_col = Vec::with_capacity(50_000); + + let categories = create_categories(playlist, &path).await; + { + for (xtream_cluster, category) in categories { + match xtream_cluster { + XtreamCluster::Live => &mut cat_live_col, + XtreamCluster::Series => &mut cat_series_col, + XtreamCluster::Video => &mut cat_vod_col, + } + .push(category); + } + } + + for plg in playlist.iter_mut() { + if plg.channels.is_empty() { + continue; + } + + for pli in &plg.channels { + let col = match pli.header.xtream_cluster { + XtreamCluster::Live => &mut live_col, + XtreamCluster::Series => &mut series_col, + XtreamCluster::Video => &mut vod_col, + }; + col.push(pli); + } + } + + let root_path = path.clone(); + let app_config = app_cfg.clone(); + for (cluster, col_path, data) in [ + (XtreamCluster::Live, get_live_cat_collection_path(&root_path), &cat_live_col), + (XtreamCluster::Video, get_vod_cat_collection_path(&root_path), &cat_vod_col), + (XtreamCluster::Series, get_series_cat_collection_path(&root_path), &cat_series_col), + ] { + if data.is_empty() { + if replace_empty_clusters.contains(cluster_flag(cluster)) { + continue; + } + if file_exists_async(&col_path).await { + continue; + } + } + let category_lock_path = target_category_lock_path(&col_path); + let lock = app_config.file_locks.write_lock(&category_lock_path).await; + match json_write_documents_to_file(&col_path, data).await { + Ok(()) => {} + Err(err) => { + errors.push(format!("Persisting collection failed: {}: {err}", col_path.display())); + } + } + drop(lock); + } + + // Process each cluster sequentially to avoid holding multiple fully + // materialized Xtream collections in memory at the same time. + for (cluster, col) in + [(XtreamCluster::Live, &live_col), (XtreamCluster::Video, &vod_col), (XtreamCluster::Series, &series_col)] + { + if col.is_empty() && replace_empty_clusters.contains(cluster_flag(cluster)) { + if let Err(error) = + replace_target_xtream_cluster_with_empty(app_cfg, &path, cluster, empty_replacement_mode.clone()).await + { + errors.push(format!("Persisting empty {cluster} target cluster failed: {error}")); + } + continue; + } + let data = col.iter().map(|item| XtreamPlaylistItem::from(&**item)).collect::>(); + if let Err(err) = write_playlists_to_file( + app_cfg, + &path, + true, + |item| item.virtual_id, + vec![(cluster, data)], + replace_empty_clusters, + ) + .await + { + errors.push(format!("Persisting collection failed:{err}")); + } + } + + if !errors.is_empty() { + return Err(TuliproxError::Config(errors.join("\n"))); + } + + Ok(()) +} + +async fn create_categories(playlist: &mut [PlaylistGroup], path: &Path) -> Vec<(XtreamCluster, CategoryEntry)> { + // preserve category_ids + let (max_cat_id, existing_cat_ids) = load_old_category_ids(path).await; + let mut cat_id_counter = max_cat_id; + + let mut new_categories: IndexMap = IndexMap::new(); + + for plg in playlist.iter_mut() { + if plg.channels.is_empty() { + continue; + } + + for channel in &mut plg.channels { + let cluster = channel.header.xtream_cluster; + let group = &channel.header.group; + + let entry = new_categories.entry((cluster, group.clone())).or_insert_with(|| { + let cat_id = existing_cat_ids.get(&(cluster, group.clone())).copied().unwrap_or_else(|| { + cat_id_counter += 1; + cat_id_counter + }); + + CategoryEntry { category_id: cat_id, category_name: group.clone(), parent_id: 0 } + }); + + channel.header.category_id = entry.category_id; + } + } + + new_categories + .into_iter() + .map(|((cluster, _group), value)| (cluster, value)) + .collect::>() +} diff --git a/backend/repository/src/xtream_repository/tests.rs b/backend/repository/src/xtream_repository/tests.rs new file mode 100644 index 000000000..11178e87d --- /dev/null +++ b/backend/repository/src/xtream_repository/tests.rs @@ -0,0 +1,29 @@ +#[cfg(unix)] +use super::refresh_staging_path; +use super::{ + count_input_xtream_cluster, get_collection_path, load_input_xtream_playlist, merge_preserved_stream_properties, + needs_update_info_details, persist_input_xtream_playlist, persist_input_xtream_playlist_cluster_to_disk, + persist_input_xtream_playlist_clusters_to_disk, persist_input_xtream_playlist_clusters_to_disk_with_operations, + persists_input_series_info, preserve_details_input_xtream_playlist_cluster_to_disk, + preserve_details_with_injected_operation_failure, publish_staged_file_same_directory, target_category_lock_path, + xtream_cluster_category_collection, xtream_get_playlist_categories, xtream_write_playlist, + xtream_write_playlist_with_injected_empty_replacement_failure, xtream_write_playlist_with_mode, + DetailPreservationOperation, PreserveDetailsOutcome, TargetEmptyPublicationHook, TargetEmptyReplacementFailure, + TargetEmptyReplacementMode, XtreamClusterPublishOutcome, XtreamClusterQualityPolicy, XtreamClusterRefreshRequest, + XtreamClusterStageOperations, XtreamRefreshLease, XtreamRefreshPaths, +}; + +mod behavior; +mod lifecycle; +mod persistence; +mod query; +mod support; + +use self::support::{ + fixed_refresh_paths, make_live_item, target_writer_config, target_writer_group, test_app_config, + write_detail_preservation_fixture, write_single_item, +}; + +mod behavior_detail_preservation; +mod behavior_quality_acceptance; +mod behavior_refresh_publication; diff --git a/backend/repository/src/xtream_repository/tests/behavior.rs b/backend/repository/src/xtream_repository/tests/behavior.rs new file mode 100644 index 000000000..941952f46 --- /dev/null +++ b/backend/repository/src/xtream_repository/tests/behavior.rs @@ -0,0 +1,265 @@ +use super::{ + get_collection_path, persist_input_xtream_playlist_cluster_to_disk, + preserve_details_with_injected_operation_failure, xtream_cluster_category_collection, DetailPreservationOperation, + PreserveDetailsOutcome, XtreamClusterPublishOutcome, +}; +use crate::BPlusTreeQuery; +use shared::{ + error::TuliproxError, + model::{ + InputType, LiveStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, + SeriesStreamProperties, StreamProperties, XtreamCluster, XtreamPlaylistItem, + }, + utils::Internable, +}; +use std::{ + fs, io, + path::Path, + process::{Child, ExitStatus}, + sync::Arc, + thread, + time::{Duration, Instant}, +}; +use tokio::io::AsyncWriteExt; +use tuliprox_core::{ + model::{AppConfig, ConfigInput}, + utils::request::DynReader, +}; + +pub(in crate::xtream_repository::tests) fn json_reader(content: &str) -> DynReader { + let (mut writer, reader) = tokio::io::duplex(4096); + let content = content.as_bytes().to_vec(); + tokio::spawn(async move { + writer.write_all(&content).await.expect("fixture should fit into duplex reader"); + writer.shutdown().await.expect("fixture writer should shut down"); + }); + Box::pin(reader) +} + +pub(in crate::xtream_repository::tests) fn disk_test_input(name: &str) -> ConfigInput { + ConfigInput { + name: name.intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("user".to_string()), + password: Some("password".to_string()), + ..ConfigInput::default() + } +} + +pub(in crate::xtream_repository::tests) fn cluster_fixture_readers( + cluster: XtreamCluster, + category_name: &str, + count: usize, + first_provider_id: u32, +) -> (DynReader, DynReader) { + let category_id = match cluster { + XtreamCluster::Live => 1_u32, + XtreamCluster::Video => 2, + XtreamCluster::Series => 3, + }; + let categories = serde_json::json!([{ + "category_id": category_id, + "category_name": category_name, + }]) + .to_string(); + let streams = (0..count) + .map(|offset| { + let provider_id = + first_provider_id + u32::try_from(offset).expect("fixture item count should fit into u32"); + let name = format!("{category_name}-{provider_id}"); + match cluster { + XtreamCluster::Live => serde_json::json!({ + "name": name, + "stream_id": provider_id, + "category_id": category_id, + "added": "0", + }), + XtreamCluster::Video => serde_json::json!({ + "name": name, + "stream_id": provider_id, + "category_id": category_id, + "added": "0", + "container_extension": "mp4", + }), + XtreamCluster::Series => serde_json::json!({ + "name": name, + "series_id": provider_id, + "category_id": category_id, + "last_modified": "0", + }), + } + }) + .collect::>(); + let streams = serde_json::Value::Array(streams).to_string(); + (json_reader(&categories), json_reader(&streams)) +} + +pub(in crate::xtream_repository::tests) fn live_fixture_stream( + provider_id: u32, + category_id: u32, + name: &str, +) -> serde_json::Value { + serde_json::json!({ + "name": name, + "stream_id": provider_id, + "category_id": category_id, + "added": "0", + }) +} + +pub(in crate::xtream_repository::tests) async fn publish_live_fixture_rows( + app_config: &Arc, + input: &ConfigInput, + threshold: u8, + categories: serde_json::Value, + streams: Vec, +) -> XtreamClusterPublishOutcome { + persist_input_xtream_playlist_cluster_to_disk( + app_config, + input, + XtreamCluster::Live, + threshold, + json_reader(&categories.to_string()), + json_reader(&serde_json::Value::Array(streams).to_string()), + ) + .await + .expect("Live fixture refresh should complete") +} + +pub(in crate::xtream_repository::tests) async fn publish_test_cluster( + app_config: &Arc, + input: &ConfigInput, + cluster: XtreamCluster, + threshold: u8, + category_name: &str, + count: usize, + first_provider_id: u32, +) -> XtreamClusterPublishOutcome { + let (categories, streams) = cluster_fixture_readers(cluster, category_name, count, first_provider_id); + persist_input_xtream_playlist_cluster_to_disk(app_config, input, cluster, threshold, categories, streams) + .await + .expect("test cluster refresh should complete") +} + +pub(in crate::xtream_repository::tests) fn active_cluster_count(storage_path: &Path, cluster: XtreamCluster) -> usize { + super::super::count_xtream_tree_entries(&super::super::xtream_get_file_path(storage_path, cluster)) + .expect("active cluster should be countable") + .expect("active cluster should exist") +} + +pub(in crate::xtream_repository::tests) fn active_category_bytes( + storage_path: &Path, + cluster: XtreamCluster, +) -> Vec { + fs::read(get_collection_path(storage_path, xtream_cluster_category_collection(cluster))) + .expect("active categories should be readable") +} + +#[derive(Debug, Eq, PartialEq)] +pub(in crate::xtream_repository::tests) struct ActiveClusterSnapshot { + pub(in crate::xtream_repository::tests) database: Vec, + pub(in crate::xtream_repository::tests) categories: Vec, +} + +pub(in crate::xtream_repository::tests) fn active_cluster_snapshot( + storage_path: &Path, + cluster: XtreamCluster, +) -> ActiveClusterSnapshot { + let database_path = super::super::xtream_get_file_path(storage_path, cluster); + ActiveClusterSnapshot { + database: fs::read(&database_path).expect("active database should be readable"), + categories: active_category_bytes(storage_path, cluster), + } +} + +pub(in crate::xtream_repository::tests) fn fail_detail_preservation_after_quality( + published_path: &Path, + staging_path: &Path, +) -> Result { + preserve_details_with_injected_operation_failure(published_path, staging_path, DetailPreservationOperation::Commit) +} + +pub(in crate::xtream_repository::tests) fn read_series_props(path: &Path, provider_id: u32) -> SeriesStreamProperties { + let mut query = BPlusTreeQuery::::try_new(path).expect("query open should succeed"); + let item = query.query_zero_copy(&provider_id).expect("query should succeed").expect("item should exist"); + match item.additional_properties { + Some(StreamProperties::Series(series)) => *series, + other => panic!("expected series stream properties, got {other:?}"), + } +} + +pub(in crate::xtream_repository::tests) fn assert_no_refresh_artifacts(storage_path: &Path) { + let entries = fs::read_dir(storage_path) + .expect("input storage should be readable") + .collect::>>() + .expect("input storage entries should be readable"); + let refresh_artifacts = entries + .into_iter() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name.contains("refresh-")) + .collect::>(); + assert!(refresh_artifacts.is_empty(), "staging artifacts survived: {refresh_artifacts:?}"); +} + +pub(in crate::xtream_repository::tests) fn wait_for_child( + mut child: Child, + timeout: Duration, +) -> io::Result { + let deadline = Instant::now() + timeout; + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(status), + Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(10)), + Ok(None) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(io::Error::new(io::ErrorKind::TimedOut, "Xtream refresh child timed out")); + } + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(error); + } + } + } +} + +pub(in crate::xtream_repository::tests) fn make_input_group( + cluster: XtreamCluster, + category_id: u32, + category_name: &str, + provider_id: u32, +) -> PlaylistGroup { + let stream_id = provider_id.to_string().intern(); + let category_name = category_name.intern(); + PlaylistGroup { + id: category_id, + title: Arc::clone(&category_name), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { + id: Arc::clone(&stream_id), + input_stream_id: stream_id, + name: format!("stream-{provider_id}").intern(), + title: format!("stream-{provider_id}").intern(), + group: category_name, + url: format!("http://provider.example/{cluster}/{provider_id}").intern(), + item_type: PlaylistItemType::from(cluster), + xtream_cluster: cluster, + category_id, + input_name: "provider-a".intern(), + ..PlaylistItemHeader::default() + }, + }], + xtream_cluster: cluster, + } +} + +pub(in crate::xtream_repository::tests) fn read_live_props(path: &Path, provider_id: u32) -> LiveStreamProperties { + let mut query = BPlusTreeQuery::::try_new(path).expect("query open should succeed"); + let item = query.query_zero_copy(&provider_id).expect("query should succeed").expect("item should exist"); + match item.additional_properties { + Some(StreamProperties::Live(live)) => *live, + other => panic!("expected live stream properties, got {other:?}"), + } +} diff --git a/backend/repository/src/xtream_repository/tests/behavior_detail_preservation.rs b/backend/repository/src/xtream_repository/tests/behavior_detail_preservation.rs new file mode 100644 index 000000000..4b40f2eed --- /dev/null +++ b/backend/repository/src/xtream_repository/tests/behavior_detail_preservation.rs @@ -0,0 +1,354 @@ +use super::{ + behavior::read_live_props, fixed_refresh_paths, make_live_item, merge_preserved_stream_properties, + needs_update_info_details, preserve_details_input_xtream_playlist_cluster_to_disk, + preserve_details_with_injected_operation_failure, write_detail_preservation_fixture, write_single_item, + DetailPreservationOperation, PreserveDetailsOutcome, +}; +use shared::{ + model::{CatchupProperties, LiveStreamProperties, SeriesStreamProperties, StreamProperties, VideoStreamProperties}, + utils::Internable, +}; +use tempfile::tempdir; + +#[test] +fn keeps_existing_details_when_new_timestamp_is_missing() { + let new_props = StreamProperties::Video(Box::new(VideoStreamProperties { + added: "".into(), + ..VideoStreamProperties::default() + })); + let old_props = StreamProperties::Video(Box::new(VideoStreamProperties { + added: "1700000000".into(), + ..VideoStreamProperties::default() + })); + + assert!(!needs_update_info_details(&new_props, &old_props)); +} + +#[test] +fn updates_details_when_new_timestamp_is_newer() { + let new_props = StreamProperties::Series(Box::new(SeriesStreamProperties { + last_modified: Some("200".into()), + ..SeriesStreamProperties::default() + })); + let old_props = StreamProperties::Series(Box::new(SeriesStreamProperties { + last_modified: Some("100".into()), + ..SeriesStreamProperties::default() + })); + + assert!(needs_update_info_details(&new_props, &old_props)); +} + +#[test] +fn does_not_update_details_when_new_timestamp_is_older() { + let new_props = StreamProperties::Series(Box::new(SeriesStreamProperties { + last_modified: Some("100".into()), + ..SeriesStreamProperties::default() + })); + let old_props = StreamProperties::Series(Box::new(SeriesStreamProperties { + last_modified: Some("200".into()), + ..SeriesStreamProperties::default() + })); + + assert!(!needs_update_info_details(&new_props, &old_props)); +} + +#[test] +fn merge_preserves_missing_live_probe_timestamps() { + let mut new_props = + StreamProperties::Live(Box::new(LiveStreamProperties { stream_id: 1, ..LiveStreamProperties::default() })); + let old_props = StreamProperties::Live(Box::new(LiveStreamProperties { + stream_id: 1, + last_probed_timestamp: Some(1_700_000_000), + last_success_timestamp: Some(1_700_000_100), + ..LiveStreamProperties::default() + })); + + let changed = merge_preserved_stream_properties(&mut new_props, &old_props); + assert!(changed); + + match new_props { + StreamProperties::Live(live) => { + assert_eq!(live.last_probed_timestamp, Some(1_700_000_000)); + assert_eq!(live.last_success_timestamp, Some(1_700_000_100)); + } + _ => panic!("expected live properties"), + } +} + +#[test] +fn merge_does_not_override_existing_live_probe_timestamps() { + let mut new_props = StreamProperties::Live(Box::new(LiveStreamProperties { + stream_id: 1, + last_probed_timestamp: Some(1_800_000_000), + last_success_timestamp: Some(1_800_000_100), + ..LiveStreamProperties::default() + })); + let old_props = StreamProperties::Live(Box::new(LiveStreamProperties { + stream_id: 1, + last_probed_timestamp: Some(1_700_000_000), + last_success_timestamp: Some(1_700_000_100), + ..LiveStreamProperties::default() + })); + + let changed = merge_preserved_stream_properties(&mut new_props, &old_props); + assert!(!changed); + + match new_props { + StreamProperties::Live(live) => { + assert_eq!(live.last_probed_timestamp, Some(1_800_000_000)); + assert_eq!(live.last_success_timestamp, Some(1_800_000_100)); + } + _ => panic!("expected live properties"), + } +} + +#[test] +fn merge_preserves_higher_learned_live_bitrate() { + let mut new_props = StreamProperties::Live(Box::new(LiveStreamProperties { + stream_id: 1, + bitrate: 1_500_000, + ..LiveStreamProperties::default() + })); + let old_props = StreamProperties::Live(Box::new(LiveStreamProperties { + stream_id: 1, + bitrate: 2_500_000, + ..LiveStreamProperties::default() + })); + + assert!(merge_preserved_stream_properties(&mut new_props, &old_props)); + match new_props { + StreamProperties::Live(live) => assert_eq!(live.bitrate, 2_500_000), + _ => panic!("expected live properties"), + } +} + +#[test] +fn merge_preserves_missing_live_catchup_properties() { + let mut new_props = + StreamProperties::Live(Box::new(LiveStreamProperties { stream_id: 1, ..LiveStreamProperties::default() })); + let old_props = StreamProperties::Live(Box::new(LiveStreamProperties { + stream_id: 1, + catchup: Some(CatchupProperties { + mode: Some("append".into()), + source: Some("?offset=-${offset}".into()), + ..CatchupProperties::default() + }), + ..LiveStreamProperties::default() + })); + + let changed = merge_preserved_stream_properties(&mut new_props, &old_props); + assert!(changed); + match new_props { + StreamProperties::Live(live) => { + let catchup = live.catchup.expect("catchup should be preserved"); + assert_eq!(catchup.mode.as_deref(), Some("append")); + assert_eq!(catchup.source.as_deref(), Some("?offset=-${offset}")); + } + _ => panic!("expected live properties"), + } +} + +#[test] +fn merge_preserves_missing_video_tmdb() { + let mut new_props = + StreamProperties::Video(Box::new(VideoStreamProperties { tmdb: None, ..VideoStreamProperties::default() })); + let old_props = StreamProperties::Video(Box::new(VideoStreamProperties { + tmdb: Some(317_981), + ..VideoStreamProperties::default() + })); + + let changed = merge_preserved_stream_properties(&mut new_props, &old_props); + assert!(changed); + match new_props { + StreamProperties::Video(video) => assert_eq!(video.tmdb, Some(317_981)), + _ => panic!("expected video properties"), + } +} + +#[test] +fn preserve_details_for_disk_cluster_copies_missing_live_probe_fields() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 3); + let provider_id = 100_u32; + + write_single_item( + &paths.published_database, + &make_live_item( + provider_id, + Some("{\"codec_name\":\"h264\"}"), + Some("{\"codec_name\":\"aac\"}"), + Some(1_700_000_000), + Some(1_700_000_100), + 2_500_000, + ), + ); + write_single_item(&paths.staging_database, &make_live_item(provider_id, None, None, None, None, 0)); + + let outcome = + preserve_details_input_xtream_playlist_cluster_to_disk(&paths.published_database, &paths.staging_database) + .expect("merge should succeed"); + assert_eq!(outcome, PreserveDetailsOutcome::Merged { scanned: 1, updated: 1 }); + + let merged = read_live_props(&paths.staging_database, provider_id); + assert_eq!(merged.video, Some("{\"codec_name\":\"h264\"}".intern())); + assert_eq!(merged.audio, Some("{\"codec_name\":\"aac\"}".intern())); + assert_eq!(merged.last_probed_timestamp, Some(1_700_000_000)); + assert_eq!(merged.last_success_timestamp, Some(1_700_000_100)); + assert_eq!(merged.bitrate, 2_500_000); +} + +#[test] +fn preserve_details_reports_missing_published_database_explicitly() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 4); + write_single_item(&paths.staging_database, &make_live_item(101, None, None, None, None, 0)); + + let outcome = + preserve_details_input_xtream_playlist_cluster_to_disk(&paths.published_database, &paths.staging_database) + .expect("a missing published database should not fail the refresh"); + + assert_eq!(outcome, PreserveDetailsOutcome::SourceMissing); +} + +#[test] +fn preserve_details_propagates_missing_staging_database() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 14); + write_single_item(&paths.published_database, &make_live_item(105, None, None, None, None, 0)); + + let error = + preserve_details_input_xtream_playlist_cluster_to_disk(&paths.published_database, &paths.staging_database) + .expect_err("a missing staging database must fail"); + let message = error.to_string(); + + assert!(message.contains("Failed to open staging Xtream tree")); + assert!(message.contains(&paths.staging_database.display().to_string())); +} + +#[test] +fn preserve_details_propagates_staging_batch_write_failure() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 16); + write_detail_preservation_fixture(&paths, 107); + + let error = preserve_details_with_injected_operation_failure( + &paths.published_database, + &paths.staging_database, + DetailPreservationOperation::BatchWrite, + ) + .expect_err("a staging batch write failure must fail the merge"); + let message = error.to_string(); + + assert!(message.contains("Failed to update staging Xtream tree")); + assert!(message.contains(&paths.staging_database.display().to_string())); + assert!(message.contains("injected BatchWrite failure")); +} + +#[test] +fn preserve_details_empty_merge_reports_zero_updates() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 7); + let item = make_live_item(104, None, None, None, None, 0); + write_single_item(&paths.published_database, &item); + write_single_item(&paths.staging_database, &item); + + let outcome = + preserve_details_input_xtream_playlist_cluster_to_disk(&paths.published_database, &paths.staging_database) + .expect("empty merge should succeed"); + + assert_eq!(outcome, PreserveDetailsOutcome::Merged { scanned: 1, updated: 0 }); +} + +#[cfg(unix)] +#[test] +fn refresh_staging_path_preserves_non_utf8_stem() { + use super::refresh_staging_path; + use std::{ + os::unix::ffi::{OsStrExt, OsStringExt}, + path::Path, + }; + use uuid::Uuid; + + let mut input_name = std::ffi::OsString::from_vec(vec![b'l', b'i', b'v', b'e', 0xff]); + input_name.push(".db"); + let published = Path::new("/tmp").join(input_name); + let generation = Uuid::from_u128(12); + + let staging = refresh_staging_path(&published, generation).expect("non-UTF-8 path should be supported"); + let bytes = staging.file_name().expect("staging file name").as_bytes(); + assert!(bytes.starts_with(&[b'l', b'i', b'v', b'e', 0xff])); + assert!(bytes.ends_with(b".db")); + assert!(bytes.windows(b".refresh-".len()).any(|window| window == b".refresh-")); +} + +#[test] +fn preserve_details_for_disk_cluster_does_not_override_existing_live_probe_fields() { + let dir = tempdir().expect("temp dir should be created"); + let old_path = dir.path().join("old_live_existing.db"); + let tmp_path = dir.path().join("tmp_live_existing.db"); + let provider_id = 200_u32; + + write_single_item( + &old_path, + &make_live_item( + provider_id, + Some("{\"codec_name\":\"h264\"}"), + Some("{\"codec_name\":\"aac\"}"), + Some(1_700_000_000), + Some(1_700_000_100), + 2_500_000, + ), + ); + write_single_item( + &tmp_path, + &make_live_item( + provider_id, + Some("{\"codec_name\":\"hevc\"}"), + Some("{\"codec_name\":\"ac3\"}"), + Some(1_800_000_000), + Some(1_800_000_100), + 3_500_000, + ), + ); + + preserve_details_input_xtream_playlist_cluster_to_disk(&old_path, &tmp_path).expect("merge should succeed"); + + let merged = read_live_props(&tmp_path, provider_id); + assert_eq!(merged.video, Some("{\"codec_name\":\"hevc\"}".intern())); + assert_eq!(merged.audio, Some("{\"codec_name\":\"ac3\"}".intern())); + assert_eq!(merged.last_probed_timestamp, Some(1_800_000_000)); + assert_eq!(merged.last_success_timestamp, Some(1_800_000_100)); + assert_eq!(merged.bitrate, 3_500_000); +} + +#[test] +fn preserve_details_for_disk_cluster_fills_only_missing_live_probe_fields() { + let dir = tempdir().expect("temp dir should be created"); + let old_path = dir.path().join("old_live_partial.db"); + let tmp_path = dir.path().join("tmp_live_partial.db"); + let provider_id = 300_u32; + + write_single_item( + &old_path, + &make_live_item( + provider_id, + Some("{\"codec_name\":\"h264\"}"), + Some("{\"codec_name\":\"aac\"}"), + Some(1_700_000_000), + Some(1_700_000_100), + 2_500_000, + ), + ); + write_single_item( + &tmp_path, + &make_live_item(provider_id, Some("{\"codec_name\":\"hevc\"}"), None, Some(1_800_000_000), None, 3_500_000), + ); + + preserve_details_input_xtream_playlist_cluster_to_disk(&old_path, &tmp_path).expect("merge should succeed"); + + let merged = read_live_props(&tmp_path, provider_id); + assert_eq!(merged.video, Some("{\"codec_name\":\"hevc\"}".intern())); + assert_eq!(merged.audio, Some("{\"codec_name\":\"aac\"}".intern())); + assert_eq!(merged.last_probed_timestamp, Some(1_800_000_000)); + assert_eq!(merged.last_success_timestamp, Some(1_700_000_100)); +} diff --git a/backend/repository/src/xtream_repository/tests/behavior_quality_acceptance.rs b/backend/repository/src/xtream_repository/tests/behavior_quality_acceptance.rs new file mode 100644 index 000000000..b320a265a --- /dev/null +++ b/backend/repository/src/xtream_repository/tests/behavior_quality_acceptance.rs @@ -0,0 +1,643 @@ +use super::{ + behavior::{ + active_category_bytes, active_cluster_count, active_cluster_snapshot, assert_no_refresh_artifacts, + cluster_fixture_readers, disk_test_input, fail_detail_preservation_after_quality, json_reader, + live_fixture_stream, publish_live_fixture_rows, publish_test_cluster, read_series_props, + }, + fixed_refresh_paths, get_collection_path, persist_input_xtream_playlist_cluster_to_disk, + persist_input_xtream_playlist_clusters_to_disk, persist_input_xtream_playlist_clusters_to_disk_with_operations, + persists_input_series_info, test_app_config, xtream_cluster_category_collection, XtreamClusterPublishOutcome, + XtreamClusterQualityPolicy, XtreamClusterRefreshRequest, XtreamClusterStageOperations, +}; +use crate::{build_input_storage_path, get_input_storage_path, BPlusTreeQuery}; +use shared::{ + model::{ + InputType, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, SeriesStreamProperties, + XtreamCluster, XtreamPlaylistItem, + }, + utils::Internable, +}; +use std::fs; +use tempfile::tempdir; +use tuliprox_core::model::{ + ClusterForceUpdate, ClusterUpdateAcceptance, ClusterUpdateRejection, ConfigInput, UpdateQualityDecision, +}; + +#[tokio::test] +async fn empty_disk_refresh_retains_published_playlist_and_catalog() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let app_config = test_app_config(directory.path()); + let input = ConfigInput { name: "empty-guard".intern(), input_type: InputType::Xtream, ..Default::default() }; + + persist_input_xtream_playlist_cluster_to_disk( + &app_config, + &input, + XtreamCluster::Live, + 0, + json_reader(r#"[{"category_id":"1","category_name":"News"}]"#), + json_reader(r#"[{"name":"Channel","stream_id":7,"category_id":"1","added":"0"}]"#), + ) + .await?; + + let storage = build_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()); + let database = super::super::xtream_get_file_path(&storage, XtreamCluster::Live); + let catalog = crate::raw_group_catalog_path(&storage, XtreamCluster::Live); + let database_before = tokio::fs::read(&database).await?; + let catalog_before = tokio::fs::read(&catalog).await?; + + let result = persist_input_xtream_playlist_cluster_to_disk( + &app_config, + &input, + XtreamCluster::Live, + 0, + json_reader("[]"), + json_reader("[]"), + ) + .await; + + assert!(result.is_err()); + assert_eq!(tokio::fs::read(database).await?, database_before); + assert_eq!(tokio::fs::read(catalog).await?, catalog_before); + Ok(()) +} + +#[tokio::test] +async fn empty_cluster_is_published_when_another_cluster_still_has_items() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let app_config = test_app_config(directory.path()); + let input = ConfigInput { name: "empty-cluster".intern(), input_type: InputType::Xtream, ..Default::default() }; + + for (cluster, category, stream) in [ + ( + XtreamCluster::Live, + r#"[{"category_id":"1","category_name":"News"}]"#, + r#"[{"name":"Channel","stream_id":7,"category_id":"1","added":"0"}]"#, + ), + ( + XtreamCluster::Video, + r#"[{"category_id":"2","category_name":"Movies"}]"#, + r#"[{"name":"Movie","stream_id":8,"category_id":"2","added":"0"}]"#, + ), + ] { + persist_input_xtream_playlist_cluster_to_disk( + &app_config, + &input, + cluster, + 0, + json_reader(category), + json_reader(stream), + ) + .await?; + } + + persist_input_xtream_playlist_cluster_to_disk( + &app_config, + &input, + XtreamCluster::Live, + 0, + json_reader("[]"), + json_reader("[]"), + ) + .await?; + + let storage = build_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()); + let playlist = + super::super::load_input_xtream_playlist(&app_config, &storage, &[XtreamCluster::Live, XtreamCluster::Video]) + .await?; + assert!(playlist.iter().all(|group| group.xtream_cluster != XtreamCluster::Live)); + assert!(playlist.iter().any(|group| group.xtream_cluster == XtreamCluster::Video)); + Ok(()) +} + +#[test] +fn disk_quality_disabled_does_not_read_staging_or_baseline() { + let directory = tempdir().expect("temp directory"); + let paths = fixed_refresh_paths(directory.path(), 18); + + let decision = super::super::evaluate_staged_xtream_cluster_quality(&paths, XtreamCluster::Live, 0) + .expect("disabled quality evaluation should not access missing files"); + + assert_eq!(decision, UpdateQualityDecision::Disabled); +} + +#[tokio::test] +async fn disk_force_publishes_an_empty_cluster_and_cleans_staging() { + let directory = tempdir().expect("temporary directory"); + let app_config = test_app_config(directory.path()); + let input = disk_test_input("force-empty"); + assert_eq!( + publish_test_cluster(&app_config, &input, XtreamCluster::Video, 0, "old-vod", 3, 2_000).await, + XtreamClusterPublishOutcome::Published + ); + let storage_path = + get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()).await.expect("force storage"); + let before = active_cluster_snapshot(&storage_path, XtreamCluster::Video); + + let result = persist_input_xtream_playlist_clusters_to_disk( + &app_config, + &input, + vec![XtreamClusterRefreshRequest { + cluster: XtreamCluster::Video, + quality: XtreamClusterQualityPolicy::Bypass { configured_threshold: 95 }, + categories: json_reader("[]"), + streams: json_reader("[]"), + }], + ) + .await; + + assert!(result.errors.is_empty()); + assert_eq!( + result.outcomes, + vec![XtreamClusterPublishOutcome::ForcePublished(ClusterForceUpdate { + cluster: XtreamCluster::Video, + candidate_count: 0, + configured_threshold: 95, + })] + ); + assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Video), 0); + assert_ne!(active_cluster_snapshot(&storage_path, XtreamCluster::Video), before); + assert_eq!( + serde_json::from_slice::(&active_category_bytes(&storage_path, XtreamCluster::Video,)) + .expect("empty category JSON"), + serde_json::json!([]) + ); + assert_no_refresh_artifacts(&storage_path); +} + +#[tokio::test] +async fn disk_force_keeps_active_files_after_a_technical_staging_failure() { + let directory = tempdir().expect("temporary directory"); + let app_config = test_app_config(directory.path()); + let input = disk_test_input("force-technical-error"); + assert_eq!( + publish_test_cluster(&app_config, &input, XtreamCluster::Live, 0, "old-live", 3, 1_000).await, + XtreamClusterPublishOutcome::Published + ); + let storage_path = + get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()).await.expect("force storage"); + let before = active_cluster_snapshot(&storage_path, XtreamCluster::Live); + + let result = persist_input_xtream_playlist_clusters_to_disk( + &app_config, + &input, + vec![XtreamClusterRefreshRequest { + cluster: XtreamCluster::Live, + quality: XtreamClusterQualityPolicy::Bypass { configured_threshold: 100 }, + categories: json_reader(r#"[{"category_id":"1","category_name":"new-live"}]"#), + streams: json_reader("{"), + }], + ) + .await; + + assert_eq!(result.errors.len(), 1); + assert!(result.outcomes.is_empty()); + assert!(result.force_updates.is_empty()); + assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Live), before); + assert_no_refresh_artifacts(&storage_path); +} + +#[tokio::test] +async fn pipeline_transparency_disk_quality_survives_post_evaluation_staging_failure() { + let directory = tempdir().expect("temporary directory"); + let app_config = test_app_config(directory.path()); + let input = disk_test_input("accepted-then-detail-error"); + assert_eq!( + publish_test_cluster(&app_config, &input, XtreamCluster::Live, 0, "old-live", 100, 1_000).await, + XtreamClusterPublishOutcome::Published + ); + let storage_path = + get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()).await.expect("storage path"); + let before = active_cluster_snapshot(&storage_path, XtreamCluster::Live); + let (categories, streams) = cluster_fixture_readers(XtreamCluster::Live, "new-live", 95, 10_000); + + let result = persist_input_xtream_playlist_clusters_to_disk_with_operations( + &app_config, + &input, + vec![XtreamClusterRefreshRequest { + cluster: XtreamCluster::Live, + quality: XtreamClusterQualityPolicy::Enforce { threshold: 90 }, + categories, + streams, + }], + XtreamClusterStageOperations { preserve_details: fail_detail_preservation_after_quality }, + ) + .await; + + assert_eq!( + result.quality_acceptances, + vec![ClusterUpdateAcceptance { + cluster: XtreamCluster::Live, + current_count: Some(100), + candidate_count: 95, + threshold: 90, + quality: Some(95), + }] + ); + assert!(result.outcomes.is_empty()); + assert_eq!(result.errors.len(), 1); + assert_eq!(result.failed_cluster, Some(XtreamCluster::Live)); + assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Live), before); + assert_no_refresh_artifacts(&storage_path); +} + +#[tokio::test] +async fn pipeline_transparency_disk_batch_keeps_prior_publish_and_all_evaluated_quality_on_later_error() { + let directory = tempdir().expect("temporary directory"); + let app_config = test_app_config(directory.path()); + let input = disk_test_input("later-cluster-publish-error"); + assert_eq!( + publish_test_cluster(&app_config, &input, XtreamCluster::Live, 0, "old-live", 100, 1_000).await, + XtreamClusterPublishOutcome::Published + ); + assert_eq!( + publish_test_cluster(&app_config, &input, XtreamCluster::Video, 0, "old-vod", 100, 2_000).await, + XtreamClusterPublishOutcome::Published + ); + let storage_path = + get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()).await.expect("storage path"); + let blocked_category_path = + get_collection_path(&storage_path, xtream_cluster_category_collection(XtreamCluster::Video)); + fs::remove_file(&blocked_category_path).expect("replace published VOD categories with a directory"); + fs::create_dir(&blocked_category_path).expect("create category publication blocker"); + let (live_categories, live_streams) = cluster_fixture_readers(XtreamCluster::Live, "new-live", 90, 10_000); + let (vod_categories, vod_streams) = cluster_fixture_readers(XtreamCluster::Video, "new-vod", 90, 20_000); + + let result = persist_input_xtream_playlist_clusters_to_disk( + &app_config, + &input, + vec![ + XtreamClusterRefreshRequest { + cluster: XtreamCluster::Live, + quality: XtreamClusterQualityPolicy::Enforce { threshold: 90 }, + categories: live_categories, + streams: live_streams, + }, + XtreamClusterRefreshRequest { + cluster: XtreamCluster::Video, + quality: XtreamClusterQualityPolicy::Enforce { threshold: 90 }, + categories: vod_categories, + streams: vod_streams, + }, + ], + ) + .await; + + assert_eq!(result.quality_acceptances.len(), 2); + assert_eq!(result.quality_acceptances[0].cluster, XtreamCluster::Live); + assert_eq!(result.quality_acceptances[1].cluster, XtreamCluster::Video); + assert_eq!(result.outcomes, vec![XtreamClusterPublishOutcome::QualityAccepted(result.quality_acceptances[0])]); + assert_eq!(result.errors.len(), 1); + assert_eq!(result.failed_cluster, Some(XtreamCluster::Video)); + assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Live), 90); + assert_no_refresh_artifacts(&storage_path); +} + +#[tokio::test] +async fn disk_quality_bootstrap_publishes_nonempty_candidate_and_rejects_empty_without_baseline() { + let populated_directory = tempdir().expect("populated bootstrap directory"); + let populated_config = test_app_config(populated_directory.path()); + let populated_input = disk_test_input("bootstrap-populated"); + + let populated_outcome = + publish_test_cluster(&populated_config, &populated_input, XtreamCluster::Live, 90, "bootstrap-live", 3, 1_000) + .await; + let populated_storage = + get_input_storage_path(&populated_input.name, populated_directory.path().to_string_lossy().as_ref()) + .await + .expect("populated bootstrap storage"); + + assert_eq!( + populated_outcome, + XtreamClusterPublishOutcome::QualityAccepted(ClusterUpdateAcceptance { + cluster: XtreamCluster::Live, + current_count: None, + candidate_count: 3, + threshold: 90, + quality: None, + }) + ); + assert_eq!(active_cluster_count(&populated_storage, XtreamCluster::Live), 3); + assert!(String::from_utf8_lossy(&active_category_bytes(&populated_storage, XtreamCluster::Live)) + .contains("bootstrap-live")); + assert_no_refresh_artifacts(&populated_storage); + + let empty_directory = tempdir().expect("empty bootstrap directory"); + let empty_config = test_app_config(empty_directory.path()); + let empty_input = disk_test_input("bootstrap-empty"); + let empty_outcome = + publish_test_cluster(&empty_config, &empty_input, XtreamCluster::Video, 90, "empty-vod", 0, 2_000).await; + let empty_storage = get_input_storage_path(&empty_input.name, empty_directory.path().to_string_lossy().as_ref()) + .await + .expect("empty bootstrap storage"); + + assert_eq!( + empty_outcome, + XtreamClusterPublishOutcome::RetainedPrevious(ClusterUpdateRejection { + cluster: XtreamCluster::Video, + current_count: 0, + candidate_count: 0, + threshold: 90, + quality: 0, + }) + ); + assert!(!super::super::xtream_get_file_path(&empty_storage, XtreamCluster::Video).exists()); + assert!(!get_collection_path(&empty_storage, xtream_cluster_category_collection(XtreamCluster::Video)).exists()); + assert_no_refresh_artifacts(&empty_storage); +} + +#[tokio::test] +async fn disk_quality_enforces_exact_90_percent_boundaries_and_retains_rejected_files() { + for (name, candidate_count, accepted) in [ + ("lower-boundary", 90, true), + ("below-lower-boundary", 89, false), + ("upper-boundary", 110, true), + ("above-upper-boundary", 111, false), + ] { + let directory = tempdir().expect("boundary directory"); + let app_config = test_app_config(directory.path()); + let input = disk_test_input(name); + assert_eq!( + publish_test_cluster(&app_config, &input, XtreamCluster::Live, 0, "old-live", 100, 10_000,).await, + XtreamClusterPublishOutcome::Published, + "baseline publish failed for {name}" + ); + let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) + .await + .expect("boundary storage"); + let before = active_cluster_snapshot(&storage_path, XtreamCluster::Live); + + let outcome = + publish_test_cluster(&app_config, &input, XtreamCluster::Live, 90, "new-live", candidate_count, 20_000) + .await; + + if accepted { + assert_eq!( + outcome, + XtreamClusterPublishOutcome::QualityAccepted(ClusterUpdateAcceptance { + cluster: XtreamCluster::Live, + current_count: Some(100), + candidate_count, + threshold: 90, + quality: Some(90), + }), + "case: {name}" + ); + assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Live), candidate_count); + assert!(String::from_utf8_lossy(&active_category_bytes(&storage_path, XtreamCluster::Live)) + .contains("new-live")); + } else { + assert_eq!( + outcome, + XtreamClusterPublishOutcome::RetainedPrevious(ClusterUpdateRejection { + cluster: XtreamCluster::Live, + current_count: 100, + candidate_count, + threshold: 90, + quality: 89, + }), + "case: {name}" + ); + assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Live), before, "case: {name}"); + } + assert_no_refresh_artifacts(&storage_path); + } +} + +#[tokio::test] +async fn disk_quality_rejects_duplicate_rows_that_represent_only_half_the_provider_ids() { + let directory = tempdir().expect("duplicate rejection directory"); + let app_config = test_app_config(directory.path()); + let input = disk_test_input("duplicate-rejection"); + let first_provider_id = 1_000_u32; + assert_eq!( + publish_test_cluster(&app_config, &input, XtreamCluster::Live, 0, "old-live", 100, first_provider_id,).await, + XtreamClusterPublishOutcome::Published + ); + let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) + .await + .expect("duplicate rejection storage"); + let before = active_cluster_snapshot(&storage_path, XtreamCluster::Live); + let streams = (0..50_u32) + .flat_map(|offset| { + let provider_id = first_provider_id + offset; + [ + live_fixture_stream(provider_id, 1, &format!("candidate-{provider_id}")), + live_fixture_stream(provider_id, 1, &format!("duplicate-{provider_id}")), + ] + }) + .collect(); + + let outcome = publish_live_fixture_rows( + &app_config, + &input, + 100, + serde_json::json!([{"category_id": 1, "category_name": "candidate-live"}]), + streams, + ) + .await; + + assert_eq!( + outcome, + XtreamClusterPublishOutcome::RetainedPrevious(ClusterUpdateRejection { + cluster: XtreamCluster::Live, + current_count: 100, + candidate_count: 50, + threshold: 100, + quality: 50, + }) + ); + assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Live), before); + assert_no_refresh_artifacts(&storage_path); +} + +#[tokio::test] +async fn pipeline_transparency_disk_quality_preserves_accepted_publish_facts() { + let directory = tempdir().expect("duplicate acceptance directory"); + let app_config = test_app_config(directory.path()); + let input = disk_test_input("duplicate-acceptance"); + let first_provider_id = 1_000_u32; + let winning_provider_id = first_provider_id + 42; + assert_eq!( + publish_test_cluster(&app_config, &input, XtreamCluster::Live, 0, "old-live", 100, first_provider_id,).await, + XtreamClusterPublishOutcome::Published + ); + let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) + .await + .expect("duplicate acceptance storage"); + let mut streams = (0..100_u32) + .map(|offset| { + let provider_id = first_provider_id + offset; + live_fixture_stream(provider_id, 1, &format!("candidate-{provider_id}")) + }) + .collect::>(); + streams.push(live_fixture_stream(winning_provider_id, 2, "winning-duplicate")); + + let outcome = publish_live_fixture_rows( + &app_config, + &input, + 100, + serde_json::json!([ + {"category_id": 1, "category_name": "candidate-live"}, + {"category_id": 2, "category_name": "winning-category"} + ]), + streams, + ) + .await; + + assert_eq!( + outcome, + XtreamClusterPublishOutcome::QualityAccepted(ClusterUpdateAcceptance { + cluster: XtreamCluster::Live, + current_count: Some(100), + candidate_count: 100, + threshold: 100, + quality: Some(100), + }) + ); + assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Live), 100); + let active_path = super::super::xtream_get_file_path(&storage_path, XtreamCluster::Live); + let mut query = + BPlusTreeQuery::::try_new(&active_path).expect("accepted Live cluster should open"); + let winner = query + .query_zero_copy(&winning_provider_id) + .expect("winner lookup should succeed") + .expect("winner should exist"); + assert_eq!(winner.name.as_ref(), "winning-duplicate"); + assert_eq!(winner.category_id, 2); + assert!(String::from_utf8_lossy(&active_category_bytes(&storage_path, XtreamCluster::Live)) + .contains("winning-category")); + assert_no_refresh_artifacts(&storage_path); +} + +#[tokio::test] +async fn disk_series_quality_compares_catalog_rows_and_ignores_embedded_episode_details() { + let directory = tempdir().expect("series population directory"); + let app_config = test_app_config(directory.path()); + let input = disk_test_input("series-logical-population"); + let provider_id = 30_000; + assert_eq!( + publish_test_cluster(&app_config, &input, XtreamCluster::Series, 0, "old-series", 1, provider_id,).await, + XtreamClusterPublishOutcome::Published + ); + let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) + .await + .expect("series population storage"); + let episodes = (0..200_u32) + .map(|id| SeriesStreamDetailEpisodeProperties { id, ..SeriesStreamDetailEpisodeProperties::default() }) + .collect(); + let enriched = SeriesStreamProperties { + series_id: provider_id, + details: Some(SeriesStreamDetailProperties::new(None, Vec::new(), Some(episodes))), + ..SeriesStreamProperties::default() + }; + persists_input_series_info(&app_config, &storage_path, XtreamCluster::Series, &input.name, provider_id, &enriched) + .await + .expect("series enrichment should persist"); + let active_path = super::super::xtream_get_file_path(&storage_path, XtreamCluster::Series); + assert_eq!( + read_series_props(&active_path, provider_id) + .details + .and_then(|details| details.episodes) + .map(|episodes| episodes.len()), + Some(200) + ); + + let equal_catalog = + publish_test_cluster(&app_config, &input, XtreamCluster::Series, 100, "equal-series", 1, provider_id).await; + + assert_eq!( + equal_catalog, + XtreamClusterPublishOutcome::QualityAccepted(ClusterUpdateAcceptance { + cluster: XtreamCluster::Series, + current_count: Some(1), + candidate_count: 1, + threshold: 100, + quality: Some(100), + }) + ); + assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Series), 1); + let accepted_snapshot = active_cluster_snapshot(&storage_path, XtreamCluster::Series); + + let different_catalog = + publish_test_cluster(&app_config, &input, XtreamCluster::Series, 100, "different-series", 2, 40_000).await; + + assert_eq!( + different_catalog, + XtreamClusterPublishOutcome::RetainedPrevious(ClusterUpdateRejection { + cluster: XtreamCluster::Series, + current_count: 1, + candidate_count: 2, + threshold: 100, + quality: 0, + }) + ); + assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Series), accepted_snapshot); + assert_no_refresh_artifacts(&storage_path); +} + +#[tokio::test] +async fn disk_quality_publishes_accepted_clusters_and_retains_rejected_cluster_independently() { + let directory = tempdir().expect("mixed cluster directory"); + let app_config = test_app_config(directory.path()); + let input = disk_test_input("mixed-clusters"); + + for (cluster, category_name, first_provider_id) in [ + (XtreamCluster::Live, "old-live", 10_000), + (XtreamCluster::Video, "old-vod", 20_000), + (XtreamCluster::Series, "old-series", 30_000), + ] { + assert_eq!( + publish_test_cluster(&app_config, &input, cluster, 0, category_name, 100, first_provider_id).await, + XtreamClusterPublishOutcome::Published + ); + } + let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) + .await + .expect("mixed cluster storage"); + let previous_vod = active_cluster_snapshot(&storage_path, XtreamCluster::Video); + + let live_outcome = publish_test_cluster(&app_config, &input, XtreamCluster::Live, 90, "new-live", 90, 40_000).await; + let vod_outcome = publish_test_cluster(&app_config, &input, XtreamCluster::Video, 90, "new-vod", 89, 50_000).await; + let series_outcome = + publish_test_cluster(&app_config, &input, XtreamCluster::Series, 90, "new-series", 110, 60_000).await; + + assert_eq!( + live_outcome, + XtreamClusterPublishOutcome::QualityAccepted(ClusterUpdateAcceptance { + cluster: XtreamCluster::Live, + current_count: Some(100), + candidate_count: 90, + threshold: 90, + quality: Some(90), + }) + ); + assert_eq!( + vod_outcome, + XtreamClusterPublishOutcome::RetainedPrevious(ClusterUpdateRejection { + cluster: XtreamCluster::Video, + current_count: 100, + candidate_count: 89, + threshold: 90, + quality: 89, + }) + ); + assert_eq!( + series_outcome, + XtreamClusterPublishOutcome::QualityAccepted(ClusterUpdateAcceptance { + cluster: XtreamCluster::Series, + current_count: Some(100), + candidate_count: 110, + threshold: 90, + quality: Some(90), + }) + ); + assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Live), 90); + assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Video), 100); + assert_eq!(active_cluster_count(&storage_path, XtreamCluster::Series), 110); + assert!(String::from_utf8_lossy(&active_category_bytes(&storage_path, XtreamCluster::Live)).contains("new-live")); + assert_eq!(active_cluster_snapshot(&storage_path, XtreamCluster::Video), previous_vod); + assert!( + String::from_utf8_lossy(&active_category_bytes(&storage_path, XtreamCluster::Series)).contains("new-series") + ); + assert_no_refresh_artifacts(&storage_path); +} diff --git a/backend/repository/src/xtream_repository/tests/behavior_refresh_publication.rs b/backend/repository/src/xtream_repository/tests/behavior_refresh_publication.rs new file mode 100644 index 000000000..d3c8a9fe0 --- /dev/null +++ b/backend/repository/src/xtream_repository/tests/behavior_refresh_publication.rs @@ -0,0 +1,388 @@ +use super::{ + behavior::{json_reader, make_input_group, read_live_props, wait_for_child}, + count_input_xtream_cluster, fixed_refresh_paths, get_collection_path, load_input_xtream_playlist, make_live_item, + persist_input_xtream_playlist, persist_input_xtream_playlist_cluster_to_disk, publish_staged_file_same_directory, + target_category_lock_path, target_writer_config, target_writer_group, test_app_config, write_single_item, + xtream_cluster_category_collection, xtream_get_playlist_categories, xtream_write_playlist, + xtream_write_playlist_with_mode, TargetEmptyPublicationHook, TargetEmptyReplacementMode, +}; +use crate::{ + bplustree::{ensure_distinct_sidecar_lock_domains, sidecar_lock_path}, + build_input_storage_path, get_file_path_for_db_index, get_input_storage_path, +}; +use shared::{ + model::{ClusterFlags, InputType, XtreamCluster}, + utils::Internable, +}; +use std::{ + env, fs, io, + path::Path, + process::{Command, Stdio}, + sync::Arc, + time::Duration, +}; +use tempfile::tempdir; +use tuliprox_core::model::ConfigInput; + +#[tokio::test] +async fn target_force_empty_category_reader_waits_through_the_backup_window() -> Result<(), Box> +{ + let directory = tempfile::tempdir()?; + let app_config = test_app_config(directory.path()); + let target = target_writer_config(); + let mut baseline = vec![ + target_writer_group(XtreamCluster::Live, 1, 101), + target_writer_group(XtreamCluster::Video, 2, 201), + target_writer_group(XtreamCluster::Series, 3, 301), + ]; + xtream_write_playlist(&app_config, &target, &mut baseline, ClusterFlags::empty()).await?; + let storage_path = { + let config = app_config.config.load(); + super::super::xtream_get_storage_path(&config, &target.name).expect("target Xtream storage") + }; + let category_path = super::super::get_vod_cat_collection_path(&storage_path); + let hook = TargetEmptyPublicationHook::new(); + let entered = Arc::clone(&hook.backup_window_entered); + let resume = Arc::clone(&hook.resume_publication); + let writer_app_config = Arc::clone(&app_config); + let writer_target = target.clone(); + let writer = tokio::spawn(async move { + let mut candidate = + vec![target_writer_group(XtreamCluster::Live, 1, 102), target_writer_group(XtreamCluster::Series, 3, 302)]; + xtream_write_playlist_with_mode( + &writer_app_config, + &writer_target, + &mut candidate, + ClusterFlags::Vod, + TargetEmptyReplacementMode::PauseDuringPublication(hook), + ) + .await + }); + + tokio::task::spawn_blocking(move || entered.wait()).await?; + let category_was_temporarily_backed_up = !category_path.exists(); + let category_lock_path = target_category_lock_path(&category_path); + let category_write_lock_was_held = app_config.file_locks.try_write_lock(&category_lock_path).await.is_err(); + + let category_read = xtream_get_playlist_categories(&app_config, &target.name, XtreamCluster::Video); + tokio::pin!(category_read); + let category_read_poll = futures::poll!(category_read.as_mut()); + let category_reader_state = match &category_read_poll { + std::task::Poll::Pending => "waiting", + std::task::Poll::Ready(None) => "missing", + std::task::Poll::Ready(Some(categories)) if categories.is_empty() => "new", + std::task::Poll::Ready(Some(_)) => "old-or-partial", + }; + + tokio::task::spawn_blocking(move || resume.wait()).await?; + writer.await??; + assert!(category_was_temporarily_backed_up, "failure hook must expose the internal backup window"); + assert!(category_write_lock_was_held, "writer must hold the target category lock during publication"); + assert_eq!(category_reader_state, "waiting", "the category reader must wait on the writer's category lock"); + let categories = category_read.await.expect("published VOD category catalog"); + assert!(categories.is_empty(), "reader must observe the complete force-empty category catalog"); + Ok(()) +} + +#[tokio::test] +async fn input_cluster_count_returns_none_for_missing_baseline() { + let directory = tempdir().expect("temp directory"); + let app_config = test_app_config(directory.path()); + let input = ConfigInput { name: "provider-a".intern(), input_type: InputType::Xtream, ..ConfigInput::default() }; + + assert_eq!( + count_input_xtream_cluster(&app_config, &input, XtreamCluster::Live) + .await + .expect("missing baseline should be readable"), + None + ); +} + +#[tokio::test] +async fn input_cluster_count_reads_the_canonical_active_raw_cluster() { + let directory = tempdir().expect("temp directory"); + let app_config = test_app_config(directory.path()); + let input = ConfigInput { name: "provider-a".intern(), input_type: InputType::Xtream, ..ConfigInput::default() }; + + let storage_path = get_input_storage_path(&input.name, directory.path().to_string_lossy().as_ref()) + .await + .expect("canonical input storage"); + write_single_item( + &super::super::xtream_get_file_path(&storage_path, XtreamCluster::Live), + &make_live_item(700, None, None, None, None, 0), + ); + + assert_eq!( + count_input_xtream_cluster(&app_config, &input, XtreamCluster::Live) + .await + .expect("active baseline should be readable"), + Some(1) + ); + assert_eq!( + count_input_xtream_cluster(&app_config, &input, XtreamCluster::Video) + .await + .expect("other cluster should remain absent"), + None + ); +} + +#[tokio::test] +async fn in_memory_fallback_keeps_colliding_category_ids_separate_by_cluster() { + let directory = tempdir().expect("temp directory"); + let app_config = test_app_config(directory.path()); + let storage_path = get_input_storage_path("provider-a", directory.path().to_string_lossy().as_ref()) + .await + .expect("canonical input storage"); + + let (_, seed_error) = persist_input_xtream_playlist( + &app_config, + &storage_path, + vec![ + make_input_group(XtreamCluster::Live, 1, "Previous Live", 100), + make_input_group(XtreamCluster::Video, 2, "Previous VOD", 200), + make_input_group(XtreamCluster::Series, 3, "Previous Series", 300), + ], + ) + .await; + assert!(seed_error.is_none(), "failed to seed persisted VOD: {seed_error:?}"); + let previous_vod = load_input_xtream_playlist(&app_config, &storage_path, &[XtreamCluster::Video]) + .await + .expect("seeded VOD should load"); + let category_path = get_collection_path(&storage_path, xtream_cluster_category_collection(XtreamCluster::Video)); + let previous_categories = fs::read(&category_path).expect("persisted VOD categories"); + + let (merged, persist_error) = persist_input_xtream_playlist( + &app_config, + &storage_path, + vec![ + make_input_group(XtreamCluster::Live, 1, "New Live", 101), + make_input_group(XtreamCluster::Series, 2, "New Series", 301), + ], + ) + .await; + + assert!(persist_error.is_none(), "failed to persist accepted clusters: {persist_error:?}"); + assert_eq!(fs::read(&category_path).expect("retained VOD categories"), previous_categories); + assert!(merged + .iter() + .all(|group| { group.channels.iter().all(|item| item.header.xtream_cluster == group.xtream_cluster) })); + + let retained_vod = merged + .iter() + .find(|group| group.xtream_cluster == XtreamCluster::Video && group.id == 2) + .expect("persisted VOD fallback"); + assert_eq!(retained_vod.title.as_ref(), "Previous VOD"); + assert_eq!(retained_vod.channels.len(), 1); + assert_eq!(retained_vod.channels[0].header.id.as_ref(), "200"); + + let accepted_live = merged + .iter() + .find(|group| group.xtream_cluster == XtreamCluster::Live && group.id == 1) + .expect("accepted Live candidate"); + assert_eq!(accepted_live.title.as_ref(), "New Live"); + assert_eq!(accepted_live.channels[0].header.id.as_ref(), "101"); + + let accepted_series = merged + .iter() + .find(|group| group.xtream_cluster == XtreamCluster::Series && group.id == 2) + .expect("accepted Series candidate sharing VOD category id"); + assert_eq!(accepted_series.title.as_ref(), "New Series"); + assert_eq!(accepted_series.channels[0].header.id.as_ref(), "301"); + + let loaded = load_input_xtream_playlist(&app_config, &storage_path, &[XtreamCluster::Video]) + .await + .expect("retained VOD should load"); + assert_eq!(loaded.len(), previous_vod.len()); + assert_eq!(loaded[0].id, previous_vod[0].id); + assert_eq!(loaded[0].title, previous_vod[0].title); + assert_eq!(loaded[0].xtream_cluster, previous_vod[0].xtream_cluster); + assert_eq!(loaded[0].channels.len(), previous_vod[0].channels.len()); + assert_eq!(loaded[0].channels[0].header.id, previous_vod[0].channels[0].header.id); + assert_eq!(loaded[0].channels[0].header.name, previous_vod[0].channels[0].header.name); + assert_eq!(loaded[0].channels[0].header.category_id, previous_vod[0].channels[0].header.category_id); + assert_eq!(loaded[0].channels[0].header.xtream_cluster, previous_vod[0].channels[0].header.xtream_cluster); +} + +#[test] +fn refresh_staging_database_uses_distinct_published_lock_domain() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 1); + + assert_eq!(paths.published_database.file_name().and_then(|name| name.to_str()), Some("live.db")); + assert_ne!(sidecar_lock_path(&paths.published_database), sidecar_lock_path(&paths.staging_database)); +} + +#[test] +fn refresh_staging_database_and_index_share_one_generation_lock_domain() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 2); + let staging_index = get_file_path_for_db_index(&paths.staging_database); + + assert_eq!(sidecar_lock_path(&paths.staging_database), sidecar_lock_path(&staging_index)); +} + +#[test] +fn colliding_staging_path_is_rejected_before_lock_acquisition() { + let dir = tempdir().expect("temp dir should be created"); + let published = dir.path().join("live.db"); + let colliding = dir.path().join("live.tmp"); + + let error = ensure_distinct_sidecar_lock_domains(&published, &colliding) + .expect_err("colliding sidecar domains must be rejected"); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); +} + +#[test] +fn sequential_refresh_generations_use_distinct_stems() { + let dir = tempdir().expect("temp dir should be created"); + let first = fixed_refresh_paths(dir.path(), 10); + let second = fixed_refresh_paths(dir.path(), 11); + + assert_ne!(first.staging_database.file_stem(), second.staging_database.file_stem()); + assert_ne!(sidecar_lock_path(&first.staging_database), sidecar_lock_path(&second.staging_database)); +} + +#[test] +fn category_publish_atomically_replaces_same_directory_file() { + let dir = tempdir().expect("temp dir should be created"); + let published = dir.path().join("cat_live.json"); + let staging = dir.path().join("cat_live.refresh-fixed.json"); + fs::write(&published, b"old").expect("published fixture should be written"); + fs::write(&staging, b"new").expect("staging fixture should be written"); + + publish_staged_file_same_directory(&staging, &published).expect("category publish should succeed"); + + assert_eq!(fs::read(&published).expect("published categories should be readable"), b"new"); + assert!(!staging.exists()); +} + +#[test] +fn category_publish_creates_missing_same_directory_file() { + let dir = tempdir().expect("temp dir should be created"); + let published = dir.path().join("cat_live.json"); + let staging = dir.path().join("cat_live.refresh-fixed.json"); + fs::write(&staging, b"new").expect("staging fixture should be written"); + + publish_staged_file_same_directory(&staging, &published).expect("category publish should succeed"); + + assert_eq!(fs::read(&published).expect("published categories should be readable"), b"new"); + assert!(!staging.exists()); +} + +#[test] +fn category_publish_rejects_different_parent_before_replace() { + let dir = tempdir().expect("temp dir should be created"); + let staging_dir = dir.path().join("staging"); + let published_dir = dir.path().join("published"); + fs::create_dir_all(&staging_dir).expect("staging directory should be created"); + fs::create_dir_all(&published_dir).expect("published directory should be created"); + let staging = staging_dir.join("cat_live.refresh-fixed.json"); + let published = published_dir.join("cat_live.json"); + fs::write(&staging, b"new").expect("staging fixture should be written"); + fs::write(&published, b"old").expect("published fixture should be written"); + + let error = publish_staged_file_same_directory(&staging, &published) + .expect_err("cross-directory category publication should fail"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(fs::read(&staging).expect("staging fixture should remain"), b"new"); + assert_eq!(fs::read(&published).expect("published fixture should remain"), b"old"); +} + +#[cfg(not(windows))] +#[test] +fn category_publish_reports_post_rename_barrier_failure_truthfully() { + let dir = tempdir().expect("temp dir should be created"); + let published = dir.path().join("cat_live.json"); + let staging = dir.path().join("cat_live.refresh-fixed.json"); + fs::write(&published, b"old").expect("published fixture should be written"); + fs::write(&staging, b"new").expect("staging fixture should be written"); + let staging_path = + tempfile::TempPath::try_from_path(&staging).expect("staging fixture should become an owned temporary path"); + + let error = super::super::publish_staged_file_with_parent_sync(staging_path, &published, |_| { + Err(io::Error::other("injected parent synchronization failure")) + }) + .expect_err("post-rename synchronization failure should be reported"); + + assert_eq!(error.kind(), io::ErrorKind::Other); + assert!(error.to_string().contains("was published, but its parent directory")); + assert_eq!(fs::read(&published).expect("published categories should be readable"), b"new"); + assert!(!staging.exists()); +} + +#[test] +fn xtream_refresh_end_to_end_child() -> io::Result<()> { + let Some(storage_root) = env::var_os("TULIPROX_XTREAM_REFRESH_TEST_ROOT") else { + return Ok(()); + }; + let storage_root = Path::new(&storage_root); + let app_config = test_app_config(storage_root); + let input = ConfigInput { name: "deadlock-test".intern(), input_type: InputType::Xtream, ..ConfigInput::default() }; + let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; + runtime.block_on(async { + for _ in 0..2 { + persist_input_xtream_playlist_cluster_to_disk( + &app_config, + &input, + XtreamCluster::Live, + 0, + json_reader(r#"[{"category_id":"1","category_name":"Sports"}]"#), + json_reader(r#"[{"name":"Live","stream_id":700,"category_id":"1","added":"0"}]"#), + ) + .await + .map_err(|error| io::Error::other(error.to_string()))?; + } + Ok::<(), io::Error>(()) + })?; + + let input_storage = build_input_storage_path(&input.name, storage_root.to_string_lossy().as_ref()); + let published = super::super::xtream_get_file_path(&input_storage, XtreamCluster::Live); + let learned = read_live_props(&published, 700); + assert_eq!(learned.video, Some("{\"codec_name\":\"h264\"}".intern())); + assert_eq!(learned.bitrate, 2_500_000); + Ok(()) +} + +#[test] +fn two_sequential_cluster_refreshes_complete_without_generation_artifacts() -> io::Result<()> { + let directory = tempfile::tempdir()?; + let input_name = "deadlock-test".intern(); + let input_storage = build_input_storage_path(&input_name, directory.path().to_string_lossy().as_ref()); + fs::create_dir_all(&input_storage)?; + let published = super::super::xtream_get_file_path(&input_storage, XtreamCluster::Live); + write_single_item( + &published, + &make_live_item( + 700, + Some("{\"codec_name\":\"h264\"}"), + Some("{\"codec_name\":\"aac\"}"), + Some(1_700_000_000), + Some(1_700_000_100), + 2_500_000, + ), + ); + + let child = Command::new(env::current_exe()?) + .arg("--exact") + .arg("xtream_repository::tests::xtream_refresh_end_to_end_child") + .arg("--nocapture") + .env("TULIPROX_XTREAM_REFRESH_TEST_ROOT", directory.path()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + let status = wait_for_child(child, Duration::from_secs(20))?; + assert!(status.success(), "Xtream refresh child failed with {status}"); + + let learned = read_live_props(&published, 700); + assert_eq!(learned.video, Some("{\"codec_name\":\"h264\"}".intern())); + assert_eq!(learned.bitrate, 2_500_000); + let entries = fs::read_dir(&input_storage)?.collect::>>()?; + let generation_artifacts = entries + .into_iter() + .filter(|entry| entry.file_name().to_string_lossy().contains(".refresh-")) + .collect::>(); + assert!(generation_artifacts.is_empty(), "generation artifacts survived successful refreshes"); + assert!(sidecar_lock_path(&published).exists()); + Ok(()) +} diff --git a/backend/repository/src/xtream_repository/tests/lifecycle.rs b/backend/repository/src/xtream_repository/tests/lifecycle.rs new file mode 100644 index 000000000..eb77daf4b --- /dev/null +++ b/backend/repository/src/xtream_repository/tests/lifecycle.rs @@ -0,0 +1,107 @@ +use super::{fixed_refresh_paths, merge_preserved_stream_properties, XtreamRefreshLease}; +use crate::{bplustree::sidecar_lock_path, cleanup_orphaned_staging_artifacts, refresh_generation_guard_path}; +use shared::model::{SeriesStreamProperties, StreamProperties}; +use std::{fs, time::Duration}; +use tempfile::tempdir; + +#[test] +fn merge_preserves_missing_series_tmdb_and_release_date() { + let mut new_props = StreamProperties::Series(Box::new(SeriesStreamProperties { + tmdb: None, + release_date: None, + ..SeriesStreamProperties::default() + })); + let old_props = StreamProperties::Series(Box::new(SeriesStreamProperties { + tmdb: Some(12345), + release_date: Some("2015-01-01".into()), + ..SeriesStreamProperties::default() + })); + + let changed = merge_preserved_stream_properties(&mut new_props, &old_props); + assert!(changed); + match new_props { + StreamProperties::Series(series) => { + assert_eq!(series.tmdb, Some(12345)); + assert_eq!(series.release_date.as_deref(), Some("2015-01-01")); + } + _ => panic!("expected series properties"), + } +} + +#[test] +fn refresh_lease_cleanup_is_idempotent_and_generation_local() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 8); + let other_paths = fixed_refresh_paths(dir.path(), 9); + let lease = XtreamRefreshLease::new(paths.clone()).expect("refresh lease should be valid"); + + fs::write(&paths.published_database, b"published").expect("published fixture should be written"); + let published_lock = sidecar_lock_path(&paths.published_database); + fs::write(&published_lock, b"").expect("published lock fixture should be written"); + for artifact in lease.0.database_artifacts.owned_paths() { + fs::write(artifact, b"staging").expect("staging artifact should be written"); + } + fs::write(&paths.staging_categories, b"staging").expect("staging category should be written"); + fs::write(&other_paths.staging_database, b"other generation").expect("other generation fixture should be written"); + + lease.cleanup_staging_artifacts().expect("first cleanup should succeed"); + lease.cleanup_staging_artifacts().expect("second cleanup should be idempotent"); + + assert!(paths.published_database.exists()); + assert!(published_lock.exists()); + assert!(other_paths.staging_database.exists()); + assert!(!paths.staging_database.exists()); + assert!(!paths.staging_categories.exists()); +} + +#[test] +fn refresh_lease_defers_cleanup_until_last_worker_clone_drops() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 13); + let guard_path = refresh_generation_guard_path(dir.path(), paths.generation); + let parent_lease = XtreamRefreshLease::new(paths.clone()).expect("refresh lease should be valid"); + let worker_lease = parent_lease.clone(); + fs::write(&paths.staging_database, b"staging").expect("staging fixture should be written"); + fs::write(&paths.staging_categories, b"categories").expect("category fixture should be written"); + + drop(parent_lease); + assert!(paths.staging_database.exists()); + assert!(paths.staging_categories.exists()); + assert!(guard_path.exists()); + + drop(worker_lease); + assert!(!paths.staging_database.exists()); + assert!(!paths.staging_categories.exists()); + assert!(!guard_path.exists()); +} + +#[test] +fn active_refresh_between_btree_batches_survives_orphan_cleanup() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 14); + let guard_path = refresh_generation_guard_path(dir.path(), paths.generation); + let lease = XtreamRefreshLease::new(paths.clone()).expect("refresh lease should be valid"); + let sidecar = sidecar_lock_path(&paths.staging_database); + fs::write(&paths.staging_database, b"staging").expect("staging fixture should be written"); + fs::write(&paths.staging_categories, b"categories").expect("category fixture should be written"); + fs::write(&sidecar, b"").expect("sidecar fixture should be written"); + + let between_batch_probe = + fs::OpenOptions::new().read(true).write(true).open(&sidecar).expect("open staging sidecar"); + between_batch_probe.try_lock().expect("staging sidecar should be unlocked between batches"); + between_batch_probe.unlock().expect("release between-batch probe"); + drop(between_batch_probe); + + cleanup_orphaned_staging_artifacts(dir.path(), Duration::ZERO); + + assert!(paths.staging_database.exists()); + assert!(paths.staging_categories.exists()); + assert!(sidecar.exists()); + assert!(guard_path.exists()); + + drop(lease); + assert!(!paths.staging_database.exists()); + assert!(!paths.staging_categories.exists()); + assert!(!sidecar.exists()); + assert!(!guard_path.exists()); +} diff --git a/backend/repository/src/xtream_repository/tests/persistence.rs b/backend/repository/src/xtream_repository/tests/persistence.rs new file mode 100644 index 000000000..029328e89 --- /dev/null +++ b/backend/repository/src/xtream_repository/tests/persistence.rs @@ -0,0 +1,81 @@ +use super::{ + fixed_refresh_paths, preserve_details_with_injected_operation_failure, target_writer_config, target_writer_group, + test_app_config, write_detail_preservation_fixture, xtream_write_playlist, + xtream_write_playlist_with_injected_empty_replacement_failure, DetailPreservationOperation, + TargetEmptyReplacementFailure, +}; +use crate::get_file_path_for_db_index; +use shared::model::{ClusterFlags, XtreamCluster}; +use std::fs; +use tempfile::tempdir; + +#[tokio::test] +async fn target_force_empty_failures_restore_database_index_and_categories() -> Result<(), Box> { + for failure in [ + TargetEmptyReplacementFailure::CategoryPersistence, + TargetEmptyReplacementFailure::BTreePersistence, + TargetEmptyReplacementFailure::Publication, + ] { + let directory = tempfile::tempdir()?; + let app_config = test_app_config(directory.path()); + let target = target_writer_config(); + let mut baseline = vec![ + target_writer_group(XtreamCluster::Live, 1, 101), + target_writer_group(XtreamCluster::Video, 2, 201), + target_writer_group(XtreamCluster::Series, 3, 301), + ]; + xtream_write_playlist(&app_config, &target, &mut baseline, ClusterFlags::empty()).await?; + let storage_path = { + let config = app_config.config.load(); + super::super::xtream_get_storage_path(&config, &target.name).expect("target Xtream storage") + }; + let database = super::super::xtream_get_file_path(&storage_path, XtreamCluster::Video); + let index = get_file_path_for_db_index(&database); + let categories = super::super::get_vod_cat_collection_path(&storage_path); + let before = + [tokio::fs::read(&database).await?, tokio::fs::read(&index).await?, tokio::fs::read(&categories).await?]; + let mut candidate = + vec![target_writer_group(XtreamCluster::Live, 1, 102), target_writer_group(XtreamCluster::Series, 3, 302)]; + + let error = xtream_write_playlist_with_injected_empty_replacement_failure( + &app_config, + &target, + &mut candidate, + ClusterFlags::Vod, + failure, + ) + .await + .expect_err("injected empty replacement must fail"); + + assert!(error.to_string().contains("target cluster failed")); + assert_eq!(tokio::fs::read(&database).await?, before[0]); + assert_eq!(tokio::fs::read(&index).await?, before[1]); + assert_eq!(tokio::fs::read(&categories).await?, before[2]); + let leaked = fs::read_dir(&storage_path)? + .filter_map(Result::ok) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name.contains("force-empty")) + .collect::>(); + assert!(leaked.is_empty(), "staging or backup files leaked after {failure:?}: {leaked:?}"); + } + Ok(()) +} + +#[test] +fn preserve_details_propagates_staging_commit_failure() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 17); + write_detail_preservation_fixture(&paths, 108); + + let error = preserve_details_with_injected_operation_failure( + &paths.published_database, + &paths.staging_database, + DetailPreservationOperation::Commit, + ) + .expect_err("a staging commit failure must fail the merge"); + let message = error.to_string(); + + assert!(message.contains("Failed to commit staging Xtream tree")); + assert!(message.contains(&paths.staging_database.display().to_string())); + assert!(message.contains("injected Commit failure")); +} diff --git a/backend/repository/src/xtream_repository/tests/query.rs b/backend/repository/src/xtream_repository/tests/query.rs new file mode 100644 index 000000000..1fd437f88 --- /dev/null +++ b/backend/repository/src/xtream_repository/tests/query.rs @@ -0,0 +1,114 @@ +use super::{ + fixed_refresh_paths, get_collection_path, make_live_item, preserve_details_input_xtream_playlist_cluster_to_disk, + preserve_details_with_injected_operation_failure, target_writer_config, target_writer_group, test_app_config, + write_detail_preservation_fixture, write_single_item, xtream_cluster_category_collection, xtream_write_playlist, + DetailPreservationOperation, +}; +use shared::model::{ClusterFlags, XtreamCluster}; +use std::fs; +use tempfile::tempdir; + +#[tokio::test] +async fn target_writer_creates_missing_empty_category_files_without_force() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let app_config = test_app_config(directory.path()); + let target = target_writer_config(); + let mut live_only = vec![target_writer_group(XtreamCluster::Live, 1, 101)]; + + xtream_write_playlist(&app_config, &target, &mut live_only, ClusterFlags::empty()).await?; + + let storage_path = { + let config = app_config.config.load(); + super::super::xtream_get_storage_path(&config, &target.name).expect("target Xtream storage") + }; + for cluster in [XtreamCluster::Video, XtreamCluster::Series] { + let categories = get_collection_path(&storage_path, xtream_cluster_category_collection(cluster)); + assert_eq!(tokio::fs::read(categories).await?, b"[]"); + assert!(!super::super::xtream_get_file_path(&storage_path, cluster).exists()); + } + Ok(()) +} + +#[tokio::test] +async fn target_writer_preserves_an_unauthorized_empty_cluster() -> Result<(), Box> { + let directory = tempfile::tempdir()?; + let app_config = test_app_config(directory.path()); + let target = target_writer_config(); + let mut baseline = vec![ + target_writer_group(XtreamCluster::Live, 1, 101), + target_writer_group(XtreamCluster::Video, 2, 201), + target_writer_group(XtreamCluster::Series, 3, 301), + ]; + xtream_write_playlist(&app_config, &target, &mut baseline, ClusterFlags::empty()).await?; + + let storage_path = { + let config = app_config.config.load(); + super::super::xtream_get_storage_path(&config, &target.name).expect("target Xtream storage") + }; + let retained_paths = [XtreamCluster::Video, XtreamCluster::Series].map(|cluster| { + ( + super::super::xtream_get_file_path(&storage_path, cluster), + get_collection_path(&storage_path, xtream_cluster_category_collection(cluster)), + ) + }); + let mut retained_before = Vec::new(); + for (database, categories) in &retained_paths { + retained_before.push((tokio::fs::read(database).await?, tokio::fs::read(categories).await?)); + } + + let mut live_only = vec![target_writer_group(XtreamCluster::Live, 1, 102)]; + xtream_write_playlist(&app_config, &target, &mut live_only, ClusterFlags::empty()).await?; + + for ((database, categories), (database_before, categories_before)) in retained_paths.iter().zip(retained_before) { + assert_eq!(tokio::fs::read(database).await?, database_before); + assert_eq!(tokio::fs::read(categories).await?, categories_before); + } + Ok(()) +} + +#[test] +fn preserve_details_propagates_corrupt_published_database() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 5); + fs::write(&paths.published_database, b"corrupt").expect("corrupt fixture should be written"); + write_single_item(&paths.staging_database, &make_live_item(102, None, None, None, None, 0)); + + let error = + preserve_details_input_xtream_playlist_cluster_to_disk(&paths.published_database, &paths.staging_database) + .expect_err("corrupt published data must fail"); + + assert!(error.to_string().contains(&paths.published_database.display().to_string())); +} + +#[test] +fn preserve_details_propagates_corrupt_staging_database() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 6); + write_single_item(&paths.published_database, &make_live_item(103, None, None, None, None, 0)); + fs::write(&paths.staging_database, b"corrupt").expect("corrupt fixture should be written"); + + let error = + preserve_details_input_xtream_playlist_cluster_to_disk(&paths.published_database, &paths.staging_database) + .expect_err("corrupt staging data must fail"); + + assert!(error.to_string().contains(&paths.staging_database.display().to_string())); +} + +#[test] +fn preserve_details_propagates_staging_query_failure() { + let dir = tempdir().expect("temp dir should be created"); + let paths = fixed_refresh_paths(dir.path(), 15); + write_detail_preservation_fixture(&paths, 106); + + let error = preserve_details_with_injected_operation_failure( + &paths.published_database, + &paths.staging_database, + DetailPreservationOperation::Query, + ) + .expect_err("a staging query failure must fail the merge"); + let message = error.to_string(); + + assert!(message.contains("Failed to query staging Xtream tree")); + assert!(message.contains(&paths.staging_database.display().to_string())); + assert!(message.contains("injected Query failure")); +} diff --git a/backend/repository/src/xtream_repository/tests/support.rs b/backend/repository/src/xtream_repository/tests/support.rs new file mode 100644 index 000000000..09de160ff --- /dev/null +++ b/backend/repository/src/xtream_repository/tests/support.rs @@ -0,0 +1,175 @@ +use super::XtreamRefreshPaths; +use crate::BPlusTreeUpdate; +use arc_swap::{ArcSwap, ArcSwapOption}; +use shared::{ + model::{ + ConfigPaths, LiveStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, + ProcessingOrder, StreamProperties, VirtualId, XtreamCluster, XtreamPlaylistItem, + }, + utils::Internable, +}; +use std::{path::Path, sync::Arc}; +use tuliprox_core::{ + model::{ + ApiProxyConfig, AppConfig, Config, ConfigTarget, CustomStreamResponse, HdHomeRunConfig, MediaToolCapabilities, + SourcesConfig, StagedFilter, TargetExecutionPlan, TargetOutput, XtreamTargetFlagsSet, XtreamTargetOutput, + }, + utils::FileLockManager, +}; +use uuid::Uuid; + +pub(in crate::xtream_repository::tests) fn test_app_config(storage_dir: &Path) -> Arc { + Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config { + storage_dir: storage_dir.to_string_lossy().into_owned(), + ..Config::default() + })), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::::default()), + api_proxy: Arc::new(ArcSwapOption::::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + }) +} + +pub(in crate::xtream_repository::tests) fn target_writer_config() -> ConfigTarget { + ConfigTarget { + curation: None, + id: 1, + enabled: true, + name: "target-empty-cluster-guard".to_string(), + options: None, + sort: None, + filter: StagedFilter::default(), + output: vec![TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::new(), + trakt: None, + filter: None, + })], + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::new(None)), + favourites: None, + processing_order: ProcessingOrder::default(), + execution_plan: TargetExecutionPlan::default(), + watch: None, + use_memory_cache: false, + } +} + +pub(in crate::xtream_repository::tests) fn target_writer_group( + cluster: XtreamCluster, + category_id: u32, + virtual_id: u32, +) -> PlaylistGroup { + let title = format!("target-{cluster}").intern(); + PlaylistGroup { + id: category_id, + title: Arc::clone(&title), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { + id: virtual_id.to_string().intern(), + input_stream_id: virtual_id.to_string().intern(), + virtual_id: VirtualId::new(virtual_id), + name: Arc::clone(&title), + title: Arc::clone(&title), + group: title, + input_name: "target-writer-input".intern(), + item_type: PlaylistItemType::from(cluster), + xtream_cluster: cluster, + category_id, + ..PlaylistItemHeader::default() + }, + }], + xtream_cluster: cluster, + } +} + +pub(in crate::xtream_repository::tests) fn make_live_item( + provider_id: u32, + video: Option<&str>, + audio: Option<&str>, + last_probed_timestamp: Option, + last_success_timestamp: Option, + bitrate: u32, +) -> XtreamPlaylistItem { + XtreamPlaylistItem { + virtual_id: VirtualId::new(provider_id), + provider_id, + name: "Live".intern(), + logo: "".intern(), + logo_small: "".intern(), + group: "group".intern(), + title: "".intern(), + parent_code: "".intern(), + rec: "".intern(), + url: "http://example.com/live.ts".intern(), + epg_channel_id: None, + xtream_cluster: XtreamCluster::Live, + additional_properties: Some(StreamProperties::Live(Box::new(LiveStreamProperties { + video: video.map(Internable::intern), + audio: audio.map(Internable::intern), + last_probed_timestamp, + last_success_timestamp, + bitrate, + ..Default::default() + }))), + item_type: shared::model::PlaylistItemType::Live, + category_id: 1, + input_name: "input_a".intern(), + channel_no: 0, + source_ordinal: 0, + input_stream_id: provider_id.to_string().intern(), + upstream_user_agent: None, + } +} + +pub(in crate::xtream_repository::tests) fn write_single_item(path: &Path, item: &XtreamPlaylistItem) { + crate::BPlusTree::::new().store(path).expect("tree creation should succeed"); + let mut tree = + BPlusTreeUpdate::::try_new_with_backoff(path).expect("tree open should succeed"); + let batch: Vec<(&u32, &XtreamPlaylistItem)> = vec![(&item.provider_id, item)]; + let prepared = BPlusTreeUpdate::::prepare_upsert_batch(&batch) + .expect("batch preparation should succeed"); + tree.upsert_batch_encoded(prepared).expect("batch upsert should succeed"); + tree.commit().expect("tree commit should succeed"); +} + +pub(in crate::xtream_repository::tests) fn fixed_refresh_paths(path: &Path, generation: u128) -> XtreamRefreshPaths { + XtreamRefreshPaths::for_generation(path, XtreamCluster::Live, Uuid::from_u128(generation)) + .expect("fixed refresh paths should be valid") +} + +pub(in crate::xtream_repository::tests) fn write_detail_preservation_fixture( + paths: &XtreamRefreshPaths, + provider_id: u32, +) { + write_single_item( + &paths.published_database, + &make_live_item( + provider_id, + Some("{\"codec_name\":\"h264\"}"), + Some("{\"codec_name\":\"aac\"}"), + Some(1_700_000_000), + Some(1_700_000_100), + 2_500_000, + ), + ); + write_single_item(&paths.staging_database, &make_live_item(provider_id, None, None, None, None, 0)); +} diff --git a/backend/session/src/active_provider_manager.rs b/backend/session/src/active_provider_manager.rs index 4bae244d0..dbc16d9b9 100644 --- a/backend/session/src/active_provider_manager.rs +++ b/backend/session/src/active_provider_manager.rs @@ -1,43 +1,22 @@ -use crate::{ - provider_leases::{ProviderAffinityTtl, ProviderLeaseTable, ProviderLeaseUsage}, - provider_lineup_manager::ProviderLineupManager, - EventManager, SharedStreamManager, -}; -use log::error; -use shared::utils::sanitize_sensitive_info; +use crate::{provider_leases::ProviderLeaseTable, provider_lineup_manager::ProviderLineupManager, SharedStreamManager}; use std::{ - cmp::Reverse, - collections::{BTreeMap, HashMap, HashSet}, + collections::HashMap, net::SocketAddr, sync::{ - atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering}, - Arc, LazyLock, OnceLock, RwLockReadGuard, RwLockWriteGuard, Weak, + atomic::{AtomicBool, AtomicU64}, + Arc, LazyLock, OnceLock, Weak, }, - time::{Duration, Instant}, + time::Duration, }; -use tokio::time::Instant as TokioInstant; use tokio_util::sync::CancellationToken; -use tuliprox_core::{ - model::{ - AllocationId, AppConfig, ConfigInput, ConnectionLifecycle, GracePeriodOptions, PlaybackKind, PlaybackLeaseId, - PlaybackRequestId, PlaybackRequestOutcome, PlaybackSelectionReason, ProviderAllocation, ProviderBindingTag, - ProviderCloseReason, ProviderConfig, ProviderHandle, SharedSubscriberId, - }, - utils::debug_if_enabled, +use tuliprox_core::model::{ + AllocationId, PlaybackKind, PlaybackRequestId, ProviderAllocation, ProviderHandle, SharedSubscriberId, }; static DUMMY_ADDR: LazyLock = LazyLock::new(|| SocketAddr::from(([127, 0, 0, 1], 0))); -type SharedConnectionId = AllocationId; -type PreemptionCandidate = (PriorityOwner, AllocationId, i8, Instant); -// Key for BTreeMap priority index: (priority, Reverse, AllocationId) -// Semantics: lower numeric priority value = higher importance (0 = highest, 127 = lowest). -// `.last()` on the BTreeMap returns the entry with the highest priority value, which is -// the lowest-importance connection and therefore the best eviction victim. -// Ties are broken by `Reverse`: among equal priority values, the oldest connection -// (smallest `created_at`) sorts last and is evicted first. -type PriorityKey = (i8, Reverse, AllocationId); const PREEMPTION_COMPLETION_TIMEOUT: Duration = Duration::from_millis(1500); + const EVICTED_PROVIDER_RELEASE_POLL_INTERVAL: Duration = Duration::from_millis(10); #[derive(Clone, Debug, PartialEq, Eq)] @@ -47,61 +26,12 @@ pub(crate) struct ProviderReleaseSnapshot { shared_subscribers: Vec, } -impl ProviderReleaseSnapshot { - pub fn is_empty(&self) -> bool { self.single_allocations.is_empty() && self.shared_subscribers.is_empty() } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ConnectionKind { Normal, Soft, } -fn is_better_preemption_candidate(current: Option, candidate: PreemptionCandidate) -> bool { - match current { - None => true, - Some((_, current_allocation_id, current_priority, current_created_at)) => { - candidate.2 > current_priority - || (candidate.2 == current_priority - && (candidate.3 < current_created_at - || (candidate.3 == current_created_at && candidate.1 < current_allocation_id))) - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum PriorityOwner { - Single(AllocationId), - Shared(SharedConnectionId), -} - -/// Resolves a public live HLS session token to its stable provider playback owner. -/// -/// The random suffix distinguishes public sessions, while the preceding client, -/// user and channel fingerprint identifies one provider playback across retries. -/// Catchup and shared-cache owners retain their own identity semantics. -fn playback_lease_owner(owner: &str) -> &str { - if owner.starts_with("m3u-catchup|") || owner.starts_with("catchup|") || owner.starts_with("hls-cache:") { - return owner; - } - let Some((base, suffix)) = owner.rsplit_once("|hls|") else { - return owner; - }; - if suffix.len() == 16 && suffix.bytes().all(|byte| byte.is_ascii_alphanumeric()) { - base - } else { - owner - } -} - -/// How an acquisition of a playback owner is tied to a provider. -enum OwnerProviderPin { - /// A confirmed or active lease pins the playback to exactly this provider. - Reserved(Arc), - /// A recently confirmed playback prefers this provider; it holds no capacity. - Preferred(Arc), -} - /// Playback identity of one provider acquisition. /// /// `owner` carries the public session identity so transport cleanup can distinguish @@ -114,135 +44,11 @@ pub struct PlaybackLeaseRef<'a> { pub request_id: PlaybackRequestId, } -impl<'a> PlaybackLeaseRef<'a> { - pub fn new(owner: &'a str, kind: PlaybackKind) -> Self { - Self { owner, kind, request_id: PlaybackRequestId::next() } - } - - /// Stable provider lease identity, independent of the public HLS token suffix. - pub fn provider_owner(&self) -> &'a str { playback_lease_owner(self.owner) } - - /// Compatibility view of a bare session owner: one request, live TS semantics. - pub fn for_session_owner(owner: &'a str) -> Self { Self::new(owner, PlaybackKind::LiveTs) } -} - -struct AcquireProviderParams<'a> { - addr: &'a SocketAddr, - priority: i8, - kind: ConnectionKind, - lease: Option>, -} - -struct ProviderAllocationGuard(Option); - -impl ProviderAllocationGuard { - fn new(allocation: ProviderAllocation) -> Self { Self(Some(allocation)) } - - fn allocation(&self) -> &ProviderAllocation { self.0.as_ref().unwrap_or(&ProviderAllocation::Exhausted) } - - fn take(&mut self) -> ProviderAllocation { self.0.take().unwrap_or(ProviderAllocation::Exhausted) } -} - -impl Drop for ProviderAllocationGuard { - fn drop(&mut self) { - if let Some(allocation) = self.0.take() { - allocation.release(); - } - } -} - -impl AcquireProviderParams<'_> { - #[inline] - fn session_owner(&self) -> Option<&str> { self.lease.map(|lease| lease.provider_owner()) } -} - -#[derive(Debug, Clone)] -struct SharedAllocation { - allocation_id: AllocationId, - origin_subscriber_id: SharedSubscriberId, - allocation: ProviderAllocation, - /// Keyed by unique subscriber id, never by socket: two external clients behind one - /// reverse proxy must not collapse into a single entry. - connections: HashMap, - priority: i8, - kind: ConnectionKind, - created_at: Instant, - cancel_token: Option, - session_owner: Option>, -} - -#[derive(Debug, Clone)] -struct ActiveConnectionInfo { - allocation_id: AllocationId, - client_addr: SocketAddr, - allocation: ProviderAllocation, - // Used to signal preemption to the consumer of this connection - cancel_token: CancellationToken, - completion_token: CancellationToken, - close_reason: Arc, - lifecycle: ConnectionLifecycle, - has_body_owner: bool, - reaper_spawned: bool, - open_generation: u64, - created_at: Instant, - priority: i8, - kind: ConnectionKind, - session_owner: Option>, - // Public session identity scopes targeted socket cleanup independently of the - // stable provider lease owner shared by entry retries. - request_owner: Option>, - playback_request_id: Option, -} - -#[derive(Debug, Clone, Copy)] -struct SharedSubscriber { - priority: i8, - kind: ConnectionKind, - /// Transport metadata and socket-wide close target only; never an identity key. - addr: SocketAddr, -} - -#[derive(Debug, Clone, Default)] -struct SharedConnections { - by_key: HashMap, SharedAllocation>, - key_by_subscriber: HashMap>, - shared_by_allocation_id: HashMap>, -} - -#[derive(Debug, Clone, Default)] -struct Connections { - // Primary map keyed by AllocationId, never by socket address. - single: HashMap, - // Secondary index for socket-wide transport close/kick actions. - single_by_addr: HashMap>, - shared: SharedConnections, - // Index to quickly find connections by provider name for preemption - // ProviderName -> Set - by_provider: HashMap, HashSet>, - // Index to find every allocation (single and shared) owned by a playback session. - // SessionOwner -> Set - by_owner: HashMap, HashSet>, - // Priority index per provider alias for O(log n) victim lookup - // ProviderName -> BTreeMap - priority_index: HashMap, BTreeMap>, - soft_priority_index: HashMap, BTreeMap>, - // Providers whose allocations were released since the last drain. - released_providers: Vec>, -} - /// Tables that record which providers' capacity a write freed. trait CapacityReleases { fn take_released_providers(&mut self) -> Vec>; } -impl CapacityReleases for Connections { - fn take_released_providers(&mut self) -> Vec> { std::mem::take(&mut self.released_providers) } -} - -impl CapacityReleases for ProviderLeaseTable { - fn take_released_providers(&mut self) -> Vec> { ProviderLeaseTable::take_released_providers(self) } -} - /// Wakes only the requests waiting for capacity on a provider whose slot or lease was freed. #[derive(Default)] pub struct ProviderCapacityNotifier { @@ -251,151 +57,11 @@ pub struct ProviderCapacityNotifier { has_staged: AtomicBool, } -impl ProviderCapacityNotifier { - fn lock_waiters(&self) -> std::sync::MutexGuard<'_, HashMap, Arc>> { - self.waiters.lock().unwrap_or_else(std::sync::PoisonError::into_inner) - } - - /// The notification fired whenever `provider` frees capacity. Provider names are bounded by - /// the configuration, so entries are kept. - pub fn subscribe(&self, provider: &Arc) -> Arc { - Arc::clone(self.lock_waiters().entry(Arc::clone(provider)).or_default()) - } - - fn notify(&self, provider: &str) { - if let Some(notify) = self.lock_waiters().get(provider) { - notify.notify_waiters(); - } - } - - /// Called while a table lock is held; [`Self::flush`] delivers after it is released. - fn stage(&self, released: Vec>) { - if released.is_empty() { - return; - } - self.staged.lock().unwrap_or_else(std::sync::PoisonError::into_inner).extend(released); - self.has_staged.store(true, Ordering::Release); - } - - fn flush(&self) { - if !self.has_staged.swap(false, Ordering::AcqRel) { - return; - } - let mut staged = std::mem::take(&mut *self.staged.lock().unwrap_or_else(std::sync::PoisonError::into_inner)); - staged.sort_unstable(); - staged.dedup(); - let waiters = self.lock_waiters(); - for provider in &staged { - if let Some(notify) = waiters.get(provider) { - notify.notify_waiters(); - } - } - } -} - -/// Write access to a capacity table that wakes the waiters of freed providers after release. -/// -/// Fields drop in declaration order: `lock` stages the freed providers while it still holds the -/// lock and then releases it; only afterwards does `_flush` wake the waiters. -struct CapacityWriteGuard<'a, T: CapacityReleases> { - lock: StagingWriteGuard<'a, T>, - _flush: FlushCapacityWaiters<'a>, -} - -struct StagingWriteGuard<'a, T: CapacityReleases> { - guard: RwLockWriteGuard<'a, T>, - notifier: &'a ProviderCapacityNotifier, -} - -impl Drop for StagingWriteGuard<'_, T> { - fn drop(&mut self) { self.notifier.stage(self.guard.take_released_providers()); } -} - -struct FlushCapacityWaiters<'a>(&'a ProviderCapacityNotifier); - -impl Drop for FlushCapacityWaiters<'_> { - fn drop(&mut self) { self.0.flush(); } -} - -impl<'a, T: CapacityReleases> CapacityWriteGuard<'a, T> { - fn new(guard: RwLockWriteGuard<'a, T>, notifier: &'a ProviderCapacityNotifier) -> Self { - Self { lock: StagingWriteGuard { guard, notifier }, _flush: FlushCapacityWaiters(notifier) } - } -} - -impl std::ops::Deref for CapacityWriteGuard<'_, T> { - type Target = T; - fn deref(&self) -> &Self::Target { &self.lock.guard } -} - -impl std::ops::DerefMut for CapacityWriteGuard<'_, T> { - fn deref_mut(&mut self) -> &mut Self::Target { &mut self.lock.guard } -} - pub struct ManagedProviderHandle { manager: Arc, handle: Option, } -impl std::fmt::Debug for ManagedProviderHandle { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ManagedProviderHandle").field("handle", &self.handle).finish_non_exhaustive() - } -} - -impl ManagedProviderHandle { - #[inline] - pub fn manager(&self) -> &Arc { &self.manager } - - #[inline] - pub fn new(manager: Arc, handle: ProviderHandle) -> Self { - Self { manager, handle: Some(handle) } - } - - #[inline] - pub fn handle(&self) -> Option<&ProviderHandle> { self.handle.as_ref() } - - #[inline] - pub fn take(&mut self) -> Option { self.handle.take() } - - #[inline] - pub fn disarm(&mut self) -> Option { self.handle.take() } - - pub fn mark_opening(&self) { - if let Some(handle) = self.handle.as_ref() { - self.manager.mark_opening(handle.allocation_id); - } - } - - pub fn register_body_owner(&self) -> bool { - if let Some(handle) = self.handle.as_ref() { - self.manager.register_body_owner(handle.allocation_id) - } else { - false - } - } - - pub fn renew_opening_tokens(&mut self) -> Option<(CancellationToken, CancellationToken)> { - if let Some(ref mut handle) = self.handle { - if let Some((cancel, completion, gen)) = self.manager.renew_opening_tokens(handle.allocation_id) { - handle.cancel_token = Some(cancel.clone()); - handle.completion_token = Some(completion.clone()); - handle.open_generation = gen; - return Some((cancel, completion)); - } - } - None - } -} - -impl Drop for ManagedProviderHandle { - fn drop(&mut self) { - if let Some(handle) = self.handle.take() { - self.manager.release_handle(&handle); - } - } -} - pub struct ActiveProviderManagerCore { // Serializes capacity transitions across counters, allocation indices and leases. capacity_transition: std::sync::Mutex<()>, @@ -409,180 +75,6 @@ pub struct ActiveProviderManagerCore { capacity: ProviderCapacityNotifier, } -impl ActiveProviderManagerCore { - /// Notification fired whenever `provider` frees a slot or ends a lease. - pub fn provider_capacity_notify(&self, provider: &Arc) -> Arc { - self.capacity.subscribe(provider) - } - - /// Serialises capacity transitions (acquire/release/reclassify) so a reclassify - /// cannot race an acquire or release. - /// - /// Lock-order invariant: this lock must be acquired **before** the connections - /// lock (`read_connections`/`write_connections`). No path may hold the - /// connections lock and then call this; doing so can deadlock. - fn lock_capacity_transition(&self) -> std::sync::MutexGuard<'_, ()> { - match self.capacity_transition.lock() { - Ok(guard) => guard, - Err(poisoned) => { - error!("Recovering poisoned active-provider capacity transition lock"); - poisoned.into_inner() - } - } - } - - fn read_connections(&self) -> RwLockReadGuard<'_, Connections> { - match self.connections.read() { - Ok(guard) => guard, - Err(poisoned) => { - error!("Recovering poisoned active-provider connection lock"); - poisoned.into_inner() - } - } - } - - fn write_connections(&self) -> CapacityWriteGuard<'_, Connections> { - let guard = match self.connections.write() { - Ok(guard) => guard, - Err(poisoned) => { - error!("Recovering poisoned active-provider connection lock"); - poisoned.into_inner() - } - }; - CapacityWriteGuard::new(guard, &self.capacity) - } - - fn write_leases(&self) -> CapacityWriteGuard<'_, ProviderLeaseTable> { - let guard = match self.leases.write() { - Ok(guard) => guard, - Err(poisoned) => { - error!("Recovering poisoned provider-lease lock"); - poisoned.into_inner() - } - }; - CapacityWriteGuard::new(guard, &self.capacity) - } - - fn read_leases(&self) -> RwLockReadGuard<'_, ProviderLeaseTable> { - match self.leases.read() { - Ok(guard) => guard, - Err(poisoned) => { - error!("Recovering poisoned provider-lease lock"); - poisoned.into_inner() - } - } - } - - pub fn complete_release(&self, alloc_id: AllocationId) { self.complete_release_with_generation(alloc_id, None); } - - pub fn complete_release_with_generation(&self, alloc_id: AllocationId, generation: Option) { - let _transition = self.lock_capacity_transition(); - let mut connections = self.write_connections(); - Self::complete_release_locked(&mut connections, alloc_id, generation); - } - - fn complete_release_locked( - connections: &mut Connections, - alloc_id: AllocationId, - expected_generation: Option, - ) -> Option { - if let Some(info) = connections.single.get(&alloc_id) { - if let Some(expected_gen) = expected_generation { - if info.open_generation != expected_gen { - return None; - } - } - } - if let Some(mut info) = connections.single.remove(&alloc_id) { - ActiveProviderManager::unindex_owner(connections, alloc_id, info.session_owner.as_ref()); - if let Some(set) = connections.single_by_addr.get_mut(&info.client_addr) { - set.remove(&alloc_id); - if set.is_empty() { - connections.single_by_addr.remove(&info.client_addr); - } - } - if !info.allocation.is_unlimited_provider() { - if let Some(name) = info.allocation.get_provider_name() { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&alloc_id); - } - ActiveProviderManager::remove_priority_entry( - connections, - &name, - &(info.priority, Reverse(info.created_at), alloc_id), - info.kind, - ); - } - } - info.lifecycle = ConnectionLifecycle::Closed; - info.cancel_token.cancel(); - info.allocation.release(); - if let Some(name) = info.allocation.get_provider_name() { - connections.released_providers.push(name); - } - Some(info.allocation) - } else { - None - } - } - - fn plan_single_release_locked(connections: &mut Connections, alloc_id: AllocationId) -> ReleaseAction { - let Some(info) = connections.single.get_mut(&alloc_id) else { - return ReleaseAction::None; - }; - if info.lifecycle == ConnectionLifecycle::Closed { - return ReleaseAction::None; - } - - info.cancel_token.cancel(); - - let completed = info.completion_token.is_cancelled(); - let needs_completion_wait = (info.lifecycle == ConnectionLifecycle::Closing - || info.lifecycle == ConnectionLifecycle::Opening - || info.has_body_owner) - && !completed; - - if needs_completion_wait { - info.lifecycle = ConnectionLifecycle::Closing; - let client_addr = info.client_addr; - let prio = info - .allocation - .get_provider_name() - .map(|name| (name, (info.priority, Reverse(info.created_at), alloc_id), info.kind)); - let already_spawned = info.reaper_spawned; - info.reaper_spawned = true; - let gen = info.open_generation; - let completion_token = info.completion_token.clone(); - - if let Some((name, key, kind)) = prio { - ActiveProviderManager::remove_priority_entry(connections, &name, &key, kind); - } - if let Some(set) = connections.single_by_addr.get_mut(&client_addr) { - set.remove(&alloc_id); - if set.is_empty() { - connections.single_by_addr.remove(&client_addr); - } - } - if already_spawned { - ReleaseAction::None - } else { - ReleaseAction::Wait(alloc_id, gen, completion_token) - } - } else { - let gen = info.open_generation; - Self::complete_release_locked(connections, alloc_id, Some(gen)); - ReleaseAction::Immediate - } - } -} - -#[derive(Debug)] -enum ReleaseAction { - None, - Immediate, - Wait(AllocationId, u64, CancellationToken), -} - #[derive(Debug)] pub(crate) enum PreemptionOutcome { Acquired(ProviderAllocation), @@ -599,5935 +91,21 @@ pub struct ActiveProviderManager { shared_stream_manager: OnceLock>, } -impl std::ops::Deref for ActiveProviderManager { - type Target = ActiveProviderManagerCore; - - #[inline] - fn deref(&self) -> &Self::Target { &self.core } -} - -impl ActiveProviderManager { - fn has_connections_for_addr(&self, addr: &SocketAddr) -> bool { - let _transition = self.lock_capacity_transition(); - let connections = self.read_connections(); - connections.single.values().any(|info| info.client_addr == *addr) - || connections - .shared - .by_key - .values() - .any(|shared| shared.connections.values().any(|subscriber| subscriber.addr == *addr)) - } - - /// Waits for every allocation currently using an address to leave the registry. - /// Admission handoffs use `wait_for_snapshot_release` to ignore later allocations at that address. - pub async fn wait_for_addr_release(&self, addr: &SocketAddr, timeout: Duration) -> bool { - let deadline = TokioInstant::now() + timeout; - loop { - if !self.has_connections_for_addr(addr) { - return true; - } - let now = TokioInstant::now(); - if now >= deadline { - return false; - } - tokio::time::sleep_until((now + EVICTED_PROVIDER_RELEASE_POLL_INTERVAL).min(deadline)).await; - } - } - - pub(crate) fn release_snapshot_for_addr(&self, addr: &SocketAddr) -> ProviderReleaseSnapshot { - let _transition = self.lock_capacity_transition(); - let connections = self.read_connections(); - let single_allocations = connections - .single - .values() - .filter_map(|info| (info.client_addr == *addr).then_some(info.allocation_id)) - .collect(); - let shared_subscribers = connections - .shared - .by_key - .values() - .flat_map(|shared| { - shared.connections.iter().filter_map(|(id, subscriber)| (subscriber.addr == *addr).then_some(*id)) - }) - .collect(); - ProviderReleaseSnapshot { addr: *addr, single_allocations, shared_subscribers } - } - - fn has_connections_from_snapshot(&self, snapshot: &ProviderReleaseSnapshot) -> bool { - let _transition = self.lock_capacity_transition(); - let connections = self.read_connections(); - snapshot.single_allocations.iter().any(|id| { - connections.single.contains_key(id) - || connections.shared.shared_by_allocation_id.get(id).is_some_and(|key| { - connections.shared.by_key.get(key).is_some_and(|shared| { - connections.shared.key_by_subscriber.get(&shared.origin_subscriber_id) == Some(key) - }) - }) - }) || snapshot.shared_subscribers.iter().any(|id| connections.shared.key_by_subscriber.contains_key(id)) - } - - /// Waits for the kicked transport's original provider allocations to leave the registry. - /// The socket close may be signalled before its response bodies and provider handles are dropped. - pub(crate) async fn wait_for_snapshot_release( - &self, - snapshot: &ProviderReleaseSnapshot, - timeout: Duration, - ) -> bool { - let deadline = TokioInstant::now() + timeout; - loop { - if !self.has_connections_from_snapshot(snapshot) { - return true; - } - let now = TokioInstant::now(); - if now >= deadline { - return false; - } - tokio::time::sleep_until((now + EVICTED_PROVIDER_RELEASE_POLL_INTERVAL).min(deadline)).await; - } - } - - fn upsert_priority_entry( - connections: &mut Connections, - provider_name: &Arc, - key: PriorityKey, - owner: PriorityOwner, - kind: ConnectionKind, - ) { - connections.priority_index.entry(provider_name.clone()).or_default().insert(key, owner); - if kind == ConnectionKind::Soft { - connections.soft_priority_index.entry(provider_name.clone()).or_default().insert(key, owner); - } - } - - fn remove_priority_entry( - connections: &mut Connections, - provider_name: &Arc, - key: &PriorityKey, - kind: ConnectionKind, - ) { - if let Some(tree) = connections.priority_index.get_mut(provider_name) { - tree.remove(key); - } - if kind == ConnectionKind::Soft { - if let Some(tree) = connections.soft_priority_index.get_mut(provider_name) { - tree.remove(key); - } - } - } - - fn index_owner(connections: &mut Connections, allocation_id: AllocationId, owner: Option<&Arc>) { - if let Some(owner) = owner { - connections.by_owner.entry(Arc::clone(owner)).or_default().insert(allocation_id); - } - } - - fn unindex_owner(connections: &mut Connections, allocation_id: AllocationId, owner: Option<&Arc>) { - if let Some(owner) = owner { - let remove = connections.by_owner.get_mut(owner).is_some_and(|set| { - set.remove(&allocation_id); - set.is_empty() - }); - if remove { - connections.by_owner.remove(owner); - } - } - } - - fn shared_effective_priority( - subscribers: &HashMap, - kind: ConnectionKind, - ) -> Option { - subscribers.values().filter(|subscriber| subscriber.kind == kind).map(|subscriber| subscriber.priority).min() - } - - fn shared_effective_kind(subscribers: &HashMap) -> ConnectionKind { - if subscribers.values().all(|subscriber| subscriber.kind == ConnectionKind::Soft) { - ConnectionKind::Soft - } else { - ConnectionKind::Normal - } - } - - pub fn new(cfg: &AppConfig, event_manager: &Arc) -> Self { - let grace_period_options = Self::get_grace_options(cfg); - let inputs = Self::get_config_inputs(cfg); - Self { - core: Arc::new(ActiveProviderManagerCore { - capacity_transition: std::sync::Mutex::new(()), - is_shutting_down: AtomicBool::new(false), - shutdown_token: CancellationToken::new(), - providers: ProviderLineupManager::new(inputs, grace_period_options, event_manager), - connections: std::sync::RwLock::new(Connections::default()), - leases: std::sync::RwLock::new(ProviderLeaseTable::with_affinity_ttl(Self::get_affinity_ttl(cfg))), - next_allocation_id: AtomicU64::new(1), - capacity: ProviderCapacityNotifier::default(), - }), - shared_stream_manager: OnceLock::new(), - } - } - - /// Routes account observations reported through `event_manager` to this manager. - /// Only the first binding wins, so test or auxiliary managers cannot hijack the runtime one. - pub fn bind_event_manager(&self, event_manager: &EventManager) { - let _ = event_manager.account_provider.set(Arc::downgrade(&self.core)); - } - - pub fn observe_account( - &self, - observation: tuliprox_core::model::ProviderAccountObservation, - ) -> Option { - self.core.providers.observe_account(observation) - } - - pub fn reset_account_health(&self, name: &Arc) { self.core.providers.forget_account_health(name); } - pub fn pending_account_observations(&self) -> Vec { - self.core.providers.pending_account_observations() - } - pub fn is_observation_pending(&self, observation: &tuliprox_core::model::ProviderAccountObservation) -> bool { - self.core.providers.is_observation_pending(observation) - } - pub fn acknowledge_account_observation(&self, observation: &tuliprox_core::model::ProviderAccountObservation) { - self.core.providers.acknowledge_account_observation(observation); - } - pub fn request_account_probe(&self, name: &Arc) { self.core.providers.request_account_probe(name); } - pub fn is_account_probe_requested(&self, name: &Arc) -> bool { - self.core.providers.is_account_probe_requested(name) - } - pub fn clear_account_probe(&self, name: &Arc) { self.core.providers.clear_account_probe(name); } - pub async fn account_health_changed(&self) { self.core.providers.account_health_changed.notified().await; } - - pub fn set_shared_stream_manager(&self, manager: &Arc) { - let _ = self.shared_stream_manager.set(Arc::downgrade(manager)); - } - - /// Closes provider admission and releases all physical allocations and leases. - pub fn shutdown(&self) { - self.shutdown_token.cancel(); - self.is_shutting_down.store(true, Ordering::Release); - let _transition = self.lock_capacity_transition(); - let connections = std::mem::take(&mut *self.write_connections()); - self.write_leases().clear(); - - for info in connections.single.into_values() { - info.cancel_token.cancel(); - info.allocation.release(); - } - for shared in connections.shared.by_key.into_values() { - if let Some(cancel_token) = shared.cancel_token { - cancel_token.cancel(); - } - shared.allocation.release(); - } - self.providers.reconcile_connections(HashMap::new()); - } - - fn get_config_inputs(cfg: &AppConfig) -> Vec> { - cfg.sources.load().inputs.iter().filter(|i| i.enabled).map(Arc::clone).collect() - } - - fn get_grace_options(cfg: &AppConfig) -> GracePeriodOptions { cfg.config.load().get_grace_options() } - - fn get_affinity_ttl(cfg: &AppConfig) -> ProviderAffinityTtl { - ProviderAffinityTtl(cfg.config.load().get_provider_affinity_ttl_secs()) - } - - pub fn update_config(&self, cfg: &AppConfig) { - let grace_period_options = Self::get_grace_options(cfg); - let inputs = Self::get_config_inputs(cfg); - self.providers.update_config(inputs, &grace_period_options); - self.write_leases().set_affinity_ttl(Self::get_affinity_ttl(cfg)); - self.reconcile_connections(); - } - - pub fn reconcile_connections(&self) { - let _transition = self.lock_capacity_transition(); - let mut counts = HashMap::, usize>::new(); - { - let connections = self.read_connections(); - - // Single connections - for info in connections.single.values() { - if let Some(name) = info.allocation.get_provider_name() { - *counts.entry(name).or_insert(0) += 1; - } - } - - // Shared connections - for shared in connections.shared.by_key.values() { - if let Some(name) = shared.allocation.get_provider_name() { - *counts.entry(name).or_insert(0) += 1; - } - } - } - - self.providers.reconcile_connections(counts); - } - - fn prune_expired_leases(leases: &mut ProviderLeaseTable) { leases.prune(TokioInstant::now()); } - - fn has_foreign_reservation(&self, provider_name: &Arc, session_owner: Option<&str>) -> bool { - if !self.providers.reservation_blocks_other_sessions(provider_name) { - return false; - } - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - leases.has_foreign_reserved_lease(provider_name, session_owner) - } - - /// Resolves how a playback owner is bound to a provider of `input_name`, with a - /// single lease-table lookup per acquisition. - fn owner_provider_pin(&self, input_name: &Arc, session_owner: &str) -> Option { - let (lease_pin, affinity_provider) = { - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - let lease_pin = leases - .lease_of_owner(session_owner) - .map(|lease| (Arc::clone(&lease.provider_name), lease.state.is_confirmed())); - (lease_pin, leases.affinity_provider_for_owner(session_owner)) - }; - // The lease lock is released before the connection lock is taken. - if let Some((provider_name, confirmed)) = lease_pin { - if self.providers.is_provider_for_input(&provider_name, input_name) - && (confirmed || self.has_active_owner_for_provider(&provider_name, session_owner)) - { - return Some(OwnerProviderPin::Reserved(provider_name)); - } - } - affinity_provider - .filter(|provider_name| self.providers.is_provider_for_input(provider_name, input_name)) - .map(OwnerProviderPin::Preferred) - } - - /// Provider a playback currently prefers on re-entry, for diagnostics and tests. - pub fn provider_affinity_for_owner(&self, session_owner: &str) -> Option> { - let session_owner = playback_lease_owner(session_owner); - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - leases.affinity_provider_for_owner(session_owner) - } - - /// A provisional lease only pins its provider while its allocation is active. - /// Once an unstarted request releases its slot, the next attempt may use another alias. - pub fn should_reuse_playback_provider(&self, session_owner: &str, provider_name: &Arc) -> bool { - let session_owner = playback_lease_owner(session_owner); - let _transition = self.lock_capacity_transition(); - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - let confirmed = leases - .lease_of_owner(session_owner) - .is_some_and(|lease| lease.provider_name == *provider_name && lease.state.is_confirmed()); - drop(leases); - confirmed || self.has_active_owner_for_provider(provider_name, session_owner) - } - - /// True when `session_owner` already backs a live allocation on `provider_name`, - /// without materialising the full owner set for a membership test. - fn has_active_owner_for_provider(&self, provider_name: &Arc, session_owner: &str) -> bool { - let connections = self.read_connections(); - connections.by_owner.get(session_owner).is_some_and(|alloc_ids| { - alloc_ids.iter().any(|id| { - if let Some(info) = connections.single.get(id) { - info.allocation.get_provider_name().as_ref() == Some(provider_name) - } else if let Some(key) = connections.shared.shared_by_allocation_id.get(id) { - connections - .shared - .by_key - .get(key) - .is_some_and(|shared| shared.allocation.get_provider_name().as_ref() == Some(provider_name)) - } else { - false - } - }) - }) - } - - fn active_reservation_owners(&self, provider_name: &Arc) -> HashSet> { - let connections = self.read_connections(); - let mut owners = HashSet::new(); - if let Some(alloc_ids) = connections.by_provider.get(provider_name) { - for id in alloc_ids { - if let Some(info) = connections.single.get(id) { - if let Some(owner) = info.session_owner.as_ref() { - owners.insert(Arc::clone(owner)); - } - } else if let Some(key) = connections.shared.shared_by_allocation_id.get(id) { - if let Some(shared) = connections.shared.by_key.get(key) { - if let Some(owner) = shared.session_owner.as_ref() { - owners.insert(Arc::clone(owner)); - } - } - } - } - } else { - for info in connections.single.values() { - if info.allocation.get_provider_name().as_ref() == Some(provider_name) { - if let Some(owner) = info.session_owner.as_ref() { - owners.insert(Arc::clone(owner)); - } - } - } - for shared in connections.shared.by_key.values() { - if shared.allocation.get_provider_name().as_ref() == Some(provider_name) { - if let Some(owner) = shared.session_owner.as_ref() { - owners.insert(Arc::clone(owner)); - } - } - } - } - owners - } - - fn reservation_capacity_usage( - &self, - provider_name: &Arc, - session_owner: Option<&str>, - ) -> Option<(usize, usize, usize)> { - let (current_connections, max_connections) = self.providers.provider_capacity(provider_name)?; - if max_connections == 0 { - return Some((current_connections, max_connections, 0)); - } - // Without any foreign reserving lease the owner analysis cannot change the result. - if !self.read_leases().has_foreign_reserved_lease(provider_name, session_owner) { - return Some((current_connections, max_connections, 0)); - } - let counted_owners = self.active_reservation_owners(provider_name); - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - let idle_foreign_reservations = leases.foreign_reserved_slots(provider_name, session_owner, &counted_owners); - - Some((current_connections, max_connections, idle_foreign_reservations)) - } - - /// Returns true when the next allocation would consume a slot kept for an idle reservation. - fn reserved_capacity_blocks_next(&self, provider_name: &Arc, session_owner: Option<&str>) -> bool { - let Some((current_connections, max_connections, idle_foreign_reservations)) = - self.reservation_capacity_usage(provider_name, session_owner) - else { - return true; - }; - max_connections > 0 - && idle_foreign_reservations > 0 - && current_connections.saturating_add(idle_foreign_reservations) >= max_connections - } - - /// Returns true when an already-counted candidate allocation consumed a slot kept for an idle reservation. - fn exceeds_reserved_capacity(&self, provider_name: &Arc, session_owner: Option<&str>) -> bool { - let Some((current_connections, max_connections, idle_foreign_reservations)) = - self.reservation_capacity_usage(provider_name, session_owner) - else { - return true; - }; - max_connections > 0 - && idle_foreign_reservations > 0 - && current_connections.saturating_add(idle_foreign_reservations) > max_connections - } - - /// Explains a reservation skip with the authoritative slot breakdown instead of - /// only the transport socket, so the decision can be audited from the logs. - fn log_reserved_capacity_skip(&self, provider_name: &Arc, params: &AcquireProviderParams<'_>) { - if !log::log_enabled!(log::Level::Debug) { - return; - } - let Some((current_connections, max_connections, foreign_reserved)) = - self.reservation_capacity_usage(provider_name, params.session_owner()) - else { - debug_if_enabled!( - "Skipping reserved provider {} (reason={}, capacity=unknown, peer_addr={}, request_id={}, playback_kind={})", - sanitize_sensitive_info(provider_name), - PlaybackSelectionReason::ReservedCapacity, - sanitize_sensitive_info(¶ms.addr.to_string()), - params.lease.map_or_else(String::new, |lease| lease.request_id.to_string()), - params.lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string()) - ); - return; - }; - let usage = self.provider_lease_usage(provider_name); - debug_if_enabled!( - "Skipping reserved provider {} (reason={}, current={}, max={}, foreign_reserved={}, active_slots={}, starting_slots={}, idle_slots={}, peer_addr={}, request_id={}, playback_kind={})", - sanitize_sensitive_info(provider_name), - PlaybackSelectionReason::ReservedCapacity, - current_connections, - max_connections, - foreign_reserved, - usage.active, - usage.starting, - usage.idle, - sanitize_sensitive_info(¶ms.addr.to_string()), - params.lease.map_or_else(String::new, |lease| lease.request_id.to_string()), - params.lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string()) - ); - } - - fn reserved_provider_names_for_other( - &self, - input_name: &Arc, - session_owner: Option<&str>, - ) -> HashSet> { - let mut reserved = HashSet::new(); - for provider_name in self.providers.provider_names_for_input(input_name) { - if self.has_foreign_reservation(&provider_name, session_owner) { - reserved.insert(provider_name); - } - } - reserved - } - - pub fn refresh_provider_reservation(&self, provider_name: &Arc, session_owner: &str, ttl_secs: u64) { - self.refresh_identified_provider_reservation(provider_name, session_owner, PlaybackKind::LiveHls, ttl_secs); - } - - /// Recreates or renews a provider reservation for an owner with an explicit kind. - /// - /// Unlike [`Self::refresh_adaptive_playback_lease`] this recreates the lease when it - /// no longer exists, which is what a failed preemption needs to restore a victim - /// reservation that [`Self::clear_identified_provider_reservation`] already removed. - pub fn refresh_identified_provider_reservation( - &self, - provider_name: &Arc, - session_owner: &str, - kind: PlaybackKind, - ttl_secs: u64, - ) { - let session_owner = playback_lease_owner(session_owner); - let _transition = self.lock_capacity_transition(); - if self.is_shutting_down.load(Ordering::Acquire) { - return; - } - let granted_ttl = self.renewal_granted_ttl(provider_name, session_owner, ttl_secs); - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - if ttl_secs == 0 { - leases.release_owner(session_owner); - return; - } - if leases.renew_current_owner(session_owner, provider_name, kind, granted_ttl).is_none() { - let req_id = PlaybackRequestId::next(); - let _ = leases.begin_owner(session_owner, provider_name, kind, req_id); - leases.renew_identified_owner(session_owner, provider_name, kind, req_id, granted_ttl); - } - } - - /// Renews the lease of an adaptive (HLS/DASH/Catchup) playback. - /// - /// Adaptive playback consists of many short requests, so the lease must stay - /// reconnect-capable: between two segment requests the slot is kept as an idle - /// lease on the same provider instead of being released and reselected. - pub fn refresh_adaptive_playback_lease( - &self, - provider_name: &Arc, - session_owner: &str, - kind: PlaybackKind, - ttl_secs: u64, - ) { - let session_owner = playback_lease_owner(session_owner); - let _transition = self.lock_capacity_transition(); - if self.is_shutting_down.load(Ordering::Acquire) { - return; - } - let granted_ttl = self.renewal_granted_ttl(provider_name, session_owner, ttl_secs); - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - leases.renew_current_owner(session_owner, provider_name, kind, granted_ttl); - } - - /// Renews the lease of a playback. A zero TTL clears it. - pub fn refresh_playback_lease(&self, provider_name: &Arc, lease_ref: &PlaybackLeaseRef<'_>, ttl_secs: u64) { - let lease_ref = PlaybackLeaseRef { owner: lease_ref.provider_owner(), ..*lease_ref }; - let _transition = self.lock_capacity_transition(); - if self.is_shutting_down.load(Ordering::Acquire) { - return; - } - let granted_ttl = self.renewal_granted_ttl(provider_name, lease_ref.owner, ttl_secs); - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - if ttl_secs == 0 { - leases.release_identified_owner(lease_ref.owner, lease_ref.request_id); - return; - } - leases.renew_identified_owner( - lease_ref.owner, - provider_name, - lease_ref.kind, - lease_ref.request_id, - granted_ttl, - ); - } - - /// The reconnect window a renewal may actually grant for `owner`. - /// - /// A lease that currently holds no reservation right must not gain one through a - /// mere activity refresh: the grant is re-checked against provider capacity with - /// the same admission rule as a confirmation, so a previously denied reservation - /// cannot be silently restored by a later refresh. - fn renewal_granted_ttl(&self, provider_name: &Arc, owner: &str, requested_ttl_secs: u64) -> u64 { - if requested_ttl_secs == 0 { - return 0; - } - let already_reserves = { - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - leases.lease_of_owner(owner).is_some_and(|lease| lease.state.is_confirmed() && lease.idle_ttl_secs > 0) - }; - if already_reserves || self.confirmation_reserve_allowed(provider_name, owner) { - requested_ttl_secs - } else { - 0 - } - } - - /// Clears an explicit owner. Public HLS tokens intentionally have no owner-wide - /// delete right: a delayed session cleanup must not erase a retry's shared lease. - /// HLS request cleanup uses request IDs or binding tags instead. - pub fn clear_provider_reservation(&self, session_owner: &str) { - if playback_lease_owner(session_owner) != session_owner { - return; - } - let _transition = self.lock_capacity_transition(); - let mut leases = self.write_leases(); - leases.forget_affinity(session_owner); - leases.release_owner(session_owner); - } - - /// Administrative end of the playback behind a session that was actually terminated - /// (terminate endpoint, kick). - /// - /// A public HLS token shares its stable owner with the retries of the same - /// client/user/channel, so it only ends the binding it acquired itself. A stale - /// token therefore never removes the lease or provider preference of a newer - /// binding. A bare owner keeps the owner-wide release semantics. - pub fn terminate_identified_playback_owner(&self, session_token: &str) -> bool { - let owner = playback_lease_owner(session_token); - let _transition = self.lock_capacity_transition(); - let mut leases = self.write_leases(); - if owner == session_token { - leases.forget_affinity(owner); - return leases.release_owner(owner).is_some(); - } - leases.terminate_identified_token(owner, session_token) - } - - /// Ends the provider preference of a failed playback, but only while it still - /// belongs to the exact binding that failed. A delayed failure of an older binding - /// leaves the successor's preference intact. - pub fn forget_identified_provider_affinity( - &self, - session_owner: &str, - provider_name: &Arc, - binding_tag: ProviderBindingTag, - ) -> bool { - let session_owner = playback_lease_owner(session_owner); - let _transition = self.lock_capacity_transition(); - self.write_leases().forget_identified_affinity(session_owner, provider_name, binding_tag) - } - - pub fn clear_identified_provider_reservation( - &self, - session_owner: &str, - provider_name: &Arc, - binding_tag: Option, - ) { - let session_owner = playback_lease_owner(session_owner); - // A delayed clear without the exact binding tag has no delete right: it must - // not fall back to an owner-wide release and erase a successor lease. - let Some(binding_tag) = binding_tag else { - return; - }; - let _transition = self.lock_capacity_transition(); - let mut leases = self.write_leases(); - leases.release_matching_owner(session_owner, provider_name, binding_tag); - } - - /// Confirms real media activity for a playback lease. Only a confirmed lease may - /// outlive its request as a reconnect slot. - pub fn confirm_playback_activity(&self, owner: &str) -> Option { self.confirm_owner(owner, None) } - - /// Confirms real media activity for a specific request ID of a playback lease. - pub fn confirm_identified_playback_activity( - &self, - owner: &str, - request_id: PlaybackRequestId, - ) -> Option { - self.confirm_owner(owner, Some(request_id)) - } - - /// Records media activity under the capacity transition. The confirmation only - /// grants a reservation right when the lease already backs a live allocation or a - /// free slot remains; otherwise the media bytes are recorded without a reservation - /// so a late cache confirmation cannot over-commit a provider at its limit. - fn confirm_owner(&self, owner: &str, request_id: Option) -> Option { - let owner = playback_lease_owner(owner); - let _transition = self.lock_capacity_transition(); - let (provider_name, wants_reservation) = { - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - let provider_name = leases.provider_for_owner(owner)?; - let wants_reservation = leases.lease_of_owner(owner).is_some_and(|lease| lease.idle_ttl_secs > 0); - (provider_name, wants_reservation) - }; - let reserve = !wants_reservation || self.confirmation_reserve_allowed(&provider_name, owner); - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - match (request_id, reserve) { - (Some(request_id), true) => leases.confirm_identified_owner(owner, request_id), - (Some(request_id), false) => leases.confirm_identified_owner_without_reservation(owner, request_id), - (None, true) => leases.confirm_owner(owner), - (None, false) => leases.confirm_owner_without_reservation(owner), - } - } - - /// True when confirming `session_owner`'s lease may reserve a provider slot: the - /// lease either already backs a live allocation, or a free slot remains below the - /// configured maximum after foreign reservations are counted. - fn confirmation_reserve_allowed(&self, provider_name: &Arc, session_owner: &str) -> bool { - if self.has_active_owner_for_provider(provider_name, session_owner) { - return true; - } - let Some((current, max, foreign)) = self.reservation_capacity_usage(provider_name, Some(session_owner)) else { - return false; - }; - max == 0 || current.saturating_add(foreign) < max - } - - /// Ends a playback request and applies the outcome-specific lease policy. - /// - /// `idle_ttl_secs` overrides the reconnect window stored on the lease; `None` - /// keeps the window the endpoint configured when it renewed the lease. - pub fn finish_playback_request(&self, owner: &str, outcome: PlaybackRequestOutcome, idle_ttl_secs: Option) { - self.finish_playback_request_inner(owner, None, outcome, idle_ttl_secs); - } - - pub fn finish_identified_playback_request( - &self, - owner: &str, - request_id: PlaybackRequestId, - outcome: PlaybackRequestOutcome, - ) { - self.finish_playback_request_inner(owner, Some(request_id), outcome, None); - } - - fn finish_playback_request_inner( - &self, - owner: &str, - request_id: Option, - outcome: PlaybackRequestOutcome, - idle_ttl_secs: Option, - ) { - let owner = playback_lease_owner(owner); - let _transition = self.lock_capacity_transition(); - // A delayed completion must not end a newer request of this playback. - // Only active connections from a DIFFERENT request ID (or shared streams) prevent lease finish. - let connections = self.read_connections(); - let has_other_active_connection = connections.by_owner.get(owner).is_some_and(|alloc_ids| { - alloc_ids.iter().any(|id| { - connections - .single - .get(id) - .is_some_and(|info| request_id.is_none() || info.playback_request_id != request_id) - || connections.shared.shared_by_allocation_id.contains_key(id) - }) - }); - if has_other_active_connection { - if let Some(req_id) = request_id { - let mut leases = self.write_leases(); - leases.detach_identified_request(owner, req_id); - } - return; - } - drop(connections); - let mut leases = self.write_leases(); - if let Some(req_id) = request_id { - leases.finish_identified_owner(owner, req_id, outcome, idle_ttl_secs); - } else { - leases.finish_owner(owner, outcome, idle_ttl_secs); - } - } - - /// Snapshot of the logical slots a provider holds, for logs and UI metrics. - pub fn provider_lease_usage(&self, provider_name: &Arc) -> ProviderLeaseUsage { - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - leases.usage(provider_name) - } - - /// Removes every expired starting/idle lease. Runs from the session GC task. - pub fn prune_expired_leases_now(&self) { - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - } - - /// The provider lease's binding tag for an owner. The HLS layer captures this - /// from the acquired handle (via `ProviderHandle::binding_tag`) and passes it - /// back to [`Self::clear_identified_provider_reservation`] so a stale detach - /// cannot delete a successor lease on the same account. - pub fn binding_tag_for_owner(&self, owner: &str) -> Option { - let owner = playback_lease_owner(owner); - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - leases.lease_of_owner(owner).map(|lease| ProviderBindingTag::new(lease.id, lease.binding_generation)) - } - - fn acquire_exact_connection_inner_no_preempt( - &self, - provider_name: &Arc, - allow_grace: bool, - params: &AcquireProviderParams<'_>, - ) -> Option { - let mut allocation = ProviderAllocationGuard::new( - self.providers.acquire_exact_connection_with_grace_override(provider_name, allow_grace), - ); - if matches!(allocation.allocation(), ProviderAllocation::Exhausted) { - return None; - } - if self.exceeds_reserved_capacity(provider_name, params.session_owner()) { - return None; - } - Some(self.register_allocation(allocation.take(), params)) - } - - fn acquire_exact_connection_inner( - &self, - provider_name: &Arc, - allow_grace: bool, - params: &AcquireProviderParams<'_>, - ) -> Option { - if let Some(handle) = self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, params) { - return Some(handle); - } - if let Some(preempted_alloc) = self.try_preempt_connection( - provider_name, - params.priority, - allow_grace, - params.kind, - params.session_owner(), - ) { - return Some(self.register_allocation(preempted_alloc, params)); - } - None - } - - fn finalize_lineup_allocation( - &self, - input_name: &Arc, - allow_grace: bool, - mut allocation: ProviderAllocationGuard, - params: &AcquireProviderParams<'_>, - ) -> ProviderHandle { - if matches!(allocation.allocation(), ProviderAllocation::GracePeriod(_)) - && self.evict_lower_priority_on_input(input_name, params.priority, params.kind, params.session_owner()) - { - let evicted_on_same = - !self.providers.is_over_limit(&allocation.allocation().get_provider_name().unwrap_or_default()); - if !evicted_on_same { - let mut new_alloc = ProviderAllocationGuard::new( - self.providers.acquire_connection_with_grace_override(input_name, allow_grace), - ); - if !matches!(new_alloc.allocation(), ProviderAllocation::Exhausted) { - if let Some(provider_name) = new_alloc.allocation().get_provider_name() { - if !self.exceeds_reserved_capacity(&provider_name, params.session_owner()) { - return self.register_allocation(new_alloc.take(), params); - } - } - } - } - } - - self.register_allocation(allocation.take(), params) - } - - fn select_victim_from_index( - &self, - index: &HashMap, BTreeMap>, - input_name: &Arc, - requester_priority: Option, - reserved_providers: &HashSet>, - ) -> Option { - let mut victim = None; - - for (prov_name, tree) in index { - if (!self.providers.is_provider_for_input(prov_name, input_name) && prov_name != input_name) - || reserved_providers.contains(prov_name) - { - continue; - } - - let Some(((victim_priority, Reverse(created_at), allocation_id), owner)) = tree.iter().next_back() else { - continue; - }; - - if let Some(req_prio) = requester_priority { - if *victim_priority <= req_prio { - continue; - } - } - - let candidate = (*owner, *allocation_id, *victim_priority, *created_at); - if is_better_preemption_candidate(victim, candidate) { - victim = Some(candidate); - } - } - - victim - } - - fn is_normal_preemption_candidate( - connections: &Connections, - provider_name: &Arc, - candidate: PreemptionCandidate, - ) -> bool { - match candidate.0 { - PriorityOwner::Single(alloc_id) => connections.single.get(&alloc_id).is_some_and(|info| { - info.kind == ConnectionKind::Normal - && info.lifecycle != ConnectionLifecycle::Closing - && info.allocation.get_provider_name().as_ref() == Some(provider_name) - }), - PriorityOwner::Shared(shared_id) => connections - .shared - .shared_by_allocation_id - .get(&shared_id) - .and_then(|key| connections.shared.by_key.get(key)) - .is_some_and(|shared| { - shared.allocation_id == candidate.1 - && shared.kind == ConnectionKind::Normal - && shared.allocation.get_provider_name().as_ref() == Some(provider_name) - }), - } - } - - fn select_preemption_candidate( - &self, - connections: &Connections, - input_name: &Arc, - new_priority: i8, - kind_needed: ConnectionKind, - reserved_providers: &HashSet>, - ) -> Option { - match kind_needed { - ConnectionKind::Normal => { - let soft_victim = self.select_victim_from_index( - &connections.soft_priority_index, - input_name, - None, - reserved_providers, - ); - if soft_victim.is_some() { - return soft_victim; - } - - let mut victim = None; - for (prov_name, tree) in &connections.priority_index { - if (!self.providers.is_provider_for_input(prov_name, input_name) && prov_name != input_name) - || reserved_providers.contains(prov_name) - { - continue; - } - - let Some(((victim_priority, Reverse(created_at), allocation_id), owner)) = tree.iter().next_back() - else { - continue; - }; - if *victim_priority <= new_priority { - continue; - } - let candidate = (*owner, *allocation_id, *victim_priority, *created_at); - if !ActiveProviderManager::is_normal_preemption_candidate(connections, prov_name, candidate) { - continue; - } - if is_better_preemption_candidate(victim, candidate) { - victim = Some(candidate); - } - } - victim - } - ConnectionKind::Soft => self.select_victim_from_index( - &connections.soft_priority_index, - input_name, - Some(new_priority), - reserved_providers, - ), - } - } - - fn acquire_connection_inner_no_preempt( - &self, - provider_or_input_name: &Arc, - allow_grace: bool, - params: &AcquireProviderParams<'_>, - ) -> Option { - if let Some(owner) = params.session_owner() { - match self.owner_provider_pin(provider_or_input_name, owner) { - Some(OwnerProviderPin::Reserved(reserved_provider)) => { - return self.acquire_exact_connection_inner_no_preempt(&reserved_provider, allow_grace, params); - } - // A continuing playback whose reconnect lease lapsed during a request gap - // returns to its provider before priority selection, but only into free - // capacity: the lineup prefers any free slot over a grace over-allocation, - // so the preference must not force one either. - Some(OwnerProviderPin::Preferred(affinity_provider)) => { - if let Some(handle) = - self.acquire_exact_connection_inner_no_preempt(&affinity_provider, false, params) - { - debug_if_enabled!( - "Provider affinity kept: provider={} owner={} playback_kind={}", - sanitize_sensitive_info(&affinity_provider), - sanitize_sensitive_info(owner), - params.lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string()) - ); - return Some(handle); - } - debug_if_enabled!( - "Provider affinity unavailable, using lineup: provider={} owner={}", - sanitize_sensitive_info(&affinity_provider), - sanitize_sensitive_info(owner) - ); - } - None => {} - } - } - - let candidate_count = self.providers.provider_names_for_input(provider_or_input_name).len(); - let attempts = candidate_count.max(1); - let mut skipped_reserved = HashSet::new(); - for _ in 0..attempts { - let allocation = - ProviderAllocationGuard::new(self.providers.acquire_connection_with_grace_override_excluding( - provider_or_input_name, - allow_grace, - &skipped_reserved, - )); - if matches!(allocation.allocation(), ProviderAllocation::Exhausted) { - break; - } - if let Some(provider_name) = allocation.allocation().get_provider_name() { - if self.exceeds_reserved_capacity(&provider_name, params.session_owner()) { - self.log_reserved_capacity_skip(&provider_name, params); - skipped_reserved.insert(provider_name); - if skipped_reserved.len() >= attempts { - break; - } - continue; - } - } - return Some(self.finalize_lineup_allocation(provider_or_input_name, allow_grace, allocation, params)); - } - - None - } - - fn acquire_connection_inner( - &self, - provider_or_input_name: &Arc, - allow_grace: bool, - params: &AcquireProviderParams<'_>, - ) -> Option { - if self.is_shutting_down.load(Ordering::Acquire) { - return None; - } - let _transition = self.lock_capacity_transition(); - if self.is_shutting_down.load(Ordering::Acquire) { - return None; - } - if let Some(handle) = self.acquire_connection_inner_no_preempt(provider_or_input_name, allow_grace, params) { - return Some(handle); - } - - if let Some(preempted_alloc) = self.try_preempt_connection( - provider_or_input_name, - params.priority, - allow_grace, - params.kind, - params.session_owner(), - ) { - return Some(self.register_allocation(preempted_alloc, params)); - } - - None - } - - fn register_allocation( - &self, - allocation: ProviderAllocation, - params: &AcquireProviderParams<'_>, - ) -> ProviderHandle { - let AcquireProviderParams { addr, priority, kind, lease } = *params; - let session_owner = lease.map(|lease| lease.provider_owner()); - let provider_name = allocation.get_provider_name().unwrap_or_default(); - let allocation_id = self.next_allocation_id.fetch_add(1, Ordering::Relaxed); - let cancel_token = CancellationToken::new(); - let completion_token = CancellationToken::new(); - let close_reason = Arc::new(AtomicU8::new(ProviderCloseReason::Unspecified as u8)); - let now = Instant::now(); - let is_unlimited = allocation.is_unlimited_provider(); - - // Claim the playback's capacity slot lease in its own lock scope so the - // connection and lease locks are never nested. The lease starts unconfirmed; - // it only reserves capacity against other playbacks once real media activity - // confirms it, which is what keeps abandoned manifest starts from blocking. - let binding_tag = if let Some(lease) = lease { - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - let id = leases.begin_owner(lease.provider_owner(), &provider_name, lease.kind, lease.request_id); - if lease.owner != lease.provider_owner() { - leases.attach_request_token(id, lease.request_id, lease.owner); - } - let generation = leases.lease(id).map_or(1, |lease| lease.binding_generation); - debug_if_enabled!( - "Playback lease began: provider={} owner={} kind={} request_id={} lease_id={} generation={} state=starting", - sanitize_sensitive_info(&provider_name), - sanitize_sensitive_info(lease.provider_owner()), - lease.kind, - lease.request_id, - id, - generation - ); - Some(ProviderBindingTag::new(id, generation)) - } else { - None - }; - - let session_owner_arc = session_owner.map(Arc::from); - let mut connections = self.write_connections(); - connections.single.insert( - allocation_id, - ActiveConnectionInfo { - allocation_id, - client_addr: *addr, - allocation: allocation.clone(), - cancel_token: cancel_token.clone(), - completion_token: completion_token.clone(), - close_reason: Arc::clone(&close_reason), - lifecycle: ConnectionLifecycle::Active, - has_body_owner: false, - reaper_spawned: false, - open_generation: 0, - created_at: now, - priority, - kind, - session_owner: session_owner_arc.clone(), - request_owner: lease.map(|lease| { - session_owner_arc - .as_ref() - .filter(|owner| owner.as_ref() == lease.owner) - .map_or_else(|| Arc::from(lease.owner), Arc::clone) - }), - playback_request_id: lease.map(|lease| lease.request_id), - }, - ); - connections.single_by_addr.entry(*addr).or_default().insert(allocation_id); - Self::index_owner(&mut connections, allocation_id, session_owner_arc.as_ref()); - - // Unlimited providers are not subject to preemption, so we deliberately - // skip populating the by_provider / priority_index / soft_priority_index - // indices. The connection itself is still tracked via `single`, so all - // capacity and lifecycle invariants hold. Without this skip, an - // unlimited provider's connection can be selected as a preemption victim - // even though it cannot be exhausted, contradicting the configured - // `max_connections: 0` semantics. - if !is_unlimited { - connections.by_provider.entry(provider_name.clone()).or_default().insert(allocation_id); - Self::upsert_priority_entry( - &mut connections, - &provider_name, - (priority, Reverse(now), allocation_id), - PriorityOwner::Single(allocation_id), - kind, - ); - } - - debug_if_enabled!( - "Added provider connection {provider_name:?} (reason={}, prio={priority}, kind={kind:?}, unlimited={is_unlimited}, allocation_id={allocation_id}, lease_id={}, request_id={}, playback_kind={}, peer_addr={})", - PlaybackSelectionReason::NewPriorityAllocation, - binding_tag.map_or_else(String::new, |tag| tag.lease_id.to_string()), - lease.map_or_else(String::new, |lease| lease.request_id.to_string()), - lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string()), - sanitize_sensitive_info(&addr.to_string()) - ); - let mut handle = ProviderHandle::new(*addr, allocation_id, allocation, Some(cancel_token)); - handle.completion_token = Some(completion_token); - handle.close_reason = close_reason; - handle.playback_request_id = lease.map(|lease| lease.request_id); - handle.binding_tag = binding_tag; - handle - } - - #[allow(clippy::too_many_lines)] - /// Evict a single lower-priority connection across the entire input lineup - /// (all provider aliases). Used when a `GracePeriod` allocation was granted. - /// Returns true if a victim was successfully evicted. - fn evict_lower_priority_on_input( - &self, - input_name: &Arc, - new_priority: i8, - kind_needed: ConnectionKind, - session_owner: Option<&str>, - ) -> bool { - let reserved_providers = self.reserved_provider_names_for_other(input_name, session_owner); - - let victim = { - let connections = self.read_connections(); - self.select_preemption_candidate(&connections, input_name, new_priority, kind_needed, &reserved_providers) - }; - - let Some((owner, alloc_id, v_prio, victim_created_at)) = victim else { - return false; - }; - match owner { - PriorityOwner::Shared(shared_id) => { - debug_if_enabled!( - "Grace-evicting shared connection (allocation_id={shared_id}, prio={v_prio}) on input {} for higher priority request (prio={})", - sanitize_sensitive_info(input_name), - new_priority - ); - - let released = { - let mut connections = self.write_connections(); - let Some(key) = connections.shared.shared_by_allocation_id.get(&shared_id).cloned() else { - return false; - }; - - let still_match = connections.shared.by_key.get(&key).is_some_and(|shared| { - shared.allocation_id == alloc_id - && shared.priority == v_prio - && shared.created_at == victim_created_at - }); - if !still_match { - return false; - } - - if let Some(shared) = connections.shared.by_key.remove(&key) { - connections.shared.shared_by_allocation_id.remove(&shared.allocation_id); - Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref()); - for subscriber_id in shared.connections.keys() { - connections.shared.key_by_subscriber.remove(subscriber_id); - } - if let Some(name) = shared.allocation.get_provider_name() { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&shared.allocation_id); - } - Self::remove_priority_entry( - &mut connections, - &name, - &(v_prio, Reverse(victim_created_at), alloc_id), - shared.kind, - ); - } - Some((key, shared.allocation, shared.cancel_token)) - } else { - return false; - } - }; - if let Some((stream_url, allocation, cancel_token)) = released { - if let Some(token) = cancel_token { - token.cancel(); - } - if let Some(ssm) = self.shared_stream_manager.get().and_then(Weak::upgrade) { - tokio::spawn(async move { - ssm.teardown_preempted_stream(&stream_url, alloc_id).await; - allocation.release(); - }); - return false; - } - allocation.release(); - } - } - PriorityOwner::Single(victim_alloc_id) => { - let mut connections = self.write_connections(); - if let Some(info) = connections.single.get(&victim_alloc_id) { - if info.priority != v_prio || info.created_at != victim_created_at { - return false; - } - } - if let Some(info) = connections.single.get_mut(&victim_alloc_id) { - debug_if_enabled!( - "Grace-evicting single connection from {} (prio={}) on input {} for higher priority request (prio={})", - sanitize_sensitive_info(&info.client_addr.to_string()), - v_prio, - sanitize_sensitive_info(input_name), - new_priority - ); - info.close_reason.store(ProviderCloseReason::PriorityPreempted as u8, Ordering::Release); - } - let action = ActiveProviderManagerCore::plan_single_release_locked(&mut connections, victim_alloc_id); - if let ReleaseAction::Wait(alloc_id, gen, completion_token) = action { - let shutdown_token = self.shutdown_token.clone(); - let core = Arc::clone(&self.core); - tokio::spawn(async move { - tokio::select! { - () = shutdown_token.cancelled() => {}, - () = completion_token.cancelled() => { - core.complete_release_with_generation(alloc_id, Some(gen)); - } - } - }); - return false; - } - } - } - - true - } - - fn try_acquire_allocation_after_freed( - &self, - input_name: &Arc, - allow_grace: bool, - reserved_providers: &HashSet>, - session_owner: Option<&str>, - ) -> Option { - let attempts = self.providers.provider_names_for_input(input_name).len().max(1); - let mut excluded_providers = reserved_providers.clone(); - for _ in 0..attempts { - let allocation = self.providers.acquire_connection_with_grace_override_excluding( - input_name, - allow_grace, - &excluded_providers, - ); - if matches!(allocation, ProviderAllocation::Exhausted) { - break; - } - if let Some(provider_name) = allocation.get_provider_name() { - if self.exceeds_reserved_capacity(&provider_name, session_owner) { - excluded_providers.insert(provider_name); - allocation.release(); - continue; - } - } - return Some(allocation); - } - None - } - - #[allow(clippy::too_many_lines)] - fn try_preempt_connection_outcome( - &self, - input_name: &Arc, - new_priority: i8, - allow_grace: bool, - kind_needed: ConnectionKind, - session_owner: Option<&str>, - ) -> PreemptionOutcome { - let reserved_providers = self.reserved_provider_names_for_other(input_name, session_owner); - let victim = { - let connections = self.read_connections(); - self.select_preemption_candidate(&connections, input_name, new_priority, kind_needed, &reserved_providers) - }; - - let Some((owner, alloc_id, v_prio, victim_created_at)) = victim else { - return PreemptionOutcome::Exhausted; - }; - - match owner { - PriorityOwner::Shared(shared_id) => { - debug_if_enabled!( - "Preempting shared connection (allocation_id={shared_id}, prio={v_prio}) for higher priority request (prio={new_priority})" - ); - let released_shared_allocation = { - let mut connections = self.write_connections(); - let Some(key) = connections.shared.shared_by_allocation_id.get(&shared_id).cloned() else { - return PreemptionOutcome::Exhausted; - }; - - let still_match = connections.shared.by_key.get(&key).is_some_and(|shared| { - shared.allocation_id == alloc_id - && shared.priority == v_prio - && shared.created_at == victim_created_at - }); - if !still_match { - return PreemptionOutcome::Exhausted; - } - - if let Some(shared) = connections.shared.by_key.remove(&key) { - connections.shared.shared_by_allocation_id.remove(&shared.allocation_id); - Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref()); - for subscriber_id in shared.connections.keys() { - connections.shared.key_by_subscriber.remove(subscriber_id); - } - - if let Some(name) = shared.allocation.get_provider_name() { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&shared.allocation_id); - } - Self::remove_priority_entry( - &mut connections, - &name, - &(v_prio, Reverse(victim_created_at), alloc_id), - shared.kind, - ); - } - Some((key, shared.allocation, shared.cancel_token)) - } else { - None - } - }; - - let Some((stream_url, allocation, cancel_token)) = released_shared_allocation else { - return PreemptionOutcome::Exhausted; - }; - - if let Some(token) = cancel_token { - token.cancel(); - } - if let Some(ssm) = self.shared_stream_manager.get().and_then(Weak::upgrade) { - // Signal completion after teardown finishes so async acquirers can wait for the - // actual upstream shutdown rather than receiving Exhausted immediately. - let done_token = CancellationToken::new(); - let done_signal = done_token.clone(); - tokio::spawn(async move { - ssm.teardown_preempted_stream(&stream_url, alloc_id).await; - allocation.release(); - done_signal.cancel(); - }); - PreemptionOutcome::PendingCompletion(None, done_token) - } else { - allocation.release(); - if let Some(alloc) = self.try_acquire_allocation_after_freed( - input_name, - allow_grace, - &reserved_providers, - session_owner, - ) { - PreemptionOutcome::Acquired(alloc) - } else { - PreemptionOutcome::Exhausted - } - } - } - PriorityOwner::Single(victim_alloc_id) => { - let (action, completion_token) = { - let mut connections = self.write_connections(); - if let Some(info) = connections.single.get(&victim_alloc_id) { - if info.priority != v_prio || info.created_at != victim_created_at { - return PreemptionOutcome::Exhausted; - } - } else { - return PreemptionOutcome::Exhausted; - } - if let Some(info) = connections.single.get_mut(&victim_alloc_id) { - debug_if_enabled!( - "Preempting single connection from {} (prio={v_prio}) for higher priority request (prio={new_priority})", - sanitize_sensitive_info(&info.client_addr.to_string()) - ); - info.close_reason.store(ProviderCloseReason::PriorityPreempted as u8, Ordering::Release); - } - let completion_token = - connections.single.get(&victim_alloc_id).map(|info| info.completion_token.clone()); - let action = - ActiveProviderManagerCore::plan_single_release_locked(&mut connections, victim_alloc_id); - (action, completion_token) - }; - - match action { - ReleaseAction::Wait(victim_id, gen, comp_token) => { - self.spawn_preemption_reaper(victim_id, gen, comp_token.clone()); - PreemptionOutcome::PendingCompletion(Some((victim_id, gen)), comp_token) - } - ReleaseAction::None => { - if let Some(token) = completion_token { - // Retrieve the current generation for the idempotent self-release path. - let victim_identity = { - let connections = self.read_connections(); - connections - .single - .get(&victim_alloc_id) - .map(|info| (victim_alloc_id, info.open_generation)) - }; - if let Some((victim_id, gen)) = victim_identity { - self.spawn_preemption_reaper(victim_id, gen, token.clone()); - } - PreemptionOutcome::PendingCompletion(victim_identity, token) - } else { - PreemptionOutcome::Exhausted - } - } - ReleaseAction::Immediate => { - if let Some(alloc) = self.try_acquire_allocation_after_freed( - input_name, - allow_grace, - &reserved_providers, - session_owner, - ) { - PreemptionOutcome::Acquired(alloc) - } else { - PreemptionOutcome::Exhausted - } - } - } - } - } - } - - fn try_preempt_connection( - &self, - input_name: &Arc, - new_priority: i8, - allow_grace: bool, - kind_needed: ConnectionKind, - session_owner: Option<&str>, - ) -> Option { - match self.try_preempt_connection_outcome(input_name, new_priority, allow_grace, kind_needed, session_owner) { - PreemptionOutcome::Acquired(alloc) => Some(alloc), - _ => None, - } - } - - pub fn acquire_exact_connection_with_grace( - &self, - provider_name: &Arc, - addr: &SocketAddr, - allow_grace: bool, - priority: i8, - kind: ConnectionKind, - ) -> Option { - self.acquire_exact_connection_with_grace_for_session(provider_name, addr, allow_grace, priority, kind, None) - } - - pub fn acquire_exact_connection_with_grace_for_session( - &self, - provider_name: &Arc, - addr: &SocketAddr, - allow_grace: bool, - priority: i8, - kind: ConnectionKind, - session_owner: Option<&str>, - ) -> Option { - if self.is_shutting_down.load(Ordering::Acquire) { - return None; - } - let _transition = self.lock_capacity_transition(); - if self.is_shutting_down.load(Ordering::Acquire) { - return None; - } - let lease = session_owner.map(PlaybackLeaseRef::for_session_owner); - self.acquire_exact_connection_inner( - provider_name, - allow_grace, - &AcquireProviderParams { addr, priority, kind, lease: lease.as_ref().copied() }, - ) - } - - pub fn force_exact_acquire_connection( - &self, - provider_name: &Arc, - addr: &SocketAddr, - priority: i8, - kind: ConnectionKind, - ) -> Option { - // Compatibility wrapper: keep the exact-provider behavior but do not over-allocate exhausted accounts. - self.acquire_exact_connection_with_grace(provider_name, addr, false, priority, kind) - } - - // Returns the next available provider connection - pub fn acquire_connection( - &self, - input_name: &Arc, - addr: &SocketAddr, - priority: i8, - kind: ConnectionKind, - ) -> Option { - self.acquire_connection_inner(input_name, true, &AcquireProviderParams { addr, priority, kind, lease: None }) - } - - /// Acquire a provider connection while explicitly controlling provider-side grace allocations. - pub fn acquire_connection_with_grace( - &self, - input_name: &Arc, - addr: &SocketAddr, - allow_grace: bool, - priority: i8, - kind: ConnectionKind, - ) -> Option { - self.acquire_connection_with_grace_for_session(input_name, addr, allow_grace, priority, kind, None) - } - - pub fn acquire_connection_with_grace_for_session( - &self, - input_name: &Arc, - addr: &SocketAddr, - allow_grace: bool, - priority: i8, - kind: ConnectionKind, - session_owner: Option<&str>, - ) -> Option { - self.acquire_connection_with_lease_for_session( - input_name, - addr, - allow_grace, - priority, - kind, - session_owner.map(PlaybackLeaseRef::for_session_owner), - ) - } - - /// Lineup acquisition with an explicit playback lease identity. - pub fn acquire_connection_with_lease_for_session( - &self, - input_name: &Arc, - addr: &SocketAddr, - allow_grace: bool, - priority: i8, - kind: ConnectionKind, - lease: Option>, - ) -> Option { - self.acquire_connection_inner(input_name, allow_grace, &AcquireProviderParams { addr, priority, kind, lease }) - } - - /// Lineup acquisition with an explicit playback lease identity, awaiting preemption completion if needed. - pub async fn acquire_connection_with_lease_for_session_await( - &self, - input_name: &Arc, - addr: &SocketAddr, - allow_grace: bool, - priority: i8, - kind: ConnectionKind, - lease: Option>, - ) -> Option { - let params = AcquireProviderParams { addr, priority, kind, lease }; - let outcome = { - if self.is_shutting_down.load(Ordering::Acquire) { - return None; - } - let _transition = self.lock_capacity_transition(); - if self.is_shutting_down.load(Ordering::Acquire) { - return None; - } - if let Some(handle) = self.acquire_connection_inner_no_preempt(input_name, allow_grace, ¶ms) { - return Some(handle); - } - self.try_preempt_connection_outcome( - input_name, - params.priority, - allow_grace, - params.kind, - params.session_owner(), - ) - }; - - match outcome { - PreemptionOutcome::Acquired(alloc) => { - let _transition = self.lock_capacity_transition(); - Some(self.register_allocation(alloc, ¶ms)) - } - PreemptionOutcome::PendingCompletion(victim_identity, completion_token) => { - let _ = tokio::time::timeout(PREEMPTION_COMPLETION_TIMEOUT, completion_token.cancelled()).await; - if self.is_shutting_down.load(Ordering::Acquire) { - return None; - } - let _transition = self.lock_capacity_transition(); - // Idempotently free the victim's slot in case the background reaper has not yet run. - // complete_release_locked is a no-op if the slot was already freed or the generation - // no longer matches. - if let Some((victim_alloc_id, victim_gen)) = victim_identity { - let mut connections = self.write_connections(); - ActiveProviderManagerCore::complete_release_locked( - &mut connections, - victim_alloc_id, - Some(victim_gen), - ); - } - self.acquire_connection_inner_no_preempt(input_name, allow_grace, ¶ms) - } - PreemptionOutcome::Exhausted => None, - } - } - - /// Exact-provider acquisition with an explicit playback lease identity. - pub fn acquire_exact_connection_with_lease_for_session( - &self, - provider_name: &Arc, - addr: &SocketAddr, - allow_grace: bool, - priority: i8, - kind: ConnectionKind, - lease: Option>, - ) -> Option { - let _transition = self.lock_capacity_transition(); - self.acquire_exact_connection_inner( - provider_name, - allow_grace, - &AcquireProviderParams { addr, priority, kind, lease }, - ) - } - - /// Exact-provider acquisition with an explicit playback lease identity, awaiting preemption completion if needed. - pub async fn acquire_exact_connection_with_lease_for_session_await( - &self, - provider_name: &Arc, - addr: &SocketAddr, - allow_grace: bool, - priority: i8, - kind: ConnectionKind, - lease: Option>, - ) -> Option { - self.acquire_exact_connection_with_lease_for_session_until( - provider_name, - addr, - allow_grace, - priority, - kind, - lease, - None, - ) - .await - } - - /// Frees a preempted victim's slot once its body completes, or after - /// `PREEMPTION_COMPLETION_TIMEOUT` at the latest. It runs independently of the preempting - /// request, so a request dropped while waiting (client disconnect, deadline, cancelled - /// future) cannot leave the victim's slot occupied. - fn spawn_preemption_reaper(&self, victim_id: AllocationId, generation: u64, completion_token: CancellationToken) { - let Ok(runtime) = tokio::runtime::Handle::try_current() else { - return; - }; - let shutdown_token = self.shutdown_token.clone(); - let core = Arc::clone(&self.core); - runtime.spawn(async move { - tokio::select! { - () = shutdown_token.cancelled() => {} - _ = tokio::time::timeout(PREEMPTION_COMPLETION_TIMEOUT, completion_token.cancelled()) => { - core.complete_release_with_generation(victim_id, Some(generation)); - } - } - }); - } - - /// Earliest instant at which a playback lease may expire and free reserved capacity, after - /// pruning the ones already expired. Lease expiry is time based and sends no notification. - pub fn next_lease_expiry(&self) -> Option { - let mut leases = self.write_leases(); - Self::prune_expired_leases(&mut leases); - leases.next_expiry() - } - - /// Like [`Self::acquire_exact_connection_with_lease_for_session_await`], but never waits past - /// `deadline`. A preemption that outlives the deadline is finished by its reaper. - #[allow(clippy::too_many_arguments)] - pub async fn acquire_exact_connection_with_lease_for_session_until( - &self, - provider_name: &Arc, - addr: &SocketAddr, - allow_grace: bool, - priority: i8, - kind: ConnectionKind, - lease: Option>, - deadline: Option, - ) -> Option { - let params = AcquireProviderParams { addr, priority, kind, lease }; - let outcome = { - if self.is_shutting_down.load(Ordering::Acquire) { - return None; - } - let _transition = self.lock_capacity_transition(); - if self.is_shutting_down.load(Ordering::Acquire) { - return None; - } - if let Some(handle) = self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, ¶ms) { - return Some(handle); - } - self.try_preempt_connection_outcome( - provider_name, - params.priority, - allow_grace, - params.kind, - params.session_owner(), - ) - }; - - match outcome { - PreemptionOutcome::Acquired(alloc) => { - let _transition = self.lock_capacity_transition(); - Some(self.register_allocation(alloc, ¶ms)) - } - PreemptionOutcome::PendingCompletion(victim_identity, completion_token) => { - let completion_deadline = TokioInstant::now() + PREEMPTION_COMPLETION_TIMEOUT; - let wait_until = deadline.map_or(completion_deadline, |deadline| deadline.min(completion_deadline)); - let completed = tokio::time::timeout_at(wait_until, completion_token.cancelled()).await.is_ok(); - if self.is_shutting_down.load(Ordering::Acquire) { - return None; - } - if !completed && wait_until < completion_deadline { - // The preemption reaper forces the victim release on time without this caller. - return None; - } - let _transition = self.lock_capacity_transition(); - if let Some((victim_alloc_id, victim_gen)) = victim_identity { - let mut connections = self.write_connections(); - ActiveProviderManagerCore::complete_release_locked( - &mut connections, - victim_alloc_id, - Some(victim_gen), - ); - } - self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, ¶ms) - } - PreemptionOutcome::Exhausted => None, - } - } - - /// Acquire a provider connection for probe tasks with configurable priority. - /// Probes never consume grace capacity. - pub fn acquire_connection_for_probe(&self, input_name: &Arc, priority: i8) -> Option { - self.acquire_connection_inner( - input_name, - false, - &AcquireProviderParams { addr: &DUMMY_ADDR, priority, kind: ConnectionKind::Normal, lease: None }, - ) - } - - /// Acquire a provider connection for background transfers (downloads/recordings). - /// Transfers participate in the same provider priority/preemption model as normal - /// streams, but they never consume grace capacity and wait externally on notifications. - pub fn acquire_connection_for_download(&self, input_name: &Arc, priority: i8) -> Option { - self.acquire_connection_inner( - input_name, - false, - &AcquireProviderParams { addr: &DUMMY_ADDR, priority, kind: ConnectionKind::Normal, lease: None }, - ) - } - - /// Transfers an already reserved recording allocation into the internal - /// playback request that opens the provider body. The worker keeps its - /// original handle so preemption can still stop the recording; releases - /// are allocation-id based and therefore remain idempotent. - pub fn claim_download_connection( - &self, - allocation_id: AllocationId, - input_name: &Arc, - ) -> Option { - let _transition = self.lock_capacity_transition(); - let mut connections = self.write_connections(); - let info = connections.single.get_mut(&allocation_id)?; - if info.lifecycle != ConnectionLifecycle::Active || info.has_body_owner { - return None; - } - let provider_name = info.allocation.get_provider_name()?; - if !self.providers.is_provider_for_input(provider_name.as_ref(), input_name.as_ref()) { - return None; - } - - // Claim under the capacity-transition lock so a duplicated internal - // request cannot attach a second provider body to this reservation. - info.lifecycle = ConnectionLifecycle::Opening; - let handle = ProviderHandle { - playback_request_id: info.playback_request_id, - binding_tag: None, - client_id: info.client_addr, - allocation_id: info.allocation_id, - allocation: info.allocation.clone(), - cancel_token: Some(info.cancel_token.clone()), - completion_token: Some(info.completion_token.clone()), - close_reason: Arc::clone(&info.close_reason), - open_generation: info.open_generation, - }; - Some(handle) - } - - // This method is used for redirects to cycle through the provider - pub fn get_next_provider(&self, provider_name: &Arc) -> Option> { - self.providers.get_next_provider(provider_name) - } - - pub fn find_provider_config(&self, provider_name: &Arc) -> Option> { - self.providers.find_provider_config(provider_name) - } - - pub fn is_provider_for_input(&self, provider_name: &Arc, input_name: &Arc) -> bool { - self.providers.is_provider_for_input(provider_name.as_ref(), input_name.as_ref()) - } - - pub fn is_provider_reserved_for_other_session( - &self, - provider_name: &Arc, - session_owner: Option<&str>, - ) -> bool { - let session_owner = session_owner.map(playback_lease_owner); - self.reserved_capacity_blocks_next(provider_name, session_owner) - } - - pub fn active_connections(&self) -> Option, usize>> { self.providers.active_connections() } - - pub fn is_over_limit(&self, provider_name: &Arc) -> bool { self.providers.is_over_limit(provider_name) } - - pub fn is_exhausted(&self, provider_name: &Arc) -> bool { self.providers.is_exhausted(provider_name) } - - /// Finds every shared subscription carried by one transport socket. - /// - /// This is an explicit socket-wide transport action (connection close, kick), not - /// the normal playback cleanup path. A single shared subscription is released - /// through `release_shared_subscriber`, which cannot touch other clients that - /// happen to arrive through the same reverse-proxy socket. - fn release_shared_by_addr(&self, addr: &SocketAddr) -> Vec { - let connections = self.read_connections(); - connections - .shared - .by_key - .values() - .flat_map(|shared| { - shared.connections.iter().filter_map(|(id, subscriber)| (subscriber.addr == *addr).then_some(*id)) - }) - .collect() - } - - /// Releases exactly one shared subscription and rebalances the shared allocation. - /// - /// Returns the allocation to release when the last subscriber left. - fn release_shared_subscriber(&self, subscriber_id: SharedSubscriberId) -> Option { - let mut connections = self.write_connections(); - - let key = connections.shared.key_by_subscriber.remove(&subscriber_id)?; - let mut shared = connections.shared.by_key.remove(&key)?; - shared.connections.remove(&subscriber_id); - - let shared_is_unlimited = shared.allocation.is_unlimited_provider(); - - if shared.connections.is_empty() { - connections.shared.shared_by_allocation_id.remove(&shared.allocation_id); - Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref()); - if !shared_is_unlimited { - if let Some(name) = shared.allocation.get_provider_name() { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&shared.allocation_id); - } - Self::remove_priority_entry( - &mut connections, - &name, - &(shared.priority, Reverse(shared.created_at), shared.allocation_id), - shared.kind, - ); - } - } - return Some(shared.allocation); - } - - // Recompute shared priority from remaining subscribers so preemption decisions - // reflect who is actually still watching the shared stream. Unlimited shared - // streams are not in the preemption index, so this rebalance is skipped. - if !shared_is_unlimited { - let old_priority = shared.priority; - let old_kind = shared.kind; - shared.kind = Self::shared_effective_kind(&shared.connections); - if let Some(new_priority) = Self::shared_effective_priority(&shared.connections, shared.kind) { - shared.priority = new_priority; - if (new_priority, shared.kind) != (old_priority, old_kind) { - if let Some(name) = shared.allocation.get_provider_name() { - Self::remove_priority_entry( - &mut connections, - &name, - &(old_priority, Reverse(shared.created_at), shared.allocation_id), - old_kind, - ); - Self::upsert_priority_entry( - &mut connections, - &name, - (new_priority, Reverse(shared.created_at), shared.allocation_id), - PriorityOwner::Shared(shared.allocation_id), - shared.kind, - ); - } - } - } - } - connections.shared.by_key.insert(key, shared); - None - } - - pub fn release_shared_connection(&self, subscriber_id: SharedSubscriberId) { - let _transition = self.lock_capacity_transition(); - self.release_shared_connection_inner(subscriber_id); - } - - fn release_shared_connection_inner(&self, subscriber_id: SharedSubscriberId) { - if let Some(allocation) = self.release_shared_subscriber(subscriber_id) { - debug_if_enabled!( - "Released last shared connection for provider {} (subscriber={subscriber_id})", - allocation.get_provider_name().unwrap_or_default() - ); - allocation.release(); - if let Some(name) = allocation.get_provider_name() { - self.capacity.notify(&name); - } - } - } - - pub fn register_body_owner(&self, alloc_id: AllocationId) -> bool { - let _transition = self.lock_capacity_transition(); - let mut connections = self.write_connections(); - if let Some(info) = connections.single.get_mut(&alloc_id) { - if info.lifecycle == ConnectionLifecycle::Closing || info.lifecycle == ConnectionLifecycle::Closed { - return false; - } - info.has_body_owner = true; - if info.lifecycle == ConnectionLifecycle::Opening { - info.lifecycle = ConnectionLifecycle::Active; - } - true - } else { - false - } - } - - pub fn mark_opening(&self, alloc_id: AllocationId) { - let _transition = self.lock_capacity_transition(); - let mut connections = self.write_connections(); - if let Some(info) = connections.single.get_mut(&alloc_id) { - if info.lifecycle != ConnectionLifecycle::Closing && info.lifecycle != ConnectionLifecycle::Closed { - info.lifecycle = ConnectionLifecycle::Opening; - } - } - } - - pub fn renew_opening_tokens(&self, alloc_id: AllocationId) -> Option<(CancellationToken, CancellationToken, u64)> { - let _transition = self.lock_capacity_transition(); - let mut connections = self.write_connections(); - if let Some(info) = connections.single.get_mut(&alloc_id) { - if info.lifecycle != ConnectionLifecycle::Closing && info.lifecycle != ConnectionLifecycle::Closed { - info.lifecycle = ConnectionLifecycle::Opening; - info.open_generation = info.open_generation.wrapping_add(1); - info.reaper_spawned = false; - let cancel_token = CancellationToken::new(); - let completion_token = CancellationToken::new(); - info.cancel_token = cancel_token.clone(); - info.completion_token = completion_token.clone(); - return Some((cancel_token, completion_token, info.open_generation)); - } - } - None - } - - pub fn mark_closing(&self, alloc_id: AllocationId) { - let _transition = self.lock_capacity_transition(); - let mut connections = self.write_connections(); - let prio = if let Some(info) = connections.single.get_mut(&alloc_id) { - info.lifecycle = ConnectionLifecycle::Closing; - info.cancel_token.cancel(); - info.allocation - .get_provider_name() - .map(|name| (name, (info.priority, Reverse(info.created_at), alloc_id), info.kind)) - } else { - None - }; - if let Some((name, key, kind)) = prio { - Self::remove_priority_entry(&mut connections, &name, &key, kind); - } - } - - pub fn release_connection(&self, addr: &SocketAddr) { - let _transition = self.lock_capacity_transition(); - let single_alloc_ids = { - let mut connections = self.write_connections(); - connections.single_by_addr.remove(addr) - }; - - if let Some(alloc_ids) = single_alloc_ids { - let mut to_reap = Vec::new(); - { - let mut connections = self.write_connections(); - for id in alloc_ids { - if let ReleaseAction::Wait(alloc_id, gen, token) = - ActiveProviderManagerCore::plan_single_release_locked(&mut connections, id) - { - to_reap.push((alloc_id, gen, token)); - } - } - } - for (id, gen, completion_token) in to_reap { - let core = Arc::clone(&self.core); - let shutdown_token = self.shutdown_token.clone(); - if let Ok(rt_handle) = tokio::runtime::Handle::try_current() { - rt_handle.spawn(async move { - tokio::select! { - () = shutdown_token.cancelled() => {}, - () = completion_token.cancelled() => { - core.complete_release_with_generation(id, Some(gen)); - } - } - }); - } - } - } - - // Shared connections carried by this transport socket. Resolve the subscriber - // ids first, then release each one precisely; two external clients behind one - // reverse proxy hold distinct subscriber ids and never overwrite each other. - for subscriber_id in self.release_shared_by_addr(addr) { - self.release_shared_connection_inner(subscriber_id); - } - } - - fn release_shared_handle_locked( - connections: &mut Connections, - handle: &ProviderHandle, - ) -> Option { - let mut released = None; - let mut released_priority_key: Option<(Arc, PriorityKey, ConnectionKind)> = None; - if let Some(key) = connections.shared.shared_by_allocation_id.remove(&handle.allocation_id) { - if let Some(shared) = connections.shared.by_key.remove(&key) { - Self::unindex_owner(connections, handle.allocation_id, shared.session_owner.as_ref()); - let pkey = (shared.priority, Reverse(shared.created_at), handle.allocation_id); - let shared_kind = shared.kind; - let shared_is_unlimited = shared.allocation.is_unlimited_provider(); - released = Some(shared.allocation); - for subscriber_id in shared.connections.keys() { - connections.shared.key_by_subscriber.remove(subscriber_id); - } - if !shared_is_unlimited { - if let Some(name) = released.as_ref().and_then(ProviderAllocation::get_provider_name) { - if let Some(list) = connections.by_provider.get_mut(&name) { - list.remove(&handle.allocation_id); - } - released_priority_key = Some((name, pkey, shared_kind)); - } - } - } - } - if let Some((name, pkey, kind)) = &released_priority_key { - Self::remove_priority_entry(connections, name, pkey, *kind); - } - if let Some(allocation) = released.as_ref() { - allocation.release(); - if let Some(name) = allocation.get_provider_name() { - connections.released_providers.push(name); - } - } - released - } - - pub fn release_handle(&self, handle: &ProviderHandle) { - let _transition = self.lock_capacity_transition(); - let wait_action = { - let mut connections = self.write_connections(); - if connections.single.contains_key(&handle.allocation_id) { - if handle.completion_token.as_ref().is_some_and(CancellationToken::is_cancelled) { - if let Some(info) = connections.single.get_mut(&handle.allocation_id) { - if info.open_generation == handle.open_generation { - info.completion_token.cancel(); - } - } - } - ActiveProviderManagerCore::plan_single_release_locked(&mut connections, handle.allocation_id) - } else { - Self::release_shared_handle_locked(&mut connections, handle); - ReleaseAction::None - } - }; - - if let ReleaseAction::Wait(alloc_id, gen, completion_token) = wait_action { - let core = Arc::clone(&self.core); - let shutdown_token = self.shutdown_token.clone(); - if let Ok(rt_handle) = tokio::runtime::Handle::try_current() { - rt_handle.spawn(async move { - tokio::select! { - () = shutdown_token.cancelled() => {}, - () = completion_token.cancelled() => { - core.complete_release_with_generation(alloc_id, Some(gen)); - } - } - }); - } - } - } - - /// Stops only stale requests of this playback, even on a multiplexed proxy socket. - pub fn release_playback_connections(&self, owner: &str, addrs: &[SocketAddr]) { - let handles = { - let connections = self.read_connections(); - let mut handles = Vec::new(); - for addr in addrs { - if let Some(alloc_ids) = connections.single_by_addr.get(addr) { - for id in alloc_ids { - if let Some(info) = connections.single.get(id) { - if info.request_owner.as_deref() == Some(owner) { - handles.push(ProviderHandle { - playback_request_id: info.playback_request_id, - binding_tag: None, - client_id: info.client_addr, - allocation_id: info.allocation_id, - allocation: info.allocation.clone(), - cancel_token: Some(info.cancel_token.clone()), - completion_token: Some(info.completion_token.clone()), - close_reason: Arc::clone(&info.close_reason), - open_generation: info.open_generation, - }); - } - } - } - } - } - handles - }; - for handle in handles { - handle.set_close_reason(ProviderCloseReason::Superseded); - if let Some(token) = &handle.cancel_token { - token.cancel(); - } - self.release_handle(&handle); - } - } - - /// Stops stale requests of this playback and awaits upstream body closure before releasing capacity. - pub async fn release_playback_connections_await(&self, owner: &str, addrs: &[SocketAddr]) { - let targets = { - let mut connections = self.write_connections(); - let mut targets = Vec::new(); - for addr in addrs { - if let Some(alloc_ids) = connections.single_by_addr.get(addr).cloned() { - for id in alloc_ids { - if let Some(info) = connections.single.get_mut(&id) { - if info.request_owner.as_deref() == Some(owner) - && info.lifecycle != ConnectionLifecycle::Closed - { - info.lifecycle = ConnectionLifecycle::Closing; - info.close_reason.store(ProviderCloseReason::Superseded as u8, Ordering::Release); - targets.push(( - info.allocation_id, - info.cancel_token.clone(), - info.completion_token.clone(), - info.priority, - info.created_at, - info.kind, - info.allocation.get_provider_name(), - info.open_generation, - )); - } - } - } - } - } - for (alloc_id, _, _, priority, created_at, kind, provider_name, _) in &targets { - if let Some(name) = provider_name { - Self::remove_priority_entry( - &mut connections, - name, - &(*priority, Reverse(*created_at), *alloc_id), - *kind, - ); - } - } - targets - }; - - if targets.is_empty() { - return; - } - - for (_, cancel_token, _, _, _, _, _, _) in &targets { - cancel_token.cancel(); - } - - let futures: Vec<_> = targets.iter().map(|(_, _, token, _, _, _, _, _)| token.cancelled()).collect(); - let _ = tokio::time::timeout(Duration::from_secs(1), futures::future::join_all(futures)).await; - - for (alloc_id, _, completion_token, _, _, _, _, gen) in targets { - if completion_token.is_cancelled() { - self.core.complete_release_with_generation(alloc_id, Some(gen)); - } else { - log::warn!("Superseded provider connection {alloc_id} did not close within deadline, retaining slot as Closing until reaped"); - // Atomically claim the reaper role under the lock before spawning, so that - // if this future was dropped between collection and here, release_handle can - // still install its own reaper (reaper_spawned remains false until now). - let should_spawn = { - let _transition = self.lock_capacity_transition(); - let mut connections = self.write_connections(); - connections.single.get_mut(&alloc_id).is_some_and(|info| { - if info.reaper_spawned { - false - } else { - info.reaper_spawned = true; - true - } - }) - }; - if should_spawn { - let core = Arc::clone(&self.core); - let shutdown_token = self.shutdown_token.clone(); - tokio::spawn(async move { - tokio::select! { - () = shutdown_token.cancelled() => {}, - () = completion_token.cancelled() => { - core.complete_release_with_generation(alloc_id, Some(gen)); - } - } - }); - } - } - } - } - - pub fn reclassify_connection(&self, addr: &SocketAddr, kind: ConnectionKind, priority: i8) -> bool { - self.reclassify_connection_for_owner(addr, None, kind, priority) - } - - pub fn reclassify_connection_for_owner( - &self, - addr: &SocketAddr, - owner: Option<&str>, - kind: ConnectionKind, - priority: i8, - ) -> bool { - let _transition = self.lock_capacity_transition(); - let mut connections = self.write_connections(); - - let Some(alloc_ids) = connections.single_by_addr.get(addr).cloned() else { - return false; - }; - let mut index_updates = Vec::new(); - for allocation_id in alloc_ids { - if let Some(info) = connections.single.get_mut(&allocation_id) { - if info.request_owner.as_deref() != owner { - continue; - } - if let Some(provider_name) = info.allocation.get_provider_name() { - let old_key = (info.priority, Reverse(info.created_at), allocation_id); - let p_owner = PriorityOwner::Single(allocation_id); - let old_kind = info.kind; - info.kind = kind; - info.priority = priority; - let new_key = (info.priority, Reverse(info.created_at), allocation_id); - // Skip preemption-index updates for unlimited providers: they - // are intentionally absent from `by_provider` / `priority_index` - // / `soft_priority_index`, so the old entry was never inserted. - // Performing the upsert here would re-introduce them into the - // preemption index and contradict `max_connections: 0`. - if !info.allocation.is_unlimited_provider() { - index_updates.push((provider_name, old_key, old_kind, new_key, p_owner, info.kind)); - } - } - } - } - for (provider_name, old_key, old_kind, new_key, p_owner, new_kind) in index_updates { - Self::remove_priority_entry(&mut connections, &provider_name, &old_key, old_kind); - Self::upsert_priority_entry(&mut connections, &provider_name, new_key, p_owner, new_kind); - } - true - } - - pub fn reclassify_shared_connection( - &self, - subscriber_id: SharedSubscriberId, - kind: ConnectionKind, - priority: i8, - ) -> bool { - let _transition = self.lock_capacity_transition(); - let mut connections = self.write_connections(); - let shared_key = connections.shared.key_by_subscriber.get(&subscriber_id).cloned(); - let Some(shared_key) = shared_key else { - return false; - }; - let Some(shared_allocation) = connections.shared.by_key.get_mut(&shared_key) else { - return false; - }; - let shared_is_unlimited = shared_allocation.allocation.is_unlimited_provider(); - - let old_priority = shared_allocation.priority; - let old_kind = shared_allocation.kind; - let Some(subscriber) = shared_allocation.connections.get_mut(&subscriber_id) else { - return false; - }; - subscriber.kind = kind; - subscriber.priority = priority; - shared_allocation.kind = Self::shared_effective_kind(&shared_allocation.connections); - if let Some(new_priority) = - Self::shared_effective_priority(&shared_allocation.connections, shared_allocation.kind) - { - shared_allocation.priority = new_priority; - } - let new_priority = shared_allocation.priority; - let new_kind = shared_allocation.kind; - let allocation_id = shared_allocation.allocation_id; - let created_at = shared_allocation.created_at; - let provider_name = shared_allocation.allocation.get_provider_name(); - let _ = shared_allocation; - - // Mirror the single-connection rule: skip priority-index rebuild for - // shared allocations backed by an unlimited provider. The kind/priority - // transition on the per-subscriber state is still applied above. - if !shared_is_unlimited { - if let Some(provider_name) = provider_name { - let owner = PriorityOwner::Shared(allocation_id); - Self::remove_priority_entry( - &mut connections, - &provider_name, - &(old_priority, Reverse(created_at), allocation_id), - old_kind, - ); - Self::upsert_priority_entry( - &mut connections, - &provider_name, - (new_priority, Reverse(created_at), allocation_id), - owner, - new_kind, - ); - } - } - - true - } - - /// Promotes exactly the allocation owned by this handle into a shared origin. - pub fn make_shared_connection( - &self, - handle: &ProviderHandle, - key: &str, - subscriber_id: SharedSubscriberId, - ) -> bool { - if self.is_shutting_down.load(Ordering::Acquire) { - return false; - } - let _transition = self.lock_capacity_transition(); - if self.is_shutting_down.load(Ordering::Acquire) { - return false; - } - let mut connections = self.write_connections(); - if connections.shared.by_key.contains_key(key) { - return false; - } - let Some(info) = connections.single.remove(&handle.allocation_id) else { - return false; - }; - if let Some(set) = connections.single_by_addr.get_mut(&info.client_addr) { - set.remove(&handle.allocation_id); - if set.is_empty() { - connections.single_by_addr.remove(&info.client_addr); - } - } - let provider_name = info.allocation.get_provider_name().unwrap_or_default(); - if !info.allocation.is_unlimited_provider() { - if let Some(list) = connections.by_provider.get_mut(&provider_name) { - list.remove(&handle.allocation_id); - } - Self::remove_priority_entry( - &mut connections, - &provider_name, - &(info.priority, Reverse(info.created_at), handle.allocation_id), - info.kind, - ); - Self::upsert_priority_entry( - &mut connections, - &provider_name, - (info.priority, Reverse(info.created_at), handle.allocation_id), - PriorityOwner::Shared(handle.allocation_id), - info.kind, - ); - } - let shared_key: Arc = Arc::from(key); - connections.shared.by_key.insert( - Arc::clone(&shared_key), - SharedAllocation { - allocation_id: handle.allocation_id, - origin_subscriber_id: subscriber_id, - allocation: info.allocation, - connections: HashMap::from([( - subscriber_id, - SharedSubscriber { priority: info.priority, kind: info.kind, addr: handle.client_id }, - )]), - priority: info.priority, - kind: info.kind, - created_at: info.created_at, - cancel_token: Some(info.cancel_token), - session_owner: info.session_owner, - }, - ); - connections.shared.key_by_subscriber.insert(subscriber_id, Arc::clone(&shared_key)); - connections.shared.shared_by_allocation_id.insert(handle.allocation_id, shared_key); - true - } - - pub fn add_shared_connection( - &self, - addr: &SocketAddr, - subscriber_id: SharedSubscriberId, - key: &str, - priority: i8, - kind: ConnectionKind, - ) -> Result<(), String> { - if self.is_shutting_down.load(Ordering::Acquire) { - return Err("Provider manager is shutting down".to_string()); - } - let _transition = self.lock_capacity_transition(); - if self.is_shutting_down.load(Ordering::Acquire) { - return Err("Provider manager is shutting down".to_string()); - } - let mut connections = self.write_connections(); - - // Extract metadata before taking a second mutable borrow on `connections`. - let metadata = connections.shared.by_key.get(key).map(|s| { - ( - s.allocation_id, - s.allocation.get_provider_name().unwrap_or_default(), - s.priority, - s.kind, - s.created_at, - s.allocation.is_unlimited_provider(), - ) - }); - - let Some((alloc_id, provider_name, old_priority, old_kind, created_at, shared_is_unlimited)) = metadata else { - let err = - format!("Failed to add shared connection for {addr}: url {} not found", sanitize_sensitive_info(key)); - error!("{err}"); - return Err(err); - }; - - debug_if_enabled!( - "Shared connection: added addr {addr} provider={} key={}", - sanitize_sensitive_info(&provider_name), - sanitize_sensitive_info(key) - ); - - let Some(shared_allocation) = connections.shared.by_key.get_mut(key) else { - let err = format!( - "Failed to add shared connection for {addr}: url {} disappeared during update", - sanitize_sensitive_info(key) - ); - error!("{err}"); - return Err(err); - }; - - shared_allocation.connections.insert(subscriber_id, SharedSubscriber { priority, kind, addr: *addr }); - let new_kind = Self::shared_effective_kind(&shared_allocation.connections); - let new_priority = Self::shared_effective_priority(&shared_allocation.connections, new_kind); - shared_allocation.kind = new_kind; - if let Some(new_priority) = new_priority { - shared_allocation.priority = new_priority; - } - let updated_kind = shared_allocation.kind; - let updated_priority = shared_allocation.priority; - let needs_reindex = (updated_priority, updated_kind) != (old_priority, old_kind); - let _ = shared_allocation; - // Skip priority-index rebuild for unlimited providers: they are not - // subject to preemption and therefore must not be in the index. - if !shared_is_unlimited && needs_reindex { - Self::remove_priority_entry( - &mut connections, - &provider_name, - &(old_priority, Reverse(created_at), alloc_id), - old_kind, - ); - Self::upsert_priority_entry( - &mut connections, - &provider_name, - (updated_priority, Reverse(created_at), alloc_id), - PriorityOwner::Shared(alloc_id), - updated_kind, - ); - } - - connections.shared.key_by_subscriber.insert(subscriber_id, Arc::from(key)); - Ok(()) - } - - pub fn get_provider_connections_count(&self) -> usize { self.providers.active_connection_count() } - - pub fn provider_capacities_for_input(&self, input_name: &Arc) -> Vec<(Arc, usize, usize)> { - let mut result = Vec::new(); - for provider_name in self.providers.provider_names_for_input(input_name) { - if let Some((current, max)) = self.providers.provider_capacity(&provider_name) { - result.push((provider_name, current, max)); - } - } - result - } -} - #[cfg(test)] -mod tests { - use super::{ActiveProviderManager, ConnectionKind, PlaybackLeaseRef}; - use crate::{ActiveUserManager, EventManager, SharedStreamManager}; - use arc_swap::{ArcSwap, ArcSwapOption}; - use shared::{ - defaults::{default_probe_user_priority, default_user_priority}, - model::{ConfigPaths, InputFetchMethod, InputType}, - utils::Internable, - }; - use std::{ - collections::{HashMap, HashSet}, - net::SocketAddr, - sync::{Arc, Weak}, - time::Duration, - }; - use tuliprox_core::{ - model::{ - AppConfig, Config, ConfigInput, ConfigInputAlias, MediaToolCapabilities, PlaybackKind, - PlaybackRequestOutcome, ProviderAllocation, SharedSubscriberId, SourcesConfig, - }, - utils::FileLockManager, - }; - - fn build_test_app_config(aliases: Option>, max_connections: u16) -> AppConfig { - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: "http://provider-1.example".to_string(), - username: Some("user1".to_string()), - password: Some("pass1".to_string()), - enabled: true, - priority: 0, - max_connections, - method: InputFetchMethod::default(), - aliases, - ..ConfigInput::default() - }); - - let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; - - AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config::default())), - sources: Arc::new(ArcSwap::from_pointee(sources)), - hdhomerun: Arc::new(ArcSwapOption::default()), - api_proxy: Arc::new(ArcSwapOption::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - } - } - - fn create_test_app_config_with_dual_provider_pool() -> AppConfig { - build_test_app_config( - Some(vec![ConfigInputAlias { - id: 2, - name: "provider_2".intern(), - url: "http://provider-2.example".to_string(), - username: Some("user2".to_string()), - password: Some("pass2".to_string()), - priority: 1, - max_connections: 1, - exp_date: None, - enabled: true, - stalker: None, - }]), - 1, - ) - } - - fn create_test_app_config_with_capacity_ordered_pool() -> AppConfig { - build_test_app_config( - Some(vec![ConfigInputAlias { - id: 2, - name: "provider_2".intern(), - url: "http://provider-2.example".to_string(), - username: Some("user2".to_string()), - password: Some("pass2".to_string()), - priority: 1, - max_connections: 1, - exp_date: None, - enabled: true, - stalker: None, - }]), - 3, - ) - } - - fn create_test_app_config_single_provider_pool() -> AppConfig { build_test_app_config(None, 1) } - - fn create_test_app_config_single_unlimited_provider_pool() -> AppConfig { build_test_app_config(None, 0) } - - /// Pool where the higher-priority provider (A) and its lower-priority alias (B) - /// each carry their own capacity, mirroring the reported A=2 / B=3 setup. - fn create_test_app_config_with_pool(primary_max: u16, alias_max: u16) -> AppConfig { - build_test_app_config( - Some(vec![ConfigInputAlias { - id: 2, - name: "provider_2".intern(), - url: "http://provider-2.example".to_string(), - username: Some("user2".to_string()), - password: Some("pass2".to_string()), - priority: 1, - max_connections: alias_max, - exp_date: None, - enabled: true, - stalker: None, - }]), - primary_max, - ) - } - - #[test] - fn shared_stream_manager_backref_is_weak_and_does_not_leak() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let shared = Arc::new(SharedStreamManager::new(Arc::clone(&provider))); - provider.set_shared_stream_manager(&shared); - - // The only strong reference to the shared manager is the local `shared`; the - // provider holds a Weak backref, so dropping `shared` releases both managers. - drop(shared); - assert!(provider.shared_stream_manager.get().and_then(Weak::upgrade).is_none()); - } - - #[tokio::test] - async fn confirmed_playbacks_fill_higher_priority_provider_before_alias() { - let app_cfg = create_test_app_config_with_pool(2, 3); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - - let mut selected = Vec::new(); - for index in 0..5 { - let owner = format!("playback-{index}"); - let addr = SocketAddr::from(([172, 18, 0, 9], 50_000 + index)); - let handle = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(&owner), - ) - .expect("pool has capacity for five confirmed playbacks"); - let provider = handle.allocation.get_provider_name().expect("provider name"); - // Only real media activity confirms a playback and lets it reserve capacity. - manager.confirm_playback_activity(&owner); - selected.push(provider.to_string()); - } - - assert_eq!(selected, ["provider_1", "provider_1", "provider_2", "provider_2", "provider_2"]); - } - - #[tokio::test] - async fn concurrent_proxy_requests_fill_exact_pool_capacity() -> Result<(), Box> { - let manager = Arc::new(ActiveProviderManager::new( - &create_test_app_config_with_pool(2, 3), - &Arc::new(EventManager::new()), - )); - let addr = SocketAddr::from(([127, 0, 0, 1], 50001)); - let mut tasks = tokio::task::JoinSet::new(); - for attempt in 0..32 { - let manager = Arc::clone(&manager); - tasks.spawn(async move { - manager.acquire_connection_with_grace_for_session( - &Arc::from("provider_1"), - &addr, - false, - 0, - ConnectionKind::Normal, - Some(&format!("playback-{attempt}")), - ) - }); - } - let mut handles = Vec::new(); - while let Some(result) = tasks.join_next().await { - if let Some(handle) = result? { - handles.push(handle); - } - } - assert_eq!(handles.len(), 5); - assert_eq!( - handles.iter().filter(|h| h.allocation.get_provider_name().as_deref() == Some("provider_1")).count(), - 2 - ); - assert_eq!( - handles.iter().filter(|h| h.allocation.get_provider_name().as_deref() == Some("provider_2")).count(), - 3 - ); - for handle in handles { - manager.release_handle(&handle); - } - assert_eq!(manager.get_provider_connections_count(), 0); - Ok(()) - } - - #[tokio::test] - async fn forced_reopen_and_late_cleanup_preserve_other_proxy_requests() -> Result<(), Box> { - let manager = - ActiveProviderManager::new(&create_test_app_config_with_pool(2, 3), &Arc::new(EventManager::new())); - let addr = SocketAddr::from(([127, 0, 0, 1], 50002)); - let input = Arc::from("provider_1"); - let first = manager - .acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("first")) - .ok_or("first allocation missing")?; - let other = manager - .acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("other")) - .ok_or("other allocation missing")?; - manager.release_playback_connections("first", &[addr]); - assert!(first.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); - assert!(!other.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); - assert_eq!(manager.get_provider_connections_count(), 1); - let replacement = manager - .acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("first")) - .ok_or("replacement missing")?; - manager.refresh_adaptive_playback_lease(&input, "first", tuliprox_core::model::PlaybackKind::LiveHls, 15); - manager.confirm_playback_activity("first"); - manager.finish_playback_request("first", PlaybackRequestOutcome::ProviderFailed, None); - assert_eq!(manager.provider_lease_usage(&input).active, 1); - manager.release_handle(&replacement); - manager.release_handle(&other); - Ok(()) - } - - #[tokio::test] - async fn release_snapshot_tracks_original_allocation_after_addr_reuse() { - let app_cfg = create_test_app_config_with_pool(2, 3); - let events = Arc::new(EventManager::new()); - let provider = ActiveProviderManager::new(&app_cfg, &events); - let addr = SocketAddr::from(([127, 0, 0, 1], 50_022)); - let input = "provider_1".intern(); - let original = provider - .acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("old")) - .expect("original allocation"); - let original_snapshot = provider.release_snapshot_for_addr(&addr); - assert!(!provider.wait_for_snapshot_release(&original_snapshot, Duration::ZERO).await); - - provider.release_handle(&original); - let replacement = provider - .acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("new")) - .expect("replacement allocation"); - let replacement_snapshot = provider.release_snapshot_for_addr(&addr); - assert!(provider.wait_for_snapshot_release(&original_snapshot, Duration::ZERO).await); - assert!(!provider.wait_for_snapshot_release(&replacement_snapshot, Duration::ZERO).await); - - let geoip = Arc::new(ArcSwapOption::default()); - let users = ActiveUserManager::new(&Config::default(), &geoip, &events); - users.set_pending_provider_release("user", original_snapshot.clone()).await; - users.set_pending_provider_release("user", replacement_snapshot.clone()).await; - assert!(!users.clear_pending_provider_release("user", &original_snapshot).await); - assert_eq!(users.pending_provider_release("user").await, Some(replacement_snapshot.clone())); - assert!(users.clear_pending_provider_release("user", &replacement_snapshot).await); - assert_eq!(users.pending_provider_release("user").await, None); - provider.release_handle(&replacement); - } - - #[tokio::test] - async fn release_snapshot_tracks_shared_subscriber_without_waiting_for_other_subscribers() { - let app_cfg = create_test_app_config_single_provider_pool(); - let events = Arc::new(EventManager::new()); - let provider = ActiveProviderManager::new(&app_cfg, &events); - let input = "provider_1".intern(); - let first_addr = SocketAddr::from(([127, 0, 0, 1], 50_023)); - let second_addr = SocketAddr::from(([127, 0, 0, 1], 50_024)); - let first = SharedSubscriberId::from_stream_uid(50_023); - let second = SharedSubscriberId::from_stream_uid(50_024); - let origin = - provider.acquire_connection(&input, &first_addr, 0, ConnectionKind::Normal).expect("shared origin"); - let before_promotion = provider.release_snapshot_for_addr(&first_addr); - assert!(provider.make_shared_connection(&origin, "shared-release", first)); - provider - .add_shared_connection(&second_addr, second, "shared-release", 0, ConnectionKind::Normal) - .expect("second subscriber"); - let snapshot = provider.release_snapshot_for_addr(&first_addr); - assert!(!provider.wait_for_snapshot_release(&before_promotion, Duration::ZERO).await); - assert!(!provider.wait_for_snapshot_release(&snapshot, Duration::ZERO).await); - - provider.release_connection(&first_addr); - assert!(provider.wait_for_snapshot_release(&before_promotion, Duration::ZERO).await); - assert!(provider.wait_for_snapshot_release(&snapshot, Duration::ZERO).await); - assert_eq!(provider.get_provider_connections_count(), 1); - provider.release_connection(&second_addr); - } - - #[tokio::test(start_paused = true)] - async fn manifest_retries_without_media_do_not_reserve_capacity() { - let app_cfg = create_test_app_config_with_pool(2, 3); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - let owner = "hls-cache:shared-session"; - let addr = SocketAddr::from(([172, 18, 0, 9], 51_000)); - - // Five manifest responses without a single media segment, all on one owner. - for _ in 0..5 { - let handle = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner), - ) - .expect("manifest start allocates on the preferred provider"); - assert_eq!(handle.allocation.get_provider_name().as_deref(), Some("provider_1")); - manager.refresh_provider_reservation(&input_name, owner, 15); - manager.release_connection(&addr); - } - - // Unconfirmed leases never reserve capacity, so an unrelated client is served - // by the higher-priority provider even though its own counters read zero. - let other = manager - .acquire_connection_with_grace_for_session( - &input_name, - &SocketAddr::from(([172, 18, 0, 9], 51_001)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some("other-client"), - ) - .expect("unrelated client must still get the preferred provider"); - assert_eq!(other.allocation.get_provider_name().as_deref(), Some("provider_1")); - - // The abandoned starts stop holding anything once their startup deadline passes. - tokio::time::advance(Duration::from_secs(6)).await; - manager.prune_expired_leases_now(); - assert!(!manager.is_provider_reserved_for_other_session(&input_name, Some("other-client"))); - } - - #[tokio::test(start_paused = true)] - async fn identified_reservation_recreates_a_cleared_lease() { - let app_cfg = create_test_app_config_with_pool(2, 3); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - let owner = "hls-cache:restore-session"; - let addr = SocketAddr::from(([172, 18, 0, 9], 53_000)); - - let handle = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner), - ) - .expect("allocate"); - manager.refresh_identified_provider_reservation( - &input_name, - owner, - tuliprox_core::model::PlaybackKind::Catchup, - 15, - ); - assert_eq!(manager.provider_lease_usage(&input_name).starting, 1); - - // Clearing removes the lease entirely; recreating must bring it back rather - // than silently no-op like a plain adaptive refresh would. - let binding_tag = manager.binding_tag_for_owner(owner); - manager.clear_identified_provider_reservation(owner, &input_name, binding_tag); - assert_eq!(manager.provider_lease_usage(&input_name).total(), 0); - - manager.refresh_identified_provider_reservation( - &input_name, - owner, - tuliprox_core::model::PlaybackKind::Catchup, - 15, - ); - assert_eq!(manager.provider_lease_usage(&input_name).starting, 1); - - manager.release_handle(&handle); - } - - #[tokio::test(start_paused = true)] - async fn reservations_do_not_survive_after_counters_reach_zero() { - let app_cfg = create_test_app_config_with_pool(2, 3); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - let owner = "finished-playback"; - let addr = SocketAddr::from(([172, 18, 0, 9], 52_000)); - - let handle = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner), - ) - .expect("first playback acquires the preferred provider"); - assert_eq!(handle.allocation.get_provider_name().as_deref(), Some("provider_1")); - manager.refresh_provider_reservation(&input_name, owner, 15); - manager.confirm_playback_activity(owner); - manager.release_connection(&addr); - - // Live TS is not reconnect capable: the confirmed lease is dropped on release, - // so the provider is immediately free again for the next higher-priority client. - manager.finish_playback_request(owner, PlaybackRequestOutcome::Completed, None); - let next = manager - .acquire_connection_with_grace_for_session( - &input_name, - &SocketAddr::from(([172, 18, 0, 9], 52_001)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some("next-client"), - ) - .expect("freed capacity must be selectable again"); - assert_eq!(next.allocation.get_provider_name().as_deref(), Some("provider_1")); - } - - #[tokio::test] - async fn two_playbacks_sharing_one_proxy_socket_keep_separate_slots() { - let app_cfg = create_test_app_config_with_pool(2, 3); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - // Both external clients arrive through the same reverse-proxy peer socket. - let proxy_addr = SocketAddr::from(([172, 18, 0, 9], 53_000)); - - let first = manager - .acquire_connection_with_grace_for_session( - &input_name, - &proxy_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some("device-one"), - ) - .expect("first device behind the proxy acquires a slot"); - let second = manager - .acquire_connection_with_grace_for_session( - &input_name, - &proxy_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some("device-two"), - ) - .expect("second device behind the proxy acquires its own slot"); - assert_ne!(first.allocation_id, second.allocation_id); - manager.confirm_playback_activity("device-one"); - manager.confirm_playback_activity("device-two"); - - // Releasing one playback must free exactly its own allocation and leave the - // other device's slot on the same transport untouched. - manager.release_handle(&first); - manager.finish_playback_request("device-one", PlaybackRequestOutcome::Completed, None); - let remaining = manager.provider_capacities_for_input(&input_name); - let primary = remaining.iter().find(|(name, _, _)| name.as_ref() == "provider_1").expect("primary pool entry"); - assert_eq!(primary.1, 1, "exactly one slot must remain in use on the shared socket"); - assert!(manager.read_leases().lease_of_owner("device-one").is_none()); - assert!(manager.read_leases().lease_of_owner("device-two").is_some()); - - manager.release_handle(&second); - manager.finish_playback_request("device-two", PlaybackRequestOutcome::Completed, None); - } - - #[tokio::test] - async fn unlimited_provider_connection_is_not_in_priority_index() { - let app_cfg = create_test_app_config_single_unlimited_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let addr_1: SocketAddr = "127.0.0.1:44001".parse().unwrap(); - let addr_2: SocketAddr = "127.0.0.1:44002".parse().unwrap(); - let addr_3: SocketAddr = "127.0.0.1:44003".parse().unwrap(); - - manager - .acquire_connection(&input_name, &addr_1, default_user_priority(), ConnectionKind::Normal) - .expect("acquire #1 on unlimited provider"); - manager - .acquire_connection(&input_name, &addr_2, default_user_priority(), ConnectionKind::Normal) - .expect("acquire #2 on unlimited provider"); - manager - .acquire_connection(&input_name, &addr_3, default_user_priority(), ConnectionKind::Normal) - .expect("acquire #3 on unlimited provider"); - - { - let connections = manager.read_connections(); - let priority_tree = connections.priority_index.get(&input_name); - assert!( - priority_tree.is_none_or(std::collections::BTreeMap::is_empty), - "unlimited provider must not be present in priority_index, found {priority_tree:?}" - ); - let soft_tree = connections.soft_priority_index.get(&input_name); - assert!( - soft_tree.is_none_or(std::collections::BTreeMap::is_empty), - "unlimited provider must not be present in soft_priority_index, found {soft_tree:?}" - ); - let by_provider = connections.by_provider.get(&input_name); - assert!( - by_provider.is_none_or(std::collections::HashSet::is_empty), - "unlimited provider must not be present in by_provider, found {by_provider:?}" - ); - } - - manager.release_connection(&addr_1); - manager.release_connection(&addr_2); - manager.release_connection(&addr_3); - } - - #[tokio::test] - async fn preemption_does_not_select_unlimited_provider_as_victim() { - let app_cfg = create_test_app_config_single_unlimited_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let low_addr: SocketAddr = "127.0.0.1:45001".parse().unwrap(); - let high_addr: SocketAddr = "127.0.0.1:45002".parse().unwrap(); - - // Acquire one low-priority and one high-priority connection on the unlimited provider. - let low = manager - .acquire_connection(&input_name, &low_addr, 50, ConnectionKind::Normal) - .expect("low-priority acquire on unlimited provider"); - let low_token = low.cancel_token.clone().expect("cancel token for low-priority connection"); - let _high = manager - .acquire_connection(&input_name, &high_addr, 0, ConnectionKind::Normal) - .expect("high-priority acquire on unlimited provider"); - - // A request at an even higher priority must not be able to preempt the unlimited - // provider's connection, because the connection is intentionally absent from the - // preemption indices. - let candidate = { - let connections = manager.read_connections(); - manager.select_preemption_candidate(&connections, &input_name, 0, ConnectionKind::Normal, &HashSet::new()) - }; - assert!( - candidate.is_none(), - "select_preemption_candidate must not return an unlimited-provider connection as a victim, got {candidate:?}" - ); - - // The low-priority connection is still alive and its cancel token has not been fired. - assert!( - !low_token.is_cancelled(), - "low-priority unlimited-provider connection must not be cancelled by preemption" - ); - - manager.release_connection(&low_addr); - manager.release_connection(&high_addr); - } - - #[tokio::test] - async fn test_force_exact_acquire_does_not_overallocate_busy_provider() { - let app_cfg = create_test_app_config_with_dual_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let client_1_addr: SocketAddr = "127.0.0.1:40001".parse().unwrap(); - let client_2_addr: SocketAddr = "127.0.0.1:40002".parse().unwrap(); - - let first_alloc = manager - .acquire_connection(&input_name, &client_1_addr, default_user_priority(), ConnectionKind::Normal) - .expect("client1 initial allocation"); - let pinned_provider = first_alloc.allocation.get_provider_name().expect("provider name expected"); - assert_eq!(pinned_provider.as_ref(), "provider_1"); - - // provider_1 has max_connections=1 and is already in use by client1 - let forced = manager.force_exact_acquire_connection( - &pinned_provider, - &client_2_addr, - default_user_priority(), - ConnectionKind::Normal, - ); - assert!(forced.is_none(), "forced exact acquire must not over-allocate busy provider"); - - manager.release_connection(&client_1_addr); - manager.release_connection(&client_2_addr); - } - - #[tokio::test] - async fn unstarted_series_retry_uses_free_alias_while_started_playback_stays_pinned() { - let app_cfg = create_test_app_config_with_dual_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input = "provider_1".intern(); - let owner = "series-playback"; - let first_addr: SocketAddr = "127.0.0.1:41010".parse().unwrap(); - let busy_addr: SocketAddr = "127.0.0.1:41011".parse().unwrap(); - let retry_addr: SocketAddr = "127.0.0.1:41012".parse().unwrap(); - - let failed_start = manager - .acquire_connection_with_lease_for_session( - &input, - &first_addr, - false, - 0, - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new(owner, PlaybackKind::Series)), - ) - .expect("first series allocation"); - assert!(manager.should_reuse_playback_provider(owner, &input)); - manager.release_handle(&failed_start); - assert!(!manager.should_reuse_playback_provider(owner, &input)); - - let busy = manager - .acquire_connection(&input, &busy_addr, 0, ConnectionKind::Normal) - .expect("other playback occupies first provider"); - let retry = manager - .acquire_connection_with_lease_for_session( - &input, - &retry_addr, - false, - 0, - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new(owner, PlaybackKind::Series)), - ) - .expect("retry should use free alias"); - let alias = retry.allocation.get_provider_name().expect("alias name"); - assert_eq!(alias.as_ref(), "provider_2"); - assert!(manager.should_reuse_playback_provider(owner, &alias)); - manager.refresh_adaptive_playback_lease(&alias, owner, PlaybackKind::Series, 15); - manager.confirm_identified_playback_activity(owner, retry.playback_request_id.expect("request id")); - manager.release_handle(&retry); - assert!(manager.should_reuse_playback_provider(owner, &alias)); - manager.release_handle(&busy); - } - - #[tokio::test] - async fn test_force_session_fallback_uses_different_provider_when_current_is_busy() { - let app_cfg = create_test_app_config_with_dual_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let client_1_addr: SocketAddr = "127.0.0.1:41001".parse().unwrap(); - let client_2_addr: SocketAddr = "127.0.0.1:41002".parse().unwrap(); - - // Step 1: Client1 starts movie -> provider_1 - let first_alloc = manager - .acquire_connection(&input_name, &client_1_addr, default_user_priority(), ConnectionKind::Normal) - .expect("client1 initial allocation"); - assert_eq!(first_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - - // Step 2: Client1 stops -> release provider_1 - manager.release_connection(&client_1_addr); - - // Step 3: Client2 starts live -> provider_1 - let live_alloc = manager - .acquire_connection(&input_name, &client_2_addr, default_user_priority(), ConnectionKind::Normal) - .expect("client2 live allocation"); - let busy_provider = live_alloc.allocation.get_provider_name().expect("provider name expected"); - assert_eq!(busy_provider.as_ref(), input_name.as_ref()); - assert!(manager.is_exhausted(&busy_provider)); - - // Step 4: Client1 restarts same movie. - // This emulates force-session fallback path by acquiring without provider grace. - let fallback_alloc = manager - .acquire_connection_with_grace(&input_name, &client_1_addr, false, 0, ConnectionKind::Normal) - .expect("client1 fallback allocation without grace"); - let fallback_provider = fallback_alloc.allocation.get_provider_name().expect("fallback provider expected"); - - assert_ne!(fallback_provider.as_ref(), busy_provider.as_ref()); - assert_eq!(fallback_provider.as_ref(), "provider_2"); - - manager.release_connection(&client_1_addr); - manager.release_connection(&client_2_addr); - } - - #[tokio::test] - async fn test_seek_reacquire_stays_on_same_provider_account_until_stop() { - let app_cfg = create_test_app_config_with_dual_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let client_1_addr: SocketAddr = "127.0.0.1:42001".parse().unwrap(); - let client_2_addr: SocketAddr = "127.0.0.1:42002".parse().unwrap(); - - // Initial playback for client1. - let first_alloc = manager - .acquire_connection(&input_name, &client_1_addr, default_user_priority(), ConnectionKind::Normal) - .expect("client1 initial allocation"); - let pinned_provider = first_alloc.allocation.get_provider_name().expect("provider name expected"); - assert_eq!(pinned_provider.as_ref(), "provider_1"); - - // Another client occupies the alternate account while client1 keeps seeking. - let second_alloc = manager - .acquire_connection(&input_name, &client_2_addr, default_user_priority(), ConnectionKind::Normal) - .expect("client2 allocation"); - let second_provider = second_alloc.allocation.get_provider_name().expect("provider name expected"); - assert_eq!(second_provider.as_ref(), "provider_2"); - - // Simulate repeated seek/range reconnects for client1: - // release old connection for the same client, then force exact pinned provider. - for _ in 0..3 { - manager.release_connection(&client_1_addr); - let seek_alloc = manager - .force_exact_acquire_connection( - &pinned_provider, - &client_1_addr, - default_user_priority(), - ConnectionKind::Normal, - ) - .expect("seek reacquire should stay on pinned provider"); - let seek_provider = seek_alloc.allocation.get_provider_name().expect("provider name expected"); - assert_eq!(seek_provider.as_ref(), pinned_provider.as_ref()); - } - - // Stream stop / cleanup. - manager.release_connection(&client_1_addr); - manager.release_connection(&client_2_addr); - } - - #[tokio::test] - async fn test_probe_preemption_releases_capacity_and_cancels_immediately() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let user_addr: SocketAddr = "127.0.0.1:43001".parse().unwrap(); - - let probe_handle = manager - .acquire_connection_for_probe(&input_name, default_probe_user_priority()) - .expect("probe allocation should succeed"); - let probe_token = probe_handle.cancel_token.clone().expect("probe handle must carry cancel token"); - - // User request should preempt probe and immediately acquire released capacity. - let user_alloc = manager - .acquire_connection_with_grace( - &input_name, - &user_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - ) - .expect("user allocation should preempt probe"); - assert_eq!(user_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - - // Cancellation happens inline during preemption; yield once to observe any deferred work. - tokio::task::yield_now().await; - assert!(probe_token.is_cancelled(), "probe token should be cancelled immediately after preemption"); - - manager.release_connection(&user_addr); - } - - #[tokio::test(start_paused = true)] - async fn test_session_provider_reservation_blocks_other_sessions_until_ttl_expires() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let owner_1 = "session-owner-1"; - let owner_2 = "session-owner-2"; - let addr_1: SocketAddr = "127.0.0.1:43101".parse().unwrap(); - let addr_2: SocketAddr = "127.0.0.1:43102".parse().unwrap(); - - manager.refresh_provider_reservation(&input_name, owner_1, 15); - manager.confirm_playback_activity(owner_1); - - let first = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr_1, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_1), - ) - .expect("reserved owner should reacquire its provider"); - assert_eq!(first.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - manager.release_connection(&addr_1); - - let blocked = manager.acquire_connection_with_grace_for_session( - &input_name, - &addr_2, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_2), - ); - assert!(blocked.is_none(), "other sessions must not take a reserved provider before TTL expiry"); - - tokio::time::advance(Duration::from_secs(16)).await; - - let second = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr_2, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_2), - ) - .expect("reservation should expire after TTL"); - assert_eq!(second.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - manager.release_connection(&addr_2); - } - - #[tokio::test(start_paused = true)] - async fn denied_confirmation_followed_by_refresh_cannot_reserve_foreign_slot() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - let owner_x = "session-owner-x"; - let owner_y = "session-owner-y"; - let owner_z = "session-owner-z"; - let addr_y: SocketAddr = "127.0.0.1:43310".parse().unwrap(); - let addr_z: SocketAddr = "127.0.0.1:43311".parse().unwrap(); - - // Y holds the only provider slot. - let y = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr_y, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_y), - ) - .expect("y acquires the single slot"); - - // X starts a lease and confirms media while Y holds the slot, so its - // reservation right is denied. A later refresh must not restore it. - manager.refresh_provider_reservation(&input_name, owner_x, 15); - manager.confirm_playback_activity(owner_x); - manager.refresh_provider_reservation(&input_name, owner_x, 15); - - manager.release_handle(&y); - - // If X's refresh had re-granted the denied reservation, Z would be blocked here. - let z = manager.acquire_connection_with_grace_for_session( - &input_name, - &addr_z, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_z), - ); - assert!(z.is_some(), "a denied reservation must not be restored by a later refresh"); - if let Some(z) = z { - manager.release_handle(&z); - } - } - - #[tokio::test(start_paused = true)] - async fn untagged_clear_cannot_delete_successor_reservation() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - let owner = "session-owner"; - let addr: SocketAddr = "127.0.0.1:43320".parse().unwrap(); - - // First incarnation acquires, then is removed entirely. - let first = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner), - ) - .expect("first acquire"); - manager.release_handle(&first); - manager.clear_provider_reservation(owner); - - // Second incarnation acquires a fresh lease on the same account. - let second = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner), - ) - .expect("second acquire"); - - // An untagged clear has no delete right and must not remove the successor. - manager.clear_identified_provider_reservation(owner, &input_name, None); - assert!( - manager.binding_tag_for_owner(owner).is_some(), - "an untagged clear must not delete the successor reservation" - ); - - manager.release_handle(&second); - manager.clear_provider_reservation(owner); - } - - #[tokio::test(start_paused = true)] - async fn session_reservations_preserve_capacity_and_priority_order() { - let app_cfg = create_test_app_config_with_capacity_ordered_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - - for index in 0..3 { - let owner = format!("session-owner-{index}"); - let addr = SocketAddr::from(([127, 0, 0, 1], 43200 + index)); - let allocation = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(&owner), - ) - .expect("preferred provider should retain free capacity for another session"); - assert_eq!(allocation.allocation.get_provider_name().as_deref(), Some("provider_1")); - manager.refresh_provider_reservation(&input_name, &owner, 15); - manager.confirm_playback_activity(&owner); - } - - let fallback_addr = SocketAddr::from(([127, 0, 0, 1], 43203)); - let fallback = manager - .acquire_connection_with_grace_for_session( - &input_name, - &fallback_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some("session-owner-3"), - ) - .expect("lower-priority provider should be used after preferred capacity is exhausted"); - assert_eq!(fallback.allocation.get_provider_name().as_deref(), Some("provider_2")); - - manager.release_connection(&SocketAddr::from(([127, 0, 0, 1], 43200))); - manager.release_connection(&fallback_addr); - - let foreign_after_release = manager - .acquire_connection_with_grace_for_session( - &input_name, - &fallback_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some("session-owner-4"), - ) - .expect("idle reservation should protect one preferred-provider slot"); - assert_eq!(foreign_after_release.allocation.get_provider_name().as_deref(), Some("provider_2")); - - let reserved_owner_addr = SocketAddr::from(([127, 0, 0, 1], 43204)); - let reserved_owner = manager - .acquire_connection_with_grace_for_session( - &input_name, - &reserved_owner_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some("session-owner-0"), - ) - .expect("reservation owner should reclaim its preferred-provider slot"); - assert_eq!(reserved_owner.allocation.get_provider_name().as_deref(), Some("provider_1")); - - for index in 1..5 { - let addr = SocketAddr::from(([127, 0, 0, 1], 43200 + index)); - manager.release_connection(&addr); - } - } - - #[tokio::test(start_paused = true)] - async fn test_unlimited_provider_reservation_does_not_block_other_sessions() { - let app_cfg = create_test_app_config_single_unlimited_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let owner_1 = "session-owner-1"; - let owner_2 = "session-owner-2"; - let addr_1: SocketAddr = "127.0.0.1:43121".parse().unwrap(); - let addr_2: SocketAddr = "127.0.0.1:43122".parse().unwrap(); - - manager.refresh_provider_reservation(&input_name, owner_1, 15); - - let first = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr_1, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_1), - ) - .expect("reserved owner should reacquire its unlimited provider"); - assert_eq!(first.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - - let second = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr_2, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_2), - ) - .expect("other sessions should not be blocked by reservations on unlimited providers"); - assert_eq!(second.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - - manager.release_connection(&addr_1); - manager.release_connection(&addr_2); - } - - #[tokio::test(start_paused = true)] - async fn different_session_cannot_take_idle_reservation_from_same_client_family() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let Ok(second_addr) = "127.0.0.1:43112".parse() else { - return; - }; - let owner_channel_1 = "client|ua|user|100"; - let owner_channel_2 = "client|ua|user|200"; - - manager.refresh_provider_reservation(&input_name, owner_channel_1, 15); - manager.confirm_playback_activity(owner_channel_1); - - let blocked = manager.acquire_connection_with_grace_for_session( - &input_name, - &second_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_channel_2), - ); - assert!(blocked.is_none(), "a related but distinct playback must not steal an idle reservation"); - - manager.clear_provider_reservation(owner_channel_1); - let acquired = manager.acquire_connection_with_grace_for_session( - &input_name, - &second_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_channel_2), - ); - assert!(acquired.is_some(), "explicitly clearing the old playback should release its provider"); - manager.release_connection(&second_addr); - } - - #[tokio::test(start_paused = true)] - async fn concurrent_same_family_playbacks_keep_independent_provider_reservations() { - let app_cfg = create_test_app_config_with_dual_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let provider_2 = "provider_2".intern(); - let Ok(first_addr) = "127.0.0.1:43121".parse() else { - return; - }; - let Ok(second_addr) = "127.0.0.1:43122".parse() else { - return; - }; - let owner_channel_1 = "proxy|player|user|100"; - let owner_channel_2 = "proxy|player|user|200"; - - let first = manager.acquire_connection_with_grace_for_session( - &input_name, - &first_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_channel_1), - ); - assert!(first.is_some(), "first playback should acquire the preferred provider"); - let Some(first) = first else { - return; - }; - assert_eq!(first.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - manager.refresh_provider_reservation(&input_name, owner_channel_1, 15); - manager.confirm_playback_activity(owner_channel_1); - manager.release_connection(&first_addr); - - let second = manager.acquire_connection_with_grace_for_session( - &input_name, - &second_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_channel_2), - ); - assert!(second.is_some(), "parallel playback should acquire the remaining provider"); - let Some(second) = second else { - return; - }; - assert_eq!(second.allocation.get_provider_name().as_deref(), Some(provider_2.as_ref())); - manager.refresh_provider_reservation(&provider_2, owner_channel_2, 15); - manager.confirm_playback_activity(owner_channel_2); - - let leases = manager.read_leases(); - assert_eq!(leases.provider_for_owner(owner_channel_1).as_deref(), Some(input_name.as_ref())); - assert_eq!(leases.provider_for_owner(owner_channel_2).as_deref(), Some(provider_2.as_ref())); - drop(leases); - - manager.clear_provider_reservation(owner_channel_2); - let leases = manager.read_leases(); - assert_eq!(leases.provider_for_owner(owner_channel_1).as_deref(), Some(input_name.as_ref())); - assert!(leases.lease_of_owner(owner_channel_2).is_none()); - drop(leases); - - manager.release_connection(&second_addr); - } - - #[tokio::test(start_paused = true)] - async fn test_clear_provider_reservation_releases_family_block() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let owner_1 = "session-owner-1"; - let owner_2 = "session-owner-2"; - let addr_2: SocketAddr = "127.0.0.1:43132".parse().unwrap(); - - manager.refresh_provider_reservation(&input_name, owner_1, 15); - manager.confirm_playback_activity(owner_1); - - let blocked = manager.acquire_connection_with_grace_for_session( - &input_name, - &addr_2, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_2), - ); - assert!(blocked.is_none(), "reservation should initially block another session"); - - manager.clear_provider_reservation(owner_1); - - let acquired = manager.acquire_connection_with_grace_for_session( - &input_name, - &addr_2, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(owner_2), - ); - assert!(acquired.is_some(), "clearing reservation should unblock the provider"); - } - - #[tokio::test] - async fn test_higher_priority_user_preempts_lower_priority_user() { - // User with priority 5 (low) is connected; user with priority -1 (high) arrives. - // The low-priority user should be preempted. - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let low_prio_addr: SocketAddr = "127.0.0.1:44001".parse().unwrap(); - let high_prio_addr: SocketAddr = "127.0.0.1:44002".parse().unwrap(); - - // Low-priority user connects (priority 5 = lower importance) - let low_alloc = manager - .acquire_connection(&input_name, &low_prio_addr, 5, ConnectionKind::Normal) - .expect("low-priority user should get connection"); - assert_eq!(low_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - - // Provider is now exhausted - assert!(manager.is_exhausted(&input_name)); - - // High-priority user arrives (priority -1 = higher importance), should preempt low-priority user - let high_alloc = manager - .acquire_connection_with_grace(&input_name, &high_prio_addr, false, -1, ConnectionKind::Normal) - .expect("high-priority user should preempt low-priority user and get connection"); - assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - - manager.release_connection(&high_prio_addr); - } - - #[tokio::test] - async fn test_same_priority_user_does_not_preempt() { - // Two users with the same priority — new one should NOT preempt the existing one. - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let user_1_addr: SocketAddr = "127.0.0.1:45001".parse().unwrap(); - let user_2_addr: SocketAddr = "127.0.0.1:45002".parse().unwrap(); - - // User 1 connects with priority 0 - let alloc1 = manager - .acquire_connection(&input_name, &user_1_addr, default_user_priority(), ConnectionKind::Normal) - .expect("user1 should get connection"); - assert_eq!(alloc1.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - - // Provider is now exhausted - assert!(manager.is_exhausted(&input_name)); - - // User 2 arrives with the same priority 0 — should NOT preempt user 1 - let alloc2 = manager.acquire_connection_with_grace( - &input_name, - &user_2_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - ); - assert!(alloc2.is_none(), "same-priority user should not preempt existing user"); - - manager.release_connection(&user_1_addr); - } - - #[tokio::test] - async fn test_lower_priority_user_does_not_preempt_higher_priority_user() { - // User with high priority is connected; user with low priority arrives — should NOT preempt. - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let high_prio_addr: SocketAddr = "127.0.0.1:46001".parse().unwrap(); - let low_prio_addr: SocketAddr = "127.0.0.1:46002".parse().unwrap(); - - // High-priority user connects (priority -10) - let alloc1 = manager - .acquire_connection(&input_name, &high_prio_addr, -10, ConnectionKind::Normal) - .expect("high-priority user should get connection"); - assert_eq!(alloc1.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - - // Provider is now exhausted - assert!(manager.is_exhausted(&input_name)); - - // Low-priority user arrives (priority 10) — should NOT preempt high-priority user - let alloc2 = - manager.acquire_connection_with_grace(&input_name, &low_prio_addr, false, 10, ConnectionKind::Normal); - assert!(alloc2.is_none(), "low-priority user should not preempt high-priority user"); - - manager.release_connection(&high_prio_addr); - } - - #[tokio::test] - async fn test_grace_period_triggers_preemption_of_lower_priority() { - // Provider full, high-prio user arrives with grace allowed, - // low-prio victim should be evicted and provider should not be over limit. - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let low_prio_addr: SocketAddr = "127.0.0.1:47001".parse().unwrap(); - let high_prio_addr: SocketAddr = "127.0.0.1:47002".parse().unwrap(); - - // Low-priority user connects (priority 20 = low importance) - let low_alloc = manager - .acquire_connection(&input_name, &low_prio_addr, 20, ConnectionKind::Normal) - .expect("low-priority user should get connection"); - assert_eq!(low_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - let low_token = low_alloc.cancel_token.clone().expect("must have cancel token"); - - // Provider is now exhausted - assert!(manager.is_exhausted(&input_name)); - - // High-priority user arrives WITH grace allowed (default streaming path) - // This should get a GracePeriod allocation and then evict the low-prio user - let high_alloc = manager - .acquire_connection(&input_name, &high_prio_addr, 0, ConnectionKind::Normal) - .expect("high-priority user should get grace allocation and evict low-prio"); - assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - - // Low-priority user's cancel token should be cancelled - assert!(low_token.is_cancelled(), "low-prio user should be cancelled after eviction"); - - // Provider should not be over limit (eviction freed a slot) - assert!(!manager.is_over_limit(&input_name), "provider should not be over limit after eviction"); - - manager.release_connection(&high_prio_addr); - } - - #[tokio::test] - async fn test_equal_priority_user_gets_grace_without_preempting_existing_stream() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let user_1_addr: SocketAddr = "127.0.0.1:48001".parse().unwrap(); - let user_2_addr: SocketAddr = "127.0.0.1:48002".parse().unwrap(); - - // User 1 connects with priority 0 - let alloc1 = manager - .acquire_connection(&input_name, &user_1_addr, default_user_priority(), ConnectionKind::Normal) - .expect("user1 should get connection"); - assert_eq!(alloc1.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - let token1 = alloc1.cancel_token.clone().expect("must have cancel token"); - - // Provider is now exhausted - assert!(manager.is_exhausted(&input_name)); - - // User 2 arrives with the same priority and should be granted grace instead of being rejected. - let alloc2 = manager - .acquire_connection(&input_name, &user_2_addr, default_user_priority(), ConnectionKind::Normal) - .expect("same-priority user should get grace allocation"); - assert!(matches!(alloc2.allocation, ProviderAllocation::GracePeriod(_))); - assert_eq!(alloc2.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - - // User 1 should NOT be cancelled, and the provider should be temporarily over limit. - assert!(!token1.is_cancelled(), "same-prio user should not be evicted"); - assert!(manager.is_over_limit(&input_name), "provider should be temporarily over limit during grace"); - - manager.release_connection(&user_1_addr); - manager.release_connection(&user_2_addr); - } - - #[tokio::test] - async fn test_higher_priority_user_preempts_first_inserted_low_priority_victim_on_exact_tie() { - let app_cfg = create_test_app_config_with_dual_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let old_low_addr: SocketAddr = "127.0.0.1:48101".parse().unwrap(); - let new_low_addr: SocketAddr = "127.0.0.1:48102".parse().unwrap(); - let high_prio_addr: SocketAddr = "127.0.0.1:48103".parse().unwrap(); - - // Place two equal-priority low-priority victims on different provider aliases. - // The test then normalizes their created_at timestamps to an exact tie so the - // selector must fall back to the final stable tie-break instead of clock order. - let old_alloc = manager - .acquire_exact_connection_with_grace( - &"provider_2".intern(), - &old_low_addr, - false, - 20, - ConnectionKind::Normal, - ) - .expect("old low-priority allocation should succeed"); - let old_token = old_alloc.cancel_token.clone().expect("old allocation should have cancel token"); - assert_eq!(old_alloc.allocation.get_provider_name().as_deref(), Some("provider_2")); - - let new_alloc = manager - .acquire_exact_connection_with_grace( - &"provider_1".intern(), - &new_low_addr, - false, - 20, - ConnectionKind::Normal, - ) - .expect("new low-priority allocation should succeed"); - let new_token = new_alloc.cancel_token.clone().expect("new allocation should have cancel token"); - assert_eq!(new_alloc.allocation.get_provider_name().as_deref(), Some("provider_1")); - - { - let mut connections = manager.write_connections(); - let old_created_at = connections - .single - .get(&old_alloc.allocation_id) - .map(|info| info.created_at) - .expect("old allocation should still be registered"); - - let (new_created_at, new_priority) = { - let info = connections - .single - .get_mut(&new_alloc.allocation_id) - .expect("new allocation should still be registered"); - let original_created_at = info.created_at; - info.created_at = old_created_at; - (original_created_at, info.priority) - }; - - let provider_name = "provider_1".intern(); - let tree = - connections.priority_index.get_mut(&provider_name).expect("priority index for provider_1 should exist"); - let owner = tree - .remove(&(new_priority, std::cmp::Reverse(new_created_at), new_alloc.allocation_id)) - .expect("new allocation should still be indexed"); - tree.insert((new_priority, std::cmp::Reverse(old_created_at), new_alloc.allocation_id), owner); - } - - assert!(manager.is_exhausted(&input_name)); - - // Higher-priority request should now select the first-inserted victim because - // priority and created_at are exactly tied across provider aliases. - let high_alloc = manager - .acquire_connection(&input_name, &high_prio_addr, 0, ConnectionKind::Normal) - .expect("higher-priority request should preempt the first-inserted low-priority victim on exact tie"); - assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some("provider_2")); - - assert!(old_token.is_cancelled(), "first-inserted low-priority victim should be canceled first on exact tie"); - assert!(!new_token.is_cancelled(), "later allocation should remain active after the stable tie-break"); - - manager.release_connection(&high_prio_addr); - manager.release_connection(&old_low_addr); - manager.release_connection(&new_low_addr); - } - - #[tokio::test] - async fn shared_promotion_and_release_preserve_other_allocations_on_same_socket( - ) -> Result<(), Box> { - let app_cfg = build_test_app_config(None, 3); - let events = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &events); - let addr = "127.0.0.1:48500".parse()?; - let input = "provider_1".intern(); - let unrelated = manager - .acquire_connection(&input, &addr, 0, ConnectionKind::Normal) - .ok_or("unrelated allocation missing")?; - let origin = - manager.acquire_connection(&input, &addr, 5, ConnectionKind::Normal).ok_or("origin allocation missing")?; - let first = SharedSubscriberId::from_stream_uid(1); - let second = SharedSubscriberId::from_stream_uid(2); - let key = "https://example.invalid/shared.ts"; - assert!(manager.make_shared_connection(&origin, key, first)); - assert_eq!(manager.get_provider_connections_count(), 2); - assert!(manager.read_connections().single.contains_key(&unrelated.allocation_id)); - assert!(manager - .connections - .read() - .unwrap() - .single_by_addr - .get(&addr) - .is_some_and(|allocs| allocs.contains(&unrelated.allocation_id))); - assert!(!unrelated.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); - manager.add_shared_connection(&addr, second, key, 9, ConnectionKind::Soft)?; - assert!(manager.reclassify_shared_connection(second, ConnectionKind::Normal, 1)); - { - let connections = manager.read_connections(); - let shared = connections.shared.by_key.get(key).ok_or("shared origin missing")?; - assert_eq!(shared.connections.get(&first).map(|subscriber| subscriber.priority), Some(5)); - assert_eq!(shared.priority, 1); - } - manager.release_shared_connection(second); - manager.release_shared_connection(second); - assert_eq!(manager.get_provider_connections_count(), 2); - assert_eq!(manager.read_connections().shared.by_key.get(key).map(|shared| shared.priority), Some(5)); - manager.release_shared_connection(first); - manager.release_handle(&origin); - assert_eq!(manager.get_provider_connections_count(), 1); - manager.release_handle(&unrelated); - assert_eq!(manager.get_provider_connections_count(), 0); - Ok(()) - } - - #[tokio::test] - async fn test_shared_priority_downgrades_after_high_priority_user_leaves() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let stream_key = "http://example.com/shared/live"; - let addr_a: SocketAddr = "127.0.0.1:48501".parse().unwrap(); - let addr_b: SocketAddr = "127.0.0.1:48502".parse().unwrap(); - - // A starts shared stream with high importance (priority 0). - let alloc_a = manager - .acquire_connection(&input_name, &addr_a, 0, ConnectionKind::Normal) - .expect("A should get initial connection"); - let shared_token = alloc_a.cancel_token.clone().expect("shared allocation should have cancel token"); - assert!(manager.make_shared_connection(&alloc_a, stream_key, SharedSubscriberId::from_stream_uid(1))); - - // B joins the same shared stream with lower importance (priority 1). - let join_result = manager.add_shared_connection( - &addr_b, - SharedSubscriberId::from_stream_uid(2), - stream_key, - 1, - ConnectionKind::Normal, - ); - assert!(join_result.is_ok(), "B should join existing shared stream, got: {join_result:?}"); - - // A leaves shared stream. Shared allocation should now inherit B's lower priority. - manager.release_connection(&addr_a); - { - let connections = manager.read_connections(); - let shared = connections.shared.by_key.get(stream_key).expect("shared entry should remain for B"); - assert_eq!(shared.priority, 1, "shared priority must downgrade to remaining subscriber priority"); - } - - // A starts another stream with higher importance and should preempt B's shared stream. - let alloc_a2 = manager - .acquire_connection(&input_name, &addr_a, 0, ConnectionKind::Normal) - .expect("A should preempt lower-priority shared stream"); - assert_eq!(alloc_a2.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - assert!(shared_token.is_cancelled(), "shared stream should be cancelled when preempted"); - assert!(!manager.is_over_limit(&input_name), "provider should not remain over limit after preemption"); - - manager.release_connection(&addr_a); - manager.release_connection(&addr_b); - } - - #[tokio::test] - async fn test_higher_priority_user_preempts_shared_stream_with_multiple_subscribers() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let stream_key = "http://example.com/shared/live/multi"; - let addr_a: SocketAddr = "127.0.0.1:48601".parse().unwrap(); - let addr_b: SocketAddr = "127.0.0.1:48602".parse().unwrap(); - let addr_high: SocketAddr = "127.0.0.1:48603".parse().unwrap(); - - let shared_alloc = manager - .acquire_connection(&input_name, &addr_a, 5, ConnectionKind::Normal) - .expect("low-priority shared stream should get initial connection"); - let shared_token = shared_alloc.cancel_token.clone().expect("shared allocation should have cancel token"); - assert!(manager.make_shared_connection(&shared_alloc, stream_key, SharedSubscriberId::from_stream_uid(1))); - - manager - .add_shared_connection( - &addr_b, - SharedSubscriberId::from_stream_uid(2), - stream_key, - 6, - ConnectionKind::Normal, - ) - .expect("second subscriber should join shared stream"); - - let high_alloc = manager - .acquire_connection(&input_name, &addr_high, 0, ConnectionKind::Normal) - .expect("higher-priority user should preempt lower-priority shared stream"); - assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); - assert!(shared_token.is_cancelled(), "shared stream should be cancelled when preempted"); - - { - let connections = manager.read_connections(); - assert!( - !connections.shared.by_key.contains_key(stream_key), - "preempted shared stream must be removed even with multiple subscribers" - ); - } - - manager.release_connection(&addr_high); - manager.release_connection(&addr_a); - manager.release_connection(&addr_b); - } - - #[tokio::test] - async fn test_btree_index_consistent_after_lifecycle() { - // Verify that the priority_index stays consistent through add, evict, release. - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let addr_a: SocketAddr = "127.0.0.1:49001".parse().unwrap(); - let addr_b: SocketAddr = "127.0.0.1:49002".parse().unwrap(); - - // Add connection A (low priority) - let alloc_a = manager.acquire_connection(&input_name, &addr_a, 10, ConnectionKind::Normal).expect("alloc_a"); - - // Check index has 1 entry - { - let connections = manager.read_connections(); - let tree = connections.priority_index.get(&input_name).expect("index for provider_1"); - assert_eq!(tree.len(), 1, "index should have 1 entry after first allocation"); - } - - // High-priority user evicts low-priority via grace path - let alloc_b = manager - .acquire_connection(&input_name, &addr_b, -5, ConnectionKind::Normal) - .expect("alloc_b should evict alloc_a"); - - // Check index: should have 1 entry (alloc_a evicted, alloc_b added) - { - let connections = manager.read_connections(); - let tree = connections.priority_index.get(&input_name).expect("index for provider_1"); - assert_eq!(tree.len(), 1, "index should have 1 entry after eviction + new allocation"); - // The remaining entry should be alloc_b - let ((prio, _, _), _) = tree.iter().next().expect("one entry"); - assert_eq!(*prio, -5, "remaining entry should be the high-prio connection"); - } - - // Release alloc_b - manager.release_handle(&alloc_b); - - // Check index: should be empty - { - let connections = manager.read_connections(); - let tree = connections.priority_index.get(&input_name); - let is_empty = tree.is_none_or(std::collections::BTreeMap::is_empty); - assert!(is_empty, "index should be empty after releasing all connections"); - } - - // Verify alloc_a handle can be safely released (already evicted - no-op) - manager.release_handle(&alloc_a); - } - - #[tokio::test] - async fn test_preemption_respects_registered_body_owner_completion() { - let app_cfg = build_test_app_config(None, 1); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let addr_1: SocketAddr = "127.0.0.1:49061".parse().unwrap(); - let addr_2: SocketAddr = "127.0.0.1:49062".parse().unwrap(); - - // 1. First request with prio 5 occupies limit=1 - let handle_1 = manager - .acquire_connection(&input_name, &addr_1, 5, ConnectionKind::Normal) - .expect("first allocation should succeed"); - - // 2. Register body owner on handle_1 - manager.register_body_owner(handle_1.allocation_id); - - // 3. Higher-priority request with prio -1 arrives without grace while handle_1 is active - let handle_2 = manager.acquire_connection_with_grace(&input_name, &addr_2, false, -1, ConnectionKind::Normal); - - // 4. Replacement must NOT be admitted before completion - assert!(handle_2.is_none(), "replacement must not be admitted before victim completion"); - assert!( - handle_1.cancel_token.as_ref().expect("cancel token").is_cancelled(), - "victim should receive cancel signal" - ); - assert_eq!( - handle_1.close_reason.load(std::sync::atomic::Ordering::Acquire), - tuliprox_core::model::ProviderCloseReason::PriorityPreempted as u8, - "victim close reason must be PriorityPreempted" - ); - - // 5. Victim completes upstream reading - handle_1.completion_token.as_ref().expect("completion token").cancel(); - - // Allow background complete_release task to run - tokio::task::yield_now().await; - tokio::time::sleep(Duration::from_millis(20)).await; - - // 6. Now replacement request can be admitted - let handle_2 = manager - .acquire_connection(&input_name, &addr_2, -1, ConnectionKind::Normal) - .expect("replacement should succeed after victim has completed release"); - - manager.release_handle(&handle_2); - } - - #[tokio::test] - async fn test_preemption_respects_opening_connection_completion() { - let app_cfg = build_test_app_config(None, 1); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let addr_1: SocketAddr = "127.0.0.1:49063".parse().unwrap(); - let addr_2: SocketAddr = "127.0.0.1:49064".parse().unwrap(); - - // 1. First request with prio 5 occupies limit=1 - let handle_1 = manager - .acquire_connection(&input_name, &addr_1, 5, ConnectionKind::Normal) - .expect("first allocation should succeed"); - manager.mark_opening(handle_1.allocation_id); - - // Verify lifecycle is Opening and body owner is not yet registered - { - let connections = manager.read_connections(); - let info = connections.single.get(&handle_1.allocation_id).expect("handle_1 exists"); - assert_eq!(info.lifecycle, tuliprox_core::model::ConnectionLifecycle::Opening); - assert!(!info.has_body_owner); - } - - // 2. Higher-priority request with prio -1 arrives without grace while handle_1 is in Opening - let handle_2 = manager.acquire_connection_with_grace(&input_name, &addr_2, false, -1, ConnectionKind::Normal); - - // 3. Replacement must NOT be admitted before completion of opening - assert!(handle_2.is_none(), "replacement must not be admitted while victim is still opening"); - assert!( - handle_1.cancel_token.as_ref().expect("cancel token").is_cancelled(), - "victim should receive cancel signal during open" - ); - assert_eq!( - handle_1.close_reason.load(std::sync::atomic::Ordering::Acquire), - tuliprox_core::model::ProviderCloseReason::PriorityPreempted as u8, - "victim close reason must be PriorityPreempted" - ); - - // 4. Victim's open future completes and signals completion - handle_1.completion_token.as_ref().expect("completion token").cancel(); - - // Allow background complete_release task to run - tokio::task::yield_now().await; - tokio::time::sleep(Duration::from_millis(20)).await; - - // 5. Now replacement request can be admitted - let handle_2 = manager - .acquire_connection(&input_name, &addr_2, -1, ConnectionKind::Normal) - .expect("replacement should succeed after victim open future has completed release"); - - manager.release_handle(&handle_2); - } - - #[tokio::test] - async fn test_owner_index_consistent_through_lifecycle() { - let app_cfg = build_test_app_config(None, 2); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let addr_a: SocketAddr = "127.0.0.1:49051".parse().unwrap(); - let addr_b: SocketAddr = "127.0.0.1:49052".parse().unwrap(); - - let owner_a = "owner-a"; - let owner_b = "owner-b"; - let alloc_a = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr_a, - false, - 0, - ConnectionKind::Normal, - Some(owner_a), - ) - .expect("alloc_a"); - let alloc_b = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr_b, - false, - 0, - ConnectionKind::Normal, - Some(owner_b), - ) - .expect("alloc_b"); - - { - let connections = manager.read_connections(); - assert_eq!(connections.by_owner.get(owner_a).map_or(0, HashSet::len), 1, "owner-a has one allocation"); - assert_eq!(connections.by_owner.get(owner_b).map_or(0, HashSet::len), 1, "owner-b has one allocation"); - assert!(connections.by_owner.get(owner_a).unwrap().contains(&alloc_a.allocation_id)); - assert!(connections.by_owner.get(owner_b).unwrap().contains(&alloc_b.allocation_id)); - } - - manager.release_handle(&alloc_a); - - { - let connections = manager.read_connections(); - assert!(!connections.by_owner.contains_key(owner_a), "owner-a index is removed after release"); - assert_eq!(connections.by_owner.get(owner_b).map_or(0, HashSet::len), 1, "owner-b still indexed"); - } - - manager.release_handle(&alloc_b); - { - let connections = manager.read_connections(); - assert!(connections.by_owner.is_empty(), "owner index is empty after all releases"); - } - } - - #[tokio::test] - async fn test_normal_connection_preempts_existing_soft_connection() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let soft_addr: SocketAddr = "127.0.0.1:49011".parse().unwrap(); - let normal_addr: SocketAddr = "127.0.0.1:49012".parse().unwrap(); - - let soft_alloc = manager - .acquire_connection(&input_name, &soft_addr, default_user_priority(), ConnectionKind::Soft) - .expect("soft allocation"); - let soft_token = soft_alloc.cancel_token.clone().expect("soft allocations expose a cancel token"); - - let normal_alloc = manager - .acquire_connection(&input_name, &normal_addr, default_user_priority(), ConnectionKind::Normal) - .expect("normal allocation should preempt soft allocation"); - - tokio::task::yield_now().await; - assert!(soft_token.is_cancelled(), "soft allocation should be preempted by normal traffic"); - assert_eq!(manager.get_provider_connections_count(), 1); - - manager.release_handle(&normal_alloc); - } - - #[tokio::test] - async fn test_higher_priority_soft_preempts_lower_priority_soft() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let low_soft_addr: SocketAddr = "127.0.0.1:49013".parse().unwrap(); - let high_soft_addr: SocketAddr = "127.0.0.1:49014".parse().unwrap(); - - let low_soft_alloc = manager - .acquire_connection(&input_name, &low_soft_addr, 10, ConnectionKind::Soft) - .expect("low-priority soft allocation"); - let low_soft_token = low_soft_alloc.cancel_token.clone().expect("soft allocations expose a cancel token"); - - let high_soft_alloc = manager - .acquire_connection(&input_name, &high_soft_addr, -5, ConnectionKind::Soft) - .expect("higher-priority soft allocation should preempt lower-priority soft allocation"); - - tokio::task::yield_now().await; - assert!(low_soft_token.is_cancelled(), "lower-priority soft allocation should be preempted"); - assert_eq!(manager.get_provider_connections_count(), 1); - - manager.release_handle(&high_soft_alloc); - } - - #[tokio::test] - async fn test_reclassify_soft_to_normal_prevents_same_priority_preemption() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let promoted_addr: SocketAddr = "127.0.0.1:49015".parse().unwrap(); - let challenger_addr: SocketAddr = "127.0.0.1:49016".parse().unwrap(); - - let promoted_alloc = manager - .acquire_connection(&input_name, &promoted_addr, default_user_priority(), ConnectionKind::Soft) - .expect("initial soft allocation"); - let promoted_token = promoted_alloc.cancel_token.clone().expect("soft allocations expose a cancel token"); - - assert!( - manager.reclassify_connection(&promoted_addr, ConnectionKind::Normal, default_user_priority()), - "soft allocation should be promotable to normal" - ); - - let challenger = manager.acquire_connection_with_grace( - &input_name, - &challenger_addr, - false, - default_user_priority(), - ConnectionKind::Normal, - ); - assert!(challenger.is_none(), "same-priority normal traffic should not preempt a promoted normal connection"); - assert!(!promoted_token.is_cancelled(), "promoted connection should remain active"); - assert_eq!(manager.get_provider_connections_count(), 1); - - manager.release_handle(&promoted_alloc); - } - - #[tokio::test] - async fn test_reclassify_connection_for_owner_preserves_other_owner_on_same_socket() { - let app_cfg = build_test_app_config(None, 2); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - - let input_name = "provider_1".intern(); - let shared_addr: SocketAddr = "127.0.0.1:49020".parse().unwrap(); - - let alloc_1 = manager - .acquire_connection_with_lease_for_session( - &input_name, - &shared_addr, - false, - -10, - ConnectionKind::Soft, - Some(PlaybackLeaseRef::new("owner-1", PlaybackKind::LiveTs)), - ) - .expect("first soft allocation"); - - let alloc_2 = manager - .acquire_connection_with_lease_for_session( - &input_name, - &shared_addr, - false, - -10, - ConnectionKind::Soft, - Some(PlaybackLeaseRef::new("owner-2", PlaybackKind::LiveTs)), - ) - .expect("second soft allocation"); - - assert!(manager.reclassify_connection_for_owner(&shared_addr, Some("owner-1"), ConnectionKind::Normal, 0)); - - { - let connections = manager.read_connections(); - let info_1 = connections.single.get(&alloc_1.allocation_id).expect("alloc_1 exists"); - assert_eq!(info_1.kind, ConnectionKind::Normal); - assert_eq!(info_1.priority, 0); - - let info_2 = connections.single.get(&alloc_2.allocation_id).expect("alloc_2 exists"); - assert_eq!(info_2.kind, ConnectionKind::Soft); - assert_eq!(info_2.priority, -10); - } - - manager.release_handle(&alloc_1); - manager.release_handle(&alloc_2); - } - - #[test] - fn recording_allocation_is_claimed_without_consuming_a_second_slot() { - let app_cfg = build_test_app_config(None, 1); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - - let reserved = - manager.acquire_connection_for_download(&input_name, 0).expect("recording reserves the only provider slot"); - assert_eq!(manager.get_provider_connections_count(), 1); - - let claimed = manager - .claim_download_connection(reserved.allocation_id, &input_name) - .expect("internal recording request claims the existing reservation"); - assert_eq!(claimed.allocation_id, reserved.allocation_id); - assert_eq!(manager.get_provider_connections_count(), 1, "claim must not allocate a second slot"); - assert!( - manager.claim_download_connection(reserved.allocation_id, &input_name).is_none(), - "one reservation can only own one provider body" - ); - - manager.complete_release(reserved.allocation_id); - assert_eq!(manager.get_provider_connections_count(), 0); - } - - /// A=2 / B=3 pool: five confirmed playbacks exhaust the pool and a sixth start is - /// rejected; releasing one confirmed playback frees exactly one slot again. - #[tokio::test] - async fn sixth_playback_rejected_after_pool_exhausted_by_confirmed_leases() { - let app_cfg = create_test_app_config_with_pool(2, 3); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - - let mut acquired = Vec::new(); - for index in 0..5u16 { - let owner = format!("playback-{index}"); - let addr = SocketAddr::from(([172, 18, 0, 9], 60_000 + index)); - let handle = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(&owner), - ) - .expect("pool has capacity for five playbacks"); - manager.confirm_playback_activity(&owner); - acquired.push((handle, owner)); - } - - let sixth = manager.acquire_connection_with_grace_for_session( - &input_name, - &SocketAddr::from(([172, 18, 0, 9], 60_005)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some("playback-5"), - ); - assert!(sixth.is_none(), "sixth start must be rejected once the pool is exhausted"); - - manager.release_handle(&acquired[0].0); - let replacement = manager.acquire_connection_with_grace_for_session( - &input_name, - &SocketAddr::from(([172, 18, 0, 9], 60_006)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some("playback-6"), - ); - assert!(replacement.is_some(), "freed capacity must be selectable again"); - manager.release_handle(&replacement.expect("replacement handle")); - for (handle, _owner) in &acquired[1..] { - manager.release_handle(handle); - } - assert_eq!(manager.get_provider_connections_count(), 0); - } - - #[test] - fn live_hls_lease_owner_preserves_other_playback_identities() { - let owner = "client|alice|42"; - assert_eq!(super::playback_lease_owner("client|alice|42|hls|0123456789abcdef"), owner); - for token in [ - owner, - "client|alice|42|hls|short", - "client|alice|42|hls|0123456789abcde!", - "m3u-catchup|client|alice|42|hls|0123456789abcdef", - "catchup|client|alice|42|hls|0123456789abcdef", - "hls-cache:client|hls|0123456789abcdef", - ] { - assert_eq!(super::playback_lease_owner(token), token); - } - assert_ne!( - super::playback_lease_owner("client|alice|42|hls|0123456789abcdef"), - super::playback_lease_owner("other-client|alice|42|hls|0123456789abcdef") - ); - } - - #[tokio::test(start_paused = true)] - async fn live_hls_retries_with_unique_tokens_keep_one_provider_lease() -> Result<(), String> { - let app_cfg = create_test_app_config_with_pool(2, 4); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input = "provider_1".intern(); - let alias = "provider_2".intern(); - let mut original_tag = None; - let mut previous_request: Option<( - String, - tuliprox_core::model::PlaybackRequestId, - Option, - )> = None; - for attempt in 0..104_u16 { - let token = format!("client|alice|42|hls|{attempt:016x}"); - let handle = manager - .acquire_connection_with_lease_for_session( - &input, - &SocketAddr::from(([172, 18, 0, 9], 55_000 + attempt)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new(&token, PlaybackKind::LiveHls)), - ) - .ok_or("retry must reuse the reserved provider")?; - let request_id = handle.playback_request_id.ok_or("identified HLS request")?; - let provider = handle.allocation.get_provider_name().ok_or("provider")?; - assert_eq!(provider, input); - let tag = handle.binding_tag.ok_or("binding tag")?; - let first_tag = original_tag.get_or_insert(tag); - assert_eq!(tag.lease_id, first_tag.lease_id); - manager.refresh_adaptive_playback_lease(&provider, &token, PlaybackKind::LiveHls, 60); - manager.confirm_identified_playback_activity(&token, request_id); - assert!(manager.should_reuse_playback_provider(&token, &provider)); - manager.release_handle(&handle); - manager.finish_identified_playback_request(&token, request_id, PlaybackRequestOutcome::Completed); - manager.clear_provider_reservation(&token); - if let Some((old_token, old_request, old_tag)) = previous_request.take() { - manager.clear_identified_provider_reservation(&old_token, &provider, old_tag); - manager.finish_identified_playback_request( - &old_token, - old_request, - PlaybackRequestOutcome::ProviderFailed, - ); - assert!(manager.confirm_identified_playback_activity(&old_token, old_request).is_none()); - } - previous_request = Some((token, request_id, handle.binding_tag)); - assert_eq!(manager.provider_lease_usage(&input).idle, 1); - assert_eq!(manager.provider_lease_usage(&alias).total(), 0); - } - let foreign = manager - .acquire_connection_with_lease_for_session( - &input, - &SocketAddr::from(([172, 18, 0, 10], 56_000)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new("other-client|alice|42|hls|0123456789abcdef", PlaybackKind::LiveHls)), - ) - .ok_or("another playback must retain free provider capacity")?; - manager.release_handle(&foreign); - tokio::time::advance(Duration::from_secs(61)).await; - assert_eq!(manager.provider_lease_usage(&input).total(), 0); - Ok(()) - } - - /// Primary `provider_1` with an alias `provider_2` behind it. - fn alias_pool(primary_max: u16, alias_max: u16) -> (ActiveProviderManager, Arc, Arc) { - let app_cfg = create_test_app_config_with_pool(primary_max, alias_max); - let manager = ActiveProviderManager::new(&app_cfg, &Arc::new(EventManager::new())); - (manager, "provider_1".intern(), "provider_2".intern()) - } - - fn acquire_live_hls_from_lineup( - manager: &ActiveProviderManager, - input: &Arc, - token: &str, - port: u16, - ) -> Result { - manager - .acquire_connection_with_lease_for_session( - input, - &SocketAddr::from(([172, 18, 0, 9], port)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new(token, PlaybackKind::LiveHls)), - ) - .ok_or_else(|| format!("no provider for {token}")) - } - - /// Starts `token` on the alias while the primary is full, confirms media, ends the - /// request cleanly, frees the primary and lets the reconnect lease lapse. - async fn live_hls_playback_on_alias_with_lapsed_lease( - manager: &ActiveProviderManager, - input: &Arc, - alias: &Arc, - token: &str, - outcome: PlaybackRequestOutcome, - ) -> Result<(), String> { - let busy_1 = acquire_live_hls_from_lineup(manager, input, "busy-1|bob|7|hls|0123456789abcdef", 50_001)?; - let busy_2 = acquire_live_hls_from_lineup(manager, input, "busy-2|carol|8|hls|0123456789abcdef", 50_002)?; - assert_eq!(busy_1.allocation.get_provider_name().as_ref(), Some(input)); - assert_eq!(busy_2.allocation.get_provider_name().as_ref(), Some(input)); - - let handle = acquire_live_hls_from_lineup(manager, input, token, 50_010)?; - let provider = handle.allocation.get_provider_name().ok_or("provider")?; - assert_eq!(&provider, alias, "full primary must fall back to the alias"); - let request_id = handle.playback_request_id.ok_or("identified request")?; - manager.refresh_adaptive_playback_lease(&provider, token, PlaybackKind::LiveHls, 15); - manager.confirm_identified_playback_activity(token, request_id).ok_or("confirmation")?; - manager.release_handle(&handle); - manager.finish_identified_playback_request(token, request_id, outcome); - - manager.release_handle(&busy_1); - manager.release_handle(&busy_2); - tokio::time::advance(Duration::from_secs(20)).await; - assert_eq!(manager.provider_lease_usage(alias).total(), 0, "reconnect lease must have lapsed"); - assert_eq!(manager.provider_lease_usage(input).total(), 0); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn live_hls_reentry_keeps_alias_affinity_after_lease_expiry() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 0); - live_hls_playback_on_alias_with_lapsed_lease( - &manager, - &input, - &alias, - "client|alice|42|hls|0123456789abcdef", - PlaybackRequestOutcome::Completed, - ) - .await?; - - let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; - assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&alias))); - // Affinity reserves nothing: another playback still gets the free primary slot. - let other = acquire_live_hls_from_lineup(&manager, &input, "client|dave|42|hls|0123456789abcdef", 50_012)?; - assert_eq!(other.allocation.get_provider_name(), Some(Arc::clone(&input))); - manager.release_handle(&reentry); - manager.release_handle(&other); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn live_hls_alias_affinity_ends_after_its_window() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 0); - live_hls_playback_on_alias_with_lapsed_lease( - &manager, - &input, - &alias, - "client|alice|42|hls|0123456789abcdef", - PlaybackRequestOutcome::Completed, - ) - .await?; - tokio::time::advance(Duration::from_secs(shared::defaults::default_provider_affinity_ttl_secs())).await; - - let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; - assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); - manager.release_handle(&reentry); - Ok(()) - } - - /// Starts `token` on the alias while the primary is full and confirms media. The - /// request stays attached; the primary is freed again before returning. - fn confirmed_alias_request( - manager: &ActiveProviderManager, - input: &Arc, - alias: &Arc, - token: &str, - ) -> Result { - let busy_1 = acquire_live_hls_from_lineup(manager, input, "busy-1|bob|7|hls|0123456789abcdef", 50_001)?; - let busy_2 = acquire_live_hls_from_lineup(manager, input, "busy-2|carol|8|hls|0123456789abcdef", 50_002)?; - let handle = acquire_live_hls_from_lineup(manager, input, token, 50_010)?; - assert_eq!(handle.allocation.get_provider_name().as_ref(), Some(alias)); - let request_id = handle.playback_request_id.ok_or("identified request")?; - manager.refresh_adaptive_playback_lease(alias, token, PlaybackKind::LiveHls, 15); - manager.confirm_identified_playback_activity(token, request_id).ok_or("confirmation")?; - manager.release_handle(&busy_1); - manager.release_handle(&busy_2); - Ok(handle) - } - - #[tokio::test(start_paused = true)] - async fn provider_failure_after_lease_expiry_ends_alias_affinity() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 0); - let token = "client|alice|42|hls|0123456789abcdef"; - let handle = confirmed_alias_request(&manager, &input, &alias, token)?; - let request_id = handle.playback_request_id.ok_or("identified request")?; - manager.release_handle(&handle); - tokio::time::advance(Duration::from_secs(20)).await; - manager.prune_expired_leases_now(); - assert_eq!(manager.provider_lease_usage(&alias).total(), 0, "lease must have expired"); - - manager.finish_identified_playback_request(token, request_id, PlaybackRequestOutcome::ProviderFailed); - - let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; - assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); - manager.release_handle(&reentry); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn delayed_failure_of_older_binding_keeps_successor_affinity() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 0); - let old_token = "client|alice|42|hls|0123456789abcdef"; - let new_token = "client|alice|42|hls|fedcba9876543210"; - - // An older binding on the primary that never confirmed media. - let old = manager - .acquire_exact_connection_with_lease_for_session( - &input, - &SocketAddr::from(([172, 18, 0, 9], 50_000)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new(old_token, PlaybackKind::LiveHls)), - ) - .ok_or("old binding")?; - let old_request = old.playback_request_id.ok_or("old request")?; - let old_tag = old.binding_tag.ok_or("old tag")?; - manager.release_handle(&old); - - let current = confirmed_alias_request(&manager, &input, &alias, new_token)?; - let current_request = current.playback_request_id.ok_or("current request")?; - manager.release_handle(¤t); - manager.finish_identified_playback_request(new_token, current_request, PlaybackRequestOutcome::Completed); - - // Delayed manifest failures of the older binding arrive after the rebind. - assert!(!manager.forget_identified_provider_affinity(old_token, &input, old_tag)); - manager.finish_identified_playback_request(old_token, old_request, PlaybackRequestOutcome::ProviderFailed); - tokio::time::advance(Duration::from_secs(20)).await; - assert_eq!(manager.provider_lease_usage(&alias).total(), 0, "lease must have expired"); - manager.finish_identified_playback_request(old_token, old_request, PlaybackRequestOutcome::ProviderFailed); - - let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|00000000000000aa", 50_011)?; - assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&alias))); - manager.release_handle(&reentry); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn administrative_termination_of_hls_token_ends_affinity_and_lease() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 0); - let token = "client|alice|42|hls|0123456789abcdef"; - let handle = confirmed_alias_request(&manager, &input, &alias, token)?; - let request_id = handle.playback_request_id.ok_or("identified request")?; - manager.release_handle(&handle); - manager.finish_identified_playback_request(token, request_id, PlaybackRequestOutcome::Completed); - assert_eq!(manager.provider_lease_usage(&alias).idle, 1); - - // The bare-owner clear has no delete right for public HLS tokens. - manager.clear_provider_reservation(token); - assert_eq!(manager.provider_lease_usage(&alias).idle, 1); - manager.terminate_identified_playback_owner(token); - assert_eq!(manager.provider_lease_usage(&alias).total(), 0); - - let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; - assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); - manager.release_handle(&reentry); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn stale_hls_token_termination_keeps_newer_binding() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 0); - let old_token = "client|alice|42|hls|0123456789abcdef"; - let new_token = "client|alice|42|hls|fedcba9876543210"; - - let old = confirmed_alias_request(&manager, &input, &alias, old_token)?; - let old_request = old.playback_request_id.ok_or("old request")?; - manager.release_handle(&old); - manager.finish_identified_playback_request(old_token, old_request, PlaybackRequestOutcome::Completed); - - // The retry re-enters the idle lease and starts a new binding incarnation. - let new = acquire_live_hls_from_lineup(&manager, &input, new_token, 50_011)?; - assert_eq!(new.allocation.get_provider_name(), Some(Arc::clone(&alias))); - assert_ne!(new.binding_tag, old.binding_tag); - let new_request = new.playback_request_id.ok_or("new request")?; - manager.refresh_adaptive_playback_lease(&alias, new_token, PlaybackKind::LiveHls, 15); - manager.confirm_identified_playback_activity(new_token, new_request).ok_or("confirmation")?; - manager.release_handle(&new); - manager.finish_identified_playback_request(new_token, new_request, PlaybackRequestOutcome::Completed); - assert_eq!(manager.provider_lease_usage(&alias).idle, 1); - - assert!(!manager.terminate_identified_playback_owner(old_token)); - assert_eq!(manager.provider_lease_usage(&alias).idle, 1, "stale token must keep the newer lease"); - assert_eq!(manager.provider_affinity_for_owner(new_token), Some(Arc::clone(&alias))); - - assert!(manager.terminate_identified_playback_owner(new_token)); - assert_eq!(manager.provider_lease_usage(&alias).total(), 0); - assert_eq!(manager.provider_affinity_for_owner(new_token), None); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn terminating_token_of_expired_binding_ends_its_affinity_only() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 0); - let token = "client|alice|42|hls|0123456789abcdef"; - let handle = confirmed_alias_request(&manager, &input, &alias, token)?; - manager.release_handle(&handle); - tokio::time::advance(Duration::from_secs(20)).await; - assert_eq!(manager.provider_lease_usage(&alias).total(), 0, "lease must have expired"); - - assert!(!manager.terminate_identified_playback_owner("client|alice|42|hls|00000000000000aa")); - assert_eq!(manager.provider_affinity_for_owner(token), Some(Arc::clone(&alias))); - assert!(manager.terminate_identified_playback_owner(token)); - assert_eq!(manager.provider_affinity_for_owner(token), None); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn finished_retry_tokens_do_not_accumulate_on_an_active_binding() -> Result<(), String> { - let app_cfg = create_test_app_config_with_pool(2, 0); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input = "provider_1".intern(); - let first = "client|alice|42|hls|0000000000000000"; - // One request stays in flight for the whole run and keeps the binding active. - let anchor = acquire_live_hls_from_lineup(&manager, &input, first, 50_000)?; - let anchor_request = anchor.playback_request_id.ok_or("anchor request")?; - manager.refresh_adaptive_playback_lease(&input, first, PlaybackKind::LiveHls, 15); - manager.confirm_identified_playback_activity(first, anchor_request).ok_or("confirmation")?; - - let mut last_token = String::new(); - for attempt in 1..=1_000_u32 { - last_token = format!("client|alice|42|hls|{attempt:016x}"); - let handle = acquire_live_hls_from_lineup(&manager, &input, &last_token, 50_001)?; - assert_eq!(handle.binding_tag, anchor.binding_tag, "retries share the active binding"); - let request_id = handle.playback_request_id.ok_or("retry request")?; - manager.refresh_adaptive_playback_lease(&input, &last_token, PlaybackKind::LiveHls, 15); - manager.release_handle(&handle); - manager.finish_identified_playback_request(&last_token, request_id, PlaybackRequestOutcome::Completed); - tokio::time::advance(Duration::from_secs(1)).await; - } - - let claims = manager.write_leases().request_token_claims(super::playback_lease_owner(first)); - assert!(claims <= 17, "only the anchor and recently finished retries may hold a claim, got {claims}"); - // The anchor keeps its claim while its request runs; a recent retry still has its cleanup right. - assert!(manager.write_leases().request_token_claims(super::playback_lease_owner(first)) >= 2); - assert!(manager.terminate_identified_playback_owner(&last_token)); - manager.release_handle(&anchor); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn affinity_never_forces_grace_over_allocation_while_primary_is_free() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 1); - live_hls_playback_on_alias_with_lapsed_lease( - &manager, - &input, - &alias, - "client|alice|42|hls|0123456789abcdef", - PlaybackRequestOutcome::Completed, - ) - .await?; - let alias_holder = manager - .acquire_exact_connection_with_lease_for_session( - &alias, - &SocketAddr::from(([172, 18, 0, 9], 50_020)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new("client|erin|9|hls|0123456789abcdef", PlaybackKind::LiveHls)), - ) - .ok_or("alias holder")?; - - // Grace is enabled by default and allowed for this request. - let reentry = manager - .acquire_connection_with_lease_for_session( - &input, - &SocketAddr::from(([172, 18, 0, 9], 50_011)), - true, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new("client|alice|42|hls|fedcba9876543210", PlaybackKind::LiveHls)), - ) - .ok_or("re-entry")?; - assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); - assert!( - matches!(reentry.allocation, ProviderAllocation::Available(_)), - "a free primary slot beats a grace allocation" - ); - manager.release_handle(&reentry); - manager.release_handle(&alias_holder); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn failure_on_fallback_provider_keeps_preference_for_preferred_provider() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 1); - let token = "client|alice|42|hls|0123456789abcdef"; - live_hls_playback_on_alias_with_lapsed_lease( - &manager, - &input, - &alias, - token, - PlaybackRequestOutcome::Completed, - ) - .await?; - let alias_holder = manager - .acquire_exact_connection_with_lease_for_session( - &alias, - &SocketAddr::from(([172, 18, 0, 9], 50_020)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new("client|erin|9|hls|0123456789abcdef", PlaybackKind::LiveHls)), - ) - .ok_or("alias holder")?; - - // The full alias sends the re-entry to the primary, which then fails before media. - let fallback = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; - assert_eq!(fallback.allocation.get_provider_name(), Some(Arc::clone(&input))); - let fallback_request = fallback.playback_request_id.ok_or("fallback request")?; - manager.release_handle(&fallback); - manager.finish_identified_playback_request( - "client|alice|42|hls|fedcba9876543210", - fallback_request, - PlaybackRequestOutcome::FailedBeforeMedia, - ); - - assert_eq!(manager.provider_affinity_for_owner(token), Some(Arc::clone(&alias))); - manager.release_handle(&alias_holder); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn live_ts_playback_records_no_provider_affinity() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 0); - let busy_1 = acquire_live_hls_from_lineup(&manager, &input, "busy-1|bob|7|hls|0123456789abcdef", 50_001)?; - let busy_2 = acquire_live_hls_from_lineup(&manager, &input, "busy-2|carol|8|hls|0123456789abcdef", 50_002)?; - let owner = "ts-session-token"; - let handle = manager - .acquire_connection_with_lease_for_session( - &input, - &SocketAddr::from(([172, 18, 0, 9], 50_010)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveTs)), - ) - .ok_or("ts playback")?; - assert_eq!(handle.allocation.get_provider_name(), Some(Arc::clone(&alias))); - let request_id = handle.playback_request_id.ok_or("ts request")?; - manager.confirm_identified_playback_activity(owner, request_id).ok_or("confirmation")?; - manager.release_handle(&handle); - manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::ClientClosed); - manager.release_handle(&busy_1); - manager.release_handle(&busy_2); - - assert_eq!(manager.provider_affinity_for_owner(owner), None, "live TS has nothing to return to"); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn huge_reconnect_and_affinity_windows_never_panic() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 0); - manager.write_leases().set_affinity_ttl(crate::provider_leases::ProviderAffinityTtl(u64::MAX)); - let token = "client|alice|42|hls|0123456789abcdef"; - let handle = acquire_live_hls_from_lineup(&manager, &input, token, 50_010)?; - let provider = handle.allocation.get_provider_name().ok_or("provider")?; - let request_id = handle.playback_request_id.ok_or("request")?; - manager.refresh_adaptive_playback_lease(&provider, token, PlaybackKind::LiveHls, u64::MAX); - manager.confirm_identified_playback_activity(token, request_id).ok_or("confirmation")?; - manager.release_handle(&handle); - manager.finish_identified_playback_request(token, request_id, PlaybackRequestOutcome::Completed); - assert_eq!(manager.provider_affinity_for_owner(token), Some(provider)); - assert_eq!(manager.provider_lease_usage(&alias).total(), 0); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn zero_provider_affinity_ttl_ends_affinity_with_reconnect_window() -> Result<(), String> { - let app_cfg = create_test_app_config_with_pool(2, 0); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - manager.write_leases().set_affinity_ttl(crate::provider_leases::ProviderAffinityTtl(0)); - let input = "provider_1".intern(); - let alias = "provider_2".intern(); - live_hls_playback_on_alias_with_lapsed_lease( - &manager, - &input, - &alias, - "client|alice|42|hls|0123456789abcdef", - PlaybackRequestOutcome::Completed, - ) - .await?; - - let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; - assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); - manager.release_handle(&reentry); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn live_hls_provider_failure_ends_alias_affinity() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 0); - live_hls_playback_on_alias_with_lapsed_lease( - &manager, - &input, - &alias, - "client|alice|42|hls|0123456789abcdef", - PlaybackRequestOutcome::ProviderFailed, - ) - .await?; - - let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; - assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); - manager.release_handle(&reentry); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn live_hls_full_affinity_provider_falls_back_to_lineup() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 1); - live_hls_playback_on_alias_with_lapsed_lease( - &manager, - &input, - &alias, - "client|alice|42|hls|0123456789abcdef", - PlaybackRequestOutcome::Completed, - ) - .await?; - let alias_holder = manager - .acquire_exact_connection_with_lease_for_session( - &alias, - &SocketAddr::from(([172, 18, 0, 9], 50_020)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new("client|erin|9|hls|0123456789abcdef", PlaybackKind::LiveHls)), - ) - .ok_or("alias holder")?; - - let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; - assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); - manager.release_handle(&reentry); - manager.release_handle(&alias_holder); - Ok(()) - } - - #[tokio::test] - async fn live_hls_socket_cleanup_only_cancels_its_public_session() -> Result<(), String> { - let app_cfg = create_test_app_config_with_pool(2, 4); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input = "provider_1".intern(); - let addr = SocketAddr::from(([172, 18, 0, 9], 55_000)); - let old = "client|alice|42|hls|0123456789abcdef"; - let new = "client|alice|42|hls|fedcba9876543210"; - let first = manager - .acquire_exact_connection_with_lease_for_session( - &input, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new(old, PlaybackKind::LiveHls)), - ) - .ok_or("old allocation")?; - let next = manager - .acquire_exact_connection_with_lease_for_session( - &input, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new(new, PlaybackKind::LiveHls)), - ) - .ok_or("new allocation")?; - assert_eq!(first.binding_tag, next.binding_tag); - assert_eq!(manager.provider_lease_usage(&input).total(), 1); - manager.release_playback_connections(old, &[addr]); - manager.release_playback_connections_await(old, &[addr]).await; - assert_eq!(manager.get_provider_connections_count(), 1); - assert!(!next.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); - manager.release_handle(&next); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn live_hls_old_token_cleanup_cannot_mutate_rebound_retry() -> Result<(), String> { - let (manager, input, alias) = alias_pool(2, 4); - let old = "client|alice|42|hls|0123456789abcdef"; - let new = "client|alice|42|hls|fedcba9876543210"; - let addr = SocketAddr::from(([172, 18, 0, 9], 55_000)); - let first = manager - .acquire_exact_connection_with_lease_for_session( - &input, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new(old, PlaybackKind::LiveHls)), - ) - .ok_or("initial allocation")?; - let old_request = first.playback_request_id.ok_or("old request")?; - manager.refresh_adaptive_playback_lease(&input, old, PlaybackKind::LiveHls, 60); - manager.confirm_identified_playback_activity(old, old_request); - manager.release_handle(&first); - let next = manager - .acquire_exact_connection_with_lease_for_session( - &alias, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new(new, PlaybackKind::LiveHls)), - ) - .ok_or("rebound allocation")?; - let new_request = next.playback_request_id.ok_or("new request")?; - assert_ne!(first.binding_tag, next.binding_tag); - manager.refresh_adaptive_playback_lease(&alias, new, PlaybackKind::LiveHls, 60); - manager.confirm_identified_playback_activity(new, new_request); - manager.release_handle(&next); - manager.finish_identified_playback_request(new, new_request, PlaybackRequestOutcome::Completed); - let before = manager.binding_tag_for_owner(new); - manager.clear_provider_reservation(old); - manager.clear_identified_provider_reservation(old, &input, first.binding_tag); - manager.finish_identified_playback_request(old, old_request, PlaybackRequestOutcome::ProviderFailed); - manager.refresh_playback_lease( - &input, - &PlaybackLeaseRef { owner: old, kind: PlaybackKind::LiveHls, request_id: old_request }, - 0, - ); - assert!(manager.confirm_identified_playback_activity(old, old_request).is_none()); - assert_eq!(manager.binding_tag_for_owner(new), before); - assert_eq!(manager.provider_lease_usage(&input).total(), 0); - assert_eq!(manager.provider_lease_usage(&alias).idle, 1); - Ok(()) - } - - /// Confirmed reconnect-capable lease, physical handle release first, then the real - /// provider-error outcome: the error must reach the lease and leave no idle reserve. - #[tokio::test(start_paused = true)] - async fn provider_error_cleanup_after_physical_release_does_not_keep_idle_lease() { - let app_cfg = create_test_app_config_with_pool(2, 3); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - let owner = "r5-error-owner"; - let addr = SocketAddr::from(([172, 18, 0, 9], 55_000)); - let handle = manager - .acquire_connection_with_lease_for_session( - &input_name, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveHls)), - ) - .expect("acquire a reconnect-capable slot"); - let request_id = handle.playback_request_id.expect("identified request id"); - manager.refresh_adaptive_playback_lease(&input_name, owner, PlaybackKind::LiveHls, 15); - manager.confirm_playback_activity(owner); - - // Physical release must not conclude the request; the outcome is decided later. - manager.release_handle(&handle); - assert_eq!( - manager.provider_lease_usage(&input_name).active, - 1, - "physical release must not finish a confirmed lease" - ); - - manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::ProviderFailed); - assert_eq!( - manager.provider_lease_usage(&input_name).total(), - 0, - "provider error must not keep an idle reconnect lease" - ); - } - - #[tokio::test(start_paused = true)] - async fn stale_refresh_does_not_recreate_terminal_request() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - let owner = "terminal-refresh-owner"; - let handle = manager - .acquire_connection_with_lease_for_session( - &input_name, - &SocketAddr::from(([172, 18, 0, 9], 55_002)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveHls)), - ) - .expect("acquire identified request"); - let request_id = handle.playback_request_id.expect("identified request id"); - let lease_ref = PlaybackLeaseRef { owner, kind: PlaybackKind::LiveHls, request_id }; - - manager.release_handle(&handle); - manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::ProviderFailed); - assert_eq!(manager.provider_lease_usage(&input_name).total(), 0); - - manager.refresh_playback_lease(&input_name, &lease_ref, 15); - manager.confirm_identified_playback_activity(owner, request_id); - assert_eq!(manager.provider_lease_usage(&input_name).total(), 0); - } - - /// Counterexample: the same sequence with a clean end keeps the configured idle window. - #[tokio::test(start_paused = true)] - async fn clean_cleanup_after_physical_release_keeps_configured_idle_window() { - let app_cfg = create_test_app_config_with_pool(2, 3); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - let owner = "r5-clean-owner"; - let addr = SocketAddr::from(([172, 18, 0, 9], 55_001)); - - let handle = manager - .acquire_connection_with_lease_for_session( - &input_name, - &addr, - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveHls)), - ) - .expect("acquire a reconnect-capable slot"); - let request_id = handle.playback_request_id.expect("identified request id"); - manager.refresh_adaptive_playback_lease(&input_name, owner, PlaybackKind::LiveHls, 15); - manager.confirm_playback_activity(owner); - - manager.release_handle(&handle); - manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::Completed); - let usage = manager.provider_lease_usage(&input_name); - assert_eq!(usage.active, 0, "clean end must move the lease out of active"); - assert_eq!(usage.idle, 1, "clean end must keep the configured reconnect window"); - } - - /// max=1: X fetch → physical release → Y acquire → X first byte within the startup - /// deadline. X's late confirmation records media but must not over-commit Y's slot. - #[tokio::test(start_paused = true)] - async fn late_first_byte_cannot_reserve_slot_taken_by_other_playback() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - - let x_handle = manager - .acquire_connection_with_lease_for_session( - &input_name, - &SocketAddr::from(([172, 18, 0, 9], 40_000)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new("x", PlaybackKind::LiveHls)), - ) - .expect("x acquires the only slot"); - manager.refresh_adaptive_playback_lease(&input_name, "x", PlaybackKind::LiveHls, 15); - manager.release_handle(&x_handle); - - let y_handle = manager - .acquire_connection_with_lease_for_session( - &input_name, - &SocketAddr::from(([172, 18, 0, 9], 40_001)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new("y", PlaybackKind::LiveHls)), - ) - .expect("y acquires the now-free slot"); - - manager.confirm_playback_activity("x"); - assert_eq!(manager.get_provider_connections_count(), 1, "y still holds the only slot"); - assert!( - !manager.is_provider_reserved_for_other_session(&input_name, Some("y")), - "x's late confirmation must not over-commit the provider" - ); - - manager.confirm_playback_activity("y"); - manager.release_handle(&y_handle); - manager.finish_playback_request("y", PlaybackRequestOutcome::Completed, None); - } - - /// Counterexample: the same X fetch → release → first-byte sequence without a - /// competing Y keeps the free slot as a real reservation. - #[tokio::test(start_paused = true)] - async fn first_byte_reserves_when_slot_still_free() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let input_name = "provider_1".intern(); - - let x_handle = manager - .acquire_connection_with_lease_for_session( - &input_name, - &SocketAddr::from(([172, 18, 0, 9], 40_002)), - false, - default_user_priority(), - ConnectionKind::Normal, - Some(PlaybackLeaseRef::new("x", PlaybackKind::LiveHls)), - ) - .expect("x acquires the only slot"); - manager.refresh_adaptive_playback_lease(&input_name, "x", PlaybackKind::LiveHls, 15); - manager.release_handle(&x_handle); - - // No competing playback took the slot, so X's first byte reserves it. - manager.confirm_playback_activity("x"); - assert!( - manager.is_provider_reserved_for_other_session(&input_name, Some("other")), - "a free slot must be reservable by the confirming playback" - ); - - manager.finish_playback_request("x", PlaybackRequestOutcome::Completed, None); - } - - /// The RAII owner releases the allocation synchronously even when dropped outside - /// a tokio runtime, so a body/context drop can never leak a provider slot. - #[test] - fn managed_handle_drop_outside_runtime_releases_allocation() { - let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); - let (manager, managed_handle) = rt.block_on(async { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let input_name = "provider_1".intern(); - - let handle = manager - .acquire_connection_with_grace_for_session( - &input_name, - &SocketAddr::from(([127, 0, 0, 1], 45_000)), - false, - 0, - ConnectionKind::Normal, - Some("managed-owner"), - ) - .expect("allocation should succeed"); - assert_eq!(manager.get_provider_connections_count(), 1); - - let managed_handle = super::ManagedProviderHandle::new(Arc::clone(&manager), handle); - (manager, managed_handle) - }); - - drop(managed_handle); - assert_eq!(manager.get_provider_connections_count(), 0); - } - - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss, clippy::cast_precision_loss)] - fn latency_percentile(sorted: &[u64], p: f64) -> u64 { - if sorted.is_empty() { - return 0; - } - let index = ((sorted.len() as f64 - 1.0) * p).round() as usize; - sorted[index.min(sorted.len() - 1)] - } - - #[cfg(target_os = "linux")] - fn resident_set_kib() -> Option { - let Ok(content) = std::fs::read_to_string("/proc/self/status") else { - return None; - }; - for line in content.lines() { - if let Some(rest) = line.strip_prefix("VmRSS:") { - return rest.split_whitespace().next().and_then(|v| v.parse::().ok()); - } - } - None - } - - #[cfg(not(target_os = "linux"))] - fn resident_set_kib() -> Option { None } - - struct LeaseWorkloadStats { - ops: u64, - elapsed_secs: f64, - acquire_p50: u64, - acquire_p95: u64, - acquire_p99: u64, - confirm_p50: u64, - confirm_p95: u64, - confirm_p99: u64, - release_p50: u64, - release_p95: u64, - release_p99: u64, - } - - #[allow(clippy::cast_possible_truncation)] - async fn measure_lease_workload( - manager: &Arc, - input_name: &Arc, - concurrency: usize, - rounds_per_task: usize, - ) -> LeaseWorkloadStats { - let samples = Arc::new(std::sync::Mutex::new(Vec::<(u64, u64, u64)>::new())); - let started = std::time::Instant::now(); - let mut tasks = tokio::task::JoinSet::new(); - for task_index in 0..concurrency { - let manager = Arc::clone(manager); - let input_name = Arc::clone(input_name); - let samples = Arc::clone(&samples); - tasks.spawn(async move { - for round in 0..rounds_per_task { - let owner = format!("bench-{task_index}-{round}"); - let addr = SocketAddr::from(([127, 0, 0, 1], 40_000 + (task_index as u16))); - let acquire_at = std::time::Instant::now(); - let Some(handle) = manager.acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - 0, - ConnectionKind::Normal, - Some(&owner), - ) else { - continue; - }; - let acquire_us = acquire_at.elapsed().as_micros() as u64; - - let confirm_at = std::time::Instant::now(); - manager.confirm_playback_activity(&owner); - let confirm_us = confirm_at.elapsed().as_micros() as u64; - - let release_at = std::time::Instant::now(); - manager.release_handle(&handle); - let release_us = release_at.elapsed().as_micros() as u64; - - // A physical release never concludes the request; model the real - // lifecycle by finishing with a clean outcome so the lease returns - // to baseline (LiveTs is not reconnect-capable, so it is removed). - manager.finish_playback_request(&owner, PlaybackRequestOutcome::Completed, None); - - samples.lock().unwrap().push((acquire_us, confirm_us, release_us)); - } - }); - } - while let Some(result) = tasks.join_next().await { - result.expect("benchmark task must not fail"); - } - - let elapsed_secs = started.elapsed().as_secs_f64(); - let samples = Arc::try_unwrap(samples).expect("samples unique").into_inner().unwrap(); - let mut acquire = Vec::with_capacity(samples.len()); - let mut confirm = Vec::with_capacity(samples.len()); - let mut release = Vec::with_capacity(samples.len()); - for (a, c, r) in samples { - acquire.push(a); - confirm.push(c); - release.push(r); - } - acquire.sort_unstable(); - confirm.sort_unstable(); - release.sort_unstable(); - let ops = acquire.len() as u64; - LeaseWorkloadStats { - ops, - elapsed_secs, - acquire_p50: latency_percentile(&acquire, 0.50), - acquire_p95: latency_percentile(&acquire, 0.95), - acquire_p99: latency_percentile(&acquire, 0.99), - confirm_p50: latency_percentile(&confirm, 0.50), - confirm_p95: latency_percentile(&confirm, 0.95), - confirm_p99: latency_percentile(&confirm, 0.99), - release_p50: latency_percentile(&release, 0.50), - release_p95: latency_percentile(&release, 0.95), - release_p99: latency_percentile(&release, 0.99), - } - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - #[ignore = "manager microbenchmark: cargo +stable test -p tuliprox-session --release -- --ignored --nocapture bench_provider_lease_microbenchmark"] - #[allow(clippy::cast_precision_loss)] - async fn bench_provider_lease_microbenchmark() { - let manager = Arc::new(ActiveProviderManager::new( - &create_test_app_config_single_unlimited_provider_pool(), - &Arc::new(EventManager::new()), - )); - let input_name: Arc = "provider_1".intern(); - - // Warm-up before measurement so allocator and lock caches are exercised. - for index in 0..64u16 { - let addr = SocketAddr::from(([127, 0, 0, 1], 40_000 + index)); - let Some(handle) = manager.acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - 0, - ConnectionKind::Normal, - None, - ) else { - continue; - }; - manager.release_handle(&handle); - } - - let baseline_rss_kib = resident_set_kib(); - eprintln!("provider lease manager microbenchmark (unlimited provider, in-process, no HTTP bodies)"); - for concurrency in [1usize, 5, 50, 200] { - let stats = measure_lease_workload(&manager, &input_name, concurrency, 10).await; - assert_eq!( - stats.ops, - (concurrency * 10) as u64, - "unexpected acquire failures at concurrency {concurrency}" - ); - let throughput = stats.ops as f64 / stats.elapsed_secs.max(f64::EPSILON); - let rss_delta_kib = resident_set_kib() - .zip(baseline_rss_kib) - .map_or_else(|| "unsupported".to_string(), |(rss, baseline)| rss.saturating_sub(baseline).to_string()); - eprintln!( - "concurrency={concurrency:>3} ops={:>5} throughput={:>9.1} ops/s | acquire p50/p95/p99={}/{}/{}us | confirm p50/p95/p99={}/{}/{}us | release p50/p95/p99={}/{}/{}us | rss_delta_kib={}", - stats.ops, - throughput, - stats.acquire_p50, - stats.acquire_p95, - stats.acquire_p99, - stats.confirm_p50, - stats.confirm_p95, - stats.confirm_p99, - stats.release_p50, - stats.release_p95, - stats.release_p99, - rss_delta_kib, - ); - } - - // Soak-style churn: every connection and lease must return to baseline. - assert_eq!(manager.get_provider_connections_count(), 0); - let usage = manager.provider_lease_usage(&input_name); - assert_eq!(usage.total(), 0, "lease table must return to baseline after churn"); - } - - #[tokio::test] - async fn closing_state_blocks_slot_release_until_completion_token_cancelled() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let input_name: Arc = "provider_1".intern(); - - let handle1 = manager - .acquire_connection_with_grace_for_session( - &input_name, - &SocketAddr::from(([127, 0, 0, 1], 46_001)), - false, - 0, - ConnectionKind::Normal, - Some("session-1"), - ) - .expect("first allocation should succeed"); - assert_eq!(manager.get_provider_connections_count(), 1); - - let completion_token = handle1.completion_token.clone().expect("completion token present"); - let alloc_id = handle1.allocation_id; - - // Mark Closing (simulating supersede / release_playback_connections_await) - manager.mark_closing(alloc_id); - - // Drop the handle while completion_token is still pending (upstream body not yet closed) - manager.release_handle(&handle1); - - // The slot must still be occupied by the Closing allocation - assert_eq!(manager.get_provider_connections_count(), 1); - - // A second start attempt must fail because the provider is at its limit (1 connection) - let handle2 = manager.acquire_connection_with_grace_for_session( - &input_name, - &SocketAddr::from(([127, 0, 0, 1], 46_002)), - false, - 0, - ConnectionKind::Normal, - Some("session-2"), - ); - assert!(handle2.is_none(), "second start must be rejected while first slot is Closing"); - - // Now signal completion (upstream body owner dropped upstream socket) - completion_token.cancel(); - - // Allow background reaper task to execute complete_release - tokio::time::sleep(Duration::from_millis(50)).await; - - assert_eq!(manager.get_provider_connections_count(), 0); - - // Second start attempt now succeeds - let handle3 = manager - .acquire_connection_with_grace_for_session( - &input_name, - &SocketAddr::from(([127, 0, 0, 1], 46_003)), - false, - 0, - ConnectionKind::Normal, - Some("session-3"), - ) - .expect("start must succeed after completion releases the slot"); - assert_eq!(manager.get_provider_connections_count(), 1); - - manager.release_handle(&handle3); - assert_eq!(manager.get_provider_connections_count(), 0); - } - - #[tokio::test] - async fn reaper_cleanup_is_idempotent_and_removes_all_indices() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let input_name: Arc = "provider_1".intern(); - - let handle = manager - .acquire_connection_with_grace_for_session( - &input_name, - &SocketAddr::from(([127, 0, 0, 1], 47_001)), - false, - 0, - ConnectionKind::Normal, - Some("session-idempotent"), - ) - .expect("allocation should succeed"); - - let completion_token = handle.completion_token.clone().expect("completion token present"); - let alloc_id = handle.allocation_id; - manager.mark_closing(alloc_id); - - manager.release_handle(&handle); - completion_token.cancel(); - - tokio::time::sleep(Duration::from_millis(50)).await; - assert_eq!(manager.get_provider_connections_count(), 0); - - // Duplicate releases must be completely safe and no-op - manager.complete_release(alloc_id); - manager.release_handle(&handle); - manager.release_connection(&SocketAddr::from(([127, 0, 0, 1], 47_001))); - assert_eq!(manager.get_provider_connections_count(), 0); - } - - #[tokio::test] - async fn capacity_release_wakes_only_waiters_of_that_provider() { - use futures::FutureExt; - let app_cfg = create_test_app_config_with_dual_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveProviderManager::new(&app_cfg, &event_manager); - let provider_1: Arc = "provider_1".intern(); - let provider_2: Arc = "provider_2".intern(); - let acquire = |provider: &Arc, port: u16| { - manager - .acquire_exact_connection_with_lease_for_session( - provider, - &SocketAddr::from(([127, 0, 0, 1], port)), - false, - 0, - ConnectionKind::Normal, - None, - ) - .expect("slot available") - }; - let handle_1 = acquire(&provider_1, 48_201); - let handle_2 = acquire(&provider_2, 48_202); - - let notify = manager.provider_capacity_notify(&provider_1); - let woken = notify.notified(); - tokio::pin!(woken); - woken.as_mut().enable(); - manager.release_handle(&handle_2); - assert!(woken.as_mut().now_or_never().is_none(), "another provider's release must not wake the waiter"); - manager.release_handle(&handle_1); - assert!(woken.as_mut().now_or_never().is_some(), "the waiter's provider release wakes it"); - } - - #[tokio::test(start_paused = true)] - async fn dropped_preempting_request_still_frees_the_victim_slot() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let input_name: Arc = "provider_1".intern(); - let victim = manager - .acquire_connection_with_grace_for_session( - &input_name, - &SocketAddr::from(([127, 0, 0, 1], 48_111)), - false, - 10, - ConnectionKind::Normal, - Some("session-victim"), - ) - .expect("victim allocation succeeds"); - manager.mark_opening(victim.allocation_id); - assert!(manager.register_body_owner(victim.allocation_id)); - - // The preempting request waits without deadline and is dropped (client disconnect). - let waiting = { - let manager = Arc::clone(&manager); - let input_name = Arc::clone(&input_name); - tokio::spawn(async move { - manager - .acquire_exact_connection_with_lease_for_session_await( - &input_name, - &SocketAddr::from(([127, 0, 0, 1], 48_112)), - false, - 0, - ConnectionKind::Normal, - None, - ) - .await - }) - }; - tokio::time::sleep(Duration::from_millis(50)).await; - assert!(victim.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); - waiting.abort(); - assert!(waiting.await.is_err_and(|err| err.is_cancelled())); - assert_eq!(manager.get_provider_connections_count(), 1, "the draining victim keeps its slot for now"); - - tokio::time::sleep(super::PREEMPTION_COMPLETION_TIMEOUT).await; - tokio::task::yield_now().await; - assert_eq!(manager.get_provider_connections_count(), 0, "the reaper frees the never-completing victim"); - } - - #[tokio::test(start_paused = true)] - async fn exact_acquire_until_stops_at_deadline_and_still_forces_victim_release() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let input_name: Arc = "provider_1".intern(); - let victim = manager - .acquire_connection_with_grace_for_session( - &input_name, - &SocketAddr::from(([127, 0, 0, 1], 48_101)), - false, - 10, - ConnectionKind::Normal, - Some("session-victim"), - ) - .expect("victim allocation succeeds"); - manager.mark_opening(victim.allocation_id); - assert!(manager.register_body_owner(victim.allocation_id)); - - // The victim never completes its body, so only the deadline can end the wait. - let started = tokio::time::Instant::now(); - let acquired = manager - .acquire_exact_connection_with_lease_for_session_until( - &input_name, - &SocketAddr::from(([127, 0, 0, 1], 48_102)), - false, - 0, - ConnectionKind::Normal, - None, - Some(started + Duration::from_millis(100)), - ) - .await; - assert!(acquired.is_none()); - assert_eq!(started.elapsed(), Duration::from_millis(100), "the caller deadline bounds the wait"); - assert!(victim.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); - assert_eq!(manager.get_provider_connections_count(), 1, "the draining victim keeps its slot for now"); - - tokio::time::sleep(super::PREEMPTION_COMPLETION_TIMEOUT).await; - tokio::task::yield_now().await; - assert_eq!(manager.get_provider_connections_count(), 0, "the forced victim release still runs"); - } - - #[tokio::test] - async fn test_single_request_preemption_awaits_victim_completion() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let input_name: Arc = "provider_1".intern(); - - let low_addr = SocketAddr::from(([127, 0, 0, 1], 48_001)); - let high_addr = SocketAddr::from(([127, 0, 0, 1], 48_002)); - - let low_handle = manager - .acquire_connection_with_grace_for_session( - &input_name, - &low_addr, - false, - 10, - ConnectionKind::Normal, - Some("session-victim"), - ) - .expect("initial low-priority allocation succeeds"); - - manager.mark_opening(low_handle.allocation_id); - assert!(manager.register_body_owner(low_handle.allocation_id)); - - let cancel_token = low_handle.cancel_token.clone().unwrap(); - let completion_token = low_handle.completion_token.clone().unwrap(); - - // Simulate upstream body owner: when cancelled, complete release after 30ms - let comp_clone = completion_token.clone(); - tokio::spawn(async move { - cancel_token.cancelled().await; - tokio::time::sleep(Duration::from_millis(30)).await; - comp_clone.cancel(); - }); - - // High priority acquire should preempt the victim, await its completion, and succeed in the same call - let high_handle = manager - .acquire_connection_with_lease_for_session_await( - &input_name, - &high_addr, - false, - 0, - ConnectionKind::Normal, - None, - ) - .await - .expect("high-priority allocation must succeed in the same call after awaiting victim completion"); - - assert_eq!(manager.get_provider_connections_count(), 1); - manager.release_handle(&high_handle); - assert_eq!(manager.get_provider_connections_count(), 0); - } - - #[tokio::test] - async fn stale_handle_does_not_cancel_newer_generation_slot() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let input_name: Arc = "provider_1".intern(); - let addr = SocketAddr::from(([127, 0, 0, 1], 49_001)); - - let handle = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - 0, - ConnectionKind::Normal, - Some("session-gen"), - ) - .expect("initial allocation succeeds"); - - manager.mark_opening(handle.allocation_id); - assert!(manager.register_body_owner(handle.allocation_id)); - - // Gen 0 has an old completion token - let old_completion_token = handle.completion_token.clone().expect("completion token present"); - - // Renew opening tokens to advance generation to 1 - let (_new_cancel, new_completion, gen) = - manager.renew_opening_tokens(handle.allocation_id).expect("renewal should succeed"); - assert_eq!(gen, 1); - - // Simulate an old handle belonging to generation 0 whose completion token was cancelled - old_completion_token.cancel(); - let stale_handle = tuliprox_core::model::ProviderHandle { - playback_request_id: handle.playback_request_id, - binding_tag: handle.binding_tag, - client_id: handle.client_id, - allocation_id: handle.allocation_id, - allocation: handle.allocation.clone(), - cancel_token: handle.cancel_token.clone(), - completion_token: Some(old_completion_token), - close_reason: Arc::clone(&handle.close_reason), - open_generation: 0, // Stale generation - }; - - // Releasing the stale handle must NOT poison/cancel the new generation's completion token - manager.release_handle(&stale_handle); - assert!( - !new_completion.is_cancelled(), - "new generation completion token must remain uncancelled after stale handle release" - ); - assert_eq!(manager.get_provider_connections_count(), 1, "slot must still be held by the active new generation"); - - // When the real new generation completes, the slot is reaped - new_completion.cancel(); - tokio::time::sleep(Duration::from_millis(50)).await; - assert_eq!(manager.get_provider_connections_count(), 0); - } - - #[tokio::test] - async fn dropped_cleanup_future_does_not_permanently_leak_slot() { - let app_cfg = create_test_app_config_single_provider_pool(); - let event_manager = Arc::new(EventManager::new()); - let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let input_name: Arc = "provider_1".intern(); - let addr = SocketAddr::from(([127, 0, 0, 1], 49_002)); - let addrs = [addr]; - - let handle = manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - 0, - ConnectionKind::Normal, - Some("session-drop"), - ) - .expect("allocation succeeds"); - - manager.mark_opening(handle.allocation_id); - assert!(manager.register_body_owner(handle.allocation_id)); - let completion_token = handle.completion_token.clone().expect("token present"); - - // Start release_playback_connections_await but drop it before completion - { - let cleanup_fut = manager.release_playback_connections_await("session-drop", &addrs); - // Poll once and drop - tokio::select! { - biased; - () = async {} => {}, - () = cleanup_fut => {}, - } - } - - // Slot must still be occupied - assert_eq!(manager.get_provider_connections_count(), 1); - - // Now the handle is released and the body owner signals completion - manager.release_handle(&handle); - completion_token.cancel(); - - // Give the background reaper task time to run - tokio::time::sleep(Duration::from_millis(50)).await; - - assert_eq!( - manager.get_provider_connections_count(), - 0, - "slot must be freed after completion and not permanently leaked in Closing state" - ); - } -} +mod tests; + +mod account_health; +mod allocation; +mod capacity; +mod leases; +mod preemption; +mod release; +mod shared; + +use self::{ + allocation::{playback_lease_owner, AcquireProviderParams, Connections}, + leases::OwnerProviderPin, + preemption::{PriorityKey, PriorityOwner}, + release::{ProviderAllocationGuard, ReleaseAction}, + shared::{SharedConnectionId, SharedConnections}, +}; diff --git a/backend/session/src/active_provider_manager/account_health.rs b/backend/session/src/active_provider_manager/account_health.rs new file mode 100644 index 000000000..9541bc929 --- /dev/null +++ b/backend/session/src/active_provider_manager/account_health.rs @@ -0,0 +1,28 @@ +use super::ActiveProviderManager; +use std::sync::Arc; + +impl ActiveProviderManager { + pub fn observe_account( + &self, + observation: tuliprox_core::model::ProviderAccountObservation, + ) -> Option { + self.core.providers.observe_account(observation) + } + + pub fn reset_account_health(&self, name: &Arc) { self.core.providers.forget_account_health(name); } + pub fn pending_account_observations(&self) -> Vec { + self.core.providers.pending_account_observations() + } + pub fn is_observation_pending(&self, observation: &tuliprox_core::model::ProviderAccountObservation) -> bool { + self.core.providers.is_observation_pending(observation) + } + pub fn acknowledge_account_observation(&self, observation: &tuliprox_core::model::ProviderAccountObservation) { + self.core.providers.acknowledge_account_observation(observation); + } + pub fn request_account_probe(&self, name: &Arc) { self.core.providers.request_account_probe(name); } + pub fn is_account_probe_requested(&self, name: &Arc) -> bool { + self.core.providers.is_account_probe_requested(name) + } + pub fn clear_account_probe(&self, name: &Arc) { self.core.providers.clear_account_probe(name); } + pub async fn account_health_changed(&self) { self.core.providers.account_health_changed.notified().await; } +} diff --git a/backend/session/src/active_provider_manager/allocation.rs b/backend/session/src/active_provider_manager/allocation.rs new file mode 100644 index 000000000..91bc3db2d --- /dev/null +++ b/backend/session/src/active_provider_manager/allocation.rs @@ -0,0 +1,962 @@ +use super::{ + ActiveProviderManager, ActiveProviderManagerCore, CapacityReleases, ConnectionKind, ManagedProviderHandle, + PlaybackLeaseRef, PreemptionOutcome, PriorityKey, PriorityOwner, ProviderAllocationGuard, ProviderCapacityNotifier, + ProviderReleaseSnapshot, SharedConnections, DUMMY_ADDR, PREEMPTION_COMPLETION_TIMEOUT, +}; +use crate::{ + provider_leases::{ProviderLeaseTable, ProviderLeaseUsage}, + provider_lineup_manager::ProviderLineupManager, + EventManager, +}; +use ::shared::utils::sanitize_sensitive_info; +use std::{ + cmp::Reverse, + collections::{BTreeMap, HashMap, HashSet}, + net::SocketAddr, + sync::{ + atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering}, + Arc, OnceLock, + }, + time::Instant, +}; +use tokio::time::Instant as TokioInstant; +use tokio_util::sync::CancellationToken; +use tuliprox_core::{ + model::{ + AllocationId, AppConfig, ConfigInput, ConnectionLifecycle, PlaybackKind, PlaybackRequestId, + PlaybackRequestOutcome, PlaybackSelectionReason, ProviderAllocation, ProviderBindingTag, ProviderCloseReason, + ProviderConfig, ProviderHandle, + }, + utils::debug_if_enabled, +}; + +impl ProviderReleaseSnapshot { + pub fn is_empty(&self) -> bool { self.single_allocations.is_empty() && self.shared_subscribers.is_empty() } +} + +/// Resolves a public live HLS session token to its stable provider playback owner. +/// +/// The random suffix distinguishes public sessions, while the preceding client, +/// user and channel fingerprint identifies one provider playback across retries. +/// Catchup and shared-cache owners retain their own identity semantics. +pub(super) fn playback_lease_owner(owner: &str) -> &str { + if owner.starts_with("m3u-catchup|") || owner.starts_with("catchup|") || owner.starts_with("hls-cache:") { + return owner; + } + let Some((base, suffix)) = owner.rsplit_once("|hls|") else { + return owner; + }; + if suffix.len() == 16 && suffix.bytes().all(|byte| byte.is_ascii_alphanumeric()) { + base + } else { + owner + } +} + +impl<'a> PlaybackLeaseRef<'a> { + pub fn new(owner: &'a str, kind: PlaybackKind) -> Self { + Self { owner, kind, request_id: PlaybackRequestId::next() } + } + + /// Stable provider lease identity, independent of the public HLS token suffix. + pub fn provider_owner(&self) -> &'a str { playback_lease_owner(self.owner) } +} + +pub(super) struct AcquireProviderParams<'a> { + pub(super) addr: &'a SocketAddr, + pub(super) priority: i8, + pub(super) kind: ConnectionKind, + pub(super) lease: Option>, +} + +impl AcquireProviderParams<'_> { + #[inline] + pub(super) fn session_owner(&self) -> Option<&str> { self.lease.map(|lease| lease.provider_owner()) } +} + +#[derive(Debug, Clone)] +pub(super) struct ActiveConnectionInfo { + pub(super) allocation_id: AllocationId, + pub(super) client_addr: SocketAddr, + pub(super) allocation: ProviderAllocation, + // Used to signal preemption to the consumer of this connection + pub(super) cancel_token: CancellationToken, + pub(super) completion_token: CancellationToken, + pub(super) close_reason: Arc, + pub(super) lifecycle: ConnectionLifecycle, + pub(super) has_body_owner: bool, + pub(super) reaper_spawned: bool, + pub(super) open_generation: u64, + pub(super) created_at: Instant, + pub(super) priority: i8, + pub(super) kind: ConnectionKind, + pub(super) session_owner: Option>, + // Public session identity scopes targeted socket cleanup independently of the + // stable provider lease owner shared by entry retries. + pub(super) request_owner: Option>, + pub(super) playback_request_id: Option, +} + +#[derive(Debug, Clone, Default)] +pub(super) struct Connections { + // Primary map keyed by AllocationId, never by socket address. + pub(super) single: HashMap, + // Secondary index for socket-wide transport close/kick actions. + pub(super) single_by_addr: HashMap>, + pub(super) shared: SharedConnections, + // Index to quickly find connections by provider name for preemption + // ProviderName -> Set + pub(super) by_provider: HashMap, HashSet>, + // Index to find every allocation (single and shared) owned by a playback session. + // SessionOwner -> Set + pub(super) by_owner: HashMap, HashSet>, + // Priority index per provider alias for O(log n) victim lookup + // ProviderName -> BTreeMap + pub(super) priority_index: HashMap, BTreeMap>, + pub(super) soft_priority_index: HashMap, BTreeMap>, + // Providers whose allocations were released since the last drain. + pub(super) released_providers: Vec>, +} + +impl CapacityReleases for Connections { + fn take_released_providers(&mut self) -> Vec> { std::mem::take(&mut self.released_providers) } +} + +impl ProviderCapacityNotifier { + fn lock_waiters(&self) -> std::sync::MutexGuard<'_, HashMap, Arc>> { + self.waiters.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + } + + /// The notification fired whenever `provider` frees capacity. Provider names are bounded by + /// the configuration, so entries are kept. + pub fn subscribe(&self, provider: &Arc) -> Arc { + Arc::clone(self.lock_waiters().entry(Arc::clone(provider)).or_default()) + } + + pub(super) fn notify(&self, provider: &str) { + if let Some(notify) = self.lock_waiters().get(provider) { + notify.notify_waiters(); + } + } + + /// Called while a table lock is held; [`Self::flush`] delivers after it is released. + pub(super) fn stage(&self, released: Vec>) { + if released.is_empty() { + return; + } + self.staged.lock().unwrap_or_else(std::sync::PoisonError::into_inner).extend(released); + self.has_staged.store(true, Ordering::Release); + } + + pub(super) fn flush(&self) { + if !self.has_staged.swap(false, Ordering::AcqRel) { + return; + } + let mut staged = std::mem::take(&mut *self.staged.lock().unwrap_or_else(std::sync::PoisonError::into_inner)); + staged.sort_unstable(); + staged.dedup(); + let waiters = self.lock_waiters(); + for provider in &staged { + if let Some(notify) = waiters.get(provider) { + notify.notify_waiters(); + } + } + } +} + +impl ManagedProviderHandle { + #[inline] + pub fn manager(&self) -> &Arc { &self.manager } + + #[inline] + pub fn new(manager: Arc, handle: ProviderHandle) -> Self { + Self { manager, handle: Some(handle) } + } + + #[inline] + pub fn handle(&self) -> Option<&ProviderHandle> { self.handle.as_ref() } + + #[inline] + pub fn take(&mut self) -> Option { self.handle.take() } + + #[inline] + pub fn disarm(&mut self) -> Option { self.handle.take() } + + pub fn mark_opening(&self) { + if let Some(handle) = self.handle.as_ref() { + self.manager.mark_opening(handle.allocation_id); + } + } + + pub fn register_body_owner(&self) -> bool { + if let Some(handle) = self.handle.as_ref() { + self.manager.register_body_owner(handle.allocation_id) + } else { + false + } + } +} + +impl std::ops::Deref for ActiveProviderManager { + type Target = ActiveProviderManagerCore; + + #[inline] + fn deref(&self) -> &Self::Target { &self.core } +} + +impl ActiveProviderManager { + fn index_owner(connections: &mut Connections, allocation_id: AllocationId, owner: Option<&Arc>) { + if let Some(owner) = owner { + connections.by_owner.entry(Arc::clone(owner)).or_default().insert(allocation_id); + } + } + + pub(super) fn unindex_owner(connections: &mut Connections, allocation_id: AllocationId, owner: Option<&Arc>) { + if let Some(owner) = owner { + let remove = connections.by_owner.get_mut(owner).is_some_and(|set| { + set.remove(&allocation_id); + set.is_empty() + }); + if remove { + connections.by_owner.remove(owner); + } + } + } + + pub fn new(cfg: &AppConfig, event_manager: &Arc) -> Self { + let grace_period_options = Self::get_grace_options(cfg); + let inputs = Self::get_config_inputs(cfg); + Self { + core: Arc::new(ActiveProviderManagerCore { + capacity_transition: std::sync::Mutex::new(()), + is_shutting_down: AtomicBool::new(false), + shutdown_token: CancellationToken::new(), + providers: ProviderLineupManager::new(inputs, grace_period_options, event_manager), + connections: std::sync::RwLock::new(Connections::default()), + leases: std::sync::RwLock::new(ProviderLeaseTable::with_affinity_ttl(Self::get_affinity_ttl(cfg))), + next_allocation_id: AtomicU64::new(1), + capacity: ProviderCapacityNotifier::default(), + }), + shared_stream_manager: OnceLock::new(), + } + } + + /// Routes account observations reported through `event_manager` to this manager. + /// Only the first binding wins, so test or auxiliary managers cannot hijack the runtime one. + pub fn bind_event_manager(&self, event_manager: &EventManager) { + let _ = event_manager.account_provider.set(Arc::downgrade(&self.core)); + } + + /// Closes provider admission and releases all physical allocations and leases. + pub fn shutdown(&self) { + self.shutdown_token.cancel(); + self.is_shutting_down.store(true, Ordering::Release); + let _transition = self.lock_capacity_transition(); + let connections = std::mem::take(&mut *self.write_connections()); + self.write_leases().clear(); + + for info in connections.single.into_values() { + info.cancel_token.cancel(); + info.allocation.release(); + } + for shared in connections.shared.by_key.into_values() { + if let Some(cancel_token) = shared.cancel_token { + cancel_token.cancel(); + } + shared.allocation.release(); + } + self.providers.reconcile_connections(HashMap::new()); + } + + fn get_config_inputs(cfg: &AppConfig) -> Vec> { + cfg.sources.load().inputs.iter().filter(|i| i.enabled).map(Arc::clone).collect() + } + + pub fn update_config(&self, cfg: &AppConfig) { + let grace_period_options = Self::get_grace_options(cfg); + let inputs = Self::get_config_inputs(cfg); + self.providers.update_config(inputs, &grace_period_options); + self.write_leases().set_affinity_ttl(Self::get_affinity_ttl(cfg)); + self.reconcile_connections(); + } + + pub fn reconcile_connections(&self) { + let _transition = self.lock_capacity_transition(); + let mut counts = HashMap::, usize>::new(); + { + let connections = self.read_connections(); + + // Single connections + for info in connections.single.values() { + if let Some(name) = info.allocation.get_provider_name() { + *counts.entry(name).or_insert(0) += 1; + } + } + + // Shared connections + for shared in connections.shared.by_key.values() { + if let Some(name) = shared.allocation.get_provider_name() { + *counts.entry(name).or_insert(0) += 1; + } + } + } + + self.providers.reconcile_connections(counts); + } + + pub(super) fn prune_expired_leases(leases: &mut ProviderLeaseTable) { leases.prune(TokioInstant::now()); } + + /// A provisional lease only pins its provider while its allocation is active. + /// Once an unstarted request releases its slot, the next attempt may use another alias. + pub fn should_reuse_playback_provider(&self, session_owner: &str, provider_name: &Arc) -> bool { + let session_owner = playback_lease_owner(session_owner); + let _transition = self.lock_capacity_transition(); + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + let confirmed = leases + .lease_of_owner(session_owner) + .is_some_and(|lease| lease.provider_name == *provider_name && lease.state.is_confirmed()); + drop(leases); + confirmed || self.has_active_owner_for_provider(provider_name, session_owner) + } + + /// Returns true when the next allocation would consume a slot kept for an idle reservation. + fn reserved_capacity_blocks_next(&self, provider_name: &Arc, session_owner: Option<&str>) -> bool { + let Some((current_connections, max_connections, idle_foreign_reservations)) = + self.reservation_capacity_usage(provider_name, session_owner) + else { + return true; + }; + max_connections > 0 + && idle_foreign_reservations > 0 + && current_connections.saturating_add(idle_foreign_reservations) >= max_connections + } + + pub(super) fn reserved_provider_names_for_other( + &self, + input_name: &Arc, + session_owner: Option<&str>, + ) -> HashSet> { + let mut reserved = HashSet::new(); + for provider_name in self.providers.provider_names_for_input(input_name) { + if self.has_foreign_reservation(&provider_name, session_owner) { + reserved.insert(provider_name); + } + } + reserved + } + + /// Renews the lease of an adaptive (HLS/DASH/Catchup) playback. + /// + /// Adaptive playback consists of many short requests, so the lease must stay + /// reconnect-capable: between two segment requests the slot is kept as an idle + /// lease on the same provider instead of being released and reselected. + pub fn refresh_adaptive_playback_lease( + &self, + provider_name: &Arc, + session_owner: &str, + kind: PlaybackKind, + ttl_secs: u64, + ) { + let session_owner = playback_lease_owner(session_owner); + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return; + } + let granted_ttl = self.renewal_granted_ttl(provider_name, session_owner, ttl_secs); + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + leases.renew_current_owner(session_owner, provider_name, kind, granted_ttl); + } + + /// Renews the lease of a playback. A zero TTL clears it. + pub fn refresh_playback_lease(&self, provider_name: &Arc, lease_ref: &PlaybackLeaseRef<'_>, ttl_secs: u64) { + let lease_ref = PlaybackLeaseRef { owner: lease_ref.provider_owner(), ..*lease_ref }; + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return; + } + let granted_ttl = self.renewal_granted_ttl(provider_name, lease_ref.owner, ttl_secs); + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + if ttl_secs == 0 { + leases.release_identified_owner(lease_ref.owner, lease_ref.request_id); + return; + } + leases.renew_identified_owner( + lease_ref.owner, + provider_name, + lease_ref.kind, + lease_ref.request_id, + granted_ttl, + ); + } + + pub fn finish_identified_playback_request( + &self, + owner: &str, + request_id: PlaybackRequestId, + outcome: PlaybackRequestOutcome, + ) { + self.finish_playback_request_inner(owner, Some(request_id), outcome, None); + } + + /// Snapshot of the logical slots a provider holds, for logs and UI metrics. + pub fn provider_lease_usage(&self, provider_name: &Arc) -> ProviderLeaseUsage { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + leases.usage(provider_name) + } + + /// Removes every expired starting/idle lease. Runs from the session GC task. + pub fn prune_expired_leases_now(&self) { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + } + + pub(super) fn acquire_exact_connection_inner( + &self, + provider_name: &Arc, + allow_grace: bool, + params: &AcquireProviderParams<'_>, + ) -> Option { + if let Some(handle) = self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, params) { + return Some(handle); + } + if let Some(preempted_alloc) = self.try_preempt_connection( + provider_name, + params.priority, + allow_grace, + params.kind, + params.session_owner(), + ) { + return Some(self.register_allocation(preempted_alloc, params)); + } + None + } + + pub(super) fn finalize_lineup_allocation( + &self, + input_name: &Arc, + allow_grace: bool, + mut allocation: ProviderAllocationGuard, + params: &AcquireProviderParams<'_>, + ) -> ProviderHandle { + if matches!(allocation.allocation(), ProviderAllocation::GracePeriod(_)) + && self.evict_lower_priority_on_input(input_name, params.priority, params.kind, params.session_owner()) + { + let evicted_on_same = + !self.providers.is_over_limit(&allocation.allocation().get_provider_name().unwrap_or_default()); + if !evicted_on_same { + let mut new_alloc = ProviderAllocationGuard::new( + self.providers.acquire_connection_with_grace_override(input_name, allow_grace), + ); + if !matches!(new_alloc.allocation(), ProviderAllocation::Exhausted) { + if let Some(provider_name) = new_alloc.allocation().get_provider_name() { + if !self.exceeds_reserved_capacity(&provider_name, params.session_owner()) { + return self.register_allocation(new_alloc.take(), params); + } + } + } + } + } + + self.register_allocation(allocation.take(), params) + } + + fn acquire_connection_inner( + &self, + provider_or_input_name: &Arc, + allow_grace: bool, + params: &AcquireProviderParams<'_>, + ) -> Option { + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + if let Some(handle) = self.acquire_connection_inner_no_preempt(provider_or_input_name, allow_grace, params) { + return Some(handle); + } + + if let Some(preempted_alloc) = self.try_preempt_connection( + provider_or_input_name, + params.priority, + allow_grace, + params.kind, + params.session_owner(), + ) { + return Some(self.register_allocation(preempted_alloc, params)); + } + + None + } + + pub(super) fn register_allocation( + &self, + allocation: ProviderAllocation, + params: &AcquireProviderParams<'_>, + ) -> ProviderHandle { + let AcquireProviderParams { addr, priority, kind, lease } = *params; + let session_owner = lease.map(|lease| lease.provider_owner()); + let provider_name = allocation.get_provider_name().unwrap_or_default(); + let allocation_id = self.next_allocation_id.fetch_add(1, Ordering::Relaxed); + let cancel_token = CancellationToken::new(); + let completion_token = CancellationToken::new(); + let close_reason = Arc::new(AtomicU8::new(ProviderCloseReason::Unspecified as u8)); + let now = Instant::now(); + let is_unlimited = allocation.is_unlimited_provider(); + + // Claim the playback's capacity slot lease in its own lock scope so the + // connection and lease locks are never nested. The lease starts unconfirmed; + // it only reserves capacity against other playbacks once real media activity + // confirms it, which is what keeps abandoned manifest starts from blocking. + let binding_tag = if let Some(lease) = lease { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + let id = leases.begin_owner(lease.provider_owner(), &provider_name, lease.kind, lease.request_id); + if lease.owner != lease.provider_owner() { + leases.attach_request_token(id, lease.request_id, lease.owner); + } + let generation = leases.lease(id).map_or(1, |lease| lease.binding_generation); + debug_if_enabled!( + "Playback lease began: provider={} owner={} kind={} request_id={} lease_id={} generation={} state=starting", + sanitize_sensitive_info(&provider_name), + sanitize_sensitive_info(lease.provider_owner()), + lease.kind, + lease.request_id, + id, + generation + ); + Some(ProviderBindingTag::new(id, generation)) + } else { + None + }; + + let session_owner_arc = session_owner.map(Arc::from); + let mut connections = self.write_connections(); + connections.single.insert( + allocation_id, + ActiveConnectionInfo { + allocation_id, + client_addr: *addr, + allocation: allocation.clone(), + cancel_token: cancel_token.clone(), + completion_token: completion_token.clone(), + close_reason: Arc::clone(&close_reason), + lifecycle: ConnectionLifecycle::Active, + has_body_owner: false, + reaper_spawned: false, + open_generation: 0, + created_at: now, + priority, + kind, + session_owner: session_owner_arc.clone(), + request_owner: lease.map(|lease| { + session_owner_arc + .as_ref() + .filter(|owner| owner.as_ref() == lease.owner) + .map_or_else(|| Arc::from(lease.owner), Arc::clone) + }), + playback_request_id: lease.map(|lease| lease.request_id), + }, + ); + connections.single_by_addr.entry(*addr).or_default().insert(allocation_id); + Self::index_owner(&mut connections, allocation_id, session_owner_arc.as_ref()); + + // Unlimited providers are not subject to preemption, so we deliberately + // skip populating the by_provider / priority_index / soft_priority_index + // indices. The connection itself is still tracked via `single`, so all + // capacity and lifecycle invariants hold. Without this skip, an + // unlimited provider's connection can be selected as a preemption victim + // even though it cannot be exhausted, contradicting the configured + // `max_connections: 0` semantics. + if !is_unlimited { + connections.by_provider.entry(provider_name.clone()).or_default().insert(allocation_id); + Self::upsert_priority_entry( + &mut connections, + &provider_name, + (priority, Reverse(now), allocation_id), + PriorityOwner::Single(allocation_id), + kind, + ); + } + + debug_if_enabled!( + "Added provider connection {provider_name:?} (reason={}, prio={priority}, kind={kind:?}, unlimited={is_unlimited}, allocation_id={allocation_id}, lease_id={}, request_id={}, playback_kind={}, peer_addr={})", + PlaybackSelectionReason::NewPriorityAllocation, + binding_tag.map_or_else(String::new, |tag| tag.lease_id.to_string()), + lease.map_or_else(String::new, |lease| lease.request_id.to_string()), + lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string()), + sanitize_sensitive_info(&addr.to_string()) + ); + let mut handle = ProviderHandle::new(*addr, allocation_id, allocation, Some(cancel_token)); + handle.completion_token = Some(completion_token); + handle.close_reason = close_reason; + handle.playback_request_id = lease.map(|lease| lease.request_id); + handle.binding_tag = binding_tag; + handle + } + + pub(super) fn try_acquire_allocation_after_freed( + &self, + input_name: &Arc, + allow_grace: bool, + reserved_providers: &HashSet>, + session_owner: Option<&str>, + ) -> Option { + let attempts = self.providers.provider_names_for_input(input_name).len().max(1); + let mut excluded_providers = reserved_providers.clone(); + for _ in 0..attempts { + let allocation = self.providers.acquire_connection_with_grace_override_excluding( + input_name, + allow_grace, + &excluded_providers, + ); + if matches!(allocation, ProviderAllocation::Exhausted) { + break; + } + if let Some(provider_name) = allocation.get_provider_name() { + if self.exceeds_reserved_capacity(&provider_name, session_owner) { + excluded_providers.insert(provider_name); + allocation.release(); + continue; + } + } + return Some(allocation); + } + None + } + + pub fn force_exact_acquire_connection( + &self, + provider_name: &Arc, + addr: &SocketAddr, + priority: i8, + kind: ConnectionKind, + ) -> Option { + // Compatibility wrapper: keep the exact-provider behavior but do not over-allocate exhausted accounts. + self.acquire_exact_connection_with_grace(provider_name, addr, false, priority, kind) + } + + // Returns the next available provider connection + pub fn acquire_connection( + &self, + input_name: &Arc, + addr: &SocketAddr, + priority: i8, + kind: ConnectionKind, + ) -> Option { + self.acquire_connection_inner(input_name, true, &AcquireProviderParams { addr, priority, kind, lease: None }) + } + + /// Lineup acquisition with an explicit playback lease identity. + pub fn acquire_connection_with_lease_for_session( + &self, + input_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + lease: Option>, + ) -> Option { + self.acquire_connection_inner(input_name, allow_grace, &AcquireProviderParams { addr, priority, kind, lease }) + } + + /// Lineup acquisition with an explicit playback lease identity, awaiting preemption completion if needed. + pub async fn acquire_connection_with_lease_for_session_await( + &self, + input_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + lease: Option>, + ) -> Option { + let params = AcquireProviderParams { addr, priority, kind, lease }; + let outcome = { + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + if let Some(handle) = self.acquire_connection_inner_no_preempt(input_name, allow_grace, ¶ms) { + return Some(handle); + } + self.try_preempt_connection_outcome( + input_name, + params.priority, + allow_grace, + params.kind, + params.session_owner(), + ) + }; + + match outcome { + PreemptionOutcome::Acquired(alloc) => { + let _transition = self.lock_capacity_transition(); + Some(self.register_allocation(alloc, ¶ms)) + } + PreemptionOutcome::PendingCompletion(victim_identity, completion_token) => { + let _ = tokio::time::timeout(PREEMPTION_COMPLETION_TIMEOUT, completion_token.cancelled()).await; + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + // Idempotently free the victim's slot in case the background reaper has not yet run. + // complete_release_locked is a no-op if the slot was already freed or the generation + // no longer matches. + if let Some((victim_alloc_id, victim_gen)) = victim_identity { + let mut connections = self.write_connections(); + ActiveProviderManagerCore::complete_release_locked( + &mut connections, + victim_alloc_id, + Some(victim_gen), + ); + } + self.acquire_connection_inner_no_preempt(input_name, allow_grace, ¶ms) + } + PreemptionOutcome::Exhausted => None, + } + } + + /// Exact-provider acquisition with an explicit playback lease identity. + pub fn acquire_exact_connection_with_lease_for_session( + &self, + provider_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + lease: Option>, + ) -> Option { + let _transition = self.lock_capacity_transition(); + self.acquire_exact_connection_inner( + provider_name, + allow_grace, + &AcquireProviderParams { addr, priority, kind, lease }, + ) + } + + /// Exact-provider acquisition with an explicit playback lease identity, awaiting preemption completion if needed. + pub async fn acquire_exact_connection_with_lease_for_session_await( + &self, + provider_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + lease: Option>, + ) -> Option { + self.acquire_exact_connection_with_lease_for_session_until( + provider_name, + addr, + allow_grace, + priority, + kind, + lease, + None, + ) + .await + } + + /// Earliest instant at which a playback lease may expire and free reserved capacity, after + /// pruning the ones already expired. Lease expiry is time based and sends no notification. + pub fn next_lease_expiry(&self) -> Option { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + leases.next_expiry() + } + + /// Like [`Self::acquire_exact_connection_with_lease_for_session_await`], but never waits past + /// `deadline`. A preemption that outlives the deadline is finished by its reaper. + #[allow(clippy::too_many_arguments)] + pub async fn acquire_exact_connection_with_lease_for_session_until( + &self, + provider_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + lease: Option>, + deadline: Option, + ) -> Option { + let params = AcquireProviderParams { addr, priority, kind, lease }; + let outcome = { + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + if let Some(handle) = self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, ¶ms) { + return Some(handle); + } + self.try_preempt_connection_outcome( + provider_name, + params.priority, + allow_grace, + params.kind, + params.session_owner(), + ) + }; + + match outcome { + PreemptionOutcome::Acquired(alloc) => { + let _transition = self.lock_capacity_transition(); + Some(self.register_allocation(alloc, ¶ms)) + } + PreemptionOutcome::PendingCompletion(victim_identity, completion_token) => { + let completion_deadline = TokioInstant::now() + PREEMPTION_COMPLETION_TIMEOUT; + let wait_until = deadline.map_or(completion_deadline, |deadline| deadline.min(completion_deadline)); + let completed = tokio::time::timeout_at(wait_until, completion_token.cancelled()).await.is_ok(); + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + if !completed && wait_until < completion_deadline { + // The preemption reaper forces the victim release on time without this caller. + return None; + } + let _transition = self.lock_capacity_transition(); + if let Some((victim_alloc_id, victim_gen)) = victim_identity { + let mut connections = self.write_connections(); + ActiveProviderManagerCore::complete_release_locked( + &mut connections, + victim_alloc_id, + Some(victim_gen), + ); + } + self.acquire_exact_connection_inner_no_preempt(provider_name, allow_grace, ¶ms) + } + PreemptionOutcome::Exhausted => None, + } + } + + /// Acquire a provider connection for probe tasks with configurable priority. + /// Probes never consume grace capacity. + pub fn acquire_connection_for_probe(&self, input_name: &Arc, priority: i8) -> Option { + self.acquire_connection_inner( + input_name, + false, + &AcquireProviderParams { addr: &DUMMY_ADDR, priority, kind: ConnectionKind::Normal, lease: None }, + ) + } + + /// Acquire a provider connection for background transfers (downloads/recordings). + /// Transfers participate in the same provider priority/preemption model as normal + /// streams, but they never consume grace capacity and wait externally on notifications. + pub fn acquire_connection_for_download(&self, input_name: &Arc, priority: i8) -> Option { + self.acquire_connection_inner( + input_name, + false, + &AcquireProviderParams { addr: &DUMMY_ADDR, priority, kind: ConnectionKind::Normal, lease: None }, + ) + } + + /// Transfers an already reserved recording allocation into the internal + /// playback request that opens the provider body. The worker keeps its + /// original handle so preemption can still stop the recording; releases + /// are allocation-id based and therefore remain idempotent. + pub fn claim_download_connection( + &self, + allocation_id: AllocationId, + input_name: &Arc, + ) -> Option { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + let info = connections.single.get_mut(&allocation_id)?; + if info.lifecycle != ConnectionLifecycle::Active || info.has_body_owner { + return None; + } + let provider_name = info.allocation.get_provider_name()?; + if !self.providers.is_provider_for_input(provider_name.as_ref(), input_name.as_ref()) { + return None; + } + + // Claim under the capacity-transition lock so a duplicated internal + // request cannot attach a second provider body to this reservation. + info.lifecycle = ConnectionLifecycle::Opening; + let handle = ProviderHandle { + playback_request_id: info.playback_request_id, + binding_tag: None, + client_id: info.client_addr, + allocation_id: info.allocation_id, + allocation: info.allocation.clone(), + cancel_token: Some(info.cancel_token.clone()), + completion_token: Some(info.completion_token.clone()), + close_reason: Arc::clone(&info.close_reason), + open_generation: info.open_generation, + }; + Some(handle) + } + + // This method is used for redirects to cycle through the provider + pub fn get_next_provider(&self, provider_name: &Arc) -> Option> { + self.providers.get_next_provider(provider_name) + } + + pub fn find_provider_config(&self, provider_name: &Arc) -> Option> { + self.providers.find_provider_config(provider_name) + } + + pub fn is_provider_for_input(&self, provider_name: &Arc, input_name: &Arc) -> bool { + self.providers.is_provider_for_input(provider_name.as_ref(), input_name.as_ref()) + } + + pub fn is_provider_reserved_for_other_session( + &self, + provider_name: &Arc, + session_owner: Option<&str>, + ) -> bool { + let session_owner = session_owner.map(playback_lease_owner); + self.reserved_capacity_blocks_next(provider_name, session_owner) + } + + pub fn active_connections(&self) -> Option, usize>> { self.providers.active_connections() } + + pub fn is_exhausted(&self, provider_name: &Arc) -> bool { self.providers.is_exhausted(provider_name) } + + pub fn register_body_owner(&self, alloc_id: AllocationId) -> bool { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get_mut(&alloc_id) { + if info.lifecycle == ConnectionLifecycle::Closing || info.lifecycle == ConnectionLifecycle::Closed { + return false; + } + info.has_body_owner = true; + if info.lifecycle == ConnectionLifecycle::Opening { + info.lifecycle = ConnectionLifecycle::Active; + } + true + } else { + false + } + } + + pub fn mark_opening(&self, alloc_id: AllocationId) { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get_mut(&alloc_id) { + if info.lifecycle != ConnectionLifecycle::Closing && info.lifecycle != ConnectionLifecycle::Closed { + info.lifecycle = ConnectionLifecycle::Opening; + } + } + } + + pub fn get_provider_connections_count(&self) -> usize { self.providers.active_connection_count() } + + pub fn provider_capacities_for_input(&self, input_name: &Arc) -> Vec<(Arc, usize, usize)> { + let mut result = Vec::new(); + for provider_name in self.providers.provider_names_for_input(input_name) { + if let Some((current, max)) = self.providers.provider_capacity(&provider_name) { + result.push((provider_name, current, max)); + } + } + result + } +} diff --git a/backend/session/src/active_provider_manager/capacity.rs b/backend/session/src/active_provider_manager/capacity.rs new file mode 100644 index 000000000..26d52173e --- /dev/null +++ b/backend/session/src/active_provider_manager/capacity.rs @@ -0,0 +1,186 @@ +use super::{ + AcquireProviderParams, ActiveProviderManager, ActiveProviderManagerCore, CapacityReleases, Connections, + ProviderCapacityNotifier, +}; +use crate::provider_leases::ProviderLeaseTable; +use ::shared::utils::sanitize_sensitive_info; +use log::error; +use std::sync::{Arc, RwLockReadGuard, RwLockWriteGuard}; +use tuliprox_core::{model::PlaybackSelectionReason, utils::debug_if_enabled}; + +/// Write access to a capacity table that wakes the waiters of freed providers after release. +/// +/// Fields drop in declaration order: `lock` stages the freed providers while it still holds the +/// lock and then releases it; only afterwards does `_flush` wake the waiters. +pub(super) struct CapacityWriteGuard<'a, T: CapacityReleases> { + lock: StagingWriteGuard<'a, T>, + _flush: FlushCapacityWaiters<'a>, +} + +struct StagingWriteGuard<'a, T: CapacityReleases> { + guard: RwLockWriteGuard<'a, T>, + notifier: &'a ProviderCapacityNotifier, +} + +impl Drop for StagingWriteGuard<'_, T> { + fn drop(&mut self) { self.notifier.stage(self.guard.take_released_providers()); } +} + +struct FlushCapacityWaiters<'a>(&'a ProviderCapacityNotifier); + +impl Drop for FlushCapacityWaiters<'_> { + fn drop(&mut self) { self.0.flush(); } +} + +impl<'a, T: CapacityReleases> CapacityWriteGuard<'a, T> { + pub(super) fn new(guard: RwLockWriteGuard<'a, T>, notifier: &'a ProviderCapacityNotifier) -> Self { + Self { lock: StagingWriteGuard { guard, notifier }, _flush: FlushCapacityWaiters(notifier) } + } +} + +impl std::ops::Deref for CapacityWriteGuard<'_, T> { + type Target = T; + fn deref(&self) -> &Self::Target { &self.lock.guard } +} + +impl std::ops::DerefMut for CapacityWriteGuard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { &mut self.lock.guard } +} + +impl ActiveProviderManagerCore { + /// Notification fired whenever `provider` frees a slot or ends a lease. + pub fn provider_capacity_notify(&self, provider: &Arc) -> Arc { + self.capacity.subscribe(provider) + } + + /// Serialises capacity transitions (acquire/release/reclassify) so a reclassify + /// cannot race an acquire or release. + /// + /// Lock-order invariant: this lock must be acquired **before** the connections + /// lock (`read_connections`/`write_connections`). No path may hold the + /// connections lock and then call this; doing so can deadlock. + pub(super) fn lock_capacity_transition(&self) -> std::sync::MutexGuard<'_, ()> { + match self.capacity_transition.lock() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Recovering poisoned active-provider capacity transition lock"); + poisoned.into_inner() + } + } + } + + pub(super) fn read_connections(&self) -> RwLockReadGuard<'_, Connections> { + match self.connections.read() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Recovering poisoned active-provider connection lock"); + poisoned.into_inner() + } + } + } + + pub(super) fn write_connections(&self) -> CapacityWriteGuard<'_, Connections> { + let guard = match self.connections.write() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Recovering poisoned active-provider connection lock"); + poisoned.into_inner() + } + }; + CapacityWriteGuard::new(guard, &self.capacity) + } + + pub(super) fn write_leases(&self) -> CapacityWriteGuard<'_, ProviderLeaseTable> { + let guard = match self.leases.write() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Recovering poisoned provider-lease lock"); + poisoned.into_inner() + } + }; + CapacityWriteGuard::new(guard, &self.capacity) + } + + pub(super) fn read_leases(&self) -> RwLockReadGuard<'_, ProviderLeaseTable> { + match self.leases.read() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Recovering poisoned provider-lease lock"); + poisoned.into_inner() + } + } + } +} + +impl ActiveProviderManager { + pub(super) fn reservation_capacity_usage( + &self, + provider_name: &Arc, + session_owner: Option<&str>, + ) -> Option<(usize, usize, usize)> { + let (current_connections, max_connections) = self.providers.provider_capacity(provider_name)?; + if max_connections == 0 { + return Some((current_connections, max_connections, 0)); + } + // Without any foreign reserving lease the owner analysis cannot change the result. + if !self.read_leases().has_foreign_reserved_lease(provider_name, session_owner) { + return Some((current_connections, max_connections, 0)); + } + let counted_owners = self.active_reservation_owners(provider_name); + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + let idle_foreign_reservations = leases.foreign_reserved_slots(provider_name, session_owner, &counted_owners); + + Some((current_connections, max_connections, idle_foreign_reservations)) + } + + /// Returns true when an already-counted candidate allocation consumed a slot kept for an idle reservation. + pub(super) fn exceeds_reserved_capacity(&self, provider_name: &Arc, session_owner: Option<&str>) -> bool { + let Some((current_connections, max_connections, idle_foreign_reservations)) = + self.reservation_capacity_usage(provider_name, session_owner) + else { + return true; + }; + max_connections > 0 + && idle_foreign_reservations > 0 + && current_connections.saturating_add(idle_foreign_reservations) > max_connections + } + + /// Explains a reservation skip with the authoritative slot breakdown instead of + /// only the transport socket, so the decision can be audited from the logs. + pub(super) fn log_reserved_capacity_skip(&self, provider_name: &Arc, params: &AcquireProviderParams<'_>) { + if !log::log_enabled!(log::Level::Debug) { + return; + } + let Some((current_connections, max_connections, foreign_reserved)) = + self.reservation_capacity_usage(provider_name, params.session_owner()) + else { + debug_if_enabled!( + "Skipping reserved provider {} (reason={}, capacity=unknown, peer_addr={}, request_id={}, playback_kind={})", + sanitize_sensitive_info(provider_name), + PlaybackSelectionReason::ReservedCapacity, + sanitize_sensitive_info(¶ms.addr.to_string()), + params.lease.map_or_else(String::new, |lease| lease.request_id.to_string()), + params.lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string()) + ); + return; + }; + let usage = self.provider_lease_usage(provider_name); + debug_if_enabled!( + "Skipping reserved provider {} (reason={}, current={}, max={}, foreign_reserved={}, active_slots={}, starting_slots={}, idle_slots={}, peer_addr={}, request_id={}, playback_kind={})", + sanitize_sensitive_info(provider_name), + PlaybackSelectionReason::ReservedCapacity, + current_connections, + max_connections, + foreign_reserved, + usage.active, + usage.starting, + usage.idle, + sanitize_sensitive_info(¶ms.addr.to_string()), + params.lease.map_or_else(String::new, |lease| lease.request_id.to_string()), + params.lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string()) + ); + } + + pub fn is_over_limit(&self, provider_name: &Arc) -> bool { self.providers.is_over_limit(provider_name) } +} diff --git a/backend/session/src/active_provider_manager/leases.rs b/backend/session/src/active_provider_manager/leases.rs new file mode 100644 index 000000000..7d4c72577 --- /dev/null +++ b/backend/session/src/active_provider_manager/leases.rs @@ -0,0 +1,390 @@ +use super::{playback_lease_owner, ActiveProviderManager, CapacityReleases, ManagedProviderHandle, PlaybackLeaseRef}; +use crate::provider_leases::{ProviderAffinityTtl, ProviderLeaseTable}; +use std::{ + collections::HashSet, + sync::{atomic::Ordering, Arc}, +}; +use tokio_util::sync::CancellationToken; +use tuliprox_core::model::{ + AllocationId, AppConfig, ConnectionLifecycle, PlaybackKind, PlaybackLeaseId, PlaybackRequestId, + PlaybackRequestOutcome, ProviderBindingTag, +}; + +/// How an acquisition of a playback owner is tied to a provider. +pub(super) enum OwnerProviderPin { + /// A confirmed or active lease pins the playback to exactly this provider. + Reserved(Arc), + /// A recently confirmed playback prefers this provider; it holds no capacity. + Preferred(Arc), +} + +impl<'a> PlaybackLeaseRef<'a> { + /// Compatibility view of a bare session owner: one request, live TS semantics. + pub fn for_session_owner(owner: &'a str) -> Self { Self::new(owner, PlaybackKind::LiveTs) } +} + +impl CapacityReleases for ProviderLeaseTable { + fn take_released_providers(&mut self) -> Vec> { ProviderLeaseTable::take_released_providers(self) } +} + +impl ManagedProviderHandle { + pub fn renew_opening_tokens(&mut self) -> Option<(CancellationToken, CancellationToken)> { + if let Some(ref mut handle) = self.handle { + if let Some((cancel, completion, gen)) = self.manager.renew_opening_tokens(handle.allocation_id) { + handle.cancel_token = Some(cancel.clone()); + handle.completion_token = Some(completion.clone()); + handle.open_generation = gen; + return Some((cancel, completion)); + } + } + None + } +} + +impl ActiveProviderManager { + pub(super) fn get_affinity_ttl(cfg: &AppConfig) -> ProviderAffinityTtl { + ProviderAffinityTtl(cfg.config.load().get_provider_affinity_ttl_secs()) + } + + pub(super) fn has_foreign_reservation(&self, provider_name: &Arc, session_owner: Option<&str>) -> bool { + if !self.providers.reservation_blocks_other_sessions(provider_name) { + return false; + } + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + leases.has_foreign_reserved_lease(provider_name, session_owner) + } + + /// Resolves how a playback owner is bound to a provider of `input_name`, with a + /// single lease-table lookup per acquisition. + pub(super) fn owner_provider_pin(&self, input_name: &Arc, session_owner: &str) -> Option { + let (lease_pin, affinity_provider) = { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + let lease_pin = leases + .lease_of_owner(session_owner) + .map(|lease| (Arc::clone(&lease.provider_name), lease.state.is_confirmed())); + (lease_pin, leases.affinity_provider_for_owner(session_owner)) + }; + // The lease lock is released before the connection lock is taken. + if let Some((provider_name, confirmed)) = lease_pin { + if self.providers.is_provider_for_input(&provider_name, input_name) + && (confirmed || self.has_active_owner_for_provider(&provider_name, session_owner)) + { + return Some(OwnerProviderPin::Reserved(provider_name)); + } + } + affinity_provider + .filter(|provider_name| self.providers.is_provider_for_input(provider_name, input_name)) + .map(OwnerProviderPin::Preferred) + } + + /// Provider a playback currently prefers on re-entry, for diagnostics and tests. + pub fn provider_affinity_for_owner(&self, session_owner: &str) -> Option> { + let session_owner = playback_lease_owner(session_owner); + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + leases.affinity_provider_for_owner(session_owner) + } + + /// True when `session_owner` already backs a live allocation on `provider_name`, + /// without materialising the full owner set for a membership test. + pub(super) fn has_active_owner_for_provider(&self, provider_name: &Arc, session_owner: &str) -> bool { + let connections = self.read_connections(); + connections.by_owner.get(session_owner).is_some_and(|alloc_ids| { + alloc_ids.iter().any(|id| { + if let Some(info) = connections.single.get(id) { + info.allocation.get_provider_name().as_ref() == Some(provider_name) + } else if let Some(key) = connections.shared.shared_by_allocation_id.get(id) { + connections + .shared + .by_key + .get(key) + .is_some_and(|shared| shared.allocation.get_provider_name().as_ref() == Some(provider_name)) + } else { + false + } + }) + }) + } + + pub(super) fn active_reservation_owners(&self, provider_name: &Arc) -> HashSet> { + let connections = self.read_connections(); + let mut owners = HashSet::new(); + if let Some(alloc_ids) = connections.by_provider.get(provider_name) { + for id in alloc_ids { + if let Some(info) = connections.single.get(id) { + if let Some(owner) = info.session_owner.as_ref() { + owners.insert(Arc::clone(owner)); + } + } else if let Some(key) = connections.shared.shared_by_allocation_id.get(id) { + if let Some(shared) = connections.shared.by_key.get(key) { + if let Some(owner) = shared.session_owner.as_ref() { + owners.insert(Arc::clone(owner)); + } + } + } + } + } else { + for info in connections.single.values() { + if info.allocation.get_provider_name().as_ref() == Some(provider_name) { + if let Some(owner) = info.session_owner.as_ref() { + owners.insert(Arc::clone(owner)); + } + } + } + for shared in connections.shared.by_key.values() { + if shared.allocation.get_provider_name().as_ref() == Some(provider_name) { + if let Some(owner) = shared.session_owner.as_ref() { + owners.insert(Arc::clone(owner)); + } + } + } + } + owners + } + + pub fn refresh_provider_reservation(&self, provider_name: &Arc, session_owner: &str, ttl_secs: u64) { + self.refresh_identified_provider_reservation(provider_name, session_owner, PlaybackKind::LiveHls, ttl_secs); + } + + /// Recreates or renews a provider reservation for an owner with an explicit kind. + /// + /// Unlike [`Self::refresh_adaptive_playback_lease`] this recreates the lease when it + /// no longer exists, which is what a failed preemption needs to restore a victim + /// reservation that [`Self::clear_identified_provider_reservation`] already removed. + pub fn refresh_identified_provider_reservation( + &self, + provider_name: &Arc, + session_owner: &str, + kind: PlaybackKind, + ttl_secs: u64, + ) { + let session_owner = playback_lease_owner(session_owner); + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return; + } + let granted_ttl = self.renewal_granted_ttl(provider_name, session_owner, ttl_secs); + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + if ttl_secs == 0 { + leases.release_owner(session_owner); + return; + } + if leases.renew_current_owner(session_owner, provider_name, kind, granted_ttl).is_none() { + let req_id = PlaybackRequestId::next(); + let _ = leases.begin_owner(session_owner, provider_name, kind, req_id); + leases.renew_identified_owner(session_owner, provider_name, kind, req_id, granted_ttl); + } + } + + /// The reconnect window a renewal may actually grant for `owner`. + /// + /// A lease that currently holds no reservation right must not gain one through a + /// mere activity refresh: the grant is re-checked against provider capacity with + /// the same admission rule as a confirmation, so a previously denied reservation + /// cannot be silently restored by a later refresh. + pub(super) fn renewal_granted_ttl(&self, provider_name: &Arc, owner: &str, requested_ttl_secs: u64) -> u64 { + if requested_ttl_secs == 0 { + return 0; + } + let already_reserves = { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + leases.lease_of_owner(owner).is_some_and(|lease| lease.state.is_confirmed() && lease.idle_ttl_secs > 0) + }; + if already_reserves || self.confirmation_reserve_allowed(provider_name, owner) { + requested_ttl_secs + } else { + 0 + } + } + + /// Clears an explicit owner. Public HLS tokens intentionally have no owner-wide + /// delete right: a delayed session cleanup must not erase a retry's shared lease. + /// HLS request cleanup uses request IDs or binding tags instead. + pub fn clear_provider_reservation(&self, session_owner: &str) { + if playback_lease_owner(session_owner) != session_owner { + return; + } + let _transition = self.lock_capacity_transition(); + let mut leases = self.write_leases(); + leases.forget_affinity(session_owner); + leases.release_owner(session_owner); + } + + /// Administrative end of the playback behind a session that was actually terminated + /// (terminate endpoint, kick). + /// + /// A public HLS token shares its stable owner with the retries of the same + /// client/user/channel, so it only ends the binding it acquired itself. A stale + /// token therefore never removes the lease or provider preference of a newer + /// binding. A bare owner keeps the owner-wide release semantics. + pub fn terminate_identified_playback_owner(&self, session_token: &str) -> bool { + let owner = playback_lease_owner(session_token); + let _transition = self.lock_capacity_transition(); + let mut leases = self.write_leases(); + if owner == session_token { + leases.forget_affinity(owner); + return leases.release_owner(owner).is_some(); + } + leases.terminate_identified_token(owner, session_token) + } + + /// Ends the provider preference of a failed playback, but only while it still + /// belongs to the exact binding that failed. A delayed failure of an older binding + /// leaves the successor's preference intact. + pub fn forget_identified_provider_affinity( + &self, + session_owner: &str, + provider_name: &Arc, + binding_tag: ProviderBindingTag, + ) -> bool { + let session_owner = playback_lease_owner(session_owner); + let _transition = self.lock_capacity_transition(); + self.write_leases().forget_identified_affinity(session_owner, provider_name, binding_tag) + } + + pub fn clear_identified_provider_reservation( + &self, + session_owner: &str, + provider_name: &Arc, + binding_tag: Option, + ) { + let session_owner = playback_lease_owner(session_owner); + // A delayed clear without the exact binding tag has no delete right: it must + // not fall back to an owner-wide release and erase a successor lease. + let Some(binding_tag) = binding_tag else { + return; + }; + let _transition = self.lock_capacity_transition(); + let mut leases = self.write_leases(); + leases.release_matching_owner(session_owner, provider_name, binding_tag); + } + + /// Confirms real media activity for a playback lease. Only a confirmed lease may + /// outlive its request as a reconnect slot. + pub fn confirm_playback_activity(&self, owner: &str) -> Option { self.confirm_owner(owner, None) } + + /// Confirms real media activity for a specific request ID of a playback lease. + pub fn confirm_identified_playback_activity( + &self, + owner: &str, + request_id: PlaybackRequestId, + ) -> Option { + self.confirm_owner(owner, Some(request_id)) + } + + /// Records media activity under the capacity transition. The confirmation only + /// grants a reservation right when the lease already backs a live allocation or a + /// free slot remains; otherwise the media bytes are recorded without a reservation + /// so a late cache confirmation cannot over-commit a provider at its limit. + fn confirm_owner(&self, owner: &str, request_id: Option) -> Option { + let owner = playback_lease_owner(owner); + let _transition = self.lock_capacity_transition(); + let (provider_name, wants_reservation) = { + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + let provider_name = leases.provider_for_owner(owner)?; + let wants_reservation = leases.lease_of_owner(owner).is_some_and(|lease| lease.idle_ttl_secs > 0); + (provider_name, wants_reservation) + }; + let reserve = !wants_reservation || self.confirmation_reserve_allowed(&provider_name, owner); + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + match (request_id, reserve) { + (Some(request_id), true) => leases.confirm_identified_owner(owner, request_id), + (Some(request_id), false) => leases.confirm_identified_owner_without_reservation(owner, request_id), + (None, true) => leases.confirm_owner(owner), + (None, false) => leases.confirm_owner_without_reservation(owner), + } + } + + /// True when confirming `session_owner`'s lease may reserve a provider slot: the + /// lease either already backs a live allocation, or a free slot remains below the + /// configured maximum after foreign reservations are counted. + fn confirmation_reserve_allowed(&self, provider_name: &Arc, session_owner: &str) -> bool { + if self.has_active_owner_for_provider(provider_name, session_owner) { + return true; + } + let Some((current, max, foreign)) = self.reservation_capacity_usage(provider_name, Some(session_owner)) else { + return false; + }; + max == 0 || current.saturating_add(foreign) < max + } + + /// Ends a playback request and applies the outcome-specific lease policy. + /// + /// `idle_ttl_secs` overrides the reconnect window stored on the lease; `None` + /// keeps the window the endpoint configured when it renewed the lease. + pub fn finish_playback_request(&self, owner: &str, outcome: PlaybackRequestOutcome, idle_ttl_secs: Option) { + self.finish_playback_request_inner(owner, None, outcome, idle_ttl_secs); + } + + pub(super) fn finish_playback_request_inner( + &self, + owner: &str, + request_id: Option, + outcome: PlaybackRequestOutcome, + idle_ttl_secs: Option, + ) { + let owner = playback_lease_owner(owner); + let _transition = self.lock_capacity_transition(); + // A delayed completion must not end a newer request of this playback. + // Only active connections from a DIFFERENT request ID (or shared streams) prevent lease finish. + let connections = self.read_connections(); + let has_other_active_connection = connections.by_owner.get(owner).is_some_and(|alloc_ids| { + alloc_ids.iter().any(|id| { + connections + .single + .get(id) + .is_some_and(|info| request_id.is_none() || info.playback_request_id != request_id) + || connections.shared.shared_by_allocation_id.contains_key(id) + }) + }); + if has_other_active_connection { + if let Some(req_id) = request_id { + let mut leases = self.write_leases(); + leases.detach_identified_request(owner, req_id); + } + return; + } + drop(connections); + let mut leases = self.write_leases(); + if let Some(req_id) = request_id { + leases.finish_identified_owner(owner, req_id, outcome, idle_ttl_secs); + } else { + leases.finish_owner(owner, outcome, idle_ttl_secs); + } + } + + /// The provider lease's binding tag for an owner. The HLS layer captures this + /// from the acquired handle (via `ProviderHandle::binding_tag`) and passes it + /// back to [`Self::clear_identified_provider_reservation`] so a stale detach + /// cannot delete a successor lease on the same account. + pub fn binding_tag_for_owner(&self, owner: &str) -> Option { + let owner = playback_lease_owner(owner); + let mut leases = self.write_leases(); + Self::prune_expired_leases(&mut leases); + leases.lease_of_owner(owner).map(|lease| ProviderBindingTag::new(lease.id, lease.binding_generation)) + } + + pub fn renew_opening_tokens(&self, alloc_id: AllocationId) -> Option<(CancellationToken, CancellationToken, u64)> { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get_mut(&alloc_id) { + if info.lifecycle != ConnectionLifecycle::Closing && info.lifecycle != ConnectionLifecycle::Closed { + info.lifecycle = ConnectionLifecycle::Opening; + info.open_generation = info.open_generation.wrapping_add(1); + info.reaper_spawned = false; + let cancel_token = CancellationToken::new(); + let completion_token = CancellationToken::new(); + info.cancel_token = cancel_token.clone(); + info.completion_token = completion_token.clone(); + return Some((cancel_token, completion_token, info.open_generation)); + } + } + None + } +} diff --git a/backend/session/src/active_provider_manager/preemption.rs b/backend/session/src/active_provider_manager/preemption.rs new file mode 100644 index 000000000..fbbd0f4e4 --- /dev/null +++ b/backend/session/src/active_provider_manager/preemption.rs @@ -0,0 +1,655 @@ +use super::{ + AcquireProviderParams, ActiveProviderManager, ActiveProviderManagerCore, ConnectionKind, Connections, + OwnerProviderPin, PlaybackLeaseRef, PreemptionOutcome, ProviderAllocationGuard, ReleaseAction, SharedConnectionId, + PREEMPTION_COMPLETION_TIMEOUT, +}; +use ::shared::utils::sanitize_sensitive_info; +use std::{ + cmp::Reverse, + collections::{BTreeMap, HashMap, HashSet}, + net::SocketAddr, + sync::{atomic::Ordering, Arc, Weak}, + time::Instant, +}; +use tokio_util::sync::CancellationToken; +use tuliprox_core::{ + model::{ + AllocationId, AppConfig, ConnectionLifecycle, GracePeriodOptions, ProviderAllocation, ProviderCloseReason, + ProviderHandle, + }, + utils::debug_if_enabled, +}; + +type PreemptionCandidate = (PriorityOwner, AllocationId, i8, Instant); + +// Key for BTreeMap priority index: (priority, Reverse, AllocationId) +// Semantics: lower numeric priority value = higher importance (0 = highest, 127 = lowest). +// `.last()` on the BTreeMap returns the entry with the highest priority value, which is +// the lowest-importance connection and therefore the best eviction victim. +// Ties are broken by `Reverse`: among equal priority values, the oldest connection +// (smallest `created_at`) sorts last and is evicted first. +pub(super) type PriorityKey = (i8, Reverse, AllocationId); + +fn is_better_preemption_candidate(current: Option, candidate: PreemptionCandidate) -> bool { + match current { + None => true, + Some((_, current_allocation_id, current_priority, current_created_at)) => { + candidate.2 > current_priority + || (candidate.2 == current_priority + && (candidate.3 < current_created_at + || (candidate.3 == current_created_at && candidate.1 < current_allocation_id))) + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum PriorityOwner { + Single(AllocationId), + Shared(SharedConnectionId), +} + +impl ActiveProviderManager { + pub(super) fn upsert_priority_entry( + connections: &mut Connections, + provider_name: &Arc, + key: PriorityKey, + owner: PriorityOwner, + kind: ConnectionKind, + ) { + connections.priority_index.entry(provider_name.clone()).or_default().insert(key, owner); + if kind == ConnectionKind::Soft { + connections.soft_priority_index.entry(provider_name.clone()).or_default().insert(key, owner); + } + } + + pub(super) fn remove_priority_entry( + connections: &mut Connections, + provider_name: &Arc, + key: &PriorityKey, + kind: ConnectionKind, + ) { + if let Some(tree) = connections.priority_index.get_mut(provider_name) { + tree.remove(key); + } + if kind == ConnectionKind::Soft { + if let Some(tree) = connections.soft_priority_index.get_mut(provider_name) { + tree.remove(key); + } + } + } + + pub(super) fn get_grace_options(cfg: &AppConfig) -> GracePeriodOptions { cfg.config.load().get_grace_options() } + + pub(super) fn acquire_exact_connection_inner_no_preempt( + &self, + provider_name: &Arc, + allow_grace: bool, + params: &AcquireProviderParams<'_>, + ) -> Option { + let mut allocation = ProviderAllocationGuard::new( + self.providers.acquire_exact_connection_with_grace_override(provider_name, allow_grace), + ); + if matches!(allocation.allocation(), ProviderAllocation::Exhausted) { + return None; + } + if self.exceeds_reserved_capacity(provider_name, params.session_owner()) { + return None; + } + Some(self.register_allocation(allocation.take(), params)) + } + + fn select_victim_from_index( + &self, + index: &HashMap, BTreeMap>, + input_name: &Arc, + requester_priority: Option, + reserved_providers: &HashSet>, + ) -> Option { + let mut victim = None; + + for (prov_name, tree) in index { + if (!self.providers.is_provider_for_input(prov_name, input_name) && prov_name != input_name) + || reserved_providers.contains(prov_name) + { + continue; + } + + let Some(((victim_priority, Reverse(created_at), allocation_id), owner)) = tree.iter().next_back() else { + continue; + }; + + if let Some(req_prio) = requester_priority { + if *victim_priority <= req_prio { + continue; + } + } + + let candidate = (*owner, *allocation_id, *victim_priority, *created_at); + if is_better_preemption_candidate(victim, candidate) { + victim = Some(candidate); + } + } + + victim + } + + fn is_normal_preemption_candidate( + connections: &Connections, + provider_name: &Arc, + candidate: PreemptionCandidate, + ) -> bool { + match candidate.0 { + PriorityOwner::Single(alloc_id) => connections.single.get(&alloc_id).is_some_and(|info| { + info.kind == ConnectionKind::Normal + && info.lifecycle != ConnectionLifecycle::Closing + && info.allocation.get_provider_name().as_ref() == Some(provider_name) + }), + PriorityOwner::Shared(shared_id) => connections + .shared + .shared_by_allocation_id + .get(&shared_id) + .and_then(|key| connections.shared.by_key.get(key)) + .is_some_and(|shared| { + shared.allocation_id == candidate.1 + && shared.kind == ConnectionKind::Normal + && shared.allocation.get_provider_name().as_ref() == Some(provider_name) + }), + } + } + + pub(super) fn select_preemption_candidate( + &self, + connections: &Connections, + input_name: &Arc, + new_priority: i8, + kind_needed: ConnectionKind, + reserved_providers: &HashSet>, + ) -> Option { + match kind_needed { + ConnectionKind::Normal => { + let soft_victim = self.select_victim_from_index( + &connections.soft_priority_index, + input_name, + None, + reserved_providers, + ); + if soft_victim.is_some() { + return soft_victim; + } + + let mut victim = None; + for (prov_name, tree) in &connections.priority_index { + if (!self.providers.is_provider_for_input(prov_name, input_name) && prov_name != input_name) + || reserved_providers.contains(prov_name) + { + continue; + } + + let Some(((victim_priority, Reverse(created_at), allocation_id), owner)) = tree.iter().next_back() + else { + continue; + }; + if *victim_priority <= new_priority { + continue; + } + let candidate = (*owner, *allocation_id, *victim_priority, *created_at); + if !ActiveProviderManager::is_normal_preemption_candidate(connections, prov_name, candidate) { + continue; + } + if is_better_preemption_candidate(victim, candidate) { + victim = Some(candidate); + } + } + victim + } + ConnectionKind::Soft => self.select_victim_from_index( + &connections.soft_priority_index, + input_name, + Some(new_priority), + reserved_providers, + ), + } + } + + pub(super) fn acquire_connection_inner_no_preempt( + &self, + provider_or_input_name: &Arc, + allow_grace: bool, + params: &AcquireProviderParams<'_>, + ) -> Option { + if let Some(owner) = params.session_owner() { + match self.owner_provider_pin(provider_or_input_name, owner) { + Some(OwnerProviderPin::Reserved(reserved_provider)) => { + return self.acquire_exact_connection_inner_no_preempt(&reserved_provider, allow_grace, params); + } + // A continuing playback whose reconnect lease lapsed during a request gap + // returns to its provider before priority selection, but only into free + // capacity: the lineup prefers any free slot over a grace over-allocation, + // so the preference must not force one either. + Some(OwnerProviderPin::Preferred(affinity_provider)) => { + if let Some(handle) = + self.acquire_exact_connection_inner_no_preempt(&affinity_provider, false, params) + { + debug_if_enabled!( + "Provider affinity kept: provider={} owner={} playback_kind={}", + sanitize_sensitive_info(&affinity_provider), + sanitize_sensitive_info(owner), + params.lease.map_or_else(|| "-".to_string(), |lease| lease.kind.to_string()) + ); + return Some(handle); + } + debug_if_enabled!( + "Provider affinity unavailable, using lineup: provider={} owner={}", + sanitize_sensitive_info(&affinity_provider), + sanitize_sensitive_info(owner) + ); + } + None => {} + } + } + + let candidate_count = self.providers.provider_names_for_input(provider_or_input_name).len(); + let attempts = candidate_count.max(1); + let mut skipped_reserved = HashSet::new(); + for _ in 0..attempts { + let allocation = + ProviderAllocationGuard::new(self.providers.acquire_connection_with_grace_override_excluding( + provider_or_input_name, + allow_grace, + &skipped_reserved, + )); + if matches!(allocation.allocation(), ProviderAllocation::Exhausted) { + break; + } + if let Some(provider_name) = allocation.allocation().get_provider_name() { + if self.exceeds_reserved_capacity(&provider_name, params.session_owner()) { + self.log_reserved_capacity_skip(&provider_name, params); + skipped_reserved.insert(provider_name); + if skipped_reserved.len() >= attempts { + break; + } + continue; + } + } + return Some(self.finalize_lineup_allocation(provider_or_input_name, allow_grace, allocation, params)); + } + + None + } + + #[allow(clippy::too_many_lines)] + /// Evict a single lower-priority connection across the entire input lineup + /// (all provider aliases). Used when a `GracePeriod` allocation was granted. + /// Returns true if a victim was successfully evicted. + pub(super) fn evict_lower_priority_on_input( + &self, + input_name: &Arc, + new_priority: i8, + kind_needed: ConnectionKind, + session_owner: Option<&str>, + ) -> bool { + let reserved_providers = self.reserved_provider_names_for_other(input_name, session_owner); + + let victim = { + let connections = self.read_connections(); + self.select_preemption_candidate(&connections, input_name, new_priority, kind_needed, &reserved_providers) + }; + + let Some((owner, alloc_id, v_prio, victim_created_at)) = victim else { + return false; + }; + match owner { + PriorityOwner::Shared(shared_id) => { + debug_if_enabled!( + "Grace-evicting shared connection (allocation_id={shared_id}, prio={v_prio}) on input {} for higher priority request (prio={})", + sanitize_sensitive_info(input_name), + new_priority + ); + + let released = { + let mut connections = self.write_connections(); + let Some(key) = connections.shared.shared_by_allocation_id.get(&shared_id).cloned() else { + return false; + }; + + let still_match = connections.shared.by_key.get(&key).is_some_and(|shared| { + shared.allocation_id == alloc_id + && shared.priority == v_prio + && shared.created_at == victim_created_at + }); + if !still_match { + return false; + } + + if let Some(shared) = connections.shared.by_key.remove(&key) { + connections.shared.shared_by_allocation_id.remove(&shared.allocation_id); + Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref()); + for subscriber_id in shared.connections.keys() { + connections.shared.key_by_subscriber.remove(subscriber_id); + } + if let Some(name) = shared.allocation.get_provider_name() { + if let Some(list) = connections.by_provider.get_mut(&name) { + list.remove(&shared.allocation_id); + } + Self::remove_priority_entry( + &mut connections, + &name, + &(v_prio, Reverse(victim_created_at), alloc_id), + shared.kind, + ); + } + Some((key, shared.allocation, shared.cancel_token)) + } else { + return false; + } + }; + if let Some((stream_url, allocation, cancel_token)) = released { + if let Some(token) = cancel_token { + token.cancel(); + } + if let Some(ssm) = self.shared_stream_manager.get().and_then(Weak::upgrade) { + tokio::spawn(async move { + ssm.teardown_preempted_stream(&stream_url, alloc_id).await; + allocation.release(); + }); + return false; + } + allocation.release(); + } + } + PriorityOwner::Single(victim_alloc_id) => { + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get(&victim_alloc_id) { + if info.priority != v_prio || info.created_at != victim_created_at { + return false; + } + } + if let Some(info) = connections.single.get_mut(&victim_alloc_id) { + debug_if_enabled!( + "Grace-evicting single connection from {} (prio={}) on input {} for higher priority request (prio={})", + sanitize_sensitive_info(&info.client_addr.to_string()), + v_prio, + sanitize_sensitive_info(input_name), + new_priority + ); + info.close_reason.store(ProviderCloseReason::PriorityPreempted as u8, Ordering::Release); + } + let action = ActiveProviderManagerCore::plan_single_release_locked(&mut connections, victim_alloc_id); + if let ReleaseAction::Wait(alloc_id, gen, completion_token) = action { + let shutdown_token = self.shutdown_token.clone(); + let core = Arc::clone(&self.core); + tokio::spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = completion_token.cancelled() => { + core.complete_release_with_generation(alloc_id, Some(gen)); + } + } + }); + return false; + } + } + } + + true + } + + #[allow(clippy::too_many_lines)] + pub(super) fn try_preempt_connection_outcome( + &self, + input_name: &Arc, + new_priority: i8, + allow_grace: bool, + kind_needed: ConnectionKind, + session_owner: Option<&str>, + ) -> PreemptionOutcome { + let reserved_providers = self.reserved_provider_names_for_other(input_name, session_owner); + let victim = { + let connections = self.read_connections(); + self.select_preemption_candidate(&connections, input_name, new_priority, kind_needed, &reserved_providers) + }; + + let Some((owner, alloc_id, v_prio, victim_created_at)) = victim else { + return PreemptionOutcome::Exhausted; + }; + + match owner { + PriorityOwner::Shared(shared_id) => { + debug_if_enabled!( + "Preempting shared connection (allocation_id={shared_id}, prio={v_prio}) for higher priority request (prio={new_priority})" + ); + let released_shared_allocation = { + let mut connections = self.write_connections(); + let Some(key) = connections.shared.shared_by_allocation_id.get(&shared_id).cloned() else { + return PreemptionOutcome::Exhausted; + }; + + let still_match = connections.shared.by_key.get(&key).is_some_and(|shared| { + shared.allocation_id == alloc_id + && shared.priority == v_prio + && shared.created_at == victim_created_at + }); + if !still_match { + return PreemptionOutcome::Exhausted; + } + + if let Some(shared) = connections.shared.by_key.remove(&key) { + connections.shared.shared_by_allocation_id.remove(&shared.allocation_id); + Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref()); + for subscriber_id in shared.connections.keys() { + connections.shared.key_by_subscriber.remove(subscriber_id); + } + + if let Some(name) = shared.allocation.get_provider_name() { + if let Some(list) = connections.by_provider.get_mut(&name) { + list.remove(&shared.allocation_id); + } + Self::remove_priority_entry( + &mut connections, + &name, + &(v_prio, Reverse(victim_created_at), alloc_id), + shared.kind, + ); + } + Some((key, shared.allocation, shared.cancel_token)) + } else { + None + } + }; + + let Some((stream_url, allocation, cancel_token)) = released_shared_allocation else { + return PreemptionOutcome::Exhausted; + }; + + if let Some(token) = cancel_token { + token.cancel(); + } + if let Some(ssm) = self.shared_stream_manager.get().and_then(Weak::upgrade) { + // Signal completion after teardown finishes so async acquirers can wait for the + // actual upstream shutdown rather than receiving Exhausted immediately. + let done_token = CancellationToken::new(); + let done_signal = done_token.clone(); + tokio::spawn(async move { + ssm.teardown_preempted_stream(&stream_url, alloc_id).await; + allocation.release(); + done_signal.cancel(); + }); + PreemptionOutcome::PendingCompletion(None, done_token) + } else { + allocation.release(); + if let Some(alloc) = self.try_acquire_allocation_after_freed( + input_name, + allow_grace, + &reserved_providers, + session_owner, + ) { + PreemptionOutcome::Acquired(alloc) + } else { + PreemptionOutcome::Exhausted + } + } + } + PriorityOwner::Single(victim_alloc_id) => { + let (action, completion_token) = { + let mut connections = self.write_connections(); + if let Some(info) = connections.single.get(&victim_alloc_id) { + if info.priority != v_prio || info.created_at != victim_created_at { + return PreemptionOutcome::Exhausted; + } + } else { + return PreemptionOutcome::Exhausted; + } + if let Some(info) = connections.single.get_mut(&victim_alloc_id) { + debug_if_enabled!( + "Preempting single connection from {} (prio={v_prio}) for higher priority request (prio={new_priority})", + sanitize_sensitive_info(&info.client_addr.to_string()) + ); + info.close_reason.store(ProviderCloseReason::PriorityPreempted as u8, Ordering::Release); + } + let completion_token = + connections.single.get(&victim_alloc_id).map(|info| info.completion_token.clone()); + let action = + ActiveProviderManagerCore::plan_single_release_locked(&mut connections, victim_alloc_id); + (action, completion_token) + }; + + match action { + ReleaseAction::Wait(victim_id, gen, comp_token) => { + self.spawn_preemption_reaper(victim_id, gen, comp_token.clone()); + PreemptionOutcome::PendingCompletion(Some((victim_id, gen)), comp_token) + } + ReleaseAction::None => { + if let Some(token) = completion_token { + // Retrieve the current generation for the idempotent self-release path. + let victim_identity = { + let connections = self.read_connections(); + connections + .single + .get(&victim_alloc_id) + .map(|info| (victim_alloc_id, info.open_generation)) + }; + if let Some((victim_id, gen)) = victim_identity { + self.spawn_preemption_reaper(victim_id, gen, token.clone()); + } + PreemptionOutcome::PendingCompletion(victim_identity, token) + } else { + PreemptionOutcome::Exhausted + } + } + ReleaseAction::Immediate => { + if let Some(alloc) = self.try_acquire_allocation_after_freed( + input_name, + allow_grace, + &reserved_providers, + session_owner, + ) { + PreemptionOutcome::Acquired(alloc) + } else { + PreemptionOutcome::Exhausted + } + } + } + } + } + } + + pub(super) fn try_preempt_connection( + &self, + input_name: &Arc, + new_priority: i8, + allow_grace: bool, + kind_needed: ConnectionKind, + session_owner: Option<&str>, + ) -> Option { + match self.try_preempt_connection_outcome(input_name, new_priority, allow_grace, kind_needed, session_owner) { + PreemptionOutcome::Acquired(alloc) => Some(alloc), + _ => None, + } + } + + pub fn acquire_exact_connection_with_grace( + &self, + provider_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + ) -> Option { + self.acquire_exact_connection_with_grace_for_session(provider_name, addr, allow_grace, priority, kind, None) + } + + pub fn acquire_exact_connection_with_grace_for_session( + &self, + provider_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + session_owner: Option<&str>, + ) -> Option { + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return None; + } + let lease = session_owner.map(PlaybackLeaseRef::for_session_owner); + self.acquire_exact_connection_inner( + provider_name, + allow_grace, + &AcquireProviderParams { addr, priority, kind, lease: lease.as_ref().copied() }, + ) + } + + /// Acquire a provider connection while explicitly controlling provider-side grace allocations. + pub fn acquire_connection_with_grace( + &self, + input_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + ) -> Option { + self.acquire_connection_with_grace_for_session(input_name, addr, allow_grace, priority, kind, None) + } + + pub fn acquire_connection_with_grace_for_session( + &self, + input_name: &Arc, + addr: &SocketAddr, + allow_grace: bool, + priority: i8, + kind: ConnectionKind, + session_owner: Option<&str>, + ) -> Option { + self.acquire_connection_with_lease_for_session( + input_name, + addr, + allow_grace, + priority, + kind, + session_owner.map(PlaybackLeaseRef::for_session_owner), + ) + } + + /// Frees a preempted victim's slot once its body completes, or after + /// `PREEMPTION_COMPLETION_TIMEOUT` at the latest. It runs independently of the preempting + /// request, so a request dropped while waiting (client disconnect, deadline, cancelled + /// future) cannot leave the victim's slot occupied. + fn spawn_preemption_reaper(&self, victim_id: AllocationId, generation: u64, completion_token: CancellationToken) { + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return; + }; + let shutdown_token = self.shutdown_token.clone(); + let core = Arc::clone(&self.core); + runtime.spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {} + _ = tokio::time::timeout(PREEMPTION_COMPLETION_TIMEOUT, completion_token.cancelled()) => { + core.complete_release_with_generation(victim_id, Some(generation)); + } + } + }); + } +} diff --git a/backend/session/src/active_provider_manager/release.rs b/backend/session/src/active_provider_manager/release.rs new file mode 100644 index 000000000..0d0dcdeb9 --- /dev/null +++ b/backend/session/src/active_provider_manager/release.rs @@ -0,0 +1,460 @@ +use super::{ + ActiveProviderManager, ActiveProviderManagerCore, Connections, ManagedProviderHandle, ProviderReleaseSnapshot, + EVICTED_PROVIDER_RELEASE_POLL_INTERVAL, +}; +use std::{ + cmp::Reverse, + net::SocketAddr, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; +use tokio::time::Instant as TokioInstant; +use tokio_util::sync::CancellationToken; +use tuliprox_core::model::{ + AllocationId, ConnectionLifecycle, ProviderAllocation, ProviderCloseReason, ProviderHandle, +}; + +pub(super) struct ProviderAllocationGuard(pub(super) Option); + +impl ProviderAllocationGuard { + pub(super) fn new(allocation: ProviderAllocation) -> Self { Self(Some(allocation)) } + + pub(super) fn allocation(&self) -> &ProviderAllocation { self.0.as_ref().unwrap_or(&ProviderAllocation::Exhausted) } + + pub(super) fn take(&mut self) -> ProviderAllocation { self.0.take().unwrap_or(ProviderAllocation::Exhausted) } +} + +impl Drop for ProviderAllocationGuard { + fn drop(&mut self) { + if let Some(allocation) = self.0.take() { + allocation.release(); + } + } +} + +impl std::fmt::Debug for ManagedProviderHandle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ManagedProviderHandle").field("handle", &self.handle).finish_non_exhaustive() + } +} + +impl Drop for ManagedProviderHandle { + fn drop(&mut self) { + if let Some(handle) = self.handle.take() { + self.manager.release_handle(&handle); + } + } +} + +impl ActiveProviderManagerCore { + pub fn complete_release(&self, alloc_id: AllocationId) { self.complete_release_with_generation(alloc_id, None); } + + pub fn complete_release_with_generation(&self, alloc_id: AllocationId, generation: Option) { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + Self::complete_release_locked(&mut connections, alloc_id, generation); + } + + pub(super) fn complete_release_locked( + connections: &mut Connections, + alloc_id: AllocationId, + expected_generation: Option, + ) -> Option { + if let Some(info) = connections.single.get(&alloc_id) { + if let Some(expected_gen) = expected_generation { + if info.open_generation != expected_gen { + return None; + } + } + } + if let Some(mut info) = connections.single.remove(&alloc_id) { + ActiveProviderManager::unindex_owner(connections, alloc_id, info.session_owner.as_ref()); + if let Some(set) = connections.single_by_addr.get_mut(&info.client_addr) { + set.remove(&alloc_id); + if set.is_empty() { + connections.single_by_addr.remove(&info.client_addr); + } + } + if !info.allocation.is_unlimited_provider() { + if let Some(name) = info.allocation.get_provider_name() { + if let Some(list) = connections.by_provider.get_mut(&name) { + list.remove(&alloc_id); + } + ActiveProviderManager::remove_priority_entry( + connections, + &name, + &(info.priority, Reverse(info.created_at), alloc_id), + info.kind, + ); + } + } + info.lifecycle = ConnectionLifecycle::Closed; + info.cancel_token.cancel(); + info.allocation.release(); + if let Some(name) = info.allocation.get_provider_name() { + connections.released_providers.push(name); + } + Some(info.allocation) + } else { + None + } + } + + pub(super) fn plan_single_release_locked(connections: &mut Connections, alloc_id: AllocationId) -> ReleaseAction { + let Some(info) = connections.single.get_mut(&alloc_id) else { + return ReleaseAction::None; + }; + if info.lifecycle == ConnectionLifecycle::Closed { + return ReleaseAction::None; + } + + info.cancel_token.cancel(); + + let completed = info.completion_token.is_cancelled(); + let needs_completion_wait = (info.lifecycle == ConnectionLifecycle::Closing + || info.lifecycle == ConnectionLifecycle::Opening + || info.has_body_owner) + && !completed; + + if needs_completion_wait { + info.lifecycle = ConnectionLifecycle::Closing; + let client_addr = info.client_addr; + let prio = info + .allocation + .get_provider_name() + .map(|name| (name, (info.priority, Reverse(info.created_at), alloc_id), info.kind)); + let already_spawned = info.reaper_spawned; + info.reaper_spawned = true; + let gen = info.open_generation; + let completion_token = info.completion_token.clone(); + + if let Some((name, key, kind)) = prio { + ActiveProviderManager::remove_priority_entry(connections, &name, &key, kind); + } + if let Some(set) = connections.single_by_addr.get_mut(&client_addr) { + set.remove(&alloc_id); + if set.is_empty() { + connections.single_by_addr.remove(&client_addr); + } + } + if already_spawned { + ReleaseAction::None + } else { + ReleaseAction::Wait(alloc_id, gen, completion_token) + } + } else { + let gen = info.open_generation; + Self::complete_release_locked(connections, alloc_id, Some(gen)); + ReleaseAction::Immediate + } + } +} + +#[derive(Debug)] +pub(super) enum ReleaseAction { + None, + Immediate, + Wait(AllocationId, u64, CancellationToken), +} + +impl ActiveProviderManager { + fn has_connections_for_addr(&self, addr: &SocketAddr) -> bool { + let _transition = self.lock_capacity_transition(); + let connections = self.read_connections(); + connections.single.values().any(|info| info.client_addr == *addr) + || connections + .shared + .by_key + .values() + .any(|shared| shared.connections.values().any(|subscriber| subscriber.addr == *addr)) + } + + /// Waits for every allocation currently using an address to leave the registry. + /// Admission handoffs use `wait_for_snapshot_release` to ignore later allocations at that address. + pub async fn wait_for_addr_release(&self, addr: &SocketAddr, timeout: Duration) -> bool { + let deadline = TokioInstant::now() + timeout; + loop { + if !self.has_connections_for_addr(addr) { + return true; + } + let now = TokioInstant::now(); + if now >= deadline { + return false; + } + tokio::time::sleep_until((now + EVICTED_PROVIDER_RELEASE_POLL_INTERVAL).min(deadline)).await; + } + } + + pub(crate) fn release_snapshot_for_addr(&self, addr: &SocketAddr) -> ProviderReleaseSnapshot { + let _transition = self.lock_capacity_transition(); + let connections = self.read_connections(); + let single_allocations = connections + .single + .values() + .filter_map(|info| (info.client_addr == *addr).then_some(info.allocation_id)) + .collect(); + let shared_subscribers = connections + .shared + .by_key + .values() + .flat_map(|shared| { + shared.connections.iter().filter_map(|(id, subscriber)| (subscriber.addr == *addr).then_some(*id)) + }) + .collect(); + ProviderReleaseSnapshot { addr: *addr, single_allocations, shared_subscribers } + } + + fn has_connections_from_snapshot(&self, snapshot: &ProviderReleaseSnapshot) -> bool { + let _transition = self.lock_capacity_transition(); + let connections = self.read_connections(); + snapshot.single_allocations.iter().any(|id| { + connections.single.contains_key(id) + || connections.shared.shared_by_allocation_id.get(id).is_some_and(|key| { + connections.shared.by_key.get(key).is_some_and(|shared| { + connections.shared.key_by_subscriber.get(&shared.origin_subscriber_id) == Some(key) + }) + }) + }) || snapshot.shared_subscribers.iter().any(|id| connections.shared.key_by_subscriber.contains_key(id)) + } + + /// Waits for the kicked transport's original provider allocations to leave the registry. + /// The socket close may be signalled before its response bodies and provider handles are dropped. + pub(crate) async fn wait_for_snapshot_release( + &self, + snapshot: &ProviderReleaseSnapshot, + timeout: Duration, + ) -> bool { + let deadline = TokioInstant::now() + timeout; + loop { + if !self.has_connections_from_snapshot(snapshot) { + return true; + } + let now = TokioInstant::now(); + if now >= deadline { + return false; + } + tokio::time::sleep_until((now + EVICTED_PROVIDER_RELEASE_POLL_INTERVAL).min(deadline)).await; + } + } + + pub fn mark_closing(&self, alloc_id: AllocationId) { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + let prio = if let Some(info) = connections.single.get_mut(&alloc_id) { + info.lifecycle = ConnectionLifecycle::Closing; + info.cancel_token.cancel(); + info.allocation + .get_provider_name() + .map(|name| (name, (info.priority, Reverse(info.created_at), alloc_id), info.kind)) + } else { + None + }; + if let Some((name, key, kind)) = prio { + Self::remove_priority_entry(&mut connections, &name, &key, kind); + } + } + + pub fn release_connection(&self, addr: &SocketAddr) { + let _transition = self.lock_capacity_transition(); + let single_alloc_ids = { + let mut connections = self.write_connections(); + connections.single_by_addr.remove(addr) + }; + + if let Some(alloc_ids) = single_alloc_ids { + let mut to_reap = Vec::new(); + { + let mut connections = self.write_connections(); + for id in alloc_ids { + if let ReleaseAction::Wait(alloc_id, gen, token) = + ActiveProviderManagerCore::plan_single_release_locked(&mut connections, id) + { + to_reap.push((alloc_id, gen, token)); + } + } + } + for (id, gen, completion_token) in to_reap { + let core = Arc::clone(&self.core); + let shutdown_token = self.shutdown_token.clone(); + if let Ok(rt_handle) = tokio::runtime::Handle::try_current() { + rt_handle.spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = completion_token.cancelled() => { + core.complete_release_with_generation(id, Some(gen)); + } + } + }); + } + } + } + + // Shared connections carried by this transport socket. Resolve the subscriber + // ids first, then release each one precisely; two external clients behind one + // reverse proxy hold distinct subscriber ids and never overwrite each other. + for subscriber_id in self.release_shared_by_addr(addr) { + self.release_shared_connection_inner(subscriber_id); + } + } + + pub fn release_handle(&self, handle: &ProviderHandle) { + let _transition = self.lock_capacity_transition(); + let wait_action = { + let mut connections = self.write_connections(); + if connections.single.contains_key(&handle.allocation_id) { + if handle.completion_token.as_ref().is_some_and(CancellationToken::is_cancelled) { + if let Some(info) = connections.single.get_mut(&handle.allocation_id) { + if info.open_generation == handle.open_generation { + info.completion_token.cancel(); + } + } + } + ActiveProviderManagerCore::plan_single_release_locked(&mut connections, handle.allocation_id) + } else { + Self::release_shared_handle_locked(&mut connections, handle); + ReleaseAction::None + } + }; + + if let ReleaseAction::Wait(alloc_id, gen, completion_token) = wait_action { + let core = Arc::clone(&self.core); + let shutdown_token = self.shutdown_token.clone(); + if let Ok(rt_handle) = tokio::runtime::Handle::try_current() { + rt_handle.spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = completion_token.cancelled() => { + core.complete_release_with_generation(alloc_id, Some(gen)); + } + } + }); + } + } + } + + /// Stops only stale requests of this playback, even on a multiplexed proxy socket. + pub fn release_playback_connections(&self, owner: &str, addrs: &[SocketAddr]) { + let handles = { + let connections = self.read_connections(); + let mut handles = Vec::new(); + for addr in addrs { + if let Some(alloc_ids) = connections.single_by_addr.get(addr) { + for id in alloc_ids { + if let Some(info) = connections.single.get(id) { + if info.request_owner.as_deref() == Some(owner) { + handles.push(ProviderHandle { + playback_request_id: info.playback_request_id, + binding_tag: None, + client_id: info.client_addr, + allocation_id: info.allocation_id, + allocation: info.allocation.clone(), + cancel_token: Some(info.cancel_token.clone()), + completion_token: Some(info.completion_token.clone()), + close_reason: Arc::clone(&info.close_reason), + open_generation: info.open_generation, + }); + } + } + } + } + } + handles + }; + for handle in handles { + handle.set_close_reason(ProviderCloseReason::Superseded); + if let Some(token) = &handle.cancel_token { + token.cancel(); + } + self.release_handle(&handle); + } + } + + /// Stops stale requests of this playback and awaits upstream body closure before releasing capacity. + pub async fn release_playback_connections_await(&self, owner: &str, addrs: &[SocketAddr]) { + let targets = { + let mut connections = self.write_connections(); + let mut targets = Vec::new(); + for addr in addrs { + if let Some(alloc_ids) = connections.single_by_addr.get(addr).cloned() { + for id in alloc_ids { + if let Some(info) = connections.single.get_mut(&id) { + if info.request_owner.as_deref() == Some(owner) + && info.lifecycle != ConnectionLifecycle::Closed + { + info.lifecycle = ConnectionLifecycle::Closing; + info.close_reason.store(ProviderCloseReason::Superseded as u8, Ordering::Release); + targets.push(( + info.allocation_id, + info.cancel_token.clone(), + info.completion_token.clone(), + info.priority, + info.created_at, + info.kind, + info.allocation.get_provider_name(), + info.open_generation, + )); + } + } + } + } + } + for (alloc_id, _, _, priority, created_at, kind, provider_name, _) in &targets { + if let Some(name) = provider_name { + Self::remove_priority_entry( + &mut connections, + name, + &(*priority, Reverse(*created_at), *alloc_id), + *kind, + ); + } + } + targets + }; + + if targets.is_empty() { + return; + } + + for (_, cancel_token, _, _, _, _, _, _) in &targets { + cancel_token.cancel(); + } + + let futures: Vec<_> = targets.iter().map(|(_, _, token, _, _, _, _, _)| token.cancelled()).collect(); + let _ = tokio::time::timeout(Duration::from_secs(1), futures::future::join_all(futures)).await; + + for (alloc_id, _, completion_token, _, _, _, _, gen) in targets { + if completion_token.is_cancelled() { + self.core.complete_release_with_generation(alloc_id, Some(gen)); + } else { + log::warn!("Superseded provider connection {alloc_id} did not close within deadline, retaining slot as Closing until reaped"); + // Atomically claim the reaper role under the lock before spawning, so that + // if this future was dropped between collection and here, release_handle can + // still install its own reaper (reaper_spawned remains false until now). + let should_spawn = { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + connections.single.get_mut(&alloc_id).is_some_and(|info| { + if info.reaper_spawned { + false + } else { + info.reaper_spawned = true; + true + } + }) + }; + if should_spawn { + let core = Arc::clone(&self.core); + let shutdown_token = self.shutdown_token.clone(); + tokio::spawn(async move { + tokio::select! { + () = shutdown_token.cancelled() => {}, + () = completion_token.cancelled() => { + core.complete_release_with_generation(alloc_id, Some(gen)); + } + } + }); + } + } + } + } +} diff --git a/backend/session/src/active_provider_manager/shared.rs b/backend/session/src/active_provider_manager/shared.rs new file mode 100644 index 000000000..6a5a34948 --- /dev/null +++ b/backend/session/src/active_provider_manager/shared.rs @@ -0,0 +1,469 @@ +use super::{ActiveProviderManager, ConnectionKind, Connections, PriorityKey, PriorityOwner}; +use crate::SharedStreamManager; +use ::shared::utils::sanitize_sensitive_info; +use log::error; +use std::{ + cmp::Reverse, + collections::HashMap, + net::SocketAddr, + sync::{atomic::Ordering, Arc}, + time::Instant, +}; +use tokio_util::sync::CancellationToken; +use tuliprox_core::{ + model::{AllocationId, ProviderAllocation, ProviderHandle, SharedSubscriberId}, + utils::debug_if_enabled, +}; + +pub(super) type SharedConnectionId = AllocationId; + +#[derive(Debug, Clone)] +pub(super) struct SharedAllocation { + pub(super) allocation_id: AllocationId, + pub(super) origin_subscriber_id: SharedSubscriberId, + pub(super) allocation: ProviderAllocation, + /// Keyed by unique subscriber id, never by socket: two external clients behind one + /// reverse proxy must not collapse into a single entry. + pub(super) connections: HashMap, + pub(super) priority: i8, + pub(super) kind: ConnectionKind, + pub(super) created_at: Instant, + pub(super) cancel_token: Option, + pub(super) session_owner: Option>, +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct SharedSubscriber { + pub(super) priority: i8, + pub(super) kind: ConnectionKind, + /// Transport metadata and socket-wide close target only; never an identity key. + pub(super) addr: SocketAddr, +} + +#[derive(Debug, Clone, Default)] +pub(super) struct SharedConnections { + pub(super) by_key: HashMap, SharedAllocation>, + pub(super) key_by_subscriber: HashMap>, + pub(super) shared_by_allocation_id: HashMap>, +} + +impl ActiveProviderManager { + fn shared_effective_priority( + subscribers: &HashMap, + kind: ConnectionKind, + ) -> Option { + subscribers.values().filter(|subscriber| subscriber.kind == kind).map(|subscriber| subscriber.priority).min() + } + + fn shared_effective_kind(subscribers: &HashMap) -> ConnectionKind { + if subscribers.values().all(|subscriber| subscriber.kind == ConnectionKind::Soft) { + ConnectionKind::Soft + } else { + ConnectionKind::Normal + } + } + + pub fn set_shared_stream_manager(&self, manager: &Arc) { + let _ = self.shared_stream_manager.set(Arc::downgrade(manager)); + } + + /// Finds every shared subscription carried by one transport socket. + /// + /// This is an explicit socket-wide transport action (connection close, kick), not + /// the normal playback cleanup path. A single shared subscription is released + /// through `release_shared_subscriber`, which cannot touch other clients that + /// happen to arrive through the same reverse-proxy socket. + pub(super) fn release_shared_by_addr(&self, addr: &SocketAddr) -> Vec { + let connections = self.read_connections(); + connections + .shared + .by_key + .values() + .flat_map(|shared| { + shared.connections.iter().filter_map(|(id, subscriber)| (subscriber.addr == *addr).then_some(*id)) + }) + .collect() + } + + /// Releases exactly one shared subscription and rebalances the shared allocation. + /// + /// Returns the allocation to release when the last subscriber left. + fn release_shared_subscriber(&self, subscriber_id: SharedSubscriberId) -> Option { + let mut connections = self.write_connections(); + + let key = connections.shared.key_by_subscriber.remove(&subscriber_id)?; + let mut shared = connections.shared.by_key.remove(&key)?; + shared.connections.remove(&subscriber_id); + + let shared_is_unlimited = shared.allocation.is_unlimited_provider(); + + if shared.connections.is_empty() { + connections.shared.shared_by_allocation_id.remove(&shared.allocation_id); + Self::unindex_owner(&mut connections, shared.allocation_id, shared.session_owner.as_ref()); + if !shared_is_unlimited { + if let Some(name) = shared.allocation.get_provider_name() { + if let Some(list) = connections.by_provider.get_mut(&name) { + list.remove(&shared.allocation_id); + } + Self::remove_priority_entry( + &mut connections, + &name, + &(shared.priority, Reverse(shared.created_at), shared.allocation_id), + shared.kind, + ); + } + } + return Some(shared.allocation); + } + + // Recompute shared priority from remaining subscribers so preemption decisions + // reflect who is actually still watching the shared stream. Unlimited shared + // streams are not in the preemption index, so this rebalance is skipped. + if !shared_is_unlimited { + let old_priority = shared.priority; + let old_kind = shared.kind; + shared.kind = Self::shared_effective_kind(&shared.connections); + if let Some(new_priority) = Self::shared_effective_priority(&shared.connections, shared.kind) { + shared.priority = new_priority; + if (new_priority, shared.kind) != (old_priority, old_kind) { + if let Some(name) = shared.allocation.get_provider_name() { + Self::remove_priority_entry( + &mut connections, + &name, + &(old_priority, Reverse(shared.created_at), shared.allocation_id), + old_kind, + ); + Self::upsert_priority_entry( + &mut connections, + &name, + (new_priority, Reverse(shared.created_at), shared.allocation_id), + PriorityOwner::Shared(shared.allocation_id), + shared.kind, + ); + } + } + } + } + connections.shared.by_key.insert(key, shared); + None + } + + pub fn release_shared_connection(&self, subscriber_id: SharedSubscriberId) { + let _transition = self.lock_capacity_transition(); + self.release_shared_connection_inner(subscriber_id); + } + + pub(super) fn release_shared_connection_inner(&self, subscriber_id: SharedSubscriberId) { + if let Some(allocation) = self.release_shared_subscriber(subscriber_id) { + debug_if_enabled!( + "Released last shared connection for provider {} (subscriber={subscriber_id})", + allocation.get_provider_name().unwrap_or_default() + ); + allocation.release(); + if let Some(name) = allocation.get_provider_name() { + self.capacity.notify(&name); + } + } + } + + pub(super) fn release_shared_handle_locked( + connections: &mut Connections, + handle: &ProviderHandle, + ) -> Option { + let mut released = None; + let mut released_priority_key: Option<(Arc, PriorityKey, ConnectionKind)> = None; + if let Some(key) = connections.shared.shared_by_allocation_id.remove(&handle.allocation_id) { + if let Some(shared) = connections.shared.by_key.remove(&key) { + Self::unindex_owner(connections, handle.allocation_id, shared.session_owner.as_ref()); + let pkey = (shared.priority, Reverse(shared.created_at), handle.allocation_id); + let shared_kind = shared.kind; + let shared_is_unlimited = shared.allocation.is_unlimited_provider(); + released = Some(shared.allocation); + for subscriber_id in shared.connections.keys() { + connections.shared.key_by_subscriber.remove(subscriber_id); + } + if !shared_is_unlimited { + if let Some(name) = released.as_ref().and_then(ProviderAllocation::get_provider_name) { + if let Some(list) = connections.by_provider.get_mut(&name) { + list.remove(&handle.allocation_id); + } + released_priority_key = Some((name, pkey, shared_kind)); + } + } + } + } + if let Some((name, pkey, kind)) = &released_priority_key { + Self::remove_priority_entry(connections, name, pkey, *kind); + } + if let Some(allocation) = released.as_ref() { + allocation.release(); + if let Some(name) = allocation.get_provider_name() { + connections.released_providers.push(name); + } + } + released + } + + pub fn reclassify_connection(&self, addr: &SocketAddr, kind: ConnectionKind, priority: i8) -> bool { + self.reclassify_connection_for_owner(addr, None, kind, priority) + } + + pub fn reclassify_connection_for_owner( + &self, + addr: &SocketAddr, + owner: Option<&str>, + kind: ConnectionKind, + priority: i8, + ) -> bool { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + + let Some(alloc_ids) = connections.single_by_addr.get(addr).cloned() else { + return false; + }; + let mut index_updates = Vec::new(); + for allocation_id in alloc_ids { + if let Some(info) = connections.single.get_mut(&allocation_id) { + if info.request_owner.as_deref() != owner { + continue; + } + if let Some(provider_name) = info.allocation.get_provider_name() { + let old_key = (info.priority, Reverse(info.created_at), allocation_id); + let p_owner = PriorityOwner::Single(allocation_id); + let old_kind = info.kind; + info.kind = kind; + info.priority = priority; + let new_key = (info.priority, Reverse(info.created_at), allocation_id); + // Skip preemption-index updates for unlimited providers: they + // are intentionally absent from `by_provider` / `priority_index` + // / `soft_priority_index`, so the old entry was never inserted. + // Performing the upsert here would re-introduce them into the + // preemption index and contradict `max_connections: 0`. + if !info.allocation.is_unlimited_provider() { + index_updates.push((provider_name, old_key, old_kind, new_key, p_owner, info.kind)); + } + } + } + } + for (provider_name, old_key, old_kind, new_key, p_owner, new_kind) in index_updates { + Self::remove_priority_entry(&mut connections, &provider_name, &old_key, old_kind); + Self::upsert_priority_entry(&mut connections, &provider_name, new_key, p_owner, new_kind); + } + true + } + + pub fn reclassify_shared_connection( + &self, + subscriber_id: SharedSubscriberId, + kind: ConnectionKind, + priority: i8, + ) -> bool { + let _transition = self.lock_capacity_transition(); + let mut connections = self.write_connections(); + let shared_key = connections.shared.key_by_subscriber.get(&subscriber_id).cloned(); + let Some(shared_key) = shared_key else { + return false; + }; + let Some(shared_allocation) = connections.shared.by_key.get_mut(&shared_key) else { + return false; + }; + let shared_is_unlimited = shared_allocation.allocation.is_unlimited_provider(); + + let old_priority = shared_allocation.priority; + let old_kind = shared_allocation.kind; + let Some(subscriber) = shared_allocation.connections.get_mut(&subscriber_id) else { + return false; + }; + subscriber.kind = kind; + subscriber.priority = priority; + shared_allocation.kind = Self::shared_effective_kind(&shared_allocation.connections); + if let Some(new_priority) = + Self::shared_effective_priority(&shared_allocation.connections, shared_allocation.kind) + { + shared_allocation.priority = new_priority; + } + let new_priority = shared_allocation.priority; + let new_kind = shared_allocation.kind; + let allocation_id = shared_allocation.allocation_id; + let created_at = shared_allocation.created_at; + let provider_name = shared_allocation.allocation.get_provider_name(); + let _ = shared_allocation; + + // Mirror the single-connection rule: skip priority-index rebuild for + // shared allocations backed by an unlimited provider. The kind/priority + // transition on the per-subscriber state is still applied above. + if !shared_is_unlimited { + if let Some(provider_name) = provider_name { + let owner = PriorityOwner::Shared(allocation_id); + Self::remove_priority_entry( + &mut connections, + &provider_name, + &(old_priority, Reverse(created_at), allocation_id), + old_kind, + ); + Self::upsert_priority_entry( + &mut connections, + &provider_name, + (new_priority, Reverse(created_at), allocation_id), + owner, + new_kind, + ); + } + } + + true + } + + /// Promotes exactly the allocation owned by this handle into a shared origin. + pub fn make_shared_connection( + &self, + handle: &ProviderHandle, + key: &str, + subscriber_id: SharedSubscriberId, + ) -> bool { + if self.is_shutting_down.load(Ordering::Acquire) { + return false; + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return false; + } + let mut connections = self.write_connections(); + if connections.shared.by_key.contains_key(key) { + return false; + } + let Some(info) = connections.single.remove(&handle.allocation_id) else { + return false; + }; + if let Some(set) = connections.single_by_addr.get_mut(&info.client_addr) { + set.remove(&handle.allocation_id); + if set.is_empty() { + connections.single_by_addr.remove(&info.client_addr); + } + } + let provider_name = info.allocation.get_provider_name().unwrap_or_default(); + if !info.allocation.is_unlimited_provider() { + if let Some(list) = connections.by_provider.get_mut(&provider_name) { + list.remove(&handle.allocation_id); + } + Self::remove_priority_entry( + &mut connections, + &provider_name, + &(info.priority, Reverse(info.created_at), handle.allocation_id), + info.kind, + ); + Self::upsert_priority_entry( + &mut connections, + &provider_name, + (info.priority, Reverse(info.created_at), handle.allocation_id), + PriorityOwner::Shared(handle.allocation_id), + info.kind, + ); + } + let shared_key: Arc = Arc::from(key); + connections.shared.by_key.insert( + Arc::clone(&shared_key), + SharedAllocation { + allocation_id: handle.allocation_id, + origin_subscriber_id: subscriber_id, + allocation: info.allocation, + connections: HashMap::from([( + subscriber_id, + SharedSubscriber { priority: info.priority, kind: info.kind, addr: handle.client_id }, + )]), + priority: info.priority, + kind: info.kind, + created_at: info.created_at, + cancel_token: Some(info.cancel_token), + session_owner: info.session_owner, + }, + ); + connections.shared.key_by_subscriber.insert(subscriber_id, Arc::clone(&shared_key)); + connections.shared.shared_by_allocation_id.insert(handle.allocation_id, shared_key); + true + } + + pub fn add_shared_connection( + &self, + addr: &SocketAddr, + subscriber_id: SharedSubscriberId, + key: &str, + priority: i8, + kind: ConnectionKind, + ) -> Result<(), String> { + if self.is_shutting_down.load(Ordering::Acquire) { + return Err("Provider manager is shutting down".to_string()); + } + let _transition = self.lock_capacity_transition(); + if self.is_shutting_down.load(Ordering::Acquire) { + return Err("Provider manager is shutting down".to_string()); + } + let mut connections = self.write_connections(); + + // Extract metadata before taking a second mutable borrow on `connections`. + let metadata = connections.shared.by_key.get(key).map(|s| { + ( + s.allocation_id, + s.allocation.get_provider_name().unwrap_or_default(), + s.priority, + s.kind, + s.created_at, + s.allocation.is_unlimited_provider(), + ) + }); + + let Some((alloc_id, provider_name, old_priority, old_kind, created_at, shared_is_unlimited)) = metadata else { + let err = + format!("Failed to add shared connection for {addr}: url {} not found", sanitize_sensitive_info(key)); + error!("{err}"); + return Err(err); + }; + + debug_if_enabled!( + "Shared connection: added addr {addr} provider={} key={}", + sanitize_sensitive_info(&provider_name), + sanitize_sensitive_info(key) + ); + + let Some(shared_allocation) = connections.shared.by_key.get_mut(key) else { + let err = format!( + "Failed to add shared connection for {addr}: url {} disappeared during update", + sanitize_sensitive_info(key) + ); + error!("{err}"); + return Err(err); + }; + + shared_allocation.connections.insert(subscriber_id, SharedSubscriber { priority, kind, addr: *addr }); + let new_kind = Self::shared_effective_kind(&shared_allocation.connections); + let new_priority = Self::shared_effective_priority(&shared_allocation.connections, new_kind); + shared_allocation.kind = new_kind; + if let Some(new_priority) = new_priority { + shared_allocation.priority = new_priority; + } + let updated_kind = shared_allocation.kind; + let updated_priority = shared_allocation.priority; + let needs_reindex = (updated_priority, updated_kind) != (old_priority, old_kind); + let _ = shared_allocation; + // Skip priority-index rebuild for unlimited providers: they are not + // subject to preemption and therefore must not be in the index. + if !shared_is_unlimited && needs_reindex { + Self::remove_priority_entry( + &mut connections, + &provider_name, + &(old_priority, Reverse(created_at), alloc_id), + old_kind, + ); + Self::upsert_priority_entry( + &mut connections, + &provider_name, + (updated_priority, Reverse(created_at), alloc_id), + PriorityOwner::Shared(alloc_id), + updated_kind, + ); + } + + connections.shared.key_by_subscriber.insert(subscriber_id, Arc::from(key)); + Ok(()) + } +} diff --git a/backend/session/src/active_provider_manager/tests.rs b/backend/session/src/active_provider_manager/tests.rs new file mode 100644 index 000000000..2b030caa3 --- /dev/null +++ b/backend/session/src/active_provider_manager/tests.rs @@ -0,0 +1,15 @@ +use super::{ActiveProviderManager, ConnectionKind, PlaybackLeaseRef}; + +mod admission; +mod behavior; +mod lifecycle; +mod retry; +mod support; +mod terminal; + +use self::support::{ + acquire_live_hls_from_lineup, alias_pool, build_test_app_config, confirmed_alias_request, + create_test_app_config_single_provider_pool, create_test_app_config_single_unlimited_provider_pool, + create_test_app_config_with_dual_provider_pool, create_test_app_config_with_pool, + live_hls_playback_on_alias_with_lapsed_lease, +}; diff --git a/backend/session/src/active_provider_manager/tests/admission.rs b/backend/session/src/active_provider_manager/tests/admission.rs new file mode 100644 index 000000000..390b83323 --- /dev/null +++ b/backend/session/src/active_provider_manager/tests/admission.rs @@ -0,0 +1,1141 @@ +use super::{ + alias_pool, build_test_app_config, create_test_app_config_single_provider_pool, + create_test_app_config_single_unlimited_provider_pool, create_test_app_config_with_dual_provider_pool, + create_test_app_config_with_pool, live_hls_playback_on_alias_with_lapsed_lease, ActiveProviderManager, + ConnectionKind, PlaybackLeaseRef, +}; +use crate::EventManager; +use shared::{defaults::default_user_priority, utils::Internable}; +use std::{collections::HashSet, net::SocketAddr, sync::Arc, time::Duration}; +use tuliprox_core::model::{ + AppConfig, ConfigInputAlias, PlaybackKind, PlaybackRequestOutcome, ProviderAllocation, SharedSubscriberId, +}; + +pub(in crate::active_provider_manager::tests) fn create_test_app_config_with_capacity_ordered_pool() -> AppConfig { + build_test_app_config( + Some(vec![ConfigInputAlias { + id: 2, + name: "provider_2".intern(), + url: "http://provider-2.example".to_string(), + username: Some("user2".to_string()), + password: Some("pass2".to_string()), + priority: 1, + max_connections: 1, + exp_date: None, + enabled: true, + stalker: None, + }]), + 3, + ) +} + +#[tokio::test] +async fn confirmed_playbacks_fill_higher_priority_provider_before_alias() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + + let mut selected = Vec::new(); + for index in 0..5 { + let owner = format!("playback-{index}"); + let addr = SocketAddr::from(([172, 18, 0, 9], 50_000 + index)); + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(&owner), + ) + .expect("pool has capacity for five confirmed playbacks"); + let provider = handle.allocation.get_provider_name().expect("provider name"); + // Only real media activity confirms a playback and lets it reserve capacity. + manager.confirm_playback_activity(&owner); + selected.push(provider.to_string()); + } + + assert_eq!(selected, ["provider_1", "provider_1", "provider_2", "provider_2", "provider_2"]); +} + +#[tokio::test] +async fn concurrent_proxy_requests_fill_exact_pool_capacity() -> Result<(), Box> { + let manager = + Arc::new(ActiveProviderManager::new(&create_test_app_config_with_pool(2, 3), &Arc::new(EventManager::new()))); + let addr = SocketAddr::from(([127, 0, 0, 1], 50001)); + let mut tasks = tokio::task::JoinSet::new(); + for attempt in 0..32 { + let manager = Arc::clone(&manager); + tasks.spawn(async move { + manager.acquire_connection_with_grace_for_session( + &Arc::from("provider_1"), + &addr, + false, + 0, + ConnectionKind::Normal, + Some(&format!("playback-{attempt}")), + ) + }); + } + let mut handles = Vec::new(); + while let Some(result) = tasks.join_next().await { + if let Some(handle) = result? { + handles.push(handle); + } + } + assert_eq!(handles.len(), 5); + assert_eq!(handles.iter().filter(|h| h.allocation.get_provider_name().as_deref() == Some("provider_1")).count(), 2); + assert_eq!(handles.iter().filter(|h| h.allocation.get_provider_name().as_deref() == Some("provider_2")).count(), 3); + for handle in handles { + manager.release_handle(&handle); + } + assert_eq!(manager.get_provider_connections_count(), 0); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn manifest_retries_without_media_do_not_reserve_capacity() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "hls-cache:shared-session"; + let addr = SocketAddr::from(([172, 18, 0, 9], 51_000)); + + // Five manifest responses without a single media segment, all on one owner. + for _ in 0..5 { + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner), + ) + .expect("manifest start allocates on the preferred provider"); + assert_eq!(handle.allocation.get_provider_name().as_deref(), Some("provider_1")); + manager.refresh_provider_reservation(&input_name, owner, 15); + manager.release_connection(&addr); + } + + // Unconfirmed leases never reserve capacity, so an unrelated client is served + // by the higher-priority provider even though its own counters read zero. + let other = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 51_001)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some("other-client"), + ) + .expect("unrelated client must still get the preferred provider"); + assert_eq!(other.allocation.get_provider_name().as_deref(), Some("provider_1")); + + // The abandoned starts stop holding anything once their startup deadline passes. + tokio::time::advance(Duration::from_secs(6)).await; + manager.prune_expired_leases_now(); + assert!(!manager.is_provider_reserved_for_other_session(&input_name, Some("other-client"))); +} + +#[tokio::test(start_paused = true)] +async fn identified_reservation_recreates_a_cleared_lease() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "hls-cache:restore-session"; + let addr = SocketAddr::from(([172, 18, 0, 9], 53_000)); + + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner), + ) + .expect("allocate"); + manager.refresh_identified_provider_reservation( + &input_name, + owner, + tuliprox_core::model::PlaybackKind::Catchup, + 15, + ); + assert_eq!(manager.provider_lease_usage(&input_name).starting, 1); + + // Clearing removes the lease entirely; recreating must bring it back rather + // than silently no-op like a plain adaptive refresh would. + let binding_tag = manager.binding_tag_for_owner(owner); + manager.clear_identified_provider_reservation(owner, &input_name, binding_tag); + assert_eq!(manager.provider_lease_usage(&input_name).total(), 0); + + manager.refresh_identified_provider_reservation( + &input_name, + owner, + tuliprox_core::model::PlaybackKind::Catchup, + 15, + ); + assert_eq!(manager.provider_lease_usage(&input_name).starting, 1); + + manager.release_handle(&handle); +} + +#[tokio::test(start_paused = true)] +async fn reservations_do_not_survive_after_counters_reach_zero() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "finished-playback"; + let addr = SocketAddr::from(([172, 18, 0, 9], 52_000)); + + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner), + ) + .expect("first playback acquires the preferred provider"); + assert_eq!(handle.allocation.get_provider_name().as_deref(), Some("provider_1")); + manager.refresh_provider_reservation(&input_name, owner, 15); + manager.confirm_playback_activity(owner); + manager.release_connection(&addr); + + // Live TS is not reconnect capable: the confirmed lease is dropped on release, + // so the provider is immediately free again for the next higher-priority client. + manager.finish_playback_request(owner, PlaybackRequestOutcome::Completed, None); + let next = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 52_001)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some("next-client"), + ) + .expect("freed capacity must be selectable again"); + assert_eq!(next.allocation.get_provider_name().as_deref(), Some("provider_1")); +} + +#[tokio::test] +async fn unlimited_provider_connection_is_not_in_priority_index() { + let app_cfg = create_test_app_config_single_unlimited_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let addr_1: SocketAddr = "127.0.0.1:44001".parse().unwrap(); + let addr_2: SocketAddr = "127.0.0.1:44002".parse().unwrap(); + let addr_3: SocketAddr = "127.0.0.1:44003".parse().unwrap(); + + manager + .acquire_connection(&input_name, &addr_1, default_user_priority(), ConnectionKind::Normal) + .expect("acquire #1 on unlimited provider"); + manager + .acquire_connection(&input_name, &addr_2, default_user_priority(), ConnectionKind::Normal) + .expect("acquire #2 on unlimited provider"); + manager + .acquire_connection(&input_name, &addr_3, default_user_priority(), ConnectionKind::Normal) + .expect("acquire #3 on unlimited provider"); + + { + let connections = manager.read_connections(); + let priority_tree = connections.priority_index.get(&input_name); + assert!( + priority_tree.is_none_or(std::collections::BTreeMap::is_empty), + "unlimited provider must not be present in priority_index, found {priority_tree:?}" + ); + let soft_tree = connections.soft_priority_index.get(&input_name); + assert!( + soft_tree.is_none_or(std::collections::BTreeMap::is_empty), + "unlimited provider must not be present in soft_priority_index, found {soft_tree:?}" + ); + let by_provider = connections.by_provider.get(&input_name); + assert!( + by_provider.is_none_or(std::collections::HashSet::is_empty), + "unlimited provider must not be present in by_provider, found {by_provider:?}" + ); + } + + manager.release_connection(&addr_1); + manager.release_connection(&addr_2); + manager.release_connection(&addr_3); +} + +#[tokio::test] +async fn preemption_does_not_select_unlimited_provider_as_victim() { + let app_cfg = create_test_app_config_single_unlimited_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let low_addr: SocketAddr = "127.0.0.1:45001".parse().unwrap(); + let high_addr: SocketAddr = "127.0.0.1:45002".parse().unwrap(); + + // Acquire one low-priority and one high-priority connection on the unlimited provider. + let low = manager + .acquire_connection(&input_name, &low_addr, 50, ConnectionKind::Normal) + .expect("low-priority acquire on unlimited provider"); + let low_token = low.cancel_token.clone().expect("cancel token for low-priority connection"); + let _high = manager + .acquire_connection(&input_name, &high_addr, 0, ConnectionKind::Normal) + .expect("high-priority acquire on unlimited provider"); + + // A request at an even higher priority must not be able to preempt the unlimited + // provider's connection, because the connection is intentionally absent from the + // preemption indices. + let candidate = { + let connections = manager.read_connections(); + manager.select_preemption_candidate(&connections, &input_name, 0, ConnectionKind::Normal, &HashSet::new()) + }; + assert!( + candidate.is_none(), + "select_preemption_candidate must not return an unlimited-provider connection as a victim, got {candidate:?}" + ); + + // The low-priority connection is still alive and its cancel token has not been fired. + assert!( + !low_token.is_cancelled(), + "low-priority unlimited-provider connection must not be cancelled by preemption" + ); + + manager.release_connection(&low_addr); + manager.release_connection(&high_addr); +} + +#[tokio::test(start_paused = true)] +async fn untagged_clear_cannot_delete_successor_reservation() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "session-owner"; + let addr: SocketAddr = "127.0.0.1:43320".parse().unwrap(); + + // First incarnation acquires, then is removed entirely. + let first = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner), + ) + .expect("first acquire"); + manager.release_handle(&first); + manager.clear_provider_reservation(owner); + + // Second incarnation acquires a fresh lease on the same account. + let second = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner), + ) + .expect("second acquire"); + + // An untagged clear has no delete right and must not remove the successor. + manager.clear_identified_provider_reservation(owner, &input_name, None); + assert!( + manager.binding_tag_for_owner(owner).is_some(), + "an untagged clear must not delete the successor reservation" + ); + + manager.release_handle(&second); + manager.clear_provider_reservation(owner); +} + +#[tokio::test(start_paused = true)] +async fn session_reservations_preserve_capacity_and_priority_order() { + let app_cfg = create_test_app_config_with_capacity_ordered_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + + for index in 0..3 { + let owner = format!("session-owner-{index}"); + let addr = SocketAddr::from(([127, 0, 0, 1], 43200 + index)); + let allocation = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(&owner), + ) + .expect("preferred provider should retain free capacity for another session"); + assert_eq!(allocation.allocation.get_provider_name().as_deref(), Some("provider_1")); + manager.refresh_provider_reservation(&input_name, &owner, 15); + manager.confirm_playback_activity(&owner); + } + + let fallback_addr = SocketAddr::from(([127, 0, 0, 1], 43203)); + let fallback = manager + .acquire_connection_with_grace_for_session( + &input_name, + &fallback_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some("session-owner-3"), + ) + .expect("lower-priority provider should be used after preferred capacity is exhausted"); + assert_eq!(fallback.allocation.get_provider_name().as_deref(), Some("provider_2")); + + manager.release_connection(&SocketAddr::from(([127, 0, 0, 1], 43200))); + manager.release_connection(&fallback_addr); + + let foreign_after_release = manager + .acquire_connection_with_grace_for_session( + &input_name, + &fallback_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some("session-owner-4"), + ) + .expect("idle reservation should protect one preferred-provider slot"); + assert_eq!(foreign_after_release.allocation.get_provider_name().as_deref(), Some("provider_2")); + + let reserved_owner_addr = SocketAddr::from(([127, 0, 0, 1], 43204)); + let reserved_owner = manager + .acquire_connection_with_grace_for_session( + &input_name, + &reserved_owner_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some("session-owner-0"), + ) + .expect("reservation owner should reclaim its preferred-provider slot"); + assert_eq!(reserved_owner.allocation.get_provider_name().as_deref(), Some("provider_1")); + + for index in 1..5 { + let addr = SocketAddr::from(([127, 0, 0, 1], 43200 + index)); + manager.release_connection(&addr); + } +} + +#[tokio::test(start_paused = true)] +async fn test_unlimited_provider_reservation_does_not_block_other_sessions() { + let app_cfg = create_test_app_config_single_unlimited_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let owner_1 = "session-owner-1"; + let owner_2 = "session-owner-2"; + let addr_1: SocketAddr = "127.0.0.1:43121".parse().unwrap(); + let addr_2: SocketAddr = "127.0.0.1:43122".parse().unwrap(); + + manager.refresh_provider_reservation(&input_name, owner_1, 15); + + let first = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr_1, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_1), + ) + .expect("reserved owner should reacquire its unlimited provider"); + assert_eq!(first.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + + let second = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr_2, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_2), + ) + .expect("other sessions should not be blocked by reservations on unlimited providers"); + assert_eq!(second.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + + manager.release_connection(&addr_1); + manager.release_connection(&addr_2); +} + +#[tokio::test(start_paused = true)] +async fn concurrent_same_family_playbacks_keep_independent_provider_reservations() { + let app_cfg = create_test_app_config_with_dual_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let provider_2 = "provider_2".intern(); + let Ok(first_addr) = "127.0.0.1:43121".parse() else { + return; + }; + let Ok(second_addr) = "127.0.0.1:43122".parse() else { + return; + }; + let owner_channel_1 = "proxy|player|user|100"; + let owner_channel_2 = "proxy|player|user|200"; + + let first = manager.acquire_connection_with_grace_for_session( + &input_name, + &first_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_channel_1), + ); + assert!(first.is_some(), "first playback should acquire the preferred provider"); + let Some(first) = first else { + return; + }; + assert_eq!(first.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + manager.refresh_provider_reservation(&input_name, owner_channel_1, 15); + manager.confirm_playback_activity(owner_channel_1); + manager.release_connection(&first_addr); + + let second = manager.acquire_connection_with_grace_for_session( + &input_name, + &second_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_channel_2), + ); + assert!(second.is_some(), "parallel playback should acquire the remaining provider"); + let Some(second) = second else { + return; + }; + assert_eq!(second.allocation.get_provider_name().as_deref(), Some(provider_2.as_ref())); + manager.refresh_provider_reservation(&provider_2, owner_channel_2, 15); + manager.confirm_playback_activity(owner_channel_2); + + let leases = manager.read_leases(); + assert_eq!(leases.provider_for_owner(owner_channel_1).as_deref(), Some(input_name.as_ref())); + assert_eq!(leases.provider_for_owner(owner_channel_2).as_deref(), Some(provider_2.as_ref())); + drop(leases); + + manager.clear_provider_reservation(owner_channel_2); + let leases = manager.read_leases(); + assert_eq!(leases.provider_for_owner(owner_channel_1).as_deref(), Some(input_name.as_ref())); + assert!(leases.lease_of_owner(owner_channel_2).is_none()); + drop(leases); + + manager.release_connection(&second_addr); +} + +#[tokio::test] +async fn test_higher_priority_user_preempts_lower_priority_user() { + // User with priority 5 (low) is connected; user with priority -1 (high) arrives. + // The low-priority user should be preempted. + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let low_prio_addr: SocketAddr = "127.0.0.1:44001".parse().unwrap(); + let high_prio_addr: SocketAddr = "127.0.0.1:44002".parse().unwrap(); + + // Low-priority user connects (priority 5 = lower importance) + let low_alloc = manager + .acquire_connection(&input_name, &low_prio_addr, 5, ConnectionKind::Normal) + .expect("low-priority user should get connection"); + assert_eq!(low_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + + // Provider is now exhausted + assert!(manager.is_exhausted(&input_name)); + + // High-priority user arrives (priority -1 = higher importance), should preempt low-priority user + let high_alloc = manager + .acquire_connection_with_grace(&input_name, &high_prio_addr, false, -1, ConnectionKind::Normal) + .expect("high-priority user should preempt low-priority user and get connection"); + assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + + manager.release_connection(&high_prio_addr); +} + +#[tokio::test] +async fn test_same_priority_user_does_not_preempt() { + // Two users with the same priority — new one should NOT preempt the existing one. + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let user_1_addr: SocketAddr = "127.0.0.1:45001".parse().unwrap(); + let user_2_addr: SocketAddr = "127.0.0.1:45002".parse().unwrap(); + + // User 1 connects with priority 0 + let alloc1 = manager + .acquire_connection(&input_name, &user_1_addr, default_user_priority(), ConnectionKind::Normal) + .expect("user1 should get connection"); + assert_eq!(alloc1.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + + // Provider is now exhausted + assert!(manager.is_exhausted(&input_name)); + + // User 2 arrives with the same priority 0 — should NOT preempt user 1 + let alloc2 = manager.acquire_connection_with_grace( + &input_name, + &user_2_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + ); + assert!(alloc2.is_none(), "same-priority user should not preempt existing user"); + + manager.release_connection(&user_1_addr); +} + +#[tokio::test] +async fn test_lower_priority_user_does_not_preempt_higher_priority_user() { + // User with high priority is connected; user with low priority arrives — should NOT preempt. + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let high_prio_addr: SocketAddr = "127.0.0.1:46001".parse().unwrap(); + let low_prio_addr: SocketAddr = "127.0.0.1:46002".parse().unwrap(); + + // High-priority user connects (priority -10) + let alloc1 = manager + .acquire_connection(&input_name, &high_prio_addr, -10, ConnectionKind::Normal) + .expect("high-priority user should get connection"); + assert_eq!(alloc1.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + + // Provider is now exhausted + assert!(manager.is_exhausted(&input_name)); + + // Low-priority user arrives (priority 10) — should NOT preempt high-priority user + let alloc2 = manager.acquire_connection_with_grace(&input_name, &low_prio_addr, false, 10, ConnectionKind::Normal); + assert!(alloc2.is_none(), "low-priority user should not preempt high-priority user"); + + manager.release_connection(&high_prio_addr); +} + +#[tokio::test] +async fn test_grace_period_triggers_preemption_of_lower_priority() { + // Provider full, high-prio user arrives with grace allowed, + // low-prio victim should be evicted and provider should not be over limit. + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let low_prio_addr: SocketAddr = "127.0.0.1:47001".parse().unwrap(); + let high_prio_addr: SocketAddr = "127.0.0.1:47002".parse().unwrap(); + + // Low-priority user connects (priority 20 = low importance) + let low_alloc = manager + .acquire_connection(&input_name, &low_prio_addr, 20, ConnectionKind::Normal) + .expect("low-priority user should get connection"); + assert_eq!(low_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + let low_token = low_alloc.cancel_token.clone().expect("must have cancel token"); + + // Provider is now exhausted + assert!(manager.is_exhausted(&input_name)); + + // High-priority user arrives WITH grace allowed (default streaming path) + // This should get a GracePeriod allocation and then evict the low-prio user + let high_alloc = manager + .acquire_connection(&input_name, &high_prio_addr, 0, ConnectionKind::Normal) + .expect("high-priority user should get grace allocation and evict low-prio"); + assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + + // Low-priority user's cancel token should be cancelled + assert!(low_token.is_cancelled(), "low-prio user should be cancelled after eviction"); + + // Provider should not be over limit (eviction freed a slot) + assert!(!manager.is_over_limit(&input_name), "provider should not be over limit after eviction"); + + manager.release_connection(&high_prio_addr); +} + +#[tokio::test] +async fn test_equal_priority_user_gets_grace_without_preempting_existing_stream() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let user_1_addr: SocketAddr = "127.0.0.1:48001".parse().unwrap(); + let user_2_addr: SocketAddr = "127.0.0.1:48002".parse().unwrap(); + + // User 1 connects with priority 0 + let alloc1 = manager + .acquire_connection(&input_name, &user_1_addr, default_user_priority(), ConnectionKind::Normal) + .expect("user1 should get connection"); + assert_eq!(alloc1.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + let token1 = alloc1.cancel_token.clone().expect("must have cancel token"); + + // Provider is now exhausted + assert!(manager.is_exhausted(&input_name)); + + // User 2 arrives with the same priority and should be granted grace instead of being rejected. + let alloc2 = manager + .acquire_connection(&input_name, &user_2_addr, default_user_priority(), ConnectionKind::Normal) + .expect("same-priority user should get grace allocation"); + assert!(matches!(alloc2.allocation, ProviderAllocation::GracePeriod(_))); + assert_eq!(alloc2.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + + // User 1 should NOT be cancelled, and the provider should be temporarily over limit. + assert!(!token1.is_cancelled(), "same-prio user should not be evicted"); + assert!(manager.is_over_limit(&input_name), "provider should be temporarily over limit during grace"); + + manager.release_connection(&user_1_addr); + manager.release_connection(&user_2_addr); +} + +#[tokio::test] +async fn test_higher_priority_user_preempts_first_inserted_low_priority_victim_on_exact_tie() { + let app_cfg = create_test_app_config_with_dual_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let old_low_addr: SocketAddr = "127.0.0.1:48101".parse().unwrap(); + let new_low_addr: SocketAddr = "127.0.0.1:48102".parse().unwrap(); + let high_prio_addr: SocketAddr = "127.0.0.1:48103".parse().unwrap(); + + // Place two equal-priority low-priority victims on different provider aliases. + // The test then normalizes their created_at timestamps to an exact tie so the + // selector must fall back to the final stable tie-break instead of clock order. + let old_alloc = manager + .acquire_exact_connection_with_grace(&"provider_2".intern(), &old_low_addr, false, 20, ConnectionKind::Normal) + .expect("old low-priority allocation should succeed"); + let old_token = old_alloc.cancel_token.clone().expect("old allocation should have cancel token"); + assert_eq!(old_alloc.allocation.get_provider_name().as_deref(), Some("provider_2")); + + let new_alloc = manager + .acquire_exact_connection_with_grace(&"provider_1".intern(), &new_low_addr, false, 20, ConnectionKind::Normal) + .expect("new low-priority allocation should succeed"); + let new_token = new_alloc.cancel_token.clone().expect("new allocation should have cancel token"); + assert_eq!(new_alloc.allocation.get_provider_name().as_deref(), Some("provider_1")); + + { + let mut connections = manager.write_connections(); + let old_created_at = connections + .single + .get(&old_alloc.allocation_id) + .map(|info| info.created_at) + .expect("old allocation should still be registered"); + + let (new_created_at, new_priority) = { + let info = connections + .single + .get_mut(&new_alloc.allocation_id) + .expect("new allocation should still be registered"); + let original_created_at = info.created_at; + info.created_at = old_created_at; + (original_created_at, info.priority) + }; + + let provider_name = "provider_1".intern(); + let tree = + connections.priority_index.get_mut(&provider_name).expect("priority index for provider_1 should exist"); + let owner = tree + .remove(&(new_priority, std::cmp::Reverse(new_created_at), new_alloc.allocation_id)) + .expect("new allocation should still be indexed"); + tree.insert((new_priority, std::cmp::Reverse(old_created_at), new_alloc.allocation_id), owner); + } + + assert!(manager.is_exhausted(&input_name)); + + // Higher-priority request should now select the first-inserted victim because + // priority and created_at are exactly tied across provider aliases. + let high_alloc = manager + .acquire_connection(&input_name, &high_prio_addr, 0, ConnectionKind::Normal) + .expect("higher-priority request should preempt the first-inserted low-priority victim on exact tie"); + assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some("provider_2")); + + assert!(old_token.is_cancelled(), "first-inserted low-priority victim should be canceled first on exact tie"); + assert!(!new_token.is_cancelled(), "later allocation should remain active after the stable tie-break"); + + manager.release_connection(&high_prio_addr); + manager.release_connection(&old_low_addr); + manager.release_connection(&new_low_addr); +} + +#[tokio::test] +async fn test_shared_priority_downgrades_after_high_priority_user_leaves() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let stream_key = "http://example.com/shared/live"; + let addr_a: SocketAddr = "127.0.0.1:48501".parse().unwrap(); + let addr_b: SocketAddr = "127.0.0.1:48502".parse().unwrap(); + + // A starts shared stream with high importance (priority 0). + let alloc_a = manager + .acquire_connection(&input_name, &addr_a, 0, ConnectionKind::Normal) + .expect("A should get initial connection"); + let shared_token = alloc_a.cancel_token.clone().expect("shared allocation should have cancel token"); + assert!(manager.make_shared_connection(&alloc_a, stream_key, SharedSubscriberId::from_stream_uid(1))); + + // B joins the same shared stream with lower importance (priority 1). + let join_result = manager.add_shared_connection( + &addr_b, + SharedSubscriberId::from_stream_uid(2), + stream_key, + 1, + ConnectionKind::Normal, + ); + assert!(join_result.is_ok(), "B should join existing shared stream, got: {join_result:?}"); + + // A leaves shared stream. Shared allocation should now inherit B's lower priority. + manager.release_connection(&addr_a); + { + let connections = manager.read_connections(); + let shared = connections.shared.by_key.get(stream_key).expect("shared entry should remain for B"); + assert_eq!(shared.priority, 1, "shared priority must downgrade to remaining subscriber priority"); + } + + // A starts another stream with higher importance and should preempt B's shared stream. + let alloc_a2 = manager + .acquire_connection(&input_name, &addr_a, 0, ConnectionKind::Normal) + .expect("A should preempt lower-priority shared stream"); + assert_eq!(alloc_a2.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + assert!(shared_token.is_cancelled(), "shared stream should be cancelled when preempted"); + assert!(!manager.is_over_limit(&input_name), "provider should not remain over limit after preemption"); + + manager.release_connection(&addr_a); + manager.release_connection(&addr_b); +} + +#[tokio::test] +async fn test_higher_priority_user_preempts_shared_stream_with_multiple_subscribers() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let stream_key = "http://example.com/shared/live/multi"; + let addr_a: SocketAddr = "127.0.0.1:48601".parse().unwrap(); + let addr_b: SocketAddr = "127.0.0.1:48602".parse().unwrap(); + let addr_high: SocketAddr = "127.0.0.1:48603".parse().unwrap(); + + let shared_alloc = manager + .acquire_connection(&input_name, &addr_a, 5, ConnectionKind::Normal) + .expect("low-priority shared stream should get initial connection"); + let shared_token = shared_alloc.cancel_token.clone().expect("shared allocation should have cancel token"); + assert!(manager.make_shared_connection(&shared_alloc, stream_key, SharedSubscriberId::from_stream_uid(1))); + + manager + .add_shared_connection(&addr_b, SharedSubscriberId::from_stream_uid(2), stream_key, 6, ConnectionKind::Normal) + .expect("second subscriber should join shared stream"); + + let high_alloc = manager + .acquire_connection(&input_name, &addr_high, 0, ConnectionKind::Normal) + .expect("higher-priority user should preempt lower-priority shared stream"); + assert_eq!(high_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + assert!(shared_token.is_cancelled(), "shared stream should be cancelled when preempted"); + + { + let connections = manager.read_connections(); + assert!( + !connections.shared.by_key.contains_key(stream_key), + "preempted shared stream must be removed even with multiple subscribers" + ); + } + + manager.release_connection(&addr_high); + manager.release_connection(&addr_a); + manager.release_connection(&addr_b); +} + +#[tokio::test] +async fn test_preemption_respects_registered_body_owner_completion() { + let app_cfg = build_test_app_config(None, 1); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let addr_1: SocketAddr = "127.0.0.1:49061".parse().unwrap(); + let addr_2: SocketAddr = "127.0.0.1:49062".parse().unwrap(); + + // 1. First request with prio 5 occupies limit=1 + let handle_1 = manager + .acquire_connection(&input_name, &addr_1, 5, ConnectionKind::Normal) + .expect("first allocation should succeed"); + + // 2. Register body owner on handle_1 + manager.register_body_owner(handle_1.allocation_id); + + // 3. Higher-priority request with prio -1 arrives without grace while handle_1 is active + let handle_2 = manager.acquire_connection_with_grace(&input_name, &addr_2, false, -1, ConnectionKind::Normal); + + // 4. Replacement must NOT be admitted before completion + assert!(handle_2.is_none(), "replacement must not be admitted before victim completion"); + assert!( + handle_1.cancel_token.as_ref().expect("cancel token").is_cancelled(), + "victim should receive cancel signal" + ); + assert_eq!( + handle_1.close_reason.load(std::sync::atomic::Ordering::Acquire), + tuliprox_core::model::ProviderCloseReason::PriorityPreempted as u8, + "victim close reason must be PriorityPreempted" + ); + + // 5. Victim completes upstream reading + handle_1.completion_token.as_ref().expect("completion token").cancel(); + + // Allow background complete_release task to run + tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(20)).await; + + // 6. Now replacement request can be admitted + let handle_2 = manager + .acquire_connection(&input_name, &addr_2, -1, ConnectionKind::Normal) + .expect("replacement should succeed after victim has completed release"); + + manager.release_handle(&handle_2); +} + +#[tokio::test] +async fn test_preemption_respects_opening_connection_completion() { + let app_cfg = build_test_app_config(None, 1); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let addr_1: SocketAddr = "127.0.0.1:49063".parse().unwrap(); + let addr_2: SocketAddr = "127.0.0.1:49064".parse().unwrap(); + + // 1. First request with prio 5 occupies limit=1 + let handle_1 = manager + .acquire_connection(&input_name, &addr_1, 5, ConnectionKind::Normal) + .expect("first allocation should succeed"); + manager.mark_opening(handle_1.allocation_id); + + // Verify lifecycle is Opening and body owner is not yet registered + { + let connections = manager.read_connections(); + let info = connections.single.get(&handle_1.allocation_id).expect("handle_1 exists"); + assert_eq!(info.lifecycle, tuliprox_core::model::ConnectionLifecycle::Opening); + assert!(!info.has_body_owner); + } + + // 2. Higher-priority request with prio -1 arrives without grace while handle_1 is in Opening + let handle_2 = manager.acquire_connection_with_grace(&input_name, &addr_2, false, -1, ConnectionKind::Normal); + + // 3. Replacement must NOT be admitted before completion of opening + assert!(handle_2.is_none(), "replacement must not be admitted while victim is still opening"); + assert!( + handle_1.cancel_token.as_ref().expect("cancel token").is_cancelled(), + "victim should receive cancel signal during open" + ); + assert_eq!( + handle_1.close_reason.load(std::sync::atomic::Ordering::Acquire), + tuliprox_core::model::ProviderCloseReason::PriorityPreempted as u8, + "victim close reason must be PriorityPreempted" + ); + + // 4. Victim's open future completes and signals completion + handle_1.completion_token.as_ref().expect("completion token").cancel(); + + // Allow background complete_release task to run + tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(20)).await; + + // 5. Now replacement request can be admitted + let handle_2 = manager + .acquire_connection(&input_name, &addr_2, -1, ConnectionKind::Normal) + .expect("replacement should succeed after victim open future has completed release"); + + manager.release_handle(&handle_2); +} + +#[tokio::test] +async fn test_normal_connection_preempts_existing_soft_connection() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let soft_addr: SocketAddr = "127.0.0.1:49011".parse().unwrap(); + let normal_addr: SocketAddr = "127.0.0.1:49012".parse().unwrap(); + + let soft_alloc = manager + .acquire_connection(&input_name, &soft_addr, default_user_priority(), ConnectionKind::Soft) + .expect("soft allocation"); + let soft_token = soft_alloc.cancel_token.clone().expect("soft allocations expose a cancel token"); + + let normal_alloc = manager + .acquire_connection(&input_name, &normal_addr, default_user_priority(), ConnectionKind::Normal) + .expect("normal allocation should preempt soft allocation"); + + tokio::task::yield_now().await; + assert!(soft_token.is_cancelled(), "soft allocation should be preempted by normal traffic"); + assert_eq!(manager.get_provider_connections_count(), 1); + + manager.release_handle(&normal_alloc); +} + +#[tokio::test] +async fn test_higher_priority_soft_preempts_lower_priority_soft() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let low_soft_addr: SocketAddr = "127.0.0.1:49013".parse().unwrap(); + let high_soft_addr: SocketAddr = "127.0.0.1:49014".parse().unwrap(); + + let low_soft_alloc = manager + .acquire_connection(&input_name, &low_soft_addr, 10, ConnectionKind::Soft) + .expect("low-priority soft allocation"); + let low_soft_token = low_soft_alloc.cancel_token.clone().expect("soft allocations expose a cancel token"); + + let high_soft_alloc = manager + .acquire_connection(&input_name, &high_soft_addr, -5, ConnectionKind::Soft) + .expect("higher-priority soft allocation should preempt lower-priority soft allocation"); + + tokio::task::yield_now().await; + assert!(low_soft_token.is_cancelled(), "lower-priority soft allocation should be preempted"); + assert_eq!(manager.get_provider_connections_count(), 1); + + manager.release_handle(&high_soft_alloc); +} + +#[tokio::test] +async fn test_reclassify_soft_to_normal_prevents_same_priority_preemption() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let promoted_addr: SocketAddr = "127.0.0.1:49015".parse().unwrap(); + let challenger_addr: SocketAddr = "127.0.0.1:49016".parse().unwrap(); + + let promoted_alloc = manager + .acquire_connection(&input_name, &promoted_addr, default_user_priority(), ConnectionKind::Soft) + .expect("initial soft allocation"); + let promoted_token = promoted_alloc.cancel_token.clone().expect("soft allocations expose a cancel token"); + + assert!( + manager.reclassify_connection(&promoted_addr, ConnectionKind::Normal, default_user_priority()), + "soft allocation should be promotable to normal" + ); + + let challenger = manager.acquire_connection_with_grace( + &input_name, + &challenger_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + ); + assert!(challenger.is_none(), "same-priority normal traffic should not preempt a promoted normal connection"); + assert!(!promoted_token.is_cancelled(), "promoted connection should remain active"); + assert_eq!(manager.get_provider_connections_count(), 1); + + manager.release_handle(&promoted_alloc); +} + +#[tokio::test(start_paused = true)] +async fn affinity_never_forces_grace_over_allocation_while_primary_is_free() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 1); + live_hls_playback_on_alias_with_lapsed_lease( + &manager, + &input, + &alias, + "client|alice|42|hls|0123456789abcdef", + PlaybackRequestOutcome::Completed, + ) + .await?; + let alias_holder = manager + .acquire_exact_connection_with_lease_for_session( + &alias, + &SocketAddr::from(([172, 18, 0, 9], 50_020)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new("client|erin|9|hls|0123456789abcdef", PlaybackKind::LiveHls)), + ) + .ok_or("alias holder")?; + + // Grace is enabled by default and allowed for this request. + let reentry = manager + .acquire_connection_with_lease_for_session( + &input, + &SocketAddr::from(([172, 18, 0, 9], 50_011)), + true, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new("client|alice|42|hls|fedcba9876543210", PlaybackKind::LiveHls)), + ) + .ok_or("re-entry")?; + assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); + assert!( + matches!(reentry.allocation, ProviderAllocation::Available(_)), + "a free primary slot beats a grace allocation" + ); + manager.release_handle(&reentry); + manager.release_handle(&alias_holder); + Ok(()) +} + +#[tokio::test] +async fn test_single_request_preemption_awaits_victim_completion() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + + let low_addr = SocketAddr::from(([127, 0, 0, 1], 48_001)); + let high_addr = SocketAddr::from(([127, 0, 0, 1], 48_002)); + + let low_handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &low_addr, + false, + 10, + ConnectionKind::Normal, + Some("session-victim"), + ) + .expect("initial low-priority allocation succeeds"); + + manager.mark_opening(low_handle.allocation_id); + assert!(manager.register_body_owner(low_handle.allocation_id)); + + let cancel_token = low_handle.cancel_token.clone().unwrap(); + let completion_token = low_handle.completion_token.clone().unwrap(); + + // Simulate upstream body owner: when cancelled, complete release after 30ms + let comp_clone = completion_token.clone(); + tokio::spawn(async move { + cancel_token.cancelled().await; + tokio::time::sleep(Duration::from_millis(30)).await; + comp_clone.cancel(); + }); + + // High priority acquire should preempt the victim, await its completion, and succeed in the same call + let high_handle = manager + .acquire_connection_with_lease_for_session_await( + &input_name, + &high_addr, + false, + 0, + ConnectionKind::Normal, + None, + ) + .await + .expect("high-priority allocation must succeed in the same call after awaiting victim completion"); + + assert_eq!(manager.get_provider_connections_count(), 1); + manager.release_handle(&high_handle); + assert_eq!(manager.get_provider_connections_count(), 0); +} diff --git a/backend/session/src/active_provider_manager/tests/behavior.rs b/backend/session/src/active_provider_manager/tests/behavior.rs new file mode 100644 index 000000000..cd0a28715 --- /dev/null +++ b/backend/session/src/active_provider_manager/tests/behavior.rs @@ -0,0 +1,1019 @@ +use super::{ + acquire_live_hls_from_lineup, alias_pool, build_test_app_config, confirmed_alias_request, + create_test_app_config_single_provider_pool, create_test_app_config_single_unlimited_provider_pool, + create_test_app_config_with_dual_provider_pool, create_test_app_config_with_pool, + live_hls_playback_on_alias_with_lapsed_lease, ActiveProviderManager, ConnectionKind, PlaybackLeaseRef, +}; +use crate::{EventManager, SharedStreamManager}; +use shared::{defaults::default_user_priority, utils::Internable}; +use std::{ + net::SocketAddr, + sync::{Arc, Weak}, + time::Duration, +}; +use tuliprox_core::model::{PlaybackKind, PlaybackRequestOutcome}; + +#[test] +fn shared_stream_manager_backref_is_weak_and_does_not_leak() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let provider = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let shared = Arc::new(SharedStreamManager::new(Arc::clone(&provider))); + provider.set_shared_stream_manager(&shared); + + // The only strong reference to the shared manager is the local `shared`; the + // provider holds a Weak backref, so dropping `shared` releases both managers. + drop(shared); + assert!(provider.shared_stream_manager.get().and_then(Weak::upgrade).is_none()); +} + +#[tokio::test] +async fn two_playbacks_sharing_one_proxy_socket_keep_separate_slots() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + // Both external clients arrive through the same reverse-proxy peer socket. + let proxy_addr = SocketAddr::from(([172, 18, 0, 9], 53_000)); + + let first = manager + .acquire_connection_with_grace_for_session( + &input_name, + &proxy_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some("device-one"), + ) + .expect("first device behind the proxy acquires a slot"); + let second = manager + .acquire_connection_with_grace_for_session( + &input_name, + &proxy_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some("device-two"), + ) + .expect("second device behind the proxy acquires its own slot"); + assert_ne!(first.allocation_id, second.allocation_id); + manager.confirm_playback_activity("device-one"); + manager.confirm_playback_activity("device-two"); + + // Releasing one playback must free exactly its own allocation and leave the + // other device's slot on the same transport untouched. + manager.release_handle(&first); + manager.finish_playback_request("device-one", PlaybackRequestOutcome::Completed, None); + let remaining = manager.provider_capacities_for_input(&input_name); + let primary = remaining.iter().find(|(name, _, _)| name.as_ref() == "provider_1").expect("primary pool entry"); + assert_eq!(primary.1, 1, "exactly one slot must remain in use on the shared socket"); + assert!(manager.read_leases().lease_of_owner("device-one").is_none()); + assert!(manager.read_leases().lease_of_owner("device-two").is_some()); + + manager.release_handle(&second); + manager.finish_playback_request("device-two", PlaybackRequestOutcome::Completed, None); +} + +#[tokio::test] +async fn test_force_exact_acquire_does_not_overallocate_busy_provider() { + let app_cfg = create_test_app_config_with_dual_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let client_1_addr: SocketAddr = "127.0.0.1:40001".parse().unwrap(); + let client_2_addr: SocketAddr = "127.0.0.1:40002".parse().unwrap(); + + let first_alloc = manager + .acquire_connection(&input_name, &client_1_addr, default_user_priority(), ConnectionKind::Normal) + .expect("client1 initial allocation"); + let pinned_provider = first_alloc.allocation.get_provider_name().expect("provider name expected"); + assert_eq!(pinned_provider.as_ref(), "provider_1"); + + // provider_1 has max_connections=1 and is already in use by client1 + let forced = manager.force_exact_acquire_connection( + &pinned_provider, + &client_2_addr, + default_user_priority(), + ConnectionKind::Normal, + ); + assert!(forced.is_none(), "forced exact acquire must not over-allocate busy provider"); + + manager.release_connection(&client_1_addr); + manager.release_connection(&client_2_addr); +} + +#[tokio::test] +async fn test_force_session_fallback_uses_different_provider_when_current_is_busy() { + let app_cfg = create_test_app_config_with_dual_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let client_1_addr: SocketAddr = "127.0.0.1:41001".parse().unwrap(); + let client_2_addr: SocketAddr = "127.0.0.1:41002".parse().unwrap(); + + // Step 1: Client1 starts movie -> provider_1 + let first_alloc = manager + .acquire_connection(&input_name, &client_1_addr, default_user_priority(), ConnectionKind::Normal) + .expect("client1 initial allocation"); + assert_eq!(first_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + + // Step 2: Client1 stops -> release provider_1 + manager.release_connection(&client_1_addr); + + // Step 3: Client2 starts live -> provider_1 + let live_alloc = manager + .acquire_connection(&input_name, &client_2_addr, default_user_priority(), ConnectionKind::Normal) + .expect("client2 live allocation"); + let busy_provider = live_alloc.allocation.get_provider_name().expect("provider name expected"); + assert_eq!(busy_provider.as_ref(), input_name.as_ref()); + assert!(manager.is_exhausted(&busy_provider)); + + // Step 4: Client1 restarts same movie. + // This emulates force-session fallback path by acquiring without provider grace. + let fallback_alloc = manager + .acquire_connection_with_grace(&input_name, &client_1_addr, false, 0, ConnectionKind::Normal) + .expect("client1 fallback allocation without grace"); + let fallback_provider = fallback_alloc.allocation.get_provider_name().expect("fallback provider expected"); + + assert_ne!(fallback_provider.as_ref(), busy_provider.as_ref()); + assert_eq!(fallback_provider.as_ref(), "provider_2"); + + manager.release_connection(&client_1_addr); + manager.release_connection(&client_2_addr); +} + +#[tokio::test] +async fn test_seek_reacquire_stays_on_same_provider_account_until_stop() { + let app_cfg = create_test_app_config_with_dual_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let client_1_addr: SocketAddr = "127.0.0.1:42001".parse().unwrap(); + let client_2_addr: SocketAddr = "127.0.0.1:42002".parse().unwrap(); + + // Initial playback for client1. + let first_alloc = manager + .acquire_connection(&input_name, &client_1_addr, default_user_priority(), ConnectionKind::Normal) + .expect("client1 initial allocation"); + let pinned_provider = first_alloc.allocation.get_provider_name().expect("provider name expected"); + assert_eq!(pinned_provider.as_ref(), "provider_1"); + + // Another client occupies the alternate account while client1 keeps seeking. + let second_alloc = manager + .acquire_connection(&input_name, &client_2_addr, default_user_priority(), ConnectionKind::Normal) + .expect("client2 allocation"); + let second_provider = second_alloc.allocation.get_provider_name().expect("provider name expected"); + assert_eq!(second_provider.as_ref(), "provider_2"); + + // Simulate repeated seek/range reconnects for client1: + // release old connection for the same client, then force exact pinned provider. + for _ in 0..3 { + manager.release_connection(&client_1_addr); + let seek_alloc = manager + .force_exact_acquire_connection( + &pinned_provider, + &client_1_addr, + default_user_priority(), + ConnectionKind::Normal, + ) + .expect("seek reacquire should stay on pinned provider"); + let seek_provider = seek_alloc.allocation.get_provider_name().expect("provider name expected"); + assert_eq!(seek_provider.as_ref(), pinned_provider.as_ref()); + } + + // Stream stop / cleanup. + manager.release_connection(&client_1_addr); + manager.release_connection(&client_2_addr); +} + +#[tokio::test(start_paused = true)] +async fn denied_confirmation_followed_by_refresh_cannot_reserve_foreign_slot() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner_x = "session-owner-x"; + let owner_y = "session-owner-y"; + let owner_z = "session-owner-z"; + let addr_y: SocketAddr = "127.0.0.1:43310".parse().unwrap(); + let addr_z: SocketAddr = "127.0.0.1:43311".parse().unwrap(); + + // Y holds the only provider slot. + let y = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr_y, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_y), + ) + .expect("y acquires the single slot"); + + // X starts a lease and confirms media while Y holds the slot, so its + // reservation right is denied. A later refresh must not restore it. + manager.refresh_provider_reservation(&input_name, owner_x, 15); + manager.confirm_playback_activity(owner_x); + manager.refresh_provider_reservation(&input_name, owner_x, 15); + + manager.release_handle(&y); + + // If X's refresh had re-granted the denied reservation, Z would be blocked here. + let z = manager.acquire_connection_with_grace_for_session( + &input_name, + &addr_z, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_z), + ); + assert!(z.is_some(), "a denied reservation must not be restored by a later refresh"); + if let Some(z) = z { + manager.release_handle(&z); + } +} + +#[tokio::test] +async fn test_reclassify_connection_for_owner_preserves_other_owner_on_same_socket() { + let app_cfg = build_test_app_config(None, 2); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let shared_addr: SocketAddr = "127.0.0.1:49020".parse().unwrap(); + + let alloc_1 = manager + .acquire_connection_with_lease_for_session( + &input_name, + &shared_addr, + false, + -10, + ConnectionKind::Soft, + Some(PlaybackLeaseRef::new("owner-1", PlaybackKind::LiveTs)), + ) + .expect("first soft allocation"); + + let alloc_2 = manager + .acquire_connection_with_lease_for_session( + &input_name, + &shared_addr, + false, + -10, + ConnectionKind::Soft, + Some(PlaybackLeaseRef::new("owner-2", PlaybackKind::LiveTs)), + ) + .expect("second soft allocation"); + + assert!(manager.reclassify_connection_for_owner(&shared_addr, Some("owner-1"), ConnectionKind::Normal, 0)); + + { + let connections = manager.read_connections(); + let info_1 = connections.single.get(&alloc_1.allocation_id).expect("alloc_1 exists"); + assert_eq!(info_1.kind, ConnectionKind::Normal); + assert_eq!(info_1.priority, 0); + + let info_2 = connections.single.get(&alloc_2.allocation_id).expect("alloc_2 exists"); + assert_eq!(info_2.kind, ConnectionKind::Soft); + assert_eq!(info_2.priority, -10); + } + + manager.release_handle(&alloc_1); + manager.release_handle(&alloc_2); +} + +#[test] +fn recording_allocation_is_claimed_without_consuming_a_second_slot() { + let app_cfg = build_test_app_config(None, 1); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + + let reserved = + manager.acquire_connection_for_download(&input_name, 0).expect("recording reserves the only provider slot"); + assert_eq!(manager.get_provider_connections_count(), 1); + + let claimed = manager + .claim_download_connection(reserved.allocation_id, &input_name) + .expect("internal recording request claims the existing reservation"); + assert_eq!(claimed.allocation_id, reserved.allocation_id); + assert_eq!(manager.get_provider_connections_count(), 1, "claim must not allocate a second slot"); + assert!( + manager.claim_download_connection(reserved.allocation_id, &input_name).is_none(), + "one reservation can only own one provider body" + ); + + manager.complete_release(reserved.allocation_id); + assert_eq!(manager.get_provider_connections_count(), 0); +} + +/// A=2 / B=3 pool: five confirmed playbacks exhaust the pool and a sixth start is +/// rejected; releasing one confirmed playback frees exactly one slot again. +#[tokio::test] +async fn sixth_playback_rejected_after_pool_exhausted_by_confirmed_leases() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + + let mut acquired = Vec::new(); + for index in 0..5u16 { + let owner = format!("playback-{index}"); + let addr = SocketAddr::from(([172, 18, 0, 9], 60_000 + index)); + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(&owner), + ) + .expect("pool has capacity for five playbacks"); + manager.confirm_playback_activity(&owner); + acquired.push((handle, owner)); + } + + let sixth = manager.acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 60_005)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some("playback-5"), + ); + assert!(sixth.is_none(), "sixth start must be rejected once the pool is exhausted"); + + manager.release_handle(&acquired[0].0); + let replacement = manager.acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 60_006)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some("playback-6"), + ); + assert!(replacement.is_some(), "freed capacity must be selectable again"); + manager.release_handle(&replacement.expect("replacement handle")); + for (handle, _owner) in &acquired[1..] { + manager.release_handle(handle); + } + assert_eq!(manager.get_provider_connections_count(), 0); +} + +#[test] +fn live_hls_lease_owner_preserves_other_playback_identities() { + let owner = "client|alice|42"; + assert_eq!(super::super::playback_lease_owner("client|alice|42|hls|0123456789abcdef"), owner); + for token in [ + owner, + "client|alice|42|hls|short", + "client|alice|42|hls|0123456789abcde!", + "m3u-catchup|client|alice|42|hls|0123456789abcdef", + "catchup|client|alice|42|hls|0123456789abcdef", + "hls-cache:client|hls|0123456789abcdef", + ] { + assert_eq!(super::super::playback_lease_owner(token), token); + } + assert_ne!( + super::super::playback_lease_owner("client|alice|42|hls|0123456789abcdef"), + super::super::playback_lease_owner("other-client|alice|42|hls|0123456789abcdef") + ); +} + +#[tokio::test(start_paused = true)] +async fn live_hls_retries_with_unique_tokens_keep_one_provider_lease() -> Result<(), String> { + let app_cfg = create_test_app_config_with_pool(2, 4); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input = "provider_1".intern(); + let alias = "provider_2".intern(); + let mut original_tag = None; + let mut previous_request: Option<( + String, + tuliprox_core::model::PlaybackRequestId, + Option, + )> = None; + for attempt in 0..104_u16 { + let token = format!("client|alice|42|hls|{attempt:016x}"); + let handle = manager + .acquire_connection_with_lease_for_session( + &input, + &SocketAddr::from(([172, 18, 0, 9], 55_000 + attempt)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(&token, PlaybackKind::LiveHls)), + ) + .ok_or("retry must reuse the reserved provider")?; + let request_id = handle.playback_request_id.ok_or("identified HLS request")?; + let provider = handle.allocation.get_provider_name().ok_or("provider")?; + assert_eq!(provider, input); + let tag = handle.binding_tag.ok_or("binding tag")?; + let first_tag = original_tag.get_or_insert(tag); + assert_eq!(tag.lease_id, first_tag.lease_id); + manager.refresh_adaptive_playback_lease(&provider, &token, PlaybackKind::LiveHls, 60); + manager.confirm_identified_playback_activity(&token, request_id); + assert!(manager.should_reuse_playback_provider(&token, &provider)); + manager.release_handle(&handle); + manager.finish_identified_playback_request(&token, request_id, PlaybackRequestOutcome::Completed); + manager.clear_provider_reservation(&token); + if let Some((old_token, old_request, old_tag)) = previous_request.take() { + manager.clear_identified_provider_reservation(&old_token, &provider, old_tag); + manager.finish_identified_playback_request(&old_token, old_request, PlaybackRequestOutcome::ProviderFailed); + assert!(manager.confirm_identified_playback_activity(&old_token, old_request).is_none()); + } + previous_request = Some((token, request_id, handle.binding_tag)); + assert_eq!(manager.provider_lease_usage(&input).idle, 1); + assert_eq!(manager.provider_lease_usage(&alias).total(), 0); + } + let foreign = manager + .acquire_connection_with_lease_for_session( + &input, + &SocketAddr::from(([172, 18, 0, 10], 56_000)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new("other-client|alice|42|hls|0123456789abcdef", PlaybackKind::LiveHls)), + ) + .ok_or("another playback must retain free provider capacity")?; + manager.release_handle(&foreign); + tokio::time::advance(Duration::from_secs(61)).await; + assert_eq!(manager.provider_lease_usage(&input).total(), 0); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn live_hls_alias_affinity_ends_after_its_window() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 0); + live_hls_playback_on_alias_with_lapsed_lease( + &manager, + &input, + &alias, + "client|alice|42|hls|0123456789abcdef", + PlaybackRequestOutcome::Completed, + ) + .await?; + tokio::time::advance(Duration::from_secs(shared::defaults::default_provider_affinity_ttl_secs())).await; + + let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; + assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); + manager.release_handle(&reentry); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn delayed_failure_of_older_binding_keeps_successor_affinity() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 0); + let old_token = "client|alice|42|hls|0123456789abcdef"; + let new_token = "client|alice|42|hls|fedcba9876543210"; + + // An older binding on the primary that never confirmed media. + let old = manager + .acquire_exact_connection_with_lease_for_session( + &input, + &SocketAddr::from(([172, 18, 0, 9], 50_000)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(old_token, PlaybackKind::LiveHls)), + ) + .ok_or("old binding")?; + let old_request = old.playback_request_id.ok_or("old request")?; + let old_tag = old.binding_tag.ok_or("old tag")?; + manager.release_handle(&old); + + let current = confirmed_alias_request(&manager, &input, &alias, new_token)?; + let current_request = current.playback_request_id.ok_or("current request")?; + manager.release_handle(¤t); + manager.finish_identified_playback_request(new_token, current_request, PlaybackRequestOutcome::Completed); + + // Delayed manifest failures of the older binding arrive after the rebind. + assert!(!manager.forget_identified_provider_affinity(old_token, &input, old_tag)); + manager.finish_identified_playback_request(old_token, old_request, PlaybackRequestOutcome::ProviderFailed); + tokio::time::advance(Duration::from_secs(20)).await; + assert_eq!(manager.provider_lease_usage(&alias).total(), 0, "lease must have expired"); + manager.finish_identified_playback_request(old_token, old_request, PlaybackRequestOutcome::ProviderFailed); + + let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|00000000000000aa", 50_011)?; + assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&alias))); + manager.release_handle(&reentry); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn administrative_termination_of_hls_token_ends_affinity_and_lease() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 0); + let token = "client|alice|42|hls|0123456789abcdef"; + let handle = confirmed_alias_request(&manager, &input, &alias, token)?; + let request_id = handle.playback_request_id.ok_or("identified request")?; + manager.release_handle(&handle); + manager.finish_identified_playback_request(token, request_id, PlaybackRequestOutcome::Completed); + assert_eq!(manager.provider_lease_usage(&alias).idle, 1); + + // The bare-owner clear has no delete right for public HLS tokens. + manager.clear_provider_reservation(token); + assert_eq!(manager.provider_lease_usage(&alias).idle, 1); + manager.terminate_identified_playback_owner(token); + assert_eq!(manager.provider_lease_usage(&alias).total(), 0); + + let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; + assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); + manager.release_handle(&reentry); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn stale_hls_token_termination_keeps_newer_binding() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 0); + let old_token = "client|alice|42|hls|0123456789abcdef"; + let new_token = "client|alice|42|hls|fedcba9876543210"; + + let old = confirmed_alias_request(&manager, &input, &alias, old_token)?; + let old_request = old.playback_request_id.ok_or("old request")?; + manager.release_handle(&old); + manager.finish_identified_playback_request(old_token, old_request, PlaybackRequestOutcome::Completed); + + // The retry re-enters the idle lease and starts a new binding incarnation. + let new = acquire_live_hls_from_lineup(&manager, &input, new_token, 50_011)?; + assert_eq!(new.allocation.get_provider_name(), Some(Arc::clone(&alias))); + assert_ne!(new.binding_tag, old.binding_tag); + let new_request = new.playback_request_id.ok_or("new request")?; + manager.refresh_adaptive_playback_lease(&alias, new_token, PlaybackKind::LiveHls, 15); + manager.confirm_identified_playback_activity(new_token, new_request).ok_or("confirmation")?; + manager.release_handle(&new); + manager.finish_identified_playback_request(new_token, new_request, PlaybackRequestOutcome::Completed); + assert_eq!(manager.provider_lease_usage(&alias).idle, 1); + + assert!(!manager.terminate_identified_playback_owner(old_token)); + assert_eq!(manager.provider_lease_usage(&alias).idle, 1, "stale token must keep the newer lease"); + assert_eq!(manager.provider_affinity_for_owner(new_token), Some(Arc::clone(&alias))); + + assert!(manager.terminate_identified_playback_owner(new_token)); + assert_eq!(manager.provider_lease_usage(&alias).total(), 0); + assert_eq!(manager.provider_affinity_for_owner(new_token), None); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn failure_on_fallback_provider_keeps_preference_for_preferred_provider() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 1); + let token = "client|alice|42|hls|0123456789abcdef"; + live_hls_playback_on_alias_with_lapsed_lease(&manager, &input, &alias, token, PlaybackRequestOutcome::Completed) + .await?; + let alias_holder = manager + .acquire_exact_connection_with_lease_for_session( + &alias, + &SocketAddr::from(([172, 18, 0, 9], 50_020)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new("client|erin|9|hls|0123456789abcdef", PlaybackKind::LiveHls)), + ) + .ok_or("alias holder")?; + + // The full alias sends the re-entry to the primary, which then fails before media. + let fallback = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; + assert_eq!(fallback.allocation.get_provider_name(), Some(Arc::clone(&input))); + let fallback_request = fallback.playback_request_id.ok_or("fallback request")?; + manager.release_handle(&fallback); + manager.finish_identified_playback_request( + "client|alice|42|hls|fedcba9876543210", + fallback_request, + PlaybackRequestOutcome::FailedBeforeMedia, + ); + + assert_eq!(manager.provider_affinity_for_owner(token), Some(Arc::clone(&alias))); + manager.release_handle(&alias_holder); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn live_ts_playback_records_no_provider_affinity() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 0); + let busy_1 = acquire_live_hls_from_lineup(&manager, &input, "busy-1|bob|7|hls|0123456789abcdef", 50_001)?; + let busy_2 = acquire_live_hls_from_lineup(&manager, &input, "busy-2|carol|8|hls|0123456789abcdef", 50_002)?; + let owner = "ts-session-token"; + let handle = manager + .acquire_connection_with_lease_for_session( + &input, + &SocketAddr::from(([172, 18, 0, 9], 50_010)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveTs)), + ) + .ok_or("ts playback")?; + assert_eq!(handle.allocation.get_provider_name(), Some(Arc::clone(&alias))); + let request_id = handle.playback_request_id.ok_or("ts request")?; + manager.confirm_identified_playback_activity(owner, request_id).ok_or("confirmation")?; + manager.release_handle(&handle); + manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::ClientClosed); + manager.release_handle(&busy_1); + manager.release_handle(&busy_2); + + assert_eq!(manager.provider_affinity_for_owner(owner), None, "live TS has nothing to return to"); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn huge_reconnect_and_affinity_windows_never_panic() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 0); + manager.write_leases().set_affinity_ttl(crate::provider_leases::ProviderAffinityTtl(u64::MAX)); + let token = "client|alice|42|hls|0123456789abcdef"; + let handle = acquire_live_hls_from_lineup(&manager, &input, token, 50_010)?; + let provider = handle.allocation.get_provider_name().ok_or("provider")?; + let request_id = handle.playback_request_id.ok_or("request")?; + manager.refresh_adaptive_playback_lease(&provider, token, PlaybackKind::LiveHls, u64::MAX); + manager.confirm_identified_playback_activity(token, request_id).ok_or("confirmation")?; + manager.release_handle(&handle); + manager.finish_identified_playback_request(token, request_id, PlaybackRequestOutcome::Completed); + assert_eq!(manager.provider_affinity_for_owner(token), Some(provider)); + assert_eq!(manager.provider_lease_usage(&alias).total(), 0); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn zero_provider_affinity_ttl_ends_affinity_with_reconnect_window() -> Result<(), String> { + let app_cfg = create_test_app_config_with_pool(2, 0); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + manager.write_leases().set_affinity_ttl(crate::provider_leases::ProviderAffinityTtl(0)); + let input = "provider_1".intern(); + let alias = "provider_2".intern(); + live_hls_playback_on_alias_with_lapsed_lease( + &manager, + &input, + &alias, + "client|alice|42|hls|0123456789abcdef", + PlaybackRequestOutcome::Completed, + ) + .await?; + + let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; + assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); + manager.release_handle(&reentry); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn live_hls_provider_failure_ends_alias_affinity() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 0); + live_hls_playback_on_alias_with_lapsed_lease( + &manager, + &input, + &alias, + "client|alice|42|hls|0123456789abcdef", + PlaybackRequestOutcome::ProviderFailed, + ) + .await?; + + let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; + assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); + manager.release_handle(&reentry); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn live_hls_full_affinity_provider_falls_back_to_lineup() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 1); + live_hls_playback_on_alias_with_lapsed_lease( + &manager, + &input, + &alias, + "client|alice|42|hls|0123456789abcdef", + PlaybackRequestOutcome::Completed, + ) + .await?; + let alias_holder = manager + .acquire_exact_connection_with_lease_for_session( + &alias, + &SocketAddr::from(([172, 18, 0, 9], 50_020)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new("client|erin|9|hls|0123456789abcdef", PlaybackKind::LiveHls)), + ) + .ok_or("alias holder")?; + + let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; + assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); + manager.release_handle(&reentry); + manager.release_handle(&alias_holder); + Ok(()) +} + +/// max=1: X fetch → physical release → Y acquire → X first byte within the startup +/// deadline. X's late confirmation records media but must not over-commit Y's slot. +#[tokio::test(start_paused = true)] +async fn late_first_byte_cannot_reserve_slot_taken_by_other_playback() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + + let x_handle = manager + .acquire_connection_with_lease_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 40_000)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new("x", PlaybackKind::LiveHls)), + ) + .expect("x acquires the only slot"); + manager.refresh_adaptive_playback_lease(&input_name, "x", PlaybackKind::LiveHls, 15); + manager.release_handle(&x_handle); + + let y_handle = manager + .acquire_connection_with_lease_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 40_001)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new("y", PlaybackKind::LiveHls)), + ) + .expect("y acquires the now-free slot"); + + manager.confirm_playback_activity("x"); + assert_eq!(manager.get_provider_connections_count(), 1, "y still holds the only slot"); + assert!( + !manager.is_provider_reserved_for_other_session(&input_name, Some("y")), + "x's late confirmation must not over-commit the provider" + ); + + manager.confirm_playback_activity("y"); + manager.release_handle(&y_handle); + manager.finish_playback_request("y", PlaybackRequestOutcome::Completed, None); +} + +/// Counterexample: the same X fetch → release → first-byte sequence without a +/// competing Y keeps the free slot as a real reservation. +#[tokio::test(start_paused = true)] +async fn first_byte_reserves_when_slot_still_free() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + + let x_handle = manager + .acquire_connection_with_lease_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 40_002)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new("x", PlaybackKind::LiveHls)), + ) + .expect("x acquires the only slot"); + manager.refresh_adaptive_playback_lease(&input_name, "x", PlaybackKind::LiveHls, 15); + manager.release_handle(&x_handle); + + // No competing playback took the slot, so X's first byte reserves it. + manager.confirm_playback_activity("x"); + assert!( + manager.is_provider_reserved_for_other_session(&input_name, Some("other")), + "a free slot must be reservable by the confirming playback" + ); + + manager.finish_playback_request("x", PlaybackRequestOutcome::Completed, None); +} + +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss, clippy::cast_precision_loss)] +pub(in crate::active_provider_manager::tests) fn latency_percentile(sorted: &[u64], p: f64) -> u64 { + if sorted.is_empty() { + return 0; + } + let index = ((sorted.len() as f64 - 1.0) * p).round() as usize; + sorted[index.min(sorted.len() - 1)] +} + +#[cfg(target_os = "linux")] +pub(in crate::active_provider_manager::tests) fn resident_set_kib() -> Option { + let Ok(content) = std::fs::read_to_string("/proc/self/status") else { + return None; + }; + for line in content.lines() { + if let Some(rest) = line.strip_prefix("VmRSS:") { + return rest.split_whitespace().next().and_then(|v| v.parse::().ok()); + } + } + None +} + +#[cfg(not(target_os = "linux"))] +pub(in crate::active_provider_manager::tests) fn resident_set_kib() -> Option { None } + +pub(in crate::active_provider_manager::tests) struct LeaseWorkloadStats { + pub(in crate::active_provider_manager::tests) ops: u64, + pub(in crate::active_provider_manager::tests) elapsed_secs: f64, + pub(in crate::active_provider_manager::tests) acquire_p50: u64, + pub(in crate::active_provider_manager::tests) acquire_p95: u64, + pub(in crate::active_provider_manager::tests) acquire_p99: u64, + pub(in crate::active_provider_manager::tests) confirm_p50: u64, + pub(in crate::active_provider_manager::tests) confirm_p95: u64, + pub(in crate::active_provider_manager::tests) confirm_p99: u64, + pub(in crate::active_provider_manager::tests) release_p50: u64, + pub(in crate::active_provider_manager::tests) release_p95: u64, + pub(in crate::active_provider_manager::tests) release_p99: u64, +} + +#[allow(clippy::cast_possible_truncation)] +pub(in crate::active_provider_manager::tests) async fn measure_lease_workload( + manager: &Arc, + input_name: &Arc, + concurrency: usize, + rounds_per_task: usize, +) -> LeaseWorkloadStats { + let samples = Arc::new(std::sync::Mutex::new(Vec::<(u64, u64, u64)>::new())); + let started = std::time::Instant::now(); + let mut tasks = tokio::task::JoinSet::new(); + for task_index in 0..concurrency { + let manager = Arc::clone(manager); + let input_name = Arc::clone(input_name); + let samples = Arc::clone(&samples); + tasks.spawn(async move { + for round in 0..rounds_per_task { + let owner = format!("bench-{task_index}-{round}"); + let addr = SocketAddr::from(([127, 0, 0, 1], 40_000 + (task_index as u16))); + let acquire_at = std::time::Instant::now(); + let Some(handle) = manager.acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + 0, + ConnectionKind::Normal, + Some(&owner), + ) else { + continue; + }; + let acquire_us = acquire_at.elapsed().as_micros() as u64; + + let confirm_at = std::time::Instant::now(); + manager.confirm_playback_activity(&owner); + let confirm_us = confirm_at.elapsed().as_micros() as u64; + + let release_at = std::time::Instant::now(); + manager.release_handle(&handle); + let release_us = release_at.elapsed().as_micros() as u64; + + // A physical release never concludes the request; model the real + // lifecycle by finishing with a clean outcome so the lease returns + // to baseline (LiveTs is not reconnect-capable, so it is removed). + manager.finish_playback_request(&owner, PlaybackRequestOutcome::Completed, None); + + samples.lock().unwrap().push((acquire_us, confirm_us, release_us)); + } + }); + } + while let Some(result) = tasks.join_next().await { + result.expect("benchmark task must not fail"); + } + + let elapsed_secs = started.elapsed().as_secs_f64(); + let samples = Arc::try_unwrap(samples).expect("samples unique").into_inner().unwrap(); + let mut acquire = Vec::with_capacity(samples.len()); + let mut confirm = Vec::with_capacity(samples.len()); + let mut release = Vec::with_capacity(samples.len()); + for (a, c, r) in samples { + acquire.push(a); + confirm.push(c); + release.push(r); + } + acquire.sort_unstable(); + confirm.sort_unstable(); + release.sort_unstable(); + let ops = acquire.len() as u64; + LeaseWorkloadStats { + ops, + elapsed_secs, + acquire_p50: latency_percentile(&acquire, 0.50), + acquire_p95: latency_percentile(&acquire, 0.95), + acquire_p99: latency_percentile(&acquire, 0.99), + confirm_p50: latency_percentile(&confirm, 0.50), + confirm_p95: latency_percentile(&confirm, 0.95), + confirm_p99: latency_percentile(&confirm, 0.99), + release_p50: latency_percentile(&release, 0.50), + release_p95: latency_percentile(&release, 0.95), + release_p99: latency_percentile(&release, 0.99), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "manager microbenchmark: cargo +stable test -p tuliprox-session --release -- --ignored --nocapture bench_provider_lease_microbenchmark"] +#[allow(clippy::cast_precision_loss)] +async fn bench_provider_lease_microbenchmark() { + let manager = Arc::new(ActiveProviderManager::new( + &create_test_app_config_single_unlimited_provider_pool(), + &Arc::new(EventManager::new()), + )); + let input_name: Arc = "provider_1".intern(); + + // Warm-up before measurement so allocator and lock caches are exercised. + for index in 0..64u16 { + let addr = SocketAddr::from(([127, 0, 0, 1], 40_000 + index)); + let Some(handle) = manager.acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + 0, + ConnectionKind::Normal, + None, + ) else { + continue; + }; + manager.release_handle(&handle); + } + + let baseline_rss_kib = resident_set_kib(); + eprintln!("provider lease manager microbenchmark (unlimited provider, in-process, no HTTP bodies)"); + for concurrency in [1usize, 5, 50, 200] { + let stats = measure_lease_workload(&manager, &input_name, concurrency, 10).await; + assert_eq!(stats.ops, (concurrency * 10) as u64, "unexpected acquire failures at concurrency {concurrency}"); + let throughput = stats.ops as f64 / stats.elapsed_secs.max(f64::EPSILON); + let rss_delta_kib = resident_set_kib() + .zip(baseline_rss_kib) + .map_or_else(|| "unsupported".to_string(), |(rss, baseline)| rss.saturating_sub(baseline).to_string()); + eprintln!( + "concurrency={concurrency:>3} ops={:>5} throughput={:>9.1} ops/s | acquire p50/p95/p99={}/{}/{}us | confirm p50/p95/p99={}/{}/{}us | release p50/p95/p99={}/{}/{}us | rss_delta_kib={}", + stats.ops, + throughput, + stats.acquire_p50, + stats.acquire_p95, + stats.acquire_p99, + stats.confirm_p50, + stats.confirm_p95, + stats.confirm_p99, + stats.release_p50, + stats.release_p95, + stats.release_p99, + rss_delta_kib, + ); + } + + // Soak-style churn: every connection and lease must return to baseline. + assert_eq!(manager.get_provider_connections_count(), 0); + let usage = manager.provider_lease_usage(&input_name); + assert_eq!(usage.total(), 0, "lease table must return to baseline after churn"); +} + +#[tokio::test] +async fn stale_handle_does_not_cancel_newer_generation_slot() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + let addr = SocketAddr::from(([127, 0, 0, 1], 49_001)); + + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + 0, + ConnectionKind::Normal, + Some("session-gen"), + ) + .expect("initial allocation succeeds"); + + manager.mark_opening(handle.allocation_id); + assert!(manager.register_body_owner(handle.allocation_id)); + + // Gen 0 has an old completion token + let old_completion_token = handle.completion_token.clone().expect("completion token present"); + + // Renew opening tokens to advance generation to 1 + let (_new_cancel, new_completion, gen) = + manager.renew_opening_tokens(handle.allocation_id).expect("renewal should succeed"); + assert_eq!(gen, 1); + + // Simulate an old handle belonging to generation 0 whose completion token was cancelled + old_completion_token.cancel(); + let stale_handle = tuliprox_core::model::ProviderHandle { + playback_request_id: handle.playback_request_id, + binding_tag: handle.binding_tag, + client_id: handle.client_id, + allocation_id: handle.allocation_id, + allocation: handle.allocation.clone(), + cancel_token: handle.cancel_token.clone(), + completion_token: Some(old_completion_token), + close_reason: Arc::clone(&handle.close_reason), + open_generation: 0, // Stale generation + }; + + // Releasing the stale handle must NOT poison/cancel the new generation's completion token + manager.release_handle(&stale_handle); + assert!( + !new_completion.is_cancelled(), + "new generation completion token must remain uncancelled after stale handle release" + ); + assert_eq!(manager.get_provider_connections_count(), 1, "slot must still be held by the active new generation"); + + // When the real new generation completes, the slot is reaped + new_completion.cancel(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(manager.get_provider_connections_count(), 0); +} diff --git a/backend/session/src/active_provider_manager/tests/lifecycle.rs b/backend/session/src/active_provider_manager/tests/lifecycle.rs new file mode 100644 index 000000000..7f835c337 --- /dev/null +++ b/backend/session/src/active_provider_manager/tests/lifecycle.rs @@ -0,0 +1,884 @@ +use super::{ + acquire_live_hls_from_lineup, alias_pool, build_test_app_config, confirmed_alias_request, + create_test_app_config_single_provider_pool, create_test_app_config_with_dual_provider_pool, + create_test_app_config_with_pool, live_hls_playback_on_alias_with_lapsed_lease, ActiveProviderManager, + ConnectionKind, PlaybackLeaseRef, +}; +use crate::{ActiveUserManager, EventManager}; +use arc_swap::ArcSwapOption; +use shared::{ + defaults::{default_probe_user_priority, default_user_priority}, + utils::Internable, +}; +use std::{collections::HashSet, net::SocketAddr, sync::Arc, time::Duration}; +use tuliprox_core::model::{Config, PlaybackKind, PlaybackRequestOutcome, SharedSubscriberId}; + +#[tokio::test] +async fn forced_reopen_and_late_cleanup_preserve_other_proxy_requests() -> Result<(), Box> { + let manager = ActiveProviderManager::new(&create_test_app_config_with_pool(2, 3), &Arc::new(EventManager::new())); + let addr = SocketAddr::from(([127, 0, 0, 1], 50002)); + let input = Arc::from("provider_1"); + let first = manager + .acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("first")) + .ok_or("first allocation missing")?; + let other = manager + .acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("other")) + .ok_or("other allocation missing")?; + manager.release_playback_connections("first", &[addr]); + assert!(first.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); + assert!(!other.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); + assert_eq!(manager.get_provider_connections_count(), 1); + let replacement = manager + .acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("first")) + .ok_or("replacement missing")?; + manager.refresh_adaptive_playback_lease(&input, "first", tuliprox_core::model::PlaybackKind::LiveHls, 15); + manager.confirm_playback_activity("first"); + manager.finish_playback_request("first", PlaybackRequestOutcome::ProviderFailed, None); + assert_eq!(manager.provider_lease_usage(&input).active, 1); + manager.release_handle(&replacement); + manager.release_handle(&other); + Ok(()) +} + +#[tokio::test] +async fn release_snapshot_tracks_original_allocation_after_addr_reuse() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let events = Arc::new(EventManager::new()); + let provider = ActiveProviderManager::new(&app_cfg, &events); + let addr = SocketAddr::from(([127, 0, 0, 1], 50_022)); + let input = "provider_1".intern(); + let original = provider + .acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("old")) + .expect("original allocation"); + let original_snapshot = provider.release_snapshot_for_addr(&addr); + assert!(!provider.wait_for_snapshot_release(&original_snapshot, Duration::ZERO).await); + + provider.release_handle(&original); + let replacement = provider + .acquire_connection_with_grace_for_session(&input, &addr, false, 0, ConnectionKind::Normal, Some("new")) + .expect("replacement allocation"); + let replacement_snapshot = provider.release_snapshot_for_addr(&addr); + assert!(provider.wait_for_snapshot_release(&original_snapshot, Duration::ZERO).await); + assert!(!provider.wait_for_snapshot_release(&replacement_snapshot, Duration::ZERO).await); + + let geoip = Arc::new(ArcSwapOption::default()); + let users = ActiveUserManager::new(&Config::default(), &geoip, &events); + users.set_pending_provider_release("user", original_snapshot.clone()).await; + users.set_pending_provider_release("user", replacement_snapshot.clone()).await; + assert!(!users.clear_pending_provider_release("user", &original_snapshot).await); + assert_eq!(users.pending_provider_release("user").await, Some(replacement_snapshot.clone())); + assert!(users.clear_pending_provider_release("user", &replacement_snapshot).await); + assert_eq!(users.pending_provider_release("user").await, None); + provider.release_handle(&replacement); +} + +#[tokio::test] +async fn release_snapshot_tracks_shared_subscriber_without_waiting_for_other_subscribers() { + let app_cfg = create_test_app_config_single_provider_pool(); + let events = Arc::new(EventManager::new()); + let provider = ActiveProviderManager::new(&app_cfg, &events); + let input = "provider_1".intern(); + let first_addr = SocketAddr::from(([127, 0, 0, 1], 50_023)); + let second_addr = SocketAddr::from(([127, 0, 0, 1], 50_024)); + let first = SharedSubscriberId::from_stream_uid(50_023); + let second = SharedSubscriberId::from_stream_uid(50_024); + let origin = provider.acquire_connection(&input, &first_addr, 0, ConnectionKind::Normal).expect("shared origin"); + let before_promotion = provider.release_snapshot_for_addr(&first_addr); + assert!(provider.make_shared_connection(&origin, "shared-release", first)); + provider + .add_shared_connection(&second_addr, second, "shared-release", 0, ConnectionKind::Normal) + .expect("second subscriber"); + let snapshot = provider.release_snapshot_for_addr(&first_addr); + assert!(!provider.wait_for_snapshot_release(&before_promotion, Duration::ZERO).await); + assert!(!provider.wait_for_snapshot_release(&snapshot, Duration::ZERO).await); + + provider.release_connection(&first_addr); + assert!(provider.wait_for_snapshot_release(&before_promotion, Duration::ZERO).await); + assert!(provider.wait_for_snapshot_release(&snapshot, Duration::ZERO).await); + assert_eq!(provider.get_provider_connections_count(), 1); + provider.release_connection(&second_addr); +} + +#[tokio::test] +async fn test_probe_preemption_releases_capacity_and_cancels_immediately() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let user_addr: SocketAddr = "127.0.0.1:43001".parse().unwrap(); + + let probe_handle = manager + .acquire_connection_for_probe(&input_name, default_probe_user_priority()) + .expect("probe allocation should succeed"); + let probe_token = probe_handle.cancel_token.clone().expect("probe handle must carry cancel token"); + + // User request should preempt probe and immediately acquire released capacity. + let user_alloc = manager + .acquire_connection_with_grace(&input_name, &user_addr, false, default_user_priority(), ConnectionKind::Normal) + .expect("user allocation should preempt probe"); + assert_eq!(user_alloc.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + + // Cancellation happens inline during preemption; yield once to observe any deferred work. + tokio::task::yield_now().await; + assert!(probe_token.is_cancelled(), "probe token should be cancelled immediately after preemption"); + + manager.release_connection(&user_addr); +} + +#[tokio::test(start_paused = true)] +async fn test_session_provider_reservation_blocks_other_sessions_until_ttl_expires() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let owner_1 = "session-owner-1"; + let owner_2 = "session-owner-2"; + let addr_1: SocketAddr = "127.0.0.1:43101".parse().unwrap(); + let addr_2: SocketAddr = "127.0.0.1:43102".parse().unwrap(); + + manager.refresh_provider_reservation(&input_name, owner_1, 15); + manager.confirm_playback_activity(owner_1); + + let first = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr_1, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_1), + ) + .expect("reserved owner should reacquire its provider"); + assert_eq!(first.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + manager.release_connection(&addr_1); + + let blocked = manager.acquire_connection_with_grace_for_session( + &input_name, + &addr_2, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_2), + ); + assert!(blocked.is_none(), "other sessions must not take a reserved provider before TTL expiry"); + + tokio::time::advance(Duration::from_secs(16)).await; + + let second = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr_2, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_2), + ) + .expect("reservation should expire after TTL"); + assert_eq!(second.allocation.get_provider_name().as_deref(), Some(input_name.as_ref())); + manager.release_connection(&addr_2); +} + +#[tokio::test(start_paused = true)] +async fn different_session_cannot_take_idle_reservation_from_same_client_family() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let Ok(second_addr) = "127.0.0.1:43112".parse() else { + return; + }; + let owner_channel_1 = "client|ua|user|100"; + let owner_channel_2 = "client|ua|user|200"; + + manager.refresh_provider_reservation(&input_name, owner_channel_1, 15); + manager.confirm_playback_activity(owner_channel_1); + + let blocked = manager.acquire_connection_with_grace_for_session( + &input_name, + &second_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_channel_2), + ); + assert!(blocked.is_none(), "a related but distinct playback must not steal an idle reservation"); + + manager.clear_provider_reservation(owner_channel_1); + let acquired = manager.acquire_connection_with_grace_for_session( + &input_name, + &second_addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_channel_2), + ); + assert!(acquired.is_some(), "explicitly clearing the old playback should release its provider"); + manager.release_connection(&second_addr); +} + +#[tokio::test(start_paused = true)] +async fn test_clear_provider_reservation_releases_family_block() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let owner_1 = "session-owner-1"; + let owner_2 = "session-owner-2"; + let addr_2: SocketAddr = "127.0.0.1:43132".parse().unwrap(); + + manager.refresh_provider_reservation(&input_name, owner_1, 15); + manager.confirm_playback_activity(owner_1); + + let blocked = manager.acquire_connection_with_grace_for_session( + &input_name, + &addr_2, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_2), + ); + assert!(blocked.is_none(), "reservation should initially block another session"); + + manager.clear_provider_reservation(owner_1); + + let acquired = manager.acquire_connection_with_grace_for_session( + &input_name, + &addr_2, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(owner_2), + ); + assert!(acquired.is_some(), "clearing reservation should unblock the provider"); +} + +#[tokio::test] +async fn shared_promotion_and_release_preserve_other_allocations_on_same_socket( +) -> Result<(), Box> { + let app_cfg = build_test_app_config(None, 3); + let events = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &events); + let addr = "127.0.0.1:48500".parse()?; + let input = "provider_1".intern(); + let unrelated = + manager.acquire_connection(&input, &addr, 0, ConnectionKind::Normal).ok_or("unrelated allocation missing")?; + let origin = + manager.acquire_connection(&input, &addr, 5, ConnectionKind::Normal).ok_or("origin allocation missing")?; + let first = SharedSubscriberId::from_stream_uid(1); + let second = SharedSubscriberId::from_stream_uid(2); + let key = "https://example.invalid/shared.ts"; + assert!(manager.make_shared_connection(&origin, key, first)); + assert_eq!(manager.get_provider_connections_count(), 2); + assert!(manager.read_connections().single.contains_key(&unrelated.allocation_id)); + assert!(manager + .connections + .read() + .unwrap() + .single_by_addr + .get(&addr) + .is_some_and(|allocs| allocs.contains(&unrelated.allocation_id))); + assert!(!unrelated.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); + manager.add_shared_connection(&addr, second, key, 9, ConnectionKind::Soft)?; + assert!(manager.reclassify_shared_connection(second, ConnectionKind::Normal, 1)); + { + let connections = manager.read_connections(); + let shared = connections.shared.by_key.get(key).ok_or("shared origin missing")?; + assert_eq!(shared.connections.get(&first).map(|subscriber| subscriber.priority), Some(5)); + assert_eq!(shared.priority, 1); + } + manager.release_shared_connection(second); + manager.release_shared_connection(second); + assert_eq!(manager.get_provider_connections_count(), 2); + assert_eq!(manager.read_connections().shared.by_key.get(key).map(|shared| shared.priority), Some(5)); + manager.release_shared_connection(first); + manager.release_handle(&origin); + assert_eq!(manager.get_provider_connections_count(), 1); + manager.release_handle(&unrelated); + assert_eq!(manager.get_provider_connections_count(), 0); + Ok(()) +} + +#[tokio::test] +async fn test_btree_index_consistent_after_lifecycle() { + // Verify that the priority_index stays consistent through add, evict, release. + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let addr_a: SocketAddr = "127.0.0.1:49001".parse().unwrap(); + let addr_b: SocketAddr = "127.0.0.1:49002".parse().unwrap(); + + // Add connection A (low priority) + let alloc_a = manager.acquire_connection(&input_name, &addr_a, 10, ConnectionKind::Normal).expect("alloc_a"); + + // Check index has 1 entry + { + let connections = manager.read_connections(); + let tree = connections.priority_index.get(&input_name).expect("index for provider_1"); + assert_eq!(tree.len(), 1, "index should have 1 entry after first allocation"); + } + + // High-priority user evicts low-priority via grace path + let alloc_b = manager + .acquire_connection(&input_name, &addr_b, -5, ConnectionKind::Normal) + .expect("alloc_b should evict alloc_a"); + + // Check index: should have 1 entry (alloc_a evicted, alloc_b added) + { + let connections = manager.read_connections(); + let tree = connections.priority_index.get(&input_name).expect("index for provider_1"); + assert_eq!(tree.len(), 1, "index should have 1 entry after eviction + new allocation"); + // The remaining entry should be alloc_b + let ((prio, _, _), _) = tree.iter().next().expect("one entry"); + assert_eq!(*prio, -5, "remaining entry should be the high-prio connection"); + } + + // Release alloc_b + manager.release_handle(&alloc_b); + + // Check index: should be empty + { + let connections = manager.read_connections(); + let tree = connections.priority_index.get(&input_name); + let is_empty = tree.is_none_or(std::collections::BTreeMap::is_empty); + assert!(is_empty, "index should be empty after releasing all connections"); + } + + // Verify alloc_a handle can be safely released (already evicted - no-op) + manager.release_handle(&alloc_a); +} + +#[tokio::test] +async fn test_owner_index_consistent_through_lifecycle() { + let app_cfg = build_test_app_config(None, 2); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + + let input_name = "provider_1".intern(); + let addr_a: SocketAddr = "127.0.0.1:49051".parse().unwrap(); + let addr_b: SocketAddr = "127.0.0.1:49052".parse().unwrap(); + + let owner_a = "owner-a"; + let owner_b = "owner-b"; + let alloc_a = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr_a, + false, + 0, + ConnectionKind::Normal, + Some(owner_a), + ) + .expect("alloc_a"); + let alloc_b = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr_b, + false, + 0, + ConnectionKind::Normal, + Some(owner_b), + ) + .expect("alloc_b"); + + { + let connections = manager.read_connections(); + assert_eq!(connections.by_owner.get(owner_a).map_or(0, HashSet::len), 1, "owner-a has one allocation"); + assert_eq!(connections.by_owner.get(owner_b).map_or(0, HashSet::len), 1, "owner-b has one allocation"); + assert!(connections.by_owner.get(owner_a).unwrap().contains(&alloc_a.allocation_id)); + assert!(connections.by_owner.get(owner_b).unwrap().contains(&alloc_b.allocation_id)); + } + + manager.release_handle(&alloc_a); + + { + let connections = manager.read_connections(); + assert!(!connections.by_owner.contains_key(owner_a), "owner-a index is removed after release"); + assert_eq!(connections.by_owner.get(owner_b).map_or(0, HashSet::len), 1, "owner-b still indexed"); + } + + manager.release_handle(&alloc_b); + { + let connections = manager.read_connections(); + assert!(connections.by_owner.is_empty(), "owner index is empty after all releases"); + } +} + +#[tokio::test(start_paused = true)] +async fn live_hls_reentry_keeps_alias_affinity_after_lease_expiry() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 0); + live_hls_playback_on_alias_with_lapsed_lease( + &manager, + &input, + &alias, + "client|alice|42|hls|0123456789abcdef", + PlaybackRequestOutcome::Completed, + ) + .await?; + + let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; + assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&alias))); + // Affinity reserves nothing: another playback still gets the free primary slot. + let other = acquire_live_hls_from_lineup(&manager, &input, "client|dave|42|hls|0123456789abcdef", 50_012)?; + assert_eq!(other.allocation.get_provider_name(), Some(Arc::clone(&input))); + manager.release_handle(&reentry); + manager.release_handle(&other); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn provider_failure_after_lease_expiry_ends_alias_affinity() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 0); + let token = "client|alice|42|hls|0123456789abcdef"; + let handle = confirmed_alias_request(&manager, &input, &alias, token)?; + let request_id = handle.playback_request_id.ok_or("identified request")?; + manager.release_handle(&handle); + tokio::time::advance(Duration::from_secs(20)).await; + manager.prune_expired_leases_now(); + assert_eq!(manager.provider_lease_usage(&alias).total(), 0, "lease must have expired"); + + manager.finish_identified_playback_request(token, request_id, PlaybackRequestOutcome::ProviderFailed); + + let reentry = acquire_live_hls_from_lineup(&manager, &input, "client|alice|42|hls|fedcba9876543210", 50_011)?; + assert_eq!(reentry.allocation.get_provider_name(), Some(Arc::clone(&input))); + manager.release_handle(&reentry); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn terminating_token_of_expired_binding_ends_its_affinity_only() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 0); + let token = "client|alice|42|hls|0123456789abcdef"; + let handle = confirmed_alias_request(&manager, &input, &alias, token)?; + manager.release_handle(&handle); + tokio::time::advance(Duration::from_secs(20)).await; + assert_eq!(manager.provider_lease_usage(&alias).total(), 0, "lease must have expired"); + + assert!(!manager.terminate_identified_playback_owner("client|alice|42|hls|00000000000000aa")); + assert_eq!(manager.provider_affinity_for_owner(token), Some(Arc::clone(&alias))); + assert!(manager.terminate_identified_playback_owner(token)); + assert_eq!(manager.provider_affinity_for_owner(token), None); + Ok(()) +} + +#[tokio::test] +async fn live_hls_socket_cleanup_only_cancels_its_public_session() -> Result<(), String> { + let app_cfg = create_test_app_config_with_pool(2, 4); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input = "provider_1".intern(); + let addr = SocketAddr::from(([172, 18, 0, 9], 55_000)); + let old = "client|alice|42|hls|0123456789abcdef"; + let new = "client|alice|42|hls|fedcba9876543210"; + let first = manager + .acquire_exact_connection_with_lease_for_session( + &input, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(old, PlaybackKind::LiveHls)), + ) + .ok_or("old allocation")?; + let next = manager + .acquire_exact_connection_with_lease_for_session( + &input, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(new, PlaybackKind::LiveHls)), + ) + .ok_or("new allocation")?; + assert_eq!(first.binding_tag, next.binding_tag); + assert_eq!(manager.provider_lease_usage(&input).total(), 1); + manager.release_playback_connections(old, &[addr]); + manager.release_playback_connections_await(old, &[addr]).await; + assert_eq!(manager.get_provider_connections_count(), 1); + assert!(!next.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); + manager.release_handle(&next); + Ok(()) +} + +/// Confirmed reconnect-capable lease, physical handle release first, then the real +/// provider-error outcome: the error must reach the lease and leave no idle reserve. +#[tokio::test(start_paused = true)] +async fn provider_error_cleanup_after_physical_release_does_not_keep_idle_lease() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "r5-error-owner"; + let addr = SocketAddr::from(([172, 18, 0, 9], 55_000)); + let handle = manager + .acquire_connection_with_lease_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveHls)), + ) + .expect("acquire a reconnect-capable slot"); + let request_id = handle.playback_request_id.expect("identified request id"); + manager.refresh_adaptive_playback_lease(&input_name, owner, PlaybackKind::LiveHls, 15); + manager.confirm_playback_activity(owner); + + // Physical release must not conclude the request; the outcome is decided later. + manager.release_handle(&handle); + assert_eq!( + manager.provider_lease_usage(&input_name).active, + 1, + "physical release must not finish a confirmed lease" + ); + + manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::ProviderFailed); + assert_eq!( + manager.provider_lease_usage(&input_name).total(), + 0, + "provider error must not keep an idle reconnect lease" + ); +} + +/// Counterexample: the same sequence with a clean end keeps the configured idle window. +#[tokio::test(start_paused = true)] +async fn clean_cleanup_after_physical_release_keeps_configured_idle_window() { + let app_cfg = create_test_app_config_with_pool(2, 3); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "r5-clean-owner"; + let addr = SocketAddr::from(([172, 18, 0, 9], 55_001)); + + let handle = manager + .acquire_connection_with_lease_for_session( + &input_name, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveHls)), + ) + .expect("acquire a reconnect-capable slot"); + let request_id = handle.playback_request_id.expect("identified request id"); + manager.refresh_adaptive_playback_lease(&input_name, owner, PlaybackKind::LiveHls, 15); + manager.confirm_playback_activity(owner); + + manager.release_handle(&handle); + manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::Completed); + let usage = manager.provider_lease_usage(&input_name); + assert_eq!(usage.active, 0, "clean end must move the lease out of active"); + assert_eq!(usage.idle, 1, "clean end must keep the configured reconnect window"); +} + +/// The RAII owner releases the allocation synchronously even when dropped outside +/// a tokio runtime, so a body/context drop can never leak a provider slot. +#[test] +fn managed_handle_drop_outside_runtime_releases_allocation() { + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + let (manager, managed_handle) = rt.block_on(async { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name = "provider_1".intern(); + + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 45_000)), + false, + 0, + ConnectionKind::Normal, + Some("managed-owner"), + ) + .expect("allocation should succeed"); + assert_eq!(manager.get_provider_connections_count(), 1); + + let managed_handle = super::super::ManagedProviderHandle::new(Arc::clone(&manager), handle); + (manager, managed_handle) + }); + + drop(managed_handle); + assert_eq!(manager.get_provider_connections_count(), 0); +} + +#[tokio::test] +async fn closing_state_blocks_slot_release_until_completion_token_cancelled() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + + let handle1 = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 46_001)), + false, + 0, + ConnectionKind::Normal, + Some("session-1"), + ) + .expect("first allocation should succeed"); + assert_eq!(manager.get_provider_connections_count(), 1); + + let completion_token = handle1.completion_token.clone().expect("completion token present"); + let alloc_id = handle1.allocation_id; + + // Mark Closing (simulating supersede / release_playback_connections_await) + manager.mark_closing(alloc_id); + + // Drop the handle while completion_token is still pending (upstream body not yet closed) + manager.release_handle(&handle1); + + // The slot must still be occupied by the Closing allocation + assert_eq!(manager.get_provider_connections_count(), 1); + + // A second start attempt must fail because the provider is at its limit (1 connection) + let handle2 = manager.acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 46_002)), + false, + 0, + ConnectionKind::Normal, + Some("session-2"), + ); + assert!(handle2.is_none(), "second start must be rejected while first slot is Closing"); + + // Now signal completion (upstream body owner dropped upstream socket) + completion_token.cancel(); + + // Allow background reaper task to execute complete_release + tokio::time::sleep(Duration::from_millis(50)).await; + + assert_eq!(manager.get_provider_connections_count(), 0); + + // Second start attempt now succeeds + let handle3 = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 46_003)), + false, + 0, + ConnectionKind::Normal, + Some("session-3"), + ) + .expect("start must succeed after completion releases the slot"); + assert_eq!(manager.get_provider_connections_count(), 1); + + manager.release_handle(&handle3); + assert_eq!(manager.get_provider_connections_count(), 0); +} + +#[tokio::test] +async fn reaper_cleanup_is_idempotent_and_removes_all_indices() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 47_001)), + false, + 0, + ConnectionKind::Normal, + Some("session-idempotent"), + ) + .expect("allocation should succeed"); + + let completion_token = handle.completion_token.clone().expect("completion token present"); + let alloc_id = handle.allocation_id; + manager.mark_closing(alloc_id); + + manager.release_handle(&handle); + completion_token.cancel(); + + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(manager.get_provider_connections_count(), 0); + + // Duplicate releases must be completely safe and no-op + manager.complete_release(alloc_id); + manager.release_handle(&handle); + manager.release_connection(&SocketAddr::from(([127, 0, 0, 1], 47_001))); + assert_eq!(manager.get_provider_connections_count(), 0); +} + +#[tokio::test] +async fn capacity_release_wakes_only_waiters_of_that_provider() { + use futures::FutureExt; + let app_cfg = create_test_app_config_with_dual_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let provider_1: Arc = "provider_1".intern(); + let provider_2: Arc = "provider_2".intern(); + let acquire = |provider: &Arc, port: u16| { + manager + .acquire_exact_connection_with_lease_for_session( + provider, + &SocketAddr::from(([127, 0, 0, 1], port)), + false, + 0, + ConnectionKind::Normal, + None, + ) + .expect("slot available") + }; + let handle_1 = acquire(&provider_1, 48_201); + let handle_2 = acquire(&provider_2, 48_202); + + let notify = manager.provider_capacity_notify(&provider_1); + let woken = notify.notified(); + tokio::pin!(woken); + woken.as_mut().enable(); + manager.release_handle(&handle_2); + assert!(woken.as_mut().now_or_never().is_none(), "another provider's release must not wake the waiter"); + manager.release_handle(&handle_1); + assert!(woken.as_mut().now_or_never().is_some(), "the waiter's provider release wakes it"); +} + +#[tokio::test(start_paused = true)] +async fn dropped_preempting_request_still_frees_the_victim_slot() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + let victim = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 48_111)), + false, + 10, + ConnectionKind::Normal, + Some("session-victim"), + ) + .expect("victim allocation succeeds"); + manager.mark_opening(victim.allocation_id); + assert!(manager.register_body_owner(victim.allocation_id)); + + // The preempting request waits without deadline and is dropped (client disconnect). + let waiting = { + let manager = Arc::clone(&manager); + let input_name = Arc::clone(&input_name); + tokio::spawn(async move { + manager + .acquire_exact_connection_with_lease_for_session_await( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 48_112)), + false, + 0, + ConnectionKind::Normal, + None, + ) + .await + }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(victim.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); + waiting.abort(); + assert!(waiting.await.is_err_and(|err| err.is_cancelled())); + assert_eq!(manager.get_provider_connections_count(), 1, "the draining victim keeps its slot for now"); + + tokio::time::sleep(super::super::PREEMPTION_COMPLETION_TIMEOUT).await; + tokio::task::yield_now().await; + assert_eq!(manager.get_provider_connections_count(), 0, "the reaper frees the never-completing victim"); +} + +#[tokio::test(start_paused = true)] +async fn exact_acquire_until_stops_at_deadline_and_still_forces_victim_release() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + let victim = manager + .acquire_connection_with_grace_for_session( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 48_101)), + false, + 10, + ConnectionKind::Normal, + Some("session-victim"), + ) + .expect("victim allocation succeeds"); + manager.mark_opening(victim.allocation_id); + assert!(manager.register_body_owner(victim.allocation_id)); + + // The victim never completes its body, so only the deadline can end the wait. + let started = tokio::time::Instant::now(); + let acquired = manager + .acquire_exact_connection_with_lease_for_session_until( + &input_name, + &SocketAddr::from(([127, 0, 0, 1], 48_102)), + false, + 0, + ConnectionKind::Normal, + None, + Some(started + Duration::from_millis(100)), + ) + .await; + assert!(acquired.is_none()); + assert_eq!(started.elapsed(), Duration::from_millis(100), "the caller deadline bounds the wait"); + assert!(victim.cancel_token.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled)); + assert_eq!(manager.get_provider_connections_count(), 1, "the draining victim keeps its slot for now"); + + tokio::time::sleep(super::super::PREEMPTION_COMPLETION_TIMEOUT).await; + tokio::task::yield_now().await; + assert_eq!(manager.get_provider_connections_count(), 0, "the forced victim release still runs"); +} + +#[tokio::test] +async fn dropped_cleanup_future_does_not_permanently_leak_slot() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let input_name: Arc = "provider_1".intern(); + let addr = SocketAddr::from(([127, 0, 0, 1], 49_002)); + let addrs = [addr]; + + let handle = manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + 0, + ConnectionKind::Normal, + Some("session-drop"), + ) + .expect("allocation succeeds"); + + manager.mark_opening(handle.allocation_id); + assert!(manager.register_body_owner(handle.allocation_id)); + let completion_token = handle.completion_token.clone().expect("token present"); + + // Start release_playback_connections_await but drop it before completion + { + let cleanup_fut = manager.release_playback_connections_await("session-drop", &addrs); + // Poll once and drop + tokio::select! { + biased; + () = async {} => {}, + () = cleanup_fut => {}, + } + } + + // Slot must still be occupied + assert_eq!(manager.get_provider_connections_count(), 1); + + // Now the handle is released and the body owner signals completion + manager.release_handle(&handle); + completion_token.cancel(); + + // Give the background reaper task time to run + tokio::time::sleep(Duration::from_millis(50)).await; + + assert_eq!( + manager.get_provider_connections_count(), + 0, + "slot must be freed after completion and not permanently leaked in Closing state" + ); +} diff --git a/backend/session/src/active_provider_manager/tests/retry.rs b/backend/session/src/active_provider_manager/tests/retry.rs new file mode 100644 index 000000000..4b083babe --- /dev/null +++ b/backend/session/src/active_provider_manager/tests/retry.rs @@ -0,0 +1,142 @@ +use super::{ + acquire_live_hls_from_lineup, alias_pool, create_test_app_config_with_dual_provider_pool, + create_test_app_config_with_pool, ActiveProviderManager, ConnectionKind, PlaybackLeaseRef, +}; +use crate::EventManager; +use shared::{defaults::default_user_priority, utils::Internable}; +use std::{net::SocketAddr, sync::Arc, time::Duration}; +use tuliprox_core::model::{PlaybackKind, PlaybackRequestOutcome}; + +#[tokio::test] +async fn unstarted_series_retry_uses_free_alias_while_started_playback_stays_pinned() { + let app_cfg = create_test_app_config_with_dual_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input = "provider_1".intern(); + let owner = "series-playback"; + let first_addr: SocketAddr = "127.0.0.1:41010".parse().unwrap(); + let busy_addr: SocketAddr = "127.0.0.1:41011".parse().unwrap(); + let retry_addr: SocketAddr = "127.0.0.1:41012".parse().unwrap(); + + let failed_start = manager + .acquire_connection_with_lease_for_session( + &input, + &first_addr, + false, + 0, + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(owner, PlaybackKind::Series)), + ) + .expect("first series allocation"); + assert!(manager.should_reuse_playback_provider(owner, &input)); + manager.release_handle(&failed_start); + assert!(!manager.should_reuse_playback_provider(owner, &input)); + + let busy = manager + .acquire_connection(&input, &busy_addr, 0, ConnectionKind::Normal) + .expect("other playback occupies first provider"); + let retry = manager + .acquire_connection_with_lease_for_session( + &input, + &retry_addr, + false, + 0, + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(owner, PlaybackKind::Series)), + ) + .expect("retry should use free alias"); + let alias = retry.allocation.get_provider_name().expect("alias name"); + assert_eq!(alias.as_ref(), "provider_2"); + assert!(manager.should_reuse_playback_provider(owner, &alias)); + manager.refresh_adaptive_playback_lease(&alias, owner, PlaybackKind::Series, 15); + manager.confirm_identified_playback_activity(owner, retry.playback_request_id.expect("request id")); + manager.release_handle(&retry); + assert!(manager.should_reuse_playback_provider(owner, &alias)); + manager.release_handle(&busy); +} + +#[tokio::test(start_paused = true)] +async fn finished_retry_tokens_do_not_accumulate_on_an_active_binding() -> Result<(), String> { + let app_cfg = create_test_app_config_with_pool(2, 0); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input = "provider_1".intern(); + let first = "client|alice|42|hls|0000000000000000"; + // One request stays in flight for the whole run and keeps the binding active. + let anchor = acquire_live_hls_from_lineup(&manager, &input, first, 50_000)?; + let anchor_request = anchor.playback_request_id.ok_or("anchor request")?; + manager.refresh_adaptive_playback_lease(&input, first, PlaybackKind::LiveHls, 15); + manager.confirm_identified_playback_activity(first, anchor_request).ok_or("confirmation")?; + + let mut last_token = String::new(); + for attempt in 1..=1_000_u32 { + last_token = format!("client|alice|42|hls|{attempt:016x}"); + let handle = acquire_live_hls_from_lineup(&manager, &input, &last_token, 50_001)?; + assert_eq!(handle.binding_tag, anchor.binding_tag, "retries share the active binding"); + let request_id = handle.playback_request_id.ok_or("retry request")?; + manager.refresh_adaptive_playback_lease(&input, &last_token, PlaybackKind::LiveHls, 15); + manager.release_handle(&handle); + manager.finish_identified_playback_request(&last_token, request_id, PlaybackRequestOutcome::Completed); + tokio::time::advance(Duration::from_secs(1)).await; + } + + let claims = manager.write_leases().request_token_claims(super::super::playback_lease_owner(first)); + assert!(claims <= 17, "only the anchor and recently finished retries may hold a claim, got {claims}"); + // The anchor keeps its claim while its request runs; a recent retry still has its cleanup right. + assert!(manager.write_leases().request_token_claims(super::super::playback_lease_owner(first)) >= 2); + assert!(manager.terminate_identified_playback_owner(&last_token)); + manager.release_handle(&anchor); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn live_hls_old_token_cleanup_cannot_mutate_rebound_retry() -> Result<(), String> { + let (manager, input, alias) = alias_pool(2, 4); + let old = "client|alice|42|hls|0123456789abcdef"; + let new = "client|alice|42|hls|fedcba9876543210"; + let addr = SocketAddr::from(([172, 18, 0, 9], 55_000)); + let first = manager + .acquire_exact_connection_with_lease_for_session( + &input, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(old, PlaybackKind::LiveHls)), + ) + .ok_or("initial allocation")?; + let old_request = first.playback_request_id.ok_or("old request")?; + manager.refresh_adaptive_playback_lease(&input, old, PlaybackKind::LiveHls, 60); + manager.confirm_identified_playback_activity(old, old_request); + manager.release_handle(&first); + let next = manager + .acquire_exact_connection_with_lease_for_session( + &alias, + &addr, + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(new, PlaybackKind::LiveHls)), + ) + .ok_or("rebound allocation")?; + let new_request = next.playback_request_id.ok_or("new request")?; + assert_ne!(first.binding_tag, next.binding_tag); + manager.refresh_adaptive_playback_lease(&alias, new, PlaybackKind::LiveHls, 60); + manager.confirm_identified_playback_activity(new, new_request); + manager.release_handle(&next); + manager.finish_identified_playback_request(new, new_request, PlaybackRequestOutcome::Completed); + let before = manager.binding_tag_for_owner(new); + manager.clear_provider_reservation(old); + manager.clear_identified_provider_reservation(old, &input, first.binding_tag); + manager.finish_identified_playback_request(old, old_request, PlaybackRequestOutcome::ProviderFailed); + manager.refresh_playback_lease( + &input, + &PlaybackLeaseRef { owner: old, kind: PlaybackKind::LiveHls, request_id: old_request }, + 0, + ); + assert!(manager.confirm_identified_playback_activity(old, old_request).is_none()); + assert_eq!(manager.binding_tag_for_owner(new), before); + assert_eq!(manager.provider_lease_usage(&input).total(), 0); + assert_eq!(manager.provider_lease_usage(&alias).idle, 1); + Ok(()) +} diff --git a/backend/session/src/active_provider_manager/tests/support.rs b/backend/session/src/active_provider_manager/tests/support.rs new file mode 100644 index 000000000..d867a2b76 --- /dev/null +++ b/backend/session/src/active_provider_manager/tests/support.rs @@ -0,0 +1,192 @@ +use super::{ActiveProviderManager, ConnectionKind, PlaybackLeaseRef}; +use crate::EventManager; +use arc_swap::{ArcSwap, ArcSwapOption}; +use shared::{ + defaults::default_user_priority, + model::{ConfigPaths, InputFetchMethod, InputType}, + utils::Internable, +}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration}; +use tuliprox_core::{ + model::{ + AppConfig, Config, ConfigInput, ConfigInputAlias, MediaToolCapabilities, PlaybackKind, PlaybackRequestOutcome, + SourcesConfig, + }, + utils::FileLockManager, +}; + +pub(in crate::active_provider_manager::tests) fn build_test_app_config( + aliases: Option>, + max_connections: u16, +) -> AppConfig { + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: "http://provider-1.example".to_string(), + username: Some("user1".to_string()), + password: Some("pass1".to_string()), + enabled: true, + priority: 0, + max_connections, + method: InputFetchMethod::default(), + aliases, + ..ConfigInput::default() + }); + + let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; + + AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(sources)), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + } +} + +pub(in crate::active_provider_manager::tests) fn create_test_app_config_with_dual_provider_pool() -> AppConfig { + build_test_app_config( + Some(vec![ConfigInputAlias { + id: 2, + name: "provider_2".intern(), + url: "http://provider-2.example".to_string(), + username: Some("user2".to_string()), + password: Some("pass2".to_string()), + priority: 1, + max_connections: 1, + exp_date: None, + enabled: true, + stalker: None, + }]), + 1, + ) +} + +pub(in crate::active_provider_manager::tests) fn create_test_app_config_single_provider_pool() -> AppConfig { + build_test_app_config(None, 1) +} + +pub(in crate::active_provider_manager::tests) fn create_test_app_config_single_unlimited_provider_pool() -> AppConfig { + build_test_app_config(None, 0) +} + +/// Pool where the higher-priority provider (A) and its lower-priority alias (B) +/// each carry their own capacity, mirroring the reported A=2 / B=3 setup. +pub(in crate::active_provider_manager::tests) fn create_test_app_config_with_pool( + primary_max: u16, + alias_max: u16, +) -> AppConfig { + build_test_app_config( + Some(vec![ConfigInputAlias { + id: 2, + name: "provider_2".intern(), + url: "http://provider-2.example".to_string(), + username: Some("user2".to_string()), + password: Some("pass2".to_string()), + priority: 1, + max_connections: alias_max, + exp_date: None, + enabled: true, + stalker: None, + }]), + primary_max, + ) +} + +/// Primary `provider_1` with an alias `provider_2` behind it. +pub(in crate::active_provider_manager::tests) fn alias_pool( + primary_max: u16, + alias_max: u16, +) -> (ActiveProviderManager, Arc, Arc) { + let app_cfg = create_test_app_config_with_pool(primary_max, alias_max); + let manager = ActiveProviderManager::new(&app_cfg, &Arc::new(EventManager::new())); + (manager, "provider_1".intern(), "provider_2".intern()) +} + +pub(in crate::active_provider_manager::tests) fn acquire_live_hls_from_lineup( + manager: &ActiveProviderManager, + input: &Arc, + token: &str, + port: u16, +) -> Result { + manager + .acquire_connection_with_lease_for_session( + input, + &SocketAddr::from(([172, 18, 0, 9], port)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(token, PlaybackKind::LiveHls)), + ) + .ok_or_else(|| format!("no provider for {token}")) +} + +/// Starts `token` on the alias while the primary is full, confirms media, ends the +/// request cleanly, frees the primary and lets the reconnect lease lapse. +pub(in crate::active_provider_manager::tests) async fn live_hls_playback_on_alias_with_lapsed_lease( + manager: &ActiveProviderManager, + input: &Arc, + alias: &Arc, + token: &str, + outcome: PlaybackRequestOutcome, +) -> Result<(), String> { + let busy_1 = acquire_live_hls_from_lineup(manager, input, "busy-1|bob|7|hls|0123456789abcdef", 50_001)?; + let busy_2 = acquire_live_hls_from_lineup(manager, input, "busy-2|carol|8|hls|0123456789abcdef", 50_002)?; + assert_eq!(busy_1.allocation.get_provider_name().as_ref(), Some(input)); + assert_eq!(busy_2.allocation.get_provider_name().as_ref(), Some(input)); + + let handle = acquire_live_hls_from_lineup(manager, input, token, 50_010)?; + let provider = handle.allocation.get_provider_name().ok_or("provider")?; + assert_eq!(&provider, alias, "full primary must fall back to the alias"); + let request_id = handle.playback_request_id.ok_or("identified request")?; + manager.refresh_adaptive_playback_lease(&provider, token, PlaybackKind::LiveHls, 15); + manager.confirm_identified_playback_activity(token, request_id).ok_or("confirmation")?; + manager.release_handle(&handle); + manager.finish_identified_playback_request(token, request_id, outcome); + + manager.release_handle(&busy_1); + manager.release_handle(&busy_2); + tokio::time::advance(Duration::from_secs(20)).await; + assert_eq!(manager.provider_lease_usage(alias).total(), 0, "reconnect lease must have lapsed"); + assert_eq!(manager.provider_lease_usage(input).total(), 0); + Ok(()) +} + +/// Starts `token` on the alias while the primary is full and confirms media. The +/// request stays attached; the primary is freed again before returning. +pub(in crate::active_provider_manager::tests) fn confirmed_alias_request( + manager: &ActiveProviderManager, + input: &Arc, + alias: &Arc, + token: &str, +) -> Result { + let busy_1 = acquire_live_hls_from_lineup(manager, input, "busy-1|bob|7|hls|0123456789abcdef", 50_001)?; + let busy_2 = acquire_live_hls_from_lineup(manager, input, "busy-2|carol|8|hls|0123456789abcdef", 50_002)?; + let handle = acquire_live_hls_from_lineup(manager, input, token, 50_010)?; + assert_eq!(handle.allocation.get_provider_name().as_ref(), Some(alias)); + let request_id = handle.playback_request_id.ok_or("identified request")?; + manager.refresh_adaptive_playback_lease(alias, token, PlaybackKind::LiveHls, 15); + manager.confirm_identified_playback_activity(token, request_id).ok_or("confirmation")?; + manager.release_handle(&busy_1); + manager.release_handle(&busy_2); + Ok(handle) +} diff --git a/backend/session/src/active_provider_manager/tests/terminal.rs b/backend/session/src/active_provider_manager/tests/terminal.rs new file mode 100644 index 000000000..369ba9592 --- /dev/null +++ b/backend/session/src/active_provider_manager/tests/terminal.rs @@ -0,0 +1,34 @@ +use super::{create_test_app_config_single_provider_pool, ActiveProviderManager, ConnectionKind, PlaybackLeaseRef}; +use crate::EventManager; +use shared::{defaults::default_user_priority, utils::Internable}; +use std::{net::SocketAddr, sync::Arc}; +use tuliprox_core::model::{PlaybackKind, PlaybackRequestOutcome}; + +#[tokio::test(start_paused = true)] +async fn stale_refresh_does_not_recreate_terminal_request() { + let app_cfg = create_test_app_config_single_provider_pool(); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveProviderManager::new(&app_cfg, &event_manager); + let input_name = "provider_1".intern(); + let owner = "terminal-refresh-owner"; + let handle = manager + .acquire_connection_with_lease_for_session( + &input_name, + &SocketAddr::from(([172, 18, 0, 9], 55_002)), + false, + default_user_priority(), + ConnectionKind::Normal, + Some(PlaybackLeaseRef::new(owner, PlaybackKind::LiveHls)), + ) + .expect("acquire identified request"); + let request_id = handle.playback_request_id.expect("identified request id"); + let lease_ref = PlaybackLeaseRef { owner, kind: PlaybackKind::LiveHls, request_id }; + + manager.release_handle(&handle); + manager.finish_identified_playback_request(owner, request_id, PlaybackRequestOutcome::ProviderFailed); + assert_eq!(manager.provider_lease_usage(&input_name).total(), 0); + + manager.refresh_playback_lease(&input_name, &lease_ref, 15); + manager.confirm_identified_playback_activity(owner, request_id); + assert_eq!(manager.provider_lease_usage(&input_name).total(), 0); +} diff --git a/backend/session/src/active_user_manager/activity.rs b/backend/session/src/active_user_manager/activity.rs new file mode 100644 index 000000000..75f739a85 --- /dev/null +++ b/backend/session/src/active_user_manager/activity.rs @@ -0,0 +1,307 @@ +use super::{ + next_session_incarnation, remember_session_addr, url_host_key, ActiveUserManager, CreateUserSessionParams, + PlaybackLifecycle, SessionChangeSignal, SessionIdentity, SocketRegistration, UserConnectionData, UserSession, + UserSessionParams, +}; +use shared::{ + model::UserConnectionPermission, + utils::{current_time_secs, sanitize_sensitive_info, Internable}, +}; +use std::{ + collections::HashMap, + net::SocketAddr, + sync::{atomic::AtomicBool, Arc}, + time::Instant, +}; +use tuliprox_core::utils::debug_if_enabled; + +impl ActiveUserManager { + pub(super) fn new_user_session(params: &UserSessionParams<'_>) -> UserSession { + let now = current_time_secs(); + UserSession { + token: params.session_token.to_string(), + transition_version: 1, + virtual_id: params.virtual_id, + provider: params.provider.intern(), + stream_url: params.stream_url.intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + provider_session_cookies: Arc::default(), + incarnation: next_session_incarnation(), + binding_generation: 0, + change_signal: SessionChangeSignal::new(params.change_wakes), + media_started: Arc::new(AtomicBool::new(false)), + user_agent_stream_index: None, + addr: *params.addr, + socket_bound: params.socket_bound, + active_addrs: vec![*params.addr], + ts: now, + started_at: now, + permission: params.connection_permission, + connection_kind: params.connection_kind, + lifecycle: PlaybackLifecycle::Prepared, + } + } + + pub async fn create_user_session(&self, request: CreateUserSessionParams<'_>) -> String { + let CreateUserSessionParams { + user, + session_token, + virtual_id, + provider, + stream_url, + addr, + connection_permission, + connection_kind, + socket_bound, + } = request; + self.gc(); + + let username = user.username.clone(); + let mut user_connections = self.write_connections().await; + let connection_data = user_connections.by_key.entry(username.clone()).or_insert_with(|| { + debug_if_enabled!("Creating first session for user {username} {}", sanitize_sensitive_info(stream_url)); + let mut data = UserConnectionData::new(0, user.max_connections, user.soft_connections); + let session = Self::new_user_session(&UserSessionParams { + change_wakes: &self.deferred_wakes, + session_token, + virtual_id, + provider, + stream_url, + addr, + connection_permission, + connection_kind, + socket_bound, + }); + data.add_session(session); + data + }); + + // If a session exists, update it + for session in &mut connection_data.sessions { + if session.token == session_token { + session.ts = current_time_secs(); + session.socket_bound = socket_bound; + remember_session_addr(session, *addr); + Self::bump_session_transition_version(session); + let mut reset_provider_session_headers = false; + if &*session.stream_url != stream_url { + // Provider cookies are host scoped: alternating child playlists on one host keep them. + reset_provider_session_headers |= url_host_key(&session.stream_url) != url_host_key(stream_url); + session.stream_url = stream_url.intern(); + } + if &*session.provider != provider { + session.switch_provider(provider.intern()); + } + if reset_provider_session_headers { + session.provider_session_headers.clear(); + session.provider_session_headers_host = None; + } + // Normalize stale lifecycle states on session refresh. + // Expired, PendingProvider, and Preserved sessions cannot stay in those states + // when a new request arrives for the same session token - the request is either + // a reactivation (Activate) or a follow-up on a still-valid logical playback. + match session.lifecycle { + PlaybackLifecycle::Expired => { + session.lifecycle = PlaybackLifecycle::Prepared; + } + // PendingProvider: pending wait continues until explicitly resolved. + // Preserved: stays preserved until explicit reactivation via activation path. + // Prepared: placeholder session, no counted lease. + // Active: session is already in a valid counted state. + // All these keep their current state - session.refresh() alone does not advance it. + #[allow(clippy::match_same_arms)] + PlaybackLifecycle::PendingProvider { .. } + | PlaybackLifecycle::Preserved + | PlaybackLifecycle::Prepared + | PlaybackLifecycle::Active => {} + PlaybackLifecycle::GraceActive => { + // GraceActive refresh keeps the provisional state. Grace window is still + // running — refresh does not advance it. The grace task will resolve it. + } + } + Self::update_session_admission(session, connection_permission, connection_kind); + debug_if_enabled!( + "Using session for user {} with url: {}", + user.username, + sanitize_sensitive_info(stream_url) + ); + return session.token.clone(); + } + } + + // If no session exists, create one + debug_if_enabled!( + "Creating session for user {} with url: {}", + user.username, + sanitize_sensitive_info(stream_url) + ); + let session = Self::new_user_session(&UserSessionParams { + change_wakes: &self.deferred_wakes, + session_token, + virtual_id, + provider, + stream_url, + addr, + connection_permission, + connection_kind, + socket_bound, + }); + let token = session.token.clone(); + connection_data.add_session(session); + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, &username); + drop(user_connections); + self.log_divergence_snapshot(divergence_snapshot).await; + token + } + + pub async fn touch_socket_activity(&self, addr: &SocketAddr) { + let now = current_time_secs(); + let mut user_connections = self.write_connections().await; + // Transport activity keeps the transport alive, not every playback that + // has previously used it. Authenticated HTTP activity updates its own user. + if let Some(registration) = user_connections.key_by_addr.get_mut(addr) { + registration.ts = now; + } + } + + pub async fn touch_http_activity(&self, username: &str, token: &str, addr: &SocketAddr) { + let now = current_time_secs(); + let mut user_connections = self.write_connections().await; + + let registration = user_connections.key_by_addr.entry(*addr).or_insert_with(SocketRegistration::anonymous); + registration.add_user(username, now); + + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return; + }; + + connection_data.ts = now; + + for session in &mut connection_data.sessions { + if session.token == token { + // Lightweight HTTP activity (for example HLS manifest reloads) refreshes + // continuity metadata only. It must not become an active stream socket: + // otherwise a manifest or probe request can steal the visible stream addr, + // and the real segment socket later migrates to that stale addr instead of + // being released/preserved. + session.ts = now; + break; + } + } + } + + /// Marks a session as ended on purpose (eviction, kick, explicit terminate), so it is not + /// recreated from a sealed playlist token. + pub async fn mark_session_ended(&self, token: &str) { + self.write_connections().await.mark_sessions_ended([token.to_string()]); + } + + /// True while a session token is blocked from recreation after an eviction, kick or terminate. + pub async fn is_session_ended(&self, token: &str) -> bool { + self.connections.read().await.ended_sessions.get(token).is_some_and(|expires_at| *expires_at > Instant::now()) + } + + pub async fn get_and_update_user_session(&self, username: &str, token: &str) -> Option { + self.update_user_session(username, token).await + } + + /// Checks an in-flight resource against the same session and account binding without touching admission. + pub async fn playback_session_is_current(&self, username: &str, token: &str, identity: SessionIdentity) -> bool { + self.connections.read().await.current_session(username, token, identity).is_some() + } + + /// Resolves once the session ended or switched accounts. Only writes that end this + /// session wake the waiter, after the write lock is released. + pub async fn wait_for_playback_session_end(&self, username: &str, token: &str, identity: SessionIdentity) { + loop { + let notify = { + let users = self.connections.read().await; + let Some((_, session)) = users.current_session(username, token, identity) else { + return; + }; + Arc::clone(&session.change_signal.notify) + }; + let changed = notify.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + // A change between the lookup and `enable` was not observed: re-check before waiting. + if !self.playback_session_is_current(username, token, identity).await { + return; + } + changed.await; + } + } + + /// Session object and account binding currently stored for `token`, in any lifecycle state. + pub async fn session_identity(&self, username: &str, token: &str) -> Option { + let users = self.connections.read().await; + users.by_key.get(username)?.sessions.iter().find(|session| session.token == token).map(UserSession::identity) + } + + pub async fn media_started_flag(&self, username: &str, token: &str) -> Option> { + let users = self.connections.read().await; + users + .by_key + .get(username)? + .sessions + .iter() + .find(|session| session.token == token) + .map(|session| Arc::clone(&session.media_started)) + } + + /// Session for target-scoped `virtual_id` and request token (used to recover leaked relative DVR segment paths). + pub async fn find_latest_session_for_target_stream( + &self, + username: &str, + target_id: u16, + input_name: &str, + virtual_id: u32, + session_token: &str, + ) -> Option { + let user_connections = self.connections.read().await; + let connection_data = user_connections.by_key.get(username)?; + connection_data + .streams + .iter() + .any(|stream| { + stream.channel.target_id == target_id + && stream.channel.input_name.as_ref() == input_name + && stream.channel.virtual_id == virtual_id + && stream.session_token.as_deref() == Some(session_token) + }) + .then_some(())?; + + connection_data + .sessions + .iter() + .find(|session| session.token == session_token && session.virtual_id == virtual_id) + .cloned() + } + + pub(super) async fn update_user_session(&self, username: &str, token: &str) -> Option { + let mut user_connections = self.write_connections().await; + + let connection_data = user_connections.by_key.get_mut(username)?; + let now = current_time_secs(); + + connection_data.ts = now; + + let session_index = connection_data.sessions.iter().position(|s| s.token == token)?; + + connection_data.sessions[session_index].ts = now; + + if connection_data.max_connections > 0 + && connection_data.sessions[session_index].permission == UserConnectionPermission::GracePeriod + && !matches!(connection_data.sessions[session_index].lifecycle, PlaybackLifecycle::PendingProvider { .. }) + { + let admission = self.check_connection_admission(username, connection_data); + connection_data.sessions[session_index].set_permission(admission.permission()); + if admission.kind().is_some() { + connection_data.sessions[session_index].connection_kind = admission.kind(); + } + } + + Some(connection_data.sessions[session_index].clone()) + } +} diff --git a/backend/session/src/active_user_manager/admission.rs b/backend/session/src/active_user_manager/admission.rs new file mode 100644 index 000000000..b95f4ea0e --- /dev/null +++ b/backend/session/src/active_user_manager/admission.rs @@ -0,0 +1,683 @@ +use super::{ + decide_connection_kind, ActiveUserManager, AdmissionRejectionReason, ConnectionAdmission, + PendingProviderWakeSource, PlaybackLifecycle, PromotionAction, UserConnectionCounts, UserConnectionData, + UserSession, USER_CON_TTL, +}; +use crate::active_provider_manager::{ConnectionKind, ProviderReleaseSnapshot}; +use jsonwebtoken::get_current_timestamp; +use log::debug; +use shared::model::{StreamInfo, UserConnectionPermission}; +use std::{collections::HashSet, sync::atomic::Ordering}; + +impl PlaybackLifecycle { + /// Returns true for lifecycle states that own a counted admission lease. + /// Both `Active` and `GraceActive` count — `GraceActive` is a provisional + /// counted state for `GraceMode::Instant` sessions. + pub fn is_counted(&self) -> bool { matches!(self, Self::Active | Self::GraceActive) } +} + +impl UserSession { + pub(super) fn set_permission(&mut self, permission: UserConnectionPermission) { + let exhausted_now = permission == UserConnectionPermission::Exhausted && self.permission != permission; + self.permission = permission; + if exhausted_now { + self.change_signal.signal(); + } + } +} + +impl ConnectionAdmission { + /// Maps a stored session permission onto an admission. An exhausted session is + /// classified as a genuine connection-limit rejection so the + /// `Exhausted ⇒ rejection_reason.is_some()` invariant always holds. + pub fn from_permission(permission: UserConnectionPermission, kind: Option) -> Self { + match permission { + UserConnectionPermission::Allowed => Self::allowed(kind), + UserConnectionPermission::GracePeriod => Self::grace_period(kind), + UserConnectionPermission::Exhausted => { + Self::exhausted(AdmissionRejectionReason::UserConnectionsExhausted, kind) + } + } + } + + pub fn permission(&self) -> UserConnectionPermission { self.permission } +} + +impl UserConnectionData { + pub(super) fn remove_streams_for_session_and_release_counted( + &mut self, + session_token: &str, + counted_kind: Option, + ) -> (u32, bool) { + let mut removed_count = 0; + let mut connection_changed = false; + let mut released_stream_kind = false; + let mut stream_idx = 0; + while stream_idx < self.streams.len() { + if self.streams[stream_idx].session_token.as_deref() != Some(session_token) { + stream_idx += 1; + continue; + } + + let uid = self.streams[stream_idx].uid; + if let Some(kind) = self.stream_kinds.remove(&uid) { + self.decrement_kind(kind); + released_stream_kind = true; + connection_changed = true; + } + self.stream_normal_priorities.remove(&uid); + self.remove_stream_request_claims(uid); + self.streams.swap_remove(stream_idx); + removed_count += 1; + } + + if let Some(kind) = counted_kind.filter(|_| !released_stream_kind) { + self.decrement_kind(kind); + connection_changed = true; + } + + (removed_count, connection_changed) + } + + pub(super) fn try_promote_soft_stream(&mut self) -> Option { + if self.counts.normal >= self.max_connections + || (u32::from(self.counts.soft)) <= u32::from(self.soft_connections) + { + return None; + } + + let candidate_uid = self + .streams + .iter() + .filter(|stream| !stream.preserved) + .filter_map(|stream| { + let kind = self.stream_kinds.get(&stream.uid).copied()?; + if kind != ConnectionKind::Soft { + return None; + } + let normal_priority = self.stream_normal_priorities.get(&stream.uid).copied().unwrap_or_default(); + Some((normal_priority, stream.ts, stream.uid, stream.addr)) + }) + .min_by_key(|(normal_priority, ts, uid, _)| (*normal_priority, *ts, *uid)); + + let (new_priority, _ts, uid, addr) = candidate_uid?; + + self.counts.normal = self.counts.normal.saturating_add(1); + if self.counts.soft > 0 { + self.counts.soft -= 1; + } + self.stream_kinds.insert(uid, ConnectionKind::Normal); + + Some(PromotionAction { addr, uid, new_priority }) + } + + pub(super) fn try_promote_soft_session_reservation(&mut self) -> bool { + if self.counts.normal >= self.max_connections + || (u32::from(self.counts.soft)) <= u32::from(self.soft_connections) + { + return false; + } + + let active_tokens = + self.streams.iter().filter_map(|stream| stream.session_token.as_deref()).collect::>(); + + let candidate_index = self.sessions.iter().position(|session| { + session.lifecycle.is_counted() + && session.connection_kind == Some(ConnectionKind::Soft) + && !active_tokens.contains(session.token.as_str()) + }); + + let Some(candidate_index) = candidate_index else { + return false; + }; + + self.counts.normal = self.counts.normal.saturating_add(1); + if self.counts.soft > 0 { + self.counts.soft -= 1; + } + self.sessions[candidate_index].connection_kind = Some(ConnectionKind::Normal); + true + } + + pub(super) fn effective_counts_for_admission(&self, exclude_session_token: Option<&str>) -> UserConnectionCounts { + let mut counts = self.counts; + let counted_tokens = self + .sessions + .iter() + .filter(|session| session.lifecycle.is_counted()) + .map(|session| session.token.as_str()) + .collect::>(); + let mut reserved_tokens = HashSet::new(); + + for stream in self.streams.iter().filter(|stream| stream.preserved) { + // Orphan preserved stream: no session token means no session to evict. + // Do not count it — it has no bearing on admission decisions. + let Some(session_token) = stream.session_token.as_deref() else { + continue; + }; + if exclude_session_token.is_some_and(|token| token == session_token) + || counted_tokens.contains(session_token) + || !reserved_tokens.insert(session_token) + { + continue; + } + + let kind = self + .sessions + .iter() + .find(|session| session.token == session_token) + .and_then(|session| session.connection_kind) + .unwrap_or(ConnectionKind::Normal); + match kind { + ConnectionKind::Normal => counts.normal = counts.normal.saturating_add(1), + ConnectionKind::Soft => counts.soft = counts.soft.saturating_add(1), + } + } + + counts + } +} + +impl ActiveUserManager { + pub(crate) async fn pending_provider_release(&self, username: &str) -> Option { + self.pending_provider_releases.lock().await.get(username).cloned() + } + + pub(crate) async fn set_pending_provider_release(&self, username: &str, snapshot: ProviderReleaseSnapshot) { + self.pending_provider_releases.lock().await.insert(username.to_owned(), snapshot); + } + + pub(crate) async fn clear_pending_provider_release( + &self, + username: &str, + snapshot: &ProviderReleaseSnapshot, + ) -> bool { + let mut pending = self.pending_provider_releases.lock().await; + if pending.get(username) != Some(snapshot) { + return false; + } + pending.remove(username); + true + } + + pub(super) fn check_connection_admission_with_counts( + &self, + username: &str, + connection_data: &mut UserConnectionData, + counts: UserConnectionCounts, + ) -> ConnectionAdmission { + let selected_kind = + decide_connection_kind(counts, connection_data.max_connections, connection_data.soft_connections); + let effective_connections = counts.normal.saturating_add(u32::from(counts.soft)); + + if let Some(kind) = selected_kind { + // Reset grace only once the user is back below the hard limit. + if effective_connections < connection_data.max_connections { + connection_data.granted_grace = false; + connection_data.grace_ts = 0; + } + return ConnectionAdmission::allowed(Some(kind)); + } + + let now = get_current_timestamp(); + // Check if user already used a grace period + if connection_data.granted_grace { + if effective_connections >= connection_data.max_connections + && now - connection_data.grace_ts <= self.grace_period_timeout_secs.load(Ordering::Relaxed) + { + // Grace timeout, still active, deny connection + debug!("User access denied, grace exhausted, too many connections: {username}"); + return ConnectionAdmission::exhausted(AdmissionRejectionReason::UserConnectionsExhausted, None); + } + // Grace timeout expired, reset grace counters + if effective_connections < connection_data.max_connections { + connection_data.granted_grace = false; + connection_data.grace_ts = 0; + } + } + + debug!("User access denied, too many connections: {username}"); + ConnectionAdmission::exhausted(AdmissionRejectionReason::UserConnectionsExhausted, None) + } + + pub(super) fn check_connection_admission( + &self, + username: &str, + connection_data: &mut UserConnectionData, + ) -> ConnectionAdmission { + self.check_connection_admission_with_counts( + username, + connection_data, + connection_data.effective_counts_for_admission(None), + ) + } + + pub async fn connection_admission( + &self, + username: &str, + max_connections: u32, + soft_connections: u16, + ) -> ConnectionAdmission { + if max_connections > 0 || soft_connections > 0 { + if let Some(connection_data) = self.write_connections().await.by_key.get_mut(username) { + connection_data.max_connections = max_connections; + connection_data.soft_connections = soft_connections; + return self.check_connection_admission(username, connection_data); + } + } + ConnectionAdmission::allowed(Some(ConnectionKind::Normal)) + } + + pub async fn connection_permission( + &self, + username: &str, + max_connections: u32, + soft_connections: u16, + ) -> UserConnectionPermission { + self.connection_admission(username, max_connections, soft_connections).await.permission + } + + pub async fn connection_admission_for_session( + &self, + username: &str, + max_connections: u32, + soft_connections: u16, + session_token: &str, + ) -> ConnectionAdmission { + if max_connections == 0 && soft_connections == 0 { + return ConnectionAdmission::allowed(Some(ConnectionKind::Normal)); + } + + let mut connections = self.write_connections().await; + let Some(connection_data) = connections.by_key.get_mut(username) else { + return ConnectionAdmission::allowed(Some(ConnectionKind::Normal)); + }; + connection_data.max_connections = max_connections; + connection_data.soft_connections = soft_connections; + + let Some(session_index) = connection_data.sessions.iter().position(|session| session.token == session_token) + else { + return self.check_connection_admission(username, connection_data); + }; + + if connection_data.sessions[session_index].lifecycle.is_counted() { + return ConnectionAdmission::allowed( + connection_data.sessions[session_index].connection_kind.or(Some(ConnectionKind::Normal)), + ); + } + + self.check_connection_admission_with_counts( + username, + connection_data, + connection_data.effective_counts_for_admission(Some(session_token)), + ) + } + + pub async fn connection_permission_for_session( + &self, + username: &str, + max_connections: u32, + soft_connections: u16, + session_token: &str, + ) -> UserConnectionPermission { + self.connection_admission_for_session(username, max_connections, soft_connections, session_token) + .await + .permission() + } + + pub async fn grant_grace(&self, username: &str) -> bool { + if self.grace_period_millis.load(Ordering::Relaxed) == 0 { + debug!("Grace grant denied, grace_period_millis is zero for {username}"); + return false; + } + let mut connections = self.write_connections().await; + if let Some(connection_data) = connections.by_key.get_mut(username) { + let now = get_current_timestamp(); + if connection_data.connections < connection_data.max_connections { + debug!( + "Grace grant denied for {username}, user not at connection limit ({}/{})", + connection_data.connections, connection_data.max_connections + ); + return false; + } + if connection_data.granted_grace + && connection_data.connections >= connection_data.max_connections + && now - connection_data.grace_ts <= self.grace_period_timeout_secs.load(Ordering::Relaxed) + { + debug!("Grace grant denied, still within active grace timeout for {username}"); + return false; + } + connection_data.granted_grace = true; + connection_data.grace_ts = now; + debug!("Granted a grace period for user access: {username}"); + return true; + } + false + } + + pub(super) fn promote_session_for_stream(connection_data: &mut UserConnectionData, stream: &StreamInfo) { + if let Some(token) = stream.session_token.as_deref() { + if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { + Self::mark_session_committed(session, ConnectionKind::Normal); + } + } + } + + pub(super) fn collect_promotions_after_capacity_release( + connection_data: &mut UserConnectionData, + ) -> Vec { + let mut promotions = Vec::new(); + while let Some(action) = connection_data.try_promote_soft_stream() { + let promoted_stream = connection_data.streams.iter().find(|stream| stream.uid == action.uid).cloned(); + if let Some(stream) = promoted_stream.as_ref() { + Self::promote_session_for_stream(connection_data, stream); + } + promotions.push(action); + } + while connection_data.try_promote_soft_session_reservation() {} + promotions + } + + pub(super) fn promote_counted_soft_session_to_normal_if_available( + connection_data: &mut UserConnectionData, + session_token: &str, + ) -> Vec { + if connection_data.max_connections > 0 && connection_data.counts.normal >= connection_data.max_connections { + return Vec::new(); + } + + let Some(session_index) = connection_data.sessions.iter().position(|session| { + session.token == session_token + && session.lifecycle.is_counted() + && session.connection_kind == Some(ConnectionKind::Soft) + }) else { + return Vec::new(); + }; + + if connection_data.counts.soft == 0 { + return Vec::new(); + } + + connection_data.counts.normal = connection_data.counts.normal.saturating_add(1); + connection_data.counts.soft = connection_data.counts.soft.saturating_sub(1); + connection_data.sessions[session_index].connection_kind = Some(ConnectionKind::Normal); + Self::bump_session_transition_version(&mut connection_data.sessions[session_index]); + + let mut promotions = Vec::new(); + for stream in + connection_data.streams.iter().filter(|stream| stream.session_token.as_deref() == Some(session_token)) + { + if connection_data.stream_kinds.get(&stream.uid) != Some(&ConnectionKind::Soft) { + continue; + } + let new_priority = connection_data.stream_normal_priorities.get(&stream.uid).copied().unwrap_or_default(); + connection_data.stream_kinds.insert(stream.uid, ConnectionKind::Normal); + promotions.push(PromotionAction { addr: stream.addr, uid: stream.uid, new_priority }); + } + promotions + } + + pub(super) fn update_session_admission( + session: &mut UserSession, + permission: UserConnectionPermission, + kind: Option, + ) { + session.set_permission(permission); + if let Some(kind) = kind { + session.connection_kind = Some(kind); + } + } + + pub(super) fn clear_session_pending_with_permission( + session: &mut UserSession, + permission: UserConnectionPermission, + wake_source: PendingProviderWakeSource, + ) { + if let PlaybackLifecycle::PendingProvider { data } = &mut session.lifecycle { + data.wake_source = Some(wake_source); + } + Self::bump_session_transition_version(session); + session.set_permission(permission); + } + + pub(super) fn clear_session_counted_without_stream(connection_data: &mut UserConnectionData, session_token: &str) { + if Self::session_has_stream(connection_data, session_token) { + return; + } + if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == session_token) { + match session.lifecycle { + PlaybackLifecycle::Active => { + session.lifecycle = PlaybackLifecycle::Preserved; + } + // GraceActive without a stream: grace failed, expire the session. + // This can happen when the grace window times out while the client + // is still connecting but hasn't opened a stream yet. + PlaybackLifecycle::GraceActive => { + session.expire(); + } + _ => {} + } + } + } + + pub(super) fn clear_session_counted(connection_data: &mut UserConnectionData, session_token: &str) { + if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == session_token) { + match session.lifecycle { + PlaybackLifecycle::Active => { + session.lifecycle = PlaybackLifecycle::Preserved; + } + PlaybackLifecycle::GraceActive => { + // GraceActive without stream: the grace failed. The stream was already + // removed (this function is called after stream removal), so expire the session. + session.expire(); + } + _ => {} + } + } + } + + pub(super) fn release_expired_session_reservations(connection_data: &mut UserConnectionData, now: u64) { + let expired_counted = connection_data + .sessions + .iter() + .filter(|session| session.lifecycle.is_counted()) + .filter(|session| now.saturating_sub(session.ts) >= USER_CON_TTL) + .filter(|session| !Self::session_has_stream(connection_data, session.token.as_str())) + .map(|session| (session.token.clone(), session.connection_kind.unwrap_or(ConnectionKind::Normal))) + .collect::>(); + + for (_, kind) in &expired_counted { + connection_data.decrement_kind(*kind); + } + for (token, _) in expired_counted { + Self::clear_session_counted_without_stream(connection_data, &token); + } + while connection_data.try_promote_soft_session_reservation() {} + } + + pub async fn connection_admission_for_session_activation( + &self, + username: &str, + max_connections: u32, + soft_connections: u16, + session_token: &str, + ) -> ConnectionAdmission { + if max_connections == 0 && soft_connections == 0 { + return ConnectionAdmission::allowed(Some(ConnectionKind::Normal)); + } + + let mut connections = self.write_connections().await; + let Some(connection_data) = connections.by_key.get_mut(username) else { + return ConnectionAdmission::allowed(Some(ConnectionKind::Normal)); + }; + connection_data.max_connections = max_connections; + connection_data.soft_connections = soft_connections; + + let Some(session_index) = connection_data.sessions.iter().position(|session| session.token == session_token) + else { + return self.check_connection_admission(username, connection_data); + }; + + // Existing counted session or any stream row for this token (including soft-preserved + // HLS/Catchup between segments): entitled to its slot. #807 switched this to + // active-only stream checks, so every LiveHls segment gap returned Exhausted + // and kick-evicted/terminated the same session (retry storm since v3.3.79). + if connection_data.sessions[session_index].lifecycle.is_counted() + || Self::session_has_stream(connection_data, session_token) + { + return ConnectionAdmission::allowed( + connection_data.sessions[session_index].connection_kind.or(Some(ConnectionKind::Normal)), + ); + } + + // Uncounted session with no stream row: run normal admission. + // Same-token soft-preserve is handled above via `session_has_stream` (3.3.78 semantics). + // Do not return Exhausted for own preserved rows — that forced self-eviction on HLS gaps. + let admission = self.check_connection_admission_with_counts( + username, + connection_data, + connection_data.effective_counts_for_admission(Some(session_token)), + ); + if admission.permission() == UserConnectionPermission::Allowed { + let session = &mut connection_data.sessions[session_index]; + Self::update_session_admission(session, admission.permission(), admission.kind()); + } + admission + } + + pub async fn release_unbound_session_reservation( + &self, + username: &str, + session_token: &str, + expected_transition_version: Option, + remove_session_if_unbound: bool, + ) { + let (connection_changed, user_removed, promotions, divergence_snapshot) = { + let mut user_connections = self.write_connections().await; + let (connection_changed, user_removed, promotions, divergence_snapshot) = { + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return; + }; + + if Self::session_has_stream(connection_data, session_token) { + return; + } + + let Some(session_index) = + connection_data.sessions.iter().position(|session| session.token == session_token) + else { + return; + }; + + if expected_transition_version + .is_some_and(|expected| connection_data.sessions[session_index].transition_version != expected) + { + return; + } + + let mut connection_changed = false; + if connection_data.sessions[session_index].lifecycle.is_counted() { + let kind = + connection_data.sessions[session_index].connection_kind.unwrap_or(ConnectionKind::Normal); + connection_data.decrement_kind(kind); + connection_data.sessions[session_index].expire(); + connection_changed = true; + } + connection_data.sessions[session_index].transition_version = + connection_data.sessions[session_index].transition_version.saturating_add(1); + + if remove_session_if_unbound { + connection_data.sessions.swap_remove(session_index); + } + + if connection_data.connections < connection_data.max_connections { + connection_data.granted_grace = false; + connection_data.grace_ts = 0; + } + + let mut promotions = Vec::new(); + while let Some(action) = connection_data.try_promote_soft_stream() { + let promoted_stream = + connection_data.streams.iter().find(|stream| stream.uid == action.uid).cloned(); + if let Some(stream) = promoted_stream.as_ref() { + Self::promote_session_for_stream(connection_data, stream); + } + promotions.push(action); + } + while connection_data.try_promote_soft_session_reservation() {} + + let user_removed = connection_data.connections == 0 + && connection_data.streams.is_empty() + && connection_data.sessions.is_empty(); + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); + + (connection_changed, user_removed, promotions, divergence_snapshot) + }; + + if user_removed { + user_connections.by_key.remove(username); + } + + (connection_changed, user_removed, promotions, divergence_snapshot) + }; + + self.log_divergence_snapshot(divergence_snapshot).await; + + if connection_changed || user_removed { + self.log_active_user().await; + } + for action in promotions { + self.emit_promotion_update(username, action).await; + } + } + + pub async fn release_session_streams_and_counted_reservation(&self, username: &str, session_token: &str) -> bool { + let (connection_changed, user_removed, promotions, divergence_snapshot) = { + let mut user_connections = self.write_connections().await; + let (connection_changed, user_removed, promotions, divergence_snapshot) = { + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return false; + }; + + let counted_kind = connection_data + .sessions + .iter() + .find(|session| session.token == session_token && session.lifecycle.is_counted()) + .and_then(|session| session.connection_kind); + let (_removed_streams, mut connection_changed) = + connection_data.remove_streams_for_session_and_release_counted(session_token, counted_kind); + Self::clear_session_counted_without_stream(connection_data, session_token); + + if connection_data.connections < connection_data.max_connections { + connection_data.granted_grace = false; + connection_data.grace_ts = 0; + } + + let promotions = Self::collect_promotions_after_capacity_release(connection_data); + let user_removed = connection_data.connections == 0 + && connection_data.streams.is_empty() + && connection_data.sessions.is_empty(); + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); + connection_changed |= !promotions.is_empty(); + + (connection_changed, user_removed, promotions, divergence_snapshot) + }; + + if user_removed { + user_connections.by_key.remove(username); + } + + (connection_changed, user_removed, promotions, divergence_snapshot) + }; + + self.log_divergence_snapshot(divergence_snapshot).await; + + if connection_changed || user_removed { + self.log_active_user().await; + } + for action in promotions { + self.emit_promotion_update(username, action).await; + } + connection_changed || user_removed + } +} diff --git a/backend/session/src/active_user_manager/connections.rs b/backend/session/src/active_user_manager/connections.rs new file mode 100644 index 000000000..ea75b90fa --- /dev/null +++ b/backend/session/src/active_user_manager/connections.rs @@ -0,0 +1,1004 @@ +use super::{ + get_adaptive_session_ttl_secs, release_session_addr, ActiveUserConnectionParams, ActiveUserManager, + AdmissionRejectionReason, ConnectionAdmission, DeferredWakes, PlaybackLifecycle, ReleasedConnection, + SessionChangeSignal, SessionIdentity, SessionProviderHeaders, StreamRequestClaim, StreamRequestDetach, UserSession, + DIVERGENCE_CACHE_CAPACITY, ENDED_SESSION_RECREATE_BLOCK, +}; +use crate::{ + active_provider_manager::ConnectionKind, connection_manager::CleanupEvent, stream::uses_direct_body_idle_timeout, + ActiveProviderManager, EventManager, +}; +use arc_swap::ArcSwapOption; +use lru::LruCache; +use shared::{ + defaults::{default_grace_period_millis, default_grace_period_timeout_secs}, + model::{ + ActiveUserConnectionChange, CustomVideoStreamType, EventMessage, StreamChannel, StreamInfo, + UserConnectionPermission, VirtualId, + }, + utils::{current_time_secs, sanitize_sensitive_info, strip_port, Internable}, +}; +use std::{ + collections::{BinaryHeap, HashMap, HashSet}, + net::SocketAddr, + sync::{ + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, + Arc, + }, + time::Instant, +}; +use tokio::sync::{mpsc, Mutex, Notify, RwLock}; +use tokio_util::sync::CancellationToken; +use tuliprox_core::{model::Config, utils::debug_if_enabled}; +use tuliprox_repository::GeoIp; + +pub(super) fn get_grace_options(config: &Config) -> (u64, u64) { + let (grace_period_millis, grace_period_timeout_secs) = + config.reverse_proxy.as_ref().and_then(|r| r.stream.as_ref()).map_or_else( + || (default_grace_period_millis(), default_grace_period_timeout_secs()), + |s| (s.grace_period_millis, s.grace_period_timeout_secs), + ); + (grace_period_millis, grace_period_timeout_secs) +} + +pub(super) fn decide_connection_kind( + counts: UserConnectionCounts, + max_connections: u32, + soft_connections: u16, +) -> Option { + if max_connections == 0 || counts.normal < max_connections { + return Some(ConnectionKind::Normal); + } + if soft_connections > 0 && counts.soft < soft_connections { + return Some(ConnectionKind::Soft); + } + None +} + +impl DeferredWakes { + pub(super) fn push(&self, notify: Arc) { + self.pending.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push(notify); + self.has_pending.store(true, Ordering::Release); + } + + pub(super) fn wake(&self) { + if !self.has_pending.swap(false, Ordering::AcqRel) { + return; + } + let pending = std::mem::take(&mut *self.pending.lock().unwrap_or_else(std::sync::PoisonError::into_inner)); + for notify in pending { + notify.notify_waiters(); + } + } +} + +impl SessionChangeSignal { + pub(super) fn new(deferred: &Arc) -> Self { + Self { notify: Arc::new(Notify::new()), deferred: Some(Arc::clone(deferred)), owner: true } + } + + pub(super) fn signal(&self) { + match &self.deferred { + Some(deferred) => deferred.push(Arc::clone(&self.notify)), + None => self.notify.notify_waiters(), + } + } +} + +impl SessionProviderHeaders { + pub(super) fn for_target(session: &UserSession, target_url: &str) -> Self { + session + .provider_session_headers_for(target_url) + .map_or(Self::NoHeaders, |headers| Self::Headers(headers.into_owned())) + } +} + +/// `host:port` of a URL, used to scope provider session cookies. +pub(super) fn url_host_key(url: &str) -> Option { + let url = url::Url::parse(url).ok()?; + Some(format!("{}://{}:{}", url.scheme(), url.host_str()?, url.port_or_known_default()?)) +} + +impl UserSession { + pub const fn identity(&self) -> SessionIdentity { + SessionIdentity { incarnation: self.incarnation, binding_generation: self.binding_generation } + } + + pub(super) fn switch_provider(&mut self, provider: Arc) { + self.provider = provider; + self.binding_generation = self.binding_generation.wrapping_add(1); + self.change_signal.signal(); + self.provider_session_headers.clear(); + self.provider_session_headers_host = None; + self.clear_provider_session_cookies(); + } + + pub(super) fn expire(&mut self) { + self.lifecycle = PlaybackLifecycle::Expired; + self.change_signal.signal(); + } + + /// Whether the session may still serve requests: neither expired nor exhausted. + pub(super) fn is_live(&self) -> bool { + !matches!(self.lifecycle, PlaybackLifecycle::Expired) && self.permission != UserConnectionPermission::Exhausted + } +} + +#[derive(Debug, Default, Clone, Copy)] +pub(super) struct UserConnectionCounts { + pub(super) normal: u32, + pub(super) soft: u16, +} + +impl ConnectionAdmission { + pub fn allowed(kind: Option) -> Self { + Self { permission: UserConnectionPermission::Allowed, kind, rejection_reason: None } + } + + pub fn grace_period(kind: Option) -> Self { + Self { permission: UserConnectionPermission::GracePeriod, kind, rejection_reason: None } + } + + pub fn exhausted(reason: AdmissionRejectionReason, kind: Option) -> Self { + Self { permission: UserConnectionPermission::Exhausted, kind, rejection_reason: Some(reason) } + } + + pub fn kind(&self) -> Option { self.kind } + + pub fn rejection_reason(&self) -> Option { self.rejection_reason } + + pub fn is_reentry_suppressed(&self) -> bool { + self.rejection_reason == Some(AdmissionRejectionReason::RecentEvictionReentry) + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct PromotionAction { + pub(super) addr: SocketAddr, + pub(super) uid: u32, + pub(super) new_priority: i8, +} + +impl StreamRequestDetach { + pub fn request_was_detached(&self) -> bool { !matches!(self, Self::NotFound) } + + pub fn stream_info(&self) -> Option<&StreamInfo> { + match self { + Self::NotFound => None, + Self::Retained(stream) | Self::Preserved(stream) | Self::Removed(stream) => Some(stream), + } + } + + pub(super) fn into_removed(self) -> Option { + match self { + Self::Removed(stream) => Some(stream), + Self::NotFound | Self::Retained(_) | Self::Preserved(_) => None, + } + } +} + +#[derive(Debug)] +pub(super) struct UserConnectionData { + pub(super) max_connections: u32, + pub(super) soft_connections: u16, + pub(super) counts: UserConnectionCounts, + pub(super) connections: u32, + pub(super) granted_grace: bool, + pub(super) grace_ts: u64, + pub(super) sessions: Vec, + pub(super) streams: Vec, + pub(super) stream_request_claims: HashMap, + pub(super) stream_kinds: HashMap, + pub(super) stream_normal_priorities: HashMap, + pub(super) ts: u64, +} + +impl UserConnectionData { + pub(super) fn new(connections: u32, max_connections: u32, soft_connections: u16) -> Self { + Self { + max_connections, + soft_connections, + counts: UserConnectionCounts::default(), + connections, + granted_grace: false, + grace_ts: 0, + sessions: Vec::new(), + streams: Vec::new(), + stream_request_claims: HashMap::new(), + stream_kinds: HashMap::new(), + stream_normal_priorities: HashMap::new(), + ts: current_time_secs(), + } + } + + pub(super) fn add_session(&mut self, session: UserSession) { + self.gc(); + self.sessions.push(session); + } + + pub(super) fn release_addr_from_sessions(&mut self, addr: &SocketAddr) -> HashMap> { + let mut migrated_addrs = HashMap::new(); + for session in &mut self.sessions { + if session.addr == *addr || session.active_addrs.contains(addr) { + migrated_addrs.insert(session.token.clone(), release_session_addr(session, addr)); + } + } + migrated_addrs + } + + pub(super) fn release_addr_from_stream_session( + &mut self, + addr: &SocketAddr, + uid: u32, + ) -> HashMap> { + let Some(token) = + self.streams.iter().find(|stream| stream.uid == uid).and_then(|stream| stream.session_token.as_deref()) + else { + return HashMap::new(); + }; + if self.streams.iter().any(|stream| { + stream.uid != uid + && !stream.preserved + && stream.addr == *addr + && stream.session_token.as_deref() == Some(token) + }) { + return HashMap::new(); + } + let Some(session) = self.sessions.iter_mut().find(|session| session.token == token) else { + return HashMap::new(); + }; + HashMap::from([(session.token.clone(), release_session_addr(session, addr))]) + } + + pub(super) fn attach_stream_request(&mut self, request_uid: u32, stream_uid: u32, addr: SocketAddr) { + self.stream_request_claims.insert(request_uid, StreamRequestClaim { stream_uid, addr }); + } + + pub(super) fn detach_stream_request(&mut self, request_uid: u32) -> Option<(u32, bool)> { + let claim = self.stream_request_claims.remove(&request_uid)?; + let is_last = !self.stream_request_claims.values().any(|other| other.stream_uid == claim.stream_uid); + Some((claim.stream_uid, is_last)) + } + + pub(super) fn increment_kind(&mut self, kind: ConnectionKind) { + self.connections = self.connections.saturating_add(1); + match kind { + ConnectionKind::Normal => { + self.counts.normal = self.counts.normal.saturating_add(1); + } + ConnectionKind::Soft => { + self.counts.soft = self.counts.soft.saturating_add(1); + } + } + } + + pub(super) fn decrement_kind(&mut self, kind: ConnectionKind) { + self.connections = self.connections.saturating_sub(1); + match kind { + ConnectionKind::Normal => { + self.counts.normal = self.counts.normal.saturating_sub(1); + } + ConnectionKind::Soft => { + self.counts.soft = self.counts.soft.saturating_sub(1); + } + } + } +} + +pub(super) fn create_socket_reentry_guard_key(username: &str, client_ip: &str, virtual_id: VirtualId) -> String { + shared::concat_string!(username, "|", client_ip, "|", &virtual_id.to_string()) +} + +impl UserConnections { + pub(super) fn mark_sessions_ended>(&mut self, tokens: I) { + let now = Instant::now(); + self.ended_sessions.retain(|_, expires_at| *expires_at > now); + let expires_at = now + ENDED_SESSION_RECREATE_BLOCK; + let tokens = tokens.into_iter().collect::>(); + for session in self.by_key.values().flat_map(|data| data.sessions.iter()) { + if tokens.contains(&session.token) { + session.change_signal.signal(); + } + } + self.ended_sessions.extend(tokens.into_iter().map(|token| (token, expires_at))); + } + + pub(super) fn is_token_ended(&self, token: &str) -> bool { + self.ended_sessions.get(token).is_some_and(|expiry| *expiry > Instant::now()) + } + + /// Returns the session while it may still serve requests: not ended, expired or exhausted. + pub(super) fn live_session(&self, username: &str, token: &str) -> Option<(&UserConnectionData, &UserSession)> { + if self.is_token_ended(token) { + return None; + } + let data = self.by_key.get(username)?; + let session = data.sessions.iter().find(|session| session.token == token && session.is_live())?; + Some((data, session)) + } + + /// Returns the live session only while this exact incarnation and binding may still serve requests. + pub(super) fn current_session( + &self, + username: &str, + token: &str, + identity: SessionIdentity, + ) -> Option<(&UserConnectionData, &UserSession)> { + self.live_session(username, token).filter(|(_, session)| session.identity() == identity) + } +} + +#[derive(Clone, Copy, Debug)] +pub(super) struct RecentWinnerProtection { + pub(super) protected_addr: SocketAddr, + pub(super) expires_at: Instant, +} + +#[derive(Debug, Default)] +pub(super) struct UserConnections { + pub(super) kicked: HashMap, + pub(super) recently_evicted_sessions: HashMap, + /// Session tokens ended by eviction, kick or explicit terminate, with their expiry. A sealed + /// playlist token of such a session must not recreate it; an expired session may. + pub(super) ended_sessions: HashMap, + pub(super) recent_socket_reentry_guards: HashMap, + pub(super) by_key: HashMap, + pub(super) key_by_addr: HashMap, +} + +#[derive(Clone, Debug, Default)] +pub(super) struct SocketRegistration { + pub(super) usernames: HashSet, + pub(super) ts: u64, +} + +impl SocketRegistration { + pub(super) fn anonymous() -> Self { Self { usernames: HashSet::new(), ts: current_time_secs() } } + + pub(super) fn add_user(&mut self, username: &str, ts: u64) { + if !username.is_empty() && !self.usernames.contains(username) { + self.usernames.insert(username.to_string()); + } + self.ts = ts; + } + + pub(super) fn remove_user(&mut self, username: &str) -> bool { + self.usernames.remove(username); + self.usernames.is_empty() + } + + pub(super) fn is_empty(&self) -> bool { self.usernames.is_empty() } + + pub(super) fn primary_username(&self) -> Option<&str> { self.usernames.iter().map(String::as_str).min() } + + pub(super) fn all_usernames(&self) -> Vec { self.usernames.iter().cloned().collect() } + + pub(super) fn into_usernames(self) -> HashSet { self.usernames } +} + +/// Write access to `UserConnections` that delivers session change signals after release. +/// +/// Fields drop in declaration order: `guard` releases the write lock before `_wake` notifies +/// the waiters signalled during this write. +pub(super) struct UserConnectionsWriteGuard<'a> { + pub(super) guard: tokio::sync::RwLockWriteGuard<'a, UserConnections>, + pub(super) _wake: WakeDeferred<'a>, +} + +pub(super) struct WakeDeferred<'a>(pub(super) &'a DeferredWakes); + +impl Drop for WakeDeferred<'_> { + fn drop(&mut self) { self.0.wake(); } +} + +impl<'a> UserConnectionsWriteGuard<'a> { + pub(super) fn new(guard: tokio::sync::RwLockWriteGuard<'a, UserConnections>, wakes: &'a DeferredWakes) -> Self { + Self { guard, _wake: WakeDeferred(wakes) } + } +} + +impl std::ops::Deref for UserConnectionsWriteGuard<'_> { + type Target = UserConnections; + fn deref(&self) -> &Self::Target { &self.guard } +} + +impl std::ops::DerefMut for UserConnectionsWriteGuard<'_> { + fn deref_mut(&mut self) -> &mut Self::Target { &mut self.guard } +} + +impl ActiveUserManager { + pub(super) async fn write_connections(&self) -> UserConnectionsWriteGuard<'_> { + UserConnectionsWriteGuard::new(self.connections.write().await, &self.deferred_wakes) + } + + pub fn shutdown(&self) { self.adaptive_expiry_cancel.cancel(); } + + pub(super) fn lookup_country(&self, client_ip: &str) -> Option { + let geoip = self.geo_ip.load(); + (*geoip).as_ref().and_then(|geoip_db| geoip_db.lookup(&strip_port(client_ip))) + } + + pub fn new(config: &Config, geoip: &Arc>, event_manager: &Arc) -> Self { + let log_active_user: bool = config.log.as_ref().is_some_and(|l| l.log_active_user); + let (grace_period_millis, grace_period_timeout_secs) = get_grace_options(config); + + Self { + grace_period_millis: AtomicU64::new(grace_period_millis), + grace_period_timeout_secs: AtomicU64::new(grace_period_timeout_secs), + adaptive_session_ttl_secs: AtomicU64::new(get_adaptive_session_ttl_secs(config)), + log_active_user: AtomicBool::new(log_active_user), + next_user_agent_stream_index: AtomicU64::new(1), + connections: RwLock::new(UserConnections::default()), + adaptive_expiry_queue: Arc::new(Mutex::new(BinaryHeap::new())), + adaptive_expiry_index: Arc::new(Mutex::new(HashMap::new())), + adaptive_expiry_notify: Arc::new(Notify::new()), + adaptive_expiry_cancel: CancellationToken::new(), + adaptive_expiry_worker_started: AtomicBool::new(false), + gc_ts: Some(AtomicU64::new(current_time_secs())), + geo_ip: Arc::clone(geoip), + event_manager: Arc::clone(event_manager), + last_logged_user_count: AtomicUsize::new(0), + last_logged_user_connection_count: AtomicUsize::new(0), + cleanup_tx: tokio::sync::OnceCell::new(), + provider_manager: tokio::sync::OnceCell::new(), + transition_gates: Mutex::new(HashMap::new()), + pending_provider_releases: Mutex::new(HashMap::new()), + dropped_cleanup_events: AtomicU64::new(0), + reentry_suppressed_total: AtomicU64::new(0), + deferred_wakes: Arc::default(), + divergence_cache: Mutex::new(LruCache::new(DIVERGENCE_CACHE_CAPACITY)), + divergence_cooldown_secs: 300, + } + } + + pub(super) fn should_reuse_stream_for_session( + existing_stream: &StreamInfo, + incoming_channel: &StreamChannel, + ) -> bool { + existing_stream.channel.item_type.requires_provider_affinity() + || incoming_channel.item_type.requires_provider_affinity() + } + + pub fn set_cleanup_sender(&self, tx: mpsc::Sender) { let _ = self.cleanup_tx.set(tx); } + + pub fn set_provider_manager(&self, provider_manager: Arc) { + let _ = self.provider_manager.set(provider_manager); + } + + /// Atomically removes all user requests and sessions during terminal shutdown. + pub async fn drain_for_shutdown(&self) -> Vec { + let connections = std::mem::take(&mut *self.write_connections().await); + self.adaptive_expiry_queue.lock().await.clear(); + self.adaptive_expiry_index.lock().await.clear(); + self.transition_gates.lock().await.clear(); + connections + .by_key + .into_values() + .flat_map(|data| data.streams.into_iter().filter(|stream| !stream.preserved)) + .collect() + } + + /// Releases an active stream for the given socket address without removing the + /// socket registration (`key_by_addr`). This is used when a stream ends while + /// the underlying HTTP connection may still remain open. + pub async fn release_stream(&self, addr: &SocketAddr) -> Option { + self.release_stream_inner(addr, None).await.into_removed() + } + + pub async fn release_stream_by_uid(&self, addr: &SocketAddr, stream_uid: u32) -> Option { + self.release_stream_request_by_uid(addr, stream_uid).await.into_removed() + } + + #[allow(clippy::too_many_lines)] + pub(super) async fn release_stream_inner(&self, addr: &SocketAddr, stream_uid: Option) -> StreamRequestDetach { + let ( + removed_stream, + username, + expiry_entry, + preserved_update, + connection_changed, + promotion, + divergence_snapshot, + ) = { + let mut user_connections = self.write_connections().await; + + let username = if let Some(uid) = stream_uid { + user_connections.by_key.iter().find_map(|(username, connection_data)| { + connection_data + .stream_request_claims + .get(&uid) + .is_some_and(|claim| { + claim.addr == *addr + && connection_data + .streams + .iter() + .any(|stream| !stream.preserved && stream.uid == claim.stream_uid) + }) + .then(|| username.clone()) + }) + } else { + let mut matching_users = user_connections + .by_key + .iter() + .filter(|(_, connection_data)| { + connection_data.streams.iter().any(|stream| !stream.preserved && stream.addr == *addr) + }) + .map(|(username, _)| username.clone()); + let first = matching_users.next(); + let second = matching_users.next(); + if second.is_some() { + None + } else { + first + } + }; + let Some(username) = username else { + return StreamRequestDetach::NotFound; + }; + + let mut removed_stream = None; + let mut expiry_entry = None; + let mut preserved_update = None; + let mut connection_changed = false; + let mut promotion = None; + if let Some(connection_data) = user_connections.by_key.get_mut(&username) { + let stream_uid = if let Some(request_uid) = stream_uid { + let Some((display_uid, is_last)) = connection_data.detach_stream_request(request_uid) else { + return StreamRequestDetach::NotFound; + }; + if !is_last { + let has_claim_on_addr = connection_data + .stream_request_claims + .values() + .any(|claim| claim.stream_uid == display_uid && claim.addr == *addr); + let migrated_session_addrs = if has_claim_on_addr { + HashMap::new() + } else { + connection_data.release_addr_from_stream_session(addr, display_uid) + }; + let next_addr = connection_data + .stream_request_claims + .values() + .find(|claim| claim.stream_uid == display_uid) + .map(|claim| claim.addr); + if let Some(stream) = + connection_data.streams.iter_mut().find(|stream| stream.uid == display_uid) + { + stream.addr = next_addr + .or_else(|| { + stream + .session_token + .as_deref() + .and_then(|token| migrated_session_addrs.get(token)) + .copied() + .flatten() + }) + .unwrap_or(stream.addr); + stream.ts = current_time_secs(); + } + let retained = connection_data.streams.iter().find(|stream| stream.uid == display_uid).cloned(); + return retained.map_or(StreamRequestDetach::NotFound, StreamRequestDetach::Retained); + } + Some(display_uid) + } else { + None + }; + let migrated_session_addrs = if let Some(uid) = stream_uid { + connection_data.release_addr_from_stream_session(addr, uid) + } else { + connection_data.release_addr_from_sessions(addr) + }; + if let Some(stream_idx) = connection_data.streams.iter().position(|stream| { + !stream.preserved + && stream_uid.map_or(stream.addr == *addr, |uid| stream.uid == uid && stream.addr == *addr) + }) { + let migrated_addr = connection_data.streams[stream_idx] + .session_token + .as_deref() + .and_then(|token| migrated_session_addrs.get(token)) + .copied() + .flatten(); + if let Some(next_addr) = migrated_addr { + connection_data.streams[stream_idx].addr = next_addr; + connection_data.streams[stream_idx].ts = current_time_secs(); + } else if Self::should_preserve_session_stream(&connection_data.streams[stream_idx]) { + let preserved_session_token = connection_data.streams[stream_idx].session_token.clone(); + if let Some(entry) = self.build_preserved_stream_expiry( + &username, + &connection_data.streams[stream_idx], + &connection_data.sessions, + ) { + if let Some(kind) = + connection_data.stream_kinds.remove(&connection_data.streams[stream_idx].uid) + { + connection_data.decrement_kind(kind); + connection_changed = true; + } + connection_data.stream_normal_priorities.remove(&connection_data.streams[stream_idx].uid); + if let Some(session_token) = preserved_session_token.as_deref() { + Self::clear_session_counted(connection_data, session_token); + } + connection_data.streams[stream_idx].preserved = true; + preserved_update = Some(connection_data.streams[stream_idx].clone()); + expiry_entry = Some(entry); + } else { + removed_stream = Some(connection_data.streams.swap_remove(stream_idx)); + } + } else { + removed_stream = Some(connection_data.streams.swap_remove(stream_idx)); + } + if let Some(removed_stream) = removed_stream.as_ref() { + connection_data.remove_stream_request_claims(removed_stream.uid); + if let Some(kind) = connection_data.stream_kinds.remove(&removed_stream.uid) { + connection_data.decrement_kind(kind); + } + connection_data.stream_normal_priorities.remove(&removed_stream.uid); + connection_changed = true; + } + if connection_data.connections < connection_data.max_connections { + connection_data.granted_grace = false; + connection_data.grace_ts = 0; + } + if removed_stream.is_some() { + if let Some(action) = connection_data.try_promote_soft_stream() { + let promoted_stream = + connection_data.streams.iter().find(|stream| stream.uid == action.uid).cloned(); + if let Some(stream) = promoted_stream.as_ref() { + Self::promote_session_for_stream(connection_data, stream); + } + promotion = Some(action); + } + if let Some(session_token) = + removed_stream.as_ref().and_then(|stream| stream.session_token.as_deref()) + { + Self::clear_session_counted_without_stream(connection_data, session_token); + } + while connection_data.try_promote_soft_session_reservation() {} + } + } + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, &username); + ( + removed_stream, + username, + expiry_entry, + preserved_update, + connection_changed, + promotion, + divergence_snapshot, + ) + } else { + (None, username, None, None, false, None, None) + } + }; + + self.log_divergence_snapshot(divergence_snapshot).await; + + if let Some(entry) = expiry_entry { + self.enqueue_adaptive_expiry(entry).await; + } + + if let Some(stream_info) = preserved_update.as_ref() { + self.event_manager + .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info.clone()))); + } + + if connection_changed { + if !username.is_empty() { + debug_if_enabled!( + "Released stream for user {username} at {}", + sanitize_sensitive_info(&addr.to_string()) + ); + } + self.log_active_user().await; + } + + if let Some(action) = promotion { + self.emit_promotion_update(&username, action).await; + } + + if let Some(stream_info) = removed_stream { + StreamRequestDetach::Removed(stream_info) + } else if let Some(stream_info) = preserved_update { + StreamRequestDetach::Preserved(stream_info) + } else { + StreamRequestDetach::NotFound + } + } + + #[allow(clippy::too_many_lines)] + pub(super) async fn release_connection_inner( + &self, + addr: &SocketAddr, + preserve_session_streams: bool, + ) -> ReleasedConnection { + let ( + addr_removed, + connection_count_changed, + disconnected_users, + removed_streams, + expiry_entries, + preserved_updates, + promotions, + ) = { + let mut user_connections = self.write_connections().await; + + let registration = user_connections.key_by_addr.remove(addr); + let had_registration = registration.is_some(); + let mut disconnected_users: HashSet = + registration.map(SocketRegistration::into_usernames).unwrap_or_default(); + disconnected_users.extend( + user_connections + .by_key + .iter() + .filter(|(_, connection_data)| { + connection_data.has_session_addr(addr) + || connection_data.streams.iter().any(|stream| stream.addr == *addr) + }) + .map(|(username, _)| username.clone()), + ); + + let mut removed_streams = Vec::new(); + let mut expiry_entries = Vec::new(); + let mut preserved_updates = Vec::new(); + let mut promotions = Vec::new(); + let mut connection_count_changed = false; + for username in &disconnected_users { + if let Some(connection_data) = user_connections.by_key.get_mut(username) { + let previous_connection_count = connection_data.connections; + let migrated_session_addrs = connection_data.release_addr_from_sessions(addr); + connection_data.stream_request_claims.retain(|_, claim| claim.addr != *addr); + let mut remaining_streams = Vec::with_capacity(connection_data.streams.len()); + let mut released_kinds = Vec::new(); + let mut removed_session_tokens = HashSet::new(); + let mut preserved_session_tokens = Vec::new(); + let now = current_time_secs(); + for mut stream_info in connection_data.streams.drain(..) { + if stream_info.addr == *addr { + let migrated_addr = stream_info + .session_token + .as_deref() + .and_then(|token| migrated_session_addrs.get(token)) + .copied() + .flatten(); + if let Some(next_addr) = migrated_addr { + stream_info.addr = next_addr; + stream_info.ts = now; + remaining_streams.push(stream_info); + } else if preserve_session_streams && Self::should_preserve_session_stream(&stream_info) { + if let Some(entry) = self.build_preserved_stream_expiry( + username, + &stream_info, + &connection_data.sessions, + ) { + if let Some(kind) = connection_data.stream_kinds.remove(&stream_info.uid) { + released_kinds.push(kind); + } + connection_data.stream_normal_priorities.remove(&stream_info.uid); + if let Some(token) = stream_info.session_token.as_ref() { + preserved_session_tokens.push(token.clone()); + } + if !stream_info.preserved { + stream_info.preserved = true; + preserved_updates.push(stream_info.clone()); + } + expiry_entries.push(entry); + remaining_streams.push(stream_info); + } else { + if let Some(kind) = connection_data.stream_kinds.remove(&stream_info.uid) { + released_kinds.push(kind); + } + connection_data.stream_normal_priorities.remove(&stream_info.uid); + if let Some(token) = stream_info.session_token.as_ref() { + removed_session_tokens.insert(token.clone()); + } + removed_streams.push(stream_info); + } + } else { + if let Some(kind) = connection_data.stream_kinds.remove(&stream_info.uid) { + released_kinds.push(kind); + } + connection_data.stream_normal_priorities.remove(&stream_info.uid); + if let Some(token) = stream_info.session_token.as_ref() { + removed_session_tokens.insert(token.clone()); + } + removed_streams.push(stream_info); + } + } else { + remaining_streams.push(stream_info); + } + } + connection_data.streams = remaining_streams; + let streams = &connection_data.streams; + connection_data + .stream_request_claims + .retain(|_, claim| streams.iter().any(|stream| stream.uid == claim.stream_uid)); + if !preserve_session_streams && !removed_session_tokens.is_empty() { + connection_data.sessions.retain(|session| !removed_session_tokens.contains(&session.token)); + } + for kind in released_kinds { + connection_data.decrement_kind(kind); + } + while let Some(action) = connection_data.try_promote_soft_stream() { + let promoted_stream = + connection_data.streams.iter().find(|stream| stream.uid == action.uid).cloned(); + if let Some(stream) = promoted_stream.as_ref() { + Self::promote_session_for_stream(connection_data, stream); + } + promotions.push((username.clone(), action)); + } + for session_token in &removed_session_tokens { + Self::clear_session_counted_without_stream(connection_data, session_token); + } + for session_token in &preserved_session_tokens { + Self::clear_session_counted(connection_data, session_token); + } + while connection_data.try_promote_soft_session_reservation() {} + + if connection_data.connections < connection_data.max_connections { + connection_data.granted_grace = false; + connection_data.grace_ts = 0; + } + connection_count_changed |= connection_data.connections != previous_connection_count; + } + } + let state_changed = had_registration || !disconnected_users.is_empty(); + ( + state_changed, + connection_count_changed, + disconnected_users, + removed_streams, + expiry_entries, + preserved_updates, + promotions, + ) + }; + + for entry in expiry_entries { + self.enqueue_adaptive_expiry(entry).await; + } + + for stream_info in preserved_updates { + self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + } + + for username in &disconnected_users { + if !username.is_empty() { + debug_if_enabled!( + "Released connection for user {username} at {}", + sanitize_sensitive_info(&addr.to_string()) + ); + } + } + if connection_count_changed { + self.log_active_user().await; + } + if addr_removed { + for (username, action) in promotions { + self.emit_promotion_update(&username, action).await; + } + } + + ReleasedConnection { + addr_removed, + removed_streams, + disconnected_users: disconnected_users.into_iter().collect(), + } + } + + pub async fn release_connection(&self, addr: &SocketAddr) -> ReleasedConnection { + let released = self.release_connection_inner(addr, true).await; + // divergence check after connection release + if released.addr_removed { + for username in &released.disconnected_users { + self.check_and_log_divergence_for_user(username).await; + } + } + released + } + + pub async fn release_connection_as_kicked(&self, addr: &SocketAddr) -> ReleasedConnection { + let released = self.release_connection_inner(addr, false).await; + // divergence check after connection release + if released.addr_removed { + for username in &released.disconnected_users { + self.check_and_log_divergence_for_user(username).await; + } + } + released + } + + pub fn update_config(&self, config: &Config) { + let log_active_user = config.log.as_ref().is_some_and(|l| l.log_active_user); + let (grace_period_millis, grace_period_timeout_secs) = get_grace_options(config); + self.grace_period_millis.store(grace_period_millis, Ordering::Relaxed); + self.grace_period_timeout_secs.store(grace_period_timeout_secs, Ordering::Relaxed); + self.adaptive_session_ttl_secs.store(get_adaptive_session_ttl_secs(config), Ordering::Relaxed); + self.log_active_user.store(log_active_user, Ordering::Relaxed); + } + + pub async fn user_connections(&self, username: &str) -> u32 { + if let Some(connection_data) = self.connections.read().await.by_key.get(username) { + return connection_data.connections; + } + 0 + } + + pub async fn update_stream_detail( + &self, + addr: &SocketAddr, + video_type: CustomVideoStreamType, + ) -> Option { + let uid = { + let connections = self.connections.read().await; + let mut streams = + connections.by_key.values().flat_map(|data| &data.streams).filter(|stream| &stream.addr == addr); + let uid = streams.next()?.uid; + if streams.next().is_some() { + return None; + } + uid + }; + self.update_stream_detail_by_uid(uid, video_type).await + } + + pub async fn update_stream_detail_by_uid(&self, uid: u32, video_type: CustomVideoStreamType) -> Option { + let mut user_connections = self.write_connections().await; + for connection_data in user_connections.by_key.values_mut() { + for stream in &mut connection_data.streams { + if stream.uid == uid { + // IMPORTANT: `resolve_disconnect_reason` in connection_manager.rs parses + // `channel.title` back via `CustomVideoStreamType::from_str` to determine QoS + // disconnect reasons. If these values change, update that function too. + stream.provider = "tuliprox".intern(); + stream.channel.title = video_type.to_string().into(); + stream.channel.group = "".intern(); + stream.channel.technical = Some(Self::custom_stream_technical_info()); + return Some(stream.clone()); + } + } + } + None + } + + pub async fn add_connection(&self, addr: &SocketAddr) { + self.gc(); + let mut user_connections = self.write_connections().await; + user_connections + .key_by_addr + .entry(*addr) + .and_modify(|registration| registration.ts = current_time_secs()) + .or_insert_with(SocketRegistration::anonymous); + } + + pub async fn update_connection(&self, update: ActiveUserConnectionParams<'_>) -> Option { + self.update_connection_with_session_registration(update, None).await + } + + pub(super) fn addr_has_direct_body_stream(connections: &UserConnections, addr: &SocketAddr) -> bool { + let Some(registration) = connections.key_by_addr.get(addr) else { + return false; + }; + registration.usernames.iter().any(|username| { + connections.by_key.get(username).is_some_and(|connection_data| { + connection_data + .streams + .iter() + .any(|stream| stream.addr == *addr && uses_direct_body_idle_timeout(&stream.channel)) + }) + }) + } + + pub async fn get_username_for_addr(&self, addr: &SocketAddr) -> Option { + self.connections + .read() + .await + .key_by_addr + .get(addr) + .and_then(SocketRegistration::primary_username) + .map(str::to_string) + } + + pub async fn get_usernames_for_addr(&self, addr: &SocketAddr) -> Vec { + self.connections.read().await.key_by_addr.get(addr).map(SocketRegistration::all_usernames).unwrap_or_default() + } +} diff --git a/backend/session/src/active_user_manager/eviction.rs b/backend/session/src/active_user_manager/eviction.rs new file mode 100644 index 000000000..1f01f3683 --- /dev/null +++ b/backend/session/src/active_user_manager/eviction.rs @@ -0,0 +1,192 @@ +use super::{ + create_socket_reentry_guard_key, uses_session_reentry_guard, ActiveUserManager, RecentWinnerProtection, + SocketRegistration, +}; +use shared::{model::VirtualId, utils::current_time_secs}; +use std::{ + collections::HashMap, + net::SocketAddr, + time::{Duration, Instant}, +}; + +impl ActiveUserManager { + pub async fn get_eviction_candidates(&self, username: &str, _client_ip: &str) -> Vec { + let connections = self.connections.read().await; + let Some(connection_data) = connections.by_key.get(username) else { + return Vec::new(); + }; + let mut addr_counts = HashMap::new(); + for stream in &connection_data.streams { + // The stream kind is the charged slot. Session lifecycle may lag behind + // the stream registration, so it cannot determine eviction eligibility. + if !stream.preserved && connection_data.stream_kinds.contains_key(&stream.uid) { + addr_counts + .entry(stream.addr) + .and_modify(|count: &mut u8| *count = count.saturating_add(1)) + .or_insert(1_u8); + } + } + let candidates: Vec<_> = connection_data + .streams + .iter() + .filter(|stream| stream.preserved || connection_data.stream_kinds.contains_key(&stream.uid)) + .filter(|stream| { + let addr_count = addr_counts.get(&stream.addr).copied().unwrap_or(0); + if stream.preserved { + // Preserved streams are always valid eviction candidates — they hold no counted + // slot. addr_count is 0 for preserved-only addresses, 1+ for addresses with + // counted competition. Either way they can be evicted. + true + } else { + // Non-preserved streams: only on singleton counted addresses. + // addr_count 0 = no counted streams at this address (shouldn't happen since + // non-preserved streams aren't preserved, but addr_count would be >= 1). + // addr_count 1 = single counted stream at address — candidate. + // addr_count > 1 = multiple counted streams — not a singleton, not candidate. + addr_count == 1 + } + }) + .map(|s| crate::EvictionCandidate { + addr: s.addr, + client_ip: s.client_ip.clone(), + virtual_id: shared::model::VirtualId::new(s.channel.virtual_id), + ts: s.ts, + uid: s.uid, + }) + .collect(); + candidates + } + + pub async fn is_user_blocked_for_stream(&self, username: &str, virtual_id: VirtualId) -> bool { + let connections = self.connections.read().await; + let now = current_time_secs(); + matches!(connections.kicked.get(username), Some((expires_at, vid)) if *vid == virtual_id && *expires_at > now) + } + + pub async fn recently_evicted_session_protected_addr(&self, session_token: &str) -> Option { + let connections = self.connections.read().await; + let protection = connections.recently_evicted_sessions.get(session_token)?; + if protection.expires_at > Instant::now() { + return Some(protection.protected_addr); + } + + let username = connections.by_key.iter().find_map(|(username, connection_data)| { + connection_data.sessions.iter().any(|session| session.token == session_token).then_some(username.as_str()) + })?; + connections + .key_by_addr + .get(&protection.protected_addr) + .filter(|registration| registration.usernames.contains(username)) + .map(|_| protection.protected_addr) + } + + pub async fn recent_socket_reentry_protected_addr( + &self, + username: &str, + client_ip: &str, + virtual_id: VirtualId, + ) -> Option { + let connections = self.connections.read().await; + let key = create_socket_reentry_guard_key(username, client_ip, virtual_id); + let protection = connections.recent_socket_reentry_guards.get(&key)?; + if protection.expires_at > Instant::now() { + Some(protection.protected_addr) + } else { + None + } + } + + pub async fn block_user_for_stream(&self, addr: &SocketAddr, virtual_id: VirtualId, blocked_secs: u64) { + let block_for_secs = blocked_secs.clamp(0, 86_400); // max 1 day; + if block_for_secs > 0 { + let mut connections = self.write_connections().await; + let now = current_time_secs(); + connections.kicked.retain(|_, (expires_at, _)| *expires_at > now); + let candidate_usernames: Vec = connections + .key_by_addr + .get(addr) + .map(|reg| reg.usernames.iter().cloned().collect()) + .unwrap_or_default(); + let target_vid = virtual_id.get(); + let expires_at = now + block_for_secs; + for username in candidate_usernames { + // Only block users who are actually watching the target channel + // at this socket address. Behind reverse proxies / NAT, multiple + // distinct users share the same SocketAddr; blocking all of them + // would be a multi-tenant isolation violation. + let is_watching_target = connections.by_key.get(&username).is_some_and(|data| { + data.streams.iter().any(|s| s.addr == *addr && s.channel.virtual_id == target_vid) + }); + if is_watching_target { + connections.kicked.insert(username, (expires_at, virtual_id)); + } + } + } + } + + pub async fn block_user_for_stream_uid(&self, uid: u32, virtual_id: VirtualId, blocked_secs: u64) { + if blocked_secs == 0 { + return; + } + let mut connections = self.write_connections().await; + let username = connections + .by_key + .iter() + .find_map(|(username, data)| data.streams.iter().any(|stream| stream.uid == uid).then(|| username.clone())); + if let Some(username) = username { + connections.kicked.insert(username, (current_time_secs() + blocked_secs.min(86_400), virtual_id)); + } + } + + pub async fn mark_recent_eviction_guard_for_addr( + &self, + addr: &SocketAddr, + protected_addr: SocketAddr, + ttl: Duration, + ) { + let mut connections = self.write_connections().await; + let now = Instant::now(); + connections.recently_evicted_sessions.retain(|_, protection| protection.expires_at > now); + connections.recent_socket_reentry_guards.retain(|_, protection| protection.expires_at > now); + + let usernames = connections.key_by_addr.get(addr).map(SocketRegistration::all_usernames).unwrap_or_default(); + if usernames.is_empty() { + return; + } + + let protection = RecentWinnerProtection { protected_addr, expires_at: now + ttl }; + let mut session_tokens = Vec::new(); + let mut socket_guard_keys = Vec::new(); + + for username in &usernames { + if let Some(connection_data) = connections.by_key.get(username) { + for stream in connection_data.streams.iter().filter(|stream| stream.addr == *addr) { + if uses_session_reentry_guard(stream) && stream.session_token.is_some() { + let Some(session_token) = stream.session_token.clone() else { + continue; + }; + session_tokens.push(session_token); + } else { + socket_guard_keys.push(create_socket_reentry_guard_key( + username, + &stream.client_ip, + shared::model::VirtualId::new(stream.channel.virtual_id), + )); + } + } + } + } + + // Evicted sessions stay ended even when the reentry guard itself is disabled. + connections.mark_sessions_ended(session_tokens.iter().cloned()); + if ttl.is_zero() { + return; + } + for session_token in session_tokens { + connections.recently_evicted_sessions.insert(session_token, protection); + } + for key in socket_guard_keys { + connections.recent_socket_reentry_guards.insert(key, protection); + } + } +} diff --git a/backend/session/src/active_user_manager/expiry.rs b/backend/session/src/active_user_manager/expiry.rs new file mode 100644 index 000000000..8341c1118 --- /dev/null +++ b/backend/session/src/active_user_manager/expiry.rs @@ -0,0 +1,370 @@ +use super::{ + ActiveUserManager, PromotionAction, UserConnectionData, UserConnections, UserConnectionsWriteGuard, UserSession, + ANON_SOCKET_TTL, DEFAULT_ACTIVE_SOCKET_TTL_SECS, USER_CON_TTL, USER_GC_TTL, USER_SESSION_LIMIT, +}; +use crate::{connection_manager::CleanupEvent, stream::DIRECT_BODY_IDLE_TIMEOUT_SECS}; +use log::debug; +use shared::{ + model::{ActiveUserConnectionChange, EventMessage, PlaylistItemType, StreamInfo}, + utils::{current_time_secs, is_catchup_session_token}, +}; +use std::{ + cmp::Reverse, + net::SocketAddr, + sync::{atomic::Ordering, Arc}, + time::{Duration, Instant}, +}; + +impl UserConnectionData { + pub(super) fn gc(&mut self) { + if self.sessions.len() > USER_SESSION_LIMIT { + self.sessions.sort_by_key(|e| std::cmp::Reverse(e.ts)); + self.sessions.truncate(USER_SESSION_LIMIT); + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub(super) struct AdaptiveExpiryEntry { + pub(super) expires_at: u64, + pub(super) username: String, + pub(super) session_token: String, + pub(super) uid: u32, +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub(super) struct AdaptiveExpiryKey { + pub(super) username: String, + pub(super) session_token: String, + pub(super) uid: u32, +} + +impl ActiveUserManager { + pub fn start_adaptive_expiry_worker(self: &Arc) { + if self + .adaptive_expiry_worker_started + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return; + } + + let manager = Arc::clone(self); + tokio::spawn(async move { + manager.run_adaptive_expiry_worker().await; + }); + } + + pub(super) fn build_preserved_stream_expiry( + &self, + username: &str, + stream: &StreamInfo, + sessions: &[UserSession], + ) -> Option { + let session_token = stream.session_token.as_deref()?; + // Catchup segment gaps can briefly lose the UserSession row; still preserve the panel + // row using the stream timestamp so Streams does not blink between archive chunks. + let session_ts = if let Some(session) = sessions.iter().find(|session| session.token == session_token) { + session.ts + } else if stream.channel.item_type == PlaylistItemType::Catchup || is_catchup_session_token(session_token) { + stream.ts + } else { + return None; + }; + + let ttl_secs = self.adaptive_session_ttl_secs.load(Ordering::Relaxed); + let expires_at = session_ts.saturating_add(ttl_secs); + Some(AdaptiveExpiryEntry { + expires_at, + username: username.to_string(), + session_token: session_token.to_string(), + uid: stream.uid, + }) + } + + pub(super) async fn enqueue_adaptive_expiry(&self, entry: AdaptiveExpiryEntry) { + let key = AdaptiveExpiryKey { + username: entry.username.clone(), + session_token: entry.session_token.clone(), + uid: entry.uid, + }; + + let mut expiry_index = self.adaptive_expiry_index.lock().await; + expiry_index.insert(key, entry.expires_at); + drop(expiry_index); + + let mut queue = self.adaptive_expiry_queue.lock().await; + let wake_worker = queue.peek().is_none_or(|current| entry.expires_at < current.0.expires_at); + queue.push(Reverse(entry)); + if wake_worker { + self.adaptive_expiry_notify.notify_one(); + } + } + + pub fn active_socket_ttl_secs(&self) -> u64 { + let configured_ttl = self.adaptive_session_ttl_secs.load(Ordering::Relaxed); + if configured_ttl == 0 { + DEFAULT_ACTIVE_SOCKET_TTL_SECS + } else { + configured_ttl + } + } + + pub async fn socket_expiry_deadline(&self, addr: &SocketAddr) -> Option { + let connections = self.connections.read().await; + let registration = connections.key_by_addr.get(addr)?; + if registration.is_empty() { + return None; + } + + let ttl_secs = self.effective_socket_ttl_secs(&connections, addr); + Some(registration.ts.saturating_add(ttl_secs)) + } + + /// Direct VOD/series bodies may stop reading while the player drains its buffer. + /// Their socket expiry must not fire before the direct-body idle timeout, or a + /// buffering player is disconnected mid-playback even though the stream is alive. + pub(super) fn effective_socket_ttl_secs(&self, connections: &UserConnections, addr: &SocketAddr) -> u64 { + let base_ttl = self.active_socket_ttl_secs(); + if Self::addr_has_direct_body_stream(connections, addr) { + base_ttl.max(DIRECT_BODY_IDLE_TIMEOUT_SECS) + } else { + base_ttl + } + } + + pub(super) fn is_preserved_stream_expired(&self, stream: &StreamInfo, sessions: &[UserSession], now: u64) -> bool { + if !stream.preserved || !Self::should_preserve_session_stream(stream) { + return false; + } + + let ttl_secs = self.adaptive_session_ttl_secs.load(Ordering::Relaxed); + let Some(session_token) = stream.session_token.as_deref() else { + return true; + }; + + let session_ts = + sessions.iter().find(|session| session.token == session_token).map_or(stream.ts, |session| session.ts); + + now.saturating_sub(session_ts) >= ttl_secs + } + + pub(super) async fn run_adaptive_expiry_worker(self: Arc) { + loop { + let next_expiry = { + let queue = self.adaptive_expiry_queue.lock().await; + queue.peek().map(|entry| entry.0.expires_at) + }; + + match next_expiry { + None => { + tokio::select! { + () = self.adaptive_expiry_notify.notified() => {} + () = self.adaptive_expiry_cancel.cancelled() => break, + } + } + Some(expires_at) => { + let now = current_time_secs(); + if expires_at <= now { + self.process_due_adaptive_expiry_entries(now).await; + continue; + } + + tokio::select! { + () = tokio::time::sleep(Duration::from_secs(expires_at.saturating_sub(now))) => {} + () = self.adaptive_expiry_notify.notified() => {} + () = self.adaptive_expiry_cancel.cancelled() => break, + } + } + } + } + } + + #[allow(clippy::too_many_lines)] + pub(super) async fn process_due_adaptive_expiry_entries(&self, now: u64) { + let mut due_entries = Vec::new(); + { + let mut queue = self.adaptive_expiry_queue.lock().await; + while let Some(entry) = queue.peek() { + if entry.0.expires_at > now { + break; + } + if let Some(Reverse(entry)) = queue.pop() { + due_entries.push(entry); + } + } + } + + if due_entries.is_empty() { + return; + } + + let usernames_to_check: std::collections::HashSet<_> = due_entries.iter().map(|e| &e.username).collect(); + + let mut removed_addrs: Vec = Vec::new(); + let mut cleanup_events: Vec<(std::net::SocketAddr, Box)> = Vec::new(); + let mut replacement_entries: Vec = Vec::new(); + let mut promotions: Vec<(String, PromotionAction)> = Vec::new(); + { + let mut expiry_index = self.adaptive_expiry_index.lock().await; + let mut user_connections = self.write_connections().await; + for entry in &due_entries { + let key = AdaptiveExpiryKey { + username: entry.username.clone(), + session_token: entry.session_token.clone(), + uid: entry.uid, + }; + let Some(current_expires_at) = expiry_index.get(&key).copied() else { + continue; + }; + if current_expires_at != entry.expires_at { + continue; + } + + let mut remove_user = false; + if let Some(connection_data) = user_connections.by_key.get_mut(&entry.username) { + let stream_idx_opt = connection_data.streams.iter().position(|stream| { + stream.uid == entry.uid + && stream.preserved + && stream.session_token.as_deref() == Some(entry.session_token.as_str()) + }); + + if let Some(stream_idx) = stream_idx_opt { + let should_remove = self.is_preserved_stream_expired( + &connection_data.streams[stream_idx], + &connection_data.sessions, + now, + ); + + if should_remove { + let addr = connection_data.streams[stream_idx].addr; + let session_token = connection_data.streams[stream_idx].session_token.clone(); + if self.cleanup_tx.get().is_some() { + cleanup_events.push((addr, Box::new(connection_data.streams[stream_idx].clone()))); + } else { + removed_addrs.push(addr); + } + let removed_stream = connection_data.streams.swap_remove(stream_idx); + connection_data.remove_stream_request_claims(removed_stream.uid); + if let Some(kind) = connection_data.stream_kinds.remove(&removed_stream.uid) { + connection_data.decrement_kind(kind); + } + connection_data.stream_normal_priorities.remove(&removed_stream.uid); + if let Some(action) = connection_data.try_promote_soft_stream() { + let promoted_stream = + connection_data.streams.iter().find(|stream| stream.uid == action.uid).cloned(); + if let Some(stream) = promoted_stream.as_ref() { + Self::promote_session_for_stream(connection_data, stream); + } + promotions.push((entry.username.clone(), action)); + } + if let Some(session_token) = session_token.as_deref() { + Self::clear_session_counted_without_stream(connection_data, session_token); + } + while connection_data.try_promote_soft_session_reservation() {} + expiry_index.remove(&key); + } else if let Some(replacement_entry) = self.build_preserved_stream_expiry( + &entry.username, + &connection_data.streams[stream_idx], + &connection_data.sessions, + ) { + if replacement_entry.expires_at != current_expires_at { + replacement_entries.push(replacement_entry); + } + } + } else { + expiry_index.remove(&key); + } + + remove_user = connection_data.connections == 0 + && connection_data.streams.is_empty() + && connection_data.sessions.is_empty(); + } else { + expiry_index.remove(&key); + } + + if remove_user { + user_connections.by_key.remove(&entry.username); + } + } + } // locks released here + + // divergence check after adaptive expiry processing + for username in usernames_to_check { + let snapshot = { + let connections = self.connections.read().await; + connections.by_key.get(username).and_then(|data| Self::collect_divergence_snapshot(data, username)) + }; + self.log_divergence_snapshot(snapshot).await; + } + + if let Some(tx) = self.cleanup_tx.get() { + for (addr, stream_info) in cleanup_events { + if tx.try_send(CleanupEvent::AdaptiveSessionExpired { stream_info }).is_err() { + self.dropped_cleanup_events.fetch_add(1, Ordering::Relaxed); + debug!("Cleanup channel unavailable, dropping adaptive session expiry"); + removed_addrs.push(addr); + } + } + } + + for entry in replacement_entries { + self.enqueue_adaptive_expiry(entry).await; + } + + for (username, action) in promotions { + self.emit_promotion_update(&username, action).await; + } + + let had_removals = !removed_addrs.is_empty(); + for addr in removed_addrs { + self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Disconnected(addr))); + } + if had_removals { + self.log_active_user().await; + } + } + + pub(super) fn gc(&self) { + if let Some(gc_ts) = &self.gc_ts { + let ts = gc_ts.load(Ordering::Acquire); + let now = current_time_secs(); + + if now.saturating_sub(ts) > USER_GC_TTL + && gc_ts.compare_exchange(ts, now, Ordering::AcqRel, Ordering::Relaxed).is_ok() + { + if let Ok(guard) = self.connections.try_write() { + let mut user_connections = UserConnectionsWriteGuard::new(guard, &self.deferred_wakes); + user_connections.kicked.retain(|_, (expires_at, _)| *expires_at > now); + let now_instant = Instant::now(); + user_connections + .recently_evicted_sessions + .retain(|_, protection| protection.expires_at > now_instant); + user_connections.ended_sessions.retain(|_, expires_at| *expires_at > now_instant); + user_connections + .recent_socket_reentry_guards + .retain(|_, protection| protection.expires_at > now_instant); + for connection_data in user_connections.by_key.values_mut() { + Self::release_expired_session_reservations(connection_data, now); + connection_data.sessions.retain(|s| now.saturating_sub(s.ts) < USER_CON_TTL); + } + user_connections.by_key.retain(|_k, v| { + v.connections > 0 || !v.streams.is_empty() || now.saturating_sub(v.ts) < USER_CON_TTL + }); + let UserConnections { ref by_key, ref mut key_by_addr, .. } = *user_connections; + key_by_addr.retain(|addr, registration| { + registration.usernames.retain(|u| { + by_key + .get(u) + .is_some_and(|d| d.has_session_addr(addr) || d.streams.iter().any(|s| s.addr == *addr)) + }); + !(registration.is_empty() && now.saturating_sub(registration.ts) >= ANON_SOCKET_TTL) + }); + } else { + // Lock contention: release the GC claim so a subsequent caller can retry immediately. + let _ = gc_ts.compare_exchange(now, ts, Ordering::AcqRel, Ordering::Relaxed); + } + } + } + } +} diff --git a/backend/session/src/active_user_manager/mod.rs b/backend/session/src/active_user_manager/mod.rs index fea49447f..97149690a 100644 --- a/backend/session/src/active_user_manager/mod.rs +++ b/backend/session/src/active_user_manager/mod.rs @@ -1,89 +1,42 @@ use crate::{ active_provider_manager::{ConnectionKind, ProviderReleaseSnapshot}, connection_manager::CleanupEvent, - stream::{uses_direct_body_idle_timeout, DIRECT_BODY_IDLE_TIMEOUT_SECS}, ActiveProviderManager, EventManager, }; use arc_swap::ArcSwapOption; -use jsonwebtoken::get_current_timestamp; -use log::{debug, info, log_enabled}; use lru::LruCache; -use shared::{ - defaults::{ - default_grace_period_millis, default_grace_period_timeout_secs, default_hls_session_ttl_secs, DASH_EXT, HLS_EXT, - }, - model::{ - ActiveUserConnectionChange, CustomVideoStreamType, EventMessage, PlaylistItemType, StreamChannel, StreamInfo, - StreamTechnicalInfo, UserConnectionPermission, VirtualId, - }, - utils::{ - current_time_secs, extract_extension_from_url, is_catchup_session_token, sanitize_sensitive_info, strip_port, - Internable, - }, -}; +use shared::model::{StreamChannel, StreamInfo, UserConnectionPermission}; use std::{ borrow::Cow, cmp::Reverse, - collections::{BinaryHeap, HashMap, HashSet}, + collections::{BinaryHeap, HashMap}, net::SocketAddr, num::NonZeroUsize, sync::{ - atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicU64, AtomicUsize}, Arc, }, - time::{Duration, Instant}, + time::Duration, }; use tokio::sync::{mpsc, Mutex, Notify, RwLock}; use tokio_util::sync::CancellationToken; -use tuliprox_core::{ - model::{Config, Fingerprint, ProxyUserCredentials}, - utils::{debug_if_enabled, utc_day_from_secs}, -}; +use tuliprox_core::model::{Fingerprint, ProxyUserCredentials}; use tuliprox_repository::GeoIp; /// Capacity of the per-user divergence cache. A constant so the conversion /// cannot fail at runtime. const DIVERGENCE_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(256).unwrap(); -const USER_GC_TTL: u64 = 900; // 15 Min -const USER_CON_TTL: u64 = 1_800; // 30 minutes +const USER_GC_TTL: u64 = 900; +// 15 Min +const USER_CON_TTL: u64 = 1_800; +// 30 minutes const USER_SESSION_LIMIT: usize = 50; -const ANON_SOCKET_TTL: u64 = 300; // 5 Min + +const ANON_SOCKET_TTL: u64 = 300; +// 5 Min const DEFAULT_ACTIVE_SOCKET_TTL_SECS: u64 = 90; -fn get_grace_options(config: &Config) -> (u64, u64) { - let (grace_period_millis, grace_period_timeout_secs) = - config.reverse_proxy.as_ref().and_then(|r| r.stream.as_ref()).map_or_else( - || (default_grace_period_millis(), default_grace_period_timeout_secs()), - |s| (s.grace_period_millis, s.grace_period_timeout_secs), - ); - (grace_period_millis, grace_period_timeout_secs) -} - -fn get_adaptive_session_ttl_secs(config: &Config) -> u64 { - config - .reverse_proxy - .as_ref() - .and_then(|r| r.stream.as_ref()) - .map_or_else(default_hls_session_ttl_secs, |s| s.hls_session_ttl_secs) -} - -fn stream_history_session_id(ts: u64, uid: u32) -> u64 { (ts << 32) | u64::from(uid) } - -fn decide_connection_kind( - counts: UserConnectionCounts, - max_connections: u32, - soft_connections: u16, -) -> Option { - if max_connections == 0 || counts.normal < max_connections { - return Some(ConnectionKind::Normal); - } - if soft_connections > 0 && counts.soft < soft_connections { - return Some(ConnectionKind::Soft); - } - None -} - #[derive(Clone, Debug, PartialEq, Eq)] pub enum PendingProviderReason { GraceHold, @@ -122,17 +75,8 @@ pub enum PlaybackLifecycle { Expired, } -impl PlaybackLifecycle { - /// Returns true for lifecycle states that own a counted admission lease. - /// Both `Active` and `GraceActive` count — `GraceActive` is a provisional - /// counted state for `GraceMode::Instant` sessions. - pub fn is_counted(&self) -> bool { matches!(self, Self::Active | Self::GraceActive) } -} - static NEXT_SESSION_INCARNATION: AtomicU64 = AtomicU64::new(1); -pub fn next_session_incarnation() -> u64 { NEXT_SESSION_INCARNATION.fetch_add(1, Ordering::Relaxed) } - /// The session object and provider binding a finite resource request was authorized for. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SessionIdentity { @@ -147,23 +91,6 @@ pub(crate) struct DeferredWakes { has_pending: AtomicBool, } -impl DeferredWakes { - fn push(&self, notify: Arc) { - self.pending.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push(notify); - self.has_pending.store(true, Ordering::Release); - } - - fn wake(&self) { - if !self.has_pending.swap(false, Ordering::AcqRel) { - return; - } - let pending = std::mem::take(&mut *self.pending.lock().unwrap_or_else(std::sync::PoisonError::into_inner)); - for notify in pending { - notify.notify_waiters(); - } - } -} - /// Wakes the in-flight requests of one session when it ends or switches accounts. /// /// Only the session stored in the connection table owns the signal: dropping it, by any removal @@ -174,41 +101,6 @@ pub struct SessionChangeSignal { owner: bool, } -impl SessionChangeSignal { - fn new(deferred: &Arc) -> Self { - Self { notify: Arc::new(Notify::new()), deferred: Some(Arc::clone(deferred)), owner: true } - } - - fn signal(&self) { - match &self.deferred { - Some(deferred) => deferred.push(Arc::clone(&self.notify)), - None => self.notify.notify_waiters(), - } - } -} - -impl Default for SessionChangeSignal { - fn default() -> Self { Self { notify: Arc::new(Notify::new()), deferred: None, owner: true } } -} - -impl Clone for SessionChangeSignal { - fn clone(&self) -> Self { Self { notify: Arc::clone(&self.notify), deferred: self.deferred.clone(), owner: false } } -} - -impl Drop for SessionChangeSignal { - fn drop(&mut self) { - if self.owner { - self.signal(); - } - } -} - -impl std::fmt::Debug for SessionChangeSignal { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SessionChangeSignal").field("owner", &self.owner).finish_non_exhaustive() - } -} - /// Provider session headers of one playback session for one target URL. #[derive(Debug, PartialEq, Eq)] pub enum SessionProviderHeaders { @@ -219,20 +111,6 @@ pub enum SessionProviderHeaders { Headers(HashMap), } -impl SessionProviderHeaders { - fn for_target(session: &UserSession, target_url: &str) -> Self { - session - .provider_session_headers_for(target_url) - .map_or(Self::NoHeaders, |headers| Self::Headers(headers.into_owned())) - } -} - -/// `host:port` of a URL, used to scope provider session cookies. -fn url_host_key(url: &str) -> Option { - let url = url::Url::parse(url).ok()?; - Some(format!("{}://{}:{}", url.scheme(), url.host_str()?, url.port_or_known_default()?)) -} - #[derive(Clone, Debug)] pub struct UserSession { pub token: String, @@ -264,97 +142,6 @@ pub struct UserSession { pub lifecycle: PlaybackLifecycle, } -/// A detached session with a fresh incarnation, for fixtures and struct-update construction. -/// Sessions in the connection table come from `create_user_session`. -impl Default for UserSession { - fn default() -> Self { - Self { - token: String::new(), - transition_version: 1, - virtual_id: 0, - provider: Arc::from(""), - stream_url: Arc::from(""), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - provider_session_cookies: Arc::default(), - incarnation: next_session_incarnation(), - binding_generation: 0, - change_signal: SessionChangeSignal::default(), - media_started: Arc::new(AtomicBool::new(false)), - user_agent_stream_index: None, - addr: SocketAddr::from(([0, 0, 0, 0], 0)), - socket_bound: false, - active_addrs: Vec::new(), - ts: 0, - started_at: 0, - permission: UserConnectionPermission::Allowed, - connection_kind: None, - lifecycle: PlaybackLifecycle::default(), - } - } -} - -impl UserSession { - pub const fn identity(&self) -> SessionIdentity { - SessionIdentity { incarnation: self.incarnation, binding_generation: self.binding_generation } - } - - fn switch_provider(&mut self, provider: Arc) { - self.provider = provider; - self.binding_generation = self.binding_generation.wrapping_add(1); - self.change_signal.signal(); - self.provider_session_headers.clear(); - self.provider_session_headers_host = None; - self.clear_provider_session_cookies(); - } - - fn expire(&mut self) { - self.lifecycle = PlaybackLifecycle::Expired; - self.change_signal.signal(); - } - - fn set_permission(&mut self, permission: UserConnectionPermission) { - let exhausted_now = permission == UserConnectionPermission::Exhausted && self.permission != permission; - self.permission = permission; - if exhausted_now { - self.change_signal.signal(); - } - } - - /// Whether the session may still serve requests: neither expired nor exhausted. - fn is_live(&self) -> bool { - !matches!(self.lifecycle, PlaybackLifecycle::Expired) && self.permission != UserConnectionPermission::Exhausted - } - - fn clear_provider_session_cookies(&mut self) { - if !self.provider_session_cookies.is_empty() { - self.provider_session_cookies = Arc::default(); - } - } - - /// Provider session headers that may be sent to `target_url`: only to the origin (scheme, - /// host and port) that set them, or unconditionally when that origin is unknown. - pub fn provider_session_headers_for(&self, target_url: &str) -> Option>> { - if !self.provider_session_cookies.is_empty() { - return self.provider_session_cookies.headers_for(target_url); - } - if self.provider_session_headers.is_empty() { - return None; - } - match self.provider_session_headers_host.as_deref() { - Some(host) => (url_host_key(target_url).as_deref() == Some(host)) - .then_some(Cow::Borrowed(&self.provider_session_headers)), - None => Some(Cow::Borrowed(&self.provider_session_headers)), - } - } -} - -#[derive(Debug, Default, Clone, Copy)] -struct UserConnectionCounts { - normal: u32, - soft: u16, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AdmissionRejectionReason { UserConnectionsExhausted, @@ -368,56 +155,6 @@ pub struct ConnectionAdmission { rejection_reason: Option, } -impl ConnectionAdmission { - pub fn allowed(kind: Option) -> Self { - Self { permission: UserConnectionPermission::Allowed, kind, rejection_reason: None } - } - - pub fn grace_period(kind: Option) -> Self { - Self { permission: UserConnectionPermission::GracePeriod, kind, rejection_reason: None } - } - - pub fn exhausted(reason: AdmissionRejectionReason, kind: Option) -> Self { - Self { permission: UserConnectionPermission::Exhausted, kind, rejection_reason: Some(reason) } - } - - /// Maps a stored session permission onto an admission. An exhausted session is - /// classified as a genuine connection-limit rejection so the - /// `Exhausted ⇒ rejection_reason.is_some()` invariant always holds. - pub fn from_permission(permission: UserConnectionPermission, kind: Option) -> Self { - match permission { - UserConnectionPermission::Allowed => Self::allowed(kind), - UserConnectionPermission::GracePeriod => Self::grace_period(kind), - UserConnectionPermission::Exhausted => { - Self::exhausted(AdmissionRejectionReason::UserConnectionsExhausted, kind) - } - } - } - - pub fn permission(&self) -> UserConnectionPermission { self.permission } - - pub fn kind(&self) -> Option { self.kind } - - pub fn rejection_reason(&self) -> Option { self.rejection_reason } - - pub fn is_reentry_suppressed(&self) -> bool { - self.rejection_reason == Some(AdmissionRejectionReason::RecentEvictionReentry) - } -} - -#[derive(Debug, Clone, Copy)] -struct PromotionAction { - addr: SocketAddr, - uid: u32, - new_priority: i8, -} - -#[derive(Debug, Clone, Copy)] -struct StreamRequestClaim { - stream_uid: u32, - addr: SocketAddr, -} - #[derive(Debug)] pub enum StreamRequestDetach { NotFound, @@ -426,380 +163,9 @@ pub enum StreamRequestDetach { Removed(StreamInfo), } -impl StreamRequestDetach { - pub fn request_was_detached(&self) -> bool { !matches!(self, Self::NotFound) } - - pub fn stream_info(&self) -> Option<&StreamInfo> { - match self { - Self::NotFound => None, - Self::Retained(stream) | Self::Preserved(stream) | Self::Removed(stream) => Some(stream), - } - } - - fn into_removed(self) -> Option { - match self { - Self::Removed(stream) => Some(stream), - Self::NotFound | Self::Retained(_) | Self::Preserved(_) => None, - } - } -} - -#[derive(Debug)] -struct UserConnectionData { - max_connections: u32, - soft_connections: u16, - counts: UserConnectionCounts, - connections: u32, - granted_grace: bool, - grace_ts: u64, - sessions: Vec, - streams: Vec, - stream_request_claims: HashMap, - stream_kinds: HashMap, - stream_normal_priorities: HashMap, - ts: u64, -} - -impl UserConnectionData { - fn new(connections: u32, max_connections: u32, soft_connections: u16) -> Self { - Self { - max_connections, - soft_connections, - counts: UserConnectionCounts::default(), - connections, - granted_grace: false, - grace_ts: 0, - sessions: Vec::new(), - streams: Vec::new(), - stream_request_claims: HashMap::new(), - stream_kinds: HashMap::new(), - stream_normal_priorities: HashMap::new(), - ts: current_time_secs(), - } - } - - fn add_session(&mut self, session: UserSession) { - self.gc(); - self.sessions.push(session); - } - fn gc(&mut self) { - if self.sessions.len() > USER_SESSION_LIMIT { - self.sessions.sort_by_key(|e| std::cmp::Reverse(e.ts)); - self.sessions.truncate(USER_SESSION_LIMIT); - } - } - - fn has_session_addr(&self, addr: &SocketAddr) -> bool { - self.sessions.iter().any(|session| session.addr == *addr || session.active_addrs.contains(addr)) - } - - fn release_addr_from_sessions(&mut self, addr: &SocketAddr) -> HashMap> { - let mut migrated_addrs = HashMap::new(); - for session in &mut self.sessions { - if session.addr == *addr || session.active_addrs.contains(addr) { - migrated_addrs.insert(session.token.clone(), release_session_addr(session, addr)); - } - } - migrated_addrs - } - - fn release_addr_from_stream_session(&mut self, addr: &SocketAddr, uid: u32) -> HashMap> { - let Some(token) = - self.streams.iter().find(|stream| stream.uid == uid).and_then(|stream| stream.session_token.as_deref()) - else { - return HashMap::new(); - }; - if self.streams.iter().any(|stream| { - stream.uid != uid - && !stream.preserved - && stream.addr == *addr - && stream.session_token.as_deref() == Some(token) - }) { - return HashMap::new(); - } - let Some(session) = self.sessions.iter_mut().find(|session| session.token == token) else { - return HashMap::new(); - }; - HashMap::from([(session.token.clone(), release_session_addr(session, addr))]) - } - - fn attach_stream_request(&mut self, request_uid: u32, stream_uid: u32, addr: SocketAddr) { - self.stream_request_claims.insert(request_uid, StreamRequestClaim { stream_uid, addr }); - } - - fn detach_stream_request(&mut self, request_uid: u32) -> Option<(u32, bool)> { - let claim = self.stream_request_claims.remove(&request_uid)?; - let is_last = !self.stream_request_claims.values().any(|other| other.stream_uid == claim.stream_uid); - Some((claim.stream_uid, is_last)) - } - - fn remove_stream_request_claims(&mut self, stream_uid: u32) { - self.stream_request_claims.retain(|_, claim| claim.stream_uid != stream_uid); - } - - fn increment_kind(&mut self, kind: ConnectionKind) { - self.connections = self.connections.saturating_add(1); - match kind { - ConnectionKind::Normal => { - self.counts.normal = self.counts.normal.saturating_add(1); - } - ConnectionKind::Soft => { - self.counts.soft = self.counts.soft.saturating_add(1); - } - } - } - - fn decrement_kind(&mut self, kind: ConnectionKind) { - self.connections = self.connections.saturating_sub(1); - match kind { - ConnectionKind::Normal => { - self.counts.normal = self.counts.normal.saturating_sub(1); - } - ConnectionKind::Soft => { - self.counts.soft = self.counts.soft.saturating_sub(1); - } - } - } - - fn remove_streams_for_session_and_release_counted( - &mut self, - session_token: &str, - counted_kind: Option, - ) -> (u32, bool) { - let mut removed_count = 0; - let mut connection_changed = false; - let mut released_stream_kind = false; - let mut stream_idx = 0; - while stream_idx < self.streams.len() { - if self.streams[stream_idx].session_token.as_deref() != Some(session_token) { - stream_idx += 1; - continue; - } - - let uid = self.streams[stream_idx].uid; - if let Some(kind) = self.stream_kinds.remove(&uid) { - self.decrement_kind(kind); - released_stream_kind = true; - connection_changed = true; - } - self.stream_normal_priorities.remove(&uid); - self.remove_stream_request_claims(uid); - self.streams.swap_remove(stream_idx); - removed_count += 1; - } - - if let Some(kind) = counted_kind.filter(|_| !released_stream_kind) { - self.decrement_kind(kind); - connection_changed = true; - } - - (removed_count, connection_changed) - } - - fn try_promote_soft_stream(&mut self) -> Option { - if self.counts.normal >= self.max_connections - || (u32::from(self.counts.soft)) <= u32::from(self.soft_connections) - { - return None; - } - - let candidate_uid = self - .streams - .iter() - .filter(|stream| !stream.preserved) - .filter_map(|stream| { - let kind = self.stream_kinds.get(&stream.uid).copied()?; - if kind != ConnectionKind::Soft { - return None; - } - let normal_priority = self.stream_normal_priorities.get(&stream.uid).copied().unwrap_or_default(); - Some((normal_priority, stream.ts, stream.uid, stream.addr)) - }) - .min_by_key(|(normal_priority, ts, uid, _)| (*normal_priority, *ts, *uid)); - - let (new_priority, _ts, uid, addr) = candidate_uid?; - - self.counts.normal = self.counts.normal.saturating_add(1); - if self.counts.soft > 0 { - self.counts.soft -= 1; - } - self.stream_kinds.insert(uid, ConnectionKind::Normal); - - Some(PromotionAction { addr, uid, new_priority }) - } - - fn try_promote_soft_session_reservation(&mut self) -> bool { - if self.counts.normal >= self.max_connections - || (u32::from(self.counts.soft)) <= u32::from(self.soft_connections) - { - return false; - } - - let active_tokens = - self.streams.iter().filter_map(|stream| stream.session_token.as_deref()).collect::>(); - - let candidate_index = self.sessions.iter().position(|session| { - session.lifecycle.is_counted() - && session.connection_kind == Some(ConnectionKind::Soft) - && !active_tokens.contains(session.token.as_str()) - }); - - let Some(candidate_index) = candidate_index else { - return false; - }; - - self.counts.normal = self.counts.normal.saturating_add(1); - if self.counts.soft > 0 { - self.counts.soft -= 1; - } - self.sessions[candidate_index].connection_kind = Some(ConnectionKind::Normal); - true - } - - fn effective_counts_for_admission(&self, exclude_session_token: Option<&str>) -> UserConnectionCounts { - let mut counts = self.counts; - let counted_tokens = self - .sessions - .iter() - .filter(|session| session.lifecycle.is_counted()) - .map(|session| session.token.as_str()) - .collect::>(); - let mut reserved_tokens = HashSet::new(); - - for stream in self.streams.iter().filter(|stream| stream.preserved) { - // Orphan preserved stream: no session token means no session to evict. - // Do not count it — it has no bearing on admission decisions. - let Some(session_token) = stream.session_token.as_deref() else { - continue; - }; - if exclude_session_token.is_some_and(|token| token == session_token) - || counted_tokens.contains(session_token) - || !reserved_tokens.insert(session_token) - { - continue; - } - - let kind = self - .sessions - .iter() - .find(|session| session.token == session_token) - .and_then(|session| session.connection_kind) - .unwrap_or(ConnectionKind::Normal); - match kind { - ConnectionKind::Normal => counts.normal = counts.normal.saturating_add(1), - ConnectionKind::Soft => counts.soft = counts.soft.saturating_add(1), - } - } - - counts - } -} - -fn create_socket_reentry_guard_key(username: &str, client_ip: &str, virtual_id: VirtualId) -> String { - shared::concat_string!(username, "|", client_ip, "|", &virtual_id.to_string()) -} - -fn is_stable_session_stream(stream: &StreamInfo) -> bool { - // Catchup-token Live/.ts segment sockets must preserve too; otherwise archive panel rows - // hard-remove every HLS chunk and Streams blinks even when frontend soft-preserve is present. - stream.channel.item_type == PlaylistItemType::Catchup - || stream.channel.item_type.is_live_adaptive() - || stream.session_token.as_deref().is_some_and(is_catchup_session_token) - || matches!( - extract_extension_from_url(stream.channel.url.as_ref()), - Some(ext) if ext == HLS_EXT || ext == DASH_EXT - ) -} - -fn uses_session_reentry_guard(stream: &StreamInfo) -> bool { - stream.channel.item_type.requires_provider_affinity() - || matches!( - extract_extension_from_url(stream.channel.url.as_ref()), - Some(ext) if ext == HLS_EXT || ext == DASH_EXT - ) -} - /// How long an ended session token blocks its recreation from a sealed playlist token. const ENDED_SESSION_RECREATE_BLOCK: Duration = Duration::from_secs(600); -impl UserConnections { - fn mark_sessions_ended>(&mut self, tokens: I) { - let now = Instant::now(); - self.ended_sessions.retain(|_, expires_at| *expires_at > now); - let expires_at = now + ENDED_SESSION_RECREATE_BLOCK; - let tokens = tokens.into_iter().collect::>(); - for session in self.by_key.values().flat_map(|data| data.sessions.iter()) { - if tokens.contains(&session.token) { - session.change_signal.signal(); - } - } - self.ended_sessions.extend(tokens.into_iter().map(|token| (token, expires_at))); - } - - fn is_token_ended(&self, token: &str) -> bool { - self.ended_sessions.get(token).is_some_and(|expiry| *expiry > Instant::now()) - } - - /// Returns the session while it may still serve requests: not ended, expired or exhausted. - fn live_session(&self, username: &str, token: &str) -> Option<(&UserConnectionData, &UserSession)> { - if self.is_token_ended(token) { - return None; - } - let data = self.by_key.get(username)?; - let session = data.sessions.iter().find(|session| session.token == token && session.is_live())?; - Some((data, session)) - } - - /// Returns the live session only while this exact incarnation and binding may still serve requests. - fn current_session( - &self, - username: &str, - token: &str, - identity: SessionIdentity, - ) -> Option<(&UserConnectionData, &UserSession)> { - self.live_session(username, token).filter(|(_, session)| session.identity() == identity) - } -} - -#[derive(Clone, Copy, Debug)] -struct RecentWinnerProtection { - protected_addr: SocketAddr, - expires_at: Instant, -} - -#[derive(Debug, Default)] -struct UserConnections { - kicked: HashMap, - recently_evicted_sessions: HashMap, - /// Session tokens ended by eviction, kick or explicit terminate, with their expiry. A sealed - /// playlist token of such a session must not recreate it; an expired session may. - ended_sessions: HashMap, - recent_socket_reentry_guards: HashMap, - by_key: HashMap, - key_by_addr: HashMap, -} - -#[derive(Clone, Debug, Default)] -struct SocketRegistration { - usernames: HashSet, - ts: u64, -} - -#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] -struct AdaptiveExpiryEntry { - expires_at: u64, - username: String, - session_token: String, - uid: u32, -} - -#[derive(Clone, Debug, Eq, PartialEq, Hash)] -struct AdaptiveExpiryKey { - username: String, - session_token: String, - uid: u32, -} - pub struct ReleasedConnection { pub addr_removed: bool, pub removed_streams: Vec, @@ -842,88 +208,6 @@ pub struct CreateUserSessionParams<'a> { pub socket_bound: bool, } -fn remember_session_addr(session: &mut UserSession, addr: SocketAddr) { - if session.socket_bound { - session.active_addrs.clear(); - } else if let Some(position) = session.active_addrs.iter().position(|active_addr| *active_addr == addr) { - session.active_addrs.remove(position); - } - session.active_addrs.push(addr); - session.addr = addr; -} - -fn release_session_addr(session: &mut UserSession, addr: &SocketAddr) -> Option { - if let Some(position) = session.active_addrs.iter().position(|active_addr| active_addr == addr) { - session.active_addrs.remove(position); - } else if session.addr != *addr { - return None; - } - - if session.addr == *addr { - if let Some(next_addr) = session.active_addrs.last().copied() { - session.addr = next_addr; - return Some(next_addr); - } - } - - None -} - -fn clear_session_addr(session: &mut UserSession, addr: &SocketAddr) -> bool { - let mut changed = false; - if let Some(position) = session.active_addrs.iter().position(|active_addr| active_addr == addr) { - session.active_addrs.remove(position); - changed = true; - } - - if session.addr == *addr { - if let Some(next_addr) = session.active_addrs.last().copied() { - session.addr = next_addr; - } else { - session.addr = SocketAddr::from(([0, 0, 0, 0], 0)); - } - changed = true; - } - - changed -} - -impl SocketRegistration { - fn anonymous() -> Self { Self { usernames: HashSet::new(), ts: current_time_secs() } } - - fn add_user(&mut self, username: &str, ts: u64) { - if !username.is_empty() && !self.usernames.contains(username) { - self.usernames.insert(username.to_string()); - } - self.ts = ts; - } - - fn remove_user(&mut self, username: &str) -> bool { - self.usernames.remove(username); - self.usernames.is_empty() - } - - fn is_empty(&self) -> bool { self.usernames.is_empty() } - - fn primary_username(&self) -> Option<&str> { self.usernames.iter().map(String::as_str).min() } - - fn all_usernames(&self) -> Vec { self.usernames.iter().cloned().collect() } - - fn into_usernames(self) -> HashSet { self.usernames } -} - -struct UserSessionParams<'a> { - change_wakes: &'a Arc, - session_token: &'a str, - virtual_id: u32, - provider: &'a str, - stream_url: &'a str, - addr: &'a SocketAddr, - connection_permission: UserConnectionPermission, - connection_kind: Option, - socket_bound: bool, -} - pub struct ActiveUserManager { grace_period_millis: AtomicU64, grace_period_timeout_secs: AtomicU64, @@ -955,3297 +239,34 @@ pub struct ActiveUserManager { deferred_wakes: Arc, } -/// Write access to `UserConnections` that delivers session change signals after release. -/// -/// Fields drop in declaration order: `guard` releases the write lock before `_wake` notifies -/// the waiters signalled during this write. -struct UserConnectionsWriteGuard<'a> { - guard: tokio::sync::RwLockWriteGuard<'a, UserConnections>, - _wake: WakeDeferred<'a>, -} - -struct WakeDeferred<'a>(&'a DeferredWakes); - -impl Drop for WakeDeferred<'_> { - fn drop(&mut self) { self.0.wake(); } -} - -impl<'a> UserConnectionsWriteGuard<'a> { - fn new(guard: tokio::sync::RwLockWriteGuard<'a, UserConnections>, wakes: &'a DeferredWakes) -> Self { - Self { guard, _wake: WakeDeferred(wakes) } - } -} - -impl std::ops::Deref for UserConnectionsWriteGuard<'_> { - type Target = UserConnections; - fn deref(&self) -> &Self::Target { &self.guard } -} - -impl std::ops::DerefMut for UserConnectionsWriteGuard<'_> { - fn deref_mut(&mut self) -> &mut Self::Target { &mut self.guard } -} - -struct DivergenceEntry { - last_logged: Instant, - count_since_last_log: u32, -} - -#[derive(Clone, Debug, Eq, PartialEq, Hash)] -enum DivergenceKind { - CountedSessionWithoutStream, - StreamWithoutCountedSession, - ConnectionCountMismatch { legacy: u32, counted: u32 }, -} - -fn divergence_key(username: &str, kind: &DivergenceKind) -> String { - match kind { - DivergenceKind::CountedSessionWithoutStream => format!("{username}:CountedSessionWithoutStream"), - DivergenceKind::StreamWithoutCountedSession => format!("{username}:StreamWithoutCountedSession"), - DivergenceKind::ConnectionCountMismatch { legacy, counted } => { - format!("{username}:ConnectionCountMismatch:{legacy}+{counted}") - } - } -} - -struct DivergenceSnapshot { - username: String, - connections: u32, - counted_sessions: usize, - streams_count: usize, - kinds: Vec, -} - -impl ActiveUserManager { - async fn write_connections(&self) -> UserConnectionsWriteGuard<'_> { - UserConnectionsWriteGuard::new(self.connections.write().await, &self.deferred_wakes) - } - - pub fn shutdown(&self) { self.adaptive_expiry_cancel.cancel(); } - - pub fn start_adaptive_expiry_worker(self: &Arc) { - if self - .adaptive_expiry_worker_started - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) - .is_err() - { - return; - } - - let manager = Arc::clone(self); - tokio::spawn(async move { - manager.run_adaptive_expiry_worker().await; - }); - } - - fn lookup_country(&self, client_ip: &str) -> Option { - let geoip = self.geo_ip.load(); - (*geoip).as_ref().and_then(|geoip_db| geoip_db.lookup(&strip_port(client_ip))) - } - - fn custom_stream_technical_info() -> StreamTechnicalInfo { - StreamTechnicalInfo { - container: String::from("mpegts"), - resolution: String::new(), - fps: String::from("30"), - video_codec: String::from("H.264"), - audio_codec: String::from("AAC"), - audio_channels: String::from("Stereo"), - } - } - - /// The bus this manager publishes on. - /// - /// Exposed so the admission path can report a refusal without - /// `AdmissionCtx` growing a second handle to the same manager. - #[must_use] - pub fn events(&self) -> &Arc { &self.event_manager } - - pub fn new(config: &Config, geoip: &Arc>, event_manager: &Arc) -> Self { - let log_active_user: bool = config.log.as_ref().is_some_and(|l| l.log_active_user); - let (grace_period_millis, grace_period_timeout_secs) = get_grace_options(config); - - Self { - grace_period_millis: AtomicU64::new(grace_period_millis), - grace_period_timeout_secs: AtomicU64::new(grace_period_timeout_secs), - adaptive_session_ttl_secs: AtomicU64::new(get_adaptive_session_ttl_secs(config)), - log_active_user: AtomicBool::new(log_active_user), - next_user_agent_stream_index: AtomicU64::new(1), - connections: RwLock::new(UserConnections::default()), - adaptive_expiry_queue: Arc::new(Mutex::new(BinaryHeap::new())), - adaptive_expiry_index: Arc::new(Mutex::new(HashMap::new())), - adaptive_expiry_notify: Arc::new(Notify::new()), - adaptive_expiry_cancel: CancellationToken::new(), - adaptive_expiry_worker_started: AtomicBool::new(false), - gc_ts: Some(AtomicU64::new(current_time_secs())), - geo_ip: Arc::clone(geoip), - event_manager: Arc::clone(event_manager), - last_logged_user_count: AtomicUsize::new(0), - last_logged_user_connection_count: AtomicUsize::new(0), - cleanup_tx: tokio::sync::OnceCell::new(), - provider_manager: tokio::sync::OnceCell::new(), - transition_gates: Mutex::new(HashMap::new()), - pending_provider_releases: Mutex::new(HashMap::new()), - dropped_cleanup_events: AtomicU64::new(0), - reentry_suppressed_total: AtomicU64::new(0), - deferred_wakes: Arc::default(), - divergence_cache: Mutex::new(LruCache::new(DIVERGENCE_CACHE_CAPACITY)), - divergence_cooldown_secs: 300, - } - } - - /// Cumulative count of admission requests quietly suppressed by the recent-eviction - /// reentry guard. A diagnostic counter, not a failure metric. - pub fn reentry_suppressed_total(&self) -> u64 { self.reentry_suppressed_total.load(Ordering::Relaxed) } - - /// Records one suppressed reentry retry. - pub fn record_reentry_suppressed(&self) { self.reentry_suppressed_total.fetch_add(1, Ordering::Relaxed); } - - fn transition_gate_key(username: &str, token: &str) -> String { - let mut key = String::with_capacity(username.len() + token.len() + 1); - key.push_str(username); - key.push('\0'); - key.push_str(token); - key - } - - fn admission_gate_key(username: &str) -> String { - let mut key = String::with_capacity(username.len() + 11); - key.push_str("admission"); - key.push('\0'); - key.push_str(username); - key - } - - fn cleanup_idle_transition_gates(transition_gates: &mut HashMap>>) { - transition_gates.retain(|_, gate| Arc::strong_count(gate) > 1); - } - - pub async fn acquire_playback_transition(&self, username: &str, token: &str) -> tokio::sync::OwnedMutexGuard<()> { - let key = Self::transition_gate_key(username, token); - let gate = { - let mut transition_gates = self.transition_gates.lock().await; - Self::cleanup_idle_transition_gates(&mut transition_gates); - Arc::clone(transition_gates.entry(key).or_insert_with(|| Arc::new(Mutex::new(())))) - }; - gate.lock_owned().await - } - - pub async fn acquire_user_admission(&self, username: &str) -> tokio::sync::OwnedMutexGuard<()> { - let key = Self::admission_gate_key(username); - let gate = { - let mut transition_gates = self.transition_gates.lock().await; - Self::cleanup_idle_transition_gates(&mut transition_gates); - Arc::clone(transition_gates.entry(key).or_insert_with(|| Arc::new(Mutex::new(())))) - }; - gate.lock_owned().await - } - - pub(crate) async fn pending_provider_release(&self, username: &str) -> Option { - self.pending_provider_releases.lock().await.get(username).cloned() - } - - pub(crate) async fn set_pending_provider_release(&self, username: &str, snapshot: ProviderReleaseSnapshot) { - self.pending_provider_releases.lock().await.insert(username.to_owned(), snapshot); - } - - pub(crate) async fn clear_pending_provider_release( - &self, - username: &str, - snapshot: &ProviderReleaseSnapshot, - ) -> bool { - let mut pending = self.pending_provider_releases.lock().await; - if pending.get(username) != Some(snapshot) { - return false; - } - pending.remove(username); - true - } - - fn should_reuse_stream_for_session(existing_stream: &StreamInfo, incoming_channel: &StreamChannel) -> bool { - existing_stream.channel.item_type.requires_provider_affinity() - || incoming_channel.item_type.requires_provider_affinity() - } - - pub fn set_cleanup_sender(&self, tx: mpsc::Sender) { let _ = self.cleanup_tx.set(tx); } - - pub fn set_provider_manager(&self, provider_manager: Arc) { - let _ = self.provider_manager.set(provider_manager); - } - - /// Collect a snapshot of all currently active streams for shutdown history recording. - pub async fn get_all_active_streams(&self) -> Vec { - let connections = self.connections.read().await; - connections - .by_key - .values() - .flat_map(|data| data.streams.iter().filter(|stream| !stream.preserved).cloned()) - .collect() - } - - /// Atomically removes all user requests and sessions during terminal shutdown. - pub async fn drain_for_shutdown(&self) -> Vec { - let connections = std::mem::take(&mut *self.write_connections().await); - self.adaptive_expiry_queue.lock().await.clear(); - self.adaptive_expiry_index.lock().await.clear(); - self.transition_gates.lock().await.clear(); - connections - .by_key - .into_values() - .flat_map(|data| data.streams.into_iter().filter(|stream| !stream.preserved)) - .collect() - } - - async fn log_active_user(&self) { - let is_log_user_enabled = self.is_log_user_enabled(); - let (user_count, user_connection_count) = { self.active_users_and_connections().await }; - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Connections( - user_count, - user_connection_count, - ))); - if !is_log_user_enabled { - return; - } - let last_user_count = self.last_logged_user_count.load(Ordering::Relaxed); - let last_connection_count = self.last_logged_user_connection_count.load(Ordering::Relaxed); - if last_user_count != user_count || last_connection_count != user_connection_count { - self.last_logged_user_count.store(user_count, Ordering::Relaxed); - self.last_logged_user_connection_count.store(user_connection_count, Ordering::Relaxed); - info!("Active Users: {user_count}, Active User Connections: {user_connection_count}"); - } - } - - async fn emit_promotion_update(&self, username: &str, action: PromotionAction) { - let maybe_stream = { - let user_connections = self.connections.read().await; - user_connections.by_key.get(username).and_then(|connection_data| { - connection_data.streams.iter().find(|stream| stream.uid == action.uid).cloned() - }) - }; - if let Some(stream_info) = maybe_stream { - if let Some(provider_manager) = self.provider_manager.get() { - if stream_info.channel.shared { - provider_manager.reclassify_shared_connection( - tuliprox_core::model::SharedSubscriberId::from_stream_uid(action.uid), - ConnectionKind::Normal, - action.new_priority, - ); - } else { - provider_manager.reclassify_connection_for_owner( - &action.addr, - stream_info.session_token.as_deref(), - ConnectionKind::Normal, - action.new_priority, - ); - } - } - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); - } - } - - /// Releases an active stream for the given socket address without removing the - /// socket registration (`key_by_addr`). This is used when a stream ends while - /// the underlying HTTP connection may still remain open. - pub async fn release_stream(&self, addr: &SocketAddr) -> Option { - self.release_stream_inner(addr, None).await.into_removed() - } - - pub async fn release_stream_by_uid(&self, addr: &SocketAddr, stream_uid: u32) -> Option { - self.release_stream_request_by_uid(addr, stream_uid).await.into_removed() - } - - pub async fn release_stream_request_by_uid(&self, addr: &SocketAddr, request_uid: u32) -> StreamRequestDetach { - self.release_stream_inner(addr, Some(request_uid)).await - } - - #[allow(clippy::too_many_lines)] - async fn release_stream_inner(&self, addr: &SocketAddr, stream_uid: Option) -> StreamRequestDetach { - let ( - removed_stream, - username, - expiry_entry, - preserved_update, - connection_changed, - promotion, - divergence_snapshot, - ) = { - let mut user_connections = self.write_connections().await; - - let username = if let Some(uid) = stream_uid { - user_connections.by_key.iter().find_map(|(username, connection_data)| { - connection_data - .stream_request_claims - .get(&uid) - .is_some_and(|claim| { - claim.addr == *addr - && connection_data - .streams - .iter() - .any(|stream| !stream.preserved && stream.uid == claim.stream_uid) - }) - .then(|| username.clone()) - }) - } else { - let mut matching_users = user_connections - .by_key - .iter() - .filter(|(_, connection_data)| { - connection_data.streams.iter().any(|stream| !stream.preserved && stream.addr == *addr) - }) - .map(|(username, _)| username.clone()); - let first = matching_users.next(); - let second = matching_users.next(); - if second.is_some() { - None - } else { - first - } - }; - let Some(username) = username else { - return StreamRequestDetach::NotFound; - }; - - let mut removed_stream = None; - let mut expiry_entry = None; - let mut preserved_update = None; - let mut connection_changed = false; - let mut promotion = None; - if let Some(connection_data) = user_connections.by_key.get_mut(&username) { - let stream_uid = if let Some(request_uid) = stream_uid { - let Some((display_uid, is_last)) = connection_data.detach_stream_request(request_uid) else { - return StreamRequestDetach::NotFound; - }; - if !is_last { - let has_claim_on_addr = connection_data - .stream_request_claims - .values() - .any(|claim| claim.stream_uid == display_uid && claim.addr == *addr); - let migrated_session_addrs = if has_claim_on_addr { - HashMap::new() - } else { - connection_data.release_addr_from_stream_session(addr, display_uid) - }; - let next_addr = connection_data - .stream_request_claims - .values() - .find(|claim| claim.stream_uid == display_uid) - .map(|claim| claim.addr); - if let Some(stream) = - connection_data.streams.iter_mut().find(|stream| stream.uid == display_uid) - { - stream.addr = next_addr - .or_else(|| { - stream - .session_token - .as_deref() - .and_then(|token| migrated_session_addrs.get(token)) - .copied() - .flatten() - }) - .unwrap_or(stream.addr); - stream.ts = current_time_secs(); - } - let retained = connection_data.streams.iter().find(|stream| stream.uid == display_uid).cloned(); - return retained.map_or(StreamRequestDetach::NotFound, StreamRequestDetach::Retained); - } - Some(display_uid) - } else { - None - }; - let migrated_session_addrs = if let Some(uid) = stream_uid { - connection_data.release_addr_from_stream_session(addr, uid) - } else { - connection_data.release_addr_from_sessions(addr) - }; - if let Some(stream_idx) = connection_data.streams.iter().position(|stream| { - !stream.preserved - && stream_uid.map_or(stream.addr == *addr, |uid| stream.uid == uid && stream.addr == *addr) - }) { - let migrated_addr = connection_data.streams[stream_idx] - .session_token - .as_deref() - .and_then(|token| migrated_session_addrs.get(token)) - .copied() - .flatten(); - if let Some(next_addr) = migrated_addr { - connection_data.streams[stream_idx].addr = next_addr; - connection_data.streams[stream_idx].ts = current_time_secs(); - } else if Self::should_preserve_session_stream(&connection_data.streams[stream_idx]) { - let preserved_session_token = connection_data.streams[stream_idx].session_token.clone(); - if let Some(entry) = self.build_preserved_stream_expiry( - &username, - &connection_data.streams[stream_idx], - &connection_data.sessions, - ) { - if let Some(kind) = - connection_data.stream_kinds.remove(&connection_data.streams[stream_idx].uid) - { - connection_data.decrement_kind(kind); - connection_changed = true; - } - connection_data.stream_normal_priorities.remove(&connection_data.streams[stream_idx].uid); - if let Some(session_token) = preserved_session_token.as_deref() { - Self::clear_session_counted(connection_data, session_token); - } - connection_data.streams[stream_idx].preserved = true; - preserved_update = Some(connection_data.streams[stream_idx].clone()); - expiry_entry = Some(entry); - } else { - removed_stream = Some(connection_data.streams.swap_remove(stream_idx)); - } - } else { - removed_stream = Some(connection_data.streams.swap_remove(stream_idx)); - } - if let Some(removed_stream) = removed_stream.as_ref() { - connection_data.remove_stream_request_claims(removed_stream.uid); - if let Some(kind) = connection_data.stream_kinds.remove(&removed_stream.uid) { - connection_data.decrement_kind(kind); - } - connection_data.stream_normal_priorities.remove(&removed_stream.uid); - connection_changed = true; - } - if connection_data.connections < connection_data.max_connections { - connection_data.granted_grace = false; - connection_data.grace_ts = 0; - } - if removed_stream.is_some() { - if let Some(action) = connection_data.try_promote_soft_stream() { - let promoted_stream = - connection_data.streams.iter().find(|stream| stream.uid == action.uid).cloned(); - if let Some(stream) = promoted_stream.as_ref() { - Self::promote_session_for_stream(connection_data, stream); - } - promotion = Some(action); - } - if let Some(session_token) = - removed_stream.as_ref().and_then(|stream| stream.session_token.as_deref()) - { - Self::clear_session_counted_without_stream(connection_data, session_token); - } - while connection_data.try_promote_soft_session_reservation() {} - } - } - let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, &username); - ( - removed_stream, - username, - expiry_entry, - preserved_update, - connection_changed, - promotion, - divergence_snapshot, - ) - } else { - (None, username, None, None, false, None, None) - } - }; - - self.log_divergence_snapshot(divergence_snapshot).await; - - if let Some(entry) = expiry_entry { - self.enqueue_adaptive_expiry(entry).await; - } - - if let Some(stream_info) = preserved_update.as_ref() { - self.event_manager - .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info.clone()))); - } - - if connection_changed { - if !username.is_empty() { - debug_if_enabled!( - "Released stream for user {username} at {}", - sanitize_sensitive_info(&addr.to_string()) - ); - } - self.log_active_user().await; - } - - if let Some(action) = promotion { - self.emit_promotion_update(&username, action).await; - } - - if let Some(stream_info) = removed_stream { - StreamRequestDetach::Removed(stream_info) - } else if let Some(stream_info) = preserved_update { - StreamRequestDetach::Preserved(stream_info) - } else { - StreamRequestDetach::NotFound - } - } - - #[allow(clippy::too_many_lines)] - async fn release_connection_inner(&self, addr: &SocketAddr, preserve_session_streams: bool) -> ReleasedConnection { - let ( - addr_removed, - connection_count_changed, - disconnected_users, - removed_streams, - expiry_entries, - preserved_updates, - promotions, - ) = { - let mut user_connections = self.write_connections().await; - - let registration = user_connections.key_by_addr.remove(addr); - let had_registration = registration.is_some(); - let mut disconnected_users: HashSet = - registration.map(SocketRegistration::into_usernames).unwrap_or_default(); - disconnected_users.extend( - user_connections - .by_key - .iter() - .filter(|(_, connection_data)| { - connection_data.has_session_addr(addr) - || connection_data.streams.iter().any(|stream| stream.addr == *addr) - }) - .map(|(username, _)| username.clone()), - ); - - let mut removed_streams = Vec::new(); - let mut expiry_entries = Vec::new(); - let mut preserved_updates = Vec::new(); - let mut promotions = Vec::new(); - let mut connection_count_changed = false; - for username in &disconnected_users { - if let Some(connection_data) = user_connections.by_key.get_mut(username) { - let previous_connection_count = connection_data.connections; - let migrated_session_addrs = connection_data.release_addr_from_sessions(addr); - connection_data.stream_request_claims.retain(|_, claim| claim.addr != *addr); - let mut remaining_streams = Vec::with_capacity(connection_data.streams.len()); - let mut released_kinds = Vec::new(); - let mut removed_session_tokens = HashSet::new(); - let mut preserved_session_tokens = Vec::new(); - let now = current_time_secs(); - for mut stream_info in connection_data.streams.drain(..) { - if stream_info.addr == *addr { - let migrated_addr = stream_info - .session_token - .as_deref() - .and_then(|token| migrated_session_addrs.get(token)) - .copied() - .flatten(); - if let Some(next_addr) = migrated_addr { - stream_info.addr = next_addr; - stream_info.ts = now; - remaining_streams.push(stream_info); - } else if preserve_session_streams && Self::should_preserve_session_stream(&stream_info) { - if let Some(entry) = self.build_preserved_stream_expiry( - username, - &stream_info, - &connection_data.sessions, - ) { - if let Some(kind) = connection_data.stream_kinds.remove(&stream_info.uid) { - released_kinds.push(kind); - } - connection_data.stream_normal_priorities.remove(&stream_info.uid); - if let Some(token) = stream_info.session_token.as_ref() { - preserved_session_tokens.push(token.clone()); - } - if !stream_info.preserved { - stream_info.preserved = true; - preserved_updates.push(stream_info.clone()); - } - expiry_entries.push(entry); - remaining_streams.push(stream_info); - } else { - if let Some(kind) = connection_data.stream_kinds.remove(&stream_info.uid) { - released_kinds.push(kind); - } - connection_data.stream_normal_priorities.remove(&stream_info.uid); - if let Some(token) = stream_info.session_token.as_ref() { - removed_session_tokens.insert(token.clone()); - } - removed_streams.push(stream_info); - } - } else { - if let Some(kind) = connection_data.stream_kinds.remove(&stream_info.uid) { - released_kinds.push(kind); - } - connection_data.stream_normal_priorities.remove(&stream_info.uid); - if let Some(token) = stream_info.session_token.as_ref() { - removed_session_tokens.insert(token.clone()); - } - removed_streams.push(stream_info); - } - } else { - remaining_streams.push(stream_info); - } - } - connection_data.streams = remaining_streams; - let streams = &connection_data.streams; - connection_data - .stream_request_claims - .retain(|_, claim| streams.iter().any(|stream| stream.uid == claim.stream_uid)); - if !preserve_session_streams && !removed_session_tokens.is_empty() { - connection_data.sessions.retain(|session| !removed_session_tokens.contains(&session.token)); - } - for kind in released_kinds { - connection_data.decrement_kind(kind); - } - while let Some(action) = connection_data.try_promote_soft_stream() { - let promoted_stream = - connection_data.streams.iter().find(|stream| stream.uid == action.uid).cloned(); - if let Some(stream) = promoted_stream.as_ref() { - Self::promote_session_for_stream(connection_data, stream); - } - promotions.push((username.clone(), action)); - } - for session_token in &removed_session_tokens { - Self::clear_session_counted_without_stream(connection_data, session_token); - } - for session_token in &preserved_session_tokens { - Self::clear_session_counted(connection_data, session_token); - } - while connection_data.try_promote_soft_session_reservation() {} - - if connection_data.connections < connection_data.max_connections { - connection_data.granted_grace = false; - connection_data.grace_ts = 0; - } - connection_count_changed |= connection_data.connections != previous_connection_count; - } - } - let state_changed = had_registration || !disconnected_users.is_empty(); - ( - state_changed, - connection_count_changed, - disconnected_users, - removed_streams, - expiry_entries, - preserved_updates, - promotions, - ) - }; - - for entry in expiry_entries { - self.enqueue_adaptive_expiry(entry).await; - } - - for stream_info in preserved_updates { - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); - } - - for username in &disconnected_users { - if !username.is_empty() { - debug_if_enabled!( - "Released connection for user {username} at {}", - sanitize_sensitive_info(&addr.to_string()) - ); - } - } - if connection_count_changed { - self.log_active_user().await; - } - if addr_removed { - for (username, action) in promotions { - self.emit_promotion_update(&username, action).await; - } - } - - ReleasedConnection { - addr_removed, - removed_streams, - disconnected_users: disconnected_users.into_iter().collect(), - } - } - - pub async fn release_connection(&self, addr: &SocketAddr) -> ReleasedConnection { - let released = self.release_connection_inner(addr, true).await; - // divergence check after connection release - if released.addr_removed { - for username in &released.disconnected_users { - self.check_and_log_divergence_for_user(username).await; - } - } - released - } - - pub async fn release_connection_as_kicked(&self, addr: &SocketAddr) -> ReleasedConnection { - let released = self.release_connection_inner(addr, false).await; - // divergence check after connection release - if released.addr_removed { - for username in &released.disconnected_users { - self.check_and_log_divergence_for_user(username).await; - } - } - released - } - - pub fn update_config(&self, config: &Config) { - let log_active_user = config.log.as_ref().is_some_and(|l| l.log_active_user); - let (grace_period_millis, grace_period_timeout_secs) = get_grace_options(config); - self.grace_period_millis.store(grace_period_millis, Ordering::Relaxed); - self.grace_period_timeout_secs.store(grace_period_timeout_secs, Ordering::Relaxed); - self.adaptive_session_ttl_secs.store(get_adaptive_session_ttl_secs(config), Ordering::Relaxed); - self.log_active_user.store(log_active_user, Ordering::Relaxed); - } - - pub async fn user_connections(&self, username: &str) -> u32 { - if let Some(connection_data) = self.connections.read().await.by_key.get(username) { - return connection_data.connections; - } - 0 - } - - fn check_connection_admission_with_counts( - &self, - username: &str, - connection_data: &mut UserConnectionData, - counts: UserConnectionCounts, - ) -> ConnectionAdmission { - let selected_kind = - decide_connection_kind(counts, connection_data.max_connections, connection_data.soft_connections); - let effective_connections = counts.normal.saturating_add(u32::from(counts.soft)); - - if let Some(kind) = selected_kind { - // Reset grace only once the user is back below the hard limit. - if effective_connections < connection_data.max_connections { - connection_data.granted_grace = false; - connection_data.grace_ts = 0; - } - return ConnectionAdmission::allowed(Some(kind)); - } - - let now = get_current_timestamp(); - // Check if user already used a grace period - if connection_data.granted_grace { - if effective_connections >= connection_data.max_connections - && now - connection_data.grace_ts <= self.grace_period_timeout_secs.load(Ordering::Relaxed) - { - // Grace timeout, still active, deny connection - debug!("User access denied, grace exhausted, too many connections: {username}"); - return ConnectionAdmission::exhausted(AdmissionRejectionReason::UserConnectionsExhausted, None); - } - // Grace timeout expired, reset grace counters - if effective_connections < connection_data.max_connections { - connection_data.granted_grace = false; - connection_data.grace_ts = 0; - } - } - - debug!("User access denied, too many connections: {username}"); - ConnectionAdmission::exhausted(AdmissionRejectionReason::UserConnectionsExhausted, None) - } - - fn check_connection_admission( - &self, - username: &str, - connection_data: &mut UserConnectionData, - ) -> ConnectionAdmission { - self.check_connection_admission_with_counts( - username, - connection_data, - connection_data.effective_counts_for_admission(None), - ) - } - - pub async fn connection_admission( - &self, - username: &str, - max_connections: u32, - soft_connections: u16, - ) -> ConnectionAdmission { - if max_connections > 0 || soft_connections > 0 { - if let Some(connection_data) = self.write_connections().await.by_key.get_mut(username) { - connection_data.max_connections = max_connections; - connection_data.soft_connections = soft_connections; - return self.check_connection_admission(username, connection_data); - } - } - ConnectionAdmission::allowed(Some(ConnectionKind::Normal)) - } - - pub async fn connection_permission( - &self, - username: &str, - max_connections: u32, - soft_connections: u16, - ) -> UserConnectionPermission { - self.connection_admission(username, max_connections, soft_connections).await.permission - } - - pub async fn connection_admission_for_session( - &self, - username: &str, - max_connections: u32, - soft_connections: u16, - session_token: &str, - ) -> ConnectionAdmission { - if max_connections == 0 && soft_connections == 0 { - return ConnectionAdmission::allowed(Some(ConnectionKind::Normal)); - } - - let mut connections = self.write_connections().await; - let Some(connection_data) = connections.by_key.get_mut(username) else { - return ConnectionAdmission::allowed(Some(ConnectionKind::Normal)); - }; - connection_data.max_connections = max_connections; - connection_data.soft_connections = soft_connections; - - let Some(session_index) = connection_data.sessions.iter().position(|session| session.token == session_token) - else { - return self.check_connection_admission(username, connection_data); - }; - - if connection_data.sessions[session_index].lifecycle.is_counted() { - return ConnectionAdmission::allowed( - connection_data.sessions[session_index].connection_kind.or(Some(ConnectionKind::Normal)), - ); - } - - self.check_connection_admission_with_counts( - username, - connection_data, - connection_data.effective_counts_for_admission(Some(session_token)), - ) - } - - pub async fn connection_permission_for_session( - &self, - username: &str, - max_connections: u32, - soft_connections: u16, - session_token: &str, - ) -> UserConnectionPermission { - self.connection_admission_for_session(username, max_connections, soft_connections, session_token) - .await - .permission() - } - - pub async fn refresh_session_connection_kind_for_origin_policy( - &self, - username: &str, - max_connections: u32, - soft_connections: u16, - session_token: &str, - ) -> Option { - if max_connections == 0 && soft_connections == 0 { - return Some(ConnectionKind::Normal); - } - - let (connection_kind, promotions, divergence_snapshot) = { - let mut connections = self.write_connections().await; - let connection_data = connections.by_key.get_mut(username)?; - connection_data.max_connections = max_connections; - connection_data.soft_connections = soft_connections; - - let session_index = connection_data.sessions.iter().position(|session| session.token == session_token)?; - - let promotions = Self::promote_counted_soft_session_to_normal_if_available(connection_data, session_token); - let connection_kind = if connection_data.sessions[session_index].lifecycle.is_counted() - || Self::session_has_stream(connection_data, session_token) - { - Some(connection_data.sessions[session_index].connection_kind.unwrap_or(ConnectionKind::Normal)) - } else { - let admission = self.check_connection_admission_with_counts( - username, - connection_data, - connection_data.effective_counts_for_admission(Some(session_token)), - ); - if admission.permission() == UserConnectionPermission::Allowed { - if let Some(kind) = admission.kind() { - Self::update_session_admission( - &mut connection_data.sessions[session_index], - admission.permission(), - Some(kind), - ); - } - admission.kind() - } else { - None - } - }; - let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); - - (connection_kind, promotions, divergence_snapshot) - }; - - self.log_divergence_snapshot(divergence_snapshot).await; - for action in promotions { - self.emit_promotion_update(username, action).await; - } - - connection_kind - } - - pub async fn get_eviction_candidates(&self, username: &str, _client_ip: &str) -> Vec { - let connections = self.connections.read().await; - let Some(connection_data) = connections.by_key.get(username) else { - return Vec::new(); - }; - let mut addr_counts = HashMap::new(); - for stream in &connection_data.streams { - // The stream kind is the charged slot. Session lifecycle may lag behind - // the stream registration, so it cannot determine eviction eligibility. - if !stream.preserved && connection_data.stream_kinds.contains_key(&stream.uid) { - addr_counts - .entry(stream.addr) - .and_modify(|count: &mut u8| *count = count.saturating_add(1)) - .or_insert(1_u8); - } - } - let candidates: Vec<_> = connection_data - .streams - .iter() - .filter(|stream| stream.preserved || connection_data.stream_kinds.contains_key(&stream.uid)) - .filter(|stream| { - let addr_count = addr_counts.get(&stream.addr).copied().unwrap_or(0); - if stream.preserved { - // Preserved streams are always valid eviction candidates — they hold no counted - // slot. addr_count is 0 for preserved-only addresses, 1+ for addresses with - // counted competition. Either way they can be evicted. - true - } else { - // Non-preserved streams: only on singleton counted addresses. - // addr_count 0 = no counted streams at this address (shouldn't happen since - // non-preserved streams aren't preserved, but addr_count would be >= 1). - // addr_count 1 = single counted stream at address — candidate. - // addr_count > 1 = multiple counted streams — not a singleton, not candidate. - addr_count == 1 - } - }) - .map(|s| crate::EvictionCandidate { - addr: s.addr, - client_ip: s.client_ip.clone(), - virtual_id: shared::model::VirtualId::new(s.channel.virtual_id), - ts: s.ts, - uid: s.uid, - }) - .collect(); - candidates - } - - pub async fn grant_grace(&self, username: &str) -> bool { - if self.grace_period_millis.load(Ordering::Relaxed) == 0 { - debug!("Grace grant denied, grace_period_millis is zero for {username}"); - return false; - } - let mut connections = self.write_connections().await; - if let Some(connection_data) = connections.by_key.get_mut(username) { - let now = get_current_timestamp(); - if connection_data.connections < connection_data.max_connections { - debug!( - "Grace grant denied for {username}, user not at connection limit ({}/{})", - connection_data.connections, connection_data.max_connections - ); - return false; - } - if connection_data.granted_grace - && connection_data.connections >= connection_data.max_connections - && now - connection_data.grace_ts <= self.grace_period_timeout_secs.load(Ordering::Relaxed) - { - debug!("Grace grant denied, still within active grace timeout for {username}"); - return false; - } - connection_data.granted_grace = true; - connection_data.grace_ts = now; - debug!("Granted a grace period for user access: {username}"); - return true; - } - false - } - - pub async fn active_users_and_connections(&self) -> (usize, usize) { - self.gc(); - let user_connections = self.connections.read().await; - user_connections - .by_key - .values() - .filter_map(|c| { - let effective = c.connections as usize; - if effective > 0 { - Some(effective) - } else { - None - } - }) - .fold((0usize, 0usize), |(user_count, conn_count), effective| (user_count + 1, conn_count + effective)) - } - - pub async fn playback_resource_counts(&self) -> (usize, usize) { - self.gc(); - let user_connections = self.connections.read().await; - user_connections.by_key.values().fold((0, 0), |(requests, sessions), connection| { - (requests + connection.stream_request_claims.len(), sessions + connection.sessions.len()) - }) - } - - pub async fn update_stream_detail( - &self, - addr: &SocketAddr, - video_type: CustomVideoStreamType, - ) -> Option { - let uid = { - let connections = self.connections.read().await; - let mut streams = - connections.by_key.values().flat_map(|data| &data.streams).filter(|stream| &stream.addr == addr); - let uid = streams.next()?.uid; - if streams.next().is_some() { - return None; - } - uid - }; - self.update_stream_detail_by_uid(uid, video_type).await - } - - pub async fn update_stream_detail_by_uid(&self, uid: u32, video_type: CustomVideoStreamType) -> Option { - let mut user_connections = self.write_connections().await; - for connection_data in user_connections.by_key.values_mut() { - for stream in &mut connection_data.streams { - if stream.uid == uid { - // IMPORTANT: `resolve_disconnect_reason` in connection_manager.rs parses - // `channel.title` back via `CustomVideoStreamType::from_str` to determine QoS - // disconnect reasons. If these values change, update that function too. - stream.provider = "tuliprox".intern(); - stream.channel.title = video_type.to_string().into(); - stream.channel.group = "".intern(); - stream.channel.technical = Some(Self::custom_stream_technical_info()); - return Some(stream.clone()); - } - } - } - None - } - - pub async fn add_connection(&self, addr: &SocketAddr) { - self.gc(); - let mut user_connections = self.write_connections().await; - user_connections - .key_by_addr - .entry(*addr) - .and_modify(|registration| registration.ts = current_time_secs()) - .or_insert_with(SocketRegistration::anonymous); - } - - pub async fn update_connection(&self, update: ActiveUserConnectionParams<'_>) -> Option { - self.update_connection_with_session_registration(update, None).await - } - - #[allow(clippy::too_many_lines)] - pub async fn update_connection_with_session_registration( - &self, - update: ActiveUserConnectionParams<'_>, - session_registration: Option<&PlaybackSessionRegistration>, - ) -> Option { - let ActiveUserConnectionParams { - uid, - meter_uid, - username, - max_connections, - soft_connections, - connection_kind, - priority, - soft_priority: _, - fingerprint, - provider, - stream_channel, - user_agent, - session_token, - } = update; - let (stream_info, divergence_snapshot, connection_count_changed) = { - let mut user_connections = self.write_connections().await; - if let Some(requirement) = session_registration { - let token = session_token?; - let (data, session) = user_connections.current_session(username, token, requirement.identity)?; - if requirement.enforce_limits - && !requirement.grace_admitted - && !session.lifecycle.is_counted() - && !Self::session_has_stream(data, token) - && decide_connection_kind( - data.effective_counts_for_admission(Some(token)), - max_connections, - soft_connections, - ) != Some(connection_kind) - { - return None; - } - } - - let now = current_time_secs(); - let registration = - user_connections.key_by_addr.entry(fingerprint.addr).or_insert_with(SocketRegistration::anonymous); - registration.add_user(username, now); - - let tracked_socket_count = user_connections.key_by_addr.len(); - let connection_data = user_connections - .by_key - .entry(username.to_string()) - .or_insert_with(|| UserConnectionData::new(0, max_connections, soft_connections)); - connection_data.max_connections = max_connections; - connection_data.soft_connections = soft_connections; - let previous_connection_count = connection_data.connections; - - if let Some(token) = session_token { - if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { - session.ts = now; - remember_session_addr(session, fingerprint.addr); - Self::bump_session_transition_version(session); - } - } - - let user_agent_string = user_agent.to_string(); - let reserved_session_kind = session_token.and_then(|token| { - connection_data - .sessions - .iter() - .find(|session| session.token == token && session.lifecycle.is_counted()) - .map(|session| session.connection_kind.unwrap_or(connection_kind)) - }); - - let existing_stream_info = connection_data - .streams - .iter() - .position(|stream_info| { - !stream_channel.shared - && match session_token { - Some(token) => { - stream_info.session_token.as_deref() == Some(token) - && Self::should_reuse_stream_for_session(stream_info, stream_channel) - } - None => stream_info.uid == uid && stream_info.session_token.is_none(), - } - }) - .map(|stream_idx| { - let session_started_at = session_token.and_then(|token| { - connection_data.sessions.iter().find(|s| s.token == token).map(|s| s.started_at) - }); - - let stream_info = &mut connection_data.streams[stream_idx]; - let client_ip = fingerprint.client_ip.clone(); - let preserve_started_at = stream_info.session_token.is_some() - && (stream_info.channel.item_type.is_live_adaptive() - || stream_channel.item_type.is_live_adaptive()); - let was_preserved = stream_info.preserved; - let old_session_id = stream_history_session_id(stream_info.ts, stream_info.uid); - stream_info.meter_uid = meter_uid; - stream_info.addr = fingerprint.addr; - stream_info.client_ip.clone_from(&client_ip); - stream_info.country_code = self.lookup_country(&client_ip); - stream_info.channel = stream_channel.clone(); - stream_info.provider = provider.clone(); - stream_info.user_agent.clone_from(&user_agent_string); - - if let Some(started_at) = session_started_at { - stream_info.started_at = started_at; - } - - if preserve_started_at { - let now = current_time_secs(); - if utc_day_from_secs(stream_info.ts) != utc_day_from_secs(now) { - stream_info.ts = now; - stream_info.previous_session_id = Some(old_session_id); - } - } else { - stream_info.ts = current_time_secs(); - } - - if let Some(token) = session_token { - stream_info.session_token = Some(token.to_string()); - } - if was_preserved { - stream_info.preserved = false; - } - connection_data.stream_normal_priorities.insert(stream_info.uid, priority); - let result = stream_info.clone(); - stream_info.previous_session_id = None; - (result, was_preserved) - }); - let (stream_info, divergence_snapshot) = if let Some((stream_info, was_preserved)) = existing_stream_info { - let effective_connection_kind = reserved_session_kind.unwrap_or(connection_kind); - connection_data.attach_stream_request(uid, stream_info.uid, fingerprint.addr); - if was_preserved { - connection_data.increment_kind(effective_connection_kind); - } - connection_data.stream_kinds.insert(stream_info.uid, effective_connection_kind); - connection_data.stream_normal_priorities.insert(stream_info.uid, priority); - if let Some(token) = session_token { - if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { - Self::mark_session_committed(session, effective_connection_kind); - } - } - let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); - (stream_info, divergence_snapshot) - } else { - let effective_connection_kind = reserved_session_kind.unwrap_or(connection_kind); - let country_code = self.lookup_country(&fingerprint.client_ip); - - let mut stream_info = StreamInfo::new(shared::model::StreamInfoParams { - uid, - meter_uid, - username, - addr: &fingerprint.addr, - client_ip: &fingerprint.client_ip, - provider, - stream_channel: stream_channel.clone(), - user_agent: user_agent_string, - country_code, - session_token, - }); - - if let Some(token) = session_token { - if let Some(session) = connection_data.sessions.iter().find(|s| s.token == token) { - stream_info.started_at = session.started_at; - } - } - - if reserved_session_kind.is_none() { - connection_data.increment_kind(effective_connection_kind); - } - connection_data.streams.push(stream_info.clone()); - connection_data.attach_stream_request(uid, stream_info.uid, fingerprint.addr); - connection_data.stream_kinds.insert(stream_info.uid, effective_connection_kind); - connection_data.stream_normal_priorities.insert(stream_info.uid, priority); - if let Some(token) = session_token { - if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { - Self::mark_session_committed(session, effective_connection_kind); - } - } - Self::log_connection_added(username, &fingerprint.addr, connection_data, tracked_socket_count); - let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); - (stream_info, divergence_snapshot) - }; - let connection_count_changed = connection_data.connections != previous_connection_count; - (stream_info, divergence_snapshot, connection_count_changed) - }; - - self.log_divergence_snapshot(divergence_snapshot).await; - - if connection_count_changed { - self.log_active_user().await; - } - - Some(stream_info) - } - - fn is_log_user_enabled(&self) -> bool { self.log_active_user.load(Ordering::Relaxed) } - - fn build_preserved_stream_expiry( - &self, - username: &str, - stream: &StreamInfo, - sessions: &[UserSession], - ) -> Option { - let session_token = stream.session_token.as_deref()?; - // Catchup segment gaps can briefly lose the UserSession row; still preserve the panel - // row using the stream timestamp so Streams does not blink between archive chunks. - let session_ts = if let Some(session) = sessions.iter().find(|session| session.token == session_token) { - session.ts - } else if stream.channel.item_type == PlaylistItemType::Catchup || is_catchup_session_token(session_token) { - stream.ts - } else { - return None; - }; - - let ttl_secs = self.adaptive_session_ttl_secs.load(Ordering::Relaxed); - let expires_at = session_ts.saturating_add(ttl_secs); - Some(AdaptiveExpiryEntry { - expires_at, - username: username.to_string(), - session_token: session_token.to_string(), - uid: stream.uid, - }) - } - - async fn enqueue_adaptive_expiry(&self, entry: AdaptiveExpiryEntry) { - let key = AdaptiveExpiryKey { - username: entry.username.clone(), - session_token: entry.session_token.clone(), - uid: entry.uid, - }; - - let mut expiry_index = self.adaptive_expiry_index.lock().await; - expiry_index.insert(key, entry.expires_at); - drop(expiry_index); - - let mut queue = self.adaptive_expiry_queue.lock().await; - let wake_worker = queue.peek().is_none_or(|current| entry.expires_at < current.0.expires_at); - queue.push(Reverse(entry)); - if wake_worker { - self.adaptive_expiry_notify.notify_one(); - } - } - - fn new_user_session(params: &UserSessionParams<'_>) -> UserSession { - let now = current_time_secs(); - UserSession { - token: params.session_token.to_string(), - transition_version: 1, - virtual_id: params.virtual_id, - provider: params.provider.intern(), - stream_url: params.stream_url.intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - provider_session_cookies: Arc::default(), - incarnation: next_session_incarnation(), - binding_generation: 0, - change_signal: SessionChangeSignal::new(params.change_wakes), - media_started: Arc::new(AtomicBool::new(false)), - user_agent_stream_index: None, - addr: *params.addr, - socket_bound: params.socket_bound, - active_addrs: vec![*params.addr], - ts: now, - started_at: now, - permission: params.connection_permission, - connection_kind: params.connection_kind, - lifecycle: PlaybackLifecycle::Prepared, - } - } - - fn promote_session_for_stream(connection_data: &mut UserConnectionData, stream: &StreamInfo) { - if let Some(token) = stream.session_token.as_deref() { - if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { - Self::mark_session_committed(session, ConnectionKind::Normal); - } - } - } - - fn collect_promotions_after_capacity_release(connection_data: &mut UserConnectionData) -> Vec { - let mut promotions = Vec::new(); - while let Some(action) = connection_data.try_promote_soft_stream() { - let promoted_stream = connection_data.streams.iter().find(|stream| stream.uid == action.uid).cloned(); - if let Some(stream) = promoted_stream.as_ref() { - Self::promote_session_for_stream(connection_data, stream); - } - promotions.push(action); - } - while connection_data.try_promote_soft_session_reservation() {} - promotions - } - - fn promote_counted_soft_session_to_normal_if_available( - connection_data: &mut UserConnectionData, - session_token: &str, - ) -> Vec { - if connection_data.max_connections > 0 && connection_data.counts.normal >= connection_data.max_connections { - return Vec::new(); - } - - let Some(session_index) = connection_data.sessions.iter().position(|session| { - session.token == session_token - && session.lifecycle.is_counted() - && session.connection_kind == Some(ConnectionKind::Soft) - }) else { - return Vec::new(); - }; - - if connection_data.counts.soft == 0 { - return Vec::new(); - } - - connection_data.counts.normal = connection_data.counts.normal.saturating_add(1); - connection_data.counts.soft = connection_data.counts.soft.saturating_sub(1); - connection_data.sessions[session_index].connection_kind = Some(ConnectionKind::Normal); - Self::bump_session_transition_version(&mut connection_data.sessions[session_index]); - - let mut promotions = Vec::new(); - for stream in - connection_data.streams.iter().filter(|stream| stream.session_token.as_deref() == Some(session_token)) - { - if connection_data.stream_kinds.get(&stream.uid) != Some(&ConnectionKind::Soft) { - continue; - } - let new_priority = connection_data.stream_normal_priorities.get(&stream.uid).copied().unwrap_or_default(); - connection_data.stream_kinds.insert(stream.uid, ConnectionKind::Normal); - promotions.push(PromotionAction { addr: stream.addr, uid: stream.uid, new_priority }); - } - promotions - } - - fn bump_session_transition_version(session: &mut UserSession) -> u64 { - session.transition_version = session.transition_version.saturating_add(1); - session.transition_version - } - - fn mark_session_committed(session: &mut UserSession, kind: ConnectionKind) { - session.connection_kind = Some(kind); - session.lifecycle = PlaybackLifecycle::Active; - Self::bump_session_transition_version(session); - } - - fn update_session_admission( - session: &mut UserSession, - permission: UserConnectionPermission, - kind: Option, - ) { - session.set_permission(permission); - if let Some(kind) = kind { - session.connection_kind = Some(kind); - } - } - - fn clear_session_pending_with_permission( - session: &mut UserSession, - permission: UserConnectionPermission, - wake_source: PendingProviderWakeSource, - ) { - if let PlaybackLifecycle::PendingProvider { data } = &mut session.lifecycle { - data.wake_source = Some(wake_source); - } - Self::bump_session_transition_version(session); - session.set_permission(permission); - } - - fn session_has_stream(connection_data: &UserConnectionData, session_token: &str) -> bool { - connection_data.streams.iter().any(|stream| stream.session_token.as_deref() == Some(session_token)) - } - - fn clear_session_counted_without_stream(connection_data: &mut UserConnectionData, session_token: &str) { - if Self::session_has_stream(connection_data, session_token) { - return; - } - if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == session_token) { - match session.lifecycle { - PlaybackLifecycle::Active => { - session.lifecycle = PlaybackLifecycle::Preserved; - } - // GraceActive without a stream: grace failed, expire the session. - // This can happen when the grace window times out while the client - // is still connecting but hasn't opened a stream yet. - PlaybackLifecycle::GraceActive => { - session.expire(); - } - _ => {} - } - } - } - - fn clear_session_counted(connection_data: &mut UserConnectionData, session_token: &str) { - if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == session_token) { - match session.lifecycle { - PlaybackLifecycle::Active => { - session.lifecycle = PlaybackLifecycle::Preserved; - } - PlaybackLifecycle::GraceActive => { - // GraceActive without stream: the grace failed. The stream was already - // removed (this function is called after stream removal), so expire the session. - session.expire(); - } - _ => {} - } - } - } - - fn release_expired_session_reservations(connection_data: &mut UserConnectionData, now: u64) { - let expired_counted = connection_data - .sessions - .iter() - .filter(|session| session.lifecycle.is_counted()) - .filter(|session| now.saturating_sub(session.ts) >= USER_CON_TTL) - .filter(|session| !Self::session_has_stream(connection_data, session.token.as_str())) - .map(|session| (session.token.clone(), session.connection_kind.unwrap_or(ConnectionKind::Normal))) - .collect::>(); - - for (_, kind) in &expired_counted { - connection_data.decrement_kind(*kind); - } - for (token, _) in expired_counted { - Self::clear_session_counted_without_stream(connection_data, &token); - } - while connection_data.try_promote_soft_session_reservation() {} - } - - pub async fn connection_admission_for_session_activation( - &self, - username: &str, - max_connections: u32, - soft_connections: u16, - session_token: &str, - ) -> ConnectionAdmission { - if max_connections == 0 && soft_connections == 0 { - return ConnectionAdmission::allowed(Some(ConnectionKind::Normal)); - } - - let mut connections = self.write_connections().await; - let Some(connection_data) = connections.by_key.get_mut(username) else { - return ConnectionAdmission::allowed(Some(ConnectionKind::Normal)); - }; - connection_data.max_connections = max_connections; - connection_data.soft_connections = soft_connections; - - let Some(session_index) = connection_data.sessions.iter().position(|session| session.token == session_token) - else { - return self.check_connection_admission(username, connection_data); - }; - - // Existing counted session or any stream row for this token (including soft-preserved - // HLS/Catchup between segments): entitled to its slot. #807 switched this to - // active-only stream checks, so every LiveHls segment gap returned Exhausted - // and kick-evicted/terminated the same session (retry storm since v3.3.79). - if connection_data.sessions[session_index].lifecycle.is_counted() - || Self::session_has_stream(connection_data, session_token) - { - return ConnectionAdmission::allowed( - connection_data.sessions[session_index].connection_kind.or(Some(ConnectionKind::Normal)), - ); - } - - // Uncounted session with no stream row: run normal admission. - // Same-token soft-preserve is handled above via `session_has_stream` (3.3.78 semantics). - // Do not return Exhausted for own preserved rows — that forced self-eviction on HLS gaps. - let admission = self.check_connection_admission_with_counts( - username, - connection_data, - connection_data.effective_counts_for_admission(Some(session_token)), - ); - if admission.permission() == UserConnectionPermission::Allowed { - let session = &mut connection_data.sessions[session_index]; - Self::update_session_admission(session, admission.permission(), admission.kind()); - } - admission - } - - pub async fn ensure_user_session_placeholder(&self, request: CreateUserSessionParams<'_>) -> u64 { - let CreateUserSessionParams { - user, - session_token, - virtual_id, - provider, - stream_url, - addr, - connection_permission, - connection_kind, - socket_bound, - } = request; - self.gc(); - - let username = user.username.clone(); - let mut user_connections = self.write_connections().await; - let connection_data = user_connections - .by_key - .entry(username.clone()) - .or_insert_with(|| UserConnectionData::new(0, user.max_connections, user.soft_connections)); - - if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == session_token) { - session.ts = current_time_secs(); - session.socket_bound = socket_bound; - remember_session_addr(session, *addr); - if session.connection_kind.is_none() { - session.connection_kind = connection_kind; - } - if session.permission == UserConnectionPermission::Exhausted { - Self::update_session_admission(session, connection_permission, None); - } - let version = Self::bump_session_transition_version(session); - let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, &username); - drop(user_connections); - self.log_divergence_snapshot(divergence_snapshot).await; - return version; - } - - let session = Self::new_user_session(&UserSessionParams { - change_wakes: &self.deferred_wakes, - session_token, - virtual_id, - provider, - stream_url, - addr, - connection_permission, - connection_kind, - socket_bound, - }); - let version = session.transition_version; - connection_data.add_session(session); - let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, &username); - drop(user_connections); - self.log_divergence_snapshot(divergence_snapshot).await; - version - } - - pub async fn release_unbound_session_reservation( - &self, - username: &str, - session_token: &str, - expected_transition_version: Option, - remove_session_if_unbound: bool, - ) { - let (connection_changed, user_removed, promotions, divergence_snapshot) = { - let mut user_connections = self.write_connections().await; - let (connection_changed, user_removed, promotions, divergence_snapshot) = { - let Some(connection_data) = user_connections.by_key.get_mut(username) else { - return; - }; - - if Self::session_has_stream(connection_data, session_token) { - return; - } - - let Some(session_index) = - connection_data.sessions.iter().position(|session| session.token == session_token) - else { - return; - }; - - if expected_transition_version - .is_some_and(|expected| connection_data.sessions[session_index].transition_version != expected) - { - return; - } - - let mut connection_changed = false; - if connection_data.sessions[session_index].lifecycle.is_counted() { - let kind = - connection_data.sessions[session_index].connection_kind.unwrap_or(ConnectionKind::Normal); - connection_data.decrement_kind(kind); - connection_data.sessions[session_index].expire(); - connection_changed = true; - } - connection_data.sessions[session_index].transition_version = - connection_data.sessions[session_index].transition_version.saturating_add(1); - - if remove_session_if_unbound { - connection_data.sessions.swap_remove(session_index); - } - - if connection_data.connections < connection_data.max_connections { - connection_data.granted_grace = false; - connection_data.grace_ts = 0; - } - - let mut promotions = Vec::new(); - while let Some(action) = connection_data.try_promote_soft_stream() { - let promoted_stream = - connection_data.streams.iter().find(|stream| stream.uid == action.uid).cloned(); - if let Some(stream) = promoted_stream.as_ref() { - Self::promote_session_for_stream(connection_data, stream); - } - promotions.push(action); - } - while connection_data.try_promote_soft_session_reservation() {} - - let user_removed = connection_data.connections == 0 - && connection_data.streams.is_empty() - && connection_data.sessions.is_empty(); - let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); - - (connection_changed, user_removed, promotions, divergence_snapshot) - }; - - if user_removed { - user_connections.by_key.remove(username); - } - - (connection_changed, user_removed, promotions, divergence_snapshot) - }; - - self.log_divergence_snapshot(divergence_snapshot).await; - - if connection_changed || user_removed { - self.log_active_user().await; - } - for action in promotions { - self.emit_promotion_update(username, action).await; - } - } - - pub async fn release_session_streams_and_counted_reservation(&self, username: &str, session_token: &str) -> bool { - let (connection_changed, user_removed, promotions, divergence_snapshot) = { - let mut user_connections = self.write_connections().await; - let (connection_changed, user_removed, promotions, divergence_snapshot) = { - let Some(connection_data) = user_connections.by_key.get_mut(username) else { - return false; - }; - - let counted_kind = connection_data - .sessions - .iter() - .find(|session| session.token == session_token && session.lifecycle.is_counted()) - .and_then(|session| session.connection_kind); - let (_removed_streams, mut connection_changed) = - connection_data.remove_streams_for_session_and_release_counted(session_token, counted_kind); - Self::clear_session_counted_without_stream(connection_data, session_token); - - if connection_data.connections < connection_data.max_connections { - connection_data.granted_grace = false; - connection_data.grace_ts = 0; - } - - let promotions = Self::collect_promotions_after_capacity_release(connection_data); - let user_removed = connection_data.connections == 0 - && connection_data.streams.is_empty() - && connection_data.sessions.is_empty(); - let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); - connection_changed |= !promotions.is_empty(); - - (connection_changed, user_removed, promotions, divergence_snapshot) - }; - - if user_removed { - user_connections.by_key.remove(username); - } - - (connection_changed, user_removed, promotions, divergence_snapshot) - }; - - self.log_divergence_snapshot(divergence_snapshot).await; - - if connection_changed || user_removed { - self.log_active_user().await; - } - for action in promotions { - self.emit_promotion_update(username, action).await; - } - connection_changed || user_removed - } - - pub async fn create_user_session(&self, request: CreateUserSessionParams<'_>) -> String { - let CreateUserSessionParams { - user, - session_token, - virtual_id, - provider, - stream_url, - addr, - connection_permission, - connection_kind, - socket_bound, - } = request; - self.gc(); - - let username = user.username.clone(); - let mut user_connections = self.write_connections().await; - let connection_data = user_connections.by_key.entry(username.clone()).or_insert_with(|| { - debug_if_enabled!("Creating first session for user {username} {}", sanitize_sensitive_info(stream_url)); - let mut data = UserConnectionData::new(0, user.max_connections, user.soft_connections); - let session = Self::new_user_session(&UserSessionParams { - change_wakes: &self.deferred_wakes, - session_token, - virtual_id, - provider, - stream_url, - addr, - connection_permission, - connection_kind, - socket_bound, - }); - data.add_session(session); - data - }); - - // If a session exists, update it - for session in &mut connection_data.sessions { - if session.token == session_token { - session.ts = current_time_secs(); - session.socket_bound = socket_bound; - remember_session_addr(session, *addr); - Self::bump_session_transition_version(session); - let mut reset_provider_session_headers = false; - if &*session.stream_url != stream_url { - // Provider cookies are host scoped: alternating child playlists on one host keep them. - reset_provider_session_headers |= url_host_key(&session.stream_url) != url_host_key(stream_url); - session.stream_url = stream_url.intern(); - } - if &*session.provider != provider { - session.switch_provider(provider.intern()); - } - if reset_provider_session_headers { - session.provider_session_headers.clear(); - session.provider_session_headers_host = None; - } - // Normalize stale lifecycle states on session refresh. - // Expired, PendingProvider, and Preserved sessions cannot stay in those states - // when a new request arrives for the same session token - the request is either - // a reactivation (Activate) or a follow-up on a still-valid logical playback. - match session.lifecycle { - PlaybackLifecycle::Expired => { - session.lifecycle = PlaybackLifecycle::Prepared; - } - // PendingProvider: pending wait continues until explicitly resolved. - // Preserved: stays preserved until explicit reactivation via activation path. - // Prepared: placeholder session, no counted lease. - // Active: session is already in a valid counted state. - // All these keep their current state - session.refresh() alone does not advance it. - #[allow(clippy::match_same_arms)] - PlaybackLifecycle::PendingProvider { .. } - | PlaybackLifecycle::Preserved - | PlaybackLifecycle::Prepared - | PlaybackLifecycle::Active => {} - PlaybackLifecycle::GraceActive => { - // GraceActive refresh keeps the provisional state. Grace window is still - // running — refresh does not advance it. The grace task will resolve it. - } - } - Self::update_session_admission(session, connection_permission, connection_kind); - debug_if_enabled!( - "Using session for user {} with url: {}", - user.username, - sanitize_sensitive_info(stream_url) - ); - return session.token.clone(); - } - } - - // If no session exists, create one - debug_if_enabled!( - "Creating session for user {} with url: {}", - user.username, - sanitize_sensitive_info(stream_url) - ); - let session = Self::new_user_session(&UserSessionParams { - change_wakes: &self.deferred_wakes, - session_token, - virtual_id, - provider, - stream_url, - addr, - connection_permission, - connection_kind, - socket_bound, - }); - let token = session.token.clone(); - connection_data.add_session(session); - let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, &username); - drop(user_connections); - self.log_divergence_snapshot(divergence_snapshot).await; - token - } - - pub async fn update_session_addr(&self, username: &str, token: &str, addr: &SocketAddr) { - let now = current_time_secs(); - let mut user_connections = self.write_connections().await; - if let Some(connection_data) = user_connections.by_key.get_mut(username) { - let update_result = if let Some(session) = connection_data.sessions.iter_mut().find(|s| s.token == token) { - let previous_addr = session.addr; - remember_session_addr(session, *addr); - session.ts = now; - Self::bump_session_transition_version(session); - for stream in &mut connection_data.streams { - if stream.addr == previous_addr && stream.session_token.as_deref() == Some(token) { - stream.addr = *addr; - stream.ts = now; - } - } - let prune_previous_registration = previous_addr != *addr - && !connection_data.has_session_addr(&previous_addr) - && !connection_data.streams.iter().any(|stream| stream.addr == previous_addr); - Some((previous_addr, prune_previous_registration)) - } else { - None - }; - - if let Some((previous_addr, prune_previous_registration)) = update_result { - let registration = - user_connections.key_by_addr.entry(*addr).or_insert_with(SocketRegistration::anonymous); - registration.add_user(username, now); - if prune_previous_registration { - let can_remove_previous = user_connections - .key_by_addr - .get_mut(&previous_addr) - .is_some_and(|registration| registration.remove_user(username)); - if can_remove_previous { - user_connections.key_by_addr.remove(&previous_addr); - } - } - debug_if_enabled!( - "Updated session {token} for {username} address {} -> {}", - sanitize_sensitive_info(&previous_addr.to_string()), - sanitize_sensitive_info(&addr.to_string()) - ); - } - } - } - - pub async fn update_session_provider_binding( - &self, - username: &str, - token: &str, - provider: Arc, - stream_url: Arc, - ) { - let now = current_time_secs(); - let mut user_connections = self.write_connections().await; - if let Some(connection_data) = user_connections.by_key.get_mut(username) { - if let Some(session) = connection_data.sessions.iter_mut().find(|s| s.token == token) { - let previous_provider = session.provider.clone(); - if session.provider != provider { - session.switch_provider(provider.clone()); - } - session.stream_url = stream_url.clone(); - session.ts = now; - Self::bump_session_transition_version(session); - for stream in &mut connection_data.streams { - if stream.session_token.as_deref() == Some(token) { - stream.provider = provider.clone(); - stream.ts = now; - } - } - debug_if_enabled!( - "Updated session {token} for {username} provider binding {} -> {}", - sanitize_sensitive_info(&previous_provider), - sanitize_sensitive_info(&provider) - ); - } - } - } - - pub async fn clear_unbound_session_addr(&self, username: &str, token: &str, addr: &SocketAddr) { - let now = current_time_secs(); - let mut user_connections = self.write_connections().await; - let Some(connection_data) = user_connections.by_key.get_mut(username) else { - return; - }; - let addr_has_active_stream_for_session = connection_data - .streams - .iter() - .any(|stream| stream.session_token.as_deref() == Some(token) && stream.addr == *addr && !stream.preserved); - if addr_has_active_stream_for_session { - return; - } - - let cleared = if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) - { - let changed = clear_session_addr(session, addr); - if changed { - Self::bump_session_transition_version(session); - } - changed - } else { - false - }; - let can_remove_registration = !connection_data.has_session_addr(addr) - && !connection_data.streams.iter().any(|stream| stream.addr == *addr); - if can_remove_registration { - let can_remove = user_connections - .key_by_addr - .get_mut(addr) - .is_some_and(|registration| registration.remove_user(username)); - if can_remove { - user_connections.key_by_addr.remove(addr); - } - } else if cleared { - if let Some(registration) = user_connections.key_by_addr.get_mut(addr) { - registration.ts = now; - } - } - } - - pub async fn mark_pending_provider( - &self, - username: &str, - token: &str, - reason_code: PendingProviderReason, - deadline: u64, - ) -> Option { - let mut user_connections = self.write_connections().await; - let connection_data = user_connections.by_key.get_mut(username)?; - let now = current_time_secs(); - if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { - let version = match session.lifecycle { - PlaybackLifecycle::PendingProvider { ref data } => data.version.saturating_add(1), - _ => 1, - }; - // Capture counted status BEFORE lifecycle transition to PendingProvider. - // is_counted() returns false for PendingProvider, so we must check first. - let kind = if session.lifecycle.is_counted() { - Some(session.connection_kind.unwrap_or(ConnectionKind::Normal)) - } else { - None - }; - session.ts = now; - Self::bump_session_transition_version(session); - Self::update_session_admission(session, UserConnectionPermission::GracePeriod, None); - session.lifecycle = PlaybackLifecycle::PendingProvider { - data: PendingProviderState { reason_code, created_at: now, deadline, version, wake_source: None }, - }; - if let Some(kind) = kind { - connection_data.decrement_kind(kind); - } - return Some(version); - } - None - } - - pub async fn activate_pending_provider( - &self, - username: &str, - token: &str, - expected_version: u64, - wake_source: PendingProviderWakeSource, - ) { - let mut user_connections = self.write_connections().await; - let Some(connection_data) = user_connections.by_key.get_mut(username) else { - return; - }; - if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { - let PlaybackLifecycle::PendingProvider { data } = &mut session.lifecycle else { - return; - }; - if data.version != expected_version { - return; - } - data.wake_source = Some(wake_source); - Self::bump_session_transition_version(session); - session.permission = UserConnectionPermission::Allowed; - session.lifecycle = PlaybackLifecycle::Active; - } - } - - /// Returns the current `transition_version` if the session is in `GraceActive` lifecycle. - /// Used by the grace task to confirm the session is still in `GraceActive` before committing. - pub async fn grace_active_version(&self, username: &str, token: &str) -> Option { - let connections = self.connections.read().await; - let connection_data = connections.by_key.get(username)?; - let session = connection_data.sessions.iter().find(|s| s.token == token)?; - if session.lifecycle == PlaybackLifecycle::GraceActive { - Some(session.transition_version) - } else { - None - } - } - - /// Marks a session as `GraceActive` — the session was granted immediate grace - /// (`GraceMode::Instant`) and is provisionally active. The session counts against - /// admission limits in this state. - /// - /// This corresponds to `Prepared -> GraceActive` in the playback state machine. - /// The session remains in `GraceActive` until either: - /// - `activate_grace_active` confirms it (grace window succeeded -> `GraceActive -> Active`) - /// - `expire_grace_active` expires it (grace window failed -> `GraceActive -> Expired`) - pub async fn mark_grace_active(&self, username: &str, token: &str) { - let mut user_connections = self.write_connections().await; - let Some(connection_data) = user_connections.by_key.get_mut(username) else { - return; - }; - let Some(session_index) = connection_data.sessions.iter().position(|session| session.token == token) else { - return; - }; - if connection_data.sessions[session_index].lifecycle == PlaybackLifecycle::GraceActive { - return; // already grace active - } - // Collect fields while only borrowing sessions. - let kind = connection_data.sessions[session_index].connection_kind.unwrap_or(ConnectionKind::Normal); - let needs_count = !connection_data.sessions[session_index].lifecycle.is_counted(); - let now = current_time_secs(); - // Now mutate. Use index access to avoid nested &mut borrows. - connection_data.sessions[session_index].ts = now; - Self::bump_session_transition_version(&mut connection_data.sessions[session_index]); - if needs_count { - connection_data.increment_kind(kind); - } - connection_data.sessions[session_index].lifecycle = PlaybackLifecycle::GraceActive; - } - - /// Activates a `GraceActive` session when the grace window resolves successfully. - /// - /// This corresponds to `GraceActive -> Active` in the playback state machine. - /// The session remains counted and the kind counts are already correct from - /// the `GraceActive` provisional state. - pub async fn activate_grace_active(&self, username: &str, token: &str, expected_version: u64) { - let mut user_connections = self.write_connections().await; - let Some(connection_data) = user_connections.by_key.get_mut(username) else { - return; - }; - if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { - if session.transition_version != expected_version { - return; - } - if session.lifecycle != PlaybackLifecycle::GraceActive { - return; - } - Self::bump_session_transition_version(session); - session.lifecycle = PlaybackLifecycle::Active; - session.permission = UserConnectionPermission::Allowed; - } - } - - /// Expires a `GraceActive` session when the grace window fails. - /// - /// This corresponds to `GraceActive -> Expired` in the playback state machine. - /// Releases the provisional counted lease. - pub async fn expire_grace_active(&self, username: &str, token: &str, expected_version: u64) { - let (connection_changed, removed_count) = { - let mut user_connections = self.write_connections().await; - let Some(connection_data) = user_connections.by_key.get_mut(username) else { - return; - }; - let Some(session_index) = connection_data.sessions.iter().position(|session| session.token == token) else { - return; - }; - - if connection_data.sessions[session_index].transition_version != expected_version { - return; - } - if connection_data.sessions[session_index].lifecycle != PlaybackLifecycle::GraceActive { - return; - } - - // Release the provisional counted lease using index-based access - // to avoid nested mutable borrows with connection_data methods. - let mut connection_changed = false; - let mut counted_kind: Option = None; - if connection_data.sessions[session_index].lifecycle.is_counted() { - counted_kind = connection_data.sessions[session_index].connection_kind; - connection_changed = true; - } - if let Some(kind) = counted_kind { - connection_data.decrement_kind(kind); - } - - // Expire the session. Lifecycle change alone handles counted state (Expired is not counted). - connection_data.sessions[session_index].expire(); - connection_data.sessions[session_index].set_permission(UserConnectionPermission::Exhausted); - Self::bump_session_transition_version(&mut connection_data.sessions[session_index]); - - // Collect addresses for stream cleanup. - let mut addrs = Vec::new(); - let session_addr = connection_data.sessions[session_index].addr; - if !session_addr.ip().is_unspecified() { - addrs.push(session_addr); - } - for addr in &connection_data.sessions[session_index].active_addrs { - if *addr != session_addr && !addrs.contains(addr) { - addrs.push(*addr); - } - } - - // Remove all streams for these addresses (never preserve on expire). - let mut removed_count = 0; - for addr in &addrs { - if let Some(stream_idx) = - connection_data.streams.iter().position(|stream| stream.addr == *addr && !stream.preserved) - { - if let Some(kind) = connection_data.stream_kinds.remove(&connection_data.streams[stream_idx].uid) { - connection_data.decrement_kind(kind); - } - connection_data.stream_normal_priorities.remove(&connection_data.streams[stream_idx].uid); - connection_data.remove_stream_request_claims(connection_data.streams[stream_idx].uid); - connection_data.streams.swap_remove(stream_idx); - removed_count += 1; - } - } - - // Reset grace if no connections remain. - if connection_data.connections == 0 && connection_data.streams.is_empty() { - connection_data.granted_grace = false; - connection_data.grace_ts = 0; - } - - let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); - drop(user_connections); - self.log_divergence_snapshot(divergence_snapshot).await; - - (connection_changed, removed_count) - }; - - if connection_changed { - self.log_active_user().await; - } - debug!("GraceActive expired for session {token} in {username}, released {removed_count} streams"); - } - - /// Terminates the session and all associated streams for a playback. - /// - /// This is the explicit `Terminate` path from the playback state machine: - /// - Removes all streams associated with this session token (never preserves) - /// - Releases the counted lease if held - /// - Sets lifecycle to `Expired` - /// - Clears pending-provider state - /// - /// Unlike `release_unbound_session_reservation`, this terminates regardless of - /// whether streams are currently active, and always removes associated streams. - /// Returns `true` only when the session existed and was removed by this call. - pub async fn terminate_session(&self, username: &str, session_token: &str) -> bool { - let (connection_changed, removed_count, promotions) = { - let mut user_connections = self.write_connections().await; - let Some(connection_data) = user_connections.by_key.get_mut(username) else { - return false; - }; - - let Some(session_index) = - connection_data.sessions.iter().position(|session| session.token == session_token) - else { - return false; - }; - - let counted_kind = connection_data.sessions[session_index] - .lifecycle - .is_counted() - .then(|| connection_data.sessions[session_index].connection_kind.unwrap_or(ConnectionKind::Normal)); - let (removed_count, connection_changed) = - connection_data.remove_streams_for_session_and_release_counted(session_token, counted_kind); - - // Expire and remove the session immediately. Unlike `release_unbound_session_reservation` - // which keeps the expired session for TTL-based GC cleanup, terminate_session explicitly - // removes the session from the list so `get_and_update_user_session` returns None. - connection_data.sessions.swap_remove(session_index); - - // Reset grace if no connections remain. - if connection_data.connections == 0 && connection_data.streams.is_empty() { - connection_data.granted_grace = false; - connection_data.grace_ts = 0; - } - - let promotions = Self::collect_promotions_after_capacity_release(connection_data); - - let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); - drop(user_connections); - self.log_divergence_snapshot(divergence_snapshot).await; - - (connection_changed, removed_count, promotions) - }; - - if connection_changed { - self.log_active_user().await; - } - for action in promotions { - self.emit_promotion_update(username, action).await; - } - debug!("Terminated session {session_token} for user {username}, released {removed_count} streams"); - true - } - - /// Terminates all sessions associated with a given socket address for a user. - /// - /// This is used when a connection is explicitly kicked — the session should be - /// expired and removed immediately rather than waiting for TTL-based GC cleanup. - /// - /// Removes all sessions whose `addr` or `active_addrs` contains `kick_addr`. - pub async fn terminate_sessions_for_addr(&self, username: &str, kick_addr: &SocketAddr) { - let (connection_changed, removed_count, promotions) = { - let mut user_connections = self.write_connections().await; - let Some(connection_data) = user_connections.by_key.get_mut(username) else { - return; - }; - - // Collect tokens of sessions associated with the kicked addr. - let tokens_to_remove: Vec = connection_data - .sessions - .iter() - .filter(|session| session.addr == *kick_addr || session.active_addrs.contains(kick_addr)) - .map(|session| session.token.clone()) - .collect(); - - if tokens_to_remove.is_empty() { - return; - } - - let mut removed_count = 0; - let mut connection_changed = false; - - for token in &tokens_to_remove { - let Some(session_index) = connection_data.sessions.iter().position(|s| s.token == *token) else { - continue; - }; - - let counted_kind = connection_data.sessions[session_index] - .lifecycle - .is_counted() - .then(|| connection_data.sessions[session_index].connection_kind.unwrap_or(ConnectionKind::Normal)); - - let (_, session_connection_changed) = - connection_data.remove_streams_for_session_and_release_counted(token, counted_kind); - connection_changed |= session_connection_changed; - - // Expire and remove the session. - connection_data.sessions.swap_remove(session_index); - removed_count += 1; - } - - // Reset grace if no connections remain. - if connection_data.connections == 0 && connection_data.streams.is_empty() { - connection_data.granted_grace = false; - connection_data.grace_ts = 0; - } - - let promotions = Self::collect_promotions_after_capacity_release(connection_data); - - let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); - user_connections.mark_sessions_ended(tokens_to_remove); - drop(user_connections); - self.log_divergence_snapshot(divergence_snapshot).await; - - (connection_changed, removed_count, promotions) - }; - - if connection_changed { - self.log_active_user().await; - } - for action in promotions { - self.emit_promotion_update(username, action).await; - } - debug!("Terminated {removed_count} sessions for user {username} at addr {kick_addr}"); - } - - pub async fn expire_pending_provider( - &self, - username: &str, - token: &str, - expected_version: u64, - wake_source: PendingProviderWakeSource, - ) { - let mut user_connections = self.write_connections().await; - let Some(connection_data) = user_connections.by_key.get_mut(username) else { - return; - }; - let Some(session_index) = connection_data.sessions.iter().position(|session| session.token == token) else { - return; - }; - let pending_version = match &connection_data.sessions[session_index].lifecycle { - PlaybackLifecycle::PendingProvider { data } => data.version, - _ => return, - }; - if pending_version != expected_version { - return; - } - // Capture counted status BEFORE lifecycle changes. - // PendingProvider is not counted (is_counted() = false), so checking here - // captures whether there is a previously-counted lease to release. - let kind_to_release = if connection_data.sessions[session_index].lifecycle.is_counted() { - Some(connection_data.sessions[session_index].connection_kind.unwrap_or(ConnectionKind::Normal)) - } else { - None - }; - let session = &mut connection_data.sessions[session_index]; - Self::clear_session_pending_with_permission(session, UserConnectionPermission::Exhausted, wake_source); - session.expire(); - if let Some(kind) = kind_to_release { - connection_data.decrement_kind(kind); - } - } - - pub async fn adaptive_session_stream_cleanup_addrs( - &self, - username: &str, - session_token: &str, - current_addr: &SocketAddr, - ) -> Vec { - let connections = self.connections.read().await; - let Some(connection_data) = connections.by_key.get(username) else { - return Vec::new(); - }; - - let mut addrs = Vec::new(); - for stream in - connection_data.streams.iter().filter(|stream| stream.session_token.as_deref() == Some(session_token)) - { - if stream.addr != *current_addr && !addrs.contains(&stream.addr) { - addrs.push(stream.addr); - } - } - let current_addr_string = current_addr.to_string(); - let current_ip = strip_port(¤t_addr_string).to_string(); - if let Some(session) = connection_data.sessions.iter().find(|session| session.token == session_token) { - for addr in &session.active_addrs { - let addr_string = addr.to_string(); - let addr_ip = strip_port(&addr_string); - if *addr != *current_addr && addr_ip == current_ip && !addrs.contains(addr) { - addrs.push(*addr); - } - } - } - addrs - } - - pub fn active_socket_ttl_secs(&self) -> u64 { - let configured_ttl = self.adaptive_session_ttl_secs.load(Ordering::Relaxed); - if configured_ttl == 0 { - DEFAULT_ACTIVE_SOCKET_TTL_SECS - } else { - configured_ttl - } - } - - pub async fn socket_expiry_deadline(&self, addr: &SocketAddr) -> Option { - let connections = self.connections.read().await; - let registration = connections.key_by_addr.get(addr)?; - if registration.is_empty() { - return None; - } - - let ttl_secs = self.effective_socket_ttl_secs(&connections, addr); - Some(registration.ts.saturating_add(ttl_secs)) - } - - /// Direct VOD/series bodies may stop reading while the player drains its buffer. - /// Their socket expiry must not fire before the direct-body idle timeout, or a - /// buffering player is disconnected mid-playback even though the stream is alive. - fn effective_socket_ttl_secs(&self, connections: &UserConnections, addr: &SocketAddr) -> u64 { - let base_ttl = self.active_socket_ttl_secs(); - if Self::addr_has_direct_body_stream(connections, addr) { - base_ttl.max(DIRECT_BODY_IDLE_TIMEOUT_SECS) - } else { - base_ttl - } - } - - fn addr_has_direct_body_stream(connections: &UserConnections, addr: &SocketAddr) -> bool { - let Some(registration) = connections.key_by_addr.get(addr) else { - return false; - }; - registration.usernames.iter().any(|username| { - connections.by_key.get(username).is_some_and(|connection_data| { - connection_data - .streams - .iter() - .any(|stream| stream.addr == *addr && uses_direct_body_idle_timeout(&stream.channel)) - }) - }) - } - - pub async fn touch_socket_activity(&self, addr: &SocketAddr) { - let now = current_time_secs(); - let mut user_connections = self.write_connections().await; - // Transport activity keeps the transport alive, not every playback that - // has previously used it. Authenticated HTTP activity updates its own user. - if let Some(registration) = user_connections.key_by_addr.get_mut(addr) { - registration.ts = now; - } - } - - pub async fn touch_http_activity(&self, username: &str, token: &str, addr: &SocketAddr) { - let now = current_time_secs(); - let mut user_connections = self.write_connections().await; - - let registration = user_connections.key_by_addr.entry(*addr).or_insert_with(SocketRegistration::anonymous); - registration.add_user(username, now); - - let Some(connection_data) = user_connections.by_key.get_mut(username) else { - return; - }; - - connection_data.ts = now; - - for session in &mut connection_data.sessions { - if session.token == token { - // Lightweight HTTP activity (for example HLS manifest reloads) refreshes - // continuity metadata only. It must not become an active stream socket: - // otherwise a manifest or probe request can steal the visible stream addr, - // and the real segment socket later migrates to that stale addr instead of - // being released/preserved. - session.ts = now; - break; - } - } - } - - /// Marks a session as ended on purpose (eviction, kick, explicit terminate), so it is not - /// recreated from a sealed playlist token. - pub async fn mark_session_ended(&self, token: &str) { - self.write_connections().await.mark_sessions_ended([token.to_string()]); - } - - /// True while a session token is blocked from recreation after an eviction, kick or terminate. - pub async fn is_session_ended(&self, token: &str) -> bool { - self.connections.read().await.ended_sessions.get(token).is_some_and(|expires_at| *expires_at > Instant::now()) - } - - pub async fn get_and_update_user_session(&self, username: &str, token: &str) -> Option { - self.update_user_session(username, token).await - } - - /// Checks an in-flight resource against the same session and account binding without touching admission. - pub async fn playback_session_is_current(&self, username: &str, token: &str, identity: SessionIdentity) -> bool { - self.connections.read().await.current_session(username, token, identity).is_some() - } - - /// Resolves once the session ended or switched accounts. Only writes that end this - /// session wake the waiter, after the write lock is released. - pub async fn wait_for_playback_session_end(&self, username: &str, token: &str, identity: SessionIdentity) { - loop { - let notify = { - let users = self.connections.read().await; - let Some((_, session)) = users.current_session(username, token, identity) else { - return; - }; - Arc::clone(&session.change_signal.notify) - }; - let changed = notify.notified(); - tokio::pin!(changed); - changed.as_mut().enable(); - // A change between the lookup and `enable` was not observed: re-check before waiting. - if !self.playback_session_is_current(username, token, identity).await { - return; - } - changed.await; - } - } - - /// Session object and account binding currently stored for `token`, in any lifecycle state. - pub async fn session_identity(&self, username: &str, token: &str) -> Option { - let users = self.connections.read().await; - users.by_key.get(username)?.sessions.iter().find(|session| session.token == token).map(UserSession::identity) - } - - /// Provider session headers valid for `target_url` now, for the session object and account - /// binding of `identity` in any lifecycle state: a manifest refresh may revive an expired - /// reservation. - pub async fn provider_headers_for_session_identity( - &self, - username: &str, - token: &str, - identity: SessionIdentity, - target_url: &str, - ) -> SessionProviderHeaders { - let users = self.connections.read().await; - users - .by_key - .get(username) - .and_then(|data| { - data.sessions.iter().find(|session| session.token == token && session.identity() == identity) - }) - .map_or(SessionProviderHeaders::NoSession, |session| { - SessionProviderHeaders::for_target(session, target_url) - }) - } - - /// Provider session headers currently valid for `target_url`, read at send time so cookies - /// rotated or expired while the request waited are honored. - pub async fn current_session_provider_headers( - &self, - username: &str, - token: &str, - identity: SessionIdentity, - target_url: &str, - ) -> SessionProviderHeaders { - let users = self.connections.read().await; - users.current_session(username, token, identity).map_or(SessionProviderHeaders::NoSession, |(_, session)| { - SessionProviderHeaders::for_target(session, target_url) - }) - } - - pub async fn media_started_flag(&self, username: &str, token: &str) -> Option> { - let users = self.connections.read().await; - users - .by_key - .get(username)? - .sessions - .iter() - .find(|session| session.token == token) - .map(|session| Arc::clone(&session.media_started)) - } - - /// Session for target-scoped `virtual_id` and request token (used to recover leaked relative DVR segment paths). - pub async fn find_latest_session_for_target_stream( - &self, - username: &str, - target_id: u16, - input_name: &str, - virtual_id: u32, - session_token: &str, - ) -> Option { - let user_connections = self.connections.read().await; - let connection_data = user_connections.by_key.get(username)?; - connection_data - .streams - .iter() - .any(|stream| { - stream.channel.target_id == target_id - && stream.channel.input_name.as_ref() == input_name - && stream.channel.virtual_id == virtual_id - && stream.session_token.as_deref() == Some(session_token) - }) - .then_some(())?; - - connection_data - .sessions - .iter() - .find(|session| session.token == session_token && session.virtual_id == virtual_id) - .cloned() - } - - pub async fn update_session_provider_headers( - &self, - username: &str, - token: &str, - provider_session_headers: &HashMap, - ) -> bool { - let mut user_connections = self.write_connections().await; - let Some(connection_data) = user_connections.by_key.get_mut(username) else { - return false; - }; - let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) else { - return false; - }; - session.provider_session_headers.clone_from(provider_session_headers); - session.provider_session_headers_host = None; - session.clear_provider_session_cookies(); - session.ts = current_time_secs(); - true - } - - pub async fn update_session_provider_response_headers_from( - &self, - username: &str, - token: &str, - response: &crate::ProviderSessionHeaders, - source_url: &str, - ) -> bool { - self.store_session_provider_response_headers(username, token, None, response, source_url).await - } - - /// Stores response cookies only while the session still has the expected account binding. - pub async fn update_current_session_provider_response_headers_from( - &self, - username: &str, - token: &str, - identity: SessionIdentity, - response: &crate::ProviderSessionHeaders, - source_url: &str, - ) -> bool { - self.store_session_provider_response_headers(username, token, Some(identity), response, source_url).await - } - - async fn store_session_provider_response_headers( - &self, - username: &str, - token: &str, - expected: Option, - response: &crate::ProviderSessionHeaders, - source_url: &str, - ) -> bool { - if response.is_empty() { - return false; - } - let Ok(source) = url::Url::parse(source_url) else { - return false; - }; - let mut users = self.write_connections().await; - if expected.is_some() && users.is_token_ended(token) { - return false; - } - let Some(session) = users - .by_key - .get_mut(username) - .and_then(|data| data.sessions.iter_mut().find(|session| session.token == token)) - else { - return false; - }; - if expected.is_some_and(|identity| session.identity() != identity || !session.is_live()) { - return false; - } - Arc::make_mut(&mut session.provider_session_cookies).update(&source, response); - // The origin-scoped cookie store is the only source once a provider response was stored. - session.provider_session_headers.clear(); - session.provider_session_headers_host = None; - session.ts = current_time_secs(); - true - } - - /// Returns the stable User-Agent suffix for a playback session, assigning one on first use. - pub async fn get_or_assign_user_agent_stream_index(&self, username: &str, token: &str) -> Option { - let mut user_connections = self.write_connections().await; - let session = - user_connections.by_key.get_mut(username)?.sessions.iter_mut().find(|session| session.token == token)?; - Some(*session.user_agent_stream_index.get_or_insert_with(|| self.next_user_agent_stream_index())) - } - - /// Allocates the next process-local, globally unique User-Agent stream index. - pub fn next_user_agent_stream_index(&self) -> u64 { - let mut current = self.next_user_agent_stream_index.load(Ordering::Relaxed); - loop { - let next = if current == 0 || current == u64::MAX { 1 } else { current + 1 }; - match self.next_user_agent_stream_index.compare_exchange_weak( - current, - next, - Ordering::Relaxed, - Ordering::Relaxed, - ) { - Ok(previous) => return if previous == 0 { 1 } else { previous }, - Err(observed) => current = observed, - } - } - } - - /// Persists a previously allocated index without replacing an existing session identity. - pub async fn set_user_agent_stream_index_if_absent(&self, username: &str, token: &str, index: u64) -> bool { - let mut user_connections = self.write_connections().await; - let Some(session) = user_connections - .by_key - .get_mut(username) - .and_then(|connection| connection.sessions.iter_mut().find(|session| session.token == token)) - else { - return false; - }; - session.user_agent_stream_index.get_or_insert(index); - true - } - - pub async fn pending_provider_version(&self, username: &str, token: &str) -> Option { - let user_connections = self.connections.read().await; - let connection_data = user_connections.by_key.get(username)?; - let session = connection_data.sessions.iter().find(|session| session.token == token)?; - match &session.lifecycle { - PlaybackLifecycle::PendingProvider { data } => Some(data.version), - _ => None, - } - } - - async fn update_user_session(&self, username: &str, token: &str) -> Option { - let mut user_connections = self.write_connections().await; - - let connection_data = user_connections.by_key.get_mut(username)?; - let now = current_time_secs(); - - connection_data.ts = now; - - let session_index = connection_data.sessions.iter().position(|s| s.token == token)?; - - connection_data.sessions[session_index].ts = now; - - if connection_data.max_connections > 0 - && connection_data.sessions[session_index].permission == UserConnectionPermission::GracePeriod - && !matches!(connection_data.sessions[session_index].lifecycle, PlaybackLifecycle::PendingProvider { .. }) - { - let admission = self.check_connection_admission(username, connection_data); - connection_data.sessions[session_index].set_permission(admission.permission()); - if admission.kind().is_some() { - connection_data.sessions[session_index].connection_kind = admission.kind(); - } - } - - Some(connection_data.sessions[session_index].clone()) - } - - pub async fn active_streams(&self) -> Vec { - self.gc(); - let user_connections = self.connections.read().await; - let mut streams = Vec::new(); - for connection_data in user_connections.by_key.values() { - for stream in &connection_data.streams { - // Keep active_streams free of preserved rows — shared-HLS join detection and - // connection accounting must not see stale archive segment leases (v3.3.81). - if !stream.preserved { - streams.push(stream.clone()); - } - } - } - streams - } - - /// Streams for the `WebUI` / `StatusCheck` snapshot. - /// - /// Includes preserved Catchup/HLS/DASH session rows so the panel keeps showing archive - /// playback between short segment sockets. Do not use this for shared-HLS accounting. - pub async fn panel_streams(&self) -> Vec { - self.gc(); - let user_connections = self.connections.read().await; - let mut streams = Vec::new(); - for connection_data in user_connections.by_key.values() { - for stream in &connection_data.streams { - if !stream.preserved || Self::should_preserve_session_stream(stream) { - streams.push(stream.clone()); - } - } - } - streams - } - - fn log_connection_added( - username: &str, - addr: &SocketAddr, - connection_data: &UserConnectionData, - tracked_socket_count: usize, - ) { - if log::log_enabled!(log::Level::Debug) { - let active_for_user = connection_data.connections; - if connection_data.max_connections > 0 && active_for_user > connection_data.max_connections { - let recent_sockets = connection_data - .streams - .iter() - .rev() - .take(3) - .map(|stream| stream.addr.to_string()) - .collect::>() - .join(", "); - let recent_sockets = if recent_sockets.is_empty() { String::from("n/a") } else { recent_sockets }; - let unique_clients = - connection_data.streams.iter().map(|stream| &stream.client_ip).collect::>().len(); - debug!( - "User {username} exceeded configured max connections ({}/{}). Unique clients: {}, recent sockets [{}]", - active_for_user, - connection_data.max_connections, - unique_clients, - recent_sockets - ); - } else { - debug_if_enabled!( - "Added new connection for {username} at {} (active user connections={active_for_user}, tracked sockets={tracked_socket_count})", - sanitize_sensitive_info(&addr.to_string()) - ); - } - } - } - - pub async fn is_user_blocked_for_stream(&self, username: &str, virtual_id: VirtualId) -> bool { - let connections = self.connections.read().await; - let now = current_time_secs(); - matches!(connections.kicked.get(username), Some((expires_at, vid)) if *vid == virtual_id && *expires_at > now) - } - - pub async fn recently_evicted_session_protected_addr(&self, session_token: &str) -> Option { - let connections = self.connections.read().await; - let protection = connections.recently_evicted_sessions.get(session_token)?; - if protection.expires_at > Instant::now() { - return Some(protection.protected_addr); - } - - let username = connections.by_key.iter().find_map(|(username, connection_data)| { - connection_data.sessions.iter().any(|session| session.token == session_token).then_some(username.as_str()) - })?; - connections - .key_by_addr - .get(&protection.protected_addr) - .filter(|registration| registration.usernames.contains(username)) - .map(|_| protection.protected_addr) - } - - pub async fn recent_socket_reentry_protected_addr( - &self, - username: &str, - client_ip: &str, - virtual_id: VirtualId, - ) -> Option { - let connections = self.connections.read().await; - let key = create_socket_reentry_guard_key(username, client_ip, virtual_id); - let protection = connections.recent_socket_reentry_guards.get(&key)?; - if protection.expires_at > Instant::now() { - Some(protection.protected_addr) - } else { - None - } - } - - pub async fn block_user_for_stream(&self, addr: &SocketAddr, virtual_id: VirtualId, blocked_secs: u64) { - let block_for_secs = blocked_secs.clamp(0, 86_400); // max 1 day; - if block_for_secs > 0 { - let mut connections = self.write_connections().await; - let now = current_time_secs(); - connections.kicked.retain(|_, (expires_at, _)| *expires_at > now); - let candidate_usernames: Vec = connections - .key_by_addr - .get(addr) - .map(|reg| reg.usernames.iter().cloned().collect()) - .unwrap_or_default(); - let target_vid = virtual_id.get(); - let expires_at = now + block_for_secs; - for username in candidate_usernames { - // Only block users who are actually watching the target channel - // at this socket address. Behind reverse proxies / NAT, multiple - // distinct users share the same SocketAddr; blocking all of them - // would be a multi-tenant isolation violation. - let is_watching_target = connections.by_key.get(&username).is_some_and(|data| { - data.streams.iter().any(|s| s.addr == *addr && s.channel.virtual_id == target_vid) - }); - if is_watching_target { - connections.kicked.insert(username, (expires_at, virtual_id)); - } - } - } - } - - pub async fn block_user_for_stream_uid(&self, uid: u32, virtual_id: VirtualId, blocked_secs: u64) { - if blocked_secs == 0 { - return; - } - let mut connections = self.write_connections().await; - let username = connections - .by_key - .iter() - .find_map(|(username, data)| data.streams.iter().any(|stream| stream.uid == uid).then(|| username.clone())); - if let Some(username) = username { - connections.kicked.insert(username, (current_time_secs() + blocked_secs.min(86_400), virtual_id)); - } - } - - pub async fn mark_recent_eviction_guard_for_addr( - &self, - addr: &SocketAddr, - protected_addr: SocketAddr, - ttl: Duration, - ) { - let mut connections = self.write_connections().await; - let now = Instant::now(); - connections.recently_evicted_sessions.retain(|_, protection| protection.expires_at > now); - connections.recent_socket_reentry_guards.retain(|_, protection| protection.expires_at > now); - - let usernames = connections.key_by_addr.get(addr).map(SocketRegistration::all_usernames).unwrap_or_default(); - if usernames.is_empty() { - return; - } - - let protection = RecentWinnerProtection { protected_addr, expires_at: now + ttl }; - let mut session_tokens = Vec::new(); - let mut socket_guard_keys = Vec::new(); - - for username in &usernames { - if let Some(connection_data) = connections.by_key.get(username) { - for stream in connection_data.streams.iter().filter(|stream| stream.addr == *addr) { - if uses_session_reentry_guard(stream) && stream.session_token.is_some() { - let Some(session_token) = stream.session_token.clone() else { - continue; - }; - session_tokens.push(session_token); - } else { - socket_guard_keys.push(create_socket_reentry_guard_key( - username, - &stream.client_ip, - shared::model::VirtualId::new(stream.channel.virtual_id), - )); - } - } - } - } - - // Evicted sessions stay ended even when the reentry guard itself is disabled. - connections.mark_sessions_ended(session_tokens.iter().cloned()); - if ttl.is_zero() { - return; - } - for session_token in session_tokens { - connections.recently_evicted_sessions.insert(session_token, protection); - } - for key in socket_guard_keys { - connections.recent_socket_reentry_guards.insert(key, protection); - } - } - - pub async fn get_username_for_addr(&self, addr: &SocketAddr) -> Option { - self.connections - .read() - .await - .key_by_addr - .get(addr) - .and_then(SocketRegistration::primary_username) - .map(str::to_string) - } - - pub async fn get_usernames_for_addr(&self, addr: &SocketAddr) -> Vec { - self.connections.read().await.key_by_addr.get(addr).map(SocketRegistration::all_usernames).unwrap_or_default() - } - - fn should_preserve_session_stream(stream: &StreamInfo) -> bool { - stream.session_token.is_some() && is_stable_session_stream(stream) - } - - fn is_preserved_stream_expired(&self, stream: &StreamInfo, sessions: &[UserSession], now: u64) -> bool { - if !stream.preserved || !Self::should_preserve_session_stream(stream) { - return false; - } - - let ttl_secs = self.adaptive_session_ttl_secs.load(Ordering::Relaxed); - let Some(session_token) = stream.session_token.as_deref() else { - return true; - }; - - let session_ts = - sessions.iter().find(|session| session.token == session_token).map_or(stream.ts, |session| session.ts); - - now.saturating_sub(session_ts) >= ttl_secs - } - - fn collect_divergence_snapshot(connection_data: &UserConnectionData, username: &str) -> Option { - log_enabled!(log::Level::Debug).then(|| Self::build_divergence_snapshot(connection_data, username)) - } - - fn build_divergence_snapshot(connection_data: &UserConnectionData, username: &str) -> DivergenceSnapshot { - let connections = connection_data.connections; - let counted_sessions = connection_data.sessions.iter().filter(|s| s.lifecycle.is_counted()).count(); - let streams_count = connection_data.streams.len(); - let mut kinds = Vec::new(); - - for session in &connection_data.sessions { - if !session.lifecycle.is_counted() { - continue; - } - if matches!(session.lifecycle, PlaybackLifecycle::PendingProvider { ref data } if data.reason_code == PendingProviderReason::GraceHold) - { - continue; - } - let has_active_stream = connection_data - .streams - .iter() - .any(|s| s.session_token.as_deref() == Some(&session.token) && !s.preserved); - if !has_active_stream { - kinds.push(DivergenceKind::CountedSessionWithoutStream); - } - } - - for stream in &connection_data.streams { - if stream.preserved { - continue; - } - let Some(token) = stream.session_token.as_deref() else { - continue; - }; - let has_counted_session = - connection_data.sessions.iter().any(|s| s.token == token && s.lifecycle.is_counted()); - if !has_counted_session { - kinds.push(DivergenceKind::StreamWithoutCountedSession); - } - } - - #[allow(clippy::cast_possible_truncation)] - let counted_sessions_u32 = counted_sessions as u32; - if connections != counted_sessions_u32 { - kinds.push(DivergenceKind::ConnectionCountMismatch { legacy: connections, counted: counted_sessions_u32 }); - } - - DivergenceSnapshot { username: username.to_string(), connections, counted_sessions, streams_count, kinds } - } - - async fn log_divergence_snapshot(&self, snapshot: Option) { - let Some(snapshot) = snapshot else { - return; - }; - let cooldown = Duration::from_secs(self.divergence_cooldown_secs); - for kind in &snapshot.kinds { - let key = divergence_key(&snapshot.username, kind); - let should_log = { - let mut cache = self.divergence_cache.lock().await; - if let Some(entry) = cache.get_mut(&key) { - if entry.last_logged.elapsed() >= cooldown { - entry.last_logged = Instant::now(); - entry.count_since_last_log = 0; - true - } else { - entry.count_since_last_log = entry.count_since_last_log.saturating_add(1); - false - } - } else { - cache.push(key, DivergenceEntry { last_logged: Instant::now(), count_since_last_log: 0 }); - true - } - }; - - if should_log { - debug!( - "ADMISSION DIVERGENCE user={} kind={kind:?} connections={} counted_sessions={} streams={}", - snapshot.username, snapshot.connections, snapshot.counted_sessions, snapshot.streams_count, - ); - } - } - } - - async fn check_and_log_divergence_for_user(&self, username: &str) { - let snapshot = { - let connections = self.connections.read().await; - let Some(data) = connections.by_key.get(username) else { - return; - }; - Self::collect_divergence_snapshot(data, username) - }; - self.log_divergence_snapshot(snapshot).await; - } - - async fn run_adaptive_expiry_worker(self: Arc) { - loop { - let next_expiry = { - let queue = self.adaptive_expiry_queue.lock().await; - queue.peek().map(|entry| entry.0.expires_at) - }; - - match next_expiry { - None => { - tokio::select! { - () = self.adaptive_expiry_notify.notified() => {} - () = self.adaptive_expiry_cancel.cancelled() => break, - } - } - Some(expires_at) => { - let now = current_time_secs(); - if expires_at <= now { - self.process_due_adaptive_expiry_entries(now).await; - continue; - } - - tokio::select! { - () = tokio::time::sleep(Duration::from_secs(expires_at.saturating_sub(now))) => {} - () = self.adaptive_expiry_notify.notified() => {} - () = self.adaptive_expiry_cancel.cancelled() => break, - } - } - } - } - } - - #[allow(clippy::too_many_lines)] - async fn process_due_adaptive_expiry_entries(&self, now: u64) { - let mut due_entries = Vec::new(); - { - let mut queue = self.adaptive_expiry_queue.lock().await; - while let Some(entry) = queue.peek() { - if entry.0.expires_at > now { - break; - } - if let Some(Reverse(entry)) = queue.pop() { - due_entries.push(entry); - } - } - } - - if due_entries.is_empty() { - return; - } - - let usernames_to_check: std::collections::HashSet<_> = due_entries.iter().map(|e| &e.username).collect(); - - let mut removed_addrs: Vec = Vec::new(); - let mut cleanup_events: Vec<(std::net::SocketAddr, Box)> = Vec::new(); - let mut replacement_entries: Vec = Vec::new(); - let mut promotions: Vec<(String, PromotionAction)> = Vec::new(); - { - let mut expiry_index = self.adaptive_expiry_index.lock().await; - let mut user_connections = self.write_connections().await; - for entry in &due_entries { - let key = AdaptiveExpiryKey { - username: entry.username.clone(), - session_token: entry.session_token.clone(), - uid: entry.uid, - }; - let Some(current_expires_at) = expiry_index.get(&key).copied() else { - continue; - }; - if current_expires_at != entry.expires_at { - continue; - } - - let mut remove_user = false; - if let Some(connection_data) = user_connections.by_key.get_mut(&entry.username) { - let stream_idx_opt = connection_data.streams.iter().position(|stream| { - stream.uid == entry.uid - && stream.preserved - && stream.session_token.as_deref() == Some(entry.session_token.as_str()) - }); - - if let Some(stream_idx) = stream_idx_opt { - let should_remove = self.is_preserved_stream_expired( - &connection_data.streams[stream_idx], - &connection_data.sessions, - now, - ); - - if should_remove { - let addr = connection_data.streams[stream_idx].addr; - let session_token = connection_data.streams[stream_idx].session_token.clone(); - if self.cleanup_tx.get().is_some() { - cleanup_events.push((addr, Box::new(connection_data.streams[stream_idx].clone()))); - } else { - removed_addrs.push(addr); - } - let removed_stream = connection_data.streams.swap_remove(stream_idx); - connection_data.remove_stream_request_claims(removed_stream.uid); - if let Some(kind) = connection_data.stream_kinds.remove(&removed_stream.uid) { - connection_data.decrement_kind(kind); - } - connection_data.stream_normal_priorities.remove(&removed_stream.uid); - if let Some(action) = connection_data.try_promote_soft_stream() { - let promoted_stream = - connection_data.streams.iter().find(|stream| stream.uid == action.uid).cloned(); - if let Some(stream) = promoted_stream.as_ref() { - Self::promote_session_for_stream(connection_data, stream); - } - promotions.push((entry.username.clone(), action)); - } - if let Some(session_token) = session_token.as_deref() { - Self::clear_session_counted_without_stream(connection_data, session_token); - } - while connection_data.try_promote_soft_session_reservation() {} - expiry_index.remove(&key); - } else if let Some(replacement_entry) = self.build_preserved_stream_expiry( - &entry.username, - &connection_data.streams[stream_idx], - &connection_data.sessions, - ) { - if replacement_entry.expires_at != current_expires_at { - replacement_entries.push(replacement_entry); - } - } - } else { - expiry_index.remove(&key); - } - - remove_user = connection_data.connections == 0 - && connection_data.streams.is_empty() - && connection_data.sessions.is_empty(); - } else { - expiry_index.remove(&key); - } - - if remove_user { - user_connections.by_key.remove(&entry.username); - } - } - } // locks released here - - // divergence check after adaptive expiry processing - for username in usernames_to_check { - let snapshot = { - let connections = self.connections.read().await; - connections.by_key.get(username).and_then(|data| Self::collect_divergence_snapshot(data, username)) - }; - self.log_divergence_snapshot(snapshot).await; - } - - if let Some(tx) = self.cleanup_tx.get() { - for (addr, stream_info) in cleanup_events { - if tx.try_send(CleanupEvent::AdaptiveSessionExpired { stream_info }).is_err() { - self.dropped_cleanup_events.fetch_add(1, Ordering::Relaxed); - debug!("Cleanup channel unavailable, dropping adaptive session expiry"); - removed_addrs.push(addr); - } - } - } - - for entry in replacement_entries { - self.enqueue_adaptive_expiry(entry).await; - } - - for (username, action) in promotions { - self.emit_promotion_update(&username, action).await; - } - - let had_removals = !removed_addrs.is_empty(); - for addr in removed_addrs { - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Disconnected(addr))); - } - if had_removals { - self.log_active_user().await; - } - } - - fn gc(&self) { - if let Some(gc_ts) = &self.gc_ts { - let ts = gc_ts.load(Ordering::Acquire); - let now = current_time_secs(); - - if now.saturating_sub(ts) > USER_GC_TTL - && gc_ts.compare_exchange(ts, now, Ordering::AcqRel, Ordering::Relaxed).is_ok() - { - if let Ok(guard) = self.connections.try_write() { - let mut user_connections = UserConnectionsWriteGuard::new(guard, &self.deferred_wakes); - user_connections.kicked.retain(|_, (expires_at, _)| *expires_at > now); - let now_instant = Instant::now(); - user_connections - .recently_evicted_sessions - .retain(|_, protection| protection.expires_at > now_instant); - user_connections.ended_sessions.retain(|_, expires_at| *expires_at > now_instant); - user_connections - .recent_socket_reentry_guards - .retain(|_, protection| protection.expires_at > now_instant); - for connection_data in user_connections.by_key.values_mut() { - Self::release_expired_session_reservations(connection_data, now); - connection_data.sessions.retain(|s| now.saturating_sub(s.ts) < USER_CON_TTL); - } - user_connections.by_key.retain(|_k, v| { - v.connections > 0 || !v.streams.is_empty() || now.saturating_sub(v.ts) < USER_CON_TTL - }); - let UserConnections { ref by_key, ref mut key_by_addr, .. } = *user_connections; - key_by_addr.retain(|addr, registration| { - registration.usernames.retain(|u| { - by_key - .get(u) - .is_some_and(|d| d.has_session_addr(addr) || d.streams.iter().any(|s| s.addr == *addr)) - }); - !(registration.is_empty() && now.saturating_sub(registration.ts) >= ANON_SOCKET_TTL) - }); - } else { - // Lock contention: release the GC claim so a subsequent caller can retry immediately. - let _ = gc_ts.compare_exchange(now, ts, Ordering::AcqRel, Ordering::Relaxed); - } - } - } - } -} - #[cfg(test)] mod tests; + +mod activity; +mod admission; +mod connections; +mod eviction; +mod expiry; +mod observability; +mod provider_headers; +mod requests; +mod session_lifecycle; + +#[cfg(test)] +use self::observability::divergence_key; +#[cfg(test)] +use self::observability::DivergenceKind; +pub use self::session_lifecycle::next_session_incarnation; +use self::{ + connections::{ + create_socket_reentry_guard_key, decide_connection_kind, url_host_key, PromotionAction, RecentWinnerProtection, + SocketRegistration, UserConnectionCounts, UserConnectionData, UserConnections, UserConnectionsWriteGuard, + }, + expiry::{AdaptiveExpiryEntry, AdaptiveExpiryKey}, + observability::DivergenceEntry, + requests::StreamRequestClaim, + session_lifecycle::{ + get_adaptive_session_ttl_secs, release_session_addr, remember_session_addr, stream_history_session_id, + uses_session_reentry_guard, UserSessionParams, + }, +}; diff --git a/backend/session/src/active_user_manager/observability.rs b/backend/session/src/active_user_manager/observability.rs new file mode 100644 index 000000000..e8f318537 --- /dev/null +++ b/backend/session/src/active_user_manager/observability.rs @@ -0,0 +1,325 @@ +use super::{ActiveUserManager, PendingProviderReason, PlaybackLifecycle, PromotionAction, UserConnectionData}; +use crate::{active_provider_manager::ConnectionKind, EventManager}; +use log::{debug, info, log_enabled}; +use shared::{ + model::{ActiveUserConnectionChange, EventMessage, StreamInfo, StreamTechnicalInfo}, + utils::sanitize_sensitive_info, +}; +use std::{ + collections::HashSet, + net::SocketAddr, + sync::{atomic::Ordering, Arc}, + time::{Duration, Instant}, +}; +use tuliprox_core::utils::debug_if_enabled; + +pub(super) struct DivergenceEntry { + pub(super) last_logged: Instant, + pub(super) count_since_last_log: u32, +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub(super) enum DivergenceKind { + CountedSessionWithoutStream, + StreamWithoutCountedSession, + ConnectionCountMismatch { legacy: u32, counted: u32 }, +} + +pub(super) fn divergence_key(username: &str, kind: &DivergenceKind) -> String { + match kind { + DivergenceKind::CountedSessionWithoutStream => format!("{username}:CountedSessionWithoutStream"), + DivergenceKind::StreamWithoutCountedSession => format!("{username}:StreamWithoutCountedSession"), + DivergenceKind::ConnectionCountMismatch { legacy, counted } => { + format!("{username}:ConnectionCountMismatch:{legacy}+{counted}") + } + } +} + +pub(super) struct DivergenceSnapshot { + pub(super) username: String, + pub(super) connections: u32, + pub(super) counted_sessions: usize, + pub(super) streams_count: usize, + pub(super) kinds: Vec, +} + +impl ActiveUserManager { + pub(super) fn custom_stream_technical_info() -> StreamTechnicalInfo { + StreamTechnicalInfo { + container: String::from("mpegts"), + resolution: String::new(), + fps: String::from("30"), + video_codec: String::from("H.264"), + audio_codec: String::from("AAC"), + audio_channels: String::from("Stereo"), + } + } + + /// The bus this manager publishes on. + /// + /// Exposed so the admission path can report a refusal without + /// `AdmissionCtx` growing a second handle to the same manager. + #[must_use] + pub fn events(&self) -> &Arc { &self.event_manager } + + /// Cumulative count of admission requests quietly suppressed by the recent-eviction + /// reentry guard. A diagnostic counter, not a failure metric. + pub fn reentry_suppressed_total(&self) -> u64 { self.reentry_suppressed_total.load(Ordering::Relaxed) } + + /// Records one suppressed reentry retry. + pub fn record_reentry_suppressed(&self) { self.reentry_suppressed_total.fetch_add(1, Ordering::Relaxed); } + + /// Collect a snapshot of all currently active streams for shutdown history recording. + pub async fn get_all_active_streams(&self) -> Vec { + let connections = self.connections.read().await; + connections + .by_key + .values() + .flat_map(|data| data.streams.iter().filter(|stream| !stream.preserved).cloned()) + .collect() + } + + pub(super) async fn log_active_user(&self) { + let is_log_user_enabled = self.is_log_user_enabled(); + let (user_count, user_connection_count) = { self.active_users_and_connections().await }; + self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Connections( + user_count, + user_connection_count, + ))); + if !is_log_user_enabled { + return; + } + let last_user_count = self.last_logged_user_count.load(Ordering::Relaxed); + let last_connection_count = self.last_logged_user_connection_count.load(Ordering::Relaxed); + if last_user_count != user_count || last_connection_count != user_connection_count { + self.last_logged_user_count.store(user_count, Ordering::Relaxed); + self.last_logged_user_connection_count.store(user_connection_count, Ordering::Relaxed); + info!("Active Users: {user_count}, Active User Connections: {user_connection_count}"); + } + } + + pub(super) async fn emit_promotion_update(&self, username: &str, action: PromotionAction) { + let maybe_stream = { + let user_connections = self.connections.read().await; + user_connections.by_key.get(username).and_then(|connection_data| { + connection_data.streams.iter().find(|stream| stream.uid == action.uid).cloned() + }) + }; + if let Some(stream_info) = maybe_stream { + if let Some(provider_manager) = self.provider_manager.get() { + if stream_info.channel.shared { + provider_manager.reclassify_shared_connection( + tuliprox_core::model::SharedSubscriberId::from_stream_uid(action.uid), + ConnectionKind::Normal, + action.new_priority, + ); + } else { + provider_manager.reclassify_connection_for_owner( + &action.addr, + stream_info.session_token.as_deref(), + ConnectionKind::Normal, + action.new_priority, + ); + } + } + self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + } + } + + pub async fn active_users_and_connections(&self) -> (usize, usize) { + self.gc(); + let user_connections = self.connections.read().await; + user_connections + .by_key + .values() + .filter_map(|c| { + let effective = c.connections as usize; + if effective > 0 { + Some(effective) + } else { + None + } + }) + .fold((0usize, 0usize), |(user_count, conn_count), effective| (user_count + 1, conn_count + effective)) + } + + pub async fn playback_resource_counts(&self) -> (usize, usize) { + self.gc(); + let user_connections = self.connections.read().await; + user_connections.by_key.values().fold((0, 0), |(requests, sessions), connection| { + (requests + connection.stream_request_claims.len(), sessions + connection.sessions.len()) + }) + } + + pub(super) fn is_log_user_enabled(&self) -> bool { self.log_active_user.load(Ordering::Relaxed) } + + pub async fn active_streams(&self) -> Vec { + self.gc(); + let user_connections = self.connections.read().await; + let mut streams = Vec::new(); + for connection_data in user_connections.by_key.values() { + for stream in &connection_data.streams { + // Keep active_streams free of preserved rows — shared-HLS join detection and + // connection accounting must not see stale archive segment leases (v3.3.81). + if !stream.preserved { + streams.push(stream.clone()); + } + } + } + streams + } + + /// Streams for the `WebUI` / `StatusCheck` snapshot. + /// + /// Includes preserved Catchup/HLS/DASH session rows so the panel keeps showing archive + /// playback between short segment sockets. Do not use this for shared-HLS accounting. + pub async fn panel_streams(&self) -> Vec { + self.gc(); + let user_connections = self.connections.read().await; + let mut streams = Vec::new(); + for connection_data in user_connections.by_key.values() { + for stream in &connection_data.streams { + if !stream.preserved || Self::should_preserve_session_stream(stream) { + streams.push(stream.clone()); + } + } + } + streams + } + + pub(super) fn log_connection_added( + username: &str, + addr: &SocketAddr, + connection_data: &UserConnectionData, + tracked_socket_count: usize, + ) { + if log::log_enabled!(log::Level::Debug) { + let active_for_user = connection_data.connections; + if connection_data.max_connections > 0 && active_for_user > connection_data.max_connections { + let recent_sockets = connection_data + .streams + .iter() + .rev() + .take(3) + .map(|stream| stream.addr.to_string()) + .collect::>() + .join(", "); + let recent_sockets = if recent_sockets.is_empty() { String::from("n/a") } else { recent_sockets }; + let unique_clients = + connection_data.streams.iter().map(|stream| &stream.client_ip).collect::>().len(); + debug!( + "User {username} exceeded configured max connections ({}/{}). Unique clients: {}, recent sockets [{}]", + active_for_user, + connection_data.max_connections, + unique_clients, + recent_sockets + ); + } else { + debug_if_enabled!( + "Added new connection for {username} at {} (active user connections={active_for_user}, tracked sockets={tracked_socket_count})", + sanitize_sensitive_info(&addr.to_string()) + ); + } + } + } + + pub(super) fn collect_divergence_snapshot( + connection_data: &UserConnectionData, + username: &str, + ) -> Option { + log_enabled!(log::Level::Debug).then(|| Self::build_divergence_snapshot(connection_data, username)) + } + + pub(super) fn build_divergence_snapshot( + connection_data: &UserConnectionData, + username: &str, + ) -> DivergenceSnapshot { + let connections = connection_data.connections; + let counted_sessions = connection_data.sessions.iter().filter(|s| s.lifecycle.is_counted()).count(); + let streams_count = connection_data.streams.len(); + let mut kinds = Vec::new(); + + for session in &connection_data.sessions { + if !session.lifecycle.is_counted() { + continue; + } + if matches!(session.lifecycle, PlaybackLifecycle::PendingProvider { ref data } if data.reason_code == PendingProviderReason::GraceHold) + { + continue; + } + let has_active_stream = connection_data + .streams + .iter() + .any(|s| s.session_token.as_deref() == Some(&session.token) && !s.preserved); + if !has_active_stream { + kinds.push(DivergenceKind::CountedSessionWithoutStream); + } + } + + for stream in &connection_data.streams { + if stream.preserved { + continue; + } + let Some(token) = stream.session_token.as_deref() else { + continue; + }; + let has_counted_session = + connection_data.sessions.iter().any(|s| s.token == token && s.lifecycle.is_counted()); + if !has_counted_session { + kinds.push(DivergenceKind::StreamWithoutCountedSession); + } + } + + #[allow(clippy::cast_possible_truncation)] + let counted_sessions_u32 = counted_sessions as u32; + if connections != counted_sessions_u32 { + kinds.push(DivergenceKind::ConnectionCountMismatch { legacy: connections, counted: counted_sessions_u32 }); + } + + DivergenceSnapshot { username: username.to_string(), connections, counted_sessions, streams_count, kinds } + } + + pub(super) async fn log_divergence_snapshot(&self, snapshot: Option) { + let Some(snapshot) = snapshot else { + return; + }; + let cooldown = Duration::from_secs(self.divergence_cooldown_secs); + for kind in &snapshot.kinds { + let key = divergence_key(&snapshot.username, kind); + let should_log = { + let mut cache = self.divergence_cache.lock().await; + if let Some(entry) = cache.get_mut(&key) { + if entry.last_logged.elapsed() >= cooldown { + entry.last_logged = Instant::now(); + entry.count_since_last_log = 0; + true + } else { + entry.count_since_last_log = entry.count_since_last_log.saturating_add(1); + false + } + } else { + cache.push(key, DivergenceEntry { last_logged: Instant::now(), count_since_last_log: 0 }); + true + } + }; + + if should_log { + debug!( + "ADMISSION DIVERGENCE user={} kind={kind:?} connections={} counted_sessions={} streams={}", + snapshot.username, snapshot.connections, snapshot.counted_sessions, snapshot.streams_count, + ); + } + } + } + + pub(super) async fn check_and_log_divergence_for_user(&self, username: &str) { + let snapshot = { + let connections = self.connections.read().await; + let Some(data) = connections.by_key.get(username) else { + return; + }; + Self::collect_divergence_snapshot(data, username) + }; + self.log_divergence_snapshot(snapshot).await; + } +} diff --git a/backend/session/src/active_user_manager/provider_headers.rs b/backend/session/src/active_user_manager/provider_headers.rs new file mode 100644 index 000000000..9d1157a96 --- /dev/null +++ b/backend/session/src/active_user_manager/provider_headers.rs @@ -0,0 +1,162 @@ +use super::{ActiveUserManager, SessionIdentity, SessionProviderHeaders}; +use shared::utils::current_time_secs; +use std::{ + collections::HashMap, + sync::{atomic::Ordering, Arc}, +}; + +impl ActiveUserManager { + /// Provider session headers valid for `target_url` now, for the session object and account + /// binding of `identity` in any lifecycle state: a manifest refresh may revive an expired + /// reservation. + pub async fn provider_headers_for_session_identity( + &self, + username: &str, + token: &str, + identity: SessionIdentity, + target_url: &str, + ) -> SessionProviderHeaders { + let users = self.connections.read().await; + users + .by_key + .get(username) + .and_then(|data| { + data.sessions.iter().find(|session| session.token == token && session.identity() == identity) + }) + .map_or(SessionProviderHeaders::NoSession, |session| { + SessionProviderHeaders::for_target(session, target_url) + }) + } + + /// Provider session headers currently valid for `target_url`, read at send time so cookies + /// rotated or expired while the request waited are honored. + pub async fn current_session_provider_headers( + &self, + username: &str, + token: &str, + identity: SessionIdentity, + target_url: &str, + ) -> SessionProviderHeaders { + let users = self.connections.read().await; + users.current_session(username, token, identity).map_or(SessionProviderHeaders::NoSession, |(_, session)| { + SessionProviderHeaders::for_target(session, target_url) + }) + } + + pub async fn update_session_provider_headers( + &self, + username: &str, + token: &str, + provider_session_headers: &HashMap, + ) -> bool { + let mut user_connections = self.write_connections().await; + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return false; + }; + let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) else { + return false; + }; + session.provider_session_headers.clone_from(provider_session_headers); + session.provider_session_headers_host = None; + session.clear_provider_session_cookies(); + session.ts = current_time_secs(); + true + } + + pub async fn update_session_provider_response_headers_from( + &self, + username: &str, + token: &str, + response: &crate::ProviderSessionHeaders, + source_url: &str, + ) -> bool { + self.store_session_provider_response_headers(username, token, None, response, source_url).await + } + + /// Stores response cookies only while the session still has the expected account binding. + pub async fn update_current_session_provider_response_headers_from( + &self, + username: &str, + token: &str, + identity: SessionIdentity, + response: &crate::ProviderSessionHeaders, + source_url: &str, + ) -> bool { + self.store_session_provider_response_headers(username, token, Some(identity), response, source_url).await + } + + pub(super) async fn store_session_provider_response_headers( + &self, + username: &str, + token: &str, + expected: Option, + response: &crate::ProviderSessionHeaders, + source_url: &str, + ) -> bool { + if response.is_empty() { + return false; + } + let Ok(source) = url::Url::parse(source_url) else { + return false; + }; + let mut users = self.write_connections().await; + if expected.is_some() && users.is_token_ended(token) { + return false; + } + let Some(session) = users + .by_key + .get_mut(username) + .and_then(|data| data.sessions.iter_mut().find(|session| session.token == token)) + else { + return false; + }; + if expected.is_some_and(|identity| session.identity() != identity || !session.is_live()) { + return false; + } + Arc::make_mut(&mut session.provider_session_cookies).update(&source, response); + // The origin-scoped cookie store is the only source once a provider response was stored. + session.provider_session_headers.clear(); + session.provider_session_headers_host = None; + session.ts = current_time_secs(); + true + } + + /// Returns the stable User-Agent suffix for a playback session, assigning one on first use. + pub async fn get_or_assign_user_agent_stream_index(&self, username: &str, token: &str) -> Option { + let mut user_connections = self.write_connections().await; + let session = + user_connections.by_key.get_mut(username)?.sessions.iter_mut().find(|session| session.token == token)?; + Some(*session.user_agent_stream_index.get_or_insert_with(|| self.next_user_agent_stream_index())) + } + + /// Allocates the next process-local, globally unique User-Agent stream index. + pub fn next_user_agent_stream_index(&self) -> u64 { + let mut current = self.next_user_agent_stream_index.load(Ordering::Relaxed); + loop { + let next = if current == 0 || current == u64::MAX { 1 } else { current + 1 }; + match self.next_user_agent_stream_index.compare_exchange_weak( + current, + next, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(previous) => return if previous == 0 { 1 } else { previous }, + Err(observed) => current = observed, + } + } + } + + /// Persists a previously allocated index without replacing an existing session identity. + pub async fn set_user_agent_stream_index_if_absent(&self, username: &str, token: &str, index: u64) -> bool { + let mut user_connections = self.write_connections().await; + let Some(session) = user_connections + .by_key + .get_mut(username) + .and_then(|connection| connection.sessions.iter_mut().find(|session| session.token == token)) + else { + return false; + }; + session.user_agent_stream_index.get_or_insert(index); + true + } +} diff --git a/backend/session/src/active_user_manager/requests.rs b/backend/session/src/active_user_manager/requests.rs new file mode 100644 index 000000000..fd8f03018 --- /dev/null +++ b/backend/session/src/active_user_manager/requests.rs @@ -0,0 +1,262 @@ +use super::{ + decide_connection_kind, remember_session_addr, stream_history_session_id, ActiveUserConnectionParams, + ActiveUserManager, PlaybackSessionRegistration, SocketRegistration, StreamRequestDetach, UserConnectionData, +}; +use shared::{model::StreamInfo, utils::current_time_secs}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; +use tokio::sync::Mutex; +use tuliprox_core::utils::utc_day_from_secs; + +#[derive(Debug, Clone, Copy)] +pub(super) struct StreamRequestClaim { + pub(super) stream_uid: u32, + pub(super) addr: SocketAddr, +} + +impl UserConnectionData { + pub(super) fn remove_stream_request_claims(&mut self, stream_uid: u32) { + self.stream_request_claims.retain(|_, claim| claim.stream_uid != stream_uid); + } +} + +impl ActiveUserManager { + pub(super) fn transition_gate_key(username: &str, token: &str) -> String { + let mut key = String::with_capacity(username.len() + token.len() + 1); + key.push_str(username); + key.push('\0'); + key.push_str(token); + key + } + + pub(super) fn admission_gate_key(username: &str) -> String { + let mut key = String::with_capacity(username.len() + 11); + key.push_str("admission"); + key.push('\0'); + key.push_str(username); + key + } + + pub(super) fn cleanup_idle_transition_gates(transition_gates: &mut HashMap>>) { + transition_gates.retain(|_, gate| Arc::strong_count(gate) > 1); + } + + pub async fn acquire_playback_transition(&self, username: &str, token: &str) -> tokio::sync::OwnedMutexGuard<()> { + let key = Self::transition_gate_key(username, token); + let gate = { + let mut transition_gates = self.transition_gates.lock().await; + Self::cleanup_idle_transition_gates(&mut transition_gates); + Arc::clone(transition_gates.entry(key).or_insert_with(|| Arc::new(Mutex::new(())))) + }; + gate.lock_owned().await + } + + pub async fn acquire_user_admission(&self, username: &str) -> tokio::sync::OwnedMutexGuard<()> { + let key = Self::admission_gate_key(username); + let gate = { + let mut transition_gates = self.transition_gates.lock().await; + Self::cleanup_idle_transition_gates(&mut transition_gates); + Arc::clone(transition_gates.entry(key).or_insert_with(|| Arc::new(Mutex::new(())))) + }; + gate.lock_owned().await + } + + pub async fn release_stream_request_by_uid(&self, addr: &SocketAddr, request_uid: u32) -> StreamRequestDetach { + self.release_stream_inner(addr, Some(request_uid)).await + } + + #[allow(clippy::too_many_lines)] + pub async fn update_connection_with_session_registration( + &self, + update: ActiveUserConnectionParams<'_>, + session_registration: Option<&PlaybackSessionRegistration>, + ) -> Option { + let ActiveUserConnectionParams { + uid, + meter_uid, + username, + max_connections, + soft_connections, + connection_kind, + priority, + soft_priority: _, + fingerprint, + provider, + stream_channel, + user_agent, + session_token, + } = update; + let (stream_info, divergence_snapshot, connection_count_changed) = { + let mut user_connections = self.write_connections().await; + if let Some(requirement) = session_registration { + let token = session_token?; + let (data, session) = user_connections.current_session(username, token, requirement.identity)?; + if requirement.enforce_limits + && !requirement.grace_admitted + && !session.lifecycle.is_counted() + && !Self::session_has_stream(data, token) + && decide_connection_kind( + data.effective_counts_for_admission(Some(token)), + max_connections, + soft_connections, + ) != Some(connection_kind) + { + return None; + } + } + + let now = current_time_secs(); + let registration = + user_connections.key_by_addr.entry(fingerprint.addr).or_insert_with(SocketRegistration::anonymous); + registration.add_user(username, now); + + let tracked_socket_count = user_connections.key_by_addr.len(); + let connection_data = user_connections + .by_key + .entry(username.to_string()) + .or_insert_with(|| UserConnectionData::new(0, max_connections, soft_connections)); + connection_data.max_connections = max_connections; + connection_data.soft_connections = soft_connections; + let previous_connection_count = connection_data.connections; + + if let Some(token) = session_token { + if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { + session.ts = now; + remember_session_addr(session, fingerprint.addr); + Self::bump_session_transition_version(session); + } + } + + let user_agent_string = user_agent.to_string(); + let reserved_session_kind = session_token.and_then(|token| { + connection_data + .sessions + .iter() + .find(|session| session.token == token && session.lifecycle.is_counted()) + .map(|session| session.connection_kind.unwrap_or(connection_kind)) + }); + + let existing_stream_info = connection_data + .streams + .iter() + .position(|stream_info| { + !stream_channel.shared + && match session_token { + Some(token) => { + stream_info.session_token.as_deref() == Some(token) + && Self::should_reuse_stream_for_session(stream_info, stream_channel) + } + None => stream_info.uid == uid && stream_info.session_token.is_none(), + } + }) + .map(|stream_idx| { + let session_started_at = session_token.and_then(|token| { + connection_data.sessions.iter().find(|s| s.token == token).map(|s| s.started_at) + }); + + let stream_info = &mut connection_data.streams[stream_idx]; + let client_ip = fingerprint.client_ip.clone(); + let preserve_started_at = stream_info.session_token.is_some() + && (stream_info.channel.item_type.is_live_adaptive() + || stream_channel.item_type.is_live_adaptive()); + let was_preserved = stream_info.preserved; + let old_session_id = stream_history_session_id(stream_info.ts, stream_info.uid); + stream_info.meter_uid = meter_uid; + stream_info.addr = fingerprint.addr; + stream_info.client_ip.clone_from(&client_ip); + stream_info.country_code = self.lookup_country(&client_ip); + stream_info.channel = stream_channel.clone(); + stream_info.provider = provider.clone(); + stream_info.user_agent.clone_from(&user_agent_string); + + if let Some(started_at) = session_started_at { + stream_info.started_at = started_at; + } + + if preserve_started_at { + let now = current_time_secs(); + if utc_day_from_secs(stream_info.ts) != utc_day_from_secs(now) { + stream_info.ts = now; + stream_info.previous_session_id = Some(old_session_id); + } + } else { + stream_info.ts = current_time_secs(); + } + + if let Some(token) = session_token { + stream_info.session_token = Some(token.to_string()); + } + if was_preserved { + stream_info.preserved = false; + } + connection_data.stream_normal_priorities.insert(stream_info.uid, priority); + let result = stream_info.clone(); + stream_info.previous_session_id = None; + (result, was_preserved) + }); + let (stream_info, divergence_snapshot) = if let Some((stream_info, was_preserved)) = existing_stream_info { + let effective_connection_kind = reserved_session_kind.unwrap_or(connection_kind); + connection_data.attach_stream_request(uid, stream_info.uid, fingerprint.addr); + if was_preserved { + connection_data.increment_kind(effective_connection_kind); + } + connection_data.stream_kinds.insert(stream_info.uid, effective_connection_kind); + connection_data.stream_normal_priorities.insert(stream_info.uid, priority); + if let Some(token) = session_token { + if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { + Self::mark_session_committed(session, effective_connection_kind); + } + } + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); + (stream_info, divergence_snapshot) + } else { + let effective_connection_kind = reserved_session_kind.unwrap_or(connection_kind); + let country_code = self.lookup_country(&fingerprint.client_ip); + + let mut stream_info = StreamInfo::new(shared::model::StreamInfoParams { + uid, + meter_uid, + username, + addr: &fingerprint.addr, + client_ip: &fingerprint.client_ip, + provider, + stream_channel: stream_channel.clone(), + user_agent: user_agent_string, + country_code, + session_token, + }); + + if let Some(token) = session_token { + if let Some(session) = connection_data.sessions.iter().find(|s| s.token == token) { + stream_info.started_at = session.started_at; + } + } + + if reserved_session_kind.is_none() { + connection_data.increment_kind(effective_connection_kind); + } + connection_data.streams.push(stream_info.clone()); + connection_data.attach_stream_request(uid, stream_info.uid, fingerprint.addr); + connection_data.stream_kinds.insert(stream_info.uid, effective_connection_kind); + connection_data.stream_normal_priorities.insert(stream_info.uid, priority); + if let Some(token) = session_token { + if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { + Self::mark_session_committed(session, effective_connection_kind); + } + } + Self::log_connection_added(username, &fingerprint.addr, connection_data, tracked_socket_count); + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); + (stream_info, divergence_snapshot) + }; + let connection_count_changed = connection_data.connections != previous_connection_count; + (stream_info, divergence_snapshot, connection_count_changed) + }; + + self.log_divergence_snapshot(divergence_snapshot).await; + + if connection_count_changed { + self.log_active_user().await; + } + + Some(stream_info) + } +} diff --git a/backend/session/src/active_user_manager/session_lifecycle.rs b/backend/session/src/active_user_manager/session_lifecycle.rs new file mode 100644 index 000000000..efd4ff9ec --- /dev/null +++ b/backend/session/src/active_user_manager/session_lifecycle.rs @@ -0,0 +1,877 @@ +use super::{ + url_host_key, ActiveUserManager, CreateUserSessionParams, DeferredWakes, PendingProviderReason, + PendingProviderState, PendingProviderWakeSource, PlaybackLifecycle, SessionChangeSignal, SocketRegistration, + UserConnectionData, UserSession, NEXT_SESSION_INCARNATION, +}; +use crate::active_provider_manager::ConnectionKind; +use log::debug; +use shared::{ + defaults::{default_hls_session_ttl_secs, DASH_EXT, HLS_EXT}, + model::{PlaylistItemType, StreamInfo, UserConnectionPermission}, + utils::{ + current_time_secs, extract_extension_from_url, is_catchup_session_token, sanitize_sensitive_info, strip_port, + }, +}; +use std::{ + borrow::Cow, + collections::HashMap, + net::SocketAddr, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; +use tokio::sync::Notify; +use tuliprox_core::{model::Config, utils::debug_if_enabled}; + +pub(super) fn get_adaptive_session_ttl_secs(config: &Config) -> u64 { + config + .reverse_proxy + .as_ref() + .and_then(|r| r.stream.as_ref()) + .map_or_else(default_hls_session_ttl_secs, |s| s.hls_session_ttl_secs) +} + +pub(super) fn stream_history_session_id(ts: u64, uid: u32) -> u64 { (ts << 32) | u64::from(uid) } + +pub fn next_session_incarnation() -> u64 { NEXT_SESSION_INCARNATION.fetch_add(1, Ordering::Relaxed) } + +impl Default for SessionChangeSignal { + fn default() -> Self { Self { notify: Arc::new(Notify::new()), deferred: None, owner: true } } +} + +impl Clone for SessionChangeSignal { + fn clone(&self) -> Self { Self { notify: Arc::clone(&self.notify), deferred: self.deferred.clone(), owner: false } } +} + +impl Drop for SessionChangeSignal { + fn drop(&mut self) { + if self.owner { + self.signal(); + } + } +} + +impl std::fmt::Debug for SessionChangeSignal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SessionChangeSignal").field("owner", &self.owner).finish_non_exhaustive() + } +} + +/// A detached session with a fresh incarnation, for fixtures and struct-update construction. +/// Sessions in the connection table come from `create_user_session`. +impl Default for UserSession { + fn default() -> Self { + Self { + token: String::new(), + transition_version: 1, + virtual_id: 0, + provider: Arc::from(""), + stream_url: Arc::from(""), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + provider_session_cookies: Arc::default(), + incarnation: next_session_incarnation(), + binding_generation: 0, + change_signal: SessionChangeSignal::default(), + media_started: Arc::new(AtomicBool::new(false)), + user_agent_stream_index: None, + addr: SocketAddr::from(([0, 0, 0, 0], 0)), + socket_bound: false, + active_addrs: Vec::new(), + ts: 0, + started_at: 0, + permission: UserConnectionPermission::Allowed, + connection_kind: None, + lifecycle: PlaybackLifecycle::default(), + } + } +} + +impl UserSession { + pub(super) fn clear_provider_session_cookies(&mut self) { + if !self.provider_session_cookies.is_empty() { + self.provider_session_cookies = Arc::default(); + } + } + + /// Provider session headers that may be sent to `target_url`: only to the origin (scheme, + /// host and port) that set them, or unconditionally when that origin is unknown. + pub fn provider_session_headers_for(&self, target_url: &str) -> Option>> { + if !self.provider_session_cookies.is_empty() { + return self.provider_session_cookies.headers_for(target_url); + } + if self.provider_session_headers.is_empty() { + return None; + } + match self.provider_session_headers_host.as_deref() { + Some(host) => (url_host_key(target_url).as_deref() == Some(host)) + .then_some(Cow::Borrowed(&self.provider_session_headers)), + None => Some(Cow::Borrowed(&self.provider_session_headers)), + } + } +} + +impl UserConnectionData { + pub(super) fn has_session_addr(&self, addr: &SocketAddr) -> bool { + self.sessions.iter().any(|session| session.addr == *addr || session.active_addrs.contains(addr)) + } +} + +pub(super) fn is_stable_session_stream(stream: &StreamInfo) -> bool { + // Catchup-token Live/.ts segment sockets must preserve too; otherwise archive panel rows + // hard-remove every HLS chunk and Streams blinks even when frontend soft-preserve is present. + stream.channel.item_type == PlaylistItemType::Catchup + || stream.channel.item_type.is_live_adaptive() + || stream.session_token.as_deref().is_some_and(is_catchup_session_token) + || matches!( + extract_extension_from_url(stream.channel.url.as_ref()), + Some(ext) if ext == HLS_EXT || ext == DASH_EXT + ) +} + +pub(super) fn uses_session_reentry_guard(stream: &StreamInfo) -> bool { + stream.channel.item_type.requires_provider_affinity() + || matches!( + extract_extension_from_url(stream.channel.url.as_ref()), + Some(ext) if ext == HLS_EXT || ext == DASH_EXT + ) +} + +pub(super) fn remember_session_addr(session: &mut UserSession, addr: SocketAddr) { + if session.socket_bound { + session.active_addrs.clear(); + } else if let Some(position) = session.active_addrs.iter().position(|active_addr| *active_addr == addr) { + session.active_addrs.remove(position); + } + session.active_addrs.push(addr); + session.addr = addr; +} + +pub(super) fn release_session_addr(session: &mut UserSession, addr: &SocketAddr) -> Option { + if let Some(position) = session.active_addrs.iter().position(|active_addr| active_addr == addr) { + session.active_addrs.remove(position); + } else if session.addr != *addr { + return None; + } + + if session.addr == *addr { + if let Some(next_addr) = session.active_addrs.last().copied() { + session.addr = next_addr; + return Some(next_addr); + } + } + + None +} + +pub(super) fn clear_session_addr(session: &mut UserSession, addr: &SocketAddr) -> bool { + let mut changed = false; + if let Some(position) = session.active_addrs.iter().position(|active_addr| active_addr == addr) { + session.active_addrs.remove(position); + changed = true; + } + + if session.addr == *addr { + if let Some(next_addr) = session.active_addrs.last().copied() { + session.addr = next_addr; + } else { + session.addr = SocketAddr::from(([0, 0, 0, 0], 0)); + } + changed = true; + } + + changed +} + +pub(super) struct UserSessionParams<'a> { + pub(super) change_wakes: &'a Arc, + pub(super) session_token: &'a str, + pub(super) virtual_id: u32, + pub(super) provider: &'a str, + pub(super) stream_url: &'a str, + pub(super) addr: &'a SocketAddr, + pub(super) connection_permission: UserConnectionPermission, + pub(super) connection_kind: Option, + pub(super) socket_bound: bool, +} + +impl ActiveUserManager { + pub async fn refresh_session_connection_kind_for_origin_policy( + &self, + username: &str, + max_connections: u32, + soft_connections: u16, + session_token: &str, + ) -> Option { + if max_connections == 0 && soft_connections == 0 { + return Some(ConnectionKind::Normal); + } + + let (connection_kind, promotions, divergence_snapshot) = { + let mut connections = self.write_connections().await; + let connection_data = connections.by_key.get_mut(username)?; + connection_data.max_connections = max_connections; + connection_data.soft_connections = soft_connections; + + let session_index = connection_data.sessions.iter().position(|session| session.token == session_token)?; + + let promotions = Self::promote_counted_soft_session_to_normal_if_available(connection_data, session_token); + let connection_kind = if connection_data.sessions[session_index].lifecycle.is_counted() + || Self::session_has_stream(connection_data, session_token) + { + Some(connection_data.sessions[session_index].connection_kind.unwrap_or(ConnectionKind::Normal)) + } else { + let admission = self.check_connection_admission_with_counts( + username, + connection_data, + connection_data.effective_counts_for_admission(Some(session_token)), + ); + if admission.permission() == UserConnectionPermission::Allowed { + if let Some(kind) = admission.kind() { + Self::update_session_admission( + &mut connection_data.sessions[session_index], + admission.permission(), + Some(kind), + ); + } + admission.kind() + } else { + None + } + }; + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); + + (connection_kind, promotions, divergence_snapshot) + }; + + self.log_divergence_snapshot(divergence_snapshot).await; + for action in promotions { + self.emit_promotion_update(username, action).await; + } + + connection_kind + } + + pub(super) fn bump_session_transition_version(session: &mut UserSession) -> u64 { + session.transition_version = session.transition_version.saturating_add(1); + session.transition_version + } + + pub(super) fn mark_session_committed(session: &mut UserSession, kind: ConnectionKind) { + session.connection_kind = Some(kind); + session.lifecycle = PlaybackLifecycle::Active; + Self::bump_session_transition_version(session); + } + + pub(super) fn session_has_stream(connection_data: &UserConnectionData, session_token: &str) -> bool { + connection_data.streams.iter().any(|stream| stream.session_token.as_deref() == Some(session_token)) + } + + pub async fn ensure_user_session_placeholder(&self, request: CreateUserSessionParams<'_>) -> u64 { + let CreateUserSessionParams { + user, + session_token, + virtual_id, + provider, + stream_url, + addr, + connection_permission, + connection_kind, + socket_bound, + } = request; + self.gc(); + + let username = user.username.clone(); + let mut user_connections = self.write_connections().await; + let connection_data = user_connections + .by_key + .entry(username.clone()) + .or_insert_with(|| UserConnectionData::new(0, user.max_connections, user.soft_connections)); + + if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == session_token) { + session.ts = current_time_secs(); + session.socket_bound = socket_bound; + remember_session_addr(session, *addr); + if session.connection_kind.is_none() { + session.connection_kind = connection_kind; + } + if session.permission == UserConnectionPermission::Exhausted { + Self::update_session_admission(session, connection_permission, None); + } + let version = Self::bump_session_transition_version(session); + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, &username); + drop(user_connections); + self.log_divergence_snapshot(divergence_snapshot).await; + return version; + } + + let session = Self::new_user_session(&UserSessionParams { + change_wakes: &self.deferred_wakes, + session_token, + virtual_id, + provider, + stream_url, + addr, + connection_permission, + connection_kind, + socket_bound, + }); + let version = session.transition_version; + connection_data.add_session(session); + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, &username); + drop(user_connections); + self.log_divergence_snapshot(divergence_snapshot).await; + version + } + + pub async fn update_session_addr(&self, username: &str, token: &str, addr: &SocketAddr) { + let now = current_time_secs(); + let mut user_connections = self.write_connections().await; + if let Some(connection_data) = user_connections.by_key.get_mut(username) { + let update_result = if let Some(session) = connection_data.sessions.iter_mut().find(|s| s.token == token) { + let previous_addr = session.addr; + remember_session_addr(session, *addr); + session.ts = now; + Self::bump_session_transition_version(session); + for stream in &mut connection_data.streams { + if stream.addr == previous_addr && stream.session_token.as_deref() == Some(token) { + stream.addr = *addr; + stream.ts = now; + } + } + let prune_previous_registration = previous_addr != *addr + && !connection_data.has_session_addr(&previous_addr) + && !connection_data.streams.iter().any(|stream| stream.addr == previous_addr); + Some((previous_addr, prune_previous_registration)) + } else { + None + }; + + if let Some((previous_addr, prune_previous_registration)) = update_result { + let registration = + user_connections.key_by_addr.entry(*addr).or_insert_with(SocketRegistration::anonymous); + registration.add_user(username, now); + if prune_previous_registration { + let can_remove_previous = user_connections + .key_by_addr + .get_mut(&previous_addr) + .is_some_and(|registration| registration.remove_user(username)); + if can_remove_previous { + user_connections.key_by_addr.remove(&previous_addr); + } + } + debug_if_enabled!( + "Updated session {token} for {username} address {} -> {}", + sanitize_sensitive_info(&previous_addr.to_string()), + sanitize_sensitive_info(&addr.to_string()) + ); + } + } + } + + pub async fn update_session_provider_binding( + &self, + username: &str, + token: &str, + provider: Arc, + stream_url: Arc, + ) { + let now = current_time_secs(); + let mut user_connections = self.write_connections().await; + if let Some(connection_data) = user_connections.by_key.get_mut(username) { + if let Some(session) = connection_data.sessions.iter_mut().find(|s| s.token == token) { + let previous_provider = session.provider.clone(); + if session.provider != provider { + session.switch_provider(provider.clone()); + } + session.stream_url = stream_url.clone(); + session.ts = now; + Self::bump_session_transition_version(session); + for stream in &mut connection_data.streams { + if stream.session_token.as_deref() == Some(token) { + stream.provider = provider.clone(); + stream.ts = now; + } + } + debug_if_enabled!( + "Updated session {token} for {username} provider binding {} -> {}", + sanitize_sensitive_info(&previous_provider), + sanitize_sensitive_info(&provider) + ); + } + } + } + + pub async fn clear_unbound_session_addr(&self, username: &str, token: &str, addr: &SocketAddr) { + let now = current_time_secs(); + let mut user_connections = self.write_connections().await; + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return; + }; + let addr_has_active_stream_for_session = connection_data + .streams + .iter() + .any(|stream| stream.session_token.as_deref() == Some(token) && stream.addr == *addr && !stream.preserved); + if addr_has_active_stream_for_session { + return; + } + + let cleared = if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) + { + let changed = clear_session_addr(session, addr); + if changed { + Self::bump_session_transition_version(session); + } + changed + } else { + false + }; + let can_remove_registration = !connection_data.has_session_addr(addr) + && !connection_data.streams.iter().any(|stream| stream.addr == *addr); + if can_remove_registration { + let can_remove = user_connections + .key_by_addr + .get_mut(addr) + .is_some_and(|registration| registration.remove_user(username)); + if can_remove { + user_connections.key_by_addr.remove(addr); + } + } else if cleared { + if let Some(registration) = user_connections.key_by_addr.get_mut(addr) { + registration.ts = now; + } + } + } + + pub async fn mark_pending_provider( + &self, + username: &str, + token: &str, + reason_code: PendingProviderReason, + deadline: u64, + ) -> Option { + let mut user_connections = self.write_connections().await; + let connection_data = user_connections.by_key.get_mut(username)?; + let now = current_time_secs(); + if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { + let version = match session.lifecycle { + PlaybackLifecycle::PendingProvider { ref data } => data.version.saturating_add(1), + _ => 1, + }; + // Capture counted status BEFORE lifecycle transition to PendingProvider. + // is_counted() returns false for PendingProvider, so we must check first. + let kind = if session.lifecycle.is_counted() { + Some(session.connection_kind.unwrap_or(ConnectionKind::Normal)) + } else { + None + }; + session.ts = now; + Self::bump_session_transition_version(session); + Self::update_session_admission(session, UserConnectionPermission::GracePeriod, None); + session.lifecycle = PlaybackLifecycle::PendingProvider { + data: PendingProviderState { reason_code, created_at: now, deadline, version, wake_source: None }, + }; + if let Some(kind) = kind { + connection_data.decrement_kind(kind); + } + return Some(version); + } + None + } + + pub async fn activate_pending_provider( + &self, + username: &str, + token: &str, + expected_version: u64, + wake_source: PendingProviderWakeSource, + ) { + let mut user_connections = self.write_connections().await; + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return; + }; + if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { + let PlaybackLifecycle::PendingProvider { data } = &mut session.lifecycle else { + return; + }; + if data.version != expected_version { + return; + } + data.wake_source = Some(wake_source); + Self::bump_session_transition_version(session); + session.permission = UserConnectionPermission::Allowed; + session.lifecycle = PlaybackLifecycle::Active; + } + } + + /// Returns the current `transition_version` if the session is in `GraceActive` lifecycle. + /// Used by the grace task to confirm the session is still in `GraceActive` before committing. + pub async fn grace_active_version(&self, username: &str, token: &str) -> Option { + let connections = self.connections.read().await; + let connection_data = connections.by_key.get(username)?; + let session = connection_data.sessions.iter().find(|s| s.token == token)?; + if session.lifecycle == PlaybackLifecycle::GraceActive { + Some(session.transition_version) + } else { + None + } + } + + /// Marks a session as `GraceActive` — the session was granted immediate grace + /// (`GraceMode::Instant`) and is provisionally active. The session counts against + /// admission limits in this state. + /// + /// This corresponds to `Prepared -> GraceActive` in the playback state machine. + /// The session remains in `GraceActive` until either: + /// - `activate_grace_active` confirms it (grace window succeeded -> `GraceActive -> Active`) + /// - `expire_grace_active` expires it (grace window failed -> `GraceActive -> Expired`) + pub async fn mark_grace_active(&self, username: &str, token: &str) { + let mut user_connections = self.write_connections().await; + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return; + }; + let Some(session_index) = connection_data.sessions.iter().position(|session| session.token == token) else { + return; + }; + if connection_data.sessions[session_index].lifecycle == PlaybackLifecycle::GraceActive { + return; // already grace active + } + // Collect fields while only borrowing sessions. + let kind = connection_data.sessions[session_index].connection_kind.unwrap_or(ConnectionKind::Normal); + let needs_count = !connection_data.sessions[session_index].lifecycle.is_counted(); + let now = current_time_secs(); + // Now mutate. Use index access to avoid nested &mut borrows. + connection_data.sessions[session_index].ts = now; + Self::bump_session_transition_version(&mut connection_data.sessions[session_index]); + if needs_count { + connection_data.increment_kind(kind); + } + connection_data.sessions[session_index].lifecycle = PlaybackLifecycle::GraceActive; + } + + /// Activates a `GraceActive` session when the grace window resolves successfully. + /// + /// This corresponds to `GraceActive -> Active` in the playback state machine. + /// The session remains counted and the kind counts are already correct from + /// the `GraceActive` provisional state. + pub async fn activate_grace_active(&self, username: &str, token: &str, expected_version: u64) { + let mut user_connections = self.write_connections().await; + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return; + }; + if let Some(session) = connection_data.sessions.iter_mut().find(|session| session.token == token) { + if session.transition_version != expected_version { + return; + } + if session.lifecycle != PlaybackLifecycle::GraceActive { + return; + } + Self::bump_session_transition_version(session); + session.lifecycle = PlaybackLifecycle::Active; + session.permission = UserConnectionPermission::Allowed; + } + } + + /// Expires a `GraceActive` session when the grace window fails. + /// + /// This corresponds to `GraceActive -> Expired` in the playback state machine. + /// Releases the provisional counted lease. + pub async fn expire_grace_active(&self, username: &str, token: &str, expected_version: u64) { + let (connection_changed, removed_count) = { + let mut user_connections = self.write_connections().await; + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return; + }; + let Some(session_index) = connection_data.sessions.iter().position(|session| session.token == token) else { + return; + }; + + if connection_data.sessions[session_index].transition_version != expected_version { + return; + } + if connection_data.sessions[session_index].lifecycle != PlaybackLifecycle::GraceActive { + return; + } + + // Release the provisional counted lease using index-based access + // to avoid nested mutable borrows with connection_data methods. + let mut connection_changed = false; + let mut counted_kind: Option = None; + if connection_data.sessions[session_index].lifecycle.is_counted() { + counted_kind = connection_data.sessions[session_index].connection_kind; + connection_changed = true; + } + if let Some(kind) = counted_kind { + connection_data.decrement_kind(kind); + } + + // Expire the session. Lifecycle change alone handles counted state (Expired is not counted). + connection_data.sessions[session_index].expire(); + connection_data.sessions[session_index].set_permission(UserConnectionPermission::Exhausted); + Self::bump_session_transition_version(&mut connection_data.sessions[session_index]); + + // Collect addresses for stream cleanup. + let mut addrs = Vec::new(); + let session_addr = connection_data.sessions[session_index].addr; + if !session_addr.ip().is_unspecified() { + addrs.push(session_addr); + } + for addr in &connection_data.sessions[session_index].active_addrs { + if *addr != session_addr && !addrs.contains(addr) { + addrs.push(*addr); + } + } + + // Remove all streams for these addresses (never preserve on expire). + let mut removed_count = 0; + for addr in &addrs { + if let Some(stream_idx) = + connection_data.streams.iter().position(|stream| stream.addr == *addr && !stream.preserved) + { + if let Some(kind) = connection_data.stream_kinds.remove(&connection_data.streams[stream_idx].uid) { + connection_data.decrement_kind(kind); + } + connection_data.stream_normal_priorities.remove(&connection_data.streams[stream_idx].uid); + connection_data.remove_stream_request_claims(connection_data.streams[stream_idx].uid); + connection_data.streams.swap_remove(stream_idx); + removed_count += 1; + } + } + + // Reset grace if no connections remain. + if connection_data.connections == 0 && connection_data.streams.is_empty() { + connection_data.granted_grace = false; + connection_data.grace_ts = 0; + } + + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); + drop(user_connections); + self.log_divergence_snapshot(divergence_snapshot).await; + + (connection_changed, removed_count) + }; + + if connection_changed { + self.log_active_user().await; + } + debug!("GraceActive expired for session {token} in {username}, released {removed_count} streams"); + } + + /// Terminates the session and all associated streams for a playback. + /// + /// This is the explicit `Terminate` path from the playback state machine: + /// - Removes all streams associated with this session token (never preserves) + /// - Releases the counted lease if held + /// - Sets lifecycle to `Expired` + /// - Clears pending-provider state + /// + /// Unlike `release_unbound_session_reservation`, this terminates regardless of + /// whether streams are currently active, and always removes associated streams. + /// Returns `true` only when the session existed and was removed by this call. + pub async fn terminate_session(&self, username: &str, session_token: &str) -> bool { + let (connection_changed, removed_count, promotions) = { + let mut user_connections = self.write_connections().await; + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return false; + }; + + let Some(session_index) = + connection_data.sessions.iter().position(|session| session.token == session_token) + else { + return false; + }; + + let counted_kind = connection_data.sessions[session_index] + .lifecycle + .is_counted() + .then(|| connection_data.sessions[session_index].connection_kind.unwrap_or(ConnectionKind::Normal)); + let (removed_count, connection_changed) = + connection_data.remove_streams_for_session_and_release_counted(session_token, counted_kind); + + // Expire and remove the session immediately. Unlike `release_unbound_session_reservation` + // which keeps the expired session for TTL-based GC cleanup, terminate_session explicitly + // removes the session from the list so `get_and_update_user_session` returns None. + connection_data.sessions.swap_remove(session_index); + + // Reset grace if no connections remain. + if connection_data.connections == 0 && connection_data.streams.is_empty() { + connection_data.granted_grace = false; + connection_data.grace_ts = 0; + } + + let promotions = Self::collect_promotions_after_capacity_release(connection_data); + + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); + drop(user_connections); + self.log_divergence_snapshot(divergence_snapshot).await; + + (connection_changed, removed_count, promotions) + }; + + if connection_changed { + self.log_active_user().await; + } + for action in promotions { + self.emit_promotion_update(username, action).await; + } + debug!("Terminated session {session_token} for user {username}, released {removed_count} streams"); + true + } + + /// Terminates all sessions associated with a given socket address for a user. + /// + /// This is used when a connection is explicitly kicked — the session should be + /// expired and removed immediately rather than waiting for TTL-based GC cleanup. + /// + /// Removes all sessions whose `addr` or `active_addrs` contains `kick_addr`. + pub async fn terminate_sessions_for_addr(&self, username: &str, kick_addr: &SocketAddr) { + let (connection_changed, removed_count, promotions) = { + let mut user_connections = self.write_connections().await; + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return; + }; + + // Collect tokens of sessions associated with the kicked addr. + let tokens_to_remove: Vec = connection_data + .sessions + .iter() + .filter(|session| session.addr == *kick_addr || session.active_addrs.contains(kick_addr)) + .map(|session| session.token.clone()) + .collect(); + + if tokens_to_remove.is_empty() { + return; + } + + let mut removed_count = 0; + let mut connection_changed = false; + + for token in &tokens_to_remove { + let Some(session_index) = connection_data.sessions.iter().position(|s| s.token == *token) else { + continue; + }; + + let counted_kind = connection_data.sessions[session_index] + .lifecycle + .is_counted() + .then(|| connection_data.sessions[session_index].connection_kind.unwrap_or(ConnectionKind::Normal)); + + let (_, session_connection_changed) = + connection_data.remove_streams_for_session_and_release_counted(token, counted_kind); + connection_changed |= session_connection_changed; + + // Expire and remove the session. + connection_data.sessions.swap_remove(session_index); + removed_count += 1; + } + + // Reset grace if no connections remain. + if connection_data.connections == 0 && connection_data.streams.is_empty() { + connection_data.granted_grace = false; + connection_data.grace_ts = 0; + } + + let promotions = Self::collect_promotions_after_capacity_release(connection_data); + + let divergence_snapshot = Self::collect_divergence_snapshot(connection_data, username); + user_connections.mark_sessions_ended(tokens_to_remove); + drop(user_connections); + self.log_divergence_snapshot(divergence_snapshot).await; + + (connection_changed, removed_count, promotions) + }; + + if connection_changed { + self.log_active_user().await; + } + for action in promotions { + self.emit_promotion_update(username, action).await; + } + debug!("Terminated {removed_count} sessions for user {username} at addr {kick_addr}"); + } + + pub async fn expire_pending_provider( + &self, + username: &str, + token: &str, + expected_version: u64, + wake_source: PendingProviderWakeSource, + ) { + let mut user_connections = self.write_connections().await; + let Some(connection_data) = user_connections.by_key.get_mut(username) else { + return; + }; + let Some(session_index) = connection_data.sessions.iter().position(|session| session.token == token) else { + return; + }; + let pending_version = match &connection_data.sessions[session_index].lifecycle { + PlaybackLifecycle::PendingProvider { data } => data.version, + _ => return, + }; + if pending_version != expected_version { + return; + } + // Capture counted status BEFORE lifecycle changes. + // PendingProvider is not counted (is_counted() = false), so checking here + // captures whether there is a previously-counted lease to release. + let kind_to_release = if connection_data.sessions[session_index].lifecycle.is_counted() { + Some(connection_data.sessions[session_index].connection_kind.unwrap_or(ConnectionKind::Normal)) + } else { + None + }; + let session = &mut connection_data.sessions[session_index]; + Self::clear_session_pending_with_permission(session, UserConnectionPermission::Exhausted, wake_source); + session.expire(); + if let Some(kind) = kind_to_release { + connection_data.decrement_kind(kind); + } + } + + pub async fn adaptive_session_stream_cleanup_addrs( + &self, + username: &str, + session_token: &str, + current_addr: &SocketAddr, + ) -> Vec { + let connections = self.connections.read().await; + let Some(connection_data) = connections.by_key.get(username) else { + return Vec::new(); + }; + + let mut addrs = Vec::new(); + for stream in + connection_data.streams.iter().filter(|stream| stream.session_token.as_deref() == Some(session_token)) + { + if stream.addr != *current_addr && !addrs.contains(&stream.addr) { + addrs.push(stream.addr); + } + } + let current_addr_string = current_addr.to_string(); + let current_ip = strip_port(¤t_addr_string).to_string(); + if let Some(session) = connection_data.sessions.iter().find(|session| session.token == session_token) { + for addr in &session.active_addrs { + let addr_string = addr.to_string(); + let addr_ip = strip_port(&addr_string); + if *addr != *current_addr && addr_ip == current_ip && !addrs.contains(addr) { + addrs.push(*addr); + } + } + } + addrs + } + + pub async fn pending_provider_version(&self, username: &str, token: &str) -> Option { + let user_connections = self.connections.read().await; + let connection_data = user_connections.by_key.get(username)?; + let session = connection_data.sessions.iter().find(|session| session.token == token)?; + match &session.lifecycle { + PlaybackLifecycle::PendingProvider { data } => Some(data.version), + _ => None, + } + } + + pub(super) fn should_preserve_session_stream(stream: &StreamInfo) -> bool { + stream.session_token.is_some() && is_stable_session_stream(stream) + } +} diff --git a/backend/session/src/active_user_manager/tests.rs b/backend/session/src/active_user_manager/tests.rs index d56ae619b..8cd943389 100644 --- a/backend/session/src/active_user_manager/tests.rs +++ b/backend/session/src/active_user_manager/tests.rs @@ -1,6550 +1,18 @@ use super::*; -use crate::EventManager; -use arc_swap::ArcSwapOption; -use futures::FutureExt; -use shared::{ - model::{PlaylistItemType, ProxyType, StreamChannel, StreamInfo, XtreamCluster}, - utils::Internable, + +mod adaptive_streams; +mod admission; +mod expiry_gc; +mod observability; +mod provider_headers; +mod reentry_protection; +mod request_claims; +mod sessions; +mod socket_activity; +mod support; + +use self::support::{ + assert_connection_ownership_invariants, assert_no_real_connection_slots, assert_preserved_session_is_uncounted, + commit_and_preserve_adaptive_session, create_provider_header_session, provider_header_test_manager, + session_identity, test_adaptive_channel, test_channel, test_series_channel, test_user_credentials, }; -use std::{borrow::Cow, collections::HashMap, sync::Arc}; -use tuliprox_core::model::{Config, Fingerprint, ProxyUserCredentials}; - -fn test_channel(virtual_id: u32) -> StreamChannel { - StreamChannel { - target_id: 1, - virtual_id, - provider_id: 1, - input_name: "input".intern(), - item_type: PlaylistItemType::Live, - cluster: XtreamCluster::Live, - group: "group".intern(), - title: "title".intern(), - url: "http://localhost/stream.ts".intern(), - shared: false, - shared_joined_existing: None, - shared_stream_id: None, - technical: None, - epg_channel_id: None, - epg_reference_ts: None, - upstream_user_agent: None, - } -} - -fn test_adaptive_channel(virtual_id: u32) -> StreamChannel { - StreamChannel { - target_id: 1, - virtual_id, - provider_id: 1, - input_name: "input".intern(), - item_type: PlaylistItemType::LiveHls, - cluster: XtreamCluster::Live, - group: "group".intern(), - title: "title".intern(), - url: "http://localhost/stream.ts".intern(), - shared: false, - shared_joined_existing: None, - shared_stream_id: None, - technical: None, - epg_channel_id: None, - epg_reference_ts: None, - upstream_user_agent: None, - } -} - -fn test_series_channel(virtual_id: u32) -> StreamChannel { - StreamChannel { - item_type: PlaylistItemType::Series, - cluster: XtreamCluster::Series, - url: "http://localhost/series/episode.mkv".intern(), - ..test_channel(virtual_id) - } -} - -#[tokio::test] -async fn target_scoped_session_lookup_does_not_use_same_virtual_id_from_other_target() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55499".parse().unwrap(); - let mut user = ProxyUserCredentials::default(); - user.username = "target-scoped-user".to_string(); - let mut target_one = test_channel(42); - target_one.target_id = 1; - let mut target_two = test_channel(42); - target_two.target_id = 2; - - let _ = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-target-one", - virtual_id: 42, - provider: "provider", - stream_url: "http://localhost/target-one.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - let _ = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-target-two", - virtual_id: 42, - provider: "provider", - stream_url: "http://localhost/target-two.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - { - let mut connections = manager.connections.write().await; - assert!(connections.by_key.contains_key(&user.username), "user connection data should exist"); - let Some(data) = connections.by_key.get_mut(&user.username) else { - return; - }; - data.streams.push(StreamInfo::new(shared::model::StreamInfoParams { - uid: 1, - meter_uid: 1, - username: &user.username, - addr: &addr, - client_ip: "127.0.0.1", - provider: "provider".intern(), - stream_channel: target_one, - user_agent: "ua".to_string(), - country_code: None, - session_token: Some("tok-target-one"), - })); - data.streams.push(StreamInfo::new(shared::model::StreamInfoParams { - uid: 2, - meter_uid: 2, - username: &user.username, - addr: &addr, - client_ip: "127.0.0.1", - provider: "provider".intern(), - stream_channel: target_two, - user_agent: "ua".to_string(), - country_code: None, - session_token: Some("tok-target-two"), - })); - } - - let session = manager.find_latest_session_for_target_stream(&user.username, 2, "input", 42, "tok-target-two").await; - assert!(session.is_some(), "target-scoped session should resolve"); - let Some(session) = session else { - return; - }; - assert_eq!(session.token, "tok-target-two"); - assert!(manager - .find_latest_session_for_target_stream(&user.username, 3, "input", 42, "tok-target-two") - .await - .is_none()); - assert!(manager - .find_latest_session_for_target_stream(&user.username, 1, "input", 42, "tok-target-two") - .await - .is_none()); -} - -#[tokio::test] -async fn user_agent_stream_index_is_stable_per_session_and_unique_between_sessions() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - let addr = SocketAddr::from(([127, 0, 0, 1], 55_500)); - let mut user = ProxyUserCredentials::default(); - user.username = "indexed-user".to_string(); - - for token in ["indexed-session-a", "indexed-session-b"] { - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: token, - virtual_id: 42, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - } - - let first = manager.get_or_assign_user_agent_stream_index(&user.username, "indexed-session-a").await; - let repeated = manager.get_or_assign_user_agent_stream_index(&user.username, "indexed-session-a").await; - let second = manager.get_or_assign_user_agent_stream_index(&user.username, "indexed-session-b").await; - - assert!(first.is_some()); - assert_eq!(repeated, first); - assert!(second.is_some()); - assert_ne!(second, first); -} - -/// Session refresh normalizes Expired -> Prepared. -/// When a new request arrives on an expired session, the lifecycle should be -/// reset to Prepared so that full activation evaluation happens. -#[tokio::test] -async fn create_user_session_normalizes_expired_lifecycle() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55400".parse().unwrap(); - let mut user = ProxyUserCredentials::default(); - user.username = "user-lifecycle-refresh".to_string(); - - // Create a session in Expired state directly via session manipulation - { - let mut connections = manager.connections.write().await; - let data = connections.by_key.entry(user.username.clone()).or_insert_with(|| UserConnectionData::new(0, 1, 0)); - data.add_session(UserSession { - token: "tok-refresh-expired".to_string(), - transition_version: 1, - virtual_id: 7001, - provider: "provider-a".intern(), - stream_url: "http://localhost/live.m3u8".intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), - user_agent_stream_index: None, - addr, - socket_bound: false, - active_addrs: vec![addr], - ts: current_time_secs(), - started_at: current_time_secs(), - permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - lifecycle: PlaybackLifecycle::Expired, - ..Default::default() - }); - } - - // Refresh the session via create_user_session - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-refresh-expired", - virtual_id: 7001, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let sessions = manager.connections.read().await; - let data = sessions.by_key.get(&user.username).expect("user data should exist"); - let session = data.sessions.iter().find(|s| s.token == "tok-refresh-expired").expect("session"); - assert_eq!( - session.lifecycle, - PlaybackLifecycle::Prepared, - "Expired session should normalize to Prepared on refresh" - ); -} - -/// Session refresh does NOT normalize `PendingProvider`. -/// A `PendingProvider` session must not be reset — pending state must continue -/// until explicitly resolved via `activate_pending_provider` or `expire_pending_provider`. -#[tokio::test] -async fn create_user_session_does_not_normalize_pending_provider_lifecycle() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55401".parse().unwrap(); - let mut user = ProxyUserCredentials::default(); - user.username = "user-pending-lifecycle".to_string(); - - // Create a session in PendingProvider state - { - let mut connections = manager.connections.write().await; - let data = connections.by_key.entry(user.username.clone()).or_insert_with(|| UserConnectionData::new(0, 1, 0)); - data.add_session(UserSession { - token: "tok-refresh-pending".to_string(), - transition_version: 1, - virtual_id: 7002, - provider: "provider-a".intern(), - stream_url: "http://localhost/live.m3u8".intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), - user_agent_stream_index: None, - addr, - socket_bound: false, - active_addrs: vec![addr], - ts: current_time_secs(), - started_at: current_time_secs(), - permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - lifecycle: PlaybackLifecycle::PendingProvider { - data: PendingProviderState { - reason_code: PendingProviderReason::GraceHold, - created_at: current_time_secs(), - deadline: current_time_secs() + 30, - version: 1, - wake_source: None, - }, - }, - ..Default::default() - }); - } - - // Refresh the session via create_user_session - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-refresh-pending", - virtual_id: 7002, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let sessions = manager.connections.read().await; - let data = sessions.by_key.get(&user.username).expect("user data should exist"); - let session = data.sessions.iter().find(|s| s.token == "tok-refresh-pending").expect("session"); - assert!( - matches!(session.lifecycle, PlaybackLifecycle::PendingProvider { .. }), - "PendingProvider session should NOT be normalized on refresh - pending wait must continue" - ); -} - -#[tokio::test] -async fn update_session_provider_headers_updates_existing_session_and_timestamp() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55402".parse().unwrap_or_else(|_| unreachable!()); - let mut user = ProxyUserCredentials::default(); - user.username = "user-provider-headers".to_string(); - - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-provider-headers", - virtual_id: 7003, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let before = manager - .get_and_update_user_session(&user.username, "tok-provider-headers") - .await - .expect("session should exist"); - let previous_ts = before.ts; - let headers = HashMap::from([(String::from("cookie"), String::from("sid=abc"))]); - - assert!(manager.update_session_provider_headers(&user.username, "tok-provider-headers", &headers).await); - - let after = manager - .get_and_update_user_session(&user.username, "tok-provider-headers") - .await - .expect("session should exist"); - assert_eq!(after.provider_session_headers, headers); - assert!(after.ts >= previous_ts); -} - -#[tokio::test] -async fn update_session_provider_headers_returns_false_for_missing_user_or_token() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - let headers = HashMap::from([(String::from("cookie"), String::from("sid=abc"))]); - - assert!(!manager.update_session_provider_headers("missing-user", "missing-token", &headers).await); - - let addr: SocketAddr = "127.0.0.1:55403".parse().unwrap_or_else(|_| unreachable!()); - let mut user = ProxyUserCredentials::default(); - user.username = "user-missing-token".to_string(); - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-existing", - virtual_id: 7004, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - assert!(!manager.update_session_provider_headers(&user.username, "tok-missing", &headers).await); -} - -#[tokio::test] -async fn create_user_session_clears_provider_headers_when_provider_or_stream_url_changes() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55404".parse().unwrap_or_else(|_| unreachable!()); - let mut user = ProxyUserCredentials::default(); - user.username = "user-provider-header-reset".to_string(); - let headers = HashMap::from([(String::from("cookie"), String::from("sid=abc"))]); - - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-reset", - virtual_id: 7005, - provider: "provider-a", - stream_url: "http://localhost/live-a.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - assert!(manager.update_session_provider_headers(&user.username, "tok-reset", &headers).await); - - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-reset", - virtual_id: 7005, - provider: "provider-b", - stream_url: "http://localhost/live-b.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let session = manager.get_and_update_user_session(&user.username, "tok-reset").await.expect("session should exist"); - assert!(session.provider_session_headers.is_empty()); -} - -/// `terminate_session` expires a session and removes it. -#[tokio::test] -async fn terminate_session_expires_and_removes_session() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55410".parse().unwrap(); - let mut user = ProxyUserCredentials::default(); - user.username = "user-terminate".to_string(); - user.max_connections = 2; - - let token = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-terminate-test", - virtual_id: 8001, - provider: "provider-terminate", - stream_url: "http://localhost/test.ts", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - // Verify session exists. - let before = manager.get_and_update_user_session(&user.username, &token).await; - assert!(before.is_some(), "session should exist before terminate"); - assert_eq!(before.as_ref().unwrap().lifecycle, PlaybackLifecycle::Prepared); - - // Terminate the session. - manager.terminate_session(&user.username, &token).await; - - // Session should be gone. - let after = manager.get_and_update_user_session(&user.username, &token).await; - assert!(after.is_none(), "session should be removed after terminate"); -} - -/// `terminate_session` releases counted lease. -#[tokio::test] -async fn terminate_session_releases_counted_lease() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55411".parse().unwrap(); - let mut user = ProxyUserCredentials::default(); - user.username = "user-terminate-counted".to_string(); - user.max_connections = 2; - - let token = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-terminate-counted", - virtual_id: 8002, - provider: "provider-terminate-counted", - stream_url: "http://localhost/test.ts", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - // Mark the session as counted and active (simulating post-admission state). - { - let mut connections = manager.connections.write().await; - let data = connections.by_key.get_mut(&user.username).unwrap(); - let session = data.sessions.iter_mut().find(|s| s.token == token).unwrap(); - // Simulate counted state by setting lifecycle to Active. - session.lifecycle = PlaybackLifecycle::Active; - data.increment_kind(ConnectionKind::Normal); - } - - // Verify counted before terminate. - { - let before = manager.get_and_update_user_session(&user.username, &token).await.unwrap(); - assert!(before.lifecycle.is_counted(), "session should be counted before terminate"); - } - - // Terminate. - manager.terminate_session(&user.username, &token).await; - - // Session should be gone. - let after = manager.get_and_update_user_session(&user.username, &token).await; - assert!(after.is_none(), "session should be removed after terminate"); -} - -#[tokio::test] -async fn terminate_session_removes_preserved_adaptive_stream() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55412".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-terminate-preserved".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = "user-terminate-preserved".to_string(); - user.max_connections = 1; - - let token = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-terminate-preserved", - virtual_id: 8003, - provider: "provider-terminate-preserved", - stream_url: "http://localhost/test.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - manager.add_connection(&addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 8003, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-terminate-preserved".intern(), - stream_channel: &test_adaptive_channel(8003), - user_agent: Cow::Borrowed("ua"), - session_token: Some(&token), - }) - .await - .expect("adaptive stream should be registered"); - - let released = manager.release_connection(&addr).await; - assert!(released.addr_removed); - assert!(released.removed_streams.is_empty(), "adaptive stream should be preserved first"); - - manager.terminate_session(&user.username, &token).await; - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user data should remain inspectable"); - assert!(connection_data.streams.is_empty(), "terminating a session must remove its preserved adaptive stream"); - assert!(connection_data.sessions.iter().all(|session| session.token != token)); -} - -#[allow(clippy::too_many_lines)] -#[tokio::test] -async fn terminate_session_promotes_soft_stream_after_releasing_capacity() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let normal_addr: SocketAddr = "127.0.0.1:55413".parse().unwrap(); - let soft_addr: SocketAddr = "127.0.0.1:55414".parse().unwrap(); - let soft_addr_two: SocketAddr = "127.0.0.1:55415".parse().unwrap(); - let normal_fp = Fingerprint::new("fp-terminate-normal".to_string(), "127.0.0.1".to_string(), normal_addr); - let soft_fp = Fingerprint::new("fp-terminate-soft".to_string(), "127.0.0.1".to_string(), soft_addr); - let soft_fp_two = Fingerprint::new("fp-terminate-soft-2".to_string(), "127.0.0.1".to_string(), soft_addr_two); - let mut user = ProxyUserCredentials::default(); - user.username = "user-terminate-promote".to_string(); - user.max_connections = 1; - user.soft_connections = 2; - - manager.add_connection(&normal_addr).await; - manager.add_connection(&soft_addr).await; - manager.add_connection(&soft_addr_two).await; - - let normal_token = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-terminate-normal", - virtual_id: 8101, - provider: "provider-normal", - stream_url: "http://localhost/normal.ts", - addr: &normal_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - let soft_token = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-terminate-soft", - virtual_id: 8102, - provider: "provider-soft", - stream_url: "http://localhost/soft.ts", - addr: &soft_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Soft), - socket_bound: false, - }) - .await; - let soft_token_two = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-terminate-soft-2", - virtual_id: 8103, - provider: "provider-soft-2", - stream_url: "http://localhost/soft-2.ts", - addr: &soft_addr_two, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Soft), - socket_bound: false, - }) - .await; - - manager - .update_connection(ActiveUserConnectionParams { - uid: 8101, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &normal_fp, - provider: "provider-normal".intern(), - stream_channel: &test_channel(8101), - user_agent: Cow::Borrowed("ua-normal"), - session_token: Some(&normal_token), - }) - .await - .expect("normal stream should be registered"); - manager - .update_connection(ActiveUserConnectionParams { - uid: 8102, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: ConnectionKind::Soft, - priority: -5, - soft_priority: 9, - fingerprint: &soft_fp, - provider: "provider-soft".intern(), - stream_channel: &test_channel(8102), - user_agent: Cow::Borrowed("ua-soft"), - session_token: Some(&soft_token), - }) - .await - .expect("soft stream should be registered"); - manager - .update_connection(ActiveUserConnectionParams { - uid: 8103, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: ConnectionKind::Soft, - priority: -3, - soft_priority: 9, - fingerprint: &soft_fp_two, - provider: "provider-soft-2".intern(), - stream_channel: &test_channel(8103), - user_agent: Cow::Borrowed("ua-soft-2"), - session_token: Some(&soft_token_two), - }) - .await - .expect("second soft stream should be registered"); - - { - let mut connections = manager.connections.write().await; - let connection_data = connections.by_key.get_mut(&user.username).expect("user data should exist"); - connection_data.soft_connections = 1; - } - - manager.terminate_session(&user.username, &normal_token).await; - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user data should remain inspectable"); - assert_eq!(connection_data.counts.normal, 1); - assert_eq!(connection_data.counts.soft, 1); - let promoted_uid = [8102_u32, 8103_u32] - .into_iter() - .find(|uid| connection_data.stream_kinds.get(uid) == Some(&ConnectionKind::Normal)) - .expect("one soft stream should be promoted to normal"); - let promoted_token = if promoted_uid == 8102 { soft_token.as_str() } else { soft_token_two.as_str() }; - let promoted_session = connection_data - .sessions - .iter() - .find(|session| session.token == promoted_token) - .expect("promoted soft session should remain"); - assert_eq!(promoted_session.connection_kind, Some(ConnectionKind::Normal)); - assert!(matches!(promoted_session.lifecycle, PlaybackLifecycle::Active)); -} - -#[tokio::test] -async fn test_multi_session_same_addr_counts_and_releases_individually() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55001".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key".to_string(), "127.0.0.1".to_string(), addr); - let username = "user1"; - - manager.add_connection(&addr).await; - - let first = manager - .update_connection(ActiveUserConnectionParams { - uid: 1, - meter_uid: 0, - username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(1001), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-1"), - }) - .await; - assert!(first.is_some()); - assert_eq!(manager.user_connections(username).await, 1); - assert_eq!(manager.connection_permission(username, 1, 0).await, UserConnectionPermission::Exhausted); - - let second = manager - .update_connection(ActiveUserConnectionParams { - uid: 2, - meter_uid: 0, - username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-b".intern(), - stream_channel: &test_channel(1002), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-2"), - }) - .await; - assert!(second.is_some()); - assert_eq!(manager.user_connections(username).await, 2); - - assert!(manager.release_stream(&addr).await.is_some()); - assert_eq!(manager.user_connections(username).await, 1); - - assert!(manager.release_stream(&addr).await.is_some()); - assert_eq!(manager.user_connections(username).await, 0); -} - -#[tokio::test] -async fn mark_pending_provider_tracks_metadata_on_session() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55021".parse().unwrap_or_else(|_| unreachable!()); - let mut user = ProxyUserCredentials::default(); - user.username = "pending-user".to_string(); - - let _ = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-pending", - virtual_id: 1001, - provider: "provider-a", - stream_url: "http://provider/live/1001.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let _ = - manager.mark_pending_provider(&user.username, "tok-pending", PendingProviderReason::GraceHold, 12_345).await; - - let session = - manager.get_and_update_user_session(&user.username, "tok-pending").await.expect("session should exist"); - let PlaybackLifecycle::PendingProvider { data: pending } = &session.lifecycle else { - panic!("pending provider should be tracked") - }; - assert!(matches!(pending.reason_code, PendingProviderReason::GraceHold)); - assert_eq!(pending.deadline, 12_345); - assert!(pending.created_at > 0); - assert_eq!(pending.version, 1); - assert!(pending.wake_source.is_none()); -} - -#[tokio::test] -async fn activate_pending_provider_clears_pending_metadata() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55022".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = Fingerprint::new("fp-pending".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = "pending-activate".to_string(); - - let _ = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-pending-activate", - virtual_id: 1002, - provider: "provider-a", - stream_url: "http://provider/live/1002.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - let _ = manager - .mark_pending_provider( - &user.username, - "tok-pending-activate", - PendingProviderReason::GraceHold, - current_time_secs().saturating_add(30), - ) - .await; - - let _ = manager - .update_connection(ActiveUserConnectionParams { - uid: 12, - meter_uid: 0, - username: &user.username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(1002), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-pending-activate"), - }) - .await; - manager - .activate_pending_provider(&user.username, "tok-pending-activate", 1, PendingProviderWakeSource::Activated) - .await; - - let session = manager - .get_and_update_user_session(&user.username, "tok-pending-activate") - .await - .expect("session should exist"); - assert!(session.lifecycle.is_counted()); - assert!( - !matches!(session.lifecycle, PlaybackLifecycle::PendingProvider { .. }), - "explicit pending resolution must clear pending provider state" - ); -} - -#[tokio::test] -async fn activate_pending_provider_ignores_stale_version_after_replacement() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55023".parse().unwrap_or_else(|_| unreachable!()); - let mut user = ProxyUserCredentials::default(); - user.username = "pending-stale".to_string(); - - let _ = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-pending-stale", - virtual_id: 1003, - provider: "provider-a", - stream_url: "http://provider/live/1003.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let first_version = manager - .mark_pending_provider(&user.username, "tok-pending-stale", PendingProviderReason::GraceHold, 5_000) - .await - .expect("first pending version should be created"); - let second_version = manager - .mark_pending_provider(&user.username, "tok-pending-stale", PendingProviderReason::GraceHold, 6_000) - .await - .expect("second pending version should replace the first"); - assert!(second_version > first_version); - - manager - .activate_pending_provider( - &user.username, - "tok-pending-stale", - first_version, - PendingProviderWakeSource::CapacityNotify, - ) - .await; - - let session = manager - .get_and_update_user_session(&user.username, "tok-pending-stale") - .await - .expect("session should still exist"); - let PlaybackLifecycle::PendingProvider { data: pending_data } = &session.lifecycle else { - panic!("session should still be in PendingProvider after stale wakeup") - }; - assert_eq!(pending_data.version, second_version); - assert!(pending_data.wake_source.is_none()); - assert_eq!(session.permission, UserConnectionPermission::GracePeriod); -} - -#[tokio::test] -async fn expire_pending_provider_marks_session_exhausted() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55024".parse().unwrap_or_else(|_| unreachable!()); - let mut user = ProxyUserCredentials::default(); - user.username = "pending-expire".to_string(); - - let _ = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-pending-expire", - virtual_id: 1004, - provider: "provider-a", - stream_url: "http://provider/live/1004.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let version = manager - .mark_pending_provider(&user.username, "tok-pending-expire", PendingProviderReason::GraceHold, 6_000) - .await - .expect("pending version should be created"); - - manager - .expire_pending_provider(&user.username, "tok-pending-expire", version, PendingProviderWakeSource::Timeout) - .await; - - let session = manager - .get_and_update_user_session(&user.username, "tok-pending-expire") - .await - .expect("session should still exist"); - assert_eq!(session.permission, UserConnectionPermission::Exhausted); - assert!(!matches!(session.lifecycle, PlaybackLifecycle::PendingProvider { .. })); - assert!(!session.lifecycle.is_counted()); -} - -#[tokio::test] -async fn expire_pending_provider_releases_counted_slot_for_pending_session() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55025".parse().unwrap_or_else(|_| unreachable!()); - let mut user = ProxyUserCredentials::default(); - user.username = "pending-expire-counted".to_string(); - user.max_connections = 1; - - let _ = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-pending-expire-counted", - virtual_id: 1005, - provider: "provider-a", - stream_url: "http://provider/live/1005.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - { - let mut connections = manager.connections.write().await; - let connection_data = - connections.by_key.get_mut(&user.username).expect("session should have created connection data"); - connection_data.increment_kind(ConnectionKind::Normal); - let session = connection_data - .sessions - .iter_mut() - .find(|session| session.token == "tok-pending-expire-counted") - .expect("session should exist"); - // Simulate a previously-counted session transitioning to PendingProvider. - // Set lifecycle to Active (is_counted() = true). The kind count is already - // incremented above via connection_data.increment_kind(). - session.lifecycle = PlaybackLifecycle::Active; - } - - assert_eq!(manager.user_connections(&user.username).await, 1); - - let version = manager - .mark_pending_provider(&user.username, "tok-pending-expire-counted", PendingProviderReason::GraceHold, 6_500) - .await - .expect("pending version should be created"); - - manager - .expire_pending_provider( - &user.username, - "tok-pending-expire-counted", - version, - PendingProviderWakeSource::Timeout, - ) - .await; - - let session = manager - .get_and_update_user_session(&user.username, "tok-pending-expire-counted") - .await - .expect("session should still exist"); - assert_eq!(session.permission, UserConnectionPermission::Exhausted); - assert!(!matches!(session.lifecycle, PlaybackLifecycle::PendingProvider { .. })); - assert!(!session.lifecycle.is_counted()); - assert_eq!(manager.user_connections(&user.username).await, 0); -} - -/// `terminate_sessions_for_addr` expires all sessions at a given addr and releases counted leases. -#[tokio::test] -async fn terminate_sessions_for_addr_expires_all_sessions_at_addr() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr_kick: SocketAddr = "127.0.0.1:55420".parse().unwrap(); - let addr_keep: SocketAddr = "127.0.0.1:55421".parse().unwrap(); - let mut user = ProxyUserCredentials::default(); - user.username = "user-kick-addr".to_string(); - user.max_connections = 4; - - // Create session at kicked addr. - let tok_kick = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-kick", - virtual_id: 1, - provider: "provider-a", - stream_url: "http://provider/live/1.m3u8", - addr: &addr_kick, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - // Create session at kept addr. - let tok_keep = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-keep", - virtual_id: 2, - provider: "provider-b", - stream_url: "http://provider/live/2.m3u8", - addr: &addr_keep, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - // Mark both sessions as counted and active. - { - let mut connections = manager.connections.write().await; - let data = connections.by_key.get_mut(&user.username).unwrap(); - for session in &mut data.sessions { - // Simulate counted state by setting lifecycle to Active. - session.lifecycle = PlaybackLifecycle::Active; - } - data.increment_kind(ConnectionKind::Normal); - data.increment_kind(ConnectionKind::Normal); - } - - assert_eq!(manager.user_connections(&user.username).await, 2); - - // Kick the addr — should terminate only the sessions at that addr. - manager.terminate_sessions_for_addr(&user.username, &addr_kick).await; - - // Session at kicked addr should be gone. - assert!( - manager.get_and_update_user_session(&user.username, &tok_kick).await.is_none(), - "kicked session should be removed" - ); - - // Session at kept addr should remain. - let kept = - manager.get_and_update_user_session(&user.username, &tok_keep).await.expect("kept session should still exist"); - assert_eq!(kept.token, tok_keep); - assert_eq!(kept.addr, addr_keep); - - // Connection count should drop by 1. - assert_eq!(manager.user_connections(&user.username).await, 1); -} - -#[tokio::test] -async fn test_grant_grace_succeeds_at_and_above_limit_without_prior_grace() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let at_limit_addr: SocketAddr = "127.0.0.1:55011".parse().unwrap(); - let at_limit_fingerprint = Fingerprint::new("fp-limit".to_string(), "127.0.0.1".to_string(), at_limit_addr); - let over_limit_addr: SocketAddr = "127.0.0.1:55012".parse().unwrap(); - let over_limit_fingerprint = Fingerprint::new("fp-over".to_string(), "127.0.0.1".to_string(), over_limit_addr); - - manager.add_connection(&at_limit_addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 10, - meter_uid: 0, - username: "at-limit", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &at_limit_fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(1010), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-limit"), - }) - .await; - - assert!(manager.grant_grace("at-limit").await); - - manager.add_connection(&over_limit_addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 11, - meter_uid: 0, - username: "over-limit", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &over_limit_fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(1011), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-over-1"), - }) - .await; - manager.add_connection(&"127.0.0.1:55013".parse().unwrap()).await; - let second_fingerprint = - Fingerprint::new("fp-over-2".to_string(), "127.0.0.1".to_string(), "127.0.0.1:55013".parse().unwrap()); - manager - .update_connection(ActiveUserConnectionParams { - uid: 12, - meter_uid: 0, - username: "over-limit", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &second_fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(1012), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-over-2"), - }) - .await; - - assert!(manager.grant_grace("over-limit").await); -} - -fn test_user_credentials(username: &str, max_connections: u32, soft_connections: u16) -> ProxyUserCredentials { - ProxyUserCredentials { - username: username.to_string(), - password: "test".to_string(), - token: None, - proxy: ProxyType::default(), - server: None, - epg_timeshift: None, - epg_request_timeshift: None, - created_at: None, - exp_date: None, - max_connections, - status: None, - output_clusters: shared::model::ClusterFlags::all(), - ui_enabled: true, - comment: None, - priority: 0, - soft_connections, - soft_priority: 0, - t_is_api_user: false, - network_access: None, - plan: None, - filter: None, - raw_output_clusters: None, - raw_max_connections: 0, - raw_soft_connections: 0, - raw_proxy: Some(ProxyType::default()), - t_filter: None, - t_has_unresolved_plan: false, - t_has_invalid_filter: false, - } -} - -fn record_owned_slot(counts: &mut UserConnectionCounts, kind: ConnectionKind) { - match kind { - ConnectionKind::Normal => counts.normal += 1, - ConnectionKind::Soft => counts.soft += 1, - } -} - -fn assert_connection_ownership_invariants(connection_data: &UserConnectionData) { - let mut owned_slots = UserConnectionCounts::default(); - - // A counted session owns one logical slot. Active streams tied to that - // session validate its kind below, but do not add another slot. - for session in connection_data.sessions.iter().filter(|session| session.lifecycle.is_counted()) { - record_owned_slot(&mut owned_slots, session.connection_kind.unwrap_or(ConnectionKind::Normal)); - } - - for stream in &connection_data.streams { - let stream_kind = connection_data.stream_kinds.get(&stream.uid); - if stream.preserved { - assert!(stream_kind.is_none(), "preserved stream {} must not own a real connection slot", stream.uid); - continue; - } - - let stream_kind = stream_kind.expect("every active stream must have a connection kind"); - let counted_session = stream.session_token.as_deref().and_then(|session_token| { - connection_data - .sessions - .iter() - .find(|session| session.token == session_token && session.lifecycle.is_counted()) - }); - if let Some(session) = counted_session { - assert_eq!( - *stream_kind, - session.connection_kind.unwrap_or(ConnectionKind::Normal), - "a counted session and its active stream must use the same slot kind" - ); - } else { - record_owned_slot(&mut owned_slots, *stream_kind); - } - } - - for uid in connection_data.stream_kinds.keys() { - assert!( - connection_data.streams.iter().any(|stream| stream.uid == *uid && !stream.preserved), - "stream kind for uid {uid} must belong to an active stream" - ); - } - - assert_eq!(connection_data.counts.normal, owned_slots.normal, "normal slots must match their owners"); - assert_eq!(connection_data.counts.soft, owned_slots.soft, "soft slots must match their owners"); - assert_eq!( - connection_data.connections, - connection_data.counts.normal + u32::from(connection_data.counts.soft), - "aggregate connection count must equal the normal and soft counters" - ); -} - -fn assert_no_real_connection_slots(connection_data: &UserConnectionData) { - assert_eq!(connection_data.connections, 0); - assert_eq!(connection_data.counts.normal, 0); - assert_eq!(connection_data.counts.soft, 0); - assert_connection_ownership_invariants(connection_data); -} - -fn assert_active_stream_kind(connection_data: &UserConnectionData, stream_uid: u32, expected_kind: ConnectionKind) { - assert!( - connection_data.streams.iter().any(|stream| stream.uid == stream_uid && !stream.preserved), - "stream {stream_uid} must remain active" - ); - assert_eq!(connection_data.stream_kinds.get(&stream_uid), Some(&expected_kind)); -} - -fn assert_single_normal_stream_slot(connection_data: &UserConnectionData, stream_uid: u32) { - assert_eq!(connection_data.connections, 1); - assert_eq!(connection_data.counts.normal, 1); - assert_eq!(connection_data.counts.soft, 0); - assert_active_stream_kind(connection_data, stream_uid, ConnectionKind::Normal); - assert_connection_ownership_invariants(connection_data); -} - -fn assert_preserved_session_is_uncounted(connection_data: &UserConnectionData, session_token: &str, stream_uid: u32) { - let session = connection_data - .sessions - .iter() - .find(|session| session.token == session_token) - .expect("preserved session must exist"); - assert_eq!(session.lifecycle, PlaybackLifecycle::Preserved); - - let stream = - connection_data.streams.iter().find(|stream| stream.uid == stream_uid).expect("preserved stream must exist"); - assert!(stream.preserved); - assert_eq!(stream.session_token.as_deref(), Some(session_token)); - assert!(!connection_data.stream_kinds.contains_key(&stream_uid)); -} - -async fn commit_and_preserve_adaptive_session( - manager: &ActiveUserManager, - user: &ProxyUserCredentials, - session_token: &str, - stream_uid: u32, - addr: SocketAddr, - connection_kind: ConnectionKind, -) { - let fingerprint = Fingerprint::new(format!("fp-preserved-{stream_uid}"), addr.ip().to_string(), addr); - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user, - session_token, - virtual_id: stream_uid, - provider: "provider-a", - stream_url: "http://localhost/live-preserved.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(connection_kind), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: stream_uid, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind, - priority: user.priority, - soft_priority: user.soft_priority, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(stream_uid), - user_agent: Cow::Borrowed("ua"), - session_token: Some(session_token), - }) - .await - .expect("adaptive session stream should bind"); - - { - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - let session = connection_data - .sessions - .iter() - .find(|session| session.token == session_token) - .expect("committed session must exist"); - assert_eq!(session.lifecycle, PlaybackLifecycle::Active); - assert_eq!(connection_data.stream_kinds.get(&stream_uid), Some(&connection_kind)); - assert_connection_ownership_invariants(connection_data); - } - - assert!(manager.release_stream(&addr).await.is_none(), "adaptive stream should be preserved"); - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_preserved_session_is_uncounted(connection_data, session_token, stream_uid); - assert_connection_ownership_invariants(connection_data); -} - -#[tokio::test] -async fn eviction_candidates_ignore_ambiguous_socket_addrs() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let shared_addr: SocketAddr = "127.0.0.1:55031".parse().unwrap(); - let unique_addr: SocketAddr = "127.0.0.1:55032".parse().unwrap(); - let shared_fp = Fingerprint::new("fp-shared".to_string(), "127.0.0.1".to_string(), shared_addr); - let unique_fp = Fingerprint::new("fp-unique".to_string(), "127.0.0.1".to_string(), unique_addr); - - manager.add_connection(&shared_addr).await; - manager.add_connection(&unique_addr).await; - - // Create sessions first so update_connection can mark them as counted. - let user = test_user_credentials("same-user", 3, 0); - for (token, addr, channel_id) in - [("tok-31", shared_addr, 1031u32), ("tok-32", shared_addr, 1032), ("tok-33", unique_addr, 1033)] - { - manager - .create_user_session(crate::CreateUserSessionParams { - user: &user, - session_token: token, - virtual_id: channel_id, - provider: "provider-a", - stream_url: "", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - } - - // update_connection marks the session as counted. - for (uid, token, fp, channel_id) in [(31, "tok-31", &shared_fp, 1031u32), (32, "tok-32", &shared_fp, 1032)] { - manager - .update_connection(ActiveUserConnectionParams { - uid, - meter_uid: 0, - username: "same-user", - max_connections: 3, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: fp, - provider: "provider-a".intern(), - stream_channel: &test_channel(channel_id), - user_agent: Cow::Borrowed("ua"), - session_token: Some(token), - }) - .await; - } - - manager - .update_connection(ActiveUserConnectionParams { - uid: 33, - meter_uid: 0, - username: "same-user", - max_connections: 3, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &unique_fp, - provider: "provider-a".intern(), - stream_channel: &test_channel(1033), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-33"), - }) - .await; - - let candidates = manager.get_eviction_candidates("same-user", "127.0.0.1").await; - assert_eq!(candidates.len(), 1); - assert_eq!(candidates[0].addr, unique_addr); -} - -#[tokio::test] -async fn eviction_candidates_include_charged_stream_with_uncounted_session() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55034".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-switch".to_string(), "10.41.41.170".to_string(), addr); - let username = "switch-user"; - manager.add_connection(&addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 34, - meter_uid: 0, - username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(1034), - user_agent: Cow::Borrowed("ua"), - session_token: Some("missing-session"), - }) - .await; - - assert_eq!(manager.user_connections(username).await, 1); - assert_eq!(manager.connection_permission(username, 1, 0).await, UserConnectionPermission::Exhausted); - - let candidates = manager.get_eviction_candidates(username, &fingerprint.client_ip).await; - assert_eq!(candidates.len(), 1); - assert_eq!(candidates[0].addr, addr); - assert_eq!(candidates[0].client_ip, fingerprint.client_ip); - - manager.release_connection_as_kicked(&addr).await; - assert_eq!(manager.connection_permission(username, 1, 0).await, UserConnectionPermission::Allowed); -} - -#[tokio::test] -async fn eviction_candidates_include_other_ips_for_user_wide_rules() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let first_addr: SocketAddr = "127.0.0.1:55041".parse().unwrap(); - let second_addr: SocketAddr = "127.0.0.1:55042".parse().unwrap(); - let first_fp = Fingerprint::new("fp-user-wide-1".to_string(), "10.0.0.1".to_string(), first_addr); - let second_fp = Fingerprint::new("fp-user-wide-2".to_string(), "10.0.0.2".to_string(), second_addr); - - manager.add_connection(&first_addr).await; - manager.add_connection(&second_addr).await; - - let user = test_user_credentials("same-user", 2, 0); - for (token, addr, channel_id) in [("tok-41", first_addr, 1041u32), ("tok-42", second_addr, 1042)] { - manager - .create_user_session(crate::CreateUserSessionParams { - user: &user, - session_token: token, - virtual_id: channel_id, - provider: "provider-a", - stream_url: "", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - } - - manager - .update_connection(ActiveUserConnectionParams { - uid: 41, - meter_uid: 0, - username: "same-user", - max_connections: 2, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &first_fp, - provider: "provider-a".intern(), - stream_channel: &test_channel(1041), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-41"), - }) - .await; - - manager - .update_connection(ActiveUserConnectionParams { - uid: 42, - meter_uid: 0, - username: "same-user", - max_connections: 2, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &second_fp, - provider: "provider-a".intern(), - stream_channel: &test_channel(1042), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-42"), - }) - .await; - - let candidates = manager.get_eviction_candidates("same-user", "10.0.0.1").await; - assert_eq!(candidates.len(), 2); - assert!(candidates.iter().any(|candidate| candidate.addr == first_addr)); - assert!(candidates.iter().any(|candidate| candidate.addr == second_addr)); -} - -#[tokio::test] -async fn eviction_candidates_include_preserved_adaptive_streams() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55043".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-preserved".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("same-user"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-preserved", - virtual_id: 1043, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 43, - meter_uid: 0, - username: "same-user", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(1043) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-preserved"), - }) - .await; - - let released = manager.release_connection(&addr).await; - assert!(released.addr_removed); - assert!(released.removed_streams.is_empty(), "adaptive stream should stay logically active"); - - let candidates = manager.get_eviction_candidates("same-user", "127.0.0.1").await; - assert_eq!(candidates.len(), 1, "preserved adaptive streams must remain evictable"); - assert_eq!(candidates[0].addr, addr); -} - -#[tokio::test] -async fn test_kicked_release_does_not_preserve_adaptive_stream() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55014".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-adaptive".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user-adaptive"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-adaptive", - virtual_id: 2014, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 14, - meter_uid: 0, - username: &user.username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(2014), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-adaptive"), - }) - .await; - - let removed = manager.release_connection_as_kicked(&addr).await; - assert!(removed.addr_removed); - assert_eq!(removed.removed_streams.len(), 1); - assert!(manager.active_streams().await.is_empty()); -} - -#[tokio::test] -async fn kicked_release_removes_preserved_adaptive_stream_without_socket_registration() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55017".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-preserved-kick".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user-preserved-kick"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-preserved-kick", - virtual_id: 2017, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 17, - meter_uid: 0, - username: &user.username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(2017), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-preserved-kick"), - }) - .await; - - let released = manager.release_connection(&addr).await; - assert!(released.addr_removed); - assert!(released.removed_streams.is_empty()); - assert!(manager.active_streams().await.is_empty()); - - let kicked = manager.release_connection_as_kicked(&addr).await; - assert!(kicked.addr_removed); - assert_eq!(kicked.removed_streams.len(), 1); - assert!(manager.active_streams().await.is_empty()); -} - -#[tokio::test] -async fn test_kicked_release_invalidates_removed_session_tokens() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let kicked_addr: SocketAddr = "127.0.0.1:55015".parse().unwrap(); - let survivor_addr: SocketAddr = "127.0.0.1:55016".parse().unwrap(); - let kicked_fingerprint = Fingerprint::new("fp-kicked".to_string(), "127.0.0.1".to_string(), kicked_addr); - let survivor_fingerprint = Fingerprint::new("fp-survivor".to_string(), "127.0.0.1".to_string(), survivor_addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("kicked-user"); - user.max_connections = 1; - - manager.add_connection(&kicked_addr).await; - manager.add_connection(&survivor_addr).await; - - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-kicked", - virtual_id: 2015, - provider: "provider-a", - stream_url: "http://localhost/live-1.ts", - addr: &kicked_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: true, - }) - .await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-survivor", - virtual_id: 2016, - provider: "provider-a", - stream_url: "http://localhost/live-2.ts", - addr: &survivor_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: true, - }) - .await; - - manager - .update_connection(ActiveUserConnectionParams { - uid: 15, - meter_uid: 0, - username: &user.username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &kicked_fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(2015), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-kicked"), - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 16, - meter_uid: 0, - username: &user.username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &survivor_fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(2016), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-survivor"), - }) - .await; - - let removed = manager.release_connection_as_kicked(&kicked_addr).await; - assert!(removed.addr_removed); - assert_eq!(removed.removed_streams.len(), 1); - assert_eq!( - manager.connection_admission_for_session(&user.username, 1, 0, "tok-kicked").await.permission, - UserConnectionPermission::Exhausted - ); - assert_eq!( - manager.connection_admission_for_session(&user.username, 1, 0, "tok-survivor").await.permission, - UserConnectionPermission::Allowed - ); -} - -#[tokio::test] -async fn test_grace_at_limit_remains_active_until_connections_drop_below_limit() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55017".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-grace".to_string(), "127.0.0.1".to_string(), addr); - - manager.add_connection(&addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 17, - meter_uid: 0, - username: "grace-at-limit", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(2017), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-grace"), - }) - .await; - - assert!(manager.grant_grace("grace-at-limit").await); - assert_eq!( - manager.connection_admission("grace-at-limit", 1, 0).await.permission, - UserConnectionPermission::Exhausted - ); - assert!(!manager.grant_grace("grace-at-limit").await); -} - -#[tokio::test] -async fn test_same_session_token_on_new_addr_reuses_logical_connection() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let first_addr: SocketAddr = "127.0.0.1:55021".parse().unwrap(); - let second_addr: SocketAddr = "127.0.0.1:55022".parse().unwrap(); - let first = Fingerprint::new("fp-key-1".to_string(), "127.0.0.1".to_string(), first_addr); - let second = Fingerprint::new("fp-key-2".to_string(), "127.0.0.1".to_string(), second_addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&first_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-hls", - virtual_id: 2001, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &first_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 0, - meter_uid: 0, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &first, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(2001), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-hls"), - }) - .await; - - assert_eq!( - manager.connection_permission_for_session("user1", 1, 0, "tok-hls").await, - UserConnectionPermission::Allowed - ); - - manager.add_connection(&second_addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 0, - meter_uid: 0, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &second, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(2001), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-hls"), - }) - .await; - - assert_eq!(manager.user_connections("user1").await, 1); - - let streams = manager.active_streams().await; - assert_eq!(streams.len(), 1); - assert_eq!(streams[0].addr, second_addr); - assert_eq!(streams[0].session_token.as_deref(), Some("tok-hls")); -} - -#[tokio::test] -async fn adaptive_session_stream_cleanup_addrs_excludes_manifest_addr_and_current_addr() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let manifest_addr: SocketAddr = "127.0.0.1:55091".parse().unwrap(); - let first_segment_addr: SocketAddr = "10.41.41.89:55092".parse().unwrap(); - let next_segment_addr: SocketAddr = "10.41.41.89:55093".parse().unwrap(); - let first_segment = Fingerprint::new("fp-segment-1".to_string(), "10.41.41.89".to_string(), first_segment_addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&manifest_addr).await; - manager.add_connection(&first_segment_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-hls-cleanup", - virtual_id: 2002, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &manifest_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 0, - meter_uid: 0, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &first_segment, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(2002) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-hls-cleanup"), - }) - .await; - - assert_eq!( - manager.adaptive_session_stream_cleanup_addrs("user1", "tok-hls-cleanup", &next_segment_addr).await, - vec![first_segment_addr] - ); -} - -#[tokio::test] -async fn adaptive_session_stream_cleanup_addrs_falls_back_to_same_ip_session_addrs() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let manifest_addr: SocketAddr = "127.0.0.1:55101".parse().unwrap(); - let first_segment_addr: SocketAddr = "10.41.41.89:55102".parse().unwrap(); - let next_segment_addr: SocketAddr = "10.41.41.89:55103".parse().unwrap(); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user2"); - user.max_connections = 1; - - manager.add_connection(&manifest_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-hls-cleanup-fallback", - virtual_id: 2003, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &manifest_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager.update_session_addr("user2", "tok-hls-cleanup-fallback", &first_segment_addr).await; - manager.update_session_addr("user2", "tok-hls-cleanup-fallback", &next_segment_addr).await; - - assert_eq!( - manager.adaptive_session_stream_cleanup_addrs("user2", "tok-hls-cleanup-fallback", &next_segment_addr).await, - vec![first_segment_addr] - ); -} - -#[tokio::test] -async fn recently_evicted_session_guard_survives_ttl_while_protected_addr_is_still_active() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let evicted_addr: SocketAddr = "127.0.0.1:55111".parse().unwrap(); - let protected_addr: SocketAddr = "127.0.0.1:55112".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-guard-session".to_string(), "127.0.0.1".to_string(), evicted_addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("guard-user"); - user.max_connections = 1; - - manager.add_connection(&evicted_addr).await; - manager.add_connection(&protected_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-guard-session", - virtual_id: 2018, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &evicted_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 18, - meter_uid: 0, - username: &user.username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(2018), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-guard-session"), - }) - .await; - - manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, Duration::from_secs(1)).await; - { - let mut connections = manager.connections.write().await; - if let Some(registration) = connections.key_by_addr.get_mut(&protected_addr) { - registration.add_user(&user.username, current_time_secs()); - } - let protection = connections - .recently_evicted_sessions - .get_mut("tok-guard-session") - .expect("recent eviction guard should exist"); - protection.expires_at = Instant::now().checked_sub(Duration::from_secs(1)).unwrap_or_else(Instant::now); - } - - assert_eq!(manager.recently_evicted_session_protected_addr("tok-guard-session").await, Some(protected_addr)); -} - -#[tokio::test] -async fn recently_evicted_vod_uses_session_reentry_guard() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let evicted_addr: SocketAddr = "127.0.0.1:55113".parse().unwrap(); - let protected_addr: SocketAddr = "127.0.0.1:55114".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-vod-guard".to_string(), "127.0.0.1".to_string(), evicted_addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("vod-guard-user"); - user.max_connections = 1; - let mut channel = test_channel(2019); - channel.item_type = PlaylistItemType::Video; - channel.cluster = XtreamCluster::Video; - channel.url = "http://localhost/movie.mkv".intern(); - - manager.add_connection(&evicted_addr).await; - manager.add_connection(&protected_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-guard-vod", - virtual_id: channel.virtual_id, - provider: "provider-a", - stream_url: channel.url.as_ref(), - addr: &evicted_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 19, - meter_uid: 0, - username: &user.username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &channel, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-guard-vod"), - }) - .await; - - manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, Duration::from_secs(10)).await; - - assert_eq!(manager.recently_evicted_session_protected_addr("tok-guard-vod").await, Some(protected_addr)); - let connections = manager.connections.read().await; - assert!( - connections.recent_socket_reentry_guards.is_empty(), - "provider-affine VOD must not be guarded by transient socket identity" - ); -} - -#[tokio::test] -async fn provider_affine_stream_without_session_token_uses_socket_reentry_fallback() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let evicted_addr: SocketAddr = "127.0.0.1:55115".parse().unwrap(); - let protected_addr: SocketAddr = "127.0.0.1:55116".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-vod-no-token".to_string(), "127.0.0.1".to_string(), evicted_addr); - let mut channel = test_channel(2020); - channel.item_type = PlaylistItemType::Video; - channel.cluster = XtreamCluster::Video; - channel.url = "http://localhost/movie-no-token.mkv".intern(); - - manager.add_connection(&evicted_addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 20, - meter_uid: 0, - username: "vod-no-token-user", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &channel, - user_agent: Cow::Borrowed("ua"), - session_token: None, - }) - .await; - - manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, Duration::from_secs(10)).await; - - assert_eq!( - manager - .recent_socket_reentry_protected_addr( - "vod-no-token-user", - "127.0.0.1", - shared::model::VirtualId::new(channel.virtual_id) - ) - .await, - Some(protected_addr) - ); -} - -#[tokio::test] -async fn socket_reentry_guard_expires_at_ttl() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let evicted_addr: SocketAddr = "127.0.0.1:55117".parse().unwrap(); - let protected_addr: SocketAddr = "127.0.0.1:55118".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-socket-ttl".to_string(), "127.0.0.1".to_string(), evicted_addr); - let channel = test_channel(2021); - - manager.add_connection(&evicted_addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 21, - meter_uid: 0, - username: "socket-ttl-user", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &channel, - user_agent: Cow::Borrowed("ua"), - session_token: None, - }) - .await; - - manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, Duration::from_secs(10)).await; - let virtual_id = shared::model::VirtualId::new(channel.virtual_id); - assert_eq!( - manager.recent_socket_reentry_protected_addr("socket-ttl-user", "127.0.0.1", virtual_id).await, - Some(protected_addr) - ); - - // Expire only the guard while the protected address stays registered: the TTL is - // authoritative and must not fall back to the still-present socket registration. - { - let mut connections = manager.connections.write().await; - let key = create_socket_reentry_guard_key("socket-ttl-user", "127.0.0.1", virtual_id); - let protection = - connections.recent_socket_reentry_guards.get_mut(&key).expect("socket reentry guard should exist"); - protection.expires_at = Instant::now().checked_sub(Duration::from_secs(1)).unwrap_or_else(Instant::now); - } - assert_eq!(manager.recent_socket_reentry_protected_addr("socket-ttl-user", "127.0.0.1", virtual_id).await, None); -} - -#[tokio::test] -async fn test_reused_logical_stream_refreshes_normal_priority() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55023".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key-2a".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-prio", - virtual_id: 2002, - provider: "provider-a", - stream_url: "http://localhost/live-prio.ts", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Soft), - socket_bound: true, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 201, - meter_uid: 0, - username: "user1", - max_connections: 1, - soft_connections: 1, - connection_kind: ConnectionKind::Soft, - priority: 8, - soft_priority: 8, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(2002), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-prio"), - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 201, - meter_uid: 0, - username: "user1", - max_connections: 1, - soft_connections: 1, - connection_kind: ConnectionKind::Soft, - priority: -7, - soft_priority: 8, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(2002), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-prio"), - }) - .await; - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get("user1").unwrap(); - assert_eq!(connection_data.stream_normal_priorities.get(&201), Some(&-7)); -} - -#[tokio::test] -async fn test_same_session_token_refreshes_meter_metadata_on_reuse() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55031".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key-3".to_string(), "127.0.0.1".to_string(), addr); - - manager.add_connection(&addr).await; - let first = manager - .update_connection(ActiveUserConnectionParams { - uid: 11, - meter_uid: 101, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(3001), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-meter"), - }) - .await - .expect("initial stream should register"); - assert_eq!(first.uid, 11); - assert_eq!(first.meter_uid, 101); - - let second = manager - .update_connection(ActiveUserConnectionParams { - uid: 22, - meter_uid: 202, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-b".intern(), - stream_channel: &test_adaptive_channel(3002), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-meter"), - }) - .await - .expect("reused stream should register"); - - assert_eq!(second.uid, 11, "logical stream identity should stay stable on session reuse"); - assert_eq!(second.meter_uid, 202, "reused stream must refresh its meter mapping"); - - let streams = manager.active_streams().await; - assert_eq!(streams.len(), 1); - assert_eq!(streams[0].uid, 11); - assert_eq!(streams[0].meter_uid, 202); - assert_eq!(streams[0].provider.as_ref(), "provider-b"); - assert_eq!(streams[0].channel.virtual_id, 3002); -} - -#[tokio::test] -async fn socket_bound_live_streams_with_colliding_token_are_tracked_separately() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let Some(addr) = "127.0.0.1:55032".parse::().ok() else { - return; - }; - let fingerprint = Fingerprint::new("fp-key-colliding".to_string(), "127.0.0.1".to_string(), addr); - - manager.add_connection(&addr).await; - let first = manager - .update_connection(ActiveUserConnectionParams { - uid: 31, - meter_uid: 301, - username: "user1", - max_connections: 0, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(3003), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-live-colliding"), - }) - .await; - let second = manager - .update_connection(ActiveUserConnectionParams { - uid: 32, - meter_uid: 302, - username: "user1", - max_connections: 0, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-b".intern(), - stream_channel: &test_channel(3003), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-live-colliding"), - }) - .await; - - assert!(first.is_some()); - assert!(second.is_some()); - - let streams = manager.active_streams().await; - assert_eq!(streams.len(), 2); - assert!(streams.iter().any(|stream| stream.uid == 31)); - assert!(streams.iter().any(|stream| stream.uid == 32)); -} - -#[tokio::test] -async fn unlimited_user_can_open_same_and_different_live_streams_from_same_ip() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let username = "unlimited-same-ip"; - let client_ip = "10.9.0.1"; - let addrs = [ - "10.9.0.1:55101".parse::().unwrap(), - "10.9.0.1:55102".parse::().unwrap(), - "10.9.0.1:55103".parse::().unwrap(), - ]; - let fingerprints = [ - Fingerprint::new("fp-unlimited-1".to_string(), client_ip.to_string(), addrs[0]), - Fingerprint::new("fp-unlimited-2".to_string(), client_ip.to_string(), addrs[1]), - Fingerprint::new("fp-unlimited-3".to_string(), client_ip.to_string(), addrs[2]), - ]; - - for addr in addrs { - manager.add_connection(&addr).await; - } - - for (idx, (fingerprint, virtual_id)) in fingerprints.iter().zip([4100, 4100, 4101]).enumerate() { - let token = format!("tok-unlimited-{idx}"); - manager - .update_connection(ActiveUserConnectionParams { - uid: 410 + u32::try_from(idx).unwrap_or_default(), - meter_uid: 0, - username, - max_connections: 0, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(virtual_id), - user_agent: Cow::Borrowed("ua"), - session_token: Some(&token), - }) - .await - .expect("unlimited stream should register"); - } - - assert_eq!(manager.user_connections(username).await, 3); - assert_eq!(manager.active_streams().await.len(), 3); - assert_eq!(manager.connection_admission(username, 0, 0).await.permission, UserConnectionPermission::Allowed); - assert_eq!( - manager.connection_admission_for_session(username, 0, 0, "tok-unlimited-new").await.permission, - UserConnectionPermission::Allowed - ); -} - -#[tokio::test] -async fn affine_request_claims_keep_playback_until_oldest_body_finishes() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let Some(addr) = "127.0.0.1:55033".parse::().ok() else { - return; - }; - let fingerprint = Fingerprint::new("fp-key-shared-addr".to_string(), "127.0.0.1".to_string(), addr); - - manager.add_connection(&addr).await; - for uid in [41, 42] { - manager - .update_connection(ActiveUserConnectionParams { - uid, - meter_uid: uid + 300, - username: "user1", - max_connections: 0, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(3004), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-live-shared-addr"), - }) - .await; - } - - assert_eq!(manager.active_streams().await.len(), 1); - assert!(manager.release_stream_by_uid(&addr, 42).await.is_none()); - - let streams = manager.active_streams().await; - assert_eq!(streams.len(), 1); - assert_eq!(streams[0].uid, 41); - - let removed = manager.release_stream_by_uid(&addr, 41).await; - assert!(removed.as_ref().is_some_and(|stream| stream.uid == 41)); - assert!(manager.active_streams().await.is_empty()); -} - -#[tokio::test] -async fn affine_request_claims_keep_playback_until_newest_body_finishes() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - let addr: SocketAddr = "127.0.0.1:55037".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-affine-cleanup-order".to_string(), "127.0.0.1".to_string(), addr); - - manager.add_connection(&addr).await; - for uid in [45, 46] { - manager - .update_connection(ActiveUserConnectionParams { - uid, - meter_uid: 0, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(3005), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-affine-cleanup-order"), - }) - .await - .expect("affine request should register"); - } - - assert!(manager.release_stream_by_uid(&addr, 45).await.is_none()); - assert_eq!(manager.active_users_and_connections().await, (1, 1)); - assert_eq!(manager.active_streams().await.len(), 1); - - let removed = manager.release_stream_by_uid(&addr, 46).await; - assert!(removed.as_ref().is_some_and(|stream| stream.uid == 45)); - assert_eq!(manager.active_users_and_connections().await, (0, 0)); - assert!(manager.active_streams().await.is_empty()); -} - -/// PR1 regression: two bodies with distinct request UIDs that map to the same -/// display stream must both clean up correctly. The current body path passes the -/// *display* UID (`stream_info.uid`) to cleanup, not the request UID. This test -/// asserts that the cleanup is request-affine — releasing the display UID twice -/// must drain both claims, and the second release must find the stream. -#[tokio::test] -async fn pr1_two_bodies_same_display_uid_both_cleanups_must_succeed() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55035".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-pr1-display-vs-request".to_string(), "127.0.0.1".to_string(), addr); - manager.add_connection(&addr).await; - - // Request 41 creates the display stream. - let first = manager - .update_connection(ActiveUserConnectionParams { - uid: 41, - meter_uid: 301, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(3006), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-pr1"), - }) - .await - .expect("first body should register"); - - // Request 42 reuses the same playback (same session token, same channel). - let second = manager - .update_connection(ActiveUserConnectionParams { - uid: 42, - meter_uid: 302, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(3006), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-pr1"), - }) - .await - .expect("second body should register"); - - // Both return the same display UID. - assert_eq!(first.uid, second.uid, "display UID must be stable across requests"); - let display_uid = first.uid; - - // Fixed behavior: the body passes its own request UID to cleanup, not the display UID. - // First body (request 41) finishes → should NOT remove the stream (request 42 still active). - let removed_first = manager.release_stream_by_uid(&addr, 41).await; - assert!(removed_first.is_none(), "first body cleanup must keep stream alive for second body"); - - assert_eq!(manager.active_streams().await.len(), 1, "stream must survive first body cleanup"); - - // Second body (request 42) finishes → should remove the stream (no more claims). - let removed_second = manager.release_stream_by_uid(&addr, 42).await; - assert!(removed_second.is_some(), "second body cleanup must find and remove the stream"); - assert_eq!(removed_second.unwrap().uid, display_uid); - - assert!(manager.active_streams().await.is_empty(), "no streams left after all bodies finish"); - assert_eq!(manager.active_users_and_connections().await, (0, 0)); -} - -#[tokio::test] -async fn release_stream_by_uid_finds_original_user_after_shared_addr_owner_changes() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55034".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-cross-user-stream".to_string(), "127.0.0.1".to_string(), addr); - manager.add_connection(&addr).await; - - for (uid, username) in [(43, "user-a"), (44, "user-b")] { - manager - .update_connection(ActiveUserConnectionParams { - uid, - meter_uid: 0, - username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_series_channel(3005), - user_agent: Cow::Borrowed("ua"), - session_token: None, - }) - .await - .expect("direct Series stream should register"); - } - - assert_eq!(manager.active_users_and_connections().await, (2, 2)); - assert_eq!(manager.active_streams().await.len(), 2); - - let removed = manager.release_stream_by_uid(&addr, 43).await; - assert!(removed.as_ref().is_some_and(|stream| stream.uid == 43)); - assert_eq!(manager.active_users_and_connections().await, (1, 1)); - let streams = manager.active_streams().await; - assert_eq!(streams.len(), 1); - assert_eq!(streams[0].uid, 44); - - assert!(manager.release_stream_by_uid(&addr, 44).await.is_some()); - assert_eq!(manager.active_users_and_connections().await, (0, 0)); - assert!(manager.active_streams().await.is_empty()); -} - -#[tokio::test] -async fn release_connection_cleans_every_user_stream_for_reused_addr_only() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let reused_addr: SocketAddr = "127.0.0.1:55035".parse().unwrap(); - let unrelated_addr: SocketAddr = "127.0.0.1:55036".parse().unwrap(); - let reused_fingerprint = Fingerprint::new("fp-cross-user-socket".to_string(), "127.0.0.1".to_string(), reused_addr); - let unrelated_fingerprint = - Fingerprint::new("fp-unrelated-socket".to_string(), "127.0.0.1".to_string(), unrelated_addr); - manager.add_connection(&reused_addr).await; - manager.add_connection(&unrelated_addr).await; - - for (uid, username, fingerprint) in [ - (45, "user-a", &reused_fingerprint), - (46, "user-b", &reused_fingerprint), - (47, "user-a", &unrelated_fingerprint), - ] { - manager - .update_connection(ActiveUserConnectionParams { - uid, - meter_uid: 0, - username, - max_connections: 2, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_series_channel(3006 + uid), - user_agent: Cow::Borrowed("ua"), - session_token: None, - }) - .await - .expect("direct Series stream should register"); - } - - let released = manager.release_connection(&reused_addr).await; - let mut removed_uids = released.removed_streams.iter().map(|stream| stream.uid).collect::>(); - removed_uids.sort_unstable(); - assert!(released.addr_removed); - assert_eq!(removed_uids, vec![45, 46]); - assert_eq!(manager.active_users_and_connections().await, (1, 1)); - let streams = manager.active_streams().await; - assert_eq!(streams.len(), 1); - assert_eq!(streams[0].uid, 47); - - manager.release_connection(&unrelated_addr).await; - assert_eq!(manager.active_users_and_connections().await, (0, 0)); - assert!(manager.active_streams().await.is_empty()); -} - -#[tokio::test] -async fn connection_counts_are_broadcast_when_active_user_logging_is_disabled() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let mut events = event_manager.get_event_channel(); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55037".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-count-events".to_string(), "127.0.0.1".to_string(), addr); - manager.add_connection(&addr).await; - manager.release_connection(&addr).await; - assert!( - tokio::time::timeout(Duration::from_millis(50), events.recv()).await.is_err(), - "closing an unowned socket must not broadcast unchanged connection counts" - ); - manager.add_connection(&addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 48, - meter_uid: 0, - username: "event-user", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_series_channel(3048), - user_agent: Cow::Borrowed("ua"), - session_token: None, - }) - .await - .expect("direct Series stream should register"); - - assert_eq!( - tokio::time::timeout(Duration::from_secs(1), events.recv()) - .await - .expect("connection count update should be broadcast") - .expect("event channel should remain open"), - EventMessage::ActiveUser(ActiveUserConnectionChange::Connections(1, 1)) - ); - - manager - .update_connection(ActiveUserConnectionParams { - uid: 48, - meter_uid: 0, - username: "event-user", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_series_channel(3048), - user_agent: Cow::Borrowed("ua"), - session_token: None, - }) - .await - .expect("same direct Series stream should be reused"); - assert!( - tokio::time::timeout(Duration::from_millis(50), events.recv()).await.is_err(), - "unchanged connection counts must not broadcast another full snapshot" - ); - - manager.release_stream_by_uid(&addr, 48).await.expect("direct Series stream should release"); - assert_eq!( - tokio::time::timeout(Duration::from_secs(1), events.recv()) - .await - .expect("released count update should be broadcast") - .expect("event channel should remain open"), - EventMessage::ActiveUser(ActiveUserConnectionChange::Connections(0, 0)) - ); -} - -#[tokio::test] -async fn test_adaptive_session_release_connection_preserves_logical_stream_and_start_time() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55041".parse().unwrap(); - let next_addr: SocketAddr = "127.0.0.1:55042".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key-4".to_string(), "127.0.0.1".to_string(), addr); - let next_fingerprint = Fingerprint::new("fp-key-5".to_string(), "127.0.0.1".to_string(), next_addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-hls", - virtual_id: 4001, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - let first = manager - .update_connection(ActiveUserConnectionParams { - uid: 44, - meter_uid: 144, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(4001) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-hls"), - }) - .await - .expect("initial adaptive session should register"); - - let released = manager.release_connection(&addr).await; - assert!(released.addr_removed); - assert!(released.removed_streams.is_empty(), "adaptive session should remain logically active"); - assert_eq!(manager.user_connections("user1").await, 0); - assert_eq!(manager.active_users_and_connections().await, (0, 0)); - assert!(manager.active_streams().await.is_empty()); - - let connections = manager.connections.read().await; - let preserved_stream = connections - .by_key - .get("user1") - .and_then(|data| data.streams.iter().find(|stream| stream.uid == 44)) - .expect("preserved adaptive stream should stay internally tracked"); - assert_eq!(preserved_stream.ts, first.ts); - assert!(preserved_stream.preserved); - drop(connections); - - manager.add_connection(&next_addr).await; - let second = manager - .update_connection(ActiveUserConnectionParams { - uid: 55, - meter_uid: 155, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &next_fingerprint, - provider: "provider-b".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveDash, ..test_channel(4002) }, - user_agent: Cow::Borrowed("ua-2"), - session_token: Some("tok-hls"), - }) - .await - .expect("adaptive session should reuse logical stream"); - - assert_eq!(second.uid, 44); - assert_eq!(second.ts, first.ts, "adaptive session duration must stay session-based"); - assert_eq!(second.addr, next_addr); - assert_eq!(second.meter_uid, 155); - assert_eq!(manager.user_connections("user1").await, 1); - - let streams = manager.active_streams().await; - assert_eq!(streams.len(), 1); - assert!(!streams[0].preserved); -} - -#[tokio::test] -async fn test_release_stream_ignores_preserved_adaptive_entry() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55051".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key-6".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-hls", - virtual_id: 5001, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 66, - meter_uid: 166, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(5001) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-hls"), - }) - .await; - - let released = manager.release_connection(&addr).await; - assert!(released.addr_removed); - assert!(released.removed_streams.is_empty()); - assert!(manager.release_stream(&addr).await.is_none()); -} - -#[tokio::test] -async fn test_preserved_adaptive_stream_is_pruned_after_session_ttl() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55061".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key-7".to_string(), "127.0.0.1".to_string(), addr); - - manager.add_connection(&addr).await; - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-expire", - virtual_id: 6001, - provider: "provider-a", - stream_url: "http://localhost/hls.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 77, - meter_uid: 177, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(6001) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-expire"), - }) - .await; - let released = manager.release_connection(&addr).await; - assert!(released.addr_removed); - - { - let mut connections = manager.connections.write().await; - let connection_data = connections.by_key.get_mut("user1").unwrap(); - let session = connection_data.sessions.iter_mut().find(|session| session.token == "tok-expire").unwrap(); - session.ts = session.ts.saturating_sub(default_hls_session_ttl_secs() + 1); - } - if let Some(gc_ts) = &manager.gc_ts { - gc_ts.store(current_time_secs().saturating_sub(USER_GC_TTL + 1), Ordering::Release); - } - - manager - .process_due_adaptive_expiry_entries(current_time_secs().saturating_add(default_hls_session_ttl_secs() + 1)) - .await; - assert!(manager.active_streams().await.is_empty()); -} - -#[tokio::test] -async fn test_due_adaptive_expiry_removal_promotes_soft_stream() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let normal_addr: SocketAddr = "127.0.0.1:55062".parse().unwrap(); - let soft_addr: SocketAddr = "127.0.0.1:55063".parse().unwrap(); - let normal_fp = Fingerprint::new("fp-key-7a".to_string(), "127.0.0.1".to_string(), normal_addr); - let soft_fp = Fingerprint::new("fp-key-7b".to_string(), "127.0.0.1".to_string(), soft_addr); - - manager.add_connection(&normal_addr).await; - manager.add_connection(&soft_addr).await; - - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - user.soft_connections = 1; - - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-expire-normal", - virtual_id: 6002, - provider: "provider-a", - stream_url: "http://localhost/hls-normal.m3u8", - addr: &normal_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 78, - meter_uid: 178, - username: "user1", - max_connections: 1, - soft_connections: 1, - connection_kind: ConnectionKind::Normal, - priority: -1, - soft_priority: 9, - fingerprint: &normal_fp, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(6002) }, - user_agent: Cow::Borrowed("ua-normal"), - session_token: Some("tok-expire-normal"), - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 79, - meter_uid: 179, - username: "user1", - max_connections: 1, - soft_connections: 1, - connection_kind: ConnectionKind::Soft, - priority: -5, - soft_priority: 9, - fingerprint: &soft_fp, - provider: "provider-a".intern(), - stream_channel: &test_channel(6003), - user_agent: Cow::Borrowed("ua-soft"), - session_token: None, - }) - .await; - - let released = manager.release_connection(&normal_addr).await; - assert!(released.addr_removed); - - { - let mut connections = manager.connections.write().await; - let connection_data = connections.by_key.get_mut("user1").unwrap(); - let session = connection_data.sessions.iter_mut().find(|session| session.token == "tok-expire-normal").unwrap(); - session.ts = session.ts.saturating_sub(default_hls_session_ttl_secs() + 1); - } - - manager - .process_due_adaptive_expiry_entries(current_time_secs().saturating_add(default_hls_session_ttl_secs() + 1)) - .await; - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get("user1").unwrap(); - assert_eq!(connection_data.stream_kinds.get(&79), Some(&ConnectionKind::Soft)); - assert!(!connection_data.stream_normal_priorities.contains_key(&78)); -} - -#[tokio::test] -async fn test_repeated_preserve_for_same_adaptive_session_keeps_single_current_expiry_index() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr_a: SocketAddr = "127.0.0.1:55071".parse().unwrap(); - let addr_b: SocketAddr = "127.0.0.1:55072".parse().unwrap(); - let fp_a = Fingerprint::new("fp-key-a".to_string(), "127.0.0.1".to_string(), addr_a); - let fp_b = Fingerprint::new("fp-key-b".to_string(), "127.0.0.1".to_string(), addr_b); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&addr_a).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-reuse", - virtual_id: 7001, - provider: "provider-a", - stream_url: "http://localhost/live-a.m3u8", - addr: &addr_a, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 88, - meter_uid: 188, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fp_a, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(7001) }, - user_agent: Cow::Borrowed("ua-a"), - session_token: Some("tok-reuse"), - }) - .await; - let released = manager.release_connection(&addr_a).await; - assert!(released.addr_removed); - - manager.add_connection(&addr_b).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 99, - meter_uid: 199, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fp_b, - provider: "provider-b".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveDash, ..test_channel(7002) }, - user_agent: Cow::Borrowed("ua-b"), - session_token: Some("tok-reuse"), - }) - .await; - let released = manager.release_connection(&addr_b).await; - assert!(released.addr_removed); - - let expiry_index = manager.adaptive_expiry_index.lock().await; - assert_eq!(expiry_index.len(), 1); - assert!(expiry_index.contains_key(&AdaptiveExpiryKey { - username: String::from("user1"), - session_token: String::from("tok-reuse"), - uid: 88, - })); -} - -#[tokio::test] -async fn test_release_stream_preserved_path_emits_connection_update_event() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - let mut events = event_manager.get_event_channel(); - - let addr: SocketAddr = "127.0.0.1:55081".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key-8".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-event", - virtual_id: 8001, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 111, - meter_uid: 211, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8001) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-event"), - }) - .await; - let _ = events.try_recv(); - - let released = manager.release_stream(&addr).await; - assert!(released.is_none(), "adaptive stream should remain logically preserved"); - - let event = events.try_recv().expect("preserved release should emit an ActiveUser event"); - assert!(matches!(event, EventMessage::ActiveUser(_))); -} - -#[tokio::test] -async fn test_release_stream_without_session_removes_adaptive_stream_instead_of_preserving() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55082".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key-9".to_string(), "127.0.0.1".to_string(), addr); - - manager.add_connection(&addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 122, - meter_uid: 222, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8002) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("missing-session"), - }) - .await; - - let released = manager.release_stream(&addr).await; - assert!(released.is_some(), "stream without schedulable expiry must be removed"); - assert!(manager.active_streams().await.is_empty()); -} - -#[tokio::test] -async fn test_due_adaptive_expiry_reschedules_when_session_timestamp_changes() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55083".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key-10".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-reschedule", - virtual_id: 8003, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 133, - meter_uid: 233, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8003) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-reschedule"), - }) - .await; - let released = manager.release_connection(&addr).await; - assert!(released.addr_removed); - - let key = - AdaptiveExpiryKey { username: String::from("user1"), session_token: String::from("tok-reschedule"), uid: 133 }; - let old_expires_at = { - let expiry_index = manager.adaptive_expiry_index.lock().await; - *expiry_index.get(&key).unwrap() - }; - - { - let mut connections = manager.connections.write().await; - let session = connections - .by_key - .get_mut("user1") - .unwrap() - .sessions - .iter_mut() - .find(|session| session.token == "tok-reschedule") - .unwrap(); - session.ts = session.ts.saturating_add(30); - } - - manager.process_due_adaptive_expiry_entries(old_expires_at).await; - - let new_expires_at = { - let expiry_index = manager.adaptive_expiry_index.lock().await; - *expiry_index.get(&key).unwrap() - }; - assert!(new_expires_at > old_expires_at); - assert!(manager.active_streams().await.is_empty()); -} - -#[tokio::test] -async fn test_due_adaptive_expiry_removes_stale_index_when_preserved_stream_missing() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55085".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key-11a".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-stale", - virtual_id: 8004, - provider: "provider-a", - stream_url: "http://localhost/stale.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 134, - meter_uid: 234, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8004) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-stale"), - }) - .await; - let released = manager.release_connection(&addr).await; - assert!(released.addr_removed); - - let key = AdaptiveExpiryKey { username: String::from("user1"), session_token: String::from("tok-stale"), uid: 134 }; - let old_expires_at = { - let expiry_index = manager.adaptive_expiry_index.lock().await; - *expiry_index.get(&key).unwrap() - }; - - { - let mut connections = manager.connections.write().await; - let connection_data = connections.by_key.get_mut("user1").unwrap(); - connection_data.streams.clear(); - } - - manager.process_due_adaptive_expiry_entries(old_expires_at).await; - - let expiry_index = manager.adaptive_expiry_index.lock().await; - assert!(!expiry_index.contains_key(&key)); -} - -#[tokio::test] -async fn test_due_adaptive_expiry_does_not_block_on_full_cleanup_channel() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55084".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key-11".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-full-channel", - virtual_id: 8004, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 144, - meter_uid: 244, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8004) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-full-channel"), - }) - .await; - let released = manager.release_connection(&addr).await; - assert!(released.addr_removed); - - { - let mut connections = manager.connections.write().await; - let session = connections - .by_key - .get_mut("user1") - .unwrap() - .sessions - .iter_mut() - .find(|session| session.token == "tok-full-channel") - .unwrap(); - session.ts = session.ts.saturating_sub(default_hls_session_ttl_secs() + 1); - } - - let (cleanup_tx, mut cleanup_rx) = mpsc::channel(1); - cleanup_tx.send(CleanupEvent::ReleaseConnection { addr }).await.expect("prefill cleanup channel"); - manager.set_cleanup_sender(cleanup_tx); - - let process_result = tokio::time::timeout( - Duration::from_millis(100), - manager.process_due_adaptive_expiry_entries( - current_time_secs().saturating_add(default_hls_session_ttl_secs() + 1), - ), - ) - .await; - - assert!(process_result.is_ok(), "adaptive expiry processing must not await while holding locks"); - - let queued_event = cleanup_rx.try_recv().expect("prefilled cleanup event should remain queued"); - assert!(matches!(queued_event, CleanupEvent::ReleaseConnection { .. })); - assert!(manager.active_streams().await.is_empty()); -} - -#[tokio::test] -async fn test_preserved_adaptive_stream_reconnect_across_day_sets_previous_session_id() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55085".parse().unwrap(); - let next_addr: SocketAddr = "127.0.0.1:55086".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-rollover-a".to_string(), "127.0.0.1".to_string(), addr); - let next_fingerprint = Fingerprint::new("fp-rollover-b".to_string(), "127.0.0.1".to_string(), next_addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-rollover", - virtual_id: 8005, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - let first = manager - .update_connection(ActiveUserConnectionParams { - uid: 145, - meter_uid: 245, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8005) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-rollover"), - }) - .await - .expect("initial adaptive session should register"); - - let released = manager.release_connection(&addr).await; - assert!(released.addr_removed); - - let forced_old_ts = { - let mut connections = manager.connections.write().await; - let stream = connections - .by_key - .get_mut("user1") - .unwrap() - .streams - .iter_mut() - .find(|stream| stream.session_token.as_deref() == Some("tok-rollover")) - .unwrap(); - stream.ts = stream.ts.saturating_sub(86_400); - stream.ts - }; - - manager.add_connection(&next_addr).await; - let second = manager - .update_connection(ActiveUserConnectionParams { - uid: 146, - meter_uid: 246, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &next_fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveDash, ..test_channel(8005) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-rollover"), - }) - .await - .expect("adaptive session should reconnect"); - - assert_eq!(second.previous_session_id, Some((forced_old_ts << 32) | u64::from(first.uid))); - assert!(second.ts > forced_old_ts); - assert_eq!(utc_day_from_secs(second.ts), utc_day_from_secs(current_time_secs())); -} - -#[tokio::test] -async fn stale_anonymous_socket_registration_is_pruned_by_gc() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let stale_addr: SocketAddr = "127.0.0.1:55011".parse().unwrap(); - let fresh_addr: SocketAddr = "127.0.0.1:55012".parse().unwrap(); - - manager.add_connection(&stale_addr).await; - { - let mut connections = manager.connections.write().await; - let registration = connections.key_by_addr.get_mut(&stale_addr).expect("socket registration should exist"); - registration.ts = registration.ts.saturating_sub(ANON_SOCKET_TTL + 1); - } - - if let Some(gc_ts) = &manager.gc_ts { - gc_ts.store(current_time_secs().saturating_sub(USER_GC_TTL + 1), Ordering::Release); - } - - manager.add_connection(&fresh_addr).await; - - let connections = manager.connections.read().await; - assert!(!connections.key_by_addr.contains_key(&stale_addr)); - assert!(connections.key_by_addr.contains_key(&fresh_addr)); -} - -#[tokio::test] -async fn named_socket_registration_exposes_expiry_deadline() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let stale_addr: SocketAddr = "127.0.0.1:55021".parse().unwrap(); - let fresh_addr: SocketAddr = "127.0.0.1:55022".parse().unwrap(); - let stale_fp = Fingerprint::new("fp-stale".to_string(), "127.0.0.1".to_string(), stale_addr); - let fresh_fp = Fingerprint::new("fp-fresh".to_string(), "127.0.0.1".to_string(), fresh_addr); - let mut stale_user = ProxyUserCredentials::default(); - stale_user.username = "user1".to_string(); - stale_user.max_connections = 1; - let mut fresh_user = ProxyUserCredentials::default(); - fresh_user.username = "user2".to_string(); - fresh_user.max_connections = 1; - - manager.add_connection(&stale_addr).await; - manager.add_connection(&fresh_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &stale_user, - session_token: "tok-stale-deadline", - virtual_id: 9201, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &stale_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .create_user_session(CreateUserSessionParams { - user: &fresh_user, - session_token: "tok-fresh-deadline", - virtual_id: 9202, - provider: "provider-b", - stream_url: "http://localhost/live.m3u8", - addr: &fresh_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 201, - meter_uid: 301, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &stale_fp, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(9201), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-stale-deadline"), - }) - .await - .expect("stale stream should register"); - manager - .update_connection(ActiveUserConnectionParams { - uid: 202, - meter_uid: 302, - username: "user2", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fresh_fp, - provider: "provider-b".intern(), - stream_channel: &test_adaptive_channel(9202), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-fresh-deadline"), - }) - .await - .expect("fresh stream should register"); - - { - let mut connections = manager.connections.write().await; - let stale_registration = connections.key_by_addr.get_mut(&stale_addr).expect("stale registration should exist"); - stale_registration.ts = stale_registration.ts.saturating_sub(DEFAULT_ACTIVE_SOCKET_TTL_SECS + 1); - } - - let stale_deadline = - manager.socket_expiry_deadline(&stale_addr).await.expect("stale named socket should have an expiry deadline"); - let fresh_deadline = - manager.socket_expiry_deadline(&fresh_addr).await.expect("fresh named socket should have an expiry deadline"); - assert!(stale_deadline < fresh_deadline); -} - -#[tokio::test] -async fn touch_http_activity_refreshes_session_and_registration_without_stream() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55024".parse().unwrap(); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-http-touch", - virtual_id: 9302, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let previous_ts = { - let mut connections = manager.connections.write().await; - let previous_ts = { - let registration = connections.key_by_addr.get_mut(&addr).expect("registration should exist"); - registration.ts = registration.ts.saturating_sub(DEFAULT_ACTIVE_SOCKET_TTL_SECS + 5); - registration.ts - }; - let connection_data = connections.by_key.get_mut("user1").expect("user should exist"); - connection_data.sessions[0].ts = - connection_data.sessions[0].ts.saturating_sub(DEFAULT_ACTIVE_SOCKET_TTL_SECS + 5); - previous_ts - }; - - manager.touch_http_activity("user1", "tok-http-touch", &addr).await; - - let connections = manager.connections.read().await; - let registration = connections.key_by_addr.get(&addr).expect("registration should still exist"); - let connection_data = connections.by_key.get("user1").expect("user should still exist"); - assert!(registration.ts > previous_ts); - assert!(connection_data.sessions[0].ts >= registration.ts); -} - -#[tokio::test] -async fn touch_http_activity_does_not_reset_stream_started_at_ts() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr1: SocketAddr = "127.0.0.1:55030".parse().unwrap(); - let addr2: SocketAddr = "127.0.0.1:55031".parse().unwrap(); - let fingerprint = Fingerprint::new("fp".to_string(), "127.0.0.1".to_string(), addr1); - let mut user = ProxyUserCredentials::default(); - user.username = "user-touch-ts".to_string(); - - manager.add_connection(&addr1).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-hls-ts", - virtual_id: 7777, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr1, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - // Simulate first HLS segment: creates the stream entry with ts = now - manager - .update_connection(ActiveUserConnectionParams { - uid: 601, - meter_uid: 701, - username: "user-touch-ts", - max_connections: 0, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(7777), - user_agent: Cow::Borrowed("player/1.0"), - session_token: Some("tok-hls-ts"), - }) - .await - .expect("stream should be created"); - - // Record the original stream start timestamp - let original_ts = { - let connections = manager.connections.read().await; - connections - .by_key - .get("user-touch-ts") - .and_then(|data| data.streams.iter().find(|s| s.session_token.as_deref() == Some("tok-hls-ts"))) - .map(|s| s.ts) - .expect("stream should exist") - }; - - // Simulate manifest re-fetch (touch_http_activity called with a new addr) - manager.touch_http_activity("user-touch-ts", "tok-hls-ts", &addr2).await; - - // stream.ts must NOT have been reset — it represents session start time shown as Duration - let connections = manager.connections.read().await; - let stream = connections - .by_key - .get("user-touch-ts") - .and_then(|data| data.streams.iter().find(|s| s.session_token.as_deref() == Some("tok-hls-ts"))) - .expect("stream should still exist"); - assert_eq!(stream.ts, original_ts, "touch_http_activity must not reset the stream start timestamp"); - // Lightweight manifest activity must not move the active stream socket. - assert_eq!(stream.addr, addr1, "touch_http_activity must not replace the active stream addr"); -} - -#[tokio::test] -async fn touch_http_activity_does_not_migrate_adaptive_stream_to_manifest_addr_on_close() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - let mut events = event_manager.get_event_channel(); - - let segment_addr: SocketAddr = "127.0.0.1:55032".parse().unwrap(); - let manifest_addr: SocketAddr = "127.0.0.1:55033".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-hls-segment".to_string(), "127.0.0.1".to_string(), segment_addr); - let mut user = ProxyUserCredentials::default(); - user.username = "user-hls-manifest-touch".to_string(); - user.max_connections = 1; - - manager.add_connection(&segment_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-hls-manifest-touch", - virtual_id: 7788, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &segment_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 602, - meter_uid: 702, - username: &user.username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(7788), - user_agent: Cow::Borrowed("player/1.0"), - session_token: Some("tok-hls-manifest-touch"), - }) - .await - .expect("stream should be created"); - - manager.touch_http_activity(&user.username, "tok-hls-manifest-touch", &manifest_addr).await; - - let released = manager.release_connection(&segment_addr).await; - assert!(released.addr_removed); - assert!(released.removed_streams.is_empty(), "adaptive close should preserve without history removal"); - assert_eq!(manager.user_connections(&user.username).await, 0); - assert!( - manager.active_streams().await.is_empty(), - "preserved rows stay out of active_streams (use panel_streams for StatusCheck)" - ); - let panel = manager.panel_streams().await; - assert_eq!(panel.len(), 1, "preserved adaptive/catchup session rows stay in panel snapshots"); - assert!(panel[0].preserved); - assert_eq!(panel[0].session_token.as_deref(), Some("tok-hls-manifest-touch")); - - let connections = manager.connections.read().await; - let data = connections.by_key.get(&user.username).expect("user should remain for preserved session"); - let stream = data - .streams - .iter() - .find(|stream| stream.session_token.as_deref() == Some("tok-hls-manifest-touch")) - .expect("preserved stream should remain internally tracked"); - assert!(stream.preserved); - assert_eq!(stream.addr, segment_addr, "closed segment must not migrate to manifest addr"); - assert!(!data.sessions[0].active_addrs.contains(&manifest_addr)); - drop(connections); - - let mut saw_preserved_update = false; - while let Ok(event) = events.try_recv() { - if matches!(event, EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream)) if stream.addr == segment_addr && stream.preserved) - { - saw_preserved_update = true; - } - } - assert!(saw_preserved_update, "preserving a stream must notify the frontend so adaptive TTL cleanup can hide it"); -} - -#[tokio::test] -async fn clear_unbound_session_addr_prunes_manifest_addr_while_stream_is_active_elsewhere() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let segment_addr: SocketAddr = "127.0.0.1:55034".parse().unwrap(); - let manifest_addr: SocketAddr = "127.0.0.1:55035".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-hls-segment-2".to_string(), "127.0.0.1".to_string(), segment_addr); - let mut user = ProxyUserCredentials::default(); - user.username = "user-hls-manifest-clear".to_string(); - user.max_connections = 1; - - manager.add_connection(&segment_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-hls-manifest-clear", - virtual_id: 7789, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &segment_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 603, - meter_uid: 703, - username: &user.username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(7789), - user_agent: Cow::Borrowed("player/1.0"), - session_token: Some("tok-hls-manifest-clear"), - }) - .await - .expect("stream should be created"); - - manager.add_connection(&manifest_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-hls-manifest-clear", - virtual_id: 7789, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &manifest_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - manager.clear_unbound_session_addr(&user.username, "tok-hls-manifest-clear", &manifest_addr).await; - - let connections = manager.connections.read().await; - assert!(!connections.key_by_addr.contains_key(&manifest_addr)); - let data = connections.by_key.get(&user.username).expect("user should exist"); - assert_eq!(data.streams[0].addr, segment_addr); - assert!(!data.sessions[0].active_addrs.contains(&manifest_addr)); -} - -#[tokio::test] -async fn clear_unbound_session_addr_prunes_touch_only_manifest_addr() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let segment_addr: SocketAddr = "127.0.0.1:55036".parse().unwrap(); - let manifest_addr: SocketAddr = "127.0.0.1:55037".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-hls-segment-3".to_string(), "127.0.0.1".to_string(), segment_addr); - let mut user = ProxyUserCredentials::default(); - user.username = "user-hls-manifest-touch-clear".to_string(); - user.max_connections = 1; - - manager.add_connection(&segment_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-hls-manifest-touch-clear", - virtual_id: 7790, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &segment_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 604, - meter_uid: 704, - username: &user.username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(7790), - user_agent: Cow::Borrowed("player/1.0"), - session_token: Some("tok-hls-manifest-touch-clear"), - }) - .await - .expect("stream should be created"); - - manager.touch_http_activity(&user.username, "tok-hls-manifest-touch-clear", &manifest_addr).await; - manager.clear_unbound_session_addr(&user.username, "tok-hls-manifest-touch-clear", &manifest_addr).await; - - let connections = manager.connections.read().await; - assert!(!connections.key_by_addr.contains_key(&manifest_addr)); - let data = connections.by_key.get(&user.username).expect("user should exist"); - assert_eq!(data.streams[0].addr, segment_addr); - assert_eq!(data.sessions[0].addr, segment_addr); - assert!(!data.sessions[0].active_addrs.contains(&manifest_addr)); -} - -#[tokio::test] -async fn socket_expiry_deadline_extends_direct_body_streams_to_the_idle_timeout() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55040".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-vod".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = "user-vod-expiry".to_string(); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-vod-expiry", - virtual_id: 8888, - provider: "provider-a", - stream_url: "http://localhost/movie.mkv", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let mut channel = test_channel(8888); - channel.item_type = PlaylistItemType::Video; - channel.cluster = XtreamCluster::Video; - channel.url = "http://localhost/movie.mkv".intern(); - - manager - .update_connection(ActiveUserConnectionParams { - uid: 602, - meter_uid: 702, - username: "user-vod-expiry", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &channel, - user_agent: Cow::Borrowed("player/1.0"), - session_token: Some("tok-vod-expiry"), - }) - .await - .expect("vod stream should be created"); - - // The player stops reading while draining its buffer. The pause already exceeds the - // short HLS session TTL but stays well within the direct-body idle timeout, so the - // socket must not be treated as expired yet. - let previous_registration_ts = { - let mut connections = manager.connections.write().await; - let registration = connections.key_by_addr.get_mut(&addr).expect("registration should exist"); - registration.ts = registration.ts.saturating_sub(default_hls_session_ttl_secs() + 5); - registration.ts - }; - - let deadline = - manager.socket_expiry_deadline(&addr).await.expect("VOD streams should stay scheduled for expiry tracking"); - - let unchanged_registration_ts = { - let connections = manager.connections.read().await; - connections.key_by_addr.get(&addr).expect("registration should still exist").ts - }; - - assert_eq!(unchanged_registration_ts, previous_registration_ts); - assert!( - deadline > current_time_secs(), - "a buffering direct-body stream must not be treated as expired after only the HLS session TTL" - ); - assert_eq!( - deadline, - previous_registration_ts.saturating_add(DIRECT_BODY_IDLE_TIMEOUT_SECS), - "direct-body sockets must use the direct-body idle timeout as their expiry allowance" - ); -} - -#[tokio::test] -async fn socket_expiry_deadline_keeps_hls_session_ttl_for_adaptive_streams() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55043".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-hls".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = "user-hls-expiry".to_string(); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-hls-expiry", - virtual_id: 8891, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - manager - .update_connection(ActiveUserConnectionParams { - uid: 605, - meter_uid: 705, - username: "user-hls-expiry", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(8891), - user_agent: Cow::Borrowed("player/1.0"), - session_token: Some("tok-hls-expiry"), - }) - .await - .expect("hls stream should be created"); - - let registration_ts = { - let mut connections = manager.connections.write().await; - let registration = connections.key_by_addr.get_mut(&addr).expect("registration should exist"); - registration.ts = registration.ts.saturating_sub(default_hls_session_ttl_secs() + 5); - registration.ts - }; - - let deadline = manager.socket_expiry_deadline(&addr).await.expect("HLS streams should stay scheduled for expiry"); - - assert_eq!( - deadline, - registration_ts.saturating_add(manager.active_socket_ttl_secs()), - "adaptive sockets keep the short HLS session TTL" - ); -} - -#[tokio::test] -async fn touch_socket_activity_refreshes_registration_without_resetting_stream_start() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55041".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-vod-touch".to_string(), "127.0.0.1".to_string(), addr); - let mut user = ProxyUserCredentials::default(); - user.username = "user-vod-touch".to_string(); - user.max_connections = 1; - - manager.add_connection(&addr).await; - let mut channel = test_channel(8889); - channel.item_type = PlaylistItemType::Video; - channel.cluster = XtreamCluster::Video; - channel.url = "http://localhost/movie-2.mkv".intern(); - - let stream = manager - .update_connection(ActiveUserConnectionParams { - uid: 603, - meter_uid: 703, - username: "user-vod-touch", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &channel, - user_agent: Cow::Borrowed("player/1.0"), - session_token: None, - }) - .await - .expect("vod stream should be created"); - - let stale_registration_ts = { - let mut connections = manager.connections.write().await; - let registration = connections.key_by_addr.get_mut(&addr).expect("registration should exist"); - registration.ts = registration.ts.saturating_sub(DEFAULT_ACTIVE_SOCKET_TTL_SECS + 5); - registration.ts - }; - - manager.touch_socket_activity(&addr).await; - - let (refreshed_registration_ts, stream_started_at) = { - let connections = manager.connections.read().await; - let registration_ts = connections.key_by_addr.get(&addr).expect("registration should still exist").ts; - let stream_started_at = connections - .by_key - .get("user-vod-touch") - .and_then(|data| data.streams.iter().find(|active| active.uid == stream.uid)) - .expect("stream should still exist") - .ts; - (registration_ts, stream_started_at) - }; - - assert!(refreshed_registration_ts > stale_registration_ts); - assert_eq!(stream_started_at, stream.ts, "body activity must not reset visible stream duration"); -} - -#[tokio::test] -async fn update_session_addr_prunes_previous_registration_for_socket_bound_session() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let old_addr: SocketAddr = "127.0.0.1:55121".parse().unwrap(); - let new_addr: SocketAddr = "127.0.0.1:55122".parse().unwrap(); - let old_fingerprint = Fingerprint::new("fp-old".to_string(), "127.0.0.1".to_string(), old_addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&old_addr).await; - manager.add_connection(&new_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-move", - virtual_id: 9101, - provider: "provider-a", - stream_url: "http://localhost/live.ts", - addr: &old_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: true, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 301, - meter_uid: 401, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &old_fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::Live, ..test_channel(9101) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-move"), - }) - .await - .expect("initial live stream should register"); - - manager.update_session_addr("user1", "tok-move", &new_addr).await; - - let connections = manager.connections.read().await; - assert!( - !connections.key_by_addr.contains_key(&old_addr), - "previous range-request socket registration should be pruned once the session moved" - ); - assert!(connections.key_by_addr.contains_key(&new_addr)); - - let connection_data = connections.by_key.get("user1").expect("user connection data"); - assert_eq!(connection_data.sessions.len(), 1); - assert_eq!(connection_data.sessions[0].addr, new_addr); - assert_eq!(connection_data.streams.len(), 1); - assert_eq!(connection_data.streams[0].addr, new_addr); -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn vod_session_survives_overlapping_and_seek_sockets() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let base_addr: SocketAddr = "127.0.0.1:55131".parse().unwrap(); - let range_addr: SocketAddr = "127.0.0.1:55132".parse().unwrap(); - let seek_addr: SocketAddr = "127.0.0.1:55133".parse().unwrap(); - let base_fingerprint = Fingerprint::new("fp-vod-base".to_string(), "127.0.0.1".to_string(), base_addr); - let range_fingerprint = Fingerprint::new("fp-vod-range".to_string(), "127.0.0.1".to_string(), range_addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&base_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-vod", - virtual_id: 9102, - provider: "provider-a", - stream_url: "http://localhost/movie.mkv", - addr: &base_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 302, - meter_uid: 402, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &base_fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::Video, ..test_channel(9102) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-vod"), - }) - .await - .expect("initial vod stream should register"); - - manager.add_connection(&range_addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 303, - meter_uid: 403, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &range_fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::Video, ..test_channel(9102) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-vod"), - }) - .await - .expect("overlapping range request should reuse the same vod session"); - - assert_eq!(manager.user_connections("user1").await, 1); - assert!(manager.release_stream(&range_addr).await.is_none()); - let released = manager.release_connection(&range_addr).await; - assert!(released.addr_removed); - assert!(released.removed_streams.is_empty()); - - { - let connections = manager.connections.read().await; - assert!(connections.key_by_addr.contains_key(&base_addr)); - let connection_data = connections.by_key.get("user1").expect("user connection data"); - assert_eq!(connection_data.sessions[0].addr, base_addr); - assert_eq!(connection_data.streams[0].addr, base_addr); - } - - manager.add_connection(&seek_addr).await; - manager.update_session_addr("user1", "tok-vod", &seek_addr).await; - - { - let connections = manager.connections.read().await; - assert!( - connections.key_by_addr.contains_key(&base_addr), - "existing vod socket must remain registered while the session spans multiple requests" - ); - assert!(connections.key_by_addr.contains_key(&seek_addr)); - - let connection_data = connections.by_key.get("user1").expect("user connection data"); - assert_eq!(connection_data.sessions[0].addr, seek_addr); - assert_eq!(connection_data.streams[0].addr, seek_addr); - } - - assert!(manager.release_stream(&seek_addr).await.is_none()); - let released = manager.release_connection(&seek_addr).await; - assert!(released.addr_removed); - assert!(released.removed_streams.is_empty()); - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get("user1").expect("user connection data"); - assert_eq!(connection_data.sessions[0].addr, base_addr); - assert_eq!(connection_data.streams[0].addr, base_addr); -} - -#[tokio::test] -async fn update_session_provider_binding_updates_session_and_streams() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55140".parse().unwrap(); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user1"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-binding", - virtual_id: 100, - provider: "account-b", - stream_url: "http://example.com/vod/movie.mkv?token=account-b", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - { - let mut connections = manager.connections.write().await; - let data = connections.by_key.get_mut("user1").expect("user connection data"); - data.streams.push(StreamInfo::new(shared::model::StreamInfoParams { - uid: 1, - meter_uid: 1, - username: &user.username, - addr: &addr, - client_ip: "127.0.0.1", - provider: "account-b".intern(), - stream_channel: test_channel(100), - user_agent: "ua".to_string(), - country_code: None, - session_token: Some("tok-binding"), - })); - } - - manager - .update_session_provider_binding( - "user1", - "tok-binding", - "account-a".intern(), - "http://example.com/vod/movie.mkv?token=account-a".into(), - ) - .await; - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get("user1").expect("user connection data"); - assert_eq!(connection_data.sessions[0].provider.as_ref(), "account-a"); - assert_eq!(connection_data.sessions[0].stream_url.as_ref(), "http://example.com/vod/movie.mkv?token=account-a"); - assert_eq!(connection_data.streams[0].provider.as_ref(), "account-a"); -} - -#[tokio::test] -async fn catchup_release_connection_preserves_logical_stream_until_session_expires() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55141".parse().unwrap(); - let next_addr: SocketAddr = "127.0.0.1:55142".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-catchup-1".to_string(), "127.0.0.1".to_string(), addr); - let next_fingerprint = Fingerprint::new("fp-catchup-2".to_string(), "127.0.0.1".to_string(), next_addr); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user-catchup"); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-catchup", - virtual_id: 9103, - provider: "provider-a", - stream_url: "http://localhost/archive.ts", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - let first = manager - .update_connection(ActiveUserConnectionParams { - uid: 304, - meter_uid: 404, - username: &user.username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::Catchup, ..test_channel(9103) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-catchup"), - }) - .await - .expect("initial catchup stream should register"); - - let released = manager.release_connection(&addr).await; - assert!(released.addr_removed); - assert!( - released.removed_streams.is_empty(), - "catchup stream should remain logically active between range requests" - ); - - assert_eq!(manager.user_connections(&user.username).await, 0); - assert!(manager.active_streams().await.is_empty()); - - let connections = manager.connections.read().await; - let preserved_stream = connections - .by_key - .get(&user.username) - .and_then(|data| data.streams.iter().find(|stream| stream.uid == first.uid)) - .expect("preserved catchup stream should stay internally tracked"); - assert!(preserved_stream.preserved); - drop(connections); - - manager.add_connection(&next_addr).await; - let second = manager - .update_connection(ActiveUserConnectionParams { - uid: 305, - meter_uid: 405, - username: &user.username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &next_fingerprint, - provider: "provider-a".intern(), - stream_channel: &StreamChannel { item_type: PlaylistItemType::Catchup, ..test_channel(9103) }, - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-catchup"), - }) - .await - .expect("catchup stream should reconnect"); - - assert_eq!(second.uid, first.uid); - assert_eq!(second.started_at, first.started_at); - assert!(!second.preserved); -} - -#[tokio::test] -async fn gc_keeps_active_ts_streams_even_when_user_timestamp_is_stale() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55013".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key-ts".to_string(), "127.0.0.1".to_string(), addr); - - manager.add_connection(&addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 144, - meter_uid: 244, - username: "user1", - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(9001), - user_agent: Cow::Borrowed("ua"), - session_token: None, - }) - .await - .expect("ts stream should register"); - - { - let mut connections = manager.connections.write().await; - let connection_data = connections.by_key.get_mut("user1").expect("user entry should exist"); - connection_data.ts = connection_data.ts.saturating_sub(USER_CON_TTL + 1); - } - - if let Some(gc_ts) = &manager.gc_ts { - gc_ts.store(current_time_secs().saturating_sub(USER_GC_TTL + 1), Ordering::Release); - } - - manager.active_streams().await; - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get("user1").expect("active user entry must survive gc"); - assert_eq!(connection_data.connections, 1); - assert_eq!(connection_data.streams.len(), 1); -} - -#[tokio::test] -async fn session_activation_keeps_first_hls_slot_uncommitted_before_stream_registration() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user-hls-reserve"); - user.max_connections = 1; - - let first_addr: SocketAddr = "127.0.0.1:55180".parse().unwrap(); - let second_addr: SocketAddr = "127.0.0.1:55181".parse().unwrap(); - - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-first", - virtual_id: 9201, - provider: "provider-a", - stream_url: "http://localhost/live-a.m3u8", - addr: &first_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-second", - virtual_id: 9202, - provider: "provider-a", - stream_url: "http://localhost/live-b.m3u8", - addr: &second_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let first_admission = - manager.connection_admission_for_session_activation(&user.username, user.max_connections, 0, "tok-first").await; - let second_admission = manager - .connection_admission_for_session_activation(&user.username, user.max_connections, 0, "tok-second") - .await; - - assert_eq!(first_admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(first_admission.kind(), Some(ConnectionKind::Normal)); - assert_eq!(second_admission.permission(), UserConnectionPermission::Allowed); - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_eq!(connection_data.connections, 0); - assert_eq!(connection_data.counts.normal, 0); - assert_eq!(connection_data.streams.len(), 0); - assert!(connection_data - .sessions - .iter() - .find(|session| session.token == "tok-first") - .is_some_and(|session| !session.lifecycle.is_counted())); - assert!(connection_data - .sessions - .iter() - .find(|session| session.token == "tok-second") - .is_some_and(|session| !session.lifecycle.is_counted())); -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn binding_reserved_sessions_keeps_hard_and_soft_counts_stable() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user-hls-soft"); - user.max_connections = 1; - user.soft_connections = 1; - - let first_addr: SocketAddr = "127.0.0.1:55182".parse().unwrap(); - let second_addr: SocketAddr = "127.0.0.1:55183".parse().unwrap(); - let first_fingerprint = Fingerprint::new("fp-hls-1".to_string(), "127.0.0.1".to_string(), first_addr); - let second_fingerprint = Fingerprint::new("fp-hls-2".to_string(), "127.0.0.1".to_string(), second_addr); - - manager.add_connection(&first_addr).await; - manager.add_connection(&second_addr).await; - - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-normal", - virtual_id: 9203, - provider: "provider-a", - stream_url: "http://localhost/live-normal.m3u8", - addr: &first_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-soft", - virtual_id: 9204, - provider: "provider-a", - stream_url: "http://localhost/live-soft.m3u8", - addr: &second_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let first_admission = manager - .connection_admission_for_session_activation( - &user.username, - user.max_connections, - user.soft_connections, - "tok-normal", - ) - .await; - assert_eq!(first_admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(first_admission.kind(), Some(ConnectionKind::Normal)); - - manager - .update_connection(ActiveUserConnectionParams { - uid: 401, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &first_fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(9203), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-normal"), - }) - .await - .expect("reserved normal session should bind"); - - let second_admission = manager - .connection_admission_for_session_activation( - &user.username, - user.max_connections, - user.soft_connections, - "tok-soft", - ) - .await; - assert_eq!(second_admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(second_admission.kind(), Some(ConnectionKind::Soft)); - - manager - .update_connection(ActiveUserConnectionParams { - uid: 402, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: ConnectionKind::Soft, - priority: 0, - soft_priority: 0, - fingerprint: &second_fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(9204), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-soft"), - }) - .await - .expect("reserved soft session should bind"); - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_eq!(connection_data.connections, 2); - assert_eq!(connection_data.counts.normal, 1); - assert_eq!(connection_data.counts.soft, 1); - assert_eq!(connection_data.streams.len(), 2); - assert_eq!( - connection_data.stream_kinds.get(&401), - Some(&ConnectionKind::Normal), - "binding a reserved normal session must not increment counts twice" - ); - assert_eq!( - connection_data.stream_kinds.get(&402), - Some(&ConnectionKind::Soft), - "binding a reserved soft session must keep the soft classification" - ); -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn origin_policy_refresh_promotes_counted_soft_session_when_hard_slot_is_available() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user-hls-policy-refresh"); - user.max_connections = 1; - user.soft_connections = 1; - - let normal_addr: SocketAddr = "127.0.0.1:55185".parse().unwrap(); - let soft_addr: SocketAddr = "127.0.0.1:55186".parse().unwrap(); - let normal_fingerprint = Fingerprint::new("fp-hls-policy-1".to_string(), "127.0.0.1".to_string(), normal_addr); - let soft_fingerprint = Fingerprint::new("fp-hls-policy-2".to_string(), "127.0.0.1".to_string(), soft_addr); - - manager.add_connection(&normal_addr).await; - manager.add_connection(&soft_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-normal", - virtual_id: 9210, - provider: "provider-a", - stream_url: "http://localhost/live-normal.m3u8", - addr: &normal_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-soft", - virtual_id: 9211, - provider: "provider-a", - stream_url: "http://localhost/live-soft.m3u8", - addr: &soft_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let normal_admission = manager - .connection_admission_for_session_activation( - &user.username, - user.max_connections, - user.soft_connections, - "tok-normal", - ) - .await; - assert_eq!(normal_admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(normal_admission.kind(), Some(ConnectionKind::Normal)); - manager - .update_connection(ActiveUserConnectionParams { - uid: 411, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &normal_fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(9210), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-normal"), - }) - .await - .expect("normal stream should bind"); - - let soft_admission = manager - .connection_admission_for_session_activation( - &user.username, - user.max_connections, - user.soft_connections, - "tok-soft", - ) - .await; - assert_eq!(soft_admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(soft_admission.kind(), Some(ConnectionKind::Soft)); - manager - .update_connection(ActiveUserConnectionParams { - uid: 412, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: ConnectionKind::Soft, - priority: 0, - soft_priority: 0, - fingerprint: &soft_fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(9211), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-soft"), - }) - .await - .expect("soft stream should bind"); - - assert!(manager.release_session_streams_and_counted_reservation(&user.username, "tok-normal").await); - { - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_eq!(connection_data.connections, 1); - assert_eq!(connection_data.counts.normal, 0); - assert_eq!(connection_data.counts.soft, 1); - assert_eq!( - connection_data - .sessions - .iter() - .find(|session| session.token == "tok-soft") - .and_then(|session| session.connection_kind), - Some(ConnectionKind::Soft) - ); - } - - let refreshed_kind = manager - .refresh_session_connection_kind_for_origin_policy( - &user.username, - user.max_connections, - user.soft_connections, - "tok-soft", - ) - .await; - assert_eq!(refreshed_kind, Some(ConnectionKind::Normal)); - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_eq!(connection_data.connections, 1); - assert_eq!(connection_data.counts.normal, 1); - assert_eq!(connection_data.counts.soft, 0); - assert_eq!( - connection_data - .sessions - .iter() - .find(|session| session.token == "tok-soft") - .and_then(|session| session.connection_kind), - Some(ConnectionKind::Normal) - ); - assert_eq!(connection_data.stream_kinds.get(&412), Some(&ConnectionKind::Normal)); -} - -#[tokio::test] -async fn origin_policy_refresh_returns_none_for_pending_grace_without_available_slot() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user-pending-grace-origin-policy"); - user.max_connections = 1; - - let active_addr: SocketAddr = "127.0.0.1:55195".parse().unwrap(); - let pending_addr: SocketAddr = "127.0.0.1:55196".parse().unwrap(); - let active_fingerprint = Fingerprint::new("active".to_string(), "127.0.0.1".to_string(), active_addr); - - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-active", - virtual_id: 9301, - provider: "provider-a", - stream_url: "http://localhost/live-active.m3u8", - addr: &active_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 9301, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &active_fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(9301), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-active"), - }) - .await - .expect("active stream should bind the only normal slot"); - - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-pending", - virtual_id: 9302, - provider: "provider-a", - stream_url: "http://localhost/live-pending.m3u8", - addr: &pending_addr, - connection_permission: UserConnectionPermission::GracePeriod, - connection_kind: None, - socket_bound: false, - }) - .await; - manager - .mark_pending_provider( - &user.username, - "tok-pending", - PendingProviderReason::GraceHold, - current_time_secs() + 30, - ) - .await - .expect("pending session should be marked"); - - let refreshed_kind = manager - .refresh_session_connection_kind_for_origin_policy( - &user.username, - user.max_connections, - user.soft_connections, - "tok-pending", - ) - .await; - - assert_eq!(refreshed_kind, None); -} - -#[tokio::test] -async fn release_unbound_session_reservation_frees_reserved_slot() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user-release-reservation"); - user.max_connections = 1; - - let addr: SocketAddr = "127.0.0.1:55184".parse().unwrap(); - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-release", - virtual_id: 9205, - provider: "provider-a", - stream_url: "http://localhost/live-release.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let admission = manager - .connection_admission_for_session_activation(&user.username, user.max_connections, 0, "tok-release") - .await; - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - - manager.release_unbound_session_reservation(&user.username, "tok-release", None, false).await; - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_eq!(connection_data.connections, 0); - assert_eq!(connection_data.counts.normal, 0); - assert_eq!(connection_data.streams.len(), 0); - assert!(connection_data - .sessions - .iter() - .find(|session| session.token == "tok-release") - .is_some_and(|session| !session.lifecycle.is_counted())); -} - -#[tokio::test] -async fn preserved_reactivation_admission_does_not_create_ownerless_counted_slot() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let user = test_user_credentials("user-preserved-activation", 1, 0); - let addr: SocketAddr = "127.0.0.1:55195".parse().unwrap(); - let session_token = "tok-preserved-activation"; - let stream_uid = 501; - - commit_and_preserve_adaptive_session(&manager, &user, session_token, stream_uid, addr, ConnectionKind::Normal) - .await; - - let virtual_admission = - manager.connection_admission(&user.username, user.max_connections, user.soft_connections).await; - assert_eq!(virtual_admission.permission(), UserConnectionPermission::Exhausted); - - let admission = manager - .connection_admission_for_session_activation( - &user.username, - user.max_connections, - user.soft_connections, - session_token, - ) - .await; - - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(admission.kind(), Some(ConnectionKind::Normal)); - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_preserved_session_is_uncounted(connection_data, session_token, stream_uid); - assert_no_real_connection_slots(connection_data); -} - -#[tokio::test] -async fn preserved_reactivation_admission_then_kicked_release_removes_state_without_ghost_counter() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let user = test_user_credentials("user-preserved-eviction", 1, 0); - let addr: SocketAddr = "127.0.0.1:55201".parse().unwrap(); - let session_token = "tok-preserved-eviction"; - let stream_uid = 601; - - commit_and_preserve_adaptive_session(&manager, &user, session_token, stream_uid, addr, ConnectionKind::Normal) - .await; - let admission = manager - .connection_admission_for_session_activation( - &user.username, - user.max_connections, - user.soft_connections, - session_token, - ) - .await; - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - { - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_preserved_session_is_uncounted(connection_data, session_token, stream_uid); - assert_no_real_connection_slots(connection_data); - } - - let released = manager.release_connection_as_kicked(&addr).await; - assert!(released.addr_removed); - assert_eq!(released.removed_streams.len(), 1); - let removed_stream = released.removed_streams.first().expect("kicked release must remove the preserved stream"); - assert_eq!(removed_stream.uid, stream_uid); - assert!(removed_stream.preserved); - assert_eq!(removed_stream.session_token.as_deref(), Some(session_token)); - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert!(connection_data.streams.iter().all(|stream| stream.uid != stream_uid)); - assert!(connection_data.sessions.iter().all(|session| session.token != session_token)); - assert!(!connection_data.stream_kinds.contains_key(&stream_uid)); - assert_no_real_connection_slots(connection_data); - drop(connections); - assert_eq!(manager.active_users_and_connections().await, (0, 0)); -} - -#[tokio::test] -async fn preserved_reactivation_admission_then_lease_idle_cleanup_leaves_counters_at_zero() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let user = test_user_credentials("user-preserved-idle-cleanup", 1, 0); - let addr: SocketAddr = "127.0.0.1:55202".parse().unwrap(); - let session_token = "tok-preserved-idle-cleanup"; - let stream_uid = 602; - - commit_and_preserve_adaptive_session(&manager, &user, session_token, stream_uid, addr, ConnectionKind::Normal) - .await; - let admission = manager - .connection_admission_for_session_activation( - &user.username, - user.max_connections, - user.soft_connections, - session_token, - ) - .await; - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - { - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_preserved_session_is_uncounted(connection_data, session_token, stream_uid); - assert_no_real_connection_slots(connection_data); - } - - // Shared-HLS lease-idle cleanup delegates to this manager operation. - let counter_changed = manager.release_session_streams_and_counted_reservation(&user.username, session_token).await; - assert!(!counter_changed, "removing an uncounted preserved stream must not change real counters"); - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert!(connection_data.streams.iter().all(|stream| stream.uid != stream_uid)); - assert!(connection_data - .sessions - .iter() - .find(|session| session.token == session_token) - .is_some_and(|session| session.lifecycle == PlaybackLifecycle::Preserved)); - assert!(!connection_data.stream_kinds.contains_key(&stream_uid)); - assert_no_real_connection_slots(connection_data); - drop(connections); - assert_eq!(manager.active_users_and_connections().await, (0, 0)); -} - -#[tokio::test] -async fn repeated_preserved_reactivation_cleanup_does_not_accumulate_connections() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let user = test_user_credentials("user-preserved-repeat", 4, 0); - for (session_token, stream_uid, addr) in - [("tok-preserved-repeat-one", 603, "127.0.0.1:55203"), ("tok-preserved-repeat-two", 604, "127.0.0.1:55204")] - { - let addr = addr.parse().unwrap(); - commit_and_preserve_adaptive_session(&manager, &user, session_token, stream_uid, addr, ConnectionKind::Normal) - .await; - - let admission = manager - .connection_admission_for_session_activation( - &user.username, - user.max_connections, - user.soft_connections, - session_token, - ) - .await; - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - - let counter_changed = - manager.release_session_streams_and_counted_reservation(&user.username, session_token).await; - assert!(!counter_changed, "cleanup must not release a slot that was never committed"); - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_no_real_connection_slots(connection_data); - drop(connections); - assert_eq!(manager.active_users_and_connections().await, (0, 0)); - } -} - -#[tokio::test] -async fn dashboard_counts_only_real_slots_during_preserved_reactivation_admission() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let user = test_user_credentials("user-preserved-dashboard", 1, 0); - let addr: SocketAddr = "127.0.0.1:55205".parse().unwrap(); - let session_token = "tok-preserved-dashboard"; - let stream_uid = 605; - - commit_and_preserve_adaptive_session(&manager, &user, session_token, stream_uid, addr, ConnectionKind::Normal) - .await; - assert_eq!(manager.active_users_and_connections().await, (0, 0)); - - let admission = manager - .connection_admission_for_session_activation( - &user.username, - user.max_connections, - user.soft_connections, - session_token, - ) - .await; - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(manager.active_users_and_connections().await, (0, 0)); - - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_preserved_session_is_uncounted(connection_data, session_token, stream_uid); - assert_no_real_connection_slots(connection_data); -} - -#[tokio::test] -async fn preserved_soft_reactivation_and_cleanup_leave_normal_slot_unchanged() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let user = test_user_credentials("user-preserved-soft", 1, 1); - let normal_addr: SocketAddr = "127.0.0.1:55206".parse().unwrap(); - let normal_fingerprint = - Fingerprint::new("fp-preserved-soft-normal".to_string(), normal_addr.ip().to_string(), normal_addr); - let soft_addr: SocketAddr = "127.0.0.1:55207".parse().unwrap(); - let normal_stream_uid = 606; - let session_token = "tok-preserved-soft"; - let soft_stream_uid = 607; - - manager.add_connection(&normal_addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: normal_stream_uid, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: ConnectionKind::Normal, - priority: user.priority, - soft_priority: user.soft_priority, - fingerprint: &normal_fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(normal_stream_uid), - user_agent: Cow::Borrowed("ua"), - session_token: None, - }) - .await - .expect("normal stream should bind"); - { - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_single_normal_stream_slot(connection_data, normal_stream_uid); - } - - commit_and_preserve_adaptive_session( - &manager, - &user, - session_token, - soft_stream_uid, - soft_addr, - ConnectionKind::Soft, - ) - .await; - { - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_preserved_session_is_uncounted(connection_data, session_token, soft_stream_uid); - assert_single_normal_stream_slot(connection_data, normal_stream_uid); - } - - let admission = manager - .connection_admission_for_session_activation( - &user.username, - user.max_connections, - user.soft_connections, - session_token, - ) - .await; - assert_eq!(admission.permission(), UserConnectionPermission::Allowed); - assert_eq!(admission.kind(), Some(ConnectionKind::Soft)); - - { - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert_preserved_session_is_uncounted(connection_data, session_token, soft_stream_uid); - assert_single_normal_stream_slot(connection_data, normal_stream_uid); - } - - let counter_changed = manager.release_session_streams_and_counted_reservation(&user.username, session_token).await; - assert!(!counter_changed, "preserved soft cleanup must not release an uncommitted slot"); - { - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert!(connection_data.streams.iter().all(|stream| stream.uid != soft_stream_uid)); - assert!(!connection_data.stream_kinds.contains_key(&soft_stream_uid)); - assert!(connection_data - .sessions - .iter() - .find(|session| session.token == session_token) - .is_some_and(|session| session.lifecycle == PlaybackLifecycle::Preserved)); - assert_single_normal_stream_slot(connection_data, normal_stream_uid); - } - - manager.release_stream_by_uid(&normal_addr, normal_stream_uid).await.expect("normal stream should release"); - let connections = manager.connections.read().await; - let connection_data = connections.by_key.get(&user.username).expect("user connection data"); - assert!(connection_data.streams.is_empty()); - assert!(connection_data.stream_kinds.is_empty()); - assert_no_real_connection_slots(connection_data); - drop(connections); - assert_eq!(manager.active_users_and_connections().await, (0, 0)); -} - -#[tokio::test] -async fn release_unbound_session_reservation_ignores_stale_transition_version() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user-stale-release"); - - let addr: SocketAddr = "127.0.0.1:55194".parse().unwrap(); - let stale_version = manager - .ensure_user_session_placeholder(CreateUserSessionParams { - user: &user, - session_token: "tok-stale-release", - virtual_id: 9206, - provider: "provider-a", - stream_url: "http://localhost/live-stale.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - let _ = manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-stale-release", - virtual_id: 9206, - provider: "provider-b", - stream_url: "http://localhost/live-stale-updated.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - manager.release_unbound_session_reservation(&user.username, "tok-stale-release", Some(stale_version), true).await; - - let session = manager - .get_and_update_user_session(&user.username, "tok-stale-release") - .await - .expect("stale rollback must not remove the newer session"); - assert!(session.transition_version > stale_version); - assert_eq!(session.provider.as_ref(), "provider-b"); - assert_eq!(session.stream_url.as_ref(), "http://localhost/live-stale-updated.m3u8"); -} - -#[tokio::test] -async fn clear_unbound_session_addr_prunes_manifest_addr_without_stream() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let first_addr: SocketAddr = "127.0.0.1:55185".parse().unwrap(); - let second_addr: SocketAddr = "127.0.0.1:55186".parse().unwrap(); - let mut user = ProxyUserCredentials::default(); - user.username = String::from("user-clear-addr"); - user.max_connections = 1; - - manager.add_connection(&first_addr).await; - manager.add_connection(&second_addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-clear-addr", - virtual_id: 9206, - provider: "provider-a", - stream_url: "http://localhost/live-clear.m3u8", - addr: &first_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-clear-addr", - virtual_id: 9206, - provider: "provider-a", - stream_url: "http://localhost/live-clear.m3u8", - addr: &second_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - manager.clear_unbound_session_addr(&user.username, "tok-clear-addr", &second_addr).await; - - let connections = manager.connections.read().await; - let session = connections - .by_key - .get(&user.username) - .and_then(|connection_data| connection_data.sessions.iter().find(|session| session.token == "tok-clear-addr")) - .expect("session should remain"); - assert_eq!(session.addr, first_addr); - assert_eq!(session.active_addrs, vec![first_addr]); -} - -#[tokio::test] -async fn get_eviction_candidates_keeps_preserved_streams_evictable() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55300".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key".to_string(), "192.168.1.100".to_string(), addr); - let username = "user-eviction-addr"; - let mut user = ProxyUserCredentials::default(); - user.username = username.to_string(); - user.max_connections = 1; - user.soft_connections = 0; - - // Create session first (HLS type = preserved after release) - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-preserved-1", - virtual_id: 5001, - provider: "provider-a", - stream_url: "http://localhost/live.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - - // Create stream + register connection - manager.add_connection(&addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 1, - meter_uid: 0, - username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(5001), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-preserved-1"), - }) - .await - .expect("first stream"); - assert_eq!(manager.user_connections(username).await, 1); - - // Release -> stream becomes preserved, session becomes uncounted - manager.release_stream(&addr).await; - assert_eq!(manager.user_connections(username).await, 0, "preserved stream should not count"); - - let candidates = manager.get_eviction_candidates(username, "192.168.1.100").await; - assert!( - candidates.iter().any(|candidate| candidate.addr == addr), - "preserved stream should remain a direct eviction candidate" - ); -} - -#[tokio::test] -async fn get_eviction_candidates_does_not_count_preserved_streams_in_addr_counts() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55801".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-preserved-no-count".to_string(), "10.0.0.5".to_string(), addr); - let username = "user-preserved-addr-count"; - let mut user = ProxyUserCredentials::default(); - user.username = username.to_string(); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-preserved-addr-count", - virtual_id: 7000, - provider: "provider-preserved", - stream_url: "http://localhost/preserved.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 7000, - meter_uid: 0, - username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-preserved".intern(), - stream_channel: &test_adaptive_channel(7000), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-preserved-addr-count"), - }) - .await - .expect("stream should be created"); - - // Release -> stream becomes preserved, session becomes uncounted - manager.release_stream(&addr).await; - - // Preserved streams do not consume a counted slot — user_connections should be 0 - assert_eq!( - manager.user_connections(username).await, - 0, - "preserved stream should not count toward active connections" - ); - - // But the preserved stream is still a valid eviction candidate (valid victim) - let candidates = manager.get_eviction_candidates(username, "10.0.0.5").await; - assert!(candidates.iter().any(|c| c.addr == addr), "preserved stream should be an eviction candidate"); -} - -#[tokio::test] -async fn connection_admission_treats_preserved_stream_as_reserved_capacity() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55305".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-preserved-admission".to_string(), "192.168.1.100".to_string(), addr); - let username = "user-preserved-admission"; - let mut user = ProxyUserCredentials::default(); - user.username = username.to_string(); - user.max_connections = 1; - - manager.add_connection(&addr).await; - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-preserved-admission", - virtual_id: 6000, - provider: "provider-a", - stream_url: "http://localhost/live-preserved.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 6000, - meter_uid: 0, - username, - max_connections: user.max_connections, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(6000), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-preserved-admission"), - }) - .await - .expect("preserved stream should be created"); - - manager.release_connection(&addr).await; - assert_eq!(manager.user_connections(username).await, 0, "preserved stream stays uncounted for active snapshots"); - - let admission = manager.connection_admission(username, user.max_connections, 0).await; - assert_eq!( - admission.permission(), - UserConnectionPermission::Exhausted, - "a preserved stream should still reserve capacity against unrelated playback admissions" - ); -} - -#[tokio::test] -async fn connection_admission_for_session_evaluates_admission_for_uncounted_session() { - // Bug: connection_admission_for_session returns Allowed for any existing session, - // even if it's uncounted (preserved). This causes strategy evaluation to be skipped. - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "127.0.0.1:55310".parse().unwrap(); - let username = "user-uncounted-admission"; - let mut user = ProxyUserCredentials::default(); - user.username = username.to_string(); - user.max_connections = 1; - user.soft_connections = 0; - - // Create session + counted stream (HLS type = preserved after release) - manager.add_connection(&addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 1, - meter_uid: 0, - username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &Fingerprint::new("fp".to_string(), "192.168.1.50".to_string(), addr), - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(6001), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-uncounted"), - }) - .await - .expect("first stream"); - - // Release to preserve (uncounted session, but counts.normal still = 1 from the stream) - manager.release_stream(&addr).await; - // After preserve: session is uncounted, stream is preserved, connections=0 - // BUT the stream was removed, so counts.normal is decremented -> counts=0 - assert_eq!(manager.user_connections(username).await, 0); - - // Add a second stream first - this uses a different session token and consumes the slot - let second_addr: SocketAddr = "192.168.1.100:55311".parse().unwrap(); - manager.add_connection(&second_addr).await; - manager - .update_connection(ActiveUserConnectionParams { - uid: 2, - meter_uid: 0, - username, - max_connections: 1, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &Fingerprint::new("fp2".to_string(), "192.168.1.100".to_string(), second_addr), - provider: "provider-b".intern(), - stream_channel: &test_channel(6002), - user_agent: Cow::Borrowed("ua"), - session_token: Some("tok-second"), - }) - .await - .expect("second stream"); - // Now user is at limit: connections=1, counts.normal=1, max_connections=1 - assert_eq!(manager.user_connections(username).await, 1); - - // connection_admission_for_session for the PRESERVED session token should return - // Exhausted so that eviction strategies can run and evict the preserved stream, - // freeing a slot for the uncounted session to reactivate - let admission = manager.connection_admission_for_session(username, 1, 0, "tok-uncounted").await; - assert_eq!( - admission.permission(), - UserConnectionPermission::Exhausted, - "uncounted session should not bypass admission when user is at limit; \ - bug: session exists -> Allowed -> strategy evaluation skipped" - ); -} - -#[tokio::test] -async fn playback_transition_gate_serializes_same_session() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); - - let first_guard = manager.acquire_playback_transition("user-gated", "tok-gated").await; - let second_manager = Arc::clone(&manager); - let waiting = tokio::spawn(async move { - let _second_guard = second_manager.acquire_playback_transition("user-gated", "tok-gated").await; - }); - - tokio::time::sleep(Duration::from_millis(25)).await; - assert!( - !waiting.is_finished(), - "same-session transition gate should block a concurrent transition until the first completes" - ); - - drop(first_guard); - tokio::time::timeout(Duration::from_millis(100), waiting) - .await - .expect("second transition should proceed once the first guard is released") - .expect("second transition task should complete"); -} - -#[tokio::test] -async fn playback_transition_gate_cleanup_removes_idle_gates_on_next_acquire() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let first_guard = manager.acquire_playback_transition("user-gated-cleanup", "tok-first").await; - assert_eq!(manager.transition_gates.lock().await.len(), 1); - drop(first_guard); - - let second_guard = manager.acquire_playback_transition("user-gated-cleanup", "tok-second").await; - assert_eq!(manager.transition_gates.lock().await.len(), 1); - drop(second_guard); -} - -#[tokio::test] -async fn check_divergence_detects_connection_count_mismatch() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); - - let addr: SocketAddr = "127.0.0.1:55902".parse().unwrap(); - let mut user = ProxyUserCredentials::default(); - user.username = "div-user-2".to_string(); - - // Create a counted session without a stream or matching legacy counter. - { - let mut connections = manager.connections.write().await; - let data = connections.by_key.entry(user.username.clone()).or_insert_with(|| UserConnectionData::new(0, 1, 0)); - data.add_session(UserSession { - token: "tok-div-2".to_string(), - transition_version: 1, - virtual_id: 9002, - provider: "provider-a".intern(), - stream_url: "http://localhost/stream.ts".intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), - user_agent_stream_index: None, - addr, - socket_bound: false, - active_addrs: vec![addr], - ts: current_time_secs(), - started_at: current_time_secs(), - permission: UserConnectionPermission::Allowed, - connection_kind: None, - lifecycle: PlaybackLifecycle::Active, - ..Default::default() - }); - } - - let connections = manager.connections.read().await; - let data = connections.by_key.get(&user.username).expect("user connection data"); - let snapshot = ActiveUserManager::build_divergence_snapshot(data, &user.username); - assert!(snapshot.kinds.contains(&DivergenceKind::CountedSessionWithoutStream)); - assert!(snapshot.kinds.contains(&DivergenceKind::ConnectionCountMismatch { legacy: 0, counted: 1 })); - drop(connections); - manager.log_divergence_snapshot(Some(snapshot)).await; -} - -#[tokio::test] -async fn check_divergence_detects_stream_without_counted_session() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); - - let addr: SocketAddr = "127.0.0.1:55903".parse().unwrap(); - let mut user = ProxyUserCredentials::default(); - user.username = "div-user-3".to_string(); - - { - let mut connections = manager.connections.write().await; - let data = connections.by_key.entry(user.username.clone()).or_insert_with(|| UserConnectionData::new(0, 1, 0)); - - // Add a session with GraceHold pending — exempt from Invariant 1 - data.add_session(UserSession { - token: "tok-div-3".to_string(), - transition_version: 1, - virtual_id: 9003, - provider: "provider-a".intern(), - stream_url: "http://localhost/stream.ts".intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), - user_agent_stream_index: None, - addr, - socket_bound: false, - active_addrs: vec![addr], - ts: current_time_secs(), - started_at: current_time_secs(), - permission: UserConnectionPermission::Allowed, - connection_kind: None, - lifecycle: PlaybackLifecycle::PendingProvider { - data: PendingProviderState { - reason_code: PendingProviderReason::GraceHold, - created_at: current_time_secs(), - deadline: current_time_secs() + 30, - version: 1, - wake_source: None, - }, - }, - ..Default::default() - }); - data.increment_kind(ConnectionKind::Normal); - - // Add a stream whose session_token doesn't match any counted session - let orphan_stream = StreamInfo::new(shared::model::StreamInfoParams { - uid: 903, - meter_uid: 0, - username: &user.username, - addr: &addr, - client_ip: "127.0.0.1", - provider: "provider-a".intern(), - stream_channel: StreamChannel { - target_id: 1, - virtual_id: 9003, - provider_id: 1, - input_name: "provider-a".intern(), - item_type: PlaylistItemType::Live, - cluster: XtreamCluster::Live, - group: "g".intern(), - title: "t".intern(), - url: "http://localhost/stream.ts".intern(), - shared: false, - shared_joined_existing: None, - shared_stream_id: None, - technical: None, - epg_channel_id: None, - epg_reference_ts: None, - upstream_user_agent: None, - }, - user_agent: "ua".to_string(), - country_code: None, - session_token: Some("tok-orphan"), - }); - data.streams.push(orphan_stream); - data.stream_kinds.insert(903, ConnectionKind::Normal); - } - - let connections = manager.connections.read().await; - let data = connections.by_key.get(&user.username).expect("user connection data"); - let snapshot = ActiveUserManager::build_divergence_snapshot(data, &user.username); - assert!(snapshot.kinds.contains(&DivergenceKind::StreamWithoutCountedSession)); - drop(connections); - manager.log_divergence_snapshot(Some(snapshot)).await; -} - -#[tokio::test] -async fn divergence_log_rate_limited_within_cooldown_window() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); - - let addr: SocketAddr = "127.0.0.1:55904".parse().unwrap(); - let mut user = ProxyUserCredentials::default(); - user.username = "div-user-4".to_string(); - - // Create mismatch - { - let mut connections = manager.connections.write().await; - let data = connections.by_key.entry(user.username.clone()).or_insert_with(|| UserConnectionData::new(0, 1, 0)); - data.increment_kind(ConnectionKind::Normal); - data.add_session(UserSession { - token: "tok-div-4".to_string(), - transition_version: 1, - virtual_id: 9004, - provider: "provider-a".intern(), - stream_url: "http://localhost/stream.ts".intern(), - provider_session_headers: HashMap::new(), - provider_session_headers_host: None, - media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), - user_agent_stream_index: None, - addr, - socket_bound: false, - active_addrs: vec![addr], - ts: current_time_secs(), - started_at: current_time_secs(), - permission: UserConnectionPermission::Allowed, - connection_kind: None, - lifecycle: PlaybackLifecycle::Prepared, - ..Default::default() - }); - } - - let connections = manager.connections.read().await; - let data = connections.by_key.get(&user.username).expect("user connection data"); - let snapshot = ActiveUserManager::build_divergence_snapshot(data, &user.username); - drop(connections); - manager.log_divergence_snapshot(Some(snapshot)).await; - let key = divergence_key(&user.username, &DivergenceKind::ConnectionCountMismatch { legacy: 1, counted: 0 }); - let first_logged = { - let cache = manager.divergence_cache.lock().await; - let entry = cache.peek(&key).expect("first divergence should populate the cache"); - assert_eq!(entry.count_since_last_log, 0); - entry.last_logged - }; - - let connections = manager.connections.read().await; - let data = connections.by_key.get(&user.username).expect("user connection data"); - let snapshot = ActiveUserManager::build_divergence_snapshot(data, &user.username); - drop(connections); - manager.log_divergence_snapshot(Some(snapshot)).await; - let connections = manager.connections.read().await; - let data = connections.by_key.get(&user.username).expect("user connection data"); - let snapshot = ActiveUserManager::build_divergence_snapshot(data, &user.username); - drop(connections); - manager.log_divergence_snapshot(Some(snapshot)).await; - let cache = manager.divergence_cache.lock().await; - let entry = cache.peek(&key).expect("repeated divergence should remain cached"); - assert_eq!(entry.count_since_last_log, 2); - assert_eq!(entry.last_logged, first_logged); -} - -#[test] -fn socket_registration_primary_username_is_deterministic() { - let mut reg = SocketRegistration::anonymous(); - reg.add_user("user_z", 100); - reg.add_user("user_a", 100); - reg.add_user("user_m", 100); - assert_eq!(reg.primary_username(), Some("user_a")); -} - -#[tokio::test] -async fn release_stream_without_uid_rejects_ambiguous_multiple_users_on_same_addr() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "192.168.1.100:12345".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key".to_string(), "192.168.1.100".to_string(), addr); - - manager.add_connection(&addr).await; - - let stream_a = manager - .update_connection(ActiveUserConnectionParams { - uid: 1, - meter_uid: 0, - username: "user_a", - max_connections: 5, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(1001), - user_agent: Cow::Borrowed("ua"), - session_token: None, - }) - .await - .expect("register user_a"); - - let stream_b = manager - .update_connection(ActiveUserConnectionParams { - uid: 2, - meter_uid: 0, - username: "user_b", - max_connections: 5, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-b".intern(), - stream_channel: &test_channel(1002), - user_agent: Cow::Borrowed("ua"), - session_token: None, - }) - .await - .expect("register user_b"); - - // Releasing without stream_uid when multiple users are on the same addr must return None (ambiguous) - let released = manager.release_stream(&addr).await; - assert!( - released.is_none(), - "releasing without uid must not arbitrarily pick a user when multiple are active on the same addr" - ); - - // But releasing with explicit stream_uid works cleanly: - let released_a = manager.release_stream_by_uid(&addr, stream_a.uid).await; - assert!(released_a.is_some(), "releasing with stream_uid succeeds"); - - // Now only user_b remains on addr. Releasing without stream_uid should now succeed: - let released_b = manager.release_stream(&addr).await; - assert!(released_b.is_some(), "releasing without stream_uid succeeds when only a single user remains on addr"); - assert_eq!(released_b.unwrap().uid, stream_b.uid); -} - -#[tokio::test] -async fn arm_eviction_protection_protects_all_users_on_addr() { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - let manager = ActiveUserManager::new(&config, &geoip, &event_manager); - - let addr: SocketAddr = "192.168.1.100:12345".parse().unwrap(); - let protected_addr: SocketAddr = "192.168.1.101:54321".parse().unwrap(); - let fingerprint = Fingerprint::new("fp-key".to_string(), "192.168.1.100".to_string(), addr); - - manager.add_connection(&addr).await; - - manager - .update_connection(ActiveUserConnectionParams { - uid: 1, - meter_uid: 0, - username: "user_a", - max_connections: 5, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_channel(2001), - user_agent: Cow::Borrowed("ua"), - session_token: None, - }) - .await - .expect("register user_a"); - - manager - .update_connection(ActiveUserConnectionParams { - uid: 2, - meter_uid: 0, - username: "user_b", - max_connections: 5, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider-b".intern(), - stream_channel: &test_channel(2002), - user_agent: Cow::Borrowed("ua"), - session_token: None, - }) - .await - .expect("register user_b"); - - manager.mark_recent_eviction_guard_for_addr(&addr, protected_addr, Duration::from_secs(60)).await; - - let connections = manager.connections.read().await; - let key_a = create_socket_reentry_guard_key("user_a", "192.168.1.100", shared::model::VirtualId::new(2001)); - let key_b = create_socket_reentry_guard_key("user_b", "192.168.1.100", shared::model::VirtualId::new(2002)); - - assert!(connections.recent_socket_reentry_guards.contains_key(&key_a), "user_a guard must be armed"); - assert!(connections.recent_socket_reentry_guards.contains_key(&key_b), "user_b guard must be armed"); -} - -fn provider_header_test_manager() -> ActiveUserManager { - let config = Config::default(); - let geoip = Arc::new(ArcSwapOption::::default()); - let event_manager = Arc::new(EventManager::new()); - ActiveUserManager::new(&config, &geoip, &event_manager) -} - -async fn create_provider_header_session(manager: &ActiveUserManager, user: &ProxyUserCredentials, url: &str) { - let addr: SocketAddr = "127.0.0.1:55420".parse().unwrap_or_else(|_| unreachable!()); - manager - .create_user_session(CreateUserSessionParams { - user, - session_token: "tok-host", - virtual_id: 7010, - provider: "provider-a", - stream_url: url, - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; -} - -/// Provider cookies survive child playlist changes on the same host and are only sent to that host. -#[tokio::test] -async fn provider_session_headers_are_scoped_to_the_host_that_set_them() { - let manager = provider_header_test_manager(); - let mut user = ProxyUserCredentials::default(); - user.username = "user-provider-header-host".to_string(); - let headers = HashMap::from([(String::from("cookie"), String::from("sid=abc"))]); - - create_provider_header_session(&manager, &user, "http://cdn.example/a/video.m3u8").await; - assert!(store_cookie_header(&manager, &user.username, "sid=abc", "http://cdn.example/a/video.m3u8").await); - create_provider_header_session(&manager, &user, "http://cdn.example/a/audio.m3u8").await; - - let session = manager.get_and_update_user_session(&user.username, "tok-host").await.expect("session exists"); - assert_eq!(session.provider_session_headers_for("http://cdn.example/a/video.m3u8").as_deref(), Some(&headers)); - assert_eq!(session.provider_session_headers_for("http://entry.example/live/1.m3u8"), None); - assert_eq!( - session.provider_session_headers_for("https://cdn.example:80/a/video.m3u8"), - None, - "same host and port with another scheme is another origin" - ); - - create_provider_header_session(&manager, &user, "http://other.example/a/video.m3u8").await; - let session = manager.get_and_update_user_session(&user.username, "tok-host").await.expect("session exists"); - assert!(session.provider_session_headers.is_empty()); -} - -/// Kicked and evicted sessions are marked ended, also with the reentry guard disabled. -#[tokio::test] -async fn kicked_and_evicted_sessions_are_marked_ended() { - let manager = provider_header_test_manager(); - let mut user = ProxyUserCredentials::default(); - user.username = "user-ended".to_string(); - create_provider_header_session(&manager, &user, "http://cdn.example/a/video.m3u8").await; - let addr: SocketAddr = "127.0.0.1:55420".parse().unwrap_or_else(|_| unreachable!()); - - assert!(!manager.is_session_ended("tok-host").await); - assert!(manager.terminate_session(&user.username, "tok-host").await); - assert!(!manager.is_session_ended("tok-host").await, "a plain terminate (e.g. manifest failure) is not an end"); - - create_provider_header_session(&manager, &user, "http://cdn.example/a/video.m3u8").await; - manager.terminate_sessions_for_addr(&user.username, &addr).await; - assert!(manager.is_session_ended("tok-host").await); - - manager.mark_session_ended("tok-explicit").await; - assert!(manager.is_session_ended("tok-explicit").await); -} - -async fn register_guarded_resource( - manager: &ActiveUserManager, - user: &ProxyUserCredentials, - token: &str, - uid: u32, - requirement: &PlaybackSessionRegistration, -) -> Option { - let addr = SocketAddr::from(([127, 0, 0, 1], 55500 + u16::try_from(uid).ok()?)); - let fingerprint = Fingerprint::new("guarded-player".to_string(), "127.0.0.1".to_string(), addr); - manager - .update_connection_with_session_registration( - ActiveUserConnectionParams { - uid, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: ConnectionKind::Normal, - priority: user.priority, - soft_priority: user.soft_priority, - fingerprint: &fingerprint, - provider: "provider-a".intern(), - stream_channel: &test_adaptive_channel(7010), - user_agent: Cow::Borrowed("guarded-player"), - session_token: Some(token), - }, - Some(requirement), - ) - .await -} - -#[tokio::test] -async fn guarded_resource_registration_rejects_removed_and_replaced_sessions() -> Result<(), Box> -{ - let manager = provider_header_test_manager(); - let mut user = ProxyUserCredentials::default(); - user.username = "registration-identity".to_string(); - user.max_connections = 1; - create_provider_header_session(&manager, &user, "http://provider.example/index.m3u8").await; - let identity = session_identity(&manager, &user.username, "tok-host").await.ok_or("session missing")?; - let requirement = PlaybackSessionRegistration { identity, enforce_limits: true, grace_admitted: false }; - assert!(manager.terminate_session(&user.username, "tok-host").await); - assert!(register_guarded_resource(&manager, &user, "tok-host", 1, &requirement).await.is_none()); - assert_eq!(manager.user_connections(&user.username).await, 0); - assert!(manager.connections.read().await.key_by_addr.is_empty(), "rejection must not mutate socket registrations"); - create_provider_header_session(&manager, &user, "http://provider.example/index.m3u8").await; - assert!(register_guarded_resource(&manager, &user, "tok-host", 2, &requirement).await.is_none()); - let current = PlaybackSessionRegistration { - identity: session_identity(&manager, &user.username, "tok-host").await.ok_or("replacement missing")?, - ..requirement - }; - assert!(register_guarded_resource(&manager, &user, "tok-host", 3, ¤t).await.is_some()); - assert!(register_guarded_resource(&manager, &user, "tok-host", 4, ¤t).await.is_some()); - assert_eq!(manager.user_connections(&user.username).await, 1, "parallel resources share one user admission"); - Ok(()) -} - -#[tokio::test] -async fn session_end_wakes_waiters_without_polling() -> Result<(), Box> { - let manager = Arc::new(provider_header_test_manager()); - let mut user = ProxyUserCredentials::default(); - user.username = "session-end-waiter".to_string(); - create_provider_header_session(&manager, &user, "http://provider.example/index.m3u8").await; - let identity = session_identity(&manager, &user.username, "tok-host").await.ok_or("session missing")?; - let waiter = { - let manager = Arc::clone(&manager); - let username = user.username.clone(); - tokio::spawn(async move { manager.wait_for_playback_session_end(&username, "tok-host", identity).await }) - }; - tokio::task::yield_now().await; - assert!(!waiter.is_finished(), "a current session must keep the waiter pending"); - assert!(manager.terminate_session(&user.username, "tok-host").await); - tokio::time::timeout(Duration::from_secs(1), waiter).await??; - Ok(()) -} - -#[tokio::test] -async fn account_switch_ends_resource_identity_and_rejects_stale_cookie_updates( -) -> Result<(), Box> { - let manager = Arc::new(provider_header_test_manager()); - let mut user = ProxyUserCredentials::default(); - user.username = "session-binding".to_string(); - let url = "http://provider.example/index.m3u8"; - create_provider_header_session(&manager, &user, url).await; - let identity = session_identity(&manager, &user.username, "tok-host").await.ok_or("session missing")?; - let cookie = |value: &str| crate::ProviderSessionHeaders { - headers: HashMap::new(), - cookies: vec![format!("sid={value}; Path=/")], - }; - assert!( - manager - .update_current_session_provider_response_headers_from( - &user.username, - "tok-host", - identity, - &cookie("old"), - url - ) - .await - ); - // Rotation while a resource waits is visible to the send-time lookup. - assert!( - manager.update_session_provider_response_headers_from(&user.username, "tok-host", &cookie("fresh"), url).await - ); - assert_eq!( - manager.current_session_provider_headers(&user.username, "tok-host", identity, url).await, - SessionProviderHeaders::Headers(HashMap::from([("cookie".to_string(), "sid=fresh".to_string())])) - ); - assert_eq!( - manager.current_session_provider_headers(&user.username, "tok-host", identity, "http://other.example/x").await, - SessionProviderHeaders::NoHeaders, - "a valid session without headers for the target is not a missing session" - ); - - let notify = { - let users = manager.connections.read().await; - let (_, session) = users.current_session(&user.username, "tok-host", identity).ok_or("session missing")?; - Arc::clone(&session.change_signal.notify) - }; - let woken = notify.notified(); - tokio::pin!(woken); - woken.as_mut().enable(); - // Writes that keep the session and binding, of this or another user, wake nobody. - manager.update_session_addr(&user.username, "tok-host", &"127.0.0.1:55499".parse()?).await; - let mut other = ProxyUserCredentials::default(); - other.username = "session-binding-other".to_string(); - let other_addr: SocketAddr = "127.0.0.1:55498".parse()?; - manager - .create_user_session(CreateUserSessionParams { - user: &other, - session_token: "tok-other", - virtual_id: 7012, - provider: "provider-a", - stream_url: url, - addr: &other_addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - assert!(manager.terminate_session(&other.username, "tok-other").await); - assert!(woken.as_mut().now_or_never().is_none(), "unrelated writes must not wake the waiter"); - let waiter = { - let manager = Arc::clone(&manager); - let username = user.username.clone(); - tokio::spawn(async move { manager.wait_for_playback_session_end(&username, "tok-host", identity).await }) - }; - tokio::task::yield_now().await; - assert!(!waiter.is_finished()); - manager.update_session_provider_binding(&user.username, "tok-host", "provider-b".intern(), url.intern()).await; - tokio::time::timeout(Duration::from_secs(1), waiter).await??; - assert!(woken.as_mut().now_or_never().is_some(), "the account switch wakes the waiter"); - assert!(!manager.playback_session_is_current(&user.username, "tok-host", identity).await); - assert_eq!( - manager.current_session_provider_headers(&user.username, "tok-host", identity, url).await, - SessionProviderHeaders::NoSession - ); - assert!( - !manager - .update_current_session_provider_response_headers_from( - &user.username, - "tok-host", - identity, - &cookie("stale"), - url - ) - .await, - "responses from the previous account must not enter the new account's jar" - ); - let switched = session_identity(&manager, &user.username, "tok-host").await.ok_or("session missing")?; - assert_eq!(switched.incarnation, identity.incarnation); - assert_ne!(switched.binding_generation, identity.binding_generation); - Ok(()) -} - -#[tokio::test] -async fn guarded_resource_registration_rechecks_user_capacity_at_commit() -> Result<(), Box> { - let manager = provider_header_test_manager(); - let mut user = ProxyUserCredentials::default(); - user.username = "registration-capacity".to_string(); - user.max_connections = 1; - create_provider_header_session(&manager, &user, "http://provider.example/index.m3u8").await; - let first = PlaybackSessionRegistration { - identity: session_identity(&manager, &user.username, "tok-host").await.ok_or("session missing")?, - enforce_limits: true, - grace_admitted: false, - }; - let addr = SocketAddr::from(([127, 0, 0, 1], 55421)); - manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "second-session", - virtual_id: 7011, - provider: "provider-a", - stream_url: "http://provider.example/other.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(ConnectionKind::Normal), - socket_bound: false, - }) - .await; - let second = PlaybackSessionRegistration { - identity: session_identity(&manager, &user.username, "second-session").await.ok_or("second session missing")?, - enforce_limits: true, - grace_admitted: false, - }; - assert!(register_guarded_resource(&manager, &user, "tok-host", 5, &first).await.is_some()); - assert!(register_guarded_resource(&manager, &user, "second-session", 6, &second).await.is_none()); - assert_eq!(manager.user_connections(&user.username).await, 1); - assert!( - manager.playback_session_is_current(&user.username, "second-session", second.identity).await, - "capacity rejection does not terminate an otherwise valid session" - ); - Ok(()) -} - -#[tokio::test] -async fn provider_session_cookies_survive_origin_changes_and_clear_on_account_switch( -) -> Result<(), Box> { - let manager = provider_header_test_manager(); - let mut user = ProxyUserCredentials::default(); - user.username = "cookie-account".to_string(); - let entry = "https://entry.example/index.m3u8"; - let cdn = "https://cdn.example/video/init.mp4"; - create_provider_header_session(&manager, &user, entry).await; - for (url, cookie) in [(entry, "sid=entry"), (cdn, "sid=cdn")] { - assert!(store_cookie_header(&manager, &user.username, cookie, url).await); - } - create_provider_header_session(&manager, &user, cdn).await; - let session = manager.get_and_update_user_session(&user.username, "tok-host").await.ok_or("session missing")?; - assert_eq!( - session.provider_session_headers_for(entry).and_then(|headers| headers.get("cookie").cloned()).as_deref(), - Some("sid=entry") - ); - assert_eq!( - session.provider_session_headers_for(cdn).and_then(|headers| headers.get("cookie").cloned()).as_deref(), - Some("sid=cdn") - ); - manager.update_session_provider_binding(&user.username, "tok-host", "provider-b".intern(), entry.intern()).await; - let session = manager.get_and_update_user_session(&user.username, "tok-host").await.ok_or("session missing")?; - assert!(session.provider_session_cookies.is_empty()); - assert!(session.provider_session_headers_for(entry).is_none()); - assert!(session.provider_session_headers_for(cdn).is_none()); - Ok(()) -} - -/// Stores the pairs of a `Cookie` header as origin-wide provider cookies set by `source_url`. -async fn store_cookie_header( - manager: &ActiveUserManager, - username: &str, - cookie_header: &str, - source_url: &str, -) -> bool { - let response = crate::ProviderSessionHeaders { - headers: HashMap::new(), - cookies: cookie_header - .split(';') - .map(str::trim) - .filter(|pair| !pair.is_empty()) - .map(|pair| format!("{pair}; Path=/")) - .collect(), - }; - manager.update_session_provider_response_headers_from(username, "tok-host", &response, source_url).await -} - -async fn session_identity(manager: &ActiveUserManager, username: &str, token: &str) -> Option { - let users = manager.connections.read().await; - users.by_key.get(username)?.sessions.iter().find(|session| session.token == token).map(UserSession::identity) -} diff --git a/backend/session/src/active_user_manager/tests/adaptive_streams.rs b/backend/session/src/active_user_manager/tests/adaptive_streams.rs new file mode 100644 index 000000000..7d43d4887 --- /dev/null +++ b/backend/session/src/active_user_manager/tests/adaptive_streams.rs @@ -0,0 +1,1024 @@ +use super::{ + assert_connection_ownership_invariants, assert_no_real_connection_slots, assert_preserved_session_is_uncounted, + commit_and_preserve_adaptive_session, test_adaptive_channel, test_channel, test_user_credentials, + ActiveUserConnectionParams, ActiveUserManager, CreateUserSessionParams, PlaybackLifecycle, UserConnectionData, +}; +use crate::{active_provider_manager::ConnectionKind, EventManager}; +use arc_swap::ArcSwapOption; +use shared::{ + model::{PlaylistItemType, StreamChannel, UserConnectionPermission}, + utils::{current_time_secs, Internable}, +}; +use std::{borrow::Cow, net::SocketAddr, sync::Arc}; +use tuliprox_core::{ + model::{Config, Fingerprint, ProxyUserCredentials}, + utils::utc_day_from_secs, +}; +use tuliprox_repository::GeoIp; + +#[tokio::test] +async fn terminate_session_removes_preserved_adaptive_stream() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55412".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-terminate-preserved".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = "user-terminate-preserved".to_string(); + user.max_connections = 1; + + let token = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-terminate-preserved", + virtual_id: 8003, + provider: "provider-terminate-preserved", + stream_url: "http://localhost/test.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + manager.add_connection(&addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 8003, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-terminate-preserved".intern(), + stream_channel: &test_adaptive_channel(8003), + user_agent: Cow::Borrowed("ua"), + session_token: Some(&token), + }) + .await + .expect("adaptive stream should be registered"); + + let released = manager.release_connection(&addr).await; + assert!(released.addr_removed); + assert!(released.removed_streams.is_empty(), "adaptive stream should be preserved first"); + + manager.terminate_session(&user.username, &token).await; + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user data should remain inspectable"); + assert!(connection_data.streams.is_empty(), "terminating a session must remove its preserved adaptive stream"); + assert!(connection_data.sessions.iter().all(|session| session.token != token)); +} + +pub(in crate::active_user_manager::tests) fn assert_active_stream_kind( + connection_data: &UserConnectionData, + stream_uid: u32, + expected_kind: ConnectionKind, +) { + assert!( + connection_data.streams.iter().any(|stream| stream.uid == stream_uid && !stream.preserved), + "stream {stream_uid} must remain active" + ); + assert_eq!(connection_data.stream_kinds.get(&stream_uid), Some(&expected_kind)); +} + +pub(in crate::active_user_manager::tests) fn assert_single_normal_stream_slot( + connection_data: &UserConnectionData, + stream_uid: u32, +) { + assert_eq!(connection_data.connections, 1); + assert_eq!(connection_data.counts.normal, 1); + assert_eq!(connection_data.counts.soft, 0); + assert_active_stream_kind(connection_data, stream_uid, ConnectionKind::Normal); + assert_connection_ownership_invariants(connection_data); +} + +#[tokio::test] +async fn eviction_candidates_include_preserved_adaptive_streams() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55043".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-preserved".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("same-user"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-preserved", + virtual_id: 1043, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 43, + meter_uid: 0, + username: "same-user", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(1043) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-preserved"), + }) + .await; + + let released = manager.release_connection(&addr).await; + assert!(released.addr_removed); + assert!(released.removed_streams.is_empty(), "adaptive stream should stay logically active"); + + let candidates = manager.get_eviction_candidates("same-user", "127.0.0.1").await; + assert_eq!(candidates.len(), 1, "preserved adaptive streams must remain evictable"); + assert_eq!(candidates[0].addr, addr); +} + +#[tokio::test] +async fn test_kicked_release_does_not_preserve_adaptive_stream() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55014".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-adaptive".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user-adaptive"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-adaptive", + virtual_id: 2014, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 14, + meter_uid: 0, + username: &user.username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(2014), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-adaptive"), + }) + .await; + + let removed = manager.release_connection_as_kicked(&addr).await; + assert!(removed.addr_removed); + assert_eq!(removed.removed_streams.len(), 1); + assert!(manager.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn test_reused_logical_stream_refreshes_normal_priority() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55023".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key-2a".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-prio", + virtual_id: 2002, + provider: "provider-a", + stream_url: "http://localhost/live-prio.ts", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Soft), + socket_bound: true, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 201, + meter_uid: 0, + username: "user1", + max_connections: 1, + soft_connections: 1, + connection_kind: ConnectionKind::Soft, + priority: 8, + soft_priority: 8, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(2002), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-prio"), + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 201, + meter_uid: 0, + username: "user1", + max_connections: 1, + soft_connections: 1, + connection_kind: ConnectionKind::Soft, + priority: -7, + soft_priority: 8, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(2002), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-prio"), + }) + .await; + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get("user1").unwrap(); + assert_eq!(connection_data.stream_normal_priorities.get(&201), Some(&-7)); +} + +#[tokio::test] +async fn test_adaptive_session_release_connection_preserves_logical_stream_and_start_time() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55041".parse().unwrap(); + let next_addr: SocketAddr = "127.0.0.1:55042".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key-4".to_string(), "127.0.0.1".to_string(), addr); + let next_fingerprint = Fingerprint::new("fp-key-5".to_string(), "127.0.0.1".to_string(), next_addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-hls", + virtual_id: 4001, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + let first = manager + .update_connection(ActiveUserConnectionParams { + uid: 44, + meter_uid: 144, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(4001) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-hls"), + }) + .await + .expect("initial adaptive session should register"); + + let released = manager.release_connection(&addr).await; + assert!(released.addr_removed); + assert!(released.removed_streams.is_empty(), "adaptive session should remain logically active"); + assert_eq!(manager.user_connections("user1").await, 0); + assert_eq!(manager.active_users_and_connections().await, (0, 0)); + assert!(manager.active_streams().await.is_empty()); + + let connections = manager.connections.read().await; + let preserved_stream = connections + .by_key + .get("user1") + .and_then(|data| data.streams.iter().find(|stream| stream.uid == 44)) + .expect("preserved adaptive stream should stay internally tracked"); + assert_eq!(preserved_stream.ts, first.ts); + assert!(preserved_stream.preserved); + drop(connections); + + manager.add_connection(&next_addr).await; + let second = manager + .update_connection(ActiveUserConnectionParams { + uid: 55, + meter_uid: 155, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &next_fingerprint, + provider: "provider-b".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveDash, ..test_channel(4002) }, + user_agent: Cow::Borrowed("ua-2"), + session_token: Some("tok-hls"), + }) + .await + .expect("adaptive session should reuse logical stream"); + + assert_eq!(second.uid, 44); + assert_eq!(second.ts, first.ts, "adaptive session duration must stay session-based"); + assert_eq!(second.addr, next_addr); + assert_eq!(second.meter_uid, 155); + assert_eq!(manager.user_connections("user1").await, 1); + + let streams = manager.active_streams().await; + assert_eq!(streams.len(), 1); + assert!(!streams[0].preserved); +} + +#[tokio::test] +async fn test_release_stream_ignores_preserved_adaptive_entry() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55051".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key-6".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-hls", + virtual_id: 5001, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 66, + meter_uid: 166, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(5001) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-hls"), + }) + .await; + + let released = manager.release_connection(&addr).await; + assert!(released.addr_removed); + assert!(released.removed_streams.is_empty()); + assert!(manager.release_stream(&addr).await.is_none()); +} + +#[tokio::test] +async fn test_release_stream_without_session_removes_adaptive_stream_instead_of_preserving() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55082".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key-9".to_string(), "127.0.0.1".to_string(), addr); + + manager.add_connection(&addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 122, + meter_uid: 222, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8002) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("missing-session"), + }) + .await; + + let released = manager.release_stream(&addr).await; + assert!(released.is_some(), "stream without schedulable expiry must be removed"); + assert!(manager.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn test_preserved_adaptive_stream_reconnect_across_day_sets_previous_session_id() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55085".parse().unwrap(); + let next_addr: SocketAddr = "127.0.0.1:55086".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-rollover-a".to_string(), "127.0.0.1".to_string(), addr); + let next_fingerprint = Fingerprint::new("fp-rollover-b".to_string(), "127.0.0.1".to_string(), next_addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-rollover", + virtual_id: 8005, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + let first = manager + .update_connection(ActiveUserConnectionParams { + uid: 145, + meter_uid: 245, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8005) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-rollover"), + }) + .await + .expect("initial adaptive session should register"); + + let released = manager.release_connection(&addr).await; + assert!(released.addr_removed); + + let forced_old_ts = { + let mut connections = manager.connections.write().await; + let stream = connections + .by_key + .get_mut("user1") + .unwrap() + .streams + .iter_mut() + .find(|stream| stream.session_token.as_deref() == Some("tok-rollover")) + .unwrap(); + stream.ts = stream.ts.saturating_sub(86_400); + stream.ts + }; + + manager.add_connection(&next_addr).await; + let second = manager + .update_connection(ActiveUserConnectionParams { + uid: 146, + meter_uid: 246, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &next_fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveDash, ..test_channel(8005) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-rollover"), + }) + .await + .expect("adaptive session should reconnect"); + + assert_eq!(second.previous_session_id, Some((forced_old_ts << 32) | u64::from(first.uid))); + assert!(second.ts > forced_old_ts); + assert_eq!(utc_day_from_secs(second.ts), utc_day_from_secs(current_time_secs())); +} + +#[tokio::test] +async fn catchup_release_connection_preserves_logical_stream_until_session_expires() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55141".parse().unwrap(); + let next_addr: SocketAddr = "127.0.0.1:55142".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-catchup-1".to_string(), "127.0.0.1".to_string(), addr); + let next_fingerprint = Fingerprint::new("fp-catchup-2".to_string(), "127.0.0.1".to_string(), next_addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user-catchup"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-catchup", + virtual_id: 9103, + provider: "provider-a", + stream_url: "http://localhost/archive.ts", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + let first = manager + .update_connection(ActiveUserConnectionParams { + uid: 304, + meter_uid: 404, + username: &user.username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::Catchup, ..test_channel(9103) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-catchup"), + }) + .await + .expect("initial catchup stream should register"); + + let released = manager.release_connection(&addr).await; + assert!(released.addr_removed); + assert!( + released.removed_streams.is_empty(), + "catchup stream should remain logically active between range requests" + ); + + assert_eq!(manager.user_connections(&user.username).await, 0); + assert!(manager.active_streams().await.is_empty()); + + let connections = manager.connections.read().await; + let preserved_stream = connections + .by_key + .get(&user.username) + .and_then(|data| data.streams.iter().find(|stream| stream.uid == first.uid)) + .expect("preserved catchup stream should stay internally tracked"); + assert!(preserved_stream.preserved); + drop(connections); + + manager.add_connection(&next_addr).await; + let second = manager + .update_connection(ActiveUserConnectionParams { + uid: 305, + meter_uid: 405, + username: &user.username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &next_fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::Catchup, ..test_channel(9103) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-catchup"), + }) + .await + .expect("catchup stream should reconnect"); + + assert_eq!(second.uid, first.uid); + assert_eq!(second.started_at, first.started_at); + assert!(!second.preserved); +} + +#[tokio::test] +async fn preserved_reactivation_admission_does_not_create_ownerless_counted_slot() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let user = test_user_credentials("user-preserved-activation", 1, 0); + let addr: SocketAddr = "127.0.0.1:55195".parse().unwrap(); + let session_token = "tok-preserved-activation"; + let stream_uid = 501; + + commit_and_preserve_adaptive_session(&manager, &user, session_token, stream_uid, addr, ConnectionKind::Normal) + .await; + + let virtual_admission = + manager.connection_admission(&user.username, user.max_connections, user.soft_connections).await; + assert_eq!(virtual_admission.permission(), UserConnectionPermission::Exhausted); + + let admission = manager + .connection_admission_for_session_activation( + &user.username, + user.max_connections, + user.soft_connections, + session_token, + ) + .await; + + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(admission.kind(), Some(ConnectionKind::Normal)); + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_preserved_session_is_uncounted(connection_data, session_token, stream_uid); + assert_no_real_connection_slots(connection_data); +} + +#[tokio::test] +async fn preserved_reactivation_admission_then_kicked_release_removes_state_without_ghost_counter() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let user = test_user_credentials("user-preserved-eviction", 1, 0); + let addr: SocketAddr = "127.0.0.1:55201".parse().unwrap(); + let session_token = "tok-preserved-eviction"; + let stream_uid = 601; + + commit_and_preserve_adaptive_session(&manager, &user, session_token, stream_uid, addr, ConnectionKind::Normal) + .await; + let admission = manager + .connection_admission_for_session_activation( + &user.username, + user.max_connections, + user.soft_connections, + session_token, + ) + .await; + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + { + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_preserved_session_is_uncounted(connection_data, session_token, stream_uid); + assert_no_real_connection_slots(connection_data); + } + + let released = manager.release_connection_as_kicked(&addr).await; + assert!(released.addr_removed); + assert_eq!(released.removed_streams.len(), 1); + let removed_stream = released.removed_streams.first().expect("kicked release must remove the preserved stream"); + assert_eq!(removed_stream.uid, stream_uid); + assert!(removed_stream.preserved); + assert_eq!(removed_stream.session_token.as_deref(), Some(session_token)); + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert!(connection_data.streams.iter().all(|stream| stream.uid != stream_uid)); + assert!(connection_data.sessions.iter().all(|session| session.token != session_token)); + assert!(!connection_data.stream_kinds.contains_key(&stream_uid)); + assert_no_real_connection_slots(connection_data); + drop(connections); + assert_eq!(manager.active_users_and_connections().await, (0, 0)); +} + +#[tokio::test] +async fn preserved_reactivation_admission_then_lease_idle_cleanup_leaves_counters_at_zero() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let user = test_user_credentials("user-preserved-idle-cleanup", 1, 0); + let addr: SocketAddr = "127.0.0.1:55202".parse().unwrap(); + let session_token = "tok-preserved-idle-cleanup"; + let stream_uid = 602; + + commit_and_preserve_adaptive_session(&manager, &user, session_token, stream_uid, addr, ConnectionKind::Normal) + .await; + let admission = manager + .connection_admission_for_session_activation( + &user.username, + user.max_connections, + user.soft_connections, + session_token, + ) + .await; + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + { + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_preserved_session_is_uncounted(connection_data, session_token, stream_uid); + assert_no_real_connection_slots(connection_data); + } + + // Shared-HLS lease-idle cleanup delegates to this manager operation. + let counter_changed = manager.release_session_streams_and_counted_reservation(&user.username, session_token).await; + assert!(!counter_changed, "removing an uncounted preserved stream must not change real counters"); + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert!(connection_data.streams.iter().all(|stream| stream.uid != stream_uid)); + assert!(connection_data + .sessions + .iter() + .find(|session| session.token == session_token) + .is_some_and(|session| session.lifecycle == PlaybackLifecycle::Preserved)); + assert!(!connection_data.stream_kinds.contains_key(&stream_uid)); + assert_no_real_connection_slots(connection_data); + drop(connections); + assert_eq!(manager.active_users_and_connections().await, (0, 0)); +} + +#[tokio::test] +async fn repeated_preserved_reactivation_cleanup_does_not_accumulate_connections() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let user = test_user_credentials("user-preserved-repeat", 4, 0); + for (session_token, stream_uid, addr) in + [("tok-preserved-repeat-one", 603, "127.0.0.1:55203"), ("tok-preserved-repeat-two", 604, "127.0.0.1:55204")] + { + let addr = addr.parse().unwrap(); + commit_and_preserve_adaptive_session(&manager, &user, session_token, stream_uid, addr, ConnectionKind::Normal) + .await; + + let admission = manager + .connection_admission_for_session_activation( + &user.username, + user.max_connections, + user.soft_connections, + session_token, + ) + .await; + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + + let counter_changed = + manager.release_session_streams_and_counted_reservation(&user.username, session_token).await; + assert!(!counter_changed, "cleanup must not release a slot that was never committed"); + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_no_real_connection_slots(connection_data); + drop(connections); + assert_eq!(manager.active_users_and_connections().await, (0, 0)); + } +} + +#[tokio::test] +async fn preserved_soft_reactivation_and_cleanup_leave_normal_slot_unchanged() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let user = test_user_credentials("user-preserved-soft", 1, 1); + let normal_addr: SocketAddr = "127.0.0.1:55206".parse().unwrap(); + let normal_fingerprint = + Fingerprint::new("fp-preserved-soft-normal".to_string(), normal_addr.ip().to_string(), normal_addr); + let soft_addr: SocketAddr = "127.0.0.1:55207".parse().unwrap(); + let normal_stream_uid = 606; + let session_token = "tok-preserved-soft"; + let soft_stream_uid = 607; + + manager.add_connection(&normal_addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: normal_stream_uid, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Normal, + priority: user.priority, + soft_priority: user.soft_priority, + fingerprint: &normal_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(normal_stream_uid), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("normal stream should bind"); + { + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_single_normal_stream_slot(connection_data, normal_stream_uid); + } + + commit_and_preserve_adaptive_session( + &manager, + &user, + session_token, + soft_stream_uid, + soft_addr, + ConnectionKind::Soft, + ) + .await; + { + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_preserved_session_is_uncounted(connection_data, session_token, soft_stream_uid); + assert_single_normal_stream_slot(connection_data, normal_stream_uid); + } + + let admission = manager + .connection_admission_for_session_activation( + &user.username, + user.max_connections, + user.soft_connections, + session_token, + ) + .await; + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(admission.kind(), Some(ConnectionKind::Soft)); + + { + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_preserved_session_is_uncounted(connection_data, session_token, soft_stream_uid); + assert_single_normal_stream_slot(connection_data, normal_stream_uid); + } + + let counter_changed = manager.release_session_streams_and_counted_reservation(&user.username, session_token).await; + assert!(!counter_changed, "preserved soft cleanup must not release an uncommitted slot"); + { + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert!(connection_data.streams.iter().all(|stream| stream.uid != soft_stream_uid)); + assert!(!connection_data.stream_kinds.contains_key(&soft_stream_uid)); + assert!(connection_data + .sessions + .iter() + .find(|session| session.token == session_token) + .is_some_and(|session| session.lifecycle == PlaybackLifecycle::Preserved)); + assert_single_normal_stream_slot(connection_data, normal_stream_uid); + } + + manager.release_stream_by_uid(&normal_addr, normal_stream_uid).await.expect("normal stream should release"); + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert!(connection_data.streams.is_empty()); + assert!(connection_data.stream_kinds.is_empty()); + assert_no_real_connection_slots(connection_data); + drop(connections); + assert_eq!(manager.active_users_and_connections().await, (0, 0)); +} + +#[tokio::test] +async fn get_eviction_candidates_keeps_preserved_streams_evictable() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55300".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key".to_string(), "192.168.1.100".to_string(), addr); + let username = "user-eviction-addr"; + let mut user = ProxyUserCredentials::default(); + user.username = username.to_string(); + user.max_connections = 1; + user.soft_connections = 0; + + // Create session first (HLS type = preserved after release) + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-preserved-1", + virtual_id: 5001, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + // Create stream + register connection + manager.add_connection(&addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 1, + meter_uid: 0, + username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(5001), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-preserved-1"), + }) + .await + .expect("first stream"); + assert_eq!(manager.user_connections(username).await, 1); + + // Release -> stream becomes preserved, session becomes uncounted + manager.release_stream(&addr).await; + assert_eq!(manager.user_connections(username).await, 0, "preserved stream should not count"); + + let candidates = manager.get_eviction_candidates(username, "192.168.1.100").await; + assert!( + candidates.iter().any(|candidate| candidate.addr == addr), + "preserved stream should remain a direct eviction candidate" + ); +} + +#[tokio::test] +async fn connection_admission_treats_preserved_stream_as_reserved_capacity() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55305".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-preserved-admission".to_string(), "192.168.1.100".to_string(), addr); + let username = "user-preserved-admission"; + let mut user = ProxyUserCredentials::default(); + user.username = username.to_string(); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-preserved-admission", + virtual_id: 6000, + provider: "provider-a", + stream_url: "http://localhost/live-preserved.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 6000, + meter_uid: 0, + username, + max_connections: user.max_connections, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(6000), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-preserved-admission"), + }) + .await + .expect("preserved stream should be created"); + + manager.release_connection(&addr).await; + assert_eq!(manager.user_connections(username).await, 0, "preserved stream stays uncounted for active snapshots"); + + let admission = manager.connection_admission(username, user.max_connections, 0).await; + assert_eq!( + admission.permission(), + UserConnectionPermission::Exhausted, + "a preserved stream should still reserve capacity against unrelated playback admissions" + ); +} diff --git a/backend/session/src/active_user_manager/tests/admission.rs b/backend/session/src/active_user_manager/tests/admission.rs new file mode 100644 index 000000000..06039af7d --- /dev/null +++ b/backend/session/src/active_user_manager/tests/admission.rs @@ -0,0 +1,763 @@ +use super::{ + test_adaptive_channel, test_channel, test_user_credentials, ActiveUserConnectionParams, ActiveUserManager, + CreateUserSessionParams, PendingProviderReason, PendingProviderWakeSource, PlaybackLifecycle, +}; +use crate::{active_provider_manager::ConnectionKind, EventManager}; +use arc_swap::ArcSwapOption; +use shared::{ + model::UserConnectionPermission, + utils::{current_time_secs, Internable}, +}; +use std::{borrow::Cow, net::SocketAddr, sync::Arc}; +use tuliprox_core::model::{Config, Fingerprint, ProxyUserCredentials}; +use tuliprox_repository::GeoIp; + +/// `terminate_session` releases counted lease. +#[tokio::test] +async fn terminate_session_releases_counted_lease() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55411".parse().unwrap(); + let mut user = ProxyUserCredentials::default(); + user.username = "user-terminate-counted".to_string(); + user.max_connections = 2; + + let token = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-terminate-counted", + virtual_id: 8002, + provider: "provider-terminate-counted", + stream_url: "http://localhost/test.ts", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + // Mark the session as counted and active (simulating post-admission state). + { + let mut connections = manager.connections.write().await; + let data = connections.by_key.get_mut(&user.username).unwrap(); + let session = data.sessions.iter_mut().find(|s| s.token == token).unwrap(); + // Simulate counted state by setting lifecycle to Active. + session.lifecycle = PlaybackLifecycle::Active; + data.increment_kind(ConnectionKind::Normal); + } + + // Verify counted before terminate. + { + let before = manager.get_and_update_user_session(&user.username, &token).await.unwrap(); + assert!(before.lifecycle.is_counted(), "session should be counted before terminate"); + } + + // Terminate. + manager.terminate_session(&user.username, &token).await; + + // Session should be gone. + let after = manager.get_and_update_user_session(&user.username, &token).await; + assert!(after.is_none(), "session should be removed after terminate"); +} + +#[tokio::test] +async fn expire_pending_provider_releases_counted_slot_for_pending_session() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55025".parse().unwrap_or_else(|_| unreachable!()); + let mut user = ProxyUserCredentials::default(); + user.username = "pending-expire-counted".to_string(); + user.max_connections = 1; + + let _ = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-pending-expire-counted", + virtual_id: 1005, + provider: "provider-a", + stream_url: "http://provider/live/1005.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + { + let mut connections = manager.connections.write().await; + let connection_data = + connections.by_key.get_mut(&user.username).expect("session should have created connection data"); + connection_data.increment_kind(ConnectionKind::Normal); + let session = connection_data + .sessions + .iter_mut() + .find(|session| session.token == "tok-pending-expire-counted") + .expect("session should exist"); + // Simulate a previously-counted session transitioning to PendingProvider. + // Set lifecycle to Active (is_counted() = true). The kind count is already + // incremented above via connection_data.increment_kind(). + session.lifecycle = PlaybackLifecycle::Active; + } + + assert_eq!(manager.user_connections(&user.username).await, 1); + + let version = manager + .mark_pending_provider(&user.username, "tok-pending-expire-counted", PendingProviderReason::GraceHold, 6_500) + .await + .expect("pending version should be created"); + + manager + .expire_pending_provider( + &user.username, + "tok-pending-expire-counted", + version, + PendingProviderWakeSource::Timeout, + ) + .await; + + let session = manager + .get_and_update_user_session(&user.username, "tok-pending-expire-counted") + .await + .expect("session should still exist"); + assert_eq!(session.permission, UserConnectionPermission::Exhausted); + assert!(!matches!(session.lifecycle, PlaybackLifecycle::PendingProvider { .. })); + assert!(!session.lifecycle.is_counted()); + assert_eq!(manager.user_connections(&user.username).await, 0); +} + +#[tokio::test] +async fn test_grant_grace_succeeds_at_and_above_limit_without_prior_grace() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let at_limit_addr: SocketAddr = "127.0.0.1:55011".parse().unwrap(); + let at_limit_fingerprint = Fingerprint::new("fp-limit".to_string(), "127.0.0.1".to_string(), at_limit_addr); + let over_limit_addr: SocketAddr = "127.0.0.1:55012".parse().unwrap(); + let over_limit_fingerprint = Fingerprint::new("fp-over".to_string(), "127.0.0.1".to_string(), over_limit_addr); + + manager.add_connection(&at_limit_addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 10, + meter_uid: 0, + username: "at-limit", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &at_limit_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(1010), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-limit"), + }) + .await; + + assert!(manager.grant_grace("at-limit").await); + + manager.add_connection(&over_limit_addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 11, + meter_uid: 0, + username: "over-limit", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &over_limit_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(1011), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-over-1"), + }) + .await; + manager.add_connection(&"127.0.0.1:55013".parse().unwrap()).await; + let second_fingerprint = + Fingerprint::new("fp-over-2".to_string(), "127.0.0.1".to_string(), "127.0.0.1:55013".parse().unwrap()); + manager + .update_connection(ActiveUserConnectionParams { + uid: 12, + meter_uid: 0, + username: "over-limit", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &second_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(1012), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-over-2"), + }) + .await; + + assert!(manager.grant_grace("over-limit").await); +} + +#[tokio::test] +async fn eviction_candidates_include_charged_stream_with_uncounted_session() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55034".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-switch".to_string(), "10.41.41.170".to_string(), addr); + let username = "switch-user"; + manager.add_connection(&addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 34, + meter_uid: 0, + username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(1034), + user_agent: Cow::Borrowed("ua"), + session_token: Some("missing-session"), + }) + .await; + + assert_eq!(manager.user_connections(username).await, 1); + assert_eq!(manager.connection_permission(username, 1, 0).await, UserConnectionPermission::Exhausted); + + let candidates = manager.get_eviction_candidates(username, &fingerprint.client_ip).await; + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].addr, addr); + assert_eq!(candidates[0].client_ip, fingerprint.client_ip); + + manager.release_connection_as_kicked(&addr).await; + assert_eq!(manager.connection_permission(username, 1, 0).await, UserConnectionPermission::Allowed); +} + +#[tokio::test] +async fn eviction_candidates_include_other_ips_for_user_wide_rules() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let first_addr: SocketAddr = "127.0.0.1:55041".parse().unwrap(); + let second_addr: SocketAddr = "127.0.0.1:55042".parse().unwrap(); + let first_fp = Fingerprint::new("fp-user-wide-1".to_string(), "10.0.0.1".to_string(), first_addr); + let second_fp = Fingerprint::new("fp-user-wide-2".to_string(), "10.0.0.2".to_string(), second_addr); + + manager.add_connection(&first_addr).await; + manager.add_connection(&second_addr).await; + + let user = test_user_credentials("same-user", 2, 0); + for (token, addr, channel_id) in [("tok-41", first_addr, 1041u32), ("tok-42", second_addr, 1042)] { + manager + .create_user_session(crate::CreateUserSessionParams { + user: &user, + session_token: token, + virtual_id: channel_id, + provider: "provider-a", + stream_url: "", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + } + + manager + .update_connection(ActiveUserConnectionParams { + uid: 41, + meter_uid: 0, + username: "same-user", + max_connections: 2, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &first_fp, + provider: "provider-a".intern(), + stream_channel: &test_channel(1041), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-41"), + }) + .await; + + manager + .update_connection(ActiveUserConnectionParams { + uid: 42, + meter_uid: 0, + username: "same-user", + max_connections: 2, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &second_fp, + provider: "provider-a".intern(), + stream_channel: &test_channel(1042), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-42"), + }) + .await; + + let candidates = manager.get_eviction_candidates("same-user", "10.0.0.1").await; + assert_eq!(candidates.len(), 2); + assert!(candidates.iter().any(|candidate| candidate.addr == first_addr)); + assert!(candidates.iter().any(|candidate| candidate.addr == second_addr)); +} + +#[tokio::test] +async fn test_grace_at_limit_remains_active_until_connections_drop_below_limit() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55017".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-grace".to_string(), "127.0.0.1".to_string(), addr); + + manager.add_connection(&addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 17, + meter_uid: 0, + username: "grace-at-limit", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(2017), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-grace"), + }) + .await; + + assert!(manager.grant_grace("grace-at-limit").await); + assert_eq!( + manager.connection_admission("grace-at-limit", 1, 0).await.permission, + UserConnectionPermission::Exhausted + ); + assert!(!manager.grant_grace("grace-at-limit").await); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn origin_policy_refresh_promotes_counted_soft_session_when_hard_slot_is_available() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user-hls-policy-refresh"); + user.max_connections = 1; + user.soft_connections = 1; + + let normal_addr: SocketAddr = "127.0.0.1:55185".parse().unwrap(); + let soft_addr: SocketAddr = "127.0.0.1:55186".parse().unwrap(); + let normal_fingerprint = Fingerprint::new("fp-hls-policy-1".to_string(), "127.0.0.1".to_string(), normal_addr); + let soft_fingerprint = Fingerprint::new("fp-hls-policy-2".to_string(), "127.0.0.1".to_string(), soft_addr); + + manager.add_connection(&normal_addr).await; + manager.add_connection(&soft_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-normal", + virtual_id: 9210, + provider: "provider-a", + stream_url: "http://localhost/live-normal.m3u8", + addr: &normal_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-soft", + virtual_id: 9211, + provider: "provider-a", + stream_url: "http://localhost/live-soft.m3u8", + addr: &soft_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let normal_admission = manager + .connection_admission_for_session_activation( + &user.username, + user.max_connections, + user.soft_connections, + "tok-normal", + ) + .await; + assert_eq!(normal_admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(normal_admission.kind(), Some(ConnectionKind::Normal)); + manager + .update_connection(ActiveUserConnectionParams { + uid: 411, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &normal_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(9210), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-normal"), + }) + .await + .expect("normal stream should bind"); + + let soft_admission = manager + .connection_admission_for_session_activation( + &user.username, + user.max_connections, + user.soft_connections, + "tok-soft", + ) + .await; + assert_eq!(soft_admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(soft_admission.kind(), Some(ConnectionKind::Soft)); + manager + .update_connection(ActiveUserConnectionParams { + uid: 412, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Soft, + priority: 0, + soft_priority: 0, + fingerprint: &soft_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(9211), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-soft"), + }) + .await + .expect("soft stream should bind"); + + assert!(manager.release_session_streams_and_counted_reservation(&user.username, "tok-normal").await); + { + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_eq!(connection_data.connections, 1); + assert_eq!(connection_data.counts.normal, 0); + assert_eq!(connection_data.counts.soft, 1); + assert_eq!( + connection_data + .sessions + .iter() + .find(|session| session.token == "tok-soft") + .and_then(|session| session.connection_kind), + Some(ConnectionKind::Soft) + ); + } + + let refreshed_kind = manager + .refresh_session_connection_kind_for_origin_policy( + &user.username, + user.max_connections, + user.soft_connections, + "tok-soft", + ) + .await; + assert_eq!(refreshed_kind, Some(ConnectionKind::Normal)); + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_eq!(connection_data.connections, 1); + assert_eq!(connection_data.counts.normal, 1); + assert_eq!(connection_data.counts.soft, 0); + assert_eq!( + connection_data + .sessions + .iter() + .find(|session| session.token == "tok-soft") + .and_then(|session| session.connection_kind), + Some(ConnectionKind::Normal) + ); + assert_eq!(connection_data.stream_kinds.get(&412), Some(&ConnectionKind::Normal)); +} + +#[tokio::test] +async fn origin_policy_refresh_returns_none_for_pending_grace_without_available_slot() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user-pending-grace-origin-policy"); + user.max_connections = 1; + + let active_addr: SocketAddr = "127.0.0.1:55195".parse().unwrap(); + let pending_addr: SocketAddr = "127.0.0.1:55196".parse().unwrap(); + let active_fingerprint = Fingerprint::new("active".to_string(), "127.0.0.1".to_string(), active_addr); + + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-active", + virtual_id: 9301, + provider: "provider-a", + stream_url: "http://localhost/live-active.m3u8", + addr: &active_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 9301, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &active_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(9301), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-active"), + }) + .await + .expect("active stream should bind the only normal slot"); + + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-pending", + virtual_id: 9302, + provider: "provider-a", + stream_url: "http://localhost/live-pending.m3u8", + addr: &pending_addr, + connection_permission: UserConnectionPermission::GracePeriod, + connection_kind: None, + socket_bound: false, + }) + .await; + manager + .mark_pending_provider( + &user.username, + "tok-pending", + PendingProviderReason::GraceHold, + current_time_secs() + 30, + ) + .await + .expect("pending session should be marked"); + + let refreshed_kind = manager + .refresh_session_connection_kind_for_origin_policy( + &user.username, + user.max_connections, + user.soft_connections, + "tok-pending", + ) + .await; + + assert_eq!(refreshed_kind, None); +} + +#[tokio::test] +async fn release_unbound_session_reservation_frees_reserved_slot() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user-release-reservation"); + user.max_connections = 1; + + let addr: SocketAddr = "127.0.0.1:55184".parse().unwrap(); + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-release", + virtual_id: 9205, + provider: "provider-a", + stream_url: "http://localhost/live-release.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let admission = manager + .connection_admission_for_session_activation(&user.username, user.max_connections, 0, "tok-release") + .await; + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + + manager.release_unbound_session_reservation(&user.username, "tok-release", None, false).await; + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_eq!(connection_data.connections, 0); + assert_eq!(connection_data.counts.normal, 0); + assert_eq!(connection_data.streams.len(), 0); + assert!(connection_data + .sessions + .iter() + .find(|session| session.token == "tok-release") + .is_some_and(|session| !session.lifecycle.is_counted())); +} + +#[tokio::test] +async fn release_unbound_session_reservation_ignores_stale_transition_version() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user-stale-release"); + + let addr: SocketAddr = "127.0.0.1:55194".parse().unwrap(); + let stale_version = manager + .ensure_user_session_placeholder(CreateUserSessionParams { + user: &user, + session_token: "tok-stale-release", + virtual_id: 9206, + provider: "provider-a", + stream_url: "http://localhost/live-stale.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let _ = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-stale-release", + virtual_id: 9206, + provider: "provider-b", + stream_url: "http://localhost/live-stale-updated.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + manager.release_unbound_session_reservation(&user.username, "tok-stale-release", Some(stale_version), true).await; + + let session = manager + .get_and_update_user_session(&user.username, "tok-stale-release") + .await + .expect("stale rollback must not remove the newer session"); + assert!(session.transition_version > stale_version); + assert_eq!(session.provider.as_ref(), "provider-b"); + assert_eq!(session.stream_url.as_ref(), "http://localhost/live-stale-updated.m3u8"); +} + +#[tokio::test] +async fn connection_admission_for_session_evaluates_admission_for_uncounted_session() { + // Bug: connection_admission_for_session returns Allowed for any existing session, + // even if it's uncounted (preserved). This causes strategy evaluation to be skipped. + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55310".parse().unwrap(); + let username = "user-uncounted-admission"; + let mut user = ProxyUserCredentials::default(); + user.username = username.to_string(); + user.max_connections = 1; + user.soft_connections = 0; + + // Create session + counted stream (HLS type = preserved after release) + manager.add_connection(&addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 1, + meter_uid: 0, + username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &Fingerprint::new("fp".to_string(), "192.168.1.50".to_string(), addr), + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(6001), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-uncounted"), + }) + .await + .expect("first stream"); + + // Release to preserve (uncounted session, but counts.normal still = 1 from the stream) + manager.release_stream(&addr).await; + // After preserve: session is uncounted, stream is preserved, connections=0 + // BUT the stream was removed, so counts.normal is decremented -> counts=0 + assert_eq!(manager.user_connections(username).await, 0); + + // Add a second stream first - this uses a different session token and consumes the slot + let second_addr: SocketAddr = "192.168.1.100:55311".parse().unwrap(); + manager.add_connection(&second_addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 2, + meter_uid: 0, + username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &Fingerprint::new("fp2".to_string(), "192.168.1.100".to_string(), second_addr), + provider: "provider-b".intern(), + stream_channel: &test_channel(6002), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-second"), + }) + .await + .expect("second stream"); + // Now user is at limit: connections=1, counts.normal=1, max_connections=1 + assert_eq!(manager.user_connections(username).await, 1); + + // connection_admission_for_session for the PRESERVED session token should return + // Exhausted so that eviction strategies can run and evict the preserved stream, + // freeing a slot for the uncounted session to reactivate + let admission = manager.connection_admission_for_session(username, 1, 0, "tok-uncounted").await; + assert_eq!( + admission.permission(), + UserConnectionPermission::Exhausted, + "uncounted session should not bypass admission when user is at limit; \ + bug: session exists -> Allowed -> strategy evaluation skipped" + ); +} diff --git a/backend/session/src/active_user_manager/tests/expiry_gc.rs b/backend/session/src/active_user_manager/tests/expiry_gc.rs new file mode 100644 index 000000000..db3637605 --- /dev/null +++ b/backend/session/src/active_user_manager/tests/expiry_gc.rs @@ -0,0 +1,826 @@ +use super::{ + test_adaptive_channel, test_channel, ActiveUserConnectionParams, ActiveUserManager, AdaptiveExpiryKey, + CreateUserSessionParams, DEFAULT_ACTIVE_SOCKET_TTL_SECS, USER_CON_TTL, USER_GC_TTL, +}; +use crate::{ + active_provider_manager::ConnectionKind, connection_manager::CleanupEvent, stream::DIRECT_BODY_IDLE_TIMEOUT_SECS, + EventManager, +}; +use arc_swap::ArcSwapOption; +use shared::{ + defaults::default_hls_session_ttl_secs, + model::{PlaylistItemType, StreamChannel, UserConnectionPermission, XtreamCluster}, + utils::{current_time_secs, Internable}, +}; +use std::{ + borrow::Cow, + net::SocketAddr, + sync::{atomic::Ordering, Arc}, + time::{Duration, Instant}, +}; +use tokio::sync::mpsc; +use tuliprox_core::model::{Config, Fingerprint, ProxyUserCredentials}; +use tuliprox_repository::GeoIp; + +#[tokio::test] +async fn recently_evicted_session_guard_survives_ttl_while_protected_addr_is_still_active() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let evicted_addr: SocketAddr = "127.0.0.1:55111".parse().unwrap(); + let protected_addr: SocketAddr = "127.0.0.1:55112".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-guard-session".to_string(), "127.0.0.1".to_string(), evicted_addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("guard-user"); + user.max_connections = 1; + + manager.add_connection(&evicted_addr).await; + manager.add_connection(&protected_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-guard-session", + virtual_id: 2018, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &evicted_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 18, + meter_uid: 0, + username: &user.username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(2018), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-guard-session"), + }) + .await; + + manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, Duration::from_secs(1)).await; + { + let mut connections = manager.connections.write().await; + if let Some(registration) = connections.key_by_addr.get_mut(&protected_addr) { + registration.add_user(&user.username, current_time_secs()); + } + let protection = connections + .recently_evicted_sessions + .get_mut("tok-guard-session") + .expect("recent eviction guard should exist"); + protection.expires_at = Instant::now().checked_sub(Duration::from_secs(1)).unwrap_or_else(Instant::now); + } + + assert_eq!(manager.recently_evicted_session_protected_addr("tok-guard-session").await, Some(protected_addr)); +} + +#[tokio::test] +async fn test_preserved_adaptive_stream_is_pruned_after_session_ttl() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55061".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key-7".to_string(), "127.0.0.1".to_string(), addr); + + manager.add_connection(&addr).await; + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-expire", + virtual_id: 6001, + provider: "provider-a", + stream_url: "http://localhost/hls.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 77, + meter_uid: 177, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(6001) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-expire"), + }) + .await; + let released = manager.release_connection(&addr).await; + assert!(released.addr_removed); + + { + let mut connections = manager.connections.write().await; + let connection_data = connections.by_key.get_mut("user1").unwrap(); + let session = connection_data.sessions.iter_mut().find(|session| session.token == "tok-expire").unwrap(); + session.ts = session.ts.saturating_sub(default_hls_session_ttl_secs() + 1); + } + if let Some(gc_ts) = &manager.gc_ts { + gc_ts.store(current_time_secs().saturating_sub(USER_GC_TTL + 1), Ordering::Release); + } + + manager + .process_due_adaptive_expiry_entries(current_time_secs().saturating_add(default_hls_session_ttl_secs() + 1)) + .await; + assert!(manager.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn test_due_adaptive_expiry_removal_promotes_soft_stream() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let normal_addr: SocketAddr = "127.0.0.1:55062".parse().unwrap(); + let soft_addr: SocketAddr = "127.0.0.1:55063".parse().unwrap(); + let normal_fp = Fingerprint::new("fp-key-7a".to_string(), "127.0.0.1".to_string(), normal_addr); + let soft_fp = Fingerprint::new("fp-key-7b".to_string(), "127.0.0.1".to_string(), soft_addr); + + manager.add_connection(&normal_addr).await; + manager.add_connection(&soft_addr).await; + + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + user.soft_connections = 1; + + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-expire-normal", + virtual_id: 6002, + provider: "provider-a", + stream_url: "http://localhost/hls-normal.m3u8", + addr: &normal_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 78, + meter_uid: 178, + username: "user1", + max_connections: 1, + soft_connections: 1, + connection_kind: ConnectionKind::Normal, + priority: -1, + soft_priority: 9, + fingerprint: &normal_fp, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(6002) }, + user_agent: Cow::Borrowed("ua-normal"), + session_token: Some("tok-expire-normal"), + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 79, + meter_uid: 179, + username: "user1", + max_connections: 1, + soft_connections: 1, + connection_kind: ConnectionKind::Soft, + priority: -5, + soft_priority: 9, + fingerprint: &soft_fp, + provider: "provider-a".intern(), + stream_channel: &test_channel(6003), + user_agent: Cow::Borrowed("ua-soft"), + session_token: None, + }) + .await; + + let released = manager.release_connection(&normal_addr).await; + assert!(released.addr_removed); + + { + let mut connections = manager.connections.write().await; + let connection_data = connections.by_key.get_mut("user1").unwrap(); + let session = connection_data.sessions.iter_mut().find(|session| session.token == "tok-expire-normal").unwrap(); + session.ts = session.ts.saturating_sub(default_hls_session_ttl_secs() + 1); + } + + manager + .process_due_adaptive_expiry_entries(current_time_secs().saturating_add(default_hls_session_ttl_secs() + 1)) + .await; + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get("user1").unwrap(); + assert_eq!(connection_data.stream_kinds.get(&79), Some(&ConnectionKind::Soft)); + assert!(!connection_data.stream_normal_priorities.contains_key(&78)); +} + +#[tokio::test] +async fn test_repeated_preserve_for_same_adaptive_session_keeps_single_current_expiry_index() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr_a: SocketAddr = "127.0.0.1:55071".parse().unwrap(); + let addr_b: SocketAddr = "127.0.0.1:55072".parse().unwrap(); + let fp_a = Fingerprint::new("fp-key-a".to_string(), "127.0.0.1".to_string(), addr_a); + let fp_b = Fingerprint::new("fp-key-b".to_string(), "127.0.0.1".to_string(), addr_b); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&addr_a).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-reuse", + virtual_id: 7001, + provider: "provider-a", + stream_url: "http://localhost/live-a.m3u8", + addr: &addr_a, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 88, + meter_uid: 188, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fp_a, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(7001) }, + user_agent: Cow::Borrowed("ua-a"), + session_token: Some("tok-reuse"), + }) + .await; + let released = manager.release_connection(&addr_a).await; + assert!(released.addr_removed); + + manager.add_connection(&addr_b).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 99, + meter_uid: 199, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fp_b, + provider: "provider-b".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveDash, ..test_channel(7002) }, + user_agent: Cow::Borrowed("ua-b"), + session_token: Some("tok-reuse"), + }) + .await; + let released = manager.release_connection(&addr_b).await; + assert!(released.addr_removed); + + let expiry_index = manager.adaptive_expiry_index.lock().await; + assert_eq!(expiry_index.len(), 1); + assert!(expiry_index.contains_key(&AdaptiveExpiryKey { + username: String::from("user1"), + session_token: String::from("tok-reuse"), + uid: 88, + })); +} + +#[tokio::test] +async fn test_due_adaptive_expiry_reschedules_when_session_timestamp_changes() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55083".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key-10".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-reschedule", + virtual_id: 8003, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 133, + meter_uid: 233, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8003) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-reschedule"), + }) + .await; + let released = manager.release_connection(&addr).await; + assert!(released.addr_removed); + + let key = + AdaptiveExpiryKey { username: String::from("user1"), session_token: String::from("tok-reschedule"), uid: 133 }; + let old_expires_at = { + let expiry_index = manager.adaptive_expiry_index.lock().await; + *expiry_index.get(&key).unwrap() + }; + + { + let mut connections = manager.connections.write().await; + let session = connections + .by_key + .get_mut("user1") + .unwrap() + .sessions + .iter_mut() + .find(|session| session.token == "tok-reschedule") + .unwrap(); + session.ts = session.ts.saturating_add(30); + } + + manager.process_due_adaptive_expiry_entries(old_expires_at).await; + + let new_expires_at = { + let expiry_index = manager.adaptive_expiry_index.lock().await; + *expiry_index.get(&key).unwrap() + }; + assert!(new_expires_at > old_expires_at); + assert!(manager.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn test_due_adaptive_expiry_removes_stale_index_when_preserved_stream_missing() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55085".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key-11a".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-stale", + virtual_id: 8004, + provider: "provider-a", + stream_url: "http://localhost/stale.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 134, + meter_uid: 234, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8004) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-stale"), + }) + .await; + let released = manager.release_connection(&addr).await; + assert!(released.addr_removed); + + let key = AdaptiveExpiryKey { username: String::from("user1"), session_token: String::from("tok-stale"), uid: 134 }; + let old_expires_at = { + let expiry_index = manager.adaptive_expiry_index.lock().await; + *expiry_index.get(&key).unwrap() + }; + + { + let mut connections = manager.connections.write().await; + let connection_data = connections.by_key.get_mut("user1").unwrap(); + connection_data.streams.clear(); + } + + manager.process_due_adaptive_expiry_entries(old_expires_at).await; + + let expiry_index = manager.adaptive_expiry_index.lock().await; + assert!(!expiry_index.contains_key(&key)); +} + +#[tokio::test] +async fn test_due_adaptive_expiry_does_not_block_on_full_cleanup_channel() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55084".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key-11".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-full-channel", + virtual_id: 8004, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 144, + meter_uid: 244, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8004) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-full-channel"), + }) + .await; + let released = manager.release_connection(&addr).await; + assert!(released.addr_removed); + + { + let mut connections = manager.connections.write().await; + let session = connections + .by_key + .get_mut("user1") + .unwrap() + .sessions + .iter_mut() + .find(|session| session.token == "tok-full-channel") + .unwrap(); + session.ts = session.ts.saturating_sub(default_hls_session_ttl_secs() + 1); + } + + let (cleanup_tx, mut cleanup_rx) = mpsc::channel(1); + cleanup_tx.send(CleanupEvent::ReleaseConnection { addr }).await.expect("prefill cleanup channel"); + manager.set_cleanup_sender(cleanup_tx); + + let process_result = tokio::time::timeout( + Duration::from_millis(100), + manager.process_due_adaptive_expiry_entries( + current_time_secs().saturating_add(default_hls_session_ttl_secs() + 1), + ), + ) + .await; + + assert!(process_result.is_ok(), "adaptive expiry processing must not await while holding locks"); + + let queued_event = cleanup_rx.try_recv().expect("prefilled cleanup event should remain queued"); + assert!(matches!(queued_event, CleanupEvent::ReleaseConnection { .. })); + assert!(manager.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn named_socket_registration_exposes_expiry_deadline() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let stale_addr: SocketAddr = "127.0.0.1:55021".parse().unwrap(); + let fresh_addr: SocketAddr = "127.0.0.1:55022".parse().unwrap(); + let stale_fp = Fingerprint::new("fp-stale".to_string(), "127.0.0.1".to_string(), stale_addr); + let fresh_fp = Fingerprint::new("fp-fresh".to_string(), "127.0.0.1".to_string(), fresh_addr); + let mut stale_user = ProxyUserCredentials::default(); + stale_user.username = "user1".to_string(); + stale_user.max_connections = 1; + let mut fresh_user = ProxyUserCredentials::default(); + fresh_user.username = "user2".to_string(); + fresh_user.max_connections = 1; + + manager.add_connection(&stale_addr).await; + manager.add_connection(&fresh_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &stale_user, + session_token: "tok-stale-deadline", + virtual_id: 9201, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &stale_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .create_user_session(CreateUserSessionParams { + user: &fresh_user, + session_token: "tok-fresh-deadline", + virtual_id: 9202, + provider: "provider-b", + stream_url: "http://localhost/live.m3u8", + addr: &fresh_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 201, + meter_uid: 301, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &stale_fp, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(9201), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-stale-deadline"), + }) + .await + .expect("stale stream should register"); + manager + .update_connection(ActiveUserConnectionParams { + uid: 202, + meter_uid: 302, + username: "user2", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fresh_fp, + provider: "provider-b".intern(), + stream_channel: &test_adaptive_channel(9202), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-fresh-deadline"), + }) + .await + .expect("fresh stream should register"); + + { + let mut connections = manager.connections.write().await; + let stale_registration = connections.key_by_addr.get_mut(&stale_addr).expect("stale registration should exist"); + stale_registration.ts = stale_registration.ts.saturating_sub(DEFAULT_ACTIVE_SOCKET_TTL_SECS + 1); + } + + let stale_deadline = + manager.socket_expiry_deadline(&stale_addr).await.expect("stale named socket should have an expiry deadline"); + let fresh_deadline = + manager.socket_expiry_deadline(&fresh_addr).await.expect("fresh named socket should have an expiry deadline"); + assert!(stale_deadline < fresh_deadline); +} + +#[tokio::test] +async fn socket_expiry_deadline_extends_direct_body_streams_to_the_idle_timeout() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55040".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-vod".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = "user-vod-expiry".to_string(); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-vod-expiry", + virtual_id: 8888, + provider: "provider-a", + stream_url: "http://localhost/movie.mkv", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let mut channel = test_channel(8888); + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + channel.url = "http://localhost/movie.mkv".intern(); + + manager + .update_connection(ActiveUserConnectionParams { + uid: 602, + meter_uid: 702, + username: "user-vod-expiry", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &channel, + user_agent: Cow::Borrowed("player/1.0"), + session_token: Some("tok-vod-expiry"), + }) + .await + .expect("vod stream should be created"); + + // The player stops reading while draining its buffer. The pause already exceeds the + // short HLS session TTL but stays well within the direct-body idle timeout, so the + // socket must not be treated as expired yet. + let previous_registration_ts = { + let mut connections = manager.connections.write().await; + let registration = connections.key_by_addr.get_mut(&addr).expect("registration should exist"); + registration.ts = registration.ts.saturating_sub(default_hls_session_ttl_secs() + 5); + registration.ts + }; + + let deadline = + manager.socket_expiry_deadline(&addr).await.expect("VOD streams should stay scheduled for expiry tracking"); + + let unchanged_registration_ts = { + let connections = manager.connections.read().await; + connections.key_by_addr.get(&addr).expect("registration should still exist").ts + }; + + assert_eq!(unchanged_registration_ts, previous_registration_ts); + assert!( + deadline > current_time_secs(), + "a buffering direct-body stream must not be treated as expired after only the HLS session TTL" + ); + assert_eq!( + deadline, + previous_registration_ts.saturating_add(DIRECT_BODY_IDLE_TIMEOUT_SECS), + "direct-body sockets must use the direct-body idle timeout as their expiry allowance" + ); +} + +#[tokio::test] +async fn socket_expiry_deadline_keeps_hls_session_ttl_for_adaptive_streams() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55043".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-hls".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = "user-hls-expiry".to_string(); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-hls-expiry", + virtual_id: 8891, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + manager + .update_connection(ActiveUserConnectionParams { + uid: 605, + meter_uid: 705, + username: "user-hls-expiry", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(8891), + user_agent: Cow::Borrowed("player/1.0"), + session_token: Some("tok-hls-expiry"), + }) + .await + .expect("hls stream should be created"); + + let registration_ts = { + let mut connections = manager.connections.write().await; + let registration = connections.key_by_addr.get_mut(&addr).expect("registration should exist"); + registration.ts = registration.ts.saturating_sub(default_hls_session_ttl_secs() + 5); + registration.ts + }; + + let deadline = manager.socket_expiry_deadline(&addr).await.expect("HLS streams should stay scheduled for expiry"); + + assert_eq!( + deadline, + registration_ts.saturating_add(manager.active_socket_ttl_secs()), + "adaptive sockets keep the short HLS session TTL" + ); +} + +#[tokio::test] +async fn gc_keeps_active_ts_streams_even_when_user_timestamp_is_stale() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55013".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key-ts".to_string(), "127.0.0.1".to_string(), addr); + + manager.add_connection(&addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 144, + meter_uid: 244, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(9001), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("ts stream should register"); + + { + let mut connections = manager.connections.write().await; + let connection_data = connections.by_key.get_mut("user1").expect("user entry should exist"); + connection_data.ts = connection_data.ts.saturating_sub(USER_CON_TTL + 1); + } + + if let Some(gc_ts) = &manager.gc_ts { + gc_ts.store(current_time_secs().saturating_sub(USER_GC_TTL + 1), Ordering::Release); + } + + manager.active_streams().await; + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get("user1").expect("active user entry must survive gc"); + assert_eq!(connection_data.connections, 1); + assert_eq!(connection_data.streams.len(), 1); +} diff --git a/backend/session/src/active_user_manager/tests/observability.rs b/backend/session/src/active_user_manager/tests/observability.rs new file mode 100644 index 000000000..04bd223c4 --- /dev/null +++ b/backend/session/src/active_user_manager/tests/observability.rs @@ -0,0 +1,303 @@ +use super::{ + assert_no_real_connection_slots, assert_preserved_session_is_uncounted, commit_and_preserve_adaptive_session, + divergence_key, test_channel, test_user_credentials, ActiveUserConnectionParams, ActiveUserManager, + CreateUserSessionParams, DivergenceKind, PendingProviderReason, PendingProviderState, PlaybackLifecycle, + UserConnectionData, UserSession, +}; +use crate::{active_provider_manager::ConnectionKind, EventManager}; +use arc_swap::ArcSwapOption; +use shared::{ + model::{EventMessage, PlaylistItemType, StreamChannel, StreamInfo, UserConnectionPermission, XtreamCluster}, + utils::{current_time_secs, Internable}, +}; +use std::{borrow::Cow, collections::HashMap, net::SocketAddr, sync::Arc}; +use tuliprox_core::model::{Config, Fingerprint, ProxyUserCredentials}; +use tuliprox_repository::GeoIp; + +#[tokio::test] +async fn test_release_stream_preserved_path_emits_connection_update_event() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + let mut events = event_manager.get_event_channel(); + + let addr: SocketAddr = "127.0.0.1:55081".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key-8".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-event", + virtual_id: 8001, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 111, + meter_uid: 211, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(8001) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-event"), + }) + .await; + let _ = events.try_recv(); + + let released = manager.release_stream(&addr).await; + assert!(released.is_none(), "adaptive stream should remain logically preserved"); + + let event = events.try_recv().expect("preserved release should emit an ActiveUser event"); + assert!(matches!(event, EventMessage::ActiveUser(_))); +} + +#[tokio::test] +async fn dashboard_counts_only_real_slots_during_preserved_reactivation_admission() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let user = test_user_credentials("user-preserved-dashboard", 1, 0); + let addr: SocketAddr = "127.0.0.1:55205".parse().unwrap(); + let session_token = "tok-preserved-dashboard"; + let stream_uid = 605; + + commit_and_preserve_adaptive_session(&manager, &user, session_token, stream_uid, addr, ConnectionKind::Normal) + .await; + assert_eq!(manager.active_users_and_connections().await, (0, 0)); + + let admission = manager + .connection_admission_for_session_activation( + &user.username, + user.max_connections, + user.soft_connections, + session_token, + ) + .await; + assert_eq!(admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(manager.active_users_and_connections().await, (0, 0)); + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_preserved_session_is_uncounted(connection_data, session_token, stream_uid); + assert_no_real_connection_slots(connection_data); +} + +#[tokio::test] +async fn check_divergence_detects_connection_count_mismatch() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); + + let addr: SocketAddr = "127.0.0.1:55902".parse().unwrap(); + let mut user = ProxyUserCredentials::default(); + user.username = "div-user-2".to_string(); + + // Create a counted session without a stream or matching legacy counter. + { + let mut connections = manager.connections.write().await; + let data = connections.by_key.entry(user.username.clone()).or_insert_with(|| UserConnectionData::new(0, 1, 0)); + data.add_session(UserSession { + token: "tok-div-2".to_string(), + transition_version: 1, + virtual_id: 9002, + provider: "provider-a".intern(), + stream_url: "http://localhost/stream.ts".intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), + user_agent_stream_index: None, + addr, + socket_bound: false, + active_addrs: vec![addr], + ts: current_time_secs(), + started_at: current_time_secs(), + permission: UserConnectionPermission::Allowed, + connection_kind: None, + lifecycle: PlaybackLifecycle::Active, + ..Default::default() + }); + } + + let connections = manager.connections.read().await; + let data = connections.by_key.get(&user.username).expect("user connection data"); + let snapshot = ActiveUserManager::build_divergence_snapshot(data, &user.username); + assert!(snapshot.kinds.contains(&DivergenceKind::CountedSessionWithoutStream)); + assert!(snapshot.kinds.contains(&DivergenceKind::ConnectionCountMismatch { legacy: 0, counted: 1 })); + drop(connections); + manager.log_divergence_snapshot(Some(snapshot)).await; +} + +#[tokio::test] +async fn check_divergence_detects_stream_without_counted_session() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); + + let addr: SocketAddr = "127.0.0.1:55903".parse().unwrap(); + let mut user = ProxyUserCredentials::default(); + user.username = "div-user-3".to_string(); + + { + let mut connections = manager.connections.write().await; + let data = connections.by_key.entry(user.username.clone()).or_insert_with(|| UserConnectionData::new(0, 1, 0)); + + // Add a session with GraceHold pending — exempt from Invariant 1 + data.add_session(UserSession { + token: "tok-div-3".to_string(), + transition_version: 1, + virtual_id: 9003, + provider: "provider-a".intern(), + stream_url: "http://localhost/stream.ts".intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), + user_agent_stream_index: None, + addr, + socket_bound: false, + active_addrs: vec![addr], + ts: current_time_secs(), + started_at: current_time_secs(), + permission: UserConnectionPermission::Allowed, + connection_kind: None, + lifecycle: PlaybackLifecycle::PendingProvider { + data: PendingProviderState { + reason_code: PendingProviderReason::GraceHold, + created_at: current_time_secs(), + deadline: current_time_secs() + 30, + version: 1, + wake_source: None, + }, + }, + ..Default::default() + }); + data.increment_kind(ConnectionKind::Normal); + + // Add a stream whose session_token doesn't match any counted session + let orphan_stream = StreamInfo::new(shared::model::StreamInfoParams { + uid: 903, + meter_uid: 0, + username: &user.username, + addr: &addr, + client_ip: "127.0.0.1", + provider: "provider-a".intern(), + stream_channel: StreamChannel { + target_id: 1, + virtual_id: 9003, + provider_id: 1, + input_name: "provider-a".intern(), + item_type: PlaylistItemType::Live, + cluster: XtreamCluster::Live, + group: "g".intern(), + title: "t".intern(), + url: "http://localhost/stream.ts".intern(), + shared: false, + shared_joined_existing: None, + shared_stream_id: None, + technical: None, + epg_channel_id: None, + epg_reference_ts: None, + upstream_user_agent: None, + }, + user_agent: "ua".to_string(), + country_code: None, + session_token: Some("tok-orphan"), + }); + data.streams.push(orphan_stream); + data.stream_kinds.insert(903, ConnectionKind::Normal); + } + + let connections = manager.connections.read().await; + let data = connections.by_key.get(&user.username).expect("user connection data"); + let snapshot = ActiveUserManager::build_divergence_snapshot(data, &user.username); + assert!(snapshot.kinds.contains(&DivergenceKind::StreamWithoutCountedSession)); + drop(connections); + manager.log_divergence_snapshot(Some(snapshot)).await; +} + +#[tokio::test] +async fn divergence_log_rate_limited_within_cooldown_window() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); + + let addr: SocketAddr = "127.0.0.1:55904".parse().unwrap(); + let mut user = ProxyUserCredentials::default(); + user.username = "div-user-4".to_string(); + + // Create mismatch + { + let mut connections = manager.connections.write().await; + let data = connections.by_key.entry(user.username.clone()).or_insert_with(|| UserConnectionData::new(0, 1, 0)); + data.increment_kind(ConnectionKind::Normal); + data.add_session(UserSession { + token: "tok-div-4".to_string(), + transition_version: 1, + virtual_id: 9004, + provider: "provider-a".intern(), + stream_url: "http://localhost/stream.ts".intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), + user_agent_stream_index: None, + addr, + socket_bound: false, + active_addrs: vec![addr], + ts: current_time_secs(), + started_at: current_time_secs(), + permission: UserConnectionPermission::Allowed, + connection_kind: None, + lifecycle: PlaybackLifecycle::Prepared, + ..Default::default() + }); + } + + let connections = manager.connections.read().await; + let data = connections.by_key.get(&user.username).expect("user connection data"); + let snapshot = ActiveUserManager::build_divergence_snapshot(data, &user.username); + drop(connections); + manager.log_divergence_snapshot(Some(snapshot)).await; + let key = divergence_key(&user.username, &DivergenceKind::ConnectionCountMismatch { legacy: 1, counted: 0 }); + let first_logged = { + let cache = manager.divergence_cache.lock().await; + let entry = cache.peek(&key).expect("first divergence should populate the cache"); + assert_eq!(entry.count_since_last_log, 0); + entry.last_logged + }; + + let connections = manager.connections.read().await; + let data = connections.by_key.get(&user.username).expect("user connection data"); + let snapshot = ActiveUserManager::build_divergence_snapshot(data, &user.username); + drop(connections); + manager.log_divergence_snapshot(Some(snapshot)).await; + let connections = manager.connections.read().await; + let data = connections.by_key.get(&user.username).expect("user connection data"); + let snapshot = ActiveUserManager::build_divergence_snapshot(data, &user.username); + drop(connections); + manager.log_divergence_snapshot(Some(snapshot)).await; + let cache = manager.divergence_cache.lock().await; + let entry = cache.peek(&key).expect("repeated divergence should remain cached"); + assert_eq!(entry.count_since_last_log, 2); + assert_eq!(entry.last_logged, first_logged); +} diff --git a/backend/session/src/active_user_manager/tests/provider_headers.rs b/backend/session/src/active_user_manager/tests/provider_headers.rs new file mode 100644 index 000000000..f6a0e90cd --- /dev/null +++ b/backend/session/src/active_user_manager/tests/provider_headers.rs @@ -0,0 +1,338 @@ +use super::{ + create_provider_header_session, provider_header_test_manager, session_identity, ActiveUserManager, + CreateUserSessionParams, SessionProviderHeaders, +}; +use crate::{active_provider_manager::ConnectionKind, EventManager}; +use arc_swap::ArcSwapOption; +use futures::FutureExt; +use shared::{model::UserConnectionPermission, utils::Internable}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration}; +use tuliprox_core::model::{Config, ProxyUserCredentials}; +use tuliprox_repository::GeoIp; + +#[tokio::test] +async fn user_agent_stream_index_is_stable_per_session_and_unique_between_sessions() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + let addr = SocketAddr::from(([127, 0, 0, 1], 55_500)); + let mut user = ProxyUserCredentials::default(); + user.username = "indexed-user".to_string(); + + for token in ["indexed-session-a", "indexed-session-b"] { + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: token, + virtual_id: 42, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + } + + let first = manager.get_or_assign_user_agent_stream_index(&user.username, "indexed-session-a").await; + let repeated = manager.get_or_assign_user_agent_stream_index(&user.username, "indexed-session-a").await; + let second = manager.get_or_assign_user_agent_stream_index(&user.username, "indexed-session-b").await; + + assert!(first.is_some()); + assert_eq!(repeated, first); + assert!(second.is_some()); + assert_ne!(second, first); +} + +#[tokio::test] +async fn update_session_provider_headers_updates_existing_session_and_timestamp() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55402".parse().unwrap_or_else(|_| unreachable!()); + let mut user = ProxyUserCredentials::default(); + user.username = "user-provider-headers".to_string(); + + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-provider-headers", + virtual_id: 7003, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let before = manager + .get_and_update_user_session(&user.username, "tok-provider-headers") + .await + .expect("session should exist"); + let previous_ts = before.ts; + let headers = HashMap::from([(String::from("cookie"), String::from("sid=abc"))]); + + assert!(manager.update_session_provider_headers(&user.username, "tok-provider-headers", &headers).await); + + let after = manager + .get_and_update_user_session(&user.username, "tok-provider-headers") + .await + .expect("session should exist"); + assert_eq!(after.provider_session_headers, headers); + assert!(after.ts >= previous_ts); +} + +#[tokio::test] +async fn update_session_provider_headers_returns_false_for_missing_user_or_token() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + let headers = HashMap::from([(String::from("cookie"), String::from("sid=abc"))]); + + assert!(!manager.update_session_provider_headers("missing-user", "missing-token", &headers).await); + + let addr: SocketAddr = "127.0.0.1:55403".parse().unwrap_or_else(|_| unreachable!()); + let mut user = ProxyUserCredentials::default(); + user.username = "user-missing-token".to_string(); + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-existing", + virtual_id: 7004, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + assert!(!manager.update_session_provider_headers(&user.username, "tok-missing", &headers).await); +} + +#[tokio::test] +async fn create_user_session_clears_provider_headers_when_provider_or_stream_url_changes() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55404".parse().unwrap_or_else(|_| unreachable!()); + let mut user = ProxyUserCredentials::default(); + user.username = "user-provider-header-reset".to_string(); + let headers = HashMap::from([(String::from("cookie"), String::from("sid=abc"))]); + + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-reset", + virtual_id: 7005, + provider: "provider-a", + stream_url: "http://localhost/live-a.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + assert!(manager.update_session_provider_headers(&user.username, "tok-reset", &headers).await); + + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-reset", + virtual_id: 7005, + provider: "provider-b", + stream_url: "http://localhost/live-b.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let session = manager.get_and_update_user_session(&user.username, "tok-reset").await.expect("session should exist"); + assert!(session.provider_session_headers.is_empty()); +} + +/// Provider cookies survive child playlist changes on the same host and are only sent to that host. +#[tokio::test] +async fn provider_session_headers_are_scoped_to_the_host_that_set_them() { + let manager = provider_header_test_manager(); + let mut user = ProxyUserCredentials::default(); + user.username = "user-provider-header-host".to_string(); + let headers = HashMap::from([(String::from("cookie"), String::from("sid=abc"))]); + + create_provider_header_session(&manager, &user, "http://cdn.example/a/video.m3u8").await; + assert!(store_cookie_header(&manager, &user.username, "sid=abc", "http://cdn.example/a/video.m3u8").await); + create_provider_header_session(&manager, &user, "http://cdn.example/a/audio.m3u8").await; + + let session = manager.get_and_update_user_session(&user.username, "tok-host").await.expect("session exists"); + assert_eq!(session.provider_session_headers_for("http://cdn.example/a/video.m3u8").as_deref(), Some(&headers)); + assert_eq!(session.provider_session_headers_for("http://entry.example/live/1.m3u8"), None); + assert_eq!( + session.provider_session_headers_for("https://cdn.example:80/a/video.m3u8"), + None, + "same host and port with another scheme is another origin" + ); + + create_provider_header_session(&manager, &user, "http://other.example/a/video.m3u8").await; + let session = manager.get_and_update_user_session(&user.username, "tok-host").await.expect("session exists"); + assert!(session.provider_session_headers.is_empty()); +} + +#[tokio::test] +async fn account_switch_ends_resource_identity_and_rejects_stale_cookie_updates( +) -> Result<(), Box> { + let manager = Arc::new(provider_header_test_manager()); + let mut user = ProxyUserCredentials::default(); + user.username = "session-binding".to_string(); + let url = "http://provider.example/index.m3u8"; + create_provider_header_session(&manager, &user, url).await; + let identity = session_identity(&manager, &user.username, "tok-host").await.ok_or("session missing")?; + let cookie = |value: &str| crate::ProviderSessionHeaders { + headers: HashMap::new(), + cookies: vec![format!("sid={value}; Path=/")], + }; + assert!( + manager + .update_current_session_provider_response_headers_from( + &user.username, + "tok-host", + identity, + &cookie("old"), + url + ) + .await + ); + // Rotation while a resource waits is visible to the send-time lookup. + assert!( + manager.update_session_provider_response_headers_from(&user.username, "tok-host", &cookie("fresh"), url).await + ); + assert_eq!( + manager.current_session_provider_headers(&user.username, "tok-host", identity, url).await, + SessionProviderHeaders::Headers(HashMap::from([("cookie".to_string(), "sid=fresh".to_string())])) + ); + assert_eq!( + manager.current_session_provider_headers(&user.username, "tok-host", identity, "http://other.example/x").await, + SessionProviderHeaders::NoHeaders, + "a valid session without headers for the target is not a missing session" + ); + + let notify = { + let users = manager.connections.read().await; + let (_, session) = users.current_session(&user.username, "tok-host", identity).ok_or("session missing")?; + Arc::clone(&session.change_signal.notify) + }; + let woken = notify.notified(); + tokio::pin!(woken); + woken.as_mut().enable(); + // Writes that keep the session and binding, of this or another user, wake nobody. + manager.update_session_addr(&user.username, "tok-host", &"127.0.0.1:55499".parse()?).await; + let mut other = ProxyUserCredentials::default(); + other.username = "session-binding-other".to_string(); + let other_addr: SocketAddr = "127.0.0.1:55498".parse()?; + manager + .create_user_session(CreateUserSessionParams { + user: &other, + session_token: "tok-other", + virtual_id: 7012, + provider: "provider-a", + stream_url: url, + addr: &other_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + assert!(manager.terminate_session(&other.username, "tok-other").await); + assert!(woken.as_mut().now_or_never().is_none(), "unrelated writes must not wake the waiter"); + let waiter = { + let manager = Arc::clone(&manager); + let username = user.username.clone(); + tokio::spawn(async move { manager.wait_for_playback_session_end(&username, "tok-host", identity).await }) + }; + tokio::task::yield_now().await; + assert!(!waiter.is_finished()); + manager.update_session_provider_binding(&user.username, "tok-host", "provider-b".intern(), url.intern()).await; + tokio::time::timeout(Duration::from_secs(1), waiter).await??; + assert!(woken.as_mut().now_or_never().is_some(), "the account switch wakes the waiter"); + assert!(!manager.playback_session_is_current(&user.username, "tok-host", identity).await); + assert_eq!( + manager.current_session_provider_headers(&user.username, "tok-host", identity, url).await, + SessionProviderHeaders::NoSession + ); + assert!( + !manager + .update_current_session_provider_response_headers_from( + &user.username, + "tok-host", + identity, + &cookie("stale"), + url + ) + .await, + "responses from the previous account must not enter the new account's jar" + ); + let switched = session_identity(&manager, &user.username, "tok-host").await.ok_or("session missing")?; + assert_eq!(switched.incarnation, identity.incarnation); + assert_ne!(switched.binding_generation, identity.binding_generation); + Ok(()) +} + +#[tokio::test] +async fn provider_session_cookies_survive_origin_changes_and_clear_on_account_switch( +) -> Result<(), Box> { + let manager = provider_header_test_manager(); + let mut user = ProxyUserCredentials::default(); + user.username = "cookie-account".to_string(); + let entry = "https://entry.example/index.m3u8"; + let cdn = "https://cdn.example/video/init.mp4"; + create_provider_header_session(&manager, &user, entry).await; + for (url, cookie) in [(entry, "sid=entry"), (cdn, "sid=cdn")] { + assert!(store_cookie_header(&manager, &user.username, cookie, url).await); + } + create_provider_header_session(&manager, &user, cdn).await; + let session = manager.get_and_update_user_session(&user.username, "tok-host").await.ok_or("session missing")?; + assert_eq!( + session.provider_session_headers_for(entry).and_then(|headers| headers.get("cookie").cloned()).as_deref(), + Some("sid=entry") + ); + assert_eq!( + session.provider_session_headers_for(cdn).and_then(|headers| headers.get("cookie").cloned()).as_deref(), + Some("sid=cdn") + ); + manager.update_session_provider_binding(&user.username, "tok-host", "provider-b".intern(), entry.intern()).await; + let session = manager.get_and_update_user_session(&user.username, "tok-host").await.ok_or("session missing")?; + assert!(session.provider_session_cookies.is_empty()); + assert!(session.provider_session_headers_for(entry).is_none()); + assert!(session.provider_session_headers_for(cdn).is_none()); + Ok(()) +} + +/// Stores the pairs of a `Cookie` header as origin-wide provider cookies set by `source_url`. +pub(in crate::active_user_manager::tests) async fn store_cookie_header( + manager: &ActiveUserManager, + username: &str, + cookie_header: &str, + source_url: &str, +) -> bool { + let response = crate::ProviderSessionHeaders { + headers: HashMap::new(), + cookies: cookie_header + .split(';') + .map(str::trim) + .filter(|pair| !pair.is_empty()) + .map(|pair| format!("{pair}; Path=/")) + .collect(), + }; + manager.update_session_provider_response_headers_from(username, "tok-host", &response, source_url).await +} diff --git a/backend/session/src/active_user_manager/tests/reentry_protection.rs b/backend/session/src/active_user_manager/tests/reentry_protection.rs new file mode 100644 index 000000000..7538dd415 --- /dev/null +++ b/backend/session/src/active_user_manager/tests/reentry_protection.rs @@ -0,0 +1,238 @@ +use super::{ + create_socket_reentry_guard_key, test_channel, ActiveUserConnectionParams, ActiveUserManager, + CreateUserSessionParams, +}; +use crate::{active_provider_manager::ConnectionKind, EventManager}; +use arc_swap::ArcSwapOption; +use shared::{ + model::{PlaylistItemType, UserConnectionPermission, XtreamCluster}, + utils::Internable, +}; +use std::{ + borrow::Cow, + net::SocketAddr, + sync::Arc, + time::{Duration, Instant}, +}; +use tuliprox_core::model::{Config, Fingerprint, ProxyUserCredentials}; +use tuliprox_repository::GeoIp; + +#[tokio::test] +async fn recently_evicted_vod_uses_session_reentry_guard() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let evicted_addr: SocketAddr = "127.0.0.1:55113".parse().unwrap(); + let protected_addr: SocketAddr = "127.0.0.1:55114".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-vod-guard".to_string(), "127.0.0.1".to_string(), evicted_addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("vod-guard-user"); + user.max_connections = 1; + let mut channel = test_channel(2019); + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + channel.url = "http://localhost/movie.mkv".intern(); + + manager.add_connection(&evicted_addr).await; + manager.add_connection(&protected_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-guard-vod", + virtual_id: channel.virtual_id, + provider: "provider-a", + stream_url: channel.url.as_ref(), + addr: &evicted_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 19, + meter_uid: 0, + username: &user.username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &channel, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-guard-vod"), + }) + .await; + + manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, Duration::from_secs(10)).await; + + assert_eq!(manager.recently_evicted_session_protected_addr("tok-guard-vod").await, Some(protected_addr)); + let connections = manager.connections.read().await; + assert!( + connections.recent_socket_reentry_guards.is_empty(), + "provider-affine VOD must not be guarded by transient socket identity" + ); +} + +#[tokio::test] +async fn provider_affine_stream_without_session_token_uses_socket_reentry_fallback() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let evicted_addr: SocketAddr = "127.0.0.1:55115".parse().unwrap(); + let protected_addr: SocketAddr = "127.0.0.1:55116".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-vod-no-token".to_string(), "127.0.0.1".to_string(), evicted_addr); + let mut channel = test_channel(2020); + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + channel.url = "http://localhost/movie-no-token.mkv".intern(); + + manager.add_connection(&evicted_addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 20, + meter_uid: 0, + username: "vod-no-token-user", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &channel, + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await; + + manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, Duration::from_secs(10)).await; + + assert_eq!( + manager + .recent_socket_reentry_protected_addr( + "vod-no-token-user", + "127.0.0.1", + shared::model::VirtualId::new(channel.virtual_id) + ) + .await, + Some(protected_addr) + ); +} + +#[tokio::test] +async fn socket_reentry_guard_expires_at_ttl() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let evicted_addr: SocketAddr = "127.0.0.1:55117".parse().unwrap(); + let protected_addr: SocketAddr = "127.0.0.1:55118".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-socket-ttl".to_string(), "127.0.0.1".to_string(), evicted_addr); + let channel = test_channel(2021); + + manager.add_connection(&evicted_addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 21, + meter_uid: 0, + username: "socket-ttl-user", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &channel, + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await; + + manager.mark_recent_eviction_guard_for_addr(&evicted_addr, protected_addr, Duration::from_secs(10)).await; + let virtual_id = shared::model::VirtualId::new(channel.virtual_id); + assert_eq!( + manager.recent_socket_reentry_protected_addr("socket-ttl-user", "127.0.0.1", virtual_id).await, + Some(protected_addr) + ); + + // Expire only the guard while the protected address stays registered: the TTL is + // authoritative and must not fall back to the still-present socket registration. + { + let mut connections = manager.connections.write().await; + let key = create_socket_reentry_guard_key("socket-ttl-user", "127.0.0.1", virtual_id); + let protection = + connections.recent_socket_reentry_guards.get_mut(&key).expect("socket reentry guard should exist"); + protection.expires_at = Instant::now().checked_sub(Duration::from_secs(1)).unwrap_or_else(Instant::now); + } + assert_eq!(manager.recent_socket_reentry_protected_addr("socket-ttl-user", "127.0.0.1", virtual_id).await, None); +} + +#[tokio::test] +async fn arm_eviction_protection_protects_all_users_on_addr() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "192.168.1.100:12345".parse().unwrap(); + let protected_addr: SocketAddr = "192.168.1.101:54321".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key".to_string(), "192.168.1.100".to_string(), addr); + + manager.add_connection(&addr).await; + + manager + .update_connection(ActiveUserConnectionParams { + uid: 1, + meter_uid: 0, + username: "user_a", + max_connections: 5, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(2001), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("register user_a"); + + manager + .update_connection(ActiveUserConnectionParams { + uid: 2, + meter_uid: 0, + username: "user_b", + max_connections: 5, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-b".intern(), + stream_channel: &test_channel(2002), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("register user_b"); + + manager.mark_recent_eviction_guard_for_addr(&addr, protected_addr, Duration::from_secs(60)).await; + + let connections = manager.connections.read().await; + let key_a = create_socket_reentry_guard_key("user_a", "192.168.1.100", shared::model::VirtualId::new(2001)); + let key_b = create_socket_reentry_guard_key("user_b", "192.168.1.100", shared::model::VirtualId::new(2002)); + + assert!(connections.recent_socket_reentry_guards.contains_key(&key_a), "user_a guard must be armed"); + assert!(connections.recent_socket_reentry_guards.contains_key(&key_b), "user_b guard must be armed"); +} diff --git a/backend/session/src/active_user_manager/tests/request_claims.rs b/backend/session/src/active_user_manager/tests/request_claims.rs new file mode 100644 index 000000000..9fc5290e8 --- /dev/null +++ b/backend/session/src/active_user_manager/tests/request_claims.rs @@ -0,0 +1,93 @@ +use super::{test_adaptive_channel, ActiveUserConnectionParams, ActiveUserManager}; +use crate::{active_provider_manager::ConnectionKind, EventManager}; +use arc_swap::ArcSwapOption; +use shared::utils::Internable; +use std::{borrow::Cow, net::SocketAddr, sync::Arc}; +use tuliprox_core::model::{Config, Fingerprint}; +use tuliprox_repository::GeoIp; + +#[tokio::test] +async fn affine_request_claims_keep_playback_until_oldest_body_finishes() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let Some(addr) = "127.0.0.1:55033".parse::().ok() else { + return; + }; + let fingerprint = Fingerprint::new("fp-key-shared-addr".to_string(), "127.0.0.1".to_string(), addr); + + manager.add_connection(&addr).await; + for uid in [41, 42] { + manager + .update_connection(ActiveUserConnectionParams { + uid, + meter_uid: uid + 300, + username: "user1", + max_connections: 0, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(3004), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-live-shared-addr"), + }) + .await; + } + + assert_eq!(manager.active_streams().await.len(), 1); + assert!(manager.release_stream_by_uid(&addr, 42).await.is_none()); + + let streams = manager.active_streams().await; + assert_eq!(streams.len(), 1); + assert_eq!(streams[0].uid, 41); + + let removed = manager.release_stream_by_uid(&addr, 41).await; + assert!(removed.as_ref().is_some_and(|stream| stream.uid == 41)); + assert!(manager.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn affine_request_claims_keep_playback_until_newest_body_finishes() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + let addr: SocketAddr = "127.0.0.1:55037".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-affine-cleanup-order".to_string(), "127.0.0.1".to_string(), addr); + + manager.add_connection(&addr).await; + for uid in [45, 46] { + manager + .update_connection(ActiveUserConnectionParams { + uid, + meter_uid: 0, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(3005), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-affine-cleanup-order"), + }) + .await + .expect("affine request should register"); + } + + assert!(manager.release_stream_by_uid(&addr, 45).await.is_none()); + assert_eq!(manager.active_users_and_connections().await, (1, 1)); + assert_eq!(manager.active_streams().await.len(), 1); + + let removed = manager.release_stream_by_uid(&addr, 46).await; + assert!(removed.as_ref().is_some_and(|stream| stream.uid == 45)); + assert_eq!(manager.active_users_and_connections().await, (0, 0)); + assert!(manager.active_streams().await.is_empty()); +} diff --git a/backend/session/src/active_user_manager/tests/sessions.rs b/backend/session/src/active_user_manager/tests/sessions.rs new file mode 100644 index 000000000..b36382704 --- /dev/null +++ b/backend/session/src/active_user_manager/tests/sessions.rs @@ -0,0 +1,1428 @@ +use super::{ + create_provider_header_session, provider_header_test_manager, session_identity, test_adaptive_channel, + test_channel, test_series_channel, ActiveUserConnectionParams, ActiveUserManager, CreateUserSessionParams, + PendingProviderReason, PendingProviderState, PendingProviderWakeSource, PlaybackLifecycle, + PlaybackSessionRegistration, UserConnectionData, UserSession, +}; +use crate::{active_provider_manager::ConnectionKind, EventManager}; +use arc_swap::ArcSwapOption; +use shared::{ + model::{ActiveUserConnectionChange, EventMessage, StreamInfo, UserConnectionPermission}, + utils::{current_time_secs, Internable}, +}; +use std::{borrow::Cow, collections::HashMap, net::SocketAddr, sync::Arc, time::Duration}; +use tuliprox_core::model::{Config, Fingerprint, ProxyUserCredentials}; +use tuliprox_repository::GeoIp; + +#[tokio::test] +async fn target_scoped_session_lookup_does_not_use_same_virtual_id_from_other_target() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55499".parse().unwrap(); + let mut user = ProxyUserCredentials::default(); + user.username = "target-scoped-user".to_string(); + let mut target_one = test_channel(42); + target_one.target_id = 1; + let mut target_two = test_channel(42); + target_two.target_id = 2; + + let _ = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-target-one", + virtual_id: 42, + provider: "provider", + stream_url: "http://localhost/target-one.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + let _ = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-target-two", + virtual_id: 42, + provider: "provider", + stream_url: "http://localhost/target-two.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + { + let mut connections = manager.connections.write().await; + assert!(connections.by_key.contains_key(&user.username), "user connection data should exist"); + let Some(data) = connections.by_key.get_mut(&user.username) else { + return; + }; + data.streams.push(StreamInfo::new(shared::model::StreamInfoParams { + uid: 1, + meter_uid: 1, + username: &user.username, + addr: &addr, + client_ip: "127.0.0.1", + provider: "provider".intern(), + stream_channel: target_one, + user_agent: "ua".to_string(), + country_code: None, + session_token: Some("tok-target-one"), + })); + data.streams.push(StreamInfo::new(shared::model::StreamInfoParams { + uid: 2, + meter_uid: 2, + username: &user.username, + addr: &addr, + client_ip: "127.0.0.1", + provider: "provider".intern(), + stream_channel: target_two, + user_agent: "ua".to_string(), + country_code: None, + session_token: Some("tok-target-two"), + })); + } + + let session = manager.find_latest_session_for_target_stream(&user.username, 2, "input", 42, "tok-target-two").await; + assert!(session.is_some(), "target-scoped session should resolve"); + let Some(session) = session else { + return; + }; + assert_eq!(session.token, "tok-target-two"); + assert!(manager + .find_latest_session_for_target_stream(&user.username, 3, "input", 42, "tok-target-two") + .await + .is_none()); + assert!(manager + .find_latest_session_for_target_stream(&user.username, 1, "input", 42, "tok-target-two") + .await + .is_none()); +} + +/// Session refresh normalizes Expired -> Prepared. +/// When a new request arrives on an expired session, the lifecycle should be +/// reset to Prepared so that full activation evaluation happens. +#[tokio::test] +async fn create_user_session_normalizes_expired_lifecycle() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55400".parse().unwrap(); + let mut user = ProxyUserCredentials::default(); + user.username = "user-lifecycle-refresh".to_string(); + + // Create a session in Expired state directly via session manipulation + { + let mut connections = manager.connections.write().await; + let data = connections.by_key.entry(user.username.clone()).or_insert_with(|| UserConnectionData::new(0, 1, 0)); + data.add_session(UserSession { + token: "tok-refresh-expired".to_string(), + transition_version: 1, + virtual_id: 7001, + provider: "provider-a".intern(), + stream_url: "http://localhost/live.m3u8".intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), + user_agent_stream_index: None, + addr, + socket_bound: false, + active_addrs: vec![addr], + ts: current_time_secs(), + started_at: current_time_secs(), + permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + lifecycle: PlaybackLifecycle::Expired, + ..Default::default() + }); + } + + // Refresh the session via create_user_session + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-refresh-expired", + virtual_id: 7001, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let sessions = manager.connections.read().await; + let data = sessions.by_key.get(&user.username).expect("user data should exist"); + let session = data.sessions.iter().find(|s| s.token == "tok-refresh-expired").expect("session"); + assert_eq!( + session.lifecycle, + PlaybackLifecycle::Prepared, + "Expired session should normalize to Prepared on refresh" + ); +} + +/// Session refresh does NOT normalize `PendingProvider`. +/// A `PendingProvider` session must not be reset — pending state must continue +/// until explicitly resolved via `activate_pending_provider` or `expire_pending_provider`. +#[tokio::test] +async fn create_user_session_does_not_normalize_pending_provider_lifecycle() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55401".parse().unwrap(); + let mut user = ProxyUserCredentials::default(); + user.username = "user-pending-lifecycle".to_string(); + + // Create a session in PendingProvider state + { + let mut connections = manager.connections.write().await; + let data = connections.by_key.entry(user.username.clone()).or_insert_with(|| UserConnectionData::new(0, 1, 0)); + data.add_session(UserSession { + token: "tok-refresh-pending".to_string(), + transition_version: 1, + virtual_id: 7002, + provider: "provider-a".intern(), + stream_url: "http://localhost/live.m3u8".intern(), + provider_session_headers: HashMap::new(), + provider_session_headers_host: None, + media_started: Arc::new(std::sync::atomic::AtomicBool::new(false)), + user_agent_stream_index: None, + addr, + socket_bound: false, + active_addrs: vec![addr], + ts: current_time_secs(), + started_at: current_time_secs(), + permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + lifecycle: PlaybackLifecycle::PendingProvider { + data: PendingProviderState { + reason_code: PendingProviderReason::GraceHold, + created_at: current_time_secs(), + deadline: current_time_secs() + 30, + version: 1, + wake_source: None, + }, + }, + ..Default::default() + }); + } + + // Refresh the session via create_user_session + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-refresh-pending", + virtual_id: 7002, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let sessions = manager.connections.read().await; + let data = sessions.by_key.get(&user.username).expect("user data should exist"); + let session = data.sessions.iter().find(|s| s.token == "tok-refresh-pending").expect("session"); + assert!( + matches!(session.lifecycle, PlaybackLifecycle::PendingProvider { .. }), + "PendingProvider session should NOT be normalized on refresh - pending wait must continue" + ); +} + +/// `terminate_session` expires a session and removes it. +#[tokio::test] +async fn terminate_session_expires_and_removes_session() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55410".parse().unwrap(); + let mut user = ProxyUserCredentials::default(); + user.username = "user-terminate".to_string(); + user.max_connections = 2; + + let token = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-terminate-test", + virtual_id: 8001, + provider: "provider-terminate", + stream_url: "http://localhost/test.ts", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + // Verify session exists. + let before = manager.get_and_update_user_session(&user.username, &token).await; + assert!(before.is_some(), "session should exist before terminate"); + assert_eq!(before.as_ref().unwrap().lifecycle, PlaybackLifecycle::Prepared); + + // Terminate the session. + manager.terminate_session(&user.username, &token).await; + + // Session should be gone. + let after = manager.get_and_update_user_session(&user.username, &token).await; + assert!(after.is_none(), "session should be removed after terminate"); +} + +#[allow(clippy::too_many_lines)] +#[tokio::test] +async fn terminate_session_promotes_soft_stream_after_releasing_capacity() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let normal_addr: SocketAddr = "127.0.0.1:55413".parse().unwrap(); + let soft_addr: SocketAddr = "127.0.0.1:55414".parse().unwrap(); + let soft_addr_two: SocketAddr = "127.0.0.1:55415".parse().unwrap(); + let normal_fp = Fingerprint::new("fp-terminate-normal".to_string(), "127.0.0.1".to_string(), normal_addr); + let soft_fp = Fingerprint::new("fp-terminate-soft".to_string(), "127.0.0.1".to_string(), soft_addr); + let soft_fp_two = Fingerprint::new("fp-terminate-soft-2".to_string(), "127.0.0.1".to_string(), soft_addr_two); + let mut user = ProxyUserCredentials::default(); + user.username = "user-terminate-promote".to_string(); + user.max_connections = 1; + user.soft_connections = 2; + + manager.add_connection(&normal_addr).await; + manager.add_connection(&soft_addr).await; + manager.add_connection(&soft_addr_two).await; + + let normal_token = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-terminate-normal", + virtual_id: 8101, + provider: "provider-normal", + stream_url: "http://localhost/normal.ts", + addr: &normal_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + let soft_token = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-terminate-soft", + virtual_id: 8102, + provider: "provider-soft", + stream_url: "http://localhost/soft.ts", + addr: &soft_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Soft), + socket_bound: false, + }) + .await; + let soft_token_two = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-terminate-soft-2", + virtual_id: 8103, + provider: "provider-soft-2", + stream_url: "http://localhost/soft-2.ts", + addr: &soft_addr_two, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Soft), + socket_bound: false, + }) + .await; + + manager + .update_connection(ActiveUserConnectionParams { + uid: 8101, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &normal_fp, + provider: "provider-normal".intern(), + stream_channel: &test_channel(8101), + user_agent: Cow::Borrowed("ua-normal"), + session_token: Some(&normal_token), + }) + .await + .expect("normal stream should be registered"); + manager + .update_connection(ActiveUserConnectionParams { + uid: 8102, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Soft, + priority: -5, + soft_priority: 9, + fingerprint: &soft_fp, + provider: "provider-soft".intern(), + stream_channel: &test_channel(8102), + user_agent: Cow::Borrowed("ua-soft"), + session_token: Some(&soft_token), + }) + .await + .expect("soft stream should be registered"); + manager + .update_connection(ActiveUserConnectionParams { + uid: 8103, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Soft, + priority: -3, + soft_priority: 9, + fingerprint: &soft_fp_two, + provider: "provider-soft-2".intern(), + stream_channel: &test_channel(8103), + user_agent: Cow::Borrowed("ua-soft-2"), + session_token: Some(&soft_token_two), + }) + .await + .expect("second soft stream should be registered"); + + { + let mut connections = manager.connections.write().await; + let connection_data = connections.by_key.get_mut(&user.username).expect("user data should exist"); + connection_data.soft_connections = 1; + } + + manager.terminate_session(&user.username, &normal_token).await; + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user data should remain inspectable"); + assert_eq!(connection_data.counts.normal, 1); + assert_eq!(connection_data.counts.soft, 1); + let promoted_uid = [8102_u32, 8103_u32] + .into_iter() + .find(|uid| connection_data.stream_kinds.get(uid) == Some(&ConnectionKind::Normal)) + .expect("one soft stream should be promoted to normal"); + let promoted_token = if promoted_uid == 8102 { soft_token.as_str() } else { soft_token_two.as_str() }; + let promoted_session = connection_data + .sessions + .iter() + .find(|session| session.token == promoted_token) + .expect("promoted soft session should remain"); + assert_eq!(promoted_session.connection_kind, Some(ConnectionKind::Normal)); + assert!(matches!(promoted_session.lifecycle, PlaybackLifecycle::Active)); +} + +#[tokio::test] +async fn mark_pending_provider_tracks_metadata_on_session() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55021".parse().unwrap_or_else(|_| unreachable!()); + let mut user = ProxyUserCredentials::default(); + user.username = "pending-user".to_string(); + + let _ = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-pending", + virtual_id: 1001, + provider: "provider-a", + stream_url: "http://provider/live/1001.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let _ = + manager.mark_pending_provider(&user.username, "tok-pending", PendingProviderReason::GraceHold, 12_345).await; + + let session = + manager.get_and_update_user_session(&user.username, "tok-pending").await.expect("session should exist"); + let PlaybackLifecycle::PendingProvider { data: pending } = &session.lifecycle else { + panic!("pending provider should be tracked") + }; + assert!(matches!(pending.reason_code, PendingProviderReason::GraceHold)); + assert_eq!(pending.deadline, 12_345); + assert!(pending.created_at > 0); + assert_eq!(pending.version, 1); + assert!(pending.wake_source.is_none()); +} + +#[tokio::test] +async fn activate_pending_provider_clears_pending_metadata() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55022".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = Fingerprint::new("fp-pending".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = "pending-activate".to_string(); + + let _ = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-pending-activate", + virtual_id: 1002, + provider: "provider-a", + stream_url: "http://provider/live/1002.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + let _ = manager + .mark_pending_provider( + &user.username, + "tok-pending-activate", + PendingProviderReason::GraceHold, + current_time_secs().saturating_add(30), + ) + .await; + + let _ = manager + .update_connection(ActiveUserConnectionParams { + uid: 12, + meter_uid: 0, + username: &user.username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(1002), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-pending-activate"), + }) + .await; + manager + .activate_pending_provider(&user.username, "tok-pending-activate", 1, PendingProviderWakeSource::Activated) + .await; + + let session = manager + .get_and_update_user_session(&user.username, "tok-pending-activate") + .await + .expect("session should exist"); + assert!(session.lifecycle.is_counted()); + assert!( + !matches!(session.lifecycle, PlaybackLifecycle::PendingProvider { .. }), + "explicit pending resolution must clear pending provider state" + ); +} + +#[tokio::test] +async fn activate_pending_provider_ignores_stale_version_after_replacement() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55023".parse().unwrap_or_else(|_| unreachable!()); + let mut user = ProxyUserCredentials::default(); + user.username = "pending-stale".to_string(); + + let _ = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-pending-stale", + virtual_id: 1003, + provider: "provider-a", + stream_url: "http://provider/live/1003.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let first_version = manager + .mark_pending_provider(&user.username, "tok-pending-stale", PendingProviderReason::GraceHold, 5_000) + .await + .expect("first pending version should be created"); + let second_version = manager + .mark_pending_provider(&user.username, "tok-pending-stale", PendingProviderReason::GraceHold, 6_000) + .await + .expect("second pending version should replace the first"); + assert!(second_version > first_version); + + manager + .activate_pending_provider( + &user.username, + "tok-pending-stale", + first_version, + PendingProviderWakeSource::CapacityNotify, + ) + .await; + + let session = manager + .get_and_update_user_session(&user.username, "tok-pending-stale") + .await + .expect("session should still exist"); + let PlaybackLifecycle::PendingProvider { data: pending_data } = &session.lifecycle else { + panic!("session should still be in PendingProvider after stale wakeup") + }; + assert_eq!(pending_data.version, second_version); + assert!(pending_data.wake_source.is_none()); + assert_eq!(session.permission, UserConnectionPermission::GracePeriod); +} + +#[tokio::test] +async fn expire_pending_provider_marks_session_exhausted() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55024".parse().unwrap_or_else(|_| unreachable!()); + let mut user = ProxyUserCredentials::default(); + user.username = "pending-expire".to_string(); + + let _ = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-pending-expire", + virtual_id: 1004, + provider: "provider-a", + stream_url: "http://provider/live/1004.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let version = manager + .mark_pending_provider(&user.username, "tok-pending-expire", PendingProviderReason::GraceHold, 6_000) + .await + .expect("pending version should be created"); + + manager + .expire_pending_provider(&user.username, "tok-pending-expire", version, PendingProviderWakeSource::Timeout) + .await; + + let session = manager + .get_and_update_user_session(&user.username, "tok-pending-expire") + .await + .expect("session should still exist"); + assert_eq!(session.permission, UserConnectionPermission::Exhausted); + assert!(!matches!(session.lifecycle, PlaybackLifecycle::PendingProvider { .. })); + assert!(!session.lifecycle.is_counted()); +} + +#[tokio::test] +async fn test_kicked_release_invalidates_removed_session_tokens() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let kicked_addr: SocketAddr = "127.0.0.1:55015".parse().unwrap(); + let survivor_addr: SocketAddr = "127.0.0.1:55016".parse().unwrap(); + let kicked_fingerprint = Fingerprint::new("fp-kicked".to_string(), "127.0.0.1".to_string(), kicked_addr); + let survivor_fingerprint = Fingerprint::new("fp-survivor".to_string(), "127.0.0.1".to_string(), survivor_addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("kicked-user"); + user.max_connections = 1; + + manager.add_connection(&kicked_addr).await; + manager.add_connection(&survivor_addr).await; + + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-kicked", + virtual_id: 2015, + provider: "provider-a", + stream_url: "http://localhost/live-1.ts", + addr: &kicked_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: true, + }) + .await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-survivor", + virtual_id: 2016, + provider: "provider-a", + stream_url: "http://localhost/live-2.ts", + addr: &survivor_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: true, + }) + .await; + + manager + .update_connection(ActiveUserConnectionParams { + uid: 15, + meter_uid: 0, + username: &user.username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &kicked_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(2015), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-kicked"), + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 16, + meter_uid: 0, + username: &user.username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &survivor_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(2016), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-survivor"), + }) + .await; + + let removed = manager.release_connection_as_kicked(&kicked_addr).await; + assert!(removed.addr_removed); + assert_eq!(removed.removed_streams.len(), 1); + assert_eq!( + manager.connection_admission_for_session(&user.username, 1, 0, "tok-kicked").await.permission, + UserConnectionPermission::Exhausted + ); + assert_eq!( + manager.connection_admission_for_session(&user.username, 1, 0, "tok-survivor").await.permission, + UserConnectionPermission::Allowed + ); +} + +#[tokio::test] +async fn test_same_session_token_refreshes_meter_metadata_on_reuse() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55031".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key-3".to_string(), "127.0.0.1".to_string(), addr); + + manager.add_connection(&addr).await; + let first = manager + .update_connection(ActiveUserConnectionParams { + uid: 11, + meter_uid: 101, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(3001), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-meter"), + }) + .await + .expect("initial stream should register"); + assert_eq!(first.uid, 11); + assert_eq!(first.meter_uid, 101); + + let second = manager + .update_connection(ActiveUserConnectionParams { + uid: 22, + meter_uid: 202, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-b".intern(), + stream_channel: &test_adaptive_channel(3002), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-meter"), + }) + .await + .expect("reused stream should register"); + + assert_eq!(second.uid, 11, "logical stream identity should stay stable on session reuse"); + assert_eq!(second.meter_uid, 202, "reused stream must refresh its meter mapping"); + + let streams = manager.active_streams().await; + assert_eq!(streams.len(), 1); + assert_eq!(streams[0].uid, 11); + assert_eq!(streams[0].meter_uid, 202); + assert_eq!(streams[0].provider.as_ref(), "provider-b"); + assert_eq!(streams[0].channel.virtual_id, 3002); +} + +#[tokio::test] +async fn unlimited_user_can_open_same_and_different_live_streams_from_same_ip() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let username = "unlimited-same-ip"; + let client_ip = "10.9.0.1"; + let addrs = [ + "10.9.0.1:55101".parse::().unwrap(), + "10.9.0.1:55102".parse::().unwrap(), + "10.9.0.1:55103".parse::().unwrap(), + ]; + let fingerprints = [ + Fingerprint::new("fp-unlimited-1".to_string(), client_ip.to_string(), addrs[0]), + Fingerprint::new("fp-unlimited-2".to_string(), client_ip.to_string(), addrs[1]), + Fingerprint::new("fp-unlimited-3".to_string(), client_ip.to_string(), addrs[2]), + ]; + + for addr in addrs { + manager.add_connection(&addr).await; + } + + for (idx, (fingerprint, virtual_id)) in fingerprints.iter().zip([4100, 4100, 4101]).enumerate() { + let token = format!("tok-unlimited-{idx}"); + manager + .update_connection(ActiveUserConnectionParams { + uid: 410 + u32::try_from(idx).unwrap_or_default(), + meter_uid: 0, + username, + max_connections: 0, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(virtual_id), + user_agent: Cow::Borrowed("ua"), + session_token: Some(&token), + }) + .await + .expect("unlimited stream should register"); + } + + assert_eq!(manager.user_connections(username).await, 3); + assert_eq!(manager.active_streams().await.len(), 3); + assert_eq!(manager.connection_admission(username, 0, 0).await.permission, UserConnectionPermission::Allowed); + assert_eq!( + manager.connection_admission_for_session(username, 0, 0, "tok-unlimited-new").await.permission, + UserConnectionPermission::Allowed + ); +} + +/// PR1 regression: two bodies with distinct request UIDs that map to the same +/// display stream must both clean up correctly. The current body path passes the +/// *display* UID (`stream_info.uid`) to cleanup, not the request UID. This test +/// asserts that the cleanup is request-affine — releasing the display UID twice +/// must drain both claims, and the second release must find the stream. +#[tokio::test] +async fn pr1_two_bodies_same_display_uid_both_cleanups_must_succeed() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55035".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-pr1-display-vs-request".to_string(), "127.0.0.1".to_string(), addr); + manager.add_connection(&addr).await; + + // Request 41 creates the display stream. + let first = manager + .update_connection(ActiveUserConnectionParams { + uid: 41, + meter_uid: 301, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(3006), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-pr1"), + }) + .await + .expect("first body should register"); + + // Request 42 reuses the same playback (same session token, same channel). + let second = manager + .update_connection(ActiveUserConnectionParams { + uid: 42, + meter_uid: 302, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(3006), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-pr1"), + }) + .await + .expect("second body should register"); + + // Both return the same display UID. + assert_eq!(first.uid, second.uid, "display UID must be stable across requests"); + let display_uid = first.uid; + + // Fixed behavior: the body passes its own request UID to cleanup, not the display UID. + // First body (request 41) finishes → should NOT remove the stream (request 42 still active). + let removed_first = manager.release_stream_by_uid(&addr, 41).await; + assert!(removed_first.is_none(), "first body cleanup must keep stream alive for second body"); + + assert_eq!(manager.active_streams().await.len(), 1, "stream must survive first body cleanup"); + + // Second body (request 42) finishes → should remove the stream (no more claims). + let removed_second = manager.release_stream_by_uid(&addr, 42).await; + assert!(removed_second.is_some(), "second body cleanup must find and remove the stream"); + assert_eq!(removed_second.unwrap().uid, display_uid); + + assert!(manager.active_streams().await.is_empty(), "no streams left after all bodies finish"); + assert_eq!(manager.active_users_and_connections().await, (0, 0)); +} + +#[tokio::test] +async fn connection_counts_are_broadcast_when_active_user_logging_is_disabled() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let mut events = event_manager.get_event_channel(); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55037".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-count-events".to_string(), "127.0.0.1".to_string(), addr); + manager.add_connection(&addr).await; + manager.release_connection(&addr).await; + assert!( + tokio::time::timeout(Duration::from_millis(50), events.recv()).await.is_err(), + "closing an unowned socket must not broadcast unchanged connection counts" + ); + manager.add_connection(&addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 48, + meter_uid: 0, + username: "event-user", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_series_channel(3048), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("direct Series stream should register"); + + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), events.recv()) + .await + .expect("connection count update should be broadcast") + .expect("event channel should remain open"), + EventMessage::ActiveUser(ActiveUserConnectionChange::Connections(1, 1)) + ); + + manager + .update_connection(ActiveUserConnectionParams { + uid: 48, + meter_uid: 0, + username: "event-user", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_series_channel(3048), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("same direct Series stream should be reused"); + assert!( + tokio::time::timeout(Duration::from_millis(50), events.recv()).await.is_err(), + "unchanged connection counts must not broadcast another full snapshot" + ); + + manager.release_stream_by_uid(&addr, 48).await.expect("direct Series stream should release"); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), events.recv()) + .await + .expect("released count update should be broadcast") + .expect("event channel should remain open"), + EventMessage::ActiveUser(ActiveUserConnectionChange::Connections(0, 0)) + ); +} + +#[tokio::test] +async fn update_session_provider_binding_updates_session_and_streams() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55140".parse().unwrap(); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-binding", + virtual_id: 100, + provider: "account-b", + stream_url: "http://example.com/vod/movie.mkv?token=account-b", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + { + let mut connections = manager.connections.write().await; + let data = connections.by_key.get_mut("user1").expect("user connection data"); + data.streams.push(StreamInfo::new(shared::model::StreamInfoParams { + uid: 1, + meter_uid: 1, + username: &user.username, + addr: &addr, + client_ip: "127.0.0.1", + provider: "account-b".intern(), + stream_channel: test_channel(100), + user_agent: "ua".to_string(), + country_code: None, + session_token: Some("tok-binding"), + })); + } + + manager + .update_session_provider_binding( + "user1", + "tok-binding", + "account-a".intern(), + "http://example.com/vod/movie.mkv?token=account-a".into(), + ) + .await; + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get("user1").expect("user connection data"); + assert_eq!(connection_data.sessions[0].provider.as_ref(), "account-a"); + assert_eq!(connection_data.sessions[0].stream_url.as_ref(), "http://example.com/vod/movie.mkv?token=account-a"); + assert_eq!(connection_data.streams[0].provider.as_ref(), "account-a"); +} + +#[tokio::test] +async fn session_activation_keeps_first_hls_slot_uncommitted_before_stream_registration() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user-hls-reserve"); + user.max_connections = 1; + + let first_addr: SocketAddr = "127.0.0.1:55180".parse().unwrap(); + let second_addr: SocketAddr = "127.0.0.1:55181".parse().unwrap(); + + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-first", + virtual_id: 9201, + provider: "provider-a", + stream_url: "http://localhost/live-a.m3u8", + addr: &first_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-second", + virtual_id: 9202, + provider: "provider-a", + stream_url: "http://localhost/live-b.m3u8", + addr: &second_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let first_admission = + manager.connection_admission_for_session_activation(&user.username, user.max_connections, 0, "tok-first").await; + let second_admission = manager + .connection_admission_for_session_activation(&user.username, user.max_connections, 0, "tok-second") + .await; + + assert_eq!(first_admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(first_admission.kind(), Some(ConnectionKind::Normal)); + assert_eq!(second_admission.permission(), UserConnectionPermission::Allowed); + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_eq!(connection_data.connections, 0); + assert_eq!(connection_data.counts.normal, 0); + assert_eq!(connection_data.streams.len(), 0); + assert!(connection_data + .sessions + .iter() + .find(|session| session.token == "tok-first") + .is_some_and(|session| !session.lifecycle.is_counted())); + assert!(connection_data + .sessions + .iter() + .find(|session| session.token == "tok-second") + .is_some_and(|session| !session.lifecycle.is_counted())); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn binding_reserved_sessions_keeps_hard_and_soft_counts_stable() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user-hls-soft"); + user.max_connections = 1; + user.soft_connections = 1; + + let first_addr: SocketAddr = "127.0.0.1:55182".parse().unwrap(); + let second_addr: SocketAddr = "127.0.0.1:55183".parse().unwrap(); + let first_fingerprint = Fingerprint::new("fp-hls-1".to_string(), "127.0.0.1".to_string(), first_addr); + let second_fingerprint = Fingerprint::new("fp-hls-2".to_string(), "127.0.0.1".to_string(), second_addr); + + manager.add_connection(&first_addr).await; + manager.add_connection(&second_addr).await; + + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-normal", + virtual_id: 9203, + provider: "provider-a", + stream_url: "http://localhost/live-normal.m3u8", + addr: &first_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-soft", + virtual_id: 9204, + provider: "provider-a", + stream_url: "http://localhost/live-soft.m3u8", + addr: &second_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let first_admission = manager + .connection_admission_for_session_activation( + &user.username, + user.max_connections, + user.soft_connections, + "tok-normal", + ) + .await; + assert_eq!(first_admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(first_admission.kind(), Some(ConnectionKind::Normal)); + + manager + .update_connection(ActiveUserConnectionParams { + uid: 401, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &first_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(9203), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-normal"), + }) + .await + .expect("reserved normal session should bind"); + + let second_admission = manager + .connection_admission_for_session_activation( + &user.username, + user.max_connections, + user.soft_connections, + "tok-soft", + ) + .await; + assert_eq!(second_admission.permission(), UserConnectionPermission::Allowed); + assert_eq!(second_admission.kind(), Some(ConnectionKind::Soft)); + + manager + .update_connection(ActiveUserConnectionParams { + uid: 402, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Soft, + priority: 0, + soft_priority: 0, + fingerprint: &second_fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(9204), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-soft"), + }) + .await + .expect("reserved soft session should bind"); + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_eq!(connection_data.connections, 2); + assert_eq!(connection_data.counts.normal, 1); + assert_eq!(connection_data.counts.soft, 1); + assert_eq!(connection_data.streams.len(), 2); + assert_eq!( + connection_data.stream_kinds.get(&401), + Some(&ConnectionKind::Normal), + "binding a reserved normal session must not increment counts twice" + ); + assert_eq!( + connection_data.stream_kinds.get(&402), + Some(&ConnectionKind::Soft), + "binding a reserved soft session must keep the soft classification" + ); +} + +#[tokio::test] +async fn playback_transition_gate_serializes_same_session() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = Arc::new(ActiveUserManager::new(&config, &geoip, &event_manager)); + + let first_guard = manager.acquire_playback_transition("user-gated", "tok-gated").await; + let second_manager = Arc::clone(&manager); + let waiting = tokio::spawn(async move { + let _second_guard = second_manager.acquire_playback_transition("user-gated", "tok-gated").await; + }); + + tokio::time::sleep(Duration::from_millis(25)).await; + assert!( + !waiting.is_finished(), + "same-session transition gate should block a concurrent transition until the first completes" + ); + + drop(first_guard); + tokio::time::timeout(Duration::from_millis(100), waiting) + .await + .expect("second transition should proceed once the first guard is released") + .expect("second transition task should complete"); +} + +#[tokio::test] +async fn playback_transition_gate_cleanup_removes_idle_gates_on_next_acquire() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let first_guard = manager.acquire_playback_transition("user-gated-cleanup", "tok-first").await; + assert_eq!(manager.transition_gates.lock().await.len(), 1); + drop(first_guard); + + let second_guard = manager.acquire_playback_transition("user-gated-cleanup", "tok-second").await; + assert_eq!(manager.transition_gates.lock().await.len(), 1); + drop(second_guard); +} + +/// Kicked and evicted sessions are marked ended, also with the reentry guard disabled. +#[tokio::test] +async fn kicked_and_evicted_sessions_are_marked_ended() { + let manager = provider_header_test_manager(); + let mut user = ProxyUserCredentials::default(); + user.username = "user-ended".to_string(); + create_provider_header_session(&manager, &user, "http://cdn.example/a/video.m3u8").await; + let addr: SocketAddr = "127.0.0.1:55420".parse().unwrap_or_else(|_| unreachable!()); + + assert!(!manager.is_session_ended("tok-host").await); + assert!(manager.terminate_session(&user.username, "tok-host").await); + assert!(!manager.is_session_ended("tok-host").await, "a plain terminate (e.g. manifest failure) is not an end"); + + create_provider_header_session(&manager, &user, "http://cdn.example/a/video.m3u8").await; + manager.terminate_sessions_for_addr(&user.username, &addr).await; + assert!(manager.is_session_ended("tok-host").await); + + manager.mark_session_ended("tok-explicit").await; + assert!(manager.is_session_ended("tok-explicit").await); +} + +pub(in crate::active_user_manager::tests) async fn register_guarded_resource( + manager: &ActiveUserManager, + user: &ProxyUserCredentials, + token: &str, + uid: u32, + requirement: &PlaybackSessionRegistration, +) -> Option { + let addr = SocketAddr::from(([127, 0, 0, 1], 55500 + u16::try_from(uid).ok()?)); + let fingerprint = Fingerprint::new("guarded-player".to_string(), "127.0.0.1".to_string(), addr); + manager + .update_connection_with_session_registration( + ActiveUserConnectionParams { + uid, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: ConnectionKind::Normal, + priority: user.priority, + soft_priority: user.soft_priority, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(7010), + user_agent: Cow::Borrowed("guarded-player"), + session_token: Some(token), + }, + Some(requirement), + ) + .await +} + +#[tokio::test] +async fn guarded_resource_registration_rejects_removed_and_replaced_sessions() -> Result<(), Box> +{ + let manager = provider_header_test_manager(); + let mut user = ProxyUserCredentials::default(); + user.username = "registration-identity".to_string(); + user.max_connections = 1; + create_provider_header_session(&manager, &user, "http://provider.example/index.m3u8").await; + let identity = session_identity(&manager, &user.username, "tok-host").await.ok_or("session missing")?; + let requirement = PlaybackSessionRegistration { identity, enforce_limits: true, grace_admitted: false }; + assert!(manager.terminate_session(&user.username, "tok-host").await); + assert!(register_guarded_resource(&manager, &user, "tok-host", 1, &requirement).await.is_none()); + assert_eq!(manager.user_connections(&user.username).await, 0); + assert!(manager.connections.read().await.key_by_addr.is_empty(), "rejection must not mutate socket registrations"); + create_provider_header_session(&manager, &user, "http://provider.example/index.m3u8").await; + assert!(register_guarded_resource(&manager, &user, "tok-host", 2, &requirement).await.is_none()); + let current = PlaybackSessionRegistration { + identity: session_identity(&manager, &user.username, "tok-host").await.ok_or("replacement missing")?, + ..requirement + }; + assert!(register_guarded_resource(&manager, &user, "tok-host", 3, ¤t).await.is_some()); + assert!(register_guarded_resource(&manager, &user, "tok-host", 4, ¤t).await.is_some()); + assert_eq!(manager.user_connections(&user.username).await, 1, "parallel resources share one user admission"); + Ok(()) +} + +#[tokio::test] +async fn session_end_wakes_waiters_without_polling() -> Result<(), Box> { + let manager = Arc::new(provider_header_test_manager()); + let mut user = ProxyUserCredentials::default(); + user.username = "session-end-waiter".to_string(); + create_provider_header_session(&manager, &user, "http://provider.example/index.m3u8").await; + let identity = session_identity(&manager, &user.username, "tok-host").await.ok_or("session missing")?; + let waiter = { + let manager = Arc::clone(&manager); + let username = user.username.clone(); + tokio::spawn(async move { manager.wait_for_playback_session_end(&username, "tok-host", identity).await }) + }; + tokio::task::yield_now().await; + assert!(!waiter.is_finished(), "a current session must keep the waiter pending"); + assert!(manager.terminate_session(&user.username, "tok-host").await); + tokio::time::timeout(Duration::from_secs(1), waiter).await??; + Ok(()) +} + +#[tokio::test] +async fn guarded_resource_registration_rechecks_user_capacity_at_commit() -> Result<(), Box> { + let manager = provider_header_test_manager(); + let mut user = ProxyUserCredentials::default(); + user.username = "registration-capacity".to_string(); + user.max_connections = 1; + create_provider_header_session(&manager, &user, "http://provider.example/index.m3u8").await; + let first = PlaybackSessionRegistration { + identity: session_identity(&manager, &user.username, "tok-host").await.ok_or("session missing")?, + enforce_limits: true, + grace_admitted: false, + }; + let addr = SocketAddr::from(([127, 0, 0, 1], 55421)); + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "second-session", + virtual_id: 7011, + provider: "provider-a", + stream_url: "http://provider.example/other.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + let second = PlaybackSessionRegistration { + identity: session_identity(&manager, &user.username, "second-session").await.ok_or("second session missing")?, + enforce_limits: true, + grace_admitted: false, + }; + assert!(register_guarded_resource(&manager, &user, "tok-host", 5, &first).await.is_some()); + assert!(register_guarded_resource(&manager, &user, "second-session", 6, &second).await.is_none()); + assert_eq!(manager.user_connections(&user.username).await, 1); + assert!( + manager.playback_session_is_current(&user.username, "second-session", second.identity).await, + "capacity rejection does not terminate an otherwise valid session" + ); + Ok(()) +} diff --git a/backend/session/src/active_user_manager/tests/socket_activity.rs b/backend/session/src/active_user_manager/tests/socket_activity.rs new file mode 100644 index 000000000..0af494fd2 --- /dev/null +++ b/backend/session/src/active_user_manager/tests/socket_activity.rs @@ -0,0 +1,1418 @@ +use super::{ + test_adaptive_channel, test_channel, test_series_channel, test_user_credentials, ActiveUserConnectionParams, + ActiveUserManager, CreateUserSessionParams, PlaybackLifecycle, SocketRegistration, ANON_SOCKET_TTL, + DEFAULT_ACTIVE_SOCKET_TTL_SECS, USER_GC_TTL, +}; +use crate::{active_provider_manager::ConnectionKind, EventManager}; +use arc_swap::ArcSwapOption; +use shared::{ + model::{ + ActiveUserConnectionChange, EventMessage, PlaylistItemType, StreamChannel, UserConnectionPermission, + XtreamCluster, + }, + utils::{current_time_secs, Internable}, +}; +use std::{ + borrow::Cow, + net::SocketAddr, + sync::{atomic::Ordering, Arc}, +}; +use tuliprox_core::model::{Config, Fingerprint, ProxyUserCredentials}; +use tuliprox_repository::GeoIp; + +#[tokio::test] +async fn test_multi_session_same_addr_counts_and_releases_individually() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55001".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key".to_string(), "127.0.0.1".to_string(), addr); + let username = "user1"; + + manager.add_connection(&addr).await; + + let first = manager + .update_connection(ActiveUserConnectionParams { + uid: 1, + meter_uid: 0, + username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(1001), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-1"), + }) + .await; + assert!(first.is_some()); + assert_eq!(manager.user_connections(username).await, 1); + assert_eq!(manager.connection_permission(username, 1, 0).await, UserConnectionPermission::Exhausted); + + let second = manager + .update_connection(ActiveUserConnectionParams { + uid: 2, + meter_uid: 0, + username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-b".intern(), + stream_channel: &test_channel(1002), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-2"), + }) + .await; + assert!(second.is_some()); + assert_eq!(manager.user_connections(username).await, 2); + + assert!(manager.release_stream(&addr).await.is_some()); + assert_eq!(manager.user_connections(username).await, 1); + + assert!(manager.release_stream(&addr).await.is_some()); + assert_eq!(manager.user_connections(username).await, 0); +} + +/// `terminate_sessions_for_addr` expires all sessions at a given addr and releases counted leases. +#[tokio::test] +async fn terminate_sessions_for_addr_expires_all_sessions_at_addr() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr_kick: SocketAddr = "127.0.0.1:55420".parse().unwrap(); + let addr_keep: SocketAddr = "127.0.0.1:55421".parse().unwrap(); + let mut user = ProxyUserCredentials::default(); + user.username = "user-kick-addr".to_string(); + user.max_connections = 4; + + // Create session at kicked addr. + let tok_kick = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-kick", + virtual_id: 1, + provider: "provider-a", + stream_url: "http://provider/live/1.m3u8", + addr: &addr_kick, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + // Create session at kept addr. + let tok_keep = manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-keep", + virtual_id: 2, + provider: "provider-b", + stream_url: "http://provider/live/2.m3u8", + addr: &addr_keep, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + // Mark both sessions as counted and active. + { + let mut connections = manager.connections.write().await; + let data = connections.by_key.get_mut(&user.username).unwrap(); + for session in &mut data.sessions { + // Simulate counted state by setting lifecycle to Active. + session.lifecycle = PlaybackLifecycle::Active; + } + data.increment_kind(ConnectionKind::Normal); + data.increment_kind(ConnectionKind::Normal); + } + + assert_eq!(manager.user_connections(&user.username).await, 2); + + // Kick the addr — should terminate only the sessions at that addr. + manager.terminate_sessions_for_addr(&user.username, &addr_kick).await; + + // Session at kicked addr should be gone. + assert!( + manager.get_and_update_user_session(&user.username, &tok_kick).await.is_none(), + "kicked session should be removed" + ); + + // Session at kept addr should remain. + let kept = + manager.get_and_update_user_session(&user.username, &tok_keep).await.expect("kept session should still exist"); + assert_eq!(kept.token, tok_keep); + assert_eq!(kept.addr, addr_keep); + + // Connection count should drop by 1. + assert_eq!(manager.user_connections(&user.username).await, 1); +} + +#[tokio::test] +async fn eviction_candidates_ignore_ambiguous_socket_addrs() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let shared_addr: SocketAddr = "127.0.0.1:55031".parse().unwrap(); + let unique_addr: SocketAddr = "127.0.0.1:55032".parse().unwrap(); + let shared_fp = Fingerprint::new("fp-shared".to_string(), "127.0.0.1".to_string(), shared_addr); + let unique_fp = Fingerprint::new("fp-unique".to_string(), "127.0.0.1".to_string(), unique_addr); + + manager.add_connection(&shared_addr).await; + manager.add_connection(&unique_addr).await; + + // Create sessions first so update_connection can mark them as counted. + let user = test_user_credentials("same-user", 3, 0); + for (token, addr, channel_id) in + [("tok-31", shared_addr, 1031u32), ("tok-32", shared_addr, 1032), ("tok-33", unique_addr, 1033)] + { + manager + .create_user_session(crate::CreateUserSessionParams { + user: &user, + session_token: token, + virtual_id: channel_id, + provider: "provider-a", + stream_url: "", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + } + + // update_connection marks the session as counted. + for (uid, token, fp, channel_id) in [(31, "tok-31", &shared_fp, 1031u32), (32, "tok-32", &shared_fp, 1032)] { + manager + .update_connection(ActiveUserConnectionParams { + uid, + meter_uid: 0, + username: "same-user", + max_connections: 3, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: fp, + provider: "provider-a".intern(), + stream_channel: &test_channel(channel_id), + user_agent: Cow::Borrowed("ua"), + session_token: Some(token), + }) + .await; + } + + manager + .update_connection(ActiveUserConnectionParams { + uid: 33, + meter_uid: 0, + username: "same-user", + max_connections: 3, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &unique_fp, + provider: "provider-a".intern(), + stream_channel: &test_channel(1033), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-33"), + }) + .await; + + let candidates = manager.get_eviction_candidates("same-user", "127.0.0.1").await; + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].addr, unique_addr); +} + +#[tokio::test] +async fn kicked_release_removes_preserved_adaptive_stream_without_socket_registration() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55017".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-preserved-kick".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user-preserved-kick"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-preserved-kick", + virtual_id: 2017, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 17, + meter_uid: 0, + username: &user.username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(2017), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-preserved-kick"), + }) + .await; + + let released = manager.release_connection(&addr).await; + assert!(released.addr_removed); + assert!(released.removed_streams.is_empty()); + assert!(manager.active_streams().await.is_empty()); + + let kicked = manager.release_connection_as_kicked(&addr).await; + assert!(kicked.addr_removed); + assert_eq!(kicked.removed_streams.len(), 1); + assert!(manager.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn test_same_session_token_on_new_addr_reuses_logical_connection() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let first_addr: SocketAddr = "127.0.0.1:55021".parse().unwrap(); + let second_addr: SocketAddr = "127.0.0.1:55022".parse().unwrap(); + let first = Fingerprint::new("fp-key-1".to_string(), "127.0.0.1".to_string(), first_addr); + let second = Fingerprint::new("fp-key-2".to_string(), "127.0.0.1".to_string(), second_addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&first_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-hls", + virtual_id: 2001, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &first_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 0, + meter_uid: 0, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &first, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(2001), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-hls"), + }) + .await; + + assert_eq!( + manager.connection_permission_for_session("user1", 1, 0, "tok-hls").await, + UserConnectionPermission::Allowed + ); + + manager.add_connection(&second_addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 0, + meter_uid: 0, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &second, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(2001), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-hls"), + }) + .await; + + assert_eq!(manager.user_connections("user1").await, 1); + + let streams = manager.active_streams().await; + assert_eq!(streams.len(), 1); + assert_eq!(streams[0].addr, second_addr); + assert_eq!(streams[0].session_token.as_deref(), Some("tok-hls")); +} + +#[tokio::test] +async fn adaptive_session_stream_cleanup_addrs_excludes_manifest_addr_and_current_addr() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let manifest_addr: SocketAddr = "127.0.0.1:55091".parse().unwrap(); + let first_segment_addr: SocketAddr = "10.41.41.89:55092".parse().unwrap(); + let next_segment_addr: SocketAddr = "10.41.41.89:55093".parse().unwrap(); + let first_segment = Fingerprint::new("fp-segment-1".to_string(), "10.41.41.89".to_string(), first_segment_addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&manifest_addr).await; + manager.add_connection(&first_segment_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-hls-cleanup", + virtual_id: 2002, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &manifest_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 0, + meter_uid: 0, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &first_segment, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::LiveHls, ..test_channel(2002) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-hls-cleanup"), + }) + .await; + + assert_eq!( + manager.adaptive_session_stream_cleanup_addrs("user1", "tok-hls-cleanup", &next_segment_addr).await, + vec![first_segment_addr] + ); +} + +#[tokio::test] +async fn adaptive_session_stream_cleanup_addrs_falls_back_to_same_ip_session_addrs() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let manifest_addr: SocketAddr = "127.0.0.1:55101".parse().unwrap(); + let first_segment_addr: SocketAddr = "10.41.41.89:55102".parse().unwrap(); + let next_segment_addr: SocketAddr = "10.41.41.89:55103".parse().unwrap(); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user2"); + user.max_connections = 1; + + manager.add_connection(&manifest_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-hls-cleanup-fallback", + virtual_id: 2003, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &manifest_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager.update_session_addr("user2", "tok-hls-cleanup-fallback", &first_segment_addr).await; + manager.update_session_addr("user2", "tok-hls-cleanup-fallback", &next_segment_addr).await; + + assert_eq!( + manager.adaptive_session_stream_cleanup_addrs("user2", "tok-hls-cleanup-fallback", &next_segment_addr).await, + vec![first_segment_addr] + ); +} + +#[tokio::test] +async fn socket_bound_live_streams_with_colliding_token_are_tracked_separately() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let Some(addr) = "127.0.0.1:55032".parse::().ok() else { + return; + }; + let fingerprint = Fingerprint::new("fp-key-colliding".to_string(), "127.0.0.1".to_string(), addr); + + manager.add_connection(&addr).await; + let first = manager + .update_connection(ActiveUserConnectionParams { + uid: 31, + meter_uid: 301, + username: "user1", + max_connections: 0, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(3003), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-live-colliding"), + }) + .await; + let second = manager + .update_connection(ActiveUserConnectionParams { + uid: 32, + meter_uid: 302, + username: "user1", + max_connections: 0, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-b".intern(), + stream_channel: &test_channel(3003), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-live-colliding"), + }) + .await; + + assert!(first.is_some()); + assert!(second.is_some()); + + let streams = manager.active_streams().await; + assert_eq!(streams.len(), 2); + assert!(streams.iter().any(|stream| stream.uid == 31)); + assert!(streams.iter().any(|stream| stream.uid == 32)); +} + +#[tokio::test] +async fn release_stream_by_uid_finds_original_user_after_shared_addr_owner_changes() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55034".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-cross-user-stream".to_string(), "127.0.0.1".to_string(), addr); + manager.add_connection(&addr).await; + + for (uid, username) in [(43, "user-a"), (44, "user-b")] { + manager + .update_connection(ActiveUserConnectionParams { + uid, + meter_uid: 0, + username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_series_channel(3005), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("direct Series stream should register"); + } + + assert_eq!(manager.active_users_and_connections().await, (2, 2)); + assert_eq!(manager.active_streams().await.len(), 2); + + let removed = manager.release_stream_by_uid(&addr, 43).await; + assert!(removed.as_ref().is_some_and(|stream| stream.uid == 43)); + assert_eq!(manager.active_users_and_connections().await, (1, 1)); + let streams = manager.active_streams().await; + assert_eq!(streams.len(), 1); + assert_eq!(streams[0].uid, 44); + + assert!(manager.release_stream_by_uid(&addr, 44).await.is_some()); + assert_eq!(manager.active_users_and_connections().await, (0, 0)); + assert!(manager.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn release_connection_cleans_every_user_stream_for_reused_addr_only() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let reused_addr: SocketAddr = "127.0.0.1:55035".parse().unwrap(); + let unrelated_addr: SocketAddr = "127.0.0.1:55036".parse().unwrap(); + let reused_fingerprint = Fingerprint::new("fp-cross-user-socket".to_string(), "127.0.0.1".to_string(), reused_addr); + let unrelated_fingerprint = + Fingerprint::new("fp-unrelated-socket".to_string(), "127.0.0.1".to_string(), unrelated_addr); + manager.add_connection(&reused_addr).await; + manager.add_connection(&unrelated_addr).await; + + for (uid, username, fingerprint) in [ + (45, "user-a", &reused_fingerprint), + (46, "user-b", &reused_fingerprint), + (47, "user-a", &unrelated_fingerprint), + ] { + manager + .update_connection(ActiveUserConnectionParams { + uid, + meter_uid: 0, + username, + max_connections: 2, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_series_channel(3006 + uid), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("direct Series stream should register"); + } + + let released = manager.release_connection(&reused_addr).await; + let mut removed_uids = released.removed_streams.iter().map(|stream| stream.uid).collect::>(); + removed_uids.sort_unstable(); + assert!(released.addr_removed); + assert_eq!(removed_uids, vec![45, 46]); + assert_eq!(manager.active_users_and_connections().await, (1, 1)); + let streams = manager.active_streams().await; + assert_eq!(streams.len(), 1); + assert_eq!(streams[0].uid, 47); + + manager.release_connection(&unrelated_addr).await; + assert_eq!(manager.active_users_and_connections().await, (0, 0)); + assert!(manager.active_streams().await.is_empty()); +} + +#[tokio::test] +async fn stale_anonymous_socket_registration_is_pruned_by_gc() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let stale_addr: SocketAddr = "127.0.0.1:55011".parse().unwrap(); + let fresh_addr: SocketAddr = "127.0.0.1:55012".parse().unwrap(); + + manager.add_connection(&stale_addr).await; + { + let mut connections = manager.connections.write().await; + let registration = connections.key_by_addr.get_mut(&stale_addr).expect("socket registration should exist"); + registration.ts = registration.ts.saturating_sub(ANON_SOCKET_TTL + 1); + } + + if let Some(gc_ts) = &manager.gc_ts { + gc_ts.store(current_time_secs().saturating_sub(USER_GC_TTL + 1), Ordering::Release); + } + + manager.add_connection(&fresh_addr).await; + + let connections = manager.connections.read().await; + assert!(!connections.key_by_addr.contains_key(&stale_addr)); + assert!(connections.key_by_addr.contains_key(&fresh_addr)); +} + +#[tokio::test] +async fn touch_http_activity_refreshes_session_and_registration_without_stream() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55024".parse().unwrap(); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-http-touch", + virtual_id: 9302, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + let previous_ts = { + let mut connections = manager.connections.write().await; + let previous_ts = { + let registration = connections.key_by_addr.get_mut(&addr).expect("registration should exist"); + registration.ts = registration.ts.saturating_sub(DEFAULT_ACTIVE_SOCKET_TTL_SECS + 5); + registration.ts + }; + let connection_data = connections.by_key.get_mut("user1").expect("user should exist"); + connection_data.sessions[0].ts = + connection_data.sessions[0].ts.saturating_sub(DEFAULT_ACTIVE_SOCKET_TTL_SECS + 5); + previous_ts + }; + + manager.touch_http_activity("user1", "tok-http-touch", &addr).await; + + let connections = manager.connections.read().await; + let registration = connections.key_by_addr.get(&addr).expect("registration should still exist"); + let connection_data = connections.by_key.get("user1").expect("user should still exist"); + assert!(registration.ts > previous_ts); + assert!(connection_data.sessions[0].ts >= registration.ts); +} + +#[tokio::test] +async fn touch_http_activity_does_not_reset_stream_started_at_ts() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr1: SocketAddr = "127.0.0.1:55030".parse().unwrap(); + let addr2: SocketAddr = "127.0.0.1:55031".parse().unwrap(); + let fingerprint = Fingerprint::new("fp".to_string(), "127.0.0.1".to_string(), addr1); + let mut user = ProxyUserCredentials::default(); + user.username = "user-touch-ts".to_string(); + + manager.add_connection(&addr1).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-hls-ts", + virtual_id: 7777, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &addr1, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + // Simulate first HLS segment: creates the stream entry with ts = now + manager + .update_connection(ActiveUserConnectionParams { + uid: 601, + meter_uid: 701, + username: "user-touch-ts", + max_connections: 0, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(7777), + user_agent: Cow::Borrowed("player/1.0"), + session_token: Some("tok-hls-ts"), + }) + .await + .expect("stream should be created"); + + // Record the original stream start timestamp + let original_ts = { + let connections = manager.connections.read().await; + connections + .by_key + .get("user-touch-ts") + .and_then(|data| data.streams.iter().find(|s| s.session_token.as_deref() == Some("tok-hls-ts"))) + .map(|s| s.ts) + .expect("stream should exist") + }; + + // Simulate manifest re-fetch (touch_http_activity called with a new addr) + manager.touch_http_activity("user-touch-ts", "tok-hls-ts", &addr2).await; + + // stream.ts must NOT have been reset — it represents session start time shown as Duration + let connections = manager.connections.read().await; + let stream = connections + .by_key + .get("user-touch-ts") + .and_then(|data| data.streams.iter().find(|s| s.session_token.as_deref() == Some("tok-hls-ts"))) + .expect("stream should still exist"); + assert_eq!(stream.ts, original_ts, "touch_http_activity must not reset the stream start timestamp"); + // Lightweight manifest activity must not move the active stream socket. + assert_eq!(stream.addr, addr1, "touch_http_activity must not replace the active stream addr"); +} + +#[tokio::test] +async fn touch_http_activity_does_not_migrate_adaptive_stream_to_manifest_addr_on_close() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + let mut events = event_manager.get_event_channel(); + + let segment_addr: SocketAddr = "127.0.0.1:55032".parse().unwrap(); + let manifest_addr: SocketAddr = "127.0.0.1:55033".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-hls-segment".to_string(), "127.0.0.1".to_string(), segment_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "user-hls-manifest-touch".to_string(); + user.max_connections = 1; + + manager.add_connection(&segment_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-hls-manifest-touch", + virtual_id: 7788, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &segment_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 602, + meter_uid: 702, + username: &user.username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(7788), + user_agent: Cow::Borrowed("player/1.0"), + session_token: Some("tok-hls-manifest-touch"), + }) + .await + .expect("stream should be created"); + + manager.touch_http_activity(&user.username, "tok-hls-manifest-touch", &manifest_addr).await; + + let released = manager.release_connection(&segment_addr).await; + assert!(released.addr_removed); + assert!(released.removed_streams.is_empty(), "adaptive close should preserve without history removal"); + assert_eq!(manager.user_connections(&user.username).await, 0); + assert!( + manager.active_streams().await.is_empty(), + "preserved rows stay out of active_streams (use panel_streams for StatusCheck)" + ); + let panel = manager.panel_streams().await; + assert_eq!(panel.len(), 1, "preserved adaptive/catchup session rows stay in panel snapshots"); + assert!(panel[0].preserved); + assert_eq!(panel[0].session_token.as_deref(), Some("tok-hls-manifest-touch")); + + let connections = manager.connections.read().await; + let data = connections.by_key.get(&user.username).expect("user should remain for preserved session"); + let stream = data + .streams + .iter() + .find(|stream| stream.session_token.as_deref() == Some("tok-hls-manifest-touch")) + .expect("preserved stream should remain internally tracked"); + assert!(stream.preserved); + assert_eq!(stream.addr, segment_addr, "closed segment must not migrate to manifest addr"); + assert!(!data.sessions[0].active_addrs.contains(&manifest_addr)); + drop(connections); + + let mut saw_preserved_update = false; + while let Ok(event) = events.try_recv() { + if matches!(event, EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream)) if stream.addr == segment_addr && stream.preserved) + { + saw_preserved_update = true; + } + } + assert!(saw_preserved_update, "preserving a stream must notify the frontend so adaptive TTL cleanup can hide it"); +} + +#[tokio::test] +async fn clear_unbound_session_addr_prunes_manifest_addr_while_stream_is_active_elsewhere() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let segment_addr: SocketAddr = "127.0.0.1:55034".parse().unwrap(); + let manifest_addr: SocketAddr = "127.0.0.1:55035".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-hls-segment-2".to_string(), "127.0.0.1".to_string(), segment_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "user-hls-manifest-clear".to_string(); + user.max_connections = 1; + + manager.add_connection(&segment_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-hls-manifest-clear", + virtual_id: 7789, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &segment_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 603, + meter_uid: 703, + username: &user.username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(7789), + user_agent: Cow::Borrowed("player/1.0"), + session_token: Some("tok-hls-manifest-clear"), + }) + .await + .expect("stream should be created"); + + manager.add_connection(&manifest_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-hls-manifest-clear", + virtual_id: 7789, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &manifest_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + manager.clear_unbound_session_addr(&user.username, "tok-hls-manifest-clear", &manifest_addr).await; + + let connections = manager.connections.read().await; + assert!(!connections.key_by_addr.contains_key(&manifest_addr)); + let data = connections.by_key.get(&user.username).expect("user should exist"); + assert_eq!(data.streams[0].addr, segment_addr); + assert!(!data.sessions[0].active_addrs.contains(&manifest_addr)); +} + +#[tokio::test] +async fn clear_unbound_session_addr_prunes_touch_only_manifest_addr() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let segment_addr: SocketAddr = "127.0.0.1:55036".parse().unwrap(); + let manifest_addr: SocketAddr = "127.0.0.1:55037".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-hls-segment-3".to_string(), "127.0.0.1".to_string(), segment_addr); + let mut user = ProxyUserCredentials::default(); + user.username = "user-hls-manifest-touch-clear".to_string(); + user.max_connections = 1; + + manager.add_connection(&segment_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-hls-manifest-touch-clear", + virtual_id: 7790, + provider: "provider-a", + stream_url: "http://localhost/live.m3u8", + addr: &segment_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 604, + meter_uid: 704, + username: &user.username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(7790), + user_agent: Cow::Borrowed("player/1.0"), + session_token: Some("tok-hls-manifest-touch-clear"), + }) + .await + .expect("stream should be created"); + + manager.touch_http_activity(&user.username, "tok-hls-manifest-touch-clear", &manifest_addr).await; + manager.clear_unbound_session_addr(&user.username, "tok-hls-manifest-touch-clear", &manifest_addr).await; + + let connections = manager.connections.read().await; + assert!(!connections.key_by_addr.contains_key(&manifest_addr)); + let data = connections.by_key.get(&user.username).expect("user should exist"); + assert_eq!(data.streams[0].addr, segment_addr); + assert_eq!(data.sessions[0].addr, segment_addr); + assert!(!data.sessions[0].active_addrs.contains(&manifest_addr)); +} + +#[tokio::test] +async fn touch_socket_activity_refreshes_registration_without_resetting_stream_start() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55041".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-vod-touch".to_string(), "127.0.0.1".to_string(), addr); + let mut user = ProxyUserCredentials::default(); + user.username = "user-vod-touch".to_string(); + user.max_connections = 1; + + manager.add_connection(&addr).await; + let mut channel = test_channel(8889); + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + channel.url = "http://localhost/movie-2.mkv".intern(); + + let stream = manager + .update_connection(ActiveUserConnectionParams { + uid: 603, + meter_uid: 703, + username: "user-vod-touch", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &channel, + user_agent: Cow::Borrowed("player/1.0"), + session_token: None, + }) + .await + .expect("vod stream should be created"); + + let stale_registration_ts = { + let mut connections = manager.connections.write().await; + let registration = connections.key_by_addr.get_mut(&addr).expect("registration should exist"); + registration.ts = registration.ts.saturating_sub(DEFAULT_ACTIVE_SOCKET_TTL_SECS + 5); + registration.ts + }; + + manager.touch_socket_activity(&addr).await; + + let (refreshed_registration_ts, stream_started_at) = { + let connections = manager.connections.read().await; + let registration_ts = connections.key_by_addr.get(&addr).expect("registration should still exist").ts; + let stream_started_at = connections + .by_key + .get("user-vod-touch") + .and_then(|data| data.streams.iter().find(|active| active.uid == stream.uid)) + .expect("stream should still exist") + .ts; + (registration_ts, stream_started_at) + }; + + assert!(refreshed_registration_ts > stale_registration_ts); + assert_eq!(stream_started_at, stream.ts, "body activity must not reset visible stream duration"); +} + +#[tokio::test] +async fn update_session_addr_prunes_previous_registration_for_socket_bound_session() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let old_addr: SocketAddr = "127.0.0.1:55121".parse().unwrap(); + let new_addr: SocketAddr = "127.0.0.1:55122".parse().unwrap(); + let old_fingerprint = Fingerprint::new("fp-old".to_string(), "127.0.0.1".to_string(), old_addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&old_addr).await; + manager.add_connection(&new_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-move", + virtual_id: 9101, + provider: "provider-a", + stream_url: "http://localhost/live.ts", + addr: &old_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: true, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 301, + meter_uid: 401, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &old_fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::Live, ..test_channel(9101) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-move"), + }) + .await + .expect("initial live stream should register"); + + manager.update_session_addr("user1", "tok-move", &new_addr).await; + + let connections = manager.connections.read().await; + assert!( + !connections.key_by_addr.contains_key(&old_addr), + "previous range-request socket registration should be pruned once the session moved" + ); + assert!(connections.key_by_addr.contains_key(&new_addr)); + + let connection_data = connections.by_key.get("user1").expect("user connection data"); + assert_eq!(connection_data.sessions.len(), 1); + assert_eq!(connection_data.sessions[0].addr, new_addr); + assert_eq!(connection_data.streams.len(), 1); + assert_eq!(connection_data.streams[0].addr, new_addr); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn vod_session_survives_overlapping_and_seek_sockets() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let base_addr: SocketAddr = "127.0.0.1:55131".parse().unwrap(); + let range_addr: SocketAddr = "127.0.0.1:55132".parse().unwrap(); + let seek_addr: SocketAddr = "127.0.0.1:55133".parse().unwrap(); + let base_fingerprint = Fingerprint::new("fp-vod-base".to_string(), "127.0.0.1".to_string(), base_addr); + let range_fingerprint = Fingerprint::new("fp-vod-range".to_string(), "127.0.0.1".to_string(), range_addr); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user1"); + user.max_connections = 1; + + manager.add_connection(&base_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-vod", + virtual_id: 9102, + provider: "provider-a", + stream_url: "http://localhost/movie.mkv", + addr: &base_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 302, + meter_uid: 402, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &base_fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::Video, ..test_channel(9102) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-vod"), + }) + .await + .expect("initial vod stream should register"); + + manager.add_connection(&range_addr).await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 303, + meter_uid: 403, + username: "user1", + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &range_fingerprint, + provider: "provider-a".intern(), + stream_channel: &StreamChannel { item_type: PlaylistItemType::Video, ..test_channel(9102) }, + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-vod"), + }) + .await + .expect("overlapping range request should reuse the same vod session"); + + assert_eq!(manager.user_connections("user1").await, 1); + assert!(manager.release_stream(&range_addr).await.is_none()); + let released = manager.release_connection(&range_addr).await; + assert!(released.addr_removed); + assert!(released.removed_streams.is_empty()); + + { + let connections = manager.connections.read().await; + assert!(connections.key_by_addr.contains_key(&base_addr)); + let connection_data = connections.by_key.get("user1").expect("user connection data"); + assert_eq!(connection_data.sessions[0].addr, base_addr); + assert_eq!(connection_data.streams[0].addr, base_addr); + } + + manager.add_connection(&seek_addr).await; + manager.update_session_addr("user1", "tok-vod", &seek_addr).await; + + { + let connections = manager.connections.read().await; + assert!( + connections.key_by_addr.contains_key(&base_addr), + "existing vod socket must remain registered while the session spans multiple requests" + ); + assert!(connections.key_by_addr.contains_key(&seek_addr)); + + let connection_data = connections.by_key.get("user1").expect("user connection data"); + assert_eq!(connection_data.sessions[0].addr, seek_addr); + assert_eq!(connection_data.streams[0].addr, seek_addr); + } + + assert!(manager.release_stream(&seek_addr).await.is_none()); + let released = manager.release_connection(&seek_addr).await; + assert!(released.addr_removed); + assert!(released.removed_streams.is_empty()); + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get("user1").expect("user connection data"); + assert_eq!(connection_data.sessions[0].addr, base_addr); + assert_eq!(connection_data.streams[0].addr, base_addr); +} + +#[tokio::test] +async fn clear_unbound_session_addr_prunes_manifest_addr_without_stream() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let first_addr: SocketAddr = "127.0.0.1:55185".parse().unwrap(); + let second_addr: SocketAddr = "127.0.0.1:55186".parse().unwrap(); + let mut user = ProxyUserCredentials::default(); + user.username = String::from("user-clear-addr"); + user.max_connections = 1; + + manager.add_connection(&first_addr).await; + manager.add_connection(&second_addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-clear-addr", + virtual_id: 9206, + provider: "provider-a", + stream_url: "http://localhost/live-clear.m3u8", + addr: &first_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-clear-addr", + virtual_id: 9206, + provider: "provider-a", + stream_url: "http://localhost/live-clear.m3u8", + addr: &second_addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + + manager.clear_unbound_session_addr(&user.username, "tok-clear-addr", &second_addr).await; + + let connections = manager.connections.read().await; + let session = connections + .by_key + .get(&user.username) + .and_then(|connection_data| connection_data.sessions.iter().find(|session| session.token == "tok-clear-addr")) + .expect("session should remain"); + assert_eq!(session.addr, first_addr); + assert_eq!(session.active_addrs, vec![first_addr]); +} + +#[tokio::test] +async fn get_eviction_candidates_does_not_count_preserved_streams_in_addr_counts() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "127.0.0.1:55801".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-preserved-no-count".to_string(), "10.0.0.5".to_string(), addr); + let username = "user-preserved-addr-count"; + let mut user = ProxyUserCredentials::default(); + user.username = username.to_string(); + user.max_connections = 1; + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-preserved-addr-count", + virtual_id: 7000, + provider: "provider-preserved", + stream_url: "http://localhost/preserved.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: 7000, + meter_uid: 0, + username, + max_connections: 1, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-preserved".intern(), + stream_channel: &test_adaptive_channel(7000), + user_agent: Cow::Borrowed("ua"), + session_token: Some("tok-preserved-addr-count"), + }) + .await + .expect("stream should be created"); + + // Release -> stream becomes preserved, session becomes uncounted + manager.release_stream(&addr).await; + + // Preserved streams do not consume a counted slot — user_connections should be 0 + assert_eq!( + manager.user_connections(username).await, + 0, + "preserved stream should not count toward active connections" + ); + + // But the preserved stream is still a valid eviction candidate (valid victim) + let candidates = manager.get_eviction_candidates(username, "10.0.0.5").await; + assert!(candidates.iter().any(|c| c.addr == addr), "preserved stream should be an eviction candidate"); +} + +#[test] +fn socket_registration_primary_username_is_deterministic() { + let mut reg = SocketRegistration::anonymous(); + reg.add_user("user_z", 100); + reg.add_user("user_a", 100); + reg.add_user("user_m", 100); + assert_eq!(reg.primary_username(), Some("user_a")); +} + +#[tokio::test] +async fn release_stream_without_uid_rejects_ambiguous_multiple_users_on_same_addr() { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + let manager = ActiveUserManager::new(&config, &geoip, &event_manager); + + let addr: SocketAddr = "192.168.1.100:12345".parse().unwrap(); + let fingerprint = Fingerprint::new("fp-key".to_string(), "192.168.1.100".to_string(), addr); + + manager.add_connection(&addr).await; + + let stream_a = manager + .update_connection(ActiveUserConnectionParams { + uid: 1, + meter_uid: 0, + username: "user_a", + max_connections: 5, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_channel(1001), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("register user_a"); + + let stream_b = manager + .update_connection(ActiveUserConnectionParams { + uid: 2, + meter_uid: 0, + username: "user_b", + max_connections: 5, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider-b".intern(), + stream_channel: &test_channel(1002), + user_agent: Cow::Borrowed("ua"), + session_token: None, + }) + .await + .expect("register user_b"); + + // Releasing without stream_uid when multiple users are on the same addr must return None (ambiguous) + let released = manager.release_stream(&addr).await; + assert!( + released.is_none(), + "releasing without uid must not arbitrarily pick a user when multiple are active on the same addr" + ); + + // But releasing with explicit stream_uid works cleanly: + let released_a = manager.release_stream_by_uid(&addr, stream_a.uid).await; + assert!(released_a.is_some(), "releasing with stream_uid succeeds"); + + // Now only user_b remains on addr. Releasing without stream_uid should now succeed: + let released_b = manager.release_stream(&addr).await; + assert!(released_b.is_some(), "releasing without stream_uid succeeds when only a single user remains on addr"); + assert_eq!(released_b.unwrap().uid, stream_b.uid); +} diff --git a/backend/session/src/active_user_manager/tests/support.rs b/backend/session/src/active_user_manager/tests/support.rs new file mode 100644 index 000000000..ac32cba02 --- /dev/null +++ b/backend/session/src/active_user_manager/tests/support.rs @@ -0,0 +1,290 @@ +use super::{ + ActiveUserConnectionParams, ActiveUserManager, CreateUserSessionParams, PlaybackLifecycle, SessionIdentity, + UserConnectionCounts, UserConnectionData, UserSession, +}; +use crate::{active_provider_manager::ConnectionKind, EventManager}; +use arc_swap::ArcSwapOption; +use shared::{ + model::{PlaylistItemType, ProxyType, StreamChannel, UserConnectionPermission, XtreamCluster}, + utils::Internable, +}; +use std::{borrow::Cow, net::SocketAddr, sync::Arc}; +use tuliprox_core::model::{Config, Fingerprint, ProxyUserCredentials}; +use tuliprox_repository::GeoIp; + +pub(in crate::active_user_manager::tests) fn test_channel(virtual_id: u32) -> StreamChannel { + StreamChannel { + target_id: 1, + virtual_id, + provider_id: 1, + input_name: "input".intern(), + item_type: PlaylistItemType::Live, + cluster: XtreamCluster::Live, + group: "group".intern(), + title: "title".intern(), + url: "http://localhost/stream.ts".intern(), + shared: false, + shared_joined_existing: None, + shared_stream_id: None, + technical: None, + epg_channel_id: None, + epg_reference_ts: None, + upstream_user_agent: None, + } +} + +pub(in crate::active_user_manager::tests) fn test_adaptive_channel(virtual_id: u32) -> StreamChannel { + StreamChannel { + target_id: 1, + virtual_id, + provider_id: 1, + input_name: "input".intern(), + item_type: PlaylistItemType::LiveHls, + cluster: XtreamCluster::Live, + group: "group".intern(), + title: "title".intern(), + url: "http://localhost/stream.ts".intern(), + shared: false, + shared_joined_existing: None, + shared_stream_id: None, + technical: None, + epg_channel_id: None, + epg_reference_ts: None, + upstream_user_agent: None, + } +} + +pub(in crate::active_user_manager::tests) fn test_series_channel(virtual_id: u32) -> StreamChannel { + StreamChannel { + item_type: PlaylistItemType::Series, + cluster: XtreamCluster::Series, + url: "http://localhost/series/episode.mkv".intern(), + ..test_channel(virtual_id) + } +} + +pub(in crate::active_user_manager::tests) fn test_user_credentials( + username: &str, + max_connections: u32, + soft_connections: u16, +) -> ProxyUserCredentials { + ProxyUserCredentials { + username: username.to_string(), + password: "test".to_string(), + token: None, + proxy: ProxyType::default(), + server: None, + epg_timeshift: None, + epg_request_timeshift: None, + created_at: None, + exp_date: None, + max_connections, + status: None, + output_clusters: shared::model::ClusterFlags::all(), + ui_enabled: true, + comment: None, + priority: 0, + soft_connections, + soft_priority: 0, + t_is_api_user: false, + network_access: None, + plan: None, + filter: None, + raw_output_clusters: None, + raw_max_connections: 0, + raw_soft_connections: 0, + raw_proxy: Some(ProxyType::default()), + t_filter: None, + t_has_unresolved_plan: false, + t_has_invalid_filter: false, + } +} + +pub(in crate::active_user_manager::tests) fn record_owned_slot( + counts: &mut UserConnectionCounts, + kind: ConnectionKind, +) { + match kind { + ConnectionKind::Normal => counts.normal += 1, + ConnectionKind::Soft => counts.soft += 1, + } +} + +pub(in crate::active_user_manager::tests) fn assert_connection_ownership_invariants( + connection_data: &UserConnectionData, +) { + let mut owned_slots = UserConnectionCounts::default(); + + // A counted session owns one logical slot. Active streams tied to that + // session validate its kind below, but do not add another slot. + for session in connection_data.sessions.iter().filter(|session| session.lifecycle.is_counted()) { + record_owned_slot(&mut owned_slots, session.connection_kind.unwrap_or(ConnectionKind::Normal)); + } + + for stream in &connection_data.streams { + let stream_kind = connection_data.stream_kinds.get(&stream.uid); + if stream.preserved { + assert!(stream_kind.is_none(), "preserved stream {} must not own a real connection slot", stream.uid); + continue; + } + + let stream_kind = stream_kind.expect("every active stream must have a connection kind"); + let counted_session = stream.session_token.as_deref().and_then(|session_token| { + connection_data + .sessions + .iter() + .find(|session| session.token == session_token && session.lifecycle.is_counted()) + }); + if let Some(session) = counted_session { + assert_eq!( + *stream_kind, + session.connection_kind.unwrap_or(ConnectionKind::Normal), + "a counted session and its active stream must use the same slot kind" + ); + } else { + record_owned_slot(&mut owned_slots, *stream_kind); + } + } + + for uid in connection_data.stream_kinds.keys() { + assert!( + connection_data.streams.iter().any(|stream| stream.uid == *uid && !stream.preserved), + "stream kind for uid {uid} must belong to an active stream" + ); + } + + assert_eq!(connection_data.counts.normal, owned_slots.normal, "normal slots must match their owners"); + assert_eq!(connection_data.counts.soft, owned_slots.soft, "soft slots must match their owners"); + assert_eq!( + connection_data.connections, + connection_data.counts.normal + u32::from(connection_data.counts.soft), + "aggregate connection count must equal the normal and soft counters" + ); +} + +pub(in crate::active_user_manager::tests) fn assert_no_real_connection_slots(connection_data: &UserConnectionData) { + assert_eq!(connection_data.connections, 0); + assert_eq!(connection_data.counts.normal, 0); + assert_eq!(connection_data.counts.soft, 0); + assert_connection_ownership_invariants(connection_data); +} + +pub(in crate::active_user_manager::tests) fn assert_preserved_session_is_uncounted( + connection_data: &UserConnectionData, + session_token: &str, + stream_uid: u32, +) { + let session = connection_data + .sessions + .iter() + .find(|session| session.token == session_token) + .expect("preserved session must exist"); + assert_eq!(session.lifecycle, PlaybackLifecycle::Preserved); + + let stream = + connection_data.streams.iter().find(|stream| stream.uid == stream_uid).expect("preserved stream must exist"); + assert!(stream.preserved); + assert_eq!(stream.session_token.as_deref(), Some(session_token)); + assert!(!connection_data.stream_kinds.contains_key(&stream_uid)); +} + +pub(in crate::active_user_manager::tests) async fn commit_and_preserve_adaptive_session( + manager: &ActiveUserManager, + user: &ProxyUserCredentials, + session_token: &str, + stream_uid: u32, + addr: SocketAddr, + connection_kind: ConnectionKind, +) { + let fingerprint = Fingerprint::new(format!("fp-preserved-{stream_uid}"), addr.ip().to_string(), addr); + + manager.add_connection(&addr).await; + manager + .create_user_session(CreateUserSessionParams { + user, + session_token, + virtual_id: stream_uid, + provider: "provider-a", + stream_url: "http://localhost/live-preserved.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(connection_kind), + socket_bound: false, + }) + .await; + manager + .update_connection(ActiveUserConnectionParams { + uid: stream_uid, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind, + priority: user.priority, + soft_priority: user.soft_priority, + fingerprint: &fingerprint, + provider: "provider-a".intern(), + stream_channel: &test_adaptive_channel(stream_uid), + user_agent: Cow::Borrowed("ua"), + session_token: Some(session_token), + }) + .await + .expect("adaptive session stream should bind"); + + { + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + let session = connection_data + .sessions + .iter() + .find(|session| session.token == session_token) + .expect("committed session must exist"); + assert_eq!(session.lifecycle, PlaybackLifecycle::Active); + assert_eq!(connection_data.stream_kinds.get(&stream_uid), Some(&connection_kind)); + assert_connection_ownership_invariants(connection_data); + } + + assert!(manager.release_stream(&addr).await.is_none(), "adaptive stream should be preserved"); + + let connections = manager.connections.read().await; + let connection_data = connections.by_key.get(&user.username).expect("user connection data"); + assert_preserved_session_is_uncounted(connection_data, session_token, stream_uid); + assert_connection_ownership_invariants(connection_data); +} + +pub(in crate::active_user_manager::tests) fn provider_header_test_manager() -> ActiveUserManager { + let config = Config::default(); + let geoip = Arc::new(ArcSwapOption::::default()); + let event_manager = Arc::new(EventManager::new()); + ActiveUserManager::new(&config, &geoip, &event_manager) +} + +pub(in crate::active_user_manager::tests) async fn create_provider_header_session( + manager: &ActiveUserManager, + user: &ProxyUserCredentials, + url: &str, +) { + let addr: SocketAddr = "127.0.0.1:55420".parse().unwrap_or_else(|_| unreachable!()); + manager + .create_user_session(CreateUserSessionParams { + user, + session_token: "tok-host", + virtual_id: 7010, + provider: "provider-a", + stream_url: url, + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(ConnectionKind::Normal), + socket_bound: false, + }) + .await; +} + +pub(in crate::active_user_manager::tests) async fn session_identity( + manager: &ActiveUserManager, + username: &str, + token: &str, +) -> Option { + let users = manager.connections.read().await; + users.by_key.get(username)?.sessions.iter().find(|session| session.token == token).map(UserSession::identity) +} diff --git a/backend/session/src/connection_manager.rs b/backend/session/src/connection_manager.rs index e46c512a3..061dc69b7 100644 --- a/backend/session/src/connection_manager.rs +++ b/backend/session/src/connection_manager.rs @@ -1,58 +1,48 @@ -use crate::{ - uses_direct_body_idle_timeout, ActiveProviderManager, ActiveUserConnectionParams, ActiveUserManager, EventManager, - ManagedProviderHandle, SharedStreamManager, -}; +use crate::{ActiveProviderManager, ActiveUserManager, EventManager, SharedStreamManager}; use arc_swap::ArcSwapOption; -use futures::future::BoxFuture; -use log::{debug, warn}; -use shared::{ - model::{ - ActiveUserConnectionChange, ConnectFailureReason, CustomVideoStreamType, DisconnectReason, EventMessage, - FailureStage, StreamChannel, StreamInfo, VirtualId, - }, - utils::sanitize_sensitive_info, -}; +use shared::model::StreamInfo; use std::{ - borrow::Cow, - cmp::Reverse, - collections::{BinaryHeap, HashMap, HashSet, VecDeque}, + collections::{HashMap, HashSet, VecDeque}, net::SocketAddr, - str::FromStr, sync::{ - atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, - Arc, Mutex, MutexGuard, + atomic::{AtomicBool, AtomicU32, AtomicU64}, + Arc, Mutex, }, - thread, time::Duration, }; -use tokio::sync::{mpsc, Notify, RwLock, RwLockReadGuard}; +use tokio::sync::{mpsc, Notify, RwLock}; use tokio_util::sync::CancellationToken; -use tuliprox_core::{ - model::{ - DisconnectQos, Fingerprint, PlaybackRequestOutcome, ProviderHandle, SharedSubscriberId, StreamHistoryConfig, - StreamHistoryRecord, - }, - utils::debug_if_enabled, -}; -use tuliprox_repository::{recover_pending_files, StreamHistoryWriter}; +use tuliprox_core::model::SharedSubscriberId; +use tuliprox_repository::StreamHistoryWriter; // Maximum number of deferred cleanup actions buffered before producers must wait/drop. const CLEANUP_QUEUE_CAPACITY: usize = 4096; + // Reserved lane for control work (HLS origin I/O, shutdown) so a saturated body-cleanup // queue cannot starve the control path that releases provider handles. const CONTROL_CLEANUP_CAPACITY: usize = 64; + pub const PROVIDER_END_NOT_SET: u8 = 0; -pub const PROVIDER_END_CLOSED: u8 = 1; // Provider EOF -pub const PROVIDER_END_ERROR: u8 = 2; // Provider Err -pub const PROVIDER_END_PREEMPTED: u8 = 3; // Preempted by higher priority + +pub const PROVIDER_END_CLOSED: u8 = 1; + +// Provider EOF +pub const PROVIDER_END_ERROR: u8 = 2; + +// Provider Err +pub const PROVIDER_END_PREEMPTED: u8 = 3; + +// Preempted by higher priority const PREEMPT_REENTRY_BLOCK_SECS: u64 = 3; + // Bounded wait for a mandatory cleanup admission right before a request registration. const CLEANUP_ADMISSION_TIMEOUT: Duration = Duration::from_secs(5); + // Rebuild the expiry heap when it grows beyond this multiple of the live index size. const SOCKET_EXPIRY_QUEUE_REBUILD_FACTOR: usize = 2; + // Avoid rebuilding the expiry heap unless it contains at least this many stale entries. const SOCKET_EXPIRY_QUEUE_REBUILD_MIN_STALE: usize = 256; -fn notify_capacity(capacity_notify: &Notify) { capacity_notify.notify_waiters(); } /// Proof that a shared-stream response owns the guaranteed terminal cleanup permit for /// its subscriber. The constructor is crate-private, so callers outside this crate cannot @@ -64,14 +54,6 @@ pub struct SharedCleanupCapability { subscriber_id: SharedSubscriberId, } -impl SharedCleanupCapability { - pub(crate) fn new(subscriber_id: SharedSubscriberId) -> Self { Self { subscriber_id } } - - pub fn stream_uid(&self) -> u32 { self.subscriber_id.stream_uid() } - - pub fn subscriber_id(&self) -> SharedSubscriberId { self.subscriber_id } -} - struct BackpressureState { overflow: VecDeque, draining: bool, @@ -85,162 +67,12 @@ struct BackpressureSender { dropped_events: AtomicU64, } -impl BackpressureSender -where - T: Send + 'static, -{ - fn new(tx: mpsc::Sender, queue_name: &'static str, overflow_capacity: usize) -> Self { - Self { - tx, - state: Arc::new(Mutex::new(BackpressureState { overflow: VecDeque::new(), draining: false })), - queue_name, - overflow_capacity, - dropped_events: AtomicU64::new(0), - } - } - - fn enqueue(&self, event: T) { - let runtime = tokio::runtime::Handle::try_current().ok(); - { - let mut state = lock_backpressure_state(self.state.as_ref()); - if state.draining { - if state.overflow.len() >= self.overflow_capacity { - let count = self.dropped_events.fetch_add(1, Ordering::Relaxed) + 1; - if count == 1 || count.is_multiple_of(1024) { - warn!( - "{} overflow buffer full (capacity={}), {} events dropped total", - self.queue_name, self.overflow_capacity, count - ); - } - return; - } - state.overflow.push_back(event); - return; - } - - match self.tx.try_send(event) { - Ok(()) => return, - Err(tokio::sync::mpsc::error::TrySendError::Full(event)) => { - state.draining = true; - if state.overflow.len() >= self.overflow_capacity { - let count = self.dropped_events.fetch_add(1, Ordering::Relaxed) + 1; - if count == 1 || count.is_multiple_of(1024) { - warn!( - "{} overflow buffer full (capacity={}), {} events dropped total", - self.queue_name, self.overflow_capacity, count - ); - } - state.draining = false; - return; - } - state.overflow.push_back(event); - } - Err(tokio::sync::mpsc::error::TrySendError::Closed(_event)) => { - debug!("{} channel closed, dropping event", self.queue_name); - return; - } - } - } - - let tx = self.tx.clone(); - let state = Arc::clone(&self.state); - let queue_name = self.queue_name; - if let Some(handle) = runtime { - handle.spawn(async move { - Self::drain_async(&tx, &state, queue_name).await; - }); - } else { - thread::spawn(move || Self::drain_blocking(&tx, &state, queue_name)); - } - } - - async fn drain_async(tx: &mpsc::Sender, state: &Arc>>, queue_name: &'static str) { - loop { - let Some(event) = Self::next_event(state) else { - break; - }; - if tx.send(event).await.is_err() { - debug!("{queue_name} channel closed while draining backpressure"); - Self::clear_and_stop(state); - break; - } - } - } - - fn drain_blocking(tx: &mpsc::Sender, state: &Arc>>, queue_name: &'static str) { - loop { - let Some(event) = Self::next_event(state) else { - break; - }; - if tx.blocking_send(event).is_err() { - warn!("{queue_name} channel closed while draining backpressure"); - Self::clear_and_stop(state); - break; - } - } - } - - fn next_event(state: &Arc>>) -> Option { - let mut state = lock_backpressure_state(state.as_ref()); - if let Some(event) = state.overflow.pop_front() { - return Some(event); - } - state.draining = false; - None - } - - fn clear_and_stop(state: &Arc>>) { - let mut state = lock_backpressure_state(state.as_ref()); - state.overflow.clear(); - state.draining = false; - } - - fn dropped_count(&self) -> u64 { self.dropped_events.load(Ordering::Relaxed) } -} - -fn lock_backpressure_state(state: &Mutex>) -> MutexGuard<'_, BackpressureState> { - match state.lock() { - Ok(guard) => guard, - Err(poisoned) => { - warn!("Backpressure queue state was poisoned, continuing with recovered state"); - poisoned.into_inner() - } - } -} - #[derive(Clone)] struct SocketActivityTracker { pending: Arc>>, notify: Arc, } -impl SocketActivityTracker { - fn new() -> Self { Self { pending: Arc::new(Mutex::new(HashSet::new())), notify: Arc::new(Notify::new()) } } - - fn track(&self, event: SocketActivityEvent) { - let mut pending = lock_socket_activity_pending(self.pending.as_ref()); - pending.insert(event); - drop(pending); - self.notify.notify_one(); - } - - fn drain(&self) -> Vec { - let mut pending = lock_socket_activity_pending(self.pending.as_ref()); - pending.drain().collect() - } - - async fn notified(&self) { self.notify.notified().await; } -} - -fn lock_socket_activity_pending( - pending: &Mutex>, -) -> MutexGuard<'_, HashSet> { - pending.lock().unwrap_or_else(|poisoned| { - warn!("Socket activity state was poisoned, continuing with recovered state"); - poisoned.into_inner() - }) -} - struct CleanupWorkerDeps { user_manager: Arc, provider_manager: Arc, @@ -250,368 +82,12 @@ struct CleanupWorkerDeps { history_writer: Arc>, } -pub enum CleanupEvent { - ReleaseSharedSubscriber { - addr: SocketAddr, - subscriber_id: SharedSubscriberId, - request_id: Option, - owner: Option>, - }, - ReleaseStream { - request_id: Option, - /// Playback owner (session token) captured at acquire, so the provider request - /// can be finished independently of whether the user claim still exists. - owner: Option>, - addr: SocketAddr, - stream_uid: Option, - provider_end_reason: u8, - reconnect_count: u8, - provider_error_class: Option<&'static str>, - provider_http_status: Option, - }, - ReleaseConnection { - addr: SocketAddr, - }, - ReleaseProviderHandle { - handle: Option, - }, - ReleaseStreamAndProviderHandle { - request_id: Option, - addr: SocketAddr, - stream_uid: Option, - handle: Option, - provider_end_reason: u8, - reconnect_count: u8, - provider_error_class: Option<&'static str>, - provider_http_status: Option, - }, - UpdateDetailAndReleaseProvider { - addr: SocketAddr, - stream_uid: Option, - video_type: CustomVideoStreamType, - handle: Option, - }, - AdaptiveSessionExpired { - stream_info: Box, - }, - /// Confirms that real provider media reached the client for a playback lease. - /// Only a confirmed lease may reserve provider capacity against other playbacks. - ConfirmPlaybackLease { - owner: Arc, - request_id: Option, - }, - /// Runs a deferred asynchronous cleanup task in the managed cleanup worker. - Defer(BoxFuture<'static, ()>), -} - -async fn handle_release_connection(deps: &CleanupWorkerDeps, addr: SocketAddr) { - release_connection_parts(deps, &addr, DisconnectReason::Cleanup, true).await; -} - -async fn release_connection_with_reason( - connection_manager: &ConnectionManager, - addr: &SocketAddr, - reason: DisconnectReason, - send_shared_stop_signal: bool, -) { - let deps = CleanupWorkerDeps { - user_manager: Arc::clone(&connection_manager.user_manager), - provider_manager: Arc::clone(&connection_manager.provider_manager), - shared_stream_manager: Arc::clone(&connection_manager.shared_stream_manager), - event_manager: Arc::clone(&connection_manager.event_manager), - capacity_notify: Arc::clone(&connection_manager.capacity_notify), - history_writer: Arc::clone(&connection_manager.history_writer), - }; - release_connection_parts(&deps, addr, reason, send_shared_stop_signal).await; -} - -async fn release_connection_parts( - deps: &CleanupWorkerDeps, - addr: &SocketAddr, - reason: DisconnectReason, - send_shared_stop_signal: bool, -) { - let removed = if matches!(reason, DisconnectReason::ClientKicked) { - deps.user_manager.release_connection_as_kicked(addr).await - } else { - deps.user_manager.release_connection(addr).await - }; - if matches!(reason, DisconnectReason::ClientKicked) { - for stream_info in &removed.removed_streams { - if let Some(session_token) = stream_info.session_token.as_deref() { - deps.provider_manager.terminate_identified_playback_owner(session_token); - } - } - // Explicitly terminate all sessions for the kicked addr. This expires them - // immediately rather than leaving them for TTL-based GC cleanup. - for username in &removed.disconnected_users { - deps.user_manager.terminate_sessions_for_addr(username, addr).await; - } - } - for stream_info in &removed.removed_streams { - let qos = deps.event_manager.read_meter_qos(stream_info.meter_uid).await; - let bytes_sent = qos.map(|qos| qos.bytes_total); - let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); - deps.event_manager.unregister_meter_client(stream_info.uid).await; - emit_disconnect_record( - &deps.history_writer, - stream_info, - reason, - &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, - None, - None, - ); - } - deps.provider_manager.release_connection(addr); - deps.shared_stream_manager.release_connection(addr, send_shared_stop_signal).await; - if removed.addr_removed && !removed.removed_streams.is_empty() { - deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Disconnected(*addr))); - } - notify_capacity(deps.capacity_notify.as_ref()); -} - -#[allow(clippy::too_many_arguments)] -async fn handle_release_stream( - deps: &CleanupWorkerDeps, - addr: SocketAddr, - stream_uid: Option, - provider_end_reason: u8, - reconnect_count: u8, - provider_error_class: Option<&'static str>, - provider_http_status: Option, - request_id: Option, - owner: Option>, -) { - if let Some(stream_info) = release_stream_with_disconnect( - deps, - addr, - stream_uid, - provider_end_reason, - reconnect_count, - provider_error_class, - provider_http_status, - request_id, - owner, - ) - .await - { - deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::DisconnectedStream { - addr: stream_info.addr, - uid: stream_info.uid, - })); - notify_capacity(deps.capacity_notify.as_ref()); - } -} - -fn handle_release_provider_handle(deps: &CleanupWorkerDeps, handle: Option) { - if let Some(handle) = handle { - deps.provider_manager.release_handle(&handle); - notify_capacity(deps.capacity_notify.as_ref()); - } -} - -#[allow(clippy::too_many_arguments)] -async fn handle_release_stream_and_provider_handle( - deps: &CleanupWorkerDeps, - addr: SocketAddr, - stream_uid: Option, - handle: Option, - provider_end_reason: u8, - reconnect_count: u8, - provider_error_class: Option<&'static str>, - provider_http_status: Option, - request_id: Option, -) { - let provider_released = if let Some(handle) = handle { - deps.provider_manager.release_handle(&handle); - true - } else { - false - }; - let stream_released = release_stream_with_disconnect( - deps, - addr, - stream_uid, - provider_end_reason, - reconnect_count, - provider_error_class, - provider_http_status, - request_id, - None, - ) - .await; - if let Some(stream_info) = stream_released.as_ref() { - deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::DisconnectedStream { - addr: stream_info.addr, - uid: stream_info.uid, - })); - } - if provider_released || stream_released.is_some() { - notify_capacity(deps.capacity_notify.as_ref()); - } -} - -async fn handle_update_detail_and_release_provider( - deps: &CleanupWorkerDeps, - addr: SocketAddr, - video_type: CustomVideoStreamType, - handle: Option, - stream_uid: Option, -) { - let stream_info = if let Some(uid) = stream_uid { - deps.user_manager.update_stream_detail_by_uid(uid, video_type).await - } else { - deps.user_manager.update_stream_detail(&addr, video_type).await - }; - if let Some(stream_info) = stream_info { - if matches!(video_type, CustomVideoStreamType::LowPriorityPreempted) { - deps.user_manager - .block_user_for_stream_uid( - stream_info.uid, - shared::model::VirtualId::new(stream_info.channel.virtual_id), - PREEMPT_REENTRY_BLOCK_SECS, - ) - .await; - } - deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); - } - if let Some(handle) = handle { - deps.provider_manager.release_handle(&handle); - notify_capacity(deps.capacity_notify.as_ref()); - } -} - -async fn handle_adaptive_session_expired(deps: &CleanupWorkerDeps, stream_info: Box) { - let qos = deps.event_manager.read_meter_qos(stream_info.meter_uid).await; - let bytes_sent = qos.map(|qos| qos.bytes_total); - let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); - deps.event_manager.unregister_meter_client(stream_info.uid).await; - emit_disconnect_record( - &deps.history_writer, - &stream_info, - DisconnectReason::SessionExpired, - &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, - None, - None, - ); - deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::DisconnectedStream { - addr: stream_info.addr, - uid: stream_info.uid, - })); - notify_capacity(deps.capacity_notify.as_ref()); -} - -#[allow(clippy::too_many_arguments)] -async fn release_stream_with_disconnect( - deps: &CleanupWorkerDeps, - addr: SocketAddr, - stream_uid: Option, - provider_end_reason: u8, - reconnect_count: u8, - provider_error_class: Option<&'static str>, - provider_http_status: Option, - request_id: Option, - owner: Option>, -) -> Option { - let detach = if let Some(stream_uid) = stream_uid { - deps.user_manager.release_stream_request_by_uid(&addr, stream_uid).await - } else { - crate::StreamRequestDetach::NotFound - }; - match detach { - crate::StreamRequestDetach::NotFound => { - debug_if_enabled!( - "Stream release skipped: no active stream for {} uid={:?}", - sanitize_sensitive_info(&addr.to_string()), - stream_uid - ); - // The provider request must still be finished even when the user claim is - // already gone: the identity travels with the cleanup event. - if let (Some(owner), Some(request_id)) = (owner.as_deref(), request_id) { - let reason = resolve_disconnect_reason_from_provider_end(provider_end_reason); - deps.provider_manager.finish_identified_playback_request( - owner, - request_id, - playback_outcome_for_reason(reason), - ); - notify_capacity(deps.capacity_notify.as_ref()); - } - None - } - crate::StreamRequestDetach::Retained(stream_info) | crate::StreamRequestDetach::Preserved(stream_info) => { - let reason = resolve_disconnect_reason(provider_end_reason, &stream_info); - if let (Some(session_token), Some(request_id)) = - (owner.as_deref().or(stream_info.session_token.as_deref()), request_id) - { - deps.provider_manager.finish_identified_playback_request( - session_token, - request_id, - playback_outcome_for_reason(reason), - ); - } - notify_capacity(deps.capacity_notify.as_ref()); - None - } - crate::StreamRequestDetach::Removed(stream_info) => { - let qos = deps.event_manager.read_meter_qos(stream_info.meter_uid).await; - let bytes_sent = qos.map(|qos| qos.bytes_total); - let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); - deps.event_manager.unregister_meter_client(stream_info.uid).await; - let reason = resolve_disconnect_reason(provider_end_reason, &stream_info); - if let (Some(session_token), Some(request_id)) = - (owner.as_deref().or(stream_info.session_token.as_deref()), request_id) - { - deps.provider_manager.finish_identified_playback_request( - session_token, - request_id, - playback_outcome_for_reason(reason), - ); - } - let provider_reconnect_count = (reconnect_count > 0).then_some(reconnect_count); - emit_disconnect_record( - &deps.history_writer, - &stream_info, - reason, - &DisconnectQos { bytes_sent, first_byte_latency_ms, provider_reconnect_count }, - provider_error_class, - provider_http_status, - ); - Some(stream_info) - } - } -} - #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] struct SocketExpiryEntry { expires_at: u64, addr: SocketAddr, } -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -enum SocketActivityEvent { - HttpActivity { addr: SocketAddr }, - DirectBodyActivity { addr: SocketAddr }, -} - -impl SocketActivityEvent { - fn addr(&self) -> SocketAddr { - match self { - Self::HttpActivity { addr, .. } | Self::DirectBodyActivity { addr, .. } => *addr, - } - } -} - -#[derive(Clone, Debug, PartialEq)] -pub enum CloseConnectionSignal { - WithReason(SocketAddr, DisconnectReason), -} - -#[derive(Debug)] -enum SocketCloseState { - Open(tokio::sync::oneshot::Sender), - Closing, -} - pub struct ConnectionManager { pub user_manager: Arc, pub provider_manager: Arc, @@ -631,27 +107,6 @@ pub struct ConnectionManager { worker_handles: std::sync::Mutex>>, } -pub struct ConnectionParams<'a> { - pub meter_uid: u32, - pub username: &'a str, - pub max_connections: u32, - pub soft_connections: u16, - pub connection_kind: crate::active_provider_manager::ConnectionKind, - pub priority: i8, - pub soft_priority: i8, - pub fingerprint: &'a Fingerprint, - pub provider: Arc, - pub stream_channel: &'a StreamChannel, - pub user_agent: Cow<'a, str>, - pub session_token: Option<&'a str>, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ConnectionHistoryMode { - EmitConnect, - RefreshOnly, -} - /// Guaranteed final cleanup for a registered request, transferred into the body. /// /// The permit was reserved before the registration mutation, so the release is @@ -671,74 +126,6 @@ pub struct OwnedRequestCleanup { finished: bool, } -impl OwnedRequestCleanup { - /// Releases the reserved cleanup right without emitting any release. - fn disarm(&mut self) { - self.finished = true; - self.permit = None; - } - - /// Releases the request claim exactly once with the actual provider outcome. - #[allow(clippy::too_many_arguments)] - pub fn finish( - &mut self, - request_id: Option, - provider_end_reason: u8, - reconnect_count: u8, - provider_error_class: Option<&'static str>, - provider_http_status: Option, - ) { - if self.finished { - return; - } - self.finished = true; - let Some(permit) = self.permit.take() else { - return; - }; - permit.send(CleanupEvent::ReleaseStream { - request_id, - owner: self.owner.clone(), - addr: self.addr, - stream_uid: Some(self.request_uid), - provider_end_reason, - reconnect_count, - provider_error_class, - provider_http_status, - }); - } -} - -impl Drop for OwnedRequestCleanup { - fn drop(&mut self) { - // Conservative fallback for a body that is dropped without an explicit finish: - // never invent a successful EOF, but still finish the provider request with its - // captured identity so an abandoned start does not linger until the startup TTL. - self.finish(self.provider_request_id, PROVIDER_END_NOT_SET, 0, None, None); - } -} - -/// Why a playback request was rejected before a body could be created. Carried on -/// `RegisteredPlaybackRequest` so the caller can surface a non-success HTTP response -/// instead of silently dropping the request. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ConnectionRejectionReason { - CleanupReceiverClosed, - CleanupAdmissionTimeout, - RegistrationFailed, -} - -impl std::fmt::Display for ConnectionRejectionReason { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::CleanupReceiverClosed => write!(f, "cleanup receiver closed"), - Self::CleanupAdmissionTimeout => write!(f, "cleanup admission timed out"), - Self::RegistrationFailed => write!(f, "connection registration failed"), - } - } -} - -impl std::error::Error for ConnectionRejectionReason {} - pub struct RegisteredPlaybackRequest { pub request_uid: u32, pub display_stream: Option, @@ -746,2271 +133,23 @@ pub struct RegisteredPlaybackRequest { rejection: Option, } -impl std::fmt::Debug for RegisteredPlaybackRequest { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RegisteredPlaybackRequest") - .field("request_uid", &self.request_uid) - .field("display_stream", &self.display_stream) - .field("rejection", &self.rejection) - .finish_non_exhaustive() - } -} - -impl RegisteredPlaybackRequest { - #[inline] - pub fn new(request_uid: u32, display_stream: Option) -> Self { - Self { request_uid, display_stream, cleanup: None, rejection: None } - } - - #[inline] - fn rejected(request_uid: u32, reason: ConnectionRejectionReason) -> Self { - Self { request_uid, display_stream: None, cleanup: None, rejection: Some(reason) } - } - - #[inline] - fn with_cleanup( - request_uid: u32, - display_stream: Option, - cleanup: Option, - ) -> Self { - Self { request_uid, display_stream, cleanup, rejection: None } - } - - /// Marks the request as taken over by a path without a body (compatibility API or - /// shared source), disabling rollback on drop. - #[inline] - pub fn commit(&mut self) { - if let Some(mut cleanup) = self.cleanup.take() { - cleanup.disarm(); - } - } - - /// Transfers the guaranteed cleanup right into the body. The body owns the returned - /// value and must finish it exactly once with the real provider outcome. - #[inline] - pub fn into_body_cleanup(&mut self) -> Option { self.cleanup.take() } - - #[inline] - pub fn display_uid(&self) -> Option { self.display_stream.as_ref().map(|s| s.uid) } - - #[inline] - pub fn rejection_reason(&self) -> Option { self.rejection } - - #[inline] - pub fn into_display_stream(self) -> Option { self.display_stream } -} - -impl ConnectionManager { - pub fn new( - user_manager: &Arc, - provider_manager: &Arc, - shared_stream_manager: &Arc, - event_manager: &Arc, - history_config: Option<&StreamHistoryConfig>, - ) -> Self { - Self::new_with_capacity( - user_manager, - provider_manager, - shared_stream_manager, - event_manager, - history_config, - CLEANUP_QUEUE_CAPACITY, - ) - } - - /// Same as [`ConnectionManager::new`], but with an explicit cleanup-queue capacity - /// derived from configuration instead of the hard-coded default. - pub fn new_with_capacity( - user_manager: &Arc, - provider_manager: &Arc, - shared_stream_manager: &Arc, - event_manager: &Arc, - history_config: Option<&StreamHistoryConfig>, - cleanup_capacity: usize, - ) -> Self { - let cleanup_capacity = cleanup_capacity.max(1); - let history_writer = Arc::new(ArcSwapOption::new(build_history_writer(history_config))); - let (close_socket_signal_tx, _) = tokio::sync::broadcast::channel(256); - let (cleanup_tx, cleanup_rx) = mpsc::channel(cleanup_capacity); - let (control_cleanup_tx, control_cleanup_rx) = mpsc::channel(CONTROL_CLEANUP_CAPACITY); - user_manager.set_cleanup_sender(cleanup_tx.clone()); - user_manager.set_provider_manager(Arc::clone(provider_manager)); - let socket_cleanup_tx = cleanup_tx.clone(); - let socket_activity_tracker = SocketActivityTracker::new(); - let capacity_notify = Arc::new(Notify::new()); - let shutdown_token = CancellationToken::new(); - let mgr = Self { - user_manager: Arc::clone(user_manager), - provider_manager: Arc::clone(provider_manager), - shared_stream_manager: Arc::clone(shared_stream_manager), - event_manager: Arc::clone(event_manager), - close_socket_signal_tx, - socket_closers: std::sync::Mutex::new(HashMap::new()), - cleanup_sender: BackpressureSender::new(cleanup_tx, "cleanup", cleanup_capacity), - control_cleanup_tx: control_cleanup_tx.clone(), - socket_activity_tracker: socket_activity_tracker.clone(), - capacity_notify: Arc::clone(&capacity_notify), - stream_uid_counter: AtomicU32::new(1), - history_writer: Arc::clone(&history_writer), - is_shutting_down: AtomicBool::new(false), - admission_gate: RwLock::new(()), - shutdown_token: shutdown_token.clone(), - worker_handles: std::sync::Mutex::new(Vec::new()), - }; - - let cleanup_handle = Self::spawn_cleanup_worker( - cleanup_rx, - control_cleanup_rx, - Arc::clone(user_manager), - Arc::clone(provider_manager), - Arc::clone(shared_stream_manager), - Arc::clone(event_manager), - Arc::clone(&capacity_notify), - history_writer, - shutdown_token.clone(), - ); - let socket_handle = Self::spawn_socket_activity_worker( - socket_activity_tracker, - Arc::clone(user_manager), - socket_cleanup_tx, - shutdown_token, - ); - mgr.worker_handles - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .extend([cleanup_handle, socket_handle]); - - mgr - } - - /// Reload the history writer on config change. Shuts down the old writer first so - /// `recover_pending_files` does not collide with an active writer. Recovery runs on the - /// blocking pool instead of a runtime worker so large pending-file sets cannot stall - /// live stream heartbeats. - pub async fn reload_history_writer(&self, config: Option<&StreamHistoryConfig>) { - let old_writer = self.history_writer.swap(None); - if let Some(w) = old_writer { - w.shutdown().await; - } - let new_writer = build_history_writer_async(config).await; - self.history_writer.store(new_writer); - } - - /// Returns a reference to the history writer. - pub fn history_writer(&self) -> &Arc> { &self.history_writer } - - fn spawn_socket_activity_worker( - activity_tracker: SocketActivityTracker, - user_manager: Arc, - cleanup_tx: mpsc::Sender, - shutdown_token: CancellationToken, - ) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - let mut expiry_queue: BinaryHeap> = BinaryHeap::new(); - let mut expiry_index: HashMap = HashMap::new(); - - loop { - Self::drain_pending_socket_activity( - &activity_tracker, - &mut expiry_queue, - &mut expiry_index, - &user_manager, - ) - .await; - - let next_expiry = expiry_queue.peek().map(|entry| entry.0.expires_at); - if let Some(expires_at) = next_expiry { - let now = shared::utils::current_time_secs(); - if expires_at <= now { - Self::process_due_socket_expiry_entries( - &mut expiry_queue, - &mut expiry_index, - now, - &user_manager, - &cleanup_tx, - ) - .await; - continue; - } - - tokio::select! { - biased; - () = shutdown_token.cancelled() => break, - () = activity_tracker.notified() => {} - () = tokio::time::sleep(Duration::from_secs(expires_at.saturating_sub(now))) => {} - } - } else { - tokio::select! { - () = shutdown_token.cancelled() => break, - () = activity_tracker.notified() => {} - } - } - } - }) - } - - async fn drain_pending_socket_activity( - activity_tracker: &SocketActivityTracker, - expiry_queue: &mut BinaryHeap>, - expiry_index: &mut HashMap, - user_manager: &Arc, - ) { - for event in activity_tracker.drain() { - Self::handle_socket_activity_event(event, expiry_queue, expiry_index, user_manager).await; - } - } - - async fn handle_socket_activity_event( - event: SocketActivityEvent, - expiry_queue: &mut BinaryHeap>, - expiry_index: &mut HashMap, - user_manager: &Arc, - ) { - let addr = event.addr(); - if matches!(event, SocketActivityEvent::DirectBodyActivity { .. }) { - user_manager.touch_socket_activity(&addr).await; - } - - if let Some(expires_at) = user_manager.socket_expiry_deadline(&addr).await { - let current = expiry_index.insert(addr, expires_at); - if current != Some(expires_at) { - expiry_queue.push(Reverse(SocketExpiryEntry { expires_at, addr })); - Self::maybe_rebuild_socket_expiry_queue(expiry_queue, expiry_index); - } - } - } - - async fn process_due_socket_expiry_entries( - expiry_queue: &mut BinaryHeap>, - expiry_index: &mut HashMap, - now: u64, - user_manager: &Arc, - cleanup_tx: &mpsc::Sender, - ) { - while let Some(entry) = expiry_queue.peek().copied() { - if entry.0.expires_at > now { - break; - } - - let Reverse(SocketExpiryEntry { expires_at, addr }) = expiry_queue.pop().unwrap_or(entry); - let Some(current_expires_at) = expiry_index.get(&addr).copied() else { - continue; - }; - if current_expires_at != expires_at { - continue; - } - - if let Some(next_expires_at) = user_manager.socket_expiry_deadline(&addr).await { - if next_expires_at > now { - expiry_index.insert(addr, next_expires_at); - expiry_queue.push(Reverse(SocketExpiryEntry { expires_at: next_expires_at, addr })); - Self::maybe_rebuild_socket_expiry_queue(expiry_queue, expiry_index); - continue; - } - } - - // Fallthrough to release connection if `None` or `< now` - - expiry_index.remove(&addr); - debug_if_enabled!( - "Socket activity deadline expired for {}, releasing connection", - sanitize_sensitive_info(&addr.to_string()) - ); - if cleanup_tx.send(CleanupEvent::ReleaseConnection { addr }).await.is_err() { - debug!("Cleanup channel closed, stopping socket expiry worker"); - break; - } - } - } - - fn maybe_rebuild_socket_expiry_queue( - expiry_queue: &mut BinaryHeap>, - expiry_index: &HashMap, - ) { - let indexed_len = expiry_index.len(); - if indexed_len == 0 { - expiry_queue.clear(); - return; - } - - let stale_entries = expiry_queue.len().saturating_sub(indexed_len); - if expiry_queue.len() <= indexed_len.saturating_mul(SOCKET_EXPIRY_QUEUE_REBUILD_FACTOR) - || stale_entries < SOCKET_EXPIRY_QUEUE_REBUILD_MIN_STALE - { - return; - } - - *expiry_queue = expiry_index - .iter() - .map(|(addr, expires_at)| Reverse(SocketExpiryEntry { expires_at: *expires_at, addr: *addr })) - .collect(); - } - - #[allow(clippy::too_many_arguments, clippy::too_many_lines)] - fn spawn_cleanup_worker( - mut rx: mpsc::Receiver, - mut control_rx: mpsc::Receiver, - user_manager: Arc, - provider_manager: Arc, - shared_stream_manager: Arc, - event_manager: Arc, - capacity_notify: Arc, - history_writer: Arc>, - shutdown_token: CancellationToken, - ) -> tokio::task::JoinHandle<()> { - let deps = CleanupWorkerDeps { - user_manager, - provider_manager, - shared_stream_manager, - event_manager, - capacity_notify, - history_writer, - }; - tokio::spawn(async move { - loop { - let event = tokio::select! { - biased; - () = shutdown_token.cancelled() => break, - control_event = control_rx.recv() => control_event, - event = rx.recv() => event, - }; - let Some(event) = event else { break }; - match event { - CleanupEvent::ReleaseSharedSubscriber { addr, subscriber_id, request_id, owner } => { - deps.shared_stream_manager.release_subscriber(subscriber_id).await; - handle_release_stream( - &deps, - addr, - Some(subscriber_id.stream_uid()), - PROVIDER_END_NOT_SET, - 0, - None, - None, - request_id, - owner, - ) - .await; - notify_capacity(deps.capacity_notify.as_ref()); - } - CleanupEvent::ReleaseConnection { addr } => { - handle_release_connection(&deps, addr).await; - } - CleanupEvent::ReleaseStream { - request_id, - owner, - addr, - stream_uid, - provider_end_reason, - reconnect_count, - provider_error_class, - provider_http_status, - } => { - handle_release_stream( - &deps, - addr, - stream_uid, - provider_end_reason, - reconnect_count, - provider_error_class, - provider_http_status, - request_id, - owner, - ) - .await; - } - CleanupEvent::ReleaseProviderHandle { handle } => { - handle_release_provider_handle(&deps, handle); - } - CleanupEvent::ReleaseStreamAndProviderHandle { - request_id, - addr, - stream_uid, - handle, - provider_end_reason, - reconnect_count, - provider_error_class, - provider_http_status, - } => { - handle_release_stream_and_provider_handle( - &deps, - addr, - stream_uid, - handle, - provider_end_reason, - reconnect_count, - provider_error_class, - provider_http_status, - request_id, - ) - .await; - } - CleanupEvent::UpdateDetailAndReleaseProvider { addr, stream_uid, video_type, handle } => { - handle_update_detail_and_release_provider(&deps, addr, video_type, handle, stream_uid).await; - } - CleanupEvent::AdaptiveSessionExpired { stream_info } => { - handle_adaptive_session_expired(&deps, stream_info).await; - } - CleanupEvent::ConfirmPlaybackLease { owner, request_id } => { - if let Some(request_id) = request_id { - deps.provider_manager.confirm_identified_playback_activity(&owner, request_id); - } else { - deps.provider_manager.confirm_playback_activity(&owner); - } - } - CleanupEvent::Defer(future) => { - future.await; - } - } - } - debug!("Cleanup worker exiting"); - }) - } - - pub fn send_cleanup(&self, event: CleanupEvent) { self.cleanup_sender.enqueue(event); } - - pub fn cleanup_tx(&self) -> mpsc::Sender { self.cleanup_sender.tx.clone() } - - /// Reserved control lane for cleanup permits that must never be starved by a - /// saturated body-cleanup queue (HLS origin I/O, shutdown). - pub fn control_cleanup_tx(&self) -> mpsc::Sender { self.control_cleanup_tx.clone() } - - pub fn dropped_cleanup_events(&self) -> u64 { - self.cleanup_sender.dropped_count() + self.user_manager.dropped_cleanup_events.load(Ordering::Relaxed) - } - - pub fn get_close_connection_channel(&self) -> tokio::sync::broadcast::Receiver { - self.close_socket_signal_tx.subscribe() - } - - pub async fn kick_connection(&self, addr: &SocketAddr, virtual_id: VirtualId, block_secs: u64) -> bool { - debug_if_enabled!( - "User {} kicked for stream with virtual_id {virtual_id} for {block_secs} seconds with addr {}.", - self.user_manager.get_username_for_addr(addr).await.unwrap_or_default(), - sanitize_sensitive_info(&addr.to_string()) - ); - if block_secs > 0 { - self.user_manager.block_user_for_stream(addr, virtual_id, block_secs).await; - } - self.release_connection_as_kicked(addr).await - } - - pub fn register_close_socket(&self, addr: SocketAddr) -> tokio::sync::oneshot::Receiver { - let (tx, rx) = tokio::sync::oneshot::channel(); - if let Ok(mut closers) = self.socket_closers.lock() { - closers.insert(addr, SocketCloseState::Open(tx)); - } - rx - } - - pub fn unregister_close_socket(&self, addr: &SocketAddr) { - if let Ok(mut closers) = self.socket_closers.lock() { - closers.remove(addr); - } - } - - pub async fn close_connection_with_reason_and_block( - &self, - addr: &SocketAddr, - virtual_id: VirtualId, - block_secs: u64, - reason: DisconnectReason, - ) -> bool { - if block_secs > 0 { - self.user_manager.block_user_for_stream(addr, virtual_id, block_secs).await; - } - self.close_connection_with_reason(addr, reason) - } - - pub fn close_connection_signal(&self, addr: &SocketAddr) -> bool { - self.close_connection_with_reason(addr, DisconnectReason::ClientClosed) - } - - pub async fn block_stream_by_uid(&self, uid: u32, virtual_id: VirtualId, block_secs: u64) { - self.user_manager.block_user_for_stream_uid(uid, virtual_id, block_secs).await; - } - - pub fn close_connection_with_reason(&self, addr: &SocketAddr, reason: DisconnectReason) -> bool { - let target_sent = if let Ok(mut closers) = self.socket_closers.lock() { - match closers.remove(addr) { - Some(SocketCloseState::Open(tx)) => { - if tx.send(reason).is_ok() { - closers.insert(*addr, SocketCloseState::Closing); - true - } else { - false - } - } - Some(SocketCloseState::Closing) => { - closers.insert(*addr, SocketCloseState::Closing); - true - } - None => false, - } - } else { - false - }; - let _ = self.close_socket_signal_tx.send(CloseConnectionSignal::WithReason(*addr, reason)); - target_sent - } - - pub async fn release_connection(&self, addr: &SocketAddr) { - release_connection_with_reason(self, addr, DisconnectReason::ClientClosed, true).await; - } - - pub async fn release_connection_with_reason(&self, addr: &SocketAddr, reason: DisconnectReason) { - release_connection_with_reason(self, addr, reason, true).await; - } - - pub async fn release_connection_as_kicked(&self, addr: &SocketAddr) -> bool { - self.release_user_sessions_only(addr).await; - let closed = self.close_connection_with_reason(addr, DisconnectReason::ClientKicked); - if !closed { - self.release_provider_deferred(addr).await; - } - closed - } - - /// Releases the provider connection for `addr` after `tcp_close_notify` is notified. - /// This defers the provider-slot release until after the TCP connection is fully closed, - /// preventing a race where a new request acquires the same provider slot while the old - /// connection is still draining buffered data (which can take 10+ seconds on a live stream). - /// The `close_connection_with_reason` call on the `ConnectionManager` must have already been - /// called before invoking this method. - pub async fn release_provider_deferred(&self, addr: &SocketAddr) { - let addr_owned = *addr; - self.provider_manager.release_connection(&addr_owned); - self.shared_stream_manager.release_connection(&addr_owned, true).await; - notify_capacity(self.capacity_notify.as_ref()); - } - - /// Cleans up user/session/stream state for a forced close without releasing the provider slot. - /// The provider release is deferred to `release_provider_deferred` which waits for the TCP - /// connection to close, preventing a race where a new request acquires the same provider - /// slot while the old connection is still draining buffered data. - /// Note: `release_connection_as_kicked` (called internally) already handles divergence checking. - pub async fn release_user_sessions_only(&self, addr: &SocketAddr) { - let removed = self.user_manager.release_connection_as_kicked(addr).await; - // Mirrors the kicked-specific steps from `release_connection_parts`. - // Provider release and capacity notification are deferred via `release_provider_deferred`. - for stream_info in &removed.removed_streams { - if let Some(session_token) = stream_info.session_token.as_deref() { - self.provider_manager.terminate_identified_playback_owner(session_token); - } - } - for username in &removed.disconnected_users { - self.user_manager.terminate_sessions_for_addr(username, addr).await; - } - for stream_info in &removed.removed_streams { - let qos = self.event_manager.read_meter_qos(stream_info.meter_uid).await; - let bytes_sent = qos.map(|qos| qos.bytes_total); - let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); - self.event_manager.unregister_meter_client(stream_info.uid).await; - emit_disconnect_record( - &self.history_writer, - stream_info, - DisconnectReason::ClientKicked, - &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, - None, - None, - ); - } - if removed.addr_removed && !removed.removed_streams.is_empty() { - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Disconnected(*addr))); - } - } - - pub async fn release_provider_connection(&self, addr: &SocketAddr) { - self.provider_manager.release_connection(addr); - self.shared_stream_manager.release_connection(addr, false).await; - notify_capacity(self.capacity_notify.as_ref()); - } - - pub async fn release_stream(&self, addr: &SocketAddr) { - if let Some(stream_info) = self.user_manager.release_stream(addr).await { - let qos = self.event_manager.read_meter_qos(stream_info.meter_uid).await; - let bytes_sent = qos.map(|qos| qos.bytes_total); - let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); - self.event_manager.unregister_meter_client(stream_info.uid).await; - emit_disconnect_record( - &self.history_writer, - &stream_info, - DisconnectReason::ClientClosed, - &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, - None, - None, - ); - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::DisconnectedStream { - addr: stream_info.addr, - uid: stream_info.uid, - })); - notify_capacity(self.capacity_notify.as_ref()); - } - } - - pub fn release_provider_handle(&self, provider_handle: Option) { - if let Some(handle) = provider_handle { - self.provider_manager.release_handle(&handle); - notify_capacity(self.capacity_notify.as_ref()); - } - } - - /// Releases a managed provider owner by value: its drop performs the synchronous - /// slot release, then the capacity waiters are notified. - pub fn release_managed_provider_handle(&self, provider_handle: Option) { - if provider_handle.is_some() { - drop(provider_handle); - notify_capacity(self.capacity_notify.as_ref()); - } - } - - pub fn next_stream_uid(&self) -> u32 { - self.stream_uid_counter - .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| { - let next = current.wrapping_add(1); - Some(if next == 0 { 1 } else { next }) - }) - .unwrap_or(1) - } - - pub fn record_connect_failed_with_provider_failure( - &self, - info: &StreamInfo, - reason: ConnectFailureReason, - failure_stage: FailureStage, - provider_http_status: Option, - provider_error_class: Option<&str>, - target_name: Option>, - ) { - let guard = self.history_writer.load(); - let Some(writer) = guard.as_ref() else { return }; - let attempt_uid = self.next_stream_uid(); - writer.send_record( - StreamHistoryRecord::from_connect_failed(info, reason, attempt_uid, failure_stage, target_name) - .with_provider_failure(provider_http_status, provider_error_class), - ); - } - - pub fn capacity_notified(&self) -> Arc { Arc::clone(&self.capacity_notify) } - - #[inline] - pub fn is_shutting_down(&self) -> bool { self.is_shutting_down.load(Ordering::Acquire) } - - pub(crate) async fn begin_admission(&self) -> Option> { - if self.is_shutting_down() { - return None; - } - let guard = self.admission_gate.read().await; - (!self.is_shutting_down()).then_some(guard) - } - - /// Emit disconnect records for all still-active streams, unregister shared streams, - /// drain queued cleanups, and flush the history writer. - /// Call once at graceful shutdown before dropping the `ConnectionManager`. - pub async fn shutdown(&self) { - self.is_shutting_down.store(true, Ordering::Release); - let _admission_closed = self.admission_gate.write().await; - self.shared_stream_manager.shutdown().await; - let active_streams = self.user_manager.drain_for_shutdown().await; - for stream_info in &active_streams { - let qos = self.event_manager.read_meter_qos(stream_info.meter_uid).await; - let bytes_sent = qos.map(|qos| qos.bytes_total); - let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); - emit_disconnect_record( - &self.history_writer, - stream_info, - DisconnectReason::Shutdown, - &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, - None, - None, - ); - self.event_manager.unregister_meter_client(stream_info.uid).await; - } - // This terminal transition does not depend on queue capacity or body-held - // cleanup permits. Later guard drops are idempotent against the emptied indices. - self.provider_manager.shutdown(); - if let Some(w) = self.history_writer.load_full() { - w.shutdown().await; - } - // Stop and join the cleanup and socket workers so no background task outlives - // the manager. Active claims were released directly above; later guard drops are - // idempotent against the emptied indices. - self.shutdown_token.cancel(); - let handles = - std::mem::take(&mut *self.worker_handles.lock().unwrap_or_else(std::sync::PoisonError::into_inner)); - for handle in handles { - let _ = handle.await; - } - } - - pub async fn add_connection(&self, addr: &SocketAddr) { self.user_manager.add_connection(addr).await; } - - pub async fn touch_http_activity(&self, username: &str, token: &str, addr: &SocketAddr) { - self.user_manager.touch_http_activity(username, token, addr).await; - self.socket_activity_tracker.track(SocketActivityEvent::HttpActivity { addr: *addr }); - } - - pub fn touch_direct_body_activity(&self, addr: &SocketAddr) { - self.socket_activity_tracker.track(SocketActivityEvent::DirectBodyActivity { addr: *addr }); - } - - pub async fn update_connection(&self, update: ConnectionParams<'_>) -> Option { - self.update_connection_with_history_mode(update, ConnectionHistoryMode::EmitConnect).await - } - - pub async fn update_connection_with_history_mode( - &self, - update: ConnectionParams<'_>, - history_mode: ConnectionHistoryMode, - ) -> Option { - let uid = self.next_stream_uid(); - let mut registered = self.update_connection_with_uid(update, history_mode, uid, None).await; - // This compatibility API has no body to hand the claim to; the caller owns the - // registered stream, so disarm the rollback before returning its metadata. - registered.commit(); - registered.into_display_stream() - } - - pub async fn update_connection_with_uid( - &self, - update: ConnectionParams<'_>, - history_mode: ConnectionHistoryMode, - uid: u32, - provider_request_id: Option, - ) -> RegisteredPlaybackRequest { - self.update_connection_with_uid_and_session_registration(update, history_mode, uid, provider_request_id, None) - .await - } - - pub async fn update_connection_with_uid_and_session_registration( - &self, - update: ConnectionParams<'_>, - history_mode: ConnectionHistoryMode, - uid: u32, - provider_request_id: Option, - session_registration: Option<&crate::PlaybackSessionRegistration>, - ) -> RegisteredPlaybackRequest { - self.update_connection_with_uid_impl( - update, - history_mode, - uid, - provider_request_id, - false, - session_registration, - ) - .await - } - - /// Registers a request whose enclosing shared-subscriber body already owns the - /// guaranteed terminal cleanup permit for this exact stream UID. - pub async fn update_connection_with_uid_using_shared_cleanup( - &self, - update: ConnectionParams<'_>, - history_mode: ConnectionHistoryMode, - capability: SharedCleanupCapability, - provider_request_id: Option, - ) -> RegisteredPlaybackRequest { - self.update_connection_with_uid_impl( - update, - history_mode, - capability.stream_uid(), - provider_request_id, - true, - None, - ) - .await - } - - async fn update_connection_with_uid_impl( - &self, - update: ConnectionParams<'_>, - history_mode: ConnectionHistoryMode, - uid: u32, - provider_request_id: Option, - cleanup_owned_by_shared_subscriber: bool, - session_registration: Option<&crate::PlaybackSessionRegistration>, - ) -> RegisteredPlaybackRequest { - let username = update.username; - let fingerprint = update.fingerprint; - let track_direct_body_activity = uses_direct_body_idle_timeout(update.stream_channel); - let Some(_admission) = self.begin_admission().await else { - warn!("Connection manager is shutting down; rejecting connection registration for user {username}"); - return RegisteredPlaybackRequest::rejected(uid, ConnectionRejectionReason::CleanupReceiverClosed); - }; - // Admission: reserve a guaranteed cleanup right before any registration mutation, - // bounded so a saturated or stalled cleanup queue cannot hang request setup. - let rollback_permit = if cleanup_owned_by_shared_subscriber { - None - } else { - match tokio::time::timeout(CLEANUP_ADMISSION_TIMEOUT, self.cleanup_tx().reserve_owned()).await { - Ok(Ok(permit)) => Some(permit), - Ok(Err(_)) => { - warn!("Cleanup receiver closed; rejecting connection registration for user {username}"); - return RegisteredPlaybackRequest::rejected(uid, ConnectionRejectionReason::CleanupReceiverClosed); - } - Err(_) => { - warn!("Cleanup admission timed out; rejecting connection registration for user {username}"); - return RegisteredPlaybackRequest::rejected( - uid, - ConnectionRejectionReason::CleanupAdmissionTimeout, - ); - } - } - }; - // The cleanup owner is armed before the first mutation, so a cancellation during - // `update_connection`, meter registration or any later await still releases the claim. - let mut cleanup = OwnedRequestCleanup { - addr: fingerprint.addr, - request_uid: uid, - provider_request_id, - owner: update.session_token.map(Arc::::from), - permit: rollback_permit, - finished: false, - }; - if let Some(stream_info) = self - .user_manager - .update_connection_with_session_registration( - ActiveUserConnectionParams { - uid, - meter_uid: update.meter_uid, - username, - max_connections: update.max_connections, - soft_connections: update.soft_connections, - connection_kind: update.connection_kind, - priority: update.priority, - soft_priority: update.soft_priority, - fingerprint, - provider: update.provider, - stream_channel: update.stream_channel, - user_agent: update.user_agent, - session_token: update.session_token, - }, - session_registration, - ) - .await - { - self.event_manager.register_meter_client(stream_info.uid, stream_info.meter_uid).await; - if history_mode == ConnectionHistoryMode::EmitConnect { - emit_connect_record(&self.history_writer, &stream_info); - } - if track_direct_body_activity { - debug_if_enabled!( - "Direct body stream registered for socket expiry: {}", - sanitize_sensitive_info(&fingerprint.addr.to_string()) - ); - self.touch_direct_body_activity(&fingerprint.addr); - } - self.event_manager - .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info.clone()))); - RegisteredPlaybackRequest::with_cleanup(uid, Some(stream_info), Some(cleanup)) - } else { - // Registration failed: nothing was inserted, so the cleanup must not fire. - cleanup.disarm(); - warn!("Failed to register connection for user {username} at {}; disconnecting client", fingerprint.addr); - RegisteredPlaybackRequest::rejected(uid, ConnectionRejectionReason::RegistrationFailed) - } - } - - // pub fn send_active_user_stats(&self, user_count: usize, user_connection_count: usize) { - // self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Connections(user_count, user_connection_count))); - // } - - pub async fn update_stream_detail(&self, addr: &SocketAddr, video_type: CustomVideoStreamType) { - if let Some(stream_info) = self.user_manager.update_stream_detail(addr, video_type).await { - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); - } - } - - pub async fn update_stream_detail_by_uid(&self, uid: u32, video_type: CustomVideoStreamType) { - if let Some(stream_info) = self.user_manager.update_stream_detail_by_uid(uid, video_type).await { - self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); - } - } -} - -/// Build a new `StreamHistoryWriter` from the given config, running file recovery first. -/// Returns `None` if history is disabled or no config is provided. -fn build_history_writer(config: Option<&StreamHistoryConfig>) -> Option> { - let cfg = config?; - if !cfg.stream_history_enabled { - return None; - } - if let Err(e) = recover_pending_files(&cfg.stream_history_directory) { - log::warn!("Stream history recovery failed: {e}"); - } - Some(Arc::new(StreamHistoryWriter::new(cfg))) -} - -/// Async variant used on hot config reload: recovery I/O and compression run on the blocking -/// pool so a runtime worker is never blocked while streams are being served. -async fn build_history_writer_async(config: Option<&StreamHistoryConfig>) -> Option> { - let cfg = config?; - if !cfg.stream_history_enabled { - return None; - } - let directory = cfg.stream_history_directory.clone(); - match tokio::task::spawn_blocking(move || recover_pending_files(&directory)).await { - Ok(Ok(())) => {} - Ok(Err(e)) => log::warn!("Stream history recovery failed: {e}"), - Err(join_err) => log::warn!("Stream history recovery task panicked: {join_err}"), - } - Some(Arc::new(StreamHistoryWriter::new(cfg))) -} - -/// Determine the disconnect reason from the provider-end signal and the stream's current state. -/// -/// Priority: If `update_stream_detail` switched the stream to custom-video mode -/// (`provider == "tuliprox"`), the video type takes precedence. The `provider_end_reason` -/// `AtomicU8` disambiguates `ChannelUnavailable` into `ProviderClosed` (EOF) vs `ProviderError` (Err). -/// -/// SAFETY: The `channel.title` strings (`channel_unavailable`, `low_priority_preempted`, etc.) -/// are wire-format identifiers shared with Serialize/Deserialize and the REST API. -/// If they ever change, update `CustomVideoStreamType::fmt`/`from_str` and this function together. -fn resolve_disconnect_reason(provider_end_reason: u8, stream_info: &StreamInfo) -> DisconnectReason { - if stream_info.provider.as_ref() == "tuliprox" { - if let Ok(video_type) = CustomVideoStreamType::from_str(&stream_info.channel.title) { - match video_type { - CustomVideoStreamType::LowPriorityPreempted => return DisconnectReason::Preempted, - CustomVideoStreamType::UserConnectionsExhausted => return DisconnectReason::UserConnectionsExhausted, - CustomVideoStreamType::ProviderConnectionsExhausted => { - return DisconnectReason::ProviderConnectionsExhausted - } - CustomVideoStreamType::ChannelUnavailable => { - return match provider_end_reason { - PROVIDER_END_CLOSED => DisconnectReason::ProviderClosed, - _ => DisconnectReason::ProviderError, - }; - } - _ => {} - } - } - } - - match provider_end_reason { - PROVIDER_END_CLOSED => DisconnectReason::ProviderClosed, - PROVIDER_END_ERROR => DisconnectReason::ProviderError, - PROVIDER_END_PREEMPTED => DisconnectReason::Preempted, - _ => DisconnectReason::ClientClosed, - } -} - -/// Resolves a disconnect reason from the provider-end signal alone, for the path where -/// the user stream claim is already gone and no `StreamInfo` is available to consult. -fn resolve_disconnect_reason_from_provider_end(provider_end_reason: u8) -> DisconnectReason { - match provider_end_reason { - PROVIDER_END_CLOSED => DisconnectReason::ProviderClosed, - PROVIDER_END_ERROR => DisconnectReason::ProviderError, - PROVIDER_END_PREEMPTED => DisconnectReason::Preempted, - _ => DisconnectReason::ClientClosed, - } -} - -/// Maps a disconnect reason onto the provider-lease outcome policy. -/// -/// Only a clean client-side end keeps a reconnect-capable lease alive; provider -/// failures, preemption, kicks and timeouts release capacity immediately. -fn playback_outcome_for_reason(reason: DisconnectReason) -> PlaybackRequestOutcome { - match reason { - DisconnectReason::ClientClosed - | DisconnectReason::DayRollover - | DisconnectReason::Cleanup - | DisconnectReason::Unknown => PlaybackRequestOutcome::ClientClosed, - DisconnectReason::Timeout => PlaybackRequestOutcome::TimedOut, - DisconnectReason::SessionExpired => PlaybackRequestOutcome::SessionExpired, - DisconnectReason::Shutdown => PlaybackRequestOutcome::ServerShutdown, - DisconnectReason::ClientKicked => PlaybackRequestOutcome::Kicked, - DisconnectReason::Preempted => PlaybackRequestOutcome::Preempted, - DisconnectReason::Provisioning - | DisconnectReason::UserConnectionsExhausted - | DisconnectReason::ProviderConnectionsExhausted => PlaybackRequestOutcome::FailedBeforeMedia, - DisconnectReason::ProviderError - | DisconnectReason::ProviderClosed - | DisconnectReason::ServerError - | DisconnectReason::IntermediateFailures(_) => PlaybackRequestOutcome::ProviderFailed, - } -} - -fn emit_connect_record(writer: &ArcSwapOption, info: &StreamInfo) { - let guard = writer.load(); - let Some(w) = guard.as_ref() else { return }; - w.send_record(StreamHistoryRecord::from_connect(info)); -} - -fn emit_disconnect_record( - writer: &ArcSwapOption, - info: &StreamInfo, - reason: DisconnectReason, - qos: &DisconnectQos, - provider_error_class: Option<&str>, - provider_http_status: Option, -) { - let guard = writer.load(); - let Some(w) = guard.as_ref() else { return }; - w.send_record( - StreamHistoryRecord::from_disconnect(info, reason, qos, resolve_disconnect_failure_stage(info, reason, qos)) - .with_provider_failure(provider_http_status, provider_error_class), - ); -} - -fn resolve_disconnect_failure_stage( - info: &StreamInfo, - reason: DisconnectReason, - qos: &DisconnectQos, -) -> Option { - match reason { - DisconnectReason::ProviderError | DisconnectReason::ProviderClosed => { - if !info.channel.shared && qos.first_byte_latency_ms.is_none() { - Some(FailureStage::FirstByte) - } else { - Some(FailureStage::Streaming) - } - } - DisconnectReason::Preempted => Some(FailureStage::Streaming), - DisconnectReason::SessionExpired => Some(FailureStage::SessionReconnect), - _ => None, - } -} - #[cfg(test)] -mod tests { - use super::*; - use crate::{ActiveProviderManager, ActiveUserManager, CreateUserSessionParams, EventManager, SharedStreamManager}; - use arc_swap::{ArcSwap, ArcSwapOption}; - use shared::{ - model::{ - ConfigPaths, InputFetchMethod, InputType, PlaylistItemType, ProxyType, StreamChannel, StreamInfo, - UserConnectionPermission, XtreamCluster, - }, - utils::Internable, - }; - use std::{collections::HashMap, net::SocketAddr, sync::Arc}; - use tokio::sync::mpsc; - use tuliprox_core::{ - model::{AppConfig, Config, ConfigInput, MediaToolCapabilities, ProxyUserCredentials, SourcesConfig}, - utils::FileLockManager, - }; - use tuliprox_repository::GeoIp; - - fn make_stream_info(provider: &str, title: &str) -> StreamInfo { - let addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); - let channel = StreamChannel { - target_id: 1, - virtual_id: 1, - provider_id: 1, - input_name: "input".intern(), - item_type: PlaylistItemType::Live, - cluster: XtreamCluster::Live, - group: "".intern(), - title: title.intern(), - url: "".intern(), - shared: false, - shared_joined_existing: None, - shared_stream_id: None, - technical: None, - epg_channel_id: None, - epg_reference_ts: None, - upstream_user_agent: None, - }; - StreamInfo::new(shared::model::StreamInfoParams { - uid: 0, - meter_uid: 0, - username: "test", - addr: &addr, - client_ip: "127.0.0.1", - provider: provider.intern(), - stream_channel: channel, - user_agent: String::new(), - country_code: None, - session_token: None, - }) - } - - fn create_test_app_config() -> AppConfig { - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: "http://provider-1.example".to_string(), - username: Some("user1".to_string()), - password: Some("pass1".to_string()), - enabled: true, - priority: 0, - max_connections: 1, - method: InputFetchMethod::default(), - aliases: None, - ..ConfigInput::default() - }); - let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; - - AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config::default())), - sources: Arc::new(ArcSwap::from_pointee(sources)), - hdhomerun: Arc::new(ArcSwapOption::default()), - api_proxy: Arc::new(ArcSwapOption::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - } - } - - fn create_test_connection_manager() -> Arc { - let app_cfg = create_test_app_config(); - let event_manager = Arc::new(EventManager::new()); - let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let shared_manager = Arc::new(SharedStreamManager::new(Arc::clone(&provider_manager))); - provider_manager.set_shared_stream_manager(&shared_manager); - - let geo_ip = Arc::new(ArcSwapOption::::default()); - let config = app_cfg.config.load(); - let user_manager = Arc::new(ActiveUserManager::new(&config, &geo_ip, &event_manager)); - - Arc::new(ConnectionManager::new(&user_manager, &provider_manager, &shared_manager, &event_manager, None)) - } - - fn create_test_proxy_user(username: &str) -> ProxyUserCredentials { - let mut user = ProxyUserCredentials::default(); - user.username = username.to_string(); - user.password = "password".to_string(); - user.proxy = ProxyType::Reverse(None); - user.max_connections = 1; - user - } - - #[tokio::test] - async fn enqueue_with_backpressure_delivers_events_after_queue_full() { - let (tx, mut rx) = mpsc::channel(1); - let sender = BackpressureSender::new(tx.clone(), "test", 2); - assert!(tx.send(1_u8).await.is_ok()); - - sender.enqueue(2_u8); - sender.enqueue(3_u8); - - assert_eq!(rx.recv().await, Some(1)); - assert_eq!(tokio::time::timeout(Duration::from_secs(1), rx.recv()).await.ok().flatten(), Some(2)); - assert_eq!(tokio::time::timeout(Duration::from_secs(1), rx.recv()).await.ok().flatten(), Some(3)); - } - - #[tokio::test] - async fn enqueue_with_backpressure_bounds_overflow_buffer() { - let (tx, mut rx) = mpsc::channel(1); - let sender = BackpressureSender::new(tx.clone(), "test", 1); - assert!(tx.send(1_u8).await.is_ok()); - - sender.enqueue(2_u8); - for _ in 0..50 { - let ready = { - let state = super::lock_backpressure_state(sender.state.as_ref()); - state.overflow.is_empty() && state.draining - }; - if ready { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - sender.enqueue(3_u8); - sender.enqueue(4_u8); - - assert_eq!(rx.recv().await, Some(1)); - assert_eq!(tokio::time::timeout(Duration::from_secs(1), rx.recv()).await.ok().flatten(), Some(2)); - assert_eq!(tokio::time::timeout(Duration::from_secs(1), rx.recv()).await.ok().flatten(), Some(3)); - let result = tokio::time::timeout(Duration::from_millis(100), rx.recv()).await; - assert!(result.is_err()); - } - - #[test] - fn socket_activity_tracker_coalesces_updates_per_socket() { - let tracker = SocketActivityTracker::new(); - let addr_one: SocketAddr = "127.0.0.1:3234".parse().unwrap_or_else(|_| unreachable!()); - let addr_two: SocketAddr = "127.0.0.1:3235".parse().unwrap_or_else(|_| unreachable!()); - - tracker.track(SocketActivityEvent::HttpActivity { addr: addr_one }); - tracker.track(SocketActivityEvent::HttpActivity { addr: addr_one }); - tracker.track(SocketActivityEvent::DirectBodyActivity { addr: addr_two }); - - let pending = tracker.drain(); - assert_eq!(pending.len(), 2); - assert!(pending.iter().any(|event| matches!( - event, - SocketActivityEvent::HttpActivity { addr } if *addr == addr_one - ))); - assert!(pending.iter().any(|event| matches!( - event, - SocketActivityEvent::DirectBodyActivity { addr } if *addr == addr_two - ))); - } - - #[tokio::test] - async fn touch_http_activity_is_processed_by_socket_activity_worker() { - let manager = create_test_connection_manager(); - let addr: SocketAddr = "127.0.0.1:3234".parse().unwrap_or_else(|_| unreachable!()); - let user = create_test_proxy_user("user1"); - - manager.add_connection(&addr).await; - let _ = manager - .user_manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-touch", - virtual_id: 1, - provider: "provider_1", - stream_url: "http://provider-1.example/live.ts", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::ConnectionKind::Normal), - socket_bound: true, - }) - .await; - - manager.touch_http_activity(&user.username, "tok-touch", &addr).await; - - assert!(tokio::time::timeout(Duration::from_secs(1), async { - loop { - if manager.user_manager.socket_expiry_deadline(&addr).await.is_some() - && manager.user_manager.get_username_for_addr(&addr).await.as_deref() - == Some(user.username.as_str()) - { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .is_ok()); - } - - #[tokio::test] - async fn kick_connection_sends_kick_close_signal() { - let manager = create_test_connection_manager(); - let mut rx = manager.get_close_connection_channel(); - let addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); - let _close_rx = manager.register_close_socket(addr); - - assert!(manager.kick_connection(&addr, shared::model::VirtualId::new(1), 0).await); - assert_eq!(rx.recv().await.ok(), Some(CloseConnectionSignal::WithReason(addr, DisconnectReason::ClientKicked))); - } - - #[tokio::test] - async fn kick_connection_returns_false_without_receivers() { - let manager = create_test_connection_manager(); - let addr: SocketAddr = "127.0.0.1:1235".parse().unwrap_or_else(|_| unreachable!()); - assert!(!manager.kick_connection(&addr, shared::model::VirtualId::new(1), 0).await); - } - - #[tokio::test] - async fn close_connection_with_reason_returns_false_for_dropped_oneshot_receiver() { - let manager = create_test_connection_manager(); - let addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); - let close_rx = manager.register_close_socket(addr); - drop(close_rx); - - assert!( - !manager.close_connection_with_reason(&addr, DisconnectReason::ClientKicked), - "must return false when target oneshot receiver was dropped" - ); - } - - #[tokio::test] - async fn close_connection_with_reason_returns_false_when_only_broadcast_receiver_exists() { - let manager = create_test_connection_manager(); - let mut broadcast_rx = manager.get_close_connection_channel(); - let addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); - - assert!( - !manager.close_connection_with_reason(&addr, DisconnectReason::ClientKicked), - "must return false when only broadcast receiver exists without registered target socket" - ); - assert_eq!( - broadcast_rx.recv().await.ok(), - Some(CloseConnectionSignal::WithReason(addr, DisconnectReason::ClientKicked)) - ); - } - - #[tokio::test] - async fn close_connection_with_reason_returns_false_for_unrelated_addr_even_with_broadcast_receivers() { - let manager = create_test_connection_manager(); - let _broadcast_rx = manager.get_close_connection_channel(); - let active_addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); - let unrelated_addr: SocketAddr = "127.0.0.1:5678".parse().unwrap_or_else(|_| unreachable!()); - - let _close_rx = manager.register_close_socket(active_addr); - - assert!( - !manager.close_connection_with_reason(&unrelated_addr, DisconnectReason::ClientKicked), - "unrelated address must not report success just because another socket is registered" - ); - assert!( - manager.close_connection_with_reason(&active_addr, DisconnectReason::ClientKicked), - "target address registered must report success" - ); - } - - #[tokio::test] - async fn release_connection_as_kicked_cleans_provider_when_unreceived() { - let manager = create_test_connection_manager(); - let addr: SocketAddr = "127.0.0.1:2236".parse().unwrap_or_else(|_| unreachable!()); - let input_name = "provider_1".intern(); - let handle = manager - .provider_manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - 0, - crate::ConnectionKind::Normal, - Some("test-owner"), - ) - .expect("acquire provider"); - assert_eq!(manager.provider_manager.get_provider_connections_count(), 1); - - let closed = manager.release_connection_as_kicked(&addr).await; - assert!(!closed, "should return false when no receiver is subscribed"); - assert_eq!(manager.provider_manager.get_provider_connections_count(), 0); - - drop(handle); - } - - #[tokio::test] - async fn release_connection_as_kicked_does_not_release_provider_prematurely_on_repeated_kick() { - let manager = create_test_connection_manager(); - let addr: SocketAddr = "127.0.0.1:2237".parse().unwrap_or_else(|_| unreachable!()); - let input_name = "provider_1".intern(); - let handle = manager - .provider_manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - 0, - crate::ConnectionKind::Normal, - Some("test-owner"), - ) - .expect("acquire provider"); - assert_eq!(manager.provider_manager.get_provider_connections_count(), 1); - - let mut close_rx = manager.register_close_socket(addr); - - // First kick sends close signal to socket and transitions to Closing - let closed_first = manager.release_connection_as_kicked(&addr).await; - assert!(closed_first, "first kick must report success when socket is registered"); - assert_eq!( - manager.provider_manager.get_provider_connections_count(), - 1, - "provider must remain allocated after first kick while socket is in Closing state" - ); - - // Second kick while socket is still Closing must not prematurely release provider - let closed_second = manager.release_connection_as_kicked(&addr).await; - assert!(closed_second, "second kick during Closing state must report success"); - assert_eq!( - manager.provider_manager.get_provider_connections_count(), - 1, - "provider must not be released prematurely by second kick before socket finishes" - ); - - // Now simulate the socket processing the signal and finishing its cleanup - let reason = close_rx.try_recv().expect("close signal must be pending in oneshot"); - assert_eq!(reason, DisconnectReason::ClientKicked); - manager.release_provider_deferred(&addr).await; - manager.unregister_close_socket(&addr); - assert_eq!( - manager.provider_manager.get_provider_connections_count(), - 0, - "provider must be released after socket finishes and unregisters" - ); - - drop(handle); - } - - #[tokio::test] - async fn release_connection_as_kicked_sends_kick_close_signal() { - let manager = create_test_connection_manager(); - let mut rx = manager.get_close_connection_channel(); - let addr: SocketAddr = "127.0.0.1:2234".parse().unwrap_or_else(|_| unreachable!()); - - manager.release_connection_as_kicked(&addr).await; - assert_eq!( - tokio::time::timeout(Duration::from_millis(100), rx.recv()).await.ok().and_then(Result::ok), - Some(CloseConnectionSignal::WithReason(addr, DisconnectReason::ClientKicked)) - ); - } - - #[tokio::test] - async fn release_connection_as_kicked_is_idempotent() { - let manager = create_test_connection_manager(); - let mut rx = manager.get_close_connection_channel(); - let addr: SocketAddr = "127.0.0.1:2235".parse().unwrap_or_else(|_| unreachable!()); - - manager.add_connection(&addr).await; - manager.release_connection_as_kicked(&addr).await; - assert_eq!( - tokio::time::timeout(Duration::from_millis(100), rx.recv()).await.ok().and_then(Result::ok), - Some(CloseConnectionSignal::WithReason(addr, DisconnectReason::ClientKicked)) - ); - - manager.release_connection_as_kicked(&addr).await; - } - - #[tokio::test] - async fn close_connection_signal_sends_generic_close_signal() { - let manager = create_test_connection_manager(); - let mut rx = manager.get_close_connection_channel(); - let addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); - let _close_rx = manager.register_close_socket(addr); - - assert!(manager.close_connection_signal(&addr)); - assert_eq!(rx.recv().await.ok(), Some(CloseConnectionSignal::WithReason(addr, DisconnectReason::ClientClosed))); - } - - #[tokio::test] - async fn provisioning_close_connection_sends_provisioning_signal() { - let manager = create_test_connection_manager(); - let mut rx = manager.get_close_connection_channel(); - let addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); - let _close_rx = manager.register_close_socket(addr); - - assert!( - manager - .close_connection_with_reason_and_block( - &addr, - shared::model::VirtualId::new(7), - 0, - DisconnectReason::Provisioning - ) - .await - ); - assert_eq!(rx.recv().await.ok(), Some(CloseConnectionSignal::WithReason(addr, DisconnectReason::Provisioning))); - } - - #[tokio::test] - async fn low_priority_preempted_cleanup_blocks_same_user_stream_reentry() { - let manager = create_test_connection_manager(); - let user = create_test_proxy_user("preempted-user"); - let addr: SocketAddr = "127.0.0.1:6234".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); - let channel = StreamChannel { - virtual_id: 409, - title: "channel-409".intern(), - ..make_stream_info("provider_1", "channel-409").channel - }; - - manager.add_connection(&addr).await; - manager - .user_manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-preempted", - virtual_id: channel.virtual_id, - provider: "provider_1", - stream_url: "http://provider-1.example/live/409.ts", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::ConnectionKind::Normal), - socket_bound: true, - }) - .await; - manager - .user_manager - .update_connection(ActiveUserConnectionParams { - uid: 9001, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: crate::ConnectionKind::Normal, - priority: 9, - soft_priority: 9, - fingerprint: &fingerprint, - provider: "provider_1".intern(), - stream_channel: &channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some("tok-preempted"), - }) - .await; - - let deps = CleanupWorkerDeps { - user_manager: Arc::clone(&manager.user_manager), - provider_manager: Arc::clone(&manager.provider_manager), - shared_stream_manager: Arc::clone(&manager.shared_stream_manager), - event_manager: Arc::clone(&manager.event_manager), - capacity_notify: Arc::clone(&manager.capacity_notify), - history_writer: Arc::clone(&manager.history_writer), - }; - - handle_update_detail_and_release_provider(&deps, addr, CustomVideoStreamType::LowPriorityPreempted, None, None) - .await; - - assert!( - manager - .user_manager - .is_user_blocked_for_stream(&user.username, shared::model::VirtualId::new(channel.virtual_id)) - .await, - "preempted playback should be blocked briefly to prevent immediate reconnect ping-pong" - ); - } - - #[tokio::test] - async fn low_priority_preempted_cleanup_blocks_same_user_hls_reentry() { - let manager = create_test_connection_manager(); - let user = create_test_proxy_user("preempted-hls-user"); - let addr: SocketAddr = "127.0.0.1:6235".parse().unwrap_or_else(|_| unreachable!()); - let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); - let mut channel = make_stream_info("provider_1", "channel-410").channel; - channel.virtual_id = 410; - channel.item_type = PlaylistItemType::LiveHls; - channel.title = "channel-410.m3u8".intern(); - channel.url = "http://provider-1.example/live/410.m3u8".intern(); - - manager.add_connection(&addr).await; - manager - .user_manager - .create_user_session(CreateUserSessionParams { - user: &user, - session_token: "tok-preempted-hls", - virtual_id: channel.virtual_id, - provider: "provider_1", - stream_url: "http://provider-1.example/live/410.m3u8", - addr: &addr, - connection_permission: UserConnectionPermission::Allowed, - connection_kind: Some(crate::ConnectionKind::Normal), - socket_bound: false, - }) - .await; - manager - .user_manager - .update_connection(ActiveUserConnectionParams { - uid: 9002, - meter_uid: 0, - username: &user.username, - max_connections: user.max_connections, - soft_connections: user.soft_connections, - connection_kind: crate::ConnectionKind::Normal, - priority: 9, - soft_priority: 9, - fingerprint: &fingerprint, - provider: "provider_1".intern(), - stream_channel: &channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some("tok-preempted-hls"), - }) - .await; - - let deps = CleanupWorkerDeps { - user_manager: Arc::clone(&manager.user_manager), - provider_manager: Arc::clone(&manager.provider_manager), - shared_stream_manager: Arc::clone(&manager.shared_stream_manager), - event_manager: Arc::clone(&manager.event_manager), - capacity_notify: Arc::clone(&manager.capacity_notify), - history_writer: Arc::clone(&manager.history_writer), - }; - - handle_update_detail_and_release_provider(&deps, addr, CustomVideoStreamType::LowPriorityPreempted, None, None) - .await; - - assert!( - manager - .user_manager - .is_user_blocked_for_stream(&user.username, shared::model::VirtualId::new(channel.virtual_id)) - .await, - "preempted HLS playback should be blocked briefly to prevent immediate reconnect ping-pong" - ); - } - - #[test] - fn test_client_closed_when_no_provider_end() { - let info = make_stream_info("some_provider", "Some Channel"); - let reason = resolve_disconnect_reason(PROVIDER_END_NOT_SET, &info); - assert_eq!(reason, DisconnectReason::ClientClosed); - } - - #[test] - fn test_client_kicked_disconnect_has_no_failure_stage() { - assert_eq!( - resolve_disconnect_failure_stage( - &make_stream_info("some_provider", "Some Channel"), - DisconnectReason::ClientKicked, - &DisconnectQos::default(), - ), - None - ); - } - - #[test] - fn test_provisioning_disconnect_has_no_failure_stage() { - assert_eq!( - resolve_disconnect_failure_stage( - &make_stream_info("some_provider", "Some Channel"), - DisconnectReason::Provisioning, - &DisconnectQos::default(), - ), - None - ); - } - - #[test] - fn test_provider_closed_on_eof() { - let info = make_stream_info("some_provider", "Some Channel"); - let reason = resolve_disconnect_reason(PROVIDER_END_CLOSED, &info); - assert_eq!(reason, DisconnectReason::ProviderClosed); - } - - #[test] - fn test_provider_error_on_err() { - let info = make_stream_info("some_provider", "Some Channel"); - let reason = resolve_disconnect_reason(PROVIDER_END_ERROR, &info); - assert_eq!(reason, DisconnectReason::ProviderError); - } - - #[test] - fn test_preempted_from_custom_video_detail() { - let info = make_stream_info("tuliprox", "low_priority_preempted"); - let reason = resolve_disconnect_reason(PROVIDER_END_NOT_SET, &info); - assert_eq!(reason, DisconnectReason::Preempted); - } - - #[test] - fn test_provider_preempted_atomic_maps_to_preempted() { - let info = make_stream_info("some_provider", "Some Channel"); - let reason = resolve_disconnect_reason(PROVIDER_END_PREEMPTED, &info); - assert_eq!(reason, DisconnectReason::Preempted); - - let reason_direct = resolve_disconnect_reason_from_provider_end(PROVIDER_END_PREEMPTED); - assert_eq!(reason_direct, DisconnectReason::Preempted); - assert_eq!(playback_outcome_for_reason(reason_direct), PlaybackRequestOutcome::Preempted); - } - - #[test] - fn test_channel_unavailable_with_eof_maps_to_provider_closed() { - let info = make_stream_info("tuliprox", "channel_unavailable"); - let reason = resolve_disconnect_reason(PROVIDER_END_CLOSED, &info); - assert_eq!(reason, DisconnectReason::ProviderClosed); - } - - #[test] - fn test_channel_unavailable_with_err_maps_to_provider_error() { - let info = make_stream_info("tuliprox", "channel_unavailable"); - let reason = resolve_disconnect_reason(PROVIDER_END_ERROR, &info); - assert_eq!(reason, DisconnectReason::ProviderError); - } - - #[test] - fn test_channel_unavailable_without_atomic_maps_to_provider_error() { - let info = make_stream_info("tuliprox", "channel_unavailable"); - let reason = resolve_disconnect_reason(PROVIDER_END_NOT_SET, &info); - assert_eq!(reason, DisconnectReason::ProviderError); - } - - #[test] - fn test_user_exhausted_custom_video_maps_to_user_connections_exhausted() { - let info = make_stream_info("tuliprox", "user_connections_exhausted"); - let reason = resolve_disconnect_reason(PROVIDER_END_NOT_SET, &info); - assert_eq!(reason, DisconnectReason::UserConnectionsExhausted); - } - - #[test] - fn test_provider_exhausted_custom_video_maps_to_provider_connections_exhausted() { - let info = make_stream_info("tuliprox", "provider_connections_exhausted"); - let reason = resolve_disconnect_reason(PROVIDER_END_NOT_SET, &info); - assert_eq!(reason, DisconnectReason::ProviderConnectionsExhausted); - } - - #[test] - fn test_unknown_tuliprox_title_falls_through_to_atomic() { - let info = make_stream_info("tuliprox", "some_unknown_video_type"); - let reason = resolve_disconnect_reason(PROVIDER_END_CLOSED, &info); - assert_eq!(reason, DisconnectReason::ProviderClosed); - } - - #[test] - fn test_provider_error_disconnect_maps_to_streaming_failure_stage() { - assert_eq!( - resolve_disconnect_failure_stage( - &make_stream_info("some_provider", "Some Channel"), - DisconnectReason::ProviderError, - &DisconnectQos { first_byte_latency_ms: Some(150), ..Default::default() }, - ), - Some(FailureStage::Streaming) - ); - } - - #[test] - fn test_session_expired_disconnect_maps_to_session_reconnect_stage() { - assert_eq!( - resolve_disconnect_failure_stage( - &make_stream_info("some_provider", "Some Channel"), - DisconnectReason::SessionExpired, - &DisconnectQos::default(), - ), - Some(FailureStage::SessionReconnect) - ); - } - - #[test] - fn test_provider_error_without_first_byte_maps_to_first_byte_stage() { - assert_eq!( - resolve_disconnect_failure_stage( - &make_stream_info("some_provider", "Some Channel"), - DisconnectReason::ProviderError, - &DisconnectQos::default(), - ), - Some(FailureStage::FirstByte) - ); - } - - #[test] - fn test_shared_provider_error_without_first_byte_stays_streaming_stage() { - let mut info = make_stream_info("some_provider", "Some Channel"); - info.channel.shared = true; - assert_eq!( - resolve_disconnect_failure_stage(&info, DisconnectReason::ProviderError, &DisconnectQos::default(),), - Some(FailureStage::Streaming) - ); - } - - #[tokio::test] - async fn deferred_cleanup_executes_in_background_worker() { - let conn_manager = create_test_connection_manager(); - let ran = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let notify = Arc::new(Notify::new()); - let ran_clone = Arc::clone(&ran); - let notify_clone = Arc::clone(¬ify); - - conn_manager.send_cleanup(CleanupEvent::Defer(Box::pin(async move { - ran_clone.store(true, Ordering::SeqCst); - notify_clone.notify_one(); - }))); - - tokio::time::timeout(Duration::from_secs(1), notify.notified()).await.expect("deferred task must execute"); - assert!(ran.load(Ordering::SeqCst)); - } - - /// Dropping a just-registered request without handing it to a body must roll the - /// claim back, so a cancellation between registration and body construction cannot - /// leak a user claim. - #[tokio::test] - async fn registration_rollback_releases_claim_when_body_never_constructed() { - let manager = create_test_connection_manager(); - let addr: SocketAddr = "127.0.0.1:56234".parse().unwrap(); - let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); - let channel = StreamChannel { - virtual_id: 410, - title: "channel-410".intern(), - ..make_stream_info("provider_1", "channel-410").channel - }; - - manager.add_connection(&addr).await; - let registered = manager - .update_connection_with_uid( - ConnectionParams { - meter_uid: 0, - username: "rollback-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider_1".intern(), - stream_channel: &channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: None, - }, - ConnectionHistoryMode::EmitConnect, - 1, - None, - ) - .await; - assert!(registered.display_stream.is_some(), "registration must succeed"); - - // Drop without committing: the rollback owner releases the claim. - drop(registered); - tokio::time::timeout(Duration::from_secs(2), async { - loop { - if manager.user_manager.active_streams().await.is_empty() { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("the rollback must release the claim"); - } - - /// The body cleanup transferred via `into_body_cleanup` releases the claim exactly - /// once with the real outcome, even under cleanup-queue pressure. - #[tokio::test] - async fn body_cleanup_finish_releases_claim_exactly_once() { - let manager = create_test_connection_manager(); - let addr: SocketAddr = "127.0.0.1:56236".parse().unwrap(); - let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); - let channel = StreamChannel { - virtual_id: 412, - title: "channel-412".intern(), - ..make_stream_info("provider_1", "channel-412").channel - }; - - manager.add_connection(&addr).await; - let mut registered = manager - .update_connection_with_uid( - ConnectionParams { - meter_uid: 0, - username: "body-cleanup-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider_1".intern(), - stream_channel: &channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: None, - }, - ConnectionHistoryMode::EmitConnect, - 1, - None, - ) - .await; - assert!(registered.display_stream.is_some(), "registration must succeed"); - - let mut cleanup = registered.into_body_cleanup().expect("body cleanup present"); - cleanup.finish(None, PROVIDER_END_CLOSED, 0, None, None); - cleanup.finish(None, PROVIDER_END_CLOSED, 0, None, None); - - tokio::time::timeout(Duration::from_secs(2), async { - loop { - if manager.user_manager.active_streams().await.is_empty() { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("the body cleanup must release the claim exactly once"); - } - - /// A saturated cleanup queue must reject registration with an admission-timeout reason - /// (rather than hanging), so the caller can surface a defined non-success response. - #[tokio::test(start_paused = true)] - async fn cleanup_admission_timeout_rejects_registration_with_reason() { - let manager = create_test_connection_manager(); - // Stall the cleanup worker on a never-resolving defer, then fill the channel so - // `reserve_owned()` blocks and the bounded admission deadline fires. - let pending = || CleanupEvent::Defer(Box::pin(std::future::pending::<()>())); - manager.send_cleanup(pending()); - for _ in 0..CLEANUP_QUEUE_CAPACITY { - manager.send_cleanup(pending()); - } - - let addr: SocketAddr = "127.0.0.1:56242".parse().unwrap(); - let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); - let channel = StreamChannel { - virtual_id: 414, - title: "channel-414".intern(), - ..make_stream_info("provider_1", "channel-414").channel - }; - - manager.add_connection(&addr).await; - let manager_for_task = Arc::clone(&manager); - let registration = tokio::spawn(async move { - manager_for_task - .update_connection_with_uid( - ConnectionParams { - meter_uid: 0, - username: "admission-timeout-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider_1".intern(), - stream_channel: &channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: None, - }, - ConnectionHistoryMode::EmitConnect, - 1, - None, - ) - .await - }); - - // Let the task reach the blocked cleanup reservation, then fire the deadline. - tokio::task::yield_now().await; - tokio::task::yield_now().await; - tokio::time::advance(CLEANUP_ADMISSION_TIMEOUT + Duration::from_secs(1)).await; - let registered = registration.await.expect("registration task must not panic"); - - assert!(registered.display_stream.is_none(), "saturated cleanup admission must be rejected"); - assert_eq!(registered.rejection_reason(), Some(ConnectionRejectionReason::CleanupAdmissionTimeout)); - } - - #[tokio::test] - async fn control_cleanup_lane_precedes_a_ready_body_cleanup_backlog() { - let manager = create_test_connection_manager(); - let release_blocker = Arc::new(Notify::new()); - let (blocker_started_tx, blocker_started_rx) = tokio::sync::oneshot::channel(); - let order = Arc::new(std::sync::Mutex::new(Vec::new())); - - manager.send_cleanup(CleanupEvent::Defer(Box::pin({ - let release_blocker = Arc::clone(&release_blocker); - let order = Arc::clone(&order); - async move { - let _ = blocker_started_tx.send(()); - release_blocker.notified().await; - order.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push('B'); - } - }))); - blocker_started_rx.await.expect("cleanup blocker must start"); - - manager.send_cleanup(CleanupEvent::Defer(Box::pin({ - let order = Arc::clone(&order); - async move { order.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push('N') } - }))); - manager - .control_cleanup_tx() - .send(CleanupEvent::Defer(Box::pin({ - let order = Arc::clone(&order); - async move { order.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push('C') } - }))) - .await - .expect("control cleanup receiver must be open"); - - release_blocker.notify_one(); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if order.lock().unwrap_or_else(std::sync::PoisonError::into_inner).len() == 3 { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("both queued cleanup events must run"); - - assert_eq!( - *order.lock().unwrap_or_else(std::sync::PoisonError::into_inner), - vec!['B', 'C', 'N'], - "a ready control cleanup must not be starved by the body cleanup backlog" - ); - } - - /// A joined shared body already holds the terminal cleanup permit. Registration must - /// reuse that ownership instead of waiting for a second permit from the same full queue. - #[tokio::test] - async fn shared_admission_does_not_wait_for_second_permit_it_prevents() { - let manager = create_test_connection_manager(); - let cleanup_tx = manager.cleanup_tx(); - let available = cleanup_tx.capacity(); - let mut permits = Vec::with_capacity(available); - for _ in 0..available { - permits.push(cleanup_tx.clone().try_reserve_owned().expect("cleanup permit")); - } - - let addr: SocketAddr = "127.0.0.1:56243".parse().expect("test socket"); - let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); - let channel = StreamChannel { - virtual_id: 415, - title: "channel-415".intern(), - shared: true, - shared_joined_existing: Some(true), - ..make_stream_info("provider_1", "channel-415").channel - }; - - manager.add_connection(&addr).await; - let mut registered = tokio::time::timeout( - Duration::from_millis(100), - manager.update_connection_with_uid_using_shared_cleanup( - ConnectionParams { - meter_uid: 0, - username: "shared-admission-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider_1".intern(), - stream_channel: &channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: None, - }, - ConnectionHistoryMode::EmitConnect, - SharedCleanupCapability::new(tuliprox_core::model::SharedSubscriberId::from_stream_uid(1)), - None, - ), - ) - .await - .expect("externally owned cleanup must not reserve another permit"); - - assert!(registered.display_stream.is_some(), "shared registration must succeed"); - assert!(registered.into_body_cleanup().is_some(), "the body still tracks its terminal outcome"); - let _ = manager.user_manager.release_stream_request_by_uid(&addr, 1).await; - drop(permits); - } - - /// A cleanup whose user row is already gone must still finish the provider request, - /// because the provider identity travels with the cleanup owner independently. - #[tokio::test] - async fn cleanup_finishes_provider_request_when_user_row_is_gone() { - let manager = create_test_connection_manager(); - let addr: SocketAddr = "127.0.0.1:56240".parse().unwrap(); - let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); - let channel = StreamChannel { - virtual_id: 413, - title: "channel-413".intern(), - ..make_stream_info("provider_1", "channel-413").channel - }; - let input_name = "provider_1".intern(); - let owner = "session-owner-g2"; - - // Acquire a provider slot with an identified lease. - let handle = manager - .provider_manager - .acquire_connection_with_grace_for_session( - &input_name, - &addr, - false, - 0, - crate::ConnectionKind::Normal, - Some(owner), - ) - .expect("acquire provider slot"); - let request_id = handle.playback_request_id.expect("identified request id"); - assert!(manager.provider_manager.provider_lease_usage(&input_name).total() > 0); - - // Register a user claim carrying the same owner and provider request id. - manager.add_connection(&addr).await; - let mut registered = manager - .update_connection_with_uid( - ConnectionParams { - meter_uid: 0, - username: "g2-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider_1".intern(), - stream_channel: &channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some(owner), - }, - ConnectionHistoryMode::EmitConnect, - 1, - Some(request_id), - ) - .await; - assert!(registered.display_stream.is_some()); - - let cleanup = registered.into_body_cleanup().expect("cleanup present"); - - // Remove the user row first, so the cleanup worker hits the NotFound path. - let _ = manager.user_manager.release_stream_request_by_uid(&addr, 1).await; - - drop(cleanup); - - tokio::time::timeout(Duration::from_secs(2), async { - loop { - if manager.provider_manager.provider_lease_usage(&input_name).total() == 0 { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("provider request must finish even without a user row"); - - manager.provider_manager.release_handle(&handle); - } - - /// Shutdown releases every active stream so no claim or provider slot survives. - #[tokio::test] - async fn shutdown_releases_active_streams() { - let manager = create_test_connection_manager(); - let addr: SocketAddr = "127.0.0.1:56235".parse().unwrap(); - let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); - let channel = StreamChannel { - virtual_id: 411, - title: "channel-411".intern(), - ..make_stream_info("provider_1", "channel-411").channel - }; - - manager.add_connection(&addr).await; - let stream_info = manager - .update_connection(ConnectionParams { - meter_uid: 0, - username: "shutdown-user", - max_connections: 1, - soft_connections: 0, - connection_kind: crate::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider_1".intern(), - stream_channel: &channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: None, - }) - .await; - assert!(stream_info.is_some(), "registration must succeed"); - assert!(!manager.user_manager.active_streams().await.is_empty()); - - manager.shutdown().await; - assert!(manager.user_manager.active_streams().await.is_empty(), "shutdown must release active streams"); - } - - #[tokio::test] - async fn shutdown_parallel_requests_on_distinct_sockets_drains_all_claims() { - let manager = create_test_connection_manager(); - let first_addr: SocketAddr = "127.0.0.1:56241".parse().unwrap(); - let second_addr: SocketAddr = "127.0.0.1:56242".parse().unwrap(); - let first_fingerprint = - tuliprox_core::model::Fingerprint::new("range-first".to_string(), "127.0.0.1".to_string(), first_addr); - let second_fingerprint = - tuliprox_core::model::Fingerprint::new("range-second".to_string(), "127.0.0.1".to_string(), second_addr); - let mut channel = make_stream_info("provider_1", "parallel-range").channel; - channel.item_type = PlaylistItemType::Video; - channel.cluster = XtreamCluster::Video; - let owner = "shutdown-parallel-range"; - let input_name = "provider_1".intern(); - let handle = manager - .provider_manager - .acquire_connection_with_grace_for_session( - &input_name, - &first_addr, - false, - 0, - crate::ConnectionKind::Normal, - Some(owner), - ) - .expect("provider allocation"); - let request_id = handle.playback_request_id.expect("provider request id"); - - manager.add_connection(&first_addr).await; - manager.add_connection(&second_addr).await; - let mut first = manager - .update_connection_with_uid( - ConnectionParams { - meter_uid: 0, - username: "shutdown-range-user", - max_connections: 2, - soft_connections: 0, - connection_kind: crate::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &first_fingerprint, - provider: Arc::clone(&input_name), - stream_channel: &channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some(owner), - }, - ConnectionHistoryMode::EmitConnect, - 1, - Some(request_id), - ) - .await; - let mut second = manager - .update_connection_with_uid( - ConnectionParams { - meter_uid: 0, - username: "shutdown-range-user", - max_connections: 2, - soft_connections: 0, - connection_kind: crate::ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &second_fingerprint, - provider: Arc::clone(&input_name), - stream_channel: &channel, - user_agent: std::borrow::Cow::Borrowed("player/1.0"), - session_token: Some(owner), - }, - ConnectionHistoryMode::EmitConnect, - 2, - None, - ) - .await; - assert!(first.display_stream.is_some()); - assert!(second.display_stream.is_some()); - assert_eq!(manager.user_manager.active_streams().await.len(), 1); - assert_eq!(manager.user_manager.playback_resource_counts().await.0, 2); - assert_eq!(manager.provider_manager.get_provider_connections_count(), 1); - - let first_cleanup = first.into_body_cleanup().expect("first cleanup"); - let second_cleanup = second.into_body_cleanup().expect("second cleanup"); - manager.shutdown().await; - - assert_eq!(manager.user_manager.playback_resource_counts().await, (0, 0)); - assert_eq!(manager.provider_manager.get_provider_connections_count(), 0); - assert_eq!(manager.provider_manager.provider_lease_usage(&input_name).total(), 0); - - drop(first_cleanup); - drop(second_cleanup); - drop(handle); - } - - #[tokio::test] - async fn shutdown_with_all_cleanup_permits_held_completes() { - let manager = create_test_connection_manager(); - let mut permits = Vec::with_capacity(CLEANUP_QUEUE_CAPACITY); - for _ in 0..CLEANUP_QUEUE_CAPACITY { - permits.push(manager.cleanup_tx().reserve_owned().await.expect("cleanup receiver open")); - } - assert_eq!(manager.cleanup_tx().capacity(), 0); - - tokio::time::timeout(Duration::from_secs(2), manager.shutdown()) - .await - .expect("shutdown must not require cleanup queue capacity"); - drop(permits); - } - - #[tokio::test] - async fn new_with_capacity_sizes_the_cleanup_queue() { - let app_cfg = create_test_app_config(); - let event_manager = Arc::new(EventManager::new()); - let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let shared_manager = Arc::new(SharedStreamManager::new(Arc::clone(&provider_manager))); - provider_manager.set_shared_stream_manager(&shared_manager); - let geo_ip = Arc::new(ArcSwapOption::::default()); - let config = app_cfg.config.load(); - let user_manager = Arc::new(ActiveUserManager::new(&config, &geo_ip, &event_manager)); - - let manager = Arc::new(ConnectionManager::new_with_capacity( - &user_manager, - &provider_manager, - &shared_manager, - &event_manager, - None, - 2, - )); - assert_eq!(manager.cleanup_tx().capacity(), 2); - - // A capacity of zero is clamped to one so the cleanup queue is always usable. - let manager = Arc::new(ConnectionManager::new_with_capacity( - &user_manager, - &provider_manager, - &shared_manager, - &event_manager, - None, - 0, - )); - assert_eq!(manager.cleanup_tx().capacity(), 1); - } -} +mod tests; + +mod backpressure; +mod cleanup; +mod history; +mod manager; +mod registration; +mod requests; +mod socket; +#[cfg(test)] +use backpressure::lock_backpressure_state; +#[cfg(test)] +use cleanup::handle_update_detail_and_release_provider; +pub use cleanup::CleanupEvent; +#[cfg(test)] +use history::resolve_disconnect_failure_stage; +pub use requests::{ConnectionHistoryMode, ConnectionParams, ConnectionRejectionReason}; +pub use socket::CloseConnectionSignal; +use socket::{SocketActivityEvent, SocketCloseState}; diff --git a/backend/session/src/connection_manager/backpressure.rs b/backend/session/src/connection_manager/backpressure.rs new file mode 100644 index 000000000..945b5c53b --- /dev/null +++ b/backend/session/src/connection_manager/backpressure.rs @@ -0,0 +1,184 @@ +use super::{ + socket::SocketActivityEvent, BackpressureSender, BackpressureState, SharedCleanupCapability, SocketActivityTracker, +}; +use log::{debug, warn}; +use std::{ + collections::{HashSet, VecDeque}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, MutexGuard, + }, + thread, +}; +use tokio::sync::{mpsc, Notify}; +use tuliprox_core::model::SharedSubscriberId; + +pub(super) fn notify_capacity(capacity_notify: &Notify) { capacity_notify.notify_waiters(); } + +impl SharedCleanupCapability { + pub(crate) fn new(subscriber_id: SharedSubscriberId) -> Self { Self { subscriber_id } } + + pub fn stream_uid(&self) -> u32 { self.subscriber_id.stream_uid() } + + pub fn subscriber_id(&self) -> SharedSubscriberId { self.subscriber_id } +} + +impl BackpressureSender +where + T: Send + 'static, +{ + pub(super) fn new(tx: mpsc::Sender, queue_name: &'static str, overflow_capacity: usize) -> Self { + Self { + tx, + state: Arc::new(Mutex::new(BackpressureState { overflow: VecDeque::new(), draining: false })), + queue_name, + overflow_capacity, + dropped_events: AtomicU64::new(0), + } + } + + pub(super) fn enqueue(&self, event: T) { + let runtime = tokio::runtime::Handle::try_current().ok(); + { + let mut state = lock_backpressure_state(self.state.as_ref()); + if state.draining { + if state.overflow.len() >= self.overflow_capacity { + let count = self.dropped_events.fetch_add(1, Ordering::Relaxed) + 1; + if count == 1 || count.is_multiple_of(1024) { + warn!( + "{} overflow buffer full (capacity={}), {} events dropped total", + self.queue_name, self.overflow_capacity, count + ); + } + return; + } + state.overflow.push_back(event); + return; + } + + match self.tx.try_send(event) { + Ok(()) => return, + Err(tokio::sync::mpsc::error::TrySendError::Full(event)) => { + state.draining = true; + if state.overflow.len() >= self.overflow_capacity { + let count = self.dropped_events.fetch_add(1, Ordering::Relaxed) + 1; + if count == 1 || count.is_multiple_of(1024) { + warn!( + "{} overflow buffer full (capacity={}), {} events dropped total", + self.queue_name, self.overflow_capacity, count + ); + } + state.draining = false; + return; + } + state.overflow.push_back(event); + } + Err(tokio::sync::mpsc::error::TrySendError::Closed(_event)) => { + debug!("{} channel closed, dropping event", self.queue_name); + return; + } + } + } + + let tx = self.tx.clone(); + let state = Arc::clone(&self.state); + let queue_name = self.queue_name; + if let Some(handle) = runtime { + handle.spawn(async move { + Self::drain_async(&tx, &state, queue_name).await; + }); + } else { + thread::spawn(move || Self::drain_blocking(&tx, &state, queue_name)); + } + } + + pub(super) async fn drain_async( + tx: &mpsc::Sender, + state: &Arc>>, + queue_name: &'static str, + ) { + loop { + let Some(event) = Self::next_event(state) else { + break; + }; + if tx.send(event).await.is_err() { + debug!("{queue_name} channel closed while draining backpressure"); + Self::clear_and_stop(state); + break; + } + } + } + + pub(super) fn drain_blocking( + tx: &mpsc::Sender, + state: &Arc>>, + queue_name: &'static str, + ) { + loop { + let Some(event) = Self::next_event(state) else { + break; + }; + if tx.blocking_send(event).is_err() { + warn!("{queue_name} channel closed while draining backpressure"); + Self::clear_and_stop(state); + break; + } + } + } + + pub(super) fn next_event(state: &Arc>>) -> Option { + let mut state = lock_backpressure_state(state.as_ref()); + if let Some(event) = state.overflow.pop_front() { + return Some(event); + } + state.draining = false; + None + } + + pub(super) fn clear_and_stop(state: &Arc>>) { + let mut state = lock_backpressure_state(state.as_ref()); + state.overflow.clear(); + state.draining = false; + } + + pub(super) fn dropped_count(&self) -> u64 { self.dropped_events.load(Ordering::Relaxed) } +} + +pub(super) fn lock_backpressure_state(state: &Mutex>) -> MutexGuard<'_, BackpressureState> { + match state.lock() { + Ok(guard) => guard, + Err(poisoned) => { + warn!("Backpressure queue state was poisoned, continuing with recovered state"); + poisoned.into_inner() + } + } +} + +impl SocketActivityTracker { + pub(super) fn new() -> Self { + Self { pending: Arc::new(Mutex::new(HashSet::new())), notify: Arc::new(Notify::new()) } + } + + pub(super) fn track(&self, event: SocketActivityEvent) { + let mut pending = lock_socket_activity_pending(self.pending.as_ref()); + pending.insert(event); + drop(pending); + self.notify.notify_one(); + } + + pub(super) fn drain(&self) -> Vec { + let mut pending = lock_socket_activity_pending(self.pending.as_ref()); + pending.drain().collect() + } + + pub(super) async fn notified(&self) { self.notify.notified().await; } +} + +fn lock_socket_activity_pending( + pending: &Mutex>, +) -> MutexGuard<'_, HashSet> { + pending.lock().unwrap_or_else(|poisoned| { + warn!("Socket activity state was poisoned, continuing with recovered state"); + poisoned.into_inner() + }) +} diff --git a/backend/session/src/connection_manager/cleanup.rs b/backend/session/src/connection_manager/cleanup.rs new file mode 100644 index 000000000..1deb00438 --- /dev/null +++ b/backend/session/src/connection_manager/cleanup.rs @@ -0,0 +1,492 @@ +use super::{ + backpressure::notify_capacity, + history::{ + emit_disconnect_record, playback_outcome_for_reason, resolve_disconnect_reason, + resolve_disconnect_reason_from_provider_end, + }, + CleanupWorkerDeps, ConnectionManager, PREEMPT_REENTRY_BLOCK_SECS, PROVIDER_END_NOT_SET, +}; +use crate::{ActiveProviderManager, ActiveUserManager, EventManager, SharedStreamManager}; +use arc_swap::ArcSwapOption; +use futures::future::BoxFuture; +use log::debug; +use shared::{ + model::{ActiveUserConnectionChange, CustomVideoStreamType, DisconnectReason, EventMessage, StreamInfo}, + utils::sanitize_sensitive_info, +}; +use std::{ + net::SocketAddr, + sync::{atomic::Ordering, Arc}, +}; +use tokio::sync::{mpsc, Notify}; +use tokio_util::sync::CancellationToken; +use tuliprox_core::{ + model::{DisconnectQos, ProviderHandle, SharedSubscriberId}, + utils::debug_if_enabled, +}; +use tuliprox_repository::StreamHistoryWriter; + +pub enum CleanupEvent { + ReleaseSharedSubscriber { + addr: SocketAddr, + subscriber_id: SharedSubscriberId, + request_id: Option, + owner: Option>, + }, + ReleaseStream { + request_id: Option, + /// Playback owner (session token) captured at acquire, so the provider request + /// can be finished independently of whether the user claim still exists. + owner: Option>, + addr: SocketAddr, + stream_uid: Option, + provider_end_reason: u8, + reconnect_count: u8, + provider_error_class: Option<&'static str>, + provider_http_status: Option, + }, + ReleaseConnection { + addr: SocketAddr, + }, + ReleaseProviderHandle { + handle: Option, + }, + ReleaseStreamAndProviderHandle { + request_id: Option, + addr: SocketAddr, + stream_uid: Option, + handle: Option, + provider_end_reason: u8, + reconnect_count: u8, + provider_error_class: Option<&'static str>, + provider_http_status: Option, + }, + UpdateDetailAndReleaseProvider { + addr: SocketAddr, + stream_uid: Option, + video_type: CustomVideoStreamType, + handle: Option, + }, + AdaptiveSessionExpired { + stream_info: Box, + }, + /// Confirms that real provider media reached the client for a playback lease. + /// Only a confirmed lease may reserve provider capacity against other playbacks. + ConfirmPlaybackLease { + owner: Arc, + request_id: Option, + }, + /// Runs a deferred asynchronous cleanup task in the managed cleanup worker. + Defer(BoxFuture<'static, ()>), +} + +async fn handle_release_connection(deps: &CleanupWorkerDeps, addr: SocketAddr) { + release_connection_parts(deps, &addr, DisconnectReason::Cleanup, true).await; +} + +pub(super) async fn release_connection_with_reason( + connection_manager: &ConnectionManager, + addr: &SocketAddr, + reason: DisconnectReason, + send_shared_stop_signal: bool, +) { + let deps = CleanupWorkerDeps { + user_manager: Arc::clone(&connection_manager.user_manager), + provider_manager: Arc::clone(&connection_manager.provider_manager), + shared_stream_manager: Arc::clone(&connection_manager.shared_stream_manager), + event_manager: Arc::clone(&connection_manager.event_manager), + capacity_notify: Arc::clone(&connection_manager.capacity_notify), + history_writer: Arc::clone(&connection_manager.history_writer), + }; + release_connection_parts(&deps, addr, reason, send_shared_stop_signal).await; +} + +pub(super) async fn release_connection_parts( + deps: &CleanupWorkerDeps, + addr: &SocketAddr, + reason: DisconnectReason, + send_shared_stop_signal: bool, +) { + let removed = if matches!(reason, DisconnectReason::ClientKicked) { + deps.user_manager.release_connection_as_kicked(addr).await + } else { + deps.user_manager.release_connection(addr).await + }; + if matches!(reason, DisconnectReason::ClientKicked) { + for stream_info in &removed.removed_streams { + if let Some(session_token) = stream_info.session_token.as_deref() { + deps.provider_manager.terminate_identified_playback_owner(session_token); + } + } + // Explicitly terminate all sessions for the kicked addr. This expires them + // immediately rather than leaving them for TTL-based GC cleanup. + for username in &removed.disconnected_users { + deps.user_manager.terminate_sessions_for_addr(username, addr).await; + } + } + for stream_info in &removed.removed_streams { + let qos = deps.event_manager.read_meter_qos(stream_info.meter_uid).await; + let bytes_sent = qos.map(|qos| qos.bytes_total); + let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); + deps.event_manager.unregister_meter_client(stream_info.uid).await; + emit_disconnect_record( + &deps.history_writer, + stream_info, + reason, + &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, + None, + None, + ); + } + deps.provider_manager.release_connection(addr); + deps.shared_stream_manager.release_connection(addr, send_shared_stop_signal).await; + if removed.addr_removed && !removed.removed_streams.is_empty() { + deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Disconnected(*addr))); + } + notify_capacity(deps.capacity_notify.as_ref()); +} + +#[allow(clippy::too_many_arguments)] +async fn handle_release_stream( + deps: &CleanupWorkerDeps, + addr: SocketAddr, + stream_uid: Option, + provider_end_reason: u8, + reconnect_count: u8, + provider_error_class: Option<&'static str>, + provider_http_status: Option, + request_id: Option, + owner: Option>, +) { + if let Some(stream_info) = release_stream_with_disconnect( + deps, + addr, + stream_uid, + provider_end_reason, + reconnect_count, + provider_error_class, + provider_http_status, + request_id, + owner, + ) + .await + { + deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::DisconnectedStream { + addr: stream_info.addr, + uid: stream_info.uid, + })); + notify_capacity(deps.capacity_notify.as_ref()); + } +} + +fn handle_release_provider_handle(deps: &CleanupWorkerDeps, handle: Option) { + if let Some(handle) = handle { + deps.provider_manager.release_handle(&handle); + notify_capacity(deps.capacity_notify.as_ref()); + } +} + +#[allow(clippy::too_many_arguments)] +async fn handle_release_stream_and_provider_handle( + deps: &CleanupWorkerDeps, + addr: SocketAddr, + stream_uid: Option, + handle: Option, + provider_end_reason: u8, + reconnect_count: u8, + provider_error_class: Option<&'static str>, + provider_http_status: Option, + request_id: Option, +) { + let provider_released = if let Some(handle) = handle { + deps.provider_manager.release_handle(&handle); + true + } else { + false + }; + let stream_released = release_stream_with_disconnect( + deps, + addr, + stream_uid, + provider_end_reason, + reconnect_count, + provider_error_class, + provider_http_status, + request_id, + None, + ) + .await; + if let Some(stream_info) = stream_released.as_ref() { + deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::DisconnectedStream { + addr: stream_info.addr, + uid: stream_info.uid, + })); + } + if provider_released || stream_released.is_some() { + notify_capacity(deps.capacity_notify.as_ref()); + } +} + +pub(super) async fn handle_update_detail_and_release_provider( + deps: &CleanupWorkerDeps, + addr: SocketAddr, + video_type: CustomVideoStreamType, + handle: Option, + stream_uid: Option, +) { + let stream_info = if let Some(uid) = stream_uid { + deps.user_manager.update_stream_detail_by_uid(uid, video_type).await + } else { + deps.user_manager.update_stream_detail(&addr, video_type).await + }; + if let Some(stream_info) = stream_info { + if matches!(video_type, CustomVideoStreamType::LowPriorityPreempted) { + deps.user_manager + .block_user_for_stream_uid( + stream_info.uid, + shared::model::VirtualId::new(stream_info.channel.virtual_id), + PREEMPT_REENTRY_BLOCK_SECS, + ) + .await; + } + deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + } + if let Some(handle) = handle { + deps.provider_manager.release_handle(&handle); + notify_capacity(deps.capacity_notify.as_ref()); + } +} + +async fn handle_adaptive_session_expired(deps: &CleanupWorkerDeps, stream_info: Box) { + let qos = deps.event_manager.read_meter_qos(stream_info.meter_uid).await; + let bytes_sent = qos.map(|qos| qos.bytes_total); + let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); + deps.event_manager.unregister_meter_client(stream_info.uid).await; + emit_disconnect_record( + &deps.history_writer, + &stream_info, + DisconnectReason::SessionExpired, + &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, + None, + None, + ); + deps.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::DisconnectedStream { + addr: stream_info.addr, + uid: stream_info.uid, + })); + notify_capacity(deps.capacity_notify.as_ref()); +} + +#[allow(clippy::too_many_arguments)] +async fn release_stream_with_disconnect( + deps: &CleanupWorkerDeps, + addr: SocketAddr, + stream_uid: Option, + provider_end_reason: u8, + reconnect_count: u8, + provider_error_class: Option<&'static str>, + provider_http_status: Option, + request_id: Option, + owner: Option>, +) -> Option { + let detach = if let Some(stream_uid) = stream_uid { + deps.user_manager.release_stream_request_by_uid(&addr, stream_uid).await + } else { + crate::StreamRequestDetach::NotFound + }; + match detach { + crate::StreamRequestDetach::NotFound => { + debug_if_enabled!( + "Stream release skipped: no active stream for {} uid={:?}", + sanitize_sensitive_info(&addr.to_string()), + stream_uid + ); + // The provider request must still be finished even when the user claim is + // already gone: the identity travels with the cleanup event. + if let (Some(owner), Some(request_id)) = (owner.as_deref(), request_id) { + let reason = resolve_disconnect_reason_from_provider_end(provider_end_reason); + deps.provider_manager.finish_identified_playback_request( + owner, + request_id, + playback_outcome_for_reason(reason), + ); + notify_capacity(deps.capacity_notify.as_ref()); + } + None + } + crate::StreamRequestDetach::Retained(stream_info) | crate::StreamRequestDetach::Preserved(stream_info) => { + let reason = resolve_disconnect_reason(provider_end_reason, &stream_info); + if let (Some(session_token), Some(request_id)) = + (owner.as_deref().or(stream_info.session_token.as_deref()), request_id) + { + deps.provider_manager.finish_identified_playback_request( + session_token, + request_id, + playback_outcome_for_reason(reason), + ); + } + notify_capacity(deps.capacity_notify.as_ref()); + None + } + crate::StreamRequestDetach::Removed(stream_info) => { + let qos = deps.event_manager.read_meter_qos(stream_info.meter_uid).await; + let bytes_sent = qos.map(|qos| qos.bytes_total); + let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); + deps.event_manager.unregister_meter_client(stream_info.uid).await; + let reason = resolve_disconnect_reason(provider_end_reason, &stream_info); + if let (Some(session_token), Some(request_id)) = + (owner.as_deref().or(stream_info.session_token.as_deref()), request_id) + { + deps.provider_manager.finish_identified_playback_request( + session_token, + request_id, + playback_outcome_for_reason(reason), + ); + } + let provider_reconnect_count = (reconnect_count > 0).then_some(reconnect_count); + emit_disconnect_record( + &deps.history_writer, + &stream_info, + reason, + &DisconnectQos { bytes_sent, first_byte_latency_ms, provider_reconnect_count }, + provider_error_class, + provider_http_status, + ); + Some(stream_info) + } + } +} + +impl ConnectionManager { + #[allow(clippy::too_many_arguments, clippy::too_many_lines)] + pub(super) fn spawn_cleanup_worker( + mut rx: mpsc::Receiver, + mut control_rx: mpsc::Receiver, + user_manager: Arc, + provider_manager: Arc, + shared_stream_manager: Arc, + event_manager: Arc, + capacity_notify: Arc, + history_writer: Arc>, + shutdown_token: CancellationToken, + ) -> tokio::task::JoinHandle<()> { + let deps = CleanupWorkerDeps { + user_manager, + provider_manager, + shared_stream_manager, + event_manager, + capacity_notify, + history_writer, + }; + tokio::spawn(async move { + loop { + let event = tokio::select! { + biased; + () = shutdown_token.cancelled() => break, + control_event = control_rx.recv() => control_event, + event = rx.recv() => event, + }; + let Some(event) = event else { break }; + match event { + CleanupEvent::ReleaseSharedSubscriber { addr, subscriber_id, request_id, owner } => { + deps.shared_stream_manager.release_subscriber(subscriber_id).await; + handle_release_stream( + &deps, + addr, + Some(subscriber_id.stream_uid()), + PROVIDER_END_NOT_SET, + 0, + None, + None, + request_id, + owner, + ) + .await; + notify_capacity(deps.capacity_notify.as_ref()); + } + CleanupEvent::ReleaseConnection { addr } => { + handle_release_connection(&deps, addr).await; + } + CleanupEvent::ReleaseStream { + request_id, + owner, + addr, + stream_uid, + provider_end_reason, + reconnect_count, + provider_error_class, + provider_http_status, + } => { + handle_release_stream( + &deps, + addr, + stream_uid, + provider_end_reason, + reconnect_count, + provider_error_class, + provider_http_status, + request_id, + owner, + ) + .await; + } + CleanupEvent::ReleaseProviderHandle { handle } => { + handle_release_provider_handle(&deps, handle); + } + CleanupEvent::ReleaseStreamAndProviderHandle { + request_id, + addr, + stream_uid, + handle, + provider_end_reason, + reconnect_count, + provider_error_class, + provider_http_status, + } => { + handle_release_stream_and_provider_handle( + &deps, + addr, + stream_uid, + handle, + provider_end_reason, + reconnect_count, + provider_error_class, + provider_http_status, + request_id, + ) + .await; + } + CleanupEvent::UpdateDetailAndReleaseProvider { addr, stream_uid, video_type, handle } => { + handle_update_detail_and_release_provider(&deps, addr, video_type, handle, stream_uid).await; + } + CleanupEvent::AdaptiveSessionExpired { stream_info } => { + handle_adaptive_session_expired(&deps, stream_info).await; + } + CleanupEvent::ConfirmPlaybackLease { owner, request_id } => { + if let Some(request_id) = request_id { + deps.provider_manager.confirm_identified_playback_activity(&owner, request_id); + } else { + deps.provider_manager.confirm_playback_activity(&owner); + } + } + CleanupEvent::Defer(future) => { + future.await; + } + } + } + debug!("Cleanup worker exiting"); + }) + } + + pub fn send_cleanup(&self, event: CleanupEvent) { self.cleanup_sender.enqueue(event); } + + pub fn cleanup_tx(&self) -> mpsc::Sender { self.cleanup_sender.tx.clone() } + + /// Reserved control lane for cleanup permits that must never be starved by a + /// saturated body-cleanup queue (HLS origin I/O, shutdown). + pub fn control_cleanup_tx(&self) -> mpsc::Sender { self.control_cleanup_tx.clone() } + + pub fn dropped_cleanup_events(&self) -> u64 { + self.cleanup_sender.dropped_count() + self.user_manager.dropped_cleanup_events.load(Ordering::Relaxed) + } +} diff --git a/backend/session/src/connection_manager/history.rs b/backend/session/src/connection_manager/history.rs new file mode 100644 index 000000000..1dbc22c6d --- /dev/null +++ b/backend/session/src/connection_manager/history.rs @@ -0,0 +1,151 @@ +use super::{PROVIDER_END_CLOSED, PROVIDER_END_ERROR, PROVIDER_END_PREEMPTED}; +use arc_swap::ArcSwapOption; +use shared::model::{CustomVideoStreamType, DisconnectReason, FailureStage, StreamInfo}; +use std::{str::FromStr, sync::Arc}; +use tuliprox_core::model::{DisconnectQos, PlaybackRequestOutcome, StreamHistoryConfig, StreamHistoryRecord}; +use tuliprox_repository::{recover_pending_files, StreamHistoryWriter}; + +/// Build a new `StreamHistoryWriter` from the given config, running file recovery first. +/// Returns `None` if history is disabled or no config is provided. +pub(super) fn build_history_writer(config: Option<&StreamHistoryConfig>) -> Option> { + let cfg = config?; + if !cfg.stream_history_enabled { + return None; + } + if let Err(e) = recover_pending_files(&cfg.stream_history_directory) { + log::warn!("Stream history recovery failed: {e}"); + } + Some(Arc::new(StreamHistoryWriter::new(cfg))) +} + +/// Async variant used on hot config reload: recovery I/O and compression run on the blocking +/// pool so a runtime worker is never blocked while streams are being served. +pub(super) async fn build_history_writer_async( + config: Option<&StreamHistoryConfig>, +) -> Option> { + let cfg = config?; + if !cfg.stream_history_enabled { + return None; + } + let directory = cfg.stream_history_directory.clone(); + match tokio::task::spawn_blocking(move || recover_pending_files(&directory)).await { + Ok(Ok(())) => {} + Ok(Err(e)) => log::warn!("Stream history recovery failed: {e}"), + Err(join_err) => log::warn!("Stream history recovery task panicked: {join_err}"), + } + Some(Arc::new(StreamHistoryWriter::new(cfg))) +} + +/// Determine the disconnect reason from the provider-end signal and the stream's current state. +/// +/// Priority: If `update_stream_detail` switched the stream to custom-video mode +/// (`provider == "tuliprox"`), the video type takes precedence. The `provider_end_reason` +/// `AtomicU8` disambiguates `ChannelUnavailable` into `ProviderClosed` (EOF) vs `ProviderError` (Err). +/// +/// SAFETY: The `channel.title` strings (`channel_unavailable`, `low_priority_preempted`, etc.) +/// are wire-format identifiers shared with Serialize/Deserialize and the REST API. +/// If they ever change, update `CustomVideoStreamType::fmt`/`from_str` and this function together. +pub(super) fn resolve_disconnect_reason(provider_end_reason: u8, stream_info: &StreamInfo) -> DisconnectReason { + if stream_info.provider.as_ref() == "tuliprox" { + if let Ok(video_type) = CustomVideoStreamType::from_str(&stream_info.channel.title) { + match video_type { + CustomVideoStreamType::LowPriorityPreempted => return DisconnectReason::Preempted, + CustomVideoStreamType::UserConnectionsExhausted => return DisconnectReason::UserConnectionsExhausted, + CustomVideoStreamType::ProviderConnectionsExhausted => { + return DisconnectReason::ProviderConnectionsExhausted + } + CustomVideoStreamType::ChannelUnavailable => { + return match provider_end_reason { + PROVIDER_END_CLOSED => DisconnectReason::ProviderClosed, + _ => DisconnectReason::ProviderError, + }; + } + _ => {} + } + } + } + + match provider_end_reason { + PROVIDER_END_CLOSED => DisconnectReason::ProviderClosed, + PROVIDER_END_ERROR => DisconnectReason::ProviderError, + PROVIDER_END_PREEMPTED => DisconnectReason::Preempted, + _ => DisconnectReason::ClientClosed, + } +} + +/// Resolves a disconnect reason from the provider-end signal alone, for the path where +/// the user stream claim is already gone and no `StreamInfo` is available to consult. +pub(super) fn resolve_disconnect_reason_from_provider_end(provider_end_reason: u8) -> DisconnectReason { + match provider_end_reason { + PROVIDER_END_CLOSED => DisconnectReason::ProviderClosed, + PROVIDER_END_ERROR => DisconnectReason::ProviderError, + PROVIDER_END_PREEMPTED => DisconnectReason::Preempted, + _ => DisconnectReason::ClientClosed, + } +} + +/// Maps a disconnect reason onto the provider-lease outcome policy. +/// +/// Only a clean client-side end keeps a reconnect-capable lease alive; provider +/// failures, preemption, kicks and timeouts release capacity immediately. +pub(super) fn playback_outcome_for_reason(reason: DisconnectReason) -> PlaybackRequestOutcome { + match reason { + DisconnectReason::ClientClosed + | DisconnectReason::DayRollover + | DisconnectReason::Cleanup + | DisconnectReason::Unknown => PlaybackRequestOutcome::ClientClosed, + DisconnectReason::Timeout => PlaybackRequestOutcome::TimedOut, + DisconnectReason::SessionExpired => PlaybackRequestOutcome::SessionExpired, + DisconnectReason::Shutdown => PlaybackRequestOutcome::ServerShutdown, + DisconnectReason::ClientKicked => PlaybackRequestOutcome::Kicked, + DisconnectReason::Preempted => PlaybackRequestOutcome::Preempted, + DisconnectReason::Provisioning + | DisconnectReason::UserConnectionsExhausted + | DisconnectReason::ProviderConnectionsExhausted => PlaybackRequestOutcome::FailedBeforeMedia, + DisconnectReason::ProviderError + | DisconnectReason::ProviderClosed + | DisconnectReason::ServerError + | DisconnectReason::IntermediateFailures(_) => PlaybackRequestOutcome::ProviderFailed, + } +} + +pub(super) fn emit_connect_record(writer: &ArcSwapOption, info: &StreamInfo) { + let guard = writer.load(); + let Some(w) = guard.as_ref() else { return }; + w.send_record(StreamHistoryRecord::from_connect(info)); +} + +pub(super) fn emit_disconnect_record( + writer: &ArcSwapOption, + info: &StreamInfo, + reason: DisconnectReason, + qos: &DisconnectQos, + provider_error_class: Option<&str>, + provider_http_status: Option, +) { + let guard = writer.load(); + let Some(w) = guard.as_ref() else { return }; + w.send_record( + StreamHistoryRecord::from_disconnect(info, reason, qos, resolve_disconnect_failure_stage(info, reason, qos)) + .with_provider_failure(provider_http_status, provider_error_class), + ); +} + +pub(super) fn resolve_disconnect_failure_stage( + info: &StreamInfo, + reason: DisconnectReason, + qos: &DisconnectQos, +) -> Option { + match reason { + DisconnectReason::ProviderError | DisconnectReason::ProviderClosed => { + if !info.channel.shared && qos.first_byte_latency_ms.is_none() { + Some(FailureStage::FirstByte) + } else { + Some(FailureStage::Streaming) + } + } + DisconnectReason::Preempted => Some(FailureStage::Streaming), + DisconnectReason::SessionExpired => Some(FailureStage::SessionReconnect), + _ => None, + } +} diff --git a/backend/session/src/connection_manager/manager.rs b/backend/session/src/connection_manager/manager.rs new file mode 100644 index 000000000..492689b88 --- /dev/null +++ b/backend/session/src/connection_manager/manager.rs @@ -0,0 +1,312 @@ +use super::{ + backpressure::notify_capacity, + cleanup::release_connection_with_reason, + history::{build_history_writer, build_history_writer_async, emit_disconnect_record}, + BackpressureSender, ConnectionManager, SocketActivityTracker, CLEANUP_QUEUE_CAPACITY, CONTROL_CLEANUP_CAPACITY, +}; +use crate::{ActiveProviderManager, ActiveUserManager, EventManager, ManagedProviderHandle, SharedStreamManager}; +use arc_swap::ArcSwapOption; +use shared::model::{ + ActiveUserConnectionChange, ConnectFailureReason, DisconnectReason, EventMessage, FailureStage, StreamInfo, +}; +use std::{ + collections::HashMap, + net::SocketAddr, + sync::{ + atomic::{AtomicBool, AtomicU32, Ordering}, + Arc, + }, +}; +use tokio::sync::{mpsc, Notify, RwLock, RwLockReadGuard}; +use tokio_util::sync::CancellationToken; +use tuliprox_core::model::{DisconnectQos, ProviderHandle, StreamHistoryConfig, StreamHistoryRecord}; +use tuliprox_repository::StreamHistoryWriter; + +impl ConnectionManager { + pub fn new( + user_manager: &Arc, + provider_manager: &Arc, + shared_stream_manager: &Arc, + event_manager: &Arc, + history_config: Option<&StreamHistoryConfig>, + ) -> Self { + Self::new_with_capacity( + user_manager, + provider_manager, + shared_stream_manager, + event_manager, + history_config, + CLEANUP_QUEUE_CAPACITY, + ) + } + + /// Same as [`ConnectionManager::new`], but with an explicit cleanup-queue capacity + /// derived from configuration instead of the hard-coded default. + pub fn new_with_capacity( + user_manager: &Arc, + provider_manager: &Arc, + shared_stream_manager: &Arc, + event_manager: &Arc, + history_config: Option<&StreamHistoryConfig>, + cleanup_capacity: usize, + ) -> Self { + let cleanup_capacity = cleanup_capacity.max(1); + let history_writer = Arc::new(ArcSwapOption::new(build_history_writer(history_config))); + let (close_socket_signal_tx, _) = tokio::sync::broadcast::channel(256); + let (cleanup_tx, cleanup_rx) = mpsc::channel(cleanup_capacity); + let (control_cleanup_tx, control_cleanup_rx) = mpsc::channel(CONTROL_CLEANUP_CAPACITY); + user_manager.set_cleanup_sender(cleanup_tx.clone()); + user_manager.set_provider_manager(Arc::clone(provider_manager)); + let socket_cleanup_tx = cleanup_tx.clone(); + let socket_activity_tracker = SocketActivityTracker::new(); + let capacity_notify = Arc::new(Notify::new()); + let shutdown_token = CancellationToken::new(); + let mgr = Self { + user_manager: Arc::clone(user_manager), + provider_manager: Arc::clone(provider_manager), + shared_stream_manager: Arc::clone(shared_stream_manager), + event_manager: Arc::clone(event_manager), + close_socket_signal_tx, + socket_closers: std::sync::Mutex::new(HashMap::new()), + cleanup_sender: BackpressureSender::new(cleanup_tx, "cleanup", cleanup_capacity), + control_cleanup_tx: control_cleanup_tx.clone(), + socket_activity_tracker: socket_activity_tracker.clone(), + capacity_notify: Arc::clone(&capacity_notify), + stream_uid_counter: AtomicU32::new(1), + history_writer: Arc::clone(&history_writer), + is_shutting_down: AtomicBool::new(false), + admission_gate: RwLock::new(()), + shutdown_token: shutdown_token.clone(), + worker_handles: std::sync::Mutex::new(Vec::new()), + }; + + let cleanup_handle = Self::spawn_cleanup_worker( + cleanup_rx, + control_cleanup_rx, + Arc::clone(user_manager), + Arc::clone(provider_manager), + Arc::clone(shared_stream_manager), + Arc::clone(event_manager), + Arc::clone(&capacity_notify), + history_writer, + shutdown_token.clone(), + ); + let socket_handle = Self::spawn_socket_activity_worker( + socket_activity_tracker, + Arc::clone(user_manager), + socket_cleanup_tx, + shutdown_token, + ); + mgr.worker_handles + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .extend([cleanup_handle, socket_handle]); + + mgr + } + + /// Reload the history writer on config change. Shuts down the old writer first so + /// `recover_pending_files` does not collide with an active writer. Recovery runs on the + /// blocking pool instead of a runtime worker so large pending-file sets cannot stall + /// live stream heartbeats. + pub async fn reload_history_writer(&self, config: Option<&StreamHistoryConfig>) { + let old_writer = self.history_writer.swap(None); + if let Some(w) = old_writer { + w.shutdown().await; + } + let new_writer = build_history_writer_async(config).await; + self.history_writer.store(new_writer); + } + + /// Returns a reference to the history writer. + pub fn history_writer(&self) -> &Arc> { &self.history_writer } + + pub async fn release_connection(&self, addr: &SocketAddr) { + release_connection_with_reason(self, addr, DisconnectReason::ClientClosed, true).await; + } + + pub async fn release_connection_with_reason(&self, addr: &SocketAddr, reason: DisconnectReason) { + release_connection_with_reason(self, addr, reason, true).await; + } + + pub async fn release_connection_as_kicked(&self, addr: &SocketAddr) -> bool { + self.release_user_sessions_only(addr).await; + let closed = self.close_connection_with_reason(addr, DisconnectReason::ClientKicked); + if !closed { + self.release_provider_deferred(addr).await; + } + closed + } + + /// Releases the provider connection for `addr` after `tcp_close_notify` is notified. + /// This defers the provider-slot release until after the TCP connection is fully closed, + /// preventing a race where a new request acquires the same provider slot while the old + /// connection is still draining buffered data (which can take 10+ seconds on a live stream). + /// The `close_connection_with_reason` call on the `ConnectionManager` must have already been + /// called before invoking this method. + pub async fn release_provider_deferred(&self, addr: &SocketAddr) { + let addr_owned = *addr; + self.provider_manager.release_connection(&addr_owned); + self.shared_stream_manager.release_connection(&addr_owned, true).await; + notify_capacity(self.capacity_notify.as_ref()); + } + + /// Cleans up user/session/stream state for a forced close without releasing the provider slot. + /// The provider release is deferred to `release_provider_deferred` which waits for the TCP + /// connection to close, preventing a race where a new request acquires the same provider + /// slot while the old connection is still draining buffered data. + /// Note: `release_connection_as_kicked` (called internally) already handles divergence checking. + pub async fn release_user_sessions_only(&self, addr: &SocketAddr) { + let removed = self.user_manager.release_connection_as_kicked(addr).await; + // Mirrors the kicked-specific steps from `release_connection_parts`. + // Provider release and capacity notification are deferred via `release_provider_deferred`. + for stream_info in &removed.removed_streams { + if let Some(session_token) = stream_info.session_token.as_deref() { + self.provider_manager.terminate_identified_playback_owner(session_token); + } + } + for username in &removed.disconnected_users { + self.user_manager.terminate_sessions_for_addr(username, addr).await; + } + for stream_info in &removed.removed_streams { + let qos = self.event_manager.read_meter_qos(stream_info.meter_uid).await; + let bytes_sent = qos.map(|qos| qos.bytes_total); + let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); + self.event_manager.unregister_meter_client(stream_info.uid).await; + emit_disconnect_record( + &self.history_writer, + stream_info, + DisconnectReason::ClientKicked, + &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, + None, + None, + ); + } + if removed.addr_removed && !removed.removed_streams.is_empty() { + self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Disconnected(*addr))); + } + } + + pub async fn release_provider_connection(&self, addr: &SocketAddr) { + self.provider_manager.release_connection(addr); + self.shared_stream_manager.release_connection(addr, false).await; + notify_capacity(self.capacity_notify.as_ref()); + } + + pub async fn release_stream(&self, addr: &SocketAddr) { + if let Some(stream_info) = self.user_manager.release_stream(addr).await { + let qos = self.event_manager.read_meter_qos(stream_info.meter_uid).await; + let bytes_sent = qos.map(|qos| qos.bytes_total); + let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); + self.event_manager.unregister_meter_client(stream_info.uid).await; + emit_disconnect_record( + &self.history_writer, + &stream_info, + DisconnectReason::ClientClosed, + &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, + None, + None, + ); + self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::DisconnectedStream { + addr: stream_info.addr, + uid: stream_info.uid, + })); + notify_capacity(self.capacity_notify.as_ref()); + } + } + + pub fn release_provider_handle(&self, provider_handle: Option) { + if let Some(handle) = provider_handle { + self.provider_manager.release_handle(&handle); + notify_capacity(self.capacity_notify.as_ref()); + } + } + + /// Releases a managed provider owner by value: its drop performs the synchronous + /// slot release, then the capacity waiters are notified. + pub fn release_managed_provider_handle(&self, provider_handle: Option) { + if provider_handle.is_some() { + drop(provider_handle); + notify_capacity(self.capacity_notify.as_ref()); + } + } + + pub fn next_stream_uid(&self) -> u32 { + self.stream_uid_counter + .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + let next = current.wrapping_add(1); + Some(if next == 0 { 1 } else { next }) + }) + .unwrap_or(1) + } + + pub fn record_connect_failed_with_provider_failure( + &self, + info: &StreamInfo, + reason: ConnectFailureReason, + failure_stage: FailureStage, + provider_http_status: Option, + provider_error_class: Option<&str>, + target_name: Option>, + ) { + let guard = self.history_writer.load(); + let Some(writer) = guard.as_ref() else { return }; + let attempt_uid = self.next_stream_uid(); + writer.send_record( + StreamHistoryRecord::from_connect_failed(info, reason, attempt_uid, failure_stage, target_name) + .with_provider_failure(provider_http_status, provider_error_class), + ); + } + + pub fn capacity_notified(&self) -> Arc { Arc::clone(&self.capacity_notify) } + + #[inline] + pub fn is_shutting_down(&self) -> bool { self.is_shutting_down.load(Ordering::Acquire) } + + pub(crate) async fn begin_admission(&self) -> Option> { + if self.is_shutting_down() { + return None; + } + let guard = self.admission_gate.read().await; + (!self.is_shutting_down()).then_some(guard) + } + + /// Emit disconnect records for all still-active streams, unregister shared streams, + /// drain queued cleanups, and flush the history writer. + /// Call once at graceful shutdown before dropping the `ConnectionManager`. + pub async fn shutdown(&self) { + self.is_shutting_down.store(true, Ordering::Release); + let _admission_closed = self.admission_gate.write().await; + self.shared_stream_manager.shutdown().await; + let active_streams = self.user_manager.drain_for_shutdown().await; + for stream_info in &active_streams { + let qos = self.event_manager.read_meter_qos(stream_info.meter_uid).await; + let bytes_sent = qos.map(|qos| qos.bytes_total); + let first_byte_latency_ms = qos.and_then(|qos| qos.first_byte_latency_ms); + emit_disconnect_record( + &self.history_writer, + stream_info, + DisconnectReason::Shutdown, + &DisconnectQos { bytes_sent, first_byte_latency_ms, ..Default::default() }, + None, + None, + ); + self.event_manager.unregister_meter_client(stream_info.uid).await; + } + // This terminal transition does not depend on queue capacity or body-held + // cleanup permits. Later guard drops are idempotent against the emptied indices. + self.provider_manager.shutdown(); + if let Some(w) = self.history_writer.load_full() { + w.shutdown().await; + } + // Stop and join the cleanup and socket workers so no background task outlives + // the manager. Active claims were released directly above; later guard drops are + // idempotent against the emptied indices. + self.shutdown_token.cancel(); + let handles = + std::mem::take(&mut *self.worker_handles.lock().unwrap_or_else(std::sync::PoisonError::into_inner)); + for handle in handles { + let _ = handle.await; + } + } +} diff --git a/backend/session/src/connection_manager/registration.rs b/backend/session/src/connection_manager/registration.rs new file mode 100644 index 000000000..2a912d907 --- /dev/null +++ b/backend/session/src/connection_manager/registration.rs @@ -0,0 +1,199 @@ +use super::{ + history::emit_connect_record, socket::SocketActivityEvent, ConnectionHistoryMode, ConnectionManager, + ConnectionParams, ConnectionRejectionReason, OwnedRequestCleanup, RegisteredPlaybackRequest, + SharedCleanupCapability, CLEANUP_ADMISSION_TIMEOUT, +}; +use crate::{uses_direct_body_idle_timeout, ActiveUserConnectionParams}; +use log::warn; +use shared::{ + model::{ActiveUserConnectionChange, CustomVideoStreamType, EventMessage, StreamInfo}, + utils::sanitize_sensitive_info, +}; +use std::{net::SocketAddr, sync::Arc}; +use tuliprox_core::utils::debug_if_enabled; + +impl ConnectionManager { + pub async fn add_connection(&self, addr: &SocketAddr) { self.user_manager.add_connection(addr).await; } + + pub async fn touch_http_activity(&self, username: &str, token: &str, addr: &SocketAddr) { + self.user_manager.touch_http_activity(username, token, addr).await; + self.socket_activity_tracker.track(SocketActivityEvent::HttpActivity { addr: *addr }); + } + + pub fn touch_direct_body_activity(&self, addr: &SocketAddr) { + self.socket_activity_tracker.track(SocketActivityEvent::DirectBodyActivity { addr: *addr }); + } + + pub async fn update_connection(&self, update: ConnectionParams<'_>) -> Option { + self.update_connection_with_history_mode(update, ConnectionHistoryMode::EmitConnect).await + } + + pub async fn update_connection_with_history_mode( + &self, + update: ConnectionParams<'_>, + history_mode: ConnectionHistoryMode, + ) -> Option { + let uid = self.next_stream_uid(); + let mut registered = self.update_connection_with_uid(update, history_mode, uid, None).await; + // This compatibility API has no body to hand the claim to; the caller owns the + // registered stream, so disarm the rollback before returning its metadata. + registered.commit(); + registered.into_display_stream() + } + + pub async fn update_connection_with_uid( + &self, + update: ConnectionParams<'_>, + history_mode: ConnectionHistoryMode, + uid: u32, + provider_request_id: Option, + ) -> RegisteredPlaybackRequest { + self.update_connection_with_uid_and_session_registration(update, history_mode, uid, provider_request_id, None) + .await + } + + pub async fn update_connection_with_uid_and_session_registration( + &self, + update: ConnectionParams<'_>, + history_mode: ConnectionHistoryMode, + uid: u32, + provider_request_id: Option, + session_registration: Option<&crate::PlaybackSessionRegistration>, + ) -> RegisteredPlaybackRequest { + self.update_connection_with_uid_impl( + update, + history_mode, + uid, + provider_request_id, + false, + session_registration, + ) + .await + } + + /// Registers a request whose enclosing shared-subscriber body already owns the + /// guaranteed terminal cleanup permit for this exact stream UID. + pub async fn update_connection_with_uid_using_shared_cleanup( + &self, + update: ConnectionParams<'_>, + history_mode: ConnectionHistoryMode, + capability: SharedCleanupCapability, + provider_request_id: Option, + ) -> RegisteredPlaybackRequest { + self.update_connection_with_uid_impl( + update, + history_mode, + capability.stream_uid(), + provider_request_id, + true, + None, + ) + .await + } + + pub(super) async fn update_connection_with_uid_impl( + &self, + update: ConnectionParams<'_>, + history_mode: ConnectionHistoryMode, + uid: u32, + provider_request_id: Option, + cleanup_owned_by_shared_subscriber: bool, + session_registration: Option<&crate::PlaybackSessionRegistration>, + ) -> RegisteredPlaybackRequest { + let username = update.username; + let fingerprint = update.fingerprint; + let track_direct_body_activity = uses_direct_body_idle_timeout(update.stream_channel); + let Some(_admission) = self.begin_admission().await else { + warn!("Connection manager is shutting down; rejecting connection registration for user {username}"); + return RegisteredPlaybackRequest::rejected(uid, ConnectionRejectionReason::CleanupReceiverClosed); + }; + // Admission: reserve a guaranteed cleanup right before any registration mutation, + // bounded so a saturated or stalled cleanup queue cannot hang request setup. + let rollback_permit = if cleanup_owned_by_shared_subscriber { + None + } else { + match tokio::time::timeout(CLEANUP_ADMISSION_TIMEOUT, self.cleanup_tx().reserve_owned()).await { + Ok(Ok(permit)) => Some(permit), + Ok(Err(_)) => { + warn!("Cleanup receiver closed; rejecting connection registration for user {username}"); + return RegisteredPlaybackRequest::rejected(uid, ConnectionRejectionReason::CleanupReceiverClosed); + } + Err(_) => { + warn!("Cleanup admission timed out; rejecting connection registration for user {username}"); + return RegisteredPlaybackRequest::rejected( + uid, + ConnectionRejectionReason::CleanupAdmissionTimeout, + ); + } + } + }; + // The cleanup owner is armed before the first mutation, so a cancellation during + // `update_connection`, meter registration or any later await still releases the claim. + let mut cleanup = OwnedRequestCleanup { + addr: fingerprint.addr, + request_uid: uid, + provider_request_id, + owner: update.session_token.map(Arc::::from), + permit: rollback_permit, + finished: false, + }; + if let Some(stream_info) = self + .user_manager + .update_connection_with_session_registration( + ActiveUserConnectionParams { + uid, + meter_uid: update.meter_uid, + username, + max_connections: update.max_connections, + soft_connections: update.soft_connections, + connection_kind: update.connection_kind, + priority: update.priority, + soft_priority: update.soft_priority, + fingerprint, + provider: update.provider, + stream_channel: update.stream_channel, + user_agent: update.user_agent, + session_token: update.session_token, + }, + session_registration, + ) + .await + { + self.event_manager.register_meter_client(stream_info.uid, stream_info.meter_uid).await; + if history_mode == ConnectionHistoryMode::EmitConnect { + emit_connect_record(&self.history_writer, &stream_info); + } + if track_direct_body_activity { + debug_if_enabled!( + "Direct body stream registered for socket expiry: {}", + sanitize_sensitive_info(&fingerprint.addr.to_string()) + ); + self.touch_direct_body_activity(&fingerprint.addr); + } + self.event_manager + .send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info.clone()))); + RegisteredPlaybackRequest::with_cleanup(uid, Some(stream_info), Some(cleanup)) + } else { + // Registration failed: nothing was inserted, so the cleanup must not fire. + cleanup.disarm(); + warn!("Failed to register connection for user {username} at {}; disconnecting client", fingerprint.addr); + RegisteredPlaybackRequest::rejected(uid, ConnectionRejectionReason::RegistrationFailed) + } + } + + // pub fn send_active_user_stats(&self, user_count: usize, user_connection_count: usize) { + // self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Connections(user_count, user_connection_count))); + // } + + pub async fn update_stream_detail(&self, addr: &SocketAddr, video_type: CustomVideoStreamType) { + if let Some(stream_info) = self.user_manager.update_stream_detail(addr, video_type).await { + self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + } + } + + pub async fn update_stream_detail_by_uid(&self, uid: u32, video_type: CustomVideoStreamType) { + if let Some(stream_info) = self.user_manager.update_stream_detail_by_uid(uid, video_type).await { + self.event_manager.send_event(EventMessage::ActiveUser(ActiveUserConnectionChange::Updated(stream_info))); + } + } +} diff --git a/backend/session/src/connection_manager/requests.rs b/backend/session/src/connection_manager/requests.rs new file mode 100644 index 000000000..9e169a657 --- /dev/null +++ b/backend/session/src/connection_manager/requests.rs @@ -0,0 +1,147 @@ +use super::{CleanupEvent, OwnedRequestCleanup, RegisteredPlaybackRequest, PROVIDER_END_NOT_SET}; +use shared::model::{StreamChannel, StreamInfo}; +use std::{borrow::Cow, sync::Arc}; +use tuliprox_core::model::Fingerprint; + +pub struct ConnectionParams<'a> { + pub meter_uid: u32, + pub username: &'a str, + pub max_connections: u32, + pub soft_connections: u16, + pub connection_kind: crate::active_provider_manager::ConnectionKind, + pub priority: i8, + pub soft_priority: i8, + pub fingerprint: &'a Fingerprint, + pub provider: Arc, + pub stream_channel: &'a StreamChannel, + pub user_agent: Cow<'a, str>, + pub session_token: Option<&'a str>, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ConnectionHistoryMode { + EmitConnect, + RefreshOnly, +} + +impl OwnedRequestCleanup { + /// Releases the reserved cleanup right without emitting any release. + pub(super) fn disarm(&mut self) { + self.finished = true; + self.permit = None; + } + + /// Releases the request claim exactly once with the actual provider outcome. + #[allow(clippy::too_many_arguments)] + pub fn finish( + &mut self, + request_id: Option, + provider_end_reason: u8, + reconnect_count: u8, + provider_error_class: Option<&'static str>, + provider_http_status: Option, + ) { + if self.finished { + return; + } + self.finished = true; + let Some(permit) = self.permit.take() else { + return; + }; + permit.send(CleanupEvent::ReleaseStream { + request_id, + owner: self.owner.clone(), + addr: self.addr, + stream_uid: Some(self.request_uid), + provider_end_reason, + reconnect_count, + provider_error_class, + provider_http_status, + }); + } +} + +impl Drop for OwnedRequestCleanup { + fn drop(&mut self) { + // Conservative fallback for a body that is dropped without an explicit finish: + // never invent a successful EOF, but still finish the provider request with its + // captured identity so an abandoned start does not linger until the startup TTL. + self.finish(self.provider_request_id, PROVIDER_END_NOT_SET, 0, None, None); + } +} + +/// Why a playback request was rejected before a body could be created. Carried on +/// `RegisteredPlaybackRequest` so the caller can surface a non-success HTTP response +/// instead of silently dropping the request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConnectionRejectionReason { + CleanupReceiverClosed, + CleanupAdmissionTimeout, + RegistrationFailed, +} + +impl std::fmt::Display for ConnectionRejectionReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::CleanupReceiverClosed => write!(f, "cleanup receiver closed"), + Self::CleanupAdmissionTimeout => write!(f, "cleanup admission timed out"), + Self::RegistrationFailed => write!(f, "connection registration failed"), + } + } +} + +impl std::error::Error for ConnectionRejectionReason {} + +impl std::fmt::Debug for RegisteredPlaybackRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RegisteredPlaybackRequest") + .field("request_uid", &self.request_uid) + .field("display_stream", &self.display_stream) + .field("rejection", &self.rejection) + .finish_non_exhaustive() + } +} + +impl RegisteredPlaybackRequest { + #[inline] + pub fn new(request_uid: u32, display_stream: Option) -> Self { + Self { request_uid, display_stream, cleanup: None, rejection: None } + } + + #[inline] + pub(super) fn rejected(request_uid: u32, reason: ConnectionRejectionReason) -> Self { + Self { request_uid, display_stream: None, cleanup: None, rejection: Some(reason) } + } + + #[inline] + pub(super) fn with_cleanup( + request_uid: u32, + display_stream: Option, + cleanup: Option, + ) -> Self { + Self { request_uid, display_stream, cleanup, rejection: None } + } + + /// Marks the request as taken over by a path without a body (compatibility API or + /// shared source), disabling rollback on drop. + #[inline] + pub fn commit(&mut self) { + if let Some(mut cleanup) = self.cleanup.take() { + cleanup.disarm(); + } + } + + /// Transfers the guaranteed cleanup right into the body. The body owns the returned + /// value and must finish it exactly once with the real provider outcome. + #[inline] + pub fn into_body_cleanup(&mut self) -> Option { self.cleanup.take() } + + #[inline] + pub fn display_uid(&self) -> Option { self.display_stream.as_ref().map(|s| s.uid) } + + #[inline] + pub fn rejection_reason(&self) -> Option { self.rejection } + + #[inline] + pub fn into_display_stream(self) -> Option { self.display_stream } +} diff --git a/backend/session/src/connection_manager/socket.rs b/backend/session/src/connection_manager/socket.rs new file mode 100644 index 000000000..6af72bf01 --- /dev/null +++ b/backend/session/src/connection_manager/socket.rs @@ -0,0 +1,269 @@ +use super::{ + CleanupEvent, ConnectionManager, SocketActivityTracker, SocketExpiryEntry, SOCKET_EXPIRY_QUEUE_REBUILD_FACTOR, + SOCKET_EXPIRY_QUEUE_REBUILD_MIN_STALE, +}; +use crate::ActiveUserManager; +use log::debug; +use shared::{ + model::{DisconnectReason, VirtualId}, + utils::sanitize_sensitive_info, +}; +use std::{ + cmp::Reverse, + collections::{BinaryHeap, HashMap}, + net::SocketAddr, + sync::Arc, + time::Duration, +}; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; +use tuliprox_core::utils::debug_if_enabled; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub(super) enum SocketActivityEvent { + HttpActivity { addr: SocketAddr }, + DirectBodyActivity { addr: SocketAddr }, +} + +impl SocketActivityEvent { + pub(super) fn addr(&self) -> SocketAddr { + match self { + Self::HttpActivity { addr, .. } | Self::DirectBodyActivity { addr, .. } => *addr, + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum CloseConnectionSignal { + WithReason(SocketAddr, DisconnectReason), +} + +#[derive(Debug)] +pub(super) enum SocketCloseState { + Open(tokio::sync::oneshot::Sender), + Closing, +} + +impl ConnectionManager { + pub(super) fn spawn_socket_activity_worker( + activity_tracker: SocketActivityTracker, + user_manager: Arc, + cleanup_tx: mpsc::Sender, + shutdown_token: CancellationToken, + ) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut expiry_queue: BinaryHeap> = BinaryHeap::new(); + let mut expiry_index: HashMap = HashMap::new(); + + loop { + Self::drain_pending_socket_activity( + &activity_tracker, + &mut expiry_queue, + &mut expiry_index, + &user_manager, + ) + .await; + + let next_expiry = expiry_queue.peek().map(|entry| entry.0.expires_at); + if let Some(expires_at) = next_expiry { + let now = shared::utils::current_time_secs(); + if expires_at <= now { + Self::process_due_socket_expiry_entries( + &mut expiry_queue, + &mut expiry_index, + now, + &user_manager, + &cleanup_tx, + ) + .await; + continue; + } + + tokio::select! { + biased; + () = shutdown_token.cancelled() => break, + () = activity_tracker.notified() => {} + () = tokio::time::sleep(Duration::from_secs(expires_at.saturating_sub(now))) => {} + } + } else { + tokio::select! { + () = shutdown_token.cancelled() => break, + () = activity_tracker.notified() => {} + } + } + } + }) + } + + pub(super) async fn drain_pending_socket_activity( + activity_tracker: &SocketActivityTracker, + expiry_queue: &mut BinaryHeap>, + expiry_index: &mut HashMap, + user_manager: &Arc, + ) { + for event in activity_tracker.drain() { + Self::handle_socket_activity_event(event, expiry_queue, expiry_index, user_manager).await; + } + } + + pub(super) async fn handle_socket_activity_event( + event: SocketActivityEvent, + expiry_queue: &mut BinaryHeap>, + expiry_index: &mut HashMap, + user_manager: &Arc, + ) { + let addr = event.addr(); + if matches!(event, SocketActivityEvent::DirectBodyActivity { .. }) { + user_manager.touch_socket_activity(&addr).await; + } + + if let Some(expires_at) = user_manager.socket_expiry_deadline(&addr).await { + let current = expiry_index.insert(addr, expires_at); + if current != Some(expires_at) { + expiry_queue.push(Reverse(SocketExpiryEntry { expires_at, addr })); + Self::maybe_rebuild_socket_expiry_queue(expiry_queue, expiry_index); + } + } + } + + pub(super) async fn process_due_socket_expiry_entries( + expiry_queue: &mut BinaryHeap>, + expiry_index: &mut HashMap, + now: u64, + user_manager: &Arc, + cleanup_tx: &mpsc::Sender, + ) { + while let Some(entry) = expiry_queue.peek().copied() { + if entry.0.expires_at > now { + break; + } + + let Reverse(SocketExpiryEntry { expires_at, addr }) = expiry_queue.pop().unwrap_or(entry); + let Some(current_expires_at) = expiry_index.get(&addr).copied() else { + continue; + }; + if current_expires_at != expires_at { + continue; + } + + if let Some(next_expires_at) = user_manager.socket_expiry_deadline(&addr).await { + if next_expires_at > now { + expiry_index.insert(addr, next_expires_at); + expiry_queue.push(Reverse(SocketExpiryEntry { expires_at: next_expires_at, addr })); + Self::maybe_rebuild_socket_expiry_queue(expiry_queue, expiry_index); + continue; + } + } + + // Fallthrough to release connection if `None` or `< now` + + expiry_index.remove(&addr); + debug_if_enabled!( + "Socket activity deadline expired for {}, releasing connection", + sanitize_sensitive_info(&addr.to_string()) + ); + if cleanup_tx.send(CleanupEvent::ReleaseConnection { addr }).await.is_err() { + debug!("Cleanup channel closed, stopping socket expiry worker"); + break; + } + } + } + + pub(super) fn maybe_rebuild_socket_expiry_queue( + expiry_queue: &mut BinaryHeap>, + expiry_index: &HashMap, + ) { + let indexed_len = expiry_index.len(); + if indexed_len == 0 { + expiry_queue.clear(); + return; + } + + let stale_entries = expiry_queue.len().saturating_sub(indexed_len); + if expiry_queue.len() <= indexed_len.saturating_mul(SOCKET_EXPIRY_QUEUE_REBUILD_FACTOR) + || stale_entries < SOCKET_EXPIRY_QUEUE_REBUILD_MIN_STALE + { + return; + } + + *expiry_queue = expiry_index + .iter() + .map(|(addr, expires_at)| Reverse(SocketExpiryEntry { expires_at: *expires_at, addr: *addr })) + .collect(); + } + + pub fn get_close_connection_channel(&self) -> tokio::sync::broadcast::Receiver { + self.close_socket_signal_tx.subscribe() + } + + pub async fn kick_connection(&self, addr: &SocketAddr, virtual_id: VirtualId, block_secs: u64) -> bool { + debug_if_enabled!( + "User {} kicked for stream with virtual_id {virtual_id} for {block_secs} seconds with addr {}.", + self.user_manager.get_username_for_addr(addr).await.unwrap_or_default(), + sanitize_sensitive_info(&addr.to_string()) + ); + if block_secs > 0 { + self.user_manager.block_user_for_stream(addr, virtual_id, block_secs).await; + } + self.release_connection_as_kicked(addr).await + } + + pub fn register_close_socket(&self, addr: SocketAddr) -> tokio::sync::oneshot::Receiver { + let (tx, rx) = tokio::sync::oneshot::channel(); + if let Ok(mut closers) = self.socket_closers.lock() { + closers.insert(addr, SocketCloseState::Open(tx)); + } + rx + } + + pub fn unregister_close_socket(&self, addr: &SocketAddr) { + if let Ok(mut closers) = self.socket_closers.lock() { + closers.remove(addr); + } + } + + pub async fn close_connection_with_reason_and_block( + &self, + addr: &SocketAddr, + virtual_id: VirtualId, + block_secs: u64, + reason: DisconnectReason, + ) -> bool { + if block_secs > 0 { + self.user_manager.block_user_for_stream(addr, virtual_id, block_secs).await; + } + self.close_connection_with_reason(addr, reason) + } + + pub fn close_connection_signal(&self, addr: &SocketAddr) -> bool { + self.close_connection_with_reason(addr, DisconnectReason::ClientClosed) + } + + pub async fn block_stream_by_uid(&self, uid: u32, virtual_id: VirtualId, block_secs: u64) { + self.user_manager.block_user_for_stream_uid(uid, virtual_id, block_secs).await; + } + + pub fn close_connection_with_reason(&self, addr: &SocketAddr, reason: DisconnectReason) -> bool { + let target_sent = if let Ok(mut closers) = self.socket_closers.lock() { + match closers.remove(addr) { + Some(SocketCloseState::Open(tx)) => { + if tx.send(reason).is_ok() { + closers.insert(*addr, SocketCloseState::Closing); + true + } else { + false + } + } + Some(SocketCloseState::Closing) => { + closers.insert(*addr, SocketCloseState::Closing); + true + } + None => false, + } + } else { + false + }; + let _ = self.close_socket_signal_tx.send(CloseConnectionSignal::WithReason(*addr, reason)); + target_sent + } +} diff --git a/backend/session/src/connection_manager/tests.rs b/backend/session/src/connection_manager/tests.rs new file mode 100644 index 000000000..dc8e97da3 --- /dev/null +++ b/backend/session/src/connection_manager/tests.rs @@ -0,0 +1,140 @@ +use super::{ + history::{playback_outcome_for_reason, resolve_disconnect_reason, resolve_disconnect_reason_from_provider_end}, + *, +}; +use crate::{ + ActiveProviderManager, ActiveUserConnectionParams, ActiveUserManager, CreateUserSessionParams, EventManager, + SharedStreamManager, +}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use shared::{ + model::{ + ConfigPaths, CustomVideoStreamType, DisconnectReason, FailureStage, InputFetchMethod, InputType, + PlaylistItemType, ProxyType, StreamChannel, StreamInfo, UserConnectionPermission, XtreamCluster, + }, + utils::Internable, +}; +use std::{ + collections::HashMap, + net::SocketAddr, + sync::{atomic::Ordering, Arc}, +}; +use tokio::sync::mpsc; +use tuliprox_core::{ + model::{ + AppConfig, Config, ConfigInput, DisconnectQos, MediaToolCapabilities, PlaybackRequestOutcome, + ProxyUserCredentials, SourcesConfig, + }, + utils::FileLockManager, +}; +use tuliprox_repository::GeoIp; + +fn make_stream_info(provider: &str, title: &str) -> StreamInfo { + let addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); + let channel = StreamChannel { + target_id: 1, + virtual_id: 1, + provider_id: 1, + input_name: "input".intern(), + item_type: PlaylistItemType::Live, + cluster: XtreamCluster::Live, + group: "".intern(), + title: title.intern(), + url: "".intern(), + shared: false, + shared_joined_existing: None, + shared_stream_id: None, + technical: None, + epg_channel_id: None, + epg_reference_ts: None, + upstream_user_agent: None, + }; + StreamInfo::new(shared::model::StreamInfoParams { + uid: 0, + meter_uid: 0, + username: "test", + addr: &addr, + client_ip: "127.0.0.1", + provider: provider.intern(), + stream_channel: channel, + user_agent: String::new(), + country_code: None, + session_token: None, + }) +} + +fn create_test_app_config() -> AppConfig { + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: "http://provider-1.example".to_string(), + username: Some("user1".to_string()), + password: Some("pass1".to_string()), + enabled: true, + priority: 0, + max_connections: 1, + method: InputFetchMethod::default(), + aliases: None, + ..ConfigInput::default() + }); + let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; + + AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(sources)), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + } +} + +fn create_test_connection_manager() -> Arc { + let app_cfg = create_test_app_config(); + let event_manager = Arc::new(EventManager::new()); + let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let shared_manager = Arc::new(SharedStreamManager::new(Arc::clone(&provider_manager))); + provider_manager.set_shared_stream_manager(&shared_manager); + + let geo_ip = Arc::new(ArcSwapOption::::default()); + let config = app_cfg.config.load(); + let user_manager = Arc::new(ActiveUserManager::new(&config, &geo_ip, &event_manager)); + + Arc::new(ConnectionManager::new(&user_manager, &provider_manager, &shared_manager, &event_manager, None)) +} + +fn create_test_proxy_user(username: &str) -> ProxyUserCredentials { + let mut user = ProxyUserCredentials::default(); + user.username = username.to_string(); + user.password = "password".to_string(); + user.proxy = ProxyType::Reverse(None); + user.max_connections = 1; + user +} + +mod admission; +mod http; +mod lifecycle; +mod playlist; +mod policy; +mod streaming; +mod terminal; +mod transport; diff --git a/backend/session/src/connection_manager/tests/admission.rs b/backend/session/src/connection_manager/tests/admission.rs new file mode 100644 index 000000000..bdaf0463e --- /dev/null +++ b/backend/session/src/connection_manager/tests/admission.rs @@ -0,0 +1,145 @@ +use super::*; + +/// A saturated cleanup queue must reject registration with an admission-timeout reason +/// (rather than hanging), so the caller can surface a defined non-success response. +#[tokio::test(start_paused = true)] +async fn cleanup_admission_timeout_rejects_registration_with_reason() { + let manager = create_test_connection_manager(); + // Stall the cleanup worker on a never-resolving defer, then fill the channel so + // `reserve_owned()` blocks and the bounded admission deadline fires. + let pending = || CleanupEvent::Defer(Box::pin(std::future::pending::<()>())); + manager.send_cleanup(pending()); + for _ in 0..CLEANUP_QUEUE_CAPACITY { + manager.send_cleanup(pending()); + } + + let addr: SocketAddr = "127.0.0.1:56242".parse().unwrap(); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let channel = StreamChannel { + virtual_id: 414, + title: "channel-414".intern(), + ..make_stream_info("provider_1", "channel-414").channel + }; + + manager.add_connection(&addr).await; + let manager_for_task = Arc::clone(&manager); + let registration = tokio::spawn(async move { + manager_for_task + .update_connection_with_uid( + ConnectionParams { + meter_uid: 0, + username: "admission-timeout-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: None, + }, + ConnectionHistoryMode::EmitConnect, + 1, + None, + ) + .await + }); + + // Let the task reach the blocked cleanup reservation, then fire the deadline. + tokio::task::yield_now().await; + tokio::task::yield_now().await; + tokio::time::advance(CLEANUP_ADMISSION_TIMEOUT + Duration::from_secs(1)).await; + let registered = registration.await.expect("registration task must not panic"); + + assert!(registered.display_stream.is_none(), "saturated cleanup admission must be rejected"); + assert_eq!(registered.rejection_reason(), Some(ConnectionRejectionReason::CleanupAdmissionTimeout)); +} + +/// A joined shared body already holds the terminal cleanup permit. Registration must +/// reuse that ownership instead of waiting for a second permit from the same full queue. +#[tokio::test] +async fn shared_admission_does_not_wait_for_second_permit_it_prevents() { + let manager = create_test_connection_manager(); + let cleanup_tx = manager.cleanup_tx(); + let available = cleanup_tx.capacity(); + let mut permits = Vec::with_capacity(available); + for _ in 0..available { + permits.push(cleanup_tx.clone().try_reserve_owned().expect("cleanup permit")); + } + + let addr: SocketAddr = "127.0.0.1:56243".parse().expect("test socket"); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let channel = StreamChannel { + virtual_id: 415, + title: "channel-415".intern(), + shared: true, + shared_joined_existing: Some(true), + ..make_stream_info("provider_1", "channel-415").channel + }; + + manager.add_connection(&addr).await; + let mut registered = tokio::time::timeout( + Duration::from_millis(100), + manager.update_connection_with_uid_using_shared_cleanup( + ConnectionParams { + meter_uid: 0, + username: "shared-admission-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: None, + }, + ConnectionHistoryMode::EmitConnect, + SharedCleanupCapability::new(tuliprox_core::model::SharedSubscriberId::from_stream_uid(1)), + None, + ), + ) + .await + .expect("externally owned cleanup must not reserve another permit"); + + assert!(registered.display_stream.is_some(), "shared registration must succeed"); + assert!(registered.into_body_cleanup().is_some(), "the body still tracks its terminal outcome"); + let _ = manager.user_manager.release_stream_request_by_uid(&addr, 1).await; + drop(permits); +} + +#[tokio::test] +async fn new_with_capacity_sizes_the_cleanup_queue() { + let app_cfg = create_test_app_config(); + let event_manager = Arc::new(EventManager::new()); + let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let shared_manager = Arc::new(SharedStreamManager::new(Arc::clone(&provider_manager))); + provider_manager.set_shared_stream_manager(&shared_manager); + let geo_ip = Arc::new(ArcSwapOption::::default()); + let config = app_cfg.config.load(); + let user_manager = Arc::new(ActiveUserManager::new(&config, &geo_ip, &event_manager)); + + let manager = Arc::new(ConnectionManager::new_with_capacity( + &user_manager, + &provider_manager, + &shared_manager, + &event_manager, + None, + 2, + )); + assert_eq!(manager.cleanup_tx().capacity(), 2); + + // A capacity of zero is clamped to one so the cleanup queue is always usable. + let manager = Arc::new(ConnectionManager::new_with_capacity( + &user_manager, + &provider_manager, + &shared_manager, + &event_manager, + None, + 0, + )); + assert_eq!(manager.cleanup_tx().capacity(), 1); +} diff --git a/backend/session/src/connection_manager/tests/http.rs b/backend/session/src/connection_manager/tests/http.rs new file mode 100644 index 000000000..b73f86fac --- /dev/null +++ b/backend/session/src/connection_manager/tests/http.rs @@ -0,0 +1,39 @@ +use super::*; + +#[tokio::test] +async fn touch_http_activity_is_processed_by_socket_activity_worker() { + let manager = create_test_connection_manager(); + let addr: SocketAddr = "127.0.0.1:3234".parse().unwrap_or_else(|_| unreachable!()); + let user = create_test_proxy_user("user1"); + + manager.add_connection(&addr).await; + let _ = manager + .user_manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-touch", + virtual_id: 1, + provider: "provider_1", + stream_url: "http://provider-1.example/live.ts", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::ConnectionKind::Normal), + socket_bound: true, + }) + .await; + + manager.touch_http_activity(&user.username, "tok-touch", &addr).await; + + assert!(tokio::time::timeout(Duration::from_secs(1), async { + loop { + if manager.user_manager.socket_expiry_deadline(&addr).await.is_some() + && manager.user_manager.get_username_for_addr(&addr).await.as_deref() == Some(user.username.as_str()) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .is_ok()); +} diff --git a/backend/session/src/connection_manager/tests/lifecycle.rs b/backend/session/src/connection_manager/tests/lifecycle.rs new file mode 100644 index 000000000..90c18955f --- /dev/null +++ b/backend/session/src/connection_manager/tests/lifecycle.rs @@ -0,0 +1,643 @@ +use super::*; + +#[tokio::test] +async fn close_connection_with_reason_returns_false_for_dropped_oneshot_receiver() { + let manager = create_test_connection_manager(); + let addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); + let close_rx = manager.register_close_socket(addr); + drop(close_rx); + + assert!( + !manager.close_connection_with_reason(&addr, DisconnectReason::ClientKicked), + "must return false when target oneshot receiver was dropped" + ); +} + +#[tokio::test] +async fn release_connection_as_kicked_cleans_provider_when_unreceived() { + let manager = create_test_connection_manager(); + let addr: SocketAddr = "127.0.0.1:2236".parse().unwrap_or_else(|_| unreachable!()); + let input_name = "provider_1".intern(); + let handle = manager + .provider_manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + 0, + crate::ConnectionKind::Normal, + Some("test-owner"), + ) + .expect("acquire provider"); + assert_eq!(manager.provider_manager.get_provider_connections_count(), 1); + + let closed = manager.release_connection_as_kicked(&addr).await; + assert!(!closed, "should return false when no receiver is subscribed"); + assert_eq!(manager.provider_manager.get_provider_connections_count(), 0); + + drop(handle); +} + +#[tokio::test] +async fn release_connection_as_kicked_does_not_release_provider_prematurely_on_repeated_kick() { + let manager = create_test_connection_manager(); + let addr: SocketAddr = "127.0.0.1:2237".parse().unwrap_or_else(|_| unreachable!()); + let input_name = "provider_1".intern(); + let handle = manager + .provider_manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + 0, + crate::ConnectionKind::Normal, + Some("test-owner"), + ) + .expect("acquire provider"); + assert_eq!(manager.provider_manager.get_provider_connections_count(), 1); + + let mut close_rx = manager.register_close_socket(addr); + + // First kick sends close signal to socket and transitions to Closing + let closed_first = manager.release_connection_as_kicked(&addr).await; + assert!(closed_first, "first kick must report success when socket is registered"); + assert_eq!( + manager.provider_manager.get_provider_connections_count(), + 1, + "provider must remain allocated after first kick while socket is in Closing state" + ); + + // Second kick while socket is still Closing must not prematurely release provider + let closed_second = manager.release_connection_as_kicked(&addr).await; + assert!(closed_second, "second kick during Closing state must report success"); + assert_eq!( + manager.provider_manager.get_provider_connections_count(), + 1, + "provider must not be released prematurely by second kick before socket finishes" + ); + + // Now simulate the socket processing the signal and finishing its cleanup + let reason = close_rx.try_recv().expect("close signal must be pending in oneshot"); + assert_eq!(reason, DisconnectReason::ClientKicked); + manager.release_provider_deferred(&addr).await; + manager.unregister_close_socket(&addr); + assert_eq!( + manager.provider_manager.get_provider_connections_count(), + 0, + "provider must be released after socket finishes and unregisters" + ); + + drop(handle); +} + +#[tokio::test] +async fn release_connection_as_kicked_sends_kick_close_signal() { + let manager = create_test_connection_manager(); + let mut rx = manager.get_close_connection_channel(); + let addr: SocketAddr = "127.0.0.1:2234".parse().unwrap_or_else(|_| unreachable!()); + + manager.release_connection_as_kicked(&addr).await; + assert_eq!( + tokio::time::timeout(Duration::from_millis(100), rx.recv()).await.ok().and_then(Result::ok), + Some(CloseConnectionSignal::WithReason(addr, DisconnectReason::ClientKicked)) + ); +} + +#[tokio::test] +async fn release_connection_as_kicked_is_idempotent() { + let manager = create_test_connection_manager(); + let mut rx = manager.get_close_connection_channel(); + let addr: SocketAddr = "127.0.0.1:2235".parse().unwrap_or_else(|_| unreachable!()); + + manager.add_connection(&addr).await; + manager.release_connection_as_kicked(&addr).await; + assert_eq!( + tokio::time::timeout(Duration::from_millis(100), rx.recv()).await.ok().and_then(Result::ok), + Some(CloseConnectionSignal::WithReason(addr, DisconnectReason::ClientKicked)) + ); + + manager.release_connection_as_kicked(&addr).await; +} + +#[tokio::test] +async fn low_priority_preempted_cleanup_blocks_same_user_stream_reentry() { + let manager = create_test_connection_manager(); + let user = create_test_proxy_user("preempted-user"); + let addr: SocketAddr = "127.0.0.1:6234".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let channel = StreamChannel { + virtual_id: 409, + title: "channel-409".intern(), + ..make_stream_info("provider_1", "channel-409").channel + }; + + manager.add_connection(&addr).await; + manager + .user_manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-preempted", + virtual_id: channel.virtual_id, + provider: "provider_1", + stream_url: "http://provider-1.example/live/409.ts", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::ConnectionKind::Normal), + socket_bound: true, + }) + .await; + manager + .user_manager + .update_connection(ActiveUserConnectionParams { + uid: 9001, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: crate::ConnectionKind::Normal, + priority: 9, + soft_priority: 9, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("tok-preempted"), + }) + .await; + + let deps = CleanupWorkerDeps { + user_manager: Arc::clone(&manager.user_manager), + provider_manager: Arc::clone(&manager.provider_manager), + shared_stream_manager: Arc::clone(&manager.shared_stream_manager), + event_manager: Arc::clone(&manager.event_manager), + capacity_notify: Arc::clone(&manager.capacity_notify), + history_writer: Arc::clone(&manager.history_writer), + }; + + handle_update_detail_and_release_provider(&deps, addr, CustomVideoStreamType::LowPriorityPreempted, None, None) + .await; + + assert!( + manager + .user_manager + .is_user_blocked_for_stream(&user.username, shared::model::VirtualId::new(channel.virtual_id)) + .await, + "preempted playback should be blocked briefly to prevent immediate reconnect ping-pong" + ); +} + +#[tokio::test] +async fn low_priority_preempted_cleanup_blocks_same_user_hls_reentry() { + let manager = create_test_connection_manager(); + let user = create_test_proxy_user("preempted-hls-user"); + let addr: SocketAddr = "127.0.0.1:6235".parse().unwrap_or_else(|_| unreachable!()); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let mut channel = make_stream_info("provider_1", "channel-410").channel; + channel.virtual_id = 410; + channel.item_type = PlaylistItemType::LiveHls; + channel.title = "channel-410.m3u8".intern(); + channel.url = "http://provider-1.example/live/410.m3u8".intern(); + + manager.add_connection(&addr).await; + manager + .user_manager + .create_user_session(CreateUserSessionParams { + user: &user, + session_token: "tok-preempted-hls", + virtual_id: channel.virtual_id, + provider: "provider_1", + stream_url: "http://provider-1.example/live/410.m3u8", + addr: &addr, + connection_permission: UserConnectionPermission::Allowed, + connection_kind: Some(crate::ConnectionKind::Normal), + socket_bound: false, + }) + .await; + manager + .user_manager + .update_connection(ActiveUserConnectionParams { + uid: 9002, + meter_uid: 0, + username: &user.username, + max_connections: user.max_connections, + soft_connections: user.soft_connections, + connection_kind: crate::ConnectionKind::Normal, + priority: 9, + soft_priority: 9, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some("tok-preempted-hls"), + }) + .await; + + let deps = CleanupWorkerDeps { + user_manager: Arc::clone(&manager.user_manager), + provider_manager: Arc::clone(&manager.provider_manager), + shared_stream_manager: Arc::clone(&manager.shared_stream_manager), + event_manager: Arc::clone(&manager.event_manager), + capacity_notify: Arc::clone(&manager.capacity_notify), + history_writer: Arc::clone(&manager.history_writer), + }; + + handle_update_detail_and_release_provider(&deps, addr, CustomVideoStreamType::LowPriorityPreempted, None, None) + .await; + + assert!( + manager + .user_manager + .is_user_blocked_for_stream(&user.username, shared::model::VirtualId::new(channel.virtual_id)) + .await, + "preempted HLS playback should be blocked briefly to prevent immediate reconnect ping-pong" + ); +} + +#[tokio::test] +async fn deferred_cleanup_executes_in_background_worker() { + let conn_manager = create_test_connection_manager(); + let ran = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let notify = Arc::new(Notify::new()); + let ran_clone = Arc::clone(&ran); + let notify_clone = Arc::clone(¬ify); + + conn_manager.send_cleanup(CleanupEvent::Defer(Box::pin(async move { + ran_clone.store(true, Ordering::SeqCst); + notify_clone.notify_one(); + }))); + + tokio::time::timeout(Duration::from_secs(1), notify.notified()).await.expect("deferred task must execute"); + assert!(ran.load(Ordering::SeqCst)); +} + +/// Dropping a just-registered request without handing it to a body must roll the +/// claim back, so a cancellation between registration and body construction cannot +/// leak a user claim. +#[tokio::test] +async fn registration_rollback_releases_claim_when_body_never_constructed() { + let manager = create_test_connection_manager(); + let addr: SocketAddr = "127.0.0.1:56234".parse().unwrap(); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let channel = StreamChannel { + virtual_id: 410, + title: "channel-410".intern(), + ..make_stream_info("provider_1", "channel-410").channel + }; + + manager.add_connection(&addr).await; + let registered = manager + .update_connection_with_uid( + ConnectionParams { + meter_uid: 0, + username: "rollback-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: None, + }, + ConnectionHistoryMode::EmitConnect, + 1, + None, + ) + .await; + assert!(registered.display_stream.is_some(), "registration must succeed"); + + // Drop without committing: the rollback owner releases the claim. + drop(registered); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if manager.user_manager.active_streams().await.is_empty() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the rollback must release the claim"); +} + +/// The body cleanup transferred via `into_body_cleanup` releases the claim exactly +/// once with the real outcome, even under cleanup-queue pressure. +#[tokio::test] +async fn body_cleanup_finish_releases_claim_exactly_once() { + let manager = create_test_connection_manager(); + let addr: SocketAddr = "127.0.0.1:56236".parse().unwrap(); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let channel = StreamChannel { + virtual_id: 412, + title: "channel-412".intern(), + ..make_stream_info("provider_1", "channel-412").channel + }; + + manager.add_connection(&addr).await; + let mut registered = manager + .update_connection_with_uid( + ConnectionParams { + meter_uid: 0, + username: "body-cleanup-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: None, + }, + ConnectionHistoryMode::EmitConnect, + 1, + None, + ) + .await; + assert!(registered.display_stream.is_some(), "registration must succeed"); + + let mut cleanup = registered.into_body_cleanup().expect("body cleanup present"); + cleanup.finish(None, PROVIDER_END_CLOSED, 0, None, None); + cleanup.finish(None, PROVIDER_END_CLOSED, 0, None, None); + + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if manager.user_manager.active_streams().await.is_empty() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the body cleanup must release the claim exactly once"); +} + +#[tokio::test] +async fn control_cleanup_lane_precedes_a_ready_body_cleanup_backlog() { + let manager = create_test_connection_manager(); + let release_blocker = Arc::new(Notify::new()); + let (blocker_started_tx, blocker_started_rx) = tokio::sync::oneshot::channel(); + let order = Arc::new(std::sync::Mutex::new(Vec::new())); + + manager.send_cleanup(CleanupEvent::Defer(Box::pin({ + let release_blocker = Arc::clone(&release_blocker); + let order = Arc::clone(&order); + async move { + let _ = blocker_started_tx.send(()); + release_blocker.notified().await; + order.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push('B'); + } + }))); + blocker_started_rx.await.expect("cleanup blocker must start"); + + manager.send_cleanup(CleanupEvent::Defer(Box::pin({ + let order = Arc::clone(&order); + async move { order.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push('N') } + }))); + manager + .control_cleanup_tx() + .send(CleanupEvent::Defer(Box::pin({ + let order = Arc::clone(&order); + async move { order.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push('C') } + }))) + .await + .expect("control cleanup receiver must be open"); + + release_blocker.notify_one(); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if order.lock().unwrap_or_else(std::sync::PoisonError::into_inner).len() == 3 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("both queued cleanup events must run"); + + assert_eq!( + *order.lock().unwrap_or_else(std::sync::PoisonError::into_inner), + vec!['B', 'C', 'N'], + "a ready control cleanup must not be starved by the body cleanup backlog" + ); +} + +/// A cleanup whose user row is already gone must still finish the provider request, +/// because the provider identity travels with the cleanup owner independently. +#[tokio::test] +async fn cleanup_finishes_provider_request_when_user_row_is_gone() { + let manager = create_test_connection_manager(); + let addr: SocketAddr = "127.0.0.1:56240".parse().unwrap(); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let channel = StreamChannel { + virtual_id: 413, + title: "channel-413".intern(), + ..make_stream_info("provider_1", "channel-413").channel + }; + let input_name = "provider_1".intern(); + let owner = "session-owner-g2"; + + // Acquire a provider slot with an identified lease. + let handle = manager + .provider_manager + .acquire_connection_with_grace_for_session( + &input_name, + &addr, + false, + 0, + crate::ConnectionKind::Normal, + Some(owner), + ) + .expect("acquire provider slot"); + let request_id = handle.playback_request_id.expect("identified request id"); + assert!(manager.provider_manager.provider_lease_usage(&input_name).total() > 0); + + // Register a user claim carrying the same owner and provider request id. + manager.add_connection(&addr).await; + let mut registered = manager + .update_connection_with_uid( + ConnectionParams { + meter_uid: 0, + username: "g2-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some(owner), + }, + ConnectionHistoryMode::EmitConnect, + 1, + Some(request_id), + ) + .await; + assert!(registered.display_stream.is_some()); + + let cleanup = registered.into_body_cleanup().expect("cleanup present"); + + // Remove the user row first, so the cleanup worker hits the NotFound path. + let _ = manager.user_manager.release_stream_request_by_uid(&addr, 1).await; + + drop(cleanup); + + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if manager.provider_manager.provider_lease_usage(&input_name).total() == 0 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("provider request must finish even without a user row"); + + manager.provider_manager.release_handle(&handle); +} + +/// Shutdown releases every active stream so no claim or provider slot survives. +#[tokio::test] +async fn shutdown_releases_active_streams() { + let manager = create_test_connection_manager(); + let addr: SocketAddr = "127.0.0.1:56235".parse().unwrap(); + let fingerprint = tuliprox_core::model::Fingerprint::new(format!("fp-{addr}"), addr.ip().to_string(), addr); + let channel = StreamChannel { + virtual_id: 411, + title: "channel-411".intern(), + ..make_stream_info("provider_1", "channel-411").channel + }; + + manager.add_connection(&addr).await; + let stream_info = manager + .update_connection(ConnectionParams { + meter_uid: 0, + username: "shutdown-user", + max_connections: 1, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: None, + }) + .await; + assert!(stream_info.is_some(), "registration must succeed"); + assert!(!manager.user_manager.active_streams().await.is_empty()); + + manager.shutdown().await; + assert!(manager.user_manager.active_streams().await.is_empty(), "shutdown must release active streams"); +} + +#[tokio::test] +async fn shutdown_parallel_requests_on_distinct_sockets_drains_all_claims() { + let manager = create_test_connection_manager(); + let first_addr: SocketAddr = "127.0.0.1:56241".parse().unwrap(); + let second_addr: SocketAddr = "127.0.0.1:56242".parse().unwrap(); + let first_fingerprint = + tuliprox_core::model::Fingerprint::new("range-first".to_string(), "127.0.0.1".to_string(), first_addr); + let second_fingerprint = + tuliprox_core::model::Fingerprint::new("range-second".to_string(), "127.0.0.1".to_string(), second_addr); + let mut channel = make_stream_info("provider_1", "parallel-range").channel; + channel.item_type = PlaylistItemType::Video; + channel.cluster = XtreamCluster::Video; + let owner = "shutdown-parallel-range"; + let input_name = "provider_1".intern(); + let handle = manager + .provider_manager + .acquire_connection_with_grace_for_session( + &input_name, + &first_addr, + false, + 0, + crate::ConnectionKind::Normal, + Some(owner), + ) + .expect("provider allocation"); + let request_id = handle.playback_request_id.expect("provider request id"); + + manager.add_connection(&first_addr).await; + manager.add_connection(&second_addr).await; + let mut first = manager + .update_connection_with_uid( + ConnectionParams { + meter_uid: 0, + username: "shutdown-range-user", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &first_fingerprint, + provider: Arc::clone(&input_name), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some(owner), + }, + ConnectionHistoryMode::EmitConnect, + 1, + Some(request_id), + ) + .await; + let mut second = manager + .update_connection_with_uid( + ConnectionParams { + meter_uid: 0, + username: "shutdown-range-user", + max_connections: 2, + soft_connections: 0, + connection_kind: crate::ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &second_fingerprint, + provider: Arc::clone(&input_name), + stream_channel: &channel, + user_agent: std::borrow::Cow::Borrowed("player/1.0"), + session_token: Some(owner), + }, + ConnectionHistoryMode::EmitConnect, + 2, + None, + ) + .await; + assert!(first.display_stream.is_some()); + assert!(second.display_stream.is_some()); + assert_eq!(manager.user_manager.active_streams().await.len(), 1); + assert_eq!(manager.user_manager.playback_resource_counts().await.0, 2); + assert_eq!(manager.provider_manager.get_provider_connections_count(), 1); + + let first_cleanup = first.into_body_cleanup().expect("first cleanup"); + let second_cleanup = second.into_body_cleanup().expect("second cleanup"); + manager.shutdown().await; + + assert_eq!(manager.user_manager.playback_resource_counts().await, (0, 0)); + assert_eq!(manager.provider_manager.get_provider_connections_count(), 0); + assert_eq!(manager.provider_manager.provider_lease_usage(&input_name).total(), 0); + + drop(first_cleanup); + drop(second_cleanup); + drop(handle); +} + +#[tokio::test] +async fn shutdown_with_all_cleanup_permits_held_completes() { + let manager = create_test_connection_manager(); + let mut permits = Vec::with_capacity(CLEANUP_QUEUE_CAPACITY); + for _ in 0..CLEANUP_QUEUE_CAPACITY { + permits.push(manager.cleanup_tx().reserve_owned().await.expect("cleanup receiver open")); + } + assert_eq!(manager.cleanup_tx().capacity(), 0); + + tokio::time::timeout(Duration::from_secs(2), manager.shutdown()) + .await + .expect("shutdown must not require cleanup queue capacity"); + drop(permits); +} diff --git a/backend/session/src/connection_manager/tests/playlist.rs b/backend/session/src/connection_manager/tests/playlist.rs new file mode 100644 index 000000000..4f5fa77cd --- /dev/null +++ b/backend/session/src/connection_manager/tests/playlist.rs @@ -0,0 +1,22 @@ +use super::*; + +#[test] +fn test_channel_unavailable_with_eof_maps_to_provider_closed() { + let info = make_stream_info("tuliprox", "channel_unavailable"); + let reason = resolve_disconnect_reason(PROVIDER_END_CLOSED, &info); + assert_eq!(reason, DisconnectReason::ProviderClosed); +} + +#[test] +fn test_channel_unavailable_with_err_maps_to_provider_error() { + let info = make_stream_info("tuliprox", "channel_unavailable"); + let reason = resolve_disconnect_reason(PROVIDER_END_ERROR, &info); + assert_eq!(reason, DisconnectReason::ProviderError); +} + +#[test] +fn test_channel_unavailable_without_atomic_maps_to_provider_error() { + let info = make_stream_info("tuliprox", "channel_unavailable"); + let reason = resolve_disconnect_reason(PROVIDER_END_NOT_SET, &info); + assert_eq!(reason, DisconnectReason::ProviderError); +} diff --git a/backend/session/src/connection_manager/tests/policy.rs b/backend/session/src/connection_manager/tests/policy.rs new file mode 100644 index 000000000..470446e69 --- /dev/null +++ b/backend/session/src/connection_manager/tests/policy.rs @@ -0,0 +1,148 @@ +use super::*; + +#[test] +fn socket_activity_tracker_coalesces_updates_per_socket() { + let tracker = SocketActivityTracker::new(); + let addr_one: SocketAddr = "127.0.0.1:3234".parse().unwrap_or_else(|_| unreachable!()); + let addr_two: SocketAddr = "127.0.0.1:3235".parse().unwrap_or_else(|_| unreachable!()); + + tracker.track(SocketActivityEvent::HttpActivity { addr: addr_one }); + tracker.track(SocketActivityEvent::HttpActivity { addr: addr_one }); + tracker.track(SocketActivityEvent::DirectBodyActivity { addr: addr_two }); + + let pending = tracker.drain(); + assert_eq!(pending.len(), 2); + assert!(pending.iter().any(|event| matches!( + event, + SocketActivityEvent::HttpActivity { addr } if *addr == addr_one + ))); + assert!(pending.iter().any(|event| matches!( + event, + SocketActivityEvent::DirectBodyActivity { addr } if *addr == addr_two + ))); +} + +#[tokio::test] +async fn kick_connection_sends_kick_close_signal() { + let manager = create_test_connection_manager(); + let mut rx = manager.get_close_connection_channel(); + let addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); + let _close_rx = manager.register_close_socket(addr); + + assert!(manager.kick_connection(&addr, shared::model::VirtualId::new(1), 0).await); + assert_eq!(rx.recv().await.ok(), Some(CloseConnectionSignal::WithReason(addr, DisconnectReason::ClientKicked))); +} + +#[tokio::test] +async fn kick_connection_returns_false_without_receivers() { + let manager = create_test_connection_manager(); + let addr: SocketAddr = "127.0.0.1:1235".parse().unwrap_or_else(|_| unreachable!()); + assert!(!manager.kick_connection(&addr, shared::model::VirtualId::new(1), 0).await); +} + +#[tokio::test] +async fn close_connection_signal_sends_generic_close_signal() { + let manager = create_test_connection_manager(); + let mut rx = manager.get_close_connection_channel(); + let addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); + let _close_rx = manager.register_close_socket(addr); + + assert!(manager.close_connection_signal(&addr)); + assert_eq!(rx.recv().await.ok(), Some(CloseConnectionSignal::WithReason(addr, DisconnectReason::ClientClosed))); +} + +#[tokio::test] +async fn provisioning_close_connection_sends_provisioning_signal() { + let manager = create_test_connection_manager(); + let mut rx = manager.get_close_connection_channel(); + let addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); + let _close_rx = manager.register_close_socket(addr); + + assert!( + manager + .close_connection_with_reason_and_block( + &addr, + shared::model::VirtualId::new(7), + 0, + DisconnectReason::Provisioning + ) + .await + ); + assert_eq!(rx.recv().await.ok(), Some(CloseConnectionSignal::WithReason(addr, DisconnectReason::Provisioning))); +} + +#[test] +fn test_client_closed_when_no_provider_end() { + let info = make_stream_info("some_provider", "Some Channel"); + let reason = resolve_disconnect_reason(PROVIDER_END_NOT_SET, &info); + assert_eq!(reason, DisconnectReason::ClientClosed); +} + +#[test] +fn test_client_kicked_disconnect_has_no_failure_stage() { + assert_eq!( + resolve_disconnect_failure_stage( + &make_stream_info("some_provider", "Some Channel"), + DisconnectReason::ClientKicked, + &DisconnectQos::default(), + ), + None + ); +} + +#[test] +fn test_provisioning_disconnect_has_no_failure_stage() { + assert_eq!( + resolve_disconnect_failure_stage( + &make_stream_info("some_provider", "Some Channel"), + DisconnectReason::Provisioning, + &DisconnectQos::default(), + ), + None + ); +} + +#[test] +fn test_provider_closed_on_eof() { + let info = make_stream_info("some_provider", "Some Channel"); + let reason = resolve_disconnect_reason(PROVIDER_END_CLOSED, &info); + assert_eq!(reason, DisconnectReason::ProviderClosed); +} + +#[test] +fn test_provider_error_on_err() { + let info = make_stream_info("some_provider", "Some Channel"); + let reason = resolve_disconnect_reason(PROVIDER_END_ERROR, &info); + assert_eq!(reason, DisconnectReason::ProviderError); +} + +#[test] +fn test_unknown_tuliprox_title_falls_through_to_atomic() { + let info = make_stream_info("tuliprox", "some_unknown_video_type"); + let reason = resolve_disconnect_reason(PROVIDER_END_CLOSED, &info); + assert_eq!(reason, DisconnectReason::ProviderClosed); +} + +#[test] +fn test_session_expired_disconnect_maps_to_session_reconnect_stage() { + assert_eq!( + resolve_disconnect_failure_stage( + &make_stream_info("some_provider", "Some Channel"), + DisconnectReason::SessionExpired, + &DisconnectQos::default(), + ), + Some(FailureStage::SessionReconnect) + ); +} + +#[test] +fn test_provider_error_without_first_byte_maps_to_first_byte_stage() { + assert_eq!( + resolve_disconnect_failure_stage( + &make_stream_info("some_provider", "Some Channel"), + DisconnectReason::ProviderError, + &DisconnectQos::default(), + ), + Some(FailureStage::FirstByte) + ); +} diff --git a/backend/session/src/connection_manager/tests/streaming.rs b/backend/session/src/connection_manager/tests/streaming.rs new file mode 100644 index 000000000..80268380c --- /dev/null +++ b/backend/session/src/connection_manager/tests/streaming.rs @@ -0,0 +1,85 @@ +use super::*; + +#[tokio::test] +async fn enqueue_with_backpressure_bounds_overflow_buffer() { + let (tx, mut rx) = mpsc::channel(1); + let sender = BackpressureSender::new(tx.clone(), "test", 1); + assert!(tx.send(1_u8).await.is_ok()); + + sender.enqueue(2_u8); + for _ in 0..50 { + let ready = { + let state = super::super::lock_backpressure_state(sender.state.as_ref()); + state.overflow.is_empty() && state.draining + }; + if ready { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + sender.enqueue(3_u8); + sender.enqueue(4_u8); + + assert_eq!(rx.recv().await, Some(1)); + assert_eq!(tokio::time::timeout(Duration::from_secs(1), rx.recv()).await.ok().flatten(), Some(2)); + assert_eq!(tokio::time::timeout(Duration::from_secs(1), rx.recv()).await.ok().flatten(), Some(3)); + let result = tokio::time::timeout(Duration::from_millis(100), rx.recv()).await; + assert!(result.is_err()); +} + +#[tokio::test] +async fn close_connection_with_reason_returns_false_when_only_broadcast_receiver_exists() { + let manager = create_test_connection_manager(); + let mut broadcast_rx = manager.get_close_connection_channel(); + let addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); + + assert!( + !manager.close_connection_with_reason(&addr, DisconnectReason::ClientKicked), + "must return false when only broadcast receiver exists without registered target socket" + ); + assert_eq!( + broadcast_rx.recv().await.ok(), + Some(CloseConnectionSignal::WithReason(addr, DisconnectReason::ClientKicked)) + ); +} + +#[tokio::test] +async fn close_connection_with_reason_returns_false_for_unrelated_addr_even_with_broadcast_receivers() { + let manager = create_test_connection_manager(); + let _broadcast_rx = manager.get_close_connection_channel(); + let active_addr: SocketAddr = "127.0.0.1:1234".parse().unwrap_or_else(|_| unreachable!()); + let unrelated_addr: SocketAddr = "127.0.0.1:5678".parse().unwrap_or_else(|_| unreachable!()); + + let _close_rx = manager.register_close_socket(active_addr); + + assert!( + !manager.close_connection_with_reason(&unrelated_addr, DisconnectReason::ClientKicked), + "unrelated address must not report success just because another socket is registered" + ); + assert!( + manager.close_connection_with_reason(&active_addr, DisconnectReason::ClientKicked), + "target address registered must report success" + ); +} + +#[test] +fn test_provider_error_disconnect_maps_to_streaming_failure_stage() { + assert_eq!( + resolve_disconnect_failure_stage( + &make_stream_info("some_provider", "Some Channel"), + DisconnectReason::ProviderError, + &DisconnectQos { first_byte_latency_ms: Some(150), ..Default::default() }, + ), + Some(FailureStage::Streaming) + ); +} + +#[test] +fn test_shared_provider_error_without_first_byte_stays_streaming_stage() { + let mut info = make_stream_info("some_provider", "Some Channel"); + info.channel.shared = true; + assert_eq!( + resolve_disconnect_failure_stage(&info, DisconnectReason::ProviderError, &DisconnectQos::default(),), + Some(FailureStage::Streaming) + ); +} diff --git a/backend/session/src/connection_manager/tests/terminal.rs b/backend/session/src/connection_manager/tests/terminal.rs new file mode 100644 index 000000000..960f028ce --- /dev/null +++ b/backend/session/src/connection_manager/tests/terminal.rs @@ -0,0 +1,33 @@ +use super::*; + +#[test] +fn test_preempted_from_custom_video_detail() { + let info = make_stream_info("tuliprox", "low_priority_preempted"); + let reason = resolve_disconnect_reason(PROVIDER_END_NOT_SET, &info); + assert_eq!(reason, DisconnectReason::Preempted); +} + +#[test] +fn test_provider_preempted_atomic_maps_to_preempted() { + let info = make_stream_info("some_provider", "Some Channel"); + let reason = resolve_disconnect_reason(PROVIDER_END_PREEMPTED, &info); + assert_eq!(reason, DisconnectReason::Preempted); + + let reason_direct = resolve_disconnect_reason_from_provider_end(PROVIDER_END_PREEMPTED); + assert_eq!(reason_direct, DisconnectReason::Preempted); + assert_eq!(playback_outcome_for_reason(reason_direct), PlaybackRequestOutcome::Preempted); +} + +#[test] +fn test_user_exhausted_custom_video_maps_to_user_connections_exhausted() { + let info = make_stream_info("tuliprox", "user_connections_exhausted"); + let reason = resolve_disconnect_reason(PROVIDER_END_NOT_SET, &info); + assert_eq!(reason, DisconnectReason::UserConnectionsExhausted); +} + +#[test] +fn test_provider_exhausted_custom_video_maps_to_provider_connections_exhausted() { + let info = make_stream_info("tuliprox", "provider_connections_exhausted"); + let reason = resolve_disconnect_reason(PROVIDER_END_NOT_SET, &info); + assert_eq!(reason, DisconnectReason::ProviderConnectionsExhausted); +} diff --git a/backend/session/src/connection_manager/tests/transport.rs b/backend/session/src/connection_manager/tests/transport.rs new file mode 100644 index 000000000..eea3f42b6 --- /dev/null +++ b/backend/session/src/connection_manager/tests/transport.rs @@ -0,0 +1,15 @@ +use super::*; + +#[tokio::test] +async fn enqueue_with_backpressure_delivers_events_after_queue_full() { + let (tx, mut rx) = mpsc::channel(1); + let sender = BackpressureSender::new(tx.clone(), "test", 2); + assert!(tx.send(1_u8).await.is_ok()); + + sender.enqueue(2_u8); + sender.enqueue(3_u8); + + assert_eq!(rx.recv().await, Some(1)); + assert_eq!(tokio::time::timeout(Duration::from_secs(1), rx.recv()).await.ok().flatten(), Some(2)); + assert_eq!(tokio::time::timeout(Duration::from_secs(1), rx.recv()).await.ok().flatten(), Some(3)); +} diff --git a/backend/session/src/streams/shared_stream_manager.rs b/backend/session/src/streams/shared_stream_manager.rs index f8fc58790..970aed3a8 100644 --- a/backend/session/src/streams/shared_stream_manager.rs +++ b/backend/session/src/streams/shared_stream_manager.rs @@ -1,42 +1,33 @@ -use crate::{ - streams::buffered_stream::CHANNEL_SIZE, ActiveProviderManager, BoxedProviderStream, CleanupEvent, - ConnectionManager, ConnectionRejectionReason, ManagedProviderHandle, SharedCleanupCapability, -}; +use crate::{ActiveProviderManager, CleanupEvent}; use bytes::Bytes; -use futures::{Stream, StreamExt}; -use log::{debug, warn}; -use shared::utils::sanitize_sensitive_info; use std::{ collections::{HashMap, VecDeque}, - fmt, - fmt::{Debug, Formatter}, - future::Future, + fmt::Debug, net::SocketAddr, pin::Pin, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }, - task::{Context, Poll}, + sync::{atomic::AtomicUsize, Arc}, }; use tokio::{ - sync::{mpsc, mpsc::Sender, oneshot, Mutex, Notify, RwLock, Semaphore}, - time::{sleep, timeout, Duration, Instant, Sleep}, + sync::{oneshot, Mutex, Notify, RwLock}, + time::{Duration, Sleep}, }; -use tokio_stream::wrappers::ReceiverStream; use tokio_util::sync::CancellationToken; -use tuliprox_core::{ - model::{AllocationId, AppConfig, Config, ProviderHandle, SharedSubscriberId, StreamError}, - utils::{debug_if_enabled, network::request::STREAM_IDLE_TIMEOUT, trace_if_enabled}, -}; +use tuliprox_core::model::{AllocationId, ProviderHandle, SharedSubscriberId}; const DEFAULT_SHARED_BUFFER_SIZE_BYTES: usize = 1024 * 1024 * 32; + const YIELD_COUNTER: usize = 64; + const MIN_BURST_BUFFER_CHUNKS: usize = 2; + const MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES: usize = 188; + const SHARED_BURST_BYTES_PER_BUFFER_SLOT: usize = 12 * 1024; + const DEFAULT_SUBSCRIBER_IDLE_TIMEOUT_SECS: u64 = 300; + const SHARED_CLEANUP_ADMISSION_TIMEOUT: Duration = Duration::from_secs(5); + // Upper bound on chunks pulled into a subscriber's live batch per read. The burst ring // is byte-bounded already; this caps how many of its entries are cloned into the // per-subscriber vector at once so a long backlog cannot balloon that vector. @@ -57,18 +48,6 @@ pub struct PendingSharedMeterRegistration { armed: bool, } -impl PendingSharedMeterRegistration { - pub fn commit(mut self) { self.armed = false; } -} - -impl Drop for PendingSharedMeterRegistration { - fn drop(&mut self) { - if self.armed { - self.manager.remove_meter_uid_if_pending(&self.stream_url, self.meter_uid); - } - } -} - /// Tracks a shared-stream meter reservation. `owner` is `None` while the meter is /// only reserved; it becomes `Some(allocation_id)` once a live shared origin adopts it. /// Pending rollbacks may only remove an entry that no origin has adopted yet. @@ -85,70 +64,6 @@ struct PendingJoinGuard { armed: bool, } -impl PendingJoinGuard { - fn new(state: &Arc) -> Self { - state.increment_pending_joins(); - Self { state: Arc::clone(state), armed: true } - } - - fn commit(mut self) { - self.armed = false; - self.state.decrement_pending_joins(); - } -} - -impl Drop for PendingJoinGuard { - fn drop(&mut self) { - if self.armed { - self.state.decrement_pending_joins(); - } - } -} - -impl PendingSharedSubscriberCleanup { - fn new( - permit: tokio::sync::mpsc::OwnedPermit, - subscriber_id: SharedSubscriberId, - addr: SocketAddr, - ) -> Self { - Self { permit: Some(permit), subscriber_id, addr, request_id: None, owner: None } - } - - pub fn set_provider_request_identity(&mut self, owner: &str, request_id: tuliprox_core::model::PlaybackRequestId) { - self.owner = Some(Arc::from(owner)); - self.request_id = Some(request_id); - } - - /// Produces the cleanup-ownership proof for this subscriber. The permit held by - /// this guard is the source of the capability. - pub fn capability(&self) -> SharedCleanupCapability { SharedCleanupCapability::new(self.subscriber_id) } - - fn take_cleanup( - &mut self, - ) -> ( - Option>, - Option, - Option>, - ) { - (self.permit.take(), self.request_id.take(), self.owner.take()) - } - - pub fn disarm(&mut self) { self.permit = None; } -} - -impl Drop for PendingSharedSubscriberCleanup { - fn drop(&mut self) { - if let Some(permit) = self.permit.take() { - permit.send(CleanupEvent::ReleaseSharedSubscriber { - addr: self.addr, - subscriber_id: self.subscriber_id, - request_id: self.request_id, - owner: self.owner.take(), - }); - } - } -} - struct ReceiverStreamWrapper { stream: S, start: Option>, @@ -163,60 +78,6 @@ struct ReceiverStreamWrapper { released: bool, } -impl ReceiverStreamWrapper { - fn release(&mut self) { - if !self.released { - self.released = true; - if let Some(permit) = self.permit.take() { - permit.send(CleanupEvent::ReleaseSharedSubscriber { - addr: self.addr, - subscriber_id: self.subscriber_id, - request_id: self.request_id, - owner: self.owner.take(), - }); - } - } - } -} - -impl Stream for ReceiverStreamWrapper -where - S: Stream + Unpin, -{ - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if self.deadline.as_mut().is_some_and(|deadline| deadline.as_mut().poll(cx).is_ready()) { - self.release(); - return Poll::Ready(None); - } - if let Some(start) = self.start.take() { - let _ = start.send(()); - } - match Pin::new(&mut self.stream).poll_next(cx) { - Poll::Ready(Some(bytes)) => Poll::Ready(Some(Ok(bytes))), - Poll::Ready(None) => Poll::Ready(None), - Poll::Pending => Poll::Pending, - } - } -} - -fn resolve_min_burst_buffer_bytes(config: &Config) -> usize { - config - .reverse_proxy - .as_ref() - .and_then(|rp| rp.stream.as_ref()) - .and_then(|stream| usize::try_from(stream.shared_burst_buffer_mb.saturating_mul(1024 * 1024)).ok()) - .unwrap_or(DEFAULT_SHARED_BUFFER_SIZE_BYTES) - .max(1) -} - -impl Drop for ReceiverStreamWrapper { - fn drop(&mut self) { self.release(); } -} - -type SubscriberId = SharedSubscriberId; - #[derive(Clone, Debug)] struct SharedSubscriber { addr: SocketAddr, @@ -241,174 +102,12 @@ struct BurstRead { skipped: u64, } -#[allow(clippy::missing_fields_in_debug)] -impl Debug for BurstBuffer { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("BurstBuffer") - .field("buffer_size", &self.buffer_size) - .field("max_chunks", &self.max_chunks) - .field("current_bytes", &self.current_bytes) - .finish() - } -} - -impl BurstBuffer { - pub fn new(buf_size: usize) -> Self { - Self { - buffer: VecDeque::new(), - buffer_size: buf_size, - max_chunks: Self::max_chunks_for_buffer_size(buf_size), - current_bytes: 0, - next_sequence: 0, - } - } - - pub fn snapshot(&self) -> (Vec, u64) { - (self.buffer.iter().map(|chunk| chunk.bytes.clone()).collect::>(), self.next_sequence) - } - - pub fn read_from_into(&self, next_sequence: u64, chunks: &mut Vec, max_chunks: usize) -> BurstRead { - chunks.clear(); - let earliest_sequence = self.buffer.front().map_or(self.next_sequence, |chunk| chunk.sequence); - let start_sequence = next_sequence.max(earliest_sequence); - let skipped = start_sequence.saturating_sub(next_sequence); - let start_index = self.start_index_for_sequence(start_sequence); - let mut read_next_sequence = start_sequence; - for chunk in self.buffer.range(start_index..).take(max_chunks) { - chunks.push(chunk.bytes.clone()); - read_next_sequence = chunk.sequence.saturating_add(1); - } - - BurstRead { next_sequence: read_next_sequence, skipped } - } - - pub fn push(&mut self, packet: Bytes) { - let packet_len = packet.len(); - while !self.buffer.is_empty() - && (self.buffer.len() >= self.max_chunks - || self.current_bytes.saturating_add(packet_len) > self.buffer_size) - { - if let Some(popped) = self.buffer.pop_front() { - self.current_bytes = self.current_bytes.saturating_sub(popped.bytes.len()); - } - } - self.current_bytes = self.current_bytes.saturating_add(packet_len); - let sequence = self.next_sequence; - self.next_sequence = self.next_sequence.saturating_add(1); - self.buffer.push_back(BufferedChunk { sequence, bytes: packet }); - } - - fn start_index_for_sequence(&self, sequence: u64) -> usize { - let mut left = 0_usize; - let mut right = self.buffer.len(); - - while left < right { - let mid = left + ((right - left) / 2); - let mid_sequence = self.buffer.get(mid).map_or(u64::MAX, |chunk| chunk.sequence); - if mid_sequence < sequence { - left = mid.saturating_add(1); - } else { - right = mid; - } - } - - left - } - - fn max_chunks_for_buffer_size(buffer_size: usize) -> usize { - buffer_size.div_ceil(MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES).max(MIN_BURST_BUFFER_CHUNKS) - } -} - -/// Terminal outcome of trying to deliver one chunk to a subscriber. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum SendOutcome { - Sent, - Cancelled, - Closed, - TimedOut, -} - /// A queue entry that releases its byte-budget permit when the client consumes (drops) it. struct BudgetedChunk { bytes: Bytes, _permit: tokio::sync::OwnedSemaphorePermit, } -async fn reserve_byte_permit( - byte_budget: &Arc, - len: usize, - cancellation_token: &CancellationToken, - progress_deadline: Instant, -) -> Result { - let permits = u32::try_from(len).unwrap_or(u32::MAX); - tokio::select! { - biased; - () = cancellation_token.cancelled() => Err(SendOutcome::Cancelled), - result = tokio::time::timeout_at(progress_deadline, byte_budget.clone().acquire_many_owned(permits)) => { - match result { - Ok(Ok(permit)) => Ok(permit), - Ok(Err(_)) => Err(SendOutcome::Closed), - Err(_) => Err(SendOutcome::TimedOut), - } - } - } -} - -async fn send_burst_buffer( - start_buffer: &[Bytes], - client_tx: &Sender, - cancellation_token: &CancellationToken, - idle_timeout: Duration, - byte_budget: &Arc, -) -> Result { - let mut sent = 0_usize; - let mut last_progress = Instant::now(); - for buf in start_buffer { - let deadline = last_progress + idle_timeout; - match send_client_chunk(client_tx, buf.clone(), cancellation_token, deadline, byte_budget).await { - SendOutcome::Sent => { - sent = sent.saturating_add(1); - last_progress = Instant::now(); - } - outcome => return Err(outcome), - } - } - Ok(sent) -} - -async fn send_client_chunk( - client_tx: &Sender, - data: Bytes, - cancellation_token: &CancellationToken, - progress_deadline: Instant, - byte_budget: &Arc, -) -> SendOutcome { - if cancellation_token.is_cancelled() { - return SendOutcome::Cancelled; - } - - let permit = match reserve_byte_permit(byte_budget, data.len(), cancellation_token, progress_deadline).await { - Ok(permit) => permit, - Err(outcome) => return outcome, - }; - - let chunk = BudgetedChunk { bytes: data, _permit: permit }; - match client_tx.try_send(chunk) { - Ok(()) => SendOutcome::Sent, - Err(mpsc::error::TrySendError::Closed(_)) => SendOutcome::Closed, - Err(mpsc::error::TrySendError::Full(chunk)) => tokio::select! { - biased; - () = cancellation_token.cancelled() => SendOutcome::Cancelled, - result = tokio::time::timeout_at(progress_deadline, client_tx.send(chunk)) => match result { - Ok(Ok(())) => SendOutcome::Sent, - Ok(Err(_)) => SendOutcome::Closed, - Err(_) => SendOutcome::TimedOut, - }, - }, - } -} - #[derive(Debug)] pub struct SharedStreamState { headers: Vec<(String, String)>, @@ -429,491 +128,6 @@ pub struct SharedStreamState { pending_joins: AtomicUsize, } -impl SharedStreamState { - fn new( - headers: Vec<(String, String)>, - buf_size: usize, - provider_guard: Option, - min_burst_buffer_size: usize, - low_priority_preempted: Option, - ) -> Self { - let base_channel_capacity = buf_size.max(CHANNEL_SIZE); - let burst_buffer_size_in_bytes = - min_burst_buffer_size.max(base_channel_capacity.saturating_mul(SHARED_BURST_BYTES_PER_BUFFER_SLOT)); - Self { - headers, - buf_size: base_channel_capacity, - provider_guard, - low_priority_preempted, - preempted_token: CancellationToken::new(), - subscribers: RwLock::new(HashMap::new()), - stop_token: CancellationToken::new(), - burst_buffer: Arc::new(Mutex::new(BurstBuffer::new(burst_buffer_size_in_bytes))), - live_notification: Arc::new(Notify::new()), - task_handles: std::sync::Mutex::new(Vec::new()), - subscriber_idle_timeout_secs: DEFAULT_SUBSCRIBER_IDLE_TIMEOUT_SECS, - subscriber_max_duration: None, - pending_joins: AtomicUsize::new(0), - } - } - - fn with_subscriber_idle_timeout_secs(mut self, secs: u64) -> Self { - if secs > 0 { - self.subscriber_idle_timeout_secs = secs; - } - self - } - - fn increment_pending_joins(&self) { self.pending_joins.fetch_add(1, Ordering::AcqRel); } - - fn decrement_pending_joins(&self) { self.pending_joins.fetch_sub(1, Ordering::AcqRel); } - - fn has_pending_joins(&self) -> bool { self.pending_joins.load(Ordering::Acquire) > 0 } - - fn lock_task_handles(&self) -> std::sync::MutexGuard<'_, Vec>> { - self.task_handles.lock().unwrap_or_else(std::sync::PoisonError::into_inner) - } - - async fn register_subscriber(&self, id: SubscriberId, addr: &SocketAddr, cancel_token: CancellationToken) { - self.subscribers.write().await.insert(id, SharedSubscriber { addr: *addr, cancel_token }); - } - - async fn cancel_subscribers(&self) { - for subscriber in self.subscribers.read().await.values() { - subscriber.cancel_token.cancel(); - } - } - - #[allow(clippy::too_many_lines)] - async fn subscribe( - self: &Arc, - addr: &SocketAddr, - subscriber_id: SubscriberId, - connection_manager: Arc, - mut pending_cleanup: PendingSharedSubscriberCleanup, - pending_join: PendingJoinGuard, - ) -> (BoxedProviderStream, Option>, SharedCleanupCapability) { - let (client_tx, client_rx) = mpsc::channel::(self.buf_size); - let cancel_token = CancellationToken::new(); - let queue_byte_budget = self.buf_size.saturating_mul(SHARED_BURST_BYTES_PER_BUFFER_SLOT); - let byte_budget = Arc::new(Semaphore::new(queue_byte_budget)); - - { - let mut handles = self.lock_task_handles(); - handles.retain(|h| !h.is_finished()); - } - - self.register_subscriber(subscriber_id, addr, cancel_token.clone()).await; - pending_join.commit(); - let (start_tx, start_rx) = oneshot::channel(); - let cleanup_manager = Arc::clone(&connection_manager); - - let client_tx_clone = client_tx.clone(); - let burst_buffer = Arc::clone(&self.burst_buffer); - let burst_buffer_for_log = Arc::clone(&self.burst_buffer); - let live_notification = Arc::clone(&self.live_notification); - let timeout_duration = Duration::from_secs(self.subscriber_idle_timeout_secs); - let idle_check_interval = Duration::from_secs(1); - let mut last_lag_log = Instant::now().checked_sub(Duration::from_secs(10)).unwrap_or_else(Instant::now); - let mut consecutive_lag_count: u32 = 0; - let subscriber_buf_size = self.buf_size; - let preempted_token = self.preempted_token.clone(); - let low_priority_preempted = self.low_priority_preempted.clone(); - let address = *addr; - let subscriber_started_at = Instant::now(); - - let handle = tokio::spawn(async move { - // Wait for the response to be polled, but respect cancellation/shutdown so a - // body that is never polled cannot keep the forwarder task alive indefinitely. - tokio::select! { - () = cancel_token.cancelled() => return, - started = start_rx => { - if started.is_err() { - return; - } - } - } - // Send-progress budget begins only once the body is actually polled. - let mut last_active = Instant::now(); - let (snapshot, mut next_sequence) = { - let buffer = burst_buffer.lock().await; - buffer.snapshot() - }; - match send_burst_buffer(&snapshot, &client_tx_clone, &cancel_token, timeout_duration, &byte_budget).await { - Ok(sent_burst_chunks) => { - drop(snapshot); - if sent_burst_chunks > 0 { - // The replay renewed its own internal progress time; carry that - // forward so the first live send does not immediately expire. - last_active = Instant::now(); - debug_if_enabled!( - "Shared stream subscriber {} replayed {sent_burst_chunks} burst chunks after {} ms", - sanitize_sensitive_info(&address.to_string()), - subscriber_started_at.elapsed().as_millis() - ); - } - } - Err(outcome) => { - drop(snapshot); - debug!( - "Shared stream subscriber {} burst replay failed ({outcome:?}); terminating", - sanitize_sensitive_info(&address.to_string()) - ); - cleanup_manager.send_cleanup(CleanupEvent::ReleaseSharedSubscriber { - addr: address, - subscriber_id, - request_id: None, - owner: None, - }); - return; - } - } - - let mut first_live_chunk_logged = false; - let mut startup_chunks_sent = 0_usize; - let mut startup_bytes_sent = 0_usize; - let mut startup_stats_logged = false; - let mut read_chunks = Vec::with_capacity(subscriber_buf_size.min(64)); - let idle_check = sleep(idle_check_interval); - tokio::pin!(idle_check); - - loop { - // Pre-create the notified future before locking the buffer to avoid - // a race where notify_waiters() fires between lock release and await. - let notified_fut = live_notification.notified(); - - let read = { - let buffer = burst_buffer.lock().await; - buffer.read_from_into(next_sequence, &mut read_chunks, MAX_SUBSCRIBER_LIVE_BATCH_CHUNKS) - }; - next_sequence = read.next_sequence; - if read.skipped > 0 { - consecutive_lag_count = consecutive_lag_count.saturating_add(1); - if last_lag_log.elapsed() > Duration::from_secs(5) { - let buffered_bytes = { - let buffer = burst_buffer_for_log.lock().await; - buffer.current_bytes - }; - warn!( - "Shared stream client lagged behind {address}. Skipped {} messages \ - (buffered {buffered_bytes} bytes, consecutive lags={consecutive_lag_count})", - read.skipped - ); - last_lag_log = Instant::now(); - } - } else if !read_chunks.is_empty() { - consecutive_lag_count = 0; - } - - trace_if_enabled!( - "shared_stream.subscribe: read {} chunks (next_seq={}, skipped={}) for {}", - read_chunks.len(), - read.next_sequence, - read.skipped, - sanitize_sensitive_info(&address.to_string()) - ); - - if !read_chunks.is_empty() { - for data in read_chunks.drain(..) { - let chunk_len = data.len(); - match send_client_chunk( - &client_tx, - data, - &cancel_token, - last_active + timeout_duration, - &byte_budget, - ) - .await - { - SendOutcome::Sent => {} - outcome => { - debug!("Shared stream client send error ({outcome:?}): {address}"); - cleanup_manager.send_cleanup(CleanupEvent::ReleaseSharedSubscriber { - addr: address, - subscriber_id, - request_id: None, - owner: None, - }); - return; - } - } - if !first_live_chunk_logged { - debug_if_enabled!( - "Shared stream subscriber {} received first live chunk after {} ms", - sanitize_sensitive_info(&address.to_string()), - subscriber_started_at.elapsed().as_millis() - ); - first_live_chunk_logged = true; - } - if !startup_stats_logged { - startup_chunks_sent = startup_chunks_sent.saturating_add(1); - startup_bytes_sent = startup_bytes_sent.saturating_add(chunk_len); - if subscriber_started_at.elapsed() >= Duration::from_secs(5) { - debug_if_enabled!( - "Shared stream subscriber {} startup throughput: chunks={} bytes={} over {} ms (queue_used={}/{})", - sanitize_sensitive_info(&address.to_string()), - startup_chunks_sent, - startup_bytes_sent, - subscriber_started_at.elapsed().as_millis(), - subscriber_buf_size.saturating_sub(client_tx_clone.capacity()), - subscriber_buf_size - ); - startup_stats_logged = true; - } - } - last_active = Instant::now(); - } - continue; - } - - tokio::select! { - biased; - - () = cancel_token.cancelled() => { - trace_if_enabled!( - "shared_stream.subscribe: cancel_received for {}", - sanitize_sensitive_info(&address.to_string()) - ); - break; - } - - () = &mut idle_check => { - if last_active.elapsed() > timeout_duration { - trace_if_enabled!( - "shared_stream.subscribe: idle_check_fired (inactivity>={}s) for {}", - timeout_duration.as_secs(), - sanitize_sensitive_info(&address.to_string()) - ); - cancel_token.cancel(); - break; - } - idle_check.as_mut().reset(Instant::now() + idle_check_interval); - } - - () = notified_fut => { - trace_if_enabled!( - "shared_stream.subscribe: empty_buffer_waiting waker for {}", - sanitize_sensitive_info(&address.to_string()) - ); - } - - () = preempted_token.cancelled() => { - trace_if_enabled!( - "shared_stream.subscribe: preempted for {}", - sanitize_sensitive_info(&address.to_string()) - ); - if let Some(mut fallback) = low_priority_preempted { - debug_if_enabled!( - "Shared stream subscriber {} switching to low_priority_preempted fallback", - sanitize_sensitive_info(&address.to_string()) - ); - while let Some(chunk) = fallback.next_chunk() { - match send_client_chunk(&client_tx, chunk, &cancel_token, last_active + timeout_duration, &byte_budget).await { - SendOutcome::Sent => last_active = Instant::now(), - outcome => { - debug!( - "Shared stream fallback send error ({outcome:?}) for {}", - sanitize_sensitive_info(&address.to_string()) - ); - break; - } - } - } - } - break; - } - } - } - - cleanup_manager.send_cleanup(CleanupEvent::ReleaseSharedSubscriber { - addr: address, - subscriber_id, - request_id: None, - owner: None, - }); - }); - - self.lock_task_handles().push(handle); - - let provider = self.provider_guard.as_ref().and_then(|h| h.allocation.get_provider_name()); - let (permit, request_id, owner) = pending_cleanup.take_cleanup(); - ( - ReceiverStreamWrapper { - stream: ReceiverStream::new(client_rx).map(|chunk| chunk.bytes), - start: Some(start_tx), - subscriber_id, - addr: *addr, - permit, - request_id, - owner, - deadline: self.subscriber_max_duration.map(|duration| Box::pin(sleep(duration))), - released: false, - } - .boxed(), - provider, - SharedCleanupCapability::new(subscriber_id), - ) - } - - #[allow(clippy::too_many_lines)] - fn broadcast(self: &Arc, stream_url: &str, bytes_stream: S, shared_streams: Arc) - where - S: Stream> + Unpin + 'static + Send, - E: std::fmt::Debug + Send, - { - let streaming_url = stream_url.to_string(); - let origin_state = Arc::clone(self); - let stop_token = self.stop_token.clone(); - let burst_buffer = Arc::clone(&self.burst_buffer); - let live_notification = Arc::clone(&self.live_notification); - let broadcast_started_at = Instant::now(); - - let broadcast_handle = tokio::spawn(async move { - let mut source_stream = std::pin::pin!(bytes_stream); - let mut counter = 0_usize; - let idle_timeout = Duration::from_secs(STREAM_IDLE_TIMEOUT); - let idle = sleep(idle_timeout); - tokio::pin!(idle); - let mut first_source_chunk_logged = false; - let mut startup_chunks_seen = 0_usize; - let mut startup_bytes_seen = 0_usize; - let mut startup_stats_logged = false; - // Track the time of the most recent upstream push so the broadcast can - // detect a stalled source before the global idle timeout fires. This is - // the diagnostic signal for H1 (broadcast stall with stale burst replay). - let mut last_push_at: Option = None; - let mut idle_warning_emitted = false; - let idle_warn_threshold = idle_timeout / 2; - - loop { - tokio::select! { - biased; - - () = stop_token.cancelled() => { - debug_if_enabled!( - "No shared stream subscribers left. Closing shared provider stream {}", - sanitize_sensitive_info(&streaming_url) - ); - break; - } - - () = &mut idle => { - debug_if_enabled!( - "Shared stream source idle timeout after {}s for {}", - STREAM_IDLE_TIMEOUT, - sanitize_sensitive_info(&streaming_url) - ); - stop_token.cancel(); - break; - } - - chunk = source_stream.next() => { - // Only successful chunks count as liveness; resetting on Err would let an - // error-spinning source dodge the idle timeout forever - if matches!(chunk, Some(Ok(_))) { - idle.as_mut().reset(Instant::now() + idle_timeout); - } - match chunk { - Some(Ok(data)) => { - let chunk_len = data.len(); - let push_seq = { - let mut buffer = burst_buffer.lock().await; - let seq = buffer.next_sequence; - buffer.push(data); - seq - }; - live_notification.notify_waiters(); - last_push_at = Some(Instant::now()); - idle_warning_emitted = false; - trace_if_enabled!( - "shared_stream.broadcast: push seq={} len={} url={}", - push_seq, - chunk_len, - sanitize_sensitive_info(&streaming_url) - ); - - if !first_source_chunk_logged { - debug_if_enabled!( - "Shared stream source produced first chunk for {} after {} ms", - sanitize_sensitive_info(&streaming_url), - broadcast_started_at.elapsed().as_millis() - ); - first_source_chunk_logged = true; - } - if !startup_stats_logged { - startup_chunks_seen = startup_chunks_seen.saturating_add(1); - startup_bytes_seen = startup_bytes_seen.saturating_add(chunk_len); - if broadcast_started_at.elapsed() >= Duration::from_secs(5) { - debug_if_enabled!( - "Shared stream source startup throughput for {}: chunks={} bytes={} over {} ms", - sanitize_sensitive_info(&streaming_url), - startup_chunks_seen, - startup_bytes_seen, - broadcast_started_at.elapsed().as_millis() - ); - startup_stats_logged = true; - } - } - - counter = counter.saturating_add(1); - if counter >= YIELD_COUNTER { - tokio::task::yield_now().await; - counter = 0; - } - } - Some(Err(e)) => { - trace_if_enabled!( - "Shared stream source error for {}: {:?}", - sanitize_sensitive_info(&streaming_url), - e - ); - tokio::task::yield_now().await; - } - None => { - debug_if_enabled!( - "Shared stream source stream ended for {}", - sanitize_sensitive_info(&streaming_url) - ); - break; - } - } - } - } - - // Edge-triggered stall warning: fires once when the broadcast has - // not seen an upstream push for STREAM_IDLE_TIMEOUT/2 seconds, then - // resets on the next successful push. Operators correlate this - // warning with the "stuck in resending same buffer" symptom (H1). - if let Some(last) = last_push_at { - let stalled_for = last.elapsed(); - if stalled_for >= idle_warn_threshold && !idle_warning_emitted { - warn!( - "shared_stream.broadcast: no upstream bytes for {}s on url={}; \ - source may be stalled. Subscribers will see cached burst until {}s timeout.", - stalled_for.as_secs(), - sanitize_sensitive_info(&streaming_url), - STREAM_IDLE_TIMEOUT - ); - idle_warning_emitted = true; - } - } - } - - debug_if_enabled!( - "Shared stream exiting for {} (last_push_age_secs={})", - sanitize_sensitive_info(&streaming_url), - last_push_at.map_or(0, |t| t.elapsed().as_secs()) - ); - shared_streams.unregister(&streaming_url, &origin_state).await; - }); - - // Keep the broadcast handle so shutdown can join it instead of leaving a detached task. - // Registration is guaranteed: a short-lived synchronous lock avoids the try_write - // failure that would otherwise drop the handle under contention. - self.lock_task_handles().push(broadcast_handle); - } -} - #[derive(Debug, Clone, Default)] struct SharedStreamsRegister { by_key: HashMap, Arc>, @@ -928,1708 +142,13 @@ pub struct SharedStreamManager { pub(crate) test_preflight_barrier: std::sync::Mutex>>, } -/// The four state handles the shared-stream paths need. -/// -/// These functions used to take the whole `AppState` and reach into four of its -/// fields. Keeping this slice explicit avoids coupling the session crate to the -/// API server state; the composition root supplies the required handles. -#[derive(Clone, Copy)] -pub struct SharedStreamCtx<'a> { - pub app_config: &'a Arc, - pub shared_stream_manager: &'a Arc, - pub active_provider: &'a Arc, - pub connection_manager: &'a Arc, -} - -impl SharedStreamManager { - pub async fn reserve_subscriber_cleanup( - connection_manager: &ConnectionManager, - subscriber_id: SharedSubscriberId, - addr: SocketAddr, - ) -> Result { - if connection_manager.is_shutting_down() { - warn!("Shared stream cleanup rejected during shutdown for subscriber {subscriber_id}"); - return Err(ConnectionRejectionReason::CleanupReceiverClosed); - } - match timeout(SHARED_CLEANUP_ADMISSION_TIMEOUT, connection_manager.cleanup_tx().reserve_owned()).await { - Ok(Ok(permit)) => Ok(PendingSharedSubscriberCleanup::new(permit, subscriber_id, addr)), - Ok(Err(_)) => { - warn!("Shared stream cleanup receiver closed; rejecting subscriber {subscriber_id}"); - Err(ConnectionRejectionReason::CleanupReceiverClosed) - } - Err(_) => { - warn!("Shared stream cleanup admission timed out; rejecting subscriber {subscriber_id}"); - Err(ConnectionRejectionReason::CleanupAdmissionTimeout) - } - } - } - - pub fn new(provider_manager: Arc) -> Self { - Self { - provider_manager, - shared_streams: RwLock::new(SharedStreamsRegister::default()), - meter_uids: std::sync::Mutex::new(HashMap::new()), - #[cfg(test)] - test_preflight_barrier: std::sync::Mutex::new(None), - } - } - - pub async fn get_shared_state(&self, stream_url: &str) -> Option> { - self.shared_streams.read().await.by_key.get(stream_url).map(Arc::clone) - } - - pub async fn get_shared_state_headers(&self, stream_url: &str) -> Option> { - self.get_shared_state(stream_url).await.map(|s| s.headers.clone()) - } - - pub async fn resource_counts(&self) -> (usize, usize) { - let register = self.shared_streams.read().await; - (register.by_key.len(), register.key_by_subscriber.len()) - } - - fn lock_meter_uids(&self) -> std::sync::MutexGuard<'_, HashMap, SharedMeterEntry>> { - self.meter_uids.lock().unwrap_or_else(|poisoned| { - warn!("Recovering poisoned shared-stream meter registry"); - poisoned.into_inner() - }) - } - - pub fn reserve_meter_uid( - self: &Arc, - stream_url: &str, - uid_factory: impl FnOnce() -> u32, - ) -> (u32, Option) { - let mut uids = self.lock_meter_uids(); - if let Some(entry) = uids.get(stream_url) { - return (entry.uid, None); - } - let stream_url: Arc = Arc::from(stream_url); - let meter_uid = uid_factory(); - uids.insert(Arc::clone(&stream_url), SharedMeterEntry { uid: meter_uid, owner: None }); - drop(uids); - ( - meter_uid, - Some(PendingSharedMeterRegistration { manager: Arc::clone(self), stream_url, meter_uid, armed: true }), - ) - } - - /// Binds the reserved meter entry to the origin that just committed its shared - /// origin. Called only while the registry write lock is held and only after the URL - /// was confirmed absent from `by_key`, so the previous origin (if any) is already - /// torn down and this new origin is the sole owner. Overwriting the owner is what - /// keeps a successor's meter from being removed by a stale predecessor teardown. - pub fn adopt_meter_uid(&self, stream_url: &str, allocation_id: AllocationId) { - let mut uids = self.lock_meter_uids(); - if let Some(entry) = uids.get_mut(stream_url) { - entry.owner = Some(allocation_id); - } - } - - fn remove_meter_uid_if_pending(&self, stream_url: &str, meter_uid: u32) { - let mut uids = self.lock_meter_uids(); - if let Some(entry) = uids.get(stream_url) { - if entry.uid == meter_uid && entry.owner.is_none() { - uids.remove(stream_url); - } - } - } - - pub fn meter_count(&self) -> usize { self.lock_meter_uids().len() } - - fn remove_meter_uid_if_owned_by(&self, stream_url: &str, allocation_id: Option) { - let mut uids = self.lock_meter_uids(); - let remove = uids.get(stream_url).is_some_and(|entry| match allocation_id { - Some(id) => entry.owner == Some(id), - None => entry.owner.is_none(), - }); - if remove { - uids.remove(stream_url); - } - } - - async fn finish_unregister(&self, stream_url: &str, state: &SharedStreamState) { - self.remove_meter_uid_if_owned_by(stream_url, state.provider_guard.as_ref().map(|handle| handle.allocation_id)); - state.cancel_subscribers().await; - state.stop_token.cancel(); - if let Some(handle) = &state.provider_guard { - self.provider_manager.release_handle(handle); - } - } - - async fn unregister(&self, stream_url: &str, expected: &Arc) { - let mut register = self.shared_streams.write().await; - let state = { - if !register.by_key.get(stream_url).is_some_and(|state| Arc::ptr_eq(state, expected)) { - return; - } - register.key_by_subscriber.retain(|_, url| url.as_ref() != stream_url); - register.by_key.remove(stream_url) - }; - drop(register); - if let Some(state) = state { - self.finish_unregister(stream_url, &state).await; - } - } - - pub async fn teardown_preempted_stream(&self, stream_url: &str, allocation_id: u64) { - let mut register = self.shared_streams.write().await; - if !register.by_key.get(stream_url).is_some_and(|state| { - state.provider_guard.as_ref().is_some_and(|handle| handle.allocation_id == allocation_id) - }) { - return; - } - let state = { - register.key_by_subscriber.retain(|_, url| url.as_ref() != stream_url); - register.by_key.remove(stream_url) - }; - drop(register); - self.remove_meter_uid_if_owned_by(stream_url, Some(allocation_id)); - if let Some(state) = state { - state.preempted_token.cancel(); - state.stop_token.cancel(); - } - } - - /// Releases all subscribers on a closed transport. Playback cleanup uses the id variant. - pub async fn release_connection(&self, addr: &SocketAddr, _send_stop_signal: bool) { - let states: Vec> = { - let register = self.shared_streams.read().await; - register.by_key.values().cloned().collect() - }; - let mut ids = Vec::new(); - for state in states { - ids.extend( - state - .subscribers - .read() - .await - .iter() - .filter_map(|(id, subscriber)| (subscriber.addr == *addr).then_some(*id)), - ); - } - for id in ids { - self.release_subscriber(id).await; - } - } - - pub async fn release_subscriber(&self, subscriber_id: SubscriberId) { - let mut register = self.shared_streams.write().await; - let stopped = { - // Registry then subscribers is the shared-stream lock order. Joining and - // removing the final subscriber must not race an origin replacement. - if let Some(url) = register.key_by_subscriber.remove(&subscriber_id) { - if let Some(state) = register.by_key.get(&url).cloned() { - let mut subscribers = state.subscribers.write().await; - if let Some(subscriber) = subscribers.remove(&subscriber_id) { - subscriber.cancel_token.cancel(); - } - if subscribers.is_empty() && !state.has_pending_joins() { - register.by_key.remove(&url); - drop(subscribers); - Some((url, state)) - } else { - None - } - } else { - None - } - } else { - None - } - }; - drop(register); - self.provider_manager.release_shared_connection(subscriber_id); - if let Some((url, state)) = stopped { - self.finish_unregister(&url, &state).await; - } - } - - pub async fn shutdown(&self) { - let stopped = { - let mut register = self.shared_streams.write().await; - register.key_by_subscriber.clear(); - register.by_key.drain().collect::>() - }; - for (url, state) in stopped { - self.finish_unregister(&url, &state).await; - // Join the broadcast and forwarder tasks for this origin so teardown is - // complete before the manager (and its runtime) is dropped. Cancellation has - // already fired, so these tasks only finish their terminal cleanup. - let handles = std::mem::take(&mut *state.lock_task_handles()); - for handle in handles { - let _ = handle.await; - } - } - self.lock_meter_uids().clear(); - } - - async fn subscribe_stream( - &self, - stream_url: &str, - addr: &SocketAddr, - subscriber_id: SubscriberId, - connection_manager: Arc, - user_priority: i8, - connection_kind: crate::active_provider_manager::ConnectionKind, - ) -> Result>, SharedCleanupCapability)>, ConnectionRejectionReason> - { - let Some(_admission) = connection_manager.begin_admission().await else { - return Err(ConnectionRejectionReason::CleanupReceiverClosed); - }; - { - let register = self.shared_streams.read().await; - if !register.by_key.contains_key(stream_url) { - return Ok(None); - } - } - #[cfg(test)] - { - let barrier = self.test_preflight_barrier.lock().ok().and_then(|g| g.clone()); - if let Some(barrier) = barrier { - barrier.wait().await; - } - } - let mut pending_cleanup = Self::reserve_subscriber_cleanup(&connection_manager, subscriber_id, *addr).await?; - // Keep the global registry lock only for the lookup and the key_by_subscriber - // commit. The origin's own subscriber/task locks are taken during subscribe, so a - // slow origin must not block joins on other URLs. - let (state, pending_join) = { - let mut register = self.shared_streams.write().await; - let Some((key, state)) = - register.by_key.get_key_value(stream_url).map(|(key, state)| (Arc::clone(key), Arc::clone(state))) - else { - pending_cleanup.disarm(); - return Ok(None); - }; - if let Err(err) = self.provider_manager.add_shared_connection( - addr, - subscriber_id, - stream_url, - user_priority, - connection_kind, - ) { - warn!("Failed joining shared stream: {}", sanitize_sensitive_info(&err)); - pending_cleanup.disarm(); - return Ok(None); - } - register.key_by_subscriber.insert(subscriber_id, key); - let pending_join = PendingJoinGuard::new(&state); - (state, pending_join) - }; - Ok(Some( - state.subscribe(addr, subscriber_id, Arc::clone(&connection_manager), pending_cleanup, pending_join).await, - )) - } - - #[allow(clippy::too_many_arguments)] - pub async fn register_shared_stream( - ctx: SharedStreamCtx<'_>, - stream_url: &str, - bytes_stream: S, - addr: &SocketAddr, - subscriber_id: SubscriberId, - headers: Vec<(String, String)>, - buffer_size: usize, - mut provider_handle: Option, - pending_cleanup: PendingSharedSubscriberCleanup, - user_priority: i8, - connection_kind: crate::active_provider_manager::ConnectionKind, - ) -> Option<(BoxedProviderStream, Option>, SharedCleanupCapability)> - where - S: Stream> + Unpin + 'static + Send, - E: std::fmt::Debug + Send, - { - let _admission = ctx.connection_manager.begin_admission().await?; - let registration_started_at = Instant::now(); - let buf_size = CHANNEL_SIZE.max(buffer_size); - let config = ctx.app_config.config.load(); - let min_buffer_bytes = resolve_min_burst_buffer_bytes(&config); - let low_priority_preempted = - ctx.app_config.custom_stream_response.load().as_ref().and_then(|c| c.low_priority_preempted.clone()); - let mut register = ctx.shared_stream_manager.shared_streams.write().await; - if let Some((key, existing)) = - register.by_key.get_key_value(stream_url).map(|(key, state)| (Arc::clone(key), Arc::clone(state))) - { - drop(provider_handle.take()); - if ctx - .active_provider - .add_shared_connection(addr, subscriber_id, stream_url, user_priority, connection_kind) - .is_err() - { - return None; - } - register.key_by_subscriber.insert(subscriber_id, key); - let pending_join = PendingJoinGuard::new(&existing); - drop(register); - let response = existing - .subscribe(addr, subscriber_id, Arc::clone(ctx.connection_manager), pending_cleanup, pending_join) - .await; - return Some(response); - } - let handle = provider_handle.as_ref().and_then(|managed| managed.handle())?; - if !ctx.active_provider.make_shared_connection(handle, stream_url, subscriber_id) { - drop(register); - drop(provider_handle.take()); - return None; - } - let allocation_id = handle.allocation_id; - let raw_handle = provider_handle.and_then(|mut managed| managed.disarm())?; - let mut shared_state = - SharedStreamState::new(headers, buf_size, Some(raw_handle), min_buffer_bytes, low_priority_preempted) - .with_subscriber_idle_timeout_secs( - config - .reverse_proxy - .as_ref() - .and_then(|reverse_proxy| reverse_proxy.stream.as_ref()) - .map_or(DEFAULT_SUBSCRIBER_IDLE_TIMEOUT_SECS, |stream| { - stream.shared_subscriber_idle_timeout_secs - }), - ); - shared_state.subscriber_max_duration = - config.sleep_timer_mins.filter(|mins| *mins > 0).map(|mins| Duration::from_secs(u64::from(mins) * 60)); - let shared_state = Arc::new(shared_state); - let stream_key: Arc = Arc::from(stream_url); - register.by_key.insert(Arc::clone(&stream_key), Arc::clone(&shared_state)); - register.key_by_subscriber.insert(subscriber_id, stream_key); - ctx.shared_stream_manager.adopt_meter_uid(stream_url, allocation_id); - let pending_join = PendingJoinGuard::new(&shared_state); - drop(register); - let subscribed_stream = shared_state - .subscribe(addr, subscriber_id, Arc::clone(ctx.connection_manager), pending_cleanup, pending_join) - .await; - debug_if_enabled!( - "Shared stream startup register+subscribe completed for {} in {} ms", - sanitize_sensitive_info(stream_url), - registration_started_at.elapsed().as_millis() - ); - shared_state.broadcast(stream_url, bytes_stream, Arc::clone(ctx.shared_stream_manager)); - Some(subscribed_stream) - } - - pub async fn subscribe_shared_stream( - ctx: SharedStreamCtx<'_>, - stream_url: &str, - addr: &SocketAddr, - subscriber_id: SubscriberId, - user_priority: i8, - connection_kind: crate::active_provider_manager::ConnectionKind, - ) -> Result>, SharedCleanupCapability)>, ConnectionRejectionReason> - { - ctx.shared_stream_manager - .subscribe_stream( - stream_url, - addr, - subscriber_id, - Arc::clone(ctx.connection_manager), - user_priority, - connection_kind, - ) - .await - } -} - #[cfg(test)] -mod tests { - use super::{ - send_client_chunk, BudgetedChunk, BurstBuffer, SendOutcome, SharedStreamManager, SharedStreamState, - CHANNEL_SIZE, MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES, SHARED_CLEANUP_ADMISSION_TIMEOUT, - }; - use crate::{ - ActiveProviderManager, ActiveUserConnectionParams, ActiveUserManager, ConnectionKind, ConnectionManager, - ConnectionRejectionReason, EventManager, ManagedProviderHandle, - }; - use arc_swap::{ArcSwap, ArcSwapOption}; - use bytes::Bytes; - use futures::StreamExt; - use shared::{ - model::{ConfigPaths, InputFetchMethod, InputType, PlaylistItemType, StreamChannel, XtreamCluster}, - utils::Internable, - }; - use std::{borrow::Cow, collections::HashMap, net::SocketAddr, sync::Arc}; - use tokio::{ - sync::{mpsc, Semaphore}, - time::{timeout, Duration, Instant}, - }; - use tokio_stream::wrappers::ReceiverStream; - use tokio_util::sync::CancellationToken; - use tuliprox_core::{ - model::{ - AppConfig, Config, ConfigInput, Fingerprint, MediaToolCapabilities, SharedSubscriberId, SourcesConfig, - }, - utils::FileLockManager, - }; - - fn create_test_app_config() -> AppConfig { create_test_app_config_with_conns(1) } - - fn create_test_app_config_with_conns(max_connections: u16) -> AppConfig { - let input = Arc::new(ConfigInput { - id: 1, - name: "provider_1".intern(), - input_type: InputType::Xtream, - headers: HashMap::default(), - url: "http://provider-1.example".to_string(), - username: Some("user1".to_string()), - password: Some("pass1".to_string()), - enabled: true, - priority: 0, - max_connections, - method: InputFetchMethod::default(), - aliases: None, - ..ConfigInput::default() - }); - - let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; - - AppConfig { - config: Arc::new(ArcSwap::from_pointee(Config::default())), - sources: Arc::new(ArcSwap::from_pointee(sources)), - hdhomerun: Arc::new(ArcSwapOption::default()), - api_proxy: Arc::new(ArcSwapOption::default()), - file_locks: Arc::new(FileLockManager::default()), - paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { - home_path: String::new(), - config_path: String::new(), - storage_path: String::new(), - config_file_path: String::new(), - sources_file_path: String::new(), - mapping_file_path: None, - mapping_files_used: None, - template_file_path: None, - template_files_used: None, - api_proxy_file_path: String::new(), - custom_stream_response_path: None, - })), - custom_stream_response: Arc::new(ArcSwapOption::default()), - access_token_secret: [0; 32], - encrypt_secret: [0; 16], - media_tools: Arc::new(MediaToolCapabilities::new()), - } - } - - fn create_test_stream_channel(url: &str) -> StreamChannel { - StreamChannel { - target_id: 1, - virtual_id: 1, - provider_id: 1, - input_name: "provider_1".intern(), - item_type: PlaylistItemType::Live, - cluster: XtreamCluster::Live, - group: "group".intern(), - title: "title".intern(), - url: url.intern(), - shared: true, - shared_joined_existing: Some(false), - shared_stream_id: None, - technical: None, - epg_channel_id: None, - epg_reference_ts: None, - upstream_user_agent: None, - } - } - - fn create_test_connection_manager( - app_cfg: &AppConfig, - event_manager: &Arc, - ) -> (Arc, Arc, Arc, Arc) { - let provider_manager = Arc::new(ActiveProviderManager::new(app_cfg, event_manager)); - let geoip = Arc::new(ArcSwapOption::default()); - let user_manager = Arc::new(ActiveUserManager::new(&Config::default(), &geoip, event_manager)); - let shared_manager = Arc::new(SharedStreamManager::new(Arc::clone(&provider_manager))); - let connection_manager = - Arc::new(ConnectionManager::new(&user_manager, &provider_manager, &shared_manager, event_manager, None)); - (provider_manager, user_manager, shared_manager, connection_manager) - } - - async fn register_user( - users: &ActiveUserManager, - addr: SocketAddr, - uid: u32, - username: &str, - stream_url: &str, - ) -> Result<(), Box> { - let fingerprint = Fingerprint::new(username.to_string(), username.to_string(), addr); - let channel = create_test_stream_channel(stream_url); - let stream = users - .update_connection(ActiveUserConnectionParams { - uid, - meter_uid: 0, - username, - max_connections: 5, - soft_connections: 0, - connection_kind: ConnectionKind::Normal, - priority: 0, - soft_priority: 0, - fingerprint: &fingerprint, - provider: "provider_1".intern(), - stream_channel: &channel, - user_agent: Cow::Borrowed("test"), - session_token: None, - }) - .await - .ok_or("user stream missing")?; - assert_eq!(stream.uid, uid); - Ok(()) - } - - #[tokio::test] - async fn subscribers_on_same_socket_do_not_replace_each_other() -> Result<(), Box> { - let state = SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None); - let addr = "127.0.0.1:41003".parse()?; - let first = CancellationToken::new(); - let second = CancellationToken::new(); - state.register_subscriber(SharedSubscriberId::from_stream_uid(1), &addr, first.clone()).await; - state.register_subscriber(SharedSubscriberId::from_stream_uid(2), &addr, second.clone()).await; - assert_eq!(state.subscribers.read().await.len(), 2); - assert!(!first.is_cancelled()); - assert!(!second.is_cancelled()); - Ok(()) - } - - #[tokio::test] - async fn send_client_chunk_returns_when_cancelled_while_queue_is_full() -> Result<(), Box> { - let (tx, _rx) = mpsc::channel::(1); - let byte_budget = Arc::new(Semaphore::new(1024)); - let queued_permit = byte_budget.clone().acquire_many_owned(6).await?; - tx.send(BudgetedChunk { bytes: Bytes::from_static(b"queued"), _permit: queued_permit }).await?; - let cancel = CancellationToken::new(); - cancel.cancel(); - let outcome = timeout( - Duration::from_secs(1), - send_client_chunk( - &tx, - Bytes::from_static(b"blocked"), - &cancel, - Instant::now() + Duration::from_secs(1), - &byte_budget, - ), - ) - .await?; - assert_eq!(outcome, SendOutcome::Cancelled); - Ok(()) - } - - #[tokio::test] - async fn send_client_chunk_respects_progress_deadline_while_queue_is_full() -> Result<(), Box> - { - let (tx, _rx) = mpsc::channel::(1); - let byte_budget = Arc::new(Semaphore::new(1024)); - let queued_permit = byte_budget.clone().acquire_many_owned(6).await?; - tx.send(BudgetedChunk { bytes: Bytes::from_static(b"queued"), _permit: queued_permit }).await?; - let cancel = CancellationToken::new(); - let deadline = Instant::now() + Duration::from_millis(50); - let outcome = timeout( - Duration::from_secs(1), - send_client_chunk(&tx, Bytes::from_static(b"blocked"), &cancel, deadline, &byte_budget), - ) - .await?; - assert_eq!(outcome, SendOutcome::TimedOut); - Ok(()) - } - - #[tokio::test] - async fn send_client_chunk_respects_byte_budget() -> Result<(), Box> { - let (tx, _rx) = mpsc::channel::(16); - // Byte budget of 4 bytes cannot accommodate a 6-byte chunk, so the acquire must - // time out rather than exceed the budget. - let byte_budget = Arc::new(Semaphore::new(4)); - let cancel = CancellationToken::new(); - let deadline = Instant::now() + Duration::from_millis(50); - let outcome = timeout( - Duration::from_secs(1), - send_client_chunk(&tx, Bytes::from_static(b"123456"), &cancel, deadline, &byte_budget), - ) - .await?; - assert_eq!(outcome, SendOutcome::TimedOut); - Ok(()) - } - - #[tokio::test] - async fn shared_clients_on_same_socket_keep_independent_streams_and_capacity( - ) -> Result<(), Box> { - let app_cfg = Arc::new(create_test_app_config()); - let events = Arc::new(EventManager::new()); - let (providers, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); - let addr = "127.0.0.1:41004".parse()?; - let first_id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let second_id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let url = "https://example.invalid/live/shared.ts"; - register_user(&users, addr, first_id.stream_uid(), "first", url).await?; - let allocation = ManagedProviderHandle::new( - Arc::clone(&providers), - providers - .acquire_connection(&"provider_1".intern(), &addr, 0, ConnectionKind::Normal) - .ok_or("provider allocation missing")?, - ); - let ctx = super::SharedStreamCtx { - app_config: &app_cfg, - shared_stream_manager: &manager, - active_provider: &providers, - connection_manager: &connections, - }; - let (tx, rx) = mpsc::channel::>(8); - let pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, first_id, addr) - .await - .map_err(|_| "shared cleanup admission failed")?; - let (mut first, _, _) = SharedStreamManager::register_shared_stream( - ctx, - url, - ReceiverStream::new(rx), - &addr, - first_id, - Vec::new(), - 8, - Some(allocation), - pending_cleanup, - 0, - ConnectionKind::Normal, - ) - .await - .ok_or("first subscription missing")?; - let (mut second, _, _) = - SharedStreamManager::subscribe_shared_stream(ctx, url, &addr, second_id, 0, ConnectionKind::Normal) - .await - .map_err(|_| "second shared cleanup admission failed")? - .ok_or("second subscription missing")?; - register_user(&users, addr, second_id.stream_uid(), "second", url).await?; - - tx.send(Ok(Bytes::from_static(b"first chunk"))).await?; - assert_eq!( - timeout(Duration::from_secs(2), first.next()).await?.ok_or("first ended")??, - Bytes::from_static(b"first chunk") - ); - assert_eq!( - timeout(Duration::from_secs(2), second.next()).await?.ok_or("second ended")??, - Bytes::from_static(b"first chunk") - ); - assert_eq!(users.active_streams().await.len(), 2); - assert_eq!(providers.get_provider_connections_count(), 1); - - drop(first); - timeout(Duration::from_secs(2), async { - while users.active_streams().await.len() != 1 { - tokio::task::yield_now().await; - } - }) - .await?; - // A delayed duplicate cleanup cannot remove the surviving subscriber. - connections.send_cleanup(crate::CleanupEvent::ReleaseSharedSubscriber { - addr, - subscriber_id: first_id, - request_id: None, - owner: None, - }); - tx.send(Ok(Bytes::from_static(b"second chunk"))).await?; - assert_eq!( - timeout(Duration::from_secs(2), second.next()).await?.ok_or("survivor ended")??, - Bytes::from_static(b"second chunk") - ); - assert_eq!(providers.get_provider_connections_count(), 1); - assert_eq!(users.active_streams().await.first().map(|stream| stream.uid), Some(second_id.stream_uid())); - - drop(second); - timeout(Duration::from_secs(2), async { - while !users.active_streams().await.is_empty() || providers.get_provider_connections_count() != 0 { - tokio::task::yield_now().await; - } - }) - .await?; - assert!(manager.get_shared_state(url).await.is_none()); - Ok(()) - } - - #[tokio::test] - async fn dropping_unpolled_shared_response_releases_its_subscription() -> Result<(), Box> { - let app_cfg = Arc::new(create_test_app_config()); - let events = Arc::new(EventManager::new()); - let (providers, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); - let addr = "127.0.0.1:41005".parse()?; - let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let url = "https://example.invalid/live/unpolled.ts"; - register_user(&users, addr, id.stream_uid(), "first", url).await?; - let allocation = ManagedProviderHandle::new( - Arc::clone(&providers), - providers - .acquire_connection(&"provider_1".intern(), &addr, 0, ConnectionKind::Normal) - .ok_or("provider allocation missing")?, - ); - let pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr) - .await - .map_err(|_| "shared cleanup admission failed")?; - let response = SharedStreamManager::register_shared_stream( - super::SharedStreamCtx { - app_config: &app_cfg, - shared_stream_manager: &manager, - active_provider: &providers, - connection_manager: &connections, - }, - url, - futures::stream::pending::>(), - &addr, - id, - Vec::new(), - 8, - Some(allocation), - pending_cleanup, - 0, - ConnectionKind::Normal, - ) - .await - .ok_or("subscription missing")?; - drop(response); - timeout(Duration::from_secs(2), async { - while !users.active_streams().await.is_empty() || providers.get_provider_connections_count() != 0 { - tokio::task::yield_now().await; - } - }) - .await?; - assert!(manager.shared_streams.read().await.key_by_subscriber.is_empty()); - Ok(()) - } - - #[tokio::test(start_paused = true)] - async fn saturated_cleanup_queue_rejects_shared_subscriber_before_registration( - ) -> Result<(), Box> { - let app_cfg = create_test_app_config(); - let events = Arc::new(EventManager::new()); - let (_, _, _, connections) = create_test_connection_manager(&app_cfg, &events); - let cleanup_tx = connections.cleanup_tx(); - let available = cleanup_tx.capacity(); - let mut permits = Vec::with_capacity(available); - for _ in 0..available { - permits.push(cleanup_tx.clone().try_reserve_owned()?); - } - - let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); - let addr = "127.0.0.1:41008".parse()?; - let subscriber_id = SharedSubscriberId::from_stream_uid(88); - let subscribe_connections = Arc::clone(&connections); - let subscribe = tokio::spawn(async move { - SharedStreamManager::reserve_subscriber_cleanup(&subscribe_connections, subscriber_id, addr).await - }); - - tokio::task::yield_now().await; - tokio::time::advance(SHARED_CLEANUP_ADMISSION_TIMEOUT + Duration::from_millis(1)).await; - assert!(subscribe.await?.is_err(), "saturated cleanup admission must reject the subscriber"); - assert!(state.subscribers.read().await.is_empty(), "rejected admission must not register a subscriber"); - assert!(state.lock_task_handles().is_empty(), "rejected admission must not spawn a forwarding task"); - - drop(permits); - Ok(()) - } - - #[tokio::test] - async fn shared_timeout_ends_only_its_subscriber() -> Result<(), Box> { - let app_cfg = create_test_app_config(); - let events = Arc::new(EventManager::new()); - let (_, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); - let addr = "127.0.0.1:41007".parse()?; - let first = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let second = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let url: Arc = Arc::from("https://example.invalid/timeout.ts"); - register_user(&users, addr, first.stream_uid(), "same-user", &url).await?; - register_user(&users, addr, second.stream_uid(), "same-user", &url).await?; - let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); - let surviving_token = CancellationToken::new(); - state.register_subscriber(first, &addr, CancellationToken::new()).await; - state.register_subscriber(second, &addr, surviving_token.clone()).await; - { - let mut register = manager.shared_streams.write().await; - register.by_key.insert(Arc::clone(&url), state); - register.key_by_subscriber.insert(first, Arc::clone(&url)); - register.key_by_subscriber.insert(second, Arc::clone(&url)); - } - let permit = connections.cleanup_tx().reserve_owned().await.ok(); - let mut response = super::ReceiverStreamWrapper { - stream: futures::stream::pending::(), - start: None, - subscriber_id: first, - addr, - permit, - request_id: None, - owner: None, - deadline: Some(Box::pin(tokio::time::sleep(Duration::ZERO))), - released: false, - }; - assert!(timeout(Duration::from_secs(2), response.next()).await?.is_none()); - timeout(Duration::from_secs(2), async { - while users.active_streams().await.len() != 1 { - tokio::task::yield_now().await; - } - }) - .await?; - assert_eq!(users.active_streams().await.first().map(|stream| stream.uid), Some(second.stream_uid())); - assert!(!surviving_token.is_cancelled()); - assert!(manager.get_shared_state(&url).await.is_some()); - Ok(()) - } - - #[tokio::test] - async fn old_origin_cannot_unregister_replacement() { - let app_cfg = create_test_app_config(); - let events = Arc::new(EventManager::new()); - let (_, _, manager, _) = create_test_connection_manager(&app_cfg, &events); - let url = "https://example.invalid/live/replaced.ts"; - let old = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); - let current = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); - manager.shared_streams.write().await.by_key.insert(Arc::from(url), Arc::clone(¤t)); - manager.unregister(url, &old).await; - assert!(manager.get_shared_state(url).await.is_some_and(|state| Arc::ptr_eq(&state, ¤t))); - assert!(!current.stop_token.is_cancelled()); - } - - #[tokio::test] - async fn duplicate_subscriber_release_preserves_other_channel_on_same_socket( - ) -> Result<(), Box> { - let app_cfg = create_test_app_config(); - let events = Arc::new(EventManager::new()); - let (_, _, manager, _) = create_test_connection_manager(&app_cfg, &events); - let addr = "127.0.0.1:41006".parse()?; - let first = SharedSubscriberId::from_stream_uid(1); - let second = SharedSubscriberId::from_stream_uid(2); - let first_url: Arc = Arc::from("https://example.invalid/first.ts"); - let second_url: Arc = Arc::from("https://example.invalid/second.ts"); - let state_a = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); - let state_b = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); - let surviving_token = CancellationToken::new(); - state_a.register_subscriber(first, &addr, CancellationToken::new()).await; - state_b.register_subscriber(second, &addr, surviving_token.clone()).await; - { - let mut register = manager.shared_streams.write().await; - register.by_key.insert(Arc::clone(&first_url), state_a); - register.by_key.insert(Arc::clone(&second_url), state_b); - register.key_by_subscriber.insert(first, Arc::clone(&first_url)); - register.key_by_subscriber.insert(second, Arc::clone(&second_url)); - } - manager.release_subscriber(first).await; - manager.release_subscriber(first).await; - assert!(manager.get_shared_state(&first_url).await.is_none()); - assert!(manager.get_shared_state(&second_url).await.is_some()); - assert!(!surviving_token.is_cancelled()); - manager.release_connection(&addr, true).await; - assert!(surviving_token.is_cancelled()); - Ok(()) - } - - #[test] - fn test_shared_state_channel_capacity_does_not_scale_with_burst_buffer_bytes() { - let min_burst_buffer_size = 12 * 1024 * 1024; - let state = SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, min_burst_buffer_size, None); - - assert_eq!(state.buf_size, CHANNEL_SIZE); - } - - #[test] - fn test_burst_buffer_eviction_is_byte_bounded() { - let mut buffer = BurstBuffer::new(10); - buffer.push(Bytes::from_static(b"12345")); - buffer.push(Bytes::from_static(b"67890")); - buffer.push(Bytes::from_static(b"abcde")); - - let mut chunks = Vec::new(); - let read = buffer.read_from_into(0, &mut chunks, usize::MAX); - - assert_eq!(buffer.current_bytes, 10); - assert_eq!(read.skipped, 1); - assert_eq!(read.next_sequence, 3); - assert_eq!(chunks.len(), 2); - } - - #[test] - fn test_burst_buffer_eviction_is_chunk_bounded_for_small_packets() { - let max_chunks = 3; - let mut buffer = BurstBuffer::new(MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES * max_chunks); - for _ in 0..max_chunks.saturating_add(1) { - buffer.push(Bytes::from_static(b"x")); - } - - let mut chunks = Vec::new(); - let read = buffer.read_from_into(0, &mut chunks, usize::MAX); - - assert_eq!(buffer.buffer.len(), max_chunks); - assert_eq!(buffer.current_bytes, max_chunks); - assert_eq!(read.skipped, 1); - assert_eq!(chunks.len(), max_chunks); - } - - #[test] - fn test_burst_buffer_chunk_bound_preserves_ts_sized_byte_capacity() { - let max_chunks = 4; - let mut buffer = BurstBuffer::new(MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES * max_chunks); - for _ in 0..max_chunks.saturating_add(1) { - buffer.push(Bytes::from(vec![0_u8; MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES])); - } - - let mut chunks = Vec::new(); - let read = buffer.read_from_into(0, &mut chunks, usize::MAX); - - assert_eq!(buffer.buffer.len(), max_chunks); - assert_eq!(buffer.current_bytes, MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES * max_chunks); - assert_eq!(read.skipped, 1); - assert_eq!(chunks.len(), max_chunks); - } - - #[test] - fn test_burst_buffer_keeps_oversized_packet_as_single_latest_chunk() { - let mut buffer = BurstBuffer::new(4); - buffer.push(Bytes::from_static(b"12345678")); - - let mut chunks = Vec::new(); - let read = buffer.read_from_into(0, &mut chunks, usize::MAX); - - assert_eq!(buffer.buffer.len(), 1); - assert_eq!(buffer.current_bytes, 8); - assert_eq!(read.next_sequence, 1); - assert_eq!(chunks.len(), 1); - } - - #[test] - fn test_burst_buffer_reads_clone_bytes_without_copying_payload() { - let mut buffer = BurstBuffer::new(1024); - let chunk = Bytes::from(vec![1_u8, 2, 3, 4]); - let ptr = chunk.as_ptr(); - buffer.push(chunk); - - let mut chunks = Vec::new(); - let _read = buffer.read_from_into(0, &mut chunks, usize::MAX); - let Some(read_chunk) = chunks.first() else { - panic!("expected one buffered chunk"); - }; - - assert_eq!(read_chunk.as_ptr(), ptr); - } - - #[test] - fn test_burst_buffer_live_batch_is_chunk_bounded() { - let mut buffer = BurstBuffer::new(4096); - for _ in 0..10 { - buffer.push(Bytes::from_static(b"x")); - } - - let mut chunks = Vec::new(); - let first = buffer.read_from_into(0, &mut chunks, 4); - assert_eq!(chunks.len(), 4); - assert_eq!(first.next_sequence, 4); - assert_eq!(first.skipped, 0); - - let second = buffer.read_from_into(first.next_sequence, &mut chunks, 4); - assert_eq!(chunks.len(), 4); - assert_eq!(second.next_sequence, 8); - - let third = buffer.read_from_into(second.next_sequence, &mut chunks, 4); - assert_eq!(chunks.len(), 2); - assert_eq!(third.next_sequence, 10); - } - - #[tokio::test] - async fn test_duplicate_release_connection_is_idempotent_with_single_subscriber() { - let app_cfg = create_test_app_config(); - let event_manager = Arc::new(EventManager::new()); - let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let shared_manager = Arc::new(SharedStreamManager::new(provider_manager)); - - let stream_url = "https://example.invalid/live/single.ts"; - let addr_1: SocketAddr = "127.0.0.1:42001".parse().unwrap_or_else(|_| unreachable!()); - let id = SharedSubscriberId::from_stream_uid(1); - let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE.max(8), None, 1024, None)); - - { - let mut reg = shared_manager.shared_streams.write().await; - reg.by_key.insert(Arc::from(stream_url), Arc::clone(&state)); - reg.key_by_subscriber.insert(id, Arc::from(stream_url)); - } - - state.register_subscriber(id, &addr_1, CancellationToken::new()).await; - - shared_manager.release_connection(&addr_1, false).await; - { - let reg = shared_manager.shared_streams.read().await; - assert!(!reg.by_key.contains_key(stream_url)); - assert!(!reg.key_by_subscriber.contains_key(&id)); - } - { - let subs = state.subscribers.read().await; - assert!(subs.is_empty()); - } - - shared_manager.release_connection(&addr_1, false).await; - { - let reg = shared_manager.shared_streams.read().await; - assert!(!reg.by_key.contains_key(stream_url)); - assert!(!reg.key_by_subscriber.contains_key(&id)); - } - { - let subs = state.subscribers.read().await; - assert!(subs.is_empty()); - } - } - - #[tokio::test] - async fn test_preempted_shared_subscriber_switches_to_low_priority_fallback() { - let app_cfg = create_test_app_config(); - let event_manager = Arc::new(EventManager::new()); - let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let geoip = Arc::new(ArcSwapOption::default()); - let user_manager = Arc::new(ActiveUserManager::new(&Config::default(), &geoip, &event_manager)); - let shared_manager = Arc::new(SharedStreamManager::new(Arc::clone(&provider_manager))); - let connection_manager = - Arc::new(ConnectionManager::new(&user_manager, &provider_manager, &shared_manager, &event_manager, None)); - - let addr: SocketAddr = "127.0.0.1:43001".parse().unwrap_or_else(|_| unreachable!()); - let mut ts_packet = vec![0_u8; 188]; - ts_packet[0] = 0x47; - - let low_priority_fallback = tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer::new(ts_packet); - let state = - Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE.max(8), None, 1024, Some(low_priority_fallback))); - - let subscriber_id = SharedSubscriberId::from_stream_uid(1); - let Ok(pending_cleanup) = - SharedStreamManager::reserve_subscriber_cleanup(&connection_manager, subscriber_id, addr).await - else { - panic!("shared subscriber admission failed"); - }; - let (mut stream, _provider, _capability) = state - .subscribe(&addr, subscriber_id, connection_manager, pending_cleanup, super::PendingJoinGuard::new(&state)) - .await; - - state.preempted_token.cancel(); - drop(state); - - let first = timeout(Duration::from_secs(2), stream.next()).await; - let Ok(maybe_chunk) = first else { panic!("timed out waiting for fallback chunk") }; - let chunk = match maybe_chunk { - Some(Ok(bytes)) => bytes, - Some(Err(err)) => panic!("fallback stream returned error: {err}"), - None => panic!("fallback stream ended unexpectedly"), - }; - assert!(!chunk.is_empty(), "fallback chunk must contain MPEG-TS bytes"); - } - - #[tokio::test(start_paused = true)] - async fn shared_subscription_admission_has_deadline_and_no_unprotected_fallback( - ) -> Result<(), Box> { - let app_cfg = create_test_app_config(); - let events = Arc::new(EventManager::new()); - let (_, _, _, connections) = create_test_connection_manager(&app_cfg, &events); - let addr: SocketAddr = "127.0.0.1:43002".parse()?; - let id = SharedSubscriberId::from_stream_uid(1); - - // Saturate the cleanup queue so the admission await blocks; the shared admission - // deadline must reject the subscriber instead of hanging or falling back. - let pending = || crate::CleanupEvent::Defer(Box::pin(std::future::pending::<()>())); - connections.send_cleanup(pending()); - for _ in 0..4096 { - connections.send_cleanup(pending()); - } - - let admission = tokio::spawn({ - let connections = Arc::clone(&connections); - async move { SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await } - }); - tokio::task::yield_now().await; - tokio::task::yield_now().await; - tokio::time::advance(super::SHARED_CLEANUP_ADMISSION_TIMEOUT + Duration::from_secs(1)).await; - let result = admission.await?; - assert!(result.is_err(), "saturated shared subscription admission must be rejected, not fall back"); - Ok(()) - } - - #[tokio::test] - async fn broadcast_does_not_emit_shared_stream_health_events() { - let app_cfg = create_test_app_config(); - let event_manager = Arc::new(EventManager::new()); - let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); - let shared_manager = Arc::new(SharedStreamManager::new(provider_manager)); - let mut events = event_manager.get_event_channel(); - let stream_url = "https://user:pass@example.invalid/live/health.ts"; - let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE.max(8), None, 1024, None)); - { - let mut reg = shared_manager.shared_streams.write().await; - reg.by_key.insert(Arc::from(stream_url), Arc::clone(&state)); - } - - let (tx, rx) = mpsc::channel::>(8); - state.broadcast(stream_url, ReceiverStream::new(rx), Arc::clone(&shared_manager)); - - assert!( - timeout(Duration::from_millis(100), events.recv()).await.is_err(), - "broadcast startup must not emit runtime events" - ); - - // Pushing data must also not emit a per-chunk health event. - tx.send(Ok(Bytes::from_static(b"payload-1"))).await.unwrap_or_else(|_| panic!("send chunk should succeed")); - assert!( - timeout(Duration::from_millis(100), events.recv()).await.is_err(), - "pushing a chunk must not emit runtime events" - ); - - drop(tx); - let ended = timeout(Duration::from_secs(2), async { - loop { - if shared_manager.get_shared_state(stream_url).await.is_none() { - break; - } - tokio::task::yield_now().await; - } - }) - .await; - assert!(ended.is_ok(), "broadcast must exit after source end"); - assert!( - timeout(Duration::from_millis(100), events.recv()).await.is_err(), - "broadcast shutdown must not emit runtime events" - ); - } - - #[tokio::test] - async fn shared_origin_abort_before_user_registration_releases_physical_and_logical_request( - ) -> Result<(), Box> { - let app_cfg = Arc::new(create_test_app_config()); - let events = Arc::new(EventManager::new()); - let (providers, users, _manager, connections) = create_test_connection_manager(&app_cfg, &events); - let addr = "127.0.0.1:41010".parse()?; - let owner = "shared-owner-1"; - let handle = providers - .acquire_connection_with_lease_for_session( - &"provider_1".intern(), - &addr, - false, - 0, - ConnectionKind::Normal, - Some(crate::active_provider_manager::PlaybackLeaseRef::new( - owner, - tuliprox_core::model::PlaybackKind::LiveTs, - )), - ) - .ok_or("provider allocation missing")?; - let request_id = handle.playback_request_id.ok_or("provider request identity missing")?; - let managed = ManagedProviderHandle::new(Arc::clone(&providers), handle); - let subscriber_id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let mut pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, subscriber_id, addr) - .await - .map_err(|err| format!("shared cleanup admission failed: {err}"))?; - pending_cleanup.set_provider_request_identity(owner, request_id); - assert_eq!(providers.get_provider_connections_count(), 1); - assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).starting, 1); - - drop(managed); - drop(pending_cleanup); - - timeout(Duration::from_secs(2), async { - while providers.provider_lease_usage(&"provider_1".intern()).total() != 0 { - tokio::task::yield_now().await; - } - }) - .await?; - assert_eq!(providers.get_provider_connections_count(), 0); - assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).total(), 0); - assert!(users.active_streams().await.is_empty()); - Ok(()) - } - - #[tokio::test] - async fn shared_origin_registration_rejected_releases_preacquired_handle() -> Result<(), Box> - { - let app_cfg = Arc::new(create_test_app_config()); - let events = Arc::new(EventManager::new()); - let (providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); - let addr = "127.0.0.1:41011".parse()?; - let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let url = "https://example.invalid/live/rejected_preacquired.ts"; - let owner = "rejected-preacquired-owner"; - let handle = providers - .acquire_connection_with_lease_for_session( - &"provider_1".intern(), - &addr, - false, - 0, - ConnectionKind::Normal, - Some(crate::active_provider_manager::PlaybackLeaseRef::new( - owner, - tuliprox_core::model::PlaybackKind::LiveTs, - )), - ) - .ok_or("provider allocation missing")?; - let request_id = handle.playback_request_id.ok_or("provider request identity missing")?; - let managed_handle = ManagedProviderHandle::new(Arc::clone(&providers), handle); - assert_eq!(providers.get_provider_connections_count(), 1); - - let dummy_state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); - manager.shared_streams.write().await.by_key.insert(Arc::from(url), dummy_state); - - let mut pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await?; - pending_cleanup.set_provider_request_identity(owner, request_id); - let res = SharedStreamManager::register_shared_stream( - super::SharedStreamCtx { - app_config: &app_cfg, - shared_stream_manager: &manager, - active_provider: &providers, - connection_manager: &connections, - }, - url, - futures::stream::pending::>(), - &addr, - id, - Vec::new(), - 8, - Some(managed_handle), - pending_cleanup, - 0, - ConnectionKind::Normal, - ) - .await; - - assert!(res.is_none()); - timeout(Duration::from_secs(2), async { - while providers.get_provider_connections_count() != 0 - || providers.provider_lease_usage(&"provider_1".intern()).total() != 0 - { - tokio::task::yield_now().await; - } - }) - .await?; - assert_eq!(providers.get_provider_connections_count(), 0); - assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).total(), 0); - Ok(()) - } - - #[tokio::test] - async fn shared_origin_abort_waiting_for_registry_lock_releases_handle() -> Result<(), Box> { - let app_cfg = Arc::new(create_test_app_config()); - let events = Arc::new(EventManager::new()); - let (providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); - let addr = "127.0.0.1:41012".parse()?; - let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let url = "https://example.invalid/live/abort_waiting.ts"; - let owner = "abort-waiting-owner"; - - let lock_guard = manager.shared_streams.write().await; - - let handle = providers - .acquire_connection_with_lease_for_session( - &"provider_1".intern(), - &addr, - false, - 0, - ConnectionKind::Normal, - Some(crate::active_provider_manager::PlaybackLeaseRef::new( - owner, - tuliprox_core::model::PlaybackKind::LiveTs, - )), - ) - .ok_or("provider allocation missing")?; - let request_id = handle.playback_request_id.ok_or("provider request identity missing")?; - let managed_handle = ManagedProviderHandle::new(Arc::clone(&providers), handle); - assert_eq!(providers.get_provider_connections_count(), 1); - - let mut pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await?; - pending_cleanup.set_provider_request_identity(owner, request_id); - - let register_task = tokio::spawn({ - let app_cfg = Arc::clone(&app_cfg); - let manager = Arc::clone(&manager); - let providers = Arc::clone(&providers); - let connections = Arc::clone(&connections); - async move { - let ctx = super::SharedStreamCtx { - app_config: &app_cfg, - shared_stream_manager: &manager, - active_provider: &providers, - connection_manager: &connections, - }; - SharedStreamManager::register_shared_stream( - ctx, - url, - futures::stream::pending::>(), - &addr, - id, - Vec::new(), - 8, - Some(managed_handle), - pending_cleanup, - 0, - ConnectionKind::Normal, - ) - .await - } - }); - - tokio::task::yield_now().await; - - register_task.abort(); - let _ = register_task.await; - - drop(lock_guard); - - timeout(Duration::from_secs(2), async { - while providers.get_provider_connections_count() != 0 - || providers.provider_lease_usage(&"provider_1".intern()).total() != 0 - { - tokio::task::yield_now().await; - } - }) - .await?; - assert_eq!(providers.get_provider_connections_count(), 0); - assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).total(), 0); - Ok(()) - } - - #[tokio::test] - async fn concurrent_shared_origin_creation_releases_only_losing_allocation( - ) -> Result<(), Box> { - let app_cfg = Arc::new(create_test_app_config_with_conns(2)); - let events = Arc::new(EventManager::new()); - let (providers, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); - let addr_1 = "127.0.0.1:41013".parse()?; - let addr_2 = "127.0.0.1:41014".parse()?; - let id_1 = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let id_2 = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let url = "https://example.invalid/live/concurrent_origin.ts"; - - let handle_1 = providers - .acquire_connection(&"provider_1".intern(), &addr_1, 0, ConnectionKind::Normal) - .ok_or("first provider allocation missing")?; - let managed_1 = ManagedProviderHandle::new(Arc::clone(&providers), handle_1); - let pending_1 = SharedStreamManager::reserve_subscriber_cleanup(&connections, id_1, addr_1).await?; - let (first_stream, _, _) = SharedStreamManager::register_shared_stream( - super::SharedStreamCtx { - app_config: &app_cfg, - shared_stream_manager: &manager, - active_provider: &providers, - connection_manager: &connections, - }, - url, - futures::stream::pending::>(), - &addr_1, - id_1, - Vec::new(), - 8, - Some(managed_1), - pending_1, - 0, - ConnectionKind::Normal, - ) - .await - .ok_or("first origin registration failed")?; - - assert_eq!(providers.get_provider_connections_count(), 1); - - let handle_2 = providers - .acquire_connection(&"provider_1".intern(), &addr_2, 0, ConnectionKind::Normal) - .ok_or("second provider allocation missing")?; - let managed_2 = ManagedProviderHandle::new(Arc::clone(&providers), handle_2); - assert_eq!(providers.get_provider_connections_count(), 2); - - let pending_2 = SharedStreamManager::reserve_subscriber_cleanup(&connections, id_2, addr_2).await?; - let (second_stream, _, _) = SharedStreamManager::register_shared_stream( - super::SharedStreamCtx { - app_config: &app_cfg, - shared_stream_manager: &manager, - active_provider: &providers, - connection_manager: &connections, - }, - url, - futures::stream::pending::>(), - &addr_2, - id_2, - Vec::new(), - 8, - Some(managed_2), - pending_2, - 0, - ConnectionKind::Normal, - ) - .await - .ok_or("second subscriber join failed")?; - - assert_eq!(providers.get_provider_connections_count(), 1); - - drop(first_stream); - drop(second_stream); - timeout(Duration::from_secs(2), async { - while providers.get_provider_connections_count() != 0 || !users.active_streams().await.is_empty() { - tokio::task::yield_now().await; - } - }) - .await?; - assert_eq!(providers.get_provider_connections_count(), 0); - Ok(()) - } - - #[tokio::test] - async fn shutdown_rejects_waiting_admission_and_drains_owned_cleanup() -> Result<(), Box> { - let app_cfg = Arc::new(create_test_app_config()); - let events = Arc::new(EventManager::new()); - let (_providers, _users, _manager, connections) = create_test_connection_manager(&app_cfg, &events); - let addr = "127.0.0.1:41015".parse()?; - let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let initial_capacity = connections.cleanup_tx().capacity(); - let held_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await?; - let admission = connections.begin_admission().await.ok_or("admission unexpectedly closed")?; - - let shutdown = tokio::spawn({ - let connections = Arc::clone(&connections); - async move { connections.shutdown().await } - }); - timeout(Duration::from_secs(2), async { - while !connections.is_shutting_down() { - tokio::task::yield_now().await; - } - }) - .await?; - assert!(connections.is_shutting_down()); - - let res = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await; - assert!( - matches!(res, Err(ConnectionRejectionReason::CleanupReceiverClosed)), - "waiting admission must be rejected after shutdown" - ); - assert!(!shutdown.is_finished(), "shutdown must wait for an admitted registration"); - - drop(held_cleanup); - drop(admission); - timeout(Duration::from_secs(2), shutdown).await??; - assert_eq!(connections.cleanup_tx().capacity(), initial_capacity); - Ok(()) - } - - #[tokio::test] - async fn shutdown_with_unpolled_shared_body_leaves_no_requests_or_subscribers( - ) -> Result<(), Box> { - let app_cfg = Arc::new(create_test_app_config()); - let events = Arc::new(EventManager::new()); - let (providers, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); - let addr = "127.0.0.1:41016".parse()?; - let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let url = "https://example.invalid/live/unpolled_shutdown.ts"; - register_user(&users, addr, id.stream_uid(), "shutdown_user", url).await?; - - let handle = providers - .acquire_connection(&"provider_1".intern(), &addr, 0, ConnectionKind::Normal) - .ok_or("provider allocation missing")?; - let managed_handle = ManagedProviderHandle::new(Arc::clone(&providers), handle); - let pending = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await?; - let (unpolled_body, _, _) = SharedStreamManager::register_shared_stream( - super::SharedStreamCtx { - app_config: &app_cfg, - shared_stream_manager: &manager, - active_provider: &providers, - connection_manager: &connections, - }, - url, - futures::stream::pending::>(), - &addr, - id, - Vec::new(), - 8, - Some(managed_handle), - pending, - 0, - ConnectionKind::Normal, - ) - .await - .ok_or("register failed")?; - - assert_eq!(providers.get_provider_connections_count(), 1); - assert_eq!(users.active_streams().await.len(), 1); - - connections.shutdown().await; - - assert_eq!(providers.get_provider_connections_count(), 0); - assert!(users.active_streams().await.is_empty()); - assert!(manager.get_shared_state(url).await.is_none()); - - drop(unpolled_body); - Ok(()) - } - - #[tokio::test] - async fn shared_cold_miss_does_not_consume_cleanup_capacity_or_emit_cleanup( - ) -> Result<(), Box> { - let app_cfg = Arc::new(create_test_app_config()); - let events = Arc::new(EventManager::new()); - let (providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); - let addr = "127.0.0.1:41017".parse()?; - let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let initial_capacity = connections.cleanup_tx().capacity(); - - let ctx = super::SharedStreamCtx { - app_config: &app_cfg, - shared_stream_manager: &manager, - active_provider: &providers, - connection_manager: &connections, - }; - - let res = SharedStreamManager::subscribe_shared_stream( - ctx, - "https://example.invalid/live/non_existent.ts", - &addr, - id, - 0, - ConnectionKind::Normal, - ) - .await?; - - assert!(res.is_none()); - assert_eq!( - connections.cleanup_tx().capacity(), - initial_capacity, - "cold miss must not consume cleanup capacity" - ); - Ok(()) - } - - #[tokio::test] - async fn shared_state_removed_between_preflight_and_commit_leaves_no_claim( - ) -> Result<(), Box> { - let app_cfg = Arc::new(create_test_app_config()); - let events = Arc::new(EventManager::new()); - let (providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); - let addr = "127.0.0.1:41018".parse()?; - let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let url = "https://example.invalid/live/removed_between.ts"; - - let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); - manager.shared_streams.write().await.by_key.insert(Arc::from(url), state); - - let barrier = Arc::new(tokio::sync::Barrier::new(2)); - if let Ok(mut lock) = manager.test_preflight_barrier.lock() { - *lock = Some(Arc::clone(&barrier)); - } - - let initial_capacity = connections.cleanup_tx().capacity(); - - let sub_task = tokio::spawn({ - let app_cfg = Arc::clone(&app_cfg); - let manager = Arc::clone(&manager); - let providers = Arc::clone(&providers); - let connections = Arc::clone(&connections); - async move { - let ctx = super::SharedStreamCtx { - app_config: &app_cfg, - shared_stream_manager: &manager, - active_provider: &providers, - connection_manager: &connections, - }; - SharedStreamManager::subscribe_shared_stream(ctx, url, &addr, id, 0, ConnectionKind::Normal).await - } - }); - - barrier.wait().await; - - { - let mut reg = manager.shared_streams.write().await; - reg.by_key.remove(url); - } - - let res = sub_task.await??; - assert!(res.is_none()); - assert_eq!(connections.cleanup_tx().capacity(), initial_capacity); - assert!(manager.get_shared_state(url).await.is_none()); - Ok(()) - } - - #[tokio::test] - async fn pending_join_keeps_origin_alive_when_last_subscriber_leaves() -> Result<(), Box> { - let app_cfg = Arc::new(create_test_app_config()); - let events = Arc::new(EventManager::new()); - let (_providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); - let addr = "127.0.0.1:41019".parse()?; - let first_id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); - let url = "https://example.invalid/live/pending_join.ts"; - - let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); - state.register_subscriber(first_id, &addr, CancellationToken::new()).await; - { - let mut reg = manager.shared_streams.write().await; - reg.by_key.insert(Arc::from(url), Arc::clone(&state)); - reg.key_by_subscriber.insert(first_id, Arc::from(url)); - } - - // Simulate a join that committed its registry entry but has not yet registered - // its subscriber in the origin. - let pending_join = super::PendingJoinGuard::new(&state); - assert!(state.has_pending_joins()); - - // The last (and only) subscriber leaves; the origin must not be torn down because - // the pending join is about to land on it. - manager.release_subscriber(first_id).await; - assert!(manager.get_shared_state(url).await.is_some(), "origin must survive a pending join"); - - // The join completes; the origin has no subscribers and no pending joins. - pending_join.commit(); - assert!(!state.has_pending_joins()); - assert!(state.subscribers.read().await.is_empty(), "origin has no subscribers after release"); - Ok(()) - } - - #[test] - fn uncommitted_shared_meter_registration_is_rolled_back() { - let app_cfg = create_test_app_config(); - let events = Arc::new(EventManager::new()); - let providers = Arc::new(ActiveProviderManager::new(&app_cfg, &events)); - let manager = Arc::new(SharedStreamManager::new(providers)); - let url = "https://example.invalid/live/meter-rollback.ts"; - - let (meter_uid, pending) = manager.reserve_meter_uid(url, || 41); - assert_eq!(meter_uid, 41); - assert_eq!(manager.meter_count(), 1); - - drop(pending); - assert_eq!(manager.meter_count(), 0); - } - - #[test] - fn pending_meter_rollback_does_not_remove_adopted_entry() { - let app_cfg = create_test_app_config(); - let events = Arc::new(EventManager::new()); - let providers = Arc::new(ActiveProviderManager::new(&app_cfg, &events)); - let manager = Arc::new(SharedStreamManager::new(providers)); - let url = "https://example.invalid/live/meter-replacement.ts"; - - // A reserved the meter; B then commits a shared origin and adopts it. - let (_, stale) = manager.reserve_meter_uid(url, || 51); - manager.adopt_meter_uid(url, 999); - - // A aborts: its pending rollback must not remove the adopted entry. - drop(stale); - assert_eq!(manager.meter_count(), 1); - assert_eq!(manager.lock_meter_uids().get(url).map(|entry| entry.uid), Some(51)); - assert_eq!(manager.lock_meter_uids().get(url).and_then(|entry| entry.owner), Some(999)); - } - - #[test] - fn successor_adoption_overwrites_stale_owner_and_survives_old_teardown() { - let app_cfg = create_test_app_config(); - let events = Arc::new(EventManager::new()); - let providers = Arc::new(ActiveProviderManager::new(&app_cfg, &events)); - let manager = Arc::new(SharedStreamManager::new(providers)); - let url = "https://example.invalid/live/meter-successor.ts"; - - let (uid, guard) = manager.reserve_meter_uid(url, || 71); - manager.adopt_meter_uid(url, 100); - // A successor origin commits the same URL and takes over the meter. - manager.adopt_meter_uid(url, 200); - assert_eq!(manager.lock_meter_uids().get(url).and_then(|entry| entry.owner), Some(200)); - - // A stale teardown by the predecessor must not remove the successor's entry. - manager.remove_meter_uid_if_owned_by(url, Some(100)); - assert_eq!(manager.meter_count(), 1); - assert_eq!(manager.lock_meter_uids().get(url).map(|entry| entry.uid), Some(uid)); - guard.expect("reservation").commit(); - } - - #[tokio::test] - async fn shutdown_clears_shared_meter_registrations() { - let app_cfg = create_test_app_config(); - let events = Arc::new(EventManager::new()); - let providers = Arc::new(ActiveProviderManager::new(&app_cfg, &events)); - let manager = Arc::new(SharedStreamManager::new(providers)); - - let (_, pending) = manager.reserve_meter_uid("https://example.invalid/live/meter-shutdown.ts", || 61); - pending.expect("new registration").commit(); - assert_eq!(manager.meter_count(), 1); - - manager.shutdown().await; - assert_eq!(manager.meter_count(), 0); - } -} +mod tests; + +mod broadcast; +mod buffer; +mod manager; +mod reader; +mod subscription; +use buffer::SubscriberId; +pub use manager::SharedStreamCtx; diff --git a/backend/session/src/streams/shared_stream_manager/broadcast.rs b/backend/session/src/streams/shared_stream_manager/broadcast.rs new file mode 100644 index 000000000..e33452492 --- /dev/null +++ b/backend/session/src/streams/shared_stream_manager/broadcast.rs @@ -0,0 +1,172 @@ +use super::{SharedStreamManager, SharedStreamState, YIELD_COUNTER}; +use bytes::Bytes; +use futures::{Stream, StreamExt}; +use log::warn; +use shared::utils::sanitize_sensitive_info; +use std::sync::Arc; +use tokio::time::{sleep, Duration, Instant}; +use tuliprox_core::utils::{debug_if_enabled, network::request::STREAM_IDLE_TIMEOUT, trace_if_enabled}; + +impl SharedStreamState { + #[allow(clippy::too_many_lines)] + pub(super) fn broadcast( + self: &Arc, + stream_url: &str, + bytes_stream: S, + shared_streams: Arc, + ) where + S: Stream> + Unpin + 'static + Send, + E: std::fmt::Debug + Send, + { + let streaming_url = stream_url.to_string(); + let origin_state = Arc::clone(self); + let stop_token = self.stop_token.clone(); + let burst_buffer = Arc::clone(&self.burst_buffer); + let live_notification = Arc::clone(&self.live_notification); + let broadcast_started_at = Instant::now(); + + let broadcast_handle = tokio::spawn(async move { + let mut source_stream = std::pin::pin!(bytes_stream); + let mut counter = 0_usize; + let idle_timeout = Duration::from_secs(STREAM_IDLE_TIMEOUT); + let idle = sleep(idle_timeout); + tokio::pin!(idle); + let mut first_source_chunk_logged = false; + let mut startup_chunks_seen = 0_usize; + let mut startup_bytes_seen = 0_usize; + let mut startup_stats_logged = false; + // Track the time of the most recent upstream push so the broadcast can + // detect a stalled source before the global idle timeout fires. This is + // the diagnostic signal for H1 (broadcast stall with stale burst replay). + let mut last_push_at: Option = None; + let mut idle_warning_emitted = false; + let idle_warn_threshold = idle_timeout / 2; + + loop { + tokio::select! { + biased; + + () = stop_token.cancelled() => { + debug_if_enabled!( + "No shared stream subscribers left. Closing shared provider stream {}", + sanitize_sensitive_info(&streaming_url) + ); + break; + } + + () = &mut idle => { + debug_if_enabled!( + "Shared stream source idle timeout after {}s for {}", + STREAM_IDLE_TIMEOUT, + sanitize_sensitive_info(&streaming_url) + ); + stop_token.cancel(); + break; + } + + chunk = source_stream.next() => { + // Only successful chunks count as liveness; resetting on Err would let an + // error-spinning source dodge the idle timeout forever + if matches!(chunk, Some(Ok(_))) { + idle.as_mut().reset(Instant::now() + idle_timeout); + } + match chunk { + Some(Ok(data)) => { + let chunk_len = data.len(); + let push_seq = { + let mut buffer = burst_buffer.lock().await; + let seq = buffer.next_sequence; + buffer.push(data); + seq + }; + live_notification.notify_waiters(); + last_push_at = Some(Instant::now()); + idle_warning_emitted = false; + trace_if_enabled!( + "shared_stream.broadcast: push seq={} len={} url={}", + push_seq, + chunk_len, + sanitize_sensitive_info(&streaming_url) + ); + + if !first_source_chunk_logged { + debug_if_enabled!( + "Shared stream source produced first chunk for {} after {} ms", + sanitize_sensitive_info(&streaming_url), + broadcast_started_at.elapsed().as_millis() + ); + first_source_chunk_logged = true; + } + if !startup_stats_logged { + startup_chunks_seen = startup_chunks_seen.saturating_add(1); + startup_bytes_seen = startup_bytes_seen.saturating_add(chunk_len); + if broadcast_started_at.elapsed() >= Duration::from_secs(5) { + debug_if_enabled!( + "Shared stream source startup throughput for {}: chunks={} bytes={} over {} ms", + sanitize_sensitive_info(&streaming_url), + startup_chunks_seen, + startup_bytes_seen, + broadcast_started_at.elapsed().as_millis() + ); + startup_stats_logged = true; + } + } + + counter = counter.saturating_add(1); + if counter >= YIELD_COUNTER { + tokio::task::yield_now().await; + counter = 0; + } + } + Some(Err(e)) => { + trace_if_enabled!( + "Shared stream source error for {}: {:?}", + sanitize_sensitive_info(&streaming_url), + e + ); + tokio::task::yield_now().await; + } + None => { + debug_if_enabled!( + "Shared stream source stream ended for {}", + sanitize_sensitive_info(&streaming_url) + ); + break; + } + } + } + } + + // Edge-triggered stall warning: fires once when the broadcast has + // not seen an upstream push for STREAM_IDLE_TIMEOUT/2 seconds, then + // resets on the next successful push. Operators correlate this + // warning with the "stuck in resending same buffer" symptom (H1). + if let Some(last) = last_push_at { + let stalled_for = last.elapsed(); + if stalled_for >= idle_warn_threshold && !idle_warning_emitted { + warn!( + "shared_stream.broadcast: no upstream bytes for {}s on url={}; \ + source may be stalled. Subscribers will see cached burst until {}s timeout.", + stalled_for.as_secs(), + sanitize_sensitive_info(&streaming_url), + STREAM_IDLE_TIMEOUT + ); + idle_warning_emitted = true; + } + } + } + + debug_if_enabled!( + "Shared stream exiting for {} (last_push_age_secs={})", + sanitize_sensitive_info(&streaming_url), + last_push_at.map_or(0, |t| t.elapsed().as_secs()) + ); + shared_streams.unregister(&streaming_url, &origin_state).await; + }); + + // Keep the broadcast handle so shutdown can join it instead of leaving a detached task. + // Registration is guaranteed: a short-lived synchronous lock avoids the try_write + // failure that would otherwise drop the handle under contention. + self.lock_task_handles().push(broadcast_handle); + } +} diff --git a/backend/session/src/streams/shared_stream_manager/buffer.rs b/backend/session/src/streams/shared_stream_manager/buffer.rs new file mode 100644 index 000000000..a11a2fcc6 --- /dev/null +++ b/backend/session/src/streams/shared_stream_manager/buffer.rs @@ -0,0 +1,181 @@ +use super::{ + BudgetedChunk, BufferedChunk, BurstBuffer, BurstRead, MIN_BURST_BUFFER_CHUNKS, + MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES, +}; +use bytes::Bytes; +use std::{ + collections::VecDeque, + fmt, + fmt::{Debug, Formatter}, + sync::Arc, +}; +use tokio::{ + sync::{mpsc, mpsc::Sender, Semaphore}, + time::{Duration, Instant}, +}; +use tokio_util::sync::CancellationToken; +use tuliprox_core::model::SharedSubscriberId; + +pub(super) type SubscriberId = SharedSubscriberId; + +#[allow(clippy::missing_fields_in_debug)] +impl Debug for BurstBuffer { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("BurstBuffer") + .field("buffer_size", &self.buffer_size) + .field("max_chunks", &self.max_chunks) + .field("current_bytes", &self.current_bytes) + .finish() + } +} + +impl BurstBuffer { + pub fn new(buf_size: usize) -> Self { + Self { + buffer: VecDeque::new(), + buffer_size: buf_size, + max_chunks: Self::max_chunks_for_buffer_size(buf_size), + current_bytes: 0, + next_sequence: 0, + } + } + + pub fn snapshot(&self) -> (Vec, u64) { + (self.buffer.iter().map(|chunk| chunk.bytes.clone()).collect::>(), self.next_sequence) + } + + pub fn read_from_into(&self, next_sequence: u64, chunks: &mut Vec, max_chunks: usize) -> BurstRead { + chunks.clear(); + let earliest_sequence = self.buffer.front().map_or(self.next_sequence, |chunk| chunk.sequence); + let start_sequence = next_sequence.max(earliest_sequence); + let skipped = start_sequence.saturating_sub(next_sequence); + let start_index = self.start_index_for_sequence(start_sequence); + let mut read_next_sequence = start_sequence; + for chunk in self.buffer.range(start_index..).take(max_chunks) { + chunks.push(chunk.bytes.clone()); + read_next_sequence = chunk.sequence.saturating_add(1); + } + + BurstRead { next_sequence: read_next_sequence, skipped } + } + + pub fn push(&mut self, packet: Bytes) { + let packet_len = packet.len(); + while !self.buffer.is_empty() + && (self.buffer.len() >= self.max_chunks + || self.current_bytes.saturating_add(packet_len) > self.buffer_size) + { + if let Some(popped) = self.buffer.pop_front() { + self.current_bytes = self.current_bytes.saturating_sub(popped.bytes.len()); + } + } + self.current_bytes = self.current_bytes.saturating_add(packet_len); + let sequence = self.next_sequence; + self.next_sequence = self.next_sequence.saturating_add(1); + self.buffer.push_back(BufferedChunk { sequence, bytes: packet }); + } + + pub(super) fn start_index_for_sequence(&self, sequence: u64) -> usize { + let mut left = 0_usize; + let mut right = self.buffer.len(); + + while left < right { + let mid = left + ((right - left) / 2); + let mid_sequence = self.buffer.get(mid).map_or(u64::MAX, |chunk| chunk.sequence); + if mid_sequence < sequence { + left = mid.saturating_add(1); + } else { + right = mid; + } + } + + left + } + + pub(super) fn max_chunks_for_buffer_size(buffer_size: usize) -> usize { + buffer_size.div_ceil(MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES).max(MIN_BURST_BUFFER_CHUNKS) + } +} + +/// Terminal outcome of trying to deliver one chunk to a subscriber. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum SendOutcome { + Sent, + Cancelled, + Closed, + TimedOut, +} + +async fn reserve_byte_permit( + byte_budget: &Arc, + len: usize, + cancellation_token: &CancellationToken, + progress_deadline: Instant, +) -> Result { + let permits = u32::try_from(len).unwrap_or(u32::MAX); + tokio::select! { + biased; + () = cancellation_token.cancelled() => Err(SendOutcome::Cancelled), + result = tokio::time::timeout_at(progress_deadline, byte_budget.clone().acquire_many_owned(permits)) => { + match result { + Ok(Ok(permit)) => Ok(permit), + Ok(Err(_)) => Err(SendOutcome::Closed), + Err(_) => Err(SendOutcome::TimedOut), + } + } + } +} + +pub(super) async fn send_burst_buffer( + start_buffer: &[Bytes], + client_tx: &Sender, + cancellation_token: &CancellationToken, + idle_timeout: Duration, + byte_budget: &Arc, +) -> Result { + let mut sent = 0_usize; + let mut last_progress = Instant::now(); + for buf in start_buffer { + let deadline = last_progress + idle_timeout; + match send_client_chunk(client_tx, buf.clone(), cancellation_token, deadline, byte_budget).await { + SendOutcome::Sent => { + sent = sent.saturating_add(1); + last_progress = Instant::now(); + } + outcome => return Err(outcome), + } + } + Ok(sent) +} + +pub(super) async fn send_client_chunk( + client_tx: &Sender, + data: Bytes, + cancellation_token: &CancellationToken, + progress_deadline: Instant, + byte_budget: &Arc, +) -> SendOutcome { + if cancellation_token.is_cancelled() { + return SendOutcome::Cancelled; + } + + let permit = match reserve_byte_permit(byte_budget, data.len(), cancellation_token, progress_deadline).await { + Ok(permit) => permit, + Err(outcome) => return outcome, + }; + + let chunk = BudgetedChunk { bytes: data, _permit: permit }; + match client_tx.try_send(chunk) { + Ok(()) => SendOutcome::Sent, + Err(mpsc::error::TrySendError::Closed(_)) => SendOutcome::Closed, + Err(mpsc::error::TrySendError::Full(chunk)) => tokio::select! { + biased; + () = cancellation_token.cancelled() => SendOutcome::Cancelled, + result = tokio::time::timeout_at(progress_deadline, client_tx.send(chunk)) => match result { + Ok(Ok(())) => SendOutcome::Sent, + Ok(Err(_)) => SendOutcome::Closed, + Err(_) => SendOutcome::TimedOut, + }, + }, + } +} diff --git a/backend/session/src/streams/shared_stream_manager/manager.rs b/backend/session/src/streams/shared_stream_manager/manager.rs new file mode 100644 index 000000000..54fac5e64 --- /dev/null +++ b/backend/session/src/streams/shared_stream_manager/manager.rs @@ -0,0 +1,422 @@ +use super::{ + buffer::SubscriberId, reader::resolve_min_burst_buffer_bytes, PendingJoinGuard, PendingSharedMeterRegistration, + PendingSharedSubscriberCleanup, SharedMeterEntry, SharedStreamManager, SharedStreamState, SharedStreamsRegister, + DEFAULT_SUBSCRIBER_IDLE_TIMEOUT_SECS, SHARED_CLEANUP_ADMISSION_TIMEOUT, +}; +use crate::{ + streams::buffered_stream::CHANNEL_SIZE, ActiveProviderManager, BoxedProviderStream, ConnectionManager, + ConnectionRejectionReason, ManagedProviderHandle, SharedCleanupCapability, +}; +use bytes::Bytes; +use futures::Stream; +use log::warn; +use shared::utils::sanitize_sensitive_info; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; +use tokio::{ + sync::RwLock, + time::{timeout, Duration, Instant}, +}; +use tuliprox_core::{ + model::{AllocationId, AppConfig, SharedSubscriberId}, + utils::debug_if_enabled, +}; + +/// The four state handles the shared-stream paths need. +/// +/// These functions used to take the whole `AppState` and reach into four of its +/// fields. Keeping this slice explicit avoids coupling the session crate to the +/// API server state; the composition root supplies the required handles. +#[derive(Clone, Copy)] +pub struct SharedStreamCtx<'a> { + pub app_config: &'a Arc, + pub shared_stream_manager: &'a Arc, + pub active_provider: &'a Arc, + pub connection_manager: &'a Arc, +} + +impl SharedStreamManager { + pub async fn reserve_subscriber_cleanup( + connection_manager: &ConnectionManager, + subscriber_id: SharedSubscriberId, + addr: SocketAddr, + ) -> Result { + if connection_manager.is_shutting_down() { + warn!("Shared stream cleanup rejected during shutdown for subscriber {subscriber_id}"); + return Err(ConnectionRejectionReason::CleanupReceiverClosed); + } + match timeout(SHARED_CLEANUP_ADMISSION_TIMEOUT, connection_manager.cleanup_tx().reserve_owned()).await { + Ok(Ok(permit)) => Ok(PendingSharedSubscriberCleanup::new(permit, subscriber_id, addr)), + Ok(Err(_)) => { + warn!("Shared stream cleanup receiver closed; rejecting subscriber {subscriber_id}"); + Err(ConnectionRejectionReason::CleanupReceiverClosed) + } + Err(_) => { + warn!("Shared stream cleanup admission timed out; rejecting subscriber {subscriber_id}"); + Err(ConnectionRejectionReason::CleanupAdmissionTimeout) + } + } + } + + pub fn new(provider_manager: Arc) -> Self { + Self { + provider_manager, + shared_streams: RwLock::new(SharedStreamsRegister::default()), + meter_uids: std::sync::Mutex::new(HashMap::new()), + #[cfg(test)] + test_preflight_barrier: std::sync::Mutex::new(None), + } + } + + pub async fn get_shared_state(&self, stream_url: &str) -> Option> { + self.shared_streams.read().await.by_key.get(stream_url).map(Arc::clone) + } + + pub async fn get_shared_state_headers(&self, stream_url: &str) -> Option> { + self.get_shared_state(stream_url).await.map(|s| s.headers.clone()) + } + + pub async fn resource_counts(&self) -> (usize, usize) { + let register = self.shared_streams.read().await; + (register.by_key.len(), register.key_by_subscriber.len()) + } + + pub(super) fn lock_meter_uids(&self) -> std::sync::MutexGuard<'_, HashMap, SharedMeterEntry>> { + self.meter_uids.lock().unwrap_or_else(|poisoned| { + warn!("Recovering poisoned shared-stream meter registry"); + poisoned.into_inner() + }) + } + + pub fn reserve_meter_uid( + self: &Arc, + stream_url: &str, + uid_factory: impl FnOnce() -> u32, + ) -> (u32, Option) { + let mut uids = self.lock_meter_uids(); + if let Some(entry) = uids.get(stream_url) { + return (entry.uid, None); + } + let stream_url: Arc = Arc::from(stream_url); + let meter_uid = uid_factory(); + uids.insert(Arc::clone(&stream_url), SharedMeterEntry { uid: meter_uid, owner: None }); + drop(uids); + ( + meter_uid, + Some(PendingSharedMeterRegistration { manager: Arc::clone(self), stream_url, meter_uid, armed: true }), + ) + } + + /// Binds the reserved meter entry to the origin that just committed its shared + /// origin. Called only while the registry write lock is held and only after the URL + /// was confirmed absent from `by_key`, so the previous origin (if any) is already + /// torn down and this new origin is the sole owner. Overwriting the owner is what + /// keeps a successor's meter from being removed by a stale predecessor teardown. + pub fn adopt_meter_uid(&self, stream_url: &str, allocation_id: AllocationId) { + let mut uids = self.lock_meter_uids(); + if let Some(entry) = uids.get_mut(stream_url) { + entry.owner = Some(allocation_id); + } + } + + pub(super) fn remove_meter_uid_if_pending(&self, stream_url: &str, meter_uid: u32) { + let mut uids = self.lock_meter_uids(); + if let Some(entry) = uids.get(stream_url) { + if entry.uid == meter_uid && entry.owner.is_none() { + uids.remove(stream_url); + } + } + } + + pub fn meter_count(&self) -> usize { self.lock_meter_uids().len() } + + pub(super) fn remove_meter_uid_if_owned_by(&self, stream_url: &str, allocation_id: Option) { + let mut uids = self.lock_meter_uids(); + let remove = uids.get(stream_url).is_some_and(|entry| match allocation_id { + Some(id) => entry.owner == Some(id), + None => entry.owner.is_none(), + }); + if remove { + uids.remove(stream_url); + } + } + + pub(super) async fn finish_unregister(&self, stream_url: &str, state: &SharedStreamState) { + self.remove_meter_uid_if_owned_by(stream_url, state.provider_guard.as_ref().map(|handle| handle.allocation_id)); + state.cancel_subscribers().await; + state.stop_token.cancel(); + if let Some(handle) = &state.provider_guard { + self.provider_manager.release_handle(handle); + } + } + + pub(super) async fn unregister(&self, stream_url: &str, expected: &Arc) { + let mut register = self.shared_streams.write().await; + let state = { + if !register.by_key.get(stream_url).is_some_and(|state| Arc::ptr_eq(state, expected)) { + return; + } + register.key_by_subscriber.retain(|_, url| url.as_ref() != stream_url); + register.by_key.remove(stream_url) + }; + drop(register); + if let Some(state) = state { + self.finish_unregister(stream_url, &state).await; + } + } + + pub async fn teardown_preempted_stream(&self, stream_url: &str, allocation_id: u64) { + let mut register = self.shared_streams.write().await; + if !register.by_key.get(stream_url).is_some_and(|state| { + state.provider_guard.as_ref().is_some_and(|handle| handle.allocation_id == allocation_id) + }) { + return; + } + let state = { + register.key_by_subscriber.retain(|_, url| url.as_ref() != stream_url); + register.by_key.remove(stream_url) + }; + drop(register); + self.remove_meter_uid_if_owned_by(stream_url, Some(allocation_id)); + if let Some(state) = state { + state.preempted_token.cancel(); + state.stop_token.cancel(); + } + } + + /// Releases all subscribers on a closed transport. Playback cleanup uses the id variant. + pub async fn release_connection(&self, addr: &SocketAddr, _send_stop_signal: bool) { + let states: Vec> = { + let register = self.shared_streams.read().await; + register.by_key.values().cloned().collect() + }; + let mut ids = Vec::new(); + for state in states { + ids.extend( + state + .subscribers + .read() + .await + .iter() + .filter_map(|(id, subscriber)| (subscriber.addr == *addr).then_some(*id)), + ); + } + for id in ids { + self.release_subscriber(id).await; + } + } + + pub async fn release_subscriber(&self, subscriber_id: SubscriberId) { + let mut register = self.shared_streams.write().await; + let stopped = { + // Registry then subscribers is the shared-stream lock order. Joining and + // removing the final subscriber must not race an origin replacement. + if let Some(url) = register.key_by_subscriber.remove(&subscriber_id) { + if let Some(state) = register.by_key.get(&url).cloned() { + let mut subscribers = state.subscribers.write().await; + if let Some(subscriber) = subscribers.remove(&subscriber_id) { + subscriber.cancel_token.cancel(); + } + if subscribers.is_empty() && !state.has_pending_joins() { + register.by_key.remove(&url); + drop(subscribers); + Some((url, state)) + } else { + None + } + } else { + None + } + } else { + None + } + }; + drop(register); + self.provider_manager.release_shared_connection(subscriber_id); + if let Some((url, state)) = stopped { + self.finish_unregister(&url, &state).await; + } + } + + pub async fn shutdown(&self) { + let stopped = { + let mut register = self.shared_streams.write().await; + register.key_by_subscriber.clear(); + register.by_key.drain().collect::>() + }; + for (url, state) in stopped { + self.finish_unregister(&url, &state).await; + // Join the broadcast and forwarder tasks for this origin so teardown is + // complete before the manager (and its runtime) is dropped. Cancellation has + // already fired, so these tasks only finish their terminal cleanup. + let handles = std::mem::take(&mut *state.lock_task_handles()); + for handle in handles { + let _ = handle.await; + } + } + self.lock_meter_uids().clear(); + } + + pub(super) async fn subscribe_stream( + &self, + stream_url: &str, + addr: &SocketAddr, + subscriber_id: SubscriberId, + connection_manager: Arc, + user_priority: i8, + connection_kind: crate::active_provider_manager::ConnectionKind, + ) -> Result>, SharedCleanupCapability)>, ConnectionRejectionReason> + { + let Some(_admission) = connection_manager.begin_admission().await else { + return Err(ConnectionRejectionReason::CleanupReceiverClosed); + }; + { + let register = self.shared_streams.read().await; + if !register.by_key.contains_key(stream_url) { + return Ok(None); + } + } + #[cfg(test)] + { + let barrier = self.test_preflight_barrier.lock().ok().and_then(|g| g.clone()); + if let Some(barrier) = barrier { + barrier.wait().await; + } + } + let mut pending_cleanup = Self::reserve_subscriber_cleanup(&connection_manager, subscriber_id, *addr).await?; + // Keep the global registry lock only for the lookup and the key_by_subscriber + // commit. The origin's own subscriber/task locks are taken during subscribe, so a + // slow origin must not block joins on other URLs. + let (state, pending_join) = { + let mut register = self.shared_streams.write().await; + let Some((key, state)) = + register.by_key.get_key_value(stream_url).map(|(key, state)| (Arc::clone(key), Arc::clone(state))) + else { + pending_cleanup.disarm(); + return Ok(None); + }; + if let Err(err) = self.provider_manager.add_shared_connection( + addr, + subscriber_id, + stream_url, + user_priority, + connection_kind, + ) { + warn!("Failed joining shared stream: {}", sanitize_sensitive_info(&err)); + pending_cleanup.disarm(); + return Ok(None); + } + register.key_by_subscriber.insert(subscriber_id, key); + let pending_join = PendingJoinGuard::new(&state); + (state, pending_join) + }; + Ok(Some( + state.subscribe(addr, subscriber_id, Arc::clone(&connection_manager), pending_cleanup, pending_join).await, + )) + } + + #[allow(clippy::too_many_arguments)] + pub async fn register_shared_stream( + ctx: SharedStreamCtx<'_>, + stream_url: &str, + bytes_stream: S, + addr: &SocketAddr, + subscriber_id: SubscriberId, + headers: Vec<(String, String)>, + buffer_size: usize, + mut provider_handle: Option, + pending_cleanup: PendingSharedSubscriberCleanup, + user_priority: i8, + connection_kind: crate::active_provider_manager::ConnectionKind, + ) -> Option<(BoxedProviderStream, Option>, SharedCleanupCapability)> + where + S: Stream> + Unpin + 'static + Send, + E: std::fmt::Debug + Send, + { + let _admission = ctx.connection_manager.begin_admission().await?; + let registration_started_at = Instant::now(); + let buf_size = CHANNEL_SIZE.max(buffer_size); + let config = ctx.app_config.config.load(); + let min_buffer_bytes = resolve_min_burst_buffer_bytes(&config); + let low_priority_preempted = + ctx.app_config.custom_stream_response.load().as_ref().and_then(|c| c.low_priority_preempted.clone()); + let mut register = ctx.shared_stream_manager.shared_streams.write().await; + if let Some((key, existing)) = + register.by_key.get_key_value(stream_url).map(|(key, state)| (Arc::clone(key), Arc::clone(state))) + { + drop(provider_handle.take()); + if ctx + .active_provider + .add_shared_connection(addr, subscriber_id, stream_url, user_priority, connection_kind) + .is_err() + { + return None; + } + register.key_by_subscriber.insert(subscriber_id, key); + let pending_join = PendingJoinGuard::new(&existing); + drop(register); + let response = existing + .subscribe(addr, subscriber_id, Arc::clone(ctx.connection_manager), pending_cleanup, pending_join) + .await; + return Some(response); + } + let handle = provider_handle.as_ref().and_then(|managed| managed.handle())?; + if !ctx.active_provider.make_shared_connection(handle, stream_url, subscriber_id) { + drop(register); + drop(provider_handle.take()); + return None; + } + let allocation_id = handle.allocation_id; + let raw_handle = provider_handle.and_then(|mut managed| managed.disarm())?; + let mut shared_state = + SharedStreamState::new(headers, buf_size, Some(raw_handle), min_buffer_bytes, low_priority_preempted) + .with_subscriber_idle_timeout_secs( + config + .reverse_proxy + .as_ref() + .and_then(|reverse_proxy| reverse_proxy.stream.as_ref()) + .map_or(DEFAULT_SUBSCRIBER_IDLE_TIMEOUT_SECS, |stream| { + stream.shared_subscriber_idle_timeout_secs + }), + ); + shared_state.subscriber_max_duration = + config.sleep_timer_mins.filter(|mins| *mins > 0).map(|mins| Duration::from_secs(u64::from(mins) * 60)); + let shared_state = Arc::new(shared_state); + let stream_key: Arc = Arc::from(stream_url); + register.by_key.insert(Arc::clone(&stream_key), Arc::clone(&shared_state)); + register.key_by_subscriber.insert(subscriber_id, stream_key); + ctx.shared_stream_manager.adopt_meter_uid(stream_url, allocation_id); + let pending_join = PendingJoinGuard::new(&shared_state); + drop(register); + let subscribed_stream = shared_state + .subscribe(addr, subscriber_id, Arc::clone(ctx.connection_manager), pending_cleanup, pending_join) + .await; + debug_if_enabled!( + "Shared stream startup register+subscribe completed for {} in {} ms", + sanitize_sensitive_info(stream_url), + registration_started_at.elapsed().as_millis() + ); + shared_state.broadcast(stream_url, bytes_stream, Arc::clone(ctx.shared_stream_manager)); + Some(subscribed_stream) + } + + pub async fn subscribe_shared_stream( + ctx: SharedStreamCtx<'_>, + stream_url: &str, + addr: &SocketAddr, + subscriber_id: SubscriberId, + user_priority: i8, + connection_kind: crate::active_provider_manager::ConnectionKind, + ) -> Result>, SharedCleanupCapability)>, ConnectionRejectionReason> + { + ctx.shared_stream_manager + .subscribe_stream( + stream_url, + addr, + subscriber_id, + Arc::clone(ctx.connection_manager), + user_priority, + connection_kind, + ) + .await + } +} diff --git a/backend/session/src/streams/shared_stream_manager/reader.rs b/backend/session/src/streams/shared_stream_manager/reader.rs new file mode 100644 index 000000000..23696205b --- /dev/null +++ b/backend/session/src/streams/shared_stream_manager/reader.rs @@ -0,0 +1,143 @@ +use super::{ + PendingJoinGuard, PendingSharedMeterRegistration, PendingSharedSubscriberCleanup, ReceiverStreamWrapper, + SharedStreamState, DEFAULT_SHARED_BUFFER_SIZE_BYTES, +}; +use crate::{CleanupEvent, SharedCleanupCapability}; +use bytes::Bytes; +use futures::Stream; +use std::{ + future::Future, + net::SocketAddr, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; +use tuliprox_core::model::{Config, SharedSubscriberId, StreamError}; + +impl PendingSharedMeterRegistration { + pub fn commit(mut self) { self.armed = false; } +} + +impl Drop for PendingSharedMeterRegistration { + fn drop(&mut self) { + if self.armed { + self.manager.remove_meter_uid_if_pending(&self.stream_url, self.meter_uid); + } + } +} + +impl PendingJoinGuard { + pub(super) fn new(state: &Arc) -> Self { + state.increment_pending_joins(); + Self { state: Arc::clone(state), armed: true } + } + + pub(super) fn commit(mut self) { + self.armed = false; + self.state.decrement_pending_joins(); + } +} + +impl Drop for PendingJoinGuard { + fn drop(&mut self) { + if self.armed { + self.state.decrement_pending_joins(); + } + } +} + +impl PendingSharedSubscriberCleanup { + pub(super) fn new( + permit: tokio::sync::mpsc::OwnedPermit, + subscriber_id: SharedSubscriberId, + addr: SocketAddr, + ) -> Self { + Self { permit: Some(permit), subscriber_id, addr, request_id: None, owner: None } + } + + pub fn set_provider_request_identity(&mut self, owner: &str, request_id: tuliprox_core::model::PlaybackRequestId) { + self.owner = Some(Arc::from(owner)); + self.request_id = Some(request_id); + } + + /// Produces the cleanup-ownership proof for this subscriber. The permit held by + /// this guard is the source of the capability. + pub fn capability(&self) -> SharedCleanupCapability { SharedCleanupCapability::new(self.subscriber_id) } + + pub(super) fn take_cleanup( + &mut self, + ) -> ( + Option>, + Option, + Option>, + ) { + (self.permit.take(), self.request_id.take(), self.owner.take()) + } + + pub fn disarm(&mut self) { self.permit = None; } +} + +impl Drop for PendingSharedSubscriberCleanup { + fn drop(&mut self) { + if let Some(permit) = self.permit.take() { + permit.send(CleanupEvent::ReleaseSharedSubscriber { + addr: self.addr, + subscriber_id: self.subscriber_id, + request_id: self.request_id, + owner: self.owner.take(), + }); + } + } +} + +impl ReceiverStreamWrapper { + pub(super) fn release(&mut self) { + if !self.released { + self.released = true; + if let Some(permit) = self.permit.take() { + permit.send(CleanupEvent::ReleaseSharedSubscriber { + addr: self.addr, + subscriber_id: self.subscriber_id, + request_id: self.request_id, + owner: self.owner.take(), + }); + } + } + } +} + +impl Stream for ReceiverStreamWrapper +where + S: Stream + Unpin, +{ + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.deadline.as_mut().is_some_and(|deadline| deadline.as_mut().poll(cx).is_ready()) { + self.release(); + return Poll::Ready(None); + } + if let Some(start) = self.start.take() { + let _ = start.send(()); + } + match Pin::new(&mut self.stream).poll_next(cx) { + Poll::Ready(Some(bytes)) => Poll::Ready(Some(Ok(bytes))), + Poll::Ready(None) => Poll::Ready(None), + Poll::Pending => Poll::Pending, + } + } +} + +pub(super) fn resolve_min_burst_buffer_bytes(config: &Config) -> usize { + config + .reverse_proxy + .as_ref() + .and_then(|rp| rp.stream.as_ref()) + .and_then(|stream| usize::try_from(stream.shared_burst_buffer_mb.saturating_mul(1024 * 1024)).ok()) + .unwrap_or(DEFAULT_SHARED_BUFFER_SIZE_BYTES) + .max(1) +} + +impl Drop for ReceiverStreamWrapper { + fn drop(&mut self) { self.release(); } +} diff --git a/backend/session/src/streams/shared_stream_manager/subscription.rs b/backend/session/src/streams/shared_stream_manager/subscription.rs new file mode 100644 index 000000000..b4863c1a5 --- /dev/null +++ b/backend/session/src/streams/shared_stream_manager/subscription.rs @@ -0,0 +1,363 @@ +use super::{ + buffer::{send_burst_buffer, send_client_chunk, SendOutcome, SubscriberId}, + BudgetedChunk, BurstBuffer, PendingJoinGuard, PendingSharedSubscriberCleanup, ReceiverStreamWrapper, + SharedStreamState, SharedSubscriber, DEFAULT_SUBSCRIBER_IDLE_TIMEOUT_SECS, MAX_SUBSCRIBER_LIVE_BATCH_CHUNKS, + SHARED_BURST_BYTES_PER_BUFFER_SLOT, +}; +use crate::{ + streams::buffered_stream::CHANNEL_SIZE, BoxedProviderStream, CleanupEvent, ConnectionManager, + SharedCleanupCapability, +}; +use futures::StreamExt; +use log::{debug, warn}; +use shared::utils::sanitize_sensitive_info; +use std::{ + collections::HashMap, + net::SocketAddr, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, +}; +use tokio::{ + sync::{mpsc, oneshot, Mutex, Notify, RwLock, Semaphore}, + time::{sleep, Duration, Instant}, +}; +use tokio_stream::wrappers::ReceiverStream; +use tokio_util::sync::CancellationToken; +use tuliprox_core::{ + model::ProviderHandle, + utils::{debug_if_enabled, trace_if_enabled}, +}; + +impl SharedStreamState { + pub(super) fn new( + headers: Vec<(String, String)>, + buf_size: usize, + provider_guard: Option, + min_burst_buffer_size: usize, + low_priority_preempted: Option, + ) -> Self { + let base_channel_capacity = buf_size.max(CHANNEL_SIZE); + let burst_buffer_size_in_bytes = + min_burst_buffer_size.max(base_channel_capacity.saturating_mul(SHARED_BURST_BYTES_PER_BUFFER_SLOT)); + Self { + headers, + buf_size: base_channel_capacity, + provider_guard, + low_priority_preempted, + preempted_token: CancellationToken::new(), + subscribers: RwLock::new(HashMap::new()), + stop_token: CancellationToken::new(), + burst_buffer: Arc::new(Mutex::new(BurstBuffer::new(burst_buffer_size_in_bytes))), + live_notification: Arc::new(Notify::new()), + task_handles: std::sync::Mutex::new(Vec::new()), + subscriber_idle_timeout_secs: DEFAULT_SUBSCRIBER_IDLE_TIMEOUT_SECS, + subscriber_max_duration: None, + pending_joins: AtomicUsize::new(0), + } + } + + pub(super) fn with_subscriber_idle_timeout_secs(mut self, secs: u64) -> Self { + if secs > 0 { + self.subscriber_idle_timeout_secs = secs; + } + self + } + + pub(super) fn increment_pending_joins(&self) { self.pending_joins.fetch_add(1, Ordering::AcqRel); } + + pub(super) fn decrement_pending_joins(&self) { self.pending_joins.fetch_sub(1, Ordering::AcqRel); } + + pub(super) fn has_pending_joins(&self) -> bool { self.pending_joins.load(Ordering::Acquire) > 0 } + + pub(super) fn lock_task_handles(&self) -> std::sync::MutexGuard<'_, Vec>> { + self.task_handles.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + } + + pub(super) async fn register_subscriber( + &self, + id: SubscriberId, + addr: &SocketAddr, + cancel_token: CancellationToken, + ) { + self.subscribers.write().await.insert(id, SharedSubscriber { addr: *addr, cancel_token }); + } + + pub(super) async fn cancel_subscribers(&self) { + for subscriber in self.subscribers.read().await.values() { + subscriber.cancel_token.cancel(); + } + } + + #[allow(clippy::too_many_lines)] + pub(super) async fn subscribe( + self: &Arc, + addr: &SocketAddr, + subscriber_id: SubscriberId, + connection_manager: Arc, + mut pending_cleanup: PendingSharedSubscriberCleanup, + pending_join: PendingJoinGuard, + ) -> (BoxedProviderStream, Option>, SharedCleanupCapability) { + let (client_tx, client_rx) = mpsc::channel::(self.buf_size); + let cancel_token = CancellationToken::new(); + let queue_byte_budget = self.buf_size.saturating_mul(SHARED_BURST_BYTES_PER_BUFFER_SLOT); + let byte_budget = Arc::new(Semaphore::new(queue_byte_budget)); + + { + let mut handles = self.lock_task_handles(); + handles.retain(|h| !h.is_finished()); + } + + self.register_subscriber(subscriber_id, addr, cancel_token.clone()).await; + pending_join.commit(); + let (start_tx, start_rx) = oneshot::channel(); + let cleanup_manager = Arc::clone(&connection_manager); + + let client_tx_clone = client_tx.clone(); + let burst_buffer = Arc::clone(&self.burst_buffer); + let burst_buffer_for_log = Arc::clone(&self.burst_buffer); + let live_notification = Arc::clone(&self.live_notification); + let timeout_duration = Duration::from_secs(self.subscriber_idle_timeout_secs); + let idle_check_interval = Duration::from_secs(1); + let mut last_lag_log = Instant::now().checked_sub(Duration::from_secs(10)).unwrap_or_else(Instant::now); + let mut consecutive_lag_count: u32 = 0; + let subscriber_buf_size = self.buf_size; + let preempted_token = self.preempted_token.clone(); + let low_priority_preempted = self.low_priority_preempted.clone(); + let address = *addr; + let subscriber_started_at = Instant::now(); + + let handle = tokio::spawn(async move { + // Wait for the response to be polled, but respect cancellation/shutdown so a + // body that is never polled cannot keep the forwarder task alive indefinitely. + tokio::select! { + () = cancel_token.cancelled() => return, + started = start_rx => { + if started.is_err() { + return; + } + } + } + // Send-progress budget begins only once the body is actually polled. + let mut last_active = Instant::now(); + let (snapshot, mut next_sequence) = { + let buffer = burst_buffer.lock().await; + buffer.snapshot() + }; + match send_burst_buffer(&snapshot, &client_tx_clone, &cancel_token, timeout_duration, &byte_budget).await { + Ok(sent_burst_chunks) => { + drop(snapshot); + if sent_burst_chunks > 0 { + // The replay renewed its own internal progress time; carry that + // forward so the first live send does not immediately expire. + last_active = Instant::now(); + debug_if_enabled!( + "Shared stream subscriber {} replayed {sent_burst_chunks} burst chunks after {} ms", + sanitize_sensitive_info(&address.to_string()), + subscriber_started_at.elapsed().as_millis() + ); + } + } + Err(outcome) => { + drop(snapshot); + debug!( + "Shared stream subscriber {} burst replay failed ({outcome:?}); terminating", + sanitize_sensitive_info(&address.to_string()) + ); + cleanup_manager.send_cleanup(CleanupEvent::ReleaseSharedSubscriber { + addr: address, + subscriber_id, + request_id: None, + owner: None, + }); + return; + } + } + + let mut first_live_chunk_logged = false; + let mut startup_chunks_sent = 0_usize; + let mut startup_bytes_sent = 0_usize; + let mut startup_stats_logged = false; + let mut read_chunks = Vec::with_capacity(subscriber_buf_size.min(64)); + let idle_check = sleep(idle_check_interval); + tokio::pin!(idle_check); + + loop { + // Pre-create the notified future before locking the buffer to avoid + // a race where notify_waiters() fires between lock release and await. + let notified_fut = live_notification.notified(); + + let read = { + let buffer = burst_buffer.lock().await; + buffer.read_from_into(next_sequence, &mut read_chunks, MAX_SUBSCRIBER_LIVE_BATCH_CHUNKS) + }; + next_sequence = read.next_sequence; + if read.skipped > 0 { + consecutive_lag_count = consecutive_lag_count.saturating_add(1); + if last_lag_log.elapsed() > Duration::from_secs(5) { + let buffered_bytes = { + let buffer = burst_buffer_for_log.lock().await; + buffer.current_bytes + }; + warn!( + "Shared stream client lagged behind {address}. Skipped {} messages \ + (buffered {buffered_bytes} bytes, consecutive lags={consecutive_lag_count})", + read.skipped + ); + last_lag_log = Instant::now(); + } + } else if !read_chunks.is_empty() { + consecutive_lag_count = 0; + } + + trace_if_enabled!( + "shared_stream.subscribe: read {} chunks (next_seq={}, skipped={}) for {}", + read_chunks.len(), + read.next_sequence, + read.skipped, + sanitize_sensitive_info(&address.to_string()) + ); + + if !read_chunks.is_empty() { + for data in read_chunks.drain(..) { + let chunk_len = data.len(); + match send_client_chunk( + &client_tx, + data, + &cancel_token, + last_active + timeout_duration, + &byte_budget, + ) + .await + { + SendOutcome::Sent => {} + outcome => { + debug!("Shared stream client send error ({outcome:?}): {address}"); + cleanup_manager.send_cleanup(CleanupEvent::ReleaseSharedSubscriber { + addr: address, + subscriber_id, + request_id: None, + owner: None, + }); + return; + } + } + if !first_live_chunk_logged { + debug_if_enabled!( + "Shared stream subscriber {} received first live chunk after {} ms", + sanitize_sensitive_info(&address.to_string()), + subscriber_started_at.elapsed().as_millis() + ); + first_live_chunk_logged = true; + } + if !startup_stats_logged { + startup_chunks_sent = startup_chunks_sent.saturating_add(1); + startup_bytes_sent = startup_bytes_sent.saturating_add(chunk_len); + if subscriber_started_at.elapsed() >= Duration::from_secs(5) { + debug_if_enabled!( + "Shared stream subscriber {} startup throughput: chunks={} bytes={} over {} ms (queue_used={}/{})", + sanitize_sensitive_info(&address.to_string()), + startup_chunks_sent, + startup_bytes_sent, + subscriber_started_at.elapsed().as_millis(), + subscriber_buf_size.saturating_sub(client_tx_clone.capacity()), + subscriber_buf_size + ); + startup_stats_logged = true; + } + } + last_active = Instant::now(); + } + continue; + } + + tokio::select! { + biased; + + () = cancel_token.cancelled() => { + trace_if_enabled!( + "shared_stream.subscribe: cancel_received for {}", + sanitize_sensitive_info(&address.to_string()) + ); + break; + } + + () = &mut idle_check => { + if last_active.elapsed() > timeout_duration { + trace_if_enabled!( + "shared_stream.subscribe: idle_check_fired (inactivity>={}s) for {}", + timeout_duration.as_secs(), + sanitize_sensitive_info(&address.to_string()) + ); + cancel_token.cancel(); + break; + } + idle_check.as_mut().reset(Instant::now() + idle_check_interval); + } + + () = notified_fut => { + trace_if_enabled!( + "shared_stream.subscribe: empty_buffer_waiting waker for {}", + sanitize_sensitive_info(&address.to_string()) + ); + } + + () = preempted_token.cancelled() => { + trace_if_enabled!( + "shared_stream.subscribe: preempted for {}", + sanitize_sensitive_info(&address.to_string()) + ); + if let Some(mut fallback) = low_priority_preempted { + debug_if_enabled!( + "Shared stream subscriber {} switching to low_priority_preempted fallback", + sanitize_sensitive_info(&address.to_string()) + ); + while let Some(chunk) = fallback.next_chunk() { + match send_client_chunk(&client_tx, chunk, &cancel_token, last_active + timeout_duration, &byte_budget).await { + SendOutcome::Sent => last_active = Instant::now(), + outcome => { + debug!( + "Shared stream fallback send error ({outcome:?}) for {}", + sanitize_sensitive_info(&address.to_string()) + ); + break; + } + } + } + } + break; + } + } + } + + cleanup_manager.send_cleanup(CleanupEvent::ReleaseSharedSubscriber { + addr: address, + subscriber_id, + request_id: None, + owner: None, + }); + }); + + self.lock_task_handles().push(handle); + + let provider = self.provider_guard.as_ref().and_then(|h| h.allocation.get_provider_name()); + let (permit, request_id, owner) = pending_cleanup.take_cleanup(); + ( + ReceiverStreamWrapper { + stream: ReceiverStream::new(client_rx).map(|chunk| chunk.bytes), + start: Some(start_tx), + subscriber_id, + addr: *addr, + permit, + request_id, + owner, + deadline: self.subscriber_max_duration.map(|duration| Box::pin(sleep(duration))), + released: false, + } + .boxed(), + provider, + SharedCleanupCapability::new(subscriber_id), + ) + } +} diff --git a/backend/session/src/streams/shared_stream_manager/tests.rs b/backend/session/src/streams/shared_stream_manager/tests.rs new file mode 100644 index 000000000..97e5e49e1 --- /dev/null +++ b/backend/session/src/streams/shared_stream_manager/tests.rs @@ -0,0 +1,146 @@ +use super::{ + buffer::{send_client_chunk, SendOutcome}, + BudgetedChunk, BurstBuffer, SharedStreamManager, SharedStreamState, MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES, + SHARED_CLEANUP_ADMISSION_TIMEOUT, +}; +use crate::{ + streams::buffered_stream::CHANNEL_SIZE, ActiveProviderManager, ActiveUserConnectionParams, ActiveUserManager, + ConnectionKind, ConnectionManager, ConnectionRejectionReason, EventManager, ManagedProviderHandle, +}; +use arc_swap::{ArcSwap, ArcSwapOption}; +use bytes::Bytes; +use futures::StreamExt; +use shared::{ + model::{ConfigPaths, InputFetchMethod, InputType, PlaylistItemType, StreamChannel, XtreamCluster}, + utils::Internable, +}; +use std::{borrow::Cow, collections::HashMap, net::SocketAddr, sync::Arc}; +use tokio::{ + sync::{mpsc, Semaphore}, + time::{timeout, Duration, Instant}, +}; +use tokio_stream::wrappers::ReceiverStream; +use tokio_util::sync::CancellationToken; +use tuliprox_core::{ + model::{AppConfig, Config, ConfigInput, Fingerprint, MediaToolCapabilities, SharedSubscriberId, SourcesConfig}, + utils::FileLockManager, +}; + +fn create_test_app_config() -> AppConfig { create_test_app_config_with_conns(1) } + +fn create_test_app_config_with_conns(max_connections: u16) -> AppConfig { + let input = Arc::new(ConfigInput { + id: 1, + name: "provider_1".intern(), + input_type: InputType::Xtream, + headers: HashMap::default(), + url: "http://provider-1.example".to_string(), + username: Some("user1".to_string()), + password: Some("pass1".to_string()), + enabled: true, + priority: 0, + max_connections, + method: InputFetchMethod::default(), + aliases: None, + ..ConfigInput::default() + }); + + let sources = SourcesConfig { inputs: vec![input], ..SourcesConfig::default() }; + + AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config::default())), + sources: Arc::new(ArcSwap::from_pointee(sources)), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + } +} + +fn create_test_stream_channel(url: &str) -> StreamChannel { + StreamChannel { + target_id: 1, + virtual_id: 1, + provider_id: 1, + input_name: "provider_1".intern(), + item_type: PlaylistItemType::Live, + cluster: XtreamCluster::Live, + group: "group".intern(), + title: "title".intern(), + url: url.intern(), + shared: true, + shared_joined_existing: Some(false), + shared_stream_id: None, + technical: None, + epg_channel_id: None, + epg_reference_ts: None, + upstream_user_agent: None, + } +} + +fn create_test_connection_manager( + app_cfg: &AppConfig, + event_manager: &Arc, +) -> (Arc, Arc, Arc, Arc) { + let provider_manager = Arc::new(ActiveProviderManager::new(app_cfg, event_manager)); + let geoip = Arc::new(ArcSwapOption::default()); + let user_manager = Arc::new(ActiveUserManager::new(&Config::default(), &geoip, event_manager)); + let shared_manager = Arc::new(SharedStreamManager::new(Arc::clone(&provider_manager))); + let connection_manager = + Arc::new(ConnectionManager::new(&user_manager, &provider_manager, &shared_manager, event_manager, None)); + (provider_manager, user_manager, shared_manager, connection_manager) +} + +async fn register_user( + users: &ActiveUserManager, + addr: SocketAddr, + uid: u32, + username: &str, + stream_url: &str, +) -> Result<(), Box> { + let fingerprint = Fingerprint::new(username.to_string(), username.to_string(), addr); + let channel = create_test_stream_channel(stream_url); + let stream = users + .update_connection(ActiveUserConnectionParams { + uid, + meter_uid: 0, + username, + max_connections: 5, + soft_connections: 0, + connection_kind: ConnectionKind::Normal, + priority: 0, + soft_priority: 0, + fingerprint: &fingerprint, + provider: "provider_1".intern(), + stream_channel: &channel, + user_agent: Cow::Borrowed("test"), + session_token: None, + }) + .await + .ok_or("user stream missing")?; + assert_eq!(stream.uid, uid); + Ok(()) +} + +mod admission; +mod lifecycle; +mod policy; +mod streaming; +mod terminal; +mod transport; diff --git a/backend/session/src/streams/shared_stream_manager/tests/admission.rs b/backend/session/src/streams/shared_stream_manager/tests/admission.rs new file mode 100644 index 000000000..50116644e --- /dev/null +++ b/backend/session/src/streams/shared_stream_manager/tests/admission.rs @@ -0,0 +1,299 @@ +use super::*; + +#[tokio::test] +async fn shared_clients_on_same_socket_keep_independent_streams_and_capacity() -> Result<(), Box> +{ + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41004".parse()?; + let first_id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let second_id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/shared.ts"; + register_user(&users, addr, first_id.stream_uid(), "first", url).await?; + let allocation = ManagedProviderHandle::new( + Arc::clone(&providers), + providers + .acquire_connection(&"provider_1".intern(), &addr, 0, ConnectionKind::Normal) + .ok_or("provider allocation missing")?, + ); + let ctx = super::super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }; + let (tx, rx) = mpsc::channel::>(8); + let pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, first_id, addr) + .await + .map_err(|_| "shared cleanup admission failed")?; + let (mut first, _, _) = SharedStreamManager::register_shared_stream( + ctx, + url, + ReceiverStream::new(rx), + &addr, + first_id, + Vec::new(), + 8, + Some(allocation), + pending_cleanup, + 0, + ConnectionKind::Normal, + ) + .await + .ok_or("first subscription missing")?; + let (mut second, _, _) = + SharedStreamManager::subscribe_shared_stream(ctx, url, &addr, second_id, 0, ConnectionKind::Normal) + .await + .map_err(|_| "second shared cleanup admission failed")? + .ok_or("second subscription missing")?; + register_user(&users, addr, second_id.stream_uid(), "second", url).await?; + + tx.send(Ok(Bytes::from_static(b"first chunk"))).await?; + assert_eq!( + timeout(Duration::from_secs(2), first.next()).await?.ok_or("first ended")??, + Bytes::from_static(b"first chunk") + ); + assert_eq!( + timeout(Duration::from_secs(2), second.next()).await?.ok_or("second ended")??, + Bytes::from_static(b"first chunk") + ); + assert_eq!(users.active_streams().await.len(), 2); + assert_eq!(providers.get_provider_connections_count(), 1); + + drop(first); + timeout(Duration::from_secs(2), async { + while users.active_streams().await.len() != 1 { + tokio::task::yield_now().await; + } + }) + .await?; + // A delayed duplicate cleanup cannot remove the surviving subscriber. + connections.send_cleanup(crate::CleanupEvent::ReleaseSharedSubscriber { + addr, + subscriber_id: first_id, + request_id: None, + owner: None, + }); + tx.send(Ok(Bytes::from_static(b"second chunk"))).await?; + assert_eq!( + timeout(Duration::from_secs(2), second.next()).await?.ok_or("survivor ended")??, + Bytes::from_static(b"second chunk") + ); + assert_eq!(providers.get_provider_connections_count(), 1); + assert_eq!(users.active_streams().await.first().map(|stream| stream.uid), Some(second_id.stream_uid())); + + drop(second); + timeout(Duration::from_secs(2), async { + while !users.active_streams().await.is_empty() || providers.get_provider_connections_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await?; + assert!(manager.get_shared_state(url).await.is_none()); + Ok(()) +} + +#[test] +fn test_shared_state_channel_capacity_does_not_scale_with_burst_buffer_bytes() { + let min_burst_buffer_size = 12 * 1024 * 1024; + let state = SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, min_burst_buffer_size, None); + + assert_eq!(state.buf_size, CHANNEL_SIZE); +} + +#[test] +fn test_burst_buffer_chunk_bound_preserves_ts_sized_byte_capacity() { + let max_chunks = 4; + let mut buffer = BurstBuffer::new(MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES * max_chunks); + for _ in 0..max_chunks.saturating_add(1) { + buffer.push(Bytes::from(vec![0_u8; MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES])); + } + + let mut chunks = Vec::new(); + let read = buffer.read_from_into(0, &mut chunks, usize::MAX); + + assert_eq!(buffer.buffer.len(), max_chunks); + assert_eq!(buffer.current_bytes, MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES * max_chunks); + assert_eq!(read.skipped, 1); + assert_eq!(chunks.len(), max_chunks); +} + +#[tokio::test(start_paused = true)] +async fn shared_subscription_admission_has_deadline_and_no_unprotected_fallback( +) -> Result<(), Box> { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let (_, _, _, connections) = create_test_connection_manager(&app_cfg, &events); + let addr: SocketAddr = "127.0.0.1:43002".parse()?; + let id = SharedSubscriberId::from_stream_uid(1); + + // Saturate the cleanup queue so the admission await blocks; the shared admission + // deadline must reject the subscriber instead of hanging or falling back. + let pending = || crate::CleanupEvent::Defer(Box::pin(std::future::pending::<()>())); + connections.send_cleanup(pending()); + for _ in 0..4096 { + connections.send_cleanup(pending()); + } + + let admission = tokio::spawn({ + let connections = Arc::clone(&connections); + async move { SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await } + }); + tokio::task::yield_now().await; + tokio::task::yield_now().await; + tokio::time::advance(super::super::SHARED_CLEANUP_ADMISSION_TIMEOUT + Duration::from_secs(1)).await; + let result = admission.await?; + assert!(result.is_err(), "saturated shared subscription admission must be rejected, not fall back"); + Ok(()) +} + +#[tokio::test] +async fn concurrent_shared_origin_creation_releases_only_losing_allocation() -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config_with_conns(2)); + let events = Arc::new(EventManager::new()); + let (providers, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr_1 = "127.0.0.1:41013".parse()?; + let addr_2 = "127.0.0.1:41014".parse()?; + let id_1 = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let id_2 = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/concurrent_origin.ts"; + + let handle_1 = providers + .acquire_connection(&"provider_1".intern(), &addr_1, 0, ConnectionKind::Normal) + .ok_or("first provider allocation missing")?; + let managed_1 = ManagedProviderHandle::new(Arc::clone(&providers), handle_1); + let pending_1 = SharedStreamManager::reserve_subscriber_cleanup(&connections, id_1, addr_1).await?; + let (first_stream, _, _) = SharedStreamManager::register_shared_stream( + super::super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }, + url, + futures::stream::pending::>(), + &addr_1, + id_1, + Vec::new(), + 8, + Some(managed_1), + pending_1, + 0, + ConnectionKind::Normal, + ) + .await + .ok_or("first origin registration failed")?; + + assert_eq!(providers.get_provider_connections_count(), 1); + + let handle_2 = providers + .acquire_connection(&"provider_1".intern(), &addr_2, 0, ConnectionKind::Normal) + .ok_or("second provider allocation missing")?; + let managed_2 = ManagedProviderHandle::new(Arc::clone(&providers), handle_2); + assert_eq!(providers.get_provider_connections_count(), 2); + + let pending_2 = SharedStreamManager::reserve_subscriber_cleanup(&connections, id_2, addr_2).await?; + let (second_stream, _, _) = SharedStreamManager::register_shared_stream( + super::super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }, + url, + futures::stream::pending::>(), + &addr_2, + id_2, + Vec::new(), + 8, + Some(managed_2), + pending_2, + 0, + ConnectionKind::Normal, + ) + .await + .ok_or("second subscriber join failed")?; + + assert_eq!(providers.get_provider_connections_count(), 1); + + drop(first_stream); + drop(second_stream); + timeout(Duration::from_secs(2), async { + while providers.get_provider_connections_count() != 0 || !users.active_streams().await.is_empty() { + tokio::task::yield_now().await; + } + }) + .await?; + assert_eq!(providers.get_provider_connections_count(), 0); + Ok(()) +} + +#[tokio::test] +async fn shutdown_rejects_waiting_admission_and_drains_owned_cleanup() -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (_providers, _users, _manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41015".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let initial_capacity = connections.cleanup_tx().capacity(); + let held_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await?; + let admission = connections.begin_admission().await.ok_or("admission unexpectedly closed")?; + + let shutdown = tokio::spawn({ + let connections = Arc::clone(&connections); + async move { connections.shutdown().await } + }); + timeout(Duration::from_secs(2), async { + while !connections.is_shutting_down() { + tokio::task::yield_now().await; + } + }) + .await?; + assert!(connections.is_shutting_down()); + + let res = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await; + assert!( + matches!(res, Err(ConnectionRejectionReason::CleanupReceiverClosed)), + "waiting admission must be rejected after shutdown" + ); + assert!(!shutdown.is_finished(), "shutdown must wait for an admitted registration"); + + drop(held_cleanup); + drop(admission); + timeout(Duration::from_secs(2), shutdown).await??; + assert_eq!(connections.cleanup_tx().capacity(), initial_capacity); + Ok(()) +} + +#[tokio::test] +async fn shared_cold_miss_does_not_consume_cleanup_capacity_or_emit_cleanup() -> Result<(), Box> +{ + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41017".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let initial_capacity = connections.cleanup_tx().capacity(); + + let ctx = super::super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }; + + let res = SharedStreamManager::subscribe_shared_stream( + ctx, + "https://example.invalid/live/non_existent.ts", + &addr, + id, + 0, + ConnectionKind::Normal, + ) + .await?; + + assert!(res.is_none()); + assert_eq!(connections.cleanup_tx().capacity(), initial_capacity, "cold miss must not consume cleanup capacity"); + Ok(()) +} diff --git a/backend/session/src/streams/shared_stream_manager/tests/lifecycle.rs b/backend/session/src/streams/shared_stream_manager/tests/lifecycle.rs new file mode 100644 index 000000000..fd11d8588 --- /dev/null +++ b/backend/session/src/streams/shared_stream_manager/tests/lifecycle.rs @@ -0,0 +1,505 @@ +use super::*; + +#[tokio::test] +async fn send_client_chunk_returns_when_cancelled_while_queue_is_full() -> Result<(), Box> { + let (tx, _rx) = mpsc::channel::(1); + let byte_budget = Arc::new(Semaphore::new(1024)); + let queued_permit = byte_budget.clone().acquire_many_owned(6).await?; + tx.send(BudgetedChunk { bytes: Bytes::from_static(b"queued"), _permit: queued_permit }).await?; + let cancel = CancellationToken::new(); + cancel.cancel(); + let outcome = timeout( + Duration::from_secs(1), + send_client_chunk( + &tx, + Bytes::from_static(b"blocked"), + &cancel, + Instant::now() + Duration::from_secs(1), + &byte_budget, + ), + ) + .await?; + assert_eq!(outcome, SendOutcome::Cancelled); + Ok(()) +} + +#[tokio::test] +async fn dropping_unpolled_shared_response_releases_its_subscription() -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41005".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/unpolled.ts"; + register_user(&users, addr, id.stream_uid(), "first", url).await?; + let allocation = ManagedProviderHandle::new( + Arc::clone(&providers), + providers + .acquire_connection(&"provider_1".intern(), &addr, 0, ConnectionKind::Normal) + .ok_or("provider allocation missing")?, + ); + let pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr) + .await + .map_err(|_| "shared cleanup admission failed")?; + let response = SharedStreamManager::register_shared_stream( + super::super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }, + url, + futures::stream::pending::>(), + &addr, + id, + Vec::new(), + 8, + Some(allocation), + pending_cleanup, + 0, + ConnectionKind::Normal, + ) + .await + .ok_or("subscription missing")?; + drop(response); + timeout(Duration::from_secs(2), async { + while !users.active_streams().await.is_empty() || providers.get_provider_connections_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await?; + assert!(manager.shared_streams.read().await.key_by_subscriber.is_empty()); + Ok(()) +} + +#[tokio::test(start_paused = true)] +async fn saturated_cleanup_queue_rejects_shared_subscriber_before_registration( +) -> Result<(), Box> { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let (_, _, _, connections) = create_test_connection_manager(&app_cfg, &events); + let cleanup_tx = connections.cleanup_tx(); + let available = cleanup_tx.capacity(); + let mut permits = Vec::with_capacity(available); + for _ in 0..available { + permits.push(cleanup_tx.clone().try_reserve_owned()?); + } + + let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + let addr = "127.0.0.1:41008".parse()?; + let subscriber_id = SharedSubscriberId::from_stream_uid(88); + let subscribe_connections = Arc::clone(&connections); + let subscribe = tokio::spawn(async move { + SharedStreamManager::reserve_subscriber_cleanup(&subscribe_connections, subscriber_id, addr).await + }); + + tokio::task::yield_now().await; + tokio::time::advance(SHARED_CLEANUP_ADMISSION_TIMEOUT + Duration::from_millis(1)).await; + assert!(subscribe.await?.is_err(), "saturated cleanup admission must reject the subscriber"); + assert!(state.subscribers.read().await.is_empty(), "rejected admission must not register a subscriber"); + assert!(state.lock_task_handles().is_empty(), "rejected admission must not spawn a forwarding task"); + + drop(permits); + Ok(()) +} + +#[tokio::test] +async fn duplicate_subscriber_release_preserves_other_channel_on_same_socket() -> Result<(), Box> +{ + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let (_, _, manager, _) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41006".parse()?; + let first = SharedSubscriberId::from_stream_uid(1); + let second = SharedSubscriberId::from_stream_uid(2); + let first_url: Arc = Arc::from("https://example.invalid/first.ts"); + let second_url: Arc = Arc::from("https://example.invalid/second.ts"); + let state_a = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + let state_b = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + let surviving_token = CancellationToken::new(); + state_a.register_subscriber(first, &addr, CancellationToken::new()).await; + state_b.register_subscriber(second, &addr, surviving_token.clone()).await; + { + let mut register = manager.shared_streams.write().await; + register.by_key.insert(Arc::clone(&first_url), state_a); + register.by_key.insert(Arc::clone(&second_url), state_b); + register.key_by_subscriber.insert(first, Arc::clone(&first_url)); + register.key_by_subscriber.insert(second, Arc::clone(&second_url)); + } + manager.release_subscriber(first).await; + manager.release_subscriber(first).await; + assert!(manager.get_shared_state(&first_url).await.is_none()); + assert!(manager.get_shared_state(&second_url).await.is_some()); + assert!(!surviving_token.is_cancelled()); + manager.release_connection(&addr, true).await; + assert!(surviving_token.is_cancelled()); + Ok(()) +} + +#[tokio::test] +async fn test_duplicate_release_connection_is_idempotent_with_single_subscriber() { + let app_cfg = create_test_app_config(); + let event_manager = Arc::new(EventManager::new()); + let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let shared_manager = Arc::new(SharedStreamManager::new(provider_manager)); + + let stream_url = "https://example.invalid/live/single.ts"; + let addr_1: SocketAddr = "127.0.0.1:42001".parse().unwrap_or_else(|_| unreachable!()); + let id = SharedSubscriberId::from_stream_uid(1); + let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE.max(8), None, 1024, None)); + + { + let mut reg = shared_manager.shared_streams.write().await; + reg.by_key.insert(Arc::from(stream_url), Arc::clone(&state)); + reg.key_by_subscriber.insert(id, Arc::from(stream_url)); + } + + state.register_subscriber(id, &addr_1, CancellationToken::new()).await; + + shared_manager.release_connection(&addr_1, false).await; + { + let reg = shared_manager.shared_streams.read().await; + assert!(!reg.by_key.contains_key(stream_url)); + assert!(!reg.key_by_subscriber.contains_key(&id)); + } + { + let subs = state.subscribers.read().await; + assert!(subs.is_empty()); + } + + shared_manager.release_connection(&addr_1, false).await; + { + let reg = shared_manager.shared_streams.read().await; + assert!(!reg.by_key.contains_key(stream_url)); + assert!(!reg.key_by_subscriber.contains_key(&id)); + } + { + let subs = state.subscribers.read().await; + assert!(subs.is_empty()); + } +} + +#[tokio::test] +async fn shared_origin_abort_before_user_registration_releases_physical_and_logical_request( +) -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, users, _manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41010".parse()?; + let owner = "shared-owner-1"; + let handle = providers + .acquire_connection_with_lease_for_session( + &"provider_1".intern(), + &addr, + false, + 0, + ConnectionKind::Normal, + Some(crate::active_provider_manager::PlaybackLeaseRef::new( + owner, + tuliprox_core::model::PlaybackKind::LiveTs, + )), + ) + .ok_or("provider allocation missing")?; + let request_id = handle.playback_request_id.ok_or("provider request identity missing")?; + let managed = ManagedProviderHandle::new(Arc::clone(&providers), handle); + let subscriber_id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let mut pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, subscriber_id, addr) + .await + .map_err(|err| format!("shared cleanup admission failed: {err}"))?; + pending_cleanup.set_provider_request_identity(owner, request_id); + assert_eq!(providers.get_provider_connections_count(), 1); + assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).starting, 1); + + drop(managed); + drop(pending_cleanup); + + timeout(Duration::from_secs(2), async { + while providers.provider_lease_usage(&"provider_1".intern()).total() != 0 { + tokio::task::yield_now().await; + } + }) + .await?; + assert_eq!(providers.get_provider_connections_count(), 0); + assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).total(), 0); + assert!(users.active_streams().await.is_empty()); + Ok(()) +} + +#[tokio::test] +async fn shared_origin_registration_rejected_releases_preacquired_handle() -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41011".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/rejected_preacquired.ts"; + let owner = "rejected-preacquired-owner"; + let handle = providers + .acquire_connection_with_lease_for_session( + &"provider_1".intern(), + &addr, + false, + 0, + ConnectionKind::Normal, + Some(crate::active_provider_manager::PlaybackLeaseRef::new( + owner, + tuliprox_core::model::PlaybackKind::LiveTs, + )), + ) + .ok_or("provider allocation missing")?; + let request_id = handle.playback_request_id.ok_or("provider request identity missing")?; + let managed_handle = ManagedProviderHandle::new(Arc::clone(&providers), handle); + assert_eq!(providers.get_provider_connections_count(), 1); + + let dummy_state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + manager.shared_streams.write().await.by_key.insert(Arc::from(url), dummy_state); + + let mut pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await?; + pending_cleanup.set_provider_request_identity(owner, request_id); + let res = SharedStreamManager::register_shared_stream( + super::super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }, + url, + futures::stream::pending::>(), + &addr, + id, + Vec::new(), + 8, + Some(managed_handle), + pending_cleanup, + 0, + ConnectionKind::Normal, + ) + .await; + + assert!(res.is_none()); + timeout(Duration::from_secs(2), async { + while providers.get_provider_connections_count() != 0 + || providers.provider_lease_usage(&"provider_1".intern()).total() != 0 + { + tokio::task::yield_now().await; + } + }) + .await?; + assert_eq!(providers.get_provider_connections_count(), 0); + assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).total(), 0); + Ok(()) +} + +#[tokio::test] +async fn shared_origin_abort_waiting_for_registry_lock_releases_handle() -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41012".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/abort_waiting.ts"; + let owner = "abort-waiting-owner"; + + let lock_guard = manager.shared_streams.write().await; + + let handle = providers + .acquire_connection_with_lease_for_session( + &"provider_1".intern(), + &addr, + false, + 0, + ConnectionKind::Normal, + Some(crate::active_provider_manager::PlaybackLeaseRef::new( + owner, + tuliprox_core::model::PlaybackKind::LiveTs, + )), + ) + .ok_or("provider allocation missing")?; + let request_id = handle.playback_request_id.ok_or("provider request identity missing")?; + let managed_handle = ManagedProviderHandle::new(Arc::clone(&providers), handle); + assert_eq!(providers.get_provider_connections_count(), 1); + + let mut pending_cleanup = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await?; + pending_cleanup.set_provider_request_identity(owner, request_id); + + let register_task = tokio::spawn({ + let app_cfg = Arc::clone(&app_cfg); + let manager = Arc::clone(&manager); + let providers = Arc::clone(&providers); + let connections = Arc::clone(&connections); + async move { + let ctx = super::super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }; + SharedStreamManager::register_shared_stream( + ctx, + url, + futures::stream::pending::>(), + &addr, + id, + Vec::new(), + 8, + Some(managed_handle), + pending_cleanup, + 0, + ConnectionKind::Normal, + ) + .await + } + }); + + tokio::task::yield_now().await; + + register_task.abort(); + let _ = register_task.await; + + drop(lock_guard); + + timeout(Duration::from_secs(2), async { + while providers.get_provider_connections_count() != 0 + || providers.provider_lease_usage(&"provider_1".intern()).total() != 0 + { + tokio::task::yield_now().await; + } + }) + .await?; + assert_eq!(providers.get_provider_connections_count(), 0); + assert_eq!(providers.provider_lease_usage(&"provider_1".intern()).total(), 0); + Ok(()) +} + +#[tokio::test] +async fn shutdown_with_unpolled_shared_body_leaves_no_requests_or_subscribers() -> Result<(), Box> +{ + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41016".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/unpolled_shutdown.ts"; + register_user(&users, addr, id.stream_uid(), "shutdown_user", url).await?; + + let handle = providers + .acquire_connection(&"provider_1".intern(), &addr, 0, ConnectionKind::Normal) + .ok_or("provider allocation missing")?; + let managed_handle = ManagedProviderHandle::new(Arc::clone(&providers), handle); + let pending = SharedStreamManager::reserve_subscriber_cleanup(&connections, id, addr).await?; + let (unpolled_body, _, _) = SharedStreamManager::register_shared_stream( + super::super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }, + url, + futures::stream::pending::>(), + &addr, + id, + Vec::new(), + 8, + Some(managed_handle), + pending, + 0, + ConnectionKind::Normal, + ) + .await + .ok_or("register failed")?; + + assert_eq!(providers.get_provider_connections_count(), 1); + assert_eq!(users.active_streams().await.len(), 1); + + connections.shutdown().await; + + assert_eq!(providers.get_provider_connections_count(), 0); + assert!(users.active_streams().await.is_empty()); + assert!(manager.get_shared_state(url).await.is_none()); + + drop(unpolled_body); + Ok(()) +} + +#[tokio::test] +async fn shared_state_removed_between_preflight_and_commit_leaves_no_claim() -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41018".parse()?; + let id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/removed_between.ts"; + + let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + manager.shared_streams.write().await.by_key.insert(Arc::from(url), state); + + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + if let Ok(mut lock) = manager.test_preflight_barrier.lock() { + *lock = Some(Arc::clone(&barrier)); + } + + let initial_capacity = connections.cleanup_tx().capacity(); + + let sub_task = tokio::spawn({ + let app_cfg = Arc::clone(&app_cfg); + let manager = Arc::clone(&manager); + let providers = Arc::clone(&providers); + let connections = Arc::clone(&connections); + async move { + let ctx = super::super::SharedStreamCtx { + app_config: &app_cfg, + shared_stream_manager: &manager, + active_provider: &providers, + connection_manager: &connections, + }; + SharedStreamManager::subscribe_shared_stream(ctx, url, &addr, id, 0, ConnectionKind::Normal).await + } + }); + + barrier.wait().await; + + { + let mut reg = manager.shared_streams.write().await; + reg.by_key.remove(url); + } + + let res = sub_task.await??; + assert!(res.is_none()); + assert_eq!(connections.cleanup_tx().capacity(), initial_capacity); + assert!(manager.get_shared_state(url).await.is_none()); + Ok(()) +} + +#[test] +fn pending_meter_rollback_does_not_remove_adopted_entry() { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let providers = Arc::new(ActiveProviderManager::new(&app_cfg, &events)); + let manager = Arc::new(SharedStreamManager::new(providers)); + let url = "https://example.invalid/live/meter-replacement.ts"; + + // A reserved the meter; B then commits a shared origin and adopts it. + let (_, stale) = manager.reserve_meter_uid(url, || 51); + manager.adopt_meter_uid(url, 999); + + // A aborts: its pending rollback must not remove the adopted entry. + drop(stale); + assert_eq!(manager.meter_count(), 1); + assert_eq!(manager.lock_meter_uids().get(url).map(|entry| entry.uid), Some(51)); + assert_eq!(manager.lock_meter_uids().get(url).and_then(|entry| entry.owner), Some(999)); +} + +#[tokio::test] +async fn shutdown_clears_shared_meter_registrations() { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let providers = Arc::new(ActiveProviderManager::new(&app_cfg, &events)); + let manager = Arc::new(SharedStreamManager::new(providers)); + + let (_, pending) = manager.reserve_meter_uid("https://example.invalid/live/meter-shutdown.ts", || 61); + pending.expect("new registration").commit(); + assert_eq!(manager.meter_count(), 1); + + manager.shutdown().await; + assert_eq!(manager.meter_count(), 0); +} diff --git a/backend/session/src/streams/shared_stream_manager/tests/policy.rs b/backend/session/src/streams/shared_stream_manager/tests/policy.rs new file mode 100644 index 000000000..2c626b9de --- /dev/null +++ b/backend/session/src/streams/shared_stream_manager/tests/policy.rs @@ -0,0 +1,38 @@ +use super::*; + +#[test] +fn uncommitted_shared_meter_registration_is_rolled_back() { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let providers = Arc::new(ActiveProviderManager::new(&app_cfg, &events)); + let manager = Arc::new(SharedStreamManager::new(providers)); + let url = "https://example.invalid/live/meter-rollback.ts"; + + let (meter_uid, pending) = manager.reserve_meter_uid(url, || 41); + assert_eq!(meter_uid, 41); + assert_eq!(manager.meter_count(), 1); + + drop(pending); + assert_eq!(manager.meter_count(), 0); +} + +#[test] +fn successor_adoption_overwrites_stale_owner_and_survives_old_teardown() { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let providers = Arc::new(ActiveProviderManager::new(&app_cfg, &events)); + let manager = Arc::new(SharedStreamManager::new(providers)); + let url = "https://example.invalid/live/meter-successor.ts"; + + let (uid, guard) = manager.reserve_meter_uid(url, || 71); + manager.adopt_meter_uid(url, 100); + // A successor origin commits the same URL and takes over the meter. + manager.adopt_meter_uid(url, 200); + assert_eq!(manager.lock_meter_uids().get(url).and_then(|entry| entry.owner), Some(200)); + + // A stale teardown by the predecessor must not remove the successor's entry. + manager.remove_meter_uid_if_owned_by(url, Some(100)); + assert_eq!(manager.meter_count(), 1); + assert_eq!(manager.lock_meter_uids().get(url).map(|entry| entry.uid), Some(uid)); + guard.expect("reservation").commit(); +} diff --git a/backend/session/src/streams/shared_stream_manager/tests/streaming.rs b/backend/session/src/streams/shared_stream_manager/tests/streaming.rs new file mode 100644 index 000000000..4d6b03876 --- /dev/null +++ b/backend/session/src/streams/shared_stream_manager/tests/streaming.rs @@ -0,0 +1,163 @@ +use super::*; + +#[tokio::test] +async fn subscribers_on_same_socket_do_not_replace_each_other() -> Result<(), Box> { + let state = SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None); + let addr = "127.0.0.1:41003".parse()?; + let first = CancellationToken::new(); + let second = CancellationToken::new(); + state.register_subscriber(SharedSubscriberId::from_stream_uid(1), &addr, first.clone()).await; + state.register_subscriber(SharedSubscriberId::from_stream_uid(2), &addr, second.clone()).await; + assert_eq!(state.subscribers.read().await.len(), 2); + assert!(!first.is_cancelled()); + assert!(!second.is_cancelled()); + Ok(()) +} + +#[tokio::test] +async fn old_origin_cannot_unregister_replacement() { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let (_, _, manager, _) = create_test_connection_manager(&app_cfg, &events); + let url = "https://example.invalid/live/replaced.ts"; + let old = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + let current = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + manager.shared_streams.write().await.by_key.insert(Arc::from(url), Arc::clone(¤t)); + manager.unregister(url, &old).await; + assert!(manager.get_shared_state(url).await.is_some_and(|state| Arc::ptr_eq(&state, ¤t))); + assert!(!current.stop_token.is_cancelled()); +} + +#[test] +fn test_burst_buffer_eviction_is_byte_bounded() { + let mut buffer = BurstBuffer::new(10); + buffer.push(Bytes::from_static(b"12345")); + buffer.push(Bytes::from_static(b"67890")); + buffer.push(Bytes::from_static(b"abcde")); + + let mut chunks = Vec::new(); + let read = buffer.read_from_into(0, &mut chunks, usize::MAX); + + assert_eq!(buffer.current_bytes, 10); + assert_eq!(read.skipped, 1); + assert_eq!(read.next_sequence, 3); + assert_eq!(chunks.len(), 2); +} + +#[test] +fn test_burst_buffer_reads_clone_bytes_without_copying_payload() { + let mut buffer = BurstBuffer::new(1024); + let chunk = Bytes::from(vec![1_u8, 2, 3, 4]); + let ptr = chunk.as_ptr(); + buffer.push(chunk); + + let mut chunks = Vec::new(); + let _read = buffer.read_from_into(0, &mut chunks, usize::MAX); + let Some(read_chunk) = chunks.first() else { + panic!("expected one buffered chunk"); + }; + + assert_eq!(read_chunk.as_ptr(), ptr); +} + +#[test] +fn test_burst_buffer_live_batch_is_chunk_bounded() { + let mut buffer = BurstBuffer::new(4096); + for _ in 0..10 { + buffer.push(Bytes::from_static(b"x")); + } + + let mut chunks = Vec::new(); + let first = buffer.read_from_into(0, &mut chunks, 4); + assert_eq!(chunks.len(), 4); + assert_eq!(first.next_sequence, 4); + assert_eq!(first.skipped, 0); + + let second = buffer.read_from_into(first.next_sequence, &mut chunks, 4); + assert_eq!(chunks.len(), 4); + assert_eq!(second.next_sequence, 8); + + let third = buffer.read_from_into(second.next_sequence, &mut chunks, 4); + assert_eq!(chunks.len(), 2); + assert_eq!(third.next_sequence, 10); +} + +#[tokio::test] +async fn broadcast_does_not_emit_shared_stream_health_events() { + let app_cfg = create_test_app_config(); + let event_manager = Arc::new(EventManager::new()); + let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let shared_manager = Arc::new(SharedStreamManager::new(provider_manager)); + let mut events = event_manager.get_event_channel(); + let stream_url = "https://user:pass@example.invalid/live/health.ts"; + let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE.max(8), None, 1024, None)); + { + let mut reg = shared_manager.shared_streams.write().await; + reg.by_key.insert(Arc::from(stream_url), Arc::clone(&state)); + } + + let (tx, rx) = mpsc::channel::>(8); + state.broadcast(stream_url, ReceiverStream::new(rx), Arc::clone(&shared_manager)); + + assert!( + timeout(Duration::from_millis(100), events.recv()).await.is_err(), + "broadcast startup must not emit runtime events" + ); + + // Pushing data must also not emit a per-chunk health event. + tx.send(Ok(Bytes::from_static(b"payload-1"))).await.unwrap_or_else(|_| panic!("send chunk should succeed")); + assert!( + timeout(Duration::from_millis(100), events.recv()).await.is_err(), + "pushing a chunk must not emit runtime events" + ); + + drop(tx); + let ended = timeout(Duration::from_secs(2), async { + loop { + if shared_manager.get_shared_state(stream_url).await.is_none() { + break; + } + tokio::task::yield_now().await; + } + }) + .await; + assert!(ended.is_ok(), "broadcast must exit after source end"); + assert!( + timeout(Duration::from_millis(100), events.recv()).await.is_err(), + "broadcast shutdown must not emit runtime events" + ); +} + +#[tokio::test] +async fn pending_join_keeps_origin_alive_when_last_subscriber_leaves() -> Result<(), Box> { + let app_cfg = Arc::new(create_test_app_config()); + let events = Arc::new(EventManager::new()); + let (_providers, _users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41019".parse()?; + let first_id = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url = "https://example.invalid/live/pending_join.ts"; + + let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + state.register_subscriber(first_id, &addr, CancellationToken::new()).await; + { + let mut reg = manager.shared_streams.write().await; + reg.by_key.insert(Arc::from(url), Arc::clone(&state)); + reg.key_by_subscriber.insert(first_id, Arc::from(url)); + } + + // Simulate a join that committed its registry entry but has not yet registered + // its subscriber in the origin. + let pending_join = super::super::PendingJoinGuard::new(&state); + assert!(state.has_pending_joins()); + + // The last (and only) subscriber leaves; the origin must not be torn down because + // the pending join is about to land on it. + manager.release_subscriber(first_id).await; + assert!(manager.get_shared_state(url).await.is_some(), "origin must survive a pending join"); + + // The join completes; the origin has no subscribers and no pending joins. + pending_join.commit(); + assert!(!state.has_pending_joins()); + assert!(state.subscribers.read().await.is_empty(), "origin has no subscribers after release"); + Ok(()) +} diff --git a/backend/session/src/streams/shared_stream_manager/tests/terminal.rs b/backend/session/src/streams/shared_stream_manager/tests/terminal.rs new file mode 100644 index 000000000..a9a3c4f6a --- /dev/null +++ b/backend/session/src/streams/shared_stream_manager/tests/terminal.rs @@ -0,0 +1,49 @@ +use super::*; + +#[tokio::test] +async fn test_preempted_shared_subscriber_switches_to_low_priority_fallback() { + let app_cfg = create_test_app_config(); + let event_manager = Arc::new(EventManager::new()); + let provider_manager = Arc::new(ActiveProviderManager::new(&app_cfg, &event_manager)); + let geoip = Arc::new(ArcSwapOption::default()); + let user_manager = Arc::new(ActiveUserManager::new(&Config::default(), &geoip, &event_manager)); + let shared_manager = Arc::new(SharedStreamManager::new(Arc::clone(&provider_manager))); + let connection_manager = + Arc::new(ConnectionManager::new(&user_manager, &provider_manager, &shared_manager, &event_manager, None)); + + let addr: SocketAddr = "127.0.0.1:43001".parse().unwrap_or_else(|_| unreachable!()); + let mut ts_packet = vec![0_u8; 188]; + ts_packet[0] = 0x47; + + let low_priority_fallback = tuliprox_mpegts::transport_stream_buffer::TransportStreamBuffer::new(ts_packet); + let state = + Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE.max(8), None, 1024, Some(low_priority_fallback))); + + let subscriber_id = SharedSubscriberId::from_stream_uid(1); + let Ok(pending_cleanup) = + SharedStreamManager::reserve_subscriber_cleanup(&connection_manager, subscriber_id, addr).await + else { + panic!("shared subscriber admission failed"); + }; + let (mut stream, _provider, _capability) = state + .subscribe( + &addr, + subscriber_id, + connection_manager, + pending_cleanup, + super::super::PendingJoinGuard::new(&state), + ) + .await; + + state.preempted_token.cancel(); + drop(state); + + let first = timeout(Duration::from_secs(2), stream.next()).await; + let Ok(maybe_chunk) = first else { panic!("timed out waiting for fallback chunk") }; + let chunk = match maybe_chunk { + Some(Ok(bytes)) => bytes, + Some(Err(err)) => panic!("fallback stream returned error: {err}"), + None => panic!("fallback stream ended unexpectedly"), + }; + assert!(!chunk.is_empty(), "fallback chunk must contain MPEG-TS bytes"); +} diff --git a/backend/session/src/streams/shared_stream_manager/tests/transport.rs b/backend/session/src/streams/shared_stream_manager/tests/transport.rs new file mode 100644 index 000000000..c54b80496 --- /dev/null +++ b/backend/session/src/streams/shared_stream_manager/tests/transport.rs @@ -0,0 +1,112 @@ +use super::*; + +#[tokio::test] +async fn send_client_chunk_respects_progress_deadline_while_queue_is_full() -> Result<(), Box> { + let (tx, _rx) = mpsc::channel::(1); + let byte_budget = Arc::new(Semaphore::new(1024)); + let queued_permit = byte_budget.clone().acquire_many_owned(6).await?; + tx.send(BudgetedChunk { bytes: Bytes::from_static(b"queued"), _permit: queued_permit }).await?; + let cancel = CancellationToken::new(); + let deadline = Instant::now() + Duration::from_millis(50); + let outcome = timeout( + Duration::from_secs(1), + send_client_chunk(&tx, Bytes::from_static(b"blocked"), &cancel, deadline, &byte_budget), + ) + .await?; + assert_eq!(outcome, SendOutcome::TimedOut); + Ok(()) +} + +#[tokio::test] +async fn send_client_chunk_respects_byte_budget() -> Result<(), Box> { + let (tx, _rx) = mpsc::channel::(16); + // Byte budget of 4 bytes cannot accommodate a 6-byte chunk, so the acquire must + // time out rather than exceed the budget. + let byte_budget = Arc::new(Semaphore::new(4)); + let cancel = CancellationToken::new(); + let deadline = Instant::now() + Duration::from_millis(50); + let outcome = timeout( + Duration::from_secs(1), + send_client_chunk(&tx, Bytes::from_static(b"123456"), &cancel, deadline, &byte_budget), + ) + .await?; + assert_eq!(outcome, SendOutcome::TimedOut); + Ok(()) +} + +#[tokio::test] +async fn shared_timeout_ends_only_its_subscriber() -> Result<(), Box> { + let app_cfg = create_test_app_config(); + let events = Arc::new(EventManager::new()); + let (_, users, manager, connections) = create_test_connection_manager(&app_cfg, &events); + let addr = "127.0.0.1:41007".parse()?; + let first = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let second = SharedSubscriberId::from_stream_uid(connections.next_stream_uid()); + let url: Arc = Arc::from("https://example.invalid/timeout.ts"); + register_user(&users, addr, first.stream_uid(), "same-user", &url).await?; + register_user(&users, addr, second.stream_uid(), "same-user", &url).await?; + let state = Arc::new(SharedStreamState::new(Vec::new(), CHANNEL_SIZE, None, 1024, None)); + let surviving_token = CancellationToken::new(); + state.register_subscriber(first, &addr, CancellationToken::new()).await; + state.register_subscriber(second, &addr, surviving_token.clone()).await; + { + let mut register = manager.shared_streams.write().await; + register.by_key.insert(Arc::clone(&url), state); + register.key_by_subscriber.insert(first, Arc::clone(&url)); + register.key_by_subscriber.insert(second, Arc::clone(&url)); + } + let permit = connections.cleanup_tx().reserve_owned().await.ok(); + let mut response = super::super::ReceiverStreamWrapper { + stream: futures::stream::pending::(), + start: None, + subscriber_id: first, + addr, + permit, + request_id: None, + owner: None, + deadline: Some(Box::pin(tokio::time::sleep(Duration::ZERO))), + released: false, + }; + assert!(timeout(Duration::from_secs(2), response.next()).await?.is_none()); + timeout(Duration::from_secs(2), async { + while users.active_streams().await.len() != 1 { + tokio::task::yield_now().await; + } + }) + .await?; + assert_eq!(users.active_streams().await.first().map(|stream| stream.uid), Some(second.stream_uid())); + assert!(!surviving_token.is_cancelled()); + assert!(manager.get_shared_state(&url).await.is_some()); + Ok(()) +} + +#[test] +fn test_burst_buffer_eviction_is_chunk_bounded_for_small_packets() { + let max_chunks = 3; + let mut buffer = BurstBuffer::new(MIN_BURST_BUFFER_CHUNK_ACCOUNTING_BYTES * max_chunks); + for _ in 0..max_chunks.saturating_add(1) { + buffer.push(Bytes::from_static(b"x")); + } + + let mut chunks = Vec::new(); + let read = buffer.read_from_into(0, &mut chunks, usize::MAX); + + assert_eq!(buffer.buffer.len(), max_chunks); + assert_eq!(buffer.current_bytes, max_chunks); + assert_eq!(read.skipped, 1); + assert_eq!(chunks.len(), max_chunks); +} + +#[test] +fn test_burst_buffer_keeps_oversized_packet_as_single_latest_chunk() { + let mut buffer = BurstBuffer::new(4); + buffer.push(Bytes::from_static(b"12345678")); + + let mut chunks = Vec::new(); + let read = buffer.read_from_into(0, &mut chunks, usize::MAX); + + assert_eq!(buffer.buffer.len(), 1); + assert_eq!(buffer.current_bytes, 8); + assert_eq!(read.next_sequence, 1); + assert_eq!(chunks.len(), 1); +} diff --git a/docs/src/configuration/shared-hls-configuration.md b/docs/src/configuration/shared-hls-configuration.md index 11c2f1ec9..74d5f2b12 100644 --- a/docs/src/configuration/shared-hls-configuration.md +++ b/docs/src/configuration/shared-hls-configuration.md @@ -246,3 +246,82 @@ clear in-memory HLS sessions, access leases, QoS state, and repair state that po Changing `rewrite_secret` is also disruptive for public HLS identifiers. Existing player URLs may reference old `proxy_session_id` or transient resource IDs and should be treated as stale. + +## Optional startup modes + +`reverse_proxy.hls_cache.startup.mode` defaults to `conservative`. Omitted startup configuration keeps the existing +READY reserve, cache delivery and repair behavior. `first_ready` lowers initial admission to one processed start +segment. Every listed successor must already have a successful HTTP 200 response and a nonempty body prefix. + +```yaml +reverse_proxy: + hls_cache: + startup: + mode: first_ready + max_progressive_segments: 16 + max_progressive_bytes_per_segment: 32MB + max_progressive_bytes_total: 128MB + max_progressive_reader_lifetime_secs: 90 + max_deferred_repairs: 32 +``` + +`progressive` lets the first full segment request consume the existing shared origin fill before origin EOF. +The complete session delivers immutable raw TS revisions, including subsequent requests, retries and ranges. +Repair and the corruption watchdog continue on the processed cache copy. Raw delivery does not correct source +timestamp or transport-stream defects; prefer `first_ready` when a source needs repair. + +Fast startup is limited to live, clear MPEG-TS objects without MAP or origin byte ranges. Archive, encrypted, +fragmented-MP4 and transient sessions use their existing path. Before publication an incompatible source can fall back to that path. +After a Fast Start manifest is published, a refresh requiring transient delivery is rejected to preserve +the bound revisions; it cannot silently change the active session representation. A progressive `Range: bytes=0-` request may receive +a complete HTTP 200 streaming response; other ranges wait for the bound revision to complete. + +The replay limits apply across sessions and remain shared during configuration reload. Byte clones stay charged +until their last reference is released. An absolute reader lifetime prevents a stalled client from retaining a +revision indefinitely. An origin drop or replay overflow after streaming begins produces a body error; it cannot turn into successful EOF +or silently change the bytes of an already published URI. Revision and repair peak reservations share the existing +disk cache limits. Conservative behavior remains the default; existing sessions keep their startup policy after +reload. Restart after changing to `conservative` when an immediate rollback of running sessions is required. + +Existing provider and fetch concurrency limits remain in effect. The initial playlist needs at least three +publishable segments. With fewer than three concurrent fills, a cold session can still wait for an earlier +origin EOF before that window is available. Raising the existing per-session concurrency limit to three is an +operator choice and must fit the provider's connection allowance; startup mode never raises it automatically. + +Measure both first data and decoded-picture time before enabling progressive mode in production. Short successful +response runs do not establish long-term player, origin-failure or resource stability. + +## Reproducible startup checks + +The testkit has isolated shared-HLS scenarios for `conservative`, `first_ready` and `progressive`, plus missing-head +scenarios for both fast modes. Existing scenarios retain their original origin behavior. The new +`origin.hls_profile` publishes six segments, sends a complete synthetic frame immediately, and delays the second +frame by `prefix_delay_millis`. `missing_head_segments` returns alternating HTTP 404/410 at the head; +`abort_segments` injects a body error after the prefix. `unknown_length` omits origin Content-Length. +These frames test HTTP delivery and cache identity, not MPEG-TS decoding or picture startup. + +```bash +cargo +stable build --package tuliprox --package tuliprox-testkit +export TULIPROX_TESTKIT_SUT_BINARY="$PWD/target/debug/tuliprox" +cargo +stable test --package tuliprox-testkit --test hls_fast_start_process -- --ignored --test-threads=1 + +target/debug/tuliprox-testkit controller \ + --scenario test/fixtures/testkit/scenarios/m3u-hls-share-progressive-startup.yml \ + --report-directory /tmp/hls-progressive-report +``` + +The process tests check prefix delivery before origin EOF, processed delivery after EOF, 404/410 recovery, +terminal errors for both known-length and chunked origin drops, byte-identical retries and HTTP 206 ranges. +They also check that successful retries and ranges do not create another origin segment download. The fixed +local timing thresholds distinguish the intentionally delayed fixture responses; they are not production SLAs. + +For a synthetic HLS playback URL, `hls-probe` emits first-byte/body-end timing and revision checks as JSON: + +```bash +target/debug/tuliprox-testkit hls-probe \ + --url "$TESTKIT_HLS_URL" --run-id "$TESTKIT_RUN_ID" --expected-marker 17 \ + --verify-revision --min-prefix-to-eof-millis 500 +``` + +Use `--expect-body-error` instead of `--verify-revision` for an injected segment-body drop. The probe keeps one +absolute 30-second deadline and a 1 MiB response bound. It does not print the playback URL in its JSON result. diff --git a/frontend/public/assets/i18n/ar.json b/frontend/public/assets/i18n/ar.json index 763a59fac..979ae54ec 100644 --- a/frontend/public/assets/i18n/ar.json +++ b/frontend/public/assets/i18n/ar.json @@ -1,4 +1,5 @@ { + "HLS_STARTUP_MODE_DESCRIPTION": "الوضع المحافظ يحافظ على سلوك البدء الحالي. وضع «أول جاهز» ينتظر مقطع بداية معالجًا. الوضع التدريجي يقدّم TS خامًا ويتطلب مصادر مناسبة؛ ويسري على الجلسات الجديدة.", "BACKOFF_JITTER_PERCENT_VALIDATION": "يجب أن يكون نطاق نسبة تباعد الارتداد بين 0 و95.", "COUNTRY": { "AD": "أندورا", @@ -904,6 +905,16 @@ "WEB_ROOT": "Web-Root: يحتوي على ملفات واجهة الويب. عادة ما يتم وضعها في نفس الدليل تحت ./web" }, "LABEL": { + "HLS_STARTUP_LIMITS": "حدود البدء السريع", + "HLS_PROGRESSIVE_SEGMENTS": "الحد الأقصى للمقاطع التدريجية", + "HLS_PROGRESSIVE_BYTES_PER_SEGMENT": "ذاكرة المخزن المؤقت لكل مقطع", + "HLS_PROGRESSIVE_BYTES_TOTAL": "إجمالي ذاكرة المخزن المؤقت", + "HLS_PROGRESSIVE_READER_LIFETIME": "أقصى عمر للقارئ (بالثواني)", + "HLS_DEFERRED_REPAIRS": "الحد الأقصى لمهام الإصلاح المعلقة", + "HLS_STARTUP_MODE": "وضع بدء HLS", + "HLS_STARTUP_MODE_CONSERVATIVE": "محافظ", + "HLS_STARTUP_MODE_FIRST_READY": "أول جاهز", + "HLS_STARTUP_MODE_PROGRESSIVE": "تدريجي", "BACK": "رجوع", "BOUQUETS": "الباقات", "ALL_TARGETS": "جميع الأهداف", diff --git a/frontend/public/assets/i18n/en.json b/frontend/public/assets/i18n/en.json index de3903c6a..872586a1e 100644 --- a/frontend/public/assets/i18n/en.json +++ b/frontend/public/assets/i18n/en.json @@ -1,4 +1,5 @@ { + "HLS_STARTUP_MODE_DESCRIPTION": "Conservative keeps the existing startup behavior. First ready waits for a processed start segment. Progressive delivers raw TS and requires suitable sources; it takes effect for new sessions.", "BACKOFF_JITTER_PERCENT_VALIDATION": "Backoff jitter percent must be between 0 and 95.", "COUNTRY": { "AD": "Andorra", @@ -904,6 +905,16 @@ "WEB_ROOT": "Web-Root: Contains the web-ui files. They are normally placed in the same directory under ./web" }, "LABEL": { + "HLS_STARTUP_LIMITS": "Fast start limits", + "HLS_PROGRESSIVE_SEGMENTS": "Maximum progressive segments", + "HLS_PROGRESSIVE_BYTES_PER_SEGMENT": "Replay memory per segment", + "HLS_PROGRESSIVE_BYTES_TOTAL": "Total replay memory", + "HLS_PROGRESSIVE_READER_LIFETIME": "Maximum reader lifetime (seconds)", + "HLS_DEFERRED_REPAIRS": "Maximum pending repair jobs", + "HLS_STARTUP_MODE": "HLS startup mode", + "HLS_STARTUP_MODE_CONSERVATIVE": "Conservative", + "HLS_STARTUP_MODE_FIRST_READY": "First ready", + "HLS_STARTUP_MODE_PROGRESSIVE": "Progressive", "ALL_TARGETS": "All targets", "ACCEPT_INSECURE_SSL_CERTIFICATES": "Accept Insecure SSL Certificates", "ACTIVE": "Active", diff --git a/frontend/public/assets/i18n/es.json b/frontend/public/assets/i18n/es.json index 4790a1fd3..3ce7476ec 100644 --- a/frontend/public/assets/i18n/es.json +++ b/frontend/public/assets/i18n/es.json @@ -1,4 +1,5 @@ { + "HLS_STARTUP_MODE_DESCRIPTION": "Conservador mantiene el comportamiento de inicio existente. Primero listo espera un segmento inicial procesado. Progresivo entrega TS sin procesar y requiere fuentes adecuadas; se aplica a las sesiones nuevas.", "BACKOFF_JITTER_PERCENT_VALIDATION": "El porcentaje de fluctuación (jitter) del retroceso debe estar entre 0 y 95.", "COUNTRY": { "AD": "Andorra", @@ -904,6 +905,16 @@ "WEB_ROOT": "Web-Root: contiene los archivos de la interfaz web. Normalmente se colocan en el mismo directorio bajo ./web" }, "LABEL": { + "HLS_STARTUP_LIMITS": "Límites de inicio rápido", + "HLS_PROGRESSIVE_SEGMENTS": "Máximo de segmentos progresivos", + "HLS_PROGRESSIVE_BYTES_PER_SEGMENT": "Memoria de búfer por segmento", + "HLS_PROGRESSIVE_BYTES_TOTAL": "Memoria de búfer total", + "HLS_PROGRESSIVE_READER_LIFETIME": "Vida máxima del lector (segundos)", + "HLS_DEFERRED_REPAIRS": "Máximo de reparaciones pendientes", + "HLS_STARTUP_MODE": "Modo de inicio HLS", + "HLS_STARTUP_MODE_CONSERVATIVE": "Conservador", + "HLS_STARTUP_MODE_FIRST_READY": "Primero listo", + "HLS_STARTUP_MODE_PROGRESSIVE": "Progresivo", "ALL_TARGETS": "Todos los destinos", "ACCEPT_INSECURE_SSL_CERTIFICATES": "Aceptar certificados SSL no seguros", "ACTIVE": "Activo", diff --git a/frontend/public/assets/i18n/ru.json b/frontend/public/assets/i18n/ru.json index 9b2e4d174..22dfbd5c9 100644 --- a/frontend/public/assets/i18n/ru.json +++ b/frontend/public/assets/i18n/ru.json @@ -1,4 +1,5 @@ { + "HLS_STARTUP_MODE_DESCRIPTION": "«Консервативный» сохраняет прежнее поведение запуска. «Первый готовый» ждёт обработанный начальный сегмент. «Прогрессивный» отдаёт необработанный TS и требует подходящих источников; действует для новых сессий.", "BACKOFF_JITTER_PERCENT_VALIDATION": "Процент дрожания (jitter) при экспоненциальной задержке должен быть от 0 до 95.", "COUNTRY": { "AD": "Андорра", @@ -904,6 +905,16 @@ "WEB_ROOT": "Web-Root: Содержит файлы веб-интерфейса. Они обычно размещаются в той же директории под ./web" }, "LABEL": { + "HLS_STARTUP_LIMITS": "Ограничения быстрого старта", + "HLS_PROGRESSIVE_SEGMENTS": "Максимум прогрессивных сегментов", + "HLS_PROGRESSIVE_BYTES_PER_SEGMENT": "Буферная память на сегмент", + "HLS_PROGRESSIVE_BYTES_TOTAL": "Общая буферная память", + "HLS_PROGRESSIVE_READER_LIFETIME": "Максимальное время жизни читателя (секунды)", + "HLS_DEFERRED_REPAIRS": "Максимум ожидающих задач восстановления", + "HLS_STARTUP_MODE": "Режим запуска HLS", + "HLS_STARTUP_MODE_CONSERVATIVE": "Консервативный", + "HLS_STARTUP_MODE_FIRST_READY": "Первый готовый", + "HLS_STARTUP_MODE_PROGRESSIVE": "Прогрессивный", "BACK": "Назад", "BOUQUETS": "Букеты", "ALL_TARGETS": "Все цели", diff --git a/frontend/src/app/components/config/reverse_proxy_config_view.rs b/frontend/src/app/components/config/reverse_proxy_config_view.rs index e0ed27fcf..d6a5940c5 100644 --- a/frontend/src/app/components/config/reverse_proxy_config_view.rs +++ b/frontend/src/app/components/config/reverse_proxy_config_view.rs @@ -30,9 +30,10 @@ use shared::{ model::{ ByteSize, CacheConfigDto, GeoIpConfigDto, GeoIpUnavailablePolicy, HlsCacheConfigDto, HlsCorruptSegmentWatchdogMode, HlsManifestRecoveryBurstConfigDto, HlsManifestRecoveryBurstLevel, - HlsSegmentRepairConfigDto, HlsSegmentRepairMode, HlsStripConfigDto, HlsStripMode, Millis, - QosAggregationConfigDto, RateLimitConfigDto, ResourceRetryConfigDto, ReverseProxyConfigDto, - ReverseProxyDisabledHeaderConfigDto, Secs, StreamBufferConfigDto, StreamConfigDto, StreamHistoryConfigDto, + HlsSegmentRepairConfigDto, HlsSegmentRepairMode, HlsStartupConfigDto, HlsStartupMode, HlsStripConfigDto, + HlsStripMode, Millis, QosAggregationConfigDto, RateLimitConfigDto, ResourceRetryConfigDto, + ReverseProxyConfigDto, ReverseProxyDisabledHeaderConfigDto, Secs, StreamBufferConfigDto, StreamConfigDto, + StreamHistoryConfigDto, }, utils::format_float_localized, }; @@ -239,6 +240,7 @@ generate_form_reducer!( OriginSegmentTimeoutMs => origin_segment_timeout_ms: Millis, SessionIdleTimeout => session_idle_timeout: Secs, SegmentRepair => segment_repair: HlsSegmentRepairConfigDto, + Startup => startup: HlsStartupConfigDto, } ); @@ -416,6 +418,70 @@ fn clamp_u8_range(value: Option, min_value: u8, max_value: u8) -> u8 { value.and_then(|value| u8::try_from(value).ok()).map_or(min_value, |value| value.clamp(min_value, max_value)) } +fn render_hls_startup_limits( + state: &UseReducerHandle, + translate: &YewI18n, + edit: bool, +) -> Html { + let number = |label: &str, field: &'static str, value: usize, set: fn(&mut HlsStartupConfigDto, usize)| { + let state = state.clone(); + if edit { + html! { +

+ | { + let mut startup = state.form.startup.clone(); + set(&mut startup, clamp_usize_min(value, 1)); + state.dispatch(HlsCacheConfigFormAction::Startup(startup)); + })} /> +
+ } + } else { + config_field_child!(translate.t(label), field, { + html! { {value.to_string()} } + }) + } + }; + let bytes = |label: &str, field: &'static str, value: &ByteSize, set: fn(&mut HlsStartupConfigDto, ByteSize)| { + let state = state.clone(); + if edit { + html! { +
+ +
+ } + } else { + config_field_child!(translate.t(label), field, { + html! { {value.to_string()} } + }) + } + }; + let startup = &state.form.startup; + html! { +
+ {translate.t("LABEL.HLS_STARTUP_LIMITS")} + {number("LABEL.HLS_PROGRESSIVE_SEGMENTS", "hls_progressive_segments", startup.max_progressive_segments, + |startup, value| startup.max_progressive_segments = value)} + {bytes("LABEL.HLS_PROGRESSIVE_BYTES_PER_SEGMENT", "hls_progressive_bytes_per_segment", &startup.max_progressive_bytes_per_segment, + |startup, value| startup.max_progressive_bytes_per_segment = value)} + {bytes("LABEL.HLS_PROGRESSIVE_BYTES_TOTAL", "hls_progressive_bytes_total", &startup.max_progressive_bytes_total, + |startup, value| startup.max_progressive_bytes_total = value)} + {number("LABEL.HLS_PROGRESSIVE_READER_LIFETIME", "hls_progressive_reader_lifetime", + usize::try_from(startup.max_progressive_reader_lifetime_secs.get()).unwrap_or(usize::MAX), + |startup, value| startup.max_progressive_reader_lifetime_secs = Secs::new(u64::try_from(value).unwrap_or(u64::MAX)))} + {number("LABEL.HLS_DEFERRED_REPAIRS", "hls_deferred_repairs", startup.max_deferred_repairs, + |startup, value| startup.max_deferred_repairs = value)} +
+ } +} + #[component] pub fn ReverseProxyConfigView() -> Html { let translate = use_translation(); @@ -802,6 +868,10 @@ pub fn ReverseProxyConfigView() -> Html {

{translate.t(LABEL_HLS_CACHE_PROXY)}

{ config_field_optional!(hls_cache_state.form, translate.t(LABEL_CACHE_PATH), cache_path) } + { config_field_child!(translate.t("LABEL.HLS_STARTUP_MODE"), "HLS_CACHE_CONFIG.STARTUP_MODE", { + html! { {hls_cache_state.form.startup.mode.to_string()} } + }) } + {render_hls_startup_limits(&hls_cache_state, &translate, false)} { config_field_child!(translate.t(LABEL_STRIP_MODE), "HLS_CACHE_CONFIG.STRIP_MODE", { html! { {hls_strip_state.form.mode.to_string()} } }) } @@ -913,6 +983,36 @@ pub fn ReverseProxyConfigView() -> Html { } }; + let selected_startup = hls_cache_state.form.startup.mode; + let startup_options = Rc::new(vec![ + DropDownOption::new( + "conservative", + html! { translate.t("LABEL.HLS_STARTUP_MODE_CONSERVATIVE") }, + selected_startup == HlsStartupMode::Conservative, + ), + DropDownOption::new( + "first_ready", + html! { translate.t("LABEL.HLS_STARTUP_MODE_FIRST_READY") }, + selected_startup == HlsStartupMode::FirstReady, + ), + DropDownOption::new( + "progressive", + html! { translate.t("LABEL.HLS_STARTUP_MODE_PROGRESSIVE") }, + selected_startup == HlsStartupMode::Progressive, + ), + ]); + let set_startup = { + let hls_cache_state = hls_cache_state.clone(); + Callback::from(move |(_, selections): (String, DropDownSelection)| { + if let DropDownSelection::Single(selection) = selections { + if let Ok(mode) = HlsStartupMode::from_str(selection.as_str()) { + let mut startup = hls_cache_state.form.startup.clone(); + startup.mode = mode; + hls_cache_state.dispatch(HlsCacheConfigFormAction::Startup(startup)); + } + } + }) + }; let selected_manifest_recovery_burst = hls_cache_state.form.manifest_recovery_burst.level; let set_manifest_recovery_burst = { let hls_cache_state = hls_cache_state.clone(); @@ -930,6 +1030,11 @@ pub fn ReverseProxyConfigView() -> Html {

{translate.t(LABEL_HLS_CACHE_PROXY)}

{ edit_field_text_option!(hls_cache_state, translate.t(LABEL_CACHE_PATH), cache_path, HlsCacheConfigFormAction::CachePath) } + { config_field_child!(translate.t("LABEL.HLS_STARTUP_MODE"), "HLS_CACHE_CONFIG.STARTUP_MODE", { + html! { , -} - -impl PartialEq for CompiledRegex { - fn eq(&self, other: &Self) -> bool { self.restr == other.restr } -} - -#[derive(Parser)] -#[grammar_inline = r#" -WHITESPACE = _{ " " | "\t" | "\r" | "\n"} -field = { ^"group" | ^"title" | ^"name" | ^"genre" | ^"url" | ^"input" | ^"caption" | ^"epgid"} -numeric_field = { ^"chno" | ^"quality" } -and = { ^"and" } -or = { ^"or" } -not = { ^"not" } -regexp = @{ "\"" ~ ( "\\\"" | (!"\"" ~ ANY) )* ~ "\"" } -number = @{ ASCII_DIGIT+ } -type_value = { ^"live" | ^"vod" | ^"movie" | ^"series" } -type_comparison = { ^"type" ~ "=" ~ type_value } -field_comparison_value = _{ regexp } -field_comparison = { field ~ "~" ~ field_comparison_value } -str_op = { "!=" | "=" | ^"contains" | ^"startswith" } -string_comparison = { field ~ str_op ~ regexp } -presence_negation = { ^"not" } -presence_op = { (^"is" ~ presence_negation?) | "!=" | "=" } -presence_comparison = { field ~ presence_op ~ ^"empty" } -num_op = { ">=" | "<=" | "!=" | ">" | "<" | "=" } -numeric_comparison = { numeric_field ~ num_op ~ number } -set_values = { regexp ~ ("," ~ regexp)* } -set_comparison = { field ~ ^"in" ~ "[" ~ set_values ~ "]" } -comparison = { field_comparison | type_comparison | numeric_comparison | set_comparison | presence_comparison | string_comparison } -bool_op = { and | or } -expr_group = { "(" ~ expr ~ ")" } -basic_expr = _{ comparison | expr_group } -not_expr = _{ not ~ basic_expr } -expr = { - not_expr ~ (bool_op ~ expr)? - | basic_expr ~ (bool_op ~ expr)* -} -stmt = { expr ~ (bool_op ~ expr)* } -main = _{ SOI ~ stmt ~ EOI } -"#] -struct FilterParser; - -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum UnaryOperator { - Not, -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum StringOperator { - Eq, - NotEq, - Contains, - StartsWith, -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum PresenceOperator { - IsEmpty, - IsNotEmpty, -} - -impl std::fmt::Display for PresenceOperator { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.write_str(match self { - Self::IsEmpty => "IS EMPTY", - Self::IsNotEmpty => "IS NOT EMPTY", - }) - } -} - -impl std::fmt::Display for StringOperator { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.write_str(match *self { - Self::Eq => "=", - Self::NotEq => "!=", - Self::Contains => "CONTAINS", - Self::StartsWith => "STARTSWITH", - }) - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum NumericOperator { - Eq, - NotEq, - Greater, - GreaterOrEqual, - Less, - LessOrEqual, -} - -impl std::fmt::Display for NumericOperator { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.write_str(match *self { - Self::Eq => "=", - Self::NotEq => "!=", - Self::Greater => ">", - Self::GreaterOrEqual => ">=", - Self::Less => "<", - Self::LessOrEqual => "<=", - }) - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum BinaryOperator { - And, - Or, -} - -impl BinaryOperator { - const OP_OR: &'static str = "OR"; - const OP_AND: &'static str = "AND"; -} - -impl std::fmt::Display for BinaryOperator { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!( - f, - "{}", - match *self { - Self::Or => Self::OP_OR, - Self::And => Self::OP_AND, - } - ) - } -} - -#[derive(Debug, Clone, PartialEq)] -pub enum Filter { - Group(Box), - FieldComparison(ItemField, CompiledRegex), - TypeComparison(ItemField, PlaylistItemType), - StringComparison(ItemField, StringOperator, String), - PresenceComparison(ItemField, PresenceOperator), - NumericComparison(ItemField, NumericOperator, u32), - SetComparison(ItemField, Vec), - UnaryExpression(UnaryOperator, Box), - BinaryExpression(Box, BinaryOperator, Box), -} - -impl Default for Filter { - fn default() -> Self { - Self::Group(Box::new(Filter::FieldComparison( - ItemField::Group, - CompiledRegex { restr: ".*".to_string(), re: crate::model::REGEX_CACHE.get_or_compile(".*").unwrap() }, - ))) - } -} - -fn get_caption<'a>(provider: &'a ValueProvider<'_>, rewc: &CompiledRegex) -> (bool, Cow<'a, str>) { - if let Some(value) = provider.get_filter_value(ItemField::Title) { - if rewc.re.is_match(&value) { - return (true, value); - } - } - - if let Some(value) = provider.get_filter_value(ItemField::Name) { - if rewc.re.is_match(&value) { - return (true, value); - } - } - (false, Cow::Borrowed("")) -} - -fn contains_ignore_ascii_case(haystack: &str, needle: &str) -> bool { - if needle.is_empty() { - return true; - } - if needle.len() > haystack.len() { - return false; - } - haystack.as_bytes().windows(needle.len()).any(|window| window.eq_ignore_ascii_case(needle.as_bytes())) -} - -fn starts_with_ignore_ascii_case(haystack: &str, needle: &str) -> bool { - haystack.len() >= needle.len() && haystack.as_bytes()[..needle.len()].eq_ignore_ascii_case(needle.as_bytes()) -} - -fn string_operator_matches(actual: &str, op: StringOperator, value: &str) -> bool { - match op { - StringOperator::Eq => actual.eq_ignore_ascii_case(value), - StringOperator::NotEq => !actual.eq_ignore_ascii_case(value), - StringOperator::Contains => contains_ignore_ascii_case(actual, value), - StringOperator::StartsWith => starts_with_ignore_ascii_case(actual, value), - } -} - -/// Caption matches against the title and falls back to the name, mirroring `get_caption`. -fn string_caption_matches(provider: &ValueProvider<'_>, op: StringOperator, value: &str) -> bool { - let title = provider.get_filter_value(ItemField::Title).unwrap_or(Cow::Borrowed("")); - let name = provider.get_filter_value(ItemField::Name).unwrap_or(Cow::Borrowed("")); - match op { - // NotEq is the negation of Eq-on-either, not "either differs" - StringOperator::NotEq => !(title.eq_ignore_ascii_case(value) || name.eq_ignore_ascii_case(value)), - StringOperator::Eq | StringOperator::Contains | StringOperator::StartsWith => { - string_operator_matches(&title, op, value) || string_operator_matches(&name, op, value) - } - } -} - -impl Filter { - pub fn filter(&self, provider: &ValueProvider) -> bool { - match self { - Self::FieldComparison(field, rewc) => { - let (is_match, value) = if field == &ItemField::Caption { - get_caption(provider, rewc) - } else if let Some(value) = provider.get_filter_value(*field) { - (rewc.re.is_match(&value), value) - } else { - (false, Cow::Borrowed("")) - }; - if log_enabled!(Level::Trace) { - if is_match { - trace!("Match found: {rewc:?} {} => {field}='{value}'", rewc.restr); - } else { - trace!("Match failed: {self}: {rewc:?} {} => {field}='{value}'", rewc.restr); - } - } - is_match - } - Self::TypeComparison(field, item_type) => { - let actual = provider.pli.header.item_type; - let is_match = match item_type { - PlaylistItemType::Live => actual.is_live(), - PlaylistItemType::Video => actual.is_video(), - PlaylistItemType::Series => { - actual.is_series() - || matches!(actual, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo) - } - _ => actual == *item_type, - }; - if log_enabled!(Level::Trace) { - if is_match { - trace!("Match found: {field:?} {}", actual.as_str()); - } else { - trace!("Match failed: {self}: {field:?} {}", actual.as_str()); - } - } - is_match - } - Self::StringComparison(field, op, value) => { - // Missing values (e.g. absent epgid) are matched as empty strings. - let is_match = if *field == ItemField::Caption { - string_caption_matches(provider, *op, value) - } else { - let actual = provider.get_filter_value(*field).unwrap_or(Cow::Borrowed("")); - string_operator_matches(&actual, *op, value) - }; - if log_enabled!(Level::Trace) { - let actual = provider.get_filter_value(*field).unwrap_or(Cow::Borrowed("")); - if is_match { - trace!("Match found: {field} {op} \"{value}\" => '{actual}'"); - } else { - trace!("Match failed: {field} {op} \"{value}\" => '{actual}'"); - } - } - is_match - } - Self::PresenceComparison(field, op) => { - let is_empty = provider.get_filter_value(*field).is_none_or(|value| value.is_empty()); - match op { - PresenceOperator::IsEmpty => is_empty, - PresenceOperator::IsNotEmpty => !is_empty, - } - } - Self::NumericComparison(field, op, value) => { - let actual = match field { - ItemField::Chno => provider.pli.header.chno, - ItemField::Quality => u32::from(provider.quality_rank()), - _ => return false, - }; - match op { - NumericOperator::Eq => actual == *value, - NumericOperator::NotEq => actual != *value, - NumericOperator::Greater => actual > *value, - NumericOperator::GreaterOrEqual => actual >= *value, - NumericOperator::Less => actual < *value, - NumericOperator::LessOrEqual => actual <= *value, - } - } - Self::SetComparison(field, values) => { - if *field == ItemField::Caption { - let title = provider.get_filter_value(ItemField::Title).unwrap_or(Cow::Borrowed("")); - let name = provider.get_filter_value(ItemField::Name).unwrap_or(Cow::Borrowed("")); - values.iter().any(|value| title.eq_ignore_ascii_case(value) || name.eq_ignore_ascii_case(value)) - } else { - let actual = provider.get_filter_value(*field).unwrap_or(Cow::Borrowed("")); - values.iter().any(|value| actual.eq_ignore_ascii_case(value)) - } - } - Self::Group(expr) => expr.filter(provider), - Self::UnaryExpression(op, expr) => match op { - UnaryOperator::Not => !expr.filter(provider), - }, - Self::BinaryExpression(left, op, right) => match op { - BinaryOperator::And => left.filter(provider) && right.filter(provider), - BinaryOperator::Or => left.filter(provider) || right.filter(provider), - }, - } - } -} - -impl Filter { - const LIVE: &'static str = "live"; - const VOD: &'static str = "vod"; - const MOVIE: &'static str = "movie"; - const SERIES: &'static str = "series"; - const UNSUPPORTED: &'static str = "unsupported"; -} - -impl std::fmt::Display for Filter { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::FieldComparison(field, rewc) => { - write!(f, "{} ~ \"{}\"", field, String::from(&rewc.restr)) - } - Self::TypeComparison(field, item_type) => { - write!( - f, - "{} = {}", - field, - match item_type { - PlaylistItemType::Live => Self::LIVE, - PlaylistItemType::Video | PlaylistItemType::LocalVideo => Self::MOVIE, - PlaylistItemType::Series - | PlaylistItemType::SeriesInfo - | PlaylistItemType::LocalSeries - | PlaylistItemType::LocalSeriesInfo => Self::SERIES, // yes series-info is handled as series in filter - _ => Self::UNSUPPORTED, - } - ) - } - Self::Group(stmt) => { - write!(f, "({stmt})") - } - Self::StringComparison(field, op, value) => { - write!(f, "{field} {op} \"{}\"", value.replace('"', "\\\"")) - } - Self::PresenceComparison(field, op) => write!(f, "{field} {op}"), - Self::NumericComparison(field, op, value) => { - write!(f, "{field} {op} {value}") - } - Self::SetComparison(field, values) => { - let rendered = - values.iter().map(|v| format!("\"{}\"", v.replace('"', "\\\""))).collect::>().join(", "); - write!(f, "{field} IN [{rendered}]") - } - Self::UnaryExpression(op, expr) => { - let flt = match op { - UnaryOperator::Not => format!("NOT {expr}"), - }; - write!(f, "{flt}") - } - Self::BinaryExpression(left, op, right) => { - write!(f, "{left} {op} {right}") - } - } - } -} - -fn get_parser_item_field(expr: &Pair) -> Result { - if expr.as_rule() == Rule::field { - let field_text = expr.as_str(); - for item in ItemField::iter() { - if field_text.eq_ignore_ascii_case(item.as_ref()) { - return Ok(item); - } - } - } - Err(TuliproxError::FilterParse(format!("unknown field: {}", expr.as_str()))) -} - -fn get_parser_regexp(expr: &Pair, templates: Option<&[PatternTemplate]>) -> Result { - if expr.as_rule() == Rule::regexp { - let full_str = expr.as_str(); - let parsed_text = &full_str[1..full_str.len() - 1]; - let regstr = apply_templates_to_pattern_single(parsed_text, templates)?; - let re = crate::model::REGEX_CACHE.get_or_compile(regstr.as_str()); - if re.is_err() { - return Err(TuliproxError::RegexCompile(regstr)); - } - let regexp = re.unwrap(); - if log_enabled!(Level::Trace) { - trace!("Created regex: {regstr}"); - } - return Ok(CompiledRegex { restr: regstr, re: regexp }); - } - Err(TuliproxError::FilterParse(format!("unknown field: {}", expr.as_str()))) -} - -fn get_parser_field_comparison( - expr: Pair, - templates: Option<&[PatternTemplate]>, -) -> Result { - let mut expr_inner = expr.into_inner(); - match get_parser_item_field(&expr_inner.next().unwrap()) { - Ok(field) => match get_parser_regexp(&expr_inner.next().unwrap(), templates) { - Ok(regexp) => Ok(Filter::FieldComparison(field, regexp)), - Err(err) => Err(err), - }, - Err(err) => Err(err), - } -} - -fn get_parser_string_value(expr: &Pair, templates: Option<&[PatternTemplate]>) -> Result { - if expr.as_rule() == Rule::regexp { - let full_str = expr.as_str(); - let parsed_text = &full_str[1..full_str.len() - 1]; - let resolved = apply_templates_to_pattern_single(parsed_text, templates)?; - return Ok(resolved.replace("\\\"", "\"")); - } - Err(TuliproxError::FilterParse(format!("expected string literal: {}", expr.as_str()))) -} - -fn get_parser_string_operator(expr: &Pair) -> Result { - let text = expr.as_str(); - if text == "=" { - Ok(StringOperator::Eq) - } else if text == "!=" { - Ok(StringOperator::NotEq) - } else if text.eq_ignore_ascii_case("contains") { - Ok(StringOperator::Contains) - } else if text.eq_ignore_ascii_case("startswith") { - Ok(StringOperator::StartsWith) - } else { - Err(TuliproxError::FilterParse(format!("unknown string operator: {text}"))) - } -} - -fn get_parser_numeric_operator(expr: &Pair) -> Result { - match expr.as_str() { - "=" => Ok(NumericOperator::Eq), - "!=" => Ok(NumericOperator::NotEq), - ">" => Ok(NumericOperator::Greater), - ">=" => Ok(NumericOperator::GreaterOrEqual), - "<" => Ok(NumericOperator::Less), - "<=" => Ok(NumericOperator::LessOrEqual), - other => Err(TuliproxError::FilterParse(format!("unknown numeric operator: {other}"))), - } -} - -fn get_parser_string_comparison( - expr: Pair, - templates: Option<&[PatternTemplate]>, -) -> Result { - let mut expr_inner = expr.into_inner(); - let field = get_parser_item_field(&expr_inner.next().unwrap())?; - let op = get_parser_string_operator(&expr_inner.next().unwrap())?; - let value = get_parser_string_value(&expr_inner.next().unwrap(), templates)?; - Ok(Filter::StringComparison(field, op, value)) -} - -fn get_parser_presence_comparison(expr: Pair) -> Result { - let mut expr_inner = expr.into_inner(); - let field = get_parser_item_field( - &expr_inner - .next() - .ok_or_else(|| TuliproxError::FilterParse("presence comparison is missing a field".to_string()))?, - )?; - let operator = expr_inner - .next() - .ok_or_else(|| TuliproxError::FilterParse("presence comparison is missing an operator".to_string()))?; - let op = if operator.as_str() == "!=" || operator.as_str().split_whitespace().count() > 1 { - PresenceOperator::IsNotEmpty - } else { - PresenceOperator::IsEmpty - }; - Ok(Filter::PresenceComparison(field, op)) -} - -fn get_parser_numeric_comparison(expr: Pair) -> Result { - let mut expr_inner = expr.into_inner(); - let field_pair = expr_inner.next().unwrap(); - let field_text = field_pair.as_str(); - let field = if field_text.eq_ignore_ascii_case("chno") { - ItemField::Chno - } else if field_text.eq_ignore_ascii_case("quality") { - ItemField::Quality - } else { - return Err(TuliproxError::FilterParse(format!("unknown numeric field: {field_text}"))); - }; - let op = get_parser_numeric_operator(&expr_inner.next().unwrap())?; - let value_pair = expr_inner.next().unwrap(); - let value = value_pair - .as_str() - .parse::() - .map_err(|_| TuliproxError::FilterParse(format!("invalid number: {}", value_pair.as_str())))?; - Ok(Filter::NumericComparison(field, op, value)) -} - -fn get_parser_set_comparison(expr: Pair, templates: Option<&[PatternTemplate]>) -> Result { - let mut expr_inner = expr.into_inner(); - let field = get_parser_item_field(&expr_inner.next().unwrap())?; - let values_pair = expr_inner.next().unwrap(); - let mut values = Vec::new(); - for value in values_pair.into_inner() { - values.push(get_parser_string_value(&value, templates)?); - } - if values.is_empty() { - return Err(TuliproxError::FilterParse("empty value list in IN comparison".to_string())); - } - Ok(Filter::SetComparison(field, values)) -} - -fn get_filter_item_type(text_item_type: &str) -> Option { - if text_item_type.eq_ignore_ascii_case("live") { - Some(PlaylistItemType::Live) - } else if text_item_type.eq_ignore_ascii_case(Filter::VOD) - || text_item_type.eq_ignore_ascii_case("video") - || text_item_type.eq_ignore_ascii_case(Filter::MOVIE) - { - Some(PlaylistItemType::Video) - } else if text_item_type.eq_ignore_ascii_case("series") { - Some(PlaylistItemType::Series) - } else if text_item_type.eq_ignore_ascii_case("series-info") { - // this is necessarry to avoid series and series-info confusion in filter! - // we can now use series for filtering series and series-info (series-info are categories) - Some(PlaylistItemType::Series) - } else { - None - } -} - -fn get_parser_type_comparison(expr: Pair) -> Result { - let expr_inner = expr.into_inner(); - let text_item_type = expr_inner.as_str(); - let item_type = get_filter_item_type(text_item_type); - item_type.map_or_else( - || Err(TuliproxError::FilterParse(format!("can't parse item type: {text_item_type}"))), - |itype| Ok(Filter::TypeComparison(ItemField::Type, itype)), - ) -} macro_rules! handle_expr { ($bop: expr, $uop: expr, $stmts: expr, $exp: expr) => {{ @@ -577,817 +22,15 @@ macro_rules! handle_expr { }}; } -fn get_parser_expression( - expr: Pair, - templates: Option<&[PatternTemplate]>, - errors: &mut Vec, -) -> Result { - let mut stmts = Vec::new(); - let pairs = expr.into_inner(); - let mut bop: Option = None; - let mut uop: Option = None; - - for pair in pairs { - match pair.as_rule() { - Rule::field_comparison => { - let comp_res = get_parser_field_comparison(pair, templates); - match comp_res { - Ok(comp) => handle_expr!(bop, uop, stmts, comp), - Err(err) => errors.push(err.to_string()), - } - } - Rule::type_comparison => { - let comp_res = get_parser_type_comparison(pair); - match comp_res { - Ok(comp) => handle_expr!(bop, uop, stmts, comp), - Err(err) => errors.push(err.to_string()), - } - } - Rule::string_comparison => match get_parser_string_comparison(pair, templates) { - Ok(comp) => handle_expr!(bop, uop, stmts, comp), - Err(err) => errors.push(err.to_string()), - }, - Rule::presence_comparison => match get_parser_presence_comparison(pair) { - Ok(comp) => handle_expr!(bop, uop, stmts, comp), - Err(err) => errors.push(err.to_string()), - }, - Rule::numeric_comparison => match get_parser_numeric_comparison(pair) { - Ok(comp) => handle_expr!(bop, uop, stmts, comp), - Err(err) => errors.push(err.to_string()), - }, - Rule::set_comparison => match get_parser_set_comparison(pair, templates) { - Ok(comp) => handle_expr!(bop, uop, stmts, comp), - Err(err) => errors.push(err.to_string()), - }, - Rule::comparison | Rule::expr => { - let expr = get_parser_expression(pair, templates, errors)?; - handle_expr!(bop, uop, stmts, expr); - } - Rule::expr_group => { - let expr = get_parser_expression(pair.into_inner().next().unwrap(), templates, errors)?; - handle_expr!(bop, uop, stmts, Filter::Group(Box::new(expr))); - } - Rule::not => { - uop = Some(UnaryOperator::Not); - } - Rule::bool_op => match get_parser_binary_op(&pair.into_inner().next().unwrap()) { - Ok(binop) => { - bop = Some(binop); - } - Err(err) => { - errors.push(format!("{err}")); - } - }, - _ => { - errors.push(format!("did not expect rule: {pair:?}")); - } - } - } - if stmts.is_empty() { - return Err(format!("Invalid Filter, could not parse {errors:?}")); - } - if stmts.len() > 1 { - return Err(format!("did not expect multiple rule: {stmts:?}, {errors:?}")); - } - - Ok(stmts.pop().unwrap()) -} - -fn get_parser_binary_op(expr: &Pair) -> Result { - match expr.as_rule() { - Rule::and => Ok(BinaryOperator::And), - Rule::or => Ok(BinaryOperator::Or), - _ => Err(TuliproxError::FilterParse(format!("Unknown binary operator {}", expr.as_str()))), - } -} - -fn unresolved_template_placeholders(input: &str) -> Vec { - let mut placeholders = IndexSet::new(); - for captures in CONSTANTS.re_template_var.captures_iter(input) { - if let Some(inner) = captures.get(1) { - let value = inner.as_str(); - if !value.is_empty() { - placeholders.insert(format!("!{value}!")); - } - } - } - placeholders.into_iter().collect() -} - -/// 1-based line/column of a filter syntax error, when the parser can locate it. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct FilterParsePosition { - pub line: usize, - pub column: usize, -} - -/// Like [`get_filter`], but reports the syntax-error position when available -/// (semantic errors such as invalid regex values have no position). -pub fn get_filter_detailed( - filter_text: &str, - templates: Option<&[PatternTemplate]>, -) -> Result)> { - let source = match apply_templates_to_pattern_single(filter_text, templates) { - Ok(source) => source, - Err(err) => return Err((err, None)), - }; - let unresolved_placeholders = unresolved_template_placeholders(&source); - if !unresolved_placeholders.is_empty() { - return Err(( - TuliproxError::FilterParse(format!( - "Unknown template placeholder(s) in filter: {}", - unresolved_placeholders.join(", ") - )), - None, - )); - } - - match FilterParser::parse(Rule::main, &source) { - Ok(pairs) => { - let mut errors = Vec::new(); - let mut result: Option = None; - let mut op: Option = None; - for pair in pairs { - match pair.as_rule() { - Rule::stmt => { - for expr in pair.into_inner() { - match expr.as_rule() { - Rule::expr => match get_parser_expression(expr, templates, &mut errors) { - Ok(expr) => match &op { - Some(binop) => { - result = Some(Filter::BinaryExpression( - Box::new(result.unwrap()), - *binop, - Box::new(expr), - )); - op = None; - } - _ => result = Some(expr), - }, - Err(err) => errors.push(err), - }, - Rule::bool_op => match get_parser_binary_op(&expr.into_inner().next().unwrap()) { - Ok(binop) => { - op = Some(binop); - } - Err(err) => { - errors.push(err.to_string()); - } - }, - _ => { - errors.push(format!("unknown expression {expr:?}")); - } - } - } - } - Rule::EOI => {} - _ => { - errors.push(format!("unknown: {}", pair.as_str())); - } - } - } - - if !errors.is_empty() { - errors.push(format!("Unable to parse filter: {filter_text}")); - return Err((TuliproxError::FilterParse(errors.join("\n")), None)); - } - - result.map_or_else( - || Err((TuliproxError::FilterParse(format!("Unable to parse filter: {filter_text}")), None)), - Ok, - ) - } - Err(err) => { - let (line, column) = match err.line_col { - pest::error::LineColLocation::Pos((line, column)) - | pest::error::LineColLocation::Span((line, column), _) => (line, column), - }; - Err((TuliproxError::FilterParse(format!("{err}")), Some(FilterParsePosition { line, column }))) - } - } -} - -pub fn get_filter(filter_text: &str, templates: Option<&[PatternTemplate]>) -> Result { - get_filter_detailed(filter_text, templates).map_err(|(err, _)| err) -} - -fn build_dependency_graph(templates: &Vec) -> Result, TuliproxError> { - let mut graph = DirectedGraph::::new(); - for template in templates { - graph.add_node(&template.name); - let mut handle_template_value = |value| { - CONSTANTS - .re_template_var - .captures_iter(value) - .filter(|caps| caps.len() > 1) - .filter_map(|caps| caps.get(1)) - .map(|caps| String::from(caps.as_str())) - .for_each(|e| { - graph.add_node(&e); - graph.add_edge(&template.name, &e); - }); - }; - match &template.value { - TemplateValue::Single(value) => handle_template_value(value), - TemplateValue::Multi(values) => values.iter().for_each(|value| handle_template_value(value)), - } - } - let cycles = graph.find_cycles(); - for cyclic in &cycles { - error!("Cyclic template dependencies detected [{}]", cyclic.join(" <-> ")); - } - if !cycles.is_empty() { - return Err(TuliproxError::FilterParse("Cyclic dependencies in templates detected!".to_string())); - } - Ok(graph) -} - -fn apply_dependency_template_value( - template_value: TemplateValue, - placeholder: &str, - dependency_value: &TemplateValue, -) -> TemplateValue { - match dependency_value { - TemplateValue::Single(dep_val) => match template_value { - TemplateValue::Single(templ_val) => { - if templ_val.contains(placeholder) { - TemplateValue::Single(templ_val.replace(placeholder, dep_val.as_str())) - } else { - TemplateValue::Single(templ_val) - } - } - TemplateValue::Multi(templ_vals) => { - let mut new_values = IndexSet::new(); - for templ_val in templ_vals { - if templ_val.contains(placeholder) { - new_values.insert(templ_val.replace(placeholder, dep_val.as_str())); - } else { - new_values.insert(templ_val); - } - } - TemplateValue::Multi(new_values.into_iter().collect()) - } - }, - TemplateValue::Multi(dep_vals) => match template_value { - TemplateValue::Single(templ_val) => { - if templ_val.contains(placeholder) { - let mut new_values = IndexSet::new(); - for dep_val in dep_vals { - new_values.insert(templ_val.replace(placeholder, dep_val.as_str())); - } - TemplateValue::Multi(new_values.into_iter().collect()) - } else { - TemplateValue::Single(templ_val) - } - } - TemplateValue::Multi(templ_vals) => { - let mut new_values = IndexSet::new(); - for templ_val in templ_vals { - if templ_val.contains(placeholder) { - for dep_val in dep_vals { - new_values.insert(templ_val.replace(placeholder, dep_val.as_str())); - } - } else { - new_values.insert(templ_val); - } - } - TemplateValue::Multi(new_values.into_iter().collect()) - } - }, - } -} - -pub fn prepare_templates(templates: &mut Vec) -> Result, TuliproxError> { - let mut seen_template_names: HashSet<&str> = HashSet::with_capacity(templates.len()); - for template in templates.iter() { - if !seen_template_names.insert(template.name.as_str()) { - return Err(TuliproxError::FilterParse(format!("Duplicate template name found: {}", template.name))); - } - } - - let graph = build_dependency_graph(templates)?; - let mut template_values = HashMap::new(); - let mut template_map = HashMap::with_capacity(templates.len()); - - for item in templates.iter_mut() { - item.prepare(); - template_values.insert(item.name.clone(), item.value.clone()); - template_map.insert(item.name.clone(), item); - } - - if let Some(dependencies) = graph.get_dependencies() { - if let Some(sorted) = graph.topological_sort() { - for template_name in sorted { - if let Some(depends_on) = dependencies.get(&template_name) { - let mut templ_value = template_values.get(&template_name).unwrap().clone(); - for dep_templ_name in depends_on { - let dep_value = template_values.get(dep_templ_name).ok_or_else(|| { - TuliproxError::FilterParse(format!("Failed to load template {dep_templ_name}")) - })?; - let dep_templ = template_map.get_mut(dep_templ_name).unwrap(); - templ_value = apply_dependency_template_value(templ_value, &dep_templ.placeholder, dep_value); - } - template_values.insert(template_name.clone(), templ_value); - } - } - - for (k, v) in template_values { - let template = template_map.get_mut(&k).unwrap(); - template.value = v; - } - } - } - let result: Vec = template_map.iter_mut().map(|(_, t)| t.clone()).collect(); - Ok(result) -} - -pub fn apply_templates_to_pattern( - pattern: &str, - templates_list: Option<&[PatternTemplate]>, - allow_multi: bool, -) -> Result { - let mut new_pattern = TemplateValue::Single(pattern.to_string()); - - if let Some(templates) = templates_list { - for template in templates { - new_pattern = apply_dependency_template_value(new_pattern, &template.placeholder, &template.value); - } - } - - if !allow_multi { - match &new_pattern { - TemplateValue::Single(_) => {} - TemplateValue::Multi(multi_vals) => match multi_vals.len().cmp(&1) { - Ordering::Less => { - return Err(TuliproxError::FilterParse(format!( - "Empty multi value templates are not supported for pattern! {pattern}" - ))); - } - Ordering::Equal => { - new_pattern = TemplateValue::Single(multi_vals.first().unwrap().to_owned()); - } - Ordering::Greater => { - return Err(TuliproxError::FilterParse(format!( - "Multi value templates are not supported for pattern! {pattern}" - ))); - } - }, - } - } - - Ok(new_pattern) -} - -pub fn apply_templates_to_pattern_single( - pattern: &str, - templates: Option<&[PatternTemplate]>, -) -> Result { - match apply_templates_to_pattern(pattern, templates, false)? { - TemplateValue::Single(value) => Ok(value), - TemplateValue::Multi(_) => { - Err(TuliproxError::FilterParse("Multi value templates are not supported for pattern!".to_string())) - } - } -} - #[cfg(test)] -mod tests { - use crate::{ - foundation::{ - filter::{get_filter, ValueProvider}, - prepare_templates, - }, - model::{ItemField, PatternTemplate, PlaylistItem, PlaylistItemHeader, PlaylistItemType, TemplateValue}, - utils::{Internable, CONSTANTS}, - }; - use std::borrow::Cow; +mod tests; - fn create_mock_pli(name: &str, group: &str) -> PlaylistItem { - PlaylistItem { header: PlaylistItemHeader { name: name.into(), group: group.intern(), ..Default::default() } } - } - - /// Parse `input` and assert that `Display` round-trips back to `input` - /// byte-for-byte. Used by the three filter-parser tests below that only - /// exercise the parser and its `Display` impl, not the runtime evaluator. - fn assert_filter_round_trip(input: &str) { - match get_filter(input, None) { - Ok(filter) => assert_eq!(format!("{filter}"), input), - Err(e) => panic!("{e}"), - } - } - - #[test] - fn test_filter_1() { - assert_filter_round_trip( - r#"(Group ~ "A" OR Group ~ "B") AND (Name ~ "C" OR Name ~ "D" OR Name ~ "E") OR (NOT (Title ~ "F") AND NOT Title ~ "K")"#, - ); - } - - #[test] - fn test_filter_2() { - assert_filter_round_trip( - r#"Group ~ "d" AND ((Name ~ "e" AND NOT ((Name ~ "c" OR Name ~ "f"))) OR (Name ~ "a" OR Name ~ "b"))"#, - ); - } - - #[test] - fn test_filter_3() { - assert_filter_round_trip( - r#"Group ~ "d" AND ((Name ~ "e" AND NOT ((Name ~ "c" OR Name ~ "f"))) OR (Name ~ "a" OR Name ~ "b")) AND (Type = movie)"#, - ); - } - - #[test] - fn test_filter_string_ops_round_trip() { - assert_filter_round_trip( - r#"Group = "Sports" AND Title != "News" OR Name CONTAINS "HD" AND Caption STARTSWITH "DE:""#, - ); - } - - #[test] - fn test_filter_numeric_round_trip() { assert_filter_round_trip(r"Chno >= 100 AND Chno < 200 OR NOT Chno = 7"); } - - #[test] - fn test_filter_set_round_trip() { - assert_filter_round_trip(r#"Group IN ["Sports", "News"] AND NOT (Name IN ["A"])"#); - } - - #[test] - fn test_filter_presence_round_trip_and_aliases() { - assert_filter_round_trip("EpgId IS EMPTY"); - assert_filter_round_trip("EpgId IS NOT EMPTY"); - - assert_eq!(get_filter("EpgId = EMPTY", None).expect("alias parses").to_string(), "EpgId IS EMPTY"); - assert_eq!(get_filter("EpgId != EMPTY", None).expect("alias parses").to_string(), "EpgId IS NOT EMPTY"); - } - - #[test] - fn test_filter_presence_eval() { - let empty = get_filter("EpgId IS EMPTY", None).expect("empty filter parses"); - let populated = get_filter("EpgId IS NOT EMPTY", None).expect("populated filter parses"); - let mut item = create_mock_pli("Channel", "Group"); - let provider = ValueProvider { pli: &item, match_as_ascii: false }; - assert!(empty.filter(&provider)); - assert!(!populated.filter(&provider)); - - item.header.epg_channel_id = Some("channel.epg".intern()); - let provider = ValueProvider { pli: &item, match_as_ascii: false }; - assert!(!empty.filter(&provider)); - assert!(populated.filter(&provider)); - - item.header.epg_channel_id = Some("".intern()); - let provider = ValueProvider { pli: &item, match_as_ascii: false }; - assert!(empty.filter(&provider)); - assert!(!populated.filter(&provider)); - } - - #[test] - fn test_filter_string_ops_eval() { - let flt = r#"Group = "sports" AND Name CONTAINS "hd" AND NOT (Name STARTSWITH "x")"#; - let filter = get_filter(flt, None).expect("filter parses"); - let matching = create_mock_pli("Channel HD", "Sports"); - assert!(filter.filter(&ValueProvider { pli: &matching, match_as_ascii: false })); - let other = create_mock_pli("Channel", "Sports"); - assert!(!filter.filter(&ValueProvider { pli: &other, match_as_ascii: false })); - } - - #[test] - fn test_filter_set_eval() { - let filter = get_filter(r#"Group IN ["Sports", "News"]"#, None).expect("filter parses"); - let matching = create_mock_pli("A", "news"); - assert!(filter.filter(&ValueProvider { pli: &matching, match_as_ascii: false })); - let other = create_mock_pli("B", "Movies"); - assert!(!filter.filter(&ValueProvider { pli: &other, match_as_ascii: false })); - } - - #[test] - fn test_filter_caption_string_ops_fall_back_to_name() { - // title and name differ; only the name matches - let mut pli = create_mock_pli("Channel HD", "Sports"); - pli.header.title = "Some Title".into(); - let provider = ValueProvider { pli: &pli, match_as_ascii: false }; - - let contains = get_filter(r#"Caption CONTAINS "hd""#, None).expect("filter parses"); - assert!(contains.filter(&provider)); - - let starts = get_filter(r#"Caption STARTSWITH "channel""#, None).expect("filter parses"); - assert!(starts.filter(&provider)); - - let eq = get_filter(r#"Caption = "channel hd""#, None).expect("filter parses"); - assert!(eq.filter(&provider)); - // NotEq is the negation of Eq-on-either - let not_eq = get_filter(r#"Caption != "channel hd""#, None).expect("filter parses"); - assert!(!not_eq.filter(&provider)); - - let set = get_filter(r#"Caption IN ["Channel HD"]"#, None).expect("filter parses"); - assert!(set.filter(&provider)); - - let no_match = get_filter(r#"Caption CONTAINS "missing""#, None).expect("filter parses"); - assert!(!no_match.filter(&provider)); - } - - #[test] - fn test_filter_chno_eval() { - let filter = get_filter("Chno >= 10 AND Chno <= 20", None).expect("filter parses"); - let mut pli = create_mock_pli("A", "G"); - pli.header.chno = 15; - assert!(filter.filter(&ValueProvider { pli: &pli, match_as_ascii: false })); - pli.header.chno = 5; - assert!(!filter.filter(&ValueProvider { pli: &pli, match_as_ascii: false })); - } - - #[test] - fn test_filter_quality_round_trip_and_eval() { - assert_filter_round_trip(r"Quality >= 3 AND NOT Quality = 5"); - - let filter = get_filter("Quality >= 3", None).expect("filter parses"); - let mut fhd = create_mock_pli("X", "G"); - fhd.header.title = "News FHD".into(); - assert!(filter.filter(&ValueProvider { pli: &fhd, match_as_ascii: false })); - let mut hd = create_mock_pli("X", "G"); - hd.header.title = "News HD".into(); - assert!(!filter.filter(&ValueProvider { pli: &hd, match_as_ascii: false })); - let unknown = create_mock_pli("News", "G"); - assert!(!filter.filter(&ValueProvider { pli: &unknown, match_as_ascii: false })); - } - - #[test] - fn test_filter_4() { - let flt = r#"NOT (Name ~ ".*24/7.*" AND Group ~ "^US.*")"#; - match get_filter(flt, None) { - Ok(filter) => { - assert_eq!(format!("{filter}"), flt); - let channels = [ - create_mock_pli("24/7: Cars", "FR Channels"), - create_mock_pli("24/7: Cars", "US Channels"), - create_mock_pli("Entertainment", "US Channels"), - ]; - let filtered: Vec<&PlaylistItem> = channels - .iter() - .filter(|&chan| { - let provider = ValueProvider { pli: chan, match_as_ascii: false }; - filter.filter(&provider) - }) - .collect(); - assert_eq!(filtered.len(), 2); - assert!(filtered.iter().any(|&chan| { - let group = chan.header.group.to_string(); - let name = chan.header.name.to_string(); - name.eq("24/7: Cars") && group.eq("FR Channels") - })); - assert!(filtered.iter().any(|&chan| { - let group = chan.header.group.to_string(); - let name = chan.header.name.to_string(); - name.eq("Entertainment") && group.eq("US Channels") - })); - assert!(!filtered.iter().any(|&chan| { - let group = chan.header.group.to_string(); - let name = chan.header.name.to_string(); - name.eq("24/7: Cars") && group.eq("US Channels") - })); - } - Err(e) => { - panic!("{e}") - } - } - } - - #[test] - fn test_filter_5() { - let flt = - r#"NOT (Name ~ "NC" OR Group ~ "GA") AND (Name ~ "NA" AND Group ~ "GA") OR (Name ~ "NB" AND Group ~ "GB")"#; - match get_filter(flt, None) { - Ok(filter) => { - assert_eq!(format!("{filter}"), flt); - let channels = [ - create_mock_pli("NA", "GA"), - create_mock_pli("NB", "GB"), - create_mock_pli("NA", "GB"), - create_mock_pli("NB", "GA"), - create_mock_pli("NC", "GA"), - create_mock_pli("NA", "GC"), - ]; - let filtered: Vec<&PlaylistItem> = channels - .iter() - .filter(|&chan| { - let provider = ValueProvider { pli: chan, match_as_ascii: false }; - filter.filter(&provider) - }) - .collect(); - assert_eq!(filtered.len(), 1); - } - Err(e) => { - panic!("{e}") - } - } - } - - #[test] - fn test_filter_6() { - let flt = r####" - Group ~ "^EU \| FRANCE.*" - OR Input ~ "hello" - OR Group ~ "^VOD \| FR.*" - OR Group ~ "\[FR\].*" - OR Group ~ "^SRS \| FR.*" - AND NOT (Group ~ ".* LQ.*" - OR Title ~ ".* LQ.*" - OR Group ~ ".* SD.*" - OR Title ~ ".* SD.*" - OR Group ~ ".* HD.*" - OR Title ~ ".* HD.*" - OR Group ~ "(?i).*sport.*" - OR Group ~ "(?i).*DAZN.*" - OR Group ~ "(?i).*EQUIPE.*" - OR Group ~ "DOM TOM.*" - OR Group ~ "(?i).*PLUTO.*" - OR Title ~ "(?i).*GOLD.*" - OR Title ~ "###.*")"####; - - match get_filter(flt, None) { - Ok(filter) => { - let result = CONSTANTS.re_whitespace.replace_all(flt, " "); - assert_eq!(format!("{filter}"), result.trim()); - } - Err(e) => { - panic!("{e}") - } - } - } - - #[test] - fn test_filter_7() { - let flt = r#"NOT (Name ~ ".*24/7.*")"#; - match get_filter(flt, None) { - Ok(filter) => { - assert_eq!(format!("{filter}"), flt); - let channels = [ - create_mock_pli("24/7: Cars", "FR Channels"), - create_mock_pli("24/7: Cars", "US Channels"), - create_mock_pli("Entertainment", "US Channels"), - ]; - let filtered: Vec<&PlaylistItem> = channels - .iter() - .filter(|&chan| { - let provider = ValueProvider { pli: chan, match_as_ascii: false }; - filter.filter(&provider) - }) - .collect(); - assert_eq!(filtered.len(), 1); - assert!(filtered.iter().any(|&chan| { - let group = chan.header.group.to_string(); - let name = chan.header.name.to_string(); - name.eq("Entertainment") && group.eq("US Channels") - })); - } - Err(e) => { - panic!("{e}") - } - } - } - - #[test] - fn test_filter_match_as_ascii() { - let flt = r#"Name ~ "Cinema""#; - match get_filter(flt, None) { - Ok(filter) => { - let chan = create_mock_pli("Cinéma", "Some Group"); - - // Without match_as_ascii (should fail) - let provider_fail = ValueProvider { pli: &chan, match_as_ascii: false }; - assert!(!filter.filter(&provider_fail)); - - // With match_as_ascii (should succeed) - let provider_success = ValueProvider { pli: &chan, match_as_ascii: true }; - assert!(filter.filter(&provider_success)); - } - Err(e) => panic!("{e}"), - } - } - - #[test] - fn filter_value_borrows_stored_field() { - let channel = create_mock_pli("Channel", "Group"); - let provider = ValueProvider { pli: &channel, match_as_ascii: false }; - - assert!(matches!(provider.get_filter_value(ItemField::Name), Some(Cow::Borrowed("Channel")))); - } - - #[test] - fn missing_genre_does_not_match_empty_pattern() { - let filter = get_filter(r#"Genre ~ "^$""#, None).expect("filter should parse"); - let channel = create_mock_pli("Channel", "Group"); - let provider = ValueProvider { pli: &channel, match_as_ascii: false }; - - assert!(!filter.filter(&provider)); - } - - #[test] - fn caption_checks_name_when_title_does_not_match() { - let filter = get_filter(r#"Caption ~ "Channel""#, None).expect("filter should parse"); - let mut channel = create_mock_pli("Channel", "Group"); - channel.header.title = "Different title".intern(); - let provider = ValueProvider { pli: &channel, match_as_ascii: false }; - - assert!(filter.filter(&provider)); - } - - #[test] - fn type_filter_matches_normalized_type_families() { - let live_filter = get_filter("Type = live", None).expect("live filter should parse"); - let series_filter = get_filter("Type = series", None).expect("series filter should parse"); - let mut channel = create_mock_pli("Channel", "Group"); - - channel.header.item_type = PlaylistItemType::LiveHls; - assert!(live_filter.filter(&ValueProvider { pli: &channel, match_as_ascii: false })); - - channel.header.item_type = PlaylistItemType::LocalSeriesInfo; - assert!(series_filter.filter(&ValueProvider { pli: &channel, match_as_ascii: false })); - } - - #[test] - fn test_filter_unknown_template_placeholder_reports_error() { - let err = get_filter("!UNKNOWN_FILTER!", None).expect_err("unknown placeholder should fail"); - let msg = err.to_string(); - assert!(msg.contains("Unknown template placeholder(s) in filter")); - assert!(msg.contains("!UNKNOWN_FILTER!")); - } - - #[test] - fn prepare_templates_does_not_duplicate_unrelated_multi_entries() { - let mut templates = vec![ - PatternTemplate { - name: "A".to_string(), - value: TemplateValue::Multi(vec!["a1".to_string(), "a2".to_string()]), - placeholder: String::new(), - }, - PatternTemplate { - name: "SEQ".to_string(), - value: TemplateValue::Multi(vec!["!A!".to_string(), "literal".to_string()]), - placeholder: String::new(), - }, - ]; - - let prepared = prepare_templates(&mut templates).expect("templates should prepare"); - let seq = prepared.iter().find(|template| template.name == "SEQ").expect("SEQ template"); - - assert_eq!(seq.value, TemplateValue::Multi(vec!["a1".to_string(), "a2".to_string(), "literal".to_string()])); - } - - #[test] - fn prepare_templates_concatenates_sequence_templates_without_cartesian_blowup() { - let mut templates = vec![ - PatternTemplate { - name: "UK".to_string(), - value: TemplateValue::Multi(vec!["uk1".to_string(), "uk2".to_string()]), - placeholder: String::new(), - }, - PatternTemplate { - name: "US".to_string(), - value: TemplateValue::Multi(vec!["us1".to_string(), "us2".to_string(), "us3".to_string()]), - placeholder: String::new(), - }, - PatternTemplate { - name: "SEQ".to_string(), - value: TemplateValue::Multi(vec!["!UK!".to_string(), "!US!".to_string(), "adult".to_string()]), - placeholder: String::new(), - }, - ]; - - let prepared = prepare_templates(&mut templates).expect("templates should prepare"); - let seq = prepared.iter().find(|template| template.name == "SEQ").expect("SEQ template"); - - assert_eq!( - seq.value, - TemplateValue::Multi(vec![ - "uk1".to_string(), - "uk2".to_string(), - "us1".to_string(), - "us2".to_string(), - "us3".to_string(), - "adult".to_string(), - ]) - ); - } - - #[test] - fn prepare_templates_promotes_single_template_to_multi_for_multi_dependency() { - let mut templates = vec![ - PatternTemplate { - name: "BASE".to_string(), - value: TemplateValue::Multi(vec!["a".to_string(), "b".to_string()]), - placeholder: String::new(), - }, - PatternTemplate { - name: "URL".to_string(), - value: TemplateValue::Single("!BASE!/stream".to_string()), - placeholder: String::new(), - }, - ]; - - let prepared = prepare_templates(&mut templates).expect("templates should prepare"); - let url = prepared.iter().find(|template| template.name == "URL").expect("URL template"); - - assert_eq!(url.value, TemplateValue::Multi(vec!["a/stream".to_string(), "b/stream".to_string()])); - } -} +mod eval; +mod expression; +mod parser; +mod templates; +pub use expression::{ + BinaryOperator, CompiledRegex, Filter, NumericOperator, PresenceOperator, StringOperator, UnaryOperator, +}; +pub use parser::{get_filter, get_filter_detailed, FilterParsePosition}; +pub use templates::{apply_templates_to_pattern, apply_templates_to_pattern_single, prepare_templates}; diff --git a/shared/src/foundation/filter/eval.rs b/shared/src/foundation/filter/eval.rs new file mode 100644 index 000000000..221589a1e --- /dev/null +++ b/shared/src/foundation/filter/eval.rs @@ -0,0 +1,160 @@ +use super::{ + BinaryOperator, CompiledRegex, Filter, ItemField, NumericOperator, PlaylistItemType, PresenceOperator, + StringOperator, UnaryOperator, +}; +use crate::foundation::value_provider::ValueProvider; +use log::{log_enabled, trace, Level}; +use std::borrow::Cow; + +fn get_caption<'a>(provider: &'a ValueProvider<'_>, rewc: &CompiledRegex) -> (bool, Cow<'a, str>) { + if let Some(value) = provider.get_filter_value(ItemField::Title) { + if rewc.re.is_match(&value) { + return (true, value); + } + } + + if let Some(value) = provider.get_filter_value(ItemField::Name) { + if rewc.re.is_match(&value) { + return (true, value); + } + } + (false, Cow::Borrowed("")) +} + +fn contains_ignore_ascii_case(haystack: &str, needle: &str) -> bool { + if needle.is_empty() { + return true; + } + if needle.len() > haystack.len() { + return false; + } + haystack.as_bytes().windows(needle.len()).any(|window| window.eq_ignore_ascii_case(needle.as_bytes())) +} + +fn starts_with_ignore_ascii_case(haystack: &str, needle: &str) -> bool { + haystack.len() >= needle.len() && haystack.as_bytes()[..needle.len()].eq_ignore_ascii_case(needle.as_bytes()) +} + +fn string_operator_matches(actual: &str, op: StringOperator, value: &str) -> bool { + match op { + StringOperator::Eq => actual.eq_ignore_ascii_case(value), + StringOperator::NotEq => !actual.eq_ignore_ascii_case(value), + StringOperator::Contains => contains_ignore_ascii_case(actual, value), + StringOperator::StartsWith => starts_with_ignore_ascii_case(actual, value), + } +} + +/// Caption matches against the title and falls back to the name, mirroring `get_caption`. +fn string_caption_matches(provider: &ValueProvider<'_>, op: StringOperator, value: &str) -> bool { + let title = provider.get_filter_value(ItemField::Title).unwrap_or(Cow::Borrowed("")); + let name = provider.get_filter_value(ItemField::Name).unwrap_or(Cow::Borrowed("")); + match op { + // NotEq is the negation of Eq-on-either, not "either differs" + StringOperator::NotEq => !(title.eq_ignore_ascii_case(value) || name.eq_ignore_ascii_case(value)), + StringOperator::Eq | StringOperator::Contains | StringOperator::StartsWith => { + string_operator_matches(&title, op, value) || string_operator_matches(&name, op, value) + } + } +} + +impl Filter { + pub fn filter(&self, provider: &ValueProvider) -> bool { + match self { + Self::FieldComparison(field, rewc) => { + let (is_match, value) = if field == &ItemField::Caption { + get_caption(provider, rewc) + } else if let Some(value) = provider.get_filter_value(*field) { + (rewc.re.is_match(&value), value) + } else { + (false, Cow::Borrowed("")) + }; + if log_enabled!(Level::Trace) { + if is_match { + trace!("Match found: {rewc:?} {} => {field}='{value}'", rewc.restr); + } else { + trace!("Match failed: {self}: {rewc:?} {} => {field}='{value}'", rewc.restr); + } + } + is_match + } + Self::TypeComparison(field, item_type) => { + let actual = provider.pli.header.item_type; + let is_match = match item_type { + PlaylistItemType::Live => actual.is_live(), + PlaylistItemType::Video => actual.is_video(), + PlaylistItemType::Series => { + actual.is_series() + || matches!(actual, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo) + } + _ => actual == *item_type, + }; + if log_enabled!(Level::Trace) { + if is_match { + trace!("Match found: {field:?} {}", actual.as_str()); + } else { + trace!("Match failed: {self}: {field:?} {}", actual.as_str()); + } + } + is_match + } + Self::StringComparison(field, op, value) => { + // Missing values (e.g. absent epgid) are matched as empty strings. + let is_match = if *field == ItemField::Caption { + string_caption_matches(provider, *op, value) + } else { + let actual = provider.get_filter_value(*field).unwrap_or(Cow::Borrowed("")); + string_operator_matches(&actual, *op, value) + }; + if log_enabled!(Level::Trace) { + let actual = provider.get_filter_value(*field).unwrap_or(Cow::Borrowed("")); + if is_match { + trace!("Match found: {field} {op} \"{value}\" => '{actual}'"); + } else { + trace!("Match failed: {field} {op} \"{value}\" => '{actual}'"); + } + } + is_match + } + Self::PresenceComparison(field, op) => { + let is_empty = provider.get_filter_value(*field).is_none_or(|value| value.is_empty()); + match op { + PresenceOperator::IsEmpty => is_empty, + PresenceOperator::IsNotEmpty => !is_empty, + } + } + Self::NumericComparison(field, op, value) => { + let actual = match field { + ItemField::Chno => provider.pli.header.chno, + ItemField::Quality => u32::from(provider.quality_rank()), + _ => return false, + }; + match op { + NumericOperator::Eq => actual == *value, + NumericOperator::NotEq => actual != *value, + NumericOperator::Greater => actual > *value, + NumericOperator::GreaterOrEqual => actual >= *value, + NumericOperator::Less => actual < *value, + NumericOperator::LessOrEqual => actual <= *value, + } + } + Self::SetComparison(field, values) => { + if *field == ItemField::Caption { + let title = provider.get_filter_value(ItemField::Title).unwrap_or(Cow::Borrowed("")); + let name = provider.get_filter_value(ItemField::Name).unwrap_or(Cow::Borrowed("")); + values.iter().any(|value| title.eq_ignore_ascii_case(value) || name.eq_ignore_ascii_case(value)) + } else { + let actual = provider.get_filter_value(*field).unwrap_or(Cow::Borrowed("")); + values.iter().any(|value| actual.eq_ignore_ascii_case(value)) + } + } + Self::Group(expr) => expr.filter(provider), + Self::UnaryExpression(op, expr) => match op { + UnaryOperator::Not => !expr.filter(provider), + }, + Self::BinaryExpression(left, op, right) => match op { + BinaryOperator::And => left.filter(provider) && right.filter(provider), + BinaryOperator::Or => left.filter(provider) || right.filter(provider), + }, + } + } +} diff --git a/shared/src/foundation/filter/expression.rs b/shared/src/foundation/filter/expression.rs new file mode 100644 index 000000000..6867ec98d --- /dev/null +++ b/shared/src/foundation/filter/expression.rs @@ -0,0 +1,178 @@ +use super::{ItemField, PlaylistItemType}; +use std::sync::Arc; + +#[derive(Debug, Clone)] +pub struct CompiledRegex { + pub restr: String, + pub re: Arc, +} + +impl PartialEq for CompiledRegex { + fn eq(&self, other: &Self) -> bool { self.restr == other.restr } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum UnaryOperator { + Not, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum StringOperator { + Eq, + NotEq, + Contains, + StartsWith, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum PresenceOperator { + IsEmpty, + IsNotEmpty, +} + +impl std::fmt::Display for PresenceOperator { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str(match self { + Self::IsEmpty => "IS EMPTY", + Self::IsNotEmpty => "IS NOT EMPTY", + }) + } +} + +impl std::fmt::Display for StringOperator { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str(match *self { + Self::Eq => "=", + Self::NotEq => "!=", + Self::Contains => "CONTAINS", + Self::StartsWith => "STARTSWITH", + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum NumericOperator { + Eq, + NotEq, + Greater, + GreaterOrEqual, + Less, + LessOrEqual, +} + +impl std::fmt::Display for NumericOperator { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str(match *self { + Self::Eq => "=", + Self::NotEq => "!=", + Self::Greater => ">", + Self::GreaterOrEqual => ">=", + Self::Less => "<", + Self::LessOrEqual => "<=", + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum BinaryOperator { + And, + Or, +} + +impl BinaryOperator { + pub(super) const OP_OR: &'static str = "OR"; + pub(super) const OP_AND: &'static str = "AND"; +} + +impl std::fmt::Display for BinaryOperator { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "{}", + match *self { + Self::Or => Self::OP_OR, + Self::And => Self::OP_AND, + } + ) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Filter { + Group(Box), + FieldComparison(ItemField, CompiledRegex), + TypeComparison(ItemField, PlaylistItemType), + StringComparison(ItemField, StringOperator, String), + PresenceComparison(ItemField, PresenceOperator), + NumericComparison(ItemField, NumericOperator, u32), + SetComparison(ItemField, Vec), + UnaryExpression(UnaryOperator, Box), + BinaryExpression(Box, BinaryOperator, Box), +} + +impl Default for Filter { + fn default() -> Self { + Self::Group(Box::new(Filter::FieldComparison( + ItemField::Group, + CompiledRegex { restr: ".*".to_string(), re: crate::model::REGEX_CACHE.get_or_compile(".*").unwrap() }, + ))) + } +} + +impl Filter { + pub(super) const LIVE: &'static str = "live"; + pub(super) const VOD: &'static str = "vod"; + pub(super) const MOVIE: &'static str = "movie"; + pub(super) const SERIES: &'static str = "series"; + pub(super) const UNSUPPORTED: &'static str = "unsupported"; +} + +impl std::fmt::Display for Filter { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::FieldComparison(field, rewc) => { + write!(f, "{} ~ \"{}\"", field, String::from(&rewc.restr)) + } + Self::TypeComparison(field, item_type) => { + write!( + f, + "{} = {}", + field, + match item_type { + PlaylistItemType::Live => Self::LIVE, + PlaylistItemType::Video | PlaylistItemType::LocalVideo => Self::MOVIE, + PlaylistItemType::Series + | PlaylistItemType::SeriesInfo + | PlaylistItemType::LocalSeries + | PlaylistItemType::LocalSeriesInfo => Self::SERIES, // yes series-info is handled as series in filter + _ => Self::UNSUPPORTED, + } + ) + } + Self::Group(stmt) => { + write!(f, "({stmt})") + } + Self::StringComparison(field, op, value) => { + write!(f, "{field} {op} \"{}\"", value.replace('"', "\\\"")) + } + Self::PresenceComparison(field, op) => write!(f, "{field} {op}"), + Self::NumericComparison(field, op, value) => { + write!(f, "{field} {op} {value}") + } + Self::SetComparison(field, values) => { + let rendered = + values.iter().map(|v| format!("\"{}\"", v.replace('"', "\\\""))).collect::>().join(", "); + write!(f, "{field} IN [{rendered}]") + } + Self::UnaryExpression(op, expr) => { + let flt = match op { + UnaryOperator::Not => format!("NOT {expr}"), + }; + write!(f, "{flt}") + } + Self::BinaryExpression(left, op, right) => { + write!(f, "{left} {op} {right}") + } + } + } +} diff --git a/shared/src/foundation/filter/parser.rs b/shared/src/foundation/filter/parser.rs new file mode 100644 index 000000000..892bcaa54 --- /dev/null +++ b/shared/src/foundation/filter/parser.rs @@ -0,0 +1,414 @@ +use super::{ + apply_templates_to_pattern_single, BinaryOperator, CompiledRegex, Filter, ItemField, NumericOperator, + PatternTemplate, PlaylistItemType, PresenceOperator, StringOperator, UnaryOperator, +}; +use crate::{error::TuliproxError, utils::CONSTANTS}; +use indexmap::IndexSet; +use log::{log_enabled, trace, Level}; +use pest::{iterators::Pair, Parser}; +use pest_derive::Parser; +use strum::IntoEnumIterator; + +#[derive(Parser)] +#[grammar_inline = r#" +WHITESPACE = _{ " " | "\t" | "\r" | "\n"} +field = { ^"group" | ^"title" | ^"name" | ^"genre" | ^"url" | ^"input" | ^"caption" | ^"epgid"} +numeric_field = { ^"chno" | ^"quality" } +and = { ^"and" } +or = { ^"or" } +not = { ^"not" } +regexp = @{ "\"" ~ ( "\\\"" | (!"\"" ~ ANY) )* ~ "\"" } +number = @{ ASCII_DIGIT+ } +type_value = { ^"live" | ^"vod" | ^"movie" | ^"series" } +type_comparison = { ^"type" ~ "=" ~ type_value } +field_comparison_value = _{ regexp } +field_comparison = { field ~ "~" ~ field_comparison_value } +str_op = { "!=" | "=" | ^"contains" | ^"startswith" } +string_comparison = { field ~ str_op ~ regexp } +presence_negation = { ^"not" } +presence_op = { (^"is" ~ presence_negation?) | "!=" | "=" } +presence_comparison = { field ~ presence_op ~ ^"empty" } +num_op = { ">=" | "<=" | "!=" | ">" | "<" | "=" } +numeric_comparison = { numeric_field ~ num_op ~ number } +set_values = { regexp ~ ("," ~ regexp)* } +set_comparison = { field ~ ^"in" ~ "[" ~ set_values ~ "]" } +comparison = { field_comparison | type_comparison | numeric_comparison | set_comparison | presence_comparison | string_comparison } +bool_op = { and | or } +expr_group = { "(" ~ expr ~ ")" } +basic_expr = _{ comparison | expr_group } +not_expr = _{ not ~ basic_expr } +expr = { + not_expr ~ (bool_op ~ expr)? + | basic_expr ~ (bool_op ~ expr)* +} +stmt = { expr ~ (bool_op ~ expr)* } +main = _{ SOI ~ stmt ~ EOI } +"#] +struct FilterParser; + +fn get_parser_item_field(expr: &Pair) -> Result { + if expr.as_rule() == Rule::field { + let field_text = expr.as_str(); + for item in ItemField::iter() { + if field_text.eq_ignore_ascii_case(item.as_ref()) { + return Ok(item); + } + } + } + Err(TuliproxError::FilterParse(format!("unknown field: {}", expr.as_str()))) +} + +fn get_parser_regexp(expr: &Pair, templates: Option<&[PatternTemplate]>) -> Result { + if expr.as_rule() == Rule::regexp { + let full_str = expr.as_str(); + let parsed_text = &full_str[1..full_str.len() - 1]; + let regstr = apply_templates_to_pattern_single(parsed_text, templates)?; + let re = crate::model::REGEX_CACHE.get_or_compile(regstr.as_str()); + if re.is_err() { + return Err(TuliproxError::RegexCompile(regstr)); + } + let regexp = re.unwrap(); + if log_enabled!(Level::Trace) { + trace!("Created regex: {regstr}"); + } + return Ok(CompiledRegex { restr: regstr, re: regexp }); + } + Err(TuliproxError::FilterParse(format!("unknown field: {}", expr.as_str()))) +} + +fn get_parser_field_comparison( + expr: Pair, + templates: Option<&[PatternTemplate]>, +) -> Result { + let mut expr_inner = expr.into_inner(); + match get_parser_item_field(&expr_inner.next().unwrap()) { + Ok(field) => match get_parser_regexp(&expr_inner.next().unwrap(), templates) { + Ok(regexp) => Ok(Filter::FieldComparison(field, regexp)), + Err(err) => Err(err), + }, + Err(err) => Err(err), + } +} + +fn get_parser_string_value(expr: &Pair, templates: Option<&[PatternTemplate]>) -> Result { + if expr.as_rule() == Rule::regexp { + let full_str = expr.as_str(); + let parsed_text = &full_str[1..full_str.len() - 1]; + let resolved = apply_templates_to_pattern_single(parsed_text, templates)?; + return Ok(resolved.replace("\\\"", "\"")); + } + Err(TuliproxError::FilterParse(format!("expected string literal: {}", expr.as_str()))) +} + +fn get_parser_string_operator(expr: &Pair) -> Result { + let text = expr.as_str(); + if text == "=" { + Ok(StringOperator::Eq) + } else if text == "!=" { + Ok(StringOperator::NotEq) + } else if text.eq_ignore_ascii_case("contains") { + Ok(StringOperator::Contains) + } else if text.eq_ignore_ascii_case("startswith") { + Ok(StringOperator::StartsWith) + } else { + Err(TuliproxError::FilterParse(format!("unknown string operator: {text}"))) + } +} + +fn get_parser_numeric_operator(expr: &Pair) -> Result { + match expr.as_str() { + "=" => Ok(NumericOperator::Eq), + "!=" => Ok(NumericOperator::NotEq), + ">" => Ok(NumericOperator::Greater), + ">=" => Ok(NumericOperator::GreaterOrEqual), + "<" => Ok(NumericOperator::Less), + "<=" => Ok(NumericOperator::LessOrEqual), + other => Err(TuliproxError::FilterParse(format!("unknown numeric operator: {other}"))), + } +} + +fn get_parser_string_comparison( + expr: Pair, + templates: Option<&[PatternTemplate]>, +) -> Result { + let mut expr_inner = expr.into_inner(); + let field = get_parser_item_field(&expr_inner.next().unwrap())?; + let op = get_parser_string_operator(&expr_inner.next().unwrap())?; + let value = get_parser_string_value(&expr_inner.next().unwrap(), templates)?; + Ok(Filter::StringComparison(field, op, value)) +} + +fn get_parser_presence_comparison(expr: Pair) -> Result { + let mut expr_inner = expr.into_inner(); + let field = get_parser_item_field( + &expr_inner + .next() + .ok_or_else(|| TuliproxError::FilterParse("presence comparison is missing a field".to_string()))?, + )?; + let operator = expr_inner + .next() + .ok_or_else(|| TuliproxError::FilterParse("presence comparison is missing an operator".to_string()))?; + let op = if operator.as_str() == "!=" || operator.as_str().split_whitespace().count() > 1 { + PresenceOperator::IsNotEmpty + } else { + PresenceOperator::IsEmpty + }; + Ok(Filter::PresenceComparison(field, op)) +} + +fn get_parser_numeric_comparison(expr: Pair) -> Result { + let mut expr_inner = expr.into_inner(); + let field_pair = expr_inner.next().unwrap(); + let field_text = field_pair.as_str(); + let field = if field_text.eq_ignore_ascii_case("chno") { + ItemField::Chno + } else if field_text.eq_ignore_ascii_case("quality") { + ItemField::Quality + } else { + return Err(TuliproxError::FilterParse(format!("unknown numeric field: {field_text}"))); + }; + let op = get_parser_numeric_operator(&expr_inner.next().unwrap())?; + let value_pair = expr_inner.next().unwrap(); + let value = value_pair + .as_str() + .parse::() + .map_err(|_| TuliproxError::FilterParse(format!("invalid number: {}", value_pair.as_str())))?; + Ok(Filter::NumericComparison(field, op, value)) +} + +fn get_parser_set_comparison(expr: Pair, templates: Option<&[PatternTemplate]>) -> Result { + let mut expr_inner = expr.into_inner(); + let field = get_parser_item_field(&expr_inner.next().unwrap())?; + let values_pair = expr_inner.next().unwrap(); + let mut values = Vec::new(); + for value in values_pair.into_inner() { + values.push(get_parser_string_value(&value, templates)?); + } + if values.is_empty() { + return Err(TuliproxError::FilterParse("empty value list in IN comparison".to_string())); + } + Ok(Filter::SetComparison(field, values)) +} + +fn get_filter_item_type(text_item_type: &str) -> Option { + if text_item_type.eq_ignore_ascii_case("live") { + Some(PlaylistItemType::Live) + } else if text_item_type.eq_ignore_ascii_case(Filter::VOD) + || text_item_type.eq_ignore_ascii_case("video") + || text_item_type.eq_ignore_ascii_case(Filter::MOVIE) + { + Some(PlaylistItemType::Video) + } else if text_item_type.eq_ignore_ascii_case("series") { + Some(PlaylistItemType::Series) + } else if text_item_type.eq_ignore_ascii_case("series-info") { + // this is necessarry to avoid series and series-info confusion in filter! + // we can now use series for filtering series and series-info (series-info are categories) + Some(PlaylistItemType::Series) + } else { + None + } +} + +fn get_parser_type_comparison(expr: Pair) -> Result { + let expr_inner = expr.into_inner(); + let text_item_type = expr_inner.as_str(); + let item_type = get_filter_item_type(text_item_type); + item_type.map_or_else( + || Err(TuliproxError::FilterParse(format!("can't parse item type: {text_item_type}"))), + |itype| Ok(Filter::TypeComparison(ItemField::Type, itype)), + ) +} + +fn get_parser_expression( + expr: Pair, + templates: Option<&[PatternTemplate]>, + errors: &mut Vec, +) -> Result { + let mut stmts = Vec::new(); + let pairs = expr.into_inner(); + let mut bop: Option = None; + let mut uop: Option = None; + + for pair in pairs { + match pair.as_rule() { + Rule::field_comparison => { + let comp_res = get_parser_field_comparison(pair, templates); + match comp_res { + Ok(comp) => handle_expr!(bop, uop, stmts, comp), + Err(err) => errors.push(err.to_string()), + } + } + Rule::type_comparison => { + let comp_res = get_parser_type_comparison(pair); + match comp_res { + Ok(comp) => handle_expr!(bop, uop, stmts, comp), + Err(err) => errors.push(err.to_string()), + } + } + Rule::string_comparison => match get_parser_string_comparison(pair, templates) { + Ok(comp) => handle_expr!(bop, uop, stmts, comp), + Err(err) => errors.push(err.to_string()), + }, + Rule::presence_comparison => match get_parser_presence_comparison(pair) { + Ok(comp) => handle_expr!(bop, uop, stmts, comp), + Err(err) => errors.push(err.to_string()), + }, + Rule::numeric_comparison => match get_parser_numeric_comparison(pair) { + Ok(comp) => handle_expr!(bop, uop, stmts, comp), + Err(err) => errors.push(err.to_string()), + }, + Rule::set_comparison => match get_parser_set_comparison(pair, templates) { + Ok(comp) => handle_expr!(bop, uop, stmts, comp), + Err(err) => errors.push(err.to_string()), + }, + Rule::comparison | Rule::expr => { + let expr = get_parser_expression(pair, templates, errors)?; + handle_expr!(bop, uop, stmts, expr); + } + Rule::expr_group => { + let expr = get_parser_expression(pair.into_inner().next().unwrap(), templates, errors)?; + handle_expr!(bop, uop, stmts, Filter::Group(Box::new(expr))); + } + Rule::not => { + uop = Some(UnaryOperator::Not); + } + Rule::bool_op => match get_parser_binary_op(&pair.into_inner().next().unwrap()) { + Ok(binop) => { + bop = Some(binop); + } + Err(err) => { + errors.push(format!("{err}")); + } + }, + _ => { + errors.push(format!("did not expect rule: {pair:?}")); + } + } + } + if stmts.is_empty() { + return Err(format!("Invalid Filter, could not parse {errors:?}")); + } + if stmts.len() > 1 { + return Err(format!("did not expect multiple rule: {stmts:?}, {errors:?}")); + } + + Ok(stmts.pop().unwrap()) +} + +fn get_parser_binary_op(expr: &Pair) -> Result { + match expr.as_rule() { + Rule::and => Ok(BinaryOperator::And), + Rule::or => Ok(BinaryOperator::Or), + _ => Err(TuliproxError::FilterParse(format!("Unknown binary operator {}", expr.as_str()))), + } +} + +fn unresolved_template_placeholders(input: &str) -> Vec { + let mut placeholders = IndexSet::new(); + for captures in CONSTANTS.re_template_var.captures_iter(input) { + if let Some(inner) = captures.get(1) { + let value = inner.as_str(); + if !value.is_empty() { + placeholders.insert(format!("!{value}!")); + } + } + } + placeholders.into_iter().collect() +} + +/// 1-based line/column of a filter syntax error, when the parser can locate it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FilterParsePosition { + pub line: usize, + pub column: usize, +} + +/// Like [`get_filter`], but reports the syntax-error position when available +/// (semantic errors such as invalid regex values have no position). +pub fn get_filter_detailed( + filter_text: &str, + templates: Option<&[PatternTemplate]>, +) -> Result)> { + let source = match apply_templates_to_pattern_single(filter_text, templates) { + Ok(source) => source, + Err(err) => return Err((err, None)), + }; + let unresolved_placeholders = unresolved_template_placeholders(&source); + if !unresolved_placeholders.is_empty() { + return Err(( + TuliproxError::FilterParse(format!( + "Unknown template placeholder(s) in filter: {}", + unresolved_placeholders.join(", ") + )), + None, + )); + } + + match FilterParser::parse(Rule::main, &source) { + Ok(pairs) => { + let mut errors = Vec::new(); + let mut result: Option = None; + let mut op: Option = None; + for pair in pairs { + match pair.as_rule() { + Rule::stmt => { + for expr in pair.into_inner() { + match expr.as_rule() { + Rule::expr => match get_parser_expression(expr, templates, &mut errors) { + Ok(expr) => match &op { + Some(binop) => { + result = Some(Filter::BinaryExpression( + Box::new(result.unwrap()), + *binop, + Box::new(expr), + )); + op = None; + } + _ => result = Some(expr), + }, + Err(err) => errors.push(err), + }, + Rule::bool_op => match get_parser_binary_op(&expr.into_inner().next().unwrap()) { + Ok(binop) => { + op = Some(binop); + } + Err(err) => { + errors.push(err.to_string()); + } + }, + _ => { + errors.push(format!("unknown expression {expr:?}")); + } + } + } + } + Rule::EOI => {} + _ => { + errors.push(format!("unknown: {}", pair.as_str())); + } + } + } + + if !errors.is_empty() { + errors.push(format!("Unable to parse filter: {filter_text}")); + return Err((TuliproxError::FilterParse(errors.join("\n")), None)); + } + + result.map_or_else( + || Err((TuliproxError::FilterParse(format!("Unable to parse filter: {filter_text}")), None)), + Ok, + ) + } + Err(err) => { + let (line, column) = match err.line_col { + pest::error::LineColLocation::Pos((line, column)) + | pest::error::LineColLocation::Span((line, column), _) => (line, column), + }; + Err((TuliproxError::FilterParse(format!("{err}")), Some(FilterParsePosition { line, column }))) + } + } +} + +pub fn get_filter(filter_text: &str, templates: Option<&[PatternTemplate]>) -> Result { + get_filter_detailed(filter_text, templates).map_err(|(err, _)| err) +} diff --git a/shared/src/foundation/filter/templates.rs b/shared/src/foundation/filter/templates.rs new file mode 100644 index 000000000..7d5b0d721 --- /dev/null +++ b/shared/src/foundation/filter/templates.rs @@ -0,0 +1,190 @@ +use super::{PatternTemplate, TemplateValue}; +use crate::{ + error::TuliproxError, + utils::{DirectedGraph, CONSTANTS}, +}; +use indexmap::IndexSet; +use log::error; +use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, +}; + +fn build_dependency_graph(templates: &Vec) -> Result, TuliproxError> { + let mut graph = DirectedGraph::::new(); + for template in templates { + graph.add_node(&template.name); + let mut handle_template_value = |value| { + CONSTANTS + .re_template_var + .captures_iter(value) + .filter(|caps| caps.len() > 1) + .filter_map(|caps| caps.get(1)) + .map(|caps| String::from(caps.as_str())) + .for_each(|e| { + graph.add_node(&e); + graph.add_edge(&template.name, &e); + }); + }; + match &template.value { + TemplateValue::Single(value) => handle_template_value(value), + TemplateValue::Multi(values) => values.iter().for_each(|value| handle_template_value(value)), + } + } + let cycles = graph.find_cycles(); + for cyclic in &cycles { + error!("Cyclic template dependencies detected [{}]", cyclic.join(" <-> ")); + } + if !cycles.is_empty() { + return Err(TuliproxError::FilterParse("Cyclic dependencies in templates detected!".to_string())); + } + Ok(graph) +} + +fn apply_dependency_template_value( + template_value: TemplateValue, + placeholder: &str, + dependency_value: &TemplateValue, +) -> TemplateValue { + match dependency_value { + TemplateValue::Single(dep_val) => match template_value { + TemplateValue::Single(templ_val) => { + if templ_val.contains(placeholder) { + TemplateValue::Single(templ_val.replace(placeholder, dep_val.as_str())) + } else { + TemplateValue::Single(templ_val) + } + } + TemplateValue::Multi(templ_vals) => { + let mut new_values = IndexSet::new(); + for templ_val in templ_vals { + if templ_val.contains(placeholder) { + new_values.insert(templ_val.replace(placeholder, dep_val.as_str())); + } else { + new_values.insert(templ_val); + } + } + TemplateValue::Multi(new_values.into_iter().collect()) + } + }, + TemplateValue::Multi(dep_vals) => match template_value { + TemplateValue::Single(templ_val) => { + if templ_val.contains(placeholder) { + let mut new_values = IndexSet::new(); + for dep_val in dep_vals { + new_values.insert(templ_val.replace(placeholder, dep_val.as_str())); + } + TemplateValue::Multi(new_values.into_iter().collect()) + } else { + TemplateValue::Single(templ_val) + } + } + TemplateValue::Multi(templ_vals) => { + let mut new_values = IndexSet::new(); + for templ_val in templ_vals { + if templ_val.contains(placeholder) { + for dep_val in dep_vals { + new_values.insert(templ_val.replace(placeholder, dep_val.as_str())); + } + } else { + new_values.insert(templ_val); + } + } + TemplateValue::Multi(new_values.into_iter().collect()) + } + }, + } +} + +pub fn prepare_templates(templates: &mut Vec) -> Result, TuliproxError> { + let mut seen_template_names: HashSet<&str> = HashSet::with_capacity(templates.len()); + for template in templates.iter() { + if !seen_template_names.insert(template.name.as_str()) { + return Err(TuliproxError::FilterParse(format!("Duplicate template name found: {}", template.name))); + } + } + + let graph = build_dependency_graph(templates)?; + let mut template_values = HashMap::new(); + let mut template_map = HashMap::with_capacity(templates.len()); + + for item in templates.iter_mut() { + item.prepare(); + template_values.insert(item.name.clone(), item.value.clone()); + template_map.insert(item.name.clone(), item); + } + + if let Some(dependencies) = graph.get_dependencies() { + if let Some(sorted) = graph.topological_sort() { + for template_name in sorted { + if let Some(depends_on) = dependencies.get(&template_name) { + let mut templ_value = template_values.get(&template_name).unwrap().clone(); + for dep_templ_name in depends_on { + let dep_value = template_values.get(dep_templ_name).ok_or_else(|| { + TuliproxError::FilterParse(format!("Failed to load template {dep_templ_name}")) + })?; + let dep_templ = template_map.get_mut(dep_templ_name).unwrap(); + templ_value = apply_dependency_template_value(templ_value, &dep_templ.placeholder, dep_value); + } + template_values.insert(template_name.clone(), templ_value); + } + } + + for (k, v) in template_values { + let template = template_map.get_mut(&k).unwrap(); + template.value = v; + } + } + } + let result: Vec = template_map.iter_mut().map(|(_, t)| t.clone()).collect(); + Ok(result) +} + +pub fn apply_templates_to_pattern( + pattern: &str, + templates_list: Option<&[PatternTemplate]>, + allow_multi: bool, +) -> Result { + let mut new_pattern = TemplateValue::Single(pattern.to_string()); + + if let Some(templates) = templates_list { + for template in templates { + new_pattern = apply_dependency_template_value(new_pattern, &template.placeholder, &template.value); + } + } + + if !allow_multi { + match &new_pattern { + TemplateValue::Single(_) => {} + TemplateValue::Multi(multi_vals) => match multi_vals.len().cmp(&1) { + Ordering::Less => { + return Err(TuliproxError::FilterParse(format!( + "Empty multi value templates are not supported for pattern! {pattern}" + ))); + } + Ordering::Equal => { + new_pattern = TemplateValue::Single(multi_vals.first().unwrap().to_owned()); + } + Ordering::Greater => { + return Err(TuliproxError::FilterParse(format!( + "Multi value templates are not supported for pattern! {pattern}" + ))); + } + }, + } + } + + Ok(new_pattern) +} + +pub fn apply_templates_to_pattern_single( + pattern: &str, + templates: Option<&[PatternTemplate]>, +) -> Result { + match apply_templates_to_pattern(pattern, templates, false)? { + TemplateValue::Single(value) => Ok(value), + TemplateValue::Multi(_) => { + Err(TuliproxError::FilterParse("Multi value templates are not supported for pattern!".to_string())) + } + } +} diff --git a/shared/src/foundation/filter/tests.rs b/shared/src/foundation/filter/tests.rs new file mode 100644 index 000000000..fb93e5e9f --- /dev/null +++ b/shared/src/foundation/filter/tests.rs @@ -0,0 +1,433 @@ +use crate::{ + foundation::{filter::get_filter, prepare_templates, value_provider::ValueProvider}, + model::{ItemField, PatternTemplate, PlaylistItem, PlaylistItemHeader, PlaylistItemType, TemplateValue}, + utils::{Internable, CONSTANTS}, +}; +use std::borrow::Cow; + +fn create_mock_pli(name: &str, group: &str) -> PlaylistItem { + PlaylistItem { header: PlaylistItemHeader { name: name.into(), group: group.intern(), ..Default::default() } } +} + +/// Parse `input` and assert that `Display` round-trips back to `input` +/// byte-for-byte. Used by the three filter-parser tests below that only +/// exercise the parser and its `Display` impl, not the runtime evaluator. +fn assert_filter_round_trip(input: &str) { + match get_filter(input, None) { + Ok(filter) => assert_eq!(format!("{filter}"), input), + Err(e) => panic!("{e}"), + } +} + +#[test] +fn test_filter_1() { + assert_filter_round_trip( + r#"(Group ~ "A" OR Group ~ "B") AND (Name ~ "C" OR Name ~ "D" OR Name ~ "E") OR (NOT (Title ~ "F") AND NOT Title ~ "K")"#, + ); +} + +#[test] +fn test_filter_2() { + assert_filter_round_trip( + r#"Group ~ "d" AND ((Name ~ "e" AND NOT ((Name ~ "c" OR Name ~ "f"))) OR (Name ~ "a" OR Name ~ "b"))"#, + ); +} + +#[test] +fn test_filter_3() { + assert_filter_round_trip( + r#"Group ~ "d" AND ((Name ~ "e" AND NOT ((Name ~ "c" OR Name ~ "f"))) OR (Name ~ "a" OR Name ~ "b")) AND (Type = movie)"#, + ); +} + +#[test] +fn test_filter_string_ops_round_trip() { + assert_filter_round_trip( + r#"Group = "Sports" AND Title != "News" OR Name CONTAINS "HD" AND Caption STARTSWITH "DE:""#, + ); +} + +#[test] +fn test_filter_numeric_round_trip() { assert_filter_round_trip(r"Chno >= 100 AND Chno < 200 OR NOT Chno = 7"); } + +#[test] +fn test_filter_set_round_trip() { assert_filter_round_trip(r#"Group IN ["Sports", "News"] AND NOT (Name IN ["A"])"#); } + +#[test] +fn test_filter_presence_round_trip_and_aliases() { + assert_filter_round_trip("EpgId IS EMPTY"); + assert_filter_round_trip("EpgId IS NOT EMPTY"); + + assert_eq!(get_filter("EpgId = EMPTY", None).expect("alias parses").to_string(), "EpgId IS EMPTY"); + assert_eq!(get_filter("EpgId != EMPTY", None).expect("alias parses").to_string(), "EpgId IS NOT EMPTY"); +} + +#[test] +fn test_filter_presence_eval() { + let empty = get_filter("EpgId IS EMPTY", None).expect("empty filter parses"); + let populated = get_filter("EpgId IS NOT EMPTY", None).expect("populated filter parses"); + let mut item = create_mock_pli("Channel", "Group"); + let provider = ValueProvider { pli: &item, match_as_ascii: false }; + assert!(empty.filter(&provider)); + assert!(!populated.filter(&provider)); + + item.header.epg_channel_id = Some("channel.epg".intern()); + let provider = ValueProvider { pli: &item, match_as_ascii: false }; + assert!(!empty.filter(&provider)); + assert!(populated.filter(&provider)); + + item.header.epg_channel_id = Some("".intern()); + let provider = ValueProvider { pli: &item, match_as_ascii: false }; + assert!(empty.filter(&provider)); + assert!(!populated.filter(&provider)); +} + +#[test] +fn test_filter_string_ops_eval() { + let flt = r#"Group = "sports" AND Name CONTAINS "hd" AND NOT (Name STARTSWITH "x")"#; + let filter = get_filter(flt, None).expect("filter parses"); + let matching = create_mock_pli("Channel HD", "Sports"); + assert!(filter.filter(&ValueProvider { pli: &matching, match_as_ascii: false })); + let other = create_mock_pli("Channel", "Sports"); + assert!(!filter.filter(&ValueProvider { pli: &other, match_as_ascii: false })); +} + +#[test] +fn test_filter_set_eval() { + let filter = get_filter(r#"Group IN ["Sports", "News"]"#, None).expect("filter parses"); + let matching = create_mock_pli("A", "news"); + assert!(filter.filter(&ValueProvider { pli: &matching, match_as_ascii: false })); + let other = create_mock_pli("B", "Movies"); + assert!(!filter.filter(&ValueProvider { pli: &other, match_as_ascii: false })); +} + +#[test] +fn test_filter_caption_string_ops_fall_back_to_name() { + // title and name differ; only the name matches + let mut pli = create_mock_pli("Channel HD", "Sports"); + pli.header.title = "Some Title".into(); + let provider = ValueProvider { pli: &pli, match_as_ascii: false }; + + let contains = get_filter(r#"Caption CONTAINS "hd""#, None).expect("filter parses"); + assert!(contains.filter(&provider)); + + let starts = get_filter(r#"Caption STARTSWITH "channel""#, None).expect("filter parses"); + assert!(starts.filter(&provider)); + + let eq = get_filter(r#"Caption = "channel hd""#, None).expect("filter parses"); + assert!(eq.filter(&provider)); + // NotEq is the negation of Eq-on-either + let not_eq = get_filter(r#"Caption != "channel hd""#, None).expect("filter parses"); + assert!(!not_eq.filter(&provider)); + + let set = get_filter(r#"Caption IN ["Channel HD"]"#, None).expect("filter parses"); + assert!(set.filter(&provider)); + + let no_match = get_filter(r#"Caption CONTAINS "missing""#, None).expect("filter parses"); + assert!(!no_match.filter(&provider)); +} + +#[test] +fn test_filter_chno_eval() { + let filter = get_filter("Chno >= 10 AND Chno <= 20", None).expect("filter parses"); + let mut pli = create_mock_pli("A", "G"); + pli.header.chno = 15; + assert!(filter.filter(&ValueProvider { pli: &pli, match_as_ascii: false })); + pli.header.chno = 5; + assert!(!filter.filter(&ValueProvider { pli: &pli, match_as_ascii: false })); +} + +#[test] +fn test_filter_quality_round_trip_and_eval() { + assert_filter_round_trip(r"Quality >= 3 AND NOT Quality = 5"); + + let filter = get_filter("Quality >= 3", None).expect("filter parses"); + let mut fhd = create_mock_pli("X", "G"); + fhd.header.title = "News FHD".into(); + assert!(filter.filter(&ValueProvider { pli: &fhd, match_as_ascii: false })); + let mut hd = create_mock_pli("X", "G"); + hd.header.title = "News HD".into(); + assert!(!filter.filter(&ValueProvider { pli: &hd, match_as_ascii: false })); + let unknown = create_mock_pli("News", "G"); + assert!(!filter.filter(&ValueProvider { pli: &unknown, match_as_ascii: false })); +} + +#[test] +fn test_filter_4() { + let flt = r#"NOT (Name ~ ".*24/7.*" AND Group ~ "^US.*")"#; + match get_filter(flt, None) { + Ok(filter) => { + assert_eq!(format!("{filter}"), flt); + let channels = [ + create_mock_pli("24/7: Cars", "FR Channels"), + create_mock_pli("24/7: Cars", "US Channels"), + create_mock_pli("Entertainment", "US Channels"), + ]; + let filtered: Vec<&PlaylistItem> = channels + .iter() + .filter(|&chan| { + let provider = ValueProvider { pli: chan, match_as_ascii: false }; + filter.filter(&provider) + }) + .collect(); + assert_eq!(filtered.len(), 2); + assert!(filtered.iter().any(|&chan| { + let group = chan.header.group.to_string(); + let name = chan.header.name.to_string(); + name.eq("24/7: Cars") && group.eq("FR Channels") + })); + assert!(filtered.iter().any(|&chan| { + let group = chan.header.group.to_string(); + let name = chan.header.name.to_string(); + name.eq("Entertainment") && group.eq("US Channels") + })); + assert!(!filtered.iter().any(|&chan| { + let group = chan.header.group.to_string(); + let name = chan.header.name.to_string(); + name.eq("24/7: Cars") && group.eq("US Channels") + })); + } + Err(e) => { + panic!("{e}") + } + } +} + +#[test] +fn test_filter_5() { + let flt = + r#"NOT (Name ~ "NC" OR Group ~ "GA") AND (Name ~ "NA" AND Group ~ "GA") OR (Name ~ "NB" AND Group ~ "GB")"#; + match get_filter(flt, None) { + Ok(filter) => { + assert_eq!(format!("{filter}"), flt); + let channels = [ + create_mock_pli("NA", "GA"), + create_mock_pli("NB", "GB"), + create_mock_pli("NA", "GB"), + create_mock_pli("NB", "GA"), + create_mock_pli("NC", "GA"), + create_mock_pli("NA", "GC"), + ]; + let filtered: Vec<&PlaylistItem> = channels + .iter() + .filter(|&chan| { + let provider = ValueProvider { pli: chan, match_as_ascii: false }; + filter.filter(&provider) + }) + .collect(); + assert_eq!(filtered.len(), 1); + } + Err(e) => { + panic!("{e}") + } + } +} + +#[test] +fn test_filter_6() { + let flt = r####" + Group ~ "^EU \| FRANCE.*" + OR Input ~ "hello" + OR Group ~ "^VOD \| FR.*" + OR Group ~ "\[FR\].*" + OR Group ~ "^SRS \| FR.*" + AND NOT (Group ~ ".* LQ.*" + OR Title ~ ".* LQ.*" + OR Group ~ ".* SD.*" + OR Title ~ ".* SD.*" + OR Group ~ ".* HD.*" + OR Title ~ ".* HD.*" + OR Group ~ "(?i).*sport.*" + OR Group ~ "(?i).*DAZN.*" + OR Group ~ "(?i).*EQUIPE.*" + OR Group ~ "DOM TOM.*" + OR Group ~ "(?i).*PLUTO.*" + OR Title ~ "(?i).*GOLD.*" + OR Title ~ "###.*")"####; + + match get_filter(flt, None) { + Ok(filter) => { + let result = CONSTANTS.re_whitespace.replace_all(flt, " "); + assert_eq!(format!("{filter}"), result.trim()); + } + Err(e) => { + panic!("{e}") + } + } +} + +#[test] +fn test_filter_7() { + let flt = r#"NOT (Name ~ ".*24/7.*")"#; + match get_filter(flt, None) { + Ok(filter) => { + assert_eq!(format!("{filter}"), flt); + let channels = [ + create_mock_pli("24/7: Cars", "FR Channels"), + create_mock_pli("24/7: Cars", "US Channels"), + create_mock_pli("Entertainment", "US Channels"), + ]; + let filtered: Vec<&PlaylistItem> = channels + .iter() + .filter(|&chan| { + let provider = ValueProvider { pli: chan, match_as_ascii: false }; + filter.filter(&provider) + }) + .collect(); + assert_eq!(filtered.len(), 1); + assert!(filtered.iter().any(|&chan| { + let group = chan.header.group.to_string(); + let name = chan.header.name.to_string(); + name.eq("Entertainment") && group.eq("US Channels") + })); + } + Err(e) => { + panic!("{e}") + } + } +} + +#[test] +fn test_filter_match_as_ascii() { + let flt = r#"Name ~ "Cinema""#; + match get_filter(flt, None) { + Ok(filter) => { + let chan = create_mock_pli("Cinéma", "Some Group"); + + // Without match_as_ascii (should fail) + let provider_fail = ValueProvider { pli: &chan, match_as_ascii: false }; + assert!(!filter.filter(&provider_fail)); + + // With match_as_ascii (should succeed) + let provider_success = ValueProvider { pli: &chan, match_as_ascii: true }; + assert!(filter.filter(&provider_success)); + } + Err(e) => panic!("{e}"), + } +} + +#[test] +fn filter_value_borrows_stored_field() { + let channel = create_mock_pli("Channel", "Group"); + let provider = ValueProvider { pli: &channel, match_as_ascii: false }; + + assert!(matches!(provider.get_filter_value(ItemField::Name), Some(Cow::Borrowed("Channel")))); +} + +#[test] +fn missing_genre_does_not_match_empty_pattern() { + let filter = get_filter(r#"Genre ~ "^$""#, None).expect("filter should parse"); + let channel = create_mock_pli("Channel", "Group"); + let provider = ValueProvider { pli: &channel, match_as_ascii: false }; + + assert!(!filter.filter(&provider)); +} + +#[test] +fn caption_checks_name_when_title_does_not_match() { + let filter = get_filter(r#"Caption ~ "Channel""#, None).expect("filter should parse"); + let mut channel = create_mock_pli("Channel", "Group"); + channel.header.title = "Different title".intern(); + let provider = ValueProvider { pli: &channel, match_as_ascii: false }; + + assert!(filter.filter(&provider)); +} + +#[test] +fn type_filter_matches_normalized_type_families() { + let live_filter = get_filter("Type = live", None).expect("live filter should parse"); + let series_filter = get_filter("Type = series", None).expect("series filter should parse"); + let mut channel = create_mock_pli("Channel", "Group"); + + channel.header.item_type = PlaylistItemType::LiveHls; + assert!(live_filter.filter(&ValueProvider { pli: &channel, match_as_ascii: false })); + + channel.header.item_type = PlaylistItemType::LocalSeriesInfo; + assert!(series_filter.filter(&ValueProvider { pli: &channel, match_as_ascii: false })); +} + +#[test] +fn test_filter_unknown_template_placeholder_reports_error() { + let err = get_filter("!UNKNOWN_FILTER!", None).expect_err("unknown placeholder should fail"); + let msg = err.to_string(); + assert!(msg.contains("Unknown template placeholder(s) in filter")); + assert!(msg.contains("!UNKNOWN_FILTER!")); +} + +#[test] +fn prepare_templates_does_not_duplicate_unrelated_multi_entries() { + let mut templates = vec![ + PatternTemplate { + name: "A".to_string(), + value: TemplateValue::Multi(vec!["a1".to_string(), "a2".to_string()]), + placeholder: String::new(), + }, + PatternTemplate { + name: "SEQ".to_string(), + value: TemplateValue::Multi(vec!["!A!".to_string(), "literal".to_string()]), + placeholder: String::new(), + }, + ]; + + let prepared = prepare_templates(&mut templates).expect("templates should prepare"); + let seq = prepared.iter().find(|template| template.name == "SEQ").expect("SEQ template"); + + assert_eq!(seq.value, TemplateValue::Multi(vec!["a1".to_string(), "a2".to_string(), "literal".to_string()])); +} + +#[test] +fn prepare_templates_concatenates_sequence_templates_without_cartesian_blowup() { + let mut templates = vec![ + PatternTemplate { + name: "UK".to_string(), + value: TemplateValue::Multi(vec!["uk1".to_string(), "uk2".to_string()]), + placeholder: String::new(), + }, + PatternTemplate { + name: "US".to_string(), + value: TemplateValue::Multi(vec!["us1".to_string(), "us2".to_string(), "us3".to_string()]), + placeholder: String::new(), + }, + PatternTemplate { + name: "SEQ".to_string(), + value: TemplateValue::Multi(vec!["!UK!".to_string(), "!US!".to_string(), "adult".to_string()]), + placeholder: String::new(), + }, + ]; + + let prepared = prepare_templates(&mut templates).expect("templates should prepare"); + let seq = prepared.iter().find(|template| template.name == "SEQ").expect("SEQ template"); + + assert_eq!( + seq.value, + TemplateValue::Multi(vec![ + "uk1".to_string(), + "uk2".to_string(), + "us1".to_string(), + "us2".to_string(), + "us3".to_string(), + "adult".to_string(), + ]) + ); +} + +#[test] +fn prepare_templates_promotes_single_template_to_multi_for_multi_dependency() { + let mut templates = vec![ + PatternTemplate { + name: "BASE".to_string(), + value: TemplateValue::Multi(vec!["a".to_string(), "b".to_string()]), + placeholder: String::new(), + }, + PatternTemplate { + name: "URL".to_string(), + value: TemplateValue::Single("!BASE!/stream".to_string()), + placeholder: String::new(), + }, + ]; + + let prepared = prepare_templates(&mut templates).expect("templates should prepare"); + let url = prepared.iter().find(|template| template.name == "URL").expect("URL template"); + + assert_eq!(url.value, TemplateValue::Multi(vec!["a/stream".to_string(), "b/stream".to_string()])); +} diff --git a/shared/src/foundation/mapper.rs b/shared/src/foundation/mapper.rs index 156dce48a..f7471a41c 100644 --- a/shared/src/foundation/mapper.rs +++ b/shared/src/foundation/mapper.rs @@ -1,80 +1,8 @@ #![allow(clippy::empty_docs)] -use crate::{ - error::TuliproxError, - foundation::mapper::EvalResult::{AnyValue, Failure, Named, Number, Undefined, Value}, - model::{FieldGetAccessor, FieldSetAccessor, PatternTemplate, PlaylistItem, PlaylistItemType, TemplateValue}, - utils::{deunicode_string, Capitalize, Internable}, -}; -use log::{debug, error, trace}; -use pest::{ - iterators::{Pair, Pairs}, - Parser, -}; -use pest_derive::Parser; -use regex::Regex; -use std::{ - borrow::Cow, - cmp::Ordering, - collections::{HashMap, HashSet}, - fmt::Display, - ops::Deref, - str::FromStr, - sync::Arc, -}; - -#[derive(Parser)] -#[grammar_inline = r##" -WHITESPACE = _{ " " | "\t"} -regex_op = _{ "~" } -null = { "null" } -identifier = @{ !null ~ (ASCII_ALPHANUMERIC | "_")+ } -var_access = { identifier ~ ("." ~ identifier)? } -string_literal = @{ "\"" ~ ( "\\\\" | "\\\"" | "\\n" | "\\t" | "\\r" | (!"\"" ~ ANY) )* ~ "\"" } -number = @{ "-"? ~ ASCII_DIGIT+ ~ ("." ~ ASCII_DIGIT+)? } -number_range_from = { number ~ ".." } -number_range_to = { ".." ~ number } -number_range_full = { number ~ ".." ~ number } -number_range_eq = { number } -number_range = _{ number_range_full | number_range_from | number_range_to | number_range_eq} -read_write_field = _{ ^"name" | ^"title" | ^"caption" | ^"group" | ^"id" | ^"chno" | ^"logo" | ^"logo_small" | ^"parent_code" | ^"audio_track" | ^"time_shift" | ^"rec" | ^"url" | ^"epg_channel_id" | ^"epg_id" | ^"genre" } -read_only_field = _{ ^"input" | ^"type" } -field = { read_write_field | read_only_field } -assignment_field = { read_write_field } -field_access = _{ "@" ~ field } -regex_source = _{ field_access | identifier } -regex_expr = { regex_source ~ regex_op ~ string_literal } -block_expr = { "{" ~ statements ~ "}" } -condition = { function_call | var_access | field_access } -assignment = { (("@" ~ assignment_field) | identifier) ~ "=" ~ expression } -expression = { assignment | map_block | match_block | for_each_block | function_call | regex_expr | string_literal | number | var_access | field_access | null | block_expr } -function_name = { "concat" | "uppercase" | "lowercase" | "capitalize" | "split" | "trim" | "print" | "number" | "first" | "template" | "replace" | "pad" | "format" | "add_favourite" } -function_call = { function_name ~ "(" ~ (expression ~ ("," ~ expression)*)? ~ ")" } -any_match = { "_" } -match_case_key = { any_match | identifier } -match_case_key_list = { match_case_key ~ ("," ~ match_case_key)* } -match_case = { match_case_key_list ~ "=>" ~ expression | "(" ~ match_case_key_list ~ ")" ~ "=>" ~ expression } -match_block = { "match" ~ "{" ~ NEWLINE* ~ (match_case ~ ("," ~ NEWLINE* ~ match_case)*)? ~ ","? ~ NEWLINE* ~ "}" } -map_case_key_list = { string_literal ~ ("|" ~ string_literal)* } -map_case_key = { any_match | number_range | map_case_key_list } -map_case = { map_case_key ~ "=>" ~ expression } -map_key = { var_access | field_access } -map_block = { "map" ~ map_key ~ "{" ~ NEWLINE* ~ (map_case ~ ("," ~ NEWLINE* ~ map_case)*)? ~ ","? ~ NEWLINE* ~ "}" } -for_each_param = { any_match | identifier } -for_each_params = { "(" ~ for_each_param ~ "," ~ for_each_param ~ ")" } -for_each_target_nested = { identifier ~ "." ~ identifier } -for_each_target_simple = { identifier } -for_each_block = { - (for_each_target_nested ~ ^".for_each" ~ "(" ~ for_each_params ~ "=>" ~ expression ~ ")") | - (for_each_target_simple ~ ^".for_each" ~ "(" ~ for_each_params ~ "=>" ~ expression ~ ")") -} -statement = _{ expression } -comment = _{ "#" ~ (!NEWLINE ~ ANY)* } -statement_reparator = _{ ";" | NEWLINE } -statements = _{ (statement_reparator* ~ (statement | comment))* ~ statement_reparator* } -main = { SOI ~ statements? ~ EOI } -"##] -struct MapperParser; +use crate::model::{FieldGetAccessor, FieldSetAccessor, PatternTemplate, PlaylistItem}; +use log::{error, trace}; +use std::{collections::HashMap, sync::Arc}; /// /// The `ValueAccessor` does not respect `match_as_ascii` when returning values. @@ -103,1055 +31,18 @@ impl ValueAccessor<'_> { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct ExprId(pub usize); - -impl Deref for ExprId { - type Target = usize; - - fn deref(&self) -> &Self::Target { &self.0 } -} - -#[derive(Debug, Clone, PartialEq)] -pub enum MatchCaseKey { - Identifier(String), - AnyMatch, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct MatchCase { - pub keys: Vec, - pub expression: ExprId, -} - -#[derive(Debug, Clone, PartialEq)] -pub enum MapCaseKey { - Text(String), - RangeFrom(f64), - RangeTo(f64), - RangeFull(f64, f64), - RangeEq(f64), - AnyMatch, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct MapCase { - pub keys: Vec, - pub expression: ExprId, -} - -#[derive(Debug, Clone, PartialEq)] -pub enum MapKey { - Identifier(String), - FieldAccess(String), - VarAccess(String, String), -} - -#[derive(Debug, Clone, PartialEq)] -pub enum ForEachKey { - Identifier(String), - VarAccess(String, String), -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ForEachExpr { - pub key_var: Option, - pub value_var: Option, - pub expression: ExprId, -} - -#[derive(Debug, Clone, PartialEq)] -pub enum BuiltInFunction { - Concat, - Uppercase, - Lowercase, - Capitalize, - Split, - Trim, - Print, - ToNumber, - First, - Template, - Replace, - Pad, - Format, - AddFavourite, -} - -impl FromStr for BuiltInFunction { - type Err = TuliproxError; - - fn from_str(s: &str) -> Result { - match s.to_lowercase().as_str() { - "concat" => Ok(Self::Concat), - "capitalize" => Ok(Self::Capitalize), - "lowercase" => Ok(Self::Lowercase), - "uppercase" => Ok(Self::Uppercase), - "split" => Ok(Self::Split), - "trim" => Ok(Self::Trim), - "print" => Ok(Self::Print), - "number" => Ok(Self::ToNumber), - "first" => Ok(Self::First), - "template" => Ok(Self::Template), - "replace" => Ok(Self::Replace), - "pad" => Ok(Self::Pad), - "format" => Ok(Self::Format), - "add_favourite" => Ok(Self::AddFavourite), - _ => Err(TuliproxError::Mapper(format!("Unknown function {s}"))), - } - } -} - -impl Display for BuiltInFunction { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(match self { - Self::Concat => "concat", - Self::Capitalize => "capitalize", - Self::Lowercase => "lowercase", - Self::Uppercase => "uppercase", - Self::Split => "split", - Self::Trim => "trim", - Self::Print => "print", - Self::ToNumber => "number", - Self::First => "first", - Self::Template => "template", - Self::Replace => "replace", - Self::Pad => "pad", - Self::Format => "format", - Self::AddFavourite => "add_favourite", - }) - } -} - -#[derive(Debug, Clone, PartialEq)] -pub enum RegexSource { - Identifier(String), - Field(String), -} - -#[derive(Debug, Clone)] -pub enum Expression { - Identifier(String), - StringLiteral(String), - NumberLiteral(f64), - FieldAccess(String), - VarAccess(String, String), - RegexExpr { field: RegexSource, pattern: String, re_pattern: Arc }, - FunctionCall { name: BuiltInFunction, args: Vec }, - Assignment { target: AssignmentTarget, expr: ExprId }, - MatchBlock(Vec), - MapBlock { key: MapKey, cases: Vec }, - ForEachBlock { key: ForEachKey, expr: ForEachExpr }, - NullValue, - Block(Vec), -} - -impl PartialEq for Expression { - fn eq(&self, other: &Self) -> bool { - use Expression::{ - Assignment, Block, FieldAccess, ForEachBlock, FunctionCall, Identifier, MapBlock, MatchBlock, NullValue, - NumberLiteral, RegexExpr, StringLiteral, VarAccess, - }; - match (self, other) { - (Identifier(a), Identifier(b)) => a == b, - (StringLiteral(a), StringLiteral(b)) => a == b, - (NumberLiteral(a), NumberLiteral(b)) => a == b, - (FieldAccess(a), FieldAccess(b)) => a == b, - (VarAccess(a1, b1), VarAccess(a2, b2)) => a1 == a2 && b1 == b2, - (RegexExpr { field: f1, pattern: p1, .. }, RegexExpr { field: f2, pattern: p2, .. }) => { - f1 == f2 && p1 == p2 - } - (FunctionCall { name: n1, args: a1 }, FunctionCall { name: n2, args: a2 }) => n1 == n2 && a1 == a2, - (Assignment { target: t1, expr: e1 }, Assignment { target: t2, expr: e2 }) => t1 == t2 && e1 == e2, - (MatchBlock(m1), MatchBlock(m2)) => m1 == m2, - (MapBlock { key: k1, cases: c1 }, MapBlock { key: k2, cases: c2 }) => k1 == k2 && c1 == c2, - (ForEachBlock { key: k1, expr: c1 }, ForEachBlock { key: k2, expr: c2 }) => k1 == k2 && c1 == c2, - (NullValue, NullValue) => true, - (Block(b1), Block(b2)) => b1 == b2, - _ => false, - } - } -} - -#[derive(Debug, Clone, PartialEq)] -pub enum AssignmentTarget { - Identifier(String), - Field(String), -} - -#[derive(Debug, Clone, PartialEq)] -pub enum Statement { - Expression(ExprId), - Comment(String), -} - #[derive(Debug, Clone, PartialEq)] pub struct MapperScript { pub expressions: Vec, pub statements: Vec, } -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct MappingDiagnostic { - pub statement: usize, - pub message: String, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct MappingOutcome { - pub changed_fields: Vec, - pub emitted_items: usize, - pub diagnostics: Vec, -} - -impl MapperScript { - pub fn eval(&self, setter: &mut ValueAccessor, templates: Option<&[PatternTemplate]>) -> MappingOutcome { - let initial_change_count = setter.changed_fields.len(); - let initial_item_count = setter.virtual_items.len(); - let ctx = &mut MapperContext::new(&self.expressions, templates); - let diagnostics = self.eval_with_context(ctx, setter); - MappingOutcome { - changed_fields: setter.changed_fields[initial_change_count..].to_vec(), - emitted_items: setter.virtual_items.len().saturating_sub(initial_item_count), - diagnostics, - } - } - - fn eval_with_context(&self, ctx: &mut MapperContext, setter: &mut ValueAccessor) -> Vec { - self.statements - .iter() - .enumerate() - .filter_map(|(statement, stmt)| { - stmt.eval(ctx, setter).map(|message| MappingDiagnostic { statement, message }) - }) - .collect() - } - - pub fn get_expr_by_id(&self, id: usize) -> Option<&Expression> { self.expressions.get(id) } -} - -impl ExprId { - pub fn eval(self, ctx: &mut MapperContext, accessor: &mut ValueAccessor) -> EvalResult { - let id = self.0; - ctx.eval_expr_by_id(id, accessor) - } -} - -impl Statement { - pub fn eval(&self, ctx: &mut MapperContext, setter: &mut ValueAccessor) -> Option { - match self { - Statement::Expression(expr_id) => { - let result = expr_id.eval(ctx, setter); - if let Failure(err) = &result { - debug!("{err}"); - Some(err.clone()) - } else { - None - } - } - Statement::Comment(_) => None, - } - } -} - -impl MapperScript { - fn validate( - expressions: &[Expression], - statements: &[Statement], - templates: Option<&[PatternTemplate]>, - ) -> Result<(), TuliproxError> { - let ctx = &mut MapperContext::new(expressions, templates); - - let mut identifiers: HashSet = HashSet::new(); - for stmt in statements { - match stmt { - Statement::Expression(expr) => { - ctx.validate_expr(*expr, &mut identifiers)?; - } - Statement::Comment(_) => {} - } - } - Ok(()) - } - - pub fn parse(input: &str, templates: Option<&[PatternTemplate]>) -> Result { - let mut parsed = MapperParser::parse(Rule::main, input).map_err(|e| TuliproxError::Mapper(format!("{e}")))?; - let program_pair = parsed.next().unwrap(); - let mut statements = Vec::new(); - let mut expressions = Vec::new(); - for stmt_pair in program_pair.into_inner() { - if let Some(stmt) = Self::parse_statement(stmt_pair, &mut expressions)? { - statements.push(stmt); - } - } - - MapperScript::validate(&expressions, &statements, templates)?; - Ok(Self { expressions, statements }) - } - fn parse_statement( - pair: Pair, - expressions: &mut Vec, - ) -> Result, TuliproxError> { - match pair.as_rule() { - Rule::expression => { - if let Some(expr) = MapperScript::parse_expression(pair, expressions)? { - expressions.push(expr); - let expr_id = ExprId(expressions.len() - 1); - Ok(Some(Statement::Expression(expr_id))) - } else { - Ok(None) - } - } - Rule::comment => Ok(Some(Statement::Comment(pair.as_str().trim().to_string()))), - - _ => { - // error!("Unknown statement rule: {:?}", pair.as_rule()); - Ok(None) - } - } - } - - fn parse_assignment( - pair: Pair, - expressions: &mut Vec, - ) -> Result, TuliproxError> { - let mut inner = pair.into_inner(); - let name = inner.next().unwrap(); - let target = match name.as_rule() { - Rule::identifier => AssignmentTarget::Identifier(name.as_str().to_string()), - Rule::assignment_field => AssignmentTarget::Field(name.as_str().to_string()), - _ => return Err(TuliproxError::Mapper(format!("Assignment target isn't supported {}", name.as_str()))), - }; - let next = inner.next().unwrap(); - if let Some(expr) = MapperScript::parse_expression(next, expressions)? { - expressions.push(expr); - let expr_id = ExprId(expressions.len() - 1); - Ok(Some(Expression::Assignment { target, expr: expr_id })) - } else { - Ok(None) - } - } - - fn parse_match_case_key(pair: Pair) -> Result { - let inner = pair.into_inner().next().unwrap(); - match inner.as_rule() { - Rule::identifier => Ok(MatchCaseKey::Identifier(inner.as_str().to_string())), - Rule::any_match => Ok(MatchCaseKey::AnyMatch), - _ => Err(TuliproxError::Mapper(format!("Unexpected match_key: {:?}", inner.as_rule()))), - } - } - - fn parse_match_case( - pair: Pair, - expressions: &mut Vec, - ) -> Result, TuliproxError> { - let mut inner = pair.into_inner(); - - let first = inner.next().unwrap(); - - let identifiers = match first.as_rule() { - Rule::match_case_key => { - vec![MapperScript::parse_match_case_key(first)?] - } - Rule::match_case_key_list => { - let mut matches = vec![]; - for arm in first.into_inner() { - if arm.as_rule() != Rule::WHITESPACE { - match MapperScript::parse_match_case_key(arm)? { - MatchCaseKey::Identifier(ident) => matches.push(MatchCaseKey::Identifier(ident)), - MatchCaseKey::AnyMatch => matches.push(MatchCaseKey::AnyMatch), - } - } - } - // we don't allow inside multi match keys AnyMatch - if matches.len() > 1 && matches.iter().filter(|&m| matches!(m, &MatchCaseKey::AnyMatch)).count() > 0 { - return Err(TuliproxError::Mapper("Unexpected match case key: _".to_string())); - } - matches - } - _ => return Err(TuliproxError::Mapper(format!("Unexpected match arm input: {:?}", first.as_rule()))), - }; - - if let Some(expr) = MapperScript::parse_expression(inner.next().unwrap(), expressions)? { - expressions.push(expr); - let expr_id = ExprId(expressions.len() - 1); - Ok(Some(MatchCase { keys: identifiers, expression: expr_id })) - } else { - Ok(None) - } - } - - fn parse_map_case_key(pair: Pair) -> Result, TuliproxError> { - let inner = pair.into_inner().next().unwrap(); - match inner.as_rule() { - Rule::map_case_key_list => { - let mut matches = vec![]; - for arm in inner.into_inner() { - match arm.as_rule() { - Rule::string_literal => { - let raw = arm.as_str().to_string(); - // remove quotes - let content = &raw[1..raw.len() - 1]; - matches.push(MapCaseKey::Text(content.to_string())); - } - _ => return Err(TuliproxError::Mapper(format!("Unexpected map key: {:?}", arm.as_rule()))), - } - } - Ok(matches) - } - Rule::number_range_full => { - let mut inner = inner.into_inner(); - let start = inner.next().unwrap().as_str().parse::().unwrap(); - let end = inner.next().unwrap().as_str().parse::().unwrap(); - Ok(vec![MapCaseKey::RangeFull(start, end)]) - } - Rule::number_range_from => { - let mut inner = inner.into_inner(); - let start = inner.next().unwrap().as_str().parse::().unwrap(); - Ok(vec![MapCaseKey::RangeFrom(start)]) - } - Rule::number_range_to => { - let mut inner = inner.into_inner(); - let to = inner.next().unwrap().as_str().parse::().unwrap(); - Ok(vec![MapCaseKey::RangeTo(to)]) - } - Rule::number_range_eq => { - let mut inner = inner.into_inner(); - let num = inner.next().unwrap().as_str().parse::().unwrap(); - Ok(vec![MapCaseKey::RangeEq(num)]) - } - Rule::any_match => Ok(vec![MapCaseKey::AnyMatch]), - _ => Err(TuliproxError::Mapper(format!("Unexpected map key: {:?}", inner.as_rule()))), - } - } - - fn parse_map_case(pair: Pair, expressions: &mut Vec) -> Result, TuliproxError> { - let mut inner = pair.into_inner(); - - let first = inner.next().unwrap(); - - let identifier = match first.as_rule() { - Rule::map_case_key => MapperScript::parse_map_case_key(first)?, - _ => return Err(TuliproxError::Mapper(format!("Unexpected match arm input: {:?}", first.as_rule()))), - }; - - if let Some(expr) = MapperScript::parse_expression(inner.next().unwrap(), expressions)? { - expressions.push(expr); - let expr_id = ExprId(expressions.len() - 1); - Ok(Some(MapCase { keys: identifier, expression: expr_id })) - } else { - Ok(None) - } - } - - fn parse_expression( - pair: Pair, - expressions: &mut Vec, - ) -> Result, TuliproxError> { - match pair.as_rule() { - Rule::assignment => { - if let Some(expr) = MapperScript::parse_assignment(pair, expressions)? { - Ok(Some(expr)) - } else { - Ok(None) - } - } - Rule::field => Ok(Some(Expression::FieldAccess(pair.as_str().trim().to_string()))), - Rule::var_access => { - let text = pair.as_str(); - if text.contains('.') { - let splitted: Vec<&str> = text.splitn(2, '.').collect(); - Ok(Some(Expression::VarAccess(splitted[0].trim().to_string(), splitted[1].trim().to_string()))) - } else { - Ok(Some(Expression::Identifier(text.trim().to_string()))) - } - } - - Rule::string_literal => { - let raw = pair.as_str(); - // remove quotes - let content = &raw[1..raw.len() - 1]; - Ok(Some(Expression::StringLiteral(content.to_string()))) - } - - Rule::number => { - let raw = pair.as_str(); - if let Number(val) = to_number(raw) { - Ok(Some(Expression::NumberLiteral(val))) - } else { - Err(TuliproxError::Mapper(format!("Invalid number {raw}"))) - } - } - - Rule::regex_expr => { - let mut inner = pair.into_inner(); - let first = inner.next().unwrap(); - let field = match first.as_rule() { - Rule::identifier => RegexSource::Identifier(first.as_str().to_string()), - Rule::field => RegexSource::Field(first.as_str().to_string()), - _ => return Err(TuliproxError::RegexCompile(first.as_str().to_string())), - }; - let pattern_raw = inner.next().unwrap().as_str(); - let pattern = &pattern_raw[1..pattern_raw.len() - 1]; // Strip quotes - match crate::model::REGEX_CACHE.get_or_compile(pattern) { - Ok(re) => Ok(Some(Expression::RegexExpr { field, pattern: pattern.to_string(), re_pattern: re })), - Err(_) => Err(TuliproxError::RegexCompile(pattern.to_string())), - } - } - - Rule::function_call => { - let mut inner = pair.into_inner(); - let fn_name = inner.next().unwrap().as_str().to_string(); - let mut args = vec![]; - for arg in inner { - if let Some(expr) = MapperScript::parse_expression(arg, expressions)? { - expressions.push(expr); - let expr_id = ExprId(expressions.len() - 1); - args.push(expr_id); - } - } - let name = BuiltInFunction::from_str(&fn_name)?; - Ok(Some(Expression::FunctionCall { name, args })) - } - - Rule::match_block => { - let case_pairs = pair.into_inner(); - let mut cases = vec![]; - for case in case_pairs { - if let Some(expr) = MapperScript::parse_match_case(case, expressions)? { - cases.push(expr); - } - } - if cases.is_empty() { - Ok(None) - } else { - Ok(Some(Expression::MatchBlock(cases))) - } - } - - Rule::map_block => Self::parse_map_block(pair.into_inner(), expressions), - - Rule::for_each_block => Self::parse_for_each_block(pair.into_inner(), expressions), - - Rule::null => Ok(Some(Expression::NullValue)), - - Rule::expression => { - let inner = pair.into_inner().next().unwrap(); - MapperScript::parse_expression(inner, expressions) - } - Rule::block_expr => { - let inner = pair.into_inner(); - let mut block_expressions = vec![]; - for expr in inner { - if let Some(expr) = MapperScript::parse_expression(expr, expressions)? { - expressions.push(expr); - let expr_id = ExprId(expressions.len() - 1); - block_expressions.push(expr_id); - } - } - Ok(Some(Expression::Block(block_expressions))) - } - _ => Err(TuliproxError::Mapper(format!("Unknown expression rule: {:?}", pair.as_rule()))), - } - } - - fn parse_map_block( - mut pairs: Pairs, - expressions: &mut Vec, - ) -> Result, TuliproxError> { - let first = pairs.next().unwrap(); - let key = match first.as_rule() { - Rule::map_key => { - if let Some(map_key) = first.into_inner().next() { - match map_key.as_rule() { - Rule::field => MapKey::FieldAccess(map_key.as_str().trim().to_string()), - Rule::var_access => { - let text = map_key.as_str(); - if text.contains('.') { - let splitted: Vec<&str> = text.splitn(2, '.').collect(); - MapKey::VarAccess(splitted[0].trim().to_string(), splitted[1].trim().to_string()) - } else { - MapKey::Identifier(text.trim().to_string()) - } - } - _ => { - return Err(TuliproxError::Mapper(format!( - "Unexpected map case key: {:?}", - map_key.as_rule() - ))) - } - } - } else { - return Err(TuliproxError::Mapper("Missing map case key".to_string())); - } - } - _ => return Err(TuliproxError::Mapper(format!("Unexpected map case key: {:?}", first.as_rule()))), - }; - let mut cases = vec![]; - for case in pairs { - if let Some(map_case) = MapperScript::parse_map_case(case, expressions)? { - cases.push(map_case); - } - } - if cases.is_empty() { - Ok(None) - } else { - Ok(Some(Expression::MapBlock { key, cases })) - } - } - - fn parse_for_each_param(pair: Pair) -> Result, TuliproxError> { - let inner = pair.into_inner().next().unwrap(); - match inner.as_rule() { - Rule::identifier => Ok(Some(inner.as_str().to_string())), - Rule::any_match => Ok(None), - _ => Err(TuliproxError::Mapper(format!("Unexpected for_each_param: {:?}", inner.as_rule()))), - } - } - - fn parse_for_each_params(pair: Pair) -> Result<(Option, Option), TuliproxError> { - let mut inner = pair.into_inner(); - let key = Self::parse_for_each_param(inner.next().unwrap())?; - let val = Self::parse_for_each_param(inner.next().unwrap())?; - - if key.is_none() && val.is_none() { - return Err(TuliproxError::Mapper("At least one parameter must be named in for_each loop".to_string())); - } - - Ok((key, val)) - } - - fn parse_for_each_block( - mut pairs: Pairs, - expressions: &mut Vec, - ) -> Result, TuliproxError> { - let first = pairs.next().unwrap(); - let key = match first.as_rule() { - Rule::for_each_target_simple => ForEachKey::Identifier(first.as_str().trim().to_string()), - Rule::for_each_target_nested => { - let text = first.as_str(); - let splitted: Vec<&str> = text.splitn(2, '.').collect(); - ForEachKey::VarAccess(splitted[0].trim().to_string(), splitted[1].trim().to_string()) - } - _ => return Err(TuliproxError::Mapper(format!("Unexpected for each target: {:?}", first.as_rule()))), - }; - - if let Some(params_pair) = pairs.next() { - // .for_each - let (key_var, value_var) = Self::parse_for_each_params(params_pair)?; - - let expr_pair = pairs.next().unwrap(); - if let Some(expr) = MapperScript::parse_expression(expr_pair, expressions)? { - expressions.push(expr); - let expr_id = ExprId(expressions.len() - 1); - return Ok(Some(Expression::ForEachBlock { - key, - expr: ForEachExpr { key_var, value_var, expression: expr_id }, - })); - } - } - - Ok(None) - } -} - pub struct MapperContext<'a> { expressions: &'a [Expression], variables: HashMap, templates: Option<&'a [PatternTemplate]>, } -impl<'a> MapperContext<'a> { - fn new(expressions: &'a [Expression], templates: Option<&'a [PatternTemplate]>) -> Self { - Self { expressions, variables: HashMap::new(), templates: templates.filter(|templates| !templates.is_empty()) } - } - - fn get_template(&self, name: &str) -> Option<&str> { - self.templates?.iter().rev().find(|template| template.name == name).and_then(|template| match &template.value { - TemplateValue::Single(v) => Some(v.as_str()), - TemplateValue::Multi(_) => None, - }) - } - - fn set_var(&mut self, name: &str, value: EvalResult) { - if let Some(current) = self.variables.get_mut(name) { - *current = value; - } else { - self.variables.insert(name.to_string(), value); - } - } - - fn has_var(&self, name: &str) -> bool { self.variables.contains_key(name) } - - fn get_var(&self, name: &str) -> &EvalResult { self.variables.get(name).unwrap_or(&Undefined) } - - fn eval_expr_by_id(&mut self, id: usize, accessor: &mut ValueAccessor) -> EvalResult { - let Some(expr) = self.expressions.get(id) else { return Undefined }; - expr.eval(self, accessor) - } - - fn validate_expr(&mut self, expr_id: ExprId, identifiers: &mut HashSet) -> Result<(), TuliproxError> { - let Some(expr) = self.expressions.get(expr_id.0) else { - return Err(TuliproxError::Mapper(format!("No matching expression found at index {}", expr_id.0))); - }; - match expr { - Expression::Identifier(ident) | Expression::VarAccess(ident, _) => { - if !identifiers.contains(ident.as_str()) { - return Err(TuliproxError::Mapper(format!("Identifier unknown {ident}, {expr:?}"))); - } - } - Expression::NullValue - | Expression::FieldAccess(_) - | Expression::StringLiteral(_) - | Expression::NumberLiteral(_) => {} - Expression::RegexExpr { field, pattern: _pattern, re_pattern: _re_pattern } => match field { - RegexSource::Identifier(ident) => { - if !identifiers.contains(ident.as_str()) { - return Err(TuliproxError::Mapper(format!("Regex identifier unknown {ident}, {expr:?}"))); - } - } - RegexSource::Field(_) => {} - }, - Expression::Assignment { target, expr } => { - match target { - AssignmentTarget::Identifier(ident) => { - identifiers.insert(ident.clone()); - } - AssignmentTarget::Field(_) => {} - } - self.validate_expr(*expr, identifiers)?; - } - Expression::FunctionCall { name, args } => { - if args.is_empty() { - return Err(TuliproxError::Mapper(format!("Function needs at least one argument {name:?}"))); - } - match name { - BuiltInFunction::ToNumber - | BuiltInFunction::Template - | BuiltInFunction::First - | BuiltInFunction::AddFavourite - if args.len() > 1 => - { - return Err(TuliproxError::Mapper(format!( - "Function accepts only one argument {:?}, {} given", - name, - args.len() - ))); - } - BuiltInFunction::Split if args.len() != 2 => { - return Err(TuliproxError::Mapper(format!( - "Function accepts two arguments {:?}, {} given", - name, - args.len() - ))); - } - BuiltInFunction::Replace if args.len() != 3 => { - return Err(TuliproxError::Mapper(format!( - "Function accepts three arguments {:?}, {} given", - name, - args.len() - ))); - } - BuiltInFunction::Pad if !(args.len() == 3 || args.len() == 4) => { - return Err(TuliproxError::Mapper(format!( - "Function accepts three or four arguments {:?}, {} given", - name, - args.len() - ))); - } - _ => {} - } - for expr_id in args { - self.validate_expr(*expr_id, identifiers)?; - } - } - Expression::MatchBlock(cases) => { - self.validate_match_block(identifiers, cases)?; - } - Expression::MapBlock { key, cases } => { - self.validate_map_block(identifiers, key, cases)?; - } - Expression::ForEachBlock { key, expr } => { - self.validate_for_each_block(identifiers, key, expr)?; - } - Expression::Block(expressions) => { - for expr_id in expressions { - self.validate_expr(*expr_id, identifiers)?; - } - } - } - Ok(()) - } - - fn validate_match_block( - &mut self, - identifiers: &mut HashSet, - cases: &Vec, - ) -> Result<(), TuliproxError> { - let mut case_keys = HashSet::new(); - for match_case in cases { - let mut any_match_count = 0; - let mut identifier_key = String::with_capacity(56); - for identifier in &match_case.keys { - match identifier { - MatchCaseKey::Identifier(ident) => { - if !identifiers.contains(ident.as_str()) { - return Err(TuliproxError::Mapper(format!("Match case identifier unknown {ident}"))); - } - identifier_key.push_str(ident.as_str()); - identifier_key.push_str(", "); - } - MatchCaseKey::AnyMatch => { - any_match_count += 1; - if any_match_count > 1 { - return Err(TuliproxError::Mapper("Match case can only have one '_'".to_string())); - } - identifier_key.push_str("_, "); - } - } - } - if case_keys.contains(&identifier_key) { - return Err(TuliproxError::Mapper(format!("Duplicate case {identifier_key}"))); - } - case_keys.insert(identifier_key); - self.validate_expr(match_case.expression, identifiers)?; - } - Ok(()) - } - - fn validate_map_block( - &mut self, - identifiers: &mut HashSet, - key: &MapKey, - cases: &Vec, - ) -> Result<(), TuliproxError> { - match key { - MapKey::Identifier(ident) | MapKey::VarAccess(ident, _) => { - if !identifiers.contains(ident.as_str()) { - return Err(TuliproxError::Mapper(format!("Map key identifier unknown {ident}"))); - } - } - MapKey::FieldAccess(_) => {} - } - let mut case_keys = HashSet::new(); - let mut any_match_count = 0; - for map_case in cases { - for key in &map_case.keys { - match key { - MapCaseKey::Text(value) => { - if case_keys.contains(value.as_str()) { - return Err(TuliproxError::Mapper(format!("Duplicate case {value}"))); - } - case_keys.insert(value.as_str()); - } - MapCaseKey::RangeEq(_) | MapCaseKey::RangeTo(_) | MapCaseKey::RangeFrom(_) => {} - MapCaseKey::RangeFull(from, to) => { - if *from > *to { - return Err(TuliproxError::Mapper(format!("Invalid range {from}..{to}"))); - } - } - MapCaseKey::AnyMatch => { - any_match_count += 1; - if any_match_count > 1 { - return Err(TuliproxError::Mapper("Map case can only have one '_'".to_string())); - } - } - } - } - self.validate_expr(map_case.expression, identifiers)?; - } - Ok(()) - } - - fn validate_for_each_block( - &mut self, - identifiers: &mut HashSet, - key: &ForEachKey, - expr: &ForEachExpr, - ) -> Result<(), TuliproxError> { - match key { - ForEachKey::Identifier(ident) | ForEachKey::VarAccess(ident, _) => { - if !identifiers.contains(ident.as_str()) { - return Err(TuliproxError::Mapper(format!("For each key identifier unknown {ident}"))); - } - } - } - let mut local_identifiers = identifiers.clone(); - - if let Some(key_var) = &expr.key_var { - if local_identifiers.contains(key_var) { - return Err(TuliproxError::Mapper(format!( - "For each key variable shadows existing identifier {key_var}" - ))); - } - local_identifiers.insert(key_var.clone()); - } - - if let Some(value_var) = &expr.value_var { - if local_identifiers.contains(value_var) { - return Err(TuliproxError::Mapper(format!( - "For each value variable shadows existing identifier {value_var}" - ))); - } - local_identifiers.insert(value_var.clone()); - } - - self.validate_expr(expr.expression, &mut local_identifiers)?; - - Ok(()) - } -} - -#[derive(Debug, Clone)] -pub enum EvalResult { - Undefined, - Value(String), - Number(f64), - Named(Vec<(String, String)>), - AnyValue, - Failure(String), -} - -fn to_number(value: &str) -> EvalResult { - match value.parse::() { - Ok(num) => Number(num), - Err(_) => Failure(format!("Invalid number: {value}")), - } -} - -fn compare_number(a: f64, b: f64) -> Ordering { - let epsilon = 1e-3; // = 0.001 - - if (a - b).abs() < epsilon { - Ordering::Equal - } else if a < b { - Ordering::Less - } else { - Ordering::Greater - } -} - -fn format_number(num: f64) -> String { - let epsilon = 1e-3; // = 0.001 - - if num.fract().abs() < epsilon { - format!("{}", num as i64) - } else { - format!("{num}") - } -} - -fn compare_tuple_vec(a: &[(String, String)], b: &[(String, String)]) -> bool { - fn to_map(vec: &[(String, String)]) -> HashMap<&str, &str> { - vec.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect() - } - - to_map(a) == to_map(b) -} - -fn match_number(num: f64, s: &str) -> bool { - if let Ok(val) = s.parse::() { - return compare_number(num, val) == Ordering::Equal; - } - false -} - -fn cmp_number(num: f64, s: &str) -> Option { - if let Ok(val) = s.parse::() { - return Some(compare_number(num, val)); - } - None -} - -impl EvalResult { - fn compare(&self, other: &EvalResult) -> Option { - match (self, other) { - (AnyValue, _) | (_, AnyValue) => Some(Ordering::Equal), - (Value(a), Value(b)) => Some(a.cmp(b)), - (Number(a), Value(b)) => cmp_number(*a, b), - (Value(a), Number(b)) => match cmp_number(*b, a) { - None => None, - Some(ord) => match ord { - Ordering::Less => Some(Ordering::Greater), - Ordering::Equal => Some(Ordering::Equal), - Ordering::Greater => Some(Ordering::Less), - }, - }, - (Number(a), Number(b)) => Some(compare_number(*a, *b)), - (Named(a), Named(b)) => { - if compare_tuple_vec(a, b) { - Some(Ordering::Equal) - } else { - None - } - } - _ => None, - } - } - - pub fn is_error(&self) -> bool { matches!(self, Failure(_)) } -} - -fn concat_args(args: &Vec) -> Vec> { - let mut result = vec![]; - - for arg in args { - match arg { - Value(value) => result.push(Cow::Borrowed(value.as_str())), - Number(value) => result.push(Cow::Owned(format_number(*value))), - Named(pairs) => { - for (i, (key, value)) in pairs.iter().enumerate() { - result.push(Cow::Borrowed(key.as_str())); - result.push(Cow::Borrowed(": ")); - result.push(Cow::Borrowed(value.as_str())); - if i < pairs.len() - 1 { - result.push(Cow::Borrowed(", ")); - } - } - } - Undefined | AnyValue | Failure(_) => {} - } - } - - result -} - -fn eval_regex(source: &str, re_pattern: &Regex, match_as_ascii: bool) -> EvalResult { - let source = if match_as_ascii { deunicode_string(source) } else { Cow::Borrowed(source) }; - let mut values = vec![]; - - let Some(caps) = re_pattern.captures(&source) else { - return Undefined; - }; - - for i in 1..caps.len() { - if let Some(value) = caps.get(i) { - values.push((i.to_string(), value.as_str().to_string())); - } - } - - for name in re_pattern.capture_names().flatten() { - if let Some(value) = caps.name(name) { - values.push((name.to_string(), value.as_str().to_string())); - } - } - - match values.len() { - 0 => Undefined, - 1 => values.pop().map_or(Undefined, |(_, value)| Value(value)), - _ => Named(values), - } -} - -fn matches_text(value: &EvalResult, expected: &str) -> bool { - match value { - AnyValue => true, - Value(actual) => actual == expected, - Number(actual) => match_number(*actual, expected), - Undefined | Named(_) | Failure(_) => false, - } -} - macro_rules! extract_evaluated_arg_value { ($evaluated_args:expr, $index:expr) => {{ if $index >= $evaluated_args.len() { @@ -1167,990 +58,22 @@ macro_rules! extract_evaluated_arg_value { }}; } -impl Expression { - pub fn eval(&self, ctx: &mut MapperContext, accessor: &mut ValueAccessor) -> EvalResult { - match self { - Expression::NullValue => Undefined, - Expression::Identifier(name) => { - if ctx.has_var(name) { - ctx.get_var(name).clone() - } else { - Failure(format!("Variable with name {name} not found.")) - } - } - Expression::FieldAccess(field) => { - if let Some(val) = accessor.get(field) { - Value(val.to_string()) - } else { - Undefined - } - } - Expression::VarAccess(name, field) => match ctx.variables.get(name) { - None => Failure(format!("Variable with name {name} not found.")), - Some(value) => match value { - Undefined => Undefined, - Value(value) if field == "1" => Value(value.clone()), - Number(_) | Value(_) => Failure(format!("Variable with name {name} has no fields.")), - Named(values) => { - for (key, val) in values { - if key == field { - return Value(val.clone()); - } - } - Failure(format!("Variable with name {name} has no field {field}.")) - } - AnyValue | Failure(_) => value.clone(), - }, - }, - Expression::StringLiteral(s) => Value(s.clone()), - Expression::NumberLiteral(num) => Number(*num), - Expression::RegexExpr { field, pattern: _pattern, re_pattern } => match field { - RegexSource::Identifier(ident) => match ctx.get_var(ident) { - Value(text) => eval_regex(text, re_pattern, accessor.match_as_ascii), - _ => Undefined, - }, - RegexSource::Field(field) => accessor - .get(field) - .map_or(Undefined, |value| eval_regex(&value, re_pattern, accessor.match_as_ascii)), - }, - Expression::Assignment { target, expr } => { - let val = expr.eval(ctx, accessor); - match target { - AssignmentTarget::Identifier(name) => { - ctx.set_var(name, val); - Undefined - } - AssignmentTarget::Field(name) => { - match val { - Value(content) => { - accessor.set(name, content.as_str()); - } - Number(num) => { - accessor.set(name, format_number(num).as_str()); - } - Named(pairs) => { - let mut result = String::with_capacity(128); - for (i, (key, value)) in pairs.iter().enumerate() { - result.push_str(key); - result.push_str(": "); - result.push_str(value); - if i < pairs.len() - 1 { - result.push_str(", "); - } - } - accessor.set(name, &result); - } - Undefined | AnyValue => {} - Failure(err) => { - return Failure(format!("Failed to set field {name} value: {err}")); - } - } - Undefined - } - } - } - Expression::FunctionCall { name, args } => { - let mut evaluated_args: Vec = args.iter().map(|a| a.eval(ctx, accessor)).collect(); - for arg in &evaluated_args { - if arg.is_error() { - return Failure(format!( - "Function '{name:?}' failed: {}", - if let Failure(msg) = arg { msg } else { "Unknown error" } - )); - } - } - evaluated_args.retain(|er| !matches!(er, Undefined | Failure(_) | AnyValue)); - if evaluated_args.is_empty() { - if matches!(name, BuiltInFunction::Print) { - trace!("[MapperScript] undefined"); - } - Undefined - } else { - match name { - BuiltInFunction::Concat => Value(concat_args(&evaluated_args).join("")), - BuiltInFunction::Uppercase => Value(concat_args(&evaluated_args).join(" ").to_uppercase()), - BuiltInFunction::Trim => Value( - concat_args(&evaluated_args) - .iter() - .map(|s| s.trim()) - .collect::>() - .join(" ") - .trim() - .to_string(), - ), - BuiltInFunction::Lowercase => Value(concat_args(&evaluated_args).join(" ").to_lowercase()), - BuiltInFunction::Capitalize => Value( - concat_args(&evaluated_args) - .iter() - .map(Capitalize::capitalize) - .collect::>() - .join(" "), - ), - BuiltInFunction::Split => { - let string = extract_evaluated_arg_value!(evaluated_args, 0); - let pattern = extract_evaluated_arg_value!(evaluated_args, 1); - - if let (Some(text), Some(pat)) = (string, pattern) { - match crate::model::REGEX_CACHE.get_or_compile(pat) { - Ok(re) => Named( - re.split(text) - .enumerate() - .map(|(i, s)| (i.to_string(), s.trim().to_string())) - .collect(), - ), - Err(e) => Failure(format!("Invalid regex pattern '{pat}': {e}")), - } - } else { - Undefined - } - } - BuiltInFunction::Print => { - trace!("[MapperScript] {}", concat_args(&evaluated_args).join("")); - Undefined - } - BuiltInFunction::ToNumber => { - let evaluated_arg = &evaluated_args[0]; - match evaluated_arg { - Value(value) => to_number(value), - _ => evaluated_arg.clone(), - } - } - BuiltInFunction::First => match evaluated_args.first() { - Some(value) => match value { - Named(values) => match values.first() { - None => Undefined, - Some((_key, val)) => Value(val.clone()), - }, - _ => value.clone(), - }, - None => Undefined, - }, - BuiltInFunction::Template => { - let value = extract_evaluated_arg_value!(evaluated_args, 0); - if let Some(val) = value { - match ctx.get_template(val) { - Some(v) => Value(v.to_string()), - None => Undefined, - } - } else { - Undefined - } - } - BuiltInFunction::Replace => { - let value = extract_evaluated_arg_value!(evaluated_args, 0); - let pattern = extract_evaluated_arg_value!(evaluated_args, 1); - let substring = extract_evaluated_arg_value!(evaluated_args, 2); - - if let (Some(text), Some(pat), Some(subst)) = (value, pattern, substring) { - Value(text.replace(pat, subst)) - } else { - evaluated_args[0].clone() - } - } - BuiltInFunction::Pad => { - let value = match &evaluated_args[0] { - Number(value) => Some(value.to_string()), - Value(value) => Some(value.clone()), - Named(values) => values.first().map(|(_key, val)| val.clone()), - _ => None, - }; - let width = match &evaluated_args[1] { - Number(value) => { - if value.is_nan() || value.is_infinite() { - 0 - } else { - value.abs().min(usize::MAX as f64) as usize - } - } - Value(value) => value.parse::().ok().unwrap_or(0), - Named(values) => { - values.first().and_then(|(_key, val)| val.parse::().ok()).unwrap_or(0) - } - _ => 0, - }; - - let fill = match &evaluated_args[2] { - Number(value) => Some(value.to_string()), - Value(value) => Some(value.clone()), - Named(values) => values.first().map(|(_key, val)| val.clone()), - _ => None, - }; - - let align = extract_evaluated_arg_value!(evaluated_args, 3); // "<", ">", "^" - - if let Some(text) = value { - let fill_char = fill.and_then(|s| s.chars().next()).unwrap_or(' '); - - let padded = if width <= text.len() { - text.clone() - } else { - let pad = width - text.len(); - if let Some(al) = align { - match al.as_str() { - "^" => { - let left = pad / 2; - let right = pad - left; - format!( - "{}{}{}", - fill_char.to_string().repeat(left), - text, - fill_char.to_string().repeat(right) - ) - } - "<" => format!("{}{}", text, fill_char.to_string().repeat(pad)), - _ => format!("{}{}", fill_char.to_string().repeat(pad), text), - } - } else { - format!("{}{}", fill_char.to_string().repeat(pad), text) - } - }; - - Value(padded) - } else { - Undefined - } - } - BuiltInFunction::Format => { - let fmt_pattern = extract_evaluated_arg_value!(evaluated_args, 0); - - if let Some(fmt_str) = fmt_pattern { - let args = evaluated_args - .iter() - .skip(1) - .map(|value| match value { - Value(value) => Some(value.clone()), - Number(value) => Some(format_number(*value)), - Named(values) => values.first().map(|(_, value)| value.clone()), - Undefined | AnyValue | Failure(_) => None, - }) - .collect::>(); - - let mut formatted = String::new(); - let mut arg_iter = args.iter(); - let mut chars = fmt_str.chars().peekable(); - - while let Some(ch) = chars.next() { - if ch == '{' && chars.peek() == Some(&'}') { - chars.next(); - if let Some(Some(arg)) = arg_iter.next() { - formatted.push_str(arg); - } else { - formatted.push_str("{}"); - } - } else { - formatted.push(ch); - } - } - - Value(formatted) - } else { - Undefined - } - } - BuiltInFunction::AddFavourite => { - let group_name = extract_evaluated_arg_value!(evaluated_args, 0); - if let Some(group) = group_name { - let item_type = accessor.pli.header.item_type; - if item_type != PlaylistItemType::Series && item_type != PlaylistItemType::LocalSeries { - let mut pli = accessor.pli.clone(); - pli.header.group = group.intern(); - pli.header.uuid = crate::utils::create_alias_uuid(&accessor.pli.header.uuid, group); - accessor.virtual_items.push((group.clone(), pli)); - } - } - Undefined - } - } - } - } - Expression::MatchBlock(cases) => { - for match_case in cases { - let mut case_keys = vec![]; - for case_key in &match_case.keys { - match case_key { - MatchCaseKey::Identifier(ident) => { - if !ctx.has_var(ident) { - return Failure(format!( - "Match case invalid! Variable with name {ident} not found." - )); - } - case_keys.push(ctx.get_var(ident).clone()); - } - MatchCaseKey::AnyMatch => case_keys.push(AnyValue), - } - } - - let mut match_count = 0; - let case_keys_len = case_keys.len(); - for case_key in case_keys { - match case_key { - Value(_) | Number(_) | Named(_) | AnyValue => match_count += 1, - Undefined | Failure(_) => {} - } - } - if match_count == case_keys_len { - return match_case.expression.eval(ctx, accessor); - } - } - Undefined - } - Expression::MapBlock { key, cases } => { - let key_value = match key { - MapKey::Identifier(ident) => { - if !ctx.has_var(ident) { - return Failure(format!("Map expression invalid! Variable with name {ident} not found.")); - } - let mut val = ctx.get_var(ident).clone(); - if accessor.match_as_ascii { - if let Value(text) = val { - val = Value(deunicode_string(&text).into_owned()); - } - } - val - } - MapKey::FieldAccess(field) => { - if let Some(mut val) = accessor.get(field) { - if accessor.match_as_ascii { - val = deunicode_string(&val).into_owned().into(); - } - Value(val.to_string()) - } else { - Undefined - } - } - MapKey::VarAccess(name, field) => match ctx.variables.get(name) { - None => Failure(format!("Variable with name {name} not found.")), - Some(value) => match value { - Undefined => Undefined, - Value(value) if field == "1" => { - let value = if accessor.match_as_ascii { - deunicode_string(value).into_owned() - } else { - value.clone() - }; - Value(value) - } - Number(_) | Value(_) => Failure(format!("Variable with name {name} has no fields.")), - Named(values) => { - for (key, val) in values { - if key == field { - let mut res_val = val.clone(); - if accessor.match_as_ascii { - res_val = deunicode_string(&res_val).into_owned(); - } - return Value(res_val); - } - } - Failure(format!("Variable with name {name} has no field {field}.")) - } - AnyValue | Failure(_) => value.clone(), - }, - }, - }; - - for map_case in cases { - let mut matches = false; - for key in &map_case.keys { - if match key { - MapCaseKey::Text(value) => matches_text(&key_value, value), - MapCaseKey::AnyMatch => true, - MapCaseKey::RangeFrom(num) => match key_value.compare(&Number(*num)) { - None => false, - Some(ord) => match ord { - Ordering::Less => false, - Ordering::Equal | Ordering::Greater => true, - }, - }, - MapCaseKey::RangeTo(num) => match key_value.compare(&Number(*num)) { - None => false, - Some(ord) => match ord { - Ordering::Equal | Ordering::Less => true, - Ordering::Greater => false, - }, - }, - MapCaseKey::RangeFull(from, to) => match key_value.compare(&Number(*from)) { - None => false, - Some(ord) => match ord { - Ordering::Less => false, - Ordering::Equal | Ordering::Greater => match key_value.compare(&Number(*to)) { - None => false, - Some(ord) => match ord { - Ordering::Equal | Ordering::Less => true, - Ordering::Greater => false, - }, - }, - }, - }, - MapCaseKey::RangeEq(num) => match key_value.compare(&Number(*num)) { - None => false, - Some(ord) => match ord { - Ordering::Equal => true, - Ordering::Less | Ordering::Greater => false, - }, - }, - } { - matches = true; - break; - } - } - - if matches { - return map_case.expression.eval(ctx, accessor); - } - } - Undefined - } - Expression::ForEachBlock { key, expr } => { - let key_value = match key { - ForEachKey::Identifier(ident) => { - if !ctx.has_var(ident) { - return Failure(format!( - "For each expression invalid! Variable with name {ident} not found." - )); - } - let v = ctx.get_var(ident); - match v { - Named(_) | AnyValue | Failure(_) => v.clone(), - Undefined => Undefined, - _ => Failure(format!("Variable with name {ident} must be a Named list.")), - } - } - ForEachKey::VarAccess(name, field) => match ctx.variables.get(name) { - None => Failure(format!("Variable with name {name} not found.")), - Some(value) => match value { - AnyValue | Failure(_) => value.clone(), - Named(values) => { - let filtered: Vec<(String, String)> = values - .iter() - .filter(|(k, _)| k == field) - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - if filtered.is_empty() { - Undefined - } else { - Named(filtered) - } - } - Undefined => Undefined, - _ => Failure(format!("Variable with name {name} must be a Named list.")), - }, - }, - }; - - let values = match key_value { - Named(key_value) => key_value, - Failure(_) => return key_value, - _ => Vec::new(), - }; - let previous_key_var = expr.key_var.as_ref().and_then(|key_var| ctx.variables.get(key_var).cloned()); - let previous_value_var = - expr.value_var.as_ref().and_then(|value_var| ctx.variables.get(value_var).cloned()); - for (k, val) in values { - if let Some(key_var) = &expr.key_var { - ctx.set_var(key_var, EvalResult::Value(k)); - } - if let Some(value_var) = &expr.value_var { - ctx.set_var(value_var, EvalResult::Value(val)); - } - expr.expression.eval(ctx, accessor); - } - if let Some(key_var) = &expr.key_var { - if let Some(previous) = previous_key_var { - ctx.set_var(key_var, previous); - } else { - ctx.variables.remove(key_var); - } - } - if let Some(value_var) = &expr.value_var { - if let Some(previous) = previous_value_var { - ctx.set_var(value_var, previous); - } else { - ctx.variables.remove(value_var); - } - } - Undefined - } - Expression::Block(expressions) => { - let mut result = Undefined; - for expr in expressions { - result = expr.eval(ctx, accessor); - } - result - } - } - } -} - #[cfg(test)] -mod tests { - use super::*; - use crate::model::{ - PlaylistItem, PlaylistItemHeader, SeriesStreamProperties, StreamProperties, VideoStreamDetailProperties, - VideoStreamProperties, - }; +mod tests; - #[test] - fn test_mapper_dsl_eval() { - let dsl = r#" - coast = @Caption ~ "(?i)\b(EAST|WEST)\b" - quality = @Caption ~ "(?i)\b([FUSL]?HD|SD|4K|1080p|720p|3840p)\b" - quality = uppercase(quality) - quality = map quality { - "SHD" => "SD", - "LHD" => "HD", - "720p" => "HD", - "1080p" => "FHD", - "4K" => "UHD", - "3840p" => "UHD", - _ => quality, - } - coast_quality = match { - (coast, quality) => concat(capitalize(coast), " ", uppercase(quality)), - coast => concat(capitalize(coast), " HD"), - quality => concat("East ", uppercase(quality)), - } - @Caption = concat("US: TNT", " ", coast_quality) - @Group = "United States - Entertainment" - "#; - - let mapper = MapperScript::parse(dsl, None).expect("Parsing failed"); - println!("Program: {mapper:?}"); - let mut channels: Vec = vec![ - ("D", "HD"), - ("A", "FHD"), - ("Z", ""), - ("K", "HD"), - ("B", "HD"), - ("A", "HD"), - ("K", "SHD"), - ("C", "LHD"), - ("L", "FHD"), - ("R", "UHD"), - ("T", "SD"), - ("A", "FHD"), - ] - .into_iter() - .map(|(name, quality)| PlaylistItem { - header: PlaylistItemHeader { title: format!("Chanel {name} [{quality}]").into(), ..Default::default() }, - }) - .collect::>(); - - for pli in &mut channels { - let mut accessor = - ValueAccessor { pli, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; - mapper.eval(&mut accessor, None); - println!("Result: {pli:?}"); - } - - // ctx.fields.insert("Caption".to_string(), "US: TNT East LHD bubble".to_string()); - // - // for stmt in &program.statements { - // //let res = stmt.eval(&mut ctx); - // println!("Statement Result: {:?}", res); - // } - // - // println!("Result variable: {:?}", ctx.variables.get("result")); - // assert_eq!(ctx.variables.get("result").unwrap(), "US: TNT East HD"); - } - - #[test] - fn test_complex() { - let script = r#" - print("LOCAL") - coast = @Caption ~ "!COAST!" - quality = uppercase(@Caption ~ "!QUALITY!") - - quality = map quality { - "SHD" | "SD" => "SD", - "LHD" | "720P" | "HD" => "HD", - "FHD" | "1080P" => "FHD", - "UHD" | "4K" | "3840P" => "UHD", - _ => quality, - } - - coast_quality = match { - (coast, quality) => concat(capitalize(coast), " ", uppercase(quality)), - quality => uppercase(quality), - _ => "HD", - } - - network = uppercase(first(@Caption ~ "(?i)\b(CBS|NBC|FOX|ABC|PBS|CW|UNIVISION)\b")) - station = map network { - "CBS" => @Caption ~ "(?i)\b(WINK|WFOR)\b", - "NBC" => @Caption ~ "(?i)\b(WBBH|WTVJ)\b", - "FOX" => @Caption ~ "(?i)\b(WFTX|WSVM)\b", - "ABC" => @Caption ~ "(?i)\b(WZVN|WPLG)\b", - "PBS" => @Caption ~ "(?i)\b(WGCU|WPBT)\b", - "CW" => @Caption ~ "(?i)\b(WINK|WSFL)\b", - "UNIVISION" => @Caption ~ "(?i)\b(WUVF|WLTV)\b", - _ => null, - } - - match { - station => { - station = uppercase(station) - @Caption = map station { - "WINK" => concat("!US_CBS_FM_PREFIX!", " ", coast_quality), - "WBBH" => concat("!US_NBC_FM_PREFIX!", " ", coast_quality), - "WFTX" => concat("!US_FOX_FM_PREFIX!", " ", coast_quality), - "WZVN" => concat("!US_ABC_FM_PREFIX!", " ", coast_quality), - "WGCU" => concat("!US_PBS_FM_PREFIX!", " ", coast_quality), - "WUVF" => concat("!US_UNIVISION_FM_PREFIX!", " ", coast_quality), - - "WFOR" => concat("!US_CBS_MIA_PREFIX!", " ", coast_quality), - "WTVJ" => concat("!US_NBC_MIA_PREFIX!", " ", coast_quality), - "WSVM" => concat("!US_FOX_MIA_PREFIX!", " ", coast_quality), - "WPLG" => concat("!US_ABC_MIA_PREFIX!", " ", coast_quality), - "WPBT" => concat("!US_PBS_MIA_PREFIX!", " ", coast_quality), - "WSFL" => concat("!US_CW_MIA_PREFIX!", " ", coast_quality), - "WLTV" => concat("!US_UNIVISION_MIA_PREFIX!", " ", coast_quality), - - _ => concat(network, " ", station, " ", coast_quality), - } - - @Group = concat("🇺🇸 > USA - ", network, " Locals") - } - } - "#; - let mapper = MapperScript::parse(script, None).expect("Parsing failed"); - println!("Program: {mapper:?}"); - } - - #[test] - fn test_mapper_format() { - let dsl = r#" - @Name = pad(1000, 10, 0); - @Title = format("Channel {} is {}", 12, "live"); - "#; - - let mapper = MapperScript::parse(dsl, None).expect("Parsing failed"); - let mut channels: Vec = vec![("D", "HD")] - .into_iter() - .map(|(name, quality)| PlaylistItem { - header: PlaylistItemHeader { title: format!("Chanel {name} [{quality}]").into(), ..Default::default() }, - }) - .collect::>(); - - for pli in &mut channels { - let mut accessor = - ValueAccessor { pli, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; - mapper.eval(&mut accessor, None); - assert_eq!(accessor.pli.header.title.as_ref(), "Channel 12 is live"); - } - } - - #[test] - fn regex_identifier_ascii_normalization_preserves_map_selection() { - let dsl = r#" - source = @Name - capture = source ~ "(Cinema)" - @Group = map capture { - "Cinema" => "matched", - _ => "missed", - } - "#; - let mapper = MapperScript::parse(dsl, None).expect("mapper should parse"); - let mut channel = - PlaylistItem { header: PlaylistItemHeader { name: "Cinéma".intern(), ..Default::default() } }; - let mut accessor = - ValueAccessor { pli: &mut channel, virtual_items: vec![], match_as_ascii: true, changed_fields: vec![] }; - - mapper.eval(&mut accessor, None); - - assert_eq!(accessor.pli.header.group.as_ref(), "matched"); - } - - #[test] - fn regex_evaluation_keeps_all_groups_from_only_the_first_match() -> Result<(), Box> { - let regex = Regex::new(r"(?P[A-Z])([A-Z])(?P[A-Z])([A-Z])(?P[A-Z])([A-Z])")?; - - let Named(values) = eval_regex("ABCDEFUVWXYZ", ®ex, false) else { - return Err("regex captures should produce a named result".into()); - }; - - assert_eq!( - values, - [ - ("1", "A"), - ("2", "B"), - ("3", "C"), - ("4", "D"), - ("5", "E"), - ("6", "F"), - ("one", "A"), - ("three", "C"), - ("five", "E"), - ] - .map(|(key, value)| (key.to_string(), value.to_string())) - ); - Ok(()) - } - - #[test] - fn regex_capture_groups_are_available_by_index_and_name() -> Result<(), Box> { - let dsl = r#" - captures = @Name ~ "(?P[A-Z])([A-Z])(?P[A-Z])([A-Z])(?P[A-Z])([A-Z])" - @Title = concat(captures.1, captures.2, captures.3, captures.4, captures.5, captures.6) - @Group = concat(captures.one, captures.three, captures.five) - "#; - let mapper = MapperScript::parse(dsl, None)?; - let mut channel = PlaylistItem { header: PlaylistItemHeader { name: "ABCDEF".intern(), ..Default::default() } }; - let mut accessor = - ValueAccessor { pli: &mut channel, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; - - mapper.eval(&mut accessor, None); - - assert_eq!(accessor.pli.header.title.as_ref(), "ABCDEF"); - assert_eq!(accessor.pli.header.group.as_ref(), "ACE"); - Ok(()) - } - - #[test] - fn single_regex_capture_is_available_as_scalar_and_by_index() -> Result<(), Box> { - let dsl = r#" - capture = @Name ~ "([A-Z]+)" - @Group = map capture.1 { - "ABC" => concat(capture, ":", capture.1), - _ => "missed", - } - "#; - let mapper = MapperScript::parse(dsl, None)?; - let mut channel = PlaylistItem { header: PlaylistItemHeader { name: "ABC".intern(), ..Default::default() } }; - let mut accessor = - ValueAccessor { pli: &mut channel, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; - - mapper.eval(&mut accessor, None); - - assert_eq!(accessor.pli.header.group.as_ref(), "ABC:ABC"); - Ok(()) - } - - #[test] - fn read_only_fields_work_in_direct_regex_and_map_expressions() -> Result<(), Box> { - let dsl = r#" - input_match = @Input ~ "(source)" - type_match = @Type ~ "(live)" - @Title = concat(@Input, ":", @Type, ":", input_match, ":", type_match) - @Group = map @Input { - "source" => "input-map", - _ => "missed", - } - @Name = map @Type { - "live" => concat(@Group, ":type-map"), - _ => "missed", - } - "#; - let mapper = MapperScript::parse(dsl, None)?; - let mut channel = PlaylistItem { - header: PlaylistItemHeader { - input_name: "source".intern(), - item_type: PlaylistItemType::Live, - ..Default::default() - }, - }; - let mut accessor = - ValueAccessor { pli: &mut channel, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; - - mapper.eval(&mut accessor, None); - - assert_eq!(accessor.pli.header.title.as_ref(), "source:live:source:live"); - assert_eq!(accessor.pli.header.group.as_ref(), "input-map"); - assert_eq!(accessor.pli.header.name.as_ref(), "input-map:type-map"); - Ok(()) - } - - #[test] - fn read_only_field_assignments_are_rejected() { - for script in [r#"@Input = "changed""#, r#"@Type = "movie""#] { - assert!(MapperScript::parse(script, None).is_err()); - } - assert!(MapperScript::parse(r#"@Group = "changed""#, None).is_ok()); - } - - #[test] - fn template_lookup_keeps_last_duplicate_value() { - let dsl = r#"@Group = template("label")"#; - let templates = vec![ - PatternTemplate { - name: "label".to_string(), - value: TemplateValue::Single("first".to_string()), - placeholder: "!label!".to_string(), - }, - PatternTemplate { - name: "label".to_string(), - value: TemplateValue::Single("second".to_string()), - placeholder: "!label!".to_string(), - }, - ]; - let mapper = MapperScript::parse(dsl, Some(&templates)).expect("mapper should parse"); - let mut channel = PlaylistItem { header: PlaylistItemHeader::default() }; - let mut accessor = - ValueAccessor { pli: &mut channel, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; - - mapper.eval(&mut accessor, Some(&templates)); - - assert_eq!(accessor.pli.header.group.as_ref(), "second"); - } - - #[test] - fn test_mapper_add_favourite() { - use crate::model::PlaylistItemType; - let dsl = r#" - add_favourite("My Favs"); - "#; - - let mapper = MapperScript::parse(dsl, None).expect("Parsing failed"); - - // Test with Video (should work) - let mut video = PlaylistItem { - header: PlaylistItemHeader { - name: "Movie 1".to_string().into(), - item_type: PlaylistItemType::Video, - ..Default::default() - }, - }; - let mut accessor = - ValueAccessor { pli: &mut video, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; - mapper.eval(&mut accessor, None); - assert_eq!(accessor.virtual_items.len(), 1); - assert_eq!(&*accessor.virtual_items[0].1.header.group, "My Favs"); - - // Test with SeriesInfo (should work) - let mut series_info = PlaylistItem { - header: PlaylistItemHeader { - name: "Series 1".to_string().into(), - item_type: PlaylistItemType::SeriesInfo, - ..Default::default() - }, - }; - let mut accessor = ValueAccessor { - pli: &mut series_info, - virtual_items: vec![], - match_as_ascii: false, - changed_fields: vec![], - }; - mapper.eval(&mut accessor, None); - assert_eq!(accessor.virtual_items.len(), 1); - - // Test with Series episode (should NOT work) - let mut episode = PlaylistItem { - header: PlaylistItemHeader { - name: "Episode 1".to_string().into(), - item_type: PlaylistItemType::Series, - ..Default::default() - }, - }; - let mut accessor = - ValueAccessor { pli: &mut episode, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; - mapper.eval(&mut accessor, None); - assert_eq!(accessor.virtual_items.len(), 0); - } - - #[test] - fn test_mapper_split_loop() { - use crate::model::PlaylistItemType; - let dsl = r#" - genres = split(@Genre, ",") - print(genres) - genres.for_each((_, gen) => { - add_favourite(concat("Genre - ", gen)) - }) - "#; - - let mapper = MapperScript::parse(dsl, None).expect("Parsing failed"); - - // Test with Video (should work) - let mut video = PlaylistItem { - header: PlaylistItemHeader { - name: "Movie 1".to_string().into(), - item_type: PlaylistItemType::Video, - additional_properties: Some(StreamProperties::Video(Box::new(VideoStreamProperties { - details: Some(VideoStreamDetailProperties { - genre: Some("A, B, C".intern()), - ..VideoStreamDetailProperties::default() - }), - ..VideoStreamProperties::default() - }))), - ..Default::default() - }, - }; - let mut accessor = - ValueAccessor { pli: &mut video, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; - mapper.eval(&mut accessor, None); - assert_eq!(accessor.virtual_items.len(), 3); - assert_eq!(&*accessor.virtual_items[0].1.header.group, "Genre - A"); - assert_eq!(&*accessor.virtual_items[1].1.header.group, "Genre - B"); - assert_eq!(&*accessor.virtual_items[2].1.header.group, "Genre - C"); - - // Test with SeriesInfo (should work) - let mut series_info = PlaylistItem { - header: PlaylistItemHeader { - name: "Series 1".to_string().into(), - item_type: PlaylistItemType::SeriesInfo, - additional_properties: Some(StreamProperties::Series(Box::new(SeriesStreamProperties { - genre: Some("A, B, C".intern()), - ..SeriesStreamProperties::default() - }))), - ..Default::default() - }, - }; - let mut accessor = ValueAccessor { - pli: &mut series_info, - virtual_items: vec![], - match_as_ascii: false, - changed_fields: vec![], - }; - mapper.eval(&mut accessor, None); - assert_eq!(accessor.virtual_items.len(), 3); - } - - #[test] - fn for_each_restores_existing_loop_variables() { - let expressions = vec![ - Expression::Identifier("value".to_string()), - Expression::ForEachBlock { - key: ForEachKey::Identifier("items".to_string()), - expr: ForEachExpr { - key_var: Some("key".to_string()), - value_var: Some("value".to_string()), - expression: ExprId(0), - }, - }, - ]; - let mut ctx = MapperContext::new(&expressions, None); - ctx.set_var("items", Named(vec![("first".to_string(), "current".to_string())])); - ctx.set_var("value", Value("outer".to_string())); - let mut item = PlaylistItem { header: PlaylistItemHeader::default() }; - let mut accessor = - ValueAccessor { pli: &mut item, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; - - ExprId(1).eval(&mut ctx, &mut accessor); - - assert!(matches!(ctx.get_var("value"), Value(value) if value == "outer")); - assert!(matches!(ctx.get_var("key"), Undefined)); - } - - #[test] - fn eval_reports_changes_and_statement_failures() { - let mapper = MapperScript::parse( - r#" - @Group = "News" - capture = @Name ~ "(?PBBC)" - @Title = capture.missing - "#, - None, - ) - .expect("script should parse"); - let mut item = PlaylistItem { header: PlaylistItemHeader { name: "BBC One".intern(), ..Default::default() } }; - let mut accessor = - ValueAccessor { pli: &mut item, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; - - let outcome = mapper.eval(&mut accessor, None); - - assert_eq!(outcome.changed_fields, ["Group"]); - assert_eq!(outcome.emitted_items, 0); - assert_eq!(outcome.diagnostics.len(), 1); - assert_eq!(outcome.diagnostics[0].statement, 2); - assert!(outcome.diagnostics[0].message.contains("has no field missing")); - } -} +mod ast; +mod builtins; +mod context; +mod diagnostics; +mod eval; +mod parser; +pub use ast::{ + AssignmentTarget, ExprId, Expression, ForEachExpr, ForEachKey, MapCase, MapCaseKey, MapKey, MatchCase, + MatchCaseKey, RegexSource, Statement, +}; +pub use builtins::BuiltInFunction; +pub use diagnostics::{MappingDiagnostic, MappingOutcome}; +#[cfg(test)] +use eval::eval_regex; +pub use eval::EvalResult; +pub use parser::Rule; diff --git a/shared/src/foundation/mapper/ast.rs b/shared/src/foundation/mapper/ast.rs new file mode 100644 index 000000000..09fae9e8c --- /dev/null +++ b/shared/src/foundation/mapper/ast.rs @@ -0,0 +1,148 @@ +use super::{BuiltInFunction, EvalResult, MapperContext, ValueAccessor}; +use crate::foundation::mapper::EvalResult::Failure; +use log::debug; +use regex::Regex; +use std::{ops::Deref, sync::Arc}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ExprId(pub usize); + +impl Deref for ExprId { + type Target = usize; + + fn deref(&self) -> &Self::Target { &self.0 } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum MatchCaseKey { + Identifier(String), + AnyMatch, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct MatchCase { + pub keys: Vec, + pub expression: ExprId, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum MapCaseKey { + Text(String), + RangeFrom(f64), + RangeTo(f64), + RangeFull(f64, f64), + RangeEq(f64), + AnyMatch, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct MapCase { + pub keys: Vec, + pub expression: ExprId, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum MapKey { + Identifier(String), + FieldAccess(String), + VarAccess(String, String), +} + +#[derive(Debug, Clone, PartialEq)] +pub enum ForEachKey { + Identifier(String), + VarAccess(String, String), +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ForEachExpr { + pub key_var: Option, + pub value_var: Option, + pub expression: ExprId, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum RegexSource { + Identifier(String), + Field(String), +} + +#[derive(Debug, Clone)] +pub enum Expression { + Identifier(String), + StringLiteral(String), + NumberLiteral(f64), + FieldAccess(String), + VarAccess(String, String), + RegexExpr { field: RegexSource, pattern: String, re_pattern: Arc }, + FunctionCall { name: BuiltInFunction, args: Vec }, + Assignment { target: AssignmentTarget, expr: ExprId }, + MatchBlock(Vec), + MapBlock { key: MapKey, cases: Vec }, + ForEachBlock { key: ForEachKey, expr: ForEachExpr }, + NullValue, + Block(Vec), +} + +impl PartialEq for Expression { + fn eq(&self, other: &Self) -> bool { + use Expression::{ + Assignment, Block, FieldAccess, ForEachBlock, FunctionCall, Identifier, MapBlock, MatchBlock, NullValue, + NumberLiteral, RegexExpr, StringLiteral, VarAccess, + }; + match (self, other) { + (Identifier(a), Identifier(b)) => a == b, + (StringLiteral(a), StringLiteral(b)) => a == b, + (NumberLiteral(a), NumberLiteral(b)) => a == b, + (FieldAccess(a), FieldAccess(b)) => a == b, + (VarAccess(a1, b1), VarAccess(a2, b2)) => a1 == a2 && b1 == b2, + (RegexExpr { field: f1, pattern: p1, .. }, RegexExpr { field: f2, pattern: p2, .. }) => { + f1 == f2 && p1 == p2 + } + (FunctionCall { name: n1, args: a1 }, FunctionCall { name: n2, args: a2 }) => n1 == n2 && a1 == a2, + (Assignment { target: t1, expr: e1 }, Assignment { target: t2, expr: e2 }) => t1 == t2 && e1 == e2, + (MatchBlock(m1), MatchBlock(m2)) => m1 == m2, + (MapBlock { key: k1, cases: c1 }, MapBlock { key: k2, cases: c2 }) => k1 == k2 && c1 == c2, + (ForEachBlock { key: k1, expr: c1 }, ForEachBlock { key: k2, expr: c2 }) => k1 == k2 && c1 == c2, + (NullValue, NullValue) => true, + (Block(b1), Block(b2)) => b1 == b2, + _ => false, + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum AssignmentTarget { + Identifier(String), + Field(String), +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Statement { + Expression(ExprId), + Comment(String), +} + +impl ExprId { + pub fn eval(self, ctx: &mut MapperContext, accessor: &mut ValueAccessor) -> EvalResult { + let id = self.0; + ctx.eval_expr_by_id(id, accessor) + } +} + +impl Statement { + pub fn eval(&self, ctx: &mut MapperContext, setter: &mut ValueAccessor) -> Option { + match self { + Statement::Expression(expr_id) => { + let result = expr_id.eval(ctx, setter); + if let Failure(err) = &result { + debug!("{err}"); + Some(err.clone()) + } else { + None + } + } + Statement::Comment(_) => None, + } + } +} diff --git a/shared/src/foundation/mapper/builtins.rs b/shared/src/foundation/mapper/builtins.rs new file mode 100644 index 000000000..5c1b3dde4 --- /dev/null +++ b/shared/src/foundation/mapper/builtins.rs @@ -0,0 +1,65 @@ +use crate::error::TuliproxError; +use std::{fmt::Display, str::FromStr}; + +#[derive(Debug, Clone, PartialEq)] +pub enum BuiltInFunction { + Concat, + Uppercase, + Lowercase, + Capitalize, + Split, + Trim, + Print, + ToNumber, + First, + Template, + Replace, + Pad, + Format, + AddFavourite, +} + +impl FromStr for BuiltInFunction { + type Err = TuliproxError; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "concat" => Ok(Self::Concat), + "capitalize" => Ok(Self::Capitalize), + "lowercase" => Ok(Self::Lowercase), + "uppercase" => Ok(Self::Uppercase), + "split" => Ok(Self::Split), + "trim" => Ok(Self::Trim), + "print" => Ok(Self::Print), + "number" => Ok(Self::ToNumber), + "first" => Ok(Self::First), + "template" => Ok(Self::Template), + "replace" => Ok(Self::Replace), + "pad" => Ok(Self::Pad), + "format" => Ok(Self::Format), + "add_favourite" => Ok(Self::AddFavourite), + _ => Err(TuliproxError::Mapper(format!("Unknown function {s}"))), + } + } +} + +impl Display for BuiltInFunction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Concat => "concat", + Self::Capitalize => "capitalize", + Self::Lowercase => "lowercase", + Self::Uppercase => "uppercase", + Self::Split => "split", + Self::Trim => "trim", + Self::Print => "print", + Self::ToNumber => "number", + Self::First => "first", + Self::Template => "template", + Self::Replace => "replace", + Self::Pad => "pad", + Self::Format => "format", + Self::AddFavourite => "add_favourite", + }) + } +} diff --git a/shared/src/foundation/mapper/context.rs b/shared/src/foundation/mapper/context.rs new file mode 100644 index 000000000..656a4fe55 --- /dev/null +++ b/shared/src/foundation/mapper/context.rs @@ -0,0 +1,255 @@ +use super::{ + AssignmentTarget, BuiltInFunction, EvalResult, ExprId, Expression, ForEachExpr, ForEachKey, MapCase, MapCaseKey, + MapKey, MapperContext, MatchCase, MatchCaseKey, RegexSource, ValueAccessor, +}; +use crate::{ + error::TuliproxError, + foundation::mapper::EvalResult::Undefined, + model::{PatternTemplate, TemplateValue}, +}; +use std::collections::{HashMap, HashSet}; + +impl<'a> MapperContext<'a> { + pub(super) fn new(expressions: &'a [Expression], templates: Option<&'a [PatternTemplate]>) -> Self { + Self { expressions, variables: HashMap::new(), templates: templates.filter(|templates| !templates.is_empty()) } + } + + pub(super) fn get_template(&self, name: &str) -> Option<&str> { + self.templates?.iter().rev().find(|template| template.name == name).and_then(|template| match &template.value { + TemplateValue::Single(v) => Some(v.as_str()), + TemplateValue::Multi(_) => None, + }) + } + + pub(super) fn set_var(&mut self, name: &str, value: EvalResult) { + if let Some(current) = self.variables.get_mut(name) { + *current = value; + } else { + self.variables.insert(name.to_string(), value); + } + } + + pub(super) fn has_var(&self, name: &str) -> bool { self.variables.contains_key(name) } + + pub(super) fn get_var(&self, name: &str) -> &EvalResult { self.variables.get(name).unwrap_or(&Undefined) } + + pub(super) fn eval_expr_by_id(&mut self, id: usize, accessor: &mut ValueAccessor) -> EvalResult { + let Some(expr) = self.expressions.get(id) else { return Undefined }; + expr.eval(self, accessor) + } + + pub(super) fn validate_expr( + &mut self, + expr_id: ExprId, + identifiers: &mut HashSet, + ) -> Result<(), TuliproxError> { + let Some(expr) = self.expressions.get(expr_id.0) else { + return Err(TuliproxError::Mapper(format!("No matching expression found at index {}", expr_id.0))); + }; + match expr { + Expression::Identifier(ident) | Expression::VarAccess(ident, _) => { + if !identifiers.contains(ident.as_str()) { + return Err(TuliproxError::Mapper(format!("Identifier unknown {ident}, {expr:?}"))); + } + } + Expression::NullValue + | Expression::FieldAccess(_) + | Expression::StringLiteral(_) + | Expression::NumberLiteral(_) => {} + Expression::RegexExpr { field, pattern: _pattern, re_pattern: _re_pattern } => match field { + RegexSource::Identifier(ident) => { + if !identifiers.contains(ident.as_str()) { + return Err(TuliproxError::Mapper(format!("Regex identifier unknown {ident}, {expr:?}"))); + } + } + RegexSource::Field(_) => {} + }, + Expression::Assignment { target, expr } => { + match target { + AssignmentTarget::Identifier(ident) => { + identifiers.insert(ident.clone()); + } + AssignmentTarget::Field(_) => {} + } + self.validate_expr(*expr, identifiers)?; + } + Expression::FunctionCall { name, args } => { + if args.is_empty() { + return Err(TuliproxError::Mapper(format!("Function needs at least one argument {name:?}"))); + } + match name { + BuiltInFunction::ToNumber + | BuiltInFunction::Template + | BuiltInFunction::First + | BuiltInFunction::AddFavourite + if args.len() > 1 => + { + return Err(TuliproxError::Mapper(format!( + "Function accepts only one argument {:?}, {} given", + name, + args.len() + ))); + } + BuiltInFunction::Split if args.len() != 2 => { + return Err(TuliproxError::Mapper(format!( + "Function accepts two arguments {:?}, {} given", + name, + args.len() + ))); + } + BuiltInFunction::Replace if args.len() != 3 => { + return Err(TuliproxError::Mapper(format!( + "Function accepts three arguments {:?}, {} given", + name, + args.len() + ))); + } + BuiltInFunction::Pad if !(args.len() == 3 || args.len() == 4) => { + return Err(TuliproxError::Mapper(format!( + "Function accepts three or four arguments {:?}, {} given", + name, + args.len() + ))); + } + _ => {} + } + for expr_id in args { + self.validate_expr(*expr_id, identifiers)?; + } + } + Expression::MatchBlock(cases) => { + self.validate_match_block(identifiers, cases)?; + } + Expression::MapBlock { key, cases } => { + self.validate_map_block(identifiers, key, cases)?; + } + Expression::ForEachBlock { key, expr } => { + self.validate_for_each_block(identifiers, key, expr)?; + } + Expression::Block(expressions) => { + for expr_id in expressions { + self.validate_expr(*expr_id, identifiers)?; + } + } + } + Ok(()) + } + + pub(super) fn validate_match_block( + &mut self, + identifiers: &mut HashSet, + cases: &Vec, + ) -> Result<(), TuliproxError> { + let mut case_keys = HashSet::new(); + for match_case in cases { + let mut any_match_count = 0; + let mut identifier_key = String::with_capacity(56); + for identifier in &match_case.keys { + match identifier { + MatchCaseKey::Identifier(ident) => { + if !identifiers.contains(ident.as_str()) { + return Err(TuliproxError::Mapper(format!("Match case identifier unknown {ident}"))); + } + identifier_key.push_str(ident.as_str()); + identifier_key.push_str(", "); + } + MatchCaseKey::AnyMatch => { + any_match_count += 1; + if any_match_count > 1 { + return Err(TuliproxError::Mapper("Match case can only have one '_'".to_string())); + } + identifier_key.push_str("_, "); + } + } + } + if case_keys.contains(&identifier_key) { + return Err(TuliproxError::Mapper(format!("Duplicate case {identifier_key}"))); + } + case_keys.insert(identifier_key); + self.validate_expr(match_case.expression, identifiers)?; + } + Ok(()) + } + + pub(super) fn validate_map_block( + &mut self, + identifiers: &mut HashSet, + key: &MapKey, + cases: &Vec, + ) -> Result<(), TuliproxError> { + match key { + MapKey::Identifier(ident) | MapKey::VarAccess(ident, _) => { + if !identifiers.contains(ident.as_str()) { + return Err(TuliproxError::Mapper(format!("Map key identifier unknown {ident}"))); + } + } + MapKey::FieldAccess(_) => {} + } + let mut case_keys = HashSet::new(); + let mut any_match_count = 0; + for map_case in cases { + for key in &map_case.keys { + match key { + MapCaseKey::Text(value) => { + if case_keys.contains(value.as_str()) { + return Err(TuliproxError::Mapper(format!("Duplicate case {value}"))); + } + case_keys.insert(value.as_str()); + } + MapCaseKey::RangeEq(_) | MapCaseKey::RangeTo(_) | MapCaseKey::RangeFrom(_) => {} + MapCaseKey::RangeFull(from, to) => { + if *from > *to { + return Err(TuliproxError::Mapper(format!("Invalid range {from}..{to}"))); + } + } + MapCaseKey::AnyMatch => { + any_match_count += 1; + if any_match_count > 1 { + return Err(TuliproxError::Mapper("Map case can only have one '_'".to_string())); + } + } + } + } + self.validate_expr(map_case.expression, identifiers)?; + } + Ok(()) + } + + pub(super) fn validate_for_each_block( + &mut self, + identifiers: &mut HashSet, + key: &ForEachKey, + expr: &ForEachExpr, + ) -> Result<(), TuliproxError> { + match key { + ForEachKey::Identifier(ident) | ForEachKey::VarAccess(ident, _) => { + if !identifiers.contains(ident.as_str()) { + return Err(TuliproxError::Mapper(format!("For each key identifier unknown {ident}"))); + } + } + } + let mut local_identifiers = identifiers.clone(); + + if let Some(key_var) = &expr.key_var { + if local_identifiers.contains(key_var) { + return Err(TuliproxError::Mapper(format!( + "For each key variable shadows existing identifier {key_var}" + ))); + } + local_identifiers.insert(key_var.clone()); + } + + if let Some(value_var) = &expr.value_var { + if local_identifiers.contains(value_var) { + return Err(TuliproxError::Mapper(format!( + "For each value variable shadows existing identifier {value_var}" + ))); + } + local_identifiers.insert(value_var.clone()); + } + + self.validate_expr(expr.expression, &mut local_identifiers)?; + + Ok(()) + } +} diff --git a/shared/src/foundation/mapper/diagnostics.rs b/shared/src/foundation/mapper/diagnostics.rs new file mode 100644 index 000000000..8b2bc91f9 --- /dev/null +++ b/shared/src/foundation/mapper/diagnostics.rs @@ -0,0 +1,12 @@ +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct MappingDiagnostic { + pub statement: usize, + pub message: String, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct MappingOutcome { + pub changed_fields: Vec, + pub emitted_items: usize, + pub diagnostics: Vec, +} diff --git a/shared/src/foundation/mapper/eval.rs b/shared/src/foundation/mapper/eval.rs new file mode 100644 index 000000000..b86c7959c --- /dev/null +++ b/shared/src/foundation/mapper/eval.rs @@ -0,0 +1,706 @@ +use super::{ + AssignmentTarget, BuiltInFunction, Expression, ForEachKey, MapCaseKey, MapKey, MapperContext, MapperScript, + MappingDiagnostic, MappingOutcome, MatchCaseKey, RegexSource, ValueAccessor, +}; +use crate::{ + foundation::mapper::EvalResult::{AnyValue, Failure, Named, Number, Undefined, Value}, + model::{PatternTemplate, PlaylistItemType}, + utils::{deunicode_string, Capitalize, Internable}, +}; +use log::trace; +use regex::Regex; +use std::{borrow::Cow, cmp::Ordering, collections::HashMap}; + +impl MapperScript { + pub fn eval(&self, setter: &mut ValueAccessor, templates: Option<&[PatternTemplate]>) -> MappingOutcome { + let initial_change_count = setter.changed_fields.len(); + let initial_item_count = setter.virtual_items.len(); + let ctx = &mut MapperContext::new(&self.expressions, templates); + let diagnostics = self.eval_with_context(ctx, setter); + MappingOutcome { + changed_fields: setter.changed_fields[initial_change_count..].to_vec(), + emitted_items: setter.virtual_items.len().saturating_sub(initial_item_count), + diagnostics, + } + } + + pub(super) fn eval_with_context( + &self, + ctx: &mut MapperContext, + setter: &mut ValueAccessor, + ) -> Vec { + self.statements + .iter() + .enumerate() + .filter_map(|(statement, stmt)| { + stmt.eval(ctx, setter).map(|message| MappingDiagnostic { statement, message }) + }) + .collect() + } + + pub fn get_expr_by_id(&self, id: usize) -> Option<&Expression> { self.expressions.get(id) } +} + +#[derive(Debug, Clone)] +pub enum EvalResult { + Undefined, + Value(String), + Number(f64), + Named(Vec<(String, String)>), + AnyValue, + Failure(String), +} + +pub(super) fn to_number(value: &str) -> EvalResult { + match value.parse::() { + Ok(num) => Number(num), + Err(_) => Failure(format!("Invalid number: {value}")), + } +} + +fn compare_number(a: f64, b: f64) -> Ordering { + let epsilon = 1e-3; // = 0.001 + + if (a - b).abs() < epsilon { + Ordering::Equal + } else if a < b { + Ordering::Less + } else { + Ordering::Greater + } +} + +fn format_number(num: f64) -> String { + let epsilon = 1e-3; // = 0.001 + + if num.fract().abs() < epsilon { + format!("{}", num as i64) + } else { + format!("{num}") + } +} + +fn compare_tuple_vec(a: &[(String, String)], b: &[(String, String)]) -> bool { + fn to_map(vec: &[(String, String)]) -> HashMap<&str, &str> { + vec.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect() + } + + to_map(a) == to_map(b) +} + +fn match_number(num: f64, s: &str) -> bool { + if let Ok(val) = s.parse::() { + return compare_number(num, val) == Ordering::Equal; + } + false +} + +fn cmp_number(num: f64, s: &str) -> Option { + if let Ok(val) = s.parse::() { + return Some(compare_number(num, val)); + } + None +} + +impl EvalResult { + pub(super) fn compare(&self, other: &EvalResult) -> Option { + match (self, other) { + (AnyValue, _) | (_, AnyValue) => Some(Ordering::Equal), + (Value(a), Value(b)) => Some(a.cmp(b)), + (Number(a), Value(b)) => cmp_number(*a, b), + (Value(a), Number(b)) => match cmp_number(*b, a) { + None => None, + Some(ord) => match ord { + Ordering::Less => Some(Ordering::Greater), + Ordering::Equal => Some(Ordering::Equal), + Ordering::Greater => Some(Ordering::Less), + }, + }, + (Number(a), Number(b)) => Some(compare_number(*a, *b)), + (Named(a), Named(b)) => { + if compare_tuple_vec(a, b) { + Some(Ordering::Equal) + } else { + None + } + } + _ => None, + } + } + + pub fn is_error(&self) -> bool { matches!(self, Failure(_)) } +} + +fn concat_args(args: &Vec) -> Vec> { + let mut result = vec![]; + + for arg in args { + match arg { + Value(value) => result.push(Cow::Borrowed(value.as_str())), + Number(value) => result.push(Cow::Owned(format_number(*value))), + Named(pairs) => { + for (i, (key, value)) in pairs.iter().enumerate() { + result.push(Cow::Borrowed(key.as_str())); + result.push(Cow::Borrowed(": ")); + result.push(Cow::Borrowed(value.as_str())); + if i < pairs.len() - 1 { + result.push(Cow::Borrowed(", ")); + } + } + } + Undefined | AnyValue | Failure(_) => {} + } + } + + result +} + +pub(super) fn eval_regex(source: &str, re_pattern: &Regex, match_as_ascii: bool) -> EvalResult { + let source = if match_as_ascii { deunicode_string(source) } else { Cow::Borrowed(source) }; + let mut values = vec![]; + + let Some(caps) = re_pattern.captures(&source) else { + return Undefined; + }; + + for i in 1..caps.len() { + if let Some(value) = caps.get(i) { + values.push((i.to_string(), value.as_str().to_string())); + } + } + + for name in re_pattern.capture_names().flatten() { + if let Some(value) = caps.name(name) { + values.push((name.to_string(), value.as_str().to_string())); + } + } + + match values.len() { + 0 => Undefined, + 1 => values.pop().map_or(Undefined, |(_, value)| Value(value)), + _ => Named(values), + } +} + +fn matches_text(value: &EvalResult, expected: &str) -> bool { + match value { + AnyValue => true, + Value(actual) => actual == expected, + Number(actual) => match_number(*actual, expected), + Undefined | Named(_) | Failure(_) => false, + } +} + +impl Expression { + pub fn eval(&self, ctx: &mut MapperContext, accessor: &mut ValueAccessor) -> EvalResult { + match self { + Expression::NullValue => Undefined, + Expression::Identifier(name) => { + if ctx.has_var(name) { + ctx.get_var(name).clone() + } else { + Failure(format!("Variable with name {name} not found.")) + } + } + Expression::FieldAccess(field) => { + if let Some(val) = accessor.get(field) { + Value(val.to_string()) + } else { + Undefined + } + } + Expression::VarAccess(name, field) => match ctx.variables.get(name) { + None => Failure(format!("Variable with name {name} not found.")), + Some(value) => match value { + Undefined => Undefined, + Value(value) if field == "1" => Value(value.clone()), + Number(_) | Value(_) => Failure(format!("Variable with name {name} has no fields.")), + Named(values) => { + for (key, val) in values { + if key == field { + return Value(val.clone()); + } + } + Failure(format!("Variable with name {name} has no field {field}.")) + } + AnyValue | Failure(_) => value.clone(), + }, + }, + Expression::StringLiteral(s) => Value(s.clone()), + Expression::NumberLiteral(num) => Number(*num), + Expression::RegexExpr { field, pattern: _pattern, re_pattern } => match field { + RegexSource::Identifier(ident) => match ctx.get_var(ident) { + Value(text) => eval_regex(text, re_pattern, accessor.match_as_ascii), + _ => Undefined, + }, + RegexSource::Field(field) => accessor + .get(field) + .map_or(Undefined, |value| eval_regex(&value, re_pattern, accessor.match_as_ascii)), + }, + Expression::Assignment { target, expr } => { + let val = expr.eval(ctx, accessor); + match target { + AssignmentTarget::Identifier(name) => { + ctx.set_var(name, val); + Undefined + } + AssignmentTarget::Field(name) => { + match val { + Value(content) => { + accessor.set(name, content.as_str()); + } + Number(num) => { + accessor.set(name, format_number(num).as_str()); + } + Named(pairs) => { + let mut result = String::with_capacity(128); + for (i, (key, value)) in pairs.iter().enumerate() { + result.push_str(key); + result.push_str(": "); + result.push_str(value); + if i < pairs.len() - 1 { + result.push_str(", "); + } + } + accessor.set(name, &result); + } + Undefined | AnyValue => {} + Failure(err) => { + return Failure(format!("Failed to set field {name} value: {err}")); + } + } + Undefined + } + } + } + Expression::FunctionCall { name, args } => { + let mut evaluated_args: Vec = args.iter().map(|a| a.eval(ctx, accessor)).collect(); + for arg in &evaluated_args { + if arg.is_error() { + return Failure(format!( + "Function '{name:?}' failed: {}", + if let Failure(msg) = arg { msg } else { "Unknown error" } + )); + } + } + evaluated_args.retain(|er| !matches!(er, Undefined | Failure(_) | AnyValue)); + if evaluated_args.is_empty() { + if matches!(name, BuiltInFunction::Print) { + trace!("[MapperScript] undefined"); + } + Undefined + } else { + match name { + BuiltInFunction::Concat => Value(concat_args(&evaluated_args).join("")), + BuiltInFunction::Uppercase => Value(concat_args(&evaluated_args).join(" ").to_uppercase()), + BuiltInFunction::Trim => Value( + concat_args(&evaluated_args) + .iter() + .map(|s| s.trim()) + .collect::>() + .join(" ") + .trim() + .to_string(), + ), + BuiltInFunction::Lowercase => Value(concat_args(&evaluated_args).join(" ").to_lowercase()), + BuiltInFunction::Capitalize => Value( + concat_args(&evaluated_args) + .iter() + .map(Capitalize::capitalize) + .collect::>() + .join(" "), + ), + BuiltInFunction::Split => { + let string = extract_evaluated_arg_value!(evaluated_args, 0); + let pattern = extract_evaluated_arg_value!(evaluated_args, 1); + + if let (Some(text), Some(pat)) = (string, pattern) { + match crate::model::REGEX_CACHE.get_or_compile(pat) { + Ok(re) => Named( + re.split(text) + .enumerate() + .map(|(i, s)| (i.to_string(), s.trim().to_string())) + .collect(), + ), + Err(e) => Failure(format!("Invalid regex pattern '{pat}': {e}")), + } + } else { + Undefined + } + } + BuiltInFunction::Print => { + trace!("[MapperScript] {}", concat_args(&evaluated_args).join("")); + Undefined + } + BuiltInFunction::ToNumber => { + let evaluated_arg = &evaluated_args[0]; + match evaluated_arg { + Value(value) => to_number(value), + _ => evaluated_arg.clone(), + } + } + BuiltInFunction::First => match evaluated_args.first() { + Some(value) => match value { + Named(values) => match values.first() { + None => Undefined, + Some((_key, val)) => Value(val.clone()), + }, + _ => value.clone(), + }, + None => Undefined, + }, + BuiltInFunction::Template => { + let value = extract_evaluated_arg_value!(evaluated_args, 0); + if let Some(val) = value { + match ctx.get_template(val) { + Some(v) => Value(v.to_string()), + None => Undefined, + } + } else { + Undefined + } + } + BuiltInFunction::Replace => { + let value = extract_evaluated_arg_value!(evaluated_args, 0); + let pattern = extract_evaluated_arg_value!(evaluated_args, 1); + let substring = extract_evaluated_arg_value!(evaluated_args, 2); + + if let (Some(text), Some(pat), Some(subst)) = (value, pattern, substring) { + Value(text.replace(pat, subst)) + } else { + evaluated_args[0].clone() + } + } + BuiltInFunction::Pad => { + let value = match &evaluated_args[0] { + Number(value) => Some(value.to_string()), + Value(value) => Some(value.clone()), + Named(values) => values.first().map(|(_key, val)| val.clone()), + _ => None, + }; + let width = match &evaluated_args[1] { + Number(value) => { + if value.is_nan() || value.is_infinite() { + 0 + } else { + value.abs().min(usize::MAX as f64) as usize + } + } + Value(value) => value.parse::().ok().unwrap_or(0), + Named(values) => { + values.first().and_then(|(_key, val)| val.parse::().ok()).unwrap_or(0) + } + _ => 0, + }; + + let fill = match &evaluated_args[2] { + Number(value) => Some(value.to_string()), + Value(value) => Some(value.clone()), + Named(values) => values.first().map(|(_key, val)| val.clone()), + _ => None, + }; + + let align = extract_evaluated_arg_value!(evaluated_args, 3); // "<", ">", "^" + + if let Some(text) = value { + let fill_char = fill.and_then(|s| s.chars().next()).unwrap_or(' '); + + let padded = if width <= text.len() { + text.clone() + } else { + let pad = width - text.len(); + if let Some(al) = align { + match al.as_str() { + "^" => { + let left = pad / 2; + let right = pad - left; + format!( + "{}{}{}", + fill_char.to_string().repeat(left), + text, + fill_char.to_string().repeat(right) + ) + } + "<" => format!("{}{}", text, fill_char.to_string().repeat(pad)), + _ => format!("{}{}", fill_char.to_string().repeat(pad), text), + } + } else { + format!("{}{}", fill_char.to_string().repeat(pad), text) + } + }; + + Value(padded) + } else { + Undefined + } + } + BuiltInFunction::Format => { + let fmt_pattern = extract_evaluated_arg_value!(evaluated_args, 0); + + if let Some(fmt_str) = fmt_pattern { + let args = evaluated_args + .iter() + .skip(1) + .map(|value| match value { + Value(value) => Some(value.clone()), + Number(value) => Some(format_number(*value)), + Named(values) => values.first().map(|(_, value)| value.clone()), + Undefined | AnyValue | Failure(_) => None, + }) + .collect::>(); + + let mut formatted = String::new(); + let mut arg_iter = args.iter(); + let mut chars = fmt_str.chars().peekable(); + + while let Some(ch) = chars.next() { + if ch == '{' && chars.peek() == Some(&'}') { + chars.next(); + if let Some(Some(arg)) = arg_iter.next() { + formatted.push_str(arg); + } else { + formatted.push_str("{}"); + } + } else { + formatted.push(ch); + } + } + + Value(formatted) + } else { + Undefined + } + } + BuiltInFunction::AddFavourite => { + let group_name = extract_evaluated_arg_value!(evaluated_args, 0); + if let Some(group) = group_name { + let item_type = accessor.pli.header.item_type; + if item_type != PlaylistItemType::Series && item_type != PlaylistItemType::LocalSeries { + let mut pli = accessor.pli.clone(); + pli.header.group = group.intern(); + pli.header.uuid = crate::utils::create_alias_uuid(&accessor.pli.header.uuid, group); + accessor.virtual_items.push((group.clone(), pli)); + } + } + Undefined + } + } + } + } + Expression::MatchBlock(cases) => { + for match_case in cases { + let mut case_keys = vec![]; + for case_key in &match_case.keys { + match case_key { + MatchCaseKey::Identifier(ident) => { + if !ctx.has_var(ident) { + return Failure(format!( + "Match case invalid! Variable with name {ident} not found." + )); + } + case_keys.push(ctx.get_var(ident).clone()); + } + MatchCaseKey::AnyMatch => case_keys.push(AnyValue), + } + } + + let mut match_count = 0; + let case_keys_len = case_keys.len(); + for case_key in case_keys { + match case_key { + Value(_) | Number(_) | Named(_) | AnyValue => match_count += 1, + Undefined | Failure(_) => {} + } + } + if match_count == case_keys_len { + return match_case.expression.eval(ctx, accessor); + } + } + Undefined + } + Expression::MapBlock { key, cases } => { + let key_value = match key { + MapKey::Identifier(ident) => { + if !ctx.has_var(ident) { + return Failure(format!("Map expression invalid! Variable with name {ident} not found.")); + } + let mut val = ctx.get_var(ident).clone(); + if accessor.match_as_ascii { + if let Value(text) = val { + val = Value(deunicode_string(&text).into_owned()); + } + } + val + } + MapKey::FieldAccess(field) => { + if let Some(mut val) = accessor.get(field) { + if accessor.match_as_ascii { + val = deunicode_string(&val).into_owned().into(); + } + Value(val.to_string()) + } else { + Undefined + } + } + MapKey::VarAccess(name, field) => match ctx.variables.get(name) { + None => Failure(format!("Variable with name {name} not found.")), + Some(value) => match value { + Undefined => Undefined, + Value(value) if field == "1" => { + let value = if accessor.match_as_ascii { + deunicode_string(value).into_owned() + } else { + value.clone() + }; + Value(value) + } + Number(_) | Value(_) => Failure(format!("Variable with name {name} has no fields.")), + Named(values) => { + for (key, val) in values { + if key == field { + let mut res_val = val.clone(); + if accessor.match_as_ascii { + res_val = deunicode_string(&res_val).into_owned(); + } + return Value(res_val); + } + } + Failure(format!("Variable with name {name} has no field {field}.")) + } + AnyValue | Failure(_) => value.clone(), + }, + }, + }; + + for map_case in cases { + let mut matches = false; + for key in &map_case.keys { + if match key { + MapCaseKey::Text(value) => matches_text(&key_value, value), + MapCaseKey::AnyMatch => true, + MapCaseKey::RangeFrom(num) => match key_value.compare(&Number(*num)) { + None => false, + Some(ord) => match ord { + Ordering::Less => false, + Ordering::Equal | Ordering::Greater => true, + }, + }, + MapCaseKey::RangeTo(num) => match key_value.compare(&Number(*num)) { + None => false, + Some(ord) => match ord { + Ordering::Equal | Ordering::Less => true, + Ordering::Greater => false, + }, + }, + MapCaseKey::RangeFull(from, to) => match key_value.compare(&Number(*from)) { + None => false, + Some(ord) => match ord { + Ordering::Less => false, + Ordering::Equal | Ordering::Greater => match key_value.compare(&Number(*to)) { + None => false, + Some(ord) => match ord { + Ordering::Equal | Ordering::Less => true, + Ordering::Greater => false, + }, + }, + }, + }, + MapCaseKey::RangeEq(num) => match key_value.compare(&Number(*num)) { + None => false, + Some(ord) => match ord { + Ordering::Equal => true, + Ordering::Less | Ordering::Greater => false, + }, + }, + } { + matches = true; + break; + } + } + + if matches { + return map_case.expression.eval(ctx, accessor); + } + } + Undefined + } + Expression::ForEachBlock { key, expr } => { + let key_value = match key { + ForEachKey::Identifier(ident) => { + if !ctx.has_var(ident) { + return Failure(format!( + "For each expression invalid! Variable with name {ident} not found." + )); + } + let v = ctx.get_var(ident); + match v { + Named(_) | AnyValue | Failure(_) => v.clone(), + Undefined => Undefined, + _ => Failure(format!("Variable with name {ident} must be a Named list.")), + } + } + ForEachKey::VarAccess(name, field) => match ctx.variables.get(name) { + None => Failure(format!("Variable with name {name} not found.")), + Some(value) => match value { + AnyValue | Failure(_) => value.clone(), + Named(values) => { + let filtered: Vec<(String, String)> = values + .iter() + .filter(|(k, _)| k == field) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + if filtered.is_empty() { + Undefined + } else { + Named(filtered) + } + } + Undefined => Undefined, + _ => Failure(format!("Variable with name {name} must be a Named list.")), + }, + }, + }; + + let values = match key_value { + Named(key_value) => key_value, + Failure(_) => return key_value, + _ => Vec::new(), + }; + let previous_key_var = expr.key_var.as_ref().and_then(|key_var| ctx.variables.get(key_var).cloned()); + let previous_value_var = + expr.value_var.as_ref().and_then(|value_var| ctx.variables.get(value_var).cloned()); + for (k, val) in values { + if let Some(key_var) = &expr.key_var { + ctx.set_var(key_var, EvalResult::Value(k)); + } + if let Some(value_var) = &expr.value_var { + ctx.set_var(value_var, EvalResult::Value(val)); + } + expr.expression.eval(ctx, accessor); + } + if let Some(key_var) = &expr.key_var { + if let Some(previous) = previous_key_var { + ctx.set_var(key_var, previous); + } else { + ctx.variables.remove(key_var); + } + } + if let Some(value_var) = &expr.value_var { + if let Some(previous) = previous_value_var { + ctx.set_var(value_var, previous); + } else { + ctx.variables.remove(value_var); + } + } + Undefined + } + Expression::Block(expressions) => { + let mut result = Undefined; + for expr in expressions { + result = expr.eval(ctx, accessor); + } + result + } + } + } +} diff --git a/shared/src/foundation/mapper/parser.rs b/shared/src/foundation/mapper/parser.rs new file mode 100644 index 000000000..0b60e5451 --- /dev/null +++ b/shared/src/foundation/mapper/parser.rs @@ -0,0 +1,468 @@ +use super::{ + eval::to_number, AssignmentTarget, BuiltInFunction, ExprId, Expression, ForEachExpr, ForEachKey, MapCase, + MapCaseKey, MapKey, MapperContext, MapperScript, MatchCase, MatchCaseKey, RegexSource, Statement, +}; +use crate::{error::TuliproxError, foundation::mapper::EvalResult::Number, model::PatternTemplate}; +use pest::{ + iterators::{Pair, Pairs}, + Parser, +}; +use pest_derive::Parser; +use std::{collections::HashSet, str::FromStr}; + +#[derive(Parser)] +#[grammar_inline = r##" +WHITESPACE = _{ " " | "\t"} +regex_op = _{ "~" } +null = { "null" } +identifier = @{ !null ~ (ASCII_ALPHANUMERIC | "_")+ } +var_access = { identifier ~ ("." ~ identifier)? } +string_literal = @{ "\"" ~ ( "\\\\" | "\\\"" | "\\n" | "\\t" | "\\r" | (!"\"" ~ ANY) )* ~ "\"" } +number = @{ "-"? ~ ASCII_DIGIT+ ~ ("." ~ ASCII_DIGIT+)? } +number_range_from = { number ~ ".." } +number_range_to = { ".." ~ number } +number_range_full = { number ~ ".." ~ number } +number_range_eq = { number } +number_range = _{ number_range_full | number_range_from | number_range_to | number_range_eq} +read_write_field = _{ ^"name" | ^"title" | ^"caption" | ^"group" | ^"id" | ^"chno" | ^"logo" | ^"logo_small" | ^"parent_code" | ^"audio_track" | ^"time_shift" | ^"rec" | ^"url" | ^"epg_channel_id" | ^"epg_id" | ^"genre" } +read_only_field = _{ ^"input" | ^"type" } +field = { read_write_field | read_only_field } +assignment_field = { read_write_field } +field_access = _{ "@" ~ field } +regex_source = _{ field_access | identifier } +regex_expr = { regex_source ~ regex_op ~ string_literal } +block_expr = { "{" ~ statements ~ "}" } +condition = { function_call | var_access | field_access } +assignment = { (("@" ~ assignment_field) | identifier) ~ "=" ~ expression } +expression = { assignment | map_block | match_block | for_each_block | function_call | regex_expr | string_literal | number | var_access | field_access | null | block_expr } +function_name = { "concat" | "uppercase" | "lowercase" | "capitalize" | "split" | "trim" | "print" | "number" | "first" | "template" | "replace" | "pad" | "format" | "add_favourite" } +function_call = { function_name ~ "(" ~ (expression ~ ("," ~ expression)*)? ~ ")" } +any_match = { "_" } +match_case_key = { any_match | identifier } +match_case_key_list = { match_case_key ~ ("," ~ match_case_key)* } +match_case = { match_case_key_list ~ "=>" ~ expression | "(" ~ match_case_key_list ~ ")" ~ "=>" ~ expression } +match_block = { "match" ~ "{" ~ NEWLINE* ~ (match_case ~ ("," ~ NEWLINE* ~ match_case)*)? ~ ","? ~ NEWLINE* ~ "}" } +map_case_key_list = { string_literal ~ ("|" ~ string_literal)* } +map_case_key = { any_match | number_range | map_case_key_list } +map_case = { map_case_key ~ "=>" ~ expression } +map_key = { var_access | field_access } +map_block = { "map" ~ map_key ~ "{" ~ NEWLINE* ~ (map_case ~ ("," ~ NEWLINE* ~ map_case)*)? ~ ","? ~ NEWLINE* ~ "}" } +for_each_param = { any_match | identifier } +for_each_params = { "(" ~ for_each_param ~ "," ~ for_each_param ~ ")" } +for_each_target_nested = { identifier ~ "." ~ identifier } +for_each_target_simple = { identifier } +for_each_block = { + (for_each_target_nested ~ ^".for_each" ~ "(" ~ for_each_params ~ "=>" ~ expression ~ ")") | + (for_each_target_simple ~ ^".for_each" ~ "(" ~ for_each_params ~ "=>" ~ expression ~ ")") +} +statement = _{ expression } +comment = _{ "#" ~ (!NEWLINE ~ ANY)* } +statement_reparator = _{ ";" | NEWLINE } +statements = _{ (statement_reparator* ~ (statement | comment))* ~ statement_reparator* } +main = { SOI ~ statements? ~ EOI } +"##] +struct MapperParser; + +impl MapperScript { + pub(super) fn validate( + expressions: &[Expression], + statements: &[Statement], + templates: Option<&[PatternTemplate]>, + ) -> Result<(), TuliproxError> { + let ctx = &mut MapperContext::new(expressions, templates); + + let mut identifiers: HashSet = HashSet::new(); + for stmt in statements { + match stmt { + Statement::Expression(expr) => { + ctx.validate_expr(*expr, &mut identifiers)?; + } + Statement::Comment(_) => {} + } + } + Ok(()) + } + + pub fn parse(input: &str, templates: Option<&[PatternTemplate]>) -> Result { + let mut parsed = MapperParser::parse(Rule::main, input).map_err(|e| TuliproxError::Mapper(format!("{e}")))?; + let program_pair = parsed.next().unwrap(); + let mut statements = Vec::new(); + let mut expressions = Vec::new(); + for stmt_pair in program_pair.into_inner() { + if let Some(stmt) = Self::parse_statement(stmt_pair, &mut expressions)? { + statements.push(stmt); + } + } + + MapperScript::validate(&expressions, &statements, templates)?; + Ok(Self { expressions, statements }) + } + pub(super) fn parse_statement( + pair: Pair, + expressions: &mut Vec, + ) -> Result, TuliproxError> { + match pair.as_rule() { + Rule::expression => { + if let Some(expr) = MapperScript::parse_expression(pair, expressions)? { + expressions.push(expr); + let expr_id = ExprId(expressions.len() - 1); + Ok(Some(Statement::Expression(expr_id))) + } else { + Ok(None) + } + } + Rule::comment => Ok(Some(Statement::Comment(pair.as_str().trim().to_string()))), + + _ => { + // error!("Unknown statement rule: {:?}", pair.as_rule()); + Ok(None) + } + } + } + + pub(super) fn parse_assignment( + pair: Pair, + expressions: &mut Vec, + ) -> Result, TuliproxError> { + let mut inner = pair.into_inner(); + let name = inner.next().unwrap(); + let target = match name.as_rule() { + Rule::identifier => AssignmentTarget::Identifier(name.as_str().to_string()), + Rule::assignment_field => AssignmentTarget::Field(name.as_str().to_string()), + _ => return Err(TuliproxError::Mapper(format!("Assignment target isn't supported {}", name.as_str()))), + }; + let next = inner.next().unwrap(); + if let Some(expr) = MapperScript::parse_expression(next, expressions)? { + expressions.push(expr); + let expr_id = ExprId(expressions.len() - 1); + Ok(Some(Expression::Assignment { target, expr: expr_id })) + } else { + Ok(None) + } + } + + pub(super) fn parse_match_case_key(pair: Pair) -> Result { + let inner = pair.into_inner().next().unwrap(); + match inner.as_rule() { + Rule::identifier => Ok(MatchCaseKey::Identifier(inner.as_str().to_string())), + Rule::any_match => Ok(MatchCaseKey::AnyMatch), + _ => Err(TuliproxError::Mapper(format!("Unexpected match_key: {:?}", inner.as_rule()))), + } + } + + pub(super) fn parse_match_case( + pair: Pair, + expressions: &mut Vec, + ) -> Result, TuliproxError> { + let mut inner = pair.into_inner(); + + let first = inner.next().unwrap(); + + let identifiers = match first.as_rule() { + Rule::match_case_key => { + vec![MapperScript::parse_match_case_key(first)?] + } + Rule::match_case_key_list => { + let mut matches = vec![]; + for arm in first.into_inner() { + if arm.as_rule() != Rule::WHITESPACE { + match MapperScript::parse_match_case_key(arm)? { + MatchCaseKey::Identifier(ident) => matches.push(MatchCaseKey::Identifier(ident)), + MatchCaseKey::AnyMatch => matches.push(MatchCaseKey::AnyMatch), + } + } + } + // we don't allow inside multi match keys AnyMatch + if matches.len() > 1 && matches.iter().filter(|&m| matches!(m, &MatchCaseKey::AnyMatch)).count() > 0 { + return Err(TuliproxError::Mapper("Unexpected match case key: _".to_string())); + } + matches + } + _ => return Err(TuliproxError::Mapper(format!("Unexpected match arm input: {:?}", first.as_rule()))), + }; + + if let Some(expr) = MapperScript::parse_expression(inner.next().unwrap(), expressions)? { + expressions.push(expr); + let expr_id = ExprId(expressions.len() - 1); + Ok(Some(MatchCase { keys: identifiers, expression: expr_id })) + } else { + Ok(None) + } + } + + pub(super) fn parse_map_case_key(pair: Pair) -> Result, TuliproxError> { + let inner = pair.into_inner().next().unwrap(); + match inner.as_rule() { + Rule::map_case_key_list => { + let mut matches = vec![]; + for arm in inner.into_inner() { + match arm.as_rule() { + Rule::string_literal => { + let raw = arm.as_str().to_string(); + // remove quotes + let content = &raw[1..raw.len() - 1]; + matches.push(MapCaseKey::Text(content.to_string())); + } + _ => return Err(TuliproxError::Mapper(format!("Unexpected map key: {:?}", arm.as_rule()))), + } + } + Ok(matches) + } + Rule::number_range_full => { + let mut inner = inner.into_inner(); + let start = inner.next().unwrap().as_str().parse::().unwrap(); + let end = inner.next().unwrap().as_str().parse::().unwrap(); + Ok(vec![MapCaseKey::RangeFull(start, end)]) + } + Rule::number_range_from => { + let mut inner = inner.into_inner(); + let start = inner.next().unwrap().as_str().parse::().unwrap(); + Ok(vec![MapCaseKey::RangeFrom(start)]) + } + Rule::number_range_to => { + let mut inner = inner.into_inner(); + let to = inner.next().unwrap().as_str().parse::().unwrap(); + Ok(vec![MapCaseKey::RangeTo(to)]) + } + Rule::number_range_eq => { + let mut inner = inner.into_inner(); + let num = inner.next().unwrap().as_str().parse::().unwrap(); + Ok(vec![MapCaseKey::RangeEq(num)]) + } + Rule::any_match => Ok(vec![MapCaseKey::AnyMatch]), + _ => Err(TuliproxError::Mapper(format!("Unexpected map key: {:?}", inner.as_rule()))), + } + } + + pub(super) fn parse_map_case( + pair: Pair, + expressions: &mut Vec, + ) -> Result, TuliproxError> { + let mut inner = pair.into_inner(); + + let first = inner.next().unwrap(); + + let identifier = match first.as_rule() { + Rule::map_case_key => MapperScript::parse_map_case_key(first)?, + _ => return Err(TuliproxError::Mapper(format!("Unexpected match arm input: {:?}", first.as_rule()))), + }; + + if let Some(expr) = MapperScript::parse_expression(inner.next().unwrap(), expressions)? { + expressions.push(expr); + let expr_id = ExprId(expressions.len() - 1); + Ok(Some(MapCase { keys: identifier, expression: expr_id })) + } else { + Ok(None) + } + } + + pub(super) fn parse_expression( + pair: Pair, + expressions: &mut Vec, + ) -> Result, TuliproxError> { + match pair.as_rule() { + Rule::assignment => { + if let Some(expr) = MapperScript::parse_assignment(pair, expressions)? { + Ok(Some(expr)) + } else { + Ok(None) + } + } + Rule::field => Ok(Some(Expression::FieldAccess(pair.as_str().trim().to_string()))), + Rule::var_access => { + let text = pair.as_str(); + if text.contains('.') { + let splitted: Vec<&str> = text.splitn(2, '.').collect(); + Ok(Some(Expression::VarAccess(splitted[0].trim().to_string(), splitted[1].trim().to_string()))) + } else { + Ok(Some(Expression::Identifier(text.trim().to_string()))) + } + } + + Rule::string_literal => { + let raw = pair.as_str(); + // remove quotes + let content = &raw[1..raw.len() - 1]; + Ok(Some(Expression::StringLiteral(content.to_string()))) + } + + Rule::number => { + let raw = pair.as_str(); + if let Number(val) = to_number(raw) { + Ok(Some(Expression::NumberLiteral(val))) + } else { + Err(TuliproxError::Mapper(format!("Invalid number {raw}"))) + } + } + + Rule::regex_expr => { + let mut inner = pair.into_inner(); + let first = inner.next().unwrap(); + let field = match first.as_rule() { + Rule::identifier => RegexSource::Identifier(first.as_str().to_string()), + Rule::field => RegexSource::Field(first.as_str().to_string()), + _ => return Err(TuliproxError::RegexCompile(first.as_str().to_string())), + }; + let pattern_raw = inner.next().unwrap().as_str(); + let pattern = &pattern_raw[1..pattern_raw.len() - 1]; // Strip quotes + match crate::model::REGEX_CACHE.get_or_compile(pattern) { + Ok(re) => Ok(Some(Expression::RegexExpr { field, pattern: pattern.to_string(), re_pattern: re })), + Err(_) => Err(TuliproxError::RegexCompile(pattern.to_string())), + } + } + + Rule::function_call => { + let mut inner = pair.into_inner(); + let fn_name = inner.next().unwrap().as_str().to_string(); + let mut args = vec![]; + for arg in inner { + if let Some(expr) = MapperScript::parse_expression(arg, expressions)? { + expressions.push(expr); + let expr_id = ExprId(expressions.len() - 1); + args.push(expr_id); + } + } + let name = BuiltInFunction::from_str(&fn_name)?; + Ok(Some(Expression::FunctionCall { name, args })) + } + + Rule::match_block => { + let case_pairs = pair.into_inner(); + let mut cases = vec![]; + for case in case_pairs { + if let Some(expr) = MapperScript::parse_match_case(case, expressions)? { + cases.push(expr); + } + } + if cases.is_empty() { + Ok(None) + } else { + Ok(Some(Expression::MatchBlock(cases))) + } + } + + Rule::map_block => Self::parse_map_block(pair.into_inner(), expressions), + + Rule::for_each_block => Self::parse_for_each_block(pair.into_inner(), expressions), + + Rule::null => Ok(Some(Expression::NullValue)), + + Rule::expression => { + let inner = pair.into_inner().next().unwrap(); + MapperScript::parse_expression(inner, expressions) + } + Rule::block_expr => { + let inner = pair.into_inner(); + let mut block_expressions = vec![]; + for expr in inner { + if let Some(expr) = MapperScript::parse_expression(expr, expressions)? { + expressions.push(expr); + let expr_id = ExprId(expressions.len() - 1); + block_expressions.push(expr_id); + } + } + Ok(Some(Expression::Block(block_expressions))) + } + _ => Err(TuliproxError::Mapper(format!("Unknown expression rule: {:?}", pair.as_rule()))), + } + } + + pub(super) fn parse_map_block( + mut pairs: Pairs, + expressions: &mut Vec, + ) -> Result, TuliproxError> { + let first = pairs.next().unwrap(); + let key = match first.as_rule() { + Rule::map_key => { + if let Some(map_key) = first.into_inner().next() { + match map_key.as_rule() { + Rule::field => MapKey::FieldAccess(map_key.as_str().trim().to_string()), + Rule::var_access => { + let text = map_key.as_str(); + if text.contains('.') { + let splitted: Vec<&str> = text.splitn(2, '.').collect(); + MapKey::VarAccess(splitted[0].trim().to_string(), splitted[1].trim().to_string()) + } else { + MapKey::Identifier(text.trim().to_string()) + } + } + _ => { + return Err(TuliproxError::Mapper(format!( + "Unexpected map case key: {:?}", + map_key.as_rule() + ))) + } + } + } else { + return Err(TuliproxError::Mapper("Missing map case key".to_string())); + } + } + _ => return Err(TuliproxError::Mapper(format!("Unexpected map case key: {:?}", first.as_rule()))), + }; + let mut cases = vec![]; + for case in pairs { + if let Some(map_case) = MapperScript::parse_map_case(case, expressions)? { + cases.push(map_case); + } + } + if cases.is_empty() { + Ok(None) + } else { + Ok(Some(Expression::MapBlock { key, cases })) + } + } + + pub(super) fn parse_for_each_param(pair: Pair) -> Result, TuliproxError> { + let inner = pair.into_inner().next().unwrap(); + match inner.as_rule() { + Rule::identifier => Ok(Some(inner.as_str().to_string())), + Rule::any_match => Ok(None), + _ => Err(TuliproxError::Mapper(format!("Unexpected for_each_param: {:?}", inner.as_rule()))), + } + } + + pub(super) fn parse_for_each_params(pair: Pair) -> Result<(Option, Option), TuliproxError> { + let mut inner = pair.into_inner(); + let key = Self::parse_for_each_param(inner.next().unwrap())?; + let val = Self::parse_for_each_param(inner.next().unwrap())?; + + if key.is_none() && val.is_none() { + return Err(TuliproxError::Mapper("At least one parameter must be named in for_each loop".to_string())); + } + + Ok((key, val)) + } + + pub(super) fn parse_for_each_block( + mut pairs: Pairs, + expressions: &mut Vec, + ) -> Result, TuliproxError> { + let first = pairs.next().unwrap(); + let key = match first.as_rule() { + Rule::for_each_target_simple => ForEachKey::Identifier(first.as_str().trim().to_string()), + Rule::for_each_target_nested => { + let text = first.as_str(); + let splitted: Vec<&str> = text.splitn(2, '.').collect(); + ForEachKey::VarAccess(splitted[0].trim().to_string(), splitted[1].trim().to_string()) + } + _ => return Err(TuliproxError::Mapper(format!("Unexpected for each target: {:?}", first.as_rule()))), + }; + + if let Some(params_pair) = pairs.next() { + // .for_each + let (key_var, value_var) = Self::parse_for_each_params(params_pair)?; + + let expr_pair = pairs.next().unwrap(); + if let Some(expr) = MapperScript::parse_expression(expr_pair, expressions)? { + expressions.push(expr); + let expr_id = ExprId(expressions.len() - 1); + return Ok(Some(Expression::ForEachBlock { + key, + expr: ForEachExpr { key_var, value_var, expression: expr_id }, + })); + } + } + + Ok(None) + } +} diff --git a/shared/src/foundation/mapper/tests.rs b/shared/src/foundation/mapper/tests.rs new file mode 100644 index 000000000..15574296f --- /dev/null +++ b/shared/src/foundation/mapper/tests.rs @@ -0,0 +1,464 @@ +use super::*; +use crate::{ + foundation::mapper::EvalResult::{Named, Undefined, Value}, + model::{ + PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamProperties, StreamProperties, TemplateValue, + VideoStreamDetailProperties, VideoStreamProperties, + }, + utils::Internable, +}; +use regex::Regex; + +#[test] +fn test_mapper_dsl_eval() { + let dsl = r#" + coast = @Caption ~ "(?i)\b(EAST|WEST)\b" + quality = @Caption ~ "(?i)\b([FUSL]?HD|SD|4K|1080p|720p|3840p)\b" + quality = uppercase(quality) + quality = map quality { + "SHD" => "SD", + "LHD" => "HD", + "720p" => "HD", + "1080p" => "FHD", + "4K" => "UHD", + "3840p" => "UHD", + _ => quality, + } + coast_quality = match { + (coast, quality) => concat(capitalize(coast), " ", uppercase(quality)), + coast => concat(capitalize(coast), " HD"), + quality => concat("East ", uppercase(quality)), + } + @Caption = concat("US: TNT", " ", coast_quality) + @Group = "United States - Entertainment" + "#; + + let mapper = MapperScript::parse(dsl, None).expect("Parsing failed"); + println!("Program: {mapper:?}"); + let mut channels: Vec = vec![ + ("D", "HD"), + ("A", "FHD"), + ("Z", ""), + ("K", "HD"), + ("B", "HD"), + ("A", "HD"), + ("K", "SHD"), + ("C", "LHD"), + ("L", "FHD"), + ("R", "UHD"), + ("T", "SD"), + ("A", "FHD"), + ] + .into_iter() + .map(|(name, quality)| PlaylistItem { + header: PlaylistItemHeader { title: format!("Chanel {name} [{quality}]").into(), ..Default::default() }, + }) + .collect::>(); + + for pli in &mut channels { + let mut accessor = ValueAccessor { pli, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; + mapper.eval(&mut accessor, None); + println!("Result: {pli:?}"); + } + + // ctx.fields.insert("Caption".to_string(), "US: TNT East LHD bubble".to_string()); + // + // for stmt in &program.statements { + // //let res = stmt.eval(&mut ctx); + // println!("Statement Result: {:?}", res); + // } + // + // println!("Result variable: {:?}", ctx.variables.get("result")); + // assert_eq!(ctx.variables.get("result").unwrap(), "US: TNT East HD"); +} + +#[test] +fn test_complex() { + let script = r#" + print("LOCAL") + coast = @Caption ~ "!COAST!" + quality = uppercase(@Caption ~ "!QUALITY!") + + quality = map quality { + "SHD" | "SD" => "SD", + "LHD" | "720P" | "HD" => "HD", + "FHD" | "1080P" => "FHD", + "UHD" | "4K" | "3840P" => "UHD", + _ => quality, + } + + coast_quality = match { + (coast, quality) => concat(capitalize(coast), " ", uppercase(quality)), + quality => uppercase(quality), + _ => "HD", + } + + network = uppercase(first(@Caption ~ "(?i)\b(CBS|NBC|FOX|ABC|PBS|CW|UNIVISION)\b")) + station = map network { + "CBS" => @Caption ~ "(?i)\b(WINK|WFOR)\b", + "NBC" => @Caption ~ "(?i)\b(WBBH|WTVJ)\b", + "FOX" => @Caption ~ "(?i)\b(WFTX|WSVM)\b", + "ABC" => @Caption ~ "(?i)\b(WZVN|WPLG)\b", + "PBS" => @Caption ~ "(?i)\b(WGCU|WPBT)\b", + "CW" => @Caption ~ "(?i)\b(WINK|WSFL)\b", + "UNIVISION" => @Caption ~ "(?i)\b(WUVF|WLTV)\b", + _ => null, + } + + match { + station => { + station = uppercase(station) + @Caption = map station { + "WINK" => concat("!US_CBS_FM_PREFIX!", " ", coast_quality), + "WBBH" => concat("!US_NBC_FM_PREFIX!", " ", coast_quality), + "WFTX" => concat("!US_FOX_FM_PREFIX!", " ", coast_quality), + "WZVN" => concat("!US_ABC_FM_PREFIX!", " ", coast_quality), + "WGCU" => concat("!US_PBS_FM_PREFIX!", " ", coast_quality), + "WUVF" => concat("!US_UNIVISION_FM_PREFIX!", " ", coast_quality), + + "WFOR" => concat("!US_CBS_MIA_PREFIX!", " ", coast_quality), + "WTVJ" => concat("!US_NBC_MIA_PREFIX!", " ", coast_quality), + "WSVM" => concat("!US_FOX_MIA_PREFIX!", " ", coast_quality), + "WPLG" => concat("!US_ABC_MIA_PREFIX!", " ", coast_quality), + "WPBT" => concat("!US_PBS_MIA_PREFIX!", " ", coast_quality), + "WSFL" => concat("!US_CW_MIA_PREFIX!", " ", coast_quality), + "WLTV" => concat("!US_UNIVISION_MIA_PREFIX!", " ", coast_quality), + + _ => concat(network, " ", station, " ", coast_quality), + } + + @Group = concat("🇺🇸 > USA - ", network, " Locals") + } + } + "#; + let mapper = MapperScript::parse(script, None).expect("Parsing failed"); + println!("Program: {mapper:?}"); +} + +#[test] +fn test_mapper_format() { + let dsl = r#" + @Name = pad(1000, 10, 0); + @Title = format("Channel {} is {}", 12, "live"); + "#; + + let mapper = MapperScript::parse(dsl, None).expect("Parsing failed"); + let mut channels: Vec = vec![("D", "HD")] + .into_iter() + .map(|(name, quality)| PlaylistItem { + header: PlaylistItemHeader { title: format!("Chanel {name} [{quality}]").into(), ..Default::default() }, + }) + .collect::>(); + + for pli in &mut channels { + let mut accessor = ValueAccessor { pli, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; + mapper.eval(&mut accessor, None); + assert_eq!(accessor.pli.header.title.as_ref(), "Channel 12 is live"); + } +} + +#[test] +fn regex_identifier_ascii_normalization_preserves_map_selection() { + let dsl = r#" + source = @Name + capture = source ~ "(Cinema)" + @Group = map capture { + "Cinema" => "matched", + _ => "missed", + } + "#; + let mapper = MapperScript::parse(dsl, None).expect("mapper should parse"); + let mut channel = PlaylistItem { header: PlaylistItemHeader { name: "Cinéma".intern(), ..Default::default() } }; + let mut accessor = + ValueAccessor { pli: &mut channel, virtual_items: vec![], match_as_ascii: true, changed_fields: vec![] }; + + mapper.eval(&mut accessor, None); + + assert_eq!(accessor.pli.header.group.as_ref(), "matched"); +} + +#[test] +fn regex_evaluation_keeps_all_groups_from_only_the_first_match() -> Result<(), Box> { + let regex = Regex::new(r"(?P[A-Z])([A-Z])(?P[A-Z])([A-Z])(?P[A-Z])([A-Z])")?; + + let Named(values) = eval_regex("ABCDEFUVWXYZ", ®ex, false) else { + return Err("regex captures should produce a named result".into()); + }; + + assert_eq!( + values, + [ + ("1", "A"), + ("2", "B"), + ("3", "C"), + ("4", "D"), + ("5", "E"), + ("6", "F"), + ("one", "A"), + ("three", "C"), + ("five", "E"), + ] + .map(|(key, value)| (key.to_string(), value.to_string())) + ); + Ok(()) +} + +#[test] +fn regex_capture_groups_are_available_by_index_and_name() -> Result<(), Box> { + let dsl = r#" + captures = @Name ~ "(?P[A-Z])([A-Z])(?P[A-Z])([A-Z])(?P[A-Z])([A-Z])" + @Title = concat(captures.1, captures.2, captures.3, captures.4, captures.5, captures.6) + @Group = concat(captures.one, captures.three, captures.five) + "#; + let mapper = MapperScript::parse(dsl, None)?; + let mut channel = PlaylistItem { header: PlaylistItemHeader { name: "ABCDEF".intern(), ..Default::default() } }; + let mut accessor = + ValueAccessor { pli: &mut channel, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; + + mapper.eval(&mut accessor, None); + + assert_eq!(accessor.pli.header.title.as_ref(), "ABCDEF"); + assert_eq!(accessor.pli.header.group.as_ref(), "ACE"); + Ok(()) +} + +#[test] +fn single_regex_capture_is_available_as_scalar_and_by_index() -> Result<(), Box> { + let dsl = r#" + capture = @Name ~ "([A-Z]+)" + @Group = map capture.1 { + "ABC" => concat(capture, ":", capture.1), + _ => "missed", + } + "#; + let mapper = MapperScript::parse(dsl, None)?; + let mut channel = PlaylistItem { header: PlaylistItemHeader { name: "ABC".intern(), ..Default::default() } }; + let mut accessor = + ValueAccessor { pli: &mut channel, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; + + mapper.eval(&mut accessor, None); + + assert_eq!(accessor.pli.header.group.as_ref(), "ABC:ABC"); + Ok(()) +} + +#[test] +fn read_only_fields_work_in_direct_regex_and_map_expressions() -> Result<(), Box> { + let dsl = r#" + input_match = @Input ~ "(source)" + type_match = @Type ~ "(live)" + @Title = concat(@Input, ":", @Type, ":", input_match, ":", type_match) + @Group = map @Input { + "source" => "input-map", + _ => "missed", + } + @Name = map @Type { + "live" => concat(@Group, ":type-map"), + _ => "missed", + } + "#; + let mapper = MapperScript::parse(dsl, None)?; + let mut channel = PlaylistItem { + header: PlaylistItemHeader { + input_name: "source".intern(), + item_type: PlaylistItemType::Live, + ..Default::default() + }, + }; + let mut accessor = + ValueAccessor { pli: &mut channel, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; + + mapper.eval(&mut accessor, None); + + assert_eq!(accessor.pli.header.title.as_ref(), "source:live:source:live"); + assert_eq!(accessor.pli.header.group.as_ref(), "input-map"); + assert_eq!(accessor.pli.header.name.as_ref(), "input-map:type-map"); + Ok(()) +} + +#[test] +fn read_only_field_assignments_are_rejected() { + for script in [r#"@Input = "changed""#, r#"@Type = "movie""#] { + assert!(MapperScript::parse(script, None).is_err()); + } + assert!(MapperScript::parse(r#"@Group = "changed""#, None).is_ok()); +} + +#[test] +fn template_lookup_keeps_last_duplicate_value() { + let dsl = r#"@Group = template("label")"#; + let templates = vec![ + PatternTemplate { + name: "label".to_string(), + value: TemplateValue::Single("first".to_string()), + placeholder: "!label!".to_string(), + }, + PatternTemplate { + name: "label".to_string(), + value: TemplateValue::Single("second".to_string()), + placeholder: "!label!".to_string(), + }, + ]; + let mapper = MapperScript::parse(dsl, Some(&templates)).expect("mapper should parse"); + let mut channel = PlaylistItem { header: PlaylistItemHeader::default() }; + let mut accessor = + ValueAccessor { pli: &mut channel, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; + + mapper.eval(&mut accessor, Some(&templates)); + + assert_eq!(accessor.pli.header.group.as_ref(), "second"); +} + +#[test] +fn test_mapper_add_favourite() { + use crate::model::PlaylistItemType; + let dsl = r#" + add_favourite("My Favs"); + "#; + + let mapper = MapperScript::parse(dsl, None).expect("Parsing failed"); + + // Test with Video (should work) + let mut video = PlaylistItem { + header: PlaylistItemHeader { + name: "Movie 1".to_string().into(), + item_type: PlaylistItemType::Video, + ..Default::default() + }, + }; + let mut accessor = + ValueAccessor { pli: &mut video, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; + mapper.eval(&mut accessor, None); + assert_eq!(accessor.virtual_items.len(), 1); + assert_eq!(&*accessor.virtual_items[0].1.header.group, "My Favs"); + + // Test with SeriesInfo (should work) + let mut series_info = PlaylistItem { + header: PlaylistItemHeader { + name: "Series 1".to_string().into(), + item_type: PlaylistItemType::SeriesInfo, + ..Default::default() + }, + }; + let mut accessor = + ValueAccessor { pli: &mut series_info, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; + mapper.eval(&mut accessor, None); + assert_eq!(accessor.virtual_items.len(), 1); + + // Test with Series episode (should NOT work) + let mut episode = PlaylistItem { + header: PlaylistItemHeader { + name: "Episode 1".to_string().into(), + item_type: PlaylistItemType::Series, + ..Default::default() + }, + }; + let mut accessor = + ValueAccessor { pli: &mut episode, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; + mapper.eval(&mut accessor, None); + assert_eq!(accessor.virtual_items.len(), 0); +} + +#[test] +fn test_mapper_split_loop() { + use crate::model::PlaylistItemType; + let dsl = r#" + genres = split(@Genre, ",") + print(genres) + genres.for_each((_, gen) => { + add_favourite(concat("Genre - ", gen)) + }) + "#; + + let mapper = MapperScript::parse(dsl, None).expect("Parsing failed"); + + // Test with Video (should work) + let mut video = PlaylistItem { + header: PlaylistItemHeader { + name: "Movie 1".to_string().into(), + item_type: PlaylistItemType::Video, + additional_properties: Some(StreamProperties::Video(Box::new(VideoStreamProperties { + details: Some(VideoStreamDetailProperties { + genre: Some("A, B, C".intern()), + ..VideoStreamDetailProperties::default() + }), + ..VideoStreamProperties::default() + }))), + ..Default::default() + }, + }; + let mut accessor = + ValueAccessor { pli: &mut video, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; + mapper.eval(&mut accessor, None); + assert_eq!(accessor.virtual_items.len(), 3); + assert_eq!(&*accessor.virtual_items[0].1.header.group, "Genre - A"); + assert_eq!(&*accessor.virtual_items[1].1.header.group, "Genre - B"); + assert_eq!(&*accessor.virtual_items[2].1.header.group, "Genre - C"); + + // Test with SeriesInfo (should work) + let mut series_info = PlaylistItem { + header: PlaylistItemHeader { + name: "Series 1".to_string().into(), + item_type: PlaylistItemType::SeriesInfo, + additional_properties: Some(StreamProperties::Series(Box::new(SeriesStreamProperties { + genre: Some("A, B, C".intern()), + ..SeriesStreamProperties::default() + }))), + ..Default::default() + }, + }; + let mut accessor = + ValueAccessor { pli: &mut series_info, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; + mapper.eval(&mut accessor, None); + assert_eq!(accessor.virtual_items.len(), 3); +} + +#[test] +fn for_each_restores_existing_loop_variables() { + let expressions = vec![ + Expression::Identifier("value".to_string()), + Expression::ForEachBlock { + key: ForEachKey::Identifier("items".to_string()), + expr: ForEachExpr { + key_var: Some("key".to_string()), + value_var: Some("value".to_string()), + expression: ExprId(0), + }, + }, + ]; + let mut ctx = MapperContext::new(&expressions, None); + ctx.set_var("items", Named(vec![("first".to_string(), "current".to_string())])); + ctx.set_var("value", Value("outer".to_string())); + let mut item = PlaylistItem { header: PlaylistItemHeader::default() }; + let mut accessor = + ValueAccessor { pli: &mut item, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; + + ExprId(1).eval(&mut ctx, &mut accessor); + + assert!(matches!(ctx.get_var("value"), Value(value) if value == "outer")); + assert!(matches!(ctx.get_var("key"), Undefined)); +} + +#[test] +fn eval_reports_changes_and_statement_failures() { + let mapper = MapperScript::parse( + r#" + @Group = "News" + capture = @Name ~ "(?PBBC)" + @Title = capture.missing + "#, + None, + ) + .expect("script should parse"); + let mut item = PlaylistItem { header: PlaylistItemHeader { name: "BBC One".intern(), ..Default::default() } }; + let mut accessor = + ValueAccessor { pli: &mut item, virtual_items: vec![], match_as_ascii: false, changed_fields: vec![] }; + + let outcome = mapper.eval(&mut accessor, None); + + assert_eq!(outcome.changed_fields, ["Group"]); + assert_eq!(outcome.emitted_items, 0); + assert_eq!(outcome.diagnostics.len(), 1); + assert_eq!(outcome.diagnostics[0].statement, 2); + assert!(outcome.diagnostics[0].message.contains("has no field missing")); +} diff --git a/shared/src/model/config/app_config.rs b/shared/src/model/config/app_config.rs index f88a052fe..f847cae88 100644 --- a/shared/src/model/config/app_config.rs +++ b/shared/src/model/config/app_config.rs @@ -28,8 +28,10 @@ mod tests { let mut app_cfg = AppConfigDto::default(); assert!(!app_cfg.is_recording_enabled()); - let mut video = VideoConfigDto::default(); - video.recording = Some(RecordingConfigDto { enabled: true, ..Default::default() }); + let video = VideoConfigDto { + recording: Some(RecordingConfigDto { enabled: true, ..Default::default() }), + ..Default::default() + }; app_cfg.config.video = Some(video); assert!(app_cfg.is_recording_enabled()); } diff --git a/shared/src/model/config/base.rs b/shared/src/model/config/base.rs index 5bf873e7d..6b02d7a06 100644 --- a/shared/src/model/config/base.rs +++ b/shared/src/model/config/base.rs @@ -2,18 +2,13 @@ use crate::{ defaults::{ default_as_true, default_connect_timeout_secs, default_custom_stream_response_error_status, default_custom_stream_response_path, default_default_user_agent, default_event_channel_capacity, - default_interner_gc_interval_secs, default_interner_gc_min_pool_size, default_main_backup_dir, - default_main_mapping_path, default_main_storage_dir, default_main_template_path, default_main_user_config_dir, - default_supported_video_extensions, is_blank_or_default_backup_dir, + default_interner_gc_interval_secs, default_interner_gc_min_pool_size, is_blank_or_default_backup_dir, is_blank_or_default_custom_stream_response_path, is_blank_or_default_mapping_path, is_blank_or_default_storage_dir, is_blank_or_default_template_path, is_blank_or_default_user_config_dir, is_default_connect_timeout_secs, is_default_custom_stream_response_error_status, is_default_event_channel_capacity, is_default_interner_gc_interval_secs, is_default_interner_gc_min_pool_size, - is_false, is_none_or_empty_metadata_update, is_true, is_zero_u32, normalize_optional_config_file_path, - normalize_optional_dir, DEFAULT_BACKUP_DIR, DEFAULT_CUSTOM_STREAM_RESPONSE_PATH, DEFAULT_STORAGE_DIR, - DEFAULT_USER_CONFIG_DIR, MAPPING_FILE, TEMPLATE_FILE, + is_false, is_none_or_empty_metadata_update, is_true, is_zero_u32, }, - error::TuliproxError, model::{ ConfigApiDto, HdHomeRunConfigDto, IpCheckConfigDto, LibraryConfigDto, LogConfigDto, MessagingConfigDto, MetadataUpdateConfigDto, ProxyConfigDto, ReverseProxyConfigDto, ScheduleConfigDto, VideoConfigDto, @@ -268,649 +263,9 @@ impl<'de> serde::Deserialize<'de> for ConfigDto { } } -// This MainConfigDto is a copy of ConfigDto simple fields for form editing. -// It has no other purpose than editing and saving the simple config values. -// `recording` is intentionally stripped here — the main-config form edits -// the simple scalar settings; DVR lives on a dedicated form/page (see -// `SchedulesConfigDto` for the parallel pattern). Add the field here only -// when the main form gains DVR editing. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -pub struct MainConfigDto { - #[serde(default, skip_serializing_if = "is_false")] - pub process_parallel: bool, - #[serde(default = "default_main_storage_dir", skip_serializing_if = "is_blank_or_default_storage_dir")] - pub storage_dir: Option, - #[serde(default = "default_default_user_agent", skip_serializing_if = "is_blank_optional_string")] - pub default_user_agent: Option, - #[serde(default = "default_main_backup_dir", skip_serializing_if = "is_blank_or_default_backup_dir")] - pub backup_dir: Option, - #[serde(default = "default_main_user_config_dir", skip_serializing_if = "is_blank_or_default_user_config_dir")] - pub user_config_dir: Option, - #[serde(default = "default_main_mapping_path", skip_serializing_if = "is_blank_or_default_mapping_path")] - pub mapping_path: Option, - #[serde(default = "default_main_template_path", skip_serializing_if = "is_blank_or_default_template_path")] - pub template_path: Option, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub custom_stream_response_path: Option, - #[serde(default, skip_serializing_if = "is_zero_u32")] - pub custom_stream_response_timeout_secs: u32, - #[serde(default = "default_as_true", skip_serializing_if = "is_true")] - pub custom_stream_response_enabled: bool, - #[serde( - default = "default_custom_stream_response_error_status", - skip_serializing_if = "is_default_custom_stream_response_error_status" - )] - pub custom_stream_response_error_status: u16, - #[serde(default, skip_serializing_if = "is_false")] - pub user_access_control: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub disk_based_processing: bool, - #[serde(default = "default_connect_timeout_secs", skip_serializing_if = "is_default_connect_timeout_secs")] - pub connect_timeout_secs: u32, - #[serde( - default = "default_interner_gc_interval_secs", - skip_serializing_if = "is_default_interner_gc_interval_secs" - )] - pub interner_gc_interval_secs: u32, - #[serde( - default = "default_interner_gc_min_pool_size", - skip_serializing_if = "is_default_interner_gc_min_pool_size" - )] - pub interner_gc_min_pool_size: u32, - #[serde(default = "default_event_channel_capacity", skip_serializing_if = "is_default_event_channel_capacity")] - pub event_channel_capacity: u32, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub sleep_timer_mins: Option, - #[serde(default, skip_serializing_if = "is_false")] - pub update_on_boot: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub config_hot_reload: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub accept_insecure_ssl_certificates: bool, -} - -impl Default for MainConfigDto { - fn default() -> Self { - MainConfigDto { - process_parallel: false, - disk_based_processing: false, - storage_dir: default_main_storage_dir(), - default_user_agent: default_default_user_agent(), - backup_dir: default_main_backup_dir(), - user_config_dir: default_main_user_config_dir(), - mapping_path: default_main_mapping_path(), - template_path: default_main_template_path(), - custom_stream_response_path: None, - custom_stream_response_timeout_secs: 0, - custom_stream_response_enabled: true, - custom_stream_response_error_status: default_custom_stream_response_error_status(), - user_access_control: false, - connect_timeout_secs: default_connect_timeout_secs(), - interner_gc_interval_secs: default_interner_gc_interval_secs(), - event_channel_capacity: default_event_channel_capacity(), - interner_gc_min_pool_size: default_interner_gc_min_pool_size(), - sleep_timer_mins: None, - update_on_boot: false, - config_hot_reload: false, - accept_insecure_ssl_certificates: false, - } - } -} - -impl From<&ConfigDto> for MainConfigDto { - fn from(config: &ConfigDto) -> Self { - // `recording` is intentionally NOT mirrored: the main-config form - // owns simple scalar settings only (see the struct-level comment). - Self { - process_parallel: config.process_parallel, - disk_based_processing: config.disk_based_processing, - storage_dir: config.storage_dir.clone(), - default_user_agent: config.default_user_agent.clone(), - backup_dir: config.backup_dir.clone(), - user_config_dir: config.user_config_dir.clone(), - mapping_path: config.mapping_path.clone(), - template_path: config.template_path.clone(), - custom_stream_response_path: config.custom_stream_response_path.clone(), - custom_stream_response_timeout_secs: config.custom_stream_response_timeout_secs, - custom_stream_response_enabled: config.custom_stream_response_enabled, - custom_stream_response_error_status: config.custom_stream_response_error_status, - user_access_control: config.user_access_control, - connect_timeout_secs: config.connect_timeout_secs, - interner_gc_interval_secs: config.interner_gc_interval_secs, - event_channel_capacity: config.event_channel_capacity, - interner_gc_min_pool_size: config.interner_gc_min_pool_size, - sleep_timer_mins: config.sleep_timer_mins, - update_on_boot: config.update_on_boot, - config_hot_reload: config.config_hot_reload, - accept_insecure_ssl_certificates: config.accept_insecure_ssl_certificates, - } - } -} - -// This SchedulesConfigDto is a copy of ConfigDto schedules fields for form editing. -// It has no other purpose than editing and saving the schedules -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] -pub struct SchedulesConfigDto { - #[serde(default)] - pub schedules: Option>, -} - -impl SchedulesConfigDto { - // Clippy's method-path suggestion here names a private module and does not - // compile; the closure is kept deliberately. - #[allow(clippy::redundant_closure_for_method_calls)] - pub fn is_empty(&self) -> bool { self.schedules.as_deref().is_none_or(|s| s.is_empty()) } -} - -impl From<&ConfigDto> for SchedulesConfigDto { - fn from(config: &ConfigDto) -> Self { Self { schedules: config.schedules.clone() } } -} - -pub struct HdHomeRunDeviceOverview { - pub enabled: bool, - pub devices: Vec, -} - -impl ConfigDto { - pub fn prepare(&mut self, include_computed: bool) -> Result<(), TuliproxError> { - self.api.prepare(); - - if is_blank_optional_string(&self.default_user_agent) { - self.default_user_agent = default_default_user_agent(); - } - if is_blank_or_default_storage_dir(&self.storage_dir) { - self.storage_dir = default_main_storage_dir(); - } - if is_blank_or_default_backup_dir(&self.backup_dir) { - self.backup_dir = default_main_backup_dir(); - } - if is_blank_or_default_user_config_dir(&self.user_config_dir) { - self.user_config_dir = default_main_user_config_dir(); - } - if is_blank_or_default_mapping_path(&self.mapping_path) { - self.mapping_path = default_main_mapping_path(); - } - if is_blank_or_default_template_path(&self.template_path) { - self.template_path = default_main_template_path(); - } - - if let Some(mins) = self.sleep_timer_mins { - if mins == 0 { - return Err(TuliproxError::ConfigBase("`sleep_timer_mins` must be > 0 when specified".to_string())); - } - } - if self.interner_gc_interval_secs == 0 { - return Err(TuliproxError::ConfigBase("`interner_gc_interval_secs` must be > 0".to_string())); - } - if self.interner_gc_min_pool_size == 0 { - return Err(TuliproxError::ConfigBase("`interner_gc_min_pool_size` must be > 0".to_string())); - } - - if self.custom_stream_response_error_status == 0 { - self.custom_stream_response_error_status = default_custom_stream_response_error_status(); - } else if !(400..=599).contains(&self.custom_stream_response_error_status) { - return Err(TuliproxError::ConfigBase(format!( - "`custom_stream_response_error_status` must be a 4xx or 5xx HTTP status, got {}", - self.custom_stream_response_error_status - ))); - } - - self.prepare_web()?; - self.prepare_hdhomerun(include_computed)?; - self.prepare_video_config()?; - self.prepare_metadata_update_config()?; - - if let Some(reverse_proxy) = self.reverse_proxy.as_mut() { - reverse_proxy.prepare(self.storage_dir.as_deref().unwrap_or_default())?; - } - if let Some(proxy) = &mut self.proxy { - proxy.prepare()?; - } - if let Some(ipcheck) = self.ipcheck.as_mut() { - ipcheck.prepare()?; - } - - if let Some(messaging) = &mut self.messaging { - messaging.prepare(include_computed)?; - } - if let Some(library) = &mut self.library { - library.playlist.prepare(); - } - - Ok(()) - } - - fn prepare_web(&mut self) -> Result<(), TuliproxError> { - if let Some(web_ui_config) = self.web_ui.as_mut() { - web_ui_config.prepare()?; - } - Ok(()) - } - - fn prepare_hdhomerun(&mut self, include_computed: bool) -> Result<(), TuliproxError> { - if let Some(hdhomerun) = &mut self.hdhomerun { - if hdhomerun.enabled { - hdhomerun.prepare(self.api.port, include_computed)?; - } - } - Ok(()) - } - - fn prepare_video_config(&mut self) -> Result<(), TuliproxError> { - match &mut self.video { - None => { - self.video = Some(VideoConfigDto { - extensions: default_supported_video_extensions(), - web_search: None, - recording: None, - }); - } - Some(video) => match video.prepare() { - Ok(()) => {} - Err(err) => return Err(err), - }, - } - - Ok(()) - } - - fn prepare_metadata_update_config(&mut self) -> Result<(), TuliproxError> { - let mut metadata_update = self.metadata_update.clone().unwrap_or_default(); - - metadata_update.prepare()?; - - if metadata_update.is_empty() { - self.metadata_update = None; - } else { - self.metadata_update = Some(metadata_update); - } - - Ok(()) - } - - pub fn is_valid(&self) -> bool { - if self.api.host.is_empty() { - return false; - } - - if let Some(video) = &self.video { - if let Some(recording) = &video.recording { - if let Some(episode_pattern) = &recording.episode_pattern { - if !episode_pattern.is_empty() { - let re = crate::model::REGEX_CACHE.get_or_compile(episode_pattern); - if re.is_err() { - return false; - } - } - } - } - } - true - } - - pub fn get_hdhr_device_overview(&self) -> Option { - self.hdhomerun.as_ref().map(|hdhr| HdHomeRunDeviceOverview { - enabled: hdhr.enabled, - devices: hdhr.devices.iter().map(|d| d.name.clone()).collect::>(), - }) - } - - pub fn update_from_main_config(&mut self, main_config: &MainConfigDto) { - // `recording` is intentionally NOT touched: this is the - // simple-form save path; DVR is edited on its own form (see - // the `MainConfigDto` comment). - self.process_parallel = main_config.process_parallel; - self.disk_based_processing = main_config.disk_based_processing; - self.storage_dir = normalize_optional_dir(&main_config.storage_dir, DEFAULT_STORAGE_DIR); - self.default_user_agent = main_config.default_user_agent.clone(); - self.backup_dir = normalize_optional_dir(&main_config.backup_dir, DEFAULT_BACKUP_DIR); - self.user_config_dir = normalize_optional_dir(&main_config.user_config_dir, DEFAULT_USER_CONFIG_DIR); - self.mapping_path = normalize_optional_config_file_path(&main_config.mapping_path, MAPPING_FILE); - self.template_path = normalize_optional_config_file_path(&main_config.template_path, TEMPLATE_FILE); - self.custom_stream_response_path = - normalize_optional_dir(&main_config.custom_stream_response_path, DEFAULT_CUSTOM_STREAM_RESPONSE_PATH); - self.custom_stream_response_timeout_secs = main_config.custom_stream_response_timeout_secs; - self.custom_stream_response_enabled = main_config.custom_stream_response_enabled; - self.custom_stream_response_error_status = main_config.custom_stream_response_error_status; - self.user_access_control = main_config.user_access_control; - self.connect_timeout_secs = main_config.connect_timeout_secs; - self.interner_gc_interval_secs = main_config.interner_gc_interval_secs; - self.interner_gc_min_pool_size = main_config.interner_gc_min_pool_size; - self.sleep_timer_mins = main_config.sleep_timer_mins; - self.update_on_boot = main_config.update_on_boot; - self.config_hot_reload = main_config.config_hot_reload; - self.accept_insecure_ssl_certificates = main_config.accept_insecure_ssl_certificates; - } - - pub fn is_geoip_enabled(&self) -> bool { - self.reverse_proxy.as_ref().is_some_and(|r| r.geoip.as_ref().is_some_and(|g| g.enabled)) - } - - pub fn is_library_enabled(&self) -> bool { self.library.as_ref().is_some_and(|l| l.enabled) } - - pub fn is_stream_history_enabled(&self) -> bool { - self.reverse_proxy.as_ref().and_then(|r| r.stream_history.as_ref()).is_some_and(|sh| sh.stream_history_enabled) - } - - pub fn is_qos_aggregation_enabled(&self) -> bool { - self.is_stream_history_enabled() - && self.reverse_proxy.as_ref().and_then(|r| r.qos_aggregation.as_ref()).is_some_and(|qos| qos.enabled) - } - - pub fn is_recording_enabled(&self) -> bool { - self.video.as_ref().and_then(|v| v.recording.as_ref()).is_some_and(|r| r.enabled) - } -} - #[cfg(test)] -mod tests { - use super::*; - use crate::{defaults::CONFIG_PATH, model::RecordingConfigDto}; - use serde_json::json; +mod tests; - #[test] - fn default_uses_connect_timeout_default_value() { - let cfg = ConfigDto::default(); - assert_eq!(cfg.connect_timeout_secs, default_connect_timeout_secs()); - assert_eq!(cfg.interner_gc_interval_secs, default_interner_gc_interval_secs()); - assert_eq!(cfg.interner_gc_min_pool_size, default_interner_gc_min_pool_size()); - } - - #[test] - fn custom_video_stream_defaults_are_true_and_502() { - let cfg = ConfigDto::default(); - assert!(cfg.custom_stream_response_enabled); - assert_eq!(cfg.custom_stream_response_error_status, 502); - } - - #[test] - fn prepare_rejects_non_4xx_5xx_custom_stream_response_error_status() { - for bad in [200u16, 100, 399, 600, 1000] { - let mut cfg = ConfigDto { custom_stream_response_error_status: bad, ..ConfigDto::default() }; - let err = cfg.prepare(false).expect_err(&format!("status {bad} must be rejected")); - let msg = format!("{err}"); - assert!(msg.contains("custom_stream_response_error_status"), "status {bad} msg: {msg}"); - } - } - - #[test] - fn prepare_accepts_4xx_and_5xx_custom_stream_response_error_status() { - for ok in [400u16, 404, 500, 502, 503, 599] { - let mut cfg = ConfigDto { custom_stream_response_error_status: ok, ..ConfigDto::default() }; - assert!(cfg.prepare(false).is_ok(), "status {ok} must be accepted"); - } - } - - #[test] - fn prepare_clamps_zero_custom_stream_response_error_status_to_default() { - let mut cfg = ConfigDto { custom_stream_response_error_status: 0, ..ConfigDto::default() }; - cfg.prepare(false).expect("zero must be silently clamped, not rejected"); - assert_eq!(cfg.custom_stream_response_error_status, 502); - } - - #[test] - fn serializing_skips_default_storage_backup_and_user_config_dirs() { - let cfg = ConfigDto { - storage_dir: Some(DEFAULT_STORAGE_DIR.to_string()), - backup_dir: Some(DEFAULT_BACKUP_DIR.to_string()), - user_config_dir: Some(DEFAULT_USER_CONFIG_DIR.to_string()), - ..ConfigDto::default() - }; - - let serialized = serde_json::to_string(&cfg).expect("config serialization should succeed"); - assert!( - !serialized.contains("\"storage_dir\""), - "expected no storage_dir field for default value, got: {serialized}" - ); - assert!( - !serialized.contains("\"backup_dir\""), - "expected no backup_dir field for default value, got: {serialized}" - ); - assert!( - !serialized.contains("\"user_config_dir\""), - "expected no user_config_dir field for default value, got: {serialized}" - ); - } - - #[test] - fn serializing_keeps_non_default_storage_and_backup_dirs() { - let cfg = ConfigDto { - storage_dir: Some("custom-storage".to_string()), - backup_dir: Some("custom-backup".to_string()), - user_config_dir: Some("custom-user-config".to_string()), - ..ConfigDto::default() - }; - - let serialized = serde_json::to_string(&cfg).expect("config serialization should succeed"); - assert!( - serialized.contains("\"storage_dir\""), - "expected storage_dir field for non-default value, got: {serialized}" - ); - assert!( - serialized.contains("\"backup_dir\""), - "expected backup_dir field for non-default value, got: {serialized}" - ); - assert!( - serialized.contains("\"user_config_dir\""), - "expected user_config_dir field for non-default value, got: {serialized}" - ); - } - - #[test] - fn main_config_from_applies_default_storage_backup_and_user_config_dirs() { - let mut cfg = ConfigDto::default(); - cfg.prepare(false).expect("prepare should succeed"); - let main = MainConfigDto::from(&cfg); - assert_eq!(main.storage_dir.as_deref(), Some(DEFAULT_STORAGE_DIR)); - assert_eq!(main.backup_dir.as_deref(), Some(DEFAULT_BACKUP_DIR)); - assert_eq!(main.user_config_dir.as_deref(), Some(DEFAULT_USER_CONFIG_DIR)); - assert_eq!(main.mapping_path.as_deref(), Some(format!("./{CONFIG_PATH}/{MAPPING_FILE}").as_str())); - assert_eq!(main.template_path.as_deref(), Some(format!("./{CONFIG_PATH}/{TEMPLATE_FILE}").as_str())); - } - - #[test] - fn update_from_main_config_omits_default_optional_paths() { - let mut cfg = ConfigDto::default(); - let main = MainConfigDto { - storage_dir: Some(DEFAULT_STORAGE_DIR.to_string()), - backup_dir: Some(DEFAULT_BACKUP_DIR.to_string()), - user_config_dir: Some(DEFAULT_USER_CONFIG_DIR.to_string()), - mapping_path: Some(format!("./{CONFIG_PATH}/{MAPPING_FILE}")), - template_path: Some(format!("./{CONFIG_PATH}/{TEMPLATE_FILE}")), - ..MainConfigDto::default() - }; - - cfg.update_from_main_config(&main); - assert!(cfg.storage_dir.is_none()); - assert!(cfg.backup_dir.is_none()); - assert!(cfg.user_config_dir.is_none()); - assert!(cfg.mapping_path.is_none()); - assert!(cfg.template_path.is_none()); - } - - #[test] - fn prepare_sets_default_optional_paths() { - let mut cfg = ConfigDto { - storage_dir: None, - backup_dir: None, - user_config_dir: None, - mapping_path: None, - template_path: None, - ..ConfigDto::default() - }; - cfg.prepare(false).expect("prepare should succeed"); - assert_eq!(cfg.storage_dir.as_deref(), Some(DEFAULT_STORAGE_DIR)); - assert_eq!(cfg.backup_dir.as_deref(), Some(DEFAULT_BACKUP_DIR)); - assert_eq!(cfg.user_config_dir.as_deref(), Some(DEFAULT_USER_CONFIG_DIR)); - assert_eq!(cfg.mapping_path.as_deref(), Some(format!("./{CONFIG_PATH}/{MAPPING_FILE}").as_str())); - assert_eq!(cfg.template_path.as_deref(), Some(format!("./{CONFIG_PATH}/{TEMPLATE_FILE}").as_str())); - } - - #[test] - fn deserializing_rejects_legacy_video_ffprobe_fields() { - let raw = json!({ - "api": { - "host": "127.0.0.1", - "port": 8901, - "web_root": "./web" - }, - "storage_dir": ".", - "video": { - "extensions": ["mp4"], - "ffprobe_enabled": true - } - }); - - let result: Result = serde_json::from_value(raw); - assert!(result.is_err(), "legacy ffprobe field under video must fail"); - let err = result.unwrap_err().to_string(); - assert!(err.contains("ffprobe_enabled"), "unexpected error text: {err}"); - } - - #[test] - fn deserializing_rejects_legacy_data_dir_alias() { - let raw = json!({ - "api": { - "host": "127.0.0.1", - "port": 8901, - "web_root": "./web" - }, - "data_dir": "." - }); - - let result: Result = serde_json::from_value(raw); - assert!(result.is_err(), "data_dir should not deserialize"); - } - - #[test] - fn deserializing_accepts_legacy_working_dir_alias() { - let raw = json!({ - "api": { - "host": "127.0.0.1", - "port": 8901, - "web_root": "./web" - }, - "working_dir": "." - }); - - let cfg: ConfigDto = serde_json::from_value(raw).expect("working_dir should deserialize as legacy alias"); - assert_eq!(cfg.storage_dir.as_deref(), Some(".")); - } - - #[test] - fn deserializing_accepts_missing_storage_dir() { - let raw = json!({ - "api": { - "host": "127.0.0.1", - "port": 8901, - "web_root": "./web" - } - }); - - let cfg: ConfigDto = serde_json::from_value(raw).expect("missing storage_dir should deserialize"); - assert!(cfg.storage_dir.is_none()); - } - - #[test] - fn stream_history_defaults_to_disabled_and_safe() { - let cfg = ConfigDto::default(); - - assert!(cfg.reverse_proxy.is_none()); - } - - #[test] - fn stream_history_deserializes_under_reverse_proxy() { - let raw = r" -api: - host: 127.0.0.1 - port: 8901 - web_root: ./web -reverse_proxy: - rewrite_secret: 00112233445566778899aabbccddeeff - stream_history: - stream_history_enabled: true - stream_history_batch_size: 64 - stream_history_retention_days: 14 - stream_history_directory: /var/lib/tuliprox/history -"; - - let cfg: ConfigDto = serde_saphyr::from_str(raw).expect("config should deserialize"); - let reverse_proxy = cfg.reverse_proxy.expect("reverse_proxy should deserialize"); - let stream_history = reverse_proxy.stream_history.expect("stream_history should deserialize"); - - assert!(stream_history.stream_history_enabled); - assert_eq!(stream_history.stream_history_batch_size, 64); - assert_eq!(stream_history.stream_history_retention_days, 14); - assert_eq!(stream_history.stream_history_directory, "/var/lib/tuliprox/history"); - } - - #[test] - fn stream_history_missing_values_keep_disabled_without_reverse_proxy() { - let raw = r" -api: - host: 127.0.0.1 - port: 8901 - web_root: ./web -"; - - let cfg: ConfigDto = serde_saphyr::from_str(raw).expect("config should deserialize"); - - assert!(cfg.reverse_proxy.is_none()); - } - - #[test] - fn stream_history_defaults_to_none_when_reverse_proxy_present_but_stream_history_omitted() { - let raw = r#" -api: - host: 127.0.0.1 - port: 8901 - web_root: ./web -reverse_proxy: - resource_rewrite_disabled: false - rewrite_secret: "00112233445566778899aabbccddeeff" -"#; - - let cfg: ConfigDto = serde_saphyr::from_str(raw).expect("config should deserialize"); - - assert!(cfg.reverse_proxy.is_some()); - assert!(cfg.reverse_proxy.as_ref().and_then(|rp| rp.stream_history.as_ref()).is_none()); - } - - #[test] - fn is_stream_history_and_qos_enabled_match_nested_config() { - let mut cfg = ConfigDto::default(); - assert!(!cfg.is_stream_history_enabled()); - assert!(!cfg.is_qos_aggregation_enabled()); - - let mut rp = ReverseProxyConfigDto::default(); - rp.stream_history = - Some(crate::model::StreamHistoryConfigDto { stream_history_enabled: true, ..Default::default() }); - cfg.reverse_proxy = Some(rp); - assert!(cfg.is_stream_history_enabled()); - assert!(!cfg.is_qos_aggregation_enabled()); - - cfg.reverse_proxy.as_mut().unwrap().qos_aggregation = - Some(crate::model::QosAggregationConfigDto { enabled: true, ..Default::default() }); - assert!(cfg.is_stream_history_enabled()); - assert!(cfg.is_qos_aggregation_enabled()); - - // Disabling stream history disables QoS aggregation as well - cfg.reverse_proxy.as_mut().unwrap().stream_history.as_mut().unwrap().stream_history_enabled = false; - assert!(!cfg.is_stream_history_enabled()); - assert!(!cfg.is_qos_aggregation_enabled()); - } - - #[test] - fn is_recording_enabled_matches_nested_config() { - let mut cfg = ConfigDto::default(); - assert!(!cfg.is_recording_enabled()); - - let mut video = VideoConfigDto::default(); - video.recording = Some(RecordingConfigDto { enabled: true, ..Default::default() }); - cfg.video = Some(video); - assert!(cfg.is_recording_enabled()); - - cfg.video.as_mut().unwrap().recording.as_mut().unwrap().enabled = false; - assert!(!cfg.is_recording_enabled()); - } -} +mod prepare; +mod views; +pub use views::{HdHomeRunDeviceOverview, MainConfigDto, SchedulesConfigDto}; diff --git a/shared/src/model/config/base/prepare.rs b/shared/src/model/config/base/prepare.rs new file mode 100644 index 000000000..a66d0d49f --- /dev/null +++ b/shared/src/model/config/base/prepare.rs @@ -0,0 +1,205 @@ +use super::{ConfigDto, HdHomeRunDeviceOverview, MainConfigDto}; +use crate::{ + defaults::{ + default_custom_stream_response_error_status, default_default_user_agent, default_main_backup_dir, + default_main_mapping_path, default_main_storage_dir, default_main_template_path, default_main_user_config_dir, + default_supported_video_extensions, is_blank_or_default_backup_dir, is_blank_or_default_mapping_path, + is_blank_or_default_storage_dir, is_blank_or_default_template_path, is_blank_or_default_user_config_dir, + normalize_optional_config_file_path, normalize_optional_dir, DEFAULT_BACKUP_DIR, + DEFAULT_CUSTOM_STREAM_RESPONSE_PATH, DEFAULT_STORAGE_DIR, DEFAULT_USER_CONFIG_DIR, MAPPING_FILE, TEMPLATE_FILE, + }, + error::TuliproxError, + model::VideoConfigDto, + utils::is_blank_optional_string, +}; + +impl ConfigDto { + pub fn prepare(&mut self, include_computed: bool) -> Result<(), TuliproxError> { + self.api.prepare(); + + if is_blank_optional_string(&self.default_user_agent) { + self.default_user_agent = default_default_user_agent(); + } + if is_blank_or_default_storage_dir(&self.storage_dir) { + self.storage_dir = default_main_storage_dir(); + } + if is_blank_or_default_backup_dir(&self.backup_dir) { + self.backup_dir = default_main_backup_dir(); + } + if is_blank_or_default_user_config_dir(&self.user_config_dir) { + self.user_config_dir = default_main_user_config_dir(); + } + if is_blank_or_default_mapping_path(&self.mapping_path) { + self.mapping_path = default_main_mapping_path(); + } + if is_blank_or_default_template_path(&self.template_path) { + self.template_path = default_main_template_path(); + } + + if let Some(mins) = self.sleep_timer_mins { + if mins == 0 { + return Err(TuliproxError::ConfigBase("`sleep_timer_mins` must be > 0 when specified".to_string())); + } + } + if self.interner_gc_interval_secs == 0 { + return Err(TuliproxError::ConfigBase("`interner_gc_interval_secs` must be > 0".to_string())); + } + if self.interner_gc_min_pool_size == 0 { + return Err(TuliproxError::ConfigBase("`interner_gc_min_pool_size` must be > 0".to_string())); + } + + if self.custom_stream_response_error_status == 0 { + self.custom_stream_response_error_status = default_custom_stream_response_error_status(); + } else if !(400..=599).contains(&self.custom_stream_response_error_status) { + return Err(TuliproxError::ConfigBase(format!( + "`custom_stream_response_error_status` must be a 4xx or 5xx HTTP status, got {}", + self.custom_stream_response_error_status + ))); + } + + self.prepare_web()?; + self.prepare_hdhomerun(include_computed)?; + self.prepare_video_config()?; + self.prepare_metadata_update_config()?; + + if let Some(reverse_proxy) = self.reverse_proxy.as_mut() { + reverse_proxy.prepare(self.storage_dir.as_deref().unwrap_or_default())?; + } + if let Some(proxy) = &mut self.proxy { + proxy.prepare()?; + } + if let Some(ipcheck) = self.ipcheck.as_mut() { + ipcheck.prepare()?; + } + + if let Some(messaging) = &mut self.messaging { + messaging.prepare(include_computed)?; + } + if let Some(library) = &mut self.library { + library.playlist.prepare(); + } + + Ok(()) + } + + pub(super) fn prepare_web(&mut self) -> Result<(), TuliproxError> { + if let Some(web_ui_config) = self.web_ui.as_mut() { + web_ui_config.prepare()?; + } + Ok(()) + } + + pub(super) fn prepare_hdhomerun(&mut self, include_computed: bool) -> Result<(), TuliproxError> { + if let Some(hdhomerun) = &mut self.hdhomerun { + if hdhomerun.enabled { + hdhomerun.prepare(self.api.port, include_computed)?; + } + } + Ok(()) + } + + pub(super) fn prepare_video_config(&mut self) -> Result<(), TuliproxError> { + match &mut self.video { + None => { + self.video = Some(VideoConfigDto { + extensions: default_supported_video_extensions(), + web_search: None, + recording: None, + }); + } + Some(video) => match video.prepare() { + Ok(()) => {} + Err(err) => return Err(err), + }, + } + + Ok(()) + } + + pub(super) fn prepare_metadata_update_config(&mut self) -> Result<(), TuliproxError> { + let mut metadata_update = self.metadata_update.clone().unwrap_or_default(); + + metadata_update.prepare()?; + + if metadata_update.is_empty() { + self.metadata_update = None; + } else { + self.metadata_update = Some(metadata_update); + } + + Ok(()) + } + + pub fn is_valid(&self) -> bool { + if self.api.host.is_empty() { + return false; + } + + if let Some(video) = &self.video { + if let Some(recording) = &video.recording { + if let Some(episode_pattern) = &recording.episode_pattern { + if !episode_pattern.is_empty() { + let re = crate::model::REGEX_CACHE.get_or_compile(episode_pattern); + if re.is_err() { + return false; + } + } + } + } + } + true + } + + pub fn get_hdhr_device_overview(&self) -> Option { + self.hdhomerun.as_ref().map(|hdhr| HdHomeRunDeviceOverview { + enabled: hdhr.enabled, + devices: hdhr.devices.iter().map(|d| d.name.clone()).collect::>(), + }) + } + + pub fn update_from_main_config(&mut self, main_config: &MainConfigDto) { + // `recording` is intentionally NOT touched: this is the + // simple-form save path; DVR is edited on its own form (see + // the `MainConfigDto` comment). + self.process_parallel = main_config.process_parallel; + self.disk_based_processing = main_config.disk_based_processing; + self.storage_dir = normalize_optional_dir(&main_config.storage_dir, DEFAULT_STORAGE_DIR); + self.default_user_agent = main_config.default_user_agent.clone(); + self.backup_dir = normalize_optional_dir(&main_config.backup_dir, DEFAULT_BACKUP_DIR); + self.user_config_dir = normalize_optional_dir(&main_config.user_config_dir, DEFAULT_USER_CONFIG_DIR); + self.mapping_path = normalize_optional_config_file_path(&main_config.mapping_path, MAPPING_FILE); + self.template_path = normalize_optional_config_file_path(&main_config.template_path, TEMPLATE_FILE); + self.custom_stream_response_path = + normalize_optional_dir(&main_config.custom_stream_response_path, DEFAULT_CUSTOM_STREAM_RESPONSE_PATH); + self.custom_stream_response_timeout_secs = main_config.custom_stream_response_timeout_secs; + self.custom_stream_response_enabled = main_config.custom_stream_response_enabled; + self.custom_stream_response_error_status = main_config.custom_stream_response_error_status; + self.user_access_control = main_config.user_access_control; + self.connect_timeout_secs = main_config.connect_timeout_secs; + self.interner_gc_interval_secs = main_config.interner_gc_interval_secs; + self.interner_gc_min_pool_size = main_config.interner_gc_min_pool_size; + self.sleep_timer_mins = main_config.sleep_timer_mins; + self.update_on_boot = main_config.update_on_boot; + self.config_hot_reload = main_config.config_hot_reload; + self.accept_insecure_ssl_certificates = main_config.accept_insecure_ssl_certificates; + } + + pub fn is_geoip_enabled(&self) -> bool { + self.reverse_proxy.as_ref().is_some_and(|r| r.geoip.as_ref().is_some_and(|g| g.enabled)) + } + + pub fn is_library_enabled(&self) -> bool { self.library.as_ref().is_some_and(|l| l.enabled) } + + pub fn is_stream_history_enabled(&self) -> bool { + self.reverse_proxy.as_ref().and_then(|r| r.stream_history.as_ref()).is_some_and(|sh| sh.stream_history_enabled) + } + + pub fn is_qos_aggregation_enabled(&self) -> bool { + self.is_stream_history_enabled() + && self.reverse_proxy.as_ref().and_then(|r| r.qos_aggregation.as_ref()).is_some_and(|qos| qos.enabled) + } + + pub fn is_recording_enabled(&self) -> bool { + self.video.as_ref().and_then(|v| v.recording.as_ref()).is_some_and(|r| r.enabled) + } +} diff --git a/shared/src/model/config/base/tests.rs b/shared/src/model/config/base/tests.rs new file mode 100644 index 000000000..daffd03c8 --- /dev/null +++ b/shared/src/model/config/base/tests.rs @@ -0,0 +1,320 @@ +use super::*; +use crate::{ + defaults::{ + CONFIG_PATH, DEFAULT_BACKUP_DIR, DEFAULT_STORAGE_DIR, DEFAULT_USER_CONFIG_DIR, MAPPING_FILE, TEMPLATE_FILE, + }, + model::RecordingConfigDto, +}; +use serde_json::json; + +#[test] +fn default_uses_connect_timeout_default_value() { + let cfg = ConfigDto::default(); + assert_eq!(cfg.connect_timeout_secs, default_connect_timeout_secs()); + assert_eq!(cfg.interner_gc_interval_secs, default_interner_gc_interval_secs()); + assert_eq!(cfg.interner_gc_min_pool_size, default_interner_gc_min_pool_size()); +} + +#[test] +fn custom_video_stream_defaults_are_true_and_502() { + let cfg = ConfigDto::default(); + assert!(cfg.custom_stream_response_enabled); + assert_eq!(cfg.custom_stream_response_error_status, 502); +} + +#[test] +fn prepare_rejects_non_4xx_5xx_custom_stream_response_error_status() { + for bad in [200u16, 100, 399, 600, 1000] { + let mut cfg = ConfigDto { custom_stream_response_error_status: bad, ..ConfigDto::default() }; + let err = cfg.prepare(false).expect_err(&format!("status {bad} must be rejected")); + let msg = format!("{err}"); + assert!(msg.contains("custom_stream_response_error_status"), "status {bad} msg: {msg}"); + } +} + +#[test] +fn prepare_accepts_4xx_and_5xx_custom_stream_response_error_status() { + for ok in [400u16, 404, 500, 502, 503, 599] { + let mut cfg = ConfigDto { custom_stream_response_error_status: ok, ..ConfigDto::default() }; + assert!(cfg.prepare(false).is_ok(), "status {ok} must be accepted"); + } +} + +#[test] +fn prepare_clamps_zero_custom_stream_response_error_status_to_default() { + let mut cfg = ConfigDto { custom_stream_response_error_status: 0, ..ConfigDto::default() }; + cfg.prepare(false).expect("zero must be silently clamped, not rejected"); + assert_eq!(cfg.custom_stream_response_error_status, 502); +} + +#[test] +fn serializing_skips_default_storage_backup_and_user_config_dirs() { + let cfg = ConfigDto { + storage_dir: Some(DEFAULT_STORAGE_DIR.to_string()), + backup_dir: Some(DEFAULT_BACKUP_DIR.to_string()), + user_config_dir: Some(DEFAULT_USER_CONFIG_DIR.to_string()), + ..ConfigDto::default() + }; + + let serialized = serde_json::to_string(&cfg).expect("config serialization should succeed"); + assert!( + !serialized.contains("\"storage_dir\""), + "expected no storage_dir field for default value, got: {serialized}" + ); + assert!( + !serialized.contains("\"backup_dir\""), + "expected no backup_dir field for default value, got: {serialized}" + ); + assert!( + !serialized.contains("\"user_config_dir\""), + "expected no user_config_dir field for default value, got: {serialized}" + ); +} + +#[test] +fn serializing_keeps_non_default_storage_and_backup_dirs() { + let cfg = ConfigDto { + storage_dir: Some("custom-storage".to_string()), + backup_dir: Some("custom-backup".to_string()), + user_config_dir: Some("custom-user-config".to_string()), + ..ConfigDto::default() + }; + + let serialized = serde_json::to_string(&cfg).expect("config serialization should succeed"); + assert!( + serialized.contains("\"storage_dir\""), + "expected storage_dir field for non-default value, got: {serialized}" + ); + assert!( + serialized.contains("\"backup_dir\""), + "expected backup_dir field for non-default value, got: {serialized}" + ); + assert!( + serialized.contains("\"user_config_dir\""), + "expected user_config_dir field for non-default value, got: {serialized}" + ); +} + +#[test] +fn main_config_from_applies_default_storage_backup_and_user_config_dirs() { + let mut cfg = ConfigDto::default(); + cfg.prepare(false).expect("prepare should succeed"); + let main = MainConfigDto::from(&cfg); + assert_eq!(main.storage_dir.as_deref(), Some(DEFAULT_STORAGE_DIR)); + assert_eq!(main.backup_dir.as_deref(), Some(DEFAULT_BACKUP_DIR)); + assert_eq!(main.user_config_dir.as_deref(), Some(DEFAULT_USER_CONFIG_DIR)); + assert_eq!(main.mapping_path.as_deref(), Some(format!("./{CONFIG_PATH}/{MAPPING_FILE}").as_str())); + assert_eq!(main.template_path.as_deref(), Some(format!("./{CONFIG_PATH}/{TEMPLATE_FILE}").as_str())); +} + +#[test] +fn update_from_main_config_omits_default_optional_paths() { + let mut cfg = ConfigDto::default(); + let main = MainConfigDto { + storage_dir: Some(DEFAULT_STORAGE_DIR.to_string()), + backup_dir: Some(DEFAULT_BACKUP_DIR.to_string()), + user_config_dir: Some(DEFAULT_USER_CONFIG_DIR.to_string()), + mapping_path: Some(format!("./{CONFIG_PATH}/{MAPPING_FILE}")), + template_path: Some(format!("./{CONFIG_PATH}/{TEMPLATE_FILE}")), + ..MainConfigDto::default() + }; + + cfg.update_from_main_config(&main); + assert!(cfg.storage_dir.is_none()); + assert!(cfg.backup_dir.is_none()); + assert!(cfg.user_config_dir.is_none()); + assert!(cfg.mapping_path.is_none()); + assert!(cfg.template_path.is_none()); +} + +#[test] +fn prepare_sets_default_optional_paths() { + let mut cfg = ConfigDto { + storage_dir: None, + backup_dir: None, + user_config_dir: None, + mapping_path: None, + template_path: None, + ..ConfigDto::default() + }; + cfg.prepare(false).expect("prepare should succeed"); + assert_eq!(cfg.storage_dir.as_deref(), Some(DEFAULT_STORAGE_DIR)); + assert_eq!(cfg.backup_dir.as_deref(), Some(DEFAULT_BACKUP_DIR)); + assert_eq!(cfg.user_config_dir.as_deref(), Some(DEFAULT_USER_CONFIG_DIR)); + assert_eq!(cfg.mapping_path.as_deref(), Some(format!("./{CONFIG_PATH}/{MAPPING_FILE}").as_str())); + assert_eq!(cfg.template_path.as_deref(), Some(format!("./{CONFIG_PATH}/{TEMPLATE_FILE}").as_str())); +} + +#[test] +fn deserializing_rejects_legacy_video_ffprobe_fields() { + let raw = json!({ + "api": { + "host": "127.0.0.1", + "port": 8901, + "web_root": "./web" + }, + "storage_dir": ".", + "video": { + "extensions": ["mp4"], + "ffprobe_enabled": true + } + }); + + let result: Result = serde_json::from_value(raw); + assert!(result.is_err(), "legacy ffprobe field under video must fail"); + let err = result.unwrap_err().to_string(); + assert!(err.contains("ffprobe_enabled"), "unexpected error text: {err}"); +} + +#[test] +fn deserializing_rejects_legacy_data_dir_alias() { + let raw = json!({ + "api": { + "host": "127.0.0.1", + "port": 8901, + "web_root": "./web" + }, + "data_dir": "." + }); + + let result: Result = serde_json::from_value(raw); + assert!(result.is_err(), "data_dir should not deserialize"); +} + +#[test] +fn deserializing_accepts_legacy_working_dir_alias() { + let raw = json!({ + "api": { + "host": "127.0.0.1", + "port": 8901, + "web_root": "./web" + }, + "working_dir": "." + }); + + let cfg: ConfigDto = serde_json::from_value(raw).expect("working_dir should deserialize as legacy alias"); + assert_eq!(cfg.storage_dir.as_deref(), Some(".")); +} + +#[test] +fn deserializing_accepts_missing_storage_dir() { + let raw = json!({ + "api": { + "host": "127.0.0.1", + "port": 8901, + "web_root": "./web" + } + }); + + let cfg: ConfigDto = serde_json::from_value(raw).expect("missing storage_dir should deserialize"); + assert!(cfg.storage_dir.is_none()); +} + +#[test] +fn stream_history_defaults_to_disabled_and_safe() { + let cfg = ConfigDto::default(); + + assert!(cfg.reverse_proxy.is_none()); +} + +#[test] +fn stream_history_deserializes_under_reverse_proxy() { + let raw = r" +api: + host: 127.0.0.1 + port: 8901 + web_root: ./web +reverse_proxy: + rewrite_secret: 00112233445566778899aabbccddeeff + stream_history: + stream_history_enabled: true + stream_history_batch_size: 64 + stream_history_retention_days: 14 + stream_history_directory: /var/lib/tuliprox/history +"; + + let cfg: ConfigDto = serde_saphyr::from_str(raw).expect("config should deserialize"); + let reverse_proxy = cfg.reverse_proxy.expect("reverse_proxy should deserialize"); + let stream_history = reverse_proxy.stream_history.expect("stream_history should deserialize"); + + assert!(stream_history.stream_history_enabled); + assert_eq!(stream_history.stream_history_batch_size, 64); + assert_eq!(stream_history.stream_history_retention_days, 14); + assert_eq!(stream_history.stream_history_directory, "/var/lib/tuliprox/history"); +} + +#[test] +fn stream_history_missing_values_keep_disabled_without_reverse_proxy() { + let raw = r" +api: + host: 127.0.0.1 + port: 8901 + web_root: ./web +"; + + let cfg: ConfigDto = serde_saphyr::from_str(raw).expect("config should deserialize"); + + assert!(cfg.reverse_proxy.is_none()); +} + +#[test] +fn stream_history_defaults_to_none_when_reverse_proxy_present_but_stream_history_omitted() { + let raw = r#" +api: + host: 127.0.0.1 + port: 8901 + web_root: ./web +reverse_proxy: + resource_rewrite_disabled: false + rewrite_secret: "00112233445566778899aabbccddeeff" +"#; + + let cfg: ConfigDto = serde_saphyr::from_str(raw).expect("config should deserialize"); + + assert!(cfg.reverse_proxy.is_some()); + assert!(cfg.reverse_proxy.as_ref().and_then(|rp| rp.stream_history.as_ref()).is_none()); +} + +#[test] +fn is_stream_history_and_qos_enabled_match_nested_config() { + let mut cfg = ConfigDto::default(); + assert!(!cfg.is_stream_history_enabled()); + assert!(!cfg.is_qos_aggregation_enabled()); + + let rp = ReverseProxyConfigDto { + stream_history: Some(crate::model::StreamHistoryConfigDto { + stream_history_enabled: true, + ..Default::default() + }), + ..Default::default() + }; + cfg.reverse_proxy = Some(rp); + assert!(cfg.is_stream_history_enabled()); + assert!(!cfg.is_qos_aggregation_enabled()); + + cfg.reverse_proxy.as_mut().unwrap().qos_aggregation = + Some(crate::model::QosAggregationConfigDto { enabled: true, ..Default::default() }); + assert!(cfg.is_stream_history_enabled()); + assert!(cfg.is_qos_aggregation_enabled()); + + // Disabling stream history disables QoS aggregation as well + cfg.reverse_proxy.as_mut().unwrap().stream_history.as_mut().unwrap().stream_history_enabled = false; + assert!(!cfg.is_stream_history_enabled()); + assert!(!cfg.is_qos_aggregation_enabled()); +} + +#[test] +fn is_recording_enabled_matches_nested_config() { + let mut cfg = ConfigDto::default(); + assert!(!cfg.is_recording_enabled()); + + let video = VideoConfigDto { + recording: Some(RecordingConfigDto { enabled: true, ..Default::default() }), + ..Default::default() + }; + cfg.video = Some(video); + assert!(cfg.is_recording_enabled()); + + cfg.video.as_mut().unwrap().recording.as_mut().unwrap().enabled = false; + assert!(!cfg.is_recording_enabled()); +} diff --git a/shared/src/model/config/base/views.rs b/shared/src/model/config/base/views.rs new file mode 100644 index 000000000..565348e43 --- /dev/null +++ b/shared/src/model/config/base/views.rs @@ -0,0 +1,158 @@ +use super::ConfigDto; +use crate::{ + defaults::{ + default_as_true, default_connect_timeout_secs, default_custom_stream_response_error_status, + default_default_user_agent, default_event_channel_capacity, default_interner_gc_interval_secs, + default_interner_gc_min_pool_size, default_main_backup_dir, default_main_mapping_path, + default_main_storage_dir, default_main_template_path, default_main_user_config_dir, + is_blank_or_default_backup_dir, is_blank_or_default_mapping_path, is_blank_or_default_storage_dir, + is_blank_or_default_template_path, is_blank_or_default_user_config_dir, is_default_connect_timeout_secs, + is_default_custom_stream_response_error_status, is_default_event_channel_capacity, + is_default_interner_gc_interval_secs, is_default_interner_gc_min_pool_size, is_false, is_true, is_zero_u32, + }, + model::ScheduleConfigDto, + utils::is_blank_optional_string, +}; + +// This MainConfigDto is a copy of ConfigDto simple fields for form editing. +// It has no other purpose than editing and saving the simple config values. +// `recording` is intentionally stripped here — the main-config form edits +// the simple scalar settings; DVR lives on a dedicated form/page (see +// `SchedulesConfigDto` for the parallel pattern). Add the field here only +// when the main form gains DVR editing. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +pub struct MainConfigDto { + #[serde(default, skip_serializing_if = "is_false")] + pub process_parallel: bool, + #[serde(default = "default_main_storage_dir", skip_serializing_if = "is_blank_or_default_storage_dir")] + pub storage_dir: Option, + #[serde(default = "default_default_user_agent", skip_serializing_if = "is_blank_optional_string")] + pub default_user_agent: Option, + #[serde(default = "default_main_backup_dir", skip_serializing_if = "is_blank_or_default_backup_dir")] + pub backup_dir: Option, + #[serde(default = "default_main_user_config_dir", skip_serializing_if = "is_blank_or_default_user_config_dir")] + pub user_config_dir: Option, + #[serde(default = "default_main_mapping_path", skip_serializing_if = "is_blank_or_default_mapping_path")] + pub mapping_path: Option, + #[serde(default = "default_main_template_path", skip_serializing_if = "is_blank_or_default_template_path")] + pub template_path: Option, + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub custom_stream_response_path: Option, + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub custom_stream_response_timeout_secs: u32, + #[serde(default = "default_as_true", skip_serializing_if = "is_true")] + pub custom_stream_response_enabled: bool, + #[serde( + default = "default_custom_stream_response_error_status", + skip_serializing_if = "is_default_custom_stream_response_error_status" + )] + pub custom_stream_response_error_status: u16, + #[serde(default, skip_serializing_if = "is_false")] + pub user_access_control: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub disk_based_processing: bool, + #[serde(default = "default_connect_timeout_secs", skip_serializing_if = "is_default_connect_timeout_secs")] + pub connect_timeout_secs: u32, + #[serde( + default = "default_interner_gc_interval_secs", + skip_serializing_if = "is_default_interner_gc_interval_secs" + )] + pub interner_gc_interval_secs: u32, + #[serde( + default = "default_interner_gc_min_pool_size", + skip_serializing_if = "is_default_interner_gc_min_pool_size" + )] + pub interner_gc_min_pool_size: u32, + #[serde(default = "default_event_channel_capacity", skip_serializing_if = "is_default_event_channel_capacity")] + pub event_channel_capacity: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sleep_timer_mins: Option, + #[serde(default, skip_serializing_if = "is_false")] + pub update_on_boot: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub config_hot_reload: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub accept_insecure_ssl_certificates: bool, +} + +impl Default for MainConfigDto { + fn default() -> Self { + MainConfigDto { + process_parallel: false, + disk_based_processing: false, + storage_dir: default_main_storage_dir(), + default_user_agent: default_default_user_agent(), + backup_dir: default_main_backup_dir(), + user_config_dir: default_main_user_config_dir(), + mapping_path: default_main_mapping_path(), + template_path: default_main_template_path(), + custom_stream_response_path: None, + custom_stream_response_timeout_secs: 0, + custom_stream_response_enabled: true, + custom_stream_response_error_status: default_custom_stream_response_error_status(), + user_access_control: false, + connect_timeout_secs: default_connect_timeout_secs(), + interner_gc_interval_secs: default_interner_gc_interval_secs(), + event_channel_capacity: default_event_channel_capacity(), + interner_gc_min_pool_size: default_interner_gc_min_pool_size(), + sleep_timer_mins: None, + update_on_boot: false, + config_hot_reload: false, + accept_insecure_ssl_certificates: false, + } + } +} + +impl From<&ConfigDto> for MainConfigDto { + fn from(config: &ConfigDto) -> Self { + // `recording` is intentionally NOT mirrored: the main-config form + // owns simple scalar settings only (see the struct-level comment). + Self { + process_parallel: config.process_parallel, + disk_based_processing: config.disk_based_processing, + storage_dir: config.storage_dir.clone(), + default_user_agent: config.default_user_agent.clone(), + backup_dir: config.backup_dir.clone(), + user_config_dir: config.user_config_dir.clone(), + mapping_path: config.mapping_path.clone(), + template_path: config.template_path.clone(), + custom_stream_response_path: config.custom_stream_response_path.clone(), + custom_stream_response_timeout_secs: config.custom_stream_response_timeout_secs, + custom_stream_response_enabled: config.custom_stream_response_enabled, + custom_stream_response_error_status: config.custom_stream_response_error_status, + user_access_control: config.user_access_control, + connect_timeout_secs: config.connect_timeout_secs, + interner_gc_interval_secs: config.interner_gc_interval_secs, + event_channel_capacity: config.event_channel_capacity, + interner_gc_min_pool_size: config.interner_gc_min_pool_size, + sleep_timer_mins: config.sleep_timer_mins, + update_on_boot: config.update_on_boot, + config_hot_reload: config.config_hot_reload, + accept_insecure_ssl_certificates: config.accept_insecure_ssl_certificates, + } + } +} + +// This SchedulesConfigDto is a copy of ConfigDto schedules fields for form editing. +// It has no other purpose than editing and saving the schedules +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] +pub struct SchedulesConfigDto { + #[serde(default)] + pub schedules: Option>, +} + +impl SchedulesConfigDto { + // Clippy's method-path suggestion here names a private module and does not + // compile; the closure is kept deliberately. + #[allow(clippy::redundant_closure_for_method_calls)] + pub fn is_empty(&self) -> bool { self.schedules.as_deref().is_none_or(|s| s.is_empty()) } +} + +impl From<&ConfigDto> for SchedulesConfigDto { + fn from(config: &ConfigDto) -> Self { Self { schedules: config.schedules.clone() } } +} + +pub struct HdHomeRunDeviceOverview { + pub enabled: bool, + pub devices: Vec, +} diff --git a/shared/src/model/config/epg.rs b/shared/src/model/config/epg.rs index e850dd90f..89a74f4ed 100644 --- a/shared/src/model/config/epg.rs +++ b/shared/src/model/config/epg.rs @@ -1,20 +1,4 @@ -use crate::{ - defaults::{ - default_ics_dummy_block_hours, default_ics_dummy_days_future, default_ics_dummy_days_past, - default_ics_dummy_min_gap_minutes, default_ics_dummy_title, default_ics_event_description, - default_ics_event_title, default_ics_max_decompressed_bytes, default_ics_max_download_bytes, - default_ics_max_events, default_ics_timezone, is_default_ics_dummy_block_hours, - is_default_ics_dummy_days_future, is_default_ics_dummy_days_past, is_default_ics_dummy_min_gap_minutes, - is_default_ics_dummy_title, is_default_ics_event_description, is_default_ics_event_title, - is_default_ics_max_decompressed_bytes, is_default_ics_max_download_bytes, is_default_ics_max_events, - is_default_ics_timezone, is_false, MAX_ICS_DAYS_FUTURE, MAX_ICS_DAYS_PAST, - MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT, MAX_ICS_DESCRIPTION_LENGTH, MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT, - MAX_ICS_EVENTS_HARD_LIMIT, MAX_ICS_SUMMARY_LENGTH, - }, - error::TuliproxError, - model::EpgSmartMatchConfigDto, - utils::{is_blank_optional_string, sanitize_sensitive_info}, -}; +use crate::{defaults::is_false, model::EpgSmartMatchConfigDto, utils::is_blank_optional_string}; const AUTO_URL: &str = "auto"; @@ -61,179 +45,6 @@ impl Default for EpgSourceDto { } } -impl EpgSourceDto { - pub fn prepare(&mut self) -> Result<(), TuliproxError> { - self.url = self.url.trim().to_string(); - self.channel_id = - self.channel_id.take().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()); - self.channel_title = - self.channel_title.take().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()); - self.match_names = self - .match_names - .drain(..) - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) - .collect(); - - match self.source_type { - EpgSourceTypeDto::Xmltv => self.prepare_xmltv(), - EpgSourceTypeDto::Ics => self.prepare_ics(), - } - } - - fn prepare_xmltv(&self) -> Result<(), TuliproxError> { - if self.url.is_empty() { - return Err(TuliproxError::ConfigEpg("XMLTV EPG source url is empty".to_string())); - } - if self.channel_id.is_some() - || self.channel_title.is_some() - || !self.match_names.is_empty() - || self.ics.is_some() - { - return Err(TuliproxError::ConfigEpg( - "channel_id, channel_title, match_names, and ics are only supported for ICS EPG sources".to_string(), - )); - } - Ok(()) - } - - fn prepare_ics(&mut self) -> Result<(), TuliproxError> { - if self.url.is_empty() { - return Err(TuliproxError::ConfigEpg("ICS EPG source url is empty".to_string())); - } - if self.url.eq_ignore_ascii_case(AUTO_URL) { - return Err(TuliproxError::ConfigEpg( - "url: auto is only supported for XMLTV EPG sources, not for ICS".to_string(), - )); - } - if let Some(rest) = strip_prefix_ignore_ascii_case(&self.url, "webcal://") { - self.url = format!("https://{rest}"); - } - if self.channel_id.as_deref().is_none_or(str::is_empty) { - return Err(TuliproxError::ConfigEpg("channel_id is required for ICS EPG sources".to_string())); - } - let ics = self.ics.get_or_insert_with(IcsEpgSourceConfigDto::default); - ics.timezone = ics.timezone.trim().to_string(); - validate_ics_config(ics)?; - validate_ics_url_scheme(&self.url) - } - - pub fn is_valid(&self) -> bool { !self.url.is_empty() } - - pub fn source_identity(&self) -> String { - match self.source_type { - EpgSourceTypeDto::Xmltv => format!("xmltv|{}", self.url), - EpgSourceTypeDto::Ics => { - format!("ics|{}|{}", self.url, self.channel_id.as_deref().unwrap_or_default()) - } - } - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct IcsEpgSourceConfigDto { - #[serde(default = "default_ics_timezone", skip_serializing_if = "is_default_ics_timezone")] - pub timezone: String, - #[serde(default, skip_serializing_if = "IcsEventMappingDto::is_default")] - pub event: IcsEventMappingDto, - #[serde(default, skip_serializing_if = "IcsDummyConfigDto::is_default")] - pub dummy: IcsDummyConfigDto, - #[serde(default, skip_serializing_if = "is_false")] - pub include_cancelled: bool, - #[serde(default = "default_ics_max_events", skip_serializing_if = "is_default_ics_max_events")] - pub max_events: usize, - #[serde(default = "default_ics_max_download_bytes", skip_serializing_if = "is_default_ics_max_download_bytes")] - pub max_download_bytes: u64, - #[serde( - default = "default_ics_max_decompressed_bytes", - skip_serializing_if = "is_default_ics_max_decompressed_bytes" - )] - pub max_decompressed_bytes: usize, -} - -impl Default for IcsEpgSourceConfigDto { - fn default() -> Self { - Self { - timezone: default_ics_timezone(), - event: IcsEventMappingDto::default(), - dummy: IcsDummyConfigDto::default(), - include_cancelled: false, - max_events: default_ics_max_events(), - max_download_bytes: default_ics_max_download_bytes(), - max_decompressed_bytes: default_ics_max_decompressed_bytes(), - } - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct IcsEventMappingDto { - #[serde(default = "default_ics_event_title", skip_serializing_if = "is_default_ics_event_title")] - pub title: String, - #[serde(default = "default_ics_event_description", skip_serializing_if = "is_default_ics_event_description")] - pub description: String, - #[serde(default, skip_serializing_if = "is_false")] - pub include_location: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub include_categories: bool, -} - -impl Default for IcsEventMappingDto { - fn default() -> Self { - Self { - title: default_ics_event_title(), - description: default_ics_event_description(), - include_location: false, - include_categories: false, - } - } -} - -impl IcsEventMappingDto { - pub fn is_default(value: &Self) -> bool { value == &Self::default() } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct IcsDummyConfigDto { - #[serde(default, skip_serializing_if = "is_false")] - pub enabled: bool, - #[serde(default = "default_ics_dummy_title", skip_serializing_if = "is_default_ics_dummy_title")] - pub title: String, - #[serde(default, skip_serializing_if = "String::is_empty")] - pub description: String, - #[serde(default = "default_ics_dummy_days_past", skip_serializing_if = "is_default_ics_dummy_days_past")] - pub days_past: u16, - #[serde(default = "default_ics_dummy_days_future", skip_serializing_if = "is_default_ics_dummy_days_future")] - pub days_future: u16, - #[serde(default = "default_ics_dummy_block_hours", skip_serializing_if = "is_default_ics_dummy_block_hours")] - pub block_hours: u8, - #[serde( - default = "default_ics_dummy_min_gap_minutes", - skip_serializing_if = "is_default_ics_dummy_min_gap_minutes" - )] - pub min_gap_minutes: u16, -} - -impl Default for IcsDummyConfigDto { - fn default() -> Self { - Self { - enabled: false, - title: default_ics_dummy_title(), - description: String::new(), - days_past: default_ics_dummy_days_past(), - days_future: default_ics_dummy_days_future(), - block_hours: default_ics_dummy_block_hours(), - min_gap_minutes: default_ics_dummy_min_gap_minutes(), - } - } -} - -impl IcsDummyConfigDto { - pub fn is_default(value: &Self) -> bool { value == &Self::default() } -} - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Default)] #[serde(deny_unknown_fields)] pub struct EpgConfigDto { @@ -245,398 +56,9 @@ pub struct EpgConfigDto { pub t_sources: Vec, } -impl EpgConfigDto { - /// Prepares the EPG configuration by resolving all source URLs into `t_sources`. - /// - /// - `create_auto_url` derives an XMLTV URL from the parent input when XMLTV `url` is `auto`. - /// - `include_computed` skips resolution when serialisation round-trips do not need computed URLs. - pub fn prepare(&mut self, create_auto_url: F, include_computed: bool) -> Result<(), TuliproxError> - where - F: Fn() -> Result, - { - if include_computed { - self.t_sources = Vec::new(); - if let Some(epg_sources) = self.sources.as_mut() { - for epg_source in epg_sources.iter_mut() { - epg_source.prepare()?; - if !epg_source.is_valid() { - continue; - } - - if epg_source.source_type == EpgSourceTypeDto::Xmltv - && epg_source.url.eq_ignore_ascii_case(AUTO_URL) - { - match create_auto_url() { - Ok(provider_url) => { - let mut resolved = epg_source.clone(); - resolved.url = provider_url; - self.t_sources.push(resolved); - } - Err(err) => return Err(TuliproxError::ConfigEpg(err.clone())), - } - } else { - self.t_sources.push(epg_source.clone()); - } - } - } - - if let Some(smart_match) = self.smart_match.as_mut() { - smart_match.prepare()?; - } - } - Ok(()) - } -} - -fn validate_ics_config(config: &IcsEpgSourceConfigDto) -> Result<(), TuliproxError> { - validate_ics_timezone(&config.timezone)?; - let block_hours = config.dummy.block_hours; - if block_hours == 0 || block_hours > 24 || 24 % block_hours != 0 { - return Err(TuliproxError::ConfigEpg(format!( - "ics.dummy.block_hours must divide 24 evenly, got {block_hours}" - ))); - } - if config.max_events == 0 { - return Err(TuliproxError::ConfigEpg("ics.max_events must be greater than 0".to_string())); - } - if config.max_events > MAX_ICS_EVENTS_HARD_LIMIT { - return Err(TuliproxError::ConfigEpg(format!("ics.max_events must not exceed {MAX_ICS_EVENTS_HARD_LIMIT}"))); - } - if config.max_download_bytes == 0 { - return Err(TuliproxError::ConfigEpg("ics.max_download_bytes must be greater than 0".to_string())); - } - if config.max_download_bytes > MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT { - return Err(TuliproxError::ConfigEpg(format!( - "ics.max_download_bytes must not exceed {MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT}" - ))); - } - if config.max_decompressed_bytes == 0 { - return Err(TuliproxError::ConfigEpg("ics.max_decompressed_bytes must be greater than 0".to_string())); - } - if config.max_decompressed_bytes > MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT { - return Err(TuliproxError::ConfigEpg(format!( - "ics.max_decompressed_bytes must not exceed {MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT}" - ))); - } - if config.dummy.days_past > MAX_ICS_DAYS_PAST { - return Err(TuliproxError::ConfigEpg(format!("ics.dummy.days_past must not exceed {MAX_ICS_DAYS_PAST}"))); - } - if config.dummy.days_future > MAX_ICS_DAYS_FUTURE { - return Err(TuliproxError::ConfigEpg(format!("ics.dummy.days_future must not exceed {MAX_ICS_DAYS_FUTURE}"))); - } - validate_text_limit("ics.event.title", &config.event.title, MAX_ICS_SUMMARY_LENGTH)?; - validate_text_limit("ics.event.description", &config.event.description, MAX_ICS_DESCRIPTION_LENGTH)?; - validate_text_limit("ics.dummy.title", &config.dummy.title, MAX_ICS_SUMMARY_LENGTH)?; - validate_text_limit("ics.dummy.description", &config.dummy.description, MAX_ICS_DESCRIPTION_LENGTH)?; - Ok(()) -} - -fn validate_ics_timezone(timezone: &str) -> Result<(), TuliproxError> { - if timezone.is_empty() { - return Err(TuliproxError::ConfigEpg("ics.timezone must not be empty".to_string())); - } - timezone - .parse::() - .map(|_| ()) - .map_err(|_| TuliproxError::ConfigEpg(format!("ics.timezone '{timezone}' is not a valid IANA timezone"))) -} - -fn validate_text_limit(field: &str, value: &str, max_len: usize) -> Result<(), TuliproxError> { - if value.len() > max_len { - return Err(TuliproxError::ConfigEpg(format!("{field} must not exceed {max_len} bytes"))); - } - Ok(()) -} - -fn validate_ics_url_scheme(url: &str) -> Result<(), TuliproxError> { - if is_absolute_local_path(url) { - return Ok(()); - } - - let Ok(parsed) = url::Url::parse(url) else { - return Ok(()); - }; - - match parsed.scheme() { - "https" | "http" | "file" | "provider" => Ok(()), - scheme => Err(TuliproxError::ConfigEpg(format!( - "Unsupported ICS url scheme '{scheme}' for {}", - sanitize_sensitive_info(url) - ))), - } -} - -fn strip_prefix_ignore_ascii_case<'a>(value: &'a str, prefix: &str) -> Option<&'a str> { - let candidate = value.get(..prefix.len())?; - if !candidate.eq_ignore_ascii_case(prefix) { - return None; - } - value.get(prefix.len()..) -} - -fn is_absolute_local_path(value: &str) -> bool { - if std::path::Path::new(value).is_absolute() { - return true; - } - - let bytes = value.as_bytes(); - let windows_drive_path = - bytes.len() >= 3 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' && matches!(bytes[2], b'\\' | b'/'); - windows_drive_path || bytes.starts_with(b"\\\\") -} - #[cfg(test)] -mod tests { - use super::*; +mod tests; - fn xmltv_source(url: &str) -> EpgSourceDto { EpgSourceDto { url: url.to_owned(), ..EpgSourceDto::default() } } - - fn ics_source(url: &str, channel_id: Option<&str>) -> EpgSourceDto { - EpgSourceDto { - source_type: EpgSourceTypeDto::Ics, - url: url.to_owned(), - channel_id: channel_id.map(ToOwned::to_owned), - ..EpgSourceDto::default() - } - } - - #[test] - fn xmltv_source_without_type_remains_valid() { - let mut cfg = - EpgConfigDto { sources: Some(vec![xmltv_source("http://example.com/xmltv.php")]), ..Default::default() }; - cfg.prepare(|| Err("no auto".to_owned()), true).expect("prepare failed"); - assert_eq!(cfg.t_sources.len(), 1); - assert_eq!(cfg.t_sources[0].source_type, EpgSourceTypeDto::Xmltv); - assert_eq!(cfg.t_sources[0].url, "http://example.com/xmltv.php"); - } - - #[test] - fn provider_scheme_kept_unresolved() { - let mut source = xmltv_source("provider://myprovider/xmltv.php?username=u&password=p"); - source.priority = 1; - source.logo_override = true; - let mut cfg = EpgConfigDto { sources: Some(vec![source]), ..Default::default() }; - cfg.prepare(|| Err("no auto".to_owned()), true).expect("prepare failed"); - assert_eq!(cfg.t_sources.len(), 1); - assert_eq!(cfg.t_sources[0].url, "provider://myprovider/xmltv.php?username=u&password=p"); - assert_eq!(cfg.t_sources[0].priority, 1); - assert!(cfg.t_sources[0].logo_override); - } - - #[test] - fn xmltv_auto_url_used() { - let mut cfg = EpgConfigDto { sources: Some(vec![xmltv_source(AUTO_URL)]), ..Default::default() }; - cfg.prepare(|| Ok("http://auto.example.com/xmltv.php?username=u&password=p".to_owned()), true) - .expect("prepare failed"); - assert_eq!(cfg.t_sources.len(), 1); - assert!(cfg.t_sources[0].url.starts_with("http://auto.example.com/")); - } - - #[test] - fn include_computed_false_skips_resolution() { - let mut cfg = - EpgConfigDto { sources: Some(vec![xmltv_source("provider://myprovider/xmltv.php")]), ..Default::default() }; - cfg.prepare(|| Err("no auto".to_owned()), false).expect("prepare with include_computed=false should succeed"); - assert!(cfg.t_sources.is_empty()); - } - - #[test] - fn ics_auto_url_is_rejected() { - let mut cfg = - EpgConfigDto { sources: Some(vec![ics_source(AUTO_URL, Some("f1.calendar"))]), ..Default::default() }; - let err = cfg.prepare(|| Ok("http://auto.example.com/xmltv.php".to_owned()), true).unwrap_err(); - assert!(err.to_string().contains("only supported for XMLTV")); - } - - #[test] - fn ics_requires_channel_id() { - let mut cfg = - EpgConfigDto { sources: Some(vec![ics_source("https://example.com/f1.ics", None)]), ..Default::default() }; - let err = cfg.prepare(|| Err("no auto".to_owned()), true).unwrap_err(); - assert!(err.to_string().contains("channel_id is required")); - } - - #[test] - fn ics_webcal_url_is_normalized_to_https() { - for url in ["webcal://example.com/f1.ics", "WEBcal://example.com/f1.ics"] { - let mut cfg = - EpgConfigDto { sources: Some(vec![ics_source(url, Some("f1.calendar"))]), ..Default::default() }; - cfg.prepare(|| Err("no auto".to_owned()), true).expect("prepare failed"); - assert_eq!(cfg.t_sources[0].url, "https://example.com/f1.ics"); - } - } - - #[test] - fn ics_unicode_local_path_is_accepted_without_panicking() { - let mut source = ics_source("äääää/calendar.ics", Some("calendar")); - source.prepare().expect("unicode local path should be valid"); - } - - #[test] - fn ics_absolute_windows_paths_are_local_sources() { - for path in [r"C:\calendar.ics", "D:/calendar.ics", r"\\server\share\calendar.ics"] { - let mut source = ics_source(path, Some("calendar")); - source.prepare().expect("absolute Windows path should be valid"); - } - } - - #[test] - fn xmltv_rejects_ics_only_fields() { - let mut with_ics_block = xmltv_source("https://example.com/epg.xml"); - with_ics_block.ics = Some(IcsEpgSourceConfigDto::default()); - let err = with_ics_block.prepare().unwrap_err(); - assert!(err.to_string().contains("only supported for ICS")); - - let mut with_channel_id = xmltv_source("https://example.com/epg.xml"); - with_channel_id.channel_id = Some("not-an-xmltv-option".to_string()); - let err = with_channel_id.prepare().unwrap_err(); - assert!(err.to_string().contains("only supported for ICS")); - } - - #[test] - fn ics_unsupported_scheme_is_rejected() { - let mut cfg = EpgConfigDto { - sources: Some(vec![ics_source("ftp://example.com/f1.ics", Some("f1.calendar"))]), - ..Default::default() - }; - let err = cfg.prepare(|| Err("no auto".to_owned()), true).unwrap_err(); - assert!(err.to_string().contains("Unsupported ICS url scheme")); - } - - #[test] - fn ics_match_names_are_trimmed_and_empty_values_removed() { - let mut source = ics_source("https://example.com/f1.ics", Some(" f1.calendar ")); - source.channel_title = Some(" Formula 1 ".to_owned()); - source.match_names = vec![" F1 ".to_owned(), String::new(), " ".to_owned(), "Formel 1".to_owned()]; - let mut cfg = EpgConfigDto { sources: Some(vec![source]), ..Default::default() }; - cfg.prepare(|| Err("no auto".to_owned()), true).expect("prepare failed"); - assert_eq!(cfg.t_sources[0].channel_id.as_deref(), Some("f1.calendar")); - assert_eq!(cfg.t_sources[0].channel_title.as_deref(), Some("Formula 1")); - assert_eq!(cfg.t_sources[0].match_names, vec!["F1", "Formel 1"]); - } - - #[test] - fn aliases_field_is_rejected() { - let yaml = r" -type: ics -url: https://example.com/f1.ics -channel_id: f1.calendar -aliases: - - F1 -"; - let err = serde_saphyr::from_str::(yaml).unwrap_err(); - assert!(err.to_string().contains("aliases") || err.to_string().contains("unknown field")); - } - - #[test] - fn ics_block_hours_must_divide_day_evenly() { - let mut valid = ics_source("https://example.com/f1.ics", Some("f1.calendar")); - valid.ics = Some(IcsEpgSourceConfigDto { - dummy: IcsDummyConfigDto { block_hours: 4, ..IcsDummyConfigDto::default() }, - ..IcsEpgSourceConfigDto::default() - }); - valid.prepare().expect("valid block size"); - - let mut invalid = ics_source("https://example.com/f1.ics", Some("f1.calendar")); - invalid.ics = Some(IcsEpgSourceConfigDto { - dummy: IcsDummyConfigDto { block_hours: 5, ..IcsDummyConfigDto::default() }, - ..IcsEpgSourceConfigDto::default() - }); - let err = invalid.prepare().unwrap_err(); - assert!(err.to_string().contains("must divide 24 evenly")); - } - - #[test] - fn invalid_ics_timezone_is_rejected() { - let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); - source.ics = - Some(IcsEpgSourceConfigDto { timezone: "Mars/Olympus".to_string(), ..IcsEpgSourceConfigDto::default() }); - - let err = source.prepare().unwrap_err(); - - assert!(err.to_string().contains("ics.timezone")); - } - - #[test] - fn ics_max_download_bytes_above_hard_cap_is_rejected() { - let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); - source.ics = Some(IcsEpgSourceConfigDto { - max_download_bytes: MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT + 1, - ..IcsEpgSourceConfigDto::default() - }); - - let err = source.prepare().unwrap_err(); - - assert!(err.to_string().contains("ics.max_download_bytes")); - } - - #[test] - fn ics_max_decompressed_bytes_above_hard_cap_is_rejected() { - let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); - source.ics = Some(IcsEpgSourceConfigDto { - max_decompressed_bytes: MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT + 1, - ..IcsEpgSourceConfigDto::default() - }); - - let err = source.prepare().unwrap_err(); - - assert!(err.to_string().contains("ics.max_decompressed_bytes")); - } - - #[test] - fn ics_max_events_above_hard_cap_is_rejected() { - let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); - source.ics = Some(IcsEpgSourceConfigDto { - max_events: MAX_ICS_EVENTS_HARD_LIMIT + 1, - ..IcsEpgSourceConfigDto::default() - }); - - let err = source.prepare().unwrap_err(); - - assert!(err.to_string().contains("ics.max_events")); - } - - #[test] - fn ics_dummy_days_above_hard_caps_are_rejected() { - let mut too_much_past = ics_source("https://example.com/f1.ics", Some("f1.calendar")); - too_much_past.ics = Some(IcsEpgSourceConfigDto { - dummy: IcsDummyConfigDto { days_past: MAX_ICS_DAYS_PAST + 1, ..IcsDummyConfigDto::default() }, - ..IcsEpgSourceConfigDto::default() - }); - let err = too_much_past.prepare().unwrap_err(); - assert!(err.to_string().contains("ics.dummy.days_past")); - - let mut too_much_future = ics_source("https://example.com/f1.ics", Some("f1.calendar")); - too_much_future.ics = Some(IcsEpgSourceConfigDto { - dummy: IcsDummyConfigDto { days_future: MAX_ICS_DAYS_FUTURE + 1, ..IcsDummyConfigDto::default() }, - ..IcsEpgSourceConfigDto::default() - }); - let err = too_much_future.prepare().unwrap_err(); - assert!(err.to_string().contains("ics.dummy.days_future")); - } - - #[test] - fn ics_extreme_template_and_dummy_texts_are_rejected() { - let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); - source.ics = Some(IcsEpgSourceConfigDto { - event: IcsEventMappingDto { - title: "x".repeat(MAX_ICS_SUMMARY_LENGTH + 1), - ..IcsEventMappingDto::default() - }, - ..IcsEpgSourceConfigDto::default() - }); - let err = source.prepare().unwrap_err(); - assert!(err.to_string().contains("ics.event.title")); - - let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); - source.ics = Some(IcsEpgSourceConfigDto { - dummy: IcsDummyConfigDto { - description: "x".repeat(MAX_ICS_DESCRIPTION_LENGTH + 1), - ..IcsDummyConfigDto::default() - }, - ..IcsEpgSourceConfigDto::default() - }); - let err = source.prepare().unwrap_err(); - assert!(err.to_string().contains("ics.dummy.description")); - } -} +mod ics; +mod prepare; +pub use ics::{IcsDummyConfigDto, IcsEpgSourceConfigDto, IcsEventMappingDto}; diff --git a/shared/src/model/config/epg/ics.rs b/shared/src/model/config/epg/ics.rs new file mode 100644 index 000000000..7ab7a5782 --- /dev/null +++ b/shared/src/model/config/epg/ics.rs @@ -0,0 +1,207 @@ +use super::prepare::is_absolute_local_path; +use crate::{ + defaults::{ + default_ics_dummy_block_hours, default_ics_dummy_days_future, default_ics_dummy_days_past, + default_ics_dummy_min_gap_minutes, default_ics_dummy_title, default_ics_event_description, + default_ics_event_title, default_ics_max_decompressed_bytes, default_ics_max_download_bytes, + default_ics_max_events, default_ics_timezone, is_default_ics_dummy_block_hours, + is_default_ics_dummy_days_future, is_default_ics_dummy_days_past, is_default_ics_dummy_min_gap_minutes, + is_default_ics_dummy_title, is_default_ics_event_description, is_default_ics_event_title, + is_default_ics_max_decompressed_bytes, is_default_ics_max_download_bytes, is_default_ics_max_events, + is_default_ics_timezone, is_false, MAX_ICS_DAYS_FUTURE, MAX_ICS_DAYS_PAST, + MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT, MAX_ICS_DESCRIPTION_LENGTH, MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT, + MAX_ICS_EVENTS_HARD_LIMIT, MAX_ICS_SUMMARY_LENGTH, + }, + error::TuliproxError, + utils::sanitize_sensitive_info, +}; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct IcsEpgSourceConfigDto { + #[serde(default = "default_ics_timezone", skip_serializing_if = "is_default_ics_timezone")] + pub timezone: String, + #[serde(default, skip_serializing_if = "IcsEventMappingDto::is_default")] + pub event: IcsEventMappingDto, + #[serde(default, skip_serializing_if = "IcsDummyConfigDto::is_default")] + pub dummy: IcsDummyConfigDto, + #[serde(default, skip_serializing_if = "is_false")] + pub include_cancelled: bool, + #[serde(default = "default_ics_max_events", skip_serializing_if = "is_default_ics_max_events")] + pub max_events: usize, + #[serde(default = "default_ics_max_download_bytes", skip_serializing_if = "is_default_ics_max_download_bytes")] + pub max_download_bytes: u64, + #[serde( + default = "default_ics_max_decompressed_bytes", + skip_serializing_if = "is_default_ics_max_decompressed_bytes" + )] + pub max_decompressed_bytes: usize, +} + +impl Default for IcsEpgSourceConfigDto { + fn default() -> Self { + Self { + timezone: default_ics_timezone(), + event: IcsEventMappingDto::default(), + dummy: IcsDummyConfigDto::default(), + include_cancelled: false, + max_events: default_ics_max_events(), + max_download_bytes: default_ics_max_download_bytes(), + max_decompressed_bytes: default_ics_max_decompressed_bytes(), + } + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct IcsEventMappingDto { + #[serde(default = "default_ics_event_title", skip_serializing_if = "is_default_ics_event_title")] + pub title: String, + #[serde(default = "default_ics_event_description", skip_serializing_if = "is_default_ics_event_description")] + pub description: String, + #[serde(default, skip_serializing_if = "is_false")] + pub include_location: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub include_categories: bool, +} + +impl Default for IcsEventMappingDto { + fn default() -> Self { + Self { + title: default_ics_event_title(), + description: default_ics_event_description(), + include_location: false, + include_categories: false, + } + } +} + +impl IcsEventMappingDto { + pub fn is_default(value: &Self) -> bool { value == &Self::default() } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct IcsDummyConfigDto { + #[serde(default, skip_serializing_if = "is_false")] + pub enabled: bool, + #[serde(default = "default_ics_dummy_title", skip_serializing_if = "is_default_ics_dummy_title")] + pub title: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub description: String, + #[serde(default = "default_ics_dummy_days_past", skip_serializing_if = "is_default_ics_dummy_days_past")] + pub days_past: u16, + #[serde(default = "default_ics_dummy_days_future", skip_serializing_if = "is_default_ics_dummy_days_future")] + pub days_future: u16, + #[serde(default = "default_ics_dummy_block_hours", skip_serializing_if = "is_default_ics_dummy_block_hours")] + pub block_hours: u8, + #[serde( + default = "default_ics_dummy_min_gap_minutes", + skip_serializing_if = "is_default_ics_dummy_min_gap_minutes" + )] + pub min_gap_minutes: u16, +} + +impl Default for IcsDummyConfigDto { + fn default() -> Self { + Self { + enabled: false, + title: default_ics_dummy_title(), + description: String::new(), + days_past: default_ics_dummy_days_past(), + days_future: default_ics_dummy_days_future(), + block_hours: default_ics_dummy_block_hours(), + min_gap_minutes: default_ics_dummy_min_gap_minutes(), + } + } +} + +impl IcsDummyConfigDto { + pub fn is_default(value: &Self) -> bool { value == &Self::default() } +} + +pub(super) fn validate_ics_config(config: &IcsEpgSourceConfigDto) -> Result<(), TuliproxError> { + validate_ics_timezone(&config.timezone)?; + let block_hours = config.dummy.block_hours; + if block_hours == 0 || block_hours > 24 || 24 % block_hours != 0 { + return Err(TuliproxError::ConfigEpg(format!( + "ics.dummy.block_hours must divide 24 evenly, got {block_hours}" + ))); + } + if config.max_events == 0 { + return Err(TuliproxError::ConfigEpg("ics.max_events must be greater than 0".to_string())); + } + if config.max_events > MAX_ICS_EVENTS_HARD_LIMIT { + return Err(TuliproxError::ConfigEpg(format!("ics.max_events must not exceed {MAX_ICS_EVENTS_HARD_LIMIT}"))); + } + if config.max_download_bytes == 0 { + return Err(TuliproxError::ConfigEpg("ics.max_download_bytes must be greater than 0".to_string())); + } + if config.max_download_bytes > MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT { + return Err(TuliproxError::ConfigEpg(format!( + "ics.max_download_bytes must not exceed {MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT}" + ))); + } + if config.max_decompressed_bytes == 0 { + return Err(TuliproxError::ConfigEpg("ics.max_decompressed_bytes must be greater than 0".to_string())); + } + if config.max_decompressed_bytes > MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT { + return Err(TuliproxError::ConfigEpg(format!( + "ics.max_decompressed_bytes must not exceed {MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT}" + ))); + } + if config.dummy.days_past > MAX_ICS_DAYS_PAST { + return Err(TuliproxError::ConfigEpg(format!("ics.dummy.days_past must not exceed {MAX_ICS_DAYS_PAST}"))); + } + if config.dummy.days_future > MAX_ICS_DAYS_FUTURE { + return Err(TuliproxError::ConfigEpg(format!("ics.dummy.days_future must not exceed {MAX_ICS_DAYS_FUTURE}"))); + } + validate_text_limit("ics.event.title", &config.event.title, MAX_ICS_SUMMARY_LENGTH)?; + validate_text_limit("ics.event.description", &config.event.description, MAX_ICS_DESCRIPTION_LENGTH)?; + validate_text_limit("ics.dummy.title", &config.dummy.title, MAX_ICS_SUMMARY_LENGTH)?; + validate_text_limit("ics.dummy.description", &config.dummy.description, MAX_ICS_DESCRIPTION_LENGTH)?; + Ok(()) +} + +fn validate_ics_timezone(timezone: &str) -> Result<(), TuliproxError> { + if timezone.is_empty() { + return Err(TuliproxError::ConfigEpg("ics.timezone must not be empty".to_string())); + } + timezone + .parse::() + .map(|_| ()) + .map_err(|_| TuliproxError::ConfigEpg(format!("ics.timezone '{timezone}' is not a valid IANA timezone"))) +} + +fn validate_text_limit(field: &str, value: &str, max_len: usize) -> Result<(), TuliproxError> { + if value.len() > max_len { + return Err(TuliproxError::ConfigEpg(format!("{field} must not exceed {max_len} bytes"))); + } + Ok(()) +} + +pub(super) fn validate_ics_url_scheme(url: &str) -> Result<(), TuliproxError> { + if is_absolute_local_path(url) { + return Ok(()); + } + + let Ok(parsed) = url::Url::parse(url) else { + return Ok(()); + }; + + match parsed.scheme() { + "https" | "http" | "file" | "provider" => Ok(()), + scheme => Err(TuliproxError::ConfigEpg(format!( + "Unsupported ICS url scheme '{scheme}' for {}", + sanitize_sensitive_info(url) + ))), + } +} + +pub(super) fn strip_prefix_ignore_ascii_case<'a>(value: &'a str, prefix: &str) -> Option<&'a str> { + let candidate = value.get(..prefix.len())?; + if !candidate.eq_ignore_ascii_case(prefix) { + return None; + } + value.get(prefix.len()..) +} diff --git a/shared/src/model/config/epg/prepare.rs b/shared/src/model/config/epg/prepare.rs new file mode 100644 index 000000000..c5cd21107 --- /dev/null +++ b/shared/src/model/config/epg/prepare.rs @@ -0,0 +1,128 @@ +use super::{ + ics::{strip_prefix_ignore_ascii_case, validate_ics_config, validate_ics_url_scheme}, + EpgConfigDto, EpgSourceDto, EpgSourceTypeDto, IcsEpgSourceConfigDto, AUTO_URL, +}; +use crate::error::TuliproxError; + +impl EpgSourceDto { + pub fn prepare(&mut self) -> Result<(), TuliproxError> { + self.url = self.url.trim().to_string(); + self.channel_id = + self.channel_id.take().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()); + self.channel_title = + self.channel_title.take().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()); + self.match_names = self + .match_names + .drain(..) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .collect(); + + match self.source_type { + EpgSourceTypeDto::Xmltv => self.prepare_xmltv(), + EpgSourceTypeDto::Ics => self.prepare_ics(), + } + } + + pub(super) fn prepare_xmltv(&self) -> Result<(), TuliproxError> { + if self.url.is_empty() { + return Err(TuliproxError::ConfigEpg("XMLTV EPG source url is empty".to_string())); + } + if self.channel_id.is_some() + || self.channel_title.is_some() + || !self.match_names.is_empty() + || self.ics.is_some() + { + return Err(TuliproxError::ConfigEpg( + "channel_id, channel_title, match_names, and ics are only supported for ICS EPG sources".to_string(), + )); + } + Ok(()) + } + + pub(super) fn prepare_ics(&mut self) -> Result<(), TuliproxError> { + if self.url.is_empty() { + return Err(TuliproxError::ConfigEpg("ICS EPG source url is empty".to_string())); + } + if self.url.eq_ignore_ascii_case(AUTO_URL) { + return Err(TuliproxError::ConfigEpg( + "url: auto is only supported for XMLTV EPG sources, not for ICS".to_string(), + )); + } + if let Some(rest) = strip_prefix_ignore_ascii_case(&self.url, "webcal://") { + self.url = format!("https://{rest}"); + } + if self.channel_id.as_deref().is_none_or(str::is_empty) { + return Err(TuliproxError::ConfigEpg("channel_id is required for ICS EPG sources".to_string())); + } + let ics = self.ics.get_or_insert_with(IcsEpgSourceConfigDto::default); + ics.timezone = ics.timezone.trim().to_string(); + validate_ics_config(ics)?; + validate_ics_url_scheme(&self.url) + } + + pub fn is_valid(&self) -> bool { !self.url.is_empty() } + + pub fn source_identity(&self) -> String { + match self.source_type { + EpgSourceTypeDto::Xmltv => format!("xmltv|{}", self.url), + EpgSourceTypeDto::Ics => { + format!("ics|{}|{}", self.url, self.channel_id.as_deref().unwrap_or_default()) + } + } + } +} + +impl EpgConfigDto { + /// Prepares the EPG configuration by resolving all source URLs into `t_sources`. + /// + /// - `create_auto_url` derives an XMLTV URL from the parent input when XMLTV `url` is `auto`. + /// - `include_computed` skips resolution when serialisation round-trips do not need computed URLs. + pub fn prepare(&mut self, create_auto_url: F, include_computed: bool) -> Result<(), TuliproxError> + where + F: Fn() -> Result, + { + if include_computed { + self.t_sources = Vec::new(); + if let Some(epg_sources) = self.sources.as_mut() { + for epg_source in epg_sources.iter_mut() { + epg_source.prepare()?; + if !epg_source.is_valid() { + continue; + } + + if epg_source.source_type == EpgSourceTypeDto::Xmltv + && epg_source.url.eq_ignore_ascii_case(AUTO_URL) + { + match create_auto_url() { + Ok(provider_url) => { + let mut resolved = epg_source.clone(); + resolved.url = provider_url; + self.t_sources.push(resolved); + } + Err(err) => return Err(TuliproxError::ConfigEpg(err.clone())), + } + } else { + self.t_sources.push(epg_source.clone()); + } + } + } + + if let Some(smart_match) = self.smart_match.as_mut() { + smart_match.prepare()?; + } + } + Ok(()) + } +} + +pub(super) fn is_absolute_local_path(value: &str) -> bool { + if std::path::Path::new(value).is_absolute() { + return true; + } + + let bytes = value.as_bytes(); + let windows_drive_path = + bytes.len() >= 3 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' && matches!(bytes[2], b'\\' | b'/'); + windows_drive_path || bytes.starts_with(b"\\\\") +} diff --git a/shared/src/model/config/epg/tests.rs b/shared/src/model/config/epg/tests.rs new file mode 100644 index 000000000..010632123 --- /dev/null +++ b/shared/src/model/config/epg/tests.rs @@ -0,0 +1,249 @@ +use super::*; +use crate::defaults::{ + MAX_ICS_DAYS_FUTURE, MAX_ICS_DAYS_PAST, MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT, MAX_ICS_DESCRIPTION_LENGTH, + MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT, MAX_ICS_EVENTS_HARD_LIMIT, MAX_ICS_SUMMARY_LENGTH, +}; + +fn xmltv_source(url: &str) -> EpgSourceDto { EpgSourceDto { url: url.to_owned(), ..EpgSourceDto::default() } } + +fn ics_source(url: &str, channel_id: Option<&str>) -> EpgSourceDto { + EpgSourceDto { + source_type: EpgSourceTypeDto::Ics, + url: url.to_owned(), + channel_id: channel_id.map(ToOwned::to_owned), + ..EpgSourceDto::default() + } +} + +#[test] +fn xmltv_source_without_type_remains_valid() { + let mut cfg = + EpgConfigDto { sources: Some(vec![xmltv_source("http://example.com/xmltv.php")]), ..Default::default() }; + cfg.prepare(|| Err("no auto".to_owned()), true).expect("prepare failed"); + assert_eq!(cfg.t_sources.len(), 1); + assert_eq!(cfg.t_sources[0].source_type, EpgSourceTypeDto::Xmltv); + assert_eq!(cfg.t_sources[0].url, "http://example.com/xmltv.php"); +} + +#[test] +fn provider_scheme_kept_unresolved() { + let mut source = xmltv_source("provider://myprovider/xmltv.php?username=u&password=p"); + source.priority = 1; + source.logo_override = true; + let mut cfg = EpgConfigDto { sources: Some(vec![source]), ..Default::default() }; + cfg.prepare(|| Err("no auto".to_owned()), true).expect("prepare failed"); + assert_eq!(cfg.t_sources.len(), 1); + assert_eq!(cfg.t_sources[0].url, "provider://myprovider/xmltv.php?username=u&password=p"); + assert_eq!(cfg.t_sources[0].priority, 1); + assert!(cfg.t_sources[0].logo_override); +} + +#[test] +fn xmltv_auto_url_used() { + let mut cfg = EpgConfigDto { sources: Some(vec![xmltv_source(AUTO_URL)]), ..Default::default() }; + cfg.prepare(|| Ok("http://auto.example.com/xmltv.php?username=u&password=p".to_owned()), true) + .expect("prepare failed"); + assert_eq!(cfg.t_sources.len(), 1); + assert!(cfg.t_sources[0].url.starts_with("http://auto.example.com/")); +} + +#[test] +fn include_computed_false_skips_resolution() { + let mut cfg = + EpgConfigDto { sources: Some(vec![xmltv_source("provider://myprovider/xmltv.php")]), ..Default::default() }; + cfg.prepare(|| Err("no auto".to_owned()), false).expect("prepare with include_computed=false should succeed"); + assert!(cfg.t_sources.is_empty()); +} + +#[test] +fn ics_auto_url_is_rejected() { + let mut cfg = EpgConfigDto { sources: Some(vec![ics_source(AUTO_URL, Some("f1.calendar"))]), ..Default::default() }; + let err = cfg.prepare(|| Ok("http://auto.example.com/xmltv.php".to_owned()), true).unwrap_err(); + assert!(err.to_string().contains("only supported for XMLTV")); +} + +#[test] +fn ics_requires_channel_id() { + let mut cfg = + EpgConfigDto { sources: Some(vec![ics_source("https://example.com/f1.ics", None)]), ..Default::default() }; + let err = cfg.prepare(|| Err("no auto".to_owned()), true).unwrap_err(); + assert!(err.to_string().contains("channel_id is required")); +} + +#[test] +fn ics_webcal_url_is_normalized_to_https() { + for url in ["webcal://example.com/f1.ics", "WEBcal://example.com/f1.ics"] { + let mut cfg = EpgConfigDto { sources: Some(vec![ics_source(url, Some("f1.calendar"))]), ..Default::default() }; + cfg.prepare(|| Err("no auto".to_owned()), true).expect("prepare failed"); + assert_eq!(cfg.t_sources[0].url, "https://example.com/f1.ics"); + } +} + +#[test] +fn ics_unicode_local_path_is_accepted_without_panicking() { + let mut source = ics_source("äääää/calendar.ics", Some("calendar")); + source.prepare().expect("unicode local path should be valid"); +} + +#[test] +fn ics_absolute_windows_paths_are_local_sources() { + for path in [r"C:\calendar.ics", "D:/calendar.ics", r"\\server\share\calendar.ics"] { + let mut source = ics_source(path, Some("calendar")); + source.prepare().expect("absolute Windows path should be valid"); + } +} + +#[test] +fn xmltv_rejects_ics_only_fields() { + let mut with_ics_block = xmltv_source("https://example.com/epg.xml"); + with_ics_block.ics = Some(IcsEpgSourceConfigDto::default()); + let err = with_ics_block.prepare().unwrap_err(); + assert!(err.to_string().contains("only supported for ICS")); + + let mut with_channel_id = xmltv_source("https://example.com/epg.xml"); + with_channel_id.channel_id = Some("not-an-xmltv-option".to_string()); + let err = with_channel_id.prepare().unwrap_err(); + assert!(err.to_string().contains("only supported for ICS")); +} + +#[test] +fn ics_unsupported_scheme_is_rejected() { + let mut cfg = EpgConfigDto { + sources: Some(vec![ics_source("ftp://example.com/f1.ics", Some("f1.calendar"))]), + ..Default::default() + }; + let err = cfg.prepare(|| Err("no auto".to_owned()), true).unwrap_err(); + assert!(err.to_string().contains("Unsupported ICS url scheme")); +} + +#[test] +fn ics_match_names_are_trimmed_and_empty_values_removed() { + let mut source = ics_source("https://example.com/f1.ics", Some(" f1.calendar ")); + source.channel_title = Some(" Formula 1 ".to_owned()); + source.match_names = vec![" F1 ".to_owned(), String::new(), " ".to_owned(), "Formel 1".to_owned()]; + let mut cfg = EpgConfigDto { sources: Some(vec![source]), ..Default::default() }; + cfg.prepare(|| Err("no auto".to_owned()), true).expect("prepare failed"); + assert_eq!(cfg.t_sources[0].channel_id.as_deref(), Some("f1.calendar")); + assert_eq!(cfg.t_sources[0].channel_title.as_deref(), Some("Formula 1")); + assert_eq!(cfg.t_sources[0].match_names, vec!["F1", "Formel 1"]); +} + +#[test] +fn aliases_field_is_rejected() { + let yaml = r" +type: ics +url: https://example.com/f1.ics +channel_id: f1.calendar +aliases: + - F1 +"; + let err = serde_saphyr::from_str::(yaml).unwrap_err(); + assert!(err.to_string().contains("aliases") || err.to_string().contains("unknown field")); +} + +#[test] +fn ics_block_hours_must_divide_day_evenly() { + let mut valid = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + valid.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { block_hours: 4, ..IcsDummyConfigDto::default() }, + ..IcsEpgSourceConfigDto::default() + }); + valid.prepare().expect("valid block size"); + + let mut invalid = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + invalid.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { block_hours: 5, ..IcsDummyConfigDto::default() }, + ..IcsEpgSourceConfigDto::default() + }); + let err = invalid.prepare().unwrap_err(); + assert!(err.to_string().contains("must divide 24 evenly")); +} + +#[test] +fn invalid_ics_timezone_is_rejected() { + let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + source.ics = + Some(IcsEpgSourceConfigDto { timezone: "Mars/Olympus".to_string(), ..IcsEpgSourceConfigDto::default() }); + + let err = source.prepare().unwrap_err(); + + assert!(err.to_string().contains("ics.timezone")); +} + +#[test] +fn ics_max_download_bytes_above_hard_cap_is_rejected() { + let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + source.ics = Some(IcsEpgSourceConfigDto { + max_download_bytes: MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT + 1, + ..IcsEpgSourceConfigDto::default() + }); + + let err = source.prepare().unwrap_err(); + + assert!(err.to_string().contains("ics.max_download_bytes")); +} + +#[test] +fn ics_max_decompressed_bytes_above_hard_cap_is_rejected() { + let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + source.ics = Some(IcsEpgSourceConfigDto { + max_decompressed_bytes: MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT + 1, + ..IcsEpgSourceConfigDto::default() + }); + + let err = source.prepare().unwrap_err(); + + assert!(err.to_string().contains("ics.max_decompressed_bytes")); +} + +#[test] +fn ics_max_events_above_hard_cap_is_rejected() { + let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + source.ics = + Some(IcsEpgSourceConfigDto { max_events: MAX_ICS_EVENTS_HARD_LIMIT + 1, ..IcsEpgSourceConfigDto::default() }); + + let err = source.prepare().unwrap_err(); + + assert!(err.to_string().contains("ics.max_events")); +} + +#[test] +fn ics_dummy_days_above_hard_caps_are_rejected() { + let mut too_much_past = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + too_much_past.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { days_past: MAX_ICS_DAYS_PAST + 1, ..IcsDummyConfigDto::default() }, + ..IcsEpgSourceConfigDto::default() + }); + let err = too_much_past.prepare().unwrap_err(); + assert!(err.to_string().contains("ics.dummy.days_past")); + + let mut too_much_future = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + too_much_future.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { days_future: MAX_ICS_DAYS_FUTURE + 1, ..IcsDummyConfigDto::default() }, + ..IcsEpgSourceConfigDto::default() + }); + let err = too_much_future.prepare().unwrap_err(); + assert!(err.to_string().contains("ics.dummy.days_future")); +} + +#[test] +fn ics_extreme_template_and_dummy_texts_are_rejected() { + let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + source.ics = Some(IcsEpgSourceConfigDto { + event: IcsEventMappingDto { title: "x".repeat(MAX_ICS_SUMMARY_LENGTH + 1), ..IcsEventMappingDto::default() }, + ..IcsEpgSourceConfigDto::default() + }); + let err = source.prepare().unwrap_err(); + assert!(err.to_string().contains("ics.event.title")); + + let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + source.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { + description: "x".repeat(MAX_ICS_DESCRIPTION_LENGTH + 1), + ..IcsDummyConfigDto::default() + }, + ..IcsEpgSourceConfigDto::default() + }); + let err = source.prepare().unwrap_err(); + assert!(err.to_string().contains("ics.dummy.description")); +} diff --git a/shared/src/model/config/hls.rs b/shared/src/model/config/hls.rs index bd0f9f459..82dad0e95 100644 --- a/shared/src/model/config/hls.rs +++ b/shared/src/model/config/hls.rs @@ -1,14 +1,10 @@ use crate::{ defaults::{ default_hls_cache_bytes, default_hls_cache_bytes_per_session, default_hls_cache_duration, - default_hls_corrupt_segment_watchdog_max_parallel_jobs, default_hls_initial_manifest_wait_timeout_secs, - default_hls_max_concurrent_segment_fetches_global, default_hls_max_concurrent_segment_fetches_per_session, - default_hls_max_segments_prefetch, default_hls_origin_manifest_timeout_ms, - default_hls_origin_segment_timeout_ms, default_hls_segment_repair_apply_to_first_segments, - default_hls_segment_repair_high_size_increase_percent, default_hls_segment_repair_low_size_increase_percent, - default_hls_segment_repair_max_parallel_repairs, default_hls_segment_repair_medium_size_increase_percent, - default_hls_segment_repair_postprocess_timeout_ms, default_hls_session_idle_timeout, DEFAULT_HLS_CACHE_BYTES, - DEFAULT_HLS_CACHE_BYTES_PER_SESSION, + default_hls_initial_manifest_wait_timeout_secs, default_hls_max_concurrent_segment_fetches_global, + default_hls_max_concurrent_segment_fetches_per_session, default_hls_max_segments_prefetch, + default_hls_origin_manifest_timeout_ms, default_hls_origin_segment_timeout_ms, + default_hls_session_idle_timeout, DEFAULT_HLS_CACHE_BYTES, DEFAULT_HLS_CACHE_BYTES_PER_SESSION, }, error::TuliproxError, model::{ @@ -46,134 +42,11 @@ impl HlsManifestRecoveryBurstConfigDto { pub const fn clean(&mut self) {} } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct HlsSegmentRepairSizeIncreaseConfigDto { - #[serde(default = "default_hls_segment_repair_low_size_increase_percent")] - pub low_percent: u8, - #[serde(default = "default_hls_segment_repair_medium_size_increase_percent")] - pub medium_percent: u8, - #[serde(default = "default_hls_segment_repair_high_size_increase_percent")] - pub high_percent: u8, -} - -impl Default for HlsSegmentRepairSizeIncreaseConfigDto { - fn default() -> Self { - Self { - low_percent: default_hls_segment_repair_low_size_increase_percent(), - medium_percent: default_hls_segment_repair_medium_size_increase_percent(), - high_percent: default_hls_segment_repair_high_size_increase_percent(), - } - } -} - -impl HlsSegmentRepairSizeIncreaseConfigDto { - pub fn is_empty(&self) -> bool { self == &Self::default() } - - pub const fn clean(&mut self) {} - - fn validate(&self) -> Result<(), TuliproxError> { - let fields = [ - ("low_percent", self.low_percent), - ("medium_percent", self.medium_percent), - ("high_percent", self.high_percent), - ]; - for (field, value) in fields { - if value > 100 { - return Err(TuliproxError::ConfigReverseProxy(format!( - "hls_cache.segment_repair.size_increase.{field} must be <= 100" - ))); - } - } - Ok(()) - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct HlsSegmentRepairConfigDto { - #[serde(default)] - pub max_level: HlsSegmentRepairMode, - #[serde(default = "default_hls_segment_repair_apply_to_first_segments")] - pub apply_to_first_segments: u8, - #[serde(default = "default_hls_segment_repair_max_parallel_repairs")] - pub max_parallel_repairs: usize, - #[serde(default = "default_hls_segment_repair_postprocess_timeout_ms")] - pub postprocess_timeout_ms: Millis, - #[serde(default, skip_serializing_if = "HlsSegmentRepairSizeIncreaseConfigDto::is_empty")] - pub size_increase: HlsSegmentRepairSizeIncreaseConfigDto, - #[serde(default, skip_serializing_if = "HlsCorruptSegmentWatchdogConfigDto::is_empty")] - pub corrupt_segment_watchdog: HlsCorruptSegmentWatchdogConfigDto, -} - -impl Default for HlsSegmentRepairConfigDto { - fn default() -> Self { - Self { - max_level: HlsSegmentRepairMode::Off, - apply_to_first_segments: default_hls_segment_repair_apply_to_first_segments(), - max_parallel_repairs: default_hls_segment_repair_max_parallel_repairs(), - postprocess_timeout_ms: default_hls_segment_repair_postprocess_timeout_ms(), - size_increase: HlsSegmentRepairSizeIncreaseConfigDto::default(), - corrupt_segment_watchdog: HlsCorruptSegmentWatchdogConfigDto::default(), - } - } -} - -impl HlsSegmentRepairConfigDto { - pub fn is_empty(&self) -> bool { self == &Self::default() } - - pub fn clean(&mut self) { - self.size_increase.clean(); - self.corrupt_segment_watchdog.clean(); - } - - fn validate(&self) -> Result<(), TuliproxError> { - if self.postprocess_timeout_ms < Millis::new(100) { - return Err(TuliproxError::ConfigReverseProxy( - "hls_cache.segment_repair.postprocess_timeout_ms must be >= 100".to_string(), - )); - } - self.size_increase.validate()?; - self.corrupt_segment_watchdog.validate() - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct HlsCorruptSegmentWatchdogConfigDto { - #[serde(default)] - pub mode: HlsCorruptSegmentWatchdogMode, - #[serde(default = "default_hls_corrupt_segment_watchdog_max_parallel_jobs")] - pub max_parallel_jobs: usize, -} - -impl Default for HlsCorruptSegmentWatchdogConfigDto { - fn default() -> Self { - Self { - mode: HlsCorruptSegmentWatchdogMode::Off, - max_parallel_jobs: default_hls_corrupt_segment_watchdog_max_parallel_jobs(), - } - } -} - -impl HlsCorruptSegmentWatchdogConfigDto { - pub fn is_empty(&self) -> bool { self == &Self::default() } - - pub const fn clean(&mut self) {} - - fn validate(&self) -> Result<(), TuliproxError> { - if self.max_parallel_jobs == 0 { - return Err(TuliproxError::ConfigReverseProxy( - "hls_cache.segment_repair.corrupt_segment_watchdog.max_parallel_jobs must be >= 1".to_string(), - )); - } - Ok(()) - } -} - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct HlsCacheConfigDto { + #[serde(default, skip_serializing_if = "HlsStartupConfigDto::is_empty")] + pub startup: HlsStartupConfigDto, #[serde(default, skip_serializing_if = "is_blank_optional_string")] pub cache_path: Option, #[serde(default)] @@ -208,6 +81,7 @@ pub struct HlsCacheConfigDto { impl Default for HlsCacheConfigDto { fn default() -> Self { Self { + startup: HlsStartupConfigDto::default(), cache_path: None, strip: HlsStripConfigDto::default(), cache_duration: default_hls_cache_duration(), @@ -263,6 +137,7 @@ impl HlsCacheConfigDto { } pub fn prepare(&mut self) -> Result<(), TuliproxError> { + self.startup.prepare()?; if let Some(cache_path) = &self.cache_path { if cache_path.is_empty() { self.cache_path = None; @@ -323,310 +198,16 @@ impl HlsCacheConfigDto { } #[cfg(test)] -mod tests { - use super::{ - ByteSize, HlsCacheConfigDto, HlsCorruptSegmentWatchdogConfigDto, HlsCorruptSegmentWatchdogMode, - HlsManifestRecoveryBurstConfigDto, HlsManifestRecoveryBurstLevel, HlsSegmentRepairConfigDto, - HlsSegmentRepairMode, HlsSegmentRepairSizeIncreaseConfigDto, HlsStripConfigDto, HlsStripMode, Millis, Secs, - }; - use crate::model::ReverseProxyConfigDto; +mod tests; - #[test] - fn hls_cache_deserializes_target_yaml() { - let yaml = r#" -rewrite_secret: 00112233445566778899aabbccddeeff -hls_cache: - cache_path: - strip: - mode: "segments" - value: 0 - cache_duration: 300 - cache_bytes: "10GB" - cache_bytes_per_session: "512MB" - max_segments_prefetch: 6 - max_concurrent_segment_fetches_per_session: 2 - max_concurrent_segment_fetches_global: 64 - origin_manifest_timeout_ms: 3000 - origin_segment_timeout_ms: 10000 - session_idle_timeout: 300 -"#; +#[cfg(test)] +mod startup_config_tests; - let cfg: ReverseProxyConfigDto = serde_saphyr::from_str(yaml).expect("reverse_proxy should deserialize"); - assert_eq!(cfg.hls_cache, Some(HlsCacheConfigDto::default())); - } - - #[test] - fn hls_cache_serializes_under_reverse_proxy() { - let cfg = ReverseProxyConfigDto { - rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), - hls_cache: Some(HlsCacheConfigDto::default()), - ..Default::default() - }; - - let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); - assert!(serialized.contains("hls_cache:"), "expected hls_cache block, got: {serialized}"); - } - - #[test] - fn hls_cache_serializes_non_default_segment_repair() { - let cfg = ReverseProxyConfigDto { - rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), - hls_cache: Some(HlsCacheConfigDto { - segment_repair: HlsSegmentRepairConfigDto { - max_level: HlsSegmentRepairMode::Medium, - apply_to_first_segments: 2, - max_parallel_repairs: 2, - postprocess_timeout_ms: Millis::new(1_500), - ..Default::default() - }, - ..Default::default() - }), - ..Default::default() - }; - - let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); - assert!(serialized.contains("segment_repair:"), "expected segment_repair block, got: {serialized}"); - assert!(serialized.contains("max_level: medium"), "expected segment repair max level, got: {serialized}"); - assert!( - !serialized.contains("mode: medium"), - "segment repair must serialize the v2 max_level field, got: {serialized}" - ); - assert!( - serialized.contains("apply_to_first_segments: 2"), - "expected segment repair first segment limit, got: {serialized}" - ); - assert!( - serialized.contains("max_parallel_repairs: 2"), - "expected segment repair parallel limit, got: {serialized}" - ); - assert!( - serialized.contains("postprocess_timeout_ms: 1500"), - "expected common post-processing timeout, got: {serialized}" - ); - } - - #[test] - fn hls_cache_serializes_non_default_corrupt_segment_watchdog_without_command_version() { - let cfg = ReverseProxyConfigDto { - rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), - hls_cache: Some(HlsCacheConfigDto { - segment_repair: HlsSegmentRepairConfigDto { - corrupt_segment_watchdog: HlsCorruptSegmentWatchdogConfigDto { - mode: HlsCorruptSegmentWatchdogMode::Sanitize, - max_parallel_jobs: 2, - }, - ..Default::default() - }, - ..Default::default() - }), - ..Default::default() - }; - - let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); - assert!(serialized.contains("corrupt_segment_watchdog:"), "expected watchdog config block, got: {serialized}"); - assert!(serialized.contains("mode: sanitize"), "expected watchdog mode, got: {serialized}"); - assert!(serialized.contains("max_parallel_jobs: 2"), "expected watchdog parallel limit, got: {serialized}"); - assert!( - !serialized.contains("command_version"), - "command_version is internal and must not serialize, got: {serialized}" - ); - } - - #[test] - fn hls_cache_serializes_non_default_manifest_recovery_burst() { - let cfg = ReverseProxyConfigDto { - rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), - hls_cache: Some(HlsCacheConfigDto { - manifest_recovery_burst: HlsManifestRecoveryBurstConfigDto { - level: HlsManifestRecoveryBurstLevel::Beast, - }, - ..Default::default() - }), - ..Default::default() - }; - - let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); - assert!( - serialized.contains("manifest_recovery_burst:"), - "expected manifest recovery burst block, got: {serialized}" - ); - assert!(serialized.contains("level: beast"), "expected burst level, got: {serialized}"); - } - - #[test] - fn hls_cache_rejects_corrupt_segment_watchdog_command_version() { - let yaml = r" -rewrite_secret: 00112233445566778899aabbccddeeff -hls_cache: - segment_repair: - corrupt_segment_watchdog: - mode: sanitize - command_version: 1 -"; - - let err = serde_saphyr::from_str::(yaml) - .expect_err("command_version must be rejected as unknown config"); - assert!(err.to_string().contains("command_version"), "unexpected error: {err}"); - } - - #[test] - fn hls_cache_serializes_non_default_segment_repair_size_increase() { - let segment_repair = HlsSegmentRepairConfigDto { - max_level: HlsSegmentRepairMode::Medium, - size_increase: HlsSegmentRepairSizeIncreaseConfigDto { medium_percent: 9, ..Default::default() }, - ..Default::default() - }; - let cfg = ReverseProxyConfigDto { - rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), - hls_cache: Some(HlsCacheConfigDto { segment_repair, ..Default::default() }), - ..Default::default() - }; - - let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); - assert!(serialized.contains("size_increase:"), "expected size increase block, got: {serialized}"); - assert!(serialized.contains("medium_percent: 9"), "expected size increase value, got: {serialized}"); - } - - #[test] - fn hls_cache_with_non_default_segment_repair_is_not_empty() { - let cfg = HlsCacheConfigDto { - segment_repair: HlsSegmentRepairConfigDto { - max_level: HlsSegmentRepairMode::Medium, - apply_to_first_segments: 1, - max_parallel_repairs: 1, - ..Default::default() - }, - ..Default::default() - }; - - assert!(!cfg.is_empty()); - } - - #[test] - fn hls_cache_rejects_unknown_fields() { - let yaml = r#" -rewrite_secret: 00112233445566778899aabbccddeeff -hls_cache: - cache_path: "/tmp/tuliprox/cache/hls" - unknown_field: true -"#; - - let err = serde_saphyr::from_str::(yaml).expect_err("unknown fields must be rejected"); - assert!(err.to_string().contains("unknown_field"), "unexpected error: {err}"); - } - - #[test] - fn hls_cache_prepare_sets_defaults_for_blank_values() { - let mut cfg = HlsCacheConfigDto { - cache_path: None, - cache_bytes: ByteSize::new(" "), - cache_bytes_per_session: ByteSize::new(""), - ..Default::default() - }; - - cfg.prepare().expect("prepare should succeed"); - - assert_eq!(cfg, HlsCacheConfigDto::default()); - } - - #[test] - fn hls_cache_prepare_accepts_default_values() { - let mut cfg = HlsCacheConfigDto::default(); - - cfg.prepare().expect("default hls cache config should be valid"); - } - - #[test] - fn hls_cache_prepare_allows_zero_prefetch() { - let mut cfg = HlsCacheConfigDto { max_segments_prefetch: 0, ..Default::default() }; - - cfg.prepare().expect("zero prefetch should be valid"); - } - - #[test] - fn hls_cache_prepare_rejects_invalid_minimums() { - let cases: [(&str, HlsCacheConfigDto); 6] = [ - ("cache_duration", HlsCacheConfigDto { cache_duration: Secs::new(0), ..Default::default() }), - ( - "max_concurrent_segment_fetches_per_session", - HlsCacheConfigDto { max_concurrent_segment_fetches_per_session: 0, ..Default::default() }, - ), - ( - "max_concurrent_segment_fetches_global", - HlsCacheConfigDto { max_concurrent_segment_fetches_global: 0, ..Default::default() }, - ), - ( - "origin_manifest_timeout_ms", - HlsCacheConfigDto { origin_manifest_timeout_ms: Millis::new(0), ..Default::default() }, - ), - ( - "origin_segment_timeout_ms", - HlsCacheConfigDto { origin_segment_timeout_ms: Millis::new(0), ..Default::default() }, - ), - ("session_idle_timeout", HlsCacheConfigDto { session_idle_timeout: Secs::new(0), ..Default::default() }), - ]; - - for (field_name, mut cfg) in cases { - let err = cfg.prepare().expect_err("invalid minimum should be rejected"); - assert!(err.to_string().contains(field_name), "expected error to mention {field_name}, got: {err}"); - } - } - - #[test] - fn hls_cache_prepare_rejects_invalid_segment_repair_limits() { - let mut too_many_first_segments = HlsCacheConfigDto::default(); - too_many_first_segments.segment_repair.apply_to_first_segments = 7; - let err = too_many_first_segments.prepare().expect_err("too many first segments should be rejected"); - assert!(err.to_string().contains("segment_repair.apply_to_first_segments"), "unexpected error: {err}"); - - let mut too_many_parallel_repairs = HlsCacheConfigDto { max_segments_prefetch: 2, ..Default::default() }; - too_many_parallel_repairs.segment_repair.max_level = HlsSegmentRepairMode::Medium; - too_many_parallel_repairs.segment_repair.max_parallel_repairs = 3; - let err = too_many_parallel_repairs.prepare().expect_err("too many parallel repairs should be rejected"); - assert!(err.to_string().contains("segment_repair.max_parallel_repairs"), "unexpected error: {err}"); - - let mut invalid_size_increase = HlsCacheConfigDto::default(); - invalid_size_increase.segment_repair.size_increase.high_percent = 101; - let err = invalid_size_increase.prepare().expect_err("size increase above 100 should be rejected"); - assert!(err.to_string().contains("segment_repair.size_increase.high_percent"), "unexpected error: {err}"); - - let mut invalid_postprocess_timeout = HlsCacheConfigDto::default(); - invalid_postprocess_timeout.segment_repair.postprocess_timeout_ms = Millis::new(99); - let err = - invalid_postprocess_timeout.prepare().expect_err("post-processing timeout below 100ms should be rejected"); - assert!(err.to_string().contains("segment_repair.postprocess_timeout_ms"), "unexpected error: {err}"); - - let mut invalid_watchdog_parallel = HlsCacheConfigDto::default(); - invalid_watchdog_parallel.segment_repair.corrupt_segment_watchdog.max_parallel_jobs = 0; - let err = invalid_watchdog_parallel.prepare().expect_err("watchdog parallelism below 1 should be rejected"); - assert!( - err.to_string().contains("segment_repair.corrupt_segment_watchdog.max_parallel_jobs"), - "unexpected error: {err}" - ); - } - - #[test] - fn hls_cache_prepare_allows_zero_segment_repair_parallelism_when_repair_is_off() { - let mut cfg = HlsCacheConfigDto::default(); - cfg.segment_repair.max_parallel_repairs = 0; - - cfg.prepare().expect("zero repair parallelism is ignored while repair is off"); - } - - #[test] - fn hls_cache_prepare_rejects_zero_segment_repair_parallelism_when_repair_is_enabled() { - let mut cfg = HlsCacheConfigDto::default(); - cfg.segment_repair.max_level = HlsSegmentRepairMode::Medium; - cfg.segment_repair.max_parallel_repairs = 0; - - let err = cfg.prepare().expect_err("enabled repair needs a positive parallel limit"); - assert!(err.to_string().contains("segment_repair.max_parallel_repairs"), "unexpected error: {err}"); - } - - #[test] - fn hls_strip_config_default_is_segments() { - let cfg = HlsStripConfigDto::default(); - assert_eq!(cfg.mode, HlsStripMode::Segments); - assert_eq!(cfg.value, 0); - } -} +mod repair; +mod startup; +pub use repair::{ + HlsCorruptSegmentWatchdogConfigDto, HlsSegmentRepairConfigDto, HlsSegmentRepairSizeIncreaseConfigDto, +}; +pub use startup::{ + HlsStartupConfigDto, HlsStartupMode, DEFAULT_HLS_PROGRESSIVE_BYTES_PER_SEGMENT, DEFAULT_HLS_PROGRESSIVE_BYTES_TOTAL, +}; diff --git a/shared/src/model/config/hls/repair.rs b/shared/src/model/config/hls/repair.rs new file mode 100644 index 000000000..e748baca5 --- /dev/null +++ b/shared/src/model/config/hls/repair.rs @@ -0,0 +1,135 @@ +use crate::{ + defaults::{ + default_hls_corrupt_segment_watchdog_max_parallel_jobs, default_hls_segment_repair_apply_to_first_segments, + default_hls_segment_repair_high_size_increase_percent, default_hls_segment_repair_low_size_increase_percent, + default_hls_segment_repair_max_parallel_repairs, default_hls_segment_repair_medium_size_increase_percent, + default_hls_segment_repair_postprocess_timeout_ms, + }, + error::TuliproxError, + model::{HlsCorruptSegmentWatchdogMode, HlsSegmentRepairMode, Millis}, +}; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct HlsSegmentRepairSizeIncreaseConfigDto { + #[serde(default = "default_hls_segment_repair_low_size_increase_percent")] + pub low_percent: u8, + #[serde(default = "default_hls_segment_repair_medium_size_increase_percent")] + pub medium_percent: u8, + #[serde(default = "default_hls_segment_repair_high_size_increase_percent")] + pub high_percent: u8, +} + +impl Default for HlsSegmentRepairSizeIncreaseConfigDto { + fn default() -> Self { + Self { + low_percent: default_hls_segment_repair_low_size_increase_percent(), + medium_percent: default_hls_segment_repair_medium_size_increase_percent(), + high_percent: default_hls_segment_repair_high_size_increase_percent(), + } + } +} + +impl HlsSegmentRepairSizeIncreaseConfigDto { + pub fn is_empty(&self) -> bool { self == &Self::default() } + + pub const fn clean(&mut self) {} + + pub(super) fn validate(&self) -> Result<(), TuliproxError> { + let fields = [ + ("low_percent", self.low_percent), + ("medium_percent", self.medium_percent), + ("high_percent", self.high_percent), + ]; + for (field, value) in fields { + if value > 100 { + return Err(TuliproxError::ConfigReverseProxy(format!( + "hls_cache.segment_repair.size_increase.{field} must be <= 100" + ))); + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct HlsSegmentRepairConfigDto { + #[serde(default)] + pub max_level: HlsSegmentRepairMode, + #[serde(default = "default_hls_segment_repair_apply_to_first_segments")] + pub apply_to_first_segments: u8, + #[serde(default = "default_hls_segment_repair_max_parallel_repairs")] + pub max_parallel_repairs: usize, + #[serde(default = "default_hls_segment_repair_postprocess_timeout_ms")] + pub postprocess_timeout_ms: Millis, + #[serde(default, skip_serializing_if = "HlsSegmentRepairSizeIncreaseConfigDto::is_empty")] + pub size_increase: HlsSegmentRepairSizeIncreaseConfigDto, + #[serde(default, skip_serializing_if = "HlsCorruptSegmentWatchdogConfigDto::is_empty")] + pub corrupt_segment_watchdog: HlsCorruptSegmentWatchdogConfigDto, +} + +impl Default for HlsSegmentRepairConfigDto { + fn default() -> Self { + Self { + max_level: HlsSegmentRepairMode::Off, + apply_to_first_segments: default_hls_segment_repair_apply_to_first_segments(), + max_parallel_repairs: default_hls_segment_repair_max_parallel_repairs(), + postprocess_timeout_ms: default_hls_segment_repair_postprocess_timeout_ms(), + size_increase: HlsSegmentRepairSizeIncreaseConfigDto::default(), + corrupt_segment_watchdog: HlsCorruptSegmentWatchdogConfigDto::default(), + } + } +} + +impl HlsSegmentRepairConfigDto { + pub fn is_empty(&self) -> bool { self == &Self::default() } + + pub fn clean(&mut self) { + self.size_increase.clean(); + self.corrupt_segment_watchdog.clean(); + } + + pub(super) fn validate(&self) -> Result<(), TuliproxError> { + if self.postprocess_timeout_ms < Millis::new(100) { + return Err(TuliproxError::ConfigReverseProxy( + "hls_cache.segment_repair.postprocess_timeout_ms must be >= 100".to_string(), + )); + } + self.size_increase.validate()?; + self.corrupt_segment_watchdog.validate() + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct HlsCorruptSegmentWatchdogConfigDto { + #[serde(default)] + pub mode: HlsCorruptSegmentWatchdogMode, + #[serde(default = "default_hls_corrupt_segment_watchdog_max_parallel_jobs")] + pub max_parallel_jobs: usize, +} + +impl Default for HlsCorruptSegmentWatchdogConfigDto { + fn default() -> Self { + Self { + mode: HlsCorruptSegmentWatchdogMode::Off, + max_parallel_jobs: default_hls_corrupt_segment_watchdog_max_parallel_jobs(), + } + } +} + +impl HlsCorruptSegmentWatchdogConfigDto { + pub fn is_empty(&self) -> bool { self == &Self::default() } + + pub const fn clean(&mut self) {} + + pub(super) fn validate(&self) -> Result<(), TuliproxError> { + if self.max_parallel_jobs == 0 { + return Err(TuliproxError::ConfigReverseProxy( + "hls_cache.segment_repair.corrupt_segment_watchdog.max_parallel_jobs must be >= 1".to_string(), + )); + } + Ok(()) + } +} diff --git a/shared/src/model/config/hls/startup.rs b/shared/src/model/config/hls/startup.rs new file mode 100644 index 000000000..b6aef504c --- /dev/null +++ b/shared/src/model/config/hls/startup.rs @@ -0,0 +1,67 @@ +use crate::{ + error::TuliproxError, + model::{ByteSize, Secs}, +}; + +pub const DEFAULT_HLS_PROGRESSIVE_BYTES_PER_SEGMENT: &str = "32MB"; +pub const DEFAULT_HLS_PROGRESSIVE_BYTES_TOTAL: &str = "128MB"; + +#[derive(Debug, Clone, Copy, Hash, serde::Serialize, serde::Deserialize, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum HlsStartupMode { + #[default] + Conservative, + FirstReady, + Progressive, +} + +crate::impl_str_enum!(HlsStartupMode, "HLS startup mode", + Conservative => "conservative", + FirstReady => "first_ready", + Progressive => "progressive", +); + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct HlsStartupConfigDto { + pub mode: HlsStartupMode, + pub max_progressive_segments: usize, + pub max_progressive_bytes_per_segment: ByteSize, + pub max_progressive_bytes_total: ByteSize, + pub max_progressive_reader_lifetime_secs: Secs, + pub max_deferred_repairs: usize, +} + +impl Default for HlsStartupConfigDto { + fn default() -> Self { + Self { + mode: HlsStartupMode::Conservative, + max_progressive_segments: 16, + max_progressive_bytes_per_segment: ByteSize::new(DEFAULT_HLS_PROGRESSIVE_BYTES_PER_SEGMENT), + max_progressive_bytes_total: ByteSize::new(DEFAULT_HLS_PROGRESSIVE_BYTES_TOTAL), + max_progressive_reader_lifetime_secs: Secs::new(90), + max_deferred_repairs: 32, + } + } +} + +impl HlsStartupConfigDto { + pub fn is_empty(&self) -> bool { self == &Self::default() } + + pub fn prepare(&self) -> Result<(), TuliproxError> { + let segment = + self.max_progressive_bytes_per_segment.parse_bytes().map_err(TuliproxError::ConfigReverseProxy)?.get(); + let total = self.max_progressive_bytes_total.parse_bytes().map_err(TuliproxError::ConfigReverseProxy)?.get(); + if self.max_progressive_segments == 0 + || self.max_deferred_repairs == 0 + || self.max_progressive_reader_lifetime_secs.get() == 0 + || segment == 0 + || total < segment + { + return Err(TuliproxError::ConfigReverseProxy( + "hls_cache.startup requires positive limits and a total byte budget at least as large as the segment budget".to_string() + )); + } + Ok(()) + } +} diff --git a/shared/src/model/config/hls/startup_config_tests.rs b/shared/src/model/config/hls/startup_config_tests.rs new file mode 100644 index 000000000..1a638366a --- /dev/null +++ b/shared/src/model/config/hls/startup_config_tests.rs @@ -0,0 +1,32 @@ +use super::*; + +#[test] +fn startup_defaults_and_modes_roundtrip() -> Result<(), Box> { + let mut legacy: HlsCacheConfigDto = serde_json::from_str("{}")?; + legacy.prepare()?; + assert_eq!(legacy.startup.mode, HlsStartupMode::Conservative); + assert!(serde_json::to_value(&legacy)?.get("startup").is_none()); + for mode in [HlsStartupMode::Conservative, HlsStartupMode::FirstReady, HlsStartupMode::Progressive] { + let startup = HlsStartupConfigDto { mode, max_progressive_segments: 7, ..Default::default() }; + startup.prepare()?; + let serialized = serde_json::to_string(&startup)?; + assert_eq!(serde_json::from_str::(&serialized)?, startup); + } + assert!(serde_json::from_str::(r#"{"mode":"unknown"}"#).is_err()); + assert!(serde_json::from_str::(r#"{"typo":true}"#).is_err()); + Ok(()) +} + +#[test] +fn invalid_startup_limits_are_rejected() { + for config in [ + HlsStartupConfigDto { max_progressive_segments: 0, ..Default::default() }, + HlsStartupConfigDto { max_progressive_bytes_per_segment: ByteSize::new("bad"), ..Default::default() }, + HlsStartupConfigDto { max_progressive_bytes_per_segment: ByteSize::new("0MB"), ..Default::default() }, + HlsStartupConfigDto { max_progressive_bytes_total: ByteSize::new("1MB"), ..Default::default() }, + HlsStartupConfigDto { max_progressive_reader_lifetime_secs: Secs::new(0), ..Default::default() }, + HlsStartupConfigDto { max_deferred_repairs: 0, ..Default::default() }, + ] { + assert!(config.prepare().is_err()); + } +} diff --git a/shared/src/model/config/hls/tests.rs b/shared/src/model/config/hls/tests.rs new file mode 100644 index 000000000..c859e55b6 --- /dev/null +++ b/shared/src/model/config/hls/tests.rs @@ -0,0 +1,305 @@ +use super::{ + HlsCacheConfigDto, HlsCorruptSegmentWatchdogConfigDto, HlsManifestRecoveryBurstConfigDto, + HlsSegmentRepairConfigDto, HlsSegmentRepairSizeIncreaseConfigDto, HlsStripConfigDto, +}; +use crate::model::{ + ByteSize, HlsCorruptSegmentWatchdogMode, HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripMode, Millis, + ReverseProxyConfigDto, Secs, +}; + +#[test] +fn hls_cache_deserializes_target_yaml() { + let yaml = r#" +rewrite_secret: 00112233445566778899aabbccddeeff +hls_cache: + cache_path: + strip: + mode: "segments" + value: 0 + cache_duration: 300 + cache_bytes: "10GB" + cache_bytes_per_session: "512MB" + max_segments_prefetch: 6 + max_concurrent_segment_fetches_per_session: 2 + max_concurrent_segment_fetches_global: 64 + origin_manifest_timeout_ms: 3000 + origin_segment_timeout_ms: 10000 + session_idle_timeout: 300 +"#; + + let cfg: ReverseProxyConfigDto = serde_saphyr::from_str(yaml).expect("reverse_proxy should deserialize"); + assert_eq!(cfg.hls_cache, Some(HlsCacheConfigDto::default())); +} + +#[test] +fn hls_cache_serializes_under_reverse_proxy() { + let cfg = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto::default()), + ..Default::default() + }; + + let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); + assert!(serialized.contains("hls_cache:"), "expected hls_cache block, got: {serialized}"); +} + +#[test] +fn hls_cache_serializes_non_default_segment_repair() { + let cfg = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto { + segment_repair: HlsSegmentRepairConfigDto { + max_level: HlsSegmentRepairMode::Medium, + apply_to_first_segments: 2, + max_parallel_repairs: 2, + postprocess_timeout_ms: Millis::new(1_500), + ..Default::default() + }, + ..Default::default() + }), + ..Default::default() + }; + + let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); + assert!(serialized.contains("segment_repair:"), "expected segment_repair block, got: {serialized}"); + assert!(serialized.contains("max_level: medium"), "expected segment repair max level, got: {serialized}"); + assert!( + !serialized.contains("mode: medium"), + "segment repair must serialize the v2 max_level field, got: {serialized}" + ); + assert!( + serialized.contains("apply_to_first_segments: 2"), + "expected segment repair first segment limit, got: {serialized}" + ); + assert!( + serialized.contains("max_parallel_repairs: 2"), + "expected segment repair parallel limit, got: {serialized}" + ); + assert!( + serialized.contains("postprocess_timeout_ms: 1500"), + "expected common post-processing timeout, got: {serialized}" + ); +} + +#[test] +fn hls_cache_serializes_non_default_corrupt_segment_watchdog_without_command_version() { + let cfg = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto { + segment_repair: HlsSegmentRepairConfigDto { + corrupt_segment_watchdog: HlsCorruptSegmentWatchdogConfigDto { + mode: HlsCorruptSegmentWatchdogMode::Sanitize, + max_parallel_jobs: 2, + }, + ..Default::default() + }, + ..Default::default() + }), + ..Default::default() + }; + + let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); + assert!(serialized.contains("corrupt_segment_watchdog:"), "expected watchdog config block, got: {serialized}"); + assert!(serialized.contains("mode: sanitize"), "expected watchdog mode, got: {serialized}"); + assert!(serialized.contains("max_parallel_jobs: 2"), "expected watchdog parallel limit, got: {serialized}"); + assert!( + !serialized.contains("command_version"), + "command_version is internal and must not serialize, got: {serialized}" + ); +} + +#[test] +fn hls_cache_serializes_non_default_manifest_recovery_burst() { + let cfg = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto { + manifest_recovery_burst: HlsManifestRecoveryBurstConfigDto { level: HlsManifestRecoveryBurstLevel::Beast }, + ..Default::default() + }), + ..Default::default() + }; + + let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); + assert!( + serialized.contains("manifest_recovery_burst:"), + "expected manifest recovery burst block, got: {serialized}" + ); + assert!(serialized.contains("level: beast"), "expected burst level, got: {serialized}"); +} + +#[test] +fn hls_cache_rejects_corrupt_segment_watchdog_command_version() { + let yaml = r" +rewrite_secret: 00112233445566778899aabbccddeeff +hls_cache: + segment_repair: + corrupt_segment_watchdog: + mode: sanitize + command_version: 1 +"; + + let err = serde_saphyr::from_str::(yaml) + .expect_err("command_version must be rejected as unknown config"); + assert!(err.to_string().contains("command_version"), "unexpected error: {err}"); +} + +#[test] +fn hls_cache_serializes_non_default_segment_repair_size_increase() { + let segment_repair = HlsSegmentRepairConfigDto { + max_level: HlsSegmentRepairMode::Medium, + size_increase: HlsSegmentRepairSizeIncreaseConfigDto { medium_percent: 9, ..Default::default() }, + ..Default::default() + }; + let cfg = ReverseProxyConfigDto { + rewrite_secret: "00112233445566778899aabbccddeeff".to_string(), + hls_cache: Some(HlsCacheConfigDto { segment_repair, ..Default::default() }), + ..Default::default() + }; + + let serialized = serde_saphyr::to_string(&cfg).expect("serialization should succeed"); + assert!(serialized.contains("size_increase:"), "expected size increase block, got: {serialized}"); + assert!(serialized.contains("medium_percent: 9"), "expected size increase value, got: {serialized}"); +} + +#[test] +fn hls_cache_with_non_default_segment_repair_is_not_empty() { + let cfg = HlsCacheConfigDto { + segment_repair: HlsSegmentRepairConfigDto { + max_level: HlsSegmentRepairMode::Medium, + apply_to_first_segments: 1, + max_parallel_repairs: 1, + ..Default::default() + }, + ..Default::default() + }; + + assert!(!cfg.is_empty()); +} + +#[test] +fn hls_cache_rejects_unknown_fields() { + let yaml = r#" +rewrite_secret: 00112233445566778899aabbccddeeff +hls_cache: + cache_path: "/tmp/tuliprox/cache/hls" + unknown_field: true +"#; + + let err = serde_saphyr::from_str::(yaml).expect_err("unknown fields must be rejected"); + assert!(err.to_string().contains("unknown_field"), "unexpected error: {err}"); +} + +#[test] +fn hls_cache_prepare_sets_defaults_for_blank_values() { + let mut cfg = HlsCacheConfigDto { + cache_path: None, + cache_bytes: ByteSize::new(" "), + cache_bytes_per_session: ByteSize::new(""), + ..Default::default() + }; + + cfg.prepare().expect("prepare should succeed"); + + assert_eq!(cfg, HlsCacheConfigDto::default()); +} + +#[test] +fn hls_cache_prepare_accepts_default_values() { + let mut cfg = HlsCacheConfigDto::default(); + + cfg.prepare().expect("default hls cache config should be valid"); +} + +#[test] +fn hls_cache_prepare_allows_zero_prefetch() { + let mut cfg = HlsCacheConfigDto { max_segments_prefetch: 0, ..Default::default() }; + + cfg.prepare().expect("zero prefetch should be valid"); +} + +#[test] +fn hls_cache_prepare_rejects_invalid_minimums() { + let cases: [(&str, HlsCacheConfigDto); 6] = [ + ("cache_duration", HlsCacheConfigDto { cache_duration: Secs::new(0), ..Default::default() }), + ( + "max_concurrent_segment_fetches_per_session", + HlsCacheConfigDto { max_concurrent_segment_fetches_per_session: 0, ..Default::default() }, + ), + ( + "max_concurrent_segment_fetches_global", + HlsCacheConfigDto { max_concurrent_segment_fetches_global: 0, ..Default::default() }, + ), + ( + "origin_manifest_timeout_ms", + HlsCacheConfigDto { origin_manifest_timeout_ms: Millis::new(0), ..Default::default() }, + ), + ( + "origin_segment_timeout_ms", + HlsCacheConfigDto { origin_segment_timeout_ms: Millis::new(0), ..Default::default() }, + ), + ("session_idle_timeout", HlsCacheConfigDto { session_idle_timeout: Secs::new(0), ..Default::default() }), + ]; + + for (field_name, mut cfg) in cases { + let err = cfg.prepare().expect_err("invalid minimum should be rejected"); + assert!(err.to_string().contains(field_name), "expected error to mention {field_name}, got: {err}"); + } +} + +#[test] +fn hls_cache_prepare_rejects_invalid_segment_repair_limits() { + let mut too_many_first_segments = HlsCacheConfigDto::default(); + too_many_first_segments.segment_repair.apply_to_first_segments = 7; + let err = too_many_first_segments.prepare().expect_err("too many first segments should be rejected"); + assert!(err.to_string().contains("segment_repair.apply_to_first_segments"), "unexpected error: {err}"); + + let mut too_many_parallel_repairs = HlsCacheConfigDto { max_segments_prefetch: 2, ..Default::default() }; + too_many_parallel_repairs.segment_repair.max_level = HlsSegmentRepairMode::Medium; + too_many_parallel_repairs.segment_repair.max_parallel_repairs = 3; + let err = too_many_parallel_repairs.prepare().expect_err("too many parallel repairs should be rejected"); + assert!(err.to_string().contains("segment_repair.max_parallel_repairs"), "unexpected error: {err}"); + + let mut invalid_size_increase = HlsCacheConfigDto::default(); + invalid_size_increase.segment_repair.size_increase.high_percent = 101; + let err = invalid_size_increase.prepare().expect_err("size increase above 100 should be rejected"); + assert!(err.to_string().contains("segment_repair.size_increase.high_percent"), "unexpected error: {err}"); + + let mut invalid_postprocess_timeout = HlsCacheConfigDto::default(); + invalid_postprocess_timeout.segment_repair.postprocess_timeout_ms = Millis::new(99); + let err = + invalid_postprocess_timeout.prepare().expect_err("post-processing timeout below 100ms should be rejected"); + assert!(err.to_string().contains("segment_repair.postprocess_timeout_ms"), "unexpected error: {err}"); + + let mut invalid_watchdog_parallel = HlsCacheConfigDto::default(); + invalid_watchdog_parallel.segment_repair.corrupt_segment_watchdog.max_parallel_jobs = 0; + let err = invalid_watchdog_parallel.prepare().expect_err("watchdog parallelism below 1 should be rejected"); + assert!( + err.to_string().contains("segment_repair.corrupt_segment_watchdog.max_parallel_jobs"), + "unexpected error: {err}" + ); +} + +#[test] +fn hls_cache_prepare_allows_zero_segment_repair_parallelism_when_repair_is_off() { + let mut cfg = HlsCacheConfigDto::default(); + cfg.segment_repair.max_parallel_repairs = 0; + + cfg.prepare().expect("zero repair parallelism is ignored while repair is off"); +} + +#[test] +fn hls_cache_prepare_rejects_zero_segment_repair_parallelism_when_repair_is_enabled() { + let mut cfg = HlsCacheConfigDto::default(); + cfg.segment_repair.max_level = HlsSegmentRepairMode::Medium; + cfg.segment_repair.max_parallel_repairs = 0; + + let err = cfg.prepare().expect_err("enabled repair needs a positive parallel limit"); + assert!(err.to_string().contains("segment_repair.max_parallel_repairs"), "unexpected error: {err}"); +} + +#[test] +fn hls_strip_config_default_is_segments() { + let cfg = HlsStripConfigDto::default(); + assert_eq!(cfg.mode, HlsStripMode::Segments); + assert_eq!(cfg.value, 0); +} diff --git a/shared/src/model/config/input.rs b/shared/src/model/config/input.rs index 1a92478a4..7c256ada1 100644 --- a/shared/src/model/config/input.rs +++ b/shared/src/model/config/input.rs @@ -1,31 +1,11 @@ use super::PanelApiConfigDto; use crate::{ - check_input_connections, check_input_credentials, - defaults::{ - default_as_true, default_probe_delay_secs, default_probe_live_interval, default_resolve_background, - default_resolve_delay_secs, default_xtream_live_stream_use_prefix, is_default_probe_delay_secs, - is_default_probe_live_interval, is_default_resolve_delay_secs, is_false, is_true, is_zero_i16, is_zero_u16, - }, + defaults::{default_as_true, is_true, is_zero_i16, is_zero_u16}, error::TuliproxError, - foundation::{get_filter, Filter}, - model::{ - config::media_server_catalog::MediaServerInputConfigDto, ClusterFlags, ConfigInputUpdateQualityDto, - EpgConfigDto, PatternTemplate, Prepare, StalkerAuthMode, StalkerInputConfigDto, - }, - utils::{ - arc_str_option_serde, arc_str_serde, arc_str_vec_serde, deserialize_timestamp, get_credentials_from_url_str, - get_trimmed_string, is_blank_optional_arc_str, is_blank_optional_string, is_non_blank_optional_string, - parse_duration_seconds, parse_provider_scheme_url_parts, sanitize_sensitive_info, trim_last_slash, Internable, - BATCH_SCHEME_PREFIX, PROVIDER_SCHEME_PREFIX, - }, + model::{config::media_server_catalog::MediaServerInputConfigDto, EpgConfigDto, StalkerInputConfigDto}, + utils::{arc_str_serde, deserialize_timestamp, is_blank_optional_string, Internable}, }; -use log::warn; -use std::{ - collections::{HashMap, HashSet}, - net::IpAddr, - sync::Arc, -}; -use strum_macros::{AsRefStr, Display, EnumIter, EnumString}; +use std::{collections::HashMap, sync::Arc}; #[macro_export] macro_rules! apply_batch_aliases { @@ -85,445 +65,6 @@ macro_rules! check_provider_scheme_url { }; } -#[derive( - Debug, - Copy, - Clone, - serde::Serialize, - serde::Deserialize, - PartialEq, - Eq, - Default, - EnumIter, - Display, - EnumString, - AsRefStr, -)] -#[strum(serialize_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum InputType { - #[default] - M3u, - Xtream, - M3uBatch, - XtreamBatch, - Stalker, - StalkerBatch, - Library, - Emby, - Jellyfin, - Plex, - Staged, -} - -#[derive(Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default, Display, EnumString)] -#[strum(serialize_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum StagedInputType { - #[default] - M3u, - Xtream, -} - -impl StagedInputType { - pub fn is_default(value: &Self) -> bool { matches!(value, Self::M3u) } - - pub const fn input_type(self) -> InputType { - match self { - Self::M3u => InputType::M3u, - Self::Xtream => InputType::Xtream, - } - } -} - -#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct ConfigInputStagedDto { - #[serde( - default, - alias = "provider", - skip_serializing_if = "is_blank_optional_arc_str", - with = "arc_str_option_serde" - )] - pub for_input: Option>, - #[serde(default)] - pub clusters: ClusterFlags, -} - -impl InputType { - pub fn is_xtream(&self) -> bool { matches!(self, Self::Xtream | Self::XtreamBatch) } - pub fn is_m3u(&self) -> bool { matches!(self, Self::M3u | Self::M3uBatch) } - pub fn is_stalker(&self) -> bool { matches!(self, Self::Stalker | Self::StalkerBatch) } - pub fn uses_standard_input_url(&self) -> bool { - matches!( - self, - Self::M3u | Self::Xtream | Self::M3uBatch | Self::XtreamBatch | Self::Stalker | Self::StalkerBatch - ) - } - pub fn is_library(&self) -> bool { matches!(self, Self::Library) } - pub fn is_media_server(&self) -> bool { matches!(self, Self::Emby | Self::Jellyfin | Self::Plex) } - pub fn is_batch(&self) -> bool { matches!(self, Self::M3uBatch | Self::XtreamBatch | Self::StalkerBatch) } - pub fn is_staged(&self) -> bool { matches!(self, Self::Staged) } - - /// Single source of truth for the categorical behavior of an input type. - /// - /// Adding a new [`InputType`] variant forces an arm here (the match is - /// exhaustive), and every site that consumes [`InputCapabilities`] — - /// persistence/load routing, probe requirements, the custom-provider - /// endpoint gate — stays in sync automatically instead of relying on a - /// parallel `match` somewhere else that is easy to forget. - #[must_use] - pub const fn capabilities(self) -> InputCapabilities { - match self { - Self::M3u | Self::M3uBatch => InputCapabilities { - persistence: InputPersistence::M3u, - requires_provider_connection_for_probe: true, - served_on_custom_provider_endpoint: true, - }, - Self::Xtream | Self::XtreamBatch => InputCapabilities { - persistence: InputPersistence::Xtream, - requires_provider_connection_for_probe: true, - served_on_custom_provider_endpoint: true, - }, - Self::Stalker | Self::StalkerBatch => InputCapabilities { - persistence: InputPersistence::Stalker, - requires_provider_connection_for_probe: true, - served_on_custom_provider_endpoint: true, - }, - Self::Library => InputCapabilities { - persistence: InputPersistence::Library, - requires_provider_connection_for_probe: false, - served_on_custom_provider_endpoint: false, - }, - Self::Emby | Self::Jellyfin | Self::Plex => InputCapabilities { - persistence: InputPersistence::MediaServer, - requires_provider_connection_for_probe: false, - served_on_custom_provider_endpoint: false, - }, - Self::Staged => InputCapabilities { - persistence: InputPersistence::M3u, - requires_provider_connection_for_probe: false, - served_on_custom_provider_endpoint: false, - }, - } - } - - /// Persistence/load backend family for this input type. - #[must_use] - pub const fn persistence(self) -> InputPersistence { self.capabilities().persistence } -} - -/// Storage/loading backend family an [`InputType`] maps onto. -/// -/// Multiple input variants collapse onto the same persistence family (for -/// example every media-server variant shares the same on-disk format), so -/// persist/load routing can match on this instead of re-listing variants. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum InputPersistence { - M3u, - Xtream, - Library, - MediaServer, - Stalker, -} - -/// Categorical capabilities of an [`InputType`], declared once in -/// [`InputType::capabilities`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct InputCapabilities { - /// Storage/loading backend family. - pub persistence: InputPersistence, - /// Whether generic stream probing must open a provider connection. - pub requires_provider_connection_for_probe: bool, - /// Whether the custom-provider HTTP endpoint can serve this input. - pub served_on_custom_provider_endpoint: bool, -} - -#[derive( - Debug, - Copy, - Clone, - serde::Serialize, - serde::Deserialize, - EnumIter, - PartialEq, - Eq, - Default, - Display, - EnumString, - AsRefStr, -)] -#[strum(serialize_all = "UPPERCASE")] -#[serde(rename_all = "UPPERCASE")] -pub enum InputFetchMethod { - #[default] - GET, - POST, -} - -impl InputFetchMethod { - pub fn is_default(value: &InputFetchMethod) -> bool { matches!(value, Self::GET) } -} - -#[derive( - Debug, - Clone, - Copy, - Default, - serde::Serialize, - serde::Deserialize, - PartialEq, - Eq, - strum_macros::Display, - strum_macros::EnumString, - strum_macros::EnumIter, -)] -#[serde(rename_all = "snake_case")] -#[strum(serialize_all = "snake_case")] -pub enum FlussonicHlsCatchup { - #[default] - Native, - BoundedArchive, -} - -impl FlussonicHlsCatchup { - pub fn is_native(value: &Self) -> bool { matches!(value, Self::Native) } -} - -pub const fn default_flussonic_hls_catchup_max_duration_secs() -> u32 { 4 * 60 * 60 } - -fn is_default_flussonic_hls_catchup_max_duration_secs(value: &u32) -> bool { - *value == default_flussonic_hls_catchup_max_duration_secs() -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ConfigInputOptionsDto { - #[serde(default, alias = "xtream_skip_live", alias = "stalker_skip_live", skip_serializing_if = "is_false")] - pub skip_live: bool, - #[serde(default, alias = "xtream_skip_vod", alias = "stalker_skip_vod", skip_serializing_if = "is_false")] - pub skip_vod: bool, - #[serde(default, alias = "xtream_skip_series", alias = "stalker_skip_series", skip_serializing_if = "is_false")] - pub skip_series: bool, - #[serde(default, skip_serializing_if = "ConfigInputUpdateQualityDto::is_disabled")] - pub update_quality: ConfigInputUpdateQualityDto, - #[serde(default = "default_xtream_live_stream_use_prefix", skip_serializing_if = "is_true")] - pub xtream_live_stream_use_prefix: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub xtream_live_stream_without_extension: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub disable_hls_streaming: bool, - #[serde(default, skip_serializing_if = "FlussonicHlsCatchup::is_native")] - pub flussonic_hls_catchup: FlussonicHlsCatchup, - #[serde( - default = "default_flussonic_hls_catchup_max_duration_secs", - skip_serializing_if = "is_default_flussonic_hls_catchup_max_duration_secs" - )] - pub flussonic_hls_catchup_max_duration_secs: u32, - #[serde(default, skip_serializing_if = "is_false")] - pub user_agent_stream_index: bool, - /// Labels fragmented-MP4 objects below Flussonic `tracks-a` paths as `audio/mp4` when the - /// provider sends no specific Content-Type. - #[serde(default, skip_serializing_if = "is_false")] - pub flussonic_hls_audio_tracks: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub resolve_tmdb: bool, - #[serde(default = "default_resolve_background", skip_serializing_if = "is_true")] - pub resolve_background: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub resolve_series: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub resolve_vod: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub probe_series: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub probe_vod: bool, - #[serde(default = "default_resolve_delay_secs", skip_serializing_if = "is_default_resolve_delay_secs")] - pub resolve_delay: u16, - #[serde(default = "default_probe_delay_secs", skip_serializing_if = "is_default_probe_delay_secs")] - pub probe_delay: u16, - #[serde(default, alias = "resolve_live", skip_serializing_if = "is_false")] - pub probe_live: bool, - #[serde( - default = "default_probe_live_interval", - alias = "resolve_live_interval_hours", - skip_serializing_if = "is_default_probe_live_interval" - )] - pub probe_live_interval_hours: u32, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub resolve_filter: Option, - #[serde(skip)] - pub t_resolve_filter: Option, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub probe_filter: Option, - #[serde(skip)] - pub t_probe_filter: Option, - /// Bulk-fetch EPG for live channels during playlist processing. - #[serde(default, skip_serializing_if = "is_false")] - pub stalker_bulk_epg: bool, -} - -impl Default for ConfigInputOptionsDto { - fn default() -> Self { - ConfigInputOptionsDto { - skip_live: false, - skip_vod: false, - skip_series: false, - update_quality: ConfigInputUpdateQualityDto::default(), - xtream_live_stream_use_prefix: default_xtream_live_stream_use_prefix(), - xtream_live_stream_without_extension: false, - disable_hls_streaming: false, - flussonic_hls_catchup: FlussonicHlsCatchup::Native, - flussonic_hls_catchup_max_duration_secs: default_flussonic_hls_catchup_max_duration_secs(), - user_agent_stream_index: false, - flussonic_hls_audio_tracks: false, - resolve_tmdb: false, - resolve_background: default_resolve_background(), - resolve_series: false, - resolve_vod: false, - probe_series: false, - probe_vod: false, - resolve_delay: default_resolve_delay_secs(), - probe_delay: default_probe_delay_secs(), - probe_live: false, - probe_live_interval_hours: default_probe_live_interval(), - resolve_filter: None, - t_resolve_filter: None, - probe_filter: None, - t_probe_filter: None, - stalker_bulk_epg: false, - } - } -} - -impl ConfigInputOptionsDto { - pub fn is_empty(&self) -> bool { - !self.skip_live - && !self.skip_vod - && !self.skip_series - && self.update_quality.is_empty() - && self.xtream_live_stream_use_prefix - && !self.xtream_live_stream_without_extension - && !self.disable_hls_streaming - && FlussonicHlsCatchup::is_native(&self.flussonic_hls_catchup) - && is_default_flussonic_hls_catchup_max_duration_secs(&self.flussonic_hls_catchup_max_duration_secs) - && !self.user_agent_stream_index - && !self.flussonic_hls_audio_tracks - && !self.resolve_tmdb - && self.resolve_background - && !self.resolve_series - && !self.resolve_vod - && !self.probe_series - && !self.probe_vod - && is_default_resolve_delay_secs(&self.resolve_delay) - && is_default_probe_delay_secs(&self.probe_delay) - && !self.probe_live - && is_default_probe_live_interval(&self.probe_live_interval_hours) - && self.resolve_filter.as_ref().is_none_or(|s| s.trim().is_empty()) - && self.probe_filter.as_ref().is_none_or(|s| s.trim().is_empty()) - && !self.stalker_bulk_epg - } - - pub fn clean(&mut self) { - self.skip_live = false; - self.skip_vod = false; - self.skip_series = false; - self.update_quality.clean(); - self.xtream_live_stream_use_prefix = default_as_true(); - self.xtream_live_stream_without_extension = false; - self.disable_hls_streaming = false; - self.flussonic_hls_catchup = FlussonicHlsCatchup::Native; - self.flussonic_hls_catchup_max_duration_secs = default_flussonic_hls_catchup_max_duration_secs(); - self.user_agent_stream_index = false; - self.flussonic_hls_audio_tracks = false; - self.resolve_tmdb = false; - self.resolve_background = default_as_true(); - self.resolve_series = false; - self.resolve_vod = false; - self.probe_series = false; - self.probe_vod = false; - self.resolve_delay = default_resolve_delay_secs(); - self.probe_delay = default_probe_delay_secs(); - self.probe_live = false; - self.probe_live_interval_hours = default_probe_live_interval(); - self.resolve_filter = None; - self.t_resolve_filter = None; - self.probe_filter = None; - self.t_probe_filter = None; - self.stalker_bulk_epg = false; - } -} - -impl Prepare for ConfigInputOptionsDto { - type Ctx<'a> = Option<&'a [PatternTemplate]>; - - fn prepare(&mut self, templates: Self::Ctx<'_>) -> Result<(), TuliproxError> { - self.update_quality.prepare(())?; - if !(1..=7 * 24 * 60 * 60).contains(&self.flussonic_hls_catchup_max_duration_secs) { - return Err(TuliproxError::ConfigInput( - "flussonic_hls_catchup_max_duration_secs must be between 1 and 604800 seconds".to_string(), - )); - } - if let Some(raw_filter) = &self.resolve_filter { - self.t_resolve_filter = Some(get_filter(raw_filter, templates)?); - } - if let Some(raw_filter) = &self.probe_filter { - self.t_probe_filter = Some(get_filter(raw_filter, templates)?); - } - Ok(()) - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ConfigInputAliasDto { - #[serde(default, skip_serializing_if = "is_zero_u16")] - pub id: u16, - #[serde(with = "arc_str_serde")] - pub name: Arc, - pub url: String, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub username: Option, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub password: Option, - #[serde(default, skip_serializing_if = "is_zero_i16")] - pub priority: i16, - #[serde(default)] - pub max_connections: u16, - #[serde(default, deserialize_with = "deserialize_timestamp", skip_serializing_if = "Option::is_none")] - pub exp_date: Option, - #[serde(default = "default_as_true", skip_serializing_if = "is_true")] - pub enabled: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stalker: Option, -} - -impl ConfigInputAliasDto { - pub fn prepare(&mut self, index: u16, input_type: &InputType) -> Result { - self.id = index + 1; - self.name = self.name.trim().intern(); - if self.name.is_empty() { - return Err(TuliproxError::ConfigInput("name for input is mandatory".to_string())); - } - self.url = self.url.trim().to_string(); - if self.url.is_empty() { - return Err(TuliproxError::ConfigInput(format!("url for input is mandatory (input: {})", self.name))); - } - check_input_credentials!(self, input_type, true, true); - check_input_connections!(self, input_type, true); - prepare_stalker_config( - &self.name, - input_type, - &mut self.stalker, - true, - self.username.as_deref(), - self.password.as_deref(), - )?; - - Ok(self.id) - } -} - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct ConfigInputDto { @@ -646,1734 +187,23 @@ impl ConfigInputDto { } } -/// Normalize a user-supplied MAC address into the canonical lowercase -/// `xx:xx:xx:xx:xx:xx` form. Accepts colon-, dash- and bare-hex formats -/// (all three are common in portal provisioning exports). Returns `None` -/// when the value is not a valid 6-octet MAC. -fn normalize_mac_address(raw: &str) -> Option { - let trimmed = raw.trim(); - let octets: Vec = if trimmed.contains(':') || trimmed.contains('-') { - trimmed.split(['-', ':']).map(str::to_string).collect() - } else if trimmed.len() == 12 { - trimmed.as_bytes().chunks(2).map(|c| String::from_utf8_lossy(c).to_string()).collect() - } else { - return None; - }; - let valid = octets.len() == 6 && octets.iter().all(|p| p.len() == 2 && p.chars().all(|c| c.is_ascii_hexdigit())); - if !valid { - return None; - } - Some(octets.join(":").to_ascii_lowercase()) -} - -fn prepare_stalker_config( - input_name: &str, - input_type: &InputType, - stalker: &mut Option, - is_alias: bool, - username: Option<&str>, - password: Option<&str>, -) -> Result<(), TuliproxError> { - if !input_type.is_stalker() { - if stalker.is_some() { - return Err(TuliproxError::ConfigInput(format!( - "stalker configuration is only valid for stalker inputs (input: {input_name})" - ))); - } - return Ok(()); - } - - // Aliases inherit the parent's stalker block when they define none of - // their own — never materialize a default here, otherwise the inherited - // config would be shadowed by an empty one in `as_input`. - let config = if is_alias { - match stalker.as_mut() { - Some(config) => config, - None => return Ok(()), - } - } else { - stalker.get_or_insert_with(StalkerInputConfigDto::default) - }; - - let mut has_mac = false; - // Only normalize an existing device block — do not materialize an empty - // one (it would be re-serialized as a noise `device: {}` block). - if let Some(device) = config.device.as_mut() { - // MAC validation — accept colon-, dash- and bare-hex formats and - // normalize to lowercase `xx:xx:xx:xx:xx:xx`. Empty MAC is allowed - // at this point (auth mode might be credentials-only). - if let Some(mac) = device.mac_address.as_ref() { - let trimmed = mac.trim(); - if !trimmed.is_empty() { - let Some(normalized) = normalize_mac_address(trimmed) else { - return Err(TuliproxError::ConfigInput(format!( - "stalker.device.mac_address must be a MAC address in XX:XX:XX:XX:XX:XX, XX-XX-XX-XX-XX-XX or bare-hex format (input: {input_name})" - ))); - }; - let first_octet = u8::from_str_radix(&normalized[..2], 16).unwrap_or_default(); - if first_octet & 1 != 0 { - return Err(TuliproxError::ConfigInput(format!( - "stalker.device.mac_address must be a unicast MAC address (input: {input_name})" - ))); - } - device.mac_address = Some(normalized); - has_mac = true; - } - } - - // Trim locale/timezone if the user supplied them. - if let Some(timezone) = device.timezone.as_mut() { - *timezone = timezone.trim().to_string(); - if timezone.is_empty() { - device.timezone = None; - } - } - if let Some(locale) = device.locale.as_mut() { - *locale = locale.trim().to_string(); - if locale.is_empty() { - device.locale = None; - } - } - if let Some(profile) = device.device_profile.as_mut() { - *profile = profile.trim().to_string(); - if profile.is_empty() { - device.device_profile = None; - } - } - } - - // Aliases may inherit the parent's identity, so only main inputs are checked. - if !is_alias && !input_type.is_batch() { - let has_credentials = username.is_some_and(|value| !value.trim().is_empty()) - && password.is_some_and(|value| !value.trim().is_empty()); - let requirement = match config.auth_mode { - StalkerAuthMode::Auto if !has_mac && !has_credentials => { - Some("stalker.device.mac_address or username/password") - } - StalkerAuthMode::MacOnly if !has_mac => Some("stalker.device.mac_address for auth_mode mac_only"), - StalkerAuthMode::CredentialsOnly if !has_credentials => { - Some("username and password for auth_mode credentials_only") - } - StalkerAuthMode::MacPlusCredentials if !has_mac || !has_credentials => { - Some("stalker.device.mac_address, username and password for auth_mode mac_plus_credentials") - } - _ => None, - }; - if let Some(requirement) = requirement { - return Err(TuliproxError::ConfigInput(format!("Stalker input '{input_name}' requires {requirement}"))); - } - } - - Ok(()) -} - -impl ConfigInputDto { - fn normalize_input_type_from_batch_url(&mut self) { - let is_batch_url = self.url.trim().starts_with(BATCH_SCHEME_PREFIX); - self.input_type = match self.input_type { - InputType::M3u | InputType::M3uBatch => { - if is_batch_url { - InputType::M3uBatch - } else { - InputType::M3u - } - } - InputType::Xtream | InputType::XtreamBatch => { - if is_batch_url { - InputType::XtreamBatch - } else { - InputType::Xtream - } - } - InputType::Stalker | InputType::StalkerBatch => { - if is_batch_url { - InputType::StalkerBatch - } else { - InputType::Stalker - } - } - InputType::Library => InputType::Library, - InputType::Emby => InputType::Emby, - InputType::Jellyfin => InputType::Jellyfin, - InputType::Plex => InputType::Plex, - InputType::Staged => InputType::Staged, - }; - } - - fn prepare_media_server_input(&mut self) -> Result<(), TuliproxError> { - if !self.input_type.is_media_server() { - return Ok(()); - } - - let trimmed_url = self.url.trim(); - if trimmed_url.starts_with(BATCH_SCHEME_PREFIX) || trimmed_url.starts_with(PROVIDER_SCHEME_PREFIX) { - return Err(TuliproxError::ConfigInput(format!( - "media-server input does not support batch:// or provider:// URLs (input: {})", - self.name - ))); - } - if self.aliases.as_ref().is_some_and(|aliases| !aliases.is_empty()) { - return Err(TuliproxError::ConfigInput(format!( - "media-server input does not support aliases (input: {})", - self.name - ))); - } - if self.epg.is_some() { - return Err(TuliproxError::ConfigInput(format!( - "media-server input does not support EPG configuration (input: {})", - self.name - ))); - } - if self.panel_api.is_some() { - return Err(TuliproxError::ConfigInput(format!( - "media-server input does not support panel_api configuration (input: {})", - self.name - ))); - } - if self.provider.as_ref().is_some_and(|provider| !provider.is_empty()) { - return Err(TuliproxError::ConfigInput(format!( - "media-server input does not support provider failover definitions (input: {})", - self.name - ))); - } - let Some(media_server) = self.media_server.as_mut() else { - return Err(TuliproxError::ConfigInput(format!( - "media_server configuration is mandatory for input type {} (input: {})", - self.input_type, self.name - ))); - }; - media_server.prepare(&self.name)?; - if media_server.libraries.is_empty() { - return Err(TuliproxError::ConfigInput(format!( - "media-server input requires at least one selected library (input: {})", - self.name - ))); - } - - match self.input_type { - InputType::Emby | InputType::Jellyfin => { - if trimmed_url.is_empty() { - return Err(TuliproxError::ConfigInput(format!( - "url is mandatory for input type {} (input: {})", - self.input_type, self.name - ))); - } - let has_login = self.username.as_ref().is_some_and(|u| !u.trim().is_empty()) - && self.password.as_ref().is_some_and(|p| !p.trim().is_empty()); - if !media_server.has_any_emby_jellyfin_auth() && !has_login { - return Err(TuliproxError::ConfigInput(format!( - "media-server input type {} requires media_server token/api_key or username/password bootstrap credentials (input: {})", - self.input_type, self.name - ))); - } - } - InputType::Plex => { - if trimmed_url.is_empty() { - if !is_non_blank_optional_string(&media_server.account_token) { - return Err(TuliproxError::ConfigInput(format!( - "media-server input type plex without input.url requires media_server.account_token for MyPlex discovery (input: {})", - self.name - ))); - } - if !media_server.has_plex_server_selector() { - return Err(TuliproxError::ConfigInput(format!( - "media-server input type plex requires a server selector such as media_server.server_id or media_server.server_name when input.url is omitted (input: {})", - self.name - ))); - } - } else if !is_non_blank_optional_string(&media_server.token) { - return Err(TuliproxError::ConfigInput(format!( - "media-server input type plex with input.url requires media_server.token for direct PMS access (input: {})", - self.name - ))); - } - } - InputType::M3u - | InputType::Xtream - | InputType::M3uBatch - | InputType::XtreamBatch - | InputType::Stalker - | InputType::StalkerBatch - | InputType::Library - | InputType::Staged => {} - } - - Ok(()) - } - - fn validate_staged_self(&mut self) -> Result<(), TuliproxError> { - if self.input_type.is_staged() { - let Some(staged) = self.staged.as_mut() else { - return Err(TuliproxError::ConfigInput(format!( - "staged input requires a provider input (input: {})", - self.name - ))); - }; - if let Some(for_input) = staged.for_input.as_ref().map(|p| p.trim()).filter(|p| !p.is_empty()) { - staged.for_input = Some(for_input.intern()); - } else { - return Err(TuliproxError::ConfigInput(format!( - "staged input requires a provider input (input: {})", - self.name - ))); - } - if staged.clusters.is_empty() { - return Err(TuliproxError::ConfigInput(format!( - "staged input requires at least one staged cluster (input: {})", - self.name - ))); - } - if self.url.trim().is_empty() { - return Err(TuliproxError::ConfigInput(format!( - "url for staged input is mandatory (input: {})", - self.name - ))); - } - if self.staged_type == StagedInputType::Xtream { - let has_credentials = self.username.as_ref().is_some_and(|u| !u.trim().is_empty()) - && self.password.as_ref().is_some_and(|p| !p.trim().is_empty()); - if !has_credentials { - return Err(TuliproxError::ConfigInput(format!( - "staged xtream input requires username and password (input: {})", - self.name - ))); - } - } - // R7 - if self.media_server.is_some() { - return Err(TuliproxError::ConfigInput(format!( - "staged input does not support media_server configuration (input: {})", - self.name - ))); - } - if self.panel_api.is_some() { - return Err(TuliproxError::ConfigInput(format!( - "staged input does not support panel_api configuration (input: {})", - self.name - ))); - } - } else if self.staged.is_some() { - return Err(TuliproxError::ConfigInput(format!( - "staged configuration is only allowed for staged inputs (input: {})", - self.name - ))); - } - Ok(()) - } - - pub fn prepare( - &mut self, - index: u16, - _include_computed: bool, - provider_names: &HashSet, - templates: Option<&[PatternTemplate]>, - ) -> Result { - self.name = self.name.trim().intern(); - if self.name.is_empty() { - return Err(TuliproxError::ConfigInput("name for input is mandatory".to_string())); - } - - if self.sequential_group == Some(0) { - return Err(TuliproxError::ConfigInput(format!( - "sequential_group must be greater than zero (input: {})", - self.name - ))); - } - if self.input_type.is_staged() && self.sequential_group.is_some() { - return Err(TuliproxError::ConfigInput(format!( - "staged input does not support sequential_group (input: {})", - self.name - ))); - } - - if self.input_type.is_staged() { - self.priority = 0; - self.max_connections = 0; - self.cache_duration = None; - self.cache_duration_seconds = 0; - } - - if let Some(duration_str) = &self.cache_duration { - self.cache_duration_seconds = self.parse_duration(duration_str)?; - } else { - self.cache_duration_seconds = 0; - } - - self.url = self.url.trim().to_string(); - self.normalize_input_type_from_batch_url(); - if let Some(options) = self.options.as_ref() { - if options.disable_hls_streaming && !self.input_type.is_xtream() { - return Err(TuliproxError::ConfigInput(format!( - "`disable_hls_streaming` is supported only for Xtream inputs (input: {})", - self.name - ))); - } - } - if let Some(media_server) = self.media_server.as_mut() { - media_server.normalize(); - } - if self.enabled { - self.prepare_media_server_input()?; - } - if self.url.starts_with(PROVIDER_SCHEME_PREFIX) && self.input_type.is_batch() { - return Err(TuliproxError::ConfigInput(format!( - "input type {} does not support provider:// URLs for batch definitions; use batch:// URL (input: {})", - self.input_type, self.name - ))); - } - - check_input_credentials!(self, self.input_type, true, false); - check_input_connections!(self, self.input_type, false); - // Always run stalker validation: it rejects a stray `stalker` block on - // non-stalker inputs and surfaces a malformed MAC at config load even - // when the input is currently disabled (consistent with aliases). - self.prepare_stalker_input()?; - self.validate_staged_self()?; - - self.persist = get_trimmed_string(self.persist.as_deref()); - check_provider_scheme_url!(self.url, provider_names); - - let mut current_index = index + 1; - self.id = current_index; - if let Some(aliases) = self.aliases.as_mut() { - let input_type = &self.input_type; - for alias in aliases { - current_index = alias.prepare(current_index, input_type)?; - check_provider_scheme_url!(alias.url.as_str(), provider_names); - } - } - - if let Some(panel_api) = self.panel_api.as_mut() { - panel_api.prepare(&self.name)?; - } - - // Validate provider:// URLs in EPG sources - if let Some(epg) = self.epg.as_ref() { - if let Some(sources) = epg.sources.as_ref() { - for epg_source in sources { - let url = epg_source.url.trim(); - check_provider_scheme_url!(url, provider_names); - } - } - } - - // Prepare filter options - if let Some(options) = self.options.as_mut() { - options.prepare(templates)?; - } - - Ok(current_index) - } - - fn parse_duration(&self, duration_str: &str) -> Result { - match parse_duration_seconds(duration_str, false) { - Some(seconds) => Ok(seconds), - None => Err(TuliproxError::ConfigInput(format!( - "Invalid cache_duration format in '{}': {}", - self.name, duration_str - ))), - } - } - - // Neue ausgelagerte Methode für die URL-Generierung - fn generate_auto_epg_url(&self) -> Result { - let get_creds = || { - if self.username.is_some() && self.password.is_some() { - return (self.username.clone(), self.password.clone(), Some(self.url.clone())); - } - - let (u, p, r) = self - .aliases - .as_ref() - .and_then(|aliases| aliases.iter().find(|a| a.enabled)) - .map_or((None, None, None), |alias| { - (alias.username.clone(), alias.password.clone(), Some(alias.url.clone())) - }); - - if u.is_some() && p.is_some() && r.is_some() { - return (u, p, r); - } - - let (u, p) = get_credentials_from_url_str(&self.url); - if u.is_some() && p.is_some() { - return (u, p, Some(self.url.clone())); - } - - self.aliases.as_ref().and_then(|aliases| aliases.iter().find(|a| a.enabled)).map_or( - (None, None, None), - |alias| { - let (u, p) = get_credentials_from_url_str(alias.url.as_str()); - (u, p, Some(alias.url.clone())) - }, - ) - }; - - let (username, password, base_url) = get_creds(); - - if username.is_none() || password.is_none() || base_url.is_none() { - Err(format!("auto_epg is enabled for input {}, but no credentials could be extracted", self.name)) - } else if let Some(base) = base_url { - let clean_base = base.split('?').next().unwrap_or(&base); - - let provider_epg_url = format!( - "{}/xmltv.php?username={}&password={}", - trim_last_slash(clean_base), - username.unwrap_or_default(), - password.unwrap_or_default() - ); - Ok(provider_epg_url) - } else { - Err(format!( - "auto_epg is enabled for input {}, but url could not be parsed {}", - self.name, - sanitize_sensitive_info(&self.url) - )) - } - } - - pub fn prepare_epg(&mut self, include_computed: bool) -> Result<(), TuliproxError> { - if let Some(mut epg) = self.epg.take() { - if self.input_type == InputType::Library { - warn!("EPG is not supported for library inputs {}, skipping", self.name); - self.epg = None; - return Ok(()); - } - - epg.prepare(|| self.generate_auto_epg_url(), include_computed)?; - epg.t_sources = { - let mut seen_sources = HashSet::new(); - epg.t_sources.drain(..).filter(|src| seen_sources.insert(src.source_identity())).collect() - }; - self.epg = Some(epg); - } - Ok(()) - } - - pub fn prepare_batch( - &mut self, - batch_aliases: Vec, - index: u16, - ) -> Result, TuliproxError> { - let idx = apply_batch_aliases!(self, batch_aliases, Some(index)); - Ok(idx) - } - - pub fn prepare_type(&mut self) -> Result<(), TuliproxError> { - self.url = self.url.trim().to_string(); - self.normalize_input_type_from_batch_url(); - if self.url.starts_with(PROVIDER_SCHEME_PREFIX) && self.input_type.is_batch() { - return Err(TuliproxError::ConfigInput(format!( - "input type {} does not support provider:// URLs for batch definitions; use batch:// URL", - self.input_type - ))); - } - Ok(()) - } - - pub fn upsert_alias(&mut self, mut alias: ConfigInputAliasDto) -> Result<(), TuliproxError> { - check_input_credentials!(alias, self.input_type, true, true); - check_input_connections!(alias, self.input_type, true); - let aliases = self.aliases.get_or_insert_with(Vec::new); - if let Some(existing) = aliases.iter_mut().find(|a| a.id == alias.id) { - *existing = alias; - } else { - aliases.push(alias); - } - Ok(()) - } - - pub fn update_account_expiration_date( - &mut self, - account_name: &str, - exp_date: i64, - disable: bool, - ) -> Result { - if self.name.as_ref() == account_name { - let changed = self.exp_date != Some(exp_date) || disable && !self.account_disabled; - self.exp_date = Some(exp_date); - self.account_disabled |= disable; - return Ok(changed); - } - let (expiration, enabled) = if let Some(alias) = self - .aliases - .as_mut() - .and_then(|aliases| aliases.iter_mut().find(|alias| alias.name.as_ref() == account_name)) - { - (&mut alias.exp_date, &mut alias.enabled) - } else { - return Err(TuliproxError::ConfigInput(format!("No input or alias found for account '{account_name}'"))); - }; - let changed = *expiration != Some(exp_date) || disable && *enabled; - *expiration = Some(exp_date); - if disable { - *enabled = false; - } - Ok(changed) - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ConfigProviderDto { - #[serde(with = "arc_str_serde")] - pub name: Arc, - #[serde(with = "arc_str_vec_serde")] - pub urls: Vec>, - #[serde(default, skip_serializing_if = "is_default_provider_url_selection_policy")] - pub provider_url_selection_policy: ProviderUrlSelectionPolicy, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub dns: Option, -} - -impl ConfigProviderDto { - pub fn prepare(&mut self) -> Result<(), TuliproxError> { - self.name = self.name.trim().intern(); - if self.name.is_empty() { - return Err(TuliproxError::ConfigInput("Name for provider is mandatory".to_string())); - } - self.urls = self.urls.drain(..).filter(|url| !url.trim().is_empty()).map(|u| u.trim().intern()).collect(); - if self.urls.is_empty() { - return Err(TuliproxError::ConfigInput("Urls for provider is mandatory".to_string())); - } - if let Some(dns) = self.dns.as_mut() { - dns.prepare()?; - } - Ok(()) - } -} - -pub const fn default_provider_dns_refresh_secs() -> u64 { 300 } -pub const fn is_default_provider_dns_refresh_secs(v: &u64) -> bool { *v == default_provider_dns_refresh_secs() } -pub fn is_default_provider_url_selection_policy(v: &ProviderUrlSelectionPolicy) -> bool { - *v == ProviderUrlSelectionPolicy::default() -} -pub fn is_default_dns_prefer(v: &DnsPrefer) -> bool { *v == DnsPrefer::default() } -pub fn is_default_on_resolve_error(v: &OnResolveErrorPolicy) -> bool { *v == OnResolveErrorPolicy::default() } -pub fn is_default_on_connect_error(v: &OnConnectErrorPolicy) -> bool { *v == OnConnectErrorPolicy::default() } - -#[derive( - Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default, EnumString, AsRefStr, Display, -)] -#[strum(serialize_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum ProviderUrlSelectionPolicy { - #[default] - ResumeLastWorking, - RestartFromFirst, -} - -#[derive( - Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default, EnumString, AsRefStr, Display, -)] -#[strum(serialize_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum DnsPrefer { - Ipv4, - Ipv6, - #[default] - System, -} - -#[derive(Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, EnumString, AsRefStr, Display)] -#[strum(serialize_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum DnsScheme { - Http, - Https, -} - -#[derive( - Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default, EnumString, AsRefStr, Display, -)] -#[strum(serialize_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum OnResolveErrorPolicy { - #[default] - KeepLastGood, - FallbackToHostname, -} - -#[derive( - Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default, EnumString, AsRefStr, Display, -)] -#[strum(serialize_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum OnConnectErrorPolicy { - #[default] - TryNextIp, - RotateProviderUrl, -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ProviderDnsDto { - #[serde(default, skip_serializing_if = "is_false")] - pub enabled: bool, - #[serde( - default = "default_provider_dns_refresh_secs", - skip_serializing_if = "is_default_provider_dns_refresh_secs" - )] - pub refresh_secs: u64, - #[serde(default, skip_serializing_if = "is_default_dns_prefer")] - pub prefer: DnsPrefer, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_addrs: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub schemes: Option>, - #[serde(default, skip_serializing_if = "is_false")] - pub keep_vhost: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub overrides: Option>>, - #[serde(default, skip_serializing_if = "is_default_on_resolve_error")] - pub on_resolve_error: OnResolveErrorPolicy, - #[serde(default, skip_serializing_if = "is_default_on_connect_error")] - pub on_connect_error: OnConnectErrorPolicy, -} - -impl Default for ProviderDnsDto { - fn default() -> Self { - Self { - enabled: false, - refresh_secs: default_provider_dns_refresh_secs(), - prefer: DnsPrefer::default(), - max_addrs: None, - schemes: None, - keep_vhost: false, - overrides: None, - on_resolve_error: OnResolveErrorPolicy::default(), - on_connect_error: OnConnectErrorPolicy::default(), - } - } -} - -impl ProviderDnsDto { - pub fn prepare(&mut self) -> Result<(), TuliproxError> { - if self.refresh_secs == 0 { - self.refresh_secs = default_provider_dns_refresh_secs(); - } else if self.refresh_secs < 10 { - // Explicit low values are honored for fast failover, but flagged - warn!( - "Provider dns refresh_secs={} is below the recommended minimum of 10s; this increases resolver load", - self.refresh_secs - ); - } - if self.max_addrs == Some(0) { - return Err(TuliproxError::ConfigInput("Provider dns max_addrs must be >= 1 when set".to_string())); - } - if let Some(schemes) = self.schemes.as_mut() { - let mut unique = Vec::with_capacity(schemes.len()); - for scheme in schemes.drain(..) { - if !unique.contains(&scheme) { - unique.push(scheme); - } - } - *schemes = unique; - if schemes.is_empty() { - self.schemes = None; - } - } - - if let Some(overrides) = self.overrides.as_mut() { - let mut normalized: HashMap> = HashMap::new(); - for (host, ips) in std::mem::take(overrides) { - let host = host.trim().to_ascii_lowercase(); - if host.is_empty() { - return Err(TuliproxError::ConfigInput( - "Provider dns overrides hostname must not be empty".to_string(), - )); - } - if ips.is_empty() { - return Err(TuliproxError::ConfigInput( - "Provider dns overrides for host '{host}' must not be empty".to_string(), - )); - } - let entry = normalized.entry(host.clone()).or_default(); - for ip in ips { - if !entry.contains(&ip) { - entry.push(ip); - } - } - } - if normalized.is_empty() { - self.overrides = None; - } else { - *overrides = normalized; - } - } - - Ok(()) - } -} - #[cfg(test)] -mod tests { - use super::*; - use crate::model::StalkerDeviceProfileDto; - - fn create_test_dto() -> ConfigInputDto { - ConfigInputDto { name: "test_input".intern(), ..ConfigInputDto::default() } - } - - fn prepare_dto(dto: &mut ConfigInputDto) -> Result { - dto.prepare(0, false, &HashSet::new(), None) - } - - #[test] - fn config_input_alias_serialization_uses_block_default_and_reads_both_styles() { - for aliases in [ - "- {name: alias-one, url: provider://example, username: alias, password: 'pass # with: {}, commas', max_connections: 1, exp_date: 1788779145}\n", - "- name: alias-one\n url: provider://example\n username: alias\n password: 'pass # with: {}, commas'\n max_connections: 1\n exp_date: 1788779145\n", - ] { - let yaml = format!( - "name: provider\ntype: xtream\nurl: provider://example\nusername: root\npassword: pass\noptions:\n update_quality:\n live: 85\n vod: 85\n series: 85\n resolve_background: false\ncache_duration: 20h\nmax_connections: 1\nexp_date: 1788779143\naliases:\n{aliases}panel_api:\n url: http://panel.example\n api_key: synthetic-key\n" - ); - let input: ConfigInputDto = serde_saphyr::from_str(&yaml).expect("both alias styles parse"); - let serialized = serde_saphyr::to_string(&input).expect("serialize input"); - assert!(serialized.contains("- name: alias-one\n")); - assert!(!serialized.contains("- {")); - let reparsed: ConfigInputDto = serde_saphyr::from_str(&serialized).expect("read serialized input"); - assert_eq!(reparsed, input); - let json = serde_json::to_string(&input).expect("JSON serialization"); - assert_eq!(serde_json::from_str::(&json).expect("JSON roundtrip"), input); - - let mut prepared = input; - prepared.prepare(0, false, &HashSet::from(["example".to_string()]), None).expect("prepare input"); - assert_eq!(prepared.cache_duration_seconds, 72_000); - let options = prepared.options.as_ref().expect("options"); - assert_eq!(options.update_quality, ConfigInputUpdateQualityDto { live: 85, vod: 85, series: 85 }); - assert!(!options.resolve_background); - assert_eq!(prepared.aliases.as_ref().expect("aliases").len(), 1); - assert!(prepared.panel_api.is_some()); - } - } - - #[test] - fn config_input_alias_serialization_preserves_empty_and_missing_lists() { - for aliases in [None, Some(Vec::new())] { - let input = ConfigInputDto { name: "provider".intern(), aliases, ..Default::default() }; - let serialized = serde_saphyr::to_string(&input).expect("serialize input"); - assert_eq!(serde_saphyr::from_str::(&serialized).expect("roundtrip"), input); - } - } - - #[test] - fn sequential_group_round_trips_yaml_and_json() { - let yaml = "name: grouped\ntype: m3u\nurl: http://example.com/list.m3u\nsequential_group: 7\n"; - let dto: ConfigInputDto = serde_saphyr::from_str(yaml).expect("sequential group should deserialize from yaml"); - assert_eq!(dto.sequential_group, Some(7)); - - let encoded_yaml = serde_saphyr::to_string(&dto).expect("sequential group should serialize to yaml"); - assert!(encoded_yaml.contains("sequential_group: 7")); - - let encoded_json = serde_json::to_string(&dto).expect("sequential group should serialize to json"); - let decoded_json: ConfigInputDto = - serde_json::from_str(&encoded_json).expect("sequential group should deserialize from json"); - assert_eq!(decoded_json.sequential_group, Some(7)); - } - - #[test] - fn sequential_group_rejects_zero_and_staged_inputs() { - let mut zero = ConfigInputDto { - name: "zero_group".intern(), - url: "http://example.com/list.m3u".to_string(), - sequential_group: Some(0), - ..ConfigInputDto::default() - }; - let err = prepare_dto(&mut zero).expect_err("group zero must be rejected"); - assert!(err.to_string().contains("sequential_group"), "Error: {err}"); - - let mut staged = ConfigInputDto { - name: "staged_group".intern(), - input_type: InputType::Staged, - url: "http://example.com/list.m3u".to_string(), - staged: Some(ConfigInputStagedDto { - for_input: Some("provider_a".intern()), - ..ConfigInputStagedDto::default() - }), - sequential_group: Some(1), - ..ConfigInputDto::default() - }; - let err = prepare_dto(&mut staged).expect_err("staged groups must be rejected"); - assert!(err.to_string().contains("sequential_group"), "Error: {err}"); - } - - #[test] - fn stalker_options_reject_retired_playback_resolution_switches() -> Result<(), serde_json::Error> { - let dto: ConfigInputOptionsDto = serde_json::from_str(r#"{"stalker_bulk_epg":true}"#)?; - assert!(dto.stalker_bulk_epg); - assert!(serde_json::from_str::(r#"{"stalker_pre_resolve_playback":true}"#).is_err()); - assert!(serde_json::from_str::(r#"{"stalker_runtime_resolve_playback":false}"#).is_err()); - Ok(()) - } - - #[test] - fn test_epg_url_from_explicit_main_credentials() { - let mut dto = create_test_dto(); - // Hier testen wir auch gleich mit, ob der Trailing Slash sauber entfernt wird! - dto.url = "http://myprovider.com/".to_string(); - dto.username = Some("hello".to_string()); - dto.password = Some("mello".to_string()); - - let result = dto.generate_auto_epg_url().unwrap(); - assert_eq!(result, "http://myprovider.com/xmltv.php?username=hello&password=mello"); - } - - #[test] - fn test_epg_url_from_enabled_alias_explicit_credentials() { - let mut dto = create_test_dto(); - dto.url = "http://main.com".to_string(); - - let alias = ConfigInputAliasDto { - enabled: true, - url: "http://alias.com".to_string(), - username: Some("alias_user".to_string()), - password: Some("alias_pass".to_string()), - ..ConfigInputAliasDto::default() - }; - - dto.aliases = Some(vec![alias]); - - let result = dto.generate_auto_epg_url().unwrap(); - // Er muss die URL und die Credentials vom Alias nehmen - assert_eq!(result, "http://alias.com/xmltv.php?username=alias_user&password=alias_pass"); - } - - #[test] - fn test_epg_url_skips_disabled_aliases() { - let mut dto = create_test_dto(); - - let alias = ConfigInputAliasDto { - enabled: false, // Alias ist deaktiviert! - url: "http://alias.com".to_string(), - username: Some("alias_user".to_string()), - password: Some("alias_pass".to_string()), - ..ConfigInputAliasDto::default() - }; - - dto.aliases = Some(vec![alias]); - - let result = dto.generate_auto_epg_url(); - // Since the main DTO is empty and alias is disabled, an error must occur - assert!(result.is_err()); - assert!(result.unwrap_err().contains("no credentials could be extracted")); - } - - #[test] - fn test_epg_url_fails_without_credentials() { - let mut dto = create_test_dto(); - dto.url = "http://nocreds.com".to_string(); - - let result = dto.generate_auto_epg_url(); - assert!(result.is_err()); - assert!(result.unwrap_err().contains("no credentials could be extracted")); - } - - #[test] - fn test_epg_url_from_main_url_query_credentials() { - let mut dto = create_test_dto(); - // Credentials stecken als Query-Parameter in der URL - dto.url = "http://myprovider.com?username=hello&password=mello".to_string(); - - let result = dto.generate_auto_epg_url().unwrap(); - - // Durch unseren sauberen "clean_base" Fix sieht die URL jetzt richtig aus! - assert_eq!(result, "http://myprovider.com/xmltv.php?username=hello&password=mello"); - } - - #[test] - fn test_epg_url_from_alias_url_query_credentials() { - let mut dto = create_test_dto(); - dto.url = "http://main.com".to_string(); - - let alias = ConfigInputAliasDto { - enabled: true, - // Credentials im Alias als Query-Parameter - url: "http://alias.com?username=alias_user&password=alias_pass".to_string(), - ..ConfigInputAliasDto::default() - }; - - dto.aliases = Some(vec![alias]); - - let result = dto.generate_auto_epg_url().unwrap(); - assert_eq!(result, "http://alias.com/xmltv.php?username=alias_user&password=alias_pass"); - } - - #[test] - fn test_epg_url_from_provider_scheme_url_query_credentials() { - let mut dto = create_test_dto(); - dto.url = "provider://myprovider".to_string(); - dto.username = Some("test".to_string()); - dto.password = Some("secret".to_string()); - - let result = dto.generate_auto_epg_url().unwrap(); - assert_eq!(result, "provider://myprovider/xmltv.php?username=test&password=secret"); - } - - #[test] - fn test_provider_dns_defaults() { - let dns = ProviderDnsDto::default(); - assert!(!dns.enabled); - assert_eq!(dns.refresh_secs, 300); - assert_eq!(dns.prefer, DnsPrefer::System); - assert_eq!(dns.on_resolve_error, OnResolveErrorPolicy::KeepLastGood); - assert_eq!(dns.on_connect_error, OnConnectErrorPolicy::TryNextIp); - assert!(dns.schemes.is_none()); - } - - #[test] - fn test_provider_url_selection_policy_defaults_to_resume_last_working() { - let provider = ConfigProviderDto { - name: "provider-a".intern(), - urls: vec!["http://primary.example.com".intern()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), - dns: None, - }; - - assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::ResumeLastWorking); - } - - #[test] - fn test_provider_url_selection_policy_can_be_set_to_restart_from_first() { - let provider = ConfigProviderDto { - name: "provider-a".intern(), - urls: vec!["http://primary.example.com".intern()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, - dns: None, - }; - - assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::RestartFromFirst); - } - - #[test] - fn test_provider_url_selection_policy_deserializes_default_when_omitted() { - let provider: ConfigProviderDto = - serde_json::from_str(r#"{"name":"provider-a","urls":["http://primary.example.com"]}"#) - .expect("provider dto should deserialize"); - - assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::ResumeLastWorking); - } - - #[test] - fn test_provider_url_selection_policy_deserializes_restart_from_first() { - let provider: ConfigProviderDto = serde_json::from_str( - r#"{"name":"provider-a","urls":["http://primary.example.com"],"provider_url_selection_policy":"restart_from_first"}"#, - ) - .expect("provider dto should deserialize"); - - assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::RestartFromFirst); - } - - #[test] - fn test_provider_url_selection_policy_default_is_omitted_on_serialize() { - let provider = ConfigProviderDto { - name: "provider-a".intern(), - urls: vec!["http://primary.example.com".intern()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, - dns: None, - }; - - let json = serde_json::to_string(&provider).expect("provider dto should serialize"); - let value: serde_json::Value = serde_json::from_str(&json).expect("serialized provider should be valid json"); - - assert!(value.get("provider_url_selection_policy").is_none()); - } - - #[test] - fn test_provider_dns_prepare_normalizes_overrides_and_preserves_low_refresh() { - let mut dns = ProviderDnsDto { - refresh_secs: 1, - schemes: Some(vec![DnsScheme::Http, DnsScheme::Http, DnsScheme::Https]), - overrides: Some(HashMap::from([( - " EXAMPLE.COM ".to_string(), - vec![ - "203.0.113.10".parse::().expect("valid ip"), - "203.0.113.10".parse::().expect("valid ip"), - ], - )])), - ..ProviderDnsDto::default() - }; - - dns.prepare().expect("dns prepare should succeed"); - - // Explicit low values are honored (only warned about), not clamped. - assert_eq!(dns.refresh_secs, 1); - assert_eq!(dns.schemes, Some(vec![DnsScheme::Http, DnsScheme::Https])); - let overrides = dns.overrides.expect("overrides should exist"); - assert_eq!(overrides.len(), 1); - assert!(overrides.contains_key("example.com")); - assert_eq!(overrides["example.com"].len(), 1); - } - - #[test] - fn prepare_switches_xtream_to_xtream_batch_when_alias_exists() { - let mut dto = ConfigInputDto { - name: "input_alias".intern(), - input_type: InputType::Xtream, - url: "batch:///tmp/input_alias.csv".to_string(), - aliases: Some(vec![ConfigInputAliasDto { - id: 1, - name: "alias_1".intern(), - url: "http://provider.example/stream".to_string(), - username: Some("u".to_string()), - password: Some("p".to_string()), - enabled: true, - ..ConfigInputAliasDto::default() - }]), - ..ConfigInputDto::default() - }; - - dto.prepare_type().expect("prepare type should succeed"); - dto.prepare(0, true, &HashSet::new(), None) - .expect("prepare should succeed and infer batch type from batch:// URL"); - assert_eq!(dto.input_type, InputType::XtreamBatch); - } - - #[test] - fn prepare_keeps_xtream_type_when_alias_exists_without_batch_url() { - let mut dto = ConfigInputDto { - name: "input_alias_http".intern(), - input_type: InputType::XtreamBatch, - url: "http://localhost:3001".to_string(), - username: Some("root_user".to_string()), - password: Some("root_pass".to_string()), - aliases: Some(vec![ConfigInputAliasDto { - id: 1, - name: "alias_1".intern(), - url: "http://provider.example/stream".to_string(), - username: Some("u".to_string()), - password: Some("p".to_string()), - enabled: true, - ..ConfigInputAliasDto::default() - }]), - ..ConfigInputDto::default() - }; - - dto.prepare_type().expect("prepare type should normalize non-batch URL to xtream"); - assert_eq!(dto.input_type, InputType::Xtream); - dto.prepare(0, true, &HashSet::new(), None).expect("prepare should succeed for regular URL with aliases"); - assert_eq!(dto.input_type, InputType::Xtream); - } - - #[test] - fn prepare_type_does_not_validate_media_server_config() { - let mut dto = ConfigInputDto { - name: "emby_media_server".intern(), - input_type: InputType::Emby, - url: "https://media.example.invalid".to_string(), - ..ConfigInputDto::default() - }; - - dto.prepare_type().expect("prepare_type only normalizes type/url"); - let err = prepare_dto(&mut dto).expect_err("full prepare should validate missing media_server block"); - assert!(err.to_string().contains("media_server configuration is mandatory")); - } - - #[test] - fn prepare_batch_url_does_not_require_xtream_credentials() { - let mut dto = ConfigInputDto { - name: "batch_no_creds".intern(), - input_type: InputType::Xtream, - url: "batch:///tmp/no-creds.csv".to_string(), - username: None, - password: None, - ..ConfigInputDto::default() - }; - - dto.prepare(0, true, &HashSet::new(), None) - .expect("batch:// input must be normalized before credential validation"); - assert_eq!(dto.input_type, InputType::XtreamBatch); - } - - #[test] - fn prepare_provider_scheme_url_is_not_treated_as_batch_input() { - let mut dto = ConfigInputDto { - name: "batch_provider".intern(), - input_type: InputType::XtreamBatch, - url: "provider://myprovider".to_string(), - username: Some("root_user".to_string()), - password: Some("root_pass".to_string()), - aliases: Some(vec![ConfigInputAliasDto { - id: 1, - name: "alias_1".intern(), - url: "http://provider.example/stream".to_string(), - username: Some("u".to_string()), - password: Some("p".to_string()), - enabled: true, - ..ConfigInputAliasDto::default() - }]), - ..ConfigInputDto::default() - }; - - let err = dto - .prepare(0, true, &HashSet::new(), None) - .expect_err("prepare should treat provider:// URL as regular input (non-batch) and validate provider"); - assert!(err.to_string().contains("Provider name myprovider is not defined"), "Error: {err}"); - } - - #[test] - fn disable_hls_streaming_rejects_non_xtream_input() -> Result<(), String> { - let mut input = ConfigInputDto { - name: "hls-only".intern(), - input_type: InputType::M3u, - url: "http://provider.example/list.m3u".to_string(), - options: Some(ConfigInputOptionsDto { disable_hls_streaming: true, ..ConfigInputOptionsDto::default() }), - ..ConfigInputDto::default() - }; - - let Err(error) = prepare_dto(&mut input) else { - return Err("non-Xtream input accepted disable_hls_streaming".to_string()); - }; - let message = error.to_string(); - - assert!(message.contains("disable_hls_streaming"), "Error: {message}"); - assert!(message.contains("Xtream"), "Error: {message}"); - Ok(()) - } - - #[test] - fn disable_hls_streaming_allows_extensionless_live_streams() -> Result<(), String> { - let mut input = ConfigInputDto { - name: "xtream".intern(), - input_type: InputType::Xtream, - url: "http://provider.example".to_string(), - username: Some("user".to_string()), - password: Some("pass".to_string()), - options: Some(ConfigInputOptionsDto { - disable_hls_streaming: true, - xtream_live_stream_without_extension: true, - ..ConfigInputOptionsDto::default() - }), - ..ConfigInputDto::default() - }; - - prepare_dto(&mut input).map(|_| ()).map_err(|error| error.to_string()) - } - - #[test] - fn prepare_rejects_missing_input_url_even_with_aliases() { - let mut dto = ConfigInputDto { - name: "xtream_missing_root_url".intern(), - input_type: InputType::Xtream, - url: String::new(), - username: Some("root_user".to_string()), - password: Some("root_pass".to_string()), - aliases: Some(vec![ConfigInputAliasDto { - id: 1, - name: "alias_1".intern(), - url: "http://alias.example".to_string(), - username: Some("alias_user".to_string()), - password: Some("alias_pass".to_string()), - enabled: true, - ..ConfigInputAliasDto::default() - }]), - ..ConfigInputDto::default() - }; - - let err = dto - .prepare(0, true, &HashSet::new(), None) - .expect_err("prepare must require root input url even when aliases are present"); - assert!(err.to_string().contains("url for input is mandatory"), "Error: {err}"); - assert!(err.to_string().contains("xtream_missing_root_url"), "Error: {err}"); - } - - #[test] - fn prepare_rejects_missing_root_credentials_for_non_batch_url_even_with_aliases() { - let mut dto = ConfigInputDto { - name: "xtream_batch_missing_root_creds".intern(), - input_type: InputType::XtreamBatch, - url: "http://root.example".to_string(), - aliases: Some(vec![ConfigInputAliasDto { - id: 1, - name: "alias_1".intern(), - url: "http://alias.example".to_string(), - username: Some("alias_user".to_string()), - password: Some("alias_pass".to_string()), - enabled: true, - ..ConfigInputAliasDto::default() - }]), - ..ConfigInputDto::default() - }; - - let err = dto - .prepare(0, true, &HashSet::new(), None) - .expect_err("prepare must require root credentials for non-batch URL"); - assert!(err.to_string().contains("for input type xtream: username and password are mandatory"), "Error: {err}"); - assert!(err.to_string().contains("xtream_batch_missing_root_creds"), "Error: {err}"); - } - - #[test] - fn prepare_rejects_xtream_batch_batch_url_with_root_credentials_even_with_aliases() { - let mut dto = ConfigInputDto { - name: "xtream_batch_with_root_creds".intern(), - input_type: InputType::XtreamBatch, - url: "batch:///tmp/aliases.csv".to_string(), - username: Some("root_user".to_string()), - password: Some("root_pass".to_string()), - aliases: Some(vec![ConfigInputAliasDto { - id: 1, - name: "alias_1".intern(), - url: "http://alias.example".to_string(), - username: Some("alias_user".to_string()), - password: Some("alias_pass".to_string()), - enabled: true, - ..ConfigInputAliasDto::default() - }]), - ..ConfigInputDto::default() - }; - - let err = dto - .prepare(0, true, &HashSet::new(), None) - .expect_err("prepare must reject root credentials when using batch:// for xtream-batch"); - assert!(err.to_string().contains("with batch:// URL should not define username or password"), "Error: {err}"); - assert!(err.to_string().contains("xtream_batch_with_root_creds"), "Error: {err}"); - } - - #[test] - fn test_staged_requires_provider() { - let mut dto = create_test_dto(); - dto.input_type = InputType::Staged; - dto.url = "http://staged.com/playlist.m3u".to_string(); - - let err = prepare_dto(&mut dto).expect_err("staged input without provider must be rejected"); - assert!(err.to_string().contains("requires a provider input"), "Error: {err}"); - } - - #[test] - fn test_staged_requires_url() { - let mut dto = create_test_dto(); - dto.input_type = InputType::Staged; - dto.staged = - Some(ConfigInputStagedDto { for_input: Some("provider_a".intern()), ..ConfigInputStagedDto::default() }); - dto.url = String::new(); - - let err = prepare_dto(&mut dto).expect_err("staged input without url must be rejected"); - assert!(err.to_string().contains("url for staged input is mandatory"), "Error: {err}"); - } - - #[test] - fn test_staged_requires_non_empty_clusters() { - let mut dto = create_test_dto(); - dto.input_type = InputType::Staged; - dto.staged = - Some(ConfigInputStagedDto { for_input: Some("provider_a".intern()), clusters: ClusterFlags::empty() }); - dto.url = "http://staged.com/playlist.m3u".to_string(); - - let err = prepare_dto(&mut dto).expect_err("staged input without clusters must be rejected"); - assert!(err.to_string().contains("requires at least one staged cluster"), "Error: {err}"); - } - - #[test] - fn test_staged_config_only_allowed_for_staged() { - let mut dto = create_test_dto(); - dto.input_type = InputType::M3u; - dto.url = "http://main.com/playlist.m3u".to_string(); - dto.staged = - Some(ConfigInputStagedDto { for_input: Some("provider_a".intern()), ..ConfigInputStagedDto::default() }); - - let err = prepare_dto(&mut dto).expect_err("non-staged input with staged config must be rejected"); - assert!(err.to_string().contains("staged configuration is only allowed for staged inputs"), "Error: {err}"); - } - - #[test] - fn test_staged_rejects_media_server() { - let mut dto = create_test_dto(); - dto.input_type = InputType::Staged; - dto.staged = - Some(ConfigInputStagedDto { for_input: Some("provider_a".intern()), ..ConfigInputStagedDto::default() }); - dto.url = "http://staged.com/playlist.m3u".to_string(); - dto.media_server = Some(MediaServerInputConfigDto::default()); - - let err = prepare_dto(&mut dto).expect_err("staged input with media_server must be rejected"); - assert!(err.to_string().contains("does not support media_server configuration"), "Error: {err}"); - } - - #[test] - fn test_staged_valid() { - let mut dto = create_test_dto(); - dto.input_type = InputType::Staged; - dto.staged = - Some(ConfigInputStagedDto { for_input: Some(" provider_a ".intern()), ..ConfigInputStagedDto::default() }); - dto.url = "http://staged.com/playlist.m3u".to_string(); - - prepare_dto(&mut dto).expect("valid staged input should prepare successfully"); - assert!(dto.input_type.is_staged()); - assert_eq!(dto.staged.as_ref().and_then(|staged| staged.for_input.as_deref()), Some("provider_a")); - } - - #[test] - fn test_staged_provider_alias_deserializes_as_for_input() { - let staged: ConfigInputStagedDto = - serde_json::from_str(r#"{"provider":"provider_a"}"#).expect("legacy provider field should deserialize"); - - assert_eq!(staged.for_input.as_deref(), Some("provider_a")); - } - - #[test] - fn test_staged_ignores_stream_runtime_fields() { - let mut dto = create_test_dto(); - dto.input_type = InputType::Staged; - dto.url = "http://staged.com/playlist.m3u".to_string(); - dto.staged = - Some(ConfigInputStagedDto { for_input: Some("provider_a".intern()), ..ConfigInputStagedDto::default() }); - dto.priority = -10; - dto.max_connections = 2; - dto.cache_duration = Some("not-a-duration".to_string()); - - prepare_dto(&mut dto).expect("staged stream runtime fields should be ignored"); - - assert_eq!(dto.priority, 0); - assert_eq!(dto.max_connections, 0); - assert_eq!(dto.cache_duration, None); - assert_eq!(dto.cache_duration_seconds, 0); - } - - #[test] - fn test_config_input_options_dto_filter_prepare_parses_valid_filter() { - let mut dto = ConfigInputOptionsDto { - resolve_filter: Some(r#"name ~ "test""#.to_string()), - ..ConfigInputOptionsDto::default() - }; - dto.prepare(None).expect("valid filter should parse"); - assert!(dto.t_resolve_filter.is_some()); - } - - #[test] - fn test_config_input_options_dto_filter_prepare_rejects_invalid_filter() { - let mut dto = ConfigInputOptionsDto { - resolve_filter: Some(r#"name ~ "["#.to_string()), // invalid regex - ..ConfigInputOptionsDto::default() - }; - let result = dto.prepare(None); - assert!(result.is_err()); - } - - #[test] - fn test_config_input_options_dto_filter_prepare_with_unknown_template_placeholder() { - let mut dto = ConfigInputOptionsDto { - resolve_filter: Some(r#"name ~ "!UNKNOWN!""#.to_string()), - ..ConfigInputOptionsDto::default() - }; - let result = dto.prepare(None); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("Unknown template placeholder")); - } - - #[test] - fn test_config_input_options_dto_filter_none_prepares_successfully() { - let mut dto = ConfigInputOptionsDto { resolve_filter: None, ..ConfigInputOptionsDto::default() }; - dto.prepare(None).expect("None filter should prepare successfully"); - assert!(dto.t_resolve_filter.is_none()); - } - - #[test] - fn config_input_options_deserializes_legacy_prefixed_skip_aliases() { - let dto: ConfigInputOptionsDto = serde_json::from_str( - r#"{ - "xtream_skip_live": true, - "stalker_skip_vod": true, - "skip_series": true - }"#, - ) - .expect("legacy aliases should deserialize"); - - assert!(dto.skip_live); - assert!(dto.skip_vod); - assert!(dto.skip_series); - } - - #[test] - fn config_input_options_serializes_shared_skip_names() { - let dto = ConfigInputOptionsDto { - skip_live: true, - skip_vod: true, - skip_series: false, - ..ConfigInputOptionsDto::default() - }; - - let value: serde_json::Value = - serde_json::from_str(&serde_json::to_string(&dto).expect("config input options should serialize")) - .expect("serialized json should parse"); - - assert_eq!(value.get("skip_live"), Some(&serde_json::Value::Bool(true))); - assert_eq!(value.get("skip_vod"), Some(&serde_json::Value::Bool(true))); - assert!(value.get("xtream_skip_live").is_none()); - assert!(value.get("stalker_skip_vod").is_none()); - } - - #[test] - fn config_input_options_without_update_quality_remains_backward_compatible() -> Result<(), serde_json::Error> { - let options: ConfigInputOptionsDto = serde_json::from_str("{}")?; - let value = serde_json::to_value(&options)?; - - assert_eq!(options.update_quality, ConfigInputUpdateQualityDto::default()); - assert!(options.is_empty()); - assert!(value.get("update_quality").is_none()); - Ok(()) - } - - #[test] - fn config_input_options_update_quality_round_trips_and_keeps_options_non_empty() -> Result<(), serde_json::Error> { - let options: ConfigInputOptionsDto = serde_json::from_value(serde_json::json!({ - "update_quality": { - "live": 95, - "vod": 90, - "series": 100 - } - }))?; - - assert!(!options.is_empty()); - assert_eq!(options.update_quality.live, 95); - assert_eq!(options.update_quality.vod, 90); - assert_eq!(options.update_quality.series, 100); - assert_eq!( - serde_json::to_value(options)?, - serde_json::json!({ - "update_quality": { - "live": 95, - "vod": 90, - "series": 100 - } - }) - ); - Ok(()) - } - - #[test] - fn config_input_options_prepare_validates_update_quality() { - let mut options: ConfigInputOptionsDto = serde_saphyr::from_str("update_quality:\n live: 101\n") - .expect("manually edited update quality should deserialize before validation"); - - let error = options.prepare(None).expect_err("invalid update quality must be rejected"); - - assert!(error.to_string().contains("options.update_quality.live")); - } - - #[test] - fn config_input_options_clean_resets_update_quality() { - let mut options = ConfigInputOptionsDto { - update_quality: ConfigInputUpdateQualityDto { live: 95, vod: 90, series: 85 }, - ..ConfigInputOptionsDto::default() - }; - - options.clean(); - - assert_eq!(options.update_quality, ConfigInputUpdateQualityDto::default()); - assert!(options.is_empty()); - } - - #[test] - fn flussonic_archive_window_config_validates_and_cleans() -> Result<(), Box> { - use crate::model::Prepare; - let mut options: ConfigInputOptionsDto = serde_json::from_str( - r#"{"flussonic_hls_catchup":"bounded_archive","flussonic_hls_catchup_max_duration_secs":1800}"#, - )?; - options.prepare(None)?; - assert_eq!(options.flussonic_hls_catchup_max_duration_secs, 1800); - assert_eq!(serde_json::from_str::(&serde_json::to_string(&options)?)?, options); - for value in [0, 604801, u32::MAX] { - options.flussonic_hls_catchup_max_duration_secs = value; - assert!(options.prepare(None).is_err()); - } - options.clean(); - assert_eq!(options.flussonic_hls_catchup_max_duration_secs, 14400); - assert!(options.is_empty()); - assert!(serde_json::to_value(options)?.get("flussonic_hls_catchup_max_duration_secs").is_none()); - Ok(()) - } - - #[test] - fn flussonic_hls_catchup_round_trips_and_cleans() -> Result<(), serde_json::Error> { - let defaults: ConfigInputOptionsDto = serde_json::from_str("{}")?; - assert_eq!(defaults.flussonic_hls_catchup, super::FlussonicHlsCatchup::Native); - assert!(serde_json::to_value(defaults)?.get("flussonic_hls_catchup").is_none()); - let mut options: ConfigInputOptionsDto = - serde_json::from_str(r#"{"flussonic_hls_catchup":"bounded_archive"}"#)?; - assert_eq!(options.flussonic_hls_catchup, super::FlussonicHlsCatchup::BoundedArchive); - assert!(!options.is_empty()); - assert_eq!(serde_json::to_value(&options)?, serde_json::json!({"flussonic_hls_catchup": "bounded_archive"})); - assert_eq!(serde_json::from_str::(&serde_json::to_string(&options)?)?, options); - assert!(serde_json::from_str::(r#"{"flussonic_hls_catchup":"unknown"}"#).is_err()); - options.clean(); - assert_eq!(options.flussonic_hls_catchup, super::FlussonicHlsCatchup::Native); - assert!(options.is_empty()); - Ok(()) - } - - #[test] - fn disable_hls_streaming_defaults_to_false() -> Result<(), serde_json::Error> { - let options: ConfigInputOptionsDto = serde_json::from_str("{}")?; - assert!(!options.disable_hls_streaming); - Ok(()) - } - - #[test] - fn disable_hls_streaming_keeps_options_non_empty_and_clean_resets_it() { - let mut options = ConfigInputOptionsDto { disable_hls_streaming: true, ..ConfigInputOptionsDto::default() }; - - assert!(!options.is_empty()); - options.clean(); - assert!(!options.disable_hls_streaming); - assert!(options.is_empty()); - } - - #[test] - fn disable_hls_streaming_round_trips() -> Result<(), serde_json::Error> { - let options = ConfigInputOptionsDto { disable_hls_streaming: true, ..ConfigInputOptionsDto::default() }; - let json = serde_json::to_string(&options)?; - let restored: ConfigInputOptionsDto = serde_json::from_str(&json)?; - - assert!(restored.disable_hls_streaming); - Ok(()) - } - - #[test] - fn flussonic_hls_audio_tracks_round_trips_defaults_off_and_cleans() -> Result<(), serde_json::Error> { - assert!(!ConfigInputOptionsDto::default().flussonic_hls_audio_tracks); - let mut options = - ConfigInputOptionsDto { flussonic_hls_audio_tracks: true, ..ConfigInputOptionsDto::default() }; - let json = serde_json::to_string(&options)?; - let restored: ConfigInputOptionsDto = serde_json::from_str(&json)?; - - assert!(restored.flussonic_hls_audio_tracks); - assert!(!options.is_empty()); - options.clean(); - assert!(!options.flussonic_hls_audio_tracks); - assert!(options.is_empty()); - Ok(()) - } - - #[test] - fn user_agent_stream_index_round_trips_and_keeps_options_non_empty() -> Result<(), serde_json::Error> { - let mut options = ConfigInputOptionsDto { user_agent_stream_index: true, ..ConfigInputOptionsDto::default() }; - let json = serde_json::to_string(&options)?; - let restored: ConfigInputOptionsDto = serde_json::from_str(&json)?; - - assert!(restored.user_agent_stream_index); - assert!(!options.is_empty()); - options.clean(); - assert!(!options.user_agent_stream_index); - assert!(options.is_empty()); - Ok(()) - } - - #[test] - fn new_stalker_input_has_configuration_block() { - let input = ConfigInputDto::new_with_type(InputType::Stalker); - - assert!(input.stalker.is_some()); - assert!(ConfigInputDto::new_with_type(InputType::M3u).stalker.is_none()); - } - - #[test] - fn stalker_mac_is_normalized_and_multicast_is_rejected() { - let mut input = ConfigInputDto::new_with_type(InputType::Stalker); - input.name = "portal".into(); - input.stalker.as_mut().expect("stalker config").device = - Some(StalkerDeviceProfileDto { mac_address: Some("00:1A:79:AA:BB:CC".to_string()), ..Default::default() }); - input.prepare_stalker_input().expect("unicast MAC should be valid"); - assert_eq!( - input - .stalker - .as_ref() - .and_then(|config| config.device.as_ref()) - .and_then(|device| device.mac_address.as_deref()), - Some("00:1a:79:aa:bb:cc") - ); - - input.stalker.as_mut().expect("stalker config").device.as_mut().expect("device").mac_address = - Some("01:00:00:00:00:00".to_string()); - assert!(input.prepare_stalker_input().is_err()); - } - - #[test] - fn stalker_main_input_requires_identity_but_batch_aliases_may_inherit() { - let mut input = ConfigInputDto::new_with_type(InputType::Stalker); - input.name = "portal".into(); - assert!(input.prepare_stalker_input().is_err()); - - let mut alias = ConfigInputAliasDto { - name: "alias".into(), - url: "http://portal.example/c/".to_string(), - ..Default::default() - }; - assert!(alias.prepare(0, &InputType::Stalker).is_ok()); - } - - #[test] - fn stalker_auth_mode_requires_matching_identity() { - for (auth_mode, has_mac, has_credentials, valid) in [ - (StalkerAuthMode::Auto, true, false, true), - (StalkerAuthMode::Auto, false, true, true), - (StalkerAuthMode::MacOnly, true, false, true), - (StalkerAuthMode::MacOnly, false, true, false), - (StalkerAuthMode::CredentialsOnly, false, true, true), - (StalkerAuthMode::CredentialsOnly, true, false, false), - (StalkerAuthMode::MacPlusCredentials, true, true, true), - (StalkerAuthMode::MacPlusCredentials, true, false, false), - ] { - let mut input = ConfigInputDto::new_with_type(InputType::Stalker); - input.name = "portal".into(); - input.username = has_credentials.then(|| "user".to_string()); - input.password = has_credentials.then(|| "password".to_string()); - input.stalker = Some(StalkerInputConfigDto { - auth_mode, - device: has_mac.then(|| StalkerDeviceProfileDto { - mac_address: Some("00:1a:79:aa:bb:cc".to_string()), - ..Default::default() - }), - ..Default::default() - }); - - assert_eq!(input.prepare_stalker_input().is_ok(), valid, "auth mode: {auth_mode}"); - } - } - - #[test] - fn non_stalker_input_rejects_stalker_configuration() { - let mut input = ConfigInputDto { - name: "m3u".into(), - stalker: Some(StalkerInputConfigDto::default()), - ..Default::default() - }; - assert!(input.prepare_stalker_input().is_err()); - } - - #[test] - fn stalker_config_round_trips_extended_device_and_size_caps() { - let mut input = ConfigInputDto::new_with_type(InputType::Stalker); - input.name = "portal".into(); - input.stalker = Some(StalkerInputConfigDto { - device: Some(StalkerDeviceProfileDto { - mac_address: Some("00:1a:79:aa:bb:cc".to_string()), - device_profile: Some("MAG254".to_string()), - serial_number: Some("serial-1".to_string()), - device_id: Some("device-1".to_string()), - device_id2: Some("device-2".to_string()), - signature: Some("sig-1".to_string()), - ..Default::default() - }), - size_caps: Some(crate::model::stalker::StalkerActionSizeCapDto { - create_link_kb: 96, - ordered_list_mb: 12, - get_epg_mb: 80, - }), - catalog_max_pages: Some(2048), - ..Default::default() - }); - - let value = serde_json::to_value(&input).expect("serialize stalker input"); - let decoded: ConfigInputDto = serde_json::from_value(value).expect("deserialize stalker input"); - let stalker = decoded.stalker.expect("stalker config"); - let device = stalker.device.expect("device"); - assert_eq!(device.device_profile.as_deref(), Some("MAG254")); - assert_eq!(device.serial_number.as_deref(), Some("serial-1")); - assert_eq!(device.device_id.as_deref(), Some("device-1")); - assert_eq!(device.device_id2.as_deref(), Some("device-2")); - assert_eq!(device.signature.as_deref(), Some("sig-1")); - let caps = stalker.size_caps.expect("size caps"); - assert_eq!(caps.create_link_kb, 96); - assert_eq!(caps.ordered_list_mb, 12); - assert_eq!(caps.get_epg_mb, 80); - assert_eq!(stalker.catalog_max_pages, Some(2048)); - } -} +mod tests; + +mod aliases; +mod kind; +mod options; +mod prepare; +mod provider; +mod stalker; +pub use aliases::ConfigInputAliasDto; +pub use kind::{ + ConfigInputStagedDto, InputCapabilities, InputFetchMethod, InputPersistence, InputType, StagedInputType, +}; +pub use options::{default_flussonic_hls_catchup_max_duration_secs, ConfigInputOptionsDto, FlussonicHlsCatchup}; +pub use provider::{ + default_provider_dns_refresh_secs, is_default_dns_prefer, is_default_on_connect_error, is_default_on_resolve_error, + is_default_provider_dns_refresh_secs, is_default_provider_url_selection_policy, ConfigProviderDto, DnsPrefer, + DnsScheme, OnConnectErrorPolicy, OnResolveErrorPolicy, ProviderDnsDto, ProviderUrlSelectionPolicy, +}; +use stalker::prepare_stalker_config; diff --git a/shared/src/model/config/input/aliases.rs b/shared/src/model/config/input/aliases.rs new file mode 100644 index 000000000..21b2a78e8 --- /dev/null +++ b/shared/src/model/config/input/aliases.rs @@ -0,0 +1,59 @@ +use super::{stalker::prepare_stalker_config, InputType}; +use crate::{ + check_input_connections, check_input_credentials, + defaults::{default_as_true, is_true, is_zero_i16, is_zero_u16}, + error::TuliproxError, + model::StalkerInputConfigDto, + utils::{arc_str_serde, deserialize_timestamp, is_blank_optional_string, Internable}, +}; +use std::sync::Arc; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ConfigInputAliasDto { + #[serde(default, skip_serializing_if = "is_zero_u16")] + pub id: u16, + #[serde(with = "arc_str_serde")] + pub name: Arc, + pub url: String, + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub username: Option, + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub password: Option, + #[serde(default, skip_serializing_if = "is_zero_i16")] + pub priority: i16, + #[serde(default)] + pub max_connections: u16, + #[serde(default, deserialize_with = "deserialize_timestamp", skip_serializing_if = "Option::is_none")] + pub exp_date: Option, + #[serde(default = "default_as_true", skip_serializing_if = "is_true")] + pub enabled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stalker: Option, +} + +impl ConfigInputAliasDto { + pub fn prepare(&mut self, index: u16, input_type: &InputType) -> Result { + self.id = index + 1; + self.name = self.name.trim().intern(); + if self.name.is_empty() { + return Err(TuliproxError::ConfigInput("name for input is mandatory".to_string())); + } + self.url = self.url.trim().to_string(); + if self.url.is_empty() { + return Err(TuliproxError::ConfigInput(format!("url for input is mandatory (input: {})", self.name))); + } + check_input_credentials!(self, input_type, true, true); + check_input_connections!(self, input_type, true); + prepare_stalker_config( + &self.name, + input_type, + &mut self.stalker, + true, + self.username.as_deref(), + self.password.as_deref(), + )?; + + Ok(self.id) + } +} diff --git a/shared/src/model/config/input/kind.rs b/shared/src/model/config/input/kind.rs new file mode 100644 index 000000000..086659600 --- /dev/null +++ b/shared/src/model/config/input/kind.rs @@ -0,0 +1,186 @@ +use crate::{ + model::ClusterFlags, + utils::{arc_str_option_serde, is_blank_optional_arc_str}, +}; +use std::sync::Arc; +use strum_macros::{AsRefStr, Display, EnumIter, EnumString}; + +#[derive( + Debug, + Copy, + Clone, + serde::Serialize, + serde::Deserialize, + PartialEq, + Eq, + Default, + EnumIter, + Display, + EnumString, + AsRefStr, +)] +#[strum(serialize_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum InputType { + #[default] + M3u, + Xtream, + M3uBatch, + XtreamBatch, + Stalker, + StalkerBatch, + Library, + Emby, + Jellyfin, + Plex, + Staged, +} + +#[derive(Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default, Display, EnumString)] +#[strum(serialize_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum StagedInputType { + #[default] + M3u, + Xtream, +} + +impl StagedInputType { + pub fn is_default(value: &Self) -> bool { matches!(value, Self::M3u) } + + pub const fn input_type(self) -> InputType { + match self { + Self::M3u => InputType::M3u, + Self::Xtream => InputType::Xtream, + } + } +} + +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ConfigInputStagedDto { + #[serde( + default, + alias = "provider", + skip_serializing_if = "is_blank_optional_arc_str", + with = "arc_str_option_serde" + )] + pub for_input: Option>, + #[serde(default)] + pub clusters: ClusterFlags, +} + +impl InputType { + pub fn is_xtream(&self) -> bool { matches!(self, Self::Xtream | Self::XtreamBatch) } + pub fn is_m3u(&self) -> bool { matches!(self, Self::M3u | Self::M3uBatch) } + pub fn is_stalker(&self) -> bool { matches!(self, Self::Stalker | Self::StalkerBatch) } + pub fn uses_standard_input_url(&self) -> bool { + matches!( + self, + Self::M3u | Self::Xtream | Self::M3uBatch | Self::XtreamBatch | Self::Stalker | Self::StalkerBatch + ) + } + pub fn is_library(&self) -> bool { matches!(self, Self::Library) } + pub fn is_media_server(&self) -> bool { matches!(self, Self::Emby | Self::Jellyfin | Self::Plex) } + pub fn is_batch(&self) -> bool { matches!(self, Self::M3uBatch | Self::XtreamBatch | Self::StalkerBatch) } + pub fn is_staged(&self) -> bool { matches!(self, Self::Staged) } + + /// Single source of truth for the categorical behavior of an input type. + /// + /// Adding a new [`InputType`] variant forces an arm here (the match is + /// exhaustive), and every site that consumes [`InputCapabilities`] — + /// persistence/load routing, probe requirements, the custom-provider + /// endpoint gate — stays in sync automatically instead of relying on a + /// parallel `match` somewhere else that is easy to forget. + #[must_use] + pub const fn capabilities(self) -> InputCapabilities { + match self { + Self::M3u | Self::M3uBatch => InputCapabilities { + persistence: InputPersistence::M3u, + requires_provider_connection_for_probe: true, + served_on_custom_provider_endpoint: true, + }, + Self::Xtream | Self::XtreamBatch => InputCapabilities { + persistence: InputPersistence::Xtream, + requires_provider_connection_for_probe: true, + served_on_custom_provider_endpoint: true, + }, + Self::Stalker | Self::StalkerBatch => InputCapabilities { + persistence: InputPersistence::Stalker, + requires_provider_connection_for_probe: true, + served_on_custom_provider_endpoint: true, + }, + Self::Library => InputCapabilities { + persistence: InputPersistence::Library, + requires_provider_connection_for_probe: false, + served_on_custom_provider_endpoint: false, + }, + Self::Emby | Self::Jellyfin | Self::Plex => InputCapabilities { + persistence: InputPersistence::MediaServer, + requires_provider_connection_for_probe: false, + served_on_custom_provider_endpoint: false, + }, + Self::Staged => InputCapabilities { + persistence: InputPersistence::M3u, + requires_provider_connection_for_probe: false, + served_on_custom_provider_endpoint: false, + }, + } + } + + /// Persistence/load backend family for this input type. + #[must_use] + pub const fn persistence(self) -> InputPersistence { self.capabilities().persistence } +} + +/// Storage/loading backend family an [`InputType`] maps onto. +/// +/// Multiple input variants collapse onto the same persistence family (for +/// example every media-server variant shares the same on-disk format), so +/// persist/load routing can match on this instead of re-listing variants. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum InputPersistence { + M3u, + Xtream, + Library, + MediaServer, + Stalker, +} + +/// Categorical capabilities of an [`InputType`], declared once in +/// [`InputType::capabilities`]. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct InputCapabilities { + /// Storage/loading backend family. + pub persistence: InputPersistence, + /// Whether generic stream probing must open a provider connection. + pub requires_provider_connection_for_probe: bool, + /// Whether the custom-provider HTTP endpoint can serve this input. + pub served_on_custom_provider_endpoint: bool, +} + +#[derive( + Debug, + Copy, + Clone, + serde::Serialize, + serde::Deserialize, + EnumIter, + PartialEq, + Eq, + Default, + Display, + EnumString, + AsRefStr, +)] +#[strum(serialize_all = "UPPERCASE")] +#[serde(rename_all = "UPPERCASE")] +pub enum InputFetchMethod { + #[default] + GET, + POST, +} + +impl InputFetchMethod { + pub fn is_default(value: &InputFetchMethod) -> bool { matches!(value, Self::GET) } +} diff --git a/shared/src/model/config/input/options.rs b/shared/src/model/config/input/options.rs new file mode 100644 index 000000000..1a4ece175 --- /dev/null +++ b/shared/src/model/config/input/options.rs @@ -0,0 +1,220 @@ +use crate::{ + defaults::{ + default_as_true, default_probe_delay_secs, default_probe_live_interval, default_resolve_background, + default_resolve_delay_secs, default_xtream_live_stream_use_prefix, is_default_probe_delay_secs, + is_default_probe_live_interval, is_default_resolve_delay_secs, is_false, is_true, + }, + error::TuliproxError, + foundation::{get_filter, Filter}, + model::{ConfigInputUpdateQualityDto, PatternTemplate, Prepare}, + utils::is_blank_optional_string, +}; + +#[derive( + Debug, + Clone, + Copy, + Default, + serde::Serialize, + serde::Deserialize, + PartialEq, + Eq, + strum_macros::Display, + strum_macros::EnumString, + strum_macros::EnumIter, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum FlussonicHlsCatchup { + #[default] + Native, + BoundedArchive, +} + +impl FlussonicHlsCatchup { + pub fn is_native(value: &Self) -> bool { matches!(value, Self::Native) } +} + +pub const fn default_flussonic_hls_catchup_max_duration_secs() -> u32 { 4 * 60 * 60 } + +fn is_default_flussonic_hls_catchup_max_duration_secs(value: &u32) -> bool { + *value == default_flussonic_hls_catchup_max_duration_secs() +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ConfigInputOptionsDto { + #[serde(default, alias = "xtream_skip_live", alias = "stalker_skip_live", skip_serializing_if = "is_false")] + pub skip_live: bool, + #[serde(default, alias = "xtream_skip_vod", alias = "stalker_skip_vod", skip_serializing_if = "is_false")] + pub skip_vod: bool, + #[serde(default, alias = "xtream_skip_series", alias = "stalker_skip_series", skip_serializing_if = "is_false")] + pub skip_series: bool, + #[serde(default, skip_serializing_if = "ConfigInputUpdateQualityDto::is_disabled")] + pub update_quality: ConfigInputUpdateQualityDto, + #[serde(default = "default_xtream_live_stream_use_prefix", skip_serializing_if = "is_true")] + pub xtream_live_stream_use_prefix: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub xtream_live_stream_without_extension: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub disable_hls_streaming: bool, + #[serde(default, skip_serializing_if = "FlussonicHlsCatchup::is_native")] + pub flussonic_hls_catchup: FlussonicHlsCatchup, + #[serde( + default = "default_flussonic_hls_catchup_max_duration_secs", + skip_serializing_if = "is_default_flussonic_hls_catchup_max_duration_secs" + )] + pub flussonic_hls_catchup_max_duration_secs: u32, + #[serde(default, skip_serializing_if = "is_false")] + pub user_agent_stream_index: bool, + /// Labels fragmented-MP4 objects below Flussonic `tracks-a` paths as `audio/mp4` when the + /// provider sends no specific Content-Type. + #[serde(default, skip_serializing_if = "is_false")] + pub flussonic_hls_audio_tracks: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub resolve_tmdb: bool, + #[serde(default = "default_resolve_background", skip_serializing_if = "is_true")] + pub resolve_background: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub resolve_series: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub resolve_vod: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub probe_series: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub probe_vod: bool, + #[serde(default = "default_resolve_delay_secs", skip_serializing_if = "is_default_resolve_delay_secs")] + pub resolve_delay: u16, + #[serde(default = "default_probe_delay_secs", skip_serializing_if = "is_default_probe_delay_secs")] + pub probe_delay: u16, + #[serde(default, alias = "resolve_live", skip_serializing_if = "is_false")] + pub probe_live: bool, + #[serde( + default = "default_probe_live_interval", + alias = "resolve_live_interval_hours", + skip_serializing_if = "is_default_probe_live_interval" + )] + pub probe_live_interval_hours: u32, + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub resolve_filter: Option, + #[serde(skip)] + pub t_resolve_filter: Option, + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub probe_filter: Option, + #[serde(skip)] + pub t_probe_filter: Option, + /// Bulk-fetch EPG for live channels during playlist processing. + #[serde(default, skip_serializing_if = "is_false")] + pub stalker_bulk_epg: bool, +} + +impl Default for ConfigInputOptionsDto { + fn default() -> Self { + ConfigInputOptionsDto { + skip_live: false, + skip_vod: false, + skip_series: false, + update_quality: ConfigInputUpdateQualityDto::default(), + xtream_live_stream_use_prefix: default_xtream_live_stream_use_prefix(), + xtream_live_stream_without_extension: false, + disable_hls_streaming: false, + flussonic_hls_catchup: FlussonicHlsCatchup::Native, + flussonic_hls_catchup_max_duration_secs: default_flussonic_hls_catchup_max_duration_secs(), + user_agent_stream_index: false, + flussonic_hls_audio_tracks: false, + resolve_tmdb: false, + resolve_background: default_resolve_background(), + resolve_series: false, + resolve_vod: false, + probe_series: false, + probe_vod: false, + resolve_delay: default_resolve_delay_secs(), + probe_delay: default_probe_delay_secs(), + probe_live: false, + probe_live_interval_hours: default_probe_live_interval(), + resolve_filter: None, + t_resolve_filter: None, + probe_filter: None, + t_probe_filter: None, + stalker_bulk_epg: false, + } + } +} + +impl ConfigInputOptionsDto { + pub fn is_empty(&self) -> bool { + !self.skip_live + && !self.skip_vod + && !self.skip_series + && self.update_quality.is_empty() + && self.xtream_live_stream_use_prefix + && !self.xtream_live_stream_without_extension + && !self.disable_hls_streaming + && FlussonicHlsCatchup::is_native(&self.flussonic_hls_catchup) + && is_default_flussonic_hls_catchup_max_duration_secs(&self.flussonic_hls_catchup_max_duration_secs) + && !self.user_agent_stream_index + && !self.flussonic_hls_audio_tracks + && !self.resolve_tmdb + && self.resolve_background + && !self.resolve_series + && !self.resolve_vod + && !self.probe_series + && !self.probe_vod + && is_default_resolve_delay_secs(&self.resolve_delay) + && is_default_probe_delay_secs(&self.probe_delay) + && !self.probe_live + && is_default_probe_live_interval(&self.probe_live_interval_hours) + && self.resolve_filter.as_ref().is_none_or(|s| s.trim().is_empty()) + && self.probe_filter.as_ref().is_none_or(|s| s.trim().is_empty()) + && !self.stalker_bulk_epg + } + + pub fn clean(&mut self) { + self.skip_live = false; + self.skip_vod = false; + self.skip_series = false; + self.update_quality.clean(); + self.xtream_live_stream_use_prefix = default_as_true(); + self.xtream_live_stream_without_extension = false; + self.disable_hls_streaming = false; + self.flussonic_hls_catchup = FlussonicHlsCatchup::Native; + self.flussonic_hls_catchup_max_duration_secs = default_flussonic_hls_catchup_max_duration_secs(); + self.user_agent_stream_index = false; + self.flussonic_hls_audio_tracks = false; + self.resolve_tmdb = false; + self.resolve_background = default_as_true(); + self.resolve_series = false; + self.resolve_vod = false; + self.probe_series = false; + self.probe_vod = false; + self.resolve_delay = default_resolve_delay_secs(); + self.probe_delay = default_probe_delay_secs(); + self.probe_live = false; + self.probe_live_interval_hours = default_probe_live_interval(); + self.resolve_filter = None; + self.t_resolve_filter = None; + self.probe_filter = None; + self.t_probe_filter = None; + self.stalker_bulk_epg = false; + } +} + +impl Prepare for ConfigInputOptionsDto { + type Ctx<'a> = Option<&'a [PatternTemplate]>; + + fn prepare(&mut self, templates: Self::Ctx<'_>) -> Result<(), TuliproxError> { + self.update_quality.prepare(())?; + if !(1..=7 * 24 * 60 * 60).contains(&self.flussonic_hls_catchup_max_duration_secs) { + return Err(TuliproxError::ConfigInput( + "flussonic_hls_catchup_max_duration_secs must be between 1 and 604800 seconds".to_string(), + )); + } + if let Some(raw_filter) = &self.resolve_filter { + self.t_resolve_filter = Some(get_filter(raw_filter, templates)?); + } + if let Some(raw_filter) = &self.probe_filter { + self.t_probe_filter = Some(get_filter(raw_filter, templates)?); + } + Ok(()) + } +} diff --git a/shared/src/model/config/input/prepare.rs b/shared/src/model/config/input/prepare.rs new file mode 100644 index 000000000..60cab2701 --- /dev/null +++ b/shared/src/model/config/input/prepare.rs @@ -0,0 +1,458 @@ +use super::{ConfigInputAliasDto, ConfigInputDto, InputType, StagedInputType}; +use crate::{ + check_input_connections, check_input_credentials, + error::TuliproxError, + model::{PatternTemplate, Prepare}, + utils::{ + get_credentials_from_url_str, get_trimmed_string, is_non_blank_optional_string, parse_duration_seconds, + parse_provider_scheme_url_parts, sanitize_sensitive_info, trim_last_slash, Internable, BATCH_SCHEME_PREFIX, + PROVIDER_SCHEME_PREFIX, + }, +}; +use log::warn; +use std::{collections::HashSet, sync::Arc}; + +impl ConfigInputDto { + pub(super) fn normalize_input_type_from_batch_url(&mut self) { + let is_batch_url = self.url.trim().starts_with(BATCH_SCHEME_PREFIX); + self.input_type = match self.input_type { + InputType::M3u | InputType::M3uBatch => { + if is_batch_url { + InputType::M3uBatch + } else { + InputType::M3u + } + } + InputType::Xtream | InputType::XtreamBatch => { + if is_batch_url { + InputType::XtreamBatch + } else { + InputType::Xtream + } + } + InputType::Stalker | InputType::StalkerBatch => { + if is_batch_url { + InputType::StalkerBatch + } else { + InputType::Stalker + } + } + InputType::Library => InputType::Library, + InputType::Emby => InputType::Emby, + InputType::Jellyfin => InputType::Jellyfin, + InputType::Plex => InputType::Plex, + InputType::Staged => InputType::Staged, + }; + } + + pub(super) fn prepare_media_server_input(&mut self) -> Result<(), TuliproxError> { + if !self.input_type.is_media_server() { + return Ok(()); + } + + let trimmed_url = self.url.trim(); + if trimmed_url.starts_with(BATCH_SCHEME_PREFIX) || trimmed_url.starts_with(PROVIDER_SCHEME_PREFIX) { + return Err(TuliproxError::ConfigInput(format!( + "media-server input does not support batch:// or provider:// URLs (input: {})", + self.name + ))); + } + if self.aliases.as_ref().is_some_and(|aliases| !aliases.is_empty()) { + return Err(TuliproxError::ConfigInput(format!( + "media-server input does not support aliases (input: {})", + self.name + ))); + } + if self.epg.is_some() { + return Err(TuliproxError::ConfigInput(format!( + "media-server input does not support EPG configuration (input: {})", + self.name + ))); + } + if self.panel_api.is_some() { + return Err(TuliproxError::ConfigInput(format!( + "media-server input does not support panel_api configuration (input: {})", + self.name + ))); + } + if self.provider.as_ref().is_some_and(|provider| !provider.is_empty()) { + return Err(TuliproxError::ConfigInput(format!( + "media-server input does not support provider failover definitions (input: {})", + self.name + ))); + } + let Some(media_server) = self.media_server.as_mut() else { + return Err(TuliproxError::ConfigInput(format!( + "media_server configuration is mandatory for input type {} (input: {})", + self.input_type, self.name + ))); + }; + media_server.prepare(&self.name)?; + if media_server.libraries.is_empty() { + return Err(TuliproxError::ConfigInput(format!( + "media-server input requires at least one selected library (input: {})", + self.name + ))); + } + + match self.input_type { + InputType::Emby | InputType::Jellyfin => { + if trimmed_url.is_empty() { + return Err(TuliproxError::ConfigInput(format!( + "url is mandatory for input type {} (input: {})", + self.input_type, self.name + ))); + } + let has_login = self.username.as_ref().is_some_and(|u| !u.trim().is_empty()) + && self.password.as_ref().is_some_and(|p| !p.trim().is_empty()); + if !media_server.has_any_emby_jellyfin_auth() && !has_login { + return Err(TuliproxError::ConfigInput(format!( + "media-server input type {} requires media_server token/api_key or username/password bootstrap credentials (input: {})", + self.input_type, self.name + ))); + } + } + InputType::Plex => { + if trimmed_url.is_empty() { + if !is_non_blank_optional_string(&media_server.account_token) { + return Err(TuliproxError::ConfigInput(format!( + "media-server input type plex without input.url requires media_server.account_token for MyPlex discovery (input: {})", + self.name + ))); + } + if !media_server.has_plex_server_selector() { + return Err(TuliproxError::ConfigInput(format!( + "media-server input type plex requires a server selector such as media_server.server_id or media_server.server_name when input.url is omitted (input: {})", + self.name + ))); + } + } else if !is_non_blank_optional_string(&media_server.token) { + return Err(TuliproxError::ConfigInput(format!( + "media-server input type plex with input.url requires media_server.token for direct PMS access (input: {})", + self.name + ))); + } + } + InputType::M3u + | InputType::Xtream + | InputType::M3uBatch + | InputType::XtreamBatch + | InputType::Stalker + | InputType::StalkerBatch + | InputType::Library + | InputType::Staged => {} + } + + Ok(()) + } + + pub(super) fn validate_staged_self(&mut self) -> Result<(), TuliproxError> { + if self.input_type.is_staged() { + let Some(staged) = self.staged.as_mut() else { + return Err(TuliproxError::ConfigInput(format!( + "staged input requires a provider input (input: {})", + self.name + ))); + }; + if let Some(for_input) = staged.for_input.as_ref().map(|p| p.trim()).filter(|p| !p.is_empty()) { + staged.for_input = Some(for_input.intern()); + } else { + return Err(TuliproxError::ConfigInput(format!( + "staged input requires a provider input (input: {})", + self.name + ))); + } + if staged.clusters.is_empty() { + return Err(TuliproxError::ConfigInput(format!( + "staged input requires at least one staged cluster (input: {})", + self.name + ))); + } + if self.url.trim().is_empty() { + return Err(TuliproxError::ConfigInput(format!( + "url for staged input is mandatory (input: {})", + self.name + ))); + } + if self.staged_type == StagedInputType::Xtream { + let has_credentials = self.username.as_ref().is_some_and(|u| !u.trim().is_empty()) + && self.password.as_ref().is_some_and(|p| !p.trim().is_empty()); + if !has_credentials { + return Err(TuliproxError::ConfigInput(format!( + "staged xtream input requires username and password (input: {})", + self.name + ))); + } + } + // R7 + if self.media_server.is_some() { + return Err(TuliproxError::ConfigInput(format!( + "staged input does not support media_server configuration (input: {})", + self.name + ))); + } + if self.panel_api.is_some() { + return Err(TuliproxError::ConfigInput(format!( + "staged input does not support panel_api configuration (input: {})", + self.name + ))); + } + } else if self.staged.is_some() { + return Err(TuliproxError::ConfigInput(format!( + "staged configuration is only allowed for staged inputs (input: {})", + self.name + ))); + } + Ok(()) + } + + pub fn prepare( + &mut self, + index: u16, + _include_computed: bool, + provider_names: &HashSet, + templates: Option<&[PatternTemplate]>, + ) -> Result { + self.name = self.name.trim().intern(); + if self.name.is_empty() { + return Err(TuliproxError::ConfigInput("name for input is mandatory".to_string())); + } + + if self.sequential_group == Some(0) { + return Err(TuliproxError::ConfigInput(format!( + "sequential_group must be greater than zero (input: {})", + self.name + ))); + } + if self.input_type.is_staged() && self.sequential_group.is_some() { + return Err(TuliproxError::ConfigInput(format!( + "staged input does not support sequential_group (input: {})", + self.name + ))); + } + + if self.input_type.is_staged() { + self.priority = 0; + self.max_connections = 0; + self.cache_duration = None; + self.cache_duration_seconds = 0; + } + + if let Some(duration_str) = &self.cache_duration { + self.cache_duration_seconds = self.parse_duration(duration_str)?; + } else { + self.cache_duration_seconds = 0; + } + + self.url = self.url.trim().to_string(); + self.normalize_input_type_from_batch_url(); + if let Some(options) = self.options.as_ref() { + if options.disable_hls_streaming && !self.input_type.is_xtream() { + return Err(TuliproxError::ConfigInput(format!( + "`disable_hls_streaming` is supported only for Xtream inputs (input: {})", + self.name + ))); + } + } + if let Some(media_server) = self.media_server.as_mut() { + media_server.normalize(); + } + if self.enabled { + self.prepare_media_server_input()?; + } + if self.url.starts_with(PROVIDER_SCHEME_PREFIX) && self.input_type.is_batch() { + return Err(TuliproxError::ConfigInput(format!( + "input type {} does not support provider:// URLs for batch definitions; use batch:// URL (input: {})", + self.input_type, self.name + ))); + } + + check_input_credentials!(self, self.input_type, true, false); + check_input_connections!(self, self.input_type, false); + // Always run stalker validation: it rejects a stray `stalker` block on + // non-stalker inputs and surfaces a malformed MAC at config load even + // when the input is currently disabled (consistent with aliases). + self.prepare_stalker_input()?; + self.validate_staged_self()?; + + self.persist = get_trimmed_string(self.persist.as_deref()); + check_provider_scheme_url!(self.url, provider_names); + + let mut current_index = index + 1; + self.id = current_index; + if let Some(aliases) = self.aliases.as_mut() { + let input_type = &self.input_type; + for alias in aliases { + current_index = alias.prepare(current_index, input_type)?; + check_provider_scheme_url!(alias.url.as_str(), provider_names); + } + } + + if let Some(panel_api) = self.panel_api.as_mut() { + panel_api.prepare(&self.name)?; + } + + // Validate provider:// URLs in EPG sources + if let Some(epg) = self.epg.as_ref() { + if let Some(sources) = epg.sources.as_ref() { + for epg_source in sources { + let url = epg_source.url.trim(); + check_provider_scheme_url!(url, provider_names); + } + } + } + + // Prepare filter options + if let Some(options) = self.options.as_mut() { + options.prepare(templates)?; + } + + Ok(current_index) + } + + pub(super) fn parse_duration(&self, duration_str: &str) -> Result { + match parse_duration_seconds(duration_str, false) { + Some(seconds) => Ok(seconds), + None => Err(TuliproxError::ConfigInput(format!( + "Invalid cache_duration format in '{}': {}", + self.name, duration_str + ))), + } + } + + // Neue ausgelagerte Methode für die URL-Generierung + pub(super) fn generate_auto_epg_url(&self) -> Result { + let get_creds = || { + if self.username.is_some() && self.password.is_some() { + return (self.username.clone(), self.password.clone(), Some(self.url.clone())); + } + + let (u, p, r) = self + .aliases + .as_ref() + .and_then(|aliases| aliases.iter().find(|a| a.enabled)) + .map_or((None, None, None), |alias| { + (alias.username.clone(), alias.password.clone(), Some(alias.url.clone())) + }); + + if u.is_some() && p.is_some() && r.is_some() { + return (u, p, r); + } + + let (u, p) = get_credentials_from_url_str(&self.url); + if u.is_some() && p.is_some() { + return (u, p, Some(self.url.clone())); + } + + self.aliases.as_ref().and_then(|aliases| aliases.iter().find(|a| a.enabled)).map_or( + (None, None, None), + |alias| { + let (u, p) = get_credentials_from_url_str(alias.url.as_str()); + (u, p, Some(alias.url.clone())) + }, + ) + }; + + let (username, password, base_url) = get_creds(); + + if username.is_none() || password.is_none() || base_url.is_none() { + Err(format!("auto_epg is enabled for input {}, but no credentials could be extracted", self.name)) + } else if let Some(base) = base_url { + let clean_base = base.split('?').next().unwrap_or(&base); + + let provider_epg_url = format!( + "{}/xmltv.php?username={}&password={}", + trim_last_slash(clean_base), + username.unwrap_or_default(), + password.unwrap_or_default() + ); + Ok(provider_epg_url) + } else { + Err(format!( + "auto_epg is enabled for input {}, but url could not be parsed {}", + self.name, + sanitize_sensitive_info(&self.url) + )) + } + } + + pub fn prepare_epg(&mut self, include_computed: bool) -> Result<(), TuliproxError> { + if let Some(mut epg) = self.epg.take() { + if self.input_type == InputType::Library { + warn!("EPG is not supported for library inputs {}, skipping", self.name); + self.epg = None; + return Ok(()); + } + + epg.prepare(|| self.generate_auto_epg_url(), include_computed)?; + epg.t_sources = { + let mut seen_sources = HashSet::new(); + epg.t_sources.drain(..).filter(|src| seen_sources.insert(src.source_identity())).collect() + }; + self.epg = Some(epg); + } + Ok(()) + } + + pub fn prepare_batch( + &mut self, + batch_aliases: Vec, + index: u16, + ) -> Result, TuliproxError> { + let idx = apply_batch_aliases!(self, batch_aliases, Some(index)); + Ok(idx) + } + + pub fn prepare_type(&mut self) -> Result<(), TuliproxError> { + self.url = self.url.trim().to_string(); + self.normalize_input_type_from_batch_url(); + if self.url.starts_with(PROVIDER_SCHEME_PREFIX) && self.input_type.is_batch() { + return Err(TuliproxError::ConfigInput(format!( + "input type {} does not support provider:// URLs for batch definitions; use batch:// URL", + self.input_type + ))); + } + Ok(()) + } + + pub fn upsert_alias(&mut self, mut alias: ConfigInputAliasDto) -> Result<(), TuliproxError> { + check_input_credentials!(alias, self.input_type, true, true); + check_input_connections!(alias, self.input_type, true); + let aliases = self.aliases.get_or_insert_with(Vec::new); + if let Some(existing) = aliases.iter_mut().find(|a| a.id == alias.id) { + *existing = alias; + } else { + aliases.push(alias); + } + Ok(()) + } + + pub fn update_account_expiration_date( + &mut self, + account_name: &str, + exp_date: i64, + disable: bool, + ) -> Result { + if self.name.as_ref() == account_name { + let changed = self.exp_date != Some(exp_date) || disable && !self.account_disabled; + self.exp_date = Some(exp_date); + self.account_disabled |= disable; + return Ok(changed); + } + let (expiration, enabled) = if let Some(alias) = self + .aliases + .as_mut() + .and_then(|aliases| aliases.iter_mut().find(|alias| alias.name.as_ref() == account_name)) + { + (&mut alias.exp_date, &mut alias.enabled) + } else { + return Err(TuliproxError::ConfigInput(format!("No input or alias found for account '{account_name}'"))); + }; + let changed = *expiration != Some(exp_date) || disable && *enabled; + *expiration = Some(exp_date); + if disable { + *enabled = false; + } + Ok(changed) + } +} diff --git a/shared/src/model/config/input/provider.rs b/shared/src/model/config/input/provider.rs new file mode 100644 index 000000000..9f24765e8 --- /dev/null +++ b/shared/src/model/config/input/provider.rs @@ -0,0 +1,206 @@ +use crate::{ + defaults::is_false, + error::TuliproxError, + utils::{arc_str_serde, arc_str_vec_serde, Internable}, +}; +use log::warn; +use std::{collections::HashMap, net::IpAddr, sync::Arc}; +use strum_macros::{AsRefStr, Display, EnumString}; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ConfigProviderDto { + #[serde(with = "arc_str_serde")] + pub name: Arc, + #[serde(with = "arc_str_vec_serde")] + pub urls: Vec>, + #[serde(default, skip_serializing_if = "is_default_provider_url_selection_policy")] + pub provider_url_selection_policy: ProviderUrlSelectionPolicy, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dns: Option, +} + +impl ConfigProviderDto { + pub fn prepare(&mut self) -> Result<(), TuliproxError> { + self.name = self.name.trim().intern(); + if self.name.is_empty() { + return Err(TuliproxError::ConfigInput("Name for provider is mandatory".to_string())); + } + self.urls = self.urls.drain(..).filter(|url| !url.trim().is_empty()).map(|u| u.trim().intern()).collect(); + if self.urls.is_empty() { + return Err(TuliproxError::ConfigInput("Urls for provider is mandatory".to_string())); + } + if let Some(dns) = self.dns.as_mut() { + dns.prepare()?; + } + Ok(()) + } +} + +pub const fn default_provider_dns_refresh_secs() -> u64 { 300 } + +pub const fn is_default_provider_dns_refresh_secs(v: &u64) -> bool { *v == default_provider_dns_refresh_secs() } + +pub fn is_default_provider_url_selection_policy(v: &ProviderUrlSelectionPolicy) -> bool { + *v == ProviderUrlSelectionPolicy::default() +} + +pub fn is_default_dns_prefer(v: &DnsPrefer) -> bool { *v == DnsPrefer::default() } + +pub fn is_default_on_resolve_error(v: &OnResolveErrorPolicy) -> bool { *v == OnResolveErrorPolicy::default() } + +pub fn is_default_on_connect_error(v: &OnConnectErrorPolicy) -> bool { *v == OnConnectErrorPolicy::default() } + +#[derive( + Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default, EnumString, AsRefStr, Display, +)] +#[strum(serialize_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum ProviderUrlSelectionPolicy { + #[default] + ResumeLastWorking, + RestartFromFirst, +} + +#[derive( + Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default, EnumString, AsRefStr, Display, +)] +#[strum(serialize_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum DnsPrefer { + Ipv4, + Ipv6, + #[default] + System, +} + +#[derive(Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, EnumString, AsRefStr, Display)] +#[strum(serialize_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum DnsScheme { + Http, + Https, +} + +#[derive( + Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default, EnumString, AsRefStr, Display, +)] +#[strum(serialize_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum OnResolveErrorPolicy { + #[default] + KeepLastGood, + FallbackToHostname, +} + +#[derive( + Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default, EnumString, AsRefStr, Display, +)] +#[strum(serialize_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum OnConnectErrorPolicy { + #[default] + TryNextIp, + RotateProviderUrl, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ProviderDnsDto { + #[serde(default, skip_serializing_if = "is_false")] + pub enabled: bool, + #[serde( + default = "default_provider_dns_refresh_secs", + skip_serializing_if = "is_default_provider_dns_refresh_secs" + )] + pub refresh_secs: u64, + #[serde(default, skip_serializing_if = "is_default_dns_prefer")] + pub prefer: DnsPrefer, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_addrs: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schemes: Option>, + #[serde(default, skip_serializing_if = "is_false")] + pub keep_vhost: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub overrides: Option>>, + #[serde(default, skip_serializing_if = "is_default_on_resolve_error")] + pub on_resolve_error: OnResolveErrorPolicy, + #[serde(default, skip_serializing_if = "is_default_on_connect_error")] + pub on_connect_error: OnConnectErrorPolicy, +} + +impl Default for ProviderDnsDto { + fn default() -> Self { + Self { + enabled: false, + refresh_secs: default_provider_dns_refresh_secs(), + prefer: DnsPrefer::default(), + max_addrs: None, + schemes: None, + keep_vhost: false, + overrides: None, + on_resolve_error: OnResolveErrorPolicy::default(), + on_connect_error: OnConnectErrorPolicy::default(), + } + } +} + +impl ProviderDnsDto { + pub fn prepare(&mut self) -> Result<(), TuliproxError> { + if self.refresh_secs == 0 { + self.refresh_secs = default_provider_dns_refresh_secs(); + } else if self.refresh_secs < 10 { + // Explicit low values are honored for fast failover, but flagged + warn!( + "Provider dns refresh_secs={} is below the recommended minimum of 10s; this increases resolver load", + self.refresh_secs + ); + } + if self.max_addrs == Some(0) { + return Err(TuliproxError::ConfigInput("Provider dns max_addrs must be >= 1 when set".to_string())); + } + if let Some(schemes) = self.schemes.as_mut() { + let mut unique = Vec::with_capacity(schemes.len()); + for scheme in schemes.drain(..) { + if !unique.contains(&scheme) { + unique.push(scheme); + } + } + *schemes = unique; + if schemes.is_empty() { + self.schemes = None; + } + } + + if let Some(overrides) = self.overrides.as_mut() { + let mut normalized: HashMap> = HashMap::new(); + for (host, ips) in std::mem::take(overrides) { + let host = host.trim().to_ascii_lowercase(); + if host.is_empty() { + return Err(TuliproxError::ConfigInput( + "Provider dns overrides hostname must not be empty".to_string(), + )); + } + if ips.is_empty() { + return Err(TuliproxError::ConfigInput( + "Provider dns overrides for host '{host}' must not be empty".to_string(), + )); + } + let entry = normalized.entry(host.clone()).or_default(); + for ip in ips { + if !entry.contains(&ip) { + entry.push(ip); + } + } + } + if normalized.is_empty() { + self.overrides = None; + } else { + *overrides = normalized; + } + } + + Ok(()) + } +} diff --git a/shared/src/model/config/input/stalker.rs b/shared/src/model/config/input/stalker.rs new file mode 100644 index 000000000..08bf03717 --- /dev/null +++ b/shared/src/model/config/input/stalker.rs @@ -0,0 +1,126 @@ +use super::InputType; +use crate::{ + error::TuliproxError, + model::{StalkerAuthMode, StalkerInputConfigDto}, +}; + +/// Normalize a user-supplied MAC address into the canonical lowercase +/// `xx:xx:xx:xx:xx:xx` form. Accepts colon-, dash- and bare-hex formats +/// (all three are common in portal provisioning exports). Returns `None` +/// when the value is not a valid 6-octet MAC. +fn normalize_mac_address(raw: &str) -> Option { + let trimmed = raw.trim(); + let octets: Vec = if trimmed.contains(':') || trimmed.contains('-') { + trimmed.split(['-', ':']).map(str::to_string).collect() + } else if trimmed.len() == 12 { + trimmed.as_bytes().chunks(2).map(|c| String::from_utf8_lossy(c).to_string()).collect() + } else { + return None; + }; + let valid = octets.len() == 6 && octets.iter().all(|p| p.len() == 2 && p.chars().all(|c| c.is_ascii_hexdigit())); + if !valid { + return None; + } + Some(octets.join(":").to_ascii_lowercase()) +} + +pub(super) fn prepare_stalker_config( + input_name: &str, + input_type: &InputType, + stalker: &mut Option, + is_alias: bool, + username: Option<&str>, + password: Option<&str>, +) -> Result<(), TuliproxError> { + if !input_type.is_stalker() { + if stalker.is_some() { + return Err(TuliproxError::ConfigInput(format!( + "stalker configuration is only valid for stalker inputs (input: {input_name})" + ))); + } + return Ok(()); + } + + // Aliases inherit the parent's stalker block when they define none of + // their own — never materialize a default here, otherwise the inherited + // config would be shadowed by an empty one in `as_input`. + let config = if is_alias { + match stalker.as_mut() { + Some(config) => config, + None => return Ok(()), + } + } else { + stalker.get_or_insert_with(StalkerInputConfigDto::default) + }; + + let mut has_mac = false; + // Only normalize an existing device block — do not materialize an empty + // one (it would be re-serialized as a noise `device: {}` block). + if let Some(device) = config.device.as_mut() { + // MAC validation — accept colon-, dash- and bare-hex formats and + // normalize to lowercase `xx:xx:xx:xx:xx:xx`. Empty MAC is allowed + // at this point (auth mode might be credentials-only). + if let Some(mac) = device.mac_address.as_ref() { + let trimmed = mac.trim(); + if !trimmed.is_empty() { + let Some(normalized) = normalize_mac_address(trimmed) else { + return Err(TuliproxError::ConfigInput(format!( + "stalker.device.mac_address must be a MAC address in XX:XX:XX:XX:XX:XX, XX-XX-XX-XX-XX-XX or bare-hex format (input: {input_name})" + ))); + }; + let first_octet = u8::from_str_radix(&normalized[..2], 16).unwrap_or_default(); + if first_octet & 1 != 0 { + return Err(TuliproxError::ConfigInput(format!( + "stalker.device.mac_address must be a unicast MAC address (input: {input_name})" + ))); + } + device.mac_address = Some(normalized); + has_mac = true; + } + } + + // Trim locale/timezone if the user supplied them. + if let Some(timezone) = device.timezone.as_mut() { + *timezone = timezone.trim().to_string(); + if timezone.is_empty() { + device.timezone = None; + } + } + if let Some(locale) = device.locale.as_mut() { + *locale = locale.trim().to_string(); + if locale.is_empty() { + device.locale = None; + } + } + if let Some(profile) = device.device_profile.as_mut() { + *profile = profile.trim().to_string(); + if profile.is_empty() { + device.device_profile = None; + } + } + } + + // Aliases may inherit the parent's identity, so only main inputs are checked. + if !is_alias && !input_type.is_batch() { + let has_credentials = username.is_some_and(|value| !value.trim().is_empty()) + && password.is_some_and(|value| !value.trim().is_empty()); + let requirement = match config.auth_mode { + StalkerAuthMode::Auto if !has_mac && !has_credentials => { + Some("stalker.device.mac_address or username/password") + } + StalkerAuthMode::MacOnly if !has_mac => Some("stalker.device.mac_address for auth_mode mac_only"), + StalkerAuthMode::CredentialsOnly if !has_credentials => { + Some("username and password for auth_mode credentials_only") + } + StalkerAuthMode::MacPlusCredentials if !has_mac || !has_credentials => { + Some("stalker.device.mac_address, username and password for auth_mode mac_plus_credentials") + } + _ => None, + }; + if let Some(requirement) = requirement { + return Err(TuliproxError::ConfigInput(format!("Stalker input '{input_name}' requires {requirement}"))); + } + } + + Ok(()) +} diff --git a/shared/src/model/config/input/tests.rs b/shared/src/model/config/input/tests.rs new file mode 100644 index 000000000..7ad28607e --- /dev/null +++ b/shared/src/model/config/input/tests.rs @@ -0,0 +1,13 @@ +use super::*; +use crate::model::{ClusterFlags, ConfigInputUpdateQualityDto, Prepare, StalkerAuthMode, StalkerDeviceProfileDto}; +use std::{collections::HashSet, net::IpAddr}; + +fn create_test_dto() -> ConfigInputDto { ConfigInputDto { name: "test_input".intern(), ..ConfigInputDto::default() } } + +fn prepare_dto(dto: &mut ConfigInputDto) -> Result { dto.prepare(0, false, &HashSet::new(), None) } + +mod aliases; +mod input; +mod provider; +mod staged_media_server; +mod stalker; diff --git a/shared/src/model/config/input/tests/aliases.rs b/shared/src/model/config/input/tests/aliases.rs new file mode 100644 index 000000000..2982e9b34 --- /dev/null +++ b/shared/src/model/config/input/tests/aliases.rs @@ -0,0 +1,328 @@ +use super::*; + +#[test] +fn config_input_alias_serialization_uses_block_default_and_reads_both_styles() { + for aliases in [ + "- {name: alias-one, url: provider://example, username: alias, password: 'pass # with: {}, commas', max_connections: 1, exp_date: 1788779145}\n", + "- name: alias-one\n url: provider://example\n username: alias\n password: 'pass # with: {}, commas'\n max_connections: 1\n exp_date: 1788779145\n", + ] { + let yaml = format!( + "name: provider\ntype: xtream\nurl: provider://example\nusername: root\npassword: pass\noptions:\n update_quality:\n live: 85\n vod: 85\n series: 85\n resolve_background: false\ncache_duration: 20h\nmax_connections: 1\nexp_date: 1788779143\naliases:\n{aliases}panel_api:\n url: http://panel.example\n api_key: synthetic-key\n" + ); + let input: ConfigInputDto = serde_saphyr::from_str(&yaml).expect("both alias styles parse"); + let serialized = serde_saphyr::to_string(&input).expect("serialize input"); + assert!(serialized.contains("- name: alias-one\n")); + assert!(!serialized.contains("- {")); + let reparsed: ConfigInputDto = serde_saphyr::from_str(&serialized).expect("read serialized input"); + assert_eq!(reparsed, input); + let json = serde_json::to_string(&input).expect("JSON serialization"); + assert_eq!(serde_json::from_str::(&json).expect("JSON roundtrip"), input); + + let mut prepared = input; + prepared.prepare(0, false, &HashSet::from(["example".to_string()]), None).expect("prepare input"); + assert_eq!(prepared.cache_duration_seconds, 72_000); + let options = prepared.options.as_ref().expect("options"); + assert_eq!(options.update_quality, ConfigInputUpdateQualityDto { live: 85, vod: 85, series: 85 }); + assert!(!options.resolve_background); + assert_eq!(prepared.aliases.as_ref().expect("aliases").len(), 1); + assert!(prepared.panel_api.is_some()); + } +} + +#[test] +fn config_input_alias_serialization_preserves_empty_and_missing_lists() { + for aliases in [None, Some(Vec::new())] { + let input = ConfigInputDto { name: "provider".intern(), aliases, ..Default::default() }; + let serialized = serde_saphyr::to_string(&input).expect("serialize input"); + assert_eq!(serde_saphyr::from_str::(&serialized).expect("roundtrip"), input); + } +} + +#[test] +fn test_epg_url_from_explicit_main_credentials() { + let mut dto = create_test_dto(); + // Hier testen wir auch gleich mit, ob der Trailing Slash sauber entfernt wird! + dto.url = "http://myprovider.com/".to_string(); + dto.username = Some("hello".to_string()); + dto.password = Some("mello".to_string()); + + let result = dto.generate_auto_epg_url().unwrap(); + assert_eq!(result, "http://myprovider.com/xmltv.php?username=hello&password=mello"); +} + +#[test] +fn test_epg_url_from_enabled_alias_explicit_credentials() { + let mut dto = create_test_dto(); + dto.url = "http://main.com".to_string(); + + let alias = ConfigInputAliasDto { + enabled: true, + url: "http://alias.com".to_string(), + username: Some("alias_user".to_string()), + password: Some("alias_pass".to_string()), + ..ConfigInputAliasDto::default() + }; + + dto.aliases = Some(vec![alias]); + + let result = dto.generate_auto_epg_url().unwrap(); + // Er muss die URL und die Credentials vom Alias nehmen + assert_eq!(result, "http://alias.com/xmltv.php?username=alias_user&password=alias_pass"); +} + +#[test] +fn test_epg_url_skips_disabled_aliases() { + let mut dto = create_test_dto(); + + let alias = ConfigInputAliasDto { + enabled: false, // Alias ist deaktiviert! + url: "http://alias.com".to_string(), + username: Some("alias_user".to_string()), + password: Some("alias_pass".to_string()), + ..ConfigInputAliasDto::default() + }; + + dto.aliases = Some(vec![alias]); + + let result = dto.generate_auto_epg_url(); + // Since the main DTO is empty and alias is disabled, an error must occur + assert!(result.is_err()); + assert!(result.unwrap_err().contains("no credentials could be extracted")); +} + +#[test] +fn test_epg_url_fails_without_credentials() { + let mut dto = create_test_dto(); + dto.url = "http://nocreds.com".to_string(); + + let result = dto.generate_auto_epg_url(); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("no credentials could be extracted")); +} + +#[test] +fn test_epg_url_from_main_url_query_credentials() { + let mut dto = create_test_dto(); + // Credentials stecken als Query-Parameter in der URL + dto.url = "http://myprovider.com?username=hello&password=mello".to_string(); + + let result = dto.generate_auto_epg_url().unwrap(); + + // Durch unseren sauberen "clean_base" Fix sieht die URL jetzt richtig aus! + assert_eq!(result, "http://myprovider.com/xmltv.php?username=hello&password=mello"); +} + +#[test] +fn test_epg_url_from_alias_url_query_credentials() { + let mut dto = create_test_dto(); + dto.url = "http://main.com".to_string(); + + let alias = ConfigInputAliasDto { + enabled: true, + // Credentials im Alias als Query-Parameter + url: "http://alias.com?username=alias_user&password=alias_pass".to_string(), + ..ConfigInputAliasDto::default() + }; + + dto.aliases = Some(vec![alias]); + + let result = dto.generate_auto_epg_url().unwrap(); + assert_eq!(result, "http://alias.com/xmltv.php?username=alias_user&password=alias_pass"); +} + +#[test] +fn test_epg_url_from_provider_scheme_url_query_credentials() { + let mut dto = create_test_dto(); + dto.url = "provider://myprovider".to_string(); + dto.username = Some("test".to_string()); + dto.password = Some("secret".to_string()); + + let result = dto.generate_auto_epg_url().unwrap(); + assert_eq!(result, "provider://myprovider/xmltv.php?username=test&password=secret"); +} + +#[test] +fn prepare_switches_xtream_to_xtream_batch_when_alias_exists() { + let mut dto = ConfigInputDto { + name: "input_alias".intern(), + input_type: InputType::Xtream, + url: "batch:///tmp/input_alias.csv".to_string(), + aliases: Some(vec![ConfigInputAliasDto { + id: 1, + name: "alias_1".intern(), + url: "http://provider.example/stream".to_string(), + username: Some("u".to_string()), + password: Some("p".to_string()), + enabled: true, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }; + + dto.prepare_type().expect("prepare type should succeed"); + dto.prepare(0, true, &HashSet::new(), None).expect("prepare should succeed and infer batch type from batch:// URL"); + assert_eq!(dto.input_type, InputType::XtreamBatch); +} + +#[test] +fn prepare_keeps_xtream_type_when_alias_exists_without_batch_url() { + let mut dto = ConfigInputDto { + name: "input_alias_http".intern(), + input_type: InputType::XtreamBatch, + url: "http://localhost:3001".to_string(), + username: Some("root_user".to_string()), + password: Some("root_pass".to_string()), + aliases: Some(vec![ConfigInputAliasDto { + id: 1, + name: "alias_1".intern(), + url: "http://provider.example/stream".to_string(), + username: Some("u".to_string()), + password: Some("p".to_string()), + enabled: true, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }; + + dto.prepare_type().expect("prepare type should normalize non-batch URL to xtream"); + assert_eq!(dto.input_type, InputType::Xtream); + dto.prepare(0, true, &HashSet::new(), None).expect("prepare should succeed for regular URL with aliases"); + assert_eq!(dto.input_type, InputType::Xtream); +} + +#[test] +fn prepare_batch_url_does_not_require_xtream_credentials() { + let mut dto = ConfigInputDto { + name: "batch_no_creds".intern(), + input_type: InputType::Xtream, + url: "batch:///tmp/no-creds.csv".to_string(), + username: None, + password: None, + ..ConfigInputDto::default() + }; + + dto.prepare(0, true, &HashSet::new(), None) + .expect("batch:// input must be normalized before credential validation"); + assert_eq!(dto.input_type, InputType::XtreamBatch); +} + +#[test] +fn prepare_provider_scheme_url_is_not_treated_as_batch_input() { + let mut dto = ConfigInputDto { + name: "batch_provider".intern(), + input_type: InputType::XtreamBatch, + url: "provider://myprovider".to_string(), + username: Some("root_user".to_string()), + password: Some("root_pass".to_string()), + aliases: Some(vec![ConfigInputAliasDto { + id: 1, + name: "alias_1".intern(), + url: "http://provider.example/stream".to_string(), + username: Some("u".to_string()), + password: Some("p".to_string()), + enabled: true, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }; + + let err = dto + .prepare(0, true, &HashSet::new(), None) + .expect_err("prepare should treat provider:// URL as regular input (non-batch) and validate provider"); + assert!(err.to_string().contains("Provider name myprovider is not defined"), "Error: {err}"); +} + +#[test] +fn prepare_rejects_missing_input_url_even_with_aliases() { + let mut dto = ConfigInputDto { + name: "xtream_missing_root_url".intern(), + input_type: InputType::Xtream, + url: String::new(), + username: Some("root_user".to_string()), + password: Some("root_pass".to_string()), + aliases: Some(vec![ConfigInputAliasDto { + id: 1, + name: "alias_1".intern(), + url: "http://alias.example".to_string(), + username: Some("alias_user".to_string()), + password: Some("alias_pass".to_string()), + enabled: true, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }; + + let err = dto + .prepare(0, true, &HashSet::new(), None) + .expect_err("prepare must require root input url even when aliases are present"); + assert!(err.to_string().contains("url for input is mandatory"), "Error: {err}"); + assert!(err.to_string().contains("xtream_missing_root_url"), "Error: {err}"); +} + +#[test] +fn prepare_rejects_missing_root_credentials_for_non_batch_url_even_with_aliases() { + let mut dto = ConfigInputDto { + name: "xtream_batch_missing_root_creds".intern(), + input_type: InputType::XtreamBatch, + url: "http://root.example".to_string(), + aliases: Some(vec![ConfigInputAliasDto { + id: 1, + name: "alias_1".intern(), + url: "http://alias.example".to_string(), + username: Some("alias_user".to_string()), + password: Some("alias_pass".to_string()), + enabled: true, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }; + + let err = dto + .prepare(0, true, &HashSet::new(), None) + .expect_err("prepare must require root credentials for non-batch URL"); + assert!(err.to_string().contains("for input type xtream: username and password are mandatory"), "Error: {err}"); + assert!(err.to_string().contains("xtream_batch_missing_root_creds"), "Error: {err}"); +} + +#[test] +fn prepare_rejects_xtream_batch_batch_url_with_root_credentials_even_with_aliases() { + let mut dto = ConfigInputDto { + name: "xtream_batch_with_root_creds".intern(), + input_type: InputType::XtreamBatch, + url: "batch:///tmp/aliases.csv".to_string(), + username: Some("root_user".to_string()), + password: Some("root_pass".to_string()), + aliases: Some(vec![ConfigInputAliasDto { + id: 1, + name: "alias_1".intern(), + url: "http://alias.example".to_string(), + username: Some("alias_user".to_string()), + password: Some("alias_pass".to_string()), + enabled: true, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }; + + let err = dto + .prepare(0, true, &HashSet::new(), None) + .expect_err("prepare must reject root credentials when using batch:// for xtream-batch"); + assert!(err.to_string().contains("with batch:// URL should not define username or password"), "Error: {err}"); + assert!(err.to_string().contains("xtream_batch_with_root_creds"), "Error: {err}"); +} + +#[test] +fn config_input_options_deserializes_legacy_prefixed_skip_aliases() { + let dto: ConfigInputOptionsDto = serde_json::from_str( + r#"{ + "xtream_skip_live": true, + "stalker_skip_vod": true, + "skip_series": true + }"#, + ) + .expect("legacy aliases should deserialize"); + + assert!(dto.skip_live); + assert!(dto.skip_vod); + assert!(dto.skip_series); +} diff --git a/shared/src/model/config/input/tests/input.rs b/shared/src/model/config/input/tests/input.rs new file mode 100644 index 000000000..62ca7079a --- /dev/null +++ b/shared/src/model/config/input/tests/input.rs @@ -0,0 +1,266 @@ +use super::*; + +#[test] +fn sequential_group_round_trips_yaml_and_json() { + let yaml = "name: grouped\ntype: m3u\nurl: http://example.com/list.m3u\nsequential_group: 7\n"; + let dto: ConfigInputDto = serde_saphyr::from_str(yaml).expect("sequential group should deserialize from yaml"); + assert_eq!(dto.sequential_group, Some(7)); + + let encoded_yaml = serde_saphyr::to_string(&dto).expect("sequential group should serialize to yaml"); + assert!(encoded_yaml.contains("sequential_group: 7")); + + let encoded_json = serde_json::to_string(&dto).expect("sequential group should serialize to json"); + let decoded_json: ConfigInputDto = + serde_json::from_str(&encoded_json).expect("sequential group should deserialize from json"); + assert_eq!(decoded_json.sequential_group, Some(7)); +} + +#[test] +fn disable_hls_streaming_rejects_non_xtream_input() -> Result<(), String> { + let mut input = ConfigInputDto { + name: "hls-only".intern(), + input_type: InputType::M3u, + url: "http://provider.example/list.m3u".to_string(), + options: Some(ConfigInputOptionsDto { disable_hls_streaming: true, ..ConfigInputOptionsDto::default() }), + ..ConfigInputDto::default() + }; + + let Err(error) = prepare_dto(&mut input) else { + return Err("non-Xtream input accepted disable_hls_streaming".to_string()); + }; + let message = error.to_string(); + + assert!(message.contains("disable_hls_streaming"), "Error: {message}"); + assert!(message.contains("Xtream"), "Error: {message}"); + Ok(()) +} + +#[test] +fn disable_hls_streaming_allows_extensionless_live_streams() -> Result<(), String> { + let mut input = ConfigInputDto { + name: "xtream".intern(), + input_type: InputType::Xtream, + url: "http://provider.example".to_string(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + options: Some(ConfigInputOptionsDto { + disable_hls_streaming: true, + xtream_live_stream_without_extension: true, + ..ConfigInputOptionsDto::default() + }), + ..ConfigInputDto::default() + }; + + prepare_dto(&mut input).map(|_| ()).map_err(|error| error.to_string()) +} + +#[test] +fn test_config_input_options_dto_filter_prepare_parses_valid_filter() { + let mut dto = ConfigInputOptionsDto { + resolve_filter: Some(r#"name ~ "test""#.to_string()), + ..ConfigInputOptionsDto::default() + }; + dto.prepare(None).expect("valid filter should parse"); + assert!(dto.t_resolve_filter.is_some()); +} + +#[test] +fn test_config_input_options_dto_filter_prepare_rejects_invalid_filter() { + let mut dto = ConfigInputOptionsDto { + resolve_filter: Some(r#"name ~ "["#.to_string()), // invalid regex + ..ConfigInputOptionsDto::default() + }; + let result = dto.prepare(None); + assert!(result.is_err()); +} + +#[test] +fn test_config_input_options_dto_filter_prepare_with_unknown_template_placeholder() { + let mut dto = ConfigInputOptionsDto { + resolve_filter: Some(r#"name ~ "!UNKNOWN!""#.to_string()), + ..ConfigInputOptionsDto::default() + }; + let result = dto.prepare(None); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("Unknown template placeholder")); +} + +#[test] +fn test_config_input_options_dto_filter_none_prepares_successfully() { + let mut dto = ConfigInputOptionsDto { resolve_filter: None, ..ConfigInputOptionsDto::default() }; + dto.prepare(None).expect("None filter should prepare successfully"); + assert!(dto.t_resolve_filter.is_none()); +} + +#[test] +fn config_input_options_serializes_shared_skip_names() { + let dto = ConfigInputOptionsDto { + skip_live: true, + skip_vod: true, + skip_series: false, + ..ConfigInputOptionsDto::default() + }; + + let value: serde_json::Value = + serde_json::from_str(&serde_json::to_string(&dto).expect("config input options should serialize")) + .expect("serialized json should parse"); + + assert_eq!(value.get("skip_live"), Some(&serde_json::Value::Bool(true))); + assert_eq!(value.get("skip_vod"), Some(&serde_json::Value::Bool(true))); + assert!(value.get("xtream_skip_live").is_none()); + assert!(value.get("stalker_skip_vod").is_none()); +} + +#[test] +fn config_input_options_without_update_quality_remains_backward_compatible() -> Result<(), serde_json::Error> { + let options: ConfigInputOptionsDto = serde_json::from_str("{}")?; + let value = serde_json::to_value(&options)?; + + assert_eq!(options.update_quality, ConfigInputUpdateQualityDto::default()); + assert!(options.is_empty()); + assert!(value.get("update_quality").is_none()); + Ok(()) +} + +#[test] +fn config_input_options_update_quality_round_trips_and_keeps_options_non_empty() -> Result<(), serde_json::Error> { + let options: ConfigInputOptionsDto = serde_json::from_value(serde_json::json!({ + "update_quality": { + "live": 95, + "vod": 90, + "series": 100 + } + }))?; + + assert!(!options.is_empty()); + assert_eq!(options.update_quality.live, 95); + assert_eq!(options.update_quality.vod, 90); + assert_eq!(options.update_quality.series, 100); + assert_eq!( + serde_json::to_value(options)?, + serde_json::json!({ + "update_quality": { + "live": 95, + "vod": 90, + "series": 100 + } + }) + ); + Ok(()) +} + +#[test] +fn config_input_options_prepare_validates_update_quality() { + let mut options: ConfigInputOptionsDto = serde_saphyr::from_str("update_quality:\n live: 101\n") + .expect("manually edited update quality should deserialize before validation"); + + let error = options.prepare(None).expect_err("invalid update quality must be rejected"); + + assert!(error.to_string().contains("options.update_quality.live")); +} + +#[test] +fn config_input_options_clean_resets_update_quality() { + let mut options = ConfigInputOptionsDto { + update_quality: ConfigInputUpdateQualityDto { live: 95, vod: 90, series: 85 }, + ..ConfigInputOptionsDto::default() + }; + + options.clean(); + + assert_eq!(options.update_quality, ConfigInputUpdateQualityDto::default()); + assert!(options.is_empty()); +} + +#[test] +fn flussonic_archive_window_config_validates_and_cleans() -> Result<(), Box> { + use crate::model::Prepare; + let mut options: ConfigInputOptionsDto = serde_json::from_str( + r#"{"flussonic_hls_catchup":"bounded_archive","flussonic_hls_catchup_max_duration_secs":1800}"#, + )?; + options.prepare(None)?; + assert_eq!(options.flussonic_hls_catchup_max_duration_secs, 1800); + assert_eq!(serde_json::from_str::(&serde_json::to_string(&options)?)?, options); + for value in [0, 604801, u32::MAX] { + options.flussonic_hls_catchup_max_duration_secs = value; + assert!(options.prepare(None).is_err()); + } + options.clean(); + assert_eq!(options.flussonic_hls_catchup_max_duration_secs, 14400); + assert!(options.is_empty()); + assert!(serde_json::to_value(options)?.get("flussonic_hls_catchup_max_duration_secs").is_none()); + Ok(()) +} + +#[test] +fn flussonic_hls_catchup_round_trips_and_cleans() -> Result<(), serde_json::Error> { + let defaults: ConfigInputOptionsDto = serde_json::from_str("{}")?; + assert_eq!(defaults.flussonic_hls_catchup, super::super::FlussonicHlsCatchup::Native); + assert!(serde_json::to_value(defaults)?.get("flussonic_hls_catchup").is_none()); + let mut options: ConfigInputOptionsDto = serde_json::from_str(r#"{"flussonic_hls_catchup":"bounded_archive"}"#)?; + assert_eq!(options.flussonic_hls_catchup, super::super::FlussonicHlsCatchup::BoundedArchive); + assert!(!options.is_empty()); + assert_eq!(serde_json::to_value(&options)?, serde_json::json!({"flussonic_hls_catchup": "bounded_archive"})); + assert_eq!(serde_json::from_str::(&serde_json::to_string(&options)?)?, options); + assert!(serde_json::from_str::(r#"{"flussonic_hls_catchup":"unknown"}"#).is_err()); + options.clean(); + assert_eq!(options.flussonic_hls_catchup, super::super::FlussonicHlsCatchup::Native); + assert!(options.is_empty()); + Ok(()) +} + +#[test] +fn disable_hls_streaming_defaults_to_false() -> Result<(), serde_json::Error> { + let options: ConfigInputOptionsDto = serde_json::from_str("{}")?; + assert!(!options.disable_hls_streaming); + Ok(()) +} + +#[test] +fn disable_hls_streaming_keeps_options_non_empty_and_clean_resets_it() { + let mut options = ConfigInputOptionsDto { disable_hls_streaming: true, ..ConfigInputOptionsDto::default() }; + + assert!(!options.is_empty()); + options.clean(); + assert!(!options.disable_hls_streaming); + assert!(options.is_empty()); +} + +#[test] +fn disable_hls_streaming_round_trips() -> Result<(), serde_json::Error> { + let options = ConfigInputOptionsDto { disable_hls_streaming: true, ..ConfigInputOptionsDto::default() }; + let json = serde_json::to_string(&options)?; + let restored: ConfigInputOptionsDto = serde_json::from_str(&json)?; + + assert!(restored.disable_hls_streaming); + Ok(()) +} + +#[test] +fn flussonic_hls_audio_tracks_round_trips_defaults_off_and_cleans() -> Result<(), serde_json::Error> { + assert!(!ConfigInputOptionsDto::default().flussonic_hls_audio_tracks); + let mut options = ConfigInputOptionsDto { flussonic_hls_audio_tracks: true, ..ConfigInputOptionsDto::default() }; + let json = serde_json::to_string(&options)?; + let restored: ConfigInputOptionsDto = serde_json::from_str(&json)?; + + assert!(restored.flussonic_hls_audio_tracks); + assert!(!options.is_empty()); + options.clean(); + assert!(!options.flussonic_hls_audio_tracks); + assert!(options.is_empty()); + Ok(()) +} + +#[test] +fn user_agent_stream_index_round_trips_and_keeps_options_non_empty() -> Result<(), serde_json::Error> { + let mut options = ConfigInputOptionsDto { user_agent_stream_index: true, ..ConfigInputOptionsDto::default() }; + let json = serde_json::to_string(&options)?; + let restored: ConfigInputOptionsDto = serde_json::from_str(&json)?; + + assert!(restored.user_agent_stream_index); + assert!(!options.is_empty()); + options.clean(); + assert!(!options.user_agent_stream_index); + assert!(options.is_empty()); + Ok(()) +} diff --git a/shared/src/model/config/input/tests/provider.rs b/shared/src/model/config/input/tests/provider.rs new file mode 100644 index 000000000..454f9562d --- /dev/null +++ b/shared/src/model/config/input/tests/provider.rs @@ -0,0 +1,96 @@ +use super::*; + +#[test] +fn test_provider_dns_defaults() { + let dns = ProviderDnsDto::default(); + assert!(!dns.enabled); + assert_eq!(dns.refresh_secs, 300); + assert_eq!(dns.prefer, DnsPrefer::System); + assert_eq!(dns.on_resolve_error, OnResolveErrorPolicy::KeepLastGood); + assert_eq!(dns.on_connect_error, OnConnectErrorPolicy::TryNextIp); + assert!(dns.schemes.is_none()); +} + +#[test] +fn test_provider_url_selection_policy_defaults_to_resume_last_working() { + let provider = ConfigProviderDto { + name: "provider-a".intern(), + urls: vec!["http://primary.example.com".intern()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), + dns: None, + }; + + assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::ResumeLastWorking); +} + +#[test] +fn test_provider_url_selection_policy_can_be_set_to_restart_from_first() { + let provider = ConfigProviderDto { + name: "provider-a".intern(), + urls: vec!["http://primary.example.com".intern()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + }; + + assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::RestartFromFirst); +} + +#[test] +fn test_provider_url_selection_policy_deserializes_default_when_omitted() { + let provider: ConfigProviderDto = + serde_json::from_str(r#"{"name":"provider-a","urls":["http://primary.example.com"]}"#) + .expect("provider dto should deserialize"); + + assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::ResumeLastWorking); +} + +#[test] +fn test_provider_url_selection_policy_deserializes_restart_from_first() { + let provider: ConfigProviderDto = serde_json::from_str( + r#"{"name":"provider-a","urls":["http://primary.example.com"],"provider_url_selection_policy":"restart_from_first"}"#, + ) + .expect("provider dto should deserialize"); + + assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::RestartFromFirst); +} + +#[test] +fn test_provider_url_selection_policy_default_is_omitted_on_serialize() { + let provider = ConfigProviderDto { + name: "provider-a".intern(), + urls: vec!["http://primary.example.com".intern()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + }; + + let json = serde_json::to_string(&provider).expect("provider dto should serialize"); + let value: serde_json::Value = serde_json::from_str(&json).expect("serialized provider should be valid json"); + + assert!(value.get("provider_url_selection_policy").is_none()); +} + +#[test] +fn test_provider_dns_prepare_normalizes_overrides_and_preserves_low_refresh() { + let mut dns = ProviderDnsDto { + refresh_secs: 1, + schemes: Some(vec![DnsScheme::Http, DnsScheme::Http, DnsScheme::Https]), + overrides: Some(HashMap::from([( + " EXAMPLE.COM ".to_string(), + vec![ + "203.0.113.10".parse::().expect("valid ip"), + "203.0.113.10".parse::().expect("valid ip"), + ], + )])), + ..ProviderDnsDto::default() + }; + + dns.prepare().expect("dns prepare should succeed"); + + // Explicit low values are honored (only warned about), not clamped. + assert_eq!(dns.refresh_secs, 1); + assert_eq!(dns.schemes, Some(vec![DnsScheme::Http, DnsScheme::Https])); + let overrides = dns.overrides.expect("overrides should exist"); + assert_eq!(overrides.len(), 1); + assert!(overrides.contains_key("example.com")); + assert_eq!(overrides["example.com"].len(), 1); +} diff --git a/shared/src/model/config/input/tests/staged_media_server.rs b/shared/src/model/config/input/tests/staged_media_server.rs new file mode 100644 index 000000000..288b97c83 --- /dev/null +++ b/shared/src/model/config/input/tests/staged_media_server.rs @@ -0,0 +1,139 @@ +use super::*; + +#[test] +fn sequential_group_rejects_zero_and_staged_inputs() { + let mut zero = ConfigInputDto { + name: "zero_group".intern(), + url: "http://example.com/list.m3u".to_string(), + sequential_group: Some(0), + ..ConfigInputDto::default() + }; + let err = prepare_dto(&mut zero).expect_err("group zero must be rejected"); + assert!(err.to_string().contains("sequential_group"), "Error: {err}"); + + let mut staged = ConfigInputDto { + name: "staged_group".intern(), + input_type: InputType::Staged, + url: "http://example.com/list.m3u".to_string(), + staged: Some(ConfigInputStagedDto { + for_input: Some("provider_a".intern()), + ..ConfigInputStagedDto::default() + }), + sequential_group: Some(1), + ..ConfigInputDto::default() + }; + let err = prepare_dto(&mut staged).expect_err("staged groups must be rejected"); + assert!(err.to_string().contains("sequential_group"), "Error: {err}"); +} + +#[test] +fn prepare_type_does_not_validate_media_server_config() { + let mut dto = ConfigInputDto { + name: "emby_media_server".intern(), + input_type: InputType::Emby, + url: "https://media.example.invalid".to_string(), + ..ConfigInputDto::default() + }; + + dto.prepare_type().expect("prepare_type only normalizes type/url"); + let err = prepare_dto(&mut dto).expect_err("full prepare should validate missing media_server block"); + assert!(err.to_string().contains("media_server configuration is mandatory")); +} + +#[test] +fn test_staged_requires_provider() { + let mut dto = create_test_dto(); + dto.input_type = InputType::Staged; + dto.url = "http://staged.com/playlist.m3u".to_string(); + + let err = prepare_dto(&mut dto).expect_err("staged input without provider must be rejected"); + assert!(err.to_string().contains("requires a provider input"), "Error: {err}"); +} + +#[test] +fn test_staged_requires_url() { + let mut dto = create_test_dto(); + dto.input_type = InputType::Staged; + dto.staged = + Some(ConfigInputStagedDto { for_input: Some("provider_a".intern()), ..ConfigInputStagedDto::default() }); + dto.url = String::new(); + + let err = prepare_dto(&mut dto).expect_err("staged input without url must be rejected"); + assert!(err.to_string().contains("url for staged input is mandatory"), "Error: {err}"); +} + +#[test] +fn test_staged_requires_non_empty_clusters() { + let mut dto = create_test_dto(); + dto.input_type = InputType::Staged; + dto.staged = Some(ConfigInputStagedDto { for_input: Some("provider_a".intern()), clusters: ClusterFlags::empty() }); + dto.url = "http://staged.com/playlist.m3u".to_string(); + + let err = prepare_dto(&mut dto).expect_err("staged input without clusters must be rejected"); + assert!(err.to_string().contains("requires at least one staged cluster"), "Error: {err}"); +} + +#[test] +fn test_staged_config_only_allowed_for_staged() { + let mut dto = create_test_dto(); + dto.input_type = InputType::M3u; + dto.url = "http://main.com/playlist.m3u".to_string(); + dto.staged = + Some(ConfigInputStagedDto { for_input: Some("provider_a".intern()), ..ConfigInputStagedDto::default() }); + + let err = prepare_dto(&mut dto).expect_err("non-staged input with staged config must be rejected"); + assert!(err.to_string().contains("staged configuration is only allowed for staged inputs"), "Error: {err}"); +} + +#[test] +fn test_staged_rejects_media_server() { + let mut dto = create_test_dto(); + dto.input_type = InputType::Staged; + dto.staged = + Some(ConfigInputStagedDto { for_input: Some("provider_a".intern()), ..ConfigInputStagedDto::default() }); + dto.url = "http://staged.com/playlist.m3u".to_string(); + dto.media_server = Some(MediaServerInputConfigDto::default()); + + let err = prepare_dto(&mut dto).expect_err("staged input with media_server must be rejected"); + assert!(err.to_string().contains("does not support media_server configuration"), "Error: {err}"); +} + +#[test] +fn test_staged_valid() { + let mut dto = create_test_dto(); + dto.input_type = InputType::Staged; + dto.staged = + Some(ConfigInputStagedDto { for_input: Some(" provider_a ".intern()), ..ConfigInputStagedDto::default() }); + dto.url = "http://staged.com/playlist.m3u".to_string(); + + prepare_dto(&mut dto).expect("valid staged input should prepare successfully"); + assert!(dto.input_type.is_staged()); + assert_eq!(dto.staged.as_ref().and_then(|staged| staged.for_input.as_deref()), Some("provider_a")); +} + +#[test] +fn test_staged_provider_alias_deserializes_as_for_input() { + let staged: ConfigInputStagedDto = + serde_json::from_str(r#"{"provider":"provider_a"}"#).expect("legacy provider field should deserialize"); + + assert_eq!(staged.for_input.as_deref(), Some("provider_a")); +} + +#[test] +fn test_staged_ignores_stream_runtime_fields() { + let mut dto = create_test_dto(); + dto.input_type = InputType::Staged; + dto.url = "http://staged.com/playlist.m3u".to_string(); + dto.staged = + Some(ConfigInputStagedDto { for_input: Some("provider_a".intern()), ..ConfigInputStagedDto::default() }); + dto.priority = -10; + dto.max_connections = 2; + dto.cache_duration = Some("not-a-duration".to_string()); + + prepare_dto(&mut dto).expect("staged stream runtime fields should be ignored"); + + assert_eq!(dto.priority, 0); + assert_eq!(dto.max_connections, 0); + assert_eq!(dto.cache_duration, None); + assert_eq!(dto.cache_duration_seconds, 0); +} diff --git a/shared/src/model/config/input/tests/stalker.rs b/shared/src/model/config/input/tests/stalker.rs new file mode 100644 index 000000000..ce0fc8289 --- /dev/null +++ b/shared/src/model/config/input/tests/stalker.rs @@ -0,0 +1,125 @@ +use super::*; + +#[test] +fn stalker_options_reject_retired_playback_resolution_switches() -> Result<(), serde_json::Error> { + let dto: ConfigInputOptionsDto = serde_json::from_str(r#"{"stalker_bulk_epg":true}"#)?; + assert!(dto.stalker_bulk_epg); + assert!(serde_json::from_str::(r#"{"stalker_pre_resolve_playback":true}"#).is_err()); + assert!(serde_json::from_str::(r#"{"stalker_runtime_resolve_playback":false}"#).is_err()); + Ok(()) +} + +#[test] +fn new_stalker_input_has_configuration_block() { + let input = ConfigInputDto::new_with_type(InputType::Stalker); + + assert!(input.stalker.is_some()); + assert!(ConfigInputDto::new_with_type(InputType::M3u).stalker.is_none()); +} + +#[test] +fn stalker_mac_is_normalized_and_multicast_is_rejected() { + let mut input = ConfigInputDto::new_with_type(InputType::Stalker); + input.name = "portal".into(); + input.stalker.as_mut().expect("stalker config").device = + Some(StalkerDeviceProfileDto { mac_address: Some("00:1A:79:AA:BB:CC".to_string()), ..Default::default() }); + input.prepare_stalker_input().expect("unicast MAC should be valid"); + assert_eq!( + input + .stalker + .as_ref() + .and_then(|config| config.device.as_ref()) + .and_then(|device| device.mac_address.as_deref()), + Some("00:1a:79:aa:bb:cc") + ); + + input.stalker.as_mut().expect("stalker config").device.as_mut().expect("device").mac_address = + Some("01:00:00:00:00:00".to_string()); + assert!(input.prepare_stalker_input().is_err()); +} + +#[test] +fn stalker_main_input_requires_identity_but_batch_aliases_may_inherit() { + let mut input = ConfigInputDto::new_with_type(InputType::Stalker); + input.name = "portal".into(); + assert!(input.prepare_stalker_input().is_err()); + + let mut alias = + ConfigInputAliasDto { name: "alias".into(), url: "http://portal.example/c/".to_string(), ..Default::default() }; + assert!(alias.prepare(0, &InputType::Stalker).is_ok()); +} + +#[test] +fn stalker_auth_mode_requires_matching_identity() { + for (auth_mode, has_mac, has_credentials, valid) in [ + (StalkerAuthMode::Auto, true, false, true), + (StalkerAuthMode::Auto, false, true, true), + (StalkerAuthMode::MacOnly, true, false, true), + (StalkerAuthMode::MacOnly, false, true, false), + (StalkerAuthMode::CredentialsOnly, false, true, true), + (StalkerAuthMode::CredentialsOnly, true, false, false), + (StalkerAuthMode::MacPlusCredentials, true, true, true), + (StalkerAuthMode::MacPlusCredentials, true, false, false), + ] { + let mut input = ConfigInputDto::new_with_type(InputType::Stalker); + input.name = "portal".into(); + input.username = has_credentials.then(|| "user".to_string()); + input.password = has_credentials.then(|| "password".to_string()); + input.stalker = Some(StalkerInputConfigDto { + auth_mode, + device: has_mac.then(|| StalkerDeviceProfileDto { + mac_address: Some("00:1a:79:aa:bb:cc".to_string()), + ..Default::default() + }), + ..Default::default() + }); + + assert_eq!(input.prepare_stalker_input().is_ok(), valid, "auth mode: {auth_mode}"); + } +} + +#[test] +fn non_stalker_input_rejects_stalker_configuration() { + let mut input = + ConfigInputDto { name: "m3u".into(), stalker: Some(StalkerInputConfigDto::default()), ..Default::default() }; + assert!(input.prepare_stalker_input().is_err()); +} + +#[test] +fn stalker_config_round_trips_extended_device_and_size_caps() { + let mut input = ConfigInputDto::new_with_type(InputType::Stalker); + input.name = "portal".into(); + input.stalker = Some(StalkerInputConfigDto { + device: Some(StalkerDeviceProfileDto { + mac_address: Some("00:1a:79:aa:bb:cc".to_string()), + device_profile: Some("MAG254".to_string()), + serial_number: Some("serial-1".to_string()), + device_id: Some("device-1".to_string()), + device_id2: Some("device-2".to_string()), + signature: Some("sig-1".to_string()), + ..Default::default() + }), + size_caps: Some(crate::model::stalker::StalkerActionSizeCapDto { + create_link_kb: 96, + ordered_list_mb: 12, + get_epg_mb: 80, + }), + catalog_max_pages: Some(2048), + ..Default::default() + }); + + let value = serde_json::to_value(&input).expect("serialize stalker input"); + let decoded: ConfigInputDto = serde_json::from_value(value).expect("deserialize stalker input"); + let stalker = decoded.stalker.expect("stalker config"); + let device = stalker.device.expect("device"); + assert_eq!(device.device_profile.as_deref(), Some("MAG254")); + assert_eq!(device.serial_number.as_deref(), Some("serial-1")); + assert_eq!(device.device_id.as_deref(), Some("device-1")); + assert_eq!(device.device_id2.as_deref(), Some("device-2")); + assert_eq!(device.signature.as_deref(), Some("sig-1")); + let caps = stalker.size_caps.expect("size caps"); + assert_eq!(caps.create_link_kb, 96); + assert_eq!(caps.ordered_list_mb, 12); + assert_eq!(caps.get_epg_mb, 80); + assert_eq!(stalker.catalog_max_pages, Some(2048)); +} diff --git a/shared/src/model/config/media_server_catalog.rs b/shared/src/model/config/media_server_catalog.rs index e67badaa8..d3ff5a419 100644 --- a/shared/src/model/config/media_server_catalog.rs +++ b/shared/src/model/config/media_server_catalog.rs @@ -280,645 +280,4 @@ impl MediaServerInputConfigDto { } #[cfg(test)] -mod tests { - use super::*; - use crate::{ - model::{ - ConfigInputAliasDto, ConfigInputDto, ConfigInputOptionsDto, ConfigProviderDto, DnsPrefer, DnsScheme, - InputType, OnConnectErrorPolicy, OnResolveErrorPolicy, Prepare, ProviderDnsDto, ProviderUrlSelectionPolicy, - }, - utils::Internable, - }; - use std::{ - collections::{HashMap, HashSet}, - net::IpAddr, - }; - - fn prepare_dto(dto: &mut ConfigInputDto) -> Result { - dto.prepare(0, false, &HashSet::new(), None) - } - - fn media_server_config_with_library() -> MediaServerInputConfigDto { - MediaServerInputConfigDto { - libraries: vec![MediaServerLibrarySelector::Name("Movies".to_string())], - ..MediaServerInputConfigDto::default() - } - } - - #[test] - fn prepare_rejects_blank_media_server_credentials_and_selectors() { - let mut emby = ConfigInputDto { - name: "emby_media_server".intern(), - input_type: InputType::Emby, - url: "https://media.example.invalid".to_string(), - media_server: Some(MediaServerInputConfigDto { - token: Some(" ".to_string()), - api_key: Some(String::new()), - ..media_server_config_with_library() - }), - ..ConfigInputDto::default() - }; - let err = prepare_dto(&mut emby).expect_err("blank token/api_key should be rejected"); - assert!(err.to_string().contains("requires media_server token/api_key")); - - let mut plex = ConfigInputDto { - name: "plex_media_server".intern(), - input_type: InputType::Plex, - media_server: Some(MediaServerInputConfigDto { - account_token: Some(" ".to_string()), - server_id: Some(" ".to_string()), - ..media_server_config_with_library() - }), - ..ConfigInputDto::default() - }; - let err = prepare_dto(&mut plex).expect_err("blank plex token should be rejected"); - assert!(err.to_string().contains("requires media_server.account_token")); - } - - #[test] - fn prepare_accepts_media_server_max_connections_as_stream_limit() { - let mut dto = ConfigInputDto { - name: "emby_media_server".intern(), - input_type: InputType::Emby, - url: "https://media.example.invalid".to_string(), - max_connections: 1, - media_server: Some(MediaServerInputConfigDto { - token: Some("token-value".to_string()), - ..media_server_config_with_library() - }), - ..ConfigInputDto::default() - }; - - prepare_dto(&mut dto).expect("media_server inputs reuse max_connections stream-limit semantics"); - } - - #[test] - fn media_server_defaults_are_conservative() { - let media_server = MediaServerInputConfigDto::default(); - - assert_eq!(media_server.catalog.page_size, 100); - assert_eq!(media_server.catalog.request_delay_ms, 250); - assert!(media_server.catalog.include_media_sources); - assert!(!media_server.catalog.include_paths); - assert!(!media_server.catalog.include_user_state); - assert!(!media_server.catalog.refresh_on_startup); - assert!(media_server.playback.direct_play_only); - assert!(!media_server.playback.allow_transcode); - assert!(!media_server.playback.preflight_streams); - assert_eq!(media_server.image_policy, MediaServerImagePolicy::ProxyOnDemand); - assert!(!media_server.allow_relay); - } - - #[test] - fn media_server_library_key_selector_accepts_numeric_yaml_scalars() { - let media_server = - serde_json::from_str::(r#"{"libraries":[{"key":10,"kind":"movies"}]}"#) - .expect("numeric YAML-like key selectors should deserialize as strings"); - - assert_eq!( - media_server.libraries, - vec![MediaServerLibrarySelector::Detailed(MediaServerLibrarySelectorDetailsDto { - key: Some("10".to_string()), - kind: Some(MediaServerLibraryKind::Movies), - ..MediaServerLibrarySelectorDetailsDto::default() - })] - ); - - let by_id = serde_json::from_str::(r#"{"libraries":[{"id":42,"kind":"tvshows"}]}"#) - .expect("numeric id selectors should deserialize as strings"); - assert_eq!( - by_id.libraries, - vec![MediaServerLibrarySelector::Detailed(MediaServerLibrarySelectorDetailsDto { - id: Some("42".to_string()), - kind: Some(MediaServerLibraryKind::TvShows), - ..MediaServerLibrarySelectorDetailsDto::default() - })] - ); - } - - #[test] - fn media_server_enrichment_block_is_not_part_of_schema() { - let err = serde_json::from_str::( - r#"{"libraries":["Movies"],"enrichment":{"ffprobe":true,"tmdb_lookup":true,"fetch_images":true}}"#, - ) - .expect_err("media_server.enrichment must not be accepted"); - - assert!(err.to_string().contains("unknown field `enrichment`")); - } - - #[test] - fn prepare_accepts_emby_media_server_with_token_and_library() { - let mut dto = ConfigInputDto { - name: "emby_media_server".intern(), - input_type: InputType::Emby, - url: " https://media.example.invalid/ ".to_string(), - media_server: Some(MediaServerInputConfigDto { - token: Some(" token-value ".to_string()), - ..media_server_config_with_library() - }), - ..ConfigInputDto::default() - }; - - prepare_dto(&mut dto).expect("emby media_server config should prepare"); - - assert_eq!(dto.url, "https://media.example.invalid/"); - assert!(dto.input_type.is_media_server()); - assert_eq!( - dto.media_server.as_ref().and_then(|media_server| media_server.token.as_deref()), - Some("token-value") - ); - } - - #[test] - fn prepare_rejects_media_server_without_media_server_block() { - let mut dto = ConfigInputDto { - name: "emby_media_server".intern(), - input_type: InputType::Emby, - url: "https://media.example.invalid".to_string(), - ..ConfigInputDto::default() - }; - - let err = prepare_dto(&mut dto).expect_err("media_server block should be mandatory"); - assert!(err.to_string().contains("media_server configuration is mandatory")); - } - - #[test] - fn prepare_allows_disabled_media_server_input_with_incomplete_config() { - let mut dto = ConfigInputDto { - name: "disabled_plex".intern(), - input_type: InputType::Plex, - enabled: false, - ..ConfigInputDto::default() - }; - - prepare_dto(&mut dto).expect("disabled media_server input should not require active playback/catalog config"); - assert_eq!(dto.input_type, InputType::Plex); - assert!(!dto.enabled); - } - - #[test] - fn prepare_normalizes_disabled_media_server_config_without_enforcing_invariants() { - let mut dto = ConfigInputDto { - name: "disabled_emby".intern(), - input_type: InputType::Emby, - enabled: false, - media_server: Some(MediaServerInputConfigDto { - token: Some(" token-value ".to_string()), - libraries: vec![MediaServerLibrarySelector::Name(" ".to_string())], - catalog: MediaServerCatalogConfigDto { page_size: 0, ..MediaServerCatalogConfigDto::default() }, - ..MediaServerInputConfigDto::default() - }), - ..ConfigInputDto::default() - }; - - prepare_dto(&mut dto).expect("disabled media_server input can preserve incomplete config for later repair"); - let media_server = dto.media_server.as_ref().expect("media_server config should be preserved"); - assert_eq!(media_server.token.as_deref(), Some("token-value")); - assert!(media_server.libraries[0].is_empty()); - } - - #[test] - fn prepare_rejects_emby_media_server_without_input_url() { - let mut dto = ConfigInputDto { - name: "emby_media_server".intern(), - input_type: InputType::Emby, - media_server: Some(MediaServerInputConfigDto { - token: Some("token-value".to_string()), - ..media_server_config_with_library() - }), - ..ConfigInputDto::default() - }; - - let err = prepare_dto(&mut dto).expect_err("emby media_server input should require a direct server URL"); - assert!(err.to_string().contains("url is mandatory for input type emby")); - } - - #[test] - fn prepare_rejects_media_server_provider_scheme_url() { - let mut dto = ConfigInputDto { - name: "emby_media_server".intern(), - input_type: InputType::Emby, - url: " provider://media-server ".to_string(), - media_server: Some(MediaServerInputConfigDto { - token: Some("token-value".to_string()), - ..media_server_config_with_library() - }), - ..ConfigInputDto::default() - }; - - let err = prepare_dto(&mut dto).expect_err("media_server input must not use provider URLs"); - assert!(err.to_string().contains("does not support batch:// or provider://")); - } - - #[test] - fn prepare_rejects_plex_without_token_or_server_selector() { - let mut without_token = ConfigInputDto { - name: "plex_media_server".intern(), - input_type: InputType::Plex, - media_server: Some(MediaServerInputConfigDto { - server_id: Some("server".to_string()), - ..media_server_config_with_library() - }), - ..ConfigInputDto::default() - }; - let err = prepare_dto(&mut without_token).expect_err("plex token should be mandatory"); - assert!(err.to_string().contains("requires media_server.account_token")); - - let mut without_selector = ConfigInputDto { - name: "plex_media_server".intern(), - input_type: InputType::Plex, - media_server: Some(MediaServerInputConfigDto { - account_token: Some("token".to_string()), - ..media_server_config_with_library() - }), - ..ConfigInputDto::default() - }; - let err = prepare_dto(&mut without_selector).expect_err("plex server selector should be mandatory"); - assert!(err.to_string().contains("requires a server selector")); - } - - #[test] - fn prepare_accepts_plex_without_input_url_when_discovery_is_configured() { - let mut dto = ConfigInputDto { - name: "plex_media_server".intern(), - input_type: InputType::Plex, - media_server: Some(MediaServerInputConfigDto { - account_token: Some("token".to_string()), - server_id: Some("server".to_string()), - ..media_server_config_with_library() - }), - ..ConfigInputDto::default() - }; - - prepare_dto(&mut dto).expect("plex discovery config should not require input.url"); - assert_eq!(dto.input_type, InputType::Plex); - } - - #[test] - fn prepare_accepts_plex_media_server_with_direct_url_without_selector() { - let mut dto = ConfigInputDto { - name: "plex_media_server".intern(), - input_type: InputType::Plex, - url: "https://plex.example.invalid".to_string(), - media_server: Some(MediaServerInputConfigDto { - token: Some("token".to_string()), - ..media_server_config_with_library() - }), - ..ConfigInputDto::default() - }; - - prepare_dto(&mut dto).expect("direct Plex URL should not require MyPlex server selector"); - assert_eq!(dto.input_type, InputType::Plex); - } - - #[test] - fn prepare_rejects_plex_direct_url_without_server_token() { - let mut dto = ConfigInputDto { - name: "plex_media_server".intern(), - input_type: InputType::Plex, - url: "https://plex.example.invalid".to_string(), - media_server: Some(MediaServerInputConfigDto { - account_token: Some("account-token".to_string()), - ..media_server_config_with_library() - }), - ..ConfigInputDto::default() - }; - - let err = prepare_dto(&mut dto).expect_err("direct Plex URL requires the PMS server token"); - assert!(err.to_string().contains("requires media_server.token")); - } - - #[test] - fn prepare_rejects_plex_discovery_without_account_token() { - let mut dto = ConfigInputDto { - name: "plex_media_server".intern(), - input_type: InputType::Plex, - media_server: Some(MediaServerInputConfigDto { - token: Some("server-token".to_string()), - server_id: Some("server".to_string()), - ..media_server_config_with_library() - }), - ..ConfigInputDto::default() - }; - - let err = prepare_dto(&mut dto).expect_err("Plex discovery requires the MyPlex account token"); - assert!(err.to_string().contains("requires media_server.account_token")); - } - - #[test] - fn test_provider_dns_defaults() { - let dns = ProviderDnsDto::default(); - assert!(!dns.enabled); - assert_eq!(dns.refresh_secs, 300); - assert_eq!(dns.prefer, DnsPrefer::System); - assert_eq!(dns.on_resolve_error, OnResolveErrorPolicy::KeepLastGood); - assert_eq!(dns.on_connect_error, OnConnectErrorPolicy::TryNextIp); - assert!(dns.schemes.is_none()); - } - - #[test] - fn test_provider_url_selection_policy_defaults_to_resume_last_working() { - let provider = ConfigProviderDto { - name: "provider-a".intern(), - urls: vec!["http://primary.example.com".intern()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), - dns: None, - }; - - assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::ResumeLastWorking); - } - - #[test] - fn test_provider_url_selection_policy_can_be_set_to_restart_from_first() { - let provider = ConfigProviderDto { - name: "provider-a".intern(), - urls: vec!["http://primary.example.com".intern()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, - dns: None, - }; - - assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::RestartFromFirst); - } - - #[test] - fn test_provider_url_selection_policy_deserializes_default_when_omitted() { - let provider: ConfigProviderDto = - serde_json::from_str(r#"{"name":"provider-a","urls":["http://primary.example.com"]}"#) - .expect("provider dto should deserialize"); - - assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::ResumeLastWorking); - } - - #[test] - fn test_provider_url_selection_policy_deserializes_restart_from_first() { - let provider: ConfigProviderDto = serde_json::from_str( - r#"{"name":"provider-a","urls":["http://primary.example.com"],"provider_url_selection_policy":"restart_from_first"}"#, - ) - .expect("provider dto should deserialize"); - - assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::RestartFromFirst); - } - - #[test] - fn test_provider_url_selection_policy_default_is_omitted_on_serialize() { - let provider = ConfigProviderDto { - name: "provider-a".intern(), - urls: vec!["http://primary.example.com".intern()], - provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, - dns: None, - }; - - let json = serde_json::to_string(&provider).expect("provider dto should serialize"); - let value: serde_json::Value = serde_json::from_str(&json).expect("serialized provider should be valid json"); - - assert!(value.get("provider_url_selection_policy").is_none()); - } - - #[test] - fn test_provider_dns_prepare_normalizes_overrides_and_clamps_refresh() { - let mut dns = ProviderDnsDto { - refresh_secs: 1, - schemes: Some(vec![DnsScheme::Http, DnsScheme::Http, DnsScheme::Https]), - overrides: Some(HashMap::from([( - " EXAMPLE.COM ".to_string(), - vec![ - "203.0.113.10".parse::().expect("valid ip"), - "203.0.113.10".parse::().expect("valid ip"), - ], - )])), - ..ProviderDnsDto::default() - }; - - dns.prepare().expect("dns prepare should succeed"); - - assert_eq!(dns.refresh_secs, 1); - assert_eq!(dns.schemes, Some(vec![DnsScheme::Http, DnsScheme::Https])); - let overrides = dns.overrides.expect("overrides should exist"); - assert_eq!(overrides.len(), 1); - assert!(overrides.contains_key("example.com")); - assert_eq!(overrides["example.com"].len(), 1); - } - - #[test] - fn prepare_switches_xtream_to_xtream_batch_when_alias_exists() { - let mut dto = ConfigInputDto { - name: "input_alias".intern(), - input_type: InputType::Xtream, - url: "batch:///tmp/input_alias.csv".to_string(), - aliases: Some(vec![ConfigInputAliasDto { - id: 1, - name: "alias_1".intern(), - url: "http://provider.example/stream".to_string(), - username: Some("u".to_string()), - password: Some("p".to_string()), - enabled: true, - ..ConfigInputAliasDto::default() - }]), - ..ConfigInputDto::default() - }; - - dto.prepare_type().expect("prepare type should succeed"); - dto.prepare(0, true, &HashSet::new(), None) - .expect("prepare should succeed and infer batch type from batch:// URL"); - assert_eq!(dto.input_type, InputType::XtreamBatch); - } - - #[test] - fn prepare_keeps_xtream_type_when_alias_exists_without_batch_url() { - let mut dto = ConfigInputDto { - name: "input_alias_http".intern(), - input_type: InputType::XtreamBatch, - url: "http://localhost:3001".to_string(), - username: Some("root_user".to_string()), - password: Some("root_pass".to_string()), - aliases: Some(vec![ConfigInputAliasDto { - id: 1, - name: "alias_1".intern(), - url: "http://provider.example/stream".to_string(), - username: Some("u".to_string()), - password: Some("p".to_string()), - enabled: true, - ..ConfigInputAliasDto::default() - }]), - ..ConfigInputDto::default() - }; - - dto.prepare_type().expect("prepare type should normalize non-batch URL to xtream"); - assert_eq!(dto.input_type, InputType::Xtream); - dto.prepare(0, true, &HashSet::new(), None).expect("prepare should succeed for regular URL with aliases"); - assert_eq!(dto.input_type, InputType::Xtream); - } - - #[test] - fn prepare_type_does_not_validate_media_server_config() { - let mut dto = ConfigInputDto { - name: "emby_media_server".intern(), - input_type: InputType::Emby, - url: "https://media.example.invalid".to_string(), - ..ConfigInputDto::default() - }; - - dto.prepare_type().expect("prepare_type only normalizes type/url"); - let err = prepare_dto(&mut dto).expect_err("full prepare should validate missing media_server block"); - assert!(err.to_string().contains("media_server configuration is mandatory")); - } - - #[test] - fn prepare_batch_url_does_not_require_xtream_credentials() { - let mut dto = ConfigInputDto { - name: "batch_no_creds".intern(), - input_type: InputType::Xtream, - url: "batch:///tmp/no-creds.csv".to_string(), - username: None, - password: None, - ..ConfigInputDto::default() - }; - - dto.prepare(0, true, &HashSet::new(), None) - .expect("batch:// input must be normalized before credential validation"); - assert_eq!(dto.input_type, InputType::XtreamBatch); - } - - #[test] - fn prepare_provider_scheme_url_is_not_treated_as_batch_input() { - let mut dto = ConfigInputDto { - name: "batch_provider".intern(), - input_type: InputType::XtreamBatch, - url: "provider://myprovider".to_string(), - username: Some("root_user".to_string()), - password: Some("root_pass".to_string()), - aliases: Some(vec![ConfigInputAliasDto { - id: 1, - name: "alias_1".intern(), - url: "http://provider.example/stream".to_string(), - username: Some("u".to_string()), - password: Some("p".to_string()), - enabled: true, - ..ConfigInputAliasDto::default() - }]), - ..ConfigInputDto::default() - }; - - let err = dto - .prepare(0, true, &HashSet::new(), None) - .expect_err("prepare should treat provider:// URL as regular input (non-batch) and validate provider"); - assert!(err.to_string().contains("Provider name myprovider is not defined"), "Error: {err}"); - } - - #[test] - fn prepare_rejects_missing_input_url_even_with_aliases() { - let mut dto = ConfigInputDto { - name: "xtream_missing_root_url".intern(), - input_type: InputType::Xtream, - url: String::new(), - username: Some("root_user".to_string()), - password: Some("root_pass".to_string()), - aliases: Some(vec![ConfigInputAliasDto { - id: 1, - name: "alias_1".intern(), - url: "http://alias.example".to_string(), - username: Some("alias_user".to_string()), - password: Some("alias_pass".to_string()), - enabled: true, - ..ConfigInputAliasDto::default() - }]), - ..ConfigInputDto::default() - }; - - let err = dto - .prepare(0, true, &HashSet::new(), None) - .expect_err("prepare must require root input url even when aliases are present"); - assert!(err.to_string().contains("url for input is mandatory"), "Error: {err}"); - assert!(err.to_string().contains("xtream_missing_root_url"), "Error: {err}"); - } - - #[test] - fn prepare_rejects_missing_root_credentials_for_non_batch_url_even_with_aliases() { - let mut dto = ConfigInputDto { - name: "xtream_batch_missing_root_creds".intern(), - input_type: InputType::XtreamBatch, - url: "http://root.example".to_string(), - aliases: Some(vec![ConfigInputAliasDto { - id: 1, - name: "alias_1".intern(), - url: "http://alias.example".to_string(), - username: Some("alias_user".to_string()), - password: Some("alias_pass".to_string()), - enabled: true, - ..ConfigInputAliasDto::default() - }]), - ..ConfigInputDto::default() - }; - - let err = dto - .prepare(0, true, &HashSet::new(), None) - .expect_err("prepare must require root credentials for non-batch URL"); - assert!(err.to_string().contains("for input type xtream: username and password are mandatory"), "Error: {err}"); - assert!(err.to_string().contains("xtream_batch_missing_root_creds"), "Error: {err}"); - } - - #[test] - fn prepare_rejects_xtream_batch_batch_url_with_root_credentials_even_with_aliases() { - let mut dto = ConfigInputDto { - name: "xtream_batch_with_root_creds".intern(), - input_type: InputType::XtreamBatch, - url: "batch:///tmp/aliases.csv".to_string(), - username: Some("root_user".to_string()), - password: Some("root_pass".to_string()), - aliases: Some(vec![ConfigInputAliasDto { - id: 1, - name: "alias_1".intern(), - url: "http://alias.example".to_string(), - username: Some("alias_user".to_string()), - password: Some("alias_pass".to_string()), - enabled: true, - ..ConfigInputAliasDto::default() - }]), - ..ConfigInputDto::default() - }; - - let err = dto - .prepare(0, true, &HashSet::new(), None) - .expect_err("prepare must reject root credentials when using batch:// for xtream-batch"); - assert!(err.to_string().contains("with batch:// URL should not define username or password"), "Error: {err}"); - assert!(err.to_string().contains("xtream_batch_with_root_creds"), "Error: {err}"); - } - - #[test] - fn test_config_input_options_dto_filter_prepare_parses_valid_filter() { - let mut dto = ConfigInputOptionsDto { - resolve_filter: Some(r#"name ~ "test""#.to_string()), - ..ConfigInputOptionsDto::default() - }; - dto.prepare(None).expect("valid filter should parse"); - assert!(dto.t_resolve_filter.is_some()); - } - - #[test] - fn test_config_input_options_dto_filter_prepare_rejects_invalid_filter() { - let mut dto = ConfigInputOptionsDto { - resolve_filter: Some(r#"name ~ "["#.to_string()), // invalid regex - ..ConfigInputOptionsDto::default() - }; - let result = dto.prepare(None); - assert!(result.is_err()); - } - - #[test] - fn test_config_input_options_dto_filter_prepare_with_unknown_template_placeholder() { - let mut dto = ConfigInputOptionsDto { - resolve_filter: Some(r#"name ~ "!UNKNOWN!""#.to_string()), - ..ConfigInputOptionsDto::default() - }; - let result = dto.prepare(None); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("Unknown template placeholder")); - } - - #[test] - fn test_config_input_options_dto_filter_none_prepares_successfully() { - let mut dto = ConfigInputOptionsDto { resolve_filter: None, ..ConfigInputOptionsDto::default() }; - dto.prepare(None).expect("None filter should prepare successfully"); - assert!(dto.t_resolve_filter.is_none()); - } -} +mod tests; diff --git a/shared/src/model/config/media_server_catalog/tests.rs b/shared/src/model/config/media_server_catalog/tests.rs new file mode 100644 index 000000000..6f5e64856 --- /dev/null +++ b/shared/src/model/config/media_server_catalog/tests.rs @@ -0,0 +1,24 @@ +use super::*; +use crate::{ + model::{ + ConfigInputAliasDto, ConfigInputDto, ConfigInputOptionsDto, ConfigProviderDto, DnsPrefer, DnsScheme, InputType, + OnConnectErrorPolicy, OnResolveErrorPolicy, Prepare, ProviderDnsDto, ProviderUrlSelectionPolicy, + }, + utils::Internable, +}; +use std::{ + collections::{HashMap, HashSet}, + net::IpAddr, +}; + +fn prepare_dto(dto: &mut ConfigInputDto) -> Result { dto.prepare(0, false, &HashSet::new(), None) } + +fn media_server_config_with_library() -> MediaServerInputConfigDto { + MediaServerInputConfigDto { + libraries: vec![MediaServerLibrarySelector::Name("Movies".to_string())], + ..MediaServerInputConfigDto::default() + } +} + +mod catalog_auth; +mod input_provider_compatibility; diff --git a/shared/src/model/config/media_server_catalog/tests/catalog_auth.rs b/shared/src/model/config/media_server_catalog/tests/catalog_auth.rs new file mode 100644 index 000000000..ab5fcda5c --- /dev/null +++ b/shared/src/model/config/media_server_catalog/tests/catalog_auth.rs @@ -0,0 +1,312 @@ +use super::*; + +#[test] +fn prepare_rejects_blank_media_server_credentials_and_selectors() { + let mut emby = ConfigInputDto { + name: "emby_media_server".intern(), + input_type: InputType::Emby, + url: "https://media.example.invalid".to_string(), + media_server: Some(MediaServerInputConfigDto { + token: Some(" ".to_string()), + api_key: Some(String::new()), + ..media_server_config_with_library() + }), + ..ConfigInputDto::default() + }; + let err = prepare_dto(&mut emby).expect_err("blank token/api_key should be rejected"); + assert!(err.to_string().contains("requires media_server token/api_key")); + + let mut plex = ConfigInputDto { + name: "plex_media_server".intern(), + input_type: InputType::Plex, + media_server: Some(MediaServerInputConfigDto { + account_token: Some(" ".to_string()), + server_id: Some(" ".to_string()), + ..media_server_config_with_library() + }), + ..ConfigInputDto::default() + }; + let err = prepare_dto(&mut plex).expect_err("blank plex token should be rejected"); + assert!(err.to_string().contains("requires media_server.account_token")); +} + +#[test] +fn prepare_accepts_media_server_max_connections_as_stream_limit() { + let mut dto = ConfigInputDto { + name: "emby_media_server".intern(), + input_type: InputType::Emby, + url: "https://media.example.invalid".to_string(), + max_connections: 1, + media_server: Some(MediaServerInputConfigDto { + token: Some("token-value".to_string()), + ..media_server_config_with_library() + }), + ..ConfigInputDto::default() + }; + + prepare_dto(&mut dto).expect("media_server inputs reuse max_connections stream-limit semantics"); +} + +#[test] +fn media_server_defaults_are_conservative() { + let media_server = MediaServerInputConfigDto::default(); + + assert_eq!(media_server.catalog.page_size, 100); + assert_eq!(media_server.catalog.request_delay_ms, 250); + assert!(media_server.catalog.include_media_sources); + assert!(!media_server.catalog.include_paths); + assert!(!media_server.catalog.include_user_state); + assert!(!media_server.catalog.refresh_on_startup); + assert!(media_server.playback.direct_play_only); + assert!(!media_server.playback.allow_transcode); + assert!(!media_server.playback.preflight_streams); + assert_eq!(media_server.image_policy, MediaServerImagePolicy::ProxyOnDemand); + assert!(!media_server.allow_relay); +} + +#[test] +fn media_server_library_key_selector_accepts_numeric_yaml_scalars() { + let media_server = + serde_json::from_str::(r#"{"libraries":[{"key":10,"kind":"movies"}]}"#) + .expect("numeric YAML-like key selectors should deserialize as strings"); + + assert_eq!( + media_server.libraries, + vec![MediaServerLibrarySelector::Detailed(MediaServerLibrarySelectorDetailsDto { + key: Some("10".to_string()), + kind: Some(MediaServerLibraryKind::Movies), + ..MediaServerLibrarySelectorDetailsDto::default() + })] + ); + + let by_id = serde_json::from_str::(r#"{"libraries":[{"id":42,"kind":"tvshows"}]}"#) + .expect("numeric id selectors should deserialize as strings"); + assert_eq!( + by_id.libraries, + vec![MediaServerLibrarySelector::Detailed(MediaServerLibrarySelectorDetailsDto { + id: Some("42".to_string()), + kind: Some(MediaServerLibraryKind::TvShows), + ..MediaServerLibrarySelectorDetailsDto::default() + })] + ); +} + +#[test] +fn media_server_enrichment_block_is_not_part_of_schema() { + let err = serde_json::from_str::( + r#"{"libraries":["Movies"],"enrichment":{"ffprobe":true,"tmdb_lookup":true,"fetch_images":true}}"#, + ) + .expect_err("media_server.enrichment must not be accepted"); + + assert!(err.to_string().contains("unknown field `enrichment`")); +} + +#[test] +fn prepare_accepts_emby_media_server_with_token_and_library() { + let mut dto = ConfigInputDto { + name: "emby_media_server".intern(), + input_type: InputType::Emby, + url: " https://media.example.invalid/ ".to_string(), + media_server: Some(MediaServerInputConfigDto { + token: Some(" token-value ".to_string()), + ..media_server_config_with_library() + }), + ..ConfigInputDto::default() + }; + + prepare_dto(&mut dto).expect("emby media_server config should prepare"); + + assert_eq!(dto.url, "https://media.example.invalid/"); + assert!(dto.input_type.is_media_server()); + assert_eq!(dto.media_server.as_ref().and_then(|media_server| media_server.token.as_deref()), Some("token-value")); +} + +#[test] +fn prepare_rejects_media_server_without_media_server_block() { + let mut dto = ConfigInputDto { + name: "emby_media_server".intern(), + input_type: InputType::Emby, + url: "https://media.example.invalid".to_string(), + ..ConfigInputDto::default() + }; + + let err = prepare_dto(&mut dto).expect_err("media_server block should be mandatory"); + assert!(err.to_string().contains("media_server configuration is mandatory")); +} + +#[test] +fn prepare_allows_disabled_media_server_input_with_incomplete_config() { + let mut dto = ConfigInputDto { + name: "disabled_plex".intern(), + input_type: InputType::Plex, + enabled: false, + ..ConfigInputDto::default() + }; + + prepare_dto(&mut dto).expect("disabled media_server input should not require active playback/catalog config"); + assert_eq!(dto.input_type, InputType::Plex); + assert!(!dto.enabled); +} + +#[test] +fn prepare_normalizes_disabled_media_server_config_without_enforcing_invariants() { + let mut dto = ConfigInputDto { + name: "disabled_emby".intern(), + input_type: InputType::Emby, + enabled: false, + media_server: Some(MediaServerInputConfigDto { + token: Some(" token-value ".to_string()), + libraries: vec![MediaServerLibrarySelector::Name(" ".to_string())], + catalog: MediaServerCatalogConfigDto { page_size: 0, ..MediaServerCatalogConfigDto::default() }, + ..MediaServerInputConfigDto::default() + }), + ..ConfigInputDto::default() + }; + + prepare_dto(&mut dto).expect("disabled media_server input can preserve incomplete config for later repair"); + let media_server = dto.media_server.as_ref().expect("media_server config should be preserved"); + assert_eq!(media_server.token.as_deref(), Some("token-value")); + assert!(media_server.libraries[0].is_empty()); +} + +#[test] +fn prepare_rejects_emby_media_server_without_input_url() { + let mut dto = ConfigInputDto { + name: "emby_media_server".intern(), + input_type: InputType::Emby, + media_server: Some(MediaServerInputConfigDto { + token: Some("token-value".to_string()), + ..media_server_config_with_library() + }), + ..ConfigInputDto::default() + }; + + let err = prepare_dto(&mut dto).expect_err("emby media_server input should require a direct server URL"); + assert!(err.to_string().contains("url is mandatory for input type emby")); +} + +#[test] +fn prepare_rejects_media_server_provider_scheme_url() { + let mut dto = ConfigInputDto { + name: "emby_media_server".intern(), + input_type: InputType::Emby, + url: " provider://media-server ".to_string(), + media_server: Some(MediaServerInputConfigDto { + token: Some("token-value".to_string()), + ..media_server_config_with_library() + }), + ..ConfigInputDto::default() + }; + + let err = prepare_dto(&mut dto).expect_err("media_server input must not use provider URLs"); + assert!(err.to_string().contains("does not support batch:// or provider://")); +} + +#[test] +fn prepare_rejects_plex_without_token_or_server_selector() { + let mut without_token = ConfigInputDto { + name: "plex_media_server".intern(), + input_type: InputType::Plex, + media_server: Some(MediaServerInputConfigDto { + server_id: Some("server".to_string()), + ..media_server_config_with_library() + }), + ..ConfigInputDto::default() + }; + let err = prepare_dto(&mut without_token).expect_err("plex token should be mandatory"); + assert!(err.to_string().contains("requires media_server.account_token")); + + let mut without_selector = ConfigInputDto { + name: "plex_media_server".intern(), + input_type: InputType::Plex, + media_server: Some(MediaServerInputConfigDto { + account_token: Some("token".to_string()), + ..media_server_config_with_library() + }), + ..ConfigInputDto::default() + }; + let err = prepare_dto(&mut without_selector).expect_err("plex server selector should be mandatory"); + assert!(err.to_string().contains("requires a server selector")); +} + +#[test] +fn prepare_accepts_plex_without_input_url_when_discovery_is_configured() { + let mut dto = ConfigInputDto { + name: "plex_media_server".intern(), + input_type: InputType::Plex, + media_server: Some(MediaServerInputConfigDto { + account_token: Some("token".to_string()), + server_id: Some("server".to_string()), + ..media_server_config_with_library() + }), + ..ConfigInputDto::default() + }; + + prepare_dto(&mut dto).expect("plex discovery config should not require input.url"); + assert_eq!(dto.input_type, InputType::Plex); +} + +#[test] +fn prepare_accepts_plex_media_server_with_direct_url_without_selector() { + let mut dto = ConfigInputDto { + name: "plex_media_server".intern(), + input_type: InputType::Plex, + url: "https://plex.example.invalid".to_string(), + media_server: Some(MediaServerInputConfigDto { + token: Some("token".to_string()), + ..media_server_config_with_library() + }), + ..ConfigInputDto::default() + }; + + prepare_dto(&mut dto).expect("direct Plex URL should not require MyPlex server selector"); + assert_eq!(dto.input_type, InputType::Plex); +} + +#[test] +fn prepare_rejects_plex_direct_url_without_server_token() { + let mut dto = ConfigInputDto { + name: "plex_media_server".intern(), + input_type: InputType::Plex, + url: "https://plex.example.invalid".to_string(), + media_server: Some(MediaServerInputConfigDto { + account_token: Some("account-token".to_string()), + ..media_server_config_with_library() + }), + ..ConfigInputDto::default() + }; + + let err = prepare_dto(&mut dto).expect_err("direct Plex URL requires the PMS server token"); + assert!(err.to_string().contains("requires media_server.token")); +} + +#[test] +fn prepare_rejects_plex_discovery_without_account_token() { + let mut dto = ConfigInputDto { + name: "plex_media_server".intern(), + input_type: InputType::Plex, + media_server: Some(MediaServerInputConfigDto { + token: Some("server-token".to_string()), + server_id: Some("server".to_string()), + ..media_server_config_with_library() + }), + ..ConfigInputDto::default() + }; + + let err = prepare_dto(&mut dto).expect_err("Plex discovery requires the MyPlex account token"); + assert!(err.to_string().contains("requires media_server.account_token")); +} + +#[test] +fn prepare_type_does_not_validate_media_server_config() { + let mut dto = ConfigInputDto { + name: "emby_media_server".intern(), + input_type: InputType::Emby, + url: "https://media.example.invalid".to_string(), + ..ConfigInputDto::default() + }; + + dto.prepare_type().expect("prepare_type only normalizes type/url"); + let err = prepare_dto(&mut dto).expect_err("full prepare should validate missing media_server block"); + assert!(err.to_string().contains("media_server configuration is mandatory")); +} diff --git a/shared/src/model/config/media_server_catalog/tests/input_provider_compatibility.rs b/shared/src/model/config/media_server_catalog/tests/input_provider_compatibility.rs new file mode 100644 index 000000000..c833725eb --- /dev/null +++ b/shared/src/model/config/media_server_catalog/tests/input_provider_compatibility.rs @@ -0,0 +1,303 @@ +use super::*; + +#[test] +fn test_provider_dns_defaults() { + let dns = ProviderDnsDto::default(); + assert!(!dns.enabled); + assert_eq!(dns.refresh_secs, 300); + assert_eq!(dns.prefer, DnsPrefer::System); + assert_eq!(dns.on_resolve_error, OnResolveErrorPolicy::KeepLastGood); + assert_eq!(dns.on_connect_error, OnConnectErrorPolicy::TryNextIp); + assert!(dns.schemes.is_none()); +} + +#[test] +fn test_provider_url_selection_policy_defaults_to_resume_last_working() { + let provider = ConfigProviderDto { + name: "provider-a".intern(), + urls: vec!["http://primary.example.com".intern()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::default(), + dns: None, + }; + + assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::ResumeLastWorking); +} + +#[test] +fn test_provider_url_selection_policy_can_be_set_to_restart_from_first() { + let provider = ConfigProviderDto { + name: "provider-a".intern(), + urls: vec!["http://primary.example.com".intern()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::RestartFromFirst, + dns: None, + }; + + assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::RestartFromFirst); +} + +#[test] +fn test_provider_url_selection_policy_deserializes_default_when_omitted() { + let provider: ConfigProviderDto = + serde_json::from_str(r#"{"name":"provider-a","urls":["http://primary.example.com"]}"#) + .expect("provider dto should deserialize"); + + assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::ResumeLastWorking); +} + +#[test] +fn test_provider_url_selection_policy_deserializes_restart_from_first() { + let provider: ConfigProviderDto = serde_json::from_str( + r#"{"name":"provider-a","urls":["http://primary.example.com"],"provider_url_selection_policy":"restart_from_first"}"#, + ) + .expect("provider dto should deserialize"); + + assert_eq!(provider.provider_url_selection_policy, ProviderUrlSelectionPolicy::RestartFromFirst); +} + +#[test] +fn test_provider_url_selection_policy_default_is_omitted_on_serialize() { + let provider = ConfigProviderDto { + name: "provider-a".intern(), + urls: vec!["http://primary.example.com".intern()], + provider_url_selection_policy: ProviderUrlSelectionPolicy::ResumeLastWorking, + dns: None, + }; + + let json = serde_json::to_string(&provider).expect("provider dto should serialize"); + let value: serde_json::Value = serde_json::from_str(&json).expect("serialized provider should be valid json"); + + assert!(value.get("provider_url_selection_policy").is_none()); +} + +#[test] +fn test_provider_dns_prepare_normalizes_overrides_and_clamps_refresh() { + let mut dns = ProviderDnsDto { + refresh_secs: 1, + schemes: Some(vec![DnsScheme::Http, DnsScheme::Http, DnsScheme::Https]), + overrides: Some(HashMap::from([( + " EXAMPLE.COM ".to_string(), + vec![ + "203.0.113.10".parse::().expect("valid ip"), + "203.0.113.10".parse::().expect("valid ip"), + ], + )])), + ..ProviderDnsDto::default() + }; + + dns.prepare().expect("dns prepare should succeed"); + + assert_eq!(dns.refresh_secs, 1); + assert_eq!(dns.schemes, Some(vec![DnsScheme::Http, DnsScheme::Https])); + let overrides = dns.overrides.expect("overrides should exist"); + assert_eq!(overrides.len(), 1); + assert!(overrides.contains_key("example.com")); + assert_eq!(overrides["example.com"].len(), 1); +} + +#[test] +fn prepare_switches_xtream_to_xtream_batch_when_alias_exists() { + let mut dto = ConfigInputDto { + name: "input_alias".intern(), + input_type: InputType::Xtream, + url: "batch:///tmp/input_alias.csv".to_string(), + aliases: Some(vec![ConfigInputAliasDto { + id: 1, + name: "alias_1".intern(), + url: "http://provider.example/stream".to_string(), + username: Some("u".to_string()), + password: Some("p".to_string()), + enabled: true, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }; + + dto.prepare_type().expect("prepare type should succeed"); + dto.prepare(0, true, &HashSet::new(), None).expect("prepare should succeed and infer batch type from batch:// URL"); + assert_eq!(dto.input_type, InputType::XtreamBatch); +} + +#[test] +fn prepare_keeps_xtream_type_when_alias_exists_without_batch_url() { + let mut dto = ConfigInputDto { + name: "input_alias_http".intern(), + input_type: InputType::XtreamBatch, + url: "http://localhost:3001".to_string(), + username: Some("root_user".to_string()), + password: Some("root_pass".to_string()), + aliases: Some(vec![ConfigInputAliasDto { + id: 1, + name: "alias_1".intern(), + url: "http://provider.example/stream".to_string(), + username: Some("u".to_string()), + password: Some("p".to_string()), + enabled: true, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }; + + dto.prepare_type().expect("prepare type should normalize non-batch URL to xtream"); + assert_eq!(dto.input_type, InputType::Xtream); + dto.prepare(0, true, &HashSet::new(), None).expect("prepare should succeed for regular URL with aliases"); + assert_eq!(dto.input_type, InputType::Xtream); +} + +#[test] +fn prepare_batch_url_does_not_require_xtream_credentials() { + let mut dto = ConfigInputDto { + name: "batch_no_creds".intern(), + input_type: InputType::Xtream, + url: "batch:///tmp/no-creds.csv".to_string(), + username: None, + password: None, + ..ConfigInputDto::default() + }; + + dto.prepare(0, true, &HashSet::new(), None) + .expect("batch:// input must be normalized before credential validation"); + assert_eq!(dto.input_type, InputType::XtreamBatch); +} + +#[test] +fn prepare_provider_scheme_url_is_not_treated_as_batch_input() { + let mut dto = ConfigInputDto { + name: "batch_provider".intern(), + input_type: InputType::XtreamBatch, + url: "provider://myprovider".to_string(), + username: Some("root_user".to_string()), + password: Some("root_pass".to_string()), + aliases: Some(vec![ConfigInputAliasDto { + id: 1, + name: "alias_1".intern(), + url: "http://provider.example/stream".to_string(), + username: Some("u".to_string()), + password: Some("p".to_string()), + enabled: true, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }; + + let err = dto + .prepare(0, true, &HashSet::new(), None) + .expect_err("prepare should treat provider:// URL as regular input (non-batch) and validate provider"); + assert!(err.to_string().contains("Provider name myprovider is not defined"), "Error: {err}"); +} + +#[test] +fn prepare_rejects_missing_input_url_even_with_aliases() { + let mut dto = ConfigInputDto { + name: "xtream_missing_root_url".intern(), + input_type: InputType::Xtream, + url: String::new(), + username: Some("root_user".to_string()), + password: Some("root_pass".to_string()), + aliases: Some(vec![ConfigInputAliasDto { + id: 1, + name: "alias_1".intern(), + url: "http://alias.example".to_string(), + username: Some("alias_user".to_string()), + password: Some("alias_pass".to_string()), + enabled: true, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }; + + let err = dto + .prepare(0, true, &HashSet::new(), None) + .expect_err("prepare must require root input url even when aliases are present"); + assert!(err.to_string().contains("url for input is mandatory"), "Error: {err}"); + assert!(err.to_string().contains("xtream_missing_root_url"), "Error: {err}"); +} + +#[test] +fn prepare_rejects_missing_root_credentials_for_non_batch_url_even_with_aliases() { + let mut dto = ConfigInputDto { + name: "xtream_batch_missing_root_creds".intern(), + input_type: InputType::XtreamBatch, + url: "http://root.example".to_string(), + aliases: Some(vec![ConfigInputAliasDto { + id: 1, + name: "alias_1".intern(), + url: "http://alias.example".to_string(), + username: Some("alias_user".to_string()), + password: Some("alias_pass".to_string()), + enabled: true, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }; + + let err = dto + .prepare(0, true, &HashSet::new(), None) + .expect_err("prepare must require root credentials for non-batch URL"); + assert!(err.to_string().contains("for input type xtream: username and password are mandatory"), "Error: {err}"); + assert!(err.to_string().contains("xtream_batch_missing_root_creds"), "Error: {err}"); +} + +#[test] +fn prepare_rejects_xtream_batch_batch_url_with_root_credentials_even_with_aliases() { + let mut dto = ConfigInputDto { + name: "xtream_batch_with_root_creds".intern(), + input_type: InputType::XtreamBatch, + url: "batch:///tmp/aliases.csv".to_string(), + username: Some("root_user".to_string()), + password: Some("root_pass".to_string()), + aliases: Some(vec![ConfigInputAliasDto { + id: 1, + name: "alias_1".intern(), + url: "http://alias.example".to_string(), + username: Some("alias_user".to_string()), + password: Some("alias_pass".to_string()), + enabled: true, + ..ConfigInputAliasDto::default() + }]), + ..ConfigInputDto::default() + }; + + let err = dto + .prepare(0, true, &HashSet::new(), None) + .expect_err("prepare must reject root credentials when using batch:// for xtream-batch"); + assert!(err.to_string().contains("with batch:// URL should not define username or password"), "Error: {err}"); + assert!(err.to_string().contains("xtream_batch_with_root_creds"), "Error: {err}"); +} + +#[test] +fn test_config_input_options_dto_filter_prepare_parses_valid_filter() { + let mut dto = ConfigInputOptionsDto { + resolve_filter: Some(r#"name ~ "test""#.to_string()), + ..ConfigInputOptionsDto::default() + }; + dto.prepare(None).expect("valid filter should parse"); + assert!(dto.t_resolve_filter.is_some()); +} + +#[test] +fn test_config_input_options_dto_filter_prepare_rejects_invalid_filter() { + let mut dto = ConfigInputOptionsDto { + resolve_filter: Some(r#"name ~ "["#.to_string()), // invalid regex + ..ConfigInputOptionsDto::default() + }; + let result = dto.prepare(None); + assert!(result.is_err()); +} + +#[test] +fn test_config_input_options_dto_filter_prepare_with_unknown_template_placeholder() { + let mut dto = ConfigInputOptionsDto { + resolve_filter: Some(r#"name ~ "!UNKNOWN!""#.to_string()), + ..ConfigInputOptionsDto::default() + }; + let result = dto.prepare(None); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("Unknown template placeholder")); +} + +#[test] +fn test_config_input_options_dto_filter_none_prepares_successfully() { + let mut dto = ConfigInputOptionsDto { resolve_filter: None, ..ConfigInputOptionsDto::default() }; + dto.prepare(None).expect("None filter should prepare successfully"); + assert!(dto.t_resolve_filter.is_none()); +} diff --git a/shared/src/model/config/messaging.rs b/shared/src/model/config/messaging.rs index b9d14757e..d3ee9e1b0 100644 --- a/shared/src/model/config/messaging.rs +++ b/shared/src/model/config/messaging.rs @@ -1,344 +1,9 @@ -use crate::{ - defaults::{default_critical_percent, default_repeat_interval_secs, default_warn_percent, is_false}, - error::TuliproxError, - utils::{is_blank_optional_str, is_blank_optional_string}, -}; - /// Placeholder substituted for a channel secret on the way out to a client. /// /// A client that sends it back unchanged means "keep what is stored"; see /// [`MessagingConfigDto::restore_redacted_secrets`]. pub const REDACTED_SECRET: &str = "********"; -/// Per-channel routing. -/// -/// Without this every enabled event went to every configured channel, so -/// "critical to Pushover, everything to Discord" was not expressible. -/// Every field is optional and an omitted block inherits the global -/// `notify_on`. -#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ChannelRoutingDto { - /// Overrides the global `notify_on` for this channel. Same glob - /// grammar. Empty means inherit. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub notify_on: Vec, - /// Drop anything below this severity: `info`, `warn`, `error`, - /// `critical`. - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub min_severity: Option, - /// `HH:MM-HH:MM` local time. Notifications inside the window are - /// deferred by the outbox, never dropped - an overnight outage must not - /// be silently invisible. - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub quiet_hours: Option, - /// Circuit breaker. On reaching it the channel sends one "suppressing - /// further notifications" message and then goes quiet for the rest of - /// the hour. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_per_hour: Option, - /// Suppress a repeated `dedup_key` for this many seconds. Generalizes - /// the disk alert's `repeat_interval_secs`, which was only available to - /// disk alerts. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub dedup_window_secs: Option, -} - -impl ChannelRoutingDto { - pub fn is_empty(&self) -> bool { - self.notify_on.is_empty() - && is_blank_optional_str(self.min_severity.as_deref()) - && is_blank_optional_str(self.quiet_hours.as_deref()) - && self.max_per_hour.is_none() - && self.dedup_window_secs.is_none() - } - - /// Reject values that would otherwise fail silently at send time. - pub fn prepare(&self) -> Result<(), TuliproxError> { - if let Some(severity) = self.min_severity.as_deref().filter(|s| !s.trim().is_empty()) { - if crate::model::notification::Severity::from_wire(severity).is_none() { - return Err(TuliproxError::Config(format!( - "messaging min_severity must be one of info, warn, error, critical; got `{severity}`" - ))); - } - } - if let Some(window) = self.quiet_hours.as_deref().filter(|s| !s.trim().is_empty()) { - if crate::model::notification::QuietHours::parse(window).is_none() { - return Err(TuliproxError::Config(format!( - "messaging quiet_hours must look like `23:00-07:00`; got `{window}`" - ))); - } - } - Ok(()) - } -} - -#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct TelegramMessagingConfigDto { - pub bot_token: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub chat_ids: Vec, - #[serde(default, skip_serializing_if = "is_false")] - pub markdown: bool, - /// Template per event, keyed by event id wire name. - /// - /// Legacy `MsgKind` names (`info`, `stats`, `disk_alert`, ...) and - /// canonical dotted ids (`recording.completed`) are both accepted; - /// `MessagingConfig::prepare` resolves them and warns on a key that - /// matches no known event. - #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] - pub templates: std::collections::HashMap, - /// Per-channel routing. Inherits the global `notify_on` when absent. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub routing: Option, -} - -impl TelegramMessagingConfigDto { - pub fn is_empty(&self) -> bool { - self.bot_token.trim().is_empty() && self.chat_ids.is_empty() && self.templates.is_empty() - } -} - -#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RestMessagingConfigDto { - pub url: String, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub method: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub headers: Vec, - /// HMAC-SHA256 signing secret. - /// - /// When set, each request carries `X-Tuliprox-Timestamp` and - /// `X-Tuliprox-Signature: sha256=` over `timestamp.body`, so the - /// receiving endpoint can verify the sender. - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub signing_secret: Option, - /// Template per event, keyed by event id wire name. - /// - /// Legacy `MsgKind` names (`info`, `stats`, `disk_alert`, ...) and - /// canonical dotted ids (`recording.completed`) are both accepted; - /// `MessagingConfig::prepare` resolves them and warns on a key that - /// matches no known event. - #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] - pub templates: std::collections::HashMap, - /// Per-channel routing. Inherits the global `notify_on` when absent. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub routing: Option, -} - -impl RestMessagingConfigDto { - pub fn is_empty(&self) -> bool { - self.url.trim().is_empty() - && is_blank_optional_str(self.method.as_deref()) - && self.headers.is_empty() - && self.templates.is_empty() - } -} - -#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct DiscordMessagingConfigDto { - pub url: String, - /// Template per event, keyed by event id wire name. - /// - /// Legacy `MsgKind` names (`info`, `stats`, `disk_alert`, ...) and - /// canonical dotted ids (`recording.completed`) are both accepted; - /// `MessagingConfig::prepare` resolves them and warns on a key that - /// matches no known event. - #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] - pub templates: std::collections::HashMap, - /// Per-channel routing. Inherits the global `notify_on` when absent. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub routing: Option, -} - -impl DiscordMessagingConfigDto { - pub fn is_empty(&self) -> bool { self.url.trim().is_empty() && self.templates.is_empty() } -} - -#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct PushoverMessagingConfigDto { - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub url: Option, - pub token: String, - pub user: String, - /// Template per event, keyed by event id wire name. - /// - /// Pushover previously had no template support at all, so every - /// notification took the built-in text - which for watch changes and - /// playlist stats was a raw `serde_json` dump pushed to a phone. - #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] - pub templates: std::collections::HashMap, - /// Per-channel routing. Inherits the global `notify_on` when absent. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub routing: Option, -} - -impl PushoverMessagingConfigDto { - pub fn is_empty(&self) -> bool { - is_blank_optional_str(self.url.as_deref()) - && self.token.trim().is_empty() - && self.user.trim().is_empty() - && self.templates.is_empty() - } -} - -/// [ntfy](https://ntfy.sh) - self-hosted push, no account, no bot token. -#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct NtfyMessagingConfigDto { - /// Server base URL, e.g. `https://ntfy.sh`. - pub url: String, - /// Topic to publish to. - pub topic: String, - /// Bearer token for a protected topic. - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub token: Option, - #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] - pub templates: std::collections::HashMap, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub routing: Option, -} - -impl NtfyMessagingConfigDto { - pub fn is_empty(&self) -> bool { self.url.trim().is_empty() && self.topic.trim().is_empty() } -} - -/// [Gotify](https://gotify.net) - same audience as ntfy, same shape. -#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct GotifyMessagingConfigDto { - /// Server base URL. - pub url: String, - /// Application token. - pub token: String, - #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] - pub templates: std::collections::HashMap, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub routing: Option, -} - -impl GotifyMessagingConfigDto { - pub fn is_empty(&self) -> bool { self.url.trim().is_empty() && self.token.trim().is_empty() } -} - -/// Slack incoming webhook. -/// -/// Not a Discord clone: Block Kit differs enough from Discord embeds that -/// reusing the Discord payload shape produces bad output. -#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct SlackMessagingConfigDto { - pub url: String, - #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] - pub templates: std::collections::HashMap, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub routing: Option, -} - -impl SlackMessagingConfigDto { - pub fn is_empty(&self) -> bool { self.url.trim().is_empty() } -} - -/// Run a local program, with the event JSON on stdin. -/// -/// The escape hatch that means nobody has to wait for a channel to be -/// added upstream. This runs arbitrary code as the tuliprox process user, -/// so it is opt-in and never configured by default. -#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct CommandMessagingConfigDto { - /// Program to run. Not passed through a shell, so no quoting rules and - /// no shell injection. - pub program: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub args: Vec, - /// Kill the child after this many seconds. Defaults to 30. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub timeout_secs: Option, - #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] - pub templates: std::collections::HashMap, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub routing: Option, -} - -impl CommandMessagingConfigDto { - pub fn is_empty(&self) -> bool { self.program.trim().is_empty() } -} - -/// Configuration for threshold-driven disk-space alerts. -/// -/// Set `messaging.disk_alert` to enable notifications whenever the filesystem -/// hosting the process working directory crosses `warn_percent` or -/// `critical_percent`. The same alert is re-sent after `repeat_interval_secs` -/// while the disk stays in the same alert state. -/// -/// All fields are optional and the alert is disabled when the whole block is -/// absent. Defaults match the values used at runtime when fields are unset. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct DiskAlertConfigDto { - /// Threshold (0-100, exclusive) below which the disk is considered normal. - /// Defaults to `80.0`. - #[serde(default = "default_warn_percent")] - pub warn_percent: f64, - /// Threshold (0-100, inclusive) at which the disk is considered full. - /// Must be `> warn_percent`. Defaults to `95.0`. - #[serde(default = "default_critical_percent")] - pub critical_percent: f64, - /// Re-arm interval in seconds. While the disk stays in the same alert - /// state, the alert is re-sent after this many seconds. Defaults to - /// `3600` (1h). - #[serde(default = "default_repeat_interval_secs")] - pub repeat_interval_secs: u64, -} - -impl DiskAlertConfigDto { - pub fn is_empty(&self) -> bool { - self.warn_percent == default_warn_percent() - && self.critical_percent == default_critical_percent() - && self.repeat_interval_secs == default_repeat_interval_secs() - } -} - -impl Default for DiskAlertConfigDto { - fn default() -> Self { - Self { - warn_percent: default_warn_percent(), - critical_percent: default_critical_percent(), - repeat_interval_secs: default_repeat_interval_secs(), - } - } -} - -impl DiskAlertConfigDto { - /// Sanity-check the thresholds. The caller surfaces the error to the user. - pub fn prepare(&self) -> Result<(), TuliproxError> { - if !(0.0..=100.0).contains(&self.warn_percent) { - return Err(TuliproxError::Config(format!( - "messaging.disk_alert.warn_percent must be in [0, 100], got {}", - self.warn_percent - ))); - } - if !(0.0..=100.0).contains(&self.critical_percent) { - return Err(TuliproxError::Config(format!( - "messaging.disk_alert.critical_percent must be in [0, 100], got {}", - self.critical_percent - ))); - } - if self.critical_percent <= self.warn_percent { - return Err(TuliproxError::Config(format!( - "messaging.disk_alert.critical_percent ({}) must be > warn_percent ({})", - self.critical_percent, self.warn_percent - ))); - } - Ok(()) - } -} - #[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct MessagingConfigDto { @@ -384,82 +49,6 @@ impl MessagingConfigDto { && (self.command.is_none() || self.command.as_ref().is_some_and(CommandMessagingConfigDto::is_empty)) } - /// Replace channel secrets with [`REDACTED_SECRET`]. - /// - /// The config GET returns `config.yml` in full to any client with - /// `ConfigRead`, which included the Telegram bot token, the Pushover - /// token and user key, and any `Authorization` header configured on the - /// REST channel. - pub fn redact_secrets(&mut self) { - if let Some(telegram) = self.telegram.as_mut() { - redact(&mut telegram.bot_token); - } - if let Some(pushover) = self.pushover.as_mut() { - redact(&mut pushover.token); - redact(&mut pushover.user); - } - if let Some(ntfy) = self.ntfy.as_mut() { - if let Some(token) = ntfy.token.as_mut() { - redact(token); - } - } - if let Some(gotify) = self.gotify.as_mut() { - redact(&mut gotify.token); - } - if let Some(rest) = self.rest.as_mut() { - if let Some(secret) = rest.signing_secret.as_mut() { - redact(secret); - } - for header in &mut rest.headers { - // `Name: value` - keep the name, mask the value. - if let Some((name, _)) = header.split_once(':') { - if is_sensitive_header(name) { - *header = format!("{name}: {REDACTED_SECRET}"); - } - } - } - } - } - - /// Put back any secret the client returned still redacted. - /// - /// Without this a round-trip through the UI would overwrite the stored - /// token with the mask, silently breaking the channel. - pub fn restore_redacted_secrets(&mut self, current: &Self) { - if let (Some(incoming), Some(stored)) = (self.telegram.as_mut(), current.telegram.as_ref()) { - restore(&mut incoming.bot_token, &stored.bot_token); - } - if let (Some(incoming), Some(stored)) = (self.pushover.as_mut(), current.pushover.as_ref()) { - restore(&mut incoming.token, &stored.token); - restore(&mut incoming.user, &stored.user); - } - if let (Some(incoming), Some(stored)) = (self.ntfy.as_mut(), current.ntfy.as_ref()) { - if let (Some(token), Some(original)) = (incoming.token.as_mut(), stored.token.as_ref()) { - restore(token, original); - } - } - if let (Some(incoming), Some(stored)) = (self.gotify.as_mut(), current.gotify.as_ref()) { - restore(&mut incoming.token, &stored.token); - } - if let (Some(incoming), Some(stored)) = (self.rest.as_mut(), current.rest.as_ref()) { - if let (Some(secret), Some(original)) = (incoming.signing_secret.as_mut(), stored.signing_secret.as_ref()) { - restore(secret, original); - } - for header in &mut incoming.headers { - let Some((name, value)) = header.split_once(':') else { continue }; - if value.trim() != REDACTED_SECRET { - continue; - } - let name = name.to_string(); - if let Some(original) = - stored.headers.iter().find(|h| h.split_once(':').is_some_and(|(n, _)| n.trim() == name.trim())) - { - *header = original.clone(); - } - } - } - } - pub fn clean(&mut self) { if self.telegram.as_ref().is_some_and(TelegramMessagingConfigDto::is_empty) { self.telegram = None; @@ -489,218 +78,16 @@ impl MessagingConfigDto { self.disk_alert = None; } } - - pub fn prepare(&mut self, _include_computed: bool) -> Result<(), TuliproxError> { - if let Some(disk) = &self.disk_alert { - disk.prepare()?; - } - for routing in [ - self.telegram.as_ref().and_then(|c| c.routing.as_ref()), - self.rest.as_ref().and_then(|c| c.routing.as_ref()), - self.discord.as_ref().and_then(|c| c.routing.as_ref()), - self.pushover.as_ref().and_then(|c| c.routing.as_ref()), - self.ntfy.as_ref().and_then(|c| c.routing.as_ref()), - self.gotify.as_ref().and_then(|c| c.routing.as_ref()), - self.slack.as_ref().and_then(|c| c.routing.as_ref()), - self.command.as_ref().and_then(|c| c.routing.as_ref()), - ] - .into_iter() - .flatten() - { - routing.prepare()?; - } - self.normalize_notify_on(); - Ok(()) - } - - /// Rewrite legacy `MsgKind` names to their canonical event ids. - /// - /// Keeps the UI's selection state consistent with the registry-driven - /// option list, and migrates an old config the next time it is saved. - /// Glob patterns are left exactly as written. - fn normalize_notify_on(&mut self) { - for entry in &mut self.notify_on { - if entry.contains('*') || entry.starts_with('!') { - continue; - } - if let Some(id) = crate::model::notification::EventId::from_wire(entry) { - *entry = id.as_str().to_string(); - } - } - } -} - -fn redact(value: &mut String) { - if !value.trim().is_empty() { - *value = REDACTED_SECRET.to_string(); - } -} - -fn restore(incoming: &mut String, stored: &str) { - if incoming.trim() == REDACTED_SECRET { - *incoming = stored.to_string(); - } -} - -/// Header names whose value is a credential. -fn is_sensitive_header(name: &str) -> bool { - const SENSITIVE: &[&str] = &["authorization", "x-api-key", "x-auth-token", "proxy-authorization", "cookie"]; - let name = name.trim(); - SENSITIVE.iter().any(|candidate| name.eq_ignore_ascii_case(candidate)) } #[cfg(test)] -mod tests { - use super::*; +mod tests; - #[test] - fn clean_preserves_disk_alert_with_default_values() { - let mut messaging = - MessagingConfigDto { disk_alert: Some(DiskAlertConfigDto::default()), ..Default::default() }; - messaging.clean(); - assert_eq!(messaging.disk_alert, None); - } - - #[test] - fn clean_preserves_disk_alert_with_non_default_values() { - let custom = DiskAlertConfigDto { warn_percent: 70.0, critical_percent: 90.0, repeat_interval_secs: 600 }; - let mut messaging = MessagingConfigDto { disk_alert: Some(custom.clone()), ..Default::default() }; - messaging.clean(); - assert_eq!(messaging.disk_alert, Some(custom)); - } - - #[test] - fn clean_strips_empty_subconfigs() { - let mut messaging = MessagingConfigDto { - telegram: Some(TelegramMessagingConfigDto::default()), - rest: Some(RestMessagingConfigDto::default()), - pushover: Some(PushoverMessagingConfigDto::default()), - discord: Some(DiscordMessagingConfigDto::default()), - ..Default::default() - }; - messaging.clean(); - assert!(messaging.telegram.is_none()); - assert!(messaging.rest.is_none()); - assert!(messaging.pushover.is_none()); - assert!(messaging.discord.is_none()); - } - - #[test] - fn prepare_rewrites_legacy_notify_on_names_to_canonical_ids() { - let mut messaging = MessagingConfigDto { - notify_on: vec!["disk_alert".to_string(), "recording_completed".to_string()], - ..Default::default() - }; - messaging.prepare(false).expect("prepare"); - assert_eq!(messaging.notify_on, vec!["system.disk.alert", "recording.completed"]); - } - - #[test] - fn prepare_leaves_glob_patterns_untouched() { - let mut messaging = MessagingConfigDto { - notify_on: vec!["recording.*".to_string(), "!system.info".to_string(), "*".to_string()], - ..Default::default() - }; - messaging.prepare(false).expect("prepare"); - assert_eq!(messaging.notify_on, vec!["recording.*", "!system.info", "*"]); - } - - #[test] - fn redaction_masks_every_channel_secret() { - let mut messaging = MessagingConfigDto { - telegram: Some(TelegramMessagingConfigDto { bot_token: "real-token".to_string(), ..Default::default() }), - pushover: Some(PushoverMessagingConfigDto { - token: "real-app".to_string(), - user: "real-user".to_string(), - ..Default::default() - }), - rest: Some(RestMessagingConfigDto { - url: "https://example.test".to_string(), - headers: vec!["Authorization: Bearer secret".to_string(), "X-Trace: keep-me".to_string()], - ..Default::default() - }), - ..Default::default() - }; - messaging.redact_secrets(); - assert_eq!(messaging.telegram.as_ref().expect("telegram").bot_token, REDACTED_SECRET); - assert_eq!(messaging.pushover.as_ref().expect("pushover").token, REDACTED_SECRET); - assert_eq!(messaging.pushover.as_ref().expect("pushover").user, REDACTED_SECRET); - let headers = &messaging.rest.as_ref().expect("rest").headers; - assert!(headers.contains(&format!("Authorization: {REDACTED_SECRET}")), "auth header not masked: {headers:?}"); - // A non-credential header keeps its value; masking everything would - // make the config unreadable for no gain. - assert!(headers.contains(&"X-Trace: keep-me".to_string()), "harmless header was masked: {headers:?}"); - } - - #[test] - fn an_empty_secret_is_not_replaced_by_a_mask() { - // Otherwise an unconfigured channel would look configured. - let mut messaging = - MessagingConfigDto { telegram: Some(TelegramMessagingConfigDto::default()), ..Default::default() }; - messaging.redact_secrets(); - assert_eq!(messaging.telegram.as_ref().expect("telegram").bot_token, ""); - } - - #[test] - fn a_returned_mask_restores_the_stored_secret() { - // Without this the UI round-trip would overwrite the real token with - // the mask and silently break the channel. - let stored = MessagingConfigDto { - telegram: Some(TelegramMessagingConfigDto { bot_token: "real-token".to_string(), ..Default::default() }), - ..Default::default() - }; - let mut incoming = MessagingConfigDto { - telegram: Some(TelegramMessagingConfigDto { bot_token: REDACTED_SECRET.to_string(), ..Default::default() }), - ..Default::default() - }; - incoming.restore_redacted_secrets(&stored); - assert_eq!(incoming.telegram.as_ref().expect("telegram").bot_token, "real-token"); - } - - #[test] - fn a_genuinely_changed_secret_is_written_through() { - let stored = MessagingConfigDto { - telegram: Some(TelegramMessagingConfigDto { bot_token: "old".to_string(), ..Default::default() }), - ..Default::default() - }; - let mut incoming = MessagingConfigDto { - telegram: Some(TelegramMessagingConfigDto { bot_token: "new".to_string(), ..Default::default() }), - ..Default::default() - }; - incoming.restore_redacted_secrets(&stored); - assert_eq!(incoming.telegram.as_ref().expect("telegram").bot_token, "new"); - } - - #[test] - fn a_returned_masked_header_restores_the_stored_one() { - let stored = MessagingConfigDto { - rest: Some(RestMessagingConfigDto { - url: "https://example.test".to_string(), - headers: vec!["Authorization: Bearer secret".to_string()], - ..Default::default() - }), - ..Default::default() - }; - let mut incoming = MessagingConfigDto { - rest: Some(RestMessagingConfigDto { - url: "https://example.test".to_string(), - headers: vec![format!("Authorization: {REDACTED_SECRET}")], - ..Default::default() - }), - ..Default::default() - }; - incoming.restore_redacted_secrets(&stored); - assert_eq!(incoming.rest.as_ref().expect("rest").headers, vec!["Authorization: Bearer secret".to_string()]); - } - - #[test] - fn is_empty_default_messaging() { - assert!(MessagingConfigDto::default().is_empty()); - } - - #[test] - fn is_empty_with_only_notify_on_is_not_empty() { - let messaging = MessagingConfigDto { notify_on: vec!["disk_alert".to_string()], ..Default::default() }; - assert!(!messaging.is_empty()); - } -} +mod channels; +mod routing; +mod secrets; +pub use channels::{ + CommandMessagingConfigDto, DiscordMessagingConfigDto, GotifyMessagingConfigDto, NtfyMessagingConfigDto, + PushoverMessagingConfigDto, RestMessagingConfigDto, SlackMessagingConfigDto, TelegramMessagingConfigDto, +}; +pub use routing::{ChannelRoutingDto, DiskAlertConfigDto}; diff --git a/shared/src/model/config/messaging/channels.rs b/shared/src/model/config/messaging/channels.rs new file mode 100644 index 000000000..24a73151e --- /dev/null +++ b/shared/src/model/config/messaging/channels.rs @@ -0,0 +1,201 @@ +use super::ChannelRoutingDto; +use crate::{ + defaults::is_false, + utils::{is_blank_optional_str, is_blank_optional_string}, +}; + +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TelegramMessagingConfigDto { + pub bot_token: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub chat_ids: Vec, + #[serde(default, skip_serializing_if = "is_false")] + pub markdown: bool, + /// Template per event, keyed by event id wire name. + /// + /// Legacy `MsgKind` names (`info`, `stats`, `disk_alert`, ...) and + /// canonical dotted ids (`recording.completed`) are both accepted; + /// `MessagingConfig::prepare` resolves them and warns on a key that + /// matches no known event. + #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] + pub templates: std::collections::HashMap, + /// Per-channel routing. Inherits the global `notify_on` when absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub routing: Option, +} + +impl TelegramMessagingConfigDto { + pub fn is_empty(&self) -> bool { + self.bot_token.trim().is_empty() && self.chat_ids.is_empty() && self.templates.is_empty() + } +} + +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct RestMessagingConfigDto { + pub url: String, + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub method: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub headers: Vec, + /// HMAC-SHA256 signing secret. + /// + /// When set, each request carries `X-Tuliprox-Timestamp` and + /// `X-Tuliprox-Signature: sha256=` over `timestamp.body`, so the + /// receiving endpoint can verify the sender. + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub signing_secret: Option, + /// Template per event, keyed by event id wire name. + /// + /// Legacy `MsgKind` names (`info`, `stats`, `disk_alert`, ...) and + /// canonical dotted ids (`recording.completed`) are both accepted; + /// `MessagingConfig::prepare` resolves them and warns on a key that + /// matches no known event. + #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] + pub templates: std::collections::HashMap, + /// Per-channel routing. Inherits the global `notify_on` when absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub routing: Option, +} + +impl RestMessagingConfigDto { + pub fn is_empty(&self) -> bool { + self.url.trim().is_empty() + && is_blank_optional_str(self.method.as_deref()) + && self.headers.is_empty() + && self.templates.is_empty() + } +} + +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct DiscordMessagingConfigDto { + pub url: String, + /// Template per event, keyed by event id wire name. + /// + /// Legacy `MsgKind` names (`info`, `stats`, `disk_alert`, ...) and + /// canonical dotted ids (`recording.completed`) are both accepted; + /// `MessagingConfig::prepare` resolves them and warns on a key that + /// matches no known event. + #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] + pub templates: std::collections::HashMap, + /// Per-channel routing. Inherits the global `notify_on` when absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub routing: Option, +} + +impl DiscordMessagingConfigDto { + pub fn is_empty(&self) -> bool { self.url.trim().is_empty() && self.templates.is_empty() } +} + +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct PushoverMessagingConfigDto { + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub url: Option, + pub token: String, + pub user: String, + /// Template per event, keyed by event id wire name. + /// + /// Pushover previously had no template support at all, so every + /// notification took the built-in text - which for watch changes and + /// playlist stats was a raw `serde_json` dump pushed to a phone. + #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] + pub templates: std::collections::HashMap, + /// Per-channel routing. Inherits the global `notify_on` when absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub routing: Option, +} + +impl PushoverMessagingConfigDto { + pub fn is_empty(&self) -> bool { + is_blank_optional_str(self.url.as_deref()) + && self.token.trim().is_empty() + && self.user.trim().is_empty() + && self.templates.is_empty() + } +} + +/// [ntfy](https://ntfy.sh) - self-hosted push, no account, no bot token. +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct NtfyMessagingConfigDto { + /// Server base URL, e.g. `https://ntfy.sh`. + pub url: String, + /// Topic to publish to. + pub topic: String, + /// Bearer token for a protected topic. + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub token: Option, + #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] + pub templates: std::collections::HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub routing: Option, +} + +impl NtfyMessagingConfigDto { + pub fn is_empty(&self) -> bool { self.url.trim().is_empty() && self.topic.trim().is_empty() } +} + +/// [Gotify](https://gotify.net) - same audience as ntfy, same shape. +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct GotifyMessagingConfigDto { + /// Server base URL. + pub url: String, + /// Application token. + pub token: String, + #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] + pub templates: std::collections::HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub routing: Option, +} + +impl GotifyMessagingConfigDto { + pub fn is_empty(&self) -> bool { self.url.trim().is_empty() && self.token.trim().is_empty() } +} + +/// Slack incoming webhook. +/// +/// Not a Discord clone: Block Kit differs enough from Discord embeds that +/// reusing the Discord payload shape produces bad output. +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SlackMessagingConfigDto { + pub url: String, + #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] + pub templates: std::collections::HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub routing: Option, +} + +impl SlackMessagingConfigDto { + pub fn is_empty(&self) -> bool { self.url.trim().is_empty() } +} + +/// Run a local program, with the event JSON on stdin. +/// +/// The escape hatch that means nobody has to wait for a channel to be +/// added upstream. This runs arbitrary code as the tuliprox process user, +/// so it is opt-in and never configured by default. +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct CommandMessagingConfigDto { + /// Program to run. Not passed through a shell, so no quoting rules and + /// no shell injection. + pub program: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub args: Vec, + /// Kill the child after this many seconds. Defaults to 30. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_secs: Option, + #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] + pub templates: std::collections::HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub routing: Option, +} + +impl CommandMessagingConfigDto { + pub fn is_empty(&self) -> bool { self.program.trim().is_empty() } +} diff --git a/shared/src/model/config/messaging/routing.rs b/shared/src/model/config/messaging/routing.rs new file mode 100644 index 000000000..0eed341a2 --- /dev/null +++ b/shared/src/model/config/messaging/routing.rs @@ -0,0 +1,180 @@ +use super::MessagingConfigDto; +use crate::{ + defaults::{default_critical_percent, default_repeat_interval_secs, default_warn_percent}, + error::TuliproxError, + utils::{is_blank_optional_str, is_blank_optional_string}, +}; + +/// Per-channel routing. +/// +/// Without this every enabled event went to every configured channel, so +/// "critical to Pushover, everything to Discord" was not expressible. +/// Every field is optional and an omitted block inherits the global +/// `notify_on`. +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ChannelRoutingDto { + /// Overrides the global `notify_on` for this channel. Same glob + /// grammar. Empty means inherit. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub notify_on: Vec, + /// Drop anything below this severity: `info`, `warn`, `error`, + /// `critical`. + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub min_severity: Option, + /// `HH:MM-HH:MM` local time. Notifications inside the window are + /// deferred by the outbox, never dropped - an overnight outage must not + /// be silently invisible. + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub quiet_hours: Option, + /// Circuit breaker. On reaching it the channel sends one "suppressing + /// further notifications" message and then goes quiet for the rest of + /// the hour. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_per_hour: Option, + /// Suppress a repeated `dedup_key` for this many seconds. Generalizes + /// the disk alert's `repeat_interval_secs`, which was only available to + /// disk alerts. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dedup_window_secs: Option, +} + +impl ChannelRoutingDto { + pub fn is_empty(&self) -> bool { + self.notify_on.is_empty() + && is_blank_optional_str(self.min_severity.as_deref()) + && is_blank_optional_str(self.quiet_hours.as_deref()) + && self.max_per_hour.is_none() + && self.dedup_window_secs.is_none() + } + + /// Reject values that would otherwise fail silently at send time. + pub fn prepare(&self) -> Result<(), TuliproxError> { + if let Some(severity) = self.min_severity.as_deref().filter(|s| !s.trim().is_empty()) { + if crate::model::notification::Severity::from_wire(severity).is_none() { + return Err(TuliproxError::Config(format!( + "messaging min_severity must be one of info, warn, error, critical; got `{severity}`" + ))); + } + } + if let Some(window) = self.quiet_hours.as_deref().filter(|s| !s.trim().is_empty()) { + if crate::model::notification::QuietHours::parse(window).is_none() { + return Err(TuliproxError::Config(format!( + "messaging quiet_hours must look like `23:00-07:00`; got `{window}`" + ))); + } + } + Ok(()) + } +} + +/// Configuration for threshold-driven disk-space alerts. +/// +/// Set `messaging.disk_alert` to enable notifications whenever the filesystem +/// hosting the process working directory crosses `warn_percent` or +/// `critical_percent`. The same alert is re-sent after `repeat_interval_secs` +/// while the disk stays in the same alert state. +/// +/// All fields are optional and the alert is disabled when the whole block is +/// absent. Defaults match the values used at runtime when fields are unset. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct DiskAlertConfigDto { + /// Threshold (0-100, exclusive) below which the disk is considered normal. + /// Defaults to `80.0`. + #[serde(default = "default_warn_percent")] + pub warn_percent: f64, + /// Threshold (0-100, inclusive) at which the disk is considered full. + /// Must be `> warn_percent`. Defaults to `95.0`. + #[serde(default = "default_critical_percent")] + pub critical_percent: f64, + /// Re-arm interval in seconds. While the disk stays in the same alert + /// state, the alert is re-sent after this many seconds. Defaults to + /// `3600` (1h). + #[serde(default = "default_repeat_interval_secs")] + pub repeat_interval_secs: u64, +} + +impl DiskAlertConfigDto { + pub fn is_empty(&self) -> bool { + self.warn_percent == default_warn_percent() + && self.critical_percent == default_critical_percent() + && self.repeat_interval_secs == default_repeat_interval_secs() + } +} + +impl Default for DiskAlertConfigDto { + fn default() -> Self { + Self { + warn_percent: default_warn_percent(), + critical_percent: default_critical_percent(), + repeat_interval_secs: default_repeat_interval_secs(), + } + } +} + +impl DiskAlertConfigDto { + /// Sanity-check the thresholds. The caller surfaces the error to the user. + pub fn prepare(&self) -> Result<(), TuliproxError> { + if !(0.0..=100.0).contains(&self.warn_percent) { + return Err(TuliproxError::Config(format!( + "messaging.disk_alert.warn_percent must be in [0, 100], got {}", + self.warn_percent + ))); + } + if !(0.0..=100.0).contains(&self.critical_percent) { + return Err(TuliproxError::Config(format!( + "messaging.disk_alert.critical_percent must be in [0, 100], got {}", + self.critical_percent + ))); + } + if self.critical_percent <= self.warn_percent { + return Err(TuliproxError::Config(format!( + "messaging.disk_alert.critical_percent ({}) must be > warn_percent ({})", + self.critical_percent, self.warn_percent + ))); + } + Ok(()) + } +} + +impl MessagingConfigDto { + pub fn prepare(&mut self, _include_computed: bool) -> Result<(), TuliproxError> { + if let Some(disk) = &self.disk_alert { + disk.prepare()?; + } + for routing in [ + self.telegram.as_ref().and_then(|c| c.routing.as_ref()), + self.rest.as_ref().and_then(|c| c.routing.as_ref()), + self.discord.as_ref().and_then(|c| c.routing.as_ref()), + self.pushover.as_ref().and_then(|c| c.routing.as_ref()), + self.ntfy.as_ref().and_then(|c| c.routing.as_ref()), + self.gotify.as_ref().and_then(|c| c.routing.as_ref()), + self.slack.as_ref().and_then(|c| c.routing.as_ref()), + self.command.as_ref().and_then(|c| c.routing.as_ref()), + ] + .into_iter() + .flatten() + { + routing.prepare()?; + } + self.normalize_notify_on(); + Ok(()) + } + + /// Rewrite legacy `MsgKind` names to their canonical event ids. + /// + /// Keeps the UI's selection state consistent with the registry-driven + /// option list, and migrates an old config the next time it is saved. + /// Glob patterns are left exactly as written. + pub(super) fn normalize_notify_on(&mut self) { + for entry in &mut self.notify_on { + if entry.contains('*') || entry.starts_with('!') { + continue; + } + if let Some(id) = crate::model::notification::EventId::from_wire(entry) { + *entry = id.as_str().to_string(); + } + } + } +} diff --git a/shared/src/model/config/messaging/secrets.rs b/shared/src/model/config/messaging/secrets.rs new file mode 100644 index 000000000..496867e70 --- /dev/null +++ b/shared/src/model/config/messaging/secrets.rs @@ -0,0 +1,98 @@ +use super::{MessagingConfigDto, REDACTED_SECRET}; + +fn redact(value: &mut String) { + if !value.trim().is_empty() { + *value = REDACTED_SECRET.to_string(); + } +} + +fn restore(incoming: &mut String, stored: &str) { + if incoming.trim() == REDACTED_SECRET { + *incoming = stored.to_string(); + } +} + +/// Header names whose value is a credential. +fn is_sensitive_header(name: &str) -> bool { + const SENSITIVE: &[&str] = &["authorization", "x-api-key", "x-auth-token", "proxy-authorization", "cookie"]; + let name = name.trim(); + SENSITIVE.iter().any(|candidate| name.eq_ignore_ascii_case(candidate)) +} + +impl MessagingConfigDto { + /// Replace channel secrets with [`REDACTED_SECRET`]. + /// + /// The config GET returns `config.yml` in full to any client with + /// `ConfigRead`, which included the Telegram bot token, the Pushover + /// token and user key, and any `Authorization` header configured on the + /// REST channel. + pub fn redact_secrets(&mut self) { + if let Some(telegram) = self.telegram.as_mut() { + redact(&mut telegram.bot_token); + } + if let Some(pushover) = self.pushover.as_mut() { + redact(&mut pushover.token); + redact(&mut pushover.user); + } + if let Some(ntfy) = self.ntfy.as_mut() { + if let Some(token) = ntfy.token.as_mut() { + redact(token); + } + } + if let Some(gotify) = self.gotify.as_mut() { + redact(&mut gotify.token); + } + if let Some(rest) = self.rest.as_mut() { + if let Some(secret) = rest.signing_secret.as_mut() { + redact(secret); + } + for header in &mut rest.headers { + // `Name: value` - keep the name, mask the value. + if let Some((name, _)) = header.split_once(':') { + if is_sensitive_header(name) { + *header = format!("{name}: {REDACTED_SECRET}"); + } + } + } + } + } + + /// Put back any secret the client returned still redacted. + /// + /// Without this a round-trip through the UI would overwrite the stored + /// token with the mask, silently breaking the channel. + pub fn restore_redacted_secrets(&mut self, current: &Self) { + if let (Some(incoming), Some(stored)) = (self.telegram.as_mut(), current.telegram.as_ref()) { + restore(&mut incoming.bot_token, &stored.bot_token); + } + if let (Some(incoming), Some(stored)) = (self.pushover.as_mut(), current.pushover.as_ref()) { + restore(&mut incoming.token, &stored.token); + restore(&mut incoming.user, &stored.user); + } + if let (Some(incoming), Some(stored)) = (self.ntfy.as_mut(), current.ntfy.as_ref()) { + if let (Some(token), Some(original)) = (incoming.token.as_mut(), stored.token.as_ref()) { + restore(token, original); + } + } + if let (Some(incoming), Some(stored)) = (self.gotify.as_mut(), current.gotify.as_ref()) { + restore(&mut incoming.token, &stored.token); + } + if let (Some(incoming), Some(stored)) = (self.rest.as_mut(), current.rest.as_ref()) { + if let (Some(secret), Some(original)) = (incoming.signing_secret.as_mut(), stored.signing_secret.as_ref()) { + restore(secret, original); + } + for header in &mut incoming.headers { + let Some((name, value)) = header.split_once(':') else { continue }; + if value.trim() != REDACTED_SECRET { + continue; + } + let name = name.to_string(); + if let Some(original) = + stored.headers.iter().find(|h| h.split_once(':').is_some_and(|(n, _)| n.trim() == name.trim())) + { + *header = original.clone(); + } + } + } + } +} diff --git a/shared/src/model/config/messaging/tests.rs b/shared/src/model/config/messaging/tests.rs new file mode 100644 index 000000000..04b75afeb --- /dev/null +++ b/shared/src/model/config/messaging/tests.rs @@ -0,0 +1,151 @@ +use super::*; + +#[test] +fn clean_preserves_disk_alert_with_default_values() { + let mut messaging = MessagingConfigDto { disk_alert: Some(DiskAlertConfigDto::default()), ..Default::default() }; + messaging.clean(); + assert_eq!(messaging.disk_alert, None); +} + +#[test] +fn clean_preserves_disk_alert_with_non_default_values() { + let custom = DiskAlertConfigDto { warn_percent: 70.0, critical_percent: 90.0, repeat_interval_secs: 600 }; + let mut messaging = MessagingConfigDto { disk_alert: Some(custom.clone()), ..Default::default() }; + messaging.clean(); + assert_eq!(messaging.disk_alert, Some(custom)); +} + +#[test] +fn clean_strips_empty_subconfigs() { + let mut messaging = MessagingConfigDto { + telegram: Some(TelegramMessagingConfigDto::default()), + rest: Some(RestMessagingConfigDto::default()), + pushover: Some(PushoverMessagingConfigDto::default()), + discord: Some(DiscordMessagingConfigDto::default()), + ..Default::default() + }; + messaging.clean(); + assert!(messaging.telegram.is_none()); + assert!(messaging.rest.is_none()); + assert!(messaging.pushover.is_none()); + assert!(messaging.discord.is_none()); +} + +#[test] +fn prepare_rewrites_legacy_notify_on_names_to_canonical_ids() { + let mut messaging = MessagingConfigDto { + notify_on: vec!["disk_alert".to_string(), "recording_completed".to_string()], + ..Default::default() + }; + messaging.prepare(false).expect("prepare"); + assert_eq!(messaging.notify_on, vec!["system.disk.alert", "recording.completed"]); +} + +#[test] +fn prepare_leaves_glob_patterns_untouched() { + let mut messaging = MessagingConfigDto { + notify_on: vec!["recording.*".to_string(), "!system.info".to_string(), "*".to_string()], + ..Default::default() + }; + messaging.prepare(false).expect("prepare"); + assert_eq!(messaging.notify_on, vec!["recording.*", "!system.info", "*"]); +} + +#[test] +fn redaction_masks_every_channel_secret() { + let mut messaging = MessagingConfigDto { + telegram: Some(TelegramMessagingConfigDto { bot_token: "real-token".to_string(), ..Default::default() }), + pushover: Some(PushoverMessagingConfigDto { + token: "real-app".to_string(), + user: "real-user".to_string(), + ..Default::default() + }), + rest: Some(RestMessagingConfigDto { + url: "https://example.test".to_string(), + headers: vec!["Authorization: Bearer secret".to_string(), "X-Trace: keep-me".to_string()], + ..Default::default() + }), + ..Default::default() + }; + messaging.redact_secrets(); + assert_eq!(messaging.telegram.as_ref().expect("telegram").bot_token, REDACTED_SECRET); + assert_eq!(messaging.pushover.as_ref().expect("pushover").token, REDACTED_SECRET); + assert_eq!(messaging.pushover.as_ref().expect("pushover").user, REDACTED_SECRET); + let headers = &messaging.rest.as_ref().expect("rest").headers; + assert!(headers.contains(&format!("Authorization: {REDACTED_SECRET}")), "auth header not masked: {headers:?}"); + // A non-credential header keeps its value; masking everything would + // make the config unreadable for no gain. + assert!(headers.contains(&"X-Trace: keep-me".to_string()), "harmless header was masked: {headers:?}"); +} + +#[test] +fn an_empty_secret_is_not_replaced_by_a_mask() { + // Otherwise an unconfigured channel would look configured. + let mut messaging = + MessagingConfigDto { telegram: Some(TelegramMessagingConfigDto::default()), ..Default::default() }; + messaging.redact_secrets(); + assert_eq!(messaging.telegram.as_ref().expect("telegram").bot_token, ""); +} + +#[test] +fn a_returned_mask_restores_the_stored_secret() { + // Without this the UI round-trip would overwrite the real token with + // the mask and silently break the channel. + let stored = MessagingConfigDto { + telegram: Some(TelegramMessagingConfigDto { bot_token: "real-token".to_string(), ..Default::default() }), + ..Default::default() + }; + let mut incoming = MessagingConfigDto { + telegram: Some(TelegramMessagingConfigDto { bot_token: REDACTED_SECRET.to_string(), ..Default::default() }), + ..Default::default() + }; + incoming.restore_redacted_secrets(&stored); + assert_eq!(incoming.telegram.as_ref().expect("telegram").bot_token, "real-token"); +} + +#[test] +fn a_genuinely_changed_secret_is_written_through() { + let stored = MessagingConfigDto { + telegram: Some(TelegramMessagingConfigDto { bot_token: "old".to_string(), ..Default::default() }), + ..Default::default() + }; + let mut incoming = MessagingConfigDto { + telegram: Some(TelegramMessagingConfigDto { bot_token: "new".to_string(), ..Default::default() }), + ..Default::default() + }; + incoming.restore_redacted_secrets(&stored); + assert_eq!(incoming.telegram.as_ref().expect("telegram").bot_token, "new"); +} + +#[test] +fn a_returned_masked_header_restores_the_stored_one() { + let stored = MessagingConfigDto { + rest: Some(RestMessagingConfigDto { + url: "https://example.test".to_string(), + headers: vec!["Authorization: Bearer secret".to_string()], + ..Default::default() + }), + ..Default::default() + }; + let mut incoming = MessagingConfigDto { + rest: Some(RestMessagingConfigDto { + url: "https://example.test".to_string(), + headers: vec![format!("Authorization: {REDACTED_SECRET}")], + ..Default::default() + }), + ..Default::default() + }; + incoming.restore_redacted_secrets(&stored); + assert_eq!(incoming.rest.as_ref().expect("rest").headers, vec!["Authorization: Bearer secret".to_string()]); +} + +#[test] +fn is_empty_default_messaging() { + assert!(MessagingConfigDto::default().is_empty()); +} + +#[test] +fn is_empty_with_only_notify_on_is_not_empty() { + let messaging = MessagingConfigDto { notify_on: vec!["disk_alert".to_string()], ..Default::default() }; + assert!(!messaging.is_empty()); +} diff --git a/shared/src/model/config/metadata_update.rs b/shared/src/model/config/metadata_update.rs index efa33a314..3900e62e5 100644 --- a/shared/src/model/config/metadata_update.rs +++ b/shared/src/model/config/metadata_update.rs @@ -1,47 +1,20 @@ -use crate::{ - defaults::{ - default_metadata_backoff_jitter_percent, default_metadata_ffprobe_analyze_duration, - default_metadata_ffprobe_live_analyze_duration, default_metadata_ffprobe_live_probe_size, - default_metadata_ffprobe_probe_size, default_metadata_max_attempts_probe, - default_metadata_max_attempts_resolve, default_metadata_max_queue_size, - default_metadata_max_resolve_retry_backoff, default_metadata_no_change_cache_ttl_secs, default_metadata_path, - default_metadata_probe_cooldown, default_metadata_probe_fairness_resolve_burst, - default_metadata_probe_retry_backoff_step_1, default_metadata_probe_retry_backoff_step_2, - default_metadata_probe_retry_backoff_step_3, default_metadata_probe_retry_load_retry_delay, - default_metadata_progress_log_interval, default_metadata_queue_log_interval, - default_metadata_resolve_exhaustion_reset_gap, default_metadata_resolve_min_retry_base, - default_metadata_retry_delay, default_metadata_tmdb_cooldown, default_metadata_worker_idle_timeout, - default_probe_user_priority, default_tmdb_api_key, default_tmdb_cache_duration_days, default_tmdb_language, - default_tmdb_match_threshold, default_tmdb_rate_limit_ms, is_default_metadata_backoff_jitter_percent, - is_default_metadata_ffprobe_analyze_duration, is_default_metadata_ffprobe_live_analyze_duration, - is_default_metadata_ffprobe_live_probe_size, is_default_metadata_ffprobe_probe_size, - is_default_metadata_max_attempts_probe, is_default_metadata_max_attempts_resolve, - is_default_metadata_max_queue_size, is_default_metadata_max_resolve_retry_backoff, - is_default_metadata_no_change_cache_ttl_secs, is_default_metadata_path, is_default_metadata_probe_cooldown, - is_default_metadata_probe_fairness_resolve_burst, is_default_metadata_probe_retry_backoff_step_1, - is_default_metadata_probe_retry_backoff_step_2, is_default_metadata_probe_retry_backoff_step_3, - is_default_metadata_probe_retry_load_retry_delay, is_default_metadata_progress_log_interval, - is_default_metadata_queue_log_interval, is_default_metadata_resolve_exhaustion_reset_gap, - is_default_metadata_resolve_min_retry_base, is_default_metadata_retry_delay, is_default_metadata_tmdb_cooldown, - is_default_metadata_worker_idle_timeout, is_default_probe_user_priority, is_default_tmdb_cache_duration_days, - is_default_tmdb_language, is_default_tmdb_match_threshold, is_default_tmdb_rate_limit_ms, is_false, - is_tmdb_default_api_key, TMDB_API_KEY, - }, - error::TuliproxError, - model::ByteSize, - utils::{deserialize_as_string, parse_duration_seconds}, +use crate::defaults::{ + default_metadata_max_queue_size, default_metadata_no_change_cache_ttl_secs, default_metadata_path, + default_metadata_probe_fairness_resolve_burst, default_metadata_retry_delay, default_metadata_worker_idle_timeout, + is_default_metadata_max_queue_size, is_default_metadata_no_change_cache_ttl_secs, is_default_metadata_path, + is_default_metadata_probe_fairness_resolve_burst, is_default_metadata_retry_delay, + is_default_metadata_worker_idle_timeout, }; const MIN_DURATION_SECS: u64 = 1; -const MIN_ATTEMPTS: u8 = 1; -const MAX_JITTER_PERCENT: u8 = 95; -const MIN_QUEUE_SIZE: usize = 1; -const DEFAULT_FFPROBE_TIMEOUT_SECS: u64 = 60; -fn default_ffprobe_timeout_secs() -> Option { Some(DEFAULT_FFPROBE_TIMEOUT_SECS) } -fn is_default_ffprobe_timeout(timeout: &Option) -> bool { - timeout.is_none_or(|value| value == DEFAULT_FFPROBE_TIMEOUT_SECS) -} +const MIN_ATTEMPTS: u8 = 1; + +const MAX_JITTER_PERCENT: u8 = 95; + +const MIN_QUEUE_SIZE: usize = 1; + +const DEFAULT_FFPROBE_TIMEOUT_SECS: u64 = 60; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] @@ -97,635 +70,11 @@ impl Default for MetadataUpdateConfigDto { } } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct MetadataLogConfigDto { - #[serde( - default = "default_metadata_queue_log_interval", - skip_serializing_if = "is_default_metadata_queue_log_interval" - )] - pub queue_interval: String, - #[serde( - default = "default_metadata_progress_log_interval", - skip_serializing_if = "is_default_metadata_progress_log_interval" - )] - pub progress_interval: String, -} - -impl Default for MetadataLogConfigDto { - fn default() -> Self { - Self { - queue_interval: default_metadata_queue_log_interval(), - progress_interval: default_metadata_progress_log_interval(), - } - } -} - -impl MetadataLogConfigDto { - pub fn is_empty(&self) -> bool { - self.queue_interval == default_metadata_queue_log_interval() - && self.progress_interval == default_metadata_progress_log_interval() - } - - fn prepare(&mut self) -> Result<(), TuliproxError> { - let queue_interval_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( - &self.queue_interval, - MIN_DURATION_SECS, - "log.queue_interval", - )?; - self.queue_interval = MetadataUpdateConfigDto::canonicalize_seconds(queue_interval_secs); - - let progress_interval_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( - &self.progress_interval, - MIN_DURATION_SECS, - "log.progress_interval", - )?; - self.progress_interval = MetadataUpdateConfigDto::canonicalize_seconds(progress_interval_secs); - - Ok(()) - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ResolveConfigDto { - #[serde( - default = "default_metadata_max_resolve_retry_backoff", - skip_serializing_if = "is_default_metadata_max_resolve_retry_backoff" - )] - pub max_retry_backoff: String, - #[serde( - default = "default_metadata_resolve_min_retry_base", - skip_serializing_if = "is_default_metadata_resolve_min_retry_base" - )] - pub min_retry_base: String, - #[serde( - default = "default_metadata_resolve_exhaustion_reset_gap", - skip_serializing_if = "is_default_metadata_resolve_exhaustion_reset_gap" - )] - pub exhaustion_reset_gap: String, - #[serde( - default = "default_metadata_max_attempts_resolve", - skip_serializing_if = "is_default_metadata_max_attempts_resolve" - )] - pub max_attempts: u8, -} - -impl Default for ResolveConfigDto { - fn default() -> Self { - Self { - max_retry_backoff: default_metadata_max_resolve_retry_backoff(), - min_retry_base: default_metadata_resolve_min_retry_base(), - exhaustion_reset_gap: default_metadata_resolve_exhaustion_reset_gap(), - max_attempts: default_metadata_max_attempts_resolve(), - } - } -} - -impl ResolveConfigDto { - pub fn is_empty(&self) -> bool { - self.max_retry_backoff == default_metadata_max_resolve_retry_backoff() - && self.min_retry_base == default_metadata_resolve_min_retry_base() - && self.exhaustion_reset_gap == default_metadata_resolve_exhaustion_reset_gap() - && self.max_attempts == default_metadata_max_attempts_resolve() - } - - fn prepare(&mut self) -> Result<(), TuliproxError> { - let max_retry_backoff_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( - &self.max_retry_backoff, - MIN_DURATION_SECS, - "resolve.max_retry_backoff", - )?; - self.max_retry_backoff = MetadataUpdateConfigDto::canonicalize_seconds(max_retry_backoff_secs); - - let min_retry_base_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( - &self.min_retry_base, - MIN_DURATION_SECS, - "resolve.min_retry_base", - )?; - self.min_retry_base = MetadataUpdateConfigDto::canonicalize_seconds(min_retry_base_secs); - - let exhaustion_reset_gap_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( - &self.exhaustion_reset_gap, - MIN_DURATION_SECS, - "resolve.exhaustion_reset_gap", - )?; - self.exhaustion_reset_gap = MetadataUpdateConfigDto::canonicalize_seconds(exhaustion_reset_gap_secs); - - self.max_attempts = self.max_attempts.max(MIN_ATTEMPTS); - - Ok(()) - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ProbeConfigDto { - #[serde(default = "default_metadata_probe_cooldown", skip_serializing_if = "is_default_metadata_probe_cooldown")] - pub cooldown: String, - #[serde( - default = "default_metadata_probe_retry_load_retry_delay", - skip_serializing_if = "is_default_metadata_probe_retry_load_retry_delay" - )] - pub retry_load_retry_delay: String, - #[serde( - default = "default_metadata_probe_retry_backoff_step_1", - skip_serializing_if = "is_default_metadata_probe_retry_backoff_step_1" - )] - pub retry_backoff_step_1: String, - #[serde( - default = "default_metadata_probe_retry_backoff_step_2", - skip_serializing_if = "is_default_metadata_probe_retry_backoff_step_2" - )] - pub retry_backoff_step_2: String, - #[serde( - default = "default_metadata_probe_retry_backoff_step_3", - skip_serializing_if = "is_default_metadata_probe_retry_backoff_step_3" - )] - pub retry_backoff_step_3: String, - #[serde( - default = "default_metadata_max_attempts_probe", - skip_serializing_if = "is_default_metadata_max_attempts_probe" - )] - pub max_attempts: u8, - #[serde( - default = "default_metadata_backoff_jitter_percent", - skip_serializing_if = "is_default_metadata_backoff_jitter_percent" - )] - pub backoff_jitter_percent: u8, - #[serde(default = "default_probe_user_priority", skip_serializing_if = "is_default_probe_user_priority")] - pub user_priority: i8, -} - -impl Default for ProbeConfigDto { - fn default() -> Self { - Self { - cooldown: default_metadata_probe_cooldown(), - retry_load_retry_delay: default_metadata_probe_retry_load_retry_delay(), - retry_backoff_step_1: default_metadata_probe_retry_backoff_step_1(), - retry_backoff_step_2: default_metadata_probe_retry_backoff_step_2(), - retry_backoff_step_3: default_metadata_probe_retry_backoff_step_3(), - max_attempts: default_metadata_max_attempts_probe(), - backoff_jitter_percent: default_metadata_backoff_jitter_percent(), - user_priority: default_probe_user_priority(), - } - } -} - -impl ProbeConfigDto { - pub fn is_empty(&self) -> bool { - self.cooldown == default_metadata_probe_cooldown() - && self.retry_load_retry_delay == default_metadata_probe_retry_load_retry_delay() - && self.retry_backoff_step_1 == default_metadata_probe_retry_backoff_step_1() - && self.retry_backoff_step_2 == default_metadata_probe_retry_backoff_step_2() - && self.retry_backoff_step_3 == default_metadata_probe_retry_backoff_step_3() - && self.max_attempts == default_metadata_max_attempts_probe() - && self.backoff_jitter_percent == default_metadata_backoff_jitter_percent() - && self.user_priority == default_probe_user_priority() - } - - fn prepare(&mut self) -> Result<(), TuliproxError> { - let cooldown_secs = - MetadataUpdateConfigDto::parse_and_clamp_duration(&self.cooldown, MIN_DURATION_SECS, "probe.cooldown")?; - self.cooldown = MetadataUpdateConfigDto::canonicalize_seconds(cooldown_secs); - - let retry_load_retry_delay_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( - &self.retry_load_retry_delay, - MIN_DURATION_SECS, - "probe.retry_load_retry_delay", - )?; - self.retry_load_retry_delay = MetadataUpdateConfigDto::canonicalize_seconds(retry_load_retry_delay_secs); - - let retry_backoff_step_1_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( - &self.retry_backoff_step_1, - MIN_DURATION_SECS, - "probe.retry_backoff_step_1", - )?; - self.retry_backoff_step_1 = MetadataUpdateConfigDto::canonicalize_seconds(retry_backoff_step_1_secs); - - let retry_backoff_step_2_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( - &self.retry_backoff_step_2, - MIN_DURATION_SECS, - "probe.retry_backoff_step_2", - )?; - self.retry_backoff_step_2 = MetadataUpdateConfigDto::canonicalize_seconds(retry_backoff_step_2_secs); - - let retry_backoff_step_3_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( - &self.retry_backoff_step_3, - MIN_DURATION_SECS, - "probe.retry_backoff_step_3", - )?; - self.retry_backoff_step_3 = MetadataUpdateConfigDto::canonicalize_seconds(retry_backoff_step_3_secs); - - self.max_attempts = self.max_attempts.max(MIN_ATTEMPTS); - self.backoff_jitter_percent = self.backoff_jitter_percent.min(MAX_JITTER_PERCENT); - - Ok(()) - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct FfprobeConfigDto { - #[serde(default, skip_serializing_if = "is_false")] - pub enabled: bool, - #[serde(default = "default_ffprobe_timeout_secs", skip_serializing_if = "is_default_ffprobe_timeout")] - pub timeout: Option, - #[serde( - default = "default_metadata_ffprobe_analyze_duration", - skip_serializing_if = "is_default_metadata_ffprobe_analyze_duration", - deserialize_with = "deserialize_as_string" - )] - pub analyze_duration: String, - #[serde( - default = "default_metadata_ffprobe_probe_size", - skip_serializing_if = "is_default_metadata_ffprobe_probe_size" - )] - pub probe_size: ByteSize, - #[serde( - default = "default_metadata_ffprobe_live_analyze_duration", - skip_serializing_if = "is_default_metadata_ffprobe_live_analyze_duration", - deserialize_with = "deserialize_as_string" - )] - pub live_analyze_duration: String, - #[serde( - default = "default_metadata_ffprobe_live_probe_size", - skip_serializing_if = "is_default_metadata_ffprobe_live_probe_size" - )] - pub live_probe_size: ByteSize, -} - -impl Default for FfprobeConfigDto { - fn default() -> Self { - Self { - enabled: false, - timeout: default_ffprobe_timeout_secs(), - analyze_duration: default_metadata_ffprobe_analyze_duration(), - probe_size: default_metadata_ffprobe_probe_size(), - live_analyze_duration: default_metadata_ffprobe_live_analyze_duration(), - live_probe_size: default_metadata_ffprobe_live_probe_size(), - } - } -} - -impl FfprobeConfigDto { - pub fn is_empty(&self) -> bool { - !self.enabled - && is_default_ffprobe_timeout(&self.timeout) - && self.analyze_duration == default_metadata_ffprobe_analyze_duration() - && self.probe_size == default_metadata_ffprobe_probe_size() - && self.live_analyze_duration == default_metadata_ffprobe_live_analyze_duration() - && self.live_probe_size == default_metadata_ffprobe_live_probe_size() - } - - pub fn clean(&mut self) { - if self.timeout.is_none_or(|timeout| timeout == 0) { - self.timeout = default_ffprobe_timeout_secs(); - } - if self.analyze_duration.trim().is_empty() { - self.analyze_duration = default_metadata_ffprobe_analyze_duration(); - } - if self.probe_size.as_str().trim().is_empty() { - self.probe_size = default_metadata_ffprobe_probe_size(); - } - if self.live_analyze_duration.trim().is_empty() { - self.live_analyze_duration = default_metadata_ffprobe_live_analyze_duration(); - } - if self.live_probe_size.as_str().trim().is_empty() { - self.live_probe_size = default_metadata_ffprobe_live_probe_size(); - } - } - - fn prepare(&mut self) -> Result<(), TuliproxError> { - self.timeout = self.timeout.map(|timeout| timeout.max(MIN_DURATION_SECS)); - - let analyze_duration_secs = MetadataUpdateConfigDto::parse_and_clamp_duration_with_required_unit( - &self.analyze_duration, - MIN_DURATION_SECS, - "ffprobe.analyze_duration", - )?; - self.analyze_duration = MetadataUpdateConfigDto::canonicalize_seconds(analyze_duration_secs); - - let probe_size_bytes = self - .probe_size - .parse_bytes() - .map_err(|err| { - TuliproxError::ConfigMetadataUpdate(format!("Invalid size for `ffprobe.probe_size`: {err}")) - })? - .at_least_1(); - self.probe_size = ByteSize::new(MetadataUpdateConfigDto::canonicalize_size_bytes(probe_size_bytes.get())); - - let live_analyze_duration_secs = MetadataUpdateConfigDto::parse_and_clamp_duration_with_required_unit( - &self.live_analyze_duration, - MIN_DURATION_SECS, - "ffprobe.live_analyze_duration", - )?; - self.live_analyze_duration = MetadataUpdateConfigDto::canonicalize_seconds(live_analyze_duration_secs); - - let live_probe_size_bytes = self - .live_probe_size - .parse_bytes() - .map_err(|err| { - TuliproxError::ConfigMetadataUpdate(format!("Invalid size for `ffprobe.live_probe_size`: {err}")) - })? - .at_least_1(); - self.live_probe_size = - ByteSize::new(MetadataUpdateConfigDto::canonicalize_size_bytes(live_probe_size_bytes.get())); - - Ok(()) - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct TmdbConfigDto { - #[serde(default)] - pub enabled: bool, - #[serde(default = "default_tmdb_api_key", skip_serializing_if = "is_tmdb_default_api_key")] - pub api_key: Option, - #[serde(default = "default_tmdb_rate_limit_ms", skip_serializing_if = "is_default_tmdb_rate_limit_ms")] - pub rate_limit_ms: u64, - #[serde(default = "default_tmdb_cache_duration_days", skip_serializing_if = "is_default_tmdb_cache_duration_days")] - pub cache_duration_days: u32, - #[serde(default = "default_tmdb_language", skip_serializing_if = "is_default_tmdb_language")] - pub language: String, - #[serde(default = "default_metadata_tmdb_cooldown", skip_serializing_if = "is_default_metadata_tmdb_cooldown")] - pub cooldown: String, - #[serde(default = "default_tmdb_match_threshold", skip_serializing_if = "is_default_tmdb_match_threshold")] - pub match_threshold: u16, -} - -impl Default for TmdbConfigDto { - fn default() -> Self { - Self { - enabled: false, - api_key: default_tmdb_api_key(), - rate_limit_ms: default_tmdb_rate_limit_ms(), - cache_duration_days: default_tmdb_cache_duration_days(), - language: default_tmdb_language(), - cooldown: default_metadata_tmdb_cooldown(), - match_threshold: default_tmdb_match_threshold(), - } - } -} - -impl TmdbConfigDto { - pub fn is_empty(&self) -> bool { - !self.enabled - && self.api_key.as_ref().is_none_or(|api_key| api_key == TMDB_API_KEY) - && self.rate_limit_ms == default_tmdb_rate_limit_ms() - && self.cache_duration_days == default_tmdb_cache_duration_days() - && self.language == default_tmdb_language() - && self.cooldown == default_metadata_tmdb_cooldown() - && self.match_threshold == default_tmdb_match_threshold() - } - - fn clean(&mut self) { - self.api_key = self.api_key.take().and_then(|api_key| { - let trimmed = api_key.trim(); - if trimmed.is_empty() || trimmed == TMDB_API_KEY { - None - } else { - Some(trimmed.to_string()) - } - }); - } - - fn prepare(&mut self) -> Result<(), TuliproxError> { - let cooldown_secs = - MetadataUpdateConfigDto::parse_and_clamp_duration(&self.cooldown, MIN_DURATION_SECS, "tmdb.cooldown")?; - self.cooldown = MetadataUpdateConfigDto::canonicalize_seconds(cooldown_secs); - self.match_threshold = self.match_threshold.clamp(0, 100); - Ok(()) - } -} - -impl MetadataUpdateConfigDto { - pub fn is_empty(&self) -> bool { - is_default_metadata_path(&self.cache_path) - && self.log.is_empty() - && self.resolve.is_empty() - && self.probe.is_empty() - && self.ffprobe.is_empty() - && self.tmdb.is_empty() - && self.retry_delay == default_metadata_retry_delay() - && self.worker_idle_timeout == default_metadata_worker_idle_timeout() - && self.max_queue_size == default_metadata_max_queue_size() - && self.no_change_cache_ttl_secs == default_metadata_no_change_cache_ttl_secs() - && self.probe_fairness_resolve_burst == default_metadata_probe_fairness_resolve_burst() - } - - pub fn clean(&mut self) { - if self.cache_path.trim().is_empty() { - self.cache_path = default_metadata_path(); - } - self.ffprobe.clean(); - self.tmdb.clean(); - } - - pub fn prepare(&mut self) -> Result<(), TuliproxError> { - if self.cache_path.trim().is_empty() { - return Err(TuliproxError::ConfigMetadataUpdate("metadata_update.cache_path cannot be empty".to_string())); - } - self.log.prepare()?; - self.resolve.prepare()?; - self.probe.prepare()?; - self.ffprobe.prepare()?; - self.tmdb.prepare()?; - - let retry_delay_secs = Self::parse_and_clamp_duration(&self.retry_delay, MIN_DURATION_SECS, "retry_delay")?; - self.retry_delay = Self::canonicalize_seconds(retry_delay_secs); - - let worker_idle_timeout_secs = - Self::parse_and_clamp_duration(&self.worker_idle_timeout, MIN_DURATION_SECS, "worker_idle_timeout")?; - self.worker_idle_timeout = Self::canonicalize_seconds(worker_idle_timeout_secs); - - self.max_queue_size = self.max_queue_size.max(MIN_QUEUE_SIZE); - self.no_change_cache_ttl_secs = self.no_change_cache_ttl_secs.max(MIN_DURATION_SECS); - self.probe_fairness_resolve_burst = self.probe_fairness_resolve_burst.max(MIN_QUEUE_SIZE); - - self.clean(); - - Ok(()) - } - - fn parse_and_clamp_duration(value: &str, min_seconds: u64, field_name: &str) -> Result { - let parsed = Self::parse_duration(value, field_name)?; - Ok(parsed.max(min_seconds)) - } - - fn parse_and_clamp_duration_with_required_unit( - value: &str, - min_seconds: u64, - field_name: &str, - ) -> Result { - let parsed = Self::parse_duration_with_required_unit(value, field_name)?; - Ok(parsed.max(min_seconds)) - } - - fn parse_duration_with_required_unit(value: &str, field_name: &str) -> Result { - if value.parse::().is_ok() { - return Err(TuliproxError::ConfigMetadataUpdate(format!( - "Invalid duration format for `{field_name}`: {value}. Use explicit unit suffix (`s`, `m`, `h`, `d`), e.g. `10s`." - ))); - } - Self::parse_duration(value, field_name) - } - - fn parse_duration(value: &str, field_name: &str) -> Result { - parse_duration_seconds(value, false).ok_or_else(|| { - crate::error::TuliproxError::ConfigMetadataUpdate(format!( - "Invalid duration format for `{field_name}`: {value}" - )) - }) - } - - fn canonicalize_seconds(seconds: u64) -> String { - if seconds.is_multiple_of(24 * 60 * 60) { - format!("{}d", seconds / (24 * 60 * 60)) - } else if seconds.is_multiple_of(60 * 60) { - format!("{}h", seconds / (60 * 60)) - } else if seconds.is_multiple_of(60) { - format!("{}m", seconds / 60) - } else { - format!("{seconds}s") - } - } - - fn canonicalize_size_bytes(bytes: u64) -> String { - if bytes.is_multiple_of(1_099_511_627_776) { - format!("{}TB", bytes / 1_099_511_627_776) - } else if bytes.is_multiple_of(1_073_741_824) { - format!("{}GB", bytes / 1_073_741_824) - } else if bytes.is_multiple_of(1_048_576) { - format!("{}MB", bytes / 1_048_576) - } else if bytes.is_multiple_of(1_024) { - format!("{}KB", bytes / 1_024) - } else { - format!("{bytes}B") - } - } -} - #[cfg(test)] -mod tests { - use super::{ByteSize, MetadataUpdateConfigDto}; +mod tests; - #[test] - fn default_config_is_empty() { - let cfg = MetadataUpdateConfigDto::default(); - assert!(cfg.is_empty()); - } - - #[test] - fn prepare_keeps_default_config_empty() { - let mut cfg = MetadataUpdateConfigDto::default(); - cfg.prepare().expect("metadata update config defaults should be valid"); - assert!(cfg.is_empty()); - } - - #[test] - fn prepare_parses_duration_suffixes() { - let mut cfg = MetadataUpdateConfigDto::default(); - cfg.log.queue_interval = "1m".to_string(); - cfg.log.progress_interval = "2h".to_string(); - cfg.probe.cooldown = "1d".to_string(); - cfg.tmdb.cooldown = "2d".to_string(); - - cfg.prepare().expect("metadata update config should parse duration values"); - - assert_eq!(cfg.log.queue_interval, "1m"); - assert_eq!(cfg.log.progress_interval, "2h"); - assert_eq!(cfg.probe.cooldown, "1d"); - assert_eq!(cfg.tmdb.cooldown, "2d"); - } - - #[test] - fn prepare_clamps_minimum_values() { - let mut cfg = MetadataUpdateConfigDto::default(); - cfg.log.queue_interval = "0".to_string(); - cfg.resolve.max_attempts = 0; - cfg.probe.max_attempts = 0; - cfg.max_queue_size = 0; - cfg.no_change_cache_ttl_secs = 0; - cfg.probe_fairness_resolve_burst = 0; - cfg.ffprobe.timeout = Some(0); - cfg.ffprobe.analyze_duration = "0s".to_string(); - cfg.ffprobe.probe_size = ByteSize::new("0"); - - cfg.prepare().expect("metadata update config should clamp minimum values"); - - assert_eq!(cfg.log.queue_interval, "1s"); - assert_eq!(cfg.resolve.max_attempts, 1); - assert_eq!(cfg.probe.max_attempts, 1); - assert_eq!(cfg.max_queue_size, 1); - assert_eq!(cfg.no_change_cache_ttl_secs, 1); - assert_eq!(cfg.probe_fairness_resolve_burst, 1); - assert_eq!(cfg.ffprobe.timeout, Some(1)); - assert_eq!(cfg.ffprobe.analyze_duration, "1s"); - assert_eq!(cfg.ffprobe.probe_size, ByteSize::new("1B")); - } - - #[test] - fn prepare_rejects_invalid_duration_unit() { - let mut cfg = MetadataUpdateConfigDto::default(); - cfg.log.queue_interval = "1w".to_string(); - - let result = cfg.prepare(); - assert!(result.is_err(), "invalid duration unit must fail"); - } - - #[test] - fn prepare_canonicalizes_to_larger_units() { - let mut cfg = MetadataUpdateConfigDto::default(); - cfg.probe.cooldown = "604800".to_string(); - cfg.tmdb.cooldown = "259200".to_string(); - cfg.worker_idle_timeout = "60".to_string(); - cfg.probe.retry_backoff_step_3 = "3600".to_string(); - cfg.ffprobe.analyze_duration = "10s".to_string(); - cfg.ffprobe.probe_size = ByteSize::new("10485760"); - cfg.ffprobe.live_analyze_duration = "5s".to_string(); - cfg.ffprobe.live_probe_size = ByteSize::new("5242880"); - - cfg.prepare().expect("metadata update config should canonicalize durations"); - - assert_eq!(cfg.probe.cooldown, "7d"); - assert_eq!(cfg.tmdb.cooldown, "3d"); - assert_eq!(cfg.worker_idle_timeout, "1m"); - assert_eq!(cfg.probe.retry_backoff_step_3, "1h"); - assert_eq!(cfg.ffprobe.analyze_duration, "10s"); - assert_eq!(cfg.ffprobe.probe_size, ByteSize::new("10MB")); - assert_eq!(cfg.ffprobe.live_analyze_duration, "5s"); - assert_eq!(cfg.ffprobe.live_probe_size, ByteSize::new("5MB")); - } - - #[test] - fn prepare_rejects_ffprobe_duration_without_unit() { - let mut cfg = MetadataUpdateConfigDto::default(); - cfg.ffprobe.analyze_duration = "10000000".to_string(); - - let result = cfg.prepare(); - assert!(result.is_err(), "numeric ffprobe analyze duration without unit must fail"); - let err_text = result.expect_err("validation should fail").to_string(); - assert!(err_text.contains("ffprobe.analyze_duration")); - } - - #[test] - fn tmdb_non_default_match_threshold_is_not_empty() { - let mut cfg = MetadataUpdateConfigDto::default(); - cfg.tmdb.match_threshold = 90; - cfg.prepare().expect("metadata update config should remain valid"); - - assert!(!cfg.tmdb.is_empty(), "tmdb config with non-default match threshold must not be empty"); - assert!(!cfg.is_empty(), "metadata update config with non-default tmdb match threshold must not be empty"); - } - - #[test] - fn prepare_clamps_tmdb_match_threshold() { - let mut cfg = MetadataUpdateConfigDto::default(); - cfg.tmdb.match_threshold = 250; - cfg.prepare().expect("metadata update config should clamp tmdb match threshold"); - - assert_eq!(cfg.tmdb.match_threshold, 100); - } -} +mod normalize; +mod policies; +mod providers; +pub use policies::{MetadataLogConfigDto, ProbeConfigDto, ResolveConfigDto}; +pub use providers::{FfprobeConfigDto, TmdbConfigDto}; diff --git a/shared/src/model/config/metadata_update/normalize.rs b/shared/src/model/config/metadata_update/normalize.rs new file mode 100644 index 000000000..cab0b4ddb --- /dev/null +++ b/shared/src/model/config/metadata_update/normalize.rs @@ -0,0 +1,121 @@ +use super::{MetadataUpdateConfigDto, MIN_DURATION_SECS, MIN_QUEUE_SIZE}; +use crate::{ + defaults::{ + default_metadata_max_queue_size, default_metadata_no_change_cache_ttl_secs, default_metadata_path, + default_metadata_probe_fairness_resolve_burst, default_metadata_retry_delay, + default_metadata_worker_idle_timeout, is_default_metadata_path, + }, + error::TuliproxError, + utils::parse_duration_seconds, +}; + +impl MetadataUpdateConfigDto { + pub fn is_empty(&self) -> bool { + is_default_metadata_path(&self.cache_path) + && self.log.is_empty() + && self.resolve.is_empty() + && self.probe.is_empty() + && self.ffprobe.is_empty() + && self.tmdb.is_empty() + && self.retry_delay == default_metadata_retry_delay() + && self.worker_idle_timeout == default_metadata_worker_idle_timeout() + && self.max_queue_size == default_metadata_max_queue_size() + && self.no_change_cache_ttl_secs == default_metadata_no_change_cache_ttl_secs() + && self.probe_fairness_resolve_burst == default_metadata_probe_fairness_resolve_burst() + } + + pub fn clean(&mut self) { + if self.cache_path.trim().is_empty() { + self.cache_path = default_metadata_path(); + } + self.ffprobe.clean(); + self.tmdb.clean(); + } + + pub fn prepare(&mut self) -> Result<(), TuliproxError> { + if self.cache_path.trim().is_empty() { + return Err(TuliproxError::ConfigMetadataUpdate("metadata_update.cache_path cannot be empty".to_string())); + } + self.log.prepare()?; + self.resolve.prepare()?; + self.probe.prepare()?; + self.ffprobe.prepare()?; + self.tmdb.prepare()?; + + let retry_delay_secs = Self::parse_and_clamp_duration(&self.retry_delay, MIN_DURATION_SECS, "retry_delay")?; + self.retry_delay = Self::canonicalize_seconds(retry_delay_secs); + + let worker_idle_timeout_secs = + Self::parse_and_clamp_duration(&self.worker_idle_timeout, MIN_DURATION_SECS, "worker_idle_timeout")?; + self.worker_idle_timeout = Self::canonicalize_seconds(worker_idle_timeout_secs); + + self.max_queue_size = self.max_queue_size.max(MIN_QUEUE_SIZE); + self.no_change_cache_ttl_secs = self.no_change_cache_ttl_secs.max(MIN_DURATION_SECS); + self.probe_fairness_resolve_burst = self.probe_fairness_resolve_burst.max(MIN_QUEUE_SIZE); + + self.clean(); + + Ok(()) + } + + pub(super) fn parse_and_clamp_duration( + value: &str, + min_seconds: u64, + field_name: &str, + ) -> Result { + let parsed = Self::parse_duration(value, field_name)?; + Ok(parsed.max(min_seconds)) + } + + pub(super) fn parse_and_clamp_duration_with_required_unit( + value: &str, + min_seconds: u64, + field_name: &str, + ) -> Result { + let parsed = Self::parse_duration_with_required_unit(value, field_name)?; + Ok(parsed.max(min_seconds)) + } + + pub(super) fn parse_duration_with_required_unit(value: &str, field_name: &str) -> Result { + if value.parse::().is_ok() { + return Err(TuliproxError::ConfigMetadataUpdate(format!( + "Invalid duration format for `{field_name}`: {value}. Use explicit unit suffix (`s`, `m`, `h`, `d`), e.g. `10s`." + ))); + } + Self::parse_duration(value, field_name) + } + + pub(super) fn parse_duration(value: &str, field_name: &str) -> Result { + parse_duration_seconds(value, false).ok_or_else(|| { + crate::error::TuliproxError::ConfigMetadataUpdate(format!( + "Invalid duration format for `{field_name}`: {value}" + )) + }) + } + + pub(super) fn canonicalize_seconds(seconds: u64) -> String { + if seconds.is_multiple_of(24 * 60 * 60) { + format!("{}d", seconds / (24 * 60 * 60)) + } else if seconds.is_multiple_of(60 * 60) { + format!("{}h", seconds / (60 * 60)) + } else if seconds.is_multiple_of(60) { + format!("{}m", seconds / 60) + } else { + format!("{seconds}s") + } + } + + pub(super) fn canonicalize_size_bytes(bytes: u64) -> String { + if bytes.is_multiple_of(1_099_511_627_776) { + format!("{}TB", bytes / 1_099_511_627_776) + } else if bytes.is_multiple_of(1_073_741_824) { + format!("{}GB", bytes / 1_073_741_824) + } else if bytes.is_multiple_of(1_048_576) { + format!("{}MB", bytes / 1_048_576) + } else if bytes.is_multiple_of(1_024) { + format!("{}KB", bytes / 1_024) + } else { + format!("{bytes}B") + } + } +} diff --git a/shared/src/model/config/metadata_update/policies.rs b/shared/src/model/config/metadata_update/policies.rs new file mode 100644 index 000000000..1c7c69d08 --- /dev/null +++ b/shared/src/model/config/metadata_update/policies.rs @@ -0,0 +1,247 @@ +use super::{MetadataUpdateConfigDto, MAX_JITTER_PERCENT, MIN_ATTEMPTS, MIN_DURATION_SECS}; +use crate::{ + defaults::{ + default_metadata_backoff_jitter_percent, default_metadata_max_attempts_probe, + default_metadata_max_attempts_resolve, default_metadata_max_resolve_retry_backoff, + default_metadata_probe_cooldown, default_metadata_probe_retry_backoff_step_1, + default_metadata_probe_retry_backoff_step_2, default_metadata_probe_retry_backoff_step_3, + default_metadata_probe_retry_load_retry_delay, default_metadata_progress_log_interval, + default_metadata_queue_log_interval, default_metadata_resolve_exhaustion_reset_gap, + default_metadata_resolve_min_retry_base, default_probe_user_priority, + is_default_metadata_backoff_jitter_percent, is_default_metadata_max_attempts_probe, + is_default_metadata_max_attempts_resolve, is_default_metadata_max_resolve_retry_backoff, + is_default_metadata_probe_cooldown, is_default_metadata_probe_retry_backoff_step_1, + is_default_metadata_probe_retry_backoff_step_2, is_default_metadata_probe_retry_backoff_step_3, + is_default_metadata_probe_retry_load_retry_delay, is_default_metadata_progress_log_interval, + is_default_metadata_queue_log_interval, is_default_metadata_resolve_exhaustion_reset_gap, + is_default_metadata_resolve_min_retry_base, is_default_probe_user_priority, + }, + error::TuliproxError, +}; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct MetadataLogConfigDto { + #[serde( + default = "default_metadata_queue_log_interval", + skip_serializing_if = "is_default_metadata_queue_log_interval" + )] + pub queue_interval: String, + #[serde( + default = "default_metadata_progress_log_interval", + skip_serializing_if = "is_default_metadata_progress_log_interval" + )] + pub progress_interval: String, +} + +impl Default for MetadataLogConfigDto { + fn default() -> Self { + Self { + queue_interval: default_metadata_queue_log_interval(), + progress_interval: default_metadata_progress_log_interval(), + } + } +} + +impl MetadataLogConfigDto { + pub fn is_empty(&self) -> bool { + self.queue_interval == default_metadata_queue_log_interval() + && self.progress_interval == default_metadata_progress_log_interval() + } + + pub(super) fn prepare(&mut self) -> Result<(), TuliproxError> { + let queue_interval_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.queue_interval, + MIN_DURATION_SECS, + "log.queue_interval", + )?; + self.queue_interval = MetadataUpdateConfigDto::canonicalize_seconds(queue_interval_secs); + + let progress_interval_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.progress_interval, + MIN_DURATION_SECS, + "log.progress_interval", + )?; + self.progress_interval = MetadataUpdateConfigDto::canonicalize_seconds(progress_interval_secs); + + Ok(()) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ResolveConfigDto { + #[serde( + default = "default_metadata_max_resolve_retry_backoff", + skip_serializing_if = "is_default_metadata_max_resolve_retry_backoff" + )] + pub max_retry_backoff: String, + #[serde( + default = "default_metadata_resolve_min_retry_base", + skip_serializing_if = "is_default_metadata_resolve_min_retry_base" + )] + pub min_retry_base: String, + #[serde( + default = "default_metadata_resolve_exhaustion_reset_gap", + skip_serializing_if = "is_default_metadata_resolve_exhaustion_reset_gap" + )] + pub exhaustion_reset_gap: String, + #[serde( + default = "default_metadata_max_attempts_resolve", + skip_serializing_if = "is_default_metadata_max_attempts_resolve" + )] + pub max_attempts: u8, +} + +impl Default for ResolveConfigDto { + fn default() -> Self { + Self { + max_retry_backoff: default_metadata_max_resolve_retry_backoff(), + min_retry_base: default_metadata_resolve_min_retry_base(), + exhaustion_reset_gap: default_metadata_resolve_exhaustion_reset_gap(), + max_attempts: default_metadata_max_attempts_resolve(), + } + } +} + +impl ResolveConfigDto { + pub fn is_empty(&self) -> bool { + self.max_retry_backoff == default_metadata_max_resolve_retry_backoff() + && self.min_retry_base == default_metadata_resolve_min_retry_base() + && self.exhaustion_reset_gap == default_metadata_resolve_exhaustion_reset_gap() + && self.max_attempts == default_metadata_max_attempts_resolve() + } + + pub(super) fn prepare(&mut self) -> Result<(), TuliproxError> { + let max_retry_backoff_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.max_retry_backoff, + MIN_DURATION_SECS, + "resolve.max_retry_backoff", + )?; + self.max_retry_backoff = MetadataUpdateConfigDto::canonicalize_seconds(max_retry_backoff_secs); + + let min_retry_base_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.min_retry_base, + MIN_DURATION_SECS, + "resolve.min_retry_base", + )?; + self.min_retry_base = MetadataUpdateConfigDto::canonicalize_seconds(min_retry_base_secs); + + let exhaustion_reset_gap_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.exhaustion_reset_gap, + MIN_DURATION_SECS, + "resolve.exhaustion_reset_gap", + )?; + self.exhaustion_reset_gap = MetadataUpdateConfigDto::canonicalize_seconds(exhaustion_reset_gap_secs); + + self.max_attempts = self.max_attempts.max(MIN_ATTEMPTS); + + Ok(()) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ProbeConfigDto { + #[serde(default = "default_metadata_probe_cooldown", skip_serializing_if = "is_default_metadata_probe_cooldown")] + pub cooldown: String, + #[serde( + default = "default_metadata_probe_retry_load_retry_delay", + skip_serializing_if = "is_default_metadata_probe_retry_load_retry_delay" + )] + pub retry_load_retry_delay: String, + #[serde( + default = "default_metadata_probe_retry_backoff_step_1", + skip_serializing_if = "is_default_metadata_probe_retry_backoff_step_1" + )] + pub retry_backoff_step_1: String, + #[serde( + default = "default_metadata_probe_retry_backoff_step_2", + skip_serializing_if = "is_default_metadata_probe_retry_backoff_step_2" + )] + pub retry_backoff_step_2: String, + #[serde( + default = "default_metadata_probe_retry_backoff_step_3", + skip_serializing_if = "is_default_metadata_probe_retry_backoff_step_3" + )] + pub retry_backoff_step_3: String, + #[serde( + default = "default_metadata_max_attempts_probe", + skip_serializing_if = "is_default_metadata_max_attempts_probe" + )] + pub max_attempts: u8, + #[serde( + default = "default_metadata_backoff_jitter_percent", + skip_serializing_if = "is_default_metadata_backoff_jitter_percent" + )] + pub backoff_jitter_percent: u8, + #[serde(default = "default_probe_user_priority", skip_serializing_if = "is_default_probe_user_priority")] + pub user_priority: i8, +} + +impl Default for ProbeConfigDto { + fn default() -> Self { + Self { + cooldown: default_metadata_probe_cooldown(), + retry_load_retry_delay: default_metadata_probe_retry_load_retry_delay(), + retry_backoff_step_1: default_metadata_probe_retry_backoff_step_1(), + retry_backoff_step_2: default_metadata_probe_retry_backoff_step_2(), + retry_backoff_step_3: default_metadata_probe_retry_backoff_step_3(), + max_attempts: default_metadata_max_attempts_probe(), + backoff_jitter_percent: default_metadata_backoff_jitter_percent(), + user_priority: default_probe_user_priority(), + } + } +} + +impl ProbeConfigDto { + pub fn is_empty(&self) -> bool { + self.cooldown == default_metadata_probe_cooldown() + && self.retry_load_retry_delay == default_metadata_probe_retry_load_retry_delay() + && self.retry_backoff_step_1 == default_metadata_probe_retry_backoff_step_1() + && self.retry_backoff_step_2 == default_metadata_probe_retry_backoff_step_2() + && self.retry_backoff_step_3 == default_metadata_probe_retry_backoff_step_3() + && self.max_attempts == default_metadata_max_attempts_probe() + && self.backoff_jitter_percent == default_metadata_backoff_jitter_percent() + && self.user_priority == default_probe_user_priority() + } + + pub(super) fn prepare(&mut self) -> Result<(), TuliproxError> { + let cooldown_secs = + MetadataUpdateConfigDto::parse_and_clamp_duration(&self.cooldown, MIN_DURATION_SECS, "probe.cooldown")?; + self.cooldown = MetadataUpdateConfigDto::canonicalize_seconds(cooldown_secs); + + let retry_load_retry_delay_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.retry_load_retry_delay, + MIN_DURATION_SECS, + "probe.retry_load_retry_delay", + )?; + self.retry_load_retry_delay = MetadataUpdateConfigDto::canonicalize_seconds(retry_load_retry_delay_secs); + + let retry_backoff_step_1_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.retry_backoff_step_1, + MIN_DURATION_SECS, + "probe.retry_backoff_step_1", + )?; + self.retry_backoff_step_1 = MetadataUpdateConfigDto::canonicalize_seconds(retry_backoff_step_1_secs); + + let retry_backoff_step_2_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.retry_backoff_step_2, + MIN_DURATION_SECS, + "probe.retry_backoff_step_2", + )?; + self.retry_backoff_step_2 = MetadataUpdateConfigDto::canonicalize_seconds(retry_backoff_step_2_secs); + + let retry_backoff_step_3_secs = MetadataUpdateConfigDto::parse_and_clamp_duration( + &self.retry_backoff_step_3, + MIN_DURATION_SECS, + "probe.retry_backoff_step_3", + )?; + self.retry_backoff_step_3 = MetadataUpdateConfigDto::canonicalize_seconds(retry_backoff_step_3_secs); + + self.max_attempts = self.max_attempts.max(MIN_ATTEMPTS); + self.backoff_jitter_percent = self.backoff_jitter_percent.min(MAX_JITTER_PERCENT); + + Ok(()) + } +} diff --git a/shared/src/model/config/metadata_update/providers.rs b/shared/src/model/config/metadata_update/providers.rs new file mode 100644 index 000000000..ff648168d --- /dev/null +++ b/shared/src/model/config/metadata_update/providers.rs @@ -0,0 +1,198 @@ +use super::{MetadataUpdateConfigDto, DEFAULT_FFPROBE_TIMEOUT_SECS, MIN_DURATION_SECS}; +use crate::{ + defaults::{ + default_metadata_ffprobe_analyze_duration, default_metadata_ffprobe_live_analyze_duration, + default_metadata_ffprobe_live_probe_size, default_metadata_ffprobe_probe_size, default_metadata_tmdb_cooldown, + default_tmdb_api_key, default_tmdb_cache_duration_days, default_tmdb_language, default_tmdb_match_threshold, + default_tmdb_rate_limit_ms, is_default_metadata_ffprobe_analyze_duration, + is_default_metadata_ffprobe_live_analyze_duration, is_default_metadata_ffprobe_live_probe_size, + is_default_metadata_ffprobe_probe_size, is_default_metadata_tmdb_cooldown, is_default_tmdb_cache_duration_days, + is_default_tmdb_language, is_default_tmdb_match_threshold, is_default_tmdb_rate_limit_ms, is_false, + is_tmdb_default_api_key, TMDB_API_KEY, + }, + error::TuliproxError, + model::ByteSize, + utils::deserialize_as_string, +}; + +fn default_ffprobe_timeout_secs() -> Option { Some(DEFAULT_FFPROBE_TIMEOUT_SECS) } + +fn is_default_ffprobe_timeout(timeout: &Option) -> bool { + timeout.is_none_or(|value| value == DEFAULT_FFPROBE_TIMEOUT_SECS) +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct FfprobeConfigDto { + #[serde(default, skip_serializing_if = "is_false")] + pub enabled: bool, + #[serde(default = "default_ffprobe_timeout_secs", skip_serializing_if = "is_default_ffprobe_timeout")] + pub timeout: Option, + #[serde( + default = "default_metadata_ffprobe_analyze_duration", + skip_serializing_if = "is_default_metadata_ffprobe_analyze_duration", + deserialize_with = "deserialize_as_string" + )] + pub analyze_duration: String, + #[serde( + default = "default_metadata_ffprobe_probe_size", + skip_serializing_if = "is_default_metadata_ffprobe_probe_size" + )] + pub probe_size: ByteSize, + #[serde( + default = "default_metadata_ffprobe_live_analyze_duration", + skip_serializing_if = "is_default_metadata_ffprobe_live_analyze_duration", + deserialize_with = "deserialize_as_string" + )] + pub live_analyze_duration: String, + #[serde( + default = "default_metadata_ffprobe_live_probe_size", + skip_serializing_if = "is_default_metadata_ffprobe_live_probe_size" + )] + pub live_probe_size: ByteSize, +} + +impl Default for FfprobeConfigDto { + fn default() -> Self { + Self { + enabled: false, + timeout: default_ffprobe_timeout_secs(), + analyze_duration: default_metadata_ffprobe_analyze_duration(), + probe_size: default_metadata_ffprobe_probe_size(), + live_analyze_duration: default_metadata_ffprobe_live_analyze_duration(), + live_probe_size: default_metadata_ffprobe_live_probe_size(), + } + } +} + +impl FfprobeConfigDto { + pub fn is_empty(&self) -> bool { + !self.enabled + && is_default_ffprobe_timeout(&self.timeout) + && self.analyze_duration == default_metadata_ffprobe_analyze_duration() + && self.probe_size == default_metadata_ffprobe_probe_size() + && self.live_analyze_duration == default_metadata_ffprobe_live_analyze_duration() + && self.live_probe_size == default_metadata_ffprobe_live_probe_size() + } + + pub fn clean(&mut self) { + if self.timeout.is_none_or(|timeout| timeout == 0) { + self.timeout = default_ffprobe_timeout_secs(); + } + if self.analyze_duration.trim().is_empty() { + self.analyze_duration = default_metadata_ffprobe_analyze_duration(); + } + if self.probe_size.as_str().trim().is_empty() { + self.probe_size = default_metadata_ffprobe_probe_size(); + } + if self.live_analyze_duration.trim().is_empty() { + self.live_analyze_duration = default_metadata_ffprobe_live_analyze_duration(); + } + if self.live_probe_size.as_str().trim().is_empty() { + self.live_probe_size = default_metadata_ffprobe_live_probe_size(); + } + } + + pub(super) fn prepare(&mut self) -> Result<(), TuliproxError> { + self.timeout = self.timeout.map(|timeout| timeout.max(MIN_DURATION_SECS)); + + let analyze_duration_secs = MetadataUpdateConfigDto::parse_and_clamp_duration_with_required_unit( + &self.analyze_duration, + MIN_DURATION_SECS, + "ffprobe.analyze_duration", + )?; + self.analyze_duration = MetadataUpdateConfigDto::canonicalize_seconds(analyze_duration_secs); + + let probe_size_bytes = self + .probe_size + .parse_bytes() + .map_err(|err| { + TuliproxError::ConfigMetadataUpdate(format!("Invalid size for `ffprobe.probe_size`: {err}")) + })? + .at_least_1(); + self.probe_size = ByteSize::new(MetadataUpdateConfigDto::canonicalize_size_bytes(probe_size_bytes.get())); + + let live_analyze_duration_secs = MetadataUpdateConfigDto::parse_and_clamp_duration_with_required_unit( + &self.live_analyze_duration, + MIN_DURATION_SECS, + "ffprobe.live_analyze_duration", + )?; + self.live_analyze_duration = MetadataUpdateConfigDto::canonicalize_seconds(live_analyze_duration_secs); + + let live_probe_size_bytes = self + .live_probe_size + .parse_bytes() + .map_err(|err| { + TuliproxError::ConfigMetadataUpdate(format!("Invalid size for `ffprobe.live_probe_size`: {err}")) + })? + .at_least_1(); + self.live_probe_size = + ByteSize::new(MetadataUpdateConfigDto::canonicalize_size_bytes(live_probe_size_bytes.get())); + + Ok(()) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TmdbConfigDto { + #[serde(default)] + pub enabled: bool, + #[serde(default = "default_tmdb_api_key", skip_serializing_if = "is_tmdb_default_api_key")] + pub api_key: Option, + #[serde(default = "default_tmdb_rate_limit_ms", skip_serializing_if = "is_default_tmdb_rate_limit_ms")] + pub rate_limit_ms: u64, + #[serde(default = "default_tmdb_cache_duration_days", skip_serializing_if = "is_default_tmdb_cache_duration_days")] + pub cache_duration_days: u32, + #[serde(default = "default_tmdb_language", skip_serializing_if = "is_default_tmdb_language")] + pub language: String, + #[serde(default = "default_metadata_tmdb_cooldown", skip_serializing_if = "is_default_metadata_tmdb_cooldown")] + pub cooldown: String, + #[serde(default = "default_tmdb_match_threshold", skip_serializing_if = "is_default_tmdb_match_threshold")] + pub match_threshold: u16, +} + +impl Default for TmdbConfigDto { + fn default() -> Self { + Self { + enabled: false, + api_key: default_tmdb_api_key(), + rate_limit_ms: default_tmdb_rate_limit_ms(), + cache_duration_days: default_tmdb_cache_duration_days(), + language: default_tmdb_language(), + cooldown: default_metadata_tmdb_cooldown(), + match_threshold: default_tmdb_match_threshold(), + } + } +} + +impl TmdbConfigDto { + pub fn is_empty(&self) -> bool { + !self.enabled + && self.api_key.as_ref().is_none_or(|api_key| api_key == TMDB_API_KEY) + && self.rate_limit_ms == default_tmdb_rate_limit_ms() + && self.cache_duration_days == default_tmdb_cache_duration_days() + && self.language == default_tmdb_language() + && self.cooldown == default_metadata_tmdb_cooldown() + && self.match_threshold == default_tmdb_match_threshold() + } + + pub(super) fn clean(&mut self) { + self.api_key = self.api_key.take().and_then(|api_key| { + let trimmed = api_key.trim(); + if trimmed.is_empty() || trimmed == TMDB_API_KEY { + None + } else { + Some(trimmed.to_string()) + } + }); + } + + pub(super) fn prepare(&mut self) -> Result<(), TuliproxError> { + let cooldown_secs = + MetadataUpdateConfigDto::parse_and_clamp_duration(&self.cooldown, MIN_DURATION_SECS, "tmdb.cooldown")?; + self.cooldown = MetadataUpdateConfigDto::canonicalize_seconds(cooldown_secs); + self.match_threshold = self.match_threshold.clamp(0, 100); + Ok(()) + } +} diff --git a/shared/src/model/config/metadata_update/tests.rs b/shared/src/model/config/metadata_update/tests.rs new file mode 100644 index 000000000..6d05d6a2b --- /dev/null +++ b/shared/src/model/config/metadata_update/tests.rs @@ -0,0 +1,120 @@ +use super::MetadataUpdateConfigDto; +use crate::model::ByteSize; + +#[test] +fn default_config_is_empty() { + let cfg = MetadataUpdateConfigDto::default(); + assert!(cfg.is_empty()); +} + +#[test] +fn prepare_keeps_default_config_empty() { + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.prepare().expect("metadata update config defaults should be valid"); + assert!(cfg.is_empty()); +} + +#[test] +fn prepare_parses_duration_suffixes() { + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.log.queue_interval = "1m".to_string(); + cfg.log.progress_interval = "2h".to_string(); + cfg.probe.cooldown = "1d".to_string(); + cfg.tmdb.cooldown = "2d".to_string(); + + cfg.prepare().expect("metadata update config should parse duration values"); + + assert_eq!(cfg.log.queue_interval, "1m"); + assert_eq!(cfg.log.progress_interval, "2h"); + assert_eq!(cfg.probe.cooldown, "1d"); + assert_eq!(cfg.tmdb.cooldown, "2d"); +} + +#[test] +fn prepare_clamps_minimum_values() { + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.log.queue_interval = "0".to_string(); + cfg.resolve.max_attempts = 0; + cfg.probe.max_attempts = 0; + cfg.max_queue_size = 0; + cfg.no_change_cache_ttl_secs = 0; + cfg.probe_fairness_resolve_burst = 0; + cfg.ffprobe.timeout = Some(0); + cfg.ffprobe.analyze_duration = "0s".to_string(); + cfg.ffprobe.probe_size = ByteSize::new("0"); + + cfg.prepare().expect("metadata update config should clamp minimum values"); + + assert_eq!(cfg.log.queue_interval, "1s"); + assert_eq!(cfg.resolve.max_attempts, 1); + assert_eq!(cfg.probe.max_attempts, 1); + assert_eq!(cfg.max_queue_size, 1); + assert_eq!(cfg.no_change_cache_ttl_secs, 1); + assert_eq!(cfg.probe_fairness_resolve_burst, 1); + assert_eq!(cfg.ffprobe.timeout, Some(1)); + assert_eq!(cfg.ffprobe.analyze_duration, "1s"); + assert_eq!(cfg.ffprobe.probe_size, ByteSize::new("1B")); +} + +#[test] +fn prepare_rejects_invalid_duration_unit() { + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.log.queue_interval = "1w".to_string(); + + let result = cfg.prepare(); + assert!(result.is_err(), "invalid duration unit must fail"); +} + +#[test] +fn prepare_canonicalizes_to_larger_units() { + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.probe.cooldown = "604800".to_string(); + cfg.tmdb.cooldown = "259200".to_string(); + cfg.worker_idle_timeout = "60".to_string(); + cfg.probe.retry_backoff_step_3 = "3600".to_string(); + cfg.ffprobe.analyze_duration = "10s".to_string(); + cfg.ffprobe.probe_size = ByteSize::new("10485760"); + cfg.ffprobe.live_analyze_duration = "5s".to_string(); + cfg.ffprobe.live_probe_size = ByteSize::new("5242880"); + + cfg.prepare().expect("metadata update config should canonicalize durations"); + + assert_eq!(cfg.probe.cooldown, "7d"); + assert_eq!(cfg.tmdb.cooldown, "3d"); + assert_eq!(cfg.worker_idle_timeout, "1m"); + assert_eq!(cfg.probe.retry_backoff_step_3, "1h"); + assert_eq!(cfg.ffprobe.analyze_duration, "10s"); + assert_eq!(cfg.ffprobe.probe_size, ByteSize::new("10MB")); + assert_eq!(cfg.ffprobe.live_analyze_duration, "5s"); + assert_eq!(cfg.ffprobe.live_probe_size, ByteSize::new("5MB")); +} + +#[test] +fn prepare_rejects_ffprobe_duration_without_unit() { + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.ffprobe.analyze_duration = "10000000".to_string(); + + let result = cfg.prepare(); + assert!(result.is_err(), "numeric ffprobe analyze duration without unit must fail"); + let err_text = result.expect_err("validation should fail").to_string(); + assert!(err_text.contains("ffprobe.analyze_duration")); +} + +#[test] +fn tmdb_non_default_match_threshold_is_not_empty() { + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.tmdb.match_threshold = 90; + cfg.prepare().expect("metadata update config should remain valid"); + + assert!(!cfg.tmdb.is_empty(), "tmdb config with non-default match threshold must not be empty"); + assert!(!cfg.is_empty(), "metadata update config with non-default tmdb match threshold must not be empty"); +} + +#[test] +fn prepare_clamps_tmdb_match_threshold() { + let mut cfg = MetadataUpdateConfigDto::default(); + cfg.tmdb.match_threshold = 250; + cfg.prepare().expect("metadata update config should clamp tmdb match threshold"); + + assert_eq!(cfg.tmdb.match_threshold, 100); +} diff --git a/shared/src/model/config/recording.rs b/shared/src/model/config/recording.rs index ddd1081f1..53582b0aa 100644 --- a/shared/src/model/config/recording.rs +++ b/shared/src/model/config/recording.rs @@ -1,135 +1,9 @@ -use crate::{ - defaults::{ - default_episode_pattern, default_recording_dir, default_supported_video_extensions, - is_blank_or_default_episode_pattern, is_blank_or_default_recording_dir, is_default_supported_video_extensions, - is_false, DEFAULT_USER_AGENT, F64_DEFAULT_EPSILON, - }, - error::TuliproxError, - utils::{is_blank_optional_str, is_blank_optional_string}, +use crate::defaults::{ + default_episode_pattern, default_recording_dir, is_blank_or_default_episode_pattern, + is_blank_or_default_recording_dir, is_false, }; use std::collections::HashMap; -const fn default_retry_backoff_initial_secs() -> u64 { 3 } -const fn default_retry_backoff_multiplier() -> f64 { 3.0 } -const fn default_retry_backoff_max_secs() -> u64 { 30 } -const fn default_retry_backoff_jitter_percent() -> u8 { 20 } -const fn default_retry_max_attempts() -> u8 { 5 } -fn is_default_retry_backoff_initial_secs(value: &u64) -> bool { *value == default_retry_backoff_initial_secs() } -fn is_default_retry_backoff_multiplier(value: &f64) -> bool { - (*value - default_retry_backoff_multiplier()).abs() < F64_DEFAULT_EPSILON -} -fn is_default_retry_backoff_max_secs(value: &u64) -> bool { *value == default_retry_backoff_max_secs() } -fn is_default_retry_backoff_jitter_percent(value: &u8) -> bool { *value == default_retry_backoff_jitter_percent() } -fn is_default_retry_max_attempts(value: &u8) -> bool { *value == default_retry_max_attempts() } -fn is_zero_u8(value: &u8) -> bool { *value == 0 } -fn is_zero_i8(value: &i8) -> bool { *value == 0 } - -// Recording configuration constants. -const DEFAULT_RECORDING_DIRECTORY_SUFFIX: &str = "recordings"; -const DEFAULT_RECORDING_TIMEZONE: &str = "UTC"; -const DEFAULT_RECORDING_FILENAME_TEMPLATE: &str = "{channel}_{program_title}_{start_time}"; -const DEFAULT_RECORDING_MAX_PRE_ROLL_SECS: u64 = 15 * 60; // 15 minutes -const DEFAULT_RECORDING_MAX_POST_ROLL_SECS: u64 = 30 * 60; // 30 minutes -const DEFAULT_RECORDING_CLEANUP_INTERVAL_SECS: u64 = 60 * 60; // 1 hour -const DEFAULT_RECORDING_DISK_SAFETY_BYTES: u64 = 1024 * 1024 * 1024; // 1 GiB -const DEFAULT_RECORDING_FALLBACK_BYTES_PER_MINUTE: u64 = 8 * 1024 * 1024; // 8 MiB -const DEFAULT_RECORDING_RETENTION_SWEEP_INTERVAL_SECS: u64 = 60 * 60; // 1 hour -const DEFAULT_RECORDING_NOTIFICATION_OUTBOX_BUFFER: usize = 1024; -const DEFAULT_RECORDING_NOTIFICATION_MAX_ATTEMPTS: u32 = 6; -const DEFAULT_RECORDING_NOTIFICATION_BACKOFF_INITIAL_SECS: u64 = 5; -const DEFAULT_RECORDING_NOTIFICATION_BACKOFF_MAX_SECS: u64 = 900; // 15 minutes -const MAX_FILENAME_TEMPLATE_BYTES: usize = 240; - -/// Container the recorder muxes into. MPEG-TS is the default; operators -/// recording H.265 or AAC-only channels may want a container that suits them -/// better. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RecordingContainerFormat { - /// MPEG-TS. Robust against truncation, so a recording killed - /// mid-stream still plays — which is why it is the default. - #[default] - Mpegts, - Matroska, - Mp4, -} - -impl RecordingContainerFormat { - /// The `-f` argument for the muxer. - pub fn ffmpeg_format(self) -> &'static str { - match self { - Self::Mpegts => "mpegts", - Self::Matroska => "matroska", - Self::Mp4 => "mp4", - } - } - - /// The extension recordings in this container get, without the dot. - pub fn file_extension(self) -> &'static str { - match self { - Self::Mpegts => "ts", - Self::Matroska => "mkv", - Self::Mp4 => "mp4", - } - } -} - -pub fn default_recording_directory(download_dir: &str) -> String { - format!("{download_dir}/{DEFAULT_RECORDING_DIRECTORY_SUFFIX}") -} - -pub fn default_recording_timezone() -> String { DEFAULT_RECORDING_TIMEZONE.to_string() } -pub fn default_recording_filename_template() -> String { DEFAULT_RECORDING_FILENAME_TEMPLATE.to_string() } -pub const fn default_recording_max_pre_roll_secs() -> u64 { DEFAULT_RECORDING_MAX_PRE_ROLL_SECS } -pub const fn default_recording_max_post_roll_secs() -> u64 { DEFAULT_RECORDING_MAX_POST_ROLL_SECS } -pub const fn default_recording_cleanup_interval_secs() -> u64 { DEFAULT_RECORDING_CLEANUP_INTERVAL_SECS } -pub const fn default_recording_disk_safety_bytes() -> u64 { DEFAULT_RECORDING_DISK_SAFETY_BYTES } -pub const fn default_recording_fallback_bytes_per_minute() -> u64 { DEFAULT_RECORDING_FALLBACK_BYTES_PER_MINUTE } - -fn is_default_recording_max_pre_roll_secs(value: &u64) -> bool { *value == default_recording_max_pre_roll_secs() } -fn is_default_recording_max_post_roll_secs(value: &u64) -> bool { *value == default_recording_max_post_roll_secs() } -pub const fn default_recording_retention_sweep_interval_secs() -> u64 { - DEFAULT_RECORDING_RETENTION_SWEEP_INTERVAL_SECS -} -pub const fn default_recording_notification_outbox_buffer() -> usize { DEFAULT_RECORDING_NOTIFICATION_OUTBOX_BUFFER } -pub const fn default_recording_notification_max_attempts() -> u32 { DEFAULT_RECORDING_NOTIFICATION_MAX_ATTEMPTS } -pub const fn default_recording_notification_backoff_initial_secs() -> u64 { - DEFAULT_RECORDING_NOTIFICATION_BACKOFF_INITIAL_SECS -} -pub const fn default_recording_notification_backoff_max_secs() -> u64 { - DEFAULT_RECORDING_NOTIFICATION_BACKOFF_MAX_SECS -} -pub const fn default_recording_enabled() -> bool { true } - -fn is_default_recording_fallback_bytes_per_minute(value: &u64) -> bool { - *value == default_recording_fallback_bytes_per_minute() -} -fn is_default_recording_retention_sweep_interval_secs(value: &u64) -> bool { - *value == default_recording_retention_sweep_interval_secs() -} -fn is_default_recording_notification_outbox_buffer(value: &usize) -> bool { - *value == default_recording_notification_outbox_buffer() -} -fn is_default_recording_notification_max_attempts(value: &u32) -> bool { - *value == default_recording_notification_max_attempts() -} -fn is_default_recording_notification_backoff_initial_secs(value: &u64) -> bool { - *value == default_recording_notification_backoff_initial_secs() -} -fn is_default_recording_notification_backoff_max_secs(value: &u64) -> bool { - *value == default_recording_notification_backoff_max_secs() -} -fn is_recording_enabled(value: &bool) -> bool { *value } -fn is_default_recording_container_format(value: &RecordingContainerFormat) -> bool { - *value == RecordingContainerFormat::default() -} -// `skip_serializing_if` predicates that distinguish "field absent" from -// "field present with a zero value". An explicit `Some(0)` is a real -// configuration choice and must round-trip; only `None` means "absent". -fn is_zero_u64_opt(value: &Option) -> bool { value.is_none() } -fn is_zero_u32_opt(value: &Option) -> bool { value.is_none() } -fn is_zero_u8_opt(value: &Option) -> bool { value.is_none() } - /// DVR recording configuration block. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] @@ -276,412 +150,30 @@ impl RecordingConfigDto { } } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RecordingRetentionConfigDto { - #[serde(default, skip_serializing_if = "is_zero_u32_opt")] - pub keep_last_per_channel: Option, - #[serde(default, skip_serializing_if = "is_zero_u32_opt")] - pub delete_after_days: Option, - /// How often the age/count sweep runs. Independent of - /// `disk.cleanup_interval_secs`, which paces the watermark check. - #[serde( - default = "default_recording_retention_sweep_interval_secs", - skip_serializing_if = "is_default_recording_retention_sweep_interval_secs" - )] - pub sweep_interval_secs: u64, -} - -impl Default for RecordingRetentionConfigDto { - fn default() -> Self { - Self { - keep_last_per_channel: None, - delete_after_days: None, - sweep_interval_secs: default_recording_retention_sweep_interval_secs(), - } - } -} - -impl RecordingRetentionConfigDto { - pub fn is_empty(&self) -> bool { - self.keep_last_per_channel.is_none() - && self.delete_after_days.is_none() - && is_default_recording_retention_sweep_interval_secs(&self.sweep_interval_secs) - } -} - -/// Lifecycle-notification delivery. The outbox worker owns these; the -/// recorder itself never blocks on a notification. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RecordingNotificationConfigDto { - /// Bounded in-memory queue between the recorder and the outbox - /// worker. A full channel drops the newest entry rather than - /// stalling a recording. - #[serde( - default = "default_recording_notification_outbox_buffer", - skip_serializing_if = "is_default_recording_notification_outbox_buffer" - )] - pub outbox_buffer: usize, - /// Delivery attempts before an entry is dead-lettered. - #[serde( - default = "default_recording_notification_max_attempts", - skip_serializing_if = "is_default_recording_notification_max_attempts" - )] - pub max_attempts: u32, - #[serde( - default = "default_recording_notification_backoff_initial_secs", - skip_serializing_if = "is_default_recording_notification_backoff_initial_secs" - )] - pub backoff_initial_secs: u64, - #[serde( - default = "default_recording_notification_backoff_max_secs", - skip_serializing_if = "is_default_recording_notification_backoff_max_secs" - )] - pub backoff_max_secs: u64, -} - -impl Default for RecordingNotificationConfigDto { - fn default() -> Self { - Self { - outbox_buffer: default_recording_notification_outbox_buffer(), - max_attempts: default_recording_notification_max_attempts(), - backoff_initial_secs: default_recording_notification_backoff_initial_secs(), - backoff_max_secs: default_recording_notification_backoff_max_secs(), - } - } -} - -impl RecordingNotificationConfigDto { - pub fn is_empty(&self) -> bool { *self == Self::default() } -} - -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RecordingDiskConfigDto { - #[serde(default, skip_serializing_if = "is_zero_u8_opt")] - pub high_water_percent: Option, - #[serde(default, skip_serializing_if = "is_zero_u8_opt")] - pub low_water_percent: Option, - #[serde(default, skip_serializing_if = "is_zero_u64_opt")] - pub cleanup_interval_secs: Option, - #[serde(default, skip_serializing_if = "is_zero_u64_opt")] - pub safety_bytes: Option, -} - -impl RecordingDiskConfigDto { - pub fn is_empty(&self) -> bool { - self.high_water_percent.is_none() - && self.low_water_percent.is_none() - && self.cleanup_interval_secs.is_none() - && self.safety_bytes.is_none() - } -} - -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RecordingQuotaConfigDto { - #[serde(default, skip_serializing_if = "is_zero_u64_opt")] - pub default_private_bytes: Option, - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub per_user_bytes: HashMap, - #[serde(default, skip_serializing_if = "is_zero_u64_opt")] - pub shared_bytes: Option, -} - -impl RecordingQuotaConfigDto { - pub fn is_empty(&self) -> bool { - self.default_private_bytes.is_none() && self.per_user_bytes.is_empty() && self.shared_bytes.is_none() - } -} - -/// Allowed placeholders in the recording filename template. -/// Order matters for error messages and the validation regex. -pub const RECORDING_FILENAME_PLACEHOLDERS: &[&str] = - &["{channel}", "{program_title}", "{start_time}", "{end_time}", "{episode}", "{owner}"]; - -/// Validates a recording filename template. Returns the cleaned template and -/// the matched placeholder set on success. -fn validate_recording_filename_template(template: &str) -> Result<(), TuliproxError> { - if template.is_empty() { - return Err(TuliproxError::ConfigRecording("recording.filename_template must not be empty".to_string())); - } - if template.len() > MAX_FILENAME_TEMPLATE_BYTES { - return Err(TuliproxError::ConfigRecording(format!( - "recording.filename_template must not exceed {MAX_FILENAME_TEMPLATE_BYTES} bytes" - ))); - } - - let bytes = template.as_bytes(); - let mut i = 0; - let mut found_placeholder = false; - while i < bytes.len() { - if bytes[i] == b'{' { - if let Some(close) = template[i + 1..].find('}') { - let end = i + 1 + close; - let placeholder = &template[i..=end]; - if !RECORDING_FILENAME_PLACEHOLDERS.contains(&placeholder) { - return Err(TuliproxError::ConfigRecording(format!( - "recording.filename_template contains unknown placeholder '{placeholder}'" - ))); - } - found_placeholder = true; - i = end + 1; - } else { - return Err(TuliproxError::ConfigRecording( - "recording.filename_template has an unmatched '{'".to_string(), - )); - } - } else if bytes[i] == b'}' { - return Err(TuliproxError::ConfigRecording("recording.filename_template has an unmatched '}'".to_string())); - } else { - i += 1; - } - } - - if !found_placeholder { - return Err(TuliproxError::ConfigRecording( - "recording.filename_template must contain at least one placeholder".to_string(), - )); - } - Ok(()) -} - -fn validate_recording_timezone(tz: &str) -> Result<(), TuliproxError> { - if tz.is_empty() { - return Err(TuliproxError::ConfigRecording("recording.timezone must not be empty".to_string())); - } - tz.parse::() - .map(|_| ()) - .map_err(|_| TuliproxError::ConfigRecording(format!("recording.timezone '{tz}' is not a valid IANA timezone"))) -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct VideoConfigDto { - #[serde( - default = "default_supported_video_extensions", - skip_serializing_if = "is_default_supported_video_extensions" - )] - pub extensions: Vec, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub web_search: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub recording: Option, -} - -impl VideoConfigDto { - pub fn is_empty(&self) -> bool { - (self.extensions.is_empty() || is_default_supported_video_extensions(&self.extensions)) - && is_blank_optional_str(self.web_search.as_deref()) - && self.recording.as_ref().is_none_or(RecordingConfigDto::is_empty) - } - - pub fn clean(&mut self) { - if let Some(recording) = self.recording.as_mut() { - recording.clean(); - } - if self.recording.as_ref().is_some_and(RecordingConfigDto::is_empty) { - self.recording = None; - } - } - - pub fn prepare(&mut self) -> Result<(), TuliproxError> { - if self.extensions.is_empty() { - self.extensions = default_supported_video_extensions(); - } - if let Some(recording) = self.recording.as_mut() { - prepare_recording_config(recording)?; - } - Ok(()) - } -} - -pub(crate) fn prepare_recording_config(recording: &mut RecordingConfigDto) -> Result<(), TuliproxError> { - if recording.headers.is_empty() { - recording.headers.insert("Accept".to_string(), "video/*".to_string()); - recording.headers.insert("User-Agent".to_string(), DEFAULT_USER_AGENT.to_string()); - } - if is_blank_or_default_episode_pattern(&recording.episode_pattern) { - recording.episode_pattern = default_episode_pattern(); - } else if let Some(pattern) = recording.episode_pattern.as_ref() { - recording.episode_pattern = Some(pattern.trim().to_string()); - } - if let Some(pattern) = recording.episode_pattern.as_ref() { - crate::model::REGEX_CACHE - .get_or_compile(pattern) - .map_err(|err| TuliproxError::RegexCompile(format!("{pattern} {err}")))?; - } - recording.retry_backoff_initial_secs = recording.retry_backoff_initial_secs.max(1); - recording.retry_backoff_multiplier = recording.retry_backoff_multiplier.max(1.0); - recording.retry_backoff_max_secs = recording.retry_backoff_max_secs.max(recording.retry_backoff_initial_secs); - recording.retry_backoff_jitter_percent = recording.retry_backoff_jitter_percent.min(95); - recording.retry_max_attempts = recording.retry_max_attempts.max(1); - - // Directory is independent of video extensions and download settings. - if let Some(dir) = recording.directory.as_ref() { - let trimmed = dir.trim(); - if trimmed.is_empty() { - recording.directory = None; - } else if trimmed != dir { - recording.directory = Some(trimmed.to_string()); - } - } - if recording.directory.is_none() { - recording.directory = default_recording_dir(); - } - - // timezone: default UTC; validate IANA. - if let Some(tz) = recording.timezone.as_ref() { - let trimmed = tz.trim(); - if trimmed.is_empty() { - recording.timezone = None; - } else if trimmed != tz { - recording.timezone = Some(trimmed.to_string()); - } - } - if let Some(tz) = recording.timezone.as_ref() { - if tz.parse::().is_err() { - // Surface a warning before the strict validator below turns - // this into a hard error, so the operator sees both signals. - log::warn!("recording.timezone '{tz}' is not a valid IANA timezone"); - } - } - if recording.timezone.is_none() { - recording.timezone = Some(default_recording_timezone()); - } - validate_recording_timezone(recording.timezone.as_deref().unwrap_or("UTC"))?; - - // filename_template: default; validate placeholders. - if let Some(template) = recording.filename_template.as_ref() { - let trimmed = template.trim(); - if trimmed != template { - recording.filename_template = Some(trimmed.to_string()); - } - } - if recording.filename_template.is_none() { - recording.filename_template = Some(default_recording_filename_template()); - } - validate_recording_filename_template(recording.filename_template.as_deref().unwrap_or(""))?; - - // padding: default <= max, fallback to defaults if missing. - if recording.default_pre_roll_secs.is_none() { - recording.default_pre_roll_secs = Some(0); - } - if recording.default_pre_roll_secs.unwrap_or(0) > recording.max_pre_roll_secs { - return Err(TuliproxError::ConfigRecording(format!( - "recording.default_pre_roll_secs ({}) must not exceed max_pre_roll_secs ({})", - recording.default_pre_roll_secs.unwrap_or(0), - recording.max_pre_roll_secs - ))); - } - if recording.default_post_roll_secs.is_none() { - recording.default_post_roll_secs = Some(0); - } - if recording.default_post_roll_secs.unwrap_or(0) > recording.max_post_roll_secs { - return Err(TuliproxError::ConfigRecording(format!( - "recording.default_post_roll_secs ({}) must not exceed max_post_roll_secs ({})", - recording.default_post_roll_secs.unwrap_or(0), - recording.max_post_roll_secs - ))); - } - - // retention: each present policy must be > 0. - if let Some(retention) = recording.retention.as_ref() { - if let Some(keep) = retention.keep_last_per_channel { - if keep == 0 { - return Err(TuliproxError::ConfigRecording( - "recording.retention.keep_last_per_channel must be > 0".to_string(), - )); - } - } - if let Some(days) = retention.delete_after_days { - if days == 0 { - return Err(TuliproxError::ConfigRecording( - "recording.retention.delete_after_days must be > 0".to_string(), - )); - } - } - if retention.sweep_interval_secs == 0 { - return Err(TuliproxError::ConfigRecording( - "recording.retention.sweep_interval_secs must be > 0".to_string(), - )); - } - if retention.keep_last_per_channel.is_none() && retention.delete_after_days.is_none() { - // A retention block that expresses no policy reads as - // "retention is configured" while nothing is ever deleted. - log::warn!( - "recording.retention has neither keep_last_per_channel nor delete_after_days; \ - no recording will ever be deleted by policy" - ); - } - } - - // notifications: an outbox that cannot hold or retry anything would - // silently drop every lifecycle notification. - if let Some(notifications) = recording.notifications.as_ref() { - if notifications.outbox_buffer == 0 { - return Err(TuliproxError::ConfigRecording( - "recording.notifications.outbox_buffer must be > 0".to_string(), - )); - } - if notifications.max_attempts == 0 { - return Err(TuliproxError::ConfigRecording("recording.notifications.max_attempts must be > 0".to_string())); - } - if notifications.backoff_initial_secs == 0 { - return Err(TuliproxError::ConfigRecording( - "recording.notifications.backoff_initial_secs must be > 0".to_string(), - )); - } - if notifications.backoff_max_secs < notifications.backoff_initial_secs { - return Err(TuliproxError::ConfigRecording(format!( - "recording.notifications.backoff_max_secs ({}) must be >= backoff_initial_secs ({})", - notifications.backoff_max_secs, notifications.backoff_initial_secs - ))); - } - } - - // disk: percentages in 0..=100, low < high, cleanup > 0, safety non-zero. - if let Some(disk) = recording.disk.as_ref() { - if let Some(high) = disk.high_water_percent { - if high > 100 { - return Err(TuliproxError::ConfigRecording(format!( - "recording.disk.high_water_percent ({high}) must be 0..=100" - ))); - } - } - if let Some(low) = disk.low_water_percent { - if low > 100 { - return Err(TuliproxError::ConfigRecording(format!( - "recording.disk.low_water_percent ({low}) must be 0..=100" - ))); - } - } - if let (Some(low), Some(high)) = (disk.low_water_percent, disk.high_water_percent) { - if low >= high { - return Err(TuliproxError::ConfigRecording(format!( - "recording.disk.low_water_percent ({low}) must be < high_water_percent ({high})" - ))); - } - } - if let Some(interval) = disk.cleanup_interval_secs { - if interval == 0 { - return Err(TuliproxError::ConfigRecording( - "recording.disk.cleanup_interval_secs must be > 0".to_string(), - )); - } - } - if let Some(safety) = disk.safety_bytes { - if safety == 0 { - return Err(TuliproxError::ConfigRecording("recording.disk.safety_bytes must be > 0".to_string())); - } - } - } - - // quota: fallback bytes per minute must be > 0. - if recording.fallback_bytes_per_minute == 0 { - return Err(TuliproxError::ConfigRecording("recording.fallback_bytes_per_minute must be > 0".to_string())); - } - - Ok(()) -} +mod defaults; +mod policies; +mod prepare; +mod video; +pub use defaults::{ + default_recording_cleanup_interval_secs, default_recording_directory, default_recording_disk_safety_bytes, + default_recording_enabled, default_recording_fallback_bytes_per_minute, default_recording_filename_template, + default_recording_max_post_roll_secs, default_recording_max_pre_roll_secs, + default_recording_notification_backoff_initial_secs, default_recording_notification_backoff_max_secs, + default_recording_notification_max_attempts, default_recording_notification_outbox_buffer, + default_recording_retention_sweep_interval_secs, default_recording_timezone, +}; +use defaults::{ + default_retry_backoff_initial_secs, default_retry_backoff_jitter_percent, default_retry_backoff_max_secs, + default_retry_backoff_multiplier, default_retry_max_attempts, is_default_recording_container_format, + is_default_recording_fallback_bytes_per_minute, is_default_recording_max_post_roll_secs, + is_default_recording_max_pre_roll_secs, is_default_retry_backoff_initial_secs, + is_default_retry_backoff_jitter_percent, is_default_retry_backoff_max_secs, is_default_retry_backoff_multiplier, + is_default_retry_max_attempts, is_recording_enabled, is_zero_i8, is_zero_u64_opt, is_zero_u8, +}; +pub use policies::{ + RecordingContainerFormat, RecordingDiskConfigDto, RecordingNotificationConfigDto, RecordingQuotaConfigDto, + RecordingRetentionConfigDto, +}; +pub(crate) use prepare::prepare_recording_config; +pub use prepare::RECORDING_FILENAME_PLACEHOLDERS; +pub use video::VideoConfigDto; diff --git a/shared/src/model/config/recording/defaults.rs b/shared/src/model/config/recording/defaults.rs new file mode 100644 index 000000000..aa914d428 --- /dev/null +++ b/shared/src/model/config/recording/defaults.rs @@ -0,0 +1,151 @@ +use super::RecordingContainerFormat; +use crate::defaults::F64_DEFAULT_EPSILON; + +pub(super) const fn default_retry_backoff_initial_secs() -> u64 { 3 } + +pub(super) const fn default_retry_backoff_multiplier() -> f64 { 3.0 } + +pub(super) const fn default_retry_backoff_max_secs() -> u64 { 30 } + +pub(super) const fn default_retry_backoff_jitter_percent() -> u8 { 20 } + +pub(super) const fn default_retry_max_attempts() -> u8 { 5 } + +pub(super) fn is_default_retry_backoff_initial_secs(value: &u64) -> bool { + *value == default_retry_backoff_initial_secs() +} + +pub(super) fn is_default_retry_backoff_multiplier(value: &f64) -> bool { + (*value - default_retry_backoff_multiplier()).abs() < F64_DEFAULT_EPSILON +} + +pub(super) fn is_default_retry_backoff_max_secs(value: &u64) -> bool { *value == default_retry_backoff_max_secs() } + +pub(super) fn is_default_retry_backoff_jitter_percent(value: &u8) -> bool { + *value == default_retry_backoff_jitter_percent() +} + +pub(super) fn is_default_retry_max_attempts(value: &u8) -> bool { *value == default_retry_max_attempts() } + +pub(super) fn is_zero_u8(value: &u8) -> bool { *value == 0 } + +pub(super) fn is_zero_i8(value: &i8) -> bool { *value == 0 } + +// Recording configuration constants. +const DEFAULT_RECORDING_DIRECTORY_SUFFIX: &str = "recordings"; + +const DEFAULT_RECORDING_TIMEZONE: &str = "UTC"; + +const DEFAULT_RECORDING_FILENAME_TEMPLATE: &str = "{channel}_{program_title}_{start_time}"; + +const DEFAULT_RECORDING_MAX_PRE_ROLL_SECS: u64 = 15 * 60; + +// 15 minutes +const DEFAULT_RECORDING_MAX_POST_ROLL_SECS: u64 = 30 * 60; + +// 30 minutes +const DEFAULT_RECORDING_CLEANUP_INTERVAL_SECS: u64 = 60 * 60; + +// 1 hour +const DEFAULT_RECORDING_DISK_SAFETY_BYTES: u64 = 1024 * 1024 * 1024; + +// 1 GiB +const DEFAULT_RECORDING_FALLBACK_BYTES_PER_MINUTE: u64 = 8 * 1024 * 1024; + +// 8 MiB +const DEFAULT_RECORDING_RETENTION_SWEEP_INTERVAL_SECS: u64 = 60 * 60; + +// 1 hour +const DEFAULT_RECORDING_NOTIFICATION_OUTBOX_BUFFER: usize = 1024; + +const DEFAULT_RECORDING_NOTIFICATION_MAX_ATTEMPTS: u32 = 6; + +const DEFAULT_RECORDING_NOTIFICATION_BACKOFF_INITIAL_SECS: u64 = 5; + +const DEFAULT_RECORDING_NOTIFICATION_BACKOFF_MAX_SECS: u64 = 900; + +// 15 minutes +pub(super) const MAX_FILENAME_TEMPLATE_BYTES: usize = 240; + +pub fn default_recording_directory(download_dir: &str) -> String { + format!("{download_dir}/{DEFAULT_RECORDING_DIRECTORY_SUFFIX}") +} + +pub fn default_recording_timezone() -> String { DEFAULT_RECORDING_TIMEZONE.to_string() } + +pub fn default_recording_filename_template() -> String { DEFAULT_RECORDING_FILENAME_TEMPLATE.to_string() } + +pub const fn default_recording_max_pre_roll_secs() -> u64 { DEFAULT_RECORDING_MAX_PRE_ROLL_SECS } + +pub const fn default_recording_max_post_roll_secs() -> u64 { DEFAULT_RECORDING_MAX_POST_ROLL_SECS } + +pub const fn default_recording_cleanup_interval_secs() -> u64 { DEFAULT_RECORDING_CLEANUP_INTERVAL_SECS } + +pub const fn default_recording_disk_safety_bytes() -> u64 { DEFAULT_RECORDING_DISK_SAFETY_BYTES } + +pub const fn default_recording_fallback_bytes_per_minute() -> u64 { DEFAULT_RECORDING_FALLBACK_BYTES_PER_MINUTE } + +pub(super) fn is_default_recording_max_pre_roll_secs(value: &u64) -> bool { + *value == default_recording_max_pre_roll_secs() +} + +pub(super) fn is_default_recording_max_post_roll_secs(value: &u64) -> bool { + *value == default_recording_max_post_roll_secs() +} + +pub const fn default_recording_retention_sweep_interval_secs() -> u64 { + DEFAULT_RECORDING_RETENTION_SWEEP_INTERVAL_SECS +} + +pub const fn default_recording_notification_outbox_buffer() -> usize { DEFAULT_RECORDING_NOTIFICATION_OUTBOX_BUFFER } + +pub const fn default_recording_notification_max_attempts() -> u32 { DEFAULT_RECORDING_NOTIFICATION_MAX_ATTEMPTS } + +pub const fn default_recording_notification_backoff_initial_secs() -> u64 { + DEFAULT_RECORDING_NOTIFICATION_BACKOFF_INITIAL_SECS +} + +pub const fn default_recording_notification_backoff_max_secs() -> u64 { + DEFAULT_RECORDING_NOTIFICATION_BACKOFF_MAX_SECS +} + +pub const fn default_recording_enabled() -> bool { true } + +pub(super) fn is_default_recording_fallback_bytes_per_minute(value: &u64) -> bool { + *value == default_recording_fallback_bytes_per_minute() +} + +pub(super) fn is_default_recording_retention_sweep_interval_secs(value: &u64) -> bool { + *value == default_recording_retention_sweep_interval_secs() +} + +pub(super) fn is_default_recording_notification_outbox_buffer(value: &usize) -> bool { + *value == default_recording_notification_outbox_buffer() +} + +pub(super) fn is_default_recording_notification_max_attempts(value: &u32) -> bool { + *value == default_recording_notification_max_attempts() +} + +pub(super) fn is_default_recording_notification_backoff_initial_secs(value: &u64) -> bool { + *value == default_recording_notification_backoff_initial_secs() +} + +pub(super) fn is_default_recording_notification_backoff_max_secs(value: &u64) -> bool { + *value == default_recording_notification_backoff_max_secs() +} + +pub(super) fn is_recording_enabled(value: &bool) -> bool { *value } + +pub(super) fn is_default_recording_container_format(value: &RecordingContainerFormat) -> bool { + *value == RecordingContainerFormat::default() +} + +// `skip_serializing_if` predicates that distinguish "field absent" from +// "field present with a zero value". An explicit `Some(0)` is a real +// configuration choice and must round-trip; only `None` means "absent". +pub(super) fn is_zero_u64_opt(value: &Option) -> bool { value.is_none() } + +pub(super) fn is_zero_u32_opt(value: &Option) -> bool { value.is_none() } + +pub(super) fn is_zero_u8_opt(value: &Option) -> bool { value.is_none() } diff --git a/shared/src/model/config/recording/policies.rs b/shared/src/model/config/recording/policies.rs new file mode 100644 index 000000000..93d727c05 --- /dev/null +++ b/shared/src/model/config/recording/policies.rs @@ -0,0 +1,164 @@ +use super::{ + default_recording_notification_backoff_initial_secs, default_recording_notification_backoff_max_secs, + default_recording_notification_max_attempts, default_recording_notification_outbox_buffer, + default_recording_retention_sweep_interval_secs, + defaults::{ + is_default_recording_notification_backoff_initial_secs, is_default_recording_notification_backoff_max_secs, + is_default_recording_notification_max_attempts, is_default_recording_notification_outbox_buffer, + is_default_recording_retention_sweep_interval_secs, is_zero_u32_opt, is_zero_u64_opt, is_zero_u8_opt, + }, +}; +use std::collections::HashMap; + +/// Container the recorder muxes into. MPEG-TS is the default; operators +/// recording H.265 or AAC-only channels may want a container that suits them +/// better. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RecordingContainerFormat { + /// MPEG-TS. Robust against truncation, so a recording killed + /// mid-stream still plays — which is why it is the default. + #[default] + Mpegts, + Matroska, + Mp4, +} + +impl RecordingContainerFormat { + /// The `-f` argument for the muxer. + pub fn ffmpeg_format(self) -> &'static str { + match self { + Self::Mpegts => "mpegts", + Self::Matroska => "matroska", + Self::Mp4 => "mp4", + } + } + + /// The extension recordings in this container get, without the dot. + pub fn file_extension(self) -> &'static str { + match self { + Self::Mpegts => "ts", + Self::Matroska => "mkv", + Self::Mp4 => "mp4", + } + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct RecordingRetentionConfigDto { + #[serde(default, skip_serializing_if = "is_zero_u32_opt")] + pub keep_last_per_channel: Option, + #[serde(default, skip_serializing_if = "is_zero_u32_opt")] + pub delete_after_days: Option, + /// How often the age/count sweep runs. Independent of + /// `disk.cleanup_interval_secs`, which paces the watermark check. + #[serde( + default = "default_recording_retention_sweep_interval_secs", + skip_serializing_if = "is_default_recording_retention_sweep_interval_secs" + )] + pub sweep_interval_secs: u64, +} + +impl Default for RecordingRetentionConfigDto { + fn default() -> Self { + Self { + keep_last_per_channel: None, + delete_after_days: None, + sweep_interval_secs: default_recording_retention_sweep_interval_secs(), + } + } +} + +impl RecordingRetentionConfigDto { + pub fn is_empty(&self) -> bool { + self.keep_last_per_channel.is_none() + && self.delete_after_days.is_none() + && is_default_recording_retention_sweep_interval_secs(&self.sweep_interval_secs) + } +} + +/// Lifecycle-notification delivery. The outbox worker owns these; the +/// recorder itself never blocks on a notification. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct RecordingNotificationConfigDto { + /// Bounded in-memory queue between the recorder and the outbox + /// worker. A full channel drops the newest entry rather than + /// stalling a recording. + #[serde( + default = "default_recording_notification_outbox_buffer", + skip_serializing_if = "is_default_recording_notification_outbox_buffer" + )] + pub outbox_buffer: usize, + /// Delivery attempts before an entry is dead-lettered. + #[serde( + default = "default_recording_notification_max_attempts", + skip_serializing_if = "is_default_recording_notification_max_attempts" + )] + pub max_attempts: u32, + #[serde( + default = "default_recording_notification_backoff_initial_secs", + skip_serializing_if = "is_default_recording_notification_backoff_initial_secs" + )] + pub backoff_initial_secs: u64, + #[serde( + default = "default_recording_notification_backoff_max_secs", + skip_serializing_if = "is_default_recording_notification_backoff_max_secs" + )] + pub backoff_max_secs: u64, +} + +impl Default for RecordingNotificationConfigDto { + fn default() -> Self { + Self { + outbox_buffer: default_recording_notification_outbox_buffer(), + max_attempts: default_recording_notification_max_attempts(), + backoff_initial_secs: default_recording_notification_backoff_initial_secs(), + backoff_max_secs: default_recording_notification_backoff_max_secs(), + } + } +} + +impl RecordingNotificationConfigDto { + pub fn is_empty(&self) -> bool { *self == Self::default() } +} + +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct RecordingDiskConfigDto { + #[serde(default, skip_serializing_if = "is_zero_u8_opt")] + pub high_water_percent: Option, + #[serde(default, skip_serializing_if = "is_zero_u8_opt")] + pub low_water_percent: Option, + #[serde(default, skip_serializing_if = "is_zero_u64_opt")] + pub cleanup_interval_secs: Option, + #[serde(default, skip_serializing_if = "is_zero_u64_opt")] + pub safety_bytes: Option, +} + +impl RecordingDiskConfigDto { + pub fn is_empty(&self) -> bool { + self.high_water_percent.is_none() + && self.low_water_percent.is_none() + && self.cleanup_interval_secs.is_none() + && self.safety_bytes.is_none() + } +} + +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct RecordingQuotaConfigDto { + #[serde(default, skip_serializing_if = "is_zero_u64_opt")] + pub default_private_bytes: Option, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub per_user_bytes: HashMap, + #[serde(default, skip_serializing_if = "is_zero_u64_opt")] + pub shared_bytes: Option, +} + +impl RecordingQuotaConfigDto { + pub fn is_empty(&self) -> bool { + self.default_private_bytes.is_none() && self.per_user_bytes.is_empty() && self.shared_bytes.is_none() + } +} diff --git a/shared/src/model/config/recording/prepare.rs b/shared/src/model/config/recording/prepare.rs new file mode 100644 index 000000000..edd626941 --- /dev/null +++ b/shared/src/model/config/recording/prepare.rs @@ -0,0 +1,260 @@ +use super::{ + default_recording_filename_template, default_recording_timezone, defaults::MAX_FILENAME_TEMPLATE_BYTES, + RecordingConfigDto, +}; +use crate::{ + defaults::{ + default_episode_pattern, default_recording_dir, is_blank_or_default_episode_pattern, DEFAULT_USER_AGENT, + }, + error::TuliproxError, +}; + +/// Allowed placeholders in the recording filename template. +/// Order matters for error messages and the validation regex. +pub const RECORDING_FILENAME_PLACEHOLDERS: &[&str] = + &["{channel}", "{program_title}", "{start_time}", "{end_time}", "{episode}", "{owner}"]; + +/// Validates a recording filename template. Returns the cleaned template and +/// the matched placeholder set on success. +fn validate_recording_filename_template(template: &str) -> Result<(), TuliproxError> { + if template.is_empty() { + return Err(TuliproxError::ConfigRecording("recording.filename_template must not be empty".to_string())); + } + if template.len() > MAX_FILENAME_TEMPLATE_BYTES { + return Err(TuliproxError::ConfigRecording(format!( + "recording.filename_template must not exceed {MAX_FILENAME_TEMPLATE_BYTES} bytes" + ))); + } + + let bytes = template.as_bytes(); + let mut i = 0; + let mut found_placeholder = false; + while i < bytes.len() { + if bytes[i] == b'{' { + if let Some(close) = template[i + 1..].find('}') { + let end = i + 1 + close; + let placeholder = &template[i..=end]; + if !RECORDING_FILENAME_PLACEHOLDERS.contains(&placeholder) { + return Err(TuliproxError::ConfigRecording(format!( + "recording.filename_template contains unknown placeholder '{placeholder}'" + ))); + } + found_placeholder = true; + i = end + 1; + } else { + return Err(TuliproxError::ConfigRecording( + "recording.filename_template has an unmatched '{'".to_string(), + )); + } + } else if bytes[i] == b'}' { + return Err(TuliproxError::ConfigRecording("recording.filename_template has an unmatched '}'".to_string())); + } else { + i += 1; + } + } + + if !found_placeholder { + return Err(TuliproxError::ConfigRecording( + "recording.filename_template must contain at least one placeholder".to_string(), + )); + } + Ok(()) +} + +fn validate_recording_timezone(tz: &str) -> Result<(), TuliproxError> { + if tz.is_empty() { + return Err(TuliproxError::ConfigRecording("recording.timezone must not be empty".to_string())); + } + tz.parse::() + .map(|_| ()) + .map_err(|_| TuliproxError::ConfigRecording(format!("recording.timezone '{tz}' is not a valid IANA timezone"))) +} + +pub(crate) fn prepare_recording_config(recording: &mut RecordingConfigDto) -> Result<(), TuliproxError> { + if recording.headers.is_empty() { + recording.headers.insert("Accept".to_string(), "video/*".to_string()); + recording.headers.insert("User-Agent".to_string(), DEFAULT_USER_AGENT.to_string()); + } + if is_blank_or_default_episode_pattern(&recording.episode_pattern) { + recording.episode_pattern = default_episode_pattern(); + } else if let Some(pattern) = recording.episode_pattern.as_ref() { + recording.episode_pattern = Some(pattern.trim().to_string()); + } + if let Some(pattern) = recording.episode_pattern.as_ref() { + crate::model::REGEX_CACHE + .get_or_compile(pattern) + .map_err(|err| TuliproxError::RegexCompile(format!("{pattern} {err}")))?; + } + recording.retry_backoff_initial_secs = recording.retry_backoff_initial_secs.max(1); + recording.retry_backoff_multiplier = recording.retry_backoff_multiplier.max(1.0); + recording.retry_backoff_max_secs = recording.retry_backoff_max_secs.max(recording.retry_backoff_initial_secs); + recording.retry_backoff_jitter_percent = recording.retry_backoff_jitter_percent.min(95); + recording.retry_max_attempts = recording.retry_max_attempts.max(1); + + // Directory is independent of video extensions and download settings. + if let Some(dir) = recording.directory.as_ref() { + let trimmed = dir.trim(); + if trimmed.is_empty() { + recording.directory = None; + } else if trimmed != dir { + recording.directory = Some(trimmed.to_string()); + } + } + if recording.directory.is_none() { + recording.directory = default_recording_dir(); + } + + // timezone: default UTC; validate IANA. + if let Some(tz) = recording.timezone.as_ref() { + let trimmed = tz.trim(); + if trimmed.is_empty() { + recording.timezone = None; + } else if trimmed != tz { + recording.timezone = Some(trimmed.to_string()); + } + } + if let Some(tz) = recording.timezone.as_ref() { + if tz.parse::().is_err() { + // Surface a warning before the strict validator below turns + // this into a hard error, so the operator sees both signals. + log::warn!("recording.timezone '{tz}' is not a valid IANA timezone"); + } + } + if recording.timezone.is_none() { + recording.timezone = Some(default_recording_timezone()); + } + validate_recording_timezone(recording.timezone.as_deref().unwrap_or("UTC"))?; + + // filename_template: default; validate placeholders. + if let Some(template) = recording.filename_template.as_ref() { + let trimmed = template.trim(); + if trimmed != template { + recording.filename_template = Some(trimmed.to_string()); + } + } + if recording.filename_template.is_none() { + recording.filename_template = Some(default_recording_filename_template()); + } + validate_recording_filename_template(recording.filename_template.as_deref().unwrap_or(""))?; + + // padding: default <= max, fallback to defaults if missing. + if recording.default_pre_roll_secs.is_none() { + recording.default_pre_roll_secs = Some(0); + } + if recording.default_pre_roll_secs.unwrap_or(0) > recording.max_pre_roll_secs { + return Err(TuliproxError::ConfigRecording(format!( + "recording.default_pre_roll_secs ({}) must not exceed max_pre_roll_secs ({})", + recording.default_pre_roll_secs.unwrap_or(0), + recording.max_pre_roll_secs + ))); + } + if recording.default_post_roll_secs.is_none() { + recording.default_post_roll_secs = Some(0); + } + if recording.default_post_roll_secs.unwrap_or(0) > recording.max_post_roll_secs { + return Err(TuliproxError::ConfigRecording(format!( + "recording.default_post_roll_secs ({}) must not exceed max_post_roll_secs ({})", + recording.default_post_roll_secs.unwrap_or(0), + recording.max_post_roll_secs + ))); + } + + // retention: each present policy must be > 0. + if let Some(retention) = recording.retention.as_ref() { + if let Some(keep) = retention.keep_last_per_channel { + if keep == 0 { + return Err(TuliproxError::ConfigRecording( + "recording.retention.keep_last_per_channel must be > 0".to_string(), + )); + } + } + if let Some(days) = retention.delete_after_days { + if days == 0 { + return Err(TuliproxError::ConfigRecording( + "recording.retention.delete_after_days must be > 0".to_string(), + )); + } + } + if retention.sweep_interval_secs == 0 { + return Err(TuliproxError::ConfigRecording( + "recording.retention.sweep_interval_secs must be > 0".to_string(), + )); + } + if retention.keep_last_per_channel.is_none() && retention.delete_after_days.is_none() { + // A retention block that expresses no policy reads as + // "retention is configured" while nothing is ever deleted. + log::warn!( + "recording.retention has neither keep_last_per_channel nor delete_after_days; \ + no recording will ever be deleted by policy" + ); + } + } + + // notifications: an outbox that cannot hold or retry anything would + // silently drop every lifecycle notification. + if let Some(notifications) = recording.notifications.as_ref() { + if notifications.outbox_buffer == 0 { + return Err(TuliproxError::ConfigRecording( + "recording.notifications.outbox_buffer must be > 0".to_string(), + )); + } + if notifications.max_attempts == 0 { + return Err(TuliproxError::ConfigRecording("recording.notifications.max_attempts must be > 0".to_string())); + } + if notifications.backoff_initial_secs == 0 { + return Err(TuliproxError::ConfigRecording( + "recording.notifications.backoff_initial_secs must be > 0".to_string(), + )); + } + if notifications.backoff_max_secs < notifications.backoff_initial_secs { + return Err(TuliproxError::ConfigRecording(format!( + "recording.notifications.backoff_max_secs ({}) must be >= backoff_initial_secs ({})", + notifications.backoff_max_secs, notifications.backoff_initial_secs + ))); + } + } + + // disk: percentages in 0..=100, low < high, cleanup > 0, safety non-zero. + if let Some(disk) = recording.disk.as_ref() { + if let Some(high) = disk.high_water_percent { + if high > 100 { + return Err(TuliproxError::ConfigRecording(format!( + "recording.disk.high_water_percent ({high}) must be 0..=100" + ))); + } + } + if let Some(low) = disk.low_water_percent { + if low > 100 { + return Err(TuliproxError::ConfigRecording(format!( + "recording.disk.low_water_percent ({low}) must be 0..=100" + ))); + } + } + if let (Some(low), Some(high)) = (disk.low_water_percent, disk.high_water_percent) { + if low >= high { + return Err(TuliproxError::ConfigRecording(format!( + "recording.disk.low_water_percent ({low}) must be < high_water_percent ({high})" + ))); + } + } + if let Some(interval) = disk.cleanup_interval_secs { + if interval == 0 { + return Err(TuliproxError::ConfigRecording( + "recording.disk.cleanup_interval_secs must be > 0".to_string(), + )); + } + } + if let Some(safety) = disk.safety_bytes { + if safety == 0 { + return Err(TuliproxError::ConfigRecording("recording.disk.safety_bytes must be > 0".to_string())); + } + } + } + + // quota: fallback bytes per minute must be > 0. + if recording.fallback_bytes_per_minute == 0 { + return Err(TuliproxError::ConfigRecording("recording.fallback_bytes_per_minute must be > 0".to_string())); + } + + Ok(()) +} diff --git a/shared/src/model/config/recording/video.rs b/shared/src/model/config/recording/video.rs new file mode 100644 index 000000000..fcaf27e8c --- /dev/null +++ b/shared/src/model/config/recording/video.rs @@ -0,0 +1,47 @@ +use super::{prepare_recording_config, RecordingConfigDto}; +use crate::{ + defaults::{default_supported_video_extensions, is_default_supported_video_extensions}, + error::TuliproxError, + utils::{is_blank_optional_str, is_blank_optional_string}, +}; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct VideoConfigDto { + #[serde( + default = "default_supported_video_extensions", + skip_serializing_if = "is_default_supported_video_extensions" + )] + pub extensions: Vec, + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub web_search: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recording: Option, +} + +impl VideoConfigDto { + pub fn is_empty(&self) -> bool { + (self.extensions.is_empty() || is_default_supported_video_extensions(&self.extensions)) + && is_blank_optional_str(self.web_search.as_deref()) + && self.recording.as_ref().is_none_or(RecordingConfigDto::is_empty) + } + + pub fn clean(&mut self) { + if let Some(recording) = self.recording.as_mut() { + recording.clean(); + } + if self.recording.as_ref().is_some_and(RecordingConfigDto::is_empty) { + self.recording = None; + } + } + + pub fn prepare(&mut self) -> Result<(), TuliproxError> { + if self.extensions.is_empty() { + self.extensions = default_supported_video_extensions(); + } + if let Some(recording) = self.recording.as_mut() { + prepare_recording_config(recording)?; + } + Ok(()) + } +} diff --git a/shared/src/model/config/target.rs b/shared/src/model/config/target.rs index b6dc4c06e..51a9896f6 100644 --- a/shared/src/model/config/target.rs +++ b/shared/src/model/config/target.rs @@ -4,34 +4,12 @@ use crate::{ is_true, is_zero_u16, }, error::TuliproxError, - foundation::{get_filter, Filter}, model::{ - ClusterFlags, ConfigFavouritesDto, ConfigRenameDto, ConfigSortDto, CurationConfigDto, HdHomeRunDeviceOverview, - PatternTemplate, Prepare, PrepareAll, ProcessingOrder, StrmExportStyle, TargetType, TraktConfigDto, + ConfigFavouritesDto, ConfigRenameDto, ConfigSortDto, CurationConfigDto, HdHomeRunDeviceOverview, + PatternTemplate, Prepare, PrepareAll, ProcessingOrder, TargetType, }, - utils::is_blank_optional_string, }; -#[derive(Debug, Clone, Default)] -pub struct ConfigTargetFilterDto { - pub processing: Option, - pub persist: Option, - pub t_processing: Option, - pub t_persist: Option, -} - -impl PartialEq for ConfigTargetFilterDto { - fn eq(&self, other: &Self) -> bool { self.processing == other.processing && self.persist == other.persist } -} - -impl From for ConfigTargetFilterDto { - fn from(processing: String) -> Self { Self { processing: Some(processing), ..Self::default() } } -} - -impl From<&str> for ConfigTargetFilterDto { - fn from(processing: &str) -> Self { processing.to_string().into() } -} - #[derive(serde::Deserialize)] #[serde(deny_unknown_fields)] struct StagedTargetFilterDto { @@ -41,363 +19,6 @@ struct StagedTargetFilterDto { persist: Option, } -#[derive(serde::Deserialize)] -#[serde(untagged)] -enum TargetFilterRepr { - Processing(String), - Staged(StagedTargetFilterDto), -} - -impl<'de> serde::Deserialize<'de> for ConfigTargetFilterDto { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - match ::deserialize(deserializer)? { - TargetFilterRepr::Processing(processing) => Ok(processing.into()), - TargetFilterRepr::Staged(staged) => { - if staged.processing.is_none() && staged.persist.is_none() { - return Err(serde::de::Error::custom("staged target filter requires at least one stage")); - } - Ok(Self { processing: staged.processing, persist: staged.persist, ..Self::default() }) - } - } - } -} - -impl ConfigTargetFilterDto { - pub const fn is_empty(&self) -> bool { self.processing.is_none() && self.persist.is_none() } - - fn prepare(&mut self, templates: Option<&[PatternTemplate]>) -> Result<(), TuliproxError> { - fn compile_filter( - value: Option<&str>, - templates: Option<&[PatternTemplate]>, - ) -> Result, TuliproxError> { - value - .map(str::trim) - .filter(|filter| !filter.is_empty()) - .map(|filter| get_filter(filter, templates)) - .transpose() - } - - self.t_processing = compile_filter(self.processing.as_deref(), templates)?; - self.t_persist = compile_filter(self.persist.as_deref(), templates)?; - Ok(()) - } -} - -impl serde::Serialize for ConfigTargetFilterDto { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - use serde::ser::SerializeStruct; - - if self.persist.is_none() { - return serializer.serialize_str(self.processing.as_deref().unwrap_or_default()); - } - let field_count = usize::from(self.processing.is_some()) + usize::from(self.persist.is_some()); - let mut state = serializer.serialize_struct("ConfigTargetFilterDto", field_count)?; - if let Some(processing) = self.processing.as_ref() { - state.serialize_field("processing", processing)?; - } - if let Some(persist) = self.persist.as_ref() { - state.serialize_field("persist", persist)?; - } - state.end() - } -} - -#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ConfigTargetShareLiveStreams { - #[serde(default, skip_serializing_if = "is_false")] - pub hls: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub mpeg_ts: bool, -} - -#[derive(serde::Deserialize)] -#[serde(untagged)] -enum ConfigTargetShareLiveStreamsCompat { - Legacy(bool), - Structured(ConfigTargetShareLiveStreams), -} - -fn deserialize_share_live_streams<'de, D>(deserializer: D) -> Result -where - D: serde::Deserializer<'de>, -{ - Ok(match ::deserialize(deserializer)? { - ConfigTargetShareLiveStreamsCompat::Legacy(enabled) => { - ConfigTargetShareLiveStreams { hls: false, mpeg_ts: enabled } - } - ConfigTargetShareLiveStreamsCompat::Structured(config) => config, - }) -} - -impl ConfigTargetShareLiveStreams { - pub fn is_empty(&self) -> bool { !self.hls && !self.mpeg_ts } -} - -/// Controls optional canonicalization of EPG data emitted for a target. -#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct EpgOutputOptions { - #[serde(default, skip_serializing_if = "is_false")] - pub lowercase_ids: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub lowercase_xmltv_display_names: bool, -} - -impl EpgOutputOptions { - pub const fn is_empty(&self) -> bool { !self.lowercase_ids && !self.lowercase_xmltv_display_names } -} - -#[derive(Debug, Copy, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum DeduplicateMatchBy { - #[default] - Caption, - Name, - Title, -} - -#[derive(Debug, Copy, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum DeduplicateKeep { - #[default] - BestQuality, - First, -} - -/// Quality-aware duplicate removal: channels with the same normalized match -/// value (quality tokens stripped) collapse to a single entry. -#[derive(Debug, Copy, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct DeduplicateConfig { - #[serde(default)] - pub match_by: DeduplicateMatchBy, - #[serde(default)] - pub keep: DeduplicateKeep, - /// Normalize accented characters in match keys ("Café HD" matches "Cafe FHD"). - #[serde(default)] - pub match_as_ascii: bool, -} - -#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct ConfigTargetOptions { - #[serde(default, skip_serializing_if = "is_false")] - pub ignore_logo: bool, - #[serde(default, skip_serializing_if = "is_false")] - #[serde(alias = "required_epg")] - pub clear_invalid_epg_ids: bool, - #[serde( - default, - deserialize_with = "deserialize_share_live_streams", - skip_serializing_if = "ConfigTargetShareLiveStreams::is_empty" - )] - pub share_live_streams: ConfigTargetShareLiveStreams, - #[serde(default, skip_serializing_if = "is_false")] - pub remove_duplicates: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub deduplicate: Option, - #[serde(default, skip_serializing_if = "EpgOutputOptions::is_empty")] - pub epg_output: EpgOutputOptions, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub force_redirect: Option, -} - -impl ConfigTargetOptions { - pub fn is_empty(&self) -> bool { - !self.ignore_logo - && !self.clear_invalid_epg_ids - && self.share_live_streams.is_empty() - && !self.remove_duplicates - && self.deduplicate.is_none() - && self.epg_output.is_empty() - && self.force_redirect.is_none_or(|f| f.has_full_flags() || f.is_empty()) - } - - pub const fn lowercase_epg_ids(&self) -> bool { self.epg_output.lowercase_ids } - - pub const fn lowercase_xmltv_display_names(&self) -> bool { self.epg_output.lowercase_xmltv_display_names } - - pub const fn clear_invalid_epg_ids(&self) -> bool { self.clear_invalid_epg_ids } - - pub fn share_live_hls_enabled(&self) -> bool { self.share_live_streams.hls } - - pub fn share_live_mpeg_ts_enabled(&self) -> bool { self.share_live_streams.mpeg_ts } - - pub fn share_live_any_enabled(&self) -> bool { self.share_live_hls_enabled() || self.share_live_mpeg_ts_enabled() } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct XtreamTargetOutputDto { - #[serde(default = "default_as_true", skip_serializing_if = "is_true")] - pub skip_live_direct_source: bool, - #[serde(default = "default_as_true", skip_serializing_if = "is_true")] - pub skip_video_direct_source: bool, - #[serde(default = "default_as_true", skip_serializing_if = "is_true")] - pub skip_series_direct_source: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub trakt: Option, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub filter: Option, - #[serde(skip)] - pub t_filter: Option, -} - -impl Default for XtreamTargetOutputDto { - fn default() -> Self { - XtreamTargetOutputDto { - skip_live_direct_source: default_as_true(), - skip_video_direct_source: default_as_true(), - skip_series_direct_source: default_as_true(), - trakt: None, - filter: None, - t_filter: None, - } - } -} - -impl XtreamTargetOutputDto { - pub fn has_any_option(&self) -> bool { - self.skip_live_direct_source - || self.skip_video_direct_source - || self.skip_series_direct_source - || self.trakt.is_some() - || self.filter.is_some() - } -} - -impl Prepare for XtreamTargetOutputDto { - type Ctx<'a> = Option<&'a [PatternTemplate]>; - - fn prepare(&mut self, templates: Self::Ctx<'_>) -> Result<(), TuliproxError> { - if let Some(raw_filter) = &self.filter { - self.t_filter = Some(get_filter(raw_filter, templates)?); - } - if let Some(trakt) = &mut self.trakt { - trakt.prepare()?; - } - Ok(()) - } -} - -#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct M3uTargetOutputDto { - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub filename: Option, - #[serde(default, skip_serializing_if = "is_false")] - pub include_type_in_url: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub mask_redirect_url: bool, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub filter: Option, - #[serde(skip)] - pub t_filter: Option, -} - -impl M3uTargetOutputDto { - pub fn has_any_option(&self) -> bool { - self.filename.is_some() || self.include_type_in_url || self.mask_redirect_url || self.filter.is_some() - } -} - -impl Prepare for M3uTargetOutputDto { - type Ctx<'a> = Option<&'a [PatternTemplate]>; - - fn prepare(&mut self, templates: Self::Ctx<'_>) -> Result<(), TuliproxError> { - if let Some(raw_filter) = &self.filter { - self.t_filter = Some(get_filter(raw_filter, templates)?); - } - Ok(()) - } -} - -#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct StrmTargetOutputDto { - pub directory: String, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub username: Option, - #[serde(default)] - pub style: StrmExportStyle, - #[serde(default, skip_serializing_if = "is_false")] - pub flat: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub underscore_whitespace: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub cleanup: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub strm_props: Option>, - #[serde(default, skip_serializing_if = "is_blank_optional_string")] - pub filter: Option, - #[serde(default, skip_serializing_if = "is_false")] - pub add_quality_to_filename: bool, - #[serde(default, skip_serializing_if = "is_false")] - pub use_metadata: bool, - - // New Fields for Metadata and Probe - #[serde(default, skip_serializing_if = "Option::is_none")] - pub probe_probe_size_bytes: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub probe_analyze_duration: Option, - - #[serde(skip)] - pub t_filter: Option, -} - -impl Prepare for StrmTargetOutputDto { - type Ctx<'a> = Option<&'a [PatternTemplate]>; - - fn prepare(&mut self, templates: Self::Ctx<'_>) -> Result<(), TuliproxError> { - if let Some(raw_filter) = &self.filter { - self.t_filter = Some(get_filter(raw_filter, templates)?); - } - Ok(()) - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct HdHomeRunTargetOutputDto { - pub device: String, - pub username: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub use_output: Option, -} - -impl Default for HdHomeRunTargetOutputDto { - fn default() -> Self { Self { device: String::new(), username: String::new(), use_output: Some(TargetType::M3u) } } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -#[serde(deny_unknown_fields, tag = "type", rename_all = "lowercase")] -pub enum TargetOutputDto { - Xtream(XtreamTargetOutputDto), - M3u(M3uTargetOutputDto), - Strm(StrmTargetOutputDto), - HdHomeRun(HdHomeRunTargetOutputDto), -} - -impl Prepare for TargetOutputDto { - type Ctx<'a> = Option<&'a [PatternTemplate]>; - - fn prepare(&mut self, templates: Self::Ctx<'_>) -> Result<(), TuliproxError> { - match self { - TargetOutputDto::Xtream(output) => output.prepare(templates), - TargetOutputDto::M3u(output) => output.prepare(templates), - TargetOutputDto::Strm(output) => output.prepare(templates), - TargetOutputDto::HdHomeRun(_) => Ok(()), - } - } -} - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct ConfigTargetDto { @@ -633,426 +254,16 @@ impl ConfigTargetDto { } #[cfg(test)] -mod tests { - use super::{ - ConfigTargetDto, ConfigTargetFilterDto, ConfigTargetOptions, ConfigTargetShareLiveStreams, EpgOutputOptions, - M3uTargetOutputDto, StrmTargetOutputDto, TargetOutputDto, XtreamTargetOutputDto, - }; - - fn target_with_outputs(output: Vec) -> ConfigTargetDto { - ConfigTargetDto { - name: "target".to_string(), - filter: "Group ~ \".*\"".into(), - output, - ..ConfigTargetDto::default() - } - } - - fn strm_with_username() -> TargetOutputDto { - TargetOutputDto::Strm(StrmTargetOutputDto { - directory: "/tmp/strm".to_string(), - username: Some("alice".to_string()), - ..StrmTargetOutputDto::default() - }) - } - - fn strm_without_username() -> TargetOutputDto { - TargetOutputDto::Strm(StrmTargetOutputDto { - directory: "/tmp/strm".to_string(), - ..StrmTargetOutputDto::default() - }) - } - - fn xtream_output() -> TargetOutputDto { TargetOutputDto::Xtream(XtreamTargetOutputDto::default()) } - - #[test] - fn shipped_tmdb_example_is_safe_by_default_and_prepares_when_explicitly_enabled() { - let sources: crate::model::SourcesConfigDto = - serde_saphyr::from_str(include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../config/source.yml"))).unwrap(); - let mut target = sources - .sources - .into_iter() - .flat_map(|source| source.targets) - .find(|target| target.name == "tmdb-discovery-example") - .expect("shipped TMDB example"); - assert!(!target.enabled); - assert!(!target.curation.as_ref().unwrap().enabled); - assert!(target.curation.as_ref().unwrap().tmdb.as_ref().unwrap().api.access_token.is_empty()); - target.prepare(1, None, None).unwrap(); - target.enabled = true; - target.curation.as_mut().unwrap().enabled = true; - target.prepare(1, None, None).unwrap(); - let yaml = serde_saphyr::to_string(&target).unwrap(); - let restored: ConfigTargetDto = serde_saphyr::from_str(&yaml).unwrap(); - assert_eq!(restored.curation, target.curation); - } - - #[test] - fn target_curation_prepares_and_round_trips_tmdb_only_and_mixed_sources() { - for with_trakt in [false, true] { - let mut value = serde_json::json!({"name": "discovery", "output": [{"type": "xtream"}, {"type": "m3u"}], - "curation": {"catalog_selection": "curated", "tmdb": {"trending": [{"kind": "movie", "time_window": "week", "limit": 37, "category_name": "TMDB"}]}}}); - if with_trakt { - value["curation"]["trakt"] = - serde_json::json!({"charts": [{"kind": "movies", "chart": "popular", "category_name": "Trakt"}]}); - } - let mut target: ConfigTargetDto = serde_json::from_value(value).unwrap(); - target.prepare(1, None, None).unwrap(); - let restored: ConfigTargetDto = serde_saphyr::from_str(&serde_saphyr::to_string(&target).unwrap()).unwrap(); - assert_eq!(restored.curation, target.curation); - assert!(matches!(&restored.output[0], TargetOutputDto::Xtream(output) if output.trakt.is_none())); - } - } - - #[test] - fn target_curation_rejects_dual_declarations_before_preparing_legacy_values() { - let mut target: ConfigTargetDto = serde_json::from_value(serde_json::json!({"name": "discovery", - "curation": {"enabled": false}, "output": [{"type": "xtream", "trakt": {"catalog_selection": "curated"}}]})) - .unwrap(); - assert!(target.prepare(1, None, None).unwrap_err().to_string().contains("output[].trakt")); - } - - #[test] - fn target_curation_m3u_only_requires_selection_only_entries() { - let mut target: ConfigTargetDto = serde_json::from_value(serde_json::json!({"name": "discovery", "output": [{"type": "m3u"}], - "curation": {"catalog_selection": "curated", "tmdb": {"trending": [{"kind": "tv", "time_window": "day", "limit": 500, "category_name": "Shows"}]}}})).unwrap(); - assert!(target.prepare(1, None, None).is_err()); - target.curation.as_mut().unwrap().tmdb.as_mut().unwrap().trending[0].create_xtream_category = false; - target.prepare(1, None, None).unwrap(); - } - - #[test] - fn target_curation_absence_and_null_keep_the_legacy_surface() { - for value in [ - serde_json::json!({"name": "discovery", "output": [{"type": "xtream", "trakt": {}}]}), - serde_json::json!({"name": "discovery", "curation": null, "output": [{"type": "xtream", "trakt": {}}]}), - ] { - let mut target: ConfigTargetDto = serde_json::from_value(value).unwrap(); - target.prepare(1, None, None).unwrap(); - let serialized = serde_json::to_value(target).unwrap(); - assert!(serialized.get("curation").is_none()); - assert!(serialized["output"][0].get("trakt").is_some()); - } - } - - #[test] - fn target_filter_string_roundtrips_as_string() { - let dto = serde_saphyr::from_str::( - r#" -name: target -filter: 'Group ~ ".*"' -output: - - type: m3u -"#, - ) - .expect("legacy target filter should deserialize"); - - assert_eq!(dto.filter.processing.as_deref(), Some(r#"Group ~ ".*""#)); - assert_eq!(dto.filter.persist, None); - let serialized = serde_saphyr::to_string(&dto).expect("target should serialize"); - assert!(serialized.contains("filter: Group ~"), "expected scalar filter, got: {serialized}"); - assert!(!serialized.contains("processing:"), "processing-only filter must stay scalar: {serialized}"); - } - - #[test] - fn target_without_filter_defaults_to_no_stages_and_omits_filter() { - let dto = serde_saphyr::from_str::( - r" -name: target -output: - - type: m3u -", - ) - .expect("target without filter should deserialize"); - - assert!(dto.filter.is_empty()); - let serialized = serde_saphyr::to_string(&dto).expect("target should serialize"); - assert!(!serialized.contains("filter:"), "empty filter should be omitted: {serialized}"); - } - - #[test] - fn target_filter_stages_roundtrip_as_mapping() { - let dto = serde_saphyr::from_str::( - r#" -name: target -filter: - processing: 'Group ~ ".*"' - persist: 'EpgId ~ ".+"' -output: - - type: m3u -"#, - ) - .expect("staged target filter should deserialize"); - - assert_eq!(dto.filter.processing.as_deref(), Some(r#"Group ~ ".*""#)); - assert_eq!(dto.filter.persist.as_deref(), Some(r#"EpgId ~ ".+""#)); - let serialized = serde_saphyr::to_string(&dto).expect("target should serialize"); - assert!(serialized.contains("processing:"), "expected staged filter mapping: {serialized}"); - assert!(serialized.contains("persist:"), "expected persist filter: {serialized}"); - } - - #[test] - fn staged_target_filter_rejects_empty_mapping() { - let result = serde_saphyr::from_str::("{}\n"); - assert!(result.is_err()); - } - - #[test] - fn staged_target_filter_accepts_missing_processing_stage() { - let filter = serde_saphyr::from_str::( - r#"persist: 'EpgId ~ ".+"' -"#, - ) - .expect("persist-only filter should deserialize"); - - assert_eq!(filter.processing, None); - assert_eq!(filter.persist.as_deref(), Some(r#"EpgId ~ ".+""#)); - } - - #[test] - fn target_filter_treats_empty_and_whitespace_stages_as_match_all() { - for value in ["", " \t\r\n "] { - let mut filter = ConfigTargetFilterDto { - processing: Some(value.to_string()), - persist: Some(value.to_string()), - ..ConfigTargetFilterDto::default() - }; - - filter.prepare(None).expect("blank filter stages should prepare"); - - assert!(filter.t_processing.is_none()); - assert!(filter.t_persist.is_none()); - } - } - - #[test] - fn processing_only_mapping_serializes_as_scalar() { - let filter = serde_saphyr::from_str::( - r#"processing: 'Group ~ ".*"' -"#, - ) - .expect("processing-only mapping should deserialize"); - - let serialized = serde_saphyr::to_string(&filter).expect("processing filter should serialize"); - assert!(!serialized.contains("processing:"), "processing-only filter must become scalar: {serialized}"); - assert!(serialized.contains("Group ~")); - } - - #[test] - fn target_filter_persist_roundtrips_as_mapping() { - let filter = ConfigTargetFilterDto { - processing: None, - persist: Some(r#"EpgId ~ ".+""#.to_string()), - ..ConfigTargetFilterDto::default() - }; - - let serialized = serde_saphyr::to_string(&filter).expect("target filter should serialize"); - assert!(!serialized.contains("processing:")); - assert!(serialized.contains("persist:")); - let reparsed = - serde_saphyr::from_str::(&serialized).expect("target filter should deserialize"); - assert_eq!(reparsed, filter); - } - - #[test] - fn strm_with_username_is_allowed_with_m3u_output() { - let mut target = - target_with_outputs(vec![TargetOutputDto::M3u(M3uTargetOutputDto::default()), strm_with_username()]); - - assert!(target.prepare(1, None, None).is_ok()); - } - - #[test] - fn strm_with_username_is_allowed_with_xtream_output() { - let mut target = target_with_outputs(vec![xtream_output(), strm_with_username()]); - - assert!(target.prepare(1, None, None).is_ok()); - } - - #[test] - fn strm_without_username_is_allowed_with_m3u_output() { - let mut target = - target_with_outputs(vec![TargetOutputDto::M3u(M3uTargetOutputDto::default()), strm_without_username()]); - - assert!(target.prepare(1, None, None).is_ok()); - } - - #[test] - fn strm_with_username_requires_m3u_or_xtream_output() { - let mut target = target_with_outputs(vec![strm_with_username()]); - - let err = target.prepare(1, None, None).expect_err("STRM username without stream output should fail"); - - assert!(err.to_string().contains("xtream or m3u output")); - } - - #[test] - fn strm_without_username_requires_m3u_or_xtream_output() { - let mut target = target_with_outputs(vec![strm_without_username()]); - - let err = target.prepare(1, None, None).expect_err("STRM without stream output should fail"); - - assert!(err.to_string().contains("xtream or m3u output")); - } - - #[test] - fn target_options_deserialize_structured_share_live_streams() { - let yaml = r" -share_live_streams: - hls: true - mpeg_ts: true -"; - - let options: ConfigTargetOptions = - serde_saphyr::from_str(yaml).expect("structured share_live_streams should deserialize"); - - assert!(options.share_live_hls_enabled()); - assert!(options.share_live_mpeg_ts_enabled()); - assert!(options.share_live_any_enabled()); - } - - #[test] - fn target_options_maps_legacy_true_share_live_streams_to_both_modes() { - let yaml = r" -share_live_streams: true -"; - - let options = serde_saphyr::from_str::(yaml); - - assert!(options.is_ok(), "legacy boolean should deserialize: {options:?}"); - if let Ok(options) = options { - assert_eq!(options.share_live_streams, ConfigTargetShareLiveStreams { hls: false, mpeg_ts: true }); - } - } - - #[test] - fn target_options_maps_legacy_false_share_live_streams_to_both_modes() { - let yaml = r" -share_live_streams: false -"; - - let options: ConfigTargetOptions = - serde_saphyr::from_str(yaml).expect("legacy false share_live_streams should deserialize"); - - assert_eq!(options.share_live_streams, ConfigTargetShareLiveStreams { hls: false, mpeg_ts: false }); - } - - #[test] - fn target_options_omit_default_share_live_streams() { - let options = ConfigTargetOptions::default(); - - assert!(options.is_empty()); - assert!(!options.clear_invalid_epg_ids()); - - let serialized = serde_saphyr::to_string(&options).expect("default options should serialize"); - assert!( - !serialized.contains("share_live_streams"), - "default share_live_streams should be omitted, got: {serialized}" - ); - } - - #[test] - fn target_options_round_trips_partial_share_live_streams() { - let options = ConfigTargetOptions { - share_live_streams: ConfigTargetShareLiveStreams { hls: true, mpeg_ts: false }, - ..ConfigTargetOptions::default() - }; - - let serialized = serde_saphyr::to_string(&options).expect("partial share_live_streams should serialize"); - let reparsed: ConfigTargetOptions = - serde_saphyr::from_str(&serialized).expect("partial share_live_streams should deserialize"); - - assert_eq!(reparsed.share_live_streams, options.share_live_streams); - } - - #[test] - fn target_options_clear_invalid_epg_ids_roundtrips_and_accepts_legacy_alias() { - let options = serde_saphyr::from_str::("required_epg: true\n") - .expect("legacy required_epg should deserialize"); - - assert!(options.clear_invalid_epg_ids()); - assert!(!options.is_empty()); - - let serialized = serde_saphyr::to_string(&options).expect("clear_invalid_epg_ids should serialize"); - assert!(serialized.contains("clear_invalid_epg_ids: true")); - assert!(!serialized.contains("required_epg:")); - } - - #[test] - fn target_options_default_epg_output_is_disabled_and_omitted() { - let options = serde_saphyr::from_str::("{}") - .expect("target options without epg_output should deserialize"); - - assert!(!options.lowercase_epg_ids()); - assert!(!options.lowercase_xmltv_display_names()); - assert!(options.epg_output.is_empty()); - assert!(options.is_empty()); - - let serialized = serde_saphyr::to_string(&options).expect("default target options should serialize"); - assert!(!serialized.contains("epg_output"), "default epg_output should be omitted, got: {serialized}"); - } - - #[test] - fn target_options_epg_output_roundtrips() { - let yaml = r" -epg_output: - lowercase_ids: true - lowercase_xmltv_display_names: true -"; - - let options = - serde_saphyr::from_str::(yaml).expect("configured epg_output should deserialize"); - - assert!(options.lowercase_epg_ids()); - assert!(options.lowercase_xmltv_display_names()); - assert!(!options.is_empty()); - - let serialized = serde_saphyr::to_string(&options).expect("configured epg_output should serialize"); - let roundtripped = serde_saphyr::from_str::(&serialized) - .expect("serialized epg_output should deserialize"); - assert_eq!(roundtripped, options); - } - - #[test] - fn target_options_epg_output_makes_options_nonempty() { - let lowercase_ids = ConfigTargetOptions { - epg_output: EpgOutputOptions { lowercase_ids: true, ..EpgOutputOptions::default() }, - ..ConfigTargetOptions::default() - }; - let lowercase_display_names = ConfigTargetOptions { - epg_output: EpgOutputOptions { lowercase_xmltv_display_names: true, ..EpgOutputOptions::default() }, - ..ConfigTargetOptions::default() - }; - - assert!(!lowercase_ids.is_empty()); - assert!(!lowercase_display_names.is_empty()); - } - - #[test] - fn target_options_reject_unknown_epg_output_fields() { - let yaml = r" -epg_output: - lowercase_id: true -"; - - let result = serde_saphyr::from_str::(yaml); - - assert!(result.is_err(), "unknown epg_output fields must be rejected"); - } - - #[test] - fn target_options_mpeg_ts_helper_keeps_existing_stream_share_semantics() { - let hls_only = ConfigTargetOptions { - share_live_streams: ConfigTargetShareLiveStreams { hls: true, mpeg_ts: false }, - ..Default::default() - }; - let mpeg_ts = ConfigTargetOptions { - share_live_streams: ConfigTargetShareLiveStreams { hls: false, mpeg_ts: true }, - ..Default::default() - }; - - assert!(hls_only.share_live_hls_enabled()); - assert!(!hls_only.share_live_mpeg_ts_enabled()); - assert!(mpeg_ts.share_live_mpeg_ts_enabled()); - } -} +mod tests; + +mod filter; +mod options; +mod output; +pub use filter::ConfigTargetFilterDto; +pub use options::{ + ConfigTargetOptions, ConfigTargetShareLiveStreams, DeduplicateConfig, DeduplicateKeep, DeduplicateMatchBy, + EpgOutputOptions, +}; +pub use output::{ + HdHomeRunTargetOutputDto, M3uTargetOutputDto, StrmTargetOutputDto, TargetOutputDto, XtreamTargetOutputDto, +}; diff --git a/shared/src/model/config/target/filter.rs b/shared/src/model/config/target/filter.rs new file mode 100644 index 000000000..5ec9f25e0 --- /dev/null +++ b/shared/src/model/config/target/filter.rs @@ -0,0 +1,93 @@ +use super::StagedTargetFilterDto; +use crate::{ + error::TuliproxError, + foundation::{get_filter, Filter}, + model::PatternTemplate, +}; + +#[derive(Debug, Clone, Default)] +pub struct ConfigTargetFilterDto { + pub processing: Option, + pub persist: Option, + pub t_processing: Option, + pub t_persist: Option, +} + +impl PartialEq for ConfigTargetFilterDto { + fn eq(&self, other: &Self) -> bool { self.processing == other.processing && self.persist == other.persist } +} + +impl From for ConfigTargetFilterDto { + fn from(processing: String) -> Self { Self { processing: Some(processing), ..Self::default() } } +} + +impl From<&str> for ConfigTargetFilterDto { + fn from(processing: &str) -> Self { processing.to_string().into() } +} + +#[derive(serde::Deserialize)] +#[serde(untagged)] +enum TargetFilterRepr { + Processing(String), + Staged(StagedTargetFilterDto), +} + +impl<'de> serde::Deserialize<'de> for ConfigTargetFilterDto { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + match ::deserialize(deserializer)? { + TargetFilterRepr::Processing(processing) => Ok(processing.into()), + TargetFilterRepr::Staged(staged) => { + if staged.processing.is_none() && staged.persist.is_none() { + return Err(serde::de::Error::custom("staged target filter requires at least one stage")); + } + Ok(Self { processing: staged.processing, persist: staged.persist, ..Self::default() }) + } + } + } +} + +impl ConfigTargetFilterDto { + pub const fn is_empty(&self) -> bool { self.processing.is_none() && self.persist.is_none() } + + pub(super) fn prepare(&mut self, templates: Option<&[PatternTemplate]>) -> Result<(), TuliproxError> { + fn compile_filter( + value: Option<&str>, + templates: Option<&[PatternTemplate]>, + ) -> Result, TuliproxError> { + value + .map(str::trim) + .filter(|filter| !filter.is_empty()) + .map(|filter| get_filter(filter, templates)) + .transpose() + } + + self.t_processing = compile_filter(self.processing.as_deref(), templates)?; + self.t_persist = compile_filter(self.persist.as_deref(), templates)?; + Ok(()) + } +} + +impl serde::Serialize for ConfigTargetFilterDto { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + + if self.persist.is_none() { + return serializer.serialize_str(self.processing.as_deref().unwrap_or_default()); + } + let field_count = usize::from(self.processing.is_some()) + usize::from(self.persist.is_some()); + let mut state = serializer.serialize_struct("ConfigTargetFilterDto", field_count)?; + if let Some(processing) = self.processing.as_ref() { + state.serialize_field("processing", processing)?; + } + if let Some(persist) = self.persist.as_ref() { + state.serialize_field("persist", persist)?; + } + state.end() + } +} diff --git a/shared/src/model/config/target/options.rs b/shared/src/model/config/target/options.rs new file mode 100644 index 000000000..8109f4e84 --- /dev/null +++ b/shared/src/model/config/target/options.rs @@ -0,0 +1,126 @@ +use crate::{defaults::is_false, model::ClusterFlags}; + +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ConfigTargetShareLiveStreams { + #[serde(default, skip_serializing_if = "is_false")] + pub hls: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub mpeg_ts: bool, +} + +#[derive(serde::Deserialize)] +#[serde(untagged)] +enum ConfigTargetShareLiveStreamsCompat { + Legacy(bool), + Structured(ConfigTargetShareLiveStreams), +} + +fn deserialize_share_live_streams<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + Ok(match ::deserialize(deserializer)? { + ConfigTargetShareLiveStreamsCompat::Legacy(enabled) => { + ConfigTargetShareLiveStreams { hls: false, mpeg_ts: enabled } + } + ConfigTargetShareLiveStreamsCompat::Structured(config) => config, + }) +} + +impl ConfigTargetShareLiveStreams { + pub fn is_empty(&self) -> bool { !self.hls && !self.mpeg_ts } +} + +/// Controls optional canonicalization of EPG data emitted for a target. +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct EpgOutputOptions { + #[serde(default, skip_serializing_if = "is_false")] + pub lowercase_ids: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub lowercase_xmltv_display_names: bool, +} + +impl EpgOutputOptions { + pub const fn is_empty(&self) -> bool { !self.lowercase_ids && !self.lowercase_xmltv_display_names } +} + +#[derive(Debug, Copy, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum DeduplicateMatchBy { + #[default] + Caption, + Name, + Title, +} + +#[derive(Debug, Copy, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum DeduplicateKeep { + #[default] + BestQuality, + First, +} + +/// Quality-aware duplicate removal: channels with the same normalized match +/// value (quality tokens stripped) collapse to a single entry. +#[derive(Debug, Copy, Clone, Default, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct DeduplicateConfig { + #[serde(default)] + pub match_by: DeduplicateMatchBy, + #[serde(default)] + pub keep: DeduplicateKeep, + /// Normalize accented characters in match keys ("Café HD" matches "Cafe FHD"). + #[serde(default)] + pub match_as_ascii: bool, +} + +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ConfigTargetOptions { + #[serde(default, skip_serializing_if = "is_false")] + pub ignore_logo: bool, + #[serde(default, skip_serializing_if = "is_false")] + #[serde(alias = "required_epg")] + pub clear_invalid_epg_ids: bool, + #[serde( + default, + deserialize_with = "deserialize_share_live_streams", + skip_serializing_if = "ConfigTargetShareLiveStreams::is_empty" + )] + pub share_live_streams: ConfigTargetShareLiveStreams, + #[serde(default, skip_serializing_if = "is_false")] + pub remove_duplicates: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deduplicate: Option, + #[serde(default, skip_serializing_if = "EpgOutputOptions::is_empty")] + pub epg_output: EpgOutputOptions, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub force_redirect: Option, +} + +impl ConfigTargetOptions { + pub fn is_empty(&self) -> bool { + !self.ignore_logo + && !self.clear_invalid_epg_ids + && self.share_live_streams.is_empty() + && !self.remove_duplicates + && self.deduplicate.is_none() + && self.epg_output.is_empty() + && self.force_redirect.is_none_or(|f| f.has_full_flags() || f.is_empty()) + } + + pub const fn lowercase_epg_ids(&self) -> bool { self.epg_output.lowercase_ids } + + pub const fn lowercase_xmltv_display_names(&self) -> bool { self.epg_output.lowercase_xmltv_display_names } + + pub const fn clear_invalid_epg_ids(&self) -> bool { self.clear_invalid_epg_ids } + + pub fn share_live_hls_enabled(&self) -> bool { self.share_live_streams.hls } + + pub fn share_live_mpeg_ts_enabled(&self) -> bool { self.share_live_streams.mpeg_ts } + + pub fn share_live_any_enabled(&self) -> bool { self.share_live_hls_enabled() || self.share_live_mpeg_ts_enabled() } +} diff --git a/shared/src/model/config/target/output.rs b/shared/src/model/config/target/output.rs new file mode 100644 index 000000000..08df2d271 --- /dev/null +++ b/shared/src/model/config/target/output.rs @@ -0,0 +1,172 @@ +use crate::{ + defaults::{default_as_true, is_false, is_true}, + error::TuliproxError, + foundation::{get_filter, Filter}, + model::{PatternTemplate, Prepare, StrmExportStyle, TargetType, TraktConfigDto}, + utils::is_blank_optional_string, +}; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct XtreamTargetOutputDto { + #[serde(default = "default_as_true", skip_serializing_if = "is_true")] + pub skip_live_direct_source: bool, + #[serde(default = "default_as_true", skip_serializing_if = "is_true")] + pub skip_video_direct_source: bool, + #[serde(default = "default_as_true", skip_serializing_if = "is_true")] + pub skip_series_direct_source: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trakt: Option, + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub filter: Option, + #[serde(skip)] + pub t_filter: Option, +} + +impl Default for XtreamTargetOutputDto { + fn default() -> Self { + XtreamTargetOutputDto { + skip_live_direct_source: default_as_true(), + skip_video_direct_source: default_as_true(), + skip_series_direct_source: default_as_true(), + trakt: None, + filter: None, + t_filter: None, + } + } +} + +impl XtreamTargetOutputDto { + pub fn has_any_option(&self) -> bool { + self.skip_live_direct_source + || self.skip_video_direct_source + || self.skip_series_direct_source + || self.trakt.is_some() + || self.filter.is_some() + } +} + +impl Prepare for XtreamTargetOutputDto { + type Ctx<'a> = Option<&'a [PatternTemplate]>; + + fn prepare(&mut self, templates: Self::Ctx<'_>) -> Result<(), TuliproxError> { + if let Some(raw_filter) = &self.filter { + self.t_filter = Some(get_filter(raw_filter, templates)?); + } + if let Some(trakt) = &mut self.trakt { + trakt.prepare()?; + } + Ok(()) + } +} + +#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct M3uTargetOutputDto { + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub filename: Option, + #[serde(default, skip_serializing_if = "is_false")] + pub include_type_in_url: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub mask_redirect_url: bool, + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub filter: Option, + #[serde(skip)] + pub t_filter: Option, +} + +impl M3uTargetOutputDto { + pub fn has_any_option(&self) -> bool { + self.filename.is_some() || self.include_type_in_url || self.mask_redirect_url || self.filter.is_some() + } +} + +impl Prepare for M3uTargetOutputDto { + type Ctx<'a> = Option<&'a [PatternTemplate]>; + + fn prepare(&mut self, templates: Self::Ctx<'_>) -> Result<(), TuliproxError> { + if let Some(raw_filter) = &self.filter { + self.t_filter = Some(get_filter(raw_filter, templates)?); + } + Ok(()) + } +} + +#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct StrmTargetOutputDto { + pub directory: String, + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub username: Option, + #[serde(default)] + pub style: StrmExportStyle, + #[serde(default, skip_serializing_if = "is_false")] + pub flat: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub underscore_whitespace: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub cleanup: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub strm_props: Option>, + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub filter: Option, + #[serde(default, skip_serializing_if = "is_false")] + pub add_quality_to_filename: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub use_metadata: bool, + + // New Fields for Metadata and Probe + #[serde(default, skip_serializing_if = "Option::is_none")] + pub probe_probe_size_bytes: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub probe_analyze_duration: Option, + + #[serde(skip)] + pub t_filter: Option, +} + +impl Prepare for StrmTargetOutputDto { + type Ctx<'a> = Option<&'a [PatternTemplate]>; + + fn prepare(&mut self, templates: Self::Ctx<'_>) -> Result<(), TuliproxError> { + if let Some(raw_filter) = &self.filter { + self.t_filter = Some(get_filter(raw_filter, templates)?); + } + Ok(()) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct HdHomeRunTargetOutputDto { + pub device: String, + pub username: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub use_output: Option, +} + +impl Default for HdHomeRunTargetOutputDto { + fn default() -> Self { Self { device: String::new(), username: String::new(), use_output: Some(TargetType::M3u) } } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields, tag = "type", rename_all = "lowercase")] +pub enum TargetOutputDto { + Xtream(XtreamTargetOutputDto), + M3u(M3uTargetOutputDto), + Strm(StrmTargetOutputDto), + HdHomeRun(HdHomeRunTargetOutputDto), +} + +impl Prepare for TargetOutputDto { + type Ctx<'a> = Option<&'a [PatternTemplate]>; + + fn prepare(&mut self, templates: Self::Ctx<'_>) -> Result<(), TuliproxError> { + match self { + TargetOutputDto::Xtream(output) => output.prepare(templates), + TargetOutputDto::M3u(output) => output.prepare(templates), + TargetOutputDto::Strm(output) => output.prepare(templates), + TargetOutputDto::HdHomeRun(_) => Ok(()), + } + } +} diff --git a/shared/src/model/config/target/tests.rs b/shared/src/model/config/target/tests.rs new file mode 100644 index 000000000..a9f50b74e --- /dev/null +++ b/shared/src/model/config/target/tests.rs @@ -0,0 +1,418 @@ +use super::{ + ConfigTargetDto, ConfigTargetFilterDto, ConfigTargetOptions, ConfigTargetShareLiveStreams, EpgOutputOptions, + M3uTargetOutputDto, StrmTargetOutputDto, TargetOutputDto, XtreamTargetOutputDto, +}; + +fn target_with_outputs(output: Vec) -> ConfigTargetDto { + ConfigTargetDto { + name: "target".to_string(), + filter: "Group ~ \".*\"".into(), + output, + ..ConfigTargetDto::default() + } +} + +fn strm_with_username() -> TargetOutputDto { + TargetOutputDto::Strm(StrmTargetOutputDto { + directory: "/tmp/strm".to_string(), + username: Some("alice".to_string()), + ..StrmTargetOutputDto::default() + }) +} + +fn strm_without_username() -> TargetOutputDto { + TargetOutputDto::Strm(StrmTargetOutputDto { directory: "/tmp/strm".to_string(), ..StrmTargetOutputDto::default() }) +} + +fn xtream_output() -> TargetOutputDto { TargetOutputDto::Xtream(XtreamTargetOutputDto::default()) } + +#[test] +fn shipped_tmdb_example_is_safe_by_default_and_prepares_when_explicitly_enabled() { + let sources: crate::model::SourcesConfigDto = + serde_saphyr::from_str(include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../config/source.yml"))).unwrap(); + let mut target = sources + .sources + .into_iter() + .flat_map(|source| source.targets) + .find(|target| target.name == "tmdb-discovery-example") + .expect("shipped TMDB example"); + assert!(!target.enabled); + assert!(!target.curation.as_ref().unwrap().enabled); + assert!(target.curation.as_ref().unwrap().tmdb.as_ref().unwrap().api.access_token.is_empty()); + target.prepare(1, None, None).unwrap(); + target.enabled = true; + target.curation.as_mut().unwrap().enabled = true; + target.prepare(1, None, None).unwrap(); + let yaml = serde_saphyr::to_string(&target).unwrap(); + let restored: ConfigTargetDto = serde_saphyr::from_str(&yaml).unwrap(); + assert_eq!(restored.curation, target.curation); +} + +#[test] +fn target_curation_prepares_and_round_trips_tmdb_only_and_mixed_sources() { + for with_trakt in [false, true] { + let mut value = serde_json::json!({"name": "discovery", "output": [{"type": "xtream"}, {"type": "m3u"}], + "curation": {"catalog_selection": "curated", "tmdb": {"trending": [{"kind": "movie", "time_window": "week", "limit": 37, "category_name": "TMDB"}]}}}); + if with_trakt { + value["curation"]["trakt"] = + serde_json::json!({"charts": [{"kind": "movies", "chart": "popular", "category_name": "Trakt"}]}); + } + let mut target: ConfigTargetDto = serde_json::from_value(value).unwrap(); + target.prepare(1, None, None).unwrap(); + let restored: ConfigTargetDto = serde_saphyr::from_str(&serde_saphyr::to_string(&target).unwrap()).unwrap(); + assert_eq!(restored.curation, target.curation); + assert!(matches!(&restored.output[0], TargetOutputDto::Xtream(output) if output.trakt.is_none())); + } +} + +#[test] +fn target_curation_rejects_dual_declarations_before_preparing_legacy_values() { + let mut target: ConfigTargetDto = serde_json::from_value(serde_json::json!({"name": "discovery", + "curation": {"enabled": false}, "output": [{"type": "xtream", "trakt": {"catalog_selection": "curated"}}]})) + .unwrap(); + assert!(target.prepare(1, None, None).unwrap_err().to_string().contains("output[].trakt")); +} + +#[test] +fn target_curation_m3u_only_requires_selection_only_entries() { + let mut target: ConfigTargetDto = serde_json::from_value(serde_json::json!({"name": "discovery", "output": [{"type": "m3u"}], + "curation": {"catalog_selection": "curated", "tmdb": {"trending": [{"kind": "tv", "time_window": "day", "limit": 500, "category_name": "Shows"}]}}})).unwrap(); + assert!(target.prepare(1, None, None).is_err()); + target.curation.as_mut().unwrap().tmdb.as_mut().unwrap().trending[0].create_xtream_category = false; + target.prepare(1, None, None).unwrap(); +} + +#[test] +fn target_curation_absence_and_null_keep_the_legacy_surface() { + for value in [ + serde_json::json!({"name": "discovery", "output": [{"type": "xtream", "trakt": {}}]}), + serde_json::json!({"name": "discovery", "curation": null, "output": [{"type": "xtream", "trakt": {}}]}), + ] { + let mut target: ConfigTargetDto = serde_json::from_value(value).unwrap(); + target.prepare(1, None, None).unwrap(); + let serialized = serde_json::to_value(target).unwrap(); + assert!(serialized.get("curation").is_none()); + assert!(serialized["output"][0].get("trakt").is_some()); + } +} + +#[test] +fn target_filter_string_roundtrips_as_string() { + let dto = serde_saphyr::from_str::( + r#" +name: target +filter: 'Group ~ ".*"' +output: + - type: m3u +"#, + ) + .expect("legacy target filter should deserialize"); + + assert_eq!(dto.filter.processing.as_deref(), Some(r#"Group ~ ".*""#)); + assert_eq!(dto.filter.persist, None); + let serialized = serde_saphyr::to_string(&dto).expect("target should serialize"); + assert!(serialized.contains("filter: Group ~"), "expected scalar filter, got: {serialized}"); + assert!(!serialized.contains("processing:"), "processing-only filter must stay scalar: {serialized}"); +} + +#[test] +fn target_without_filter_defaults_to_no_stages_and_omits_filter() { + let dto = serde_saphyr::from_str::( + r" +name: target +output: + - type: m3u +", + ) + .expect("target without filter should deserialize"); + + assert!(dto.filter.is_empty()); + let serialized = serde_saphyr::to_string(&dto).expect("target should serialize"); + assert!(!serialized.contains("filter:"), "empty filter should be omitted: {serialized}"); +} + +#[test] +fn target_filter_stages_roundtrip_as_mapping() { + let dto = serde_saphyr::from_str::( + r#" +name: target +filter: + processing: 'Group ~ ".*"' + persist: 'EpgId ~ ".+"' +output: + - type: m3u +"#, + ) + .expect("staged target filter should deserialize"); + + assert_eq!(dto.filter.processing.as_deref(), Some(r#"Group ~ ".*""#)); + assert_eq!(dto.filter.persist.as_deref(), Some(r#"EpgId ~ ".+""#)); + let serialized = serde_saphyr::to_string(&dto).expect("target should serialize"); + assert!(serialized.contains("processing:"), "expected staged filter mapping: {serialized}"); + assert!(serialized.contains("persist:"), "expected persist filter: {serialized}"); +} + +#[test] +fn staged_target_filter_rejects_empty_mapping() { + let result = serde_saphyr::from_str::("{}\n"); + assert!(result.is_err()); +} + +#[test] +fn staged_target_filter_accepts_missing_processing_stage() { + let filter = serde_saphyr::from_str::( + r#"persist: 'EpgId ~ ".+"' +"#, + ) + .expect("persist-only filter should deserialize"); + + assert_eq!(filter.processing, None); + assert_eq!(filter.persist.as_deref(), Some(r#"EpgId ~ ".+""#)); +} + +#[test] +fn target_filter_treats_empty_and_whitespace_stages_as_match_all() { + for value in ["", " \t\r\n "] { + let mut filter = ConfigTargetFilterDto { + processing: Some(value.to_string()), + persist: Some(value.to_string()), + ..ConfigTargetFilterDto::default() + }; + + filter.prepare(None).expect("blank filter stages should prepare"); + + assert!(filter.t_processing.is_none()); + assert!(filter.t_persist.is_none()); + } +} + +#[test] +fn processing_only_mapping_serializes_as_scalar() { + let filter = serde_saphyr::from_str::( + r#"processing: 'Group ~ ".*"' +"#, + ) + .expect("processing-only mapping should deserialize"); + + let serialized = serde_saphyr::to_string(&filter).expect("processing filter should serialize"); + assert!(!serialized.contains("processing:"), "processing-only filter must become scalar: {serialized}"); + assert!(serialized.contains("Group ~")); +} + +#[test] +fn target_filter_persist_roundtrips_as_mapping() { + let filter = ConfigTargetFilterDto { + processing: None, + persist: Some(r#"EpgId ~ ".+""#.to_string()), + ..ConfigTargetFilterDto::default() + }; + + let serialized = serde_saphyr::to_string(&filter).expect("target filter should serialize"); + assert!(!serialized.contains("processing:")); + assert!(serialized.contains("persist:")); + let reparsed = + serde_saphyr::from_str::(&serialized).expect("target filter should deserialize"); + assert_eq!(reparsed, filter); +} + +#[test] +fn strm_with_username_is_allowed_with_m3u_output() { + let mut target = + target_with_outputs(vec![TargetOutputDto::M3u(M3uTargetOutputDto::default()), strm_with_username()]); + + assert!(target.prepare(1, None, None).is_ok()); +} + +#[test] +fn strm_with_username_is_allowed_with_xtream_output() { + let mut target = target_with_outputs(vec![xtream_output(), strm_with_username()]); + + assert!(target.prepare(1, None, None).is_ok()); +} + +#[test] +fn strm_without_username_is_allowed_with_m3u_output() { + let mut target = + target_with_outputs(vec![TargetOutputDto::M3u(M3uTargetOutputDto::default()), strm_without_username()]); + + assert!(target.prepare(1, None, None).is_ok()); +} + +#[test] +fn strm_with_username_requires_m3u_or_xtream_output() { + let mut target = target_with_outputs(vec![strm_with_username()]); + + let err = target.prepare(1, None, None).expect_err("STRM username without stream output should fail"); + + assert!(err.to_string().contains("xtream or m3u output")); +} + +#[test] +fn strm_without_username_requires_m3u_or_xtream_output() { + let mut target = target_with_outputs(vec![strm_without_username()]); + + let err = target.prepare(1, None, None).expect_err("STRM without stream output should fail"); + + assert!(err.to_string().contains("xtream or m3u output")); +} + +#[test] +fn target_options_deserialize_structured_share_live_streams() { + let yaml = r" +share_live_streams: + hls: true + mpeg_ts: true +"; + + let options: ConfigTargetOptions = + serde_saphyr::from_str(yaml).expect("structured share_live_streams should deserialize"); + + assert!(options.share_live_hls_enabled()); + assert!(options.share_live_mpeg_ts_enabled()); + assert!(options.share_live_any_enabled()); +} + +#[test] +fn target_options_maps_legacy_true_share_live_streams_to_both_modes() { + let yaml = r" +share_live_streams: true +"; + + let options = serde_saphyr::from_str::(yaml); + + assert!(options.is_ok(), "legacy boolean should deserialize: {options:?}"); + if let Ok(options) = options { + assert_eq!(options.share_live_streams, ConfigTargetShareLiveStreams { hls: false, mpeg_ts: true }); + } +} + +#[test] +fn target_options_maps_legacy_false_share_live_streams_to_both_modes() { + let yaml = r" +share_live_streams: false +"; + + let options: ConfigTargetOptions = + serde_saphyr::from_str(yaml).expect("legacy false share_live_streams should deserialize"); + + assert_eq!(options.share_live_streams, ConfigTargetShareLiveStreams { hls: false, mpeg_ts: false }); +} + +#[test] +fn target_options_omit_default_share_live_streams() { + let options = ConfigTargetOptions::default(); + + assert!(options.is_empty()); + assert!(!options.clear_invalid_epg_ids()); + + let serialized = serde_saphyr::to_string(&options).expect("default options should serialize"); + assert!( + !serialized.contains("share_live_streams"), + "default share_live_streams should be omitted, got: {serialized}" + ); +} + +#[test] +fn target_options_round_trips_partial_share_live_streams() { + let options = ConfigTargetOptions { + share_live_streams: ConfigTargetShareLiveStreams { hls: true, mpeg_ts: false }, + ..ConfigTargetOptions::default() + }; + + let serialized = serde_saphyr::to_string(&options).expect("partial share_live_streams should serialize"); + let reparsed: ConfigTargetOptions = + serde_saphyr::from_str(&serialized).expect("partial share_live_streams should deserialize"); + + assert_eq!(reparsed.share_live_streams, options.share_live_streams); +} + +#[test] +fn target_options_clear_invalid_epg_ids_roundtrips_and_accepts_legacy_alias() { + let options = serde_saphyr::from_str::("required_epg: true\n") + .expect("legacy required_epg should deserialize"); + + assert!(options.clear_invalid_epg_ids()); + assert!(!options.is_empty()); + + let serialized = serde_saphyr::to_string(&options).expect("clear_invalid_epg_ids should serialize"); + assert!(serialized.contains("clear_invalid_epg_ids: true")); + assert!(!serialized.contains("required_epg:")); +} + +#[test] +fn target_options_default_epg_output_is_disabled_and_omitted() { + let options = serde_saphyr::from_str::("{}") + .expect("target options without epg_output should deserialize"); + + assert!(!options.lowercase_epg_ids()); + assert!(!options.lowercase_xmltv_display_names()); + assert!(options.epg_output.is_empty()); + assert!(options.is_empty()); + + let serialized = serde_saphyr::to_string(&options).expect("default target options should serialize"); + assert!(!serialized.contains("epg_output"), "default epg_output should be omitted, got: {serialized}"); +} + +#[test] +fn target_options_epg_output_roundtrips() { + let yaml = r" +epg_output: + lowercase_ids: true + lowercase_xmltv_display_names: true +"; + + let options = + serde_saphyr::from_str::(yaml).expect("configured epg_output should deserialize"); + + assert!(options.lowercase_epg_ids()); + assert!(options.lowercase_xmltv_display_names()); + assert!(!options.is_empty()); + + let serialized = serde_saphyr::to_string(&options).expect("configured epg_output should serialize"); + let roundtripped = + serde_saphyr::from_str::(&serialized).expect("serialized epg_output should deserialize"); + assert_eq!(roundtripped, options); +} + +#[test] +fn target_options_epg_output_makes_options_nonempty() { + let lowercase_ids = ConfigTargetOptions { + epg_output: EpgOutputOptions { lowercase_ids: true, ..EpgOutputOptions::default() }, + ..ConfigTargetOptions::default() + }; + let lowercase_display_names = ConfigTargetOptions { + epg_output: EpgOutputOptions { lowercase_xmltv_display_names: true, ..EpgOutputOptions::default() }, + ..ConfigTargetOptions::default() + }; + + assert!(!lowercase_ids.is_empty()); + assert!(!lowercase_display_names.is_empty()); +} + +#[test] +fn target_options_reject_unknown_epg_output_fields() { + let yaml = r" +epg_output: + lowercase_id: true +"; + + let result = serde_saphyr::from_str::(yaml); + + assert!(result.is_err(), "unknown epg_output fields must be rejected"); +} + +#[test] +fn target_options_mpeg_ts_helper_keeps_existing_stream_share_semantics() { + let hls_only = ConfigTargetOptions { + share_live_streams: ConfigTargetShareLiveStreams { hls: true, mpeg_ts: false }, + ..Default::default() + }; + let mpeg_ts = ConfigTargetOptions { + share_live_streams: ConfigTargetShareLiveStreams { hls: false, mpeg_ts: true }, + ..Default::default() + }; + + assert!(hls_only.share_live_hls_enabled()); + assert!(!hls_only.share_live_mpeg_ts_enabled()); + assert!(mpeg_ts.share_live_mpeg_ts_enabled()); +} diff --git a/shared/src/model/config/tmdb_curation.rs b/shared/src/model/config/tmdb_curation.rs index 162849ed5..ceca32cda 100644 --- a/shared/src/model/config/tmdb_curation.rs +++ b/shared/src/model/config/tmdb_curation.rs @@ -101,6 +101,25 @@ impl Default for TmdbCurationConfigDto { fn default() -> Self { Self { enabled: true, api: TmdbCurationApiConfigDto::default(), trending: Vec::new() } } } +impl TmdbCurationConfigDto { + pub(super) fn prepare(&mut self, curation_enabled: bool) -> Result<(), TuliproxError> { + self.api.access_token = self.api.access_token.trim().to_string(); + for selector in &mut self.trending { + // Even disabled editor drafts must retain a valid acquisition bound. + if !is_valid_tmdb_trending_limit(selector.limit) { + return Err(TuliproxError::Config("TMDB trending limit must be an integer in 1..=500".to_string())); + } + prepare_selector_category( + &mut selector.category_name, + selector.create_xtream_category, + curation_enabled && self.enabled, + "TMDB trending", + )?; + } + Ok(()) + } +} + #[cfg(test)] mod tests { use crate::model::ConfigTargetDto; @@ -186,22 +205,3 @@ mod tests { } } } - -impl TmdbCurationConfigDto { - pub(super) fn prepare(&mut self, curation_enabled: bool) -> Result<(), TuliproxError> { - self.api.access_token = self.api.access_token.trim().to_string(); - for selector in &mut self.trending { - // Even disabled editor drafts must retain a valid acquisition bound. - if !is_valid_tmdb_trending_limit(selector.limit) { - return Err(TuliproxError::Config("TMDB trending limit must be an integer in 1..=500".to_string())); - } - prepare_selector_category( - &mut selector.category_name, - selector.create_xtream_category, - curation_enabled && self.enabled, - "TMDB trending", - )?; - } - Ok(()) - } -} diff --git a/shared/src/model/event.rs b/shared/src/model/event.rs index 6e0d24815..1675d03f7 100644 --- a/shared/src/model/event.rs +++ b/shared/src/model/event.rs @@ -9,741 +9,6 @@ //! The bus implementation (`EventManager`) stays in `session`, where the //! stream-meter registry it feeds also lives. -use crate::model::{ - auth_audit::{AuthAuditEvent, AuthAuditOutcome}, - notification::{registry, EventId, Severity}, - stats::SourceStats, - ActiveUserConnectionChange, ConfigReloadFailure, ConfigType, ConnectionDenied, DiskAlert, LibraryScanProgressEvent, - LibraryScanSummaryStatus, MetadataUpdateFailure, MsgKind, NotificationDeadLetter, Permission, - PlaylistGroupsChanged, PlaylistUpdateProgressEvent, PlaylistUpdateRunId, PlaylistUpdateRunOrder, - PlaylistUpdateState, ProviderAccountEvent, ProviderAccountState, ProviderFetchFailure, ProviderPoolExhausted, - ProviderPriorityFallback, RecordingLifecycleMessage, ScheduledTaskFailure, ServerLifecycleEvent, - ServerLifecycleState, StreamProbeFailure, SystemInfo, UserLifecycleEvent, UserLifecycleState, WatchChanges, - WatchDisabled, WatchUnmatched, -}; -use std::sync::Arc; - -/// Everything that happens in the server that someone outside the emitting -/// module might want to know about. -/// -#[allow(clippy::large_enum_variant)] -#[derive(Clone, Debug, PartialEq)] -pub enum EventMessage { - ServerError(String), - /// The server finished starting, or is stopping. One variant, two kinds - - /// see [`EventMessage::kind`]. - ServerLifecycle(ServerLifecycleEvent), - ActiveUser(ActiveUserConnectionChange), - ActiveProvider(Arc, usize), - ConfigChange(ConfigType), - PlaylistUpdate(PlaylistUpdateSummary), - PlaylistUpdateProgress(PlaylistUpdateProgressEvent), - SystemInfoUpdate(Arc), - LibraryScanProgress(LibraryScanProgressEvent), - RecordingChanged, - /// Bytes moved, nothing else. Separate from `RecordingChanged` so a - /// session can rate limit it without also delaying a state transition. - RecordingProgress, - RecordingRulesChanged, - InputMetadataUpdatesCompleted(Arc), - InputMetadataUpdatesStarted(Arc), - /// A metadata update cycle ended with tasks it could not finish. - /// - /// `InputMetadataUpdatesCompleted` only fires when a cycle drained *with - /// changes*, so without this an input whose resolves always fail emits a - /// start and then silence for as long as it stays broken. - InputMetadataUpdatesFailed(MetadataUpdateFailure), - - // The lifecycle events below reached the notification pipeline directly, - // never the bus, so nothing that subscribes here could see them. Each is - // low-frequency, and each already had a registered notification id. - /// Disk usage crossed the warn or critical threshold. - DiskAlert(DiskAlert), - /// A watched config file could not be reloaded. Distinct from - /// `ServerError`: a plugin or operator can subscribe to "my config - /// stopped loading" without taking every server error. - ConfigReloadFailed(ConfigReloadFailure), - /// A target's `watch` config saw its group membership change. - PlaylistWatchChanged(WatchChanges), - /// A target gained or lost whole groups between refreshes. - PlaylistGroupsChanged(PlaylistGroupsChanged), - /// A target's `watch` config is configured but not working. - PlaylistWatchDisabled(WatchDisabled), - /// `watch` patterns that matched no group in the refreshed playlist. - PlaylistWatchUnmatched(WatchUnmatched), - /// A recording started, finished or failed. One variant, three kinds - - /// see [`EventMessage::kind`] - so a subscriber can ask for failures - /// alone. - RecordingLifecycle(RecordingLifecycleMessage), - /// A provider account changed status, is about to expire, or has. - ProviderAccount(ProviderAccountEvent), - /// An input's playlist fetch failed. - ProviderFetchFailed(ProviderFetchFailure), - /// Every provider behind an input was at capacity. - ProviderPoolExhausted(ProviderPoolExhausted), - /// An input started being served from a different priority group. - ProviderPriorityFallback(ProviderPriorityFallback), - - /// An API-proxy user was created, changed or removed. One variant, - /// three kinds - see [`EventMessage::kind`] - so a subscriber can ask - /// for deletions alone. - UserLifecycle(UserLifecycleEvent), - /// A user was refused a connection because their limits were full. - ConnectionDenied(ConnectionDenied), - - /// A stream probe returned no metadata. There is no success - /// counterpart; see [`StreamProbeFailure`]. - StreamProbeFailed(StreamProbeFailure), - - /// A scheduled task could not complete. - ScheduledTaskFailed(ScheduledTaskFailure), - - /// A notification ran out of delivery attempts and was dropped. - /// - /// On the bus for plugins and the status endpoint; deliberately *not* - /// notifiable - see [`NotificationDeadLetter`] for why routing this - /// through the outbox that just failed would loop. - NotificationDeadLettered(NotificationDeadLetter), - - /// An authentication decision: a sign-in, a rejected sign-in, a - /// throttled attempt, or a permission denial. One variant, four kinds - - /// see [`EventMessage::kind`] - so a subscriber can ask for the failures - /// without being woken by every successful sign-in. - AuthAudit(AuthAuditEvent), -} - -/// Somewhere an [`EventMessage`] can be published. -/// -/// A bound, not a trait object: emitters are generic over their sink and -/// monomorphise against the one they were built with. Three things fall out -/// of that: -/// -/// * the pipeline no longer threads `Option>` around and -/// branches on it at every emit site - [`NoopSink`] is the absent case, -/// and its `emit` is an empty function the optimiser deletes outright; -/// * tests can assert on what was emitted without standing up a broadcast -/// channel and racing a subscriber; -/// * `dvr` and the notification path can name a sink without depending on -/// the streaming-session runtime that implements it. -/// -/// Implementations must not block. The bus is reached from the streaming -/// data path, and an emitter that can stall is an emitter that will. -pub trait EventSink: Send + Sync { - /// Publish. Best-effort by contract: no subscribers, a full buffer or a - /// closed channel are all normal and none of them are the emitter's - /// problem. - fn emit(&self, event: EventMessage); - /// Forward a confirmed account response to the runtime provider manager. - /// Sinks without a provider runtime deliberately ignore the observation. - fn observe_provider_account( - &self, - _name: Arc, - _identity: super::ProviderAccountIdentity, - _status: Option, - _exp_date: Option, - ) { - } -} - -/// The sink that drops everything. -/// -/// Replaces the `Option` at call sites that only ever had a `None` case -/// because tests and one-shot CLI runs have no bus. Monomorphised against -/// this, every emit site compiles to nothing. -#[derive(Debug, Clone, Copy, Default)] -pub struct NoopSink; - -impl EventSink for NoopSink { - fn emit(&self, _event: EventMessage) {} -} - -impl EventSink for Arc { - fn emit(&self, event: EventMessage) { (**self).emit(event); } - fn observe_provider_account( - &self, - name: Arc, - identity: super::ProviderAccountIdentity, - status: Option, - exp_date: Option, - ) { - (**self).observe_provider_account(name, identity, status, exp_date); - } -} - -impl EventSink for &T { - fn emit(&self, event: EventMessage) { (**self).emit(event); } - fn observe_provider_account( - &self, - name: Arc, - identity: super::ProviderAccountIdentity, - status: Option, - exp_date: Option, - ) { - (**self).observe_provider_account(name, identity, status, exp_date); - } -} - -/// Which event this is, without its payload. -/// -/// Subscribers used to discriminate by exhaustively matching `EventMessage`, -/// once per subscriber: the websocket mapped variants to permissions, the -/// notification bridge mapped them to notification ids with four arms -/// returning `None`, and the wire layer mapped them to `ProtocolMessage`. -/// Adding a variant compiled cleanly while silently reaching none of them. -/// -/// Everything that is a property *of the event* rather than of one consumer -/// hangs off this type instead, so a new variant has one place to declare -/// what it is and every consumer picks it up. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum EventKind { - ServerError, - ServerStarted, - ServerShutdown, - ActiveUser, - ActiveProvider, - ConfigChange, - PlaylistUpdate, - PlaylistUpdateProgress, - SystemInfoUpdate, - LibraryScanProgress, - /// A scan that ended in an error. Separate from the progress kind so a - /// subscriber can take scan failures without the tick firehose. - LibraryScanFailed, - RecordingChanged, - RecordingRulesChanged, - InputMetadataUpdatesCompleted, - InputMetadataUpdatesStarted, - InputMetadataUpdatesFailed, - DiskAlert, - ConfigReloadFailed, - PlaylistWatchChanged, - PlaylistGroupsChanged, - PlaylistWatchDisabled, - PlaylistWatchUnmatched, - RecordingStarted, - RecordingCompleted, - RecordingFailed, - ProviderAccountStatus, - ProviderAccountExpiring, - ProviderAccountExpired, - ProviderFetchFailed, - ProviderPoolExhausted, - ProviderPriorityFallback, - UserCreated, - UserUpdated, - UserDeleted, - ConnectionDenied, - StreamProbeFailed, - NotificationDeadLettered, - ScheduledTaskFailed, - AuthSignInSucceeded, - AuthSignInFailed, - AuthSignInThrottled, - AuthPermissionDenied, - /// Payload-free progress nudge, appended last so no existing kind's bit - /// position shifts. - RecordingProgress, -} - -impl EventKind { - /// Every kind, in declaration order. - /// - /// The mask type below indexes into this, so the order is load-bearing: - /// it is the bit order, not just a listing. - pub const ALL: [Self; 43] = [ - Self::ServerError, - Self::ServerStarted, - Self::ServerShutdown, - Self::ActiveUser, - Self::ActiveProvider, - Self::ConfigChange, - Self::PlaylistUpdate, - Self::PlaylistUpdateProgress, - Self::SystemInfoUpdate, - Self::LibraryScanProgress, - Self::LibraryScanFailed, - Self::RecordingChanged, - Self::RecordingRulesChanged, - Self::InputMetadataUpdatesCompleted, - Self::InputMetadataUpdatesStarted, - Self::InputMetadataUpdatesFailed, - Self::DiskAlert, - Self::ConfigReloadFailed, - Self::PlaylistWatchChanged, - Self::PlaylistGroupsChanged, - Self::PlaylistWatchDisabled, - Self::PlaylistWatchUnmatched, - Self::RecordingStarted, - Self::RecordingCompleted, - Self::RecordingFailed, - Self::ProviderAccountStatus, - Self::ProviderAccountExpiring, - Self::ProviderAccountExpired, - Self::ProviderFetchFailed, - Self::ProviderPoolExhausted, - Self::ProviderPriorityFallback, - Self::UserCreated, - Self::UserUpdated, - Self::UserDeleted, - Self::ConnectionDenied, - Self::StreamProbeFailed, - Self::NotificationDeadLettered, - Self::ScheduledTaskFailed, - Self::AuthSignInSucceeded, - Self::AuthSignInFailed, - Self::AuthSignInThrottled, - Self::AuthPermissionDenied, - Self::RecordingProgress, - ]; - - /// This kind's bit position. - /// - /// `u64`, not `u32`: the taxonomy passed 23 kinds and the headroom above - /// it is where a mask migration would have to happen *after* operators - /// had written subscriptions into config. Widening now is free. - #[must_use] - pub const fn bit(self) -> u64 { 1 << (self as u32) } - - /// The permission a websocket session must hold to receive this kind. - /// - /// Lives here rather than in the websocket handler because it is a fact - /// about the event: "who may see a download delta" does not change with - /// the transport carrying it. - #[must_use] - pub const fn required_permission(self) -> Permission { - match self { - Self::RecordingChanged - | Self::RecordingProgress - | Self::RecordingRulesChanged - | Self::RecordingStarted - | Self::RecordingCompleted - | Self::RecordingFailed => Permission::RecordingRead, - Self::PlaylistUpdate - | Self::PlaylistUpdateProgress - | Self::PlaylistWatchChanged - | Self::PlaylistGroupsChanged - | Self::PlaylistWatchDisabled - | Self::PlaylistWatchUnmatched => Permission::PlaylistWrite, - Self::LibraryScanProgress | Self::LibraryScanFailed => Permission::LibraryWrite, - // Who may hear that an account was created is the same question - // as who may list accounts. - Self::UserCreated | Self::UserUpdated | Self::UserDeleted => Permission::UserRead, - // Who was turned away is the same question as who signed in, and - // strictly narrower than the system-wide read. - Self::ConnectionDenied => Permission::UserRead, - // Who signed in, who failed to, and who was refused: the same - // question as who may read the user list, and strictly narrower - // than the system-wide read the other operational events take. - Self::AuthSignInSucceeded - | Self::AuthSignInFailed - | Self::AuthSignInThrottled - | Self::AuthPermissionDenied => Permission::UserRead, - Self::ServerError - | Self::ServerStarted - | Self::ServerShutdown - | Self::ActiveUser - | Self::ActiveProvider - | Self::ConfigChange - | Self::SystemInfoUpdate - | Self::InputMetadataUpdatesCompleted - | Self::InputMetadataUpdatesStarted - | Self::InputMetadataUpdatesFailed - | Self::DiskAlert - | Self::ConfigReloadFailed - | Self::ProviderAccountStatus - | Self::ProviderAccountExpiring - | Self::ProviderAccountExpired - | Self::ProviderFetchFailed - | Self::ProviderPoolExhausted - | Self::ProviderPriorityFallback - // Sits with the metadata-update events it is produced by. - | Self::StreamProbeFailed - | Self::NotificationDeadLettered - | Self::ScheduledTaskFailed => Permission::SystemRead, - } - } - - /// Does this kind fire many times per operation? - /// - /// True for progress ticks, incremental deltas and the periodic - /// system-info sample. A bus that coalesces needs to know which messages - /// are safe to supersede, and a subscriber sizing its buffer needs to - /// know which ones will fill it. - /// - /// This is a statement about rate, not about whether anyone wants the - /// event: the notification bridge decides notifiability separately, - /// because that also depends on whether a terminal counterpart exists. - #[must_use] - pub const fn is_high_frequency(self) -> bool { - matches!( - self, - Self::PlaylistUpdateProgress | Self::LibraryScanProgress | Self::RecordingProgress | Self::SystemInfoUpdate - ) - } - - /// Does this kind describe current state rather than an occurrence? - /// - /// A latched kind's newest message is the whole truth - the last - /// `SystemInfo` sample *is* the system info - so it is worth retaining - /// for a subscriber that connects later. An occurrence is not: replaying - /// "a playlist update finished" to a session that was not there when it - /// happened would be a lie. - /// - /// This is what a cold websocket connect should be handed instead of - /// waiting up to three seconds for the next sample. - #[must_use] - pub const fn is_latched(self) -> bool { matches!(self, Self::SystemInfoUpdate | Self::ActiveProvider) } - - /// Is this kind a payload-free nudge that can be coalesced? - /// - /// True only where N occurrences and one are indistinguishable to every - /// consumer: the event carries no data and everyone who receives it - /// responds by re-reading current state. Deleting a recording emits - /// `RecordingChanged` and `RecordingRulesChanged` back to back, and a - /// bulk operation emits one per item; each makes the Web UI re-fetch the - /// same snapshot. - /// - /// Never true for an event carrying a payload, however repetitive - a - /// dropped progress tick loses the message it carried. - #[must_use] - pub const fn is_coalescable(self) -> bool { matches!(self, Self::RecordingChanged | Self::RecordingRulesChanged) } - - /// Stable wire name. - /// - /// Plugins are compiled against these strings and operators write them - /// into subscription config, so - like a notification channel id - a - /// released name must not change. - #[must_use] - pub const fn as_wire_name(self) -> &'static str { - match self { - Self::ServerError => "server.error", - Self::ServerStarted => "system.started", - Self::ServerShutdown => "system.shutdown", - Self::ActiveUser => "user.connection.changed", - Self::ActiveProvider => "provider.connection.changed", - Self::ConfigChange => "config.changed", - Self::PlaylistUpdate => "playlist.update", - Self::PlaylistUpdateProgress => "playlist.update.progress", - Self::SystemInfoUpdate => "system.info", - Self::LibraryScanProgress => "library.scan.progress", - Self::LibraryScanFailed => "library.scan.failed", - Self::RecordingChanged => "recording.changed", - Self::RecordingProgress => "recording.progress", - Self::RecordingRulesChanged => "recording.rules.changed", - Self::InputMetadataUpdatesCompleted => "metadata.update.completed", - Self::InputMetadataUpdatesStarted => "metadata.update.started", - Self::InputMetadataUpdatesFailed => "metadata.update.failed", - Self::DiskAlert => "system.disk.alert", - Self::ConfigReloadFailed => "config.reload.failed", - Self::PlaylistWatchChanged => "playlist.watch.changed", - Self::PlaylistGroupsChanged => "playlist.groups.changed", - Self::PlaylistWatchDisabled => "playlist.watch.disabled", - Self::PlaylistWatchUnmatched => "playlist.watch.unmatched", - Self::RecordingStarted => "recording.started", - Self::RecordingCompleted => "recording.completed", - Self::RecordingFailed => "recording.failed", - Self::ProviderAccountStatus => "provider.account.status", - Self::ProviderAccountExpiring => "provider.account.expiring", - Self::ProviderAccountExpired => "provider.account.expired", - Self::ProviderFetchFailed => "provider.fetch.failed", - Self::ProviderPoolExhausted => "provider.pool.exhausted", - Self::ProviderPriorityFallback => "provider.priority.fallback", - Self::UserCreated => "user.created", - Self::UserUpdated => "user.updated", - Self::UserDeleted => "user.deleted", - Self::ConnectionDenied => "user.connection.denied", - Self::StreamProbeFailed => "stream.probe.failed", - Self::NotificationDeadLettered => "notification.dead_lettered", - Self::ScheduledTaskFailed => "scheduled_task.failed", - Self::AuthSignInSucceeded => "auth.sign_in.succeeded", - Self::AuthSignInFailed => "auth.sign_in.failed", - Self::AuthSignInThrottled => "auth.sign_in.throttled", - Self::AuthPermissionDenied => "auth.permission.denied", - } - } - - /// Parse a wire name back. Unknown names are `None` rather than a - /// fallback variant, so a typo in a subscription is visible. - #[must_use] - pub fn from_wire_name(name: &str) -> Option { Self::ALL.into_iter().find(|kind| kind.as_wire_name() == name) } -} - -impl EventMessage { - /// Which event this is. - #[must_use] - pub const fn kind(&self) -> EventKind { - match self { - Self::ServerError(_) => EventKind::ServerError, - Self::ServerLifecycle(event) => match event.state { - ServerLifecycleState::Started => EventKind::ServerStarted, - ServerLifecycleState::ShuttingDown => EventKind::ServerShutdown, - }, - Self::ActiveUser(_) => EventKind::ActiveUser, - Self::ActiveProvider(_, _) => EventKind::ActiveProvider, - Self::ConfigChange(_) => EventKind::ConfigChange, - Self::PlaylistUpdate(_) => EventKind::PlaylistUpdate, - Self::PlaylistUpdateProgress(_) => EventKind::PlaylistUpdateProgress, - Self::SystemInfoUpdate(_) => EventKind::SystemInfoUpdate, - // One payload, two kinds. The emitter sends exactly one event per - // scan - a success or a failure - so the status is the whole - // discriminant. - Self::LibraryScanProgress(event) => match event.summary.status { - LibraryScanSummaryStatus::Success => EventKind::LibraryScanProgress, - LibraryScanSummaryStatus::Error => EventKind::LibraryScanFailed, - }, - Self::RecordingChanged => EventKind::RecordingChanged, - Self::RecordingProgress => EventKind::RecordingProgress, - Self::RecordingRulesChanged => EventKind::RecordingRulesChanged, - Self::InputMetadataUpdatesCompleted(_) => EventKind::InputMetadataUpdatesCompleted, - Self::InputMetadataUpdatesStarted(_) => EventKind::InputMetadataUpdatesStarted, - Self::InputMetadataUpdatesFailed(_) => EventKind::InputMetadataUpdatesFailed, - Self::DiskAlert(_) => EventKind::DiskAlert, - Self::ConfigReloadFailed(_) => EventKind::ConfigReloadFailed, - Self::PlaylistWatchChanged(_) => EventKind::PlaylistWatchChanged, - Self::PlaylistGroupsChanged(_) => EventKind::PlaylistGroupsChanged, - Self::PlaylistWatchDisabled(_) => EventKind::PlaylistWatchDisabled, - Self::PlaylistWatchUnmatched(_) => EventKind::PlaylistWatchUnmatched, - // One payload, three kinds: a subscriber that only cares about - // failures should not be woken for every completed recording. - Self::RecordingLifecycle(msg) => match msg.event { - MsgKind::RecordingStarted => EventKind::RecordingStarted, - MsgKind::RecordingCompleted => EventKind::RecordingCompleted, - // `RecordingLifecycleMessage::event` is typed as the whole - // `MsgKind`; anything that is not a start or a completion is - // reported as a failure rather than silently miscategorised. - _ => EventKind::RecordingFailed, - }, - Self::ProviderAccount(event) => match event.state { - ProviderAccountState::StatusChanged => EventKind::ProviderAccountStatus, - ProviderAccountState::Expiring => EventKind::ProviderAccountExpiring, - ProviderAccountState::Expired => EventKind::ProviderAccountExpired, - }, - Self::ProviderFetchFailed(_) => EventKind::ProviderFetchFailed, - Self::ProviderPoolExhausted(_) => EventKind::ProviderPoolExhausted, - Self::ProviderPriorityFallback(_) => EventKind::ProviderPriorityFallback, - Self::UserLifecycle(event) => match event.state { - UserLifecycleState::Created => EventKind::UserCreated, - UserLifecycleState::Updated => EventKind::UserUpdated, - UserLifecycleState::Deleted => EventKind::UserDeleted, - }, - Self::ConnectionDenied(_) => EventKind::ConnectionDenied, - Self::StreamProbeFailed(_) => EventKind::StreamProbeFailed, - Self::NotificationDeadLettered(_) => EventKind::NotificationDeadLettered, - Self::ScheduledTaskFailed(_) => EventKind::ScheduledTaskFailed, - Self::AuthAudit(event) => match event.outcome { - AuthAuditOutcome::SignInSucceeded => EventKind::AuthSignInSucceeded, - AuthAuditOutcome::SignInFailed => EventKind::AuthSignInFailed, - AuthAuditOutcome::SignInThrottled => EventKind::AuthSignInThrottled, - AuthAuditOutcome::PermissionDenied => EventKind::AuthPermissionDenied, - }, - } - } - - /// How bad this particular occurrence is. - /// - /// Depends on the payload, not just the kind: a playlist update that - /// failed is an error and one that succeeded is not. - #[must_use] - pub fn severity(&self) -> Severity { - match self { - // The registry cannot answer this either: whether a fetch failure - // needs a human is a property of the classification, not the id. - Self::ProviderFetchFailed(failure) if !failure.needs_operator => Severity::Warn, - // The only case the registry cannot answer: a partial refresh - // and a clean one share `PLAYLIST_UPDATE_COMPLETED`, but a - // partial one is not a clean success. - Self::PlaylistUpdate(summary) if summary.state == PlaylistUpdateState::Partial => Severity::Warn, - // Everything else takes the severity its registered event - // declares, so there is no second severity table to drift. - _ => self.notification_id().map_or(Severity::Info, registry::default_severity), - } - } - - /// The notification this event becomes, if it is notifiable at all. - /// - /// On `EventMessage` rather than `EventKind` because two of them depend - /// on the payload: a playlist update that failed is a different - /// notification from one that succeeded, not merely a more severe one. - /// - /// The bridge used to hold this table itself, so the event taxonomy and - /// the notification taxonomy drifted independently. Now the event says - /// what it is and the bridge only decides how to word it. - /// - /// `None` is the honest answer for the high-frequency kinds: they fire - /// many times per operation and their terminal counterparts carry the - /// news. - #[must_use] - pub const fn notification_id(&self) -> Option { - Some(match self { - Self::ServerError(_) => registry::SYSTEM_ERROR, - Self::ServerLifecycle(event) => match event.state { - ServerLifecycleState::Started => registry::SYSTEM_STARTED, - ServerLifecycleState::ShuttingDown => registry::SYSTEM_SHUTDOWN, - }, - Self::PlaylistUpdate(summary) => match summary.state { - PlaylistUpdateState::Success | PlaylistUpdateState::Partial => registry::PLAYLIST_UPDATE_COMPLETED, - PlaylistUpdateState::Failure => registry::PLAYLIST_UPDATE_FAILED, - }, - Self::ConfigChange(_) => registry::CONFIG_CHANGED, - // A failed scan used to take `LIBRARY_SCAN_COMPLETED` like any - // other, so operators were told "A local library scan finished" - // at info severity when it had not. - Self::LibraryScanProgress(event) => match event.summary.status { - LibraryScanSummaryStatus::Success => registry::LIBRARY_SCAN_COMPLETED, - LibraryScanSummaryStatus::Error => registry::LIBRARY_SCAN_FAILED, - }, - Self::InputMetadataUpdatesStarted(_) => registry::METADATA_UPDATE_STARTED, - Self::InputMetadataUpdatesCompleted(_) => registry::METADATA_UPDATE_COMPLETED, - Self::InputMetadataUpdatesFailed(_) => registry::METADATA_UPDATE_FAILED, - Self::DiskAlert(_) => registry::SYSTEM_DISK_ALERT, - Self::ConfigReloadFailed(_) => registry::CONFIG_RELOAD_FAILED, - Self::PlaylistWatchChanged(_) => registry::PLAYLIST_WATCH_CHANGED, - Self::PlaylistGroupsChanged(_) => registry::PLAYLIST_GROUPS_CHANGED, - Self::PlaylistWatchDisabled(_) => registry::PLAYLIST_WATCH_DISABLED, - Self::PlaylistWatchUnmatched(_) => registry::PLAYLIST_WATCH_UNMATCHED, - Self::RecordingLifecycle(msg) => match msg.event { - MsgKind::RecordingStarted => registry::RECORDING_STARTED, - MsgKind::RecordingCompleted => registry::RECORDING_COMPLETED, - _ => registry::RECORDING_FAILED, - }, - Self::ProviderAccount(event) => match event.state { - ProviderAccountState::StatusChanged => registry::PROVIDER_ACCOUNT_STATUS, - ProviderAccountState::Expiring => registry::PROVIDER_ACCOUNT_EXPIRING, - ProviderAccountState::Expired => registry::PROVIDER_ACCOUNT_EXPIRED, - }, - Self::ProviderFetchFailed(_) => registry::PROVIDER_FETCH_FAILED, - Self::ProviderPoolExhausted(_) => registry::PROVIDER_POOL_EXHAUSTED, - Self::ProviderPriorityFallback(_) => registry::PROVIDER_PRIORITY_FALLBACK, - Self::UserLifecycle(event) => match event.state { - UserLifecycleState::Created => registry::USER_CREATED, - UserLifecycleState::Updated => registry::USER_UPDATED, - UserLifecycleState::Deleted => registry::USER_DELETED, - }, - Self::ConnectionDenied(_) => registry::USER_CONNECTION_DENIED, - Self::StreamProbeFailed(_) => registry::STREAM_PROBE_FAILED, - Self::NotificationDeadLettered(_) => registry::NOTIFICATION_DEAD_LETTERED, - Self::ScheduledTaskFailed(_) => registry::SCHEDULED_TASK_FAILED, - Self::AuthAudit(event) => match event.outcome { - AuthAuditOutcome::SignInSucceeded => registry::AUTH_SIGN_IN_SUCCEEDED, - AuthAuditOutcome::SignInFailed => registry::AUTH_SIGN_IN_FAILED, - AuthAuditOutcome::SignInThrottled => registry::AUTH_SIGN_IN_THROTTLED, - AuthAuditOutcome::PermissionDenied => registry::AUTH_PERMISSION_DENIED, - }, - Self::ActiveUser(_) => registry::USER_CONNECTION_CHANGED, - Self::ActiveProvider(_, _) => registry::PROVIDER_CONNECTIONS_CHANGED, - Self::RecordingChanged => registry::RECORDING_QUEUE_CHANGED, - Self::RecordingRulesChanged => registry::RECORDING_RULES_CHANGED, - Self::PlaylistUpdateProgress(_) | Self::SystemInfoUpdate(_) | Self::RecordingProgress => return None, - }) - } - - /// The event as JSON, for consumers that do not speak Rust types. - /// - /// Plugins are handed a wire name and a JSON payload rather than the - /// enum, so this is what a plugin host serialises. Defining it here means - /// a new variant arrives with its payload shape already decided, instead - /// of the host growing a second `match` over the whole taxonomy. - /// - /// Never fails: a payload that will not serialise degrades to `null` - /// rather than dropping the event, because a plugin that learns a - /// refresh finished but not its statistics is still better served than - /// one that hears nothing. - #[must_use] - pub fn payload(&self) -> serde_json::Value { - fn encode(value: &T) -> serde_json::Value { - serde_json::to_value(value).unwrap_or(serde_json::Value::Null) - } - match self { - Self::ServerError(error) => serde_json::json!({ "error": error }), - Self::ServerLifecycle(event) => encode(event), - Self::ActiveUser(change) => encode(change), - Self::ActiveProvider(name, connections) => { - serde_json::json!({ "provider": name.as_ref(), "connections": connections }) - } - // `ConfigType` is not `Serialize`; its display form is the - // stable name operators already see in the Web UI. - Self::ConfigChange(config_type) => serde_json::json!({ "config_type": config_type.to_string() }), - Self::PlaylistUpdate(summary) => encode(summary), - Self::PlaylistUpdateProgress(progress) => encode(progress), - Self::SystemInfoUpdate(info) => encode(info.as_ref()), - Self::LibraryScanProgress(progress) => encode(progress), - Self::RecordingChanged | Self::RecordingProgress | Self::RecordingRulesChanged => serde_json::Value::Null, - Self::InputMetadataUpdatesStarted(input) | Self::InputMetadataUpdatesCompleted(input) => { - serde_json::json!({ "input": input.as_ref() }) - } - Self::InputMetadataUpdatesFailed(failure) => encode(failure), - Self::DiskAlert(alert) => encode(alert), - Self::ConfigReloadFailed(failure) => encode(failure), - Self::PlaylistWatchChanged(changes) => encode(changes), - Self::PlaylistGroupsChanged(changes) => encode(changes), - Self::PlaylistWatchDisabled(disabled) => encode(disabled), - Self::PlaylistWatchUnmatched(unmatched) => encode(unmatched), - Self::RecordingLifecycle(msg) => encode(msg), - Self::ProviderAccount(event) => encode(event), - Self::ProviderFetchFailed(failure) => encode(failure), - Self::ProviderPoolExhausted(exhausted) => encode(exhausted), - Self::ProviderPriorityFallback(fallback) => encode(fallback), - Self::UserLifecycle(event) => encode(event), - Self::ConnectionDenied(denied) => encode(denied), - Self::StreamProbeFailed(failure) => encode(failure), - Self::NotificationDeadLettered(dead_letter) => encode(dead_letter), - Self::ScheduledTaskFailed(failure) => encode(failure), - Self::AuthAudit(event) => encode(event), - } - } - - /// See [`EventKind::required_permission`]. - #[must_use] - pub const fn required_permission(&self) -> Permission { self.kind().required_permission() } - - /// See [`EventKind::is_high_frequency`]. - #[must_use] - pub const fn is_high_frequency(&self) -> bool { self.kind().is_high_frequency() } -} - -/// What one playlist refresh did. -/// -/// `PlaylistUpdate` used to carry the state alone, so "the refresh finished" -/// reached the bus but what it actually did did not - the run summary went -/// straight to the notification layer as a second, separate message with the -/// same registered id. Subscribers saw an outcome with no detail, operators -/// received two notifications per refresh, and a plugin asking for -/// `playlist.update` got a bare enum. -#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct PlaylistUpdateSummary { - /// Stable identity of the processing run. Missing only on legacy/test-only events. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub run_id: Option, - /// Actual serial execution order. Missing only on legacy/test-only events. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub execution_order: Option, - pub state: PlaylistUpdateState, - /// Per-source statistics. Empty when the run produced none. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub stats: Vec, - /// The aggregated error text, when the run reported errors. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -impl PlaylistUpdateSummary { - /// A summary carrying only an outcome - the timeout and panic paths, - /// which have no statistics to report. - #[must_use] - pub fn state_only(state: PlaylistUpdateState) -> Self { - Self { run_id: None, execution_order: None, state, stats: Vec::new(), error: None } - } - - #[must_use] - pub fn for_run( - run_id: PlaylistUpdateRunId, - execution_order: PlaylistUpdateRunOrder, - state: PlaylistUpdateState, - ) -> Self { - Self { run_id: Some(run_id), execution_order: Some(execution_order), state, stats: Vec::new(), error: None } - } -} - /// A set of [`EventKind`]s, as one word. /// /// `get_event_channel` handed every subscriber the firehose, and each one @@ -753,125 +18,12 @@ impl PlaylistUpdateSummary { #[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] pub struct EventKindMask(u64); -impl EventKindMask { - /// Nothing. - pub const NONE: Self = Self(0); - - /// Everything, including kinds added after this mask was written. - pub const ALL: Self = Self(u64::MAX); - - /// An empty mask to build on. - #[must_use] - pub const fn new() -> Self { Self::NONE } - - /// This mask plus `kind`. - #[must_use] - pub const fn with(self, kind: EventKind) -> Self { Self(self.0 | kind.bit()) } - - /// Is `kind` in this mask? - #[must_use] - pub const fn contains(self, kind: EventKind) -> bool { self.0 & kind.bit() != 0 } - - /// Is this mask empty? A subscriber with nothing selected should not be - /// spawned at all. - #[must_use] - pub const fn is_empty(self) -> bool { self.0 == 0 } - - /// The union of two masks - how a plugin host builds one mask covering - /// every loaded plugin's subscriptions. - #[must_use] - pub const fn union(self, other: Self) -> Self { Self(self.0 | other.0) } -} - -impl EventKindMask { - /// Build a mask from wire names - a plugin manifest's `events.*` - /// subscription list, or an operator's config. - /// - /// Returns the mask alongside the names that matched nothing, so a typo - /// in a subscription can be reported rather than silently subscribing to - /// less than was asked for. - #[must_use] - pub fn from_wire_names<'a, I>(names: I) -> (Self, Vec<&'a str>) - where - I: IntoIterator, - { - let mut mask = Self::NONE; - let mut unknown = Vec::new(); - for name in names { - match EventKind::from_wire_name(name) { - Some(kind) => mask = mask.with(kind), - None => unknown.push(name), - } - } - (mask, unknown) - } - - /// The kinds in this mask. - #[must_use] - pub fn kinds(self) -> Vec { EventKind::ALL.into_iter().filter(|kind| self.contains(*kind)).collect() } -} - -impl FromIterator for EventKindMask { - fn from_iter>(iter: I) -> Self { iter.into_iter().fold(Self::NONE, Self::with) } -} - #[cfg(test)] -mod tests { - use super::{EventKind, EventKindMask}; +mod tests; - #[test] - fn every_kind_has_a_distinct_bit() { - let mut seen = 0u64; - for kind in EventKind::ALL { - assert_eq!(seen & kind.bit(), 0, "{kind:?} shares a bit with an earlier kind"); - seen |= kind.bit(); - } - } - - /// `EventKindMask` is a `u64`, so the taxonomy cannot outgrow 64 kinds - /// without the mask type changing with it. It was a `u32` at 23 kinds; - /// the widening happened while no operator had a subscription list to - /// migrate, which is the only cheap time to do it. - #[test] - fn the_taxonomy_still_fits_in_the_mask() { - assert!(EventKind::ALL.len() <= 64, "EventKindMask needs a wider integer"); - } - - #[test] - fn wire_names_are_unique_and_round_trip() { - let mut names: Vec<&str> = EventKind::ALL.iter().map(|kind| kind.as_wire_name()).collect(); - names.sort_unstable(); - let count = names.len(); - names.dedup(); - assert_eq!(names.len(), count, "two kinds share a wire name"); - - for kind in EventKind::ALL { - assert_eq!(EventKind::from_wire_name(kind.as_wire_name()), Some(kind)); - } - } - - #[test] - fn an_unknown_subscription_name_is_reported_not_ignored() { - let (mask, unknown) = EventKindMask::from_wire_names(["playlist.update", "playlist.updat", "config.changed"]); - - assert!(mask.contains(EventKind::PlaylistUpdate)); - assert!(mask.contains(EventKind::ConfigChange)); - assert_eq!(unknown, vec!["playlist.updat"], "a typo must surface, not silently narrow the subscription"); - } - - #[test] - fn a_mask_round_trips_through_its_kinds() { - let mask = EventKindMask::from_iter([EventKind::ServerError, EventKind::RecordingChanged]); - assert_eq!(mask.kinds(), vec![EventKind::ServerError, EventKind::RecordingChanged]); - assert!(!mask.contains(EventKind::SystemInfoUpdate)); - assert!(!mask.is_empty()); - assert!(EventKindMask::NONE.is_empty()); - } - - #[test] - fn all_matches_every_kind() { - for kind in EventKind::ALL { - assert!(EventKindMask::ALL.contains(kind)); - } - } -} +mod kind; +mod message; +mod sink; +pub use kind::EventKind; +pub use message::{EventMessage, PlaylistUpdateSummary}; +pub use sink::{EventSink, NoopSink}; diff --git a/shared/src/model/event/kind.rs b/shared/src/model/event/kind.rs new file mode 100644 index 000000000..e2be56f79 --- /dev/null +++ b/shared/src/model/event/kind.rs @@ -0,0 +1,349 @@ +use super::EventKindMask; +use crate::model::Permission; + +/// Which event this is, without its payload. +/// +/// Subscribers used to discriminate by exhaustively matching `EventMessage`, +/// once per subscriber: the websocket mapped variants to permissions, the +/// notification bridge mapped them to notification ids with four arms +/// returning `None`, and the wire layer mapped them to `ProtocolMessage`. +/// Adding a variant compiled cleanly while silently reaching none of them. +/// +/// Everything that is a property *of the event* rather than of one consumer +/// hangs off this type instead, so a new variant has one place to declare +/// what it is and every consumer picks it up. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum EventKind { + ServerError, + ServerStarted, + ServerShutdown, + ActiveUser, + ActiveProvider, + ConfigChange, + PlaylistUpdate, + PlaylistUpdateProgress, + SystemInfoUpdate, + LibraryScanProgress, + /// A scan that ended in an error. Separate from the progress kind so a + /// subscriber can take scan failures without the tick firehose. + LibraryScanFailed, + RecordingChanged, + RecordingRulesChanged, + InputMetadataUpdatesCompleted, + InputMetadataUpdatesStarted, + InputMetadataUpdatesFailed, + DiskAlert, + ConfigReloadFailed, + PlaylistWatchChanged, + PlaylistGroupsChanged, + PlaylistWatchDisabled, + PlaylistWatchUnmatched, + RecordingStarted, + RecordingCompleted, + RecordingFailed, + ProviderAccountStatus, + ProviderAccountExpiring, + ProviderAccountExpired, + ProviderFetchFailed, + ProviderPoolExhausted, + ProviderPriorityFallback, + UserCreated, + UserUpdated, + UserDeleted, + ConnectionDenied, + StreamProbeFailed, + NotificationDeadLettered, + ScheduledTaskFailed, + AuthSignInSucceeded, + AuthSignInFailed, + AuthSignInThrottled, + AuthPermissionDenied, + /// Payload-free progress nudge, appended last so no existing kind's bit + /// position shifts. + RecordingProgress, +} + +impl EventKind { + /// Every kind, in declaration order. + /// + /// The mask type below indexes into this, so the order is load-bearing: + /// it is the bit order, not just a listing. + pub const ALL: [Self; 43] = [ + Self::ServerError, + Self::ServerStarted, + Self::ServerShutdown, + Self::ActiveUser, + Self::ActiveProvider, + Self::ConfigChange, + Self::PlaylistUpdate, + Self::PlaylistUpdateProgress, + Self::SystemInfoUpdate, + Self::LibraryScanProgress, + Self::LibraryScanFailed, + Self::RecordingChanged, + Self::RecordingRulesChanged, + Self::InputMetadataUpdatesCompleted, + Self::InputMetadataUpdatesStarted, + Self::InputMetadataUpdatesFailed, + Self::DiskAlert, + Self::ConfigReloadFailed, + Self::PlaylistWatchChanged, + Self::PlaylistGroupsChanged, + Self::PlaylistWatchDisabled, + Self::PlaylistWatchUnmatched, + Self::RecordingStarted, + Self::RecordingCompleted, + Self::RecordingFailed, + Self::ProviderAccountStatus, + Self::ProviderAccountExpiring, + Self::ProviderAccountExpired, + Self::ProviderFetchFailed, + Self::ProviderPoolExhausted, + Self::ProviderPriorityFallback, + Self::UserCreated, + Self::UserUpdated, + Self::UserDeleted, + Self::ConnectionDenied, + Self::StreamProbeFailed, + Self::NotificationDeadLettered, + Self::ScheduledTaskFailed, + Self::AuthSignInSucceeded, + Self::AuthSignInFailed, + Self::AuthSignInThrottled, + Self::AuthPermissionDenied, + Self::RecordingProgress, + ]; + + /// This kind's bit position. + /// + /// `u64`, not `u32`: the taxonomy passed 23 kinds and the headroom above + /// it is where a mask migration would have to happen *after* operators + /// had written subscriptions into config. Widening now is free. + #[must_use] + pub const fn bit(self) -> u64 { 1 << (self as u32) } + + /// The permission a websocket session must hold to receive this kind. + /// + /// Lives here rather than in the websocket handler because it is a fact + /// about the event: "who may see a download delta" does not change with + /// the transport carrying it. + #[must_use] + pub const fn required_permission(self) -> Permission { + match self { + Self::RecordingChanged + | Self::RecordingProgress + | Self::RecordingRulesChanged + | Self::RecordingStarted + | Self::RecordingCompleted + | Self::RecordingFailed => Permission::RecordingRead, + Self::PlaylistUpdate + | Self::PlaylistUpdateProgress + | Self::PlaylistWatchChanged + | Self::PlaylistGroupsChanged + | Self::PlaylistWatchDisabled + | Self::PlaylistWatchUnmatched => Permission::PlaylistWrite, + Self::LibraryScanProgress | Self::LibraryScanFailed => Permission::LibraryWrite, + // Who may hear that an account was created is the same question + // as who may list accounts. + Self::UserCreated | Self::UserUpdated | Self::UserDeleted => Permission::UserRead, + // Who was turned away is the same question as who signed in, and + // strictly narrower than the system-wide read. + Self::ConnectionDenied => Permission::UserRead, + // Who signed in, who failed to, and who was refused: the same + // question as who may read the user list, and strictly narrower + // than the system-wide read the other operational events take. + Self::AuthSignInSucceeded + | Self::AuthSignInFailed + | Self::AuthSignInThrottled + | Self::AuthPermissionDenied => Permission::UserRead, + Self::ServerError + | Self::ServerStarted + | Self::ServerShutdown + | Self::ActiveUser + | Self::ActiveProvider + | Self::ConfigChange + | Self::SystemInfoUpdate + | Self::InputMetadataUpdatesCompleted + | Self::InputMetadataUpdatesStarted + | Self::InputMetadataUpdatesFailed + | Self::DiskAlert + | Self::ConfigReloadFailed + | Self::ProviderAccountStatus + | Self::ProviderAccountExpiring + | Self::ProviderAccountExpired + | Self::ProviderFetchFailed + | Self::ProviderPoolExhausted + | Self::ProviderPriorityFallback + // Sits with the metadata-update events it is produced by. + | Self::StreamProbeFailed + | Self::NotificationDeadLettered + | Self::ScheduledTaskFailed => Permission::SystemRead, + } + } + + /// Does this kind fire many times per operation? + /// + /// True for progress ticks, incremental deltas and the periodic + /// system-info sample. A bus that coalesces needs to know which messages + /// are safe to supersede, and a subscriber sizing its buffer needs to + /// know which ones will fill it. + /// + /// This is a statement about rate, not about whether anyone wants the + /// event: the notification bridge decides notifiability separately, + /// because that also depends on whether a terminal counterpart exists. + #[must_use] + pub const fn is_high_frequency(self) -> bool { + matches!( + self, + Self::PlaylistUpdateProgress | Self::LibraryScanProgress | Self::RecordingProgress | Self::SystemInfoUpdate + ) + } + + /// Does this kind describe current state rather than an occurrence? + /// + /// A latched kind's newest message is the whole truth - the last + /// `SystemInfo` sample *is* the system info - so it is worth retaining + /// for a subscriber that connects later. An occurrence is not: replaying + /// "a playlist update finished" to a session that was not there when it + /// happened would be a lie. + /// + /// This is what a cold websocket connect should be handed instead of + /// waiting up to three seconds for the next sample. + #[must_use] + pub const fn is_latched(self) -> bool { matches!(self, Self::SystemInfoUpdate | Self::ActiveProvider) } + + /// Is this kind a payload-free nudge that can be coalesced? + /// + /// True only where N occurrences and one are indistinguishable to every + /// consumer: the event carries no data and everyone who receives it + /// responds by re-reading current state. Deleting a recording emits + /// `RecordingChanged` and `RecordingRulesChanged` back to back, and a + /// bulk operation emits one per item; each makes the Web UI re-fetch the + /// same snapshot. + /// + /// Never true for an event carrying a payload, however repetitive - a + /// dropped progress tick loses the message it carried. + #[must_use] + pub const fn is_coalescable(self) -> bool { matches!(self, Self::RecordingChanged | Self::RecordingRulesChanged) } + + /// Stable wire name. + /// + /// Plugins are compiled against these strings and operators write them + /// into subscription config, so - like a notification channel id - a + /// released name must not change. + #[must_use] + pub const fn as_wire_name(self) -> &'static str { + match self { + Self::ServerError => "server.error", + Self::ServerStarted => "system.started", + Self::ServerShutdown => "system.shutdown", + Self::ActiveUser => "user.connection.changed", + Self::ActiveProvider => "provider.connection.changed", + Self::ConfigChange => "config.changed", + Self::PlaylistUpdate => "playlist.update", + Self::PlaylistUpdateProgress => "playlist.update.progress", + Self::SystemInfoUpdate => "system.info", + Self::LibraryScanProgress => "library.scan.progress", + Self::LibraryScanFailed => "library.scan.failed", + Self::RecordingChanged => "recording.changed", + Self::RecordingProgress => "recording.progress", + Self::RecordingRulesChanged => "recording.rules.changed", + Self::InputMetadataUpdatesCompleted => "metadata.update.completed", + Self::InputMetadataUpdatesStarted => "metadata.update.started", + Self::InputMetadataUpdatesFailed => "metadata.update.failed", + Self::DiskAlert => "system.disk.alert", + Self::ConfigReloadFailed => "config.reload.failed", + Self::PlaylistWatchChanged => "playlist.watch.changed", + Self::PlaylistGroupsChanged => "playlist.groups.changed", + Self::PlaylistWatchDisabled => "playlist.watch.disabled", + Self::PlaylistWatchUnmatched => "playlist.watch.unmatched", + Self::RecordingStarted => "recording.started", + Self::RecordingCompleted => "recording.completed", + Self::RecordingFailed => "recording.failed", + Self::ProviderAccountStatus => "provider.account.status", + Self::ProviderAccountExpiring => "provider.account.expiring", + Self::ProviderAccountExpired => "provider.account.expired", + Self::ProviderFetchFailed => "provider.fetch.failed", + Self::ProviderPoolExhausted => "provider.pool.exhausted", + Self::ProviderPriorityFallback => "provider.priority.fallback", + Self::UserCreated => "user.created", + Self::UserUpdated => "user.updated", + Self::UserDeleted => "user.deleted", + Self::ConnectionDenied => "user.connection.denied", + Self::StreamProbeFailed => "stream.probe.failed", + Self::NotificationDeadLettered => "notification.dead_lettered", + Self::ScheduledTaskFailed => "scheduled_task.failed", + Self::AuthSignInSucceeded => "auth.sign_in.succeeded", + Self::AuthSignInFailed => "auth.sign_in.failed", + Self::AuthSignInThrottled => "auth.sign_in.throttled", + Self::AuthPermissionDenied => "auth.permission.denied", + } + } + + /// Parse a wire name back. Unknown names are `None` rather than a + /// fallback variant, so a typo in a subscription is visible. + #[must_use] + pub fn from_wire_name(name: &str) -> Option { Self::ALL.into_iter().find(|kind| kind.as_wire_name() == name) } +} + +impl EventKindMask { + /// Nothing. + pub const NONE: Self = Self(0); + + /// Everything, including kinds added after this mask was written. + pub const ALL: Self = Self(u64::MAX); + + /// An empty mask to build on. + #[must_use] + pub const fn new() -> Self { Self::NONE } + + /// This mask plus `kind`. + #[must_use] + pub const fn with(self, kind: EventKind) -> Self { Self(self.0 | kind.bit()) } + + /// Is `kind` in this mask? + #[must_use] + pub const fn contains(self, kind: EventKind) -> bool { self.0 & kind.bit() != 0 } + + /// Is this mask empty? A subscriber with nothing selected should not be + /// spawned at all. + #[must_use] + pub const fn is_empty(self) -> bool { self.0 == 0 } + + /// The union of two masks - how a plugin host builds one mask covering + /// every loaded plugin's subscriptions. + #[must_use] + pub const fn union(self, other: Self) -> Self { Self(self.0 | other.0) } +} + +impl EventKindMask { + /// Build a mask from wire names - a plugin manifest's `events.*` + /// subscription list, or an operator's config. + /// + /// Returns the mask alongside the names that matched nothing, so a typo + /// in a subscription can be reported rather than silently subscribing to + /// less than was asked for. + #[must_use] + pub fn from_wire_names<'a, I>(names: I) -> (Self, Vec<&'a str>) + where + I: IntoIterator, + { + let mut mask = Self::NONE; + let mut unknown = Vec::new(); + for name in names { + match EventKind::from_wire_name(name) { + Some(kind) => mask = mask.with(kind), + None => unknown.push(name), + } + } + (mask, unknown) + } + + /// The kinds in this mask. + #[must_use] + pub fn kinds(self) -> Vec { EventKind::ALL.into_iter().filter(|kind| self.contains(*kind)).collect() } +} + +impl FromIterator for EventKindMask { + fn from_iter>(iter: I) -> Self { iter.into_iter().fold(Self::NONE, Self::with) } +} diff --git a/shared/src/model/event/message.rs b/shared/src/model/event/message.rs new file mode 100644 index 000000000..0d1ffeb5d --- /dev/null +++ b/shared/src/model/event/message.rs @@ -0,0 +1,379 @@ +use super::EventKind; +use crate::model::{ + auth_audit::{AuthAuditEvent, AuthAuditOutcome}, + notification::{registry, EventId, Severity}, + stats::SourceStats, + ActiveUserConnectionChange, ConfigReloadFailure, ConfigType, ConnectionDenied, DiskAlert, LibraryScanProgressEvent, + LibraryScanSummaryStatus, MetadataUpdateFailure, MsgKind, NotificationDeadLetter, Permission, + PlaylistGroupsChanged, PlaylistUpdateProgressEvent, PlaylistUpdateRunId, PlaylistUpdateRunOrder, + PlaylistUpdateState, ProviderAccountEvent, ProviderAccountState, ProviderFetchFailure, ProviderPoolExhausted, + ProviderPriorityFallback, RecordingLifecycleMessage, ScheduledTaskFailure, ServerLifecycleEvent, + ServerLifecycleState, StreamProbeFailure, SystemInfo, UserLifecycleEvent, UserLifecycleState, WatchChanges, + WatchDisabled, WatchUnmatched, +}; +use std::sync::Arc; + +/// Everything that happens in the server that someone outside the emitting +/// module might want to know about. +/// +#[allow(clippy::large_enum_variant)] +#[derive(Clone, Debug, PartialEq)] +pub enum EventMessage { + ServerError(String), + /// The server finished starting, or is stopping. One variant, two kinds - + /// see [`EventMessage::kind`]. + ServerLifecycle(ServerLifecycleEvent), + ActiveUser(ActiveUserConnectionChange), + ActiveProvider(Arc, usize), + ConfigChange(ConfigType), + PlaylistUpdate(PlaylistUpdateSummary), + PlaylistUpdateProgress(PlaylistUpdateProgressEvent), + SystemInfoUpdate(Arc), + LibraryScanProgress(LibraryScanProgressEvent), + RecordingChanged, + /// Bytes moved, nothing else. Separate from `RecordingChanged` so a + /// session can rate limit it without also delaying a state transition. + RecordingProgress, + RecordingRulesChanged, + InputMetadataUpdatesCompleted(Arc), + InputMetadataUpdatesStarted(Arc), + /// A metadata update cycle ended with tasks it could not finish. + /// + /// `InputMetadataUpdatesCompleted` only fires when a cycle drained *with + /// changes*, so without this an input whose resolves always fail emits a + /// start and then silence for as long as it stays broken. + InputMetadataUpdatesFailed(MetadataUpdateFailure), + + // The lifecycle events below reached the notification pipeline directly, + // never the bus, so nothing that subscribes here could see them. Each is + // low-frequency, and each already had a registered notification id. + /// Disk usage crossed the warn or critical threshold. + DiskAlert(DiskAlert), + /// A watched config file could not be reloaded. Distinct from + /// `ServerError`: a plugin or operator can subscribe to "my config + /// stopped loading" without taking every server error. + ConfigReloadFailed(ConfigReloadFailure), + /// A target's `watch` config saw its group membership change. + PlaylistWatchChanged(WatchChanges), + /// A target gained or lost whole groups between refreshes. + PlaylistGroupsChanged(PlaylistGroupsChanged), + /// A target's `watch` config is configured but not working. + PlaylistWatchDisabled(WatchDisabled), + /// `watch` patterns that matched no group in the refreshed playlist. + PlaylistWatchUnmatched(WatchUnmatched), + /// A recording started, finished or failed. One variant, three kinds - + /// see [`EventMessage::kind`] - so a subscriber can ask for failures + /// alone. + RecordingLifecycle(RecordingLifecycleMessage), + /// A provider account changed status, is about to expire, or has. + ProviderAccount(ProviderAccountEvent), + /// An input's playlist fetch failed. + ProviderFetchFailed(ProviderFetchFailure), + /// Every provider behind an input was at capacity. + ProviderPoolExhausted(ProviderPoolExhausted), + /// An input started being served from a different priority group. + ProviderPriorityFallback(ProviderPriorityFallback), + + /// An API-proxy user was created, changed or removed. One variant, + /// three kinds - see [`EventMessage::kind`] - so a subscriber can ask + /// for deletions alone. + UserLifecycle(UserLifecycleEvent), + /// A user was refused a connection because their limits were full. + ConnectionDenied(ConnectionDenied), + + /// A stream probe returned no metadata. There is no success + /// counterpart; see [`StreamProbeFailure`]. + StreamProbeFailed(StreamProbeFailure), + + /// A scheduled task could not complete. + ScheduledTaskFailed(ScheduledTaskFailure), + + /// A notification ran out of delivery attempts and was dropped. + /// + /// On the bus for plugins and the status endpoint; deliberately *not* + /// notifiable - see [`NotificationDeadLetter`] for why routing this + /// through the outbox that just failed would loop. + NotificationDeadLettered(NotificationDeadLetter), + + /// An authentication decision: a sign-in, a rejected sign-in, a + /// throttled attempt, or a permission denial. One variant, four kinds - + /// see [`EventMessage::kind`] - so a subscriber can ask for the failures + /// without being woken by every successful sign-in. + AuthAudit(AuthAuditEvent), +} + +impl EventMessage { + /// Which event this is. + #[must_use] + pub const fn kind(&self) -> EventKind { + match self { + Self::ServerError(_) => EventKind::ServerError, + Self::ServerLifecycle(event) => match event.state { + ServerLifecycleState::Started => EventKind::ServerStarted, + ServerLifecycleState::ShuttingDown => EventKind::ServerShutdown, + }, + Self::ActiveUser(_) => EventKind::ActiveUser, + Self::ActiveProvider(_, _) => EventKind::ActiveProvider, + Self::ConfigChange(_) => EventKind::ConfigChange, + Self::PlaylistUpdate(_) => EventKind::PlaylistUpdate, + Self::PlaylistUpdateProgress(_) => EventKind::PlaylistUpdateProgress, + Self::SystemInfoUpdate(_) => EventKind::SystemInfoUpdate, + // One payload, two kinds. The emitter sends exactly one event per + // scan - a success or a failure - so the status is the whole + // discriminant. + Self::LibraryScanProgress(event) => match event.summary.status { + LibraryScanSummaryStatus::Success => EventKind::LibraryScanProgress, + LibraryScanSummaryStatus::Error => EventKind::LibraryScanFailed, + }, + Self::RecordingChanged => EventKind::RecordingChanged, + Self::RecordingProgress => EventKind::RecordingProgress, + Self::RecordingRulesChanged => EventKind::RecordingRulesChanged, + Self::InputMetadataUpdatesCompleted(_) => EventKind::InputMetadataUpdatesCompleted, + Self::InputMetadataUpdatesStarted(_) => EventKind::InputMetadataUpdatesStarted, + Self::InputMetadataUpdatesFailed(_) => EventKind::InputMetadataUpdatesFailed, + Self::DiskAlert(_) => EventKind::DiskAlert, + Self::ConfigReloadFailed(_) => EventKind::ConfigReloadFailed, + Self::PlaylistWatchChanged(_) => EventKind::PlaylistWatchChanged, + Self::PlaylistGroupsChanged(_) => EventKind::PlaylistGroupsChanged, + Self::PlaylistWatchDisabled(_) => EventKind::PlaylistWatchDisabled, + Self::PlaylistWatchUnmatched(_) => EventKind::PlaylistWatchUnmatched, + // One payload, three kinds: a subscriber that only cares about + // failures should not be woken for every completed recording. + Self::RecordingLifecycle(msg) => match msg.event { + MsgKind::RecordingStarted => EventKind::RecordingStarted, + MsgKind::RecordingCompleted => EventKind::RecordingCompleted, + // `RecordingLifecycleMessage::event` is typed as the whole + // `MsgKind`; anything that is not a start or a completion is + // reported as a failure rather than silently miscategorised. + _ => EventKind::RecordingFailed, + }, + Self::ProviderAccount(event) => match event.state { + ProviderAccountState::StatusChanged => EventKind::ProviderAccountStatus, + ProviderAccountState::Expiring => EventKind::ProviderAccountExpiring, + ProviderAccountState::Expired => EventKind::ProviderAccountExpired, + }, + Self::ProviderFetchFailed(_) => EventKind::ProviderFetchFailed, + Self::ProviderPoolExhausted(_) => EventKind::ProviderPoolExhausted, + Self::ProviderPriorityFallback(_) => EventKind::ProviderPriorityFallback, + Self::UserLifecycle(event) => match event.state { + UserLifecycleState::Created => EventKind::UserCreated, + UserLifecycleState::Updated => EventKind::UserUpdated, + UserLifecycleState::Deleted => EventKind::UserDeleted, + }, + Self::ConnectionDenied(_) => EventKind::ConnectionDenied, + Self::StreamProbeFailed(_) => EventKind::StreamProbeFailed, + Self::NotificationDeadLettered(_) => EventKind::NotificationDeadLettered, + Self::ScheduledTaskFailed(_) => EventKind::ScheduledTaskFailed, + Self::AuthAudit(event) => match event.outcome { + AuthAuditOutcome::SignInSucceeded => EventKind::AuthSignInSucceeded, + AuthAuditOutcome::SignInFailed => EventKind::AuthSignInFailed, + AuthAuditOutcome::SignInThrottled => EventKind::AuthSignInThrottled, + AuthAuditOutcome::PermissionDenied => EventKind::AuthPermissionDenied, + }, + } + } + + /// How bad this particular occurrence is. + /// + /// Depends on the payload, not just the kind: a playlist update that + /// failed is an error and one that succeeded is not. + #[must_use] + pub fn severity(&self) -> Severity { + match self { + // The registry cannot answer this either: whether a fetch failure + // needs a human is a property of the classification, not the id. + Self::ProviderFetchFailed(failure) if !failure.needs_operator => Severity::Warn, + // The only case the registry cannot answer: a partial refresh + // and a clean one share `PLAYLIST_UPDATE_COMPLETED`, but a + // partial one is not a clean success. + Self::PlaylistUpdate(summary) if summary.state == PlaylistUpdateState::Partial => Severity::Warn, + // Everything else takes the severity its registered event + // declares, so there is no second severity table to drift. + _ => self.notification_id().map_or(Severity::Info, registry::default_severity), + } + } + + /// The notification this event becomes, if it is notifiable at all. + /// + /// On `EventMessage` rather than `EventKind` because two of them depend + /// on the payload: a playlist update that failed is a different + /// notification from one that succeeded, not merely a more severe one. + /// + /// The bridge used to hold this table itself, so the event taxonomy and + /// the notification taxonomy drifted independently. Now the event says + /// what it is and the bridge only decides how to word it. + /// + /// `None` is the honest answer for the high-frequency kinds: they fire + /// many times per operation and their terminal counterparts carry the + /// news. + #[must_use] + pub const fn notification_id(&self) -> Option { + Some(match self { + Self::ServerError(_) => registry::SYSTEM_ERROR, + Self::ServerLifecycle(event) => match event.state { + ServerLifecycleState::Started => registry::SYSTEM_STARTED, + ServerLifecycleState::ShuttingDown => registry::SYSTEM_SHUTDOWN, + }, + Self::PlaylistUpdate(summary) => match summary.state { + PlaylistUpdateState::Success | PlaylistUpdateState::Partial => registry::PLAYLIST_UPDATE_COMPLETED, + PlaylistUpdateState::Failure => registry::PLAYLIST_UPDATE_FAILED, + }, + Self::ConfigChange(_) => registry::CONFIG_CHANGED, + // A failed scan used to take `LIBRARY_SCAN_COMPLETED` like any + // other, so operators were told "A local library scan finished" + // at info severity when it had not. + Self::LibraryScanProgress(event) => match event.summary.status { + LibraryScanSummaryStatus::Success => registry::LIBRARY_SCAN_COMPLETED, + LibraryScanSummaryStatus::Error => registry::LIBRARY_SCAN_FAILED, + }, + Self::InputMetadataUpdatesStarted(_) => registry::METADATA_UPDATE_STARTED, + Self::InputMetadataUpdatesCompleted(_) => registry::METADATA_UPDATE_COMPLETED, + Self::InputMetadataUpdatesFailed(_) => registry::METADATA_UPDATE_FAILED, + Self::DiskAlert(_) => registry::SYSTEM_DISK_ALERT, + Self::ConfigReloadFailed(_) => registry::CONFIG_RELOAD_FAILED, + Self::PlaylistWatchChanged(_) => registry::PLAYLIST_WATCH_CHANGED, + Self::PlaylistGroupsChanged(_) => registry::PLAYLIST_GROUPS_CHANGED, + Self::PlaylistWatchDisabled(_) => registry::PLAYLIST_WATCH_DISABLED, + Self::PlaylistWatchUnmatched(_) => registry::PLAYLIST_WATCH_UNMATCHED, + Self::RecordingLifecycle(msg) => match msg.event { + MsgKind::RecordingStarted => registry::RECORDING_STARTED, + MsgKind::RecordingCompleted => registry::RECORDING_COMPLETED, + _ => registry::RECORDING_FAILED, + }, + Self::ProviderAccount(event) => match event.state { + ProviderAccountState::StatusChanged => registry::PROVIDER_ACCOUNT_STATUS, + ProviderAccountState::Expiring => registry::PROVIDER_ACCOUNT_EXPIRING, + ProviderAccountState::Expired => registry::PROVIDER_ACCOUNT_EXPIRED, + }, + Self::ProviderFetchFailed(_) => registry::PROVIDER_FETCH_FAILED, + Self::ProviderPoolExhausted(_) => registry::PROVIDER_POOL_EXHAUSTED, + Self::ProviderPriorityFallback(_) => registry::PROVIDER_PRIORITY_FALLBACK, + Self::UserLifecycle(event) => match event.state { + UserLifecycleState::Created => registry::USER_CREATED, + UserLifecycleState::Updated => registry::USER_UPDATED, + UserLifecycleState::Deleted => registry::USER_DELETED, + }, + Self::ConnectionDenied(_) => registry::USER_CONNECTION_DENIED, + Self::StreamProbeFailed(_) => registry::STREAM_PROBE_FAILED, + Self::NotificationDeadLettered(_) => registry::NOTIFICATION_DEAD_LETTERED, + Self::ScheduledTaskFailed(_) => registry::SCHEDULED_TASK_FAILED, + Self::AuthAudit(event) => match event.outcome { + AuthAuditOutcome::SignInSucceeded => registry::AUTH_SIGN_IN_SUCCEEDED, + AuthAuditOutcome::SignInFailed => registry::AUTH_SIGN_IN_FAILED, + AuthAuditOutcome::SignInThrottled => registry::AUTH_SIGN_IN_THROTTLED, + AuthAuditOutcome::PermissionDenied => registry::AUTH_PERMISSION_DENIED, + }, + Self::ActiveUser(_) => registry::USER_CONNECTION_CHANGED, + Self::ActiveProvider(_, _) => registry::PROVIDER_CONNECTIONS_CHANGED, + Self::RecordingChanged => registry::RECORDING_QUEUE_CHANGED, + Self::RecordingRulesChanged => registry::RECORDING_RULES_CHANGED, + Self::PlaylistUpdateProgress(_) | Self::SystemInfoUpdate(_) | Self::RecordingProgress => return None, + }) + } + + /// The event as JSON, for consumers that do not speak Rust types. + /// + /// Plugins are handed a wire name and a JSON payload rather than the + /// enum, so this is what a plugin host serialises. Defining it here means + /// a new variant arrives with its payload shape already decided, instead + /// of the host growing a second `match` over the whole taxonomy. + /// + /// Never fails: a payload that will not serialise degrades to `null` + /// rather than dropping the event, because a plugin that learns a + /// refresh finished but not its statistics is still better served than + /// one that hears nothing. + #[must_use] + pub fn payload(&self) -> serde_json::Value { + fn encode(value: &T) -> serde_json::Value { + serde_json::to_value(value).unwrap_or(serde_json::Value::Null) + } + match self { + Self::ServerError(error) => serde_json::json!({ "error": error }), + Self::ServerLifecycle(event) => encode(event), + Self::ActiveUser(change) => encode(change), + Self::ActiveProvider(name, connections) => { + serde_json::json!({ "provider": name.as_ref(), "connections": connections }) + } + // `ConfigType` is not `Serialize`; its display form is the + // stable name operators already see in the Web UI. + Self::ConfigChange(config_type) => serde_json::json!({ "config_type": config_type.to_string() }), + Self::PlaylistUpdate(summary) => encode(summary), + Self::PlaylistUpdateProgress(progress) => encode(progress), + Self::SystemInfoUpdate(info) => encode(info.as_ref()), + Self::LibraryScanProgress(progress) => encode(progress), + Self::RecordingChanged | Self::RecordingProgress | Self::RecordingRulesChanged => serde_json::Value::Null, + Self::InputMetadataUpdatesStarted(input) | Self::InputMetadataUpdatesCompleted(input) => { + serde_json::json!({ "input": input.as_ref() }) + } + Self::InputMetadataUpdatesFailed(failure) => encode(failure), + Self::DiskAlert(alert) => encode(alert), + Self::ConfigReloadFailed(failure) => encode(failure), + Self::PlaylistWatchChanged(changes) => encode(changes), + Self::PlaylistGroupsChanged(changes) => encode(changes), + Self::PlaylistWatchDisabled(disabled) => encode(disabled), + Self::PlaylistWatchUnmatched(unmatched) => encode(unmatched), + Self::RecordingLifecycle(msg) => encode(msg), + Self::ProviderAccount(event) => encode(event), + Self::ProviderFetchFailed(failure) => encode(failure), + Self::ProviderPoolExhausted(exhausted) => encode(exhausted), + Self::ProviderPriorityFallback(fallback) => encode(fallback), + Self::UserLifecycle(event) => encode(event), + Self::ConnectionDenied(denied) => encode(denied), + Self::StreamProbeFailed(failure) => encode(failure), + Self::NotificationDeadLettered(dead_letter) => encode(dead_letter), + Self::ScheduledTaskFailed(failure) => encode(failure), + Self::AuthAudit(event) => encode(event), + } + } + + /// See [`EventKind::required_permission`]. + #[must_use] + pub const fn required_permission(&self) -> Permission { self.kind().required_permission() } + + /// See [`EventKind::is_high_frequency`]. + #[must_use] + pub const fn is_high_frequency(&self) -> bool { self.kind().is_high_frequency() } +} + +/// What one playlist refresh did. +/// +/// `PlaylistUpdate` used to carry the state alone, so "the refresh finished" +/// reached the bus but what it actually did did not - the run summary went +/// straight to the notification layer as a second, separate message with the +/// same registered id. Subscribers saw an outcome with no detail, operators +/// received two notifications per refresh, and a plugin asking for +/// `playlist.update` got a bare enum. +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct PlaylistUpdateSummary { + /// Stable identity of the processing run. Missing only on legacy/test-only events. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub run_id: Option, + /// Actual serial execution order. Missing only on legacy/test-only events. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub execution_order: Option, + pub state: PlaylistUpdateState, + /// Per-source statistics. Empty when the run produced none. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub stats: Vec, + /// The aggregated error text, when the run reported errors. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl PlaylistUpdateSummary { + /// A summary carrying only an outcome - the timeout and panic paths, + /// which have no statistics to report. + #[must_use] + pub fn state_only(state: PlaylistUpdateState) -> Self { + Self { run_id: None, execution_order: None, state, stats: Vec::new(), error: None } + } + + #[must_use] + pub fn for_run( + run_id: PlaylistUpdateRunId, + execution_order: PlaylistUpdateRunOrder, + state: PlaylistUpdateState, + ) -> Self { + Self { run_id: Some(run_id), execution_order: Some(execution_order), state, stats: Vec::new(), error: None } + } +} diff --git a/shared/src/model/event/sink.rs b/shared/src/model/event/sink.rs new file mode 100644 index 000000000..8fb68515c --- /dev/null +++ b/shared/src/model/event/sink.rs @@ -0,0 +1,73 @@ +use super::EventMessage; +use std::sync::Arc; + +/// Somewhere an [`EventMessage`] can be published. +/// +/// A bound, not a trait object: emitters are generic over their sink and +/// monomorphise against the one they were built with. Three things fall out +/// of that: +/// +/// * the pipeline no longer threads `Option>` around and +/// branches on it at every emit site - [`NoopSink`] is the absent case, +/// and its `emit` is an empty function the optimiser deletes outright; +/// * tests can assert on what was emitted without standing up a broadcast +/// channel and racing a subscriber; +/// * `dvr` and the notification path can name a sink without depending on +/// the streaming-session runtime that implements it. +/// +/// Implementations must not block. The bus is reached from the streaming +/// data path, and an emitter that can stall is an emitter that will. +pub trait EventSink: Send + Sync { + /// Publish. Best-effort by contract: no subscribers, a full buffer or a + /// closed channel are all normal and none of them are the emitter's + /// problem. + fn emit(&self, event: EventMessage); + /// Forward a confirmed account response to the runtime provider manager. + /// Sinks without a provider runtime deliberately ignore the observation. + fn observe_provider_account( + &self, + _name: Arc, + _identity: super::super::ProviderAccountIdentity, + _status: Option, + _exp_date: Option, + ) { + } +} + +/// The sink that drops everything. +/// +/// Replaces the `Option` at call sites that only ever had a `None` case +/// because tests and one-shot CLI runs have no bus. Monomorphised against +/// this, every emit site compiles to nothing. +#[derive(Debug, Clone, Copy, Default)] +pub struct NoopSink; + +impl EventSink for NoopSink { + fn emit(&self, _event: EventMessage) {} +} + +impl EventSink for Arc { + fn emit(&self, event: EventMessage) { (**self).emit(event); } + fn observe_provider_account( + &self, + name: Arc, + identity: super::super::ProviderAccountIdentity, + status: Option, + exp_date: Option, + ) { + (**self).observe_provider_account(name, identity, status, exp_date); + } +} + +impl EventSink for &T { + fn emit(&self, event: EventMessage) { (**self).emit(event); } + fn observe_provider_account( + &self, + name: Arc, + identity: super::super::ProviderAccountIdentity, + status: Option, + exp_date: Option, + ) { + (**self).observe_provider_account(name, identity, status, exp_date); + } +} diff --git a/shared/src/model/event/tests.rs b/shared/src/model/event/tests.rs new file mode 100644 index 000000000..286eb0943 --- /dev/null +++ b/shared/src/model/event/tests.rs @@ -0,0 +1,57 @@ +use super::{EventKind, EventKindMask}; + +#[test] +fn every_kind_has_a_distinct_bit() { + let mut seen = 0u64; + for kind in EventKind::ALL { + assert_eq!(seen & kind.bit(), 0, "{kind:?} shares a bit with an earlier kind"); + seen |= kind.bit(); + } +} + +/// `EventKindMask` is a `u64`, so the taxonomy cannot outgrow 64 kinds +/// without the mask type changing with it. It was a `u32` at 23 kinds; +/// the widening happened while no operator had a subscription list to +/// migrate, which is the only cheap time to do it. +#[test] +fn the_taxonomy_still_fits_in_the_mask() { + assert!(EventKind::ALL.len() <= 64, "EventKindMask needs a wider integer"); +} + +#[test] +fn wire_names_are_unique_and_round_trip() { + let mut names: Vec<&str> = EventKind::ALL.iter().map(|kind| kind.as_wire_name()).collect(); + names.sort_unstable(); + let count = names.len(); + names.dedup(); + assert_eq!(names.len(), count, "two kinds share a wire name"); + + for kind in EventKind::ALL { + assert_eq!(EventKind::from_wire_name(kind.as_wire_name()), Some(kind)); + } +} + +#[test] +fn an_unknown_subscription_name_is_reported_not_ignored() { + let (mask, unknown) = EventKindMask::from_wire_names(["playlist.update", "playlist.updat", "config.changed"]); + + assert!(mask.contains(EventKind::PlaylistUpdate)); + assert!(mask.contains(EventKind::ConfigChange)); + assert_eq!(unknown, vec!["playlist.updat"], "a typo must surface, not silently narrow the subscription"); +} + +#[test] +fn a_mask_round_trips_through_its_kinds() { + let mask = EventKindMask::from_iter([EventKind::ServerError, EventKind::RecordingChanged]); + assert_eq!(mask.kinds(), vec![EventKind::ServerError, EventKind::RecordingChanged]); + assert!(!mask.contains(EventKind::SystemInfoUpdate)); + assert!(!mask.is_empty()); + assert!(EventKindMask::NONE.is_empty()); +} + +#[test] +fn all_matches_every_kind() { + for kind in EventKind::ALL { + assert!(EventKindMask::ALL.contains(kind)); + } +} diff --git a/shared/src/model/notification.rs b/shared/src/model/notification.rs index ce4cabe52..7c512d066 100644 --- a/shared/src/model/notification.rs +++ b/shared/src/model/notification.rs @@ -18,8 +18,6 @@ //! through [`LEGACY_ALIASES`], so an existing `config.yml` keeps working //! untouched. -use std::fmt; - /// Stable dotted identity of a notification event. /// /// The inner string is the wire form: it appears in `notify_on` patterns, @@ -32,441 +30,12 @@ use std::fmt; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct EventId(&'static str); -impl EventId { - /// Declare an event id. `const` so the registry is built at compile time. - #[must_use] - pub const fn new(id: &'static str) -> Self { Self(id) } - - #[must_use] - pub const fn as_str(&self) -> &'static str { self.0 } - - /// The `domain` part - everything before the first dot. - /// - /// Used to group events in the UI and to answer "does this subscription - /// cover the whole domain". - #[must_use] - pub fn domain(&self) -> &'static str { - match self.0.split_once('.') { - Some((domain, _)) => domain, - None => self.0, - } - } - - /// Filename-safe form: dots become underscores. - /// - /// `recording.completed` yields `recording_completed`, which is exactly - /// the legacy template filename for that kind - so recording templates - /// are discovered by the canonical id with no alias lookup needed. - #[must_use] - pub fn file_stem(&self) -> String { self.0.replace('.', "_") } - - /// `{prefix}_{file_stem}.templ`, the on-disk template name. - #[must_use] - pub fn template_filename(&self, prefix: &str) -> String { format!("{prefix}_{}.templ", self.file_stem()) } - - /// Resolve a wire string to a known event id. - /// - /// Accepts the canonical dotted id and every legacy `MsgKind` wire - /// name. Returns `None` for an id that is not in the registry, which is - /// what lets config validation reject a typo instead of silently - /// subscribing to an event that will never fire. - #[must_use] - pub fn from_wire(s: &str) -> Option { - if let Some((_, id)) = LEGACY_ALIASES.iter().find(|(legacy, _)| legacy.eq_ignore_ascii_case(s)) { - return Some(*id); - } - registry::ALL.iter().find(|d| d.id.0.eq_ignore_ascii_case(s)).map(|d| d.id) - } -} - -impl fmt::Display for EventId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.0) } -} - -impl serde::Serialize for EventId { - fn serialize(&self, s: S) -> Result { s.serialize_str(self.0) } -} - -impl<'de> serde::Deserialize<'de> for EventId { - fn deserialize>(d: D) -> Result { - let raw = >::deserialize(d)?; - // An unknown id in a persisted outbox entry must not fail the whole - // file - see `UNKNOWN`. Config validation rejects typos separately, - // where an error message can actually reach the user. - Ok(Self::from_wire(&raw).unwrap_or(registry::UNKNOWN)) - } -} - -/// How much the operator is expected to care. -/// -/// Ordered, so a channel can subscribe with `min_severity` and get -/// everything at or above it. -#[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - Hash, - PartialOrd, - Ord, - Default, - serde::Serialize, - serde::Deserialize, - strum_macros::Display, - strum_macros::EnumString, - strum_macros::IntoStaticStr, -)] -#[serde(rename_all = "snake_case")] -#[strum(serialize_all = "snake_case", ascii_case_insensitive)] -pub enum Severity { - /// Something finished normally. Safe to route nowhere. - #[default] - Info, - /// Degraded but self-correcting, or a threshold approached. - Warn, - /// An operation failed. Someone should look, eventually. - Error, - /// Service is impaired now. Worth waking someone. - Critical, -} - -impl Severity { - #[must_use] - pub const fn wire_name(self) -> &'static str { - match self { - Self::Info => "info", - Self::Warn => "warn", - Self::Error => "error", - Self::Critical => "critical", - } - } - - #[must_use] - pub fn from_wire(s: &str) -> Option { s.parse().ok() } -} - -/// One entry in the event registry. -#[derive(Debug, Clone, Copy)] -pub struct EventDescriptor { - pub id: EventId, - /// Severity the emitter uses unless it overrides per-occurrence. - pub severity: Severity, - /// One line, rendered into the docs table and the UI picker. - pub description: &'static str, -} - -/// Legacy `MsgKind` wire name -> canonical id. -/// -/// Every entry here is load-bearing for an existing `config.yml`. Removing -/// one silently stops honouring a `notify_on` line that used to work, so -/// entries are append-only. -pub const LEGACY_ALIASES: &[(&str, EventId)] = &[ - ("info", registry::SYSTEM_INFO), - ("stats", registry::PLAYLIST_UPDATE_COMPLETED), - ("error", registry::SYSTEM_ERROR), - ("watch", registry::PLAYLIST_WATCH_CHANGED), - ("disk_alert", registry::SYSTEM_DISK_ALERT), - ("diskalert", registry::SYSTEM_DISK_ALERT), - ("recording_started", registry::RECORDING_STARTED), - ("recordingstarted", registry::RECORDING_STARTED), - ("recording_completed", registry::RECORDING_COMPLETED), - ("recordingcompleted", registry::RECORDING_COMPLETED), - ("recording_failed", registry::RECORDING_FAILED), - ("recordingfailed", registry::RECORDING_FAILED), -]; - /// The known event ids. /// /// Adding an event is a const here plus an entry in [`ALL`] - and then one /// emit call at the site that knows the event happened. Nothing else in /// the notification path needs to change. -pub mod registry { - use super::{EventDescriptor, EventId, Severity}; - - /// Fallback for an id read back from a persisted outbox that this build - /// does not know. Lets an outbox written by a newer version round-trip - /// through an older one instead of failing the whole file. - pub const UNKNOWN: EventId = EventId::new("unknown"); - - // ---- system --------------------------------------------------------- - pub const SYSTEM_INFO: EventId = EventId::new("system.info"); - pub const SYSTEM_ERROR: EventId = EventId::new("system.error"); - pub const SYSTEM_DISK_ALERT: EventId = EventId::new("system.disk.alert"); - pub const SYSTEM_STARTED: EventId = EventId::new("system.started"); - pub const SYSTEM_SHUTDOWN: EventId = EventId::new("system.shutdown"); - - // ---- playlist ------------------------------------------------------- - pub const PLAYLIST_UPDATE_COMPLETED: EventId = EventId::new("playlist.update.completed"); - pub const PLAYLIST_UPDATE_FAILED: EventId = EventId::new("playlist.update.failed"); - pub const PLAYLIST_WATCH_CHANGED: EventId = EventId::new("playlist.watch.changed"); - pub const PLAYLIST_GROUPS_CHANGED: EventId = EventId::new("playlist.groups.changed"); - pub const PLAYLIST_WATCH_DISABLED: EventId = EventId::new("playlist.watch.disabled"); - pub const PLAYLIST_WATCH_UNMATCHED: EventId = EventId::new("playlist.watch.unmatched"); - - // ---- recording ------------------------------------------------------ - pub const RECORDING_STARTED: EventId = EventId::new("recording.started"); - pub const RECORDING_COMPLETED: EventId = EventId::new("recording.completed"); - pub const RECORDING_FAILED: EventId = EventId::new("recording.failed"); - - // ---- provider ------------------------------------------------------- - pub const PROVIDER_ACCOUNT_STATUS: EventId = EventId::new("provider.account.status_changed"); - pub const PROVIDER_ACCOUNT_EXPIRING: EventId = EventId::new("provider.account.expiring"); - pub const PROVIDER_ACCOUNT_EXPIRED: EventId = EventId::new("provider.account.expired"); - pub const PROVIDER_FETCH_FAILED: EventId = EventId::new("provider.fetch.failed"); - pub const PROVIDER_POOL_EXHAUSTED: EventId = EventId::new("provider.pool.exhausted"); - pub const PROVIDER_PRIORITY_FALLBACK: EventId = EventId::new("provider.priority.fallback"); - - // ---- config --------------------------------------------------------- - pub const CONFIG_CHANGED: EventId = EventId::new("config.changed"); - pub const CONFIG_RELOAD_FAILED: EventId = EventId::new("config.reload_failed"); - - // ---- library -------------------------------------------------------- - pub const LIBRARY_SCAN_COMPLETED: EventId = EventId::new("library.scan.completed"); - pub const LIBRARY_SCAN_FAILED: EventId = EventId::new("library.scan.failed"); - - // ---- metadata ------------------------------------------------------- - pub const METADATA_UPDATE_STARTED: EventId = EventId::new("metadata.update.started"); - pub const METADATA_UPDATE_COMPLETED: EventId = EventId::new("metadata.update.completed"); - pub const METADATA_UPDATE_FAILED: EventId = EventId::new("metadata.update.failed"); - - // ---- users and connections ------------------------------------------ - /// High frequency. Subscribe deliberately. - pub const USER_CONNECTION_CHANGED: EventId = EventId::new("user.connection.changed"); - /// High frequency. Subscribe deliberately. - pub const PROVIDER_CONNECTIONS_CHANGED: EventId = EventId::new("provider.connections.changed"); - /// A user was refused a connection because their limits were full. Not - /// high frequency: reported per refusal, and a refusal is not routine. - pub const USER_CONNECTION_DENIED: EventId = EventId::new("user.connection.denied"); - - // ---- user accounts --------------------------------------------------- - pub const USER_CREATED: EventId = EventId::new("user.created"); - pub const USER_UPDATED: EventId = EventId::new("user.updated"); - pub const USER_DELETED: EventId = EventId::new("user.deleted"); - - // ---- authentication -------------------------------------------------- - pub const AUTH_SIGN_IN_SUCCEEDED: EventId = EventId::new("auth.sign_in.succeeded"); - pub const AUTH_SIGN_IN_FAILED: EventId = EventId::new("auth.sign_in.failed"); - pub const AUTH_SIGN_IN_THROTTLED: EventId = EventId::new("auth.sign_in.throttled"); - pub const AUTH_PERMISSION_DENIED: EventId = EventId::new("auth.permission.denied"); - - // ---- streams --------------------------------------------------------- - pub const STREAM_PROBE_FAILED: EventId = EventId::new("stream.probe.failed"); - - // ---- dvr ------------------------------------------------------------ - pub const RECORDING_QUEUE_CHANGED: EventId = EventId::new("recording.queue.changed"); - pub const RECORDING_RULES_CHANGED: EventId = EventId::new("recording.rules.changed"); - - // ---- scheduler ------------------------------------------------------- - pub const SCHEDULED_TASK_FAILED: EventId = EventId::new("scheduled_task.failed"); - - // ---- notification self-reporting ------------------------------------ - /// A notification was permanently lost. Must never route back through - /// the channel that dropped it. - pub const NOTIFICATION_DEAD_LETTERED: EventId = EventId::new("notification.dead_lettered"); - - /// Every registered event, in display order. - pub const ALL: &[EventDescriptor] = &[ - EventDescriptor { id: SYSTEM_INFO, severity: Severity::Info, description: "A general informational message." }, - EventDescriptor { id: SYSTEM_ERROR, severity: Severity::Error, description: "A general error message." }, - EventDescriptor { - id: SYSTEM_DISK_ALERT, - severity: Severity::Warn, - description: "Disk usage crossed the warn or critical threshold.", - }, - EventDescriptor { - id: SYSTEM_STARTED, - severity: Severity::Info, - description: "The server finished starting up.", - }, - EventDescriptor { - id: SYSTEM_SHUTDOWN, - severity: Severity::Info, - description: "The server is shutting down cleanly.", - }, - EventDescriptor { - id: PLAYLIST_UPDATE_COMPLETED, - severity: Severity::Info, - description: "A playlist update finished; carries per-source statistics.", - }, - EventDescriptor { - id: PLAYLIST_UPDATE_FAILED, - severity: Severity::Error, - description: "A playlist update failed.", - }, - EventDescriptor { - id: PLAYLIST_WATCH_CHANGED, - severity: Severity::Info, - description: "Channels were added to or removed from a watched group.", - }, - EventDescriptor { - id: PLAYLIST_GROUPS_CHANGED, - severity: Severity::Info, - description: "Groups were added to or removed from a target.", - }, - EventDescriptor { - id: PLAYLIST_WATCH_DISABLED, - severity: Severity::Warn, - description: "A target's watch configuration is set but not working.", - }, - EventDescriptor { - id: PLAYLIST_WATCH_UNMATCHED, - severity: Severity::Warn, - description: "Watch patterns matched no group in the refreshed playlist.", - }, - EventDescriptor { id: RECORDING_STARTED, severity: Severity::Info, description: "A recording started." }, - EventDescriptor { id: RECORDING_COMPLETED, severity: Severity::Info, description: "A recording completed." }, - EventDescriptor { id: RECORDING_FAILED, severity: Severity::Error, description: "A recording failed." }, - EventDescriptor { - id: PROVIDER_ACCOUNT_STATUS, - severity: Severity::Warn, - description: "A provider reported a changed account status.", - }, - EventDescriptor { - id: PROVIDER_ACCOUNT_EXPIRING, - severity: Severity::Warn, - description: "A provider account is approaching its expiry date.", - }, - EventDescriptor { - id: PROVIDER_ACCOUNT_EXPIRED, - severity: Severity::Error, - description: "A provider account has expired.", - }, - EventDescriptor { - id: PROVIDER_FETCH_FAILED, - severity: Severity::Error, - description: "An input's playlist could not be fetched.", - }, - EventDescriptor { - id: PROVIDER_POOL_EXHAUSTED, - severity: Severity::Warn, - description: "Every provider behind an input was at capacity.", - }, - EventDescriptor { - id: PROVIDER_PRIORITY_FALLBACK, - severity: Severity::Warn, - description: "An input started being served from a different provider priority group.", - }, - EventDescriptor { - id: CONFIG_CHANGED, - severity: Severity::Info, - description: "A configuration file was changed and reloaded.", - }, - EventDescriptor { - id: CONFIG_RELOAD_FAILED, - severity: Severity::Error, - description: "A configuration file changed but could not be reloaded.", - }, - EventDescriptor { - id: LIBRARY_SCAN_COMPLETED, - severity: Severity::Info, - description: "A local library scan finished.", - }, - EventDescriptor { - id: LIBRARY_SCAN_FAILED, - severity: Severity::Error, - description: "A local library scan could not complete.", - }, - EventDescriptor { - id: METADATA_UPDATE_STARTED, - severity: Severity::Info, - description: "A metadata update started for an input.", - }, - EventDescriptor { - id: METADATA_UPDATE_COMPLETED, - severity: Severity::Info, - description: "A metadata update finished for an input.", - }, - EventDescriptor { - id: METADATA_UPDATE_FAILED, - severity: Severity::Error, - description: "A metadata update cycle ended with tasks it could not finish.", - }, - EventDescriptor { - id: USER_CONNECTION_CHANGED, - severity: Severity::Info, - description: "A user connected or disconnected. High frequency - subscribe deliberately.", - }, - EventDescriptor { - id: PROVIDER_CONNECTIONS_CHANGED, - severity: Severity::Info, - description: "A provider's active connection count changed. High frequency - subscribe deliberately.", - }, - EventDescriptor { - id: USER_CONNECTION_DENIED, - severity: Severity::Warn, - description: "A user was refused a connection because their limits were full.", - }, - EventDescriptor { - id: USER_CREATED, - severity: Severity::Info, - description: "An API-proxy user account was created.", - }, - EventDescriptor { - id: USER_UPDATED, - severity: Severity::Info, - description: "An API-proxy user account was changed.", - }, - EventDescriptor { - id: USER_DELETED, - severity: Severity::Warn, - description: "An API-proxy user account was deleted.", - }, - EventDescriptor { - id: AUTH_SIGN_IN_SUCCEEDED, - severity: Severity::Info, - description: "A principal signed in and was issued a token.", - }, - EventDescriptor { - id: AUTH_SIGN_IN_FAILED, - severity: Severity::Warn, - description: "A sign-in was rejected. Deduplicated per principal and address, so a password-guessing run notifies once rather than per attempt.", - }, - EventDescriptor { - id: AUTH_SIGN_IN_THROTTLED, - severity: Severity::Warn, - description: "A sign-in was refused without checking credentials because the caller is backing off after repeated failures.", - }, - EventDescriptor { - id: AUTH_PERMISSION_DENIED, - severity: Severity::Warn, - description: "An authenticated principal asked for something its permissions do not cover.", - }, - EventDescriptor { - id: STREAM_PROBE_FAILED, - severity: Severity::Warn, - description: "A stream probe returned no metadata; the stream may be dead. Deduplicated per input, so a provider outage notifies once rather than per channel.", - }, - EventDescriptor { - id: RECORDING_QUEUE_CHANGED, - severity: Severity::Info, - description: "The recording queue changed.", - }, - EventDescriptor { - id: RECORDING_RULES_CHANGED, - severity: Severity::Info, - description: "The recording rule set changed.", - }, - EventDescriptor { - id: SCHEDULED_TASK_FAILED, - severity: Severity::Error, - description: "A scheduled task could not complete.", - }, - EventDescriptor { - id: NOTIFICATION_DEAD_LETTERED, - severity: Severity::Error, - description: "A notification was permanently undeliverable and has been dropped.", - }, - ]; - - /// Look up the descriptor for an id. - #[must_use] - pub fn describe(id: EventId) -> Option<&'static EventDescriptor> { ALL.iter().find(|d| d.id == id) } - - /// Default severity for an id; `Info` for anything unregistered. - #[must_use] - pub fn default_severity(id: EventId) -> Severity { describe(id).map_or(Severity::Info, |d| d.severity) } -} +pub mod registry; // --------------------------------------------------------------------------- // Subscriptions @@ -495,340 +64,14 @@ pub struct EventPattern { raw: String, } -#[derive(Debug, Clone, PartialEq, Eq)] -enum Segment { - /// A literal segment. - Literal(String), - /// `*` in an interior position - matches exactly one segment. - One, - /// A trailing `.*` - matches one or more remaining segments. - Rest, -} - -impl fmt::Display for EventPattern { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&self.raw) } -} - -impl EventPattern { - /// Parse a pattern. Never fails: an empty pattern simply matches nothing, - /// which is the safe reading of a blank config line. - #[must_use] - pub fn parse(raw: &str) -> Self { - let trimmed = raw.trim(); - let (negated, body) = match trimmed.strip_prefix('!') { - Some(rest) => (true, rest.trim()), - None => (false, trimmed), - }; - let parts: Vec<&str> = if body.is_empty() { Vec::new() } else { body.split('.').collect() }; - let last = parts.len().saturating_sub(1); - let segments = parts - .iter() - .enumerate() - .map(|(i, part)| match *part { - "*" if i == last => Segment::Rest, - "*" => Segment::One, - literal => Segment::Literal(literal.to_string()), - }) - .collect(); - Self { segments, negated, raw: trimmed.to_string() } - } - - /// The pattern as written, for round-tripping back into config. - #[must_use] - pub fn as_str(&self) -> &str { &self.raw } - - /// Does this pattern cover `id`? Ignores negation - the caller combines. - #[must_use] - pub fn matches(&self, id: EventId) -> bool { Self::match_segments(&self.segments, id.as_str()) } - - fn match_segments(pattern: &[Segment], id: &str) -> bool { - let mut parts = id.split('.'); - for (i, segment) in pattern.iter().enumerate() { - match segment { - // A trailing `*` swallows every remaining segment, and - // requires at least one so `recording.*` does not match a - // bare `recording`. - Segment::Rest => { - // A lone `*` pattern matches everything, including a - // single-segment id. - return if i == 0 { true } else { parts.next().is_some() }; - } - Segment::One => { - if parts.next().is_none() { - return false; - } - } - Segment::Literal(want) => match parts.next() { - Some(got) if got.eq_ignore_ascii_case(want) => {} - _ => return false, - }, - } - } - // Every pattern segment consumed; the id must be exhausted too. - parts.next().is_none() - } -} - /// A parsed `notify_on` list. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct EventSubscription { patterns: Vec, } -impl EventSubscription { - #[must_use] - pub fn parse, S: AsRef>(raw: I) -> Self { - Self { patterns: raw.into_iter().map(|s| EventPattern::parse(s.as_ref())).collect() } - } - - /// The parsed patterns, for round-tripping back into config. - #[must_use] - pub fn patterns(&self) -> &[EventPattern] { &self.patterns } - - /// `true` when nothing is subscribed - the caller can skip all work. - #[must_use] - pub fn is_empty(&self) -> bool { self.patterns.is_empty() } - - /// At least one positive pattern matches and no negative pattern does. - #[must_use] - pub fn matches(&self, id: EventId) -> bool { - let mut included = false; - for pattern in &self.patterns { - if pattern.matches(id) { - if pattern.negated { - return false; - } - included = true; - } - } - included - } - - /// Patterns that match no registered event. - /// - /// Config validation surfaces these as a warning: a typo in `notify_on` - /// is otherwise indistinguishable from an event that simply never fires. - #[must_use] - pub fn unmatched_patterns(&self) -> Vec<&str> { - self.patterns - .iter() - .filter(|p| !registry::ALL.iter().any(|d| p.matches(d.id))) - .map(EventPattern::as_str) - .collect() - } -} - #[cfg(test)] -mod tests { - use super::{registry, EventId, EventPattern, EventSubscription, Severity, LEGACY_ALIASES}; - - #[test] - fn every_legacy_msgkind_wire_name_still_resolves() { - // Load-bearing for existing config.yml files: each of these was a - // valid `notify_on` entry before the open-world refactor. - for legacy in [ - "info", - "stats", - "error", - "watch", - "disk_alert", - "recording_started", - "recording_completed", - "recording_failed", - ] { - assert!(EventId::from_wire(legacy).is_some(), "legacy wire name {legacy} no longer resolves"); - } - } - - #[test] - fn legacy_alias_targets_are_all_registered() { - for (legacy, id) in LEGACY_ALIASES { - assert!(registry::describe(*id).is_some(), "alias {legacy} points at unregistered id {id}"); - } - } - - #[test] - fn canonical_ids_resolve_and_round_trip() { - for descriptor in registry::ALL { - let resolved = EventId::from_wire(descriptor.id.as_str()); - assert_eq!(resolved, Some(descriptor.id), "id {} did not round-trip", descriptor.id); - } - } - - #[test] - fn unknown_wire_name_does_not_resolve() { - assert_eq!(EventId::from_wire("recording.definitely_not_a_real_event"), None); - } - - #[test] - fn recording_template_filenames_match_the_legacy_names() { - // The legacy on-disk names were `telegram_recording_completed.templ` - // and friends. The canonical dotted ids produce the same stems, so - // existing template files keep being discovered. - assert_eq!(registry::RECORDING_COMPLETED.template_filename("telegram"), "telegram_recording_completed.templ"); - assert_eq!(registry::RECORDING_STARTED.template_filename("discord"), "discord_recording_started.templ"); - assert_eq!(registry::RECORDING_FAILED.template_filename("rest"), "rest_recording_failed.templ"); - } - - #[test] - fn domain_is_the_part_before_the_first_dot() { - assert_eq!(registry::RECORDING_COMPLETED.domain(), "recording"); - assert_eq!(registry::SYSTEM_DISK_ALERT.domain(), "system"); - assert_eq!(registry::UNKNOWN.domain(), "unknown"); - } - - #[test] - fn star_matches_every_registered_event() { - let sub = EventSubscription::parse(["*"]); - for descriptor in registry::ALL { - assert!(sub.matches(descriptor.id), "`*` failed to match {}", descriptor.id); - } - } - - #[test] - fn prefix_pattern_matches_the_domain_and_nothing_else() { - let sub = EventSubscription::parse(["recording.*"]); - assert!(sub.matches(registry::RECORDING_COMPLETED)); - assert!(sub.matches(registry::RECORDING_FAILED)); - assert!(!sub.matches(registry::SYSTEM_INFO)); - assert!(!sub.matches(registry::PLAYLIST_UPDATE_COMPLETED)); - } - - #[test] - fn prefix_pattern_requires_at_least_one_further_segment() { - // `recording.*` must not match a hypothetical bare `recording`. - assert!(!EventPattern::parse("recording.*").matches(EventId::new("recording"))); - } - - #[test] - fn exact_pattern_matches_only_itself() { - let sub = EventSubscription::parse(["recording.completed"]); - assert!(sub.matches(registry::RECORDING_COMPLETED)); - assert!(!sub.matches(registry::RECORDING_STARTED)); - } - - #[test] - fn interior_wildcard_matches_exactly_one_segment() { - let pattern = EventPattern::parse("provider.*.expired"); - assert!(pattern.matches(registry::PROVIDER_ACCOUNT_EXPIRED)); - // One segment, not several. - assert!(!pattern.matches(EventId::new("provider.a.b.expired"))); - // And not zero. - assert!(!pattern.matches(EventId::new("provider.expired"))); - } - - #[test] - fn negation_removes_from_a_wider_match() { - let sub = EventSubscription::parse(["*", "!system.info"]); - assert!(sub.matches(registry::SYSTEM_ERROR)); - assert!(!sub.matches(registry::SYSTEM_INFO), "negated pattern did not exclude"); - } - - #[test] - fn negation_wins_regardless_of_order() { - let before = EventSubscription::parse(["!recording.started", "recording.*"]); - let after = EventSubscription::parse(["recording.*", "!recording.started"]); - assert!(!before.matches(registry::RECORDING_STARTED)); - assert!(!after.matches(registry::RECORDING_STARTED)); - assert!(before.matches(registry::RECORDING_COMPLETED)); - assert!(after.matches(registry::RECORDING_COMPLETED)); - } - - #[test] - fn a_subscription_of_only_negations_matches_nothing() { - // No positive pattern means nothing was opted in to. - let sub = EventSubscription::parse(["!system.info"]); - assert!(!sub.matches(registry::SYSTEM_ERROR)); - assert!(!sub.matches(registry::SYSTEM_INFO)); - } - - #[test] - fn empty_subscription_matches_nothing() { - let sub = EventSubscription::parse(Vec::::new()); - assert!(sub.is_empty()); - assert!(!sub.matches(registry::SYSTEM_INFO)); - } - - #[test] - fn blank_pattern_matches_nothing_rather_than_everything() { - // A stray empty line in config must not silently subscribe to all. - let sub = EventSubscription::parse([" "]); - assert!(!sub.matches(registry::SYSTEM_INFO)); - } - - #[test] - fn unmatched_patterns_flags_typos_only() { - let sub = EventSubscription::parse(["recording.*", "recroding.completed", "*"]); - assert_eq!(sub.unmatched_patterns(), vec!["recroding.completed"]); - } - - #[test] - fn severity_orders_from_info_up_to_critical() { - assert!(Severity::Info < Severity::Warn); - assert!(Severity::Warn < Severity::Error); - assert!(Severity::Error < Severity::Critical); - } - - #[test] - fn severity_wire_names_round_trip() { - for severity in [Severity::Info, Severity::Warn, Severity::Error, Severity::Critical] { - assert_eq!(Severity::from_wire(severity.wire_name()), Some(severity)); - } - } - - #[test] - fn unknown_event_id_deserializes_to_unknown_instead_of_failing() { - // A newer build's outbox entry must not poison the whole file when - // read back by an older one. - let id: EventId = serde_json::from_str("\"some.future.event\"").expect("must not fail"); - assert_eq!(id, registry::UNKNOWN); - } - - #[test] - fn event_id_serializes_as_its_wire_string() { - let json = serde_json::to_string(®istry::RECORDING_COMPLETED).expect("serialize"); - assert_eq!(json, "\"recording.completed\""); - } - - /// The docs carry a table of every event id. A registry entry that never - /// reaches the table is undiscoverable, and a table row for an id that no - /// longer exists is a lie - so the two are checked against each other - /// rather than maintained by hand. - #[test] - fn every_registered_event_appears_in_the_docs_table() { - const DOCS: &str = include_str!("../../../docs/src/configuration/config.md"); - let table = DOCS - .split_once("") - .and_then(|(_, rest)| rest.split_once("")) - .map(|(table, _)| table) - .expect("docs must contain the generated event table markers"); - - for descriptor in registry::ALL { - let row = format!("| `{}` | {} | {} |", descriptor.id, descriptor.severity, descriptor.description); - assert!( - table.contains(&row), - "docs/src/configuration/config.md is missing this row - add it inside the generated table:\n{row}" - ); - } - - // And nothing in the table that the registry no longer knows about. - for line in table.lines().filter(|line| line.trim_start().starts_with("| `")) { - let id = line.trim_start().trim_start_matches("| `").split('`').next().unwrap_or_default(); - assert!( - EventId::from_wire(id).is_some(), - "docs list `{id}`, which is not a registered event - remove the row or register the event" - ); - } - } - - #[test] - fn registry_has_no_duplicate_ids() { - let mut seen = std::collections::HashSet::new(); - for descriptor in registry::ALL { - assert!(seen.insert(descriptor.id), "duplicate registry entry for {}", descriptor.id); - } - } -} +mod tests; // --------------------------------------------------------------------------- // Quiet hours @@ -846,111 +89,11 @@ pub struct QuietHours { end_min: u16, } -impl QuietHours { - /// Parse `HH:MM-HH:MM`. Returns `None` for anything malformed, which - /// config validation turns into an error the operator sees. - #[must_use] - pub fn parse(raw: &str) -> Option { - let (start, end) = raw.trim().split_once('-')?; - Some(Self { start_min: parse_hhmm(start)?, end_min: parse_hhmm(end)? }) - } - - /// Is `minutes_since_midnight` inside the window? - /// - /// Handles the wrapping case (`23:00-07:00`), which is the one people - /// actually configure. - #[must_use] - pub fn contains(&self, minutes_since_midnight: u16) -> bool { - if self.start_min == self.end_min { - // A zero-width window silences nothing. Treating it as "always" - // would mute a channel completely on a typo. - return false; - } - if self.start_min < self.end_min { - (self.start_min..self.end_min).contains(&minutes_since_midnight) - } else { - minutes_since_midnight >= self.start_min || minutes_since_midnight < self.end_min - } - } - - /// Minutes from `minutes_since_midnight` until the window ends. - #[must_use] - pub fn minutes_until_end(&self, minutes_since_midnight: u16) -> u16 { - if !self.contains(minutes_since_midnight) { - return 0; - } - if self.end_min > minutes_since_midnight { - self.end_min - minutes_since_midnight - } else { - (24 * 60 - minutes_since_midnight) + self.end_min - } - } -} - -fn parse_hhmm(raw: &str) -> Option { - let (h, m) = raw.trim().split_once(':')?; - let h: u16 = h.trim().parse().ok()?; - let m: u16 = m.trim().parse().ok()?; - if h > 23 || m > 59 { - return None; - } - Some(h * 60 + m) -} - #[cfg(test)] -mod quiet_hours_tests { - use super::QuietHours; +mod quiet_hours_tests; - #[test] - fn a_same_day_window_contains_only_its_own_range() { - let window = QuietHours::parse("09:00-17:00").expect("parse"); - assert!(!window.contains(8 * 60 + 59)); - assert!(window.contains(9 * 60)); - assert!(window.contains(16 * 60 + 59)); - // End is exclusive, so 17:00 sharp is out. - assert!(!window.contains(17 * 60)); - } - - #[test] - fn an_overnight_window_wraps_past_midnight() { - // The case people actually configure. - let window = QuietHours::parse("23:00-07:00").expect("parse"); - assert!(window.contains(23 * 60)); - assert!(window.contains(0)); - assert!(window.contains(6 * 60 + 59)); - assert!(!window.contains(7 * 60)); - assert!(!window.contains(12 * 60)); - } - - #[test] - fn a_zero_width_window_silences_nothing() { - // Treating it as "always" would mute the channel entirely on a typo. - let window = QuietHours::parse("08:00-08:00").expect("parse"); - for minute in [0, 8 * 60, 12 * 60, 23 * 60 + 59] { - assert!(!window.contains(minute), "zero-width window muted {minute}"); - } - } - - #[test] - fn minutes_until_end_handles_both_directions() { - let same_day = QuietHours::parse("09:00-17:00").expect("parse"); - assert_eq!(same_day.minutes_until_end(10 * 60), 7 * 60); - assert_eq!(same_day.minutes_until_end(18 * 60), 0, "outside the window there is nothing to wait for"); - - let overnight = QuietHours::parse("23:00-07:00").expect("parse"); - assert_eq!(overnight.minutes_until_end(23 * 60), 8 * 60); - assert_eq!(overnight.minutes_until_end(60), 6 * 60); - } - - #[test] - fn malformed_windows_are_rejected_rather_than_guessed() { - for bad in ["", "23:00", "25:00-07:00", "23:60-07:00", "23-07", "abc-def", "23:00_07:00"] { - assert!(QuietHours::parse(bad).is_none(), "accepted malformed window `{bad}`"); - } - } - - #[test] - fn whitespace_is_tolerated() { - assert_eq!(QuietHours::parse(" 23:00 - 07:00 "), QuietHours::parse("23:00-07:00")); - } -} +mod identity; +mod quiet_hours; +mod subscription; +pub use identity::{EventDescriptor, Severity, LEGACY_ALIASES}; +use subscription::Segment; diff --git a/shared/src/model/notification/identity.rs b/shared/src/model/notification/identity.rs new file mode 100644 index 000000000..613860805 --- /dev/null +++ b/shared/src/model/notification/identity.rs @@ -0,0 +1,146 @@ +use super::{registry, EventId}; +use std::fmt; + +impl EventId { + /// Declare an event id. `const` so the registry is built at compile time. + #[must_use] + pub const fn new(id: &'static str) -> Self { Self(id) } + + #[must_use] + pub const fn as_str(&self) -> &'static str { self.0 } + + /// The `domain` part - everything before the first dot. + /// + /// Used to group events in the UI and to answer "does this subscription + /// cover the whole domain". + #[must_use] + pub fn domain(&self) -> &'static str { + match self.0.split_once('.') { + Some((domain, _)) => domain, + None => self.0, + } + } + + /// Filename-safe form: dots become underscores. + /// + /// `recording.completed` yields `recording_completed`, which is exactly + /// the legacy template filename for that kind - so recording templates + /// are discovered by the canonical id with no alias lookup needed. + #[must_use] + pub fn file_stem(&self) -> String { self.0.replace('.', "_") } + + /// `{prefix}_{file_stem}.templ`, the on-disk template name. + #[must_use] + pub fn template_filename(&self, prefix: &str) -> String { format!("{prefix}_{}.templ", self.file_stem()) } + + /// Resolve a wire string to a known event id. + /// + /// Accepts the canonical dotted id and every legacy `MsgKind` wire + /// name. Returns `None` for an id that is not in the registry, which is + /// what lets config validation reject a typo instead of silently + /// subscribing to an event that will never fire. + #[must_use] + pub fn from_wire(s: &str) -> Option { + if let Some((_, id)) = LEGACY_ALIASES.iter().find(|(legacy, _)| legacy.eq_ignore_ascii_case(s)) { + return Some(*id); + } + registry::ALL.iter().find(|d| d.id.0.eq_ignore_ascii_case(s)).map(|d| d.id) + } +} + +impl fmt::Display for EventId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.0) } +} + +impl serde::Serialize for EventId { + fn serialize(&self, s: S) -> Result { s.serialize_str(self.0) } +} + +impl<'de> serde::Deserialize<'de> for EventId { + fn deserialize>(d: D) -> Result { + let raw = >::deserialize(d)?; + // An unknown id in a persisted outbox entry must not fail the whole + // file - see `UNKNOWN`. Config validation rejects typos separately, + // where an error message can actually reach the user. + Ok(Self::from_wire(&raw).unwrap_or(registry::UNKNOWN)) + } +} + +/// How much the operator is expected to care. +/// +/// Ordered, so a channel can subscribe with `min_severity` and get +/// everything at or above it. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + Default, + serde::Serialize, + serde::Deserialize, + strum_macros::Display, + strum_macros::EnumString, + strum_macros::IntoStaticStr, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case", ascii_case_insensitive)] +pub enum Severity { + /// Something finished normally. Safe to route nowhere. + #[default] + Info, + /// Degraded but self-correcting, or a threshold approached. + Warn, + /// An operation failed. Someone should look, eventually. + Error, + /// Service is impaired now. Worth waking someone. + Critical, +} + +impl Severity { + #[must_use] + pub const fn wire_name(self) -> &'static str { + match self { + Self::Info => "info", + Self::Warn => "warn", + Self::Error => "error", + Self::Critical => "critical", + } + } + + #[must_use] + pub fn from_wire(s: &str) -> Option { s.parse().ok() } +} + +/// One entry in the event registry. +#[derive(Debug, Clone, Copy)] +pub struct EventDescriptor { + pub id: EventId, + /// Severity the emitter uses unless it overrides per-occurrence. + pub severity: Severity, + /// One line, rendered into the docs table and the UI picker. + pub description: &'static str, +} + +/// Legacy `MsgKind` wire name -> canonical id. +/// +/// Every entry here is load-bearing for an existing `config.yml`. Removing +/// one silently stops honouring a `notify_on` line that used to work, so +/// entries are append-only. +pub const LEGACY_ALIASES: &[(&str, EventId)] = &[ + ("info", registry::SYSTEM_INFO), + ("stats", registry::PLAYLIST_UPDATE_COMPLETED), + ("error", registry::SYSTEM_ERROR), + ("watch", registry::PLAYLIST_WATCH_CHANGED), + ("disk_alert", registry::SYSTEM_DISK_ALERT), + ("diskalert", registry::SYSTEM_DISK_ALERT), + ("recording_started", registry::RECORDING_STARTED), + ("recordingstarted", registry::RECORDING_STARTED), + ("recording_completed", registry::RECORDING_COMPLETED), + ("recordingcompleted", registry::RECORDING_COMPLETED), + ("recording_failed", registry::RECORDING_FAILED), + ("recordingfailed", registry::RECORDING_FAILED), +]; diff --git a/shared/src/model/notification/quiet_hours.rs b/shared/src/model/notification/quiet_hours.rs new file mode 100644 index 000000000..333c9b820 --- /dev/null +++ b/shared/src/model/notification/quiet_hours.rs @@ -0,0 +1,52 @@ +use super::QuietHours; + +impl QuietHours { + /// Parse `HH:MM-HH:MM`. Returns `None` for anything malformed, which + /// config validation turns into an error the operator sees. + #[must_use] + pub fn parse(raw: &str) -> Option { + let (start, end) = raw.trim().split_once('-')?; + Some(Self { start_min: parse_hhmm(start)?, end_min: parse_hhmm(end)? }) + } + + /// Is `minutes_since_midnight` inside the window? + /// + /// Handles the wrapping case (`23:00-07:00`), which is the one people + /// actually configure. + #[must_use] + pub fn contains(&self, minutes_since_midnight: u16) -> bool { + if self.start_min == self.end_min { + // A zero-width window silences nothing. Treating it as "always" + // would mute a channel completely on a typo. + return false; + } + if self.start_min < self.end_min { + (self.start_min..self.end_min).contains(&minutes_since_midnight) + } else { + minutes_since_midnight >= self.start_min || minutes_since_midnight < self.end_min + } + } + + /// Minutes from `minutes_since_midnight` until the window ends. + #[must_use] + pub fn minutes_until_end(&self, minutes_since_midnight: u16) -> u16 { + if !self.contains(minutes_since_midnight) { + return 0; + } + if self.end_min > minutes_since_midnight { + self.end_min - minutes_since_midnight + } else { + (24 * 60 - minutes_since_midnight) + self.end_min + } + } +} + +fn parse_hhmm(raw: &str) -> Option { + let (h, m) = raw.trim().split_once(':')?; + let h: u16 = h.trim().parse().ok()?; + let m: u16 = m.trim().parse().ok()?; + if h > 23 || m > 59 { + return None; + } + Some(h * 60 + m) +} diff --git a/shared/src/model/notification/quiet_hours_tests.rs b/shared/src/model/notification/quiet_hours_tests.rs new file mode 100644 index 000000000..98d801a38 --- /dev/null +++ b/shared/src/model/notification/quiet_hours_tests.rs @@ -0,0 +1,54 @@ +use super::QuietHours; + +#[test] +fn a_same_day_window_contains_only_its_own_range() { + let window = QuietHours::parse("09:00-17:00").expect("parse"); + assert!(!window.contains(8 * 60 + 59)); + assert!(window.contains(9 * 60)); + assert!(window.contains(16 * 60 + 59)); + // End is exclusive, so 17:00 sharp is out. + assert!(!window.contains(17 * 60)); +} + +#[test] +fn an_overnight_window_wraps_past_midnight() { + // The case people actually configure. + let window = QuietHours::parse("23:00-07:00").expect("parse"); + assert!(window.contains(23 * 60)); + assert!(window.contains(0)); + assert!(window.contains(6 * 60 + 59)); + assert!(!window.contains(7 * 60)); + assert!(!window.contains(12 * 60)); +} + +#[test] +fn a_zero_width_window_silences_nothing() { + // Treating it as "always" would mute the channel entirely on a typo. + let window = QuietHours::parse("08:00-08:00").expect("parse"); + for minute in [0, 8 * 60, 12 * 60, 23 * 60 + 59] { + assert!(!window.contains(minute), "zero-width window muted {minute}"); + } +} + +#[test] +fn minutes_until_end_handles_both_directions() { + let same_day = QuietHours::parse("09:00-17:00").expect("parse"); + assert_eq!(same_day.minutes_until_end(10 * 60), 7 * 60); + assert_eq!(same_day.minutes_until_end(18 * 60), 0, "outside the window there is nothing to wait for"); + + let overnight = QuietHours::parse("23:00-07:00").expect("parse"); + assert_eq!(overnight.minutes_until_end(23 * 60), 8 * 60); + assert_eq!(overnight.minutes_until_end(60), 6 * 60); +} + +#[test] +fn malformed_windows_are_rejected_rather_than_guessed() { + for bad in ["", "23:00", "25:00-07:00", "23:60-07:00", "23-07", "abc-def", "23:00_07:00"] { + assert!(QuietHours::parse(bad).is_none(), "accepted malformed window `{bad}`"); + } +} + +#[test] +fn whitespace_is_tolerated() { + assert_eq!(QuietHours::parse(" 23:00 - 07:00 "), QuietHours::parse("23:00-07:00")); +} diff --git a/shared/src/model/notification/registry.rs b/shared/src/model/notification/registry.rs new file mode 100644 index 000000000..8ff8a2991 --- /dev/null +++ b/shared/src/model/notification/registry.rs @@ -0,0 +1,284 @@ +use super::{EventDescriptor, EventId, Severity}; + +/// Fallback for an id read back from a persisted outbox that this build +/// does not know. Lets an outbox written by a newer version round-trip +/// through an older one instead of failing the whole file. +pub const UNKNOWN: EventId = EventId::new("unknown"); + +// ---- system --------------------------------------------------------- +pub const SYSTEM_INFO: EventId = EventId::new("system.info"); +pub const SYSTEM_ERROR: EventId = EventId::new("system.error"); +pub const SYSTEM_DISK_ALERT: EventId = EventId::new("system.disk.alert"); +pub const SYSTEM_STARTED: EventId = EventId::new("system.started"); +pub const SYSTEM_SHUTDOWN: EventId = EventId::new("system.shutdown"); + +// ---- playlist ------------------------------------------------------- +pub const PLAYLIST_UPDATE_COMPLETED: EventId = EventId::new("playlist.update.completed"); +pub const PLAYLIST_UPDATE_FAILED: EventId = EventId::new("playlist.update.failed"); +pub const PLAYLIST_WATCH_CHANGED: EventId = EventId::new("playlist.watch.changed"); +pub const PLAYLIST_GROUPS_CHANGED: EventId = EventId::new("playlist.groups.changed"); +pub const PLAYLIST_WATCH_DISABLED: EventId = EventId::new("playlist.watch.disabled"); +pub const PLAYLIST_WATCH_UNMATCHED: EventId = EventId::new("playlist.watch.unmatched"); + +// ---- recording ------------------------------------------------------ +pub const RECORDING_STARTED: EventId = EventId::new("recording.started"); +pub const RECORDING_COMPLETED: EventId = EventId::new("recording.completed"); +pub const RECORDING_FAILED: EventId = EventId::new("recording.failed"); + +// ---- provider ------------------------------------------------------- +pub const PROVIDER_ACCOUNT_STATUS: EventId = EventId::new("provider.account.status_changed"); +pub const PROVIDER_ACCOUNT_EXPIRING: EventId = EventId::new("provider.account.expiring"); +pub const PROVIDER_ACCOUNT_EXPIRED: EventId = EventId::new("provider.account.expired"); +pub const PROVIDER_FETCH_FAILED: EventId = EventId::new("provider.fetch.failed"); +pub const PROVIDER_POOL_EXHAUSTED: EventId = EventId::new("provider.pool.exhausted"); +pub const PROVIDER_PRIORITY_FALLBACK: EventId = EventId::new("provider.priority.fallback"); + +// ---- config --------------------------------------------------------- +pub const CONFIG_CHANGED: EventId = EventId::new("config.changed"); +pub const CONFIG_RELOAD_FAILED: EventId = EventId::new("config.reload_failed"); + +// ---- library -------------------------------------------------------- +pub const LIBRARY_SCAN_COMPLETED: EventId = EventId::new("library.scan.completed"); +pub const LIBRARY_SCAN_FAILED: EventId = EventId::new("library.scan.failed"); + +// ---- metadata ------------------------------------------------------- +pub const METADATA_UPDATE_STARTED: EventId = EventId::new("metadata.update.started"); +pub const METADATA_UPDATE_COMPLETED: EventId = EventId::new("metadata.update.completed"); +pub const METADATA_UPDATE_FAILED: EventId = EventId::new("metadata.update.failed"); + +// ---- users and connections ------------------------------------------ +/// High frequency. Subscribe deliberately. +pub const USER_CONNECTION_CHANGED: EventId = EventId::new("user.connection.changed"); +/// High frequency. Subscribe deliberately. +pub const PROVIDER_CONNECTIONS_CHANGED: EventId = EventId::new("provider.connections.changed"); +/// A user was refused a connection because their limits were full. Not +/// high frequency: reported per refusal, and a refusal is not routine. +pub const USER_CONNECTION_DENIED: EventId = EventId::new("user.connection.denied"); + +// ---- user accounts --------------------------------------------------- +pub const USER_CREATED: EventId = EventId::new("user.created"); +pub const USER_UPDATED: EventId = EventId::new("user.updated"); +pub const USER_DELETED: EventId = EventId::new("user.deleted"); + +// ---- authentication -------------------------------------------------- +pub const AUTH_SIGN_IN_SUCCEEDED: EventId = EventId::new("auth.sign_in.succeeded"); +pub const AUTH_SIGN_IN_FAILED: EventId = EventId::new("auth.sign_in.failed"); +pub const AUTH_SIGN_IN_THROTTLED: EventId = EventId::new("auth.sign_in.throttled"); +pub const AUTH_PERMISSION_DENIED: EventId = EventId::new("auth.permission.denied"); + +// ---- streams --------------------------------------------------------- +pub const STREAM_PROBE_FAILED: EventId = EventId::new("stream.probe.failed"); + +// ---- dvr ------------------------------------------------------------ +pub const RECORDING_QUEUE_CHANGED: EventId = EventId::new("recording.queue.changed"); +pub const RECORDING_RULES_CHANGED: EventId = EventId::new("recording.rules.changed"); + +// ---- scheduler ------------------------------------------------------- +pub const SCHEDULED_TASK_FAILED: EventId = EventId::new("scheduled_task.failed"); + +// ---- notification self-reporting ------------------------------------ +/// A notification was permanently lost. Must never route back through +/// the channel that dropped it. +pub const NOTIFICATION_DEAD_LETTERED: EventId = EventId::new("notification.dead_lettered"); + +/// Every registered event, in display order. +pub const ALL: &[EventDescriptor] = &[ + EventDescriptor { id: SYSTEM_INFO, severity: Severity::Info, description: "A general informational message." }, + EventDescriptor { id: SYSTEM_ERROR, severity: Severity::Error, description: "A general error message." }, + EventDescriptor { + id: SYSTEM_DISK_ALERT, + severity: Severity::Warn, + description: "Disk usage crossed the warn or critical threshold.", + }, + EventDescriptor { + id: SYSTEM_STARTED, + severity: Severity::Info, + description: "The server finished starting up.", + }, + EventDescriptor { + id: SYSTEM_SHUTDOWN, + severity: Severity::Info, + description: "The server is shutting down cleanly.", + }, + EventDescriptor { + id: PLAYLIST_UPDATE_COMPLETED, + severity: Severity::Info, + description: "A playlist update finished; carries per-source statistics.", + }, + EventDescriptor { + id: PLAYLIST_UPDATE_FAILED, + severity: Severity::Error, + description: "A playlist update failed.", + }, + EventDescriptor { + id: PLAYLIST_WATCH_CHANGED, + severity: Severity::Info, + description: "Channels were added to or removed from a watched group.", + }, + EventDescriptor { + id: PLAYLIST_GROUPS_CHANGED, + severity: Severity::Info, + description: "Groups were added to or removed from a target.", + }, + EventDescriptor { + id: PLAYLIST_WATCH_DISABLED, + severity: Severity::Warn, + description: "A target's watch configuration is set but not working.", + }, + EventDescriptor { + id: PLAYLIST_WATCH_UNMATCHED, + severity: Severity::Warn, + description: "Watch patterns matched no group in the refreshed playlist.", + }, + EventDescriptor { id: RECORDING_STARTED, severity: Severity::Info, description: "A recording started." }, + EventDescriptor { id: RECORDING_COMPLETED, severity: Severity::Info, description: "A recording completed." }, + EventDescriptor { id: RECORDING_FAILED, severity: Severity::Error, description: "A recording failed." }, + EventDescriptor { + id: PROVIDER_ACCOUNT_STATUS, + severity: Severity::Warn, + description: "A provider reported a changed account status.", + }, + EventDescriptor { + id: PROVIDER_ACCOUNT_EXPIRING, + severity: Severity::Warn, + description: "A provider account is approaching its expiry date.", + }, + EventDescriptor { + id: PROVIDER_ACCOUNT_EXPIRED, + severity: Severity::Error, + description: "A provider account has expired.", + }, + EventDescriptor { + id: PROVIDER_FETCH_FAILED, + severity: Severity::Error, + description: "An input's playlist could not be fetched.", + }, + EventDescriptor { + id: PROVIDER_POOL_EXHAUSTED, + severity: Severity::Warn, + description: "Every provider behind an input was at capacity.", + }, + EventDescriptor { + id: PROVIDER_PRIORITY_FALLBACK, + severity: Severity::Warn, + description: "An input started being served from a different provider priority group.", + }, + EventDescriptor { + id: CONFIG_CHANGED, + severity: Severity::Info, + description: "A configuration file was changed and reloaded.", + }, + EventDescriptor { + id: CONFIG_RELOAD_FAILED, + severity: Severity::Error, + description: "A configuration file changed but could not be reloaded.", + }, + EventDescriptor { + id: LIBRARY_SCAN_COMPLETED, + severity: Severity::Info, + description: "A local library scan finished.", + }, + EventDescriptor { + id: LIBRARY_SCAN_FAILED, + severity: Severity::Error, + description: "A local library scan could not complete.", + }, + EventDescriptor { + id: METADATA_UPDATE_STARTED, + severity: Severity::Info, + description: "A metadata update started for an input.", + }, + EventDescriptor { + id: METADATA_UPDATE_COMPLETED, + severity: Severity::Info, + description: "A metadata update finished for an input.", + }, + EventDescriptor { + id: METADATA_UPDATE_FAILED, + severity: Severity::Error, + description: "A metadata update cycle ended with tasks it could not finish.", + }, + EventDescriptor { + id: USER_CONNECTION_CHANGED, + severity: Severity::Info, + description: "A user connected or disconnected. High frequency - subscribe deliberately.", + }, + EventDescriptor { + id: PROVIDER_CONNECTIONS_CHANGED, + severity: Severity::Info, + description: "A provider's active connection count changed. High frequency - subscribe deliberately.", + }, + EventDescriptor { + id: USER_CONNECTION_DENIED, + severity: Severity::Warn, + description: "A user was refused a connection because their limits were full.", + }, + EventDescriptor { + id: USER_CREATED, + severity: Severity::Info, + description: "An API-proxy user account was created.", + }, + EventDescriptor { + id: USER_UPDATED, + severity: Severity::Info, + description: "An API-proxy user account was changed.", + }, + EventDescriptor { + id: USER_DELETED, + severity: Severity::Warn, + description: "An API-proxy user account was deleted.", + }, + EventDescriptor { + id: AUTH_SIGN_IN_SUCCEEDED, + severity: Severity::Info, + description: "A principal signed in and was issued a token.", + }, + EventDescriptor { + id: AUTH_SIGN_IN_FAILED, + severity: Severity::Warn, + description: "A sign-in was rejected. Deduplicated per principal and address, so a password-guessing run notifies once rather than per attempt.", + }, + EventDescriptor { + id: AUTH_SIGN_IN_THROTTLED, + severity: Severity::Warn, + description: "A sign-in was refused without checking credentials because the caller is backing off after repeated failures.", + }, + EventDescriptor { + id: AUTH_PERMISSION_DENIED, + severity: Severity::Warn, + description: "An authenticated principal asked for something its permissions do not cover.", + }, + EventDescriptor { + id: STREAM_PROBE_FAILED, + severity: Severity::Warn, + description: "A stream probe returned no metadata; the stream may be dead. Deduplicated per input, so a provider outage notifies once rather than per channel.", + }, + EventDescriptor { + id: RECORDING_QUEUE_CHANGED, + severity: Severity::Info, + description: "The recording queue changed.", + }, + EventDescriptor { + id: RECORDING_RULES_CHANGED, + severity: Severity::Info, + description: "The recording rule set changed.", + }, + EventDescriptor { + id: SCHEDULED_TASK_FAILED, + severity: Severity::Error, + description: "A scheduled task could not complete.", + }, + EventDescriptor { + id: NOTIFICATION_DEAD_LETTERED, + severity: Severity::Error, + description: "A notification was permanently undeliverable and has been dropped.", + }, + ]; + +/// Look up the descriptor for an id. +#[must_use] +pub fn describe(id: EventId) -> Option<&'static EventDescriptor> { ALL.iter().find(|d| d.id == id) } + +/// Default severity for an id; `Info` for anything unregistered. +#[must_use] +pub fn default_severity(id: EventId) -> Severity { describe(id).map_or(Severity::Info, |d| d.severity) } diff --git a/shared/src/model/notification/subscription.rs b/shared/src/model/notification/subscription.rs new file mode 100644 index 000000000..35da7bba4 --- /dev/null +++ b/shared/src/model/notification/subscription.rs @@ -0,0 +1,119 @@ +use super::{registry, EventId, EventPattern, EventSubscription}; +use std::fmt; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum Segment { + /// A literal segment. + Literal(String), + /// `*` in an interior position - matches exactly one segment. + One, + /// A trailing `.*` - matches one or more remaining segments. + Rest, +} + +impl fmt::Display for EventPattern { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&self.raw) } +} + +impl EventPattern { + /// Parse a pattern. Never fails: an empty pattern simply matches nothing, + /// which is the safe reading of a blank config line. + #[must_use] + pub fn parse(raw: &str) -> Self { + let trimmed = raw.trim(); + let (negated, body) = match trimmed.strip_prefix('!') { + Some(rest) => (true, rest.trim()), + None => (false, trimmed), + }; + let parts: Vec<&str> = if body.is_empty() { Vec::new() } else { body.split('.').collect() }; + let last = parts.len().saturating_sub(1); + let segments = parts + .iter() + .enumerate() + .map(|(i, part)| match *part { + "*" if i == last => Segment::Rest, + "*" => Segment::One, + literal => Segment::Literal(literal.to_string()), + }) + .collect(); + Self { segments, negated, raw: trimmed.to_string() } + } + + /// The pattern as written, for round-tripping back into config. + #[must_use] + pub fn as_str(&self) -> &str { &self.raw } + + /// Does this pattern cover `id`? Ignores negation - the caller combines. + #[must_use] + pub fn matches(&self, id: EventId) -> bool { Self::match_segments(&self.segments, id.as_str()) } + + pub(super) fn match_segments(pattern: &[Segment], id: &str) -> bool { + let mut parts = id.split('.'); + for (i, segment) in pattern.iter().enumerate() { + match segment { + // A trailing `*` swallows every remaining segment, and + // requires at least one so `recording.*` does not match a + // bare `recording`. + Segment::Rest => { + // A lone `*` pattern matches everything, including a + // single-segment id. + return if i == 0 { true } else { parts.next().is_some() }; + } + Segment::One => { + if parts.next().is_none() { + return false; + } + } + Segment::Literal(want) => match parts.next() { + Some(got) if got.eq_ignore_ascii_case(want) => {} + _ => return false, + }, + } + } + // Every pattern segment consumed; the id must be exhausted too. + parts.next().is_none() + } +} + +impl EventSubscription { + #[must_use] + pub fn parse, S: AsRef>(raw: I) -> Self { + Self { patterns: raw.into_iter().map(|s| EventPattern::parse(s.as_ref())).collect() } + } + + /// The parsed patterns, for round-tripping back into config. + #[must_use] + pub fn patterns(&self) -> &[EventPattern] { &self.patterns } + + /// `true` when nothing is subscribed - the caller can skip all work. + #[must_use] + pub fn is_empty(&self) -> bool { self.patterns.is_empty() } + + /// At least one positive pattern matches and no negative pattern does. + #[must_use] + pub fn matches(&self, id: EventId) -> bool { + let mut included = false; + for pattern in &self.patterns { + if pattern.matches(id) { + if pattern.negated { + return false; + } + included = true; + } + } + included + } + + /// Patterns that match no registered event. + /// + /// Config validation surfaces these as a warning: a typo in `notify_on` + /// is otherwise indistinguishable from an event that simply never fires. + #[must_use] + pub fn unmatched_patterns(&self) -> Vec<&str> { + self.patterns + .iter() + .filter(|p| !registry::ALL.iter().any(|d| p.matches(d.id))) + .map(EventPattern::as_str) + .collect() + } +} diff --git a/shared/src/model/notification/tests.rs b/shared/src/model/notification/tests.rs new file mode 100644 index 000000000..394bf4b22 --- /dev/null +++ b/shared/src/model/notification/tests.rs @@ -0,0 +1,208 @@ +use super::{registry, EventId, EventPattern, EventSubscription, Severity, LEGACY_ALIASES}; + +#[test] +fn every_legacy_msgkind_wire_name_still_resolves() { + // Load-bearing for existing config.yml files: each of these was a + // valid `notify_on` entry before the open-world refactor. + for legacy in [ + "info", + "stats", + "error", + "watch", + "disk_alert", + "recording_started", + "recording_completed", + "recording_failed", + ] { + assert!(EventId::from_wire(legacy).is_some(), "legacy wire name {legacy} no longer resolves"); + } +} + +#[test] +fn legacy_alias_targets_are_all_registered() { + for (legacy, id) in LEGACY_ALIASES { + assert!(registry::describe(*id).is_some(), "alias {legacy} points at unregistered id {id}"); + } +} + +#[test] +fn canonical_ids_resolve_and_round_trip() { + for descriptor in registry::ALL { + let resolved = EventId::from_wire(descriptor.id.as_str()); + assert_eq!(resolved, Some(descriptor.id), "id {} did not round-trip", descriptor.id); + } +} + +#[test] +fn unknown_wire_name_does_not_resolve() { + assert_eq!(EventId::from_wire("recording.definitely_not_a_real_event"), None); +} + +#[test] +fn recording_template_filenames_match_the_legacy_names() { + // The legacy on-disk names were `telegram_recording_completed.templ` + // and friends. The canonical dotted ids produce the same stems, so + // existing template files keep being discovered. + assert_eq!(registry::RECORDING_COMPLETED.template_filename("telegram"), "telegram_recording_completed.templ"); + assert_eq!(registry::RECORDING_STARTED.template_filename("discord"), "discord_recording_started.templ"); + assert_eq!(registry::RECORDING_FAILED.template_filename("rest"), "rest_recording_failed.templ"); +} + +#[test] +fn domain_is_the_part_before_the_first_dot() { + assert_eq!(registry::RECORDING_COMPLETED.domain(), "recording"); + assert_eq!(registry::SYSTEM_DISK_ALERT.domain(), "system"); + assert_eq!(registry::UNKNOWN.domain(), "unknown"); +} + +#[test] +fn star_matches_every_registered_event() { + let sub = EventSubscription::parse(["*"]); + for descriptor in registry::ALL { + assert!(sub.matches(descriptor.id), "`*` failed to match {}", descriptor.id); + } +} + +#[test] +fn prefix_pattern_matches_the_domain_and_nothing_else() { + let sub = EventSubscription::parse(["recording.*"]); + assert!(sub.matches(registry::RECORDING_COMPLETED)); + assert!(sub.matches(registry::RECORDING_FAILED)); + assert!(!sub.matches(registry::SYSTEM_INFO)); + assert!(!sub.matches(registry::PLAYLIST_UPDATE_COMPLETED)); +} + +#[test] +fn prefix_pattern_requires_at_least_one_further_segment() { + // `recording.*` must not match a hypothetical bare `recording`. + assert!(!EventPattern::parse("recording.*").matches(EventId::new("recording"))); +} + +#[test] +fn exact_pattern_matches_only_itself() { + let sub = EventSubscription::parse(["recording.completed"]); + assert!(sub.matches(registry::RECORDING_COMPLETED)); + assert!(!sub.matches(registry::RECORDING_STARTED)); +} + +#[test] +fn interior_wildcard_matches_exactly_one_segment() { + let pattern = EventPattern::parse("provider.*.expired"); + assert!(pattern.matches(registry::PROVIDER_ACCOUNT_EXPIRED)); + // One segment, not several. + assert!(!pattern.matches(EventId::new("provider.a.b.expired"))); + // And not zero. + assert!(!pattern.matches(EventId::new("provider.expired"))); +} + +#[test] +fn negation_removes_from_a_wider_match() { + let sub = EventSubscription::parse(["*", "!system.info"]); + assert!(sub.matches(registry::SYSTEM_ERROR)); + assert!(!sub.matches(registry::SYSTEM_INFO), "negated pattern did not exclude"); +} + +#[test] +fn negation_wins_regardless_of_order() { + let before = EventSubscription::parse(["!recording.started", "recording.*"]); + let after = EventSubscription::parse(["recording.*", "!recording.started"]); + assert!(!before.matches(registry::RECORDING_STARTED)); + assert!(!after.matches(registry::RECORDING_STARTED)); + assert!(before.matches(registry::RECORDING_COMPLETED)); + assert!(after.matches(registry::RECORDING_COMPLETED)); +} + +#[test] +fn a_subscription_of_only_negations_matches_nothing() { + // No positive pattern means nothing was opted in to. + let sub = EventSubscription::parse(["!system.info"]); + assert!(!sub.matches(registry::SYSTEM_ERROR)); + assert!(!sub.matches(registry::SYSTEM_INFO)); +} + +#[test] +fn empty_subscription_matches_nothing() { + let sub = EventSubscription::parse(Vec::::new()); + assert!(sub.is_empty()); + assert!(!sub.matches(registry::SYSTEM_INFO)); +} + +#[test] +fn blank_pattern_matches_nothing_rather_than_everything() { + // A stray empty line in config must not silently subscribe to all. + let sub = EventSubscription::parse([" "]); + assert!(!sub.matches(registry::SYSTEM_INFO)); +} + +#[test] +fn unmatched_patterns_flags_typos_only() { + let sub = EventSubscription::parse(["recording.*", "recroding.completed", "*"]); + assert_eq!(sub.unmatched_patterns(), vec!["recroding.completed"]); +} + +#[test] +fn severity_orders_from_info_up_to_critical() { + assert!(Severity::Info < Severity::Warn); + assert!(Severity::Warn < Severity::Error); + assert!(Severity::Error < Severity::Critical); +} + +#[test] +fn severity_wire_names_round_trip() { + for severity in [Severity::Info, Severity::Warn, Severity::Error, Severity::Critical] { + assert_eq!(Severity::from_wire(severity.wire_name()), Some(severity)); + } +} + +#[test] +fn unknown_event_id_deserializes_to_unknown_instead_of_failing() { + // A newer build's outbox entry must not poison the whole file when + // read back by an older one. + let id: EventId = serde_json::from_str("\"some.future.event\"").expect("must not fail"); + assert_eq!(id, registry::UNKNOWN); +} + +#[test] +fn event_id_serializes_as_its_wire_string() { + let json = serde_json::to_string(®istry::RECORDING_COMPLETED).expect("serialize"); + assert_eq!(json, "\"recording.completed\""); +} + +/// The docs carry a table of every event id. A registry entry that never +/// reaches the table is undiscoverable, and a table row for an id that no +/// longer exists is a lie - so the two are checked against each other +/// rather than maintained by hand. +#[test] +fn every_registered_event_appears_in_the_docs_table() { + const DOCS: &str = include_str!("../../../../docs/src/configuration/config.md"); + let table = DOCS + .split_once("") + .and_then(|(_, rest)| rest.split_once("")) + .map(|(table, _)| table) + .expect("docs must contain the generated event table markers"); + + for descriptor in registry::ALL { + let row = format!("| `{}` | {} | {} |", descriptor.id, descriptor.severity, descriptor.description); + assert!( + table.contains(&row), + "docs/src/configuration/config.md is missing this row - add it inside the generated table:\n{row}" + ); + } + + // And nothing in the table that the registry no longer knows about. + for line in table.lines().filter(|line| line.trim_start().starts_with("| `")) { + let id = line.trim_start().trim_start_matches("| `").split('`').next().unwrap_or_default(); + assert!( + EventId::from_wire(id).is_some(), + "docs list `{id}`, which is not a registered event - remove the row or register the event" + ); + } +} + +#[test] +fn registry_has_no_duplicate_ids() { + let mut seen = std::collections::HashSet::new(); + for descriptor in registry::ALL { + assert!(seen.insert(descriptor.id), "duplicate registry entry for {}", descriptor.id); + } +} diff --git a/shared/src/model/playlist.rs b/shared/src/model/playlist.rs index 972389b37..a75659906 100644 --- a/shared/src/model/playlist.rs +++ b/shared/src/model/playlist.rs @@ -1,493 +1,5 @@ -use crate::{ - create_bitset, - error::TuliproxError, - model::{ - stalker::StalkerStreamKind, stalker_item::StalkerPlaylistItem, xtream_const, CatchupAttribute, - CatchupProperties, ClusterFlags, CommonPlaylistItem, ConfigTargetOptions, EpisodeStreamProperties, HeaderField, - SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamProperties, VirtualId, XtreamInfoDocument, - }, - utils::{ - arc_str_option_serde, arc_str_serde, concat_path, extract_extension_from_url, generate_provider_playlist_uuid, - generate_runtime_playlist_uuid, get_provider_id, seal_web_ui_resource_url, Internable, - }, -}; -use dashmap::DashMap; -use serde::{Deserialize, Serialize}; -use std::{ - fmt::{Display, Formatter, Write}, - str::FromStr, - sync::Arc, -}; -use strum_macros::{AsRefStr, Display, EnumIter, EnumString}; -// https://de.wikipedia.org/wiki/M3U -// https://siptv.eu/howto/playlist.html - -#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq, Serialize, Deserialize, Default, Display, EnumString, AsRefStr)] -#[repr(u8)] -pub enum XtreamCluster { - #[default] - #[strum(serialize = "Live", serialize = "live")] - Live = 1, - - #[strum(serialize = "Video", serialize = "movie", serialize = "vod", serialize = "video")] - Video = 2, - - #[strum(serialize = "Series", serialize = "series")] - Series = 3, -} - -impl XtreamCluster { - pub fn as_str(&self) -> &str { self.as_ref() } - - /// True when this cluster is the Xtream `Series` cluster. Used in - /// bucket-key computations and dispatch sites that previously - /// spelled out `== XtreamCluster::Series` inline. - pub fn is_series(self) -> bool { matches!(self, Self::Series) } - - pub fn as_stream_type(&self) -> &str { - match self { - Self::Live => "live", - Self::Video => "movie", - Self::Series => "series", - } - } - - /// Returns the xtream `player_api` info action and the stream-id query field for this cluster. - /// - /// Keeps the per-cluster `(action, id_field)` mapping attached to the enum so call sites read a - /// single property instead of re-deriving it with a local `match`. - pub fn info_action_and_id_field(&self) -> (&'static str, &'static str) { - match self { - Self::Live => (xtream_const::XC_ACTION_GET_LIVE_INFO, xtream_const::XC_LIVE_ID), - Self::Video => (xtream_const::XC_ACTION_GET_VOD_INFO, xtream_const::XC_VOD_ID), - Self::Series => (xtream_const::XC_ACTION_GET_SERIES_INFO, xtream_const::XC_SERIES_ID), - } - } -} - -/// Every item type belongs to exactly one cluster, so this cannot fail. -/// -/// This used to be a `TryFrom` whose every arm returned `Ok`, and the phantom -/// error spread `.unwrap_or(Live)`, `.unwrap_or_default()` and `.ok()` across 17 -/// call sites in four crates. See [`PlaylistItemType::cluster`]. -impl From for XtreamCluster { - #[inline] - fn from(item_type: PlaylistItemType) -> Self { item_type.cluster() } -} - -#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq, Serialize, Deserialize, Default, EnumIter)] -#[repr(u8)] -pub enum PlaylistItemType { - #[default] - #[serde(alias = "live")] - Live = 1, - #[serde(alias = "video")] - Video = 2, - #[serde(alias = "series")] - Series = 3, // xtream series description - #[serde(alias = "series_info")] - SeriesInfo = 4, // xtream series info fetched for series description - #[serde(alias = "catchup")] - Catchup = 5, - #[serde(alias = "live_unknown")] - LiveUnknown = 6, // No Provider id - #[serde(alias = "live_hls")] - LiveHls = 7, // m3u8 entry - #[serde(alias = "live_dash")] - LiveDash = 8, // mpd - #[serde(alias = "local_video")] - LocalVideo = 9, - #[serde(alias = "local_series")] - LocalSeries = 10, - #[serde(alias = "local_series_info")] - LocalSeriesInfo = 11, -} - -impl From for PlaylistItemType { - fn from(xtream_cluster: XtreamCluster) -> Self { - match xtream_cluster { - XtreamCluster::Live => Self::Live, - XtreamCluster::Video => Self::Video, - XtreamCluster::Series => Self::SeriesInfo, - } - } -} - -impl FromStr for PlaylistItemType { - type Err = TuliproxError; - - fn from_str(s: &str) -> Result { - match s { - "Live" => Ok(PlaylistItemType::Live), - "Video" => Ok(PlaylistItemType::Video), - "LocalVideo" => Ok(PlaylistItemType::LocalVideo), - "Series" => Ok(PlaylistItemType::Series), - "SeriesInfo" => Ok(PlaylistItemType::SeriesInfo), - "LocalSeries" => Ok(PlaylistItemType::LocalSeries), - "LocalSeriesInfo" => Ok(PlaylistItemType::LocalSeriesInfo), - "Catchup" => Ok(PlaylistItemType::Catchup), - "LiveUnknown" => Ok(PlaylistItemType::LiveUnknown), - "LiveHls" => Ok(PlaylistItemType::LiveHls), - "LiveDash" => Ok(PlaylistItemType::LiveDash), - _ => Err(TuliproxError::Config(format!("Invalid PlaylistItemType: {s}"))), - } - } -} - -impl PlaylistItemType { - const LIVE: &'static str = "live"; - const VIDEO: &'static str = "video"; - const SERIES: &'static str = "series"; - const SERIES_INFO: &'static str = "series-info"; - const CATCHUP: &'static str = "catchup"; - - pub fn is_local(&self) -> bool { - matches!(self, PlaylistItemType::LocalVideo | PlaylistItemType::LocalSeries | PlaylistItemType::LocalSeriesInfo) - } - - pub fn is_live(&self) -> bool { - matches!( - self, - PlaylistItemType::Live - | PlaylistItemType::LiveDash - | PlaylistItemType::LiveHls - | PlaylistItemType::LiveUnknown - ) - } - - pub fn is_live_adaptive(&self) -> bool { matches!(self, PlaylistItemType::LiveHls | PlaylistItemType::LiveDash) } - - /// True for VOD item types (`Video` or `LocalVideo`). - pub fn is_video(&self) -> bool { matches!(self, PlaylistItemType::Video | PlaylistItemType::LocalVideo) } - - /// True for concrete series item types (`Series` or `LocalSeries`); excludes the `SeriesInfo` containers. - pub fn is_series(&self) -> bool { matches!(self, PlaylistItemType::Series | PlaylistItemType::LocalSeries) } - - /// Controls address tracking only. - /// Do not use this to decide whether a playback request should use session-based admission - /// or whether a logical playback must stay on the same provider account. - pub fn uses_socket_bound_session(&self) -> bool { - matches!(self, PlaylistItemType::Live | PlaylistItemType::LiveUnknown) - } - - /// Controls whether follow-up requests for the same logical playback must stay on the - /// same provider account. - /// This is separate from both session admission and socket binding. - pub fn requires_provider_affinity(&self) -> bool { - matches!( - self, - PlaylistItemType::LiveHls - | PlaylistItemType::LiveDash - | PlaylistItemType::Video - | PlaylistItemType::LocalVideo - | PlaylistItemType::Series - | PlaylistItemType::SeriesInfo - | PlaylistItemType::LocalSeries - | PlaylistItemType::LocalSeriesInfo - | PlaylistItemType::Catchup - ) - } - - pub fn as_u8(self) -> u8 { self as u8 } - - pub fn as_str(&self) -> &'static str { - match self { - Self::Live | Self::LiveHls | Self::LiveDash | Self::LiveUnknown => Self::LIVE, - Self::Video | Self::LocalVideo => Self::VIDEO, - Self::Series | Self::LocalSeries => Self::SERIES, - Self::SeriesInfo | Self::LocalSeriesInfo => Self::SERIES_INFO, - Self::Catchup => Self::CATCHUP, - } - } - - /// Returns a cached interned `Arc` of this type's label. - /// - /// `intern()` performs an interner hash-map lookup on every call. Because the - /// label is one of only five fixed values, this caches the interned `Arc` per - /// label in a `OnceLock` and returns a cheap `Arc::clone`, avoiding repeated - /// interner lookups on hot sort/filter paths. - pub fn interned_label(&self) -> Arc { - static CACHE: [std::sync::OnceLock>; 5] = [const { std::sync::OnceLock::new() }; 5]; - let idx = match self { - Self::Live | Self::LiveHls | Self::LiveDash | Self::LiveUnknown => 0, - Self::Video | Self::LocalVideo => 1, - Self::Series | Self::LocalSeries => 2, - Self::SeriesInfo | Self::LocalSeriesInfo => 3, - Self::Catchup => 4, - }; - Arc::clone(CACHE[idx].get_or_init(|| self.as_str().intern())) - } - - /// The cluster this item type belongs to. - /// - /// The one place the item-type-to-cluster relation is written down. It used - /// to be encoded twice -- here and in a `TryFrom` impl -- with nothing - /// keeping the two in agreement. - #[inline] - pub const fn cluster(self) -> XtreamCluster { - match self { - Self::Live | Self::LiveHls | Self::LiveDash | Self::LiveUnknown => XtreamCluster::Live, - Self::Catchup | Self::Video | Self::LocalVideo => XtreamCluster::Video, - Self::Series | Self::LocalSeries | Self::SeriesInfo | Self::LocalSeriesInfo => XtreamCluster::Series, - } - } - - #[inline] - pub const fn is_cluster(&self, cluster: XtreamCluster) -> bool { self.cluster() as u8 == cluster as u8 } -} - -impl Display for PlaylistItemType { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.as_str()) } -} - -impl Internable for PlaylistItemType { - fn intern(self) -> Arc { self.as_str().intern() } -} - -impl Internable for XtreamCluster { - fn intern(self) -> Arc { self.as_str().intern() } -} - #[derive(Copy, Clone, Default, Debug)] pub struct PlaylistItemTypeSet(u16); -impl PlaylistItemTypeSet { - #[inline] - pub fn empty() -> Self { Self(0) } - - #[inline] - pub fn from_item(item: PlaylistItemType) -> Self { - let bit = 1u16 << ((item as u8) - 1); - Self(bit) - } - - #[inline] - pub fn insert(&mut self, item: PlaylistItemType) { self.0 |= 1u16 << ((item as u8) - 1); } - - #[inline] - pub fn remove(&mut self, item: PlaylistItemType) { self.0 &= !(1u16 << ((item as u8) - 1)); } - - #[inline] - pub fn is_set(&self, item: PlaylistItemType) -> bool { (self.0 & (1u16 << ((item as u8) - 1))) != 0 } - - #[inline] - pub fn bits(self) -> u16 { self.0 } -} - -/// A field's value, borrowed where possible. -/// -/// The point of the enum is that reading a field never forces an allocation: -/// the `&str`-keyed accessor had to return `Arc`, so `chno` and `type` -/// went through `.to_string().intern()` — a heap allocation *and* an interner -/// write lock — on every read, per item, per rule. -pub enum FieldRef<'a> { - /// An interned field. Cloning is a refcount bump. - Shared(&'a Arc), - /// A borrowed string that is not interned, e.g. `item_type.as_str()`. - Str(&'a str), - /// A numeric field, kept numeric. - Num(u32), -} - -impl FieldRef<'_> { - /// Borrow as a string, formatting a numeric field into an owned buffer only - /// when there is one. - pub fn as_cow(&self) -> std::borrow::Cow<'_, str> { - match self { - Self::Shared(value) => std::borrow::Cow::Borrowed(value.as_ref()), - Self::Str(value) => std::borrow::Cow::Borrowed(value), - Self::Num(value) => std::borrow::Cow::Owned(value.to_string()), - } - } - - /// Materialise as an interned `Arc`. - /// - /// This is what the `&str` compatibility shims call, so their behaviour — - /// including interning numbers — is bit-for-bit what it was before. - pub fn to_arc(&self) -> Arc { - match self { - Self::Shared(value) => Arc::clone(value), - Self::Str(value) => value.intern(), - Self::Num(value) => value.to_string().intern(), - } - } -} - -/// Read a field by typed key. Matches on a discriminant rather than walking a -/// chain of case-insensitive string comparisons. -pub trait FieldGet { - fn get(&self, field: crate::model::HeaderField) -> Option>; -} - -/// Write a field by typed key. -pub trait FieldSet { - fn set(&mut self, field: crate::model::HeaderField, value: &str) -> bool; -} - -/// Read a field by name. -/// -/// Retained for callers whose field name genuinely arrives as a string (the M3U -/// resource endpoint, for one). Implemented as a shim over [`FieldGet`]. -pub trait FieldGetAccessor { - fn get_field(&self, field: &str) -> Option>; -} - -/// Write a field by name. Shim over [`FieldSet`]. -pub trait FieldSetAccessor { - fn set_field(&mut self, field: &str, value: &str) -> bool; -} - -pub trait PlaylistEntry: Send + Sync { - fn get_virtual_id(&self) -> VirtualId; - /// Returns the immutable stream identifier captured from the input playlist before target mappings. - fn get_input_stream_id(&self) -> Option>; - fn get_provider_id(&self) -> Option; - fn get_category_id(&self) -> Option; - fn get_provider_url(&self) -> Arc; - fn get_uuid(&self) -> UUIDType; - fn get_item_type(&self) -> PlaylistItemType; - fn get_group(&self) -> Arc; - fn get_name(&self) -> Arc; - fn get_resolved_info_document(&self, options: &XtreamMappingOptions) -> Option; - fn get_additional_properties(&self) -> Option<&StreamProperties>; - fn get_additional_properties_mut(&mut self) -> Option<&mut StreamProperties>; - fn get_upstream_user_agent(&self) -> Option<&str>; -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PlaylistItemHeader { - #[serde(skip)] - pub uuid: UUIDType, // calculated - #[serde(with = "arc_str_serde")] - pub id: Arc, // provider id - #[serde(with = "arc_str_serde")] - pub name: Arc, - #[serde(with = "arc_str_serde")] - pub logo: Arc, - #[serde(with = "arc_str_serde")] - pub logo_small: Arc, - #[serde(with = "arc_str_serde")] - pub group: Arc, - #[serde(with = "arc_str_serde")] - pub title: Arc, - #[serde(with = "arc_str_serde")] - pub parent_code: Arc, - #[serde(with = "arc_str_serde")] - pub audio_track: Arc, - #[serde(with = "arc_str_serde")] - pub time_shift: Arc, - #[serde(with = "arc_str_serde")] - pub rec: Arc, - #[serde(with = "arc_str_serde")] - pub url: Arc, - #[serde(default, with = "arc_str_option_serde")] - pub epg_channel_id: Option>, - #[serde(with = "arc_str_serde")] - pub input_name: Arc, - #[serde(default)] - pub additional_properties: Option, - - // 4-byte aligned - pub virtual_id: VirtualId, // virtual id - pub chno: u32, - #[serde(default)] - pub category_id: u32, - #[serde(default)] - pub source_ordinal: u32, - - // 1-byte aligned - pub xtream_cluster: XtreamCluster, - #[serde(default)] - pub item_type: PlaylistItemType, - /// Stable provider/origin ID captured before any target transformation. - #[serde(default, with = "arc_str_serde")] - pub input_stream_id: Arc, - #[serde(default, rename = "source_user_agent", alias = "upstream_user_agent", with = "arc_str_option_serde")] - pub upstream_user_agent: Option>, -} - -impl Default for PlaylistItemHeader { - fn default() -> Self { - Self { - uuid: UUIDType::default(), - id: "".intern(), - virtual_id: VirtualId::default(), - name: "".intern(), - chno: 0, - logo: "".intern(), - logo_small: "".intern(), - group: "".intern(), - title: "".intern(), - parent_code: "".intern(), - audio_track: "".intern(), - time_shift: "".intern(), - rec: "".intern(), - url: "".intern(), - epg_channel_id: None, - xtream_cluster: XtreamCluster::default(), - additional_properties: None, - item_type: PlaylistItemType::default(), - category_id: 0, - input_name: "".intern(), - source_ordinal: 0, - input_stream_id: "".intern(), - upstream_user_agent: None, - } - } -} - -impl PlaylistItemHeader { - /// Captures the input playlist item ID at the input-processing boundary. - /// - /// This must run before target transformations and must not be used to recover an identity - /// from a target-mutated `id`. - pub fn freeze_input_stream_id(&mut self) { - if self.input_stream_id.is_empty() && !self.id.is_empty() { - self.input_stream_id = Arc::clone(&self.id); - } - } - - /// Returns the input stream ID captured at the input-processing boundary. - /// - /// Legacy fallback must be resolved before a target transformation starts. Once an item is - /// represented by this header, the mutable target-facing `id` is never a valid fallback. - pub fn get_input_stream_id(&self) -> Option> { - (!self.input_stream_id.is_empty()).then(|| Arc::clone(&self.input_stream_id)) - } - - #[inline] - pub fn gen_uuid(&mut self) { - self.uuid = generate_runtime_playlist_uuid(&self.input_name, &self.id, self.item_type, &self.url); - } - - #[inline] - pub const fn get_uuid(&self) -> &UUIDType { &self.uuid } - - pub fn get_provider_id(&mut self) -> Option { - match get_provider_id(&self.id, &self.url) { - None => None, - Some(newid) => { - self.id = newid.to_string().intern(); - Some(newid) - } - } - } - - #[inline] - pub fn get_name(&self) -> Arc { - if self.title.is_empty() { - Arc::clone(&self.name) - } else { - Arc::clone(&self.title) - } - } - - #[inline] - pub fn get_container_extension(&self) -> Option> { - self.additional_properties - .as_ref() - .and_then(super::stream_properties::StreamProperties::get_container_extension) - } -} macro_rules! to_m3u_non_empty_fields { ($header:expr, $line:expr, $(($prop:ident, $field:expr)),*;) => { @@ -509,1996 +21,22 @@ macro_rules! to_m3u_resource_non_empty_fields { }; } -impl crate::model::FieldGet for crate::model::PlaylistItemHeader { - fn get(&self, field: HeaderField) -> Option> { - match field { - HeaderField::Id => Some(FieldRef::Shared(&self.id)), - HeaderField::Title => Some(FieldRef::Shared(&self.title)), - HeaderField::Name => Some(FieldRef::Shared(&self.name)), - HeaderField::Logo => Some(FieldRef::Shared(&self.logo)), - HeaderField::LogoSmall => Some(FieldRef::Shared(&self.logo_small)), - HeaderField::ParentCode => Some(FieldRef::Shared(&self.parent_code)), - HeaderField::AudioTrack => Some(FieldRef::Shared(&self.audio_track)), - HeaderField::TimeShift => Some(FieldRef::Shared(&self.time_shift)), - HeaderField::Rec => Some(FieldRef::Shared(&self.rec)), - HeaderField::Url => Some(FieldRef::Shared(&self.url)), - HeaderField::Group => Some(FieldRef::Shared(&self.group)), - HeaderField::Caption => { - Some(FieldRef::Shared(if self.title.is_empty() { &self.name } else { &self.title })) - } - HeaderField::Input => Some(FieldRef::Shared(&self.input_name)), - HeaderField::Type => Some(FieldRef::Str(self.item_type.as_str())), - HeaderField::EpgChannelId => self.epg_channel_id.as_ref().map(FieldRef::Shared), - HeaderField::Chno => Some(FieldRef::Num(self.chno)), - HeaderField::Genre => { - self.additional_properties.as_ref().and_then(StreamProperties::genre).map(FieldRef::Shared) - } - // Not carried by the header. - HeaderField::ProviderId => None, - } - } -} - -impl crate::model::FieldSet for crate::model::PlaylistItemHeader { - fn set(&mut self, field: HeaderField, value: &str) -> bool { - match field { - HeaderField::Id => self.id = value.intern(), - HeaderField::Title => self.title = value.intern(), - HeaderField::Name => self.name = value.intern(), - HeaderField::Logo => self.logo = value.intern(), - HeaderField::LogoSmall => self.logo_small = value.intern(), - HeaderField::ParentCode => self.parent_code = value.intern(), - HeaderField::AudioTrack => self.audio_track = value.intern(), - HeaderField::TimeShift => self.time_shift = value.intern(), - HeaderField::Rec => self.rec = value.intern(), - HeaderField::Url => self.url = value.intern(), - HeaderField::Group => self.group = value.intern(), - HeaderField::Caption => { - let interned = value.intern(); - self.title = Arc::clone(&interned); - self.name = interned; - } - HeaderField::EpgChannelId => self.epg_channel_id = Some(value.intern()), - HeaderField::Chno => match value.parse::() { - Ok(parsed) => self.chno = parsed, - Err(_) => return false, - }, - HeaderField::Genre => return crate::set_genre!(self, value), - // Read-only, or not carried by the header. - HeaderField::Input | HeaderField::Type | HeaderField::ProviderId => return false, - } - true - } -} - -impl crate::model::FieldGetAccessor for crate::model::PlaylistItemHeader { - #[inline] - fn get_field(&self, field: &str) -> Option> { - use crate::model::FieldGet; - self.get(HeaderField::parse(field)?).map(|value| value.to_arc()) - } -} - -impl crate::model::FieldSetAccessor for crate::model::PlaylistItemHeader { - #[inline] - fn set_field(&mut self, field: &str, value: &str) -> bool { - use crate::model::FieldSet; - HeaderField::parse(field).is_some_and(|field| self.set(field, value)) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct M3uPlaylistItem { - pub virtual_id: VirtualId, - #[serde(with = "arc_str_serde")] - pub provider_id: Arc, - #[serde(with = "arc_str_serde")] - pub name: Arc, - pub chno: u32, - #[serde(with = "arc_str_serde")] - pub logo: Arc, - #[serde(with = "arc_str_serde")] - pub logo_small: Arc, - #[serde(with = "arc_str_serde")] - pub group: Arc, - #[serde(with = "arc_str_serde")] - pub title: Arc, - #[serde(with = "arc_str_serde")] - pub parent_code: Arc, - #[serde(with = "arc_str_serde")] - pub audio_track: Arc, - #[serde(with = "arc_str_serde")] - pub time_shift: Arc, - #[serde(with = "arc_str_serde")] - pub rec: Arc, - #[serde(with = "arc_str_serde")] - pub url: Arc, - #[serde(default, with = "arc_str_option_serde")] - pub epg_channel_id: Option>, - #[serde(with = "arc_str_serde")] - pub input_name: Arc, - pub item_type: PlaylistItemType, - #[serde(skip)] - pub t_stream_url: Arc, - #[serde(skip)] - pub t_resource_url: Option, - #[serde(skip)] - pub t_catchup_source: Option>, - #[serde(skip)] - pub t_catchup_mode: Option>, - #[serde(default)] - pub source_ordinal: u32, - #[serde(default)] - pub additional_properties: Option, - /// Stable provider/origin ID captured before any target transformation. - #[serde(default, with = "arc_str_serde")] - pub input_stream_id: Arc, - #[serde(default, rename = "source_user_agent", alias = "upstream_user_agent", with = "arc_str_option_serde")] - pub upstream_user_agent: Option>, -} - -fn write_m3u_attr(line: &mut String, name: &str, value: &str) { let _ = write!(line, " {name}=\"{value}\""); } - -fn append_catchup_attribute(line: &mut String, name: &str, value: Option<&Arc>) { - if let Some(value) = value.filter(|value| !value.is_empty()) { - write_m3u_attr(line, name, value); - } -} - -fn append_extra_catchup_attributes(line: &mut String, attributes: &[CatchupAttribute]) { - for attribute in attributes { - if !attribute.name.is_empty() && !attribute.value.is_empty() { - write_m3u_attr(line, attribute.name.as_ref(), attribute.value.as_ref()); - } - } -} - -fn append_m3u_catchup_attributes( - line: &mut String, - catchup: &CatchupProperties, - rewritten_mode: Option<&Arc>, - rewritten_source: Option<&Arc>, -) { - if let Some(mode) = rewritten_mode.filter(|mode| !mode.is_empty()) { - write_m3u_attr(line, "catchup", mode.as_ref()); - } else { - append_catchup_attribute(line, "catchup", catchup.mode.as_ref()); - } - append_catchup_attribute(line, "catchup-days", catchup.days.as_ref()); - if let Some(source) = rewritten_source.filter(|source| !source.is_empty()) { - write_m3u_attr(line, "catchup-source", source.as_ref()); - } else { - append_catchup_attribute(line, "catchup-source", catchup.source.as_ref()); - } - append_catchup_attribute(line, "catchup-time", catchup.time.as_ref()); - append_catchup_attribute(line, "catchup-correction", catchup.correction.as_ref()); - if !catchup.is_flussonic() || rewritten_mode.is_none_or(|mode| mode.is_empty()) { - append_catchup_attribute(line, "catchup-type", catchup.catchup_type.as_ref()); - } - append_extra_catchup_attributes(line, &catchup.extra_attributes); -} - -fn append_unified_catchup_type_attributes( - line: &mut String, - catchup: &CatchupProperties, - catchup_type: &str, - rewritten_source: Option<&Arc>, - emit_source: bool, -) { - // Unified export: always `catchup-type=...`, never `catchup=`. - write_m3u_attr(line, "catchup-type", catchup_type); - append_catchup_attribute(line, "catchup-days", catchup.days.as_ref()); - if emit_source { - if let Some(source) = rewritten_source.filter(|source| !source.is_empty()) { - write_m3u_attr(line, "catchup-source", source.as_ref()); - } else { - append_catchup_attribute(line, "catchup-source", catchup.source.as_ref()); - } - } - append_catchup_attribute(line, "catchup-time", catchup.time.as_ref()); - append_catchup_attribute(line, "catchup-correction", catchup.correction.as_ref()); - append_extra_catchup_attributes(line, &catchup.extra_attributes); -} - -impl M3uPlaylistItem { - pub fn to_m3u(&self, target_options: Option<&ConfigTargetOptions>, rewrite_urls: bool) -> String { - let options = target_options.as_ref(); - let ignore_logo = options.is_some_and(|o| o.ignore_logo); - let mut line = String::with_capacity(256); - let _ = write!( - &mut line, - "#EXTINF:-1 tvg-id=\"{}\" tvg-name=\"{}\" group-title=\"{}\"", - self.epg_channel_id.as_ref().map_or("", |o| o.as_ref()), - self.name, - self.group - ); - - if !ignore_logo { - if let (true, Some(resource_url)) = (rewrite_urls, self.t_resource_url.as_ref()) { - to_m3u_resource_non_empty_fields!(self, resource_url, line, (logo, "tvg-logo"), (logo_small, "tvg-logo-small");); - } else { - to_m3u_non_empty_fields!(self, line, (logo, "tvg-logo"), (logo_small, "tvg-logo-small");); - } - } - - if self.chno != 0 { - let _ = write!(line, " tvg-chno=\"{}\"", self.chno); - } - let flussonic_mode = self.additional_properties.as_ref().and_then(|props| match props { - StreamProperties::Live(live) => { - live.catchup.as_ref().and_then(CatchupProperties::native_flussonic_player_mode) - } - _ => None, - }); - let append_type = self.additional_properties.as_ref().and_then(|props| match props { - StreamProperties::Live(live) => live.catchup.as_ref().and_then(CatchupProperties::append_player_type), - _ => None, - }); - // Emit timeshift only when the source had it (non-empty). Never invent catchup attrs. - to_m3u_non_empty_fields!(self, line, - (parent_code, "parent-code"), - (audio_track, "audio-track"), - (time_shift, "timeshift"), - (rec, "tvg-rec");); - if let Some(StreamProperties::Live(live)) = self.additional_properties.as_ref() { - if let Some(catchup) = live.catchup.as_ref() { - let has_rewritten_catchup = self.t_catchup_mode.as_ref().is_some_and(|mode| !mode.is_empty()) - || self.t_catchup_source.as_ref().is_some_and(|source| !source.is_empty()); - if has_rewritten_catchup { - append_m3u_catchup_attributes( - &mut line, - catchup, - self.t_catchup_mode.as_ref(), - self.t_catchup_source.as_ref(), - ); - } else if let Some(mode) = flussonic_mode { - // Unify catchup=/catchup-type=flussonic* to catchup-type only. - // No catchup=, no shift-style catchup-source, no invented days. - append_unified_catchup_type_attributes(&mut line, catchup, mode, None, false); - } else if let Some(append_type) = append_type { - // Unify catchup=/catchup-type=append to catchup-type="append" only. - append_unified_catchup_type_attributes( - &mut line, - catchup, - append_type, - self.t_catchup_source.as_ref(), - true, - ); - } else { - append_m3u_catchup_attributes( - &mut line, - catchup, - self.t_catchup_mode.as_ref(), - self.t_catchup_source.as_ref(), - ); - } - } - } - - let _ = writeln!(&mut line, ",{}", self.title); - if let Some(user_agent) = - self.upstream_user_agent.as_deref().filter(|value| !value.is_empty() && !value.contains(['\r', '\n'])) - { - let _ = writeln!(&mut line, "#EXTVLCOPT:http-user-agent={user_agent}"); - } - let url = if self.t_stream_url.is_empty() { &self.url } else { &self.t_stream_url }; - line.push_str(url); - line - } - - pub fn to_common(&self) -> CommonPlaylistItem { - CommonPlaylistItem { - virtual_id: self.virtual_id, - provider_id: Arc::clone(&self.provider_id), - name: Arc::clone(&self.name), - chno: self.chno, - logo: Arc::clone(&self.logo), - logo_small: Arc::clone(&self.logo_small), - group: Arc::clone(&self.group), - title: Arc::clone(&self.title), - parent_code: Arc::clone(&self.parent_code), - audio_track: Arc::clone(&self.audio_track), - time_shift: Arc::clone(&self.time_shift), - rec: Arc::clone(&self.rec), - url: Arc::clone(&self.url), - input_name: Arc::clone(&self.input_name), - item_type: self.item_type, - epg_channel_id: self.epg_channel_id.clone(), - xtream_cluster: Some(self.item_type.cluster()), - additional_properties: self.additional_properties.clone(), - category_id: None, - } - } -} - -impl PlaylistEntry for M3uPlaylistItem { - #[inline] - fn get_virtual_id(&self) -> VirtualId { self.virtual_id } - - fn get_input_stream_id(&self) -> Option> { - let input_stream_id = if self.input_stream_id.is_empty() { &self.provider_id } else { &self.input_stream_id }; - (!input_stream_id.is_empty()).then(|| Arc::clone(input_stream_id)) - } - - fn get_upstream_user_agent(&self) -> Option<&str> { self.upstream_user_agent.as_deref() } - - fn get_provider_id(&self) -> Option { get_provider_id(&self.provider_id, &self.url) } - #[inline] - fn get_category_id(&self) -> Option { None } - #[inline] - fn get_provider_url(&self) -> Arc { Arc::clone(&self.url) } - - fn get_uuid(&self) -> UUIDType { - generate_runtime_playlist_uuid(&self.input_name, &self.provider_id, self.item_type, &self.url) - } - - #[inline] - fn get_item_type(&self) -> PlaylistItemType { self.item_type } - - #[inline] - fn get_group(&self) -> Arc { Arc::clone(&self.group) } - - #[inline] - fn get_name(&self) -> Arc { - if self.title.is_empty() { - Arc::clone(&self.name) - } else { - Arc::clone(&self.title) - } - } - - #[inline] - fn get_resolved_info_document(&self, _options: &XtreamMappingOptions) -> Option { None } - #[inline] - fn get_additional_properties(&self) -> Option<&StreamProperties> { self.additional_properties.as_ref() } - #[inline] - fn get_additional_properties_mut(&mut self) -> Option<&mut StreamProperties> { self.additional_properties.as_mut() } -} - -impl crate::model::FieldGet for M3uPlaylistItem { - fn get(&self, field: HeaderField) -> Option> { - match field { - HeaderField::ProviderId => Some(FieldRef::Shared(&self.provider_id)), - HeaderField::Title => Some(FieldRef::Shared(&self.title)), - HeaderField::Name => Some(FieldRef::Shared(&self.name)), - HeaderField::Logo => Some(FieldRef::Shared(&self.logo)), - HeaderField::LogoSmall => Some(FieldRef::Shared(&self.logo_small)), - HeaderField::ParentCode => Some(FieldRef::Shared(&self.parent_code)), - HeaderField::AudioTrack => Some(FieldRef::Shared(&self.audio_track)), - HeaderField::TimeShift => Some(FieldRef::Shared(&self.time_shift)), - HeaderField::Rec => Some(FieldRef::Shared(&self.rec)), - HeaderField::Url => Some(FieldRef::Shared(&self.url)), - HeaderField::Group => Some(FieldRef::Shared(&self.group)), - HeaderField::Caption => { - Some(FieldRef::Shared(if self.title.is_empty() { &self.name } else { &self.title })) - } - HeaderField::EpgChannelId => self.epg_channel_id.as_ref().map(FieldRef::Shared), - HeaderField::Chno => Some(FieldRef::Num(self.chno)), - // Deliberately not addressable by name on an M3U item, even though the - // struct carries input_name, item_type and additional_properties. The - // M3U resource endpoint resolves a URL path segment through - // `get_field`, so making these resolvable would turn a 404 into a - // response. Preserved from the string-keyed accessor this replaces. - HeaderField::Id | HeaderField::Input | HeaderField::Type | HeaderField::Genre => None, - } - } -} - -impl crate::model::FieldGetAccessor for M3uPlaylistItem { - #[inline] - fn get_field(&self, field: &str) -> Option> { - use crate::model::FieldGet; - let field = HeaderField::parse(field)?; - let value = self.get(field)?.to_arc(); - Some(if matches!(field, HeaderField::Logo | HeaderField::LogoSmall) { - super::persisted_resource_arc(&value) - } else { - value - }) - } -} - -impl From for CommonPlaylistItem { - fn from(item: M3uPlaylistItem) -> Self { item.to_common() } -} - -create_bitset!( - u8, - XtreamMappingFlags, - SkipLiveDirectSource, - SkipVideoDirectSource, - SkipSeriesDirectSource, - RewriteResourceUrl -); - -pub struct XtreamMappingOptions { - pub flags: XtreamMappingFlagsSet, - pub force_redirect: Option, - pub reverse_item_types: PlaylistItemTypeSet, - pub resource_proxy_item_types: PlaylistItemTypeSet, - pub username: String, - pub password: String, - pub base_url: String, - pub web_ui_request: bool, - pub encrypt_secret: [u8; 16], - pub resource_host_policies: Arc>, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ResourceOutputPolicy { - Direct, - Proxy, - Blocked, -} - -impl XtreamMappingOptions { - #[inline] - pub fn is_reverse(&self, item_type: PlaylistItemType) -> bool { self.reverse_item_types.is_set(item_type) } - - fn is_trusted_web_ui_resource_path(resource_url: &str) -> bool { - const TRUSTED_WEB_UI_RESOURCE_PREFIXES: [&str; 1] = ["/api/v1/library/thumbnail/"]; - TRUSTED_WEB_UI_RESOURCE_PREFIXES.iter().any(|prefix| resource_url.starts_with(prefix)) - } - - fn build_reverse_proxy_base_url( - &self, - xtream_cluster: XtreamCluster, - item_type: PlaylistItemType, - virtual_id: VirtualId, - force_proxy: bool, - ) -> Option { - let proxy_resource = self.resource_proxy_item_types.is_set(item_type); - if force_proxy || (proxy_resource && self.flags.contains(XtreamMappingFlags::RewriteResourceUrl)) { - Some(format!( - "{}/resource/{}/{}/{}/{}", - self.base_url, - xtream_cluster.as_stream_type(), - self.username, - self.password, - virtual_id - )) - } else { - None - } - } - - fn resource_output_policy(&self, resource_url: &str) -> ResourceOutputPolicy { - if self.flags.contains(XtreamMappingFlags::RewriteResourceUrl) { - return ResourceOutputPolicy::Proxy; - } - if resource_url.starts_with("media-server://image/") { - return ResourceOutputPolicy::Proxy; - } - let Ok(url) = url::Url::parse(resource_url) else { - return if resource_url.starts_with("//") - || resource_url.starts_with("http:") - || resource_url.starts_with("https:") - { - ResourceOutputPolicy::Blocked - } else { - ResourceOutputPolicy::Direct - }; - }; - if !matches!(url.scheme(), "http" | "https") { - return ResourceOutputPolicy::Blocked; - } - let Some(host) = url.host_str() else { - return ResourceOutputPolicy::Blocked; - }; - // A host missing from the prepared set still needs a proxy link; a new resource - // field must not expose its origin before the output classifier knows the host. - self.resource_host_policies.get(host).map_or(ResourceOutputPolicy::Proxy, |entry| *entry) - } - - pub fn get_resource_url( - &self, - xtream_cluster: XtreamCluster, - item_type: PlaylistItemType, - virtual_id: VirtualId, - resource_url: &str, - resource_field: &str, - ) -> String { - self.rewrite_resource_url_inner(xtream_cluster, item_type, virtual_id, resource_url, |base| { - format!("{base}/{resource_field}") - }) - } - - pub fn get_bd_path_resource_url( - &self, - xtream_cluster: XtreamCluster, - item_type: PlaylistItemType, - virtual_id: VirtualId, - resource_url: &str, - resource_field: &str, - index: usize, - ) -> String { - self.rewrite_resource_url_inner(xtream_cluster, item_type, virtual_id, resource_url, |base| { - format!("{base}/{resource_field}{}_{index}", xtream_const::XC_PROP_BACKDROP_PATH) - }) - } - - fn rewrite_resource_url_inner( - &self, - xtream_cluster: XtreamCluster, - item_type: PlaylistItemType, - virtual_id: VirtualId, - resource_url: &str, - proxy_suffix: impl FnOnce(&str) -> String, - ) -> String { - let Some(resource_url) = super::persisted_resource_url(resource_url) else { - return String::new(); - }; - let resource_url = resource_url.as_ref(); - if !self.web_ui_request && Self::is_trusted_web_ui_resource_path(resource_url) { - return concat_path(&self.base_url, resource_url); - } - - if self.web_ui_request { - if resource_url.is_empty() { - return resource_url.to_string(); - } - if Self::is_trusted_web_ui_resource_path(resource_url) { - return resource_url.to_string(); - } - return concat_path(&self.base_url, &seal_web_ui_resource_url(&self.encrypt_secret, resource_url)); - } - - let policy = self.resource_output_policy(resource_url); - if policy == ResourceOutputPolicy::Blocked { - return String::new(); - } - let rewrite_url = self.build_reverse_proxy_base_url( - xtream_cluster, - item_type, - virtual_id, - policy == ResourceOutputPolicy::Proxy, - ); - - if let Some(url) = rewrite_url { - if resource_url.starts_with("http://") - || resource_url.starts_with("https://") - || resource_url.starts_with("media-server://image/") - { - return proxy_suffix(&url); - } - } - resource_url.to_string() - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct XtreamPlaylistItem { - pub virtual_id: VirtualId, - pub provider_id: u32, - #[serde(with = "arc_str_serde")] - pub name: Arc, - #[serde(with = "arc_str_serde")] - pub logo: Arc, - #[serde(with = "arc_str_serde")] - pub logo_small: Arc, - #[serde(with = "arc_str_serde")] - pub group: Arc, - #[serde(with = "arc_str_serde")] - pub title: Arc, - #[serde(with = "arc_str_serde")] - pub parent_code: Arc, - #[serde(with = "arc_str_serde")] - pub rec: Arc, - #[serde(with = "arc_str_serde")] - pub url: Arc, - #[serde(default, with = "arc_str_option_serde")] - pub epg_channel_id: Option>, - pub xtream_cluster: XtreamCluster, - #[serde(default)] - pub additional_properties: Option, - pub item_type: PlaylistItemType, - pub category_id: u32, - #[serde(with = "arc_str_serde")] - pub input_name: Arc, - pub channel_no: u32, - #[serde(default)] - pub source_ordinal: u32, - /// Stable provider/origin ID captured before any target transformation. - #[serde(default, with = "arc_str_serde")] - pub input_stream_id: Arc, - #[serde(default, rename = "source_user_agent", alias = "upstream_user_agent", with = "arc_str_option_serde")] - pub upstream_user_agent: Option>, -} - -impl XtreamPlaylistItem { - pub fn to_common(&self) -> CommonPlaylistItem { - CommonPlaylistItem { - virtual_id: self.virtual_id, - provider_id: self.provider_id.intern(), - name: self.name.clone(), - chno: self.channel_no, - logo: self.logo.clone(), - logo_small: self.logo_small.clone(), - group: self.group.clone(), - title: self.title.clone(), - parent_code: self.parent_code.clone(), - audio_track: "".intern(), - time_shift: "".intern(), - rec: self.rec.clone(), - url: self.url.clone(), - input_name: self.input_name.clone(), - item_type: self.item_type, - epg_channel_id: self.epg_channel_id.clone(), - xtream_cluster: Some(self.xtream_cluster), - additional_properties: self.additional_properties.clone(), - category_id: Some(self.category_id), - } - } - - pub fn get_container_extension(&self) -> Option> { - match self.additional_properties { - None => None, - Some(ref props) => match props { - StreamProperties::Live(_) => Some("ts".intern()), - StreamProperties::Video(video) => Some(Arc::clone(&video.container_extension)), - StreamProperties::Series(_) => None, - StreamProperties::Episode(episode) => Some(Arc::clone(&episode.container_extension)), - }, - } - } - - #[inline] - pub fn has_details(&self) -> bool { - self.additional_properties.as_ref().is_some_and(super::stream_properties::StreamProperties::has_details) - } - - pub fn visit_resource_urls(&self, mut visit: impl FnMut(&str)) { - let mut visit_persisted = |value: &str| { - if let Some(url) = super::persisted_resource_url(value) { - visit(url.as_ref()); - } - }; - visit_persisted(&self.logo); - visit_persisted(&self.logo_small); - match self.additional_properties.as_ref() { - Some(StreamProperties::Live(live)) => visit_persisted(&live.stream_icon), - Some(StreamProperties::Video(video)) => { - visit_persisted(&video.stream_icon); - if let Some(details) = video.details.as_ref() { - if let Some(url) = details.cover_big.as_deref() { - visit_persisted(url); - } - if let Some(url) = details.movie_image.as_deref() { - visit_persisted(url); - } - if let Some(backdrops) = details.backdrop_path.as_ref() { - for url in backdrops { - visit_persisted(url); - } - } - } - } - Some(StreamProperties::Series(series)) => { - visit_persisted(&series.cover); - if let Some(backdrops) = series.backdrop_path.as_ref() { - for url in backdrops { - visit_persisted(url); - } - } - if let Some(details) = series.details.as_ref() { - if let Some(seasons) = details.seasons.as_ref() { - for season in seasons { - for url in [&season.cover, &season.cover_tmdb, &season.cover_big].into_iter().flatten() { - visit_persisted(url); - } - } - } - if let Some(episodes) = details.episodes.as_ref() { - for episode in episodes { - visit_persisted(&episode.movie_image); - } - } - } - } - Some(StreamProperties::Episode(episode)) => visit_persisted(&episode.movie_image), - None => {} - } - } - - pub fn resolve_resource_url(&self, field: &str) -> Option> { - let bytes = field.as_bytes(); - let url = if bytes.eq_ignore_ascii_case(b"logo") && !self.logo.is_empty() { - Some(Arc::clone(&self.logo)) - } else if bytes.eq_ignore_ascii_case(b"logo_small") && !self.logo_small.is_empty() { - Some(Arc::clone(&self.logo_small)) - } else { - self.additional_properties.as_ref().and_then(|a| a.resolve_resource_url(field)) - }?; - super::persisted_resource_url(&url).map(|decoded| match decoded { - std::borrow::Cow::Borrowed(_) => Arc::clone(&url), - std::borrow::Cow::Owned(value) => Arc::from(value), - }) - } -} - -impl PlaylistEntry for XtreamPlaylistItem { - #[inline] - fn get_virtual_id(&self) -> VirtualId { self.virtual_id } - fn get_input_stream_id(&self) -> Option> { - if self.input_stream_id.is_empty() { - (self.provider_id > 0).then(|| self.provider_id.to_string().intern()) - } else { - Some(Arc::clone(&self.input_stream_id)) - } - } - - fn get_upstream_user_agent(&self) -> Option<&str> { self.upstream_user_agent.as_deref() } - #[inline] - fn get_provider_id(&self) -> Option { Some(self.provider_id) } - #[inline] - fn get_category_id(&self) -> Option { Some(self.category_id) } - #[inline] - fn get_provider_url(&self) -> Arc { Arc::clone(&self.url) } - - #[inline] - fn get_uuid(&self) -> UUIDType { - generate_runtime_playlist_uuid(&self.input_name, &self.provider_id.to_string(), self.item_type, &self.url) - } - #[inline] - fn get_item_type(&self) -> PlaylistItemType { self.item_type } - #[inline] - fn get_group(&self) -> Arc { Arc::clone(&self.group) } - #[inline] - fn get_name(&self) -> Arc { - if self.title.is_empty() { - Arc::clone(&self.name) - } else { - Arc::clone(&self.title) - } - } - - fn get_resolved_info_document(&self, options: &XtreamMappingOptions) -> Option { - if self.has_details() { - self.additional_properties.as_ref().map(|p| { - p.to_info_document( - options, - self.get_item_type(), - self.get_virtual_id(), - self.get_category_id().unwrap_or(0), - ) - }) - } else { - None - } - } - - #[inline] - fn get_additional_properties(&self) -> Option<&StreamProperties> { self.additional_properties.as_ref() } - #[inline] - fn get_additional_properties_mut(&mut self) -> Option<&mut StreamProperties> { self.additional_properties.as_mut() } -} - -impl From for CommonPlaylistItem { - fn from(item: XtreamPlaylistItem) -> Self { item.to_common() } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PlaylistItem { - #[serde(flatten)] - pub header: PlaylistItemHeader, -} - -impl PlaylistItem { - fn get_additional_properties(header: &PlaylistItemHeader) -> Option { - match &header.additional_properties { - Some(props) => Some(props.clone()), - None => { - match header.xtream_cluster { - XtreamCluster::Live => None, - XtreamCluster::Video => { - let container_extension = extract_extension_from_url(&header.url) - .map(|e| e.strip_prefix('.').unwrap_or(e).to_string()) - .unwrap_or_default(); - Some(StreamProperties::Video(Box::new(VideoStreamProperties { - name: header.name.clone(), - category_id: header.category_id, - stream_id: header.virtual_id.get(), - stream_icon: "".intern(), - direct_source: "".intern(), - custom_sid: None, - added: "".intern(), - container_extension: container_extension.intern(), - rating: None, - rating_5based: None, - stream_type: Some("movie".intern()), - trailer: None, - tmdb: None, - is_adult: 0, - details: None, - }))) - } - XtreamCluster::Series => { - if header.item_type == PlaylistItemType::Series { - let container_extension = extract_extension_from_url(&header.url) - .map(|e| e.strip_prefix('.').unwrap_or(e).to_string()) - .unwrap_or_default(); - // TODO maybe from link ? like s01e02 or something like this - Some(StreamProperties::Episode(Box::new(EpisodeStreamProperties { - episode_id: 0, - episode: 0, - season: 0, - added: None, - release_date: None, - series_release_date: None, - plot: None, - tmdb: None, - movie_image: "".intern(), - container_extension: container_extension.intern(), - audio: None, - video: None, - }))) - } else if header.item_type == PlaylistItemType::SeriesInfo { - Some(StreamProperties::Series(Box::new(SeriesStreamProperties { - name: header.name.clone(), - category_id: header.category_id, - tmdb: None, - series_id: 0, - backdrop_path: None, - cast: "".intern(), - cover: "".intern(), - director: "".intern(), - episode_run_time: None, - genre: None, - last_modified: None, - plot: None, - rating: 0.0, - rating_5based: 0.0, - release_date: None, - youtube_trailer: "".intern(), - details: None, - }))) - } else { - None - } - } - } - } - } - } - - pub fn has_details(&self) -> bool { - self.header.additional_properties.as_ref().is_some_and(super::stream_properties::StreamProperties::has_details) - } - - pub fn get_tmdb_id(&self) -> Option { - self.header.additional_properties.as_ref().and_then(super::stream_properties::StreamProperties::get_tmdb_id) - } -} - -impl From<&PlaylistItem> for XtreamPlaylistItem { - fn from(item: &PlaylistItem) -> Self { - let header = &item.header; - let input_stream_id = Arc::clone(&header.input_stream_id); - let missing_live_input_identity = - input_stream_id.is_empty() && (header.item_type.is_live() || header.item_type == PlaylistItemType::Catchup); - let provider_id = - if missing_live_input_identity { 0 } else { get_provider_id(&header.id, &header.url).unwrap_or_default() }; - let additional_properties = PlaylistItem::get_additional_properties(header); - - XtreamPlaylistItem { - virtual_id: header.virtual_id, - provider_id, - name: if header.item_type == PlaylistItemType::Series { - Arc::clone(&header.title) - } else { - Arc::clone(&header.name) - }, - logo: Arc::clone(&header.logo), - logo_small: Arc::clone(&header.logo_small), - group: Arc::clone(&header.group), - title: Arc::clone(&header.title), - parent_code: Arc::clone(&header.parent_code), - rec: Arc::clone(&header.rec), - url: Arc::clone(&header.url), - epg_channel_id: header.epg_channel_id.clone(), - xtream_cluster: header.xtream_cluster, - additional_properties, - item_type: header.item_type, - category_id: header.category_id, - input_name: Arc::clone(&header.input_name), - channel_no: header.chno, - source_ordinal: header.source_ordinal, - input_stream_id, - upstream_user_agent: header.upstream_user_agent.clone(), - } - } -} - -impl From<&PlaylistItem> for M3uPlaylistItem { - fn from(item: &PlaylistItem) -> Self { - let header = &item.header; - let input_stream_id = Arc::clone(&header.input_stream_id); - let missing_live_input_identity = - input_stream_id.is_empty() && (header.item_type.is_live() || header.item_type == PlaylistItemType::Catchup); - M3uPlaylistItem { - virtual_id: header.virtual_id, - provider_id: if missing_live_input_identity { "".intern() } else { Arc::clone(&header.id) }, - name: if header.item_type == PlaylistItemType::Series { - Arc::clone(&header.title) - } else { - Arc::clone(&header.name) - }, - chno: header.chno, - logo: Arc::clone(&header.logo), - logo_small: Arc::clone(&header.logo_small), - group: Arc::clone(&header.group), - title: Arc::clone(&header.title), - parent_code: Arc::clone(&header.parent_code), - audio_track: Arc::clone(&header.audio_track), - time_shift: Arc::clone(&header.time_shift), - rec: Arc::clone(&header.rec), - url: Arc::clone(&header.url), - epg_channel_id: header.epg_channel_id.clone(), - input_name: Arc::clone(&header.input_name), - item_type: header.item_type, - t_stream_url: Arc::clone(&header.url), - t_resource_url: None, - t_catchup_source: None, - t_catchup_mode: None, - source_ordinal: header.source_ordinal, - additional_properties: header.additional_properties.clone(), - input_stream_id, - upstream_user_agent: header.upstream_user_agent.clone(), - } - } -} - -impl From<&PlaylistItem> for CommonPlaylistItem { - fn from(item: &PlaylistItem) -> Self { - let header = &item.header; - - let additional_properties = PlaylistItem::get_additional_properties(header); - - CommonPlaylistItem { - virtual_id: header.virtual_id, - provider_id: Arc::clone(&header.id), - name: if header.item_type == PlaylistItemType::Series { - Arc::clone(&header.title) - } else { - Arc::clone(&header.name) - }, - logo: header.logo.clone(), - logo_small: header.logo_small.clone(), - group: Arc::clone(&header.group), - title: header.title.clone(), - parent_code: header.parent_code.clone(), - audio_track: header.audio_track.clone(), - time_shift: header.time_shift.clone(), - rec: header.rec.clone(), - url: header.url.clone(), - epg_channel_id: header.epg_channel_id.clone(), - xtream_cluster: Some(header.xtream_cluster), - additional_properties, - item_type: header.item_type, - category_id: Some(header.category_id), - input_name: Arc::clone(&header.input_name), - chno: header.chno, - } - } -} - -impl From<&XtreamPlaylistItem> for PlaylistItem { - fn from(item: &XtreamPlaylistItem) -> Self { - let input_stream_id = item.get_input_stream_id(); - let header = PlaylistItemHeader { - uuid: item.get_uuid(), - virtual_id: item.virtual_id, - id: if item.provider_id == 0 && input_stream_id.is_none() { - "".intern() - } else { - item.provider_id.to_string().intern() - }, - name: item.name.clone(), - title: item.title.clone(), - logo: item.logo.clone(), - logo_small: item.logo_small.clone(), - group: item.group.clone(), - parent_code: item.parent_code.clone(), - rec: item.rec.clone(), - url: item.url.clone(), - epg_channel_id: item.epg_channel_id.clone(), - xtream_cluster: item.xtream_cluster, - item_type: item.item_type, - category_id: item.category_id, - input_name: item.input_name.clone(), - chno: item.channel_no, - audio_track: "".intern(), - time_shift: "".intern(), - additional_properties: item.additional_properties.clone(), - source_ordinal: item.source_ordinal, - input_stream_id: input_stream_id.unwrap_or_else(|| "".intern()), - upstream_user_agent: item.upstream_user_agent.clone(), - }; - - PlaylistItem { header } - } -} - -impl From<&M3uPlaylistItem> for PlaylistItem { - fn from(item: &M3uPlaylistItem) -> Self { - let header = PlaylistItemHeader { - uuid: item.get_uuid(), - virtual_id: item.virtual_id, - id: item.provider_id.clone(), - name: item.name.clone(), - title: item.title.clone(), - logo: item.logo.clone(), - logo_small: item.logo_small.clone(), - group: item.group.clone(), - parent_code: item.parent_code.clone(), - rec: item.rec.clone(), - url: item.url.clone(), - epg_channel_id: item.epg_channel_id.clone(), - xtream_cluster: item.item_type.cluster(), - item_type: item.item_type, - category_id: 0, - input_name: item.input_name.clone(), - chno: item.chno, - audio_track: item.audio_track.clone(), - time_shift: item.time_shift.clone(), - additional_properties: item.additional_properties.clone(), - source_ordinal: item.source_ordinal, - input_stream_id: item.get_input_stream_id().unwrap_or_else(|| "".intern()), - upstream_user_agent: item.upstream_user_agent.clone(), - }; - - PlaylistItem { header } - } -} - -impl PlaylistItem { - /// Canonical `StalkerPlaylistItem` → `PlaylistItem` conversion. - /// - /// This is the single conversion used by both the download path - /// (processor) and the disk-load path so the generated identity is stable - /// across processing modes: the UUID is always seeded with the owning - /// input name and the `input_name` header field is always populated. - /// The group falls back to a cluster default only when the portal did - /// not supply a category name. - pub fn from_stalker(item: &StalkerPlaylistItem, input_name: &str) -> Self { - let item_type = match item.stream_kind { - StalkerStreamKind::Live | StalkerStreamKind::Archive => PlaylistItemType::Live, - StalkerStreamKind::Movie => PlaylistItemType::Video, - StalkerStreamKind::Episode => { - if item.is_series_root() { - PlaylistItemType::SeriesInfo - } else { - PlaylistItemType::Series - } - } - }; - let xtream_cluster = match item.stream_kind { - StalkerStreamKind::Live | StalkerStreamKind::Archive => XtreamCluster::Live, - StalkerStreamKind::Movie => XtreamCluster::Video, - StalkerStreamKind::Episode => XtreamCluster::Series, - }; - let stream_id_str: Arc = Internable::intern(item.stream_id.to_string()); - let logo: Arc = item.logo_url.clone().unwrap_or_else(|| Internable::intern(String::new())); - // Keep unresolved commands private; playback resolves them from the persisted Stalker metadata. - let url = Arc::clone(&item.stream_url); - let group: Arc = if item.category_name.is_empty() { - let fallback = match item.stream_kind { - StalkerStreamKind::Live | StalkerStreamKind::Archive => "Live", - StalkerStreamKind::Movie => "Movies", - StalkerStreamKind::Episode => "Series", - }; - Internable::intern(fallback.to_string()) - } else { - Arc::clone(&item.category_name) - }; - let header = PlaylistItemHeader { - uuid: generate_provider_playlist_uuid(input_name, &stream_id_str, item_type), - virtual_id: VirtualId::new(item.stream_id), - id: Arc::clone(&stream_id_str), - name: Arc::clone(&item.name), - title: Arc::clone(&item.name), - logo: Arc::clone(&logo), - logo_small: Internable::intern(String::new()), - group, - parent_code: Internable::intern(String::new()), - audio_track: Internable::intern(String::new()), - time_shift: Internable::intern(String::new()), - rec: Internable::intern(String::new()), - url, - epg_channel_id: item.epg_channel_id.clone(), - item_type, - xtream_cluster, - additional_properties: None, - input_name: Internable::intern(input_name.to_string()), - chno: item.number, - category_id: item.category_id, - source_ordinal: 0, - input_stream_id: stream_id_str, - upstream_user_agent: None, - }; - - PlaylistItem { header } - } -} - -impl PlaylistEntry for PlaylistItem { - #[inline] - fn get_virtual_id(&self) -> VirtualId { self.header.virtual_id } - - #[inline] - fn get_input_stream_id(&self) -> Option> { self.header.get_input_stream_id() } - - fn get_upstream_user_agent(&self) -> Option<&str> { self.header.upstream_user_agent.as_deref() } - - fn get_provider_id(&self) -> Option { - let header = &self.header; - get_provider_id(&header.id, &header.url) - } - - #[inline] - fn get_category_id(&self) -> Option { Some(self.header.category_id) } - - #[inline] - fn get_provider_url(&self) -> Arc { Arc::clone(&self.header.url) } - - #[inline] - fn get_uuid(&self) -> UUIDType { - let header = &self.header; - generate_runtime_playlist_uuid(&header.input_name, &header.id, header.item_type, &header.url) - } - - #[inline] - fn get_item_type(&self) -> PlaylistItemType { self.header.item_type } - - #[inline] - fn get_group(&self) -> Arc { Arc::clone(&self.header.group) } - - #[inline] - fn get_name(&self) -> Arc { self.header.get_name() } - - fn get_resolved_info_document(&self, options: &XtreamMappingOptions) -> Option { - if self.has_details() { - self.header.additional_properties.as_ref().map(|p| { - p.to_info_document( - options, - self.get_item_type(), - self.get_virtual_id(), - self.get_category_id().unwrap_or(0), - ) - }) - } else { - None - } - } - - fn get_additional_properties(&self) -> Option<&StreamProperties> { self.header.additional_properties.as_ref() } - #[inline] - fn get_additional_properties_mut(&mut self) -> Option<&mut StreamProperties> { - self.header.additional_properties.as_mut() - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PlaylistGroup { - pub id: u32, - #[serde(with = "arc_str_serde")] - pub title: Arc, - pub channels: Vec, - pub xtream_cluster: XtreamCluster, -} - -impl PlaylistGroup { - #[inline] - pub fn on_load(&mut self) { - for pl in &mut self.channels { - pl.header.freeze_input_stream_id(); - pl.header.gen_uuid(); - pl.header.category_id = self.id; - } - } - - #[inline] - pub fn filter_count(&self, filter: F) -> usize - where - F: Fn(&PlaylistItem) -> bool, - { - self.channels.iter().filter(|&c| filter(c)).count() - } -} - #[cfg(test)] -mod tests { - use super::*; - use crate::model::{ - CatchupAttribute, CatchupProperties, LiveStreamProperties, PlaylistItemType, StreamProperties, XtreamCluster, - XtreamMappingFlags, - }; - - #[test] - fn cluster_is_the_single_source_of_truth_for_every_item_type() { - use strum::IntoEnumIterator; - - for item_type in PlaylistItemType::iter() { - let cluster = item_type.cluster(); - - // `From` and the inherent method cannot disagree: one delegates. - assert_eq!(XtreamCluster::from(item_type), cluster, "{item_type:?}"); - - // is_cluster agrees with cluster() for the right one and rejects the - // other two. This is what used to be a separately written match. - assert!(item_type.is_cluster(cluster), "{item_type:?} should be in its own cluster"); - for other in [XtreamCluster::Live, XtreamCluster::Video, XtreamCluster::Series] { - assert_eq!(item_type.is_cluster(other), other == cluster, "{item_type:?} vs {other:?}"); - } - } - } - - #[test] - fn header_field_parse_round_trips_every_variant() { - for field in [ - HeaderField::Id, - HeaderField::ProviderId, - HeaderField::Title, - HeaderField::Name, - HeaderField::Logo, - HeaderField::LogoSmall, - HeaderField::ParentCode, - HeaderField::AudioTrack, - HeaderField::TimeShift, - HeaderField::Rec, - HeaderField::Url, - HeaderField::Group, - HeaderField::Caption, - HeaderField::Input, - HeaderField::Type, - HeaderField::EpgChannelId, - HeaderField::Chno, - HeaderField::Genre, - ] { - assert_eq!(HeaderField::parse(field.as_str()), Some(field), "{field} did not round-trip"); - // Lookup stayed case-insensitive. - assert_eq!(HeaderField::parse(&field.as_str().to_uppercase()), Some(field)); - } - assert_eq!(HeaderField::parse("epg_id"), Some(HeaderField::EpgChannelId), "legacy alias must still parse"); - assert_eq!(HeaderField::parse("input_stream_id"), None); - assert_eq!(HeaderField::parse(""), None); - } - - #[test] - fn m3u_item_exposes_provider_id_but_not_the_header_only_fields() { - // The M3U resource endpoint resolves a URL path segment through - // `get_field`, so which names resolve is externally visible behaviour. - // M3uPlaylistItem carries input_name, item_type and additional_properties, - // but none of them were ever addressable by name and must stay that way. - let mut header = PlaylistItemHeader { chno: 7, ..PlaylistItemHeader::default() }; - header.name = "Channel".intern(); - let item = M3uPlaylistItem::from(&PlaylistItem { header }); - - assert_eq!(item.get_field("name").as_deref(), Some("Channel")); - assert_eq!(item.get_field("chno").as_deref(), Some("7")); - assert_eq!(item.get_field("caption").as_deref(), Some("Channel")); - - for absent in ["id", "input", "type", "genre", "not_a_field"] { - assert!(item.get_field(absent).is_none(), "{absent} must not resolve on an M3U item"); - } - } - - #[test] - fn from_stalker_uuid_is_seeded_with_input_name_not_category() { - let mut item = StalkerPlaylistItem { - stream_id: 42, - category_name: Internable::intern("News".to_string()), - ..StalkerPlaylistItem::default() - }; - let converted_a = PlaylistItem::from_stalker(&item, "input_a"); - // A category rename must not change the identity of the channel. - item.category_name = Internable::intern("World News".to_string()); - let converted_b = PlaylistItem::from_stalker(&item, "input_a"); - assert_eq!(converted_a.header.uuid, converted_b.header.uuid); - // A different input owning the same stream id must produce a distinct identity. - let converted_c = PlaylistItem::from_stalker(&item, "input_b"); - assert_ne!(converted_a.header.uuid, converted_c.header.uuid); - } - - #[test] - fn from_stalker_populates_input_name_and_group() { - let item = StalkerPlaylistItem { - stream_id: 7, - stream_kind: StalkerStreamKind::Movie, - category_name: Internable::intern("Action".to_string()), - cmd: Internable::intern("ffmpeg http://streams.example/movie/7".to_string()), - ..StalkerPlaylistItem::default() - }; - let converted = PlaylistItem::from_stalker(&item, "my_input"); - assert_eq!(&*converted.header.input_name, "my_input"); - assert_eq!(&*converted.header.input_stream_id, "7"); - assert!(converted.header.url.is_empty()); - assert_eq!(&*converted.header.group, "Action"); - assert_eq!(converted.header.item_type, PlaylistItemType::Video); - } - - #[test] - fn from_stalker_group_falls_back_to_cluster_default() { - let item = StalkerPlaylistItem { - stream_id: 7, - stream_kind: StalkerStreamKind::Movie, - ..StalkerPlaylistItem::default() - }; - let converted = PlaylistItem::from_stalker(&item, "my_input"); - assert_eq!(&*converted.header.group, "Movies"); - } - - fn sample_options() -> XtreamMappingOptions { - XtreamMappingOptions { - base_url: "/api/v1/playlist/resource".to_string(), - username: "user".to_string(), - password: "pass".to_string(), - force_redirect: None, - reverse_item_types: PlaylistItemTypeSet::from_item(PlaylistItemType::Live), - resource_proxy_item_types: PlaylistItemTypeSet::from_item(PlaylistItemType::Live), - web_ui_request: true, - flags: XtreamMappingFlags::RewriteResourceUrl.into(), - encrypt_secret: [3u8; 16], - resource_host_policies: Arc::default(), - } - } - - #[test] - fn disabled_resource_rewrite_proxies_private_xtream_images() { - let mut options = sample_options(); - options.web_ui_request = false; - options.base_url = "https://proxy.example".to_string(); - options.flags = XtreamMappingFlagsSet::new(); - options.resource_host_policies.insert("192.168.1.20".to_string(), ResourceOutputPolicy::Proxy); - options.resource_host_policies.insert("8.8.8.8".to_string(), ResourceOutputPolicy::Direct); - options.resource_host_policies.insert("127.0.0.1".to_string(), ResourceOutputPolicy::Blocked); - - assert_eq!( - options.get_resource_url( - XtreamCluster::Live, - PlaylistItemType::Live, - VirtualId::new(1), - "http://192.168.1.20/logo.png", - "logo", - ), - "https://proxy.example/resource/live/user/pass/1/logo" - ); - assert_eq!( - options.get_resource_url( - XtreamCluster::Live, - PlaylistItemType::Live, - VirtualId::new(1), - "http://8.8.8.8/logo.png", - "logo", - ), - "http://8.8.8.8/logo.png" - ); - assert_eq!( - options.get_resource_url( - XtreamCluster::Live, - PlaylistItemType::Live, - VirtualId::new(1), - "http://127.0.0.1/logo.png", - "logo", - ), - "" - ); - assert_eq!( - options.get_bd_path_resource_url( - XtreamCluster::Live, - PlaylistItemType::Live, - VirtualId::new(1), - "http://192.168.1.20/backdrop.png", - "", - 0, - ), - "https://proxy.example/resource/live/user/pass/1/backdrop_path_0" - ); - - options.resource_proxy_item_types = PlaylistItemTypeSet::empty(); - assert_eq!( - options.get_resource_url( - XtreamCluster::Live, - PlaylistItemType::Live, - VirtualId::new(1), - "http://192.168.1.20/logo.png", - "logo", - ), - "https://proxy.example/resource/live/user/pass/1/logo" - ); - } - - #[test] - fn get_resource_url_keeps_internal_paths_for_web_ui_requests() { - let options = sample_options(); - - assert_eq!( - options.get_resource_url( - XtreamCluster::Series, - PlaylistItemType::Series, - VirtualId::new(1), - "/api/v1/library/thumbnail/abc", - "logo", - ), - "/api/v1/library/thumbnail/abc", - ); - } - - #[test] - fn get_bd_path_resource_url_keeps_internal_thumbnail_paths_for_web_ui_requests() { - let options = sample_options(); - - assert_eq!( - options.get_bd_path_resource_url( - XtreamCluster::Series, - PlaylistItemType::Series, - VirtualId::new(1), - "/api/v1/library/thumbnail/backdrop", - "backdrop_", - 0, - ), - "/api/v1/library/thumbnail/backdrop", - ); - } - - #[test] - fn get_resource_url_absolutizes_internal_thumbnail_paths_for_xtream_clients() { - let mut options = sample_options(); - options.web_ui_request = false; - options.base_url = "http://proxy.example/base".to_string(); - options.reverse_item_types = PlaylistItemTypeSet::empty(); - options.resource_proxy_item_types = PlaylistItemTypeSet::empty(); - - assert_eq!( - options.get_resource_url( - XtreamCluster::Series, - PlaylistItemType::LocalSeries, - VirtualId::new(1), - "/api/v1/library/thumbnail/abc", - "logo", - ), - "http://proxy.example/base/api/v1/library/thumbnail/abc", - ); - } - - #[test] - fn get_bd_path_resource_url_absolutizes_internal_thumbnail_paths_for_xtream_clients() { - let mut options = sample_options(); - options.web_ui_request = false; - options.base_url = "http://proxy.example/base".to_string(); - options.reverse_item_types = PlaylistItemTypeSet::empty(); - options.resource_proxy_item_types = PlaylistItemTypeSet::empty(); - - assert_eq!( - options.get_bd_path_resource_url( - XtreamCluster::Series, - PlaylistItemType::LocalSeries, - VirtualId::new(1), - "/api/v1/library/thumbnail/backdrop", - "backdrop_", - 0, - ), - "http://proxy.example/base/api/v1/library/thumbnail/backdrop", - ); - } - - #[test] - fn get_resource_url_obfuscates_protocol_relative_urls_for_web_ui_requests() { - let options = sample_options(); - let resource_url = "//cdn.example.com/poster.jpg"; - - assert_eq!( - options.get_resource_url( - XtreamCluster::Series, - PlaylistItemType::Series, - VirtualId::new(1), - resource_url, - "logo", - ), - concat_path(&options.base_url, &seal_web_ui_resource_url(&options.encrypt_secret, resource_url)), - ); - } - - #[test] - fn get_resource_url_rewrites_redirect_resources_for_xtream_clients() { - let mut options = sample_options(); - options.web_ui_request = false; - options.base_url = "http://proxy.example/iptv".to_string(); - options.reverse_item_types = PlaylistItemTypeSet::empty(); - options.resource_proxy_item_types = PlaylistItemTypeSet::from_item(PlaylistItemType::Live); - - assert_eq!( - options.get_resource_url( - XtreamCluster::Live, - PlaylistItemType::Live, - VirtualId::new(2017), - "https://provider.example/logo.png", - "logo", - ), - "http://proxy.example/iptv/resource/live/user/pass/2017/logo", - ); - } - - #[test] - fn get_resource_url_does_not_bypass_untrusted_root_relative_paths_for_web_ui_requests() { - let options = sample_options(); - let resource_url = "/provider-controlled/poster.jpg"; - - assert_eq!( - options.get_resource_url( - XtreamCluster::Series, - PlaylistItemType::Series, - VirtualId::new(1), - resource_url, - "logo", - ), - concat_path(&options.base_url, &seal_web_ui_resource_url(&options.encrypt_secret, resource_url)), - ); - } - - #[test] - fn get_resource_url_does_not_trust_absolute_urls_containing_internal_thumbnail_path() { - let options = sample_options(); - let resource_url = "https://provider.example/api/v1/library/thumbnail/abc"; - - assert_eq!( - options.get_resource_url( - XtreamCluster::Series, - PlaylistItemType::Series, - VirtualId::new(1), - resource_url, - "logo", - ), - concat_path(&options.base_url, &seal_web_ui_resource_url(&options.encrypt_secret, resource_url)), - ); - } - - #[test] - fn xtream_playlist_item_conversion_falls_back_to_numeric_id_in_url() { - let mut item = PlaylistItem { - header: PlaylistItemHeader { - id: "channel-alpha".intern(), - url: "http://provider.example/live/user/pass/12345.ts".intern(), - input_name: "input".intern(), - item_type: PlaylistItemType::Live, - xtream_cluster: XtreamCluster::Live, - ..PlaylistItemHeader::default() - }, - }; - item.header.freeze_input_stream_id(); - - let xtream_item = XtreamPlaylistItem::from(&item); - assert_eq!(xtream_item.provider_id, 12345); - } - - #[test] - fn xtream_playlist_item_preserves_numeric_input_stream_id_as_string() { - let mut item = PlaylistItem { - header: PlaylistItemHeader { - id: "80510".intern(), - virtual_id: VirtualId::new(1001), - url: "http://provider.example/live/user/pass/80510.ts".intern(), - input_name: "input".intern(), - item_type: PlaylistItemType::Live, - xtream_cluster: XtreamCluster::Live, - ..PlaylistItemHeader::default() - }, - }; - item.header.freeze_input_stream_id(); - - let xtream_item = XtreamPlaylistItem::from(&item); - - assert_eq!(xtream_item.provider_id, 80510); - assert_eq!(xtream_item.input_stream_id.as_ref(), "80510"); - assert_eq!(xtream_item.get_input_stream_id().as_deref(), Some("80510")); - } - - #[test] - fn alphanumeric_m3u_input_stream_id_survives_xtream_materialization() { - let mut source = PlaylistItem { - header: PlaylistItemHeader { - id: "channel-alpha".intern(), - url: "http://provider.example/live/user/pass/12345.ts".intern(), - input_name: "input".intern(), - item_type: PlaylistItemType::Live, - xtream_cluster: XtreamCluster::Live, - ..PlaylistItemHeader::default() - }, - }; - source.header.freeze_input_stream_id(); - - let m3u_item = M3uPlaylistItem::from(&source); - let common_item = PlaylistItem::from(&m3u_item); - let xtream_item = XtreamPlaylistItem::from(&common_item); - - assert_eq!(m3u_item.input_stream_id.as_ref(), "channel-alpha"); - assert_eq!(xtream_item.provider_id, 12345); - assert_eq!(xtream_item.input_stream_id.as_ref(), "channel-alpha"); - } - - #[test] - fn url_hash_input_stream_id_is_preserved_verbatim() { - let mut source = PlaylistItem { - header: PlaylistItemHeader { - id: "d34db33f".intern(), - url: "http://provider.example/live/channel.m3u8".intern(), - ..PlaylistItemHeader::default() - }, - }; - source.header.freeze_input_stream_id(); - - let m3u_item = M3uPlaylistItem::from(&source); - - assert_eq!(m3u_item.input_stream_id.as_ref(), "d34db33f"); - } - - #[test] - fn target_field_mapping_cannot_change_frozen_input_stream_id() { - let mut header = PlaylistItemHeader { id: "origin-alpha".intern(), ..PlaylistItemHeader::default() }; - header.freeze_input_stream_id(); - - assert!(header.set_field("id", "target-id")); - - assert_eq!(header.id.as_ref(), "target-id"); - assert_eq!(header.input_stream_id.as_ref(), "origin-alpha"); - assert!(!header.set_field("input_stream_id", "unexpected")); - assert_eq!(header.input_stream_id.as_ref(), "origin-alpha"); - } - - #[test] - fn legacy_playlist_items_fall_back_to_provider_id_without_virtual_id() { - let mut source = PlaylistItem { - header: PlaylistItemHeader { - id: "legacy-alpha".intern(), - virtual_id: VirtualId::new(7001), - url: "http://provider.example/live/user/pass/80510.ts".intern(), - ..PlaylistItemHeader::default() - }, - }; - source.header.freeze_input_stream_id(); - let mut m3u_item = M3uPlaylistItem::from(&source); - m3u_item.input_stream_id = "".intern(); - let mut xtream_item = XtreamPlaylistItem::from(&source); - xtream_item.input_stream_id = "".intern(); - - assert_eq!(m3u_item.get_input_stream_id().as_deref(), Some("legacy-alpha")); - assert_eq!(xtream_item.get_input_stream_id().as_deref(), Some("80510")); - xtream_item.provider_id = 0; - xtream_item.url = "http://provider.example/live/channel.ts".intern(); - assert_eq!(xtream_item.get_input_stream_id(), None); - - let mut legacy_common_item = PlaylistItem::from(&xtream_item); - assert!(legacy_common_item.header.id.is_empty()); - assert_eq!(legacy_common_item.get_input_stream_id(), None); - legacy_common_item.header.id = "target-mapped-id".intern(); - let rematerialized_m3u_item = M3uPlaylistItem::from(&legacy_common_item); - let rematerialized_xtream_item = XtreamPlaylistItem::from(&legacy_common_item); - assert_eq!(legacy_common_item.header.id.as_ref(), "target-mapped-id"); - assert_eq!(legacy_common_item.get_input_stream_id(), None); - assert!(rematerialized_m3u_item.provider_id.is_empty()); - assert_eq!(rematerialized_m3u_item.get_input_stream_id(), None); - assert_eq!(rematerialized_xtream_item.provider_id, 0); - assert_eq!(rematerialized_xtream_item.get_input_stream_id(), None); - - xtream_item.input_stream_id = "explicit-origin-alpha".intern(); - assert_eq!(xtream_item.get_input_stream_id().as_deref(), Some("explicit-origin-alpha")); - let identified_common_item = PlaylistItem::from(&xtream_item); - assert_eq!(identified_common_item.header.input_stream_id.as_ref(), "explicit-origin-alpha"); - assert_eq!(identified_common_item.get_input_stream_id().as_deref(), Some("explicit-origin-alpha")); - - let identified_m3u_item = M3uPlaylistItem::from(&identified_common_item); - let identified_xtream_item = XtreamPlaylistItem::from(&identified_common_item); - assert_eq!(identified_m3u_item.input_stream_id.as_ref(), "explicit-origin-alpha"); - assert_eq!(identified_m3u_item.get_input_stream_id().as_deref(), Some("explicit-origin-alpha")); - assert_eq!(identified_xtream_item.provider_id, 0); - assert_eq!(identified_xtream_item.input_stream_id.as_ref(), "explicit-origin-alpha"); - assert_eq!(identified_xtream_item.get_input_stream_id().as_deref(), Some("explicit-origin-alpha")); - } - - #[test] - fn m3u_to_m3u_emits_tvg_id_from_epg_channel_id() { - let item = M3uPlaylistItem { - virtual_id: VirtualId::default(), - provider_id: "prov1".intern(), - input_stream_id: "prov1".intern(), - upstream_user_agent: None, - name: "Test Channel".intern(), - chno: 0, - logo: "".intern(), - logo_small: "".intern(), - group: "Test Group".intern(), - title: "Test Title".intern(), - parent_code: "".intern(), - audio_track: "".intern(), - time_shift: "".intern(), - rec: "".intern(), - url: "http://example.com/stream".intern(), - epg_channel_id: Some("epg_channel_123".intern()), - input_name: "test".intern(), - item_type: PlaylistItemType::Live, - t_stream_url: "".intern(), - t_resource_url: None, - t_catchup_source: None, - t_catchup_mode: None, - source_ordinal: 0, - additional_properties: None, - }; - - let output = item.to_m3u(None, false); - assert!(output.contains(r#"tvg-id="epg_channel_123""#), "M3U output must contain tvg-id from epg_channel_id"); - } - - #[test] - fn m3u_to_m3u_preserves_mixed_case_tvg_id() { - // EPG matching is case-insensitive, so ids are no longer lowercased when parsed. - // The M3U tvg-id output must preserve the channel's original source case. - let item = M3uPlaylistItem { - virtual_id: VirtualId::default(), - provider_id: "prov1".intern(), - input_stream_id: "prov1".intern(), - upstream_user_agent: None, - name: "Test Channel".intern(), - chno: 0, - logo: "".intern(), - logo_small: "".intern(), - group: "Test Group".intern(), - title: "Test Title".intern(), - parent_code: "".intern(), - audio_track: "".intern(), - time_shift: "".intern(), - rec: "".intern(), - url: "http://example.com/stream".intern(), - epg_channel_id: Some("CNN.us".intern()), - input_name: "test".intern(), - item_type: PlaylistItemType::Live, - t_stream_url: "".intern(), - t_resource_url: None, - t_catchup_source: None, - t_catchup_mode: None, - source_ordinal: 0, - additional_properties: None, - }; - - let output = item.to_m3u(None, false); - assert!(output.contains(r#"tvg-id="CNN.us""#), "M3U tvg-id must preserve original case, got: {output}"); - } - - #[test] - fn m3u_to_m3u_emits_tvg_chno_from_chno() { - let item = M3uPlaylistItem { - virtual_id: VirtualId::default(), - provider_id: "prov1".intern(), - input_stream_id: "prov1".intern(), - upstream_user_agent: None, - name: "Test Channel".intern(), - chno: 42, - logo: "".intern(), - logo_small: "".intern(), - group: "Test Group".intern(), - title: "Test Title".intern(), - parent_code: "".intern(), - audio_track: "".intern(), - time_shift: "".intern(), - rec: "".intern(), - url: "http://example.com/stream".intern(), - epg_channel_id: None, - input_name: "test".intern(), - item_type: PlaylistItemType::Live, - t_stream_url: "".intern(), - t_resource_url: None, - t_catchup_source: None, - t_catchup_mode: None, - source_ordinal: 0, - additional_properties: None, - }; - - let output = item.to_m3u(None, false); - assert!(output.contains(r#"tvg-chno="42""#), "M3U output must contain tvg-chno from chno"); - } - - #[test] - fn m3u_to_m3u_omits_tvg_chno_when_chno_is_zero() { - let item = M3uPlaylistItem { - virtual_id: VirtualId::default(), - provider_id: "prov1".intern(), - input_stream_id: "prov1".intern(), - upstream_user_agent: None, - name: "Test Channel".intern(), - chno: 0, - logo: "".intern(), - logo_small: "".intern(), - group: "Test Group".intern(), - title: "Test Title".intern(), - parent_code: "".intern(), - audio_track: "".intern(), - time_shift: "".intern(), - rec: "".intern(), - url: "http://example.com/stream".intern(), - epg_channel_id: None, - input_name: "test".intern(), - item_type: PlaylistItemType::Live, - t_stream_url: "".intern(), - t_resource_url: None, - t_catchup_source: None, - t_catchup_mode: None, - source_ordinal: 0, - additional_properties: None, - }; - - let output = item.to_m3u(None, false); - assert!(!output.contains("tvg-chno"), "M3U output must not contain tvg-chno when chno is 0"); - } - - #[test] - fn m3u_to_m3u_preserves_catchup_attributes() { - let item = M3uPlaylistItem { - virtual_id: VirtualId::default(), - provider_id: "prov1".intern(), - input_stream_id: "prov1".intern(), - upstream_user_agent: None, - name: "Test Channel".intern(), - chno: 0, - logo: "".intern(), - logo_small: "".intern(), - group: "Test Group".intern(), - title: "Test Title".intern(), - parent_code: "".intern(), - audio_track: "".intern(), - time_shift: "".intern(), - rec: "".intern(), - url: "http://example.com/stream".intern(), - epg_channel_id: Some("channel1".intern()), - input_name: "test".intern(), - item_type: PlaylistItemType::Live, - t_stream_url: "".intern(), - t_resource_url: None, - t_catchup_source: None, - t_catchup_mode: None, - source_ordinal: 0, - additional_properties: Some(StreamProperties::Live(Box::new(LiveStreamProperties { - catchup: Some(CatchupProperties { - mode: Some("append".intern()), - days: Some("7".intern()), - source: Some("?offset=-${offset}&utcstart=${timestamp}".intern()), - correction: Some("-2.0".intern()), - catchup_type: Some("xc".intern()), - extra_attributes: vec![CatchupAttribute { name: "catchup-extra".intern(), value: "keep".intern() }], - ..CatchupProperties::default() - }), - ..LiveStreamProperties::default() - }))), - }; - - let output = item.to_m3u(None, false); - assert!(output.contains(r#"catchup="append""#)); - assert!(output.contains(r#"catchup-days="7""#)); - assert!(output.contains(r#"catchup-source="?offset=-${offset}&utcstart=${timestamp}""#)); - assert!(output.contains(r#"catchup-correction="-2.0""#)); - assert!(output.contains(r#"catchup-type="xc""#)); - assert!(output.contains(r#"catchup-extra="keep""#)); - } - - #[test] - fn m3u_to_m3u_unifies_append_to_catchup_type_only() { - let item = M3uPlaylistItem { - virtual_id: VirtualId::default(), - provider_id: "prov1".intern(), - input_stream_id: "prov1".intern(), - upstream_user_agent: None, - name: "Test Channel".intern(), - chno: 0, - logo: "".intern(), - logo_small: "".intern(), - group: "Test Group".intern(), - title: "Test Title".intern(), - parent_code: "".intern(), - audio_track: "".intern(), - time_shift: "".intern(), - rec: "".intern(), - url: "http://example.com/stream".intern(), - epg_channel_id: Some("channel1".intern()), - input_name: "test".intern(), - item_type: PlaylistItemType::Live, - t_stream_url: "".intern(), - t_resource_url: None, - t_catchup_source: None, - t_catchup_mode: None, - source_ordinal: 0, - additional_properties: Some(StreamProperties::Live(Box::new(LiveStreamProperties { - catchup: Some(CatchupProperties { - mode: Some("append".intern()), - days: Some("7".intern()), - ..CatchupProperties::default() - }), - ..LiveStreamProperties::default() - }))), - }; - - let output = item.to_m3u(None, false); - assert!(!output.contains(r#"catchup="append""#)); - assert!(output.contains(r#"catchup-type="append""#)); - assert!(output.contains(r#"catchup-days="7""#)); - assert!(!output.contains(r#"catchup-type="xc""#)); - } - - #[test] - fn m3u_to_m3u_uses_rewritten_catchup_mode_and_source() { - let item = M3uPlaylistItem { - virtual_id: VirtualId::default(), - provider_id: "prov1".intern(), - input_stream_id: "prov1".intern(), - upstream_user_agent: None, - name: "Test Channel".intern(), - chno: 0, - logo: "".intern(), - logo_small: "".intern(), - group: "Test Group".intern(), - title: "Test Title".intern(), - parent_code: "".intern(), - audio_track: "".intern(), - time_shift: "".intern(), - rec: "".intern(), - url: "http://example.com/stream".intern(), - epg_channel_id: Some("channel1".intern()), - input_name: "test".intern(), - item_type: PlaylistItemType::Live, - t_stream_url: "".intern(), - t_resource_url: None, - t_catchup_source: Some("http://proxy.example/m3u-catchup/token?v0={utc}".intern()), - t_catchup_mode: Some("default".intern()), - source_ordinal: 0, - additional_properties: Some(StreamProperties::Live(Box::new(LiveStreamProperties { - catchup: Some(CatchupProperties { - mode: Some("append".intern()), - source: Some("?offset=-${offset}".intern()), - ..CatchupProperties::default() - }), - ..LiveStreamProperties::default() - }))), - }; - - let output = item.to_m3u(None, false); - assert!(output.contains(r#"catchup="default""#)); - assert!(output.contains(r#"catchup-source="http://proxy.example/m3u-catchup/token?v0={utc}""#)); - assert!(!output.contains(r#"catchup-source="?offset=-${offset}""#)); - } - - #[test] - fn m3u_to_m3u_flussonic_source_override_uses_rewritten_mode() { - let mut item = M3uPlaylistItem::from(&PlaylistItem { header: PlaylistItemHeader::default() }); - item.url = "https://provider.example/channel/mono.m3u8?token=secret".intern(); - item.t_stream_url = "http://proxy.example/m3u-stream/alice/pass/42/index.m3u8".intern(); - item.t_catchup_mode = Some("default".intern()); - item.t_catchup_source = Some("http://proxy.example/m3u-catchup/token?v0={utc}&v1={duration}".intern()); - item.additional_properties = Some(StreamProperties::Live(Box::new(LiveStreamProperties { - catchup: Some(CatchupProperties { - mode: Some("fs".intern()), - catchup_type: Some("flussonic".intern()), - source: Some("https://provider.example/channel/archive-{utc}-{duration}.m3u8?token=secret".intern()), - ..CatchupProperties::default() - }), - ..LiveStreamProperties::default() - }))); - let output = item.to_m3u(None, false); - assert!(output.contains(r#"catchup="default""#)); - assert!(output.contains("v0={utc}&v1={duration}")); - assert!(!output.contains("catchup-type=")); - assert!(!output.contains("token=secret")); - assert!(output.ends_with("/42/index.m3u8")); - item.t_catchup_mode = None; - item.t_catchup_source = None; - let native = item.to_m3u(None, false); - assert!(native.contains(r#"catchup-type="flussonic""#)); - assert!(!native.contains("catchup-source=")); - } - - #[test] - fn m3u_to_m3u_emits_only_configured_upstream_user_agent() { - let mut item = M3uPlaylistItem::from(&PlaylistItem { header: PlaylistItemHeader::default() }); - item.upstream_user_agent = Some("Provider-UA".intern()); - - assert!(item.to_m3u(None, false).contains("#EXTVLCOPT:http-user-agent=Provider-UA\n")); - item.upstream_user_agent = None; - assert!(!item.to_m3u(None, false).contains("#EXTVLCOPT:http-user-agent=")); - } -} +mod tests; + +mod fields; +mod group; +mod header; +mod item; +mod kind; +mod m3u; +mod xtream; +pub use fields::{FieldGet, FieldGetAccessor, FieldRef, FieldSet, FieldSetAccessor}; +pub use group::PlaylistGroup; +pub use header::PlaylistItemHeader; +pub use item::{PlaylistEntry, PlaylistItem}; +pub use kind::{PlaylistItemType, XtreamCluster}; +pub use m3u::M3uPlaylistItem; +pub use xtream::{ + ResourceOutputPolicy, XtreamMappingFlags, XtreamMappingFlagsSet, XtreamMappingOptions, XtreamPlaylistItem, +}; diff --git a/shared/src/model/playlist/fields.rs b/shared/src/model/playlist/fields.rs new file mode 100644 index 000000000..c31f68add --- /dev/null +++ b/shared/src/model/playlist/fields.rs @@ -0,0 +1,65 @@ +use crate::utils::Internable; +use std::sync::Arc; + +/// A field's value, borrowed where possible. +/// +/// The point of the enum is that reading a field never forces an allocation: +/// the `&str`-keyed accessor had to return `Arc`, so `chno` and `type` +/// went through `.to_string().intern()` — a heap allocation *and* an interner +/// write lock — on every read, per item, per rule. +pub enum FieldRef<'a> { + /// An interned field. Cloning is a refcount bump. + Shared(&'a Arc), + /// A borrowed string that is not interned, e.g. `item_type.as_str()`. + Str(&'a str), + /// A numeric field, kept numeric. + Num(u32), +} + +impl FieldRef<'_> { + /// Borrow as a string, formatting a numeric field into an owned buffer only + /// when there is one. + pub fn as_cow(&self) -> std::borrow::Cow<'_, str> { + match self { + Self::Shared(value) => std::borrow::Cow::Borrowed(value.as_ref()), + Self::Str(value) => std::borrow::Cow::Borrowed(value), + Self::Num(value) => std::borrow::Cow::Owned(value.to_string()), + } + } + + /// Materialise as an interned `Arc`. + /// + /// This is what the `&str` compatibility shims call, so their behaviour — + /// including interning numbers — is bit-for-bit what it was before. + pub fn to_arc(&self) -> Arc { + match self { + Self::Shared(value) => Arc::clone(value), + Self::Str(value) => value.intern(), + Self::Num(value) => value.to_string().intern(), + } + } +} + +/// Read a field by typed key. Matches on a discriminant rather than walking a +/// chain of case-insensitive string comparisons. +pub trait FieldGet { + fn get(&self, field: crate::model::HeaderField) -> Option>; +} + +/// Write a field by typed key. +pub trait FieldSet { + fn set(&mut self, field: crate::model::HeaderField, value: &str) -> bool; +} + +/// Read a field by name. +/// +/// Retained for callers whose field name genuinely arrives as a string (the M3U +/// resource endpoint, for one). Implemented as a shim over [`FieldGet`]. +pub trait FieldGetAccessor { + fn get_field(&self, field: &str) -> Option>; +} + +/// Write a field by name. Shim over [`FieldSet`]. +pub trait FieldSetAccessor { + fn set_field(&mut self, field: &str, value: &str) -> bool; +} diff --git a/shared/src/model/playlist/group.rs b/shared/src/model/playlist/group.rs new file mode 100644 index 000000000..eeb81b53e --- /dev/null +++ b/shared/src/model/playlist/group.rs @@ -0,0 +1,32 @@ +use super::{PlaylistItem, XtreamCluster}; +use crate::utils::arc_str_serde; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PlaylistGroup { + pub id: u32, + #[serde(with = "arc_str_serde")] + pub title: Arc, + pub channels: Vec, + pub xtream_cluster: XtreamCluster, +} + +impl PlaylistGroup { + #[inline] + pub fn on_load(&mut self) { + for pl in &mut self.channels { + pl.header.freeze_input_stream_id(); + pl.header.gen_uuid(); + pl.header.category_id = self.id; + } + } + + #[inline] + pub fn filter_count(&self, filter: F) -> usize + where + F: Fn(&PlaylistItem) -> bool, + { + self.channels.iter().filter(|&c| filter(c)).count() + } +} diff --git a/shared/src/model/playlist/header.rs b/shared/src/model/playlist/header.rs new file mode 100644 index 000000000..e2ab44f17 --- /dev/null +++ b/shared/src/model/playlist/header.rs @@ -0,0 +1,221 @@ +use super::{FieldRef, PlaylistItemType, XtreamCluster}; +use crate::{ + model::{HeaderField, StreamProperties, UUIDType, VirtualId}, + utils::{arc_str_option_serde, arc_str_serde, generate_runtime_playlist_uuid, get_provider_id, Internable}, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PlaylistItemHeader { + #[serde(skip)] + pub uuid: UUIDType, // calculated + #[serde(with = "arc_str_serde")] + pub id: Arc, // provider id + #[serde(with = "arc_str_serde")] + pub name: Arc, + #[serde(with = "arc_str_serde")] + pub logo: Arc, + #[serde(with = "arc_str_serde")] + pub logo_small: Arc, + #[serde(with = "arc_str_serde")] + pub group: Arc, + #[serde(with = "arc_str_serde")] + pub title: Arc, + #[serde(with = "arc_str_serde")] + pub parent_code: Arc, + #[serde(with = "arc_str_serde")] + pub audio_track: Arc, + #[serde(with = "arc_str_serde")] + pub time_shift: Arc, + #[serde(with = "arc_str_serde")] + pub rec: Arc, + #[serde(with = "arc_str_serde")] + pub url: Arc, + #[serde(default, with = "arc_str_option_serde")] + pub epg_channel_id: Option>, + #[serde(with = "arc_str_serde")] + pub input_name: Arc, + #[serde(default)] + pub additional_properties: Option, + + // 4-byte aligned + pub virtual_id: VirtualId, // virtual id + pub chno: u32, + #[serde(default)] + pub category_id: u32, + #[serde(default)] + pub source_ordinal: u32, + + // 1-byte aligned + pub xtream_cluster: XtreamCluster, + #[serde(default)] + pub item_type: PlaylistItemType, + /// Stable provider/origin ID captured before any target transformation. + #[serde(default, with = "arc_str_serde")] + pub input_stream_id: Arc, + #[serde(default, rename = "source_user_agent", alias = "upstream_user_agent", with = "arc_str_option_serde")] + pub upstream_user_agent: Option>, +} + +impl Default for PlaylistItemHeader { + fn default() -> Self { + Self { + uuid: UUIDType::default(), + id: "".intern(), + virtual_id: VirtualId::default(), + name: "".intern(), + chno: 0, + logo: "".intern(), + logo_small: "".intern(), + group: "".intern(), + title: "".intern(), + parent_code: "".intern(), + audio_track: "".intern(), + time_shift: "".intern(), + rec: "".intern(), + url: "".intern(), + epg_channel_id: None, + xtream_cluster: XtreamCluster::default(), + additional_properties: None, + item_type: PlaylistItemType::default(), + category_id: 0, + input_name: "".intern(), + source_ordinal: 0, + input_stream_id: "".intern(), + upstream_user_agent: None, + } + } +} + +impl PlaylistItemHeader { + /// Captures the input playlist item ID at the input-processing boundary. + /// + /// This must run before target transformations and must not be used to recover an identity + /// from a target-mutated `id`. + pub fn freeze_input_stream_id(&mut self) { + if self.input_stream_id.is_empty() && !self.id.is_empty() { + self.input_stream_id = Arc::clone(&self.id); + } + } + + /// Returns the input stream ID captured at the input-processing boundary. + /// + /// Legacy fallback must be resolved before a target transformation starts. Once an item is + /// represented by this header, the mutable target-facing `id` is never a valid fallback. + pub fn get_input_stream_id(&self) -> Option> { + (!self.input_stream_id.is_empty()).then(|| Arc::clone(&self.input_stream_id)) + } + + #[inline] + pub fn gen_uuid(&mut self) { + self.uuid = generate_runtime_playlist_uuid(&self.input_name, &self.id, self.item_type, &self.url); + } + + #[inline] + pub const fn get_uuid(&self) -> &UUIDType { &self.uuid } + + pub fn get_provider_id(&mut self) -> Option { + match get_provider_id(&self.id, &self.url) { + None => None, + Some(newid) => { + self.id = newid.to_string().intern(); + Some(newid) + } + } + } + + #[inline] + pub fn get_name(&self) -> Arc { + if self.title.is_empty() { + Arc::clone(&self.name) + } else { + Arc::clone(&self.title) + } + } + + #[inline] + pub fn get_container_extension(&self) -> Option> { + self.additional_properties + .as_ref() + .and_then(super::super::stream_properties::StreamProperties::get_container_extension) + } +} + +impl crate::model::FieldGet for crate::model::PlaylistItemHeader { + fn get(&self, field: HeaderField) -> Option> { + match field { + HeaderField::Id => Some(FieldRef::Shared(&self.id)), + HeaderField::Title => Some(FieldRef::Shared(&self.title)), + HeaderField::Name => Some(FieldRef::Shared(&self.name)), + HeaderField::Logo => Some(FieldRef::Shared(&self.logo)), + HeaderField::LogoSmall => Some(FieldRef::Shared(&self.logo_small)), + HeaderField::ParentCode => Some(FieldRef::Shared(&self.parent_code)), + HeaderField::AudioTrack => Some(FieldRef::Shared(&self.audio_track)), + HeaderField::TimeShift => Some(FieldRef::Shared(&self.time_shift)), + HeaderField::Rec => Some(FieldRef::Shared(&self.rec)), + HeaderField::Url => Some(FieldRef::Shared(&self.url)), + HeaderField::Group => Some(FieldRef::Shared(&self.group)), + HeaderField::Caption => { + Some(FieldRef::Shared(if self.title.is_empty() { &self.name } else { &self.title })) + } + HeaderField::Input => Some(FieldRef::Shared(&self.input_name)), + HeaderField::Type => Some(FieldRef::Str(self.item_type.as_str())), + HeaderField::EpgChannelId => self.epg_channel_id.as_ref().map(FieldRef::Shared), + HeaderField::Chno => Some(FieldRef::Num(self.chno)), + HeaderField::Genre => { + self.additional_properties.as_ref().and_then(StreamProperties::genre).map(FieldRef::Shared) + } + // Not carried by the header. + HeaderField::ProviderId => None, + } + } +} + +impl crate::model::FieldSet for crate::model::PlaylistItemHeader { + fn set(&mut self, field: HeaderField, value: &str) -> bool { + match field { + HeaderField::Id => self.id = value.intern(), + HeaderField::Title => self.title = value.intern(), + HeaderField::Name => self.name = value.intern(), + HeaderField::Logo => self.logo = value.intern(), + HeaderField::LogoSmall => self.logo_small = value.intern(), + HeaderField::ParentCode => self.parent_code = value.intern(), + HeaderField::AudioTrack => self.audio_track = value.intern(), + HeaderField::TimeShift => self.time_shift = value.intern(), + HeaderField::Rec => self.rec = value.intern(), + HeaderField::Url => self.url = value.intern(), + HeaderField::Group => self.group = value.intern(), + HeaderField::Caption => { + let interned = value.intern(); + self.title = Arc::clone(&interned); + self.name = interned; + } + HeaderField::EpgChannelId => self.epg_channel_id = Some(value.intern()), + HeaderField::Chno => match value.parse::() { + Ok(parsed) => self.chno = parsed, + Err(_) => return false, + }, + HeaderField::Genre => return crate::set_genre!(self, value), + // Read-only, or not carried by the header. + HeaderField::Input | HeaderField::Type | HeaderField::ProviderId => return false, + } + true + } +} + +impl crate::model::FieldGetAccessor for crate::model::PlaylistItemHeader { + #[inline] + fn get_field(&self, field: &str) -> Option> { + use crate::model::FieldGet; + self.get(HeaderField::parse(field)?).map(|value| value.to_arc()) + } +} + +impl crate::model::FieldSetAccessor for crate::model::PlaylistItemHeader { + #[inline] + fn set_field(&mut self, field: &str, value: &str) -> bool { + use crate::model::FieldSet; + HeaderField::parse(field).is_some_and(|field| self.set(field, value)) + } +} diff --git a/shared/src/model/playlist/item.rs b/shared/src/model/playlist/item.rs new file mode 100644 index 000000000..a6e5c9dc6 --- /dev/null +++ b/shared/src/model/playlist/item.rs @@ -0,0 +1,439 @@ +use super::{ + M3uPlaylistItem, PlaylistItemHeader, PlaylistItemType, XtreamCluster, XtreamMappingOptions, XtreamPlaylistItem, +}; +use crate::{ + model::{ + stalker::StalkerStreamKind, stalker_item::StalkerPlaylistItem, CommonPlaylistItem, EpisodeStreamProperties, + SeriesStreamProperties, StreamProperties, UUIDType, VideoStreamProperties, VirtualId, XtreamInfoDocument, + }, + utils::{ + extract_extension_from_url, generate_provider_playlist_uuid, generate_runtime_playlist_uuid, get_provider_id, + Internable, + }, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +pub trait PlaylistEntry: Send + Sync { + fn get_virtual_id(&self) -> VirtualId; + /// Returns the immutable stream identifier captured from the input playlist before target mappings. + fn get_input_stream_id(&self) -> Option>; + fn get_provider_id(&self) -> Option; + fn get_category_id(&self) -> Option; + fn get_provider_url(&self) -> Arc; + fn get_uuid(&self) -> UUIDType; + fn get_item_type(&self) -> PlaylistItemType; + fn get_group(&self) -> Arc; + fn get_name(&self) -> Arc; + fn get_resolved_info_document(&self, options: &XtreamMappingOptions) -> Option; + fn get_additional_properties(&self) -> Option<&StreamProperties>; + fn get_additional_properties_mut(&mut self) -> Option<&mut StreamProperties>; + fn get_upstream_user_agent(&self) -> Option<&str>; +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PlaylistItem { + #[serde(flatten)] + pub header: PlaylistItemHeader, +} + +impl PlaylistItem { + pub(super) fn get_additional_properties(header: &PlaylistItemHeader) -> Option { + match &header.additional_properties { + Some(props) => Some(props.clone()), + None => { + match header.xtream_cluster { + XtreamCluster::Live => None, + XtreamCluster::Video => { + let container_extension = extract_extension_from_url(&header.url) + .map(|e| e.strip_prefix('.').unwrap_or(e).to_string()) + .unwrap_or_default(); + Some(StreamProperties::Video(Box::new(VideoStreamProperties { + name: header.name.clone(), + category_id: header.category_id, + stream_id: header.virtual_id.get(), + stream_icon: "".intern(), + direct_source: "".intern(), + custom_sid: None, + added: "".intern(), + container_extension: container_extension.intern(), + rating: None, + rating_5based: None, + stream_type: Some("movie".intern()), + trailer: None, + tmdb: None, + is_adult: 0, + details: None, + }))) + } + XtreamCluster::Series => { + if header.item_type == PlaylistItemType::Series { + let container_extension = extract_extension_from_url(&header.url) + .map(|e| e.strip_prefix('.').unwrap_or(e).to_string()) + .unwrap_or_default(); + // TODO maybe from link ? like s01e02 or something like this + Some(StreamProperties::Episode(Box::new(EpisodeStreamProperties { + episode_id: 0, + episode: 0, + season: 0, + added: None, + release_date: None, + series_release_date: None, + plot: None, + tmdb: None, + movie_image: "".intern(), + container_extension: container_extension.intern(), + audio: None, + video: None, + }))) + } else if header.item_type == PlaylistItemType::SeriesInfo { + Some(StreamProperties::Series(Box::new(SeriesStreamProperties { + name: header.name.clone(), + category_id: header.category_id, + tmdb: None, + series_id: 0, + backdrop_path: None, + cast: "".intern(), + cover: "".intern(), + director: "".intern(), + episode_run_time: None, + genre: None, + last_modified: None, + plot: None, + rating: 0.0, + rating_5based: 0.0, + release_date: None, + youtube_trailer: "".intern(), + details: None, + }))) + } else { + None + } + } + } + } + } + } + + pub fn has_details(&self) -> bool { + self.header + .additional_properties + .as_ref() + .is_some_and(super::super::stream_properties::StreamProperties::has_details) + } + + pub fn get_tmdb_id(&self) -> Option { + self.header + .additional_properties + .as_ref() + .and_then(super::super::stream_properties::StreamProperties::get_tmdb_id) + } +} + +impl From<&PlaylistItem> for XtreamPlaylistItem { + fn from(item: &PlaylistItem) -> Self { + let header = &item.header; + let input_stream_id = Arc::clone(&header.input_stream_id); + let missing_live_input_identity = + input_stream_id.is_empty() && (header.item_type.is_live() || header.item_type == PlaylistItemType::Catchup); + let provider_id = + if missing_live_input_identity { 0 } else { get_provider_id(&header.id, &header.url).unwrap_or_default() }; + let additional_properties = PlaylistItem::get_additional_properties(header); + + XtreamPlaylistItem { + virtual_id: header.virtual_id, + provider_id, + name: if header.item_type == PlaylistItemType::Series { + Arc::clone(&header.title) + } else { + Arc::clone(&header.name) + }, + logo: Arc::clone(&header.logo), + logo_small: Arc::clone(&header.logo_small), + group: Arc::clone(&header.group), + title: Arc::clone(&header.title), + parent_code: Arc::clone(&header.parent_code), + rec: Arc::clone(&header.rec), + url: Arc::clone(&header.url), + epg_channel_id: header.epg_channel_id.clone(), + xtream_cluster: header.xtream_cluster, + additional_properties, + item_type: header.item_type, + category_id: header.category_id, + input_name: Arc::clone(&header.input_name), + channel_no: header.chno, + source_ordinal: header.source_ordinal, + input_stream_id, + upstream_user_agent: header.upstream_user_agent.clone(), + } + } +} + +impl From<&PlaylistItem> for M3uPlaylistItem { + fn from(item: &PlaylistItem) -> Self { + let header = &item.header; + let input_stream_id = Arc::clone(&header.input_stream_id); + let missing_live_input_identity = + input_stream_id.is_empty() && (header.item_type.is_live() || header.item_type == PlaylistItemType::Catchup); + M3uPlaylistItem { + virtual_id: header.virtual_id, + provider_id: if missing_live_input_identity { "".intern() } else { Arc::clone(&header.id) }, + name: if header.item_type == PlaylistItemType::Series { + Arc::clone(&header.title) + } else { + Arc::clone(&header.name) + }, + chno: header.chno, + logo: Arc::clone(&header.logo), + logo_small: Arc::clone(&header.logo_small), + group: Arc::clone(&header.group), + title: Arc::clone(&header.title), + parent_code: Arc::clone(&header.parent_code), + audio_track: Arc::clone(&header.audio_track), + time_shift: Arc::clone(&header.time_shift), + rec: Arc::clone(&header.rec), + url: Arc::clone(&header.url), + epg_channel_id: header.epg_channel_id.clone(), + input_name: Arc::clone(&header.input_name), + item_type: header.item_type, + t_stream_url: Arc::clone(&header.url), + t_resource_url: None, + t_catchup_source: None, + t_catchup_mode: None, + source_ordinal: header.source_ordinal, + additional_properties: header.additional_properties.clone(), + input_stream_id, + upstream_user_agent: header.upstream_user_agent.clone(), + } + } +} + +impl From<&PlaylistItem> for CommonPlaylistItem { + fn from(item: &PlaylistItem) -> Self { + let header = &item.header; + + let additional_properties = PlaylistItem::get_additional_properties(header); + + CommonPlaylistItem { + virtual_id: header.virtual_id, + provider_id: Arc::clone(&header.id), + name: if header.item_type == PlaylistItemType::Series { + Arc::clone(&header.title) + } else { + Arc::clone(&header.name) + }, + logo: header.logo.clone(), + logo_small: header.logo_small.clone(), + group: Arc::clone(&header.group), + title: header.title.clone(), + parent_code: header.parent_code.clone(), + audio_track: header.audio_track.clone(), + time_shift: header.time_shift.clone(), + rec: header.rec.clone(), + url: header.url.clone(), + epg_channel_id: header.epg_channel_id.clone(), + xtream_cluster: Some(header.xtream_cluster), + additional_properties, + item_type: header.item_type, + category_id: Some(header.category_id), + input_name: Arc::clone(&header.input_name), + chno: header.chno, + } + } +} + +impl From<&XtreamPlaylistItem> for PlaylistItem { + fn from(item: &XtreamPlaylistItem) -> Self { + let input_stream_id = item.get_input_stream_id(); + let header = PlaylistItemHeader { + uuid: item.get_uuid(), + virtual_id: item.virtual_id, + id: if item.provider_id == 0 && input_stream_id.is_none() { + "".intern() + } else { + item.provider_id.to_string().intern() + }, + name: item.name.clone(), + title: item.title.clone(), + logo: item.logo.clone(), + logo_small: item.logo_small.clone(), + group: item.group.clone(), + parent_code: item.parent_code.clone(), + rec: item.rec.clone(), + url: item.url.clone(), + epg_channel_id: item.epg_channel_id.clone(), + xtream_cluster: item.xtream_cluster, + item_type: item.item_type, + category_id: item.category_id, + input_name: item.input_name.clone(), + chno: item.channel_no, + audio_track: "".intern(), + time_shift: "".intern(), + additional_properties: item.additional_properties.clone(), + source_ordinal: item.source_ordinal, + input_stream_id: input_stream_id.unwrap_or_else(|| "".intern()), + upstream_user_agent: item.upstream_user_agent.clone(), + }; + + PlaylistItem { header } + } +} + +impl From<&M3uPlaylistItem> for PlaylistItem { + fn from(item: &M3uPlaylistItem) -> Self { + let header = PlaylistItemHeader { + uuid: item.get_uuid(), + virtual_id: item.virtual_id, + id: item.provider_id.clone(), + name: item.name.clone(), + title: item.title.clone(), + logo: item.logo.clone(), + logo_small: item.logo_small.clone(), + group: item.group.clone(), + parent_code: item.parent_code.clone(), + rec: item.rec.clone(), + url: item.url.clone(), + epg_channel_id: item.epg_channel_id.clone(), + xtream_cluster: item.item_type.cluster(), + item_type: item.item_type, + category_id: 0, + input_name: item.input_name.clone(), + chno: item.chno, + audio_track: item.audio_track.clone(), + time_shift: item.time_shift.clone(), + additional_properties: item.additional_properties.clone(), + source_ordinal: item.source_ordinal, + input_stream_id: item.get_input_stream_id().unwrap_or_else(|| "".intern()), + upstream_user_agent: item.upstream_user_agent.clone(), + }; + + PlaylistItem { header } + } +} + +impl PlaylistItem { + /// Canonical `StalkerPlaylistItem` → `PlaylistItem` conversion. + /// + /// This is the single conversion used by both the download path + /// (processor) and the disk-load path so the generated identity is stable + /// across processing modes: the UUID is always seeded with the owning + /// input name and the `input_name` header field is always populated. + /// The group falls back to a cluster default only when the portal did + /// not supply a category name. + pub fn from_stalker(item: &StalkerPlaylistItem, input_name: &str) -> Self { + let item_type = match item.stream_kind { + StalkerStreamKind::Live | StalkerStreamKind::Archive => PlaylistItemType::Live, + StalkerStreamKind::Movie => PlaylistItemType::Video, + StalkerStreamKind::Episode => { + if item.is_series_root() { + PlaylistItemType::SeriesInfo + } else { + PlaylistItemType::Series + } + } + }; + let xtream_cluster = match item.stream_kind { + StalkerStreamKind::Live | StalkerStreamKind::Archive => XtreamCluster::Live, + StalkerStreamKind::Movie => XtreamCluster::Video, + StalkerStreamKind::Episode => XtreamCluster::Series, + }; + let stream_id_str: Arc = Internable::intern(item.stream_id.to_string()); + let logo: Arc = item.logo_url.clone().unwrap_or_else(|| Internable::intern(String::new())); + // Keep unresolved commands private; playback resolves them from the persisted Stalker metadata. + let url = Arc::clone(&item.stream_url); + let group: Arc = if item.category_name.is_empty() { + let fallback = match item.stream_kind { + StalkerStreamKind::Live | StalkerStreamKind::Archive => "Live", + StalkerStreamKind::Movie => "Movies", + StalkerStreamKind::Episode => "Series", + }; + Internable::intern(fallback.to_string()) + } else { + Arc::clone(&item.category_name) + }; + let header = PlaylistItemHeader { + uuid: generate_provider_playlist_uuid(input_name, &stream_id_str, item_type), + virtual_id: VirtualId::new(item.stream_id), + id: Arc::clone(&stream_id_str), + name: Arc::clone(&item.name), + title: Arc::clone(&item.name), + logo: Arc::clone(&logo), + logo_small: Internable::intern(String::new()), + group, + parent_code: Internable::intern(String::new()), + audio_track: Internable::intern(String::new()), + time_shift: Internable::intern(String::new()), + rec: Internable::intern(String::new()), + url, + epg_channel_id: item.epg_channel_id.clone(), + item_type, + xtream_cluster, + additional_properties: None, + input_name: Internable::intern(input_name.to_string()), + chno: item.number, + category_id: item.category_id, + source_ordinal: 0, + input_stream_id: stream_id_str, + upstream_user_agent: None, + }; + + PlaylistItem { header } + } +} + +impl PlaylistEntry for PlaylistItem { + #[inline] + fn get_virtual_id(&self) -> VirtualId { self.header.virtual_id } + + #[inline] + fn get_input_stream_id(&self) -> Option> { self.header.get_input_stream_id() } + + fn get_upstream_user_agent(&self) -> Option<&str> { self.header.upstream_user_agent.as_deref() } + + fn get_provider_id(&self) -> Option { + let header = &self.header; + get_provider_id(&header.id, &header.url) + } + + #[inline] + fn get_category_id(&self) -> Option { Some(self.header.category_id) } + + #[inline] + fn get_provider_url(&self) -> Arc { Arc::clone(&self.header.url) } + + #[inline] + fn get_uuid(&self) -> UUIDType { + let header = &self.header; + generate_runtime_playlist_uuid(&header.input_name, &header.id, header.item_type, &header.url) + } + + #[inline] + fn get_item_type(&self) -> PlaylistItemType { self.header.item_type } + + #[inline] + fn get_group(&self) -> Arc { Arc::clone(&self.header.group) } + + #[inline] + fn get_name(&self) -> Arc { self.header.get_name() } + + fn get_resolved_info_document(&self, options: &XtreamMappingOptions) -> Option { + if self.has_details() { + self.header.additional_properties.as_ref().map(|p| { + p.to_info_document( + options, + self.get_item_type(), + self.get_virtual_id(), + self.get_category_id().unwrap_or(0), + ) + }) + } else { + None + } + } + + fn get_additional_properties(&self) -> Option<&StreamProperties> { self.header.additional_properties.as_ref() } + #[inline] + fn get_additional_properties_mut(&mut self) -> Option<&mut StreamProperties> { + self.header.additional_properties.as_mut() + } +} diff --git a/shared/src/model/playlist/kind.rs b/shared/src/model/playlist/kind.rs new file mode 100644 index 000000000..0d7746097 --- /dev/null +++ b/shared/src/model/playlist/kind.rs @@ -0,0 +1,261 @@ +use super::PlaylistItemTypeSet; +use crate::{error::TuliproxError, model::xtream_const, utils::Internable}; +use serde::{Deserialize, Serialize}; +use std::{ + fmt::{Display, Formatter}, + str::FromStr, + sync::Arc, +}; +use strum_macros::{AsRefStr, Display, EnumIter, EnumString}; + +// https://de.wikipedia.org/wiki/M3U +// https://siptv.eu/howto/playlist.html + +#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq, Serialize, Deserialize, Default, Display, EnumString, AsRefStr)] +#[repr(u8)] +pub enum XtreamCluster { + #[default] + #[strum(serialize = "Live", serialize = "live")] + Live = 1, + + #[strum(serialize = "Video", serialize = "movie", serialize = "vod", serialize = "video")] + Video = 2, + + #[strum(serialize = "Series", serialize = "series")] + Series = 3, +} + +impl XtreamCluster { + pub fn as_str(&self) -> &str { self.as_ref() } + + /// True when this cluster is the Xtream `Series` cluster. Used in + /// bucket-key computations and dispatch sites that previously + /// spelled out `== XtreamCluster::Series` inline. + pub fn is_series(self) -> bool { matches!(self, Self::Series) } + + pub fn as_stream_type(&self) -> &str { + match self { + Self::Live => "live", + Self::Video => "movie", + Self::Series => "series", + } + } + + /// Returns the xtream `player_api` info action and the stream-id query field for this cluster. + /// + /// Keeps the per-cluster `(action, id_field)` mapping attached to the enum so call sites read a + /// single property instead of re-deriving it with a local `match`. + pub fn info_action_and_id_field(&self) -> (&'static str, &'static str) { + match self { + Self::Live => (xtream_const::XC_ACTION_GET_LIVE_INFO, xtream_const::XC_LIVE_ID), + Self::Video => (xtream_const::XC_ACTION_GET_VOD_INFO, xtream_const::XC_VOD_ID), + Self::Series => (xtream_const::XC_ACTION_GET_SERIES_INFO, xtream_const::XC_SERIES_ID), + } + } +} + +/// Every item type belongs to exactly one cluster, so this cannot fail. +/// +/// This used to be a `TryFrom` whose every arm returned `Ok`, and the phantom +/// error spread `.unwrap_or(Live)`, `.unwrap_or_default()` and `.ok()` across 17 +/// call sites in four crates. See [`PlaylistItemType::cluster`]. +impl From for XtreamCluster { + #[inline] + fn from(item_type: PlaylistItemType) -> Self { item_type.cluster() } +} + +#[derive(Debug, Copy, Clone, Eq, Hash, PartialEq, Serialize, Deserialize, Default, EnumIter)] +#[repr(u8)] +pub enum PlaylistItemType { + #[default] + #[serde(alias = "live")] + Live = 1, + #[serde(alias = "video")] + Video = 2, + #[serde(alias = "series")] + Series = 3, // xtream series description + #[serde(alias = "series_info")] + SeriesInfo = 4, // xtream series info fetched for series description + #[serde(alias = "catchup")] + Catchup = 5, + #[serde(alias = "live_unknown")] + LiveUnknown = 6, // No Provider id + #[serde(alias = "live_hls")] + LiveHls = 7, // m3u8 entry + #[serde(alias = "live_dash")] + LiveDash = 8, // mpd + #[serde(alias = "local_video")] + LocalVideo = 9, + #[serde(alias = "local_series")] + LocalSeries = 10, + #[serde(alias = "local_series_info")] + LocalSeriesInfo = 11, +} + +impl From for PlaylistItemType { + fn from(xtream_cluster: XtreamCluster) -> Self { + match xtream_cluster { + XtreamCluster::Live => Self::Live, + XtreamCluster::Video => Self::Video, + XtreamCluster::Series => Self::SeriesInfo, + } + } +} + +impl FromStr for PlaylistItemType { + type Err = TuliproxError; + + fn from_str(s: &str) -> Result { + match s { + "Live" => Ok(PlaylistItemType::Live), + "Video" => Ok(PlaylistItemType::Video), + "LocalVideo" => Ok(PlaylistItemType::LocalVideo), + "Series" => Ok(PlaylistItemType::Series), + "SeriesInfo" => Ok(PlaylistItemType::SeriesInfo), + "LocalSeries" => Ok(PlaylistItemType::LocalSeries), + "LocalSeriesInfo" => Ok(PlaylistItemType::LocalSeriesInfo), + "Catchup" => Ok(PlaylistItemType::Catchup), + "LiveUnknown" => Ok(PlaylistItemType::LiveUnknown), + "LiveHls" => Ok(PlaylistItemType::LiveHls), + "LiveDash" => Ok(PlaylistItemType::LiveDash), + _ => Err(TuliproxError::Config(format!("Invalid PlaylistItemType: {s}"))), + } + } +} + +impl PlaylistItemType { + pub(super) const LIVE: &'static str = "live"; + pub(super) const VIDEO: &'static str = "video"; + pub(super) const SERIES: &'static str = "series"; + pub(super) const SERIES_INFO: &'static str = "series-info"; + pub(super) const CATCHUP: &'static str = "catchup"; + + pub fn is_local(&self) -> bool { + matches!(self, PlaylistItemType::LocalVideo | PlaylistItemType::LocalSeries | PlaylistItemType::LocalSeriesInfo) + } + + pub fn is_live(&self) -> bool { + matches!( + self, + PlaylistItemType::Live + | PlaylistItemType::LiveDash + | PlaylistItemType::LiveHls + | PlaylistItemType::LiveUnknown + ) + } + + pub fn is_live_adaptive(&self) -> bool { matches!(self, PlaylistItemType::LiveHls | PlaylistItemType::LiveDash) } + + /// True for VOD item types (`Video` or `LocalVideo`). + pub fn is_video(&self) -> bool { matches!(self, PlaylistItemType::Video | PlaylistItemType::LocalVideo) } + + /// True for concrete series item types (`Series` or `LocalSeries`); excludes the `SeriesInfo` containers. + pub fn is_series(&self) -> bool { matches!(self, PlaylistItemType::Series | PlaylistItemType::LocalSeries) } + + /// Controls address tracking only. + /// Do not use this to decide whether a playback request should use session-based admission + /// or whether a logical playback must stay on the same provider account. + pub fn uses_socket_bound_session(&self) -> bool { + matches!(self, PlaylistItemType::Live | PlaylistItemType::LiveUnknown) + } + + /// Controls whether follow-up requests for the same logical playback must stay on the + /// same provider account. + /// This is separate from both session admission and socket binding. + pub fn requires_provider_affinity(&self) -> bool { + matches!( + self, + PlaylistItemType::LiveHls + | PlaylistItemType::LiveDash + | PlaylistItemType::Video + | PlaylistItemType::LocalVideo + | PlaylistItemType::Series + | PlaylistItemType::SeriesInfo + | PlaylistItemType::LocalSeries + | PlaylistItemType::LocalSeriesInfo + | PlaylistItemType::Catchup + ) + } + + pub fn as_u8(self) -> u8 { self as u8 } + + pub fn as_str(&self) -> &'static str { + match self { + Self::Live | Self::LiveHls | Self::LiveDash | Self::LiveUnknown => Self::LIVE, + Self::Video | Self::LocalVideo => Self::VIDEO, + Self::Series | Self::LocalSeries => Self::SERIES, + Self::SeriesInfo | Self::LocalSeriesInfo => Self::SERIES_INFO, + Self::Catchup => Self::CATCHUP, + } + } + + /// Returns a cached interned `Arc` of this type's label. + /// + /// `intern()` performs an interner hash-map lookup on every call. Because the + /// label is one of only five fixed values, this caches the interned `Arc` per + /// label in a `OnceLock` and returns a cheap `Arc::clone`, avoiding repeated + /// interner lookups on hot sort/filter paths. + pub fn interned_label(&self) -> Arc { + static CACHE: [std::sync::OnceLock>; 5] = [const { std::sync::OnceLock::new() }; 5]; + let idx = match self { + Self::Live | Self::LiveHls | Self::LiveDash | Self::LiveUnknown => 0, + Self::Video | Self::LocalVideo => 1, + Self::Series | Self::LocalSeries => 2, + Self::SeriesInfo | Self::LocalSeriesInfo => 3, + Self::Catchup => 4, + }; + Arc::clone(CACHE[idx].get_or_init(|| self.as_str().intern())) + } + + /// The cluster this item type belongs to. + /// + /// The one place the item-type-to-cluster relation is written down. It used + /// to be encoded twice -- here and in a `TryFrom` impl -- with nothing + /// keeping the two in agreement. + #[inline] + pub const fn cluster(self) -> XtreamCluster { + match self { + Self::Live | Self::LiveHls | Self::LiveDash | Self::LiveUnknown => XtreamCluster::Live, + Self::Catchup | Self::Video | Self::LocalVideo => XtreamCluster::Video, + Self::Series | Self::LocalSeries | Self::SeriesInfo | Self::LocalSeriesInfo => XtreamCluster::Series, + } + } + + #[inline] + pub const fn is_cluster(&self, cluster: XtreamCluster) -> bool { self.cluster() as u8 == cluster as u8 } +} + +impl Display for PlaylistItemType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.as_str()) } +} + +impl Internable for PlaylistItemType { + fn intern(self) -> Arc { self.as_str().intern() } +} + +impl Internable for XtreamCluster { + fn intern(self) -> Arc { self.as_str().intern() } +} + +impl PlaylistItemTypeSet { + #[inline] + pub fn empty() -> Self { Self(0) } + + #[inline] + pub fn from_item(item: PlaylistItemType) -> Self { + let bit = 1u16 << ((item as u8) - 1); + Self(bit) + } + + #[inline] + pub fn insert(&mut self, item: PlaylistItemType) { self.0 |= 1u16 << ((item as u8) - 1); } + + #[inline] + pub fn remove(&mut self, item: PlaylistItemType) { self.0 &= !(1u16 << ((item as u8) - 1)); } + + #[inline] + pub fn is_set(&self, item: PlaylistItemType) -> bool { (self.0 & (1u16 << ((item as u8) - 1))) != 0 } + + #[inline] + pub fn bits(self) -> u16 { self.0 } +} diff --git a/shared/src/model/playlist/m3u.rs b/shared/src/model/playlist/m3u.rs new file mode 100644 index 000000000..53150affc --- /dev/null +++ b/shared/src/model/playlist/m3u.rs @@ -0,0 +1,325 @@ +use super::{FieldRef, PlaylistEntry, PlaylistItemType, XtreamMappingOptions}; +use crate::{ + model::{ + CatchupAttribute, CatchupProperties, CommonPlaylistItem, ConfigTargetOptions, HeaderField, StreamProperties, + UUIDType, VirtualId, XtreamInfoDocument, + }, + utils::{arc_str_option_serde, arc_str_serde, generate_runtime_playlist_uuid, get_provider_id}, +}; +use serde::{Deserialize, Serialize}; +use std::{fmt::Write, sync::Arc}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct M3uPlaylistItem { + pub virtual_id: VirtualId, + #[serde(with = "arc_str_serde")] + pub provider_id: Arc, + #[serde(with = "arc_str_serde")] + pub name: Arc, + pub chno: u32, + #[serde(with = "arc_str_serde")] + pub logo: Arc, + #[serde(with = "arc_str_serde")] + pub logo_small: Arc, + #[serde(with = "arc_str_serde")] + pub group: Arc, + #[serde(with = "arc_str_serde")] + pub title: Arc, + #[serde(with = "arc_str_serde")] + pub parent_code: Arc, + #[serde(with = "arc_str_serde")] + pub audio_track: Arc, + #[serde(with = "arc_str_serde")] + pub time_shift: Arc, + #[serde(with = "arc_str_serde")] + pub rec: Arc, + #[serde(with = "arc_str_serde")] + pub url: Arc, + #[serde(default, with = "arc_str_option_serde")] + pub epg_channel_id: Option>, + #[serde(with = "arc_str_serde")] + pub input_name: Arc, + pub item_type: PlaylistItemType, + #[serde(skip)] + pub t_stream_url: Arc, + #[serde(skip)] + pub t_resource_url: Option, + #[serde(skip)] + pub t_catchup_source: Option>, + #[serde(skip)] + pub t_catchup_mode: Option>, + #[serde(default)] + pub source_ordinal: u32, + #[serde(default)] + pub additional_properties: Option, + /// Stable provider/origin ID captured before any target transformation. + #[serde(default, with = "arc_str_serde")] + pub input_stream_id: Arc, + #[serde(default, rename = "source_user_agent", alias = "upstream_user_agent", with = "arc_str_option_serde")] + pub upstream_user_agent: Option>, +} + +fn write_m3u_attr(line: &mut String, name: &str, value: &str) { let _ = write!(line, " {name}=\"{value}\""); } + +fn append_catchup_attribute(line: &mut String, name: &str, value: Option<&Arc>) { + if let Some(value) = value.filter(|value| !value.is_empty()) { + write_m3u_attr(line, name, value); + } +} + +fn append_extra_catchup_attributes(line: &mut String, attributes: &[CatchupAttribute]) { + for attribute in attributes { + if !attribute.name.is_empty() && !attribute.value.is_empty() { + write_m3u_attr(line, attribute.name.as_ref(), attribute.value.as_ref()); + } + } +} + +fn append_m3u_catchup_attributes( + line: &mut String, + catchup: &CatchupProperties, + rewritten_mode: Option<&Arc>, + rewritten_source: Option<&Arc>, +) { + if let Some(mode) = rewritten_mode.filter(|mode| !mode.is_empty()) { + write_m3u_attr(line, "catchup", mode.as_ref()); + } else { + append_catchup_attribute(line, "catchup", catchup.mode.as_ref()); + } + append_catchup_attribute(line, "catchup-days", catchup.days.as_ref()); + if let Some(source) = rewritten_source.filter(|source| !source.is_empty()) { + write_m3u_attr(line, "catchup-source", source.as_ref()); + } else { + append_catchup_attribute(line, "catchup-source", catchup.source.as_ref()); + } + append_catchup_attribute(line, "catchup-time", catchup.time.as_ref()); + append_catchup_attribute(line, "catchup-correction", catchup.correction.as_ref()); + if !catchup.is_flussonic() || rewritten_mode.is_none_or(|mode| mode.is_empty()) { + append_catchup_attribute(line, "catchup-type", catchup.catchup_type.as_ref()); + } + append_extra_catchup_attributes(line, &catchup.extra_attributes); +} + +fn append_unified_catchup_type_attributes( + line: &mut String, + catchup: &CatchupProperties, + catchup_type: &str, + rewritten_source: Option<&Arc>, + emit_source: bool, +) { + // Unified export: always `catchup-type=...`, never `catchup=`. + write_m3u_attr(line, "catchup-type", catchup_type); + append_catchup_attribute(line, "catchup-days", catchup.days.as_ref()); + if emit_source { + if let Some(source) = rewritten_source.filter(|source| !source.is_empty()) { + write_m3u_attr(line, "catchup-source", source.as_ref()); + } else { + append_catchup_attribute(line, "catchup-source", catchup.source.as_ref()); + } + } + append_catchup_attribute(line, "catchup-time", catchup.time.as_ref()); + append_catchup_attribute(line, "catchup-correction", catchup.correction.as_ref()); + append_extra_catchup_attributes(line, &catchup.extra_attributes); +} + +impl M3uPlaylistItem { + pub fn to_m3u(&self, target_options: Option<&ConfigTargetOptions>, rewrite_urls: bool) -> String { + let options = target_options.as_ref(); + let ignore_logo = options.is_some_and(|o| o.ignore_logo); + let mut line = String::with_capacity(256); + let _ = write!( + &mut line, + "#EXTINF:-1 tvg-id=\"{}\" tvg-name=\"{}\" group-title=\"{}\"", + self.epg_channel_id.as_ref().map_or("", |o| o.as_ref()), + self.name, + self.group + ); + + if !ignore_logo { + if let (true, Some(resource_url)) = (rewrite_urls, self.t_resource_url.as_ref()) { + to_m3u_resource_non_empty_fields!(self, resource_url, line, (logo, "tvg-logo"), (logo_small, "tvg-logo-small");); + } else { + to_m3u_non_empty_fields!(self, line, (logo, "tvg-logo"), (logo_small, "tvg-logo-small");); + } + } + + if self.chno != 0 { + let _ = write!(line, " tvg-chno=\"{}\"", self.chno); + } + let flussonic_mode = self.additional_properties.as_ref().and_then(|props| match props { + StreamProperties::Live(live) => { + live.catchup.as_ref().and_then(CatchupProperties::native_flussonic_player_mode) + } + _ => None, + }); + let append_type = self.additional_properties.as_ref().and_then(|props| match props { + StreamProperties::Live(live) => live.catchup.as_ref().and_then(CatchupProperties::append_player_type), + _ => None, + }); + // Emit timeshift only when the source had it (non-empty). Never invent catchup attrs. + to_m3u_non_empty_fields!(self, line, + (parent_code, "parent-code"), + (audio_track, "audio-track"), + (time_shift, "timeshift"), + (rec, "tvg-rec");); + if let Some(StreamProperties::Live(live)) = self.additional_properties.as_ref() { + if let Some(catchup) = live.catchup.as_ref() { + let has_rewritten_catchup = self.t_catchup_mode.as_ref().is_some_and(|mode| !mode.is_empty()) + || self.t_catchup_source.as_ref().is_some_and(|source| !source.is_empty()); + if has_rewritten_catchup { + append_m3u_catchup_attributes( + &mut line, + catchup, + self.t_catchup_mode.as_ref(), + self.t_catchup_source.as_ref(), + ); + } else if let Some(mode) = flussonic_mode { + // Unify catchup=/catchup-type=flussonic* to catchup-type only. + // No catchup=, no shift-style catchup-source, no invented days. + append_unified_catchup_type_attributes(&mut line, catchup, mode, None, false); + } else if let Some(append_type) = append_type { + // Unify catchup=/catchup-type=append to catchup-type="append" only. + append_unified_catchup_type_attributes( + &mut line, + catchup, + append_type, + self.t_catchup_source.as_ref(), + true, + ); + } else { + append_m3u_catchup_attributes( + &mut line, + catchup, + self.t_catchup_mode.as_ref(), + self.t_catchup_source.as_ref(), + ); + } + } + } + + let _ = writeln!(&mut line, ",{}", self.title); + if let Some(user_agent) = + self.upstream_user_agent.as_deref().filter(|value| !value.is_empty() && !value.contains(['\r', '\n'])) + { + let _ = writeln!(&mut line, "#EXTVLCOPT:http-user-agent={user_agent}"); + } + let url = if self.t_stream_url.is_empty() { &self.url } else { &self.t_stream_url }; + line.push_str(url); + line + } + + pub fn to_common(&self) -> CommonPlaylistItem { + CommonPlaylistItem { + virtual_id: self.virtual_id, + provider_id: Arc::clone(&self.provider_id), + name: Arc::clone(&self.name), + chno: self.chno, + logo: Arc::clone(&self.logo), + logo_small: Arc::clone(&self.logo_small), + group: Arc::clone(&self.group), + title: Arc::clone(&self.title), + parent_code: Arc::clone(&self.parent_code), + audio_track: Arc::clone(&self.audio_track), + time_shift: Arc::clone(&self.time_shift), + rec: Arc::clone(&self.rec), + url: Arc::clone(&self.url), + input_name: Arc::clone(&self.input_name), + item_type: self.item_type, + epg_channel_id: self.epg_channel_id.clone(), + xtream_cluster: Some(self.item_type.cluster()), + additional_properties: self.additional_properties.clone(), + category_id: None, + } + } +} + +impl PlaylistEntry for M3uPlaylistItem { + #[inline] + fn get_virtual_id(&self) -> VirtualId { self.virtual_id } + + fn get_input_stream_id(&self) -> Option> { + let input_stream_id = if self.input_stream_id.is_empty() { &self.provider_id } else { &self.input_stream_id }; + (!input_stream_id.is_empty()).then(|| Arc::clone(input_stream_id)) + } + + fn get_upstream_user_agent(&self) -> Option<&str> { self.upstream_user_agent.as_deref() } + + fn get_provider_id(&self) -> Option { get_provider_id(&self.provider_id, &self.url) } + #[inline] + fn get_category_id(&self) -> Option { None } + #[inline] + fn get_provider_url(&self) -> Arc { Arc::clone(&self.url) } + + fn get_uuid(&self) -> UUIDType { + generate_runtime_playlist_uuid(&self.input_name, &self.provider_id, self.item_type, &self.url) + } + + #[inline] + fn get_item_type(&self) -> PlaylistItemType { self.item_type } + + #[inline] + fn get_group(&self) -> Arc { Arc::clone(&self.group) } + + #[inline] + fn get_name(&self) -> Arc { + if self.title.is_empty() { + Arc::clone(&self.name) + } else { + Arc::clone(&self.title) + } + } + + #[inline] + fn get_resolved_info_document(&self, _options: &XtreamMappingOptions) -> Option { None } + #[inline] + fn get_additional_properties(&self) -> Option<&StreamProperties> { self.additional_properties.as_ref() } + #[inline] + fn get_additional_properties_mut(&mut self) -> Option<&mut StreamProperties> { self.additional_properties.as_mut() } +} + +impl crate::model::FieldGet for M3uPlaylistItem { + fn get(&self, field: HeaderField) -> Option> { + match field { + HeaderField::ProviderId => Some(FieldRef::Shared(&self.provider_id)), + HeaderField::Title => Some(FieldRef::Shared(&self.title)), + HeaderField::Name => Some(FieldRef::Shared(&self.name)), + HeaderField::Logo => Some(FieldRef::Shared(&self.logo)), + HeaderField::LogoSmall => Some(FieldRef::Shared(&self.logo_small)), + HeaderField::ParentCode => Some(FieldRef::Shared(&self.parent_code)), + HeaderField::AudioTrack => Some(FieldRef::Shared(&self.audio_track)), + HeaderField::TimeShift => Some(FieldRef::Shared(&self.time_shift)), + HeaderField::Rec => Some(FieldRef::Shared(&self.rec)), + HeaderField::Url => Some(FieldRef::Shared(&self.url)), + HeaderField::Group => Some(FieldRef::Shared(&self.group)), + HeaderField::Caption => { + Some(FieldRef::Shared(if self.title.is_empty() { &self.name } else { &self.title })) + } + HeaderField::EpgChannelId => self.epg_channel_id.as_ref().map(FieldRef::Shared), + HeaderField::Chno => Some(FieldRef::Num(self.chno)), + // Deliberately not addressable by name on an M3U item, even though the + // struct carries input_name, item_type and additional_properties. The + // M3U resource endpoint resolves a URL path segment through + // `get_field`, so making these resolvable would turn a 404 into a + // response. Preserved from the string-keyed accessor this replaces. + HeaderField::Id | HeaderField::Input | HeaderField::Type | HeaderField::Genre => None, + } + } +} + +impl crate::model::FieldGetAccessor for M3uPlaylistItem { + #[inline] + fn get_field(&self, field: &str) -> Option> { + use crate::model::FieldGet; + let field = HeaderField::parse(field)?; + let value = self.get(field)?.to_arc(); + Some(if matches!(field, HeaderField::Logo | HeaderField::LogoSmall) { + super::super::persisted_resource_arc(&value) + } else { + value + }) + } +} + +impl From for CommonPlaylistItem { + fn from(item: M3uPlaylistItem) -> Self { item.to_common() } +} diff --git a/shared/src/model/playlist/tests.rs b/shared/src/model/playlist/tests.rs new file mode 100644 index 000000000..f42a83ce9 --- /dev/null +++ b/shared/src/model/playlist/tests.rs @@ -0,0 +1,30 @@ +use super::*; +use crate::{ + model::{ + stalker::StalkerStreamKind, stalker_item::StalkerPlaylistItem, CatchupAttribute, CatchupProperties, + HeaderField, LiveStreamProperties, PlaylistItemType, StreamProperties, VirtualId, XtreamCluster, + XtreamMappingFlags, + }, + utils::{concat_path, seal_web_ui_resource_url, Internable}, +}; +use std::sync::Arc; + +fn sample_options() -> XtreamMappingOptions { + XtreamMappingOptions { + base_url: "/api/v1/playlist/resource".to_string(), + username: "user".to_string(), + password: "pass".to_string(), + force_redirect: None, + reverse_item_types: PlaylistItemTypeSet::from_item(PlaylistItemType::Live), + resource_proxy_item_types: PlaylistItemTypeSet::from_item(PlaylistItemType::Live), + web_ui_request: true, + flags: XtreamMappingFlags::RewriteResourceUrl.into(), + encrypt_secret: [3u8; 16], + resource_host_policies: Arc::default(), + } +} + +mod groups; +mod identity; +mod m3u; +mod xtream; diff --git a/shared/src/model/playlist/tests/groups.rs b/shared/src/model/playlist/tests/groups.rs new file mode 100644 index 000000000..1bac8be11 --- /dev/null +++ b/shared/src/model/playlist/tests/groups.rs @@ -0,0 +1,26 @@ +use super::*; + +#[test] +fn from_stalker_populates_input_name_and_group() { + let item = StalkerPlaylistItem { + stream_id: 7, + stream_kind: StalkerStreamKind::Movie, + category_name: Internable::intern("Action".to_string()), + cmd: Internable::intern("ffmpeg http://streams.example/movie/7".to_string()), + ..StalkerPlaylistItem::default() + }; + let converted = PlaylistItem::from_stalker(&item, "my_input"); + assert_eq!(&*converted.header.input_name, "my_input"); + assert_eq!(&*converted.header.input_stream_id, "7"); + assert!(converted.header.url.is_empty()); + assert_eq!(&*converted.header.group, "Action"); + assert_eq!(converted.header.item_type, PlaylistItemType::Video); +} + +#[test] +fn from_stalker_group_falls_back_to_cluster_default() { + let item = + StalkerPlaylistItem { stream_id: 7, stream_kind: StalkerStreamKind::Movie, ..StalkerPlaylistItem::default() }; + let converted = PlaylistItem::from_stalker(&item, "my_input"); + assert_eq!(&*converted.header.group, "Movies"); +} diff --git a/shared/src/model/playlist/tests/identity.rs b/shared/src/model/playlist/tests/identity.rs new file mode 100644 index 000000000..51de82924 --- /dev/null +++ b/shared/src/model/playlist/tests/identity.rs @@ -0,0 +1,179 @@ +use super::*; + +#[test] +fn cluster_is_the_single_source_of_truth_for_every_item_type() { + use strum::IntoEnumIterator; + + for item_type in PlaylistItemType::iter() { + let cluster = item_type.cluster(); + + // `From` and the inherent method cannot disagree: one delegates. + assert_eq!(XtreamCluster::from(item_type), cluster, "{item_type:?}"); + + // is_cluster agrees with cluster() for the right one and rejects the + // other two. This is what used to be a separately written match. + assert!(item_type.is_cluster(cluster), "{item_type:?} should be in its own cluster"); + for other in [XtreamCluster::Live, XtreamCluster::Video, XtreamCluster::Series] { + assert_eq!(item_type.is_cluster(other), other == cluster, "{item_type:?} vs {other:?}"); + } + } +} + +#[test] +fn from_stalker_uuid_is_seeded_with_input_name_not_category() { + let mut item = StalkerPlaylistItem { + stream_id: 42, + category_name: Internable::intern("News".to_string()), + ..StalkerPlaylistItem::default() + }; + let converted_a = PlaylistItem::from_stalker(&item, "input_a"); + // A category rename must not change the identity of the channel. + item.category_name = Internable::intern("World News".to_string()); + let converted_b = PlaylistItem::from_stalker(&item, "input_a"); + assert_eq!(converted_a.header.uuid, converted_b.header.uuid); + // A different input owning the same stream id must produce a distinct identity. + let converted_c = PlaylistItem::from_stalker(&item, "input_b"); + assert_ne!(converted_a.header.uuid, converted_c.header.uuid); +} + +#[test] +fn xtream_playlist_item_conversion_falls_back_to_numeric_id_in_url() { + let mut item = PlaylistItem { + header: PlaylistItemHeader { + id: "channel-alpha".intern(), + url: "http://provider.example/live/user/pass/12345.ts".intern(), + input_name: "input".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + item.header.freeze_input_stream_id(); + + let xtream_item = XtreamPlaylistItem::from(&item); + assert_eq!(xtream_item.provider_id, 12345); +} + +#[test] +fn xtream_playlist_item_preserves_numeric_input_stream_id_as_string() { + let mut item = PlaylistItem { + header: PlaylistItemHeader { + id: "80510".intern(), + virtual_id: VirtualId::new(1001), + url: "http://provider.example/live/user/pass/80510.ts".intern(), + input_name: "input".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + item.header.freeze_input_stream_id(); + + let xtream_item = XtreamPlaylistItem::from(&item); + + assert_eq!(xtream_item.provider_id, 80510); + assert_eq!(xtream_item.input_stream_id.as_ref(), "80510"); + assert_eq!(xtream_item.get_input_stream_id().as_deref(), Some("80510")); +} + +#[test] +fn alphanumeric_m3u_input_stream_id_survives_xtream_materialization() { + let mut source = PlaylistItem { + header: PlaylistItemHeader { + id: "channel-alpha".intern(), + url: "http://provider.example/live/user/pass/12345.ts".intern(), + input_name: "input".intern(), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + source.header.freeze_input_stream_id(); + + let m3u_item = M3uPlaylistItem::from(&source); + let common_item = PlaylistItem::from(&m3u_item); + let xtream_item = XtreamPlaylistItem::from(&common_item); + + assert_eq!(m3u_item.input_stream_id.as_ref(), "channel-alpha"); + assert_eq!(xtream_item.provider_id, 12345); + assert_eq!(xtream_item.input_stream_id.as_ref(), "channel-alpha"); +} + +#[test] +fn url_hash_input_stream_id_is_preserved_verbatim() { + let mut source = PlaylistItem { + header: PlaylistItemHeader { + id: "d34db33f".intern(), + url: "http://provider.example/live/channel.m3u8".intern(), + ..PlaylistItemHeader::default() + }, + }; + source.header.freeze_input_stream_id(); + + let m3u_item = M3uPlaylistItem::from(&source); + + assert_eq!(m3u_item.input_stream_id.as_ref(), "d34db33f"); +} + +#[test] +fn target_field_mapping_cannot_change_frozen_input_stream_id() { + let mut header = PlaylistItemHeader { id: "origin-alpha".intern(), ..PlaylistItemHeader::default() }; + header.freeze_input_stream_id(); + + assert!(header.set_field("id", "target-id")); + + assert_eq!(header.id.as_ref(), "target-id"); + assert_eq!(header.input_stream_id.as_ref(), "origin-alpha"); + assert!(!header.set_field("input_stream_id", "unexpected")); + assert_eq!(header.input_stream_id.as_ref(), "origin-alpha"); +} + +#[test] +fn legacy_playlist_items_fall_back_to_provider_id_without_virtual_id() { + let mut source = PlaylistItem { + header: PlaylistItemHeader { + id: "legacy-alpha".intern(), + virtual_id: VirtualId::new(7001), + url: "http://provider.example/live/user/pass/80510.ts".intern(), + ..PlaylistItemHeader::default() + }, + }; + source.header.freeze_input_stream_id(); + let mut m3u_item = M3uPlaylistItem::from(&source); + m3u_item.input_stream_id = "".intern(); + let mut xtream_item = XtreamPlaylistItem::from(&source); + xtream_item.input_stream_id = "".intern(); + + assert_eq!(m3u_item.get_input_stream_id().as_deref(), Some("legacy-alpha")); + assert_eq!(xtream_item.get_input_stream_id().as_deref(), Some("80510")); + xtream_item.provider_id = 0; + xtream_item.url = "http://provider.example/live/channel.ts".intern(); + assert_eq!(xtream_item.get_input_stream_id(), None); + + let mut legacy_common_item = PlaylistItem::from(&xtream_item); + assert!(legacy_common_item.header.id.is_empty()); + assert_eq!(legacy_common_item.get_input_stream_id(), None); + legacy_common_item.header.id = "target-mapped-id".intern(); + let rematerialized_m3u_item = M3uPlaylistItem::from(&legacy_common_item); + let rematerialized_xtream_item = XtreamPlaylistItem::from(&legacy_common_item); + assert_eq!(legacy_common_item.header.id.as_ref(), "target-mapped-id"); + assert_eq!(legacy_common_item.get_input_stream_id(), None); + assert!(rematerialized_m3u_item.provider_id.is_empty()); + assert_eq!(rematerialized_m3u_item.get_input_stream_id(), None); + assert_eq!(rematerialized_xtream_item.provider_id, 0); + assert_eq!(rematerialized_xtream_item.get_input_stream_id(), None); + + xtream_item.input_stream_id = "explicit-origin-alpha".intern(); + assert_eq!(xtream_item.get_input_stream_id().as_deref(), Some("explicit-origin-alpha")); + let identified_common_item = PlaylistItem::from(&xtream_item); + assert_eq!(identified_common_item.header.input_stream_id.as_ref(), "explicit-origin-alpha"); + assert_eq!(identified_common_item.get_input_stream_id().as_deref(), Some("explicit-origin-alpha")); + + let identified_m3u_item = M3uPlaylistItem::from(&identified_common_item); + let identified_xtream_item = XtreamPlaylistItem::from(&identified_common_item); + assert_eq!(identified_m3u_item.input_stream_id.as_ref(), "explicit-origin-alpha"); + assert_eq!(identified_m3u_item.get_input_stream_id().as_deref(), Some("explicit-origin-alpha")); + assert_eq!(identified_xtream_item.provider_id, 0); + assert_eq!(identified_xtream_item.input_stream_id.as_ref(), "explicit-origin-alpha"); + assert_eq!(identified_xtream_item.get_input_stream_id().as_deref(), Some("explicit-origin-alpha")); +} diff --git a/shared/src/model/playlist/tests/m3u.rs b/shared/src/model/playlist/tests/m3u.rs new file mode 100644 index 000000000..bf46a90e7 --- /dev/null +++ b/shared/src/model/playlist/tests/m3u.rs @@ -0,0 +1,358 @@ +use super::*; + +#[test] +fn header_field_parse_round_trips_every_variant() { + for field in [ + HeaderField::Id, + HeaderField::ProviderId, + HeaderField::Title, + HeaderField::Name, + HeaderField::Logo, + HeaderField::LogoSmall, + HeaderField::ParentCode, + HeaderField::AudioTrack, + HeaderField::TimeShift, + HeaderField::Rec, + HeaderField::Url, + HeaderField::Group, + HeaderField::Caption, + HeaderField::Input, + HeaderField::Type, + HeaderField::EpgChannelId, + HeaderField::Chno, + HeaderField::Genre, + ] { + assert_eq!(HeaderField::parse(field.as_str()), Some(field), "{field} did not round-trip"); + // Lookup stayed case-insensitive. + assert_eq!(HeaderField::parse(&field.as_str().to_uppercase()), Some(field)); + } + assert_eq!(HeaderField::parse("epg_id"), Some(HeaderField::EpgChannelId), "legacy alias must still parse"); + assert_eq!(HeaderField::parse("input_stream_id"), None); + assert_eq!(HeaderField::parse(""), None); +} + +#[test] +fn m3u_item_exposes_provider_id_but_not_the_header_only_fields() { + // The M3U resource endpoint resolves a URL path segment through + // `get_field`, so which names resolve is externally visible behaviour. + // M3uPlaylistItem carries input_name, item_type and additional_properties, + // but none of them were ever addressable by name and must stay that way. + let mut header = PlaylistItemHeader { chno: 7, ..PlaylistItemHeader::default() }; + header.name = "Channel".intern(); + let item = M3uPlaylistItem::from(&PlaylistItem { header }); + + assert_eq!(item.get_field("name").as_deref(), Some("Channel")); + assert_eq!(item.get_field("chno").as_deref(), Some("7")); + assert_eq!(item.get_field("caption").as_deref(), Some("Channel")); + + for absent in ["id", "input", "type", "genre", "not_a_field"] { + assert!(item.get_field(absent).is_none(), "{absent} must not resolve on an M3U item"); + } +} + +#[test] +fn m3u_to_m3u_emits_tvg_id_from_epg_channel_id() { + let item = M3uPlaylistItem { + virtual_id: VirtualId::default(), + provider_id: "prov1".intern(), + input_stream_id: "prov1".intern(), + upstream_user_agent: None, + name: "Test Channel".intern(), + chno: 0, + logo: "".intern(), + logo_small: "".intern(), + group: "Test Group".intern(), + title: "Test Title".intern(), + parent_code: "".intern(), + audio_track: "".intern(), + time_shift: "".intern(), + rec: "".intern(), + url: "http://example.com/stream".intern(), + epg_channel_id: Some("epg_channel_123".intern()), + input_name: "test".intern(), + item_type: PlaylistItemType::Live, + t_stream_url: "".intern(), + t_resource_url: None, + t_catchup_source: None, + t_catchup_mode: None, + source_ordinal: 0, + additional_properties: None, + }; + + let output = item.to_m3u(None, false); + assert!(output.contains(r#"tvg-id="epg_channel_123""#), "M3U output must contain tvg-id from epg_channel_id"); +} + +#[test] +fn m3u_to_m3u_preserves_mixed_case_tvg_id() { + // EPG matching is case-insensitive, so ids are no longer lowercased when parsed. + // The M3U tvg-id output must preserve the channel's original source case. + let item = M3uPlaylistItem { + virtual_id: VirtualId::default(), + provider_id: "prov1".intern(), + input_stream_id: "prov1".intern(), + upstream_user_agent: None, + name: "Test Channel".intern(), + chno: 0, + logo: "".intern(), + logo_small: "".intern(), + group: "Test Group".intern(), + title: "Test Title".intern(), + parent_code: "".intern(), + audio_track: "".intern(), + time_shift: "".intern(), + rec: "".intern(), + url: "http://example.com/stream".intern(), + epg_channel_id: Some("CNN.us".intern()), + input_name: "test".intern(), + item_type: PlaylistItemType::Live, + t_stream_url: "".intern(), + t_resource_url: None, + t_catchup_source: None, + t_catchup_mode: None, + source_ordinal: 0, + additional_properties: None, + }; + + let output = item.to_m3u(None, false); + assert!(output.contains(r#"tvg-id="CNN.us""#), "M3U tvg-id must preserve original case, got: {output}"); +} + +#[test] +fn m3u_to_m3u_emits_tvg_chno_from_chno() { + let item = M3uPlaylistItem { + virtual_id: VirtualId::default(), + provider_id: "prov1".intern(), + input_stream_id: "prov1".intern(), + upstream_user_agent: None, + name: "Test Channel".intern(), + chno: 42, + logo: "".intern(), + logo_small: "".intern(), + group: "Test Group".intern(), + title: "Test Title".intern(), + parent_code: "".intern(), + audio_track: "".intern(), + time_shift: "".intern(), + rec: "".intern(), + url: "http://example.com/stream".intern(), + epg_channel_id: None, + input_name: "test".intern(), + item_type: PlaylistItemType::Live, + t_stream_url: "".intern(), + t_resource_url: None, + t_catchup_source: None, + t_catchup_mode: None, + source_ordinal: 0, + additional_properties: None, + }; + + let output = item.to_m3u(None, false); + assert!(output.contains(r#"tvg-chno="42""#), "M3U output must contain tvg-chno from chno"); +} + +#[test] +fn m3u_to_m3u_omits_tvg_chno_when_chno_is_zero() { + let item = M3uPlaylistItem { + virtual_id: VirtualId::default(), + provider_id: "prov1".intern(), + input_stream_id: "prov1".intern(), + upstream_user_agent: None, + name: "Test Channel".intern(), + chno: 0, + logo: "".intern(), + logo_small: "".intern(), + group: "Test Group".intern(), + title: "Test Title".intern(), + parent_code: "".intern(), + audio_track: "".intern(), + time_shift: "".intern(), + rec: "".intern(), + url: "http://example.com/stream".intern(), + epg_channel_id: None, + input_name: "test".intern(), + item_type: PlaylistItemType::Live, + t_stream_url: "".intern(), + t_resource_url: None, + t_catchup_source: None, + t_catchup_mode: None, + source_ordinal: 0, + additional_properties: None, + }; + + let output = item.to_m3u(None, false); + assert!(!output.contains("tvg-chno"), "M3U output must not contain tvg-chno when chno is 0"); +} + +#[test] +fn m3u_to_m3u_preserves_catchup_attributes() { + let item = M3uPlaylistItem { + virtual_id: VirtualId::default(), + provider_id: "prov1".intern(), + input_stream_id: "prov1".intern(), + upstream_user_agent: None, + name: "Test Channel".intern(), + chno: 0, + logo: "".intern(), + logo_small: "".intern(), + group: "Test Group".intern(), + title: "Test Title".intern(), + parent_code: "".intern(), + audio_track: "".intern(), + time_shift: "".intern(), + rec: "".intern(), + url: "http://example.com/stream".intern(), + epg_channel_id: Some("channel1".intern()), + input_name: "test".intern(), + item_type: PlaylistItemType::Live, + t_stream_url: "".intern(), + t_resource_url: None, + t_catchup_source: None, + t_catchup_mode: None, + source_ordinal: 0, + additional_properties: Some(StreamProperties::Live(Box::new(LiveStreamProperties { + catchup: Some(CatchupProperties { + mode: Some("append".intern()), + days: Some("7".intern()), + source: Some("?offset=-${offset}&utcstart=${timestamp}".intern()), + correction: Some("-2.0".intern()), + catchup_type: Some("xc".intern()), + extra_attributes: vec![CatchupAttribute { name: "catchup-extra".intern(), value: "keep".intern() }], + ..CatchupProperties::default() + }), + ..LiveStreamProperties::default() + }))), + }; + + let output = item.to_m3u(None, false); + assert!(output.contains(r#"catchup="append""#)); + assert!(output.contains(r#"catchup-days="7""#)); + assert!(output.contains(r#"catchup-source="?offset=-${offset}&utcstart=${timestamp}""#)); + assert!(output.contains(r#"catchup-correction="-2.0""#)); + assert!(output.contains(r#"catchup-type="xc""#)); + assert!(output.contains(r#"catchup-extra="keep""#)); +} + +#[test] +fn m3u_to_m3u_unifies_append_to_catchup_type_only() { + let item = M3uPlaylistItem { + virtual_id: VirtualId::default(), + provider_id: "prov1".intern(), + input_stream_id: "prov1".intern(), + upstream_user_agent: None, + name: "Test Channel".intern(), + chno: 0, + logo: "".intern(), + logo_small: "".intern(), + group: "Test Group".intern(), + title: "Test Title".intern(), + parent_code: "".intern(), + audio_track: "".intern(), + time_shift: "".intern(), + rec: "".intern(), + url: "http://example.com/stream".intern(), + epg_channel_id: Some("channel1".intern()), + input_name: "test".intern(), + item_type: PlaylistItemType::Live, + t_stream_url: "".intern(), + t_resource_url: None, + t_catchup_source: None, + t_catchup_mode: None, + source_ordinal: 0, + additional_properties: Some(StreamProperties::Live(Box::new(LiveStreamProperties { + catchup: Some(CatchupProperties { + mode: Some("append".intern()), + days: Some("7".intern()), + ..CatchupProperties::default() + }), + ..LiveStreamProperties::default() + }))), + }; + + let output = item.to_m3u(None, false); + assert!(!output.contains(r#"catchup="append""#)); + assert!(output.contains(r#"catchup-type="append""#)); + assert!(output.contains(r#"catchup-days="7""#)); + assert!(!output.contains(r#"catchup-type="xc""#)); +} + +#[test] +fn m3u_to_m3u_uses_rewritten_catchup_mode_and_source() { + let item = M3uPlaylistItem { + virtual_id: VirtualId::default(), + provider_id: "prov1".intern(), + input_stream_id: "prov1".intern(), + upstream_user_agent: None, + name: "Test Channel".intern(), + chno: 0, + logo: "".intern(), + logo_small: "".intern(), + group: "Test Group".intern(), + title: "Test Title".intern(), + parent_code: "".intern(), + audio_track: "".intern(), + time_shift: "".intern(), + rec: "".intern(), + url: "http://example.com/stream".intern(), + epg_channel_id: Some("channel1".intern()), + input_name: "test".intern(), + item_type: PlaylistItemType::Live, + t_stream_url: "".intern(), + t_resource_url: None, + t_catchup_source: Some("http://proxy.example/m3u-catchup/token?v0={utc}".intern()), + t_catchup_mode: Some("default".intern()), + source_ordinal: 0, + additional_properties: Some(StreamProperties::Live(Box::new(LiveStreamProperties { + catchup: Some(CatchupProperties { + mode: Some("append".intern()), + source: Some("?offset=-${offset}".intern()), + ..CatchupProperties::default() + }), + ..LiveStreamProperties::default() + }))), + }; + + let output = item.to_m3u(None, false); + assert!(output.contains(r#"catchup="default""#)); + assert!(output.contains(r#"catchup-source="http://proxy.example/m3u-catchup/token?v0={utc}""#)); + assert!(!output.contains(r#"catchup-source="?offset=-${offset}""#)); +} + +#[test] +fn m3u_to_m3u_flussonic_source_override_uses_rewritten_mode() { + let mut item = M3uPlaylistItem::from(&PlaylistItem { header: PlaylistItemHeader::default() }); + item.url = "https://provider.example/channel/mono.m3u8?token=secret".intern(); + item.t_stream_url = "http://proxy.example/m3u-stream/alice/pass/42/index.m3u8".intern(); + item.t_catchup_mode = Some("default".intern()); + item.t_catchup_source = Some("http://proxy.example/m3u-catchup/token?v0={utc}&v1={duration}".intern()); + item.additional_properties = Some(StreamProperties::Live(Box::new(LiveStreamProperties { + catchup: Some(CatchupProperties { + mode: Some("fs".intern()), + catchup_type: Some("flussonic".intern()), + source: Some("https://provider.example/channel/archive-{utc}-{duration}.m3u8?token=secret".intern()), + ..CatchupProperties::default() + }), + ..LiveStreamProperties::default() + }))); + let output = item.to_m3u(None, false); + assert!(output.contains(r#"catchup="default""#)); + assert!(output.contains("v0={utc}&v1={duration}")); + assert!(!output.contains("catchup-type=")); + assert!(!output.contains("token=secret")); + assert!(output.ends_with("/42/index.m3u8")); + item.t_catchup_mode = None; + item.t_catchup_source = None; + let native = item.to_m3u(None, false); + assert!(native.contains(r#"catchup-type="flussonic""#)); + assert!(!native.contains("catchup-source=")); +} + +#[test] +fn m3u_to_m3u_emits_only_configured_upstream_user_agent() { + let mut item = M3uPlaylistItem::from(&PlaylistItem { header: PlaylistItemHeader::default() }); + item.upstream_user_agent = Some("Provider-UA".intern()); + + assert!(item.to_m3u(None, false).contains("#EXTVLCOPT:http-user-agent=Provider-UA\n")); + item.upstream_user_agent = None; + assert!(!item.to_m3u(None, false).contains("#EXTVLCOPT:http-user-agent=")); +} diff --git a/shared/src/model/playlist/tests/xtream.rs b/shared/src/model/playlist/tests/xtream.rs new file mode 100644 index 000000000..ddf1267eb --- /dev/null +++ b/shared/src/model/playlist/tests/xtream.rs @@ -0,0 +1,211 @@ +use super::*; + +#[test] +fn disabled_resource_rewrite_proxies_private_xtream_images() { + let mut options = sample_options(); + options.web_ui_request = false; + options.base_url = "https://proxy.example".to_string(); + options.flags = XtreamMappingFlagsSet::new(); + options.resource_host_policies.insert("192.168.1.20".to_string(), ResourceOutputPolicy::Proxy); + options.resource_host_policies.insert("8.8.8.8".to_string(), ResourceOutputPolicy::Direct); + options.resource_host_policies.insert("127.0.0.1".to_string(), ResourceOutputPolicy::Blocked); + + assert_eq!( + options.get_resource_url( + XtreamCluster::Live, + PlaylistItemType::Live, + VirtualId::new(1), + "http://192.168.1.20/logo.png", + "logo", + ), + "https://proxy.example/resource/live/user/pass/1/logo" + ); + assert_eq!( + options.get_resource_url( + XtreamCluster::Live, + PlaylistItemType::Live, + VirtualId::new(1), + "http://8.8.8.8/logo.png", + "logo", + ), + "http://8.8.8.8/logo.png" + ); + assert_eq!( + options.get_resource_url( + XtreamCluster::Live, + PlaylistItemType::Live, + VirtualId::new(1), + "http://127.0.0.1/logo.png", + "logo", + ), + "" + ); + assert_eq!( + options.get_bd_path_resource_url( + XtreamCluster::Live, + PlaylistItemType::Live, + VirtualId::new(1), + "http://192.168.1.20/backdrop.png", + "", + 0, + ), + "https://proxy.example/resource/live/user/pass/1/backdrop_path_0" + ); + + options.resource_proxy_item_types = PlaylistItemTypeSet::empty(); + assert_eq!( + options.get_resource_url( + XtreamCluster::Live, + PlaylistItemType::Live, + VirtualId::new(1), + "http://192.168.1.20/logo.png", + "logo", + ), + "https://proxy.example/resource/live/user/pass/1/logo" + ); +} + +#[test] +fn get_resource_url_keeps_internal_paths_for_web_ui_requests() { + let options = sample_options(); + + assert_eq!( + options.get_resource_url( + XtreamCluster::Series, + PlaylistItemType::Series, + VirtualId::new(1), + "/api/v1/library/thumbnail/abc", + "logo", + ), + "/api/v1/library/thumbnail/abc", + ); +} + +#[test] +fn get_bd_path_resource_url_keeps_internal_thumbnail_paths_for_web_ui_requests() { + let options = sample_options(); + + assert_eq!( + options.get_bd_path_resource_url( + XtreamCluster::Series, + PlaylistItemType::Series, + VirtualId::new(1), + "/api/v1/library/thumbnail/backdrop", + "backdrop_", + 0, + ), + "/api/v1/library/thumbnail/backdrop", + ); +} + +#[test] +fn get_resource_url_absolutizes_internal_thumbnail_paths_for_xtream_clients() { + let mut options = sample_options(); + options.web_ui_request = false; + options.base_url = "http://proxy.example/base".to_string(); + options.reverse_item_types = PlaylistItemTypeSet::empty(); + options.resource_proxy_item_types = PlaylistItemTypeSet::empty(); + + assert_eq!( + options.get_resource_url( + XtreamCluster::Series, + PlaylistItemType::LocalSeries, + VirtualId::new(1), + "/api/v1/library/thumbnail/abc", + "logo", + ), + "http://proxy.example/base/api/v1/library/thumbnail/abc", + ); +} + +#[test] +fn get_bd_path_resource_url_absolutizes_internal_thumbnail_paths_for_xtream_clients() { + let mut options = sample_options(); + options.web_ui_request = false; + options.base_url = "http://proxy.example/base".to_string(); + options.reverse_item_types = PlaylistItemTypeSet::empty(); + options.resource_proxy_item_types = PlaylistItemTypeSet::empty(); + + assert_eq!( + options.get_bd_path_resource_url( + XtreamCluster::Series, + PlaylistItemType::LocalSeries, + VirtualId::new(1), + "/api/v1/library/thumbnail/backdrop", + "backdrop_", + 0, + ), + "http://proxy.example/base/api/v1/library/thumbnail/backdrop", + ); +} + +#[test] +fn get_resource_url_obfuscates_protocol_relative_urls_for_web_ui_requests() { + let options = sample_options(); + let resource_url = "//cdn.example.com/poster.jpg"; + + assert_eq!( + options.get_resource_url( + XtreamCluster::Series, + PlaylistItemType::Series, + VirtualId::new(1), + resource_url, + "logo", + ), + concat_path(&options.base_url, &seal_web_ui_resource_url(&options.encrypt_secret, resource_url)), + ); +} + +#[test] +fn get_resource_url_rewrites_redirect_resources_for_xtream_clients() { + let mut options = sample_options(); + options.web_ui_request = false; + options.base_url = "http://proxy.example/iptv".to_string(); + options.reverse_item_types = PlaylistItemTypeSet::empty(); + options.resource_proxy_item_types = PlaylistItemTypeSet::from_item(PlaylistItemType::Live); + + assert_eq!( + options.get_resource_url( + XtreamCluster::Live, + PlaylistItemType::Live, + VirtualId::new(2017), + "https://provider.example/logo.png", + "logo", + ), + "http://proxy.example/iptv/resource/live/user/pass/2017/logo", + ); +} + +#[test] +fn get_resource_url_does_not_bypass_untrusted_root_relative_paths_for_web_ui_requests() { + let options = sample_options(); + let resource_url = "/provider-controlled/poster.jpg"; + + assert_eq!( + options.get_resource_url( + XtreamCluster::Series, + PlaylistItemType::Series, + VirtualId::new(1), + resource_url, + "logo", + ), + concat_path(&options.base_url, &seal_web_ui_resource_url(&options.encrypt_secret, resource_url)), + ); +} + +#[test] +fn get_resource_url_does_not_trust_absolute_urls_containing_internal_thumbnail_path() { + let options = sample_options(); + let resource_url = "https://provider.example/api/v1/library/thumbnail/abc"; + + assert_eq!( + options.get_resource_url( + XtreamCluster::Series, + PlaylistItemType::Series, + VirtualId::new(1), + resource_url, + "logo", + ), + concat_path(&options.base_url, &seal_web_ui_resource_url(&options.encrypt_secret, resource_url)), + ); +} diff --git a/shared/src/model/playlist/xtream.rs b/shared/src/model/playlist/xtream.rs new file mode 100644 index 000000000..d5d8f3bbe --- /dev/null +++ b/shared/src/model/playlist/xtream.rs @@ -0,0 +1,389 @@ +use super::{PlaylistEntry, PlaylistItemType, PlaylistItemTypeSet, XtreamCluster}; +use crate::{ + create_bitset, + model::{ + xtream_const, ClusterFlags, CommonPlaylistItem, StreamProperties, UUIDType, VirtualId, XtreamInfoDocument, + }, + utils::{ + arc_str_option_serde, arc_str_serde, concat_path, generate_runtime_playlist_uuid, seal_web_ui_resource_url, + Internable, + }, +}; +use dashmap::DashMap; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +create_bitset!( + u8, + XtreamMappingFlags, + SkipLiveDirectSource, + SkipVideoDirectSource, + SkipSeriesDirectSource, + RewriteResourceUrl +); + +pub struct XtreamMappingOptions { + pub flags: XtreamMappingFlagsSet, + pub force_redirect: Option, + pub reverse_item_types: PlaylistItemTypeSet, + pub resource_proxy_item_types: PlaylistItemTypeSet, + pub username: String, + pub password: String, + pub base_url: String, + pub web_ui_request: bool, + pub encrypt_secret: [u8; 16], + pub resource_host_policies: Arc>, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ResourceOutputPolicy { + Direct, + Proxy, + Blocked, +} + +impl XtreamMappingOptions { + #[inline] + pub fn is_reverse(&self, item_type: PlaylistItemType) -> bool { self.reverse_item_types.is_set(item_type) } + + pub(super) fn is_trusted_web_ui_resource_path(resource_url: &str) -> bool { + const TRUSTED_WEB_UI_RESOURCE_PREFIXES: [&str; 1] = ["/api/v1/library/thumbnail/"]; + TRUSTED_WEB_UI_RESOURCE_PREFIXES.iter().any(|prefix| resource_url.starts_with(prefix)) + } + + pub(super) fn build_reverse_proxy_base_url( + &self, + xtream_cluster: XtreamCluster, + item_type: PlaylistItemType, + virtual_id: VirtualId, + force_proxy: bool, + ) -> Option { + let proxy_resource = self.resource_proxy_item_types.is_set(item_type); + if force_proxy || (proxy_resource && self.flags.contains(XtreamMappingFlags::RewriteResourceUrl)) { + Some(format!( + "{}/resource/{}/{}/{}/{}", + self.base_url, + xtream_cluster.as_stream_type(), + self.username, + self.password, + virtual_id + )) + } else { + None + } + } + + pub(super) fn resource_output_policy(&self, resource_url: &str) -> ResourceOutputPolicy { + if self.flags.contains(XtreamMappingFlags::RewriteResourceUrl) { + return ResourceOutputPolicy::Proxy; + } + if resource_url.starts_with("media-server://image/") { + return ResourceOutputPolicy::Proxy; + } + let Ok(url) = url::Url::parse(resource_url) else { + return if resource_url.starts_with("//") + || resource_url.starts_with("http:") + || resource_url.starts_with("https:") + { + ResourceOutputPolicy::Blocked + } else { + ResourceOutputPolicy::Direct + }; + }; + if !matches!(url.scheme(), "http" | "https") { + return ResourceOutputPolicy::Blocked; + } + let Some(host) = url.host_str() else { + return ResourceOutputPolicy::Blocked; + }; + // A host missing from the prepared set still needs a proxy link; a new resource + // field must not expose its origin before the output classifier knows the host. + self.resource_host_policies.get(host).map_or(ResourceOutputPolicy::Proxy, |entry| *entry) + } + + pub fn get_resource_url( + &self, + xtream_cluster: XtreamCluster, + item_type: PlaylistItemType, + virtual_id: VirtualId, + resource_url: &str, + resource_field: &str, + ) -> String { + self.rewrite_resource_url_inner(xtream_cluster, item_type, virtual_id, resource_url, |base| { + format!("{base}/{resource_field}") + }) + } + + pub fn get_bd_path_resource_url( + &self, + xtream_cluster: XtreamCluster, + item_type: PlaylistItemType, + virtual_id: VirtualId, + resource_url: &str, + resource_field: &str, + index: usize, + ) -> String { + self.rewrite_resource_url_inner(xtream_cluster, item_type, virtual_id, resource_url, |base| { + format!("{base}/{resource_field}{}_{index}", xtream_const::XC_PROP_BACKDROP_PATH) + }) + } + + pub(super) fn rewrite_resource_url_inner( + &self, + xtream_cluster: XtreamCluster, + item_type: PlaylistItemType, + virtual_id: VirtualId, + resource_url: &str, + proxy_suffix: impl FnOnce(&str) -> String, + ) -> String { + let Some(resource_url) = super::super::persisted_resource_url(resource_url) else { + return String::new(); + }; + let resource_url = resource_url.as_ref(); + if !self.web_ui_request && Self::is_trusted_web_ui_resource_path(resource_url) { + return concat_path(&self.base_url, resource_url); + } + + if self.web_ui_request { + if resource_url.is_empty() { + return resource_url.to_string(); + } + if Self::is_trusted_web_ui_resource_path(resource_url) { + return resource_url.to_string(); + } + return concat_path(&self.base_url, &seal_web_ui_resource_url(&self.encrypt_secret, resource_url)); + } + + let policy = self.resource_output_policy(resource_url); + if policy == ResourceOutputPolicy::Blocked { + return String::new(); + } + let rewrite_url = self.build_reverse_proxy_base_url( + xtream_cluster, + item_type, + virtual_id, + policy == ResourceOutputPolicy::Proxy, + ); + + if let Some(url) = rewrite_url { + if resource_url.starts_with("http://") + || resource_url.starts_with("https://") + || resource_url.starts_with("media-server://image/") + { + return proxy_suffix(&url); + } + } + resource_url.to_string() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct XtreamPlaylistItem { + pub virtual_id: VirtualId, + pub provider_id: u32, + #[serde(with = "arc_str_serde")] + pub name: Arc, + #[serde(with = "arc_str_serde")] + pub logo: Arc, + #[serde(with = "arc_str_serde")] + pub logo_small: Arc, + #[serde(with = "arc_str_serde")] + pub group: Arc, + #[serde(with = "arc_str_serde")] + pub title: Arc, + #[serde(with = "arc_str_serde")] + pub parent_code: Arc, + #[serde(with = "arc_str_serde")] + pub rec: Arc, + #[serde(with = "arc_str_serde")] + pub url: Arc, + #[serde(default, with = "arc_str_option_serde")] + pub epg_channel_id: Option>, + pub xtream_cluster: XtreamCluster, + #[serde(default)] + pub additional_properties: Option, + pub item_type: PlaylistItemType, + pub category_id: u32, + #[serde(with = "arc_str_serde")] + pub input_name: Arc, + pub channel_no: u32, + #[serde(default)] + pub source_ordinal: u32, + /// Stable provider/origin ID captured before any target transformation. + #[serde(default, with = "arc_str_serde")] + pub input_stream_id: Arc, + #[serde(default, rename = "source_user_agent", alias = "upstream_user_agent", with = "arc_str_option_serde")] + pub upstream_user_agent: Option>, +} + +impl XtreamPlaylistItem { + pub fn to_common(&self) -> CommonPlaylistItem { + CommonPlaylistItem { + virtual_id: self.virtual_id, + provider_id: self.provider_id.intern(), + name: self.name.clone(), + chno: self.channel_no, + logo: self.logo.clone(), + logo_small: self.logo_small.clone(), + group: self.group.clone(), + title: self.title.clone(), + parent_code: self.parent_code.clone(), + audio_track: "".intern(), + time_shift: "".intern(), + rec: self.rec.clone(), + url: self.url.clone(), + input_name: self.input_name.clone(), + item_type: self.item_type, + epg_channel_id: self.epg_channel_id.clone(), + xtream_cluster: Some(self.xtream_cluster), + additional_properties: self.additional_properties.clone(), + category_id: Some(self.category_id), + } + } + + pub fn get_container_extension(&self) -> Option> { + match self.additional_properties { + None => None, + Some(ref props) => match props { + StreamProperties::Live(_) => Some("ts".intern()), + StreamProperties::Video(video) => Some(Arc::clone(&video.container_extension)), + StreamProperties::Series(_) => None, + StreamProperties::Episode(episode) => Some(Arc::clone(&episode.container_extension)), + }, + } + } + + #[inline] + pub fn has_details(&self) -> bool { + self.additional_properties.as_ref().is_some_and(super::super::stream_properties::StreamProperties::has_details) + } + + pub fn visit_resource_urls(&self, mut visit: impl FnMut(&str)) { + let mut visit_persisted = |value: &str| { + if let Some(url) = super::super::persisted_resource_url(value) { + visit(url.as_ref()); + } + }; + visit_persisted(&self.logo); + visit_persisted(&self.logo_small); + match self.additional_properties.as_ref() { + Some(StreamProperties::Live(live)) => visit_persisted(&live.stream_icon), + Some(StreamProperties::Video(video)) => { + visit_persisted(&video.stream_icon); + if let Some(details) = video.details.as_ref() { + if let Some(url) = details.cover_big.as_deref() { + visit_persisted(url); + } + if let Some(url) = details.movie_image.as_deref() { + visit_persisted(url); + } + if let Some(backdrops) = details.backdrop_path.as_ref() { + for url in backdrops { + visit_persisted(url); + } + } + } + } + Some(StreamProperties::Series(series)) => { + visit_persisted(&series.cover); + if let Some(backdrops) = series.backdrop_path.as_ref() { + for url in backdrops { + visit_persisted(url); + } + } + if let Some(details) = series.details.as_ref() { + if let Some(seasons) = details.seasons.as_ref() { + for season in seasons { + for url in [&season.cover, &season.cover_tmdb, &season.cover_big].into_iter().flatten() { + visit_persisted(url); + } + } + } + if let Some(episodes) = details.episodes.as_ref() { + for episode in episodes { + visit_persisted(&episode.movie_image); + } + } + } + } + Some(StreamProperties::Episode(episode)) => visit_persisted(&episode.movie_image), + None => {} + } + } + + pub fn resolve_resource_url(&self, field: &str) -> Option> { + let bytes = field.as_bytes(); + let url = if bytes.eq_ignore_ascii_case(b"logo") && !self.logo.is_empty() { + Some(Arc::clone(&self.logo)) + } else if bytes.eq_ignore_ascii_case(b"logo_small") && !self.logo_small.is_empty() { + Some(Arc::clone(&self.logo_small)) + } else { + self.additional_properties.as_ref().and_then(|a| a.resolve_resource_url(field)) + }?; + super::super::persisted_resource_url(&url).map(|decoded| match decoded { + std::borrow::Cow::Borrowed(_) => Arc::clone(&url), + std::borrow::Cow::Owned(value) => Arc::from(value), + }) + } +} + +impl PlaylistEntry for XtreamPlaylistItem { + #[inline] + fn get_virtual_id(&self) -> VirtualId { self.virtual_id } + fn get_input_stream_id(&self) -> Option> { + if self.input_stream_id.is_empty() { + (self.provider_id > 0).then(|| self.provider_id.to_string().intern()) + } else { + Some(Arc::clone(&self.input_stream_id)) + } + } + + fn get_upstream_user_agent(&self) -> Option<&str> { self.upstream_user_agent.as_deref() } + #[inline] + fn get_provider_id(&self) -> Option { Some(self.provider_id) } + #[inline] + fn get_category_id(&self) -> Option { Some(self.category_id) } + #[inline] + fn get_provider_url(&self) -> Arc { Arc::clone(&self.url) } + + #[inline] + fn get_uuid(&self) -> UUIDType { + generate_runtime_playlist_uuid(&self.input_name, &self.provider_id.to_string(), self.item_type, &self.url) + } + #[inline] + fn get_item_type(&self) -> PlaylistItemType { self.item_type } + #[inline] + fn get_group(&self) -> Arc { Arc::clone(&self.group) } + #[inline] + fn get_name(&self) -> Arc { + if self.title.is_empty() { + Arc::clone(&self.name) + } else { + Arc::clone(&self.title) + } + } + + fn get_resolved_info_document(&self, options: &XtreamMappingOptions) -> Option { + if self.has_details() { + self.additional_properties.as_ref().map(|p| { + p.to_info_document( + options, + self.get_item_type(), + self.get_virtual_id(), + self.get_category_id().unwrap_or(0), + ) + }) + } else { + None + } + } + + #[inline] + fn get_additional_properties(&self) -> Option<&StreamProperties> { self.additional_properties.as_ref() } + #[inline] + fn get_additional_properties_mut(&mut self) -> Option<&mut StreamProperties> { self.additional_properties.as_mut() } +} + +impl From for CommonPlaylistItem { + fn from(item: XtreamPlaylistItem) -> Self { item.to_common() } +} diff --git a/shared/src/model/playlist_info_document.rs b/shared/src/model/playlist_info_document.rs index 93096ce00..c8cda290c 100644 --- a/shared/src/model/playlist_info_document.rs +++ b/shared/src/model/playlist_info_document.rs @@ -1,27 +1,5 @@ -use crate::{ - concat_string, - model::{ - info_doc_utils::InfoDocUtils, PlaylistItemType, SeriesStreamDetailEpisodeProperties, - SeriesStreamDetailSeasonProperties, SeriesStreamProperties, StreamProperties, VideoStreamProperties, VirtualId, - XtreamCluster, XtreamMappingFlags, XtreamMappingOptions, - }, - utils::{ - arc_str_null_is_none_serde, arc_str_option_null_if_empty_serde, arc_str_option_serde, arc_str_serde, - arc_str_vec_serde, Internable, - }, -}; -use indexmap::IndexMap; +use crate::model::{PlaylistItemType, StreamProperties, VirtualId, XtreamMappingOptions}; use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::sync::Arc; - -#[inline] -fn build_season_episode_field(season: u32, episode: u32, field: &str) -> String { - concat_string!("nfo_ep_", &season.to_string(), "_", &episode.to_string(), "_", field) -} - -#[inline] -fn build_season_field(season: u32, field: &str) -> String { concat_string!("nfo_s_", &season.to_string(), "_", field) } #[allow(clippy::large_enum_variant)] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] @@ -39,194 +17,6 @@ impl Default for XtreamInfoDocument { fn default() -> Self { Self::Empty(XtreamEmptyDoc {}) } } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct XtreamVideoInfoDoc { - pub info: XtreamVideoInfoData, - pub movie_data: XtreamVideoMovieData, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct XtreamVideoInfoData { - #[serde(with = "arc_str_serde")] - pub kinopoisk_url: Arc, - #[serde(with = "arc_str_serde")] - pub tmdb_id: Arc, - #[serde(with = "arc_str_serde")] - pub name: Arc, - #[serde(with = "arc_str_serde")] - pub o_name: Arc, - #[serde(with = "arc_str_serde")] - pub cover_big: Arc, - #[serde(with = "arc_str_serde")] - pub movie_image: Arc, - #[serde(rename = "releasedate", with = "arc_str_serde")] - pub release_date: Arc, - pub episode_run_time: u32, - #[serde(with = "arc_str_serde")] - pub youtube_trailer: Arc, - #[serde(with = "arc_str_serde")] - pub director: Arc, - #[serde(with = "arc_str_serde")] - pub actors: Arc, - #[serde(with = "arc_str_serde")] - pub cast: Arc, - #[serde(with = "arc_str_serde")] - pub description: Arc, - #[serde(with = "arc_str_serde")] - pub plot: Arc, - #[serde(with = "arc_str_serde")] - pub age: Arc, - #[serde(with = "arc_str_serde")] - pub mpaa_rating: Arc, - pub rating_count_kinopoisk: u32, - #[serde(with = "arc_str_serde")] - pub country: Arc, - #[serde(with = "arc_str_serde")] - pub genre: Arc, - #[serde(with = "arc_str_vec_serde")] - pub backdrop_path: Vec>, - #[serde(with = "arc_str_serde")] - pub duration_secs: Arc, - #[serde(with = "arc_str_serde")] - pub duration: Arc, - pub video: Value, - pub audio: Value, - pub bitrate: u32, - #[serde(with = "arc_str_serde")] - pub rating: Arc, - #[serde(with = "arc_str_serde")] - pub runtime: Arc, - #[serde(with = "arc_str_serde")] - pub status: Arc, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct XtreamVideoMovieData { - pub stream_id: u32, - #[serde(with = "arc_str_serde")] - pub name: Arc, - #[serde(with = "arc_str_serde")] - pub added: Arc, - #[serde(with = "arc_str_serde")] - pub category_id: Arc, - pub category_ids: Vec, - #[serde(with = "arc_str_null_is_none_serde")] - pub container_extension: Arc, - #[serde(default, with = "arc_str_option_null_if_empty_serde")] - pub custom_sid: Option>, - #[serde(with = "arc_str_serde")] - pub direct_source: Arc, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct XtreamSeriesInfoDoc { - #[serde(default)] - pub seasons: Vec, - pub info: XtreamSeriesInfoData, - pub episodes: IndexMap>, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct XtreamSeriesSeasonInfoDoc { - #[serde(default, with = "arc_str_serde")] - pub name: Arc, - #[serde(default, with = "arc_str_serde")] - pub episode_count: Arc, - #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] - pub overview: Option>, - #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] - pub air_date: Option>, - #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] - pub cover: Option>, - #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] - pub cover_tmdb: Option>, - #[serde(default)] - pub season_number: u32, - #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] - pub cover_big: Option>, - #[serde(default, rename = "releaseDate", with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] - pub release_date: Option>, - #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] - pub duration: Option>, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct XtreamSeriesInfoData { - #[serde(with = "arc_str_serde")] - pub name: Arc, - #[serde(with = "arc_str_serde")] - pub cover: Arc, - #[serde(with = "arc_str_serde")] - pub plot: Arc, - #[serde(with = "arc_str_serde")] - pub cast: Arc, - #[serde(with = "arc_str_serde")] - pub director: Arc, - #[serde(with = "arc_str_serde")] - pub genre: Arc, - #[serde(rename = "releaseDate", with = "arc_str_serde")] - pub release_date_alternate: Arc, - #[serde(with = "arc_str_serde")] - pub release_date: Arc, - #[serde(with = "arc_str_serde")] - pub last_modified: Arc, - #[serde(with = "arc_str_serde")] - pub rating: Arc, - #[serde(with = "arc_str_serde")] - pub rating_5based: Arc, - #[serde(with = "arc_str_vec_serde")] - pub backdrop_path: Vec>, - #[serde(with = "arc_str_serde")] - pub tmdb: Arc, - #[serde(with = "arc_str_serde")] - pub youtube_trailer: Arc, - #[serde(with = "arc_str_serde")] - pub episode_run_time: Arc, - #[serde(with = "arc_str_serde")] - pub category_id: Arc, - pub category_ids: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct XtreamSeriesEpisodeInfoDoc { - #[serde(with = "arc_str_serde")] - pub id: Arc, - pub episode_num: u32, - #[serde(with = "arc_str_serde")] - pub title: Arc, - #[serde(with = "arc_str_null_is_none_serde")] - pub container_extension: Arc, - pub info: XtreamSeriesEpisodeInfoData, - #[serde(default, with = "arc_str_option_null_if_empty_serde")] - pub custom_sid: Option>, - #[serde(with = "arc_str_serde")] - pub added: Arc, - pub season: u32, - #[serde(with = "arc_str_serde")] - pub direct_source: Arc, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct XtreamSeriesEpisodeInfoData { - #[serde(with = "arc_str_serde")] - pub air_date: Arc, - #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] - pub crew: Option>, - pub rating: f64, - #[serde(rename = "id")] - pub tmdb_id: u32, - #[serde(default, with = "arc_str_option_serde")] - pub plot: Option>, - #[serde(with = "arc_str_serde")] - pub movie_image: Arc, - pub duration_secs: u32, - #[serde(with = "arc_str_serde")] - pub duration: Arc, - pub video: Value, - pub audio: Value, - pub bitrate: u32, -} - impl StreamProperties { pub fn to_info_document( &self, @@ -260,410 +50,16 @@ impl StreamProperties { } } } - - fn series_to_info_document( - &self, - options: &XtreamMappingOptions, - series: &SeriesStreamProperties, - item_type: PlaylistItemType, - virtual_id: VirtualId, - category_id: u32, - ) -> XtreamSeriesInfoDoc { - XtreamSeriesInfoDoc { - seasons: if let Some(seasons) = series.details.as_ref().and_then(|d| d.seasons.as_ref()) { - self.series_seasons_to_info_document(options, item_type, virtual_id, seasons) - } else { - Vec::new() - }, - info: XtreamSeriesInfoData { - name: Arc::clone(&series.name), - cover: options - .get_resource_url(XtreamCluster::Series, item_type, virtual_id, series.cover.as_ref(), "cover") - .intern(), - plot: series.plot.as_ref().map_or_else(|| "".intern(), Arc::clone), - cast: Arc::clone(&series.cast), - director: Arc::clone(&series.director), - genre: series.genre.as_ref().map_or_else(|| "".intern(), Arc::clone), - release_date: series.release_date.as_ref().map_or_else(|| "".intern(), Arc::clone), - release_date_alternate: series.release_date.as_ref().map_or_else(|| "".intern(), Arc::clone), - last_modified: series.last_modified.as_ref().map_or_else(|| "".intern(), Arc::clone), - rating: InfoDocUtils::limited(series.rating).intern(), - rating_5based: InfoDocUtils::limited(series.rating_5based).intern(), - backdrop_path: series.backdrop_path.as_ref().map_or_else(Vec::new, |b| { - b.iter() - .enumerate() - .map(|(idx, p)| { - options - .get_bd_path_resource_url(XtreamCluster::Series, item_type, virtual_id, p, "", idx) - .intern() - }) - .collect() - }), - tmdb: series.tmdb.map_or_else(|| "".intern(), |value| value.to_string().intern()), - youtube_trailer: Arc::clone(&series.youtube_trailer), - episode_run_time: series.episode_run_time.as_ref().map_or_else(|| "".intern(), Arc::clone), - category_id: category_id.to_string().intern(), - category_ids: vec![category_id], - }, - episodes: if let Some(episodes) = series.details.as_ref().and_then(|d| d.episodes.as_ref()) { - self.series_episodes_to_info_document(options, item_type, virtual_id, episodes) - } else { - IndexMap::new() - }, - } - } - - fn video_to_info_document( - &self, - options: &XtreamMappingOptions, - video: &VideoStreamProperties, - item_type: PlaylistItemType, - virtual_id: VirtualId, - category_id: u32, - ) -> XtreamVideoInfoDoc { - let stream_icon = options - .get_resource_url(XtreamCluster::Video, item_type, virtual_id, &self.get_stream_icon(), "logo") - .intern(); - let empty_str = "".intern(); - let zero_str = "0".intern(); - - let info = if let Some(details) = video.details.as_ref() { - XtreamVideoInfoData { - kinopoisk_url: details.kinopoisk_url.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - tmdb_id: video.tmdb.unwrap_or_default().to_string().intern(), - name: Arc::clone(&video.name), - o_name: details.o_name.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - cover_big: options - .get_resource_url( - XtreamCluster::Video, - item_type, - virtual_id, - details.cover_big.as_ref().map_or("", Arc::as_ref), - "nfo_cover_big", - ) - .intern(), - movie_image: options - .get_resource_url( - XtreamCluster::Video, - item_type, - virtual_id, - details.movie_image.as_ref().map_or("", Arc::as_ref), - "nfo_movie_image", - ) - .intern(), - release_date: details.release_date.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - episode_run_time: details.episode_run_time.unwrap_or_default(), - youtube_trailer: details.youtube_trailer.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - director: details.director.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - actors: details.actors.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - cast: details.cast.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - description: details.description.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - plot: details.plot.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - age: details.age.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - mpaa_rating: details.mpaa_rating.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - rating_count_kinopoisk: details.rating_count_kinopoisk, - country: details.country.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - genre: details.genre.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - backdrop_path: details.backdrop_path.as_deref().map_or_else(Vec::new, |b| { - b.iter() - .enumerate() - .map(|(idx, p)| { - options - .get_bd_path_resource_url(XtreamCluster::Video, item_type, virtual_id, p, "nfo_", idx) - .intern() - }) - .collect() - }), - duration_secs: details.duration_secs.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - duration: details.duration.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - video: InfoDocUtils::build_value(details.video.as_ref().map(Arc::as_ref)), - audio: InfoDocUtils::build_value(details.audio.as_ref().map(Arc::as_ref)), - bitrate: details.bitrate, - rating: InfoDocUtils::limited(video.rating.unwrap_or_default()).intern(), - runtime: details.runtime.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - status: details.status.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), - } - } else { - XtreamVideoInfoData { - kinopoisk_url: Arc::clone(&empty_str), - tmdb_id: video.tmdb.unwrap_or_default().to_string().intern(), - name: Arc::clone(&video.name), - o_name: Arc::clone(&video.name), - cover_big: stream_icon.clone(), - movie_image: stream_icon.clone(), - release_date: Arc::clone(&empty_str), - episode_run_time: 0, - youtube_trailer: Arc::clone(&empty_str), - director: Arc::clone(&empty_str), - actors: Arc::clone(&empty_str), - cast: Arc::clone(&empty_str), - description: Arc::clone(&empty_str), - plot: Arc::clone(&empty_str), - age: Arc::clone(&empty_str), - mpaa_rating: Arc::clone(&empty_str), - rating_count_kinopoisk: 0, - country: Arc::clone(&empty_str), - genre: Arc::clone(&empty_str), - backdrop_path: vec![Arc::clone(&stream_icon)], - duration_secs: Arc::clone(&zero_str), - duration: Arc::clone(&zero_str), - video: Value::Array(Vec::new()), - audio: Value::Array(Vec::new()), - bitrate: 0, - rating: InfoDocUtils::limited(video.rating.unwrap_or_default()).intern(), - runtime: zero_str, - status: "Released".intern(), - } - }; - - XtreamVideoInfoDoc { - info, - movie_data: XtreamVideoMovieData { - stream_id: virtual_id.get(), - name: Arc::clone(&video.name), - added: Arc::clone(&video.added), - category_id: category_id.to_string().intern(), - category_ids: vec![category_id], - container_extension: Arc::clone(&video.container_extension), - custom_sid: video.custom_sid.as_ref().map(Arc::clone), - direct_source: if options.flags.contains(XtreamMappingFlags::SkipVideoDirectSource) { - Arc::clone(&empty_str) - } else { - Arc::clone(&video.direct_source) - }, - }, - } - } - - fn series_seasons_to_info_document( - &self, - options: &XtreamMappingOptions, - item_type: PlaylistItemType, - virtual_id: VirtualId, - seasons: &[SeriesStreamDetailSeasonProperties], - ) -> Vec { - seasons - .iter() - .map(|season| XtreamSeriesSeasonInfoDoc { - name: Arc::clone(&season.name), - season_number: season.season_number, - episode_count: season.episode_count.intern(), - overview: season.overview.as_ref().map(|v| { - if v.starts_with("http://") || v.starts_with("https://") { - options - .get_resource_url( - XtreamCluster::Series, - item_type, - virtual_id, - v, - &build_season_field(season.season_number, "overview"), - ) - .intern() - } else { - Arc::clone(v) - } - }), - air_date: season.air_date.as_ref().map(Arc::clone), - cover: season.cover.as_ref().map(|v| { - options - .get_resource_url( - XtreamCluster::Series, - item_type, - virtual_id, - v, - &build_season_field(season.season_number, "cover"), - ) - .intern() - }), - cover_tmdb: season.cover_tmdb.as_ref().map(|v| { - options - .get_resource_url( - XtreamCluster::Series, - item_type, - virtual_id, - v, - &build_season_field(season.season_number, "cover_tmdb"), - ) - .intern() - }), - cover_big: season.cover_big.as_ref().map(|v| { - options - .get_resource_url( - XtreamCluster::Series, - item_type, - virtual_id, - v, - &build_season_field(season.season_number, "cover_big"), - ) - .intern() - }), - release_date: season.air_date.as_ref().map(Arc::clone), - duration: season.duration.as_ref().map(Arc::clone), - }) - .collect() - } - - fn series_episodes_to_info_document( - &self, - options: &XtreamMappingOptions, - item_type: PlaylistItemType, - virtual_id: VirtualId, - episodes: &[SeriesStreamDetailEpisodeProperties], - ) -> IndexMap> { - let empty_str = "".intern(); - let mut map: IndexMap> = IndexMap::new(); - for ep in episodes { - let doc = XtreamSeriesEpisodeInfoDoc { - id: ep.id.intern(), - episode_num: ep.episode_num, - title: Arc::clone(&ep.title), - container_extension: Arc::clone(&ep.container_extension), - info: XtreamSeriesEpisodeInfoData { - tmdb_id: ep.tmdb.unwrap_or_default(), - air_date: Arc::clone(&ep.release_date), - crew: ep.crew.as_ref().map(Arc::clone), - plot: ep.plot.as_ref().map(Arc::clone), - rating: ep.rating.unwrap_or_default(), - movie_image: options - .get_resource_url( - XtreamCluster::Series, - item_type, - virtual_id, - &ep.movie_image, - &build_season_episode_field(ep.season, ep.episode_num, "movie_image"), - ) - .intern(), - duration: Arc::clone(&ep.duration), - duration_secs: ep.duration_secs, - video: InfoDocUtils::build_value(ep.video.as_ref().map(Arc::as_ref)), - audio: InfoDocUtils::build_value(ep.audio.as_ref().map(Arc::as_ref)), - bitrate: ep.bitrate, - }, - custom_sid: ep.custom_sid.as_ref().map(Arc::clone), - added: Arc::clone(&ep.added), - season: ep.season, - direct_source: if options.flags.contains(XtreamMappingFlags::SkipSeriesDirectSource) { - Arc::clone(&empty_str) - } else { - Arc::clone(&ep.direct_source) - }, - }; - map.entry(ep.season).or_default().push(doc); - } - - map.into_iter().map(|(season, episodes)| (season.to_string(), episodes)).collect() - } -} - -impl Default for XtreamVideoInfoDoc { - fn default() -> Self { - let empty_str = "".intern(); - Self { - info: XtreamVideoInfoData { - kinopoisk_url: Arc::clone(&empty_str), - tmdb_id: Arc::clone(&empty_str), - name: Arc::clone(&empty_str), - o_name: Arc::clone(&empty_str), - cover_big: Arc::clone(&empty_str), - movie_image: Arc::clone(&empty_str), - release_date: Arc::clone(&empty_str), - episode_run_time: 0, - youtube_trailer: Arc::clone(&empty_str), - director: Arc::clone(&empty_str), - actors: Arc::clone(&empty_str), - cast: Arc::clone(&empty_str), - description: Arc::clone(&empty_str), - plot: Arc::clone(&empty_str), - age: Arc::clone(&empty_str), - mpaa_rating: Arc::clone(&empty_str), - rating_count_kinopoisk: 0, - country: Arc::clone(&empty_str), - genre: Arc::clone(&empty_str), - backdrop_path: vec![], - duration_secs: "0".intern(), - duration: Arc::clone(&empty_str), - video: Value::Array(Vec::new()), - audio: Value::Array(Vec::new()), - bitrate: 0, - rating: Arc::clone(&empty_str), - runtime: Arc::clone(&empty_str), - status: Arc::clone(&empty_str), - }, - movie_data: XtreamVideoMovieData { - stream_id: 0, - name: Arc::clone(&empty_str), - added: Arc::clone(&empty_str), - category_id: Arc::clone(&empty_str), - category_ids: vec![], - container_extension: Arc::clone(&empty_str), - custom_sid: None, - direct_source: empty_str, - }, - } - } } #[cfg(test)] -mod tests { - use super::*; - use crate::model::{PlaylistItemTypeSet, SeriesStreamDetailProperties, VirtualId}; +mod tests; - fn sample_options() -> XtreamMappingOptions { - XtreamMappingOptions { - flags: XtreamMappingFlags::RewriteResourceUrl.into(), - force_redirect: None, - reverse_item_types: PlaylistItemTypeSet::empty(), - resource_proxy_item_types: PlaylistItemTypeSet::empty(), - username: "user".to_string(), - password: "pass".to_string(), - base_url: "http://proxy.example".to_string(), - web_ui_request: false, - encrypt_secret: [0u8; 16], - resource_host_policies: Arc::default(), - } - } +mod series; +mod video; - #[test] - fn series_episode_info_document_includes_episode_plot() { - let properties = StreamProperties::Series(Box::new(SeriesStreamProperties { - name: "Example Show".into(), - series_id: 1, - details: Some(SeriesStreamDetailProperties { - year: None, - seasons: None, - episodes: Some(vec![SeriesStreamDetailEpisodeProperties { - id: 101, - episode_num: 1, - season: 1, - title: "Pilot".into(), - container_extension: "mkv".into(), - custom_sid: None, - added: "1700000000".into(), - direct_source: "".into(), - tmdb: None, - release_date: "2024-01-01".into(), - series_release_date: None, - plot: Some("Episode plot".into()), - crew: None, - duration_secs: 0, - duration: "".into(), - movie_image: "".into(), - bitrate: 0, - rating: None, - video: None, - audio: None, - }]), - }), - ..SeriesStreamProperties::default() - })); - - let XtreamInfoDocument::Series(doc) = - properties.to_info_document(&sample_options(), PlaylistItemType::SeriesInfo, VirtualId::new(42), 7) - else { - panic!("expected series info document"); - }; - - let episode = doc.episodes.get("1").and_then(|episodes| episodes.first()).expect("episode missing"); - assert_eq!(episode.info.plot.as_deref(), Some("Episode plot")); - } -} +pub use series::{ + XtreamSeriesEpisodeInfoData, XtreamSeriesEpisodeInfoDoc, XtreamSeriesInfoData, XtreamSeriesInfoDoc, + XtreamSeriesSeasonInfoDoc, +}; +pub use video::{XtreamVideoInfoData, XtreamVideoInfoDoc, XtreamVideoMovieData}; diff --git a/shared/src/model/playlist_info_document/series.rs b/shared/src/model/playlist_info_document/series.rs new file mode 100644 index 000000000..c92e708f0 --- /dev/null +++ b/shared/src/model/playlist_info_document/series.rs @@ -0,0 +1,306 @@ +use crate::{ + concat_string, + model::{ + info_doc_utils::InfoDocUtils, PlaylistItemType, SeriesStreamDetailEpisodeProperties, + SeriesStreamDetailSeasonProperties, SeriesStreamProperties, StreamProperties, VirtualId, XtreamCluster, + XtreamMappingFlags, XtreamMappingOptions, + }, + utils::{ + arc_str_null_is_none_serde, arc_str_option_null_if_empty_serde, arc_str_option_serde, arc_str_serde, + arc_str_vec_serde, Internable, + }, +}; +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::sync::Arc; + +#[inline] +fn build_season_episode_field(season: u32, episode: u32, field: &str) -> String { + concat_string!("nfo_ep_", &season.to_string(), "_", &episode.to_string(), "_", field) +} + +#[inline] +fn build_season_field(season: u32, field: &str) -> String { concat_string!("nfo_s_", &season.to_string(), "_", field) } + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct XtreamSeriesInfoDoc { + #[serde(default)] + pub seasons: Vec, + pub info: XtreamSeriesInfoData, + pub episodes: IndexMap>, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct XtreamSeriesSeasonInfoDoc { + #[serde(default, with = "arc_str_serde")] + pub name: Arc, + #[serde(default, with = "arc_str_serde")] + pub episode_count: Arc, + #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] + pub overview: Option>, + #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] + pub air_date: Option>, + #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] + pub cover: Option>, + #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] + pub cover_tmdb: Option>, + #[serde(default)] + pub season_number: u32, + #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] + pub cover_big: Option>, + #[serde(default, rename = "releaseDate", with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] + pub release_date: Option>, + #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] + pub duration: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct XtreamSeriesInfoData { + #[serde(with = "arc_str_serde")] + pub name: Arc, + #[serde(with = "arc_str_serde")] + pub cover: Arc, + #[serde(with = "arc_str_serde")] + pub plot: Arc, + #[serde(with = "arc_str_serde")] + pub cast: Arc, + #[serde(with = "arc_str_serde")] + pub director: Arc, + #[serde(with = "arc_str_serde")] + pub genre: Arc, + #[serde(rename = "releaseDate", with = "arc_str_serde")] + pub release_date_alternate: Arc, + #[serde(with = "arc_str_serde")] + pub release_date: Arc, + #[serde(with = "arc_str_serde")] + pub last_modified: Arc, + #[serde(with = "arc_str_serde")] + pub rating: Arc, + #[serde(with = "arc_str_serde")] + pub rating_5based: Arc, + #[serde(with = "arc_str_vec_serde")] + pub backdrop_path: Vec>, + #[serde(with = "arc_str_serde")] + pub tmdb: Arc, + #[serde(with = "arc_str_serde")] + pub youtube_trailer: Arc, + #[serde(with = "arc_str_serde")] + pub episode_run_time: Arc, + #[serde(with = "arc_str_serde")] + pub category_id: Arc, + pub category_ids: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct XtreamSeriesEpisodeInfoDoc { + #[serde(with = "arc_str_serde")] + pub id: Arc, + pub episode_num: u32, + #[serde(with = "arc_str_serde")] + pub title: Arc, + #[serde(with = "arc_str_null_is_none_serde")] + pub container_extension: Arc, + pub info: XtreamSeriesEpisodeInfoData, + #[serde(default, with = "arc_str_option_null_if_empty_serde")] + pub custom_sid: Option>, + #[serde(with = "arc_str_serde")] + pub added: Arc, + pub season: u32, + #[serde(with = "arc_str_serde")] + pub direct_source: Arc, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct XtreamSeriesEpisodeInfoData { + #[serde(with = "arc_str_serde")] + pub air_date: Arc, + #[serde(default, with = "arc_str_option_serde", skip_serializing_if = "Option::is_none")] + pub crew: Option>, + pub rating: f64, + #[serde(rename = "id")] + pub tmdb_id: u32, + #[serde(default, with = "arc_str_option_serde")] + pub plot: Option>, + #[serde(with = "arc_str_serde")] + pub movie_image: Arc, + pub duration_secs: u32, + #[serde(with = "arc_str_serde")] + pub duration: Arc, + pub video: Value, + pub audio: Value, + pub bitrate: u32, +} + +impl StreamProperties { + pub(super) fn series_to_info_document( + &self, + options: &XtreamMappingOptions, + series: &SeriesStreamProperties, + item_type: PlaylistItemType, + virtual_id: VirtualId, + category_id: u32, + ) -> XtreamSeriesInfoDoc { + XtreamSeriesInfoDoc { + seasons: if let Some(seasons) = series.details.as_ref().and_then(|d| d.seasons.as_ref()) { + self.series_seasons_to_info_document(options, item_type, virtual_id, seasons) + } else { + Vec::new() + }, + info: XtreamSeriesInfoData { + name: Arc::clone(&series.name), + cover: options + .get_resource_url(XtreamCluster::Series, item_type, virtual_id, series.cover.as_ref(), "cover") + .intern(), + plot: series.plot.as_ref().map_or_else(|| "".intern(), Arc::clone), + cast: Arc::clone(&series.cast), + director: Arc::clone(&series.director), + genre: series.genre.as_ref().map_or_else(|| "".intern(), Arc::clone), + release_date: series.release_date.as_ref().map_or_else(|| "".intern(), Arc::clone), + release_date_alternate: series.release_date.as_ref().map_or_else(|| "".intern(), Arc::clone), + last_modified: series.last_modified.as_ref().map_or_else(|| "".intern(), Arc::clone), + rating: InfoDocUtils::limited(series.rating).intern(), + rating_5based: InfoDocUtils::limited(series.rating_5based).intern(), + backdrop_path: series.backdrop_path.as_ref().map_or_else(Vec::new, |b| { + b.iter() + .enumerate() + .map(|(idx, p)| { + options + .get_bd_path_resource_url(XtreamCluster::Series, item_type, virtual_id, p, "", idx) + .intern() + }) + .collect() + }), + tmdb: series.tmdb.map_or_else(|| "".intern(), |value| value.to_string().intern()), + youtube_trailer: Arc::clone(&series.youtube_trailer), + episode_run_time: series.episode_run_time.as_ref().map_or_else(|| "".intern(), Arc::clone), + category_id: category_id.to_string().intern(), + category_ids: vec![category_id], + }, + episodes: if let Some(episodes) = series.details.as_ref().and_then(|d| d.episodes.as_ref()) { + self.series_episodes_to_info_document(options, item_type, virtual_id, episodes) + } else { + IndexMap::new() + }, + } + } + + pub(super) fn series_seasons_to_info_document( + &self, + options: &XtreamMappingOptions, + item_type: PlaylistItemType, + virtual_id: VirtualId, + seasons: &[SeriesStreamDetailSeasonProperties], + ) -> Vec { + seasons + .iter() + .map(|season| XtreamSeriesSeasonInfoDoc { + name: Arc::clone(&season.name), + season_number: season.season_number, + episode_count: season.episode_count.intern(), + overview: season.overview.as_ref().map(|v| { + if v.starts_with("http://") || v.starts_with("https://") { + options + .get_resource_url( + XtreamCluster::Series, + item_type, + virtual_id, + v, + &build_season_field(season.season_number, "overview"), + ) + .intern() + } else { + Arc::clone(v) + } + }), + air_date: season.air_date.as_ref().map(Arc::clone), + cover: season.cover.as_ref().map(|v| { + options + .get_resource_url( + XtreamCluster::Series, + item_type, + virtual_id, + v, + &build_season_field(season.season_number, "cover"), + ) + .intern() + }), + cover_tmdb: season.cover_tmdb.as_ref().map(|v| { + options + .get_resource_url( + XtreamCluster::Series, + item_type, + virtual_id, + v, + &build_season_field(season.season_number, "cover_tmdb"), + ) + .intern() + }), + cover_big: season.cover_big.as_ref().map(|v| { + options + .get_resource_url( + XtreamCluster::Series, + item_type, + virtual_id, + v, + &build_season_field(season.season_number, "cover_big"), + ) + .intern() + }), + release_date: season.air_date.as_ref().map(Arc::clone), + duration: season.duration.as_ref().map(Arc::clone), + }) + .collect() + } + + pub(super) fn series_episodes_to_info_document( + &self, + options: &XtreamMappingOptions, + item_type: PlaylistItemType, + virtual_id: VirtualId, + episodes: &[SeriesStreamDetailEpisodeProperties], + ) -> IndexMap> { + let empty_str = "".intern(); + let mut map: IndexMap> = IndexMap::new(); + for ep in episodes { + let doc = XtreamSeriesEpisodeInfoDoc { + id: ep.id.intern(), + episode_num: ep.episode_num, + title: Arc::clone(&ep.title), + container_extension: Arc::clone(&ep.container_extension), + info: XtreamSeriesEpisodeInfoData { + tmdb_id: ep.tmdb.unwrap_or_default(), + air_date: Arc::clone(&ep.release_date), + crew: ep.crew.as_ref().map(Arc::clone), + plot: ep.plot.as_ref().map(Arc::clone), + rating: ep.rating.unwrap_or_default(), + movie_image: options + .get_resource_url( + XtreamCluster::Series, + item_type, + virtual_id, + &ep.movie_image, + &build_season_episode_field(ep.season, ep.episode_num, "movie_image"), + ) + .intern(), + duration: Arc::clone(&ep.duration), + duration_secs: ep.duration_secs, + video: InfoDocUtils::build_value(ep.video.as_ref().map(Arc::as_ref)), + audio: InfoDocUtils::build_value(ep.audio.as_ref().map(Arc::as_ref)), + bitrate: ep.bitrate, + }, + custom_sid: ep.custom_sid.as_ref().map(Arc::clone), + added: Arc::clone(&ep.added), + season: ep.season, + direct_source: if options.flags.contains(XtreamMappingFlags::SkipSeriesDirectSource) { + Arc::clone(&empty_str) + } else { + Arc::clone(&ep.direct_source) + }, + }; + map.entry(ep.season).or_default().push(doc); + } + + map.into_iter().map(|(season, episodes)| (season.to_string(), episodes)).collect() + } +} diff --git a/shared/src/model/playlist_info_document/tests.rs b/shared/src/model/playlist_info_document/tests.rs new file mode 100644 index 000000000..aebc16629 --- /dev/null +++ b/shared/src/model/playlist_info_document/tests.rs @@ -0,0 +1,65 @@ +use super::*; +use crate::model::{ + PlaylistItemTypeSet, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, SeriesStreamProperties, + VirtualId, XtreamMappingFlags, +}; +use std::sync::Arc; + +fn sample_options() -> XtreamMappingOptions { + XtreamMappingOptions { + flags: XtreamMappingFlags::RewriteResourceUrl.into(), + force_redirect: None, + reverse_item_types: PlaylistItemTypeSet::empty(), + resource_proxy_item_types: PlaylistItemTypeSet::empty(), + username: "user".to_string(), + password: "pass".to_string(), + base_url: "http://proxy.example".to_string(), + web_ui_request: false, + encrypt_secret: [0u8; 16], + resource_host_policies: Arc::default(), + } +} + +#[test] +fn series_episode_info_document_includes_episode_plot() { + let properties = StreamProperties::Series(Box::new(SeriesStreamProperties { + name: "Example Show".into(), + series_id: 1, + details: Some(SeriesStreamDetailProperties { + year: None, + seasons: None, + episodes: Some(vec![SeriesStreamDetailEpisodeProperties { + id: 101, + episode_num: 1, + season: 1, + title: "Pilot".into(), + container_extension: "mkv".into(), + custom_sid: None, + added: "1700000000".into(), + direct_source: "".into(), + tmdb: None, + release_date: "2024-01-01".into(), + series_release_date: None, + plot: Some("Episode plot".into()), + crew: None, + duration_secs: 0, + duration: "".into(), + movie_image: "".into(), + bitrate: 0, + rating: None, + video: None, + audio: None, + }]), + }), + ..SeriesStreamProperties::default() + })); + + let XtreamInfoDocument::Series(doc) = + properties.to_info_document(&sample_options(), PlaylistItemType::SeriesInfo, VirtualId::new(42), 7) + else { + panic!("expected series info document"); + }; + + let episode = doc.episodes.get("1").and_then(|episodes| episodes.first()).expect("episode missing"); + assert_eq!(episode.info.plot.as_deref(), Some("Episode plot")); +} diff --git a/shared/src/model/playlist_info_document/video.rs b/shared/src/model/playlist_info_document/video.rs new file mode 100644 index 000000000..3bc641985 --- /dev/null +++ b/shared/src/model/playlist_info_document/video.rs @@ -0,0 +1,263 @@ +use crate::{ + model::{ + info_doc_utils::InfoDocUtils, PlaylistItemType, StreamProperties, VideoStreamProperties, VirtualId, + XtreamCluster, XtreamMappingFlags, XtreamMappingOptions, + }, + utils::{ + arc_str_null_is_none_serde, arc_str_option_null_if_empty_serde, arc_str_serde, arc_str_vec_serde, Internable, + }, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::sync::Arc; + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct XtreamVideoInfoDoc { + pub info: XtreamVideoInfoData, + pub movie_data: XtreamVideoMovieData, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct XtreamVideoInfoData { + #[serde(with = "arc_str_serde")] + pub kinopoisk_url: Arc, + #[serde(with = "arc_str_serde")] + pub tmdb_id: Arc, + #[serde(with = "arc_str_serde")] + pub name: Arc, + #[serde(with = "arc_str_serde")] + pub o_name: Arc, + #[serde(with = "arc_str_serde")] + pub cover_big: Arc, + #[serde(with = "arc_str_serde")] + pub movie_image: Arc, + #[serde(rename = "releasedate", with = "arc_str_serde")] + pub release_date: Arc, + pub episode_run_time: u32, + #[serde(with = "arc_str_serde")] + pub youtube_trailer: Arc, + #[serde(with = "arc_str_serde")] + pub director: Arc, + #[serde(with = "arc_str_serde")] + pub actors: Arc, + #[serde(with = "arc_str_serde")] + pub cast: Arc, + #[serde(with = "arc_str_serde")] + pub description: Arc, + #[serde(with = "arc_str_serde")] + pub plot: Arc, + #[serde(with = "arc_str_serde")] + pub age: Arc, + #[serde(with = "arc_str_serde")] + pub mpaa_rating: Arc, + pub rating_count_kinopoisk: u32, + #[serde(with = "arc_str_serde")] + pub country: Arc, + #[serde(with = "arc_str_serde")] + pub genre: Arc, + #[serde(with = "arc_str_vec_serde")] + pub backdrop_path: Vec>, + #[serde(with = "arc_str_serde")] + pub duration_secs: Arc, + #[serde(with = "arc_str_serde")] + pub duration: Arc, + pub video: Value, + pub audio: Value, + pub bitrate: u32, + #[serde(with = "arc_str_serde")] + pub rating: Arc, + #[serde(with = "arc_str_serde")] + pub runtime: Arc, + #[serde(with = "arc_str_serde")] + pub status: Arc, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct XtreamVideoMovieData { + pub stream_id: u32, + #[serde(with = "arc_str_serde")] + pub name: Arc, + #[serde(with = "arc_str_serde")] + pub added: Arc, + #[serde(with = "arc_str_serde")] + pub category_id: Arc, + pub category_ids: Vec, + #[serde(with = "arc_str_null_is_none_serde")] + pub container_extension: Arc, + #[serde(default, with = "arc_str_option_null_if_empty_serde")] + pub custom_sid: Option>, + #[serde(with = "arc_str_serde")] + pub direct_source: Arc, +} + +impl Default for XtreamVideoInfoDoc { + fn default() -> Self { + let empty_str = "".intern(); + Self { + info: XtreamVideoInfoData { + kinopoisk_url: Arc::clone(&empty_str), + tmdb_id: Arc::clone(&empty_str), + name: Arc::clone(&empty_str), + o_name: Arc::clone(&empty_str), + cover_big: Arc::clone(&empty_str), + movie_image: Arc::clone(&empty_str), + release_date: Arc::clone(&empty_str), + episode_run_time: 0, + youtube_trailer: Arc::clone(&empty_str), + director: Arc::clone(&empty_str), + actors: Arc::clone(&empty_str), + cast: Arc::clone(&empty_str), + description: Arc::clone(&empty_str), + plot: Arc::clone(&empty_str), + age: Arc::clone(&empty_str), + mpaa_rating: Arc::clone(&empty_str), + rating_count_kinopoisk: 0, + country: Arc::clone(&empty_str), + genre: Arc::clone(&empty_str), + backdrop_path: vec![], + duration_secs: "0".intern(), + duration: Arc::clone(&empty_str), + video: Value::Array(Vec::new()), + audio: Value::Array(Vec::new()), + bitrate: 0, + rating: Arc::clone(&empty_str), + runtime: Arc::clone(&empty_str), + status: Arc::clone(&empty_str), + }, + movie_data: XtreamVideoMovieData { + stream_id: 0, + name: Arc::clone(&empty_str), + added: Arc::clone(&empty_str), + category_id: Arc::clone(&empty_str), + category_ids: vec![], + container_extension: Arc::clone(&empty_str), + custom_sid: None, + direct_source: empty_str, + }, + } + } +} + +impl StreamProperties { + pub(super) fn video_to_info_document( + &self, + options: &XtreamMappingOptions, + video: &VideoStreamProperties, + item_type: PlaylistItemType, + virtual_id: VirtualId, + category_id: u32, + ) -> XtreamVideoInfoDoc { + let stream_icon = options + .get_resource_url(XtreamCluster::Video, item_type, virtual_id, &self.get_stream_icon(), "logo") + .intern(); + let empty_str = "".intern(); + let zero_str = "0".intern(); + + let info = if let Some(details) = video.details.as_ref() { + XtreamVideoInfoData { + kinopoisk_url: details.kinopoisk_url.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + tmdb_id: video.tmdb.unwrap_or_default().to_string().intern(), + name: Arc::clone(&video.name), + o_name: details.o_name.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + cover_big: options + .get_resource_url( + XtreamCluster::Video, + item_type, + virtual_id, + details.cover_big.as_ref().map_or("", Arc::as_ref), + "nfo_cover_big", + ) + .intern(), + movie_image: options + .get_resource_url( + XtreamCluster::Video, + item_type, + virtual_id, + details.movie_image.as_ref().map_or("", Arc::as_ref), + "nfo_movie_image", + ) + .intern(), + release_date: details.release_date.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + episode_run_time: details.episode_run_time.unwrap_or_default(), + youtube_trailer: details.youtube_trailer.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + director: details.director.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + actors: details.actors.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + cast: details.cast.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + description: details.description.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + plot: details.plot.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + age: details.age.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + mpaa_rating: details.mpaa_rating.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + rating_count_kinopoisk: details.rating_count_kinopoisk, + country: details.country.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + genre: details.genre.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + backdrop_path: details.backdrop_path.as_deref().map_or_else(Vec::new, |b| { + b.iter() + .enumerate() + .map(|(idx, p)| { + options + .get_bd_path_resource_url(XtreamCluster::Video, item_type, virtual_id, p, "nfo_", idx) + .intern() + }) + .collect() + }), + duration_secs: details.duration_secs.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + duration: details.duration.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + video: InfoDocUtils::build_value(details.video.as_ref().map(Arc::as_ref)), + audio: InfoDocUtils::build_value(details.audio.as_ref().map(Arc::as_ref)), + bitrate: details.bitrate, + rating: InfoDocUtils::limited(video.rating.unwrap_or_default()).intern(), + runtime: details.runtime.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + status: details.status.as_ref().map_or_else(|| Arc::clone(&empty_str), Arc::clone), + } + } else { + XtreamVideoInfoData { + kinopoisk_url: Arc::clone(&empty_str), + tmdb_id: video.tmdb.unwrap_or_default().to_string().intern(), + name: Arc::clone(&video.name), + o_name: Arc::clone(&video.name), + cover_big: stream_icon.clone(), + movie_image: stream_icon.clone(), + release_date: Arc::clone(&empty_str), + episode_run_time: 0, + youtube_trailer: Arc::clone(&empty_str), + director: Arc::clone(&empty_str), + actors: Arc::clone(&empty_str), + cast: Arc::clone(&empty_str), + description: Arc::clone(&empty_str), + plot: Arc::clone(&empty_str), + age: Arc::clone(&empty_str), + mpaa_rating: Arc::clone(&empty_str), + rating_count_kinopoisk: 0, + country: Arc::clone(&empty_str), + genre: Arc::clone(&empty_str), + backdrop_path: vec![Arc::clone(&stream_icon)], + duration_secs: Arc::clone(&zero_str), + duration: Arc::clone(&zero_str), + video: Value::Array(Vec::new()), + audio: Value::Array(Vec::new()), + bitrate: 0, + rating: InfoDocUtils::limited(video.rating.unwrap_or_default()).intern(), + runtime: zero_str, + status: "Released".intern(), + } + }; + + XtreamVideoInfoDoc { + info, + movie_data: XtreamVideoMovieData { + stream_id: virtual_id.get(), + name: Arc::clone(&video.name), + added: Arc::clone(&video.added), + category_id: category_id.to_string().intern(), + category_ids: vec![category_id], + container_extension: Arc::clone(&video.container_extension), + custom_sid: video.custom_sid.as_ref().map(Arc::clone), + direct_source: if options.flags.contains(XtreamMappingFlags::SkipVideoDirectSource) { + Arc::clone(&empty_str) + } else { + Arc::clone(&video.direct_source) + }, + }, + } + } +} diff --git a/shared/src/model/recording.rs b/shared/src/model/recording.rs index 67a099c03..5a57182e5 100644 --- a/shared/src/model/recording.rs +++ b/shared/src/model/recording.rs @@ -7,758 +7,21 @@ use super::{ transfer::{TaskPriorityDto, TransferStatusDto}, user_id::UserId, }; -use std::fmt; - -/// Media kind of a recording task. Determined by the server from the -/// resolved catalog item; never accepted from a client. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RecordingKind { - /// Scheduled live capture. Executed with ffmpeg, never resumable. - Live, - /// Video on demand. Executed over resumable HTTP. - Vod, - /// Series episode. Executed over resumable HTTP. - Series, -} - -impl RecordingKind { - /// Live is captured within a programme window; VOD and Series run to EOF. - pub fn is_scheduled(self) -> bool { matches!(self, Self::Live) } - - /// Only VOD and Series may be paused and resumed. - pub fn is_resumable(self) -> bool { matches!(self, Self::Vod | Self::Series) } - - pub fn as_str(self) -> &'static str { - match self { - Self::Live => "live", - Self::Vod => "vod", - Self::Series => "series", - } - } -} - -impl fmt::Display for RecordingKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.as_str()) } -} - -/// Which controls a recording currently offers. -/// -/// Computed server-side from the transition graph and carried on the DTO so a -/// client renders buttons for exactly the commands the server would accept. -// One flag per control is the point: the set maps 1:1 onto the buttons a -// client renders, and collapsing it would hide which command was refused. -#[allow(clippy::struct_excessive_bools)] -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct RecordingAllowedActions { - pub pause: bool, - pub resume: bool, - pub cancel: bool, - pub retry: bool, - pub edit: bool, - pub remove: bool, -} - -/// The caller's own quota, as carried in the recording snapshot. -/// -/// Only the caller's private pool and the shared pool: there is no -/// per-user breakdown here, because one user's consumption is not another -/// user's business. -#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct RecordingQuotaSummaryDto { - pub private_used_bytes: u64, - pub private_limit_bytes: Option, - pub shared_used_bytes: u64, - pub shared_limit_bytes: Option, -} - -/// Owner-safe public projection of a recording task. This is the only -/// recording shape crossing the API or WebSocket boundary. It never carries -/// the source URL, provider/source identifiers, configured headers, -/// absolute/relative/partial paths, transfer validators, or another user's -/// owner identifier. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct RecordingTaskDto { - pub id: String, - pub title: String, - pub kind: RecordingKind, - pub priority: TaskPriorityDto, - pub status: TransferStatusDto, - pub retry_attempts: u8, - pub transferred_bytes: u64, - #[serde(skip_serializing_if = "Option::is_none")] - pub total_bytes: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub next_retry_at: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - /// The provider ignored a byte-range request. Restarting from zero needs - /// the viewer's explicit confirmation before the partial is discarded. - #[serde(default)] - pub restart_from_beginning_required: bool, - /// `Some` only when the viewer owns the task; never another user's id. - #[serde(skip_serializing_if = "Option::is_none")] - pub owner_id: Option, - pub visibility: RecordingVisibility, - #[serde(skip_serializing_if = "Option::is_none")] - pub channel_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub channel_name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub program_title: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub program_start: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub program_end: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub scheduled_start: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub scheduled_end: Option, - pub pre_roll_secs: u64, - pub post_roll_secs: u64, - #[serde(skip_serializing_if = "Option::is_none")] - pub completed_at: Option, - /// Filename component of the final relative path. The server does not - /// expose the directory structure or the recording root. - #[serde(skip_serializing_if = "Option::is_none")] - pub filename: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub epg: Option, - /// Immutable provenance for tasks generated by a recurring rule. - #[serde(skip_serializing_if = "Option::is_none")] - pub rule_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub occurrence_key: Option, - /// What the server would accept for this recording right now, before the - /// viewer's permissions are applied. - #[serde(default)] - pub allowed_actions: RecordingAllowedActions, -} - -impl RecordingTaskDto { - /// Lifecycle partition for the UI: non-terminal tasks are current, - /// terminal ones are completed. - pub fn is_terminal(&self) -> bool { - matches!(self.status, TransferStatusDto::Completed | TransferStatusDto::Failed | TransferStatusDto::Cancelled) - } - - /// Effective duration of a scheduled window, when both bounds are known. - pub fn scheduled_duration_secs(&self) -> Option { - let (start, end) = self.scheduled_start.zip(self.scheduled_end)?; - u64::try_from(end.saturating_sub(start)).ok() - } -} - -/// Recording owner. Every task is owned by exactly one user; there is no -/// unowned or admin-legacy record. -/// -/// Adjacently tagged so the newtype `User(UserId)` serializes with the -/// content field name `user`. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(tag = "kind", content = "user", rename_all = "snake_case")] -pub enum RecordingOwner { - User(UserId), -} - -impl RecordingOwner { - pub fn user_id(&self) -> &UserId { - let Self::User(uid) = self; - uid - } -} - -/// Recording visibility. Shared recordings are visible to every user with -/// `recording.read`; private recordings are visible only to the owner. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RecordingVisibility { - #[default] - Private, - Shared, -} - -/// Server-owned source identifiers. The stream URL is never accepted from a -/// client; the server resolves it from these identifiers at execute time. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct RecordingSource { - pub target_id: String, - pub virtual_id: String, - pub input_name: String, - #[serde(default)] - pub cluster: super::XtreamCluster, -} - -impl RecordingSource { - pub fn new(target_id: impl Into, virtual_id: impl Into, input_name: impl Into) -> Self { - Self { - target_id: target_id.into(), - virtual_id: virtual_id.into(), - input_name: input_name.into(), - cluster: super::XtreamCluster::Live, - } - } - - pub fn with_cluster(mut self, cluster: super::XtreamCluster) -> Self { - self.cluster = cluster; - self - } -} - -/// Server-owned source identifiers as a client submits them. -/// -/// This is the request half of [`RecordingSource`]: the client names the -/// catalog item, and the server resolves the stream URL from it at execute -/// time. A URL is never accepted from a client. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct RecordingSourceRequest { - pub target_id: String, - pub virtual_id: String, - pub cluster: super::XtreamCluster, - pub input_name: String, -} - -/// The one create-recording wire contract, shared by the REST handler and -/// the frontend client so the two cannot drift. -/// -/// Unknown fields are rejected: a client that sends a field this build does -/// not understand has a different idea of what it is asking for, and -/// silently ignoring it would record the wrong thing. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct CreateRecordingRequest { - pub source: RecordingSourceRequest, - pub program_title: String, - /// Programme interval, Live only. Rejected for VOD and series. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub program_start: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub program_end: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub pre_roll_secs: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub post_roll_secs: Option, - pub visibility: RecordingVisibility, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub channel_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub channel_name: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub epg: Option, -} - -/// EPG episode metadata with tri-state airing provenance. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, Default)] -pub struct EpgEpisodeMetadata { - pub programme_id: Option, - pub series_id: Option, - pub episode_id: Option, - pub season: Option, - pub episode: Option, - #[serde(default)] - pub airing: AiringStatus, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum AiringStatus { - #[default] - Unknown, - New, - Repeat, -} - -/// Lifecycle notification marker persisted alongside the task. At-most-once -/// delivery is enforced by recording this marker in the same queue mutation -/// as the lifecycle transition; the notification adapter is gated on it. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum NotificationMarkerKind { - Started, - Completed, - Failed, -} - -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct NotificationMarker { - pub kind: NotificationMarkerKind, - pub attempted_at: i64, -} - -impl NotificationMarker { - pub fn new(kind: NotificationMarkerKind, attempted_at: i64) -> Self { Self { kind, attempted_at } } -} - -/// Recurring-rule provenance. `rule_id` and `occurrence_key` are immutable for -/// the life of a task; the rule may be deleted but the task retains the -/// historical values. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, Default)] -pub struct RecordingProvenance { - pub rule_id: Option, - pub occurrence_key: Option, -} - -/// Terminal state of a task before deletion began. Only `Completed`, `Failed`, -/// and `Cancelled` are valid previous states; the worker rejects transitions -/// from `Scheduled`/`Queued`/`Running`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum DeletionPreviousState { - Completed, - Failed, - Cancelled, -} - -impl DeletionPreviousState { - pub fn as_str(&self) -> &'static str { - match self { - Self::Completed => "completed", - Self::Failed => "failed", - Self::Cancelled => "cancelled", - } - } -} - -/// Recording metadata. Every recording task carries exactly one of these; -/// the programme interval fields are populated for `RecordingKind::Live` -/// only. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct RecordingMetadata { - pub owner: RecordingOwner, - pub visibility: RecordingVisibility, - /// Server-owned source identity. Never client supplied. - pub source: RecordingSource, - /// Original EPG interval (Unix seconds). Live only. - pub program_start: Option, - pub program_end: Option, - /// Effective padded interval (Unix seconds). When the program is - /// currently airing, this is clamped to `now..scheduled_end`. Live only. - pub scheduled_start: Option, - pub scheduled_end: Option, - pub pre_roll_secs: u64, - pub post_roll_secs: u64, - pub channel_id: Option, - pub channel_name: Option, - pub program_title: Option, - pub epg: Option, - /// Playlist group of the recorded item. Organised layouts file the - /// recording under it. - pub group: Option, - /// Series name shared by every episode of a series. Series only. - pub series_name: Option, - #[serde(default)] - pub provenance: RecordingProvenance, - /// Final relative path below the recording root. `None` until the task - /// is reserved or finalized. - pub relative_path: Option, - /// Worker-owned partial path while a recording is in progress. - pub partial_relative_path: Option, - /// Strong `ETag` captured from the first response, so a resume can prove - /// the provider has not replaced the file underneath a partial download. - /// Weak validators are never stored: they only promise semantic - /// equivalence, so two matching weak tags can still be different bytes. - /// Internal; never projected into a DTO. - #[serde(default)] - pub resume_etag: Option, - /// `Last-Modified` captured from the first response. Used only when no - /// strong `ETag` was offered. - #[serde(default)] - pub resume_last_modified: Option, - /// Conservative quota reservation. `0` once the task completes. - pub reserved_bytes: u64, - /// Measured partial/final size. Authoritative once the task completes. - pub measured_bytes: u64, - /// Actual completion timestamp. - pub completed_at: Option, - /// Persisted at-most-once notification markers. - #[serde(default)] - pub notification_markers: Vec, - /// Terminal-only previous state while a deletion is in progress. `None` - /// outside the `Deleting` transitional state. - pub deleting_previous_state: Option, -} - -impl RecordingMetadata { - /// Build metadata for a scheduled live recording. - pub fn new_live( - owner: RecordingOwner, - visibility: RecordingVisibility, - source: RecordingSource, - program_start: i64, - program_end: i64, - pre_roll_secs: u64, - post_roll_secs: u64, - ) -> Self { - let scheduled_start = program_start.saturating_sub(pre_roll_secs as i64); - let scheduled_end = program_end.saturating_add(post_roll_secs as i64); - Self { - owner, - visibility, - source, - program_start: Some(program_start), - program_end: Some(program_end), - scheduled_start: Some(scheduled_start), - scheduled_end: Some(scheduled_end), - pre_roll_secs, - post_roll_secs, - channel_id: None, - channel_name: None, - program_title: None, - epg: None, - provenance: RecordingProvenance::default(), - relative_path: None, - partial_relative_path: None, - resume_etag: None, - resume_last_modified: None, - reserved_bytes: 0, - measured_bytes: 0, - completed_at: None, - notification_markers: Vec::new(), - group: None, - series_name: None, - deleting_previous_state: None, - } - } - - /// Build metadata for an immediate VOD or series-episode transfer. These - /// have no programme window and no padding. - pub fn new_media( - owner: RecordingOwner, - visibility: RecordingVisibility, - source: RecordingSource, - title: String, - ) -> Self { - Self { - program_start: None, - program_end: None, - scheduled_start: None, - scheduled_end: None, - program_title: Some(title), - ..Self::new_live(owner, visibility, source, 0, 0, 0, 0) - } - } - - pub fn is_deleting(&self) -> bool { self.deleting_previous_state.is_some() } - - pub fn owner_id(&self) -> &UserId { self.owner.user_id() } - - /// Filename component of the final relative path, if one is reserved. - pub fn filename(&self) -> Option<&str> { self.relative_path.as_deref().and_then(extract_filename_component) } -} - -fn extract_filename_component(relative_path: &str) -> Option<&str> { - let trimmed = relative_path.trim_end_matches('/'); - if trimmed.is_empty() { - return None; - } - std::path::Path::new(trimmed).file_name().and_then(|s| s.to_str()) -} - -/// Execution state of a recording task as persisted. -/// -/// Distinct from [`TransferStatusDto`](super::transfer::TransferStatusDto), -/// which is the wire projection: this one is written to the recording -/// repository, so renaming a variant is a storage-format change. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] -pub enum RecordingTaskState { - #[default] - Queued, - Scheduled, - WaitingForCapacity, - RetryWaiting, - Running, - Paused, - /// Cancellation was requested; the worker still owns the file and the - /// provider slot. Deliberately not terminal -- treating it as terminal - /// lets a deletion unlink the partial the worker is still writing. - Cancelling, - Completed, - Failed, - Cancelled, -} - -impl RecordingTaskState { - pub fn is_terminal(self) -> bool { matches!(self, Self::Completed | Self::Failed | Self::Cancelled) } - - /// Stable label used by the edit rules, logs and errors. - pub fn label(self) -> &'static str { - match self { - Self::Queued => "Queued", - Self::Scheduled => "Scheduled", - Self::WaitingForCapacity => "WaitingForCapacity", - Self::RetryWaiting => "RetryWaiting", - Self::Running => "Running", - Self::Paused => "Paused", - Self::Cancelling => "Cancelling", - Self::Completed => "Completed", - Self::Failed => "Failed", - Self::Cancelled => "Cancelled", - } - } -} - -impl From for super::transfer::TransferStatusDto { - fn from(value: RecordingTaskState) -> Self { - match value { - RecordingTaskState::Queued => Self::Queued, - RecordingTaskState::Scheduled => Self::Scheduled, - RecordingTaskState::WaitingForCapacity => Self::WaitingForCapacity, - RecordingTaskState::RetryWaiting => Self::RetryWaiting, - RecordingTaskState::Running => Self::Running, - RecordingTaskState::Paused => Self::Paused, - RecordingTaskState::Cancelling => Self::Cancelling, - RecordingTaskState::Completed => Self::Completed, - RecordingTaskState::Failed => Self::Failed, - RecordingTaskState::Cancelled => Self::Cancelled, - } - } -} - -/// Monotonic revision counter for the persisted recording queue. Increments -/// once per committed queue mutation. Stored alongside the queue snapshot. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(transparent)] -pub struct QueueRevision(pub u64); - -impl fmt::Display for QueueRevision { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) } -} #[cfg(test)] -mod tests { - use super::*; +mod tests; - fn test_source() -> RecordingSource { RecordingSource::new("target-1", "v-1", "input-a") } - - fn test_owner() -> RecordingOwner { RecordingOwner::User(UserId::from("web:abc")) } - - #[test] - fn legacy_recording_source_defaults_to_live_cluster() { - let source: RecordingSource = - serde_json::from_str(r#"{"target_id":"target","virtual_id":"42","input_name":"input-a"}"#) - .expect("deserialize source"); - - assert_eq!(source.cluster, super::super::XtreamCluster::Live); - } - - #[test] - fn recording_kind_serializes_as_snake_case() { - assert_eq!(serde_json::to_string(&RecordingKind::Vod).expect("serialize"), "\"vod\""); - assert_eq!(serde_json::to_string(&RecordingKind::Series).expect("serialize"), "\"series\""); - assert_eq!(serde_json::to_string(&RecordingKind::Live).expect("serialize"), "\"live\""); - } - - fn create_request_json() -> &'static str { - r#"{"source":{"target_id":"t","virtual_id":"42","cluster":"Video","input_name":"in"},"program_title":"Film","visibility":"private"}"# - } - - #[test] - fn create_request_round_trips_its_minimal_form() { - let request: CreateRecordingRequest = serde_json::from_str(create_request_json()).expect("deserialize"); - assert_eq!(request.visibility, RecordingVisibility::Private); - assert_eq!(request.program_start, None); - assert_eq!(serde_json::to_string(&request).expect("serialize"), create_request_json()); - } - - #[test] - fn create_request_rejects_an_unknown_field() { - // A client that sends a field this build does not understand has a - // different idea of what it is asking for; recording the wrong thing - // is worse than refusing. - let body = r#"{"source":{"target_id":"t","virtual_id":"42","cluster":"Video","input_name":"in"}, - "program_title":"Film","visibility":"private","url":"http://evil/stream.ts"}"#; - let error = serde_json::from_str::(body).expect_err("unknown field must be rejected"); - assert!(error.to_string().contains("url"), "{error}"); - } - - #[test] - fn create_request_rejects_an_unknown_visibility() { - // The visibility is the server's enum on the wire, so a typo in the - // frontend cannot reach the server as a valid body. - let body = r#"{"source":{"target_id":"t","virtual_id":"42","cluster":"Video","input_name":"in"}, - "program_title":"Film","visibility":"pubic"}"#; - assert!(serde_json::from_str::(body).is_err()); - } - - #[test] - fn create_request_rejects_an_unknown_source_field() { - let body = r#"{"source":{"target_id":"t","virtual_id":"42","cluster":"Video","input_name":"in", - "stream_url":"http://evil/"},"program_title":"Film","visibility":"private"}"#; - assert!(serde_json::from_str::(body).is_err()); - } - - #[test] - fn only_live_is_scheduled_and_only_media_is_resumable() { - assert!(RecordingKind::Live.is_scheduled()); - assert!(!RecordingKind::Live.is_resumable()); - for kind in [RecordingKind::Vod, RecordingKind::Series] { - assert!(!kind.is_scheduled(), "{kind} must not be scheduled"); - assert!(kind.is_resumable(), "{kind} must be resumable"); - } - } - - #[test] - fn live_metadata_carries_padded_interval_and_source() { - let meta = RecordingMetadata::new_live( - test_owner(), - RecordingVisibility::Private, - test_source(), - 1_700_000_000, - 1_700_000_900, - 60, - 120, - ); - assert_eq!(meta.scheduled_start, Some(1_700_000_000 - 60)); - assert_eq!(meta.scheduled_end, Some(1_700_000_900 + 120)); - assert_eq!(meta.pre_roll_secs, 60); - assert_eq!(meta.post_roll_secs, 120); - assert!(!meta.is_deleting()); - } - - #[test] - fn media_metadata_has_owner_and_source_without_live_interval() { - let source = test_source().with_cluster(super::super::XtreamCluster::Video); - let meta = - RecordingMetadata::new_media(test_owner(), RecordingVisibility::Private, source, "Movie".to_string()); - - assert_eq!(meta.program_title.as_deref(), Some("Movie")); - assert!(meta.program_start.is_none()); - assert!(meta.program_end.is_none()); - assert!(meta.scheduled_start.is_none()); - assert!(meta.scheduled_end.is_none()); - assert_eq!(meta.pre_roll_secs, 0); - assert_eq!(meta.post_roll_secs, 0); - } - - #[test] - fn recording_owner_round_trips_as_adjacent_tag() { - let owner = RecordingOwner::User(UserId::from("web:alice")); - let json = serde_json::to_string(&owner).expect("serialize"); - assert_eq!(json, r#"{"kind":"user","user":"web:alice"}"#); - let restored: RecordingOwner = serde_json::from_str(&json).expect("deserialize"); - assert_eq!(restored, owner); - assert_eq!(restored.user_id(), &UserId::from("web:alice")); - } - - #[test] - fn deleting_state_round_trip() { - let mut meta = - RecordingMetadata::new_media(test_owner(), RecordingVisibility::Private, test_source(), "M".to_string()); - assert!(!meta.is_deleting()); - meta.deleting_previous_state = Some(DeletionPreviousState::Completed); - assert!(meta.is_deleting()); - let json = serde_json::to_string(&meta).expect("serialize"); - let restored: RecordingMetadata = serde_json::from_str(&json).expect("deserialize"); - assert_eq!(restored.deleting_previous_state, Some(DeletionPreviousState::Completed)); - } - - #[test] - fn metadata_filename_is_the_last_relative_path_component() { - let mut meta = - RecordingMetadata::new_media(test_owner(), RecordingVisibility::Private, test_source(), "M".to_string()); - assert_eq!(meta.filename(), None); - meta.relative_path = Some("users/web:abc/Pilot/pilot.ts".to_string()); - assert_eq!(meta.filename(), Some("pilot.ts")); - } - - #[test] - fn notification_marker_serialization() { - let marker = NotificationMarker::new(NotificationMarkerKind::Completed, 1_700_000_000); - let json = serde_json::to_string(&marker).expect("serialize"); - assert!(json.contains("\"completed\""), "{json}"); - } - - #[test] - fn queue_revision_default_is_zero() { - assert_eq!(QueueRevision::default().0, 0); - assert_eq!(format!("{}", QueueRevision(42)), "42"); - } - - // --- RecordingTaskDto --- - - fn make_dto() -> RecordingTaskDto { - RecordingTaskDto { - id: "rec-1".to_string(), - title: "Pilot".to_string(), - kind: RecordingKind::Series, - priority: TaskPriorityDto::Normal, - status: TransferStatusDto::Running, - retry_attempts: 1, - transferred_bytes: 128, - total_bytes: Some(4096), - next_retry_at: None, - error: None, - restart_from_beginning_required: false, - owner_id: Some(UserId::from("web:abc")), - visibility: RecordingVisibility::Private, - channel_id: None, - channel_name: None, - program_title: Some("Pilot".to_string()), - program_start: None, - program_end: None, - scheduled_start: None, - scheduled_end: None, - pre_roll_secs: 0, - post_roll_secs: 0, - completed_at: None, - filename: Some("pilot.mkv".to_string()), - epg: Some(EpgEpisodeMetadata { - programme_id: None, - series_id: Some("series-1".to_string()), - episode_id: Some("ep-1".to_string()), - season: Some(1), - episode: Some(2), - airing: AiringStatus::New, - }), - rule_id: None, - occurrence_key: None, - allowed_actions: RecordingAllowedActions::default(), - } - } - - #[test] - fn recording_task_dto_round_trips_preserving_kind() { - let dto = make_dto(); - let json = serde_json::to_string(&dto).expect("serialize"); - let restored: RecordingTaskDto = serde_json::from_str(&json).expect("deserialize"); - assert_eq!(restored, dto); - assert_eq!(restored.kind, RecordingKind::Series); - } - - #[test] - fn recording_task_dto_has_no_nested_transfer_or_recording_wrapper() { - let json = serde_json::to_value(make_dto()).expect("serialize"); - assert!(json.get("recording").is_none(), "no nested recording wrapper"); - assert!(json.get("transfer").is_none(), "no nested transfer wrapper"); - assert_eq!(json.get("kind").and_then(serde_json::Value::as_str), Some("series")); - } - - #[test] - fn terminal_partition_matches_completed_failed_cancelled() { - for status in [TransferStatusDto::Completed, TransferStatusDto::Failed, TransferStatusDto::Cancelled] { - assert!(RecordingTaskDto { status, ..make_dto() }.is_terminal(), "{status:?} is terminal"); - } - for status in [ - TransferStatusDto::Scheduled, - TransferStatusDto::Queued, - TransferStatusDto::WaitingForCapacity, - TransferStatusDto::RetryWaiting, - TransferStatusDto::Running, - TransferStatusDto::Paused, - ] { - assert!(!RecordingTaskDto { status, ..make_dto() }.is_terminal(), "{status:?} is current"); - } - } - - #[test] - fn scheduled_duration_is_derived_from_the_padded_window() { - let dto = RecordingTaskDto { - kind: RecordingKind::Live, - scheduled_start: Some(1_700_000_000), - scheduled_end: Some(1_700_003_600), - ..make_dto() - }; - assert_eq!(dto.scheduled_duration_secs(), Some(3_600)); - assert_eq!(make_dto().scheduled_duration_secs(), None); - } -} +mod metadata; +mod request; +mod task; +pub use metadata::{ + AiringStatus, DeletionPreviousState, EpgEpisodeMetadata, NotificationMarker, NotificationMarkerKind, + RecordingMetadata, RecordingProvenance, +}; +pub use request::{ + CreateRecordingRequest, RecordingOwner, RecordingSource, RecordingSourceRequest, RecordingVisibility, +}; +pub use task::{ + QueueRevision, RecordingAllowedActions, RecordingKind, RecordingQuotaSummaryDto, RecordingTaskDto, + RecordingTaskState, +}; diff --git a/shared/src/model/recording/metadata.rs b/shared/src/model/recording/metadata.rs new file mode 100644 index 000000000..71dc3e9fb --- /dev/null +++ b/shared/src/model/recording/metadata.rs @@ -0,0 +1,208 @@ +use super::{RecordingOwner, RecordingSource, RecordingVisibility, UserId}; + +/// EPG episode metadata with tri-state airing provenance. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, Default)] +pub struct EpgEpisodeMetadata { + pub programme_id: Option, + pub series_id: Option, + pub episode_id: Option, + pub season: Option, + pub episode: Option, + #[serde(default)] + pub airing: AiringStatus, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum AiringStatus { + #[default] + Unknown, + New, + Repeat, +} + +/// Lifecycle notification marker persisted alongside the task. At-most-once +/// delivery is enforced by recording this marker in the same queue mutation +/// as the lifecycle transition; the notification adapter is gated on it. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum NotificationMarkerKind { + Started, + Completed, + Failed, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct NotificationMarker { + pub kind: NotificationMarkerKind, + pub attempted_at: i64, +} + +impl NotificationMarker { + pub fn new(kind: NotificationMarkerKind, attempted_at: i64) -> Self { Self { kind, attempted_at } } +} + +/// Recurring-rule provenance. `rule_id` and `occurrence_key` are immutable for +/// the life of a task; the rule may be deleted but the task retains the +/// historical values. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, Default)] +pub struct RecordingProvenance { + pub rule_id: Option, + pub occurrence_key: Option, +} + +/// Terminal state of a task before deletion began. Only `Completed`, `Failed`, +/// and `Cancelled` are valid previous states; the worker rejects transitions +/// from `Scheduled`/`Queued`/`Running`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum DeletionPreviousState { + Completed, + Failed, + Cancelled, +} + +impl DeletionPreviousState { + pub fn as_str(&self) -> &'static str { + match self { + Self::Completed => "completed", + Self::Failed => "failed", + Self::Cancelled => "cancelled", + } + } +} + +/// Recording metadata. Every recording task carries exactly one of these; +/// the programme interval fields are populated for `RecordingKind::Live` +/// only. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct RecordingMetadata { + pub owner: RecordingOwner, + pub visibility: RecordingVisibility, + /// Server-owned source identity. Never client supplied. + pub source: RecordingSource, + /// Original EPG interval (Unix seconds). Live only. + pub program_start: Option, + pub program_end: Option, + /// Effective padded interval (Unix seconds). When the program is + /// currently airing, this is clamped to `now..scheduled_end`. Live only. + pub scheduled_start: Option, + pub scheduled_end: Option, + pub pre_roll_secs: u64, + pub post_roll_secs: u64, + pub channel_id: Option, + pub channel_name: Option, + pub program_title: Option, + pub epg: Option, + /// Playlist group of the recorded item. Organised layouts file the + /// recording under it. + pub group: Option, + /// Series name shared by every episode of a series. Series only. + pub series_name: Option, + #[serde(default)] + pub provenance: RecordingProvenance, + /// Final relative path below the recording root. `None` until the task + /// is reserved or finalized. + pub relative_path: Option, + /// Worker-owned partial path while a recording is in progress. + pub partial_relative_path: Option, + /// Strong `ETag` captured from the first response, so a resume can prove + /// the provider has not replaced the file underneath a partial download. + /// Weak validators are never stored: they only promise semantic + /// equivalence, so two matching weak tags can still be different bytes. + /// Internal; never projected into a DTO. + #[serde(default)] + pub resume_etag: Option, + /// `Last-Modified` captured from the first response. Used only when no + /// strong `ETag` was offered. + #[serde(default)] + pub resume_last_modified: Option, + /// Conservative quota reservation. `0` once the task completes. + pub reserved_bytes: u64, + /// Measured partial/final size. Authoritative once the task completes. + pub measured_bytes: u64, + /// Actual completion timestamp. + pub completed_at: Option, + /// Persisted at-most-once notification markers. + #[serde(default)] + pub notification_markers: Vec, + /// Terminal-only previous state while a deletion is in progress. `None` + /// outside the `Deleting` transitional state. + pub deleting_previous_state: Option, +} + +impl RecordingMetadata { + /// Build metadata for a scheduled live recording. + pub fn new_live( + owner: RecordingOwner, + visibility: RecordingVisibility, + source: RecordingSource, + program_start: i64, + program_end: i64, + pre_roll_secs: u64, + post_roll_secs: u64, + ) -> Self { + let scheduled_start = program_start.saturating_sub(pre_roll_secs as i64); + let scheduled_end = program_end.saturating_add(post_roll_secs as i64); + Self { + owner, + visibility, + source, + program_start: Some(program_start), + program_end: Some(program_end), + scheduled_start: Some(scheduled_start), + scheduled_end: Some(scheduled_end), + pre_roll_secs, + post_roll_secs, + channel_id: None, + channel_name: None, + program_title: None, + epg: None, + provenance: RecordingProvenance::default(), + relative_path: None, + partial_relative_path: None, + resume_etag: None, + resume_last_modified: None, + reserved_bytes: 0, + measured_bytes: 0, + completed_at: None, + notification_markers: Vec::new(), + group: None, + series_name: None, + deleting_previous_state: None, + } + } + + /// Build metadata for an immediate VOD or series-episode transfer. These + /// have no programme window and no padding. + pub fn new_media( + owner: RecordingOwner, + visibility: RecordingVisibility, + source: RecordingSource, + title: String, + ) -> Self { + Self { + program_start: None, + program_end: None, + scheduled_start: None, + scheduled_end: None, + program_title: Some(title), + ..Self::new_live(owner, visibility, source, 0, 0, 0, 0) + } + } + + pub fn is_deleting(&self) -> bool { self.deleting_previous_state.is_some() } + + pub fn owner_id(&self) -> &UserId { self.owner.user_id() } + + /// Filename component of the final relative path, if one is reserved. + pub fn filename(&self) -> Option<&str> { self.relative_path.as_deref().and_then(extract_filename_component) } +} + +fn extract_filename_component(relative_path: &str) -> Option<&str> { + let trimmed = relative_path.trim_end_matches('/'); + if trimmed.is_empty() { + return None; + } + std::path::Path::new(trimmed).file_name().and_then(|s| s.to_str()) +} diff --git a/shared/src/model/recording/request.rs b/shared/src/model/recording/request.rs new file mode 100644 index 000000000..ce1272f44 --- /dev/null +++ b/shared/src/model/recording/request.rs @@ -0,0 +1,99 @@ +use super::{EpgEpisodeMetadata, UserId}; + +/// Recording owner. Every task is owned by exactly one user; there is no +/// unowned or admin-legacy record. +/// +/// Adjacently tagged so the newtype `User(UserId)` serializes with the +/// content field name `user`. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(tag = "kind", content = "user", rename_all = "snake_case")] +pub enum RecordingOwner { + User(UserId), +} + +impl RecordingOwner { + pub fn user_id(&self) -> &UserId { + let Self::User(uid) = self; + uid + } +} + +/// Recording visibility. Shared recordings are visible to every user with +/// `recording.read`; private recordings are visible only to the owner. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RecordingVisibility { + #[default] + Private, + Shared, +} + +/// Server-owned source identifiers. The stream URL is never accepted from a +/// client; the server resolves it from these identifiers at execute time. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct RecordingSource { + pub target_id: String, + pub virtual_id: String, + pub input_name: String, + #[serde(default)] + pub cluster: super::super::XtreamCluster, +} + +impl RecordingSource { + pub fn new(target_id: impl Into, virtual_id: impl Into, input_name: impl Into) -> Self { + Self { + target_id: target_id.into(), + virtual_id: virtual_id.into(), + input_name: input_name.into(), + cluster: super::super::XtreamCluster::Live, + } + } + + pub fn with_cluster(mut self, cluster: super::super::XtreamCluster) -> Self { + self.cluster = cluster; + self + } +} + +/// Server-owned source identifiers as a client submits them. +/// +/// This is the request half of [`RecordingSource`]: the client names the +/// catalog item, and the server resolves the stream URL from it at execute +/// time. A URL is never accepted from a client. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecordingSourceRequest { + pub target_id: String, + pub virtual_id: String, + pub cluster: super::super::XtreamCluster, + pub input_name: String, +} + +/// The one create-recording wire contract, shared by the REST handler and +/// the frontend client so the two cannot drift. +/// +/// Unknown fields are rejected: a client that sends a field this build does +/// not understand has a different idea of what it is asking for, and +/// silently ignoring it would record the wrong thing. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CreateRecordingRequest { + pub source: RecordingSourceRequest, + pub program_title: String, + /// Programme interval, Live only. Rejected for VOD and series. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub program_start: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub program_end: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pre_roll_secs: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub post_roll_secs: Option, + pub visibility: RecordingVisibility, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub channel_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub channel_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub epg: Option, +} diff --git a/shared/src/model/recording/task.rs b/shared/src/model/recording/task.rs new file mode 100644 index 000000000..76fcd881e --- /dev/null +++ b/shared/src/model/recording/task.rs @@ -0,0 +1,212 @@ +use super::{EpgEpisodeMetadata, RecordingVisibility, TaskPriorityDto, TransferStatusDto, UserId}; +use std::fmt; + +/// Media kind of a recording task. Determined by the server from the +/// resolved catalog item; never accepted from a client. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RecordingKind { + /// Scheduled live capture. Executed with ffmpeg, never resumable. + Live, + /// Video on demand. Executed over resumable HTTP. + Vod, + /// Series episode. Executed over resumable HTTP. + Series, +} + +impl RecordingKind { + /// Live is captured within a programme window; VOD and Series run to EOF. + pub fn is_scheduled(self) -> bool { matches!(self, Self::Live) } + + /// Only VOD and Series may be paused and resumed. + pub fn is_resumable(self) -> bool { matches!(self, Self::Vod | Self::Series) } + + pub fn as_str(self) -> &'static str { + match self { + Self::Live => "live", + Self::Vod => "vod", + Self::Series => "series", + } + } +} + +impl fmt::Display for RecordingKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.as_str()) } +} + +/// Which controls a recording currently offers. +/// +/// Computed server-side from the transition graph and carried on the DTO so a +/// client renders buttons for exactly the commands the server would accept. +// One flag per control is the point: the set maps 1:1 onto the buttons a +// client renders, and collapsing it would hide which command was refused. +#[allow(clippy::struct_excessive_bools)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct RecordingAllowedActions { + pub pause: bool, + pub resume: bool, + pub cancel: bool, + pub retry: bool, + pub edit: bool, + pub remove: bool, +} + +/// The caller's own quota, as carried in the recording snapshot. +/// +/// Only the caller's private pool and the shared pool: there is no +/// per-user breakdown here, because one user's consumption is not another +/// user's business. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct RecordingQuotaSummaryDto { + pub private_used_bytes: u64, + pub private_limit_bytes: Option, + pub shared_used_bytes: u64, + pub shared_limit_bytes: Option, +} + +/// Owner-safe public projection of a recording task. This is the only +/// recording shape crossing the API or WebSocket boundary. It never carries +/// the source URL, provider/source identifiers, configured headers, +/// absolute/relative/partial paths, transfer validators, or another user's +/// owner identifier. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct RecordingTaskDto { + pub id: String, + pub title: String, + pub kind: RecordingKind, + pub priority: TaskPriorityDto, + pub status: TransferStatusDto, + pub retry_attempts: u8, + pub transferred_bytes: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub total_bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub next_retry_at: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// The provider ignored a byte-range request. Restarting from zero needs + /// the viewer's explicit confirmation before the partial is discarded. + #[serde(default)] + pub restart_from_beginning_required: bool, + /// `Some` only when the viewer owns the task; never another user's id. + #[serde(skip_serializing_if = "Option::is_none")] + pub owner_id: Option, + pub visibility: RecordingVisibility, + #[serde(skip_serializing_if = "Option::is_none")] + pub channel_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub channel_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub program_title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub program_start: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub program_end: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub scheduled_start: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub scheduled_end: Option, + pub pre_roll_secs: u64, + pub post_roll_secs: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub completed_at: Option, + /// Filename component of the final relative path. The server does not + /// expose the directory structure or the recording root. + #[serde(skip_serializing_if = "Option::is_none")] + pub filename: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub epg: Option, + /// Immutable provenance for tasks generated by a recurring rule. + #[serde(skip_serializing_if = "Option::is_none")] + pub rule_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub occurrence_key: Option, + /// What the server would accept for this recording right now, before the + /// viewer's permissions are applied. + #[serde(default)] + pub allowed_actions: RecordingAllowedActions, +} + +impl RecordingTaskDto { + /// Lifecycle partition for the UI: non-terminal tasks are current, + /// terminal ones are completed. + pub fn is_terminal(&self) -> bool { + matches!(self.status, TransferStatusDto::Completed | TransferStatusDto::Failed | TransferStatusDto::Cancelled) + } + + /// Effective duration of a scheduled window, when both bounds are known. + pub fn scheduled_duration_secs(&self) -> Option { + let (start, end) = self.scheduled_start.zip(self.scheduled_end)?; + u64::try_from(end.saturating_sub(start)).ok() + } +} + +/// Execution state of a recording task as persisted. +/// +/// Distinct from [`TransferStatusDto`](crate::model::TransferStatusDto), +/// which is the wire projection: this one is written to the recording +/// repository, so renaming a variant is a storage-format change. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] +pub enum RecordingTaskState { + #[default] + Queued, + Scheduled, + WaitingForCapacity, + RetryWaiting, + Running, + Paused, + /// Cancellation was requested; the worker still owns the file and the + /// provider slot. Deliberately not terminal -- treating it as terminal + /// lets a deletion unlink the partial the worker is still writing. + Cancelling, + Completed, + Failed, + Cancelled, +} + +impl RecordingTaskState { + pub fn is_terminal(self) -> bool { matches!(self, Self::Completed | Self::Failed | Self::Cancelled) } + + /// Stable label used by the edit rules, logs and errors. + pub fn label(self) -> &'static str { + match self { + Self::Queued => "Queued", + Self::Scheduled => "Scheduled", + Self::WaitingForCapacity => "WaitingForCapacity", + Self::RetryWaiting => "RetryWaiting", + Self::Running => "Running", + Self::Paused => "Paused", + Self::Cancelling => "Cancelling", + Self::Completed => "Completed", + Self::Failed => "Failed", + Self::Cancelled => "Cancelled", + } + } +} + +impl From for super::super::transfer::TransferStatusDto { + fn from(value: RecordingTaskState) -> Self { + match value { + RecordingTaskState::Queued => Self::Queued, + RecordingTaskState::Scheduled => Self::Scheduled, + RecordingTaskState::WaitingForCapacity => Self::WaitingForCapacity, + RecordingTaskState::RetryWaiting => Self::RetryWaiting, + RecordingTaskState::Running => Self::Running, + RecordingTaskState::Paused => Self::Paused, + RecordingTaskState::Cancelling => Self::Cancelling, + RecordingTaskState::Completed => Self::Completed, + RecordingTaskState::Failed => Self::Failed, + RecordingTaskState::Cancelled => Self::Cancelled, + } + } +} + +/// Monotonic revision counter for the persisted recording queue. Increments +/// once per committed queue mutation. Stored alongside the queue snapshot. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(transparent)] +pub struct QueueRevision(pub u64); + +impl fmt::Display for QueueRevision { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) } +} diff --git a/shared/src/model/recording/tests.rs b/shared/src/model/recording/tests.rs new file mode 100644 index 000000000..19ddeb970 --- /dev/null +++ b/shared/src/model/recording/tests.rs @@ -0,0 +1,234 @@ +use super::*; + +fn test_source() -> RecordingSource { RecordingSource::new("target-1", "v-1", "input-a") } + +fn test_owner() -> RecordingOwner { RecordingOwner::User(UserId::from("web:abc")) } + +#[test] +fn legacy_recording_source_defaults_to_live_cluster() { + let source: RecordingSource = + serde_json::from_str(r#"{"target_id":"target","virtual_id":"42","input_name":"input-a"}"#) + .expect("deserialize source"); + + assert_eq!(source.cluster, super::super::XtreamCluster::Live); +} + +#[test] +fn recording_kind_serializes_as_snake_case() { + assert_eq!(serde_json::to_string(&RecordingKind::Vod).expect("serialize"), "\"vod\""); + assert_eq!(serde_json::to_string(&RecordingKind::Series).expect("serialize"), "\"series\""); + assert_eq!(serde_json::to_string(&RecordingKind::Live).expect("serialize"), "\"live\""); +} + +fn create_request_json() -> &'static str { + r#"{"source":{"target_id":"t","virtual_id":"42","cluster":"Video","input_name":"in"},"program_title":"Film","visibility":"private"}"# +} + +#[test] +fn create_request_round_trips_its_minimal_form() { + let request: CreateRecordingRequest = serde_json::from_str(create_request_json()).expect("deserialize"); + assert_eq!(request.visibility, RecordingVisibility::Private); + assert_eq!(request.program_start, None); + assert_eq!(serde_json::to_string(&request).expect("serialize"), create_request_json()); +} + +#[test] +fn create_request_rejects_an_unknown_field() { + // A client that sends a field this build does not understand has a + // different idea of what it is asking for; recording the wrong thing + // is worse than refusing. + let body = r#"{"source":{"target_id":"t","virtual_id":"42","cluster":"Video","input_name":"in"}, + "program_title":"Film","visibility":"private","url":"http://evil/stream.ts"}"#; + let error = serde_json::from_str::(body).expect_err("unknown field must be rejected"); + assert!(error.to_string().contains("url"), "{error}"); +} + +#[test] +fn create_request_rejects_an_unknown_visibility() { + // The visibility is the server's enum on the wire, so a typo in the + // frontend cannot reach the server as a valid body. + let body = r#"{"source":{"target_id":"t","virtual_id":"42","cluster":"Video","input_name":"in"}, + "program_title":"Film","visibility":"pubic"}"#; + assert!(serde_json::from_str::(body).is_err()); +} + +#[test] +fn create_request_rejects_an_unknown_source_field() { + let body = r#"{"source":{"target_id":"t","virtual_id":"42","cluster":"Video","input_name":"in", + "stream_url":"http://evil/"},"program_title":"Film","visibility":"private"}"#; + assert!(serde_json::from_str::(body).is_err()); +} + +#[test] +fn only_live_is_scheduled_and_only_media_is_resumable() { + assert!(RecordingKind::Live.is_scheduled()); + assert!(!RecordingKind::Live.is_resumable()); + for kind in [RecordingKind::Vod, RecordingKind::Series] { + assert!(!kind.is_scheduled(), "{kind} must not be scheduled"); + assert!(kind.is_resumable(), "{kind} must be resumable"); + } +} + +#[test] +fn live_metadata_carries_padded_interval_and_source() { + let meta = RecordingMetadata::new_live( + test_owner(), + RecordingVisibility::Private, + test_source(), + 1_700_000_000, + 1_700_000_900, + 60, + 120, + ); + assert_eq!(meta.scheduled_start, Some(1_700_000_000 - 60)); + assert_eq!(meta.scheduled_end, Some(1_700_000_900 + 120)); + assert_eq!(meta.pre_roll_secs, 60); + assert_eq!(meta.post_roll_secs, 120); + assert!(!meta.is_deleting()); +} + +#[test] +fn media_metadata_has_owner_and_source_without_live_interval() { + let source = test_source().with_cluster(super::super::XtreamCluster::Video); + let meta = RecordingMetadata::new_media(test_owner(), RecordingVisibility::Private, source, "Movie".to_string()); + + assert_eq!(meta.program_title.as_deref(), Some("Movie")); + assert!(meta.program_start.is_none()); + assert!(meta.program_end.is_none()); + assert!(meta.scheduled_start.is_none()); + assert!(meta.scheduled_end.is_none()); + assert_eq!(meta.pre_roll_secs, 0); + assert_eq!(meta.post_roll_secs, 0); +} + +#[test] +fn recording_owner_round_trips_as_adjacent_tag() { + let owner = RecordingOwner::User(UserId::from("web:alice")); + let json = serde_json::to_string(&owner).expect("serialize"); + assert_eq!(json, r#"{"kind":"user","user":"web:alice"}"#); + let restored: RecordingOwner = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(restored, owner); + assert_eq!(restored.user_id(), &UserId::from("web:alice")); +} + +#[test] +fn deleting_state_round_trip() { + let mut meta = + RecordingMetadata::new_media(test_owner(), RecordingVisibility::Private, test_source(), "M".to_string()); + assert!(!meta.is_deleting()); + meta.deleting_previous_state = Some(DeletionPreviousState::Completed); + assert!(meta.is_deleting()); + let json = serde_json::to_string(&meta).expect("serialize"); + let restored: RecordingMetadata = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(restored.deleting_previous_state, Some(DeletionPreviousState::Completed)); +} + +#[test] +fn metadata_filename_is_the_last_relative_path_component() { + let mut meta = + RecordingMetadata::new_media(test_owner(), RecordingVisibility::Private, test_source(), "M".to_string()); + assert_eq!(meta.filename(), None); + meta.relative_path = Some("users/web:abc/Pilot/pilot.ts".to_string()); + assert_eq!(meta.filename(), Some("pilot.ts")); +} + +#[test] +fn notification_marker_serialization() { + let marker = NotificationMarker::new(NotificationMarkerKind::Completed, 1_700_000_000); + let json = serde_json::to_string(&marker).expect("serialize"); + assert!(json.contains("\"completed\""), "{json}"); +} + +#[test] +fn queue_revision_default_is_zero() { + assert_eq!(QueueRevision::default().0, 0); + assert_eq!(format!("{}", QueueRevision(42)), "42"); +} + +// --- RecordingTaskDto --- + +fn make_dto() -> RecordingTaskDto { + RecordingTaskDto { + id: "rec-1".to_string(), + title: "Pilot".to_string(), + kind: RecordingKind::Series, + priority: TaskPriorityDto::Normal, + status: TransferStatusDto::Running, + retry_attempts: 1, + transferred_bytes: 128, + total_bytes: Some(4096), + next_retry_at: None, + error: None, + restart_from_beginning_required: false, + owner_id: Some(UserId::from("web:abc")), + visibility: RecordingVisibility::Private, + channel_id: None, + channel_name: None, + program_title: Some("Pilot".to_string()), + program_start: None, + program_end: None, + scheduled_start: None, + scheduled_end: None, + pre_roll_secs: 0, + post_roll_secs: 0, + completed_at: None, + filename: Some("pilot.mkv".to_string()), + epg: Some(EpgEpisodeMetadata { + programme_id: None, + series_id: Some("series-1".to_string()), + episode_id: Some("ep-1".to_string()), + season: Some(1), + episode: Some(2), + airing: AiringStatus::New, + }), + rule_id: None, + occurrence_key: None, + allowed_actions: RecordingAllowedActions::default(), + } +} + +#[test] +fn recording_task_dto_round_trips_preserving_kind() { + let dto = make_dto(); + let json = serde_json::to_string(&dto).expect("serialize"); + let restored: RecordingTaskDto = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(restored, dto); + assert_eq!(restored.kind, RecordingKind::Series); +} + +#[test] +fn recording_task_dto_has_no_nested_transfer_or_recording_wrapper() { + let json = serde_json::to_value(make_dto()).expect("serialize"); + assert!(json.get("recording").is_none(), "no nested recording wrapper"); + assert!(json.get("transfer").is_none(), "no nested transfer wrapper"); + assert_eq!(json.get("kind").and_then(serde_json::Value::as_str), Some("series")); +} + +#[test] +fn terminal_partition_matches_completed_failed_cancelled() { + for status in [TransferStatusDto::Completed, TransferStatusDto::Failed, TransferStatusDto::Cancelled] { + assert!(RecordingTaskDto { status, ..make_dto() }.is_terminal(), "{status:?} is terminal"); + } + for status in [ + TransferStatusDto::Scheduled, + TransferStatusDto::Queued, + TransferStatusDto::WaitingForCapacity, + TransferStatusDto::RetryWaiting, + TransferStatusDto::Running, + TransferStatusDto::Paused, + ] { + assert!(!RecordingTaskDto { status, ..make_dto() }.is_terminal(), "{status:?} is current"); + } +} + +#[test] +fn scheduled_duration_is_derived_from_the_padded_window() { + let dto = RecordingTaskDto { + kind: RecordingKind::Live, + scheduled_start: Some(1_700_000_000), + scheduled_end: Some(1_700_003_600), + ..make_dto() + }; + assert_eq!(dto.scheduled_duration_secs(), Some(3_600)); + assert_eq!(make_dto().scheduled_duration_secs(), None); +} diff --git a/shared/src/model/stream_properties.rs b/shared/src/model/stream_properties.rs index aa68c5fd3..ef3813ab1 100644 --- a/shared/src/model/stream_properties.rs +++ b/shared/src/model/stream_properties.rs @@ -1,488 +1,7 @@ -use crate::{ - model::{info_doc_utils::InfoDocUtils, PlaylistEntry, XtreamSeriesInfo, XtreamSeriesInfoDoc, XtreamVideoInfo}, - utils::{ - arc_str_none_default_on_null, arc_str_null_is_none_serde, arc_str_option_null_if_empty_serde, - arc_str_option_serde, deserialize_as_option_arc_str, deserialize_as_string_array, - deserialize_json_as_opt_string, deserialize_number_from_string, deserialize_number_from_string_or_zero, - serialize_json_as_opt_string, Internable, CONSTANTS, - }, -}; -use log::warn; +use crate::utils::Internable; use serde::{Deserialize, Serialize}; use std::sync::Arc; -#[derive(Default, Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct CatchupAttribute { - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub name: Arc, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub value: Arc, -} - -#[derive(Default, Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct CatchupProperties { - #[serde(default, with = "arc_str_option_serde")] - pub mode: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub days: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub source: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub time: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub correction: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub catchup_type: Option>, - #[serde(default)] - pub extra_attributes: Vec, - /// Origin window for an automatically generated bounded Flussonic source. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub flussonic_archive_max_duration_secs: Option, -} - -impl CatchupProperties { - pub fn is_empty(&self) -> bool { - self.mode.is_none() - && self.days.is_none() - && self.source.is_none() - && self.time.is_none() - && self.correction.is_none() - && self.catchup_type.is_none() - && self.extra_attributes.is_empty() - && self.flussonic_archive_max_duration_secs.is_none() - } - - /// Prefer `catchup-type` when both fields are present because providers may leave - /// a stale `catchup` mode alongside the authoritative player type. - pub fn effective_mode(&self) -> Option<&str> { - self.catchup_type - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .or_else(|| self.mode.as_deref().map(str::trim).filter(|value| !value.is_empty())) - } - - fn is_flussonic_mode_str(mode: Option<&str>) -> bool { - mode.is_some_and(|m| { - matches!(m.trim().to_ascii_lowercase().as_str(), "flussonic" | "flussonic-hls" | "flussonic-ts" | "fs") - }) - } - - /// Prefer `catchup-type` when both are set so a leftover `catchup="shift"`/`append` - /// from a provider cannot steal Flussonic path-rewrite channels (v3.3.81 behavior). - pub fn is_flussonic(&self) -> bool { Self::is_flussonic_mode_str(self.effective_mode()) } - - pub fn native_flussonic_player_mode(&self) -> Option<&'static str> { - if !self.is_flussonic() { - return None; - } - let raw = self.effective_mode().unwrap_or("flussonic"); - if raw.eq_ignore_ascii_case("flussonic-ts") { - Some("flussonic-ts") - } else { - Some("flussonic") - } - } - - /// Canonical append label for unified M3U export (`catchup-type="append"` only). - pub fn append_player_type(&self) -> Option<&'static str> { - if self.native_flussonic_player_mode().is_some() { - return None; - } - self.effective_mode()?.eq_ignore_ascii_case("append").then_some("append") - } -} - -fn format_episode_code(season: u32, episode: u32) -> Option { - match (season, episode) { - (0, 0) => None, - (season, 0) => Some(format!("S{season:02}")), - (0, episode) => Some(format!("E{episode:02}")), - (season, episode) => Some(format!("S{season:02}E{episode:02}")), - } -} - -fn title_contains_episode_code(title: &str) -> bool { CONSTANTS.re_episode_code.is_match(title) } - -fn first_series_episode_value(series: &SeriesStreamProperties, selector: F) -> Option -where - F: FnOnce(&SeriesStreamDetailEpisodeProperties) -> Option, -{ - series - .details - .as_ref() - .and_then(|details| details.episodes.as_ref()) - .and_then(|episodes| episodes.first()) - .and_then(selector) -} - -pub fn normalize_episode_title(raw_title: &Arc, series_name: &Arc, season: u32, episode: u32) -> Arc { - let title = raw_title.trim(); - let series_name = series_name.trim(); - let Some(code) = format_episode_code(season, episode) else { - return if title.is_empty() { series_name.intern() } else { title.intern() }; - }; - - if title.is_empty() || (!series_name.is_empty() && title.eq_ignore_ascii_case(series_name)) { - return code.into(); - } - - if title_contains_episode_code(title) { - return title.intern(); - } - - format!("{code} - {title}").into() -} - -#[derive(Default, Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct LiveStreamProperties { - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub name: Arc, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub category_id: u32, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub stream_id: u32, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub stream_icon: Arc, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub direct_source: Arc, - #[serde(default, with = "arc_str_option_null_if_empty_serde")] - pub custom_sid: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub added: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub stream_type: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub epg_channel_id: Option>, - #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub tv_archive: Option, - #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub tv_archive_duration: Option, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub is_adult: i32, - // New fields for probing - #[serde( - default, - serialize_with = "serialize_json_as_opt_string", - deserialize_with = "deserialize_json_as_opt_string" - )] - pub video: Option>, - #[serde( - default, - serialize_with = "serialize_json_as_opt_string", - deserialize_with = "deserialize_json_as_opt_string" - )] - pub audio: Option>, - #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub last_probed_timestamp: Option, - #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub last_success_timestamp: Option, - #[serde(default)] - pub catchup: Option, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub bitrate: u32, -} - -impl LiveStreamProperties { - /// Merges metadata learned from probing or playback without replacing newer provider data. - pub fn merge_learned_metadata_from(&mut self, previous: &Self) -> bool { - let mut changed = false; - - if self.video.is_none() && previous.video.is_some() { - self.video.clone_from(&previous.video); - changed = true; - } - - if self.audio.is_none() && previous.audio.is_some() { - self.audio.clone_from(&previous.audio); - changed = true; - } - - if previous.bitrate > self.bitrate { - self.bitrate = previous.bitrate; - changed = true; - } - - if self.last_probed_timestamp.is_none() && previous.last_probed_timestamp.is_some() { - self.last_probed_timestamp = previous.last_probed_timestamp; - changed = true; - } - - if self.last_success_timestamp.is_none() && previous.last_success_timestamp.is_some() { - self.last_success_timestamp = previous.last_success_timestamp; - changed = true; - } - - changed - } -} - -#[derive(Default, Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct VideoStreamDetailProperties { - #[serde(default, with = "arc_str_option_serde")] - pub kinopoisk_url: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub o_name: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub cover_big: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub movie_image: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub release_date: Option>, - #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub episode_run_time: Option, - #[serde(default, with = "arc_str_option_serde")] - pub youtube_trailer: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub director: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub actors: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub cast: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub description: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub plot: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub age: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub mpaa_rating: Option>, - #[serde(default)] - pub rating_count_kinopoisk: u32, - #[serde(default, with = "arc_str_option_serde")] - pub country: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub genre: Option>, - #[serde(default, deserialize_with = "deserialize_as_string_array")] - pub backdrop_path: Option>>, - #[serde(default, with = "arc_str_option_serde")] - pub duration_secs: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub duration: Option>, - #[serde( - default, - serialize_with = "serialize_json_as_opt_string", - deserialize_with = "deserialize_json_as_opt_string" - )] - pub video: Option>, - #[serde( - default, - serialize_with = "serialize_json_as_opt_string", - deserialize_with = "deserialize_json_as_opt_string" - )] - pub audio: Option>, - #[serde(default)] - pub bitrate: u32, - #[serde(default, with = "arc_str_option_serde")] - pub runtime: Option>, - #[serde(default, with = "arc_str_option_serde")] - pub status: Option>, -} - -#[derive(Default, Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct VideoStreamProperties { - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub name: Arc, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub category_id: u32, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub stream_id: u32, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub stream_icon: Arc, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub direct_source: Arc, - #[serde(default, with = "arc_str_option_null_if_empty_serde")] - pub custom_sid: Option>, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub added: Arc, - #[serde(default, deserialize_with = "arc_str_null_is_none_serde::deserialize")] - pub container_extension: Arc, - #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub rating: Option, - #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub rating_5based: Option, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub stream_type: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub trailer: Option>, - #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub tmdb: Option, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub is_adult: i32, - #[serde(default)] - pub details: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct SeriesStreamDetailSeasonProperties { - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub name: Arc, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub season_number: u32, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub episode_count: u32, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub overview: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub air_date: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub cover: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub cover_tmdb: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub cover_big: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub duration: Option>, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] -pub struct SeriesStreamDetailEpisodeProperties { - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub id: u32, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub episode_num: u32, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub season: u32, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub title: Arc, - #[serde(default, deserialize_with = "arc_str_null_is_none_serde::deserialize")] - pub container_extension: Arc, - #[serde(default, with = "arc_str_option_null_if_empty_serde")] - pub custom_sid: Option>, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub added: Arc, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub direct_source: Arc, - #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub tmdb: Option, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub release_date: Arc, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub series_release_date: Option>, // Global series release date - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub plot: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub crew: Option>, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub duration_secs: u32, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub duration: Arc, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub movie_image: Arc, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub bitrate: u32, - #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub rating: Option, - #[serde( - default, - serialize_with = "serialize_json_as_opt_string", - deserialize_with = "deserialize_json_as_opt_string" - )] - pub video: Option>, - #[serde( - default, - serialize_with = "serialize_json_as_opt_string", - deserialize_with = "deserialize_json_as_opt_string" - )] - pub audio: Option>, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct SeriesStreamDetailProperties { - #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub year: Option, - #[serde(default)] - pub seasons: Option>, - #[serde(default)] - pub episodes: Option>, -} - -impl SeriesStreamDetailProperties { - /// Builds a `SeriesStreamDetailProperties`. Empty `seasons` is normalized to - /// `None` so downstream code can short-circuit; `episodes` is kept as-is. - pub fn new( - year: Option, - seasons: Vec, - episodes: Option>, - ) -> Self { - Self { year, seasons: if seasons.is_empty() { None } else { Some(seasons) }, episodes } - } -} - -#[derive(Default, Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct SeriesStreamProperties { - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub name: Arc, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub category_id: u32, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub series_id: u32, - #[serde(default, deserialize_with = "deserialize_as_string_array")] - pub backdrop_path: Option>>, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub cast: Arc, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub cover: Arc, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub director: Arc, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub episode_run_time: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub genre: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub last_modified: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub plot: Option>, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub rating: f64, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub rating_5based: f64, - #[serde(default, with = "arc_str_option_serde")] - pub release_date: Option>, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub youtube_trailer: Arc, - #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub tmdb: Option, - #[serde(default)] - pub details: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct EpisodeStreamProperties { - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub episode_id: u32, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub episode: u32, - #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] - pub season: u32, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub added: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub release_date: Option>, - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub series_release_date: Option>, // Global series release date - #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] - pub plot: Option>, - #[serde(default, deserialize_with = "deserialize_number_from_string")] - pub tmdb: Option, - #[serde(default, deserialize_with = "arc_str_none_default_on_null")] - pub movie_image: Arc, - #[serde(default, deserialize_with = "arc_str_null_is_none_serde::deserialize")] - pub container_extension: Arc, - #[serde( - default, - serialize_with = "serialize_json_as_opt_string", - deserialize_with = "deserialize_json_as_opt_string" - )] - pub video: Option>, - #[serde( - default, - serialize_with = "serialize_json_as_opt_string", - deserialize_with = "deserialize_json_as_opt_string" - )] - pub audio: Option>, -} - #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub enum StreamProperties { Live(Box), @@ -803,692 +322,20 @@ impl StreamProperties { } } -/// Field names accepted by resource-proxy endpoints. This is deliberately defined beside the -/// resource traversal and selector so adding a new endpoint-visible resource has one review site. -/// -/// Stream URLs are not resources: serving them here would sidestep the stream routes and their -/// admission accounting. -pub fn is_resource_field_name(field: &str) -> bool { - if matches!(field, "logo" | "logo_small" | "cover" | "movie_image" | "nfo_cover_big" | "nfo_movie_image") - || field.starts_with("backdrop_path") - || field.starts_with("nfo_backdrop_path") - { - return true; - } - parse_season_field(field).is_some_and(|(_, field)| matches!(field.as_str(), "cover" | "cover_tmdb" | "cover_big")) - || parse_season_episode_field(field).is_some_and(|(_, _, field)| field == "movie_image") -} - -fn parse_season_field(s: &str) -> Option<(u32, String)> { - let mut parts = s.split('_'); - - let (prefix, suffix) = (parts.next()?, parts.next()?); - if prefix != "nfo" || suffix != "s" { - return None; - } - - let season: u32 = parts.next()?.parse().ok()?; - - let kind = join_parts(parts, '_'); - if kind.is_empty() { - return None; - } - - Some((season, kind)) -} - -fn parse_season_episode_field(s: &str) -> Option<(u32, u32, String)> { - let mut parts = s.split('_'); - - let (prefix, ep) = (parts.next()?, parts.next()?); - if prefix != "nfo" || ep != "ep" { - return None; - } - - let season: u32 = parts.next()?.parse().ok()?; - let episode: u32 = parts.next()?.parse().ok()?; - - let kind = join_parts(parts, '_'); - if kind.is_empty() { - return None; - } - - Some((season, episode, kind)) -} - -fn join_parts<'a>(parts: impl Iterator, separator: char) -> String { - let mut result = String::new(); - for part in parts { - if !result.is_empty() { - result.push(separator); - } - result.push_str(part); - } - result -} - -impl VideoStreamProperties { - fn from_info_base(info: &XtreamVideoInfo) -> VideoStreamProperties { - VideoStreamProperties { - name: info.info.name.clone(), - category_id: info.movie_data.category_id, - stream_id: info.movie_data.stream_id, - stream_icon: info.info.movie_image.as_ref().map_or_else(|| "".intern(), Clone::clone), - direct_source: info.movie_data.direct_source.clone(), - custom_sid: info.movie_data.custom_sid.clone(), - added: info.movie_data.added.clone(), - container_extension: info.movie_data.container_extension.clone(), - rating: None, // from PlaylistItem - rating_5based: None, // from PlaylistItem - stream_type: Some("movie".intern()), - trailer: info.info.youtube_trailer.clone(), - tmdb: info.info.tmdb_id.parse::().ok(), - is_adult: 0, // from PlaylistItem - details: Some(VideoStreamDetailProperties { - kinopoisk_url: info.info.kinopoisk_url.clone(), - o_name: info.info.o_name.clone(), - cover_big: info.info.cover_big.clone(), - movie_image: info.info.movie_image.clone(), - release_date: info.info.releasedate.clone(), - episode_run_time: info.info.episode_run_time, - youtube_trailer: info.info.youtube_trailer.clone(), - director: info.info.director.clone(), - actors: info.info.actors.clone(), - cast: info.info.cast.clone(), - description: info.info.description.clone(), - plot: info.info.plot.clone(), - age: info.info.age.clone(), - mpaa_rating: info.info.mpaa_rating.clone(), - rating_count_kinopoisk: info.info.rating_count_kinopoisk, - country: info.info.country.clone(), - genre: info.info.genre.clone(), - backdrop_path: info.info.backdrop_path.clone(), - duration_secs: info.info.duration_secs.clone(), - duration: info.info.duration.clone(), - video: info.info.video.clone(), - audio: info.info.audio.clone(), - bitrate: info.info.bitrate, - runtime: info.info.runtime.clone(), - status: info.info.status.clone(), - }), - } - } - - pub fn from_info

(info: &XtreamVideoInfo, pli: &P) -> VideoStreamProperties - where - P: PlaylistEntry, - { - let mut props = VideoStreamProperties::from_info_base(info); - - if let Some(StreamProperties::Video(video)) = pli.get_additional_properties() { - props.rating = video.rating; - props.rating_5based = video.rating_5based; - props.stream_type = video.stream_type.clone(); - if props.tmdb.is_none() { - props.tmdb = video.tmdb; - } - if props.trailer.is_none() { - props.trailer = video.trailer.clone(); - } - props.is_adult = video.is_adult; - } - - props - } - - pub fn from_info_without_existing(info: &XtreamVideoInfo) -> VideoStreamProperties { - VideoStreamProperties::from_info_base(info) - } -} - -impl SeriesStreamProperties { - fn from_info_base(info: &XtreamSeriesInfo, series_id: u32) -> SeriesStreamProperties { - SeriesStreamProperties { - name: info.info.name.clone(), - category_id: info.info.category_id, - series_id, - backdrop_path: info.info.backdrop_path.clone(), - cast: info.info.cast.clone(), - cover: info.info.cover.clone(), - director: info.info.director.clone(), - episode_run_time: Some(info.info.episode_run_time.clone()), - genre: Some(info.info.genre.clone()), - last_modified: Some(info.info.last_modified.clone()), - plot: Some(info.info.plot.clone()), - rating: info.info.rating, - rating_5based: info.info.rating_5based, - release_date: Some(info.info.release_date.clone()), - youtube_trailer: info.info.youtube_trailer.clone(), - tmdb: info.info.tmdb, - details: Some(SeriesStreamDetailProperties { - year: InfoDocUtils::extract_year_from_release_date(&info.info.release_date), - seasons: info.seasons.as_ref().map(|list| { - let mut seasons: Vec = list - .iter() - .map(|s| SeriesStreamDetailSeasonProperties { - name: s.name.clone(), - season_number: s.season_number, - episode_count: s.episode_count, - overview: Some(s.overview.clone()), - air_date: Some(s.air_date.clone()), - cover: Some(s.cover.clone()), - cover_tmdb: Some(s.cover_tmdb.clone()), - cover_big: Some(s.cover_big.clone()), - duration: Some(s.duration.clone()), - }) - .collect(); - seasons.sort_by_key(|season| season.season_number); - seasons - }), - episodes: info.episodes.as_ref().map(|list| { - let mut episodes: Vec = list - .iter() - .map(|e| SeriesStreamDetailEpisodeProperties { - id: e.id, - episode_num: e.episode_num, - season: e.season, - title: normalize_episode_title(&e.title, &info.info.name, e.season, e.episode_num), - container_extension: e.container_extension.clone(), - custom_sid: e.custom_sid.clone(), - added: e.added.clone(), - direct_source: e.direct_source.clone(), - tmdb: info.info.tmdb, - release_date: e.info.as_ref().map(|i| i.air_date.clone()).unwrap_or_default(), - series_release_date: None, - plot: e.info.as_ref().and_then(|i| i.plot.clone()), - crew: e.info.as_ref().map(|i| i.crew.clone()), - duration_secs: e.info.as_ref().map(|i| i.duration_secs).unwrap_or_default(), - duration: e.info.as_ref().map(|i| i.duration.clone()).unwrap_or_default(), - movie_image: e.info.as_ref().map(|i| i.movie_image.clone()).unwrap_or_default(), - bitrate: e.info.as_ref().map(|i| i.bitrate).unwrap_or_default(), - rating: e.info.as_ref().map(|i| i.rating), - video: e.info.as_ref().map(|i| i.video.clone()).unwrap_or_default(), - audio: e.info.as_ref().map(|i| i.audio.clone()).unwrap_or_default(), - }) - .collect(); - episodes.sort_by_key(|episode| (episode.season, episode.episode_num)); - episodes - }), - }), - } - } - - pub fn from_info

(info: &XtreamSeriesInfo, pli: &P) -> SeriesStreamProperties - where - P: PlaylistEntry, - { - Self::from_info_base(info, pli.get_virtual_id().get()) - } - - pub fn from_info_without_existing(info: &XtreamSeriesInfo, series_id: u32) -> SeriesStreamProperties { - Self::from_info_base(info, series_id) - } - - pub fn from_info_doc(info: &XtreamSeriesInfoDoc, series_id: u32) -> SeriesStreamProperties { - let tmdb = info.info.tmdb.parse::().ok(); - SeriesStreamProperties { - name: info.info.name.clone(), - category_id: info.info.category_id.parse::().unwrap_or_else(|_| { - warn!("Failed to parse category_id {}", info.info.category_id); - 0 - }), - series_id, - backdrop_path: Some(info.info.backdrop_path.clone()), - cast: info.info.cast.clone(), - cover: info.info.cover.clone(), - director: info.info.director.clone(), - episode_run_time: Some(info.info.episode_run_time.clone()), - genre: Some(info.info.genre.clone()), - last_modified: Some(info.info.last_modified.clone()), - plot: Some(info.info.plot.clone()), - rating: info.info.rating.parse::().unwrap_or_else(|_| { - warn!("Failed to parse rating {}", info.info.rating); - 0.0 - }), - rating_5based: info.info.rating_5based.parse::().unwrap_or_else(|_| { - warn!("Failed to parse rating_5based {}", info.info.rating_5based); - 0.0 - }), - release_date: Some(info.info.release_date.clone()), - youtube_trailer: info.info.youtube_trailer.clone(), - tmdb, - details: Some(SeriesStreamDetailProperties { - year: InfoDocUtils::extract_year_from_release_date(&info.info.release_date), - seasons: { - let mut seasons: Vec = info - .seasons - .iter() - .map(|s| SeriesStreamDetailSeasonProperties { - name: s.name.clone(), - season_number: s.season_number, - episode_count: s.episode_count.parse::().unwrap_or_else(|_| { - warn!("Failed to parse episode_count {}", s.episode_count); - 0 - }), - overview: s.overview.clone(), - air_date: s.air_date.clone(), - cover: s.cover.clone(), - cover_tmdb: s.cover_tmdb.clone(), - cover_big: s.cover_big.clone(), - duration: s.duration.clone(), - }) - .collect(); - seasons.sort_by_key(|season| season.season_number); - Some(seasons) - }, - episodes: { - let mut episodes: Vec = info - .episodes - .iter() - .flat_map(|(_, list)| list.iter()) - .map(|e| SeriesStreamDetailEpisodeProperties { - id: e.id.parse::().unwrap_or_else(|_| { - warn!("Failed to parse episode id {}", e.id); - 0 - }), - episode_num: e.episode_num, - season: e.season, - title: normalize_episode_title(&e.title, &info.info.name, e.season, e.episode_num), - container_extension: e.container_extension.clone(), - custom_sid: e.custom_sid.clone(), - added: e.added.clone(), - direct_source: e.direct_source.clone(), - tmdb, - release_date: e.info.air_date.clone(), - series_release_date: None, - plot: e.info.plot.clone(), - crew: e.info.crew.clone(), - duration_secs: e.info.duration_secs, - duration: e.info.duration.clone(), - movie_image: e.info.movie_image.clone(), - bitrate: e.info.bitrate, - rating: Some(e.info.rating), - video: None, - audio: None, - }) - .collect(); - episodes.sort_by_key(|episode| (episode.season, episode.episode_num)); - Some(episodes) - }, - }), - } - } -} - -impl EpisodeStreamProperties { - pub fn from_series( - series: &SeriesStreamProperties, - episode: &SeriesStreamDetailEpisodeProperties, - ) -> EpisodeStreamProperties { - EpisodeStreamProperties { - episode_id: episode.id, - episode: episode.episode_num, - season: episode.season, - added: Some(episode.added.clone()), - release_date: Some(episode.release_date.clone()), - // Inherit global release date from the Series object if available - series_release_date: series.release_date.clone(), - plot: episode.plot.clone(), - tmdb: episode.tmdb.or(series.tmdb), - movie_image: episode.movie_image.clone(), - container_extension: episode.container_extension.clone(), - video: episode.video.clone(), - audio: episode.audio.clone(), - } - } -} - -fn non_empty_arc(s: &Arc) -> Option> { - if s.is_empty() { - None - } else { - Some(Arc::clone(s)) - } -} - #[cfg(test)] -mod tests { - use super::*; - use crate::model::{PlaylistItemType, VirtualId, XtreamCluster, XtreamPlaylistItem}; - use serde_json::json; +mod tests; - #[test] - fn genre_reads_from_wherever_the_variant_keeps_it() { - let mut video = StreamProperties::Video(Box::default()); - assert_eq!(video.genre(), None, "video with no details has no genre"); - assert!(video.set_genre("Action"), "video accepts a genre"); - assert_eq!(video.genre().map(Arc::as_ref), Some("Action"), "video keeps it under details"); - // Setting again goes through the existing details rather than replacing them. - assert!(video.set_genre("Drama")); - assert_eq!(video.genre().map(Arc::as_ref), Some("Drama")); - - let mut series = StreamProperties::Series(Box::default()); - assert_eq!(series.genre(), None); - assert!(series.set_genre("Comedy"), "series accepts a genre"); - assert_eq!(series.genre().map(Arc::as_ref), Some("Comedy"), "series keeps it inline"); - } - - #[test] - fn live_and_episode_have_no_genre_and_refuse_to_take_one() { - // EpisodeStreamProperties has no Default impl, but every field is - // #[serde(default)], so an empty object builds one. - let episode: EpisodeStreamProperties = - serde_json::from_value(json!({})).expect("episode properties default from an empty object"); - for mut props in [StreamProperties::Live(Box::default()), StreamProperties::Episode(Box::new(episode))] { - assert_eq!(props.genre(), None); - assert!(!props.set_genre("Action"), "this variant has no genre to set"); - assert_eq!(props.genre(), None); - } - } - - fn sample_video_info(tmdb_id: &str) -> XtreamVideoInfo { - serde_json::from_value(json!({ - "info": { - "name": "Fetched Movie", - "tmdb_id": tmdb_id, - "movie_image": "https://img.example/movie.jpg", - "video": { "codec": "h264" }, - "audio": { "codec": "aac" } - }, - "movie_data": { - "name": "Fetched Movie", - "category_id": "99", - "stream_id": "4242", - "direct_source": "https://cdn.example/stream.mkv", - "added": "1700000000", - "container_extension": "mkv" - } - })) - .expect("sample XtreamVideoInfo should deserialize") - } - - fn existing_video_item() -> XtreamPlaylistItem { - XtreamPlaylistItem { - virtual_id: VirtualId::new(1), - provider_id: 1001, - name: "Existing".into(), - logo: "".into(), - logo_small: "".into(), - group: "".into(), - title: "Existing title".into(), - parent_code: "".into(), - rec: "".into(), - url: "https://provider.example/stream".into(), - epg_channel_id: None, - xtream_cluster: XtreamCluster::Video, - additional_properties: Some(StreamProperties::Video(Box::new(VideoStreamProperties { - rating: Some(7.5), - rating_5based: Some(3.7), - stream_type: Some("legacy-type".into()), - trailer: Some("legacy-trailer".into()), - tmdb: Some(7788), - is_adult: 1, - ..Default::default() - }))), - item_type: PlaylistItemType::Video, - category_id: 99, - input_name: "input".into(), - channel_no: 0, - source_ordinal: 0, - input_stream_id: "1001".into(), - upstream_user_agent: None, - } - } - - #[test] - fn from_info_without_existing_keeps_fetched_metadata() { - let info = sample_video_info("12345"); - - let props = VideoStreamProperties::from_info_without_existing(&info); - let details = props.details.expect("details should be present"); - - assert_eq!(props.name.as_ref(), "Fetched Movie"); - assert_eq!(props.stream_id, 4242); - assert_eq!(props.tmdb, Some(12345)); - assert_eq!(props.stream_type.as_deref(), Some("movie")); - assert!(details.video.is_some()); - assert!(details.audio.is_some()); - } - - #[test] - fn from_info_with_existing_keeps_existing_overrides() { - let info = sample_video_info(""); - let existing = existing_video_item(); - - let props = VideoStreamProperties::from_info(&info, &existing); - - assert_eq!(props.rating, Some(7.5)); - assert_eq!(props.rating_5based, Some(3.7)); - assert_eq!(props.stream_type.as_deref(), Some("legacy-type")); - assert_eq!(props.trailer.as_deref(), Some("legacy-trailer")); - assert_eq!(props.tmdb, Some(7788)); - assert_eq!(props.is_adult, 1); - } - - #[test] - fn from_series_info_preserves_episode_plot() { - let info: XtreamSeriesInfo = serde_json::from_value(json!({ - "info": { - "name": "Example Show" - }, - "episodes": { - "1": [{ - "id": 101, - "episode_num": 1, - "season": 1, - "title": "Pilot", - "container_extension": "mkv", - "info": { - "air_date": "2024-01-01", - "plot": "Episode plot", - "movie_image": "https://img.example/still.jpg" - } - }] - } - })) - .expect("sample XtreamSeriesInfo should deserialize"); - - let props = SeriesStreamProperties::from_info_without_existing(&info, 7); - let episode = props - .details - .and_then(|details| details.episodes) - .and_then(|episodes| episodes.first().cloned()) - .expect("episode should be mapped"); - - assert_eq!(episode.plot.as_deref(), Some("Episode plot")); - } - - #[test] - fn normalize_episode_title_injects_missing_episode_code() { - let normalized = normalize_episode_title(&"Pilot".into(), &"Example Show".into(), 1, 2); - assert_eq!(normalized.as_ref(), "S01E02 - Pilot"); - } - - #[test] - fn normalize_episode_title_replaces_series_name_only_with_episode_code() { - let normalized = normalize_episode_title(&"Example Show".into(), &"Example Show".into(), 1, 2); - assert_eq!(normalized.as_ref(), "S01E02"); - } - - #[test] - fn normalize_episode_title_keeps_existing_episode_code() { - let normalized = normalize_episode_title(&"S01E02".into(), &"Example Show".into(), 1, 2); - assert_eq!(normalized.as_ref(), "S01E02"); - } - - #[test] - fn catchup_messagepack_round_trip_preserves_archive_window() -> Result<(), Box> { - for window in [None, Some(14400)] { - let catchup = CatchupProperties { - mode: Some("fs".into()), - days: Some("10".into()), - source: Some("https://host/channel/archive-{utc}-{duration}.m3u8?token=a%2Fb".into()), - time: None, - correction: Some("0".into()), - catchup_type: Some("flussonic".into()), - extra_attributes: vec![super::CatchupAttribute { name: "catchup-extra".into(), value: "keep".into() }], - flussonic_archive_max_duration_secs: window, - }; - let encoded = rmp_serde::to_vec(&catchup)?; - let restored: CatchupProperties = rmp_serde::from_slice(&encoded)?; - assert_eq!(restored, catchup); - - let properties = StreamProperties::Live(Box::new(LiveStreamProperties { - catchup: Some(catchup), - ..LiveStreamProperties::default() - })); - let encoded = rmp_serde::to_vec(&properties)?; - let restored: StreamProperties = rmp_serde::from_slice(&encoded)?; - assert_eq!(restored, properties); - } - Ok(()) - } - - #[test] - fn catchup_messagepack_reads_legacy_records_without_changing_encoding() -> Result<(), Box> { - // The seven-field positional layout is independent of the current struct definition. - let legacy_fields = ( - Some("fs"), - Some("10"), - Some("https://host/channel/archive-{utc}-{duration}.m3u8?token=a%2Fb"), - None::<&str>, - Some("0"), - Some("flussonic"), - vec![("catchup-extra", "keep")], - ); - let encoded = rmp_serde::to_vec(&legacy_fields)?; - let restored: CatchupProperties = rmp_serde::from_slice(&encoded)?; - assert_eq!(restored.flussonic_archive_max_duration_secs, None); - assert_eq!(restored.mode.as_deref(), Some("fs")); - assert_eq!(restored.days.as_deref(), Some("10")); - assert_eq!(restored.source.as_deref(), legacy_fields.2); - assert_eq!(restored.time, None); - assert_eq!(restored.correction.as_deref(), Some("0")); - assert_eq!(restored.catchup_type.as_deref(), Some("flussonic")); - assert_eq!( - restored.extra_attributes, - vec![super::CatchupAttribute { name: "catchup-extra".into(), value: "keep".into() }] - ); - assert_eq!(rmp_serde::to_vec(&restored)?, encoded); - Ok(()) - } - - #[test] - fn live_catchup_only_counts_as_details() { - let props = StreamProperties::Live(Box::new(LiveStreamProperties { - catchup: Some(CatchupProperties { mode: Some("append".into()), ..CatchupProperties::default() }), - ..LiveStreamProperties::default() - })); - - assert!(props.has_details()); - } - - #[test] - fn native_flussonic_mode_is_canonical_and_respects_mode_precedence() { - for (value, expected) in [ - ("fs", Some("flussonic")), - ("FLUSSONIC", Some("flussonic")), - ("flussonic-hls", Some("flussonic")), - ("Flussonic-TS", Some("flussonic-ts")), - ("append", None), - ] { - let catchup = CatchupProperties { mode: Some(value.into()), ..CatchupProperties::default() }; - assert_eq!(catchup.native_flussonic_player_mode(), expected); - } - - // BitTV often ships catchup="append"/"shift" with catchup-type="flussonic". - // catchup-type must win so path-rewrite archive stays Flussonic (v3.3.81). - let conflicting = CatchupProperties { - mode: Some("append".into()), - catchup_type: Some("flussonic".into()), - ..CatchupProperties::default() - }; - assert_eq!(conflicting.effective_mode(), Some("flussonic")); - assert!(conflicting.is_flussonic()); - assert_eq!(conflicting.native_flussonic_player_mode(), Some("flussonic")); - - let shift_type_wins = CatchupProperties { - mode: Some("flussonic".into()), - catchup_type: Some("shift".into()), - ..CatchupProperties::default() - }; - assert_eq!(shift_type_wins.effective_mode(), Some("shift")); - assert!(!shift_type_wins.is_flussonic()); - assert_eq!(shift_type_wins.native_flussonic_player_mode(), None); - - let whitespace_type = CatchupProperties { - mode: Some("append".into()), - catchup_type: Some(" ".into()), - ..CatchupProperties::default() - }; - assert_eq!(whitespace_type.effective_mode(), Some("append")); - assert_eq!(whitespace_type.append_player_type(), Some("append")); - } - - #[test] - fn positive_live_bitrate_counts_as_details() { - let props = StreamProperties::Live(Box::new(LiveStreamProperties { - bitrate: 2_500_000, - ..LiveStreamProperties::default() - })); - - assert!(props.has_details()); - } - - #[test] - fn live_bitrate_deserializes_from_string_and_defaults_when_missing() { - let measured: LiveStreamProperties = - serde_json::from_value(json!({ "bitrate": "2500000" })).expect("string bitrate should deserialize"); - let unknown: LiveStreamProperties = - serde_json::from_value(json!({})).expect("missing bitrate should deserialize"); - - assert_eq!(measured.bitrate, 2_500_000); - assert_eq!(unknown.bitrate, 0); - } - - #[test] - fn merge_learned_live_metadata_preserves_existing_values_and_higher_bitrate() { - let mut current = LiveStreamProperties { - video: Some("current-video".into()), - bitrate: 1_500_000, - last_probed_timestamp: Some(200), - ..LiveStreamProperties::default() - }; - let previous = LiveStreamProperties { - video: Some("previous-video".into()), - audio: Some("previous-audio".into()), - bitrate: 2_500_000, - last_probed_timestamp: Some(100), - last_success_timestamp: Some(150), - ..LiveStreamProperties::default() - }; - - assert!(current.merge_learned_metadata_from(&previous)); - assert_eq!(current.video.as_deref(), Some("current-video")); - assert_eq!(current.audio.as_deref(), Some("previous-audio")); - assert_eq!(current.bitrate, 2_500_000); - assert_eq!(current.last_probed_timestamp, Some(200)); - assert_eq!(current.last_success_timestamp, Some(150)); - } - - #[test] - fn resource_field_names_exclude_stream_urls() { - for resource in [ - "logo", - "logo_small", - "cover", - "movie_image", - "nfo_cover_big", - "backdrop_path", - "backdrop_path1", - "nfo_backdrop_path", - "nfo_s_2_cover", - "nfo_ep_2_5_movie_image", - ] { - assert!(is_resource_field_name(resource), "{resource}"); - } - for not_a_resource in ["url", "name", "chno", "caption", "epg_channel_id", "stream_url", "nfo_s_2_title"] { - assert!(!is_resource_field_name(not_a_resource), "{not_a_resource}"); - } - } -} +mod catchup; +mod live; +mod resources; +mod series; +mod video; +pub use catchup::{CatchupAttribute, CatchupProperties}; +pub use live::LiveStreamProperties; +pub use resources::is_resource_field_name; +use series::{first_series_episode_value, non_empty_arc, parse_season_episode_field, parse_season_field}; +pub use series::{ + normalize_episode_title, EpisodeStreamProperties, SeriesStreamDetailEpisodeProperties, + SeriesStreamDetailProperties, SeriesStreamDetailSeasonProperties, SeriesStreamProperties, +}; +pub use video::{VideoStreamDetailProperties, VideoStreamProperties}; diff --git a/shared/src/model/stream_properties/catchup.rs b/shared/src/model/stream_properties/catchup.rs new file mode 100644 index 000000000..cbe9fd38c --- /dev/null +++ b/shared/src/model/stream_properties/catchup.rs @@ -0,0 +1,85 @@ +use crate::utils::{arc_str_none_default_on_null, arc_str_option_serde}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +#[derive(Default, Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct CatchupAttribute { + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub name: Arc, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub value: Arc, +} + +#[derive(Default, Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct CatchupProperties { + #[serde(default, with = "arc_str_option_serde")] + pub mode: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub days: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub source: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub time: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub correction: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub catchup_type: Option>, + #[serde(default)] + pub extra_attributes: Vec, + /// Origin window for an automatically generated bounded Flussonic source. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub flussonic_archive_max_duration_secs: Option, +} + +impl CatchupProperties { + pub fn is_empty(&self) -> bool { + self.mode.is_none() + && self.days.is_none() + && self.source.is_none() + && self.time.is_none() + && self.correction.is_none() + && self.catchup_type.is_none() + && self.extra_attributes.is_empty() + && self.flussonic_archive_max_duration_secs.is_none() + } + + /// Prefer `catchup-type` when both fields are present because providers may leave + /// a stale `catchup` mode alongside the authoritative player type. + pub fn effective_mode(&self) -> Option<&str> { + self.catchup_type + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .or_else(|| self.mode.as_deref().map(str::trim).filter(|value| !value.is_empty())) + } + + pub(super) fn is_flussonic_mode_str(mode: Option<&str>) -> bool { + mode.is_some_and(|m| { + matches!(m.trim().to_ascii_lowercase().as_str(), "flussonic" | "flussonic-hls" | "flussonic-ts" | "fs") + }) + } + + /// Prefer `catchup-type` when both are set so a leftover `catchup="shift"`/`append` + /// from a provider cannot steal Flussonic path-rewrite channels (v3.3.81 behavior). + pub fn is_flussonic(&self) -> bool { Self::is_flussonic_mode_str(self.effective_mode()) } + + pub fn native_flussonic_player_mode(&self) -> Option<&'static str> { + if !self.is_flussonic() { + return None; + } + let raw = self.effective_mode().unwrap_or("flussonic"); + if raw.eq_ignore_ascii_case("flussonic-ts") { + Some("flussonic-ts") + } else { + Some("flussonic") + } + } + + /// Canonical append label for unified M3U export (`catchup-type="append"` only). + pub fn append_player_type(&self) -> Option<&'static str> { + if self.native_flussonic_player_mode().is_some() { + return None; + } + self.effective_mode()?.eq_ignore_ascii_case("append").then_some("append") + } +} diff --git a/shared/src/model/stream_properties/live.rs b/shared/src/model/stream_properties/live.rs new file mode 100644 index 000000000..c5fa7d38d --- /dev/null +++ b/shared/src/model/stream_properties/live.rs @@ -0,0 +1,91 @@ +use super::CatchupProperties; +use crate::utils::{ + arc_str_none_default_on_null, arc_str_option_null_if_empty_serde, deserialize_as_option_arc_str, + deserialize_json_as_opt_string, deserialize_number_from_string, deserialize_number_from_string_or_zero, + serialize_json_as_opt_string, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +#[derive(Default, Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct LiveStreamProperties { + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub name: Arc, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub category_id: u32, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub stream_id: u32, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub stream_icon: Arc, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub direct_source: Arc, + #[serde(default, with = "arc_str_option_null_if_empty_serde")] + pub custom_sid: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub added: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub stream_type: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub epg_channel_id: Option>, + #[serde(default, deserialize_with = "deserialize_number_from_string")] + pub tv_archive: Option, + #[serde(default, deserialize_with = "deserialize_number_from_string")] + pub tv_archive_duration: Option, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub is_adult: i32, + // New fields for probing + #[serde( + default, + serialize_with = "serialize_json_as_opt_string", + deserialize_with = "deserialize_json_as_opt_string" + )] + pub video: Option>, + #[serde( + default, + serialize_with = "serialize_json_as_opt_string", + deserialize_with = "deserialize_json_as_opt_string" + )] + pub audio: Option>, + #[serde(default, deserialize_with = "deserialize_number_from_string")] + pub last_probed_timestamp: Option, + #[serde(default, deserialize_with = "deserialize_number_from_string")] + pub last_success_timestamp: Option, + #[serde(default)] + pub catchup: Option, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub bitrate: u32, +} + +impl LiveStreamProperties { + /// Merges metadata learned from probing or playback without replacing newer provider data. + pub fn merge_learned_metadata_from(&mut self, previous: &Self) -> bool { + let mut changed = false; + + if self.video.is_none() && previous.video.is_some() { + self.video.clone_from(&previous.video); + changed = true; + } + + if self.audio.is_none() && previous.audio.is_some() { + self.audio.clone_from(&previous.audio); + changed = true; + } + + if previous.bitrate > self.bitrate { + self.bitrate = previous.bitrate; + changed = true; + } + + if self.last_probed_timestamp.is_none() && previous.last_probed_timestamp.is_some() { + self.last_probed_timestamp = previous.last_probed_timestamp; + changed = true; + } + + if self.last_success_timestamp.is_none() && previous.last_success_timestamp.is_some() { + self.last_success_timestamp = previous.last_success_timestamp; + changed = true; + } + + changed + } +} diff --git a/shared/src/model/stream_properties/resources.rs b/shared/src/model/stream_properties/resources.rs new file mode 100644 index 000000000..24969d7bb --- /dev/null +++ b/shared/src/model/stream_properties/resources.rs @@ -0,0 +1,17 @@ +use super::series::{parse_season_episode_field, parse_season_field}; + +/// Field names accepted by resource-proxy endpoints. This is deliberately defined beside the +/// resource traversal and selector so adding a new endpoint-visible resource has one review site. +/// +/// Stream URLs are not resources: serving them here would sidestep the stream routes and their +/// admission accounting. +pub fn is_resource_field_name(field: &str) -> bool { + if matches!(field, "logo" | "logo_small" | "cover" | "movie_image" | "nfo_cover_big" | "nfo_movie_image") + || field.starts_with("backdrop_path") + || field.starts_with("nfo_backdrop_path") + { + return true; + } + parse_season_field(field).is_some_and(|(_, field)| matches!(field.as_str(), "cover" | "cover_tmdb" | "cover_big")) + || parse_season_episode_field(field).is_some_and(|(_, _, field)| field == "movie_image") +} diff --git a/shared/src/model/stream_properties/series.rs b/shared/src/model/stream_properties/series.rs new file mode 100644 index 000000000..c66362876 --- /dev/null +++ b/shared/src/model/stream_properties/series.rs @@ -0,0 +1,476 @@ +use crate::{ + model::{info_doc_utils::InfoDocUtils, PlaylistEntry, XtreamSeriesInfo, XtreamSeriesInfoDoc}, + utils::{ + arc_str_none_default_on_null, arc_str_null_is_none_serde, arc_str_option_null_if_empty_serde, + arc_str_option_serde, deserialize_as_option_arc_str, deserialize_as_string_array, + deserialize_json_as_opt_string, deserialize_number_from_string, deserialize_number_from_string_or_zero, + serialize_json_as_opt_string, Internable, CONSTANTS, + }, +}; +use log::warn; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +fn format_episode_code(season: u32, episode: u32) -> Option { + match (season, episode) { + (0, 0) => None, + (season, 0) => Some(format!("S{season:02}")), + (0, episode) => Some(format!("E{episode:02}")), + (season, episode) => Some(format!("S{season:02}E{episode:02}")), + } +} + +fn title_contains_episode_code(title: &str) -> bool { CONSTANTS.re_episode_code.is_match(title) } + +pub(super) fn first_series_episode_value(series: &SeriesStreamProperties, selector: F) -> Option +where + F: FnOnce(&SeriesStreamDetailEpisodeProperties) -> Option, +{ + series + .details + .as_ref() + .and_then(|details| details.episodes.as_ref()) + .and_then(|episodes| episodes.first()) + .and_then(selector) +} + +pub fn normalize_episode_title(raw_title: &Arc, series_name: &Arc, season: u32, episode: u32) -> Arc { + let title = raw_title.trim(); + let series_name = series_name.trim(); + let Some(code) = format_episode_code(season, episode) else { + return if title.is_empty() { series_name.intern() } else { title.intern() }; + }; + + if title.is_empty() || (!series_name.is_empty() && title.eq_ignore_ascii_case(series_name)) { + return code.into(); + } + + if title_contains_episode_code(title) { + return title.intern(); + } + + format!("{code} - {title}").into() +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct SeriesStreamDetailSeasonProperties { + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub name: Arc, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub season_number: u32, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub episode_count: u32, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub overview: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub air_date: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub cover: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub cover_tmdb: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub cover_big: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub duration: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] +pub struct SeriesStreamDetailEpisodeProperties { + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub id: u32, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub episode_num: u32, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub season: u32, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub title: Arc, + #[serde(default, deserialize_with = "arc_str_null_is_none_serde::deserialize")] + pub container_extension: Arc, + #[serde(default, with = "arc_str_option_null_if_empty_serde")] + pub custom_sid: Option>, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub added: Arc, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub direct_source: Arc, + #[serde(default, deserialize_with = "deserialize_number_from_string")] + pub tmdb: Option, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub release_date: Arc, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub series_release_date: Option>, // Global series release date + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub plot: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub crew: Option>, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub duration_secs: u32, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub duration: Arc, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub movie_image: Arc, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub bitrate: u32, + #[serde(default, deserialize_with = "deserialize_number_from_string")] + pub rating: Option, + #[serde( + default, + serialize_with = "serialize_json_as_opt_string", + deserialize_with = "deserialize_json_as_opt_string" + )] + pub video: Option>, + #[serde( + default, + serialize_with = "serialize_json_as_opt_string", + deserialize_with = "deserialize_json_as_opt_string" + )] + pub audio: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct SeriesStreamDetailProperties { + #[serde(default, deserialize_with = "deserialize_number_from_string")] + pub year: Option, + #[serde(default)] + pub seasons: Option>, + #[serde(default)] + pub episodes: Option>, +} + +impl SeriesStreamDetailProperties { + /// Builds a `SeriesStreamDetailProperties`. Empty `seasons` is normalized to + /// `None` so downstream code can short-circuit; `episodes` is kept as-is. + pub fn new( + year: Option, + seasons: Vec, + episodes: Option>, + ) -> Self { + Self { year, seasons: if seasons.is_empty() { None } else { Some(seasons) }, episodes } + } +} + +#[derive(Default, Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct SeriesStreamProperties { + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub name: Arc, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub category_id: u32, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub series_id: u32, + #[serde(default, deserialize_with = "deserialize_as_string_array")] + pub backdrop_path: Option>>, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub cast: Arc, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub cover: Arc, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub director: Arc, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub episode_run_time: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub genre: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub last_modified: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub plot: Option>, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub rating: f64, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub rating_5based: f64, + #[serde(default, with = "arc_str_option_serde")] + pub release_date: Option>, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub youtube_trailer: Arc, + #[serde(default, deserialize_with = "deserialize_number_from_string")] + pub tmdb: Option, + #[serde(default)] + pub details: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct EpisodeStreamProperties { + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub episode_id: u32, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub episode: u32, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub season: u32, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub added: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub release_date: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub series_release_date: Option>, // Global series release date + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub plot: Option>, + #[serde(default, deserialize_with = "deserialize_number_from_string")] + pub tmdb: Option, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub movie_image: Arc, + #[serde(default, deserialize_with = "arc_str_null_is_none_serde::deserialize")] + pub container_extension: Arc, + #[serde( + default, + serialize_with = "serialize_json_as_opt_string", + deserialize_with = "deserialize_json_as_opt_string" + )] + pub video: Option>, + #[serde( + default, + serialize_with = "serialize_json_as_opt_string", + deserialize_with = "deserialize_json_as_opt_string" + )] + pub audio: Option>, +} + +pub(super) fn parse_season_field(s: &str) -> Option<(u32, String)> { + let mut parts = s.split('_'); + + let (prefix, suffix) = (parts.next()?, parts.next()?); + if prefix != "nfo" || suffix != "s" { + return None; + } + + let season: u32 = parts.next()?.parse().ok()?; + + let kind = join_parts(parts, '_'); + if kind.is_empty() { + return None; + } + + Some((season, kind)) +} + +pub(super) fn parse_season_episode_field(s: &str) -> Option<(u32, u32, String)> { + let mut parts = s.split('_'); + + let (prefix, ep) = (parts.next()?, parts.next()?); + if prefix != "nfo" || ep != "ep" { + return None; + } + + let season: u32 = parts.next()?.parse().ok()?; + let episode: u32 = parts.next()?.parse().ok()?; + + let kind = join_parts(parts, '_'); + if kind.is_empty() { + return None; + } + + Some((season, episode, kind)) +} + +fn join_parts<'a>(parts: impl Iterator, separator: char) -> String { + let mut result = String::new(); + for part in parts { + if !result.is_empty() { + result.push(separator); + } + result.push_str(part); + } + result +} + +impl SeriesStreamProperties { + pub(super) fn from_info_base(info: &XtreamSeriesInfo, series_id: u32) -> SeriesStreamProperties { + SeriesStreamProperties { + name: info.info.name.clone(), + category_id: info.info.category_id, + series_id, + backdrop_path: info.info.backdrop_path.clone(), + cast: info.info.cast.clone(), + cover: info.info.cover.clone(), + director: info.info.director.clone(), + episode_run_time: Some(info.info.episode_run_time.clone()), + genre: Some(info.info.genre.clone()), + last_modified: Some(info.info.last_modified.clone()), + plot: Some(info.info.plot.clone()), + rating: info.info.rating, + rating_5based: info.info.rating_5based, + release_date: Some(info.info.release_date.clone()), + youtube_trailer: info.info.youtube_trailer.clone(), + tmdb: info.info.tmdb, + details: Some(SeriesStreamDetailProperties { + year: InfoDocUtils::extract_year_from_release_date(&info.info.release_date), + seasons: info.seasons.as_ref().map(|list| { + let mut seasons: Vec = list + .iter() + .map(|s| SeriesStreamDetailSeasonProperties { + name: s.name.clone(), + season_number: s.season_number, + episode_count: s.episode_count, + overview: Some(s.overview.clone()), + air_date: Some(s.air_date.clone()), + cover: Some(s.cover.clone()), + cover_tmdb: Some(s.cover_tmdb.clone()), + cover_big: Some(s.cover_big.clone()), + duration: Some(s.duration.clone()), + }) + .collect(); + seasons.sort_by_key(|season| season.season_number); + seasons + }), + episodes: info.episodes.as_ref().map(|list| { + let mut episodes: Vec = list + .iter() + .map(|e| SeriesStreamDetailEpisodeProperties { + id: e.id, + episode_num: e.episode_num, + season: e.season, + title: normalize_episode_title(&e.title, &info.info.name, e.season, e.episode_num), + container_extension: e.container_extension.clone(), + custom_sid: e.custom_sid.clone(), + added: e.added.clone(), + direct_source: e.direct_source.clone(), + tmdb: info.info.tmdb, + release_date: e.info.as_ref().map(|i| i.air_date.clone()).unwrap_or_default(), + series_release_date: None, + plot: e.info.as_ref().and_then(|i| i.plot.clone()), + crew: e.info.as_ref().map(|i| i.crew.clone()), + duration_secs: e.info.as_ref().map(|i| i.duration_secs).unwrap_or_default(), + duration: e.info.as_ref().map(|i| i.duration.clone()).unwrap_or_default(), + movie_image: e.info.as_ref().map(|i| i.movie_image.clone()).unwrap_or_default(), + bitrate: e.info.as_ref().map(|i| i.bitrate).unwrap_or_default(), + rating: e.info.as_ref().map(|i| i.rating), + video: e.info.as_ref().map(|i| i.video.clone()).unwrap_or_default(), + audio: e.info.as_ref().map(|i| i.audio.clone()).unwrap_or_default(), + }) + .collect(); + episodes.sort_by_key(|episode| (episode.season, episode.episode_num)); + episodes + }), + }), + } + } + + pub fn from_info

(info: &XtreamSeriesInfo, pli: &P) -> SeriesStreamProperties + where + P: PlaylistEntry, + { + Self::from_info_base(info, pli.get_virtual_id().get()) + } + + pub fn from_info_without_existing(info: &XtreamSeriesInfo, series_id: u32) -> SeriesStreamProperties { + Self::from_info_base(info, series_id) + } + + pub fn from_info_doc(info: &XtreamSeriesInfoDoc, series_id: u32) -> SeriesStreamProperties { + let tmdb = info.info.tmdb.parse::().ok(); + SeriesStreamProperties { + name: info.info.name.clone(), + category_id: info.info.category_id.parse::().unwrap_or_else(|_| { + warn!("Failed to parse category_id {}", info.info.category_id); + 0 + }), + series_id, + backdrop_path: Some(info.info.backdrop_path.clone()), + cast: info.info.cast.clone(), + cover: info.info.cover.clone(), + director: info.info.director.clone(), + episode_run_time: Some(info.info.episode_run_time.clone()), + genre: Some(info.info.genre.clone()), + last_modified: Some(info.info.last_modified.clone()), + plot: Some(info.info.plot.clone()), + rating: info.info.rating.parse::().unwrap_or_else(|_| { + warn!("Failed to parse rating {}", info.info.rating); + 0.0 + }), + rating_5based: info.info.rating_5based.parse::().unwrap_or_else(|_| { + warn!("Failed to parse rating_5based {}", info.info.rating_5based); + 0.0 + }), + release_date: Some(info.info.release_date.clone()), + youtube_trailer: info.info.youtube_trailer.clone(), + tmdb, + details: Some(SeriesStreamDetailProperties { + year: InfoDocUtils::extract_year_from_release_date(&info.info.release_date), + seasons: { + let mut seasons: Vec = info + .seasons + .iter() + .map(|s| SeriesStreamDetailSeasonProperties { + name: s.name.clone(), + season_number: s.season_number, + episode_count: s.episode_count.parse::().unwrap_or_else(|_| { + warn!("Failed to parse episode_count {}", s.episode_count); + 0 + }), + overview: s.overview.clone(), + air_date: s.air_date.clone(), + cover: s.cover.clone(), + cover_tmdb: s.cover_tmdb.clone(), + cover_big: s.cover_big.clone(), + duration: s.duration.clone(), + }) + .collect(); + seasons.sort_by_key(|season| season.season_number); + Some(seasons) + }, + episodes: { + let mut episodes: Vec = info + .episodes + .iter() + .flat_map(|(_, list)| list.iter()) + .map(|e| SeriesStreamDetailEpisodeProperties { + id: e.id.parse::().unwrap_or_else(|_| { + warn!("Failed to parse episode id {}", e.id); + 0 + }), + episode_num: e.episode_num, + season: e.season, + title: normalize_episode_title(&e.title, &info.info.name, e.season, e.episode_num), + container_extension: e.container_extension.clone(), + custom_sid: e.custom_sid.clone(), + added: e.added.clone(), + direct_source: e.direct_source.clone(), + tmdb, + release_date: e.info.air_date.clone(), + series_release_date: None, + plot: e.info.plot.clone(), + crew: e.info.crew.clone(), + duration_secs: e.info.duration_secs, + duration: e.info.duration.clone(), + movie_image: e.info.movie_image.clone(), + bitrate: e.info.bitrate, + rating: Some(e.info.rating), + video: None, + audio: None, + }) + .collect(); + episodes.sort_by_key(|episode| (episode.season, episode.episode_num)); + Some(episodes) + }, + }), + } + } +} + +impl EpisodeStreamProperties { + pub fn from_series( + series: &SeriesStreamProperties, + episode: &SeriesStreamDetailEpisodeProperties, + ) -> EpisodeStreamProperties { + EpisodeStreamProperties { + episode_id: episode.id, + episode: episode.episode_num, + season: episode.season, + added: Some(episode.added.clone()), + release_date: Some(episode.release_date.clone()), + // Inherit global release date from the Series object if available + series_release_date: series.release_date.clone(), + plot: episode.plot.clone(), + tmdb: episode.tmdb.or(series.tmdb), + movie_image: episode.movie_image.clone(), + container_extension: episode.container_extension.clone(), + video: episode.video.clone(), + audio: episode.audio.clone(), + } + } +} + +pub(super) fn non_empty_arc(s: &Arc) -> Option> { + if s.is_empty() { + None + } else { + Some(Arc::clone(s)) + } +} diff --git a/shared/src/model/stream_properties/tests.rs b/shared/src/model/stream_properties/tests.rs new file mode 100644 index 000000000..7f9fd04ea --- /dev/null +++ b/shared/src/model/stream_properties/tests.rs @@ -0,0 +1,342 @@ +use super::*; +use crate::model::{PlaylistItemType, VirtualId, XtreamCluster, XtreamPlaylistItem, XtreamSeriesInfo, XtreamVideoInfo}; +use serde_json::json; + +#[test] +fn genre_reads_from_wherever_the_variant_keeps_it() { + let mut video = StreamProperties::Video(Box::default()); + assert_eq!(video.genre(), None, "video with no details has no genre"); + assert!(video.set_genre("Action"), "video accepts a genre"); + assert_eq!(video.genre().map(Arc::as_ref), Some("Action"), "video keeps it under details"); + // Setting again goes through the existing details rather than replacing them. + assert!(video.set_genre("Drama")); + assert_eq!(video.genre().map(Arc::as_ref), Some("Drama")); + + let mut series = StreamProperties::Series(Box::default()); + assert_eq!(series.genre(), None); + assert!(series.set_genre("Comedy"), "series accepts a genre"); + assert_eq!(series.genre().map(Arc::as_ref), Some("Comedy"), "series keeps it inline"); +} + +#[test] +fn live_and_episode_have_no_genre_and_refuse_to_take_one() { + // EpisodeStreamProperties has no Default impl, but every field is + // #[serde(default)], so an empty object builds one. + let episode: EpisodeStreamProperties = + serde_json::from_value(json!({})).expect("episode properties default from an empty object"); + for mut props in [StreamProperties::Live(Box::default()), StreamProperties::Episode(Box::new(episode))] { + assert_eq!(props.genre(), None); + assert!(!props.set_genre("Action"), "this variant has no genre to set"); + assert_eq!(props.genre(), None); + } +} + +fn sample_video_info(tmdb_id: &str) -> XtreamVideoInfo { + serde_json::from_value(json!({ + "info": { + "name": "Fetched Movie", + "tmdb_id": tmdb_id, + "movie_image": "https://img.example/movie.jpg", + "video": { "codec": "h264" }, + "audio": { "codec": "aac" } + }, + "movie_data": { + "name": "Fetched Movie", + "category_id": "99", + "stream_id": "4242", + "direct_source": "https://cdn.example/stream.mkv", + "added": "1700000000", + "container_extension": "mkv" + } + })) + .expect("sample XtreamVideoInfo should deserialize") +} + +fn existing_video_item() -> XtreamPlaylistItem { + XtreamPlaylistItem { + virtual_id: VirtualId::new(1), + provider_id: 1001, + name: "Existing".into(), + logo: "".into(), + logo_small: "".into(), + group: "".into(), + title: "Existing title".into(), + parent_code: "".into(), + rec: "".into(), + url: "https://provider.example/stream".into(), + epg_channel_id: None, + xtream_cluster: XtreamCluster::Video, + additional_properties: Some(StreamProperties::Video(Box::new(VideoStreamProperties { + rating: Some(7.5), + rating_5based: Some(3.7), + stream_type: Some("legacy-type".into()), + trailer: Some("legacy-trailer".into()), + tmdb: Some(7788), + is_adult: 1, + ..Default::default() + }))), + item_type: PlaylistItemType::Video, + category_id: 99, + input_name: "input".into(), + channel_no: 0, + source_ordinal: 0, + input_stream_id: "1001".into(), + upstream_user_agent: None, + } +} + +#[test] +fn from_info_without_existing_keeps_fetched_metadata() { + let info = sample_video_info("12345"); + + let props = VideoStreamProperties::from_info_without_existing(&info); + let details = props.details.expect("details should be present"); + + assert_eq!(props.name.as_ref(), "Fetched Movie"); + assert_eq!(props.stream_id, 4242); + assert_eq!(props.tmdb, Some(12345)); + assert_eq!(props.stream_type.as_deref(), Some("movie")); + assert!(details.video.is_some()); + assert!(details.audio.is_some()); +} + +#[test] +fn from_info_with_existing_keeps_existing_overrides() { + let info = sample_video_info(""); + let existing = existing_video_item(); + + let props = VideoStreamProperties::from_info(&info, &existing); + + assert_eq!(props.rating, Some(7.5)); + assert_eq!(props.rating_5based, Some(3.7)); + assert_eq!(props.stream_type.as_deref(), Some("legacy-type")); + assert_eq!(props.trailer.as_deref(), Some("legacy-trailer")); + assert_eq!(props.tmdb, Some(7788)); + assert_eq!(props.is_adult, 1); +} + +#[test] +fn from_series_info_preserves_episode_plot() { + let info: XtreamSeriesInfo = serde_json::from_value(json!({ + "info": { + "name": "Example Show" + }, + "episodes": { + "1": [{ + "id": 101, + "episode_num": 1, + "season": 1, + "title": "Pilot", + "container_extension": "mkv", + "info": { + "air_date": "2024-01-01", + "plot": "Episode plot", + "movie_image": "https://img.example/still.jpg" + } + }] + } + })) + .expect("sample XtreamSeriesInfo should deserialize"); + + let props = SeriesStreamProperties::from_info_without_existing(&info, 7); + let episode = props + .details + .and_then(|details| details.episodes) + .and_then(|episodes| episodes.first().cloned()) + .expect("episode should be mapped"); + + assert_eq!(episode.plot.as_deref(), Some("Episode plot")); +} + +#[test] +fn normalize_episode_title_injects_missing_episode_code() { + let normalized = normalize_episode_title(&"Pilot".into(), &"Example Show".into(), 1, 2); + assert_eq!(normalized.as_ref(), "S01E02 - Pilot"); +} + +#[test] +fn normalize_episode_title_replaces_series_name_only_with_episode_code() { + let normalized = normalize_episode_title(&"Example Show".into(), &"Example Show".into(), 1, 2); + assert_eq!(normalized.as_ref(), "S01E02"); +} + +#[test] +fn normalize_episode_title_keeps_existing_episode_code() { + let normalized = normalize_episode_title(&"S01E02".into(), &"Example Show".into(), 1, 2); + assert_eq!(normalized.as_ref(), "S01E02"); +} + +#[test] +fn catchup_messagepack_round_trip_preserves_archive_window() -> Result<(), Box> { + for window in [None, Some(14400)] { + let catchup = CatchupProperties { + mode: Some("fs".into()), + days: Some("10".into()), + source: Some("https://host/channel/archive-{utc}-{duration}.m3u8?token=a%2Fb".into()), + time: None, + correction: Some("0".into()), + catchup_type: Some("flussonic".into()), + extra_attributes: vec![super::CatchupAttribute { name: "catchup-extra".into(), value: "keep".into() }], + flussonic_archive_max_duration_secs: window, + }; + let encoded = rmp_serde::to_vec(&catchup)?; + let restored: CatchupProperties = rmp_serde::from_slice(&encoded)?; + assert_eq!(restored, catchup); + + let properties = StreamProperties::Live(Box::new(LiveStreamProperties { + catchup: Some(catchup), + ..LiveStreamProperties::default() + })); + let encoded = rmp_serde::to_vec(&properties)?; + let restored: StreamProperties = rmp_serde::from_slice(&encoded)?; + assert_eq!(restored, properties); + } + Ok(()) +} + +#[test] +fn catchup_messagepack_reads_legacy_records_without_changing_encoding() -> Result<(), Box> { + // The seven-field positional layout is independent of the current struct definition. + let legacy_fields = ( + Some("fs"), + Some("10"), + Some("https://host/channel/archive-{utc}-{duration}.m3u8?token=a%2Fb"), + None::<&str>, + Some("0"), + Some("flussonic"), + vec![("catchup-extra", "keep")], + ); + let encoded = rmp_serde::to_vec(&legacy_fields)?; + let restored: CatchupProperties = rmp_serde::from_slice(&encoded)?; + assert_eq!(restored.flussonic_archive_max_duration_secs, None); + assert_eq!(restored.mode.as_deref(), Some("fs")); + assert_eq!(restored.days.as_deref(), Some("10")); + assert_eq!(restored.source.as_deref(), legacy_fields.2); + assert_eq!(restored.time, None); + assert_eq!(restored.correction.as_deref(), Some("0")); + assert_eq!(restored.catchup_type.as_deref(), Some("flussonic")); + assert_eq!( + restored.extra_attributes, + vec![super::CatchupAttribute { name: "catchup-extra".into(), value: "keep".into() }] + ); + assert_eq!(rmp_serde::to_vec(&restored)?, encoded); + Ok(()) +} + +#[test] +fn live_catchup_only_counts_as_details() { + let props = StreamProperties::Live(Box::new(LiveStreamProperties { + catchup: Some(CatchupProperties { mode: Some("append".into()), ..CatchupProperties::default() }), + ..LiveStreamProperties::default() + })); + + assert!(props.has_details()); +} + +#[test] +fn native_flussonic_mode_is_canonical_and_respects_mode_precedence() { + for (value, expected) in [ + ("fs", Some("flussonic")), + ("FLUSSONIC", Some("flussonic")), + ("flussonic-hls", Some("flussonic")), + ("Flussonic-TS", Some("flussonic-ts")), + ("append", None), + ] { + let catchup = CatchupProperties { mode: Some(value.into()), ..CatchupProperties::default() }; + assert_eq!(catchup.native_flussonic_player_mode(), expected); + } + + // BitTV often ships catchup="append"/"shift" with catchup-type="flussonic". + // catchup-type must win so path-rewrite archive stays Flussonic (v3.3.81). + let conflicting = CatchupProperties { + mode: Some("append".into()), + catchup_type: Some("flussonic".into()), + ..CatchupProperties::default() + }; + assert_eq!(conflicting.effective_mode(), Some("flussonic")); + assert!(conflicting.is_flussonic()); + assert_eq!(conflicting.native_flussonic_player_mode(), Some("flussonic")); + + let shift_type_wins = CatchupProperties { + mode: Some("flussonic".into()), + catchup_type: Some("shift".into()), + ..CatchupProperties::default() + }; + assert_eq!(shift_type_wins.effective_mode(), Some("shift")); + assert!(!shift_type_wins.is_flussonic()); + assert_eq!(shift_type_wins.native_flussonic_player_mode(), None); + + let whitespace_type = CatchupProperties { + mode: Some("append".into()), + catchup_type: Some(" ".into()), + ..CatchupProperties::default() + }; + assert_eq!(whitespace_type.effective_mode(), Some("append")); + assert_eq!(whitespace_type.append_player_type(), Some("append")); +} + +#[test] +fn positive_live_bitrate_counts_as_details() { + let props = StreamProperties::Live(Box::new(LiveStreamProperties { + bitrate: 2_500_000, + ..LiveStreamProperties::default() + })); + + assert!(props.has_details()); +} + +#[test] +fn live_bitrate_deserializes_from_string_and_defaults_when_missing() { + let measured: LiveStreamProperties = + serde_json::from_value(json!({ "bitrate": "2500000" })).expect("string bitrate should deserialize"); + let unknown: LiveStreamProperties = serde_json::from_value(json!({})).expect("missing bitrate should deserialize"); + + assert_eq!(measured.bitrate, 2_500_000); + assert_eq!(unknown.bitrate, 0); +} + +#[test] +fn merge_learned_live_metadata_preserves_existing_values_and_higher_bitrate() { + let mut current = LiveStreamProperties { + video: Some("current-video".into()), + bitrate: 1_500_000, + last_probed_timestamp: Some(200), + ..LiveStreamProperties::default() + }; + let previous = LiveStreamProperties { + video: Some("previous-video".into()), + audio: Some("previous-audio".into()), + bitrate: 2_500_000, + last_probed_timestamp: Some(100), + last_success_timestamp: Some(150), + ..LiveStreamProperties::default() + }; + + assert!(current.merge_learned_metadata_from(&previous)); + assert_eq!(current.video.as_deref(), Some("current-video")); + assert_eq!(current.audio.as_deref(), Some("previous-audio")); + assert_eq!(current.bitrate, 2_500_000); + assert_eq!(current.last_probed_timestamp, Some(200)); + assert_eq!(current.last_success_timestamp, Some(150)); +} + +#[test] +fn resource_field_names_exclude_stream_urls() { + for resource in [ + "logo", + "logo_small", + "cover", + "movie_image", + "nfo_cover_big", + "backdrop_path", + "backdrop_path1", + "nfo_backdrop_path", + "nfo_s_2_cover", + "nfo_ep_2_5_movie_image", + ] { + assert!(is_resource_field_name(resource), "{resource}"); + } + for not_a_resource in ["url", "name", "chno", "caption", "epg_channel_id", "stream_url", "nfo_s_2_title"] { + assert!(!is_resource_field_name(not_a_resource), "{not_a_resource}"); + } +} diff --git a/shared/src/model/stream_properties/video.rs b/shared/src/model/stream_properties/video.rs new file mode 100644 index 000000000..30c60f7c2 --- /dev/null +++ b/shared/src/model/stream_properties/video.rs @@ -0,0 +1,182 @@ +use super::StreamProperties; +use crate::{ + model::{PlaylistEntry, XtreamVideoInfo}, + utils::{ + arc_str_none_default_on_null, arc_str_null_is_none_serde, arc_str_option_null_if_empty_serde, + arc_str_option_serde, deserialize_as_option_arc_str, deserialize_as_string_array, + deserialize_json_as_opt_string, deserialize_number_from_string, deserialize_number_from_string_or_zero, + serialize_json_as_opt_string, Internable, + }, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +#[derive(Default, Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct VideoStreamDetailProperties { + #[serde(default, with = "arc_str_option_serde")] + pub kinopoisk_url: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub o_name: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub cover_big: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub movie_image: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub release_date: Option>, + #[serde(default, deserialize_with = "deserialize_number_from_string")] + pub episode_run_time: Option, + #[serde(default, with = "arc_str_option_serde")] + pub youtube_trailer: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub director: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub actors: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub cast: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub description: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub plot: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub age: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub mpaa_rating: Option>, + #[serde(default)] + pub rating_count_kinopoisk: u32, + #[serde(default, with = "arc_str_option_serde")] + pub country: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub genre: Option>, + #[serde(default, deserialize_with = "deserialize_as_string_array")] + pub backdrop_path: Option>>, + #[serde(default, with = "arc_str_option_serde")] + pub duration_secs: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub duration: Option>, + #[serde( + default, + serialize_with = "serialize_json_as_opt_string", + deserialize_with = "deserialize_json_as_opt_string" + )] + pub video: Option>, + #[serde( + default, + serialize_with = "serialize_json_as_opt_string", + deserialize_with = "deserialize_json_as_opt_string" + )] + pub audio: Option>, + #[serde(default)] + pub bitrate: u32, + #[serde(default, with = "arc_str_option_serde")] + pub runtime: Option>, + #[serde(default, with = "arc_str_option_serde")] + pub status: Option>, +} + +#[derive(Default, Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct VideoStreamProperties { + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub name: Arc, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub category_id: u32, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub stream_id: u32, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub stream_icon: Arc, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub direct_source: Arc, + #[serde(default, with = "arc_str_option_null_if_empty_serde")] + pub custom_sid: Option>, + #[serde(default, deserialize_with = "arc_str_none_default_on_null")] + pub added: Arc, + #[serde(default, deserialize_with = "arc_str_null_is_none_serde::deserialize")] + pub container_extension: Arc, + #[serde(default, deserialize_with = "deserialize_number_from_string")] + pub rating: Option, + #[serde(default, deserialize_with = "deserialize_number_from_string")] + pub rating_5based: Option, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub stream_type: Option>, + #[serde(default, deserialize_with = "deserialize_as_option_arc_str")] + pub trailer: Option>, + #[serde(default, deserialize_with = "deserialize_number_from_string")] + pub tmdb: Option, + #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] + pub is_adult: i32, + #[serde(default)] + pub details: Option, +} + +impl VideoStreamProperties { + pub(super) fn from_info_base(info: &XtreamVideoInfo) -> VideoStreamProperties { + VideoStreamProperties { + name: info.info.name.clone(), + category_id: info.movie_data.category_id, + stream_id: info.movie_data.stream_id, + stream_icon: info.info.movie_image.as_ref().map_or_else(|| "".intern(), Clone::clone), + direct_source: info.movie_data.direct_source.clone(), + custom_sid: info.movie_data.custom_sid.clone(), + added: info.movie_data.added.clone(), + container_extension: info.movie_data.container_extension.clone(), + rating: None, // from PlaylistItem + rating_5based: None, // from PlaylistItem + stream_type: Some("movie".intern()), + trailer: info.info.youtube_trailer.clone(), + tmdb: info.info.tmdb_id.parse::().ok(), + is_adult: 0, // from PlaylistItem + details: Some(VideoStreamDetailProperties { + kinopoisk_url: info.info.kinopoisk_url.clone(), + o_name: info.info.o_name.clone(), + cover_big: info.info.cover_big.clone(), + movie_image: info.info.movie_image.clone(), + release_date: info.info.releasedate.clone(), + episode_run_time: info.info.episode_run_time, + youtube_trailer: info.info.youtube_trailer.clone(), + director: info.info.director.clone(), + actors: info.info.actors.clone(), + cast: info.info.cast.clone(), + description: info.info.description.clone(), + plot: info.info.plot.clone(), + age: info.info.age.clone(), + mpaa_rating: info.info.mpaa_rating.clone(), + rating_count_kinopoisk: info.info.rating_count_kinopoisk, + country: info.info.country.clone(), + genre: info.info.genre.clone(), + backdrop_path: info.info.backdrop_path.clone(), + duration_secs: info.info.duration_secs.clone(), + duration: info.info.duration.clone(), + video: info.info.video.clone(), + audio: info.info.audio.clone(), + bitrate: info.info.bitrate, + runtime: info.info.runtime.clone(), + status: info.info.status.clone(), + }), + } + } + + pub fn from_info

(info: &XtreamVideoInfo, pli: &P) -> VideoStreamProperties + where + P: PlaylistEntry, + { + let mut props = VideoStreamProperties::from_info_base(info); + + if let Some(StreamProperties::Video(video)) = pli.get_additional_properties() { + props.rating = video.rating; + props.rating_5based = video.rating_5based; + props.stream_type = video.stream_type.clone(); + if props.tmdb.is_none() { + props.tmdb = video.tmdb; + } + if props.trailer.is_none() { + props.trailer = video.trailer.clone(); + } + props.is_adult = video.is_adult; + } + + props + } + + pub fn from_info_without_existing(info: &XtreamVideoInfo) -> VideoStreamProperties { + VideoStreamProperties::from_info_base(info) + } +} diff --git a/shared/src/model/user_settings.rs b/shared/src/model/user_settings.rs index 2fea5e010..069abb421 100644 --- a/shared/src/model/user_settings.rs +++ b/shared/src/model/user_settings.rs @@ -92,7 +92,7 @@ mod tests { layout.column_order = vec!["future".into(), "future".into()]; assert_eq!(layout.validate(true), Err("settings_payload_invalid")); assert_eq!(layout.validate(false), Ok(())); - layout.column_order = vec!["".into(), "".into()]; + layout.column_order = vec![String::new(), String::new()]; assert_eq!(layout.validate(true), Err("settings_payload_invalid")); assert_eq!(layout.validate(false), Err("settings_limits_exceeded")); layout.column_visibility.clear(); diff --git a/shared/src/utils/hash_utils.rs b/shared/src/utils/hash_utils.rs index 048a43a84..a8c099af9 100644 --- a/shared/src/utils/hash_utils.rs +++ b/shared/src/utils/hash_utils.rs @@ -304,568 +304,4 @@ fn parse_sxxeyy_fallback(token: &str) -> Option<(u32, u32)> { } #[cfg(test)] -mod tests { - use super::*; - use crate::utils::{constants::EPISODE_PATTERN, CONSTANTS}; - use regex::Regex; - - // ── extract_id_from_url ──────────────────────────────────────────── - - #[test] - fn numeric_id_with_ts_extension() { - assert_eq!(extract_id_from_url("http://server.com/live/user/pass/950327.ts"), "950327"); - } - - #[test] - fn numeric_id_with_mkv_extension() { - assert_eq!(extract_id_from_url("http://server.com/movie/user/pass/12345.mkv"), "12345"); - } - - #[test] - fn numeric_id_without_extension() { - assert_eq!(extract_id_from_url("http://server.com/live/user/pass/950327"), "950327"); - } - - #[test] - fn numeric_id_with_query_string() { - assert_eq!( - extract_id_from_url("http://server.com/live/user/pass/1905905.ts?expires=1712345678&token=abc"), - "1905905" - ); - } - - #[test] - fn numeric_id_with_trailing_slash() { - assert_eq!(extract_id_from_url("http://server.com/live/user/pass/42/"), "42"); - } - - #[test] - fn non_numeric_last_segment_returns_hash() { - let result = extract_id_from_url("http://server.com/live/user/pass/hello.ts"); - assert!(!result.chars().all(|c| c.is_ascii_digit()), "should be a hash, not a numeric id"); - assert!(!result.is_empty()); - } - - #[test] - fn mixed_alphanumeric_segment_returns_hash() { - let result = extract_id_from_url("http://server.com/live/user/pass/abc123.ts"); - assert!(!result.chars().all(|c| c.is_ascii_digit()), "mixed segment should not be treated as numeric id"); - } - - #[test] - fn no_slash_returns_hash() { - let result = extract_id_from_url("12345"); - assert!(!result.chars().all(|c| c.is_ascii_digit()), "bare number without slash must not be parsed as id"); - } - - #[test] - fn empty_url_returns_hash() { - let result = extract_id_from_url(""); - assert!(!result.is_empty()); - } - - #[test] - fn only_slashes_returns_hash() { - let result = extract_id_from_url("///"); - assert!(!result.is_empty()); - } - - #[test] - fn numeric_id_after_scheme() { - assert_eq!(extract_id_from_url("http://server.com/42"), "42"); - } - - #[test] - fn same_url_same_hash() { - let a = extract_id_from_url("http://server.com/stream/hello.m3u8"); - let b = extract_id_from_url("http://server.com/stream/hello.m3u8"); - assert_eq!(a, b, "same non-numeric url should produce identical hash"); - } - - #[test] - fn different_url_different_hash() { - let a = extract_id_from_url("http://a.com/hello.ts"); - let b = extract_id_from_url("http://b.com/world.ts"); - assert_ne!(a, b); - } - - #[test] - fn query_string_ignored_for_numeric_extraction() { - // The query string should not interfere with numeric extraction - assert_eq!(extract_id_from_url("http://server.com/99999.ts?token=xyz"), "99999"); - } - - #[test] - fn fragment_after_numeric_id() { - // Fragments don't contain '?' so they stay in the path — still numeric after '/' - assert_eq!(extract_id_from_url("http://server.com/live/55555.ts#start"), "55555"); - } - - #[test] - fn non_numeric_query() { - assert_eq!( - extract_id_from_url("http://server.com/player_api.php?username=123443&password=1000"), - "7D2F521803B23A43" - ); - } - - // ── extract_numeric_id_from_url ──────────────────────────────────── - - #[test] - fn numeric_url_id_standard() { - assert_eq!(extract_numeric_id_from_url("http://server.com/live/user/pass/950327.ts"), Some(950327)); - } - - #[test] - fn numeric_url_id_no_extension() { - assert_eq!(extract_numeric_id_from_url("http://server.com/live/user/pass/42"), Some(42)); - } - - #[test] - fn numeric_url_id_with_query() { - assert_eq!( - extract_numeric_id_from_url("http://server.com/live/user/pass/1905905.ts?token=abc&expires=999"), - Some(1905905) - ); - } - - #[test] - fn numeric_url_id_trailing_slash() { - assert_eq!(extract_numeric_id_from_url("http://server.com/path/123/"), Some(123)); - } - - #[test] - fn numeric_url_id_zero() { - assert_eq!(extract_numeric_id_from_url("http://server.com/live/0.ts"), Some(0)); - } - - #[test] - fn numeric_url_id_non_numeric_segment() { - assert_eq!(extract_numeric_id_from_url("http://server.com/live/hello.ts"), None); - } - - #[test] - fn numeric_url_id_mixed_alphanumeric() { - assert_eq!(extract_numeric_id_from_url("http://server.com/live/abc123.ts"), None); - } - - #[test] - fn numeric_url_id_no_slash() { - assert_eq!(extract_numeric_id_from_url("12345"), None); - } - - #[test] - fn numeric_url_id_empty() { - assert_eq!(extract_numeric_id_from_url(""), None); - } - - #[test] - fn numeric_url_id_query_digits_not_matched() { - assert_eq!(extract_numeric_id_from_url("http://server.com/player_api.php?username=123"), None); - } - - #[test] - fn numeric_url_id_overflow_returns_none() { - // u32::MAX = 4294967295, this exceeds it - assert_eq!(extract_numeric_id_from_url("http://server.com/live/99999999999.ts"), None); - } - - #[test] - fn numeric_url_id_fragment() { - assert_eq!(extract_numeric_id_from_url("http://server.com/live/55555.ts#start"), Some(55555)); - } - - #[test] - fn numeric_url_id_fragment_no_extension() { - assert_eq!(extract_numeric_id_from_url("http://server.com/live/55555#start"), Some(55555)); - } - - #[test] - fn numeric_url_id_trailing_slash_before_query() { - assert_eq!(extract_numeric_id_from_url("http://server.com/123/?foo=bar"), Some(123)); - } - - #[test] - fn numeric_url_id_trailing_slash_before_fragment() { - assert_eq!(extract_numeric_id_from_url("http://server.com/456/#section"), Some(456)); - } - - // ── hash_bytes / hash_string ───────────────────────────────────── - - #[test] - fn hash_bytes_deterministic() { - let a = hash_bytes(b"hello"); - let b = hash_bytes(b"hello"); - assert_eq!(a, b); - } - - #[test] - fn hash_bytes_different_input_different_output() { - assert_ne!(hash_bytes(b"hello"), hash_bytes(b"world")); - } - - #[test] - fn hash_string_matches_hash_bytes() { - assert_eq!(hash_string("test"), hash_bytes(b"test")); - } - - // ── short_hash ─────────────────────────────────────────────────── - - #[test] - fn short_hash_deterministic() { - assert_eq!(short_hash("abc"), short_hash("abc")); - } - - #[test] - fn short_hash_length_is_16_hex_chars() { - // 8 bytes -> 16 hex characters - assert_eq!(short_hash("anything").len(), 16); - } - - #[test] - fn short_hash_different_input() { - assert_ne!(short_hash("abc"), short_hash("xyz")); - } - - // ── hex_encode / hex_decode ────────────────────────────────────── - - #[test] - fn hex_encode_known_value() { - assert_eq!(hex_encode(&[0xDE, 0xAD, 0xBE, 0xEF]), "DEADBEEF"); - } - - #[test] - fn hex_encode_empty() { - assert_eq!(hex_encode(&[]), ""); - } - - #[test] - fn hex_decode_known_value() { - assert_eq!(hex_decode("DEADBEEF").unwrap(), vec![0xDE, 0xAD, 0xBE, 0xEF]); - } - - #[test] - fn hex_decode_lowercase() { - assert_eq!(hex_decode("deadbeef").unwrap(), vec![0xDE, 0xAD, 0xBE, 0xEF]); - } - - #[test] - fn hex_decode_empty() { - assert_eq!(hex_decode("").unwrap(), Vec::::new()); - } - - #[test] - fn hex_decode_odd_length_error() { - assert!(hex_decode("ABC").is_err()); - } - - #[test] - fn hex_decode_invalid_char_error() { - assert!(hex_decode("ZZZZ").is_err()); - } - - #[test] - fn hex_roundtrip() { - let original = vec![0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF]; - assert_eq!(hex_decode(&hex_encode(&original)).unwrap(), original); - } - - // ── hash_string_as_hex ─────────────────────────────────────────── - - #[test] - fn hash_string_as_hex_deterministic() { - assert_eq!(hash_string_as_hex("url"), hash_string_as_hex("url")); - } - - #[test] - fn hash_string_as_hex_length() { - // UUIDType is 32 bytes (blake3) -> 64 hex chars - assert_eq!(hash_string_as_hex("test").len(), 64); - } - - // ── get_provider_id ────────────────────────────────────────────── - - #[test] - fn get_provider_id_parses_numeric_string() { - assert_eq!(get_provider_id("12345", "http://x.com/other.ts"), Some(12345)); - } - - #[test] - fn get_provider_id_falls_back_to_url() { - assert_eq!(get_provider_id("not_a_number", "http://x.com/live/999.ts"), Some(999)); - } - - #[test] - fn get_provider_id_none_when_both_fail() { - assert_eq!(get_provider_id("abc", "http://x.com/live/hello.ts"), None); - } - - #[test] - fn get_provider_id_empty_string_falls_back() { - assert_eq!(get_provider_id("", "http://x.com/42.ts"), Some(42)); - } - - #[test] - fn get_provider_id_prefers_parsed_string() { - // provider_id string wins even if URL has a different numeric id - assert_eq!(get_provider_id("100", "http://x.com/live/200.ts"), Some(100)); - } - - // ── provider/local playlist uuid ──────────────────────────────── - - #[test] - fn generate_provider_playlist_uuid_deterministic() { - let a = generate_provider_playlist_uuid("k", "1", PlaylistItemType::Live); - let b = generate_provider_playlist_uuid("k", "1", PlaylistItemType::Live); - assert_eq!(a, b); - } - - #[test] - fn generate_provider_playlist_uuid_different_key() { - let a = generate_provider_playlist_uuid("k1", "1", PlaylistItemType::Live); - let b = generate_provider_playlist_uuid("k2", "1", PlaylistItemType::Live); - assert_ne!(a, b); - } - - #[test] - fn generate_local_playlist_uuid_uses_url_path() { - let a = generate_local_playlist_uuid("k", PlaylistItemType::LocalSeries, "http://x.com/path/to/stream"); - let b = generate_local_playlist_uuid("k", PlaylistItemType::LocalSeries, "http://y.com/path/to/stream"); - assert_eq!(a, b); - } - - #[test] - fn generate_local_playlist_uuid_is_deterministic() { - let a = generate_local_playlist_uuid("k", PlaylistItemType::LocalSeries, "http://x.com/stream"); - let b = generate_local_playlist_uuid("k", PlaylistItemType::LocalSeries, "http://x.com/stream"); - assert_eq!(a, b); - } - - #[test] - fn generate_local_playlist_uuid_keeps_local_series_info_and_episode_distinct() { - let url = "file:///library/show/Season%201/Episode%2001.mkv"; - let series_info = generate_local_playlist_uuid("library", PlaylistItemType::LocalSeriesInfo, url); - let episode = generate_local_playlist_uuid("library", PlaylistItemType::LocalSeries, url); - - assert_ne!(series_info, episode); - } - - #[test] - fn generate_provider_playlist_uuid_different_type() { - let a = generate_provider_playlist_uuid("k", "1", PlaylistItemType::Live); - let b = generate_provider_playlist_uuid("k", "1", PlaylistItemType::Video); - assert_ne!(a, b); - } - - #[test] - fn generate_runtime_playlist_uuid_dispatches_by_item_type() { - let provider = generate_runtime_playlist_uuid("input", "100", PlaylistItemType::Series, "file:///ignored.mkv"); - let local = - generate_runtime_playlist_uuid("input", "100", PlaylistItemType::LocalSeries, "file:///ignored.mkv"); - - assert_ne!(provider, local); - } - - // ── u32_to_base64 / base64_to_u32 ─────────────────────────────── - - #[test] - fn base64_roundtrip_zero() { - assert_eq!(base64_to_u32(&u32_to_base64(0)), Some(0)); - } - - #[test] - fn base64_roundtrip_max() { - assert_eq!(base64_to_u32(&u32_to_base64(u32::MAX)), Some(u32::MAX)); - } - - #[test] - fn base64_roundtrip_arbitrary() { - assert_eq!(base64_to_u32(&u32_to_base64(950327)), Some(950327)); - } - - #[test] - fn base64_to_u32_invalid_input() { - assert_eq!(base64_to_u32("!!!"), None); - } - - #[test] - fn base64_to_u32_wrong_length() { - assert_eq!(base64_to_u32("AAAAAAA"), None); - } - - #[test] - fn base64_to_u32_empty() { - assert_eq!(base64_to_u32(""), None); - } - - // ── parse_uuid_hex ─────────────────────────────────────────────── - - #[test] - fn parse_uuid_hex_valid() { - let result = parse_uuid_hex("550e8400-e29b-41d4-a716-446655440000"); - assert_eq!( - result, - Some([0x55, 0x0e, 0x84, 0x00, 0xe2, 0x9b, 0x41, 0xd4, 0xa7, 0x16, 0x44, 0x66, 0x55, 0x44, 0x00, 0x00]) - ); - } - - #[test] - fn parse_uuid_hex_wrong_length() { - assert_eq!(parse_uuid_hex("550e8400-e29b-41d4-a716"), None); - } - - #[test] - fn parse_uuid_hex_no_hyphens_wrong_format() { - // 32 hex chars without hyphens -> length 32 ≠ 36 - assert_eq!(parse_uuid_hex("550e8400e29b41d4a716446655440000"), None); - } - - #[test] - fn parse_uuid_hex_invalid_hex_char() { - assert_eq!(parse_uuid_hex("ZZZZZZZZ-ZZZZ-ZZZZ-ZZZZ-ZZZZZZZZZZZZ"), None); - } - - #[test] - fn parse_uuid_hex_empty() { - assert_eq!(parse_uuid_hex(""), None); - } - - // ── create_alias_uuid ──────────────────────────────────────────── - - #[test] - fn create_alias_uuid_deterministic() { - let base = hash_string("base"); - let a = create_alias_uuid(&base, "mapping1"); - let b = create_alias_uuid(&base, "mapping1"); - assert_eq!(a, b); - } - - #[test] - fn create_alias_uuid_different_mapping() { - let base = hash_string("base"); - let a = create_alias_uuid(&base, "mapping1"); - let b = create_alias_uuid(&base, "mapping2"); - assert_ne!(a, b); - } - - #[test] - fn create_alias_uuid_different_base() { - let a = create_alias_uuid(&hash_string("base1"), "m"); - let b = create_alias_uuid(&hash_string("base2"), "m"); - assert_ne!(a, b); - } - - #[test] - fn create_alias_uuid_differs_from_base() { - let base = hash_string("base"); - let alias = create_alias_uuid(&base, "mapping"); - assert_ne!(base, alias); - } - - /// Swapping `to_string()` for `as_str()` in the UUID hashers must not change a - /// single byte — every persisted playlist UUID depends on it. Guards against a - /// future `Display` impl that stops delegating to `as_str()`. - #[test] - fn item_type_label_is_hash_stable_against_display() { - for item_type in [ - PlaylistItemType::Live, - PlaylistItemType::LiveHls, - PlaylistItemType::Video, - PlaylistItemType::LocalVideo, - PlaylistItemType::LocalSeries, - PlaylistItemType::LocalSeriesInfo, - PlaylistItemType::Series, - PlaylistItemType::SeriesInfo, - PlaylistItemType::Catchup, - ] { - assert_eq!(item_type.as_str(), item_type.to_string(), "{item_type:?}"); - - let mut expected = blake3::Hasher::new(); - expected.update(b"key"); - expected.update(b"provider"); - expected.update(item_type.to_string().as_bytes()); - assert_eq!( - generate_provider_playlist_uuid("key", "provider", item_type), - UUIDType(expected.finalize().into()), - "{item_type:?}" - ); - } - } - - #[test] - fn fnv1a_32_is_deterministic_and_nonzero() { - let a = fnv1a_32("input_a:Show Name:https://example.test/series/ep1"); - let b = fnv1a_32("input_a:Show Name:https://example.test/series/ep1"); - assert_eq!(a, b); - assert!(fnv1a_32("") > 0); - assert!(fnv1a_32("0") > 0); - assert_ne!(a, fnv1a_32("input_a:Show Name:https://example.test/series/ep2")); - } - - #[test] - fn fnv1a_32_matches_known_vector() { - // FNV-1a 32-bit of "" is 0x811c9dc5 (offset basis). Our implementation - // forces the result to `>= 1` so the sentinel-zero guard kicks in. - assert_eq!(fnv1a_32(""), 0x811c9dc5); - } - - #[test] - fn fnv1a_32_parts_matches_joined_string() { - let joined = fnv1a_32("a:b:c"); - let parts = fnv1a_32_parts(&["a", "b", "c"]); - assert_eq!(joined, parts); - // Empty parts list still produces the offset-basis sentinel. - assert_eq!(fnv1a_32_parts(&[]), fnv1a_32("")); - // Single part produces no separator byte. - assert_eq!(fnv1a_32_parts(&["only"]), fnv1a_32("only")); - } - - #[test] - fn stable_episode_storage_id_keys_separate_contexts() { - // Same (series, season, episode_num, episode_id) tuple must hash - // identically regardless of how the caller order is. - let a = stable_episode_storage_id(1, 1, "10", 1); - let b = stable_episode_storage_id(1, 1, "10", 1); - assert_eq!(a, b); - // Different episode_num must not collide. - assert_ne!(a, stable_episode_storage_id(1, 1, "10", 2)); - // Different series must not collide. - assert_ne!(a, stable_episode_storage_id(2, 1, "10", 1)); - } - - #[test] - fn parse_season_episode_extracts_with_default_pattern() { - let pattern = Regex::new(EPISODE_PATTERN).unwrap(); - assert_eq!(parse_season_episode("Show S02E05 [1080p]", &pattern), Some((2, 5))); - assert_eq!(parse_season_episode("Show s8e2", &pattern), Some((8, 2))); - assert_eq!(parse_season_episode("Show without episode", &pattern), None); - // Pattern only matches the digits grouped after S and E. - assert_eq!(parse_season_episode("Pilot", &pattern), None); - } - - #[test] - fn parse_season_episode_legacy_user_pattern_with_wrapped_episode_capture() { - // A user-configured pattern that still wraps the whole - // `SxxEyy` token inside a single named capture (the - // pre-refactor format) must keep working — the function - // falls back to splitting the captured string. - let legacy = Regex::new(r".*(?P[Ss]\d{1,2}(.*?)[Ee]\d{1,2}).*").unwrap(); - assert_eq!(parse_season_episode("Show S02E05 [1080p]", &legacy), Some((2, 5))); - assert_eq!(parse_season_episode("Show s8e2", &legacy), Some((8, 2))); - } - - #[test] - fn parse_season_episode_handles_verbose_and_fallback_shapes() { - // The CONSTANTS regex covers the four historical shapes. - let pattern = Regex::new(CONSTANTS.re_episode_code.as_str()).unwrap(); - // Verbose `Season X Episode Y` - assert_eq!(parse_season_episode("Show - Season 2 Episode 5", &pattern), Some((2, 5)),); - assert_eq!(parse_season_episode("Show Season 02 Episode 05", &pattern), Some((2, 5)),); - // `NxNN` - assert_eq!(parse_season_episode("Show 1x05", &pattern), Some((1, 5))); - // Bare `Episode Y` falls back to season 1. - assert_eq!(parse_season_episode("Show Episode 7", &pattern), Some((1, 7))); - // No match → None. - assert_eq!(parse_season_episode("Pilot", &pattern), None); - } -} +mod tests; diff --git a/shared/src/utils/hash_utils/tests.rs b/shared/src/utils/hash_utils/tests.rs new file mode 100644 index 000000000..450b401e1 --- /dev/null +++ b/shared/src/utils/hash_utils/tests.rs @@ -0,0 +1,6 @@ +use super::*; +use crate::utils::{constants::EPISODE_PATTERN, CONSTANTS}; +use regex::Regex; + +mod episodes; +mod hashing_ids; diff --git a/shared/src/utils/hash_utils/tests/episodes.rs b/shared/src/utils/hash_utils/tests/episodes.rs new file mode 100644 index 000000000..05757f773 --- /dev/null +++ b/shared/src/utils/hash_utils/tests/episodes.rs @@ -0,0 +1,59 @@ +use super::*; + +#[test] +fn generate_local_playlist_uuid_keeps_local_series_info_and_episode_distinct() { + let url = "file:///library/show/Season%201/Episode%2001.mkv"; + let series_info = generate_local_playlist_uuid("library", PlaylistItemType::LocalSeriesInfo, url); + let episode = generate_local_playlist_uuid("library", PlaylistItemType::LocalSeries, url); + + assert_ne!(series_info, episode); +} + +#[test] +fn stable_episode_storage_id_keys_separate_contexts() { + // Same (series, season, episode_num, episode_id) tuple must hash + // identically regardless of how the caller order is. + let a = stable_episode_storage_id(1, 1, "10", 1); + let b = stable_episode_storage_id(1, 1, "10", 1); + assert_eq!(a, b); + // Different episode_num must not collide. + assert_ne!(a, stable_episode_storage_id(1, 1, "10", 2)); + // Different series must not collide. + assert_ne!(a, stable_episode_storage_id(2, 1, "10", 1)); +} + +#[test] +fn parse_season_episode_extracts_with_default_pattern() { + let pattern = Regex::new(EPISODE_PATTERN).unwrap(); + assert_eq!(parse_season_episode("Show S02E05 [1080p]", &pattern), Some((2, 5))); + assert_eq!(parse_season_episode("Show s8e2", &pattern), Some((8, 2))); + assert_eq!(parse_season_episode("Show without episode", &pattern), None); + // Pattern only matches the digits grouped after S and E. + assert_eq!(parse_season_episode("Pilot", &pattern), None); +} + +#[test] +fn parse_season_episode_legacy_user_pattern_with_wrapped_episode_capture() { + // A user-configured pattern that still wraps the whole + // `SxxEyy` token inside a single named capture (the + // pre-refactor format) must keep working — the function + // falls back to splitting the captured string. + let legacy = Regex::new(r".*(?P[Ss]\d{1,2}(.*?)[Ee]\d{1,2}).*").unwrap(); + assert_eq!(parse_season_episode("Show S02E05 [1080p]", &legacy), Some((2, 5))); + assert_eq!(parse_season_episode("Show s8e2", &legacy), Some((8, 2))); +} + +#[test] +fn parse_season_episode_handles_verbose_and_fallback_shapes() { + // The CONSTANTS regex covers the four historical shapes. + let pattern = Regex::new(CONSTANTS.re_episode_code.as_str()).unwrap(); + // Verbose `Season X Episode Y` + assert_eq!(parse_season_episode("Show - Season 2 Episode 5", &pattern), Some((2, 5)),); + assert_eq!(parse_season_episode("Show Season 02 Episode 05", &pattern), Some((2, 5)),); + // `NxNN` + assert_eq!(parse_season_episode("Show 1x05", &pattern), Some((1, 5))); + // Bare `Episode Y` falls back to season 1. + assert_eq!(parse_season_episode("Show Episode 7", &pattern), Some((1, 7))); + // No match → None. + assert_eq!(parse_season_episode("Pilot", &pattern), None); +} diff --git a/shared/src/utils/hash_utils/tests/hashing_ids.rs b/shared/src/utils/hash_utils/tests/hashing_ids.rs new file mode 100644 index 000000000..a02479b4e --- /dev/null +++ b/shared/src/utils/hash_utils/tests/hashing_ids.rs @@ -0,0 +1,502 @@ +use super::*; + +// ── extract_id_from_url ──────────────────────────────────────────── + +#[test] +fn numeric_id_with_ts_extension() { + assert_eq!(extract_id_from_url("http://server.com/live/user/pass/950327.ts"), "950327"); +} + +#[test] +fn numeric_id_with_mkv_extension() { + assert_eq!(extract_id_from_url("http://server.com/movie/user/pass/12345.mkv"), "12345"); +} + +#[test] +fn numeric_id_without_extension() { + assert_eq!(extract_id_from_url("http://server.com/live/user/pass/950327"), "950327"); +} + +#[test] +fn numeric_id_with_query_string() { + assert_eq!( + extract_id_from_url("http://server.com/live/user/pass/1905905.ts?expires=1712345678&token=abc"), + "1905905" + ); +} + +#[test] +fn numeric_id_with_trailing_slash() { + assert_eq!(extract_id_from_url("http://server.com/live/user/pass/42/"), "42"); +} + +#[test] +fn non_numeric_last_segment_returns_hash() { + let result = extract_id_from_url("http://server.com/live/user/pass/hello.ts"); + assert!(!result.chars().all(|c| c.is_ascii_digit()), "should be a hash, not a numeric id"); + assert!(!result.is_empty()); +} + +#[test] +fn mixed_alphanumeric_segment_returns_hash() { + let result = extract_id_from_url("http://server.com/live/user/pass/abc123.ts"); + assert!(!result.chars().all(|c| c.is_ascii_digit()), "mixed segment should not be treated as numeric id"); +} + +#[test] +fn no_slash_returns_hash() { + let result = extract_id_from_url("12345"); + assert!(!result.chars().all(|c| c.is_ascii_digit()), "bare number without slash must not be parsed as id"); +} + +#[test] +fn empty_url_returns_hash() { + let result = extract_id_from_url(""); + assert!(!result.is_empty()); +} + +#[test] +fn only_slashes_returns_hash() { + let result = extract_id_from_url("///"); + assert!(!result.is_empty()); +} + +#[test] +fn numeric_id_after_scheme() { + assert_eq!(extract_id_from_url("http://server.com/42"), "42"); +} + +#[test] +fn same_url_same_hash() { + let a = extract_id_from_url("http://server.com/stream/hello.m3u8"); + let b = extract_id_from_url("http://server.com/stream/hello.m3u8"); + assert_eq!(a, b, "same non-numeric url should produce identical hash"); +} + +#[test] +fn different_url_different_hash() { + let a = extract_id_from_url("http://a.com/hello.ts"); + let b = extract_id_from_url("http://b.com/world.ts"); + assert_ne!(a, b); +} + +#[test] +fn query_string_ignored_for_numeric_extraction() { + // The query string should not interfere with numeric extraction + assert_eq!(extract_id_from_url("http://server.com/99999.ts?token=xyz"), "99999"); +} + +#[test] +fn fragment_after_numeric_id() { + // Fragments don't contain '?' so they stay in the path — still numeric after '/' + assert_eq!(extract_id_from_url("http://server.com/live/55555.ts#start"), "55555"); +} + +#[test] +fn non_numeric_query() { + assert_eq!( + extract_id_from_url("http://server.com/player_api.php?username=123443&password=1000"), + "7D2F521803B23A43" + ); +} + +// ── extract_numeric_id_from_url ──────────────────────────────────── + +#[test] +fn numeric_url_id_standard() { + assert_eq!(extract_numeric_id_from_url("http://server.com/live/user/pass/950327.ts"), Some(950327)); +} + +#[test] +fn numeric_url_id_no_extension() { + assert_eq!(extract_numeric_id_from_url("http://server.com/live/user/pass/42"), Some(42)); +} + +#[test] +fn numeric_url_id_with_query() { + assert_eq!( + extract_numeric_id_from_url("http://server.com/live/user/pass/1905905.ts?token=abc&expires=999"), + Some(1905905) + ); +} + +#[test] +fn numeric_url_id_trailing_slash() { + assert_eq!(extract_numeric_id_from_url("http://server.com/path/123/"), Some(123)); +} + +#[test] +fn numeric_url_id_zero() { + assert_eq!(extract_numeric_id_from_url("http://server.com/live/0.ts"), Some(0)); +} + +#[test] +fn numeric_url_id_non_numeric_segment() { + assert_eq!(extract_numeric_id_from_url("http://server.com/live/hello.ts"), None); +} + +#[test] +fn numeric_url_id_mixed_alphanumeric() { + assert_eq!(extract_numeric_id_from_url("http://server.com/live/abc123.ts"), None); +} + +#[test] +fn numeric_url_id_no_slash() { + assert_eq!(extract_numeric_id_from_url("12345"), None); +} + +#[test] +fn numeric_url_id_empty() { + assert_eq!(extract_numeric_id_from_url(""), None); +} + +#[test] +fn numeric_url_id_query_digits_not_matched() { + assert_eq!(extract_numeric_id_from_url("http://server.com/player_api.php?username=123"), None); +} + +#[test] +fn numeric_url_id_overflow_returns_none() { + // u32::MAX = 4294967295, this exceeds it + assert_eq!(extract_numeric_id_from_url("http://server.com/live/99999999999.ts"), None); +} + +#[test] +fn numeric_url_id_fragment() { + assert_eq!(extract_numeric_id_from_url("http://server.com/live/55555.ts#start"), Some(55555)); +} + +#[test] +fn numeric_url_id_fragment_no_extension() { + assert_eq!(extract_numeric_id_from_url("http://server.com/live/55555#start"), Some(55555)); +} + +#[test] +fn numeric_url_id_trailing_slash_before_query() { + assert_eq!(extract_numeric_id_from_url("http://server.com/123/?foo=bar"), Some(123)); +} + +#[test] +fn numeric_url_id_trailing_slash_before_fragment() { + assert_eq!(extract_numeric_id_from_url("http://server.com/456/#section"), Some(456)); +} + +// ── hash_bytes / hash_string ───────────────────────────────────── + +#[test] +fn hash_bytes_deterministic() { + let a = hash_bytes(b"hello"); + let b = hash_bytes(b"hello"); + assert_eq!(a, b); +} + +#[test] +fn hash_bytes_different_input_different_output() { + assert_ne!(hash_bytes(b"hello"), hash_bytes(b"world")); +} + +#[test] +fn hash_string_matches_hash_bytes() { + assert_eq!(hash_string("test"), hash_bytes(b"test")); +} + +// ── short_hash ─────────────────────────────────────────────────── + +#[test] +fn short_hash_deterministic() { + assert_eq!(short_hash("abc"), short_hash("abc")); +} + +#[test] +fn short_hash_length_is_16_hex_chars() { + // 8 bytes -> 16 hex characters + assert_eq!(short_hash("anything").len(), 16); +} + +#[test] +fn short_hash_different_input() { + assert_ne!(short_hash("abc"), short_hash("xyz")); +} + +// ── hex_encode / hex_decode ────────────────────────────────────── + +#[test] +fn hex_encode_known_value() { + assert_eq!(hex_encode(&[0xDE, 0xAD, 0xBE, 0xEF]), "DEADBEEF"); +} + +#[test] +fn hex_encode_empty() { + assert_eq!(hex_encode(&[]), ""); +} + +#[test] +fn hex_decode_known_value() { + assert_eq!(hex_decode("DEADBEEF").unwrap(), vec![0xDE, 0xAD, 0xBE, 0xEF]); +} + +#[test] +fn hex_decode_lowercase() { + assert_eq!(hex_decode("deadbeef").unwrap(), vec![0xDE, 0xAD, 0xBE, 0xEF]); +} + +#[test] +fn hex_decode_empty() { + assert_eq!(hex_decode("").unwrap(), Vec::::new()); +} + +#[test] +fn hex_decode_odd_length_error() { + assert!(hex_decode("ABC").is_err()); +} + +#[test] +fn hex_decode_invalid_char_error() { + assert!(hex_decode("ZZZZ").is_err()); +} + +#[test] +fn hex_roundtrip() { + let original = vec![0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF]; + assert_eq!(hex_decode(&hex_encode(&original)).unwrap(), original); +} + +// ── hash_string_as_hex ─────────────────────────────────────────── + +#[test] +fn hash_string_as_hex_deterministic() { + assert_eq!(hash_string_as_hex("url"), hash_string_as_hex("url")); +} + +#[test] +fn hash_string_as_hex_length() { + // UUIDType is 32 bytes (blake3) -> 64 hex chars + assert_eq!(hash_string_as_hex("test").len(), 64); +} + +// ── get_provider_id ────────────────────────────────────────────── + +#[test] +fn get_provider_id_parses_numeric_string() { + assert_eq!(get_provider_id("12345", "http://x.com/other.ts"), Some(12345)); +} + +#[test] +fn get_provider_id_falls_back_to_url() { + assert_eq!(get_provider_id("not_a_number", "http://x.com/live/999.ts"), Some(999)); +} + +#[test] +fn get_provider_id_none_when_both_fail() { + assert_eq!(get_provider_id("abc", "http://x.com/live/hello.ts"), None); +} + +#[test] +fn get_provider_id_empty_string_falls_back() { + assert_eq!(get_provider_id("", "http://x.com/42.ts"), Some(42)); +} + +#[test] +fn get_provider_id_prefers_parsed_string() { + // provider_id string wins even if URL has a different numeric id + assert_eq!(get_provider_id("100", "http://x.com/live/200.ts"), Some(100)); +} + +// ── provider/local playlist uuid ──────────────────────────────── + +#[test] +fn generate_provider_playlist_uuid_deterministic() { + let a = generate_provider_playlist_uuid("k", "1", PlaylistItemType::Live); + let b = generate_provider_playlist_uuid("k", "1", PlaylistItemType::Live); + assert_eq!(a, b); +} + +#[test] +fn generate_provider_playlist_uuid_different_key() { + let a = generate_provider_playlist_uuid("k1", "1", PlaylistItemType::Live); + let b = generate_provider_playlist_uuid("k2", "1", PlaylistItemType::Live); + assert_ne!(a, b); +} + +#[test] +fn generate_local_playlist_uuid_uses_url_path() { + let a = generate_local_playlist_uuid("k", PlaylistItemType::LocalSeries, "http://x.com/path/to/stream"); + let b = generate_local_playlist_uuid("k", PlaylistItemType::LocalSeries, "http://y.com/path/to/stream"); + assert_eq!(a, b); +} + +#[test] +fn generate_local_playlist_uuid_is_deterministic() { + let a = generate_local_playlist_uuid("k", PlaylistItemType::LocalSeries, "http://x.com/stream"); + let b = generate_local_playlist_uuid("k", PlaylistItemType::LocalSeries, "http://x.com/stream"); + assert_eq!(a, b); +} + +#[test] +fn generate_provider_playlist_uuid_different_type() { + let a = generate_provider_playlist_uuid("k", "1", PlaylistItemType::Live); + let b = generate_provider_playlist_uuid("k", "1", PlaylistItemType::Video); + assert_ne!(a, b); +} + +#[test] +fn generate_runtime_playlist_uuid_dispatches_by_item_type() { + let provider = generate_runtime_playlist_uuid("input", "100", PlaylistItemType::Series, "file:///ignored.mkv"); + let local = generate_runtime_playlist_uuid("input", "100", PlaylistItemType::LocalSeries, "file:///ignored.mkv"); + + assert_ne!(provider, local); +} + +// ── u32_to_base64 / base64_to_u32 ─────────────────────────────── + +#[test] +fn base64_roundtrip_zero() { + assert_eq!(base64_to_u32(&u32_to_base64(0)), Some(0)); +} + +#[test] +fn base64_roundtrip_max() { + assert_eq!(base64_to_u32(&u32_to_base64(u32::MAX)), Some(u32::MAX)); +} + +#[test] +fn base64_roundtrip_arbitrary() { + assert_eq!(base64_to_u32(&u32_to_base64(950327)), Some(950327)); +} + +#[test] +fn base64_to_u32_invalid_input() { + assert_eq!(base64_to_u32("!!!"), None); +} + +#[test] +fn base64_to_u32_wrong_length() { + assert_eq!(base64_to_u32("AAAAAAA"), None); +} + +#[test] +fn base64_to_u32_empty() { + assert_eq!(base64_to_u32(""), None); +} + +// ── parse_uuid_hex ─────────────────────────────────────────────── + +#[test] +fn parse_uuid_hex_valid() { + let result = parse_uuid_hex("550e8400-e29b-41d4-a716-446655440000"); + assert_eq!( + result, + Some([0x55, 0x0e, 0x84, 0x00, 0xe2, 0x9b, 0x41, 0xd4, 0xa7, 0x16, 0x44, 0x66, 0x55, 0x44, 0x00, 0x00]) + ); +} + +#[test] +fn parse_uuid_hex_wrong_length() { + assert_eq!(parse_uuid_hex("550e8400-e29b-41d4-a716"), None); +} + +#[test] +fn parse_uuid_hex_no_hyphens_wrong_format() { + // 32 hex chars without hyphens -> length 32 ≠ 36 + assert_eq!(parse_uuid_hex("550e8400e29b41d4a716446655440000"), None); +} + +#[test] +fn parse_uuid_hex_invalid_hex_char() { + assert_eq!(parse_uuid_hex("ZZZZZZZZ-ZZZZ-ZZZZ-ZZZZ-ZZZZZZZZZZZZ"), None); +} + +#[test] +fn parse_uuid_hex_empty() { + assert_eq!(parse_uuid_hex(""), None); +} + +// ── create_alias_uuid ──────────────────────────────────────────── + +#[test] +fn create_alias_uuid_deterministic() { + let base = hash_string("base"); + let a = create_alias_uuid(&base, "mapping1"); + let b = create_alias_uuid(&base, "mapping1"); + assert_eq!(a, b); +} + +#[test] +fn create_alias_uuid_different_mapping() { + let base = hash_string("base"); + let a = create_alias_uuid(&base, "mapping1"); + let b = create_alias_uuid(&base, "mapping2"); + assert_ne!(a, b); +} + +#[test] +fn create_alias_uuid_different_base() { + let a = create_alias_uuid(&hash_string("base1"), "m"); + let b = create_alias_uuid(&hash_string("base2"), "m"); + assert_ne!(a, b); +} + +#[test] +fn create_alias_uuid_differs_from_base() { + let base = hash_string("base"); + let alias = create_alias_uuid(&base, "mapping"); + assert_ne!(base, alias); +} + +/// Swapping `to_string()` for `as_str()` in the UUID hashers must not change a +/// single byte — every persisted playlist UUID depends on it. Guards against a +/// future `Display` impl that stops delegating to `as_str()`. +#[test] +fn item_type_label_is_hash_stable_against_display() { + for item_type in [ + PlaylistItemType::Live, + PlaylistItemType::LiveHls, + PlaylistItemType::Video, + PlaylistItemType::LocalVideo, + PlaylistItemType::LocalSeries, + PlaylistItemType::LocalSeriesInfo, + PlaylistItemType::Series, + PlaylistItemType::SeriesInfo, + PlaylistItemType::Catchup, + ] { + assert_eq!(item_type.as_str(), item_type.to_string(), "{item_type:?}"); + + let mut expected = blake3::Hasher::new(); + expected.update(b"key"); + expected.update(b"provider"); + expected.update(item_type.to_string().as_bytes()); + assert_eq!( + generate_provider_playlist_uuid("key", "provider", item_type), + UUIDType(expected.finalize().into()), + "{item_type:?}" + ); + } +} + +#[test] +fn fnv1a_32_is_deterministic_and_nonzero() { + let a = fnv1a_32("input_a:Show Name:https://example.test/series/ep1"); + let b = fnv1a_32("input_a:Show Name:https://example.test/series/ep1"); + assert_eq!(a, b); + assert!(fnv1a_32("") > 0); + assert!(fnv1a_32("0") > 0); + assert_ne!(a, fnv1a_32("input_a:Show Name:https://example.test/series/ep2")); +} + +#[test] +fn fnv1a_32_matches_known_vector() { + // FNV-1a 32-bit of "" is 0x811c9dc5 (offset basis). Our implementation + // forces the result to `>= 1` so the sentinel-zero guard kicks in. + assert_eq!(fnv1a_32(""), 0x811c9dc5); +} + +#[test] +fn fnv1a_32_parts_matches_joined_string() { + let joined = fnv1a_32("a:b:c"); + let parts = fnv1a_32_parts(&["a", "b", "c"]); + assert_eq!(joined, parts); + // Empty parts list still produces the offset-basis sentinel. + assert_eq!(fnv1a_32_parts(&[]), fnv1a_32("")); + // Single part produces no separator byte. + assert_eq!(fnv1a_32_parts(&["only"]), fnv1a_32("only")); +} diff --git a/shared/src/utils/string_interner.rs b/shared/src/utils/string_interner.rs index 968d68eb2..7a6b75d6b 100644 --- a/shared/src/utils/string_interner.rs +++ b/shared/src/utils/string_interner.rs @@ -3,80 +3,8 @@ //! Provides a global string interner to deduplicate frequently repeated //! strings like `input_name` and `group` in playlist items. -use crate::model::UUIDType; -use serde::{ - de::{IgnoredAny, MapAccess, SeqAccess, Visitor}, - Deserialize, Deserializer, Serializer, -}; -use std::{ - borrow::Cow, - collections::HashSet, - fmt, - sync::{Arc, LazyLock, RwLock}, -}; - -/// Global interning pool. -/// -/// ## Performance (millions of entries) -/// -/// * **Happy-path (already-interned):** one `RwLock::read()` + hash lookup + -/// `Arc::clone` (single atomic increment). Multiple threads can read -/// concurrently without blocking each other. -/// * **First-time intern:** upgrades to a write lock, double-checks, and -/// inserts. This only happens *once per unique string value*, so the write -/// path is not on the hot parse loop. -/// * **`deserialize_string` vs `deserialize_any`:** the former is *faster* -/// because saphyr skips bool / int / float parsing attempts and hands the -/// raw scalar text directly to the visitor. -/// * **Pruning:** call `interner_gc()` periodically (e.g. after a full -/// playlist reload) to release strings that are only referenced by the -/// pool itself. -static INTERNER: LazyLock>>> = LazyLock::new(|| RwLock::new(HashSet::new())); - -pub trait Internable { - fn intern(self) -> Arc; -} - -impl Internable for &Arc { - fn intern(self) -> Arc { Arc::clone(self) } -} - -impl Internable for &Cow<'_, str> { - fn intern(self) -> Arc { - match self { - Cow::Borrowed(s) => intern_str(s), - Cow::Owned(s) => intern_string(s.clone()), - } - } -} - -impl Internable for &UUIDType { - fn intern(self) -> Arc { intern_string(self.to_string()) } -} - -impl Internable for String { - fn intern(self) -> Arc { intern_string(self) } -} - -impl Internable for &String { - fn intern(self) -> Arc { intern_str(self.as_str()) } -} - -impl Internable for &str { - fn intern(self) -> Arc { intern_str(self) } -} - -impl Internable for u32 { - fn intern(self) -> Arc { intern_string(self.to_string()) } -} - -impl Internable for u64 { - fn intern(self) -> Arc { intern_string(self.to_string()) } -} - -impl Internable for i64 { - fn intern(self) -> Arc { intern_string(self.to_string()) } -} +use serde::{Deserialize, Deserializer, Serializer}; +use std::sync::Arc; macro_rules! intern_impl { ($s:expr, $create:expr) => {{ @@ -97,396 +25,6 @@ macro_rules! intern_impl { }}; } -/// Interns a string slice. -fn intern_str(s: &str) -> Arc { intern_impl!(s, Arc::from(s)) } - -/// Interns an owned string. -fn intern_string(s: String) -> Arc { intern_impl!(s.as_str(), Arc::from(s)) } - -/// Returns the current number of strings held in the interning pool. -/// Uses a read lock and is safe to call on hot paths for threshold checks. -pub fn interner_len() -> usize { INTERNER.read().map_or(0, |g| g.len()) } - -/// Garbage collection: removes strings that are only referenced by the cache. -pub fn interner_gc() -> usize { - if let Ok(mut guard) = INTERNER.write() { - let before = guard.len(); - guard.retain(|s| Arc::strong_count(s) > 1); - let removed = before - guard.len(); - if removed > 0 { - log::debug!("Pruned {removed} unused interned strings ({} remaining)", guard.len()); - } - return removed; - } - 0 -} - -/// Convert an `f64` that reached `visit_f64` into a round-trip-safe string. -/// -/// Special values are emitted as `"infinity"`, `"-infinity"`, and `"nan"`. -/// `serde_saphyr` re-serializes these ambiguous scalars quoted, so they -/// survive a YAML round-trip as strings. -/// -/// This is a safety-net for paths that intentionally accept typed numeric -/// scalars and normalize them into strings. -#[inline] -fn f64_to_str(v: f64) -> String { - if v.is_infinite() { - if v.is_sign_positive() { - "infinity".to_owned() - } else { - "-infinity".to_owned() - } - } else if v.is_nan() { - "nan".to_owned() - } else { - v.to_string() - } -} - -#[inline] -// This intentionally only normalizes the lowercase dot-prefixed spellings that -// serde_saphyr emits for special float scalars. Other YAML 1.1 variants such as -// `.Inf`, `.INF`, or `.NaN` are out of scope here because they are not produced -// by the current parser path. -fn normalize_scalar_string(value: &str) -> &str { - match value { - ".inf" => "infinity", - "-.inf" => "-infinity", - ".nan" => "nan", - _ => value, - } -} - -/// Returns `true` for values that should be treated as absent. -/// -/// Covers: -/// - empty string (`""`) -/// - JSON/YAML null literals (`"null"`, `"~"`) -/// -/// Generic, field-agnostic. Callers decide whether `true` maps to `""` -/// (for non-optional `Arc`) or `None` (for `Option>`). -/// Case-sensitive on purpose: provider-supplied `"NULL"` or `"Null"` are real -/// values and must be preserved verbatim. -#[inline] -pub fn is_nullish(value: &str) -> bool { value.is_empty() || value == "~" || value.eq_ignore_ascii_case("null") } - -// -// Two reusable visitor types live here so that multiple public entry-points -// can share them without code duplication: -// -// ArcStrVisitor -> Arc (null/empty -> "") -// OptionArcStrVisitor -> Option> (null/empty -> None) -// -// `ArcStrVisitor::visit_some` uses `deserialize_any(self)`, so numeric/bool -// scalars can flow into the typed `visit_*` methods. The tradeoff is that raw -// numeric notation may be normalized (for example `1e2` becomes `"100"`). -// -// `OptionArcStrVisitor::visit_some` intentionally diverges and uses -// `deserialize_any` so JSON/YAML numeric inputs can flow into `visit_i64`, -// `visit_u64` or `visit_f64`. The tradeoff is that this path no longer forces -// raw-scalar preservation in the same way as `ArcStrVisitor`. - -/// Visitor that produces `Arc`, mapping null / empty -> `""`. -struct ArcStrVisitor; - -impl<'de> Visitor<'de> for ArcStrVisitor { - type Value = Arc; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { f.write_str("a string, number, boolean, or null") } - - fn visit_str(self, v: &str) -> Result { - Ok(normalize_scalar_string(v).intern()) - } - fn visit_string(self, v: String) -> Result { - Ok(normalize_scalar_string(v.as_str()).intern()) - } - fn visit_bool(self, v: bool) -> Result { Ok(v.to_string().intern()) } - fn visit_i64(self, v: i64) -> Result { Ok(v.to_string().intern()) } - fn visit_u64(self, v: u64) -> Result { Ok(v.to_string().intern()) } - fn visit_f64(self, v: f64) -> Result { Ok(f64_to_str(v).intern()) } - fn visit_unit(self) -> Result { Ok("".intern()) } - fn visit_none(self) -> Result { Ok("".intern()) } - fn visit_some>(self, d: D) -> Result { d.deserialize_any(self) } -} - -/// Visitor that produces `Option>`, mapping null / empty -> `None`. -struct OptionArcStrVisitor; - -impl<'de> Visitor<'de> for OptionArcStrVisitor { - type Value = Option>; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("a string, number, boolean, null, or empty") - } - - fn visit_str(self, v: &str) -> Result { - let normalized = normalize_scalar_string(v); - if normalized.is_empty() { - Ok(None) - } else { - Ok(Some(normalized.intern())) - } - } - fn visit_string(self, v: String) -> Result { - let normalized = normalize_scalar_string(v.as_str()); - if normalized.is_empty() { - Ok(None) - } else { - Ok(Some(normalized.intern())) - } - } - fn visit_bool(self, v: bool) -> Result { Ok(Some(v.to_string().intern())) } - fn visit_i64(self, v: i64) -> Result { Ok(Some(v.to_string().intern())) } - fn visit_u64(self, v: u64) -> Result { Ok(Some(v.to_string().intern())) } - fn visit_f64(self, v: f64) -> Result { Ok(Some(f64_to_str(v).intern())) } - fn visit_unit(self) -> Result { Ok(None) } - fn visit_none(self) -> Result { Ok(None) } - fn visit_some>(self, d: D) -> Result { d.deserialize_any(self) } - fn visit_seq>(self, mut seq: A) -> Result { - while seq.next_element::()?.is_some() {} - log::debug!("ignored sequence while deserializing string interner, returning None"); - Ok(None) - } - fn visit_map>(self, mut map: A) -> Result { - while map.next_entry::()?.is_some() {} - log::debug!("ignored map while deserializing string interner, returning None"); - Ok(None) - } -} - -pub mod arc_str_vec_serde { - // These serde helper modules deliberately re-use the parent module's imports. - // Clippy's expansion of this glob names private sibling modules and does not - // compile, so the glob is kept. - #[allow(clippy::wildcard_imports)] - use super::*; - use serde::ser::SerializeSeq; - - pub fn serialize(value: &Vec>, serializer: S) -> Result - where - S: Serializer, - { - let mut seq = serializer.serialize_seq(Some(value.len()))?; - for s in value { - seq.serialize_element(s.as_ref())?; - } - seq.end() - } - - pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> - where - D: Deserializer<'de>, - { - let vec = Vec::::deserialize(deserializer)?; - Ok(vec.into_iter().map(super::Internable::intern).collect()) - } -} - -pub mod arc_str_serde { - // These serde helper modules deliberately re-use the parent module's imports. - // Clippy's expansion of this glob names private sibling modules and does not - // compile, so the glob is kept. - #[allow(clippy::wildcard_imports)] - use super::*; - - pub fn serialize(value: &Arc, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(value) - } - - /// Deserialize a scalar as an interned `Arc`. - /// - /// This goes through `deserialize_option(ArcStrVisitor)`, and - /// `ArcStrVisitor::visit_some` uses `deserialize_any`. That allows numeric - /// JSON/YAML scalars to be accepted, but raw numeric notation may be lost - /// during normalization (for example `1e2` becomes `"100"`). - pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - { - deserializer.deserialize_option(ArcStrVisitor) - } -} - -pub mod arc_str_option_serde { - // These serde helper modules deliberately re-use the parent module's imports. - // Clippy's expansion of this glob names private sibling modules and does not - // compile, so the glob is kept. - #[allow(clippy::wildcard_imports)] - use super::*; - - pub fn serialize(value: &Option>, serializer: S) -> Result - where - S: Serializer, - { - match value { - Some(s) => serializer.serialize_str(s), - None => serializer.serialize_none(), - } - } - - pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> - where - D: Deserializer<'de>, - { - deserializer.deserialize_option(OptionArcStrVisitor) - } - - pub fn serialize_null_if_empty(value: &Option>, serializer: S) -> Result - where - S: Serializer, - { - match value { - None => serializer.serialize_none(), - Some(s) if s.is_empty() => serializer.serialize_none(), - Some(s) => serializer.serialize_str(s), - } - } -} - -pub mod arc_str_option_null_if_empty_serde { - pub use super::arc_str_option_serde::{deserialize, serialize_null_if_empty as serialize}; -} - -/// Visitor that mirrors `ArcStrVisitor` but collapses `"null"`, `""`, and `~` -/// to an empty `Arc`. Generic over the field, not specific to -/// `container_extension`. -struct NullishArcStrVisitor; - -impl<'de> Visitor<'de> for NullishArcStrVisitor { - type Value = Arc; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("a string, number, boolean, or null (literal \"null\"/\"~\"/empty mapped to empty)") - } - - fn visit_str(self, v: &str) -> Result { - if is_nullish(v) { - Ok("".intern()) - } else { - Ok(normalize_scalar_string(v).intern()) - } - } - fn visit_string(self, v: String) -> Result { - if is_nullish(&v) { - Ok("".intern()) - } else { - Ok(normalize_scalar_string(&v).intern()) - } - } - fn visit_bool(self, v: bool) -> Result { Ok(v.to_string().intern()) } - fn visit_i64(self, v: i64) -> Result { Ok(v.to_string().intern()) } - fn visit_u64(self, v: u64) -> Result { Ok(v.to_string().intern()) } - fn visit_f64(self, v: f64) -> Result { Ok(f64_to_str(v).intern()) } - fn visit_unit(self) -> Result { Ok("".intern()) } - fn visit_none(self) -> Result { Ok("".intern()) } - fn visit_some>(self, d: D) -> Result { d.deserialize_any(self) } -} - -/// Visitor that mirrors `OptionArcStrVisitor` but collapses `"null"`, `""`, -/// and `~` to `None`. Generic over the field. -struct NullishOptionArcStrVisitor; - -impl<'de> Visitor<'de> for NullishOptionArcStrVisitor { - type Value = Option>; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("a string, number, boolean, null, or empty (literal \"null\"/\"~\"/empty mapped to None)") - } - - fn visit_str(self, v: &str) -> Result { - if is_nullish(v) { - Ok(None) - } else { - Ok(Some(normalize_scalar_string(v).intern())) - } - } - fn visit_string(self, v: String) -> Result { - if is_nullish(&v) { - Ok(None) - } else { - Ok(Some(normalize_scalar_string(&v).intern())) - } - } - fn visit_bool(self, v: bool) -> Result { Ok(Some(v.to_string().intern())) } - fn visit_i64(self, v: i64) -> Result { Ok(Some(v.to_string().intern())) } - fn visit_u64(self, v: u64) -> Result { Ok(Some(v.to_string().intern())) } - fn visit_f64(self, v: f64) -> Result { Ok(Some(f64_to_str(v).intern())) } - fn visit_unit(self) -> Result { Ok(None) } - fn visit_none(self) -> Result { Ok(None) } - fn visit_some>(self, d: D) -> Result { d.deserialize_any(self) } - fn visit_seq>(self, mut seq: A) -> Result { - while seq.next_element::()?.is_some() {} - log::debug!("ignored sequence while deserializing nullish arc_str, returning None"); - Ok(None) - } - fn visit_map>(self, mut map: A) -> Result { - while map.next_entry::()?.is_some() {} - log::debug!("ignored map while deserializing nullish arc_str, returning None"); - Ok(None) - } -} - -/// Generic serde for `Arc` that treats JSON null, YAML null/`~`, empty -/// string, and the literal four-character string `"null"` as empty. -/// -/// Field-agnostic. Apply with `#[serde(with = "arc_str_null_is_none_serde")]`. -/// -/// Use instead of `arc_str_serde` whenever the underlying value might be -/// reported as `"null"` by a misbehaving provider. -pub mod arc_str_null_is_none_serde { - // These serde helper modules deliberately re-use the parent module's imports. - // Clippy's expansion of this glob names private sibling modules and does not - // compile, so the glob is kept. - #[allow(clippy::wildcard_imports)] - use super::*; - - pub fn serialize(value: &Arc, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(value) - } - - pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - { - deserializer.deserialize_option(NullishArcStrVisitor) - } -} - -/// Generic serde for `Option>` that treats JSON null, YAML null/`~`, -/// empty string, and the literal four-character string `"null"` as `None`. On -/// serialization, `None`, empty, and the literal `"null"` all become JSON null. -pub mod arc_str_null_is_none_option_serde { - // These serde helper modules deliberately re-use the parent module's imports. - // Clippy's expansion of this glob names private sibling modules and does not - // compile, so the glob is kept. - #[allow(clippy::wildcard_imports)] - use super::*; - - pub fn serialize(value: &Option>, serializer: S) -> Result - where - S: Serializer, - { - match value { - Some(s) if !super::is_nullish(s) => serializer.serialize_str(s), - _ => serializer.serialize_none(), - } - } - - pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> - where - D: Deserializer<'de>, - { - deserializer.deserialize_option(NullishOptionArcStrVisitor) - } -} - // // Reuses `ArcStrVisitor` / `OptionArcStrVisitor` via `deserialize_option`: // - null / ~ / empty -> visit_none / visit_unit -> "" / None @@ -497,190 +35,17 @@ pub mod arc_str_null_is_none_option_serde { pub use arc_str_default_on_null as arc_str_none_default_on_null; -pub fn arc_str_default_on_null<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - deserializer.deserialize_option(ArcStrVisitor) -} - -pub fn deserialize_as_option_arc_str<'de, D>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, -{ - deserializer.deserialize_option(OptionArcStrVisitor) -} - #[cfg(test)] -mod tests { - // These serde helper modules deliberately re-use the parent module's imports. - // Clippy's expansion of this glob names private sibling modules and does not - // compile, so the glob is kept. - #[allow(clippy::wildcard_imports)] - use super::*; +mod tests; - #[derive(Debug, serde::Deserialize)] - struct ArcStrHolder { - #[serde(default, with = "arc_str_serde")] - value: Arc, - } - - #[derive(Debug, serde::Deserialize)] - struct OptArcStrHolder { - #[serde(default, with = "arc_str_option_serde")] - value: Option>, - } - - #[test] - fn arc_str_serde_preserves_yaml_infinity_literal_as_string() { - let parsed: ArcStrHolder = serde_saphyr::from_str("value: infinity\n").unwrap(); - assert_eq!(parsed.value.as_ref(), "infinity"); - } - - #[test] - fn arc_str_serde_preserves_yaml_numeric_like_word_as_string() { - let parsed: ArcStrHolder = serde_saphyr::from_str("value: 01abc\n").unwrap(); - assert_eq!(parsed.value.as_ref(), "01abc"); - } - - #[test] - fn arc_str_serde_accepts_json_integer() { - let parsed: ArcStrHolder = serde_json::from_str(r#"{"value":1285728}"#).unwrap(); - assert_eq!(parsed.value.as_ref(), "1285728"); - } - - #[test] - fn arc_str_option_serde_accepts_json_integer() { - let parsed: OptArcStrHolder = serde_json::from_str(r#"{"value":8169}"#).unwrap(); - assert_eq!(parsed.value.as_deref(), Some("8169")); - } - - #[test] - fn arc_str_option_serde_accepts_json_string() { - let parsed: OptArcStrHolder = serde_json::from_str(r#"{"value":"8169"}"#).unwrap(); - assert_eq!(parsed.value.as_deref(), Some("8169")); - } - - #[test] - fn arc_str_option_serde_maps_empty_string_to_none() { - let parsed: OptArcStrHolder = serde_json::from_str(r#"{"value":""}"#).unwrap(); - assert_eq!(parsed.value, None); - } - - #[test] - fn arc_str_serde_normalizes_json_scientific_notation_numbers() { - let parsed: ArcStrHolder = serde_json::from_str(r#"{"value":1e2}"#).unwrap(); - assert_eq!(parsed.value.as_ref(), "100"); - } - - #[test] - fn arc_str_serde_normalizes_yaml_special_float_scalars() { - let parsed_inf: ArcStrHolder = serde_saphyr::from_str("value: .inf\n").unwrap(); - let parsed_neg_inf: ArcStrHolder = serde_saphyr::from_str("value: -.inf\n").unwrap(); - let parsed_nan: ArcStrHolder = serde_saphyr::from_str("value: .nan\n").unwrap(); - - assert_eq!(parsed_inf.value.as_ref(), "infinity"); - assert_eq!(parsed_neg_inf.value.as_ref(), "-infinity"); - assert_eq!(parsed_nan.value.as_ref(), "nan"); - } - - #[test] - fn arc_str_option_serde_normalizes_yaml_special_float_scalars() { - let parsed_inf: OptArcStrHolder = serde_saphyr::from_str("value: .inf\n").unwrap(); - let parsed_neg_inf: OptArcStrHolder = serde_saphyr::from_str("value: -.inf\n").unwrap(); - let parsed_nan: OptArcStrHolder = serde_saphyr::from_str("value: .nan\n").unwrap(); - - assert_eq!(parsed_inf.value.as_deref(), Some("infinity")); - assert_eq!(parsed_neg_inf.value.as_deref(), Some("-infinity")); - assert_eq!(parsed_nan.value.as_deref(), Some("nan")); - } - - #[derive(Debug, serde::Deserialize, serde::Serialize)] - struct NullishArcStrHolder { - #[serde(default, with = "arc_str_null_is_none_serde")] - value: Arc, - } - - #[derive(Debug, serde::Deserialize, serde::Serialize)] - struct NullishOptArcStrHolder { - #[serde(default, with = "arc_str_null_is_none_option_serde")] - value: Option>, - } - - #[test] - fn null_is_none_arc_str_treats_literal_null_string_as_empty() { - let parsed: NullishArcStrHolder = serde_json::from_str(r#"{"value":"null"}"#).unwrap(); - assert!(parsed.value.is_empty()); - } - - #[test] - fn null_is_none_arc_str_treats_yaml_null_scalar_as_empty() { - let parsed: NullishArcStrHolder = serde_saphyr::from_str("value: null\n").unwrap(); - assert!(parsed.value.is_empty()); - let parsed_tilde: NullishArcStrHolder = serde_saphyr::from_str("value: ~\n").unwrap(); - assert!(parsed_tilde.value.is_empty()); - } - - #[test] - fn null_is_none_arc_str_treats_json_null_as_empty() { - let parsed: NullishArcStrHolder = serde_json::from_str(r#"{"value":null}"#).unwrap(); - assert!(parsed.value.is_empty()); - } - - #[test] - fn null_is_none_arc_str_preserves_real_extensions() { - for ext in ["mkv", "mp4", "ts", "avi"] { - let json = format!(r#"{{"value":"{ext}"}}"#); - let parsed: NullishArcStrHolder = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed.value.as_ref(), ext, "extension {ext} must survive"); - } - } - - #[test] - fn null_is_none_option_treats_literal_null_string_as_none() { - let parsed: NullishOptArcStrHolder = serde_json::from_str(r#"{"value":"null"}"#).unwrap(); - assert_eq!(parsed.value, None); - } - - #[test] - fn null_is_none_option_treats_json_null_as_none() { - let parsed: NullishOptArcStrHolder = serde_json::from_str(r#"{"value":null}"#).unwrap(); - assert_eq!(parsed.value, None); - } - - #[test] - fn null_is_none_option_treats_empty_string_as_none() { - let parsed: NullishOptArcStrHolder = serde_json::from_str(r#"{"value":""}"#).unwrap(); - assert_eq!(parsed.value, None); - } - - #[test] - fn null_is_none_option_preserves_real_extensions() { - for ext in ["mkv", "mp4", "ts"] { - let json = format!(r#"{{"value":"{ext}"}}"#); - let parsed: NullishOptArcStrHolder = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed.value.as_deref(), Some(ext), "extension {ext} must survive"); - } - } - - #[test] - fn null_is_none_option_serializes_none_and_literal_null_as_json_null() { - let none_value: NullishOptArcStrHolder = NullishOptArcStrHolder { value: None }; - assert_eq!(serde_json::to_string(&none_value).unwrap(), r#"{"value":null}"#); - - let literal_null: NullishOptArcStrHolder = NullishOptArcStrHolder { value: Some("null".into()) }; - assert_eq!(serde_json::to_string(&literal_null).unwrap(), r#"{"value":null}"#); - - let empty: NullishOptArcStrHolder = NullishOptArcStrHolder { value: Some("".into()) }; - assert_eq!(serde_json::to_string(&empty).unwrap(), r#"{"value":null}"#); - } - - #[test] - fn null_is_none_arc_str_roundtrips_real_extensions_byte_for_byte() { - let holder = NullishArcStrHolder { value: "mkv".into() }; - let json = serde_json::to_string(&holder).unwrap(); - assert_eq!(json, r#"{"value":"mkv"}"#); - let back: NullishArcStrHolder = serde_json::from_str(&json).unwrap(); - assert_eq!(back.value.as_ref(), "mkv"); - } -} +mod pool; +mod scalar; +mod serde_adapters; +pub use pool::{interner_gc, interner_len, Internable}; +pub use scalar::is_nullish; +pub use serde_adapters::{ + arc_str_default_on_null, arc_str_null_is_none_option_serde, arc_str_null_is_none_serde, + arc_str_option_null_if_empty_serde, arc_str_option_serde, arc_str_serde, arc_str_vec_serde, + deserialize_as_option_arc_str, +}; +use serde_adapters::{ArcStrVisitor, NullishArcStrVisitor, NullishOptionArcStrVisitor, OptionArcStrVisitor}; diff --git a/shared/src/utils/string_interner/pool.rs b/shared/src/utils/string_interner/pool.rs new file mode 100644 index 000000000..b1635c715 --- /dev/null +++ b/shared/src/utils/string_interner/pool.rs @@ -0,0 +1,93 @@ +use crate::model::UUIDType; +use std::{ + borrow::Cow, + collections::HashSet, + sync::{Arc, LazyLock, RwLock}, +}; + +/// Global interning pool. +/// +/// ## Performance (millions of entries) +/// +/// * **Happy-path (already-interned):** one `RwLock::read()` + hash lookup + +/// `Arc::clone` (single atomic increment). Multiple threads can read +/// concurrently without blocking each other. +/// * **First-time intern:** upgrades to a write lock, double-checks, and +/// inserts. This only happens *once per unique string value*, so the write +/// path is not on the hot parse loop. +/// * **`deserialize_string` vs `deserialize_any`:** the former is *faster* +/// because saphyr skips bool / int / float parsing attempts and hands the +/// raw scalar text directly to the visitor. +/// * **Pruning:** call `interner_gc()` periodically (e.g. after a full +/// playlist reload) to release strings that are only referenced by the +/// pool itself. +pub(super) static INTERNER: LazyLock>>> = LazyLock::new(|| RwLock::new(HashSet::new())); + +pub trait Internable { + fn intern(self) -> Arc; +} + +impl Internable for &Arc { + fn intern(self) -> Arc { Arc::clone(self) } +} + +impl Internable for &Cow<'_, str> { + fn intern(self) -> Arc { + match self { + Cow::Borrowed(s) => intern_str(s), + Cow::Owned(s) => intern_string(s.clone()), + } + } +} + +impl Internable for &UUIDType { + fn intern(self) -> Arc { intern_string(self.to_string()) } +} + +impl Internable for String { + fn intern(self) -> Arc { intern_string(self) } +} + +impl Internable for &String { + fn intern(self) -> Arc { intern_str(self.as_str()) } +} + +impl Internable for &str { + fn intern(self) -> Arc { intern_str(self) } +} + +impl Internable for u32 { + fn intern(self) -> Arc { intern_string(self.to_string()) } +} + +impl Internable for u64 { + fn intern(self) -> Arc { intern_string(self.to_string()) } +} + +impl Internable for i64 { + fn intern(self) -> Arc { intern_string(self.to_string()) } +} + +/// Interns a string slice. +fn intern_str(s: &str) -> Arc { intern_impl!(s, Arc::from(s)) } + +/// Interns an owned string. +fn intern_string(s: String) -> Arc { intern_impl!(s.as_str(), Arc::from(s)) } + +/// Returns the current number of strings held in the interning pool. +/// Uses a read lock and is safe to call on hot paths for threshold checks. +pub fn interner_len() -> usize { INTERNER.read().map_or(0, |g| g.len()) } + +/// Garbage collection: removes strings that are only referenced by the cache. +pub fn interner_gc() -> usize { + if let Ok(mut guard) = INTERNER.write() { + let before = guard.len(); + guard.retain(|s| Arc::strong_count(s) > 1); + let removed = before - guard.len(); + if removed > 0 { + log::debug!("Pruned {removed} unused interned strings ({} remaining)", guard.len()); + } + return removed; + } + 0 +} diff --git a/shared/src/utils/string_interner/scalar.rs b/shared/src/utils/string_interner/scalar.rs new file mode 100644 index 000000000..c12419bd0 --- /dev/null +++ b/shared/src/utils/string_interner/scalar.rs @@ -0,0 +1,49 @@ +/// Convert an `f64` that reached `visit_f64` into a round-trip-safe string. +/// +/// Special values are emitted as `"infinity"`, `"-infinity"`, and `"nan"`. +/// `serde_saphyr` re-serializes these ambiguous scalars quoted, so they +/// survive a YAML round-trip as strings. +/// +/// This is a safety-net for paths that intentionally accept typed numeric +/// scalars and normalize them into strings. +#[inline] +pub(super) fn f64_to_str(v: f64) -> String { + if v.is_infinite() { + if v.is_sign_positive() { + "infinity".to_owned() + } else { + "-infinity".to_owned() + } + } else if v.is_nan() { + "nan".to_owned() + } else { + v.to_string() + } +} + +#[inline] +// This intentionally only normalizes the lowercase dot-prefixed spellings that +// serde_saphyr emits for special float scalars. Other YAML 1.1 variants such as +// `.Inf`, `.INF`, or `.NaN` are out of scope here because they are not produced +// by the current parser path. +pub(super) fn normalize_scalar_string(value: &str) -> &str { + match value { + ".inf" => "infinity", + "-.inf" => "-infinity", + ".nan" => "nan", + _ => value, + } +} + +/// Returns `true` for values that should be treated as absent. +/// +/// Covers: +/// - empty string (`""`) +/// - JSON/YAML null literals (`"null"`, `"~"`) +/// +/// Generic, field-agnostic. Callers decide whether `true` maps to `""` +/// (for non-optional `Arc`) or `None` (for `Option>`). +/// Case-sensitive on purpose: provider-supplied `"NULL"` or `"Null"` are real +/// values and must be preserved verbatim. +#[inline] +pub fn is_nullish(value: &str) -> bool { value.is_empty() || value == "~" || value.eq_ignore_ascii_case("null") } diff --git a/shared/src/utils/string_interner/serde_adapters.rs b/shared/src/utils/string_interner/serde_adapters.rs new file mode 100644 index 000000000..18302b4ad --- /dev/null +++ b/shared/src/utils/string_interner/serde_adapters.rs @@ -0,0 +1,340 @@ +use super::{ + is_nullish, + scalar::{f64_to_str, normalize_scalar_string}, + Internable, +}; +use serde::{ + de::{IgnoredAny, MapAccess, SeqAccess, Visitor}, + Deserializer, +}; +use std::{fmt, sync::Arc}; + +// +// Two reusable visitor types live here so that multiple public entry-points +// can share them without code duplication: +// +// ArcStrVisitor -> Arc (null/empty -> "") +// OptionArcStrVisitor -> Option> (null/empty -> None) +// +// `ArcStrVisitor::visit_some` uses `deserialize_any(self)`, so numeric/bool +// scalars can flow into the typed `visit_*` methods. The tradeoff is that raw +// numeric notation may be normalized (for example `1e2` becomes `"100"`). +// +// `OptionArcStrVisitor::visit_some` intentionally diverges and uses +// `deserialize_any` so JSON/YAML numeric inputs can flow into `visit_i64`, +// `visit_u64` or `visit_f64`. The tradeoff is that this path no longer forces +// raw-scalar preservation in the same way as `ArcStrVisitor`. + +/// Visitor that produces `Arc`, mapping null / empty -> `""`. +pub(super) struct ArcStrVisitor; + +impl<'de> Visitor<'de> for ArcStrVisitor { + type Value = Arc; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { f.write_str("a string, number, boolean, or null") } + + fn visit_str(self, v: &str) -> Result { + Ok(normalize_scalar_string(v).intern()) + } + fn visit_string(self, v: String) -> Result { + Ok(normalize_scalar_string(v.as_str()).intern()) + } + fn visit_bool(self, v: bool) -> Result { Ok(v.to_string().intern()) } + fn visit_i64(self, v: i64) -> Result { Ok(v.to_string().intern()) } + fn visit_u64(self, v: u64) -> Result { Ok(v.to_string().intern()) } + fn visit_f64(self, v: f64) -> Result { Ok(f64_to_str(v).intern()) } + fn visit_unit(self) -> Result { Ok("".intern()) } + fn visit_none(self) -> Result { Ok("".intern()) } + fn visit_some>(self, d: D) -> Result { d.deserialize_any(self) } +} + +/// Visitor that produces `Option>`, mapping null / empty -> `None`. +pub(super) struct OptionArcStrVisitor; + +impl<'de> Visitor<'de> for OptionArcStrVisitor { + type Value = Option>; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a string, number, boolean, null, or empty") + } + + fn visit_str(self, v: &str) -> Result { + let normalized = normalize_scalar_string(v); + if normalized.is_empty() { + Ok(None) + } else { + Ok(Some(normalized.intern())) + } + } + fn visit_string(self, v: String) -> Result { + let normalized = normalize_scalar_string(v.as_str()); + if normalized.is_empty() { + Ok(None) + } else { + Ok(Some(normalized.intern())) + } + } + fn visit_bool(self, v: bool) -> Result { Ok(Some(v.to_string().intern())) } + fn visit_i64(self, v: i64) -> Result { Ok(Some(v.to_string().intern())) } + fn visit_u64(self, v: u64) -> Result { Ok(Some(v.to_string().intern())) } + fn visit_f64(self, v: f64) -> Result { Ok(Some(f64_to_str(v).intern())) } + fn visit_unit(self) -> Result { Ok(None) } + fn visit_none(self) -> Result { Ok(None) } + fn visit_some>(self, d: D) -> Result { d.deserialize_any(self) } + fn visit_seq>(self, mut seq: A) -> Result { + while seq.next_element::()?.is_some() {} + log::debug!("ignored sequence while deserializing string interner, returning None"); + Ok(None) + } + fn visit_map>(self, mut map: A) -> Result { + while map.next_entry::()?.is_some() {} + log::debug!("ignored map while deserializing string interner, returning None"); + Ok(None) + } +} + +pub mod arc_str_vec_serde { + // These serde helper modules deliberately re-use the parent module's imports. + // Clippy's expansion of this glob names private sibling modules and does not + // compile, so the glob is kept. + #[allow(clippy::wildcard_imports)] + use super::super::*; + use serde::ser::SerializeSeq; + + pub fn serialize(value: &Vec>, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(Some(value.len()))?; + for s in value { + seq.serialize_element(s.as_ref())?; + } + seq.end() + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> + where + D: Deserializer<'de>, + { + let vec = Vec::::deserialize(deserializer)?; + Ok(vec.into_iter().map(super::super::Internable::intern).collect()) + } +} + +pub mod arc_str_serde { + // These serde helper modules deliberately re-use the parent module's imports. + // Clippy's expansion of this glob names private sibling modules and does not + // compile, so the glob is kept. + #[allow(clippy::wildcard_imports)] + use super::super::*; + + pub fn serialize(value: &Arc, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(value) + } + + /// Deserialize a scalar as an interned `Arc`. + /// + /// This goes through `deserialize_option(ArcStrVisitor)`, and + /// `ArcStrVisitor::visit_some` uses `deserialize_any`. That allows numeric + /// JSON/YAML scalars to be accepted, but raw numeric notation may be lost + /// during normalization (for example `1e2` becomes `"100"`). + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + deserializer.deserialize_option(ArcStrVisitor) + } +} + +pub mod arc_str_option_serde { + // These serde helper modules deliberately re-use the parent module's imports. + // Clippy's expansion of this glob names private sibling modules and does not + // compile, so the glob is kept. + #[allow(clippy::wildcard_imports)] + use super::super::*; + + pub fn serialize(value: &Option>, serializer: S) -> Result + where + S: Serializer, + { + match value { + Some(s) => serializer.serialize_str(s), + None => serializer.serialize_none(), + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> + where + D: Deserializer<'de>, + { + deserializer.deserialize_option(OptionArcStrVisitor) + } + + pub fn serialize_null_if_empty(value: &Option>, serializer: S) -> Result + where + S: Serializer, + { + match value { + None => serializer.serialize_none(), + Some(s) if s.is_empty() => serializer.serialize_none(), + Some(s) => serializer.serialize_str(s), + } + } +} + +pub mod arc_str_option_null_if_empty_serde { + pub use super::super::arc_str_option_serde::{deserialize, serialize_null_if_empty as serialize}; +} + +/// Visitor that mirrors `ArcStrVisitor` but collapses `"null"`, `""`, and `~` +/// to an empty `Arc`. Generic over the field, not specific to +/// `container_extension`. +pub(super) struct NullishArcStrVisitor; + +impl<'de> Visitor<'de> for NullishArcStrVisitor { + type Value = Arc; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a string, number, boolean, or null (literal \"null\"/\"~\"/empty mapped to empty)") + } + + fn visit_str(self, v: &str) -> Result { + if is_nullish(v) { + Ok("".intern()) + } else { + Ok(normalize_scalar_string(v).intern()) + } + } + fn visit_string(self, v: String) -> Result { + if is_nullish(&v) { + Ok("".intern()) + } else { + Ok(normalize_scalar_string(&v).intern()) + } + } + fn visit_bool(self, v: bool) -> Result { Ok(v.to_string().intern()) } + fn visit_i64(self, v: i64) -> Result { Ok(v.to_string().intern()) } + fn visit_u64(self, v: u64) -> Result { Ok(v.to_string().intern()) } + fn visit_f64(self, v: f64) -> Result { Ok(f64_to_str(v).intern()) } + fn visit_unit(self) -> Result { Ok("".intern()) } + fn visit_none(self) -> Result { Ok("".intern()) } + fn visit_some>(self, d: D) -> Result { d.deserialize_any(self) } +} + +/// Visitor that mirrors `OptionArcStrVisitor` but collapses `"null"`, `""`, +/// and `~` to `None`. Generic over the field. +pub(super) struct NullishOptionArcStrVisitor; + +impl<'de> Visitor<'de> for NullishOptionArcStrVisitor { + type Value = Option>; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a string, number, boolean, null, or empty (literal \"null\"/\"~\"/empty mapped to None)") + } + + fn visit_str(self, v: &str) -> Result { + if is_nullish(v) { + Ok(None) + } else { + Ok(Some(normalize_scalar_string(v).intern())) + } + } + fn visit_string(self, v: String) -> Result { + if is_nullish(&v) { + Ok(None) + } else { + Ok(Some(normalize_scalar_string(&v).intern())) + } + } + fn visit_bool(self, v: bool) -> Result { Ok(Some(v.to_string().intern())) } + fn visit_i64(self, v: i64) -> Result { Ok(Some(v.to_string().intern())) } + fn visit_u64(self, v: u64) -> Result { Ok(Some(v.to_string().intern())) } + fn visit_f64(self, v: f64) -> Result { Ok(Some(f64_to_str(v).intern())) } + fn visit_unit(self) -> Result { Ok(None) } + fn visit_none(self) -> Result { Ok(None) } + fn visit_some>(self, d: D) -> Result { d.deserialize_any(self) } + fn visit_seq>(self, mut seq: A) -> Result { + while seq.next_element::()?.is_some() {} + log::debug!("ignored sequence while deserializing nullish arc_str, returning None"); + Ok(None) + } + fn visit_map>(self, mut map: A) -> Result { + while map.next_entry::()?.is_some() {} + log::debug!("ignored map while deserializing nullish arc_str, returning None"); + Ok(None) + } +} + +/// Generic serde for `Arc` that treats JSON null, YAML null/`~`, empty +/// string, and the literal four-character string `"null"` as empty. +/// +/// Field-agnostic. Apply with `#[serde(with = "arc_str_null_is_none_serde")]`. +/// +/// Use instead of `arc_str_serde` whenever the underlying value might be +/// reported as `"null"` by a misbehaving provider. +pub mod arc_str_null_is_none_serde { + // These serde helper modules deliberately re-use the parent module's imports. + // Clippy's expansion of this glob names private sibling modules and does not + // compile, so the glob is kept. + #[allow(clippy::wildcard_imports)] + use super::super::*; + + pub fn serialize(value: &Arc, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(value) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + deserializer.deserialize_option(NullishArcStrVisitor) + } +} + +/// Generic serde for `Option>` that treats JSON null, YAML null/`~`, +/// empty string, and the literal four-character string `"null"` as `None`. On +/// serialization, `None`, empty, and the literal `"null"` all become JSON null. +pub mod arc_str_null_is_none_option_serde { + // These serde helper modules deliberately re-use the parent module's imports. + // Clippy's expansion of this glob names private sibling modules and does not + // compile, so the glob is kept. + #[allow(clippy::wildcard_imports)] + use super::super::*; + + pub fn serialize(value: &Option>, serializer: S) -> Result + where + S: Serializer, + { + match value { + Some(s) if !super::super::is_nullish(s) => serializer.serialize_str(s), + _ => serializer.serialize_none(), + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> + where + D: Deserializer<'de>, + { + deserializer.deserialize_option(NullishOptionArcStrVisitor) + } +} + +pub fn arc_str_default_on_null<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + deserializer.deserialize_option(ArcStrVisitor) +} + +pub fn deserialize_as_option_arc_str<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + deserializer.deserialize_option(OptionArcStrVisitor) +} diff --git a/shared/src/utils/string_interner/tests.rs b/shared/src/utils/string_interner/tests.rs new file mode 100644 index 000000000..9094fdf43 --- /dev/null +++ b/shared/src/utils/string_interner/tests.rs @@ -0,0 +1,170 @@ +// These serde helper modules deliberately re-use the parent module's imports. +// Clippy's expansion of this glob names private sibling modules and does not +// compile, so the glob is kept. +#[allow(clippy::wildcard_imports)] +use super::*; + +#[derive(Debug, serde::Deserialize)] +struct ArcStrHolder { + #[serde(default, with = "arc_str_serde")] + value: Arc, +} + +#[derive(Debug, serde::Deserialize)] +struct OptArcStrHolder { + #[serde(default, with = "arc_str_option_serde")] + value: Option>, +} + +#[test] +fn arc_str_serde_preserves_yaml_infinity_literal_as_string() { + let parsed: ArcStrHolder = serde_saphyr::from_str("value: infinity\n").unwrap(); + assert_eq!(parsed.value.as_ref(), "infinity"); +} + +#[test] +fn arc_str_serde_preserves_yaml_numeric_like_word_as_string() { + let parsed: ArcStrHolder = serde_saphyr::from_str("value: 01abc\n").unwrap(); + assert_eq!(parsed.value.as_ref(), "01abc"); +} + +#[test] +fn arc_str_serde_accepts_json_integer() { + let parsed: ArcStrHolder = serde_json::from_str(r#"{"value":1285728}"#).unwrap(); + assert_eq!(parsed.value.as_ref(), "1285728"); +} + +#[test] +fn arc_str_option_serde_accepts_json_integer() { + let parsed: OptArcStrHolder = serde_json::from_str(r#"{"value":8169}"#).unwrap(); + assert_eq!(parsed.value.as_deref(), Some("8169")); +} + +#[test] +fn arc_str_option_serde_accepts_json_string() { + let parsed: OptArcStrHolder = serde_json::from_str(r#"{"value":"8169"}"#).unwrap(); + assert_eq!(parsed.value.as_deref(), Some("8169")); +} + +#[test] +fn arc_str_option_serde_maps_empty_string_to_none() { + let parsed: OptArcStrHolder = serde_json::from_str(r#"{"value":""}"#).unwrap(); + assert_eq!(parsed.value, None); +} + +#[test] +fn arc_str_serde_normalizes_json_scientific_notation_numbers() { + let parsed: ArcStrHolder = serde_json::from_str(r#"{"value":1e2}"#).unwrap(); + assert_eq!(parsed.value.as_ref(), "100"); +} + +#[test] +fn arc_str_serde_normalizes_yaml_special_float_scalars() { + let parsed_inf: ArcStrHolder = serde_saphyr::from_str("value: .inf\n").unwrap(); + let parsed_neg_inf: ArcStrHolder = serde_saphyr::from_str("value: -.inf\n").unwrap(); + let parsed_nan: ArcStrHolder = serde_saphyr::from_str("value: .nan\n").unwrap(); + + assert_eq!(parsed_inf.value.as_ref(), "infinity"); + assert_eq!(parsed_neg_inf.value.as_ref(), "-infinity"); + assert_eq!(parsed_nan.value.as_ref(), "nan"); +} + +#[test] +fn arc_str_option_serde_normalizes_yaml_special_float_scalars() { + let parsed_inf: OptArcStrHolder = serde_saphyr::from_str("value: .inf\n").unwrap(); + let parsed_neg_inf: OptArcStrHolder = serde_saphyr::from_str("value: -.inf\n").unwrap(); + let parsed_nan: OptArcStrHolder = serde_saphyr::from_str("value: .nan\n").unwrap(); + + assert_eq!(parsed_inf.value.as_deref(), Some("infinity")); + assert_eq!(parsed_neg_inf.value.as_deref(), Some("-infinity")); + assert_eq!(parsed_nan.value.as_deref(), Some("nan")); +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +struct NullishArcStrHolder { + #[serde(default, with = "arc_str_null_is_none_serde")] + value: Arc, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +struct NullishOptArcStrHolder { + #[serde(default, with = "arc_str_null_is_none_option_serde")] + value: Option>, +} + +#[test] +fn null_is_none_arc_str_treats_literal_null_string_as_empty() { + let parsed: NullishArcStrHolder = serde_json::from_str(r#"{"value":"null"}"#).unwrap(); + assert!(parsed.value.is_empty()); +} + +#[test] +fn null_is_none_arc_str_treats_yaml_null_scalar_as_empty() { + let parsed: NullishArcStrHolder = serde_saphyr::from_str("value: null\n").unwrap(); + assert!(parsed.value.is_empty()); + let parsed_tilde: NullishArcStrHolder = serde_saphyr::from_str("value: ~\n").unwrap(); + assert!(parsed_tilde.value.is_empty()); +} + +#[test] +fn null_is_none_arc_str_treats_json_null_as_empty() { + let parsed: NullishArcStrHolder = serde_json::from_str(r#"{"value":null}"#).unwrap(); + assert!(parsed.value.is_empty()); +} + +#[test] +fn null_is_none_arc_str_preserves_real_extensions() { + for ext in ["mkv", "mp4", "ts", "avi"] { + let json = format!(r#"{{"value":"{ext}"}}"#); + let parsed: NullishArcStrHolder = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.value.as_ref(), ext, "extension {ext} must survive"); + } +} + +#[test] +fn null_is_none_option_treats_literal_null_string_as_none() { + let parsed: NullishOptArcStrHolder = serde_json::from_str(r#"{"value":"null"}"#).unwrap(); + assert_eq!(parsed.value, None); +} + +#[test] +fn null_is_none_option_treats_json_null_as_none() { + let parsed: NullishOptArcStrHolder = serde_json::from_str(r#"{"value":null}"#).unwrap(); + assert_eq!(parsed.value, None); +} + +#[test] +fn null_is_none_option_treats_empty_string_as_none() { + let parsed: NullishOptArcStrHolder = serde_json::from_str(r#"{"value":""}"#).unwrap(); + assert_eq!(parsed.value, None); +} + +#[test] +fn null_is_none_option_preserves_real_extensions() { + for ext in ["mkv", "mp4", "ts"] { + let json = format!(r#"{{"value":"{ext}"}}"#); + let parsed: NullishOptArcStrHolder = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.value.as_deref(), Some(ext), "extension {ext} must survive"); + } +} + +#[test] +fn null_is_none_option_serializes_none_and_literal_null_as_json_null() { + let none_value: NullishOptArcStrHolder = NullishOptArcStrHolder { value: None }; + assert_eq!(serde_json::to_string(&none_value).unwrap(), r#"{"value":null}"#); + + let literal_null: NullishOptArcStrHolder = NullishOptArcStrHolder { value: Some("null".into()) }; + assert_eq!(serde_json::to_string(&literal_null).unwrap(), r#"{"value":null}"#); + + let empty: NullishOptArcStrHolder = NullishOptArcStrHolder { value: Some("".into()) }; + assert_eq!(serde_json::to_string(&empty).unwrap(), r#"{"value":null}"#); +} + +#[test] +fn null_is_none_arc_str_roundtrips_real_extensions_byte_for_byte() { + let holder = NullishArcStrHolder { value: "mkv".into() }; + let json = serde_json::to_string(&holder).unwrap(); + assert_eq!(json, r#"{"value":"mkv"}"#); + let back: NullishArcStrHolder = serde_json::from_str(&json).unwrap(); + assert_eq!(back.value.as_ref(), "mkv"); +} diff --git a/test/fixtures/testkit/scenarios/m3u-hls-share-conservative-startup.yml b/test/fixtures/testkit/scenarios/m3u-hls-share-conservative-startup.yml new file mode 100644 index 000000000..fb5de90dd --- /dev/null +++ b/test/fixtures/testkit/scenarios/m3u-hls-share-conservative-startup.yml @@ -0,0 +1,50 @@ +schema_version: 1 +name: m3u-hls-share-conservative-startup +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + fixture_stream: + share_live_hls: true + share_live_mpeg_ts: false + hls_startup_mode: conservative + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: m3u-hls-share-conservative-startup + hls_profile: + prefix_delay_millis: 1000 + window_segments: 6 + missing_head_segments: 0 +actors: + - id: viewer + agent: local + username: viewer + user_agent: tuliprox-testkit/viewer + - id: second-viewer + agent: local + username: other + user_agent: tuliprox-testkit/second-viewer +channels: + shared-channel: { origin_marker: 17, protocol: hls } +steps: + - command_id: first-viewer-starts + start: { actor: viewer, playback_id: first, session_group: first, channel: shared-channel } + await: { valid_frames: 2 } + assert_origin: { no_limit_rejections: true } + - command_id: second-viewer-joins + start: { actor: second-viewer, playback_id: second, session_group: second, channel: shared-channel } + await: { valid_frames: 2 } + assert_origin: { no_limit_rejections: true } + - command_id: first-viewer-stops + stop: { playback_id: first } + - command_id: second-viewer-stops + stop: { playback_id: second } + assert_runtime: { active_users: 0, active_user_connections: 0 } +policy_contract: + user_access_control: true + users: + viewer: { max_connections: 4 } + other: { max_connections: 4 } + provider_max_connections: 8 diff --git a/test/fixtures/testkit/scenarios/m3u-hls-share-first-ready-missing-head.yml b/test/fixtures/testkit/scenarios/m3u-hls-share-first-ready-missing-head.yml new file mode 100644 index 000000000..d0cb541da --- /dev/null +++ b/test/fixtures/testkit/scenarios/m3u-hls-share-first-ready-missing-head.yml @@ -0,0 +1,50 @@ +schema_version: 1 +name: m3u-hls-share-first-ready-missing-head +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + fixture_stream: + share_live_hls: true + share_live_mpeg_ts: false + hls_startup_mode: first_ready + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: m3u-hls-share-first-ready-missing-head + hls_profile: + prefix_delay_millis: 1000 + window_segments: 6 + missing_head_segments: 2 +actors: + - id: viewer + agent: local + username: viewer + user_agent: tuliprox-testkit/viewer + - id: second-viewer + agent: local + username: other + user_agent: tuliprox-testkit/second-viewer +channels: + shared-channel: { origin_marker: 17, protocol: hls } +steps: + - command_id: first-viewer-starts + start: { actor: viewer, playback_id: first, session_group: first, channel: shared-channel } + await: { valid_frames: 2 } + assert_origin: { no_limit_rejections: true } + - command_id: second-viewer-joins + start: { actor: second-viewer, playback_id: second, session_group: second, channel: shared-channel } + await: { valid_frames: 2 } + assert_origin: { no_limit_rejections: true } + - command_id: first-viewer-stops + stop: { playback_id: first } + - command_id: second-viewer-stops + stop: { playback_id: second } + assert_runtime: { active_users: 0, active_user_connections: 0 } +policy_contract: + user_access_control: true + users: + viewer: { max_connections: 4 } + other: { max_connections: 4 } + provider_max_connections: 8 diff --git a/test/fixtures/testkit/scenarios/m3u-hls-share-first-ready-startup.yml b/test/fixtures/testkit/scenarios/m3u-hls-share-first-ready-startup.yml new file mode 100644 index 000000000..9ecdd791c --- /dev/null +++ b/test/fixtures/testkit/scenarios/m3u-hls-share-first-ready-startup.yml @@ -0,0 +1,50 @@ +schema_version: 1 +name: m3u-hls-share-first-ready-startup +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + fixture_stream: + share_live_hls: true + share_live_mpeg_ts: false + hls_startup_mode: first_ready + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: m3u-hls-share-first-ready-startup + hls_profile: + prefix_delay_millis: 1000 + window_segments: 6 + missing_head_segments: 0 +actors: + - id: viewer + agent: local + username: viewer + user_agent: tuliprox-testkit/viewer + - id: second-viewer + agent: local + username: other + user_agent: tuliprox-testkit/second-viewer +channels: + shared-channel: { origin_marker: 17, protocol: hls } +steps: + - command_id: first-viewer-starts + start: { actor: viewer, playback_id: first, session_group: first, channel: shared-channel } + await: { valid_frames: 2 } + assert_origin: { no_limit_rejections: true } + - command_id: second-viewer-joins + start: { actor: second-viewer, playback_id: second, session_group: second, channel: shared-channel } + await: { valid_frames: 2 } + assert_origin: { no_limit_rejections: true } + - command_id: first-viewer-stops + stop: { playback_id: first } + - command_id: second-viewer-stops + stop: { playback_id: second } + assert_runtime: { active_users: 0, active_user_connections: 0 } +policy_contract: + user_access_control: true + users: + viewer: { max_connections: 4 } + other: { max_connections: 4 } + provider_max_connections: 8 diff --git a/test/fixtures/testkit/scenarios/m3u-hls-share-progressive-missing-head.yml b/test/fixtures/testkit/scenarios/m3u-hls-share-progressive-missing-head.yml new file mode 100644 index 000000000..8e78eb951 --- /dev/null +++ b/test/fixtures/testkit/scenarios/m3u-hls-share-progressive-missing-head.yml @@ -0,0 +1,50 @@ +schema_version: 1 +name: m3u-hls-share-progressive-missing-head +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + fixture_stream: + share_live_hls: true + share_live_mpeg_ts: false + hls_startup_mode: progressive + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: m3u-hls-share-progressive-missing-head + hls_profile: + prefix_delay_millis: 1000 + window_segments: 6 + missing_head_segments: 2 +actors: + - id: viewer + agent: local + username: viewer + user_agent: tuliprox-testkit/viewer + - id: second-viewer + agent: local + username: other + user_agent: tuliprox-testkit/second-viewer +channels: + shared-channel: { origin_marker: 17, protocol: hls } +steps: + - command_id: first-viewer-starts + start: { actor: viewer, playback_id: first, session_group: first, channel: shared-channel } + await: { valid_frames: 2 } + assert_origin: { no_limit_rejections: true } + - command_id: second-viewer-joins + start: { actor: second-viewer, playback_id: second, session_group: second, channel: shared-channel } + await: { valid_frames: 2 } + assert_origin: { no_limit_rejections: true } + - command_id: first-viewer-stops + stop: { playback_id: first } + - command_id: second-viewer-stops + stop: { playback_id: second } + assert_runtime: { active_users: 0, active_user_connections: 0 } +policy_contract: + user_access_control: true + users: + viewer: { max_connections: 4 } + other: { max_connections: 4 } + provider_max_connections: 8 diff --git a/test/fixtures/testkit/scenarios/m3u-hls-share-progressive-startup.yml b/test/fixtures/testkit/scenarios/m3u-hls-share-progressive-startup.yml new file mode 100644 index 000000000..28613f614 --- /dev/null +++ b/test/fixtures/testkit/scenarios/m3u-hls-share-progressive-startup.yml @@ -0,0 +1,50 @@ +schema_version: 1 +name: m3u-hls-share-progressive-startup +tuliprox: + base_url: http://fixture.invalid + execution_mode: isolated_fixture + fixture_stream: + share_live_hls: true + share_live_mpeg_ts: false + hls_startup_mode: progressive + bootstrap: + command: "${env:TULIPROX_TESTKIT_SUT_BINARY}" + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + readiness_url: "{base_url}/healthcheck" +origin: + run_id: m3u-hls-share-progressive-startup + hls_profile: + prefix_delay_millis: 1000 + window_segments: 6 + missing_head_segments: 0 +actors: + - id: viewer + agent: local + username: viewer + user_agent: tuliprox-testkit/viewer + - id: second-viewer + agent: local + username: other + user_agent: tuliprox-testkit/second-viewer +channels: + shared-channel: { origin_marker: 17, protocol: hls } +steps: + - command_id: first-viewer-starts + start: { actor: viewer, playback_id: first, session_group: first, channel: shared-channel } + await: { valid_frames: 2 } + assert_origin: { no_limit_rejections: true } + - command_id: second-viewer-joins + start: { actor: second-viewer, playback_id: second, session_group: second, channel: shared-channel } + await: { valid_frames: 2 } + assert_origin: { no_limit_rejections: true } + - command_id: first-viewer-stops + stop: { playback_id: first } + - command_id: second-viewer-stops + stop: { playback_id: second } + assert_runtime: { active_users: 0, active_user_connections: 0 } +policy_contract: + user_access_control: true + users: + viewer: { max_connections: 4 } + other: { max_connections: 4 } + provider_max_connections: 8 diff --git a/tools/tuliprox-testkit/src/bootstrap.rs b/tools/tuliprox-testkit/src/bootstrap.rs index 4d06e9be8..e8875d809 100644 --- a/tools/tuliprox-testkit/src/bootstrap.rs +++ b/tools/tuliprox-testkit/src/bootstrap.rs @@ -62,10 +62,13 @@ impl FixturePaths { let config_file = root.join("config.yml"); let source_file = root.join("source.yml"); let api_proxy_file = root.join("api-proxy.yml"); - std::fs::write( - &config_file, - render_config(tuliprox_address, &web_root, &storage_dir, &backup_dir, &history_dir, policy)?, - )?; + let mut config = render_config(tuliprox_address, &web_root, &storage_dir, &backup_dir, &history_dir, policy)?; + if let Some(mode) = fixture_stream.hls_startup_mode { + let _ = writeln!(config, + " hls_cache:\n cache_path: {}\n max_concurrent_segment_fetches_per_session: 3\n startup:\n mode: {mode}", + yaml_scalar(&root.join("hls-cache"))); + } + std::fs::write(&config_file, config)?; std::fs::write( &source_file, render_sources(run_id, origin_address, channels, policy, fixture_stream, add_xtream_output, input), @@ -360,6 +363,11 @@ impl IsolatedFixture { origin_cmd.arg("--hls-markers").arg(hls_markers.iter().map(u32::to_string).collect::>().join(",")); } if let Some(origin_cfg) = origin_config { + if let Some(profile) = &origin_cfg.hls_profile { + origin_cmd.arg("--hls-profile").arg( + serde_json::to_string(profile).map_err(|error| TestkitError::Configuration(error.to_string()))?, + ); + } if let Some(limit) = origin_cfg.account_limit { origin_cmd.arg("--account-limit").arg(limit.to_string()); } @@ -662,7 +670,11 @@ mod tests { #[test] fn render_sources_with_mpeg_ts_sharing_disabled() { - let opts = crate::config::FixtureStreamOptions { share_live_hls: true, share_live_mpeg_ts: false }; + let opts = crate::config::FixtureStreamOptions { + share_live_hls: true, + share_live_mpeg_ts: false, + hls_startup_mode: None, + }; let source = render_sources("run", "127.0.0.1:9910".parse().unwrap(), &HashMap::new(), None, &opts, false, &m3u_input()); assert!(source.contains("mpeg_ts: false")); diff --git a/tools/tuliprox-testkit/src/config.rs b/tools/tuliprox-testkit/src/config.rs index 3b5590222..df69bfdab 100644 --- a/tools/tuliprox-testkit/src/config.rs +++ b/tools/tuliprox-testkit/src/config.rs @@ -197,6 +197,8 @@ pub struct OriginConfig { pub account_limit: Option, #[serde(default)] pub limit_mode: Option, + #[serde(default)] + pub hls_profile: Option, } #[derive(Debug, Clone, Deserialize)] @@ -238,10 +240,12 @@ pub enum PlaybackEndpoint { pub struct FixtureStreamOptions { pub share_live_hls: bool, pub share_live_mpeg_ts: bool, + #[serde(default)] + pub hls_startup_mode: Option, } impl Default for FixtureStreamOptions { - fn default() -> Self { Self { share_live_hls: true, share_live_mpeg_ts: true } } + fn default() -> Self { Self { share_live_hls: true, share_live_mpeg_ts: true, hls_startup_mode: None } } } #[derive(Debug, Clone, Deserialize)] @@ -500,6 +504,9 @@ impl Scenario { if let Some(contract) = &self.policy_contract { validate_provider_pool_contract(contract)?; } + if let Some(profile) = self.origin.as_ref().and_then(|origin| origin.hls_profile.as_ref()) { + profile.validate()?; + } let actor_ids = self.actors.iter().map(|actor| actor.id.as_str()).collect::>(); if actor_ids.len() != self.actors.len() || actor_ids.iter().any(|id| id.is_empty()) { return Err(TestkitError::Configuration("actor IDs must be unique and non-empty".to_owned())); @@ -982,7 +989,7 @@ steps: } #[test] - fn stalker_fixture_validation_rejects_unknown_presets_and_markers() -> Result<(), TestkitError> { + fn stalker_fixture_validation_rejects_unknown_presets_and_markers() { let mut scenario = Scenario { schema_version: 1, name: "stalker".to_owned(), @@ -1027,7 +1034,6 @@ steps: scenario.tuliprox.stalker.refuse_create_link_markers.clear(); scenario.tuliprox.stalker.refuse_create_link_once = true; assert!(scenario.validate().is_err(), "portal refusals require a stalker input"); - Ok(()) } #[test] diff --git a/tools/tuliprox-testkit/src/hls_probe.rs b/tools/tuliprox-testkit/src/hls_probe.rs new file mode 100644 index 000000000..d0c84bf0a --- /dev/null +++ b/tools/tuliprox-testkit/src/hls_probe.rs @@ -0,0 +1,365 @@ +//! Synthetic HLS timing and representation checks. These measure bytes, not decoded pictures. +use crate::{ + frame::{Frame, FrameDecoder, FrameValidator}, + protocol::{OriginStreamId, RunId}, + TestkitError, +}; +use bytes::{Bytes, BytesMut}; +use futures::{Stream, StreamExt}; +use serde::{Deserialize, Serialize}; +use std::{fmt, time::Duration}; + +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StartupMode { + Conservative, + FirstReady, + Progressive, +} + +impl fmt::Display for StartupMode { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Conservative => "conservative", + Self::FirstReady => "first_ready", + Self::Progressive => "progressive", + }) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct HlsOriginProfile { + pub window_segments: u32, + pub prefix_delay_millis: u64, + pub missing_head_segments: u32, + pub abort_segments: Vec, + pub unknown_length: bool, +} + +impl Default for HlsOriginProfile { + fn default() -> Self { + Self { + window_segments: 6, + prefix_delay_millis: 1000, + missing_head_segments: 0, + abort_segments: Vec::new(), + unknown_length: false, + } + } +} + +impl HlsOriginProfile { + pub fn validate(&self) -> Result<(), TestkitError> { + if !(3..=64).contains(&self.window_segments) + || self.missing_head_segments >= self.window_segments + || self.prefix_delay_millis > 10_000 + || self.abort_segments.len() > 64 + { + return Err(TestkitError::Configuration( + "HLS profile requires 3..64 segments, a shorter missing head, at most 64 aborts and delay <= 10000 ms" + .to_owned(), + )); + } + Ok(()) + } +} + +pub fn segment_parts(run: &RunId, marker: u32, segment: u64) -> Result<(Bytes, Bytes), TestkitError> { + let first = + segment.checked_mul(2).ok_or_else(|| TestkitError::Protocol("HLS frame sequence overflow".to_owned()))?; + let second = + first.checked_add(1).ok_or_else(|| TestkitError::Protocol("HLS frame sequence overflow".to_owned()))?; + let stream = OriginStreamId::new(format!("hls-presentation-{marker}")); + let mut prefix = BytesMut::new(); + let mut tail = BytesMut::new(); + Frame::synthetic(run, &stream, marker, first).encode(&mut prefix)?; + Frame::synthetic(run, &stream, marker, second).encode(&mut tail)?; + Ok((prefix.freeze(), tail.freeze())) +} + +pub fn segment_stream( + parts: (Bytes, Bytes), + profile: &HlsOriginProfile, + sequence: u64, +) -> impl Stream> + Send + 'static { + let delay = Duration::from_millis(profile.prefix_delay_millis); + let abort = profile.abort_segments.contains(&sequence); + futures::stream::once(async move { Ok(parts.0) }).chain(futures::stream::once(async move { + tokio::time::sleep(delay).await; + if abort { + Err(std::io::Error::new(std::io::ErrorKind::ConnectionReset, "injected HLS origin body drop")) + } else { + Ok(parts.1) + } + })) +} + +#[derive(Debug, Default)] +pub struct ProbeOptions { + pub verify_revision: bool, + pub expect_body_error: bool, + pub max_first_byte_millis: Option, + pub min_prefix_to_eof_millis: Option, +} + +#[derive(Debug, Serialize)] +pub struct ProbeResult { + pub entry_to_first_byte_millis: u64, + pub entry_to_body_end_millis: u64, + pub prefix_to_body_end_millis: u64, + pub received_bytes: usize, + pub valid_frames: u64, + pub body_error: bool, + pub retry_identical: bool, + pub range_identical: bool, +} + +const PROBE_TIMEOUT: Duration = Duration::from_secs(30); +const MAX_BODY: usize = 1024 * 1024; + +/// All requests share one absolute deadline. Errors after a prefix must be observable as errors. +pub async fn probe(url: &str, run: &RunId, marker: u32, options: &ProbeOptions) -> Result { + if options.verify_revision && options.expect_body_error { + return Err(TestkitError::Configuration("revision verification requires a complete body".to_owned())); + } + tokio::time::timeout(PROBE_TIMEOUT, probe_inner(url, run, marker, options)) + .await + .map_err(|_| TestkitError::Protocol("HLS probe exceeded absolute deadline".to_owned()))? +} + +async fn first_segment(client: &reqwest::Client, url: &str) -> Result { + let mut url = url::Url::parse(url).map_err(|error| TestkitError::Configuration(error.to_string()))?; + for _ in 0..8 { + let response = client.get(url).send().await?.error_for_status()?; + url = response.url().clone(); + let manifest = response.text().await?; + let uri = manifest + .lines() + .map(str::trim) + .find(|line| !line.is_empty() && !line.starts_with('#')) + .ok_or_else(|| TestkitError::Protocol("HLS probe found no segment URI".to_owned()))?; + url = url.join(uri).map_err(|error| TestkitError::Configuration(error.to_string()))?; + if !manifest.lines().any(|line| line.starts_with("#EXT-X-STREAM-INF")) { + return Ok(url); + } + } + Err(TestkitError::Protocol("HLS probe master nesting limit".to_owned())) +} + +async fn collect(response: reqwest::Response) -> Result { + let mut body = response.bytes_stream(); + let mut data = BytesMut::new(); + while let Some(chunk) = body.next().await { + let chunk = chunk?; + if chunk.len() > MAX_BODY.saturating_sub(data.len()) { + return Err(TestkitError::Protocol("HLS probe body exceeds 1 MiB limit".to_owned())); + } + data.extend_from_slice(&chunk); + } + Ok(data.freeze()) +} + +async fn verify_revision(client: &reqwest::Client, url: &url::Url, bytes: &[u8]) -> Result<(), TestkitError> { + let retry = collect(client.get(url.clone()).send().await?.error_for_status()?).await?; + if retry != bytes { + return Err(TestkitError::Protocol("same HLS URI returned different retry bytes".to_owned())); + } + let end = bytes.len().saturating_sub(1).min(63); + let response = client + .get(url.clone()) + .header(reqwest::header::RANGE, format!("bytes=0-{end}")) + .send() + .await? + .error_for_status()?; + if response.status() != reqwest::StatusCode::PARTIAL_CONTENT { + return Err(TestkitError::Protocol( + "bound HLS revision did not return HTTP 206 for a completed range".to_owned(), + )); + } + let expected = format!("bytes 0-{end}/{}", bytes.len()); + if response.headers().get(reqwest::header::CONTENT_RANGE).and_then(|value| value.to_str().ok()) + != Some(expected.as_str()) + { + return Err(TestkitError::Protocol("bound HLS revision returned a different Content-Range".to_owned())); + } + if collect(response).await?.as_ref() != &bytes[..=end] { + return Err(TestkitError::Protocol("HLS range offsets do not match the original revision".to_owned())); + } + Ok(()) +} + +async fn probe_inner(url: &str, run: &RunId, marker: u32, options: &ProbeOptions) -> Result { + let started = tokio::time::Instant::now(); + let client = reqwest::Client::builder().http1_only().build()?; + let segment_url = first_segment(&client, url).await?; + let response = client.get(segment_url.clone()).send().await?.error_for_status()?; + let expected_length = response.content_length(); + let mut stream = response.bytes_stream(); + let mut first_byte = None; + let mut body_error = false; + let mut bytes = BytesMut::new(); + while let Some(chunk) = stream.next().await { + if let Ok(chunk) = chunk { + if !chunk.is_empty() { + first_byte.get_or_insert_with(tokio::time::Instant::now); + } + if chunk.len() > MAX_BODY.saturating_sub(bytes.len()) { + return Err(TestkitError::Protocol("HLS probe body exceeds 1 MiB limit".to_owned())); + } + bytes.extend_from_slice(&chunk); + } else { + body_error = true; + break; + } + } + let ended = tokio::time::Instant::now(); + let first_byte = first_byte.ok_or_else(|| TestkitError::Protocol("HLS probe received no prefix".to_owned()))?; + if body_error != options.expect_body_error { + return Err(TestkitError::Protocol(format!( + "HLS body error: expected {}, observed {body_error}", + options.expect_body_error + ))); + } + if !body_error && expected_length.is_some_and(|length| length != bytes.len() as u64) { + return Err(TestkitError::Protocol("HLS response ended before Content-Length".to_owned())); + } + let mut validator = FrameValidator::new(run, marker); + let mut frames = 0; + for frame in FrameDecoder::default().push(&bytes)? { + validator.validate(&frame)?; + frames += 1; + } + if frames == 0 { + return Err(TestkitError::Protocol("HLS probe found no valid synthetic frame".to_owned())); + } + let millis = |duration: Duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX); + let mut result = ProbeResult { + entry_to_first_byte_millis: millis(first_byte - started), + entry_to_body_end_millis: millis(ended - started), + prefix_to_body_end_millis: millis(ended - first_byte), + received_bytes: bytes.len(), + valid_frames: frames, + body_error, + retry_identical: false, + range_identical: false, + }; + if options.max_first_byte_millis.is_some_and(|max| result.entry_to_first_byte_millis > max) + || options.min_prefix_to_eof_millis.is_some_and(|min| result.prefix_to_body_end_millis < min) + { + return Err(TestkitError::Protocol("HLS probe timing assertion failed".to_owned())); + } + if options.verify_revision { + verify_revision(&client, &segment_url, &bytes).await?; + result.retry_identical = true; + result.range_identical = true; + } + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{ + body::Body, + http::{HeaderMap, StatusCode}, + response::IntoResponse, + routing::get, + Router, + }; + + async fn server(abort: bool, change_retry: bool) -> Result<(String, tokio::task::JoinHandle<()>), TestkitError> { + let run = RunId::new("hls-probe-test"); + let parts = segment_parts(&run, 17, 0)?; + let attempts = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let app = Router::new().route("/index.m3u8", get(|| async { "#EXTM3U\n#EXTINF:4,\n0.ts\n" })).route( + "/0.ts", + get(move |headers: HeaderMap| { + let parts = parts.clone(); + let attempts = std::sync::Arc::clone(&attempts); + async move { + if headers.contains_key("range") { + let mut response = (StatusCode::PARTIAL_CONTENT, parts.0.slice(..64)).into_response(); + response.headers_mut().insert( + "content-range", + format!("bytes 0-63/{}", parts.0.len() + parts.1.len()) + .parse() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, + ); + return Ok::<_, StatusCode>(response); + } + let attempt = attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if change_retry && attempt > 0 { + return Ok("changed revision".into_response()); + } + let profile = HlsOriginProfile { + prefix_delay_millis: 100, + abort_segments: if abort { vec![0] } else { Vec::new() }, + ..HlsOriginProfile::default() + }; + Ok(Body::from_stream(segment_stream(parts, &profile, 0)).into_response()) + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let url = format!("http://{}/index.m3u8", listener.local_addr()?); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + Ok((url, server)) + } + + #[tokio::test] + async fn measures_prefix_before_eof_and_checks_retry_and_range() -> Result<(), TestkitError> { + let (url, server) = server(false, false).await?; + let result = probe( + &url, + &RunId::new("hls-probe-test"), + 17, + &ProbeOptions { verify_revision: true, min_prefix_to_eof_millis: Some(50), ..ProbeOptions::default() }, + ) + .await; + server.abort(); + let result = result?; + assert_eq!(result.valid_frames, 2); + assert!(result.retry_identical && result.range_identical); + Ok(()) + } + + #[tokio::test] + async fn truncated_chunked_body_is_an_error_after_valid_prefix() -> Result<(), TestkitError> { + let (url, server) = server(true, false).await?; + let result = probe( + &url, + &RunId::new("hls-probe-test"), + 17, + &ProbeOptions { expect_body_error: true, ..ProbeOptions::default() }, + ) + .await; + server.abort(); + assert!(result?.body_error); + Ok(()) + } + + #[tokio::test] + async fn detects_new_revision_on_same_uri() -> Result<(), TestkitError> { + let (url, server) = server(false, true).await?; + let result = probe( + &url, + &RunId::new("hls-probe-test"), + 17, + &ProbeOptions { verify_revision: true, ..ProbeOptions::default() }, + ) + .await; + server.abort(); + assert!(matches!(result, Err(TestkitError::Protocol(message)) if message.contains("different retry bytes"))); + Ok(()) + } + + #[test] + fn rejects_unbounded_or_empty_profiles() { + assert!(HlsOriginProfile { window_segments: 2, ..HlsOriginProfile::default() }.validate().is_err()); + assert!(HlsOriginProfile { missing_head_segments: 6, ..HlsOriginProfile::default() }.validate().is_err()); + assert!(HlsOriginProfile::default().validate().is_ok()); + } +} diff --git a/tools/tuliprox-testkit/src/lib.rs b/tools/tuliprox-testkit/src/lib.rs index f3adf046f..0e7048fb1 100644 --- a/tools/tuliprox-testkit/src/lib.rs +++ b/tools/tuliprox-testkit/src/lib.rs @@ -9,6 +9,7 @@ pub mod discovery; pub mod faults; pub mod frame; pub mod hls; +pub mod hls_probe; pub mod observation; pub mod oracle; pub mod origin_events; diff --git a/tools/tuliprox-testkit/src/main.rs b/tools/tuliprox-testkit/src/main.rs index 255dd6682..0c4439441 100644 --- a/tools/tuliprox-testkit/src/main.rs +++ b/tools/tuliprox-testkit/src/main.rs @@ -119,6 +119,9 @@ enum Command { /// Publish these catalog markers as HLS instead of MPEG-TS. #[arg(long, value_delimiter = ',')] hls_markers: Vec, + /// JSON HLS fault/timing profile; omitted preserves the legacy origin. + #[arg(long)] + hls_profile: Option, #[arg(long)] account_limit: Option, #[arg(long, default_value = "observe_only")] @@ -143,6 +146,23 @@ enum Command { #[arg(long)] run_id: Option, }, + /// Measure first HLS bytes and verify retry/range identity or a terminal body error. + HlsProbe { + #[arg(long)] + url: String, + #[arg(long, default_value = "local-run")] + run_id: String, + #[arg(long, default_value_t = 17)] + expected_marker: u32, + #[arg(long)] + verify_revision: bool, + #[arg(long)] + expect_body_error: bool, + #[arg(long)] + max_first_byte_millis: Option, + #[arg(long)] + min_prefix_to_eof_millis: Option, + }, /// Read and validate one framed playback URL; useful on a remote listener host. Agent { #[arg(long)] @@ -168,6 +188,7 @@ struct OriginState { bitrate: u32, markers: Arc>, hls_markers: Arc>, + hls_profile: Option>, stream_counter: Arc>, observations: Arc>>, faults: Arc>, @@ -296,6 +317,19 @@ async fn main() -> ExitCode { } } +fn parse_hls_profile( + value: Option<&str>, +) -> Result>, TestkitError> { + value + .map(|value| { + let profile: tuliprox_testkit::hls_probe::HlsOriginProfile = + serde_json::from_str(value).map_err(|error| TestkitError::Configuration(error.to_string()))?; + profile.validate()?; + Ok(Arc::new(profile)) + }) + .transpose() +} + async fn run(cli: Cli) -> Result { match cli.command { Command::Origin { @@ -305,6 +339,7 @@ async fn run(cli: Cli) -> Result { bitrate, mut markers, hls_markers, + hls_profile, account_limit, limit_mode, stalker_refuse_create_link_once, @@ -316,6 +351,7 @@ async fn run(cli: Cli) -> Result { if markers.is_empty() { return Err(TestkitError::Configuration("origin requires at least one channel marker".to_owned())); } + let hls_profile = parse_hls_profile(hls_profile.as_deref())?; let mode = match limit_mode.as_str() { "reject_new" => OriginLimitMode::RejectNew, "evict_oldest" => OriginLimitMode::EvictOldest, @@ -331,6 +367,7 @@ async fn run(cli: Cli) -> Result { bitrate, markers: Arc::new(markers), hls_markers: Arc::new(hls_markers.into_iter().collect()), + hls_profile, stream_counter: Arc::new(Mutex::new(0)), observations: Arc::new(Mutex::new(Vec::new())), faults: Arc::new(Mutex::new(FaultSchedule::default())), @@ -345,6 +382,29 @@ async fn run(cli: Cli) -> Result { ) .await } + Command::HlsProbe { + url, + run_id, + expected_marker, + verify_revision, + expect_body_error, + max_first_byte_millis, + min_prefix_to_eof_millis, + } => { + let options = tuliprox_testkit::hls_probe::ProbeOptions { + verify_revision, + expect_body_error, + max_first_byte_millis, + min_prefix_to_eof_millis, + }; + let result = + tuliprox_testkit::hls_probe::probe(&url, &RunId::new(run_id), expected_marker, &options).await?; + println!( + "{}", + serde_json::to_string_pretty(&result).map_err(|error| TestkitError::Protocol(error.to_string()))? + ); + Ok(RunExit::Passed) + } Command::Agent { url, expected_marker, frames, hls, controller, agent_id, run_id } => { if let Some(controller) = controller { return run_controlled_agent(&controller, AgentId::new(agent_id), RunId::new(run_id)).await; @@ -1138,10 +1198,20 @@ async fn hls( let encoded = url::form_urlencoded::byte_serialize(account.as_bytes()).collect::(); format!("&token={encoded}") }); - let playlist = format!( + let playlist = if let Some(profile) = &state.hls_profile { + let first = sequence.saturating_sub(1) / 10; + let mut playlist = + format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:{first}\n"); + for seq in first..first + u64::from(profile.window_segments) { + let _ = writeln!(playlist, "#EXTINF:4.0,\n{seq}.ts?run={}{account_query}", state.run_id.0); + } + playlist + } else { + format!( "#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:1\n#EXT-X-MEDIA-SEQUENCE:{first}\n#EXTINF:1.0,\n{first}.ts?run={}{account_query}\n#EXTINF:1.0,\n{}.ts?run={}{account_query}\n", state.run_id.0, *sequence, state.run_id.0 - ); + ) + }; return hls_guarded_response( Bytes::from(playlist), "application/vnd.apple.mpegurl", @@ -1155,6 +1225,15 @@ async fn hls( let Some(sequence) = segment else { return StatusCode::NOT_FOUND.into_response(); }; + if let Some(profile) = &state.hls_profile { + if sequence < u64::from(profile.missing_head_segments) { + let status = if sequence.is_multiple_of(2) { StatusCode::NOT_FOUND } else { StatusCode::GONE }; + let mut response = hls_guarded_response(Bytes::new(), "video/mp2t", conn_id, req_id, &state.tracker).await; + *response.status_mut() = status; + return response; + } + return profiled_hls_response(&state, profile, marker, sequence, conn_id, req_id).await; + } let frame = Frame::synthetic(&state.run_id, &OriginStreamId::new(format!("hls-presentation-{marker}")), marker, sequence); let mut encoded = BytesMut::new(); @@ -1164,6 +1243,47 @@ async fn hls( hls_guarded_response(encoded.freeze(), "video/mp2t", conn_id, req_id, &state.tracker).await } +async fn profiled_hls_response( + state: &OriginState, + profile: &tuliprox_testkit::hls_probe::HlsOriginProfile, + marker: u32, + sequence: u64, + conn_id: u64, + req_id: u64, +) -> Response { + let Ok(parts) = tuliprox_testkit::hls_probe::segment_parts(&state.run_id, marker, sequence) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let length = (parts.0.len() + parts.1.len()) as u64; + let _evict_rx = + state.tracker.on_body_started(conn_id, req_id, if profile.unknown_length { None } else { Some(length) }).await; + let bytes_emitted = Arc::new(AtomicU64::new(0)); + let guard = Arc::new(BodyDropGuard { + conn_id, + req_id, + tracker: state.tracker.clone(), + bytes_emitted: Arc::clone(&bytes_emitted), + start_time: Instant::now(), + closed: Arc::new(AtomicBool::new(false)), + evicted: Arc::new(AtomicBool::new(false)), + }); + let stream = tuliprox_testkit::hls_probe::segment_stream(parts, profile, sequence).map(move |item| { + let _keep_guard = &guard; + if let Ok(chunk) = &item { + bytes_emitted.fetch_add(chunk.len() as u64, Ordering::Relaxed); + } + item + }); + let mut response = Response::new(Body::from_stream(stream)); + response.headers_mut().insert(header::CONTENT_TYPE, HeaderValue::from_static("video/mp2t")); + if !profile.unknown_length { + if let Ok(value) = HeaderValue::from_str(&length.to_string()) { + response.headers_mut().insert(header::CONTENT_LENGTH, value); + } + } + response +} + async fn vod( Path(object): Path, Query(query): Query>, @@ -3475,14 +3595,14 @@ mod tests { ]); let mut seen = HashSet::new(); let session_id = observe_new_stream_session(&streams, "alice", &mut seen); - assert_eq!(session_id, Some((123_u64 << 32) | 17)); + assert_eq!(session_id, Some((123_u64 << 32) | 0x11)); let history = serde_json::json!({"items": [ - {"session_id": (123_u64 << 32) | 18, "disconnect_reason": "client_kicked"}, - {"session_id": (123_u64 << 32) | 17, "disconnect_reason": "client_closed"} + {"session_id": (123_u64 << 32) | 0x12, "disconnect_reason": "client_kicked"}, + {"session_id": (123_u64 << 32) | 0x11, "disconnect_reason": "client_closed"} ]}); assert!(!history_confirms_kick(&history, session_id.unwrap_or_default())); let kicked = serde_json::json!({"items": [ - {"session_id": (123_u64 << 32) | 17, "disconnect_reason": "client_kicked"} + {"session_id": (123_u64 << 32) | 0x11, "disconnect_reason": "client_kicked"} ]}); assert!(history_confirms_kick(&kicked, session_id.unwrap_or_default())); } @@ -3536,6 +3656,7 @@ mod tests { bitrate: 64_000, markers: Arc::new(vec![17]), hls_markers: Arc::new(HashSet::new()), + hls_profile: None, stream_counter: Arc::new(Mutex::new(4)), observations: Arc::new(Mutex::new(Vec::new())), faults: Arc::new(Mutex::new(FaultSchedule::default())), diff --git a/tools/tuliprox-testkit/tests/hls_fast_start_process.rs b/tools/tuliprox-testkit/tests/hls_fast_start_process.rs new file mode 100644 index 000000000..6ccfa159b --- /dev/null +++ b/tools/tuliprox-testkit/tests/hls_fast_start_process.rs @@ -0,0 +1,152 @@ +//! Opt-in real-process checks. Synthetic frames exercise HTTP/cache behavior, not a TS decoder. +use std::{collections::HashMap, path::Path}; +use tuliprox_testkit::{ + bootstrap::{fixture_password, FixtureInputPlan, IsolatedFixture}, + config::{BootstrapConfig, Channel, FixtureInputType, FixtureStalker, FixtureStreamOptions, OriginConfig}, + discovery::VirtualIdMap, + hls_probe::{probe, HlsOriginProfile, ProbeOptions, StartupMode}, + protocol::RunId, + TestkitError, +}; + +async fn discover_playlist(client: &reqwest::Client, url: &str) -> Result { + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); + loop { + let attempt = tokio::time::timeout_at(deadline, async { + let playlist = client.get(url).send().await?.error_for_status()?.text().await?; + VirtualIdMap::from_m3u(&playlist) + }) + .await; + if let Ok(Ok(urls)) = attempt { + return Ok(urls); + } + // Readiness precedes the initial catalog commit; discovery may observe an unfinished playlist. + if tokio::time::Instant::now() >= deadline { + return Err(TestkitError::Protocol("fixture playlist discovery exceeded absolute deadline".to_owned())); + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } +} + +async fn run(mode: StartupMode, missing: u32, abort: bool, unknown_length: bool) -> Result<(), TestkitError> { + let sut = std::env::var("TULIPROX_TESTKIT_SUT_BINARY") + .map_err(|_| TestkitError::Configuration("set TULIPROX_TESTKIT_SUT_BINARY to the built SUT".to_owned()))?; + let testkit = env!("CARGO_BIN_EXE_tuliprox-testkit"); + let run_id = format!("hls-fast-start-{mode}-{missing}-{abort}-{unknown_length}"); + let profile = HlsOriginProfile { + missing_head_segments: missing, + unknown_length, + abort_segments: if abort { vec![0] } else { Vec::new() }, + ..HlsOriginProfile::default() + }; + let origin = + OriginConfig { run_id: run_id.clone(), account_limit: None, limit_mode: None, hls_profile: Some(profile) }; + let bootstrap = BootstrapConfig { + command: sut, + arguments: ["-s", "-c", "{config_file}", "-i", "{source_file}", "-a", "{api_proxy_file}"] + .map(str::to_owned) + .to_vec(), + readiness_url: "{base_url}/healthcheck".to_owned(), + readiness_timeout_millis: 30_000, + }; + let channels = HashMap::from([("channel".to_owned(), Channel { origin_marker: 17, protocol: "hls".to_owned() })]); + let options = FixtureStreamOptions { hls_startup_mode: Some(mode), ..FixtureStreamOptions::default() }; + let fixture = IsolatedFixture::start( + &bootstrap, + Path::new(env!("CARGO_MANIFEST_DIR")), + Path::new(testkit), + &run_id, + Some(&origin), + None, + &channels, + &options, + false, + &FixtureInputPlan { input_type: FixtureInputType::M3u, stalker: &FixtureStalker::default() }, + ) + .await?; + let result = async { + let client = reqwest::Client::new(); + let playlist_url = format!("{}/m3u?username=testkit&password={}", fixture.base_url, fixture_password()); + let urls = discover_playlist(&client, &playlist_url).await?; + let progressive = matches!(mode, StartupMode::Progressive); + let result = probe( + urls.playback_url(17)?, + &RunId::new(&run_id), + 17, + &ProbeOptions { + verify_revision: !abort, + expect_body_error: abort, + max_first_byte_millis: if progressive { Some(900) } else { None }, + min_prefix_to_eof_millis: if progressive { Some(500) } else { None }, + }, + ) + .await?; + if !progressive && result.entry_to_first_byte_millis < 800 { + return Err(TestkitError::Protocol("processed response preceded origin EOF".to_owned())); + } + let events: Vec = client + .get(format!("{}/v1/runs/{run_id}/events", fixture.origin_control_url)) + .send() + .await? + .error_for_status()? + .json() + .await?; + let mut segment_counts = HashMap::new(); + for event in events { + if let tuliprox_testkit::origin_events::OriginEventKind::RequestStarted { path, .. } = event.kind { + if Path::new(&path).extension().is_some_and(|extension| extension.eq_ignore_ascii_case("ts")) { + *segment_counts.entry(path).or_insert(0_usize) += 1; + } + } + } + if !abort + && segment_counts.iter().any(|(path, count)| { + *count > 1 && !(missing > 0 && (path.ends_with("/0.ts") || path.ends_with("/1.ts"))) + }) + { + return Err(TestkitError::Protocol("retry or range started a second origin segment download".to_owned())); + } + println!("{mode} missing={missing} abort={abort} unknown_length={unknown_length}: {result:?}"); + Ok(()) + } + .await; + let cleanup = fixture.stop().await; + result?; + cleanup +} + +#[tokio::test] +#[ignore = "requires built SUT via TULIPROX_TESTKIT_SUT_BINARY"] +async fn conservative_waits_for_complete_origin_and_preserves_ranges() -> Result<(), TestkitError> { + run(StartupMode::Conservative, 0, false, false).await +} +#[tokio::test] +#[ignore = "requires built SUT via TULIPROX_TESTKIT_SUT_BINARY"] +async fn first_ready_delivers_processed_revision() -> Result<(), TestkitError> { + run(StartupMode::FirstReady, 0, false, false).await +} +#[tokio::test] +#[ignore = "requires built SUT via TULIPROX_TESTKIT_SUT_BINARY"] +async fn progressive_delivers_prefix_before_origin_eof() -> Result<(), TestkitError> { + run(StartupMode::Progressive, 0, false, false).await +} +#[tokio::test] +#[ignore = "requires built SUT via TULIPROX_TESTKIT_SUT_BINARY"] +async fn first_ready_skips_404_and_410_head() -> Result<(), TestkitError> { + run(StartupMode::FirstReady, 2, false, false).await +} +#[tokio::test] +#[ignore = "requires built SUT via TULIPROX_TESTKIT_SUT_BINARY"] +async fn progressive_skips_404_and_410_head() -> Result<(), TestkitError> { + run(StartupMode::Progressive, 2, false, false).await +} +#[tokio::test] +#[ignore = "requires built SUT via TULIPROX_TESTKIT_SUT_BINARY"] +async fn progressive_known_length_drop_is_not_successful_eof() -> Result<(), TestkitError> { + run(StartupMode::Progressive, 0, true, false).await +} +#[tokio::test] +#[ignore = "requires built SUT via TULIPROX_TESTKIT_SUT_BINARY"] +async fn progressive_chunked_drop_is_not_successful_eof() -> Result<(), TestkitError> { + run(StartupMode::Progressive, 0, true, true).await +}